diff --git a/.clang-format b/.clang-format
index 1cc151e2adcc5..6a3de86ab27a4 100644
--- a/.clang-format
+++ b/.clang-format
@@ -481,6 +481,7 @@ ForEachMacros:
   - 'genradix_for_each'
   - 'genradix_for_each_from'
   - 'genradix_for_each_reverse'
+  - 'guard'
   - 'hash_for_each'
   - 'hash_for_each_possible'
   - 'hash_for_each_possible_rcu'
@@ -674,6 +675,7 @@ ForEachMacros:
   - 'rq_list_for_each'
   - 'rq_list_for_each_safe'
   - 'sample_read_group__for_each'
+  - 'scoped_guard'
   - 'scsi_for_each_prot_sg'
   - 'scsi_for_each_sg'
   - 'sctp_for_each_hentry'
diff --git a/.mailmap b/.mailmap
index bd6a72f29d9cc..c358f21d2a3be 100644
--- a/.mailmap
+++ b/.mailmap
@@ -263,8 +263,9 @@ Enric Balletbo i Serra <eballetbo@kernel.org> <enric.balletbo@collabora.com>
 Enric Balletbo i Serra <eballetbo@kernel.org> <eballetbo@iseebcn.com>
 Erik Kaneda <erik.kaneda@intel.com> <erik.schmauss@intel.com>
 Ethan Carter Edwards <ethan@ethancedwards.com> Ethan Edwards <ethancarteredwards@gmail.com>
-Eugen Hristev <eugen.hristev@linaro.org> <eugen.hristev@microchip.com>
-Eugen Hristev <eugen.hristev@linaro.org> <eugen.hristev@collabora.com>
+Eugen Hristev <ehristev@kernel.org> <eugen.hristev@microchip.com>
+Eugen Hristev <ehristev@kernel.org> <eugen.hristev@linaro.org>
+Eugen Hristev <ehristev@kernel.org> <eugen.hristev@collabora.com>
 Evgeniy Polyakov <johnpol@2ka.mipt.ru>
 Ezequiel Garcia <ezequiel@vanguardiasur.com.ar> <ezequiel@collabora.com>
 Faith Ekstrand <faith.ekstrand@collabora.com> <jason@jlekstrand.net>
@@ -339,6 +340,7 @@ Henrik Rydberg <rydberg@bitmath.org>
 Herbert Xu <herbert@gondor.apana.org.au>
 Huacai Chen <chenhuacai@kernel.org> <chenhc@lemote.com>
 Huacai Chen <chenhuacai@kernel.org> <chenhuacai@loongson.cn>
+Ian Ray <ian.ray@gehealthcare.com> <ian.ray@ge.com>
 Ignat Korchagin <ignat@linux.win> <ignat@cloudflare.com>
 Igor Korotin <igor.korotin@linux.dev> <igor.korotin.linux@gmail.com>
 Ike Panhc <ikepanhc@gmail.com> <ike.pan@canonical.com>
@@ -433,6 +435,7 @@ John Stultz <johnstul@us.ibm.com>
 Jonas Gorski <jonas.gorski@gmail.com> <jogo@openwrt.org>
 Jonathan Cameron <jic23@kernel.org> <jonathan.cameron@huawei.com>
 Jordan Crouse <jordan@cosmicpenguin.net> <jcrouse@codeaurora.org>
+Jorge Ramirez-Ortiz <jorge.ramirez@oss.qualcomm.com> <jorge.ramirez-ortiz@linaro.org>
 <josh@joshtriplett.org> <josh@freedesktop.org>
 <josh@joshtriplett.org> <josh@kernel.org>
 <josh@joshtriplett.org> <josht@linux.vnet.ibm.com>
diff --git a/CREDITS b/CREDITS
index 17962bdd6dbdd..2454f8f84fd36 100644
--- a/CREDITS
+++ b/CREDITS
@@ -197,6 +197,9 @@ S: Hauptstrasse 19
 S: 79837 St. Blasien
 S: Germany
 
+N: Ferenc Bakonyi
+D: Hercules graphics adapter framebuffer driver
+
 N: Krishna Balasubramanian
 E: balasub@cis.ohio-state.edu
 D: Wrote SYS V IPC (part of standard kernel since 0.99.10)
@@ -2241,6 +2244,7 @@ S: Canada
 N: Krzysztof Kozlowski
 E: krzk@kernel.org
 D: NFC network subsystem and drivers maintainer
+D: Samsung S2M/S5M Multifunction PMIC device drivers for Exynos platforms
 
 N: Christian Krafft
 D: PowerPC Cell support
@@ -3368,6 +3372,10 @@ N: Anil Ravindranath
 E: anil_ravindranath@pmc-sierra.com
 D: PMC-Sierra MaxRAID driver
 
+N: Dwaipayan Ray
+E: dwaipayanray1@gmail.com
+D: checkpatch improvements
+
 N: Eric S. Raymond
 E: esr@thyrsus.com
 W: http://www.tuxedo.org/~esr/
diff --git a/Documentation/ABI/obsolete/sysfs-driver-ivpu b/Documentation/ABI/obsolete/sysfs-driver-ivpu
new file mode 100644
index 0000000000000..b906e71497293
--- /dev/null
+++ b/Documentation/ABI/obsolete/sysfs-driver-ivpu
@@ -0,0 +1,30 @@
+What:		/sys/bus/pci/drivers/intel_vpu/.../sched_mode
+Date:		October 2024
+KernelVersion:	6.12
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Current NPU scheduling mode. Returns one of the following strings:
+		- "HW" - Hardware Scheduler mode
+		- "OS" - Operating System Scheduler mode
+		Read-only.
+		Deprecated since the "OS" scheduling mode is not usable
+		and will be removed from future versions of the driver.
+		Will be removed in 2027
+
+What:		/sys/bus/pci/drivers/intel_vpu/.../npu_max_frequency_mhz
+Date:		April 2025
+KernelVersion:	6.15
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Legacy alias for /sys/bus/pci/drivers/intel_vpu/.../freq/hw_max_freq.
+		Shows maximum frequency in MHz of the NPU's data processing unit.
+		Read-only.
+		Will be removed in 2027
+
+What:		/sys/bus/pci/drivers/intel_vpu/.../npu_current_frequency_mhz
+Date:		April 2025
+KernelVersion:	6.15
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Legacy alias for /sys/bus/pci/drivers/intel_vpu/.../freq/current_freq.
+		Shows current frequency in MHz of the NPU's data processing unit.
+		The value is read only when the device is active; otherwise it returns 0.
+		Read-only.
+		Will be removed in 2027
diff --git a/Documentation/ABI/testing/configfs-thunderbolt_stream b/Documentation/ABI/testing/configfs-thunderbolt_stream
new file mode 100644
index 0000000000000..7abc6b73a1e4c
--- /dev/null
+++ b/Documentation/ABI/testing/configfs-thunderbolt_stream
@@ -0,0 +1,83 @@
+What:		/sys/kernel/config/thunderbolt/stream/<xdomain>.<service>
+Date:		Sep 2026
+KernelVersion:	v7.2
+Contact:	Mika Westerberg <mika.westerberg@linux.intel.com>
+Description:
+		Configuration group for a stream Thunderbolt/USB4
+		service. It is possible to create groups even if there
+		is no connection yet to the other host. Once a
+		connection established and there is stream service on
+		the remote side that matches, this configuration is
+		applied to it.
+
+		To find the service name you can run tblist from tbtools [1]:
+
+		  # tblist -A
+		  ...
+		  Domain 0 Route 3 Index 0: stream
+
+		[1] https://github.com/intel/tbtools
+
+What:		/sys/kernel/config/thunderbolt/stream/<xdomain>.<service>/$name
+Date:		Sep 2026
+KernelVersion:	v7.2
+Contact:	Mika Westerberg <mika.westerberg@linux.intel.com>
+Description:
+		Creates new stream with $name and fills it with the
+		default values. If there is an advertised remote stream
+		with the same name, uses its values as the default.
+
+What:		/sys/kernel/config/thunderbolt/stream/<xdomain>.<service>/$name/index
+Date:		Sep 2026
+KernelVersion:	v7.2
+Contact:	Mika Westerberg <mika.westerberg@linux.intel.com>
+Description:
+		This matches the X in /dev/tbstreamX and allows userspace
+		to map the configfs directory to the corresponding character
+		device.
+
+What:		/sys/kernel/config/thunderbolt/stream/<xdomain>.<service>/$name/in_hopid
+Date:		Sep 2026
+KernelVersion:	v7.2
+Contact:	Mika Westerberg <mika.westerberg@linux.intel.com>
+Description:
+		In HopID used with the read path of the tunnel. Available HopIDs
+		for tunneling start from 8. You can pass also -1 for automatic
+		allocation. The allocated value can be read here. Writing 0 will
+		de-allocate if the stream is not in use.
+
+		To figure out the maximum HopID you can run tbget from
+		tbtools for the lane adapter. For example below we check
+		for lane adapter number 1 (first USB4 port):
+
+		  # tbget -r 0 -a 1 -D ADP_CS_5.Max\ Input\ HopID
+		  19
+
+		This allows to use anything between 8 and 19 inclusive.
+
+What:		/sys/kernel/config/thunderbolt/stream/<xdomain>.<service>/$name/out_hopid
+Date:		Sep 2026
+KernelVersion:	v7.2
+Contact:	Mika Westerberg <mika.westerberg@linux.intel.com>
+Description:
+		Out HopID used with the write path of the tunnel. Available HopIDs
+		for tunneling start from 8. You can pass also -1 for automatic
+		allocation. The allocated value can be read here. Writing 0 will
+		de-allocate if the stream is not in use. See @in_hopid
+		for how to figure out the maximum HopID.
+
+What:		/sys/kernel/config/thunderbolt/stream/<xdomain>.<service>/$name/ring_size
+Date:		Sep 2026
+KernelVersion:	v7.2
+Contact:	Mika Westerberg <mika.westerberg@linux.intel.com>
+Description:
+		Size of the TX/RX rings. Can be adjusted between 32 and
+		4096. The default is 256.
+
+What:		/sys/kernel/config/thunderbolt/stream/<xdomain>.<service>/$name/throttling
+Date:		Sep 2026
+KernelVersion:	v7.2
+Contact:	Mika Westerberg <mika.westerberg@linux.intel.com>
+Description:
+		Interrupt throttling rate in ns. Lower values can give
+		better latency. The default is 8192 ns.
diff --git a/Documentation/ABI/testing/se-cdev b/Documentation/ABI/testing/se-cdev
new file mode 100644
index 0000000000000..c6b8e16bda78d
--- /dev/null
+++ b/Documentation/ABI/testing/se-cdev
@@ -0,0 +1,44 @@
+What:		/dev/<se>_mu[0-9]+_ch[0-9]+
+Date:		Mar 2025
+KernelVersion:	6.8
+Contact:	linux-imx@nxp.com, pankaj.gupta@nxp.com
+Description:
+		NXP offers multiple hardware IP(s) for secure enclaves like EdgeLock-
+		Enclave(ELE), SECO. The character device file descriptors
+		/dev/<se>_mu*_ch* are the interface between userspace NXP's secure-
+		enclave shared library and the kernel driver.
+
+		The ioctl(2)-based ABI is defined and documented in
+		[include]<linux/firmware/imx/ele_mu_ioctl.h>.
+		ioctl(s) are used primarily for:
+
+			- shared memory management
+			- allocation of I/O buffers
+			- getting mu info
+			- setting a dev-ctx as receiver to receive all the commands from FW
+			- getting SoC info
+			- send command and receive command response
+
+		The following file operations are supported:
+
+		open(2)
+		  Currently the only useful flags are O_RDWR.
+
+		read(2)
+		  Every read() from the opened character device context is waiting on
+		  wait_event_interruptible, that gets set by the registered mailbox callback
+		  function, indicating a message received from the firmware on message-
+		  unit.
+
+		write(2)
+		  Every write() to the opened character device context needs to acquire
+		  mailbox_lock before sending message on to the message unit.
+
+		close(2)
+		  Stops and frees up the I/O contexts that were associated
+		  with the file descriptor.
+
+Users:		https://github.com/nxp-imx/imx-secure-enclave.git,
+		https://github.com/nxp-imx/imx-smw.git,
+		crypto/skcipher,
+		drivers/nvmem/imx-ocotp-ele.c
diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio
index 4fc9f6bd42812..925a33fd309af 100644
--- a/Documentation/ABI/testing/sysfs-bus-iio
+++ b/Documentation/ABI/testing/sysfs-bus-iio
@@ -764,10 +764,13 @@ Contact:	linux-iio@vger.kernel.org
 Description:
 		Specifies the output powerdown mode.
 		DAC output stage is disconnected from the amplifier and
+		500ohm_to_gnd: connected to ground via a 500Ohm resistor,
 		1kohm_to_gnd: connected to ground via an 1kOhm resistor,
 		2.5kohm_to_gnd: connected to ground via a 2.5kOhm resistor,
+		3.85kohm_to_gnd: connected to ground via a 3.85kOhm resistor,
 		6kohm_to_gnd: connected to ground via a 6kOhm resistor,
 		7.7kohm_to_gnd: connected to ground via a 7.7kOhm resistor,
+		16kohm_to_gnd: connected to ground via a 16kOhm resistor,
 		20kohm_to_gnd: connected to ground via a 20kOhm resistor,
 		32kohm_to_gnd: connected to ground via a 32kOhm resistor,
 		42kohm_to_gnd: connected to ground via a 42kOhm resistor,
@@ -1510,21 +1513,24 @@ Contact:	linux-iio@vger.kernel.org
 Description:
 		Description of the scan element data storage within the buffer
 		and hence the form in which it is read from user-space.
-		Form is [be|le]:[s|u]bits/storagebits[>>shift].
-		be or le specifies big or little endian. s or u specifies if
-		signed (2's complement) or unsigned. bits is the number of bits
-		of data and storagebits is the space (after padding) that it
-		occupies in the buffer. shift if specified, is the shift that
-		needs to be applied prior to masking out unused bits. Some
-		devices put their data in the middle of the transferred elements
-		with additional information on both sides.  Note that some
-		devices will have additional information in the unused bits
-		so to get a clean value, the bits value must be used to mask
-		the buffer output value appropriately.  The storagebits value
-		also specifies the data alignment.  So s48/64>>2 will be a
-		signed 48 bit integer stored in a 64 bit location aligned to
-		a 64 bit boundary. To obtain the clean value, shift right 2
-		and apply a mask to zero the top 16 bits of the result.
+		Form is [be|le]:[f|s|u]bits/storagebits[>>shift].
+		be or le specifies big or little endian. f means floating-point
+		(IEEE 754 binary format), s means signed (2's complement), u means
+		unsigned. bits is the number of bits of data and storagebits is the
+		space (after padding) that it occupies in the buffer; when using a
+		floating-point format, bits must be one of the width values defined
+		in the IEEE 754 standard for binary interchange formats (e.g. 16
+		indicates the binary16 format for half-precision numbers). shift,
+		if specified, is the shift that needs to be applied prior to
+		masking out unused bits. Some devices put their data in the middle
+		of the transferred elements with additional information on both
+		sides. Note that some devices will have additional information in
+		the unused bits, so to get a clean value the bits value must be
+		used to mask the buffer output value appropriately. The storagebits
+		value also specifies the data alignment. So s48/64>>2 will be a
+		signed 48 bit integer stored in a 64 bit location aligned to a 64
+		bit boundary. To obtain the clean value, shift right 2 and apply a
+		mask to zero the top 16 bits of the result.
 		For other storage combinations this attribute will be extended
 		appropriately.
 
@@ -1752,6 +1758,21 @@ Description:
 		measurement from channel Y. Units after application of scale and
 		offset are milliamps.
 
+What:		/sys/bus/iio/devices/iio:deviceX/in_rot_quaternionaxis_raw
+KernelVersion:	7.1
+Contact:	linux-iio@vger.kernel.org
+Description:
+		Raw value of {x, y, z} components of the quaternion vector. These
+		components represent the axis about which a rotation occurs, and are
+		subject to the following constraints:
+
+		- the quaternion vector is normalized, i.e. w^2 + x^2 + y^2 + z^2 = 1
+		- the rotation angle is within the [-pi, pi] range, i.e. the w
+		  component (which represents the amount of rotation) is non-negative
+
+		These constraints allow the w value to be calculated from the other
+		components: w = sqrt(1 - (x^2 + y^2 + z^2)).
+
 What:		/sys/.../iio:deviceX/in_energy_en
 What:		/sys/.../iio:deviceX/in_distance_en
 What:		/sys/.../iio:deviceX/in_velocity_sqrt(x^2+y^2+z^2)_en
diff --git a/Documentation/ABI/testing/sysfs-class-led b/Documentation/ABI/testing/sysfs-class-led
index 0313b82644f24..d4c918cc11a12 100644
--- a/Documentation/ABI/testing/sysfs-class-led
+++ b/Documentation/ABI/testing/sysfs-class-led
@@ -22,8 +22,8 @@ Description:
 		  For additional details please refer to
 		  Documentation/leds/leds-class-multicolor.rst.
 
-		The value is between 0 and
-		/sys/class/leds/<led>/max_brightness.
+		The value is between 0 and /sys/class/leds/<led>/max_brightness
+		and is represented by as a decimal.
 
 		Writing 0 to this file clears active trigger.
 
diff --git a/Documentation/ABI/testing/sysfs-class-led-multicolor b/Documentation/ABI/testing/sysfs-class-led-multicolor
index 16fc827b10cb6..197da3e775b4a 100644
--- a/Documentation/ABI/testing/sysfs-class-led-multicolor
+++ b/Documentation/ABI/testing/sysfs-class-led-multicolor
@@ -16,9 +16,22 @@ Date:		March 2020
 KernelVersion:	5.9
 Contact:	Dan Murphy <dmurphy@ti.com>
 Description:	read/write
-		This file contains array of integers. Order of components is
-		described by the multi_index array. The maximum intensity should
-		not exceed /sys/class/leds/<led>/max_brightness.
+		This file contains an array of integers. The order of components
+		is described by the multi_index array. The maximum intensity value
+		supported by each color component is described by the multi_max_intensity
+		file. Writing intensity values larger than the maximum value of a
+		given color component will result in those values being clamped.
+
+		For additional details please refer to
+		Documentation/leds/leds-class-multicolor.rst.
+
+What:		/sys/class/leds/<led>/multi_max_intensity
+Date:		March 2026
+KernelVersion:	7.1
+Contact:	Armin Wolf <W_Armin@gmx.de>
+Description:	read
+		This file contains an array of integers describing the maximum
+		intensity value for each intensity component.
 
 		For additional details please refer to
 		Documentation/leds/leds-class-multicolor.rst.
diff --git a/Documentation/ABI/testing/sysfs-driver-ivpu b/Documentation/ABI/testing/sysfs-driver-ivpu
new file mode 100644
index 0000000000000..91685774edfcc
--- /dev/null
+++ b/Documentation/ABI/testing/sysfs-driver-ivpu
@@ -0,0 +1,65 @@
+What:		/sys/bus/pci/drivers/intel_vpu/.../npu_busy_time_us
+Date:		May 2024
+KernelVersion:	6.11
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Time in microseconds that the device spent executing jobs. The time is
+		counted when and only when there are jobs submitted to firmware. This time
+		can be used to measure the utilization of NPU, either by calculating the
+		difference between two timepoints or monitoring utilization percentage by
+		reading periodically. Recommended read period is 1 second to avoid impact
+		on job submission performance. Read-only.
+
+What:		/sys/bus/pci/drivers/intel_vpu/.../npu_memory_utilization
+Date:		Jan 2025
+KernelVersion:	6.15
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Current NPU memory utilization in bytes. Reports the total size of all
+		resident buffer objects allocated for NPU use. Read-only.
+
+What:		/sys/bus/pci/drivers/intel_vpu/.../freq/hw_min_freq
+Date:		April 2026
+KernelVersion:	7.2
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Minimum frequency in MHz supported by the NPU hardware. This is a
+		hardware capability and cannot be changed. Read-only.
+
+What:		/sys/bus/pci/drivers/intel_vpu/.../freq/hw_efficient_freq
+Date:		April 2026
+KernelVersion:	7.2
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Most efficient operating frequency in MHz for the NPU. This represents
+		the frequency at which the NPU operates most efficiently in terms of power
+		and performance. Read-only.
+
+What:		/sys/bus/pci/drivers/intel_vpu/.../freq/hw_max_freq
+Date:		April 2026
+KernelVersion:	7.2
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Maximum frequency in MHz supported by the NPU hardware. This is a
+		hardware capability and cannot be changed. Read-only.
+
+What:		/sys/bus/pci/drivers/intel_vpu/.../freq/current_freq
+Date:		April 2026
+KernelVersion:	7.2
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Current operating frequency in MHz of the NPU. The value is valid only
+		when the device is active; returns 0 when idle. The actual frequency may
+		be lower than the requested range due to power or thermal constraints.
+		Read-only.
+
+What:		/sys/bus/pci/drivers/intel_vpu/.../freq/set_min_freq
+Date:		April 2026
+KernelVersion:	7.2
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Configured minimum operating frequency in MHz (50XX devices and newer).
+		Values written are clamped to hardware limits (hw_min_freq to hw_max_freq).
+		If set_min_freq exceeds set_max_freq, the driver clamps set_min_freq to
+		set_max_freq when selecting the operating frequency. Read-write.
+
+What:		/sys/bus/pci/drivers/intel_vpu/.../freq/set_max_freq
+Date:		April 2026
+KernelVersion:	7.2
+Contact:	dri-devel@lists.freedesktop.org
+Description:	Configured maximum operating frequency in MHz (50XX devices and newer).
+		Values written are clamped to hardware limits (hw_min_freq to hw_max_freq).
+		Read-write.
diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs
index 423ec40e2e4e2..1b58c029abd0d 100644
--- a/Documentation/ABI/testing/sysfs-fs-f2fs
+++ b/Documentation/ABI/testing/sysfs-fs-f2fs
@@ -270,7 +270,8 @@ Description:	Shows all enabled kernel features.
 		inode_checksum, flexible_inline_xattr, quota_ino,
 		inode_crtime, lost_found, verity, sb_checksum,
 		casefold, readonly, compression, test_dummy_encryption_v2,
-		atomic_write, pin_file, encrypted_casefold, linear_lookup.
+		atomic_write, pin_file, encrypted_casefold, linear_lookup,
+		fserror.
 
 What:		/sys/fs/f2fs/<disk>/inject_rate
 Date:		May 2016
@@ -1000,4 +1001,4 @@ Contact:	"Chao Yu" <chao@kernel.org>
 Description:	It can be used to tune priority of f2fs critical task, e.g. f2fs_ckpt, f2fs_gc
 		threads, limitation as below:
 		- it requires user has CAP_SYS_NICE capability.
-		- the range is [100, 139], by default the value is 100.
+		- the range is [100, 139], by default the value is 120.
diff --git a/Documentation/ABI/testing/sysfs-kernel-mm-damon b/Documentation/ABI/testing/sysfs-kernel-mm-damon
index 2424237ebb105..ee29d4e204ffa 100644
--- a/Documentation/ABI/testing/sysfs-kernel-mm-damon
+++ b/Documentation/ABI/testing/sysfs-kernel-mm-damon
@@ -84,6 +84,13 @@ Description:	Writing an integer to this file sets the 'address unit'
 		parameter of the given operations set of the context.  Reading
 		the file returns the last-written 'address unit' value.
 
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/pause
+Date:		Mar 2026
+Contact:	SeongJae Park <sj@kernel.org>
+Description:	Writing a boolean keyword to this file sets the 'pause' request
+		parameter for the context.  Reading the file returns the
+		last-written 'pause' value.
+
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/monitoring_attrs/intervals/sample_us
 Date:		Mar 2022
 Contact:	SeongJae Park <sj@kernel.org>
@@ -322,6 +329,18 @@ Contact:	SeongJae Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the
 		goal-based effective quota auto-tuning algorithm to use.
 
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/fail_charge_num
+Date:		Mar 2026
+Contact:	SeongJae Park <sj@kernel.org>
+Description:	Writing to and reading from this file sets and gets the
+		action-failed memory quota charging ratio numerator.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/fail_charge_denom
+Date:		Mar 2026
+Contact:	SeongJae Park <sj@kernel.org>
+Description:	Writing to and reading from this file sets and gets the
+		action-failed memory quota charging ratio denominator.
+
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/quotas/weights/sz_permil
 Date:		Mar 2022
 Contact:	SeongJae Park <sj@kernel.org>
@@ -377,15 +396,20 @@ Contact:	SeongJae Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the low
 		watermark of the scheme in permil.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/nr_filters
-Date:		Dec 2022
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters
+Date:		Feb 2025
+Contact:	SeongJae Park <sj@kernel.org>
+Description:	Directory for DAMON core layer-handled DAMOS filters.
+
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/nr_filters
+Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	Writing a number 'N' to this file creates the number of
 		directories for setting filters of the scheme named '0' to
-		'N-1' under the filters/ directory.
+		'N-1' under the core_filters/ directory.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/<F>/type
-Date:		Dec 2022
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/type
+Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	Writing to and reading from this file sets and gets the type of
 		the memory of the interest.  'anon' for anonymous pages,
@@ -393,77 +417,78 @@ Description:	Writing to and reading from this file sets and gets the type of
 		'addr' for address range (an open-ended interval), or 'target'
 		for DAMON monitoring target can be written and read.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/<F>/memcg_path
-Date:		Dec 2022
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/memcg_path
+Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	If 'memcg' is written to the 'type' file, writing to and
 		reading from this file sets and gets the path to the memory
 		cgroup of the interest.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/<F>/addr_start
-Date:		Jul 2023
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/addr_start
+Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	If 'addr' is written to the 'type' file, writing to or reading
 		from this file sets or gets the start address of the address
 		range for the filter.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/<F>/addr_end
-Date:		Jul 2023
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/addr_end
+Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	If 'addr' is written to the 'type' file, writing to or reading
 		from this file sets or gets the end address of the address
 		range for the filter.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/<F>/min
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/min
 Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	If 'hugepage_size' is written to the 'type' file, writing to
 		or reading from this file sets or gets the minimum size of the
 		hugepage for the filter.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/<F>/max
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/max
 Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	If 'hugepage_size' is written to the 'type' file, writing to
 		or reading from this file sets or gets the maximum size of the
 		hugepage for the filter.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/<F>/target_idx
-Date:		Dec 2022
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/target_idx
+Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	If 'target' is written to the 'type' file, writing to or
 		reading from this file sets or gets the index of the DAMON
 		monitoring target of the interest.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/<F>/matching
-Date:		Dec 2022
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/matching
+Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	Writing 'Y' or 'N' to this file sets whether the filter is for
 		the memory of the 'type', or all except the 'type'.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters/<F>/allow
-Date:		Jan 2025
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters/<F>/allow
+Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	Writing 'Y' or 'N' to this file sets whether to allow or reject
 		applying the scheme's action to the memory that satisfies the
 		'type' and the 'matching' of the directory.
 
-What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters
-Date:		Feb 2025
-Contact:	SeongJae Park <sj@kernel.org>
-Description:	Directory for DAMON core layer-handled DAMOS filters.  Files
-		under this directory works same to those of
-		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters
-		directory.
-
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/ops_filters
 Date:		Feb 2025
 Contact:	SeongJae Park <sj@kernel.org>
 Description:	Directory for DAMON operations set layer-handled DAMOS filters.
 		Files under this directory works same to those of
-		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters
+		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/core_filters
 		directory.
 
+What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/filters
+Date:		Dec 2022
+Contact:	SeongJae Park <sj@kernel.org>
+Description:	Directory for DAMOS filters.  Files under this directory works
+		same to those of
+		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/{core,ops}_filters
+		directory.  This is deprecated.  Use the core_filters and
+		ops_filters instead.
+
 What:		/sys/kernel/mm/damon/admin/kdamonds/<K>/contexts/<C>/schemes/<S>/dests/nr_dests
 Date:		Jul 2025
 Contact:	SeongJae Park <sj@kernel.org>
diff --git a/Documentation/PCI/pci.rst b/Documentation/PCI/pci.rst
index f4d2662871ab1..be35e9a1ee75f 100644
--- a/Documentation/PCI/pci.rst
+++ b/Documentation/PCI/pci.rst
@@ -338,7 +338,7 @@ the PCI_IRQ_MSI and PCI_IRQ_MSIX flags will fail, so try to always
 specify PCI_IRQ_INTX as well.
 
 Drivers that have different interrupt handlers for MSI/MSI-X and
-legacy INTx should chose the right one based on the msi_enabled
+legacy INTx should choose the right one based on the msi_enabled
 and msix_enabled flags in the pci_dev structure after calling
 pci_alloc_irq_vectors.
 
diff --git a/Documentation/PCI/pciebus-howto.rst b/Documentation/PCI/pciebus-howto.rst
index 375d9ce171f62..9cc133ccdeece 100644
--- a/Documentation/PCI/pciebus-howto.rst
+++ b/Documentation/PCI/pciebus-howto.rst
@@ -97,7 +97,7 @@ register its service with the PCI Express Port Bus driver (see
 section 5.2.1 & 5.2.2). It is important that a service driver
 initializes the pcie_port_service_driver data structure, included in
 header file /include/linux/pcieport_if.h, before calling these APIs.
-Failure to do so will result an identity mismatch, which prevents
+Failure to do so will result in an identity mismatch, which prevents
 the PCI Express Port Bus driver from loading a service driver.
 
 pcie_port_service_register
diff --git a/Documentation/RCU/Design/Requirements/Requirements.rst b/Documentation/RCU/Design/Requirements/Requirements.rst
index 4d886e7c7a956..8a216e4a46a7d 100644
--- a/Documentation/RCU/Design/Requirements/Requirements.rst
+++ b/Documentation/RCU/Design/Requirements/Requirements.rst
@@ -206,7 +206,7 @@ non-\ ``NULL``, locklessly accessing the ``->a`` and ``->b`` fields.
 
        1 bool add_gp_buggy(int a, int b)
        2 {
-       3   p = kmalloc(sizeof(*p), GFP_KERNEL);
+       3   p = kmalloc_obj(*p);
        4   if (!p)
        5     return -ENOMEM;
        6   spin_lock(&gp_lock);
@@ -228,7 +228,7 @@ their rights to reorder this code as follows:
 
        1 bool add_gp_buggy_optimized(int a, int b)
        2 {
-       3   p = kmalloc(sizeof(*p), GFP_KERNEL);
+       3   p = kmalloc_obj(*p);
        4   if (!p)
        5     return -ENOMEM;
        6   spin_lock(&gp_lock);
@@ -264,7 +264,7 @@ shows an example of insertion:
 
        1 bool add_gp(int a, int b)
        2 {
-       3   p = kmalloc(sizeof(*p), GFP_KERNEL);
+       3   p = kmalloc_obj(*p);
        4   if (!p)
        5     return -ENOMEM;
        6   spin_lock(&gp_lock);
diff --git a/Documentation/RCU/listRCU.rst b/Documentation/RCU/listRCU.rst
index d8bb98623c124..48c7272a4cccb 100644
--- a/Documentation/RCU/listRCU.rst
+++ b/Documentation/RCU/listRCU.rst
@@ -276,7 +276,7 @@ The RCU version of audit_upd_rule() is as follows::
 
 		list_for_each_entry(e, list, list) {
 			if (!audit_compare_rule(rule, &e->rule)) {
-				ne = kmalloc(sizeof(*entry), GFP_ATOMIC);
+				ne = kmalloc_obj(*entry, GFP_ATOMIC);
 				if (ne == NULL)
 					return -ENOMEM;
 				audit_copy_rule(&ne->rule, &e->rule);
diff --git a/Documentation/RCU/whatisRCU.rst b/Documentation/RCU/whatisRCU.rst
index a1582bd653d11..e6767baff2d34 100644
--- a/Documentation/RCU/whatisRCU.rst
+++ b/Documentation/RCU/whatisRCU.rst
@@ -468,7 +468,7 @@ uses of RCU may be found in listRCU.rst and NMI-RCU.rst.
 		struct foo *new_fp;
 		struct foo *old_fp;
 
-		new_fp = kmalloc(sizeof(*new_fp), GFP_KERNEL);
+		new_fp = kmalloc_obj(*new_fp);
 		spin_lock(&foo_mutex);
 		old_fp = rcu_dereference_protected(gbl_foo, lockdep_is_held(&foo_mutex));
 		*new_fp = *old_fp;
@@ -570,7 +570,7 @@ The foo_update_a() function might then be written as follows::
 		struct foo *new_fp;
 		struct foo *old_fp;
 
-		new_fp = kmalloc(sizeof(*new_fp), GFP_KERNEL);
+		new_fp = kmalloc_obj(*new_fp);
 		spin_lock(&foo_mutex);
 		old_fp = rcu_dereference_protected(gbl_foo, lockdep_is_held(&foo_mutex));
 		*new_fp = *old_fp;
diff --git a/Documentation/accel/amdxdna/amdnpu.rst b/Documentation/accel/amdxdna/amdnpu.rst
index 42e54904f9a87..064973bf48939 100644
--- a/Documentation/accel/amdxdna/amdnpu.rst
+++ b/Documentation/accel/amdxdna/amdnpu.rst
@@ -270,6 +270,31 @@ MERT can report various kinds of telemetry information like the following:
 * Deep Sleep counter
 * etc.
 
+.. _amdxdna-usage-stats:
+
+Amdxdna DRM client usage stats implementation
+=============================================
+
+The amdxdna driver implements the DRM client usage stats specification as
+documented in :ref:`drm-client-usage-stats`.
+
+Example of the output showing the implemented key value pairs:
+
+::
+
+        pos:	0
+        flags:	0100002
+        mnt_id:	29
+        ino:	939
+        drm-driver:	amdxdna_accel_driver
+        drm-client-id:	3219
+        drm-pdev:	0000:c5:00.1
+        amdxdna_accel_driver-heap-alloc:	60 KiB
+        amdxdna_accel_driver-internal-alloc:	67588 KiB
+        amdxdna_accel_driver-external-alloc:	0
+        drm-total-memory:	67632 KiB
+        drm-shared-memory:	0
+
 
 References
 ==========
diff --git a/Documentation/admin-guide/LSM/Hornet.rst b/Documentation/admin-guide/LSM/Hornet.rst
new file mode 100644
index 0000000000000..0ade4c17374c6
--- /dev/null
+++ b/Documentation/admin-guide/LSM/Hornet.rst
@@ -0,0 +1,323 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+======
+Hornet
+======
+
+Hornet is a Linux Security Module that provides extensible signature
+verification for eBPF programs. This is selectable at build-time with
+``CONFIG_SECURITY_HORNET``.
+
+Overview
+========
+
+Hornet addresses concerns from users who require strict audit trails and
+verification guarantees for eBPF programs, especially in
+security-sensitive environments. Many production systems need assurance
+that only authorized, unmodified eBPF programs are loaded into the
+kernel. Hornet provides this assurance through cryptographic signature
+verification.
+
+When an eBPF program is loaded via the ``bpf()`` syscall, Hornet
+verifies a PKCS#7 signature attached to the program instructions. The
+signature is checked against whichever keyring was specified by the user
+existing kernel cryptographic infrastructure. In addition to signing the
+program bytecode, Hornet supports signing SHA-256 hashes of associated
+BPF maps, enabling integrity verification of map contents at load time
+and at runtime.
+
+After verification, Hornet classifies the program into one of the
+following integrity states and passes the result to a downstream LSM hook
+(``bpf_prog_load_post_integrity``), allowing other security modules to
+make policy decisions based on the verification outcome:
+
+``LSM_INT_VERDICT_OK``
+  The program signature and all map hashes verified successfully.
+
+``LSM_INT_VERDICT_UNSIGNED``
+  No signature was provided with the program.
+
+``LSM_INT_VERDICT_PARTIALSIG``
+  The program signature verified, but the signature did not contain
+  hornet map hash data.
+
+``LSM_INT_VERDICT_UNKNOWNKEY``
+  The keyring requested by the user is invalid.
+
+``LSM_INT_VERDICT_FAULT``
+  A system error occurred during verification.
+
+``LSM_INT_VERDICT_UNEXPECTED``
+  An unexpected map hash value was encountered.
+
+``LSM_INT_VERDICT_BADSIG``
+  The signature or a map hash failed verification.
+
+Hornet itself does not enforce a policy on whether unsigned or partially
+signed programs should be rejected. It delegates that decision to
+downstream LSMs via the ``bpf_prog_load_post_integrity`` hook, making it
+a composable building block in a larger security architecture.
+
+Use Cases
+=========
+
+- **Locked-down production environments**: Ensure only eBPF programs
+  signed by a trusted authority can be loaded, preventing unauthorized
+  or tampered programs from running in the kernel.
+
+- **Audit and compliance**: Provide cryptographic evidence that loaded
+  eBPF programs match their expected build artifacts, supporting
+  compliance requirements in regulated industries.
+
+- **Supply chain integrity**: Verify that eBPF programs and their
+  associated map data have not been modified since they were built and
+  signed, protecting against supply chain attacks.
+
+Threat Model
+============
+
+Hornet protects against the following threats:
+
+- **Unauthorized eBPF program loading**: Programs that have not been
+  signed by a trusted key will be reported as unsigned or badly signed.
+
+- **Tampering with program instructions**: Any modification to the eBPF
+  bytecode after signing will cause signature verification to fail.
+
+- **Tampering with map data**: When map hashes are included in the
+  signature, Hornet verifies that frozen BPF maps match their expected
+  SHA-256 hashes at load time. Maps are also re-verified before program
+  execution via ``BPF_PROG_RUN``.
+
+Hornet does **not** protect against:
+
+- Compromise of the signing key itself.
+- Attacks that occur after a program has been loaded and verified.
+- Programs loaded by the kernel itself (kernel-internal loads bypass
+  the ``BPF_PROG_RUN`` map check).
+
+Known Limitations
+=================
+
+- Hornet requires programs to use :doc:`light skeletons
+  </bpf/libbpf/libbpf_naming_convention>` (lskels) for the signing
+  workflow, as the tooling operates on lskel-generated headers.
+
+- A maximum of 64 maps per program can be tracked for hash
+  verification.
+
+- Map hash verification requires the maps to be frozen before loading.
+  Maps that are not frozen at load time will cause verification to fail
+  when their hashes are included in the signature.
+
+- The only hashing algorithm available is SHA256 due to it be hardcoded
+  in the bpf subsystem.
+
+- Hornet guarantees that the signed program runs only with signed map
+  data. It does not guarantee positional binding of maps to specific
+  fd_array slots.
+
+- BPF_MAP_TYPE_PROG_ARRAY maps must be frozen for Hornet to verify
+  them. Unfrozen prog array maps are not covered by verification.
+
+Configuration
+=============
+
+Build Configuration
+-------------------
+
+Enable Hornet by setting the following kernel configuration option::
+
+  CONFIG_SECURITY_HORNET=y
+
+This option is found under :menuselection:`Security options --> Hornet
+support` and depends on ``CONFIG_SECURITY``.
+
+When enabled, Hornet is included in the default LSM initialization order
+and will appear in ``/sys/kernel/security/lsm``.
+
+Architecture
+============
+
+Signature Verification Flow
+---------------------------
+
+The following describes what happens when a userspace program calls
+``bpf(BPF_PROG_LOAD, ...)`` with a signature attached:
+
+1. The ``bpf_prog_load_integrity`` LSM hook is invoked.
+
+2. Hornet reads the signature from the userspace buffer specified by
+   ``attr->signature`` (with length ``attr->signature_size``).
+
+3. The PKCS#7 signature is verified against the program instructions
+   using ``verify_pkcs7_message_sig()`` with the user specified keyring.
+
+4. The PKCS#7 message is parsed and its trust chain is validated via
+   ``validate_pkcs7_trust()``.
+
+5. Hornet extracts the authenticated attribute identified by
+   ``OID_hornet_data`` (OID ``2.25.316487325684022475439036912669789383960``)
+   from the PKCS#7 message. This attribute contains an ASN.1-encoded set
+   of map index/hash pairs.
+
+6. For each map hash entry, Hornet retrieves the corresponding BPF map
+   via its file descriptor, confirms it is frozen, computes its SHA-256
+   hash, and compares it against the signed hash.
+
+7. The resulting integrity verdict is passed to the
+   ``bpf_prog_load_post_integrity`` hook so that downstream LSMs can
+   enforce policy.
+
+Runtime Map Verification
+------------------------
+
+When ``bpf(BPF_PROG_RUN, ...)`` is called from userspace, Hornet
+re-verifies the hashes of all maps associated with the program. This
+ensures that map contents have not been modified between program load
+and execution. If any map hash no longer matches, the ``BPF_PROG_RUN``
+command is denied.
+
+Userspace Interface
+-------------------
+
+Signatures are passed to the kernel through fields in ``union bpf_attr``
+when using the ``BPF_PROG_LOAD`` command:
+
+``signature``
+  A pointer to a userspace buffer containing the PKCS#7 signature.
+
+``signature_size``
+  The size of the signature buffer in bytes.
+
+ASN.1 Schema
+------------
+
+Map hashes are encoded as a signed attribute in the PKCS#7 message using
+the following ASN.1 schema::
+
+  HornetData ::= SET OF Map
+
+  Map ::= SEQUENCE {
+      index   INTEGER,
+      sha     OCTET STRING
+  }
+
+Each ``Map`` entry contains the index of the map in the program's
+``fd_array`` and its expected SHA-256 hash. A zero-length ``sha`` field
+indicates that the map at that index should be skipped during
+verification.
+
+Tooling
+=======
+
+Helper scripts and a signature generation tool are provided in
+``scripts/hornet/`` to support the development of signed eBPF light
+skeletons.
+
+gen_sig
+-------
+
+``gen_sig`` is a C program (using OpenSSL) that creates a PKCS#7
+signature over eBPF program instructions and optionally includes
+SHA-256 hashes of BPF maps as signed attributes.
+
+Usage::
+
+  gen_sig --data <instructions.bin> \
+          --cert <signer.crt> \
+          --key <signer.key> \
+          [--pass <passphrase>] \
+          --out <signature.p7b> \
+          [--add <mapfile.bin>:<index> ...]
+
+``--data``
+  Path to the binary file containing eBPF program instructions to sign.
+
+``--cert``
+  Path to the signing certificate (PEM or DER format).
+
+``--key``
+  Path to the private key (PEM or DER format).
+
+``--pass``
+  Optional passphrase for the private key.
+
+``--out``
+  Path to write the output PKCS#7 signature.
+
+``--add``
+  Attach a map hash as a signed attribute. The argument is a path to a
+  binary map file followed by a colon and the map's index in the
+  ``fd_array``. This option may be specified multiple times.
+
+extract-skel.sh
+---------------
+
+Extracts a named field from an autogenerated eBPF lskel header file.
+Used internally by other helper scripts.
+
+extract-insn.sh
+---------------
+
+Extracts the eBPF program instructions (``opts_insn``) from an lskel
+header into a binary file suitable for signing with ``gen_sig``.
+
+extract-map.sh
+--------------
+
+Extracts the map data (``opts_data``) from an lskel header into a
+binary file suitable for hashing with ``gen_sig``.
+
+write-sig.sh
+------------
+
+Replaces the signature data in an lskel header with a new signature
+from a binary file. This is used to embed a freshly generated signature
+back into the header after signing.
+
+Signing Workflow
+================
+
+A typical workflow for building and signing an eBPF light skeleton is:
+
+1. **Compile the eBPF program**::
+
+     clang -O2 -target bpf -c program.bpf.c -o program.bpf.o
+
+2. **Generate the light skeleton header** using ``bpftool``::
+
+     bpftool gen skeleton -S program.bpf.o > loader.h
+
+3. **Extract instructions and map data** from the generated header::
+
+     scripts/hornet/extract-insn.sh loader.h > insn.bin
+     scripts/hornet/extract-map.sh loader.h > map.bin
+
+4. **Generate the signature** with ``gen_sig``::
+
+     scripts/hornet/gen_sig \
+       --key signing_key.pem \
+       --cert signing_key.x509 \
+       --data insn.bin \
+       --add map.bin:0 \
+       --out sig.bin
+
+5. **Embed the signature** back into the header::
+
+     scripts/hornet/write-sig.sh loader.h sig.bin > signed_loader.h
+
+6. **Build the loader program** using the signed header::
+
+     cc -o loader loader.c -lbpf
+
+The resulting loader program will pass the embedded signature to the
+kernel when loading the eBPF program, enabling Hornet to verify it.
+
+Testing
+=======
+
+Self-tests are provided in ``tools/testing/selftests/hornet/``. The test
+suite builds a minimal eBPF program (``trivial.bpf.c``), signs it using
+the workflow described above, and verifies that the signed program loads
+successfully.
diff --git a/Documentation/admin-guide/LSM/index.rst b/Documentation/admin-guide/LSM/index.rst
index b44ef68f6e4da..57f6e9fbe5fd1 100644
--- a/Documentation/admin-guide/LSM/index.rst
+++ b/Documentation/admin-guide/LSM/index.rst
@@ -49,3 +49,4 @@ subdirectories.
    SafeSetID
    ipe
    landlock
+   Hornet
diff --git a/Documentation/admin-guide/LSM/ipe.rst b/Documentation/admin-guide/LSM/ipe.rst
index a756d81585317..d68ba9d98859e 100644
--- a/Documentation/admin-guide/LSM/ipe.rst
+++ b/Documentation/admin-guide/LSM/ipe.rst
@@ -559,7 +559,8 @@ policy. Two properties are built-into the policy parser: 'op' and 'action'.
 The other properties are used to restrict immutable security properties
 about the files being evaluated. Currently those properties are:
 '``boot_verified``', '``dmverity_signature``', '``dmverity_roothash``',
-'``fsverity_signature``', '``fsverity_digest``'. A description of all
+'``fsverity_signature``', '``fsverity_digest``', '``bpf_signature``',
+'``bpf_keyring``', '``bpf_kernel``'. A description of all
 properties supported by IPE are listed below:
 
 op
@@ -603,6 +604,14 @@ as the first token. IPE supports the following operations:
       Controls loading IMA certificates through the Kconfigs,
       ``CONFIG_IMA_X509_PATH`` and ``CONFIG_EVM_X509_PATH``.
 
+   ``BPF_PROG_LOAD``:
+
+      Pertains to BPF programs being loaded via the ``bpf()`` syscall.
+      This operation is used in conjunction with the ``bpf_signature``,
+      ``bpf_keyring``, and ``bpf_kernel`` properties to control BPF
+      program loading based on integrity verification provided by the
+      Hornet LSM.
+
 action
 ~~~~~~
 
@@ -713,6 +722,105 @@ fsverity_signature
 
       fsverity_signature=(TRUE|FALSE)
 
+bpf_signature
+~~~~~~~~~~~~~
+
+   This property can be utilized for authorization of BPF program loads based
+   on the integrity verdict provided by the Hornet LSM. When a BPF program is
+   loaded, Hornet performs cryptographic verification of the program's PKCS#7
+   signature (if present) and passes an integrity verdict to IPE via the
+   ``security_bpf_prog_load_post_integrity`` hook. IPE can then allow or deny
+   the load based on the verdict.
+
+   This property depends on ``SECURITY_HORNET`` and is controlled by the
+   ``IPE_PROP_BPF_SIGNATURE`` config option.
+   The format of this property is::
+
+      bpf_signature=(NONE|OK|UNSIGNED|PARTIALSIG|UNKNOWNKEY|UNEXPECTED|FAULT|BADSIG)
+
+   The possible values correspond to the integrity verdicts from Hornet:
+
+      ``NONE``
+
+         No integrity verdict was set (default/uninitialized).
+
+      ``OK``
+
+         The BPF program's signature and all map hashes were successfully
+         verified.
+
+      ``UNSIGNED``
+
+         No signature was provided with the BPF program.
+
+      ``PARTIALSIG``
+
+         The program signature was verified, but no authenticated map hash
+         data was present.
+
+      ``UNKNOWNKEY``
+
+         The keyring requested by the user is invalid.
+
+      ``UNEXPECTED``
+
+         An unexpected map hash value was encountered during verification.
+
+      ``FAULT``
+
+         A system error occurred during signature verification.
+
+      ``BADSIG``
+
+         The signature or hash verification failed.
+
+bpf_keyring
+~~~~~~~~~~~~
+
+   This property can be utilized for authorization of BPF program loads based
+   on the keyring specified in the ``bpf_attr`` during the ``BPF_PROG_LOAD``
+   syscall. This allows policies to restrict which keyring must be used for
+   signature verification of BPF programs.
+
+   This property shares the ``IPE_PROP_BPF_SIGNATURE`` config option with
+   ``bpf_signature``.
+   The format of this property is::
+
+      bpf_keyring=(BUILTIN|SECONDARY|PLATFORM)
+
+   The possible values correspond to the system keyrings:
+
+      ``BUILTIN``
+
+         The builtin trusted keyring (``.builtin_trusted_keys``), which
+         contains keys embedded at kernel compile time.
+
+      ``SECONDARY``
+
+         The secondary trusted keyring (``.secondary_trusted_keys``), which
+         includes both builtin trusted keys and keys added at runtime.
+
+      ``PLATFORM``
+
+         The platform keyring (``.platform``), which contains keys provided
+         by the platform firmware (e.g. UEFI db keys).
+
+bpf_kernel
+~~~~~~~~~~
+
+   This property can be utilized for authorization of BPF program loads based
+   on whether the load originated from kernel space or user space. The BPF
+   light skeleton infrastructure performs a secondary kernel-originated program
+   load that will not carry a signature. This property allows policies to
+   permit such kernel-originated loads while still requiring signatures for
+   user-space loads.
+
+   This property shares the ``IPE_PROP_BPF_SIGNATURE`` config option with
+   ``bpf_signature``.
+   The format of this property is::
+
+      bpf_kernel=(TRUE|FALSE)
+
 Policy Examples
 ---------------
 
@@ -788,6 +896,58 @@ Allow execution of a specific fs-verity file
 
    op=EXECUTE fsverity_digest=sha256:fd88f2b8824e197f850bf4c5109bea5cf0ee38104f710843bb72da796ba5af9e action=ALLOW
 
+Allow only signed BPF programs
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+   policy_name=Allow_Signed_BPF policy_version=0.0.0
+   DEFAULT action=ALLOW
+
+   DEFAULT op=BPF_PROG_LOAD action=DENY
+   op=BPF_PROG_LOAD bpf_kernel=TRUE action=ALLOW
+   op=BPF_PROG_LOAD bpf_signature=OK action=ALLOW
+
+This policy allows all other operations but restricts BPF program loading
+to only programs that either originate from kernel space (e.g. light skeleton
+reloads) or have a valid signature verified by the Hornet LSM. Unsigned or
+improperly signed BPF programs from user space will be denied.
+
+Allow signed BPF programs from a specific keyring
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+   policy_name=Allow_BPF_Builtin_Keyring policy_version=0.0.0
+   DEFAULT action=ALLOW
+
+   DEFAULT op=BPF_PROG_LOAD action=DENY
+   op=BPF_PROG_LOAD bpf_kernel=TRUE action=ALLOW
+   op=BPF_PROG_LOAD bpf_signature=OK bpf_keyring=BUILTIN action=ALLOW
+
+This policy further restricts BPF program loading to only accept programs
+whose signatures were verified using the builtin trusted keyring. Programs
+signed against the secondary or platform keyrings will be denied, providing
+tighter control over which signing keys are acceptable.
+
+Allow signed BPF programs with relaxed partial signatures
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+   policy_name=Allow_BPF_Partial policy_version=0.0.0
+   DEFAULT action=ALLOW
+
+   DEFAULT op=BPF_PROG_LOAD action=DENY
+   op=BPF_PROG_LOAD bpf_kernel=TRUE action=ALLOW
+   op=BPF_PROG_LOAD bpf_signature=OK action=ALLOW
+   op=BPF_PROG_LOAD bpf_signature=PARTIALSIG action=ALLOW
+
+This policy allows BPF programs that have been fully verified (``OK``) as
+well as programs with a valid program signature but without authenticated
+map hash data (``PARTIALSIG``). This can be useful during development or
+for programs that do not use maps.
+
 Additional Information
 ----------------------
 
diff --git a/Documentation/admin-guide/binfmt-misc.rst b/Documentation/admin-guide/binfmt-misc.rst
index 59cd902e35497..c0a34fbf8022b 100644
--- a/Documentation/admin-guide/binfmt-misc.rst
+++ b/Documentation/admin-guide/binfmt-misc.rst
@@ -68,10 +68,10 @@ Here is what the fields mean:
 	    Legacy behavior of binfmt_misc is to pass the full path
             of the binary to the interpreter as an argument. When this flag is
             included, binfmt_misc will open the file for reading and pass its
-            descriptor as an argument, instead of the full path, thus allowing
-            the interpreter to execute non-readable binaries. This feature
-            should be used with care - the interpreter has to be trusted not to
-            emit the contents of the non-readable binary.
+            descriptor into the auxilary vector with the key "AT_EXECFD", thus
+            allowing the interpreter to execute non-readable binaries. This
+            feature should be used with care - the interpreter has to be trusted
+            not to emit the contents of the non-readable binary.
       ``C`` - credentials
             Currently, the behavior of binfmt_misc is to calculate
             the credentials and security token of the new process according to
diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst
index 6efd0095ed995..993446ab66d0f 100644
--- a/Documentation/admin-guide/cgroup-v2.rst
+++ b/Documentation/admin-guide/cgroup-v2.rst
@@ -2785,6 +2785,59 @@ RDMA Interface Files
 	  mlx4_0 hca_handle=1 hca_object=20
 	  ocrdma1 hca_handle=1 hca_object=23
 
+  rdma.peak
+	A read-only nested-keyed file that exists for all the cgroups
+	except root.  It shows the historical high watermark of
+	resource usage per device since the cgroup was created.
+
+	An example for mlx4 and ocrdma device follows::
+
+	  mlx4_0 hca_handle=1 hca_object=20
+	  ocrdma1 hca_handle=0 hca_object=23
+
+  rdma.events
+	A read-only nested-keyed file which exists on non-root
+	cgroups.  The following nested keys are defined.
+
+	  max
+		The number of times a process in this cgroup or its
+		descendants attempted an RDMA resource allocation that
+		was rejected because a rdma.max limit in the subtree
+		was reached.  This is a hierarchical counter: the event
+		is propagated upward to all ancestor cgroups.  A value
+		change in this file generates a file modified event.
+
+	  alloc_fail
+		The number of RDMA resource allocation attempts that
+		originated in this cgroup or its descendants and failed
+		due to a rdma.max limit being reached.  This is a
+		hierarchical counter propagated upward.
+
+	An example for mlx4 device follows::
+
+	  mlx4_0 hca_handle.max=5 hca_handle.alloc_fail=3 hca_object.max=0 hca_object.alloc_fail=0
+
+  rdma.events.local
+	Similar to rdma.events but the fields in the file are local
+	to the cgroup i.e. not hierarchical.  The file modified event
+	generated on this file reflects only the local events.
+
+	The following nested keys are defined.
+
+	  max
+		The number of times a process in this cgroup or its
+		descendants attempted an RDMA resource allocation that
+		was rejected because this cgroup's own rdma.max limit
+		was reached.
+	  alloc_fail
+		The number of RDMA resource allocation attempts
+		originating from this cgroup that failed due to this
+		cgroup's or an ancestor's rdma.max limit.
+
+	An example for mlx4 device follows::
+
+	  mlx4_0 hca_handle.max=5 hca_handle.alloc_fail=0 hca_object.max=0 hca_object.alloc_fail=0
+
 DMEM
 ----
 
diff --git a/Documentation/admin-guide/device-mapper/dm-inlinecrypt.rst b/Documentation/admin-guide/device-mapper/dm-inlinecrypt.rst
new file mode 100644
index 0000000000000..9b3069a5ec188
--- /dev/null
+++ b/Documentation/admin-guide/device-mapper/dm-inlinecrypt.rst
@@ -0,0 +1,123 @@
+==============
+dm-inlinecrypt
+==============
+
+Device-Mapper's "inlinecrypt" target provides transparent encryption of block devices
+using the inline encryption hardware.
+
+For a more detailed description of inline encryption, see:
+https://docs.kernel.org/block/inline-encryption.html
+
+Parameters::
+
+	      <cipher> <key> <iv_offset> <device path> \
+	      <offset> [<#opt_params> <opt_params>]
+
+<cipher>
+    Encryption cipher type.
+
+    The cipher specifications format is::
+
+       cipher
+
+    Examples::
+
+       aes-xts-plain64
+
+    The cipher type corresponds to the encryption modes supported by
+    inline crypto in the block layer. Currently, only
+    BLK_ENCRYPTION_MODE_AES_256_XTS (i.e. aes-xts-plain64) is supported.
+
+<key>
+    Key used for encryption. It is encoded either as a hexadecimal number
+    or it can be passed as <key_string> prefixed with single colon
+    character (':') for keys residing in kernel keyring service.
+    You can only use key sizes that are valid for the selected cipher.
+    Note that the size in bytes of a valid key must be in bellow range.
+
+        [BLK_CRYPTO_KEY_TYPE_RAW, BLK_CRYPTO_KEY_TYPE_HW_WRAPPED]
+
+<key_string>
+    The kernel keyring key is identified by string in following format:
+    <key_size>:<key_type>:<key_description>.
+
+<key_size>
+    The encryption key size in bytes. The kernel key payload size must match
+    the value passed in <key_size>.
+
+<key_type>
+    Either 'logon', or 'trusted' kernel key type.
+
+<key_description>
+    The kernel keyring key description inlinecrypt target should look for
+    when loading key of <key_type>.
+
+<iv_offset>
+    The IV offset is a sector count that is added to the sector number
+    before creating the IV.
+
+<device path>
+    This is the device that is going to be used as backend and contains the
+    encrypted data.  You can specify it as a path like /dev/xxx or a device
+    number <major>:<minor>.
+
+<offset>
+    Starting sector within the device where the encrypted data begins.
+
+<#opt_params>
+    Number of optional parameters. If there are no optional parameters,
+    the optional parameters section can be skipped or #opt_params can be zero.
+    Otherwise #opt_params is the number of following arguments.
+
+    Example of optional parameters section:
+        allow_discards sector_size:4096 iv_large_sectors
+
+allow_discards
+    Block discard requests (a.k.a. TRIM) are passed through the inlinecrypt
+    device. The default is to ignore discard requests.
+
+    WARNING: Assess the specific security risks carefully before enabling this
+    option.  For example, allowing discards on encrypted devices may lead to
+    the leak of information about the ciphertext device (filesystem type,
+    used space etc.) if the discarded blocks can be located easily on the
+    device later.
+
+sector_size:<bytes>
+    Use <bytes> as the encryption unit instead of 512 bytes sectors.
+    This option can be in range 512 - 4096 bytes and must be power of two.
+    Virtual device will announce this size as a minimal IO and logical sector.
+
+iv_large_sectors
+    Use <sector_size>-based sector numbers for IV generation instead of
+    512-byte sectors.
+
+    For dm-inlinecrypt, this flag must be specified when <sector_size>
+    is larger than 512 bytes. The legacy 512-byte-based IV behavior is
+    not supported.
+
+    When specified, if <sector_size> is 4096 bytes, plain64 IV for the
+    second sector will be 1, and <iv_offset> must be a multiple of
+    <sector_size> (in 512-byte units).
+
+Example scripts
+===============
+Currently, dm-inlinecrypt devices must be set up directly using dmsetup.
+There is no userspace support yet to integrate dm-inlinecrypt with LUKS
+or cryptsetup. In particular, cryptsetup currently only supports
+dm-crypt, and cannot be used to create dm-inlinecrypt mappings.
+
+The following examples demonstrate how to create dm-inlinecrypt devices
+using dmsetup
+
+::
+
+	#!/bin/sh
+	# Create a inlinecrypt device using dmsetup
+	dmsetup create inlinecrypt1 --table "0 `blockdev --getsz $1` inlinecrypt aes-xts-plain64 babebabebabebabebabebabebabebabebabebabebabebabebabebabebabebabe 0 $1 0"
+
+::
+
+	#!/bin/sh
+	# Create a inlinecrypt device using dmsetup when encryption key is stored in keyring service
+	dmsetup create inlinecrypt2 --table "0 `blockdev --getsz $1` inlinecrypt aes-xts-plain64 :64:logon:fde:dminlinecrypt_test_key 0 $1 0"
+
diff --git a/Documentation/admin-guide/device-mapper/index.rst b/Documentation/admin-guide/device-mapper/index.rst
index 030d854628ac2..73c1f20021343 100644
--- a/Documentation/admin-guide/device-mapper/index.rst
+++ b/Documentation/admin-guide/device-mapper/index.rst
@@ -15,6 +15,7 @@ Device Mapper
     dm-flakey
     dm-ima
     dm-init
+    dm-inlinecrypt
     dm-integrity
     dm-io
     dm-log
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 4d0f545fb3ec5..6f0910abdd259 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -24,7 +24,6 @@
 	IP_PNP	IP DHCP, BOOTP, or RARP is enabled.
 	IPV6	IPv6 support is enabled.
 	ISAPNP	ISA PnP code is enabled.
-	ISDN	Appropriate ISDN support is enabled.
 	ISOL	CPU Isolation is enabled.
 	JOY	Appropriate joystick support is enabled.
 	KGDB	Kernel debugger support is enabled.
@@ -36,7 +35,6 @@
 	M68k	M68k architecture is enabled.
 			These options have more detailed description inside of
 			Documentation/arch/m68k/kernel-options.rst.
-	MDA	MDA console support is enabled.
 	MIPS	MIPS architecture is enabled.
 	MOUSE	Appropriate mouse support is enabled.
 	MSI	Message Signaled Interrupts (PCI).
@@ -1246,8 +1244,9 @@ Kernel parameters
 			deferred probe to give up waiting on dependencies to
 			probe. Only specific dependencies (subsystems or
 			drivers) that have opted in will be ignored. A timeout
-			of 0 will timeout at the end of initcalls. If the time
-			out hasn't expired, it'll be restarted by each
+			of 0 will timeout at the end of initcalls; a negative
+			value is treated as an infinite timeout value. If the
+			timeout hasn't expired, it'll be restarted by each
 			successful driver registration. This option will also
 			dump out devices still on the deferred probe list after
 			retrying.
@@ -2100,6 +2099,10 @@ Kernel parameters
 			Format: nn[KMGTPE] or (node format)
 				<node>:nn[KMGTPE][,<node>:nn[KMGTPE]]
 
+			The size must be a multiple of the gigantic page size.
+			When using node format, this applies to each per-node size.
+			Missaligned values are dropped with a warning.
+
 			Reserve a CMA area of given size and allocate gigantic
 			hugepages using the CMA allocator. If enabled, the
 			boot-time allocation of gigantic hugepages is skipped.
@@ -2245,9 +2248,6 @@ Kernel parameters
 			syscalls, essentially overriding IA32_EMULATION_DEFAULT_DISABLED at
 			boot time. When false, unconditionally disables IA32 emulation.
 
-	icn=		[HW,ISDN]
-			Format: <io>[,<membase>[,<icn_id>[,<icn_id2>]]]
-
 
 	idle=		[X86,EARLY]
 			Format: idle=poll, idle=halt, idle=nomwait
@@ -2755,7 +2755,6 @@ Kernel parameters
 			Format: <RDP>,<reset>,<pci_scan>,<verbosity>
 
 	isolcpus=	[KNL,SMP,ISOL] Isolate a given set of CPUs from disturbance.
-			[Deprecated - use cpusets instead]
 			Format: [flag-list,]<cpu-list>
 
 			Specify one or more CPUs to isolate from disturbances
@@ -2780,11 +2779,10 @@ Kernel parameters
 			  Isolate from the general SMP balancing and scheduling
 			  algorithms. Note that performing domain isolation this way
 			  is irreversible: it's not possible to bring back a CPU to
-			  the domains once isolated through isolcpus. It's strongly
-			  advised to use cpusets instead to disable scheduler load
-			  balancing through the "cpuset.sched_load_balance" file.
-			  It offers a much more flexible interface where CPUs can
-			  move in and out of an isolated set anytime.
+			  the domains once isolated through this boot time
+			  configuration. Use cpusets for a dynamic configuration
+			  which can be altered at runtime. For details see
+			  Documentation/admin-guide/cpu-isolation.rst.
 
 			  You can move a process onto or off an "isolated" CPU via
 			  the CPU affinity syscalls or cpuset.
@@ -3816,10 +3814,6 @@ Kernel parameters
 	md=		[HW] RAID subsystems devices and level
 			See Documentation/admin-guide/md.rst.
 
-	mdacon=		[MDA]
-			Format: <first>,<last>
-			Specifies range of consoles to be captured by the MDA.
-
 	mds=		[X86,INTEL,EARLY]
 			Control mitigation for the Micro-architectural Data
 			Sampling (MDS) vulnerability.
@@ -4266,12 +4260,8 @@ Kernel parameters
 
 	n2=		[NET] SDL Inc. RISCom/N2 synchronous serial card
 
-	netdev=		[NET] Network devices parameters
+	netdev=		[NET] NE2000 ISA network devices parameters
 			Format: <irq>,<io>,<mem_start>,<mem_end>,<name>
-			Note that mem_start is often overloaded to mean
-			something different and driver-specific.
-			This usage is only documented in each driver source
-			file if at all.
 
 	netpoll.carrier_timeout=
 			[NET] Specifies amount of time (in seconds) that
@@ -4437,10 +4427,6 @@ Kernel parameters
 			These settings can be accessed at runtime via
 			the nmi_watchdog and hardlockup_panic sysctls.
 
-	no387		[BUGS=X86-32] Tells the kernel to use the 387 maths
-			emulation library even if a 387 maths coprocessor
-			is present.
-
 	no4lvl		[RISCV,EARLY] Disable 4-level and 5-level paging modes.
 			Forces kernel to use 3-level paging instead.
 
@@ -4644,7 +4630,7 @@ Kernel parameters
 	nosmt		[KNL,MIPS,PPC,EARLY] Disable symmetric multithreading (SMT).
 			Equivalent to smt=1.
 
-			[KNL,LOONGARCH,X86,PPC,S390] Disable symmetric multithreading (SMT).
+			[KNL,LOONGARCH,X86,ARM64,PPC,S390] Disable symmetric multithreading (SMT).
 			nosmt=force: Force disable SMT, cannot be undone
 				     via the sysfs control file.
 
@@ -5037,8 +5023,6 @@ Kernel parameters
 			the specified number of seconds.  This is to be used if
 			your oopses keep scrolling off the screen.
 
-	pcbit=		[HW,ISDN]
-
 	pci=option[,option...]	[PCI,EARLY] various PCI subsystem options.
 
 				Some options herein operate on a specific device
@@ -7236,6 +7220,18 @@ Kernel parameters
 			Not specifying this option is equivalent to
 			spec_store_bypass_disable=auto.
 
+	split_llc=
+			[X86,EARLY] Split the LLC N-ways
+
+			When set, the LLC is split this many ways by matching
+			'core_id % n'. This is setup before SMP bringup and
+			used during SMP bringup before it knows the full
+			topology. If your core count doesn't nicely divide by
+			the number given, you get to keep the pieces.
+
+			This is mostly a debug feature to emulate multiple LLCs
+			on hardware that only have a single LLC.
+
 	split_lock_detect=
 			[X86] Enable split lock detection or bus lock detection
 
@@ -7888,6 +7884,22 @@ Kernel parameters
 			first trust source as a backend which is initialized
 			successfully during iteration.
 
+	trusted.debug=	[KEYS]
+			Format: <bool>
+			Enable trusted keys debug traces at runtime when
+			CONFIG_TRUSTED_KEYS_DEBUG=y.
+
+			To make the traces visible after enabling the option,
+			use trusted.dyndbg='+p' as needed. By convention,
+			the subsystem uses pr_debug() for these traces.
+
+			SAFETY: The traces can leak sensitive data, so be
+			cautious before enabling this. They remain inactive
+			unless this parameter is set this option to  a true
+			value.
+
+			Default: false
+
 	trusted.rng=	[KEYS]
 			Format: <string>
 			The RNG used to generate key material for trusted keys.
diff --git a/Documentation/admin-guide/laptops/uniwill-laptop.rst b/Documentation/admin-guide/laptops/uniwill-laptop.rst
index 561334865feb7..1f3ca84c7d88b 100644
--- a/Documentation/admin-guide/laptops/uniwill-laptop.rst
+++ b/Documentation/admin-guide/laptops/uniwill-laptop.rst
@@ -43,6 +43,11 @@ Support for changing the platform performance mode is currently not implemented.
 Battery Charging Control
 ------------------------
 
+.. warning:: Some devices do not properly implement the charging threshold interface. Forcing
+             the driver to enable access to said interface on such devices might damage the
+             battery [1]_. Because of this the driver will not enable said feature even when
+             using the ``force`` module parameter.
+
 The ``uniwill-laptop`` driver supports controlling the battery charge limit. This happens over
 the standard ``charge_control_end_threshold`` power supply sysfs attribute. All values
 between 1 and 100 percent are supported.
@@ -70,3 +75,8 @@ The ``uniwill-laptop`` driver allows to set the configurable TGP for devices wit
 allow it.
 
 See Documentation/ABI/testing/sysfs-driver-uniwill-laptop for details.
+
+References
+==========
+
+.. [1] https://www.reddit.com/r/XMG_gg/comments/ld9yyf/battery_limit_hidden_function_discovered_on/
diff --git a/Documentation/admin-guide/media/amdisp4-1.rst b/Documentation/admin-guide/media/amdisp4-1.rst
new file mode 100644
index 0000000000000..878141154f962
--- /dev/null
+++ b/Documentation/admin-guide/media/amdisp4-1.rst
@@ -0,0 +1,63 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. include:: <isonum.txt>
+
+====================================
+AMD Image Signal Processor (amdisp4)
+====================================
+
+Introduction
+============
+
+This file documents the driver for the AMD ISP4 that is part of
+AMD Ryzen AI Max 300 Series.
+
+The driver is located under drivers/media/platform/amd/isp4 and uses
+the Media-Controller API.
+
+The driver exposes one video capture device to userspace and provide
+web camera like interface. Internally the video device is connected
+to the isp4 sub-device responsible for communication with the CCPU FW.
+
+Topology
+========
+
+.. _amdisp4_topology_graph:
+
+.. kernel-figure:: amdisp4.dot
+     :alt:   Diagram of the media pipeline topology
+     :align: center
+
+
+
+The driver has 1 sub-device: Representing isp4 image signal processor.
+The driver has 1 video device: Capture device for retrieving images.
+
+- ISP4 Image Signal Processing Subdevice Node
+
+---------------------------------------------
+
+The isp4 is represented as a single V4L2 subdev, the sub-device does not
+provide interface to the user space. The sub-device is connected to one video node
+(isp4_capture) with immutable active link. The sub-device represents ISP with
+connected sensor similar to smart cameras (sensors with integrated ISP).
+sub-device has only one link to the video device for capturing the frames.
+The sub-device communicates with CCPU FW for streaming configuration and
+buffer management.
+
+
+- isp4_capture - Frames Capture Video Node
+
+------------------------------------------
+
+Isp4_capture is a capture device to capture frames to memory.
+The entity is connected to isp4 sub-device. The video device
+provides web camera like interface to userspace. It supports
+mmap and dma buf types of memory.
+
+Capturing Video Frames Example
+==============================
+
+.. code-block:: bash
+
+         v4l2-ctl "-d" "/dev/video0" "--set-fmt-video=width=1920,height=1080,pixelformat=NV12" "--stream-mmap" "--stream-count=10"
diff --git a/Documentation/admin-guide/media/amdisp4.dot b/Documentation/admin-guide/media/amdisp4.dot
new file mode 100644
index 0000000000000..978f30c1a31a4
--- /dev/null
+++ b/Documentation/admin-guide/media/amdisp4.dot
@@ -0,0 +1,6 @@
+digraph board {
+	rankdir=TB
+	n00000001 [label="{{} | amd isp4\n | {<port0> 0}}", shape=Mrecord, style=filled, fillcolor=green]
+	n00000001:port0 -> n00000003 [style=bold]
+	n00000003 [label="Preview\n/dev/video0", shape=box, style=filled, fillcolor=yellow]
+}
diff --git a/Documentation/admin-guide/media/v4l-drivers.rst b/Documentation/admin-guide/media/v4l-drivers.rst
index d31da8e0a54f2..4621eae9fa1ed 100644
--- a/Documentation/admin-guide/media/v4l-drivers.rst
+++ b/Documentation/admin-guide/media/v4l-drivers.rst
@@ -9,6 +9,7 @@ Video4Linux (V4L) driver-specific documentation
 .. toctree::
 	:maxdepth: 2
 
+	amdisp4-1
 	bttv
 	c3-isp
 	cafe_ccic
diff --git a/Documentation/admin-guide/mm/damon/lru_sort.rst b/Documentation/admin-guide/mm/damon/lru_sort.rst
index 14cc6b2db8973..b93ca9b0853d2 100644
--- a/Documentation/admin-guide/mm/damon/lru_sort.rst
+++ b/Documentation/admin-guide/mm/damon/lru_sort.rst
@@ -75,7 +75,7 @@ Make DAMON_LRU_SORT reads the input parameters again, except ``enabled``.
 
 Input parameters that updated while DAMON_LRU_SORT is running are not applied
 by default.  Once this parameter is set as ``Y``, DAMON_LRU_SORT reads values
-of parametrs except ``enabled`` again.  Once the re-reading is done, this
+of parameters except ``enabled`` again.  Once the re-reading is done, this
 parameter is set as ``N``.  If invalid parameters are found while the
 re-reading, DAMON_LRU_SORT will be disabled.
 
@@ -246,7 +246,8 @@ monitor_region_start
 Start of target memory region in physical address.
 
 The start physical address of memory region that DAMON_LRU_SORT will do work
-against.  By default, biggest System RAM is used as the region.
+against.  By default, the system's entire physical memory is used as the
+region.
 
 monitor_region_end
 ------------------
@@ -254,7 +255,8 @@ monitor_region_end
 End of target memory region in physical address.
 
 The end physical address of memory region that DAMON_LRU_SORT will do work
-against.  By default, biggest System RAM is used as the region.
+against.  By default, the system's entire physical memory is used as the
+region.
 
 addr_unit
 ---------
diff --git a/Documentation/admin-guide/mm/damon/reclaim.rst b/Documentation/admin-guide/mm/damon/reclaim.rst
index d7a0225b49508..ec7e3e32b4ac6 100644
--- a/Documentation/admin-guide/mm/damon/reclaim.rst
+++ b/Documentation/admin-guide/mm/damon/reclaim.rst
@@ -67,7 +67,7 @@ Make DAMON_RECLAIM reads the input parameters again, except ``enabled``.
 
 Input parameters that updated while DAMON_RECLAIM is running are not applied
 by default.  Once this parameter is set as ``Y``, DAMON_RECLAIM reads values
-of parametrs except ``enabled`` again.  Once the re-reading is done, this
+of parameters except ``enabled`` again.  Once the re-reading is done, this
 parameter is set as ``N``.  If invalid parameters are found while the
 re-reading, DAMON_RECLAIM will be disabled.
 
@@ -85,6 +85,17 @@ identifies the region as cold, and reclaims it.
 
 120 seconds by default.
 
+autotune_monitoring_intervals
+-----------------------------
+
+If this parameter is set as ``Y``, DAMON_RECLAIM automatically tunes DAMON's
+sampling and aggregation intervals.  The auto-tuning aims to capture meaningful
+amount of access events in each DAMON-snapshot, while keeping the sampling
+interval 5 milliseconds in minimum, and 10 seconds in maximum.  Setting this as
+``N`` disables the auto-tuning.
+
+Disabled by default.
+
 quota_ms
 --------
 
@@ -229,7 +240,8 @@ Start of target memory region in physical address.
 
 The start physical address of memory region that DAMON_RECLAIM will do work
 against.  That is, DAMON_RECLAIM will find cold memory regions in this region
-and reclaims.  By default, biggest System RAM is used as the region.
+and reclaims.  By default, the system's entire physical memory is used as the
+region.
 
 monitor_region_end
 ------------------
@@ -238,7 +250,8 @@ End of target memory region in physical address.
 
 The end physical address of memory region that DAMON_RECLAIM will do work
 against.  That is, DAMON_RECLAIM will find cold memory regions in this region
-and reclaims.  By default, biggest System RAM is used as the region.
+and reclaims.  By default, the system's entire physical memory is used as the
+region.
 
 addr_unit
 ---------
diff --git a/Documentation/admin-guide/mm/damon/stat.rst b/Documentation/admin-guide/mm/damon/stat.rst
index c4b14daeb2dd6..46c5dd96aa2ed 100644
--- a/Documentation/admin-guide/mm/damon/stat.rst
+++ b/Documentation/admin-guide/mm/damon/stat.rst
@@ -89,3 +89,10 @@ percentiles of the idle time values via this read-only parameter.  Reading the
 parameter returns 101 idle time values in milliseconds, separated by comma.
 Each value represents 0-th, 1st, 2nd, 3rd, ..., 99th and 100th percentile idle
 times.
+
+kdamond_pid
+-----------
+
+PID of the DAMON thread.
+
+If DAMON_STAT is enabled, this becomes the PID of the worker thread.  Else, -1.
diff --git a/Documentation/admin-guide/mm/damon/usage.rst b/Documentation/admin-guide/mm/damon/usage.rst
index 534e1199cf091..11c75a598393c 100644
--- a/Documentation/admin-guide/mm/damon/usage.rst
+++ b/Documentation/admin-guide/mm/damon/usage.rst
@@ -66,7 +66,8 @@ comma (",").
     │ :ref:`kdamonds <sysfs_kdamonds>`/nr_kdamonds
     │ │ :ref:`0 <sysfs_kdamond>`/state,pid,refresh_ms
     │ │ │ :ref:`contexts <sysfs_contexts>`/nr_contexts
-    │ │ │ │ :ref:`0 <sysfs_context>`/avail_operations,operations,addr_unit
+    │ │ │ │ :ref:`0 <sysfs_context>`/avail_operations,operations,addr_unit,
+    │ │ │ │   pause
     │ │ │ │ │ :ref:`monitoring_attrs <sysfs_monitoring_attrs>`/
     │ │ │ │ │ │ intervals/sample_us,aggr_us,update_us
     │ │ │ │ │ │ │ intervals_goal/access_bp,aggrs,min_sample_us,max_sample_us
@@ -83,7 +84,9 @@ comma (",").
     │ │ │ │ │ │ │ │ sz/min,max
     │ │ │ │ │ │ │ │ nr_accesses/min,max
     │ │ │ │ │ │ │ │ age/min,max
-    │ │ │ │ │ │ │ :ref:`quotas <sysfs_quotas>`/ms,bytes,reset_interval_ms,effective_bytes,goal_tuner
+    │ │ │ │ │ │ │ :ref:`quotas <sysfs_quotas>`/ms,bytes,reset_interval_ms,
+    │ │ │ │ │ │ │     effective_bytes,goal_tuner,
+    │ │ │ │ │ │ │     fail_charge_num,fail_charge_denom
     │ │ │ │ │ │ │ │ weights/sz_permil,nr_accesses_permil,age_permil
     │ │ │ │ │ │ │ │ :ref:`goals <sysfs_schemes_quota_goals>`/nr_goals
     │ │ │ │ │ │ │ │ │ 0/target_metric,target_value,current_value,nid,path
@@ -194,9 +197,9 @@ details).  At the moment, only one context per kdamond is supported, so only
 contexts/<N>/
 -------------
 
-In each context directory, three files (``avail_operations``, ``operations``
-and ``addr_unit``) and three directories (``monitoring_attrs``, ``targets``,
-and ``schemes``) exist.
+In each context directory, four files (``avail_operations``, ``operations``,
+``addr_unit`` and ``pause``) and three directories (``monitoring_attrs``,
+``targets``, and ``schemes``) exist.
 
 DAMON supports multiple types of :ref:`monitoring operations
 <damon_design_configurable_operations_set>`, including those for virtual address
@@ -214,6 +217,9 @@ reading from the ``operations`` file.
 ``addr_unit`` file is for setting and getting the :ref:`address unit
 <damon_design_addr_unit>` parameter of the operations set.
 
+``pause`` file is for setting and getting the :ref:`pause request
+<damon_design_execution_model_and_data_structures>` parameter of the context.
+
 .. _sysfs_monitoring_attrs:
 
 contexts/<N>/monitoring_attrs/
@@ -377,9 +383,10 @@ schemes/<N>/quotas/
 The directory for the :ref:`quotas <damon_design_damos_quotas>` of the given
 DAMON-based operation scheme.
 
-Under ``quotas`` directory, five files (``ms``, ``bytes``,
-``reset_interval_ms``, ``effective_bytes`` and ``goal_tuner``) and two
-directories (``weights`` and ``goals``) exist.
+Under ``quotas`` directory, seven files (``ms``, ``bytes``,
+``reset_interval_ms``, ``effective_bytes``, ``goal_tuner``, ``fail_charge_num``
+and ``fail_charge_denom``) and two directories (``weights`` and ``goals``)
+exist.
 
 You can set the ``time quota`` in milliseconds, ``size quota`` in bytes, and
 ``reset interval`` in milliseconds by writing the values to the three files,
@@ -398,6 +405,13 @@ the background design of the feature and the name of the selectable algorithms.
 Refer to :ref:`goals directory <sysfs_schemes_quota_goals>` for the goals
 setup.
 
+You can set the action-failed memory quota charging ratio by writing the
+numerator and the denominator for the ratio to ``fail_charge_num`` and
+``fail_charge_denom`` files, respectively.  Reading those files will return the
+current set values.  Refer to :ref:`design
+<damon_design_damos_quotas_failed_memory_charging_ratio>` for more details of
+the ratio feature.
+
 The time quota is internally transformed to a size quota.  Between the
 transformed size quota and user-specified size quota, smaller one is applied.
 Based on the user-specified :ref:`goal <sysfs_schemes_quota_goals>`, the
@@ -471,10 +485,10 @@ directory can be used for installing filters regardless of their handled
 layers.  Filters that requested by ``core_filters`` and ``ops_filters`` will be
 installed before those of ``filters``.  All three directories have same files.
 
-Use of ``filters`` directory can make expecting evaluation orders of given
-filters with the files under directory bit confusing.  Users are hence
-recommended to use ``core_filters`` and ``ops_filters`` directories.  The
-``filters`` directory could be deprecated in future.
+Use of ``filters`` directory can make filters evaluation orders confusing to
+expect.  For this reason, ``filters`` directory is deprecated.  It is still
+functioning, but is scheduled for removal in the near future.  Users should use
+``core_filters`` and ``ops_filters`` directories instead.
 
 In the beginning, the directory has only one file, ``nr_filters``.  Writing a
 number (``N``) to the file creates the number of child directories named ``0``
diff --git a/Documentation/admin-guide/mm/hugetlbpage.rst b/Documentation/admin-guide/mm/hugetlbpage.rst
index 67a941903fd22..3cc15d800be11 100644
--- a/Documentation/admin-guide/mm/hugetlbpage.rst
+++ b/Documentation/admin-guide/mm/hugetlbpage.rst
@@ -455,7 +455,7 @@ used to change the file attributes on hugetlbfs.
 Also, it is important to note that no such mount command is required if
 applications are going to use only shmat/shmget system calls or mmap with
 MAP_HUGETLB.  For an example of how to use mmap with MAP_HUGETLB see
-:ref:`map_hugetlb <map_hugetlb>` below.
+:ref:`examples <examples>` below.
 
 Users who wish to use hugetlb memory via shared memory segment should be
 members of a supplementary group and system admin needs to configure that gid
@@ -473,16 +473,13 @@ a hugetlb page and the length is smaller than the hugepage size.
 Examples
 ========
 
-.. _map_hugetlb:
+.. _examples:
 
-``map_hugetlb``
-	see tools/testing/selftests/mm/map_hugetlb.c
+``hugetlb-shm``
+	see tools/testing/selftests/mm/hugetlb-shm.c
 
-``hugepage-shm``
-	see tools/testing/selftests/mm/hugepage-shm.c
-
-``hugepage-mmap``
-	see tools/testing/selftests/mm/hugepage-mmap.c
+``hugetlb-mmap``
+	see tools/testing/selftests/mm/hugetlb-mmap.c
 
 The `libhugetlbfs`_  library provides a wide range of userspace tools
 to help with huge page usability, environment setup, and control.
diff --git a/Documentation/admin-guide/mm/transhuge.rst b/Documentation/admin-guide/mm/transhuge.rst
index 5fbc3d89bb073..fc0127a36ef61 100644
--- a/Documentation/admin-guide/mm/transhuge.rst
+++ b/Documentation/admin-guide/mm/transhuge.rst
@@ -63,7 +63,8 @@ often.
 THP can be enabled system wide or restricted to certain tasks or even
 memory ranges inside task's address space. Unless THP is completely
 disabled, there is ``khugepaged`` daemon that scans memory and
-collapses sequences of basic pages into PMD-sized huge pages.
+collapses sequences of basic pages into huge pages of either PMD size
+or mTHP sizes, if the system is configured to do so.
 
 The THP behaviour is controlled via :ref:`sysfs <thp_sysfs>`
 interface and using madvise(2) and prctl(2) system calls.
@@ -219,10 +220,10 @@ this behaviour by writing 0 to shrink_underused, and enable it by writing
 	echo 0 > /sys/kernel/mm/transparent_hugepage/shrink_underused
 	echo 1 > /sys/kernel/mm/transparent_hugepage/shrink_underused
 
-khugepaged will be automatically started when PMD-sized THP is enabled
+khugepaged will be automatically started when any THP size is enabled
 (either of the per-size anon control or the top-level control are set
 to "always" or "madvise"), and it'll be automatically shutdown when
-PMD-sized THP is disabled (when both the per-size anon control and the
+all THP sizes are disabled (when both the per-size anon control and the
 top-level control are "never")
 
 process THP controls
@@ -264,11 +265,6 @@ support the following arguments::
 Khugepaged controls
 -------------------
 
-.. note::
-   khugepaged currently only searches for opportunities to collapse to
-   PMD-sized THP and no attempt is made to collapse to other THP
-   sizes.
-
 khugepaged runs usually at low frequency so while one may not want to
 invoke defrag algorithms synchronously during the page faults, it
 should be worth invoking defrag at least in khugepaged. However it's
@@ -296,11 +292,11 @@ allocation failure to throttle the next allocation attempt::
 The khugepaged progress can be seen in the number of pages collapsed (note
 that this counter may not be an exact count of the number of pages
 collapsed, since "collapsed" could mean multiple things: (1) A PTE mapping
-being replaced by a PMD mapping, or (2) All 4K physical pages replaced by
-one 2M hugepage. Each may happen independently, or together, depending on
-the type of memory and the failures that occur. As such, this value should
-be interpreted roughly as a sign of progress, and counters in /proc/vmstat
-consulted for more accurate accounting)::
+being replaced by a PMD mapping, or (2) physical pages replaced by one
+hugepage of various sizes (PMD-sized or mTHP). Each may happen independently,
+or together, depending on the type of memory and the failures that occur.
+As such, this value should be interpreted roughly as a sign of progress,
+and counters in /proc/vmstat consulted for more accurate accounting)::
 
 	/sys/kernel/mm/transparent_hugepage/khugepaged/pages_collapsed
 
@@ -308,16 +304,20 @@ for each pass::
 
 	/sys/kernel/mm/transparent_hugepage/khugepaged/full_scans
 
-``max_ptes_none`` specifies how many extra small pages (that are
-not already mapped) can be allocated when collapsing a group
-of small pages into one large page::
+``max_ptes_none`` specifies how many empty (none/zero) pages are allowed
+when collapsing a group of small pages into one large page::
 
 	/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none
 
-A higher value leads to use additional memory for programs.
-A lower value leads to gain less thp performance. Value of
-max_ptes_none can waste cpu time very little, you can
-ignore it.
+For PMD-sized THP collapse, this directly limits the number of empty pages
+allowed in the 2MB region.
+
+For mTHP collapse, only 0 or (HPAGE_PMD_NR - 1) are supported. Any other value
+will emit a warning and no mTHP collapse will be attempted.
+
+A higher value allows more empty pages, potentially leading to more memory
+usage but better THP performance. A lower value is more conservative and
+may result in fewer THP collapses.
 
 ``max_ptes_swap`` specifies how many pages can be brought in from
 swap when collapsing a group of pages into a transparent huge page::
@@ -337,6 +337,15 @@ that THP is shared. Exceeding the number would block the collapse::
 
 A higher value may increase memory footprint for some workloads.
 
+.. note::
+   For mTHP collapse, khugepaged does not support collapsing regions that
+   contain shared or swapped out pages, as this could lead to continuous
+   promotion to higher orders. The collapse will fail if any shared or
+   swapped PTEs are encountered during the scan.
+
+   Currently, madvise_collapse only supports collapsing to PMD-sized THPs
+   and does not attempt mTHP collapses.
+
 Boot parameters
 ===============
 
@@ -639,6 +648,14 @@ anon_fault_fallback_charge
 	instead falls back to using huge pages with lower orders or
 	small pages even though the allocation was successful.
 
+collapse_alloc
+	is incremented every time a huge page is successfully allocated for a
+	khugepaged collapse.
+
+collapse_alloc_failed
+	is incremented every time a huge page allocation fails during a
+	khugepaged collapse.
+
 zswpout
 	is incremented every time a huge page is swapped out to zswap in one
 	piece without splitting.
@@ -706,6 +723,20 @@ nr_anon_partially_mapped
        an anonymous THP as "partially mapped" and count it here, even though it
        is not actually partially mapped anymore.
 
+collapse_exceed_none_pte
+       The number of collapse attempts that failed due to exceeding the
+       max_ptes_none threshold.
+
+collapse_exceed_swap_pte
+       The number of collapse attempts that failed due to exceeding the
+       max_ptes_swap threshold. For non-PMD orders this occurs if a mTHP range
+       contains at least one swap PTE.
+
+collapse_exceed_shared_pte
+       The number of collapse attempts that failed due to exceeding the
+       max_ptes_shared threshold. For non-PMD orders this occurs if a mTHP range
+       contains at least one shared PTE.
+
 As the system ages, allocating huge pages may be expensive as the
 system uses memory compaction to copy data around memory to free a
 huge page for use. There are some counters in ``/proc/vmstat`` to help
diff --git a/Documentation/admin-guide/pm/amd-pstate.rst b/Documentation/admin-guide/pm/amd-pstate.rst
index f8e7050fc7623..a95e2ebce0057 100644
--- a/Documentation/admin-guide/pm/amd-pstate.rst
+++ b/Documentation/admin-guide/pm/amd-pstate.rst
@@ -358,9 +358,9 @@ Dynamic energy performance profile
 The amd-pstate driver supports dynamically selecting the energy performance
 profile based on whether the machine is running on AC or DC power.
 
-Whether this behavior is enabled by default depends on the kernel
-config option `CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP`. This behavior can also be overridden
-at runtime by the sysfs file ``/sys/devices/system/cpu/cpufreq/policyX/dynamic_epp``.
+Whether this behavior is enabled by default depends on the kernel command line option
+``amd_dynamic_epp`` is set. This behavior can also be overridden
+at runtime by the sysfs file ``/sys/devices/system/cpu/amd_pstate/dynamic_epp``.
 
 When set to enabled, the driver will select a different energy performance
 profile when the machine is running on battery or AC power. The driver will
@@ -485,9 +485,8 @@ kernel parameter ``amd_prefcore=disable``.
 ``amd_dynamic_epp``
 
 When AMD pstate is in auto mode, dynamic EPP will control whether the kernel
-autonomously changes the EPP mode. The default is configured by
-``CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP`` but can be explicitly enabled with
-``amd_dynamic_epp=enable`` or disabled with ``amd_dynamic_epp=disable``.
+autonomously changes the EPP mode. The default is disabled. It can be enabled
+with the kernel parameter ``amd_dynamic_epp=enable``.
 
 User Space Interface in ``sysfs`` - General
 ===========================================
diff --git a/Documentation/admin-guide/pm/intel_pstate.rst b/Documentation/admin-guide/pm/intel_pstate.rst
index fde967b0c2e0e..25fe5d88fea6c 100644
--- a/Documentation/admin-guide/pm/intel_pstate.rst
+++ b/Documentation/admin-guide/pm/intel_pstate.rst
@@ -355,11 +355,12 @@ HyperThreading (HT) in the context of Intel processors, is enabled on at least
 one core, ``intel_pstate`` assigns performance-based priorities to CPUs.  Namely,
 the priority of a given CPU reflects its highest HWP performance level which
 causes the CPU scheduler to generally prefer more performant CPUs, so the less
-performant CPUs are used when the other ones are fully loaded.  However, SMT
-siblings (that is, logical CPUs sharing one physical core) are treated in a
-special way such that if one of them is in use, the effective priority of the
-other ones is lowered below the priorities of the CPUs located in the other
-physical cores.
+performant CPUs are used when the other ones are fully loaded.  SMT siblings
+(that is, logical CPUs sharing one physical core) are given the same priority.
+The scheduler can pull tasks from lower-priority cores and place them on any
+sibling.  Since the scheduler spreads tasks among physical cores, tasks will be
+placed on the SMT siblings of physical cores only after all physical cores are
+busy.
 
 This approach maximizes performance in the majority of cases, but unfortunately
 it also leads to excessive energy usage in some important scenarios, like video
diff --git a/Documentation/admin-guide/pm/intel_uncore_frequency_scaling.rst b/Documentation/admin-guide/pm/intel_uncore_frequency_scaling.rst
index d367ba4d744af..b43ad4d5e333b 100644
--- a/Documentation/admin-guide/pm/intel_uncore_frequency_scaling.rst
+++ b/Documentation/admin-guide/pm/intel_uncore_frequency_scaling.rst
@@ -88,8 +88,15 @@ and "fabric_cluster_id" in the directory.
 
 Attributes in each directory:
 
+``instance_id``
+	This attribute is used to get die indices in userspace mapped MMIO
+	blocks. Indices are local to a single TPMI partition. Needed for direct
+	TPMI register access.
+
 ``domain_id``
 	This attribute is used to get the power domain id of this instance.
+	Indices are unique in all TPMI partitions on a given CPU package. Can be
+	used to map compute dies to corresponding CPUs.
 
 ``die_id``
 	This attribute is used to get the Linux die id of this instance.
diff --git a/Documentation/admin-guide/reporting-issues.rst b/Documentation/admin-guide/reporting-issues.rst
index 16a66a1f1975f..87dd874fffcf2 100644
--- a/Documentation/admin-guide/reporting-issues.rst
+++ b/Documentation/admin-guide/reporting-issues.rst
@@ -129,7 +129,7 @@ After these preparations you'll now enter the main part:
    situations; during the merge window that actually might be even the best
    approach, but in that development phase it can be an even better idea to
    suspend your efforts for a few days anyway. Whatever version you choose,
-   ideally use a 'vanilla' build. Ignoring these advices will dramatically
+   ideally use a 'vanilla' build. Ignoring all of this advice will dramatically
    increase the risk your report will be rejected or ignored.
 
  * Ensure the kernel you just installed does not 'taint' itself when
@@ -795,7 +795,7 @@ Install a fresh kernel for testing
     situations; during the merge window that actually might be even the best
     approach, but in that development phase it can be an even better idea to
     suspend your efforts for a few days anyway. Whatever version you choose,
-    ideally use a 'vanilla' built. Ignoring these advices will dramatically
+    ideally use a 'vanilla' built. Ignoring all of this advice will dramatically
     increase the risk your report will be rejected or ignored.*
 
 As mentioned in the detailed explanation for the first step already: Like most
diff --git a/Documentation/admin-guide/thunderbolt.rst b/Documentation/admin-guide/thunderbolt.rst
index 89df26553aa03..91a6cb1099889 100644
--- a/Documentation/admin-guide/thunderbolt.rst
+++ b/Documentation/admin-guide/thunderbolt.rst
@@ -373,6 +373,67 @@ port which are named like ``thunderbolt0`` and so on. From this point
 you can either use standard userspace tools like ``ip`` to
 configure the interface or let your GUI handle it automatically.
 
+Streaming data directly over Thunderbolt cable
+----------------------------------------------
+In addition to Thunderbolt networking (aka. USB4NET) Linux supports
+streaming data directly over a cable as well (aka. USB4STREAM). This is
+possible through ``thunderbolt-stream`` driver.
+
+Similarly to ``thunderbolt-net`` you load the driver first on one end::
+
+  host1 # modprobe thunderbolt-stream
+
+Then you configure it via ``ConfigFS``::
+
+  host1 # cd /sys/kernel/config/thunderbolt/stream
+  host1 # mkdir -p 0-1.0/data
+  host1 # cd 0-1.0
+  host1 # echo -1 > data/in_hopid
+  host1 # echo -1 > data/out_hopid
+
+This information is automatically announced to the other side via
+XDomain properties so if you have cable connected the other side knows
+that there is a stream named ``data`` available and can configure it for
+you automatically::
+
+  host2 # cd /sys/kernel/config/thunderbolt/stream
+  host2 # mkdir -p 0-3.0/data
+
+Here we used auto-configuration but you can configure it manually too.
+In that case you need to fill ``in_hopid`` and ``out_hopid`` accordingly.
+If you set them to ``-1`` the next available HopID is used which is
+typically what we want.
+
+Once they are configured you can use ``/dev/tbstreamX`` on both sides to
+transfer data::
+
+  host2 # cat /dev/tbstream0
+  host1 # dmesg > /dev/tbstream0
+
+Once you are done with the stream you can remove them::
+
+  host2 # cd /sys/kernel/config/thunderbolt/stream
+  host2 # rmdir -p 0-1.0/data
+  host1 # cd /sys/kernel/config/thunderbolt/stream
+  host1 # rmdir -p 0-3.0/data
+
+Since streams are essentially files you can use any existing application
+that supports ``read(2)`` and ``write(2)`` in some form.
+
+It is possible to have more than one stream and you can have both stream
+and ``thunderbolt-net`` in use simultaneously. For example we can create
+two streams with name ``control`` and ``data`` like this::
+
+  host1 # cd /sys/kernel/config/thunderbolt/stream
+  host1 # mkdir 0-1.0
+  host1 # cd 0-1.0
+  host1 # mkdir control
+  host1 # mkdir data
+
+Then you have ``/dev/tbstream0`` for ``control`` and ``/dev/tbstream1``
+for ``data``. Before you can use them you need to configure them as
+shown above for the one stream case.
+
 Forcing power
 -------------
 Many OEMs include a method that can be used to force the power of a
diff --git a/Documentation/admin-guide/workload-tracing.rst b/Documentation/admin-guide/workload-tracing.rst
index 35963491b9f19..314e5f03474e6 100644
--- a/Documentation/admin-guide/workload-tracing.rst
+++ b/Documentation/admin-guide/workload-tracing.rst
@@ -202,6 +202,15 @@ database. To get out of this mode press ctrl+d. -p option is used to
 specify the number of file path components to display. -p10 is optimal
 for browsing kernel sources.
 
+Alternatively, the kernel build system can generate the cscope database::
+
+  make cscope
+
+To exclude directories from the generated database, pass IGNORE_DIRS to
+the cscope target. For example, to exclude Documentation/, run::
+
+  make IGNORE_DIRS="Documentation" cscope
+
 What is perf and how do we use it?
 ==================================
 
@@ -243,13 +252,21 @@ which can help mitigate performance regressions. It also acts as a common
 benchmarking framework, enabling developers to easily create test cases,
 integrate transparently, and use performance-rich tooling.
 
-"perf bench all" command runs the following benchmarks:
+"perf bench all" runs all available benchmarks in the perf bench
+framework. The exact set of benchmarks depends on the perf version and on
+the features enabled when perf was built.
+
+To list the benchmark collections available on the current system, run::
+
+  perf bench
+
+To list benchmarks in a collection, run::
+
+  perf bench <collection>
+
+For example, to list the benchmarks in the mem collection, run::
 
- * sched/messaging
- * sched/pipe
- * syscall/basic
- * mem/memcpy
- * mem/memset
+  perf bench mem
 
 What is stress-ng and how do we use it?
 =======================================
@@ -271,17 +288,17 @@ exercised:
 
 The following command runs the stressor::
 
-  stress-ng --netdev 1 -t 60 --metrics command.
+  stress-ng --netdev 1 -t 60 --metrics
 
 We can use the perf record command to record the events and information
 associated with a process. This command records the profiling data in the
 perf.data file in the same directory.
 
 Using the following commands you can record the events associated with the
-netdev stressor, view the generated report perf.data and annotate the to
-view the statistics of each instruction of the program::
+netdev stressor, view the generated report perf.data and annotate the output
+to view the statistics of each instruction of the program::
 
-  perf record stress-ng --netdev 1 -t 60 --metrics command.
+  perf record -- stress-ng --netdev 1 -t 60 --metrics
   perf report
   perf annotate
 
@@ -349,13 +366,13 @@ times each system call is invoked, and the corresponding Linux subsystem.
 +-------------------+-----------+-----------------+-------------------------+
 | geteuid           | 1         | Process Mgmt.   | sys_geteuid()           |
 +-------------------+-----------+-----------------+-------------------------+
-| getegid           | 1         | Process Mgmt.   | sys_getegid             |
+| getegid           | 1         | Process Mgmt.   | sys_getegid()           |
 +-------------------+-----------+-----------------+-------------------------+
 | close             | 49951     | Filesystem      | sys_close()             |
 +-------------------+-----------+-----------------+-------------------------+
 | pipe              | 604       | Filesystem      | sys_pipe()              |
 +-------------------+-----------+-----------------+-------------------------+
-| openat            | 48560     | Filesystem      | sys_opennat()           |
+| openat            | 48560     | Filesystem      | sys_openat()            |
 +-------------------+-----------+-----------------+-------------------------+
 | fstat             | 8338      | Filesystem      | sys_fstat()             |
 +-------------------+-----------+-----------------+-------------------------+
diff --git a/Documentation/arch/arm/index.rst b/Documentation/arch/arm/index.rst
index afe17db294c4a..b15621093f7a2 100644
--- a/Documentation/arch/arm/index.rst
+++ b/Documentation/arch/arm/index.rst
@@ -75,3 +75,5 @@ SoC-specific documents
    sti/overview
 
    vfp/release-notes
+
+   zte/index
diff --git a/Documentation/arch/arm/zte/index.rst b/Documentation/arch/arm/zte/index.rst
new file mode 100644
index 0000000000000..0ed80b60b7463
--- /dev/null
+++ b/Documentation/arch/arm/zte/index.rst
@@ -0,0 +1,10 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+=======
+ZTE SoC
+=======
+
+.. toctree::
+   :maxdepth: 1
+
+   zx297520v3
diff --git a/Documentation/arch/arm/zte/zx297520v3.rst b/Documentation/arch/arm/zte/zx297520v3.rst
new file mode 100644
index 0000000000000..2122887e391aa
--- /dev/null
+++ b/Documentation/arch/arm/zte/zx297520v3.rst
@@ -0,0 +1,167 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+====================================
+Booting Linux on ZTE zx297520v3 SoCs
+====================================
+
+...............................................................................
+
+Author:	Stefan Dösinger
+
+Date  : 27 Jan 2026
+
+1. Hardware description
+---------------------------
+Zx297520v3 SoCs use a 64 bit capable Cortex-A53 CPU and GICv3, although they
+run in arm32 mode only. The CPU has support EL3, but no hypervisor (EL2) and
+it seems to lack VFP and NEON.
+
+The SoC is used in a number of cheap LTE to WiFi routers, both battery powered
+MiFis and stationary CPEs. In addition to the CPU these devices usually have
+64 MB Ram (although some is shared with the LTE chip), 128 MB NAND flash, an
+SDIO connected RTL8192-type Wifi chip limited to 2.4 ghz operation, USB 2,
+and buttons. Devices with as low as 32 MB or as high as 128 MB ram exist, as
+do devices with 8 or 16 MB of NOR flash.
+
+Some devices, especially the stationary ones, have 100 mbit Ethernet and an
+Ethernet switch.
+
+Usually the devices have LEDs for status indication, although some have SPI or
+I2C connected displays
+
+Some have an SD card slot. If it exists, it is a better choice for the root
+file system because it easily outperforms the built-in NAND.
+
+The LTE interface runs on a separate DSP called ZSP880. It is probably derived
+from LSI ZSPs and has an undocumented instruction set. The ZSP communicates
+with the main CPU via SRAM and DRAM and a mailbox hardware that can generate
+IRQs on either ends.
+
+There is also a Cortex M0 CPU, which is responsible for early HW initialization
+and starting the Cortex A53 CPU. It does not have any essential purpose once
+U-Boot is started. A SRAM-Based handover protocol exists to run custom code on
+this CPU.
+
+2. Booting via USB
+---------------------------
+
+The Boot ROM has support for booting custom code via USB. This mode can be
+entered by connecting a Boot PIN to GND or by modifying the third byte on NAND
+(set it to anything other than 0x5A aka 'Z'). A free software tool to start
+custom U-Boot and kernels can be found here:
+
+https://github.com/zx297520v3-mainline/zx297520v3-loader
+
+If USB download mode is entered but no boot commands are sent through USB, the
+device will proceed to boot normally after a few seconds. It is therefore
+possible to enable USB boot permanently and still leave the default boot files
+in place.
+
+https://github.com/zx297520v3-mainline/u-boot-mainline
+
+Contains an U-Boot version that can be used with the USB loader and sets up the
+CPU and interrupt controller to comply with Linux's booting requirements.
+
+3. Building for built-in U-Boot
+---------------------------
+The devices come with an ancient U-Boot that loads legacy uImages from NAND and
+boots them without a chance for the user to interrupt. The images are stored in
+files ap_cpuap.bin and ap_recovery.bin on a jffs2 partition named imagefs,
+usually mtd4. A file named "fotaflag" switches between the two modes.
+
+In addition to the uImage header, those files have a 384 byte signature header,
+which is used for authenticating the images on some devices. Most devices have
+this authentication disabled and it is enough to pad the uImage files with 384
+zero bytes.
+
+Builtin U-Boot also poorly sets up the CPU. Read the next section for details
+on this. It has no support for loading DTBs, so CONFIG_ARM_APPENDED_DTB is
+needed.
+
+So to build an image that boots from NAND the following steps are necessary:
+
+1) Patch the assembly code from section 3 into arch/arm/kernel/head.S.
+2) make zx29_defconfig
+3) make [-j x]
+4) cat arch/arm/boot/zImage arch/arm/boot/dts/zte/[device].dtb > kernel+dtb
+5) mkimage -A arm -O linux -T kernel -C none -a 0x20008000 -d kernel+dtb uimg
+6) dd if=/dev/zero bs=1 count=384 of=ap_recovery.bin
+7) cat uimg >> ap_recovery.bin
+8) Place this file onto imagefs on the device. Delete ap_cpuap.bin if the
+free space is not enough.
+9) Create the file fotaflag: echo -n FOTA-RECOVERY > fotaflag
+
+For development, booting ap_recovery.bin is recommended because the normal boot
+mode arms the watchdog before starting the kernel.
+
+4. CPU and GIC Setup
+---------------------------
+
+Generally CPU and GICv3 need to be set up according to the requirements spelled
+out in Documentation/arch/arm64/booting.rst. For zx297520v3 this means:
+
+1. GICD_CTLR.DS=1 to disable GIC security
+2. Enable access to ICC_SRE
+3. Disable trapping IRQs into monitor mode
+4. Configure EL2 and below to run in insecure mode.
+5. Configure timer PPIs to active-low.
+
+The kernel sources provided by ZTE do not boot either (interrupts do not work
+at all). They are incomplete in other aspects too, so it is assumed that there
+is some workaround similar to the one described in this document somewhere in
+the binary blobs.
+
+The assembly code below is given as an example of how to achieve this:
+
+```
+#include <linux/irqchip/arm-gic-v3.h>
+#include <asm/assembler.h>
+#include <asm/cp15.h>
+
+@ Detect sane bootloaders and skip the hack
+ldr	r3, =0xf2000000
+ldr	r3, [r3]
+ldr	r4, =(GICD_CTLR_ARE_NS | GICD_CTLR_DS)
+cmp	r3, r4
+beq	skip_zx_hack
+@ This allows EL1 to handle ints hat are normally handled by EL2/3.
+ldr	r3, =0xf2000000
+str     r4, [r3]
+
+cps     #MON_MODE
+
+@ Work in non-secure physical address space: SCR_EL3.NS = 1. At least the UART
+@ seems to respond only to non-secure addresses. I have taken insipiration from
+@ Raspberry pi's armstub7.S here.
+mov	r3, #0x131			@ non-secure, Make F, A bits in CPSR writeable
+					@ Allow hypervisor call.
+mcr     p15, 0, r3, c1, c1, 0
+
+@ AP_PPI_MODE_REG: Configure timer PPIs (10, 11, 13, 14) to active-low.
+ldr	r3, =0xF22020a8
+ldr	r4, =0x50
+str	r4, [r3]
+ldr	r3, =0xF22020ac
+ldr	r4, =0x14
+str	r4, [r3]
+
+@ Enable EL2 access to ICC_SRE (bit 3, ICC_SRE_EL3.Enable). Enable system reg
+@ access to GICv3 registers (bit 0, ICC_SRE_EL3.SRE) for EL1 and EL3.
+mrc	p15, 6, r3, c12, c12, 5         @ ICC_SRE_EL3
+orr	r3, #0x9                        @ FIXME: No defines for SRE_EL3 values?
+mcr	p15, 6, r3, c12, c12, 5
+mrc	p15, 0, r3, c12, c12, 5         @ ICC_SRE_EL1
+orr	r3, #(ICC_SRE_EL1_SRE)
+mcr	p15, 0, r3, c12, c12, 5
+
+@ Like ICC_SRE_EL3, enable EL1 access to ICC_SRE and system register access
+@ for EL2.
+mrc	p15, 4, r3, c12, c9, 5          @ ICC_SRE_EL2 aka ICC_HSRE
+orr	r3, r3, #(ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE)
+mcr	p15, 4, r3, c12, c9, 5
+isb
+
+@ Back to SVC mode
+cps	#SVC_MODE
+skip_zx_hack:
+```
diff --git a/Documentation/arch/arm64/elf_hwcaps.rst b/Documentation/arch/arm64/elf_hwcaps.rst
index 97315ae6c0dae..07ff9ea1d6055 100644
--- a/Documentation/arch/arm64/elf_hwcaps.rst
+++ b/Documentation/arch/arm64/elf_hwcaps.rst
@@ -451,6 +451,30 @@ HWCAP3_LS64
     of CPU. User should only use ld64b/st64b on supported target (device)
     memory location, otherwise fallback to the non-atomic alternatives.
 
+HWCAP3_SVE_B16MM
+    Functionality implied by ID_AA64ZFR0_EL1.B16B16 == 0b0011
+
+HWCAP3_SVE2P3
+    Functionality implied by ID_AA64ZFR0_EL1.SVEver == 0b0100
+
+HWCAP3_SME_LUT6
+    Functionality implied by ID_AA64SMFR0_EL1.LUT6 == 0b1
+
+HWCAP3_SME2P3
+    Functionality implied by ID_AA64SMFR0_EL1.SMEver == 0b0100
+
+HWCAP3_F16MM
+    Functionality implied by ID_AA64FPFR0_EL1.F16MM2 == 0b1
+
+HWCAP3_F16F32DOT
+    Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0010
+
+HWCAP3_F16F32MM
+    Functionality implied by ID_AA64ISAR0_EL1.FHM == 0b0011
+
+HWCAP3_SVE_LUT6
+    Functionality implied by ID_AA64ISAR2_EL1.LUT == 0b0010 and
+    ID_AA64PFR0_EL1.SVE == 0b0001.
 
 4. Unused AT_HWCAP bits
 -----------------------
diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst
index 211119ce7adc7..046a7fa470633 100644
--- a/Documentation/arch/arm64/silicon-errata.rst
+++ b/Documentation/arch/arm64/silicon-errata.rst
@@ -116,7 +116,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A73      | #858921         | ARM64_ERRATUM_858921        |
 +----------------+-----------------+-----------------+-----------------------------+
-| ARM            | Cortex-A76      | #1188873,1418040| ARM64_ERRATUM_1418040       |
+| ARM            | Cortex-A76      | #1188873,       | ARM64_ERRATUM_1418040       |
+|                |                 | #1418040        |                             |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A76      | #1165522        | ARM64_ERRATUM_1165522       |
 +----------------+-----------------+-----------------+-----------------------------+
@@ -136,7 +137,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A78      | #3324344        | ARM64_ERRATUM_3194386       |
 +----------------+-----------------+-----------------+-----------------------------+
-| ARM            | Cortex-A78C     | #3324346,3324347| ARM64_ERRATUM_3194386       |
+| ARM            | Cortex-A78C     | #3324346,       | ARM64_ERRATUM_3194386       |
+|                |                 | #3324347        |                             |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A710     | #2119858        | ARM64_ERRATUM_2119858       |
 +----------------+-----------------+-----------------+-----------------------------+
@@ -172,11 +174,11 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-X925     | #3324334        | ARM64_ERRATUM_3194386       |
 +----------------+-----------------+-----------------+-----------------------------+
-| ARM            | Neoverse-N1     | #1188873,1418040| ARM64_ERRATUM_1418040       |
+| ARM            | Neoverse-N1     | #1188873,       | ARM64_ERRATUM_1418040       |
+|                |                 | #1418040        |                             |
 +----------------+-----------------+-----------------+-----------------------------+
-| ARM            | Neoverse-N1     | #1349291        | N/A                         |
-+----------------+-----------------+-----------------+-----------------------------+
-| ARM            | Neoverse-N1     | #1490853        | N/A                         |
+| ARM            | Neoverse-N1     | #1349291,       | N/A                         |
+|                |                 | #1490853        |                             |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N1     | #1542419        | ARM64_ERRATUM_1542419       |
 +----------------+-----------------+-----------------+-----------------------------+
@@ -204,10 +206,13 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | C1-Pro          | #4193714        | ARM64_ERRATUM_4193714       |
 +----------------+-----------------+-----------------+-----------------------------+
-| ARM            | MMU-500         | #841119,826419  | ARM_SMMU_MMU_500_CPRE_ERRATA|
-|                |                 | #562869,1047329 |                             |
+| ARM            | MMU-500         | #562869,        | ARM_SMMU_MMU_500_CPRE_ERRATA|
+|                |                 | #841119,        |                             |
+|                |                 | #826419,        |                             |
+|                |                 | #1047329        |                             |
 +----------------+-----------------+-----------------+-----------------------------+
-| ARM            | MMU-600         | #1076982,1209401| N/A                         |
+| ARM            | MMU-600         | #1076982,       | N/A                         |
+|                |                 | #1209401        |                             |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | MMU-700         | #2133013,       | N/A                         |
 |                |                 | #2268618,       |                             |
@@ -230,11 +235,13 @@ stable kernels.
 | Broadcom       | Brahma-B53      | N/A             | ARM64_ERRATUM_843419        |
 +----------------+-----------------+-----------------+-----------------------------+
 +----------------+-----------------+-----------------+-----------------------------+
-| Cavium         | ThunderX ITS    | #22375,24313    | CAVIUM_ERRATUM_22375        |
+| Cavium         | ThunderX ITS    | #22375,         | CAVIUM_ERRATUM_22375        |
+|                |                 | #24313          |                             |
 +----------------+-----------------+-----------------+-----------------------------+
 | Cavium         | ThunderX ITS    | #23144          | CAVIUM_ERRATUM_23144        |
 +----------------+-----------------+-----------------+-----------------------------+
-| Cavium         | ThunderX GICv3  | #23154,38545    | CAVIUM_ERRATUM_23154        |
+| Cavium         | ThunderX GICv3  | #23154,         | CAVIUM_ERRATUM_23154        |
+|                |                 | #38545          |                             |
 +----------------+-----------------+-----------------+-----------------------------+
 | Cavium         | ThunderX GICv3  | #38539          | N/A                         |
 +----------------+-----------------+-----------------+-----------------------------+
@@ -244,9 +251,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | Cavium         | ThunderX SMMUv2 | #27704          | N/A                         |
 +----------------+-----------------+-----------------+-----------------------------+
-| Cavium         | ThunderX2 SMMUv3| #74             | N/A                         |
-+----------------+-----------------+-----------------+-----------------------------+
-| Cavium         | ThunderX2 SMMUv3| #126            | N/A                         |
+| Cavium         | ThunderX2 SMMUv3| #74,            | N/A                         |
+|                |                 | #126            |                             |
 +----------------+-----------------+-----------------+-----------------------------+
 | Cavium         | ThunderX2 Core  | #219            | CAVIUM_TX2_ERRATUM_219      |
 +----------------+-----------------+-----------------+-----------------------------+
@@ -258,11 +264,9 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | NVIDIA         | T241 GICv3/4.x  | T241-FABRIC-4   | N/A                         |
 +----------------+-----------------+-----------------+-----------------------------+
-| NVIDIA         | T241 MPAM       | T241-MPAM-1     | N/A                         |
-+----------------+-----------------+-----------------+-----------------------------+
-| NVIDIA         | T241 MPAM       | T241-MPAM-4     | N/A                         |
-+----------------+-----------------+-----------------+-----------------------------+
-| NVIDIA         | T241 MPAM       | T241-MPAM-6     | N/A                         |
+| NVIDIA         | T241 MPAM       | T241-MPAM-1,    | N/A                         |
+|                |                 | T241-MPAM-4,    |                             |
+|                |                 | T241-MPAM-6     |                             |
 +----------------+-----------------+-----------------+-----------------------------+
 +----------------+-----------------+-----------------+-----------------------------+
 | Freescale/NXP  | LS2080A/LS1043A | A-008585        | FSL_ERRATUM_A008585         |
@@ -270,9 +274,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | Hisilicon      | Hip0{5,6,7}     | #161010101      | HISILICON_ERRATUM_161010101 |
 +----------------+-----------------+-----------------+-----------------------------+
-| Hisilicon      | Hip0{6,7}       | #161010701      | N/A                         |
-+----------------+-----------------+-----------------+-----------------------------+
-| Hisilicon      | Hip0{6,7}       | #161010803      | N/A                         |
+| Hisilicon      | Hip0{6,7}       | #161010701,     | N/A                         |
+|                |                 | #161010803      |                             |
 +----------------+-----------------+-----------------+-----------------------------+
 | Hisilicon      | Hip07           | #161600802      | HISILICON_ERRATUM_161600802 |
 +----------------+-----------------+-----------------+-----------------------------+
diff --git a/Documentation/arch/m68k/kernel-options.rst b/Documentation/arch/m68k/kernel-options.rst
index 2008a20b43295..c59e7aaed44ec 100644
--- a/Documentation/arch/m68k/kernel-options.rst
+++ b/Documentation/arch/m68k/kernel-options.rst
@@ -244,23 +244,7 @@ drive (with "root=").
 3) General Device Options (Amiga and Atari)
 ===========================================
 
-3.1) ether=
------------
-
-:Syntax: ether=[<irq>[,<base_addr>[,<mem_start>[,<mem_end>]]]],<dev-name>
-
-<dev-name> is the name of a net driver, as specified in
-drivers/net/Space.c in the Linux source. Most prominent are eth0, ...
-eth3, sl0, ... sl3, ppp0, ..., ppp3, dummy, and lo.
-
-The non-ethernet drivers (sl, ppp, dummy, lo) obviously ignore the
-settings by this options. Also, the existing ethernet drivers for
-Linux/m68k (ariadne, a2065, hydra) don't use them because Zorro boards
-are really Plug-'n-Play, so the "ether=" option is useless altogether
-for Linux/m68k.
-
-
-3.2) hd=
+3.1) hd=
 --------
 
 :Syntax: hd=<cylinders>,<heads>,<sectors>
@@ -273,7 +257,7 @@ itself. It exists just for the case that this fails for one of your
 disks.
 
 
-3.3) max_scsi_luns=
+3.2) max_scsi_luns=
 -------------------
 
 :Syntax: max_scsi_luns=<n>
@@ -284,7 +268,7 @@ be scanned. Valid values for <n> are between 1 and 8. Default is 8 if
 configuration, else 1.
 
 
-3.4) st=
+3.3) st=
 --------
 
 :Syntax: st=<buffer_size>,[<write_thres>,[<max_buffers>]]
@@ -297,7 +281,7 @@ total number of buffers. <max_buffer> limits the total number of
 buffers allocated for all tape devices.
 
 
-3.5) dmasound=
+3.4) dmasound=
 --------------
 
 :Syntax: dmasound=[<buffers>,<buffer-size>[,<catch-radius>]]
diff --git a/Documentation/arch/s390/s390dbf.rst b/Documentation/arch/s390/s390dbf.rst
index aad6d88974fe5..034374c88dbaf 100644
--- a/Documentation/arch/s390/s390dbf.rst
+++ b/Documentation/arch/s390/s390dbf.rst
@@ -106,6 +106,36 @@ the ``debug_stoppable`` sysctl. If you set ``debug_stoppable`` to 0 the debug
 feature cannot be stopped. If the debug feature is already stopped, it
 will stay deactivated.
 
+Kernel parameters
+-----------------
+The size and log levels of debug logs can be configured early during boot using
+the ``s390dbf`` kernel parameter. The parameter accepts a debug log name, a log
+level, and a log size, separated by colon characters (``:``). To configure only
+a single attribute, either the log level or the log size may be omitted.
+
+To configure multiple debug logs, the parameter may be specified multiple times,
+or multiple parameter sets may be provided in a single instance, separated by
+commas.
+
+Parameter format::
+
+	s390dbf=<name|pattern>:[<level>|-]:[<pages>][,...]
+
+where:
+
+- ``name`` specifies either an exact debug log name or a shell-style wildcard
+  pattern
+- ``level`` specifies the log level, or ``-`` to completely deactivate the log
+- ``pages`` specifies the debug area size in pages
+
+Example::
+
+	s390dbf=cio*:6:128,sclp_err::2
+
+This example sets the log level to 6 and the log size to 128 pages for all debug
+logs whose names start with ``cio``. It also sets the log level of the
+``sclp_err`` debug log to 2.
+
 Kernel Interfaces:
 ------------------
 
diff --git a/Documentation/arch/x86/tdx.rst b/Documentation/arch/x86/tdx.rst
index ff6b110291bc6..1a3b5bac10216 100644
--- a/Documentation/arch/x86/tdx.rst
+++ b/Documentation/arch/x86/tdx.rst
@@ -138,13 +138,6 @@ If the platform has such erratum, the kernel prints additional message in
 machine check handler to tell user the machine check may be caused by
 kernel bug on TDX private memory.
 
-Kexec
-~~~~~~~
-
-Currently kexec doesn't work on the TDX platforms with the aforementioned
-erratum.  It fails when loading the kexec kernel image.  Otherwise it
-works normally.
-
 Interaction vs S3 and deeper states
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/Documentation/block/biovecs.rst b/Documentation/block/biovecs.rst
index 11126ed6f40f7..b9dc0c9dbee44 100644
--- a/Documentation/block/biovecs.rst
+++ b/Documentation/block/biovecs.rst
@@ -135,6 +135,7 @@ Usage of helpers:
 	bio_first_bvec_all()
 	bio_first_page_all()
 	bio_first_folio_all()
+	bio_last_bvec_all()
 
 * The following helpers iterate over single-page segment. The passed 'struct
   bio_vec' will contain a single-page IO vector during the iteration::
diff --git a/Documentation/block/data-integrity.rst b/Documentation/block/data-integrity.rst
index 99905e880a0e5..b7b10c8abbcc2 100644
--- a/Documentation/block/data-integrity.rst
+++ b/Documentation/block/data-integrity.rst
@@ -154,7 +154,7 @@ bio_free() will automatically free the bip.
 ----------------
 
 Block devices can set up the integrity information in the integrity
-sub-struture of the queue_limits structure.
+sub-structure of the queue_limits structure.
 
 Layered block devices will need to pick a profile that's appropriate
 for all subdevices.  queue_limits_stack_integrity() can help with that.  DM
diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst
index 3b60583f5db23..3f05f17990ade 100644
--- a/Documentation/bpf/btf.rst
+++ b/Documentation/bpf/btf.rst
@@ -97,10 +97,8 @@ Each type contains the following common data::
     struct btf_type {
         __u32 name_off;
         /* "info" bits arrangement
-         * bits  0-15: vlen (e.g. # of struct's members)
-         * bits 16-23: unused
-         * bits 24-28: kind (e.g. int, ptr, array...etc)
-         * bits 29-30: unused
+         * bits  0-23: vlen (e.g. # of struct's members)
+         * bits 24-30: kind (e.g. int, ptr, array...etc)
          * bit     31: kind_flag, currently used by
          *             struct, union, enum, fwd, enum64,
          *             decl_tag and type_tag
diff --git a/Documentation/core-api/cpu_hotplug.rst b/Documentation/core-api/cpu_hotplug.rst
index 9b4afca9fd09c..6de26d1c6a9ad 100644
--- a/Documentation/core-api/cpu_hotplug.rst
+++ b/Documentation/core-api/cpu_hotplug.rst
@@ -45,11 +45,6 @@ Command Line Switches
 
   This option is limited to the X86 and S390 architecture.
 
-``cpu0_hotplug``
-  Allow to shutdown CPU0.
-
-  This option is limited to the X86 architecture.
-
 CPU maps
 ========
 
diff --git a/Documentation/core-api/housekeeping.rst b/Documentation/core-api/housekeeping.rst
index 92c6e53cea755..ccb0a88b9cb36 100644
--- a/Documentation/core-api/housekeeping.rst
+++ b/Documentation/core-api/housekeeping.rst
@@ -99,7 +99,7 @@ the same RCU read side critical section.
 A typical layout example would look like this on the update side
 (``housekeeping_update()``)::
 
-	rcu_assign_pointer(housekeeping_cpumasks[type], trial);
+	rcu_assign_pointer(housekeeping.cpumasks[type], trial);
 	synchronize_rcu();
 	flush_workqueue(example_workqueue);
 
diff --git a/Documentation/core-api/kref.rst b/Documentation/core-api/kref.rst
index 8db9ff03d952e..233d52d2393db 100644
--- a/Documentation/core-api/kref.rst
+++ b/Documentation/core-api/kref.rst
@@ -40,7 +40,7 @@ kref_init as so::
 
      struct my_data *data;
 
-     data = kmalloc(sizeof(*data), GFP_KERNEL);
+     data = kmalloc_obj(*data);
      if (!data)
             return -ENOMEM;
      kref_init(&data->refcount);
@@ -100,7 +100,7 @@ thread to process::
 	int rv = 0;
 	struct my_data *data;
 	struct task_struct *task;
-	data = kmalloc(sizeof(*data), GFP_KERNEL);
+	data = kmalloc_obj(*data);
 	if (!data)
 		return -ENOMEM;
 	kref_init(&data->refcount);
diff --git a/Documentation/core-api/list.rst b/Documentation/core-api/list.rst
index 241464ca05498..479aa91cc3958 100644
--- a/Documentation/core-api/list.rst
+++ b/Documentation/core-api/list.rst
@@ -112,7 +112,7 @@ list:
 
           /* State 1 */
 
-          grock = kzalloc(sizeof(*grock), GFP_KERNEL);
+          grock = kzalloc_obj(*grock);
           if (!grock)
                   return -ENOMEM;
           grock->name = "Grock";
@@ -123,7 +123,7 @@ list:
 
           /* State 2 */
 
-          dimitri = kzalloc(sizeof(*dimitri), GFP_KERNEL);
+          dimitri = kzalloc_obj(*dimitri);
           if (!dimitri)
                   return -ENOMEM;
           dimitri->name = "Dimitri";
@@ -752,7 +752,7 @@ This is because list_splice() did not reinitialize the list_head it took
 entries from, leaving its pointer pointing into what is now a different list.
 
 If we want to avoid this situation, list_splice_init() can be used. It does the
-same thing as list_splice(), except reinitalizes the donor list_head after the
+same thing as list_splice(), except reinitializes the donor list_head after the
 transplant.
 
 Concurrency considerations
diff --git a/Documentation/crypto/api-samples.rst b/Documentation/crypto/api-samples.rst
index e923f17bc2bd5..388bb7d7a460b 100644
--- a/Documentation/crypto/api-samples.rst
+++ b/Documentation/crypto/api-samples.rst
@@ -159,7 +159,7 @@ Code Example For Random Number Generator Usage
     static int get_random_numbers(u8 *buf, unsigned int len)
     {
         struct crypto_rng *rng = NULL;
-        char *drbg = "drbg_nopr_sha256"; /* Hash DRBG with SHA-256, no PR */
+        char *drbg = "stdrng";
         int ret;
 
         if (!buf || !len) {
diff --git a/Documentation/crypto/async-tx-api.rst b/Documentation/crypto/async-tx-api.rst
index f88a7809385e2..49fcfc66314ac 100644
--- a/Documentation/crypto/async-tx-api.rst
+++ b/Documentation/crypto/async-tx-api.rst
@@ -82,9 +82,9 @@ xor_val   xor a series of source buffers and set a flag if the
 pq	  generate the p+q (raid6 syndrome) from a series of source buffers
 pq_val    validate that a p and or q buffer are in sync with a given series of
 	  sources
-datap	  (raid6_datap_recov) recover a raid6 data block and the p block
+datap	  (raid6_recov_datap) recover a raid6 data block and the p block
 	  from the given sources
-2data	  (raid6_2data_recov) recover 2 raid6 data blocks from the given
+2data	  (raid6_recov_2data) recover 2 raid6 data blocks from the given
 	  sources
 ========  ====================================================================
 
diff --git a/Documentation/crypto/krb5.rst b/Documentation/crypto/krb5.rst
index beffa0133446d..f62e07ac68114 100644
--- a/Documentation/crypto/krb5.rst
+++ b/Documentation/crypto/krb5.rst
@@ -158,13 +158,22 @@ returned.
 When a message has been received, the location and size of the data with the
 message can be determined by calling::
 
-	void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
-					   enum krb5_crypto_mode mode,
-					   size_t *_offset, size_t *_len);
+	int crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
+					  enum krb5_crypto_mode mode,
+					  size_t *_offset, size_t *_len);
 
 The caller provides the offset and length of the message to the function, which
 then alters those values to indicate the region containing the data (plus any
-padding).  It is up to the caller to determine how much padding there is.
+padding).  It is up to the caller to determine how much padding there is.  The
+function returns an error if the length is too small or if the mode is
+unsupported.  An additional function::
+
+	int crypto_krb5_check_data_len(const struct krb5_enctype *krb5,
+				       enum krb5_crypto_mode mode,
+				       size_t len, size_t min_content);
+
+is provided to just do a basic check that the decrypted/verified message would
+have a sufficient minimum payload.
 
 Preparation Functions
 ---------------------
diff --git a/Documentation/crypto/userspace-if.rst b/Documentation/crypto/userspace-if.rst
index 021759198fe77..ea1b1b3f4049f 100644
--- a/Documentation/crypto/userspace-if.rst
+++ b/Documentation/crypto/userspace-if.rst
@@ -4,26 +4,68 @@ User Space Interface
 Introduction
 ------------
 
-The concepts of the kernel crypto API visible to kernel space is fully
-applicable to the user space interface as well. Therefore, the kernel
-crypto API high level discussion for the in-kernel use cases applies
-here as well.
-
-The major difference, however, is that user space can only act as a
-consumer and never as a provider of a transformation or cipher
-algorithm.
-
-The following covers the user space interface exported by the kernel
-crypto API. A working example of this description is libkcapi that can
-be obtained from [1]. That library can be used by user space
-applications that require cryptographic services from the kernel.
-
-Some details of the in-kernel kernel crypto API aspects do not apply to
-user space, however. This includes the difference between synchronous
-and asynchronous invocations. The user space API call is fully
-synchronous.
-
-[1] https://www.chronox.de/libkcapi/index.html
+AF_ALG provides unprivileged userspace programs access to arbitrary hash,
+symmetric cipher, AEAD, and RNG algorithms that are implemented in kernel-mode
+code.
+
+AF_ALG is insecure and is deprecated. Originally added to the kernel in 2010,
+most kernel developers now consider it to be a mistake.
+
+AF_ALG continues to be supported only for backwards compatibility. On systems
+where no programs using AF_ALG remain, the support for it should be disabled by
+disabling ``CONFIG_CRYPTO_USER_API_*``.
+
+Deprecation
+-----------
+
+AF_ALG was originally intended to provide userspace programs access to crypto
+accelerators that they wouldn't otherwise have access to.
+
+However, that capability turned out to not be useful on very many systems. More
+significantly, the actual implementation exposes a vastly greater amount of
+functionality than that. It actually provides access to all software algorithms.
+
+This includes arbitrary compositions of different algorithms created via a
+complex template system, as well as algorithms that only make sense as internal
+implementation details of other algorithms. It also includes full zero-copy
+support, which is difficult for the kernel to implement securely.
+
+Ultimately, these algorithms are just math computations. They use the same
+instructions that userspace programs already have access to, just accessed in a
+much more convoluted and less efficient way.
+
+Indeed, userspace code is nearly always what is being used anyway. These same
+algorithms are widely implemented in userspace crypto libraries.
+
+Meanwhile, AF_ALG hasn't been withstanding modern vulnerability discovery tools
+such as syzbot and large language models. It receives a steady stream of CVEs.
+Some of the examples include:
+
+- CVE-2026-31677
+- CVE-2026-31431 (https://copy.fail)
+- CVE-2025-38079
+- CVE-2025-37808
+- CVE-2024-26824
+- CVE-2022-48781
+- CVE-2019-8912
+- CVE-2018-14619
+- CVE-2017-18075
+- CVE-2017-17806
+- CVE-2017-17805
+- CVE-2016-10147
+- CVE-2015-8970
+- CVE-2015-3331
+- CVE-2014-9644
+- CVE-2013-7421
+- CVE-2011-4081
+
+It is recommended that, whenever possible, userspace programs be migrated to
+userspace crypto code (which again, is what is normally used anyway) and
+``CONFIG_CRYPTO_USER_API_*`` be disabled.  On systems that use SELinux, SELinux
+can also be used to restrict the use of AF_ALG to trusted programs.
+
+The remainder of this documentation provides the historical documentation for
+the deprecated AF_ALG interface.
 
 User Space API General Remarks
 ------------------------------
@@ -297,7 +339,7 @@ follows:
     struct sockaddr_alg sa = {
         .salg_family = AF_ALG,
         .salg_type = "rng", /* this selects the random number generator */
-        .salg_name = "drbg_nopr_sha256" /* this is the RNG name */
+        .salg_name = "stdrng" /* this is the RNG name */
     };
 
 
@@ -327,33 +369,10 @@ CRYPTO_USER_API_RNG_CAVP option:
 Zero-Copy Interface
 -------------------
 
-In addition to the send/write/read/recv system call family, the AF_ALG
-interface can be accessed with the zero-copy interface of
-splice/vmsplice. As the name indicates, the kernel tries to avoid a copy
-operation into kernel space.
-
-The zero-copy operation requires data to be aligned at the page
-boundary. Non-aligned data can be used as well, but may require more
-operations of the kernel which would defeat the speed gains obtained
-from the zero-copy interface.
-
-The system-inherent limit for the size of one zero-copy operation is 16
-pages. If more data is to be sent to AF_ALG, user space must slice the
-input into segments with a maximum size of 16 pages.
-
-Zero-copy can be used with the following code example (a complete
-working example is provided with libkcapi):
-
-::
-
-    int pipes[2];
-
-    pipe(pipes);
-    /* input data in iov */
-    vmsplice(pipes[1], iov, iovlen, SPLICE_F_GIFT);
-    /* opfd is the file descriptor returned from accept() system call */
-    splice(pipes[0], NULL, opfd, NULL, ret, 0);
-    read(opfd, out, outlen);
+AF_ALG used to have zero-copy support, but it was removed due to it being a
+frequent source of vulnerabilities.  For backwards compatibility the splice()
+and sendfile() system calls are still supported, but the kernel will make an
+internal copy of the data before passing it to the crypto code.
 
 
 Setsockopt Interface
diff --git a/Documentation/dev-tools/checkpatch.rst b/Documentation/dev-tools/checkpatch.rst
index dccede68698ca..6139a08c34cd8 100644
--- a/Documentation/dev-tools/checkpatch.rst
+++ b/Documentation/dev-tools/checkpatch.rst
@@ -184,6 +184,13 @@ Available options:
    Override checking of perl version.  Runtime errors may be encountered after
    enabling this flag if the perl version does not meet the minimum specified.
 
+ - --spdx-cxx-comments
+
+   Don't force C comments ``/* */`` for SPDX license (required by old
+   toolchains), allow also C++ comments ``//``.
+
+   NOTE: it should *not* be used for Linux mainline.
+
  - --codespell
 
    Use the codespell dictionary for checking spelling errors.
@@ -210,6 +217,13 @@ Available options:
 
    Display the help text.
 
+Configuration file
+==================
+
+Default configuration options can be stored in ``.checkpatch.conf``, search
+path: ``.:$HOME:.scripts`` or in a directory specified by ``$CHECKPATCH_CONFIG_DIR``
+environment variable (falling back to the default search path).
+
 Message Levels
 ==============
 
diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst
index 54d9ee28de982..8e71e1e75b5bc 100644
--- a/Documentation/dev-tools/context-analysis.rst
+++ b/Documentation/dev-tools/context-analysis.rst
@@ -17,7 +17,7 @@ features. To enable for Clang, configure the kernel with::
 
     CONFIG_WARN_CONTEXT_ANALYSIS=y
 
-The feature requires Clang 22 or later.
+The feature requires Clang 23 or later.
 
 The analysis is *opt-in by default*, and requires declaring which modules and
 subsystems should be analyzed in the respective `Makefile`::
diff --git a/Documentation/dev-tools/kcov.rst b/Documentation/dev-tools/kcov.rst
index 8127849d40f59..54a0baeb917a2 100644
--- a/Documentation/dev-tools/kcov.rst
+++ b/Documentation/dev-tools/kcov.rst
@@ -246,6 +246,8 @@ KCOV supports collecting remote coverage from the following contexts:
 2. Local kernel background tasks. These are spawned when a userspace process
    interacts with some kernel interface and are usually killed when the process
    exits (e.g. vhost workers).
+   This can be combined with another KCOV instance that is configured for normal
+   coverage collection.
 
 3. Soft interrupts.
 
@@ -262,6 +264,9 @@ gets saved to the ``kcov_handle`` field in the current ``task_struct`` and
 needs to be passed to the newly spawned local tasks via custom kernel code
 modifications. Those tasks should in turn use the passed handle in their
 ``kcov_remote_start`` and ``kcov_remote_stop`` annotations.
+In the kernel, common handles are wrapped in a ``kcov_common_handle_id``, which
+consumes no space in builds without ``CONFIG_KCOV``; subsystems that integrate
+with this mechanism should not need to use any ``#ifdef CONFIG_KCOV`` or such.
 
 KCOV follows a predefined format for both global and common handles. Each
 handle is a ``u64`` integer. Currently, only the one top and the lower 4 bytes
diff --git a/Documentation/dev-tools/kselftest.rst b/Documentation/dev-tools/kselftest.rst
index 18c2da67fae42..d7bfe320338ce 100644
--- a/Documentation/dev-tools/kselftest.rst
+++ b/Documentation/dev-tools/kselftest.rst
@@ -126,6 +126,18 @@ dedicated skiplist::
 See the top-level tools/testing/selftests/Makefile for the list of all
 possible targets.
 
+Requiring all targets to build successfully
+===========================================
+
+By default, the build succeeds as long as at least one target builds
+without error. Set ``FORCE_TARGETS=1`` to instead require every target to
+build successfully; make will abort as soon as any target fails::
+
+  $ make -C tools/testing/selftests FORCE_TARGETS=1
+
+This applies to both the ``all`` and ``install`` targets and is useful in
+CI environments where a silent partial build would be misleading.
+
 Running the full range hotplug selftests
 ========================================
 
diff --git a/Documentation/dev-tools/kunit/usage.rst b/Documentation/dev-tools/kunit/usage.rst
index ebd06f5ea4550..1c78dfff94e8a 100644
--- a/Documentation/dev-tools/kunit/usage.rst
+++ b/Documentation/dev-tools/kunit/usage.rst
@@ -157,6 +157,50 @@ Alternatively, one can take full control over the error message by using
 	if (some_setup_function())
 		KUNIT_FAIL(test, "Failed to setup thing for testing");
 
+Suppressing warning backtraces
+------------------------------
+
+Some unit tests trigger warning backtraces either intentionally or as a side
+effect. Such backtraces are normally undesirable since they distract from
+the actual test and may result in the impression that there is a problem.
+
+Backtraces can be suppressed with **task-scoped suppression**: while
+suppression is active on the current task, the backtrace and stack dump from
+``WARN*()``, ``WARN_ON*()``, and related macros on that task are suppressed.
+Two API forms are available.
+
+- Scoped suppression is the simplest form. Wrap the code that triggers
+  warnings in a ``kunit_warning_suppress()`` block:
+
+.. code-block:: c
+
+	static void some_test(struct kunit *test)
+	{
+		kunit_warning_suppress(test) {
+			trigger_backtrace();
+			KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+		}
+	}
+
+.. note::
+   The warning count must be checked inside the block; the suppression handle
+   is not accessible after the block exits.
+
+- Direct functions return an explicit handle pointer. Use them when the handle
+  needs to be retained or passed across helper functions:
+
+.. code-block:: c
+
+	static void some_test(struct kunit *test)
+	{
+		struct kunit_suppressed_warning *w;
+
+		w = kunit_start_suppress_warning(test);
+		trigger_backtrace();
+		kunit_end_suppress_warning(test, w);
+
+		KUNIT_EXPECT_EQ(test, kunit_suppressed_warning_count(w), 1);
+	}
 
 Test Suites
 ~~~~~~~~~~~
@@ -1211,4 +1255,4 @@ For example:
 		dev_managed_string = devm_kstrdup(fake_device, "Hello, World!");
 
 		// Everything is cleaned up automatically when the test ends.
-	}
\ No newline at end of file
+	}
diff --git a/Documentation/devicetree/bindings/arm/altera.yaml b/Documentation/devicetree/bindings/arm/altera.yaml
index 206686f3eebce..0b418d9d60dbd 100644
--- a/Documentation/devicetree/bindings/arm/altera.yaml
+++ b/Documentation/devicetree/bindings/arm/altera.yaml
@@ -111,10 +111,17 @@ properties:
           - enum:
               - intel,socfpga-agilex5-socdk
               - intel,socfpga-agilex5-socdk-013b
+              - intel,socfpga-agilex5-socdk-debug
               - intel,socfpga-agilex5-socdk-modular
               - intel,socfpga-agilex5-socdk-nand
           - const: intel,socfpga-agilex5
 
+      - description: Agilex7m boards
+        items:
+          - enum:
+              - altr,socfpga-agilex7m-socdk
+          - const: altr,socfpga-agilex7m
+
       - description: SoCFPGA VT
         items:
           - const: altr,socfpga-vt
diff --git a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
index 8ec7a3e74a21d..b553e3415ce93 100644
--- a/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
+++ b/Documentation/devicetree/bindings/arm/aspeed/aspeed.yaml
@@ -95,6 +95,7 @@ properties:
               - facebook,greatlakes-bmc
               - facebook,harma-bmc
               - facebook,minerva-cmc
+              - facebook,sanmiguel-bmc
               - facebook,santabarbara-bmc
               - facebook,yosemite4-bmc
               - facebook,yosemite5-bmc
diff --git a/Documentation/devicetree/bindings/arm/cpus.yaml b/Documentation/devicetree/bindings/arm/cpus.yaml
index 5f5ff5e51e519..5be89c5840262 100644
--- a/Documentation/devicetree/bindings/arm/cpus.yaml
+++ b/Documentation/devicetree/bindings/arm/cpus.yaml
@@ -223,6 +223,7 @@ properties:
           - qcom,oryon-1-2
           - qcom,oryon-1-3
           - qcom,oryon-1-4
+          - qcom,oryon-1-5
           - qcom,oryon-2-1
           - qcom,oryon-2-2
           - qcom,oryon-2-3
diff --git a/Documentation/devicetree/bindings/arm/fsl.yaml b/Documentation/devicetree/bindings/arm/fsl.yaml
index 0023cd1268075..4d167cf392830 100644
--- a/Documentation/devicetree/bindings/arm/fsl.yaml
+++ b/Documentation/devicetree/bindings/arm/fsl.yaml
@@ -688,7 +688,7 @@ properties:
           - const: phytec,imx6ul-pcl063   # PHYTEC phyCORE-i.MX 6UL
           - const: fsl,imx6ul
 
-      - description: i.MX6UL Variscite VAR-SOM-MX6 Boards
+      - description: i.MX6UL Variscite VAR-SOM-6UL Boards
         items:
           - const: variscite,mx6ulconcerto
           - const: variscite,var-som-imx6ul
@@ -797,6 +797,12 @@ properties:
           - const: phytec,imx6ull-pcl063        # PHYTEC phyCORE-i.MX 6ULL
           - const: fsl,imx6ull
 
+      - description: i.MX6ULL Variscite VAR-SOM-6UL Boards
+        items:
+          - const: variscite,mx6ullconcerto  # Variscite VAR-SOM-6UL dev kit board
+          - const: variscite,var-som-imx6ull # Variscite VAR-SOM-6UL SoM (6ULL variant)
+          - const: fsl,imx6ull
+
       - description: i.MX6ULL Boards with Toradex Colibri iMX6ULL Modules
         items:
           - enum:
@@ -1025,6 +1031,7 @@ properties:
               - toradex,verdin-imx8mm-nonwifi-ivy    # Verdin iMX8M Mini Module on Ivy
               - toradex,verdin-imx8mm-nonwifi-mallow # Verdin iMX8M Mini Module on Mallow
               - toradex,verdin-imx8mm-nonwifi-yavia  # Verdin iMX8M Mini Module on Yavia
+              - toradex,verdin-imx8mm-nonwifi-zinnia # Verdin iMX8M Mini Module on Zinnia
           - const: toradex,verdin-imx8mm-nonwifi     # Verdin iMX8M Mini Module without Wi-Fi / BT
           - const: toradex,verdin-imx8mm             # Verdin iMX8M Mini Module
           - const: fsl,imx8mm
@@ -1037,6 +1044,7 @@ properties:
               - toradex,verdin-imx8mm-wifi-ivy    # Verdin iMX8M Mini Wi-Fi / BT Module on Ivy
               - toradex,verdin-imx8mm-wifi-mallow # Verdin iMX8M Mini Wi-Fi / BT Module on Mallow
               - toradex,verdin-imx8mm-wifi-yavia  # Verdin iMX8M Mini Wi-Fi / BT Module on Yavia
+              - toradex,verdin-imx8mm-wifi-zinnia # Verdin iMX8M Mini Wi-Fi / BT Module on Zinnia
           - const: toradex,verdin-imx8mm-wifi     # Verdin iMX8M Mini Wi-Fi / BT Module
           - const: toradex,verdin-imx8mm          # Verdin iMX8M Mini Module
           - const: fsl,imx8mm
@@ -1271,6 +1279,7 @@ properties:
               - toradex,verdin-imx8mp-nonwifi-ivy    # Verdin iMX8M Plus Module on Ivy
               - toradex,verdin-imx8mp-nonwifi-mallow # Verdin iMX8M Plus Module on Mallow
               - toradex,verdin-imx8mp-nonwifi-yavia  # Verdin iMX8M Plus Module on Yavia
+              - toradex,verdin-imx8mp-nonwifi-zinnia # Verdin iMX8M Plus Module on Zinnia
           - const: toradex,verdin-imx8mp-nonwifi     # Verdin iMX8M Plus Module without Wi-Fi / BT
           - const: toradex,verdin-imx8mp             # Verdin iMX8M Plus Module
           - const: fsl,imx8mp
@@ -1283,6 +1292,7 @@ properties:
               - toradex,verdin-imx8mp-wifi-ivy    # Verdin iMX8M Plus Wi-Fi / BT Module on Ivy
               - toradex,verdin-imx8mp-wifi-mallow # Verdin iMX8M Plus Wi-Fi / BT Module on Mallow
               - toradex,verdin-imx8mp-wifi-yavia  # Verdin iMX8M Plus Wi-Fi / BT Module on Yavia
+              - toradex,verdin-imx8mp-wifi-zinnia # Verdin iMX8M Plus Wi-Fi / BT Module on Zinnia
           - const: toradex,verdin-imx8mp-wifi     # Verdin iMX8M Plus Wi-Fi / BT Module
           - const: toradex,verdin-imx8mp          # Verdin iMX8M Plus Module
           - const: fsl,imx8mp
@@ -1458,6 +1468,7 @@ properties:
       - description: i.MX91 based Boards
         items:
           - enum:
+              - fsl,imx91-9x9-qsb         # i.MX91 9x9 QSB Board
               - fsl,imx91-11x11-evk       # i.MX91 11x11 EVK Board
               - fsl,imx91-11x11-frdm      # FRDM i.MX91 Development Board
               - fsl,imx91-11x11-frdm-s    # FRDM i.MX91S Development Board
@@ -1482,6 +1493,7 @@ properties:
       - description: i.MX95 based Boards
         items:
           - enum:
+              - fsl,imx95-15x15-ab2       # i.MX95 15x15 Audio Board V2
               - fsl,imx95-15x15-evk       # i.MX95 15x15 EVK Board
               - fsl,imx95-15x15-frdm      # i.MX95 15x15 FRDM Board
               - fsl,imx95-19x19-evk       # i.MX95 19x19 EVK Board
@@ -1515,6 +1527,7 @@ properties:
               - toradex,verdin-imx95-nonwifi-ivy    # Verdin iMX95 Module on Ivy
               - toradex,verdin-imx95-nonwifi-mallow # Verdin iMX95 Module on Mallow
               - toradex,verdin-imx95-nonwifi-yavia  # Verdin iMX95 Module on Yavia
+              - toradex,verdin-imx95-nonwifi-zinnia # Verdin iMX95 Module on Zinnia
           - const: toradex,verdin-imx95-nonwifi     # Verdin iMX95 Module without Wi-Fi / BT
           - const: toradex,verdin-imx95             # Verdin iMX95 Module
           - const: fsl,imx95
@@ -1527,6 +1540,7 @@ properties:
               - toradex,verdin-imx95-wifi-ivy     # Verdin iMX95 Wi-Fi / BT Module on Ivy
               - toradex,verdin-imx95-wifi-mallow  # Verdin iMX95 Wi-Fi / BT Module on Mallow
               - toradex,verdin-imx95-wifi-yavia   # Verdin iMX95 Wi-Fi / BT Module on Yavia
+              - toradex,verdin-imx95-wifi-zinnia  # Verdin iMX95 Wi-Fi / BT Module on Zinnia
           - const: toradex,verdin-imx95-wifi      # Verdin iMX95 Wi-Fi / BT Module
           - const: toradex,verdin-imx95           # Verdin iMX95 Module
           - const: fsl,imx95
@@ -1614,6 +1628,18 @@ properties:
           - const: variscite,var-dart-mx91 # Variscite DART-MX91 SOM
           - const: fsl,imx91
 
+      - description: Variscite VAR-SOM-MX91 based boards
+        items:
+          - const: variscite,var-som-mx91-symphony # Variscite VAR-SOM-MX91 on Symphony
+          - const: variscite,var-som-mx91 # Variscite VAR-SOM-MX91
+          - const: fsl,imx91
+
+      - description: Variscite DART-MX93 based boards
+        items:
+          - const: variscite,var-dart-mx93-sonata # Variscite DART-MX93 on Sonata Development Board
+          - const: variscite,var-dart-mx93 # Variscite DART-MX93 SOM
+          - const: fsl,imx93
+
       - description: Variscite VAR-SOM-MX93 based boards
         items:
           - const: variscite,var-som-mx93-symphony
diff --git a/Documentation/devicetree/bindings/arm/qcom.yaml b/Documentation/devicetree/bindings/arm/qcom.yaml
index b4943123d2e42..50cc18a6ec5ed 100644
--- a/Documentation/devicetree/bindings/arm/qcom.yaml
+++ b/Documentation/devicetree/bindings/arm/qcom.yaml
@@ -79,6 +79,7 @@ properties:
       - items:
           - enum:
               - fairphone,fp6
+              - nothing,asteroids
           - const: qcom,milos
 
       - items:
@@ -365,6 +366,11 @@ properties:
               - qcom,ipq9574-ap-al02-c9
           - const: qcom,ipq9574
 
+      - items:
+          - enum:
+              - qcom,ipq9650-rdp488
+          - const: qcom,ipq9650
+
       - items:
           - enum:
               - qcom,kaanapali-mtp
@@ -822,6 +828,14 @@ properties:
           - const: google,zombie-sku514
           - const: qcom,sc7280
 
+      - description: Xiaomi Poco F1
+        items:
+          - enum:
+              - xiaomi,beryllium-ebbg
+              - xiaomi,beryllium-tianma
+          - const: xiaomi,beryllium
+          - const: qcom,sdm845
+
       - items:
           - enum:
               - lenovo,flex-5g
@@ -972,8 +986,6 @@ properties:
               - sony,akatsuki-row
               - sony,apollo-row
               - thundercomm,db845c
-              - xiaomi,beryllium
-              - xiaomi,beryllium-ebbg
               - xiaomi,polaris
           - const: qcom,sdm845
 
@@ -1040,6 +1052,7 @@ properties:
 
       - items:
           - enum:
+              - motorola,dubai
               - nothing,spacewar
           - const: qcom,sm7325
 
@@ -1168,6 +1181,10 @@ properties:
           - const: microsoft,denali
           - const: qcom,x1e80100
 
+      - items:
+          - const: microsoft,surface-pro-12in
+          - const: qcom,x1p42100
+
       - items:
           - enum:
               - qcom,purwa-iot-evk
diff --git a/Documentation/devicetree/bindings/arm/st,nomadik.yaml b/Documentation/devicetree/bindings/arm/st,nomadik.yaml
new file mode 100644
index 0000000000000..d2167908ff600
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/st,nomadik.yaml
@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/st,nomadik.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ST Nomadik SoC based Boards
+
+maintainers:
+  - Linus Walleij <linusw@kernel.org>
+
+description:
+  Boards using the Nomadik SoC platform
+
+properties:
+  $nodename:
+    const: '/'
+  compatible:
+    enum:
+      - st,nomadik-nhk-15
+      - calaosystems,usb-s8815
+
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/arm/ste-nomadik.txt b/Documentation/devicetree/bindings/arm/ste-nomadik.txt
deleted file mode 100644
index 2fdff5a806cfb..0000000000000
--- a/Documentation/devicetree/bindings/arm/ste-nomadik.txt
+++ /dev/null
@@ -1,38 +0,0 @@
-ST-Ericsson Nomadik Device Tree Bindings
-
-For various board the "board" node may contain specific properties
-that pertain to this particular board, such as board-specific GPIOs.
-
-Required root node property: src
-- Nomadik System and reset controller used for basic chip control, clock
-  and reset line control.
-- compatible: must be "stericsson,nomadik,src"
-
-Boards with the Nomadik SoC include:
-
-Nomadik NHK-15 board manufactured by ST Microelectronics:
-
-Required root node property:
-
-compatible="st,nomadik-nhk-15";
-
-S8815 "MiniKit" manufactured by Calao Systems:
-
-Required root node property:
-
-compatible="calaosystems,usb-s8815";
-
-Required node: usb-s8815
-
-Example:
-
-usb-s8815 {
-		ethernet-gpio {
-			gpios = <&gpio3 19 0x1>;
-			interrupts = <19 0x1>;
-			interrupt-parent = <&gpio3>;
-		};
-		mmcsd-gpio {
-			gpios = <&gpio3 16 0x1>;
-		};
-};
diff --git a/Documentation/devicetree/bindings/arm/ti/k3.yaml b/Documentation/devicetree/bindings/arm/ti/k3.yaml
index 2a6a9441c23de..69b5441cbf1a9 100644
--- a/Documentation/devicetree/bindings/arm/ti/k3.yaml
+++ b/Documentation/devicetree/bindings/arm/ti/k3.yaml
@@ -97,6 +97,13 @@ properties:
           - const: toradex,verdin-am62          # Verdin AM62 Module
           - const: ti,am625
 
+      - description: K3 AM625 SoC on TQ-Systems TQMa62xx SoM
+        items:
+          - enum:
+              - tq,am625-tqma6254-mba62xx # MBa62xx base board
+          - const: tq,am625-tqma6254
+          - const: ti,am625
+
       - description: K3 AM62P5 SoC Toradex Verdin Modules and Carrier Boards
         items:
           - enum:
diff --git a/Documentation/devicetree/bindings/arm/zte.yaml b/Documentation/devicetree/bindings/arm/zte.yaml
new file mode 100644
index 0000000000000..f028d2cec7ab2
--- /dev/null
+++ b/Documentation/devicetree/bindings/arm/zte.yaml
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/arm/zte.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ZTE zx platforms
+
+maintainers:
+  - Stefan Dösinger <stefandoesinger@gmail.com>
+
+description: |
+  ARM platforms using SoCs designed by ZTE. Currently this supports devices
+  based on the zx297520v3 SoC which is found in LTE routers.
+
+properties:
+  $nodename:
+    const: "/"
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - dlink,dwr932m
+          - const: zte,zx297520v3
+
+additionalProperties: true
diff --git a/Documentation/devicetree/bindings/bus/fsl,spba-bus.yaml b/Documentation/devicetree/bindings/bus/fsl,spba-bus.yaml
index 00bbde203f598..4808065fc9115 100644
--- a/Documentation/devicetree/bindings/bus/fsl,spba-bus.yaml
+++ b/Documentation/devicetree/bindings/bus/fsl,spba-bus.yaml
@@ -26,8 +26,10 @@ select:
     compatible:
       contains:
         enum:
+          - fsl,aipi-bus
           - fsl,aips
           - fsl,emi
+          - fsl,emi-bus
           - fsl,spba-bus
   required:
     - compatible
@@ -39,8 +41,10 @@ properties:
   compatible:
     items:
       - enum:
+          - fsl,aipi-bus
           - fsl,aips
           - fsl,emi
+          - fsl,emi-bus
           - fsl,spba-bus
       - const: simple-bus
 
diff --git a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
index 995d578157810..1fc5411fe948a 100644
--- a/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
+++ b/Documentation/devicetree/bindings/cache/qcom,llcc.yaml
@@ -21,6 +21,7 @@ properties:
   compatible:
     enum:
       - qcom,glymur-llcc
+      - qcom,hawi-llcc
       - qcom,ipq5424-llcc
       - qcom,kaanapali-llcc
       - qcom,qcs615-llcc
@@ -35,6 +36,7 @@ properties:
       - qcom,sc8280xp-llcc
       - qcom,sdm670-llcc
       - qcom,sdm845-llcc
+      - qcom,shikra-llcc
       - qcom,sm6350-llcc
       - qcom,sm7150-llcc
       - qcom,sm8150-llcc
@@ -57,6 +59,11 @@ properties:
   interrupts:
     maxItems: 1
 
+  memory-region:
+    maxItems: 1
+    description: handle to a reserved-memory node used for firmware-populated
+      SLC/SCT shared memory.
+
   nvmem-cells:
     items:
       - description: Reference to an nvmem node for multi channel DDR
@@ -206,6 +213,7 @@ allOf:
             enum:
               - qcom,sc7280-llcc
               - qcom,sdm670-llcc
+              - qcom,shikra-llcc
     then:
       properties:
         reg:
@@ -318,6 +326,7 @@ allOf:
           contains:
             enum:
               - qcom,kaanapali-llcc
+              - qcom,hawi-llcc
               - qcom,sm8450-llcc
               - qcom,sm8550-llcc
               - qcom,sm8650-llcc
@@ -340,6 +349,18 @@ allOf:
             - const: llcc3_base
             - const: llcc_broadcast_base
             - const: llcc_broadcast_and_base
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,hawi-llcc
+    then:
+      required:
+        - memory-region
+    else:
+      properties:
+        memory-region: false
 
 additionalProperties: false
 
diff --git a/Documentation/devicetree/bindings/clock/marvell,pxa1908.yaml b/Documentation/devicetree/bindings/clock/marvell,pxa1908.yaml
index 6f3a8578fe2a6..0db5504013d5e 100644
--- a/Documentation/devicetree/bindings/clock/marvell,pxa1908.yaml
+++ b/Documentation/devicetree/bindings/clock/marvell,pxa1908.yaml
@@ -37,6 +37,9 @@ properties:
   '#power-domain-cells':
     const: 1
 
+  '#reset-cells':
+    const: 1
+
 required:
   - compatible
   - reg
@@ -44,16 +47,27 @@ required:
 
 additionalProperties: false
 
-if:
-  not:
-    properties:
-      compatible:
-        contains:
-          const: marvell,pxa1908-apmu
-
-then:
-  properties:
-    '#power-domain-cells': false
+allOf:
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              const: marvell,pxa1908-apmu
+    then:
+      properties:
+        '#power-domain-cells': false
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              enum:
+                - marvell,pxa1908-apbc
+                - marvell,pxa1908-apbcp
+    then:
+      properties:
+        '#reset-cells': false
 
 examples:
   # APMU block:
diff --git a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml
index ef2b1e2044309..a177a1934b19f 100644
--- a/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,dispcc-sm6125.yaml
@@ -42,12 +42,6 @@ properties:
       - const: cfg_ahb_clk
       - const: gcc_disp_gpll0_div_clk_src
 
-  '#clock-cells':
-    const: 1
-
-  '#power-domain-cells':
-    const: 1
-
   power-domains:
     description:
       A phandle and PM domain specifier for the CX power domain.
@@ -58,18 +52,16 @@ properties:
       A phandle to an OPP node describing the power domain's performance point.
     maxItems: 1
 
-  reg:
-    maxItems: 1
-
 required:
   - compatible
-  - reg
   - clocks
   - clock-names
-  - '#clock-cells'
   - '#power-domain-cells'
 
-additionalProperties: false
+allOf:
+  - $ref: qcom,gcc.yaml#
+
+unevaluatedProperties: false
 
 examples:
   - |
@@ -101,6 +93,7 @@ examples:
       power-domains = <&rpmpd SM6125_VDDCX>;
 
       #clock-cells = <1>;
+      #reset-cells = <1>;
       #power-domain-cells = <1>;
     };
 ...
diff --git a/Documentation/devicetree/bindings/clock/qcom,hawi-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,hawi-gcc.yaml
new file mode 100644
index 0000000000000..4f428c0f7286e
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,hawi-gcc.yaml
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,hawi-gcc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Global Clock & Reset Controller on Hawi
+
+maintainers:
+  - Vivek Aknurwar <vivek.aknurwar@oss.qualcomm.com>
+
+description: |
+  Qualcomm global clock control module provides the clocks, resets and power
+  domains on Hawi.
+
+  See also: include/dt-bindings/clock/qcom,hawi-gcc.h
+
+properties:
+  compatible:
+    const: qcom,hawi-gcc
+
+  clocks:
+    items:
+      - description: Board XO source
+      - description: Board Always On XO source
+      - description: Sleep clock source
+      - description: PCIE 0 Pipe clock source
+      - description: PCIE 1 Pipe clock source
+      - description: UFS PHY RX symbol 0 clock
+      - description: UFS PHY RX symbol 1 clock
+      - description: UFS PHY TX symbol 0 clock
+      - description: USB3 PHY wrapper pipe clock
+
+required:
+  - compatible
+  - clocks
+  - '#power-domain-cells'
+
+allOf:
+  - $ref: qcom,gcc.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,rpmh.h>
+    clock-controller@100000 {
+      compatible = "qcom,hawi-gcc";
+      reg = <0x00100000 0x1f4200>;
+      clocks = <&rpmhcc RPMH_CXO_CLK>,
+               <&rpmhcc RPMH_CXO_CLK_A>,
+               <&sleep_clk>,
+               <&pcie0_phy>,
+               <&pcie1_phy>,
+               <&ufs_mem_phy 0>,
+               <&ufs_mem_phy 1>,
+               <&ufs_mem_phy 2>,
+               <&usb_1_qmpphy>;
+      #clock-cells = <1>;
+      #reset-cells = <1>;
+      #power-domain-cells = <1>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml
index de338c05190fb..b9c3650e5c4cb 100644
--- a/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,ipq9574-cmn-pll.yaml
@@ -25,6 +25,7 @@ properties:
   compatible:
     enum:
       - qcom,ipq5018-cmn-pll
+      - qcom,ipq5332-cmn-pll
       - qcom,ipq5424-cmn-pll
       - qcom,ipq6018-cmn-pll
       - qcom,ipq8074-cmn-pll
diff --git a/Documentation/devicetree/bindings/clock/qcom,ipq9650-gcc.yaml b/Documentation/devicetree/bindings/clock/qcom,ipq9650-gcc.yaml
new file mode 100644
index 0000000000000..f33105217a062
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,ipq9650-gcc.yaml
@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,ipq9650-gcc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Global Clock & Reset Controller on IPQ9650
+
+maintainers:
+  - Bjorn Andersson <andersson@kernel.org>
+  - Kathiravan Thirumoorthy <kathiravan.thirumoorthy@oss.qualcomm.com>
+
+description: |
+  Qualcomm global clock control module provides the clocks, resets and power
+  domains on IPQ9650
+
+  See also:
+    include/dt-bindings/clock/qcom,ipq9650-gcc.h
+    include/dt-bindings/reset/qcom,ipq9650-gcc.h
+
+properties:
+  compatible:
+    const: qcom,ipq9650-gcc
+
+  clocks:
+    items:
+      - description: Board XO source
+      - description: Sleep clock source
+      - description: PCIE30 PHY0 pipe clock source
+      - description: PCIE30 PHY1 pipe clock source
+      - description: PCIE30 PHY2 pipe clock source
+      - description: PCIE30 PHY3 pipe clock source
+      - description: PCIE30 PHY4 pipe clock source
+      - description: USB PCIE wrapper pipe clock source
+      - description: NSS common clock source
+
+  '#power-domain-cells': false
+
+  '#interconnect-cells':
+    const: 1
+
+required:
+  - compatible
+  - clocks
+
+allOf:
+  - $ref: qcom,gcc.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    clock-controller@1800000 {
+      compatible = "qcom,ipq9650-gcc";
+      reg = <0x01800000 0x40000>;
+      clocks = <&xo_board_clk>,
+               <&sleep_clk>,
+               <&pcie30_phy0_pipe_clk>,
+               <&pcie30_phy1_pipe_clk>,
+               <&pcie30_phy2_pipe_clk>,
+               <&pcie30_phy3_pipe_clk>,
+               <&pcie30_phy4_pipe_clk>,
+               <&usb3phy_0_cc_pipe_clk>,
+               <&nss_cmn_clk>;
+      #clock-cells = <1>;
+      #reset-cells = <1>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml b/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml
index 466c884aa2bab..e868963f659b6 100644
--- a/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,kaanapali-gxclkctl.yaml
@@ -44,7 +44,7 @@ required:
   - power-domains
   - '#power-domain-cells'
 
-unevaluatedProperties: false
+additionalProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/clock/qcom,milos-gxclkctl.yaml b/Documentation/devicetree/bindings/clock/qcom,milos-gxclkctl.yaml
new file mode 100644
index 0000000000000..fbcb5d3f3e3d1
--- /dev/null
+++ b/Documentation/devicetree/bindings/clock/qcom,milos-gxclkctl.yaml
@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/clock/qcom,milos-gxclkctl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Graphics Power Domain Controller on Milos
+
+maintainers:
+  - Luca Weiss <luca.weiss@fairphone.com>
+
+description: |
+  Qualcomm GX(graphics) is a clock controller which has PLLs, clocks and
+  Power domains (GDSC). This module provides the power domains control
+  of gxclkctl on Qualcomm SoCs which helps the recovery of Graphics subsystem.
+
+  See also:
+    include/dt-bindings/clock/qcom,kaanapali-gxclkctl.h
+
+properties:
+  compatible:
+    enum:
+      - qcom,milos-gxclkctl
+
+  reg:
+    maxItems: 1
+
+  power-domains:
+    description:
+      Power domains required for the clock controller to operate
+    items:
+      - description: GFX power domain
+      - description: GPUCC(CX) power domain
+
+  '#power-domain-cells':
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - power-domains
+  - '#power-domain-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/power/qcom,rpmhpd.h>
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        clock-controller@3d64000 {
+            compatible = "qcom,milos-gxclkctl";
+            reg = <0x0 0x03d64000 0x0 0x6000>;
+            power-domains = <&rpmhpd RPMHPD_GFX>,
+                            <&gpucc 0>;
+            #power-domain-cells = <1>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml
index a2c404a579812..d344b33860429 100644
--- a/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,rpmhcc.yaml
@@ -19,6 +19,7 @@ properties:
     enum:
       - qcom,eliza-rpmh-clk
       - qcom,glymur-rpmh-clk
+      - qcom,hawi-rpmh-clk
       - qcom,kaanapali-rpmh-clk
       - qcom,milos-rpmh-clk
       - qcom,nord-rpmh-clk
diff --git a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml
index 1ccdf4b0f5dd3..08824f8489735 100644
--- a/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml
+++ b/Documentation/devicetree/bindings/clock/qcom,sm8550-tcsr.yaml
@@ -17,6 +17,7 @@ description: |
   See also:
   - include/dt-bindings/clock/qcom,eliza-tcsr.h
   - include/dt-bindings/clock/qcom,glymur-tcsr.h
+  - include/dt-bindings/clock/qcom,hawi-tcsrcc.h
   - include/dt-bindings/clock/qcom,nord-tcsrcc.h
   - include/dt-bindings/clock/qcom,sm8550-tcsr.h
   - include/dt-bindings/clock/qcom,sm8650-tcsr.h
@@ -28,6 +29,7 @@ properties:
       - enum:
           - qcom,eliza-tcsr
           - qcom,glymur-tcsr
+          - qcom,hawi-tcsrcc
           - qcom,kaanapali-tcsr
           - qcom,milos-tcsr
           - qcom,nord-tcsrcc
diff --git a/Documentation/devicetree/bindings/clock/renesas,cpg-clocks.yaml b/Documentation/devicetree/bindings/clock/renesas,cpg-clocks.yaml
index a0e09b7002f07..703b5bf26717c 100644
--- a/Documentation/devicetree/bindings/clock/renesas,cpg-clocks.yaml
+++ b/Documentation/devicetree/bindings/clock/renesas,cpg-clocks.yaml
@@ -41,7 +41,7 @@ properties:
 
   clock-output-names:
     minItems: 3
-    maxItems: 17
+    maxItems: 19
 
   renesas,mode:
     description: Board-specific settings of the MD_CK* bits on R-Mobile A1
@@ -90,6 +90,8 @@ allOf:
             - const: zx
             - const: zs
             - const: hp
+            - const: ztr
+            - const: zt
 
   - if:
       properties:
@@ -123,6 +125,8 @@ allOf:
             - const: zb
             - const: m3
             - const: cp
+            - const: ztr
+            - const: zt
 
       required:
         - renesas,mode
@@ -240,6 +244,6 @@ examples:
             #clock-cells = <1>;
             clock-output-names = "system", "pllc0", "pllc1", "pllc2", "r",
                                  "usb24s", "i", "zg", "b", "m1", "hp", "hpp",
-                                 "usbp", "s", "zb", "m3", "cp";
+                                 "usbp", "s", "zb", "m3", "cp", "ztr", "zt";
             renesas,mode = <0x05>;
     };
diff --git a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
index 876bf90ed96ef..ccb6b8dd8e116 100644
--- a/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml
@@ -30,6 +30,16 @@ properties:
     maxItems: 1
 
   clocks:
+    minItems: 1
+    maxItems: 2
+
+  clock-names:
+    minItems: 1
+    items:
+      - const: core
+      - const: iface
+
+  power-domains:
     maxItems: 1
 
   operating-points-v2: true
@@ -44,6 +54,25 @@ required:
 
 additionalProperties: false
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,eliza-inline-crypto-engine
+              - qcom,milos-inline-crypto-engine
+
+    then:
+      required:
+        - power-domains
+        - clock-names
+      properties:
+        clocks:
+          minItems: 2
+        clock-names:
+          minItems: 2
+
 examples:
   - |
     #include <dt-bindings/clock/qcom,sm8550-gcc.h>
@@ -52,7 +81,11 @@ examples:
       compatible = "qcom,sm8550-inline-crypto-engine",
                    "qcom,inline-crypto-engine";
       reg = <0x01d88000 0x8000>;
-      clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
+      clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>,
+               <&gcc GCC_UFS_PHY_AHB_CLK>;
+      clock-names = "core",
+                    "iface";
+      power-domains = <&gcc UFS_PHY_GDSC>;
 
       operating-points-v2 = <&ice_opp_table>;
 
diff --git a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml
index 41402599e9ab8..d054cc114707b 100644
--- a/Documentation/devicetree/bindings/crypto/qcom,prng.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom,prng.yaml
@@ -17,11 +17,13 @@ properties:
           - qcom,prng-ee  # 8996 and later using EE
       - items:
           - enum:
+              - qcom,glymur-trng
               - qcom,ipq5332-trng
               - qcom,ipq5424-trng
               - qcom,ipq9574-trng
               - qcom,kaanapali-trng
               - qcom,milos-trng
+              - qcom,nord-trng
               - qcom,qcs615-trng
               - qcom,qcs8300-trng
               - qcom,sa8255p-trng
diff --git a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml
index 79d5be2548bc5..08febd66c22ba 100644
--- a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml
+++ b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml
@@ -45,7 +45,11 @@ properties:
 
       - items:
           - enum:
+              - qcom,eliza-qce
+              - qcom,glymur-qce
               - qcom,kaanapali-qce
+              - qcom,milos-qce
+              - qcom,nord-qce
               - qcom,qcs615-qce
               - qcom,qcs8300-qce
               - qcom,sa8775p-qce
diff --git a/Documentation/devicetree/bindings/display/bridge/lontium,lt9211.yaml b/Documentation/devicetree/bindings/display/bridge/lontium,lt9211.yaml
index 9a6e9b25d14a9..7cfe92a8bcd72 100644
--- a/Documentation/devicetree/bindings/display/bridge/lontium,lt9211.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/lontium,lt9211.yaml
@@ -36,18 +36,56 @@ properties:
 
     properties:
       port@0:
-        $ref: /schemas/graph.yaml#/properties/port
+        $ref: /schemas/graph.yaml#/$defs/port-base
+        unevaluatedProperties: false
         description:
           Primary MIPI DSI port-1 for MIPI input or
           LVDS port-1 for LVDS input or DPI input.
 
+        properties:
+          endpoint:
+            $ref: /schemas/media/video-interfaces.yaml#
+            unevaluatedProperties: false
+
+            properties:
+              data-lanes:
+                description: array of physical DSI data lane indexes.
+                minItems: 1
+                items:
+                  - const: 1
+                  - const: 2
+                  - const: 3
+                  - const: 4
+
+            required:
+              - data-lanes
+
       port@1:
-        $ref: /schemas/graph.yaml#/properties/port
+        $ref: /schemas/graph.yaml#/$defs/port-base
+        unevaluatedProperties: false
         description:
           Additional MIPI port-2 for MIPI input or LVDS port-2
           for LVDS input. Used in combination with primary
           port-1 to drive higher resolution displays
 
+        properties:
+          endpoint:
+            $ref: /schemas/media/video-interfaces.yaml#
+            unevaluatedProperties: false
+
+            properties:
+              data-lanes:
+                description: array of physical DSI data lane indexes.
+                minItems: 1
+                items:
+                  - const: 1
+                  - const: 2
+                  - const: 3
+                  - const: 4
+
+            required:
+              - data-lanes
+
       port@2:
         $ref: /schemas/graph.yaml#/properties/port
         description:
@@ -99,6 +137,7 @@ examples:
             reg = <0>;
 
             endpoint {
+              data-lanes = <1 2 3 4>;
               remote-endpoint = <&dsi0_out>;
             };
           };
diff --git a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml
index 7586d681bcc6b..0363201f0e619 100644
--- a/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/lvds-codec.yaml
@@ -34,6 +34,7 @@ properties:
       - items:
           - enum:
               - doestek,dtc34lm85am # For the Doestek DTC34LM85AM Flat Panel Display (FPD) Transmitter
+              - idt,v103      # For the Triple 10-BIT LVDS Transmitter
               - onnn,fin3385  # OnSemi FIN3385
               - ti,ds90c185   # For the TI DS90C185 FPD-Link Serializer
               - ti,ds90c187   # For the TI DS90C187 FPD-Link Serializer
diff --git a/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml b/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml
index e6808419f6254..7636c24906ba0 100644
--- a/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/simple-bridge.yaml
@@ -30,6 +30,7 @@ properties:
           - algoltek,ag6311
           - asl-tek,cs5263
           - dumb-vga-dac
+          - mstar,tsumu88adt3-lf-1
           - parade,ps185hdm
           - radxa,ra620
           - realtek,rtd2171
diff --git a/Documentation/devicetree/bindings/display/bridge/solomon,ssd2825.yaml b/Documentation/devicetree/bindings/display/bridge/solomon,ssd2825.yaml
index e2d293d623b8d..7606454930314 100644
--- a/Documentation/devicetree/bindings/display/bridge/solomon,ssd2825.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/solomon,ssd2825.yaml
@@ -10,6 +10,7 @@ maintainers:
   - Svyatoslav Ryhel <clamor95@gmail.com>
 
 allOf:
+  - $ref: /schemas/display/dsi-controller.yaml#
   - $ref: /schemas/spi/spi-peripheral-props.yaml#
 
 properties:
@@ -86,7 +87,7 @@ required:
   - compatible
   - ports
 
-additionalProperties: false
+unevaluatedProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/display/bridge/waveshare,dsi2dpi.yaml b/Documentation/devicetree/bindings/display/bridge/waveshare,dsi2dpi.yaml
index 3820dd7e11af1..4d34a92192bf0 100644
--- a/Documentation/devicetree/bindings/display/bridge/waveshare,dsi2dpi.yaml
+++ b/Documentation/devicetree/bindings/display/bridge/waveshare,dsi2dpi.yaml
@@ -10,11 +10,14 @@ maintainers:
   - Joseph Guo <qijian.guo@nxp.com>
 
 description:
-  Waveshare bridge board is part of Waveshare panel which converts DSI to DPI.
+  Waveshare bridge board is part of Waveshare panel which converts DSI to DPI
+  or LVDS.
 
 properties:
   compatible:
-    const: waveshare,dsi2dpi
+    enum:
+      - waveshare,dsi2dpi
+      - waveshare,dsi2lvds
 
   reg:
     maxItems: 1
@@ -53,7 +56,7 @@ properties:
       port@1:
         $ref: /schemas/graph.yaml#/properties/port
         description:
-          Video port for MIPI DPI output panel.
+          Video port for MIPI DPI or LVDS output to the panel.
 
     required:
       - port@0
diff --git a/Documentation/devicetree/bindings/display/imx/fsl,imx-parallel-display.yaml b/Documentation/devicetree/bindings/display/imx/fsl,imx-parallel-display.yaml
index bbcfe7e2958b7..b0c5869771fae 100644
--- a/Documentation/devicetree/bindings/display/imx/fsl,imx-parallel-display.yaml
+++ b/Documentation/devicetree/bindings/display/imx/fsl,imx-parallel-display.yaml
@@ -42,6 +42,17 @@ properties:
     unevaluatedProperties: false
     description: output port connected to a panel
 
+  port:
+    $ref: /schemas/graph.yaml#/properties/port
+    unevaluatedProperties: false
+    deprecated: true
+    description: input port connected to the IPU display interface, see port@0
+
+  display-timings:
+    $ref: /schemas/display/panel/display-timings.yaml#
+    unevaluatedProperties: false
+    deprecated: true
+
 required:
   - compatible
 
diff --git a/Documentation/devicetree/bindings/display/imx/fsl,imx8qxp-dc-command-sequencer.yaml b/Documentation/devicetree/bindings/display/imx/fsl,imx8qxp-dc-command-sequencer.yaml
index 27118f4c0d281..fd095e5742c51 100644
--- a/Documentation/devicetree/bindings/display/imx/fsl,imx8qxp-dc-command-sequencer.yaml
+++ b/Documentation/devicetree/bindings/display/imx/fsl,imx8qxp-dc-command-sequencer.yaml
@@ -41,7 +41,7 @@ properties:
       - const: sw3
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
+    maxItems: 1
     description: phandle pointing to the mmio-sram device node
 
 required:
diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,ovl-2l.yaml b/Documentation/devicetree/bindings/display/mediatek/mediatek,ovl-2l.yaml
index bacdfe7d08a61..ac0d924a451b4 100644
--- a/Documentation/devicetree/bindings/display/mediatek/mediatek,ovl-2l.yaml
+++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,ovl-2l.yaml
@@ -45,9 +45,8 @@ properties:
       - description: OVL-2L Clock
 
   iommus:
-    description:
-      This property should point to the respective IOMMU block with master port as argument,
-      see Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml for details.
+    minItems: 1
+    maxItems: 2
 
   mediatek,gce-client-reg:
     description: The register of client driver can be configured by gce with
diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,ovl.yaml b/Documentation/devicetree/bindings/display/mediatek/mediatek,ovl.yaml
index 679f731f0f15f..4df5c7b410c69 100644
--- a/Documentation/devicetree/bindings/display/mediatek/mediatek,ovl.yaml
+++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,ovl.yaml
@@ -66,9 +66,8 @@ properties:
       - description: OVL Clock
 
   iommus:
-    description:
-      This property should point to the respective IOMMU block with master port as argument,
-      see Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml for details.
+    minItems: 1
+    maxItems: 2
 
   mediatek,gce-client-reg:
     description: The register of client driver can be configured by gce with
diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,rdma.yaml b/Documentation/devicetree/bindings/display/mediatek/mediatek,rdma.yaml
index cb187a95c11ea..d914c06640dfe 100644
--- a/Documentation/devicetree/bindings/display/mediatek/mediatek,rdma.yaml
+++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,rdma.yaml
@@ -65,9 +65,7 @@ properties:
       - description: RDMA Clock
 
   iommus:
-    description:
-      This property should point to the respective IOMMU block with master port as argument,
-      see Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml for details.
+    maxItems: 1
 
   mediatek,rdma-fifo-size:
     description:
diff --git a/Documentation/devicetree/bindings/display/mediatek/mediatek,wdma.yaml b/Documentation/devicetree/bindings/display/mediatek/mediatek,wdma.yaml
index 816841a96133d..97d052b0fb614 100644
--- a/Documentation/devicetree/bindings/display/mediatek/mediatek,wdma.yaml
+++ b/Documentation/devicetree/bindings/display/mediatek/mediatek,wdma.yaml
@@ -45,9 +45,7 @@ properties:
       - description: WDMA Clock
 
   iommus:
-    description:
-      This property should point to the respective IOMMU block with master port as argument,
-      see Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml for details.
+    maxItems: 1
 
   mediatek,gce-client-reg:
     description: The register of client driver can be configured by gce with
diff --git a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml
index 8239adb7f7d3e..094a6383bb779 100644
--- a/Documentation/devicetree/bindings/display/msm/dp-controller.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dp-controller.yaml
@@ -219,6 +219,7 @@ allOf:
           - required:
               - "#sound-dai-cells"
       else:
+        $ref: /schemas/sound/dai-common.yaml#
         properties:
           aux-bus: false
         required:
@@ -243,7 +244,7 @@ allOf:
         clocks:
           minItems: 5
           maxItems: 5
-        clocks-names:
+        clock-names:
           minItems: 5
           maxItems: 5
 
@@ -264,7 +265,7 @@ allOf:
         clocks:
           minItems: 5
           maxItems: 6
-        clocks-names:
+        clock-names:
           minItems: 5
           maxItems: 6
 
@@ -277,7 +278,6 @@ allOf:
               - qcom,sc8180x-dp
               - qcom,sdm845-dp
               - qcom,sm8350-dp
-              - qcom,sm8650-dp
     then:
       properties:
         reg:
@@ -286,6 +286,24 @@ allOf:
         clocks:
           minItems: 6
           maxItems: 6
+        clock-names:
+          minItems: 6
+          maxItems: 6
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,sm8650-dp
+    then:
+      properties:
+        reg:
+          minItems: 5
+          maxItems: 9
+        clocks:
+          minItems: 6
+          maxItems: 6
         clocks-names:
           minItems: 6
           maxItems: 6
@@ -306,7 +324,7 @@ allOf:
         clocks:
           minItems: 6
           maxItems: 8
-        clocks-names:
+        clock-names:
           minItems: 6
           maxItems: 8
 
@@ -326,7 +344,7 @@ allOf:
         clocks:
           minItems: 5
           maxItems: 6
-        clocks-names:
+        clock-names:
           minItems: 5
           maxItems: 6
 
diff --git a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml
index a24fcb9144181..dbc0613e427ed 100644
--- a/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml
+++ b/Documentation/devicetree/bindings/display/msm/dsi-controller-main.yaml
@@ -16,6 +16,7 @@ properties:
           - enum:
               - qcom,apq8064-dsi-ctrl
               - qcom,kaanapali-dsi-ctrl
+              - qcom,milos-dsi-ctrl
               - qcom,msm8226-dsi-ctrl
               - qcom,msm8916-dsi-ctrl
               - qcom,msm8953-dsi-ctrl
@@ -339,6 +340,7 @@ allOf:
         compatible:
           contains:
             enum:
+              - qcom,milos-dsi-ctrl
               - qcom,msm8998-dsi-ctrl
               - qcom,sa8775p-dsi-ctrl
               - qcom,sar2130p-dsi-ctrl
diff --git a/Documentation/devicetree/bindings/display/msm/gpu.yaml b/Documentation/devicetree/bindings/display/msm/gpu.yaml
index 04b2328903ca1..358759fad8dc7 100644
--- a/Documentation/devicetree/bindings/display/msm/gpu.yaml
+++ b/Documentation/devicetree/bindings/display/msm/gpu.yaml
@@ -84,13 +84,9 @@ properties:
     maxItems: 64
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle-array
     minItems: 1
     maxItems: 4
-    items:
-      maxItems: 1
-    description: |
-      phandles to one or more reserved on-chip SRAM regions.
+    description:
       phandle to the On Chip Memory (OCMEM) that's present on some a3xx and
       a4xx Snapdragon SoCs. See
       Documentation/devicetree/bindings/sram/qcom,ocmem.yaml
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,eliza-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,eliza-mdss.yaml
index 47938d13d1ca8..bd4ba91a171f6 100644
--- a/Documentation/devicetree/bindings/display/msm/qcom,eliza-mdss.yaml
+++ b/Documentation/devicetree/bindings/display/msm/qcom,eliza-mdss.yaml
@@ -119,7 +119,7 @@ examples:
         mdss_mdp: display-controller@ae01000 {
             compatible = "qcom,eliza-dpu";
             reg = <0x0ae01000 0x93000>,
-                  <0x0aeb0000 0x2008>;
+                  <0x0aeb0000 0x3000>;
             reg-names = "mdp",
                         "vbif";
 
@@ -304,7 +304,7 @@ examples:
         mdss_dsi0_phy: phy@ae95000 {
             compatible = "qcom,eliza-dsi-phy-4nm", "qcom,sm8650-dsi-phy-4nm";
             reg = <0x0ae95000 0x200>,
-                  <0x0ae95200 0x280>,
+                  <0x0ae95200 0x300>,
                   <0x0ae95500 0x400>;
             reg-names = "dsi_phy",
                         "dsi_phy_lane",
@@ -388,7 +388,7 @@ examples:
         mdss_dsi1_phy: phy@ae97000 {
             compatible = "qcom,eliza-dsi-phy-4nm", "qcom,sm8650-dsi-phy-4nm";
             reg = <0x0ae97000 0x200>,
-                  <0x0ae97200 0x280>,
+                  <0x0ae97200 0x300>,
                   <0x0ae97500 0x400>;
             reg-names = "dsi_phy",
                         "dsi_phy_lane",
@@ -407,11 +407,15 @@ examples:
 
         displayport-controller@af54000 {
             compatible = "qcom,eliza-dp", "qcom,sm8650-dp";
-            reg = <0xaf54000 0x104>,
-                  <0xaf54200 0xc0>,
-                  <0xaf55000 0x770>,
-                  <0xaf56000 0x9c>,
-                  <0xaf57000 0x9c>;
+            reg = <0x0af54000 0x200>,
+                  <0x0af54200 0x200>,
+                  <0x0af55000 0xc00>,
+                  <0x0af56000 0x400>,
+                  <0x0af57000 0x400>,
+                  <0x0af58000 0x400>,
+                  <0x0af59000 0x400>,
+                  <0x0af5a000 0x600>,
+                  <0x0af5b000 0x600>;
 
             interrupts-extended = <&mdss 12>;
 
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,milos-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,milos-mdss.yaml
new file mode 100644
index 0000000000000..7010ffa0ae350
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/msm/qcom,milos-mdss.yaml
@@ -0,0 +1,286 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/msm/qcom,milos-mdss.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Milos Display MDSS
+
+maintainers:
+  - Luca Weiss <luca.weiss@fairphone.com>
+
+description:
+  Milos MSM Mobile Display Subsystem(MDSS), which encapsulates sub-blocks like
+  DPU display controller, DSI and DP interfaces etc.
+
+$ref: /schemas/display/msm/mdss-common.yaml#
+
+properties:
+  compatible:
+    const: qcom,milos-mdss
+
+  clocks:
+    items:
+      - description: Display AHB
+      - description: Display hf AXI
+      - description: Display core
+
+  iommus:
+    maxItems: 1
+
+  interconnects:
+    items:
+      - description: Interconnect path from mdp0 port to the data bus
+      - description: Interconnect path from CPU to the reg bus
+
+  interconnect-names:
+    items:
+      - const: mdp0-mem
+      - const: cpu-cfg
+
+patternProperties:
+  "^display-controller@[0-9a-f]+$":
+    type: object
+    additionalProperties: true
+    properties:
+      compatible:
+        const: qcom,milos-dpu
+
+  "^displayport-controller@[0-9a-f]+$":
+    type: object
+    additionalProperties: true
+    properties:
+      compatible:
+        const: qcom,milos-dp
+
+  "^dsi@[0-9a-f]+$":
+    type: object
+    additionalProperties: true
+    properties:
+      compatible:
+        contains:
+          const: qcom,milos-dsi-ctrl
+
+  "^phy@[0-9a-f]+$":
+    type: object
+    additionalProperties: true
+    properties:
+      compatible:
+        const: qcom,milos-dsi-phy-4nm
+
+required:
+  - compatible
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,dsi-phy-28nm.h>
+    #include <dt-bindings/clock/qcom,milos-dispcc.h>
+    #include <dt-bindings/clock/qcom,milos-gcc.h>
+    #include <dt-bindings/clock/qcom,rpmh.h>
+    #include <dt-bindings/interconnect/qcom,icc.h>
+    #include <dt-bindings/interconnect/qcom,milos-rpmh.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/power/qcom,rpmhpd.h>
+
+    display-subsystem@ae00000 {
+        compatible = "qcom,milos-mdss";
+        reg = <0x0ae00000 0x1000>;
+        reg-names = "mdss";
+
+        interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH 0>;
+
+        clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>,
+                 <&gcc GCC_DISP_HF_AXI_CLK>,
+                 <&dispcc DISP_CC_MDSS_MDP_CLK>;
+
+        resets = <&dispcc DISP_CC_MDSS_CORE_BCR>;
+
+        interconnects = <&mmss_noc MASTER_MDP QCOM_ICC_TAG_ALWAYS
+                         &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+                        <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+                         &cnoc_main SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>;
+        interconnect-names = "mdp0-mem",
+                             "cpu-cfg";
+
+        power-domains = <&dispcc DISP_CC_MDSS_CORE_GDSC>;
+
+        iommus = <&apps_smmu 0x1c00 0x2>;
+
+        interrupt-controller;
+        #interrupt-cells = <1>;
+
+        #address-cells = <1>;
+        #size-cells = <1>;
+        ranges;
+
+        display-controller@ae01000 {
+            compatible = "qcom,milos-dpu";
+            reg = <0x0ae01000 0x8f000>,
+                  <0x0aeb0000 0x3000>;
+            reg-names = "mdp",
+                        "vbif";
+
+            interrupts-extended = <&mdss 0>;
+
+            clocks = <&gcc GCC_DISP_HF_AXI_CLK>,
+                     <&dispcc DISP_CC_MDSS_AHB_CLK>,
+                     <&dispcc DISP_CC_MDSS_MDP_LUT_CLK>,
+                     <&dispcc DISP_CC_MDSS_MDP_CLK>,
+                     <&dispcc DISP_CC_MDSS_VSYNC_CLK>;
+            clock-names = "nrt_bus",
+                          "iface",
+                          "lut",
+                          "core",
+                          "vsync";
+
+            assigned-clocks = <&dispcc DISP_CC_MDSS_VSYNC_CLK>;
+            assigned-clock-rates = <19200000>;
+
+            operating-points-v2 = <&mdp_opp_table>;
+
+            power-domains = <&rpmhpd RPMHPD_CX>;
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+
+                    dpu_intf1_out: endpoint {
+                        remote-endpoint = <&mdss_dsi0_in>;
+                    };
+                };
+            };
+
+            mdp_opp_table: opp-table {
+                compatible = "operating-points-v2";
+
+                opp-200000000 {
+                    opp-hz = /bits/ 64 <200000000>;
+                    required-opps = <&rpmhpd_opp_low_svs>;
+                };
+
+                opp-342000000 {
+                    opp-hz = /bits/ 64 <342000000>;
+                    required-opps = <&rpmhpd_opp_svs>;
+                };
+
+                opp-402000000 {
+                    opp-hz = /bits/ 64 <402000000>;
+                    required-opps = <&rpmhpd_opp_svs_l1>;
+                };
+
+                opp-535000000 {
+                    opp-hz = /bits/ 64 <535000000>;
+                    required-opps = <&rpmhpd_opp_nom>;
+                };
+
+                opp-600000000 {
+                    opp-hz = /bits/ 64 <600000000>;
+                    required-opps = <&rpmhpd_opp_nom_l1>;
+                };
+
+                opp-630000000 {
+                    opp-hz = /bits/ 64 <630000000>;
+                    required-opps = <&rpmhpd_opp_turbo>;
+                };
+            };
+        };
+
+        dsi@ae94000 {
+            compatible = "qcom,milos-dsi-ctrl", "qcom,mdss-dsi-ctrl";
+            reg = <0x0ae94000 0x1000>;
+            reg-names = "dsi_ctrl";
+
+            interrupts-extended = <&mdss 4>;
+
+            clocks = <&dispcc DISP_CC_MDSS_BYTE0_CLK>,
+                     <&dispcc DISP_CC_MDSS_BYTE0_INTF_CLK>,
+                     <&dispcc DISP_CC_MDSS_PCLK0_CLK>,
+                     <&dispcc DISP_CC_MDSS_ESC0_CLK>,
+                     <&dispcc DISP_CC_MDSS_AHB_CLK>,
+                     <&gcc GCC_DISP_HF_AXI_CLK>;
+            clock-names = "byte",
+                          "byte_intf",
+                          "pixel",
+                          "core",
+                          "iface",
+                          "bus";
+
+            assigned-clocks = <&dispcc DISP_CC_MDSS_BYTE0_CLK_SRC>,
+                              <&dispcc DISP_CC_MDSS_PCLK0_CLK_SRC>;
+            assigned-clock-parents = <&mdss_dsi0_phy DSI_BYTE_PLL_CLK>,
+                                     <&mdss_dsi0_phy DSI_PIXEL_PLL_CLK>;
+
+            operating-points-v2 = <&mdss_dsi_opp_table>;
+
+            power-domains = <&rpmhpd RPMHPD_CX>;
+
+            phys = <&mdss_dsi0_phy>;
+            phy-names = "dsi";
+
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+
+                    mdss_dsi0_in: endpoint {
+                        remote-endpoint = <&dpu_intf1_out>;
+                    };
+                };
+
+                port@1 {
+                    reg = <1>;
+
+                    mdss_dsi0_out: endpoint {
+                    };
+                };
+            };
+
+            mdss_dsi_opp_table: opp-table {
+                compatible = "operating-points-v2";
+
+                opp-187500000 {
+                    opp-hz = /bits/ 64 <187500000>;
+                    required-opps = <&rpmhpd_opp_low_svs>;
+                };
+
+                opp-300000000 {
+                    opp-hz = /bits/ 64 <300000000>;
+                    required-opps = <&rpmhpd_opp_svs>;
+                };
+
+                opp-358000000 {
+                    opp-hz = /bits/ 64 <358000000>;
+                    required-opps = <&rpmhpd_opp_svs_l1>;
+                };
+            };
+        };
+
+        mdss_dsi0_phy: phy@ae95000 {
+            compatible = "qcom,milos-dsi-phy-4nm";
+            reg = <0x0ae95000 0x200>,
+                  <0x0ae95200 0x300>,
+                  <0x0ae95500 0x400>;
+            reg-names = "dsi_phy",
+                        "dsi_phy_lane",
+                        "dsi_pll";
+
+            clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>,
+                     <&rpmhcc RPMH_CXO_CLK>;
+            clock-names = "iface",
+                          "ref";
+
+            #clock-cells = <1>;
+            #phy-cells = <0>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml
index e2730a2f25cfb..6c827cf9692b9 100644
--- a/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml
+++ b/Documentation/devicetree/bindings/display/msm/qcom,sa8775p-mdss.yaml
@@ -200,9 +200,11 @@ examples:
                   <0x0aec2000 0x1c8>;
 
             clocks = <&dispcc0 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>,
-                     <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>;
+                     <&dispcc0 MDSS_DISP_CC_MDSS_AHB_CLK>,
+                     <&gcc GCC_EDP_REF_CLKREF_EN>;
             clock-names = "aux",
-                          "cfg_ahb";
+                          "cfg_ahb",
+                          "ref";
 
             #clock-cells = <1>;
             #phy-cells = <0>;
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml
index dccac525d202c..aa0cf0ec5b93f 100644
--- a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml
+++ b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-dpu.yaml
@@ -18,6 +18,7 @@ properties:
           - qcom,eliza-dpu
           - qcom,glymur-dpu
           - qcom,kaanapali-dpu
+          - qcom,milos-dpu
           - qcom,sa8775p-dpu
           - qcom,sm8650-dpu
           - qcom,sm8750-dpu
@@ -70,7 +71,7 @@ examples:
     display-controller@ae01000 {
         compatible = "qcom,sm8650-dpu";
         reg = <0x0ae01000 0x8f000>,
-              <0x0aeb0000 0x2008>;
+              <0x0aeb0000 0x3000>;
         reg-names = "mdp", "vbif";
 
         clocks = <&gcc_axi_clk>,
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-mdss.yaml
index a1c53e1910330..0f7f79527748e 100644
--- a/Documentation/devicetree/bindings/display/msm/qcom,sm8650-mdss.yaml
+++ b/Documentation/devicetree/bindings/display/msm/qcom,sm8650-mdss.yaml
@@ -112,7 +112,7 @@ examples:
         display-controller@ae01000 {
             compatible = "qcom,sm8650-dpu";
             reg = <0x0ae01000 0x8f000>,
-                  <0x0aeb0000 0x2008>;
+                  <0x0aeb0000 0x3000>;
             reg-names = "mdp", "vbif";
 
             clocks = <&gcc_axi_clk>,
diff --git a/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml b/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml
index a38c2261ef1ac..46dc0d28da29a 100644
--- a/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml
+++ b/Documentation/devicetree/bindings/display/msm/qcom,sm8750-mdss.yaml
@@ -117,7 +117,7 @@ examples:
             display-controller@ae01000 {
                 compatible = "qcom,sm8750-dpu";
                 reg = <0x0ae01000 0x93000>,
-                      <0x0aeb0000 0x2008>;
+                      <0x0aeb0000 0x3000>;
                 reg-names = "mdp",
                             "vbif";
 
@@ -389,11 +389,15 @@ examples:
 
             displayport-controller@af54000 {
                 compatible = "qcom,sm8750-dp", "qcom,sm8650-dp";
-                reg = <0xaf54000 0x104>,
-                      <0xaf54200 0xc0>,
-                      <0xaf55000 0x770>,
-                      <0xaf56000 0x9c>,
-                      <0xaf57000 0x9c>;
+                reg = <0x0af54000 0x200>,
+                      <0x0af54200 0x200>,
+                      <0x0af55000 0xc00>,
+                      <0x0af56000 0x400>,
+                      <0x0af57000 0x400>,
+                      <0x0af58000 0x400>,
+                      <0x0af59000 0x400>,
+                      <0x0af5a000 0x600>,
+                      <0x0af5b000 0x600>;
 
                 interrupts-extended = <&mdss 12>;
 
diff --git a/Documentation/devicetree/bindings/display/panel/chipwealth,ch13726a.yaml b/Documentation/devicetree/bindings/display/panel/chipwealth,ch13726a.yaml
new file mode 100644
index 0000000000000..c8d7b61037e62
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/chipwealth,ch13726a.yaml
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/chipwealth,ch13726a.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Chip Wealth Technology CH13726A AMOLED driver
+
+maintainers:
+  - Neil Armstrong <neil.armstrong@linaro.org>
+
+description:
+  Chip Wealth Technology CH13726A is a single-chip solution
+  for AMOLED connected using a MIPI-DSI video interface.
+
+allOf:
+  - $ref: panel-common.yaml#
+
+properties:
+  compatible:
+    items:
+      - const: ayntec,thor-panel-bottom
+      - const: chipwealth,ch13726a
+
+  reg:
+    maxItems: 1
+    description: DSI virtual channel
+
+  vdd-supply: true
+  vddio-supply: true
+  vdd1v2-supply: true
+  avdd-supply: true
+
+  port: true
+  reset-gpios: true
+  rotation: true
+
+required:
+  - compatible
+  - reg
+  - vdd-supply
+  - vddio-supply
+  - vdd1v2-supply
+  - avdd-supply
+  - reset-gpios
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+        panel@0 {
+            compatible = "ayntec,thor-panel-bottom", "chipwealth,ch13726a";
+            reg = <0>;
+            vdd1v2-supply = <&vreg_l11b_1p2>;
+            vddio-supply = <&vdd_disp_1v8>;
+            vdd-supply = <&vreg_l13b_3p0>;
+            avdd-supply = <&vdd_disp2_2v8>;
+            reset-gpios = <&tlmm 133 GPIO_ACTIVE_HIGH>;
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/display/panel/focaltech,ota7290b.yaml b/Documentation/devicetree/bindings/display/panel/focaltech,ota7290b.yaml
new file mode 100644
index 0000000000000..db6775f4d75ca
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/focaltech,ota7290b.yaml
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/focaltech,ota7290b.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Focaltech OTA7290B DSI panels
+
+maintainers:
+  - Dmitry Baryshkov <dmitry.baryshkov@oss.qualcomm.com>
+
+allOf:
+  - $ref: panel-common.yaml#
+
+properties:
+  compatible:
+    const: waveshare,8.8-dsi-touch-a
+
+  reg:
+    maxItems: 1
+
+  vdd-supply:
+    description: supply regulator for VDD, usually 3.3V
+
+  vdda-supply:
+    description: supply regulator for VDDA, 7-10V
+
+  vcc-supply:
+    description: supply regulator for VCCIO, usually 1.5V
+
+  reset-gpios: true
+  backlight: true
+  rotation: true
+  port: true
+
+required:
+  - compatible
+  - reg
+  - vdd-supply
+  - vcc-supply
+  - reset-gpios
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        panel@0 {
+            compatible = "waveshare,8.8-dsi-touch-a";
+            reg = <0>;
+            vdd-supply = <&vdd>;
+            vcc-supply = <&vccio>;
+            reset-gpios = <&gpio 1 GPIO_ACTIVE_HIGH>;
+            backlight = <&backlight>;
+
+            port {
+                endpoint {
+                    remote-endpoint = <&mipi_out_panel>;
+                };
+            };
+        };
+    };
+
+...
+
diff --git a/Documentation/devicetree/bindings/display/panel/himax,hx83102.yaml b/Documentation/devicetree/bindings/display/panel/himax,hx83102.yaml
index 66404b425af35..7667428bf9a88 100644
--- a/Documentation/devicetree/bindings/display/panel/himax,hx83102.yaml
+++ b/Documentation/devicetree/bindings/display/panel/himax,hx83102.yaml
@@ -30,6 +30,8 @@ properties:
           - starry,2082109qfh040022-50e
           # STARRY himax83102-j02 10.51" WUXGA TFT LCD panel
           - starry,himax83102-j02
+          # Waveshare 12.3-DSI-TOUCH-A panel
+          - waveshare,12.3-dsi-touch-a
       - const: himax,hx83102
 
   reg:
diff --git a/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml b/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml
index 84e840e0224f2..83c343b028355 100644
--- a/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml
+++ b/Documentation/devicetree/bindings/display/panel/himax,hx8394.yaml
@@ -23,6 +23,8 @@ properties:
               - hannstar,hsd060bhw4
               - microchip,ac40t08a-mipi-panel
               - powkiddy,x55-panel
+              - waveshare,5.0-dsi-touch-a
+              - waveshare,5.5-dsi-touch-a
           - const: himax,hx8394
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/display/panel/ilitek,ili9881c.yaml b/Documentation/devicetree/bindings/display/panel/ilitek,ili9881c.yaml
index d979701a00a8a..42e35986fbf60 100644
--- a/Documentation/devicetree/bindings/display/panel/ilitek,ili9881c.yaml
+++ b/Documentation/devicetree/bindings/display/panel/ilitek,ili9881c.yaml
@@ -24,6 +24,7 @@ properties:
           - raspberrypi,dsi-7inch
           - startek,kd050hdfia020
           - tdo,tl050hdv35
+          - waveshare,7.0-dsi-touch-a
           - wanchanglong,w552946aaa
           - wanchanglong,w552946aba
       - const: ilitek,ili9881c
@@ -34,6 +35,7 @@ properties:
   backlight: true
   port: true
   power-supply: true
+  iovcc-supply: true
   reset-gpios: true
   rotation: true
 
diff --git a/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml b/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml
index e39efb44ed42c..4eae802de9fd5 100644
--- a/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml
+++ b/Documentation/devicetree/bindings/display/panel/jadard,jd9365da-h3.yaml
@@ -24,6 +24,12 @@ properties:
           - radxa,display-10hd-ad001
           - radxa,display-8hd-ad002
           - taiguanck,xti05101-01a
+          - waveshare,3.4-dsi-touch-c
+          - waveshare,4.0-dsi-touch-c
+          - waveshare,8.0-dsi-touch-a
+          - waveshare,9.0-dsi-touch-b
+          - waveshare,10.1-dsi-touch-a
+          - waveshare,10.1-dsi-touch-b
       - const: jadard,jd9365da-h3
 
   reg:
diff --git a/Documentation/devicetree/bindings/display/panel/novatek,nt35532.yaml b/Documentation/devicetree/bindings/display/panel/novatek,nt35532.yaml
new file mode 100644
index 0000000000000..ff6fdad7febfa
--- /dev/null
+++ b/Documentation/devicetree/bindings/display/panel/novatek,nt35532.yaml
@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/display/panel/novatek,nt35532.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Novatek NT35532-based DSI display panels
+
+maintainers:
+  - Cristian Cozzolino <cristian_ci@protonmail.com>
+
+allOf:
+  - $ref: panel-common.yaml#
+
+properties:
+  compatible:
+    items:
+      - enum:
+          - flipkart,rimob-panel-nt35532-cs
+      - const: novatek,nt35532
+
+  reg:
+    maxItems: 1
+
+  backlight: true
+  reset-gpios: true
+
+  avdd-supply:
+    description: positive boost supply regulator
+
+  avee-supply:
+    description: negative boost supply regulator
+
+  vci-supply:
+    description: regulator that supplies the analog voltage
+
+  vddam-supply:
+    description: power supply for MIPI interface
+
+  vddi-supply:
+    description: regulator that supplies the I/O voltage
+
+  port: true
+
+required:
+  - compatible
+  - reg
+  - reset-gpios
+  - vddi-supply
+  - port
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    dsi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        panel@0 {
+            compatible = "flipkart,rimob-panel-nt35532-cs", "novatek,nt35532";
+            reg = <0>;
+
+            backlight = <&pmi8950_wled>;
+            reset-gpios = <&tlmm 61 GPIO_ACTIVE_LOW>;
+            avdd-supply = <&lab>;
+            avee-supply = <&ibb>;
+            vci-supply = <&pm8953_l17>;
+            vddi-supply = <&pm8953_l6>;
+
+            port {
+                panel_in: endpoint {
+                    remote-endpoint = <&dsi0_out>;
+                };
+            };
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml
index cc8d795df732c..6d4133b91e7cc 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-simple-dsi.yaml
@@ -63,6 +63,8 @@ properties:
       - samsung,s6e3fa7-ams559nk06
         # Shangai Top Display Optoelectronics 7" TL070WSH30 1024x600 TFT LCD panel
       - tdo,tl070wsh30
+        # Team Source Display Technology 7" TST070WSBE-196C 1024x600 TFT LCD panel
+      - team-source-display,tst070wsbe-196c
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
index 3e41ed0ef5d51..44ec7388d8512 100644
--- a/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
+++ b/Documentation/devicetree/bindings/display/panel/panel-simple.yaml
@@ -97,6 +97,8 @@ properties:
       - dataimage,fg1001l0dsswmg01
         # DataImage, Inc. 7" WVGA (800x480) TFT LCD panel with 24-bit parallel interface.
       - dataimage,scf0700c48ggu18
+        # Displaytech DT050BTFT-PTS 5.0" 800x480 TFT LCD Panel
+      - displaytech,dt050btft-pts
         # DLC Display Co. DLC1010GIG 10.1" WXGA TFT LCD Panel
       - dlc,dlc1010gig
         # Emerging Display Technology Corp. 3.5" QVGA TFT LCD panel
@@ -304,6 +306,8 @@ properties:
       - shelly,sca07010-bfn-lnn
         # Starry KR070PE2T 7" WVGA TFT LCD panel
       - starry,kr070pe2t
+        # Startek KD070HDFLD092 7" WSVGA TFT LCD panel
+      - startek,kd070hdfld092
         # Startek KD070WVFPA043-C069A 7" TFT LCD panel
       - startek,kd070wvfpa
         # Team Source Display Technology TST043015CMHX 4.3" WQVGA TFT LCD panel
@@ -341,10 +345,38 @@ properties:
       - vivax,tpc9150-panel
         # VXT 800x480 color TFT LCD panel
       - vxt,vl050-8048nt-c01
+        # Waveshare 10.1" WXGA (1280x800) LCD panel
+      - waveshare,10.1inch-c-panel
+        # Waveshare 11.9" (320x1480) LCD panel
+      - waveshare,11.9inch-panel
         # Waveshare 13.3" FHD (1920x1080) LCD panel
       - waveshare,13.3inch-panel
+        # Waveshare 2.8" VGA (480x640) LCD panel
+      - waveshare,2.8inch-panel
+        # Waveshare 3.4" (800x800) LCD panel
+      - waveshare,3.4inch-c-panel
+        # Waveshare 4.0" WVGA (480x800) LCD panel
+      - waveshare,4.0inch-panel
+        # Waveshare 4.0" (720x720) LCD panel
+      - waveshare,4.0inch-c-panel
+        # Waveshare 5.0" WSVGA (1024x600) LCD panel
+      - waveshare,5.0inch-c-panel
+        # Waveshare 5.0" HD 720p (720x1280) LCD panel
+      - waveshare,5.0inch-d-panel
+        # Waveshare 6.25" (720x1560) LCD panel
+      - waveshare,6.25inch-panel
         # Waveshare 7.0" WSVGA (1024x600) LCD panel
       - waveshare,7.0inch-c-panel
+        # Waveshare 7.0" WXGA (1280x800) LCD panel
+      - waveshare,7.0inch-e-panel
+        # Waveshare 7.0" HD 720p (720x1280) LCD panel
+      - waveshare,7.0inch-h-panel
+        # Waveshare 7.9" (400x1280) LCD panel
+      - waveshare,7.9inch-panel
+        # Waveshare 8.0" WXGA (1280x800) LCD panel
+      - waveshare,8.0inch-c-panel
+        # Waveshare 8.8" (480x1920) LCD panel
+      - waveshare,8.8inch-panel
         # Winstar Display Corporation 3.5" QVGA (320x240) TFT LCD panel
       - winstar,wf35ltiacd
         # Yes Optoelectronics YTC700TLAG-05-201C 7" TFT LCD panel
diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3399-cdn-dp.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3399-cdn-dp.yaml
index 1a33128e77f58..195f665970bf9 100644
--- a/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3399-cdn-dp.yaml
+++ b/Documentation/devicetree/bindings/display/rockchip/rockchip,rk3399-cdn-dp.yaml
@@ -41,7 +41,9 @@ properties:
     minItems: 1
     items:
       - description: Extcon device providing the cable state for DP PHY device 0
+        maxItems: 1
       - description: Extcon device providing the cable state for DP PHY device 1
+        maxItems: 1
     description:
       List of phandle to the extcon device providing the cable state for the DP PHY.
 
diff --git a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
index 6493a6968bb4b..0923fb189ada9 100644
--- a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
+++ b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
@@ -23,6 +23,8 @@ properties:
           - qcom,bam-v1.4.0
           # MSM8916, SDM630
           - qcom,bam-v1.7.0
+          # Kaanapali
+          - qcom,bam-v2.0.0
       - items:
           - enum:
               # SDM845, SM6115, SM8150, SM8250 and QCM2290
@@ -118,4 +120,23 @@ examples:
         #dma-cells = <1>;
         qcom,ee = <0>;
     };
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        dma-controller@1dc4000 {
+            compatible = "qcom,bam-v2.0.0";
+            reg = <0x0 0x01dc4000 0x0 0x22000>;
+            interrupts = <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+            #dma-cells = <1>;
+            iommus = <&apps_smmu 0xc0 0>, <&apps_smmu 0xc1 0>;
+            qcom,ee = <0>;
+            qcom,num-ees = <4>;
+            num-channels = <20>;
+            qcom,controlled-remotely;
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml
index fde1df035ad12..8f9a552fe30e8 100644
--- a/Documentation/devicetree/bindings/dma/qcom,gpi.yaml
+++ b/Documentation/devicetree/bindings/dma/qcom,gpi.yaml
@@ -24,7 +24,9 @@ properties:
           - qcom,sm6350-gpi-dma
       - items:
           - enum:
+              - qcom,eliza-gpi-dma
               - qcom,glymur-gpi-dma
+              - qcom,hawi-gpi-dma
               - qcom,kaanapali-gpi-dma
               - qcom,milos-gpi-dma
               - qcom,qcm2290-gpi-dma
diff --git a/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml b/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml
index ec06235baf5ca..62ce6d81526b4 100644
--- a/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml
+++ b/Documentation/devicetree/bindings/dma/spacemit,k1-pdma.yaml
@@ -14,7 +14,9 @@ allOf:
 
 properties:
   compatible:
-    const: spacemit,k1-pdma
+    enum:
+      - spacemit,k1-pdma
+      - spacemit,k3-pdma
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml b/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml
index 607da11e7baa9..d8f92838f4c9d 100644
--- a/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml
+++ b/Documentation/devicetree/bindings/dma/stericsson,dma40.yaml
@@ -136,13 +136,9 @@ properties:
     maxItems: 1
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle-array
-    description: A phandle array with inner size 1 (no arg cells).
-      First phandle is the LCPA (Logical Channel Parameter Address) memory.
-      Second phandle is the  LCLA (Logical Channel Link base Address) memory.
-    maxItems: 2
     items:
-      maxItems: 1
+      - description: LCPA (Logical Channel Parameter Address) memory.
+      - description: LCLA (Logical Channel Link base Address) memory.
 
   memcpy-channels:
     $ref: /schemas/types.yaml#/definitions/uint32-array
diff --git a/Documentation/devicetree/bindings/firmware/fsl,imx-se.yaml b/Documentation/devicetree/bindings/firmware/fsl,imx-se.yaml
new file mode 100644
index 0000000000000..fa81adbf9b80d
--- /dev/null
+++ b/Documentation/devicetree/bindings/firmware/fsl,imx-se.yaml
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/firmware/fsl,imx-se.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: NXP i.MX HW Secure Enclave(s) EdgeLock Enclave
+
+maintainers:
+  - Pankaj Gupta <pankaj.gupta@nxp.com>
+
+description: |
+  NXP's SoC may contain one or multiple embedded secure-enclave HW
+  IP(s) like i.MX EdgeLock Enclave, V2X etc. These NXP's HW IP(s)
+  enables features like
+    - Hardware Security Module (HSM),
+    - Security Hardware Extension (SHE), and
+    - Vehicular to Anything (V2X)
+
+  Communication interface to the secure-enclaves(se) is based on the
+  messaging unit(s).
+
+properties:
+  compatible:
+    enum:
+      - fsl,imx8ulp-se-ele-hsm
+      - fsl,imx93-se-ele-hsm
+      - fsl,imx95-se-ele-hsm
+
+  mboxes:
+    items:
+      - description: mailbox phandle to send message to se firmware
+      - description: mailbox phandle to receive message from se firmware
+
+  mbox-names:
+    items:
+      - const: tx
+      - const: rx
+
+  memory-region:
+    maxItems: 1
+
+  sram:
+    maxItems: 1
+
+required:
+  - compatible
+  - mboxes
+  - mbox-names
+
+allOf:
+  # memory-region
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - fsl,imx8ulp-se-ele-hsm
+              - fsl,imx93-se-ele-hsm
+    then:
+      required:
+        - memory-region
+    else:
+      properties:
+        memory-region: false
+
+  # sram
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - fsl,imx8ulp-se-ele-hsm
+    then:
+      required:
+        - sram
+
+    else:
+      properties:
+        sram: false
+
+additionalProperties: false
+
+examples:
+  - |
+    secure-enclave {
+      compatible = "fsl,imx95-se-ele-hsm";
+      mboxes = <&ele_mu0 0 0>, <&ele_mu0 1 0>;
+      mbox-names = "tx", "rx";
+    };
+...
diff --git a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
index 7918d31f58b4a..25f62bacbc919 100644
--- a/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
+++ b/Documentation/devicetree/bindings/firmware/qcom,scm.yaml
@@ -25,6 +25,7 @@ properties:
           - qcom,scm-apq8084
           - qcom,scm-eliza
           - qcom,scm-glymur
+          - qcom,scm-hawi
           - qcom,scm-ipq4019
           - qcom,scm-ipq5018
           - qcom,scm-ipq5210
@@ -49,6 +50,7 @@ properties:
           - qcom,scm-msm8994
           - qcom,scm-msm8996
           - qcom,scm-msm8998
+          - qcom,scm-nord
           - qcom,scm-qcm2290
           - qcom,scm-qcs615
           - qcom,scm-qcs8300
@@ -208,6 +210,7 @@ allOf:
             contains:
               enum:
                 - qcom,scm-eliza
+                - qcom,scm-hawi
                 - qcom,scm-kaanapali
                 - qcom,scm-milos
                 - qcom,scm-sm8450
diff --git a/Documentation/devicetree/bindings/fpga/altera-pr-ip.txt b/Documentation/devicetree/bindings/fpga/altera-pr-ip.txt
deleted file mode 100644
index 52a294cf27305..0000000000000
--- a/Documentation/devicetree/bindings/fpga/altera-pr-ip.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-Altera Arria10 Partial Reconfiguration IP
-
-Required properties:
-- compatible : should contain "altr,a10-pr-ip"
-- reg        : base address and size for memory mapped io.
-
-Example:
-
-	fpga_mgr: fpga-mgr@ff20c000 {
-		compatible = "altr,a10-pr-ip";
-		reg = <0xff20c000 0x10>;
-	};
diff --git a/Documentation/devicetree/bindings/fpga/altera-socfpga-fpga-mgr.txt b/Documentation/devicetree/bindings/fpga/altera-socfpga-fpga-mgr.txt
deleted file mode 100644
index d52f3340414d0..0000000000000
--- a/Documentation/devicetree/bindings/fpga/altera-socfpga-fpga-mgr.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-Altera SOCFPGA FPGA Manager
-
-Required properties:
-- compatible : should contain "altr,socfpga-fpga-mgr"
-- reg        : base address and size for memory mapped io.
-               - The first index is for FPGA manager register access.
-               - The second index is for writing FPGA configuration data.
-- interrupts : interrupt for the FPGA Manager device.
-
-Example:
-
-	hps_0_fpgamgr: fpgamgr@ff706000 {
-		compatible = "altr,socfpga-fpga-mgr";
-		reg = <0xFF706000 0x1000
-		       0xFFB90000 0x1000>;
-		interrupts = <0 175 4>;
-	};
diff --git a/Documentation/devicetree/bindings/fpga/altr,a10-pr-ip.yaml b/Documentation/devicetree/bindings/fpga/altr,a10-pr-ip.yaml
new file mode 100644
index 0000000000000..1f4df40308bd0
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/altr,a10-pr-ip.yaml
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/fpga/altr,a10-pr-ip.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Altera Arria10 Partial Reconfiguration IP
+
+maintainers:
+  - Matthew Gerlach <matthew.gerlach@linux.intel.com>
+
+description:
+  The Altera Arria 10 Partial Reconfiguration IP core allows the host
+  processor to perform partial reconfiguration of the FPGA fabric.
+
+properties:
+  compatible:
+    const: altr,a10-pr-ip
+
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    fpga-mgr@ff20c000 {
+        compatible = "altr,a10-pr-ip";
+        reg = <0xff20c000 0x10>;
+    };
diff --git a/Documentation/devicetree/bindings/fpga/altr,socfpga-fpga-mgr.yaml b/Documentation/devicetree/bindings/fpga/altr,socfpga-fpga-mgr.yaml
new file mode 100644
index 0000000000000..9bcc1200d61d0
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/altr,socfpga-fpga-mgr.yaml
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/fpga/altr,socfpga-fpga-mgr.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Altera SOCFPGA FPGA Manager
+
+maintainers:
+  - Steffen Trumtrar <s.trumtrar@pengutronix.de>
+
+properties:
+  compatible:
+    const: altr,socfpga-fpga-mgr
+
+  reg:
+    items:
+      - description: FPGA manager register access
+      - description: Writing FPGA configuration data
+
+  interrupts:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+additionalProperties: false
+
+examples:
+  - |
+    fpgamgr@ff706000 {
+        compatible = "altr,socfpga-fpga-mgr";
+        reg = <0xff706000 0x1000>,
+              <0xffb90000 0x1000>;
+        interrupts = <0 175 4>;
+    };
diff --git a/Documentation/devicetree/bindings/fpga/efinix,trion-config.yaml b/Documentation/devicetree/bindings/fpga/efinix,trion-config.yaml
new file mode 100644
index 0000000000000..7c7444ff9c3ad
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/efinix,trion-config.yaml
@@ -0,0 +1,98 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/fpga/efinix,trion-config.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Efinix SPI FPGA Manager
+
+maintainers:
+  - Ian Dannapel <iansdannapel@gmail.com>
+
+description: |
+  Efinix FPGAs (Trion, Topaz, and Titanium families) support loading bitstreams
+  through "SPI Passive Mode".
+  Additional pin hogs for bus width configuration should be set
+  elsewhere, if necessary.
+
+  References:
+  - https://www.efinixinc.com/docs/an006-configuring-trion-fpgas-v6.3.pdf
+  - https://www.efinixinc.com/docs/an033-configuring-titanium-fpgas-v2.8.pdf
+  - https://www.efinixinc.com/docs/an061-configuring-topaz-fpgas-v1.1.pdf
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - efinix,titanium-config
+              - efinix,topaz-config
+          - const: efinix,trion-config
+      - const: efinix,trion-config
+
+  spi-cpha: true
+
+  spi-cpol: true
+
+  spi-max-frequency:
+    maximum: 25000000
+
+  reg:
+    maxItems: 1
+
+  reset-gpios:
+    description:
+      reset and re-configuration trigger pin (low active)
+    maxItems: 1
+
+  cdone-gpios:
+    description:
+      optional configuration done status pin (high active)
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - reset-gpios
+  - spi-cpha
+  - spi-cpol
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    spi {
+      #address-cells = <1>;
+      #size-cells = <0>;
+      cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>;
+      fpga-mgr@0 {
+        compatible = "efinix,trion-config";
+        reg = <0>;
+        spi-max-frequency = <25000000>;
+        spi-cpha;
+        spi-cpol;
+        reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>;
+        cdone-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>;
+      };
+    };
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    spi {
+      #address-cells = <1>;
+      #size-cells = <0>;
+      cs-gpios = <&gpio5 13 GPIO_ACTIVE_LOW>;
+      fpga-mgr@0 {
+        compatible = "efinix,titanium-config", "efinix,trion-config";
+        reg = <0>;
+        spi-max-frequency = <25000000>;
+        spi-cpha;
+        spi-cpol;
+        reset-gpios = <&gpio4 17 GPIO_ACTIVE_LOW>;
+        cdone-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>;
+      };
+    };
+...
diff --git a/Documentation/devicetree/bindings/fpga/technologic,ts7300-fpga.yaml b/Documentation/devicetree/bindings/fpga/technologic,ts7300-fpga.yaml
new file mode 100644
index 0000000000000..c93e3a1a135bb
--- /dev/null
+++ b/Documentation/devicetree/bindings/fpga/technologic,ts7300-fpga.yaml
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/fpga/technologic,ts7300-fpga.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Technologic Systems TS-7300 FPGA Manager
+
+maintainers:
+  - Florian Fainelli <f.fainelli@gmail.com>
+
+description:
+  FPGA manager for the Altera Cyclone II FPGA on Technologic Systems
+  TS-7300 board. The FPGA is programmed via the memory-mapped interface
+  implemented in the CPLD.
+
+properties:
+  compatible:
+    const: technologic,ts7300-fpga
+
+  reg:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    fpga-mgr@13c00000 {
+        compatible = "technologic,ts7300-fpga";
+        reg = <0x13c00000 0x2>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/gpio/fairchild,74hc595.yaml b/Documentation/devicetree/bindings/gpio/fairchild,74hc595.yaml
index 23410aeca3000..451538df63f72 100644
--- a/Documentation/devicetree/bindings/gpio/fairchild,74hc595.yaml
+++ b/Documentation/devicetree/bindings/gpio/fairchild,74hc595.yaml
@@ -45,6 +45,18 @@ properties:
     $ref: /schemas/types.yaml#/definitions/uint32
     description: Number of daisy-chained shift registers
 
+  lines-initial-states:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      Bitmask that specifies the initial state of each output line, written
+      by the driver before the gpiochip is registered. Bit N corresponds to
+      GPIO line N, following the convention already documented for
+      nxp,pcf8575. Because the 74HC595/74LVC594 family is push-pull output
+      only, a bit set to zero drives the line low and a bit set to one
+      drives it high. The bitmask covers up to 32 lines (four cascaded
+      registers); outputs beyond that come up zeroed. When the property is
+      absent all outputs come up low, preserving the previous behaviour.
+
   enable-gpios:
     description: GPIO connected to the OE (Output Enable) pin.
     maxItems: 1
@@ -79,6 +91,7 @@ examples:
             gpio-controller;
             #gpio-cells = <2>;
             registers-number = <4>;
+            lines-initial-states = <0xffff0000>;
             spi-max-frequency = <100000>;
         };
     };
diff --git a/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml b/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml
index 5e2496379a3c8..de24bb361e9f6 100644
--- a/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml
+++ b/Documentation/devicetree/bindings/gpio/gpio-zynq.yaml
@@ -12,10 +12,11 @@ maintainers:
 properties:
   compatible:
     enum:
+      - xlnx,eio-gpio-1.0
+      - xlnx,pmc-gpio-1.0
+      - xlnx,versal-gpio-1.0
       - xlnx,zynq-gpio-1.0
       - xlnx,zynqmp-gpio-1.0
-      - xlnx,versal-gpio-1.0
-      - xlnx,pmc-gpio-1.0
 
   reg:
     maxItems: 1
@@ -30,7 +31,7 @@ properties:
 
   gpio-line-names:
     description: strings describing the names of each gpio line
-    minItems: 58
+    minItems: 52
     maxItems: 174
 
   interrupt-controller: true
@@ -89,6 +90,16 @@ allOf:
           minItems: 116
           maxItems: 116
 
+  - if:
+      properties:
+        compatible:
+          enum:
+            - xlnx,eio-gpio-1.0
+    then:
+      properties:
+        gpio-line-names:
+          maxItems: 52
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/gpio/nvidia,tegra186-gpio.yaml b/Documentation/devicetree/bindings/gpio/nvidia,tegra186-gpio.yaml
index 17748dd1015d7..adeb3b3a2902d 100644
--- a/Documentation/devicetree/bindings/gpio/nvidia,tegra186-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/nvidia,tegra186-gpio.yaml
@@ -85,6 +85,8 @@ properties:
       - nvidia,tegra194-gpio-aon
       - nvidia,tegra234-gpio
       - nvidia,tegra234-gpio-aon
+      - nvidia,tegra238-gpio
+      - nvidia,tegra238-gpio-aon
       - nvidia,tegra256-gpio
       - nvidia,tegra264-gpio
       - nvidia,tegra264-gpio-uphy
@@ -163,6 +165,7 @@ allOf:
               - nvidia,tegra186-gpio
               - nvidia,tegra194-gpio
               - nvidia,tegra234-gpio
+              - nvidia,tegra238-gpio
               - nvidia,tegra256-gpio
               - nvidia,tegra264-gpio
               - nvidia,tegra264-gpio-uphy
@@ -180,6 +183,7 @@ allOf:
               - nvidia,tegra186-gpio-aon
               - nvidia,tegra194-gpio-aon
               - nvidia,tegra234-gpio-aon
+              - nvidia,tegra238-gpio-aon
               - nvidia,tegra264-gpio-aon
     then:
       properties:
@@ -192,6 +196,8 @@ allOf:
         compatible:
           contains:
             enum:
+              - nvidia,tegra238-gpio
+              - nvidia,tegra238-gpio-aon
               - nvidia,tegra264-gpio
               - nvidia,tegra264-gpio-uphy
               - nvidia,tegra264-gpio-aon
diff --git a/Documentation/devicetree/bindings/gpio/snps,dw-apb-gpio.yaml b/Documentation/devicetree/bindings/gpio/snps,dw-apb-gpio.yaml
index bba6f5b6606fd..55069533f6d91 100644
--- a/Documentation/devicetree/bindings/gpio/snps,dw-apb-gpio.yaml
+++ b/Documentation/devicetree/bindings/gpio/snps,dw-apb-gpio.yaml
@@ -95,6 +95,12 @@ patternProperties:
       '#interrupt-cells':
         const: 2
 
+    patternProperties:
+      "^.+-hog(-[0-9]+)?$":
+        type: object
+        required:
+          - gpio-hog
+
     required:
       - compatible
       - reg
diff --git a/Documentation/devicetree/bindings/gpio/waveshare,dsi-touch-gpio.yaml b/Documentation/devicetree/bindings/gpio/waveshare,dsi-touch-gpio.yaml
new file mode 100644
index 0000000000000..091e1fffcd476
--- /dev/null
+++ b/Documentation/devicetree/bindings/gpio/waveshare,dsi-touch-gpio.yaml
@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/gpio/waveshare,dsi-touch-gpio.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Waveshare GPIO controller on DSI TOUCH panels
+
+maintainers:
+  - Dmitry Baryshkov <dmitry.baryshkov@oss.qualcomm.com>
+
+description:
+  Waveshare DSI TOUCH panel kits contain separate GPIO controller for toggling
+  power supplies and panel / touchscreen resets.
+
+properties:
+  compatible:
+    const: waveshare,dsi-touch-gpio
+
+  reg:
+    maxItems: 1
+
+  gpio-controller: true
+
+  '#gpio-cells':
+    const: 2
+
+required:
+  - compatible
+  - reg
+  - gpio-controller
+  - "#gpio-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        wsgpio: gpio@45 {
+            compatible = "waveshare,dsi-touch-gpio";
+            reg = <0x45>;
+            gpio-controller;
+            #gpio-cells = <2>;
+        };
+    };
+
+    panel_avdd: regulator-panel-avdd {
+        compatible = "regulator-fixed";
+        regulator-name = "panel-avdd";
+        gpios = <&wsgpio 0 GPIO_ACTIVE_HIGH>;
+        enable-active-high;
+    };
+
+    panel_iovcc: regulator-panel-iovcc {
+        compatible = "regulator-fixed";
+        regulator-name = "panel-iovcc";
+        gpios = <&wsgpio 4 GPIO_ACTIVE_HIGH>;
+        enable-active-high;
+    };
+
+    panel_vcc: regulator-panel-vcc {
+        compatible = "regulator-fixed";
+        regulator-name = "panel-vcc";
+        gpios = <&wsgpio 8 GPIO_ACTIVE_HIGH>;
+        enable-active-high;
+        regulator-always-on;
+    };
+...
diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.yaml b/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.yaml
index db49b8ff8c748..9db9f84ad964b 100644
--- a/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.yaml
+++ b/Documentation/devicetree/bindings/gpu/arm,mali-bifrost.yaml
@@ -26,6 +26,7 @@ properties:
               - realtek,rtd1619-mali
               - renesas,r9a07g044-mali
               - renesas,r9a07g054-mali
+              - renesas,r9a08g046-mali
               - renesas,r9a09g047-mali
               - renesas,r9a09g056-mali
               - renesas,r9a09g057-mali
@@ -150,6 +151,7 @@ allOf:
             enum:
               - renesas,r9a07g044-mali
               - renesas,r9a07g054-mali
+              - renesas,r9a08g046-mali
               - renesas,r9a09g047-mali
               - renesas,r9a09g056-mali
               - renesas,r9a09g057-mali
diff --git a/Documentation/devicetree/bindings/hwinfo/ti,k3-socinfo.yaml b/Documentation/devicetree/bindings/hwinfo/ti,k3-socinfo.yaml
index dada28b47ea07..2900224aac743 100644
--- a/Documentation/devicetree/bindings/hwinfo/ti,k3-socinfo.yaml
+++ b/Documentation/devicetree/bindings/hwinfo/ti,k3-socinfo.yaml
@@ -15,6 +15,9 @@ description: |
   represented by CTRLMMR_xxx_JTAGID register which contains information about
   SoC id and revision.
 
+  On some SoCs like AM62P, the silicon revision is determined by reading
+  alternative registers via NVMEM cells.
+
 properties:
   $nodename:
     pattern: "^chipid@[0-9a-f]+$"
@@ -26,6 +29,14 @@ properties:
   reg:
     maxItems: 1
 
+  nvmem-cells:
+    items:
+      - description: Alternate silicon revision register
+
+  nvmem-cell-names:
+    items:
+      - const: gpsw1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/hwmon/lm75.yaml b/Documentation/devicetree/bindings/hwmon/lm75.yaml
index 0b9fda81e3ec5..b48bf3fd721fe 100644
--- a/Documentation/devicetree/bindings/hwmon/lm75.yaml
+++ b/Documentation/devicetree/bindings/hwmon/lm75.yaml
@@ -54,6 +54,13 @@ properties:
   interrupts:
     maxItems: 1
 
+  ti,alert-polarity-active-high:
+    description: Alert pin is asserted based on the value of alert polarity
+      bit of configuration register. Default value is normal (0 which maps to
+      active-low). The other value is inverted (1 which maps to active-high).
+      Specify this property to set the alert polarity to active-high.
+    $ref: /schemas/types.yaml#/definitions/flag
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/hwmon/microchip,emc2305.yaml b/Documentation/devicetree/bindings/hwmon/microchip,emc2305.yaml
index d3f06ebc19fa2..8c2548539d7fd 100644
--- a/Documentation/devicetree/bindings/hwmon/microchip,emc2305.yaml
+++ b/Documentation/devicetree/bindings/hwmon/microchip,emc2305.yaml
@@ -54,6 +54,12 @@ patternProperties:
           The fan number used to determine the associated PWM channel.
         maxItems: 1
 
+      fan-shutdown-percent:
+        description:
+          PWM duty cycle in percent applied to the fan during shutdown.
+        minimum: 0
+        maximum: 100
+
     required:
       - reg
 
@@ -80,12 +86,14 @@ examples:
             fan@0 {
                 reg = <0x0>;
                 pwms = <&fan_controller 26000 PWM_POLARITY_INVERTED 1>;
+                fan-shutdown-percent = <100>;
                 #cooling-cells = <2>;
             };
 
             fan@1 {
                 reg = <0x1>;
                 pwms = <&fan_controller 26000 0 1>;
+                fan-shutdown-percent = <50>;
                 #cooling-cells = <2>;
             };
 
diff --git a/Documentation/devicetree/bindings/hwmon/nsa320-mcu.txt b/Documentation/devicetree/bindings/hwmon/nsa320-mcu.txt
deleted file mode 100644
index 0863e067c85b0..0000000000000
--- a/Documentation/devicetree/bindings/hwmon/nsa320-mcu.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-Bindings for the fan / temperature monitor microcontroller used on
-the Zyxel NSA 320 and several subsequent models.
-
-Required properties:
-- compatible	: "zyxel,nsa320-mcu"
-- data-gpios	: The GPIO pin connected to the data line on the MCU
-- clk-gpios	: The GPIO pin connected to the clock line on the MCU
-- act-gpios	: The GPIO pin connected to the active line on the MCU
-
-Example:
-
-	hwmon {
-		compatible = "zyxel,nsa320-mcu";
-		pinctrl-0 = <&pmx_mcu_data &pmx_mcu_clk &pmx_mcu_act>;
-		pinctrl-names = "default";
-
-		data-gpios = <&gpio0 14 GPIO_ACTIVE_HIGH>;
-		clk-gpios = <&gpio0 16 GPIO_ACTIVE_HIGH>;
-		act-gpios = <&gpio0 17 GPIO_ACTIVE_LOW>;
-	};
diff --git a/Documentation/devicetree/bindings/hwmon/pmbus/adi,max20830.yaml b/Documentation/devicetree/bindings/hwmon/pmbus/adi,max20830.yaml
new file mode 100644
index 0000000000000..1625dd59417f1
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/pmbus/adi,max20830.yaml
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/pmbus/adi,max20830.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices MAX20830 Step-Down Switching Regulator with PMBus
+
+maintainers:
+  - Alexis Czezar Torreno <alexisczezar.torreno@analog.com>
+
+description: |
+  The MAX20830 is a fully integrated step-down DC-DC switching regulator with
+  PMBus interface. It provides 2.7V to 16V input, 0.4V to 5.8V adjustable
+  output, and up to 30A output current. It allows monitoring of input/output
+  voltage, output current and temperature through the PMBus serial interface.
+  Datasheet:
+    https://www.analog.com/en/products/max20830.html
+
+allOf:
+  - $ref: /schemas/regulator/regulator.yaml#
+
+properties:
+  compatible:
+    const: adi,max20830
+
+  reg:
+    maxItems: 1
+
+  vddh-supply:
+    description:
+      Phandle to the regulator that provides the VDDH power supply.
+
+  avdd-supply:
+    description:
+      Phandle to the regulator that provides the AVDD power supply.
+
+  ldoin-supply:
+    description:
+      Optional 2.5V to 5.5V LDO input supply.
+
+  pwr-good-gpios:
+    description:
+      GPIO connected to the power-good status output pin.
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - vddh-supply
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        regulator@30 {
+            compatible = "adi,max20830";
+            reg = <0x30>;
+            vddh-supply = <&vddh>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/hwmon/zyxel,nsa320-mcu.yaml b/Documentation/devicetree/bindings/hwmon/zyxel,nsa320-mcu.yaml
new file mode 100644
index 0000000000000..a111f8125e09e
--- /dev/null
+++ b/Documentation/devicetree/bindings/hwmon/zyxel,nsa320-mcu.yaml
@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/hwmon/zyxel,nsa320-mcu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ZyXEL NSA320 MCU
+
+maintainers:
+  - Adam Baker <linux@baker-net.org.uk>
+  - Guenter Roeck <linux@roeck-us.net>
+
+description:
+  The ZyXEL NSA320 uses a dedicated microcontroller to manage system-critical
+  functions like fan speed and power monitoring. It is connected to the SoC
+  via a GPIO-based serial protocol.
+
+properties:
+  compatible:
+    const: zyxel,nsa320-mcu
+
+  data-gpios:
+    maxItems: 1
+    description: GPIO pin connected to the data line on the MCU.
+
+  clk-gpios:
+    maxItems: 1
+    description: GPIO pin connected to the clock line on the MCU.
+
+  act-gpios:
+    maxItems: 1
+    description: GPIO pin connected to the active line on the MCU.
+
+required:
+  - compatible
+  - data-gpios
+  - clk-gpios
+  - act-gpios
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    hwmon {
+        compatible = "zyxel,nsa320-mcu";
+        pinctrl-0 = <&pmx_mcu_data &pmx_mcu_clk &pmx_mcu_act>;
+        pinctrl-names = "default";
+
+        data-gpios = <&gpio0 14 GPIO_ACTIVE_HIGH>;
+        clk-gpios = <&gpio0 16 GPIO_ACTIVE_HIGH>;
+        act-gpios = <&gpio0 17 GPIO_ACTIVE_LOW>;
+    };
diff --git a/Documentation/devicetree/bindings/i2c/i2c-davinci.txt b/Documentation/devicetree/bindings/i2c/i2c-davinci.txt
deleted file mode 100644
index 6590501c53d4e..0000000000000
--- a/Documentation/devicetree/bindings/i2c/i2c-davinci.txt
+++ /dev/null
@@ -1,43 +0,0 @@
-* Texas Instruments Davinci/Keystone I2C
-
-This file provides information, what the device node for the
-davinci/keystone i2c interface contains.
-
-Required properties:
-- compatible: "ti,davinci-i2c" or "ti,keystone-i2c";
-- reg : Offset and length of the register set for the device
-- clocks: I2C functional clock phandle.
-	  For 66AK2G this property should be set per binding,
-	  Documentation/devicetree/bindings/clock/ti,sci-clk.yaml
-
-SoC-specific Required Properties:
-
-The following are mandatory properties for Keystone 2 66AK2G SoCs only:
-
-- power-domains:	Should contain a phandle to a PM domain provider node
-			and an args specifier containing the I2C device id
-			value. This property is as per the binding,
-			Documentation/devicetree/bindings/soc/ti/sci-pm-domain.yaml
-
-Recommended properties :
-- interrupts : standard interrupt property.
-- clock-frequency : desired I2C bus clock frequency in Hz.
-- ti,has-pfunc: boolean; if defined, it indicates that SoC supports PFUNC
-	registers. PFUNC registers allow to switch I2C pins to function as
-	GPIOs, so they can be toggled manually.
-
-Example (enbw_cmc board):
-	i2c@1c22000 {
-		compatible = "ti,davinci-i2c";
-		reg = <0x22000 0x1000>;
-		clock-frequency = <100000>;
-		interrupts = <15>;
-		interrupt-parent = <&intc>;
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		dtt@48 {
-			compatible = "national,lm75";
-			reg = <0x48>;
-		};
-	};
diff --git a/Documentation/devicetree/bindings/i2c/i2c-demux-pinctrl.yaml b/Documentation/devicetree/bindings/i2c/i2c-demux-pinctrl.yaml
index 1eaf00b90a77a..deca72bfc8cf3 100644
--- a/Documentation/devicetree/bindings/i2c/i2c-demux-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/i2c/i2c-demux-pinctrl.yaml
@@ -40,6 +40,7 @@ properties:
 
   i2c-parent:
     $ref: /schemas/types.yaml#/definitions/phandle-array
+    minItems: 2
     items:
       maxItems: 1
     description:
diff --git a/Documentation/devicetree/bindings/i2c/loongson,ls2x-i2c.yaml b/Documentation/devicetree/bindings/i2c/loongson,ls2x-i2c.yaml
index 67882ec6e06af..ee09c6d9c5f0b 100644
--- a/Documentation/devicetree/bindings/i2c/loongson,ls2x-i2c.yaml
+++ b/Documentation/devicetree/bindings/i2c/loongson,ls2x-i2c.yaml
@@ -16,6 +16,7 @@ properties:
   compatible:
     enum:
       - loongson,ls2k-i2c
+      - loongson,ls2k0300-i2c
       - loongson,ls7a-i2c
 
   reg:
@@ -24,6 +25,9 @@ properties:
   interrupts:
     maxItems: 1
 
+  clocks:
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/i2c/opencores,i2c-ocores.yaml b/Documentation/devicetree/bindings/i2c/opencores,i2c-ocores.yaml
index d9ef867290112..736e9a3eab8a9 100644
--- a/Documentation/devicetree/bindings/i2c/opencores,i2c-ocores.yaml
+++ b/Documentation/devicetree/bindings/i2c/opencores,i2c-ocores.yaml
@@ -95,8 +95,8 @@ examples:
       interrupts = <10>;
       opencores,ip-clock-frequency = <20000000>;
 
-      reg-shift = <0>;	/* 8 bit registers */
-      reg-io-width = <1>;	/* 8 bit read/write */
+      reg-shift = <0>; /* 8 bit registers */
+      reg-io-width = <1>; /* 8 bit read/write */
     };
 
     i2c@b0000000 {
@@ -108,7 +108,7 @@ examples:
       clocks = <&osc>;
       clock-frequency = <400000>; /* i2c bus frequency 400 KHz */
 
-      reg-shift = <0>;	/* 8 bit registers */
-      reg-io-width = <1>;	/* 8 bit read/write */
+      reg-shift = <0>; /* 8 bit registers */
+      reg-io-width = <1>; /* 8 bit read/write */
     };
 ...
diff --git a/Documentation/devicetree/bindings/i2c/ti,davinci-i2c.yaml b/Documentation/devicetree/bindings/i2c/ti,davinci-i2c.yaml
new file mode 100644
index 0000000000000..b94ac06dce366
--- /dev/null
+++ b/Documentation/devicetree/bindings/i2c/ti,davinci-i2c.yaml
@@ -0,0 +1,70 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/i2c/ti,davinci-i2c.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments DaVinci/Keystone I2C
+
+maintainers:
+  - Chaitanya Sabnis <chaitanya.msabnis@gmail.com>
+
+allOf:
+  - $ref: /schemas/i2c/i2c-controller.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,keystone-i2c
+    then:
+      required:
+        - power-domains
+
+properties:
+  compatible:
+    enum:
+      - ti,davinci-i2c
+      - ti,keystone-i2c
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  power-domains:
+    maxItems: 1
+
+  ti,has-pfunc:
+    description:
+      Indicates that the SoC supports PFUNC registers, allowing I2C pins 
+      to function as GPIOs for manual toggling.
+    type: boolean
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c@1c22000 {
+        compatible = "ti,davinci-i2c";
+        reg = <0x01c22000 0x1000>;
+        clocks = <&i2c_clk>;
+        clock-frequency = <100000>;
+        interrupts = <15>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        sensor@48 {
+            compatible = "national,lm75";
+            reg = <0x48>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
index 79df2696ef249..9c6a56c7c8ef1 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
@@ -18,7 +18,11 @@ description: |
   service a wide variety of precision, wide bandwidth data acquisition
   applications.
 
+  The AD4880 is a dual-channel variant with two independent ADC channels,
+  each with its own SPI configuration interface.
+
   https://www.analog.com/media/en/technical-documentation/data-sheets/ad4080.pdf
+  https://www.analog.com/media/en/technical-documentation/data-sheets/ad4880.pdf
 
 $ref: /schemas/spi/spi-peripheral-props.yaml#
 
@@ -34,9 +38,15 @@ properties:
       - adi,ad4086
       - adi,ad4087
       - adi,ad4088
+      - adi,ad4880
 
   reg:
-    maxItems: 1
+    minItems: 1
+    maxItems: 2
+    description:
+      SPI chip select(s). For single-channel devices, one chip select.
+      For multi-channel devices like AD4880, two chip selects are required
+      as each channel has its own SPI configuration interface.
 
   spi-max-frequency:
     description: Configuration of the SPI bus.
@@ -60,7 +70,10 @@ properties:
   vrefin-supply: true
 
   io-backends:
-    maxItems: 1
+    minItems: 1
+    items:
+      - description: Backend for channel A (primary)
+      - description: Backend for channel B (secondary)
 
   adi,lvds-cnv-enable:
     description: Enable the LVDS signal type on the CNV pin. Default is CMOS.
@@ -81,6 +94,25 @@ required:
   - vdd33-supply
   - vrefin-supply
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: adi,ad4880
+    then:
+      properties:
+        reg:
+          minItems: 2
+        io-backends:
+          minItems: 2
+    else:
+      properties:
+        reg:
+          maxItems: 1
+        io-backends:
+          maxItems: 1
+
 additionalProperties: false
 
 examples:
@@ -101,4 +133,21 @@ examples:
           io-backends = <&iio_backend>;
         };
     };
+  - |
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        adc@0 {
+          compatible = "adi,ad4880";
+          reg = <0>, <1>;
+          spi-max-frequency = <10000000>;
+          vdd33-supply = <&vdd33>;
+          vddldo-supply = <&vddldo>;
+          vrefin-supply = <&vrefin>;
+          clocks = <&cnv>;
+          clock-names = "cnv";
+          io-backends = <&iio_backend_cha>, <&iio_backend_chb>;
+        };
+    };
 ...
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml
index d00690a8d3fbd..cc38617bb8295 100644
--- a/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4130.yaml
@@ -5,19 +5,30 @@
 $id: http://devicetree.org/schemas/iio/adc/adi,ad4130.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Analog Devices AD4130 ADC device driver
+title: Analog Devices AD4130 family ADCs
 
 maintainers:
   - Cosmin Tanislav <cosmin.tanislav@analog.com>
 
 description: |
-  Bindings for the Analog Devices AD4130 ADC. Datasheet can be found here:
+  Bindings for the Analog Devices AD4130 family ADCs.
+  Datasheets can be found here:
+    https://www.analog.com/media/en/technical-documentation/data-sheets/AD4129-4.pdf
+    https://www.analog.com/media/en/technical-documentation/data-sheets/AD4129-8.pdf
+    https://www.analog.com/media/en/technical-documentation/data-sheets/AD4130-4.pdf
     https://www.analog.com/media/en/technical-documentation/data-sheets/AD4130-8.pdf
+    https://www.analog.com/media/en/technical-documentation/data-sheets/AD4131-4.pdf
+    https://www.analog.com/media/en/technical-documentation/data-sheets/AD4131-8.pdf
 
 properties:
   compatible:
     enum:
+      - adi,ad4129-4
+      - adi,ad4129-8
+      - adi,ad4130-4
       - adi,ad4130
+      - adi,ad4131-4
+      - adi,ad4131-8
 
   reg:
     maxItems: 1
@@ -32,6 +43,10 @@ properties:
 
   interrupts:
     maxItems: 1
+    description: |
+      Data Ready / FIFO interrupt. For devices with FIFO support, the
+      interrupt polarity specified here is inverted when the device enters
+      FIFO mode, and normal for data ready.
 
   interrupt-names:
     description: |
diff --git a/Documentation/devicetree/bindings/iio/adc/adi,ad4691.yaml b/Documentation/devicetree/bindings/iio/adc/adi,ad4691.yaml
new file mode 100644
index 0000000000000..af28a0c1cfa9e
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/adc/adi,ad4691.yaml
@@ -0,0 +1,180 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/adc/adi,ad4691.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices AD4691 Family Multichannel SAR ADCs
+
+maintainers:
+  - Radu Sabau <radu.sabau@analog.com>
+
+description: |
+  The AD4691 family are high-speed, low-power, multichannel successive
+  approximation register (SAR) analog-to-digital converters (ADCs) with
+  an SPI-compatible serial interface. The ADC supports CNV Burst Mode,
+  where an external PWM drives the CNV pin, and Manual Mode, where CNV
+  is directly tied to the SPI chip-select.
+
+  Datasheets:
+    * https://www.analog.com/en/products/ad4691.html
+    * https://www.analog.com/en/products/ad4692.html
+    * https://www.analog.com/en/products/ad4693.html
+    * https://www.analog.com/en/products/ad4694.html
+
+$ref: /schemas/spi/spi-peripheral-props.yaml#
+
+properties:
+  compatible:
+    enum:
+      - adi,ad4691
+      - adi,ad4692
+      - adi,ad4693
+      - adi,ad4694
+
+  reg:
+    maxItems: 1
+
+  spi-max-frequency:
+    maximum: 40000000
+
+  spi-cpol: true
+  spi-cpha: true
+
+  avdd-supply:
+    description: Analog power supply (4.5V to 5.5V).
+
+  vdd-supply:
+    description:
+      External 1.8V digital core supply. When present, the internal LDO is
+      disabled (LDO_EN = 0). Mutually exclusive with ldo-in-supply.
+
+  ldo-in-supply:
+    description:
+      LDO input supply (2.4V to 5.5V). When present and vdd-supply is absent,
+      the internal LDO generates 1.8V VDD from this input (LDO_EN = 1).
+      Mutually exclusive with vdd-supply.
+
+  vio-supply:
+    description: I/O voltage supply (1.71V to 1.89V or VDD).
+
+  ref-supply:
+    description: External reference voltage supply (2.4V to 5.25V).
+
+  refin-supply:
+    description: Internal reference buffer input supply.
+
+  reset-gpios:
+    description:
+      GPIO line controlling the hardware reset pin (active-low).
+    maxItems: 1
+
+  pwms:
+    description:
+      PWM connected to the CNV pin. When present, selects CNV Burst Mode where
+      the PWM drives the conversion rate. When absent, Manual Mode is used
+      (CNV tied to SPI CS).
+    maxItems: 1
+
+  interrupts:
+    description:
+      Interrupt lines connected to the ADC GP pins. Each GP pin can be
+      physically wired to an interrupt-capable input on the SoC.
+    maxItems: 4
+
+  interrupt-names:
+    description: Names of the interrupt lines, matching the GP pin names.
+    minItems: 1
+    maxItems: 4
+    items:
+      enum:
+        - gp0
+        - gp1
+        - gp2
+        - gp3
+
+  gpio-controller: true
+
+  '#gpio-cells':
+    const: 2
+
+  '#trigger-source-cells':
+    description:
+      This node can act as a trigger source. The single cell in a consumer
+      reference specifies the GP pin number (0-3) used as the trigger output.
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - avdd-supply
+  - vio-supply
+
+allOf:
+  # vdd-supply and ldo-in-supply are mutually exclusive, one is required:
+  # either an external 1.8V VDD is provided or the internal LDO is fed from
+  # ldo-in-supply to generate VDD.
+  - oneOf:
+      - required:
+          - vdd-supply
+      - required:
+          - ldo-in-supply
+  # ref-supply and refin-supply are mutually exclusive, one is required
+  - oneOf:
+      - required:
+          - ref-supply
+      - required:
+          - refin-supply
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    /* AD4692 in CNV Burst Mode with SPI offload */
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        adc@0 {
+            compatible = "adi,ad4692";
+            reg = <0>;
+            spi-cpol;
+            spi-cpha;
+            spi-max-frequency = <40000000>;
+
+            avdd-supply = <&avdd_supply>;
+            ldo-in-supply = <&avdd_supply>;
+            vio-supply = <&vio_supply>;
+            ref-supply = <&ref_5v>;
+
+            reset-gpios = <&gpio0 15 GPIO_ACTIVE_LOW>;
+
+            pwms = <&pwm_gen 0 0>;
+
+            #trigger-source-cells = <1>;
+        };
+    };
+
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    /* AD4692 in Manual Mode (CNV tied to SPI CS) */
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        adc@0 {
+            compatible = "adi,ad4692";
+            reg = <0>;
+            spi-cpol;
+            spi-cpha;
+            spi-max-frequency = <31250000>;
+
+            avdd-supply = <&avdd_supply>;
+            ldo-in-supply = <&avdd_supply>;
+            vio-supply = <&vio_supply>;
+            refin-supply = <&refin_supply>;
+
+            reset-gpios = <&gpio0 15 GPIO_ACTIVE_LOW>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/iio/adc/allwinner,sun20i-d1-gpadc.yaml b/Documentation/devicetree/bindings/iio/adc/allwinner,sun20i-d1-gpadc.yaml
index da605a051b949..6467800d30e27 100644
--- a/Documentation/devicetree/bindings/iio/adc/allwinner,sun20i-d1-gpadc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/allwinner,sun20i-d1-gpadc.yaml
@@ -14,6 +14,7 @@ properties:
     oneOf:
       - enum:
           - allwinner,sun20i-d1-gpadc
+          - allwinner,sun55i-a523-gpadc
       - items:
           - enum:
               - allwinner,sun50i-h616-gpadc
@@ -29,7 +30,12 @@ properties:
     const: 0
 
   clocks:
-    maxItems: 1
+    minItems: 1
+    maxItems: 2
+
+  clock-names:
+    minItems: 1
+    maxItems: 2
 
   interrupts:
     maxItems: 1
@@ -40,6 +46,30 @@ properties:
   resets:
     maxItems: 1
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          enum:
+            - allwinner,sun55i-a523-gpadc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: Bus clock
+            - description: Module clock
+        clock-names:
+          items:
+            - const: bus
+            - const: mod
+      required:
+        - clock-names
+    else:
+      properties:
+        clocks:
+          maxItems: 1
+        clock-names: false
+
 patternProperties:
   "^channel@[0-9a-f]+$":
     $ref: adc.yaml
diff --git a/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml b/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml
index 9c57eb13f8928..9134bbe413796 100644
--- a/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml
@@ -10,14 +10,9 @@ maintainers:
   - Andreas Klinger <ak@it-klinger.de>
 
 description: |
-  Bit-banging driver using two GPIOs:
-  - sck-gpio gives a clock to the sensor with 24 cycles for data retrieval
-    and up to 3 cycles for selection of the input channel and gain for the
-    next measurement
-  - dout-gpio is the sensor data the sensor responds to the clock
-
-  Specifications about the driver can be found at:
-  http://www.aviaic.com/ENProducts.aspx
+  The HX711 is a 24-bit ADC with selectable gain (32/64/128) and two
+  differential input channels. Channel A supports gain 64 and 128;
+  channel B supports gain 32.
 
 properties:
   compatible:
@@ -26,23 +21,34 @@ properties:
 
   sck-gpios:
     description:
-      Definition of the GPIO for the clock (output). In the datasheet it is
-      named PD_SCK
+      GPIO for the clock output (PD_SCK in the datasheet).
     maxItems: 1
 
   dout-gpios:
     description:
-      Definition of the GPIO for the data-out sent by the sensor in
-      response to the clock (input).
-      See Documentation/devicetree/bindings/gpio/gpio.txt for information
-      on how to specify a consumer gpio.
+      GPIO for the data output from the sensor (DOUT in the datasheet).
     maxItems: 1
 
   avdd-supply:
     description:
-      Definition of the regulator used as analog supply
+      Analog supply voltage (AVDD).
+
+  vsup-supply:
+    description:
+      Supply voltage for the on-chip regulator (VSUP).
+
+  rate-gpios:
+    description:
+      GPIO connected to the RATE pin. When driven low the output data
+      rate is 10 SPS; when driven high it is 80 SPS. If omitted the
+      RATE pin state is determined by the board wiring.
+    maxItems: 1
 
   clock-frequency:
+    description:
+      Controls the SCK bit-bang timing. The value is used to derive the
+      delay between SCK edges; keep the SCK high time below 60 us to
+      avoid triggering chip power-down mode.
     minimum: 20000
     maximum: 2500000
     default: 400000
@@ -62,6 +68,7 @@ examples:
         compatible = "avia,hx711";
         sck-gpios = <&gpio3 10 GPIO_ACTIVE_HIGH>;
         dout-gpios = <&gpio0 7 GPIO_ACTIVE_HIGH>;
+        rate-gpios = <&gpio0 9 GPIO_ACTIVE_HIGH>;
         avdd-supply = <&avdd>;
         clock-frequency = <100000>;
     };
diff --git a/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml b/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml
index c978c3a3e31af..63aac8de22ad2 100644
--- a/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/qcom,pm8018-adc.yaml
@@ -78,6 +78,10 @@ patternProperties:
       reg:
         maxItems: 1
 
+      label:
+        description: |
+          Unique name to identify which channel this is.
+
       qcom,decimation:
         $ref: /schemas/types.yaml#/definitions/uint32
         description: |
@@ -130,36 +134,47 @@ examples:
 
             vcoin: adc-channel@0 {
                 reg = <0x00 0x00>;
+                label = "vcoin";
             };
             vbat: adc-channel@1 {
                 reg = <0x00 0x01>;
+                label = "vbat";
             };
             dcin: adc-channel@2 {
                 reg = <0x00 0x02>;
+                label = "dcin";
             };
             ichg: adc-channel@3 {
                 reg = <0x00 0x03>;
+                label = "ichg";
             };
             vph_pwr: adc-channel@4 {
                 reg = <0x00 0x04>;
+                label = "vph_pwr";
             };
             usb_vbus: adc-channel@a {
                 reg = <0x00 0x0a>;
+                label = "usb_vbus";
             };
             die_temp: adc-channel@b {
                 reg = <0x00 0x0b>;
+                label = "die_temp";
             };
             ref_625mv: adc-channel@c {
                 reg = <0x00 0x0c>;
+                label = "ref_625mv";
             };
             ref_1250mv: adc-channel@d {
                 reg = <0x00 0x0d>;
+                label = "ref_1250mv";
             };
             ref_325mv: adc-channel@e {
                 reg = <0x00 0x0e>;
+                label = "ref_325mv";
             };
             ref_muxoff: adc-channel@f {
                 reg = <0x00 0x0f>;
+                label = "ref_muxoff";
             };
         };
     };
diff --git a/Documentation/devicetree/bindings/iio/adc/st,spear600-adc.yaml b/Documentation/devicetree/bindings/iio/adc/st,spear600-adc.yaml
index dd9ec30387033..c8b8f28541ce2 100644
--- a/Documentation/devicetree/bindings/iio/adc/st,spear600-adc.yaml
+++ b/Documentation/devicetree/bindings/iio/adc/st,spear600-adc.yaml
@@ -65,5 +65,5 @@ examples:
         interrupt-parent = <&vic1>;
         interrupts = <6>;
         sampling-frequency = <5000000>;
-        vref-external = <2500>;	/* 2.5V VRef */
+        vref-external = <2500>; /* 2.5V VRef */
     };
diff --git a/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml b/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml
index 40d87346ff4cf..a5b0debe85b1a 100644
--- a/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml
+++ b/Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml
@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
 title: Sensirion SCD30 carbon dioxide sensor
 
 maintainers:
-  - Tomasz Duszynski <tomasz.duszynski@octakon.com>
+  - Maxwell Doose <m32285159@gmail.com>
 
 description: |
   Air quality sensor capable of measuring co2 concentration, temperature
diff --git a/Documentation/devicetree/bindings/iio/dac/adi,ad5706r.yaml b/Documentation/devicetree/bindings/iio/dac/adi,ad5706r.yaml
new file mode 100644
index 0000000000000..19cc744a9f0fc
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/dac/adi,ad5706r.yaml
@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/dac/adi,ad5706r.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Analog Devices AD5706R 4-Channel Current Output DAC
+
+maintainers:
+  - Alexis Czezar Torreno <alexisczezar.torreno@analog.com>
+
+description: |
+  The AD5706R is a 4-channel, 16-bit resolution, current output
+  digital-to-analog converter (DAC) with programmable output current
+  ranges (50mA, 150mA, 200mA, 300mA), an integrated 2.5V voltage
+  reference, and load DAC, A/B toggle, and dither functions.
+
+  Datasheet:
+    https://www.analog.com/en/products/ad5706r.html
+
+properties:
+  compatible:
+    enum:
+      - adi,ad5706r
+
+  reg:
+    maxItems: 1
+
+  avdd-supply:
+    description: Analog power supply (2.9V to 3.6V).
+
+  iovdd-supply:
+    description: Logic power supply (1.14V to 1.89V).
+
+  pvdd0-supply:
+    description: Power supply for IDAC0 channel (1.65V to AVDD).
+
+  pvdd1-supply:
+    description: Power supply for IDAC1 channel (1.65V to AVDD).
+
+  pvdd2-supply:
+    description: Power supply for IDAC2 channel (1.65V to AVDD).
+
+  pvdd3-supply:
+    description: Power supply for IDAC3 channel (1.65V to AVDD).
+
+  vref-supply:
+    description:
+      Optional external 2.5V voltage reference. If not provided, the
+      internal 2.5V reference is used.
+
+  pwms:
+    maxItems: 1
+    description:
+      Optional PWM connected to the LDAC/TGP/DCK pin for hardware
+      triggered DAC updates, toggle, or dither clock generation.
+
+  reset-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the active low RESET pin. If not provided,
+      software reset is used.
+
+  enable-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the active low OUT_EN pin. Controls whether
+      the current outputs are enabled or in high-Z/ground state.
+
+required:
+  - compatible
+  - reg
+  - avdd-supply
+  - iovdd-supply
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        dac@0 {
+            compatible = "adi,ad5706r";
+            reg = <0>;
+            avdd-supply = <&avdd>;
+            iovdd-supply = <&iovdd>;
+            pvdd0-supply = <&pvdd>;
+            pvdd1-supply = <&pvdd>;
+            pvdd2-supply = <&pvdd>;
+            pvdd3-supply = <&pvdd>;
+            vref-supply = <&vref>;
+            spi-max-frequency = <50000000>;
+            pwms = <&pwm0 0 1000000 0>;
+            reset-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>;
+            enable-gpios = <&gpio0 12 GPIO_ACTIVE_LOW>;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml b/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml
index d2466aa6bda21..d131f136bd151 100644
--- a/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml
+++ b/Documentation/devicetree/bindings/iio/dac/microchip,mcp47feb02.yaml
@@ -161,8 +161,8 @@ patternProperties:
     properties:
       reg:
         description: The channel number.
-        minItems: 1
-        maxItems: 8
+        minimum: 0
+        maximum: 7
 
       label:
         description: Unique name to identify which channel this is.
@@ -280,23 +280,23 @@ examples:
 
         #address-cells = <1>;
         #size-cells = <0>;
-        dac@0 {
-          compatible = "microchip,mcp47feb02";
-          reg = <0>;
-          vdd-supply = <&vdac_vdd>;
-          vref-supply = <&vref_reg>;
+        dac@60 {
+            compatible = "microchip,mcp47feb02";
+            reg = <0x60>;
+            vdd-supply = <&vdac_vdd>;
+            vref-supply = <&vref_reg>;
 
-          #address-cells = <1>;
-          #size-cells = <0>;
-          channel@0 {
-            reg = <0>;
-            label = "Adjustable_voltage_ch0";
-          };
+            #address-cells = <1>;
+            #size-cells = <0>;
+            channel@0 {
+                reg = <0>;
+                label = "Adjustable_voltage_ch0";
+            };
 
-          channel@1 {
-            reg = <0x1>;
-            label = "Adjustable_voltage_ch1";
-          };
-      };
+            channel@1 {
+                reg = <0x1>;
+                label = "Adjustable_voltage_ch1";
+            };
+        };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml b/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml
index fcbd4b430e48e..8e094651381c8 100644
--- a/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml
+++ b/Documentation/devicetree/bindings/iio/gyroscope/bosch,bmg160.yaml
@@ -26,6 +26,9 @@ properties:
   vdd-supply: true
   vddio-supply: true
 
+  mount-matrix:
+    description: an optional 3x3 mounting rotation matrix.
+
   spi-max-frequency:
     maximum: 10000000
 
@@ -56,6 +59,9 @@ examples:
             reg = <0x69>;
             interrupt-parent = <&gpio6>;
             interrupts = <18 IRQ_TYPE_EDGE_RISING>;
+            mount-matrix = "0", "1", "0",
+                           "1", "0", "0",
+                           "0", "0", "1";
         };
     };
 ...
diff --git a/Documentation/devicetree/bindings/iio/light/tsl2772.yaml b/Documentation/devicetree/bindings/iio/light/tsl2772.yaml
index d812298579444..9921ccaa64a06 100644
--- a/Documentation/devicetree/bindings/iio/light/tsl2772.yaml
+++ b/Documentation/devicetree/bindings/iio/light/tsl2772.yaml
@@ -26,6 +26,8 @@ properties:
       - amstaos,tmd2672
       - amstaos,tsl2772
       - amstaos,tmd2772
+      - avago,apds9900
+      - avago,apds9901
       - avago,apds9930
 
   reg:
diff --git a/Documentation/devicetree/bindings/iio/magnetometer/memsic,mmc5983.yaml b/Documentation/devicetree/bindings/iio/magnetometer/memsic,mmc5983.yaml
new file mode 100644
index 0000000000000..4a4df6bb70fea
--- /dev/null
+++ b/Documentation/devicetree/bindings/iio/magnetometer/memsic,mmc5983.yaml
@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iio/magnetometer/memsic,mmc5983.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MEMSIC MMC5983MA 3-axis magnetic sensor
+
+maintainers:
+  - Vladislav Kulikov <vlad.kulikov.c@gmail.com>
+
+properties:
+  compatible:
+    const: memsic,mmc5983
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  vdd-supply:
+    description: Regulator that provides power to the sensor
+
+  vddio-supply:
+    description: Regulator that provides power to the digital interface and INT pin
+
+required:
+  - compatible
+  - reg
+  - vdd-supply
+
+allOf:
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        magnetometer@30 {
+            compatible = "memsic,mmc5983";
+            reg = <0x30>;
+            vdd-supply = <&vdd_3v3_reg>;
+            vddio-supply = <&vdd_3v3_reg>;
+        };
+    };
+  - |
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        magnetometer@0 {
+            compatible = "memsic,mmc5983";
+            reg = <0>;
+            spi-max-frequency = <10000000>;
+            vdd-supply = <&vdd_3v3_reg>;
+            vddio-supply = <&vdd_3v3_reg>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/input/atmel,maxtouch.yaml b/Documentation/devicetree/bindings/input/atmel,maxtouch.yaml
index 9bf07acea5999..26ea78df27c4c 100644
--- a/Documentation/devicetree/bindings/input/atmel,maxtouch.yaml
+++ b/Documentation/devicetree/bindings/input/atmel,maxtouch.yaml
@@ -88,8 +88,7 @@ properties:
       - 2 # ATMEL_MXT_WAKEUP_GPIO
     default: 0
 
-  wakeup-source:
-    type: boolean
+  wakeup-source: true
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/input/imx-keypad.yaml b/Documentation/devicetree/bindings/input/imx-keypad.yaml
index b110eb1f3358a..175256cb1295a 100644
--- a/Documentation/devicetree/bindings/input/imx-keypad.yaml
+++ b/Documentation/devicetree/bindings/input/imx-keypad.yaml
@@ -66,20 +66,20 @@ examples:
         clocks = <&clks 0>;
         pinctrl-names = "default";
         pinctrl-0 = <&pinctrl_kpp_1>;
-        linux,keymap = <0x00000067	/* KEY_UP */
-                        0x0001006c	/* KEY_DOWN */
-                        0x00020072	/* KEY_VOLUMEDOWN */
-                        0x00030066	/* KEY_HOME */
-                        0x0100006a	/* KEY_RIGHT */
-                        0x01010069	/* KEY_LEFT */
-                        0x0102001c	/* KEY_ENTER */
-                        0x01030073	/* KEY_VOLUMEUP */
-                        0x02000040	/* KEY_F6 */
-                        0x02010042	/* KEY_F8 */
-                        0x02020043	/* KEY_F9 */
-                        0x02030044	/* KEY_F10 */
-                        0x0300003b	/* KEY_F1 */
-                        0x0301003c	/* KEY_F2 */
-                        0x0302003d	/* KEY_F3 */
-                        0x03030074>;	/* KEY_POWER */
+        linux,keymap = <0x00000067 /* KEY_UP */
+                        0x0001006c /* KEY_DOWN */
+                        0x00020072 /* KEY_VOLUMEDOWN */
+                        0x00030066 /* KEY_HOME */
+                        0x0100006a /* KEY_RIGHT */
+                        0x01010069 /* KEY_LEFT */
+                        0x0102001c /* KEY_ENTER */
+                        0x01030073 /* KEY_VOLUMEUP */
+                        0x02000040 /* KEY_F6 */
+                        0x02010042 /* KEY_F8 */
+                        0x02020043 /* KEY_F9 */
+                        0x02030044 /* KEY_F10 */
+                        0x0300003b /* KEY_F1 */
+                        0x0301003c /* KEY_F2 */
+                        0x0302003d /* KEY_F3 */
+                        0x03030074>; /* KEY_POWER */
     };
diff --git a/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml b/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml
index 7ade03f1b32b8..2d762193f1c07 100644
--- a/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml
+++ b/Documentation/devicetree/bindings/input/microchip,cap11xx.yaml
@@ -197,12 +197,12 @@ examples:
         microchip,sensitivity-delta-sense = <16>;
         microchip,input-threshold = <21>, <18>, <46>, <46>, <46>, <21>;
 
-        linux,keycodes = <103>,	/* KEY_UP */
-                         <106>,	/* KEY_RIGHT */
-                         <108>,	/* KEY_DOWN */
-                         <105>,	/* KEY_LEFT */
-                         <109>,	/* KEY_PAGEDOWN */
-                         <104>;	/* KEY_PAGEUP */
+        linux,keycodes = <103>, /* KEY_UP */
+                         <106>, /* KEY_RIGHT */
+                         <108>, /* KEY_DOWN */
+                         <105>, /* KEY_LEFT */
+                         <109>, /* KEY_PAGEDOWN */
+                         <104>; /* KEY_PAGEUP */
 
         #address-cells = <1>;
         #size-cells = <0>;
diff --git a/Documentation/devicetree/bindings/input/touchscreen/st,stmfts.yaml b/Documentation/devicetree/bindings/input/touchscreen/st,stmfts.yaml
index 12256ae7df90d..64c4f24ea3dd0 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/st,stmfts.yaml
+++ b/Documentation/devicetree/bindings/input/touchscreen/st,stmfts.yaml
@@ -40,6 +40,10 @@ properties:
   vdd-supply:
     description: Power supply
 
+  reset-gpios:
+    description: Reset GPIO (active-low)
+    maxItems: 1
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/input/touchscreen/ti,ads7843.yaml b/Documentation/devicetree/bindings/input/touchscreen/ti,ads7843.yaml
index 8f6335d7da1c5..1b58fc263dce8 100644
--- a/Documentation/devicetree/bindings/input/touchscreen/ti,ads7843.yaml
+++ b/Documentation/devicetree/bindings/input/touchscreen/ti,ads7843.yaml
@@ -165,9 +165,9 @@ examples:
 
         touchscreen@0 {
             compatible = "ti,tsc2046";
-            reg = <0>;	/* CS0 */
+            reg = <0>; /* CS0 */
             interrupt-parent = <&gpio1>;
-            interrupts = <8 0>;	/* BOOT6 / GPIO 8 */
+            interrupts = <8 0>; /* BOOT6 / GPIO 8 */
             pendown-gpio = <&gpio1 8 0>;
             spi-max-frequency = <1000000>;
             vcc-supply = <&reg_vcc3>;
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,hawi-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,hawi-rpmh.yaml
new file mode 100644
index 0000000000000..49a2dca5db62a
--- /dev/null
+++ b/Documentation/devicetree/bindings/interconnect/qcom,hawi-rpmh.yaml
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interconnect/qcom,hawi-rpmh.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm RPMh Network-On-Chip Interconnect on Hawi
+
+maintainers:
+  - Vivek Aknurwar <vivek.aknurwar@oss.qualcomm.com>
+
+description: |
+  RPMh interconnect providers support system bandwidth requirements through
+  RPMh hardware accelerators known as Bus Clock Manager (BCM). The provider is
+  able to communicate with the BCM through the Resource State Coordinator (RSC)
+  associated with each execution environment. Provider nodes must point to at
+  least one RPMh device child node pertaining to their RSC and each provider
+  can map to multiple RPMh resources.
+
+  See also: include/dt-bindings/interconnect/qcom,hawi-rpmh.h
+
+properties:
+  compatible:
+    enum:
+      - qcom,hawi-aggre1-noc
+      - qcom,hawi-clk-virt
+      - qcom,hawi-cnoc-main
+      - qcom,hawi-gem-noc
+      - qcom,hawi-llclpi-noc
+      - qcom,hawi-lpass-ag-noc
+      - qcom,hawi-lpass-lpiaon-noc
+      - qcom,hawi-lpass-lpicx-noc
+      - qcom,hawi-mc-virt
+      - qcom,hawi-mmss-noc
+      - qcom,hawi-nsp-noc
+      - qcom,hawi-pcie-anoc
+      - qcom,hawi-stdst-cfg
+      - qcom,hawi-stdst-main
+      - qcom,hawi-system-noc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 2
+    maxItems: 3
+
+required:
+  - compatible
+
+allOf:
+  - $ref: qcom,rpmh-common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,hawi-clk-virt
+              - qcom,hawi-mc-virt
+    then:
+      properties:
+        reg: false
+    else:
+      required:
+        - reg
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,hawi-pcie-anoc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre-NOC PCIe AXI clock
+            - description: cfg-NOC PCIe a-NOC AHB clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,hawi-aggre1-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre UFS PHY AXI clock
+            - description: aggre USB3 PRIM AXI clock
+            - description: RPMH CC IPA clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,hawi-aggre1-noc
+              - qcom,hawi-pcie-anoc
+    then:
+      required:
+        - clocks
+    else:
+      properties:
+        clocks: false
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    soc {
+      #address-cells = <2>;
+      #size-cells = <2>;
+
+      clk_virt: interconnect-0 {
+        compatible = "qcom,hawi-clk-virt";
+        #interconnect-cells = <2>;
+        qcom,bcm-voters = <&apps_bcm_voter>;
+      };
+
+      aggre_noc: interconnect@f00000 {
+        compatible = "qcom,hawi-aggre1-noc";
+        reg = <0x0 0xf00000 0x0 0x54400>;
+        #interconnect-cells = <2>;
+        clocks = <&gcc_aggre_ufs_phy_axi_clk>,
+                 <&gcc_aggre_usb3_prim_axi_clk>,
+                 <&rpmhcc_ipa_clk>;
+        qcom,bcm-voters = <&apps_bcm_voter>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
index ce79521bb1ef2..82b1d94d3010a 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,msm8998-bwmon.yaml
@@ -26,6 +26,7 @@ properties:
       - items:
           - enum:
               - qcom,glymur-cpu-bwmon
+              - qcom,hawi-cpu-bwmon
               - qcom,kaanapali-cpu-bwmon
               - qcom,qcm2290-cpu-bwmon
               - qcom,qcs615-cpu-bwmon
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,nord-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,nord-rpmh.yaml
new file mode 100644
index 0000000000000..3650d3d5b9185
--- /dev/null
+++ b/Documentation/devicetree/bindings/interconnect/qcom,nord-rpmh.yaml
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interconnect/qcom,nord-rpmh.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm RPMh Network-On-Chip Interconnect on Nord
+
+maintainers:
+  - Odelu Kukatla <odelu.kukatla@oss.qualcomm.com>
+
+description: |
+  RPMh interconnect providers support system bandwidth requirements through
+  RPMh hardware accelerators known as Bus Clock Manager (BCM). The provider is
+  able to communicate with the BCM through the Resource State Coordinator (RSC)
+  associated with each execution environment. Provider nodes must point to at
+  least one RPMh device child node pertaining to their RSC and each provider
+  can map to multiple RPMh resources.
+
+  See also: include/dt-bindings/interconnect/qcom,nord-rpmh.h
+
+properties:
+  compatible:
+    enum:
+      - qcom,nord-aggre1-noc
+      - qcom,nord-aggre1-noc-tile
+      - qcom,nord-aggre2-noc
+      - qcom,nord-aggre2-noc-tile
+      - qcom,nord-clk-virt
+      - qcom,nord-cnoc-cfg
+      - qcom,nord-cnoc-main
+      - qcom,nord-hpass-ag-noc
+      - qcom,nord-hscnoc
+      - qcom,nord-mc-virt
+      - qcom,nord-mmss-noc
+      - qcom,nord-nsp-data-noc-0
+      - qcom,nord-nsp-data-noc-1
+      - qcom,nord-nsp-data-noc-2
+      - qcom,nord-nsp-data-noc-3
+      - qcom,nord-pcie-cfg
+      - qcom,nord-pcie-data-inbound
+      - qcom,nord-pcie-data-outbound
+      - qcom,nord-system-noc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 1
+    maxItems: 4
+
+required:
+  - compatible
+
+allOf:
+  - $ref: qcom,rpmh-common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,nord-clk-virt
+              - qcom,nord-mc-virt
+    then:
+      properties:
+        reg: false
+    else:
+      required:
+        - reg
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,nord-aggre1-noc-tile
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre UFS PHY AXI clock
+            - description: aggre USB2 AXI clock
+            - description: aggre USB3 PRIM AXI clock
+            - description: aggre USB3 SEC AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,nord-aggre2-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: RPMH CC IPA clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,nord-aggre1-noc-tile
+              - qcom,nord-aggre2-noc
+    then:
+      required:
+        - clocks
+    else:
+      properties:
+        clocks: false
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    clk_virt: interconnect-clk-virt {
+      compatible = "qcom,nord-clk-virt";
+      #interconnect-cells = <2>;
+      qcom,bcm-voters = <&apps_bcm_voter>;
+    };
+
+    aggre1_noc_tile: interconnect@1720000 {
+      compatible = "qcom,nord-aggre1-noc-tile";
+      reg = <0x01720000 0x23400>;
+      #interconnect-cells = <2>;
+      qcom,bcm-voters = <&apps_bcm_voter>;
+      clocks = <&ne_gcc_aggre_noc_ufs_phy_axi_clk>,
+               <&ne_gcc_aggre_noc_usb2_axi_clk>,
+               <&ne_gcc_aggre_noc_usb3_prim_axi_clk>,
+               <&ne_gcc_aggre_noc_usb3_sec_axi_clk>;
+    };
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,sdm660.yaml b/Documentation/devicetree/bindings/interconnect/qcom,sdm660.yaml
index 8f6bc6399626b..51428a2b0ce09 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,sdm660.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,sdm660.yaml
@@ -79,6 +79,19 @@ allOf:
             - const: aggre2_usb3_axi
             - const: cfg_noc_usb2_axi
 
+  - if:
+      properties:
+        compatible:
+          enum:
+            - qcom,sdm660-bimc
+            - qcom,sdm660-cnoc
+            - qcom,sdm660-gnoc
+            - qcom,sdm660-snoc
+    then:
+      properties:
+        clocks: false
+        clock-names: false
+
 examples:
   - |
     #include <dt-bindings/clock/qcom,gcc-sdm660.h>
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,shikra.yaml b/Documentation/devicetree/bindings/interconnect/qcom,shikra.yaml
new file mode 100644
index 0000000000000..a0c26de94ccf7
--- /dev/null
+++ b/Documentation/devicetree/bindings/interconnect/qcom,shikra.yaml
@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interconnect/qcom,shikra.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Shikra Network-On-Chip interconnect
+
+maintainers:
+  - Raviteja Laggyshetty <raviteja.laggyshetty@oss.qualcomm.com>
+
+description:
+  The Qualcomm Shikra interconnect providers support adjusting the
+  bandwidth requirements between the various NoC fabrics.
+
+properties:
+  compatible:
+    enum:
+      - qcom,shikra-config-noc
+      - qcom,shikra-mem-noc-core
+      - qcom,shikra-sys-noc
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    minItems: 1
+    maxItems: 4
+
+  clock-names:
+    minItems: 1
+    maxItems: 4
+
+# Child node's properties
+patternProperties:
+  '^interconnect-[a-z0-9]+$':
+    type: object
+    description:
+      The interconnect providers do not have a separate QoS register space,
+      but share parent's space.
+
+    $ref: qcom,rpm-common.yaml#
+
+    properties:
+      compatible:
+        enum:
+          - qcom,shikra-clk-virt
+          - qcom,shikra-mc-virt
+          - qcom,shikra-mmrt-virt
+          - qcom,shikra-mmnrt-virt
+
+    required:
+      - compatible
+
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+
+allOf:
+  - $ref: qcom,rpm-common.yaml#
+  - if:
+      properties:
+        compatible:
+          const: qcom,shikra-mem-noc-core
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: GPU-NoC AXI clock
+
+        clock-names:
+          items:
+            - const: gpu_axi
+      patternProperties:
+        '^interconnect-[a-z0-9]+$': false
+
+  - if:
+      properties:
+        compatible:
+          const: qcom,shikra-sys-noc
+
+    then:
+      properties:
+        clocks:
+          items:
+            - description: EMAC0-NoC AXI clock.
+            - description: EMAC1-NoC AXI clock.
+            - description: USB2-NoC AXI clock.
+            - description: USB3-NoC AXI clock.
+
+        clock-names:
+          items:
+            - const: emac0_axi
+            - const: emac1_axi
+            - const: usb2_axi
+            - const: usb3_axi
+
+  - if:
+      properties:
+        compatible:
+          const: qcom,shikra-config-noc
+
+    then:
+      properties:
+        clocks: false
+        clock-names: false
+      patternProperties:
+        '^interconnect-[a-z0-9]+$': false
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    interconnect@1880000 {
+        compatible = "qcom,shikra-sys-noc";
+        reg = <0x01880000 0x6a080>;
+        #interconnect-cells = <2>;
+        clocks = <&gcc_emac0_axi_sys_noc_clk>,
+                 <&gcc_emac1_axi_sys_noc_clk>,
+                 <&gcc_sys_noc_usb2_prim_axi_clk>,
+                 <&gcc_sys_noc_usb3_prim_axi_clk>;
+        clock-names = "emac0_axi",
+                      "emac1_axi",
+                      "usb2_axi",
+                      "usb3_axi";
+
+        interconnect-clk {
+            compatible = "qcom,shikra-clk-virt";
+            #interconnect-cells = <2>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,sm6115.yaml b/Documentation/devicetree/bindings/interconnect/qcom,sm6115.yaml
index 14b1a0b08e736..cdae0acf3a1da 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,sm6115.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,sm6115.yaml
@@ -62,23 +62,33 @@ allOf:
   - if:
       properties:
         compatible:
-          const: qcom,sm6115-cnoc
+          const: qcom,sm6115-bimc
+    then:
+      properties:
+        clocks: false
+        clock-names: false
+      patternProperties:
+        '^interconnect-[a-z0-9]+$': false
 
+  - if:
+      properties:
+        compatible:
+          const: qcom,sm6115-cnoc
     then:
       properties:
         clocks:
           items:
             - description: USB-NoC AXI clock
-
         clock-names:
           items:
             - const: usb_axi
+      patternProperties:
+        '^interconnect-[a-z0-9]+$': false
 
   - if:
       properties:
         compatible:
           const: qcom,sm6115-snoc
-
     then:
       properties:
         clocks:
@@ -87,7 +97,6 @@ allOf:
             - description: UFS-NoC AXI clock.
             - description: USB-NoC AXI clock.
             - description: IPA clock.
-
         clock-names:
           items:
             - const: cpu_axi
@@ -95,20 +104,6 @@ allOf:
             - const: usb_axi
             - const: ipa
 
-  - if:
-      properties:
-        compatible:
-          enum:
-            - qcom,sm6115-bimc
-            - qcom,sm6115-clk-virt
-            - qcom,sm6115-mmrt-virt
-            - qcom,sm6115-mmnrt-virt
-
-    then:
-      properties:
-        clocks: false
-        clock-names: false
-
 unevaluatedProperties: false
 
 examples:
@@ -149,4 +144,6 @@ examples:
         compatible = "qcom,sm6115-cnoc";
         reg = <0x01900000 0x8200>;
         #interconnect-cells = <1>;
+        clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>;
+        clock-names = "usb_axi";
     };
diff --git a/Documentation/devicetree/bindings/interconnect/qcom,x1e80100-rpmh.yaml b/Documentation/devicetree/bindings/interconnect/qcom,x1e80100-rpmh.yaml
index 0840b0ec6e276..27d9234bc762a 100644
--- a/Documentation/devicetree/bindings/interconnect/qcom,x1e80100-rpmh.yaml
+++ b/Documentation/devicetree/bindings/interconnect/qcom,x1e80100-rpmh.yaml
@@ -46,6 +46,10 @@ properties:
   reg:
     maxItems: 1
 
+  clocks:
+    minItems: 1
+    maxItems: 6
+
 required:
   - compatible
 
@@ -65,6 +69,63 @@ allOf:
       required:
         - reg
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,x1e80100-aggre1-noc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre UFS PHY AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,x1e80100-usb-north-anoc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre USB2 PRIM AXI clock
+            - description: aggre USB3 MP AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,x1e80100-usb-south-anoc
+    then:
+      properties:
+        clocks:
+          items:
+            - description: aggre USB3 PRIM AXI clock
+            - description: aggre USB3 SEC AXI clock
+            - description: aggre USB3 TERT AXI clock
+            - description: aggre USB4_0 AXI clock
+            - description: aggre USB4_1 AXI clock
+            - description: aggre USB4_2 AXI clock
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - qcom,x1e80100-aggre1-noc
+              - qcom,x1e80100-usb-north-anoc
+              - qcom,x1e80100-usb-south-anoc
+    then:
+      required:
+        - clocks
+    else:
+      properties:
+        clocks: false
+
 unevaluatedProperties: false
 
 examples:
@@ -80,4 +141,5 @@ examples:
       reg = <0x016e0000 0x14400>;
       #interconnect-cells = <2>;
       qcom,bcm-voters = <&apps_bcm_voter>;
+      clocks = <&gcc_aggre_ufs_phy_axi_clk>;
     };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml
index d0fad930de9d9..d26671913e895 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/amlogic,meson-gpio-intc.yaml
@@ -38,6 +38,8 @@ properties:
               - amlogic,a4-gpio-intc
               - amlogic,a4-gpio-ao-intc
               - amlogic,a5-gpio-intc
+              - amlogic,a9-gpio-intc
+              - amlogic,a9-gpio-ao-intc
               - amlogic,c3-gpio-intc
               - amlogic,s6-gpio-intc
               - amlogic,s7-gpio-intc
@@ -56,7 +58,7 @@ properties:
   amlogic,channel-interrupts:
     description: Array with the upstream hwirq numbers
     minItems: 2
-    maxItems: 12
+    maxItems: 20
     $ref: /schemas/types.yaml#/definitions/uint32-array
 
 required:
@@ -76,9 +78,20 @@ then:
     amlogic,channel-interrupts:
       maxItems: 2
 else:
-  properties:
-    amlogic,channel-interrupts:
-      minItems: 8
+  if:
+    properties:
+      compatible:
+        contains:
+          const: amlogic,a9-gpio-ao-intc
+  then:
+    properties:
+      amlogic,channel-interrupts:
+        minItems: 20
+  else:
+    properties:
+      amlogic,channel-interrupts:
+        minItems: 8
+        maxItems: 12
 
 additionalProperties: false
 
diff --git a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml
deleted file mode 100644
index 258d21fe6e359..0000000000000
--- a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml
+++ /dev/null
@@ -1,90 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/interrupt-controller/aspeed,ast2700-intc.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: Aspeed AST2700 Interrupt Controller
-
-description:
-  This interrupt controller hardware is second level interrupt controller that
-  is hooked to a parent interrupt controller. It's useful to combine multiple
-  interrupt sources into 1 interrupt to parent interrupt controller.
-
-maintainers:
-  - Kevin Chen <kevin_chen@aspeedtech.com>
-
-properties:
-  compatible:
-    enum:
-      - aspeed,ast2700-intc-ic
-
-  reg:
-    maxItems: 1
-
-  interrupt-controller: true
-
-  '#interrupt-cells':
-    const: 1
-    description:
-      The first cell is the IRQ number, the second cell is the trigger
-      type as defined in interrupt.txt in this directory.
-
-  interrupts:
-    minItems: 1
-    maxItems: 10
-    description: |
-      Depend to which INTC0 or INTC1 used.
-      INTC0 and INTC1 are two kinds of interrupt controller with enable and raw
-      status registers for use.
-      INTC0 is used to assert GIC if interrupt in INTC1 asserted.
-      INTC1 is used to assert INTC0 if interrupt of modules asserted.
-      +-----+   +-------+     +---------+---module0
-      | GIC |---| INTC0 |--+--| INTC1_0 |---module2
-      |     |   |       |  |  |         |---...
-      +-----+   +-------+  |  +---------+---module31
-                           |
-                           |   +---------+---module0
-                           +---| INTC1_1 |---module2
-                           |   |         |---...
-                           |   +---------+---module31
-                          ...
-                           |   +---------+---module0
-                           +---| INTC1_5 |---module2
-                               |         |---...
-                               +---------+---module31
-
-required:
-  - compatible
-  - reg
-  - interrupt-controller
-  - '#interrupt-cells'
-  - interrupts
-
-additionalProperties: false
-
-examples:
-  - |
-    #include <dt-bindings/interrupt-controller/arm-gic.h>
-
-    bus {
-        #address-cells = <2>;
-        #size-cells = <2>;
-
-        interrupt-controller@12101b00 {
-            compatible = "aspeed,ast2700-intc-ic";
-            reg = <0 0x12101b00 0 0x10>;
-            #interrupt-cells = <1>;
-            interrupt-controller;
-            interrupts = <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>,
-                         <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>;
-        };
-    };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-interrupt.yaml b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-interrupt.yaml
new file mode 100644
index 0000000000000..a62f0fd2435ba
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-interrupt.yaml
@@ -0,0 +1,188 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/aspeed,ast2700-interrupt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ASPEED AST2700 Interrupt Controllers (INTC0/INTC1)
+
+description: |
+  The ASPEED AST2700 SoC integrates two interrupt controller designs:
+
+    - INTC0: Primary controller that routes interrupt sources to upstream,
+      processor-specific interrupt controllers
+
+    - INTC1: Secondary controller whose interrupt outputs feed into INTC0
+
+  The SoC contains four processors to which interrupts can be routed:
+
+    - PSP: Primary Service Processor (Cortex-A35)
+    - SSP: Secondary Service Processor (Cortex-M4)
+    - TSP: Tertiary Service Processor (Cortex-M4)
+    - BMCU: Boot MCU (a RISC-V microcontroller)
+
+  The following diagram illustrates the overall architecture of the
+  ASPEED AST2700 interrupt controllers:
+
+                  +-----------+                +-----------+
+                  |   INTC0   |                | INTC1(0)  |
+                  +-----------+                +-----------+
+                  |   Router  | +-----------+  |   Router  |
+                  | out   int | +Peripheral +  | out   int |
+  +-----------+   |  0     0  <-+Controllers+  | INTM      | +-----------+
+  |PSP GIC  <-|---+  .     .  | +-----------+  |  .     .  <-+Peripheral +
+  +-----------+   |  .     .  |                |  .     .  | +Controllers+
+  +-----------+   |  .     .  |                |  .     .  | +-----------+
+  |SSP NVIC <-|---+  .     .  <----------------+  .     .  |
+  +-----------+   |  .     .  |                |  .     .  |
+  +-----------+   |  .     .  <--------        |  .     .  |
+  |TSP NVIC <-|---+  .     .  |       |    ----+  .     .  |
+  +-----------+   |  .     .  |       |    |   |  O     P  |
+                  |  .     .  |       |    |   +-----------+
+                  |  .     .  <----   |    --------------------
+                  |  .     .  |   |   |        +-----------+  |
+                  |  M     N  |   |   ---------+  INTC1(1) |  |
+                  +-----------+   |            +-----------+  |
+                                  |                  .        |
+                                  |            +-----------+  |
+                                  -------------+  INTC1(N) |  |
+                                               +-----------+  |
+  +--------------+                                            |
+  + BMCU APLIC <-+---------------------------------------------
+  +--------------+
+
+  INTC0 supports:
+    - 128 local peripheral interrupt inputs
+    - Fan-in from up to three INTC1 instances via banked interrupt lines (INTM)
+    - Local peripheral interrupt outputs
+    - Merged interrupt outputs
+    - Software interrupt outputs (SWINT)
+    - Configurable interrupt routes targeting the PSP, SSP, and TSP
+
+  INTC1 supports:
+    - 192 local peripheral interrupt inputs
+    - Banked interrupt outputs (INTM, 5 x 6 banks x 32 interrupts per bank)
+    - Configurable interrupt routes targeting the PSP, SSP, TSP, and BMCU
+
+  One INTC1 instance is always present, on the SoC's IO die. A further two
+  instances may be attached to the SoC's one INTC0 instance via LTPI (LVDS
+  Tunneling Protocol & Interface).
+
+  Interrupt numbering model
+  -------------------------
+  The binding uses a controller-local numbering model. Peripheral device
+  nodes use the INTCx local interrupt number (hwirq) in their 'interrupts' or
+  'interrupts-extended' properties.
+
+  For AST2700, INTC0 exposes the following (inclusive) input ranges:
+
+    - 000..479: Independent interrupts
+    - 480..489: INTM0-INTM9
+    - 490..499: INTM10-INTM19
+    - 500..509: INTM20-INTM29
+    - 510..519: INTM30-INTM39
+    - 520..529: INTM40-INTM49
+
+  INTC0's (inclusive) output ranges are as follows:
+
+    - 000..127: 1:1 local peripheral interrupt output to PSP
+    - 144..151: Software interrupts from the SSP output to PSP
+    - 152..159: Software interrupts from the TSP output to PSP
+    - 192..201: INTM0-INTM9 banked outputs to PSP
+    - 208..217: INTM30-INTM39 banked outputs to PSP
+    - 224..233: INTM40-INTM49 banked outputs to PSP
+    - 256..383: 1:1 local peripheral interrupt output to SSP
+    - 384..393: INTM10-INTM19 banked outputs to SSP
+    - 400..407: Software interrupts from the PSP output to SSP
+    - 408..415: Software interrupts from the TSP output to SSP
+    - 426..553: 1:1 local peripheral interrupt output to TSP
+    - 554..563: INTM20-INTM29 banked outputs to TSP
+    - 570..577: Software interrupts from the PSP output to TSP
+    - 578..585: Software interrupts from the SSP output to TSP
+
+  Inputs and outputs for INTC1 instances are context-dependent. However, for the
+  first instance of INTC1, the (inclusive) output ranges are:
+
+    - 00..05: INTM0-INTM5
+    - 10..15: INTM10-INTM15
+    - 20..25: INTM20-INTM25
+    - 30..35: INTM30-INTM35
+    - 40..45: INTM40-INTM45
+    - 50..50: BootMCU
+
+maintainers:
+  - Ryan Chen <ryan_chen@aspeedtech.com>
+  - Andrew Jeffery <andrew@codeconstruct.com.au>
+
+properties:
+  compatible:
+    enum:
+      - aspeed,ast2700-intc0
+      - aspeed,ast2700-intc1
+
+  reg:
+    maxItems: 1
+
+  interrupt-controller: true
+
+  '#interrupt-cells':
+    const: 1
+    description: Single cell encoding the INTC local interrupt number (hwirq).
+
+  aspeed,interrupt-ranges:
+    description: |
+      Describes how ranges of controller output pins are routed to a parent
+      interrupt controller.
+
+      Each range entry is encoded as:
+
+        <out count phandle parent-specifier...>
+
+      where:
+        - out:     First controller interrupt output index in the range.
+        - count:   Number of consecutive controller interrupt outputs and parent
+                   interrupt inputs in this range.
+        - phandle: Phandle to the parent interrupt controller node.
+        - parent-specifier: Interrupt specifier, as defined by the parent
+                            interrupt controller binding.
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    minItems: 3
+    items:
+      description: Range descriptors with a parent interrupt specifier.
+
+required:
+  - compatible
+  - reg
+  - interrupt-controller
+  - '#interrupt-cells'
+  - aspeed,interrupt-ranges
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    interrupt-controller@12100000 {
+        compatible = "aspeed,ast2700-intc0";
+        reg = <0x12100000 0x3b00>;
+        interrupt-parent = <&gic>;
+        interrupt-controller;
+        #interrupt-cells = <1>;
+
+        aspeed,interrupt-ranges =
+          <0 128 &gic GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+          <144 8 &gic GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+          <152 8 &gic GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>,
+          <192 10 &gic GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+          <208 10 &gic GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>,
+          <224 10 &gic GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>,
+          <256 128 &ssp_nvic 0 0>,
+          <384 10 &ssp_nvic 160 0>,
+          <400 8 &ssp_nvic 144 0>,
+          <408 8 &ssp_nvic 152 0>,
+          <426 128 &tsp_nvic 0 0>,
+          <554 10 &tsp_nvic 160 0>,
+          <570 8 &tsp_nvic 144 0>,
+          <578 8 &tsp_nvic 152 0>;
+    };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/econet,en751221-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/econet,en751221-intc.yaml
index 5536319c49c31..44c09785e6bb6 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/econet,en751221-intc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/econet,en751221-intc.yaml
@@ -52,6 +52,25 @@ properties:
         - description: primary per-CPU IRQ
         - description: shadow IRQ number
 
+  econet,cpu-interrupt-map:
+    $ref: /schemas/types.yaml#/definitions/uint32-matrix
+    description:
+      When running in VEIC mode, the hardware re-routes interrupts from the
+      CPU interrupt controller core to the "external" interrupt controller
+      (this device). It then prioritizes them and sends them back to the CPU
+      along with its own interrupts. The CPU hardware handles interrupts using
+      a special dispatch table (the normal interrupt handler is not invoked).
+      In this interrupt controller, the CPU interrupts are renumbered as they
+      are merged with this controller's own hardware interrupts.
+
+      This is the inverse of an interrupt-map, mapping which interrupts from
+      this controller must be routed back to the CPU interrupt domain for
+      correct handling there.
+    items:
+      items:
+        - description: The interrupt number as received in this controller
+        - description: The interrupt number to be dispatched on the CPU intc
+
 required:
   - compatible
   - reg
@@ -74,5 +93,6 @@ examples:
         interrupts = <2>;
 
         econet,shadow-interrupts = <7 2>, <8 3>, <13 12>, <30 29>;
+        econet,cpu-interrupt-map = <7 0>, <8 1>;
     };
 ...
diff --git a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
index b4942881b9c96..07a46c5457a49 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/qcom,pdc.yaml
@@ -31,6 +31,7 @@ properties:
           - qcom,hawi-pdc
           - qcom,kaanapali-pdc
           - qcom,milos-pdc
+          - qcom,nord-pdc
           - qcom,qcs615-pdc
           - qcom,qcs8300-pdc
           - qcom,qdu1000-pdc
diff --git a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
index e0267223887ec..639bbeb1f6bdf 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
+++ b/Documentation/devicetree/bindings/interrupt-controller/sifive,plic-1.0.0.yaml
@@ -71,6 +71,7 @@ properties:
       - items:
           - enum:
               - allwinner,sun20i-d1-plic
+              - canaan,k230-plic
               - sophgo,cv1800b-plic
               - sophgo,cv1812h-plic
               - sophgo,sg2002-plic
diff --git a/Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml
deleted file mode 100644
index ada5788602d65..0000000000000
--- a/Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml
+++ /dev/null
@@ -1,61 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
-%YAML 1.2
----
-$id: http://devicetree.org/schemas/interrupt-controller/starfive,jh8100-intc.yaml#
-$schema: http://devicetree.org/meta-schemas/core.yaml#
-
-title: StarFive External Interrupt Controller
-
-description:
-  StarFive SoC JH8100 contain a external interrupt controller. It can be used
-  to handle high-level input interrupt signals. It also send the output
-  interrupt signal to RISC-V PLIC.
-
-maintainers:
-  - Changhuang Liang <changhuang.liang@starfivetech.com>
-
-properties:
-  compatible:
-    const: starfive,jh8100-intc
-
-  reg:
-    maxItems: 1
-
-  clocks:
-    description: APB clock for the interrupt controller
-    maxItems: 1
-
-  resets:
-    description: APB reset for the interrupt controller
-    maxItems: 1
-
-  interrupts:
-    maxItems: 1
-
-  interrupt-controller: true
-
-  "#interrupt-cells":
-    const: 1
-
-required:
-  - compatible
-  - reg
-  - clocks
-  - resets
-  - interrupts
-  - interrupt-controller
-  - "#interrupt-cells"
-
-additionalProperties: false
-
-examples:
-  - |
-    interrupt-controller@12260000 {
-      compatible = "starfive,jh8100-intc";
-      reg = <0x12260000 0x10000>;
-      clocks = <&syscrg_ne 76>;
-      resets = <&syscrg_ne 13>;
-      interrupts = <45>;
-      interrupt-controller;
-      #interrupt-cells = <1>;
-    };
diff --git a/Documentation/devicetree/bindings/interrupt-controller/starfive,jhb100-intc.yaml b/Documentation/devicetree/bindings/interrupt-controller/starfive,jhb100-intc.yaml
new file mode 100644
index 0000000000000..d8a0a3862ae2c
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/starfive,jhb100-intc.yaml
@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/interrupt-controller/starfive,jhb100-intc.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: StarFive External Interrupt Controller
+
+description:
+  StarFive SoC JHB100 contain a external interrupt controller. It can be used
+  to handle high-level input interrupt signals. It also send the output
+  interrupt signal to RISC-V PLIC.
+
+maintainers:
+  - Changhuang Liang <changhuang.liang@starfivetech.com>
+
+properties:
+  compatible:
+    const: starfive,jhb100-intc
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  interrupt-controller: true
+
+  "#interrupt-cells":
+    const: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - interrupt-controller
+  - "#interrupt-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    interrupt-controller@12260000 {
+      compatible = "starfive,jhb100-intc";
+      reg = <0x12260000 0x10000>;
+      interrupts = <45>;
+      interrupt-controller;
+      #interrupt-cells = <1>;
+    };
diff --git a/Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml b/Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml
new file mode 100644
index 0000000000000..d3ce9e603b61d
--- /dev/null
+++ b/Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/iommu/verisilicon,iommu.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Verisilicon IOMMU
+
+maintainers:
+  - Benjamin Gaignard <benjamin.gaignard@collabora.com>
+
+description: |+
+  A Versilicon iommu translates io virtual addresses to physical addresses for
+  its associated video decoder.
+
+properties:
+  compatible:
+    items:
+      - const: rockchip,rk3588-av1-iommu
+      - const: verisilicon,iommu-1.2
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: Core clock
+      - description: Interface clock
+
+  clock-names:
+    items:
+      - const: core
+      - const: iface
+
+  "#iommu-cells":
+    const: 0
+
+  power-domains:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - clocks
+  - clock-names
+  - "#iommu-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/rockchip,rk3588-cru.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    bus {
+      #address-cells = <2>;
+      #size-cells = <2>;
+
+      iommu@fdca0000 {
+        compatible = "rockchip,rk3588-av1-iommu","verisilicon,iommu-1.2";
+        reg = <0x0 0xfdca0000 0x0 0x600>;
+        interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH 0>;
+        clocks = <&cru ACLK_AV1>, <&cru PCLK_AV1>;
+        clock-names = "core", "iface";
+        #iommu-cells = <0>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/leds/backlight/maxim,max25014.yaml b/Documentation/devicetree/bindings/leds/backlight/maxim,max25014.yaml
new file mode 100644
index 0000000000000..d00be2e081938
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/backlight/maxim,max25014.yaml
@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/leds/backlight/maxim,max25014.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Maxim max25014 backlight controller
+
+maintainers:
+  - Maud Spierings <maudspierings@gocontroll.com>
+
+properties:
+  compatible:
+    enum:
+      - maxim,max25014
+
+  reg:
+    maxItems: 1
+
+  default-brightness:
+    minimum: 0
+    maximum: 100
+    default: 50
+
+  enable-gpios:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  power-supply:
+    description: Regulator which controls the boost converter input rail.
+
+  pwms:
+    maxItems: 1
+
+  maxim,iset:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    maximum: 15
+    default: 11
+    description:
+      Value of the ISET field in the ISET register. This controls the current
+      scale of the outputs, a higher number means more current.
+
+  maxim,strings:
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    description:
+      A 4-bit bitfield that describes which led strings to turn on.
+    minItems: 4
+    maxItems: 4
+    items:
+      maximum: 1
+    default:
+      [1 1 1 1]
+
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        backlight@6f {
+            compatible = "maxim,max25014";
+            reg = <0x6f>;
+            default-brightness = <50>;
+            enable-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>;
+            interrupt-parent = <&gpio1>;
+            interrupts = <2 IRQ_TYPE_EDGE_FALLING>;
+            power-supply = <&reg_backlight>;
+            pwms = <&pwm1>;
+            maxim,iset = <7>;
+            maxim,strings = <1 1 1 0>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/leds/irled/ir-spi-led.yaml b/Documentation/devicetree/bindings/leds/irled/ir-spi-led.yaml
index 72cadebf6e3e3..0297bfbb2750d 100644
--- a/Documentation/devicetree/bindings/leds/irled/ir-spi-led.yaml
+++ b/Documentation/devicetree/bindings/leds/irled/ir-spi-led.yaml
@@ -25,7 +25,7 @@ properties:
 
   duty-cycle:
     $ref: /schemas/types.yaml#/definitions/uint8
-    enum: [50, 60, 70, 75, 80, 90]
+    enum: [30, 50, 60, 70, 75, 80, 90]
     description:
       Percentage of one period in which the signal is active.
 
diff --git a/Documentation/devicetree/bindings/leds/leds-lp55xx.yaml b/Documentation/devicetree/bindings/leds/leds-lp55xx.yaml
index fe8aaecf30108..67637efac3783 100644
--- a/Documentation/devicetree/bindings/leds/leds-lp55xx.yaml
+++ b/Documentation/devicetree/bindings/leds/leds-lp55xx.yaml
@@ -183,7 +183,7 @@ examples:
             compatible = "ti,lp8501";
             reg = <0x32>;
             clock-mode = /bits/ 8 <2>;
-            pwr-sel = /bits/ 8 <3>;	/* D1~9 connected to VOUT */
+            pwr-sel = /bits/ 8 <3>; /* D1~9 connected to VOUT */
             ti,charge-pump-mode = <LP55XX_CP_BYPASS>;
 
             led@0 {
diff --git a/Documentation/devicetree/bindings/leds/ti,lm3560.yaml b/Documentation/devicetree/bindings/leds/ti,lm3560.yaml
new file mode 100644
index 0000000000000..6cf8cf91ab2e2
--- /dev/null
+++ b/Documentation/devicetree/bindings/leds/ti,lm3560.yaml
@@ -0,0 +1,163 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/leds/ti,lm3560.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI LM3560 Synchronous Boost Flash Driver
+
+maintainers:
+  - Svyatoslav Ryhel <clamor95@gmail.com>
+
+description:
+  The LM3560 is a 2-MHz fixed frequency synchronous boost converter with two
+  1000-mA constant current drivers for high-current white LEDs. The dual high-
+  side current sources allow for grounded cathode LED operation and can be
+  tied together for providing flash currents at up to 2 A through a single LED.
+  An adaptive regulation method ensures the current for each LED remains in
+  regulation and maximizes efficiency.
+
+properties:
+  compatible:
+    enum:
+      - ti,lm3559
+      - ti,lm3560
+
+  reg:
+    maxItems: 1
+
+  '#address-cells':
+    const: 1
+
+  '#size-cells':
+    const: 0
+
+  enable-gpios:
+    description: GPIO connected to the HWEN pin.
+    maxItems: 1
+
+  vin-supply:
+    description: Supply connected to the IN line.
+
+  flash-max-timeout-us:
+    minimum: 32000
+    maximum: 1024000
+    default: 32000
+
+  ti,peak-current-microamp:
+    description:
+      The LM3560 features 4 selectable current limits 1.6A, 2.3A, 3A, and 3.6A
+      (in case of LM3559 - 1.4A, 2.1A, 2.7A, and 3.2A). When the current limit
+      is reached, the LM3559/LM3560 stops switching for the remainder of the
+      switching cycle.
+
+patternProperties:
+  '^led@[01]$':
+    type: object
+    $ref: /schemas/leds/common.yaml#
+    description: LED control bank nodes.
+    unevaluatedProperties: false
+
+    properties:
+      reg:
+        description: Control bank selection (0 = bank A, 1 = bank B).
+        maximum: 1
+
+    required:
+      - reg
+      - flash-max-microamp
+      - led-max-microamp
+
+allOf:
+  - $ref: /schemas/leds/common.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,lm3559
+    then:
+      properties:
+        ti,peak-current-microamp:
+          enum: [1400000, 2100000, 2700000, 3200000]
+          default: 1400000
+      patternProperties:
+        '^led@[01]$':
+          properties:
+            flash-max-microamp:
+              minimum: 56250
+              maximum: 900000
+            led-max-microamp:
+              minimum: 28125
+              maximum: 225000
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: ti,lm3560
+    then:
+      properties:
+        ti,peak-current-microamp:
+          enum: [1600000, 2300000, 3000000, 3600000]
+          default: 1600000
+      patternProperties:
+        '^led@[01]$':
+          properties:
+            flash-max-microamp:
+              minimum: 62500
+              maximum: 1000000
+            led-max-microamp:
+              minimum: 31250
+              maximum: 250000
+
+required:
+  - compatible
+  - reg
+  - '#address-cells'
+  - '#size-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+    #include <dt-bindings/leds/common.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        led-controller@53 {
+            compatible = "ti,lm3560";
+            reg = <0x53>;
+
+            enable-gpios = <&gpio 28 GPIO_ACTIVE_HIGH>;
+            vin-supply = <&vdd_3v3_sys>;
+
+            flash-max-timeout-us = <1024000>;
+            ti,peak-current-microamp = <1600000>;
+
+            #address-cells = <1>;
+            #size-cells = <0>;
+
+            led@0 {
+                reg = <0>;
+
+                function = LED_FUNCTION_FLASH;
+                color = <LED_COLOR_ID_WHITE>;
+
+                flash-max-microamp = <562500>;
+                led-max-microamp = <156250>;
+            };
+
+            led@1 {
+                reg = <1>;
+
+                function = LED_FUNCTION_FLASH;
+                color = <LED_COLOR_ID_YELLOW>;
+
+                flash-max-microamp = <562500>;
+                led-max-microamp = <156250>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml
index f40dc90483272..1b4ef0688ca79 100644
--- a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml
+++ b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml
@@ -49,6 +49,7 @@ properties:
               - qcom,qcs615-apss-shared
               - qcom,sc7180-apss-shared
               - qcom,sc8180x-apss-shared
+              - qcom,shikra-apss-shared
               - qcom,sm7150-apss-shared
               - qcom,sm8150-apss-shared
           - const: qcom,sdm845-apss-shared
diff --git a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml
index 90bfde66cc4a6..03359479d926a 100644
--- a/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml
+++ b/Documentation/devicetree/bindings/mailbox/qcom,cpucp-mbox.yaml
@@ -19,7 +19,9 @@ properties:
       - items:
           - enum:
               - qcom,glymur-cpucp-mbox
+              - qcom,hawi-cpucp-mbox
               - qcom,kaanapali-cpucp-mbox
+              - qcom,nord-cpucp-mbox
               - qcom,sm8750-cpucp-mbox
           - const: qcom,x1e80100-cpucp-mbox
       - enum:
diff --git a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml
index f5c584cf2146d..fae20a6615f6b 100644
--- a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml
+++ b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml
@@ -26,6 +26,7 @@ properties:
       - enum:
           - qcom,eliza-ipcc
           - qcom,glymur-ipcc
+          - qcom,hawi-ipcc
           - qcom,kaanapali-ipcc
           - qcom,milos-ipcc
           - qcom,qcs8300-ipcc
diff --git a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml
index 541325f900a1d..01f2afa023f04 100644
--- a/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml
+++ b/Documentation/devicetree/bindings/media/allwinner,sun4i-a10-video-engine.yaml
@@ -63,6 +63,16 @@ properties:
       CMA pool to use for buffers allocation instead of the default
       CMA pool.
 
+  # FIXME: This should be made required eventually once every SoC will
+  # have the MBUS declared.
+  interconnects:
+    maxItems: 1
+
+  # FIXME: This should be made required eventually once every SoC will
+  # have the MBUS declared.
+  interconnect-names:
+    const: dma-mem
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/media/cnm,wave521c.yaml b/Documentation/devicetree/bindings/media/cnm,wave521c.yaml
index 6a11c1d11fb5f..6cd33dfd095d6 100644
--- a/Documentation/devicetree/bindings/media/cnm,wave521c.yaml
+++ b/Documentation/devicetree/bindings/media/cnm,wave521c.yaml
@@ -37,7 +37,7 @@ properties:
     maxItems: 1
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
+    maxItems: 1
     description:
       The VPU uses the SRAM to store some of the reference data instead of
       storing it on DMA memory. It is mainly used for the purpose of reducing
diff --git a/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml b/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml
index ebc615584f921..4505e66876a29 100644
--- a/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml
+++ b/Documentation/devicetree/bindings/media/mediatek,vcodec-encoder.yaml
@@ -160,7 +160,7 @@ examples:
 
     vcodec_enc_vp8: vcodec@19002000 {
       compatible = "mediatek,mt8173-vcodec-enc-vp8";
-      reg =  <0x19002000 0x1000>;	/* VENC_LT_SYS */
+      reg =  <0x19002000 0x1000>; /* VENC_LT_SYS */
       interrupts = <GIC_SPI 202 IRQ_TYPE_LEVEL_LOW>;
       iommus = <&iommu M4U_PORT_VENC_RCPU_SET2>,
              <&iommu M4U_PORT_VENC_REC_FRM_SET2>,
diff --git a/Documentation/devicetree/bindings/media/mediatek,vcodec-subdev-decoder.yaml b/Documentation/devicetree/bindings/media/mediatek,vcodec-subdev-decoder.yaml
index bf8082d87ac03..d1d209cbbd43a 100644
--- a/Documentation/devicetree/bindings/media/mediatek,vcodec-subdev-decoder.yaml
+++ b/Documentation/devicetree/bindings/media/mediatek,vcodec-subdev-decoder.yaml
@@ -232,7 +232,7 @@ examples:
             #address-cells = <2>;
             #size-cells = <2>;
             ranges = <0 0 0 0x16000000 0 0x40000>;
-            reg = <0 0x16000000 0 0x1000>;		/* VDEC_SYS */
+            reg = <0 0x16000000 0 0x1000>; /* VDEC_SYS */
             video-codec@10000 {
                 compatible = "mediatek,mtk-vcodec-lat";
                 reg = <0 0x10000 0 0x800>;
diff --git a/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml b/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml
index 18cc6315a8212..6ba668aa633d7 100644
--- a/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml
+++ b/Documentation/devicetree/bindings/media/nxp,imx8-jpeg.yaml
@@ -56,10 +56,10 @@ properties:
     maxItems: 5               # Wrapper and 4 slots
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
+    maxItems: 1
     description:
-      Optional phandle to a reserved on-chip SRAM regions. The SRAM can
-      be used for descriptor storage, which may improve bus utilization.
+      The SRAM can be used for descriptor storage, which may improve bus
+      utilization.
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml b/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml
index bfd8b1ad47312..b21bed3148484 100644
--- a/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml
+++ b/Documentation/devicetree/bindings/media/qcom,sc7180-venus.yaml
@@ -91,6 +91,21 @@ properties:
     deprecated: true
     additionalProperties: false
 
+  video-firmware:
+    type: object
+    additionalProperties: false
+
+    description: |
+      Firmware subnode is needed when the platform does not
+      have TrustZone.
+
+    properties:
+      iommus:
+        maxItems: 1
+
+    required:
+      - iommus
+
 required:
   - compatible
   - power-domain-names
diff --git a/Documentation/devicetree/bindings/media/qcom,sc7280-venus.yaml b/Documentation/devicetree/bindings/media/qcom,sc7280-venus.yaml
index 413c5b4ee6504..9725fcb761dc0 100644
--- a/Documentation/devicetree/bindings/media/qcom,sc7280-venus.yaml
+++ b/Documentation/devicetree/bindings/media/qcom,sc7280-venus.yaml
@@ -43,8 +43,7 @@ properties:
       - const: vcodec_bus
 
   iommus:
-    minItems: 1
-    maxItems: 2
+    maxItems: 1
 
   interconnects:
     maxItems: 2
@@ -120,12 +119,7 @@ examples:
                         <&mmss_noc MASTER_VIDEO_P0 0 &mc_virt SLAVE_EBI1 0>;
         interconnect-names = "cpu-cfg", "video-mem";
 
-        iommus = <&apps_smmu 0x2180 0x20>,
-                 <&apps_smmu 0x2184 0x20>;
+        iommus = <&apps_smmu 0x2180 0x20>;
 
         memory-region = <&video_mem>;
-
-        video-firmware {
-            iommus = <&apps_smmu 0x21a2 0x0>;
-        };
     };
diff --git a/Documentation/devicetree/bindings/media/qcom,sm6350-camss.yaml b/Documentation/devicetree/bindings/media/qcom,sm6350-camss.yaml
new file mode 100644
index 0000000000000..96974d90d8c43
--- /dev/null
+++ b/Documentation/devicetree/bindings/media/qcom,sm6350-camss.yaml
@@ -0,0 +1,471 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/media/qcom,sm6350-camss.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm SM6350 Camera Subsystem (CAMSS)
+
+maintainers:
+  - Luca Weiss <luca.weiss@fairphone.com>
+
+description:
+  The CAMSS IP is a CSI decoder and ISP present on Qualcomm platforms.
+
+properties:
+  compatible:
+    const: qcom,sm6350-camss
+
+  reg:
+    maxItems: 24
+
+  reg-names:
+    items:
+      - const: csid0
+      - const: csid1
+      - const: csid2
+      - const: csid_lite
+      - const: csiphy0
+      - const: csiphy1
+      - const: csiphy2
+      - const: csiphy3
+      - const: vfe0
+      - const: vfe1
+      - const: vfe2
+      - const: vfe_lite
+      - const: a5_csr
+      - const: a5_qgic
+      - const: a5_sierra
+      - const: bps
+      - const: camnoc
+      - const: core_top_csr_tcsr
+      - const: cpas_cdm
+      - const: cpas_top
+      - const: ipe
+      - const: jpeg_dma
+      - const: jpeg_enc
+      - const: lrme
+
+  clocks:
+    maxItems: 39
+
+  clock-names:
+    items:
+      - const: cam_axi
+      - const: soc_ahb
+      - const: camnoc_axi
+      - const: core_ahb
+      - const: cpas_ahb
+      - const: csiphy0
+      - const: csiphy0_timer
+      - const: csiphy1
+      - const: csiphy1_timer
+      - const: csiphy2
+      - const: csiphy2_timer
+      - const: csiphy3
+      - const: csiphy3_timer
+      - const: vfe0_axi
+      - const: vfe0
+      - const: vfe0_cphy_rx
+      - const: vfe0_csid
+      - const: vfe1_axi
+      - const: vfe1
+      - const: vfe1_cphy_rx
+      - const: vfe1_csid
+      - const: vfe2_axi
+      - const: vfe2
+      - const: vfe2_cphy_rx
+      - const: vfe2_csid
+      - const: vfe_lite
+      - const: vfe_lite_cphy_rx
+      - const: vfe_lite_csid
+      - const: bps
+      - const: bps_ahb
+      - const: bps_areg
+      - const: bps_axi
+      - const: icp
+      - const: ipe0
+      - const: ipe0_ahb
+      - const: ipe0_areg
+      - const: ipe0_axi
+      - const: jpeg
+      - const: lrme
+
+  interrupts:
+    maxItems: 18
+
+  interrupt-names:
+    items:
+      - const: csid0
+      - const: csid1
+      - const: csid2
+      - const: csid_lite
+      - const: csiphy0
+      - const: csiphy1
+      - const: csiphy2
+      - const: csiphy3
+      - const: vfe0
+      - const: vfe1
+      - const: vfe2
+      - const: vfe_lite
+      - const: a5
+      - const: cpas
+      - const: cpas_cdm
+      - const: jpeg_dma
+      - const: jpeg_enc
+      - const: lrme
+
+  interconnects:
+    maxItems: 4
+
+  interconnect-names:
+    items:
+      - const: ahb
+      - const: hf_mnoc
+      - const: sf_mnoc
+      - const: sf_icp_mnoc
+
+  iommus:
+    maxItems: 14
+
+  power-domains:
+    maxItems: 6
+
+  power-domain-names:
+    items:
+      - const: ife0
+      - const: ife1
+      - const: ife2
+      - const: top
+      - const: bps
+      - const: ipe
+
+  vdd-csiphy0-0p9-supply:
+    description:
+      Phandle to a 0.9V regulator supply to CSIPHY0.
+
+  vdd-csiphy0-1p25-supply:
+    description:
+      Phandle to a 1.25V regulator supply to CSIPHY0.
+
+  vdd-csiphy1-0p9-supply:
+    description:
+      Phandle to a 0.9V regulator supply to CSIPHY1.
+
+  vdd-csiphy1-1p25-supply:
+    description:
+      Phandle to a 1.25V regulator supply to CSIPHY1.
+
+  vdd-csiphy2-0p9-supply:
+    description:
+      Phandle to a 0.9V regulator supply to CSIPHY2.
+
+  vdd-csiphy2-1p25-supply:
+    description:
+      Phandle to a 1.25V regulator supply to CSIPHY2.
+
+  vdd-csiphy3-0p9-supply:
+    description:
+      Phandle to a 0.9V regulator supply to CSIPHY3.
+
+  vdd-csiphy3-1p25-supply:
+    description:
+      Phandle to a 1.25V regulator supply to CSIPHY3.
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+
+    description:
+      CSI input ports.
+
+    patternProperties:
+      "^port@[0-3]$":
+        $ref: /schemas/graph.yaml#/$defs/port-base
+        unevaluatedProperties: false
+
+        description:
+          Input port for receiving CSI data from a CSIPHY.
+
+        properties:
+          endpoint:
+            $ref: video-interfaces.yaml#
+            unevaluatedProperties: false
+
+            properties:
+              data-lanes:
+                minItems: 1
+                maxItems: 4
+
+              bus-type:
+                enum:
+                  - 1 # MEDIA_BUS_TYPE_CSI2_CPHY
+                  - 4 # MEDIA_BUS_TYPE_CSI2_DPHY
+
+            required:
+              - data-lanes
+
+required:
+  - compatible
+  - reg
+  - reg-names
+  - clocks
+  - clock-names
+  - interrupts
+  - interrupt-names
+  - interconnects
+  - interconnect-names
+  - iommus
+  - power-domains
+  - power-domain-names
+  - ports
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,gcc-sm6350.h>
+    #include <dt-bindings/clock/qcom,sm6350-camcc.h>
+    #include <dt-bindings/interconnect/qcom,icc.h>
+    #include <dt-bindings/interconnect/qcom,sm6350.h>
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/media/video-interfaces.h>
+    #include <dt-bindings/power/qcom-rpmpd.h>
+
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        isp@acb3000 {
+            compatible = "qcom,sm6350-camss";
+
+            reg = <0x0 0x0acb3000 0x0 0x1000>,
+                  <0x0 0x0acba000 0x0 0x1000>,
+                  <0x0 0x0acc1000 0x0 0x1000>,
+                  <0x0 0x0acc8000 0x0 0x1000>,
+                  <0x0 0x0ac65000 0x0 0x1000>,
+                  <0x0 0x0ac66000 0x0 0x1000>,
+                  <0x0 0x0ac67000 0x0 0x1000>,
+                  <0x0 0x0ac68000 0x0 0x1000>,
+                  <0x0 0x0acaf000 0x0 0x4000>,
+                  <0x0 0x0acb6000 0x0 0x4000>,
+                  <0x0 0x0acbd000 0x0 0x4000>,
+                  <0x0 0x0acc4000 0x0 0x4000>,
+                  <0x0 0x0ac18000 0x0 0x3000>,
+                  <0x0 0x0ac00000 0x0 0x6000>,
+                  <0x0 0x0ac10000 0x0 0x8000>,
+                  <0x0 0x0ac6f000 0x0 0x8000>,
+                  <0x0 0x0ac42000 0x0 0x4600>,
+                  <0x0 0x01fc0000 0x0 0x40000>,
+                  <0x0 0x0ac48000 0x0 0x1000>,
+                  <0x0 0x0ac40000 0x0 0x1000>,
+                  <0x0 0x0ac87000 0x0 0xa000>,
+                  <0x0 0x0ac52000 0x0 0x4000>,
+                  <0x0 0x0ac4e000 0x0 0x4000>,
+                  <0x0 0x0ac6b000 0x0 0xa00>;
+            reg-names = "csid0",
+                        "csid1",
+                        "csid2",
+                        "csid_lite",
+                        "csiphy0",
+                        "csiphy1",
+                        "csiphy2",
+                        "csiphy3",
+                        "vfe0",
+                        "vfe1",
+                        "vfe2",
+                        "vfe_lite",
+                        "a5_csr",
+                        "a5_qgic",
+                        "a5_sierra",
+                        "bps",
+                        "camnoc",
+                        "core_top_csr_tcsr",
+                        "cpas_cdm",
+                        "cpas_top",
+                        "ipe",
+                        "jpeg_dma",
+                        "jpeg_enc",
+                        "lrme";
+
+            clocks = <&gcc GCC_CAMERA_AXI_CLK>,
+                     <&camcc CAMCC_SOC_AHB_CLK>,
+                     <&camcc CAMCC_CAMNOC_AXI_CLK>,
+                     <&camcc CAMCC_CORE_AHB_CLK>,
+                     <&camcc CAMCC_CPAS_AHB_CLK>,
+                     <&camcc CAMCC_CSIPHY0_CLK>,
+                     <&camcc CAMCC_CSI0PHYTIMER_CLK>,
+                     <&camcc CAMCC_CSIPHY1_CLK>,
+                     <&camcc CAMCC_CSI1PHYTIMER_CLK>,
+                     <&camcc CAMCC_CSIPHY2_CLK>,
+                     <&camcc CAMCC_CSI2PHYTIMER_CLK>,
+                     <&camcc CAMCC_CSIPHY3_CLK>,
+                     <&camcc CAMCC_CSI3PHYTIMER_CLK>,
+                     <&camcc CAMCC_IFE_0_AXI_CLK>,
+                     <&camcc CAMCC_IFE_0_CLK>,
+                     <&camcc CAMCC_IFE_0_CPHY_RX_CLK>,
+                     <&camcc CAMCC_IFE_0_CSID_CLK>,
+                     <&camcc CAMCC_IFE_1_AXI_CLK>,
+                     <&camcc CAMCC_IFE_1_CLK>,
+                     <&camcc CAMCC_IFE_1_CPHY_RX_CLK>,
+                     <&camcc CAMCC_IFE_1_CSID_CLK>,
+                     <&camcc CAMCC_IFE_2_AXI_CLK>,
+                     <&camcc CAMCC_IFE_2_CLK>,
+                     <&camcc CAMCC_IFE_2_CPHY_RX_CLK>,
+                     <&camcc CAMCC_IFE_2_CSID_CLK>,
+                     <&camcc CAMCC_IFE_LITE_CLK>,
+                     <&camcc CAMCC_IFE_LITE_CPHY_RX_CLK>,
+                     <&camcc CAMCC_IFE_LITE_CSID_CLK>,
+                     <&camcc CAMCC_BPS_CLK>,
+                     <&camcc CAMCC_BPS_AHB_CLK>,
+                     <&camcc CAMCC_BPS_AREG_CLK>,
+                     <&camcc CAMCC_BPS_AXI_CLK>,
+                     <&camcc CAMCC_ICP_CLK>,
+                     <&camcc CAMCC_IPE_0_CLK>,
+                     <&camcc CAMCC_IPE_0_AHB_CLK>,
+                     <&camcc CAMCC_IPE_0_AREG_CLK>,
+                     <&camcc CAMCC_IPE_0_AXI_CLK>,
+                     <&camcc CAMCC_JPEG_CLK>,
+                     <&camcc CAMCC_LRME_CLK>;
+            clock-names = "cam_axi",
+                          "soc_ahb",
+                          "camnoc_axi",
+                          "core_ahb",
+                          "cpas_ahb",
+                          "csiphy0",
+                          "csiphy0_timer",
+                          "csiphy1",
+                          "csiphy1_timer",
+                          "csiphy2",
+                          "csiphy2_timer",
+                          "csiphy3",
+                          "csiphy3_timer",
+                          "vfe0_axi",
+                          "vfe0",
+                          "vfe0_cphy_rx",
+                          "vfe0_csid",
+                          "vfe1_axi",
+                          "vfe1",
+                          "vfe1_cphy_rx",
+                          "vfe1_csid",
+                          "vfe2_axi",
+                          "vfe2",
+                          "vfe2_cphy_rx",
+                          "vfe2_csid",
+                          "vfe_lite",
+                          "vfe_lite_cphy_rx",
+                          "vfe_lite_csid",
+                          "bps",
+                          "bps_ahb",
+                          "bps_areg",
+                          "bps_axi",
+                          "icp",
+                          "ipe0",
+                          "ipe0_ahb",
+                          "ipe0_areg",
+                          "ipe0_axi",
+                          "jpeg",
+                          "lrme";
+
+            interrupts = <GIC_SPI 464 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 466 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 717 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 473 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 477 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 478 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 479 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 461 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 465 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 467 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 718 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 472 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 463 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 459 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 469 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 475 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 474 IRQ_TYPE_EDGE_RISING>,
+                         <GIC_SPI 476 IRQ_TYPE_EDGE_RISING>;
+            interrupt-names = "csid0",
+                              "csid1",
+                              "csid2",
+                              "csid_lite",
+                              "csiphy0",
+                              "csiphy1",
+                              "csiphy2",
+                              "csiphy3",
+                              "vfe0",
+                              "vfe1",
+                              "vfe2",
+                              "vfe_lite",
+                              "a5",
+                              "cpas",
+                              "cpas_cdm",
+                              "jpeg_dma",
+                              "jpeg_enc",
+                              "lrme";
+
+            interconnects = <&gem_noc MASTER_AMPSS_M0 QCOM_ICC_TAG_ACTIVE_ONLY
+                             &config_noc SLAVE_CAMERA_CFG QCOM_ICC_TAG_ACTIVE_ONLY>,
+                            <&mmss_noc MASTER_CAMNOC_HF QCOM_ICC_TAG_ALWAYS
+                             &clk_virt SLAVE_EBI_CH0 QCOM_ICC_TAG_ALWAYS>,
+                            <&mmss_noc MASTER_CAMNOC_SF QCOM_ICC_TAG_ALWAYS
+                             &clk_virt SLAVE_EBI_CH0 QCOM_ICC_TAG_ALWAYS>,
+                            <&mmss_noc MASTER_CAMNOC_ICP QCOM_ICC_TAG_ALWAYS
+                             &clk_virt SLAVE_EBI_CH0 QCOM_ICC_TAG_ALWAYS>;
+            interconnect-names = "ahb",
+                                 "hf_mnoc",
+                                 "sf_mnoc",
+                                 "sf_icp_mnoc";
+
+            iommus = <&apps_smmu 0x820 0xc0>,
+                     <&apps_smmu 0x840 0x0>,
+                     <&apps_smmu 0x860 0xc0>,
+                     <&apps_smmu 0x880 0x0>,
+                     <&apps_smmu 0xc40 0x20>,
+                     <&apps_smmu 0xc60 0x20>,
+                     <&apps_smmu 0xc80 0x0>,
+                     <&apps_smmu 0xca2 0x0>,
+                     <&apps_smmu 0xcc0 0x20>,
+                     <&apps_smmu 0xce0 0x20>,
+                     <&apps_smmu 0xd00 0x20>,
+                     <&apps_smmu 0xd20 0x20>,
+                     <&apps_smmu 0xd40 0x20>,
+                     <&apps_smmu 0xd60 0x20>;
+
+            power-domains = <&camcc IFE_0_GDSC>,
+                            <&camcc IFE_1_GDSC>,
+                            <&camcc IFE_2_GDSC>,
+                            <&camcc TITAN_TOP_GDSC>,
+                            <&camcc BPS_GDSC>,
+                            <&camcc IPE_0_GDSC>;
+            power-domain-names = "ife0",
+                                 "ife1",
+                                 "ife2",
+                                 "top",
+                                 "bps",
+                                 "ipe";
+
+            vdd-csiphy0-0p9-supply = <&vreg_l18a>;
+            vdd-csiphy0-1p25-supply = <&vreg_l22a>;
+            vdd-csiphy1-0p9-supply = <&vreg_l18a>;
+            vdd-csiphy1-1p25-supply = <&vreg_l22a>;
+            vdd-csiphy2-0p9-supply = <&vreg_l18a>;
+            vdd-csiphy2-1p25-supply = <&vreg_l22a>;
+            vdd-csiphy3-0p9-supply = <&vreg_l18a>;
+            vdd-csiphy3-1p25-supply = <&vreg_l22a>;
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+
+                    csiphy0_ep: endpoint {
+                        data-lanes = <0 1 2 3>;
+                        bus-type = <MEDIA_BUS_TYPE_CSI2_DPHY>;
+                        remote-endpoint = <&sensor_ep>;
+                    };
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/media/qcom,sm8250-venus.yaml b/Documentation/devicetree/bindings/media/qcom,sm8250-venus.yaml
index da54493220c9d..43a10d9f664eb 100644
--- a/Documentation/devicetree/bindings/media/qcom,sm8250-venus.yaml
+++ b/Documentation/devicetree/bindings/media/qcom,sm8250-venus.yaml
@@ -22,7 +22,7 @@ properties:
 
   power-domains:
     minItems: 2
-    maxItems: 3
+    maxItems: 4
 
   power-domain-names:
     minItems: 2
@@ -30,6 +30,7 @@ properties:
       - const: venus
       - const: vcodec0
       - const: mx
+      - const: mmcx
 
   clocks:
     maxItems: 3
@@ -114,8 +115,12 @@ examples:
         interrupts = <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>;
         power-domains = <&videocc MVS0C_GDSC>,
                         <&videocc MVS0_GDSC>,
-                        <&rpmhpd RPMHPD_MX>;
-        power-domain-names = "venus", "vcodec0", "mx";
+                        <&rpmhpd RPMHPD_MX>,
+                        <&rpmhpd RPMHPD_MMCX>;
+        power-domain-names = "venus",
+                             "vcodec0",
+                             "mx",
+                             "mmcx";
 
         clocks = <&gcc GCC_VIDEO_AXI0_CLK>,
                  <&videocc VIDEO_CC_MVS0C_CLK>,
diff --git a/Documentation/devicetree/bindings/media/qcom,venus-common.yaml b/Documentation/devicetree/bindings/media/qcom,venus-common.yaml
index 3153d91f9d18a..59a3fde846d21 100644
--- a/Documentation/devicetree/bindings/media/qcom,venus-common.yaml
+++ b/Documentation/devicetree/bindings/media/qcom,venus-common.yaml
@@ -47,21 +47,6 @@ properties:
     minItems: 1
     maxItems: 4
 
-  video-firmware:
-    type: object
-    additionalProperties: false
-
-    description: |
-      Firmware subnode is needed when the platform does not
-      have TrustZone.
-
-    properties:
-      iommus:
-        maxItems: 1
-
-    required:
-      - iommus
-
 required:
   - reg
   - clocks
diff --git a/Documentation/devicetree/bindings/media/renesas,fcp.yaml b/Documentation/devicetree/bindings/media/renesas,fcp.yaml
index b5eff6fec8a98..5e11ae0ee4564 100644
--- a/Documentation/devicetree/bindings/media/renesas,fcp.yaml
+++ b/Documentation/devicetree/bindings/media/renesas,fcp.yaml
@@ -30,6 +30,8 @@ properties:
               - renesas,r9a07g043u-fcpvd # RZ/G2UL
               - renesas,r9a07g044-fcpvd # RZ/G2{L,LC}
               - renesas,r9a07g054-fcpvd # RZ/V2L
+              - renesas,r9a08g046-fcpvd # RZ/G3L
+              - renesas,r9a09g047-fcpvd # RZ/G3E
               - renesas,r9a09g056-fcpvd # RZ/V2N
               - renesas,r9a09g057-fcpvd # RZ/V2H(P)
           - const: renesas,fcpv         # Generic FCP for VSP fallback
@@ -77,6 +79,8 @@ allOf:
               - renesas,r9a07g043u-fcpvd
               - renesas,r9a07g044-fcpvd
               - renesas,r9a07g054-fcpvd
+              - renesas,r9a08g046-fcpvd
+              - renesas,r9a09g047-fcpvd
               - renesas,r9a09g056-fcpvd
               - renesas,r9a09g057-fcpvd
     then:
diff --git a/Documentation/devicetree/bindings/media/renesas,vsp1.yaml b/Documentation/devicetree/bindings/media/renesas,vsp1.yaml
index 07a97dd87a5bc..803358780f01b 100644
--- a/Documentation/devicetree/bindings/media/renesas,vsp1.yaml
+++ b/Documentation/devicetree/bindings/media/renesas,vsp1.yaml
@@ -25,6 +25,8 @@ properties:
           - enum:
               - renesas,r9a07g043u-vsp2   # RZ/G2UL
               - renesas,r9a07g054-vsp2    # RZ/V2L
+              - renesas,r9a08g046-vsp2    # RZ/G3L
+              - renesas,r9a09g047-vsp2    # RZ/G3E
               - renesas,r9a09g056-vsp2    # RZ/V2N
               - renesas,r9a09g057-vsp2    # RZ/V2H(P)
           - const: renesas,r9a07g044-vsp2 # RZ/G2L fallback
diff --git a/Documentation/devicetree/bindings/media/rockchip,rk3568-mipi-csi2.yaml b/Documentation/devicetree/bindings/media/rockchip,rk3568-mipi-csi2.yaml
index 4ac4a3b6f4064..8bfad0fca3b7d 100644
--- a/Documentation/devicetree/bindings/media/rockchip,rk3568-mipi-csi2.yaml
+++ b/Documentation/devicetree/bindings/media/rockchip,rk3568-mipi-csi2.yaml
@@ -16,9 +16,15 @@ description:
 
 properties:
   compatible:
-    enum:
-      - fsl,imx93-mipi-csi2
-      - rockchip,rk3568-mipi-csi2
+    oneOf:
+      - enum:
+          - fsl,imx93-mipi-csi2
+          - fsl,imx95-mipi-csi2
+          - rockchip,rk3568-mipi-csi2
+      - items:
+          - enum:
+              - rockchip,rk3588-mipi-csi2
+          - const: rockchip,rk3568-mipi-csi2
 
   reg:
     maxItems: 1
@@ -135,6 +141,21 @@ allOf:
         clock-names:
           minItems: 2
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: fsl,imx95-mipi-csi2
+    then:
+      properties:
+        interrupts:
+          maxItems: 1
+        interrupt-names: false
+        clocks:
+          maxItems: 1
+        clock-names:
+          maxItems: 1
+
 examples:
   - |
     #include <dt-bindings/clock/rk3568-cru.h>
diff --git a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml
index 42022401d0ffa..4f38a0ef29d84 100644
--- a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml
+++ b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml
@@ -91,9 +91,8 @@ properties:
     maxItems: 1
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
-    description: |
-      phandle to a reserved on-chip SRAM regions.
+    maxItems: 1
+    description:
       Some SoCs, like rk3588 provide on-chip SRAM to store temporary
       buffers during decoding.
 
diff --git a/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml b/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml
index d9fbb90b0977d..7c2ddd27780f6 100644
--- a/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml
+++ b/Documentation/devicetree/bindings/media/st,stm32-dcmi.yaml
@@ -47,10 +47,10 @@ properties:
     maxItems: 1
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
+    maxItems: 1
     description:
-      phandle to a reserved SRAM region which is used as temporary
-      storage memory between DMA and MDMA engines.
+      SRAM region which is used as temporary storage memory between DMA and
+      MDMA engines.
 
   port:
     $ref: /schemas/graph.yaml#/$defs/port-base
diff --git a/Documentation/devicetree/bindings/media/ti,j721e-csi2rx-shim.yaml b/Documentation/devicetree/bindings/media/ti,j721e-csi2rx-shim.yaml
index b9f033f2f3ce4..bf62998b0445a 100644
--- a/Documentation/devicetree/bindings/media/ti,j721e-csi2rx-shim.yaml
+++ b/Documentation/devicetree/bindings/media/ti,j721e-csi2rx-shim.yaml
@@ -20,11 +20,44 @@ properties:
     const: ti,j721e-csi2rx-shim
 
   dmas:
-    maxItems: 1
+    minItems: 1
+    maxItems: 32
 
   dma-names:
+    minItems: 1
     items:
       - const: rx0
+      - const: rx1
+      - const: rx2
+      - const: rx3
+      - const: rx4
+      - const: rx5
+      - const: rx6
+      - const: rx7
+      - const: rx8
+      - const: rx9
+      - const: rx10
+      - const: rx11
+      - const: rx12
+      - const: rx13
+      - const: rx14
+      - const: rx15
+      - const: rx16
+      - const: rx17
+      - const: rx18
+      - const: rx19
+      - const: rx20
+      - const: rx21
+      - const: rx22
+      - const: rx23
+      - const: rx24
+      - const: rx25
+      - const: rx26
+      - const: rx27
+      - const: rx28
+      - const: rx29
+      - const: rx30
+      - const: rx31
 
   reg:
     maxItems: 1
@@ -62,8 +95,8 @@ examples:
 
     ti_csi2rx0: ticsi2rx@4500000 {
         compatible = "ti,j721e-csi2rx-shim";
-        dmas = <&main_udmap 0x4940>;
-        dma-names = "rx0";
+        dmas = <&main_udmap 0x4940>, <&main_udmap 0x4941>;
+        dma-names = "rx0", "rx1";
         reg = <0x4500000 0x1000>;
         power-domains = <&k3_pds 26 TI_SCI_PD_EXCLUSIVE>;
         #address-cells = <1>;
diff --git a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-emc.yaml b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-emc.yaml
index f5f03bf364137..9398aae490933 100644
--- a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-emc.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-emc.yaml
@@ -16,7 +16,9 @@ description: |
 
 properties:
   compatible:
-    const: nvidia,tegra124-emc
+    enum:
+      - nvidia,tegra114-emc
+      - nvidia,tegra124-emc
 
   reg:
     maxItems: 1
@@ -29,6 +31,9 @@ properties:
     items:
       - const: emc
 
+  interrupts:
+    maxItems: 1
+
   "#interconnect-cells":
     const: 0
 
@@ -164,153 +169,12 @@ patternProperties:
           nvidia,emc-configuration:
             description:
               EMC timing characterization data. These are the registers (see
-              section "15.6.2 EMC Registers" in the TRM) whose values need to
+              section "20.11.2 EMC Registers" in the Tegra114 TRM or section
+              "15.6.2 EMC Registers" in the Tegra124 TRM) whose values need to
               be specified, according to the board documentation.
             $ref: /schemas/types.yaml#/definitions/uint32-array
-            items:
-              - description: EMC_RC
-              - description: EMC_RFC
-              - description: EMC_RFC_SLR
-              - description: EMC_RAS
-              - description: EMC_RP
-              - description: EMC_R2W
-              - description: EMC_W2R
-              - description: EMC_R2P
-              - description: EMC_W2P
-              - description: EMC_RD_RCD
-              - description: EMC_WR_RCD
-              - description: EMC_RRD
-              - description: EMC_REXT
-              - description: EMC_WEXT
-              - description: EMC_WDV
-              - description: EMC_WDV_MASK
-              - description: EMC_QUSE
-              - description: EMC_QUSE_WIDTH
-              - description: EMC_IBDLY
-              - description: EMC_EINPUT
-              - description: EMC_EINPUT_DURATION
-              - description: EMC_PUTERM_EXTRA
-              - description: EMC_PUTERM_WIDTH
-              - description: EMC_PUTERM_ADJ
-              - description: EMC_CDB_CNTL_1
-              - description: EMC_CDB_CNTL_2
-              - description: EMC_CDB_CNTL_3
-              - description: EMC_QRST
-              - description: EMC_QSAFE
-              - description: EMC_RDV
-              - description: EMC_RDV_MASK
-              - description: EMC_REFRESH
-              - description: EMC_BURST_REFRESH_NUM
-              - description: EMC_PRE_REFRESH_REQ_CNT
-              - description: EMC_PDEX2WR
-              - description: EMC_PDEX2RD
-              - description: EMC_PCHG2PDEN
-              - description: EMC_ACT2PDEN
-              - description: EMC_AR2PDEN
-              - description: EMC_RW2PDEN
-              - description: EMC_TXSR
-              - description: EMC_TXSRDLL
-              - description: EMC_TCKE
-              - description: EMC_TCKESR
-              - description: EMC_TPD
-              - description: EMC_TFAW
-              - description: EMC_TRPAB
-              - description: EMC_TCLKSTABLE
-              - description: EMC_TCLKSTOP
-              - description: EMC_TREFBW
-              - description: EMC_FBIO_CFG6
-              - description: EMC_ODT_WRITE
-              - description: EMC_ODT_READ
-              - description: EMC_FBIO_CFG5
-              - description: EMC_CFG_DIG_DLL
-              - description: EMC_CFG_DIG_DLL_PERIOD
-              - description: EMC_DLL_XFORM_DQS0
-              - description: EMC_DLL_XFORM_DQS1
-              - description: EMC_DLL_XFORM_DQS2
-              - description: EMC_DLL_XFORM_DQS3
-              - description: EMC_DLL_XFORM_DQS4
-              - description: EMC_DLL_XFORM_DQS5
-              - description: EMC_DLL_XFORM_DQS6
-              - description: EMC_DLL_XFORM_DQS7
-              - description: EMC_DLL_XFORM_DQS8
-              - description: EMC_DLL_XFORM_DQS9
-              - description: EMC_DLL_XFORM_DQS10
-              - description: EMC_DLL_XFORM_DQS11
-              - description: EMC_DLL_XFORM_DQS12
-              - description: EMC_DLL_XFORM_DQS13
-              - description: EMC_DLL_XFORM_DQS14
-              - description: EMC_DLL_XFORM_DQS15
-              - description: EMC_DLL_XFORM_QUSE0
-              - description: EMC_DLL_XFORM_QUSE1
-              - description: EMC_DLL_XFORM_QUSE2
-              - description: EMC_DLL_XFORM_QUSE3
-              - description: EMC_DLL_XFORM_QUSE4
-              - description: EMC_DLL_XFORM_QUSE5
-              - description: EMC_DLL_XFORM_QUSE6
-              - description: EMC_DLL_XFORM_QUSE7
-              - description: EMC_DLL_XFORM_ADDR0
-              - description: EMC_DLL_XFORM_ADDR1
-              - description: EMC_DLL_XFORM_ADDR2
-              - description: EMC_DLL_XFORM_ADDR3
-              - description: EMC_DLL_XFORM_ADDR4
-              - description: EMC_DLL_XFORM_ADDR5
-              - description: EMC_DLL_XFORM_QUSE8
-              - description: EMC_DLL_XFORM_QUSE9
-              - description: EMC_DLL_XFORM_QUSE10
-              - description: EMC_DLL_XFORM_QUSE11
-              - description: EMC_DLL_XFORM_QUSE12
-              - description: EMC_DLL_XFORM_QUSE13
-              - description: EMC_DLL_XFORM_QUSE14
-              - description: EMC_DLL_XFORM_QUSE15
-              - description: EMC_DLI_TRIM_TXDQS0
-              - description: EMC_DLI_TRIM_TXDQS1
-              - description: EMC_DLI_TRIM_TXDQS2
-              - description: EMC_DLI_TRIM_TXDQS3
-              - description: EMC_DLI_TRIM_TXDQS4
-              - description: EMC_DLI_TRIM_TXDQS5
-              - description: EMC_DLI_TRIM_TXDQS6
-              - description: EMC_DLI_TRIM_TXDQS7
-              - description: EMC_DLI_TRIM_TXDQS8
-              - description: EMC_DLI_TRIM_TXDQS9
-              - description: EMC_DLI_TRIM_TXDQS10
-              - description: EMC_DLI_TRIM_TXDQS11
-              - description: EMC_DLI_TRIM_TXDQS12
-              - description: EMC_DLI_TRIM_TXDQS13
-              - description: EMC_DLI_TRIM_TXDQS14
-              - description: EMC_DLI_TRIM_TXDQS15
-              - description: EMC_DLL_XFORM_DQ0
-              - description: EMC_DLL_XFORM_DQ1
-              - description: EMC_DLL_XFORM_DQ2
-              - description: EMC_DLL_XFORM_DQ3
-              - description: EMC_DLL_XFORM_DQ4
-              - description: EMC_DLL_XFORM_DQ5
-              - description: EMC_DLL_XFORM_DQ6
-              - description: EMC_DLL_XFORM_DQ7
-              - description: EMC_XM2CMDPADCTRL
-              - description: EMC_XM2CMDPADCTRL4
-              - description: EMC_XM2CMDPADCTRL5
-              - description: EMC_XM2DQPADCTRL2
-              - description: EMC_XM2DQPADCTRL3
-              - description: EMC_XM2CLKPADCTRL
-              - description: EMC_XM2CLKPADCTRL2
-              - description: EMC_XM2COMPPADCTRL
-              - description: EMC_XM2VTTGENPADCTRL
-              - description: EMC_XM2VTTGENPADCTRL2
-              - description: EMC_XM2VTTGENPADCTRL3
-              - description: EMC_XM2DQSPADCTRL3
-              - description: EMC_XM2DQSPADCTRL4
-              - description: EMC_XM2DQSPADCTRL5
-              - description: EMC_XM2DQSPADCTRL6
-              - description: EMC_DSR_VTTGEN_DRV
-              - description: EMC_TXDSRVTTGEN
-              - description: EMC_FBIO_SPARE
-              - description: EMC_ZCAL_WAIT_CNT
-              - description: EMC_MRS_WAIT_CNT2
-              - description: EMC_CTT
-              - description: EMC_CTT_DURATION
-              - description: EMC_CFG_PIPE
-              - description: EMC_DYN_SELF_REF_CONTROL
-              - description: EMC_QPOP
+            minItems: 97
+            maxItems: 143
 
         required:
           - clock-frequency
@@ -318,9 +182,7 @@ patternProperties:
           - nvidia,emc-auto-cal-config2
           - nvidia,emc-auto-cal-config3
           - nvidia,emc-auto-cal-interval
-          - nvidia,emc-bgbias-ctl0
           - nvidia,emc-cfg
-          - nvidia,emc-cfg-2
           - nvidia,emc-ctt-term-ctrl
           - nvidia,emc-mode-1
           - nvidia,emc-mode-2
@@ -344,6 +206,22 @@ required:
   - "#interconnect-cells"
   - operating-points-v2
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - nvidia,tegra124-emc
+    then:
+      patternProperties:
+        "^emc-timings-[0-9]+$":
+          patternProperties:
+            "^timing-[0-9]+$":
+              required:
+                - nvidia,emc-bgbias-ctl0
+                - nvidia,emc-cfg-2
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-mc.yaml b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-mc.yaml
index 7b18b4d11e0ae..f8747cebb6808 100644
--- a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-mc.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra124-mc.yaml
@@ -19,7 +19,9 @@ description: |
 
 properties:
   compatible:
-    const: nvidia,tegra124-mc
+    enum:
+      - nvidia,tegra114-mc
+      - nvidia,tegra124-mc
 
   reg:
     maxItems: 1
@@ -64,29 +66,12 @@ patternProperties:
 
           nvidia,emem-configuration:
             $ref: /schemas/types.yaml#/definitions/uint32-array
-            description: |
+            description:
               Values to be written to the EMEM register block. See section
-              "15.6.1 MC Registers" in the TRM.
-            items:
-              - description: MC_EMEM_ARB_CFG
-              - description: MC_EMEM_ARB_OUTSTANDING_REQ
-              - description: MC_EMEM_ARB_TIMING_RCD
-              - description: MC_EMEM_ARB_TIMING_RP
-              - description: MC_EMEM_ARB_TIMING_RC
-              - description: MC_EMEM_ARB_TIMING_RAS
-              - description: MC_EMEM_ARB_TIMING_FAW
-              - description: MC_EMEM_ARB_TIMING_RRD
-              - description: MC_EMEM_ARB_TIMING_RAP2PRE
-              - description: MC_EMEM_ARB_TIMING_WAP2PRE
-              - description: MC_EMEM_ARB_TIMING_R2R
-              - description: MC_EMEM_ARB_TIMING_W2W
-              - description: MC_EMEM_ARB_TIMING_R2W
-              - description: MC_EMEM_ARB_TIMING_W2R
-              - description: MC_EMEM_ARB_DA_TURNS
-              - description: MC_EMEM_ARB_DA_COVERS
-              - description: MC_EMEM_ARB_MISC0
-              - description: MC_EMEM_ARB_MISC1
-              - description: MC_EMEM_ARB_RING1_THROTTLE
+              "20.11.1 MC Registers" in the Tegea114 TRM or
+              "15.6.1 MC Registers" in the Tegra124 TRM.
+            minItems: 18
+            maxItems: 19
 
         required:
           - clock-frequency
diff --git a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra186-mc.yaml b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra186-mc.yaml
index 7b03b589168b1..6c374e2b1543b 100644
--- a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra186-mc.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra186-mc.yaml
@@ -32,6 +32,7 @@ properties:
           - nvidia,tegra186-mc
           - nvidia,tegra194-mc
           - nvidia,tegra234-mc
+          - nvidia,tegra238-mc
           - nvidia,tegra264-mc
 
   reg:
@@ -266,6 +267,36 @@ allOf:
 
         interrupt-names: false
 
+  - if:
+      properties:
+        compatible:
+          const: nvidia,tegra238-mc
+    then:
+      properties:
+        reg:
+          minItems: 10
+          maxItems: 10
+          description: 9 memory controller channels and 1 for stream-id registers
+
+        reg-names:
+          items:
+            - const: sid
+            - const: broadcast
+            - const: ch0
+            - const: ch1
+            - const: ch2
+            - const: ch3
+            - const: ch4
+            - const: ch5
+            - const: ch6
+            - const: ch7
+
+        interrupts:
+          items:
+            - description: MC general interrupt
+
+        interrupt-names: false
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra20-mc.yaml b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra20-mc.yaml
index 55caf69053995..a33913fcd11f8 100644
--- a/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra20-mc.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/nvidia,tegra20-mc.yaml
@@ -66,8 +66,8 @@ examples:
   - |
     memory-controller@7000f000 {
         compatible = "nvidia,tegra20-mc-gart";
-        reg = <0x7000f000 0x400>,	/* Controller registers */
-              <0x58000000 0x02000000>;	/* GART aperture */
+        reg = <0x7000f000 0x400>, /* Controller registers */
+              <0x58000000 0x02000000>; /* GART aperture */
         clocks = <&clock_controller 32>;
         clock-names = "mc";
 
diff --git a/Documentation/devicetree/bindings/memory-controllers/samsung,exynos5422-dmc.yaml b/Documentation/devicetree/bindings/memory-controllers/samsung,exynos5422-dmc.yaml
index 783ac984d8984..4cd5af38abce1 100644
--- a/Documentation/devicetree/bindings/memory-controllers/samsung,exynos5422-dmc.yaml
+++ b/Documentation/devicetree/bindings/memory-controllers/samsung,exynos5422-dmc.yaml
@@ -128,7 +128,7 @@ examples:
                       "mout_mx_mspll_ccore",
                       "mout_mclk_cdrex";
         operating-points-v2 = <&dmc_opp_table>;
-        devfreq-events = <&ppmu_event3_dmc0_0>,	<&ppmu_event3_dmc0_1>,
+        devfreq-events = <&ppmu_event3_dmc0_0>, <&ppmu_event3_dmc0_1>,
                          <&ppmu_event3_dmc1_0>, <&ppmu_event3_dmc1_1>;
         device-handle = <&samsung_K3QF2F20DB>;
         vdd-supply = <&buck1_reg>;
diff --git a/Documentation/devicetree/bindings/mfd/aspeed,ast2x00-scu.yaml b/Documentation/devicetree/bindings/mfd/aspeed,ast2x00-scu.yaml
index a87f31fce0195..0d5e168b0309c 100644
--- a/Documentation/devicetree/bindings/mfd/aspeed,ast2x00-scu.yaml
+++ b/Documentation/devicetree/bindings/mfd/aspeed,ast2x00-scu.yaml
@@ -46,6 +46,18 @@ properties:
   '#reset-cells':
     const: 1
 
+  memory-region:
+    items:
+      - description: Region mapped through the first SSP address window.
+      - description: Region mapped through the second SSP address window.
+      - description: Region mapped through the TSP address window.
+
+  memory-region-names:
+    items:
+      - const: ssp-0
+      - const: ssp-1
+      - const: tsp
+
 patternProperties:
   '^p2a-control@[0-9a-f]+$':
     description: >
@@ -87,6 +99,7 @@ patternProperties:
             - aspeed,ast2400-pinctrl
             - aspeed,ast2500-pinctrl
             - aspeed,ast2600-pinctrl
+            - aspeed,ast2700-soc0-pinctrl
 
     required:
       - compatible
@@ -156,6 +169,30 @@ required:
   - '#clock-cells'
   - '#reset-cells'
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - aspeed,ast2700-scu0
+              - aspeed,ast2700-scu1
+    then:
+      patternProperties:
+        '^p2a-control@[0-9a-f]+$': false
+        '^smp-memram@[0-9a-f]+$': false
+
+  - if:
+      not:
+        properties:
+          compatible:
+            contains:
+              const: aspeed,ast2700-scu0
+    then:
+      properties:
+        memory-region: false
+        memory-region-names: false
+
 additionalProperties: false
 
 examples:
@@ -180,4 +217,81 @@ examples:
             reg = <0x7c 0x4>, <0x150 0x8>;
         };
     };
+
+  - |
+    / {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        reserved-memory {
+            #address-cells = <2>;
+            #size-cells = <2>;
+            ranges;
+
+            ssp_region_0: memory@400000000 {
+                reg = <0x4 0x00000000 0x0 0x01000000>;
+                no-map;
+            };
+
+            ssp_region_1: memory@401000000 {
+                reg = <0x4 0x01000000 0x0 0x01000000>;
+                no-map;
+            };
+
+            tsp_region: memory@402000000 {
+                reg = <0x4 0x02000000 0x0 0x01000000>;
+                no-map;
+            };
+        };
+
+        bus {
+            #address-cells = <2>;
+            #size-cells = <2>;
+
+            syscon@12c02000 {
+                compatible = "aspeed,ast2700-scu0", "syscon", "simple-mfd";
+                reg = <0 0x12c02000 0 0x1000>;
+                ranges = <0x0 0x0 0x12c02000 0x1000>;
+                #address-cells = <1>;
+                #size-cells = <1>;
+                #clock-cells = <1>;
+                #reset-cells = <1>;
+
+                memory-region = <&ssp_region_0>, <&ssp_region_1>,
+                                <&tsp_region>;
+                memory-region-names = "ssp-0", "ssp-1", "tsp";
+
+                silicon-id@0 {
+                    compatible = "aspeed,ast2700-silicon-id", "aspeed,silicon-id";
+                    reg = <0x0 0x4>;
+                };
+
+                interrupt-controller@1b0 {
+                    compatible = "aspeed,ast2700-scu-ic0";
+                    reg = <0x1b0 0x4>;
+                    #interrupt-cells = <1>;
+                    interrupts-extended = <&intc0 97>;
+                    interrupt-controller;
+                };
+
+                interrupt-controller@1e0 {
+                    compatible = "aspeed,ast2700-scu-ic1";
+                    reg = <0x1e0 0x4>;
+                    #interrupt-cells = <1>;
+                    interrupts-extended = <&intc0 98>;
+                    interrupt-controller;
+                };
+
+                pinctrl@400 {
+                    compatible = "aspeed,ast2700-soc0-pinctrl";
+                    reg = <0x400 0x318>;
+                    emmc-state {
+                        function = "EMMC";
+                        groups = "EMMCG1";
+                    };
+                };
+            };
+        };
+    };
+
 ...
diff --git a/Documentation/devicetree/bindings/mfd/hisilicon,hi655x-pmic.yaml b/Documentation/devicetree/bindings/mfd/hisilicon,hi655x-pmic.yaml
new file mode 100644
index 0000000000000..6f28f472e0f58
--- /dev/null
+++ b/Documentation/devicetree/bindings/mfd/hisilicon,hi655x-pmic.yaml
@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/mfd/hisilicon,hi655x-pmic.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Hisilicon Hi655x Power Management Integrated Circuit
+
+maintainers:
+  - Chen Feng <puck.chen@hisilicon.com>
+  - Daniel Lezcano <daniel.lezcano@kernel.org>
+
+description:
+  The hardware layout for access PMIC Hi655x from AP SoC Hi6220.
+  Between PMIC Hi655x and Hi6220, the physical signal channel is SSI.
+  We can use memory-mapped I/O to communicate.
+
+properties:
+  compatible:
+    const: hisilicon,hi655x-pmic
+
+  reg:
+    maxItems: 1
+
+  interrupt-controller: true
+
+  '#interrupt-cells':
+    const: 2
+
+  pmic-gpios:
+    maxItems: 1
+    description: The GPIO used by PMIC IRQ
+
+  '#clock-cells':
+    const: 0
+
+  clock-output-names:
+    maxItems: 1
+
+  regulators:
+    type: object
+    additionalProperties: false
+
+    patternProperties:
+      '^LDO(2|7|10|13|14|15|17|19|21|22)$':
+        type: object
+        $ref: /schemas/regulator/regulator.yaml#
+        unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+  - interrupt-controller
+  - '#interrupt-cells'
+  - pmic-gpios
+  - '#clock-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    pmic: pmic@f8000000 {
+        compatible = "hisilicon,hi655x-pmic";
+        reg = <0xf8000000 0x1000>;
+        #clock-cells = <0>;
+        interrupt-controller;
+        #interrupt-cells = <2>;
+        pmic-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>;
+
+        regulators {
+            ldo2: LDO2 {
+                regulator-name = "LDO2_2V8";
+                regulator-min-microvolt = <2500000>;
+                regulator-max-microvolt = <3200000>;
+                regulator-enable-ramp-delay = <120>;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/mfd/hisilicon,hi655x.txt b/Documentation/devicetree/bindings/mfd/hisilicon,hi655x.txt
deleted file mode 100644
index 9630ac0e4b56e..0000000000000
--- a/Documentation/devicetree/bindings/mfd/hisilicon,hi655x.txt
+++ /dev/null
@@ -1,33 +0,0 @@
-Hisilicon Hi655x Power Management Integrated Circuit (PMIC)
-
-The hardware layout for access PMIC Hi655x from AP SoC Hi6220.
-Between PMIC Hi655x and Hi6220, the physical signal channel is SSI.
-We can use memory-mapped I/O to communicate.
-
-+----------------+             +-------------+
-|                |             |             |
-|    Hi6220      |   SSI bus   |   Hi655x    |
-|                |-------------|             |
-|                |(REGMAP_MMIO)|             |
-+----------------+             +-------------+
-
-Required properties:
-- compatible:           Should be "hisilicon,hi655x-pmic".
-- reg:                  Base address of PMIC on Hi6220 SoC.
-- interrupt-controller: Hi655x has internal IRQs (has own IRQ domain).
-- pmic-gpios:           The GPIO used by PMIC IRQ.
-- #clock-cells:		From common clock binding; shall be set to 0
-
-Optional properties:
-- clock-output-names: From common clock binding to override the
-  default output clock name
-
-Example:
-	pmic: pmic@f8000000 {
-		compatible = "hisilicon,hi655x-pmic";
-		reg = <0x0 0xf8000000 0x0 0x1000>;
-		interrupt-controller;
-		#interrupt-cells = <2>;
-		pmic-gpios = <&gpio1 2 GPIO_ACTIVE_HIGH>;
-		#clock-cells = <0>;
-	}
diff --git a/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml b/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml
index 084960fd5a1fd..1f135618e3b6f 100644
--- a/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml
+++ b/Documentation/devicetree/bindings/mfd/khadas,mcu.yaml
@@ -18,6 +18,7 @@ properties:
   compatible:
     enum:
       - khadas,mcu # MCU revision is discoverable
+      - khadas,vim4-mcu # Different MCU variant, not discoverable
 
   "#cooling-cells": # Only needed for boards having FAN control feature
     const: 2
@@ -25,10 +26,27 @@ properties:
   reg:
     maxItems: 1
 
+  fan-supply:
+    description: Phandle to the regulator that powers the fan.
+    $ref: /schemas/types.yaml#/definitions/phandle
+
 required:
   - compatible
   - reg
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: khadas,vim4-mcu
+    then:
+      required:
+        - fan-supply
+    else:
+      properties:
+        fan-supply: false
+
 additionalProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml b/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml
index 05c121b0cb3d8..3cbc0dc12c319 100644
--- a/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml
+++ b/Documentation/devicetree/bindings/mfd/mediatek,mt6397.yaml
@@ -44,6 +44,10 @@ properties:
           - enum:
               - mediatek,mt6366
           - const: mediatek,mt6358
+      - items:
+          - enum:
+              - mediatek,mt6365
+          - const: mediatek,mt6359
 
   interrupts:
     maxItems: 1
@@ -70,6 +74,8 @@ properties:
               - mediatek,mt6397-rtc
           - items:
               - enum:
+                  - mediatek,mt6359-rtc
+                  - mediatek,mt6365-rtc
                   - mediatek,mt6366-rtc
               - const: mediatek,mt6358-rtc
 
@@ -98,6 +104,10 @@ properties:
               - enum:
                   - mediatek,mt6366-regulator
               - const: mediatek,mt6358-regulator
+          - items:
+              - enum:
+                  - mediatek,mt6365-regulator
+              - const: mediatek,mt6359-regulator
 
     required:
       - compatible
@@ -124,6 +134,10 @@ properties:
               - enum:
                   - mediatek,mt6366-sound
               - const: mediatek,mt6358-sound
+          - items:
+              - enum:
+                  - mediatek,mt6365-codec
+              - const: mediatek,mt6359-codec
 
     required:
       - compatible
@@ -225,12 +239,62 @@ properties:
     description:
       Pin controller
 
+  vsys-smps-supply:
+    description: Supply for regulator control logic
+
+patternProperties:
+  "^vsys-v[a-z]+[0-9]*-supply$":
+    description: Supplies for PMIC buck regulators
+  "^vs(ys|[12])-ldo[1-9]-supply$":
+    description: Supplies for PMIC LDO regulators
+
 required:
   - compatible
   - regulators
 
 additionalProperties: false
 
+allOf:
+  - if:
+      properties:
+        "compatible":
+          contains:
+            const: mediatek,mt6359
+    then:
+      properties:
+        vsys-ldo1-supply:
+          description: Supply for LDOs vcn33_[12], vio28, vfe28, vibr
+        vsys-ldo2-supply:
+          description: Supply for LDOs vaux18, vbif28, vxo22, vrfck, vrfck_1,
+            vemc, vsim1, vsim2, vusb
+        vsys-vcore-supply:
+          description: Supply for buck regulator vcore
+        vsys-vgpu11-supply:
+          description: Supply for buck regulator vgpu11
+        vsys-vmodem-supply:
+          description: Supply for buck regulator vmodem
+        vsys-vpa-supply:
+          description: Supply for buck regulator vpa
+        vsys-vproc1-supply:
+          description: Supply for buck regulator vproc1
+        vsys-vproc2-supply:
+          description: Supply for buck regulator vproc2
+        vsys-vpu-supply:
+          description: Supply for buck regulator vpu
+        vsys-vs1-supply:
+          description: Supply for buck regulator vs1
+        vsys-vs2-supply:
+          description: Supply for buck regulator vs2
+        vs1-ldo1-supply:
+          description: Supply for LDOs vaud18, vcamio, vm18, vufs
+        vs1-ldo2-supply:
+          description: Supply for LDOs vcn18, vefuse, vio18, vrf18
+        vs2-ldo1-supply:
+          description:
+            Supply for LDOs vsram_proc1, vsram_proc2, vsram_others, vsram_md
+        vs2-ldo2-supply:
+          description: Supply for LDOs va09, va12, vcn13, vrf12
+
 examples:
   - |
     #include <dt-bindings/interrupt-controller/arm-gic.h>
diff --git a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml
index 14ae3f00ef7e0..23317d1b381ca 100644
--- a/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml
+++ b/Documentation/devicetree/bindings/mfd/qcom,tcsr.yaml
@@ -19,6 +19,7 @@ properties:
       - enum:
           - qcom,msm8976-tcsr
           - qcom,msm8998-tcsr
+          - qcom,nord-tcsr
           - qcom,qcm2290-tcsr
           - qcom,qcs404-tcsr
           - qcom,qcs615-tcsr
diff --git a/Documentation/devicetree/bindings/mfd/rockchip,rk816.yaml b/Documentation/devicetree/bindings/mfd/rockchip,rk816.yaml
index 0676890f101e5..a58d9455a1a54 100644
--- a/Documentation/devicetree/bindings/mfd/rockchip,rk816.yaml
+++ b/Documentation/devicetree/bindings/mfd/rockchip,rk816.yaml
@@ -44,8 +44,7 @@ properties:
     description:
       Telling whether or not this PMIC is controlling the system power.
 
-  wakeup-source:
-    type: boolean
+  wakeup-source: true
 
   vcc1-supply:
     description:
diff --git a/Documentation/devicetree/bindings/mfd/syscon.yaml b/Documentation/devicetree/bindings/mfd/syscon.yaml
index e228670880631..9c81010d5a746 100644
--- a/Documentation/devicetree/bindings/mfd/syscon.yaml
+++ b/Documentation/devicetree/bindings/mfd/syscon.yaml
@@ -221,6 +221,7 @@ properties:
               - nxp,s32g3-gpr
               - qcom,apq8064-mmss-sfpb
               - qcom,apq8064-sps-sic
+              - renesas,r9a08g046-lvds-cmn
               - rockchip,px30-qos
               - rockchip,rk3036-qos
               - rockchip,rk3066-qos
diff --git a/Documentation/devicetree/bindings/mfd/ti,bq25703a.yaml b/Documentation/devicetree/bindings/mfd/ti,bq25703a.yaml
index ba14663c9266a..cdce83f058041 100644
--- a/Documentation/devicetree/bindings/mfd/ti,bq25703a.yaml
+++ b/Documentation/devicetree/bindings/mfd/ti,bq25703a.yaml
@@ -4,17 +4,16 @@
 $id: http://devicetree.org/schemas/mfd/ti,bq25703a.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: BQ25703A Charger Manager/Buck/Boost Converter
+title: BQ257xx Charger Manager/Buck/Boost Converter
 
 maintainers:
   - Chris Morgan <macromorgan@hotmail.com>
 
-allOf:
-  - $ref: /schemas/power/supply/power-supply.yaml#
-
 properties:
   compatible:
-    const: ti,bq25703a
+    enum:
+      - ti,bq25703a
+      - ti,bq25792
 
   reg:
     const: 0x6b
@@ -25,7 +24,6 @@ properties:
       powering the device.
     minimum: 50000
     maximum: 6400000
-    default: 3250000
 
   interrupts:
     maxItems: 1
@@ -57,11 +55,11 @@ properties:
             minimum: 0
             maximum: 6350000
           regulator-min-microvolt:
-            minimum: 4480000
-            maximum: 20800000
+            minimum: 2800000
+            maximum: 22000000
           regulator-max-microvolt:
-            minimum: 4480000
-            maximum: 20800000
+            minimum: 2800000
+            maximum: 22000000
           enable-gpios:
             description:
               The BQ25703 may require both a register write and a GPIO
@@ -74,6 +72,61 @@ properties:
           - regulator-min-microvolt
           - regulator-max-microvolt
 
+allOf:
+  - $ref: /schemas/power/supply/power-supply.yaml#
+  - if:
+      properties:
+        compatible:
+          const: ti,bq25703a
+    then:
+      properties:
+        input-current-limit-microamp:
+          minimum: 50000
+          maximum: 6400000
+          default: 3250000
+        regulators:
+          properties:
+            vbus:
+              properties:
+                regulator-min-microamp:
+                  minimum: 0
+                  maximum: 6350000
+                regulator-max-microamp:
+                  minimum: 0
+                  maximum: 6350000
+                regulator-min-microvolt:
+                  minimum: 4480000
+                  maximum: 20800000
+                regulator-max-microvolt:
+                  minimum: 4480000
+                  maximum: 20800000
+  - if:
+      properties:
+        compatible:
+          const: ti,bq25792
+    then:
+      properties:
+        input-current-limit-microamp:
+          minimum: 100000
+          maximum: 3300000
+          default: 3000000
+        regulators:
+          properties:
+            vbus:
+              properties:
+                regulator-min-microamp:
+                  minimum: 0
+                  maximum: 3320000
+                regulator-max-microamp:
+                  minimum: 0
+                  maximum: 3320000
+                regulator-min-microvolt:
+                  minimum: 2800000
+                  maximum: 22000000
+                regulator-max-microvolt:
+                  minimum: 2800000
+                  maximum: 22000000
+
 unevaluatedProperties: false
 
 required:
diff --git a/Documentation/devicetree/bindings/misc/fsl,qoriq-mc.yaml b/Documentation/devicetree/bindings/misc/fsl,qoriq-mc.yaml
index df45ff56d4445..801f305e03746 100644
--- a/Documentation/devicetree/bindings/misc/fsl,qoriq-mc.yaml
+++ b/Documentation/devicetree/bindings/misc/fsl,qoriq-mc.yaml
@@ -166,7 +166,7 @@ examples:
          */
         ranges = <0x0 0x0 0x8 0x0c000000 0x4000000
                  0x1 0x0 0x8 0x18000000 0x8000000>;
-        
+
         /* define map for ICIDs 23-64 */
         iommu-map = <23 &smmu 23 41>;
         /* define msi map for ICIDs 23-64 */
diff --git a/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml b/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml
index ca830dd06de2d..2876fdd7c6e66 100644
--- a/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml
+++ b/Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml
@@ -25,6 +25,7 @@ properties:
       - items:
           - enum:
               - qcom,glymur-fastrpc
+              - qcom,hawi-fastrpc
           - const: qcom,kaanapali-fastrpc
 
   label:
diff --git a/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml b/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml
index 41c9b22523e7d..e447579e0f22e 100644
--- a/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml
+++ b/Documentation/devicetree/bindings/mmc/hisilicon,hi3798cv200-dw-mshc.yaml
@@ -39,10 +39,11 @@ properties:
     $ref: /schemas/types.yaml#/definitions/phandle-array
     description: |
       DWMMC core on Hi3798MV2x SoCs has a delay-locked-loop(DLL) attached to card data input path.
-      It is integrated into CRG core on the SoC and has to be controlled during tuning.
+      It is integrated into CRG core on the SoC and has to be controlled during tuning
     items:
-      - description: A phandle pointed to the CRG syscon node
-      - description: Sample DLL register offset in CRG address space
+      - items:
+          - description: A phandle pointed to the CRG syscon node
+          - description: Sample DLL register offset in CRG address space
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml
index 64fac0d11329a..4d66966ce2900 100644
--- a/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml
+++ b/Documentation/devicetree/bindings/mmc/renesas,sdhi.yaml
@@ -52,6 +52,7 @@ properties:
               - renesas,sdhi-r8a77980  # R-Car V3H
               - renesas,sdhi-r8a77990  # R-Car E3
               - renesas,sdhi-r8a77995  # R-Car D3
+              - renesas,sdhi-r8a779md  # R-Car M3Le
           - const: renesas,rcar-gen3-sdhi # R-Car Gen3 or RZ/G2
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/mmc/spacemit,sdhci.yaml b/Documentation/devicetree/bindings/mmc/spacemit,sdhci.yaml
index 9a055d963a7f0..34d202af909f2 100644
--- a/Documentation/devicetree/bindings/mmc/spacemit,sdhci.yaml
+++ b/Documentation/devicetree/bindings/mmc/spacemit,sdhci.yaml
@@ -44,6 +44,18 @@ properties:
       - const: axi
       - const: sdh
 
+  pinctrl-names:
+    minItems: 1
+    items:
+      - const: default
+      - const: uhs
+
+  pinctrl-0:
+    description: Default pinctrl state for 3.3V operation
+
+  pinctrl-1:
+    description: Optional pinctrl state for 1.8V UHS operation with "uhs" name
+
 required:
   - compatible
   - reg
@@ -62,4 +74,7 @@ examples:
       interrupt-parent = <&plic>;
       clocks = <&clk_apmu 10>, <&clk_apmu 13>;
       clock-names = "core", "io";
+      pinctrl-names = "default", "uhs";
+      pinctrl-0 = <&sdhci_default_cfg>;
+      pinctrl-1 = <&sdhci_uhs_cfg>;
     };
diff --git a/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6855-bt.yaml b/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6855-bt.yaml
index 45630067d3c8e..0beda26ae8bb4 100644
--- a/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6855-bt.yaml
+++ b/Documentation/devicetree/bindings/net/bluetooth/qcom,wcn6855-bt.yaml
@@ -50,9 +50,6 @@ properties:
     description: VDD_RFA_1P7 supply regulator handle
     deprecated: true
 
-  vddrfa1p8-supply:
-    description: VDD_RFA_1P8 supply regulator handle
-
   vddrfacmn-supply:
     description: VDD_RFA_CMN supply regulator handle
 
@@ -68,7 +65,7 @@ required:
   - vddbtcmx-supply
   - vddrfa0p8-supply
   - vddrfa1p2-supply
-  - vddrfa1p8-supply
+  - vddrfa1p7-supply
   - vddrfacmn-supply
   - vddwlcx-supply
   - vddwlmx-supply
@@ -91,7 +88,7 @@ examples:
             vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>;
             vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
             vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-            vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+            vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
             vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>;
             vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
             vddwlmx-supply = <&vreg_pmu_wlmx_0p8>;
diff --git a/Documentation/devicetree/bindings/net/mediatek,net.yaml b/Documentation/devicetree/bindings/net/mediatek,net.yaml
index cc346946291af..6bbd83c6aaf74 100644
--- a/Documentation/devicetree/bindings/net/mediatek,net.yaml
+++ b/Documentation/devicetree/bindings/net/mediatek,net.yaml
@@ -67,8 +67,7 @@ properties:
       - const: ppe
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
-    description: phandle to mmio SRAM
+    maxItems: 1
 
   mediatek,ethsys:
     $ref: /schemas/types.yaml#/definitions/phandle
diff --git a/Documentation/devicetree/bindings/net/microchip,lan95xx.yaml b/Documentation/devicetree/bindings/net/microchip,lan95xx.yaml
index accff93d38f80..b9c3940090402 100644
--- a/Documentation/devicetree/bindings/net/microchip,lan95xx.yaml
+++ b/Documentation/devicetree/bindings/net/microchip,lan95xx.yaml
@@ -20,6 +20,8 @@ properties:
   compatible:
     items:
       - enum:
+          - usb424,7500   # SMSC LAN7500 USB Ethernet Device
+          - usb424,7505   # SMSC LAN7505 USB Ethernet Device
           - usb424,9500   # SMSC9500 USB Ethernet Device
           - usb424,9505   # SMSC9505 USB Ethernet Device
           - usb424,9530   # SMSC LAN9530 USB Ethernet Device
diff --git a/Documentation/devicetree/bindings/net/microchip,lan966x-switch.yaml b/Documentation/devicetree/bindings/net/microchip,lan966x-switch.yaml
index 306ef9ecf2b91..0f0f35865ef41 100644
--- a/Documentation/devicetree/bindings/net/microchip,lan966x-switch.yaml
+++ b/Documentation/devicetree/bindings/net/microchip,lan966x-switch.yaml
@@ -17,7 +17,7 @@ description: |
 
 properties:
   $nodename:
-    pattern: "^switch@[0-9a-f]+$"
+    pattern: "^(ethernet-)?switch@[0-9a-f]+$"
 
   compatible:
     const: microchip,lan966x-switch
@@ -70,7 +70,7 @@ properties:
     additionalProperties: false
 
     patternProperties:
-      "^port@[0-9a-f]+$":
+      "^(ethernet-)?port@[0-9a-f]+$":
         type: object
 
         $ref: /schemas/net/ethernet-controller.yaml#
@@ -138,7 +138,7 @@ additionalProperties: false
 examples:
   - |
     #include <dt-bindings/interrupt-controller/arm-gic.h>
-    switch: switch@e0000000 {
+    switch: ethernet-switch@e0000000 {
       compatible = "microchip,lan966x-switch";
       reg =  <0xe0000000 0x0100000>,
              <0xe2000000 0x0800000>;
@@ -151,14 +151,14 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
 
-        port0: port@0 {
+        port0: ethernet-port@0 {
           reg = <0>;
           phy-handle = <&phy0>;
           phys = <&serdes 0 0>;
           phy-mode = "gmii";
         };
 
-        port1: port@1 {
+        port1: ethernet-port@1 {
           reg = <1>;
           sfp = <&sfp_eth1>;
           managed = "in-band-status";
diff --git a/Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml b/Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml
index b30544410d092..33746c238513d 100644
--- a/Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml
+++ b/Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml
@@ -42,7 +42,19 @@ properties:
     description: State bits used by the AP to signal the modem.
     items:
       - description: Power control
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
       - description: Power control acknowledgment
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: Names for the state bits used by the AP to signal the modem.
diff --git a/Documentation/devicetree/bindings/net/qcom,ipa.yaml b/Documentation/devicetree/bindings/net/qcom,ipa.yaml
index fdeaa81b96454..68ec76fe44735 100644
--- a/Documentation/devicetree/bindings/net/qcom,ipa.yaml
+++ b/Documentation/devicetree/bindings/net/qcom,ipa.yaml
@@ -128,7 +128,19 @@ properties:
     description: State bits used in by the AP to signal the modem.
     items:
       - description: Whether the "ipa-clock-enabled" state bit is valid
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
       - description: Whether the IPA clock is enabled (if valid)
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
diff --git a/Documentation/devicetree/bindings/net/ti,icssg-prueth.yaml b/Documentation/devicetree/bindings/net/ti,icssg-prueth.yaml
index c296e57118484..883033b19b8f3 100644
--- a/Documentation/devicetree/bindings/net/ti,icssg-prueth.yaml
+++ b/Documentation/devicetree/bindings/net/ti,icssg-prueth.yaml
@@ -21,7 +21,7 @@ properties:
       - ti,am654-sr1-icssg-prueth  # for AM65x SoC family, SR1.0
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
+    maxItems: 1
     description:
       phandle to MSMC SRAM node
 
diff --git a/Documentation/devicetree/bindings/net/ti,icssm-prueth.yaml b/Documentation/devicetree/bindings/net/ti,icssm-prueth.yaml
index a98ad45ca66f2..9370c43bc66a6 100644
--- a/Documentation/devicetree/bindings/net/ti,icssm-prueth.yaml
+++ b/Documentation/devicetree/bindings/net/ti,icssm-prueth.yaml
@@ -24,7 +24,7 @@ properties:
       - ti,am3359-prueth   # for AM33x SoC family
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
+    maxItems: 1
     description:
       phandle to OCMC SRAM node
 
diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.yaml b/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.yaml
index c21d66c7cd558..d4f4d72ee0d38 100644
--- a/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/qcom,ath10k.yaml
@@ -158,7 +158,13 @@ properties:
     description: State bits used by the AP to signal the WLAN Q6.
     items:
       - description: Signal bits used to enable/disable low power mode
-                     on WCN in the case of WoW (Wake on Wireless).
+          on WCN in the case of WoW (Wake on Wireless).
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output.
diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml b/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml
index 0cc1dbf2beefa..d4aa56e2f8233 100644
--- a/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/qcom,ath11k.yaml
@@ -80,7 +80,13 @@ properties:
     description: State bits used by the AP to signal the WLAN Q6.
     items:
       - description: Signal bits used to enable/disable low power mode
-                     on WCN6750 in the case of WoW (Wake on Wireless).
+          on WCN6750 in the case of WoW (Wake on Wireless).
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output.
diff --git a/Documentation/devicetree/bindings/net/wireless/qcom,ipq5332-wifi.yaml b/Documentation/devicetree/bindings/net/wireless/qcom,ipq5332-wifi.yaml
index 37d8a0da77809..18cd91e2728cf 100644
--- a/Documentation/devicetree/bindings/net/wireless/qcom,ipq5332-wifi.yaml
+++ b/Documentation/devicetree/bindings/net/wireless/qcom,ipq5332-wifi.yaml
@@ -168,8 +168,26 @@ properties:
     description: States used by the AP to signal the remote processor
     items:
       - description: Shutdown WCSS pd
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
       - description: Stop WCSS pd
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
       - description: Spawn WCSS pd
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description:
diff --git a/Documentation/devicetree/bindings/net/wireless/st,stlc4560.yaml b/Documentation/devicetree/bindings/net/wireless/st,stlc4560.yaml
new file mode 100644
index 0000000000000..a32265c07350d
--- /dev/null
+++ b/Documentation/devicetree/bindings/net/wireless/st,stlc4560.yaml
@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/net/wireless/st,stlc4560.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ST/Intersil/Conexant stlc45xx/p54spi/cx3110x SPI wireless device
+
+maintainers:
+  - Christian Lamparter <chunkeey@gmail.com>
+
+description:
+  The SPI variant of the Intersil Prism54 wireless device was sold
+  under a variety of names, including Conexant CX3110x and
+  ST Microelectronics STLC5460.
+
+allOf:
+  - $ref: ieee80211.yaml#
+  - $ref: /schemas/spi/spi-peripheral-props.yaml#
+
+properties:
+  compatible:
+    oneOf:
+      - const: st,stlc4560
+      - items:
+          - enum:
+              - cnxt,3110x
+              - isil,p54spi
+              - st,stlc4550
+          - const: st,stlc4560
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  powerdown-gpios:
+    maxItems: 1
+
+required:
+  - compatible
+  - reg
+  - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    spi {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        wifi@0 {
+            compatible = "st,stlc4560";
+            reg = <0>;
+            spi-max-frequency = <48000000>;
+            interrupts-extended = <&gpio 23>;
+            powerdown-gpios = <&gpio 1>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml b/Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml
new file mode 100644
index 0000000000000..c52f8d4bec392
--- /dev/null
+++ b/Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml
@@ -0,0 +1,67 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/nvmem/airoha,smc-efuses.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Airoha SMC eFuses
+
+description: |
+  Airoha new SoC AN7581 expose banks of eFuse accessible
+  via specific SMC commands.
+
+  2 different bank of eFuse or 64 cells of 32 bit are exposed
+  read-only used to give information on HW Revision, PHY Calibration,
+  Device Model, Private Key...
+
+maintainers:
+  - Christian Marangi <ansuelsmth@gmail.com>
+
+properties:
+  compatible:
+    enum:
+      - airoha,an7581-efuses
+
+  "#address-cells":
+    const: 1
+
+  "#size-cells":
+    const: 0
+
+patternProperties:
+  '^efuse-bank@[0-1]$':
+    type: object
+
+    allOf:
+      - $ref: nvmem.yaml#
+
+    properties:
+      reg:
+        description: Identify the eFuse bank.
+        enum: [0, 1]
+
+    required:
+      - reg
+
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - '#address-cells'
+  - '#size-cells'
+
+additionalProperties: false
+
+examples:
+  - |
+    efuse {
+        compatible = "airoha,an7581-efuses";
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        efuse-bank@0 {
+           reg = <0>;
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml b/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml
index f97c6beb4766a..c03e96afe5641 100644
--- a/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml
+++ b/Documentation/devicetree/bindings/nvmem/microchip,lan9662-otpc.yaml
@@ -25,6 +25,7 @@ properties:
           - const: microchip,lan9662-otpc
       - enum:
           - microchip,lan9662-otpc
+          - microchip,lan9691-otpc
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
index 2ab047f2bb69d..8134ddb54e130 100644
--- a/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
+++ b/Documentation/devicetree/bindings/nvmem/qcom,qfprom.yaml
@@ -19,6 +19,8 @@ properties:
       - enum:
           - qcom,apq8064-qfprom
           - qcom,apq8084-qfprom
+          - qcom,glymur-qfprom
+          - qcom,hawi-qfprom
           - qcom,ipq5018-qfprom
           - qcom,ipq5332-qfprom
           - qcom,ipq5424-qfprom
@@ -27,6 +29,7 @@ properties:
           - qcom,ipq8074-qfprom
           - qcom,ipq9574-qfprom
           - qcom,kaanapali-qfprom
+          - qcom,milos-qfprom
           - qcom,msm8226-qfprom
           - qcom,msm8916-qfprom
           - qcom,msm8917-qfprom
@@ -48,6 +51,7 @@ properties:
           - qcom,sdm630-qfprom
           - qcom,sdm670-qfprom
           - qcom,sdm845-qfprom
+          - qcom,shikra-qfprom
           - qcom,sm6115-qfprom
           - qcom,sm6350-qfprom
           - qcom,sm6375-qfprom
diff --git a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml
index 9d1349855b422..e8b8131f5f23b 100644
--- a/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/fsl,imx6q-pcie.yaml
@@ -66,16 +66,34 @@ properties:
       - const: dma
 
   reset-gpio:
+    deprecated: true
     description: Should specify the GPIO for controlling the PCI bus device
       reset signal. It's not polarity aware and defaults to active-low reset
       sequence (L=reset state, H=operation state) (optional required).
+      This property is deprecated, instead of referencing this property from the
+      host bridge node, use the reset-gpios property from the root port node.
 
   reset-gpio-active-high:
+    deprecated: true
     description: If present then the reset sequence using the GPIO
       specified in the "reset-gpio" property is reversed (H=reset state,
       L=operation state) (optional required).
+      This property is deprecated along with the reset-gpio property above, use
+      the reset-gpios property from the root port node.
     type: boolean
 
+  pcie@0:
+    description:
+      Describe the i.MX6 PCIe Root Port.
+    type: object
+    $ref: /schemas/pci/pci-pci-bridge.yaml#
+
+    properties:
+      reg:
+        maxItems: 1
+
+    unevaluatedProperties: false
+
 required:
   - compatible
   - reg
@@ -236,6 +254,7 @@ unevaluatedProperties: false
 examples:
   - |
     #include <dt-bindings/clock/imx6qdl-clock.h>
+    #include <dt-bindings/gpio/gpio.h>
     #include <dt-bindings/interrupt-controller/arm-gic.h>
 
     pcie: pcie@1ffc000 {
@@ -262,5 +281,18 @@ examples:
                 <&clks IMX6QDL_CLK_LVDS1_GATE>,
                 <&clks IMX6QDL_CLK_PCIE_REF_125M>;
         clock-names = "pcie", "pcie_bus", "pcie_phy";
+
+        pcie_port0: pcie@0 {
+            compatible = "pciclass,0604";
+            device_type = "pci";
+            reg = <0x0 0x0 0x0 0x0 0x0>;
+            bus-range = <0x01 0xff>;
+
+            #address-cells = <3>;
+            #size-cells = <2>;
+            ranges;
+
+            reset-gpios = <&gpio7 12 GPIO_ACTIVE_LOW>;
+        };
     };
 ...
diff --git a/Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml b/Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml
index 54e2890ae6314..394bb46b38e60 100644
--- a/Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml
@@ -27,16 +27,20 @@ properties:
       - const: snps,dw-pcie
 
   reg:
+    minItems: 3
     items:
       - description: Controller control and status registers.
       - description: PCIe configuration registers.
       - description: Controller application registers.
+      - description: Internal Address Translation Unit (iATU) registers.
 
   reg-names:
+    minItems: 3
     items:
       - const: dbi
       - const: config
       - const: app
+      - const: atu
 
   ranges:
     maxItems: 1
@@ -95,8 +99,9 @@ examples:
       #size-cells = <2>;
       reg = <0xd0e00000 0x1000>,
             <0xd2000000 0x800000>,
-            <0xd0a41000 0x1000>;
-      reg-names = "dbi", "config", "app";
+            <0xd0a41000 0x1000>,
+            <0xd0ec0000 0x1000>;
+      reg-names = "dbi", "config", "app", "atu";
       linux,pci-domain = <0>;
       max-link-speed = <4>;
       bus-range = <0x00 0x08>;
diff --git a/Documentation/devicetree/bindings/pci/mediatek-pcie-mt7623.yaml b/Documentation/devicetree/bindings/pci/mediatek-pcie-mt7623.yaml
index e33bcc216e30c..542252168388b 100644
--- a/Documentation/devicetree/bindings/pci/mediatek-pcie-mt7623.yaml
+++ b/Documentation/devicetree/bindings/pci/mediatek-pcie-mt7623.yaml
@@ -125,8 +125,8 @@ examples:
             phy-names = "pcie-phy0", "pcie-phy1", "pcie-phy2";
             power-domains = <&scpsys MT2701_POWER_DOMAIN_HIF>;
             bus-range = <0x00 0xff>;
-            ranges = <0x81000000 0 0x1a160000 0 0x1a160000 0 0x00010000>,	/* I/O space */
-                     <0x83000000 0 0x60000000 0 0x60000000 0 0x10000000>;	/* memory space */
+            ranges = <0x81000000 0 0x1a160000 0 0x1a160000 0 0x00010000>, /* I/O space */
+                     <0x83000000 0 0x60000000 0 0x60000000 0 0x10000000>; /* memory space */
 
             pcie@0,0 {
                 device_type = "pci";
diff --git a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
index a67108c48feb8..90086909e9210 100644
--- a/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/renesas,r9a08g045-pcie.yaml
@@ -4,21 +4,27 @@
 $id: http://devicetree.org/schemas/pci/renesas,r9a08g045-pcie.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Renesas RZ/G3S PCIe host controller
+title: Renesas RZ/G3S PCIe host controller (and similar SoCs)
 
 maintainers:
   - Claudiu Beznea <claudiu.beznea.uj@bp.renesas.com>
 
-description:
-  Renesas RZ/G3{E,S} PCIe host controllers comply with PCIe
-  Base Specification 4.0 and support up to 5 GT/s (Gen2) for RZ/G3S and
-  up to 8 GT/s (Gen3) for RZ/G3E.
+description: |
+  PCIe host controller found in Renesas RZ/G3S and similar SoCs complies
+  with PCIe Base Specification 4.0 and supports different link speeds
+  depending on the SoC variant:
+    - Gen2 (5 GT/s): RZ/G3S
+    - Gen3 (8 GT/s): RZ/G3E, RZ/V2N
 
 properties:
   compatible:
-    enum:
-      - renesas,r9a08g045-pcie # RZ/G3S
-      - renesas,r9a09g047-pcie # RZ/G3E
+    oneOf:
+      - enum:
+          - renesas,r9a08g045-pcie # RZ/G3S
+          - renesas,r9a09g047-pcie # RZ/G3E
+      - items:
+          - const: renesas,r9a09g056-pcie # RZ/V2N
+          - const: renesas,r9a09g047-pcie
 
   reg:
     maxItems: 1
@@ -152,6 +158,7 @@ patternProperties:
         enum:
           - 0x0033
           - 0x0039
+          - 0x003b
 
       clocks:
         items:
diff --git a/Documentation/devicetree/bindings/pci/samsung,exynos-pcie.yaml b/Documentation/devicetree/bindings/pci/samsung,exynos-pcie.yaml
index f20ed7e709f7c..57a8f163dda59 100644
--- a/Documentation/devicetree/bindings/pci/samsung,exynos-pcie.yaml
+++ b/Documentation/devicetree/bindings/pci/samsung,exynos-pcie.yaml
@@ -109,7 +109,7 @@ examples:
         num-lanes = <1>;
         num-viewport = <3>;
         bus-range = <0x00 0xff>;
-        ranges = <0x81000000 0 0	  0x0c001000 0 0x00010000>,
+        ranges = <0x81000000 0 0   0x0c001000 0 0x00010000>,
                  <0x82000000 0 0x0c011000 0x0c011000 0 0x03feefff>;
         vdd10-supply = <&ldo6_reg>;
         vdd18-supply = <&ldo7_reg>;
diff --git a/Documentation/devicetree/bindings/pci/sophgo,sg2042-pcie-host.yaml b/Documentation/devicetree/bindings/pci/sophgo,sg2042-pcie-host.yaml
index f8b7ca57fff14..ab482488b0475 100644
--- a/Documentation/devicetree/bindings/pci/sophgo,sg2042-pcie-host.yaml
+++ b/Documentation/devicetree/bindings/pci/sophgo,sg2042-pcie-host.yaml
@@ -30,6 +30,8 @@ properties:
   device-id:
     const: 0x2042
 
+  dma-coherent: true
+
   msi-parent: true
 
 allOf:
@@ -60,5 +62,6 @@ examples:
       vendor-id = <0x1f1c>;
       device-id = <0x2042>;
       cdns,no-bar-match-nbits = <48>;
+      dma-coherent;
       msi-parent = <&msi>;
     };
diff --git a/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml b/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml
index fae4660647809..b3ad05d90201c 100644
--- a/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml
+++ b/Documentation/devicetree/bindings/pci/toshiba,tc9563.yaml
@@ -49,8 +49,9 @@ properties:
       A phandle to the parent I2C node and the slave address of the device
       used to configure tc9563 to change FTS, tx amplitude etc.
     items:
-      - description: Phandle to the I2C controller node
-      - description: I2C slave address
+      - items:
+          - description: Phandle to the I2C controller node
+          - description: I2C slave address
 
 patternProperties:
   "^pcie@[1-3],0$":
diff --git a/Documentation/devicetree/bindings/pci/ultrarisc,dp1000-pcie.yaml b/Documentation/devicetree/bindings/pci/ultrarisc,dp1000-pcie.yaml
new file mode 100644
index 0000000000000..512b935bf5d1e
--- /dev/null
+++ b/Documentation/devicetree/bindings/pci/ultrarisc,dp1000-pcie.yaml
@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pci/ultrarisc,dp1000-pcie.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: UltraRISC DP1000 PCIe Host Controller
+
+description:
+  UltraRISC DP1000 SoC PCIe host controller is based on the DesignWare PCIe IP.
+
+maintainers:
+  - Xincheng Zhang <zhangxincheng@ultrarisc.com>
+  - Jia Wang <wangjia@ultrarisc.com>
+
+allOf:
+  - $ref: /schemas/pci/snps,dw-pcie.yaml#
+
+properties:
+  compatible:
+    const: ultrarisc,dp1000-pcie
+
+  reg:
+    items:
+      - description: Data Bus Interface (DBI) registers.
+      - description: PCIe configuration space region.
+
+  reg-names:
+    items:
+      - const: dbi
+      - const: config
+
+  num-lanes:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [4, 16]
+    description: Number of lanes to use.
+
+  interrupts:
+    items:
+      - description: MSI interrupt
+      - description: Legacy INTA interrupt
+      - description: Legacy INTB interrupt
+      - description: Legacy INTC interrupt
+      - description: Legacy INTD interrupt
+
+  interrupt-names:
+    items:
+      - const: msi
+      - const: inta
+      - const: intb
+      - const: intc
+      - const: intd
+
+required:
+  - compatible
+  - reg
+  - reg-names
+  - interrupts
+  - interrupt-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    soc {
+      #address-cells = <2>;
+      #size-cells = <2>;
+
+      pcie@21000000 {
+        compatible = "ultrarisc,dp1000-pcie";
+        reg = <0x0 0x21000000 0x0 0x01000000>,
+              <0x0 0x4fff0000 0x0 0x00010000>;
+        reg-names = "dbi", "config";
+        ranges = <0x81000000 0x0 0x4fbf0000 0x0 0x4fbf0000 0x0 0x00400000>,
+                 <0x82000000 0x0 0x40000000 0x0 0x40000000 0x0 0x0fbf0000>,
+                 <0xc3000000 0x40 0x00000000 0x40 0x00000000 0xd 0x00000000>;
+        #address-cells = <3>;
+        #size-cells = <2>;
+        #interrupt-cells = <1>;
+        device_type = "pci";
+        dma-coherent;
+        bus-range = <0x0 0xff>;
+        num-lanes = <16>;
+        interrupt-parent = <&plic>;
+        interrupts = <43>, <44>, <45>, <46>, <47>;
+        interrupt-names = "msi", "inta", "intb", "intc", "intd";
+        interrupt-map-mask = <0x0 0x0 0x0 0x7>;
+        interrupt-map = <0x0 0x0 0x0 0x1 &plic 44>,
+                        <0x0 0x0 0x0 0x2 &plic 45>,
+                        <0x0 0x0 0x0 0x3 &plic 46>,
+                        <0x0 0x0 0x0 0x4 &plic 47>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/phy/axiado,ax3000-emmc-phy.yaml b/Documentation/devicetree/bindings/phy/axiado,ax3000-emmc-phy.yaml
new file mode 100644
index 0000000000000..61700b80e93f7
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/axiado,ax3000-emmc-phy.yaml
@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/phy/axiado,ax3000-emmc-phy.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Axiado AX3000 Arasan eMMC PHY
+
+maintainers:
+  - SriNavmani A <srinavmani@axiado.com>
+  - Tzu-Hao Wei <twei@axiado.com>
+  - Prasad Bolisetty <pbolisetty@axiado.com>
+
+properties:
+  compatible:
+    const: axiado,ax3000-emmc-phy
+
+  reg:
+    maxItems: 1
+
+  "#phy-cells":
+    const: 0
+
+required:
+  - compatible
+  - reg
+  - "#phy-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    phy@80801c00 {
+        compatible = "axiado,ax3000-emmc-phy";
+        reg = <0x80801c00 0x1000>;
+        #phy-cells = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/phy/econet,en751221-pcie-phy.yaml b/Documentation/devicetree/bindings/phy/econet,en751221-pcie-phy.yaml
new file mode 100644
index 0000000000000..987d396c1c642
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/econet,en751221-pcie-phy.yaml
@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/phy/econet,en751221-pcie-phy.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: EcoNet PCI-Express PHY for EcoNet EN751221 and EN7528
+
+maintainers:
+  - Caleb James DeLisle <cjd@cjdns.fr>
+
+description:
+  The PCIe PHY supports physical layer functionality for PCIe Gen1 and
+  Gen1/Gen2 ports. On these SoCs, port 0 is a Gen1-only port while
+  port 1 is Gen1/Gen2 capable.
+
+properties:
+  compatible:
+    enum:
+      - econet,en751221-pcie-gen1
+      - econet,en751221-pcie-gen2
+      - econet,en7528-pcie-gen1
+      - econet,en7528-pcie-gen2
+
+  reg:
+    maxItems: 1
+
+  "#phy-cells":
+    const: 0
+
+required:
+  - compatible
+  - reg
+  - "#phy-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    soc {
+      #address-cells = <1>;
+      #size-cells = <1>;
+
+      pcie-phy@1faf2000 {
+        compatible = "econet,en7528-pcie-gen1";
+        reg = <0x1faf2000 0x1000>;
+        #phy-cells = <0>;
+      };
+    };
+...
diff --git a/Documentation/devicetree/bindings/phy/fsl,lynx-28g.yaml b/Documentation/devicetree/bindings/phy/fsl,lynx-28g.yaml
index e96229c2f8fb7..d73591315d4b9 100644
--- a/Documentation/devicetree/bindings/phy/fsl,lynx-28g.yaml
+++ b/Documentation/devicetree/bindings/phy/fsl,lynx-28g.yaml
@@ -9,10 +9,37 @@ title: Freescale Lynx 28G SerDes PHY
 maintainers:
   - Ioana Ciornei <ioana.ciornei@nxp.com>
 
+description:
+  The Lynx 28G is a multi-lane, multi-protocol SerDes (PCIe, SATA, Ethernet)
+  present in multiple instances on NXP LX2160A and LX2162A SoCs. All instances
+  share a common register map and programming model, however they differ in
+  supported protocols per lane in a way that is not detectable by said
+  programming model without prior knowledge. The distinction is made through
+  the compatible string.
+
 properties:
   compatible:
-    enum:
-      - fsl,lynx-28g
+    oneOf:
+      - const: fsl,lynx-28g
+        deprecated: true
+        description:
+          Legacy compatibility string for Lynx 28G SerDes. Any assumption
+          regarding whether a certain lane supports a certain protocol may
+          be incorrect. Deprecated except when used as a fallback. Use
+          device-specific strings instead.
+      - items:
+          - const: fsl,lx2160a-serdes1
+          - const: fsl,lynx-28g
+      - items:
+          - const: fsl,lx2160a-serdes2
+          - const: fsl,lynx-28g
+      - items:
+          - const: fsl,lx2162a-serdes1
+          - const: fsl,lynx-28g
+      - items:
+          - const: fsl,lx2162a-serdes2
+          - const: fsl,lynx-28g
+      - const: fsl,lx2160a-serdes3
 
   reg:
     maxItems: 1
@@ -51,6 +78,21 @@ required:
   - reg
   - "#phy-cells"
 
+allOf:
+  # LX2162A SerDes 1 has fewer lanes than the others
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: fsl,lx2162a-serdes1
+    then:
+      patternProperties:
+        "^phy@[0-7]$":
+          properties:
+            reg:
+              minimum: 4
+              maximum: 7
+
 additionalProperties: false
 
 examples:
@@ -60,7 +102,7 @@ examples:
       #size-cells = <2>;
 
       serdes@1ea0000 {
-        compatible = "fsl,lynx-28g";
+        compatible = "fsl,lx2160a-serdes1", "fsl,lynx-28g";
         reg = <0x0 0x1ea0000 0x0 0x1e30>;
         #address-cells = <1>;
         #size-cells = <0>;
diff --git a/Documentation/devicetree/bindings/phy/qcom,dsi-phy-7nm.yaml b/Documentation/devicetree/bindings/phy/qcom,dsi-phy-7nm.yaml
index 966c70d746aa4..f397ba3fa84ac 100644
--- a/Documentation/devicetree/bindings/phy/qcom,dsi-phy-7nm.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,dsi-phy-7nm.yaml
@@ -20,6 +20,7 @@ properties:
               - qcom,dsi-phy-7nm
               - qcom,dsi-phy-7nm-8150
               - qcom,kaanapali-dsi-phy-3nm
+              - qcom,milos-dsi-phy-4nm
               - qcom,sa8775p-dsi-phy-5nm
               - qcom,sar2130p-dsi-phy-5nm
               - qcom,sc7280-dsi-phy-7nm
diff --git a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml
index 4a1daae3d8d47..0bf8bf4f66acf 100644
--- a/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,edp-phy.yaml
@@ -74,6 +74,7 @@ allOf:
         compatible:
           enum:
             - qcom,glymur-dp-phy
+            - qcom,sa8775p-edp-phy
             - qcom,x1e80100-dp-phy
     then:
       properties:
diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml
index 9616c736b6d46..b2c5c9a375a35 100644
--- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml
@@ -31,6 +31,7 @@ properties:
       - items:
           - enum:
               - qcom,eliza-qmp-ufs-phy
+              - qcom,nord-qmp-ufs-phy
           - const: qcom,sm8650-qmp-ufs-phy
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml
index 3d537b7f99859..4eff92343ce4f 100644
--- a/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml
@@ -16,6 +16,10 @@ description:
 properties:
   compatible:
     oneOf:
+      - items:
+          - enum:
+              - qcom,eliza-qmp-usb3-dp-phy
+          - const: qcom,sm8650-qmp-usb3-dp-phy
       - items:
           - enum:
               - qcom,kaanapali-qmp-usb3-dp-phy
diff --git a/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml
index 854f70af0a6c1..096f6b546632e 100644
--- a/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/qcom,snps-eusb2-phy.yaml
@@ -17,6 +17,7 @@ properties:
     oneOf:
       - items:
           - enum:
+              - qcom,eliza-snps-eusb2-phy
               - qcom,milos-snps-eusb2-phy
               - qcom,sar2130p-snps-eusb2-phy
               - qcom,sdx75-snps-eusb2-phy
diff --git a/Documentation/devicetree/bindings/phy/rockchip,inno-usb2phy.yaml b/Documentation/devicetree/bindings/phy/rockchip,inno-usb2phy.yaml
index 58e735b5dd05a..f50fc69fbbe49 100644
--- a/Documentation/devicetree/bindings/phy/rockchip,inno-usb2phy.yaml
+++ b/Documentation/devicetree/bindings/phy/rockchip,inno-usb2phy.yaml
@@ -20,6 +20,7 @@ properties:
       - rockchip,rk3328-usb2phy
       - rockchip,rk3366-usb2phy
       - rockchip,rk3399-usb2phy
+      - rockchip,rk3528-usb2phy
       - rockchip,rk3562-usb2phy
       - rockchip,rk3568-usb2phy
       - rockchip,rk3576-usb2phy
@@ -41,11 +42,15 @@ properties:
     maxItems: 3
 
   clock-names:
-    minItems: 1
-    items:
+    oneOf:
       - const: phyclk
-      - const: aclk
-      - const: aclk_slv
+      - items:
+          - const: phyclk
+          - const: pclk
+      - items:
+          - const: phyclk
+          - const: aclk
+          - const: aclk_slv
 
   assigned-clocks:
     description:
@@ -65,6 +70,9 @@ properties:
     description: Muxed interrupt for both ports
     maxItems: 1
 
+  power-domains:
+    maxItems: 1
+
   resets:
     maxItems: 2
 
@@ -145,6 +153,21 @@ anyOf:
       - host-port
 
 allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - rockchip,rk3528-usb2phy
+              - rockchip,rk3568-usb2phy
+              - rockchip,rv1108-usb2phy
+    then:
+      required:
+        - rockchip,usbgrf
+    else:
+      properties:
+        rockchip,usbgrf: false
+
   - if:
       properties:
         compatible:
@@ -204,6 +227,19 @@ allOf:
         clock-names:
           maxItems: 1
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - rockchip,rk3528-usb2phy
+    then:
+      properties:
+        clocks:
+          minItems: 2
+        clock-names:
+          minItems: 2
+
   - if:
       properties:
         compatible:
diff --git a/Documentation/devicetree/bindings/phy/spacemit,usb2-phy.yaml b/Documentation/devicetree/bindings/phy/spacemit,usb2-phy.yaml
index 43eaca90d88c0..18025e5f60d69 100644
--- a/Documentation/devicetree/bindings/phy/spacemit,usb2-phy.yaml
+++ b/Documentation/devicetree/bindings/phy/spacemit,usb2-phy.yaml
@@ -4,14 +4,16 @@
 $id: http://devicetree.org/schemas/phy/spacemit,usb2-phy.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: SpacemiT K1 SoC USB 2.0 PHY
+title: SpacemiT K1/K3 SoC USB 2.0 PHY
 
 maintainers:
   - Ze Huang <huang.ze@linux.dev>
 
 properties:
   compatible:
-    const: spacemit,k1-usb2-phy
+    enum:
+      - spacemit,k1-usb2-phy
+      - spacemit,k3-usb2-phy
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/phy/ti,ds125df111.yaml b/Documentation/devicetree/bindings/phy/ti,ds125df111.yaml
new file mode 100644
index 0000000000000..ca4605f1d6641
--- /dev/null
+++ b/Documentation/devicetree/bindings/phy/ti,ds125df111.yaml
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/phy/ti,ds125df111.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: TI DS125DF111 Retimer PHY
+
+description:
+  This binding describes the TI DS125DF111 multi-protocol Retimer PHY.
+
+maintainers:
+  - Ioana Ciornei <ioana.ciornei@nxp.com>
+
+properties:
+  compatible:
+    const: ti,ds125df111
+
+  reg:
+    maxItems: 1
+
+  "#phy-cells":
+    const: 1
+    description: |
+      The phandle's argument in the PHY specifier selects one of the two
+      channels of the retimer
+
+required:
+  - compatible
+  - reg
+  - "#phy-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    i2c {
+      #address-cells = <1>;
+      #size-cells = <0>;
+
+      phy@18 {
+        compatible = "ti,ds125df111";
+        reg = <0x18>;
+        #phy-cells = <1>;
+      };
+    };
diff --git a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml
index be41b4547ec6d..60b644a4c6390 100644
--- a/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml
+++ b/Documentation/devicetree/bindings/phy/ti,phy-gmii-sel.yaml
@@ -47,15 +47,20 @@ description: |
 
 properties:
   compatible:
-    enum:
-      - ti,am3352-phy-gmii-sel
-      - ti,dra7xx-phy-gmii-sel
-      - ti,am43xx-phy-gmii-sel
-      - ti,dm814-phy-gmii-sel
-      - ti,am654-phy-gmii-sel
-      - ti,j7200-cpsw5g-phy-gmii-sel
-      - ti,j721e-cpsw9g-phy-gmii-sel
-      - ti,j784s4-cpsw9g-phy-gmii-sel
+    oneOf:
+      - enum:
+          - ti,am3352-phy-gmii-sel
+          - ti,dra7xx-phy-gmii-sel
+          - ti,am43xx-phy-gmii-sel
+          - ti,dm814-phy-gmii-sel
+          - ti,am654-phy-gmii-sel
+          - ti,j7200-cpsw5g-phy-gmii-sel
+          - ti,j721e-cpsw9g-phy-gmii-sel
+          - ti,j784s4-cpsw9g-phy-gmii-sel
+      - items:
+          - enum:
+              - ti,j722s-phy-gmii-sel
+          - const: ti,am654-phy-gmii-sel
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml
index 3f16ff14484d2..0653252c18d8e 100644
--- a/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml
+++ b/Documentation/devicetree/bindings/phy/ti,phy-j721e-wiz.yaml
@@ -12,13 +12,18 @@ maintainers:
 
 properties:
   compatible:
-    enum:
-      - ti,j721e-wiz-16g
-      - ti,j721e-wiz-10g
-      - ti,j721s2-wiz-10g
-      - ti,am64-wiz-10g
-      - ti,j7200-wiz-10g
-      - ti,j784s4-wiz-10g
+    oneOf:
+      - enum:
+          - ti,j721e-wiz-16g
+          - ti,j721e-wiz-10g
+          - ti,j721s2-wiz-10g
+          - ti,am64-wiz-10g
+          - ti,j7200-wiz-10g
+          - ti,j784s4-wiz-10g
+      - items:
+          - enum:
+              - ti,j722s-wiz-10g
+          - const: ti,am64-wiz-10g
 
   power-domains:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc0-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc0-pinctrl.yaml
new file mode 100644
index 0000000000000..407a64f28d497
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc0-pinctrl.yaml
@@ -0,0 +1,188 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/aspeed,ast2700-soc0-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: ASPEED AST2700 SoC0 Pin Controller
+
+maintainers:
+  - Billy Tsai <billy_tsai@aspeedtech.com>
+
+description: >
+  The AST2700 features a dual-SoC architecture with two interconnected SoCs,
+  each having its own System Control Unit (SCU) for independent pin control.
+  This pin controller manages the pin multiplexing for SoC0.
+
+  The SoC0 pin controller manages pin functions including eMMC, VGA DDC,
+  dual USB3/USB2 ports (A and B), JTAG, and PCIe root complex interfaces.
+
+properties:
+  compatible:
+    const: aspeed,ast2700-soc0-pinctrl
+
+  reg:
+    maxItems: 1
+
+patternProperties:
+  '-state$':
+    description: |
+      Pin control state.
+
+      If 'function' is present, the node describes a pinmux state and must
+      specify 'groups'.
+
+      For pin configuration, exactly one of 'groups' or 'pins' must be
+      specified in each state node. Group-level configuration applies to all
+      pins in the group. Pin-level configuration may be supplied in a
+      separate state node for individual pins; when both group-level and
+      pin-level configuration apply to the same pin, the pin-level
+      configuration takes precedence.
+
+    type: object
+    allOf:
+      - $ref: pinmux-node.yaml#
+      - $ref: pincfg-node.yaml#
+      - if:
+          required:
+            - function
+        then:
+          required:
+            - groups
+      - oneOf:
+          - required:
+              - groups
+          - required:
+              - pins
+
+    additionalProperties: false
+
+    properties:
+      function:
+        enum:
+          - EMMC
+          - JTAGDDR
+          - JTAGM0
+          - JTAGPCIEA
+          - JTAGPCIEB
+          - JTAGPSP
+          - JTAGSSP
+          - JTAGTSP
+          - JTAGUSB3A
+          - JTAGUSB3B
+          - PCIERC0PERST
+          - PCIERC1PERST
+          - TSPRSTN
+          - UFSCLKI
+          - USB2AD0
+          - USB2AD1
+          - USB2AH
+          - USB2AHP
+          - USB2AHPD0
+          - USB2AXH
+          - USB2AXH2B
+          - USB2AXHD1
+          - USB2AXHP
+          - USB2AXHP2B
+          - USB2AXHPD1
+          - USB2BD0
+          - USB2BD1
+          - USB2BH
+          - USB2BHP
+          - USB2BHPD0
+          - USB2BXH
+          - USB2BXH2A
+          - USB2BXHD1
+          - USB2BXHP
+          - USB2BXHP2A
+          - USB2BXHPD1
+          - USB3AXH
+          - USB3AXH2B
+          - USB3AXHD
+          - USB3AXHP
+          - USB3AXHP2B
+          - USB3AXHPD
+          - USB3BXH
+          - USB3BXH2A
+          - USB3BXHD
+          - USB3BXHP
+          - USB3BXHP2A
+          - USB3BXHPD
+          - VB
+          - VGADDC
+
+      groups:
+        enum:
+          - EMMCCDN
+          - EMMCG1
+          - EMMCG4
+          - EMMCG8
+          - EMMCWPN
+          - JTAG0
+          - PCIERC0PERST
+          - PCIERC1PERST
+          - TSPRSTN
+          - UFSCLKI
+          - USB2A
+          - USB2AAP
+          - USB2ABP
+          - USB2ADAP
+          - USB2AH
+          - USB2AHAP
+          - USB2B
+          - USB2BAP
+          - USB2BBP
+          - USB2BDBP
+          - USB2BH
+          - USB2BHBP
+          - USB3A
+          - USB3AAP
+          - USB3ABP
+          - USB3B
+          - USB3BAP
+          - USB3BBP
+          - VB0
+          - VB1
+          - VGADDC
+
+      pins:
+        enum:
+          - AB13
+          - AB14
+          - AC13
+          - AC14
+          - AD13
+          - AD14
+          - AE13
+          - AE14
+          - AE15
+          - AF13
+          - AF14
+          - AF15
+
+      drive-strength:
+        enum: [3, 6, 8, 11, 16, 18, 20, 23, 30, 32, 33, 35, 37, 38, 39, 41]
+
+      bias-disable: true
+      bias-pull-up: true
+      bias-pull-down: true
+
+required:
+  - compatible
+  - reg
+
+allOf:
+  - $ref: pinctrl.yaml#
+
+additionalProperties: false
+
+examples:
+  - |
+    pinctrl@400 {
+        compatible = "aspeed,ast2700-soc0-pinctrl";
+        reg = <0x400 0x318>;
+        emmc-state {
+            function = "EMMC";
+            groups = "EMMCG1";
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt65xx-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt65xx-pinctrl.yaml
index aa71398cf522f..1468c6f87cfa6 100644
--- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt65xx-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt65xx-pinctrl.yaml
@@ -17,6 +17,7 @@ properties:
     enum:
       - mediatek,mt2701-pinctrl
       - mediatek,mt2712-pinctrl
+      - mediatek,mt6392-pinctrl
       - mediatek,mt6397-pinctrl
       - mediatek,mt7623-pinctrl
       - mediatek,mt8127-pinctrl
diff --git a/Documentation/devicetree/bindings/pinctrl/mediatek,mt8188-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8188-pinctrl.yaml
index e994b0c70dbfd..1cf06e46f7bb0 100644
--- a/Documentation/devicetree/bindings/pinctrl/mediatek,mt8188-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/mediatek,mt8188-pinctrl.yaml
@@ -67,6 +67,11 @@ properties:
 
 # PIN CONFIGURATION NODES
 patternProperties:
+  "-hog(-[0-9]+)?$":
+    type: object
+    required:
+      - gpio-hog
+
   '-pins$':
     type: object
     additionalProperties: false
diff --git a/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux-aon.yaml b/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux-aon.yaml
index db8224dfba2c1..4910dc8e8aebd 100644
--- a/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux-aon.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux-aon.yaml
@@ -58,7 +58,11 @@ patternProperties:
                     drive_soc_gpio27_pee6, drive_ao_retention_n_pee2,
                     drive_vcomp_alert_pee1, drive_hdmi_cec_pgg0 ]
 
-unevaluatedProperties: false
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux.yaml b/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux.yaml
index f5a3a881dec4f..52b3d40e8839a 100644
--- a/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/nvidia,tegra234-pinmux.yaml
@@ -115,7 +115,11 @@ patternProperties:
                     drive_sdmmc1_dat2_pj4, drive_sdmmc1_dat1_pj3,
                     drive_sdmmc1_dat0_pj2 ]
 
-unevaluatedProperties: false
+required:
+  - compatible
+  - reg
+
+additionalProperties: false
 
 examples:
   - |
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl-multiplexer.yaml b/Documentation/devicetree/bindings/pinctrl/pinctrl-multiplexer.yaml
new file mode 100644
index 0000000000000..2b0385ed879b7
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl-multiplexer.yaml
@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/pinctrl-multiplexer.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Generic pinctrl device for on-board MUX Chips
+
+maintainers:
+  - Frank Li <Frank.Li@nxp.com>
+
+description:
+  Generic pinctrl device for on-board MUX Chips, which switch SoC signals
+  between different peripherals (e.g. MMC and UART).
+
+  The MUX select lines are often driven by a I2C GPIO expander.
+
+properties:
+  compatible:
+    const: pinctrl-multiplexer
+
+patternProperties:
+  '-grp$':
+    type: object
+    additionalProperties: false
+    properties:
+      mux-states:
+        maxItems: 1
+
+    required:
+      - mux-states
+
+required:
+  - compatible
+
+allOf:
+  - $ref: pinctrl.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    pinctrl-mux {
+        compatible = "pinctrl-multiplexer";
+
+        uart-grp {
+            mux-states = <&mux 0>;
+        };
+
+        spi-grp {
+            mux-states = <&mux 1>;
+        };
+
+        i2c-grp {
+            mux-states = <&mux 2>;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/pinctrl.yaml
index 290438826c507..20176bf307475 100644
--- a/Documentation/devicetree/bindings/pinctrl/pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/pinctrl.yaml
@@ -27,7 +27,7 @@ description: |
 
 properties:
   $nodename:
-    pattern: "^(pinctrl|pinmux)(@[0-9a-f]+)?$"
+    pattern: "^(pinctrl|pinmux)(@[0-9a-f]+|-[a-z0-9]+)?$"
 
   "#pinctrl-cells":
     description: >
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,eliza-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,eliza-tlmm.yaml
index 2826504264879..fa01775292775 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,eliza-tlmm.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,eliza-tlmm.yaml
@@ -86,16 +86,22 @@ $defs:
                 qdss_gpio_tracectl, qdss_gpio_tracedata, qlink_big_enable,
                 qlink_big_request, qlink_little_enable,
                 qlink_little_request, qlink_wmss, qspi0, qspi_clk,
-                qspi_cs, qup1_se0, qup1_se1, qup1_se2, qup1_se3, qup1_se4,
-                qup1_se5, qup1_se6, qup1_se7, qup2_se0, qup2_se1,
-                qup2_se2, qup2_se3, qup2_se4, qup2_se5, qup2_se6,
-                qup2_se7, resout_gpio, sd_write_protect, sdc1, sdc2,
-                sdc2_fb_clk, tb_trig_sdc1, tb_trig_sdc2, tmess_prng0,
-                tmess_prng1, tmess_prng2, tmess_prng3, tsense_pwm1,
-                tsense_pwm2, tsense_pwm3, tsense_pwm4, uim0_clk,
-                uim0_data, uim0_present, uim0_reset, uim1_clk, uim1_data,
-                uim1_present, uim1_reset, usb0_hs, usb_phy, vfr_0, vfr_1,
-                vsense_trigger_mirnat, wcn_sw_ctrl ]
+                qspi_cs, qup1_se0, qup1_se1, qup1_se2, qup1_se2_l2_mira,
+                qup1_se2_l2_mirb, qup1_se2_l3_mira, qup1_se2_l3_mirb,
+                qup1_se3, qup1_se4_l0, qup1_se4_l1, qup1_se4_l2,
+                qup1_se4_l3, qup1_se5, qup1_se6, qup1_se6_l1_mira,
+                qup1_se6_l1_mirb, qup1_se6_l3_mira, qup1_se6_l3_mirb,
+                qup1_se7, qup1_se7_l0_mira, qup1_se7_l0_mirb,
+                qup1_se7_l1_mira, qup1_se7_l1_mirb, qup2_se0, qup2_se1,
+                qup2_se2, qup2_se3, qup2_se3_l0_mira, qup2_se3_l0_mirb,
+                qup2_se3_l1_mira, qup2_se3_l1_mirb, qup2_se4, qup2_se5,
+                qup2_se6, qup2_se7, resout_gpio, sd_write_protect, sdc1,
+                sdc2, sdc2_fb_clk, tb_trig_sdc1, tb_trig_sdc2,
+                tmess_prng0, tmess_prng1, tmess_prng2, tmess_prng3,
+                tsense_pwm1, tsense_pwm2, tsense_pwm3, tsense_pwm4,
+                uim0_clk, uim0_data, uim0_present, uim0_reset, uim1_clk,
+                uim1_data, uim1_present, uim1_reset, usb0_hs, usb_phy,
+                vfr_0, vfr_1, vsense_trigger_mirnat, wcn_sw_ctrl ]
     required:
       - pins
 
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,ipq4019-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,ipq4019-pinctrl.yaml
index cc5de9f776801..de9a3e67e1bbe 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,ipq4019-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,ipq4019-pinctrl.yaml
@@ -36,11 +36,6 @@ patternProperties:
             $ref: "#/$defs/qcom-ipq4019-tlmm-state"
         additionalProperties: false
 
-  "-hog(-[0-9]+)?$":
-    type: object
-    required:
-      - gpio-hog
-
 $defs:
   qcom-ipq4019-tlmm-state:
     type: object
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,ipq9650-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,ipq9650-tlmm.yaml
new file mode 100644
index 0000000000000..549eaa6aa11b4
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,ipq9650-tlmm.yaml
@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,ipq9650-tlmm.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm IPQ9650 TLMM pin controller
+
+maintainers:
+  - Bjorn Andersson <andersson@kernel.org>
+  - Kathiravan Thirumoorthy <kathiravan.thirumoorthy@oss.qualcomm.com>
+
+description:
+  Top Level Mode Multiplexer pin controller in Qualcomm IPQ9650 SoC.
+
+allOf:
+  - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml#
+
+properties:
+  compatible:
+    const: qcom,ipq9650-tlmm
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  gpio-reserved-ranges:
+    minItems: 1
+    maxItems: 27
+
+  gpio-line-names:
+    maxItems: 54
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-ipq9650-tlmm-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-ipq9650-tlmm-state"
+        additionalProperties: false
+
+$defs:
+  qcom-ipq9650-tlmm-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          pattern: "^gpio([0-9]|[1-4][0-9]|5[0-3])$"
+        minItems: 1
+        maxItems: 36
+
+      function:
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+
+        enum: [ atest_char_start, atest_char_status0, atest_char_status1,
+                atest_char_status2, atest_char_status3, atest_tic_en,
+                audio_pri_mclk_in0, audio_pri_mclk_out0, audio_pri_mclk_in1,
+                audio_pri_mclk_out1, audio_pri, audio_sec, audio_sec_mclk_in0,
+                audio_sec_mclk_out0, audio_sec_mclk_in1, audio_sec_mclk_out1,
+                core_voltage_0, core_voltage_1, core_voltage_2, core_voltage_3,
+                core_voltage_4, cri_rng0, cri_rng1, cri_rng2, dbg_out_clk,
+                gcc_plltest_bypassnl, gcc_plltest_resetn, gcc_tlmm, gpio,
+                mdc_mst, mdc_slv0, mdc_slv1, mdio_mst, mdio_slv, mdio_slv0,
+                mdio_slv1, pcie0_clk_req_n, pcie0_wake, pcie1_clk_req_n,
+                pcie1_wake, pcie2_clk_req_n, pcie2_wake, pcie3_clk_req_n,
+                pcie3_wake, pcie4_clk_req_n, pcie4_wake, pll_bist_sync,
+                pll_test, pwm, qdss_cti_trig_in_a0, qdss_cti_trig_in_a1,
+                qdss_cti_trig_in_b0, qdss_cti_trig_in_b1, qdss_cti_trig_out_a0,
+                qdss_cti_trig_out_a1, qdss_cti_trig_out_b0, qdss_cti_trig_out_b1,
+                qdss_traceclk_a, qdss_tracectl_a, qdss_tracedata_a, qspi_data,
+                qspi_clk, qspi_cs_n, qup_se0, qup_se1, qup_se2, qup_se3,
+                qup_se4, qup_se5, qup_se6, qup_se7, resout, rx_los0, rx_los1,
+                rx_los2, sdc_clk, sdc_cmd, sdc_data, tsens_max, tsn ]
+
+    required:
+      - pins
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    tlmm: pinctrl@1000000 {
+        compatible = "qcom,ipq9650-tlmm";
+        reg = <0x01000000 0x300000>;
+        gpio-controller;
+        #gpio-cells = <2>;
+        gpio-ranges = <&tlmm 0 0 54>;
+        interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH>;
+        interrupt-controller;
+        #interrupt-cells = <2>;
+
+        qup-uart1-default-state {
+            pins = "gpio43", "gpio44";
+            function = "qup_se6";
+            drive-strength = <8>;
+            bias-pull-down;
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml
new file mode 100644
index 0000000000000..4bb511719f313
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml
@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,nord-tlmm.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm Technologies, Inc. SA8797P TLMM block
+
+maintainers:
+  - Bartosz Golaszewski <brgl@kernel.org>
+
+description:
+  Top Level Mode Multiplexer pin controller in Qualcomm SA8797P SoC.
+
+allOf:
+  - $ref: /schemas/pinctrl/qcom,tlmm-common.yaml#
+
+properties:
+  compatible:
+    const: qcom,nord-tlmm
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  gpio-reserved-ranges:
+    minItems: 1
+    maxItems: 90
+
+  gpio-line-names:
+    maxItems: 181
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-nord-tlmm-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-nord-tlmm-state"
+        additionalProperties: false
+
+$defs:
+  qcom-nord-tlmm-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,tlmm-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          oneOf:
+            - pattern: "^gpio([0-9]|[1-9][0-9]|1[0-7][0-9]|180)$"
+            - enum: [ ufs_reset ]
+        minItems: 1
+        maxItems: 16
+
+      function:
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+
+        enum: [ aoss_cti, atest_char, atest_usb20, atest_usb21,
+                aud_intfc0_clk, aud_intfc0_data, aud_intfc0_ws,
+                aud_intfc10_clk, aud_intfc10_data, aud_intfc10_ws,
+                aud_intfc1_clk, aud_intfc1_data, aud_intfc1_ws,
+                aud_intfc2_clk, aud_intfc2_data, aud_intfc2_ws,
+                aud_intfc3_clk, aud_intfc3_data, aud_intfc3_ws,
+                aud_intfc4_clk, aud_intfc4_data, aud_intfc4_ws,
+                aud_intfc5_clk, aud_intfc5_data, aud_intfc5_ws,
+                aud_intfc6_clk, aud_intfc6_data, aud_intfc6_ws,
+                aud_intfc7_clk, aud_intfc7_data, aud_intfc7_ws,
+                aud_intfc8_clk, aud_intfc8_data, aud_intfc8_ws,
+                aud_intfc9_clk, aud_intfc9_data, aud_intfc9_ws,
+                aud_mclk0_mira, aud_mclk0_mirb, aud_mclk1_mira, aud_mclk1_mirb,
+                aud_mclk2_mira, aud_mclk2_mirb, aud_refclk0, aud_refclk1,
+                bist_done, ccu_async_in, ccu_i2c_scl, ccu_i2c_sda, ccu_timer,
+                clink_debug, dbg_out, dbg_out_clk,
+                ddr_bist_complete, ddr_bist_fail, ddr_bist_start, ddr_bist_stop,
+                ddr_pxi, dp_rx0, dp_rx00, dp_rx01, dp_rx0_mute, dp_rx1, dp_rx10,
+                dp_rx11, dp_rx1_mute,
+                edp0_hot, edp0_lcd, edp1_hot, edp1_lcd, edp2_hot, edp2_lcd,
+                edp3_hot, edp3_lcd,
+                emac0_mcg, emac0_mdc, emac0_mdio, emac0_ptp, emac1_mcg,
+                emac1_mdc, emac1_mdio, emac1_ptp,
+                gcc_gp1_clk, gcc_gp2_clk, gcc_gp3_clk, gcc_gp4_clk, gcc_gp5_clk,
+                gcc_gp6_clk, gcc_gp7_clk, gcc_gp8_clk, jitter_bist, lbist_pass,
+                mbist_pass, mdp0_vsync_out, mdp1_vsync_out, mdp_vsync_e,
+                mdp_vsync_p, mdp_vsync_s,
+                pcie0_clk_req_n, pcie1_clk_req_n, pcie2_clk_req_n,
+                pcie3_clk_req_n, phase_flag, pll_bist_sync, pll_clk_aux,
+                prng_rosc0, prng_rosc1, pwrbrk_i_n, qdss, qdss_cti, qspi,
+                qup0_se0, qup0_se1, qup0_se2, qup0_se3, qup0_se4, qup0_se5,
+                qup1_se0, qup1_se1, qup1_se3, qup1_se2, qup1_se4, qup1_se5,
+                qup1_se6, qup2_se0, qup2_se1, qup2_se2, qup2_se3, qup2_se4,
+                qup2_se5, qup2_se6,
+                sailss_ospi, sdc4_clk, sdc4_cmd, sdc4_data, smb_alert,
+                smb_alert_n, smb_clk, smb_dat, tb_trig_sdc4, tmess_prng0,
+                tmess_prng1, tsc_timer, tsense_pwm, usb0_hs,
+                usb0_phy_ps, usb1_hs, usb1_phy_ps, usb2_hs, usxgmii0_phy,
+                usxgmii1_phy, vsense_trigger_mirnat, wcn_sw, wcn_sw_ctrl]
+
+    required:
+      - pins
+
+required:
+  - compatible
+  - reg
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+
+    tlmm: pinctrl@f100000 {
+        compatible = "qcom,nord-tlmm";
+        reg = <0x0f100000 0xc0000>;
+        interrupts = <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>;
+        gpio-controller;
+        #gpio-cells = <2>;
+        interrupt-controller;
+        #interrupt-cells = <2>;
+        gpio-ranges = <&tlmm 0 0 181>;
+        wakeup-parent = <&pdc>;
+
+        qup_uart15_default: qup-uart15-default-state {
+            pins = "gpio147", "gpio148";
+            function = "qup2_se2";
+            drive-strength = <2>;
+            bias-disable;
+        };
+    };
+...
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sdm845-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sdm845-pinctrl.yaml
index 4fcac2e55b554..3b33daedc018a 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,sdm845-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,sdm845-pinctrl.yaml
@@ -42,11 +42,6 @@ patternProperties:
             $ref: "#/$defs/qcom-sdm845-tlmm-state"
         additionalProperties: false
 
-  "-hog(-[0-9]+)?$":
-    type: object
-    required:
-      - gpio-hog
-
 $defs:
   qcom-sdm845-tlmm-state:
     type: object
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,sm6350-lpass-lpi-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,sm6350-lpass-lpi-pinctrl.yaml
new file mode 100644
index 0000000000000..4903b2d37d89f
--- /dev/null
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,sm6350-lpass-lpi-pinctrl.yaml
@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pinctrl/qcom,sm6350-lpass-lpi-pinctrl.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm SM6350 SoC LPASS LPI TLMM
+
+maintainers:
+  - Luca Weiss <luca.weiss@fairphone.com>
+
+description:
+  Top Level Mode Multiplexer pin controller in the Low Power Audio SubSystem
+  (LPASS) Low Power Island (LPI) of Qualcomm SM6350 SoC.
+
+properties:
+  compatible:
+    const: qcom,sm6350-lpass-lpi-pinctrl
+
+  reg:
+    items:
+      - description: LPASS LPI TLMM Control and Status registers
+      - description: LPASS LPI MCC registers
+
+  clocks:
+    items:
+      - description: LPASS Core voting clock
+      - description: LPASS Audio voting clock
+
+  clock-names:
+    items:
+      - const: core
+      - const: audio
+
+patternProperties:
+  "-state$":
+    oneOf:
+      - $ref: "#/$defs/qcom-sm6350-lpass-state"
+      - patternProperties:
+          "-pins$":
+            $ref: "#/$defs/qcom-sm6350-lpass-state"
+        additionalProperties: false
+
+$defs:
+  qcom-sm6350-lpass-state:
+    type: object
+    description:
+      Pinctrl node's client devices use subnodes for desired pin configuration.
+      Client device subnodes use below standard properties.
+    $ref: qcom,lpass-lpi-common.yaml#/$defs/qcom-tlmm-state
+    unevaluatedProperties: false
+
+    properties:
+      pins:
+        description:
+          List of gpio pins affected by the properties specified in this
+          subnode.
+        items:
+          pattern: "^gpio([0-9]|1[0-4])$"
+
+      function:
+        enum: [ dmic1_clk, dmic1_data, dmic2_clk, dmic2_data, dmic3_clk,
+                dmic3_data, gpio, i2s1_clk, i2s1_data, i2s1_ws, i2s2_clk,
+                i2s2_data, i2s2_ws, qua_mi2s_data, qua_mi2s_sclk, qua_mi2s_ws,
+                swr_rx_clk, swr_rx_data, swr_tx_clk, swr_tx_data, wsa_swr_clk,
+                wsa_swr_data ]
+        description:
+          Specify the alternative function to be configured for the specified
+          pins.
+
+allOf:
+  - $ref: qcom,lpass-lpi-common.yaml#
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/sound/qcom,q6dsp-lpass-ports.h>
+
+    lpass_tlmm: pinctrl@33c0000 {
+        compatible = "qcom,sm6350-lpass-lpi-pinctrl";
+        reg = <0x033c0000 0x20000>,
+              <0x03550000 0x10000>;
+        gpio-controller;
+        #gpio-cells = <2>;
+        gpio-ranges = <&lpass_tlmm 0 0 15>;
+
+        clocks = <&q6afecc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+                 <&q6afecc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+        clock-names = "core",
+                      "audio";
+
+        i2s1_active: i2s1-active-state {
+            clk-pins {
+                pins = "gpio6";
+                function = "i2s1_clk";
+                drive-strength = <8>;
+                bias-disable;
+                output-high;
+            };
+
+            ws-pins {
+                pins = "gpio7";
+                function = "i2s1_ws";
+                drive-strength = <8>;
+                bias-disable;
+                output-high;
+            };
+
+            data-pins {
+                pins = "gpio8", "gpio9";
+                function = "i2s1_data";
+                drive-strength = <8>;
+                bias-disable;
+                output-high;
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml b/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml
index aae3dcf6cac88..aec72e8c06214 100644
--- a/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/qcom,tlmm-common.yaml
@@ -51,6 +51,12 @@ properties:
       should not be accessed by the OS. Please see the ../gpio/gpio.txt for more
       information.
 
+patternProperties:
+  "-hog(-[0-9]+)?$":
+    type: object
+    required:
+      - gpio-hog
+
 allOf:
   - $ref: pinctrl.yaml#
 
diff --git a/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml b/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml
index 00c05243b9a47..fb1fe1ea759fc 100644
--- a/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml
+++ b/Documentation/devicetree/bindings/pinctrl/renesas,rzg2l-pinctrl.yaml
@@ -26,6 +26,7 @@ properties:
               - renesas,r9a07g043-pinctrl # RZ/G2UL{Type-1,Type-2} and RZ/Five
               - renesas,r9a07g044-pinctrl # RZ/G2{L,LC}
               - renesas,r9a08g045-pinctrl # RZ/G3S
+              - renesas,r9a08g046-pinctrl # RZ/G3L
               - renesas,r9a09g047-pinctrl # RZ/G3E
               - renesas,r9a09g056-pinctrl # RZ/V2N
               - renesas,r9a09g057-pinctrl # RZ/V2H(P)
@@ -78,6 +79,26 @@ properties:
           - description: PFC main reset
           - description: Reset for the control register related to WDTUDFCA and WDTUDFFCM pins
 
+  reset-names:
+    oneOf:
+      - items:
+          - const: rstn
+          - const: port
+          - const: spare
+      - items:
+          - const: main
+          - const: error
+
+  renesas,clonech:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    items:
+      - items:
+          - description: phandle to system controller
+          - description: offset of clone channel control register
+    description:
+      Phandle and offset to the system controller containing the clone channel
+      control values.
+
 additionalProperties:
   anyOf:
     - type: object
@@ -140,6 +161,15 @@ additionalProperties:
 allOf:
   - $ref: pinctrl.yaml#
 
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r9a08g046-pinctrl
+    then:
+      required:
+        - renesas,clonech
+
   - if:
       properties:
         compatible:
@@ -152,10 +182,14 @@ allOf:
       properties:
         resets:
           maxItems: 2
+        reset-names:
+          maxItems: 2
     else:
       properties:
         resets:
           minItems: 3
+        reset-names:
+          minItems: 3
 
 required:
   - compatible
@@ -187,6 +221,7 @@ examples:
             resets = <&cpg R9A07G044_GPIO_RSTN>,
                      <&cpg R9A07G044_GPIO_PORT_RESETN>,
                      <&cpg R9A07G044_GPIO_SPARE_RESETN>;
+            reset-names = "rstn", "port", "spare";
             power-domains = <&cpg>;
 
             scif0_pins: serial0 {
diff --git a/Documentation/devicetree/bindings/power/power-domain.yaml b/Documentation/devicetree/bindings/power/power-domain.yaml
index b1147dbf2e738..163b0af158fd2 100644
--- a/Documentation/devicetree/bindings/power/power-domain.yaml
+++ b/Documentation/devicetree/bindings/power/power-domain.yaml
@@ -68,6 +68,21 @@ properties:
       by the given provider should be subdomains of the domain specified
       by this binding.
 
+  power-domains-child-ids:
+    $ref: /schemas/types.yaml#/definitions/uint32-array
+    description:
+      An array of child domain IDs that correspond to the power-domains
+      property. This property is only applicable to power domain providers
+      with "#power-domain-cells" > 0 (i.e., providers that supply multiple
+      power domains). It specifies which of the provider's child domains
+      should be associated with each parent domain listed in the power-domains
+      property. The number of elements in this array must match the number of
+      phandles in the power-domains property. Each element specifies the child
+      domain ID (index) that should be made a child domain of the corresponding
+      parent domain. This enables hierarchical power domain structures where
+      different child domains from the same provider can have different
+      parent domains.
+
 required:
   - "#power-domain-cells"
 
@@ -133,3 +148,22 @@ examples:
             min-residency-us = <7000>;
         };
     };
+
+  - |
+    // Example: SCMI domain 15 -> MAIN_PD, SCMI domain 19 -> WKUP_PD
+    MAIN_PD: power-controller-main {
+        compatible = "foo,power-controller";
+        #power-domain-cells = <0>;
+    };
+
+    WKUP_PD: power-controller-wkup {
+        compatible = "foo,power-controller";
+        #power-domain-cells = <0>;
+    };
+
+    scmi_pds: power-controller-scmi {
+        compatible = "foo,power-controller";
+        #power-domain-cells = <1>;
+        power-domains = <&MAIN_PD>, <&WKUP_PD>;
+        power-domains-child-ids = <15>, <19>;
+    };
diff --git a/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml b/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml
index 8df327e528107..f1422a401b6ba 100644
--- a/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml
+++ b/Documentation/devicetree/bindings/pwm/marvell,pxa-pwm.yaml
@@ -15,7 +15,9 @@ allOf:
       properties:
         compatible:
           contains:
-            const: spacemit,k1-pwm
+            enum:
+              - spacemit,k1-pwm
+              - spacemit,k3-pwm
     then:
       properties:
         "#pwm-cells":
@@ -26,6 +28,26 @@ allOf:
           const: 1
           description: |
             Used for specifying the period length in nanoseconds.
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - spacemit,k3-pwm
+    then:
+      required:
+        - clock-names
+      properties:
+        clocks:
+          minItems: 2
+        clock-names:
+          minItems: 2
+    else:
+      properties:
+        clocks:
+          maxItems: 1
+        clock-names:
+          maxItems: 1
 
 properties:
   compatible:
@@ -36,7 +58,9 @@ properties:
           - marvell,pxa168-pwm
           - marvell,pxa910-pwm
       - items:
-          - const: spacemit,k1-pwm
+          - enum:
+              - spacemit,k1-pwm
+              - spacemit,k3-pwm
           - const: marvell,pxa910-pwm
 
   reg:
@@ -47,7 +71,18 @@ properties:
     description: Number of cells in a pwm specifier.
 
   clocks:
-    maxItems: 1
+    minItems: 1
+    items:
+      - description: The function clock
+      - description: An optional bus clock
+
+  clock-names:
+    minItems: 1
+    maxItems: 2
+    oneOf:
+      - items:
+          - const: func
+          - const: bus
 
   resets:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/pwm/qcom,ipq6018-pwm.yaml b/Documentation/devicetree/bindings/pwm/qcom,ipq6018-pwm.yaml
new file mode 100644
index 0000000000000..f9f1f652e7527
--- /dev/null
+++ b/Documentation/devicetree/bindings/pwm/qcom,ipq6018-pwm.yaml
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/pwm/qcom,ipq6018-pwm.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Qualcomm IPQ6018 PWM controller
+
+maintainers:
+  - George Moussalem <george.moussalem@outlook.com>
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - qcom,ipq5018-pwm
+              - qcom,ipq5332-pwm
+              - qcom,ipq9574-pwm
+          - const: qcom,ipq6018-pwm
+      - const: qcom,ipq6018-pwm
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  "#pwm-cells":
+    const: 3
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - "#pwm-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/clock/qcom,gcc-ipq6018.h>
+
+    pwm: pwm@1941010 {
+        compatible = "qcom,ipq6018-pwm";
+        reg = <0x01941010 0x20>;
+        clocks = <&gcc GCC_ADSS_PWM_CLK>;
+        assigned-clocks = <&gcc GCC_ADSS_PWM_CLK>;
+        assigned-clock-rates = <100000000>;
+        #pwm-cells = <3>;
+    };
diff --git a/Documentation/devicetree/bindings/regulator/mt6359-regulator.yaml b/Documentation/devicetree/bindings/regulator/mt6359-regulator.yaml
index fe4ac9350ba03..e3c20412c8a73 100644
--- a/Documentation/devicetree/bindings/regulator/mt6359-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/mt6359-regulator.yaml
@@ -18,84 +18,50 @@ patternProperties:
   "^buck_v(s1|gpu11|modem|pu|core|s2|pa|proc2|proc1|core_sshub)$":
     type: object
     $ref: regulator.yaml#
-
-    properties:
-      regulator-name:
-        pattern: "^v(s1|gpu11|modem|pu|core|s2|pa|proc2|proc1|core_sshub)$"
-
     unevaluatedProperties: false
 
   "^ldo_v(ibr|rf12|usb|camio|efuse|xo22)$":
     type: object
     $ref: regulator.yaml#
-
-    properties:
-      regulator-name:
-        pattern: "^v(ibr|rf12|usb|camio|efuse|xo22)$"
-
     unevaluatedProperties: false
 
   "^ldo_v(rfck|emc|a12|a09|ufs|bbck)$":
     type: object
     $ref: regulator.yaml#
-
-    properties:
-      regulator-name:
-        pattern: "^v(rfck|emc|a12|a09|ufs|bbck)$"
-
     unevaluatedProperties: false
 
-  "^ldo_vcn(18|13|33_1_bt|13_1_wifi|33_2_bt|33_2_wifi)$":
+  "^ldo_vcn(18|13|33_[12])$":
     type: object
     $ref: regulator.yaml#
-
-    properties:
-      regulator-name:
-        pattern: "^vcn(18|13|33_1_bt|13_1_wifi|33_2_bt|33_2_wifi)$"
-
     unevaluatedProperties: false
 
   "^ldo_vsram_(proc2|others|md|proc1|others_sshub)$":
     type: object
     $ref: regulator.yaml#
-
-    properties:
-      regulator-name:
-        pattern: "^vsram_(proc2|others|md|proc1|others_sshub)$"
-
     unevaluatedProperties: false
 
   "^ldo_v(fe|bif|io)28$":
     type: object
     $ref: regulator.yaml#
-
-    properties:
-      regulator-name:
-        pattern: "^v(fe|bif|io)28$"
-
     unevaluatedProperties: false
 
   "^ldo_v(aud|io|aux|rf|m)18$":
     type: object
     $ref: regulator.yaml#
-
-    properties:
-      regulator-name:
-        pattern: "^v(aud|io|aux|rf|m)18$"
-
     unevaluatedProperties: false
 
   "^ldo_vsim[12]$":
     type: object
     $ref: regulator.yaml#
+    unevaluatedProperties: false
 
-    properties:
-      regulator-name:
-        pattern: "^vsim[12]$"
-
-    required:
-      - regulator-name
-
+  "^ldo_vcn33_[12]_(bt|wifi)$":
+    type: object
+    $ref: regulator.yaml#
+    description:
+      vcn33_[12]_(bt|wifi) are incorrect representations.
+      Use vcn33_[12] instead.
+    deprecated: true
     unevaluatedProperties: false
 
 additionalProperties: false
@@ -246,13 +212,8 @@ examples:
           regulator-min-microvolt = <900000>;
           regulator-max-microvolt = <1300000>;
         };
-        mt6359_vcn33_1_bt_ldo_reg: ldo_vcn33_1_bt {
-          regulator-name = "vcn33_1_bt";
-          regulator-min-microvolt = <2800000>;
-          regulator-max-microvolt = <3500000>;
-        };
-        mt6359_vcn33_1_wifi_ldo_reg: ldo_vcn33_1_wifi {
-          regulator-name = "vcn33_1_wifi";
+        mt6359_vcn33_1_bt_ldo_reg: ldo_vcn33_1 {
+          regulator-name = "vcn33_1";
           regulator-min-microvolt = <2800000>;
           regulator-max-microvolt = <3500000>;
         };
@@ -313,13 +274,8 @@ examples:
           regulator-min-microvolt = <2500000>;
           regulator-max-microvolt = <3300000>;
         };
-        mt6359_vcn33_2_bt_ldo_reg: ldo_vcn33_2_bt {
-          regulator-name = "vcn33_2_bt";
-          regulator-min-microvolt = <2800000>;
-          regulator-max-microvolt = <3500000>;
-        };
-        mt6359_vcn33_2_wifi_ldo_reg: ldo_vcn33_2_wifi {
-          regulator-name = "vcn33_2_wifi";
+        mt6359_vcn33_2_bt_ldo_reg: ldo_vcn33_2 {
+          regulator-name = "vcn33_2";
           regulator-min-microvolt = <2800000>;
           regulator-max-microvolt = <3500000>;
         };
diff --git a/Documentation/devicetree/bindings/regulator/mt6360-regulator.yaml b/Documentation/devicetree/bindings/regulator/mt6360-regulator.yaml
index 9c879bc3c360f..cbb74e8e875de 100644
--- a/Documentation/devicetree/bindings/regulator/mt6360-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/mt6360-regulator.yaml
@@ -17,6 +17,10 @@ properties:
   compatible:
     const: mediatek,mt6360-regulator
 
+  BUCK1_VIN-supply:
+    description: Input supply phandle(s) for BUCK1
+  BUCK2_VIN-supply:
+    description: Input supply phandle(s) for BUCK2
   LDO_VIN1-supply:
     description: Input supply phandle(s) for LDO1/2/3
   LDO_VIN2-supply:
diff --git a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml
index 58bb0ad5dda44..d93594304651c 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,rpmh-regulator.yaml
@@ -51,6 +51,7 @@ description: |
       For PM8450, smps1 - smps6, ldo1 - ldo4
       For PM8550, smps1 - smps6, ldo1 - ldo17, bob1 - bob2
       For PM8998, smps1 - smps13, ldo1 - ldo28, lvs1 - lvs2
+      For PMAU0102, smps1 - smps8, ldo1 - ldo3
       For PMH0101, ldo1 - ldo18, bob1 - bob2
       For PMH0104, smps1 - smps4
       For PMH0110, smps1 - smps10, ldo1 - ldo4
@@ -87,6 +88,7 @@ properties:
       - qcom,pm8550ve-rpmh-regulators
       - qcom,pm8550vs-rpmh-regulators
       - qcom,pm8998-rpmh-regulators
+      - qcom,pmau0102-rpmh-regulators
       - qcom,pmc8180-rpmh-regulators
       - qcom,pmc8180c-rpmh-regulators
       - qcom,pmc8380-rpmh-regulators
@@ -417,6 +419,7 @@ allOf:
       properties:
         compatible:
           enum:
+            - qcom,pmau0102-rpmh-regulators
             - qcom,pmc8380-rpmh-regulators
             - qcom,pm8550ve-rpmh-regulators
     then:
diff --git a/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml b/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml
index b7241ce975b96..0fa0db7eef23c 100644
--- a/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml
+++ b/Documentation/devicetree/bindings/regulator/qcom,smd-rpm-regulator.yaml
@@ -35,6 +35,9 @@ description:
 
   For pm660l s1, s2, s3, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, bob
 
+  For pm8150, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, l1, l2, l3, l4, l5, l6,
+  l7, l8, l9, l10, l11, l12, l13, l14, l15, l16, l17, l18
+
   For pm8226, s1, s2, s3, s4, s5, l1, l2, l3, l4, l5, l6, l7, l8, l9, l10,
   l11, l12, l13, l14, l15, l16, l17, l18, l19, l20, l21, l22, l23, l24, l25,
   l26, l27, l28, lvs1
@@ -91,6 +94,7 @@ properties:
       - qcom,rpm-pm6125-regulators
       - qcom,rpm-pm660-regulators
       - qcom,rpm-pm660l-regulators
+      - qcom,rpm-pm8150-regulators
       - qcom,rpm-pm8226-regulators
       - qcom,rpm-pm8841-regulators
       - qcom,rpm-pm8909-regulators
diff --git a/Documentation/devicetree/bindings/remoteproc/amlogic,meson-mx-ao-arc.yaml b/Documentation/devicetree/bindings/remoteproc/amlogic,meson-mx-ao-arc.yaml
index 76e8ca44906ac..3f710433e9377 100644
--- a/Documentation/devicetree/bindings/remoteproc/amlogic,meson-mx-ao-arc.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/amlogic,meson-mx-ao-arc.yaml
@@ -48,12 +48,7 @@ properties:
     minItems: 1
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
-    description:
-      phandles to a reserved SRAM region which is used as the memory of
-      the ARC core. The region should be defined as child nodes of the
-      AHB SRAM node as per the generic bindings in
-      Documentation/devicetree/bindings/sram/sram.yaml
+    maxItems: 1
 
   amlogic,secbus2:
     $ref: /schemas/types.yaml#/definitions/phandle
diff --git a/Documentation/devicetree/bindings/remoteproc/fsl,imx-rproc.yaml b/Documentation/devicetree/bindings/remoteproc/fsl,imx-rproc.yaml
index ce8ec0119469c..c18f71b648890 100644
--- a/Documentation/devicetree/bindings/remoteproc/fsl,imx-rproc.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/fsl,imx-rproc.yaml
@@ -28,6 +28,9 @@ properties:
       - fsl,imx8qxp-cm4
       - fsl,imx8ulp-cm33
       - fsl,imx93-cm33
+      - fsl,imx94-cm33s
+      - fsl,imx94-cm70
+      - fsl,imx94-cm71
       - fsl,imx95-cm7
 
   clocks:
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,msm8916-mss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,msm8916-mss-pil.yaml
index faf2712e3d272..4049157dd83d7 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,msm8916-mss-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,msm8916-mss-pil.yaml
@@ -108,6 +108,12 @@ properties:
     description: States used by the AP to signal the Hexagon core
     items:
       - description: Stop modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: Names of the states used by the AP to signal the Hexagon core
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,msm8996-mss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,msm8996-mss-pil.yaml
index 1b65813cc8adf..4a1b439f985e3 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,msm8996-mss-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,msm8996-mss-pil.yaml
@@ -101,6 +101,13 @@ properties:
     description: States used by the AP to signal the Hexagon core
     items:
       - description: Stop modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point or Shared
+              Memory Manager device handling the communication with a remote
+              processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: Names of the states used by the AP to signal the Hexagon core
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml
index 68c17bf18987c..4607b459131b4 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,pas-common.yaml
@@ -60,6 +60,12 @@ properties:
     description: States used by the AP to signal the Hexagon core
     items:
       - description: Stop the modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-cdsp-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-cdsp-pil.yaml
index bca59394aef49..e5f5f92987e1d 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-cdsp-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,qcs404-cdsp-pil.yaml
@@ -92,6 +92,12 @@ properties:
     description: States used by the AP to signal the Hexagon core
     items:
       - description: Stop the modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-mss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-mss-pil.yaml
index 7c9accac92d08..21c82cd3be037 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-mss-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7180-mss-pil.yaml
@@ -133,6 +133,12 @@ properties:
     description: States used by the AP to signal the Hexagon core
     items:
       - description: Stop the modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-adsp-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-adsp-pil.yaml
index 94ca7a0cc203f..23b8e3079f3b1 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-adsp-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-adsp-pil.yaml
@@ -91,6 +91,12 @@ properties:
     description: States used by the AP to signal the Hexagon core
     items:
       - description: Stop the modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-mss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-mss-pil.yaml
index f349c303fa07e..43dfb90ac18d4 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-mss-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-mss-pil.yaml
@@ -147,6 +147,12 @@ properties:
     description: States used by the AP to signal the Hexagon core
     items:
       - description: Stop the modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-wpss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-wpss-pil.yaml
index f4118b2da5f65..f3f3432948edc 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-wpss-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc7280-wpss-pil.yaml
@@ -104,6 +104,12 @@ properties:
     description: States used by the AP to signal the Hexagon core
     items:
       - description: Stop the modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sdm845-adsp-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sdm845-adsp-pil.yaml
index a3c74871457fc..9666ebf1e7b6b 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,sdm845-adsp-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,sdm845-adsp-pil.yaml
@@ -92,6 +92,12 @@ properties:
     description: States used by the AP to signal the Hexagon core
     items:
       - description: Stop the modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.yaml
index 117fb4d0c4ad2..a55e55f5f0145 100644
--- a/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/qcom,wcnss-pil.yaml
@@ -84,6 +84,12 @@ properties:
       States used by the AP to signal the WCNSS core that it should shutdown
     items:
       - description: Stop the modem
+        items:
+          - description: Phandle to the Shared Memory Point 2 Point device
+              handling the communication with a remote processor
+          - description: Single bit index to toggle in the value sent to
+              the remote processor
+            maximum: 32
 
   qcom,smem-state-names:
     description: The names of the state bits used for SMP2P output
diff --git a/Documentation/devicetree/bindings/remoteproc/ti,k3-dsp-rproc.yaml b/Documentation/devicetree/bindings/remoteproc/ti,k3-dsp-rproc.yaml
index b51bb863d759e..8b1ed384ef22e 100644
--- a/Documentation/devicetree/bindings/remoteproc/ti,k3-dsp-rproc.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/ti,k3-dsp-rproc.yaml
@@ -75,16 +75,8 @@ properties:
 # --------------------
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle-array
     minItems: 1
     maxItems: 4
-    items:
-      maxItems: 1
-    description: |
-      phandles to one or more reserved on-chip SRAM regions. The regions
-      should be defined as child nodes of the respective SRAM node, and
-      should be defined as per the generic bindings in,
-      Documentation/devicetree/bindings/sram/sram.yaml
 
 allOf:
   - if:
diff --git a/Documentation/devicetree/bindings/remoteproc/ti,k3-r5f-rproc.yaml b/Documentation/devicetree/bindings/remoteproc/ti,k3-r5f-rproc.yaml
index 775e9b3a19387..14e6b2f817b31 100644
--- a/Documentation/devicetree/bindings/remoteproc/ti,k3-r5f-rproc.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/ti,k3-r5f-rproc.yaml
@@ -224,16 +224,8 @@ patternProperties:
           at 0x0) or 0 (BTCM at 0x0), default value is 1 if omitted.
 
       sram:
-        $ref: /schemas/types.yaml#/definitions/phandle-array
         minItems: 1
         maxItems: 4
-        items:
-          maxItems: 1
-        description: |
-          phandles to one or more reserved on-chip SRAM regions. The regions
-          should be defined as child nodes of the respective SRAM node, and
-          should be defined as per the generic bindings in,
-          Documentation/devicetree/bindings/sram/sram.yaml
 
     required:
       - compatible
diff --git a/Documentation/devicetree/bindings/remoteproc/xlnx,zynqmp-r5fss.yaml b/Documentation/devicetree/bindings/remoteproc/xlnx,zynqmp-r5fss.yaml
index ee63c03949c93..c7d5e58330d64 100644
--- a/Documentation/devicetree/bindings/remoteproc/xlnx,zynqmp-r5fss.yaml
+++ b/Documentation/devicetree/bindings/remoteproc/xlnx,zynqmp-r5fss.yaml
@@ -106,20 +106,13 @@ patternProperties:
           - const: rx
 
       sram:
-        $ref: /schemas/types.yaml#/definitions/phandle-array
         minItems: 1
         maxItems: 8
-        items:
-          maxItems: 1
-        description: |
+        description:
           phandles to one or more reserved on-chip SRAM regions. Other than TCM,
           the RPU can execute instructions and access data from the OCM memory,
           the main DDR memory, and other system memories.
 
-          The regions should be defined as child nodes of the respective SRAM
-          node, and should be defined as per the generic bindings in
-          Documentation/devicetree/bindings/sram/sram.yaml
-
       memory-region:
         description: |
           List of phandles to the reserved memory regions associated with the
diff --git a/Documentation/devicetree/bindings/riscv/canaan.yaml b/Documentation/devicetree/bindings/riscv/canaan.yaml
index 41fd11f70a49b..f9854ff43ac68 100644
--- a/Documentation/devicetree/bindings/riscv/canaan.yaml
+++ b/Documentation/devicetree/bindings/riscv/canaan.yaml
@@ -10,7 +10,7 @@ maintainers:
   - Damien Le Moal <dlemoal@kernel.org>
 
 description:
-  Canaan Kendryte K210 SoC-based boards
+  Canaan Kendryte SoC-based boards
 
 properties:
   $nodename:
@@ -42,6 +42,12 @@ properties:
       - items:
           - const: canaan,kendryte-k210
 
+      - items:
+          - enum:
+              - canaan,canmv-k230
+              - canaan,k230-usip-lp3-evb
+          - const: canaan,kendryte-k230
+
 additionalProperties: true
 
 ...
diff --git a/Documentation/devicetree/bindings/riscv/spacemit.yaml b/Documentation/devicetree/bindings/riscv/spacemit.yaml
index b958b94a924de..af8030242bdc8 100644
--- a/Documentation/devicetree/bindings/riscv/spacemit.yaml
+++ b/Documentation/devicetree/bindings/riscv/spacemit.yaml
@@ -29,6 +29,7 @@ properties:
           - const: spacemit,k1
       - items:
           - enum:
+              - deepcomputing,fml13v05
               - spacemit,k3-pico-itx
           - const: spacemit,k3
 
diff --git a/Documentation/devicetree/bindings/rng/mtk-rng.yaml b/Documentation/devicetree/bindings/rng/mtk-rng.yaml
index 7e8dc62e5d3a6..38e67861b88fa 100644
--- a/Documentation/devicetree/bindings/rng/mtk-rng.yaml
+++ b/Documentation/devicetree/bindings/rng/mtk-rng.yaml
@@ -11,12 +11,13 @@ maintainers:
 
 properties:
   $nodename:
-    pattern: "^rng@[0-9a-f]+$"
+    pattern: "^rng(@[0-9a-f]+)?$"
 
   compatible:
     oneOf:
       - enum:
           - mediatek,mt7623-rng
+          - mediatek,mt7981-rng
       - items:
           - enum:
               - mediatek,mt7622-rng
@@ -25,6 +26,11 @@ properties:
               - mediatek,mt8365-rng
               - mediatek,mt8516-rng
           - const: mediatek,mt7623-rng
+      - items:
+          - enum:
+              - mediatek,mt7987-rng
+              - mediatek,mt7988-rng
+          - const: mediatek,mt7981-rng
 
   reg:
     maxItems: 1
@@ -38,18 +44,37 @@ properties:
 
 required:
   - compatible
-  - reg
-  - clocks
-  - clock-names
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: mediatek,mt7981-rng
+    then:
+      properties:
+        reg: false
+        clocks: false
+        clock-names: false
+    else:
+      required:
+        - reg
+        - clocks
+        - clock-names
 
 additionalProperties: false
 
 examples:
   - |
     #include <dt-bindings/clock/mt2701-clk.h>
-    rng: rng@1020f000 {
-            compatible = "mediatek,mt7623-rng";
-            reg = <0x1020f000 0x1000>;
-            clocks = <&infracfg CLK_INFRA_TRNG>;
-            clock-names = "rng";
+
+    rng@1020f000 {
+        compatible = "mediatek,mt7623-rng";
+        reg = <0x1020f000 0x1000>;
+        clocks = <&infracfg CLK_INFRA_TRNG>;
+        clock-names = "rng";
+    };
+  - |
+    rng {
+        compatible = "mediatek,mt7981-rng";
     };
diff --git a/Documentation/devicetree/bindings/serial/qcom,sa8255p-geni-uart.yaml b/Documentation/devicetree/bindings/serial/qcom,sa8255p-geni-uart.yaml
index c8f01923cb25d..8496f822dfa5c 100644
--- a/Documentation/devicetree/bindings/serial/qcom,sa8255p-geni-uart.yaml
+++ b/Documentation/devicetree/bindings/serial/qcom,sa8255p-geni-uart.yaml
@@ -14,9 +14,16 @@ allOf:
 
 properties:
   compatible:
-    enum:
-      - qcom,sa8255p-geni-uart
-      - qcom,sa8255p-geni-debug-uart
+    oneOf:
+      - enum:
+          - qcom,sa8255p-geni-uart
+          - qcom,sa8255p-geni-debug-uart
+      - items:
+          - const: qcom,sa8797p-geni-uart
+          - const: qcom,sa8255p-geni-uart
+      - items:
+          - const: qcom,sa8797p-geni-debug-uart
+          - const: qcom,sa8255p-geni-debug-uart
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml
index 685c1eceb7825..49f51b0028795 100644
--- a/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml
+++ b/Documentation/devicetree/bindings/serial/snps-dw-apb-uart.yaml
@@ -78,6 +78,7 @@ properties:
               - starfive,jh7100-hsuart
               - starfive,jh7100-uart
               - starfive,jh7110-uart
+              - ultrarisc,dp1000-uart
           - const: snps,dw-apb-uart
       - const: snps,dw-apb-uart
 
diff --git a/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-irqmux.yaml b/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-irqmux.yaml
index 51164772724f5..419b32e2df936 100644
--- a/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-irqmux.yaml
+++ b/Documentation/devicetree/bindings/soc/microchip/microchip,mpfs-irqmux.yaml
@@ -26,7 +26,11 @@ description: |
 
 properties:
   compatible:
-    const: microchip,mpfs-irqmux
+    oneOf:
+      - items:
+          - const: microchip,pic64gx-irqmux
+          - const: microchip,mpfs-irqmux
+      - const: microchip,mpfs-irqmux
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml
index c5c1bac2db013..8eaa04431d745 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,aoss-qmp.yaml
@@ -25,7 +25,9 @@ properties:
   compatible:
     items:
       - enum:
+          - qcom,eliza-aoss-qmp
           - qcom,glymur-aoss-qmp
+          - qcom,hawi-aoss-qmp
           - qcom,kaanapali-aoss-qmp
           - qcom,milos-aoss-qmp
           - qcom,qcs615-aoss-qmp
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml
index ff01d2f3ee5be..2db4288a8a54a 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,pmic-glink.yaml
@@ -32,6 +32,11 @@ properties:
               - qcom,sm8450-pmic-glink
               - qcom,sm8550-pmic-glink
           - const: qcom,pmic-glink
+      - items:
+          - enum:
+              - qcom,hawi-pmic-glink
+          - const: qcom,kaanapali-pmic-glink
+          - const: qcom,pmic-glink
       - items:
           - enum:
               - qcom,sm7325-pmic-glink
@@ -39,6 +44,7 @@ properties:
           - const: qcom,pmic-glink
       - items:
           - enum:
+              - qcom,eliza-pmic-glink
               - qcom,milos-pmic-glink
               - qcom,sm8650-pmic-glink
               - qcom,sm8750-pmic-glink
diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,sa8255p-geni-se-qup.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,sa8255p-geni-se-qup.yaml
index 352af3426d344..9c38ba59662b7 100644
--- a/Documentation/devicetree/bindings/soc/qcom/qcom,sa8255p-geni-se-qup.yaml
+++ b/Documentation/devicetree/bindings/soc/qcom/qcom,sa8255p-geni-se-qup.yaml
@@ -19,7 +19,11 @@ description:
 
 properties:
   compatible:
-    const: qcom,sa8255p-geni-se-qup
+    oneOf:
+      - const: qcom,sa8255p-geni-se-qup
+      - items:
+          - const: qcom,sa8797p-geni-se-qup
+          - const: qcom,sa8255p-geni-se-qup
 
   reg:
     description: QUP wrapper common register address and length.
@@ -49,7 +53,11 @@ patternProperties:
 
     properties:
       compatible:
-        const: qcom,sa8255p-geni-spi
+        oneOf:
+          - const: qcom,sa8255p-geni-spi
+          - items:
+              - const: qcom,sa8797p-geni-spi
+              - const: qcom,sa8255p-geni-spi
 
   "i2c@[0-9a-f]+$":
     type: object
@@ -58,7 +66,11 @@ patternProperties:
 
     properties:
       compatible:
-        const: qcom,sa8255p-geni-i2c
+        oneOf:
+          - const: qcom,sa8255p-geni-i2c
+          - items:
+              - const: qcom,sa8797p-geni-i2c
+              - const: qcom,sa8255p-geni-i2c
 
   "serial@[0-9a-f]+$":
     type: object
@@ -67,9 +79,16 @@ patternProperties:
 
     properties:
       compatible:
-        enum:
-          - qcom,sa8255p-geni-uart
-          - qcom,sa8255p-geni-debug-uart
+        oneOf:
+          - enum:
+              - qcom,sa8255p-geni-uart
+              - qcom,sa8255p-geni-debug-uart
+          - items:
+              - const: qcom,sa8797p-geni-uart
+              - const: qcom,sa8255p-geni-uart
+          - items:
+              - const: qcom,sa8797p-geni-debug-uart
+              - const: qcom,sa8255p-geni-debug-uart
 
 required:
   - compatible
diff --git a/Documentation/devicetree/bindings/soc/renesas/renesas,r8a78000-mfis.yaml b/Documentation/devicetree/bindings/soc/renesas/renesas,r8a78000-mfis.yaml
new file mode 100644
index 0000000000000..eef8c0a59e9c0
--- /dev/null
+++ b/Documentation/devicetree/bindings/soc/renesas/renesas,r8a78000-mfis.yaml
@@ -0,0 +1,187 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/soc/renesas/renesas,r8a78000-mfis.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Renesas MFIS (Multifunctional Interface) controller
+
+maintainers:
+  - Wolfram Sang <wsa+renesas@sang-engineering.com>
+
+description:
+  The Renesas Multifunctional Interface (MFIS) provides various functionality
+  like mailboxes, hardware spinlocks, product identification, error injection,
+  error detection and such. Parts of it can be used for communication between
+  different CPU cores. Those cores can be in various domains like AP, RT, or
+  SCP. Often multiple domain-specific MFIS instances exist in one SoC.
+
+properties:
+  compatible:
+    enum:
+      - renesas,r8a78000-mfis       # R-Car X5H (AP<->AP, with PRR)
+      - renesas,r8a78000-mfis-scp   # R-Car X5H (AP<->SCP, without PRR)
+
+  reg:
+    maxItems: 2
+
+  reg-names:
+    items:
+      - const: common
+      - const: mboxes
+
+  interrupts:
+    minItems: 32
+    maxItems: 128
+    description:
+      The interrupts raised by the remote doorbells.
+
+  interrupt-names:
+    minItems: 32
+    maxItems: 128
+    description:
+      An interrupt name is constructed with the prefix 'ch'. Then, the
+      channel number as specified in the documentation of the SoC. Finally,
+      the letter 'i' if the interrupt is raised by the IICR register. Or 'e'
+      if it is raised by the EICR register.
+
+  "#hwlock-cells":
+    const: 1
+
+  "#mbox-cells":
+    const: 2
+    description:
+      The first cell is the channel number as specified in the documentation
+      of the SoC. The second cell may specify flags as described in the file
+      <dt-bindings/soc/renesas,r8a78000-mfis.h>.
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r8a78000-mfis
+    then:
+      properties:
+        interrupts:
+          minItems: 128
+        interrupt-names:
+          minItems: 128
+          items:
+            pattern: "^ch[0-9]+[ie]$"
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: renesas,r8a78000-mfis-scp
+    then:
+      properties:
+        interrupts:
+          maxItems: 32
+        interrupt-names:
+          maxItems: 32
+          items:
+            pattern: "^ch[0-9]+i$"
+
+required:
+  - compatible
+  - reg
+  - reg-names
+  - interrupts
+  - interrupt-names
+  - "#hwlock-cells"
+  - "#mbox-cells"
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    system-controller@189e0000 {
+        compatible = "renesas,r8a78000-mfis";
+        reg = <0x189e0000 0x1000>, <0x18800000 0x40000>;
+        reg-names = "common", "mboxes";
+        interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 108 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 126 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 127 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 128 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 129 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 132 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 134 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 137 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 153 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 154 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 155 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 156 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 157 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 158 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 159 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 161 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 162 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 163 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 164 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 165 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 166 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 167 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 168 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 169 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 170 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 171 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 172 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 173 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 174 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 175 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 177 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 178 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 179 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 180 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 181 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 182 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 183 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 184 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 185 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 186 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 187 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 188 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 189 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 190 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 191 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 192 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 193 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 194 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 195 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 196 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 197 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 198 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 199 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 200 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 201 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 202 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 203 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 204 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 205 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 206 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 207 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 208 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 209 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 210 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 211 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 213 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 214 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 215 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 216 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 217 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 218 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 219 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 220 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 221 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 222 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 223 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 225 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 226 IRQ_TYPE_LEVEL_HIGH>,
+                     <GIC_SPI 227 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 228 IRQ_TYPE_LEVEL_HIGH>;
+        interrupt-names = "ch0i", "ch0e", "ch1i", "ch1e", "ch2i", "ch2e", "ch3i", "ch3e",
+                          "ch4i", "ch4e", "ch5i", "ch5e", "ch6i", "ch6e", "ch7i", "ch7e",
+                          "ch8i", "ch8e", "ch9i", "ch9e", "ch10i", "ch10e", "ch11i", "ch11e",
+                          "ch12i", "ch12e", "ch13i", "ch13e", "ch14i", "ch14e", "ch15i", "ch15e",
+                          "ch16i", "ch16e", "ch17i", "ch17e", "ch18i", "ch18e", "ch19i", "ch19e",
+                          "ch20i", "ch20e", "ch21i", "ch21e", "ch22i", "ch22e", "ch23i", "ch23e",
+                          "ch24i", "ch24e", "ch25i", "ch25e", "ch26i", "ch26e", "ch27i", "ch27e",
+                          "ch28i", "ch28e", "ch29i", "ch29e", "ch30i", "ch30e", "ch31i", "ch31e",
+                          "ch32i", "ch32e", "ch33i", "ch33e", "ch34i", "ch34e", "ch35i", "ch35e",
+                          "ch36i", "ch36e", "ch37i", "ch37e", "ch38i", "ch38e", "ch39i", "ch39e",
+                          "ch40i", "ch40e", "ch41i", "ch41e", "ch42i", "ch42e", "ch43i", "ch43e",
+                          "ch44i", "ch44e", "ch45i", "ch45e", "ch46i", "ch46e", "ch47i", "ch47e",
+                          "ch48i", "ch48e", "ch49i", "ch49e", "ch50i", "ch50e", "ch51i", "ch51e",
+                          "ch52i", "ch52e", "ch53i", "ch53e", "ch54i", "ch54e", "ch55i", "ch55e",
+                          "ch56i", "ch56e", "ch57i", "ch57e", "ch58i", "ch58e", "ch59i", "ch59e",
+                          "ch60i", "ch60e", "ch61i", "ch61e", "ch62i", "ch62e", "ch63i", "ch63e";
+        #hwlock-cells = <1>;
+        #mbox-cells = <2>;
+    };
diff --git a/Documentation/devicetree/bindings/sound/davinci-mcasp-audio.yaml b/Documentation/devicetree/bindings/sound/davinci-mcasp-audio.yaml
index 87559d0d079a7..f8a602cee37b1 100644
--- a/Documentation/devicetree/bindings/sound/davinci-mcasp-audio.yaml
+++ b/Documentation/devicetree/bindings/sound/davinci-mcasp-audio.yaml
@@ -248,13 +248,13 @@ examples:
       reg-names = "mpu";
       interrupts = <82>, <83>;
       interrupt-names = "tx", "rx";
-      op-mode = <0>;		/* MCASP_IIS_MODE */
+      op-mode = <0>; /* MCASP_IIS_MODE */
       tdm-slots = <2>;
       ti,async-mode;
       dmas = <&main_udmap 0xc400>, <&main_udmap 0x4400>;
       dma-names = "tx", "rx";
       serial-dir = <
-          0 0 0 0	/* 0: INACTIVE, 1: TX, 2: RX */
+          0 0 0 0 /* 0: INACTIVE, 1: TX, 2: RX */
           0 0 0 0
           0 0 0 1
           2 0 0 0 >;
diff --git a/Documentation/devicetree/bindings/sound/fsl,sai.yaml b/Documentation/devicetree/bindings/sound/fsl,sai.yaml
index 83b5ea5f3d70e..ba65b3f3d0662 100644
--- a/Documentation/devicetree/bindings/sound/fsl,sai.yaml
+++ b/Documentation/devicetree/bindings/sound/fsl,sai.yaml
@@ -162,6 +162,13 @@ properties:
       of transmitter.
     type: boolean
 
+  fsl,sai-bit-clock-swap:
+    description:
+      Enable Bit Clock Swap, which swaps the bit clock used by the transmitter
+      or receiver in asynchronous mode, i.e. makes transmitter use RX_BCLK and
+      TX_SYNC, and vice versa, makes receiver use TX_BCLK and RX_SYNC.
+    type: boolean
+
   fsl,shared-interrupt:
     description: Interrupt is shared with other modules.
     type: boolean
diff --git a/Documentation/devicetree/bindings/sound/gpio-audio-amp.yaml b/Documentation/devicetree/bindings/sound/gpio-audio-amp.yaml
new file mode 100644
index 0000000000000..3690f3d1628c9
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/gpio-audio-amp.yaml
@@ -0,0 +1,270 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/gpio-audio-amp.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Audio amplifier driven by GPIOs
+
+maintainers:
+  - Herve Codina <herve.codina@bootlin.com>
+
+description: |
+  Audio GPIO amplifiers are driven by GPIO in order to control the gain value
+  of the amplifier, its mute function and/or its bypass function.
+
+  Those amplifiers are based on discrete components (analog switches, op-amps
+  and more) where some of them, mostly analog switches, are controlled by GPIOs
+  to adjust the gain value of the whole amplifier and/or to control
+  the mute and/or bypass function.
+
+  For instance, the following piece of hardware is a GPIO amplifier
+
+                                         +5VA
+                                           ^
+                                        |\ |
+                                        | \
+        Vin >---------------------------|+ \
+                                        |   +-------+-----> Vout
+                .--\/\/\/--+------------|- /        |
+                |          |            | /         |
+                v          |            |/ |        |
+               GND         o               v        |
+                            \             GND       |
+       gpio >----------->    \                      |
+                         o    o                     |
+                         |    |                     |
+                         |    '--\/\/\/--.          |
+                         |               +--\/\/\/--'
+                         '---------------'
+
+properties:
+  compatible:
+    oneOf:
+      - const: gpio-audio-amp-mono
+        description:
+          A single channel amplifier. All features apply to this sole channel.
+
+      - const: gpio-audio-amp-stereo
+        description:
+          A dual channel amplifier (left and right). All features apply to both
+          channels producing the same effect on both channels at the same time.
+
+  vdd-supply:
+    description: Main power supply of the amplifier
+
+  vddio-supply:
+    description: Power supply related to the control path
+
+  vdda1-supply:
+    description: Analog power supply
+
+  vdda2-supply:
+    description: Additional analog power supply
+
+  mute-gpios:
+    description: GPIO to control the mute function
+    maxItems: 1
+
+  bypass-gpios:
+    description: GPIO to control the bypass function
+    maxItems: 1
+
+  gain-gpios:
+    description: |
+      GPIOs to control the amplifier gain
+
+      The gain value is computed from GPIOs value from 0 to 2^N-1 with N the
+      number of GPIO described. The first GPIO described is the lsb of the gain
+      value.
+
+      For instance assuming 2 gpios
+         gain-gpios = <&gpio1 GPIO_ACTIVE_HIGH> <&gpio2 GPIO_ACTIVE_HIGH>;
+      The gain value will be the following:
+
+          gpio1 | gpio2 | gain
+          ------+-------+-----
+            0   |    0  | 0b00 -> 0
+            1   |    0  | 0b01 -> 1
+            0   |    1  | 0b10 -> 2
+            1   |    1  | 0b11 -> 3
+          ------+-------+-----
+
+      Note: The gain value, bits set to 1 or 0, indicate the state active (bit
+            set) or the state inactive (bit unset) of the related GPIO. The
+            physical voltage corresponding to this active/inactive state is
+            given by the GPIO_ACTIVE_HIGH and GPIO_ACTIVE_LOW flags.
+
+    minItems: 1
+    maxItems: 16
+
+  gain-ranges:
+    $ref: /schemas/types.yaml#/definitions/int32-matrix
+    description: |
+      A list of one or more ranges of possible values. Each range is defined by
+      the first and last point in the range. Each point is defined by the pair
+      (GPIOs value, Gain in 0.01 dB unit).
+
+      Ranges can be contiguous or holes can be present between ranges if some
+      gpios value should not be used. Also in a range the first point and the
+      last point can be identical. In that case, the range contains only one
+      item, the given point.
+
+    items:
+      items:
+        - description: GPIOs value of the first point in the range
+        - description: Gain in 0.01 dB unit of the first point in the range
+        - description: GPIOs value of the last point in the range
+        - description: Gain in 0.01 dB unit of the last point in the range
+      description: |
+        A range defines a linear function (linear in dB) from the first point
+        to the last point, both included. The number of items in the range is
+          N = abs(first_point.gpio_value - last_point.gpio_value) + 1
+
+        It allows to define the gain range from the first_point.gain to
+        the last_point.gain, both points included.
+
+             Gain (0.01 dB unit)
+               ^
+               |                      last
+               +- - - - - - - - - - + point
+               |                 +  .
+               |              +     .
+               |           +        .
+               +- - - - +           .
+               |  first .           .
+               |  point .           .
+               |        .           .
+               +--------+-----------+---> gpios
+                                          value
+
+        Note: Even if first_point.gpio_value is lower than last_point.gpio_value
+              and first_point.gain is lower than last_point.gain in the above
+              graphic, all combination of values are supported leading to an
+              increasing or a decreasing linear segment.
+
+    minItems: 1
+    maxItems: 65536
+
+  gain-labels:
+    $ref: /schemas/types.yaml#/definitions/string-array
+    minItems: 2
+    maxItems: 65536
+    description: |
+      List of the gain labels attached to the combination of GPIOs controlling
+      the gain. The first label is related to the gain value 0, the second label
+      is related to the gain value 1 and so on.
+
+      With 2 GPIOs controlling the gain, GPIOs value can be 0, 1, 2 and 3.
+      Assuming that gain value set the hardware according to the following
+      table:
+
+         GPIOs | Hardware
+         value | amplification
+         ------+--------------
+           0   | Low
+           1   | Middle
+           2   | High
+           3   | Max
+         ------+--------------
+
+      The description using gain labels can be:
+        gain-labels = "Low", "Middle", "High", "Max";
+
+dependencies:
+  gain-ranges: [ gain-gpios ]
+  gain-labels: [ gain-gpios ]
+
+required:
+  - compatible
+  - vdd-supply
+
+anyOf:
+  - required:
+      - gain-gpios
+  - required:
+      - mute-gpios
+  - required:
+      - bypass-gpios
+
+allOf:
+  - $ref: component-common.yaml#
+  - if:
+      required:
+        - gain-ranges
+    then:
+      properties:
+        gain-labels: false
+  - if:
+      required:
+        - gain-labels
+    then:
+      properties:
+        gain-ranges: false
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    /* Gain controlled by gpios */
+    amplifier-0 {
+        compatible = "gpio-audio-amp-mono";
+        vdd-supply = <&regulator>;
+        gain-gpios = <&gpio 0 GPIO_ACTIVE_HIGH>, <&gpio 1 GPIO_ACTIVE_HIGH>;
+    };
+
+    /* Gain controlled by gpio using a simple range on a stereo amplifier */
+    amplifier-1 {
+        compatible = "gpio-audio-amp-stereo";
+        vdd-supply = <&regulator>;
+        gain-gpios = <&gpio 0 GPIO_ACTIVE_HIGH>, <&gpio 1 GPIO_ACTIVE_HIGH>;
+        gain-ranges = <0 (-300) 3 600>;
+    };
+
+    /* Gain controlled by gpio with labels */
+    amplifier-3 {
+        compatible = "gpio-audio-amp-mono";
+        vdd-supply = <&regulator>;
+        gain-gpios = <&gpio 0 GPIO_ACTIVE_HIGH>;
+        gain-labels = "Low", "High";
+    };
+
+    /* A mutable stereo amplifier without any gain control */
+    amplifier-4 {
+        compatible = "gpio-audio-amp-stereo";
+        vdd-supply = <&regulator>;
+        mute-gpios = <&gpio 0 GPIO_ACTIVE_HIGH>;
+    };
+
+    /*
+     * Several supplies, gain controlled using more complex ranges, mute and
+     * bypass.
+     *
+     * Assuming 3 gpios for controlling the gain with the following table
+     *   gpios value    Gain
+     *      0b000       Do not use (gpios value not allowed)
+     *      0b001       - 3dB
+     *      0b010       + 3dB
+     *      0b011       + 10dB
+     *      0b100       Do not use (gpios value not allowed)
+     *      0b101       + 6dB
+     *      0b110       + 7dB
+     *      0b111       + 8dB
+     */
+    amplifier-5 {
+        compatible = "gpio-audio-amp-mono";
+        vdd-supply = <&regulator>;
+        vddio-supply = <&regulator1>;
+        vdda1-supply = <&regulator2>;
+        gain-gpios = <&gpio 0 GPIO_ACTIVE_HIGH>,
+                     <&gpio 1 GPIO_ACTIVE_HIGH>,
+                     <&gpio 2 GPIO_ACTIVE_HIGH>;
+        gain-ranges = <1 (-300) 2 300>,
+                      <3 1000   3 1000>,
+                      <5 600    7 800>;
+        mute-gpios = <&gpio 3 GPIO_ACTIVE_HIGH>;
+        bypass-gpios = <&gpio 4 GPIO_ACTIVE_HIGH>;
+    };
+...
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt2701-audio.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt2701-audio.yaml
index 45382c4d86aa3..30f331366566e 100644
--- a/Documentation/devicetree/bindings/sound/mediatek,mt2701-audio.yaml
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt2701-audio.yaml
@@ -32,6 +32,7 @@ properties:
     maxItems: 1
 
   clocks:
+    minItems: 34
     items:
       - description: audio infra sys clock
       - description: top audio mux 1
@@ -67,8 +68,13 @@ properties:
       - description: top audio a1 sys pd
       - description: top audio a2 sys pd
       - description: audio merge interface pd
+      - description: HADDS2 PLL 294 MHz (HDMI audio path root)
+      - description: HDMI audio interface pd
+      - description: S/PDIF interface pd
+      - description: audio APLL root pd
 
   clock-names:
+    minItems: 34
     items:
       - const: infra_sys_audio_clk
       - const: top_audio_mux1_sel
@@ -104,6 +110,10 @@ properties:
       - const: audio_a1sys_pd
       - const: audio_a2sys_pd
       - const: audio_mrgif_pd
+      - const: hadds2pll_294m
+      - const: audio_hdmi_pd
+      - const: audio_spdf_pd
+      - const: audio_apll_pd
 
 required:
   - compatible
@@ -113,4 +123,17 @@ required:
   - clocks
   - clock-names
 
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: mediatek,mt7622-audio
+    then:
+      properties:
+        clocks:
+          maxItems: 34
+        clock-names:
+          maxItems: 34
+
 additionalProperties: false
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt2701-hdmi-audio.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt2701-hdmi-audio.yaml
new file mode 100644
index 0000000000000..9d5a8166e51ff
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt2701-hdmi-audio.yaml
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/mediatek,mt2701-hdmi-audio.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek MT2701/MT7623N HDMI audio output
+
+maintainers:
+  - Daniel Golle <daniel@makrotopia.org>
+
+description:
+  Sound card routing the MT2701/MT7623N Audio Front End HDMI
+  playback path to the on-chip HDMI transmitter. The AFE
+  provides the DMA memif and internal I2S engine; the HDMI
+  transmitter acts as the audio codec on the serialised link.
+
+properties:
+  compatible:
+    oneOf:
+      - const: mediatek,mt2701-hdmi-audio
+      - items:
+          - const: mediatek,mt7623n-hdmi-audio
+          - const: mediatek,mt2701-hdmi-audio
+
+  mediatek,platform:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: Phandle of the MT2701/MT7623N AFE platform node.
+
+  mediatek,audio-codec:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: Phandle of the HDMI transmitter acting as audio codec.
+
+required:
+  - compatible
+  - mediatek,platform
+  - mediatek,audio-codec
+
+additionalProperties: false
+
+examples:
+  - |
+    sound-hdmi {
+        compatible = "mediatek,mt7623n-hdmi-audio",
+                     "mediatek,mt2701-hdmi-audio";
+        mediatek,platform = <&afe>;
+        mediatek,audio-codec = <&hdmi0>;
+    };
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt6351-sound.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt6351-sound.yaml
new file mode 100644
index 0000000000000..b422e238b5127
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt6351-sound.yaml
@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/mediatek,mt6351-sound.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek MT6351 Audio CODEC
+
+maintainers:
+  - KaiChieh Chuang <kaichieh.chuang@mediatek.com>
+
+description:
+  MT6351 Audio CODEC is a part of the MediaTek MT6351 PMIC.
+  It communicates with the SoC through the MediaTek PMIC wrapper(pwrap).
+
+allOf:
+  - $ref: dai-common.yaml#
+
+properties:
+  compatible:
+    const: mediatek,mt6351-sound
+
+  "#sound-dai-cells":
+    const: 0
+
+required:
+  - compatible
+  - "#sound-dai-cells"
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    sound {
+       compatible = "mediatek,mt6351-sound";
+       #sound-dai-cells = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt8173-rt5650-rt5514.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt8173-rt5650-rt5514.yaml
index ed698c9ff42b0..becc7a11f8dc1 100644
--- a/Documentation/devicetree/bindings/sound/mediatek,mt8173-rt5650-rt5514.yaml
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt8173-rt5650-rt5514.yaml
@@ -18,7 +18,9 @@ properties:
     description: Phandles of rt5650 and rt5514 codecs
     items:
       - description: phandle of rt5650 codec
+        maxItems: 1
       - description: phandle of rt5514 codec
+        maxItems: 1
 
   mediatek,platform:
     $ref: /schemas/types.yaml#/definitions/phandle
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt8196-afe.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt8196-afe.yaml
new file mode 100644
index 0000000000000..949f8622baf9f
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt8196-afe.yaml
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/mediatek,mt8196-afe.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek Audio Front End PCM controller for MT8196
+
+maintainers:
+  - Darren Ye <darren.ye@mediatek.com>
+
+properties:
+  compatible:
+    const: mediatek,mt8196-afe
+
+  reg:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  memory-region:
+    maxItems: 1
+
+  power-domains:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: mux for audio intbus
+      - description: mux for audio engen1
+      - description: mux for audio engen2
+      - description: mux for audio h
+      - description: audio apll1 clock
+      - description: audio apll2 clock
+      - description: audio apll12 divide for i2sin0
+      - description: audio apll12 divide for i2sin1
+      - description: audio apll12 divide for fmi2s
+      - description: audio apll12 divide for tdmout mck
+      - description: audio apll12 divide for tdmout bck
+      - description: mux for adsp clock
+
+  clock-names:
+    items:
+      - const: top_aud_intbus
+      - const: top_aud_eng1
+      - const: top_aud_eng2
+      - const: top_aud_h
+      - const: apll1
+      - const: apll2
+      - const: apll12_div_i2sin0
+      - const: apll12_div_i2sin1
+      - const: apll12_div_fmi2s
+      - const: apll12_div_tdmout_m
+      - const: apll12_div_tdmout_b
+      - const: top_adsp
+
+required:
+  - compatible
+  - reg
+  - interrupts
+  - memory-region
+  - power-domains
+  - clocks
+  - clock-names
+
+additionalProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/interrupt-controller/arm-gic.h>
+    #include <dt-bindings/interrupt-controller/irq.h>
+
+    soc {
+        #address-cells = <2>;
+        #size-cells = <2>;
+
+        afe@1a110000 {
+            compatible = "mediatek,mt8196-afe";
+            reg = <0 0x1a110000 0 0x9000>;
+            interrupts = <GIC_SPI 351 IRQ_TYPE_LEVEL_HIGH 0>;
+            memory-region = <&afe_dma_mem_reserved>;
+            power-domains = <&scpsys 14>; //MT8196_POWER_DOMAIN_AUDIO
+            pinctrl-names = "default";
+            pinctrl-0 = <&aud_pins_default>;
+            clocks = <&vlp_cksys_clk 40>, //CLK_VLP_CK_AUD_INTBUS_SEL
+                     <&vlp_cksys_clk 38>, //CLK_VLP_CK_AUD_ENGEN1_SEL
+                     <&vlp_cksys_clk 39>, //CLK_VLP_CK_AUD_ENGEN2_SEL
+                     <&vlp_cksys_clk 37>, //CLK_VLP_CK_AUDIO_H_SEL
+                     <&vlp_cksys_clk 0>, //CLK_VLP_CK_VLP_APLL1
+                     <&vlp_cksys_clk 1>, //CLK_VLP_CK_VLP_APLL2
+                     <&cksys_clk 80>, //CLK_CK_APLL12_CK_DIV_I2SIN0
+                     <&cksys_clk 81>, //CLK_CK_APLL12_CK_DIV_I2SIN1
+                     <&cksys_clk 92>, //CLK_CK_APLL12_CK_DIV_FMI2S
+                     <&cksys_clk 93>, //CLK_CK_APLL12_CK_DIV_TDMOUT_M
+                     <&cksys_clk 94>, //CLK_CK_APLL12_CK_DIV_TDMOUT_B
+                     <&cksys_clk 45>; //CLK_CK_ADSP_SEL
+            clock-names = "top_aud_intbus",
+                          "top_aud_eng1",
+                          "top_aud_eng2",
+                          "top_aud_h",
+                          "apll1",
+                          "apll2",
+                          "apll12_div_i2sin0",
+                          "apll12_div_i2sin1",
+                          "apll12_div_fmi2s",
+                          "apll12_div_tdmout_m",
+                          "apll12_div_tdmout_b",
+                          "top_adsp";
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/sound/mediatek,mt8196-nau8825.yaml b/Documentation/devicetree/bindings/sound/mediatek,mt8196-nau8825.yaml
new file mode 100644
index 0000000000000..83350faa1e299
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/mediatek,mt8196-nau8825.yaml
@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/mediatek,mt8196-nau8825.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: MediaTek MT8196 ASoC sound card
+
+maintainers:
+  - Darren Ye <darren.ye@mediatek.com>
+
+allOf:
+  - $ref: sound-card-common.yaml#
+
+properties:
+  compatible:
+    enum:
+      - mediatek,mt8196-nau8825-sound
+      - mediatek,mt8196-rt5682s-sound
+      - mediatek,mt8196-rt5650-sound
+
+  mediatek,platform:
+    $ref: /schemas/types.yaml#/definitions/phandle
+    description: The phandle of MT8188 ASoC platform.
+
+patternProperties:
+  "^dai-link-[0-9]+$":
+    type: object
+    description:
+      Container for dai-link level properties and CODEC sub-nodes.
+
+    properties:
+      link-name:
+        description:
+          This property corresponds to the name of the BE dai-link to which
+          we are going to update parameters in this node.
+        items:
+          enum:
+            - TDM_DPTX_BE
+            - I2SOUT6_BE
+            - I2SIN6_BE
+            - I2SOUT4_BE
+            - I2SOUT3_BE
+
+      codec:
+        description: Holds subnode which indicates codec dai.
+        type: object
+        additionalProperties: false
+        properties:
+          sound-dai:
+            minItems: 1
+            maxItems: 2
+        required:
+          - sound-dai
+
+      dai-format:
+        description: audio format.
+        items:
+          enum:
+            - i2s
+            - right_j
+            - left_j
+            - dsp_a
+            - dsp_b
+
+      mediatek,clk-provider:
+        $ref: /schemas/types.yaml#/definitions/string
+        description: Indicates dai-link clock master.
+        enum:
+          - cpu
+          - codec
+
+    additionalProperties: false
+
+    required:
+      - link-name
+
+required:
+  - compatible
+  - mediatek,platform
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    sound {
+        compatible = "mediatek,mt8196-nau8825-sound";
+        model = "mt8196-nau8825";
+        mediatek,platform = <&afe>;
+        dai-link-0 {
+            link-name = "I2SOUT6_BE";
+            dai-format = "i2s";
+            mediatek,clk-provider = "cpu";
+            codec {
+                sound-dai = <&nau8825>;
+            };
+        };
+    };
+
+...
diff --git a/Documentation/devicetree/bindings/sound/mt6351.txt b/Documentation/devicetree/bindings/sound/mt6351.txt
deleted file mode 100644
index 7fb2cb99245ed..0000000000000
--- a/Documentation/devicetree/bindings/sound/mt6351.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-Mediatek MT6351 Audio Codec
-
-The communication between MT6351 and SoC is through Mediatek PMIC wrapper.
-For more detail, please visit Mediatek PMIC wrapper documentation.
-
-Must be a child node of PMIC wrapper.
-
-Required properties:
-
-- compatible : "mediatek,mt6351-sound".
-
-Example:
-
-mt6351_snd {
-	compatible = "mediatek,mt6351-sound";
-};
diff --git a/Documentation/devicetree/bindings/sound/samsung,tm2.yaml b/Documentation/devicetree/bindings/sound/samsung,tm2.yaml
index 67586ba3e0a0f..985b7d29cd337 100644
--- a/Documentation/devicetree/bindings/sound/samsung,tm2.yaml
+++ b/Documentation/devicetree/bindings/sound/samsung,tm2.yaml
@@ -45,8 +45,12 @@ properties:
     description: Phandles to the I2S controllers.
     $ref: /schemas/types.yaml#/definitions/phandle-array
     items:
-      - description: Phandle to I2S0.
-      - description: Phandle to I2S1.
+      - items:
+          - description: Phandle to I2S0
+          - description: Unused
+      - items:
+          - description: Phandle to I2S1
+          - description: Unused
 
   mic-bias-gpios:
     description: GPIO pin that enables the Main Mic bias regulator.
diff --git a/Documentation/devicetree/bindings/sound/simple-card.yaml b/Documentation/devicetree/bindings/sound/simple-card.yaml
index a14716b2732f0..23310f303be37 100644
--- a/Documentation/devicetree/bindings/sound/simple-card.yaml
+++ b/Documentation/devicetree/bindings/sound/simple-card.yaml
@@ -315,7 +315,7 @@ examples:
         #address-cells = <1>;
         #size-cells = <0>;
 
-        simple-audio-card,dai-link@0 {		/* I2S - HDMI */
+        simple-audio-card,dai-link@0 {  /* I2S - HDMI */
             reg = <0>;
             format = "i2s";
             cpu {
@@ -326,7 +326,7 @@ examples:
             };
         };
 
-        simple-audio-card,dai-link@1 {		/* S/PDIF - HDMI */
+        simple-audio-card,dai-link@1 {  /* S/PDIF - HDMI */
             reg = <1>;
             cpu {
                 sound-dai = <&audio1>;
@@ -336,7 +336,7 @@ examples:
             };
         };
 
-        simple-audio-card,dai-link@2 {		/* S/PDIF - S/PDIF */
+        simple-audio-card,dai-link@2 {  /* S/PDIF - S/PDIF */
             reg = <2>;
             cpu {
                 sound-dai = <&audio2>;
diff --git a/Documentation/devicetree/bindings/sound/ti,tas67524.yaml b/Documentation/devicetree/bindings/sound/ti,tas67524.yaml
new file mode 100644
index 0000000000000..812a4d39e2a5c
--- /dev/null
+++ b/Documentation/devicetree/bindings/sound/ti,tas67524.yaml
@@ -0,0 +1,280 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sound/ti,tas67524.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Texas Instruments TAS67524 Audio Amplifier
+
+maintainers:
+  - Sen Wang <sen@ti.com>
+
+description:
+  The TAS67524 is a four-channel, digital-input, automotive
+  Class-D audio amplifier with load diagnostics and an integrated
+  DSP for audio processing.
+
+allOf:
+  - $ref: dai-common.yaml#
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - ti,tas6754
+          - const: ti,tas67524
+      - const: ti,tas67524
+
+  reg:
+    maxItems: 1
+
+  '#sound-dai-cells':
+    const: 1
+    description: |
+      The device exposes three DAIs, selected by index.
+        0 - Standard Audio Path (Playback)
+        1 - Low-Latency Playback Path (Playback)
+        2 - Sensory Feedback (Capture - Vpredict and Isense)
+      By default, all four channels of each DAI are active.
+
+  interrupts:
+    maxItems: 1
+    description:
+      Active-low falling-edge interrupt from the FAULT pin. When provided,
+      the driver uses IRQ-driven fault reporting instead of polling.
+
+  powerdown-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the PD pin, active low. Controls the internal
+      digital circuitry power state. When asserted the device enters
+      full power-down mode and all register state is lost. Can be omitted if
+      PD pin is hardwired or externally controlled.
+
+  standby-gpios:
+    maxItems: 1
+    description:
+      GPIO connected to the STBY pin, active low. Controls the analog
+      power stage. When asserted the device enters Deep Sleep mode but
+      remains I2C-accessible with registers retained. Can be omitted if
+      STBY pin is tied to PD or hardwired.
+
+  dvdd-supply:
+    description:
+      Digital logic supply (1.62 V to 3.6 V). All three supply rails must
+      be within their recommended operating ranges before the PD pin is
+      released.
+
+  pvdd-supply:
+    description:
+      Output FET power supply (4.5 V to 19 V). All three supply rails must
+      be within their recommended operating ranges before the PD pin is
+      released.
+
+  vbat-supply:
+    description:
+      Battery supply for the Class-D output stage (4.5 V to 19 V). Optional
+      when PVDD and VBAT are connected to the same supply rail. When absent,
+      VBAT is assumed hardwired to PVDD.
+
+  ti,fast-boot:
+    type: boolean
+    description:
+      Skip DC load diagnostic sweep at power-on to reduce boot latency.
+      Automatic diagnostics after fault conditions remain enabled. Hardware
+      overcurrent protection is always active.
+
+  ti,audio-slot-no:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      TDM slot offset for the standard audio playback path via SDIN1. A value
+      of 4 maps to slot 4. If omitted, slot assignment is derived from the
+      tx_mask provided via set_tdm_slot(). Without either property, no slot
+      mapping is configured.
+
+  ti,llp-slot-no:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description:
+      TDM slot offset for the low-latency playback path via SDIN1. If omitted,
+      slot assignment is derived from the tx_mask provided via set_tdm_slot().
+      Without either property, no slot mapping is configured. Disabled outside
+      of LLP mode, and only relevant for TDM formats.
+
+  ti,vpredict-slot-no:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: |
+      In TDM mode, enables Vpredict output and assigns its starting slot;
+      four consecutive slots carry Vpredict Ch1-4 on SDOUT1. May coexist
+      with ti,isense-slot-no using separate non-overlapping slots.
+
+      In I2S mode, enables Vpredict output on SDOUT1 (Ch1/Ch2) and SDOUT2
+      (Ch3/Ch4). The slot value is unused. Requires a GPIO configured as
+      sdout2 for Ch3/Ch4; without it only Ch1/Ch2 are output. Mutually
+      exclusive with ti,isense-slot-no; if both are set, Vpredict takes
+      priority.
+
+      Irrelevant in Left-J and Right-J modes.
+
+  ti,isense-slot-no:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    description: |
+      In TDM mode, enables Isense output and assigns its starting slot;
+      four consecutive slots carry Isense Ch1-4 on SDOUT1. May coexist
+      with ti,vpredict-slot-no using separate non-overlapping slots.
+
+      In I2S mode, enables Isense output on SDOUT1 (Ch1/Ch2) and SDOUT2
+      (Ch3/Ch4). The slot value is unused. Requires a GPIO configured as
+      SDOUT2 for Ch3/Ch4; without it only Ch1/Ch2 are output. Mutually
+      exclusive with ti,vpredict-slot-no; Vpredict takes priority if both
+      are set.
+
+      Irrelevant in Left-J and Right-J modes.
+
+  ti,gpio1-function:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      Function for the GPIO_1 pin. When omitted, GPIO_1 remains in its
+      power-on default state.
+    enum:
+      - low            # Output: driven low
+      - auto-mute      # Output: high when all channels are auto-muted
+      - auto-mute-ch4  # Output: high when channel 4 is auto-muted
+      - auto-mute-ch3  # Output: high when channel 3 is auto-muted
+      - auto-mute-ch2  # Output: high when channel 2 is auto-muted
+      - auto-mute-ch1  # Output: high when channel 1 is auto-muted
+      - sdout2         # Output: Routes secondary serial data output 2
+      - sdout1         # Output: Re-routes secondary serial data output 1
+      - warn           # Output: warning signal (OTW, CBC)
+      - fault          # Output: fault signal (OTSD, OC, DC)
+      - clock-sync     # Output: clock synchronisation
+      - invalid-clock  # Output: high when clock is invalid
+      - high           # Output: driven high
+      - mute           # Input: external mute control
+      - phase-sync     # Input: phase synchronisation
+      - sdin2          # Input: secondary SDIN2 for I2S/LJ/RJ ch3/ch4
+      - deep-sleep     # Input: asserted transitions device to Deep Sleep
+      - hiz            # Input: asserted transitions device to Hi-Z
+      - play           # Input: asserted transitions device to Play
+      - sleep          # Input: asserted transitions device to Sleep
+
+  ti,gpio2-function:
+    $ref: /schemas/types.yaml#/definitions/string
+    description:
+      Function for the GPIO_2 pin. When omitted, GPIO_2 remains in its
+      power-on default state.
+    enum:
+      - low            # Output: driven low
+      - auto-mute      # Output: high when all channels are auto-muted
+      - auto-mute-ch4  # Output: high when channel 4 is auto-muted
+      - auto-mute-ch3  # Output: high when channel 3 is auto-muted
+      - auto-mute-ch2  # Output: high when channel 2 is auto-muted
+      - auto-mute-ch1  # Output: high when channel 1 is auto-muted
+      - sdout2         # Output: Routes secondary serial data output 2
+      - sdout1         # Output: Re-routes secondary serial data output 1
+      - warn           # Output: warning signal (OTW, CBC)
+      - fault          # Output: fault signal (OTSD, OC, DC)
+      - clock-sync     # Output: clock synchronisation
+      - invalid-clock  # Output: high when clock is invalid
+      - high           # Output: driven high
+      - mute           # Input: external mute control
+      - phase-sync     # Input: phase synchronisation
+      - sdin2          # Input: secondary SDIN2 for I2S/LJ/RJ ch3/ch4
+      - deep-sleep     # Input: asserted transitions device to Deep Sleep
+      - hiz            # Input: asserted transitions device to Hi-Z
+      - play           # Input: asserted transitions device to Play
+      - sleep          # Input: asserted transitions device to Sleep
+
+  ports:
+    $ref: /schemas/graph.yaml#/properties/ports
+    properties:
+      port@0:
+        $ref: audio-graph-port.yaml#
+        unevaluatedProperties: false
+        description: Standard audio playback port (DAI 0).
+
+      port@1:
+        $ref: audio-graph-port.yaml#
+        unevaluatedProperties: false
+        description: Low-latency playback port (LLP) (DAI 1).
+
+      port@2:
+        $ref: audio-graph-port.yaml#
+        unevaluatedProperties: false
+        description: Sensory feedback capture port (DAI 2).
+
+  port:
+    $ref: audio-graph-port.yaml#
+    unevaluatedProperties: false
+
+required:
+  - compatible
+  - reg
+  - '#sound-dai-cells'
+  - dvdd-supply
+  - pvdd-supply
+
+anyOf:
+  - required: [powerdown-gpios]
+  - required: [standby-gpios]
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    #include <dt-bindings/gpio/gpio.h>
+
+    i2c {
+        #address-cells = <1>;
+        #size-cells = <0>;
+
+        amplifier@70 {
+            compatible = "ti,tas67524";
+            reg = <0x70>;
+            #sound-dai-cells = <1>;
+            sound-name-prefix = "TAS0";
+
+            standby-gpios = <&main_gpio0 33 GPIO_ACTIVE_LOW>;
+
+            dvdd-supply = <&dvdd_1v8>;
+            pvdd-supply = <&pvdd_12v>;
+            vbat-supply = <&vbat_12v>;
+
+            ti,audio-slot-no = <0>;
+            ti,llp-slot-no = <4>;
+            ti,vpredict-slot-no = <0>;
+            ti,isense-slot-no = <4>;
+
+            ti,gpio2-function = "warn";
+
+            ports {
+                #address-cells = <1>;
+                #size-cells = <0>;
+
+                port@0 {
+                    reg = <0>;
+
+                    tas0_audio_ep: endpoint {
+                        dai-format = "dsp_b";
+                        remote-endpoint = <&be_tas0_audio_ep>;
+                    };
+                };
+
+                port@1 {
+                    reg = <1>;
+
+                    tas0_anc_ep: endpoint {
+                        remote-endpoint = <&be_tas0_anc_ep>;
+                    };
+                };
+
+                port@2 {
+                    reg = <2>;
+
+                    tas0_fb_ep: endpoint {
+                        remote-endpoint = <&be_tas0_fb_ep>;
+                    };
+                };
+            };
+        };
+    };
diff --git a/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.yaml b/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.yaml
index 1696ac46a660e..ee2199027e895 100644
--- a/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.yaml
+++ b/Documentation/devicetree/bindings/spi/qcom,spi-qcom-qspi.yaml
@@ -13,13 +13,11 @@ description: The QSPI controller allows SPI protocol communication in single,
   dual, or quad wire transmission modes for read/write access to slaves such
   as NOR flash.
 
-allOf:
-  - $ref: /schemas/spi/spi-controller.yaml#
-
 properties:
   compatible:
     items:
       - enum:
+          - qcom,qcs615-qspi
           - qcom,sc7180-qspi
           - qcom,sc7280-qspi
           - qcom,sdm845-qspi
@@ -67,6 +65,23 @@ required:
   - clock-names
   - clocks
 
+allOf:
+  - $ref: /schemas/spi/spi-controller.yaml#
+  - if:
+      properties:
+        compatible:
+          contains:
+            const: qcom,qcs615-qspi
+    then:
+      properties:
+        interconnects:
+          minItems: 2
+        interconnect-names:
+          minItems: 2
+      required:
+        - interconnects
+        - interconnect-names
+
 unevaluatedProperties: false
 
 examples:
diff --git a/Documentation/devicetree/bindings/spi/spacemit,k1-spi.yaml b/Documentation/devicetree/bindings/spi/spacemit,k1-spi.yaml
new file mode 100644
index 0000000000000..e82c7f8d0b981
--- /dev/null
+++ b/Documentation/devicetree/bindings/spi/spacemit,k1-spi.yaml
@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/spi/spacemit,k1-spi.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: SpacemiT K1 SoC Serial Peripheral Interface (SPI)
+
+maintainers:
+  - Alex Elder <elder@kernel.org>
+
+description:
+  The SpacemiT K1 SoC implements a SPI controller that has two 32-entry
+  FIFOs, for transmit and receive.  Details are currently available in
+  section 18.2.1 of the K1 User Manual, found in the SpacemiT Keystone
+  K1 Documentation[1].  The controller transfers words using PIO.  DMA
+  transfers are supported as well, if both TX and RX DMA channels are
+  specified,
+
+  [1] https://developer.spacemit.com/documentation
+
+allOf:
+  - $ref: /schemas/spi/spi-controller.yaml#
+
+properties:
+  compatible:
+    const: spacemit,k1-spi
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    items:
+      - description: Core clock
+      - description: Bus clock
+
+  clock-names:
+    items:
+      - const: core
+      - const: bus
+
+  resets:
+    maxItems: 1
+
+  interrupts:
+    maxItems: 1
+
+  dmas:
+    items:
+      - description: RX DMA channel
+      - description: TX DMA channel
+
+  dma-names:
+    items:
+      - const: rx
+      - const: tx
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - clock-names
+  - resets
+  - interrupts
+
+unevaluatedProperties: false
+
+examples:
+  - |
+
+    #include <dt-bindings/clock/spacemit,k1-syscon.h>
+    spi@d401c000 {
+        compatible = "spacemit,k1-spi";
+        reg = <0xd401c000 0x30>;
+        #address-cells = <1>;
+        #size-cells = <0>;
+        clocks = <&syscon_apbc CLK_SSP3>,
+                 <&syscon_apbc CLK_SSP3_BUS>;
+        clock-names = "core", "bus";
+        resets = <&syscon_apbc RESET_SSP3>;
+        interrupts = <55>;
+        dmas = <&pdma 20>, <&pdma 19>;
+        dma-names = "rx", "tx";
+    };
diff --git a/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml b/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml
index 472e92974714b..6d7d595e4ab31 100644
--- a/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml
+++ b/Documentation/devicetree/bindings/spi/st,stm32-spi.yaml
@@ -89,12 +89,10 @@ properties:
       - const: rxm2m
 
   sram:
-    $ref: /schemas/types.yaml#/definitions/phandle
-    description: |
-      Phandles to a reserved SRAM region which is used as temporary
-      storage memory between DMA and MDMA engines.
-      The region should be defined as child node of the AHB SRAM node
-      as per the generic bindings in Documentation/devicetree/bindings/sram/sram.yaml
+    maxItems: 1
+    description:
+      SRAM region which is used as temporary storage memory between DMA and
+      MDMA engines.
 
   power-domains:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/spi/st,stm32mp25-ospi.yaml b/Documentation/devicetree/bindings/spi/st,stm32mp25-ospi.yaml
index 272bc308726b2..b6be47f67fcb5 100644
--- a/Documentation/devicetree/bindings/spi/st,stm32mp25-ospi.yaml
+++ b/Documentation/devicetree/bindings/spi/st,stm32mp25-ospi.yaml
@@ -49,8 +49,9 @@ properties:
     description: configure OCTOSPI delay block.
     $ref: /schemas/types.yaml#/definitions/phandle-array
     items:
-      - description: phandle to syscfg
-      - description: register offset within syscfg
+      - items:
+          - description: phandle to syscfg
+          - description: register offset within syscfg
 
   access-controllers:
     description: phandle to the rifsc device to check access right
diff --git a/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml b/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml
index e7f7cf72719ea..6e6ab2168a2a2 100644
--- a/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml
+++ b/Documentation/devicetree/bindings/sram/allwinner,sun4i-a10-system-control.yaml
@@ -80,6 +80,7 @@ patternProperties:
               - const: allwinner,sun4i-a10-sram-c1
               - const: allwinner,sun4i-a10-sram-d
               - const: allwinner,sun50i-a64-sram-c
+              - const: allwinner,sun50i-h616-ve-sram
               - items:
                   - enum:
                       - allwinner,sun5i-a13-sram-a3-a4
@@ -103,7 +104,9 @@ patternProperties:
                       - allwinner,sun7i-a20-sram-d
                   - const: allwinner,sun4i-a10-sram-d
               - items:
-                  - const: allwinner,sun50i-h6-sram-c
+                  - enum:
+                      - allwinner,sun50i-h6-sram-c
+                      - allwinner,sun50i-h616-sram-c
                   - const: allwinner,sun50i-a64-sram-c
 
 required:
diff --git a/Documentation/devicetree/bindings/sram/sram-consumer.yaml b/Documentation/devicetree/bindings/sram/sram-consumer.yaml
new file mode 100644
index 0000000000000..f00087bd2879b
--- /dev/null
+++ b/Documentation/devicetree/bindings/sram/sram-consumer.yaml
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/sram/sram-consumer.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: SRAM Consumers
+
+maintainers:
+  - Rob Herring <robh@kernel.org>
+
+select: true
+
+properties:
+  sram:
+    description:
+      Phandles to one or more reserved on-chip SRAM regions. The regions
+      should be defined as child nodes of the respective SRAM node, and
+      should be defined as per the generic bindings in,
+      Documentation/devicetree/bindings/sram/sram.yaml
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    items:
+      maxItems: 1
+
+additionalProperties: true
+...
diff --git a/Documentation/devicetree/bindings/sram/sram.yaml b/Documentation/devicetree/bindings/sram/sram.yaml
index 8985f89170beb..eb695698a03e4 100644
--- a/Documentation/devicetree/bindings/sram/sram.yaml
+++ b/Documentation/devicetree/bindings/sram/sram.yaml
@@ -34,6 +34,7 @@ properties:
         - nvidia,tegra186-sysram
         - nvidia,tegra194-sysram
         - nvidia,tegra234-sysram
+        - qcom,eliza-imem
         - qcom,hawi-imem
         - qcom,kaanapali-imem
         - qcom,milos-imem
@@ -83,6 +84,7 @@ patternProperties:
             - allwinner,sun4i-a10-sram-d
             - allwinner,sun9i-a80-smp-sram
             - allwinner,sun50i-a64-sram-c
+            - allwinner,sun50i-h616-ve-sram
             - amlogic,meson8-ao-arc-sram
             - amlogic,meson8b-ao-arc-sram
             - amlogic,meson8-smp-sram
diff --git a/Documentation/devicetree/bindings/submitting-patches.rst b/Documentation/devicetree/bindings/submitting-patches.rst
index 81e27e50f9057..2a5533f68830b 100644
--- a/Documentation/devicetree/bindings/submitting-patches.rst
+++ b/Documentation/devicetree/bindings/submitting-patches.rst
@@ -64,9 +64,10 @@ I. For patch submitters
 
   7) DTS is treated in general as driver-independent hardware description, thus
      any DTS patches, regardless whether using existing or new bindings, should
-     be placed at the end of patchset to indicate no dependency of drivers on
-     the DTS.  DTS will be anyway applied through separate tree or branch, so
-     different order would indicate the series is non-bisectable.
+     be a separate posting or, when combined with driver patches, placed at the
+     end of the patchset to indicate no dependency of drivers on the DTS.  DTS
+     will be anyway applied through separate tree or branch, so different order
+     would indicate the series is non-bisectable.
 
      If a driver subsystem maintainer prefers to apply entire set, instead of
      their relevant portion of patchset, please split the DTS patches into
diff --git a/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml b/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml
index f1853daec2f9a..3e2725c569953 100644
--- a/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml
+++ b/Documentation/devicetree/bindings/timer/allwinner,sun5i-a13-hstimer.yaml
@@ -15,9 +15,13 @@ properties:
     oneOf:
       - const: allwinner,sun5i-a13-hstimer
       - const: allwinner,sun7i-a20-hstimer
+      - const: allwinner,sun20i-d1-hstimer
       - items:
           - const: allwinner,sun6i-a31-hstimer
           - const: allwinner,sun7i-a20-hstimer
+      - items:
+          - const: allwinner,sun50i-h616-hstimer
+          - const: allwinner,sun20i-d1-hstimer
 
   reg:
     maxItems: 1
@@ -45,7 +49,10 @@ required:
 if:
   properties:
     compatible:
-      const: allwinner,sun5i-a13-hstimer
+      anyOf:
+        - const: allwinner,sun5i-a13-hstimer
+        - contains:
+            const: allwinner,sun20i-d1-hstimer
 
 then:
   properties:
diff --git a/Documentation/devicetree/bindings/timer/fsl,imxgpt.yaml b/Documentation/devicetree/bindings/timer/fsl,imxgpt.yaml
index 9898dc7ea97be..6d41fb1203793 100644
--- a/Documentation/devicetree/bindings/timer/fsl,imxgpt.yaml
+++ b/Documentation/devicetree/bindings/timer/fsl,imxgpt.yaml
@@ -14,6 +14,7 @@ properties:
     oneOf:
       - const: fsl,imx1-gpt
       - const: fsl,imx21-gpt
+      - const: fsl,imx25-epit
       - items:
           - const: fsl,imx27-gpt
           - const: fsl,imx21-gpt
diff --git a/Documentation/devicetree/bindings/timer/renesas,rz-mtu3.yaml b/Documentation/devicetree/bindings/timer/renesas,rz-mtu3.yaml
index 3ad10c5b66ba5..ecff2912d812b 100644
--- a/Documentation/devicetree/bindings/timer/renesas,rz-mtu3.yaml
+++ b/Documentation/devicetree/bindings/timer/renesas,rz-mtu3.yaml
@@ -112,6 +112,8 @@ properties:
           - renesas,r9a07g043-mtu3  # RZ/{G2UL,Five}
           - renesas,r9a07g044-mtu3  # RZ/G2{L,LC}
           - renesas,r9a07g054-mtu3  # RZ/V2L
+          - renesas,r9a09g077-mtu3  # RZ/T2H
+          - renesas,r9a09g087-mtu3  # RZ/N2H
       - const: renesas,rz-mtu3
 
   reg:
@@ -162,7 +164,6 @@ properties:
       - description: MTU8.TGRC input capture/compare match
       - description: MTU8.TGRD input capture/compare match
       - description: MTU8.TCNT overflow
-      - description: MTU8.TCNT underflow
 
   interrupt-names:
     items:
@@ -209,7 +210,6 @@ properties:
       - const: tgic8
       - const: tgid8
       - const: tciv8
-      - const: tciu8
 
   clocks:
     maxItems: 1
@@ -233,7 +233,22 @@ required:
   - interrupt-names
   - clocks
   - power-domains
-  - resets
+
+allOf:
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - renesas,r9a07g043-mtu3
+              - renesas,r9a07g044-mtu3
+              - renesas,r9a07g054-mtu3
+    then:
+      required:
+        - resets
+    else:
+      properties:
+        resets: false
 
 additionalProperties: false
 
@@ -287,8 +302,7 @@ examples:
                    <GIC_SPI 209 IRQ_TYPE_EDGE_RISING>,
                    <GIC_SPI 210 IRQ_TYPE_EDGE_RISING>,
                    <GIC_SPI 211 IRQ_TYPE_EDGE_RISING>,
-                   <GIC_SPI 212 IRQ_TYPE_EDGE_RISING>,
-                   <GIC_SPI 213 IRQ_TYPE_EDGE_RISING>;
+                   <GIC_SPI 212 IRQ_TYPE_EDGE_RISING>;
       interrupt-names = "tgia0", "tgib0", "tgic0", "tgid0", "tciv0", "tgie0",
                         "tgif0",
                         "tgia1", "tgib1", "tciv1", "tciu1",
@@ -298,7 +312,7 @@ examples:
                         "tgiu5", "tgiv5", "tgiw5",
                         "tgia6", "tgib6", "tgic6", "tgid6", "tciv6",
                         "tgia7", "tgib7", "tgic7", "tgid7", "tciv7",
-                        "tgia8", "tgib8", "tgic8", "tgid8", "tciv8", "tciu8";
+                        "tgia8", "tgib8", "tgic8", "tgid8", "tciv8";
       clocks = <&cpg CPG_MOD R9A07G044_MTU_X_MCK_MTU3>;
       power-domains = <&cpg>;
       resets = <&cpg R9A07G044_MTU_X_PRESET_MTU3>;
diff --git a/Documentation/devicetree/bindings/timer/sifive,clint.yaml b/Documentation/devicetree/bindings/timer/sifive,clint.yaml
index 3c16b260db040..a0ba8b572b21d 100644
--- a/Documentation/devicetree/bindings/timer/sifive,clint.yaml
+++ b/Documentation/devicetree/bindings/timer/sifive,clint.yaml
@@ -38,6 +38,7 @@ properties:
               - starfive,jh7100-clint   # StarFive JH7100
               - starfive,jh7110-clint   # StarFive JH7110
               - starfive,jh8100-clint   # StarFive JH8100
+              - starfive,jhb100-clint   # StarFive JHB100
               - tenstorrent,blackhole-clint # Tenstorrent Blackhole
           - const: sifive,clint0        # SiFive CLINT v0 IP block
       - items:
@@ -49,6 +50,7 @@ properties:
       - items:
           - enum:
               - allwinner,sun20i-d1-clint
+              - canaan,k230-clint
               - sophgo,cv1800b-clint
               - sophgo,cv1812h-clint
               - sophgo,sg2002-clint
@@ -72,22 +74,6 @@ properties:
     minItems: 1
     maxItems: 4095
 
-  sifive,fine-ctr-bits:
-    maximum: 15
-    description: The width in bits of the fine counter.
-
-if:
-  properties:
-    compatible:
-      contains:
-        const: sifive,clint2
-then:
-  required:
-    - sifive,fine-ctr-bits
-else:
-  properties:
-    sifive,fine-ctr-bits: false
-
 additionalProperties: false
 
 required:
diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml
index 23fd4513933a4..1b35b411e73bb 100644
--- a/Documentation/devicetree/bindings/trivial-devices.yaml
+++ b/Documentation/devicetree/bindings/trivial-devices.yaml
@@ -67,8 +67,10 @@ properties:
           - arduino,unoq-mcu
             # Temperature monitoring of Astera Labs PT5161L PCIe retimer
           - asteralabs,pt5161l
-            # i2c h/w elliptic curve crypto module
+            # ATECC508A - i2c h/w elliptic curve crypto module
           - atmel,atecc508a
+            # ATECC608B - i2c h/w elliptic curve crypto module
+          - atmel,atecc608b
             # ATSHA204 - i2c h/w symmetric crypto module
           - atmel,atsha204
             # ATSHA204A - i2c h/w symmetric crypto module
@@ -99,6 +101,8 @@ properties:
           - delta,dps800
             # Delta Electronics DPS920AB 920W 54V Power Supply
           - delta,dps920ab
+            # 600W Non-isolated 1/8th Brick DC/DC Power Modules
+          - delta,e50sn12051
             # 1/4 Brick DC/DC Regulated Power Module
           - delta,q54sj108a2
             # 1300W 1/4 Brick DC/DC Regulated Power Module
@@ -192,6 +196,8 @@ properties:
           - jedec,spd5118
             # Linear Technology LTC2488
           - lineartechnology,ltc2488
+            # Regulated 12V, 860W, Digital DC/DC Power Module
+          - luxshare,lx1308
             # 5 Bit Programmable, Pulse-Width Modulator
           - maxim,ds1050
             # 10 kOhm digital potentiometer with I2C interface
@@ -336,6 +342,8 @@ properties:
           - mps,mp29612
             # Monolithic Power Systems Inc. multi-phase controller mp29816
           - mps,mp29816
+            # Monolithic Power Systems Inc. multi-phase controller mp2985
+          - mps,mp2985
             # Monolithic Power Systems Inc. multi-phase controller mp2993
           - mps,mp2993
             # Monolithic Power Systems Inc. hot-swap protection device
@@ -352,6 +360,8 @@ properties:
           - mps,mp9941
             # Monolithic Power Systems Inc. digital step-down converter mp9945
           - mps,mp9945
+            # Murata D1U74T-W power supply unit
+          - murata,d1u74t
             # Temperature sensor with integrated fan control
           - national,lm63
             # Temperature sensor with integrated fan control
diff --git a/Documentation/devicetree/bindings/ufs/qcom,sa8255p-ufshc.yaml b/Documentation/devicetree/bindings/ufs/qcom,sa8255p-ufshc.yaml
index 75fae9f1eba7e..db165a235cb6d 100644
--- a/Documentation/devicetree/bindings/ufs/qcom,sa8255p-ufshc.yaml
+++ b/Documentation/devicetree/bindings/ufs/qcom,sa8255p-ufshc.yaml
@@ -11,7 +11,11 @@ maintainers:
 
 properties:
   compatible:
-    const: qcom,sa8255p-ufshc
+    oneOf:
+      - const: qcom,sa8255p-ufshc
+      - items:
+          - const: qcom,sa8797p-ufshc
+          - const: qcom,sa8255p-ufshc
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml b/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml
index f28641c6e68fe..900d93b675cdf 100644
--- a/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml
+++ b/Documentation/devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml
@@ -17,6 +17,7 @@ select:
         enum:
           - qcom,eliza-ufshc
           - qcom,kaanapali-ufshc
+          - qcom,nord-ufshc
           - qcom,sm8650-ufshc
           - qcom,sm8750-ufshc
   required:
@@ -28,6 +29,7 @@ properties:
       - enum:
           - qcom,eliza-ufshc
           - qcom,kaanapali-ufshc
+          - qcom,nord-ufshc
           - qcom,sm8650-ufshc
           - qcom,sm8750-ufshc
       - const: qcom,ufshc
@@ -74,6 +76,7 @@ allOf:
           contains:
             enum:
               - qcom,eliza-ufshc
+              - qcom,nord-ufshc
     then:
       properties:
         reg:
diff --git a/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml b/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml
index a7eb7ad85a94e..710ce493f3b67 100644
--- a/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml
+++ b/Documentation/devicetree/bindings/ufs/samsung,exynos-ufs.yaml
@@ -19,6 +19,7 @@ properties:
       - samsung,exynos7-ufs
       - samsung,exynosautov9-ufs
       - samsung,exynosautov9-ufs-vh
+      - samsung,exynosautov920-ufs
       - tesla,fsd-ufs
 
   reg:
diff --git a/Documentation/devicetree/bindings/usb/cdns,usb3.yaml b/Documentation/devicetree/bindings/usb/cdns,usb3.yaml
index a199e5ba64161..2d95fb7321aff 100644
--- a/Documentation/devicetree/bindings/usb/cdns,usb3.yaml
+++ b/Documentation/devicetree/bindings/usb/cdns,usb3.yaml
@@ -4,11 +4,17 @@
 $id: http://devicetree.org/schemas/usb/cdns,usb3.yaml#
 $schema: http://devicetree.org/meta-schemas/core.yaml#
 
-title: Cadence USBSS-DRD controller
+title: Cadence USBSS and USBSSP DRD controller
 
 maintainers:
   - Pawel Laszczak <pawell@cadence.com>
 
+description:
+  Cadence USB dual-role controller. Covers USBSS (SuperSpeed, USB 3.0) and
+  USBSSP (SuperSpeed Plus, USB 3.1 gen2x1). Both variants share the same
+  DRD/OTG register interface, so the driver auto-detects the controller
+  version at runtime.
+
 properties:
   compatible:
     const: cdns,usb3
@@ -49,7 +55,7 @@ properties:
       cdns3 to type C connector.
 
   maximum-speed:
-    enum: [super-speed, high-speed, full-speed]
+    enum: [super-speed-plus, super-speed, high-speed, full-speed]
 
   phys:
     minItems: 1
diff --git a/Documentation/devicetree/bindings/usb/chipidea,usb2-common.yaml b/Documentation/devicetree/bindings/usb/chipidea,usb2-common.yaml
index 10020af15afc4..e6a5e79df3484 100644
--- a/Documentation/devicetree/bindings/usb/chipidea,usb2-common.yaml
+++ b/Documentation/devicetree/bindings/usb/chipidea,usb2-common.yaml
@@ -97,7 +97,9 @@ properties:
     minItems: 1
     items:
       - description: vbus extcon
+        maxItems: 1
       - description: id extcon
+        maxItems: 1
 
   phy-clkgate-delay-us:
     description:
diff --git a/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.yaml b/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.yaml
index 691d6cf02c272..fec04702f5300 100644
--- a/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.yaml
+++ b/Documentation/devicetree/bindings/usb/ci-hdrc-usb2.yaml
@@ -61,9 +61,10 @@ properties:
       offset, and phy index
     $ref: /schemas/types.yaml#/definitions/phandle-array
     items:
-      - description: phandle to TCSR node
-      - description: register offset
-      - description: phy index
+      - items:
+          - description: phandle to TCSR node
+          - description: register offset
+          - description: phy index
 
   nvidia,phy:
     description: phandle of usb phy that connects to the port. Use "phys" instead.
diff --git a/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml b/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml
index 9a94b2a74a1eb..e8b8c03f87a0e 100644
--- a/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml
+++ b/Documentation/devicetree/bindings/usb/genesys,gl850g.yaml
@@ -15,6 +15,7 @@ properties:
       - usb5e3,608
       - usb5e3,610
       - usb5e3,620
+      - usb5e3,625
       - usb5e3,626
 
   reg: true
@@ -26,6 +27,10 @@ properties:
     description:
       The regulator that provides 3.3V or 5.0V core power to the hub.
 
+  vdd12-supply:
+    description:
+      The regulator that provides 1.2V power to the hub.
+
   peer-hub: true
 
   ports:
@@ -56,6 +61,7 @@ allOf:
       properties:
         peer-hub: false
         vdd-supply: false
+        vdd12-supply: false
 
   - if:
       properties:
@@ -68,6 +74,18 @@ allOf:
       properties:
         peer-hub: true
         vdd-supply: true
+        vdd12-supply: false
+
+  - if:
+      properties:
+        compatible:
+          contains:
+            enum:
+              - usb5e3,625
+    then:
+      properties:
+        peer-hub: true
+        vdd-supply: true
 
 unevaluatedProperties: false
 
diff --git a/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml b/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml
index 7ded363845184..e3357f5abe2b1 100644
--- a/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml
+++ b/Documentation/devicetree/bindings/usb/richtek,rt1711h.yaml
@@ -43,8 +43,7 @@ properties:
   vbus-supply:
     description: VBUS power supply
 
-  wakeup-source:
-    type: boolean
+  wakeup-source: true
 
   connector:
     type: object
diff --git a/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml b/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml
index 653a89586f4e9..0aaaadb584e46 100644
--- a/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml
+++ b/Documentation/devicetree/bindings/usb/ti,j721e-usb.yaml
@@ -88,7 +88,7 @@ examples:
             power-domains = <&k3_pds 288 TI_SCI_PD_EXCLUSIVE>;
             clocks = <&k3_clks 288 15>, <&k3_clks 288 3>;
             clock-names = "ref", "lpm";
-            assigned-clocks = <&k3_clks 288 15>;	/* USB2_REFCLK */
+            assigned-clocks = <&k3_clks 288 15>; /* USB2_REFCLK */
             assigned-clock-parents = <&k3_clks 288 16>; /* HFOSC0 */
             #address-cells = <2>;
             #size-cells = <2>;
@@ -99,9 +99,9 @@ examples:
                         <0x00 0x6010000 0x00 0x10000>,
                         <0x00 0x6020000 0x00 0x10000>;
                   reg-names = "otg", "xhci", "dev";
-                  interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>,	/* irq.0 */
-                               <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>,	/* irq.6 */
-                               <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;	/* otgirq.0 */
+                  interrupts = <GIC_SPI 96 IRQ_TYPE_LEVEL_HIGH>, /* irq.0 */
+                               <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>, /* irq.6 */
+                               <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>; /* otgirq.0 */
                   interrupt-names = "host",
                                     "peripheral",
                                     "otg";
diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml
index 28784d66ae7ba..757029fcb2977 100644
--- a/Documentation/devicetree/bindings/vendor-prefixes.yaml
+++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml
@@ -437,6 +437,8 @@ patternProperties:
     description: Diodes, Inc.
   "^dioo,.*":
     description: Dioo Microcircuit Co., Ltd
+  "^displaytech,.*":
+    description: Displaytech Ltd.
   "^djn,.*":
     description: Shenzhen DJN Optronics Technology Co., Ltd
   "^dlc,.*":
@@ -489,6 +491,8 @@ patternProperties:
     description: Emtop Embedded Solutions
   "^eeti,.*":
     description: eGalax_eMPIA Technology Inc
+  "^efinix,.*":
+    description: Efinix, Inc.
   "^egnite,.*":
     description: egnite GmbH
   "^einfochips,.*":
@@ -981,6 +985,8 @@ patternProperties:
     description: Shenzhen Luckfox Technology Co., Ltd.
   "^lunzn,.*":
     description: Shenzhen Lunzn Technology Co., Ltd.
+  "^luxshare,.*":
+    description: Luxshare-ICT Co., Ltd.
   "^luxul,.*":
     description: Lagrand | AV
   "^lwn,.*":
diff --git a/Documentation/devicetree/bindings/watchdog/andestech,ae350-wdt.yaml b/Documentation/devicetree/bindings/watchdog/andestech,ae350-wdt.yaml
new file mode 100644
index 0000000000000..f1107c5527887
--- /dev/null
+++ b/Documentation/devicetree/bindings/watchdog/andestech,ae350-wdt.yaml
@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/watchdog/andestech,ae350-wdt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Andes ATCWDT200 Watchdog Timer
+
+maintainers:
+  - CL Wang <cl634@andestech.com>
+
+allOf:
+  - $ref: watchdog.yaml#
+
+properties:
+  compatible:
+    oneOf:
+      - items:
+          - enum:
+              - andestech,qilai-wdt
+          - const: andestech,ae350-wdt
+      - const: andestech,ae350-wdt
+
+  reg:
+    maxItems: 1
+
+  clocks:
+    maxItems: 1
+
+  andestech,clock-source:
+    $ref: /schemas/types.yaml#/definitions/uint32
+    enum: [0, 1]
+    description: |
+      Select the clock source for the watchdog timer.
+      0 - External clock
+      1 - P clock
+
+required:
+  - compatible
+  - reg
+  - clocks
+  - andestech,clock-source
+
+unevaluatedProperties: false
+
+examples:
+  - |
+    watchdog@f0500000 {
+        compatible = "andestech,ae350-wdt";
+        reg = <0xf0500000 0x20>;
+        clocks = <&clk_wdt>;
+        andestech,clock-source = <0>;
+    };
diff --git a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml
index 05602678c070d..845b5e8b5abc3 100644
--- a/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/apple,wdt.yaml
@@ -16,7 +16,9 @@ properties:
   compatible:
     oneOf:
       - items:
-          - const: apple,t6020-wdt
+          - enum:
+              - apple,t6020-wdt
+              - apple,t8122-wdt
           - const: apple,t8103-wdt
       - items:
           - enum:
diff --git a/Documentation/devicetree/bindings/watchdog/kontron,sl28cpld-wdt.yaml b/Documentation/devicetree/bindings/watchdog/kontron,sl28cpld-wdt.yaml
index 0821ba0e84a3c..872a8471ef65f 100644
--- a/Documentation/devicetree/bindings/watchdog/kontron,sl28cpld-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/kontron,sl28cpld-wdt.yaml
@@ -18,12 +18,7 @@ allOf:
 
 properties:
   compatible:
-    oneOf:
-      - items:
-          - enum:
-              - kontron,sa67mcu-wdt
-          - const: kontron,sl28cpld-wdt
-      - const: kontron,sl28cpld-wdt
+    const: kontron,sl28cpld-wdt
 
   reg:
     maxItems: 1
diff --git a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml
index 9f861045b71e8..d647bf5f19a4b 100644
--- a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml
@@ -18,15 +18,18 @@ properties:
       - items:
           - enum:
               - qcom,apss-wdt-glymur
+              - qcom,apss-wdt-hawi
               - qcom,kpss-wdt-ipq4019
               - qcom,apss-wdt-ipq5018
               - qcom,apss-wdt-ipq5332
               - qcom,apss-wdt-ipq5424
               - qcom,apss-wdt-ipq9574
+              - qcom,apss-wdt-ipq9650
               - qcom,apss-wdt-kaanapali
               - qcom,apss-wdt-msm8226
               - qcom,apss-wdt-msm8974
               - qcom,apss-wdt-msm8994
+              - qcom,apss-wdt-nord
               - qcom,apss-wdt-qcm2290
               - qcom,apss-wdt-qcs404
               - qcom,apss-wdt-qcs615
@@ -40,6 +43,7 @@ properties:
               - qcom,apss-wdt-sdm845
               - qcom,apss-wdt-sdx55
               - qcom,apss-wdt-sdx65
+              - qcom,apss-wdt-shikra
               - qcom,apss-wdt-sm6115
               - qcom,apss-wdt-sm6350
               - qcom,apss-wdt-sm8150
@@ -84,6 +88,12 @@ properties:
     minItems: 1
     maxItems: 5
 
+  sram:
+    maxItems: 1
+    description:
+      A reference to an region residing in IMEM(on-chip SRAM), which contains
+      the system restart reason value populated by the bootloader.
+
 required:
   - compatible
   - reg
diff --git a/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml
index 099200c4f1364..975c5aa4d747f 100644
--- a/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml
@@ -89,11 +89,11 @@ examples:
   - |
     #include <dt-bindings/clock/renesas,r9a09g057-cpg.h>
 
-    watchdog@11c00400 {
+    watchdog@14400000 {
             compatible = "renesas,r9a09g057-wdt";
-            reg = <0x11c00400 0x400>;
-            clocks = <&cpg CPG_MOD 0x4b>, <&cpg CPG_MOD 0x4c>;
+            reg = <0x14400000 0x400>;
+            clocks = <&cpg CPG_MOD 0x4d>, <&cpg CPG_MOD 0x4e>;
             clock-names = "pclk", "oscclk";
-            resets = <&cpg 0x75>;
+            resets = <&cpg 0x76>;
             power-domains = <&cpg>;
     };
diff --git a/Documentation/devicetree/bindings/watchdog/renesas,rzg2l-wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,rzg2l-wdt.yaml
index a4d06c9c8b86b..40b4fb26d9c57 100644
--- a/Documentation/devicetree/bindings/watchdog/renesas,rzg2l-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/renesas,rzg2l-wdt.yaml
@@ -18,6 +18,7 @@ properties:
               - renesas,r9a07g044-wdt    # RZ/G2{L,LC}
               - renesas,r9a07g054-wdt    # RZ/V2L
               - renesas,r9a08g045-wdt    # RZ/G3S
+              - renesas,r9a08g046-wdt    # RZ/G3L
           - const: renesas,rzg2l-wdt
 
       - items:
diff --git a/Documentation/devicetree/bindings/watchdog/renesas,rzn1-wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,rzn1-wdt.yaml
index 7e3ee533cd568..0e4b5b529e9c5 100644
--- a/Documentation/devicetree/bindings/watchdog/renesas,rzn1-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/renesas,rzn1-wdt.yaml
@@ -29,7 +29,6 @@ properties:
 required:
   - compatible
   - reg
-  - interrupts
   - clocks
 
 allOf:
diff --git a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml
index 609e98cdaaffd..9d25f5e497e23 100644
--- a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml
@@ -29,10 +29,12 @@ properties:
               - rockchip,rk3368-wdt
               - rockchip,rk3399-wdt
               - rockchip,rk3506-wdt
+              - rockchip,rk3528-wdt
               - rockchip,rk3562-wdt
               - rockchip,rk3568-wdt
               - rockchip,rk3576-wdt
               - rockchip,rk3588-wdt
+              - rockchip,rv1103b-wdt
               - rockchip,rv1108-wdt
               - rockchip,rv1126-wdt
           - const: snps,dw-wdt
diff --git a/Documentation/devicetree/bindings/watchdog/technologic,ts4800-wdt.yaml b/Documentation/devicetree/bindings/watchdog/technologic,ts4800-wdt.yaml
new file mode 100644
index 0000000000000..5c2541ac60cf3
--- /dev/null
+++ b/Documentation/devicetree/bindings/watchdog/technologic,ts4800-wdt.yaml
@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+%YAML 1.2
+---
+$id: http://devicetree.org/schemas/watchdog/technologic,ts4800-wdt.yaml#
+$schema: http://devicetree.org/meta-schemas/core.yaml#
+
+title: Technologic Systems TS-4800 Watchdog
+
+maintainers:
+  - Eduard Bostina <egbostina@gmail.com>
+
+properties:
+  compatible:
+    const: technologic,ts4800-wdt
+
+  syscon:
+    $ref: /schemas/types.yaml#/definitions/phandle-array
+    items:
+      - items:
+          - description: Phandle to the FPGA's syscon
+          - description: Offset to the watchdog register
+    description: Phandle / integers array that points to the syscon node which
+      describes the FPGA's syscon registers.
+
+required:
+  - compatible
+  - syscon
+
+allOf:
+  - $ref: watchdog.yaml#
+
+unevaluatedProperties: false
+
+examples:
+  - |
+      watchdog {
+        compatible = "technologic,ts4800-wdt";
+        syscon = <&syscon 0xe>;
+        timeout-sec = <10>;
+      };
diff --git a/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml
index 62ddc284a524c..2966e5bfb6c09 100644
--- a/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml
+++ b/Documentation/devicetree/bindings/watchdog/ti,rti-wdt.yaml
@@ -23,6 +23,7 @@ allOf:
 properties:
   compatible:
     enum:
+      - ti,am62l-rti-wdt
       - ti,j7-rti-wdt
 
   reg:
diff --git a/Documentation/devicetree/bindings/watchdog/ts4800-wdt.txt b/Documentation/devicetree/bindings/watchdog/ts4800-wdt.txt
deleted file mode 100644
index 8f6caad4258d1..0000000000000
--- a/Documentation/devicetree/bindings/watchdog/ts4800-wdt.txt
+++ /dev/null
@@ -1,25 +0,0 @@
-Technologic Systems Watchdog
-
-Required properties:
-- compatible: must be "technologic,ts4800-wdt"
-- syscon: phandle / integer array that points to the syscon node which
-          describes the FPGA's syscon registers.
-          - phandle to FPGA's syscon
-          - offset to the watchdog register
-
-Optional property:
-- timeout-sec: contains the watchdog timeout in seconds.
-
-Example:
-
-syscon: syscon@b0010000 {
-	compatible = "syscon", "simple-mfd";
-	reg = <0xb0010000 0x3d>;
-	reg-io-width = <2>;
-
-	wdt@e {
-		compatible = "technologic,ts4800-wdt";
-		syscon = <&syscon 0xe>;
-		timeout-sec = <10>;
-	};
-}
diff --git a/Documentation/devicetree/bindings/writing-bindings.rst b/Documentation/devicetree/bindings/writing-bindings.rst
index 667816dd7d504..1a51764833a1f 100644
--- a/Documentation/devicetree/bindings/writing-bindings.rst
+++ b/Documentation/devicetree/bindings/writing-bindings.rst
@@ -53,7 +53,17 @@ Properties
    - DON'T use wildcards or device-family names in compatible strings.
 
    - DO use fallback compatibles when devices are the same as or a superset of
-     prior implementations.
+     prior implementations. Fallback compatibles are applicable especially
+     when sharing a programming interface or when able to discover the
+     variants.
+
+   - DON'T add fake fallback compatibles when software cannot use such to match
+     and bind to a device, and still operate correctly.
+
+   - DO use the commit message to explain why devices that may appear
+     compatible in a diff (e.g. no differences in property use, same handling
+     by the software) but are not made compatible in the binding, are not
+     compatible.
 
    - DO add new compatibles in case there are new features or bugs.
 
diff --git a/Documentation/driver-api/dpll.rst b/Documentation/driver-api/dpll.rst
index 93c191b2d0898..bae14766d4f7b 100644
--- a/Documentation/driver-api/dpll.rst
+++ b/Documentation/driver-api/dpll.rst
@@ -65,35 +65,43 @@ request, where user provides attributes that result in single pin match.
 Pin selection
 =============
 
-In general, selected pin (the one which signal is driving the dpll
-device) can be obtained from ``DPLL_A_PIN_STATE`` attribute, and only
-one pin shall be in ``DPLL_PIN_STATE_CONNECTED`` state for any dpll
-device.
+Pin state (``DPLL_A_PIN_STATE``) reflects the administrative intent set
+by the user. Pin operational state (``DPLL_A_PIN_OPERSTATE``) reflects
+what the hardware is actually doing with the pin.
 
 Pin selection can be done either manually or automatically, depending
 on hardware capabilities and active dpll device work mode
 (``DPLL_A_MODE`` attribute). The consequence is that there are
-differences for each mode in terms of available pin states, as well as
-for the states the user can request for a dpll device.
+differences for each mode in terms of available pin states the user can
+request for a dpll device.
 
-In manual mode (``DPLL_MODE_MANUAL``) the user can request or receive
-one of following pin states:
+In manual mode (``DPLL_MODE_MANUAL``) the user can request one of
+following pin states:
 
-- ``DPLL_PIN_STATE_CONNECTED`` - the pin is used to drive dpll device
-- ``DPLL_PIN_STATE_DISCONNECTED`` - the pin is not used to drive dpll
+- ``DPLL_PIN_STATE_CONNECTED`` - the pin is selected to drive dpll
   device
+- ``DPLL_PIN_STATE_DISCONNECTED`` - the pin is not selected to drive
+  dpll device
 
-In automatic mode (``DPLL_MODE_AUTOMATIC``) the user can request or
-receive one of following pin states:
+In automatic mode (``DPLL_MODE_AUTOMATIC``) the user can request one of
+following pin states:
 
 - ``DPLL_PIN_STATE_SELECTABLE`` - the pin shall be considered as valid
   input for automatic selection algorithm
 - ``DPLL_PIN_STATE_DISCONNECTED`` - the pin shall be not considered as
   a valid input for automatic selection algorithm
 
-In automatic mode (``DPLL_MODE_AUTOMATIC``) the user can only receive
-pin state ``DPLL_PIN_STATE_CONNECTED`` once automatic selection
-algorithm locks a dpll device with one of the inputs.
+The actual hardware status of a pin is reported via the operational
+state (``DPLL_A_PIN_OPERSTATE``) attribute nested under the parent
+device:
+
+- ``DPLL_PIN_OPERSTATE_ACTIVE`` - pin is qualified and actively used
+  by the DPLL
+- ``DPLL_PIN_OPERSTATE_STANDBY`` - pin is qualified but not actively
+  used by the DPLL
+- ``DPLL_PIN_OPERSTATE_NO_SIGNAL`` - pin does not have a valid signal
+- ``DPLL_PIN_OPERSTATE_QUAL_FAILED`` - pin signal failed qualification
+  checks
 
 Shared pins
 ===========
@@ -250,6 +258,26 @@ in the ``DPLL_A_PIN_PHASE_OFFSET`` attribute.
   ``DPLL_A_PHASE_OFFSET_MONITOR`` attr state of a feature
   =============================== ========================
 
+Fractional frequency offset
+===========================
+
+The fractional frequency offset (FFO) is reported through two attributes
+that carry the same measurement at different precisions:
+
+- ``DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET`` in PPM (parts per million)
+- ``DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET_PPT`` in PPT (parts per trillion)
+
+Both attributes appear at the top level of a pin and inside each
+``pin-parent-device`` nest. Two FFO types are defined:
+
+- ``DPLL_FFO_PORT_RXTX_RATE`` - RX vs TX symbol rate offset (top-level)
+- ``DPLL_FFO_PIN_DEVICE`` - pin vs parent DPLL offset (per-parent)
+
+The driver declares which types it supports via the ``supported_ffo``
+bitmask in ``struct dpll_pin_ops``. The core only calls the ``ffo_get``
+callback for types the driver has opted into. The requested type is
+passed to the driver in the ``struct dpll_ffo_param``.
+
 Frequency monitor
 =================
 
diff --git a/Documentation/driver-api/eisa.rst b/Documentation/driver-api/eisa.rst
index 3563e5f7e98d6..e98b21b608657 100644
--- a/Documentation/driver-api/eisa.rst
+++ b/Documentation/driver-api/eisa.rst
@@ -1,3 +1,5 @@
+.. SPDX-License-Identifier: GPL-2.0
+
 ================
 EISA bus support
 ================
diff --git a/Documentation/driver-api/firmware/other_interfaces.rst b/Documentation/driver-api/firmware/other_interfaces.rst
index 06ac89adaafba..6c6fa9a0ba1df 100644
--- a/Documentation/driver-api/firmware/other_interfaces.rst
+++ b/Documentation/driver-api/firmware/other_interfaces.rst
@@ -49,3 +49,136 @@ of the requests on to a secure monitor (EL3).
 
 .. kernel-doc:: drivers/firmware/stratix10-svc.c
    :export:
+
+NXP Secure Enclave Firmware Interface
+=====================================
+
+Introduction
+------------
+The NXP's i.MX HW IP like EdgeLock Enclave, V2X etc., creates an embedded secure
+enclave within the SoC boundary to enable features like:
+
+- Hardware Security Module (HSM)
+- Security Hardware Extension (SHE)
+- Vehicular to Anything (V2X)
+
+Each of the above features is enabled through dedicated NXP H/W IP on the SoC.
+On a single SoC, multiple hardware IP (or can say more than one secure enclave)
+can exist.
+
+NXP SoCs enabled with the such secure enclaves(SEs) IPs are:
+i.MX93, i.MX8ULP
+
+To communicate with one or more co-existing SE(s) on SoC, there is/are dedicated
+messaging units(MU) per SE. Each co-existing SE can have one or multiple exclusive
+MUs, dedicated to itself. None of the MU is shared between two SEs. Communication
+of the MU is realized using the mailbox driver. Each secure enclave can cater to
+multiple clients by virtue of these exclusive MUs. Also, they can distinguish
+transactions originating from these clients based on the MU used and core security
+state. The communication between the clients and secure enclaves is in the form of
+a command/response mechanism. Each client could expose a specific set of secure enclave
+features to the higher layers, based on the commands supported by that client. For
+example, the secure enclave could simultaneously support an OPTEE TA and Linux
+middleware as clients. Each of these clients can expose a specific set of secure
+enclave features based on the command set supported by them.
+
+NXP Secure Enclave(SE) Interface
+--------------------------------
+MU(s) is/are not shared between SE(s). But for an SoC like i.MX95 which has
+multiple SE(s) like HSM, V2X-HSM, V2X-SHE, all the SE(s) and their interfaces 'se-if'
+that is/are dedicated to a particular SE will be enumerated and provisioned using the
+single compatible node("fsl,imx95-se").
+
+Each 'se-if' comprises two layers:
+
+- (C_DEV Layer) User-Space software-access interface.
+- (Service Layer) OS-level software-access interface.
+
+::
+
+   +--------------------------------------------+
+   |            Character Device(C_DEV)         |
+   |                                            |
+   |   +---------+ +---------+     +---------+  |
+   |   | misc #1 | | misc #2 | ... | misc #n |  |
+   |   |  dev    | |  dev    |     | dev     |  |
+   |   +---------+ +---------+     +---------+  |
+   |        +-------------------------+         |
+   |        | Misc. Dev Synchr. Logic |         |
+   |        +-------------------------+         |
+   |                                            |
+   +--------------------------------------------+
+
+   +--------------------------------------------+
+   |               Service Layer                |
+   |                                            |
+   |      +-----------------------------+       |
+   |      | Message Serialization Logic |       |
+   |      +-----------------------------+       |
+   |          +---------------+                 |
+   |          |  imx-mailbox  |                 |
+   |          |   mailbox.c   |                 |
+   |          +---------------+                 |
+   |                                            |
+   +--------------------------------------------+
+
+- service layer:
+  This layer is responsible for ensuring the communication protocol that is defined
+  for communication with firmware.
+
+  FW Communication protocol ensures two things:
+
+  - Serializing the messages to be sent over an MU.
+  - FW can handle one command message at a time.
+
+- c_dev:
+  This layer offers character device contexts, created as '/dev/<se>_mux_chx'.
+  Using these multiple device contexts that are multiplexed over a single MU,
+  userspace application(s) can call fops like write/read to send the command message,
+  and read back the command response message to/from Firmware.
+  fops like read & write use the above defined service layer API(s) to communicate with
+  Firmware.
+
+  Misc-device(/dev/<se>_mux_chn) synchronization protocol::
+
+                                Non-Secure               +   Secure
+                                                         |
+                                                         |
+                +-----------+      +-------------+       |
+                | se_ctrl.c +<---->+imx-mailbox.c|       |
+                |           |      |  mailbox.c  +<-->+------+    +------+
+                +-----+-----+      +-------------+    | MU X +<-->+ ELE |
+                      |                               +------+    +------+
+                      +----------------+                 |
+                      |                |                 |
+                      v                v                 |
+                  logical           logical              |
+                  receiver          waiter               |
+                     +                 +                 |
+                     |                 |                 |
+                     |                 |                 |
+                     |            +----+------+          |
+                     |            |           |          |
+                     |            |           |          |
+              device_ctx     device_ctx     device_ctx   |
+                                                         |
+                User 0        User 1       User Y        |
+                +------+      +------+     +------+      |
+                |misc.c|      |misc.c|     |misc.c|      |
+   kernel space +------+      +------+     +------+      |
+                                                         |
+   +---------------------------------------------------- |
+                    |             |           |          |
+   userspace   /dev/ele_muXch0    |           |          |
+                          /dev/ele_muXch1     |          |
+                                        /dev/ele_muXchY  |
+                                                         |
+
+When a user sends a command to the firmware, it registers its device_ctx
+as waiter of a response from firmware.
+
+Enclave's Firmware owns the storage management over a Linux filesystem.
+For this c_dev provisions a dedicated slave device called "receiver".
+
+.. kernel-doc:: drivers/firmware/imx/se_ctrl.c
+   :export:
diff --git a/Documentation/driver-api/iio/buffers.rst b/Documentation/driver-api/iio/buffers.rst
index 63f364e862d1c..8779022e3da50 100644
--- a/Documentation/driver-api/iio/buffers.rst
+++ b/Documentation/driver-api/iio/buffers.rst
@@ -37,9 +37,10 @@ directory contains attributes of the following form:
 * :file:`index`, the scan_index of the channel.
 * :file:`type`, description of the scan element data storage within the buffer
   and hence the form in which it is read from user space.
-  Format is [be|le]:[s|u]bits/storagebits[Xrepeat][>>shift] .
+  Format is [be|le]:[f|s|u]bits/storagebits[Xrepeat][>>shift] .
 
   * *be* or *le*, specifies big or little endian.
+  * *f*, specifies if floating-point.
   * *s* or *u*, specifies if signed (2's complement) or unsigned.
   * *bits*, is the number of valid data bits.
   * *storagebits*, is the number of bits (after padding) that it occupies in the
@@ -78,7 +79,7 @@ fields in iio_chan_spec definition::
    /* other members */
            int scan_index
            struct {
-                   char sign;
+                   char format;
                    u8 realbits;
                    u8 storagebits;
                    u8 shift;
@@ -98,7 +99,7 @@ following channel definition::
 		   /* other stuff here */
 		   .scan_index = 0,
 		   .scan_type = {
-		           .sign = 's',
+		           .format = IIO_SCAN_FORMAT_SIGNED_INT,
 			   .realbits = 12,
 			   .storagebits = 16,
 			   .shift = 4,
diff --git a/Documentation/driver-api/iio/triggered-buffers.rst b/Documentation/driver-api/iio/triggered-buffers.rst
index 417555dbbdf4f..23b82357eba64 100644
--- a/Documentation/driver-api/iio/triggered-buffers.rst
+++ b/Documentation/driver-api/iio/triggered-buffers.rst
@@ -43,7 +43,7 @@ A typical triggered buffer setup looks like this::
     }
 
     /* setup triggered buffer, usually in probe function */
-    iio_triggered_buffer_setup(indio_dev, sensor_iio_polfunc,
+    iio_triggered_buffer_setup(indio_dev, sensor_iio_pollfunc,
                                sensor_trigger_handler,
                                sensor_buffer_setup_ops);
 
diff --git a/Documentation/driver-api/mailbox.rst b/Documentation/driver-api/mailbox.rst
index 463dd032b96cd..d46c60bd5df64 100644
--- a/Documentation/driver-api/mailbox.rst
+++ b/Documentation/driver-api/mailbox.rst
@@ -87,8 +87,8 @@ a message and a callback function to the API and return immediately).
 		struct async_pkt ap;
 		struct sync_pkt sp;
 
-		dc_sync = kzalloc(sizeof(*dc_sync), GFP_KERNEL);
-		dc_async = kzalloc(sizeof(*dc_async), GFP_KERNEL);
+		dc_sync = kzalloc_obj(*dc_sync);
+		dc_async = kzalloc_obj(*dc_async);
 
 		/* Populate non-blocking mode client */
 		dc_async->cl.dev = &pdev->dev;
diff --git a/Documentation/driver-api/media/camera-sensor.rst b/Documentation/driver-api/media/camera-sensor.rst
index 94bd1dae82d5c..c8552f70f4961 100644
--- a/Documentation/driver-api/media/camera-sensor.rst
+++ b/Documentation/driver-api/media/camera-sensor.rst
@@ -114,7 +114,7 @@ of the device. This is because the power state of the device is only changed
 after the power state transition has taken place. The ``s_ctrl`` callback can be
 used to obtain device's power state after the power state transition:
 
-.. c:function:: int pm_runtime_get_if_in_use(struct device *dev);
+.. c:function:: int pm_runtime_get_if_active(struct device *dev);
 
 The function returns a non-zero value if it succeeded getting the power count or
 runtime PM was disabled, in either of which cases the driver may proceed to
diff --git a/Documentation/driver-api/media/tx-rx.rst b/Documentation/driver-api/media/tx-rx.rst
index 22e1b13ecde9a..7df2407817b35 100644
--- a/Documentation/driver-api/media/tx-rx.rst
+++ b/Documentation/driver-api/media/tx-rx.rst
@@ -93,7 +93,8 @@ where
    * - variable or constant
      - description
    * - link_freq
-     - The value of the ``V4L2_CID_LINK_FREQ`` integer64 menu item.
+     - The value of the :ref:`V4L2_CID_LINK_FREQ <v4l2-cid-link-freq>` integer64
+       menu item.
    * - nr_of_lanes
      - Number of data lanes used on the CSI-2 link.
    * - 2
diff --git a/Documentation/driver-api/media/v4l2-fh.rst b/Documentation/driver-api/media/v4l2-fh.rst
index a934caa483a43..ff1fdd8975040 100644
--- a/Documentation/driver-api/media/v4l2-fh.rst
+++ b/Documentation/driver-api/media/v4l2-fh.rst
@@ -42,7 +42,7 @@ Example:
 
 		...
 
-		my_fh = kzalloc(sizeof(*my_fh), GFP_KERNEL);
+		my_fh = kzalloc_obj(*my_fh);
 
 		...
 
diff --git a/Documentation/filesystems/adding-new-filesystems.rst b/Documentation/filesystems/adding-new-filesystems.rst
new file mode 100644
index 0000000000000..a3d0bf16f73a0
--- /dev/null
+++ b/Documentation/filesystems/adding-new-filesystems.rst
@@ -0,0 +1,195 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+.. _adding_new_filesystems:
+
+Adding New Filesystems
+======================
+
+This document describes what is involved in adding a new filesystem to the
+Linux kernel.
+
+Every filesystem merged into the kernel becomes the collective responsibility
+of the VFS maintainers and the wider filesystem development community.
+Experience has shown that filesystems which become unmaintained impose a
+significant and ongoing burden: they are hard or impossible to test, they
+block infrastructure changes because someone must update or preserve old APIs
+for code that nobody is actively looking after, and they accumulate unfixed
+bugs.  The requirements and expectations described here are informed by this
+experience and are intended to ensure that new filesystems enter the kernel
+on a sustainable footing.
+
+
+Do You Need a New In-Kernel Filesystem?
+---------------------------------------
+
+Before proposing a new in-kernel filesystem, consider whether one of the
+alternatives might be more appropriate.
+
+ - If an existing in-kernel filesystem covers the same use case, improving it
+   is generally preferred over adding a new implementation.  The kernel
+   community favors incremental improvement over parallel implementations.
+
+ - If the filesystem serves a niche audience or has a small user base, a FUSE
+   (Filesystem in Userspace) implementation may be a better fit.  FUSE
+   filesystems avoid the long-term kernel maintenance commitment and can be
+   developed and released on their own schedule.
+
+ - If kernel-level performance, reliability, or integration is genuinely
+   required, make the case explicitly.  Explain who the users are, what the
+   use case is, and why a FUSE implementation would not be sufficient.
+
+
+Technical Requirements
+----------------------
+
+New filesystems must use current kernel interfaces and practices.
+Submitting a filesystem built on outdated APIs creates an unacceptable
+maintenance debt and is likely to face pushback during review.
+
+Use modern VFS interfaces
+  Do not use interfaces listed in
+  :ref:`Documentation/process/deprecated.rst <deprecated>`.
+
+  Use folios rather than raw page operations for page cache management and
+  iomap rather than buffer heads for block mapping and I/O.  See
+  ``Documentation/filesystems/iomap/index.rst`` for iomap documentation.
+
+  Block-based filesystems that need functionality not currently provided by
+  iomap should be prepared to explain why adding that functionality to iomap
+  is infeasible, rather than reimplementing their own block mapping layer.
+
+  Network filesystems should consider using the netfs library
+  (``Documentation/filesystems/netfs_library.rst``), or be prepared to explain
+  why it is not a good fit.
+
+Provide userspace utilities
+  A ``mkfs`` tool is expected so that the filesystem can be created and used
+  by testers and users.  A ``fsck`` tool is strongly recommended; while not
+  strictly required for every filesystem type, the ability to verify
+  consistency and repair corruption is an important part of a mature
+  filesystem.
+
+Be testable
+  The filesystem must be testable in a meaningful way.  The
+  `fstests <https://git.kernel.org/pub/scm/fs/xfs/xfstests-dev.git>`_
+  framework (also known as xfstests) is the standard testing infrastructure
+  for Linux filesystems and its use is highly recommended.  At a minimum,
+  there must be a credible and documented way to test the filesystem and
+  detect regressions.  When submitting, include a summary of test results
+  indicating which tests pass, fail, or are not applicable.
+
+Provide documentation
+  A documentation file under ``Documentation/filesystems/`` describing the
+  filesystem, its on-disk format, mount options, and any notable design
+  decisions is recommended.
+
+
+Community and Maintainership Expectations
+-----------------------------------------
+
+Merging a filesystem is a long-term commitment.  The kernel community
+needs confidence that the filesystem will be actively maintained after it
+is merged.
+
+Identified maintainers
+  The submission must include a ``MAINTAINERS`` entry with at least one
+  maintainer (``M:``), a mailing list (``L:``), and a git tree (``T:``).
+  Having two or more maintainers is strongly preferred so that coverage
+  does not depend on a single person.  The maintainers are expected to be
+  the primary points of contact for the filesystem going forward.
+
+Demonstrated commitment
+  A track record of maintaining kernel code -- for example, in other
+  subsystems -- significantly strengthens the case for a new filesystem.
+  Maintainers who are already known and trusted within the community face
+  less friction during review.
+
+Sustained backing
+  Major filesystems in Linux have organizational or corporate support behind
+  their development.  Filesystems that depend entirely on volunteer effort
+  face higher scrutiny about their long-term viability.
+
+Responsiveness
+  The maintainer is expected to respond to bug reports, address review
+  feedback, and adapt the filesystem to VFS infrastructure changes such as
+  folio conversions, iomap migration, and mount API updates.  Unresponsive
+  maintainership is one of the primary reasons filesystems end up on the
+  path to deprecation.
+
+User base
+  Clearly describe who the users of this filesystem are and the scale of the
+  user base.  Filesystems with a very small or unclear user base face a
+  harder path to acceptance and a higher risk of future deprecation.
+
+Building your track record
+  A practical way to demonstrate many of the qualities above is to maintain
+  the filesystem out-of-tree for a period before requesting a merge.  This
+  shows sustained commitment, builds a visible user base, and gives reviewers
+  confidence that the code and its maintainer will persist after merging.
+  That said, it is recognized that for some filesystems the user base grows
+  significantly only after upstreaming, so a compelling case for expected
+  adoption can substitute for a large existing user base.
+
+
+Submission Process
+------------------
+
+This section covers what is specific to filesystem submissions, over and
+above the normal submission advice in
+:ref:`Documentation/process/submitting-patches.rst <submittingpatches>` and
+:ref:`Documentation/process/submit-checklist.rst <submitchecklist>`.
+
+ - Send patches to the linux-fsdevel mailing list
+   (``linux-fsdevel@vger.kernel.org``).  CC the relevant VFS maintainers as
+   listed in the ``MAINTAINERS`` file under
+   ``FILESYSTEMS (VFS and infrastructure)``.
+
+ - Structure the submission logically.  It is neither acceptable to send one
+   large patch containing the entire filesystem, nor is a replay of the full
+   development history helpful to reviewers.  Instead, split the series by
+   topic -- for example: superblock and mount handling, inode operations,
+   directory operations, address space operations, and so on -- so that each
+   patch is reviewable in isolation.
+
+ - Separate any filesystem-specific ioctls into their own patches with
+   dedicated justification.  Interfaces beyond those already common across
+   other filesystems will receive additional scrutiny because they are hard
+   to maintain and may conflict with future generic interfaces.
+
+ - Expect thorough review.  Filesystem code interacts deeply with the VFS,
+   memory management, and block layers, so reviewers will examine the code
+   carefully.  Address all review feedback and be prepared for multiple
+   revision cycles.
+
+ - It may be appropriate to mark the filesystem as experimental in its Kconfig
+   help text for the first few releases to set expectations while the code
+   stabilizes in-tree.
+
+
+Ongoing Obligations
+-------------------
+
+Merging is not the finish line.  Maintaining a filesystem in the kernel is an
+ongoing commitment.
+
+ - Adapt to VFS infrastructure changes.  The VFS layer evolves continuously;
+   maintainers are expected to keep up with conversions such as folio
+   migration, iomap adoption, and mount API updates.
+
+ - Maintain test coverage.  As test suites evolve, the filesystem's test
+   results should be kept current.
+
+ - Handle security issues and regression promptly.  Both those reported
+   by ordinary users and those reported by test bots and fuzzing tools.
+   The filesystem must handle corrupted input gracefully without corrupting
+   memory, hanging, or crashing the kernel.
+
+ - Engage with the wider filesystem community.  Participate on linux-fsdevel,
+   share approaches to common problems, and look for opportunities to reuse
+   shared infrastructure.  It is inappropriate to develop in isolation on a
+   private list and surface patches only at merge time.
+
+ - Filesystems that become unmaintained -- where the maintainer stops
+   responding, infrastructure changes go unadapted, and testing becomes
+   impossible -- are candidates for deprecation and eventual removal from
+   the kernel.
diff --git a/Documentation/filesystems/cramfs.rst b/Documentation/filesystems/cramfs.rst
index afbdbde98bd25..221c0bf91b9c8 100644
--- a/Documentation/filesystems/cramfs.rst
+++ b/Documentation/filesystems/cramfs.rst
@@ -28,8 +28,10 @@ Only the low 8 bits of gid are stored.  The current version of
 mkcramfs simply truncates to 8 bits, which is a potential security
 issue.
 
-Hard links are supported, but hard linked files
-will still have a link count of 1 in the cramfs image.
+Hard links are not preserved.  mkcramfs deduplicates files with
+identical content, but two names for the same on-disk inode in the
+source tree become two separate (content-shared) entries in the
+image, and cramfs always reports a link count of 1.
 
 Cramfs directories have no ``.`` or ``..`` entries.  Directories (like
 every other file on cramfs) always have a link count of 1.  (There's
@@ -40,12 +42,16 @@ No timestamps are stored in a cramfs, so these default to the epoch
 the update lasts only as long as the inode is cached in memory, after
 which the timestamp reverts to 1970, i.e. moves backwards in time.
 
-Currently, cramfs must be written and read with architectures of the
-same endianness, and can be read only by kernels with PAGE_SIZE
-== 4096.  At least the latter of these is a bug, but it hasn't been
-decided what the best fix is.  For the moment if you have larger pages
-you can just change the #define in mkcramfs.c, so long as you don't
-mind the filesystem becoming unreadable to future kernels.
+The on-disk layout is host-endian: the kernel does not swab, and
+refuses to mount an image whose endianness does not match the CPU.
+For cross-builds, mkcramfs -B / -L forces the output endianness so
+that a host of one endianness can produce an image for a target of
+the other.
+
+The on-disk block size is fixed at 4096 bytes.  On systems with a
+larger PAGE_SIZE you can change the #define in mkcramfs.c, with the
+caveat that the resulting image will only be readable on kernels
+configured for the same PAGE_SIZE.
 
 
 Memory Mapped cramfs image
diff --git a/Documentation/filesystems/f2fs.rst b/Documentation/filesystems/f2fs.rst
index 7e40316312867..5bc37a1c4e511 100644
--- a/Documentation/filesystems/f2fs.rst
+++ b/Documentation/filesystems/f2fs.rst
@@ -137,6 +137,15 @@ noacl			 Disable POSIX Access Control List. Note: acl is enabled
 active_logs=%u		 Support configuring the number of active logs. In the
 			 current design, f2fs supports only 2, 4, and 6 logs.
 			 Default number is 6.
+			 When the underlying block device exposes write
+			 streams, the default active_logs=6 configuration
+			 maps hot, warm, and cold DATA writes to streams 1,
+			 2, and 3, respectively. If only one or two write
+			 streams are available, f2fs falls back to mapping
+			 all DATA writes to stream 1 or mapping hot/warm
+			 to stream 1 and cold to stream 2. If no write
+			 streams are exposed, f2fs leaves the stream
+			 unset.
 disable_ext_identify	 Disable the extension list configured by mkfs, so f2fs
 			 is not aware of cold files such as media files.
 inline_xattr		 Enable the inline xattrs feature.
@@ -1044,11 +1053,14 @@ page allocation for significant performance gains. To minimize code complexity,
 this support is currently excluded from the write path, which requires handling
 complex optimizations such as compression and block allocation modes.
 
-This optional feature is triggered only when a file's immutable bit is set.
-Consequently, F2FS will return EOPNOTSUPP if a user attempts to open a cached
-file with write permissions, even immediately after clearing the bit. Write
-access is only restored once the cached inode is dropped. The usage flow is
-demonstrated below:
+This optional feature is triggered by two mechanisms: the file's immutable bit
+or a specific xattr flag. In both cases, F2FS ensures data integrity by
+restricting the file to a read-only state while large folios are active.
+
+1. Immutable Bit Approach:
+Triggered when the FS_IMMUTABLE_FL is set. This is a strict enforcement
+where the file cannot be modified at all until the bit is cleared and
+the cached inode is dropped.
 
 .. code-block::
 
@@ -1078,3 +1090,31 @@ demonstrated below:
    Written 4096 bytes with pattern = zero, total_time = 29 us, max_latency = 28 us
 
    # rm /data/testfile_read_seq
+
+2. XATTR fadvise Approach:
+A more flexible registration via extended attributes.
+
+.. code-block::
+
+    enum {
+        F2FS_XATTR_FADV_LARGEFOLIO,
+    };
+    unsigned int value = BIT(F2FS_XATTR_FADV_LARGEFOLIO);
+
+    /* Registers the inode number for large folio support in the subsystem.*/
+    # setxattr(file, "user.fadvise", &value, sizeof(unsigned int), 0)
+
+    /* The file must be made Read-Only to transition into the large folio path. */
+    # fchmod(0400, fd)
+
+    /* clean up dirty inode state. */
+    # fsync(fd)
+
+    /* Drop the inode cache.
+    # close(fd)
+
+    /* f2fs_iget() instantiates the inode with large folio support.*/
+    # open()
+
+    /* Returns -EOPNOTSUPP or error to protect the large folio cache.*/
+    # open(WRITE), mkwrite on mmap, or chmod(WRITE)
diff --git a/Documentation/filesystems/index.rst b/Documentation/filesystems/index.rst
index fc7254d01a2b2..1f71cf1595476 100644
--- a/Documentation/filesystems/index.rst
+++ b/Documentation/filesystems/index.rst
@@ -43,6 +43,7 @@ algorithms work.
    caching/index
 
    porting
+   adding-new-filesystems
 
 Filesystem support layers
 =========================
diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst
index 8421ea21bd35e..277e49314d324 100644
--- a/Documentation/filesystems/locking.rst
+++ b/Documentation/filesystems/locking.rst
@@ -584,7 +584,7 @@ write_info:	yes		dqonoff_sem
 FS recursion means calling ->quota_read() and ->quota_write() from superblock
 operations.
 
-More details about quota locking can be found in fs/dquot.c.
+More details about quota locking can be found in fs/quota/dquot.c.
 
 vm_operations_struct
 ====================
diff --git a/Documentation/filesystems/porting.rst b/Documentation/filesystems/porting.rst
index fdf074429cd3a..f546b1d3897fa 100644
--- a/Documentation/filesystems/porting.rst
+++ b/Documentation/filesystems/porting.rst
@@ -1297,7 +1297,6 @@ Several functions are renamed:
 -  kern_path_locked -> start_removing_path
 -  kern_path_create -> start_creating_path
 -  user_path_create -> start_creating_user_path
--  user_path_locked_at -> start_removing_user_path_at
 -  done_path_create -> end_creating_path
 
 ---
diff --git a/Documentation/filesystems/proc.rst b/Documentation/filesystems/proc.rst
index db6167befb7b2..2ccd5b2dfdd33 100644
--- a/Documentation/filesystems/proc.rst
+++ b/Documentation/filesystems/proc.rst
@@ -23,13 +23,13 @@ fixes/update part 1.1  Stefani Seibold <stefani@seibold.net>    June 9 2009
   1	Collecting System Information
   1.1	Process-Specific Subdirectories
   1.2	Kernel data
-  1.3	IDE devices in /proc/ide
-  1.4	Networking info in /proc/net
-  1.5	SCSI info
-  1.6	Parallel port info in /proc/parport
-  1.7	TTY info in /proc/tty
-  1.8	Miscellaneous kernel statistics in /proc/stat
-  1.9	Ext4 file system parameters
+  1.3	Networking info in /proc/net
+  1.4	SCSI info
+  1.5	Parallel port info in /proc/parport
+  1.6	TTY info in /proc/tty
+  1.7	Miscellaneous kernel statistics in /proc/stat
+  1.8	Ext4 file system parameters
+  1.9	/proc/consoles - Shows registered system consoles
 
   2	Modifying System Parameters
 
@@ -52,6 +52,7 @@ fixes/update part 1.1  Stefani Seibold <stefani@seibold.net>    June 9 2009
 
   4	Configuring procfs
   4.1	Mount options
+  4.2	Mount restrictions
 
   5	Filesystem behavior
 
@@ -118,7 +119,7 @@ PTRACE_MODE_ATTACH permissions; CAP_PERFMON capability does not grant access
 to /proc/PID/mem for other processes.
 
 Note that an open file descriptor to /proc/<pid> or to any of its
-contained files or subdirectories does not prevent <pid> being reused
+contained files or subdirectories does not prevent <pid> from being reused
 for some other process in the event that <pid> exits. Operations on
 open /proc/<pid> file descriptors corresponding to dead processes
 never act on any new process that the kernel may, through chance, have
@@ -2211,7 +2212,7 @@ the process is maintaining.  Example output::
      | lr-------- 1 root root 64 Jan 27 11:24 400000-41a000 -> /usr/bin/ls
 
 The name of a link represents the virtual memory bounds of a mapping, i.e.
-vm_area_struct::vm_start-vm_area_struct::vm_end.
+vm_area_struct::vm_start - vm_area_struct::vm_end.
 
 The main purpose of the map_files is to retrieve a set of memory mapped
 files in a fast way instead of parsing /proc/<pid>/maps or
@@ -2425,7 +2426,9 @@ prohibited by hidepid=.  If you use some daemon like identd which needs to learn
 information about processes information, just add identd to this group.
 
 subset=pid hides all top level files and directories in the procfs that
-are not related to tasks.
+are not related to tasks. This option cannot be changed on an existing
+procfs instance because overmounts that existed before the change could
+otherwise remain reachable after the top level procfs entries are hidden.
 
 pidns= specifies a pid namespace (either as a string path to something like
 `/proc/$pid/ns/pid`, or a file descriptor when using `FSCONFIG_SET_FD`) that
@@ -2434,6 +2437,20 @@ will use the calling process's active pid namespace. Note that the pid
 namespace of an existing procfs instance cannot be modified (attempting to do
 so will give an `-EBUSY` error).
 
+4.2	Mount restrictions
+--------------------------
+
+If user namespaces are in use, the kernel additionally checks the instances of
+procfs available to the mounter and will not allow procfs to be mounted if:
+
+  1. This mount is not fully visible unless the new procfs is going to be
+     mounted with subset=pid option.
+
+     a. Its root directory is not the root directory of the filesystem.
+     b. If any file or non-empty procfs directory is hidden by another mount.
+
+  2. A new mount overrides the readonly option or any option from atime family.
+
 Chapter 5: Filesystem behavior
 ==============================
 
diff --git a/Documentation/filesystems/resctrl.rst b/Documentation/filesystems/resctrl.rst
index b003bed339fdd..e4b66af55ffba 100644
--- a/Documentation/filesystems/resctrl.rst
+++ b/Documentation/filesystems/resctrl.rst
@@ -427,9 +427,9 @@ with the following files:
 
 	Two MBM events are supported by default: mbm_local_bytes and mbm_total_bytes.
 	Each MBM event's sub-directory contains a file named "event_filter" that is
-	used to view and modify which memory transactions the MBM event is configured
-	with. The file is accessible only when "mbm_event" counter assignment mode is
-	enabled.
+	used to view and (if writable) modify which memory transactions the MBM event
+	is configured with. The file is accessible only when "mbm_event" counter
+	assignment mode is enabled.
 
 	List of memory transaction types supported:
 
@@ -454,9 +454,8 @@ with the following files:
 	  # cat /sys/fs/resctrl/info/L3_MON/event_configs/mbm_local_bytes/event_filter
 	  local_reads,local_non_temporal_writes,local_reads_slow_memory
 
-	Modify the event configuration by writing to the "event_filter" file within
-	the "event_configs" directory. The read/write "event_filter" file contains the
-	configuration of the event that reflects which memory transactions are counted by it.
+	The memory transactions the MBM event is configured with can be changed
+	if "event_filter" is writable.
 
 	For example::
 
@@ -480,6 +479,12 @@ with the following files:
 	"1":
 		Auto assignment is enabled.
 
+	Automatic counter assignment is done with best effort. If auto
+	assignment is enabled but there are not enough available counters then
+	monitor group creation could succeed while one or more events belonging
+	to the group may not have a counter assigned in all domains. Consult
+	mbm_L3_assignments for counter assignment states of the new groups.
+
 	Example::
 
 	  # echo 0 > /sys/fs/resctrl/info/L3_MON/mbm_assign_on_mkdir
@@ -570,6 +575,11 @@ All groups contain the following files:
 	then the task must already belong to the CTRL_MON parent of this
 	group. The task is removed from any previous MON group.
 
+	When writing to this file, a task id of 0 is interpreted as the
+	task id of the currently running task. On reading the file, a task
+	id of 0 will never be shown and there is no representation of the
+	idle tasks. Instead, a CPU's idle task is always considered as a
+	member of the group owning the CPU.
 
 "cpus":
 	Reading this file shows a bitmask of the logical CPUs owned by
diff --git a/Documentation/firmware-guide/acpi/dsd/motorcomm-yt8xxx-phy.rst b/Documentation/firmware-guide/acpi/dsd/motorcomm-yt8xxx-phy.rst
new file mode 100644
index 0000000000000..d64a396fac815
--- /dev/null
+++ b/Documentation/firmware-guide/acpi/dsd/motorcomm-yt8xxx-phy.rst
@@ -0,0 +1,107 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+======================================
+Motorcomm yt8xxx PHY properties (_DSD)
+======================================
+
+This document describes ACPI _DSD device properties for Motorcomm yt8xxx
+Ethernet PHYs supported by the in-kernel driver in
+``drivers/net/phy/motorcomm.c``.
+
+The properties are exposed on the PHY device object under the MDIO bus ACPI
+device (the same objects that are registered via
+``fwnode_mdiobus_register_phy()``). MAC-side connection properties such as
+``phy-handle`` and ``phy-mode`` are documented in [acpi-mdio-phy]_.
+
+Property names and semantics are intentionally aligned with the Devicetree
+binding [motorcomm-yt8xxx]_ so that the same driver code path
+(``device_property_*`` on ``&phydev->mdio.dev``) can consume firmware
+described either as Devicetree or ACPI _DSD.
+
+UUID and placement
+==================
+
+Per [acpi-dsd-properties-rules]_ and [acpi-mdio-phy]_, properties must be
+placed in an _DSD package using the standard Device Properties UUID::
+
+	ToUUID("daffd814-6eba-4d8c-8a91-bc9bbf4aa301")
+
+Properties
+==========
+
+Unless noted otherwise, integer properties use the same allowed values and
+defaults as [motorcomm-yt8xxx]_.
+
+``rx-internal-delay-ps`` (u32, optional)
+  RGMII RX internal delay in picoseconds. Only meaningful when the link is
+  using RGMII modes with RX internal delay; see [motorcomm-yt8xxx]_.
+
+``tx-internal-delay-ps`` (u32, optional)
+  RGMII TX internal delay in picoseconds. Only meaningful when the link is
+  using RGMII modes with TX internal delay; see [motorcomm-yt8xxx]_.
+
+``motorcomm,clk-out-frequency-hz`` (u32, optional)
+  Clock output frequency on the PHY clock output pin. Allowed values and
+  default are defined in [motorcomm-yt8xxx]_.
+
+``motorcomm,keep-pll-enabled`` (boolean, optional)
+  If true, keep the PLL enabled even when there is no link (useful for using
+  the clock output without an Ethernet link). See [motorcomm-yt8xxx]_.
+
+``motorcomm,auto-sleep-disabled`` (boolean, optional)
+  If true, disable the PHY auto-sleep behavior described in
+  [motorcomm-yt8xxx]_.
+
+``motorcomm,rx-clk-drv-microamp`` (u32, optional)
+  Drive strength for the ``rx_clk`` RGMII pad in microamps. Allowed values
+  depend on the configured RGMII LDO voltage; see [motorcomm-yt8xxx]_.
+
+``motorcomm,rx-data-drv-microamp`` (u32, optional)
+  Drive strength for the ``rx_data`` and ``rx_ctl`` RGMII pads in microamps.
+  See [motorcomm-yt8xxx]_.
+
+``motorcomm,tx-clk-adj-enabled`` (boolean, optional)
+  Enables adjustments related to ``motorcomm,tx-clk-*-inverted`` usage; see
+  [motorcomm-yt8xxx]_.
+
+``motorcomm,tx-clk-10-inverted`` (boolean, optional)
+``motorcomm,tx-clk-100-inverted`` (boolean, optional)
+``motorcomm,tx-clk-1000-inverted`` (boolean, optional)
+  Per-speed TX clock inversion options; see [motorcomm-yt8xxx]_.
+
+ASL example (illustrative)
+==========================
+
+The example below only shows PHY-local _DSD properties. A real platform
+still needs a MAC ``phy-handle`` and ``phy-mode`` package as in
+[acpi-mdio-phy]_.
+
+.. code-block:: none
+
+	Scope (\_SB.MDI0)
+	{
+		Device (PHY4)
+		{
+			Name (_ADR, 0x4)
+
+			Name (_DSD, Package () {
+				ToUUID("daffd814-6eba-4d8c-8a91-bc9bbf4aa301"),
+				Package () {
+					Package (2) { "rx-internal-delay-ps", 2100 },
+					Package (2) { "tx-internal-delay-ps", 150 },
+					Package (2) { "motorcomm,clk-out-frequency-hz", 0 },
+					Package (2) { "motorcomm,keep-pll-enabled", 1 },
+					Package (2) { "motorcomm,auto-sleep-disabled", 1 },
+				}
+			})
+		}
+	}
+
+References
+==========
+
+.. [acpi-mdio-phy] Documentation/firmware-guide/acpi/dsd/phy.rst
+.. [acpi-dsd-properties-rules]
+   Documentation/firmware-guide/acpi/DSD-properties-rules.rst
+.. [motorcomm-yt8xxx]
+   Documentation/devicetree/bindings/net/motorcomm,yt8xxx.yaml
diff --git a/Documentation/firmware-guide/acpi/index.rst b/Documentation/firmware-guide/acpi/index.rst
index b246902f523fe..db9eb7229948c 100644
--- a/Documentation/firmware-guide/acpi/index.rst
+++ b/Documentation/firmware-guide/acpi/index.rst
@@ -11,6 +11,7 @@ ACPI Support
    dsd/graph
    dsd/data-node-references
    dsd/leds
+   dsd/motorcomm-yt8xxx-phy
    dsd/phy
    enumeration
    osi
diff --git a/Documentation/gpu/amdgpu/amdgpu-glossary.rst b/Documentation/gpu/amdgpu/amdgpu-glossary.rst
index 033167025fcca..d553dd599c966 100644
--- a/Documentation/gpu/amdgpu/amdgpu-glossary.rst
+++ b/Documentation/gpu/amdgpu/amdgpu-glossary.rst
@@ -233,8 +233,15 @@ we have a dedicated glossary for Display Core at
     TC
       Texture Cache
 
+    TCC
+      Texture Cache per Channel - L2 cache attached to the memory channels.
+      May be used when shader cores are accessing memory.
+      Despite "Texture" in the name, this is used by any kind of memory access.
+      TCCs may be mapped to TCPs, depending on the architecture.
+
     TCP (AMDGPU)
-      Texture Cache per Pipe. Even though the name "Texture" is part of this
+      Texture Cache per Pipe - L1 cache attached to each CU.
+      Even though the name "Texture" is part of this
       acronym, the TCP represents the path to memory shaders; i.e., it is not
       related to texture. The name is a leftover from older designs where shader
       stages had different cache designs; it refers to the L1 cache in older
diff --git a/Documentation/gpu/amdgpu/index.rst b/Documentation/gpu/amdgpu/index.rst
index 8732084186a48..b2ab182236efb 100644
--- a/Documentation/gpu/amdgpu/index.rst
+++ b/Documentation/gpu/amdgpu/index.rst
@@ -23,3 +23,4 @@ Next (GCN), Radeon DNA (RDNA), and Compute DNA (CDNA) architectures.
    debugfs
    process-isolation
    amdgpu-glossary
+   ptl
diff --git a/Documentation/gpu/amdgpu/ptl.rst b/Documentation/gpu/amdgpu/ptl.rst
new file mode 100644
index 0000000000000..c7f16dea79546
--- /dev/null
+++ b/Documentation/gpu/amdgpu/ptl.rst
@@ -0,0 +1,94 @@
+=======================================
+Peak Tops Limiter (PTL) sysfs Interface
+=======================================
+
+Overview
+--------
+The Peak Tops Limiter (PTL) sysfs interface enables users to control and
+configure the PTL feature for each GPU individually.  All PTL-related
+sysfs files are located under `/sys/class/drm/cardX/device/ptl/`, where
+`X` is the GPU index.  Through these files, users can enable or disable
+PTL, set preferred data formats, and query supported formats for each GPU.
+
+PTL sysfs files
+----------------
+The following files are available under `/sys/class/drm/cardX/device/ptl/`:
+
+- `ptl_enable`
+- `ptl_format`
+- `ptl_supported_formats`
+
+PTL Enable/Disable
+------------------
+File: `ptl_enable`
+Type: Read/Write (rw)
+
+Read: Returns the current PTL status as a string: `enabled` if PTL
+is active, or `disabled` if inactive.
+
+Write:
+
+- Write `1` or `enabled` to enable PTL
+- Write `0` or `disabled` to disable PTL
+
+Examples::
+
+    # Query PTL status
+    cat /sys/class/drm/card1/device/ptl/ptl_enable
+    # Output: enabled
+
+    # Enable PTL
+    sudo bash -c "echo 1 > /sys/class/drm/card1/device/ptl/ptl_enable"
+
+    # Disable PTL
+    sudo bash -c "echo 0 > /sys/class/drm/card1/device/ptl/ptl_enable"
+
+PTL Format (Preferred Data Formats)
+-----------------------------------
+File: `ptl_format`
+Type: Read/Write (rw)
+
+Read: Returns the two preferred formats, e.g. `I8,F32`.
+
+Write: Accepts two formats separated by a comma, e.g. `I8,F32`.
+
+- Both formats must be supported and different.
+- If an invalid format is provided (not supported, or both formats are the
+  same), the driver will return "write error: Invalid argument".
+
+Examples::
+
+    # Query PTL formats
+    cat /sys/class/drm/card1/device/ptl/ptl_format
+    # Output: I8,F32
+
+    # Set PTL formats
+    sudo bash -c "echo I8,F32 > /sys/class/drm/card1/device/ptl/ptl_format"
+
+Supported Formats
+-----------------
+File: `ptl_supported_formats`
+Type: Read-only (r)
+
+Read: Returns a comma-separated list of supported formats, e.g.
+`I8,F16,BF16,F32,F64`.
+
+Example::
+
+    # Check supported formats
+    cat /sys/class/drm/card1/device/ptl/ptl_supported_formats
+    # Output: I8,F16,BF16,F32,F64
+
+Behavioral Notes
+----------------
+- PTL formats can only be set when PTL is enabled.
+- If PTL is disabled, `ptl_format` returns `N/A`.
+- Only two formats can be set at a time, and they must be from the supported set and different..
+- All commands support per-GPU targeting.
+- Root permission is required to enable/disable PTL or change formats.
+- If the hardware does not support PTL, the PTL sysfs directory will not
+  be created.
+
+Implementation
+--------------
+The PTL sysfs nodes are implemented in `drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c`.
diff --git a/Documentation/gpu/driver-uapi.rst b/Documentation/gpu/driver-uapi.rst
index 1f15a8ca12651..627fc68c7a21a 100644
--- a/Documentation/gpu/driver-uapi.rst
+++ b/Documentation/gpu/driver-uapi.rst
@@ -2,6 +2,8 @@
 DRM Driver uAPI
 ===============
 
+.. contents::
+
 drm/i915 uAPI
 =============
 
diff --git a/Documentation/gpu/drivers.rst b/Documentation/gpu/drivers.rst
index 2e13e0ad7e88b..20d2c454aa1d9 100644
--- a/Documentation/gpu/drivers.rst
+++ b/Documentation/gpu/drivers.rst
@@ -8,6 +8,7 @@ GPU Driver Documentation
    amdgpu/index
    i915
    imagination/index
+   intel-display/index
    mcde
    meson
    nouveau
diff --git a/Documentation/gpu/drm-internals.rst b/Documentation/gpu/drm-internals.rst
index 94f93fd3b8a0a..a3ce25a36f1d7 100644
--- a/Documentation/gpu/drm-internals.rst
+++ b/Documentation/gpu/drm-internals.rst
@@ -18,6 +18,8 @@ event handling, memory management, output management, framebuffer
 management, command submission & fencing, suspend/resume support, and
 DMA services.
 
+.. contents::
+
 Driver Initialization
 =====================
 
diff --git a/Documentation/gpu/drm-kms-helpers.rst b/Documentation/gpu/drm-kms-helpers.rst
index b4a9e5ae81f6e..80453dda33b84 100644
--- a/Documentation/gpu/drm-kms-helpers.rst
+++ b/Documentation/gpu/drm-kms-helpers.rst
@@ -33,6 +33,8 @@ There are a few areas these helpers can grouped into:
   pipeline: Planes, handling rectangles for visibility checking and scissoring,
   flip queues and assorted bits.
 
+.. contents::
+
 Modeset Helper Reference for Common Vtables
 ===========================================
 
diff --git a/Documentation/gpu/drm-kms.rst b/Documentation/gpu/drm-kms.rst
index 2292e65f044c3..fa69a5450f967 100644
--- a/Documentation/gpu/drm-kms.rst
+++ b/Documentation/gpu/drm-kms.rst
@@ -1,3 +1,6 @@
+
+.. _drm-kms:
+
 =========================
 Kernel Mode Setting (KMS)
 =========================
@@ -15,6 +18,8 @@ be setup by initializing the following fields.
 -  struct drm_mode_config_funcs \*funcs;
    Mode setting functions.
 
+.. contents::
+
 Overview
 ========
 
@@ -206,11 +211,11 @@ Atomic Mode Setting
           style=dashed
           label="Free-standing state"
 
-          "drm_atomic_state" -> "duplicated drm_plane_state A"
-          "drm_atomic_state" -> "duplicated drm_plane_state B"
-          "drm_atomic_state" -> "duplicated drm_crtc_state"
-          "drm_atomic_state" -> "duplicated drm_connector_state"
-          "drm_atomic_state" -> "duplicated driver private state"
+          "drm_atomic_commit" -> "duplicated drm_plane_state A"
+          "drm_atomic_commit" -> "duplicated drm_plane_state B"
+          "drm_atomic_commit" -> "duplicated drm_crtc_state"
+          "drm_atomic_commit" -> "duplicated drm_connector_state"
+          "drm_atomic_commit" -> "duplicated driver private state"
       }
 
       subgraph cluster_current {
@@ -230,7 +235,7 @@ Atomic Mode Setting
           "driver private object" -> "driver private state"
       }
 
-      "drm_atomic_state" -> "drm_device" [label="atomic_commit"]
+      "drm_atomic_commit" -> "drm_device" [label="atomic_commit"]
       "duplicated drm_plane_state A" -> "drm_device"[style=invis]
    }
 
@@ -265,7 +270,7 @@ Taken all together there's two consequences for the atomic design:
   drm_private_state<drm_private_state>`.
 
 - An atomic update is assembled and validated as an entirely free-standing pile
-  of structures within the :c:type:`drm_atomic_state <drm_atomic_state>`
+  of structures within the :c:type:`drm_atomic_commit <drm_atomic_commit>`
   container. Driver private state structures are also tracked in the same
   structure; see the next chapter.  Only when a state is committed is it applied
   to the driver and modeset objects. This way rolling back an update boils down
diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst
index 32fb506db05b5..2dea94f77d52d 100644
--- a/Documentation/gpu/drm-mm.rst
+++ b/Documentation/gpu/drm-mm.rst
@@ -25,6 +25,8 @@ share it. GEM has simpler initialization and execution requirements than
 TTM, but has no video RAM management capabilities and is thus limited to
 UMA devices.
 
+.. contents::
+
 The Translation Table Manager (TTM)
 ===================================
 
diff --git a/Documentation/gpu/drm-ras.rst b/Documentation/gpu/drm-ras.rst
index 70b246a78fc8a..83c21853b74b6 100644
--- a/Documentation/gpu/drm-ras.rst
+++ b/Documentation/gpu/drm-ras.rst
@@ -24,6 +24,8 @@ Key Goals:
   nodes for different IP blocks, sub-blocks, or other logical subdivisions
   as applicable.
 
+.. contents::
+
 Nodes
 =====
 
@@ -52,6 +54,8 @@ User space tools can:
   as a parameter.
 * Query specific error counter values with the ``get-error-counter`` command, using both
   ``node-id`` and ``error-id`` as parameters.
+* Clear specific error counters with the ``clear-error-counter`` command, using both
+  ``node-id`` and ``error-id`` as parameters.
 
 YAML-based Interface
 --------------------
@@ -101,3 +105,9 @@ Example: Query an error counter for a given node
     sudo ynl --family drm_ras --do get-error-counter --json '{"node-id":0, "error-id":1}'
     {'error-id': 1, 'error-name': 'error_name1', 'error-value': 0}
 
+Example: Clear an error counter for a given node
+
+.. code-block:: bash
+
+    sudo ynl --family drm_ras --do clear-error-counter --json '{"node-id":0, "error-id":1}'
+    None
diff --git a/Documentation/gpu/drm-uapi.rst b/Documentation/gpu/drm-uapi.rst
index 579e87cb9ff75..93df92c4ac8cf 100644
--- a/Documentation/gpu/drm-uapi.rst
+++ b/Documentation/gpu/drm-uapi.rst
@@ -16,6 +16,8 @@ management, and output management.
 Cover generic ioctls and sysfs layout here. We only need high-level
 info, since man pages should cover the rest.
 
+.. contents::
+
 libdrm Device Lookup
 ====================
 
@@ -118,6 +120,10 @@ is already rather painful for the DRM subsystem, with multiple different uAPIs
 for the same thing co-existing. If we add a few more complete mistakes into the
 mix every year it would be entirely unmanageable.
 
+The DRM subsystem has however no concern with independent closed-source
+userspace implementations. To officialize that position, the DRM uAPI headers
+are covered by the MIT license.
+
 .. _drm_render_node:
 
 Render nodes
@@ -568,7 +574,7 @@ ENOSPC:
 EPERM/EACCES:
         Returned for an operation that is valid, but needs more privileges.
         E.g. root-only or much more common, DRM master-only operations return
-        this when called by unpriviledged clients. There's no clear
+        this when called by unprivileged clients. There's no clear
         difference between EACCES and EPERM.
 
 ENODEV:
@@ -761,4 +767,4 @@ Stable uAPI events
 From ``drivers/gpu/drm/scheduler/gpu_scheduler_trace.h``
 
 .. kernel-doc::  drivers/gpu/drm/scheduler/gpu_scheduler_trace.h
-   :doc: uAPI trace events
\ No newline at end of file
+   :doc: uAPI trace events
diff --git a/Documentation/gpu/drm-usage-stats.rst b/Documentation/gpu/drm-usage-stats.rst
index 63d6b2abe5adf..70b7cfcc194f7 100644
--- a/Documentation/gpu/drm-usage-stats.rst
+++ b/Documentation/gpu/drm-usage-stats.rst
@@ -16,6 +16,8 @@ output is split between common and driver specific parts. Having said that,
 wherever possible effort should still be made to standardise as much as
 possible.
 
+.. contents::
+
 File format specification
 =========================
 
@@ -215,3 +217,4 @@ Driver specific implementations
 * :ref:`panfrost-usage-stats`
 * :ref:`panthor-usage-stats`
 * :ref:`xe-usage-stats`
+* :ref:`amdxdna-usage-stats`
diff --git a/Documentation/gpu/i915.rst b/Documentation/gpu/i915.rst
index eba09c3ddce42..0c9d687585331 100644
--- a/Documentation/gpu/i915.rst
+++ b/Documentation/gpu/i915.rst
@@ -1,3 +1,6 @@
+
+.. _drm/i915:
+
 ===========================
  drm/i915 Intel GFX Driver
 ===========================
@@ -7,6 +10,9 @@ models) integrated GFX chipsets with both Intel display and rendering
 blocks. This excludes a set of SoC platforms with an SGX rendering unit,
 those have basic support through the gma500 drm driver.
 
+The display, or :ref:`drm-kms`, support for drm/i915 is provided by
+:ref:`drm/intel-display`, and shared with :ref:`drm/xe <drm/xe>`.
+
 Core Driver Infrastructure
 ==========================
 
@@ -64,200 +70,6 @@ Workarounds
 .. kernel-doc:: drivers/gpu/drm/i915/gt/intel_workarounds.c
    :doc: Hardware workarounds
 
-Display Hardware Handling
-=========================
-
-This section covers everything related to the display hardware including
-the mode setting infrastructure, plane, sprite and cursor handling and
-display, output probing and related topics.
-
-Mode Setting Infrastructure
----------------------------
-
-The i915 driver is thus far the only DRM driver which doesn't use the
-common DRM helper code to implement mode setting sequences. Thus it has
-its own tailor-made infrastructure for executing a display configuration
-change.
-
-Frontbuffer Tracking
---------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_frontbuffer.c
-   :doc: frontbuffer tracking
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_frontbuffer.h
-   :internal:
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_frontbuffer.c
-   :internal:
-
-Display FIFO Underrun Reporting
--------------------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_fifo_underrun.c
-   :doc: fifo underrun handling
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_fifo_underrun.c
-   :internal:
-
-Plane Configuration
--------------------
-
-This section covers plane configuration and composition with the primary
-plane, sprites, cursors and overlays. This includes the infrastructure
-to do atomic vsync'ed updates of all this state and also tightly coupled
-topics like watermark setup and computation, framebuffer compression and
-panel self refresh.
-
-Atomic Plane Helpers
---------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_plane.c
-   :doc: atomic plane helpers
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_plane.c
-   :internal:
-
-Asynchronous Page Flip
-----------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_display.c
-   :doc: asynchronous flip implementation
-
-Output Probing
---------------
-
-This section covers output probing and related infrastructure like the
-hotplug interrupt storm detection and mitigation code. Note that the
-i915 driver still uses most of the common DRM helper code for output
-probing, so those sections fully apply.
-
-Hotplug
--------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_hotplug.c
-   :doc: Hotplug
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_hotplug.c
-   :internal:
-
-High Definition Audio
----------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_audio.c
-   :doc: High Definition Audio over HDMI and Display Port
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_audio.c
-   :internal:
-
-.. kernel-doc:: include/drm/intel/i915_component.h
-   :internal:
-
-Intel HDMI LPE Audio Support
-----------------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_lpe_audio.c
-   :doc: LPE Audio integration for HDMI or DP playback
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_lpe_audio.c
-   :internal:
-
-Panel Self Refresh PSR (PSR/SRD)
---------------------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_psr.c
-   :doc: Panel Self Refresh (PSR/SRD)
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_psr.c
-   :internal:
-
-Frame Buffer Compression (FBC)
-------------------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_fbc.c
-   :doc: Frame Buffer Compression (FBC)
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_fbc.c
-   :internal:
-
-Display Refresh Rate Switching (DRRS)
--------------------------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
-   :doc: Display Refresh Rate Switching (DRRS)
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
-   :internal:
-
-DPIO
-----
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dpio_phy.c
-   :doc: DPIO
-
-DMC Firmware Support
---------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dmc.c
-   :doc: DMC Firmware Support
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dmc.c
-   :internal:
-
-DMC Flip Queue
---------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_flipq.c
-   :doc: DMC Flip Queue
-
-DMC wakelock support
---------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dmc_wl.c
-   :doc: DMC wakelock support
-
-Video BIOS Table (VBT)
-----------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_bios.c
-   :doc: Video BIOS Table (VBT)
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_bios.c
-   :internal:
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_vbt_defs.h
-   :internal:
-
-Display clocks
---------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_cdclk.c
-   :doc: CDCLK / RAWCLK
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_cdclk.c
-   :internal:
-
-Display PLLs
-------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dpll_mgr.c
-   :doc: Display PLLs
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dpll_mgr.c
-   :internal:
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dpll_mgr.h
-   :internal:
-
-Display State Buffer
---------------------
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dsb.c
-   :doc: DSB
-
-.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dsb.c
-   :internal:
-
 GT Programming
 ==============
 
@@ -568,7 +380,7 @@ The HuC FW layout is the same as the GuC one, see `GuC Firmware Layout`_
 
 DMC
 ---
-See `DMC Firmware Support`_
+See :ref:`drm/intel-display/dmc`.
 
 Tracing
 =======
diff --git a/Documentation/gpu/index.rst b/Documentation/gpu/index.rst
index 5d708a106b3fa..65bf3b26e4f4c 100644
--- a/Documentation/gpu/index.rst
+++ b/Documentation/gpu/index.rst
@@ -3,6 +3,7 @@ GPU Driver Developer's Guide
 ============================
 
 .. toctree::
+   :maxdepth: 2
 
    introduction
    drm-internals
diff --git a/Documentation/gpu/intel-display/async-flip.rst b/Documentation/gpu/intel-display/async-flip.rst
new file mode 100644
index 0000000000000..40f93e885bb78
--- /dev/null
+++ b/Documentation/gpu/intel-display/async-flip.rst
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Asynchronous Page Flip
+======================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_display.c
+   :doc: asynchronous flip implementation
diff --git a/Documentation/gpu/intel-display/atomic.rst b/Documentation/gpu/intel-display/atomic.rst
new file mode 100644
index 0000000000000..43a473181e7a1
--- /dev/null
+++ b/Documentation/gpu/intel-display/atomic.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Atomic Modeset Support
+======================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_atomic.c
+   :doc: atomic modeset support
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_atomic.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/audio.rst b/Documentation/gpu/intel-display/audio.rst
new file mode 100644
index 0000000000000..eef95df75f8d2
--- /dev/null
+++ b/Documentation/gpu/intel-display/audio.rst
@@ -0,0 +1,23 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+High Definition Audio
+=====================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_audio.c
+   :doc: High Definition Audio over HDMI and Display Port
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_audio.c
+   :internal:
+
+.. kernel-doc:: include/drm/intel/i915_component.h
+   :internal:
+
+Intel HDMI LPE Audio Support
+============================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_lpe_audio.c
+   :doc: LPE Audio integration for HDMI or DP playback
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_lpe_audio.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/casf.rst b/Documentation/gpu/intel-display/casf.rst
new file mode 100644
index 0000000000000..406778ccd94ca
--- /dev/null
+++ b/Documentation/gpu/intel-display/casf.rst
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Content Adaptive Sharpness Filter (CASF)
+========================================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_casf.c
+   :doc: Content Adaptive Sharpness Filter (CASF)
diff --git a/Documentation/gpu/intel-display/cdclk.rst b/Documentation/gpu/intel-display/cdclk.rst
new file mode 100644
index 0000000000000..a66d623b0ec9b
--- /dev/null
+++ b/Documentation/gpu/intel-display/cdclk.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Display clocks
+==============
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_cdclk.c
+   :doc: CDCLK / RAWCLK
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_cdclk.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/cmtg.rst b/Documentation/gpu/intel-display/cmtg.rst
new file mode 100644
index 0000000000000..04edd0bd165d2
--- /dev/null
+++ b/Documentation/gpu/intel-display/cmtg.rst
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Common Primary Timing Generator (CMTG)
+======================================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_cmtg.c
+   :doc: Common Primary Timing Generator (CMTG)
diff --git a/Documentation/gpu/intel-display/dmc.rst b/Documentation/gpu/intel-display/dmc.rst
new file mode 100644
index 0000000000000..4368da4c70489
--- /dev/null
+++ b/Documentation/gpu/intel-display/dmc.rst
@@ -0,0 +1,26 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+.. _drm/intel-display/dmc:
+
+DMC Firmware Support
+====================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dmc.c
+   :doc: DMC Firmware Support
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dmc.c
+   :internal:
+
+
+DMC Flip Queue
+==============
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_flipq.c
+   :doc: DMC Flip Queue
+
+DMC wakelock support
+====================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dmc_wl.c
+   :doc: DMC wakelock support
diff --git a/Documentation/gpu/intel-display/dpio.rst b/Documentation/gpu/intel-display/dpio.rst
new file mode 100644
index 0000000000000..84d92ac162f85
--- /dev/null
+++ b/Documentation/gpu/intel-display/dpio.rst
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+DPIO
+====
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dpio_phy.c
+   :doc: DPIO
diff --git a/Documentation/gpu/intel-display/dpll.rst b/Documentation/gpu/intel-display/dpll.rst
new file mode 100644
index 0000000000000..c750352e0ae50
--- /dev/null
+++ b/Documentation/gpu/intel-display/dpll.rst
@@ -0,0 +1,14 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Display PLLs
+============
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+   :doc: Display PLLs
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+   :internal:
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dpll_mgr.h
+   :internal:
diff --git a/Documentation/gpu/intel-display/drrs.rst b/Documentation/gpu/intel-display/drrs.rst
new file mode 100644
index 0000000000000..a5aaba63d6b95
--- /dev/null
+++ b/Documentation/gpu/intel-display/drrs.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Display Refresh Rate Switching (DRRS)
+=====================================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
+   :doc: Display Refresh Rate Switching (DRRS)
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_drrs.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/dsb.rst b/Documentation/gpu/intel-display/dsb.rst
new file mode 100644
index 0000000000000..857aca59995ad
--- /dev/null
+++ b/Documentation/gpu/intel-display/dsb.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Display State Buffer
+====================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dsb.c
+   :doc: DSB
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_dsb.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/fbc.rst b/Documentation/gpu/intel-display/fbc.rst
new file mode 100644
index 0000000000000..de9e19021f507
--- /dev/null
+++ b/Documentation/gpu/intel-display/fbc.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Frame Buffer Compression (FBC)
+==============================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_fbc.c
+   :doc: Frame Buffer Compression (FBC)
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_fbc.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/fifo-underrun.rst b/Documentation/gpu/intel-display/fifo-underrun.rst
new file mode 100644
index 0000000000000..5d8f019215069
--- /dev/null
+++ b/Documentation/gpu/intel-display/fifo-underrun.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Display FIFO Underrun Reporting
+===============================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_fifo_underrun.c
+   :doc: fifo underrun handling
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_fifo_underrun.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/frontbuffer.rst b/Documentation/gpu/intel-display/frontbuffer.rst
new file mode 100644
index 0000000000000..7ae38e0827bf1
--- /dev/null
+++ b/Documentation/gpu/intel-display/frontbuffer.rst
@@ -0,0 +1,14 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Frontbuffer Tracking
+====================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_frontbuffer.c
+   :doc: frontbuffer tracking
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_frontbuffer.h
+   :internal:
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_frontbuffer.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/hotplug.rst b/Documentation/gpu/intel-display/hotplug.rst
new file mode 100644
index 0000000000000..f33bc0087c278
--- /dev/null
+++ b/Documentation/gpu/intel-display/hotplug.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Hotplug
+=======
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_hotplug.c
+   :doc: Hotplug
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_hotplug.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/index.rst b/Documentation/gpu/intel-display/index.rst
new file mode 100644
index 0000000000000..01c3d1e576b71
--- /dev/null
+++ b/Documentation/gpu/intel-display/index.rst
@@ -0,0 +1,44 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+.. _drm/intel-display:
+
+====================
+Intel Display Driver
+====================
+
+The Intel display driver provides the display, or :ref:`drm-kms`, support for
+both the :ref:`drm/xe <drm/xe>` and :ref:`drm/i915 <drm/i915>` Intel GPU
+drivers.
+
+The source code currently resides under ``drivers/gpu/drm/i915/display`` due to
+historical reasons, and it's compiled separately into both drm/xe and drm/i915
+kernel modules.
+
+The drm/xe and drm/i915 drivers are the "core" or "parent" drivers for display,
+as they initialize and own the drm device, and pass that on to the display
+driver. The display driver isn't an independent driver in that sense.
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Detailed display topics
+
+   async-flip
+   atomic
+   audio
+   casf
+   cdclk
+   cmtg
+   dmc
+   dpio
+   dpll
+   drrs
+   dsb
+   fbc
+   fifo-underrun
+   frontbuffer
+   hotplug
+   plane
+   psr
+   snps-phy
+   vbt
diff --git a/Documentation/gpu/intel-display/plane.rst b/Documentation/gpu/intel-display/plane.rst
new file mode 100644
index 0000000000000..59932a82051b5
--- /dev/null
+++ b/Documentation/gpu/intel-display/plane.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Atomic Plane Helpers
+====================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_plane.c
+   :doc: atomic plane helpers
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_plane.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/psr.rst b/Documentation/gpu/intel-display/psr.rst
new file mode 100644
index 0000000000000..63e56abcdd564
--- /dev/null
+++ b/Documentation/gpu/intel-display/psr.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Panel Self Refresh PSR (PSR/SRD)
+================================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_psr.c
+   :doc: Panel Self Refresh (PSR/SRD)
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_psr.c
+   :internal:
diff --git a/Documentation/gpu/intel-display/snps-phy.rst b/Documentation/gpu/intel-display/snps-phy.rst
new file mode 100644
index 0000000000000..c9e333fa7f62e
--- /dev/null
+++ b/Documentation/gpu/intel-display/snps-phy.rst
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Synopsis PHY support
+====================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_snps_phy.c
+   :doc: Synopsis PHY support
diff --git a/Documentation/gpu/intel-display/vbt.rst b/Documentation/gpu/intel-display/vbt.rst
new file mode 100644
index 0000000000000..be69f7fd7b396
--- /dev/null
+++ b/Documentation/gpu/intel-display/vbt.rst
@@ -0,0 +1,14 @@
+.. SPDX-License-Identifier: MIT
+.. Copyright © 2026 Intel Corporation
+
+Video BIOS Table (VBT)
+======================
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_bios.c
+   :doc: Video BIOS Table (VBT)
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_bios.c
+   :internal:
+
+.. kernel-doc:: drivers/gpu/drm/i915/display/intel_vbt_defs.h
+   :internal:
diff --git a/Documentation/gpu/introduction.rst b/Documentation/gpu/introduction.rst
index d8f519693fc2c..64074ac22d9b7 100644
--- a/Documentation/gpu/introduction.rst
+++ b/Documentation/gpu/introduction.rst
@@ -16,6 +16,8 @@ found in current kernels.
 
 [Insert diagram of typical DRM stack here]
 
+.. contents::
+
 Style Guidelines
 ================
 
diff --git a/Documentation/gpu/komeda-kms.rst b/Documentation/gpu/komeda-kms.rst
index eaea40eb725b7..9c07459406590 100644
--- a/Documentation/gpu/komeda-kms.rst
+++ b/Documentation/gpu/komeda-kms.rst
@@ -367,7 +367,7 @@ So, one KMS-Obj represents a sub-pipeline of komeda resources.
 So, for komeda, we treat KMS crtc/plane/connector as users of pipeline and
 component, and at any one time a pipeline/component only can be used by one
 user. And pipeline/component will be treated as private object of DRM-KMS; the
-state will be managed by drm_atomic_state as well.
+state will be managed by drm_atomic_commit as well.
 
 How to map plane to Layer(input) pipeline
 -----------------------------------------
@@ -416,8 +416,8 @@ Add :c:type:`drm_private_obj` to :c:type:`komeda_component`, :c:type:`komeda_pip
         ...
     }
 
-Tracking component_state/pipeline_state by drm_atomic_state
------------------------------------------------------------
+Tracking component_state/pipeline_state by drm_atomic_commit
+------------------------------------------------------------
 
 Add :c:type:`drm_private_state` and user to :c:type:`komeda_component_state`,
 :c:type:`komeda_pipeline_state`
@@ -454,7 +454,7 @@ similar, usually including the following steps:
                   put the data flow into next stage.
          Setup 2: check user_state with component features and capabilities to see
                   if requirements can be met; if not, return fail.
-         Setup 3: get component_state from drm_atomic_state, and try set to set
+         Setup 3: get component_state from drm_atomic_commit, and try set to set
                   user to component; fail if component has been assigned to another
                   user already.
          Setup 3: configure the component_state, like set its input component,
diff --git a/Documentation/gpu/nova/core/vbios.rst b/Documentation/gpu/nova/core/vbios.rst
index efd40087480c4..a4fe63422ede9 100644
--- a/Documentation/gpu/nova/core/vbios.rst
+++ b/Documentation/gpu/nova/core/vbios.rst
@@ -46,12 +46,71 @@ region is only accessible to heavy-secure ucode.
    are of type 0xE0 and can be identified as such. This could be subject to change
    in future generations.
 
+IFR Header
+----------
+On Kepler and later GPUs, the ROM begins with an Init-from-ROM (IFR) header
+rather than a standard PCI ROM signature (0xAA55). The driver must parse the
+IFR header to find where the PCI ROM images actually start.
+
+Init-from-ROM (IFR) is a special GPU feature used for power management
+on some Nvidia GPUs.  It references data in the VBIOS for its operation,
+but for drivers the important piece is a header that precedes the
+VBIOS PCI Expansion ROM.
+
+Most such GPUs do not need to parse the IFR header in order to find the
+VBIOS, but the Nvidia GA100 is the exception.  GA100 lacks a display engine,
+so the PRAMIN method (which reads the VBIOS from VRAM via display hardware)
+is unavailable, forcing the driver to read the ROM directly via PROM.
+On other similar GPUs, either PRAMIN succeeds before PROM is tried, or the
+IFR hardware has already applied the ROM offset so that PROM reads
+transparently skip the IFR header.
+
+The driver should first check for the standard 0xAA55 signature at offset 0.
+If found, there is no IFR header and the PCI ROM images start at
+offset 0. If not found, check for the IFR signature and parse the header to
+determine the PCI ROM image offset.
+
+Fixed Header Format
+~~~~~~~~~~~~~~~~~~~
+
+The IFR header begins with four 32-bit words at fixed offsets::
+
+    Offset  Name     Fields
+    ------  -------  ------
+    0x00    FIXED0   bits 31:0  - Signature (must be 0x4947564E, ASCII "NVGI")
+    0x04    FIXED1   bit  31    - Reserved
+                     bits 30:16 - FIXED_DATA_SIZE Fixed data size (offset to extended section)
+                     bits 15:8  - VERSIONSW Software version
+                     bits 7:0   - Reserved
+    0x08    FIXED2   bit  31    - Reserved
+                     bits 30:20 - Reserved (zero)
+                     bits 19:0  - TOTAL_DATA_SIZE Total data size
+
+Finding the PCI ROM Image Offset
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The method to find this offset depends on `VERSIONSW`.
+
+- **Version 1 and 2**: Read `FIXED_DATA_SIZE` from `FIXED1` to get the extended
+  section offset. The PCI ROM image is the 32-bit value at `FIXED_DATA_SIZE + 4`.
+
+- **Version 3**: Read `TOTAL_DATA_SIZE` from `FIXED2`. The 32-bit value at that
+  offset is a flash status offset. Add 4096 to get the ROM directory offset,
+  `ROM_DIRECTORY_OFFSET`.  The ROM directory must have signature 0x44524652
+  (ASCII "RFRD"). The PCI ROM image offset is the 32-bit value at
+  `ROM_DIRECTORY_OFFSET + 8`.
+
+The PCI ROM image offset must be 4-byte aligned. All offsets are relative to the
+start of ROM (BAR0 + 0x300000).
+
 VBIOS ROM Layout
 ----------------
-The VBIOS layout is roughly a series of concatenated images laid out as follows::
+The VBIOS (PCI Expansion ROM) is a series of concatenated images laid out as
+follows. On GPUs with an IFR header, this layout begins at the image offset
+determined by parsing the IFR header. On older GPUs, it begins at offset 0::
 
     +----------------------------------------------------------------------------+
-    | VBIOS (Starting at ROM_OFFSET: 0x300000)                                   |
+    | VBIOS (Starting at ROM_OFFSET: 0x300000 + IFR image offset)                |
     +----------------------------------------------------------------------------+
     | +-----------------------------------------------+                          |
     | | PciAt Image (Type 0x00)                       |                          |
diff --git a/Documentation/gpu/rfc/index.rst b/Documentation/gpu/rfc/index.rst
index ef19b0ba2a3ea..26a7ebe6fb445 100644
--- a/Documentation/gpu/rfc/index.rst
+++ b/Documentation/gpu/rfc/index.rst
@@ -18,23 +18,9 @@ host such documentation:
 
 .. toctree::
 
-    gpusvm.rst
-
-.. toctree::
-
-    i915_gem_lmem.rst
-
-.. toctree::
-
-    i915_scheduler.rst
-
-.. toctree::
-
-    i915_small_bar.rst
-
-.. toctree::
-
-    i915_vm_bind.rst
-
-.. toctree::
-    color_pipeline.rst
\ No newline at end of file
+   gpusvm
+   i915_gem_lmem
+   i915_scheduler
+   i915_small_bar
+   i915_vm_bind
+   color_pipeline
diff --git a/Documentation/gpu/todo.rst b/Documentation/gpu/todo.rst
index bc9f14c8a2ec2..cdddf8db35f53 100644
--- a/Documentation/gpu/todo.rst
+++ b/Documentation/gpu/todo.rst
@@ -152,29 +152,6 @@ Contact: Simona Vetter, respective driver maintainers
 
 Level: Advanced
 
-Rename drm_atomic_state
------------------------
-
-The KMS framework uses two slightly different definitions for the ``state``
-concept. For a given object (plane, CRTC, encoder, etc., so
-``drm_$OBJECT_state``), the state is the entire state of that object. However,
-at the device level, ``drm_atomic_state`` refers to a state update for a
-limited number of objects.
-
-The state isn't the entire device state, but only the full state of some
-objects in that device. This is confusing to newcomers, and
-``drm_atomic_state`` should be renamed to something clearer like
-``drm_atomic_commit``.
-
-In addition to renaming the structure itself, it would also imply renaming some
-related functions (``drm_atomic_state_alloc``, ``drm_atomic_state_get``,
-``drm_atomic_state_put``, ``drm_atomic_state_init``,
-``__drm_atomic_state_free``, etc.).
-
-Contact: Maxime Ripard <mripard@kernel.org>
-
-Level: Advanced
-
 Fallout from atomic KMS
 -----------------------
 
diff --git a/Documentation/gpu/xe/index.rst b/Documentation/gpu/xe/index.rst
index bc432c95d1a35..665c0e93601c5 100644
--- a/Documentation/gpu/xe/index.rst
+++ b/Documentation/gpu/xe/index.rst
@@ -1,5 +1,7 @@
 .. SPDX-License-Identifier: (GPL-2.0+ OR MIT)
 
+.. _drm/xe:
+
 =======================
 drm/xe Intel GFX Driver
 =======================
@@ -8,6 +10,9 @@ The drm/xe driver supports some future GFX cards with rendering, display,
 compute and media. Support for currently available platforms like TGL, ADL,
 DG2, etc is provided to prototype the driver.
 
+The display, or :ref:`drm-kms`, support for drm/xe is provided by
+:ref:`drm/intel-display`, and shared with :ref:`drm/i915 <drm/i915>`.
+
 .. toctree::
    :titlesonly:
 
@@ -29,3 +34,4 @@ DG2, etc is provided to prototype the driver.
    xe_device
    xe-drm-usage-stats.rst
    xe_configfs
+   xe_gt_stats
diff --git a/Documentation/gpu/xe/xe_gt_stats.rst b/Documentation/gpu/xe/xe_gt_stats.rst
new file mode 100644
index 0000000000000..5ff806abaddb8
--- /dev/null
+++ b/Documentation/gpu/xe/xe_gt_stats.rst
@@ -0,0 +1,11 @@
+.. SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+
+================
+Xe GT Statistics
+================
+
+.. kernel-doc:: drivers/gpu/drm/xe/xe_gt_stats.c
+   :doc: Xe GT Statistics
+
+.. kernel-doc:: drivers/gpu/drm/xe/xe_gt_stats_types.h
+   :internal:
diff --git a/Documentation/hwmon/adt7411.rst b/Documentation/hwmon/adt7411.rst
index 57ad16fb216af..28b6f3cb6b86c 100644
--- a/Documentation/hwmon/adt7411.rst
+++ b/Documentation/hwmon/adt7411.rst
@@ -30,11 +30,36 @@ Check the datasheet for details.
 sysfs-Interface
 ---------------
 
-================ =================
-in0_input	 vdd voltage input
-in[1-8]_input	 analog 1-8 input
-temp1_input	 temperature input
-================ =================
+================    =================================
+in0_input           vdd voltage input
+in0_min             vdd low limit
+in0_max             vdd high limit
+in0_alarm           vdd alarm
+in[1-8]_input       analog 1-8 input
+in[1-8]_min         analog input low limit
+in[1-8]_max         analog input high limit
+in[1-8]_alarm       analog input alarm
+temp1_input         internal temperature input
+temp1_min           internal temperature low limit
+temp1_max           internal temperature high limit
+temp1_min_alarm     internal temperature low alarm
+temp1_max_alarm     internal temperature high alarm
+================    =================================
+
+If the external temperature sensor is enabled, the following attributes are
+also available:
+
+================ ================================================
+temp2_input      external temperature input
+temp2_min        external temperature low limit
+temp2_max        external temperature high limit
+temp2_min_alarm  external temperature low alarm
+temp2_max_alarm  external temperature high alarm
+temp2_fault      external temperature sensor fault
+================ ================================================
+
+If the external temperature sensor is enabled, analog inputs in1 and in2 are
+not available.
 
 Besides standard interfaces, this driver adds (0 = off, 1 = on):
 
@@ -47,4 +72,4 @@ Besides standard interfaces, this driver adds (0 = off, 1 = on):
 Notes
 -----
 
-SPI, external temperature sensor and limit registers are not supported yet.
+SPI is not supported yet.
diff --git a/Documentation/hwmon/arctic_fan_controller.rst b/Documentation/hwmon/arctic_fan_controller.rst
new file mode 100644
index 0000000000000..b5be88ae464d7
--- /dev/null
+++ b/Documentation/hwmon/arctic_fan_controller.rst
@@ -0,0 +1,56 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+Kernel driver arctic_fan_controller
+=====================================
+
+Supported devices:
+
+* ARCTIC Fan Controller (USB HID, VID 0x3904, PID 0xF001)
+
+Author: Aureo Serrano de Souza <aureo.serrano@arctic.de>
+
+Description
+-----------
+
+This driver provides hwmon support for the ARCTIC Fan Controller, a USB
+Custom HID device with 10 fan channels. The device sends IN reports about
+once per second containing current RPM values (bytes 11-30, 10 x uint16 LE).
+Fan speed control is manual-only: the device does not change PWM
+autonomously; it only applies a new duty cycle when it receives an OUT
+report from the host.
+
+After the device applies an OUT report, it sends back a 2-byte ACK IN
+report (Report ID 0x02, byte 1 = 0x00 on success) confirming the command
+was applied.
+
+Usage notes
+-----------
+
+Since it is a USB device, hotplug is supported. The device is autodetected.
+
+The device does not support GET_REPORT, so the driver cannot read back the
+current hardware PWM state at probe time. The cached PWM values (readable
+via pwm[1-10]) start at 0 and reflect only values that have been
+successfully written. Because each OUT report carries all 10 channel values,
+writing a single channel also sends the cached values for all other channels.
+Users should set all channels to the desired values before relying on the
+cached state.
+
+On system suspend, the device may lose power and reset its PWM channels to
+hardware defaults. The driver clears its cached duty values on resume so
+that reads reflect the unknown hardware state rather than stale pre-suspend
+values. Userspace is responsible for re-applying the desired duty cycles
+after resume.
+
+Sysfs entries
+-------------
+
+================ ==============================================================
+fan[1-10]_input  Fan speed in RPM (read-only). Updated from IN reports at ~1 Hz.
+pwm[1-10]        PWM duty cycle (0-255). Write: sends an OUT report setting the
+                 duty cycle (scaled from 0-255 to 0-100% for the device);
+                 the cached value is updated only after the device ACKs the
+                 command with a success status. Read: returns the last
+                 successfully written value; initialized to 0 at driver load
+                 and after resume (hardware state unknown).
+================ ==============================================================
diff --git a/Documentation/hwmon/coretemp.rst b/Documentation/hwmon/coretemp.rst
index 7a5fbb37b0f33..f63b21f24d42c 100644
--- a/Documentation/hwmon/coretemp.rst
+++ b/Documentation/hwmon/coretemp.rst
@@ -22,8 +22,7 @@ Supported chips:
 
 	       Intel 64 and IA-32 Architectures Software Developer's Manual
 	       Volume 3A: System Programming Guide
-
-	       http://softwarecommunity.intel.com/Wiki/Mobility/720.htm
+	       (https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html)
 
 Author: Rudolf Marek
 
@@ -51,12 +50,13 @@ All Sysfs entries are named with their core_id (represented here by 'X').
 
 ================= ========================================================
 tempX_input	  Core temperature (in millidegrees Celsius).
-tempX_max	  All cooling devices should be turned on (on Core2).
+tempX_max	  Maximum recommended operating temperature (Tcontrol).
+		  All cooling devices should be turned on.
 tempX_crit	  Maximum junction temperature (in millidegrees Celsius).
 tempX_crit_alarm  Set when Out-of-spec bit is set, never clears.
 		  Correct CPU operation is no longer guaranteed.
 tempX_label	  Contains string "Core X", where X is processor
-		  number. For Package temp, this will be "Physical id Y",
+		  number. For Package temp, this will be "Package id Y",
 		  where Y is the package number.
 ================= ========================================================
 
diff --git a/Documentation/hwmon/d1u74t.rst b/Documentation/hwmon/d1u74t.rst
new file mode 100644
index 0000000000000..3a9eedbda4837
--- /dev/null
+++ b/Documentation/hwmon/d1u74t.rst
@@ -0,0 +1,81 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+Kernel driver d1u74t
+====================
+
+Supported chips:
+
+  * Murata D1U74T
+
+    Prefix: 'd1u74t'
+
+    Addresses scanned: -
+
+    Datasheet: Publicly available at the Murata website
+
+Authors:
+    Abdurrahman Hussain <abdurrahman@nexthop.ai>
+
+
+Description
+-----------
+
+This driver implements support for Murata D1U74T Power Supply with
+PMBus support.
+
+The driver is a client driver to the core PMBus driver.
+Please see Documentation/hwmon/pmbus.rst for details on PMBus client drivers.
+
+
+Usage Notes
+-----------
+
+This driver does not auto-detect devices. You will have to instantiate the
+devices explicitly. Please see Documentation/i2c/instantiating-devices.rst for
+details.
+
+
+Sysfs entries
+-------------
+
+======================= ======================================================
+curr1_label		"iin"
+curr1_input		Measured input current
+curr1_alarm		Input current alarm
+curr1_rated_max		Maximum rated input current
+
+curr2_label		"iout1"
+curr2_input		Measured output current
+curr2_max		Maximum output current
+curr2_max_alarm		Output current high alarm
+curr2_crit		Critical high output current
+curr2_crit_alarm	Output current critical high alarm
+curr2_rated_max		Maximum rated output current
+
+in1_label		"vin"
+in1_input		Measured input voltage
+in1_alarm		Input voltage alarm
+in1_rated_min		Minimum rated input voltage
+in1_rated_max		Maximum rated input voltage
+
+in2_label		"vout1"
+in2_input		Measured output voltage
+in2_alarm		Output voltage alarm
+in2_rated_min		Minimum rated output voltage
+in2_rated_max		Maximum rated output voltage
+
+power1_label		"pin"
+power1_input		Measured input power
+power1_alarm		Input power alarm
+power1_rated_max	Maximum rated input power
+
+temp[1-3]_input		Measured temperature
+temp[1-3]_max		Maximum temperature
+temp[1-3]_max_alarm	Maximum temperature alarm
+temp[1-3]_rated_max	Maximum rated temperature
+
+fan1_alarm		Fan 1 warning
+fan1_fault		Fan 1 fault
+fan1_input		Fan 1 speed in RPM
+fan1_target		Fan 1 target
+======================= ======================================================
diff --git a/Documentation/hwmon/e50sn12051.rst b/Documentation/hwmon/e50sn12051.rst
new file mode 100644
index 0000000000000..cbbfa7895d829
--- /dev/null
+++ b/Documentation/hwmon/e50sn12051.rst
@@ -0,0 +1,81 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+Kernel driver e50sn12051
+========================
+
+Supported chips:
+
+  * Delta E50SN12051
+
+    Prefix: 'e50sn12051'
+
+    Addresses scanned: -
+
+    Datasheet:
+
+      Provided by Delta upon request and NDA
+
+Description
+-----------
+
+E50SN12051 is a 600W non-isolated 1/8th brick DC-DC power module.
+
+This driver supports the E50SN12051 PMBus compliant monitor device.
+
+The device provides measurements for:
+
+* Input voltage
+* Output current
+* Output voltage
+* Temperature
+
+The driver is based on the PMBus core and exposes standard hwmon
+sysfs attributes.
+
+Sysfs attributes
+----------------
+
+======================= =======================================================
+curr1_label             "iout1"
+curr1_crit              Critical maximum current.
+                        From IOUT_OC_FAULT_LIMIT register.
+curr1_crit_alarm        Current critical high alarm.
+                        From IOUT_OC_FAULT status.
+curr1_input             Measured current.
+                        From READ_IOUT register.
+curr1_max               Maximum current.
+                        From IOUT_OC_WARN_LIMIT register.
+curr1_max_alarm         Current high alarm.
+                        From IOUT_OC_WARN_LIMIT status.
+
+in1_label               "vin"
+in1_alarm               Input voltage alarm.
+in1_input               Measured voltage.
+                        From READ_VIN register.
+
+in2_label               "vout1"
+in2_crit                Critical maximum voltage.
+                        From VOUT_OV_FAULT_LIMIT register.
+in2_crit_alarm          Voltage critical high alarm.
+                        From VOLTAGE_OV_FAULT status.
+in2_input               Measured voltage.
+                        From READ_VOUT register.
+in2_max                 Maximum voltage.
+                        From VOUT_OV_WARN_LIMIT register.
+in2_max_alarm           Voltage high alarm.
+                        From VOLTAGE_OV_WARNING status.
+
+temp1_crit              Critical high temperature.
+                        From OT_FAULT_LIMIT register.
+temp1_crit_alarm        Module temperature critical high alarm.
+                        Set by comparing READ_TEMPERATURE_1 with OT_FAULT_LIMIT
+                        if TEMP_OT_FAULT status is set.
+temp1_input             Measured module's hot spot temperature.
+                        From READ_TEMPERATURE_1 register.
+temp1_max               Maximum temperature.
+                        From OT_WARN_LIMIT register.
+temp1_max_alarm         Module temperature high alarm.
+                        Set by comparing READ_TEMPERATURE_1 with OT_WARN_LIMIT if
+                        TEMP_OT_WARNING status is set.
+======================= =======================================================
+
diff --git a/Documentation/hwmon/htu31.rst b/Documentation/hwmon/htu31.rst
index ccde84264643d..9ab774dcf65de 100644
--- a/Documentation/hwmon/htu31.rst
+++ b/Documentation/hwmon/htu31.rst
@@ -35,3 +35,10 @@ temp1_input:        temperature input
 humidity1_input:    humidity input
 heater_enable:      heater control
 =================== =================
+
+debugfs-Interface
+-----------------
+
+=================== =========================================
+serial_number:      unique serial number of the sensor
+=================== =========================================
diff --git a/Documentation/hwmon/hwmon-kernel-api.rst b/Documentation/hwmon/hwmon-kernel-api.rst
index 1d7f1397a8274..9fcde32a140df 100644
--- a/Documentation/hwmon/hwmon-kernel-api.rst
+++ b/Documentation/hwmon/hwmon-kernel-api.rst
@@ -85,9 +85,10 @@ removal.
 When using ``[devm_]hwmon_device_register_with_info()`` to register the
 hardware monitoring device, accesses using the associated access functions
 are serialised by the hardware monitoring core. If a driver needs locking
-for other functions such as interrupt handlers or for attributes which are
-fully implemented in the driver, hwmon_lock() and hwmon_unlock() can be used
-to ensure that calls to those functions are serialized.
+for other functions such as interrupt handlers, attributes which are fully
+implemented in the driver, or debugfs functions, hwmon_lock() and hwmon_unlock()
+can be used to ensure that calls to those functions are serialized. Those
+functions also support guard() and scoped_guard() variants.
 
 Using devm_hwmon_device_register_with_info()
 --------------------------------------------
diff --git a/Documentation/hwmon/index.rst b/Documentation/hwmon/index.rst
index 8b655e5d6b68b..e880c6ca84f03 100644
--- a/Documentation/hwmon/index.rst
+++ b/Documentation/hwmon/index.rst
@@ -43,6 +43,7 @@ Hardware Monitoring Kernel Drivers
    amc6821
    aps-379
    aquacomputer_d5next
+   arctic_fan_controller
    asb100
    asc7621
    aspeed-g6-pwm-tach
@@ -60,6 +61,7 @@ Hardware Monitoring Kernel Drivers
    corsair-psu
    cros_ec_hwmon
    crps
+   d1u74t
    da9052
    da9055
    dell-smm-hwmon
@@ -68,6 +70,7 @@ Hardware Monitoring Kernel Drivers
    drivetemp
    ds1621
    ds620
+   e50sn12051
    emc1403
    emc2103
    emc2305
@@ -146,6 +149,7 @@ Hardware Monitoring Kernel Drivers
    ltc4261
    ltc4282
    ltc4286
+   lx1308
    macsmc-hwmon
    max127
    max15301
@@ -158,6 +162,7 @@ Hardware Monitoring Kernel Drivers
    max197
    max20730
    max20751
+   max20830
    max31722
    max31730
    max31760
@@ -185,6 +190,7 @@ Hardware Monitoring Kernel Drivers
    mp2925
    mp29502
    mp2975
+   mp2985
    mp2993
    mp5023
    mp5920
diff --git a/Documentation/hwmon/lm75.rst b/Documentation/hwmon/lm75.rst
index 4269da04508ef..ca46754e028b3 100644
--- a/Documentation/hwmon/lm75.rst
+++ b/Documentation/hwmon/lm75.rst
@@ -181,3 +181,19 @@ is supported by this driver, other specific enhancements are not.
 
 The LM77 is not supported, contrary to what we pretended for a long time.
 Both chips are simply not compatible, value encoding differs.
+
+sysfs-Interface
+---------------
+
+The following list includes the sysfs attributes that the driver provides, their
+permissions and a short description:
+
+=============================== ======= ===========================================
+Name                            Perm    Description
+=============================== ======= ===========================================
+temp1_input                     RO      Temperature input
+temp1_label                     RO      Descriptive name for the sensor
+temp1_max                       RW      Maximum temperature
+temp1_max_hyst                  RW      Maximum hysteresis temperature
+update_interval                 RW      Update conversions interval in milliseconds
+=============================== ======= ===========================================
diff --git a/Documentation/hwmon/lx1308.rst b/Documentation/hwmon/lx1308.rst
new file mode 100644
index 0000000000000..c1b72e1647c52
--- /dev/null
+++ b/Documentation/hwmon/lx1308.rst
@@ -0,0 +1,90 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+Kernel driver lx1308
+====================
+
+Supported chips:
+
+  * Luxshare LX1308
+
+    Prefixes: 'lx1308'
+
+    Addresses scanned: -
+
+    Datasheet: Datasheet is not publicly available.
+
+Author: Brian Chiang <chiang.brian@inventec.com>
+
+
+Description
+-----------
+
+The LX1308 is a high-efficiency, non-isolated, regulated 12V, 860W,
+digital DC/DC power module. The module operates from a 40V to 60V DC
+primary bus and provides a 12V regulated output voltage. It can deliver
+up to 860W continuous and 1300W in transient.
+
+The module has slow OCP and fast OCP. If the module output current is higher
+than slow OCP set point and the lasting time is also longer than the delay,
+the module will shut down and retry 3 time, if the fault still exists then
+module enter latch mode.
+
+If the module output current is higher than fast OCP set point then it shut
+down and enter latch mode.
+
+The driver is a client driver to the core PMBus driver.
+Please see Documentation/hwmon/pmbus.rst for details on PMBus client drivers.
+
+
+Usage Notes
+-----------
+
+This driver does not auto-detect devices. You will have to instantiate the
+devices explicitly. Please see Documentation/i2c/instantiating-devices.rst for
+details.
+
+
+Sysfs entries
+-------------
+
+======================= ======================================================
+curr1_alarm             Input current alarm
+curr1_input             Input current (IIN)
+curr1_label             "iin"
+curr2_crit              Output over current fault threshold (slow OCP, 60ms delay)
+curr2_crit_alarm        Output over current fault alarm
+curr2_input             Output current (IOUT)
+curr2_label             "iout1"
+curr2_max               Output over current warning threshold (slow OCP, 60ms delay)
+curr2_max_alarm         Output over current warning alarm
+in1_crit                Input over voltage fault threshold
+in1_crit_alarm          Input over voltage fault alarm
+in1_input               Input voltage (VIN)
+in1_label               "vin"
+in1_lcrit               Input under voltage fault threshold
+in1_lcrit_alarm         Input under voltage fault alarm
+in1_max                 Input over voltage warning threshold
+in1_max_alarm           Input over voltage warning alarm
+in1_min                 Input under voltage warning threshold
+in1_min_alarm           Input under voltage warning alarm
+in2_crit                Output over voltage fault threshold
+in2_crit_alarm          Output over voltage fault alarm
+in2_input               Output voltage (VOUT)
+in2_label               "vout1"
+in2_lcrit               Output under voltage fault threshold
+in2_lcrit_alarm         Output under voltage fault alarm
+in2_max                 Output over voltage warning threshold
+in2_max_alarm           Output over voltage warning alarm
+in2_min                 Output under voltage warning threshold
+in2_min_alarm           Output under voltage warning alarm
+power1_alarm            Input power alarm
+power1_input            Input power (PIN)
+power1_label            "pin"
+power2_input            Output power (POUT)
+power2_label            "pout1"
+temp1_crit              Over temperature fault threshold
+temp1_crit_alarm        Over temperature fault alarm
+temp1_input             Module hot spot temperature
+temp1_max               Over temperature warning threshold
+temp1_max_alarm         Over temperature warning alarm
+======================= ======================================================
diff --git a/Documentation/hwmon/max20830.rst b/Documentation/hwmon/max20830.rst
new file mode 100644
index 0000000000000..936e409dcc5c0
--- /dev/null
+++ b/Documentation/hwmon/max20830.rst
@@ -0,0 +1,49 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver max20830
+======================
+
+Supported chips:
+
+  * Analog Devices MAX20830
+
+    Prefix: 'max20830'
+
+    Addresses scanned: -
+
+    Datasheet: https://www.analog.com/media/en/technical-documentation/data-sheets/max20830.pdf
+
+Author:
+
+  - Alexis Czezar Torreno <alexisczezar.torreno@analog.com>
+
+
+Description
+-----------
+
+This driver supports hardware monitoring for Analog Devices MAX20830
+Step-Down Switching Regulator with PMBus Interface.
+
+The MAX20830 is a 2.7V to 16V, 30A fully integrated step-down DC-DC switching
+regulator. Through the PMBus interface, the device can monitor input/output
+voltages, output current and temperature.
+
+The driver is a client driver to the core PMBus driver. Please see
+Documentation/hwmon/pmbus.rst for details on PMBus client drivers.
+
+Sysfs entries
+-------------
+
+================= ========================================
+in1_label         "vin"
+in1_input         Measured input voltage
+in1_alarm         Input voltage alarm
+in2_label         "vout1"
+in2_input         Measured output voltage
+in2_alarm         Output voltage alarm
+curr1_label       "iout1"
+curr1_input       Measured output current
+curr1_alarm       Output current alarm
+temp1_input       Measured temperature
+temp1_alarm       Chip temperature alarm
+================= ========================================
diff --git a/Documentation/hwmon/max31730.rst b/Documentation/hwmon/max31730.rst
index 1c5a32b641879..0936ba2eac247 100644
--- a/Documentation/hwmon/max31730.rst
+++ b/Documentation/hwmon/max31730.rst
@@ -1,4 +1,4 @@
-Kernel driver max31790
+Kernel driver max31730
 ======================
 
 Supported chips:
diff --git a/Documentation/hwmon/mp2985.rst b/Documentation/hwmon/mp2985.rst
new file mode 100644
index 0000000000000..87a39c8a300c6
--- /dev/null
+++ b/Documentation/hwmon/mp2985.rst
@@ -0,0 +1,147 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Kernel driver mp2985
+====================
+
+Supported chips:
+
+  * MPS mp2985
+
+    Prefix: 'mp2985'
+
+Author:
+
+	Wensheng Wang <wenswang@yeah.net>
+
+Description
+-----------
+
+This driver implements support for Monolithic Power Systems, Inc. (MPS)
+MP2985 Dual Loop Digital Multi-phase Controller.
+
+Device compliant with:
+
+- PMBus rev 1.3 interface.
+
+The driver exports the following attributes via the 'sysfs' files
+for input voltage:
+
+**in1_input**
+
+**in1_label**
+
+**in1_crit**
+
+**in1_crit_alarm**
+
+**in1_lcrit**
+
+**in1_lcrit_alarm**
+
+**in1_max**
+
+**in1_max_alarm**
+
+**in1_min**
+
+**in1_min_alarm**
+
+The driver provides the following attributes for output voltage:
+
+**in2_input**
+
+**in2_label**
+
+**in2_crit**
+
+**in2_crit_alarm**
+
+**in2_lcrit**
+
+**in2_lcrit_alarm**
+
+**in3_input**
+
+**in3_label**
+
+**in3_crit**
+
+**in3_crit_alarm**
+
+**in3_lcrit**
+
+**in3_lcrit_alarm**
+
+The driver provides the following attributes for input current:
+
+**curr1_input**
+
+**curr1_label**
+
+The driver provides the following attributes for output current:
+
+**curr2_input**
+
+**curr2_label**
+
+**curr2_crit**
+
+**curr2_crit_alarm**
+
+**curr2_max**
+
+**curr2_max_alarm**
+
+**curr3_input**
+
+**curr3_label**
+
+**curr3_crit**
+
+**curr3_crit_alarm**
+
+**curr3_max**
+
+**curr3_max_alarm**
+
+The driver provides the following attributes for input power:
+
+**power1_input**
+
+**power1_label**
+
+**power2_input**
+
+**power2_label**
+
+The driver provides the following attributes for output power:
+
+**power3_input**
+
+**power3_label**
+
+**power4_input**
+
+**power4_label**
+
+The driver provides the following attributes for temperature:
+
+**temp1_input**
+
+**temp1_crit**
+
+**temp1_crit_alarm**
+
+**temp1_max**
+
+**temp1_max_alarm**
+
+**temp2_input**
+
+**temp2_crit**
+
+**temp2_crit_alarm**
+
+**temp2_max**
+
+**temp2_max_alarm**
diff --git a/Documentation/hwmon/nct7802.rst b/Documentation/hwmon/nct7802.rst
index 8b7365a7cb322..366050ea595cc 100644
--- a/Documentation/hwmon/nct7802.rst
+++ b/Documentation/hwmon/nct7802.rst
@@ -24,6 +24,22 @@ speed sensors.
 
 Smart Fan™ speed control is available via pwmX_auto_point attributes.
 
+Sysfs Attributes
+----------------
+
+Sysfs attributes unique to this chip are documented below. For common
+attributes, see Documentation/hwmon/sysfs-interface.rst.
+
+step_up_time
+			Time interval between successive duty cycle increases
+			when in Smart Fan mode. Specified in milliseconds and
+			rounded to intervals of 100 in the range 100-25500.
+
+step_down_time
+			Time interval between successive duty cycle decreases
+			when in Smart Fan mode. Specified in milliseconds and
+			rounded to intervals of 100 in the range 100-25500.
+
 Tested Boards and BIOS Versions
 -------------------------------
 
diff --git a/Documentation/hwmon/raspberrypi-hwmon.rst b/Documentation/hwmon/raspberrypi-hwmon.rst
index 8038ade36490a..db315184b8616 100644
--- a/Documentation/hwmon/raspberrypi-hwmon.rst
+++ b/Documentation/hwmon/raspberrypi-hwmon.rst
@@ -20,6 +20,17 @@ undervoltage conditions.
 Sysfs entries
 -------------
 
-======================= ==================
+======================= ======================================================
+in0_input		Core voltage in millivolts
+in1_input		SDRAM controller voltage in millivolts
+in2_input		SDRAM I/O voltage in millivolts
+in3_input		SDRAM PHY voltage in millivolts
+in0_label		"core"
+in1_label		"sdram_c"
+in2_label		"sdram_i"
+in3_label		"sdram_p"
 in0_lcrit_alarm		Undervoltage alarm
-======================= ==================
+======================= ======================================================
+
+The voltage inputs and labels are only exposed if the firmware reports support
+for the corresponding voltage ID.
diff --git a/Documentation/iio/ad4691.rst b/Documentation/iio/ad4691.rst
new file mode 100644
index 0000000000000..5df786260190e
--- /dev/null
+++ b/Documentation/iio/ad4691.rst
@@ -0,0 +1,204 @@
+.. SPDX-License-Identifier: GPL-2.0-only
+
+=============
+AD4691 driver
+=============
+
+ADC driver for Analog Devices Inc. AD4691 family of multichannel SAR ADCs.
+The module name is ``ad4691``.
+
+
+Supported devices
+=================
+
+The following chips are supported by this driver:
+
+* `AD4691 <https://www.analog.com/en/products/ad4691.html>`_ — 16-channel, 500 kSPS
+* `AD4692 <https://www.analog.com/en/products/ad4692.html>`_ — 16-channel, 1 MSPS
+* `AD4693 <https://www.analog.com/en/products/ad4693.html>`_ — 8-channel, 500 kSPS
+* `AD4694 <https://www.analog.com/en/products/ad4694.html>`_ — 8-channel, 1 MSPS
+
+
+IIO channels
+============
+
+Each physical ADC input maps to one IIO voltage channel. The AD4691 and AD4692
+expose 16 channels (``voltage0`` through ``voltage15``); the AD4693 and AD4694
+expose 8 channels (``voltage0`` through ``voltage7``).
+
+All channels share a common scale (``in_voltage_scale``), derived from the
+reference voltage. Each channel independently exposes:
+
+* ``in_voltageN_raw`` — single-shot ADC result
+* ``in_voltageN_sampling_frequency`` — per-channel effective output rate,
+  defined as the internal oscillator frequency divided by the channel's
+  oversampling ratio. Writing this attribute selects the nearest achievable
+  rate for the current OSR; the value read back reflects the actual rate after
+  snapping to the closest valid oscillator entry.
+* ``in_voltageN_sampling_frequency_available`` — list of achievable effective
+  rates for the channel's current oversampling ratio. The list updates
+  dynamically when the oversampling ratio changes.
+
+The following attributes are only available in CNV Burst Mode:
+
+* ``in_voltageN_oversampling_ratio`` — per-channel hardware oversampling depth;
+  see `Oversampling`_ below.
+* ``in_voltageN_oversampling_ratio_available`` — valid ratios: 1, 2, 4, 8, 16,
+  32.
+
+
+Operating modes
+===============
+
+The driver supports two operating modes, selected automatically from the
+device tree at probe time.
+
+Manual Mode
+-----------
+
+Selected when no ``pwms`` property is present in the device tree. The CNV pin
+is tied to the SPI chip-select: every CS assertion triggers a conversion and
+returns the previous result. A user-defined IIO trigger (e.g. hrtimer trigger)
+drives the buffer.
+
+Oversampling is not supported in Manual Mode.
+
+CNV Burst Mode
+--------------
+
+Selected when a ``pwms`` property is present in the device tree. A PWM drives
+the CNV pin at the configured conversion rate. A GP pin wired to the SoC and
+declared in the device tree signals DATA_READY at the end of each burst,
+triggering a readout of all active channel results into the IIO buffer.
+
+The buffer output rate is controlled by the ``sampling_frequency`` attribute
+on the IIO buffer. In practice the PWM rate should be set low enough to allow
+the SPI readout to complete before the next conversion burst begins.
+
+Autonomous Mode (idle / single-shot)
+-------------------------------------
+
+When the IIO buffer is disabled, ``in_voltageN_raw`` reads perform a single
+conversion on the requested channel using the internal oscillator. The
+oscillator is started and stopped around each read to save power.
+
+
+Oversampling
+============
+
+In CNV Burst Mode each channel has an independent hardware accumulator that
+averages a configurable number of successive conversions. The result is always
+returned as a 16-bit mean, so ``realbits`` and ``storagebits`` are unaffected
+by the oversampling ratio. Valid ratios are 1, 2, 4, 8, 16 and 32; the default
+is 1 (no averaging). Oversampling is not supported in Manual Mode.
+
+.. code-block:: bash
+
+    # Set oversampling ratio to 16 on channel 0
+    echo 16 > /sys/bus/iio/devices/iio:device0/in_voltage0_oversampling_ratio
+
+    # Read the resulting effective sampling frequency
+    cat /sys/bus/iio/devices/iio:device0/in_voltage0_sampling_frequency
+
+Writing ``oversampling_ratio`` stores the new depth for that channel;
+the internal oscillator is unaffected. The effective rate read back via
+``in_voltageN_sampling_frequency`` becomes ``osc_freq / new_osr``
+automatically. ``oversampling_ratio`` and ``sampling_frequency`` are
+orthogonal: one controls averaging depth, the other controls the oscillator.
+
+All channels share one internal oscillator. Writing ``sampling_frequency`` for
+any channel updates the oscillator and therefore affects the effective rate
+read back from all other channels.
+
+
+Reference voltage
+=================
+
+The driver supports two reference configurations, mutually exclusive:
+
+* **External reference** (``ref-supply``): a voltage between 2.4 V and 5.25 V
+  supplied externally.
+* **Buffered internal reference** (``refin-supply``): an internal reference
+  buffer is enabled by the driver.
+
+Exactly one of ``ref-supply`` or ``refin-supply`` must be present in the
+device tree. The reference voltage determines the full-scale range reported
+via ``in_voltage_scale``.
+
+
+LDO supply
+==========
+
+The chip contains an internal LDO that powers part of the analog front-end.
+The supply configuration is mutually exclusive:
+
+* **External VDD** (``vdd-supply``): an external 1.8 V supply is used directly;
+  the internal LDO is disabled.
+* **Internal LDO** (``ldo-in-supply``): the internal LDO is enabled and fed
+  from the ``ldo-in`` regulator. Use this when no external 1.8 V VDD is present.
+
+Exactly one of ``vdd-supply`` or ``ldo-in-supply`` must be provided.
+
+
+Reset
+=====
+
+The driver supports two reset mechanisms:
+
+* **Hardware reset** (``reset-gpios`` in device tree): asserted at probe by
+  the reset controller framework.
+* **Software reset** (fallback when ``reset-gpios`` is absent): written
+  automatically at probe.
+
+
+GP pins and interrupts
+======================
+
+The chip exposes up to four general-purpose (GP) pins. In CNV Burst Mode
+(non-offload), one GP pin must be wired to an interrupt-capable SoC input and
+declared in the device tree using the ``interrupts`` and ``interrupt-names``
+properties. The ``interrupt-names`` value identifies which GP pin is used
+(``"gp0"`` through ``"gp3"``).
+
+Example device tree fragment::
+
+    adc@0 {
+        compatible = "adi,ad4692";
+        ...
+        interrupts = <17 IRQ_TYPE_LEVEL_HIGH>;
+        interrupt-parent = <&gpio0>;
+        interrupt-names = "gp0";
+    };
+
+
+SPI offload support
+===================
+
+When a SPI offload engine (e.g. the AXI SPI Engine) is present, the driver
+uses DMA-backed transfers for CPU-independent, high-throughput data capture.
+SPI offload is detected automatically at probe; if no offload hardware is
+available the driver falls back to the software triggered-buffer path.
+
+Two SPI offload sub-modes exist:
+
+CNV Burst offload
+-----------------
+
+Used when a ``pwms`` property is present and SPI offload is available. The PWM
+drives CNV at the configured rate; on DATA_READY the offload engine reads all
+active channel results and streams them directly to the IIO DMA buffer with no
+CPU involvement. The GP pin used as DATA_READY trigger is supplied by the
+trigger-source consumer at buffer enable time; no ``interrupt-names`` entry is
+required.
+
+Manual offload
+--------------
+
+Used when no ``pwms`` property is present and SPI offload is available. A
+periodic SPI offload trigger controls the conversion rate and the offload engine
+streams results directly to the IIO DMA buffer.
+
+The ``sampling_frequency`` attribute on the IIO buffer controls the trigger
+rate (in Hz). The initial rate is 100 kHz.
+
+Oversampling is not supported in Manual Mode.
diff --git a/Documentation/iio/adxl313.rst b/Documentation/iio/adxl313.rst
index 966e72c0109aa..3641193bf6477 100644
--- a/Documentation/iio/adxl313.rst
+++ b/Documentation/iio/adxl313.rst
@@ -11,7 +11,7 @@ This driver supports Analog Device's ADXL313 on SPI/I2C bus.
 
 * `ADXL313 <https://www.analog.com/ADXL313>`_
 
-The ADXL313is a low noise density, low power, 3-axis accelerometer with
+The ADXL313 is a low noise density, low power, 3-axis accelerometer with
 selectable measurement ranges. The ADXL313 supports the ±0.5 g, ±1 g, ±2 g and
 ±4 g ranges.
 
@@ -38,7 +38,7 @@ specific device folder path ``/sys/bus/iio/devices/iio:deviceX``.
 +---------------------------------------------------+----------------------------------------------------------+
 | in_accel_x_raw                                    | Raw X-axis accelerometer channel value.                  |
 +---------------------------------------------------+----------------------------------------------------------+
-| in_accel_y_calibbias                              | y-axis acceleration offset correction                    |
+| in_accel_y_calibbias                              | Calibration offset for the Y-axis accelerometer channel. |
 +---------------------------------------------------+----------------------------------------------------------+
 | in_accel_y_raw                                    | Raw Y-axis accelerometer channel value.                  |
 +---------------------------------------------------+----------------------------------------------------------+
@@ -112,9 +112,9 @@ apply the following formula:
 Where _offset and _scale are device attributes. If no _offset attribute is
 present, simply assume its value is 0.
 
-The ADXL313 driver offers data for a single types of channels, the table below
-shows the measurement units for the processed value, which are defined by the
-IIO framework:
+The ADXL313 driver offers data for multiple channels of a single type.
+The table below shows the measurement units for the processed value,
+which are defined by the IIO framework:
 
 +-------------------------------------+---------------------------+
 | Channel type                        | Measurement unit          |
diff --git a/Documentation/iio/adxl345.rst b/Documentation/iio/adxl345.rst
index 978f746a81985..0aa33a8524128 100644
--- a/Documentation/iio/adxl345.rst
+++ b/Documentation/iio/adxl345.rst
@@ -47,7 +47,7 @@ specific device folder path ``/sys/bus/iio/devices/iio:deviceX``.
 +-------------------------------------------+----------------------------------------------------------+
 | in_accel_x_raw                            | Raw X-axis accelerometer channel value.                  |
 +-------------------------------------------+----------------------------------------------------------+
-| in_accel_y_calibbias                      | Y-axis acceleration offset correction                    |
+| in_accel_y_calibbias                      | Calibration offset for the Y-axis accelerometer channel. |
 +-------------------------------------------+----------------------------------------------------------+
 | in_accel_y_raw                            | Raw Y-axis accelerometer channel value.                  |
 +-------------------------------------------+----------------------------------------------------------+
diff --git a/Documentation/iio/adxl380.rst b/Documentation/iio/adxl380.rst
index 61cafa2f98bf3..654d4c0e84a19 100644
--- a/Documentation/iio/adxl380.rst
+++ b/Documentation/iio/adxl380.rst
@@ -51,7 +51,7 @@ specific device folder path ``/sys/bus/iio/devices/iio:deviceX``.
 +---------------------------------------------------+----------------------------------------------------------+
 | in_accel_x_raw                                    | Raw X-axis accelerometer channel value.                  |
 +---------------------------------------------------+----------------------------------------------------------+
-| in_accel_y_calibbias                              | y-axis acceleration offset correction                    |
+| in_accel_y_calibbias                              | Calibration offset for the Y-axis accelerometer channel. |
 +---------------------------------------------------+----------------------------------------------------------+
 | in_accel_y_raw                                    | Raw Y-axis accelerometer channel value.                  |
 +---------------------------------------------------+----------------------------------------------------------+
diff --git a/Documentation/iio/iio_devbuf.rst b/Documentation/iio/iio_devbuf.rst
index dca1f0200b0dd..e91730fa3cea4 100644
--- a/Documentation/iio/iio_devbuf.rst
+++ b/Documentation/iio/iio_devbuf.rst
@@ -83,9 +83,10 @@ and the relevant _type attributes to establish the data storage format.
 
 Read-only attribute containing the description of the scan element data storage
 within the buffer and hence the form in which it is read from userspace. Format
-is [be|le]:[s|u]bits/storagebits[Xrepeat][>>shift], where:
+is [be|le]:[f|s|u]bits/storagebits[Xrepeat][>>shift], where:
 
 - **be** or **le** specifies big or little-endian.
+- **f** specifies if floating-point.
 - **s** or **u** specifies if signed (2's complement) or unsigned.
 - **bits** is the number of valid data bits.
 - **storagebits** is the number of bits (after padding) that it occupies in the
diff --git a/Documentation/iio/index.rst b/Documentation/iio/index.rst
index ba3e609c6a13c..007e0a1fcc5a2 100644
--- a/Documentation/iio/index.rst
+++ b/Documentation/iio/index.rst
@@ -23,6 +23,7 @@ Industrial I/O Kernel Drivers
    ad4000
    ad4030
    ad4062
+   ad4691
    ad4695
    ad7191
    ad7380
diff --git a/Documentation/kbuild/makefiles.rst b/Documentation/kbuild/makefiles.rst
index 24a4708d26e8e..7521cae7d56f1 100644
--- a/Documentation/kbuild/makefiles.rst
+++ b/Documentation/kbuild/makefiles.rst
@@ -1285,8 +1285,39 @@ Example::
 In this example, the file target maketools will be processed
 before descending down in the subdirectories.
 
-See also chapter XXX-TODO that describes how kbuild supports
-generating offset header files.
+Generating offset header files
+------------------------------
+
+The ``include/generated/asm-offsets.h`` header exposes C structure
+member offsets and other compile-time constants to assembly code. It
+is generated from ``arch/$(SRCARCH)/kernel/asm-offsets.c``.
+
+The source file uses ``DEFINE()``, ``OFFSET()``, ``BLANK()`` and
+``COMMENT()`` from ``<linux/kbuild.h>``. These emit marker strings
+through inline asm that Kbuild extracts from the compiled assembly
+output.
+
+Example::
+
+  #include <linux/kbuild.h>
+  #include <linux/sched.h>
+
+  int main(void)
+  {
+          OFFSET(TSK_ACTIVE_MM, task_struct, active_mm);
+          DEFINE(THREAD_SIZE, THREAD_SIZE);
+          BLANK();
+          return 0;
+  }
+
+The rules are defined in the top-level ``Kbuild`` and
+``scripts/Makefile.lib``. The header is built during Kbuild's
+``prepare`` phase, after ``archprepare`` and before descending into
+subdirectories.
+
+The same mechanism generates ``include/generated/bounds.h`` from
+``kernel/bounds.c`` and ``include/generated/rq-offsets.h`` from
+``kernel/sched/rq-offsets.c``.
 
 List directories to visit when descending
 -----------------------------------------
@@ -1690,9 +1721,3 @@ Credits
 - Updates by Kai Germaschewski <kai@tp1.ruhr-uni-bochum.de>
 - Updates by Sam Ravnborg <sam@ravnborg.org>
 - Language QA by Jan Engelhardt <jengelh@gmx.de>
-
-TODO
-====
-
-- Generating offset header files.
-- Add more variables to chapters 7 or 9?
diff --git a/Documentation/kernel-hacking/locking.rst b/Documentation/kernel-hacking/locking.rst
index dff0646a717bf..c969c76ef7cb4 100644
--- a/Documentation/kernel-hacking/locking.rst
+++ b/Documentation/kernel-hacking/locking.rst
@@ -442,7 +442,7 @@ to protect the cache and all the objects within it. Here's the code::
     {
             struct object *obj;
 
-            if ((obj = kmalloc(sizeof(*obj), GFP_KERNEL)) == NULL)
+            if ((obj = kmalloc_obj(*obj)) == NULL)
                     return -ENOMEM;
 
             strscpy(obj->name, name, sizeof(obj->name));
@@ -517,7 +517,7 @@ which are taken away, and the ``+`` are lines which are added.
              struct object *obj;
     +        unsigned long flags;
 
-             if ((obj = kmalloc(sizeof(*obj), GFP_KERNEL)) == NULL)
+             if ((obj = kmalloc_obj(*obj)) == NULL)
                      return -ENOMEM;
     @@ -63,30 +64,33 @@
              obj->id = id;
diff --git a/Documentation/leds/leds-class-multicolor.rst b/Documentation/leds/leds-class-multicolor.rst
index c6b47b4093c47..68340644f80b4 100644
--- a/Documentation/leds/leds-class-multicolor.rst
+++ b/Documentation/leds/leds-class-multicolor.rst
@@ -25,10 +25,14 @@ color name to indexed value.
 The ``multi_index`` file is an array that contains the string list of the colors as
 they are defined in each ``multi_*`` array file.
 
-The ``multi_intensity`` is an array that can be read or written to for the
+The ``multi_intensity`` file is an array that can be read or written to for the
 individual color intensities.  All elements within this array must be written in
 order for the color LED intensities to be updated.
 
+The ``multi_max_intensity`` file is an array that contains the maximum intensity
+value supported by each color intensity. Intensity values above this will be
+automatically clamped into the supported range.
+
 Directory Layout Example
 ========================
 .. code-block:: console
@@ -38,6 +42,7 @@ Directory Layout Example
     -r--r--r--    1 root     root          4096 Oct 19 16:16 max_brightness
     -r--r--r--    1 root     root          4096 Oct 19 16:16 multi_index
     -rw-r--r--    1 root     root          4096 Oct 19 16:16 multi_intensity
+    -r--r--r--    1 root     root          4096 Oct 19 16:16 multi_max_intensity
 
 ..
 
@@ -104,3 +109,17 @@ the color LED group.
     128
 
 ..
+
+Writing intensity values larger than the maximum specified in ``multi_max_intensity``
+will result in those values being clamped into the supported range.
+
+.. code-block:: console
+
+   # cat /sys/class/leds/multicolor:status/multi_max_intensity
+   255 255 255
+
+   # echo 512 512 512 > /sys/class/leds/multicolor:status/multi_intensity
+   # cat /sys/class/leds/multicolor:status/multi_intensity
+   255 255 255
+
+..
diff --git a/Documentation/leds/leds-class.rst b/Documentation/leds/leds-class.rst
index 5db620ed27aa2..3913966cfdac0 100644
--- a/Documentation/leds/leds-class.rst
+++ b/Documentation/leds/leds-class.rst
@@ -116,6 +116,69 @@ above leaves scope for further attributes should they be needed. If sections
 of the name don't apply, just leave that section blank.
 
 
+Keyboard backlight control LED Device Naming
+============================================
+
+For backlit keyboards with a single brightness / color settings a single
+(multicolor) LED class device should be used to allow userspace to change
+the backlight brightness (and if possible the color). This LED class device
+must use "kbd_backlight" for the function part of the LED class device name.
+IOW the name must end with ":kbd_backlight".
+
+For backlit keyboards with multiple control zones, one (multicolor) LED class
+device should be used per zone. These LED class devices' name must follow:
+
+	"<devicename>:<color>:kbd_zoned_backlight-<zone_name>"
+
+and <devicename> must be the same for all zones of the same keyboard.
+
+<zone_name> should be descriptive of which part of the keyboard backlight
+the zone covers and should be suitable for userspace to show to an end user
+in an UI for controlling the zones.
+
+Where possible <zone_name> should be a value already used by other
+zoned keyboards with a similar or identical zone layout, e.g.:
+
+<devicename>:<color>:kbd_zoned_backlight-right
+<devicename>:<color>:kbd_zoned_backlight-middle
+<devicename>:<color>:kbd_zoned_backlight-left
+<devicename>:<color>:kbd_zoned_backlight-corners
+<devicename>:<color>:kbd_zoned_backlight-wasd
+
+or:
+
+<devicename>:<color>:kbd_zoned_backlight-main
+<devicename>:<color>:kbd_zoned_backlight-cursor
+<devicename>:<color>:kbd_zoned_backlight-numpad
+<devicename>:<color>:kbd_zoned_backlight-corners
+<devicename>:<color>:kbd_zoned_backlight-wasd
+
+Note that this is intended for keyboards with a limited number of zones,
+keyboards with per key addressable backlighting must not use LED class devices
+since the sysfs API is not suitable for rapidly change multiple LEDs in one
+"commit" as is necessary to do animations / special effects on such keyboards.
+
+An exception to the rule that all zones must follow:
+
+	"<devicename>:<color>:kbd_zoned_backlight-<zone_name>"
+
+is made for the special case where there is a single big zone which controls
+the backlighting of almost all of the keyboard and there are some small areas
+with separate control, like just the 4 cursor keys, or the WASD keys. In this
+case the main zone should use 'kbd_backlight' for the function part of the name
+for compatibility with (older) userspace code which is not aware of
+the "kbd_zoned_backlight-<zone_name>" function naming scheme.
+
+While the smaller zones should use the new zoned naming scheme. Such a setup
+would result in e.g.:
+
+<devicename>:<color>:kbd_backlight
+<devicename>:<color>:kbd_zoned_backlight-wasd
+
+"kbd_zoned_backlight-<zone_name>" aware userspace should be aware of this
+exception and check for a main zone with a "kbd_backlight" function-name.
+
+
 Brightness setting API
 ======================
 
diff --git a/Documentation/leds/leds-lp5812.rst b/Documentation/leds/leds-lp5812.rst
index c2a6368d51493..12e757d45c3a2 100644
--- a/Documentation/leds/leds-lp5812.rst
+++ b/Documentation/leds/leds-lp5812.rst
@@ -20,7 +20,7 @@ Sysfs Interface
 ===============
 
 This driver uses the standard multicolor LED class interfaces defined
-in Documentation/ABI/testing/sysfs-class-led-multicolor.rst.
+in Documentation/ABI/testing/sysfs-class-led-multicolor.
 
 Each LP5812 LED output appears under ``/sys/class/leds/`` with its
 assigned label (for example ``LED_A``).
diff --git a/Documentation/locking/locktypes.rst b/Documentation/locking/locktypes.rst
index 37b6a5670c2fa..ac1ad722a9e7e 100644
--- a/Documentation/locking/locktypes.rst
+++ b/Documentation/locking/locktypes.rst
@@ -498,7 +498,7 @@ allocating memory.  Thus, on a non-PREEMPT_RT kernel the following code
 works perfectly::
 
   raw_spin_lock(&lock);
-  p = kmalloc(sizeof(*p), GFP_ATOMIC);
+  p = kmalloc_obj(*p, GFP_ATOMIC);
 
 But this code fails on PREEMPT_RT kernels because the memory allocator is
 fully preemptible and therefore cannot be invoked from truly atomic
@@ -507,7 +507,7 @@ while holding normal non-raw spinlocks because they do not disable
 preemption on PREEMPT_RT kernels::
 
   spin_lock(&lock);
-  p = kmalloc(sizeof(*p), GFP_ATOMIC);
+  p = kmalloc_obj(*p, GFP_ATOMIC);
 
 
 bit spinlocks
diff --git a/Documentation/maintainer/maintainer-entry-profile.rst b/Documentation/maintainer/maintainer-entry-profile.rst
index 6020d188e13de..58e2af333692d 100644
--- a/Documentation/maintainer/maintainer-entry-profile.rst
+++ b/Documentation/maintainer/maintainer-entry-profile.rst
@@ -92,24 +92,8 @@ full series, or privately send a reminder email. This section might also
 list how review works for this code area and methods to get feedback
 that are not directly from the maintainer.
 
-Existing profiles
------------------
-
-For now, existing maintainer profiles are listed here; we will likely want
-to do something different in the near future.
-
-.. toctree::
-   :maxdepth: 1
-
-   ../doc-guide/maintainer-profile
-   ../nvdimm/maintainer-entry-profile
-   ../arch/riscv/patch-acceptance
-   ../process/maintainer-soc
-   ../process/maintainer-soc-clean-dts
-   ../driver-api/media/maintainer-entry-profile
-   ../process/maintainer-netdev
-   ../driver-api/vfio-pci-device-specific-driver-acceptance
-   ../nvme/feature-and-quirk-policy
-   ../filesystems/nfs/nfsd-maintainer-entry-profile
-   ../filesystems/xfs/xfs-maintainer-entry-profile
-   ../mm/damon/maintainer-profile
+Maintainer Handbooks
+--------------------
+
+For examples of other subsystem handbooks see
+Documentation/process/maintainer-handbooks.rst.
diff --git a/Documentation/misc-devices/apds990x.rst b/Documentation/misc-devices/apds990x.rst
deleted file mode 100644
index e2f75577f731b..0000000000000
--- a/Documentation/misc-devices/apds990x.rst
+++ /dev/null
@@ -1,128 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-======================
-Kernel driver apds990x
-======================
-
-Supported chips:
-Avago APDS990X
-
-Data sheet:
-Not freely available
-
-Author:
-Samu Onkalo <samu.p.onkalo@nokia.com>
-
-Description
------------
-
-APDS990x is a combined ambient light and proximity sensor. ALS and proximity
-functionality are highly connected. ALS measurement path must be running
-while the proximity functionality is enabled.
-
-ALS produces raw measurement values for two channels: Clear channel
-(infrared + visible light) and IR only. However, threshold comparisons happen
-using clear channel only. Lux value and the threshold level on the HW
-might vary quite much depending the spectrum of the light source.
-
-Driver makes necessary conversions to both directions so that user handles
-only lux values. Lux value is calculated using information from the both
-channels. HW threshold level is calculated from the given lux value to match
-with current type of the lightning. Sometimes inaccuracy of the estimations
-lead to false interrupt, but that doesn't harm.
-
-ALS contains 4 different gain steps. Driver automatically
-selects suitable gain step. After each measurement, reliability of the results
-is estimated and new measurement is triggered if necessary.
-
-Platform data can provide tuned values to the conversion formulas if
-values are known. Otherwise plain sensor default values are used.
-
-Proximity side is little bit simpler. There is no need for complex conversions.
-It produces directly usable values.
-
-Driver controls chip operational state using pm_runtime framework.
-Voltage regulators are controlled based on chip operational state.
-
-SYSFS
------
-
-
-chip_id
-	RO - shows detected chip type and version
-
-power_state
-	RW - enable / disable chip. Uses counting logic
-
-	     1 enables the chip
-	     0 disables the chip
-lux0_input
-	RO - measured lux value
-
-	     sysfs_notify called when threshold interrupt occurs
-
-lux0_sensor_range
-	RO - lux0_input max value.
-
-	     Actually never reaches since sensor tends
-	     to saturate much before that. Real max value varies depending
-	     on the light spectrum etc.
-
-lux0_rate
-	RW - measurement rate in Hz
-
-lux0_rate_avail
-	RO - supported measurement rates
-
-lux0_calibscale
-	RW - calibration value.
-
-	     Set to neutral value by default.
-	     Output results are multiplied with calibscale / calibscale_default
-	     value.
-
-lux0_calibscale_default
-	RO - neutral calibration value
-
-lux0_thresh_above_value
-	RW - HI level threshold value.
-
-	     All results above the value
-	     trigs an interrupt. 65535 (i.e. sensor_range) disables the above
-	     interrupt.
-
-lux0_thresh_below_value
-	RW - LO level threshold value.
-
-	     All results below the value
-	     trigs an interrupt. 0 disables the below interrupt.
-
-prox0_raw
-	RO - measured proximity value
-
-	     sysfs_notify called when threshold interrupt occurs
-
-prox0_sensor_range
-	RO - prox0_raw max value (1023)
-
-prox0_raw_en
-	RW - enable / disable proximity - uses counting logic
-
-	     - 1 enables the proximity
-	     - 0 disables the proximity
-
-prox0_reporting_mode
-	RW - trigger / periodic.
-
-	     In "trigger" mode the driver tells two possible
-	     values: 0 or prox0_sensor_range value. 0 means no proximity,
-	     1023 means proximity. This causes minimal number of interrupts.
-	     In "periodic" mode the driver reports all values above
-	     prox0_thresh_above. This causes more interrupts, but it can give
-	     _rough_ estimate about the distance.
-
-prox0_reporting_mode_avail
-	RO - accepted values to prox0_reporting_mode (trigger, periodic)
-
-prox0_thresh_above_value
-	RW - threshold level which trigs proximity events.
diff --git a/Documentation/misc-devices/index.rst b/Documentation/misc-devices/index.rst
index 081e79415e386..f911edaecbfad 100644
--- a/Documentation/misc-devices/index.rst
+++ b/Documentation/misc-devices/index.rst
@@ -13,7 +13,6 @@ fit into other categories.
 
    ad525x_dpot
    amd-sbi
-   apds990x
    bh1770glc
    c2port
    dw-xdata-pcie
diff --git a/Documentation/mm/damon/design.rst b/Documentation/mm/damon/design.rst
index afc7d52bda2f7..fa7392b5a331d 100644
--- a/Documentation/mm/damon/design.rst
+++ b/Documentation/mm/damon/design.rst
@@ -19,6 +19,13 @@ types of monitoring.
 To know how user-space can do the configurations and start/stop DAMON, refer to
 :ref:`DAMON sysfs interface <sysfs_interface>` documentation.
 
+Users can also request each context execution to be paused and resumed.  When
+it is paused, the kdamond does nothing other than applying online parameter
+update.
+
+To know how user-space can pause/resume each context, refer to :ref:`DAMON
+sysfs context <sysfs_context>` usage documentation.
+
 
 Overall Architecture
 ====================
@@ -474,6 +481,10 @@ that supports each action are as below.
    Supported by ``vaddr`` and ``fvaddr`` operations set. When
    TRANSPARENT_HUGEPAGE is disabled, the application of the action will just
    fail.
+ - ``collapse``: Call ``madvise()`` for the region with ``MADV_COLLAPSE``.
+   Supported by ``vaddr`` and ``fvaddr`` operations set. When
+   TRANSPARENT_HUGEPAGE is disabled, the application of the action will just
+   fail.
  - ``lru_prio``: Prioritize the region on its LRU lists.
    Supported by ``paddr`` operations set.
  - ``lru_deprio``: Deprioritize the region on its LRU lists.
@@ -565,6 +576,28 @@ interface <sysfs_interface>`, refer to :ref:`weights <sysfs_quotas>` part of
 the documentation.
 
 
+.. _damon_design_damos_quotas_failed_memory_charging_ratio:
+
+Action-failed Memory Charging Ratio
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+DAMOS action to a given region can fail for some subsets of the memory of the
+region.  For example, if the action is ``pageout`` and the region has some
+unreclaimable pages, applying the action to the pages will fail.  The amount of
+system resource that is taken for such failed action applications is usually
+different from that for successful action applications.  For such cases, users
+can set different charging ratio for such failed memory.  The ratio can be
+specified using ``fail_charge_num`` and ``fail_charge_denom`` parameters.  The
+two parameters represent the numerator and denominator of the ratio.  The
+feature is enabled only if ``fail_charge_denom`` is not zero.
+
+For example, let's suppose a DAMOS action is applied to a region of 1,000 MiB
+size.  The action is successfully applied to only 700 MiB of the region.
+``fail_charge_num`` and ``fail_charge_denom`` are set to ``1`` and ``1024``,
+respectively.  Then only 700 MiB and 300 KiB of size (``700 MiB + 300 MiB * 1 /
+1024``) will be charged.
+
+
 .. _damon_design_damos_quotas_auto_tuning:
 
 Aim-oriented Feedback-driven Auto-tuning
diff --git a/Documentation/mm/damon/maintainer-profile.rst b/Documentation/mm/damon/maintainer-profile.rst
index bcb9798a27a86..fb2fa00cc9aa1 100644
--- a/Documentation/mm/damon/maintainer-profile.rst
+++ b/Documentation/mm/damon/maintainer-profile.rst
@@ -100,3 +100,24 @@ There is also a public Google `calendar
 <https://calendar.google.com/calendar/u/0?cid=ZDIwOTA4YTMxNjc2MDQ3NTIyMmUzYTM5ZmQyM2U4NDA0ZGIwZjBiYmJlZGQxNDM0MmY4ZTRjOTE0NjdhZDRiY0Bncm91cC5jYWxlbmRhci5nb29nbGUuY29t>`_
 that has the events.  Anyone can subscribe to it.  DAMON maintainer will also
 provide periodic reminders to the mailing list (damon@lists.linux.dev).
+
+AI Review
+---------
+
+For patches that are publicly posted to DAMON mailing list
+(damon@lists.linux.dev), AI reviews of the patches will be available at
+sashiko.dev.  The reviews could also be sent as mails to the author of the
+patch.
+
+Patch authors are encouraged to check the AI reviews and share their opinions.
+The sharing could be done as a reply to the mail thread.  Consider reducing the
+recipients list for such sharing, since some people are not really interested
+in AI reviews.  As a rule of thumb, drop stable@vger.kernel.org and individuals
+except DAMON maintainer.
+
+`hkml` also provides a `feature
+<https://github.com/sjp38/hackermail/blob/master/USAGE.md#forwarding-sashikodev-statuscomments-to-mailing-list>`_
+for such sharing.  Please feel free to use the feature.
+
+It is only an optional recommendation.  DAMON maintainer could also ask any
+question about the AI reviews, though.
diff --git a/Documentation/netlink/specs/dpll.yaml b/Documentation/netlink/specs/dpll.yaml
index 40465a3d7fc20..91a172617b3a9 100644
--- a/Documentation/netlink/specs/dpll.yaml
+++ b/Documentation/netlink/specs/dpll.yaml
@@ -212,6 +212,27 @@ definitions:
         name: selectable
         doc: pin enabled for automatic input selection
     render-max: true
+  -
+    type: enum
+    name: pin-operstate
+    doc: |
+      defines possible operational states of a pin with respect to its
+      parent DPLL device, valid values for DPLL_A_PIN_OPERSTATE attribute
+    entries:
+      -
+        name: active
+        doc: pin is qualified and actively used by the DPLL
+        value: 1
+      -
+        name: standby
+        doc: pin is qualified but not actively used by the DPLL
+      -
+        name: no-signal
+        doc: pin does not have a valid signal
+      -
+        name: qual-failed
+        doc: pin signal failed qualification (e.g. frequency or phase monitor)
+    render-max: true
   -
     type: flags
     name: pin-capabilities
@@ -427,12 +448,14 @@ attribute-sets:
         name: fractional-frequency-offset
         type: sint
         doc: |
-          The FFO (Fractional Frequency Offset) between the RX and TX
-          symbol rate on the media associated with the pin:
-          (rx_frequency-tx_frequency)/rx_frequency
+          The FFO (Fractional Frequency Offset) of the pin.
+          At top level this represents the RX vs TX symbol rate
+          offset on the media associated with the pin. Inside
+          the pin-parent-device nest it represents the frequency
+          offset between the pin and its parent DPLL device.
           Value is in PPM (parts per million).
-          This may be implemented for example for pin of type
-          PIN_TYPE_SYNCE_ETH_PORT.
+          This is a lower-precision version of
+          fractional-frequency-offset-ppt.
       -
         name: esync-frequency
         type: u64
@@ -471,12 +494,14 @@ attribute-sets:
         name: fractional-frequency-offset-ppt
         type: sint
         doc: |
-          The FFO (Fractional Frequency Offset) of the pin with respect to
-          the nominal frequency.
-          Value = (frequency_measured - frequency_nominal) / frequency_nominal
+          The FFO (Fractional Frequency Offset) of the pin.
+          At top level this represents the RX vs TX symbol rate
+          offset on the media associated with the pin. Inside
+          the pin-parent-device nest it represents the frequency
+          offset between the pin and its parent DPLL device.
           Value is in PPT (parts per trillion, 10^-12).
-          Note: This attribute provides higher resolution than the standard
-          fractional-frequency-offset (which is in PPM).
+          This is a higher-precision version of
+          fractional-frequency-offset.
       -
         name: measured-frequency
         type: u64
@@ -488,6 +513,14 @@ attribute-sets:
           Value of (DPLL_A_PIN_MEASURED_FREQUENCY %
           DPLL_PIN_MEASURED_FREQUENCY_DIVIDER) is a fractional part
           of a measured frequency value.
+      -
+        name: operstate
+        type: u32
+        enum: pin-operstate
+        doc: |
+          Operational state of the pin with respect to its parent DPLL
+          device. Unlike state (which reflects the administrative intent),
+          operstate reflects the actual hardware status.
 
   -
     name: pin-parent-device
@@ -501,8 +534,14 @@ attribute-sets:
         name: prio
       -
         name: state
+      -
+        name: operstate
       -
         name: phase-offset
+      -
+        name: fractional-frequency-offset
+      -
+        name: fractional-frequency-offset-ppt
   -
     name: pin-parent-pin
     subset-of: pin
diff --git a/Documentation/netlink/specs/drm_ras.yaml b/Documentation/netlink/specs/drm_ras.yaml
index 79af25dac3c57..e113056f8c016 100644
--- a/Documentation/netlink/specs/drm_ras.yaml
+++ b/Documentation/netlink/specs/drm_ras.yaml
@@ -99,7 +99,7 @@ operations:
       flags: [admin-perm]
       do:
         request:
-          attributes:
+          attributes: &id-attrs
             - node-id
             - error-id
         reply:
@@ -113,3 +113,14 @@ operations:
             - node-id
         reply:
           attributes: *errorinfo
+    -
+      name: clear-error-counter
+      doc: >-
+           Clear error counter for a given node.
+           The request includes the error-id and node-id of the
+           counter to be cleared.
+      attribute-set: error-counter-attrs
+      flags: [admin-perm]
+      do:
+        request:
+          attributes: *id-attrs
diff --git a/Documentation/netlink/specs/netdev.yaml b/Documentation/netlink/specs/netdev.yaml
index b93beb247a112..a1f4c5a561e9d 100644
--- a/Documentation/netlink/specs/netdev.yaml
+++ b/Documentation/netlink/specs/netdev.yaml
@@ -649,6 +649,9 @@ operations:
             - dmabuf
             - io-uring
       dump:
+        request:
+          attributes:
+            - ifindex
         reply: *pp-reply
       config-cond: page-pool
     -
@@ -692,6 +695,9 @@ operations:
             - recycle-ring-full
             - recycle-released-refcnt
       dump:
+        request:
+          attributes:
+            - info
         reply: *pp-stats-reply
       config-cond: page-pool-stats
     -
diff --git a/Documentation/netlink/specs/nfsd.yaml b/Documentation/netlink/specs/nfsd.yaml
index 8ab43c8253b2e..25497b5331857 100644
--- a/Documentation/netlink/specs/nfsd.yaml
+++ b/Documentation/netlink/specs/nfsd.yaml
@@ -6,7 +6,51 @@ uapi-header: linux/nfsd_netlink.h
 
 doc: NFSD configuration over generic netlink.
 
+definitions:
+  -
+    type: flags
+    name: cache-type
+    entries: [svc_export, expkey]
+  -
+    type: flags
+    name: export-flags
+    doc: These flags are ordered to match the NFSEXP_* flags in include/linux/nfsd/export.h
+    entries:
+      - readonly
+      - insecure-port
+      - rootsquash
+      - allsquash
+      - async
+      - gathered-writes
+      - noreaddirplus
+      - security-label
+      - sign-fh
+      - nohide
+      - nosubtreecheck
+      - noauthnlm
+      - msnfs
+      - fsid
+      - crossmount
+      - noacl
+      - v4root
+      - pnfs
+  -
+    type: flags
+    name: xprtsec-mode
+    doc: These flags are ordered to match the NFSEXP_XPRTSEC_* flags in include/linux/nfsd/export.h
+    entries:
+      - none
+      - tls
+      - mtls
+
 attribute-sets:
+  -
+    name: cache-notify
+    attributes:
+      -
+        name: cache-type
+        type: u32
+        enum: cache-type
   -
     name: rpc-status
     attributes:
@@ -132,6 +176,131 @@ attribute-sets:
       -
         name: npools
         type: u32
+  -
+    name: fslocation
+    attributes:
+      -
+        name: host
+        type: string
+      -
+        name: path
+        type: string
+  -
+    name: fslocations
+    attributes:
+      -
+        name: location
+        type: nest
+        nested-attributes: fslocation
+        multi-attr: true
+  -
+    name: auth-flavor
+    attributes:
+      -
+        name: pseudoflavor
+        type: u32
+      -
+        name: flags
+        type: u32
+        enum: export-flags
+        enum-as-flags: true
+  -
+    name: svc-export
+    attributes:
+      -
+        name: seqno
+        type: u64
+      -
+        name: client
+        type: string
+      -
+        name: path
+        type: string
+      -
+        name: negative
+        type: flag
+      -
+        name: expiry
+        type: u64
+      -
+        name: anon-uid
+        type: u32
+      -
+        name: anon-gid
+        type: u32
+      -
+        name: fslocations
+        type: nest
+        nested-attributes: fslocations
+      -
+        name: uuid
+        type: binary
+      -
+        name: secinfo
+        type: nest
+        nested-attributes: auth-flavor
+        multi-attr: true
+      -
+        name: xprtsec
+        type: u32
+        enum: xprtsec-mode
+        multi-attr: true
+      -
+        name: flags
+        type: u32
+        enum: export-flags
+        enum-as-flags: true
+      -
+        name: fsid
+        type: s32
+  -
+    name: svc-export-reqs
+    attributes:
+      -
+        name: requests
+        type: nest
+        nested-attributes: svc-export
+        multi-attr: true
+  -
+    name: expkey
+    attributes:
+      -
+        name: seqno
+        type: u64
+      -
+        name: client
+        type: string
+      -
+        name: fsidtype
+        type: u8
+      -
+        name: fsid
+        type: binary
+      -
+        name: negative
+        type: flag
+      -
+        name: expiry
+        type: u64
+      -
+        name: path
+        type: string
+  -
+    name: expkey-reqs
+    attributes:
+      -
+        name: requests
+        type: nest
+        nested-attributes: expkey
+        multi-attr: true
+  -
+    name: cache-flush
+    attributes:
+      -
+        name: mask
+        type: u32
+        enum: cache-type
+        enum-as-flags: true
 
 operations:
   list:
@@ -233,3 +402,63 @@ operations:
           attributes:
             - mode
             - npools
+    -
+      name: cache-notify
+      doc: Notification that there are cache requests that need servicing
+      attribute-set: cache-notify
+      mcgrp: exportd
+      event:
+        attributes:
+          - cache-type
+    -
+      name: svc-export-get-reqs
+      doc: Dump all pending svc_export requests
+      attribute-set: svc-export-reqs
+      flags: [admin-perm]
+      dump:
+          reply:
+            attributes:
+              - requests
+    -
+      name: svc-export-set-reqs
+      doc: Respond to one or more svc_export requests
+      attribute-set: svc-export-reqs
+      flags: [admin-perm]
+      do:
+          request:
+            attributes:
+              - requests
+    -
+      name: expkey-get-reqs
+      doc: Dump all pending expkey requests
+      attribute-set: expkey-reqs
+      flags: [admin-perm]
+      dump:
+          reply:
+            attributes:
+              - requests
+    -
+      name: expkey-set-reqs
+      doc: Respond to one or more expkey requests
+      attribute-set: expkey-reqs
+      flags: [admin-perm]
+      do:
+          request:
+            attributes:
+              - requests
+    -
+      name: cache-flush
+      doc: Flush nfsd caches (svc_export and/or expkey)
+      attribute-set: cache-flush
+      flags: [admin-perm]
+      do:
+        request:
+          attributes:
+            - mask
+
+mcast-groups:
+  list:
+    -
+      name: none
+    -
+      name: exportd
diff --git a/Documentation/netlink/specs/rt-link.yaml b/Documentation/netlink/specs/rt-link.yaml
index f23aa5f229c50..79c89f2044152 100644
--- a/Documentation/netlink/specs/rt-link.yaml
+++ b/Documentation/netlink/specs/rt-link.yaml
@@ -1700,6 +1700,9 @@ attribute-sets:
       -
         name: backup-nhid
         type: u32
+      -
+        name: neigh-forward-grat
+        type: u8
   -
     name: linkinfo-gre-attrs
     name-prefix: ifla-gre-
diff --git a/Documentation/netlink/specs/sunrpc_cache.yaml b/Documentation/netlink/specs/sunrpc_cache.yaml
new file mode 100644
index 0000000000000..f22ff22b9418f
--- /dev/null
+++ b/Documentation/netlink/specs/sunrpc_cache.yaml
@@ -0,0 +1,149 @@
+# SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+---
+name: sunrpc
+protocol: genetlink
+uapi-header: linux/sunrpc_netlink.h
+
+doc: SUNRPC cache upcall support over generic netlink.
+
+definitions:
+  -
+    type: flags
+    name: cache-type
+    entries: [ip_map, unix_gid]
+
+attribute-sets:
+  -
+    name: cache-notify
+    attributes:
+      -
+        name: cache-type
+        type: u32
+        enum: cache-type
+  -
+    name: ip-map
+    attributes:
+      -
+        name: seqno
+        type: u64
+      -
+        name: class
+        type: string
+      -
+        name: addr
+        type: string
+      -
+        name: domain
+        type: string
+      -
+        name: negative
+        type: flag
+      -
+        name: expiry
+        type: u64
+  -
+    name: ip-map-reqs
+    attributes:
+      -
+        name: requests
+        type: nest
+        nested-attributes: ip-map
+        multi-attr: true
+  -
+    name: unix-gid
+    attributes:
+      -
+        name: seqno
+        type: u64
+      -
+        name: uid
+        type: u32
+      -
+        name: gids
+        type: u32
+        multi-attr: true
+      -
+        name: negative
+        type: flag
+      -
+        name: expiry
+        type: u64
+  -
+    name: unix-gid-reqs
+    attributes:
+      -
+        name: requests
+        type: nest
+        nested-attributes: unix-gid
+        multi-attr: true
+  -
+    name: cache-flush
+    attributes:
+      -
+        name: mask
+        type: u32
+        enum: cache-type
+        enum-as-flags: true
+
+operations:
+  list:
+    -
+      name: cache-notify
+      doc: Notification that there are cache requests that need servicing
+      attribute-set: cache-notify
+      mcgrp: exportd
+      event:
+        attributes:
+          - cache-type
+    -
+      name: ip-map-get-reqs
+      doc: Dump all pending ip_map requests
+      attribute-set: ip-map-reqs
+      flags: [admin-perm]
+      dump:
+          reply:
+            attributes:
+              - requests
+    -
+      name: ip-map-set-reqs
+      doc: Respond to one or more ip_map requests
+      attribute-set: ip-map-reqs
+      flags: [admin-perm]
+      do:
+          request:
+            attributes:
+              - requests
+    -
+      name: unix-gid-get-reqs
+      doc: Dump all pending unix_gid requests
+      attribute-set: unix-gid-reqs
+      flags: [admin-perm]
+      dump:
+          reply:
+            attributes:
+              - requests
+    -
+      name: unix-gid-set-reqs
+      doc: Respond to one or more unix_gid requests
+      attribute-set: unix-gid-reqs
+      flags: [admin-perm]
+      do:
+          request:
+            attributes:
+              - requests
+    -
+      name: cache-flush
+      doc: Flush sunrpc caches (ip_map and/or unix_gid)
+      attribute-set: cache-flush
+      flags: [admin-perm]
+      do:
+        request:
+          attributes:
+            - mask
+
+mcast-groups:
+  list:
+    -
+      name: none
+    -
+      name: exportd
diff --git a/Documentation/networking/device_drivers/ethernet/cirrus/cs89x0.rst b/Documentation/networking/device_drivers/ethernet/cirrus/cs89x0.rst
deleted file mode 100644
index e5c283940ac54..0000000000000
--- a/Documentation/networking/device_drivers/ethernet/cirrus/cs89x0.rst
+++ /dev/null
@@ -1,647 +0,0 @@
-.. SPDX-License-Identifier: GPL-2.0
-
-================================================
-Cirrus Logic LAN CS8900/CS8920 Ethernet Adapters
-================================================
-
-.. note::
-
-   This document was contributed by Cirrus Logic for kernel 2.2.5.  This version
-   has been updated for 2.3.48 by Andrew Morton.
-
-   Still, this is too outdated! A major cleanup is needed here.
-
-Cirrus make a copy of this driver available at their website, as
-described below.  In general, you should use the driver version which
-comes with your Linux distribution.
-
-
-Linux Network Interface Driver ver. 2.00 <kernel 2.3.48>
-
-
-.. TABLE OF CONTENTS
-
-   1.0 CIRRUS LOGIC LAN CS8900/CS8920 ETHERNET ADAPTERS
-	1.1 Product Overview
-	1.2 Driver Description
-	    1.2.1 Driver Name
-	    1.2.2 File in the Driver Package
-	1.3 System Requirements
-	1.4 Licensing Information
-
-   2.0 ADAPTER INSTALLATION and CONFIGURATION
-	2.1 CS8900-based Adapter Configuration
-	2.2 CS8920-based Adapter Configuration
-
-   3.0 LOADING THE DRIVER AS A MODULE
-
-   4.0 COMPILING THE DRIVER
-	4.1 Compiling the Driver as a Loadable Module
-	4.2 Compiling the driver to support memory mode
-	4.3 Compiling the driver to support Rx DMA
-
-   5.0 TESTING AND TROUBLESHOOTING
-	5.1 Known Defects and Limitations
-	5.2 Testing the Adapter
-	    5.2.1 Diagnostic Self-Test
-	    5.2.2 Diagnostic Network Test
-	5.3 Using the Adapter's LEDs
-	5.4 Resolving I/O Conflicts
-
-   6.0 TECHNICAL SUPPORT
-	6.1 Contacting Cirrus Logic's Technical Support
-	6.2 Information Required Before Contacting Technical Support
-	6.3 Obtaining the Latest Driver Version
-	6.4 Current maintainer
-	6.5 Kernel boot parameters
-
-
-1. Cirrus Logic LAN CS8900/CS8920 Ethernet Adapters
-===================================================
-
-
-1.1. Product Overview
-=====================
-
-The CS8900-based ISA Ethernet Adapters from Cirrus Logic follow
-IEEE 802.3 standards and support half or full-duplex operation in ISA bus
-computers on 10 Mbps Ethernet networks.  The adapters are designed for operation
-in 16-bit ISA or EISA bus expansion slots and are available in
-10BaseT-only or 3-media configurations (10BaseT, 10Base2, and AUI for 10Base-5
-or fiber networks).
-
-CS8920-based adapters are similar to the CS8900-based adapter with additional
-features for Plug and Play (PnP) support and Wakeup Frame recognition.  As
-such, the configuration procedures differ somewhat between the two types of
-adapters.  Refer to the "Adapter Configuration" section for details on
-configuring both types of adapters.
-
-
-1.2. Driver Description
-=======================
-
-The CS8900/CS8920 Ethernet Adapter driver for Linux supports the Linux
-v2.3.48 or greater kernel.  It can be compiled directly into the kernel
-or loaded at run-time as a device driver module.
-
-1.2.1 Driver Name: cs89x0
-
-1.2.2 Files in the Driver Archive:
-
-The files in the driver at Cirrus' website include:
-
-  ===================  ====================================================
-  readme.txt           this file
-  build                batch file to compile cs89x0.c.
-  cs89x0.c             driver C code
-  cs89x0.h             driver header file
-  cs89x0.o             pre-compiled module (for v2.2.5 kernel)
-  config/Config.in     sample file to include cs89x0 driver in the kernel.
-  config/Makefile      sample file to include cs89x0 driver in the kernel.
-  config/Space.c       sample file to include cs89x0 driver in the kernel.
-  ===================  ====================================================
-
-
-
-1.3. System Requirements
-------------------------
-
-The following hardware is required:
-
-   * Cirrus Logic LAN (CS8900/20-based) Ethernet ISA Adapter
-
-   * IBM or IBM-compatible PC with:
-     * An 80386 or higher processor
-     * 16 bytes of contiguous IO space available between 210h - 370h
-     * One available IRQ (5,10,11,or 12 for the CS8900, 3-7,9-15 for CS8920).
-
-   * Appropriate cable (and connector for AUI, 10BASE-2) for your network
-     topology.
-
-The following software is required:
-
-* LINUX kernel version 2.3.48 or higher
-
-   * CS8900/20 Setup Utility (DOS-based)
-
-   * LINUX kernel sources for your kernel (if compiling into kernel)
-
-   * GNU Toolkit (gcc and make) v2.6 or above (if compiling into kernel
-     or a module)
-
-
-
-1.4. Licensing Information
---------------------------
-
-This program is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free Software
-Foundation, version 1.
-
-This program is distributed in the hope that it will be useful, but WITHOUT
-ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
-more details.
-
-For a full copy of the GNU General Public License, write to the Free Software
-Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-
-
-
-2. Adapter Installation and Configuration
-=========================================
-
-Both the CS8900 and CS8920-based adapters can be configured using parameters
-stored in an on-board EEPROM. You must use the DOS-based CS8900/20 Setup
-Utility if you want to change the adapter's configuration in EEPROM.
-
-When loading the driver as a module, you can specify many of the adapter's
-configuration parameters on the command-line to override the EEPROM's settings
-or for interface configuration when an EEPROM is not used. (CS8920-based
-adapters must use an EEPROM.) See Section 3.0 LOADING THE DRIVER AS A MODULE.
-
-Since the CS8900/20 Setup Utility is a DOS-based application, you must install
-and configure the adapter in a DOS-based system using the CS8900/20 Setup
-Utility before installation in the target LINUX system.  (Not required if
-installing a CS8900-based adapter and the default configuration is acceptable.)
-
-
-2.1. CS8900-based Adapter Configuration
----------------------------------------
-
-CS8900-based adapters shipped from Cirrus Logic have been configured
-with the following "default" settings::
-
-  Operation Mode:      Memory Mode
-  IRQ:                 10
-  Base I/O Address:    300
-  Memory Base Address: D0000
-  Optimization:	       DOS Client
-  Transmission Mode:   Half-duplex
-  BootProm:            None
-  Media Type:	       Autodetect (3-media cards) or
-		       10BASE-T (10BASE-T only adapter)
-
-You should only change the default configuration settings if conflicts with
-another adapter exists. To change the adapter's configuration, run the
-CS8900/20 Setup Utility.
-
-
-2.2. CS8920-based Adapter Configuration
----------------------------------------
-
-CS8920-based adapters are shipped from Cirrus Logic configured as Plug
-and Play (PnP) enabled.  However, since the cs89x0 driver does NOT
-support PnP, you must install the CS8920 adapter in a DOS-based PC and
-run the CS8900/20 Setup Utility to disable PnP and configure the
-adapter before installation in the target Linux system.  Failure to do
-this will leave the adapter inactive and the driver will be unable to
-communicate with the adapter.
-
-::
-
-	****************************************************************
-	*                    CS8920-BASED ADAPTERS:                    *
-	*                                                              *
-	* CS8920-BASED ADAPTERS ARE PLUG and PLAY ENABLED BY DEFAULT.  *
-	* THE CS89X0 DRIVER DOES NOT SUPPORT PnP. THEREFORE, YOU MUST  *
-	* RUN THE CS8900/20 SETUP UTILITY TO DISABLE PnP SUPPORT AND   *
-	* TO ACTIVATE THE ADAPTER.                                     *
-	****************************************************************
-
-
-
-
-3. Loading the Driver as a Module
-=================================
-
-If the driver is compiled as a loadable module, you can load the driver module
-with the 'modprobe' command.  Many of the adapter's configuration parameters can
-be specified as command-line arguments to the load command.  This facility
-provides a means to override the EEPROM's settings or for interface
-configuration when an EEPROM is not used.
-
-Example::
-
-    insmod cs89x0.o io=0x200 irq=0xA media=aui
-
-This example loads the module and configures the adapter to use an IO port base
-address of 200h, interrupt 10, and use the AUI media connection.  The following
-configuration options are available on the command line::
-
-  io=###               - specify IO address (200h-360h)
-  irq=##               - specify interrupt level
-  use_dma=1            - Enable DMA
-  dma=#                - specify dma channel (Driver is compiled to support
-			 Rx DMA only)
-  dmasize=# (16 or 64) - DMA size 16K or 64K.  Default value is set to 16.
-  media=rj45           - specify media type
-   or media=bnc
-   or media=aui
-   or media=auto
-  duplex=full          - specify forced half/full/autonegotiate duplex
-   or duplex=half
-   or duplex=auto
-  debug=#              - debug level (only available if the driver was compiled
-			 for debugging)
-
-**Notes:**
-
-a) If an EEPROM is present, any specified command-line parameter
-   will override the corresponding configuration value stored in
-   EEPROM.
-
-b) The "io" parameter must be specified on the command-line.
-
-c) The driver's hardware probe routine is designed to avoid
-   writing to I/O space until it knows that there is a cs89x0
-   card at the written addresses.  This could cause problems
-   with device probing.  To avoid this behaviour, add one
-   to the ``io=`` module parameter.  This doesn't actually change
-   the I/O address, but it is a flag to tell the driver
-   to partially initialise the hardware before trying to
-   identify the card.  This could be dangerous if you are
-   not sure that there is a cs89x0 card at the provided address.
-
-   For example, to scan for an adapter located at IO base 0x300,
-   specify an IO address of 0x301.
-
-d) The "duplex=auto" parameter is only supported for the CS8920.
-
-e) The minimum command-line configuration required if an EEPROM is
-   not present is:
-
-   io
-   irq
-   media type (no autodetect)
-
-f) The following additional parameters are CS89XX defaults (values
-   used with no EEPROM or command-line argument).
-
-   * DMA Burst = enabled
-   * IOCHRDY Enabled = enabled
-   * UseSA = enabled
-   * CS8900 defaults to half-duplex if not specified on command-line
-   * CS8920 defaults to autoneg if not specified on command-line
-   * Use reset defaults for other config parameters
-   * dma_mode = 0
-
-g) You can use ifconfig to set the adapter's Ethernet address.
-
-h) Many Linux distributions use the 'modprobe' command to load
-   modules.  This program uses the '/etc/conf.modules' file to
-   determine configuration information which is passed to a driver
-   module when it is loaded.  All the configuration options which are
-   described above may be placed within /etc/conf.modules.
-
-   For example::
-
-     > cat /etc/conf.modules
-     ...
-     alias eth0 cs89x0
-     options cs89x0 io=0x0200 dma=5 use_dma=1
-     ...
-
-   In this example we are telling the module system that the
-   ethernet driver for this machine should use the cs89x0 driver.  We
-   are asking 'modprobe' to pass the 'io', 'dma' and 'use_dma'
-   arguments to the driver when it is loaded.
-
-i) Cirrus recommend that the cs89x0 use the ISA DMA channels 5, 6 or
-   7.  You will probably find that other DMA channels will not work.
-
-j) The cs89x0 supports DMA for receiving only.  DMA mode is
-   significantly more efficient.  Flooding a 400 MHz Celeron machine
-   with large ping packets consumes 82% of its CPU capacity in non-DMA
-   mode.  With DMA this is reduced to 45%.
-
-k) If your Linux kernel was compiled with inbuilt plug-and-play
-   support you will be able to find information about the cs89x0 card
-   with the command::
-
-     cat /proc/isapnp
-
-l) If during DMA operation you find erratic behavior or network data
-   corruption you should use your PC's BIOS to slow the EISA bus clock.
-
-m) If the cs89x0 driver is compiled directly into the kernel
-   (non-modular) then its I/O address is automatically determined by
-   ISA bus probing.  The IRQ number, media options, etc are determined
-   from the card's EEPROM.
-
-n) If the cs89x0 driver is compiled directly into the kernel, DMA
-   mode may be selected by providing the kernel with a boot option
-   'cs89x0_dma=N' where 'N' is the desired DMA channel number (5, 6 or 7).
-
-   Kernel boot options may be provided on the LILO command line::
-
-	LILO boot: linux cs89x0_dma=5
-
-   or they may be placed in /etc/lilo.conf::
-
-	image=/boot/bzImage-2.3.48
-	  append="cs89x0_dma=5"
-	  label=linux
-	  root=/dev/hda5
-	  read-only
-
-   The DMA Rx buffer size is hardwired to 16 kbytes in this mode.
-   (64k mode is not available).
-
-
-4. Compiling the Driver
-=======================
-
-The cs89x0 driver can be compiled directly into the kernel or compiled into
-a loadable device driver module.
-
-Just use the standard way to configure the driver and compile the Kernel.
-
-
-4.1. Compiling the Driver to Support Rx DMA
--------------------------------------------
-
-The compile-time optionality for DMA was removed in the 2.3 kernel
-series.  DMA support is now unconditionally part of the driver.  It is
-enabled by the 'use_dma=1' module option.
-
-
-5. Testing and Troubleshooting
-==============================
-
-5.1. Known Defects and Limitations
-----------------------------------
-
-Refer to the RELEASE.TXT file distributed as part of this archive for a list of
-known defects, driver limitations, and work arounds.
-
-
-5.2. Testing the Adapter
-------------------------
-
-Once the adapter has been installed and configured, the diagnostic option of
-the CS8900/20 Setup Utility can be used to test the functionality of the
-adapter and its network connection.  Use the diagnostics 'Self Test' option to
-test the functionality of the adapter with the hardware configuration you have
-assigned. You can use the diagnostics 'Network Test' to test the ability of the
-adapter to communicate across the Ethernet with another PC equipped with a
-CS8900/20-based adapter card (it must also be running the CS8900/20 Setup
-Utility).
-
-.. note::
-
-	 The Setup Utility's diagnostics are designed to run in a
-	 DOS-only operating system environment.  DO NOT run the diagnostics
-	 from a DOS or command prompt session under Windows 95, Windows NT,
-	 OS/2, or other operating system.
-
-To run the diagnostics tests on the CS8900/20 adapter:
-
-   1.  Boot DOS on the PC and start the CS8900/20 Setup Utility.
-
-   2.  The adapter's current configuration is displayed.  Hit the ENTER key to
-       get to the main menu.
-
-   4.  Select 'Diagnostics' (ALT-G) from the main menu.
-       * Select 'Self-Test' to test the adapter's basic functionality.
-       * Select 'Network Test' to test the network connection and cabling.
-
-
-5.2.1. Diagnostic Self-test
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The diagnostic self-test checks the adapter's basic functionality as well as
-its ability to communicate across the ISA bus based on the system resources
-assigned during hardware configuration.  The following tests are performed:
-
-   * IO Register Read/Write Test
-
-     The IO Register Read/Write test insures that the CS8900/20 can be
-     accessed in IO mode, and that the IO base address is correct.
-
-   * Shared Memory Test
-
-     The Shared Memory test insures the CS8900/20 can be accessed in memory
-     mode and that the range of memory addresses assigned does not conflict
-     with other devices in the system.
-
-   * Interrupt Test
-
-     The Interrupt test insures there are no conflicts with the assigned IRQ
-     signal.
-
-   * EEPROM Test
-
-     The EEPROM test insures the EEPROM can be read.
-
-   * Chip RAM Test
-
-     The Chip RAM test insures the 4K of memory internal to the CS8900/20 is
-     working properly.
-
-   * Internal Loop-back Test
-
-     The Internal Loop Back test insures the adapter's transmitter and
-     receiver are operating properly.  If this test fails, make sure the
-     adapter's cable is connected to the network (check for LED activity for
-     example).
-
-   * Boot PROM Test
-
-     The Boot PROM  test insures the Boot PROM is present, and can be read.
-     Failure indicates the Boot PROM  was not successfully read due to a
-     hardware problem or due to a conflicts on the Boot PROM address
-     assignment. (Test only applies if the adapter is configured to use the
-     Boot PROM option.)
-
-Failure of a test item indicates a possible system resource conflict with
-another device on the ISA bus.  In this case, you should use the Manual Setup
-option to reconfigure the adapter by selecting a different value for the system
-resource that failed.
-
-
-5.2.2. Diagnostic Network Test
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-The Diagnostic Network Test verifies a working network connection by
-transferring data between two CS8900/20 adapters installed in different PCs
-on the same network. (Note: the diagnostic network test should not be run
-between two nodes across a router.)
-
-This test requires that each of the two PCs have a CS8900/20-based adapter
-installed and have the CS8900/20 Setup Utility running.  The first PC is
-configured as a Responder and the other PC is configured as an Initiator.
-Once the Initiator is started, it sends data frames to the Responder which
-returns the frames to the Initiator.
-
-The total number of frames received and transmitted are displayed on the
-Initiator's display, along with a count of the number of frames received and
-transmitted OK or in error.  The test can be terminated anytime by the user at
-either PC.
-
-To setup the Diagnostic Network Test:
-
-    1.  Select a PC with a CS8900/20-based adapter and a known working network
-	connection to act as the Responder.  Run the CS8900/20 Setup Utility
-	and select 'Diagnostics -> Network Test -> Responder' from the main
-	menu.  Hit ENTER to start the Responder.
-
-    2.  Return to the PC with the CS8900/20-based adapter you want to test and
-	start the CS8900/20 Setup Utility.
-
-    3.  From the main menu, Select 'Diagnostic -> Network Test -> Initiator'.
-	Hit ENTER to start the test.
-
-You may stop the test on the Initiator at any time while allowing the Responder
-to continue running.  In this manner, you can move to additional PCs and test
-them by starting the Initiator on another PC without having to stop/start the
-Responder.
-
-
-
-5.3. Using the Adapter's LEDs
------------------------------
-
-The 2 and 3-media adapters have two LEDs visible on the back end of the board
-located near the 10Base-T connector.
-
-Link Integrity LED: A "steady" ON of the green LED indicates a valid 10Base-T
-connection.  (Only applies to 10Base-T.  The green LED has no significance for
-a 10Base-2 or AUI connection.)
-
-TX/RX LED: The yellow LED lights briefly each time the adapter transmits or
-receives data. (The yellow LED will appear to "flicker" on a typical network.)
-
-
-5.4. Resolving I/O Conflicts
-----------------------------
-
-An IO conflict occurs when two or more adapter use the same ISA resource (IO
-address, memory address or IRQ).  You can usually detect an IO conflict in one
-of four ways after installing and or configuring the CS8900/20-based adapter:
-
-    1.  The system does not boot properly (or at all).
-
-    2.  The driver cannot communicate with the adapter, reporting an "Adapter
-	not found" error message.
-
-    3.  You cannot connect to the network or the driver will not load.
-
-    4.  If you have configured the adapter to run in memory mode but the driver
-	reports it is using IO mode when loading, this is an indication of a
-	memory address conflict.
-
-If an IO conflict occurs, run the CS8900/20 Setup Utility and perform a
-diagnostic self-test.  Normally, the ISA resource in conflict will fail the
-self-test.  If so, reconfigure the adapter selecting another choice for the
-resource in conflict.  Run the diagnostics again to check for further IO
-conflicts.
-
-In some cases, such as when the PC will not boot, it may be necessary to remove
-the adapter and reconfigure it by installing it in another PC to run the
-CS8900/20 Setup Utility.  Once reinstalled in the target system, run the
-diagnostics self-test to ensure the new configuration is free of conflicts
-before loading the driver again.
-
-When manually configuring the adapter, keep in mind the typical ISA system
-resource usage as indicated in the tables below.
-
-::
-
-  I/O Address    	Device                        IRQ      Device
-  -----------    	--------                      ---      --------
-     200-20F       	Game I/O adapter               3       COM2, Bus Mouse
-     230-23F       	Bus Mouse                      4       COM1
-     270-27F       	LPT3: third parallel port      5       LPT2
-     2F0-2FF       	COM2: second serial port       6       Floppy Disk controller
-     320-32F       	Fixed disk controller          7       LPT1
-							 8       Real-time Clock
-						     9       EGA/VGA display adapter
-						    12       Mouse (PS/2)
-  Memory Address  Device                          13       Math Coprocessor
-  --------------  ---------------------           14       Hard Disk controller
-  A000-BFFF	EGA Graphics Adapter
-  A000-C7FF	VGA Graphics Adapter
-  B000-BFFF	Mono Graphics Adapter
-  B800-BFFF	Color Graphics Adapter
-  E000-FFFF	AT BIOS
-
-
-
-
-6. Technical Support
-====================
-
-6.1. Contacting Cirrus Logic's Technical Support
-------------------------------------------------
-
-Cirrus Logic's CS89XX Technical Application Support can be reached at::
-
-  Telephone  :(800) 888-5016 (from inside U.S. and Canada)
-	     :(512) 442-7555 (from outside the U.S. and Canada)
-  Fax        :(512) 912-3871
-  Email      :ethernet@crystal.cirrus.com
-  WWW        :http://www.cirrus.com
-
-
-6.2. Information Required before Contacting Technical Support
--------------------------------------------------------------
-
-Before contacting Cirrus Logic for technical support, be prepared to provide as
-Much of the following information as possible.
-
-1.) Adapter type (CRD8900, CDB8900, CDB8920, etc.)
-
-2.) Adapter configuration
-
-    * IO Base, Memory Base, IO or memory mode enabled, IRQ, DMA channel
-    * Plug and Play enabled/disabled (CS8920-based adapters only)
-    * Configured for media auto-detect or specific media type (which type).
-
-3.) PC System's Configuration
-
-    * Plug and Play system (yes/no)
-    * BIOS (make and version)
-    * System make and model
-    * CPU (type and speed)
-    * System RAM
-    * SCSI Adapter
-
-4.) Software
-
-    * CS89XX driver and version
-    * Your network operating system and version
-    * Your system's OS version
-    * Version of all protocol support files
-
-5.) Any Error Message displayed.
-
-
-
-6.3 Obtaining the Latest Driver Version
----------------------------------------
-
-You can obtain the latest CS89XX drivers and support software from Cirrus Logic's
-Web site.  You can also contact Cirrus Logic's Technical Support (email:
-ethernet@crystal.cirrus.com) and request that you be registered for automatic
-software-update notification.
-
-Cirrus Logic maintains a web page at http://www.cirrus.com with the
-latest drivers and technical publications.
-
-
-6.4. Current maintainer
------------------------
-
-In February 2000 the maintenance of this driver was assumed by Andrew
-Morton.
-
-6.5 Kernel module parameters
-----------------------------
-
-For use in embedded environments with no cs89x0 EEPROM, the kernel boot
-parameter ``cs89x0_media=`` has been implemented.  Usage is::
-
-	cs89x0_media=rj45    or
-	cs89x0_media=aui     or
-	cs89x0_media=bnc
diff --git a/Documentation/networking/device_drivers/ethernet/index.rst b/Documentation/networking/device_drivers/ethernet/index.rst
index 64621c21fd786..fd3be5d20397c 100644
--- a/Documentation/networking/device_drivers/ethernet/index.rst
+++ b/Documentation/networking/device_drivers/ethernet/index.rst
@@ -18,7 +18,6 @@ Contents:
    amd/pds_vfio_pci
    aquantia/atlantic
    chelsio/cxgb
-   cirrus/cs89x0
    dlink/dl2k
    davicom/dm9000
    dec/dmfe
diff --git a/Documentation/networking/devlink/stmmac.rst b/Documentation/networking/devlink/stmmac.rst
index 47e3ff10bc082..fbaa81ea782d5 100644
--- a/Documentation/networking/devlink/stmmac.rst
+++ b/Documentation/networking/devlink/stmmac.rst
@@ -24,7 +24,7 @@ The ``stmmac`` driver implements the following driver-specific parameters.
      - runtime
      - Enable the Coarse timestamping mode, as defined in the DWMAC TRM.
        A detailed explanation of this timestamping mode can be found in the
-       Socfpga Functionnal Description [1].
+       Socfpga Functional Description [1].
 
        In Coarse mode, the ptp clock is expected to be fed by a high-precision
        clock that is externally adjusted, and the subsecond increment used for
diff --git a/Documentation/networking/ip-sysctl.rst b/Documentation/networking/ip-sysctl.rst
index 2e3a746fcc6da..4c6a35d07a08a 100644
--- a/Documentation/networking/ip-sysctl.rst
+++ b/Documentation/networking/ip-sysctl.rst
@@ -489,7 +489,7 @@ tcp_ecn - INTEGER
 tcp_ecn_option - INTEGER
 	Control Accurate ECN (AccECN) option sending when AccECN has been
 	successfully negotiated during handshake. Send logic inhibits
-	sending AccECN options regarless of this setting when no AccECN
+	sending AccECN options regardless of this setting when no AccECN
 	option has been seen for the reverse direction.
 
 	Possible values are:
diff --git a/Documentation/networking/net_cachelines/net_device.rst b/Documentation/networking/net_cachelines/net_device.rst
index 1c19bb7705dfa..7b3392553fd60 100644
--- a/Documentation/networking/net_cachelines/net_device.rst
+++ b/Documentation/networking/net_cachelines/net_device.rst
@@ -10,7 +10,7 @@ Type                                Name                        fastpath_tx_acce
 =================================== =========================== =================== =================== ===================================================================================
 unsigned_long:32                    priv_flags                  read_mostly                             __dev_queue_xmit(tx)
 unsigned_long:1                     lltx                        read_mostly                             HARD_TX_LOCK,HARD_TX_TRYLOCK,HARD_TX_UNLOCK(tx)
-unsigned long:1                     netmem_tx:1;                read_mostly
+unsigned_long:2                     netmem_tx:2;                read_mostly
 char                                name[16]
 struct netdev_name_node*            name_node
 struct dev_ifalias*                 ifalias
diff --git a/Documentation/networking/net_cachelines/tcp_sock.rst b/Documentation/networking/net_cachelines/tcp_sock.rst
index fecf61166a54e..0f6088c4ab8bb 100644
--- a/Documentation/networking/net_cachelines/tcp_sock.rst
+++ b/Documentation/networking/net_cachelines/tcp_sock.rst
@@ -13,16 +13,16 @@ u16                           tcp_header_len          read_mostly         read_m
 u16                           gso_segs                read_mostly                             tcp_xmit_size_goal
 __be32                        pred_flags              read_write          read_mostly         tcp_select_window(tx);tcp_rcv_established(rx)
 u64                           bytes_received                              read_write          tcp_rcv_nxt_update(rx)
-u32                           segs_in                                     read_write          tcp_v6_rcv(rx)
+u32                           segs_in                 read_write          read_write          tcp_segs_in(),tcp_v6_rcv(rx),tcp_v4_rcv()
 u32                           data_segs_in                                read_write          tcp_v6_rcv(rx)
 u32                           rcv_nxt                 read_mostly         read_write          tcp_cleanup_rbuf,tcp_send_ack,tcp_inq_hint,tcp_transmit_skb,tcp_receive_window(tx);tcp_v6_do_rcv,tcp_rcv_established,tcp_data_queue,tcp_receive_window,tcp_rcv_nxt_update(write)(rx)
 u32                           copied_seq                                  read_mostly         tcp_cleanup_rbuf,tcp_rcv_space_adjust,tcp_inq_hint
 u32                           rcv_wup                                     read_write          __tcp_cleanup_rbuf,tcp_receive_window,tcp_receive_established
 u32                           snd_nxt                 read_write          read_mostly         tcp_rate_check_app_limited,__tcp_transmit_skb,tcp_event_new_data_sent(write)(tx);tcp_rcv_established,tcp_ack,tcp_clean_rtx_queue(rx)
-u32                           segs_out                read_write                              __tcp_transmit_skb
+u32                           segs_out                read_write          read_write          __tcp_transmit_skb
 u32                           data_segs_out           read_write                              __tcp_transmit_skb,tcp_update_skb_after_send
 u64                           bytes_sent              read_write                              __tcp_transmit_skb
-u64                           bytes_acked                                 read_write          tcp_snd_una_update/tcp_ack
+u64                           bytes_acked             read_write                              tcp_snd_una_update/tcp_ack
 u32                           dsack_dups
 u32                           snd_una                 read_mostly         read_write          tcp_wnd_end,tcp_urg_mode,tcp_minshall_check,tcp_cwnd_validate(tx);tcp_ack,tcp_may_update_window,tcp_clean_rtx_queue(write),tcp_ack_tstamp(rx)
 u32                           snd_sml                 read_write                              tcp_minshall_check,tcp_minshall_update
@@ -99,8 +99,8 @@ u32                           snd_cwnd_stamp
 u32                           prior_cwnd
 u32                           prr_delivered
 u32                           prr_out                 read_mostly         read_mostly         tcp_rate_skb_sent,tcp_newly_delivered(tx);tcp_ack,tcp_rate_gen,tcp_clean_rtx_queue(rx)
-u32                           delivered               read_mostly         read_write          tcp_rate_skb_sent, tcp_newly_delivered(tx);tcp_ack, tcp_rate_gen, tcp_clean_rtx_queue (rx)
-u32                           delivered_ce            read_mostly         read_write          tcp_rate_skb_sent(tx);tcp_rate_gen(rx)
+u32                           delivered               read_write                              tcp_rate_skb_sent, tcp_newly_delivered(tx);tcp_ack, tcp_rate_gen, tcp_clean_rtx_queue (rx)
+u32                           delivered_ce            read_write                              tcp_rate_skb_sent(tx);tcp_rate_gen(rx)
 u32                           received_ce             read_mostly         read_write
 u32[3]                        received_ecn_bytes      read_mostly         read_write
 u8:4                          received_ce_pending     read_mostly         read_write
diff --git a/Documentation/networking/netmem.rst b/Documentation/networking/netmem.rst
index b63aded463370..217869d1108dd 100644
--- a/Documentation/networking/netmem.rst
+++ b/Documentation/networking/netmem.rst
@@ -95,4 +95,10 @@ Driver TX Requirements
    netdev@, or reach out to the maintainers and/or almasrymina@google.com for
    help adding the netmem API.
 
-2. Driver should declare support by setting `netdev->netmem_tx = true`
+2. Driver should declare support by setting `netdev->netmem_tx` to the
+   appropriate mode:
+
+   - `NETMEM_TX_DMA`: for physical devices that perform DMA.
+
+   - `NETMEM_TX_NO_DMA`: for virtual or passthrough devices that do
+     not DMA, but still support handling of netmem-backed skbs.
diff --git a/Documentation/networking/smc-sysctl.rst b/Documentation/networking/smc-sysctl.rst
index a8b4f357174ef..93cc6244f6c85 100644
--- a/Documentation/networking/smc-sysctl.rst
+++ b/Documentation/networking/smc-sysctl.rst
@@ -86,7 +86,7 @@ smcr_max_send_wr - INTEGER
 	Please be aware that all the buffers need to be allocated as a physically
 	continuous array in which each element is a single buffer and has the size
 	of SMC_WR_BUF_SIZE (48) bytes. If the allocation fails, we keep retrying
-	with half of the buffer count until it is ether successful or (unlikely)
+	with half of the buffer count until it is either successful or (unlikely)
 	we dip below the old hard coded value which is 16 where we give up much
 	like before having this control.
 
@@ -100,14 +100,14 @@ smcr_max_recv_wr - INTEGER
 	depending on the workload it can be a bottleneck in a sense that threads
 	have to wait for work request buffers to become available. Before the
 	introduction of this control the maximal number of work request buffers
-	available on the receive path used to be hard coded to 16. With this control
+	available on the receive path used to be hard coded to 48. With this control
 	it becomes configurable. The acceptable range is between 2 and 2048.
 
 	Please be aware that all the buffers need to be allocated as a physically
 	continuous array in which each element is a single buffer and has the size
 	of SMC_WR_BUF_SIZE (48) bytes. If the allocation fails, we keep retrying
-	with half of the buffer count until it is ether successful or (unlikely)
-	we dip below the old hard coded value which is 16 where we give up much
+	with half of the buffer count until it is either successful or (unlikely)
+	we dip below the old hard coded value which is 48 where we give up much
 	like before having this control.
 
 	Default: 48
diff --git a/Documentation/networking/tcp_ao.rst b/Documentation/networking/tcp_ao.rst
index d5b6d0df63c35..55304037aa814 100644
--- a/Documentation/networking/tcp_ao.rst
+++ b/Documentation/networking/tcp_ao.rst
@@ -7,9 +7,9 @@ TCP Authentication Option Linux implementation (RFC5925)
 TCP Authentication Option (TCP-AO) provides a TCP extension aimed at verifying
 segments between trusted peers. It adds a new TCP header option with
 a Message Authentication Code (MAC). MACs are produced from the content
-of a TCP segment using a hashing function with a password known to both peers.
+of a TCP segment using a key known to both peers.
 The intent of TCP-AO is to deprecate TCP-MD5 providing better security,
-key rotation and support for a variety of hashing algorithms.
+key rotation and support for a variety of MAC algorithms.
 
 1. Introduction
 ===============
@@ -19,16 +19,18 @@ key rotation and support for a variety of hashing algorithms.
  +----------------------+------------------------+-----------------------+
  |                      |       TCP-MD5          |         TCP-AO        |
  +======================+========================+=======================+
- |Supported hashing     |MD5                     |Must support HMAC-SHA1 |
- |algorithms            |(cryptographically weak)|(chosen-prefix attacks)|
- |                      |                        |and CMAC-AES-128 (only |
- |                      |                        |side-channel attacks). |
- |                      |                        |May support any hashing|
- |                      |                        |algorithm.             |
+ |Supported MAC         |MD5 of data and key     |HMAC-SHA-1-96 and      |
+ |algorithms            |(cryptographically weak)|AES-128-CMAC-96.       |
+ |                      |                        |Implementations are    |
+ |                      |                        |permitted to support   |
+ |                      |                        |additional algorithms. |
  +----------------------+------------------------+-----------------------+
- |Length of MACs (bytes)|16                      |Typically 12-16.       |
- |                      |                        |Other variants that fit|
- |                      |                        |TCP header permitted.  |
+ |Length of MACs (bytes)|16                      |12 for HMAC-SHA-1-96   |
+ |                      |                        |and AES-128-CMAC-96.   |
+ |                      |                        |Implementations are    |
+ |                      |                        |permitted to support   |
+ |                      |                        |any MAC length that    |
+ |                      |                        |fits in the TCP header.|
  +----------------------+------------------------+-----------------------+
  |Number of keys per    |1                       |Many                   |
  |TCP connection        |                        |                       |
@@ -296,6 +298,20 @@ userspace manage TCP-AO on a per-socket basis. In order to add/delete MKTs
 It is not allowed to add a key on an established non-TCP-AO connection
 as well as to remove the last key from TCP-AO connection.
 
+``TCP_AO_ADD_KEY`` allows the MAC algorithm and MAC length to be selected.
+Linux supports the mandatory-to-implement algorithms HMAC-SHA-1-96 and
+AES-128-CMAC-96. In addition, as Linux extensions, it supports:
+
+- HMAC-SHA256. Linux uses HMAC-SHA256 in the same way as HMAC-SHA1; this
+  includes omitting an explicit entropy extraction step. To work around the
+  missing entropy extraction, users should provide keys with full entropy. The
+  implementation is interoperable with other implementations of HMAC-SHA256 for
+  TCP-AO only when they have implemented the key derivation the same way (and
+  also the same MAC length is selected on each side).
+
+- Any MAC length for any of the supported MAC algorithms, provided it fits in
+  the TCP header and is at least 4 bytes.
+
 ``setsockopt(TCP_AO_DEL_KEY)`` command may specify ``tcp_ao_del::current_key``
 + ``tcp_ao_del::set_current`` and/or ``tcp_ao_del::rnext``
 + ``tcp_ao_del::set_rnext`` which makes such delete "forced": it
diff --git a/Documentation/process/coding-style.rst b/Documentation/process/coding-style.rst
index 35b381230f6e4..a3bf75dc7c880 100644
--- a/Documentation/process/coding-style.rst
+++ b/Documentation/process/coding-style.rst
@@ -936,7 +936,7 @@ used.
 ---------------------
 
 The kernel provides the following general purpose memory allocators:
-kmalloc(), kzalloc(), kmalloc_array(), kcalloc(), vmalloc(), and
+kmalloc(), kzalloc(), kmalloc_objs(), kzalloc_objs(), vmalloc(), and
 vzalloc().  Please refer to the API documentation for further information
 about them.  :ref:`Documentation/core-api/memory-allocation.rst
 <memory_allocation>`
@@ -945,7 +945,7 @@ The preferred form for passing a size of a struct is the following:
 
 .. code-block:: c
 
-	p = kmalloc(sizeof(*p), ...);
+	p = kmalloc_obj(*p, ...);
 
 The alternative form where struct name is spelled out hurts readability and
 introduces an opportunity for a bug when the pointer variable type is changed
@@ -959,13 +959,13 @@ The preferred form for allocating an array is the following:
 
 .. code-block:: c
 
-	p = kmalloc_array(n, sizeof(...), ...);
+	p = kmalloc_objs(*ptr, n, ...);
 
 The preferred form for allocating a zeroed array is the following:
 
 .. code-block:: c
 
-	p = kcalloc(n, sizeof(...), ...);
+	p = kzalloc_objs(*ptr, n, ...);
 
 Both forms check for overflow on the allocation size n * sizeof(...),
 and return NULL if that occurred.
diff --git a/Documentation/process/deprecated.rst b/Documentation/process/deprecated.rst
index fed56864d0363..ac75b7ecac470 100644
--- a/Documentation/process/deprecated.rst
+++ b/Documentation/process/deprecated.rst
@@ -392,13 +392,14 @@ allocations. For example, these open coded assignments::
 
 become, respectively::
 
-	ptr = kmalloc_obj(*ptr, gfp);
-	ptr = kzalloc_obj(*ptr, gfp);
-	ptr = kmalloc_objs(*ptr, count, gfp);
-	ptr = kzalloc_objs(*ptr, count, gfp);
-	ptr = kmalloc_flex(*ptr, flex_member, count, gfp);
-	__auto_type ptr = kmalloc_obj(struct foo, gfp);
-
+	ptr = kmalloc_obj(*ptr [, gfp] );
+	ptr = kzalloc_obj(*ptr [, gfp] );
+	ptr = kmalloc_objs(*ptr, count [, gfp] );
+	ptr = kzalloc_objs(*ptr, count [, gfp] );
+	ptr = kmalloc_flex(*ptr, flex_member, count [, gfp] );
+	__auto_type ptr = kmalloc_obj(struct foo [, gfp] );
+
+The argument gfp is optional, the default value is GFP_KERNEL.
 If `ptr->flex_member` is annotated with __counted_by(), the allocation
 will automatically fail if `count` is larger than the maximum
 representable value that can be stored in the counter member associated
diff --git a/Documentation/process/maintainer-handbooks.rst b/Documentation/process/maintainer-handbooks.rst
index 3d72ad25fc6ac..3821e78aefc0a 100644
--- a/Documentation/process/maintainer-handbooks.rst
+++ b/Documentation/process/maintainer-handbooks.rst
@@ -7,14 +7,13 @@ The purpose of this document is to provide subsystem specific information
 which is supplementary to the general development process handbook
 :ref:`Documentation/process <development_process_main>`.
 
-Contents:
+For developers, see below for all the known subsystem specific guides.
+If the subsystem you are contributing to does not have a guide listed
+here, it is fair to seek clarification of questions raised in
+Documentation/maintainer/maintainer-entry-profile.rst.
 
-.. toctree::
-   :numbered:
-   :maxdepth: 2
+For maintainers, consider documenting additional requirements and
+expectations if submissions routinely overlook specific submission
+criteria. See Documentation/maintainer/maintainer-entry-profile.rst.
 
-   maintainer-netdev
-   maintainer-soc
-   maintainer-soc-clean-dts
-   maintainer-tip
-   maintainer-kvm-x86
+.. maintainers-profile-toc::
diff --git a/Documentation/process/maintainers.rst b/Documentation/process/maintainers.rst
index 6174cfb4138ff..5d1b1464c3aea 100644
--- a/Documentation/process/maintainers.rst
+++ b/Documentation/process/maintainers.rst
@@ -1 +1,3 @@
+.. SPDX-License-Identifier: GPL-2.0
+
 .. maintainers-include::
diff --git a/Documentation/scheduler/sched-deadline.rst b/Documentation/scheduler/sched-deadline.rst
index 3ad93cd7b59ad..9019b66f6a5ba 100644
--- a/Documentation/scheduler/sched-deadline.rst
+++ b/Documentation/scheduler/sched-deadline.rst
@@ -685,7 +685,7 @@ Deadline Task Scheduling
 
  Deadline tasks cannot have a cpu affinity mask smaller than the root domain they
  are created on. So, using ``sched_setaffinity(2)`` won't work. Instead, the
- the deadline task should be created in a restricted root domain. This can be
+ deadline task should be created in a restricted root domain. This can be
  done using the cpuset controller of either cgroup v1 (deprecated) or cgroup v2.
  See :ref:`Documentation/admin-guide/cgroup-v1/cpusets.rst <cpusets>` and
  :ref:`Documentation/admin-guide/cgroup-v2.rst <cgroup-v2>` for more information.
diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst
index 03d595d178ea4..c4f59c08d8a4f 100644
--- a/Documentation/scheduler/sched-ext.rst
+++ b/Documentation/scheduler/sched-ext.rst
@@ -339,6 +339,11 @@ The following briefly shows how a waking task is scheduled and executed.
      leaves (e.g., when ``ops.dispatch()`` moves it to a terminal DSQ, or
      on property change / sleep).
 
+   Note that ``ops.enqueue()`` can be called multiple times in a row without
+   an intervening call to ``ops.dequeue()``. This can happen, for example,
+   when a task on a user-created DSQ is re-enqueued using
+   ``scx_bpf_dsq_reenq()``. The task stays in BPF custody the entire time.
+
    When a task leaves BPF scheduler custody, ``ops.dequeue()`` is invoked.
    The dequeue can happen for different reasons, distinguished by flags:
 
@@ -503,7 +508,7 @@ Where to Look
     custom DSQ.
 
   * ``scx_qmap[.bpf].c``: A multi-level FIFO scheduler supporting five
-    levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``.
+    levels of priority implemented with arena-backed doubly-linked lists.
 
   * ``scx_central[.bpf].c``: A central FIFO scheduler where all scheduling
     decisions are made on one CPU, demonstrating ``LOCAL_ON`` dispatching,
diff --git a/Documentation/scsi/st.rst b/Documentation/scsi/st.rst
index b4a092faa9c82..539ff06daf5e9 100644
--- a/Documentation/scsi/st.rst
+++ b/Documentation/scsi/st.rst
@@ -93,7 +93,7 @@ optionally written. In both cases end of data is signified by
 returning zero bytes for two consecutive reads.
 
 Writing filemarks without the immediate bit set in the SCSI command block acts
-as a synchronization point, i.e., all remaining data form the drive buffers is
+as a synchronization point, i.e., all remaining data from the drive buffers is
 written to tape before the command returns. This makes sure that write errors
 are caught at that point, but this takes time. In some applications, several
 consecutive files must be written fast. The MTWEOFI operation can be used to
diff --git a/Documentation/security/credentials.rst b/Documentation/security/credentials.rst
index d0191c8b8060e..4996838491b10 100644
--- a/Documentation/security/credentials.rst
+++ b/Documentation/security/credentials.rst
@@ -189,9 +189,9 @@ The Linux kernel supports the following types of credentials:
      be searched for the desired key.  Each process may subscribe to a number
      of keyrings:
 
-	Per-thread keying
-	Per-process keyring
-	Per-session keyring
+	- Per-thread keyring
+	- Per-process keyring
+	- Per-session keyring
 
      When a process accesses a key, if not already present, it will normally be
      cached on one of these keyrings for future accesses to find.
diff --git a/Documentation/security/ipe.rst b/Documentation/security/ipe.rst
index 5eb3e6265fbde..c51dcb16a377b 100644
--- a/Documentation/security/ipe.rst
+++ b/Documentation/security/ipe.rst
@@ -412,6 +412,73 @@ a standard securityfs policy tree::
 
 The policy is stored in the ``->i_private`` data of the MyPolicy inode.
 
+BPF/Hornet Integration
+~~~~~~~~~~~~~~~~~~~~~~
+
+IPE integrates with the Hornet LSM to enforce integrity policies on BPF
+program loading. Hornet performs cryptographic verification of BPF program
+signatures (PKCS#7 with authenticated attributes containing map hashes)
+and produces an ``enum lsm_integrity_verdict``. IPE acts as the policy
+enforcer: it stores Hornet's verdict in a per-program LSM blob and later
+evaluates it against the active IPE policy when the BPF program load is
+finalized.
+
+Enforcement is split across two LSM hooks so that signature verification
+and policy evaluation are cleanly separated. This also lets IPE evaluate
+policy uniformly even when no integrity provider ran.
+
+The hook flow is:
+
+  1. User space (or the kernel, via the BPF light skeleton) invokes
+     ``BPF_PROG_LOAD`` through the ``bpf()`` syscall.
+  2. Hornet's ``bpf_prog_load_integrity`` hook runs first and calls
+     ``hornet_check_program()`` to verify the program's PKCS#7 signature
+     against ``attr->keyring_id`` and to check the signed map hashes
+     (decoded from the ``OID_hornet_data`` authenticated attribute)
+     against the live hashes of the frozen maps referenced from
+     ``attr->fd_array``. The function produces one of
+     ``LSM_INT_VERDICT_OK``, ``LSM_INT_VERDICT_UNSIGNED``,
+     ``LSM_INT_VERDICT_BADSIG``, ``LSM_INT_VERDICT_PARTIALSIG``,
+     ``LSM_INT_VERDICT_UNKNOWNKEY``, ``LSM_INT_VERDICT_UNEXPECTED``, or
+     ``LSM_INT_VERDICT_FAULT``.
+  3. Hornet calls ``security_bpf_prog_load_post_integrity()`` with the
+     resulting verdict and its ``lsm_id``. IPE's
+     ``ipe_bpf_prog_load_post_integrity`` handler does **not** enforce
+     policy here; it only stashes the verdict in IPE's per-program blob
+     and returns ``0``. This keeps the integrity step decoupled from
+     policy evaluation and allows multiple providers to coexist without
+     short-circuiting each other.
+  4. The core BPF load path then invokes the standard ``bpf_prog_load``
+     LSM hook. IPE's ``ipe_bpf_prog_load`` reads the verdict back out of
+     the per-program blob, populates an ``ipe_eval_ctx``, and calls
+     ``ipe_evaluate_event()`` against the active policy's
+     ``BPF_PROG_LOAD`` rules. A deny verdict returns ``-EACCES`` and
+     aborts the load.
+
+If no integrity provider populated the blob (e.g. Hornet is not enabled,
+or the load came from a path Hornet does not cover), the verdict defaults
+to ``LSM_INT_VERDICT_NONE`` and IPE evaluates accordingly. Policy can
+therefore express "deny anything Hornet did not vouch for".
+
+Three properties are available for BPF policy rules:
+
+  - ``bpf_signature``: Matches against the integrity verdict (OK, UNSIGNED,
+    BADSIG, etc.)
+  - ``bpf_keyring``: Matches against the keyring specified in ``bpf_attr``
+    (BUILTIN, SECONDARY, PLATFORM)
+  - ``bpf_kernel``: Matches whether the load originated from kernel space
+    (TRUE/FALSE). This is important because the BPF light skeleton
+    infrastructure performs a secondary kernel-originated program load that
+    does not carry a signature.
+
+All three properties are gated on ``CONFIG_IPE_PROP_BPF_SIGNATURE`` which
+depends on ``CONFIG_SECURITY_HORNET``.
+
+The evaluation context (``struct ipe_eval_ctx``) carries three BPF-specific
+fields: ``bpf_verdict`` (the integrity verdict enum), ``bpf_keyring_id``
+(the ``s32`` keyring ID from ``bpf_attr``), and ``bpf_kernel`` (bool
+indicating kernel origin).
+
 Tests
 -----
 
@@ -439,6 +506,7 @@ IPE has KUnit Tests for the policy parser. Recommended kunitconfig::
   CONFIG_IPE_PROP_DM_VERITY_SIGNATURE=y
   CONFIG_IPE_PROP_FS_VERITY=y
   CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG=y
+  CONFIG_IPE_PROP_BPF_SIGNATURE=y
   CONFIG_SECURITY_IPE_KUNIT_TEST=y
 
 In addition, IPE has a python based integration
diff --git a/Documentation/sound/alsa-configuration.rst b/Documentation/sound/alsa-configuration.rst
index f75f087639415..394cf87f340b4 100644
--- a/Documentation/sound/alsa-configuration.rst
+++ b/Documentation/sound/alsa-configuration.rst
@@ -1097,7 +1097,7 @@ output (with ``--no-upload`` option) to kernel bugzilla or alsa-devel
 ML (see the section `Links and Addresses`_).
 
 ``power_save`` and ``power_save_controller`` options are for power-saving
-mode.  See powersave.rst for details.
+mode.  See Documentation/sound/designs/powersave.rst for details.
 
 Note 2: If you get click noises on output, try the module option
 ``position_fix=1`` or ``2``.  ``position_fix=1`` will use the SD_LPIB
@@ -1168,7 +1168,7 @@ line_outs_monitor
 enable_monitor
     Enable Analog Out on Channel 63/64 by default.
 
-See hdspm.rst for details.
+See Documentation/sound/cards/hdspm.rst for details.
 
 Module snd-ice1712
 ------------------
diff --git a/Documentation/sound/codecs/index.rst b/Documentation/sound/codecs/index.rst
index 2cb95d87bbef2..7594d0a38d6bc 100644
--- a/Documentation/sound/codecs/index.rst
+++ b/Documentation/sound/codecs/index.rst
@@ -7,3 +7,4 @@ Codec-Specific Information
    :maxdepth: 2
 
    cs35l56
+   tas675x
diff --git a/Documentation/sound/codecs/tas675x.rst b/Documentation/sound/codecs/tas675x.rst
new file mode 100644
index 0000000000000..c08b0e392306e
--- /dev/null
+++ b/Documentation/sound/codecs/tas675x.rst
@@ -0,0 +1,820 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+======================================
+TAS675x Codec Mixer Controls
+======================================
+
+This document describes the ALSA mixer controls for the TAS675x
+4-channel amplifier driver.
+
+For device tree bindings, see:
+Documentation/devicetree/bindings/sound/ti,tas67524.yaml
+
+DSP Signal Path Mode
+====================
+
+DSP Signal Path Mode
+--------------------
+
+:Description: Signal processing mode selection.
+:Type:        Enumerated
+:Default:     Normal
+:Options:     Normal, LLP, FFLP
+:Register:    0x32 bits [1:0]
+
+Normal
+  Full DSP with all features available.
+
+LLP (Low Latency Path)
+  Bypasses DSP processing. DSP protection features (Thermal Foldback,
+  PVDD Foldback, Clip Detect) and Real-Time Load Diagnostics unavailable.
+
+FFLP (Full Feature Low Latency Path)
+  Reduced latency. Real-Time Load Diagnostics unavailable.
+
+The following controls are unavailable in LLP mode:
+``Thermal Foldback Switch``, ``PVDD Foldback Switch``,
+``DC Blocker Bypass Switch``, ``Clip Detect Switch``, ``Audio SDOUT Switch``.
+
+The following controls require Normal mode (unavailable in FFLP and LLP):
+``CHx RTLDG Switch``, ``RTLDG Clip Mask Switch``, ``ISENSE Calibration Switch``,
+``RTLDG Open Load Threshold``, ``RTLDG Short Load Threshold``,
+``CHx RTLDG Impedance``.
+
+Volume Controls
+===============
+
+Analog Playback Volume
+----------------------
+
+:Description: Analog output gain for all channels (CH1/CH2 and CH3/CH4 pairs).
+:Type:        Volume (TLV)
+:Default:     0 dB
+:Range:       -15.5 dB to 0 dB (0.5 dB steps)
+:Register:    0x4A (CH1/CH2), 0x4B (CH3/CH4)
+
+Analog Gain Ramp Step
+---------------------
+
+:Description: Anti-pop ramp step duration for analog gain transitions.
+:Type:        Enumerated
+:Default:     15us
+:Options:     15us, 60us, 200us, 400us
+:Register:    0x4E bits [3:2]
+
+CHx Digital Playback Volume
+---------------------------
+
+:Description: Per-channel digital volume control (x = 1, 2, 3, 4).
+:Type:        Volume (TLV)
+:Default:     0 dB
+:Range:       -103 dB to 0 dB (0.5 dB steps)
+:Bounds:      0x30 (min/mute) to 0xFF (max)
+:Register:    0x40 (CH1), 0x41 (CH2), 0x42 (CH3), 0x43 (CH4)
+
+Volume Ramp Down Rate
+---------------------
+
+:Description: Update frequency during mute transition.
+:Type:        Enumerated
+:Default:     16 FS
+:Options:     4 FS, 16 FS, 32 FS, Instant
+:Register:    0x44 bits [7:6]
+
+Volume Ramp Down Step
+---------------------
+
+:Description: dB change per update during mute.
+:Type:        Enumerated
+:Default:     0.5dB
+:Options:     4dB, 2dB, 1dB, 0.5dB
+:Register:    0x44 bits [5:4]
+
+Volume Ramp Up Rate
+-------------------
+
+:Description: Update frequency during unmute transition.
+:Type:        Enumerated
+:Default:     16 FS
+:Options:     4 FS, 16 FS, 32 FS, Instant
+:Register:    0x44 bits [3:2]
+
+Volume Ramp Up Step
+-------------------
+
+:Description: dB change per update during unmute.
+:Type:        Enumerated
+:Default:     0.5dB
+:Options:     4dB, 2dB, 1dB, 0.5dB
+:Register:    0x44 bits [1:0]
+
+CH1/2 Volume Combine
+--------------------
+
+:Description: Links digital volume controls for CH1 and CH2.
+:Type:        Enumerated
+:Default:     Independent
+:Options:     Independent, CH2 follows CH1, CH1 follows CH2
+:Register:    0x46 bits [1:0]
+
+CH3/4 Volume Combine
+--------------------
+
+:Description: Links digital volume controls for CH3 and CH4.
+:Type:        Enumerated
+:Default:     Independent
+:Options:     Independent, CH4 follows CH3, CH3 follows CH4
+:Register:    0x46 bits [3:2]
+
+Auto Mute & Silence Detection
+==============================
+
+CHx Auto Mute Switch
+--------------------
+
+:Description: Enables automatic muting on zero-signal detection (x = 1, 2, 3, 4).
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Register:    0x47 bit [0] (CH1), bit [1] (CH2), bit [2] (CH3), bit [3] (CH4)
+
+Auto Mute Combine Switch
+------------------------
+
+:Description: Coordinated muting behaviour across all channels.
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Behavior:    Disabled: channels mute independently when their signal is zero.
+              Enabled: all channels mute together only when all detect zero
+              signal; unmute when any channel has non-zero signal.
+:Register:    0x47 bit [4]
+
+CHx Auto Mute Time
+------------------
+
+:Description: Duration of zero signal before muting triggers (x = 1, 2, 3, 4).
+:Type:        Enumerated
+:Default:     11.5ms
+:Options:     11.5ms, 53ms, 106.5ms, 266.5ms, 535ms, 1065ms, 2665ms, 5330ms
+:Register:    0x48 bits [7:4] (CH1), bits [3:0] (CH2),
+              0x49 bits [7:4] (CH3), bits [3:0] (CH4)
+:Note:        Values are at 96 kHz. At 48 kHz, times are doubled.
+
+Clock & EMI Management
+======================
+
+Spread Spectrum Mode
+--------------------
+
+:Description: Frequency dithering mode to reduce peak EMI.
+:Type:        Enumerated
+:Default:     Disabled
+:Options:     Disabled, Triangle, Random, Triangle and Random
+:Register:    0x61 bits [1:0]
+
+SS Triangle Range
+-----------------
+
+:Description: Frequency deviation range for Triangle spread spectrum.
+:Type:        Enumerated
+:Default:     6.5%
+:Options:     6.5%, 13.5%, 5%, 10%
+:Register:    0x62 bits [1:0]
+:Note:        Applies only when Spread Spectrum Mode includes Triangle.
+
+SS Random Range
+---------------
+
+:Description: Frequency deviation range for Random spread spectrum.
+:Type:        Enumerated
+:Default:     0.83%
+:Options:     0.83%, 2.50%, 5.83%, 12.50%, 25.83%
+:Register:    0x62 bits [6:4]
+:Note:        Applies only when Spread Spectrum Mode includes Random.
+
+SS Random Dwell Range
+---------------------
+
+:Description: Dwell time range for Random spread spectrum (FSS = spread
+              spectrum modulation frequency).
+:Type:        Enumerated
+:Default:     1/FSS to 2/FSS
+:Options:     1/FSS to 2/FSS, 1/FSS to 4/FSS, 1/FSS to 8/FSS, 1/FSS to 15/FSS
+:Register:    0x62 bits [3:2]
+:Note:        Applies only when Spread Spectrum Mode includes Random.
+
+SS Triangle Dwell Min
+---------------------
+
+:Description: Minimum dwell time at Triangle spread spectrum frequency extremes.
+:Type:        Integer
+:Default:     0
+:Range:       0 to 15 (0 = feature disabled)
+:Register:    0x66 bits [7:4]
+:Note:        Counts in FSS clock cycles. The modulator holds the extreme
+              frequency for at least this many FSS cycles before reversing.
+              When Dwell Min equals Dwell Max, the dwell feature is inactive.
+              For FSS values at each PWM frequency refer to the "Spread
+              Spectrum" section of the TRM.
+
+SS Triangle Dwell Max
+---------------------
+
+:Description: Maximum dwell time at Triangle spread spectrum frequency extremes.
+:Type:        Integer
+:Default:     0
+:Range:       0 to 15 (0 = feature disabled)
+:Register:    0x66 bits [3:0]
+:Note:        Counts in FSS clock cycles. Must be >= Dwell Min. When Dwell Max
+              equals Dwell Min, the dwell feature is inactive.
+
+Hardware Protection
+===================
+
+OTSD Auto Recovery Switch
+--------------------------
+
+:Description: Enables automatic recovery from over-temperature shutdown.
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Register:    0x8F bit [1]
+:Note:        When disabled, manual fault clearing is required after OTSD events.
+
+Overcurrent Limit Level
+-----------------------
+
+:Description: Current-limit trip point sensitivity.
+:Type:        Enumerated
+:Default:     Level 4
+:Options:     Level 4, Level 3, Level 2, Level 1
+:Register:    0x55 bits [1:0]
+:Note:        Level 4 is the least sensitive (highest trip current); Level 1 is
+              the most sensitive. The exact ILIM values depend on operating
+              conditions (PVDD voltage, switching frequency, and temperature).
+              Refer to the Electrical Characteristics table and the
+              "Overcurrent Limit (Cycle-By-Cycle)" section of the TRM.
+
+CHx OTW Threshold
+-----------------
+
+:Description: Over-temperature warning threshold per channel (x = 1, 2, 3, 4).
+:Type:        Enumerated
+:Default:     >95C
+:Options:     Disabled, >95C, >110C, >125C, >135C, >145C, >155C, >165C
+:Register:    0xE2 bits [6:4] (CH1), bits [2:0] (CH2),
+              0xE3 bits [6:4] (CH3), bits [2:0] (CH4)
+
+Temperature and Voltage Monitoring
+===================================
+
+PVDD Sense
+----------
+
+:Description: Supply voltage sense register.
+:Type:        Integer (read-only)
+:Range:       0 to 255
+:Conversion:  value × 0.19 V
+:Register:    0x74
+
+Global Temperature
+------------------
+
+:Description: Global die temperature sense register.
+:Type:        Integer (read-only)
+:Range:       0 to 255
+:Conversion:  (value × 0.5 °C) − 50 °C
+:Register:    0x75
+
+CHx Temperature Range
+---------------------
+
+:Description: Per-channel coarse temperature range indicator (x = 1, 2, 3, 4).
+:Type:        Integer (read-only)
+:Range:       0 to 3
+:Mapping:     0 = <80 °C, 1 = 80–100 °C, 2 = 100–120 °C, 3 = >120 °C
+:Register:    0xBB bits [7:6] (CH1), bits [5:4] (CH2),
+              0xBC bits [3:2] (CH3), bits [1:0] (CH4)
+
+Load Diagnostics
+================
+
+The TAS675x provides three load diagnostic modes:
+
+DC Load Diagnostics (DC LDG)
+  Measures DC resistance to detect S2G (short-to-ground), S2P
+  (short-to-power), OL (open load), and SL (shorted load) faults.
+
+AC Load Diagnostics (AC LDG)
+  Measures complex AC impedance at a configurable frequency. Detects
+  capacitive loads and tweeter configurations.
+
+Real-Time Load Diagnostics (RTLDG)
+  Monitors impedance continuously during playback using a pilot tone.
+  Normal DSP mode only, at 48 kHz or 96 kHz.
+
+Fast Boot Mode
+--------------
+
+By default the device runs DC load diagnostics at initialization before
+accepting audio. Setting ``ti,fast-boot`` in the device tree bypasses this
+initial diagnostic run for faster startup. Automatic diagnostics after
+fault recovery remain enabled.
+
+DC Load Diagnostics
+-------------------
+
+The ``CHx DC LDG Report`` 4-bit fault field uses the following encoding:
+
+  ======  ===========  ===================================================
+  Bit     Fault        Description
+  ======  ===========  ===================================================
+  [3]     S2G          Short-to-Ground
+  [2]     S2P          Short-to-Power
+  [1]     OL           Open Load
+  [0]     SL           Shorted Load
+  ======  ===========  ===================================================
+
+DC LDG Trigger
+~~~~~~~~~~~~~~
+
+:Description: Triggers manual DC load diagnostics on all channels.
+:Type:        Boolean (write-only)
+:Note:        Returns -EBUSY if any DAI stream (playback or capture) is active.
+              The driver manages all channel state transitions. Blocks until
+              diagnostics complete or time out (300 ms).
+
+DC LDG Auto Diagnostics Switch
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Enables automatic DC diagnostics after fault recovery.
+:Type:        Boolean Switch
+:Default:     Enabled (1)
+:Register:    0xB0 bit [0]
+:Note:        Active-low, when enabled, affected channels re-run diagnostics after
+              fault recovery and retry approximately every 750 ms until resolved.
+
+CHx LO LDG Switch
+~~~~~~~~~~~~~~~~~
+
+:Description: Enables line output load detection per channel (x = 1, 2, 3, 4).
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Register:    0xB1 bit [3] (CH1), bit [2] (CH2), bit [1] (CH3), bit [0] (CH4)
+:Note:        When enabled and DC diagnostics report OL, the device tests for
+              a high-impedance line output load.
+
+DC LDG SLOL Ramp Time
+~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Voltage ramp time for shorted-load and open-load detection.
+:Type:        Enumerated
+:Default:     15 ms
+:Options:     15 ms, 30 ms, 10 ms, 20 ms
+:Register:    0xB2 bits [7:6]
+
+DC LDG SLOL Settling Time
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Settling time for shorted-load and open-load detection.
+:Type:        Enumerated
+:Default:     10 ms
+:Options:     10 ms, 5 ms, 20 ms, 15 ms
+:Register:    0xB2 bits [5:4]
+
+DC LDG S2PG Ramp Time
+~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Voltage ramp time for short-to-power and short-to-ground detection.
+:Type:        Enumerated
+:Default:     5 ms
+:Options:     5 ms, 2.5 ms, 10 ms, 15 ms
+:Register:    0xB2 bits [3:2]
+
+DC LDG S2PG Settling Time
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Settling time for short-to-power and short-to-ground detection.
+:Type:        Enumerated
+:Default:     10 ms
+:Options:     10 ms, 5 ms, 20 ms, 30 ms
+:Register:    0xB2 bits [1:0]
+
+CHx DC LDG SL Threshold
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Shorted-load detection threshold per channel (x = 1, 2, 3, 4).
+:Type:        Enumerated
+:Default:     1 Ohm
+:Options:     0.5 Ohm, 1 Ohm, 1.5 Ohm, 2 Ohm, 2.5 Ohm,
+              3 Ohm, 3.5 Ohm, 4 Ohm, 4.5 Ohm, 5 Ohm
+:Register:    0xB3 bits [7:4] (CH1), bits [3:0] (CH2),
+              0xB4 bits [7:4] (CH3), bits [3:0] (CH4)
+
+DC LDG Result
+~~~~~~~~~~~~~
+
+:Description: Overall DC diagnostic result register.
+:Type:        Integer (read-only)
+:Range:       0x00 to 0xFF
+:Register:    0xC2
+:Bit Encoding:
+
+  ========  =====================================================
+  Bits      Description
+  ========  =====================================================
+  [7:4]     Line output detection result, one bit per channel
+  [3:0]     DC diagnostic pass/fail per channel (1=pass, 0=fail)
+  ========  =====================================================
+
+CHx DC LDG Report
+~~~~~~~~~~~~~~~~~
+
+:Description: DC diagnostic fault status per channel (x = 1, 2, 3, 4).
+:Type:        Integer (read-only)
+:Range:       0x0 to 0xF
+:Register:    0xC0 bits [7:4] (CH1), bits [3:0] (CH2),
+              0xC1 bits [7:4] (CH3), bits [3:0] (CH4)
+:Note:        See fault bit encoding table at the start of this section.
+
+CHx LO LDG Report
+~~~~~~~~~~~~~~~~~
+
+:Description: Line output load detection result per channel (x = 1, 2, 3, 4).
+:Type:        Boolean (read-only)
+:Values:      0 = not detected, 1 = line output load detected
+:Register:    0xC2 bit [7] (CH1), bit [6] (CH2), bit [5] (CH3), bit [4] (CH4)
+
+CHx DC Resistance
+~~~~~~~~~~~~~~~~~
+
+:Description: Measured DC load resistance per channel (x = 1, 2, 3, 4).
+:Type:        Float (read-only, displayed in ohms)
+:Resolution:  0.1 ohm per code (10-bit value)
+:Range:       0.0 to 102.3 ohms
+:Register:    0xD9 bits [7:6]/[5:4]/[3:2]/[1:0] (MSB for CH1–CH4),
+              0xDA (CH1 LSB), 0xDB (CH2 LSB), 0xDC (CH3 LSB), 0xDD (CH4 LSB)
+
+AC Load Diagnostics
+-------------------
+
+AC LDG Trigger
+~~~~~~~~~~~~~~
+
+:Description: Triggers AC impedance measurement on all channels.
+:Type:        Boolean (write-only)
+:Note:        Returns -EBUSY if any DAI stream (playback or capture) is active.
+              The driver transitions all channels to SLEEP state before starting
+              the measurement. Blocks until diagnostics complete or time out.
+
+AC LDG Gain
+~~~~~~~~~~~
+
+:Description: Measurement resolution for AC diagnostics.
+:Type:        Boolean Switch
+:Default:     1 (Gain 8)
+:Values:      0 = 0.8 ohm/code (Gain 1), 1 = 0.1 ohm/code (Gain 8)
+:Register:    0xB5 bit [4]
+:Note:        Gain 8 recommended for load impedances below 8 ohms.
+
+AC LDG Test Frequency
+~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Test signal frequency for AC impedance measurement.
+:Type:        Integer
+:Default:     200 (0xC8 = 18.75 kHz)
+:Range:       0x01 to 0xFF (0x00 reserved)
+:Formula:     Frequency = 93.75 Hz × register value
+:Register:    0xB8
+
+CHx AC LDG Real / CHx AC LDG Imag
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Real and imaginary AC impedance components per channel
+              (x = 1, 2, 3, 4).
+:Type:        Integer (read-only)
+:Range:       0x00 to 0xFF (8-bit signed)
+:Register:    0xC3 (CH1 Real), 0xC4 (CH1 Imag), 0xC5 (CH2 Real), 0xC6 (CH2 Imag),
+              0xC7 (CH3 Real), 0xC8 (CH3 Imag), 0xC9 (CH4 Real), 0xCA (CH4 Imag)
+:Note:        Scale set by AC LDG Gain.
+
+Speaker Protection & Detection
+-------------------------------
+
+Tweeter Detection Switch
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Enables tweeter detection using the AC impedance magnitude comparator.
+:Type:        Boolean Switch
+:Default:     Enabled (1)
+:Register:    0xB6 bit [0]
+:Note:        The underlying register bit is TWEETER DETECT DISABLE (active-low).
+              Control value 1 = detection enabled (register bit 0), 0 = disabled
+              (register bit 1).
+
+Tweeter Detect Threshold
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Magnitude threshold for tweeter detection.
+:Type:        Integer
+:Default:     0
+:Range:       0x00 to 0xFF
+:Resolution:  0.8 ohm/code (AC LDG Gain=0) or 0.1 ohm/code (AC LDG Gain=1)
+:Register:    0xB7
+
+CHx Tweeter Detect Report
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+:Description: Tweeter detection result per channel (x = 1, 2, 3, 4).
+:Type:        Boolean (read-only)
+:Values:      0 = no tweeter detected, 1 = tweeter detected
+:Register:    0xCB bit [3] (CH1), bit [2] (CH2), bit [1] (CH3), bit [0] (CH4)
+
+DSP Protection Features
+=======================
+
+These controls are unavailable in LLP mode.
+
+Thermal Foldback Switch
+-----------------------
+
+:Description: Enables dynamic gain reduction based on die temperature.
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Register:    0x3A bit [0]
+
+PVDD Foldback Switch
+--------------------
+
+:Description: Enables automatic gain limiting when supply voltage drops
+              (Automatic Gain Limiter).
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Register:    0x3A bit [4]
+
+DC Blocker Bypass Switch
+------------------------
+
+:Description: Bypasses the DC-blocking high-pass filter.
+:Type:        Boolean Switch
+:Default:     Not bypassed (0)
+:Register:    0x39 bit [0]
+
+Clip Detect Switch
+------------------
+
+:Description: Enables DSP-based clip detection (Pseudo-Analog Clip Detect).
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Register:    0x93 bit [6]
+
+Audio SDOUT Switch
+------------------
+
+:Description: Routes post-processed audio to the SDOUT pin instead of
+              Vpredict data.
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Register:    0x3A bit [5]
+:Note:        When enabled, replaces Vpredict data on SDOUT with post-processed
+              SDIN. All SDOUT configurations that apply to Vpredict also apply
+              to SDIN-to-SDOUT transmission.
+
+Real-Time Load Diagnostics
+===========================
+
+These controls require Normal DSP mode at 48 kHz or 96 kHz. They are
+unavailable at 192 kHz and in FFLP and LLP modes.
+
+
+CHx RTLDG Switch
+----------------
+
+:Description: Enables real-time impedance monitoring during playback
+              (x = 1, 2, 3, 4).
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Register:    0x37 bit [3] (CH1), bit [2] (CH2), bit [1] (CH3), bit [0] (CH4)
+
+RTLDG Clip Mask Switch
+----------------------
+
+:Description: Suppresses impedance updates during clipping events.
+:Type:        Boolean Switch
+:Default:     Enabled (1)
+:Register:    0x37 bit [4]
+
+ISENSE Calibration Switch
+--------------------------
+
+:Description: Enables current sense calibration for accurate impedance
+              measurements.
+:Type:        Boolean Switch
+:Default:     Disabled (0)
+:Register:    0x5B bit [3]
+
+RTLDG Open Load Threshold
+--------------------------
+
+:Description: DSP coefficient for open load fault detection threshold.
+:Type:        DSP coefficient (extended control)
+:Register:    DSP Book 0x8C, page 0x22, 0x98
+
+RTLDG Short Load Threshold
+---------------------------
+
+:Description: DSP coefficient for shorted load fault detection threshold.
+:Type:        DSP coefficient (extended control)
+:Register:    DSP Book 0x8C, page 0x22, 0x9C
+
+CHx RTLDG Impedance
+-------------------
+
+:Description: Real-time load impedance per channel (x = 1, 2, 3, 4).
+:Type:        Float (read-only, displayed in ohms)
+:Register:    0xD1–0xD2 (CH1), 0xD3–0xD4 (CH2), 0xD5–0xD6 (CH3), 0xD7–0xD8 (CH4)
+:Note:        Valid only during PLAY state with RTLDG enabled at 48 or
+              96 kHz. Holds stale data in SLEEP, MUTE, or Hi-Z states.
+
+Fault Monitoring
+================
+
+The driver monitors faults via IRQ or periodic polling (fallback when
+no ``interrupts`` DT property is present). Detected faults are logged
+and cleared so affected channels can recover.
+
+Kernel log messages use the format::
+
+  tas675x <addr>: <Name> Latched: 0x<value>
+
+Critical faults are logged at ``CRIT`` level; warnings at ``WARN``.
+
+The following fault registers are monitored:
+
+Critical Faults
+---------------
+
+These faults place affected channels into the FAULT state. The driver
+issues fault clear (register 0x01 bit 3) to allow recovery.
+
+Overtemperature Shutdown (0x87)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  ========  ==========================================
+  Bits      Description
+  ========  ==========================================
+  [4]       Global OTSD
+  [3:0]     CH1-CH4 OTSD respectively
+  ========  ==========================================
+
+Fires when die temperature exceeds the OTSD threshold. If auto-recovery
+is enabled (``OTSD Auto Recovery Switch``), channels return to their
+previous state after cooling. Otherwise fault clear is required.
+
+Overcurrent / DC Fault (0x8E)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  ========  ==========================================
+  Bits      Description
+  ========  ==========================================
+  [7:4]     Overcurrent Shutdown, CH1-CH4 respectively
+  [3:0]     DC Fault, CH1-CH4 respectively
+  ========  ==========================================
+
+Overcurrent shutdown occurs when output current reaches the shutdown
+threshold (e.g. output short to GND). DC fault fires when DC offset
+exceeds the DCFAULT threshold.
+
+Real-Time Load Diagnostic Fault (0x8B)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  ========  ==========================================
+  Bits      Description
+  ========  ==========================================
+  [7:4]     Shorted Load faults, CH1-CH4 respectively
+  [3:0]     Open Load faults, CH1-CH4 respectively
+  ========  ==========================================
+
+Reported during playback when RTLDG detects impedance outside the
+configured open/short thresholds. Requires RTLDG and current sense to
+be enabled, and a sampling rate of 48 kHz or 96 kHz.
+
+CBC Fault/Warning (0x8D)
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+  ========  ==========================================
+  Bits      Description
+  ========  ==========================================
+  [7:4]     Load current warning, CH1-CH4 respectively
+  [3:0]     Load current fault, CH1-CH4 respectively
+  ========  ==========================================
+
+Cycle-by-cycle current limiting events. A warning fires when the
+current limit is active for at least 25% of a 21.3 ms window. A fault
+fires if the warning persists for 170.4 ms continuously.
+
+Warnings
+--------
+
+These conditions are logged but do not place channels into FAULT state.
+Corresponding register bits are cleared when read. Nonetheless, the driver
+still clears all events as warnings also latch the (mapped) FAULT pin.
+
+Power Fault (0x86)
+~~~~~~~~~~~~~~~~~~
+
+  ========  ==========================================
+  Bits      Description
+  ========  ==========================================
+  [7]       DVDD Power-On-Reset
+  [4]       DVDD Undervoltage
+  [3]       PVDD Overvoltage
+  [2]       VBAT Overvoltage
+  [1]       PVDD Undervoltage
+  [0]       VBAT Undervoltage
+  ========  ==========================================
+
+Supply-related events. DVDD POR at bit 7 is expected after every
+power-up and can be ignored.
+
+CP / OUTM Soft Short Fault (0x7D)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  ========  ==========================================
+  Bits      Description
+  ========  ==========================================
+  [5:2]     OUTM soft short, CH4-CH1 respectively
+  [1]       Charge Pump UVLO (latched)
+  [0]       Charge Pump UVLO (unlatched/live status)
+  ========  ==========================================
+
+Per-channel out_minus soft short faults and charge pump undervoltage lockout.
+
+Clock Fault (0x8A)
+~~~~~~~~~~~~~~~~~~
+
+  ========  ==========================================
+  Bits      Description
+  ========  ==========================================
+  [0]       Clock error
+  ========  ==========================================
+
+Fires on clock halt or unsupported SCLK/FSYNC ratio. Only logged when
+a stream is active (suppressed during idle to avoid noise from normal
+stream stop/start transitions). See `Clock Fault Behaviour`_ below.
+
+Overtemperature Warning (0x88)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  ========  ==========================================
+  Bits      Description
+  ========  ==========================================
+  [4]       Global OTW
+  [3:0]     CH1-CH4 OTW respectively
+  ========  ==========================================
+
+Fires when temperature crosses the OTW threshold. The device continues
+to operate. Only logged on change to avoid log spam.
+
+Clip Warning (0x89)
+~~~~~~~~~~~~~~~~~~~
+
+  ========  ==========================================
+  Bits      Description
+  ========  ==========================================
+  [3:0]     CH1-CH4 clip detected respectively
+  ========  ==========================================
+
+Fires when the audio signal magnitude exceeds the clip detection
+threshold. Requires Clip Detect to be enabled. Only logged when a
+stream is active.
+
+Driver Known Limitations
+========================
+
+Clock Fault Behaviour
+---------------------
+
+On Stream Stop
+~~~~~~~~~~~~~~
+
+Every time a playback stream stops the FAULT pin briefly asserts. The
+ASoC PCM trigger stop sequence calls DAIs in reverse order, so the CPU
+DAI stops SCLK before the codec can transition to sleep. The device
+detects the clock halt and latches ``CLK_FAULT_LATCHED``, which asserts
+the FAULT pin. The driver clears the latch in the ``mute_stream``
+callback that follows, so the FAULT pin flicker lasts only a few
+milliseconds. Audio output is not affected and no kernel log message
+is produced.
+
+On Rapid Rate Switching
+~~~~~~~~~~~~~~~~~~~~~~~
+
+When streams are started in rapid succession, an intermittent
+``Clock Fault Latched: 0x01`` message may appear in the kernel log.
+A (conservative) ~0.5 second gap between sessions eliminates this.
+
+References
+==========
+
+- TAS675x Technical Reference Manual: SLOU589A
+- Device Tree Bindings: Documentation/devicetree/bindings/sound/ti,tas67524.yaml
+- ALSA Control Name Conventions: Documentation/sound/designs/control-names.rst
diff --git a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst
index 895752cbcedd6..86b64de031402 100644
--- a/Documentation/sound/kernel-api/writing-an-alsa-driver.rst
+++ b/Documentation/sound/kernel-api/writing-an-alsa-driver.rst
@@ -266,7 +266,7 @@ to details explained in the following section.
               ....
 
               /* allocate a chip-specific data with zero filled */
-              chip = kzalloc(sizeof(*chip), GFP_KERNEL);
+              chip = kzalloc_obj(*chip);
               if (chip == NULL)
                       return -ENOMEM;
 
@@ -628,7 +628,7 @@ After allocating a card instance via :c:func:`snd_card_new()`
   err = snd_card_new(&pci->dev, index[dev], id[dev], THIS_MODULE,
                      0, &card);
   .....
-  chip = kzalloc(sizeof(*chip), GFP_KERNEL);
+  chip = kzalloc_obj(*chip);
 
 The chip record should have the field to hold the card pointer at least,
 
@@ -747,7 +747,7 @@ destructor and PCI entries. Example code is shown first, below::
                       return -ENXIO;
               }
 
-              chip = kzalloc(sizeof(*chip), GFP_KERNEL);
+              chip = kzalloc_obj(*chip);
               if (chip == NULL) {
                       pci_disable_device(pci);
                       return -ENOMEM;
@@ -1737,7 +1737,7 @@ callback::
   {
           struct my_pcm_data *data;
           ....
-          data = kmalloc(sizeof(*data), GFP_KERNEL);
+          data = kmalloc_obj(*data);
           substream->runtime->private_data = data;
           ....
   }
@@ -3301,7 +3301,7 @@ You can then pass any pointer value to the ``private_data``. If you
 assign private data, you should define a destructor, too. The
 destructor function is set in the ``private_free`` field::
 
-  struct mydata *p = kmalloc(sizeof(*p), GFP_KERNEL);
+  struct mydata *p = kmalloc_obj(*p);
   hw->private_data = p;
   hw->private_free = mydata_free;
 
@@ -3833,7 +3833,7 @@ chip data individually::
           err = snd_card_new(&pci->dev, index[dev], id[dev], THIS_MODULE,
                              0, &card);
           ....
-          chip = kzalloc(sizeof(*chip), GFP_KERNEL);
+          chip = kzalloc_obj(*chip);
           ....
           card->private_data = chip;
           ....
diff --git a/Documentation/sound/soc/index.rst b/Documentation/sound/soc/index.rst
index 8bed8f8f48dae..22878d73f7a54 100644
--- a/Documentation/sound/soc/index.rst
+++ b/Documentation/sound/soc/index.rst
@@ -2,7 +2,7 @@
 ALSA SoC Layer
 ==============
 
-The documentation is spilt into the following sections:-
+The documentation is split into the following sections:-
 
 .. toctree::
    :maxdepth: 2
diff --git a/Documentation/sound/soc/platform.rst b/Documentation/sound/soc/platform.rst
index bd21d0a4dd9b0..53caba6ce4cea 100644
--- a/Documentation/sound/soc/platform.rst
+++ b/Documentation/sound/soc/platform.rst
@@ -75,4 +75,4 @@ Each SoC DSP driver usually supplies the following features :-
 3. DMA IO to/from DSP buffers (if applicable)
 4. Definition of DSP front end (FE) PCM devices.
 
-Please see DPCM.txt for a description of item 4.
+Please see dpcm.rst for a description of item 4.
diff --git a/Documentation/sound/soc/usb.rst b/Documentation/sound/soc/usb.rst
index 94c12f9d9dd12..9cb860547f22d 100644
--- a/Documentation/sound/soc/usb.rst
+++ b/Documentation/sound/soc/usb.rst
@@ -103,7 +103,7 @@ Returns 0 on success, and -EOPNOTSUPP on failure.
   - ``usbdev``: the usb device that was discovered
   - ``sdev``: capabilities of the device
 
-**snd_soc_usb_connect()** notifies the ASoC USB DCPM BE DAI link of a USB
+**snd_soc_usb_connect()** notifies the ASoC USB DPCM BE DAI link of a USB
 audio device detection.  This can be utilized in the BE DAI
 driver to keep track of available USB audio devices.  This is intended
 to be called by the USB offload driver residing in USB SND.
@@ -118,7 +118,7 @@ Returns 0 on success, negative error code on failure.
   - ``usbdev``: the usb device that was removed
   - ``sdev``: capabilities to free
 
-**snd_soc_usb_disconnect()** notifies the ASoC USB DCPM BE DAI link of a USB
+**snd_soc_usb_disconnect()** notifies the ASoC USB DPCM BE DAI link of a USB
 audio device removal.  This is intended to be called by the USB offload
 driver that resides in USB SND.
 
diff --git a/Documentation/sphinx-static/custom.css b/Documentation/sphinx-static/custom.css
index f91393426a506..5aa0a1ed9864c 100644
--- a/Documentation/sphinx-static/custom.css
+++ b/Documentation/sphinx-static/custom.css
@@ -30,7 +30,7 @@ img.logo {
     margin-bottom: 20px;
 }
 
-/* The default is to use -1em, wich makes it override text */
+/* The default is to use -1em, which makes it override text */
 li { text-indent: 0em; }
 
 /*
diff --git a/Documentation/sphinx/maintainers_include.py b/Documentation/sphinx/maintainers_include.py
index 519ad18685b23..be8e566e03635 100755
--- a/Documentation/sphinx/maintainers_include.py
+++ b/Documentation/sphinx/maintainers_include.py
@@ -1,197 +1,410 @@
 #!/usr/bin/env python
 # SPDX-License-Identifier: GPL-2.0
 # -*- coding: utf-8; mode: python -*-
-# pylint: disable=R0903, C0330, R0914, R0912, E0401
+# pylint: disable=C0209, C0301, E0401, R0022, R0902, R0903, R0912, R0914
 
 """
-    maintainers-include
-    ~~~~~~~~~~~~~~~~~~~
+Implementation of the ``maintainers-include`` reST-directive.
 
-    Implementation of the ``maintainers-include`` reST-directive.
+:copyright:  Copyright (C) 2019  Kees Cook <keescook@chromium.org>
+:license:    GPL Version 2, June 1991 see linux/COPYING for details.
 
-    :copyright:  Copyright (C) 2019  Kees Cook <keescook@chromium.org>
-    :license:    GPL Version 2, June 1991 see linux/COPYING for details.
-
-    The ``maintainers-include`` reST-directive performs extensive parsing
-    specific to the Linux kernel's standard "MAINTAINERS" file, in an
-    effort to avoid needing to heavily mark up the original plain text.
+The ``maintainers-include`` reST-directive performs extensive parsing
+specific to the Linux kernel's standard "MAINTAINERS" file, in an
+effort to avoid needing to heavily mark up the original plain text.
 """
 
-import sys
-import re
 import os.path
+import re
+
+from glob import glob
 
 from docutils import statemachine
-from docutils.parsers.rst import Directive
 from docutils.parsers.rst.directives.misc import Include
 
-def ErrorString(exc):  # Shamelessly stolen from docutils
-    return f'{exc.__class__.__name}: {exc}'
-
-__version__  = '1.0'
+#
+# Base URL for intersphinx-like links to maintainer profiles
+#
+KERNELDOC_URL = "https://docs.kernel.org/"
+
+__version__ = "1.0"
+
+maint_parser = None  # pylint: disable=C0103
+
+JS_FILTER = """
+(function() {
+  function filterTable(table) {
+    const filter = document.getElementById("filter-table").value.trim();
+    const rows = table.querySelectorAll("tbody tr");
+    for (let i = 0; i < rows.length; i++) {
+      const tds = rows[i].getElementsByTagName("td");
+      let match = false;
+      for (let j = 0; j < tds.length; j++) {
+        const cellText = (tds[j].textContent || tds[j].innerText);
+        if (cellText.includes(filter)) {
+          match = true;
+          break;
+        }
+      }
+      rows[i].style.display = match ? "table-row" : "none";
+    }
+  }
+  function addInput() {
+    const table = document.getElementById("maintainers-table");
+    if (!table) return;
+    let input = document.getElementById("filter-table");
+    if (!input) {
+      const filt_div = document.createElement('div');
+      filt_div.innerHTML = `
+        <p>Filter:
+          <input type="search" id="filter-table" placeholder="search string"/>
+          subsystem or property (case-sensitive)
+        </p>
+      `;
+      table.parentNode.insertBefore(filt_div, table);
+      const input = document.getElementById("filter-table")
+      input.addEventListener('input', () => filterTable(table));
+    }
+  }
+  if (document.readyState === 'loading') {
+    document.addEventListener('DOMContentLoaded', addInput);
+  } else {
+    addInput();
+  }
+})();
+"""
 
-def setup(app):
-    app.add_directive("maintainers-include", MaintainersInclude)
-    return dict(
-        version = __version__,
-        parallel_read_safe = True,
-        parallel_write_safe = True
-    )
 
-class MaintainersInclude(Include):
-    """MaintainersInclude (``maintainers-include``) directive"""
-    required_arguments = 0
+# Shamelessly stolen from docutils
+def ErrorString(exc):  # pylint: disable=C0103, C0116
+    return f"{exc.__class__.__name}: {exc}"  # pylint: disable=W0212
 
-    def parse_maintainers(self, path):
-        """Parse all the MAINTAINERS lines into ReST for human-readability"""
+class MaintainersParser:
+    """Parse MAINTAINERS file(s) content"""
 
-        result = list()
-        result.append(".. _maintainers:")
-        result.append("")
+    def __init__(self, base_dir, app_dir, path):
+        self.path = path
 
         # Poor man's state machine.
-        descriptions = False
-        maintainers = False
-        subsystems = False
+        self.descriptions = False
+        self.maintainers = False
+        self.subsystems = False
 
-        # Field letter to field name mapping.
-        field_letter = None
-        fields = dict()
+        self.subsystem_name = None
+
+        self.base_dir = base_dir
+        self.app_dir = app_dir
+
+        self.re_doc = re.compile(r'(Documentation/(\S*)\.rst)')
+
+        #
+        # Output variables with maintainers content to be stored
+        #
+        self.profile_toc = set()
+        self.profile_entries = {}
+        self.header = ""
+        self.maint_entries = {}
+        self.fields = {}
 
         prev = None
-        field_prev = ""
-        field_content = ""
-
-        for line in open(path):
-            # Have we reached the end of the preformatted Descriptions text?
-            if descriptions and line.startswith('Maintainers'):
-                descriptions = False
-                # Ensure a blank line following the last "|"-prefixed line.
-                result.append("")
-
-            # Start subsystem processing? This is to skip processing the text
-            # between the Maintainers heading and the first subsystem name.
-            if maintainers and not subsystems:
-                if re.search('^[A-Z0-9]', line):
-                    subsystems = True
-
-            # Drop needless input whitespace.
-            line = line.rstrip()
-
-            # Linkify all non-wildcard refs to ReST files in Documentation/.
-            pat = r'(Documentation/([^\s\?\*]*)\.rst)'
-            m = re.search(pat, line)
+        with open(path, "r", encoding="utf-8") as fp:
+            for line in fp:
+                if self.descriptions:
+                    self.parse_descriptions(line)
+                elif self.maintainers and not self.subsystems:
+                    if re.search('^[A-Z0-9]', line):
+                        self.subsystems = True
+                        self.parse_subsystems(line)
+                    else:
+                        self.header += line
+                elif self.subsystems:
+                    self.parse_subsystems(line)
+                else:
+                    self.header += line
+
+                # Update the state machine when we find heading separators.
+                if line.startswith("----------"):
+                    if prev.startswith("Descriptions"):
+                        self.descriptions = True
+                    if prev.startswith("Maintainers"):
+                        self.maintainers = True
+
+                # Retain previous line for state machine transitions.
+                prev = line
+
+    def get_entries(self, text):
+        """Generate refs to ReST files in Documentation/"""
+
+        if "Documentation/" not in text:
+            return None
+
+        if "*" in text or "?" in text:
+            m = self.re_doc.search(text)
+            if not m:
+                return None
+
+            doc_list = glob(m.group(1), root_dir=self.base_dir)
+        else:
+            doc_list = [text]
+
+        entries = {}
+        for doc in doc_list:
+            m = self.re_doc.search(doc)
             if m:
-                # maintainers.rst is in a subdirectory, so include "../".
-                line = re.sub(pat, ':doc:`%s <../%s>`' % (m.group(2), m.group(2)), line)
-
-            # Check state machine for output rendering behavior.
-            output = None
-            if descriptions:
-                # Escape the escapes in preformatted text.
-                output = "| %s" % (line.replace("\\", "\\\\"))
-                # Look for and record field letter to field name mappings:
-                #   R: Designated *reviewer*: FullName <address@domain>
-                m = re.search(r"\s(\S):\s", line)
-                if m:
-                    field_letter = m.group(1)
-                if field_letter and not field_letter in fields:
-                    m = re.search(r"\*([^\*]+)\*", line)
-                    if m:
-                        fields[field_letter] = m.group(1)
-            elif subsystems:
-                # Skip empty lines: subsystem parser adds them as needed.
-                if len(line) == 0:
-                    continue
-                # Subsystem fields are batched into "field_content"
-                if line[1] != ':':
-                    # Render a subsystem entry as:
-                    #   SUBSYSTEM NAME
-                    #   ~~~~~~~~~~~~~~
-
-                    # Flush pending field content.
-                    output = field_content + "\n\n"
-                    field_content = ""
-
-                    # Collapse whitespace in subsystem name.
-                    heading = re.sub(r"\s+", " ", line)
-                    output = output + "%s\n%s" % (heading, "~" * len(heading))
-                    field_prev = ""
+                fname = m.group(1)
+                ename = m.group(2)
+
+                entry = os.path.relpath(self.base_dir + fname, self.app_dir)
+                entry = entry.removesuffix(".rst")
+
+                if entry.startswith("../"):
+                    html = KERNELDOC_URL + ename + ".html"
+                    entries[entry] = f'`{ename} <{html}>`_'
                 else:
-                    # Render a subsystem field as:
-                    #   :Field: entry
-                    #           entry...
-                    field, details = line.split(':', 1)
-                    details = details.strip()
-
-                    # Mark paths (and regexes) as literal text for improved
-                    # readability and to escape any escapes.
-                    if field in ['F', 'N', 'X', 'K']:
-                        # But only if not already marked :)
-                        if not ':doc:' in details:
-                            details = '``%s``' % (details)
-
-                    # Comma separate email field continuations.
-                    if field == field_prev and field_prev in ['M', 'R', 'L']:
-                        field_content = field_content + ","
-
-                    # Do not repeat field names, so that field entries
-                    # will be collapsed together.
-                    if field != field_prev:
-                        output = field_content + "\n"
-                        field_content = ":%s:" % (fields.get(field, field))
-                    field_content = field_content + "\n\t%s" % (details)
-                    field_prev = field
+                    entries[entry] = f':doc:`{ename} </{entry}>`'
+
+        return entries
+
+    def linkify(self, text):
+        """Return a list of doc files converted to cross-references"""
+
+        entries = self.get_entries(text)
+        if not entries:
+            return text
+
+        return self.re_doc.sub(", ".join(entries.values()), text)
+
+    def parse_descriptions(self, line):
+        """Handle contents of the descriptions section."""
+
+        # Have we reached the end of the preformatted Descriptions text?
+        if line.startswith("Maintainers"):
+            self.descriptions = False
+            self.header += "\n" + line
+            return
+
+        # Look for and record field letter to field name mappings:
+        #   R: Designated *reviewer*: FullName <address@domain>
+        m = re.match(r"\s+(\S):\s+(\S+)", line)
+        if m:
+            field = m.group(1)
+            details = m.group(2)
+
+            if field not in self.fields:
+                m = re.search(r"\*([^\*]+)\*", line)
+                if m:
+                    self.fields[field] = m.group(1)
+            elif field in ['F', 'N', 'X', 'K']:
+                line = line.replace(details, f'``{details}``')
+
+        self.header += "| " + self.linkify(line)
+
+
+    def parse_subsystems(self, line):
+        """Handle contents of the per-subsystem sections."""
+
+        # Drop needless input whitespace.
+        line = line.rstrip()
+
+        # Skip empty lines: subsystem parser adds them as needed.
+        if not line:
+            return
+
+        if line[1] != ':':
+            self.subsystem_name = re.sub(r"\s+", " ", self.linkify(line))
+            return
+
+        # Render a subsystem field as:
+        #   :Field: entry
+        #           entry...
+        field, details = line.split(":", 1)
+        details = details.strip()
+
+        #
+        # Handle profile entries - either as files or as https refs
+        #
+        if field == "P":
+            entries = self.get_entries(details)
+            if entries:
+                for e, link in entries.items():
+                    if "html" not in link:
+                        self.profile_toc.add(e)
+
+                    self.profile_entries[self.subsystem_name] = link
+
+                details = ", ".join(entries.values())
             else:
-                output = line
+                match = re.match(r"(https?://.*)", details)
+                if match:
+                    entry = match.group(1).strip()
+                    self.profile_entries[self.subsystem_name] = entry
+                else:
+                    self.profile_entries[self.subsystem_name] = f"``{details}``"
+
+                details = self.linkify(details)
+        else:
+            details = self.linkify(details)
+
+        #
+        # Mark paths (and regexes) as literal text for improved
+        # readability and to escape any escapes.
+        #
+        if field in ['F', 'N', 'X', 'K']:
+            # But only if not already marked :)
+            if ':doc:' not in details and "http" not in details:
+                details = '``%s``' % (details)
 
-            # Re-split on any added newlines in any above parsing.
-            if output != None:
-                for separated in output.split('\n'):
-                    result.append(separated)
+        if self.subsystem_name not in self.maint_entries:
+            self.maint_entries[self.subsystem_name] = {}
 
-            # Update the state machine when we find heading separators.
-            if line.startswith('----------'):
-                if prev.startswith('Descriptions'):
-                    descriptions = True
-                if prev.startswith('Maintainers'):
-                    maintainers = True
+        if field not in self.maint_entries[self.subsystem_name]:
+            self.maint_entries[self.subsystem_name][field] = []
 
-            # Retain previous line for state machine transitions.
-            prev = line
+        self.maint_entries[self.subsystem_name][field].append(details)
 
-        # Flush pending field contents.
-        if field_content != "":
-            for separated in field_content.split('\n'):
-                result.append(separated)
+        self.field_prev = field
+
+
+class MaintainersInclude(Include):
+    """MaintainersInclude (``maintainers-include``) directive"""
+
+    required_arguments = 0
+
+    def emit(self):
+        """Parse all the MAINTAINERS lines into ReST for human-readability"""
+        path = maint_parser.path
+        output = ".. _maintainers:\n\n"
+        output += maint_parser.header
+
+        output += ".. _maintainers_table:\n\n"
+        output += ".. flat-table::\n"
+        output += "  :header-rows: 1\n\n"
+        output += "  * - Subsystem\n"
+        output += "    - Properties\n\n"
+
+        self.state.document['maintainers_included'] = True
+
+        for name, fields in sorted(maint_parser.maint_entries.items()):
+            output += f"  * - {name}\n"
+            tag = "-"
+            for field, lines in fields.items():
+                field_name = maint_parser.fields.get(field, field)
+
+                output += f"    {tag} :{field_name}:\n        "
+                output += ",\n        ".join(lines) + "\n"
+                tag = " "
+
+            output += "\n"
 
-        output = "\n".join(result)
         # For debugging the pre-rendered results...
         #print(output, file=open("/tmp/MAINTAINERS.rst", "w"))
 
-        self.state_machine.insert_input(
-          statemachine.string2lines(output), path)
+        self.state.document.settings.record_dependencies.add(path)
+        self.state_machine.insert_input(statemachine.string2lines(output), path)
 
     def run(self):
         """Include the MAINTAINERS file as part of this reST file."""
         if not self.state.document.settings.file_insertion_enabled:
             raise self.warning('"%s" directive disabled.' % self.name)
 
-        # Walk up source path directories to find Documentation/../
-        path = self.state_machine.document.attributes['source']
-        path = os.path.realpath(path)
-        tail = path
-        while tail != "Documentation" and tail != "":
-            (path, tail) = os.path.split(path)
+        try:
+            self.emit()
+        except IOError as error:
+            raise self.severe('Problems with "%s" directive path:\n%s.' %
+                      (self.name, ErrorString(error)))
+
+        return []
+
+
+class MaintainersProfile(Include):
+    """Generate a list with all maintainer's profiles"""
+
+    required_arguments = 0
+
+    def emit(self):
+        """Parse all the MAINTAINERS lines looking for profile entries"""
+        env = self.state.document.settings.env
+        docdir = os.path.dirname(os.path.join(env.srcdir, env.docname))
+        path = maint_parser.path
+
+        #
+        # Produce a list with all maintainer profiles, sorted by subsystem name
+        #
+        output = ""
+        for profile, entry in sorted(maint_parser.profile_entries.items()):
+            name = profile.title()
+
+            if entry.startswith("http"):
+                output += f"- `{name} <{entry}>`_\n"
+            elif entry.startswith("`"):
+                output += f"- {name}: {entry}\n"
+                self.warning(f"{profile}: Invalid 'P' tag: {entry}\n")
+            else:
+                output += f"- {entry}\n"
+
+        #
+        # Create a hidden TOC table with all profiles. That allows adding
+        # profiles without needing to add them on any index.rst file.
+        #
+        output += "\n.. toctree::\n"
+        output += "   :hidden:\n\n"
+
+        for f in sorted(maint_parser.profile_toc):
+            fname = os.path.join(maint_parser.base_dir, "Documentation", f)
+            fname = os.path.relpath(fname, docdir)
+            output += f"   {fname}\n"
+
+        output += "\n"
+
+        # For debugging the pre-rendered results...
+        #print(output, file=open("/tmp/profiles.rst", "w"))
 
-        # Append "MAINTAINERS"
-        path = os.path.join(path, "MAINTAINERS")
+        self.state.document.settings.record_dependencies.add(path)
+        self.state_machine.insert_input(statemachine.string2lines(output), path)
+
+    def run(self):
+        """Include the MAINTAINERS file as part of this reST file."""
+        if not self.state.document.settings.file_insertion_enabled:
+            raise self.warning('"%s" directive disabled.' % self.name)
 
         try:
-            self.state.document.settings.record_dependencies.add(path)
-            lines = self.parse_maintainers(path)
+            self.emit()
         except IOError as error:
             raise self.severe('Problems with "%s" directive path:\n%s.' %
                       (self.name, ErrorString(error)))
 
         return []
+
+
+# pylint: disable=W0613
+def add_filter_script(app, pagename, templatename, context, doctree):
+    """Add Filter javascript only to maintainers page"""
+
+    if doctree and doctree.get('maintainers_included'):
+        app.add_js_file(None, body=JS_FILTER)
+
+
+def setup(app):
+    """Setup Sphinx extension"""
+    global maint_parser  # pylint: disable=W0603
+
+    app_dir = os.path.abspath(app.srcdir)
+    match = re.match(r"(.*/)Documentation", app_dir)
+    if not match:
+        raise ValueError('Documentation directory not found.')
+
+    base_dir = match.group(1)
+    path = os.path.join(base_dir, "MAINTAINERS")
+
+    maint_parser = MaintainersParser(base_dir, app_dir, path)
+
+    app.add_directive("maintainers-include", MaintainersInclude)
+    app.add_directive("maintainers-profile-toc", MaintainersProfile)
+
+    app.connect("html-page-context", add_filter_script)
+
+    return {
+        "version": __version__,
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }
diff --git a/Documentation/spi/spi-summary.rst b/Documentation/spi/spi-summary.rst
index 6e21e6f86912b..ee34796cfbb64 100644
--- a/Documentation/spi/spi-summary.rst
+++ b/Documentation/spi/spi-summary.rst
@@ -249,7 +249,7 @@ And SOC-specific utility code might look something like::
 	{
 		struct mysoc_spi_data *pdata2;
 
-		pdata2 = kmalloc(sizeof *pdata2, GFP_KERNEL);
+		pdata2 = kmalloc_obj(*pdata2);
 		*pdata2 = pdata;
 		...
 		if (n == 2) {
@@ -373,7 +373,7 @@ a bus (appearing under /sys/class/spi_master).
 			return -ENODEV;
 
 		/* get memory for driver's per-chip state */
-		chip = kzalloc(sizeof *chip, GFP_KERNEL);
+		chip = kzalloc(*chip);
 		if (!chip)
 			return -ENOMEM;
 		spi_set_drvdata(spi, chip);
diff --git a/Documentation/staging/crc32.rst b/Documentation/staging/crc32.rst
index 64f3dd430a6ca..fc0d9564b99c1 100644
--- a/Documentation/staging/crc32.rst
+++ b/Documentation/staging/crc32.rst
@@ -119,7 +119,7 @@ the byte-at-a-time table method, popularized by Dilip V. Sarwate,
 v.31 no.8 (August 1988) p. 1008-1013.
 
 Here, rather than just shifting one bit of the remainder to decide
-in the correct multiple to subtract, we can shift a byte at a time.
+on the correct multiple to subtract, we can shift a byte at a time.
 This produces a 40-bit (rather than a 33-bit) intermediate remainder,
 and the correct multiple of the polynomial to subtract is found using
 a 256-entry lookup table indexed by the high 8 bits.
diff --git a/Documentation/staging/lzo.rst b/Documentation/staging/lzo.rst
index f65b515230146..2d48b2667dd22 100644
--- a/Documentation/staging/lzo.rst
+++ b/Documentation/staging/lzo.rst
@@ -75,7 +75,7 @@ Description
      are called under the assumption that a certain number of bytes follow
      because it has already been guaranteed before parsing the instructions.
      They just have to "refill" this credit if they consume extra bytes. This
-     is an implementation design choice independent on the algorithm or
+     is an implementation design choice independent of the algorithm or
      encoding.
 
 Versions
diff --git a/Documentation/staging/remoteproc.rst b/Documentation/staging/remoteproc.rst
index 5c226fa076d6b..c117b060e76c7 100644
--- a/Documentation/staging/remoteproc.rst
+++ b/Documentation/staging/remoteproc.rst
@@ -24,7 +24,7 @@ handlers, and then all rpmsg drivers will then just work
 (for more information about the virtio-based rpmsg bus and its drivers,
 please read Documentation/staging/rpmsg.rst).
 Registration of other types of virtio devices is now also possible. Firmwares
-just need to publish what kind of virtio devices do they support, and then
+just need to publish what kind of virtio devices they support, and then
 remoteproc will add those devices. This makes it possible to reuse the
 existing virtio drivers with remote processor backends at a minimal development
 cost.
diff --git a/Documentation/staging/static-keys.rst b/Documentation/staging/static-keys.rst
index b0a519f456cfa..e8dc3a87c3817 100644
--- a/Documentation/staging/static-keys.rst
+++ b/Documentation/staging/static-keys.rst
@@ -90,7 +90,7 @@ out-of-line true branch. Thus, changing branch direction is expensive but
 branch selection is basically 'free'. That is the basic tradeoff of this
 optimization.
 
-This lowlevel patching mechanism is called 'jump label patching', and it gives
+This low-level patching mechanism is called 'jump label patching', and it gives
 the basis for the static keys facility.
 
 Static key label API, usage and examples
diff --git a/Documentation/trace/histogram-design.rst b/Documentation/trace/histogram-design.rst
index e92f56ebd0b50..41a726cd35369 100644
--- a/Documentation/trace/histogram-design.rst
+++ b/Documentation/trace/histogram-design.rst
@@ -247,7 +247,7 @@ field's size and offset, is used to grab that subkey's data from the
 current trace record.
 
 Note, the hist field function use to be a function pointer in the
-hist_field stucture. Due to spectre mitigation, it was converted into
+hist_field structure. Due to spectre mitigation, it was converted into
 a fn_num and hist_fn_call() is used to call the associated hist field
 function that corresponds to the fn_num of the hist_field structure.
 
diff --git a/Documentation/trace/rv/monitor_sched.rst b/Documentation/trace/rv/monitor_sched.rst
index 0b96d6e147c65..d3ba7edc202f6 100644
--- a/Documentation/trace/rv/monitor_sched.rst
+++ b/Documentation/trace/rv/monitor_sched.rst
@@ -36,7 +36,7 @@ Specifications
 --------------
 
 The specifications included in sched are currently a work in progress, adapting the ones
-defined in by Daniel Bristot in [1].
+defined by Daniel Bristot in [1]_.
 
 Currently we included the following:
 
@@ -365,4 +365,7 @@ constraints when processing the events::
 References
 ----------
 
-[1] - https://bristot.me/linux-task-model
+.. [1] Daniel Bristot de Oliveira et al.:
+       `A thread synchronization model for the PREEMPT_RT Linux kernel
+       <https://www.iris.sssup.it/bitstream/11382/533630/1/Elsevier-JSA-2020.pdf>`_,
+       J. Syst. Archit., 2020.
diff --git a/Documentation/translations/it_IT/kernel-hacking/locking.rst b/Documentation/translations/it_IT/kernel-hacking/locking.rst
index 4c21cf60f775a..205a232c98e23 100644
--- a/Documentation/translations/it_IT/kernel-hacking/locking.rst
+++ b/Documentation/translations/it_IT/kernel-hacking/locking.rst
@@ -462,7 +462,7 @@ e tutti gli oggetti che contiene. Ecco il codice::
     {
             struct object *obj;
 
-            if ((obj = kmalloc(sizeof(*obj), GFP_KERNEL)) == NULL)
+            if ((obj = kmalloc_obj(*obj)) == NULL)
                     return -ENOMEM;
 
             strscpy(obj->name, name, sizeof(obj->name));
@@ -537,7 +537,7 @@ sono quelle rimosse, mentre quelle ``+`` sono quelle aggiunte.
              struct object *obj;
     +        unsigned long flags;
 
-             if ((obj = kmalloc(sizeof(*obj), GFP_KERNEL)) == NULL)
+             if ((obj = kmalloc_obj(*obj)) == NULL)
                      return -ENOMEM;
     @@ -63,30 +64,33 @@
              obj->id = id;
diff --git a/Documentation/translations/it_IT/locking/locktypes.rst b/Documentation/translations/it_IT/locking/locktypes.rst
index 1c7056283b9de..d5fa36aa05cc7 100644
--- a/Documentation/translations/it_IT/locking/locktypes.rst
+++ b/Documentation/translations/it_IT/locking/locktypes.rst
@@ -488,7 +488,7 @@ o rwlock_t. Per esempio, la sezione critica non deve fare allocazioni di
 memoria. Su un kernel non-PREEMPT_RT il seguente codice funziona perfettamente::
 
   raw_spin_lock(&lock);
-  p = kmalloc(sizeof(*p), GFP_ATOMIC);
+  p = kmalloc_obj(*p, GFP_ATOMIC);
 
 Ma lo stesso codice non funziona su un kernel PREEMPT_RT perché l'allocatore di
 memoria può essere oggetto di prelazione e quindi non può essere chiamato in un
@@ -497,7 +497,7 @@ trattiene un blocco *non-raw* perché non disabilitano la prelazione sui kernel
 PREEMPT_RT::
 
   spin_lock(&lock);
-  p = kmalloc(sizeof(*p), GFP_ATOMIC);
+  p = kmalloc_obj(*p, GFP_ATOMIC);
 
 
 bit spinlocks
diff --git a/Documentation/translations/it_IT/process/coding-style.rst b/Documentation/translations/it_IT/process/coding-style.rst
index c0dc786b84749..2a499412a2e3d 100644
--- a/Documentation/translations/it_IT/process/coding-style.rst
+++ b/Documentation/translations/it_IT/process/coding-style.rst
@@ -943,7 +943,7 @@ Il modo preferito per passare la dimensione di una struttura è il seguente:
 
 .. code-block:: c
 
-	p = kmalloc(sizeof(*p), ...);
+	p = kmalloc_obj(*p, ...);
 
 La forma alternativa, dove il nome della struttura viene scritto interamente,
 peggiora la leggibilità e introduce possibili bachi quando il tipo di
diff --git a/Documentation/translations/ja_JP/process/submitting-patches.rst b/Documentation/translations/ja_JP/process/submitting-patches.rst
index 9d63220abd15b..928e38a8d34dc 100644
--- a/Documentation/translations/ja_JP/process/submitting-patches.rst
+++ b/Documentation/translations/ja_JP/process/submitting-patches.rst
@@ -83,19 +83,18 @@ Linux の多くの環境は、上流から特定のパッチだけを取り込
 詳しく説明してください。コードが意図したとおりに動作していることを
 レビューアが確認できるよう、変更内容を平易な言葉で書き下すことが重要です。
 
-パッチの説明が Linux のソースコード管理システム ``git`` の「コミットログ」
-としてそのまま取り込める形で書かれていれば、メンテナは助かります。
-詳細は原文の該当節 ("The canonical patch format") を参照してください。
+パッチ説明を Linux のソースコード管理システム ``git`` の
+「コミットログ」としてそのまま取り込める形で書けば、メンテナは
+助かります。詳細は原文の該当節 ("The canonical patch format") を
+参照してください。
 
 .. TODO: Convert to file-local cross-reference when the destination is
    translated.
 
 1 つのパッチでは 1 つの問題だけを解決してください。記述が長くなり
-始めたら、それはパッチを分割すべきサインです。
-詳細は原文の該当節 ("Separate your changes") を参照してください。
+始めたら、パッチを分割すべきサインです。詳細は `変更を分割する`_ を
+参照してください。
 
-.. TODO: Convert to file-local cross-reference when the destination is
-   translated.
 
 パッチまたはパッチシリーズを投稿／再投稿する際は、その完全な
 説明と、それを正当化する理由を含めてください。単に「これはパッチ
@@ -179,3 +178,117 @@ URL は禁止です。
 
     $ git log -1 --pretty=fixes 54a4f0239f2e
     Fixes: 54a4f0239f2e ("KVM: MMU: make kvm_mmu_zap_page() return the number of pages it actually freed")
+
+変更を分割する
+--------------
+
+各 **論理的な変更** は、個別のパッチに分けてください。
+
+たとえば、単一のドライバに対する変更にバグ修正と性能改善の
+両方が含まれるなら、それらは 2 つ以上のパッチに分けてください。
+変更に API の更新と、その新しい API を使う新しいドライバが
+含まれるなら、それらは 2 つのパッチに分けてください。
+
+一方、多数のファイルに対して単一の変更を行う場合は、それらを
+1 つのパッチにまとめてください。つまり、1 つの論理的な変更は
+1 つのパッチに含めるべきです。
+
+覚えておくべき点は、各パッチがレビューアに理解しやすく、
+検証できる変更であるべきだということです。各パッチは、
+それ自体の妥当性で正当化できなければなりません。
+
+変更を完成させるために、あるパッチが別のパッチに依存するなら、
+それでも構いません。単に、パッチの説明に
+**"this patch depends on patch X"** と記してください。
+
+変更を一連のパッチに分ける際は、シリーズ中の各パッチを
+適用した後でも、カーネルが正しくビルドされ、正常に動作することを
+特に注意して確認してください。問題の追跡に ``git bisect`` を
+使う開発者は、あなたのパッチシリーズを任意の地点で分割する
+ことがあります。途中でバグを持ち込めば、彼らに感謝されることは
+ないでしょう。
+
+パッチセットをこれ以上小さくできないなら、一度に投稿するのは
+15 個程度までにして、レビューと統合を待ってください。
+
+
+変更のスタイルを確認する
+------------------------
+
+パッチに基本的なスタイル違反がないか確認してください。詳細は
+Documentation/process/coding-style.rst を参照してください。
+これを怠ると、単にレビューアの時間を無駄にするだけでなく、
+パッチはおそらく読まれもせずに却下されます。
+
+大きな例外が 1 つあります。コードをあるファイルから別の
+ファイルへ移動する場合です。このときは、コードを移動する
+その同じパッチの中で、移動したコードを一切変更してはいけません。
+そうすることで、コードの移動という行為と、あなたの変更とを
+明確に区別できます。これは実際の差分のレビューを大いに助け、
+ツールがコード自体の履歴をより適切に追跡できるようにします。
+
+提出前に、パッチスタイルチェッカー
+(``scripts/checkpatch.pl``) でパッチを確認してください。
+ただし、スタイルチェッカーは指針として見るべきであり、
+人間の判断に取って代わるものではないことに注意してください。
+違反があっても、その方がコードの見栄えがよいなら、
+そのままにしておくのが最善でしょう。
+
+チェッカーは 3 つのレベルで報告します:
+
+ - ERROR: 間違っている可能性が非常に高いもの
+ - WARNING: 慎重なレビューを要するもの
+ - CHECK: 検討を要するもの
+
+パッチに残した違反については、すべて理由を説明できなければ
+なりません。
+
+
+パッチの宛先を選択する
+----------------------
+
+各パッチでは、そのコードを保守する適切なサブシステムメンテナと
+メーリングリストを、必ず Cc に入れてください。誰がその
+メンテナかは、MAINTAINERS ファイルとソースコードの改訂履歴を
+調べて確認してください。この段階では ``scripts/get_maintainer.pl``
+が非常に役立ちます（パッチへのパスを引数として
+``scripts/get_maintainer.pl`` に渡してください）。作業中の
+サブシステムのメンテナが見つからない場合は、Andrew Morton
+(akpm@linux-foundation.org) が最後の手段となるメンテナです。
+
+すべてのパッチでは、デフォルトで linux-kernel@vger.kernel.org を
+使うべきですが、このリストの流量が多いため、目を通さなくなった
+開発者も少なくありません。とはいえ、無関係なメーリングリストや
+無関係な人々にスパムを送らないでください。
+
+カーネル関連のメーリングリストの多くは kernel.org で運営されており、
+その一覧は https://subspace.kernel.org で確認できます。ただし、
+他所で運営されているカーネル関連のメーリングリストもあります。
+
+Linux カーネルに採用されるすべての変更の最終的な裁定者は
+Linus Torvalds です。彼のメールアドレスは
+<torvalds@linux-foundation.org> です。Linus は大量のメールを
+受け取っており、現時点では彼に直接届くパッチはごくわずかなので、
+通常は彼にメールを送ることを極力避けてください。
+
+悪用可能なセキュリティバグを修正するパッチがあるなら、
+そのパッチを security@kernel.org に送ってください。深刻なバグに
+ついては、ディストリビュータがユーザーにパッチを配布できるよう、
+短期間の embargo が検討される場合があります。そのような場合、
+そのパッチを公開メーリングリストに送るべきではありません。
+Documentation/process/security-bugs.rst も参照してください。
+
+リリース済みカーネルの深刻なバグを修正するパッチは、次のような行を
+パッチの sign-off 欄に入れることで、stable メンテナへ向けてください::
+
+  Cc: stable@vger.kernel.org
+
+これはメールの受信者ではないことに注意してください。また、
+この文書に加えて Documentation/process/stable-kernel-rules.rst も
+読んでください。
+
+変更がユーザーランドとカーネルのインターフェースに影響する場合は、
+MAINTAINERS ファイルに記載されている MAN-PAGES メンテナに
+man-pages パッチ、少なくとも変更の通知を送って、情報が
+マニュアルページに反映されるようにしてください。ユーザー空間 API の
+変更は、linux-api@vger.kernel.org にも Cc してください。
diff --git a/Documentation/translations/pt_BR/index.rst b/Documentation/translations/pt_BR/index.rst
index 4a094d8b794fa..77c1a1cdc963e 100644
--- a/Documentation/translations/pt_BR/index.rst
+++ b/Documentation/translations/pt_BR/index.rst
@@ -67,6 +67,7 @@ kernel e sobre como ver seu trabalho integrado.
    :maxdepth: 1
 
    Introdução <process/1.Intro>
+   Regras de licenciamento <process/license-rules>
    Como começar <process/howto>
    Requisitos mínimos <process/changes>
    Conclave (Continuidade do projeto) <process/conclave>
diff --git a/Documentation/translations/pt_BR/process/license-rules.rst b/Documentation/translations/pt_BR/process/license-rules.rst
new file mode 100644
index 0000000000000..1e395dfea8759
--- /dev/null
+++ b/Documentation/translations/pt_BR/process/license-rules.rst
@@ -0,0 +1,483 @@
+.. SPDX-License-Identifier: GPL-2.0
+
+Regras de licenciamento do kernel Linux
+=======================================
+
+O Kernel Linux é fornecido apenas sob os termos da licença GNU General Public
+License versão 2 (GPL-2.0), conforme estabelecido em LICENSES/preferred/GPL-2.0,
+com uma exceção explícita para syscalls descrita em
+LICENSES/exceptions/Linux-syscall-note, conforme descrito no arquivo COPYING.
+
+Este arquivo de documentação fornece uma descrição de como cada arquivo-fonte
+deve ser anotado para tornar sua licença clara e inequívoca. Ele não substitui
+a licença do Kernel.
+
+A licença descrita no arquivo COPYING aplica-se ao código-fonte do kernel como
+um todo, embora arquivos-fonte individuais possam ter uma licença diferente, a
+qual deve ser obrigatoriamente compatível com a GPL-2.0::
+
+    GPL-1.0+  :  GNU General Public License v1.0 ou posterior
+    GPL-2.0+  :  GNU General Public License v2.0 ou posterior
+    LGPL-2.0  :  GNU Library General Public License v2 apenas
+    LGPL-2.0+ :  GNU Library General Public License v2 ou posterior
+    LGPL-2.1  :  GNU Lesser General Public License v2.1 apenas
+    LGPL-2.1+ :  GNU Lesser General Public License v2.1 ou posterior
+
+Além disso, arquivos individuais podem ser fornecidos sob uma licença dupla
+(dual license), por exemplo, uma das variantes GPL compatíveis e,
+alternativamente, sob uma licença permissiva como BSD, MIT, etc.
+
+Os arquivos de cabeçalho da API do espaço do usuário (UAPI), que descrevem a
+interface dos programas do espaço do usuário com o kernel, são um caso especial.
+De acordo com a nota no arquivo COPYING do kernel, a interface de syscall é um
+limite claro, que não estende os requisitos da GPL a qualquer software que a
+utilize para se comunicar com o kernel. Como os cabeçalhos UAPI devem ser
+passíveis de inclusão em quaisquer arquivos-fonte que criem um executável
+executado no kernel Linux, a exceção deve ser documentada por uma expressão de
+licença especial.
+
+A forma comum de expressar a licença de um arquivo-fonte é adicionar o texto
+padrão (boilerplate) correspondente no comentário inicial do arquivo. Devido a
+variações de formatação, erros de digitação, etc., esses "textos padrão" são
+difíceis de validar por ferramentas usadas no contexto de conformidade de
+licença.
+
+Uma alternativa aos textos padrão é o uso de identificadores de licença
+Software Package Data Exchange (SPDX) em cada arquivo-fonte. Os identificadores
+de licença SPDX são abreviações precisas e analisáveis por máquina para a
+licença sob a qual o conteúdo do arquivo é contribuído. Os identificadores de
+licença SPDX são gerenciados pelo Grupo de Trabalho SPDX na Linux Foundation e
+foram acordados por parceiros em toda a indústria, fornecedores de ferramentas
+e equipes jurídicas. Para mais informações, consulte https://spdx.org/
+
+O kernel Linux exige o identificador SPDX preciso em todos os arquivos-fonte.
+Os identificadores válidos usados no kernel são explicados na seção
+`Identificadores de Licença`_ e foram obtidos da lista de licenças
+oficial do  SPDX em https://spdx.org/licenses/ junto com os textos das licenças.
+
+Sintaxe do identificador de licença
+-----------------------------------
+
+1. Posicionamento:
+
+   O identificador de licença SPDX em arquivos do kernel deve ser adicionado na
+   primeira linha possível do arquivo que possa conter um comentário. Para a
+   maioria dos arquivos, esta é a primeira linha, exceto para scripts que
+   requerem o '#!CAMINHO_PARA_INTERPRETADOR' na primeira linha. Para esses
+   scripts, o identificador de licença SPDX deve ser colocado na segunda linha.
+
+   A linha do identificador de licença pode então ser seguida por uma ou
+   múltiplas linhas de SPDX-FileCopyrightText, se desejado.
+
+2. Estilo:
+
+   O identificador de licença SPDX é adicionado na forma de um comentário. O
+   estilo do comentário depende do tipo de arquivo::
+
+      Fonte C:    // SPDX-License-Identifier: <Expressão de Licença SPDX>
+      Cabeçalho C:/* SPDX-License-Identifier: <Expressão de Licença SPDX> */
+      ASM:        /* SPDX-License-Identifier: <Expressão de Licença SPDX> */
+      scripts:    # SPDX-License-Identifier: <Expressão de Licença SPDX>
+      .rst:       .. SPDX-License-Identifier: <Expressão de Licença SPDX>
+      .dts{i}:    // SPDX-License-Identifier: <Expressão de Licença SPDX>
+
+   Se uma ferramenta específica não conseguir lidar com o estilo de comentário
+   padrão, então deve ser utilizado o mecanismo de comentário apropriado que a
+   ferramenta aceite. Este é o motivo para ter o comentário no estilo ``/* */``
+   em arquivos de cabeçalho C. Foi observada uma quebra de build com arquivos
+   .lds gerados, onde o 'ld' falhou ao analisar o comentário C++. Isso já foi
+   corrigido, mas ainda existem ferramentas de assembler mais antigas que não
+   conseguem lidar com comentários no estilo C++.
+
+3. Sintaxe:
+
+   Uma <Expressão de Licença SPDX> pode ser um identificador SPDX simplificado
+   encontrado na Lista de Licenças SPDX, ou a combinação de dois desses
+   identificadores separados por "WITH", caso uma exceção de licença se aplique.
+   Quando múltiplas licenças são aplicáveis, a expressão utiliza as palavras-chave
+   "AND" ou "OR" para separar as sub-expressões, que devem ser delimitadas
+   por parênteses "(", ")".
+
+   Para licenças como [L]GPL, utiliza-se o sufixo "+" para indicar a opção
+   'ou posterior'::
+
+      // SPDX-License-Identifier: GPL-2.0+
+      // SPDX-License-Identifier: LGPL-2.1+
+
+   O termo "WITH" deve ser usado sempre que houver um modificador necessário
+   para a licença. Por exemplo, os arquivos UAPI do kernel Linux utilizam a
+   expressão::
+
+      // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
+      // SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note
+
+   Outros exemplos de uso da cláusula "WITH" para exceções no kernel são::
+
+      // SPDX-License-Identifier: GPL-2.0 WITH mif-exception
+      // SPDX-License-Identifier: GPL-2.0+ WITH GCC-exception-2.0
+
+   As exceções só podem ser aplicadas a identificadores de licença específicos.
+   Os identificadores válidos estão listados nas tags do arquivo de texto de
+   cada exceção. Para detalhes, veja o ponto `Exceções`_ no capítulo
+   `Identificadores de Licença`_.
+
+   O termo "OR" deve ser usado se o arquivo possuir licenciamento duplo (dual
+   licensed) e apenas uma das licenças puder ser selecionada. Por exemplo,
+   alguns arquivos dtsi estão disponíveis sob licença dupla::
+
+      // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+
+   Exemplos de expressões para arquivos com licenciamento duplo no kernel::
+
+      // SPDX-License-Identifier: GPL-2.0 OR MIT
+      // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
+      // SPDX-License-Identifier: GPL-2.0 OR Apache-2.0
+      // SPDX-License-Identifier: GPL-2.0 OR MPL-1.1
+      // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT
+      // SPDX-License-Identifier: GPL-1.0+ OR BSD-3-Clause OR OpenSSL
+
+   O termo "AND" deve ser usado se o arquivo possuir múltiplas licenças cujos
+   termos devem ser aplicados simultaneamente. Por exemplo, se um código herdado
+   de outro projeto foi incorporado ao kernel, mas os termos da licença original
+   ainda precisam ser respeitados::
+
+      // SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) AND MIT
+
+   Outro exemplo onde ambos os conjuntos de termos devem ser cumpridos é::
+
+      // SPDX-License-Identifier: GPL-1.0+ AND LGPL-2.1+
+
+Identificadores de Licença
+--------------------------
+
+As licenças atualmente em uso, bem como as licenças para código adicionado ao
+kernel, podem ser divididas em:
+
+4. _`Licenças preferenciais`:
+
+   Sempre que possível, estas licenças devem ser utilizadas, pois são conhecidas
+   por serem totalmente compatíveis e amplamente usadas. Estas licenças estão
+   disponíveis no diretório::
+
+      LICENSES/preferred/
+
+   na árvore de diretórios do código-fonte do kernel.
+
+   Os arquivos neste diretório contêm o texto completo da licença e as
+   `Metatags`_. Os nomes dos arquivos são idênticos ao identificador de licença
+   SPDX que deve ser utilizado para a licença nos arquivos-fonte.
+
+   Exemplos::
+
+      LICENSES/preferred/GPL-2.0
+
+   Contém o texto da licença GPL versão 2 e as metatags obrigatórias::
+
+      LICENSES/preferred/MIT
+
+   Contém o texto da licença MIT e as metatags obrigatórias.
+
+   _`Metatags`:
+
+   As seguintes metatags devem estar presentes em um arquivo de licença:
+
+   - Valid-License-Identifier:
+
+     Uma ou mais linhas que declaram quais Identificadores de Licença são válidos
+     dentro do projeto para referenciar este texto de licença específico.
+     Geralmente, trata-se de um único identificador válido, mas, por exemplo,
+     para licenças com as opções 'ou posterior' (or later), dois identificadores
+     são válidos.
+
+   - SPDX-URL:
+
+     A URL da página SPDX que contém informações adicionais relacionadas à licença.
+
+   - Usage-Guidance:
+
+     Texto livre para conselhos de uso. O texto deve incluir exemplos corretos
+     para os identificadores de licença SPDX, conforme eles devem ser colocados
+     nos arquivos-fonte de acordo com as diretrizes de
+     `Sintaxe do identificador de licença`_.
+
+   - License-Text:
+
+     Todo o texto após esta tag é tratado como o texto original da licença.
+
+   Exemplos de formato de arquivo::
+
+      Valid-License-Identifier: GPL-2.0
+      Valid-License-Identifier: GPL-2.0+
+      SPDX-URL: https://spdx.org/licenses/GPL-2.0.html
+      Usage-Guide:
+        Para usar esta licença no código-fonte, coloque um dos seguintes pares
+        SPDX tag/valor em um comentário, de acordo com as diretrizes de
+        posicionamento na documentação das regras de licenciamento.
+        Para 'GNU General Public License (GPL) version 2 only', use:
+          SPDX-License-Identifier: GPL-2.0
+        Para 'GNU General Public License (GPL) version 2 or any later version', use:
+          SPDX-License-Identifier: GPL-2.0+
+      License-Text:
+        Texto completo da licença
+
+   ::
+
+      Valid-License-Identifier: MIT
+      SPDX-URL: https://spdx.org/licenses/MIT.html
+      Usage-Guide:
+        Para usar esta licença no código-fonte, coloque o seguinte par SPDX
+        tag/valor em um comentário, de acordo com as diretrizes de
+        posicionamento na documentação das regras de licenciamento.
+          SPDX-License-Identifier: MIT
+      License-Text:
+        Texto completo da licença
+
+5. Licenças obsoletas:
+
+   Estas licenças devem ser utilizadas apenas para código já existente ou para
+   a importação de código de outros projetos. Estas licenças estão disponíveis
+   no diretório::
+
+      LICENSES/deprecated/
+
+   na árvore de fontes do kernel.
+
+   Os arquivos neste diretório contêm o texto completo da licença e as
+   `Metatags`_. Os nomes dos arquivos são idênticos ao identificador de
+   licença SPDX que deve ser utilizado para a licença nos arquivos fonte.
+
+   Exemplos::
+
+      LICENSES/deprecated/ISC
+
+   Contém o texto da licença *Internet Systems Consortium* e as metatags
+   necessárias::
+
+      LICENSES/deprecated/GPL-1.0
+
+   Contém o texto da licença GPL versão 1 e as metatags necessárias.
+
+   Metatags:
+
+   Os requisitos de metatags para "outras" licenças são idênticos aos
+   requisitos das `Licenças preferenciais`_.
+
+   Exemplo de formato de arquivo::
+
+      Valid-License-Identifier: ISC
+      SPDX-URL: https://spdx.org/licenses/ISC.html
+      Usage-Guide:
+        O uso desta licença no kernel para novos códigos é desencorajado
+        e deve ser utilizada exclusivamente para importar código de um
+        projeto já existente.
+        Para usar esta licença no código-fonte, coloque o seguinte par
+        tag/valor SPDX em um comentário, seguindo as diretrizes de
+        posicionamento na documentação das regras de licenciamento.
+          SPDX-License-Identifier: ISC
+      License-Text:
+        Texto completo da licença
+
+6. Apenas Licenciamento Duplo
+
+   Estas licenças devem ser usadas apenas para o licenciamento duplo de código
+   com outra licença, além de uma licença preferencial. Estas licenças estão
+   disponíveis no diretório::
+
+      LICENSES/dual/
+
+   No código-fonte do kernel.
+
+   Os arquivos neste diretório contêm o texto completo da licença e as
+   `Metatags`_. Os nomes dos arquivos são idênticos ao identificador de licença
+   SPDX que deve ser usado para a licença nos arquivos fonte.
+
+   Exemplos::
+
+      LICENSES/dual/MPL-1.1
+
+   Contém o texto da licença Mozilla Public License versão 1.1 e as metatags
+   necessárias::
+
+      LICENSES/dual/Apache-2.0
+
+   Contém o texto da licença Apache License versão 2.0 e as metatags
+   necessárias.
+
+   Metatags:
+
+   Os requisitos de metatags para 'outras' licenças são idênticos aos
+   requisitos das `Licenças preferenciais`_.
+
+   Exemplo de formato de arquivo::
+
+      Valid-License-Identifier: MPL-1.1
+      SPDX-URL: https://spdx.org/licenses/MPL-1.1.html
+      Usage-Guide:
+        NÃO use. A licença MPL-1.1 não é compatível com a GPL2. Ela só pode ser
+        usada para arquivos com licenciamento duplo onde a outra licença seja
+        compatível com a GPL2.
+        Se você acabar utilizando-a, ela DEVE ser usada em conjunto com uma
+        licença compatível com a GPL2 utilizando "OR".
+        Para usar a Mozilla Public License versão 1.1, coloque o seguinte par
+        tag/valor SPDX em um comentário, de acordo com as diretrizes de
+        posicionamento na documentação das regras de licenciamento:
+      SPDX-License-Identifier: MPL-1.1
+      License-Text:
+        Texto completo da licença
+
+|
+
+7. _`Exceções`:
+
+Algumas licenças podem ser alteradas com exceções que concedem certos direitos
+   que a licença original não concede. Estas exceções estão disponíveis no
+   diretório::
+
+      LICENSES/exceptions/
+
+   no código-fonte do kernel. Os arquivos neste diretório contêm o texto completo
+   da exceção e as `Metatags de Exceção`_ necessárias.
+
+   Exemplos::
+
+      LICENSES/exceptions/Linux-syscall-note
+
+   Contém a exceção de syscall do Linux, conforme documentado no arquivo COPYING
+   do kernel Linux, que é usada para arquivos de cabeçalho UAPI.
+   ex: /\* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note \*/::
+
+      LICENSES/exceptions/GCC-exception-2.0
+
+   Contém a 'exceção de vinculação' do GCC, que permite
+   vincular qualquer binário, independente de sua licença, à versão compilada
+   de um arquivo marcado com esta exceção. Isso é necessário para criar
+   executáveis funcionais a partir de código-fonte que não seja compatível
+   com a GPL.
+
+   _`Metatags de Exceção`:
+
+   As seguintes meta tags devem estar disponíveis em um arquivo de exceção:
+
+   - SPDX-Exception-Identifier:
+
+     Um identificador de exceção que pode ser usado com identificadores de
+     licença SPDX.
+
+   - SPDX-URL:
+
+     A URL da página SPDX que contém informações adicionais relacionadas
+     à exceção.
+
+   - SPDX-Licenses:
+
+     Uma lista separada por vírgulas de identificadores de licença SPDX para os
+     quais a exceção pode ser usada.
+
+   - Usage-Guidance:
+
+     Texto de formato livre para conselhos de uso. O texto deve ser seguido por
+     exemplos corretos para os identificadores de licença SPDX, conforme devem
+     ser colocados nos arquivos fonte de acordo com as diretrizes de
+     `Sintaxe do identificador de licença`_.
+
+   - Exception-Text:
+
+     Todo o texto após esta tag é tratado como o texto original da exceção.
+
+   Exemplos de formato de arquivo::
+
+      SPDX-Exception-Identifier: Linux-syscall-note
+      SPDX-URL: https://spdx.org/licenses/Linux-syscall-note.html
+      SPDX-Licenses: GPL-2.0, GPL-2.0+, GPL-1.0+, LGPL-2.0, LGPL-2.0+, LGPL-2.1, LGPL-2.1+
+      Usage-Guidance:
+        Esta exceção é usada em conjunto com uma das SPDX-Licenses acima para
+        marcar arquivos de cabeçalho de API do espaço do usuário (uapi), para que
+        possam ser incluídos em código de aplicativo de espaço do usuário que não
+        esteja em conformidade com a GPL.
+        Para usar esta exceção, adicione-a com a palavra-chave WITH a um dos
+        identificadores na tag SPDX-Licenses:
+          SPDX-License-Identifier: <SPDX-License> WITH Linux-syscall-note
+      Exception-Text:
+        Texto completo da exceção
+
+Exemplos de formato de arquivo::
+
+      SPDX-Exception-Identifier: GCC-exception-2.0
+      SPDX-URL: https://spdx.org/licenses/GCC-exception-2.0.html
+      SPDX-Licenses: GPL-2.0, GPL-2.0+
+      Usage-Guidance:
+        A "GCC Runtime Library exception 2.0" é usada em conjunto com uma das
+        SPDX-Licenses acima para código importado da biblioteca de tempo de
+        execução (runtime) do GCC.
+        Para usar esta exceção, adicione-a com a palavra-chave WITH a um dos
+        identificadores na tag SPDX-Licenses:
+          SPDX-License-Identifier: <SPDX-License> WITH GCC-exception-2.0
+      Exception-Text:
+        Texto completo da exceção
+
+Todos os identificadores de licença e exceções SPDX devem ter um arquivo
+correspondente nos subdiretórios LICENSES. Isso é necessário para permitir a
+verificação por ferramentas (ex: checkpatch.pl) e para que as licenças estejam
+prontas para leitura e extração diretamente da fonte, o que é recomendado por
+várias organizações de FOSS (Software Livre e de Código Aberto), como a
+`iniciativa REUSE da FSFE <https://reuse.software/>`_.
+
+_`MODULE_LICENSE`
+-----------------
+
+   Módulos carregáveis do kernel também exigem uma tag MODULE_LICENSE(). Esta tag
+   não substitui as informações adequadas de licença do código-fonte
+   (SPDX-License-Identifier), nem é de forma alguma relevante para expressar ou
+   determinar a licença exata sob a qual o código-fonte do módulo é fornecido.
+
+   O único propósito desta tag é fornecer informações suficientes ao carregador
+   de módulos do kernel e às ferramentas de espaço do usuário sobre o módulo ser
+   software livre ou proprietário.
+
+   As strings de licença válidas para MODULE_LICENSE() são::
+
+      ============================= =============================================
+      "GPL"                         O módulo está licenciado sob a GPL versão 2.
+                                    Isso não expressa nenhuma distinção entre
+                                    GPL-2.0-only ou GPL-2.0-or-later. A informação
+                                    exata da licença só pode ser determinada por
+                                    meio das informações de licença nos arquivos
+                                    fonte correspondentes.
+
+      "GPL v2"                      O mesmo que "GPL". Existe por razões
+                                    históricas.
+
+      "GPL and additional rights"   Variante histórica para expressar que o fonte
+                                    do módulo possui licenciamento duplo sob uma
+                                    variante da GPL v2 e a licença MIT. Por favor,
+                                    não use em códigos novos.
+
+      "Dual MIT/GPL"                A maneira correta de expressar que o módulo
+                                    possui licenciamento duplo sob uma escolha de
+                                    variante GPL v2 ou licença MIT.
+
+      "Dual BSD/GPL"                O módulo possui licenciamento duplo sob uma
+                                    escolha de variante GPL v2 ou licença BSD. A
+                                    variante exata da licença BSD só pode ser
+                                    determinada por meio das informações de
+                                    licença nos arquivos fonte correspondentes.
+
+      "Dual MPL/GPL"                O módulo possui licenciamento duplo sob uma
+                                    escolha de variante GPL v2 ou Mozilla Public
+                                    License (MPL). A variante exata da licença
+                                    MPL só pode ser determinada por meio das
+                                    informações de licença nos arquivos fonte
+                                    correspondentes.
+
+      "Proprietary"                 O módulo está sob uma licença proprietária.
+                                    "Proprietary" deve ser entendido apenas como
+                                    "A licença não é compatível com GPLv2". Esta
+                                    string é exclusiva para módulos de terceiros
+                                    não compatíveis com GPL2 e não pode ser usada
+                                    para módulos que tenham seu código-fonte na
+                                    árvore do kernel. Módulos marcados dessa forma
+                                    contaminam (tainting) o kernel com a flag 'P'
+                                    quando carregados, e o carregador de módulos
+                                    recusa-se a vincular tais módulos a símbolos
+                                    exportados com EXPORT_SYMBOL_GPL().
+      ============================= =============================================
\ No newline at end of file
diff --git a/Documentation/translations/pt_BR/process/maintainer-handbooks.rst b/Documentation/translations/pt_BR/process/maintainer-handbooks.rst
index ba36df8eeaf18..bf7a381472e23 100644
--- a/Documentation/translations/pt_BR/process/maintainer-handbooks.rst
+++ b/Documentation/translations/pt_BR/process/maintainer-handbooks.rst
@@ -16,3 +16,4 @@ Conteúdos:
    maintainer-netdev
    maintainer-soc
    maintainer-soc-clean-dts
+   maintainer-kvm-x86
diff --git a/Documentation/translations/sp_SP/process/coding-style.rst b/Documentation/translations/sp_SP/process/coding-style.rst
index 7d63aa8426e63..44c93d5f6beb3 100644
--- a/Documentation/translations/sp_SP/process/coding-style.rst
+++ b/Documentation/translations/sp_SP/process/coding-style.rst
@@ -955,7 +955,7 @@ La forma preferida para pasar el tamaño de una estructura es la siguiente:
 
 .. code-block:: c
 
-	p = kmalloc(sizeof(*p), ...);
+	p = kmalloc_obj(*p, ...);
 
 La forma alternativa donde se deletrea el nombre de la estructura perjudica
 la legibilidad, y presenta una oportunidad para un error cuando se cambia
diff --git a/Documentation/translations/zh_CN/core-api/kref.rst b/Documentation/translations/zh_CN/core-api/kref.rst
index b9902af310c55..8a0c030ec1d31 100644
--- a/Documentation/translations/zh_CN/core-api/kref.rst
+++ b/Documentation/translations/zh_CN/core-api/kref.rst
@@ -52,7 +52,7 @@ kref可以出现在数据结构体中的任何地方。
 
      struct my_data *data;
 
-     data = kmalloc(sizeof(*data), GFP_KERNEL);
+     data = kmalloc_obj(*data);
      if (!data)
             return -ENOMEM;
      kref_init(&data->refcount);
@@ -106,7 +106,7 @@ Kref规则
 	int rv = 0;
 	struct my_data *data;
 	struct task_struct *task;
-	data = kmalloc(sizeof(*data), GFP_KERNEL);
+	data = kmalloc_obj(*data);
 	if (!data)
 		return -ENOMEM;
 	kref_init(&data->refcount);
diff --git a/Documentation/translations/zh_CN/networking/netmem.rst b/Documentation/translations/zh_CN/networking/netmem.rst
index fe351a240f021..320f3eacf51be 100644
--- a/Documentation/translations/zh_CN/networking/netmem.rst
+++ b/Documentation/translations/zh_CN/networking/netmem.rst
@@ -89,4 +89,9 @@ dma-mapping API 去处理。
 使用某个还不存在的 netmem API，你可以自行添加并提交到 netdev@，也可以联系维护
 人员或者发送邮件至 almasrymina@google.com 寻求帮助。
 
-2. 驱动程序应通过设置 netdev->netmem_tx = true 来表明自身支持 netmem 功能。
+2. 驱动程序应将 `netdev->netmem_tx` 设置为适当的模式：
+
+   - `NETMEM_TX_DMA`：适用于执行 DMA 的物理设备。
+
+   - `NETMEM_TX_NO_DMA`：适用于不执行 DMA 的虚拟或透传设备，但仍支持
+     处理 netmem 支持的 skb。
diff --git a/Documentation/translations/zh_CN/process/coding-style.rst b/Documentation/translations/zh_CN/process/coding-style.rst
index 5a342a024c01e..55d5da974d89f 100644
--- a/Documentation/translations/zh_CN/process/coding-style.rst
+++ b/Documentation/translations/zh_CN/process/coding-style.rst
@@ -813,7 +813,7 @@ Documentation/translations/zh_CN/core-api/memory-allocation.rst 。
 
 .. code-block:: c
 
-	p = kmalloc(sizeof(*p), ...);
+	p = kmalloc_obj(*p, ...);
 
 另外一种传递方式中，sizeof 的操作数是结构体的名字，这样会降低可读性，并且可能
 会引入 bug。有可能指针变量类型被改变时，而对应的传递给内存分配函数的 sizeof
diff --git a/Documentation/translations/zh_CN/video4linux/v4l2-framework.txt b/Documentation/translations/zh_CN/video4linux/v4l2-framework.txt
index f0be21a60a0fb..05414579efaca 100644
--- a/Documentation/translations/zh_CN/video4linux/v4l2-framework.txt
+++ b/Documentation/translations/zh_CN/video4linux/v4l2-framework.txt
@@ -799,7 +799,7 @@ int my_open(struct file *file)
 
 	...
 
-	my_fh = kzalloc(sizeof(*my_fh), GFP_KERNEL);
+	my_fh = kzalloc_obj(*my_fh);
 
 	...
 
diff --git a/Documentation/translations/zh_TW/process/coding-style.rst b/Documentation/translations/zh_TW/process/coding-style.rst
index e2ba97b3d8bbf..63c78982a1afc 100644
--- a/Documentation/translations/zh_TW/process/coding-style.rst
+++ b/Documentation/translations/zh_TW/process/coding-style.rst
@@ -827,7 +827,7 @@ Documentation/translations/zh_CN/core-api/memory-allocation.rst 。
 
 .. code-block:: c
 
-	p = kmalloc(sizeof(*p), ...);
+	p = kmalloc_obj(*p, ...);
 
 另外一種傳遞方式中，sizeof 的操作數是結構體的名字，這樣會降低可讀性，並且可能
 會引入 bug。有可能指針變量類型被改變時，而對應的傳遞給內存分配函數的 sizeof
diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst
index 331223761ffff..58716f4f4834f 100644
--- a/Documentation/userspace-api/ioctl/ioctl-number.rst
+++ b/Documentation/userspace-api/ioctl/ioctl-number.rst
@@ -271,7 +271,6 @@ Code  Seq#    Include File                                             Comments
 'm'   00-09  linux/mmtimer.h                                           conflict!
 'm'   all    linux/mtio.h                                              conflict!
 'm'   all    linux/soundcard.h                                         conflict!
-'m'   all    linux/synclink.h                                          conflict!
 'm'   00-19  drivers/message/fusion/mptctl.h                           conflict!
 'm'   00     drivers/scsi/megaraid/megaraid_ioctl.h                    conflict!
 'n'   00-7F  linux/ncp_fs.h and fs/ncpfs/ioctl.c
diff --git a/Documentation/userspace-api/landlock.rst b/Documentation/userspace-api/landlock.rst
index fd8b78c31f2fe..45861fa75685b 100644
--- a/Documentation/userspace-api/landlock.rst
+++ b/Documentation/userspace-api/landlock.rst
@@ -8,7 +8,7 @@ Landlock: unprivileged access control
 =====================================
 
 :Author: Mickaël Salaün
-:Date: March 2026
+:Date: May 2026
 
 The goal of Landlock is to enable restriction of ambient rights (e.g. global
 filesystem or network access) for a set of processes.  Because Landlock
@@ -155,7 +155,7 @@ this file descriptor.
 
 .. code-block:: c
 
-    int err;
+    int err = 0;
     struct landlock_path_beneath_attr path_beneath = {
         .allowed_access =
             LANDLOCK_ACCESS_FS_EXECUTE |
@@ -163,25 +163,29 @@ this file descriptor.
             LANDLOCK_ACCESS_FS_READ_DIR,
     };
 
-    path_beneath.parent_fd = open("/usr", O_PATH | O_CLOEXEC);
-    if (path_beneath.parent_fd < 0) {
-        perror("Failed to open file");
-        close(ruleset_fd);
-        return 1;
-    }
-    err = landlock_add_rule(ruleset_fd, LANDLOCK_RULE_PATH_BENEATH,
-                            &path_beneath, 0);
-    close(path_beneath.parent_fd);
-    if (err) {
-        perror("Failed to update ruleset");
-        close(ruleset_fd);
-        return 1;
+    path_beneath.allowed_access &= ruleset_attr.handled_access_fs;
+    if (path_beneath.allowed_access) {
+        path_beneath.parent_fd = open("/usr", O_PATH | O_CLOEXEC);
+        if (path_beneath.parent_fd < 0) {
+            perror("Failed to open file");
+            close(ruleset_fd);
+            return 1;
+        }
+        err = landlock_add_rule(ruleset_fd, LANDLOCK_RULE_PATH_BENEATH,
+                                &path_beneath, 0);
+        close(path_beneath.parent_fd);
+        if (err) {
+            perror("Failed to update ruleset");
+            close(ruleset_fd);
+            return 1;
+        }
     }
 
-It may also be required to create rules following the same logic as explained
-for the ruleset creation, by filtering access rights according to the Landlock
-ABI version.  In this example, this is not required because all of the requested
-``allowed_access`` rights are already available in ABI 1.
+As shown above, masking the rule's ``allowed_access`` against the ruleset's
+``handled_access_*`` is the recommended best-effort pattern: rights the running
+kernel does not support are dropped (the compatibility switch above already
+cleared them in ``handled_access_*``), and the rule is skipped if no supported
+right remains.
 
 For network access-control, we can add a set of rules that allow to use a port
 number for a specific action: HTTPS connections.
@@ -193,8 +197,10 @@ number for a specific action: HTTPS connections.
         .port = 443,
     };
 
-    err = landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT,
-                            &net_port, 0);
+    net_port.allowed_access &= ruleset_attr.handled_access_net;
+    if (net_port.allowed_access)
+        err = landlock_add_rule(ruleset_fd, LANDLOCK_RULE_NET_PORT,
+                                &net_port, 0);
 
 When passing a non-zero ``flags`` argument to ``landlock_restrict_self()``, a
 similar backwards compatibility check is needed for the restrict flags
diff --git a/Documentation/userspace-api/media/cec/cec.h.rst.exceptions b/Documentation/userspace-api/media/cec/cec.h.rst.exceptions
index 65e8be062bdbe..5d6f7747c0234 100644
--- a/Documentation/userspace-api/media/cec/cec.h.rst.exceptions
+++ b/Documentation/userspace-api/media/cec/cec.h.rst.exceptions
@@ -524,6 +524,29 @@ ignore define CEC_OP_AUD_OUT_COMPENSATED_DELAY
 ignore define CEC_OP_AUD_OUT_COMPENSATED_NO_DELAY
 ignore define CEC_OP_AUD_OUT_COMPENSATED_PARTIAL_DELAY
 
+ignore define CEC_MSG_REQUEST_LIP_SUPPORT
+ignore define CEC_MSG_REPORT_LIP_SUPPORT
+ignore define CEC_MSG_REQUEST_AUDIO_AND_VIDEO_LATENCY
+
+ignore define CEC_OP_HDR_FORMAT_GAMMA_SDR
+ignore define CEC_OP_HDR_FORMAT_GAMMA_HDR
+ignore define CEC_OP_HDR_FORMAT_PQ
+ignore define CEC_OP_HDR_FORMAT_HLG
+ignore define CEC_OP_HDR_FORMAT_DYNAMIC_HDR_TYPE_1
+ignore define CEC_OP_HDR_FORMAT_DYNAMIC_HDR_TYPE_2
+ignore define CEC_OP_HDR_FORMAT_DYNAMIC_HDR_TYPE_4
+ignore define CEC_OP_HDR_FORMAT_DV_SINK_LED
+ignore define CEC_OP_HDR_FORMAT_DV_SOURCE_LED
+ignore define CEC_OP_HDR_FORMAT_HDR10PLUS
+ignore define CEC_OP_HDR_FORMAT_ETSI_TS_103_433
+
+ignore define CEC_MSG_REPORT_AUDIO_AND_VIDEO_LATENCY
+ignore define CEC_MSG_REQUEST_AUDIO_LATENCY
+ignore define CEC_MSG_REPORT_AUDIO_LATENCY
+ignore define CEC_MSG_REQUEST_VIDEO_LATENCY
+ignore define CEC_MSG_REPORT_VIDEO_LATENCY
+ignore define CEC_MSG_UPDATE_SQID
+
 ignore define CEC_MSG_CDC_MESSAGE
 
 ignore define CEC_MSG_CDC_HEC_INQUIRE_STATE
diff --git a/Documentation/userspace-api/media/drivers/uvcvideo.rst b/Documentation/userspace-api/media/drivers/uvcvideo.rst
index dbb30ad389ae4..b09d2f8ba66ec 100644
--- a/Documentation/userspace-api/media/drivers/uvcvideo.rst
+++ b/Documentation/userspace-api/media/drivers/uvcvideo.rst
@@ -109,6 +109,8 @@ IOCTL reference
 UVCIOC_CTRL_MAP - Map a UVC control to a V4L2 control
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
+**This IOCTL is deprecated and will be eventually removed**
+
 Argument: struct uvc_xu_control_mapping
 
 **Description**:
diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst
index 3b1e05c6eb139..d7a3b8ef03a78 100644
--- a/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst
+++ b/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst
@@ -1877,7 +1877,7 @@ params syntax' of the :ref:`vp9` specification for more details.
     :stub-columns: 0
     :widths:       1 1 2
 
-    * - __u8
+    * - __s16
       - ``feature_data[8][4]``
       - Data attached to each feature. Data entry is only valid if the feature
         is enabled. The array shall be indexed with segment number as the first dimension
diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst
index c8890cb5e00ac..ab865a1a6ba92 100644
--- a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst
+++ b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst
@@ -737,6 +737,12 @@ enum v4l2_mpeg_video_frame_skip_mode -
     Enable writing sample aspect ratio in the Video Usability
     Information. Applicable to the H264 encoder.
 
+``V4L2_CID_MPEG_VIDEO_BACKGROUND_DETECTION (boolean)``
+    If enabled, the encoder detect a background region in frame and
+    use low bits or skip mode to encode the background region.
+    If a lot of scenes are stationary or background, It may help to
+    reduce the video bitrate. Applicable to the encoder.
+
 .. _v4l2-mpeg-video-h264-vui-sar-idc:
 
 ``V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_IDC``
diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-image-process.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-image-process.rst
index 6d516f041ca25..57c2baf34acf0 100644
--- a/Documentation/userspace-api/media/v4l/ext-ctrls-image-process.rst
+++ b/Documentation/userspace-api/media/v4l/ext-ctrls-image-process.rst
@@ -24,7 +24,10 @@ Image Process Control IDs
 .. _v4l2-cid-link-freq:
 
 ``V4L2_CID_LINK_FREQ (integer menu)``
-    The frequency of the data bus (e.g. parallel or CSI-2).
+    The fundamental frequency of the operating symbol rate (serial interfaces
+    such as CSI-2) or the sampling rate (parallel interfaces such as DVP or
+    Bt.565) of the data interface. For CSI-2, the frequency is equal to
+    _1 / (2 * UI)_.
 
 .. _v4l2-cid-pixel-rate:
 
diff --git a/Documentation/userspace-api/netlink/intro.rst b/Documentation/userspace-api/netlink/intro.rst
index aacffade8f84c..ca60abe94e3d3 100644
--- a/Documentation/userspace-api/netlink/intro.rst
+++ b/Documentation/userspace-api/netlink/intro.rst
@@ -526,8 +526,8 @@ of the recvmsg() system call, *not* a Netlink header).
 
 Upon truncation the remaining part of the message is discarded.
 
-Netlink expects that the user buffer will be at least 8kB or a page
-size of the CPU architecture, whichever is bigger. Particular Netlink
+Netlink expects that the user buffer will be at most 8kB or a page
+size of the CPU architecture, whichever is smaller. Particular Netlink
 families may, however, require a larger buffer. 32kB buffer is recommended
 for most efficient handling of dumps (larger buffer fits more dumped
 objects and therefore fewer recvmsg() calls are needed).
diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst
index 52bbbb553ce10..adfb04bbf3db9 100644
--- a/Documentation/virt/kvm/api.rst
+++ b/Documentation/virt/kvm/api.rst
@@ -4944,10 +4944,13 @@ Errors:
   #define KVM_STATE_NESTED_FORMAT_SVM		1
 
   #define KVM_STATE_NESTED_VMX_VMCS_SIZE	0x1000
+  #define KVM_STATE_NESTED_SVM_VMCB_SIZE	0x1000
 
   #define KVM_STATE_NESTED_VMX_SMM_GUEST_MODE	0x00000001
   #define KVM_STATE_NESTED_VMX_SMM_VMXON	0x00000002
 
+  #define KVM_STATE_NESTED_GIF_SET		0x00000100
+
   #define KVM_STATE_VMX_PREEMPTION_TIMER_DEADLINE 0x00000001
 
   struct kvm_vmx_nested_state_hdr {
@@ -4962,11 +4965,20 @@ Errors:
 	__u64 preemption_timer_deadline;
   };
 
+  struct kvm_svm_nested_state_hdr {
+	__u64 vmcb_pa;
+	__u64 gpat;
+  };
+
   struct kvm_vmx_nested_state_data {
 	__u8 vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];
 	__u8 shadow_vmcs12[KVM_STATE_NESTED_VMX_VMCS_SIZE];
   };
 
+  struct kvm_svm_nested_state_data {
+	__u8 vmcb12[KVM_STATE_NESTED_SVM_VMCB_SIZE];
+  };
+
 This ioctl copies the vcpu's nested virtualization state from the kernel to
 userspace.
 
@@ -8553,6 +8565,20 @@ KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM   By default, KVM relaxes the consisten
                                            bit to be cleared.  Note that the vmcs02
                                            bit is still completely controlled by the
                                            host, regardless of the quirk setting.
+
+KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT        By default, KVM for nested SVM guests
+                                           shares the IA32_PAT MSR between L1 and
+                                           L2. This is legacy behavior and does
+                                           not match the AMD architecture
+                                           specification. When this quirk is
+                                           disabled and nested paging (NPT) is
+                                           enabled for L2, KVM correctly
+                                           virtualizes a separate guest PAT
+                                           register for L2, using the g_pat
+                                           field in the VMCB. When NPT is
+                                           disabled for L2, L1 and L2 continue
+                                           to share the IA32_PAT MSR regardless
+                                           of the quirk setting.
 ========================================   ================================================
 
 7.32 KVM_CAP_MAX_VCPU_ID
diff --git a/Documentation/virt/kvm/x86/mmu.rst b/Documentation/virt/kvm/x86/mmu.rst
index 2b3b6d442302b..666aa179601a1 100644
--- a/Documentation/virt/kvm/x86/mmu.rst
+++ b/Documentation/virt/kvm/x86/mmu.rst
@@ -184,10 +184,8 @@ Shadow pages contain the following information:
     Contains the value of efer.nx for which the page is valid.
   role.cr0_wp:
     Contains the value of cr0.wp for which the page is valid.
-  role.smep_andnot_wp:
-    Contains the value of cr4.smep && !cr0.wp for which the page is valid
-    (pages for which this is true are different from other pages; see the
-    treatment of cr0.wp=0 below).
+  role.cr4_smep:
+    Contains the value of cr4.smep for which the page is valid.
   role.smap_andnot_wp:
     Contains the value of cr4.smap && !cr0.wp for which the page is valid
     (pages for which this is true are different from other pages; see the
@@ -435,8 +433,8 @@ from being written by the kernel after cr0.wp has changed to 1, we make
 the value of cr0.wp part of the page role.  This means that an spte created
 with one value of cr0.wp cannot be used when cr0.wp has a different value -
 it will simply be missed by the shadow page lookup code.  A similar issue
-exists when an spte created with cr0.wp=0 and cr4.smep=0 is used after
-changing cr4.smep to 1.  To avoid this, the value of !cr0.wp && cr4.smep
+exists when an spte created with cr0.wp=0 and cr4.smap=0 is used after
+changing cr4.smap to 1.  To avoid this, the value of !cr0.wp && cr4.smap
 is also made a part of the page role.
 
 Large pages
diff --git a/Documentation/watchdog/convert_drivers_to_kernel_api.rst b/Documentation/watchdog/convert_drivers_to_kernel_api.rst
index e83609a5d0071..004fe6ef76e94 100644
--- a/Documentation/watchdog/convert_drivers_to_kernel_api.rst
+++ b/Documentation/watchdog/convert_drivers_to_kernel_api.rst
@@ -11,12 +11,12 @@ This document shall guide you for this task. The necessary steps are described
 as well as things to look out for.
 
 
-Remove the file_operations struct
+Remove the struct file_operations
 ---------------------------------
 
 Old drivers define their own file_operations for actions like open(), write(),
 etc... These are now handled by the framework and just call the driver when
-needed. So, in general, the 'file_operations' struct and assorted functions can
+needed. So, in general, the struct file_operations and assorted functions can
 go. Only very few driver-specific details have to be moved to other functions.
 Here is a overview of the functions and probably needed actions:
 
@@ -36,7 +36,7 @@ Here is a overview of the functions and probably needed actions:
   from the driver:
 
 	WDIOC_GETSUPPORT:
-		Returns the mandatory watchdog_info struct from the driver
+		Returns the mandatory struct watchdog_info from the driver
 
 	WDIOC_GETSTATUS:
 		Needs the status-callback defined, otherwise returns 0
@@ -88,8 +88,8 @@ refactoring. The rest can go.
 Remove the miscdevice
 ---------------------
 
-Since the file_operations are gone now, you can also remove the 'struct
-miscdevice'. The framework will create it on watchdog_dev_register() called by
+Since the file_operations are gone now, you can also remove the struct
+miscdevice. The framework will create it on watchdog_dev_register() called by
 watchdog_register_device()::
 
   -static struct miscdevice s3c2410wdt_miscdev = {
@@ -113,10 +113,10 @@ them. Includes can be removed, too. For example::
 Add the watchdog operations
 ---------------------------
 
-All possible callbacks are defined in 'struct watchdog_ops'. You can find it
-explained in 'watchdog-kernel-api.txt' in this directory. start() and
+All possible callbacks are defined in struct watchdog_ops. You can find it
+explained in watchdog-kernel-api.rst in this directory. start() and
 owner must be set, the rest are optional. You will easily find corresponding
-functions in the old driver. Note that you will now get a pointer to the
+functions in the old driver. Note that you will now get a pointer to the struct
 watchdog_device as a parameter to these functions, so you probably have to
 change the function header. Other changes are most likely not needed, because
 here simply happens the direct hardware access. If you have device-specific
@@ -151,12 +151,12 @@ A typical function-header change looks like::
 Add the watchdog device
 -----------------------
 
-Now we need to create a 'struct watchdog_device' and populate it with the
-necessary information for the framework. The struct is also explained in detail
-in 'watchdog-kernel-api.txt' in this directory. We pass it the mandatory
-watchdog_info struct and the newly created watchdog_ops. Often, old drivers
+Now we need to create a struct watchdog_device and populate it with the
+necessary information for the framework. The structure is also explained in detail
+in watchdog-kernel-api.rst in this directory. We pass it the mandatory
+struct watchdog_info and the newly created struct watchdog_ops. Often, old drivers
 have their own record-keeping for things like bootstatus and timeout using
-static variables. Those have to be converted to use the members in
+static variables. Those have to be converted to use the members in struct
 watchdog_device. Note that the timeout values are unsigned int. Some drivers
 use signed int, so this has to be converted, too.
 
@@ -174,7 +174,7 @@ Handle the 'nowayout' feature
 A few drivers use nowayout statically, i.e. there is no module parameter for it
 and only CONFIG_WATCHDOG_NOWAYOUT determines if the feature is going to be
 used. This needs to be converted by initializing the status variable of the
-watchdog_device like this::
+struct watchdog_device like this::
 
         .status = WATCHDOG_NOWAYOUT_INIT_STATUS,
 
diff --git a/Documentation/watchdog/ics-wdt.rst b/Documentation/watchdog/ics-wdt.rst
new file mode 100644
index 0000000000000..b0dc29e56358d
--- /dev/null
+++ b/Documentation/watchdog/ics-wdt.rst
@@ -0,0 +1,65 @@
+.. SPDX-License-Identifier: GPL-2.0-or-later
+
+==========================================
+Industrial Computer Source Watchdog Driver
+==========================================
+
+Last Reviewed: 10/05/2007
+
+Alan Cox <alan@lxorguk.ukuu.org.uk>
+
+	- ICS	WDT501-P
+	- ICS	WDT501-P (no fan tachometer)
+	- ICS	WDT500-P
+
+All the interfaces provide /dev/watchdog, which when open must be written
+to within a timeout or the machine will reboot. Each write delays the reboot
+time another timeout. In the case of the software watchdog the ability to
+reboot will depend on the state of the machines and interrupts. The hardware
+boards physically pull the machine down off their own onboard timers and
+will reboot from almost anything.
+
+A second temperature monitoring interface is available on the WDT501P cards.
+This provides /dev/temperature. This is the machine internal temperature in
+degrees Fahrenheit. Each read returns a single byte giving the temperature.
+
+The third interface logs kernel messages on additional alert events.
+
+The ICS ISA-bus wdt card cannot be safely probed for. Instead you need to
+pass IO address and IRQ boot parameters.  E.g.::
+
+	wdt.io=0x240 wdt.irq=11
+
+Other "wdt" driver parameters are:
+
+	===========	======================================================
+	heartbeat	Watchdog heartbeat in seconds (default 60)
+	nowayout	Watchdog cannot be stopped once started (kernel
+			build parameter)
+	tachometer	WDT501-P Fan Tachometer support (0=disable, default=0)
+	type		WDT501-P Card type (500 or 501, default=500)
+	===========	======================================================
+
+Features
+--------
+
+================   =======	   =======
+		   WDT501P	   WDT500P
+================   =======	   =======
+Reboot Timer	   X               X
+External Reboot	   X	           X
+I/O Port Monitor   o		   o
+Temperature	   X		   o
+Fan Speed          X		   o
+Power Under	   X               o
+Power Over         X               o
+Overheat           X               o
+================   =======	   =======
+
+The external event interfaces on the WDT boards are not currently supported.
+Minor numbers are however allocated for it.
+
+
+Example Watchdog Driver:
+
+	see samples/watchdog/watchdog-simple.c
diff --git a/Documentation/watchdog/index.rst b/Documentation/watchdog/index.rst
index 1cea24681e6bd..dbc702b31a43e 100644
--- a/Documentation/watchdog/index.rst
+++ b/Documentation/watchdog/index.rst
@@ -4,15 +4,25 @@
 Watchdog Support
 ================
 
+Kernel infrastructure
+=====================
+
 .. toctree::
     :maxdepth: 1
 
-    hpwdt
-    mlx-wdt
-    pcwd-watchdog
     watchdog-api
     watchdog-kernel-api
-    watchdog-parameters
     watchdog-pm
-    wdt
     convert_drivers_to_kernel_api
+
+Driver specific
+===============
+
+.. toctree::
+    :maxdepth: 1
+
+    watchdog-parameters
+    hpwdt
+    ics-wdt
+    mlx-wdt
+    pcwd-watchdog
diff --git a/Documentation/watchdog/mlx-wdt.rst b/Documentation/watchdog/mlx-wdt.rst
index 35e690dea9db6..2da36c7628900 100644
--- a/Documentation/watchdog/mlx-wdt.rst
+++ b/Documentation/watchdog/mlx-wdt.rst
@@ -28,7 +28,7 @@ Type 3:
   Same as Type 2 with extended maximum timeout period.
   Maximum timeout is 65535 sec.
 
-Type 1 HW watchdog implementation exist in old systems and
+Type 1 HW watchdog implementation exists in old systems and
 all new systems have type 2 HW watchdog.
 Two types of HW implementation have also different register map.
 
@@ -48,7 +48,7 @@ which is optional.
 Watchdog can be started during a probe, in this case it will be
 pinged by watchdog core before watchdog device will be opened by
 user space application.
-Watchdog can be initialised in nowayout way, i.e. oncse started
+Watchdog can be initialised in nowayout mode, i.e. once started
 it can't be stopped.
 
 This mlx-wdt driver supports both HW watchdog implementations.
diff --git a/Documentation/watchdog/pcwd-watchdog.rst b/Documentation/watchdog/pcwd-watchdog.rst
index 151505c856f63..5cd45abe246a6 100644
--- a/Documentation/watchdog/pcwd-watchdog.rst
+++ b/Documentation/watchdog/pcwd-watchdog.rst
@@ -29,7 +29,7 @@ Documentation and Driver by Ken Hollis <kenji@bitgate.com>
  to run the program with an "&" to run it in the background!)
 
  If you want to write a program to be compatible with the PC Watchdog
- driver, simply use of modify the watchdog test program:
+ driver, simply use or modify the watchdog test program:
  tools/testing/selftests/watchdog/watchdog-test.c
 
 
@@ -37,16 +37,23 @@ Documentation and Driver by Ken Hollis <kenji@bitgate.com>
 
 	WDIOC_GETSUPPORT
 		This returns the support of the card itself.  This
-		returns in structure "PCWDS" which returns:
+		returns in structure watchdog_info:
+
+                        identity = "PCWD"
+
+			options = list of supported options::
+                                  WDIOF_OVERHEAT
+                                  WDIOF_CARDRESET
+                                  WDIOF_KEEPALIVEPING
+                                  WDIOF_SETTIMEOUT
+                                  WDIOF_MAGICCLOSE
 
-			options = WDIOS_TEMPPANIC
-				  (This card supports temperature)
 			firmware_version = xxxx
 				  (Firmware version of the card)
 
 	WDIOC_GETSTATUS
 		This returns the status of the card, with the bits of
-		WDIOF_* bitwise-anded into the value.  (The comments
+		WDIOF_* bitwise-ored into the value.  (The comments
 		are in include/uapi/linux/watchdog.h)
 
 	WDIOC_GETBOOTSTATUS
@@ -55,7 +62,7 @@ Documentation and Driver by Ken Hollis <kenji@bitgate.com>
 
 	WDIOC_GETTEMP
 		This returns the temperature of the card.  (You can also
-		read /dev/watchdog, which gives a temperature update
+		read /dev/temperature, which gives a temperature update
 		every second.)
 
 	WDIOC_SETOPTIONS
diff --git a/Documentation/watchdog/watchdog-api.rst b/Documentation/watchdog/watchdog-api.rst
index 78e228c272cf8..0e6e526d7b957 100644
--- a/Documentation/watchdog/watchdog-api.rst
+++ b/Documentation/watchdog/watchdog-api.rst
@@ -39,12 +39,12 @@ The simplest API
 
 All drivers support the basic mode of operation, where the watchdog
 activates as soon as /dev/watchdog is opened and will reboot unless
-the watchdog is pinged within a certain time, this time is called the
+the watchdog is pinged within a certain time; this time is called the
 timeout or margin.  The simplest way to ping the watchdog is to write
 some data to the device.  So a very simple watchdog daemon would look
 like this source file:  see samples/watchdog/watchdog-simple.c
 
-A more advanced driver could for example check that a HTTP server is
+A more advanced driver could for example check that an HTTP server is
 still responding before doing the write call to ping the watchdog.
 
 When the device is closed, the watchdog is disabled, unless the "Magic
@@ -87,13 +87,13 @@ replaced with::
 		sleep(10);
 	}
 
-the argument to the ioctl is ignored.
+The argument to the ioctl is ignored.
 
 Setting and getting the timeout
 ===============================
 
 For some drivers it is possible to modify the watchdog timeout on the
-fly with the SETTIMEOUT ioctl, those drivers have the WDIOF_SETTIMEOUT
+fly with the SETTIMEOUT ioctl; those drivers have the WDIOF_SETTIMEOUT
 flag set in their option field.  The argument is an integer
 representing the timeout in seconds.  The driver returns the real
 timeout used in the same variable, and this timeout might differ from
@@ -110,7 +110,7 @@ Starting with the Linux 2.4.18 kernel, it is possible to query the
 current timeout using the GETTIMEOUT ioctl::
 
     ioctl(fd, WDIOC_GETTIMEOUT, &timeout);
-    printf("The timeout was is %d seconds\n", timeout);
+    printf("The timeout is %d seconds\n", timeout);
 
 Pretimeouts
 ===========
@@ -133,7 +133,7 @@ seconds.  Setting a pretimeout to zero disables it.
 There is also a get function for getting the pretimeout::
 
     ioctl(fd, WDIOC_GETPRETIMEOUT, &timeout);
-    printf("The pretimeout was is %d seconds\n", timeout);
+    printf("The pretimeout is %d seconds\n", timeout);
 
 Not all watchdog drivers will support a pretimeout.
 
@@ -145,28 +145,28 @@ before the system will reboot. The WDIOC_GETTIMELEFT is the ioctl
 that returns the number of seconds before reboot::
 
     ioctl(fd, WDIOC_GETTIMELEFT, &timeleft);
-    printf("The timeout was is %d seconds\n", timeleft);
+    printf("The timeout is %d seconds\n", timeleft);
 
 Environmental monitoring
 ========================
 
-All watchdog drivers are required return more information about the system,
-some do temperature, fan and power level monitoring, some can tell you
+All watchdog drivers are required to return more information about the system.
+Some do temperature, fan and power level monitoring; some can tell you
 the reason for the last reboot of the system.  The GETSUPPORT ioctl is
 available to ask what the device can do::
 
 	struct watchdog_info ident;
 	ioctl(fd, WDIOC_GETSUPPORT, &ident);
 
-the fields returned in the ident struct are:
+the fields returned in the struct watchdog_info are:
 
 	================	=============================================
-        identity		a string identifying the watchdog driver
+	identity		a string identifying the watchdog driver
 	firmware_version	the firmware version of the card if available
 	options			a flags describing what the device supports
 	================	=============================================
 
-the options field can have the following bits set, and describes what
+The options field can have the following bits set, and describes what
 kind of information that the GET_STATUS and GET_BOOT_STATUS ioctls can
 return.
 
@@ -175,13 +175,13 @@ return.
 	================	=========================
 
 The machine was last rebooted by the watchdog because the thermal limit was
-exceeded:
+exceeded.
 
 	==============		==========
 	WDIOF_FANFAULT		Fan failed
 	==============		==========
 
-A system fan monitored by the watchdog card has failed
+A system fan monitored by the watchdog card has failed.
 
 	=============		================
 	WDIOF_EXTERN1		External relay 1
@@ -195,26 +195,26 @@ a reset.
 	WDIOF_EXTERN2		External relay 2
 	=============		================
 
-External monitoring relay/source 2 was triggered
+External monitoring relay/source 2 was triggered.
 
 	================	=====================
 	WDIOF_POWERUNDER	Power bad/power fault
 	================	=====================
 
-The machine is showing an undervoltage status
+The machine is showing an undervoltage status.
 
 	===============		=============================
 	WDIOF_CARDRESET		Card previously reset the CPU
 	===============		=============================
 
-The last reboot was caused by the watchdog card
+The last reboot was caused by the watchdog card.
 
 	================	=====================
 	WDIOF_POWEROVER		Power over voltage
 	================	=====================
 
 The machine is showing an overvoltage status. Note that if one level is
-under and one over both bits will be set - this may seem odd but makes
+under and one over, both bits will be set - this may seem odd but makes
 sense.
 
 	===================	=====================
@@ -227,12 +227,14 @@ The watchdog saw a keepalive ping since it was last queried.
 	WDIOF_SETTIMEOUT	Can set/get the timeout
 	================	=======================
 
-The watchdog can do pretimeouts.
+The watchdog can get/set the timeout.
 
 	================	================================
 	WDIOF_PRETIMEOUT	Pretimeout (in seconds), get/set
 	================	================================
 
+The watchdog can do pretimeouts.
+
 
 For those drivers that return any bits set in the option field, the
 GETSTATUS and GETBOOTSTATUS ioctls can be used to ask for the current
@@ -255,7 +257,7 @@ returned value is the temperature in degrees Fahrenheit::
     ioctl(fd, WDIOC_GETTEMP, &temperature);
 
 Finally the SETOPTIONS ioctl can be used to control some aspects of
-the cards operation::
+the card's operation::
 
     int options = 0;
     ioctl(fd, WDIOC_SETOPTIONS, &options);
@@ -268,4 +270,13 @@ The following options are available:
 	WDIOS_TEMPPANIC		Kernel panic on temperature trip
 	=================	================================
 
-[FIXME -- better explanations]
+The WDIOS_DISABLECARD option allows you to stop the watchdog timer via
+software. Note that this will only work if the "nowayout" module parameter
+(or CONFIG_WATCHDOG_NOWAYOUT) is not set.
+
+The WDIOS_ENABLECARD option turns on the watchdog timer.
+
+The WDIOS_TEMPPANIC option is used by some drivers (like the pcwd driver)
+to trigger a system halt (typically a kernel panic or power off) when the
+temperature trip point is reached. This ensures that the system is halted
+immediately in case of an overheat condition.
diff --git a/Documentation/watchdog/watchdog-kernel-api.rst b/Documentation/watchdog/watchdog-kernel-api.rst
index 5649c54cf6fbe..b6a70e6d28d87 100644
--- a/Documentation/watchdog/watchdog-kernel-api.rst
+++ b/Documentation/watchdog/watchdog-kernel-api.rst
@@ -38,8 +38,8 @@ The watchdog_unregister_device routine deregisters a registered watchdog timer
 device. The parameter of this routine is the pointer to the registered
 watchdog_device structure.
 
-The watchdog subsystem includes an registration deferral mechanism,
-which allows you to register an watchdog as early as you wish during
+The watchdog subsystem includes a registration deferral mechanism,
+which allows you to register a watchdog as early as you wish during
 the boot process.
 
 The watchdog device structure looks like this::
@@ -60,13 +60,14 @@ The watchdog device structure looks like this::
 	unsigned int max_hw_heartbeat_ms;
 	struct notifier_block reboot_nb;
 	struct notifier_block restart_nb;
+	struct notifier_block pm_nb;
 	void *driver_data;
 	struct watchdog_core_data *wd_data;
 	unsigned long status;
 	struct list_head deferred;
   };
 
-It contains following fields:
+It contains the following fields:
 
 * id: set by watchdog_register_device, id 0 is special. It has both a
   /dev/watchdog0 cdev (dynamic major, minor 0) as well as the old
@@ -105,6 +106,8 @@ It contains following fields:
   internal use only. If a watchdog is capable of restarting the machine, it
   should define ops->restart. Priority can be changed through
   watchdog_set_restart_priority.
+* pm_nb: coordinates watchdog_dev_suspend/resume to cancel a ping worker
+  during suspend and restore it during resume.
 * bootstatus: status of the device after booting (reported with watchdog
   WDIOF_* status bits).
 * driver_data: a pointer to the drivers private data of a watchdog device.
@@ -204,7 +207,7 @@ they are supported. These optional routines/operations are:
   If the watchdog driver does not have to perform any action but setting the
   watchdog_device.timeout, this callback can be omitted.
 
-  If set_timeout is not provided but, WDIOF_SETTIMEOUT is set, the watchdog
+  If set_timeout is not provided but WDIOF_SETTIMEOUT is set, the watchdog
   infrastructure updates the timeout value of the watchdog_device internally
   to the requested value.
 
@@ -220,7 +223,7 @@ they are supported. These optional routines/operations are:
   the watchdog". A value of 0 disables pretimeout notification.
 
   (Note: the WDIOF_PRETIMEOUT needs to be set in the options field of the
-  watchdog's info structure).
+  watchdog's info structure.)
 
   If the watchdog driver does not have to perform any action but setting the
   watchdog_device.pretimeout, this callback can be omitted. That means if
@@ -239,7 +242,7 @@ they are supported. These optional routines/operations are:
 The status bits should (preferably) be set with the set_bit and clear_bit alike
 bit-operations. The status bits that are defined are:
 
-* WDOG_ACTIVE: this status bit indicates whether or not a watchdog timer device
+* WDOG_ACTIVE: this status bit indicates whether a watchdog timer device
   is active or not from user perspective. User space is expected to send
   heartbeat requests to the driver while this flag is set.
 * WDOG_NO_WAY_OUT: this bit stores the nowayout setting for the watchdog.
@@ -254,10 +257,13 @@ bit-operations. The status bits that are defined are:
   then opening /dev/watchdog will skip the start operation but send a keepalive
   request instead.
 
+Helper Functions
+~~~~~~~~~~~~~~~~
+
   To set the WDOG_NO_WAY_OUT status bit (before registering your watchdog
   timer device) you can either:
 
-  * set it statically in your watchdog_device struct with
+  * set it statically in your struct watchdog_device with
 
 	.status = WATCHDOG_NOWAYOUT_INIT_STATUS,
 
@@ -331,7 +337,7 @@ To raise a pretimeout notification, the following function should be used::
   void watchdog_notify_pretimeout(struct watchdog_device *wdd)
 
 The function can be called in the interrupt context. If watchdog pretimeout
-governor framework (kbuild CONFIG_WATCHDOG_PRETIMEOUT_GOV symbol) is enabled,
+governor framework (kconfig CONFIG_WATCHDOG_PRETIMEOUT_GOV symbol) is enabled,
 an action is taken by a preconfigured pretimeout governor preassigned to
 the watchdog device. If watchdog pretimeout governor framework is not
 enabled, watchdog_notify_pretimeout() prints a notification message to
diff --git a/Documentation/watchdog/watchdog-parameters.rst b/Documentation/watchdog/watchdog-parameters.rst
index 773241ed99867..ec5f67c060cbd 100644
--- a/Documentation/watchdog/watchdog-parameters.rst
+++ b/Documentation/watchdog/watchdog-parameters.rst
@@ -14,13 +14,22 @@ modules.
 -------------------------------------------------
 
 watchdog core:
+    handle_boot_enabled:
+	Watchdog core auto-updates boot-enabled watchdogs before userspace
+	takes over. Default is set by the kconfig option
+	CONFIG_WATCHDOG_HANDLE_BOOT_ENABLED.
+
     open_timeout:
 	Maximum time, in seconds, for which the watchdog framework will take
 	care of pinging a running hardware watchdog until userspace opens the
 	corresponding /dev/watchdogN device. A value of 0 means an infinite
 	timeout. Setting this to a non-zero value can be useful to ensure that
 	either userspace comes up properly, or the board gets reset and allows
-	fallback logic in the bootloader to try something else.
+	fallback logic in the bootloader to try something else. Default is set
+	by the kconfig option CONFIG_WATCHDOG_OPEN_TIMEOUT.
+
+    stop_on_reboot:
+	Stops watchdogs on reboot (0 = keep watching, 1 = stop).
 
 -------------------------------------------------
 
@@ -284,16 +293,6 @@ ixp4xx_wdt:
 
 -------------------------------------------------
 
-machzwd:
-    nowayout:
-	Watchdog cannot be stopped once started
-	(default=kernel config parameter)
-    action:
-	after watchdog resets, generate:
-	0 = RESET(*)  1 = SMI  2 = NMI  3 = SCI
-
--------------------------------------------------
-
 max63xx_wdt:
     heartbeat:
 	Watchdog heartbeat period in seconds from 1 to 60, default 60
@@ -532,15 +531,6 @@ sc1200wdt:
 
 -------------------------------------------------
 
-sc520_wdt:
-    timeout:
-	Watchdog timeout in seconds. (1 <= timeout <= 3600, default=30)
-    nowayout:
-	Watchdog cannot be stopped once started
-	(default=kernel config parameter)
-
--------------------------------------------------
-
 sch311x_wdt:
     force_id:
 	Override the detected device ID
diff --git a/Documentation/watchdog/wdt.rst b/Documentation/watchdog/wdt.rst
deleted file mode 100644
index d97b0361535b0..0000000000000
--- a/Documentation/watchdog/wdt.rst
+++ /dev/null
@@ -1,63 +0,0 @@
-============================================================
-WDT Watchdog Timer Interfaces For The Linux Operating System
-============================================================
-
-Last Reviewed: 10/05/2007
-
-Alan Cox <alan@lxorguk.ukuu.org.uk>
-
-	- ICS	WDT501-P
-	- ICS	WDT501-P (no fan tachometer)
-	- ICS	WDT500-P
-
-All the interfaces provide /dev/watchdog, which when open must be written
-to within a timeout or the machine will reboot. Each write delays the reboot
-time another timeout. In the case of the software watchdog the ability to
-reboot will depend on the state of the machines and interrupts. The hardware
-boards physically pull the machine down off their own onboard timers and
-will reboot from almost anything.
-
-A second temperature monitoring interface is available on the WDT501P cards.
-This provides /dev/temperature. This is the machine internal temperature in
-degrees Fahrenheit. Each read returns a single byte giving the temperature.
-
-The third interface logs kernel messages on additional alert events.
-
-The ICS ISA-bus wdt card cannot be safely probed for. Instead you need to
-pass IO address and IRQ boot parameters.  E.g.::
-
-	wdt.io=0x240 wdt.irq=11
-
-Other "wdt" driver parameters are:
-
-	===========	======================================================
-	heartbeat	Watchdog heartbeat in seconds (default 60)
-	nowayout	Watchdog cannot be stopped once started (kernel
-			build parameter)
-	tachometer	WDT501-P Fan Tachometer support (0=disable, default=0)
-	type		WDT501-P Card type (500 or 501, default=500)
-	===========	======================================================
-
-Features
---------
-
-================   =======	   =======
-		   WDT501P	   WDT500P
-================   =======	   =======
-Reboot Timer	   X               X
-External Reboot	   X	           X
-I/O Port Monitor   o		   o
-Temperature	   X		   o
-Fan Speed          X		   o
-Power Under	   X               o
-Power Over         X               o
-Overheat           X               o
-================   =======	   =======
-
-The external event interfaces on the WDT boards are not currently supported.
-Minor numbers are however allocated for it.
-
-
-Example Watchdog Driver:
-
-	see samples/watchdog/watchdog-simple.c
diff --git a/Documentation/wmi/devices/uniwill-laptop.rst b/Documentation/wmi/devices/uniwill-laptop.rst
index e246bf293450b..65583b2397061 100644
--- a/Documentation/wmi/devices/uniwill-laptop.rst
+++ b/Documentation/wmi/devices/uniwill-laptop.rst
@@ -189,7 +189,7 @@ Indexed IO
 
 Indexed IO with IO ports with a granularity of a single byte can be performed using the ``RIOP``
 (read) and ``WIOP`` (write) ACPI control methods. Those ACPI methods are unused because they
-provide no benifit when compared to the native IO port access functions provided by the kernel.
+provide no benefit when compared to the native IO port access functions provided by the kernel.
 
 Special thanks go to github user `pobrn` which developed the
 `qc71_laptop <https://github.com/pobrn/qc71_laptop>`_ driver on which this driver is partly based.
diff --git a/MAINTAINERS b/MAINTAINERS
index c2c6d79275c6e..e706b468a53fd 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -25,9 +25,9 @@ Descriptions of section entries and preferred order
 	C: URI for *chat* protocol, server and channel where developers
 	   usually hang out, for example irc://server/channel.
 	P: *Subsystem Profile* document for more details submitting
-	   patches to the given subsystem. This is either an in-tree file,
-	   or a URI. See Documentation/maintainer/maintainer-entry-profile.rst
-	   for details.
+	   patches to the given subsystem. This is either an in-tree .rst file
+       inside Documentation/, or a URI.
+       See Documentation/maintainer/maintainer-entry-profile.rst for details.
 	T: *SCM* tree type and location.
 	   Type is one of: git, hg, quilt, stgit, topgit
 	F: *Files* and directories wildcard patterns.
@@ -297,6 +297,7 @@ F:	include/linux/acpi.h
 F:	include/linux/fwnode.h
 F:	include/linux/fw_table.h
 F:	lib/fw_table.c
+F:	rust/helpers/acpi.c
 F:	rust/kernel/acpi.rs
 F:	tools/power/acpi/
 
@@ -639,7 +640,7 @@ F:	drivers/iio/accel/adxl355_i2c.c
 F:	drivers/iio/accel/adxl355_spi.c
 
 ADXL367 THREE-AXIS DIGITAL ACCELEROMETER DRIVER
-M:	Cosmin Tanislav <cosmin.tanislav@analog.com>
+M:	Marcelo Schmitt <marcelo.schmitt@analog.com>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
@@ -808,6 +809,14 @@ S:	Maintained
 F:	Documentation/i2c/busses/i2c-ali1563.rst
 F:	drivers/i2c/busses/i2c-ali1563.c
 
+ALIBABA ELASTIC ETHERNET ADAPTER DRIVER
+M:	Xuan Zhuo <xuanzhuo@linux.alibaba.com>
+M:	Wen Gu <guwen@linux.alibaba.com>
+R:	Philo Lu <lulie@linux.alibaba.com>
+L:	netdev@vger.kernel.org
+S:	Maintained
+F:	drivers/net/ethernet/alibaba/eea
+
 ALIBABA ELASTIC RDMA DRIVER
 M:	Cheng Xu <chengyou@linux.alibaba.com>
 M:	Kai Shen <kaishen@linux.alibaba.com>
@@ -1121,10 +1130,9 @@ S:	Orphan
 F:	drivers/usb/gadget/udc/amd5536udc.*
 
 AMD GEODE PROCESSOR/CHIPSET SUPPORT
-M:	Andres Salomon <dilinger@queued.net>
 L:	linux-geode@lists.infradead.org (moderated for non-subscribers)
-S:	Supported
-W:	http://www.amd.com/us-en/ConnectivitySolutions/TechnicalResources/0,,50_2334_2452_11363,00.html
+S:	Orphaned
+F:	arch/x86/platform/geode/
 F:	arch/x86/include/asm/geode.h
 F:	drivers/char/hw_random/geode-rng.c
 F:	drivers/crypto/geode*
@@ -1158,6 +1166,31 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git
 F:	drivers/iommu/amd/
 F:	include/linux/amd-iommu.h
 
+AMD ISP4 DRIVER
+M:	Bin Du <bin.du@amd.com>
+M:	Nirujogi Pratap <pratap.nirujogi@amd.com>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+T:	git git://linuxtv.org/media.git
+F:	Documentation/admin-guide/media/amdisp4-1.rst
+F:	Documentation/admin-guide/media/amdisp4.dot
+F:	drivers/media/platform/amd/Kconfig
+F:	drivers/media/platform/amd/Makefile
+F:	drivers/media/platform/amd/isp4/Kconfig
+F:	drivers/media/platform/amd/isp4/Makefile
+F:	drivers/media/platform/amd/isp4/isp4.c
+F:	drivers/media/platform/amd/isp4/isp4.h
+F:	drivers/media/platform/amd/isp4/isp4_debug.c
+F:	drivers/media/platform/amd/isp4/isp4_debug.h
+F:	drivers/media/platform/amd/isp4/isp4_fw_cmd_resp.h
+F:	drivers/media/platform/amd/isp4/isp4_hw_reg.h
+F:	drivers/media/platform/amd/isp4/isp4_interface.c
+F:	drivers/media/platform/amd/isp4/isp4_interface.h
+F:	drivers/media/platform/amd/isp4/isp4_subdev.c
+F:	drivers/media/platform/amd/isp4/isp4_subdev.h
+F:	drivers/media/platform/amd/isp4/isp4_video.c
+F:	drivers/media/platform/amd/isp4/isp4_video.h
+
 AMD KFD
 M:	Felix Kuehling <Felix.Kuehling@amd.com>
 L:	amd-gfx@lists.freedesktop.org
@@ -1458,7 +1491,7 @@ F:	Documentation/devicetree/bindings/iio/adc/adi,ad4080.yaml
 F:	drivers/iio/adc/ad4080.c
 
 ANALOG DEVICES INC AD4130 DRIVER
-M:	Cosmin Tanislav <cosmin.tanislav@analog.com>
+M:	Marcelo Schmitt <marcelo.schmitt@analog.com>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
@@ -1482,6 +1515,15 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/iio/adc/adi,ad4170-4.yaml
 F:	drivers/iio/adc/ad4170-4.c
 
+ANALOG DEVICES INC AD4691 DRIVER
+M:	Radu Sabau <radu.sabau@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/iio/adc/adi,ad4691.yaml
+F:	drivers/iio/adc/ad4691.c
+F:	drivers/iio/adc/ad4691.rst
+
 ANALOG DEVICES INC AD4695 DRIVER
 M:	Michael Hennerich <michael.hennerich@analog.com>
 M:	Nuno Sá <nuno.sa@analog.com>
@@ -1504,6 +1546,14 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/iio/adc/adi,ad4851.yaml
 F:	drivers/iio/adc/ad4851.c
 
+ANALOG DEVICES INC AD5706R DRIVER
+M:	Alexis Czezar Torreno <alexisczezar.torreno@analog.com>
+L:	linux-iio@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/iio/dac/adi,ad5706r.yaml
+F:	drivers/iio/dac/ad5706r.c
+
 ANALOG DEVICES INC AD7091R DRIVER
 M:	Marcelo Schmitt <marcelo.schmitt@analog.com>
 L:	linux-iio@vger.kernel.org
@@ -1546,7 +1596,7 @@ F:	Documentation/devicetree/bindings/iio/dac/adi,ad7293.yaml
 F:	drivers/iio/dac/ad7293.c
 
 ANALOG DEVICES INC AD74115 DRIVER
-M:	Cosmin Tanislav <cosmin.tanislav@analog.com>
+M:	Marcelo Schmitt <marcelo.schmitt@analog.com>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
@@ -1554,7 +1604,7 @@ F:	Documentation/devicetree/bindings/iio/addac/adi,ad74115.yaml
 F:	drivers/iio/addac/ad74115.c
 
 ANALOG DEVICES INC AD74413R DRIVER
-M:	Cosmin Tanislav <cosmin.tanislav@analog.com>
+M:	Marcelo Schmitt <marcelo.schmitt@analog.com>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 W:	https://ez.analog.com/linux-software-drivers
@@ -1850,8 +1900,9 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	drivers/dma/dma-axi-dmac.c
 
 ANALOG DEVICES INC IIO DRIVERS
-M:	Lars-Peter Clausen <lars@metafoo.de>
+M:	Nuno Sá <nuno.sa@analog.com>
 M:	Michael Hennerich <Michael.Hennerich@analog.com>
+L:	linux@analog.com
 S:	Supported
 W:	http://wiki.analog.com/
 W:	https://ez.analog.com/linux-software-drivers
@@ -1880,6 +1931,12 @@ S:	Supported
 F:	Documentation/devicetree/bindings/spi/andestech,ae350-spi.yaml
 F:	drivers/spi/spi-atcspi200.c
 
+ANDES ATCWDT200 WATCHDOG DRIVER
+M:	CL Wang <cl634@andestech.com>
+S:	Supported
+F:	Documentation/devicetree/bindings/watchdog/andestech,ae350-wdt.yaml
+F:	drivers/watchdog/atcwdt200_wdt.c
+
 ANDROID DRIVERS
 M:	Greg Kroah-Hartman <gregkh@linuxfoundation.org>
 M:	Arve Hjønnevåg <arve@android.com>
@@ -2070,6 +2127,13 @@ S:	Maintained
 F:	drivers/net/arcnet/
 F:	include/uapi/linux/if_arcnet.h
 
+ARCTIC FAN CONTROLLER DRIVER
+M:	Aureo Serrano de Souza <aureo.serrano@arctic.de>
+L:	linux-hwmon@vger.kernel.org
+S:	Maintained
+F:	Documentation/hwmon/arctic_fan_controller.rst
+F:	drivers/hwmon/arctic_fan_controller.c
+
 ARM AND ARM64 SoC SUB-ARCHITECTURES (COMMON PARTS)
 M:	Arnd Bergmann <arnd@arndb.de>
 M:	Krzysztof Kozlowski <krzk@kernel.org>
@@ -3161,7 +3225,7 @@ M:	Linus Walleij <linusw@kernel.org>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-nomadik.git
-F:	Documentation/devicetree/bindings/arm/ste-*
+F:	Documentation/devicetree/bindings/arm/st,nomadik.yaml
 F:	Documentation/devicetree/bindings/arm/ux500.yaml
 F:	Documentation/devicetree/bindings/arm/ux500/
 F:	Documentation/devicetree/bindings/gpio/st,nomadik-gpio.yaml
@@ -3343,6 +3407,7 @@ F:	arch/arm64/boot/dts/qcom/
 F:	drivers/bus/qcom*
 F:	drivers/firmware/qcom/
 F:	drivers/soc/qcom/
+F:	drivers/watchdog/gunyah_wdt.c
 F:	include/dt-bindings/arm/qcom,ids.h
 F:	include/dt-bindings/firmware/qcom,scm.h
 F:	include/dt-bindings/soc/qcom*
@@ -3367,7 +3432,9 @@ F:	drivers/irqchip/irq-rda-intc.c
 F:	drivers/tty/serial/rda-uart.c
 
 ARM/REALTEK ARCHITECTURE
-M:	Andreas Färber <afaerber@suse.de>
+M:	James Tai <james.tai@realtek.com>
+M:	Yu-Chun Lin <eleanor.lin@realtek.com>
+R:	Andreas Färber <afaerber@suse.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 L:	linux-realtek-soc@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
@@ -3375,6 +3442,7 @@ F:	Documentation/devicetree/bindings/arm/realtek.yaml
 F:	arch/arm/boot/dts/realtek/
 F:	arch/arm/mach-realtek/
 F:	arch/arm64/boot/dts/realtek/
+F:	drivers/pinctrl/realtek/
 
 ARM/RISC-V/RENESAS ARCHITECTURE
 M:	Geert Uytterhoeven <geert+renesas@glider.be>
@@ -3431,6 +3499,8 @@ F:	drivers/i2c/busses/i2c-rk3x.c
 F:	drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c
 F:	sound/soc/rockchip/
 N:	rockchip
+K:	\b[Rr]ock[Cc]hip\b
+K:	(?:\b|_)ROCKCHIP(?:\b|_)
 
 ARM/SAMSUNG S3C, S5P AND EXYNOS ARM ARCHITECTURES
 M:	Krzysztof Kozlowski <krzk@kernel.org>
@@ -3788,6 +3858,15 @@ F:	drivers/video/fbdev/vt8500lcdfb.*
 F:	drivers/video/fbdev/wm8505fb*
 F:	drivers/video/fbdev/wmt_ge_rops.*
 
+ARM/ZTE ZX29 SOC SUPPORT
+M:	Stefan Dösinger <stefandoesinger@gmail.com>
+L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
+S:	Odd fixes
+F:	Documentation/arch/arm/zte/
+F:	Documentation/devicetree/bindings/arm/zte.yaml
+F:	arch/arm/boot/dts/zte/
+F:	arch/arm/mach-zte/
+
 ARM/ZYNQ ARCHITECTURE
 M:	Michal Simek <michal.simek@amd.com>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
@@ -4298,6 +4377,12 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/iio/adc/avia-hx711.yaml
 F:	drivers/iio/adc/hx711.c
 
+AVMATRIX HWS CAPTURE DRIVER
+M:	Ben Hoff <hoff.benjamin.k@gmail.com>
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	drivers/media/pci/hws/
+
 AWINIC AW99706 WLED BACKLIGHT DRIVER
 M:	Junjie Cao <caojunjie650@gmail.com>
 S:	Maintained
@@ -4338,6 +4423,16 @@ W:	https://ez.analog.com/linux-software-drivers
 F:	Documentation/devicetree/bindings/hwmon/adi,axi-fan-control.yaml
 F:	drivers/hwmon/axi-fan-control.c
 
+AXIADO EMMC PHY DRIVER
+M:	SriNavmani A <srinavmani@axiado.com>
+M:	Tzu-Hao Wei <twei@axiado.com>
+M:	Prasad Bolisetty <pbolisetty@axiado.com>
+L:	linux-phy@lists.infradead.org (moderated for non-subscribers)
+S:	Maintained
+F:	Documentation/devicetree/bindings/phy/axiado,ax3000-emmc-phy.yaml
+F:	drivers/phy/axiado/Kconfig
+F:	drivers/phy/axiado/phy-axiado-emmc.c
+
 AXI SPI ENGINE
 M:	Michael Hennerich <michael.hennerich@analog.com>
 M:	Nuno Sá <nuno.sa@analog.com>
@@ -4867,6 +4962,7 @@ F:	kernel/bpf/*iter.c
 BPF [L7 FRAMEWORK] (sockmap)
 M:	John Fastabend <john.fastabend@gmail.com>
 M:	Jakub Sitnicki <jakub@cloudflare.com>
+M:	Jiayuan Chen <jiayuan.chen@linux.dev>
 L:	netdev@vger.kernel.org
 L:	bpf@vger.kernel.org
 S:	Maintained
@@ -5966,8 +6062,6 @@ F:	drivers/input/keyboard/charlieplex_keypad.c
 CHECKPATCH
 M:	Andy Whitcroft <apw@canonical.com>
 M:	Joe Perches <joe@perches.com>
-R:	Dwaipayan Ray <dwaipayanray1@gmail.com>
-R:	Lukas Bulwahn <lukas.bulwahn@gmail.com>
 S:	Maintained
 F:	scripts/checkpatch.pl
 
@@ -6479,6 +6573,7 @@ F:	drivers/gpio/gpio-cgbc.c
 F:	drivers/hwmon/cgbc-hwmon.c
 F:	drivers/i2c/busses/i2c-cgbc.c
 F:	drivers/mfd/cgbc-core.c
+F:	drivers/video/backlight/cgbc_bl.c
 F:	drivers/watchdog/cgbc_wdt.c
 F:	include/linux/mfd/cgbc.h
 
@@ -7846,11 +7941,13 @@ F:	drivers/soc/ti/smartreflex.c
 F:	include/linux/power/smartreflex.h
 
 DRM ACCEL DRIVERS FOR INTEL VPU
-M:	Maciej Falkowski <maciej.falkowski@linux.intel.com>
 M:	Karol Wachowski <karol.wachowski@linux.intel.com>
+M:	Andrzej Kacprowski <andrzej.kacprowski@linux.intel.com>
 L:	dri-devel@lists.freedesktop.org
 S:	Supported
 T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
+F:	Documentation/ABI/obsolete/sysfs-driver-ivpu
+F:	Documentation/ABI/testing/sysfs-driver-ivpu
 F:	drivers/accel/ivpu/
 F:	include/uapi/drm/ivpu_accel.h
 
@@ -8159,6 +8256,12 @@ T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	Documentation/devicetree/bindings/display/panel/novatek,nt35510.yaml
 F:	drivers/gpu/drm/panel/panel-novatek-nt35510.c
 
+DRM DRIVER FOR NOVATEK NT35532 PANELS
+M:	Cristian Cozzolino <cristian_ci@protonmail.com>
+S:	Maintained
+F:	Documentation/devicetree/bindings/display/panel/novatek,nt35532.yaml
+F:	drivers/gpu/drm/panel/panel-novatek-nt35532.c
+
 DRM DRIVER FOR NOVATEK NT35560 PANELS
 M:	Linus Walleij <linusw@kernel.org>
 S:	Maintained
@@ -8567,6 +8670,7 @@ M:	Robert Foss <rfoss@kernel.org>
 R:	Laurent Pinchart <Laurent.pinchart@ideasonboard.com>
 R:	Jonas Karlman <jonas@kwiboo.se>
 R:	Jernej Skrabec <jernej.skrabec@gmail.com>
+R:	Luca Ceresoli <luca.ceresoli@bootlin.com>
 S:	Maintained
 T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	Documentation/devicetree/bindings/display/bridge/
@@ -8890,6 +8994,8 @@ S:	Supported
 T:	git https://gitlab.freedesktop.org/drm/misc/kernel.git
 F:	drivers/gpu/drm/drm_gpuvm.c
 F:	include/drm/drm_gpuvm.h
+F:	rust/helpers/drm_gpuvm.c
+F:	rust/kernel/drm/gpuvm/
 
 DRM LOG
 M:	Jocelyn Falempe <jfalempe@redhat.com>
@@ -9152,6 +9258,13 @@ F:	drivers/irqchip/irq-econet-en751221.c
 F:	include/dt-bindings/clock/econet,en751221-scu.h
 F:	include/dt-bindings/reset/econet,en751221-scu.h
 
+ECONET PCIE PHY DRIVER
+M:	Caleb James DeLisle <cjd@cjdns.fr>
+L:	linux-mips@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/phy/econet,en751221-pcie-phy.yaml
+F:	drivers/phy/phy-econet-pcie.c
+
 ECRYPT FILE SYSTEM
 M:	Tyler Hicks <code@tyhicks.com>
 L:	ecryptfs@vger.kernel.org
@@ -9862,7 +9975,7 @@ F:	include/uapi/scsi/fc/
 
 FILE LOCKING (flock() and fcntl()/lockf())
 M:	Jeff Layton <jlayton@kernel.org>
-M:	Chuck Lever <chuck.lever@oracle.com>
+M:	Chuck Lever <cel@kernel.org>
 R:	Alexander Aring <alex.aring@gmail.com>
 L:	linux-fsdevel@vger.kernel.org
 S:	Maintained
@@ -9903,7 +10016,7 @@ F:	init/do_mounts*
 F:	init/*initramfs*
 
 FILESYSTEMS [EXPORTFS]
-M:	Chuck Lever <chuck.lever@oracle.com>
+M:	Chuck Lever <cel@kernel.org>
 M:	Jeff Layton <jlayton@kernel.org>
 R:	Amir Goldstein <amir73il@gmail.com>
 L:	linux-fsdevel@vger.kernel.org
@@ -9912,7 +10025,7 @@ S:	Supported
 F:	Documentation/filesystems/nfs/exporting.rst
 F:	fs/exportfs/
 F:	fs/fhandle.c
-F:	include/linux/exportfs.h
+F:	include/linux/exportfs*.h
 
 FILESYSTEMS [IDMAPPED MOUNTS]
 M:	Christian Brauner <brauner@kernel.org>
@@ -10583,10 +10696,10 @@ L:	netdev@vger.kernel.org
 S:	Maintained
 F:	drivers/net/ethernet/fungible/
 
-FUSE: FILESYSTEM IN USERSPACE
+FUSE FILESYSTEM [CORE]
 M:	Miklos Szeredi <miklos@szeredi.hu>
-L:	linux-fsdevel@vger.kernel.org
-S:	Maintained
+L:	fuse-devel@lists.linux.dev
+S:	Supported
 W:	https://github.com/libfuse/
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git
 F:	Documentation/filesystems/fuse/*
@@ -10594,6 +10707,26 @@ F:	fs/fuse/
 F:	include/uapi/linux/fuse.h
 F:	tools/testing/selftests/filesystems/fuse/
 
+FUSE FILESYSTEM [IO-URING]
+M:	Bernd Schubert <bernd@bsbernd.com>
+M:	Joanne Koong <joannelkoong@gmail.com>
+L:	fuse-devel@lists.linux.dev
+S:	Maintained
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git
+F:	Documentation/filesystems/fuse/fuse-io-uring.rst
+F:	fs/fuse/dev_uring.c
+F:	fs/fuse/dev_uring_i.h
+
+FUSE FILESYSTEM [PASSTHROUGH]
+M:	Amir Goldstein <amir73il@gmail.com>
+L:	fuse-devel@lists.linux.dev
+S:	Maintained
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git
+F:	Documentation/filesystems/fuse/fuse-passthrough.rst
+F:	fs/fuse/backing.c
+F:	fs/fuse/iomode.c
+F:	fs/fuse/passthrough.c
+
 FUTEX SUBSYSTEM
 M:	Thomas Gleixner <tglx@kernel.org>
 M:	Ingo Molnar <mingo@redhat.com>
@@ -10832,7 +10965,7 @@ F:	include/linux/generic-radix-tree.h
 F:	lib/generic-radix-tree.c
 
 GENERIC RESISTIVE TOUCHSCREEN ADC DRIVER
-M:	Eugen Hristev <eugen.hristev@microchip.com>
+M:	Eugen Hristev <ehristev@kernel.org>
 L:	linux-input@vger.kernel.org
 S:	Maintained
 F:	drivers/input/touchscreen/resistive-adc-touch.c
@@ -11251,7 +11384,7 @@ Q:	http://patchwork.linuxtv.org/project/linux-media/list/
 F:	drivers/media/usb/hackrf/
 
 HANDSHAKE UPCALL FOR TRANSPORT LAYER SECURITY
-M:	Chuck Lever <chuck.lever@oracle.com>
+M:	Chuck Lever <cel@kernel.org>
 L:	kernel-tls-handshake@lists.linux.dev
 L:	netdev@vger.kernel.org
 S:	Maintained
@@ -11382,13 +11515,6 @@ F:	Documentation/filesystems/hfsplus.rst
 F:	fs/hfsplus/
 F:	include/linux/hfs_common.h
 
-HGA FRAMEBUFFER DRIVER
-M:	Ferenc Bakonyi <fero@drama.obuda.kando.hu>
-L:	linux-nvidia@lists.surfsouth.com
-S:	Maintained
-W:	http://drama.obuda.kando.hu/~fero/cgi-bin/hgafb.shtml
-F:	drivers/video/fbdev/hgafb.c
-
 HIBERNATION (aka Software Suspend, aka swsusp)
 M:	"Rafael J. Wysocki" <rafael@kernel.org>
 R:	Pavel Machek <pavel@kernel.org>
@@ -11634,7 +11760,7 @@ F:	Documentation/devicetree/bindings/net/hisilicon*.txt
 F:	drivers/net/ethernet/hisilicon/
 
 HISILICON PMU DRIVER
-M:	Jonathan Cameron <jonathan.cameron@huawei.com>
+M:	Yushan Wang <wangyushan12@huawei.com>
 S:	Supported
 W:	http://www.hisilicon.com
 F:	Documentation/admin-guide/perf/hisi-pcie-pmu.rst
@@ -11763,6 +11889,15 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/iio/pressure/honeywell,mprls0025pa.yaml
 F:	drivers/iio/pressure/mprls0025pa*
 
+HORNET SECURITY MODULE
+M:	Blaise Boscaccy <bboscaccy@linux.microsoft.com>
+L:	linux-security-module@vger.kernel.org
+S:	Supported
+T:	git https://github.com/blaiseboscaccy/hornet.git
+F:	Documentation/admin-guide/LSM/Hornet.rst
+F:	scripts/hornet/
+F:	security/hornet/
+
 HP BIOSCFG DRIVER
 M:	Jorge Lopez <jorge.lopez2@hp.com>
 L:	platform-driver-x86@vger.kernel.org
@@ -13869,6 +14004,7 @@ M:	Pratyush Yadav <pratyush@kernel.org>
 R:	Dave Young <ruirui.yang@linux.dev>
 L:	kexec@lists.infradead.org
 S:	Maintained
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
 F:	Documentation/admin-guide/kdump/
 F:	fs/proc/vmcore.c
 F:	include/linux/crash_core.h
@@ -13937,7 +14073,7 @@ F:	scripts/Makefile.randstruct
 F:	security/Kconfig.hardening
 K:	\b(add|choose)_random_kstack_offset\b
 K:	\b__check_(object_size|heap_object)\b
-K:	\b__counted_by(_le|_be)?\b
+K:	\b__counted_by(_le|_be|_ptr)?\b
 
 KERNEL JANITORS
 L:	kernel-janitors@vger.kernel.org
@@ -13945,7 +14081,7 @@ S:	Odd Fixes
 W:	http://kernelnewbies.org/KernelJanitors
 
 KERNEL NFSD, SUNRPC, AND LOCKD SERVERS
-M:	Chuck Lever <chuck.lever@oracle.com>
+M:	Chuck Lever <cel@kernel.org>
 M:	Jeff Layton <jlayton@kernel.org>
 R:	NeilBrown <neil@brown.name>
 R:	Olga Kornievskaia <okorniev@redhat.com>
@@ -14186,6 +14322,7 @@ M:	Pasha Tatashin <pasha.tatashin@soleen.com>
 M:	Pratyush Yadav <pratyush@kernel.org>
 L:	kexec@lists.infradead.org
 W:	http://kernel.org/pub/linux/utils/kernel/kexec/
+T:	git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
 F:	include/linux/kexec.h
 F:	include/uapi/linux/kexec.h
 F:	kernel/kexec*
@@ -14902,6 +15039,7 @@ LIVE UPDATE
 M:	Pasha Tatashin <pasha.tatashin@soleen.com>
 M:	Mike Rapoport <rppt@kernel.org>
 M:	Pratyush Yadav <pratyush@kernel.org>
+L:	kexec@lists.infradead.org
 L:	linux-kernel@vger.kernel.org
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git
@@ -15028,7 +15166,7 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git
 F:	drivers/hid/hid-lg-g15.c
 
 LONTIUM LT8912B MIPI TO HDMI BRIDGE
-M:	Adrien Grassein <adrien.grassein@gmail.com>
+M:	Francesco Dolcini <francesco@dolcini.it>
 S:	Maintained
 F:	Documentation/devicetree/bindings/display/bridge/lontium,lt8912b.yaml
 F:	drivers/gpu/drm/bridge/lontium-lt8912b.c
@@ -15066,6 +15204,7 @@ M:	Binbin Zhou <zhoubinbin@loongson.cn>
 L:	linux-i2c@vger.kernel.org
 S:	Maintained
 F:	Documentation/devicetree/bindings/i2c/loongson,ls2x-i2c.yaml
+F:	drivers/i2c/busses/i2c-ls2x-v2.c
 F:	drivers/i2c/busses/i2c-ls2x.c
 
 LOONGSON PWM DRIVER
@@ -15353,7 +15492,7 @@ T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git
 F:	Documentation/networking/mac80211-injection.rst
 F:	Documentation/networking/mac80211_hwsim/mac80211_hwsim.rst
-F:	drivers/net/wireless/virtual/mac80211_hwsim.[ch]
+F:	drivers/net/wireless/virtual/mac80211_hwsim*.[ch]
 F:	include/net/mac80211.h
 F:	net/mac80211/
 
@@ -15672,6 +15811,15 @@ F:	Documentation/devicetree/bindings/hwmon/pmbus/adi,max17616.yaml
 F:	Documentation/hwmon/max17616.rst
 F:	drivers/hwmon/pmbus/max17616.c
 
+MAX20830 HARDWARE MONITOR DRIVER
+M:	Alexis Czezar Torreno <alexisczezar.torreno@analog.com>
+L:	linux-hwmon@vger.kernel.org
+S:	Supported
+W:	https://ez.analog.com/linux-software-drivers
+F:	Documentation/devicetree/bindings/hwmon/pmbus/adi,max20830.yaml
+F:	Documentation/hwmon/max20830.rst
+F:	drivers/hwmon/pmbus/max20830.c
+
 MAX2175 SDR TUNER DRIVER
 M:	Ramesh Shanmugasundaram <rashanmu@gmail.com>
 L:	linux-media@vger.kernel.org
@@ -15682,6 +15830,12 @@ F:	Documentation/userspace-api/media/drivers/max2175.rst
 F:	drivers/media/i2c/max2175*
 F:	include/uapi/linux/max2175.h
 
+MAX25014 BACKLIGHT DRIVER
+M:	Maud Spierings <maudspierings@gocontroll.com>
+S:	Maintained
+F:	Documentation/devicetree/bindings/leds/backlight/maxim,max25014.yaml
+F:	drivers/video/backlight/max25014.c
+
 MAX31335 RTC DRIVER
 M:	Antoniu Miclaus <antoniu.miclaus@analog.com>
 L:	linux-rtc@vger.kernel.org
@@ -16501,7 +16655,7 @@ F:	drivers/usb/mtu3/
 
 MEGACHIPS STDPXXXX-GE-B850V3-FW LVDS/DP++ BRIDGES
 M:	Peter Senna Tschudin <peter.senna@gmail.com>
-M:	Ian Ray <ian.ray@ge.com>
+M:	Ian Ray <ian.ray@gehealthcare.com>
 M:	Martyn Welch <martyn.welch@collabora.co.uk>
 S:	Maintained
 F:	Documentation/devicetree/bindings/display/bridge/megachips-stdpxxxx-ge-b850v3-fw.txt
@@ -17180,6 +17334,13 @@ F:	drivers/mtd/
 F:	include/linux/mtd/
 F:	include/uapi/mtd/
 
+MEMSIC MMC5983 MAGNETOMETER DRIVER
+M:	Vladislav Kulikov <vlad.kulikov.c@gmail.com>
+L:	linux-iio@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/iio/magnetometer/memsic,mmc5983.yaml
+F:	drivers/iio/magnetometer/mmc5983.c
+
 MEN A21 WATCHDOG DRIVER
 M:	Johannes Thumshirn <morbidrsa@gmail.com>
 L:	linux-watchdog@vger.kernel.org
@@ -17340,7 +17501,7 @@ F:	Documentation/devicetree/bindings/sound/mikroe,mikroe-proto.txt
 F:	sound/soc/atmel
 
 MICROCHIP CSI2DC DRIVER
-M:	Eugen Hristev <eugen.hristev@microchip.com>
+M:	Eugen Hristev <ehristev@kernel.org>
 L:	linux-media@vger.kernel.org
 S:	Supported
 F:	Documentation/devicetree/bindings/media/microchip,csi2dc.yaml
@@ -17367,19 +17528,17 @@ F:	drivers/i2c/busses/i2c-at91-*.c
 F:	drivers/i2c/busses/i2c-at91.h
 
 MICROCHIP ISC DRIVER
-M:	Eugen Hristev <eugen.hristev@microchip.com>
+M:	Eugen Hristev <ehristev@kernel.org>
 L:	linux-media@vger.kernel.org
 S:	Supported
 F:	Documentation/devicetree/bindings/media/atmel,isc.yaml
 F:	Documentation/devicetree/bindings/media/microchip,xisc.yaml
 F:	drivers/media/platform/microchip/microchip-isc*
 F:	drivers/media/platform/microchip/microchip-sama*-isc*
-F:	drivers/staging/media/deprecated/atmel/atmel-isc*
-F:	drivers/staging/media/deprecated/atmel/atmel-sama*-isc*
 F:	include/linux/atmel-isc-media.h
 
 MICROCHIP ISI DRIVER
-M:	Eugen Hristev <eugen.hristev@microchip.com>
+M:	Eugen Hristev <ehristev@kernel.org>
 L:	linux-media@vger.kernel.org
 S:	Supported
 F:	drivers/media/platform/atmel/atmel-isi.c
@@ -17569,7 +17728,7 @@ F:	Documentation/devicetree/bindings/display/bridge/microchip,sam9x75-lvds.yaml
 F:	drivers/gpu/drm/bridge/microchip-lvds.c
 
 MICROCHIP SAMA5D2-COMPATIBLE ADC DRIVER
-M:	Eugen Hristev <eugen.hristev@microchip.com>
+M:	Eugen Hristev <ehristev@kernel.org>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 F:	Documentation/devicetree/bindings/iio/adc/atmel,sama5d2-adc.yaml
@@ -17958,6 +18117,7 @@ F:	arch/mips/boot/dts/mobileye/
 F:	arch/mips/configs/eyeq*_defconfig
 F:	arch/mips/mobileye/board-epm5.its.S
 F:	drivers/clk/clk-eyeq.c
+F:	drivers/phy/phy-eyeq5-eth.c
 F:	drivers/pinctrl/pinctrl-eyeq5.c
 F:	drivers/reset/reset-eyeq.c
 F:	include/dt-bindings/clock/mobileye,eyeq5-clk.h
@@ -18080,6 +18240,13 @@ S:	Maintained
 F:	Documentation/hwmon/mp29502.rst
 F:	drivers/hwmon/pmbus/mp29502.c
 
+MPS MP2985 DRIVER
+M:	Wensheng Wang <wenswang@yeah.net>
+L:	linux-hwmon@vger.kernel.org
+S:	Maintained
+F:	Documentation/hwmon/mp2985.rst
+F:	drivers/hwmon/pmbus/mp2985.c
+
 MPS MP2993 DRIVER
 M:	Noah Wang <noahwang.wang@outlook.com>
 L:	linux-hwmon@vger.kernel.org
@@ -18256,6 +18423,13 @@ F:	drivers/mux/
 F:	include/dt-bindings/mux/
 F:	include/linux/mux/
 
+MURATA D1U74T PSU DRIVER
+M:	Abdurrahman Hussain <abdurrahman@nexthop.ai>
+L:	linux-hwmon@vger.kernel.org
+S:	Maintained
+F:	Documentation/hwmon/d1u74t.rst
+F:	drivers/hwmon/pmbus/d1u74t.c
+
 MUSB MULTIPOINT HIGH SPEED DUAL-ROLE CONTROLLER
 M:	Bin Liu <b-liu@ti.com>
 L:	linux-usb@vger.kernel.org
@@ -18481,7 +18655,6 @@ F:	include/linux/fddidevice.h
 F:	include/linux/if_*
 F:	include/linux/inetdevice.h
 F:	include/linux/netdev*
-F:	include/linux/platform_data/wiznet.h
 F:	include/uapi/linux/cn_proc.h
 F:	include/uapi/linux/ethtool_netlink*
 F:	include/uapi/linux/if_*
@@ -19455,7 +19628,6 @@ F:	include/misc/ocxl*
 F:	include/uapi/misc/ocxl.h
 
 OMAP AUDIO SUPPORT
-M:	Peter Ujfalusi <peter.ujfalusi@gmail.com>
 M:	Jarkko Nikula <jarkko.nikula@bitmer.com>
 L:	linux-sound@vger.kernel.org
 L:	linux-omap@vger.kernel.org
@@ -19923,6 +20095,12 @@ S:	Maintained
 F:	drivers/mtd/nand/onenand/
 F:	include/linux/mtd/onenand*.h
 
+ONEXPLAYER HID DRIVER
+M:	Derek J. Clark <derekjohn.clark@gmail.com>
+L:	linux-input@vger.kernel.org
+S:	Maintained
+F:	drivers/hid/hid-oxp.c
+
 ONEXPLAYER PLATFORM EC DRIVER
 M:	Antheas Kapenekakis <lkml@antheas.dev>
 M:	Derek John Clark <derekjohn.clark@gmail.com>
@@ -20131,6 +20309,7 @@ M:	Christian Lamparter <chunkeey@googlemail.com>
 L:	linux-wireless@vger.kernel.org
 S:	Maintained
 W:	https://wireless.wiki.kernel.org/en/users/Drivers/p54
+F:	Documentation/devicetree/bindings/net/wireless/st,stlc4560.yaml
 F:	drivers/net/wireless/intersil/
 
 PACKET SOCKETS
@@ -20737,15 +20916,13 @@ F:	Documentation/devicetree/bindings/pci/intel,keembay-pcie*
 F:	drivers/pci/controller/dwc/pcie-keembay.c
 
 PCIE DRIVER FOR INTEL LGM GW SOC
-M:	Chuanhua Lei <lchuanhua@maxlinear.com>
 L:	linux-pci@vger.kernel.org
-S:	Maintained
+S:	Orphan
 F:	Documentation/devicetree/bindings/pci/intel-gw-pcie.yaml
 F:	drivers/pci/controller/dwc/pcie-intel-gw.c
 
 PCIE DRIVER FOR MEDIATEK
 M:	Ryder Lee <ryder.lee@mediatek.com>
-M:	Jianjun Wang <jianjun.wang@mediatek.com>
 L:	linux-pci@vger.kernel.org
 L:	linux-mediatek@lists.infradead.org (moderated for non-subscribers)
 S:	Supported
@@ -20803,6 +20980,14 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/pci/starfive,jh7110-pcie.yaml
 F:	drivers/pci/controller/plda/pcie-starfive.c
 
+PCIE DRIVER FOR ULTRARISC DP1000
+M:	Xincheng Zhang <zhangxincheng@ultrarisc.com>
+M:	Jia Wang <wangjia@ultrarisc.com>
+L:	linux-pci@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/pci/ultrarisc,dp1000-pcie.yaml
+F:	drivers/pci/controller/dwc/pcie-ultrarisc.c
+
 PCIE ENDPOINT DRIVER FOR QUALCOMM
 M:	Manivannan Sadhasivam <mani@kernel.org>
 L:	linux-pci@vger.kernel.org
@@ -21854,10 +22039,9 @@ F:	drivers/bluetooth/btqcomsmd.c
 F:	drivers/bluetooth/hci_qca.c
 
 QUALCOMM CAMERA SUBSYSTEM DRIVER
-M:	Robert Foss <rfoss@kernel.org>
-M:	Todor Tomov <todor.too@gmail.com>
 M:	Bryan O'Donoghue <bryan.odonoghue@linaro.org>
 R:	Vladimir Zapolskiy <vladimir.zapolskiy@linaro.org>
+R:	Loic Poulain <loic.poulain@oss.qualcomm.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 F:	Documentation/admin-guide/media/qcom_camss.rst
@@ -21941,6 +22125,14 @@ F:	Documentation/devicetree/bindings/misc/qcom,fastrpc.yaml
 F:	drivers/misc/fastrpc.c
 F:	include/uapi/misc/fastrpc.h
 
+QUALCOMM HAMOA EMBEDDED CONTROLLER DRIVER
+M:	Anvesh Jain P <anvesh.p@oss.qualcomm.com>
+M:	Sibi Sankar <sibi.sankar@oss.qualcomm.com>
+L:	linux-arm-msm@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/embedded-controller/qcom,hamoa-crd-ec.yaml
+F:	drivers/platform/arm64/qcom-hamoa-ec.c
+
 QUALCOMM HEXAGON ARCHITECTURE
 M:	Brian Cain <brian.cain@oss.qualcomm.com>
 L:	linux-hexagon@vger.kernel.org
@@ -22639,7 +22831,7 @@ F:	Documentation/devicetree/bindings/mtd/renesas-nandc.yaml
 F:	drivers/mtd/nand/raw/renesas-nand-controller.c
 
 RENESAS R-CAR GYROADC DRIVER
-M:	Marek Vasut <marek.vasut@gmail.com>
+M:	Marek Vasut <marek.vasut+renesas@mailbox.org>
 L:	linux-iio@vger.kernel.org
 S:	Supported
 F:	Documentation/devicetree/bindings/iio/adc/renesas,rcar-gyroadc.yaml
@@ -22703,6 +22895,15 @@ S:	Supported
 F:	Documentation/devicetree/bindings/timer/renesas,rz-mtu3.yaml
 F:	drivers/counter/rz-mtu3-cnt.c
 
+RENESAS RZ/G2L / RZ/V2H(P) CRU
+M:	Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>
+M:	Jacopo Mondi <jacopo.mondi@ideasonboard.com>
+L:	linux-renesas-soc@vger.kernel.org
+L:	linux-media@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/media/renesas,rzg2l-cru.yaml
+F:	drivers/media/platform/renesas/rzg2l-cru/
+
 RENESAS RZ/T2H / RZ/N2H A/D DRIVER
 M:	Cosmin Tanislav <cosmin-gabriel.tanislav.xa@renesas.com>
 L:	linux-iio@vger.kernel.org
@@ -22771,6 +22972,7 @@ F:	drivers/net/ethernet/stmicro/stmmac/dwmac-renesas-gbeth.c
 
 RENESAS RZ/V2H(P) INPUT VIDEO CONTROL BLOCK DRIVER
 M:	Daniel Scally <dan.scally@ideasonboard.com>
+M:	Jacopo Mondi <jacopo.mondi@ideasonboard.com>
 L:	linux-media@vger.kernel.org
 S:	Maintained
 F:	Documentation/devicetree/bindings/media/renesas,r9a09g057-ivc.yaml
@@ -23447,7 +23649,7 @@ S:	Maintained
 W:	https://rust-for-linux.com/pin-init
 B:	https://github.com/Rust-for-Linux/pin-init/issues
 C:	zulip://rust-for-linux.zulipchat.com
-P:	rust/pin-init/CONTRIBUTING.md
+P:	https://github.com/Rust-for-Linux/pin-init/blob/main/CONTRIBUTING.md
 T:	git https://github.com/Rust-for-Linux/linux.git pin-init-next
 F:	rust/kernel/init.rs
 F:	rust/pin-init/
@@ -23757,8 +23959,7 @@ S:	Maintained
 F:	drivers/platform/x86/samsung-laptop.c
 
 SAMSUNG MULTIFUNCTION PMIC DEVICE DRIVERS
-M:	Krzysztof Kozlowski <krzk@kernel.org>
-R:	André Draszik <andre.draszik@linaro.org>
+M:	André Draszik <andre.draszik@linaro.org>
 L:	linux-kernel@vger.kernel.org
 L:	linux-samsung-soc@vger.kernel.org
 S:	Maintained
@@ -24120,7 +24321,7 @@ F:	drivers/mmc/host/sdhci*
 
 SECURE DIGITAL HOST CONTROLLER INTERFACE (SDHCI) MICROCHIP DRIVER
 M:	Aubin Constans <aubin.constans@microchip.com>
-R:	Eugen Hristev <eugen.hristev@collabora.com>
+R:	Eugen Hristev <ehristev@kernel.org>
 L:	linux-mmc@vger.kernel.org
 S:	Supported
 F:	drivers/mmc/host/sdhci-of-at91.c
@@ -24230,7 +24431,7 @@ F:	Documentation/devicetree/bindings/iio/chemical/senseair,sunrise.yaml
 F:	drivers/iio/chemical/sunrise_co2.c
 
 SENSIRION SCD30 CARBON DIOXIDE SENSOR DRIVER
-M:	Tomasz Duszynski <tomasz.duszynski@octakon.com>
+M:	Maxwell Doose <m32285159@gmail.com>
 S:	Maintained
 F:	Documentation/devicetree/bindings/iio/chemical/sensirion,scd30.yaml
 F:	drivers/iio/chemical/scd30.h
@@ -24821,7 +25022,7 @@ F:	drivers/md/md*
 F:	drivers/md/raid*
 F:	include/linux/raid/
 F:	include/uapi/linux/raid/
-F:	lib/raid6/
+F:	lib/raid/raid6/
 
 SOLIDRUN CLEARFOG SUPPORT
 M:	Russell King <linux@armlinux.org.uk>
@@ -25032,6 +25233,13 @@ F:	include/sound/dmaengine_pcm.h
 F:	sound/core/pcm_dmaengine.c
 F:	sound/soc/soc-generic-dmaengine-pcm.c
 
+SOUND - SOC LAYER / GPIO AUDIO AMPLIFIER
+M:	Herve Codina <herve.codina@bootlin.com>
+L:	linux-sound@vger.kernel.org
+S:	Maintained
+F:	Documentation/devicetree/bindings/sound/gpio-audio-amp.yaml
+F:	sound/soc/codecs/simple-amplifier.c
+
 SOUND - SOC LAYER / DYNAMIC AUDIO POWER MANAGEMENT (ASoC)
 M:	Liam Girdwood <lgirdwood@gmail.com>
 M:	Mark Brown <broonie@kernel.org>
@@ -25580,11 +25788,11 @@ F:	Documentation/devicetree/bindings/phy/starfive,jh7110-usb-phy.yaml
 F:	drivers/phy/starfive/phy-jh7110-pcie.c
 F:	drivers/phy/starfive/phy-jh7110-usb.c
 
-STARFIVE JH8100 EXTERNAL INTERRUPT CONTROLLER DRIVER
+STARFIVE JHB100 EXTERNAL INTERRUPT CONTROLLER DRIVER
 M:	Changhuang Liang <changhuang.liang@starfivetech.com>
 S:	Supported
-F:	Documentation/devicetree/bindings/interrupt-controller/starfive,jh8100-intc.yaml
-F:	drivers/irqchip/irq-starfive-jh8100-intc.c
+F:	Documentation/devicetree/bindings/interrupt-controller/starfive,jhb100-intc.yaml
+F:	drivers/irqchip/irq-starfive-jhb100-intc.c
 
 STATIC BRANCH/CALL
 M:	Peter Zijlstra <peterz@infradead.org>
@@ -26358,7 +26566,7 @@ F:	arch/xtensa/
 F:	drivers/irqchip/irq-xtensa-*
 
 TEXAS INSTRUMENTS ASoC DRIVERS
-M:	Peter Ujfalusi <peter.ujfalusi@gmail.com>
+M:	Sen Wang <sen@ti.com>
 L:	linux-sound@vger.kernel.org
 S:	Maintained
 F:	Documentation/devicetree/bindings/sound/davinci-mcasp-audio.yaml
@@ -26368,17 +26576,20 @@ TEXAS INSTRUMENTS AUDIO (ASoC/HDA) DRIVERS
 M:	Shenghao Ding <shenghao-ding@ti.com>
 M:	Kevin Lu <kevin-lu@ti.com>
 M:	Baojun Xu <baojun.xu@ti.com>
+M:	Sen Wang <sen@ti.com>
 L:	linux-sound@vger.kernel.org
 S:	Maintained
 F:	Documentation/devicetree/bindings/sound/ti,tas2552.yaml
 F:	Documentation/devicetree/bindings/sound/ti,tas2562.yaml
 F:	Documentation/devicetree/bindings/sound/ti,tas2770.yaml
 F:	Documentation/devicetree/bindings/sound/ti,tas27xx.yaml
+F:	Documentation/devicetree/bindings/sound/ti,tas67524.yaml
 F:	Documentation/devicetree/bindings/sound/ti,tpa6130a2.yaml
 F:	Documentation/devicetree/bindings/sound/ti,pcm1681.yaml
 F:	Documentation/devicetree/bindings/sound/ti,pcm3168a.yaml
 F:	Documentation/devicetree/bindings/sound/ti,tlv320*.yaml
 F:	Documentation/devicetree/bindings/sound/ti,tlv320adcx140.yaml
+F:	Documentation/sound/codecs/tas675x*
 F:	include/sound/tas2*.h
 F:	include/sound/tlv320*.h
 F:	sound/hda/codecs/side-codecs/tas2781_hda_i2c.c
@@ -26392,6 +26603,7 @@ F:	sound/soc/codecs/pcm3168a*.*
 F:	sound/soc/codecs/pcm5102a.c
 F:	sound/soc/codecs/pcm512x*.*
 F:	sound/soc/codecs/tas2*.*
+F:	sound/soc/codecs/tas675x*.*
 F:	sound/soc/codecs/tlv320*.*
 F:	sound/soc/codecs/tpa6130a2.*
 
@@ -26684,7 +26896,7 @@ M:	Bartosz Golaszewski <brgl@kernel.org>
 L:	linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
 S:	Maintained
 T:	git git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git
-F:	Documentation/devicetree/bindings/i2c/i2c-davinci.txt
+F:	Documentation/devicetree/bindings/i2c/ti,davinci-i2c.yaml
 F:	arch/arm/boot/dts/ti/davinci/
 F:	arch/arm/mach-davinci/
 F:	drivers/i2c/busses/i2c-davinci.c
@@ -26713,6 +26925,13 @@ T:	git git://linuxtv.org/mhadli/v4l-dvb-davinci_devices.git
 F:	drivers/media/platform/ti/davinci/
 F:	include/media/davinci/
 
+TI DS125DF111 RETIMER PHY DRIVER
+M:	Ioana Ciornei <ioana.ciornei@nxp.com>
+L:	linux-phy@lists.infradead.org (moderated for non-subscribers)
+S:	Maintained
+F:	Documentation/devicetree/bindings/phy/ti,ds125df111.yaml
+F:	drivers/phy/ti/phy-ds125df111.c
+
 TI ENHANCED CAPTURE (eCAP) DRIVER
 M:	Vignesh Raghavendra <vigneshr@ti.com>
 R:	Julien Panis <jpanis@baylibre.com>
@@ -26860,12 +27079,6 @@ S:	Maintained
 F:	Documentation/devicetree/bindings/iio/adc/ti,tsc2046.yaml
 F:	drivers/iio/adc/ti-tsc2046.c
 
-TI TWL4030 SERIES SOC CODEC DRIVER
-M:	Peter Ujfalusi <peter.ujfalusi@gmail.com>
-L:	linux-sound@vger.kernel.org
-S:	Maintained
-F:	sound/soc/codecs/twl4030*
-
 TI VPE/CAL DRIVERS
 M:	Yemike Abhilash Chandra <y-abhilashchandra@ti.com>
 L:	linux-media@vger.kernel.org
@@ -27959,6 +28172,14 @@ F:	drivers/media/v4l2-core/v4l2-isp.c
 F:	include/media/v4l2-isp.h
 F:	include/uapi/linux/media/v4l2-isp.h
 
+VERISILICON IOMMU DRIVER
+M:	Benjamin Gaignard <benjamin.gaignard@collabora.com>
+L:	iommu@lists.linux.dev
+S:	Maintained
+F:	Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml
+F:	drivers/iommu/vsi-iommu.c
+F:	include/linux/vsi-iommu.h
+
 VF610 NAND DRIVER
 M:	Stefan Agner <stefan@agner.ch>
 L:	linux-mtd@lists.infradead.org
@@ -28872,6 +29093,7 @@ R:	Ahmed S. Darwish <darwi@linutronix.de>
 L:	x86-cpuid@lists.linux.dev
 S:	Maintained
 W:	https://x86-cpuid.org
+F:	arch/x86/include/asm/cpuid/leaf_types.h
 F:	tools/arch/x86/kcpuid/
 
 X86 ENTRY CODE
@@ -28891,6 +29113,7 @@ S:	Maintained
 F:	Documentation/admin-guide/hw-vuln/
 F:	arch/x86/include/asm/nospec-branch.h
 F:	arch/x86/kernel/cpu/bugs.c
+F:	scripts/update-intel-ucode-defs.py
 
 X86 MCE INFRASTRUCTURE
 M:	Tony Luck <tony.luck@intel.com>
@@ -29300,7 +29523,7 @@ F:	Documentation/devicetree/bindings/memory-controllers/xlnx,zynqmp-ocmc-1.0.yam
 F:	drivers/edac/zynqmp_edac.c
 
 XILINX ZYNQMP PSGTR PHY DRIVER
-M:	Laurent Pinchart <laurent.pinchart@ideasonboard.com>
+M:	Tomi Valkeinen <tomi.valkeinen@ideasonboard.com>
 L:	linux-kernel@vger.kernel.org
 S:	Supported
 T:	git https://github.com/Xilinx/linux-xlnx.git
diff --git a/Makefile b/Makefile
index 9f59598d3a085..b51435b8050fc 100644
--- a/Makefile
+++ b/Makefile
@@ -826,12 +826,6 @@ endif # KBUILD_EXTMOD
 # Defaults to vmlinux, but the arch makefile usually adds further targets
 all: vmlinux
 
-CFLAGS_GCOV	:= -fprofile-arcs -ftest-coverage
-ifdef CONFIG_CC_IS_GCC
-CFLAGS_GCOV	+= -fno-tree-loop-im
-endif
-export CFLAGS_GCOV
-
 # The arch Makefiles can override CC_FLAGS_FTRACE. We may also append it later.
 ifdef CONFIG_FUNCTION_TRACER
   CC_FLAGS_FTRACE := -pg
@@ -993,6 +987,11 @@ KBUILD_CFLAGS	+= $(CC_AUTO_VAR_INIT_ZERO_ENABLER)
 endif
 endif
 
+ifdef CONFIG_KMALLOC_PARTITION_TYPED
+# KMALLOC_PARTITION_CACHES_NR + 1
+KBUILD_CFLAGS	+= -falloc-token-max=16
+endif
+
 ifdef CONFIG_CC_IS_CLANG
 ifdef CONFIG_CC_HAS_COUNTED_BY_PTR
 KBUILD_CFLAGS	+= -fexperimental-late-parse-attributes
@@ -1149,6 +1148,27 @@ endif
 # Ensure compilers do not transform certain loops into calls to wcslen()
 KBUILD_CFLAGS += -fno-builtin-wcslen
 
+CFLAGS_GCOV	:= -fprofile-arcs -ftest-coverage
+ifdef CONFIG_CC_IS_GCC
+CFLAGS_GCOV	+= -fno-tree-loop-im
+# Use atomic counter updates to avoid concurrent-access crashes in GCOV.
+# Only enable if -fprofile-update=prefer-atomic does not introduce new
+# undefined symbols (e.g. libatomic calls that the kernel cannot link).
+CFLAGS_GCOV	+= $(call try-run,\
+	echo 'long long x; void f(void){x++;}' | \
+	$(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -w -fprofile-arcs \
+	-ftest-coverage -x c - -c -o "$$TMP.base" && \
+	echo 'long long x; void f(void){x++;}' | \
+	$(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -w -fprofile-arcs \
+	-ftest-coverage -fprofile-update=prefer-atomic \
+	-x c - -c -o "$$TMP" && \
+	$(NM) "$$TMP.base" | grep ' U ' > "$$TMP.ubase" || true ; \
+	$(NM) "$$TMP" | grep ' U ' > "$$TMP.utest" || true ; \
+	cmp -s "$$TMP.ubase" "$$TMP.utest",\
+	-fprofile-update=prefer-atomic)
+endif
+export CFLAGS_GCOV
+
 # change __FILE__ to the relative path to the source directory
 ifdef building_out_of_srctree
 KBUILD_CPPFLAGS += -fmacro-prefix-map=$(srcroot)/=
diff --git a/Next/SHA1s b/Next/SHA1s
new file mode 100644
index 0000000000000..f503b03a8240f
--- /dev/null
+++ b/Next/SHA1s
@@ -0,0 +1,425 @@
+Name		SHA1
+----		----
+origin		8bc67e4db64aa72732c474b44ea8622062c903f0
+ext4-fixes	981fcc5674e67158d24d23e841523eccba19d0e7
+vfs-brauner-fixes	45205929a3a3310e4978f4097d8ed4fca36b2c32
+fscrypt-current	254f49634ee16a731174d2ae34bc50bd5f45e731
+fsverity-current	0c8c88b8eb82a2a41bec5f17c076d6312dc40316
+btrfs-fixes	5f10ad875f9eea64638e48f086c0de608cb879e9
+vfs-fixes	49c5d168a3a8f4eb27d44a2a22b7e8a856ca601f
+erofs-fixes	79b09c54c6563df9846ca3094bcfd72082c3e1d7
+nfsd-fixes	4f8ef58c10bfe5f86a643c7c8331b37e69e3dae1
+v9fs-fixes	028ef9c96e96197026887c0f092424679298aae8
+overlayfs-fixes	1f6ee9be92f8df85a8c9a5a78c20fd39c0c21a95
+fscrypt		254f49634ee16a731174d2ae34bc50bd5f45e731
+btrfs		6bf684b8823552b99c86bf791b22f622934ee771
+ceph		b01cb51ba5652b6bda9cb05b0ab667abd5ddeb18
+cifs		19d8c94787da866318eead464fb9c9d70815af56
+configfs	6363844fdbbb76afe1d44d678fe0746390204a5f
+ecryptfs	4d97a37e2cedb008eb39354b2a4e2af9ae8bdbe7
+dlm		e61113cfcaf71cbdf4c17e3d086d8fb7f92c62bf
+erofs		a5242d37c83abe86df95c6941e2ace9f9055ffcb
+exfat		ec5e84e1605d242caf401b93e879424f7498b576
+ext3		5f0419457f89dce1a3f1c8e62a3adf2f39ab8168
+ext4		981fcc5674e67158d24d23e841523eccba19d0e7
+f2fs		520760b9f9156bf9698de38dc44c614fad68a1f9
+fsverity	254f49634ee16a731174d2ae34bc50bd5f45e731
+fuse		040d71ac647099f7bb5e480fcd74ba68abc3cb18
+gfs2		f9c9ec2c319f843b70ecdf939d48b52d189bc081
+jfs		dad98c5b2a05ef744af4c884c97066a3c8cdad61
+ksmbd		1a2ab0feaa23147e347b4d4cb79cb3fc392118eb
+nfs		7fd2df204f342fc17d1a0bfcd474b24232fb0f32
+nfs-anna	4529e0015432977af3ecc3b9f940fc2a1ef1b265
+nfsd		dc7a9c0a4a32ea2abab9daf1a541fa99a6d3749c
+ntfs		29abfa1b2fb4425a9cba286367ead451b67d7975
+ntfs3		4a107029611613911425517c4d28b1840cd9d449
+orangefs	e61bc5e4d87433c8759e7dc92bb640ef71a8970c
+overlayfs	1f6ee9be92f8df85a8c9a5a78c20fd39c0c21a95
+ubifs		11efa98bcc0d00271e8f66c7c940c284f973f39d
+v9fs		8fc518e489c1386fd0cf7f4256d055960ed6a2e4
+v9fs-ericvh	028ef9c96e96197026887c0f092424679298aae8
+xfs		509fdeb3326be0db055e88d0f689a3888f147f90
+zonefs		3a8389d42bdf4213730f4067f8bfa78bae6564ef
+vfs-brauner	1202b9c177076b8c00ba826717a8d7023e4c8f98
+vfs		e82ff9b6cb767fc823d5e461034f32388c4092a2
+mm-hotfixes	abd2c27f2cd99fc51ab746f2a7bf94f50425092c
+fs-current	3b67b13aeb5675b3fcb99a60bc054a20cbcb220f
+kbuild-current	f2be8dd21b3f4d8674478aeea33b9d072be41abf
+clang-fixes	bc5ffe737f56ee2734597069ed71f48830549483
+arc-current	7aaa8047eafd0bd628065b15757d9b48c5f9c07d
+arm-current	75f9a484e817adea211c73f89ed938a2b2f90953
+arm64-fixes	2ccd8ff980b50e842481bae71102fa3883fc4377
+arm-soc-fixes	1fcf4149418e7a8f8253dd74059d56340795503f
+davinci-current	254f49634ee16a731174d2ae34bc50bd5f45e731
+drivers-memory-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+sophgo-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+sophgo-soc-fixes	0af2f6be1b4281385b618cb86ad946eded089ac8
+m68k-current	cb39cf99d88e8f2c68a41fb3bb01c4f40a8fcc30
+powerpc-fixes	5200f5f493f79f14bbdc349e402a40dfb32f23c8
+s390-fixes	540f4a4f6ef806a28e794001bb4beac4840a6090
+net		1bbf0ced1d9db73ac7893c2187f3459288603e0d
+bpf		49b18315be4eecfc36b75f4aecb4d40a87d68a20
+ipsec		79d8be262377f7112cfa3088dfc4142d5a2533f3
+netfilter	fa997ddef508b1b37b2fe4d2dad7c4b70958335e
+ipvs		93d809adc13001e9d3a3ceb8d1e60fae2fb740d6
+wireless	dc14686f27df6454b13b16ad1c9203ab3e9b0375
+ath		dc14686f27df6454b13b16ad1c9203ab3e9b0375
+iwlwifi		734a4e051b9767f439137940095d63afbfed0745
+wpan		8ce4f287524c74a118b0af1eebd4b24a8efca57a
+rdma-fixes	5b74373390113fba798a76b483837029ab010fef
+sound-current	60a1969fae6209644698fca91c185d153674f631
+sound-asoc-fixes	298a43b54432fbc3a32949a94c72544ee18c8c00
+regmap-fixes	4b7774eeab8d66c22f02f5c120602df154a87e12
+regulator-fixes	f9b2d3b703d13df50c630997dfdc25648e96db0d
+spi-fixes	ea6ec3343e05f7937a53eb6d7617b3abdb4abc19
+pci-current	ed831e7ea1a860bdbab3eadeb95f7f73e9d212df
+driver-core.current	237557b8a81ab948e8332f7c0058e758f081c0a3
+tty.current	452d6fa37ae9b021f4f6d397dbae077f7296f6f4
+usb.current	288a81a8507052bcfbf884d39a463c44c42c5fd9
+usb-serial-fixes	689f2facc689c8add11d7ff69fbbad17d65ee596
+phy		ea17fc4d7dc2ba6459b1a318962960520201baf1
+staging.current	514ab98364595007d4557ecc85d7e5f012c504d3
+iio-fixes	bbc109ae655c8a5d4db117fea01947eb6bf51e71
+watchdog-fixes	367937951ffcaf5bd02c446c443407dfe8877886
+counter-current	254f49634ee16a731174d2ae34bc50bd5f45e731
+char-misc.current	329db447f8f02e82ff3acf130c892e1a4063d493
+soundwire-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+thunderbolt-fixes	928abe19fbf0127003abcb1ea69cabc1c897d0ab
+input-current	16ca52bc209fa4bf9239cd9e5643e95533476b58
+crypto-current	d1fa83ecac31093a550534a79a33bc7f4ba8fc10
+libcrypto-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+vfio-fixes	702809dabdecca807bdd50cfdcc1c980feb2ba62
+kselftest-fixes	465b05bae5ac553c13315681c1490dc565337771
+dmaengine-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+backlight-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+mtd-fixes	135ac3b84bcedae1860e7a9512d63166f42b736e
+mfd-fixes	aa6e63373d91f82628b20f6fae7ffe9b97b568a5
+v4l-dvb-fixes	646ebdd3105809d84ed04aa9e92e47e89cc44502
+reset-fixes	9797524ef2b69c6b187b55bd844eb72a8c1cbd99
+mips-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+at91-fixes	765aaba18413a66f6c8fe8416336ca9b3dd98a79
+omap-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+kvm-fixes	2d5d3fc593c9b7e41bee86175d7b9e11f470072e
+kvms390-fixes	ac1283cf1d3f8bfe0701ae528c7b715852792812
+kvm-arm-fixes	1702da76e017ae0fbe1a92b07bc332972c293e89
+hwmon-fixes	5200f5f493f79f14bbdc349e402a40dfb32f23c8
+nvdimm-fixes	a8aec14230322ed8f1e8042b6d656c1631d41163
+cxl-fixes	5d6919055dec134de3c40167a490f33c74c12581
+dma-mapping-fixes	af0c3f05866237f7592219bfe05387bc3bfc99b5
+drivers-x86-fixes	2a9ca96225ccc0bacb81f152482fe23175d37ac3
+samsung-krzk-fixes	dce77afa1e18f32624c431de494bad670f624338
+pinctrl-samsung-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+devicetree-fixes	4b0a3ff9f4178a0765d54c0dd09e9839526b08ba
+dt-krzk-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+scsi-fixes	b71cb088b2e3427924a470fc43e7aedb8a40d2e3
+drm-fixes	aee43aaf26c7f162780ab77ce952bae6a1b354bd
+drm-intel-fixes	4703049f768fc1c1caac754134118bee1a3af189
+mmc-fixes	5a52c5701a67d5176eb1afbf1bdaf7d6dfeec597
+rtc-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+gnss-fixes	5d6919055dec134de3c40167a490f33c74c12581
+hyperv-fixes	8b35874f56ded0cc1a90a25b87411249a86246cd
+risc-v-fixes	b69bcb13ed7024a84d6cd8ad330f1e32782fcf28
+riscv-dt-fixes	0df8aa2b9aec5cd21e8c71d9cc1227e57bea43b3
+riscv-soc-fixes	75ef233975589d9a8c88bc8822a7c725c71ff650
+fpga-fixes	19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+spdx		254f49634ee16a731174d2ae34bc50bd5f45e731
+gpio-brgl-fixes	be460cedb67ab803c1bebceac19b1d44acb85d30
+gpio-intel-fixes	3bb62e3f99a557d257e5f5a803200051b7de3afa
+pinctrl-intel-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+auxdisplay-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+kunit-fixes	8f80b5b227ef9ea422080487715c841856339aed
+memblock-fixes	34d885baf7cc7b47dc6ce35fc3540d93c54a1c55
+renesas-fixes	8a78d1c454b3cc7b66d52582474e8b3732833352
+perf-current	5a433107fab621f4e7379ccba6e52b5b1601046c
+efi-fixes	d8809f6931065cbbf3554647a50a65a471ab5983
+battery-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+iommufd-fixes	be93d186ae88a92e7aa77e122d4e661fa57b1e39
+rust-fixes	84a0f7caafc679f763d3868635837e22bb89651a
+w1-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+pmdomain-fixes	5d6919055dec134de3c40167a490f33c74c12581
+i2c-host-fixes	30792d12842901f5276f466a960962d5bfa15cc8
+i2c-rust-fixes	7ca6d1cfec80ebe46cc063f3284c5896c344d9a1
+sparc-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+clk-fixes	de019f203b0d472c98ead4081ad4f05d92c9b826
+thead-clk-fixes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+tenstorrent-clk-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+pwrseq-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+thead-dt-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+ftrace-fixes	1650a1b6cb1ae6cb99bb4fce21b30ebdf9fc238e
+ring-buffer-fixes	92d5a606721f759ebebf448b3bd2b7a781d50bd0
+trace-fixes	55a0005518195fdea1fd2991b07644f8dc97ea8e
+tracefs-fixes	07004a8c4b572171934390148ee48c4175c77eed
+spacemit-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+tip-fixes	de5cda262357b3912b944dff8a61649c44827e40
+slab-fixes	67ea9d353d0ba12bdbc9183ff568dead9e949b80
+kexec-fixes	028ef9c96e96197026887c0f092424679298aae8
+liveupdate-fixes	ec4084bc445027a52f600e30a976928be1ba1950
+drm-msm-fixes	db339b6bc9f234b4883eb02946ea01d8d9faa11c
+uml-fixes	254f49634ee16a731174d2ae34bc50bd5f45e731
+fwctl-fixes	e7537735028c3ad4b0bfc02ff8fa2a1a28aa04fe
+devsec-tsm-fixes	c3fd16c3b98ed726294feab2f94f876290bf7b61
+drm-rust-fixes	0a69ac25bd596d50823d530d0a2004336668c0df
+tenstorrent-dt-fixes	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+nfc-fixes	f23bf992d65a42007c517b060ca35cebdea3525a
+drm-misc-fixes	9af1b6e175c82daf4b423da339a722d8e67a735a
+rust		fc1ce3afa2e61b4b15e71436ece91b0441a9f4f0
+rust-interop	05f7e89ab9731565d8a62e3b5d1ec206485eeb0b
+rust-alloc	ec51531fb8efda9b75316111c040dd756f815b0c
+rust-io		86731a2a651e58953fc949573895f2fa6d456841
+rust-pin-init	6fb5912c92926025a7e205d53feb73c3478c79a1
+rust-timekeeping	ddb1444d3335129ae87d9796ab1debf41c0ee51b
+rust-xarray	c455f19bbe6104debd980bb15515faf716bd81b8
+rust-analyzer	5d6919055dec134de3c40167a490f33c74c12581
+mm-stable	27fa82620cbaa89a7fc11ac3057701d598813e87
+mm-nonmm-stable	27fa82620cbaa89a7fc11ac3057701d598813e87
+mm-unstable	e9add7501ad3297dad9b90ce201266830a68ab47
+mm-nonmm-unstable	d067a83c8063d1bdcbd9af8e1326d846f85138b8
+kbuild		2c31897a17e55a6da529b4e797e98c6febc60fd2
+clang-format	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+perf		85cc481af5ffa23809b7bb100aa774172369983c
+compiler-attributes	8f0b4cce4481fb22653697cced8d0d04027cb1e8
+dma-mapping	bebaaca682b12b0f907e31b07bb1fee42be88a25
+asm-generic	adbbd9714f8058730f93c8df5c5bf1679456424b
+nocache-cleanup	809b997a5ce945ab470f70c187048fe4f5df20bf
+alpha		d58041d2c63e09a1c9083e0e9f4151e487c4e16a
+arm		c6e61c06d6061750597e79c598acb5dead44c35b
+arm64		c754aa6b881ade764510b8539a6a313326501e3d
+arm-perf	3ef020a3c9ebd1e3814d900e996fa29ce678c0d0
+arm-soc		8dbb80525730a16c7b38b07c2400df36e11ed605
+amlogic		3878c0539707a4611aae680ff7d836fc9178c9b7
+asahi-soc	d31647c698f632e0c08080a760e004ba4d71f8dd
+at91		9a6b14b29aa2a8fb2ec45eb6eb3ffe7c0ff42e14
+bmc		66d729c23ec5543fb43659a6fc120b24103eda97
+broadcom	e8f908b91550dad3274b5ab3bb0e779ca9d80aff
+cix		7fd2df204f342fc17d1a0bfcd474b24232fb0f32
+davinci		254f49634ee16a731174d2ae34bc50bd5f45e731
+drivers-memory	0a783a6686c96847d74874dbea38fa0e6b1f6c12
+fsl		c99d8ac7b355cbbdbb3f4d8b9a9b46de05a6fee9
+imx-mxs		b3c1d1631f097619f8091f0293e027c4301285d6
+mediatek	12e53eb041a53db8e53067722c2c08d4b33eaadc
+mvebu		ae4fe907b7705d9f7af7a0e6114bb6607e703c13
+omap		e28bf943a8c772c66cce977833607f4c7232edce
+qcom		a83b64da812b6ff1123e124ebefded4a9243b720
+renesas		1dbfc4ee3e4825c650f3bd4d54063546ee059d5b
+reset		d373605cd514837d8a6de3d00c786d4bae6dbaf8
+rockchip	2e030c87599d15d364d1c9a2c0198151e09c3a2a
+samsung-krzk	446ccff1b11b73a53cf35c4c15b1d5a8225d8dee
+scmi		8408dd4800b6c15becfc07d7da18cd156da4a0a6
+sophgo		9aa6068586a9b4cd34cf04f8dee72a7283ab4ae4
+sophgo-soc	c8754c7deab4cbfa947fa2d656cbaf83771828ef
+spacemit	e2518e8cb1dde64af2d1bb246639bb7ef7523f7a
+stm32		a0d6c2a06fffff47bcca4d5bfdab4cc428a315fc
+sunxi		d55695a22b5d7b918479d5b818d757da6c1205a7
+tee		4d88259524b0605fe09e51be5d992e156e9cd221
+tegra		20e4d2163a23dcac417671d8cb49e7eb49e67a95
+tenstorrent-dt	ad3cb813231312adc4eda9cae0787204781d9a36
+thead-dt	74ec3d52c0035b662ec295bef2bbffad68446391
+ti		e84bac36f60fec7271ead8cdef735ccbad480ca8
+xilinx		fdee7c66c0d7b6869c36b9f9a915abf29ab5b550
+socfpga		7557d67a6057362a65ad03a03cbefe7f03523003
+clk		9781f9635d693704e116695a3563f3059a2926e6
+clk-imx		d16f57caa78776e6e8a88b96cb2597797b376138
+clk-renesas	4f42053949324867dc40d67829f18a01539e6322
+thead-clk	baf4fc7c03bd0f68c768cfe27829674bd060c6b4
+tenstorrent-clk	23c8ebc952849b3ba47d04d0ec95daf5cc136061
+csky		817d21578d51e801df58ab012654486a71073074
+loongarch	5a873d77ba792410a796595a917be6a440f9b7d2
+m68k		e321561b915378d2e9eacd1bc4d7fb2338d09961
+m68knommu	16b47102d9c3b5a5af60a2139926b5501e06d56c
+microblaze	6de23f81a5e08be8fbf5e8d7e9febc72a5b5f27f
+mips		254f49634ee16a731174d2ae34bc50bd5f45e731
+openrisc	0dfffa5479d6260d04d021f69203b1926f73d889
+parisc-hd	5200f5f493f79f14bbdc349e402a40dfb32f23c8
+powerpc		5200f5f493f79f14bbdc349e402a40dfb32f23c8
+risc-v		f2abc305aa93f5b12d5c929d7a9c1cf7d7fee8af
+riscv-dt	3869417f8996fb076998342d0db560fdd795a3a6
+riscv-soc	b5865650b20972847dd932a7b75e7734b7c4400c
+s390		2d5fe189176283772f34bc39939f0c03f9c48d0a
+sh		b0aa5e4b087b686575f1b31ce54048b4d059b7b8
+sparc		d844152d85cfcc7d2ef9430a25882604c12da279
+uml		254f49634ee16a731174d2ae34bc50bd5f45e731
+xtensa		84477867e122251b8c47db240a8d699b0836aa6f
+fs-next		68fa42a818989a8ff0eea89f8d96d50c65acc6fd
+printk		e56185668dc12983dd0e75e38ed6dea98b01d2d2
+pci		6b96e84b16c1e731a33d448badcdf557c201f867
+pstore		24b8f8dcb9a139a36cf48bfbe935e8dc1f33ed79
+hid		5d1aae9252b4632ef942017fdd18b9993a5a22ef
+i2c		023453cb7eb0f53c5dc36babed8e706c1b0b0187
+i2c-host	70dc84bd48dccc711261c4abfd96da4aa0f2eecf
+i2c-rust	254f49634ee16a731174d2ae34bc50bd5f45e731
+i3c		1d78a8fc97c133b8aee54993a83f86b68ed2fdb8
+dmi		1afafbaf749d8e8ec53f8e38efdc731131902b5b
+hwmon-staging	95f61e6af7d25a5043a0ac945f87ced1c598d31c
+jc_docs		4f07ec51d453a00358f627263768a987a05cf046
+v4l-dvb		bb16e2895a4161aa90a884bc2e9ef3f229f40e46
+v4l-dvb-next	adc218676eef25575469234709c2d87185ca223a
+pm		b2f26730279583c6a64d3e246c5591db3aa15237
+cpufreq-arm	9a303892acac58e77704d336418fa9ae92e3f70f
+cpupower	254f49634ee16a731174d2ae34bc50bd5f45e731
+devfreq		c096be11c2a9d4acca8e75f1f8edeb744aceb521
+pmdomain	fba9c703c18459e936c365442667c08ca40dadcb
+opp		69f888381d2ecbe18ed9f112c096f8fd3623db98
+thermal		bf746e2a41efd98668c97759e06d436ae5af5a82
+rdma		67464f388d52ec172be62c99fc43697437ffa384
+net-next	830d8771ae3c7bc90a62dde76a6556e612529fbc
+bpf-next	3db0419c0147c3b91e2df475a1d34bcf0f645930
+ipsec-next	a77d172177f3754ebd70123c78c75a6efa9eec2a
+mlx5-next	02c54621e81ccdc1907e2d735bcda751f2caade1
+netfilter-next	8b2feced65cd3aa0597d596ed5733a1abd4c4d78
+ipvs-next	8b2feced65cd3aa0597d596ed5733a1abd4c4d78
+bluetooth	0b580042a1a5478b2cf5f1ff66372b75392ad2a4
+wireless-next	1d174fec87850e1005db9b106f84bbbb19cb59b9
+ath-next	30d516006fa1f72f957c18c6171f5680dcdebfb0
+iwlwifi-next	05cfe4277e92b6da941a23ef96dce32cced7abf2
+wpan-next	1dd9291eb90378f8096669d9cb33761fd84599e0
+wpan-staging	1dd9291eb90378f8096669d9cb33761fd84599e0
+mtd		87eee18f2974c0b1d854b6228408b93c6bc91744
+nand		9e3997d3ab661f99828fb86487be56c25e166410
+spi-nor		254f49634ee16a731174d2ae34bc50bd5f45e731
+crypto		6c9dddeb582fde005360f4fe02c760d45ca05fb5
+libcrypto	cf52058dcdd96420cfc38ee284c5ac077901ea61
+drm		4b18a9d44e38c9aa63ef8bff8658ca142e89c343
+drm-exynos	3a8660878839faadb4f1a6dd72c3179c1df56787
+drm-misc	a48bbcc7ac739e93562d6148c6fa504c2e9f22f8
+amdgpu		4cdbba5a16aaf16513b69cc332b1d6f971a44b52
+drm-intel	c987f8c20946d41b9f191fdd2d7601e65e8cc120
+drm-msm		db339b6bc9f234b4883eb02946ea01d8d9faa11c
+drm-msm-lumag	df7cfac4b205763a6e712f2f8614494801a97a39
+drm-xe		339fa0be9e4a5d69fa47e91f4a36574224fb478f
+drm-rust	8bfe9d72cf2064f679c4192dba84be79eb70675d
+drm-nova	93296e9d9528f0d87f2cf3fee494599060a0f14a
+etnaviv		6bde14ba5f7ef59e103ac317df6cc5ac4291ff4a
+fbdev		63a347282ebeace9f2f6affe035c905777835b23
+regmap		4b126e54759a2010116ac4290a9e4ef7bf3b4368
+sound		2ee646353cd5fadc42b002e54ed7e4fc19676689
+ieee1394	8208d94f149a53311ac7687c051cb3a6d58063f7
+sound-asoc	5ce08d8617b6f80b52e0ab00c10ebc97f9107701
+modules		663385f9155f27892a97a5824006f806a32eb8dc
+input		b7710233c16c1f42e0b58a1f0485658fc4fa61c1
+block		c16700adbbe93a2a0ebf65aa7bd9af5791b0a7ab
+device-mapper	5aa0f9231cbacade065cedd8e9b5ebd067231171
+libata		0926c1b731b73e7296e4ad4f7d055918e16ae6ab
+pcmcia		b3c26ea81ccc522e77ed0b1707add61fc9206216
+mmc		ee06a067864b0be72cb3c3d2b43bb39b94524eb1
+mfd		29305fb8c8ac7bca36d151c4d58a6d7486a8b13c
+backlight	acea35b567d0f610d533ca661f4106ac12583f78
+battery		254f49634ee16a731174d2ae34bc50bd5f45e731
+regulator	0e8ef95038ef2cb620c33307b3bd92da597dec5a
+security	1abe66291664ccad573625dfd6f2295f71e1331c
+apparmor	254f49634ee16a731174d2ae34bc50bd5f45e731
+integrity	11143a19f5b8dc8f414deab87571134f9f447313
+selinux		ad5a51dd2f48e57d3d4986355716e75212853dab
+smack		b78fede1c69a090d377bf80417ce1f7f7f314534
+tomoyo		5e6978414efa55e2bdf1aea3ede199a1c7c12dd8
+tpmdd-tpm	fd1ac8d2bae6801ef83d5c4de3ccfc86803266cb
+tpmdd-keys	26cb5b4b32ff1ad708515d5228f28930822bf6fd
+watchdog	c574bdb524095d24169e229b2e3b9318c72e733a
+iommu		f4f5d162619bd437fdd31367c035c63f564d0719
+audit		5200f5f493f79f14bbdc349e402a40dfb32f23c8
+devicetree	98e2a77b5d2c19f7a1bf2924ed2701035f214411
+dt-krzk		254f49634ee16a731174d2ae34bc50bd5f45e731
+mailbox		5e4907c4908bff6570f48aff86fef424e74c051f
+spi		0c5b5c40dc31089e8e59d53a32313baa50bc0809
+tip		564844fbd167a335c5698579362b0d3fea8890ee
+kexec		028ef9c96e96197026887c0f092424679298aae8
+liveupdate	34e8f02817e31826e76bb2ded48bf28fe921f20b
+clockevents	f182fa740218dec7ead6275b2e096da1642272b5
+edac		4b049cd5e5194c8e58cd978d2dc9d3bcf67c84f6
+ftrace		4df143bbee63abc163226f639b69c7dfda292b74
+rcu		ed062c41dfda2de8d1712c91e089303dae013bb7
+paulmck		48f7a50c027dd2abb9e7b8a6ecc8e531d87f2c21
+kvm		b7fbe9a1bf9ee6c967ef77d366ca58c35fcf1887
+kvm-arm		9b46f887829f7d220f7ebb2640b2a183c3ae1dfe
+kvms390		9b8e8aad5896d66005d29920cb1643076a20b172
+kvm-ppc		5200f5f493f79f14bbdc349e402a40dfb32f23c8
+kvm-riscv	ddbf9c76c4020bf63a0799b00faad40caa3de6c2
+kvm-x86		a9512a611bd030088f13477258d1f8103cceaa40
+xen-tip		4594437880ce347ac8438758fd91543f70da1aa9
+percpu		8f0b4cce4481fb22653697cced8d0d04027cb1e8
+workqueues	2ccd533ef91537db4a9c38bf5b45242e987b8089
+sched-ext	8ad190500ea5410b46904a1f7419c78a8afc037a
+drivers-x86	5fdac9983681f743cfaa89414ea154a2c5fd39c4
+chrome-platform	ace8ee71e8ef4d6ef5a6501469df95237e1f7406
+chrome-platform-firmware	86e3bbc716332600e9e087f8a4889f4d9714a99c
+hsi		254f49634ee16a731174d2ae34bc50bd5f45e731
+leds-lj		25025253476a64c186592d952c27f24bc3490e42
+ipmi		8a8ba84886c905bd43dbdc3b3f78c7ba8f8661f3
+driver-core	454257f6d124a92342dcbb7710c03dd6ef96c731
+usb		e18b0dbb6ec6030e3369e5dfb2520d7bdbde20d1
+thunderbolt	e24f3c0df48378214d9a67c5048d0faca144b163
+usb-serial	e690592f98c81416c5be5ce65828aa4665abc302
+tty		16e95bfb79b5d9d01dc7651d98caf3c2ace331cd
+char-misc	5d6919055dec134de3c40167a490f33c74c12581
+coresight	a5dd853fb7774c9543aed272a8614c15ebce3173
+fastrpc		8d712bcdd1ef73bf790f01039508fb4e9c076bb1
+fpga		3c310d945b9490b3417eba7d58d7c7fbb3156081
+icc		a24b55c5c084f6bf5a808ec1525f25d6c6a4ebe1
+iio		8678fb54958893818ddeccd05fea560a4e1fc759
+phy-next	293e19f416fa3f233a2fb013258f7abcb39ad6ed
+soundwire	8a7fe10eec64bfb7cf4091bca540de4c55d56bfa
+extcon		254f49634ee16a731174d2ae34bc50bd5f45e731
+gnss		5d6919055dec134de3c40167a490f33c74c12581
+vfio		b9285405c5f6144f4444f97bf3048d865e11cc1d
+w1		254f49634ee16a731174d2ae34bc50bd5f45e731
+spmi		254f49634ee16a731174d2ae34bc50bd5f45e731
+staging		6abf0b2df0b1c2205a4c0591425e6461afa62edb
+counter-next	ee4133b921a17b239ca617e9b0c16cad1277e9fd
+mux		59b723cd2adbac2a34fc8e12c74ae26ae45bf230
+dmaengine	362ee0c0dc522bcf585bde59ceba2038ec583b7d
+cgroup		936f0880adaf8edab431e64503a5686a8d79e54e
+scsi		b71cb088b2e3427924a470fc43e7aedb8a40d2e3
+scsi-mkp	016d484531e3169cd7bcb26e0ac2c5523080809f
+vhost		ebcff9dacaf2c1418f8bc927388186d7d3674603
+rpmsg		54dacf6efe7196c1cd8ae4b5c691579d0510a8bd
+gpio-brgl	c010a78304a648fd13556aff63e60a83f35d23c9
+gpio-intel	3bb62e3f99a557d257e5f5a803200051b7de3afa
+pinctrl		73d4991a6949eedb51e442d4e81415017d85975b
+pinctrl-intel	2b9ef70d3abe1eaa9b37253fd7765cf40ff2a5ad
+pinctrl-renesas	c7c8352fe569d17e3d379a83075a8ea12168526f
+pinctrl-samsung	254f49634ee16a731174d2ae34bc50bd5f45e731
+pwm		3936b25815ee686a273ca7bbdc9ae19af5e608a3
+ktest		932cdaf3e273a2727e77af97f79f12577174c5a0
+kselftest	5f4974231fb3656c15a82faece9f2303b8c18a14
+kunit		254f49634ee16a731174d2ae34bc50bd5f45e731
+kunit-next	5c1553dd5db306d84d539f84fa184d4694d6d050
+livepatching	8f7168335cb2e438668c5d94eea76621c9a10edd
+rtc		b72386864481cf7fb6153842d22561ac3032302f
+nvdimm		45df9111692c62d5f09fc4345ae36dae31024797
+at24		b4c01497688528fc04c717842f8f310569f52629
+ntb		7b3302c687cae50d9506c77ad9d40f8c242e9c79
+seccomp		41fa04327384148b0e2e828c9be9862c5240e9fa
+slimbus		254f49634ee16a731174d2ae34bc50bd5f45e731
+nvmem		69679101cc8935b9f4032678c12612ea584a573a
+hyperv		5170a82e89211d876af17bf3d94a511fb2bb4921
+auxdisplay	4dc76c305a73bacaf330bebf723a181427bb4540
+kgdb		fdbdd0ccb30af18d3b29e714ac8d5ab6163279e0
+hmm		19272b37aa4f83ca52bdf9c16d5d81bdd1354494
+cfi		254f49634ee16a731174d2ae34bc50bd5f45e731
+mhi		519ddf194b158b91439319f6b977b8a465fda0fb
+memblock	d5759519805c54786c00765ca1303e6d7a0676ca
+cxl		6c724ce0ec6ed8608917673bbb363b208ce2644c
+zstd		65d1f5507ed2c78c64fce40e44e5574a9419eb09
+efi		48a428215782321b56956974f23593e40ce84b7a
+unicode		bcfee135d584714c2130031c7e28aafa91057b9a
+slab		7140d88bb82330d3249805a3c4b79d0a97b5928a
+random		bb9ff576fdff48c242876f55098a3ee20a29df5d
+landlock	9c5b83756e7b7eab35335da0d5c02a8854bbf416
+sysctl		f63a9df7e3f9f842945d292a19d9938924f066f9
+execve		9bf092c97b86af63694d9902b9e14047214ba76d
+bitmap		09472f591aa0b72c2dd6c693f48b2d6fea66c7ba
+hte		005b25ad11717139ca5c14c9f06c2a60855c2afd
+kspp		9a21dcb870e1343e874a641161df428d226d5992
+nolibc		136ca91411b0b637e862eb7b1cce2a56853edd17
+iommufd		254f49634ee16a731174d2ae34bc50bd5f45e731
+turbostat	d052a81a8a23cfa0dac24b91931c8bc4d3779d8b
+pwrseq		513f49c33e91e58975ada7967b44512179f0e703
+capabilities-next	071588136007482d70fd2667b827036bc60b1f8f
+ipe		028ef9c96e96197026887c0f092424679298aae8
+kcsan		07a1a6562ce29e2e0c134a57882d6e52e8758492
+crc		254f49634ee16a731174d2ae34bc50bd5f45e731
+keys-next	965e9a2cf23b066d8bdeb690dff9cd7089c5f667
+fwctl		254f49634ee16a731174d2ae34bc50bd5f45e731
+devsec-tsm	3177779ae17db4c66c851f799505fb95c7530c03
+hisilicon	b419e155e31d9e23c1aa29a37182889955189382
+kthread		fa39ec4f89f2637ed1cdbcde3656825951787668
+nfc		9705c9f36439fed3d85a0ba20d4429c4fbd0fdba
diff --git a/Next/Trees b/Next/Trees
new file mode 100644
index 0000000000000..426994346c42e
--- /dev/null
+++ b/Next/Trees
@@ -0,0 +1,425 @@
+Trees included into this release:
+
+Name		Url
+----		---
+origin		https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git#master
+ext4-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#fixes
+vfs-brauner-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.fixes
+fscrypt-current	https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-current
+fsverity-current	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-current
+btrfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#next-fixes
+vfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#fixes
+erofs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#fixes
+nfsd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-fixes
+v9fs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#fixes/next
+overlayfs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#ovl-fixes
+fscrypt		https://git.kernel.org/pub/scm/fs/fscrypt/linux.git#for-next
+btrfs		https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git#for-next
+ceph		https://github.com/ceph/ceph-client.git#master
+cifs		git://git.samba.org/sfrench/cifs-2.6.git#for-next
+configfs	https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git#configfs-next
+ecryptfs	https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git#next
+dlm		https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git#next
+erofs		https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git#dev
+exfat		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git#dev
+ext3		https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git#for_next
+ext4		https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git#dev
+f2fs		https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git#dev
+fsverity	https://git.kernel.org/pub/scm/fs/fsverity/linux.git#for-next
+fuse		https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git#for-next
+gfs2		https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git#for-next
+jfs		https://github.com/kleikamp/linux-shaggy.git#jfs-next
+ksmbd		https://github.com/smfrench/smb3-kernel.git#ksmbd-for-next
+nfs		git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git#linux-next
+nfs-anna	git://git.linux-nfs.org/projects/anna/linux-nfs.git#linux-next
+nfsd		https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux#nfsd-next
+ntfs		https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git#ntfs-next
+ntfs3		https://github.com/Paragon-Software-Group/linux-ntfs3.git#master
+orangefs	https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git#for-next
+overlayfs	https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git#overlayfs-next
+ubifs		https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git#next
+v9fs		https://github.com/martinetd/linux#9p-next
+v9fs-ericvh	https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git#ericvh/for-next
+xfs		https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git#for-next
+zonefs		https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git#for-next
+vfs-brauner	https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git#vfs.all
+vfs		https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git#for-next
+mm-hotfixes	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-hotfixes-unstable
+kbuild-current	https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-fixes-for-next
+clang-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/nathan/linux.git#clang-fixes-for-next
+arc-current	https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git#for-curr
+arm-current	https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#fixes
+arm64-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/fixes
+arm-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#arm/fixes
+davinci-current	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-current
+drivers-memory-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#fixes
+sophgo-fixes	https://github.com/sophgo/linux.git#fixes
+sophgo-soc-fixes	https://github.com/sophgo/linux.git#soc-fixes
+m68k-current	https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-linus
+powerpc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#fixes
+s390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#fixes
+net		https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git#main
+bpf		https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/#master
+ipsec		https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git#master
+netfilter	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git#main
+ipvs		https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git#main
+wireless	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git#for-next
+ath		https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-current
+iwlwifi		https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#fixes
+wpan		https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git#master
+rdma-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-rc
+sound-current	https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-linus
+sound-asoc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-linus
+regmap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-linus
+regulator-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-linus
+spi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-linus
+pci-current	https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#for-linus
+driver-core.current	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-linus
+tty.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-linus
+usb.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-linus
+usb-serial-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-linus
+phy		https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#fixes
+staging.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-linus
+iio-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#fixes-togreg
+watchdog-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#watchdog
+counter-current	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-current
+char-misc.current	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-linus
+soundwire-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#fixes
+thunderbolt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#fixes
+input-current	https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#for-linus
+crypto-current	https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git#master
+libcrypto-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-fixes
+vfio-fixes	https://github.com/awilliam/linux-vfio.git#for-linus
+kselftest-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#fixes
+dmaengine-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#fixes
+backlight-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-fixes
+mtd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/fixes
+mfd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-fixes
+v4l-dvb-fixes	git://linuxtv.org/media-ci/media-pending.git#fixes
+reset-fixes	https://git.pengutronix.de/git/pza/linux#reset/fixes
+mips-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-fixes
+at91-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-fixes
+omap-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#fixes
+kvm-fixes	git://git.kernel.org/pub/scm/virt/kvm/kvm.git#master
+kvms390-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#master
+kvm-arm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#fixes
+hwmon-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon
+nvdimm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-fixes
+cxl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#fixes
+dma-mapping-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-fixes
+drivers-x86-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#fixes
+samsung-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#fixes
+pinctrl-samsung-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#fixes
+devicetree-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#dt/linus
+dt-krzk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#fixes
+scsi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#fixes
+drm-fixes	https://gitlab.freedesktop.org/drm/kernel.git#drm-fixes
+drm-intel-fixes	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next-fixes
+mmc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#fixes
+rtc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-fixes
+gnss-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-linus
+hyperv-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-fixes
+risc-v-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#fixes
+riscv-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-fixes
+riscv-soc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-fixes
+fpga-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#fixes
+spdx		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git#spdx-linus
+gpio-brgl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-current
+gpio-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#fixes
+pinctrl-intel-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#fixes
+auxdisplay-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#fixes
+kunit-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit-fixes
+memblock-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#fixes
+renesas-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#fixes
+perf-current	https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git#perf-tools
+efi-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#urgent
+battery-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#fixes
+iommufd-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-rc
+rust-fixes	https://github.com/Rust-for-Linux/linux.git#rust-fixes
+w1-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#fixes
+pmdomain-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#fixes
+i2c-host-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-host-fixes
+i2c-rust-fixes	https://github.com/ikrtn/rust-for-linux#rust-i2c-fixes
+sparc-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-linus
+clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-fixes
+thead-clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-fixes
+tenstorrent-clk-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-clk-fixes
+pwrseq-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-current
+thead-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-fixes
+ftrace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ftrace/fixes
+ring-buffer-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#ring-buffer/fixes
+trace-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#trace/fixes
+tracefs-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#tracefs/fixes
+spacemit-fixes	https://github.com/spacemit-com/linux#fixes
+tip-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#tip/urgent
+slab-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git#slab/for-next-fixes
+kexec-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#kexec-fixes
+liveupdate-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#fixes
+drm-msm-fixes	https://gitlab.freedesktop.org/drm/msm.git#msm-fixes
+uml-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#fixes
+fwctl-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-rc
+devsec-tsm-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#fixes
+drm-rust-fixes	https://gitlab.freedesktop.org/drm/rust/kernel.git#for-linux-next-fixes
+tenstorrent-dt-fixes	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-dt-fixes
+nfc-fixes	https://codeberg.org/linux-nfc/linux.git#for-linus
+drm-misc-fixes	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next-fixes
+rust		https://github.com/Rust-for-Linux/linux.git#rust-next
+rust-interop	https://github.com/Rust-for-Linux/linux.git#interop-next
+rust-alloc	https://github.com/Rust-for-Linux/linux.git#alloc-next
+rust-io		https://github.com/Rust-for-Linux/linux.git#io-next
+rust-pin-init	https://github.com/Rust-for-Linux/linux.git#pin-init-next
+rust-timekeeping	https://github.com/Rust-for-Linux/linux.git#timekeeping-next
+rust-xarray	https://github.com/Rust-for-Linux/linux.git#xarray-next
+rust-analyzer	https://github.com/Rust-for-Linux/linux.git#rust-analyzer-next
+mm-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-stable
+mm-nonmm-stable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-stable
+mm-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-unstable
+mm-nonmm-unstable	https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm#mm-nonmm-unstable
+kbuild		https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git#kbuild-for-next
+clang-format	https://github.com/ojeda/linux.git#clang-format
+perf		https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git#perf-tools-next
+compiler-attributes	https://github.com/ojeda/linux.git#compiler-attributes
+dma-mapping	https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git#dma-mapping-for-next
+asm-generic	https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic#master
+nocache-cleanup	https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git#nocache-cleanup
+alpha		https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git#alpha-next
+arm		https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git#for-next
+arm64		https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux#for-next/core
+arm-perf	https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git#for-next/perf
+arm-soc		https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git#for-next
+amlogic		https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git#for-next
+asahi-soc	https://github.com/AsahiLinux/linux.git#asahi-soc/for-next
+at91		https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git#at91-next
+bmc		https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git#for-next
+broadcom	https://github.com/Broadcom/stblinux.git#next
+cix		https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git#for-next
+davinci		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#davinci/for-next
+drivers-memory	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git#for-next
+fsl		https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git#soc_fsl
+imx-mxs		https://git.kernel.org/pub/scm/linux/kernel/git/frank.li/linux.git#for-next
+mediatek	https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git#for-next
+mvebu		https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git#for-next
+omap		https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git#for-next
+qcom		https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git#for-next
+renesas		https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git#next
+reset		https://git.pengutronix.de/git/pza/linux#reset/next
+rockchip	https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git#for-next
+samsung-krzk	https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git#for-next
+scmi		https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git#for-linux-next
+sophgo		https://github.com/sophgo/linux.git#for-next
+sophgo-soc	https://github.com/sophgo/linux.git#soc-for-next
+spacemit	https://github.com/spacemit-com/linux#for-next
+stm32		https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git#stm32-next
+sunxi		https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git#sunxi/for-next
+tee		https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git#next
+tegra		https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git#for-next
+tenstorrent-dt	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-dt-for-next
+thead-dt	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-dt-for-next
+ti		https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git#ti-next
+xilinx		https://github.com/Xilinx/linux-xlnx.git#for-next
+socfpga		https://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux.git#for-next
+clk		https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git#clk-next
+clk-imx		https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git#for-next
+clk-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-clk
+thead-clk	https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git#thead-clk-for-next
+tenstorrent-clk	https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git#tenstorrent-clk-for-next
+csky		https://github.com/c-sky/csky-linux.git#linux-next
+loongarch	https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git#loongarch-next
+m68k		https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git#for-next
+m68knommu	https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git#for-next
+microblaze	git://git.monstr.eu/linux-2.6-microblaze.git#next
+mips		https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git#mips-next
+openrisc	https://github.com/openrisc/linux.git#for-next
+parisc-hd	https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git#for-next
+powerpc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#next
+risc-v		https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git#for-next
+riscv-dt	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-dt-for-next
+riscv-soc	https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git#riscv-soc-for-next
+s390		https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git#for-next
+sh		https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git#for-next
+sparc		https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git#for-next
+uml		https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git#next
+xtensa		https://github.com/jcmvbkbc/linux-xtensa.git#xtensa-for-next
+printk		https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git#for-next
+pci		https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git#next
+pstore		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/pstore
+hid		https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git#for-next
+i2c		https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git#i2c/for-next
+i2c-host	https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git#i2c/i2c-host
+i2c-rust	https://github.com/ikrtn/rust-for-linux#rust-i2c-next
+i3c		https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git#i3c/next
+dmi		https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git#dmi-for-next
+hwmon-staging	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#hwmon-next
+jc_docs		git://git.lwn.net/linux.git#docs-next
+v4l-dvb		git://linuxtv.org/media-ci/media-pending.git#next
+v4l-dvb-next	git://linuxtv.org/mchehab/media-next.git#master
+pm		https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git#linux-next
+cpufreq-arm	https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#cpufreq/arm/linux-next
+cpupower	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git#cpupower
+devfreq		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git#devfreq-next
+pmdomain	https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git#next
+opp		https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git#opp/linux-next
+thermal		https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git#thermal/linux-next
+rdma		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#for-next
+net-next	https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git#main
+bpf-next	https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git#for-next
+ipsec-next	https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git#master
+mlx5-next	https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git#mlx5-next
+netfilter-next	https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git#main
+ipvs-next	https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git#main
+bluetooth	https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git#master
+wireless-next	https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git#for-next
+ath-next	https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git#for-next
+iwlwifi-next	https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git#next
+wpan-next	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#master
+wpan-staging	https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git#staging
+mtd		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#mtd/next
+nand		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#nand/next
+spi-nor		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#spi-nor/next
+crypto		https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git#master
+libcrypto	https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#libcrypto-next
+drm		https://gitlab.freedesktop.org/drm/kernel.git#drm-next
+drm-exynos	https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git#for-linux-next
+drm-misc	https://gitlab.freedesktop.org/drm/misc/kernel.git#for-linux-next
+amdgpu		https://gitlab.freedesktop.org/agd5f/linux.git#drm-next
+drm-intel	https://gitlab.freedesktop.org/drm/i915/kernel.git#for-linux-next
+drm-msm		https://gitlab.freedesktop.org/drm/msm.git#msm-next
+drm-msm-lumag	https://gitlab.freedesktop.org/lumag/msm.git#msm-next-lumag
+drm-xe		https://gitlab.freedesktop.org/drm/xe/kernel.git#drm-xe-next
+drm-rust	https://gitlab.freedesktop.org/drm/rust/kernel.git#for-linux-next
+drm-nova	https://gitlab.freedesktop.org/drm/nova.git#nova-next
+etnaviv		https://git.pengutronix.de/git/lst/linux#etnaviv/next
+fbdev		https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git#for-next
+regmap		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git#for-next
+sound		https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git#for-next
+ieee1394	https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git#for-next
+sound-asoc	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git#for-next
+modules		https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git#modules-next
+input		https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git#next
+block		https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git#for-next
+device-mapper	https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git#for-next
+libata		https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux#for-next
+pcmcia		https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git#pcmcia-next
+mmc		https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git#next
+mfd		https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git#for-mfd-next
+backlight	https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git#for-backlight-next
+battery		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git#for-next
+regulator	https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git#for-next
+security	https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git#next
+apparmor	https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor#apparmor-next
+integrity	https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity#next-integrity
+selinux		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git#next
+smack		https://github.com/cschaufler/smack-next#next
+tomoyo		git://git.code.sf.net/p/tomoyo/tomoyo.git#master
+tpmdd-tpm	https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git#for-next-tpm
+tpmdd-keys	https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git#for-next-keys
+watchdog	https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git#watchdog-next
+iommu		https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git#next
+audit		https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git#next
+devicetree	https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git#for-next
+dt-krzk		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git#for-next
+mailbox		https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git#for-next
+spi		https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git#for-next
+tip		https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git#master
+kexec		https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#kexec-next
+liveupdate	https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git#next
+clockevents	https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git#timers/drivers/next
+edac		https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git#edac-for-next
+ftrace		https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git#for-next
+rcu		https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux#next
+paulmck		https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git#non-rcu/next
+kvm		git://git.kernel.org/pub/scm/virt/kvm/kvm.git#next
+kvm-arm		https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git#next
+kvms390		https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git#next
+kvm-ppc		https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git#topic/ppc-kvm
+kvm-riscv	https://github.com/kvm-riscv/linux.git#riscv_kvm_next
+kvm-x86		https://github.com/kvm-x86/linux.git#next
+xen-tip		https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git#linux-next
+percpu		https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git#for-next
+workqueues	https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git#for-next
+sched-ext	https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git#for-next
+drivers-x86	https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git#for-next
+chrome-platform	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-next
+chrome-platform-firmware	https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git#for-firmware-next
+hsi		https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git#for-next
+leds-lj		https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git#for-leds-next
+ipmi		https://github.com/cminyard/linux-ipmi.git#for-next
+driver-core	https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git#driver-core-next
+usb		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git#usb-next
+thunderbolt	https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git#next
+usb-serial	https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git#usb-next
+tty		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git#tty-next
+char-misc	https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git#char-misc-next
+coresight	https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git#next
+fastrpc		https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git#for-next
+fpga		https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git#for-next
+icc		https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git#icc-next
+iio		https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git#togreg
+phy-next	https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git#next
+soundwire	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git#next
+extcon		https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git#extcon-next
+gnss		https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git#gnss-next
+vfio		https://github.com/awilliam/linux-vfio.git#next
+w1		https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git#for-next
+spmi		https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git#spmi-next
+staging		https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git#staging-next
+counter-next	https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git#counter-next
+mux		https://gitlab.com/peda-linux/mux.git#for-next
+dmaengine	https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git#next
+cgroup		https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git#for-next
+scsi		https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git#for-next
+scsi-mkp	https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git#for-next
+vhost		https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git#linux-next
+rpmsg		https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git#for-next
+gpio-brgl	https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#gpio/for-next
+gpio-intel	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git#for-next
+pinctrl		https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git#for-next
+pinctrl-intel	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git#for-next
+pinctrl-renesas	https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git#renesas-pinctrl
+pinctrl-samsung	https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git#for-next
+pwm		https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git#pwm/for-next
+ktest		https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git#for-next
+kselftest	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#next
+kunit		https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#test
+kunit-next	https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git#kunit
+livepatching	https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git#for-next
+rtc		https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git#rtc-next
+nvdimm		https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git#libnvdimm-for-next
+at24		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#at24/for-next
+ntb		https://github.com/jonmason/ntb.git#ntb-next
+seccomp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/seccomp
+slimbus		https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git#for-next
+nvmem		https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git#for-next
+hyperv		https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git#hyperv-next
+auxdisplay	https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git#for-next
+kgdb		https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git#kgdb/for-next
+hmm		https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git#hmm
+cfi		https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git#cfi/next
+mhi		https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git#mhi-next
+memblock	https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git#for-next
+cxl		https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git#next
+zstd		https://github.com/terrelln/linux.git#zstd-next
+efi		https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git#next
+unicode		https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git#for-next
+slab		https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git#slab/for-next
+random		https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git#master
+landlock	https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git#next
+sysctl		https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git#sysctl-next
+execve		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/execve
+bitmap		https://github.com/norov/linux.git#bitmap-for-next
+hte		https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git#for-next
+kspp		https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git#for-next/kspp
+nolibc		https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git#for-next
+iommufd		https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git#for-next
+turbostat	https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git#next
+pwrseq		https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git#pwrseq/for-next
+capabilities-next	https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git#caps-next
+ipe		https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git#next
+kcsan		https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git#next
+crc		https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git#crc-next
+keys-next	https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git#keys-next
+fwctl		https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git#for-next
+devsec-tsm	https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git#next
+hisilicon	https://github.com/hisilicon/linux-hisi.git#for-next
+kthread		https://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git#for-next
+nfc		https://codeberg.org/linux-nfc/linux.git#for-next
diff --git a/Next/merge.log b/Next/merge.log
new file mode 100644
index 0000000000000..cc391a3fdea14
--- /dev/null
+++ b/Next/merge.log
@@ -0,0 +1,11172 @@
+$ date -R
+Thu, 21 May 2026 09:41:26 +0100
+$ git checkout master
+Already on 'master'
+$ git reset --hard stable
+Updating files:  57% (4114/7187)
Updating files:  58% (4169/7187)
Updating files:  59% (4241/7187)
Updating files:  60% (4313/7187)
Updating files:  61% (4385/7187)
Updating files:  62% (4456/7187)
Updating files:  63% (4528/7187)
Updating files:  64% (4600/7187)
Updating files:  65% (4672/7187)
Updating files:  66% (4744/7187)
Updating files:  67% (4816/7187)
Updating files:  68% (4888/7187)
Updating files:  69% (4960/7187)
Updating files:  70% (5031/7187)
Updating files:  71% (5103/7187)
Updating files:  72% (5175/7187)
Updating files:  73% (5247/7187)
Updating files:  74% (5319/7187)
Updating files:  75% (5391/7187)
Updating files:  76% (5463/7187)
Updating files:  77% (5534/7187)
Updating files:  78% (5606/7187)
Updating files:  79% (5678/7187)
Updating files:  80% (5750/7187)
Updating files:  81% (5822/7187)
Updating files:  82% (5894/7187)
Updating files:  83% (5966/7187)
Updating files:  84% (6038/7187)
Updating files:  85% (6109/7187)
Updating files:  86% (6181/7187)
Updating files:  87% (6253/7187)
Updating files:  88% (6325/7187)
Updating files:  89% (6397/7187)
Updating files:  90% (6469/7187)
Updating files:  91% (6541/7187)
Updating files:  92% (6613/7187)
Updating files:  93% (6684/7187)
Updating files:  94% (6756/7187)
Updating files:  95% (6828/7187)
Updating files:  96% (6900/7187)
Updating files:  97% (6972/7187)
Updating files:  98% (7044/7187)
Updating files:  99% (7116/7187)
Updating files: 100% (7187/7187)
Updating files: 100% (7187/7187), done.
+HEAD is now at 27fa82620cbaa Merge tag 'ata-7.1-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/libata/linux
+Merging origin/master (8bc67e4db64aa Merge tag 'erofs-for-7.1-rc5-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git origin/master
+Updating 27fa82620cbaa..8bc67e4db64aa
+Fast-forward (no commit created; -m option ignored)
+ fs/erofs/xattr.c        |  4 +---
+ fs/erofs/zdata.c        | 15 ++++++++-------
+ kernel/rcu/srcutree.c   | 12 ++++++------
+ security/lsm_syscalls.c |  9 ++++++++-
+ 4 files changed, 23 insertions(+), 17 deletions(-)
+Merging ext4-fixes/fixes (981fcc5674e67 jbd2: fix deadlock in jbd2_journal_cancel_revoke())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4-fixes/fixes
+Already up to date.
+Merging vfs-brauner-fixes/vfs.fixes (45205929a3a33 Merge patch series "netfs: Miscellaneous fixes")
+$ git merge -m Merge branch 'vfs.fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner-fixes/vfs.fixes
+Already up to date.
+Merging fscrypt-current/for-current (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt-current/for-current
+Already up to date.
+Merging fsverity-current/for-current (0c8c88b8eb82a ovl: fix verity lazy-load guard broken by fsverity_active() semantic change)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity-current/for-current
+Already up to date.
+Merging btrfs-fixes/next-fixes (5f10ad875f9ee Merge branch 'misc-7.1' into next-fixes)
+$ git merge -m Merge branch 'next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs-fixes/next-fixes
+Merge made by the 'ort' strategy.
+ fs/btrfs/extent_map.c |   4 +-
+ fs/btrfs/file.c       |  11 ++++--
+ fs/btrfs/fs.h         |   1 +
+ fs/btrfs/inode.c      |   4 +-
+ fs/btrfs/qgroup.c     | 101 +++++++++++++++++++++++++++++++++-----------------
+ 5 files changed, 83 insertions(+), 38 deletions(-)
+Merging vfs-fixes/fixes (49c5d168a3a8f udf: fix nls leak on udf_fill_super() failure)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs-fixes/fixes
+Auto-merging fs/udf/super.c
+Merge made by the 'ort' strategy.
+ fs/udf/super.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging erofs-fixes/fixes (79b09c54c6563 erofs: fix metabuf leak in inode xattr initialization)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs-fixes/fixes
+Already up to date.
+Merging nfsd-fixes/nfsd-fixes (4f8ef58c10bfe NFSD: Fix infinite loop in layout state revocation)
+$ git merge -m Merge branch 'nfsd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd-fixes/nfsd-fixes
+Already up to date.
+Merging v9fs-fixes/fixes/next (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'fixes/next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-fixes/fixes/next
+Already up to date.
+Merging overlayfs-fixes/ovl-fixes (1f6ee9be92f8d ovl: make fsync after metadata copy-up opt-in mount option)
+$ git merge -m Merge branch 'ovl-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs-fixes/ovl-fixes
+Already up to date.
+Merging fscrypt/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fscrypt/linux.git fscrypt/for-next
+Already up to date.
+Merging btrfs/for-next (6bf684b882355 Merge branch 'for-next-next-v7.0-20260408' into for-next-20260408)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kdave/linux.git btrfs/for-next
+Merge made by the 'ort' strategy.
+Merging ceph/master (b01cb51ba5652 rbd: switch to dynamic root device)
+$ git merge -m Merge branch 'master' of https://github.com/ceph/ceph-client.git ceph/master
+Merge made by the 'ort' strategy.
+ drivers/block/rbd.c | 43 +++++++++++++++----------------------------
+ 1 file changed, 15 insertions(+), 28 deletions(-)
+Merging cifs/for-next (19d8c94787da8 smb: client: detect short folioq copy in cifs_copy_folioq_to_iter())
+$ git merge -m Merge branch 'for-next' of git://git.samba.org/sfrench/cifs-2.6.git cifs/for-next
+Auto-merging fs/smb/client/cifsfs.c
+Auto-merging fs/smb/client/inode.c
+Auto-merging fs/smb/client/smb2ops.c
+Merge made by the 'ort' strategy.
+ fs/smb/client/cached_dir.c    |   4 +
+ fs/smb/client/cifs_spnego.c   |  16 +++
+ fs/smb/client/cifs_swn.c      | 316 ++++++++++++++++++++++++++++++++++--------
+ fs/smb/client/cifsacl.c       | 196 +++++++++++++++++---------
+ fs/smb/client/cifsfs.c        |   2 +
+ fs/smb/client/connect.c       |   5 +
+ fs/smb/client/fs_context.c    |   2 +-
+ fs/smb/client/inode.c         |  30 +++-
+ fs/smb/client/netlink.c       |   6 +-
+ fs/smb/client/smb2ops.c       |  22 ++-
+ fs/smb/client/smb2transport.c |   2 +
+ fs/smb/client/trace.h         |   2 +
+ fs/smb/smbdirect/debug.c      |   2 +-
+ 13 files changed, 474 insertions(+), 131 deletions(-)
+Merging configfs/configfs-next (6363844fdbbb7 samples: configfs: Constify struct configfs_item_operations and configfs_group_operations)
+$ git merge -m Merge branch 'configfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/a.hindborg/linux.git configfs/configfs-next
+Already up to date.
+Merging ecryptfs/next (4d97a37e2cedb ecryptfs: remove redundant variable found_auth_tok)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/tyhicks/ecryptfs.git ecryptfs/next
+Merge made by the 'ort' strategy.
+ fs/ecryptfs/keystore.c | 16 +++++-----------
+ 1 file changed, 5 insertions(+), 11 deletions(-)
+Merging dlm/next (e61113cfcaf71 dlm: init per node debugfs before add to node hash)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm.git dlm/next
+Merge made by the 'ort' strategy.
+ fs/dlm/lowcomms.c | 12 ++++++++----
+ fs/dlm/memory.c   |  4 +++-
+ fs/dlm/midcomms.c | 21 +++++++++++++--------
+ 3 files changed, 24 insertions(+), 13 deletions(-)
+Merging erofs/dev (a5242d37c83ab erofs: error out obviously illegal extents in advance)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git erofs/dev
+Already up to date.
+Merging exfat/dev (ec5e84e1605d2 exfat: simplify exfat_lookup())
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/exfat.git exfat/dev
+Merge made by the 'ort' strategy.
+ fs/exfat/dir.c   |  4 +++-
+ fs/exfat/namei.c | 56 +++++++++++++-------------------------------------------
+ fs/exfat/nls.c   | 19 +++++++++++++------
+ 3 files changed, 29 insertions(+), 50 deletions(-)
+Merging ext3/for_next (5f0419457f89d udf: validate free block extents against the partition length)
+$ git merge -m Merge branch 'for_next' of https://git.kernel.org/pub/scm/linux/kernel/git/jack/linux-fs.git ext3/for_next
+Merge made by the 'ort' strategy.
+ fs/udf/balloc.c | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+Merging ext4/dev (981fcc5674e67 jbd2: fix deadlock in jbd2_journal_cancel_revoke())
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/tytso/ext4.git ext4/dev
+Already up to date.
+Merging f2fs/dev (520760b9f9156 f2fs: optimize representative type determination in GC)
+$ git merge -m Merge branch 'dev' of https://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs.git f2fs/dev
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-fs-f2fs |  5 ++--
+ Documentation/filesystems/f2fs.rst      | 50 ++++++++++++++++++++++++++++----
+ fs/f2fs/checkpoint.c                    |  2 +-
+ fs/f2fs/compress.c                      |  2 ++
+ fs/f2fs/data.c                          | 38 ++++++++++++++++++++++--
+ fs/f2fs/dir.c                           |  2 ++
+ fs/f2fs/extent_cache.c                  | 19 +++++-------
+ fs/f2fs/f2fs.h                          | 32 +++++++++++++++++----
+ fs/f2fs/file.c                          | 13 +++++++--
+ fs/f2fs/gc.c                            | 11 +++++--
+ fs/f2fs/inline.c                        |  3 ++
+ fs/f2fs/inode.c                         | 24 ++++++++++++++--
+ fs/f2fs/node.c                          | 14 ++++++++-
+ fs/f2fs/node.h                          |  8 ++++--
+ fs/f2fs/recovery.c                      |  2 ++
+ fs/f2fs/segment.c                       | 23 +++++++++++++++
+ fs/f2fs/super.c                         | 51 +++++++++++++++++++++++++++++++--
+ fs/f2fs/sysfs.c                         |  2 ++
+ fs/f2fs/verity.c                        |  2 ++
+ fs/f2fs/xattr.c                         | 41 +++++++++++++++++++++++++-
+ fs/f2fs/xattr.h                         |  6 ++++
+ include/trace/events/f2fs.h             | 28 ++++++++++++++++++
+ 22 files changed, 336 insertions(+), 42 deletions(-)
+Merging fsverity/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/fsverity/linux.git fsverity/for-next
+Already up to date.
+Merging fuse/for-next (040d71ac64709 fuse: use current creds for backing files)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszeredi/fuse.git fuse/for-next
+Auto-merging MAINTAINERS
+Auto-merging fs/fuse/file.c
+Merge made by the 'ort' strategy.
+ MAINTAINERS           |   26 +-
+ fs/fuse/Makefile      |    3 +-
+ fs/fuse/args.h        |   65 +++
+ fs/fuse/backing.c     |    3 +-
+ fs/fuse/control.c     |   16 +-
+ fs/fuse/cuse.c        |   28 +-
+ fs/fuse/dev.c         | 1271 ++++++++++++++++++-------------------------------
+ fs/fuse/dev.h         |  110 +++++
+ fs/fuse/dev_uring.c   |  143 +++---
+ fs/fuse/dev_uring_i.h |   39 +-
+ fs/fuse/dir.c         |   12 +-
+ fs/fuse/file.c        |  146 +-----
+ fs/fuse/fuse_dev_i.h  |  350 +++++++++++++-
+ fs/fuse/fuse_i.h      |  862 +++++++++++----------------------
+ fs/fuse/fuse_trace.h  |    4 +-
+ fs/fuse/inode.c       |  263 +++-------
+ fs/fuse/notify.c      |  434 +++++++++++++++++
+ fs/fuse/poll.c        |  141 ++++++
+ fs/fuse/req.c         |   99 ++++
+ fs/fuse/req_timeout.c |  148 ++++++
+ fs/fuse/sysctl.c      |    1 +
+ fs/fuse/sysctl.h      |    9 +
+ fs/fuse/virtio_fs.c   |   21 +-
+ 23 files changed, 2320 insertions(+), 1874 deletions(-)
+ create mode 100644 fs/fuse/args.h
+ create mode 100644 fs/fuse/dev.h
+ create mode 100644 fs/fuse/notify.c
+ create mode 100644 fs/fuse/poll.c
+ create mode 100644 fs/fuse/req.c
+ create mode 100644 fs/fuse/req_timeout.c
+ create mode 100644 fs/fuse/sysctl.h
+Merging gfs2/for-next (f9c9ec2c319f8 gfs2: fix use-after-free in gfs2_qd_dealloc)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gfs2/linux-gfs2.git gfs2/for-next
+Merge made by the 'ort' strategy.
+ fs/gfs2/quota.c | 36 ++++++++++++++++++++++++------------
+ fs/gfs2/super.c |  1 +
+ 2 files changed, 25 insertions(+), 12 deletions(-)
+Merging jfs/jfs-next (dad98c5b2a05e jfs: avoid -Wtautological-constant-out-of-range-compare warning again)
+$ git merge -m Merge branch 'jfs-next' of https://github.com/kleikamp/linux-shaggy.git jfs/jfs-next
+Already up to date.
+Merging ksmbd/ksmbd-for-next (1a2ab0feaa231 Merge tag 'v7.1-rc4-ksmbd-server-fixes' of git://git.samba.org/ksmbd)
+$ git merge -m Merge branch 'ksmbd-for-next' of https://github.com/smfrench/smb3-kernel.git ksmbd/ksmbd-for-next
+Already up to date.
+Merging nfs/linux-next (7fd2df204f342 Linux 7.1-rc2)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/trondmy/nfs-2.6.git nfs/linux-next
+Already up to date.
+Merging nfs-anna/linux-next (4529e00154329 NFS: Fix NFS KConfig typos)
+$ git merge -m Merge branch 'linux-next' of git://git.linux-nfs.org/projects/anna/linux-nfs.git nfs-anna/linux-next
+Already up to date.
+Merging nfsd/nfsd-next (dc7a9c0a4a32e NFSD: Update my maintainer email addresses)
+$ git merge -m Merge branch 'nfsd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux nfsd/nfsd-next
+Auto-merging MAINTAINERS
+Auto-merging fs/nfsd/nfs4state.c
+Auto-merging net/sunrpc/cache.c
+Merge made by the 'ort' strategy.
+ Documentation/netlink/specs/nfsd.yaml         | 229 +++++++++
+ Documentation/netlink/specs/sunrpc_cache.yaml | 149 ++++++
+ MAINTAINERS                                   |   8 +-
+ fs/nfs/dns_resolve.c                          |   2 +-
+ fs/nfsd/export.c                              | 714 +++++++++++++++++++++++++-
+ fs/nfsd/netlink.c                             |  93 ++++
+ fs/nfsd/netlink.h                             |  18 +
+ fs/nfsd/nfs4idmap.c                           |   4 +-
+ fs/nfsd/nfs4state.c                           |   9 +-
+ fs/nfsd/nfsctl.c                              |  79 +++
+ fs/nfsd/nfsd.h                                |  17 +-
+ include/linux/sunrpc/cache.h                  |  15 +-
+ include/uapi/linux/nfsd_netlink.h             | 132 +++++
+ include/uapi/linux/sunrpc_netlink.h           |  84 +++
+ net/sunrpc/Makefile                           |   2 +-
+ net/sunrpc/auth_gss/svcauth_gss.c             |   2 +-
+ net/sunrpc/cache.c                            | 127 ++++-
+ net/sunrpc/netlink.c                          |  97 ++++
+ net/sunrpc/netlink.h                          |  35 ++
+ net/sunrpc/sunrpc_syms.c                      |  10 +
+ net/sunrpc/svc_xprt.c                         |  46 +-
+ net/sunrpc/svcauth_unix.c                     | 516 ++++++++++++++++++-
+ 22 files changed, 2336 insertions(+), 52 deletions(-)
+ create mode 100644 Documentation/netlink/specs/sunrpc_cache.yaml
+ create mode 100644 include/uapi/linux/sunrpc_netlink.h
+ create mode 100644 net/sunrpc/netlink.c
+ create mode 100644 net/sunrpc/netlink.h
+Merging ntfs/ntfs-next (29abfa1b2fb44 ntfs: remove unsupported quota handling)
+$ git merge -m Merge branch 'ntfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linkinjeon/ntfs.git ntfs/ntfs-next
+Merge made by the 'ort' strategy.
+ fs/ntfs/Makefile |   2 +-
+ fs/ntfs/attrib.c |   3 +-
+ fs/ntfs/quota.c  |  95 ------------------------------------------------
+ fs/ntfs/quota.h  |  15 --------
+ fs/ntfs/super.c  | 109 +------------------------------------------------------
+ fs/ntfs/volume.h |   7 ----
+ 6 files changed, 4 insertions(+), 227 deletions(-)
+ delete mode 100644 fs/ntfs/quota.c
+ delete mode 100644 fs/ntfs/quota.h
+Merging ntfs3/master (4a10702961161 fs/ntfs3: preserve non-DOS attribute bits in system.dos_attrib)
+$ git merge -m Merge branch 'master' of https://github.com/Paragon-Software-Group/linux-ntfs3.git ntfs3/master
+Merge made by the 'ort' strategy.
+ fs/ntfs3/dir.c     | 22 ++++++++++++++++------
+ fs/ntfs3/fslog.c   |  5 ++++-
+ fs/ntfs3/index.c   | 46 +++++++++++++++++++++++++++++++++++++++++++---
+ fs/ntfs3/lznt.c    |  2 +-
+ fs/ntfs3/namei.c   |  2 +-
+ fs/ntfs3/ntfs_fs.h |  3 ++-
+ fs/ntfs3/run.c     |  9 +++++++--
+ fs/ntfs3/xattr.c   |  4 +++-
+ 8 files changed, 77 insertions(+), 16 deletions(-)
+$ git am -3 ../patches/0001-ntfs3-Fix-merge-issue-with-ntfs3.patch
+Applying: ntfs3: Fix merge issue with ntfs3
+Using index info to reconstruct a base tree...
+M	fs/ntfs3/inode.c
+Falling back to patching base and 3-way merge...
+Auto-merging fs/ntfs3/inode.c
+No changes -- Patch already applied.
+Merging orangefs/for-next (e61bc5e4d8743    bufmap: manage as folios, V2.)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hubcap/linux.git orangefs/for-next
+Already up to date.
+Merging overlayfs/overlayfs-next (1f6ee9be92f8d ovl: make fsync after metadata copy-up opt-in mount option)
+$ git merge -m Merge branch 'overlayfs-next' of https://git.kernel.org/pub/scm/linux/kernel/git/overlayfs/vfs.git overlayfs/overlayfs-next
+Already up to date.
+Merging ubifs/next (11efa98bcc0d0 ubi: ubi.h: fix kernel-doc warnings)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rw/ubifs.git ubifs/next
+Auto-merging drivers/mtd/ubi/ubi.h
+Merge made by the 'ort' strategy.
+ drivers/mtd/ubi/ubi.h | 9 ++++++---
+ 1 file changed, 6 insertions(+), 3 deletions(-)
+Merging v9fs/9p-next (8fc518e489c13 9p/trans_xen: replace simple_strto* with kstrtouint)
+$ git merge -m Merge branch '9p-next' of https://github.com/martinetd/linux v9fs/9p-next
+Already up to date.
+Merging v9fs-ericvh/ericvh/for-next (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'ericvh/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ericvh/v9fs.git v9fs-ericvh/ericvh/for-next
+Already up to date.
+Merging xfs/for-next (509fdeb3326be xfs: Fix typo in comment)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/fs/xfs/xfs-linux.git xfs/for-next
+Already up to date.
+Merging zonefs/for-next (3a8389d42bdf4 zonefs: handle integer overflow in zonefs_fname_to_fno)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dlemoal/zonefs.git zonefs/for-next
+Merge made by the 'ort' strategy.
+ fs/zonefs/super.c | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+Merging vfs-brauner/vfs.all (1202b9c177076 Merge branch 'vfs-7.2.misc' into vfs.all)
+$ git merge -m Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git vfs-brauner/vfs.all
+Auto-merging MAINTAINERS
+Auto-merging fs/btrfs/inode.c
+Auto-merging fs/exfat/namei.c
+Auto-merging fs/iomap/direct-io.c
+Auto-merging fs/nfsd/export.c
+Auto-merging fs/ntfs3/namei.c
+Auto-merging fs/ntfs3/ntfs_fs.h
+Auto-merging fs/smb/client/cifsfs.c
+Auto-merging fs/smb/client/inode.c
+Auto-merging fs/xfs/xfs_inode.c
+Auto-merging lib/rhashtable.c
+Merge made by the 'ort' strategy.
+ .../filesystems/adding-new-filesystems.rst         |  195 ++++
+ Documentation/filesystems/index.rst                |    1 +
+ Documentation/filesystems/porting.rst              |    1 -
+ Documentation/filesystems/proc.rst                 |   19 +-
+ MAINTAINERS                                        |    2 +-
+ arch/alpha/include/uapi/asm/errno.h                |    2 +
+ arch/alpha/include/uapi/asm/fcntl.h                |    1 +
+ arch/mips/include/uapi/asm/errno.h                 |    2 +
+ arch/parisc/include/uapi/asm/errno.h               |    2 +
+ arch/parisc/include/uapi/asm/fcntl.h               |    1 +
+ arch/powerpc/platforms/cell/spufs/file.c           |    2 +-
+ arch/sparc/include/uapi/asm/errno.h                |    2 +
+ arch/sparc/include/uapi/asm/fcntl.h                |    1 +
+ fs/affs/affs.h                                     |    5 -
+ fs/affs/super.c                                    |    6 +-
+ fs/befs/linuxvfs.c                                 |    3 +-
+ fs/bfs/inode.c                                     |    3 +-
+ fs/bpf_fs_kfuncs.c                                 |   10 +-
+ fs/btrfs/inode.c                                   |    2 +-
+ fs/ceph/file.c                                     |    4 +
+ fs/ceph/mds_client.c                               |    2 +-
+ fs/coredump.c                                      |    3 +-
+ fs/dcache.c                                        |   22 +-
+ fs/eventpoll.c                                     | 1207 +++++++++++++-------
+ fs/exec.c                                          |    6 +-
+ fs/exfat/exfat_fs.h                                |    2 +
+ fs/exfat/file.c                                    |   18 +-
+ fs/exfat/namei.c                                   |    1 +
+ fs/ext4/ialloc.c                                   |    4 +-
+ fs/fat/fat.h                                       |    3 +
+ fs/fat/file.c                                      |   36 +
+ fs/fat/namei_msdos.c                               |    1 +
+ fs/fat/namei_vfat.c                                |    1 +
+ fs/fcntl.c                                         |    4 +-
+ fs/file.c                                          |   17 +-
+ fs/file_attr.c                                     |   16 +-
+ fs/filesystems.c                                   |  332 ++++--
+ fs/gfs2/inode.c                                    |    6 +
+ fs/hfs/dir.c                                       |    1 +
+ fs/hfs/hfs_fs.h                                    |    2 +
+ fs/hfs/inode.c                                     |   14 +
+ fs/hfsplus/inode.c                                 |   16 +-
+ fs/hpfs/inode.c                                    |    2 +-
+ fs/hpfs/super.c                                    |    3 +-
+ fs/inode.c                                         |  100 +-
+ fs/iomap/buffered-io.c                             |    3 +-
+ fs/iomap/direct-io.c                               |    3 -
+ fs/iomap/iter.c                                    |   12 +-
+ fs/isofs/dir.c                                     |   16 +
+ fs/isofs/inode.c                                   |    3 +-
+ fs/isofs/isofs.h                                   |    3 +
+ fs/jfs/super.c                                     |    3 +-
+ fs/minix/inode.c                                   |    3 +-
+ fs/mount.h                                         |    4 +
+ fs/namei.c                                         |   29 +-
+ fs/namespace.c                                     |   34 +-
+ fs/nfs/client.c                                    |   21 +-
+ fs/nfs/dir.c                                       |    4 +
+ fs/nfs/inode.c                                     |   19 +-
+ fs/nfs/internal.h                                  |    3 +
+ fs/nfs/namespace.c                                 |    2 +
+ fs/nfs/nfs3proc.c                                  |    2 +
+ fs/nfs/nfs3xdr.c                                   |    7 +-
+ fs/nfs/nfs4proc.c                                  |   10 +-
+ fs/nfs/proc.c                                      |    3 +
+ fs/nfs/symlink.c                                   |    3 +
+ fs/nfsd/blocklayout.c                              |   37 +-
+ fs/nfsd/export.c                                   |    3 +-
+ fs/nfsd/nfs3proc.c                                 |   36 +-
+ fs/nfsd/nfs4layouts.c                              |   29 +-
+ fs/nfsd/nfs4xdr.c                                  |   52 +-
+ fs/nfsd/vfs.c                                      |   88 ++
+ fs/nfsd/vfs.h                                      |    3 +
+ fs/nfsd/xdr3.h                                     |    4 +-
+ fs/ntfs3/file.c                                    |   29 +
+ fs/ntfs3/namei.c                                   |    1 +
+ fs/ntfs3/ntfs_fs.h                                 |    1 +
+ fs/ntfs3/super.c                                   |    8 +-
+ fs/ocfs2/super.c                                   |    1 -
+ fs/omfs/inode.c                                    |    6 +-
+ fs/open.c                                          |   17 +-
+ fs/proc/generic.c                                  |   10 +
+ fs/proc/internal.h                                 |    3 +
+ fs/proc/proc_net.c                                 |    8 +
+ fs/proc/root.c                                     |   24 +-
+ fs/qnx4/inode.c                                    |    3 +-
+ fs/smb/client/cifsfs.c                             |   53 +
+ fs/smb/client/cifsfs.h                             |    3 +
+ fs/smb/client/dir.c                                |   18 +-
+ fs/smb/client/inode.c                              |    2 +-
+ fs/smb/client/namespace.c                          |    1 +
+ fs/smb/server/smb2pdu.c                            |   30 +-
+ fs/super.c                                         |    1 -
+ fs/sync.c                                          |    3 +-
+ fs/sysfs/mount.c                                   |   18 +-
+ fs/ubifs/super.c                                   |    2 +-
+ fs/vboxsf/dir.c                                    |    1 +
+ fs/vboxsf/file.c                                   |    6 +-
+ fs/vboxsf/super.c                                  |    7 +
+ fs/vboxsf/utils.c                                  |   30 +
+ fs/vboxsf/vfsmod.h                                 |    6 +
+ fs/xfs/libxfs/xfs_inode_util.c                     |    2 +
+ fs/xfs/xfs_export.c                                |    4 +-
+ fs/xfs/xfs_inode.c                                 |    2 +-
+ fs/xfs/xfs_ioctl.c                                 |   22 +-
+ fs/xfs/xfs_pnfs.c                                  |   44 +-
+ fs/xfs/xfs_pnfs.h                                  |   11 +-
+ fs/xfs/xfs_trace.h                                 |    2 +-
+ include/linux/dcache.h                             |    4 +-
+ include/linux/exportfs.h                           |   25 +-
+ include/linux/exportfs_block.h                     |   88 ++
+ include/linux/fcntl.h                              |    4 +-
+ include/linux/fileattr.h                           |    3 +-
+ include/linux/fs.h                                 |   16 +-
+ include/linux/fs/super_types.h                     |    2 +-
+ include/linux/iomap.h                              |   10 -
+ include/linux/kstrtox.h                            |    9 +-
+ include/linux/namei.h                              |    1 -
+ include/linux/nfs_fs_sb.h                          |    2 +-
+ include/linux/nfs_xdr.h                            |    2 +
+ include/linux/proc_fs.h                            |   13 +
+ include/linux/rhashtable-types.h                   |   22 +-
+ include/linux/sockptr.h                            |   28 +-
+ include/linux/uaccess.h                            |   65 +-
+ include/trace/events/filelock.h                    |    2 +-
+ include/uapi/asm-generic/errno.h                   |    2 +
+ include/uapi/asm-generic/fcntl.h                   |    8 +
+ include/uapi/linux/fs.h                            |    7 +
+ init/initramfs.c                                   |   68 +-
+ init/initramfs_test.c                              |   80 +-
+ kernel/acct.c                                      |    2 +-
+ lib/iov_iter.c                                     |    8 +-
+ lib/rhashtable.c                                   |   17 +-
+ lib/vsprintf.c                                     |    7 -
+ security/landlock/fs.c                             |    2 +-
+ tools/arch/alpha/include/uapi/asm/errno.h          |    2 +
+ tools/arch/mips/include/uapi/asm/errno.h           |    2 +
+ tools/arch/parisc/include/uapi/asm/errno.h         |    2 +
+ tools/arch/sparc/include/uapi/asm/errno.h          |    2 +
+ tools/include/uapi/asm-generic/errno.h             |    2 +
+ tools/include/uapi/linux/openat2.h                 |   43 +
+ .../selftests/{ => filesystems}/openat2/.gitignore |    0
+ .../selftests/{ => filesystems}/openat2/Makefile   |    9 +-
+ .../selftests/filesystems/openat2/emptypath_test.c |   55 +
+ .../selftests/filesystems/openat2/helpers.h        |  135 +++
+ .../{ => filesystems}/openat2/openat2_test.c       |  239 ++--
+ .../filesystems/openat2/rename_attack_test.c       |  159 +++
+ .../{ => filesystems}/openat2/resolve_test.c       |  354 +++---
+ .../selftests/namespaces/listns_efault_test.c      |   33 +-
+ tools/testing/selftests/namespaces/nsid_test.c     |   14 +-
+ tools/testing/selftests/openat2/helpers.c          |  109 --
+ tools/testing/selftests/openat2/helpers.h          |  108 --
+ .../testing/selftests/openat2/rename_attack_test.c |  160 ---
+ tools/testing/selftests/pid_namespace/pid_max.c    |  156 ++-
+ 154 files changed, 3379 insertions(+), 1603 deletions(-)
+ create mode 100644 Documentation/filesystems/adding-new-filesystems.rst
+ create mode 100644 include/linux/exportfs_block.h
+ create mode 100644 tools/include/uapi/linux/openat2.h
+ rename tools/testing/selftests/{ => filesystems}/openat2/.gitignore (100%)
+ rename tools/testing/selftests/{ => filesystems}/openat2/Makefile (65%)
+ create mode 100644 tools/testing/selftests/filesystems/openat2/emptypath_test.c
+ create mode 100644 tools/testing/selftests/filesystems/openat2/helpers.h
+ rename tools/testing/selftests/{ => filesystems}/openat2/openat2_test.c (67%)
+ create mode 100644 tools/testing/selftests/filesystems/openat2/rename_attack_test.c
+ rename tools/testing/selftests/{ => filesystems}/openat2/resolve_test.c (74%)
+ delete mode 100644 tools/testing/selftests/openat2/helpers.c
+ delete mode 100644 tools/testing/selftests/openat2/helpers.h
+ delete mode 100644 tools/testing/selftests/openat2/rename_attack_test.c
+$ git am -3 ../patches/0001-ntfs-Fixup-merge-issue-with-vfs.all-changes.patch
+Applying: ntfs: Fixup merge issue with vfs.all changes
+Using index info to reconstruct a base tree...
+M	fs/ntfs/mft.c
+Falling back to patching base and 3-way merge...
+Auto-merging fs/ntfs/mft.c
+No changes -- Patch already applied.
+$ git am -3 ../patches/0001-openat-Fix-collision-between-OPENAT2_REGULAR-and-O_E.patch
+Applying: openat: Fix collision between OPENAT2_REGULAR and O_EMPTYPATH
+$ git reset HEAD^
+Unstaged changes after reset:
+M	fs/fcntl.c
+M	include/uapi/asm-generic/fcntl.h
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[fs-next 52f573e5a817e] Merge branch 'vfs.all' of https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git
+ Date: Thu May 21 09:56:28 2026 +0100
+Merging vfs/for-next (e82ff9b6cb767 Merge branches 'work.coda' and 'work.dcache-busy-wait' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git vfs/for-next
+Merge made by the 'ort' strategy.
+Merging mm-hotfixes/mm-hotfixes-unstable (abd2c27f2cd99 mm-hugetlb-avoid-false-positive-lockdep-assertion-fix)
+$ git merge -m Merge branch 'mm-hotfixes-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-hotfixes/mm-hotfixes-unstable
+Merge made by the 'ort' strategy.
+ .mailmap                                         |   6 +-
+ MAINTAINERS                                      |  14 +--
+ drivers/block/zram/zram_drv.c                    |   6 +-
+ drivers/dax/bus.c                                |   7 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c |   2 +-
+ fs/hugetlbfs/inode.c                             |  46 +++-----
+ include/linux/hugetlb.h                          |   8 +-
+ include/linux/hugetlb_inline.h                   |  14 +--
+ include/linux/mm.h                               |  27 ++++-
+ include/linux/page-flags.h                       |   6 --
+ include/net/netmem.h                             |  19 +---
+ ipc/shm.c                                        |  10 +-
+ ipc/util.c                                       |   2 +-
+ kernel/fork.c                                    |  11 +-
+ mm/damon/sysfs-schemes.c                         |   8 +-
+ mm/hugetlb.c                                     | 132 ++++++++++++-----------
+ mm/memcontrol.c                                  |  75 +++++++++----
+ mm/memfd.c                                       |  12 +--
+ mm/migrate_device.c                              |   4 +-
+ mm/page_alloc.c                                  |  13 +--
+ mm/rmap.c                                        |   2 +
+ mm/swap_cgroup.c                                 |   2 +
+ mm/userfaultfd.c                                 |  99 +++++++++++++----
+ mm/vmalloc.c                                     |   2 +-
+ net/core/netmem_priv.h                           |  23 ++--
+ net/core/page_pool.c                             |  24 +----
+ 26 files changed, 316 insertions(+), 258 deletions(-)
+Merging fs-current (3b67b13aeb567 Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git)
+$ git merge -m Merge branch 'fs-current' of linux-next fs-current
+Merge made by the 'ort' strategy.
+ fs/btrfs/extent_map.c |   4 +-
+ fs/btrfs/file.c       |  11 ++++--
+ fs/btrfs/fs.h         |   1 +
+ fs/btrfs/inode.c      |   4 +-
+ fs/btrfs/qgroup.c     | 101 +++++++++++++++++++++++++++++++++-----------------
+ fs/udf/super.c        |   2 +-
+ 6 files changed, 84 insertions(+), 39 deletions(-)
+Merging kbuild-current/kbuild-fixes-for-next (f2be8dd21b3f4 kbuild: pacman-pkg: make "rc" releases adhere to pacman versioning scheme)
+$ git merge -m Merge branch 'kbuild-fixes-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild-current/kbuild-fixes-for-next
+Merge made by the 'ort' strategy.
+Merging clang-fixes/clang-fixes-for-next (bc5ffe737f56e Disable -Wattribute-alias for clang-23 and newer)
+$ git merge -m Merge branch 'clang-fixes-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nathan/linux.git clang-fixes/clang-fixes-for-next
+Merge made by the 'ort' strategy.
+ arch/riscv/include/asm/syscall_wrapper.h | 2 ++
+ include/linux/compat.h                   | 2 ++
+ include/linux/compiler-clang.h           | 6 ++++++
+ include/linux/compiler_types.h           | 4 ++++
+ include/linux/syscalls.h                 | 2 ++
+ 5 files changed, 16 insertions(+)
+Merging arc-current/for-curr (7aaa8047eafd0 Linux 7.0-rc6)
+$ git merge -m Merge branch 'for-curr' of https://git.kernel.org/pub/scm/linux/kernel/git/vgupta/arc.git arc-current/for-curr
+Already up to date.
+Merging arm-current/fixes (75f9a484e817a ARM: 9472/1: fix race condition on PG_dcache_clean in __sync_icache_dcache())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm-current/fixes
+Already up to date.
+Merging arm64-fixes/for-next/fixes (2ccd8ff980b50 arm64: probes: Handle probes on hinted conditional branch instructions)
+$ git merge -m Merge branch 'for-next/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64-fixes/for-next/fixes
+Merge made by the 'ort' strategy.
+ arch/arm64/include/asm/insn.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging arm-soc-fixes/arm/fixes (1fcf4149418e7 Merge tag 'riscv-dt-fixes-for-v7.1-rc3' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux into arm/fixes)
+$ git merge -m Merge branch 'arm/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc-fixes/arm/fixes
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |   5 +-
+ arch/arm/boot/dts/renesas/r7s72100-genmai.dts      |   3 -
+ arch/arm/boot/dts/renesas/r7s72100-rskrza1.dts     |   2 -
+ arch/arm/boot/dts/renesas/r7s72100.dtsi            |   2 +-
+ arch/arm/boot/dts/renesas/r8a7778.dtsi             |   2 +-
+ arch/arm/boot/dts/renesas/r8a7779.dtsi             |   2 +-
+ arch/arm/boot/dts/renesas/r8a7792.dtsi             |   2 +-
+ arch/arm/mach-versatile/integrator_cp.c            |  13 +-
+ .../dts/renesas/draak-ebisu-panel-aa104xd12.dtso   |   5 +
+ arch/arm64/boot/dts/renesas/r8a78000.dtsi          |   8 +-
+ arch/arm64/boot/dts/renesas/r9a09g056.dtsi         |   1 +
+ arch/arm64/boot/dts/renesas/r9a09g057.dtsi         |   2 +
+ .../boot/dts/renesas/rz-smarc-cru-csi-ov5645.dtsi  |   5 +
+ .../boot/dts/renesas/rz-smarc-du-adv7513.dtsi      |   5 +
+ .../boot/dts/renesas/salvator-panel-aa104xd12.dtso |   5 +
+ .../boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi |  10 --
+ .../boot/dts/microchip/mpfs-icicle-kit-prod.dts    |  10 ++
+ arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts  |  19 +++
+ arch/riscv/boot/dts/starfive/jh7110-common.dtsi    |  27 +---
+ arch/riscv/boot/dts/starfive/jh7110.dtsi           |  28 ----
+ drivers/firmware/arm_ffa/bus.c                     |   4 +-
+ drivers/firmware/arm_ffa/driver.c                  | 146 ++++++++++++++-------
+ drivers/firmware/psci/psci.c                       |  10 ++
+ 23 files changed, 184 insertions(+), 132 deletions(-)
+Merging davinci-current/davinci/for-current (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'davinci/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci-current/davinci/for-current
+Already up to date.
+Merging drivers-memory-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory-fixes/fixes
+Already up to date.
+Merging sophgo-fixes/fixes (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://github.com/sophgo/linux.git sophgo-fixes/fixes
+Already up to date.
+Merging sophgo-soc-fixes/soc-fixes (0af2f6be1b428 Linux 6.15-rc1)
+$ git merge -m Merge branch 'soc-fixes' of https://github.com/sophgo/linux.git sophgo-soc-fixes/soc-fixes
+Already up to date.
+Merging m68k-current/for-linus (cb39cf99d88e8 m68k: sun3: Replace vsprintf() with bounded vsnprintf())
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k-current/for-linus
+Already up to date.
+Merging powerpc-fixes/fixes (5200f5f493f79 Linux 7.1-rc4)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc-fixes/fixes
+Already up to date.
+Merging s390-fixes/fixes (540f4a4f6ef80 s390/topology: Use zero-based numbering for containing entities)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390-fixes/fixes
+Merge made by the 'ort' strategy.
+ arch/s390/kernel/perf_pai.c | 31 +++++++++++++++++++++----------
+ arch/s390/kernel/topology.c | 10 +++++++---
+ drivers/s390/cio/chsc.c     |  4 ++--
+ drivers/s390/cio/chsc_sch.c | 20 ++++++++++----------
+ drivers/s390/cio/scm.c      |  2 +-
+ 5 files changed, 41 insertions(+), 26 deletions(-)
+Merging net/main (1bbf0ced1d9db tcp: fix stale per-CPU tcp_tw_isn leak enabling ISN prediction)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net.git net/main
+Merge made by the 'ort' strategy.
+ Documentation/crypto/krb5.rst                      |  17 +-
+ crypto/krb5/krb5_api.c                             |  54 +++++-
+ drivers/bluetooth/btintel_pcie.c                   |  20 +--
+ drivers/bluetooth/btintel_pcie.h                   |   3 -
+ drivers/bluetooth/btmtk.c                          |   6 +-
+ drivers/bluetooth/hci_ldisc.c                      |  48 +++++-
+ drivers/bluetooth/hci_qca.c                        |  33 ++--
+ drivers/dpll/zl3073x/dpll.c                        |   6 +-
+ drivers/infiniband/ulp/ipoib/ipoib_main.c          |   8 +-
+ drivers/net/dsa/mt7530.c                           | 160 ++++++++++--------
+ drivers/net/ethernet/amd/pds_core/debugfs.c        |   7 +-
+ drivers/net/ethernet/amd/pds_core/dev.c            |  11 +-
+ drivers/net/ethernet/atheros/ag71xx.c              |   3 +
+ drivers/net/ethernet/intel/ice/ice_main.c          |   8 +-
+ drivers/net/ethernet/intel/ice/ice_ptp_hw.c        |  33 +++-
+ drivers/net/ethernet/intel/ice/virt/queues.c       |   2 +-
+ drivers/net/ethernet/intel/igc/igc_tsn.c           |   9 +-
+ drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c  |   1 +
+ drivers/net/ethernet/marvell/octeontx2/af/cgx.c    |   7 +-
+ drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c |   2 +
+ .../ethernet/marvell/octeontx2/nic/otx2_common.c   |   2 +
+ drivers/net/ethernet/marvell/octeontx2/nic/rep.c   |   3 +-
+ .../ethernet/mellanox/mlx5/core/en/reporter_tx.c   |   6 +-
+ .../ethernet/mellanox/mlx5/core/en_accel/ipsec.c   |   7 +-
+ .../net/ethernet/mellanox/mlx5/core/en_ethtool.c   |   5 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c  |   2 +-
+ .../net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c |   3 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.c  |  21 +++
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.h  |   1 +
+ drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c  |   5 +
+ drivers/net/ethernet/microsoft/mana/hw_channel.c   |  23 +--
+ drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c |   8 +-
+ drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h |   1 +
+ drivers/net/ifb.c                                  |  11 +-
+ drivers/net/ovpn/io.c                              |  12 +-
+ drivers/net/ovpn/main.c                            |  12 +-
+ drivers/net/ovpn/netlink.c                         |   8 +-
+ drivers/net/ovpn/peer.c                            |  23 ++-
+ drivers/net/ovpn/peer.h                            |   1 -
+ drivers/net/ovpn/stats.h                           |  16 ++
+ drivers/net/ovpn/tcp.c                             |  19 ++-
+ drivers/net/ovpn/udp.c                             |   2 +-
+ drivers/net/phy/phy-c45.c                          |   8 +
+ drivers/net/phy/phy_device.c                       |   6 +-
+ drivers/net/pse-pd/pse_core.c                      |   2 +-
+ include/crypto/krb5.h                              |   9 +-
+ include/linux/soc/airoha/airoha_offload.h          |   6 +-
+ include/net/bluetooth/bluetooth.h                  |   1 +
+ include/net/ip_vs.h                                |   3 +-
+ include/net/neighbour.h                            |   8 +-
+ include/net/net_shaper.h                           |   1 +
+ include/net/netfilter/nf_queue.h                   |   1 +
+ include/net/tcp.h                                  |   7 +-
+ include/trace/events/rxrpc.h                       |   1 +
+ net/appletalk/aarp.c                               |   5 +
+ net/batman-adv/distributed-arp-table.c             |   3 +
+ net/batman-adv/fragmentation.c                     |  58 ++++++-
+ net/batman-adv/gateway_client.c                    |   4 +
+ net/batman-adv/tp_meter.c                          |  36 ++--
+ net/batman-adv/translation-table.c                 |  55 ++++--
+ net/batman-adv/types.h                             |   9 +-
+ net/bluetooth/af_bluetooth.c                       |  99 ++++++++---
+ net/bluetooth/bnep/core.c                          |   2 +-
+ net/bluetooth/hci_sync.c                           |   6 +-
+ net/bluetooth/iso.c                                |  14 +-
+ net/bluetooth/l2cap_core.c                         |   2 +-
+ net/bluetooth/l2cap_sock.c                         |  51 +++++-
+ net/bluetooth/mgmt.c                               |   6 +
+ net/bluetooth/rfcomm/sock.c                        |   9 +-
+ net/bluetooth/sco.c                                |   9 +-
+ net/bridge/br_multicast.c                          |  22 ++-
+ net/bridge/br_netfilter_hooks.c                    |   6 +-
+ net/core/devmem.c                                  |  11 ++
+ net/core/skbuff.c                                  |   2 +
+ net/core/skmsg.c                                   |   9 +-
+ net/hsr/hsr_framereg.c                             |   4 +-
+ net/ipv4/icmp.c                                    |   3 +
+ net/ipv4/raw.c                                     |   2 +-
+ net/ipv4/route.c                                   |   2 +-
+ net/ipv4/tcp.c                                     |   3 -
+ net/ipv4/tcp_input.c                               |  15 +-
+ net/ipv4/tcp_ipv4.c                                |   3 +-
+ net/ipv4/udp_offload.c                             |  22 +--
+ net/ipv6/exthdrs.c                                 |  15 +-
+ net/ipv6/netfilter/ip6t_hbh.c                      |   4 +
+ net/ipv6/tcp_ipv6.c                                |   3 +-
+ net/mptcp/pm.c                                     |  56 ++++--
+ net/mptcp/protocol.c                               |  25 ++-
+ net/mptcp/sockopt.c                                |  10 +-
+ net/netfilter/ipset/ip_set_core.c                  |   5 +-
+ net/netfilter/ipset/ip_set_hash_gen.h              |  57 ++++---
+ net/netfilter/ipset/ip_set_hash_ipmark.c           |   6 +-
+ net/netfilter/ipset/ip_set_hash_ipport.c           |   5 +-
+ net/netfilter/ipset/ip_set_hash_ipportip.c         |   5 +-
+ net/netfilter/ipset/ip_set_hash_ipportnet.c        |   5 +-
+ net/netfilter/ipvs/ip_vs_ctl.c                     | 187 ++++++++++++++-------
+ net/netfilter/nf_conntrack_helper.c                |  13 +-
+ net/netfilter/nf_queue.c                           |   4 +-
+ net/netfilter/nfnetlink_queue.c                    |   2 +
+ net/netfilter/nft_inner.c                          |   3 +-
+ net/openvswitch/vport-netdev.c                     |   8 +
+ net/rds/tcp.c                                      |   9 +-
+ net/rxrpc/ar-internal.h                            |  14 +-
+ net/rxrpc/call_event.c                             |  22 +--
+ net/rxrpc/call_object.c                            |   2 +
+ net/rxrpc/conn_event.c                             |  30 ++--
+ net/rxrpc/insecure.c                               |   8 +-
+ net/rxrpc/recvmsg.c                                |  68 ++++++--
+ net/rxrpc/rxgk.c                                   | 160 ++++++++----------
+ net/rxrpc/rxgk_app.c                               |  46 +++--
+ net/rxrpc/rxgk_common.h                            |  66 ++++----
+ net/rxrpc/rxkad.c                                  | 115 +++++--------
+ net/shaper/shaper.c                                |  98 ++++++-----
+ net/tls/tls_sw.c                                   |  20 ++-
+ net/unix/af_unix.c                                 |  11 +-
+ net/vmw_vsock/virtio_transport_common.c            |  83 ++++-----
+ net/vmw_vsock/vmci_transport.c                     |   2 +-
+ .../selftests/bpf/prog_tests/sockmap_ktls.c        | 103 ++++++++++++
+ .../selftests/bpf/progs/test_sockmap_ktls.c        |  21 +++
+ .../selftests/net/forwarding/bridge_vlan_mcast.sh  |  30 +++-
+ tools/testing/selftests/net/mptcp/mptcp_connect.sh |   6 +-
+ tools/testing/selftests/net/mptcp/mptcp_join.sh    |  31 ++++
+ tools/testing/selftests/net/mptcp/mptcp_lib.sh     |  10 +-
+ .../selftests/net/ovpn/test-close-socket.sh        |   2 +-
+ tools/testing/selftests/net/ovpn/test-mark.sh      |   6 +-
+ tools/testing/selftests/net/ovpn/test.sh           |   4 +-
+ 126 files changed, 1673 insertions(+), 861 deletions(-)
+Merging bpf/master (49b18315be4ee bpf: Reject NULL data/sig in bpf_verify_pkcs7_signature)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git/ bpf/master
+Merge made by the 'ort' strategy.
+ include/linux/bpf.h                                |  8 ++-
+ include/linux/bpf_verifier.h                       |  2 +
+ include/linux/filter.h                             |  3 -
+ kernel/bpf/arraymap.c                              |  2 +-
+ kernel/bpf/cfg.c                                   | 13 ++++-
+ kernel/bpf/core.c                                  | 27 ++++++---
+ kernel/bpf/fixups.c                                | 13 +++--
+ kernel/bpf/helpers.c                               |  5 ++
+ kernel/bpf/syscall.c                               | 26 +++++++++
+ kernel/bpf/verifier.c                              | 27 +++++++--
+ .../selftests/bpf/prog_tests/kfunc_dynptr_param.c  |  2 +-
+ .../bpf/prog_tests/percpu_array_inner_map.c        | 57 +++++++++++++++++++
+ tools/testing/selftests/bpf/prog_tests/verifier.c  |  2 +
+ .../testing/selftests/bpf/progs/exceptions_fail.c  | 22 ++++++++
+ .../selftests/bpf/progs/verifier_call_large_imm.c  | 66 ++++++++++++++++++++++
+ 15 files changed, 250 insertions(+), 25 deletions(-)
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/percpu_array_inner_map.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_call_large_imm.c
+Merging ipsec/master (79d8be262377f xfrm: ah: use skb_to_full_sk in async output callbacks)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec.git ipsec/master
+Merge made by the 'ort' strategy.
+ include/net/xfrm.h     |  3 ++-
+ net/ipv4/ah4.c         |  2 +-
+ net/ipv6/ah6.c         |  2 +-
+ net/key/af_key.c       |  6 +++---
+ net/xfrm/xfrm_ipcomp.c | 12 ++++++++----
+ net/xfrm/xfrm_policy.c |  2 +-
+ net/xfrm/xfrm_state.c  | 23 ++++++++++++++++++-----
+ net/xfrm/xfrm_user.c   |  5 ++---
+ 8 files changed, 36 insertions(+), 19 deletions(-)
+Merging netfilter/main (fa997ddef508b dpll: zl3073x: fix memory leak on pin registration failure)
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf.git netfilter/main
+Already up to date.
+Merging ipvs/main (93d809adc1300 Merge branch 'vsock-virtio-fix-vsockmon-tap-skb-construction')
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs.git ipvs/main
+Already up to date.
+Merging wireless/for-next (dc14686f27df6 wifi: cfg80211: wext: validate chandef in monitor mode)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless.git wireless/for-next
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/ath/ath10k/wmi.c              |  17 ++-
+ drivers/net/wireless/ath/ath11k/dp_rx.c            |   9 +-
+ drivers/net/wireless/ath/ath11k/hal.c              |  14 ++-
+ drivers/net/wireless/ath/ath11k/hal_rx.c           |   5 +-
+ drivers/net/wireless/ath/ath11k/testmode.c         |   1 +
+ drivers/net/wireless/ath/ath11k/wmi.c              | 131 ++++++++++++++++++---
+ drivers/net/wireless/ath/ath12k/mac.c              |   8 +-
+ drivers/net/wireless/intel/iwlwifi/mld/constants.h |   4 +-
+ drivers/net/wireless/intel/iwlwifi/mld/d3.c        |   6 +-
+ drivers/net/wireless/intel/iwlwifi/mld/link.c      |  13 +-
+ drivers/net/wireless/intel/iwlwifi/mld/tx.c        |  15 ++-
+ drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c  |  27 +++--
+ drivers/net/wireless/intel/iwlwifi/mvm/utils.c     |  14 +--
+ .../intel/iwlwifi/pcie/gen1_2/trans-gen2.c         |   6 +-
+ drivers/net/wireless/microchip/wilc1000/wlan.c     |   2 +-
+ net/mac80211/cfg.c                                 |   5 +-
+ net/mac80211/mlme.c                                |   5 +-
+ net/mac80211/parse.c                               | 107 ++++++++++-------
+ net/mac80211/rx.c                                  |   6 +-
+ net/wireless/scan.c                                |   3 +
+ net/wireless/wext-compat.c                         |   2 +
+ 21 files changed, 276 insertions(+), 124 deletions(-)
+Merging ath/for-current (dc14686f27df6 wifi: cfg80211: wext: validate chandef in monitor mode)
+$ git merge -m Merge branch 'for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath/for-current
+Already up to date.
+Merging iwlwifi/fixes (734a4e051b976 wifi: iwlwifi: mld: disconnect only after 6 beacons without Rx)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi/fixes
+Already up to date.
+Merging wpan/master (8ce4f287524c7 net: libwx: fix firmware mailbox abnormal return)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan.git wpan/master
+Already up to date.
+Merging rdma-fixes/for-rc (5b74373390113 RDMA/rtrs: Fix use-after-free in path file creation cleanup)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma-fixes/for-rc
+Merge made by the 'ort' strategy.
+ drivers/infiniband/core/ib_core_uverbs.c           |  87 ++++++++++++
+ drivers/infiniband/core/nldev.c                    |   3 +
+ drivers/infiniband/core/uverbs.h                   |  34 +++++
+ drivers/infiniband/core/uverbs_ioctl.c             | 148 ++-------------------
+ drivers/infiniband/hw/bnxt_re/ib_verbs.c           |   2 +-
+ drivers/infiniband/hw/mana/main.c                  |   1 +
+ drivers/infiniband/sw/siw/siw_qp_rx.c              |  15 +++
+ drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c       |   2 +-
+ include/rdma/uverbs_ioctl.h                        |   1 +
+ tools/testing/selftests/rdma/rxe_ipv6.sh           |   6 +-
+ .../selftests/rdma/rxe_rping_between_netns.sh      |   7 +
+ .../selftests/rdma/rxe_socket_with_netns.sh        |   6 +
+ .../selftests/rdma/rxe_test_NETDEV_UNREGISTER.sh   |   6 +-
+ 13 files changed, 178 insertions(+), 140 deletions(-)
+Merging sound-current/for-linus (60a1969fae620 ALSA: seq: Serialize UMP output teardown with event_input)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound-current/for-linus
+Merge made by the 'ort' strategy.
+ sound/core/pcm_drm_eld.c          |   4 +-
+ sound/core/pcm_lib.c              |   3 ++
+ sound/core/seq/seq_ports.c        |   7 ++-
+ sound/core/seq/seq_ump_client.c   |  22 ++++++--
+ sound/core/timer.c                |  19 +++++--
+ sound/hda/codecs/realtek/alc269.c |  18 ++++---
+ sound/hda/common/auto_parser.c    |   4 +-
+ sound/hda/common/controller.c     |  28 +++-------
+ sound/hda/common/hda_controller.h |  14 ++++-
+ sound/hda/controllers/intel.c     | 106 +++++++++++++++++++++++++++-----------
+ sound/hda/controllers/intel.h     |  15 ++++--
+ sound/pci/asihpi/hpicmn.c         |   6 +++
+ sound/usb/misc/ua101.c            |   5 +-
+ sound/usb/mixer_scarlett2.c       |   7 ++-
+ 14 files changed, 175 insertions(+), 83 deletions(-)
+Merging sound-asoc-fixes/for-linus (298a43b54432f ASoC: soc-utils: Add missing va_end in snd_soc_ret())
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc-fixes/for-linus
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../sound/mediatek,mt8173-rt5650-rt5514.yaml       |  2 +
+ MAINTAINERS                                        |  9 +--
+ drivers/spi/spi-amd.c                              |  2 +-
+ sound/soc/amd/acp-config.c                         |  7 +++
+ sound/soc/amd/acp/acp-sdw-legacy-mach.c            |  2 +-
+ sound/soc/codecs/Kconfig                           |  3 -
+ sound/soc/codecs/cs35l56-sdw.c                     |  5 +-
+ sound/soc/codecs/cs35l56.c                         | 13 ++--
+ sound/soc/codecs/fs210x.c                          |  2 +-
+ sound/soc/fsl/fsl_sai.c                            | 43 ++++++++++---
+ sound/soc/intel/common/soc-acpi-intel-arl-match.c  | 71 +++++++++++++++++-----
+ sound/soc/intel/common/soc-acpi-intel-ptl-match.c  | 21 +++++++
+ sound/soc/qcom/qdsp6/q6apm-dai.c                   |  7 ++-
+ sound/soc/sdw_utils/soc_sdw_utils.c                | 14 +++--
+ sound/soc/soc-utils.c                              |  1 +
+ sound/soc/sof/amd/acp.c                            |  2 +-
+ sound/soc/ti/omap-dmic.c                           |  2 +-
+ 17 files changed, 156 insertions(+), 50 deletions(-)
+Merging regmap-fixes/for-linus (4b7774eeab8d6 regmap: sdw-mbq: Fix spelling mistake "undeferable" -> "undeferrable")
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap-fixes/for-linus
+Already up to date.
+Merging regulator-fixes/for-linus (f9b2d3b703d13 regulator: tps65219: fix irq_data.rdev not being assigned)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator-fixes/for-linus
+Merge made by the 'ort' strategy.
+ drivers/regulator/Kconfig              |   2 +-
+ drivers/regulator/tps65219-regulator.c | 135 +++++++++++++++++++++++----------
+ 2 files changed, 96 insertions(+), 41 deletions(-)
+Merging spi-fixes/for-linus (ea6ec3343e05f spi: ti-qspi: fix use-after-free after DMA setup failure)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi-fixes/for-linus
+Merge made by the 'ort' strategy.
+ drivers/spi/spi-ep93xx.c   | 2 ++
+ drivers/spi/spi-mtk-snfi.c | 2 +-
+ drivers/spi/spi-qup.c      | 3 +++
+ drivers/spi/spi-sprd.c     | 3 ++-
+ drivers/spi/spi-ti-qspi.c  | 1 +
+ 5 files changed, 9 insertions(+), 2 deletions(-)
+Merging pci-current/for-linus (ed831e7ea1a86 PCI: brcmstb: Assign pcie->gen from of_pci_get_max_link_speed())
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci-current/for-linus
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                           | 4 +---
+ drivers/pci/controller/pcie-brcmstb.c | 4 +++-
+ 2 files changed, 4 insertions(+), 4 deletions(-)
+Merging driver-core.current/driver-core-linus (237557b8a81ab sysfs: don't remove existing directory on update failure)
+$ git merge -m Merge branch 'driver-core-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core.current/driver-core-linus
+Merge made by the 'ort' strategy.
+ drivers/base/platform.c | 16 +++++++++++++++-
+ fs/sysfs/group.c        |  2 +-
+ 2 files changed, 16 insertions(+), 2 deletions(-)
+Merging tty.current/tty-linus (452d6fa37ae9b serial: qcom_geni: fix kfifo underflow when flush precedes DMA completion IRQ)
+$ git merge -m Merge branch 'tty-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty.current/tty-linus
+Merge made by the 'ort' strategy.
+ drivers/tty/serial/fsl_lpuart.c       | 15 ++++++++++++---
+ drivers/tty/serial/pch_uart.c         | 19 +++++++++++++------
+ drivers/tty/serial/qcom_geni_serial.c | 16 ++++++++++++++--
+ drivers/tty/serial/sh-sci.c           |  2 +-
+ drivers/tty/serial/zs.c               |  4 ++--
+ include/linux/tty_port.h              |  2 +-
+ 6 files changed, 43 insertions(+), 15 deletions(-)
+Merging usb.current/usb-linus (288a81a850705 usb: typec: ucsi: validate connector number in ucsi_connector_change())
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb.current/usb-linus
+Merge made by the 'ort' strategy.
+ drivers/thunderbolt/property.c            | 32 +++++++++++++++------
+ drivers/usb/core/config.c                 | 46 +++++++++++++------------------
+ drivers/usb/core/hcd.c                    |  4 +--
+ drivers/usb/gadget/composite.c            |  5 +++-
+ drivers/usb/typec/altmodes/displayport.c  |  2 ++
+ drivers/usb/typec/tcpm/tcpci_maxim_core.c |  9 ++++++
+ drivers/usb/typec/tcpm/tcpm.c             | 20 ++++++++------
+ drivers/usb/typec/tcpm/wcove.c            | 13 +++++----
+ drivers/usb/typec/tipd/core.c             |  1 +
+ drivers/usb/typec/ucsi/displayport.c      |  4 +++
+ drivers/usb/typec/ucsi/ucsi.c             | 11 +++++++-
+ drivers/usb/typec/ucsi/ucsi_ccg.c         |  5 ++++
+ 12 files changed, 97 insertions(+), 55 deletions(-)
+Merging usb-serial-fixes/usb-linus (689f2facc689c USB: serial: option: add missing RSVD(5) flag for Rolling RW135R-GL)
+$ git merge -m Merge branch 'usb-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial-fixes/usb-linus
+Merge made by the 'ort' strategy.
+ drivers/usb/serial/belkin_sa.c       |  3 +++
+ drivers/usb/serial/digi_acceleport.c | 23 +++++++++++++++++++++--
+ drivers/usb/serial/keyspan.c         |  4 ++++
+ drivers/usb/serial/mct_u232.c        | 26 ++++++++++++++++----------
+ drivers/usb/serial/option.c          |  9 ++++++++-
+ 5 files changed, 52 insertions(+), 13 deletions(-)
+Merging phy/fixes (ea17fc4d7dc2b phy: qcom: qmp-usbc: Fix out-of-bounds array access in dp swing config)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy/fixes
+Merge made by the 'ort' strategy.
+ drivers/phy/apple/atc.c                    |  27 +++-
+ drivers/phy/eswin/phy-eic7700-sata.c       |   4 +-
+ drivers/phy/marvell/phy-mvebu-a3700-utmi.c |   5 +-
+ drivers/phy/qualcomm/phy-qcom-edp.c        | 224 +++++++++++++++++++++++------
+ drivers/phy/qualcomm/phy-qcom-qmp-ufs.c    |   1 +
+ drivers/phy/qualcomm/phy-qcom-qmp-usbc.c   |   2 +-
+ drivers/phy/samsung/phy-exynos5-usbdrd.c   |   7 +-
+ drivers/phy/spacemit/phy-k1-usb2.c         |   1 -
+ drivers/phy/tegra/xusb-tegra186.c          |  33 ++++-
+ drivers/phy/tegra/xusb.h                   |   1 +
+ 10 files changed, 240 insertions(+), 65 deletions(-)
+Merging staging.current/staging-linus (514ab98364595 staging: rtl8723bs: fix buffer over-read in rtw_update_protection)
+$ git merge -m Merge branch 'staging-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging.current/staging-linus
+Merge made by the 'ort' strategy.
+ drivers/staging/rtl8723bs/core/rtw_mlme.c | 10 ++++++++--
+ 1 file changed, 8 insertions(+), 2 deletions(-)
+Merging iio-fixes/fixes-togreg (bbc109ae655c8 iio: temperature: tmp006: use devm_iio_trigger_register)
+$ git merge -m Merge branch 'fixes-togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio-fixes/fixes-togreg
+Merge made by the 'ort' strategy.
+ drivers/counter/counter-core.c                 |  3 +-
+ drivers/iio/adc/Kconfig                        |  2 +
+ drivers/iio/adc/ad4695.c                       | 23 ++++-------
+ drivers/iio/adc/meson_saradc.c                 |  4 +-
+ drivers/iio/adc/mt6359-auxadc.c                |  1 +
+ drivers/iio/adc/npcm_adc.c                     | 25 ++++--------
+ drivers/iio/adc/nxp-sar-adc.c                  | 24 +++++++++--
+ drivers/iio/adc/qcom-spmi-adc5-gen3.c          |  2 +-
+ drivers/iio/adc/ti-ads1298.c                   |  7 +++-
+ drivers/iio/adc/viperboard_adc.c               |  4 +-
+ drivers/iio/adc/xilinx-xadc-core.c             | 11 ++++-
+ drivers/iio/buffer/industrialio-hw-consumer.c  |  6 ++-
+ drivers/iio/chemical/mhz19b.c                  | 17 ++++++++
+ drivers/iio/chemical/scd30_core.c              |  2 +-
+ drivers/iio/common/ssp_sensors/ssp_dev.c       |  1 +
+ drivers/iio/dac/ad3530r.c                      | 54 ++++++++++++++++++-------
+ drivers/iio/dac/ad5686.c                       | 56 +++++++++++++++++++-------
+ drivers/iio/dac/ad5686.h                       |  1 +
+ drivers/iio/dac/max5821.c                      |  9 ++++-
+ drivers/iio/gyro/adis16260.c                   |  3 ++
+ drivers/iio/gyro/bmg160_core.c                 | 16 +++++++-
+ drivers/iio/gyro/itg3200_buffer.c              |  2 +-
+ drivers/iio/imu/adis16550.c                    |  2 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c |  2 +-
+ drivers/iio/industrialio-buffer.c              |  1 +
+ drivers/iio/inkern.c                           |  6 ++-
+ drivers/iio/light/acpi-als.c                   |  6 ++-
+ drivers/iio/light/cm3323.c                     |  5 +--
+ drivers/iio/light/veml6030.c                   |  8 ++--
+ drivers/iio/light/veml6070.c                   | 14 +------
+ drivers/iio/light/veml6075.c                   |  8 +++-
+ drivers/iio/magnetometer/st_magn_core.c        | 13 +++++-
+ drivers/iio/pressure/bmp280-core.c             |  4 +-
+ drivers/iio/proximity/vl53l0x-i2c.c            |  7 ++--
+ drivers/iio/resolver/ad2s1210.c                |  2 +-
+ drivers/iio/temperature/tmp006.c               |  2 +-
+ drivers/iio/temperature/tsys01.c               |  2 +-
+ 37 files changed, 242 insertions(+), 113 deletions(-)
+Merging watchdog-fixes/watchdog (367937951ffca watchdog: ni903x_wdt: Check ACPI_COMPANION() against NULL)
+$ git merge -m Merge branch 'watchdog' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git watchdog-fixes/watchdog
+Merge made by the 'ort' strategy.
+ drivers/watchdog/ni903x_wdt.c | 7 ++++++-
+ drivers/watchdog/s32g_wdt.c   | 3 +--
+ 2 files changed, 7 insertions(+), 3 deletions(-)
+Merging counter-current/counter-current (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'counter-current' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-current/counter-current
+Already up to date.
+Merging char-misc.current/char-misc-linus (329db447f8f02 Merge tag 'iio-fixes-for-7.1a' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/jic23/iio into char-misc-linus)
+$ git merge -m Merge branch 'char-misc-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc.current/char-misc-linus
+Already up to date.
+Merging soundwire-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire-fixes/fixes
+Already up to date.
+Merging thunderbolt-fixes/fixes (928abe19fbf01 thunderbolt: property: Cap recursion depth in __tb_property_parse_dir())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt-fixes/fixes
+Already up to date.
+Merging input-current/for-linus (16ca52bc209fa Input: synaptics - add LEN2058 to SMBus passlist for ThinkPad E490)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input-current/for-linus
+Merge made by the 'ort' strategy.
+ drivers/input/gameport/fm801-gp.c          |  4 ++--
+ drivers/input/joystick/xpad.c              | 14 ++++++++++----
+ drivers/input/misc/atlas_btns.c            |  6 +++++-
+ drivers/input/mouse/elan_i2c_core.c        |  8 ++++++++
+ drivers/input/mouse/elan_i2c_i2c.c         |  2 +-
+ drivers/input/mouse/synaptics.c            |  1 +
+ drivers/input/touchscreen/atmel_mxt_ts.c   | 13 ++++++++++---
+ drivers/input/touchscreen/usbtouchscreen.c |  5 +++++
+ 8 files changed, 42 insertions(+), 11 deletions(-)
+Merging crypto-current/master (d1fa83ecac310 rhashtable: Add bucket_table_free_atomic() helper)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6.git crypto-current/master
+Already up to date.
+Merging libcrypto-fixes/libcrypto-fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'libcrypto-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto-fixes/libcrypto-fixes
+Already up to date.
+Merging vfio-fixes/for-linus (702809dabdecc vfio/pci: Check BAR resources before exporting a DMABUF)
+$ git merge -m Merge branch 'for-linus' of https://github.com/awilliam/linux-vfio.git vfio-fixes/for-linus
+Already up to date.
+Merging kselftest-fixes/fixes (465b05bae5ac5 selftests: harness: Restore order of test functions)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest-fixes/fixes
+Already up to date.
+Merging dmaengine-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine-fixes/fixes
+Already up to date.
+Merging backlight-fixes/for-backlight-fixes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-backlight-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight-fixes/for-backlight-fixes
+Already up to date.
+Merging mtd-fixes/mtd/fixes (135ac3b84bced mtd: spinand: winbond: Fix ODTR write VCR on W35NxxJW)
+$ git merge -m Merge branch 'mtd/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd-fixes/mtd/fixes
+Already up to date.
+Merging mfd-fixes/for-mfd-fixes (aa6e63373d91f mfd: ls2kbmc: Use PCI API instead of direct accesses)
+$ git merge -m Merge branch 'for-mfd-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd-fixes/for-mfd-fixes
+Merge made by the 'ort' strategy.
+Merging v4l-dvb-fixes/fixes (646ebdd310580 media: rc: ttusbir: fix inverted error logic)
+$ git merge -m Merge branch 'fixes' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb-fixes/fixes
+Already up to date.
+Merging reset-fixes/reset/fixes (9797524ef2b69 reset: amlogic: t7: Fix null reset ops)
+$ git merge -m Merge branch 'reset/fixes' of https://git.pengutronix.de/git/pza/linux reset-fixes/reset/fixes
+Already up to date.
+Merging mips-fixes/mips-fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'mips-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips-fixes/mips-fixes
+Already up to date.
+Merging at91-fixes/at91-fixes (765aaba18413a ARM: dts: microchip: sam9x7: fix GMAC clock configuration)
+$ git merge -m Merge branch 'at91-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91-fixes/at91-fixes
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/microchip/sam9x7.dtsi | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+Merging omap-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap-fixes/fixes
+Already up to date.
+Merging kvm-fixes/master (2d5d3fc593c9b KVM: VMX: introduce module parameter to disable CET)
+$ git merge -m Merge branch 'master' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm-fixes/master
+Already up to date.
+Merging kvms390-fixes/master (ac1283cf1d3f8 KVM: s390: Properly reset zero bit in PGSTE)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390-fixes/master
+Auto-merging arch/s390/kvm/dat.h
+Auto-merging arch/s390/kvm/gaccess.c
+Auto-merging arch/s390/kvm/gmap.c
+Auto-merging arch/s390/kvm/gmap.h
+Merge made by the 'ort' strategy.
+ arch/s390/kvm/dat.c     |  1 +
+ arch/s390/kvm/dat.h     |  3 ++-
+ arch/s390/kvm/gaccess.c |  1 +
+ arch/s390/kvm/gmap.c    | 18 +++++++++------
+ arch/s390/kvm/gmap.h    | 61 ++++++++++++++++++++++++++++++++++++++++++++++---
+ 5 files changed, 73 insertions(+), 11 deletions(-)
+Merging kvm-arm-fixes/fixes (1702da76e017a KVM: arm64: Fix nVHE/pKVM hyp tracing error on invalid desc)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm-fixes/fixes
+Merge made by the 'ort' strategy.
+ arch/arm64/kvm/arm.c            | 4 +++-
+ arch/arm64/kvm/hyp/nvhe/trace.c | 9 +++++++--
+ arch/arm64/kvm/vgic/vgic-its.c  | 4 ++++
+ 3 files changed, 14 insertions(+), 3 deletions(-)
+Merging hwmon-fixes/hwmon (5200f5f493f79 Linux 7.1-rc4)
+$ git merge -m Merge branch 'hwmon' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-fixes/hwmon
+Already up to date.
+Merging nvdimm-fixes/libnvdimm-fixes (a8aec14230322 nvdimm/bus: Fix potential use after free in asynchronous initialization)
+$ git merge -m Merge branch 'libnvdimm-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm-fixes/libnvdimm-fixes
+Already up to date.
+Merging cxl-fixes/fixes (5d6919055dec1 Linux 7.1-rc3)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl-fixes/fixes
+Already up to date.
+Merging dma-mapping-fixes/dma-mapping-fixes (af0c3f0586623 dma-mapping: move dma_map_resource() sanity check into debug code)
+$ git merge -m Merge branch 'dma-mapping-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping-fixes/dma-mapping-fixes
+Merge made by the 'ort' strategy.
+ kernel/dma/debug.c   | 9 ++++++++-
+ kernel/dma/direct.c  | 4 ++--
+ kernel/dma/mapping.c | 4 ----
+ 3 files changed, 10 insertions(+), 7 deletions(-)
+Merging drivers-x86-fixes/fixes (2a9ca96225ccc platform/x86: bitland-mifs-wmi: add CONFIG_LEDS_CLASS dependency)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86-fixes/fixes
+Merge made by the 'ort' strategy.
+ .../admin-guide/laptops/uniwill-laptop.rst         |  10 ++
+ .../platform/surface/surface_aggregator_registry.c |   2 -
+ drivers/platform/surface/surfacepro3_button.c      |  10 +-
+ drivers/platform/x86/Kconfig                       |   1 +
+ drivers/platform/x86/acer-wireless.c               |   8 +-
+ drivers/platform/x86/adv_swbutton.c                |   6 +-
+ drivers/platform/x86/asus-armoury.c                |  16 ++-
+ drivers/platform/x86/asus-armoury.h                | 113 +++++++++++++++++++++
+ drivers/platform/x86/asus-laptop.c                 |   6 +-
+ drivers/platform/x86/dell/dell-rbtn.c              |   6 +-
+ drivers/platform/x86/eeepc-laptop.c                |   6 +-
+ drivers/platform/x86/fujitsu-laptop.c              |  12 ++-
+ drivers/platform/x86/fujitsu-tablet.c              |   6 +-
+ drivers/platform/x86/hp/hp-wmi.c                   |   4 +
+ drivers/platform/x86/hp/hp_accel.c                 |   3 +
+ drivers/platform/x86/intel/hid.c                   |   6 +-
+ drivers/platform/x86/intel/int1092/intel_sar.c     |   7 +-
+ drivers/platform/x86/intel/rst.c                   |   6 +-
+ drivers/platform/x86/intel/smartconnect.c          |   6 +-
+ drivers/platform/x86/intel/vbtn.c                  |   6 +-
+ drivers/platform/x86/intel/vsec.c                  |  54 +++++-----
+ drivers/platform/x86/lg-laptop.c                   |   9 +-
+ drivers/platform/x86/panasonic-laptop.c            |   6 +-
+ drivers/platform/x86/sony-laptop.c                 |  12 ++-
+ drivers/platform/x86/system76_acpi.c               |   6 +-
+ drivers/platform/x86/toshiba_acpi.c                |   6 +-
+ drivers/platform/x86/toshiba_bluetooth.c           |   6 +-
+ drivers/platform/x86/toshiba_haps.c                |   6 +-
+ drivers/platform/x86/uniwill/uniwill-acpi.c        |  47 ++++++++-
+ drivers/platform/x86/wireless-hotkey.c             |   9 +-
+ 30 files changed, 338 insertions(+), 63 deletions(-)
+Merging samsung-krzk-fixes/fixes (dce77afa1e18f Merge branch 'fixes-clk' into fixes)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/clk/samsung/clk-gs101.c                    |   2 +-
+ drivers/firmware/samsung/exynos-acpm-dvfs.c        |   3 +
+ drivers/firmware/samsung/exynos-acpm.c             | 145 ++++++++++++++-------
+ .../linux/firmware/samsung/exynos-acpm-protocol.h  |   3 +-
+ 4 files changed, 107 insertions(+), 46 deletions(-)
+Merging pinctrl-samsung-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung-fixes/fixes
+Already up to date.
+Merging devicetree-fixes/dt/linus (4b0a3ff9f4178 of: cpu: add check in __of_find_n_match_cpu_property())
+$ git merge -m Merge branch 'dt/linus' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree-fixes/dt/linus
+Merge made by the 'ort' strategy.
+ drivers/of/cpu.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging dt-krzk-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk-fixes/fixes
+Already up to date.
+Merging scsi-fixes/fixes (b71cb088b2e34 scsi: target: tcm_loop: Fix NULL ptr dereference)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/scsi/isci/host.c           |  3 +++
+ drivers/scsi/sd.c                  |  3 +--
+ drivers/target/loopback/tcm_loop.c | 12 ++++++++++--
+ 3 files changed, 14 insertions(+), 4 deletions(-)
+Merging drm-fixes/drm-fixes (aee43aaf26c7f Merge tag 'amd-drm-fixes-7.1-2026-05-20' of https://gitlab.freedesktop.org/agd5f/linux into drm-fixes)
+$ git merge -m Merge branch 'drm-fixes' of https://gitlab.freedesktop.org/drm/kernel.git drm-fixes/drm-fixes
+Merge made by the 'ort' strategy.
+ .../bindings/display/msm/dp-controller.yaml        |  28 ++-
+ .../bindings/display/msm/qcom,eliza-mdss.yaml      |  20 +-
+ .../bindings/display/msm/qcom,sm8650-dpu.yaml      |   2 +-
+ .../bindings/display/msm/qcom,sm8650-mdss.yaml     |   2 +-
+ .../bindings/display/msm/qcom,sm8750-mdss.yaml     |  16 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |   1 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c         |  25 ++
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h         |   1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |   9 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c      |   2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c            |   8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c        |   9 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.h         |   3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c          |  13 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c            |   2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          | 265 ++++++++++-----------
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h          |  16 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c    |  57 +++--
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |   2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c            |   7 +-
+ drivers/gpu/drm/amd/amdgpu/vce_v1_0.c              |  64 +++--
+ drivers/gpu/drm/amd/amdgpu/vce_v2_0.c              |   9 +-
+ drivers/gpu/drm/amd/amdgpu/vce_v3_0.c              |   2 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_chardev.c           |  22 ++
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |   9 +
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c    |  11 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |   1 +
+ drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c |   9 +
+ .../drm/amd/display/dc/bios/bios_parser_helper.c   |   9 +-
+ drivers/gpu/drm/amd/display/dc/core/dc.c           |   6 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c     |   2 +
+ drivers/gpu/drm/i915/display/intel_display_types.h |   1 +
+ drivers/gpu/drm/i915/display/intel_dp.c            |   2 +-
+ drivers/gpu/drm/i915/display/intel_dpcd.h          |  15 ++
+ drivers/gpu/drm/i915/display/intel_plane.c         |   2 +-
+ drivers/gpu/drm/i915/display/intel_psr.c           |  44 +++-
+ drivers/gpu/drm/msm/adreno/a6xx_gpu.c              |   7 +-
+ drivers/gpu/drm/msm/adreno/a6xx_hfi.c              |   2 +
+ drivers/gpu/drm/msm/adreno/adreno_device.c         |   2 +-
+ drivers/gpu/drm/msm/adreno/adreno_gpu.c            |   8 +-
+ .../drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h |   2 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c        |  12 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c      |   3 +-
+ drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c  |  28 ++-
+ drivers/gpu/drm/msm/dsi/dsi_host.c                 |   1 +
+ drivers/gpu/drm/msm/msm_gem_shrinker.c             |  40 ++--
+ drivers/gpu/drm/msm/msm_gpu.c                      |   4 +-
+ drivers/gpu/drm/msm/msm_iommu.c                    |   5 +-
+ drivers/gpu/drm/radeon/evergreen_cs.c              |   6 +-
+ 49 files changed, 523 insertions(+), 293 deletions(-)
+ create mode 100644 drivers/gpu/drm/i915/display/intel_dpcd.h
+Merging drm-intel-fixes/for-linux-next-fixes (4703049f768fc drm/i915/psr: Apply Intel DPCD workaround when SDP on prior line used)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel-fixes/for-linux-next-fixes
+Already up to date.
+Merging mmc-fixes/fixes (5a52c5701a67d mmc: core: Fix host controller programming for fixed driver type)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/mmc/core/mmc.c | 4 +++-
+ 1 file changed, 3 insertions(+), 1 deletion(-)
+Merging rtc-fixes/rtc-fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'rtc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc-fixes/rtc-fixes
+Already up to date.
+Merging gnss-fixes/gnss-linus (5d6919055dec1 Linux 7.1-rc3)
+$ git merge -m Merge branch 'gnss-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss-fixes/gnss-linus
+Already up to date.
+Merging hyperv-fixes/hyperv-fixes (8b35874f56ded drm/hyperv: During panic do VMBus unload after frame buffer is flushed)
+$ git merge -m Merge branch 'hyperv-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv-fixes/hyperv-fixes
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/hyperv/hyperv_drm_drv.c     |   5 +
+ drivers/gpu/drm/hyperv/hyperv_drm_modeset.c |  15 +--
+ drivers/hv/channel_mgmt.c                   |   1 +
+ drivers/hv/hv.c                             |   3 +
+ drivers/hv/hv_kvp.c                         |  27 +++--
+ drivers/hv/hyperv_vmbus.h                   |   1 -
+ drivers/hv/mshv_debugfs.c                   |   7 +-
+ drivers/hv/mshv_synic.c                     | 156 ++++++++++++++++++----------
+ drivers/hv/vmbus_drv.c                      |  25 +++--
+ include/hyperv/hvhdk.h                      |   1 +
+ include/linux/hyperv.h                      |   3 +
+ 11 files changed, 160 insertions(+), 84 deletions(-)
+Merging risc-v-fixes/fixes (b69bcb13ed702 riscv: misaligned: Make enabling delegation depend on NONPORTABLE)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v-fixes/fixes
+Already up to date.
+Merging riscv-dt-fixes/riscv-dt-fixes (0df8aa2b9aec5 riscv: dts: microchip: fix icicle i2c pinctrl configuration)
+$ git merge -m Merge branch 'riscv-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt-fixes/riscv-dt-fixes
+Already up to date.
+Merging riscv-soc-fixes/riscv-soc-fixes (75ef233975589 soc: microchip: mpfs-sys-controller: fix resource leak on probe error)
+$ git merge -m Merge branch 'riscv-soc-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc-fixes/riscv-soc-fixes
+Merge made by the 'ort' strategy.
+ drivers/soc/microchip/mpfs-sys-controller.c | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+Merging fpga-fixes/fixes (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga-fixes/fixes
+Already up to date.
+Merging spdx/spdx-linus (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'spdx-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/spdx.git spdx/spdx-linus
+Already up to date.
+Merging gpio-brgl-fixes/gpio/for-current (be460cedb67ab gpio: pca953x: propagate regulator_enable() error from resume)
+$ git merge -m Merge branch 'gpio/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl-fixes/gpio/for-current
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpio-pca953x.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging gpio-intel-fixes/fixes (3bb62e3f99a55 gpiolib: acpi: Only trigger ActiveBoth interrupts on boot)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/gpio/gpiolib-acpi-core.c | 19 +++++++++++++++----
+ 1 file changed, 15 insertions(+), 4 deletions(-)
+Merging pinctrl-intel-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel-fixes/fixes
+Already up to date.
+Merging auxdisplay-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay-fixes/fixes
+Already up to date.
+Merging kunit-fixes/kunit-fixes (8f80b5b227ef9 kunit: config: KUNIT_DEBUGFS should depend on DEBUG_FS)
+$ git merge -m Merge branch 'kunit-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-fixes/kunit-fixes
+Already up to date.
+Merging memblock-fixes/fixes (34d885baf7cc7 tools/testing/memblock: fix stale NUMA reservation tests)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock-fixes/fixes
+Merge made by the 'ort' strategy.
+ mm/numa_emulation.c                                | 17 +++++++++++++----
+ tools/testing/memblock/README                      |  5 +----
+ tools/testing/memblock/TODO                        |  4 ++--
+ tools/testing/memblock/tests/alloc_exact_nid_api.c |  6 +++---
+ tools/testing/memblock/tests/alloc_nid_api.c       |  6 +++---
+ 5 files changed, 22 insertions(+), 16 deletions(-)
+Merging renesas-fixes/fixes (8a78d1c454b3c Merge branch 'renesas-pinctrl-fixes-for-v7.1' into renesas-fixes)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas-fixes/fixes
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/renesas/pinctrl-rzg2l.c | 23 +++++++++++++++--------
+ 1 file changed, 15 insertions(+), 8 deletions(-)
+Merging perf-current/perf-tools (5a433107fab62 perf trace: Update beautifier script for clone flags)
+$ git merge -m Merge branch 'perf-tools' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools.git perf-current/perf-tools
+Already up to date.
+Merging efi-fixes/urgent (d8809f6931065 efi: sysfb_efi: Extend quirk to cover IdeaPad Duet 3 10IGL5-LTE)
+$ git merge -m Merge branch 'urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi-fixes/urgent
+Merge made by the 'ort' strategy.
+ drivers/firmware/efi/efi.c       | 28 ++++++++++++++++------------
+ drivers/firmware/efi/sysfb_efi.c |  9 ++++++---
+ include/linux/efi.h              |  2 +-
+ 3 files changed, 23 insertions(+), 16 deletions(-)
+Merging battery-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery-fixes/fixes
+Already up to date.
+Merging iommufd-fixes/for-rc (be93d186ae88a iommufd: Use sizeof(*hdr) instead of sizeof(hdr) in veventq read)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd-fixes/for-rc
+Merge made by the 'ort' strategy.
+ drivers/iommu/iommufd/eventq.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging rust-fixes/rust-fixes (84a0f7caafc67 ARM: Do not select HAVE_RUST when KASAN is enabled)
+$ git merge -m Merge branch 'rust-fixes' of https://github.com/Rust-for-Linux/linux.git rust-fixes/rust-fixes
+Merge made by the 'ort' strategy.
+ arch/arm/Kconfig | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging w1-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1-fixes/fixes
+Already up to date.
+Merging pmdomain-fixes/fixes (5d6919055dec1 Linux 7.1-rc3)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain-fixes/fixes
+Already up to date.
+Merging i2c-host-fixes/i2c/i2c-host-fixes (30792d1284290 i2c: tegra: make tegra_i2c_mutex_unlock() return void)
+$ git merge -m Merge branch 'i2c/i2c-host-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-host-fixes/i2c/i2c-host-fixes
+Merge made by the 'ort' strategy.
+ drivers/i2c/busses/i2c-tegra.c | 19 +++++++++----------
+ 1 file changed, 9 insertions(+), 10 deletions(-)
+Merging i2c-rust-fixes/rust-i2c-fixes (7ca6d1cfec80e Merge tag 'powerpc-7.0-4' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux)
+$ git merge -m Merge branch 'rust-i2c-fixes' of https://github.com/ikrtn/rust-for-linux i2c-rust-fixes/rust-i2c-fixes
+Already up to date.
+Merging sparc-fixes/for-linus (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-linus' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc-fixes/for-linus
+Already up to date.
+Merging clk-fixes/clk-fixes (de019f203b0d4 clk: rk808: fix OF node reference imbalance)
+$ git merge -m Merge branch 'clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk-fixes/clk-fixes
+Already up to date.
+Merging thead-clk-fixes/thead-clk-fixes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'thead-clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk-fixes/thead-clk-fixes
+Already up to date.
+Merging tenstorrent-clk-fixes/tenstorrent-clk-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'tenstorrent-clk-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-clk-fixes/tenstorrent-clk-fixes
+Already up to date.
+Merging pwrseq-fixes/pwrseq/for-current (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'pwrseq/for-current' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq-fixes/pwrseq/for-current
+Already up to date.
+Merging thead-dt-fixes/thead-dt-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'thead-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt-fixes/thead-dt-fixes
+Already up to date.
+Merging ftrace-fixes/ftrace/fixes (1650a1b6cb1ae fgraph: Check ftrace_pids_enabled on registration for early filtering)
+$ git merge -m Merge branch 'ftrace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace-fixes/ftrace/fixes
+Already up to date.
+Merging ring-buffer-fixes/ring-buffer/fixes (92d5a606721f7 ring-buffer: Do not double count the reader_page)
+$ git merge -m Merge branch 'ring-buffer/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ring-buffer-fixes/ring-buffer/fixes
+Already up to date.
+Merging trace-fixes/trace/fixes (55a0005518195 tracing: Fix desc in error path for the trace remote test module)
+$ git merge -m Merge branch 'trace/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git trace-fixes/trace/fixes
+Already up to date.
+Merging tracefs-fixes/tracefs/fixes (07004a8c4b572 eventfs: Hold eventfs_mutex and SRCU when remount walks events)
+$ git merge -m Merge branch 'tracefs/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git tracefs-fixes/tracefs/fixes
+Already up to date.
+Merging spacemit-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://github.com/spacemit-com/linux spacemit-fixes/fixes
+Already up to date.
+Merging tip-fixes/tip/urgent (de5cda262357b Merge branch into tip/master: 'x86/urgent')
+$ git merge -m Merge branch 'tip/urgent' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip-fixes/tip/urgent
+Merge made by the 'ort' strategy.
+ arch/x86/entry/Makefile                 |  2 +-
+ arch/x86/entry/common.c                 | 61 +++++++++++++++++++++++++++++++++
+ arch/x86/entry/entry.S                  | 46 +++++++++++++++++++++++++
+ arch/x86/entry/entry_64_fred.S          |  1 -
+ arch/x86/entry/vdso/vma.c               |  2 +-
+ arch/x86/include/asm/desc.h             |  4 +++
+ arch/x86/include/asm/desc_defs.h        |  2 +-
+ arch/x86/include/asm/entry-common.h     |  2 ++
+ arch/x86/include/asm/fred.h             |  1 -
+ arch/x86/kernel/cpu/cpuid-deps.c        |  1 +
+ arch/x86/kernel/idt.c                   | 15 ++++++++
+ arch/x86/kernel/nmi.c                   |  1 -
+ arch/x86/kvm/vmx/vmenter.S              | 46 -------------------------
+ arch/x86/kvm/vmx/vmx.c                  | 19 ++--------
+ drivers/virt/coco/sev-guest/sev-guest.c | 10 ++++--
+ lib/debugobjects.c                      | 18 +++++++---
+ 16 files changed, 155 insertions(+), 76 deletions(-)
+ create mode 100644 arch/x86/entry/common.c
+Merging slab-fixes/slab/for-next-fixes (67ea9d353d0ba mm/slub: hold cpus_read_lock around flush_rcu_sheaves_on_cache())
+$ git merge -m Merge branch 'slab/for-next-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git slab-fixes/slab/for-next-fixes
+Auto-merging mm/slub.c
+Merge made by the 'ort' strategy.
+ mm/slab_common.c | 2 ++
+ mm/slub.c        | 1 +
+ 2 files changed, 3 insertions(+)
+Merging kexec-fixes/kexec-fixes (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'kexec-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git kexec-fixes/kexec-fixes
+Already up to date.
+Merging liveupdate-fixes/fixes (ec4084bc44502 kho: fix order calculation for kho_unpreserve_pages())
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git liveupdate-fixes/fixes
+Auto-merging kernel/liveupdate/kexec_handover.c
+Merge made by the 'ort' strategy.
+ include/linux/kho/abi/kexec_handover.h |  4 +--
+ kernel/liveupdate/kexec_handover.c     | 56 +++++++++++++++++++---------------
+ kernel/liveupdate/luo_core.c           |  4 +++
+ 3 files changed, 38 insertions(+), 26 deletions(-)
+Merging drm-msm-fixes/msm-fixes (db339b6bc9f23 dt-bindings: display/msm: Fix typo in clock-names property)
+$ git merge -m Merge branch 'msm-fixes' of https://gitlab.freedesktop.org/drm/msm.git drm-msm-fixes/msm-fixes
+Already up to date.
+Merging uml-fixes/fixes (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml-fixes/fixes
+Already up to date.
+Merging fwctl-fixes/for-rc (e7537735028c3 fwctl: pds: Validate RPC input size before parsing)
+$ git merge -m Merge branch 'for-rc' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl-fixes/for-rc
+Merge made by the 'ort' strategy.
+ drivers/fwctl/pds/main.c | 3 +++
+ 1 file changed, 3 insertions(+)
+Merging devsec-tsm-fixes/fixes (c3fd16c3b98ed virt: tdx-guest: Fix handling of host controlled 'quote' buffer length)
+$ git merge -m Merge branch 'fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm-fixes/fixes
+Already up to date.
+Merging drm-rust-fixes/for-linux-next-fixes (0a69ac25bd596 rust: drm: fix unsound initialization in drm::Device::new)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust-fixes/for-linux-next-fixes
+Already up to date.
+Merging tenstorrent-dt-fixes/tenstorrent-dt-fixes (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'tenstorrent-dt-fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-dt-fixes/tenstorrent-dt-fixes
+Already up to date.
+Merging nfc-fixes/for-linus (f23bf992d65a4 nfc: nxp-nci: i2c: use rising-edge IRQ on ACPI systems)
+$ git merge -m Merge branch 'for-linus' of https://codeberg.org/linux-nfc/linux.git nfc-fixes/for-linus
+Merge made by the 'ort' strategy.
+ drivers/nfc/nxp-nci/i2c.c | 21 ++++++++++++++++++++-
+ net/nfc/hci/core.c        | 10 ++++++++++
+ net/nfc/llcp_core.c       | 11 +++++++++++
+ net/nfc/llcp_sock.c       |  2 ++
+ net/nfc/nci/hci.c         | 10 ++++++++++
+ 5 files changed, 53 insertions(+), 1 deletion(-)
+Merging drm-misc-fixes/for-linux-next-fixes (9af1b6e175c82 drm/virtio: use uninterruptible resv lock for plane updates)
+$ git merge -m Merge branch 'for-linux-next-fixes' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc-fixes/for-linux-next-fixes
+Auto-merging drivers/gpu/drm/drm_gem.c
+Auto-merging drivers/gpu/drm/msm/msm_gem_shrinker.c
+Merge made by the 'ort' strategy.
+ drivers/accel/amdxdna/amdxdna_gem.c                |  9 +++-
+ drivers/accel/amdxdna/amdxdna_gem.h                |  2 +
+ drivers/accel/amdxdna/amdxdna_ubuf.c               | 50 ----------------------
+ drivers/gpu/drm/bridge/chipone-icn6211.c           |  4 +-
+ drivers/gpu/drm/bridge/ite-it66121.c               |  5 +++
+ .../drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c   | 16 ++++---
+ drivers/gpu/drm/drm_drv.c                          |  2 +
+ drivers/gpu/drm/drm_gem.c                          | 36 +++++++---------
+ drivers/gpu/drm/msm/msm_drv.c                      | 11 +++--
+ drivers/gpu/drm/msm/msm_drv.h                      |  7 ---
+ drivers/gpu/drm/msm/msm_gem.c                      | 33 +++++++-------
+ drivers/gpu/drm/msm/msm_gem_shrinker.c             |  4 +-
+ drivers/gpu/drm/msm/msm_gem_submit.c               |  6 +--
+ drivers/gpu/drm/msm/msm_gem_vma.c                  | 12 +++---
+ drivers/gpu/drm/msm/msm_ringbuffer.c               |  6 +--
+ drivers/gpu/drm/v3d/v3d_sched.c                    | 16 +------
+ drivers/gpu/drm/v3d/v3d_submit.c                   | 22 ++++++++--
+ drivers/gpu/drm/virtio/virtgpu_drv.h               |  1 +
+ drivers/gpu/drm/virtio/virtgpu_gem.c               | 17 ++++++++
+ drivers/gpu/drm/virtio/virtgpu_plane.c             | 10 ++++-
+ include/drm/drm_device.h                           |  7 +++
+ include/drm/drm_gem.h                              | 20 ++++-----
+ 22 files changed, 143 insertions(+), 153 deletions(-)
+Merging rust/rust-next (fc1ce3afa2e61 rust: fmt: use vertical import style)
+$ git merge -m Merge branch 'rust-next' of https://github.com/Rust-for-Linux/linux.git rust/rust-next
+Merge made by the 'ort' strategy.
+ rust/kernel/fmt.rs     | 19 +++++++++++++++++--
+ rust/kernel/prelude.rs |  3 +++
+ 2 files changed, 20 insertions(+), 2 deletions(-)
+Merging rust-interop/interop-next (05f7e89ab9731 Linux 6.19)
+$ git merge -m Merge branch 'interop-next' of https://github.com/Rust-for-Linux/linux.git rust-interop/interop-next
+Already up to date.
+Merging rust-alloc/alloc-next (ec51531fb8efd rust: alloc: cleanup doctest imports to "kernel vertical" style)
+$ git merge -m Merge branch 'alloc-next' of https://github.com/Rust-for-Linux/linux.git rust-alloc/alloc-next
+Merge made by the 'ort' strategy.
+ rust/kernel/alloc.rs                |  8 +++-
+ rust/kernel/alloc/allocator.rs      | 32 ++++++++++-----
+ rust/kernel/alloc/allocator/iter.rs |  8 +++-
+ rust/kernel/alloc/kbox.rs           | 74 ++++++++++++++++++++++-----------
+ rust/kernel/alloc/kvec.rs           | 81 ++++++++++++++++++++++++++++---------
+ rust/kernel/alloc/kvec/errors.rs    |  6 ++-
+ rust/kernel/alloc/layout.rs         | 10 ++++-
+ 7 files changed, 161 insertions(+), 58 deletions(-)
+Merging rust-io/io-next (86731a2a651e5 Linux 6.16-rc3)
+$ git merge -m Merge branch 'io-next' of https://github.com/Rust-for-Linux/linux.git rust-io/io-next
+Already up to date.
+Merging rust-pin-init/pin-init-next (6fb5912c92926 rust: pin-init: internal: project using full slot)
+$ git merge -m Merge branch 'pin-init-next' of https://github.com/Rust-for-Linux/linux.git rust-pin-init/pin-init-next
+Merge made by the 'ort' strategy.
+ rust/pin-init/README.md                       |   2 +-
+ rust/pin-init/examples/big_struct_in_place.rs |   3 -
+ rust/pin-init/examples/error.rs               |   2 +
+ rust/pin-init/examples/linked_list.rs         |   2 -
+ rust/pin-init/examples/mutex.rs               |   4 +-
+ rust/pin-init/examples/pthread_mutex.rs       |   4 +-
+ rust/pin-init/examples/static_init.rs         |   4 +-
+ rust/pin-init/internal/src/diagnostics.rs     |  14 ++
+ rust/pin-init/internal/src/init.rs            | 175 ++++++++------------
+ rust/pin-init/internal/src/lib.rs             |   1 -
+ rust/pin-init/internal/src/pin_data.rs        | 228 +++++++++++---------------
+ rust/pin-init/internal/src/zeroable.rs        |   2 +-
+ rust/pin-init/src/__internal.rs               | 215 ++++++++++++++++++------
+ rust/pin-init/src/lib.rs                      | 103 +++++-------
+ 14 files changed, 390 insertions(+), 369 deletions(-)
+Merging rust-timekeeping/timekeeping-next (ddb1444d33351 hrtimer: add usage examples to documentation)
+$ git merge -m Merge branch 'timekeeping-next' of https://github.com/Rust-for-Linux/linux.git rust-timekeeping/timekeeping-next
+Already up to date.
+Merging rust-xarray/xarray-next (c455f19bbe610 rust: xarray: add __rust_helper to helpers)
+$ git merge -m Merge branch 'xarray-next' of https://github.com/Rust-for-Linux/linux.git rust-xarray/xarray-next
+Already up to date.
+Merging rust-analyzer/rust-analyzer-next (5d6919055dec1 Linux 7.1-rc3)
+$ git merge -m Merge branch 'rust-analyzer-next' of https://github.com/Rust-for-Linux/linux.git rust-analyzer/rust-analyzer-next
+Already up to date.
+Merging mm-stable/mm-stable (27fa82620cbaa Merge tag 'ata-7.1-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/libata/linux)
+$ git merge -m Merge branch 'mm-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-stable/mm-stable
+Already up to date.
+Merging mm-nonmm-stable/mm-nonmm-stable (27fa82620cbaa Merge tag 'ata-7.1-rc5' of git://git.kernel.org/pub/scm/linux/kernel/git/libata/linux)
+$ git merge -m Merge branch 'mm-nonmm-stable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-stable/mm-nonmm-stable
+Already up to date.
+Merging mm-unstable/mm-unstable (e9add7501ad32 mm/page_alloc: document that alloc_pages_nolock() uses RCU)
+$ git merge -m Merge branch 'mm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-unstable/mm-unstable
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-kernel-mm-damon    |  81 ++-
+ Documentation/admin-guide/kernel-parameters.txt    |   4 +
+ Documentation/admin-guide/mm/damon/lru_sort.rst    |   8 +-
+ Documentation/admin-guide/mm/damon/reclaim.rst     |  19 +-
+ Documentation/admin-guide/mm/damon/stat.rst        |   7 +
+ Documentation/admin-guide/mm/damon/usage.rst       |  38 +-
+ Documentation/admin-guide/mm/hugetlbpage.rst       |  15 +-
+ Documentation/admin-guide/mm/transhuge.rst         |  71 +-
+ Documentation/mm/damon/design.rst                  |  33 +
+ Documentation/mm/damon/maintainer-profile.rst      |  21 +
+ arch/arm64/mm/mmu.c                                |   5 +-
+ arch/loongarch/mm/init.c                           |   5 +-
+ arch/powerpc/mm/init_64.c                          |   8 -
+ arch/powerpc/mm/mem.c                              |   5 +-
+ arch/powerpc/platforms/pseries/hotplug-memory.c    |  14 +-
+ arch/riscv/mm/init.c                               |   5 +-
+ arch/s390/mm/init.c                                |   5 +-
+ arch/s390/mm/vmem.c                                |   3 +-
+ arch/sh/mm/cache-sh4.c                             |   2 +-
+ arch/sparc/mm/init_64.c                            |  20 -
+ arch/x86/hyperv/mmu.c                              |   4 +-
+ arch/x86/include/asm/tlb.h                         |  19 +-
+ arch/x86/include/asm/tlbflush.h                    |   6 +-
+ arch/x86/kernel/smpboot.c                          |   1 +
+ arch/x86/mm/init_64.c                              |   5 +-
+ arch/x86/mm/tlb.c                                  |  39 +-
+ drivers/base/memory.c                              |  38 +-
+ drivers/base/node.c                                |   6 +-
+ drivers/dax/kmem.c                                 |   6 +
+ drivers/gpu/drm/drm_managed.c                      |   4 +-
+ drivers/s390/char/sclp_mem.c                       |  17 +-
+ fs/btrfs/defrag.c                                  |   3 -
+ fs/inode.c                                         |   3 -
+ fs/open.c                                          |  27 -
+ fs/proc/task_mmu.c                                 | 195 ++++--
+ fs/userfaultfd.c                                   |   2 +
+ include/asm-generic/pgalloc.h                      |   2 +-
+ include/asm-generic/tlb.h                          |  17 +
+ include/linux/alloc_tag.h                          |   4 +-
+ include/linux/bootmem_info.h                       |   1 -
+ include/linux/damon.h                              |  46 +-
+ include/linux/fs.h                                 |   5 -
+ include/linux/gfp.h                                |   4 +
+ include/linux/gfp_types.h                          |   6 +-
+ include/linux/highmem-internal.h                   |   2 +-
+ include/linux/huge_mm.h                            |  30 +-
+ include/linux/memory.h                             |   7 +-
+ include/linux/memory_hotplug.h                     |   8 +-
+ include/linux/mm.h                                 |  18 -
+ include/linux/mmu_notifier.h                       |   4 +-
+ include/linux/mmzone.h                             |   9 +-
+ include/linux/pageblock-flags.h                    |   6 +-
+ include/linux/pagemap.h                            |  58 +-
+ include/linux/shmem_fs.h                           |   2 +-
+ include/linux/thread_info.h                        |   2 +-
+ include/linux/vmpressure.h                         |   9 +-
+ include/trace/events/huge_memory.h                 |  37 +-
+ include/trace/events/vmscan.h                      |  52 ++
+ kernel/bpf/arena.c                                 |   2 +-
+ kernel/fork.c                                      |   5 +-
+ kernel/power/snapshot.c                            |   8 +-
+ lib/alloc_tag.c                                    | 147 ++--
+ lib/maple_tree.c                                   |   7 +-
+ lib/test_hmm.c                                     |   4 +-
+ lib/test_meminit.c                                 |   2 +-
+ mm/Kconfig                                         |  18 +-
+ mm/bootmem_info.c                                  |  25 +-
+ mm/compaction.c                                    |   3 +-
+ mm/damon/core.c                                    | 349 ++++++++--
+ mm/damon/lru_sort.c                                |  61 +-
+ mm/damon/ops-common.c                              |   9 +-
+ mm/damon/paddr.c                                   |   8 +-
+ mm/damon/reclaim.c                                 | 100 ++-
+ mm/damon/stat.c                                    |  92 ++-
+ mm/damon/sysfs-schemes.c                           |  67 +-
+ mm/damon/sysfs.c                                   |  31 +
+ mm/damon/tests/core-kunit.h                        |  38 +-
+ mm/damon/vaddr.c                                   |  10 +-
+ mm/filemap.c                                       |  81 +--
+ mm/gup.c                                           |  21 +-
+ mm/huge_memory.c                                   | 157 +++--
+ mm/hugetlb.c                                       |   7 +-
+ mm/hugetlb_cma.c                                   |  35 +-
+ mm/internal.h                                      |  14 +-
+ mm/kfence/kfence_test.c                            |   2 +-
+ mm/khugepaged.c                                    | 747 +++++++++++++++------
+ mm/kmemleak.c                                      | 148 +++-
+ mm/madvise.c                                       |  60 +-
+ mm/memcontrol.c                                    |   5 +-
+ mm/memory-failure.c                                |   8 +-
+ mm/memory.c                                        |  57 +-
+ mm/memory_hotplug.c                                |  34 +-
+ mm/memremap.c                                      |   4 +-
+ mm/migrate.c                                       |  48 +-
+ mm/migrate_device.c                                |   9 +-
+ mm/mm_init.c                                       |  58 +-
+ mm/mmap.c                                          |   2 +-
+ mm/mmu_gather.c                                    |  15 +
+ mm/mseal.c                                         |   5 +-
+ mm/page_alloc.c                                    | 371 +++++++---
+ mm/page_io.c                                       |  42 +-
+ mm/page_isolation.c                                |  69 +-
+ mm/page_owner.c                                    |   2 +-
+ mm/percpu-internal.h                               |  30 +-
+ mm/percpu-stats.c                                  |   2 +-
+ mm/percpu.c                                        | 318 +++++----
+ mm/readahead.c                                     |  36 +-
+ mm/rmap.c                                          |   8 +-
+ mm/shmem.c                                         |   5 +-
+ mm/shrinker.c                                      |  18 +-
+ mm/sparse-vmemmap.c                                |  75 ++-
+ mm/sparse.c                                        |  68 +-
+ mm/swap.c                                          |  43 +-
+ mm/swapfile.c                                      |  43 +-
+ mm/truncate.c                                      |   8 +-
+ mm/vmalloc.c                                       |  29 +-
+ mm/vmpressure.c                                    |  15 +-
+ mm/vmscan.c                                        | 362 +++++-----
+ net/rds/tcp_recv.c                                 |   2 +-
+ tools/mm/page-types.c                              |   8 +-
+ tools/mm/slabinfo.c                                |   8 +-
+ tools/testing/selftests/cgroup/lib/cgroup_util.c   |  18 +-
+ .../selftests/cgroup/lib/include/cgroup_util.h     |   4 +-
+ tools/testing/selftests/cgroup/test_core.c         |   2 +-
+ tools/testing/selftests/cgroup/test_freezer.c      |   2 +-
+ tools/testing/selftests/cgroup/test_kmem.c         |  13 +-
+ tools/testing/selftests/cgroup/test_memcontrol.c   |  19 +-
+ tools/testing/selftests/cgroup/test_zswap.c        | 181 +++--
+ tools/testing/selftests/damon/_damon_sysfs.py      |  31 +-
+ .../selftests/damon/drgn_dump_damon_status.py      |   3 +
+ tools/testing/selftests/damon/sysfs.py             |  56 +-
+ tools/testing/selftests/mm/.gitignore              |   4 +
+ tools/testing/selftests/mm/Makefile                |  23 +-
+ tools/testing/selftests/mm/check_config.sh         |   2 +-
+ tools/testing/selftests/mm/compaction_test.c       | 115 +---
+ tools/testing/selftests/mm/cow.c                   |  31 +-
+ tools/testing/selftests/mm/droppable.c             |   9 +-
+ tools/testing/selftests/mm/folio_split_race_test.c |  25 +-
+ tools/testing/selftests/mm/guard-regions.c         |  20 +-
+ tools/testing/selftests/mm/gup_longterm.c          |   5 +-
+ tools/testing/selftests/mm/gup_test.c              |  15 +
+ tools/testing/selftests/mm/hmm-tests.c             |  26 +-
+ tools/testing/selftests/mm/hugepage-mmap.c         |  78 ---
+ .../mm/{thp_settings.c => hugepage_settings.c}     | 316 +++++++--
+ .../mm/{thp_settings.h => hugepage_settings.h}     |  82 ++-
+ tools/testing/selftests/mm/hugetlb-madvise.c       | 219 +++---
+ tools/testing/selftests/mm/hugetlb-mmap.c          | 143 ++++
+ .../mm/{hugepage-mremap.c => hugetlb-mremap.c}     |  15 +-
+ tools/testing/selftests/mm/hugetlb-read-hwpoison.c | 121 ++--
+ .../selftests/mm/{hugepage-shm.c => hugetlb-shm.c} |  73 +-
+ tools/testing/selftests/mm/hugetlb-soft-offline.c  |  45 +-
+ .../mm/{hugepage-vmemmap.c => hugetlb-vmemmap.c}   |  46 +-
+ tools/testing/selftests/mm/hugetlb_dio.c           |  15 +-
+ .../selftests/mm/hugetlb_fault_after_madv.c        |   9 +-
+ tools/testing/selftests/mm/hugetlb_madv_vs_map.c   |  22 +-
+ tools/testing/selftests/mm/khugepaged.c            | 575 +++++++++-------
+ tools/testing/selftests/mm/ksft_kmemleak_dedup.sh  | 222 ++++++
+ tools/testing/selftests/mm/ksm_functional_tests.c  |  19 +-
+ tools/testing/selftests/mm/ksm_tests.c             | 184 +++--
+ tools/testing/selftests/mm/madv_populate.c         |   2 +-
+ tools/testing/selftests/mm/map_hugetlb.c           |  88 ---
+ tools/testing/selftests/mm/migration.c             | 147 ++--
+ tools/testing/selftests/mm/mlock2-tests.c          |  86 ++-
+ tools/testing/selftests/mm/mremap_test.c           | 109 +--
+ tools/testing/selftests/mm/pagemap_ioctl.c         |  13 +-
+ tools/testing/selftests/mm/pkey-helpers.h          |  15 +-
+ tools/testing/selftests/mm/prctl_thp_disable.c     |   2 +-
+ tools/testing/selftests/mm/process_madv.c          |  28 +
+ tools/testing/selftests/mm/protection_keys.c       | 130 ++--
+ tools/testing/selftests/mm/run_vmtests.sh          | 187 +-----
+ tools/testing/selftests/mm/soft-dirty.c            |   6 +-
+ tools/testing/selftests/mm/split_huge_page_test.c  |  12 +-
+ tools/testing/selftests/mm/thuge-gen.c             |  96 +--
+ tools/testing/selftests/mm/transhuge-stress.c      |   2 +-
+ tools/testing/selftests/mm/uffd-common.h           |  18 +-
+ tools/testing/selftests/mm/uffd-stress.c           |  43 +-
+ tools/testing/selftests/mm/uffd-unit-tests.c       | 138 ++--
+ tools/testing/selftests/mm/uffd-wp-mremap.c        |  20 +-
+ tools/testing/selftests/mm/va_high_addr_switch.c   |  42 +-
+ tools/testing/selftests/mm/va_high_addr_switch.sh  |  41 +-
+ tools/testing/selftests/mm/vm_util.c               | 136 ++--
+ tools/testing/selftests/mm/vm_util.h               |  15 +-
+ tools/testing/selftests/proc/proc-maps-race.c      | 295 ++++++--
+ 183 files changed, 5867 insertions(+), 3730 deletions(-)
+ delete mode 100644 tools/testing/selftests/mm/hugepage-mmap.c
+ rename tools/testing/selftests/mm/{thp_settings.c => hugepage_settings.c} (58%)
+ rename tools/testing/selftests/mm/{thp_settings.h => hugepage_settings.h} (51%)
+ create mode 100644 tools/testing/selftests/mm/hugetlb-mmap.c
+ rename tools/testing/selftests/mm/{hugepage-mremap.c => hugetlb-mremap.c} (92%)
+ rename tools/testing/selftests/mm/{hugepage-shm.c => hugetlb-shm.c} (55%)
+ rename tools/testing/selftests/mm/{hugepage-vmemmap.c => hugetlb-vmemmap.c} (76%)
+ create mode 100755 tools/testing/selftests/mm/ksft_kmemleak_dedup.sh
+ delete mode 100644 tools/testing/selftests/mm/map_hugetlb.c
+Merging mm-nonmm-unstable/mm-nonmm-unstable (d067a83c8063d string: use min in sized_strscpy)
+$ git merge -m Merge branch 'mm-nonmm-unstable' of https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm mm-nonmm-unstable/mm-nonmm-unstable
+Auto-merging MAINTAINERS
+Auto-merging lib/alloc_tag.c
+Auto-merging mm/Kconfig
+Merge made by the 'ort' strategy.
+ .clang-format                                      |   2 +
+ .mailmap                                           |   1 +
+ CREDITS                                            |   4 +
+ Documentation/crypto/async-tx-api.rst              |   4 +-
+ Documentation/dev-tools/checkpatch.rst             |  14 +
+ Documentation/dev-tools/kcov.rst                   |   5 +
+ MAINTAINERS                                        |   4 +-
+ Makefile                                           |  27 +-
+ arch/arc/include/asm/uaccess.h                     |   3 +-
+ arch/arm/include/asm/uaccess.h                     |   3 +-
+ arch/arm64/include/asm/uaccess.h                   |   3 +-
+ arch/hexagon/include/asm/uaccess.h                 |   3 +-
+ arch/loongarch/include/asm/uaccess.h               |   3 +-
+ arch/m68k/include/asm/uaccess.h                    |   3 +-
+ arch/microblaze/include/asm/uaccess.h              |   3 +-
+ arch/mips/include/asm/uaccess.h                    |   3 +-
+ arch/nios2/include/asm/uaccess.h                   |   3 +-
+ arch/openrisc/include/asm/uaccess.h                |   3 +-
+ arch/parisc/include/asm/uaccess.h                  |   3 +-
+ arch/riscv/Kconfig                                 |   3 -
+ arch/riscv/include/asm/asm-prototypes.h            |   4 +
+ arch/riscv/kernel/image-vars.h                     |   9 +
+ arch/riscv/lib/Makefile                            |   1 +
+ arch/riscv/lib/ashldi3.S                           |  36 ++
+ arch/riscv/lib/ashrdi3.S                           |  36 ++
+ arch/riscv/lib/lshrdi3.S                           |  36 ++
+ arch/s390/include/asm/uaccess.h                    |   3 +-
+ arch/sh/include/asm/uaccess.h                      |   3 +-
+ arch/sparc/include/asm/uaccess_32.h                |   3 +-
+ arch/sparc/include/asm/uaccess_64.h                |   3 +-
+ arch/um/include/asm/uaccess.h                      |   3 +-
+ arch/xtensa/include/asm/uaccess.h                  |   3 +-
+ certs/Kconfig                                      |  14 +-
+ crypto/async_tx/async_pq.c                         |   9 +-
+ crypto/async_tx/async_raid6_recov.c                |   9 +-
+ drivers/dma/bcm-sba-raid.c                         |   1 +
+ drivers/firmware/efi/libstub/Makefile              |   3 +-
+ drivers/md/raid5.c                                 |   4 +-
+ drivers/rapidio/devices/tsi721.c                   |   3 +-
+ drivers/usb/usbip/usbip_common.h                   |  29 +-
+ drivers/usb/usbip/vhci_rx.c                        |   4 +-
+ drivers/usb/usbip/vhci_sysfs.c                     |   2 +-
+ drivers/vhost/vhost.h                              |   2 +-
+ fs/Kconfig                                         |   6 +-
+ fs/btrfs/raid56.c                                  |   8 +-
+ fs/fat/fat_test.c                                  |  33 ++
+ fs/ocfs2/file.c                                    |  23 +-
+ fs/ocfs2/inode.c                                   | 151 ++++++++-
+ fs/ocfs2/journal.c                                 |   7 +-
+ fs/ocfs2/ocfs2.h                                   |   2 -
+ fs/ocfs2/quota_local.c                             |   2 +-
+ fs/ocfs2/super.c                                   |   2 -
+ fs/ocfs2/sysfile.c                                 |   9 +-
+ fs/ocfs2/xattr.c                                   | 123 ++++---
+ fs/proc/base.c                                     |  63 ++--
+ fs/proc/generic.c                                  |  10 +-
+ include/asm-generic/uaccess.h                      |   3 +-
+ include/linux/cnt32_to_63.h                        | 104 ------
+ include/linux/codetag.h                            |   3 +-
+ include/linux/init.h                               |   2 +-
+ include/linux/kcov.h                               |  14 +-
+ include/linux/llist.h                              |   4 +-
+ include/linux/raid/pq.h                            | 216 +-----------
+ include/linux/raid/pq_tables.h                     |  19 ++
+ include/linux/sched.h                              |   3 +
+ include/linux/skbuff.h                             |  14 +-
+ include/linux/types.h                              |   6 +
+ include/linux/uaccess.h                            |  25 +-
+ init/Kconfig                                       |  24 +-
+ kernel/kcov.c                                      | 100 +++---
+ kernel/taskstats.c                                 |  62 ++--
+ lib/Kconfig                                        |  13 +-
+ lib/Kconfig.debug                                  |  10 +
+ lib/Makefile                                       |   1 -
+ lib/alloc_tag.c                                    |   8 +-
+ lib/base64.c                                       |   5 +-
+ lib/bug.c                                          |  80 ++---
+ lib/cmdline.c                                      |  30 +-
+ lib/codetag.c                                      |  12 +-
+ lib/error-inject.c                                 |   4 +-
+ lib/idr.c                                          |  55 ++-
+ lib/iov_iter.c                                     |  54 +--
+ lib/kstrtox.c                                      |  37 +-
+ lib/raid/Kconfig                                   |  33 ++
+ lib/raid/Makefile                                  |   2 +-
+ lib/{ => raid}/raid6/.gitignore                    |   0
+ lib/raid/raid6/Makefile                            | 128 +++++++
+ lib/raid/raid6/algos.c                             | 377 +++++++++++++++++++++
+ lib/raid/raid6/algos.h                             |  41 +++
+ lib/{raid6 => raid/raid6/arm}/neon.c               |  23 +-
+ lib/{raid6 => raid/raid6/arm}/neon.h               |   0
+ lib/{raid6 => raid/raid6/arm}/neon.uc              |   2 +-
+ lib/raid/raid6/arm/pq_arch.h                       |  21 ++
+ lib/{raid6 => raid/raid6/arm}/recov_neon.c         |  27 +-
+ lib/{raid6 => raid/raid6/arm}/recov_neon_inner.c   |   2 +-
+ lib/raid/raid6/arm64/pq_arch.h                     |   1 +
+ lib/{ => raid}/raid6/int.uc                        |  10 +-
+ .../raid6/loongarch}/loongarch_simd.c              |  31 +-
+ lib/raid/raid6/loongarch/pq_arch.h                 |  23 ++
+ .../raid6/loongarch}/recov_loongarch_simd.c        |  39 +--
+ lib/{ => raid}/raid6/mktables.c                    |  28 +-
+ lib/{raid6 => raid/raid6/powerpc}/altivec.uc       |  32 +-
+ lib/raid/raid6/powerpc/pq_arch.h                   |  32 ++
+ lib/{raid6 => raid/raid6/powerpc}/vpermxor.uc      |  29 +-
+ lib/{ => raid}/raid6/recov.c                       |  62 +---
+ lib/raid/raid6/riscv/pq_arch.h                     |  21 ++
+ lib/{raid6 => raid/raid6/riscv}/recov_rvv.c        |  14 +-
+ lib/{raid6 => raid/raid6/riscv}/rvv.c              |   0
+ lib/{raid6 => raid/raid6/riscv}/rvv.h              |  26 +-
+ lib/raid/raid6/s390/pq_arch.h                      |  15 +
+ lib/{raid6 => raid/raid6/s390}/recov_s390xc.c      |  14 +-
+ lib/{raid6 => raid/raid6/s390}/s390vx.uc           |  15 +-
+ lib/raid/raid6/tests/Makefile                      |   3 +
+ lib/raid/raid6/tests/raid6_kunit.c                 | 321 ++++++++++++++++++
+ lib/{ => raid}/raid6/unroll.awk                    |   0
+ lib/{raid6 => raid/raid6/x86}/avx2.c               |  49 +--
+ lib/{raid6 => raid/raid6/x86}/avx512.c             |  61 ++--
+ lib/{raid6 => raid/raid6/x86}/mmx.c                |  39 +--
+ lib/raid/raid6/x86/pq_arch.h                       |  96 ++++++
+ lib/{raid6 => raid/raid6/x86}/recov_avx2.c         |  22 +-
+ lib/{raid6 => raid/raid6/x86}/recov_avx512.c       |  26 +-
+ lib/{raid6 => raid/raid6/x86}/recov_ssse3.c        |  23 +-
+ lib/{raid6 => raid/raid6/x86}/sse1.c               |  49 +--
+ lib/{raid6 => raid/raid6/x86}/sse2.c               |  47 +--
+ lib/raid6/Makefile                                 |  83 -----
+ lib/raid6/algos.c                                  | 291 ----------------
+ lib/raid6/loongarch.h                              |  38 ---
+ lib/raid6/test/.gitignore                          |   3 -
+ lib/raid6/test/Makefile                            | 156 ---------
+ lib/raid6/test/test.c                              | 152 ---------
+ lib/raid6/x86.h                                    |  75 ----
+ lib/seq_buf.c                                      |   1 +
+ lib/string.c                                       |   8 +-
+ lib/test-kstrtox.c                                 |   6 +
+ lib/test_ida.c                                     |  14 +
+ lib/tests/Makefile                                 |   1 +
+ lib/tests/cmdline_kunit.c                          | 118 +++++++
+ lib/tests/seq_buf_kunit.c                          |  34 ++
+ lib/tests/shdi3_kunit.c                            | 175 ++++++++++
+ lib/tests/string_helpers_kunit.c                   |  15 +-
+ lib/tests/uuid_kunit.c                             |  56 +++
+ lib/usercopy.c                                     |   4 +-
+ mm/Kconfig                                         |   4 +-
+ mm/kfence/core.c                                   |   5 +
+ rust/helpers/uaccess.c                             |   2 +-
+ scripts/bloat-o-meter                              |   1 +
+ scripts/checkpatch.pl                              |  47 ++-
+ scripts/get_maintainer.pl                          |  71 ++--
+ tools/accounting/getdelays.c                       |   8 +-
+ tools/testing/selftests/acct/.gitignore            |   3 +-
+ tools/testing/selftests/acct/Makefile              |   7 +-
+ .../selftests/acct/taskstats_fill_stats_tgid.c     | 375 ++++++++++++++++++++
+ tools/testing/selftests/filelock/.gitignore        |   1 +
+ tools/testing/selftests/filelock/ofdlocks.c        |  94 +++--
+ .../selftests/perf_events/watermark_signal.c       |   2 +-
+ 155 files changed, 3111 insertions(+), 2187 deletions(-)
+ create mode 100644 arch/riscv/lib/ashldi3.S
+ create mode 100644 arch/riscv/lib/ashrdi3.S
+ create mode 100644 arch/riscv/lib/lshrdi3.S
+ delete mode 100644 include/linux/cnt32_to_63.h
+ create mode 100644 include/linux/raid/pq_tables.h
+ rename lib/{ => raid}/raid6/.gitignore (100%)
+ create mode 100644 lib/raid/raid6/Makefile
+ create mode 100644 lib/raid/raid6/algos.c
+ create mode 100644 lib/raid/raid6/algos.h
+ rename lib/{raid6 => raid/raid6/arm}/neon.c (79%)
+ rename lib/{raid6 => raid/raid6/arm}/neon.h (100%)
+ rename lib/{raid6 => raid/raid6/arm}/neon.uc (99%)
+ create mode 100644 lib/raid/raid6/arm/pq_arch.h
+ rename lib/{raid6 => raid/raid6/arm}/recov_neon.c (81%)
+ rename lib/{raid6 => raid/raid6/arm}/recov_neon_inner.c (99%)
+ create mode 100644 lib/raid/raid6/arm64/pq_arch.h
+ rename lib/{ => raid}/raid6/int.uc (96%)
+ rename lib/{raid6 => raid/raid6/loongarch}/loongarch_simd.c (96%)
+ create mode 100644 lib/raid/raid6/loongarch/pq_arch.h
+ rename lib/{raid6 => raid/raid6/loongarch}/recov_loongarch_simd.c (96%)
+ rename lib/{ => raid}/raid6/mktables.c (83%)
+ rename lib/{raid6 => raid/raid6/powerpc}/altivec.uc (84%)
+ create mode 100644 lib/raid/raid6/powerpc/pq_arch.h
+ rename lib/{raid6 => raid/raid6/powerpc}/vpermxor.uc (82%)
+ rename lib/{ => raid}/raid6/recov.c (60%)
+ create mode 100644 lib/raid/raid6/riscv/pq_arch.h
+ rename lib/{raid6 => raid/raid6/riscv}/recov_rvv.c (95%)
+ rename lib/{raid6 => raid/raid6/riscv}/rvv.c (100%)
+ rename lib/{raid6 => raid/raid6/riscv}/rvv.h (70%)
+ create mode 100644 lib/raid/raid6/s390/pq_arch.h
+ rename lib/{raid6 => raid/raid6/s390}/recov_s390xc.c (90%)
+ rename lib/{raid6 => raid/raid6/s390}/s390vx.uc (93%)
+ create mode 100644 lib/raid/raid6/tests/Makefile
+ create mode 100644 lib/raid/raid6/tests/raid6_kunit.c
+ rename lib/{ => raid}/raid6/unroll.awk (100%)
+ rename lib/{raid6 => raid/raid6/x86}/avx2.c (94%)
+ rename lib/{raid6 => raid/raid6/x86}/avx512.c (93%)
+ rename lib/{raid6 => raid/raid6/x86}/mmx.c (81%)
+ create mode 100644 lib/raid/raid6/x86/pq_arch.h
+ rename lib/{raid6 => raid/raid6/x86}/recov_avx2.c (95%)
+ rename lib/{raid6 => raid/raid6/x86}/recov_avx512.c (94%)
+ rename lib/{raid6 => raid/raid6/x86}/recov_ssse3.c (95%)
+ rename lib/{raid6 => raid/raid6/x86}/sse1.c (78%)
+ rename lib/{raid6 => raid/raid6/x86}/sse2.c (94%)
+ delete mode 100644 lib/raid6/Makefile
+ delete mode 100644 lib/raid6/algos.c
+ delete mode 100644 lib/raid6/loongarch.h
+ delete mode 100644 lib/raid6/test/.gitignore
+ delete mode 100644 lib/raid6/test/Makefile
+ delete mode 100644 lib/raid6/test/test.c
+ delete mode 100644 lib/raid6/x86.h
+ create mode 100644 lib/tests/shdi3_kunit.c
+ create mode 100644 tools/testing/selftests/acct/taskstats_fill_stats_tgid.c
+ create mode 100644 tools/testing/selftests/filelock/.gitignore
+Merging kbuild/kbuild-for-next (2c31897a17e55 kbuild: pacman-pkg: package unstripped vDSO libraries)
+$ git merge -m Merge branch 'kbuild-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/kbuild/linux.git kbuild/kbuild-for-next
+Auto-merging scripts/package/PKGBUILD
+Merge made by the 'ort' strategy.
+ Documentation/kbuild/makefiles.rst | 41 ++++++++++++++++++++++++++++++--------
+ scripts/Makefile.modfinal          | 10 +---------
+ scripts/kconfig/conf.c             |  6 ++----
+ scripts/package/PKGBUILD           |  3 +++
+ 4 files changed, 39 insertions(+), 21 deletions(-)
+Merging clang-format/clang-format (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'clang-format' of https://github.com/ojeda/linux.git clang-format/clang-format
+Already up to date.
+Merging perf/perf-tools-next (85cc481af5ffa perf build: Prefix SCRIPTS with output directory to fix continuous rebuilds)
+$ git merge -m Merge branch 'perf-tools-next' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git perf/perf-tools-next
+Auto-merging tools/perf/Makefile.perf
+CONFLICT (content): Merge conflict in tools/perf/Makefile.perf
+Auto-merging tools/perf/builtin-trace.c
+CONFLICT (content): Merge conflict in tools/perf/builtin-trace.c
+Auto-merging tools/perf/trace/beauty/beauty.h
+Recorded preimage for 'tools/perf/Makefile.perf'
+Recorded preimage for 'tools/perf/builtin-trace.c'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'tools/perf/Makefile.perf'.
+Recorded resolution for 'tools/perf/builtin-trace.c'.
+[master 61da860eee079] Merge branch 'perf-tools-next' of https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git
+$ git diff -M --stat --summary HEAD^..
+ tools/build/Makefile.feature                       |  12 +-
+ tools/build/feature/Makefile                       |  40 +-
+ tools/build/feature/test-libunwind-aarch64.c       |  27 -
+ tools/build/feature/test-libunwind-arm.c           |  28 -
+ .../feature/test-libunwind-debug-frame-aarch64.c   |  17 -
+ .../build/feature/test-libunwind-debug-frame-arm.c |  17 -
+ tools/build/feature/test-libunwind-debug-frame.c   |   1 -
+ tools/build/feature/test-libunwind-x86.c           |  28 -
+ tools/build/feature/test-libunwind-x86_64.c        |  28 -
+ tools/build/feature/test-libunwind.c               |   1 -
+ tools/perf/.clang-format                           |  20 +
+ tools/perf/.gitignore                              |   1 +
+ tools/perf/Build                                   |   2 +
+ tools/perf/Documentation/perf-trace.txt            |   3 +
+ tools/perf/Makefile.config                         | 249 +++---
+ tools/perf/Makefile.perf                           | 431 +----------
+ tools/perf/arch/arm/util/Build                     |   2 -
+ tools/perf/arch/arm/util/unwind-libunwind.c        |  50 --
+ tools/perf/arch/arm64/util/Build                   |   2 -
+ tools/perf/arch/arm64/util/machine.c               |  12 -
+ tools/perf/arch/arm64/util/unwind-libunwind.c      |  17 -
+ tools/perf/arch/common.c                           |   4 +-
+ tools/perf/arch/loongarch/util/Build               |   2 -
+ tools/perf/arch/loongarch/util/unwind-libunwind.c  |  82 --
+ tools/perf/arch/mips/Build                         |   1 -
+ tools/perf/arch/mips/util/Build                    |   1 -
+ tools/perf/arch/mips/util/unwind-libunwind.c       |  22 -
+ tools/perf/arch/powerpc/util/Build                 |   1 -
+ tools/perf/arch/powerpc/util/unwind-libunwind.c    |  92 ---
+ tools/perf/arch/x86/tests/topdown.c                | 166 +++-
+ tools/perf/arch/x86/util/Build                     |   3 -
+ tools/perf/arch/x86/util/unwind-libunwind.c        | 115 ---
+ tools/perf/bench/Build                             |   6 +
+ .../{util => bench}/bpf_skel/bench_uprobe.bpf.c    |   0
+ tools/perf/bench/inject-buildid.c                  |   9 +-
+ tools/perf/bench/uprobe.c                          |   2 +-
+ tools/perf/bpf_skel.mak                            | 109 +++
+ tools/perf/builtin-annotate.c                      |  28 +-
+ tools/perf/builtin-buildid-list.c                  |   4 +-
+ tools/perf/builtin-c2c.c                           |   6 +-
+ tools/perf/builtin-diff.c                          |   5 +-
+ tools/perf/builtin-inject.c                        | 247 ++++--
+ tools/perf/builtin-kmem.c                          |  95 ++-
+ tools/perf/builtin-kvm.c                           |  22 +-
+ tools/perf/builtin-kwork.c                         | 104 +--
+ tools/perf/builtin-lock.c                          | 142 ++--
+ tools/perf/builtin-mem.c                           |   1 -
+ tools/perf/builtin-record.c                        |  14 +-
+ tools/perf/builtin-report.c                        |  42 +-
+ tools/perf/builtin-sched.c                         | 394 ++++++----
+ tools/perf/builtin-script.c                        |  26 +-
+ tools/perf/builtin-timechart.c                     | 164 ++--
+ tools/perf/builtin-top.c                           |  19 +-
+ tools/perf/builtin-trace.c                         | 207 +++--
+ tools/perf/pmu-events/Build                        |  26 +-
+ tools/perf/pmu-events/amd_metrics.py               |   2 +-
+ tools/perf/pmu-events/jevents.py                   |  23 +-
+ tools/perf/pmu-events/metric.py                    |   1 -
+ tools/perf/tests/Build                             |   1 +
+ tools/perf/tests/builtin-test.c                    |   1 +
+ tools/perf/tests/dlfilter-test.c                   |   8 +-
+ tools/perf/tests/hists_cumulate.c                  |   1 -
+ tools/perf/tests/hists_filter.c                    |   1 -
+ tools/perf/tests/hists_output.c                    |   1 -
+ tools/perf/tests/make                              |   6 +-
+ tools/perf/tests/mmap-basic.c                      |   4 +-
+ tools/perf/tests/openat-syscall-tp-fields.c        |   7 +-
+ tools/perf/tests/pmu-events.c                      |  24 +-
+ tools/perf/tests/sample-parsing.c                  |   5 +-
+ tools/perf/tests/shell/ftrace.sh                   |   7 +-
+ tools/perf/tests/shell/trace_summary.sh            |   2 +-
+ tools/perf/tests/switch-tracking.c                 |   9 +-
+ tools/perf/tests/tests.h                           |   1 +
+ tools/perf/tests/uncore-event-sorting.c            | 176 +++++
+ tools/perf/trace/beauty/Build                      | 276 +++++++
+ tools/perf/trace/beauty/arch_errno_names.c         |   2 +
+ tools/perf/trace/beauty/arch_errno_names.sh        |   2 +-
+ tools/perf/trace/beauty/beauty.h                   |  60 ++
+ tools/perf/trace/beauty/eventfd.c                  |   6 +-
+ tools/perf/trace/beauty/fcntl.c                    |  16 +
+ tools/perf/trace/beauty/fsconfig.c                 |   5 +
+ tools/perf/trace/beauty/futex_op.c                 |   5 +-
+ tools/perf/trace/beauty/futex_val3.c               |   5 +-
+ tools/perf/trace/beauty/mmap.c                     |  24 +-
+ tools/perf/trace/beauty/mode_t.c                   |   6 +-
+ tools/perf/trace/beauty/msg_flags.c                |   8 +-
+ tools/perf/trace/beauty/open_flags.c               |   2 +
+ tools/perf/trace/beauty/perf_event_open.c          |  21 +-
+ tools/perf/trace/beauty/pid.c                      |   5 +-
+ tools/perf/trace/beauty/sched_policy.c             |   8 +-
+ tools/perf/trace/beauty/seccomp.c                  |  12 +-
+ tools/perf/trace/beauty/signum.c                   |   6 +-
+ tools/perf/trace/beauty/socket_type.c              |   6 +-
+ tools/perf/{util => trace/beauty}/syscalltbl.c     |   0
+ tools/perf/{util => trace/beauty}/syscalltbl.h     |   0
+ tools/perf/trace/beauty/tracepoints/Build          |  21 +
+ tools/perf/trace/beauty/waitid_options.c           |   8 +-
+ tools/perf/util/Build                              |  22 +-
+ tools/perf/util/annotate-data.c                    |  27 +-
+ tools/perf/util/annotate.c                         |  19 +-
+ tools/perf/util/annotate.h                         |   6 +-
+ tools/perf/util/arm-spe-decoder/Build              |   2 +
+ tools/perf/util/arm-spe-decoder/arm-spe-decoder.c  |  17 +-
+ tools/perf/util/arm-spe-decoder/arm-spe-decoder.h  |   3 +-
+ .../util/arm-spe-decoder/arm-spe-pkt-decoder.c     | 144 ++--
+ .../util/arm-spe-decoder/arm-spe-pkt-decoder.h     |   5 +-
+ tools/perf/util/arm-spe.c                          |  90 ++-
+ .../perf/util/arm64-frame-pointer-unwind-support.c |   6 +
+ .../perf/util/arm64-frame-pointer-unwind-support.h |   2 +
+ tools/perf/util/bpf-trace-summary.c                |   2 +-
+ tools/perf/util/build-id.c                         |   3 +-
+ tools/perf/util/build-id.h                         |   7 +-
+ tools/perf/util/callchain.c                        |   8 +-
+ tools/perf/util/callchain.h                        |   8 +-
+ tools/perf/util/config.c                           |   4 +
+ tools/perf/util/cs-etm.c                           |  28 +-
+ tools/perf/util/data-convert-bt.c                  |   2 +-
+ tools/perf/util/data-convert-json.c                |   5 +-
+ tools/perf/util/db-export.c                        |  13 +-
+ tools/perf/util/db-export.h                        |   3 +-
+ tools/perf/util/debuginfo.c                        |   9 +-
+ tools/perf/util/dwarf-aux.c                        |  39 +-
+ tools/perf/util/dwarf-aux.h                        |   5 +
+ tools/perf/util/env.c                              |   4 -
+ tools/perf/util/env.h                              |   1 +
+ tools/perf/util/evsel.c                            | 137 +++-
+ tools/perf/util/evsel.h                            |  12 +-
+ tools/perf/util/hist.c                             |  26 +-
+ tools/perf/util/hist.h                             |   3 +-
+ tools/perf/util/intel-bts.c                        |   3 +-
+ tools/perf/util/intel-pt.c                         |  37 +-
+ tools/perf/util/intel-tpebs.c                      |   3 +-
+ tools/perf/util/jitdump.c                          |   2 +-
+ tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c     |  19 +-
+ tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c |  17 +-
+ tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c   |  17 +-
+ tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c     |  17 +-
+ tools/perf/util/kvm-stat-arch/kvm-stat-s390.c      |  20 +-
+ tools/perf/util/kvm-stat-arch/kvm-stat-x86.c       |  70 +-
+ tools/perf/util/kvm-stat.c                         |  19 +-
+ tools/perf/util/kvm-stat.h                         |  18 +-
+ tools/perf/util/kwork.h                            |   9 +-
+ tools/perf/util/libdw.c                            |  72 +-
+ tools/perf/util/libunwind-arch/Build               |  11 +
+ tools/perf/util/libunwind-arch/libunwind-arch.c    | 319 ++++++++
+ tools/perf/util/libunwind-arch/libunwind-arch.h    | 296 ++++++++
+ tools/perf/util/libunwind-arch/libunwind-arm.c     | 290 +++++++
+ tools/perf/util/libunwind-arch/libunwind-arm64.c   | 289 +++++++
+ tools/perf/util/libunwind-arch/libunwind-i386.c    | 312 ++++++++
+ .../perf/util/libunwind-arch/libunwind-loongarch.c | 297 ++++++++
+ tools/perf/util/libunwind-arch/libunwind-mips.c    | 299 ++++++++
+ tools/perf/util/libunwind-arch/libunwind-ppc32.c   | 303 ++++++++
+ tools/perf/util/libunwind-arch/libunwind-ppc64.c   | 303 ++++++++
+ tools/perf/util/libunwind-arch/libunwind-riscv.c   | 297 ++++++++
+ tools/perf/util/libunwind-arch/libunwind-s390.c    | 299 ++++++++
+ tools/perf/util/libunwind-arch/libunwind-x86_64.c  | 320 ++++++++
+ tools/perf/util/libunwind/arm64.c                  |  40 -
+ tools/perf/util/libunwind/x86_32.c                 |  41 -
+ tools/perf/util/machine.c                          |  14 +-
+ tools/perf/util/machine.h                          |   3 -
+ tools/perf/util/maps.c                             |  34 +-
+ tools/perf/util/maps.h                             |   4 +-
+ tools/perf/util/metricgroup.c                      |  11 +-
+ tools/perf/util/perf-regs-arch/perf_regs_riscv.c   | 128 ++++
+ tools/perf/util/perf_regs.c                        |   3 +
+ tools/perf/util/perf_regs.h                        |   1 +
+ tools/perf/util/pmus.c                             |  18 +-
+ tools/perf/util/pmus.h                             |   1 +
+ tools/perf/util/probe-finder.c                     | 102 ++-
+ tools/perf/util/s390-sample-raw.c                  |  31 +-
+ .../perf/util/scripting-engines/trace-event-perl.c |  23 +-
+ .../util/scripting-engines/trace-event-python.c    |  47 +-
+ tools/perf/util/session.c                          |  38 +-
+ tools/perf/util/srcline.c                          |   9 +-
+ tools/perf/util/srcline.h                          |   1 +
+ tools/perf/util/symbol_conf.h                      |  10 +
+ tools/perf/util/synthetic-events.c                 |  74 +-
+ tools/perf/util/synthetic-events.h                 |   8 +-
+ tools/perf/util/thread.c                           |  32 +-
+ tools/perf/util/tool.c                             |  10 +-
+ tools/perf/util/tool.h                             |   4 +-
+ tools/perf/util/trace-event-scripting.c            |   5 +-
+ tools/perf/util/trace-event.h                      |   3 -
+ tools/perf/util/unwind-libdw.c                     |  20 +-
+ tools/perf/util/unwind-libunwind-local.c           | 831 ---------------------
+ tools/perf/util/unwind-libunwind.c                 | 709 ++++++++++++++++--
+ tools/perf/util/unwind.c                           | 104 +++
+ tools/perf/util/unwind.h                           |  81 +-
+ 188 files changed, 7547 insertions(+), 3446 deletions(-)
+ delete mode 100644 tools/build/feature/test-libunwind-aarch64.c
+ delete mode 100644 tools/build/feature/test-libunwind-arm.c
+ delete mode 100644 tools/build/feature/test-libunwind-debug-frame-aarch64.c
+ delete mode 100644 tools/build/feature/test-libunwind-debug-frame-arm.c
+ delete mode 100644 tools/build/feature/test-libunwind-x86.c
+ delete mode 100644 tools/build/feature/test-libunwind-x86_64.c
+ create mode 100644 tools/perf/.clang-format
+ delete mode 100644 tools/perf/arch/arm/util/unwind-libunwind.c
+ delete mode 100644 tools/perf/arch/arm64/util/machine.c
+ delete mode 100644 tools/perf/arch/arm64/util/unwind-libunwind.c
+ delete mode 100644 tools/perf/arch/loongarch/util/unwind-libunwind.c
+ delete mode 100644 tools/perf/arch/mips/Build
+ delete mode 100644 tools/perf/arch/mips/util/Build
+ delete mode 100644 tools/perf/arch/mips/util/unwind-libunwind.c
+ delete mode 100644 tools/perf/arch/powerpc/util/unwind-libunwind.c
+ delete mode 100644 tools/perf/arch/x86/util/unwind-libunwind.c
+ rename tools/perf/{util => bench}/bpf_skel/bench_uprobe.bpf.c (100%)
+ create mode 100644 tools/perf/bpf_skel.mak
+ create mode 100644 tools/perf/tests/uncore-event-sorting.c
+ create mode 100644 tools/perf/trace/beauty/fsconfig.c
+ rename tools/perf/{util => trace/beauty}/syscalltbl.c (100%)
+ rename tools/perf/{util => trace/beauty}/syscalltbl.h (100%)
+ create mode 100644 tools/perf/util/libunwind-arch/Build
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-arch.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-arch.h
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-arm.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-arm64.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-i386.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-loongarch.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-mips.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-ppc32.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-ppc64.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-riscv.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-s390.c
+ create mode 100644 tools/perf/util/libunwind-arch/libunwind-x86_64.c
+ delete mode 100644 tools/perf/util/libunwind/arm64.c
+ delete mode 100644 tools/perf/util/libunwind/x86_32.c
+ delete mode 100644 tools/perf/util/unwind-libunwind-local.c
+ create mode 100644 tools/perf/util/unwind.c
+Merging compiler-attributes/compiler-attributes (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'compiler-attributes' of https://github.com/ojeda/linux.git compiler-attributes/compiler-attributes
+Already up to date.
+Merging dma-mapping/dma-mapping-for-next (bebaaca682b12 dma-contiguous: add kconfig option to setup numa cma area if not configured explicitly)
+$ git merge -m Merge branch 'dma-mapping-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mszyprowski/linux.git dma-mapping/dma-mapping-for-next
+Auto-merging kernel/dma/debug.c
+Auto-merging kernel/dma/mapping.c
+Merge made by the 'ort' strategy.
+ drivers/ntb/ntb_transport.c | 10 +++---
+ kernel/dma/Kconfig          | 10 ++++++
+ kernel/dma/contiguous.c     | 13 +++++--
+ kernel/dma/debug.c          | 87 +++++++++++++++++++++++++++++++--------------
+ kernel/dma/debug.h          | 25 ++++++-------
+ kernel/dma/mapping.c        | 10 +++---
+ 6 files changed, 106 insertions(+), 49 deletions(-)
+Merging asm-generic/master (adbbd9714f805 scripts: headers_install.sh: Remove config leak ignore machinery)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/arnd/asm-generic asm-generic/master
+Already up to date.
+Merging nocache-cleanup/nocache-cleanup (809b997a5ce94 x86-64/arm64/powerpc: clean up and rename __copy_from_user_flushcache)
+$ git merge -m Merge branch 'nocache-cleanup' of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git nocache-cleanup/nocache-cleanup
+Already up to date.
+Merging alpha/alpha-next (d58041d2c63e0 MAINTAINERS: Add Magnus Lindholm as maintainer for alpha port)
+$ git merge -m Merge branch 'alpha-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mattst88/alpha.git alpha/alpha-next
+Already up to date.
+Merging arm/for-next (c6e61c06d6061 ARM: 9463/1: Allow to enable RT)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rmk/linux.git arm/for-next
+Already up to date.
+Merging arm64/for-next/core (c754aa6b881ad Merge branches 'for-next/cpufeature', 'for-next/errata', 'for-next/misc', 'for-next/mm', 'for-next/mpam', 'for-next/perf', 'for-next/selftests' and 'for-next/sysregs' into for-next/core)
+$ git merge -m Merge branch 'for-next/core' of https://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux arm64/for-next/core
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/mm/mmu.c
+Auto-merging drivers/resctrl/mpam_devices.c
+Merge made by the 'ort' strategy.
+ Documentation/arch/arm64/elf_hwcaps.rst            |  24 +++++
+ Documentation/arch/arm64/silicon-errata.rst        |  47 +++++----
+ MAINTAINERS                                        |   2 +-
+ arch/arm64/include/asm/cpufeature.h                |   7 ++
+ arch/arm64/include/asm/daifflags.h                 |  10 +-
+ arch/arm64/include/asm/el2_setup.h                 |   4 +-
+ arch/arm64/include/asm/esr.h                       |   2 +
+ arch/arm64/include/asm/linkage.h                   |   2 +
+ arch/arm64/include/asm/pgtable.h                   |   4 +-
+ arch/arm64/include/uapi/asm/hwcap.h                |   8 ++
+ arch/arm64/kernel/cpufeature.c                     |  26 ++++-
+ arch/arm64/kernel/cpuinfo.c                        |   8 ++
+ arch/arm64/kernel/irq.c                            |  29 ++++--
+ arch/arm64/kernel/proton-pack.c                    |  17 +--
+ arch/arm64/kernel/smp.c                            |  14 ++-
+ arch/arm64/mm/mmu.c                                |  12 +--
+ arch/arm64/tools/sysreg                            |  29 ++++++
+ drivers/perf/Kconfig                               |   4 +-
+ drivers/resctrl/mpam_devices.c                     |  23 +++-
+ tools/testing/selftests/arm64/abi/hwcap.c          | 116 +++++++++++++++++++++
+ .../testing/selftests/arm64/signal/test_signals.h  |   2 +
+ .../selftests/arm64/signal/test_signals_utils.c    |   3 +
+ .../selftests/arm64/signal/test_signals_utils.h    |  16 +++
+ .../signal/testcases/poe_missing_poe_context.c     |  73 +++++++++++++
+ .../selftests/arm64/signal/testcases/poe_restore.c |  64 ++++++++++++
+ .../selftests/arm64/signal/testcases/poe_siginfo.c |  15 ---
+ tools/testing/selftests/mm/pkey-arm64.h            |   3 +-
+ 27 files changed, 481 insertions(+), 83 deletions(-)
+ create mode 100644 tools/testing/selftests/arm64/signal/testcases/poe_missing_poe_context.c
+ create mode 100644 tools/testing/selftests/arm64/signal/testcases/poe_restore.c
+Merging arm-perf/for-next/perf (3ef020a3c9ebd perf: qcom: Unify user-visible "Qualcomm" name)
+$ git merge -m Merge branch 'for-next/perf' of https://git.kernel.org/pub/scm/linux/kernel/git/will/linux.git arm-perf/for-next/perf
+Already up to date.
+Merging arm-soc/for-next (8dbb80525730a soc: document merges)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/soc/soc.git arm-soc/for-next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm/Kconfig
+Merge made by the 'ort' strategy.
+ Documentation/arch/arm/index.rst                   |   2 +
+ Documentation/arch/arm/zte/index.rst               |  10 +
+ Documentation/arch/arm/zte/zx297520v3.rst          | 167 ++++++++
+ Documentation/devicetree/bindings/arm/zte.yaml     |  26 ++
+ .../bindings/clock/renesas,cpg-clocks.yaml         |   8 +-
+ .../bindings/pinctrl/renesas,rzg2l-pinctrl.yaml    |  35 ++
+ .../soc/renesas/renesas,r8a78000-mfis.yaml         | 187 +++++++++
+ MAINTAINERS                                        |   9 +
+ arch/arm/Kconfig                                   |   2 +
+ arch/arm/Makefile                                  |   1 +
+ arch/arm/arm-soc-for-next-contents.txt             |  62 +++
+ arch/arm/boot/dts/Makefile                         |   1 +
+ .../intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi  |   6 +-
+ arch/arm/boot/dts/renesas/r8a73a4.dtsi             | 114 +++++-
+ arch/arm/boot/dts/renesas/r8a7740.dtsi             | 116 +++++-
+ arch/arm/boot/dts/zte/Makefile                     |   3 +
+ arch/arm/boot/dts/zte/zx297520v3-dlink-dwr932m.dts |  22 ++
+ arch/arm/boot/dts/zte/zx297520v3.dtsi              | 103 +++++
+ arch/arm/configs/at91_dt_defconfig                 |   1 -
+ arch/arm/configs/hisi_defconfig                    |   1 -
+ arch/arm/configs/multi_v5_defconfig                |   1 -
+ arch/arm/configs/multi_v7_defconfig                | 103 +++--
+ arch/arm/configs/pxa_defconfig                     |   2 -
+ arch/arm/configs/qcom_defconfig                    |  15 -
+ arch/arm/configs/sama5_defconfig                   |   1 -
+ arch/arm/configs/sama7_defconfig                   |   1 -
+ arch/arm/configs/socfpga_defconfig                 |   1 -
+ arch/arm/configs/spear13xx_defconfig               |   1 -
+ arch/arm/configs/spear3xx_defconfig                |   1 -
+ arch/arm/configs/spear6xx_defconfig                |   1 -
+ arch/arm/mach-mv78xx0/Kconfig                      |   1 +
+ arch/arm/mach-orion5x/Kconfig                      |   1 +
+ arch/arm/mach-pxa/Kconfig                          |   1 +
+ arch/arm/mach-pxa/pxa25x.c                         |   3 +
+ arch/arm/mach-pxa/pxa27x.c                         |   3 +
+ arch/arm/mach-pxa/spitz.c                          |  11 +-
+ arch/arm/mach-s3c/Kconfig.s3c64xx                  |   1 +
+ arch/arm/mach-sa1100/Kconfig                       |   1 +
+ arch/arm/mach-zte/Kconfig                          |  29 ++
+ arch/arm/mach-zte/Makefile                         |   2 +
+ arch/arm/mach-zte/zx297520v3.c                     |  16 +
+ arch/arm64/Kconfig.platforms                       |   1 -
+ .../marvell/mmp/pxa1908-samsung-coreprimevelte.dts |  11 +-
+ arch/arm64/boot/dts/marvell/mmp/pxa1908.dtsi       |   5 +-
+ arch/arm64/boot/dts/realtek/kent.dtsi              |  20 +
+ arch/arm64/boot/dts/renesas/ebisu.dtsi             |  34 +-
+ arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi  |  10 +-
+ .../arm64/boot/dts/renesas/r8a77960-salvator-x.dts |   4 +
+ .../boot/dts/renesas/r8a77960-salvator-xs.dts      |   4 +
+ arch/arm64/boot/dts/renesas/r8a77960-ulcb.dts      |   4 +
+ .../boot/dts/renesas/r8a77961-salvator-xs.dts      |   4 +
+ arch/arm64/boot/dts/renesas/r8a77961-ulcb.dts      |   4 +
+ arch/arm64/boot/dts/renesas/r9a07g043.dtsi         |   1 +
+ arch/arm64/boot/dts/renesas/r9a07g044.dtsi         |   1 +
+ arch/arm64/boot/dts/renesas/r9a07g054.dtsi         |   1 +
+ arch/arm64/boot/dts/renesas/r9a08g045.dtsi         |   1 +
+ arch/arm64/boot/dts/renesas/r9a08g046.dtsi         | 402 +++++++++++++++++++-
+ arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts |  13 +
+ arch/arm64/boot/dts/renesas/r9a09g047.dtsi         |   1 +
+ arch/arm64/boot/dts/renesas/r9a09g056.dtsi         |   1 +
+ arch/arm64/boot/dts/renesas/r9a09g057.dtsi         |   1 +
+ arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi   | 132 +++++++
+ arch/arm64/boot/dts/renesas/salvator-common.dtsi   |  26 +-
+ arch/arm64/configs/defconfig                       | 421 ++++++++++-----------
+ drivers/base/platform-msi.c                        |   4 -
+ drivers/bus/fsl-mc/dprc-driver.c                   |  14 +-
+ drivers/bus/fsl-mc/fsl-mc-bus.c                    |  50 ++-
+ drivers/bus/fsl-mc/fsl-mc-msi.c                    | 166 ++------
+ drivers/bus/fsl-mc/fsl-mc-private.h                |   2 +-
+ drivers/irqchip/Kconfig                            |   6 -
+ drivers/irqchip/Makefile                           |   1 -
+ drivers/irqchip/irq-gic-its-msi-parent.c           |   7 +-
+ drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c        | 168 --------
+ drivers/soc/fsl/qe/qe.c                            |   3 +
+ drivers/soc/fsl/qe/qe_ports_ic.c                   |  23 +-
+ drivers/soc/renesas/Kconfig                        |   9 +
+ drivers/soc/renesas/Makefile                       |   1 +
+ drivers/soc/renesas/r9a08g045-sysc.c               |  41 +-
+ drivers/soc/renesas/r9a08g046-sysc.c               |  31 +-
+ drivers/soc/renesas/r9a09g047-sys.c                |  42 +-
+ drivers/soc/renesas/r9a09g056-sys.c                |  41 +-
+ drivers/soc/renesas/r9a09g057-sys.c                |  52 +--
+ drivers/soc/renesas/rcar-mfis.c                    | 344 +++++++++++++++++
+ drivers/soc/renesas/renesas-soc.c                  |  10 +-
+ include/dt-bindings/clock/r8a73a4-clock.h          |   2 +
+ include/dt-bindings/clock/r8a7740-clock.h          |   2 +
+ .../pinctrl/renesas,r9a08g046-pinctrl.h            |  38 ++
+ include/dt-bindings/soc/renesas,r8a78000-mfis.h    |  28 ++
+ include/linux/fsl/mc.h                             |   6 +-
+ include/linux/irqdomain_defs.h                     |   1 -
+ include/uapi/linux/fsl_hypervisor.h                |   4 +-
+ 91 files changed, 2431 insertions(+), 839 deletions(-)
+ create mode 100644 Documentation/arch/arm/zte/index.rst
+ create mode 100644 Documentation/arch/arm/zte/zx297520v3.rst
+ create mode 100644 Documentation/devicetree/bindings/arm/zte.yaml
+ create mode 100644 Documentation/devicetree/bindings/soc/renesas/renesas,r8a78000-mfis.yaml
+ create mode 100644 arch/arm/arm-soc-for-next-contents.txt
+ create mode 100644 arch/arm/boot/dts/zte/Makefile
+ create mode 100644 arch/arm/boot/dts/zte/zx297520v3-dlink-dwr932m.dts
+ create mode 100644 arch/arm/boot/dts/zte/zx297520v3.dtsi
+ create mode 100644 arch/arm/mach-zte/Kconfig
+ create mode 100644 arch/arm/mach-zte/Makefile
+ create mode 100644 arch/arm/mach-zte/zx297520v3.c
+ delete mode 100644 drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c
+ create mode 100644 drivers/soc/renesas/rcar-mfis.c
+ create mode 100644 include/dt-bindings/pinctrl/renesas,r9a08g046-pinctrl.h
+ create mode 100644 include/dt-bindings/soc/renesas,r8a78000-mfis.h
+Merging amlogic/for-next (3878c0539707a Merge branch 'v7.2/arm64-dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/amlogic/linux.git amlogic/for-next
+Merge made by the 'ort' strategy.
+ .../dts/amlogic/amlogic-t7-a311d2-khadas-vim4.dts  | 215 +++++++++
+ arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi        | 482 +++++++++++++++++++++
+ .../dts/amlogic/meson-gxl-s905d-phicomm-n1.dts     |  15 +
+ .../dts/amlogic/meson-s4-s905y4-khadas-vim1s.dts   |  81 ++++
+ arch/arm64/boot/dts/amlogic/meson-s4.dtsi          |  45 ++
+ 5 files changed, 838 insertions(+)
+Merging asahi-soc/asahi-soc/for-next (d31647c698f63 Merge branch 'apple-soc/drivers-6.20' into asahi-soc/for-next)
+$ git merge -m Merge branch 'asahi-soc/for-next' of https://github.com/AsahiLinux/linux.git asahi-soc/asahi-soc/for-next
+Merge made by the 'ort' strategy.
+Merging at91/at91-next (9a6b14b29aa2a Merge branch 'at91-fixes' into at91-next)
+$ git merge -m Merge branch 'at91-next' of https://git.kernel.org/pub/scm/linux/kernel/git/at91/linux.git at91/at91-next
+Merge made by the 'ort' strategy.
+ drivers/clk/at91/sam9x7.c                         | 21 ++++++++++-----------
+ drivers/clk/microchip/Makefile                    |  2 +-
+ drivers/clk/microchip/{clk-core.c => clk-pic32.c} |  2 +-
+ drivers/clk/microchip/{clk-core.h => clk-pic32.h} |  0
+ drivers/clk/microchip/clk-pic32mzda.c             |  2 +-
+ 5 files changed, 13 insertions(+), 14 deletions(-)
+ rename drivers/clk/microchip/{clk-core.c => clk-pic32.c} (99%)
+ rename drivers/clk/microchip/{clk-core.h => clk-pic32.h} (100%)
+Merging bmc/for-next (66d729c23ec55 Merge branches 'aspeed/drivers', 'aspeed/arm/dt', 'aspeed/fixes/drivers' and 'nuvoton/arm64/dt' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bmc/linux.git bmc/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/arm/aspeed/aspeed.yaml     |   1 +
+ arch/arm/boot/dts/aspeed/Makefile                  |   1 +
+ .../dts/aspeed/aspeed-bmc-facebook-anacapa.dts     |  34 +-
+ .../dts/aspeed/aspeed-bmc-facebook-sanmiguel.dts   | 668 +++++++++++++++++++++
+ .../dts/aspeed/aspeed-bmc-facebook-yosemite5.dts   |   5 +
+ .../boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts |  15 +
+ arch/arm/boot/dts/aspeed/aspeed-g6.dtsi            | 552 ++++++++++-------
+ arch/arm64/boot/dts/nuvoton/ma35d1-iot-512m.dts    |  25 +
+ arch/arm64/boot/dts/nuvoton/ma35d1-som-256m.dts    |  24 +
+ arch/arm64/boot/dts/nuvoton/ma35d1.dtsi            |  52 ++
+ drivers/soc/aspeed/Kconfig                         |   1 -
+ drivers/soc/aspeed/aspeed-lpc-ctrl.c               |   2 +-
+ drivers/soc/aspeed/aspeed-lpc-snoop.c              |   2 +-
+ drivers/soc/aspeed/aspeed-p2a-ctrl.c               |   2 +-
+ 14 files changed, 1142 insertions(+), 242 deletions(-)
+ create mode 100644 arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-sanmiguel.dts
+Merging broadcom/next (e8f908b91550d firmware: raspberrypi: Change dependency to ARCH_BCM2835 and COMPILE_TEST)
+$ git merge -m Merge branch 'next' of https://github.com/Broadcom/stblinux.git broadcom/next
+Merge made by the 'ort' strategy.
+ drivers/firmware/Kconfig         |  5 +-
+ drivers/soc/bcm/brcmstb/common.c | 98 +++++++++++++++++++++-------------------
+ 2 files changed, 55 insertions(+), 48 deletions(-)
+Merging cix/for-next (7fd2df204f342 Linux 7.1-rc2)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix.git cix/for-next
+Already up to date.
+Merging davinci/davinci/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'davinci/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git davinci/davinci/for-next
+Already up to date.
+Merging drivers-memory/for-next (0a783a6686c96 memory: tegra114-emc: Simplify tegra114_emc_interconnect_init() error message)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-mem-ctrl.git drivers-memory/for-next
+Merge made by the 'ort' strategy.
+ .../memory-controllers/nvidia,tegra124-emc.yaml    |  174 +--
+ .../memory-controllers/nvidia,tegra124-mc.yaml     |   31 +-
+ .../memory-controllers/nvidia,tegra186-mc.yaml     |   31 +
+ drivers/memory/atmel-ebi.c                         |    3 +-
+ drivers/memory/tegra/Kconfig                       |   19 +
+ drivers/memory/tegra/Makefile                      |    3 +
+ drivers/memory/tegra/mc.c                          |   40 +-
+ drivers/memory/tegra/mc.h                          |    9 +
+ drivers/memory/tegra/tegra-emc-common.c            |  143 +++
+ drivers/memory/tegra/tegra-emc-common.h            |   46 +
+ drivers/memory/tegra/tegra114-emc.c                | 1351 ++++++++++++++++++++
+ drivers/memory/tegra/tegra114.c                    |  193 +++
+ drivers/memory/tegra/tegra124-emc.c                |  107 +-
+ drivers/memory/tegra/tegra186.c                    |    4 +-
+ drivers/memory/tegra/tegra20-emc.c                 |  110 +-
+ drivers/memory/tegra/tegra234.c                    |    2 +-
+ drivers/memory/tegra/tegra238.c                    |  391 ++++++
+ drivers/memory/tegra/tegra30-emc.c                 |  107 +-
+ include/dt-bindings/memory/nvidia,tegra238-mc.h    |   74 ++
+ include/dt-bindings/memory/tegra114-mc.h           |   67 +
+ include/soc/tegra/mc.h                             |    2 +-
+ 21 files changed, 2418 insertions(+), 489 deletions(-)
+ create mode 100644 drivers/memory/tegra/tegra-emc-common.c
+ create mode 100644 drivers/memory/tegra/tegra-emc-common.h
+ create mode 100644 drivers/memory/tegra/tegra114-emc.c
+ create mode 100644 drivers/memory/tegra/tegra238.c
+ create mode 100644 include/dt-bindings/memory/nvidia,tegra238-mc.h
+Merging fsl/soc_fsl (c99d8ac7b355c bus: fsl-mc: wait for the MC firmware to complete its boot)
+$ git merge -m Merge branch 'soc_fsl' of https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux.git fsl/soc_fsl
+Merge made by the 'ort' strategy.
+Merging imx-mxs/for-next (b3c1d1631f097 Merge branches 'imx/bindings', 'imx/dt', 'imx/dt64', 'imx/firmware-ele', 'imx/fixes' and 'imx/soc' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/frank.li/linux.git imx-mxs/for-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/se-cdev                  |   44 +
+ Documentation/devicetree/bindings/arm/fsl.yaml     |   28 +-
+ .../devicetree/bindings/bus/fsl,spba-bus.yaml      |    4 +
+ .../devicetree/bindings/firmware/fsl,imx-se.yaml   |   91 ++
+ .../driver-api/firmware/other_interfaces.rst       |  133 +++
+ arch/arm/boot/dts/nxp/imx/Makefile                 |    3 +
+ arch/arm/boot/dts/nxp/imx/imx25.dtsi               |    1 -
+ arch/arm/boot/dts/nxp/imx/imx35.dtsi               |    1 -
+ .../arm/boot/dts/nxp/imx/imx6ul-var-som-audio.dtsi |   30 +
+ .../boot/dts/nxp/imx/imx6ul-var-som-common.dtsi    |  187 +++
+ .../nxp/imx/imx6ul-var-som-concerto-common.dtsi    |  215 ++++
+ .../dts/nxp/imx/imx6ul-var-som-concerto-full.dts   |   22 +
+ .../boot/dts/nxp/imx/imx6ul-var-som-concerto.dts   |  318 +-----
+ .../arm/boot/dts/nxp/imx/imx6ul-var-som-enet1.dtsi |   44 +
+ .../arm/boot/dts/nxp/imx/imx6ul-var-som-enet2.dtsi |   79 ++
+ .../dts/nxp/imx/imx6ul-var-som-lvds-panel.dtsi     |  112 ++
+ arch/arm/boot/dts/nxp/imx/imx6ul-var-som-sd.dtsi   |   27 +
+ arch/arm/boot/dts/nxp/imx/imx6ul-var-som-wifi.dtsi |   75 ++
+ arch/arm/boot/dts/nxp/imx/imx6ul-var-som.dtsi      |  219 +---
+ arch/arm/boot/dts/nxp/imx/imx6ul.dtsi              |    1 +
+ .../dts/nxp/imx/imx6ull-var-som-concerto-full.dts  |   22 +
+ .../boot/dts/nxp/imx/imx6ull-var-som-concerto.dts  |   21 +
+ arch/arm/boot/dts/nxp/imx/imx6ull-var-som.dtsi     |   36 +
+ arch/arm64/boot/dts/freescale/Makefile             |   32 +-
+ arch/arm64/boot/dts/freescale/imx-m2-pcie.dtso     |   15 +
+ arch/arm64/boot/dts/freescale/imx8dxl-evk.dts      |    2 -
+ .../arm64/boot/dts/freescale/imx8mm-kontron-bl.dts |   10 +-
+ .../boot/dts/freescale/imx8mm-kontron-osm-s.dtsi   |    7 +-
+ .../dts/freescale/imx8mm-verdin-nonwifi-zinnia.dts |   21 +
+ .../dts/freescale/imx8mm-verdin-wifi-zinnia.dts    |   21 +
+ .../boot/dts/freescale/imx8mm-verdin-zinnia.dtsi   |  383 +++++++
+ arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi   |   16 +-
+ .../dts/freescale/imx8mn-vhip4-evalboard-v1.dts    |    2 +-
+ .../dts/freescale/imx8mn-vhip4-evalboard-v2.dts    |    2 +-
+ arch/arm64/boot/dts/freescale/imx8mp-ab2.dts       |    2 +-
+ arch/arm64/boot/dts/freescale/imx8mp-evk.dts       |    4 +-
+ .../boot/dts/freescale/imx8mp-kontron-osm-s.dtsi   |    7 +-
+ .../dts/freescale/imx8mp-phyboard-pollux-rdk.dts   |   62 +
+ ...x8p-ml81-moduline-display-106-av101hdt-a10.dtso |   24 +
+ ...x8p-ml81-moduline-display-106-av123z7m-n17.dtso |   19 +-
+ .../dts/freescale/imx8mp-verdin-nonwifi-zinnia.dts |   21 +
+ .../dts/freescale/imx8mp-verdin-wifi-zinnia.dts    |   21 +
+ .../boot/dts/freescale/imx8mp-verdin-zinnia.dtsi   |  422 +++++++
+ arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi   |   14 +-
+ .../boot/dts/freescale/imx8qm-tqma8qm-mba8x.dts    |  871 ++++++++++++++
+ arch/arm64/boot/dts/freescale/imx8qm-tqma8qm.dtsi  |  325 ++++++
+ arch/arm64/boot/dts/freescale/imx8qxp-mek.dts      |    1 -
+ arch/arm64/boot/dts/freescale/imx8ulp-evk.dts      |    4 +-
+ arch/arm64/boot/dts/freescale/imx8ulp.dtsi         |   66 ++
+ arch/arm64/boot/dts/freescale/imx8x-colibri.dtsi   |    4 +-
+ arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts  |    1 -
+ arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts    |  425 +++++++
+ .../boot/dts/freescale/imx91-phyboard-segin.dts    |   24 +-
+ .../boot/dts/freescale/imx91-phycore-som.dtsi      |   39 +-
+ .../boot/dts/freescale/imx91-var-dart-sonata.dts   |    9 +
+ .../boot/dts/freescale/imx91-var-som-symphony.dts  |  527 +++++++++
+ arch/arm64/boot/dts/freescale/imx91-var-som.dtsi   |  456 ++++++++
+ .../boot/dts/freescale/imx93-11x11-evk-common.dtsi |    1 -
+ .../imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtsi       |  110 ++
+ .../imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso       |  106 +-
+ .../freescale/imx93-9x9-qsb-tianma-tm050rdh03.dtso |   14 +
+ arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts    |    1 -
+ .../boot/dts/freescale/imx93-phyboard-nash.dts     |   39 +-
+ .../boot/dts/freescale/imx93-phyboard-segin.dts    |   24 +-
+ .../boot/dts/freescale/imx93-phycore-som.dtsi      |   39 +-
+ .../boot/dts/freescale/imx93-var-dart-sonata.dts   |  654 +++++++++++
+ arch/arm64/boot/dts/freescale/imx93-var-dart.dtsi  |  461 ++++++++
+ arch/arm64/boot/dts/freescale/imx94.dtsi           |   88 ++
+ arch/arm64/boot/dts/freescale/imx943-evk.dts       |   83 ++
+ arch/arm64/boot/dts/freescale/imx943.dtsi          |   75 ++
+ arch/arm64/boot/dts/freescale/imx95-15x15-ab2.dts  |  669 +++++++++++
+ arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts  |    4 +-
+ arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts  |    2 +-
+ .../boot/dts/freescale/imx95-toradex-smarc-dev.dts |    5 +
+ .../boot/dts/freescale/imx95-toradex-smarc.dtsi    |   46 +-
+ .../arm64/boot/dts/freescale/imx95-verdin-ivy.dtsi |    1 -
+ .../dts/freescale/imx95-verdin-nonwifi-zinnia.dts  |   21 +
+ .../dts/freescale/imx95-verdin-wifi-zinnia.dts     |   21 +
+ .../boot/dts/freescale/imx95-verdin-zinnia.dtsi    |  429 +++++++
+ arch/arm64/boot/dts/freescale/imx95-verdin.dtsi    |   18 +-
+ arch/arm64/boot/dts/freescale/imx95.dtsi           |   86 +-
+ arch/arm64/boot/dts/freescale/imx952.dtsi          |   36 +
+ drivers/firmware/imx/Kconfig                       |   13 +
+ drivers/firmware/imx/Makefile                      |    2 +
+ drivers/firmware/imx/ele_base_msg.c                |  298 +++++
+ drivers/firmware/imx/ele_base_msg.h                |  117 ++
+ drivers/firmware/imx/ele_common.c                  |  373 ++++++
+ drivers/firmware/imx/ele_common.h                  |   51 +
+ drivers/firmware/imx/se_ctrl.c                     | 1193 ++++++++++++++++++++
+ drivers/firmware/imx/se_ctrl.h                     |  129 +++
+ drivers/firmware/imx/sm-misc.c                     |    2 +-
+ drivers/soc/imx/soc-imx8m.c                        |    2 +-
+ include/linux/firmware/imx/se_api.h                |   14 +
+ include/uapi/linux/se_ioctl.h                      |   97 ++
+ 94 files changed, 10193 insertions(+), 703 deletions(-)
+ create mode 100644 Documentation/ABI/testing/se-cdev
+ create mode 100644 Documentation/devicetree/bindings/firmware/fsl,imx-se.yaml
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ul-var-som-audio.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ul-var-som-common.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto-common.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto-full.dts
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ul-var-som-enet1.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ul-var-som-enet2.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ul-var-som-lvds-panel.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ul-var-som-sd.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ul-var-som-wifi.dtsi
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ull-var-som-concerto-full.dts
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ull-var-som-concerto.dts
+ create mode 100644 arch/arm/boot/dts/nxp/imx/imx6ull-var-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx-m2-pcie.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-verdin-nonwifi-zinnia.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-verdin-wifi-zinnia.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mm-verdin-zinnia.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-verdin-nonwifi-zinnia.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-verdin-wifi-zinnia.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8mp-verdin-zinnia.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8qm-tqma8qm-mba8x.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx8qm-tqma8qm.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-var-som-symphony.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx91-var-som.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-9x9-qsb-tianma-tm050rdh03.dtso
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-var-dart-sonata.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx93-var-dart.dtsi
+ create mode 100644 arch/arm64/boot/dts/freescale/imx95-15x15-ab2.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx95-verdin-nonwifi-zinnia.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx95-verdin-wifi-zinnia.dts
+ create mode 100644 arch/arm64/boot/dts/freescale/imx95-verdin-zinnia.dtsi
+ create mode 100644 drivers/firmware/imx/ele_base_msg.c
+ create mode 100644 drivers/firmware/imx/ele_base_msg.h
+ create mode 100644 drivers/firmware/imx/ele_common.c
+ create mode 100644 drivers/firmware/imx/ele_common.h
+ create mode 100644 drivers/firmware/imx/se_ctrl.c
+ create mode 100644 drivers/firmware/imx/se_ctrl.h
+ create mode 100644 include/linux/firmware/imx/se_api.h
+ create mode 100644 include/uapi/linux/se_ioctl.h
+Merging mediatek/for-next (12e53eb041a53 Merge branch 'v7.1-next/dts64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mediatek/linux.git mediatek/for-next
+Auto-merging arch/arm64/configs/defconfig
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/mediatek/mt6589.dtsi             |  11 ++
+ arch/arm/boot/dts/mediatek/mt7623.dtsi             |  64 ++++-----
+ arch/arm/boot/dts/mediatek/mt8135.dtsi             |   2 +-
+ arch/arm64/boot/dts/mediatek/Makefile              |   8 ++
+ arch/arm64/boot/dts/mediatek/mt6359.dtsi           |  22 +--
+ arch/arm64/boot/dts/mediatek/mt6365.dtsi           |  26 ++++
+ arch/arm64/boot/dts/mediatek/mt7981b.dtsi          |  15 ++
+ .../mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso |  20 +++
+ .../mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso |  20 +++
+ .../dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi  |  15 +-
+ .../boot/dts/mediatek/mt8186-corsola-voltorb.dtsi  |   4 +
+ arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi    |  41 ++++++
+ arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi   |  82 ++++++++++-
+ arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi    | 154 +++++++++++++--------
+ .../boot/dts/mediatek/mt8390-genio-700-evk.dts     |   7 +
+ .../boot/dts/mediatek/mt8390-genio-common.dtsi     |  80 ++++++++++-
+ .../boot/dts/mediatek/mt8390-grinn-genio-som.dtsi  |   2 +-
+ .../boot/dts/mediatek/mt8390-tungsten-smarc.dtsi   | 122 +++++++++++++++-
+ .../boot/dts/mediatek/mt8395-genio-common.dtsi     |  46 ++++--
+ .../mediatek/mt8395-radxa-nio-12l-8-hd-panel.dtso  |   4 +-
+ .../boot/dts/mediatek/mt8395-radxa-nio-12l.dts     |   8 +-
+ arch/arm64/configs/defconfig                       |   2 +
+ drivers/soc/mediatek/mt8167-mmsys.h                |  11 +-
+ drivers/soc/mediatek/mtk-mmsys.c                   |   8 +-
+ drivers/soc/mediatek/mtk-mutex.c                   |   5 +-
+ include/linux/soc/mediatek/mtk-cmdq.h              |  24 ++++
+ 26 files changed, 645 insertions(+), 158 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/mediatek/mt6365.dtsi
+ create mode 100644 arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso
+ create mode 100644 arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso
+Merging mvebu/for-next (ae4fe907b7705 Merge branch 'mvebu/dt64' into mvebu/for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gclement/mvebu.git mvebu/for-next
+Merge made by the 'ort' strategy.
+Merging omap/for-next (e28bf943a8c77 Merge branch 'omap-for-v7.2/soc' into tmp/omap-next-20260511.095135)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/khilman/linux-omap.git omap/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/ti/omap/am335x-sl50.dts          |  4 +--
+ arch/arm/boot/dts/ti/omap/dm8168-evm.dts           |  4 +++
+ .../boot/dts/ti/omap/motorola-cpcap-mapphone.dtsi  |  6 ++++-
+ arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts  |  4 +--
+ arch/arm/mach-omap2/omap_device.c                  | 29 ++++++++--------------
+ arch/arm/mach-omap2/omap_device.h                  |  4 +--
+ 6 files changed, 25 insertions(+), 26 deletions(-)
+Merging qcom/for-next (a83b64da812b6 Merge branches 'arm64-defconfig-fixes-for-7.1', 'arm64-fixes-for-7.1', 'arm64-for-7.2', 'clk-fixes-for-7.1', 'clk-for-7.2', 'drivers-fixes-for-7.1' and 'drivers-for-7.2' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/qcom/linux.git qcom/for-next
+Auto-merging arch/arm64/configs/defconfig
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/qcom.yaml    |   21 +-
+ .../devicetree/bindings/cache/qcom,llcc.yaml       |   21 +
+ .../bindings/clock/qcom,dispcc-sm6125.yaml         |   17 +-
+ .../devicetree/bindings/clock/qcom,hawi-gcc.yaml   |   63 +
+ .../bindings/clock/qcom,ipq9574-cmn-pll.yaml       |    1 +
+ .../bindings/clock/qcom,ipq9650-gcc.yaml           |   68 +
+ .../bindings/clock/qcom,kaanapali-gxclkctl.yaml    |    2 +-
+ .../bindings/clock/qcom,milos-gxclkctl.yaml        |   61 +
+ .../devicetree/bindings/clock/qcom,rpmhcc.yaml     |    1 +
+ .../bindings/clock/qcom,sm8550-tcsr.yaml           |    2 +
+ .../bindings/crypto/qcom,inline-crypto-engine.yaml |   35 +-
+ .../devicetree/bindings/firmware/qcom,scm.yaml     |    3 +
+ .../bindings/net/bluetooth/qcom,wcn6855-bt.yaml    |    7 +-
+ .../bindings/soc/qcom/qcom,aoss-qmp.yaml           |    2 +
+ .../bindings/soc/qcom/qcom,pmic-glink.yaml         |    6 +
+ .../soc/qcom/qcom,sa8255p-geni-se-qup.yaml         |   31 +-
+ arch/arm64/boot/dts/qcom/Makefile                  |    7 +
+ arch/arm64/boot/dts/qcom/agatti.dtsi               |   63 +
+ arch/arm64/boot/dts/qcom/eliza.dtsi                |   32 +
+ arch/arm64/boot/dts/qcom/glymur-crd.dts            |    4 -
+ arch/arm64/boot/dts/qcom/glymur-crd.dtsi           |   33 +-
+ arch/arm64/boot/dts/qcom/glymur.dtsi               |  439 ++-
+ arch/arm64/boot/dts/qcom/hamoa.dtsi                |   77 +-
+ arch/arm64/boot/dts/qcom/ipq5424.dtsi              |    4 +-
+ arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts        |   79 +
+ arch/arm64/boot/dts/qcom/ipq9650.dtsi              |  384 ++
+ arch/arm64/boot/dts/qcom/kaanapali-mtp.dts         |   27 +
+ arch/arm64/boot/dts/qcom/kaanapali-qrd.dts         |   27 +
+ arch/arm64/boot/dts/qcom/kaanapali.dtsi            |  162 +-
+ arch/arm64/boot/dts/qcom/kodiak.dtsi               |   11 +-
+ .../boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso    |   74 +
+ arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi   |  110 +
+ arch/arm64/boot/dts/qcom/lemans.dtsi               |  981 ++++--
+ arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts   |   31 +-
+ .../boot/dts/qcom/milos-nothing-asteroids.dts      |  964 ++++++
+ arch/arm64/boot/dts/qcom/milos.dtsi                |   34 +-
+ arch/arm64/boot/dts/qcom/monaco.dtsi               |   29 +
+ arch/arm64/boot/dts/qcom/msm8998.dtsi              |    8 +-
+ arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi       |    2 +-
+ arch/arm64/boot/dts/qcom/qcs615-ride.dts           |   14 +-
+ arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts       |   22 +
+ arch/arm64/boot/dts/qcom/sc7180.dtsi               |    9 +-
+ arch/arm64/boot/dts/qcom/sc8180x.dtsi              |    2 +-
+ arch/arm64/boot/dts/qcom/sc8280xp-crd.dts          |    2 +-
+ .../boot/dts/qcom/sc8280xp-huawei-gaokun3.dts      |    2 +-
+ .../dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts     |    2 +-
+ .../boot/dts/qcom/sc8280xp-microsoft-blackrock.dts |    2 +-
+ arch/arm64/boot/dts/qcom/sc8280xp.dtsi             |   27 +
+ arch/arm64/boot/dts/qcom/sdm630.dtsi               |   40 +-
+ arch/arm64/boot/dts/qcom/sdm660.dtsi               |   36 +-
+ arch/arm64/boot/dts/qcom/sdm670-google-sargo.dts   |  100 +
+ arch/arm64/boot/dts/qcom/sdm670.dtsi               |  119 +-
+ .../qcom/sdm845-db845c-navigation-mezzanine.dtso   |    5 +
+ arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi |   31 +
+ arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi     |   22 +-
+ arch/arm64/boot/dts/qcom/sdm845-lg-judyln.dts      |    4 +-
+ arch/arm64/boot/dts/qcom/sdm845-lg-judyp.dts       |    4 +-
+ .../arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi |   79 +-
+ .../boot/dts/qcom/sdm845-samsung-starqltechn.dts   |    7 +-
+ arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts  |  127 +-
+ .../dts/qcom/sdm845-xiaomi-beryllium-common.dtsi   |   17 +
+ .../boot/dts/qcom/sdm845-xiaomi-beryllium-ebbg.dts |    2 +-
+ .../dts/qcom/sdm845-xiaomi-beryllium-tianma.dts    |    2 +-
+ .../boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts  |   94 +
+ arch/arm64/boot/dts/qcom/sm6350.dtsi               |   66 +
+ arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts | 1456 ++++++++
+ arch/arm64/boot/dts/qcom/sm8450-hdk.dts            |    2 +-
+ arch/arm64/boot/dts/qcom/sm8450.dtsi               |    2 +-
+ arch/arm64/boot/dts/qcom/sm8550.dtsi               |  367 ++
+ arch/arm64/boot/dts/qcom/sm8750.dtsi               |   54 +-
+ arch/arm64/boot/dts/qcom/talos-evk.dts             |   56 +
+ arch/arm64/boot/dts/qcom/talos.dtsi                |  134 +
+ arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi        |    7 +-
+ arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi   |    2 +-
+ .../boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi   |    2 +-
+ arch/arm64/configs/defconfig                       |    1 +
+ drivers/clk/qcom/Kconfig                           |   26 +
+ drivers/clk/qcom/Makefile                          |    5 +-
+ drivers/clk/qcom/clk-alpha-pll.h                   |    6 +
+ drivers/clk/qcom/clk-rpmh.c                        |   41 +-
+ drivers/clk/qcom/dispcc-x1e80100.c                 |    4 +-
+ drivers/clk/qcom/gcc-hawi.c                        | 3657 ++++++++++++++++++++
+ drivers/clk/qcom/gcc-ipq9650.c                     | 3445 ++++++++++++++++++
+ drivers/clk/qcom/gcc-nord.c                        |    2 +-
+ drivers/clk/qcom/gpucc-sm8750.c                    |    4 +-
+ drivers/clk/qcom/gxclkctl-kaanapali.c              |    1 +
+ drivers/clk/qcom/ipq-cmn-pll.c                     |   30 +-
+ drivers/clk/qcom/negcc-nord.c                      |    2 +-
+ drivers/clk/qcom/nwgcc-nord.c                      |    4 +-
+ drivers/clk/qcom/segcc-nord.c                      |    2 +-
+ drivers/clk/qcom/tcsrcc-hawi.c                     |  158 +
+ drivers/firmware/qcom/qcom_scm.c                   |    1 +
+ drivers/mmc/host/sdhci-msm.c                       |   12 +-
+ drivers/soc/qcom/ice.c                             |   66 +-
+ drivers/soc/qcom/llcc-qcom.c                       |  305 +-
+ drivers/soc/qcom/qcom_pd_mapper.c                  |    8 +
+ drivers/soc/qcom/smem.c                            |   56 +-
+ drivers/soc/qcom/socinfo.c                         |   23 +
+ drivers/soc/qcom/wcnss_ctrl.c                      |   11 +-
+ drivers/ufs/host/ufs-qcom.c                        |   12 +-
+ include/dt-bindings/arm/qcom,ids.h                 |   11 +
+ include/dt-bindings/clock/qcom,hawi-gcc.h          |  253 ++
+ include/dt-bindings/clock/qcom,hawi-tcsrcc.h       |   16 +
+ include/dt-bindings/clock/qcom,ipq5332-cmn-pll.h   |   19 +
+ include/dt-bindings/clock/qcom,ipq9650-gcc.h       |  172 +
+ include/dt-bindings/clock/qcom,rpmh.h              |    2 +
+ include/dt-bindings/firmware/qcom,scm.h            |    1 +
+ include/dt-bindings/reset/qcom,ipq9650-gcc.h       |  215 ++
+ include/linux/soc/qcom/llcc-qcom.h                 |   44 +-
+ include/soc/qcom/qcom-spmi-pmic.h                  |   15 +-
+ 110 files changed, 14699 insertions(+), 773 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,hawi-gcc.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,ipq9650-gcc.yaml
+ create mode 100644 Documentation/devicetree/bindings/clock/qcom,milos-gxclkctl.yaml
+ create mode 100644 arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/ipq9650.dtsi
+ create mode 100644 arch/arm64/boot/dts/qcom/milos-nothing-asteroids.dts
+ create mode 100644 arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts
+ create mode 100644 drivers/clk/qcom/gcc-hawi.c
+ create mode 100644 drivers/clk/qcom/gcc-ipq9650.c
+ create mode 100644 drivers/clk/qcom/tcsrcc-hawi.c
+ create mode 100644 include/dt-bindings/clock/qcom,hawi-gcc.h
+ create mode 100644 include/dt-bindings/clock/qcom,hawi-tcsrcc.h
+ create mode 100644 include/dt-bindings/clock/qcom,ipq5332-cmn-pll.h
+ create mode 100644 include/dt-bindings/clock/qcom,ipq9650-gcc.h
+ create mode 100644 include/dt-bindings/reset/qcom,ipq9650-gcc.h
+Merging renesas/next (1dbfc4ee3e482 Merge branch 'renesas-dts-for-v7.2' into renesas-next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel.git renesas/next
+Merge made by the 'ort' strategy.
+Merging reset/reset/next (d373605cd5148 Merge tag 'reset-fixes-for-v7.0-2' into reset/next)
+$ git merge -m Merge branch 'reset/next' of https://git.pengutronix.de/git/pza/linux reset/reset/next
+Already up to date.
+Merging rockchip/for-next (2e030c87599d1 Merge branch 'v7.2-clk/next' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mmind/linux-rockchip.git rockchip/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                                        |   2 +
+ .../dts/rockchip/rk3288-veyron-analog-audio.dtsi   |   2 -
+ arch/arm/boot/dts/rockchip/rk3288-veyron.dtsi      |   5 +
+ arch/arm64/boot/dts/rockchip/px30-cobra.dtsi       |   2 +-
+ arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi    |   2 +-
+ arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi       |   5 +
+ arch/arm64/boot/dts/rockchip/rk3528.dtsi           |   8 +
+ arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts     |   2 +-
+ arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dts |   6 +-
+ arch/arm64/boot/dts/rockchip/rk3568-rock-3b.dts    |  20 +-
+ .../boot/dts/rockchip/rk3576-100ask-dshanpi-a1.dts |  10 +
+ .../boot/dts/rockchip/rk3576-armsom-sige5.dts      |  10 +
+ arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts   |   9 +
+ arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts   |   9 +
+ .../boot/dts/rockchip/rk3576-khadas-edge-2l.dts    | 538 +++++++++++++++++++++
+ .../boot/dts/rockchip/rk3576-luckfox-core3576.dtsi |  22 +-
+ arch/arm64/boot/dts/rockchip/rk3576-nanopi-m5.dts  |   9 +
+ .../arm64/boot/dts/rockchip/rk3576-nanopi-r76s.dts |  16 +-
+ arch/arm64/boot/dts/rockchip/rk3576-roc-pc.dts     |  10 +
+ arch/arm64/boot/dts/rockchip/rk3576-rock-4d.dts    |  10 +
+ arch/arm64/boot/dts/rockchip/rk3576.dtsi           |   6 +-
+ .../boot/dts/rockchip/rk3588-armsom-sige7.dts      |  11 +
+ arch/arm64/boot/dts/rockchip/rk3588-armsom-w3.dts  |  19 +
+ arch/arm64/boot/dts/rockchip/rk3588-base.dtsi      |  64 ++-
+ .../boot/dts/rockchip/rk3588-coolpi-cm5-evb.dts    |  18 +-
+ .../dts/rockchip/rk3588-coolpi-cm5-genbook.dts     |  12 +-
+ .../arm64/boot/dts/rockchip/rk3588-coolpi-cm5.dtsi |   4 +
+ .../boot/dts/rockchip/rk3588-edgeble-neu6a-io.dtsi |   1 +
+ arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts   |  20 +
+ arch/arm64/boot/dts/rockchip/rk3588-evb2-v10.dts   |  11 +
+ arch/arm64/boot/dts/rockchip/rk3588-fet3588-c.dtsi |   1 +
+ .../dts/rockchip/rk3588-firefly-core-3588j.dtsi    |   1 +
+ .../dts/rockchip/rk3588-firefly-icore-3588q.dtsi   |   1 +
+ .../boot/dts/rockchip/rk3588-firefly-itx-3588j.dts |  12 +
+ .../rockchip/rk3588-friendlyelec-cm3588-nas.dts    |  18 +
+ .../arm64/boot/dts/rockchip/rk3588-h96-max-v58.dts |  11 +
+ arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts     |  24 +-
+ .../arm64/boot/dts/rockchip/rk3588-mnt-reform2.dts |  12 +
+ arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi |  21 +
+ .../boot/dts/rockchip/rk3588-orangepi-5-max.dts    |  18 +-
+ .../boot/dts/rockchip/rk3588-orangepi-5-plus.dts   |  20 +
+ .../boot/dts/rockchip/rk3588-orangepi-5-ultra.dts  |  11 +-
+ .../arm64/boot/dts/rockchip/rk3588-orangepi-5.dtsi |   1 +
+ arch/arm64/boot/dts/rockchip/rk3588-roc-rt.dts     |  19 +
+ arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts |  47 +-
+ .../boot/dts/rockchip/rk3588-rock-5b-5bp-5t.dtsi   |  19 +-
+ .../boot/dts/rockchip/rk3588-tiger-haikou.dts      |   4 +-
+ arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi     |  20 +-
+ .../arm64/boot/dts/rockchip/rk3588-toybrick-x0.dts |   1 +
+ .../arm64/boot/dts/rockchip/rk3588-turing-rk1.dtsi |   2 +
+ arch/arm64/boot/dts/rockchip/rk3588s-coolpi-4b.dts |  16 +
+ arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts  |   1 +
+ .../boot/dts/rockchip/rk3588s-gameforce-ace.dts    |  15 +-
+ .../boot/dts/rockchip/rk3588s-indiedroid-nova.dts  |  13 +-
+ .../boot/dts/rockchip/rk3588s-khadas-edge2.dts     |  15 +-
+ .../arm64/boot/dts/rockchip/rk3588s-nanopi-r6.dtsi |  11 +
+ arch/arm64/boot/dts/rockchip/rk3588s-odroid-m2.dts |  10 +
+ .../boot/dts/rockchip/rk3588s-orangepi-5.dtsi      |  11 +
+ .../dts/rockchip/rk3588s-orangepi-cm5-base.dts     |  12 +-
+ .../boot/dts/rockchip/rk3588s-orangepi-cm5.dtsi    |   3 +
+ .../boot/dts/rockchip/rk3588s-radxa-cm5-io.dts     |  10 +
+ arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts    |  17 +-
+ arch/arm64/boot/dts/rockchip/rk3588s-rock-5a.dts   |  13 +-
+ arch/arm64/boot/dts/rockchip/rk3588s-rock-5c.dts   |  13 +-
+ drivers/clk/Makefile                               |   2 +-
+ drivers/clk/rockchip/Kconfig                       |   2 +-
+ drivers/clk/rockchip/clk-rk3588.c                  |  16 +
+ drivers/clk/rockchip/clk.c                         |  25 +-
+ drivers/clk/rockchip/clk.h                         |   3 +
+ include/dt-bindings/clock/rockchip,rk3588-cru.h    |   4 +
+ include/soc/rockchip/rk3588_grf.h                  |   2 +
+ 71 files changed, 1243 insertions(+), 97 deletions(-)
+Merging samsung-krzk/for-next (446ccff1b11b7 Merge branch 'next/clk' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux.git samsung-krzk/for-next
+Merge made by the 'ort' strategy.
+ arch/arm/mach-s3c/mach-crag6410.c                  | 24 ++++++++++++-------
+ arch/arm64/boot/dts/exynos/exynos850.dtsi          |  7 ++++++
+ arch/arm64/boot/dts/exynos/exynosautov920-sadk.dts |  8 +++++++
+ arch/arm64/boot/dts/exynos/exynosautov920.dtsi     | 27 ++++++++++++++++++++++
+ drivers/clk/samsung/clk-exynos850.c                |  5 ++--
+ 5 files changed, 61 insertions(+), 10 deletions(-)
+Merging scmi/for-linux-next (8408dd4800b6c Merge branches 'for-next/scmi/updates' and 'for-next/ffa/updates' of ssh://gitolite.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux into for-linux-next)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux.git scmi/for-linux-next
+Auto-merging drivers/firmware/imx/sm-misc.c
+Merge made by the 'ort' strategy.
+ drivers/clk/clk-scmi.c                             |  48 +---
+ drivers/clk/clk-scpi.c                             |   2 +-
+ drivers/firmware/arm_ffa/bus.c                     |   3 +-
+ drivers/firmware/arm_ffa/common.h                  |   4 +-
+ drivers/firmware/arm_ffa/driver.c                  |  91 +++++--
+ drivers/firmware/arm_ffa/smccc.c                   |   2 +-
+ drivers/firmware/arm_scmi/base.c                   |  25 +-
+ drivers/firmware/arm_scmi/clock.c                  | 301 ++++++++++++++++-----
+ drivers/firmware/arm_scmi/common.h                 | 165 ++++++++++-
+ drivers/firmware/arm_scmi/driver.c                 |  98 ++++---
+ drivers/firmware/arm_scmi/powercap.c               |  18 +-
+ drivers/firmware/arm_scmi/protocols.h              |  13 +-
+ drivers/firmware/arm_scmi/quirks.c                 |  17 +-
+ drivers/firmware/arm_scmi/sensors.c                |  12 +-
+ drivers/firmware/arm_scmi/transports/optee.c       |  46 +++-
+ drivers/firmware/arm_scmi/transports/virtio.c      |  52 +++-
+ .../firmware/arm_scmi/vendors/imx/imx-sm-misc.c    |  86 ++++++
+ drivers/firmware/imx/sm-misc.c                     |  73 +++++
+ drivers/firmware/smccc/soc_id.c                    |   2 +-
+ include/linux/arm-smccc.h                          |   5 +
+ include/linux/arm_ffa.h                            |   4 +-
+ include/linux/scmi_imx_protocol.h                  |  14 +
+ include/linux/scmi_protocol.h                      |  36 +--
+ 23 files changed, 882 insertions(+), 235 deletions(-)
+Merging sophgo/for-next (9aa6068586a9b Merge branch 'dt/riscv' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/sophgo/linux.git sophgo/for-next
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/sophgo/cv180x.dtsi | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging sophgo-soc/soc-for-next (c8754c7deab4c soc: sophgo: cv1800: rtcsys: New driver (handling RTC only))
+$ git merge -m Merge branch 'soc-for-next' of https://github.com/sophgo/linux.git sophgo-soc/soc-for-next
+Already up to date.
+Merging spacemit/for-next (e2518e8cb1dde riscv: dts: spacemit: enable QSPI for OrangePi RV2)
+$ git merge -m Merge branch 'for-next' of https://github.com/spacemit-com/linux spacemit/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/riscv/spacemit.yaml        |   1 +
+ arch/riscv/boot/dts/spacemit/Makefile              |   1 +
+ arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts    |  33 ++-
+ arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts     | 309 ++++++++++++++++++++-
+ arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts   | 292 +++++++++++++++++++
+ arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi       |  60 ++++
+ arch/riscv/boot/dts/spacemit/k1.dtsi               |  30 +-
+ .../dts/spacemit/k3-deepcomputing-fml13v05.dts     |  31 +++
+ arch/riscv/boot/dts/spacemit/k3-pico-itx.dts       |  24 ++
+ arch/riscv/boot/dts/spacemit/k3.dtsi               |  33 ++-
+ 10 files changed, 808 insertions(+), 6 deletions(-)
+ create mode 100644 arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts
+Merging stm32/stm32-next (a0d6c2a06ffff ARM: dts: stm32: stm32mp15x-mecio1-io: Move expander gpio-line-names to board files)
+$ git merge -m Merge branch 'stm32-next' of https://git.kernel.org/pub/scm/linux/kernel/git/atorgue/stm32.git stm32/stm32-next
+Merge made by the 'ort' strategy.
+ arch/arm/boot/dts/st/stm32mp135f-dhcor-dhsbc.dts |   4 +
+ arch/arm/boot/dts/st/stm32mp135f-dk.dts          |  28 ++++-
+ arch/arm/boot/dts/st/stm32mp151c-mecio1r0.dts    | 128 ++++++++++++++++++++
+ arch/arm/boot/dts/st/stm32mp153c-mecio1r1.dts    | 144 ++++++++++++++++++++++
+ arch/arm/boot/dts/st/stm32mp15x-mecio1-io.dtsi   | 146 +++--------------------
+ 5 files changed, 316 insertions(+), 134 deletions(-)
+Merging sunxi/sunxi/for-next (d55695a22b5d7 Merge branch 'sunxi/dt-for-7.2' into sunxi/for-next)
+$ git merge -m Merge branch 'sunxi/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sunxi/linux.git sunxi/sunxi/for-next
+Merge made by the 'ort' strategy.
+ .../sram/allwinner,sun4i-a10-system-control.yaml   |   5 +-
+ Documentation/devicetree/bindings/sram/sram.yaml   |   1 +
+ arch/arm/boot/dts/allwinner/sun8i-t113s.dtsi       |   4 +
+ arch/arm64/boot/dts/allwinner/sun50i-h616.dtsi     |  38 +++-
+ arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi  |   9 +
+ drivers/soc/sunxi/sunxi_sram.c                     | 196 +++++++++++++--------
+ 6 files changed, 174 insertions(+), 79 deletions(-)
+Merging tee/next (4d88259524b06 Merge branches 'tee_fixes_for_v7.1', 'optee_for_v7.2', 'optee_fix_for_v7.1', 'amdtee_for_v7.2' and 'qcomtee_for_v7.1' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/jenswi/linux-tee.git tee/next
+Merge made by the 'ort' strategy.
+ drivers/tee/amdtee/amdtee_private.h |  16 -----
+ drivers/tee/amdtee/call.c           |   2 +-
+ drivers/tee/amdtee/core.c           |  52 ++--------------
+ drivers/tee/optee/call.c            |   3 +-
+ drivers/tee/optee/optee_msg.h       |  50 ++++++++-------
+ drivers/tee/optee/optee_private.h   | 120 +++++++++++++++++++-----------------
+ drivers/tee/optee/supp.c            | 107 ++++++++++++++++++++++----------
+ drivers/tee/qcomtee/core.c          |   6 +-
+ drivers/tee/tee_core.c              |  56 ++++++++---------
+ drivers/tee/tee_shm.c               |   2 +-
+ include/uapi/linux/tee.h            |   1 +
+ 11 files changed, 204 insertions(+), 211 deletions(-)
+Merging tegra/for-next (20e4d2163a23d Merge branch for-7.2/arm64/dt into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git tegra/for-next
+Merge made by the 'ort' strategy.
+ arch/arm64/boot/dts/nvidia/tegra194.dtsi |  9 +++++++++
+ arch/arm64/boot/dts/nvidia/tegra234.dtsi | 24 ++++++++++++++++++++----
+ drivers/bus/tegra-aconnect.c             | 14 ++++++--------
+ 3 files changed, 35 insertions(+), 12 deletions(-)
+Merging tenstorrent-dt/tenstorrent-dt-for-next (ad3cb81323131 riscv: dts: tenstorrent: Add PMU node to blackhole for Linux perf support)
+$ git merge -m Merge branch 'tenstorrent-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-dt/tenstorrent-dt-for-next
+Merge made by the 'ort' strategy.
+ arch/riscv/boot/dts/tenstorrent/blackhole.dtsi | 48 ++++++++++++++++++++++++++
+ 1 file changed, 48 insertions(+)
+Merging thead-dt/thead-dt-for-next (74ec3d52c0035 riscv: dts: thead: beaglev-ahead: enable HDMI output)
+$ git merge -m Merge branch 'thead-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-dt/thead-dt-for-next
+Already up to date.
+Merging ti/ti-next (e84bac36f60fe Merge branches 'ti-k3-dts-next' and 'ti-drivers-soc-next' into ti-next)
+$ git merge -m Merge branch 'ti-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ti/linux.git ti/ti-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/ti/k3.yaml   |    7 +
+ .../devicetree/bindings/hwinfo/ti,k3-socinfo.yaml  |   11 +
+ arch/arm64/boot/dts/ti/Makefile                    |   18 +
+ arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts           |    1 +
+ .../dts/ti/k3-am62-phyboard-lyra-oldi-lcd185.dtso  |  185 ++++
+ .../boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts      | 1033 ++++++++++++++++++++
+ arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi      |  360 +++++++
+ .../boot/dts/ti/k3-am625-verdin-dsi-to-hdmi.dtso   |   94 ++
+ arch/arm64/boot/dts/ti/k3-am62a7-sk.dts            |    2 +
+ arch/arm64/boot/dts/ti/k3-j721s2-som-p0.dtsi       |    1 +
+ arch/arm64/boot/dts/ti/k3-j722s-main.dtsi          |   24 +-
+ arch/arm64/boot/dts/ti/k3-j722s-mcu.dtsi           |   15 +
+ arch/arm64/boot/dts/ti/k3-j722s-wakeup.dtsi        |   18 +
+ arch/arm64/boot/dts/ti/k3-j722s.dtsi               |    2 +
+ drivers/soc/ti/k3-ringacc.c                        |    4 +-
+ drivers/soc/ti/k3-socinfo.c                        |   41 +-
+ drivers/soc/ti/knav_qmss_queue.c                   |    7 +-
+ include/linux/soc/ti/knav_dma.h                    |   11 +-
+ 18 files changed, 1803 insertions(+), 31 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/ti/k3-am62-phyboard-lyra-oldi-lcd185.dtso
+ create mode 100644 arch/arm64/boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts
+ create mode 100644 arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi
+ create mode 100644 arch/arm64/boot/dts/ti/k3-am625-verdin-dsi-to-hdmi.dtso
+ create mode 100644 arch/arm64/boot/dts/ti/k3-j722s-mcu.dtsi
+ create mode 100644 arch/arm64/boot/dts/ti/k3-j722s-wakeup.dtsi
+Merging xilinx/for-next (fdee7c66c0d7b soc: xilinx: Shutdown and free rx mailbox channel)
+$ git merge -m Merge branch 'for-next' of https://github.com/Xilinx/linux-xlnx.git xilinx/for-next
+Merge made by the 'ort' strategy.
+ drivers/soc/xilinx/zynqmp_power.c | 47 +++++++++++++++++----------------------
+ 1 file changed, 20 insertions(+), 27 deletions(-)
+Merging socfpga/for-next (7557d67a60573 ARM: dts: socfpga: arria10: Increase JFFS2 rootfs partition size)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dinguyen/linux.git socfpga/for-next
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/altera.yaml  |   7 ++
+ arch/arm/boot/dts/intel/socfpga/socfpga.dtsi       |   4 +-
+ .../boot/dts/intel/socfpga/socfpga_arria10.dtsi    |   4 +-
+ .../intel/socfpga/socfpga_arria10_socdk_nand.dts   |   4 +-
+ arch/arm64/boot/dts/intel/Makefile                 |   1 +
+ .../arm64/boot/dts/intel/socfpga_agilex3_socdk.dts |   2 +
+ arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi     |   4 +-
+ .../arm64/boot/dts/intel/socfpga_agilex5_socdk.dts |   4 +-
+ .../boot/dts/intel/socfpga_agilex5_socdk_013b.dts  |   4 +-
+ .../dts/intel/socfpga_agilex5_socdk_modular.dts    |   4 +-
+ .../boot/dts/intel/socfpga_agilex5_socdk_nand.dts  |   4 +-
+ .../boot/dts/intel/socfpga_agilex7m_socdk.dts      | 103 +++++++++++++++++++++
+ 12 files changed, 131 insertions(+), 14 deletions(-)
+ create mode 100644 arch/arm64/boot/dts/intel/socfpga_agilex7m_socdk.dts
+Merging clk/clk-next (9781f9635d693 Merge branch 'clk-fixes' into clk-next)
+$ git merge -m Merge branch 'clk-next' of https://git.kernel.org/pub/scm/linux/kernel/git/clk/linux.git clk/clk-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/clock/marvell,pxa1908.yaml | 34 ++++++---
+ drivers/clk/bcm/clk-iproc-asiu.c                   | 27 +++-----
+ drivers/clk/clk-axi-clkgen.c                       |  5 +-
+ drivers/clk/clk-bulk.c                             | 14 ++--
+ drivers/clk/clk-max77686.c                         | 32 ++++-----
+ drivers/clk/hisilicon/clk.c                        |  4 +-
+ drivers/clk/hisilicon/clkdivider-hi6220.c          | 26 +++----
+ drivers/clk/mediatek/clk-mt7988-infracfg.c         | 80 ++++++++++------------
+ drivers/clk/mediatek/clk-mt8192.c                  |  4 +-
+ drivers/clk/mediatek/clk-mux.h                     |  5 ++
+ drivers/clk/mmp/clk-pxa1908-apbc.c                 | 58 ++++++++++++----
+ drivers/clk/mmp/clk-pxa1908-apbcp.c                | 31 ++++++++-
+ drivers/clk/mvebu/common.c                         | 21 +++---
+ drivers/clk/nxp/clk-lpc18xx-ccu.c                  | 14 ++--
+ drivers/clk/visconti/pll.c                         | 17 ++---
+ include/linux/adi-axi-common.h                     |  2 +
+ 16 files changed, 210 insertions(+), 164 deletions(-)
+Merging clk-imx/for-next (d16f57caa7877 clk: imx8mq: Correct the CSI PHY sels)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelvesa/linux.git clk-imx/for-next
+Already up to date.
+Merging clk-renesas/renesas-clk (4f42053949324 clk: renesas: r8a73a4: Add ZT/ZTR trace clocks)
+$ git merge -m Merge branch 'renesas-clk' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git clk-renesas/renesas-clk
+Merge made by the 'ort' strategy.
+ drivers/clk/renesas/clk-r8a73a4.c      |   2 +
+ drivers/clk/renesas/clk-r8a7740.c      |   2 +
+ drivers/clk/renesas/r9a08g046-cpg.c    | 358 +++++++++++++++++++++++++++++++++
+ drivers/clk/renesas/r9a09g047-cpg.c    |  84 ++++++++
+ drivers/clk/renesas/r9a09g077-cpg.c    |   1 +
+ drivers/clk/renesas/renesas-cpg-mssr.c |   3 +
+ drivers/clk/renesas/rzg2l-cpg.c        |  68 ++++++-
+ drivers/clk/renesas/rzg2l-cpg.h        |  11 +-
+ drivers/clk/renesas/rzv2h-cpg.c        | 181 +++++++++++++++++
+ drivers/clk/renesas/rzv2h-cpg.h        |  12 ++
+ include/linux/clk/renesas.h            |  20 ++
+ 11 files changed, 737 insertions(+), 5 deletions(-)
+Merging thead-clk/thead-clk-for-next (baf4fc7c03bd0 clk: thead: th1520-ap: Support CPU frequency scaling)
+$ git merge -m Merge branch 'thead-clk-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fustini/linux.git thead-clk/thead-clk-for-next
+Already up to date.
+Merging tenstorrent-clk/tenstorrent-clk-for-next (23c8ebc952849 clk: tenstorrent: Add Atlantis clock controller driver)
+$ git merge -m Merge branch 'tenstorrent-clk-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tenstorrent/linux.git tenstorrent-clk/tenstorrent-clk-for-next
+Already up to date.
+Merging csky/linux-next (817d21578d51e csky: Remove compile warning for CONFIG_SMP)
+$ git merge -m Merge branch 'linux-next' of https://github.com/c-sky/csky-linux.git csky/linux-next
+Already up to date.
+Merging loongarch/loongarch-next (5a873d77ba792 LoongArch: KVM: Move unconditional delay into timer clear scenery)
+$ git merge -m Merge branch 'loongarch-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chenhuacai/linux-loongson.git loongarch/loongarch-next
+Already up to date.
+Merging m68k/for-next (e321561b91537 dio: Update DIO_SCMAX comment)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/linux-m68k.git m68k/for-next
+Merge made by the 'ort' strategy.
+ drivers/dio/dio.c           | 15 ++++++++-------
+ drivers/nubus/nubus.c       | 19 ++++++++-----------
+ drivers/zorro/zorro-sysfs.c | 15 ++++++++-------
+ include/linux/dio.h         |  4 +---
+ 4 files changed, 25 insertions(+), 28 deletions(-)
+Merging m68knommu/for-next (16b47102d9c3b m68k: coldfire: use ColdFire specifc IO access in SoC code)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gerg/m68knommu.git m68knommu/for-next
+Merge made by the 'ort' strategy.
+ arch/m68k/Kconfig.cpu                 |   1 +
+ arch/m68k/Kconfig.machine             |   8 +--
+ arch/m68k/coldfire/amcore.c           |   4 +-
+ arch/m68k/coldfire/clk.c              |   8 +--
+ arch/m68k/coldfire/device.c           |   8 +--
+ arch/m68k/coldfire/dma_timer.c        |  12 ++--
+ arch/m68k/coldfire/intc-2.c           |  30 ++++-----
+ arch/m68k/coldfire/intc-5249.c        |  10 +--
+ arch/m68k/coldfire/intc-525x.c        |  12 ++--
+ arch/m68k/coldfire/intc-5272.c        |  20 +++---
+ arch/m68k/coldfire/intc-simr.c        |  38 +++++------
+ arch/m68k/coldfire/intc.c             |  28 ++++----
+ arch/m68k/coldfire/m5206.c            |   2 +-
+ arch/m68k/coldfire/m520x.c            |  26 ++++----
+ arch/m68k/coldfire/m523x.c            |  12 ++--
+ arch/m68k/coldfire/m5249.c            |  16 ++---
+ arch/m68k/coldfire/m525x.c            |  12 ++--
+ arch/m68k/coldfire/m5272.c            |  16 ++---
+ arch/m68k/coldfire/m527x.c            |  40 +++++------
+ arch/m68k/coldfire/m528x.c            |  28 ++++----
+ arch/m68k/coldfire/m5307.c            |   2 +-
+ arch/m68k/coldfire/m53xx.c            | 122 +++++++++++++++++-----------------
+ arch/m68k/coldfire/m5407.c            |   2 +-
+ arch/m68k/coldfire/m5441x.c           |  12 ++--
+ arch/m68k/coldfire/m54xx.c            |  18 ++---
+ arch/m68k/coldfire/nettel.c           |  16 ++---
+ arch/m68k/coldfire/pci.c              |  56 ++++++++--------
+ arch/m68k/coldfire/pit.c              |  20 +++---
+ arch/m68k/coldfire/reset.c            |   4 +-
+ arch/m68k/coldfire/sltimers.c         |  18 ++---
+ arch/m68k/coldfire/stmark2.c          |  16 ++---
+ arch/m68k/coldfire/timers.c           |  30 ++++-----
+ arch/m68k/configs/m5208evb_defconfig  |   9 +--
+ arch/m68k/configs/m5249evb_defconfig  |   9 +--
+ arch/m68k/configs/m5272c3_defconfig   |   9 +--
+ arch/m68k/configs/m5275evb_defconfig  |   9 +--
+ arch/m68k/configs/m5307c3_defconfig   |  10 +--
+ arch/m68k/configs/m5329evb_defconfig  |  61 +++++++++++++++++
+ arch/m68k/configs/m5407c3_defconfig   |   9 ++-
+ arch/m68k/configs/m54418evb_defconfig |  70 +++++++++++++++++++
+ arch/m68k/configs/m5475evb_defconfig  |  12 ++--
+ arch/m68k/configs/nettel_defconfig    |  56 ++++++++++++++++
+ arch/m68k/configs/stmark2_defconfig   |  33 +++++----
+ arch/m68k/include/asm/io_no.h         |  16 +++++
+ arch/m68k/include/asm/mcfgpio.h       |  12 ++--
+ arch/m68k/include/asm/nettel.h        |   4 +-
+ 46 files changed, 590 insertions(+), 376 deletions(-)
+ create mode 100644 arch/m68k/configs/m5329evb_defconfig
+ create mode 100644 arch/m68k/configs/m54418evb_defconfig
+ create mode 100644 arch/m68k/configs/nettel_defconfig
+Merging microblaze/next (6de23f81a5e08 Linux 7.0-rc1)
+$ git merge -m Merge branch 'next' of git://git.monstr.eu/linux-2.6-microblaze.git microblaze/next
+Already up to date.
+Merging mips/mips-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'mips-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mips/linux.git mips/mips-next
+Already up to date.
+Merging openrisc/for-next (0dfffa5479d62 openrisc: define arch-specific version of nop())
+$ git merge -m Merge branch 'for-next' of https://github.com/openrisc/linux.git openrisc/for-next
+Already up to date.
+Merging parisc-hd/for-next (5200f5f493f79 Linux 7.1-rc4)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/parisc-linux.git parisc-hd/for-next
+Already up to date.
+Merging powerpc/next (5200f5f493f79 Linux 7.1-rc4)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git powerpc/next
+Already up to date.
+Merging risc-v/for-next (f2abc305aa93f riscv: Define __riscv_copy_{,vec_}{words,bytes}_unaligned() using SYM_TYPED_FUNC_START)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git risc-v/for-next
+Already up to date.
+Merging riscv-dt/riscv-dt-for-next (3869417f8996f riscv: dts: microchip: remove redudant enabling of syscontroller)
+$ git merge -m Merge branch 'riscv-dt-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-dt/riscv-dt-for-next
+Merge made by the 'ort' strategy.
+ .../riscv/boot/dts/microchip/mpfs-beaglev-fire.dts | 37 ++++++------
+ arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts   |  4 --
+ .../boot/dts/microchip/mpfs-icicle-kit-common.dtsi |  4 --
+ arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts  |  4 --
+ arch/riscv/boot/dts/microchip/mpfs-polarberry.dts  |  4 --
+ arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts     |  4 --
+ arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts     |  4 --
+ arch/riscv/boot/dts/microchip/mpfs.dtsi            |  6 +-
+ .../boot/dts/microchip/pic64gx-curiosity-kit.dts   | 67 ++++++++++++----------
+ arch/riscv/boot/dts/microchip/pic64gx.dtsi         | 46 +++++++++++----
+ 10 files changed, 95 insertions(+), 85 deletions(-)
+Merging riscv-soc/riscv-soc-for-next (b5865650b2097 Merge branch 'riscv-soc-drivers-for-next' into riscv-soc-for-next)
+$ git merge -m Merge branch 'riscv-soc-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git riscv-soc/riscv-soc-for-next
+Auto-merging Documentation/devicetree/bindings/timer/sifive,clint.yaml
+Merge made by the 'ort' strategy.
+ .../interrupt-controller/sifive,plic-1.0.0.yaml    |   1 +
+ .../devicetree/bindings/riscv/canaan.yaml          |   8 +-
+ .../soc/microchip/microchip,mpfs-irqmux.yaml       |   6 +-
+ .../devicetree/bindings/timer/sifive,clint.yaml    |   1 +
+ arch/riscv/boot/dts/canaan/Makefile                |   2 +
+ arch/riscv/boot/dts/canaan/k230-canmv.dts          | 332 +++++++++++++++++++++
+ arch/riscv/boot/dts/canaan/k230-evb.dts            |  28 ++
+ arch/riscv/boot/dts/canaan/k230-pinctrl.h          |  18 ++
+ arch/riscv/boot/dts/canaan/k230.dtsi               | 157 ++++++++++
+ 9 files changed, 551 insertions(+), 2 deletions(-)
+ create mode 100644 arch/riscv/boot/dts/canaan/k230-canmv.dts
+ create mode 100644 arch/riscv/boot/dts/canaan/k230-evb.dts
+ create mode 100644 arch/riscv/boot/dts/canaan/k230-pinctrl.h
+ create mode 100644 arch/riscv/boot/dts/canaan/k230.dtsi
+Merging s390/for-next (2d5fe18917628 Merge branch 'fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/s390/linux.git s390/for-next
+Auto-merging arch/s390/mm/init.c
+Merge made by the 'ort' strategy.
+ Documentation/arch/s390/s390dbf.rst    |  30 ++++++++
+ arch/s390/Kconfig                      |   1 +
+ arch/s390/include/asm/barrier.h        |  12 ++--
+ arch/s390/include/asm/debug.h          |   1 +
+ arch/s390/include/asm/processor.h      |   3 +-
+ arch/s390/include/asm/sclp.h           |   1 +
+ arch/s390/include/asm/vdso/processor.h |   4 +-
+ arch/s390/kernel/debug.c               | 125 +++++++++++++++++++++++++++++----
+ arch/s390/kernel/process.c             |   3 -
+ arch/s390/kernel/vmlinux.lds.S         |   7 ++
+ arch/s390/mm/init.c                    |   2 +
+ drivers/s390/char/sclp_pci.c           |   1 +
+ drivers/s390/cio/device.c              |   2 +-
+ drivers/s390/crypto/ap_bus.c           |  17 +++++
+ drivers/s390/crypto/ap_bus.h           |   2 +
+ drivers/s390/crypto/ap_queue.c         |   2 +
+ 16 files changed, 184 insertions(+), 29 deletions(-)
+Merging sh/for-next (b0aa5e4b087b6 sh: Fix fallout from ZERO_PAGE consolidation)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/glaubitz/sh-linux.git sh/for-next
+Already up to date.
+Merging sparc/for-next (d844152d85cfc sparc: remove unused variable strtab)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/alarsson/linux-sparc.git sparc/for-next
+Already up to date.
+Merging uml/next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/uml/linux.git uml/next
+Already up to date.
+Merging xtensa/xtensa-for-next (84477867e1222 tty: xtensa/iss: use strnlen to improve iss_console_write)
+$ git merge -m Merge branch 'xtensa-for-next' of https://github.com/jcmvbkbc/linux-xtensa.git xtensa/xtensa-for-next
+Merge made by the 'ort' strategy.
+ arch/xtensa/platforms/iss/console.c | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+Merging fs-next (68fa42a818989 Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs.git)
+$ git merge -m Merge branch 'fs-next' of linux-next fs-next
+Auto-merging MAINTAINERS
+Auto-merging fs/inode.c
+Auto-merging fs/ocfs2/super.c
+Auto-merging fs/open.c
+Auto-merging fs/proc/generic.c
+Auto-merging include/linux/fs.h
+Auto-merging include/linux/uaccess.h
+Auto-merging lib/iov_iter.c
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-fs-f2fs            |    5 +-
+ .../filesystems/adding-new-filesystems.rst         |  195 +++
+ Documentation/filesystems/f2fs.rst                 |   50 +-
+ Documentation/filesystems/index.rst                |    1 +
+ Documentation/filesystems/porting.rst              |    1 -
+ Documentation/filesystems/proc.rst                 |   19 +-
+ Documentation/netlink/specs/nfsd.yaml              |  229 ++++
+ Documentation/netlink/specs/sunrpc_cache.yaml      |  149 +++
+ MAINTAINERS                                        |   36 +-
+ arch/alpha/include/uapi/asm/errno.h                |    2 +
+ arch/alpha/include/uapi/asm/fcntl.h                |    1 +
+ arch/mips/include/uapi/asm/errno.h                 |    2 +
+ arch/parisc/include/uapi/asm/errno.h               |    2 +
+ arch/parisc/include/uapi/asm/fcntl.h               |    1 +
+ arch/powerpc/platforms/cell/spufs/file.c           |    2 +-
+ arch/sparc/include/uapi/asm/errno.h                |    2 +
+ arch/sparc/include/uapi/asm/fcntl.h                |    1 +
+ drivers/block/rbd.c                                |   43 +-
+ drivers/mtd/ubi/ubi.h                              |    9 +-
+ fs/affs/affs.h                                     |    5 -
+ fs/affs/super.c                                    |    6 +-
+ fs/befs/linuxvfs.c                                 |    3 +-
+ fs/bfs/inode.c                                     |    3 +-
+ fs/bpf_fs_kfuncs.c                                 |   10 +-
+ fs/btrfs/inode.c                                   |    2 +-
+ fs/ceph/file.c                                     |    4 +
+ fs/ceph/mds_client.c                               |    2 +-
+ fs/coredump.c                                      |    3 +-
+ fs/dcache.c                                        |   22 +-
+ fs/dlm/lowcomms.c                                  |   12 +-
+ fs/dlm/memory.c                                    |    4 +-
+ fs/dlm/midcomms.c                                  |   21 +-
+ fs/ecryptfs/keystore.c                             |   16 +-
+ fs/eventpoll.c                                     | 1207 ++++++++++++-------
+ fs/exec.c                                          |    6 +-
+ fs/exfat/dir.c                                     |    4 +-
+ fs/exfat/exfat_fs.h                                |    2 +
+ fs/exfat/file.c                                    |   18 +-
+ fs/exfat/namei.c                                   |   57 +-
+ fs/exfat/nls.c                                     |   19 +-
+ fs/ext4/ialloc.c                                   |    4 +-
+ fs/f2fs/checkpoint.c                               |    2 +-
+ fs/f2fs/compress.c                                 |    2 +
+ fs/f2fs/data.c                                     |   38 +-
+ fs/f2fs/dir.c                                      |    2 +
+ fs/f2fs/extent_cache.c                             |   19 +-
+ fs/f2fs/f2fs.h                                     |   32 +-
+ fs/f2fs/file.c                                     |   13 +-
+ fs/f2fs/gc.c                                       |   11 +-
+ fs/f2fs/inline.c                                   |    3 +
+ fs/f2fs/inode.c                                    |   24 +-
+ fs/f2fs/node.c                                     |   14 +-
+ fs/f2fs/node.h                                     |    8 +-
+ fs/f2fs/recovery.c                                 |    2 +
+ fs/f2fs/segment.c                                  |   23 +
+ fs/f2fs/super.c                                    |   51 +-
+ fs/f2fs/sysfs.c                                    |    2 +
+ fs/f2fs/verity.c                                   |    2 +
+ fs/f2fs/xattr.c                                    |   41 +-
+ fs/f2fs/xattr.h                                    |    6 +
+ fs/fat/fat.h                                       |    3 +
+ fs/fat/file.c                                      |   36 +
+ fs/fat/namei_msdos.c                               |    1 +
+ fs/fat/namei_vfat.c                                |    1 +
+ fs/fcntl.c                                         |    4 +-
+ fs/file.c                                          |   17 +-
+ fs/file_attr.c                                     |   16 +-
+ fs/filesystems.c                                   |  332 +++--
+ fs/fuse/Makefile                                   |    3 +-
+ fs/fuse/args.h                                     |   65 +
+ fs/fuse/backing.c                                  |    3 +-
+ fs/fuse/control.c                                  |   16 +-
+ fs/fuse/cuse.c                                     |   28 +-
+ fs/fuse/dev.c                                      | 1271 ++++++++------------
+ fs/fuse/dev.h                                      |  110 ++
+ fs/fuse/dev_uring.c                                |  143 ++-
+ fs/fuse/dev_uring_i.h                              |   39 +-
+ fs/fuse/dir.c                                      |   12 +-
+ fs/fuse/file.c                                     |  146 +--
+ fs/fuse/fuse_dev_i.h                               |  350 +++++-
+ fs/fuse/fuse_i.h                                   |  862 +++++--------
+ fs/fuse/fuse_trace.h                               |    4 +-
+ fs/fuse/inode.c                                    |  263 +---
+ fs/fuse/notify.c                                   |  434 +++++++
+ fs/fuse/poll.c                                     |  141 +++
+ fs/fuse/req.c                                      |   99 ++
+ fs/fuse/req_timeout.c                              |  148 +++
+ fs/fuse/sysctl.c                                   |    1 +
+ fs/fuse/sysctl.h                                   |    9 +
+ fs/fuse/virtio_fs.c                                |   21 +-
+ fs/gfs2/inode.c                                    |    6 +
+ fs/gfs2/quota.c                                    |   36 +-
+ fs/gfs2/super.c                                    |    1 +
+ fs/hfs/dir.c                                       |    1 +
+ fs/hfs/hfs_fs.h                                    |    2 +
+ fs/hfs/inode.c                                     |   14 +
+ fs/hfsplus/inode.c                                 |   16 +-
+ fs/hpfs/inode.c                                    |    2 +-
+ fs/hpfs/super.c                                    |    3 +-
+ fs/inode.c                                         |  100 +-
+ fs/iomap/buffered-io.c                             |    3 +-
+ fs/iomap/direct-io.c                               |    3 -
+ fs/iomap/iter.c                                    |   12 +-
+ fs/isofs/dir.c                                     |   16 +
+ fs/isofs/inode.c                                   |    3 +-
+ fs/isofs/isofs.h                                   |    3 +
+ fs/jfs/super.c                                     |    3 +-
+ fs/minix/inode.c                                   |    3 +-
+ fs/mount.h                                         |    4 +
+ fs/namei.c                                         |   29 +-
+ fs/namespace.c                                     |   34 +-
+ fs/nfs/client.c                                    |   21 +-
+ fs/nfs/dir.c                                       |    4 +
+ fs/nfs/dns_resolve.c                               |    2 +-
+ fs/nfs/inode.c                                     |   19 +-
+ fs/nfs/internal.h                                  |    3 +
+ fs/nfs/namespace.c                                 |    2 +
+ fs/nfs/nfs3proc.c                                  |    2 +
+ fs/nfs/nfs3xdr.c                                   |    7 +-
+ fs/nfs/nfs4proc.c                                  |   10 +-
+ fs/nfs/proc.c                                      |    3 +
+ fs/nfs/symlink.c                                   |    3 +
+ fs/nfsd/blocklayout.c                              |   37 +-
+ fs/nfsd/export.c                                   |  717 ++++++++++-
+ fs/nfsd/netlink.c                                  |   93 ++
+ fs/nfsd/netlink.h                                  |   18 +
+ fs/nfsd/nfs3proc.c                                 |   36 +-
+ fs/nfsd/nfs4idmap.c                                |    4 +-
+ fs/nfsd/nfs4layouts.c                              |   29 +-
+ fs/nfsd/nfs4state.c                                |    9 +-
+ fs/nfsd/nfs4xdr.c                                  |   52 +-
+ fs/nfsd/nfsctl.c                                   |   79 ++
+ fs/nfsd/nfsd.h                                     |   17 +-
+ fs/nfsd/vfs.c                                      |   88 ++
+ fs/nfsd/vfs.h                                      |    3 +
+ fs/nfsd/xdr3.h                                     |    4 +-
+ fs/ntfs/Makefile                                   |    2 +-
+ fs/ntfs/attrib.c                                   |    3 +-
+ fs/ntfs/quota.c                                    |   95 --
+ fs/ntfs/quota.h                                    |   15 -
+ fs/ntfs/super.c                                    |  109 +-
+ fs/ntfs/volume.h                                   |    7 -
+ fs/ntfs3/dir.c                                     |   22 +-
+ fs/ntfs3/file.c                                    |   29 +
+ fs/ntfs3/fslog.c                                   |    5 +-
+ fs/ntfs3/index.c                                   |   46 +-
+ fs/ntfs3/lznt.c                                    |    2 +-
+ fs/ntfs3/namei.c                                   |    3 +-
+ fs/ntfs3/ntfs_fs.h                                 |    4 +-
+ fs/ntfs3/run.c                                     |    9 +-
+ fs/ntfs3/super.c                                   |    8 +-
+ fs/ntfs3/xattr.c                                   |    4 +-
+ fs/ocfs2/super.c                                   |    1 -
+ fs/omfs/inode.c                                    |    6 +-
+ fs/open.c                                          |   17 +-
+ fs/proc/generic.c                                  |   10 +
+ fs/proc/internal.h                                 |    3 +
+ fs/proc/proc_net.c                                 |    8 +
+ fs/proc/root.c                                     |   24 +-
+ fs/qnx4/inode.c                                    |    3 +-
+ fs/smb/client/cached_dir.c                         |    4 +
+ fs/smb/client/cifs_spnego.c                        |   16 +
+ fs/smb/client/cifs_swn.c                           |  316 ++++-
+ fs/smb/client/cifsacl.c                            |  196 +--
+ fs/smb/client/cifsfs.c                             |   55 +
+ fs/smb/client/cifsfs.h                             |    3 +
+ fs/smb/client/connect.c                            |    5 +
+ fs/smb/client/dir.c                                |   18 +-
+ fs/smb/client/fs_context.c                         |    2 +-
+ fs/smb/client/inode.c                              |   32 +-
+ fs/smb/client/namespace.c                          |    1 +
+ fs/smb/client/netlink.c                            |    6 +-
+ fs/smb/client/smb2ops.c                            |   22 +-
+ fs/smb/client/smb2transport.c                      |    2 +
+ fs/smb/client/trace.h                              |    2 +
+ fs/smb/server/smb2pdu.c                            |   30 +-
+ fs/smb/smbdirect/debug.c                           |    2 +-
+ fs/super.c                                         |    1 -
+ fs/sync.c                                          |    3 +-
+ fs/sysfs/mount.c                                   |   18 +-
+ fs/ubifs/super.c                                   |    2 +-
+ fs/udf/balloc.c                                    |    5 +-
+ fs/vboxsf/dir.c                                    |    1 +
+ fs/vboxsf/file.c                                   |    6 +-
+ fs/vboxsf/super.c                                  |    7 +
+ fs/vboxsf/utils.c                                  |   30 +
+ fs/vboxsf/vfsmod.h                                 |    6 +
+ fs/xfs/libxfs/xfs_inode_util.c                     |    2 +
+ fs/xfs/xfs_export.c                                |    4 +-
+ fs/xfs/xfs_inode.c                                 |    2 +-
+ fs/xfs/xfs_ioctl.c                                 |   22 +-
+ fs/xfs/xfs_pnfs.c                                  |   44 +-
+ fs/xfs/xfs_pnfs.h                                  |   11 +-
+ fs/xfs/xfs_trace.h                                 |    2 +-
+ fs/zonefs/super.c                                  |    6 +-
+ include/linux/dcache.h                             |    4 +-
+ include/linux/exportfs.h                           |   25 +-
+ include/linux/exportfs_block.h                     |   88 ++
+ include/linux/fcntl.h                              |    4 +-
+ include/linux/fileattr.h                           |    3 +-
+ include/linux/fs.h                                 |   16 +-
+ include/linux/fs/super_types.h                     |    2 +-
+ include/linux/iomap.h                              |   10 -
+ include/linux/kstrtox.h                            |    9 +-
+ include/linux/namei.h                              |    1 -
+ include/linux/nfs_fs_sb.h                          |    2 +-
+ include/linux/nfs_xdr.h                            |    2 +
+ include/linux/proc_fs.h                            |   13 +
+ include/linux/rhashtable-types.h                   |   22 +-
+ include/linux/sockptr.h                            |   28 +-
+ include/linux/sunrpc/cache.h                       |   15 +-
+ include/linux/uaccess.h                            |   65 +-
+ include/trace/events/f2fs.h                        |   28 +
+ include/trace/events/filelock.h                    |    2 +-
+ include/uapi/asm-generic/errno.h                   |    2 +
+ include/uapi/asm-generic/fcntl.h                   |    8 +
+ include/uapi/linux/fs.h                            |    7 +
+ include/uapi/linux/nfsd_netlink.h                  |  132 ++
+ include/uapi/linux/sunrpc_netlink.h                |   84 ++
+ init/initramfs.c                                   |   68 +-
+ init/initramfs_test.c                              |   80 +-
+ kernel/acct.c                                      |    2 +-
+ lib/iov_iter.c                                     |    8 +-
+ lib/rhashtable.c                                   |   17 +-
+ lib/vsprintf.c                                     |    7 -
+ net/sunrpc/Makefile                                |    2 +-
+ net/sunrpc/auth_gss/svcauth_gss.c                  |    2 +-
+ net/sunrpc/cache.c                                 |  127 +-
+ net/sunrpc/netlink.c                               |   97 ++
+ net/sunrpc/netlink.h                               |   35 +
+ net/sunrpc/sunrpc_syms.c                           |   10 +
+ net/sunrpc/svc_xprt.c                              |   46 +-
+ net/sunrpc/svcauth_unix.c                          |  516 +++++++-
+ security/landlock/fs.c                             |    2 +-
+ tools/arch/alpha/include/uapi/asm/errno.h          |    2 +
+ tools/arch/mips/include/uapi/asm/errno.h           |    2 +
+ tools/arch/parisc/include/uapi/asm/errno.h         |    2 +
+ tools/arch/sparc/include/uapi/asm/errno.h          |    2 +
+ tools/include/uapi/asm-generic/errno.h             |    2 +
+ tools/include/uapi/linux/openat2.h                 |   43 +
+ .../selftests/{ => filesystems}/openat2/.gitignore |    0
+ .../selftests/{ => filesystems}/openat2/Makefile   |    9 +-
+ .../selftests/filesystems/openat2/emptypath_test.c |   55 +
+ .../selftests/filesystems/openat2/helpers.h        |  135 +++
+ .../{ => filesystems}/openat2/openat2_test.c       |  239 ++--
+ .../filesystems/openat2/rename_attack_test.c       |  159 +++
+ .../{ => filesystems}/openat2/resolve_test.c       |  354 +++---
+ .../selftests/namespaces/listns_efault_test.c      |   33 +-
+ tools/testing/selftests/namespaces/nsid_test.c     |   14 +-
+ tools/testing/selftests/openat2/helpers.c          |  109 --
+ tools/testing/selftests/openat2/helpers.h          |  108 --
+ .../testing/selftests/openat2/rename_attack_test.c |  160 ---
+ tools/testing/selftests/pid_namespace/pid_max.c    |  156 ++-
+ 253 files changed, 9038 insertions(+), 4065 deletions(-)
+ create mode 100644 Documentation/filesystems/adding-new-filesystems.rst
+ create mode 100644 Documentation/netlink/specs/sunrpc_cache.yaml
+ create mode 100644 fs/fuse/args.h
+ create mode 100644 fs/fuse/dev.h
+ create mode 100644 fs/fuse/notify.c
+ create mode 100644 fs/fuse/poll.c
+ create mode 100644 fs/fuse/req.c
+ create mode 100644 fs/fuse/req_timeout.c
+ create mode 100644 fs/fuse/sysctl.h
+ delete mode 100644 fs/ntfs/quota.c
+ delete mode 100644 fs/ntfs/quota.h
+ create mode 100644 include/linux/exportfs_block.h
+ create mode 100644 include/uapi/linux/sunrpc_netlink.h
+ create mode 100644 net/sunrpc/netlink.c
+ create mode 100644 net/sunrpc/netlink.h
+ create mode 100644 tools/include/uapi/linux/openat2.h
+ rename tools/testing/selftests/{ => filesystems}/openat2/.gitignore (100%)
+ rename tools/testing/selftests/{ => filesystems}/openat2/Makefile (65%)
+ create mode 100644 tools/testing/selftests/filesystems/openat2/emptypath_test.c
+ create mode 100644 tools/testing/selftests/filesystems/openat2/helpers.h
+ rename tools/testing/selftests/{ => filesystems}/openat2/openat2_test.c (67%)
+ create mode 100644 tools/testing/selftests/filesystems/openat2/rename_attack_test.c
+ rename tools/testing/selftests/{ => filesystems}/openat2/resolve_test.c (74%)
+ delete mode 100644 tools/testing/selftests/openat2/helpers.c
+ delete mode 100644 tools/testing/selftests/openat2/helpers.h
+ delete mode 100644 tools/testing/selftests/openat2/rename_attack_test.c
+Merging printk/for-next (e56185668dc12 lib/vsprintf: Limit the returning size to INT_MAX)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/printk/linux.git printk/for-next
+Auto-merging lib/vsprintf.c
+Merge made by the 'ort' strategy.
+ lib/vsprintf.c | 51 +++++++++++++++++++++++++++------------------------
+ 1 file changed, 27 insertions(+), 24 deletions(-)
+Merging pci/next (6b96e84b16c1e Merge branch 'pci/misc')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git pci/next
+Auto-merging MAINTAINERS
+Auto-merging drivers/pci/pci-driver.c
+Auto-merging drivers/pci/pci.c
+Auto-merging drivers/pci/setup-res.c
+Merge made by the 'ort' strategy.
+ Documentation/PCI/pci.rst                          |   2 +-
+ Documentation/PCI/pciebus-howto.rst                |   2 +-
+ .../devicetree/bindings/pci/fsl,imx6q-pcie.yaml    |  32 ++
+ .../devicetree/bindings/pci/intel-gw-pcie.yaml     |   9 +-
+ .../bindings/pci/renesas,r9a08g045-pcie.yaml       |  23 +-
+ .../bindings/pci/sophgo,sg2042-pcie-host.yaml      |   3 +
+ .../bindings/pci/ultrarisc,dp1000-pcie.yaml        |  93 ++++
+ MAINTAINERS                                        |   8 +
+ arch/alpha/include/asm/pci.h                       |  13 +-
+ arch/alpha/kernel/pci-sysfs.c                      | 388 ++++++++------
+ arch/powerpc/include/asm/machdep.h                 |   2 +-
+ arch/powerpc/include/asm/pci.h                     |   2 -
+ arch/powerpc/kernel/pci-common.c                   |   2 +-
+ arch/powerpc/platforms/powernv/pci-sriov.c         |   4 +-
+ arch/powerpc/platforms/powernv/pci.h               |   5 +-
+ arch/powerpc/platforms/pseries/setup.c             |   5 +-
+ drivers/misc/pci_endpoint_test.c                   |   6 +-
+ drivers/pci/Kconfig                                |  57 --
+ drivers/pci/bus.c                                  |   1 -
+ drivers/pci/controller/dwc/Kconfig                 |  13 +
+ drivers/pci/controller/dwc/Makefile                |   1 +
+ drivers/pci/controller/dwc/pci-dra7xx.c            |  11 +-
+ drivers/pci/controller/dwc/pci-imx6.c              | 150 ++++--
+ drivers/pci/controller/dwc/pcie-al.c               |  12 +-
+ drivers/pci/controller/dwc/pcie-amd-mdb.c          |  10 +-
+ drivers/pci/controller/dwc/pcie-artpec6.c          |   9 +-
+ .../pci/controller/dwc/pcie-designware-debugfs.c   |  42 +-
+ drivers/pci/controller/dwc/pcie-designware-ep.c    | 122 ++++-
+ drivers/pci/controller/dwc/pcie-designware-host.c  |  23 +-
+ drivers/pci/controller/dwc/pcie-designware-plat.c  |   8 +-
+ drivers/pci/controller/dwc/pcie-designware.c       |  51 +-
+ drivers/pci/controller/dwc/pcie-designware.h       |  29 +
+ drivers/pci/controller/dwc/pcie-eswin.c            |   3 +-
+ drivers/pci/controller/dwc/pcie-fu740.c            |   1 -
+ drivers/pci/controller/dwc/pcie-intel-gw.c         |  74 ++-
+ drivers/pci/controller/dwc/pcie-keembay.c          |   9 +-
+ drivers/pci/controller/dwc/pcie-nxp-s32g.c         |   3 +-
+ drivers/pci/controller/dwc/pcie-qcom-common.c      |  40 +-
+ drivers/pci/controller/dwc/pcie-qcom-ep.c          |   6 +-
+ drivers/pci/controller/dwc/pcie-qcom.c             | 304 ++++++++---
+ drivers/pci/controller/dwc/pcie-tegra194.c         |  21 +-
+ drivers/pci/controller/dwc/pcie-ultrarisc.c        | 175 ++++++
+ drivers/pci/controller/dwc/pcie-visconti.c         |   1 -
+ drivers/pci/controller/pci-host-common.c           | 247 +++++++++
+ drivers/pci/controller/pci-host-common.h           |  31 ++
+ drivers/pci/controller/pci-loongson.c              |  67 ++-
+ drivers/pci/controller/pci-mvebu.c                 |   3 +-
+ drivers/pci/controller/pcie-altera.c               |  36 +-
+ drivers/pci/controller/pcie-iproc-bcma.c           |   2 +-
+ drivers/pci/controller/pcie-iproc-platform.c       |   2 +-
+ drivers/pci/controller/pcie-iproc.c                |   1 -
+ drivers/pci/controller/pcie-iproc.h                |   2 -
+ drivers/pci/controller/pcie-mediatek-gen3.c        |  42 +-
+ drivers/pci/endpoint/functions/pci-epf-test.c      |  86 ++-
+ drivers/pci/endpoint/functions/pci-epf-vntb.c      |  61 ++-
+ drivers/pci/endpoint/pci-ep-msi.c                  | 175 +++++-
+ drivers/pci/endpoint/pci-epc-core.c                |  80 +++
+ drivers/pci/ide.c                                  |   6 +-
+ drivers/pci/iov.c                                  |  16 +-
+ drivers/pci/msi/msi.c                              |  11 +-
+ drivers/pci/p2pdma.c                               |   6 +-
+ drivers/pci/pci-driver.c                           |  10 +-
+ drivers/pci/pci-sysfs.c                            | 594 ++++++++++++---------
+ drivers/pci/pci.c                                  |  80 ++-
+ drivers/pci/pci.h                                  |  46 +-
+ drivers/pci/pcie/aspm.c                            |  40 ++
+ drivers/pci/pcie/ptm.c                             |   3 +-
+ drivers/pci/probe.c                                |   7 +-
+ drivers/pci/pwrctrl/core.c                         | 116 ++--
+ drivers/pci/quirks.c                               |  55 +-
+ drivers/pci/rebar.c                                |  12 +-
+ drivers/pci/remove.c                               |   3 -
+ drivers/pci/rom.c                                  | 149 +++++-
+ drivers/pci/setup-bus.c                            |  50 +-
+ drivers/pci/setup-cardbus.c                        |   5 +-
+ drivers/pci/setup-res.c                            |  37 ++
+ drivers/pci/switch/switchtec.c                     |  16 +
+ drivers/pci/tph.c                                  |  10 +-
+ include/linux/ioport.h                             |   2 +-
+ include/linux/pci-epc.h                            |  54 ++
+ include/linux/pci-epf.h                            |  31 +-
+ include/linux/pci.h                                |  50 +-
+ include/linux/switchtec.h                          |   1 +
+ kernel/resource.c                                  |   2 +-
+ 84 files changed, 2903 insertions(+), 1119 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pci/ultrarisc,dp1000-pcie.yaml
+ create mode 100644 drivers/pci/controller/dwc/pcie-ultrarisc.c
+Merging pstore/for-next/pstore (24b8f8dcb9a13 pstore/ftrace: Factor KASLR offset in the core kernel instruction addresses)
+$ git merge -m Merge branch 'for-next/pstore' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git pstore/for-next/pstore
+Already up to date.
+Merging hid/for-next (5d1aae9252b46 Merge branches 'for-7.1/upstream-fixes', 'for-7.2/core', 'for-7.2/multitouch', 'for-7.2/oxp', 'for-7.2/playstation', 'for-7.2/rakk' and 'for-7.2/sony' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hid/hid.git hid/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS                   |    6 +
+ drivers/hid/Kconfig           |   22 +
+ drivers/hid/Makefile          |    2 +
+ drivers/hid/hid-ids.h         |   11 +
+ drivers/hid/hid-multitouch.c  |  146 +++-
+ drivers/hid/hid-oxp.c         | 1580 +++++++++++++++++++++++++++++++++++++++++
+ drivers/hid/hid-playstation.c |   31 +
+ drivers/hid/hid-rakk.c        |   75 ++
+ drivers/hid/hid-sony.c        |   17 +-
+ drivers/hid/usbhid/hid-core.c |   17 +-
+ drivers/hid/usbhid/usbkbd.c   |   15 +-
+ drivers/hid/usbhid/usbmouse.c |   15 +-
+ 12 files changed, 1906 insertions(+), 31 deletions(-)
+ create mode 100644 drivers/hid/hid-oxp.c
+ create mode 100644 drivers/hid/hid-rakk.c
+Merging i2c/i2c/for-next (023453cb7eb0f i2c: smbus: fix a potential uninitialization bug)
+$ git merge -m Merge branch 'i2c/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wsa/linux.git i2c/i2c/for-next
+Merge made by the 'ort' strategy.
+ drivers/i2c/i2c-core-smbus.c | 1 +
+ 1 file changed, 1 insertion(+)
+Merging i2c-host/i2c/i2c-host (70dc84bd48dcc dt-bindings: i2c: convert davinci i2c to dt-schema)
+$ git merge -m Merge branch 'i2c/i2c-host' of https://git.kernel.org/pub/scm/linux/kernel/git/andi.shyti/linux.git i2c-host/i2c/i2c-host
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/i2c/i2c-davinci.txt        |  43 --
+ .../devicetree/bindings/i2c/loongson,ls2x-i2c.yaml |   4 +
+ .../devicetree/bindings/i2c/ti,davinci-i2c.yaml    |  70 +++
+ MAINTAINERS                                        |   3 +-
+ drivers/i2c/busses/Kconfig                         |  11 +
+ drivers/i2c/busses/Makefile                        |   1 +
+ drivers/i2c/busses/i2c-designware-common.c         |   4 +-
+ drivers/i2c/busses/i2c-ls2x-v2.c                   | 544 +++++++++++++++++++++
+ 8 files changed, 634 insertions(+), 46 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/i2c/i2c-davinci.txt
+ create mode 100644 Documentation/devicetree/bindings/i2c/ti,davinci-i2c.yaml
+ create mode 100644 drivers/i2c/busses/i2c-ls2x-v2.c
+Merging i2c-rust/rust-i2c-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'rust-i2c-next' of https://github.com/ikrtn/rust-for-linux i2c-rust/rust-i2c-next
+Already up to date.
+Merging i3c/i3c/next (1d78a8fc97c13 i3c: dw-i3c-master: Fix IBI count register selection for versalnet)
+$ git merge -m Merge branch 'i3c/next' of https://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux.git i3c/i3c/next
+Merge made by the 'ort' strategy.
+ drivers/i3c/master.c                | 26 +++++++++++++-------------
+ drivers/i3c/master/dw-i3c-master.c  | 21 ++++++++++++++++++++-
+ drivers/i3c/master/dw-i3c-master.h  |  1 +
+ drivers/i3c/master/svc-i3c-master.c | 25 +++++++++++++++++++++----
+ 4 files changed, 55 insertions(+), 18 deletions(-)
+Merging dmi/dmi-for-next (1afafbaf749d8 firmware/dmi: Include product_family info to modalias)
+$ git merge -m Merge branch 'dmi-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jdelvare/staging.git dmi/dmi-for-next
+Already up to date.
+Merging hwmon-staging/hwmon-next (95f61e6af7d25 hwmon: (pmbus/adm1266) add rtc debugfs entry)
+$ git merge -m Merge branch 'hwmon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git hwmon-staging/hwmon-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/hwmon/lm75.yaml  |   7 +
+ .../bindings/hwmon/microchip,emc2305.yaml          |   8 +
+ .../devicetree/bindings/hwmon/nsa320-mcu.txt       |  20 -
+ .../bindings/hwmon/pmbus/adi,max20830.yaml         |  66 ++++
+ .../bindings/hwmon/zyxel,nsa320-mcu.yaml           |  54 +++
+ .../devicetree/bindings/trivial-devices.yaml       |   8 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |   2 +
+ Documentation/hwmon/adt7411.rst                    |  37 +-
+ Documentation/hwmon/arctic_fan_controller.rst      |  56 +++
+ Documentation/hwmon/coretemp.rst                   |   8 +-
+ Documentation/hwmon/d1u74t.rst                     |  81 +++++
+ Documentation/hwmon/e50sn12051.rst                 |  81 +++++
+ Documentation/hwmon/htu31.rst                      |   7 +
+ Documentation/hwmon/hwmon-kernel-api.rst           |   7 +-
+ Documentation/hwmon/index.rst                      |   6 +
+ Documentation/hwmon/lm75.rst                       |  16 +
+ Documentation/hwmon/lx1308.rst                     |  90 +++++
+ Documentation/hwmon/max20830.rst                   |  49 +++
+ Documentation/hwmon/max31730.rst                   |   2 +-
+ Documentation/hwmon/mp2985.rst                     | 147 ++++++++
+ Documentation/hwmon/nct7802.rst                    |  16 +
+ Documentation/hwmon/raspberrypi-hwmon.rst          |  15 +-
+ MAINTAINERS                                        |  30 ++
+ drivers/hwmon/Kconfig                              |  12 +
+ drivers/hwmon/Makefile                             |   1 +
+ drivers/hwmon/ad7414.c                             |   4 +-
+ drivers/hwmon/ad7418.c                             |   6 +-
+ drivers/hwmon/adc128d818.c                         |   2 +-
+ drivers/hwmon/adm1025.c                            |   4 +-
+ drivers/hwmon/adm1026.c                            |   2 +-
+ drivers/hwmon/adm1029.c                            |   2 +-
+ drivers/hwmon/adm1031.c                            |   4 +-
+ drivers/hwmon/adm1177.c                            |   4 +-
+ drivers/hwmon/adm9240.c                            |   6 +-
+ drivers/hwmon/ads7828.c                            |   4 +-
+ drivers/hwmon/ads7871.c                            | 108 +++---
+ drivers/hwmon/adt7410.c                            |   8 +-
+ drivers/hwmon/adt7411.c                            |  12 +-
+ drivers/hwmon/adt7462.c                            |   2 +-
+ drivers/hwmon/adt7470.c                            |   2 +-
+ drivers/hwmon/adt7475.c                            |   8 +-
+ drivers/hwmon/aht10.c                              |   8 +-
+ drivers/hwmon/amc6821.c                            |   2 +-
+ drivers/hwmon/applesmc.c                           |   2 +-
+ drivers/hwmon/arctic_fan_controller.c              | 374 +++++++++++++++++++
+ drivers/hwmon/asb100.c                             |   2 +-
+ drivers/hwmon/asc7621.c                            |   6 +-
+ drivers/hwmon/atxp1.c                              |   2 +-
+ drivers/hwmon/chipcap2.c                           |  16 +-
+ drivers/hwmon/coretemp.c                           |  42 +--
+ drivers/hwmon/cros_ec_hwmon.c                      |   2 +-
+ drivers/hwmon/dme1737.c                            |   4 +-
+ drivers/hwmon/ds1621.c                             |  10 +-
+ drivers/hwmon/ds620.c                              |   4 +-
+ drivers/hwmon/emc1403.c                            |  24 +-
+ drivers/hwmon/emc2103.c                            |   2 +-
+ drivers/hwmon/emc2305.c                            |  62 +++-
+ drivers/hwmon/emc6w201.c                           |   2 +-
+ drivers/hwmon/f75375s.c                            |   6 +-
+ drivers/hwmon/fschmd.c                             |  14 +-
+ drivers/hwmon/ftsteutates.c                        |   2 +-
+ drivers/hwmon/g760a.c                              |   2 +-
+ drivers/hwmon/g762.c                               |   6 +-
+ drivers/hwmon/gl518sm.c                            |   2 +-
+ drivers/hwmon/gl520sm.c                            |   2 +-
+ drivers/hwmon/hih6130.c                            |   2 +-
+ drivers/hwmon/hs3001.c                             |   4 +-
+ drivers/hwmon/htu31.c                              |   2 +-
+ drivers/hwmon/ina209.c                             |   2 +-
+ drivers/hwmon/ina238.c                             |  12 +-
+ drivers/hwmon/ina2xx.c                             |  26 +-
+ drivers/hwmon/ina3221.c                            |   2 +-
+ drivers/hwmon/intel-m10-bmc-hwmon.c                |   2 +-
+ drivers/hwmon/isl28022.c                           |   2 +-
+ drivers/hwmon/jc42.c                               |   2 +-
+ drivers/hwmon/lineage-pem.c                        |   4 +-
+ drivers/hwmon/lm63.c                               |   6 +-
+ drivers/hwmon/lm73.c                               |   2 +-
+ drivers/hwmon/lm75.c                               |  92 +++--
+ drivers/hwmon/lm77.c                               |   2 +-
+ drivers/hwmon/lm78.c                               |   4 +-
+ drivers/hwmon/lm80.c                               |   4 +-
+ drivers/hwmon/lm83.c                               |   4 +-
+ drivers/hwmon/lm85.c                               |  24 +-
+ drivers/hwmon/lm87.c                               |   4 +-
+ drivers/hwmon/lm90.c                               | 129 ++++---
+ drivers/hwmon/lm92.c                               |   6 +-
+ drivers/hwmon/lm93.c                               |   4 +-
+ drivers/hwmon/lm95234.c                            |   4 +-
+ drivers/hwmon/lm95241.c                            |   4 +-
+ drivers/hwmon/lm95245.c                            |   4 +-
+ drivers/hwmon/ltc2945.c                            |   2 +-
+ drivers/hwmon/ltc2947-i2c.c                        |   4 +-
+ drivers/hwmon/ltc2990.c                            |   4 +-
+ drivers/hwmon/ltc2991.c                            |   4 +-
+ drivers/hwmon/ltc2992.c                            |   4 +-
+ drivers/hwmon/ltc4151.c                            |   2 +-
+ drivers/hwmon/ltc4215.c                            |   2 +-
+ drivers/hwmon/ltc4222.c                            |   2 +-
+ drivers/hwmon/ltc4245.c                            |   2 +-
+ drivers/hwmon/ltc4260.c                            |   2 +-
+ drivers/hwmon/ltc4261.c                            |   4 +-
+ drivers/hwmon/max127.c                             |   2 +-
+ drivers/hwmon/max16065.c                           |  12 +-
+ drivers/hwmon/max1619.c                            |   2 +-
+ drivers/hwmon/max1668.c                            |   6 +-
+ drivers/hwmon/max31730.c                           |   2 +-
+ drivers/hwmon/max31760.c                           |   2 +-
+ drivers/hwmon/max31790.c                           |   2 +-
+ drivers/hwmon/max31827.c                           |   6 +-
+ drivers/hwmon/max6620.c                            |   2 +-
+ drivers/hwmon/max6621.c                            |   2 +-
+ drivers/hwmon/max6639.c                            |   2 +-
+ drivers/hwmon/max6650.c                            |   4 +-
+ drivers/hwmon/max6697.c                            |  20 +-
+ drivers/hwmon/mc34vr500.c                          |   4 +-
+ drivers/hwmon/mcp3021.c                            |   4 +-
+ drivers/hwmon/mcp9982.c                            |  25 +-
+ drivers/hwmon/nct6775-i2c.c                        |  26 +-
+ drivers/hwmon/nct7802.c                            |  93 ++++-
+ drivers/hwmon/nct7904.c                            |   4 +-
+ drivers/hwmon/nsa320-hwmon.c                       |   2 +-
+ drivers/hwmon/pcf8591.c                            |   2 +-
+ drivers/hwmon/pmbus/Kconfig                        |  46 +++
+ drivers/hwmon/pmbus/Makefile                       |   5 +
+ drivers/hwmon/pmbus/acbel-fsg032.c                 |   4 +-
+ drivers/hwmon/pmbus/adm1266.c                      | 221 +++++++++--
+ drivers/hwmon/pmbus/adm1275.c                      |  20 +-
+ drivers/hwmon/pmbus/aps-379.c                      |   4 +-
+ drivers/hwmon/pmbus/bel-pfe.c                      |   6 +-
+ drivers/hwmon/pmbus/bpa-rs600.c                    |   6 +-
+ drivers/hwmon/pmbus/crps.c                         |   4 +-
+ drivers/hwmon/pmbus/d1u74t.c                       |  88 +++++
+ drivers/hwmon/pmbus/delta-ahe50dc-fan.c            |   2 +-
+ drivers/hwmon/pmbus/dps920ab.c                     |   4 +-
+ drivers/hwmon/pmbus/e50sn12051.c                   |  52 +++
+ drivers/hwmon/pmbus/fsp-3y.c                       |   4 +-
+ drivers/hwmon/pmbus/hac300s.c                      |   4 +-
+ drivers/hwmon/pmbus/ibm-cffps.c                    |   8 +-
+ drivers/hwmon/pmbus/ina233.c                       |   4 +-
+ drivers/hwmon/pmbus/inspur-ipsps.c                 |   4 +-
+ drivers/hwmon/pmbus/ir35221.c                      |   4 +-
+ drivers/hwmon/pmbus/ir36021.c                      |   4 +-
+ drivers/hwmon/pmbus/ir38064.c                      |  10 +-
+ drivers/hwmon/pmbus/irps5401.c                     |   4 +-
+ drivers/hwmon/pmbus/isl68137.c                     |  92 ++---
+ drivers/hwmon/pmbus/lm25066.c                      |  10 +-
+ drivers/hwmon/pmbus/lt3074.c                       |   4 +-
+ drivers/hwmon/pmbus/lt7182s.c                      |   4 +-
+ drivers/hwmon/pmbus/ltc2978.c                      |  60 +--
+ drivers/hwmon/pmbus/ltc3815.c                      |   2 +-
+ drivers/hwmon/pmbus/lx1308.c                       | 204 +++++++++++
+ drivers/hwmon/pmbus/max15301.c                     |   8 +-
+ drivers/hwmon/pmbus/max16064.c                     |   4 +-
+ drivers/hwmon/pmbus/max16601.c                     |  10 +-
+ drivers/hwmon/pmbus/max17616.c                     |   2 +-
+ drivers/hwmon/pmbus/max20730.c                     |  10 +-
+ drivers/hwmon/pmbus/max20751.c                     |   4 +-
+ drivers/hwmon/pmbus/max20830.c                     | 110 ++++++
+ drivers/hwmon/pmbus/max31785.c                     |   8 +-
+ drivers/hwmon/pmbus/max34440.c                     |  18 +-
+ drivers/hwmon/pmbus/max8688.c                      |   2 +-
+ drivers/hwmon/pmbus/mp2856.c                       |   6 +-
+ drivers/hwmon/pmbus/mp2869.c                       |  22 +-
+ drivers/hwmon/pmbus/mp2888.c                       |   4 +-
+ drivers/hwmon/pmbus/mp2891.c                       |   2 +-
+ drivers/hwmon/pmbus/mp2925.c                       |   6 +-
+ drivers/hwmon/pmbus/mp29502.c                      |   4 +-
+ drivers/hwmon/pmbus/mp2975.c                       |   8 +-
+ drivers/hwmon/pmbus/mp2985.c                       | 402 +++++++++++++++++++++
+ drivers/hwmon/pmbus/mp2993.c                       |   2 +-
+ drivers/hwmon/pmbus/mp5920.c                       |   2 +-
+ drivers/hwmon/pmbus/mp5926.c                       |   4 +-
+ drivers/hwmon/pmbus/mp5990.c                       |   4 +-
+ drivers/hwmon/pmbus/mp9941.c                       |   2 +-
+ drivers/hwmon/pmbus/mp9945.c                       |   4 +-
+ drivers/hwmon/pmbus/mpq7932.c                      |   6 +-
+ drivers/hwmon/pmbus/mpq8785.c                      |  10 +-
+ drivers/hwmon/pmbus/pim4328.c                      |  18 +-
+ drivers/hwmon/pmbus/pli1209bc.c                    |   4 +-
+ drivers/hwmon/pmbus/pm6764tr.c                     |   4 +-
+ drivers/hwmon/pmbus/pmbus.c                        |  60 +--
+ drivers/hwmon/pmbus/pxe1610.c                      |   8 +-
+ drivers/hwmon/pmbus/q54sj108a2.c                   |   8 +-
+ drivers/hwmon/pmbus/stef48h28.c                    |   4 +-
+ drivers/hwmon/pmbus/stpddc60.c                     |   6 +-
+ drivers/hwmon/pmbus/tda38640.c                     |   4 +-
+ drivers/hwmon/pmbus/tps25990.c                     |   4 +-
+ drivers/hwmon/pmbus/tps40422.c                     |   4 +-
+ drivers/hwmon/pmbus/tps53679.c                     |  18 +-
+ drivers/hwmon/pmbus/tps546d24.c                    |   4 +-
+ drivers/hwmon/pmbus/ucd9000.c                      |  16 +-
+ drivers/hwmon/pmbus/ucd9200.c                      |  18 +-
+ drivers/hwmon/pmbus/xdp710.c                       |   2 +-
+ drivers/hwmon/pmbus/xdp720.c                       |   4 +-
+ drivers/hwmon/pmbus/xdpe12284.c                    |   8 +-
+ drivers/hwmon/pmbus/xdpe152c4.c                    |   6 +-
+ drivers/hwmon/pmbus/xdpe1a2g7b.c                   |   6 +-
+ drivers/hwmon/pmbus/zl6100.c                       |  44 +--
+ drivers/hwmon/powr1220.c                           |   4 +-
+ drivers/hwmon/pt5161l.c                            |   4 +-
+ drivers/hwmon/raspberrypi-hwmon.c                  | 140 ++++++-
+ drivers/hwmon/sbtsi_temp.c                         |   4 +-
+ drivers/hwmon/sg2042-mcu.c                         |   2 +-
+ drivers/hwmon/sht21.c                              |   6 +-
+ drivers/hwmon/sht3x.c                              |   8 +-
+ drivers/hwmon/sht4x.c                              |   4 +-
+ drivers/hwmon/shtc1.c                              |   6 +-
+ drivers/hwmon/smsc47m192.c                         |   2 +-
+ drivers/hwmon/spd5118.c                            |   2 +-
+ drivers/hwmon/stts751.c                            |   2 +-
+ drivers/hwmon/tc654.c                              |   6 +-
+ drivers/hwmon/tc74.c                               |   4 +-
+ drivers/hwmon/thmc50.c                             |   4 +-
+ drivers/hwmon/tmp102.c                             |  11 +-
+ drivers/hwmon/tmp103.c                             |   2 +-
+ drivers/hwmon/tmp108.c                             |   8 +-
+ drivers/hwmon/tmp401.c                             |  10 +-
+ drivers/hwmon/tmp421.c                             |  10 +-
+ drivers/hwmon/tmp464.c                             |   4 +-
+ drivers/hwmon/tmp513.c                             |   4 +-
+ drivers/hwmon/tsc1641.c                            |   2 +-
+ drivers/hwmon/w83773g.c                            |   2 +-
+ drivers/hwmon/w83781d.c                            |   8 +-
+ drivers/hwmon/w83791d.c                            |   2 +-
+ drivers/hwmon/w83792d.c                            |   2 +-
+ drivers/hwmon/w83793.c                             |   2 +-
+ drivers/hwmon/w83795.c                             |   4 +-
+ drivers/hwmon/w83l785ts.c                          |   2 +-
+ drivers/hwmon/w83l786ng.c                          |   2 +-
+ include/linux/hwmon.h                              |   3 +
+ include/soc/bcm2835/raspberrypi-firmware.h         |  25 ++
+ 232 files changed, 3417 insertions(+), 893 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/hwmon/nsa320-mcu.txt
+ create mode 100644 Documentation/devicetree/bindings/hwmon/pmbus/adi,max20830.yaml
+ create mode 100644 Documentation/devicetree/bindings/hwmon/zyxel,nsa320-mcu.yaml
+ create mode 100644 Documentation/hwmon/arctic_fan_controller.rst
+ create mode 100644 Documentation/hwmon/d1u74t.rst
+ create mode 100644 Documentation/hwmon/e50sn12051.rst
+ create mode 100644 Documentation/hwmon/lx1308.rst
+ create mode 100644 Documentation/hwmon/max20830.rst
+ create mode 100644 Documentation/hwmon/mp2985.rst
+ create mode 100644 drivers/hwmon/arctic_fan_controller.c
+ create mode 100644 drivers/hwmon/pmbus/d1u74t.c
+ create mode 100644 drivers/hwmon/pmbus/e50sn12051.c
+ create mode 100644 drivers/hwmon/pmbus/lx1308.c
+ create mode 100644 drivers/hwmon/pmbus/max20830.c
+ create mode 100644 drivers/hwmon/pmbus/mp2985.c
+Merging jc_docs/docs-next (4f07ec51d453a Merge branch 'docs-mw' into docs-next)
+$ git merge -m Merge branch 'docs-next' of git://git.lwn.net/linux.git jc_docs/docs-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging Documentation/filesystems/proc.rst
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../RCU/Design/Requirements/Requirements.rst       |   6 +-
+ Documentation/RCU/listRCU.rst                      |   2 +-
+ Documentation/RCU/whatisRCU.rst                    |   4 +-
+ Documentation/admin-guide/binfmt-misc.rst          |   8 +-
+ Documentation/admin-guide/kernel-parameters.txt    |  18 +-
+ Documentation/admin-guide/reporting-issues.rst     |   4 +-
+ Documentation/admin-guide/workload-tracing.rst     |  41 +-
+ Documentation/block/data-integrity.rst             |   2 +-
+ Documentation/core-api/cpu_hotplug.rst             |   5 -
+ Documentation/core-api/housekeeping.rst            |   2 +-
+ Documentation/core-api/kref.rst                    |   4 +-
+ Documentation/core-api/list.rst                    |   6 +-
+ Documentation/driver-api/eisa.rst                  |   2 +
+ Documentation/driver-api/mailbox.rst               |   4 +-
+ Documentation/driver-api/media/v4l2-fh.rst         |   2 +-
+ Documentation/filesystems/cramfs.rst               |  22 +-
+ Documentation/filesystems/locking.rst              |   2 +-
+ Documentation/filesystems/proc.rst                 |  18 +-
+ Documentation/gpu/drm-uapi.rst                     |   2 +-
+ Documentation/kernel-hacking/locking.rst           |   4 +-
+ Documentation/locking/locktypes.rst                |   4 +-
+ .../maintainer/maintainer-entry-profile.rst        |  24 +-
+ Documentation/process/coding-style.rst             |   8 +-
+ Documentation/process/deprecated.rst               |  13 +-
+ Documentation/process/maintainer-handbooks.rst     |  17 +-
+ Documentation/process/maintainers.rst              |   2 +
+ Documentation/scheduler/sched-deadline.rst         |   2 +-
+ Documentation/security/credentials.rst             |   6 +-
+ .../sound/kernel-api/writing-an-alsa-driver.rst    |  12 +-
+ Documentation/sphinx-static/custom.css             |   2 +-
+ Documentation/sphinx/maintainers_include.py        | 521 +++++++++++++++------
+ Documentation/spi/spi-summary.rst                  |   4 +-
+ Documentation/staging/crc32.rst                    |   2 +-
+ Documentation/staging/lzo.rst                      |   2 +-
+ Documentation/staging/remoteproc.rst               |   2 +-
+ Documentation/staging/static-keys.rst              |   2 +-
+ Documentation/trace/histogram-design.rst           |   2 +-
+ Documentation/trace/rv/monitor_sched.rst           |   7 +-
+ .../translations/it_IT/kernel-hacking/locking.rst  |   4 +-
+ .../translations/it_IT/locking/locktypes.rst       |   4 +-
+ .../translations/it_IT/process/coding-style.rst    |   2 +-
+ .../ja_JP/process/submitting-patches.rst           | 127 ++++-
+ Documentation/translations/pt_BR/index.rst         |   1 +
+ .../translations/pt_BR/process/license-rules.rst   | 483 +++++++++++++++++++
+ .../pt_BR/process/maintainer-handbooks.rst         |   1 +
+ .../translations/sp_SP/process/coding-style.rst    |   2 +-
+ Documentation/translations/zh_CN/core-api/kref.rst |   4 +-
+ .../translations/zh_CN/process/coding-style.rst    |   2 +-
+ .../zh_CN/video4linux/v4l2-framework.txt           |   2 +-
+ .../translations/zh_TW/process/coding-style.rst    |   2 +-
+ Documentation/watchdog/mlx-wdt.rst                 |   4 +-
+ Documentation/watchdog/pcwd-watchdog.rst           |  19 +-
+ Documentation/watchdog/watchdog-api.rst            |  38 +-
+ Documentation/watchdog/watchdog-kernel-api.rst     |  20 +-
+ Documentation/watchdog/watchdog-parameters.rst     |  11 +-
+ MAINTAINERS                                        |   8 +-
+ fs/cramfs/README                                   |  92 +---
+ tools/lib/python/kdoc/kdoc_parser.py               |  54 ++-
+ tools/lib/python/kdoc/xforms_lists.py              |  22 +-
+ tools/unittests/test_cmatch.py                     |   1 +
+ 60 files changed, 1247 insertions(+), 446 deletions(-)
+ create mode 100644 Documentation/translations/pt_BR/process/license-rules.rst
+Merging v4l-dvb/next (bb16e2895a416 media: renesas: vsp1: Drop deprecated vsp1_du_setup_lif() function)
+$ git merge -m Merge branch 'next' of git://linuxtv.org/media-ci/media-pending.git v4l-dvb/next
+Auto-merging MAINTAINERS
+Auto-merging drivers/media/platform/qcom/camss/camss-csid-gen3.c
+Auto-merging drivers/media/platform/qcom/camss/camss-csiphy.c
+Auto-merging drivers/media/platform/qcom/camss/camss.c
+Auto-merging drivers/media/platform/qcom/iris/Kconfig
+Auto-merging drivers/media/platform/qcom/iris/iris_buffer.c
+Auto-merging drivers/media/platform/qcom/iris/iris_core.c
+Auto-merging drivers/media/platform/qcom/iris/iris_hfi_common.c
+Auto-merging drivers/media/platform/qcom/iris/iris_vdec.c
+Auto-merging drivers/media/platform/qcom/iris/iris_venc.c
+Auto-merging drivers/media/platform/qcom/iris/iris_vidc.c
+Auto-merging drivers/media/platform/qcom/iris/iris_vpu_common.c
+Auto-merging drivers/media/platform/qcom/iris/iris_vpu_common.h
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/media/amdisp4-1.rst      |   63 +
+ Documentation/admin-guide/media/amdisp4.dot        |    6 +
+ Documentation/admin-guide/media/v4l-drivers.rst    |    1 +
+ .../media/allwinner,sun4i-a10-video-engine.yaml    |   10 +
+ .../bindings/media/qcom,sc7180-venus.yaml          |   15 +
+ .../bindings/media/qcom,sc7280-venus.yaml          |   10 +-
+ .../bindings/media/qcom,sm6350-camss.yaml          |  471 +++++
+ .../bindings/media/qcom,sm8250-venus.yaml          |   11 +-
+ .../bindings/media/qcom,venus-common.yaml          |   15 -
+ .../devicetree/bindings/media/renesas,fcp.yaml     |    4 +
+ .../devicetree/bindings/media/renesas,vsp1.yaml    |    2 +
+ .../bindings/media/rockchip,rk3568-mipi-csi2.yaml  |   27 +-
+ .../bindings/media/ti,j721e-csi2rx-shim.yaml       |   39 +-
+ Documentation/driver-api/media/camera-sensor.rst   |    2 +-
+ Documentation/driver-api/media/tx-rx.rst           |    3 +-
+ .../userspace-api/media/cec/cec.h.rst.exceptions   |   23 +
+ .../userspace-api/media/drivers/uvcvideo.rst       |    2 +
+ .../media/v4l/ext-ctrls-codec-stateless.rst        |    2 +-
+ .../userspace-api/media/v4l/ext-ctrls-codec.rst    |    6 +
+ .../media/v4l/ext-ctrls-image-process.rst          |    5 +-
+ MAINTAINERS                                        |   46 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c      |    4 +-
+ drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c       |    4 +-
+ drivers/media/cec/core/cec-adap.c                  |    9 +
+ drivers/media/cec/i2c/tda9950.c                    |    2 +-
+ drivers/media/cec/platform/cros-ec/cros-ec-cec.c   |    6 +
+ drivers/media/cec/platform/seco/seco-cec.c         |    6 +-
+ drivers/media/common/videobuf2/videobuf2-core.c    |   12 +-
+ drivers/media/dvb-frontends/a8293.c                |    4 +-
+ drivers/media/dvb-frontends/af9013.c               |    4 +-
+ drivers/media/dvb-frontends/af9033.c               |    4 +-
+ drivers/media/dvb-frontends/au8522_decoder.c       |    4 +-
+ drivers/media/dvb-frontends/cxd2099.c              |    4 +-
+ drivers/media/dvb-frontends/cxd2820r_core.c        |    4 +-
+ drivers/media/dvb-frontends/dvb-pll.c              |   44 +-
+ drivers/media/dvb-frontends/helene.c               |    4 +-
+ drivers/media/dvb-frontends/lgdt3306a.c            |    4 +-
+ drivers/media/dvb-frontends/lgdt330x.c             |    4 +-
+ drivers/media/dvb-frontends/m88ds3103.c            |   10 +-
+ drivers/media/dvb-frontends/mn88443x.c             |    8 +-
+ drivers/media/dvb-frontends/mn88472.c              |    4 +-
+ drivers/media/dvb-frontends/mn88473.c              |    4 +-
+ drivers/media/dvb-frontends/mxl692.c               |    4 +-
+ drivers/media/dvb-frontends/rtl2830.c              |    4 +-
+ drivers/media/dvb-frontends/rtl2832.c              |    8 +-
+ drivers/media/dvb-frontends/rtl2832_sdr.c          |   19 +-
+ drivers/media/dvb-frontends/si2165.c               |    4 +-
+ drivers/media/dvb-frontends/si2168.c               |    4 +-
+ drivers/media/dvb-frontends/sp2.c                  |    4 +-
+ drivers/media/dvb-frontends/stv090x.c              |    4 +-
+ drivers/media/dvb-frontends/stv6110x.c             |    4 +-
+ drivers/media/dvb-frontends/tc90522.c              |    6 +-
+ drivers/media/dvb-frontends/tda10071.c             |    4 +-
+ drivers/media/dvb-frontends/ts2020.c               |    6 +-
+ drivers/media/i2c/Kconfig                          |   12 +-
+ drivers/media/i2c/ad5820.c                         |    4 +-
+ drivers/media/i2c/adp1653.c                        |    2 +-
+ drivers/media/i2c/adv7170.c                        |    4 +-
+ drivers/media/i2c/adv7175.c                        |    4 +-
+ drivers/media/i2c/adv7180.c                        |   24 +-
+ drivers/media/i2c/adv7183.c                        |    4 +-
+ drivers/media/i2c/adv7343.c                        |    4 +-
+ drivers/media/i2c/adv7393.c                        |    4 +-
+ drivers/media/i2c/adv7511-v4l2.c                   |    2 +-
+ drivers/media/i2c/adv7604.c                        |   14 +-
+ drivers/media/i2c/adv7842.c                        |    2 +-
+ drivers/media/i2c/ak881x.c                         |    4 +-
+ drivers/media/i2c/alvium-csi2.c                    |   21 +-
+ drivers/media/i2c/bt819.c                          |    6 +-
+ drivers/media/i2c/bt856.c                          |    2 +-
+ drivers/media/i2c/bt866.c                          |    2 +-
+ drivers/media/i2c/cs3308.c                         |    2 +-
+ drivers/media/i2c/cs5345.c                         |    2 +-
+ drivers/media/i2c/cs53l32a.c                       |    2 +-
+ drivers/media/i2c/cx25840/cx25840-core.c           |    2 +-
+ drivers/media/i2c/ds90ub913.c                      |    4 +-
+ drivers/media/i2c/ds90ub953.c                      |    6 +-
+ drivers/media/i2c/ds90ub960.c                      |    8 +-
+ drivers/media/i2c/dw9714.c                         |    2 +-
+ drivers/media/i2c/dw9719.c                         |   10 +
+ drivers/media/i2c/et8ek8/et8ek8_driver.c           |    2 +-
+ drivers/media/i2c/imx219.c                         |   45 +-
+ drivers/media/i2c/imx274.c                         |   60 +-
+ drivers/media/i2c/imx334.c                         |   93 -
+ drivers/media/i2c/imx335.c                         |   87 -
+ drivers/media/i2c/imx412.c                         |   82 -
+ drivers/media/i2c/ir-kbd-i2c.c                     |    6 +-
+ drivers/media/i2c/isl7998x.c                       |    2 +-
+ drivers/media/i2c/ks0127.c                         |    6 +-
+ drivers/media/i2c/lm3560.c                         |  383 +++-
+ drivers/media/i2c/lm3646.c                         |    4 +-
+ drivers/media/i2c/m52790.c                         |    2 +-
+ drivers/media/i2c/max2175.c                        |    4 +-
+ drivers/media/i2c/ml86v7667.c                      |    4 +-
+ drivers/media/i2c/msp3400-driver.c                 |    2 +-
+ drivers/media/i2c/mt9m001.c                        |    2 +-
+ drivers/media/i2c/mt9m111.c                        |    2 +-
+ drivers/media/i2c/mt9p031.c                        |    3 +-
+ drivers/media/i2c/mt9t112.c                        |    2 +-
+ drivers/media/i2c/mt9v011.c                        |    2 +-
+ drivers/media/i2c/ov13858.c                        |    4 +-
+ drivers/media/i2c/ov13b10.c                        |   47 +-
+ drivers/media/i2c/ov2640.c                         |    2 +-
+ drivers/media/i2c/ov2659.c                         |    2 +-
+ drivers/media/i2c/ov5640.c                         |    4 +-
+ drivers/media/i2c/ov5645.c                         |    4 +-
+ drivers/media/i2c/ov5647.c                         |    2 +-
+ drivers/media/i2c/ov7640.c                         |    2 +-
+ drivers/media/i2c/ov7670.c                         |    4 +-
+ drivers/media/i2c/ov772x.c                         |    2 +-
+ drivers/media/i2c/ov7740.c                         |    2 +-
+ drivers/media/i2c/ov9282.c                         |   67 -
+ drivers/media/i2c/ov9640.c                         |    2 +-
+ drivers/media/i2c/ov9650.c                         |    4 +-
+ drivers/media/i2c/rj54n1cb0c.c                     |    2 +-
+ drivers/media/i2c/s5c73m3/s5c73m3-core.c           |    2 +-
+ drivers/media/i2c/s5k5baf.c                        |    2 +-
+ drivers/media/i2c/saa6588.c                        |    2 +-
+ drivers/media/i2c/saa6752hs.c                      |    2 +-
+ drivers/media/i2c/saa7110.c                        |    2 +-
+ drivers/media/i2c/saa7115.c                        |   14 +-
+ drivers/media/i2c/saa7127.c                        |   10 +-
+ drivers/media/i2c/saa717x.c                        |    2 +-
+ drivers/media/i2c/saa7185.c                        |    2 +-
+ drivers/media/i2c/sony-btf-mpx.c                   |    2 +-
+ drivers/media/i2c/tc358743.c                       |    4 +-
+ drivers/media/i2c/tda1997x.c                       |    6 +-
+ drivers/media/i2c/tda7432.c                        |    2 +-
+ drivers/media/i2c/tda9840.c                        |    2 +-
+ drivers/media/i2c/tea6415c.c                       |    2 +-
+ drivers/media/i2c/tea6420.c                        |    2 +-
+ drivers/media/i2c/ths7303.c                        |    6 +-
+ drivers/media/i2c/ths8200.c                        |    4 +-
+ drivers/media/i2c/tlv320aic23b.c                   |    2 +-
+ drivers/media/i2c/tvaudio.c                        |    2 +-
+ drivers/media/i2c/tvp514x.c                        |   63 +-
+ drivers/media/i2c/tvp5150.c                        |    2 +-
+ drivers/media/i2c/tvp7002.c                        |    2 +-
+ drivers/media/i2c/tw2804.c                         |    2 +-
+ drivers/media/i2c/tw9900.c                         |    2 +-
+ drivers/media/i2c/tw9903.c                         |    2 +-
+ drivers/media/i2c/tw9906.c                         |    2 +-
+ drivers/media/i2c/tw9910.c                         |    2 +-
+ drivers/media/i2c/uda1342.c                        |    2 +-
+ drivers/media/i2c/upd64031a.c                      |    2 +-
+ drivers/media/i2c/upd64083.c                       |    2 +-
+ drivers/media/i2c/video-i2c.c                      |   13 +-
+ drivers/media/i2c/vp27smpx.c                       |    2 +-
+ drivers/media/i2c/vpx3220.c                        |    6 +-
+ drivers/media/i2c/wm8739.c                         |    2 +-
+ drivers/media/i2c/wm8775.c                         |    2 +-
+ drivers/media/pci/Kconfig                          |    1 +
+ drivers/media/pci/Makefile                         |    1 +
+ drivers/media/pci/cx23885/cx23885-core.c           |   14 +-
+ drivers/media/pci/dm1105/dm1105.c                  |    8 +-
+ drivers/media/pci/hws/Kconfig                      |   11 +
+ drivers/media/pci/hws/Makefile                     |    4 +
+ drivers/media/pci/hws/hws.h                        |  173 ++
+ drivers/media/pci/hws/hws_irq.c                    |  269 +++
+ drivers/media/pci/hws/hws_irq.h                    |   10 +
+ drivers/media/pci/hws/hws_pci.c                    |  865 +++++++++
+ drivers/media/pci/hws/hws_reg.h                    |  136 ++
+ drivers/media/pci/hws/hws_v4l2_ioctl.c             |  919 +++++++++
+ drivers/media/pci/hws/hws_v4l2_ioctl.h             |   36 +
+ drivers/media/pci/hws/hws_video.c                  | 1490 +++++++++++++++
+ drivers/media/pci/hws/hws_video.h                  |   29 +
+ drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c   |   24 +-
+ drivers/media/pci/ivtv/ivtv-ioctl.c                |   12 +-
+ drivers/media/pci/mgb4/mgb4_vin.c                  |    8 +-
+ drivers/media/pci/mgb4/mgb4_vout.c                 |    8 +-
+ drivers/media/pci/saa7134/saa7134-video.c          |   25 +-
+ drivers/media/pci/solo6x10/Kconfig                 |    1 -
+ drivers/media/platform/Kconfig                     |    1 +
+ drivers/media/platform/Makefile                    |    1 +
+ drivers/media/platform/amd/Kconfig                 |    3 +
+ drivers/media/platform/amd/Makefile                |    3 +
+ drivers/media/platform/amd/isp4/Kconfig            |   17 +
+ drivers/media/platform/amd/isp4/Makefile           |   10 +
+ drivers/media/platform/amd/isp4/isp4.c             |  240 +++
+ drivers/media/platform/amd/isp4/isp4.h             |   20 +
+ drivers/media/platform/amd/isp4/isp4_debug.c       |  271 +++
+ drivers/media/platform/amd/isp4/isp4_debug.h       |   41 +
+ drivers/media/platform/amd/isp4/isp4_fw_cmd_resp.h |  318 ++++
+ drivers/media/platform/amd/isp4/isp4_hw_reg.h      |  124 ++
+ drivers/media/platform/amd/isp4/isp4_interface.c   |  832 ++++++++
+ drivers/media/platform/amd/isp4/isp4_interface.h   |  144 ++
+ drivers/media/platform/amd/isp4/isp4_subdev.c      | 1047 ++++++++++
+ drivers/media/platform/amd/isp4/isp4_subdev.h      |  127 ++
+ drivers/media/platform/amd/isp4/isp4_video.c       |  797 ++++++++
+ drivers/media/platform/amd/isp4/isp4_video.h       |   57 +
+ .../media/platform/amlogic/c3/isp/c3-isp-params.c  |    4 +
+ .../media/platform/arm/mali-c55/mali-c55-core.c    |   24 +-
+ drivers/media/platform/arm/mali-c55/mali-c55-isp.c |    8 +
+ .../media/platform/arm/mali-c55/mali-c55-resizer.c |    1 +
+ drivers/media/platform/arm/mali-c55/mali-c55-tpg.c |    1 +
+ drivers/media/platform/aspeed/aspeed-video.c       |    1 +
+ drivers/media/platform/cadence/Kconfig             |    1 +
+ drivers/media/platform/cadence/cdns-csi2rx.c       |  455 +++--
+ .../platform/chips-media/wave5/wave5-helper.c      |    4 +-
+ .../platform/chips-media/wave5/wave5-helper.h      |    2 +-
+ .../media/platform/chips-media/wave5/wave5-hw.c    |    7 +-
+ .../platform/chips-media/wave5/wave5-vpu-dec.c     |   47 +-
+ .../platform/chips-media/wave5/wave5-vpu-enc.c     |   73 +-
+ .../platform/chips-media/wave5/wave5-vpuapi.c      |   11 +-
+ .../platform/chips-media/wave5/wave5-vpuapi.h      |    2 +
+ drivers/media/platform/marvell/cafe-driver.c       |    1 +
+ .../media/platform/mediatek/jpeg/mtk_jpeg_core.c   |    3 +-
+ drivers/media/platform/nuvoton/npcm-video.c        |   34 +-
+ drivers/media/platform/nvidia/tegra-vde/Kconfig    |    1 +
+ .../media/platform/nxp/imx8-isi/imx8-isi-core.c    |    4 +-
+ .../media/platform/nxp/imx8-isi/imx8-isi-core.h    |   16 +
+ .../platform/nxp/imx8-isi/imx8-isi-crossbar.c      |    1 +
+ drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c  |    9 +-
+ drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c |   11 +-
+ .../media/platform/nxp/imx8-isi/imx8-isi-pipe.c    |   20 +-
+ .../media/platform/nxp/imx8-isi/imx8-isi-video.c   |    6 +-
+ drivers/media/platform/qcom/camss/Makefile         |   12 +-
+ drivers/media/platform/qcom/camss/camss-csid-340.c |   18 +-
+ drivers/media/platform/qcom/camss/camss-csid-680.c |   44 +-
+ .../media/platform/qcom/camss/camss-csid-gen2.c    |   47 +-
+ .../media/platform/qcom/camss/camss-csid-gen3.c    |   42 +-
+ drivers/media/platform/qcom/camss/camss-csid.c     |   45 +-
+ drivers/media/platform/qcom/camss/camss-csid.h     |    1 +
+ .../platform/qcom/camss/camss-csiphy-3ph-1-0.c     |  125 ++
+ drivers/media/platform/qcom/camss/camss-csiphy.c   |    1 +
+ drivers/media/platform/qcom/camss/camss-csiphy.h   |    2 +
+ drivers/media/platform/qcom/camss/camss-format.c   |   14 +
+ drivers/media/platform/qcom/camss/camss-format.h   |    1 +
+ drivers/media/platform/qcom/camss/camss-tpg-gen1.c |  231 +++
+ drivers/media/platform/qcom/camss/camss-tpg.c      |  519 +++++
+ drivers/media/platform/qcom/camss/camss-tpg.h      |  118 ++
+ drivers/media/platform/qcom/camss/camss-vfe.c      |   31 +-
+ drivers/media/platform/qcom/camss/camss-video.c    |   19 +-
+ drivers/media/platform/qcom/camss/camss.c          |  433 ++++-
+ drivers/media/platform/qcom/camss/camss.h          |    6 +
+ drivers/media/platform/qcom/iris/Kconfig           |    1 +
+ drivers/media/platform/qcom/iris/Makefile          |    9 +-
+ drivers/media/platform/qcom/iris/iris_buffer.c     |   88 +-
+ drivers/media/platform/qcom/iris/iris_common.c     |    8 +-
+ drivers/media/platform/qcom/iris/iris_core.c       |    3 +-
+ drivers/media/platform/qcom/iris/iris_core.h       |   14 +-
+ drivers/media/platform/qcom/iris/iris_ctrls.c      |   54 +-
+ drivers/media/platform/qcom/iris/iris_firmware.c   |   13 +-
+ drivers/media/platform/qcom/iris/iris_hfi_common.c |    6 +-
+ drivers/media/platform/qcom/iris/iris_hfi_common.h |   13 +-
+ .../iris/{iris_platform_gen1.c => iris_hfi_gen1.c} |  170 +-
+ drivers/media/platform/qcom/iris/iris_hfi_gen1.h   |    6 +-
+ .../platform/qcom/iris/iris_hfi_gen1_command.c     |   43 +-
+ .../platform/qcom/iris/iris_hfi_gen1_response.c    |   11 +-
+ .../iris/{iris_platform_gen2.c => iris_hfi_gen2.c} |  435 +----
+ drivers/media/platform/qcom/iris/iris_hfi_gen2.h   |    5 +-
+ .../platform/qcom/iris/iris_hfi_gen2_command.c     |  115 +-
+ .../platform/qcom/iris/iris_hfi_gen2_packet.c      |   18 +-
+ .../platform/qcom/iris/iris_hfi_gen2_response.c    |   11 +-
+ drivers/media/platform/qcom/iris/iris_instance.h   |    4 +
+ .../platform/qcom/iris/iris_platform_common.h      |  118 +-
+ .../platform/qcom/iris/iris_platform_sm8250.h      |   29 +
+ .../platform/qcom/iris/iris_platform_sm8550.h      |   31 +
+ .../media/platform/qcom/iris/iris_platform_vpu2.c  |  132 ++
+ .../media/platform/qcom/iris/iris_platform_vpu3x.c |  219 +++
+ drivers/media/platform/qcom/iris/iris_probe.c      |   19 +-
+ drivers/media/platform/qcom/iris/iris_utils.c      |    5 +-
+ drivers/media/platform/qcom/iris/iris_vb2.c        |    2 +-
+ drivers/media/platform/qcom/iris/iris_vdec.c       |    6 +-
+ drivers/media/platform/qcom/iris/iris_venc.c       |    4 +-
+ drivers/media/platform/qcom/iris/iris_vidc.c       |   14 +-
+ drivers/media/platform/qcom/iris/iris_vpu_common.c |   11 +-
+ drivers/media/platform/qcom/iris/iris_vpu_common.h |    2 +
+ drivers/media/platform/qcom/venus/core.c           |   13 +-
+ drivers/media/platform/qcom/venus/core.h           |   12 +
+ drivers/media/platform/qcom/venus/pm_helpers.c     |    8 +-
+ .../platform/renesas/rzg2l-cru/rzg2l-cru-regs.h    |    4 +-
+ .../media/platform/renesas/rzg2l-cru/rzg2l-cru.h   |   29 +-
+ .../media/platform/renesas/rzg2l-cru/rzg2l-video.c |  328 ++--
+ .../platform/renesas/rzv2h-ivc/rzv2h-ivc-video.c   |   31 +-
+ drivers/media/platform/renesas/vsp1/vsp1_brx.c     |   33 +-
+ drivers/media/platform/renesas/vsp1/vsp1_clu.c     |   15 +-
+ drivers/media/platform/renesas/vsp1/vsp1_dl.c      |   63 +-
+ drivers/media/platform/renesas/vsp1/vsp1_drm.c     |  263 +--
+ drivers/media/platform/renesas/vsp1/vsp1_entity.c  |   27 +-
+ drivers/media/platform/renesas/vsp1/vsp1_hgo.c     |   10 +-
+ drivers/media/platform/renesas/vsp1/vsp1_hgt.c     |   16 +-
+ drivers/media/platform/renesas/vsp1/vsp1_histo.c   |   55 +-
+ drivers/media/platform/renesas/vsp1/vsp1_hsit.c    |   15 +-
+ drivers/media/platform/renesas/vsp1/vsp1_lut.c     |   15 +-
+ drivers/media/platform/renesas/vsp1/vsp1_pipe.c    |   60 +-
+ drivers/media/platform/renesas/vsp1/vsp1_rwpf.c    |   44 +-
+ drivers/media/platform/renesas/vsp1/vsp1_sru.c     |   13 +-
+ drivers/media/platform/renesas/vsp1/vsp1_uds.c     |   13 +-
+ drivers/media/platform/renesas/vsp1/vsp1_uif.c     |   29 +-
+ drivers/media/platform/renesas/vsp1/vsp1_video.c   |  157 +-
+ drivers/media/platform/renesas/vsp1/vsp1_wpf.c     |   30 +-
+ .../media/platform/rockchip/rkisp1/rkisp1-params.c |   68 +
+ .../media/platform/rockchip/rkisp1/rkisp1-regs.h   |   15 +-
+ .../platform/rockchip/rkvdec/rkvdec-bitwriter.h    |   39 +
+ .../platform/rockchip/rkvdec/rkvdec-h264-common.c  |   51 +-
+ .../platform/rockchip/rkvdec/rkvdec-h264-common.h  |   40 +-
+ .../media/platform/rockchip/rkvdec/rkvdec-h264.c   |  109 +-
+ .../platform/rockchip/rkvdec/rkvdec-hevc-common.c  |   93 +-
+ .../platform/rockchip/rkvdec/rkvdec-hevc-common.h  |   57 +-
+ .../media/platform/rockchip/rkvdec/rkvdec-hevc.c   |  171 +-
+ .../platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c |  337 ++--
+ .../platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c |  502 +++--
+ drivers/media/platform/st/stm32/stm32-dcmi.c       |    1 +
+ .../st/stm32/stm32-dcmipp/dcmipp-bytecap.c         |    6 +-
+ drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c |    6 +-
+ drivers/media/platform/synopsys/dw-mipi-csi2rx.c   |  128 +-
+ .../media/platform/synopsys/hdmirx/snps_hdmirx.c   |    4 +-
+ drivers/media/platform/ti/Kconfig                  |    1 +
+ drivers/media/platform/ti/davinci/vpif_capture.c   |    2 +-
+ .../media/platform/ti/j721e-csi2rx/j721e-csi2rx.c  | 1232 +++++++++---
+ drivers/media/platform/ti/vpe/vip.c                |   16 +-
+ drivers/media/platform/ti/vpe/vpe.c                |    3 +-
+ drivers/media/platform/verisilicon/hantro_hw.h     |    1 -
+ drivers/media/platform/verisilicon/hantro_v4l2.c   |    6 +-
+ drivers/media/radio/radio-si476x.c                 |    1 +
+ drivers/media/radio/radio-tea5764.c                |    2 +-
+ drivers/media/radio/saa7706h.c                     |    4 +-
+ drivers/media/radio/si470x/radio-si470x-i2c.c      |    2 +-
+ drivers/media/radio/si470x/radio-si470x-usb.c      |   11 +-
+ drivers/media/radio/si4713/si4713.c                |    2 +-
+ drivers/media/radio/tef6862.c                      |    4 +-
+ drivers/media/rc/imon.c                            |    4 +
+ drivers/media/rc/imon_raw.c                        |   18 +-
+ drivers/media/rc/ir_toy.c                          |   23 +-
+ drivers/media/rc/mceusb.c                          |    2 +
+ drivers/media/test-drivers/vidtv/vidtv_bridge.c    |    4 +-
+ drivers/media/test-drivers/vidtv/vidtv_demod.c     |    4 +-
+ drivers/media/test-drivers/vidtv/vidtv_mux.c       |    8 +-
+ drivers/media/test-drivers/vidtv/vidtv_tuner.c     |    4 +-
+ drivers/media/test-drivers/vimc/vimc-core.c        |    1 +
+ drivers/media/test-drivers/visl/visl-core.c        |    4 +
+ drivers/media/test-drivers/vivid/vivid-core.c      |    6 +-
+ drivers/media/test-drivers/vivid/vivid-ctrls.c     |   15 +-
+ drivers/media/test-drivers/vivid/vivid-vid-cap.c   |   38 +-
+ drivers/media/test-drivers/vivid/vivid-vid-cap.h   |    1 +
+ drivers/media/test-drivers/vivid/vivid-vid-out.c   |    6 +
+ drivers/media/tuners/e4000.c                       |    4 +-
+ drivers/media/tuners/fc2580.c                      |    4 +-
+ drivers/media/tuners/m88rs6000t.c                  |    4 +-
+ drivers/media/tuners/mt2060.c                      |    4 +-
+ drivers/media/tuners/mxl301rf.c                    |    4 +-
+ drivers/media/tuners/qm1d1b0004.c                  |    4 +-
+ drivers/media/tuners/qm1d1c0042.c                  |    4 +-
+ drivers/media/tuners/si2157.c                      |   10 +-
+ drivers/media/tuners/tda18212.c                    |    4 +-
+ drivers/media/tuners/tda18250.c                    |    4 +-
+ drivers/media/tuners/tua9001.c                     |    4 +-
+ drivers/media/usb/airspy/airspy.c                  |    8 +-
+ drivers/media/usb/cx231xx/cx231xx-cards.c          |   16 +-
+ drivers/media/usb/em28xx/em28xx-video.c            |    4 +-
+ drivers/media/usb/go7007/s2250-board.c             |    2 +-
+ drivers/media/usb/gspca/gspca.c                    |   17 +-
+ drivers/media/usb/gspca/sonixb.c                   |    2 +-
+ drivers/media/usb/gspca/touptek.c                  |   17 +-
+ drivers/media/usb/hdpvr/hdpvr-core.c               |   26 +-
+ drivers/media/usb/msi2500/msi2500.c                |   32 +-
+ drivers/media/usb/pwc/pwc-if.c                     |   13 +-
+ drivers/media/usb/s2255/s2255drv.c                 |   12 +-
+ drivers/media/usb/uvc/uvc_ctrl.c                   |  212 +++
+ drivers/media/usb/uvc/uvc_driver.c                 |    4 +
+ drivers/media/usb/uvc/uvc_v4l2.c                   |   11 +
+ drivers/media/usb/uvc/uvc_video.c                  |  109 +-
+ drivers/media/usb/uvc/uvcvideo.h                   |    2 +
+ drivers/media/v4l2-core/tuner-core.c               |    2 +-
+ drivers/media/v4l2-core/v4l2-common.c              |   14 +-
+ drivers/media/v4l2-core/v4l2-ctrls-core.c          |   13 +
+ drivers/media/v4l2-core/v4l2-ctrls-defs.c          |    2 +
+ drivers/media/v4l2-core/v4l2-ctrls-request.c       |   14 +-
+ drivers/media/v4l2-core/v4l2-dev.c                 |    5 +
+ drivers/media/v4l2-core/v4l2-subdev.c              |   43 +-
+ .../x86/intel/int3472/tps68470_board_data.c        |  160 +-
+ drivers/staging/media/Kconfig                      |    4 -
+ drivers/staging/media/Makefile                     |    1 -
+ drivers/staging/media/atomisp/i2c/atomisp-gc2235.c |   31 +-
+ drivers/staging/media/atomisp/i2c/ov2722.h         |  649 +------
+ .../staging/media/atomisp/include/linux/atomisp.h  |  182 +-
+ .../atomisp/include/linux/atomisp_gmin_platform.h  |    7 +-
+ .../media/atomisp/include/linux/atomisp_platform.h |   42 +-
+ drivers/staging/media/atomisp/pci/atomisp_cmd.c    |  198 +-
+ drivers/staging/media/atomisp/pci/atomisp_cmd.h    |    2 +-
+ .../media/atomisp/pci/atomisp_compat_css20.c       |   26 +-
+ .../media/atomisp/pci/atomisp_compat_css20.h       |    2 +-
+ .../media/atomisp/pci/atomisp_csi2_bridge.c        |    2 +-
+ .../media/atomisp/pci/atomisp_gmin_platform.c      |    6 +-
+ drivers/staging/media/atomisp/pci/atomisp_subdev.c |   20 +-
+ drivers/staging/media/atomisp/pci/atomisp_subdev.h |   12 +-
+ drivers/staging/media/atomisp/pci/atomisp_v4l2.c   |    5 +-
+ .../pci/base/circbuf/interface/ia_css_circbuf.h    |    9 +-
+ .../base/circbuf/interface/ia_css_circbuf_desc.h   |   11 +-
+ .../media/atomisp/pci/base/refcount/src/refcount.c |    3 +-
+ .../atomisp/pci/camera/pipe/src/pipe_stagedesc.c   |    9 +-
+ .../media/atomisp/pci/camera/pipe/src/pipe_util.c  |    3 +-
+ .../host/isys_stream2mmio_private.h                |    3 +-
+ .../hive_isp_css_common/host/event_fifo_private.h  |    3 +-
+ .../atomisp/pci/hive_isp_css_common/host/gdc.c     |   22 +-
+ .../pci/hive_isp_css_common/host/input_formatter.c |    3 +-
+ .../pci/hive_isp_css_common/host/input_system.c    |   14 +-
+ .../atomisp/pci/hive_isp_css_common/host/irq.c     |   30 +-
+ .../atomisp/pci/hive_isp_css_common/host/mmu.c     |    3 +-
+ .../atomisp/pci/hive_isp_css_common/host/vmem.c    |    4 +-
+ .../pci/hive_isp_css_include/host/mmu_public.h     |   87 +-
+ .../pci/hive_isp_css_include/math_support.h        |    6 -
+ drivers/staging/media/atomisp/pci/hmm/hmm_bo.c     |   27 +-
+ drivers/staging/media/atomisp/pci/ia_css_types.h   |   14 +-
+ .../pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h     |    8 +-
+ .../pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c  |   26 +-
+ .../pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h  |   20 +-
+ .../pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c   |   17 +-
+ .../pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h   |   13 +-
+ .../pci/isp/kernels/bh/bh_2/ia_css_bh.host.c       |   14 +-
+ .../pci/isp/kernels/bh/bh_2/ia_css_bh.host.h       |   14 +-
+ .../pci/isp/kernels/bnlm/ia_css_bnlm.host.c        |    6 +-
+ .../pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c    |    7 +-
+ .../pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c  |   14 +-
+ .../pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c    |   13 +-
+ .../pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c  |    7 +-
+ .../pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c  |    3 +-
+ .../isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c   |    6 +-
+ .../isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c    |   13 +-
+ .../pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c     |   25 +-
+ .../media/atomisp/pci/runtime/binary/src/binary.c  |  114 +-
+ .../atomisp/pci/runtime/debug/src/ia_css_debug.c   |   17 +-
+ .../atomisp/pci/runtime/isp_param/src/isp_param.c  |   24 +-
+ .../atomisp/pci/runtime/isys/src/virtual_isys.c    |    3 +-
+ .../atomisp/pci/runtime/pipeline/src/pipeline.c    |    3 +-
+ .../media/atomisp/pci/runtime/queue/src/queue.c    |    4 +-
+ drivers/staging/media/atomisp/pci/sh_css.c         |  158 +-
+ .../staging/media/atomisp/pci/sh_css_firmware.c    |    6 +-
+ .../staging/media/atomisp/pci/sh_css_param_dvs.c   |   12 +-
+ .../media/atomisp/pci/sh_css_param_shading.c       |   22 +-
+ drivers/staging/media/atomisp/pci/sh_css_params.c  |   74 +-
+ drivers/staging/media/atomisp/pci/sh_css_sp.c      |   41 +-
+ drivers/staging/media/atomisp/pci/system_local.c   |    3 +-
+ drivers/staging/media/av7110/av7110.c              |   49 +-
+ drivers/staging/media/av7110/av7110.h              |    4 +-
+ drivers/staging/media/av7110/av7110_av.c           |   89 +-
+ drivers/staging/media/av7110/av7110_ca.c           |    3 +-
+ drivers/staging/media/av7110/av7110_hw.c           |   46 -
+ drivers/staging/media/av7110/av7110_hw.h           |   71 +-
+ drivers/staging/media/av7110/av7110_ir.c           |    4 +-
+ drivers/staging/media/av7110/av7110_v4l.c          |   22 +-
+ drivers/staging/media/deprecated/atmel/Kconfig     |   47 -
+ drivers/staging/media/deprecated/atmel/Makefile    |    8 -
+ drivers/staging/media/deprecated/atmel/TODO        |   34 -
+ .../media/deprecated/atmel/atmel-isc-base.c        | 2008 --------------------
+ .../staging/media/deprecated/atmel/atmel-isc-clk.c |  311 ---
+ .../media/deprecated/atmel/atmel-isc-regs.h        |  413 ----
+ drivers/staging/media/deprecated/atmel/atmel-isc.h |  362 ----
+ .../media/deprecated/atmel/atmel-sama5d2-isc.c     |  644 -------
+ .../media/deprecated/atmel/atmel-sama7g5-isc.c     |  607 ------
+ drivers/staging/media/ipu3/ipu3-css-params.c       |    8 +-
+ drivers/staging/media/ipu7/abi/ipu7_fw_boot_abi.h  |   78 +-
+ .../staging/media/ipu7/abi/ipu7_fw_common_abi.h    |    4 +-
+ drivers/staging/media/ipu7/abi/ipu7_fw_msg_abi.h   |   12 +-
+ drivers/staging/media/ipu7/ipu7-buttress-regs.h    |   10 +-
+ drivers/staging/media/ipu7/ipu7.c                  |   22 +-
+ drivers/staging/media/sunxi/cedrus/cedrus.c        |    5 +-
+ drivers/staging/media/sunxi/cedrus/cedrus_h264.c   |    3 +
+ drivers/staging/media/tegra-video/tegra210.c       |    3 +-
+ drivers/staging/media/tegra-video/vi.c             |    4 +-
+ include/linux/usb/uvc.h                            |   10 +
+ include/media/i2c/lm3560.h                         |   84 -
+ include/media/media-entity.h                       |   10 +-
+ include/media/videobuf2-core.h                     |    8 +-
+ include/media/vsp1.h                               |    5 +-
+ include/uapi/linux/cec-funcs.h                     |  182 ++
+ include/uapi/linux/cec.h                           |   31 +-
+ include/uapi/linux/rkisp1-config.h                 |  113 +-
+ include/uapi/linux/v4l2-controls.h                 |    2 +
+ 470 files changed, 16552 insertions(+), 10447 deletions(-)
+ create mode 100644 Documentation/admin-guide/media/amdisp4-1.rst
+ create mode 100644 Documentation/admin-guide/media/amdisp4.dot
+ create mode 100644 Documentation/devicetree/bindings/media/qcom,sm6350-camss.yaml
+ create mode 100644 drivers/media/pci/hws/Kconfig
+ create mode 100644 drivers/media/pci/hws/Makefile
+ create mode 100644 drivers/media/pci/hws/hws.h
+ create mode 100644 drivers/media/pci/hws/hws_irq.c
+ create mode 100644 drivers/media/pci/hws/hws_irq.h
+ create mode 100644 drivers/media/pci/hws/hws_pci.c
+ create mode 100644 drivers/media/pci/hws/hws_reg.h
+ create mode 100644 drivers/media/pci/hws/hws_v4l2_ioctl.c
+ create mode 100644 drivers/media/pci/hws/hws_v4l2_ioctl.h
+ create mode 100644 drivers/media/pci/hws/hws_video.c
+ create mode 100644 drivers/media/pci/hws/hws_video.h
+ create mode 100644 drivers/media/platform/amd/Kconfig
+ create mode 100644 drivers/media/platform/amd/Makefile
+ create mode 100644 drivers/media/platform/amd/isp4/Kconfig
+ create mode 100644 drivers/media/platform/amd/isp4/Makefile
+ create mode 100644 drivers/media/platform/amd/isp4/isp4.c
+ create mode 100644 drivers/media/platform/amd/isp4/isp4.h
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_debug.c
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_debug.h
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_fw_cmd_resp.h
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_hw_reg.h
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_interface.c
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_interface.h
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_subdev.c
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_subdev.h
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_video.c
+ create mode 100644 drivers/media/platform/amd/isp4/isp4_video.h
+ create mode 100644 drivers/media/platform/qcom/camss/camss-tpg-gen1.c
+ create mode 100644 drivers/media/platform/qcom/camss/camss-tpg.c
+ create mode 100644 drivers/media/platform/qcom/camss/camss-tpg.h
+ rename drivers/media/platform/qcom/iris/{iris_platform_gen1.c => iris_hfi_gen1.c} (60%)
+ rename drivers/media/platform/qcom/iris/{iris_platform_gen2.c => iris_hfi_gen2.c} (60%)
+ create mode 100644 drivers/media/platform/qcom/iris/iris_platform_sm8250.h
+ create mode 100644 drivers/media/platform/qcom/iris/iris_platform_sm8550.h
+ create mode 100644 drivers/media/platform/qcom/iris/iris_platform_vpu2.c
+ create mode 100644 drivers/media/platform/qcom/iris/iris_platform_vpu3x.c
+ create mode 100644 drivers/media/platform/rockchip/rkvdec/rkvdec-bitwriter.h
+ delete mode 100644 drivers/staging/media/deprecated/atmel/Kconfig
+ delete mode 100644 drivers/staging/media/deprecated/atmel/Makefile
+ delete mode 100644 drivers/staging/media/deprecated/atmel/TODO
+ delete mode 100644 drivers/staging/media/deprecated/atmel/atmel-isc-base.c
+ delete mode 100644 drivers/staging/media/deprecated/atmel/atmel-isc-clk.c
+ delete mode 100644 drivers/staging/media/deprecated/atmel/atmel-isc-regs.h
+ delete mode 100644 drivers/staging/media/deprecated/atmel/atmel-isc.h
+ delete mode 100644 drivers/staging/media/deprecated/atmel/atmel-sama5d2-isc.c
+ delete mode 100644 drivers/staging/media/deprecated/atmel/atmel-sama7g5-isc.c
+ delete mode 100644 include/media/i2c/lm3560.h
+Merging v4l-dvb-next/master (adc218676eef2 Linux 6.12)
+$ git merge -m Merge branch 'master' of git://linuxtv.org/mchehab/media-next.git v4l-dvb-next/master
+Already up to date.
+Merging pm/linux-next (b2f2673027958 Merge branch 'fixes' into linux-next)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rafael/linux-pm.git pm/linux-next
+Auto-merging include/linux/hwmon.h
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/pm/amd-pstate.rst   |  11 +-
+ Documentation/admin-guide/pm/intel_pstate.rst |  11 +-
+ drivers/acpi/battery.c                        |   4 +-
+ drivers/acpi/pmic/intel_pmic.c                |  37 ++-
+ drivers/cpufreq/Kconfig.x86                   |  12 -
+ drivers/cpufreq/amd-pstate-ut.c               |  36 ++-
+ drivers/cpufreq/amd-pstate.c                  |  27 +-
+ drivers/cpufreq/intel_pstate.c                |   3 +-
+ drivers/hwmon/hwmon.c                         |   6 +-
+ drivers/thermal/thermal_core.c                | 342 ++++++++++++--------------
+ drivers/thermal/thermal_core.h                |   2 -
+ drivers/thermal/thermal_hwmon.c               | 228 ++++++-----------
+ include/linux/hwmon.h                         |   3 +-
+ tools/power/pm-graph/sleepgraph.py            |   2 +-
+ 14 files changed, 311 insertions(+), 413 deletions(-)
+Merging cpufreq-arm/cpufreq/arm/linux-next (9a303892acac5 cpufreq: cppc: mask Desired_Excursion when autonomous selection is enabled)
+$ git merge -m Merge branch 'cpufreq/arm/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git cpufreq-arm/cpufreq/arm/linux-next
+Auto-merging drivers/cpufreq/amd-pstate.c
+Merge made by the 'ort' strategy.
+ drivers/cpufreq/Kconfig.arm         |  2 +-
+ drivers/cpufreq/amd-pstate.c        | 26 ++++++++++----------------
+ drivers/cpufreq/apple-soc-cpufreq.c |  6 ++----
+ drivers/cpufreq/cppc_cpufreq.c      | 29 ++++++++++++++++++++++++++++-
+ drivers/cpufreq/qcom-cpufreq-hw.c   |  1 -
+ 5 files changed, 41 insertions(+), 23 deletions(-)
+Merging cpupower/cpupower (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'cpupower' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux.git cpupower/cpupower
+Already up to date.
+Merging devfreq/devfreq-next (c096be11c2a9d PM / devfreq: Fix governor_store() failing when device has no current governor)
+$ git merge -m Merge branch 'devfreq-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/linux.git devfreq/devfreq-next
+Merge made by the 'ort' strategy.
+ drivers/devfreq/devfreq-event.c      | 23 +++++++++--------
+ drivers/devfreq/devfreq.c            | 50 +++++++-----------------------------
+ drivers/devfreq/event/rockchip-dfi.c |  4 ++-
+ 3 files changed, 25 insertions(+), 52 deletions(-)
+Merging pmdomain/next (fba9c703c1845 pmdomain: arm_scmi: add support for domain hierarchies)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/linux-pm.git pmdomain/next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/power/power-domain.yaml    |  34 +++++
+ drivers/pmdomain/arm/scmi_pm_domain.c              |  15 +-
+ drivers/pmdomain/core.c                            | 167 +++++++++++++++++++++
+ include/linux/pm_domain.h                          |  16 ++
+ 4 files changed, 231 insertions(+), 1 deletion(-)
+Merging opp/opp/linux-next (69f888381d2ec OPP: of: Fix potential memory leak in opp_parse_supplies())
+$ git merge -m Merge branch 'opp/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vireshk/pm.git opp/opp/linux-next
+Merge made by the 'ort' strategy.
+ drivers/opp/core.c | 5 ++---
+ drivers/opp/of.c   | 3 ++-
+ 2 files changed, 4 insertions(+), 4 deletions(-)
+Merging thermal/thermal/linux-next (bf746e2a41efd thermal: renesas: rzg3e: Remove stale @trim_offset kernel-doc entry)
+$ git merge -m Merge branch 'thermal/linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/thermal/linux.git thermal/thermal/linux-next
+Already up to date.
+Merging rdma/for-next (67464f388d52e RDMA/cma: Constify struct configfs_item_operations and configfs_group_operations)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git rdma/for-next
+Auto-merging drivers/infiniband/core/addr.c
+Auto-merging drivers/infiniband/hw/bnxt_re/ib_verbs.c
+Auto-merging drivers/infiniband/hw/hns/hns_roce_qp.c
+Auto-merging drivers/infiniband/hw/hns/hns_roce_srq.c
+Auto-merging drivers/infiniband/hw/mana/cq.c
+Auto-merging drivers/infiniband/hw/mana/qp.c
+CONFLICT (content): Merge conflict in drivers/infiniband/hw/mana/qp.c
+Auto-merging drivers/infiniband/hw/mlx4/srq.c
+CONFLICT (content): Merge conflict in drivers/infiniband/hw/mlx4/srq.c
+Auto-merging drivers/infiniband/hw/mlx5/main.c
+Auto-merging drivers/infiniband/hw/mlx5/qp.c
+Auto-merging drivers/infiniband/hw/mlx5/umr.c
+Auto-merging drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+Auto-merging drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+Auto-merging drivers/infiniband/sw/rxe/rxe_recv.c
+Resolved 'drivers/infiniband/hw/mana/qp.c' using previous resolution.
+Resolved 'drivers/infiniband/hw/mlx4/srq.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master fa54452554cf7] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git
+$ git diff -M --stat --summary HEAD^..
+ drivers/infiniband/core/addr.c                     |   2 +-
+ drivers/infiniband/core/cma_configfs.c             |   6 +-
+ drivers/infiniband/hw/bng_re/bng_fw.c              |   5 +-
+ drivers/infiniband/hw/bnxt_re/ib_verbs.c           |   2 +-
+ drivers/infiniband/hw/cxgb4/cq.c                   |  11 +-
+ drivers/infiniband/hw/cxgb4/provider.c             |  14 +-
+ drivers/infiniband/hw/cxgb4/qp.c                   |  10 +-
+ drivers/infiniband/hw/efa/Kconfig                  |   3 +-
+ drivers/infiniband/hw/efa/efa_admin_cmds_defs.h    |   3 -
+ drivers/infiniband/hw/efa/efa_admin_defs.h         |  15 +-
+ drivers/infiniband/hw/efa/efa_com.c                |  50 +++-
+ drivers/infiniband/hw/efa/efa_com.h                |   4 +-
+ drivers/infiniband/hw/efa/efa_verbs.c              |  87 ++-----
+ drivers/infiniband/hw/erdma/erdma_verbs.c          |  13 +-
+ drivers/infiniband/hw/hns/hns_roce_ah.c            |   4 +-
+ drivers/infiniband/hw/hns/hns_roce_cq.c            |   3 +-
+ drivers/infiniband/hw/hns/hns_roce_debugfs.c       | 287 ++++++++++++++++++++-
+ drivers/infiniband/hw/hns/hns_roce_debugfs.h       |  26 ++
+ drivers/infiniband/hw/hns/hns_roce_device.h        |  21 ++
+ drivers/infiniband/hw/hns/hns_roce_hem.c           |   8 +-
+ drivers/infiniband/hw/hns/hns_roce_hw_v2.c         |  86 +++++-
+ drivers/infiniband/hw/hns/hns_roce_hw_v2.h         | 125 +++++++++
+ drivers/infiniband/hw/hns/hns_roce_main.c          |   3 +-
+ drivers/infiniband/hw/hns/hns_roce_pd.c            |   8 +-
+ drivers/infiniband/hw/hns/hns_roce_qp.c            |  13 +-
+ drivers/infiniband/hw/hns/hns_roce_srq.c           |   6 +-
+ drivers/infiniband/hw/ionic/ionic_controlpath.c    |   8 +-
+ drivers/infiniband/hw/irdma/verbs.c                |  48 ++--
+ drivers/infiniband/hw/mana/cq.c                    |   6 +-
+ drivers/infiniband/hw/mana/qp.c                    |  22 +-
+ drivers/infiniband/hw/mlx4/alias_GUID.c            |   2 +-
+ drivers/infiniband/hw/mlx4/cq.c                    |   7 +-
+ drivers/infiniband/hw/mlx4/main.c                  |  31 ++-
+ drivers/infiniband/hw/mlx4/qp.c                    |   9 +-
+ drivers/infiniband/hw/mlx4/srq.c                   |  12 +-
+ drivers/infiniband/hw/mlx5/ah.c                    |   2 +-
+ drivers/infiniband/hw/mlx5/cq.c                    |  10 +-
+ drivers/infiniband/hw/mlx5/devx.c                  |  66 +++--
+ drivers/infiniband/hw/mlx5/main.c                  |  31 ++-
+ drivers/infiniband/hw/mlx5/mr.c                    |   2 +-
+ drivers/infiniband/hw/mlx5/qp.c                    |  25 +-
+ drivers/infiniband/hw/mlx5/srq.c                   |   7 +-
+ drivers/infiniband/hw/mlx5/umr.c                   |   2 +-
+ drivers/infiniband/hw/mthca/mthca_provider.c       |  40 ++-
+ drivers/infiniband/hw/ocrdma/ocrdma_verbs.c        |  31 +--
+ drivers/infiniband/hw/qedr/verbs.c                 |  48 +---
+ drivers/infiniband/hw/usnic/usnic_ib_verbs.c       |  13 +-
+ drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c       |   7 +-
+ drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c       |   8 +-
+ drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c      |   6 +-
+ drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c    |  11 +-
+ drivers/infiniband/sw/rdmavt/cq.c                  |   2 +-
+ drivers/infiniband/sw/rdmavt/qp.c                  |   3 +-
+ drivers/infiniband/sw/rdmavt/srq.c                 |  19 +-
+ drivers/infiniband/sw/rxe/Makefile                 |   1 +
+ drivers/infiniband/sw/rxe/rxe_hw_counters.c        |   2 +
+ drivers/infiniband/sw/rxe/rxe_hw_counters.h        |   2 +
+ drivers/infiniband/sw/rxe/rxe_loc.h                |   6 +
+ drivers/infiniband/sw/rxe/rxe_mad.c                |  97 +++++++
+ drivers/infiniband/sw/rxe/rxe_mcast.c              |   4 +-
+ drivers/infiniband/sw/rxe/rxe_net.c                |   9 +-
+ drivers/infiniband/sw/rxe/rxe_recv.c               |   2 +
+ drivers/infiniband/sw/rxe/rxe_verbs.c              |   5 +-
+ drivers/infiniband/sw/rxe/rxe_verbs.h              |  12 +-
+ drivers/infiniband/sw/siw/siw.h                    |   5 +-
+ drivers/infiniband/sw/siw/siw_mem.c                |  56 ++--
+ drivers/infiniband/sw/siw/siw_mem.h                |   5 +-
+ drivers/infiniband/sw/siw/siw_verbs.c              |  10 +-
+ drivers/infiniband/ulp/srpt/ib_srpt.c              |   5 +-
+ tools/testing/selftests/rdma/Makefile              |   3 +-
+ .../testing/selftests/rdma/rxe_sent_rcvd_bytes.sh  |  75 ++++++
+ 71 files changed, 1112 insertions(+), 480 deletions(-)
+ create mode 100644 drivers/infiniband/sw/rxe/rxe_mad.c
+ create mode 100755 tools/testing/selftests/rdma/rxe_sent_rcvd_bytes.sh
+Merging net-next/main (830d8771ae3c7 Merge branch 'udp_tunnel-speed-up-udp-tunnel-device-destruction-part-ii')
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git net-next/main
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Auto-merging drivers/dpll/zl3073x/dpll.c
+Auto-merging drivers/infiniband/sw/rxe/rxe_net.c
+Auto-merging drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+Auto-merging drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+Auto-merging drivers/net/ovpn/udp.c
+Auto-merging include/linux/skbuff.h
+Auto-merging include/net/tcp.h
+Auto-merging net/bridge/br_multicast.c
+Auto-merging net/core/devmem.c
+Auto-merging net/core/skbuff.c
+Auto-merging net/ipv4/tcp.c
+Auto-merging net/ipv4/tcp_input.c
+Auto-merging net/ipv6/tcp_ipv6.c
+Auto-merging net/mac80211/cfg.c
+Auto-merging net/mac80211/mlme.c
+Auto-merging net/mac80211/parse.c
+Auto-merging net/mac80211/rx.c
+Auto-merging net/wireless/wext-compat.c
+Auto-merging tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh
+Auto-merging tools/testing/selftests/net/mptcp/mptcp_join.sh
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt    |    6 +-
+ Documentation/arch/m68k/kernel-options.rst         |   24 +-
+ .../devicetree/bindings/net/microchip,lan95xx.yaml |    2 +
+ .../bindings/net/microchip,lan966x-switch.yaml     |   10 +-
+ Documentation/driver-api/dpll.rst                  |   58 +-
+ .../acpi/dsd/motorcomm-yt8xxx-phy.rst              |  107 ++
+ Documentation/firmware-guide/acpi/index.rst        |    1 +
+ Documentation/netlink/specs/dpll.yaml              |   59 +-
+ Documentation/netlink/specs/netdev.yaml            |    6 +
+ Documentation/netlink/specs/rt-link.yaml           |    3 +
+ .../device_drivers/ethernet/cirrus/cs89x0.rst      |  647 ----------
+ .../networking/device_drivers/ethernet/index.rst   |    1 -
+ Documentation/networking/devlink/stmmac.rst        |    2 +-
+ Documentation/networking/ip-sysctl.rst             |    2 +-
+ .../networking/net_cachelines/net_device.rst       |    2 +-
+ .../networking/net_cachelines/tcp_sock.rst         |   10 +-
+ Documentation/networking/netmem.rst                |    8 +-
+ Documentation/networking/smc-sysctl.rst            |    8 +-
+ Documentation/networking/tcp_ao.rst                |   38 +-
+ .../translations/zh_CN/networking/netmem.rst       |    7 +-
+ Documentation/userspace-api/netlink/intro.rst      |    4 +-
+ MAINTAINERS                                        |   11 +-
+ drivers/atm/solos-pci.c                            |    4 +-
+ drivers/dibs/Kconfig                               |   10 +-
+ drivers/dpll/dpll_core.c                           |    3 +-
+ drivers/dpll/dpll_netlink.c                        |   57 +-
+ drivers/dpll/dpll_nl.c                             |    5 +-
+ drivers/dpll/dpll_nl.h                             |    2 +-
+ drivers/dpll/zl3073x/chan.c                        |   31 +-
+ drivers/dpll/zl3073x/chan.h                        |   14 +
+ drivers/dpll/zl3073x/core.c                        |   45 -
+ drivers/dpll/zl3073x/dpll.c                        |  148 ++-
+ drivers/dpll/zl3073x/i2c.c                         |   10 +-
+ drivers/dpll/zl3073x/ref.h                         |   14 -
+ drivers/dpll/zl3073x/regs.h                        |   24 +-
+ drivers/gpio/gpio-ath79.c                          |   57 +-
+ drivers/infiniband/hw/mlx5/counters.c              |    4 +-
+ drivers/infiniband/hw/mlx5/ib_rep.c                |    6 +-
+ drivers/infiniband/sw/rxe/rxe_net.c                |    6 +-
+ drivers/infiniband/sw/rxe/rxe_ns.c                 |    4 +-
+ drivers/net/Kconfig                                |    7 -
+ drivers/net/Makefile                               |    1 -
+ drivers/net/Space.c                                |  237 ----
+ drivers/net/amt.c                                  |   80 +-
+ drivers/net/arcnet/com20020-pci.c                  |  242 ++--
+ drivers/net/bareudp.c                              |   52 +-
+ drivers/net/can/sja1000/plx_pci.c                  |  167 +--
+ drivers/net/dsa/lan9303_i2c.c                      |    2 +-
+ drivers/net/dsa/microchip/ksz8.c                   |  487 +++++--
+ drivers/net/dsa/microchip/ksz8.h                   |   62 +-
+ drivers/net/dsa/microchip/ksz8863_smi.c            |    8 +-
+ drivers/net/dsa/microchip/ksz9477.c                |  323 ++++-
+ drivers/net/dsa/microchip/ksz9477.h                |   46 +-
+ drivers/net/dsa/microchip/ksz9477_i2c.c            |   12 +-
+ drivers/net/dsa/microchip/ksz_common.c             |  758 ++---------
+ drivers/net/dsa/microchip/ksz_common.h             |  111 +-
+ drivers/net/dsa/microchip/ksz_spi.c                |    8 +-
+ drivers/net/dsa/microchip/lan937x.h                |   21 +-
+ drivers/net/dsa/microchip/lan937x_main.c           |  157 ++-
+ drivers/net/dsa/mv88e6xxx/chip.c                   |   24 +-
+ drivers/net/dsa/mv88e6xxx/chip.h                   |    4 -
+ drivers/net/dsa/mv88e6xxx/port.c                   |   55 +-
+ drivers/net/dsa/mv88e6xxx/port.h                   |   11 +-
+ drivers/net/dsa/ocelot/felix.c                     |    3 +-
+ drivers/net/dsa/realtek/rtl8365mb.c                |   14 +
+ drivers/net/dsa/sja1105/sja1105_main.c             |    3 +-
+ drivers/net/dsa/xrs700x/xrs700x_i2c.c              |    4 +-
+ drivers/net/dsa/yt921x.c                           |  507 +++++++-
+ drivers/net/dsa/yt921x.h                           |   87 +-
+ drivers/net/dummy.c                                |    6 +-
+ drivers/net/ethernet/3com/3c59x.c                  |   80 +-
+ drivers/net/ethernet/3com/typhoon.c                |   75 +-
+ drivers/net/ethernet/8390/Kconfig                  |    1 -
+ drivers/net/ethernet/8390/ne.c                     |  237 +++-
+ drivers/net/ethernet/8390/ne2k-pci.c               |   24 +-
+ drivers/net/ethernet/Kconfig                       |    1 +
+ drivers/net/ethernet/Makefile                      |    1 +
+ drivers/net/ethernet/adaptec/starfire.c            |    4 +-
+ drivers/net/ethernet/agere/et131x.c                |    6 +-
+ drivers/net/ethernet/airoha/airoha_eth.c           |   37 +-
+ drivers/net/ethernet/airoha/airoha_eth.h           |    8 +-
+ drivers/net/ethernet/airoha/airoha_ppe.c           |   15 +-
+ drivers/net/ethernet/alibaba/Kconfig               |   28 +
+ drivers/net/ethernet/alibaba/Makefile              |    5 +
+ drivers/net/ethernet/alibaba/eea/Makefile          |    9 +
+ drivers/net/ethernet/alibaba/eea/eea_adminq.c      |  542 ++++++++
+ drivers/net/ethernet/alibaba/eea/eea_adminq.h      |   83 ++
+ drivers/net/ethernet/alibaba/eea/eea_desc.h        |  138 ++
+ drivers/net/ethernet/alibaba/eea/eea_ethtool.c     |  273 ++++
+ drivers/net/ethernet/alibaba/eea/eea_ethtool.h     |   48 +
+ drivers/net/ethernet/alibaba/eea/eea_net.c         |  887 +++++++++++++
+ drivers/net/ethernet/alibaba/eea/eea_net.h         |  198 +++
+ drivers/net/ethernet/alibaba/eea/eea_pci.c         |  744 +++++++++++
+ drivers/net/ethernet/alibaba/eea/eea_pci.h         |   73 ++
+ drivers/net/ethernet/alibaba/eea/eea_ring.c        |  249 ++++
+ drivers/net/ethernet/alibaba/eea/eea_ring.h        |   99 ++
+ drivers/net/ethernet/alibaba/eea/eea_rx.c          |  814 ++++++++++++
+ drivers/net/ethernet/alibaba/eea/eea_tx.c          |  500 ++++++++
+ drivers/net/ethernet/atheros/atlx/atl2.c           |   37 +-
+ drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c   |  260 ++--
+ drivers/net/ethernet/broadcom/bnx2.c               |   62 +-
+ drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c   |   50 +-
+ drivers/net/ethernet/broadcom/bnxt/bnxt.c          |    3 +-
+ drivers/net/ethernet/cavium/liquidio/lio_main.c    |   10 +-
+ drivers/net/ethernet/cavium/liquidio/lio_vf_main.c |    7 +-
+ drivers/net/ethernet/chelsio/cxgb/common.h         |    2 +-
+ drivers/net/ethernet/chelsio/cxgb/subr.c           |    2 +-
+ drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c    |    4 +-
+ drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c    |    4 +-
+ .../net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c    |    4 +-
+ drivers/net/ethernet/cirrus/Kconfig                |   20 +-
+ drivers/net/ethernet/cirrus/cs89x0.c               |  636 +--------
+ drivers/net/ethernet/dec/tulip/de2104x.c           |    6 +-
+ drivers/net/ethernet/dec/tulip/dmfe.c              |   12 +-
+ drivers/net/ethernet/dec/tulip/tulip_core.c        |   78 +-
+ drivers/net/ethernet/dec/tulip/uli526x.c           |    6 +-
+ drivers/net/ethernet/dec/tulip/winbond-840.c       |   13 +-
+ drivers/net/ethernet/dlink/dl2k.h                  |   12 +-
+ drivers/net/ethernet/dlink/sundance.c              |   14 +-
+ drivers/net/ethernet/fealnx.c                      |    8 +-
+ drivers/net/ethernet/google/gve/gve_main.c         |   17 +-
+ drivers/net/ethernet/hisilicon/hibmcge/hbg_main.c  |    2 +-
+ drivers/net/ethernet/hisilicon/hns3/hns3_enet.c    |   50 +-
+ .../ethernet/hisilicon/hns3/hns3pf/hclge_main.c    |   18 +-
+ .../ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c  |   12 +-
+ drivers/net/ethernet/huawei/hinic/hinic_main.c     |   12 +-
+ drivers/net/ethernet/huawei/hinic3/hinic3_lld.c    |    7 +-
+ drivers/net/ethernet/intel/e100.c                  |    9 +-
+ drivers/net/ethernet/intel/e1000e/netdev.c         |  471 +++++--
+ drivers/net/ethernet/intel/fm10k/fm10k_pci.c       |   10 +-
+ drivers/net/ethernet/intel/i40e/i40e_main.c        |   59 +-
+ drivers/net/ethernet/intel/i40e/i40e_ptp.c         |    9 +-
+ drivers/net/ethernet/intel/i40e/i40e_register.h    |   10 +
+ drivers/net/ethernet/intel/iavf/iavf_main.c        |   10 +-
+ drivers/net/ethernet/intel/ice/ice_dpll.c          |    2 +-
+ drivers/net/ethernet/intel/ice/ice_ethtool.c       |    2 +-
+ drivers/net/ethernet/intel/ice/ice_fw_update.c     |    2 +-
+ drivers/net/ethernet/intel/igb/igb_main.c          |   66 +-
+ drivers/net/ethernet/intel/igbvf/netdev.c          |    4 +-
+ drivers/net/ethernet/intel/igc/igc_main.c          |   34 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_e610.c      |   73 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_e610.h      |    1 +
+ drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c   |  224 +++-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_main.c      |  136 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_type.h      |    2 +
+ drivers/net/ethernet/intel/ixgbe/ixgbe_type_e610.h |   32 +-
+ drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c      |    2 +-
+ drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c  |   49 +-
+ drivers/net/ethernet/mellanox/mlx4/main.c          |    6 +-
+ drivers/net/ethernet/mellanox/mlx5/core/alloc.c    |  387 +++++-
+ drivers/net/ethernet/mellanox/mlx5/core/cmd.c      |    1 +
+ drivers/net/ethernet/mellanox/mlx5/core/debugfs.c  |   39 +-
+ drivers/net/ethernet/mellanox/mlx5/core/dpll.c     |    6 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c   |    9 -
+ drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h   |    2 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/rss.c   |   29 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en/rss.h   |    6 +-
+ .../net/ethernet/mellanox/mlx5/core/en/rx_res.c    |   40 +-
+ .../net/ethernet/mellanox/mlx5/core/en/rx_res.h    |    4 +-
+ .../ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c |    5 +-
+ .../mellanox/mlx5/core/en_accel/ktls_txrx.h        |   12 +-
+ .../net/ethernet/mellanox/mlx5/core/en_ethtool.c   |  108 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_main.c  |   53 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_tx.c    |    4 +-
+ drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c  |   35 +-
+ .../ethernet/mellanox/mlx5/core/esw/devlink_port.c |    7 +-
+ .../net/ethernet/mellanox/mlx5/core/esw/ipsec.c    |    4 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.c  |  115 +-
+ drivers/net/ethernet/mellanox/mlx5/core/eswitch.h  |   21 +-
+ .../ethernet/mellanox/mlx5/core/eswitch_offloads.c |  345 +++--
+ drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c  |  169 ++-
+ drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h  |    5 +
+ .../net/ethernet/mellanox/mlx5/core/lag/mpesw.c    |   18 +-
+ .../net/ethernet/mellanox/mlx5/core/lib/devcom.c   |    8 +
+ .../net/ethernet/mellanox/mlx5/core/lib/devcom.h   |    1 +
+ .../mellanox/mlx5/core/lib/ipsec_fs_roce.c         |   10 +
+ drivers/net/ethernet/mellanox/mlx5/core/main.c     |   43 +-
+ .../net/ethernet/mellanox/mlx5/core/mlx5_core.h    |    3 +
+ .../net/ethernet/mellanox/mlx5/core/pagealloc.c    |  250 +++-
+ .../net/ethernet/mellanox/mlx5/core/sf/devlink.c   |    5 +
+ .../mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h    |    8 -
+ drivers/net/ethernet/mellanox/mlx5/core/sriov.c    |    7 +-
+ .../ethernet/mellanox/mlx5/core/steering/hws/bwc.c |   12 +
+ .../mellanox/mlx5/core/steering/hws/table.c        |    3 +
+ .../mellanox/mlx5/core/steering/hws/vport.c        |    2 +-
+ .../mellanox/mlx5/core/steering/sws/dr_types.h     |    1 -
+ drivers/net/ethernet/mellanox/mlxsw/minimal.c      |    2 +-
+ drivers/net/ethernet/mellanox/mlxsw/reg.h          |    8 +
+ drivers/net/ethernet/mellanox/mlxsw/spectrum.c     |   16 +-
+ .../net/ethernet/mellanox/mlxsw/spectrum_ethtool.c |    4 +
+ drivers/net/ethernet/meta/fbnic/fbnic_csr.h        |    1 +
+ drivers/net/ethernet/meta/fbnic/fbnic_mdio.c       |  137 +-
+ drivers/net/ethernet/meta/fbnic/fbnic_netdev.c     |    2 +-
+ drivers/net/ethernet/micrel/ksz884x.c              |    8 +-
+ drivers/net/ethernet/microsoft/mana/gdma_main.c    |    3 +
+ drivers/net/ethernet/microsoft/mana/mana_bpf.c     |    2 +-
+ drivers/net/ethernet/microsoft/mana/mana_en.c      |   67 +-
+ drivers/net/ethernet/microsoft/mana/mana_ethtool.c |    2 +-
+ drivers/net/ethernet/mucse/rnpgbe/rnpgbe_main.c    |   10 +-
+ drivers/net/ethernet/natsemi/natsemi.c             |    4 +-
+ drivers/net/ethernet/netronome/nfp/nfp_main.c      |   65 +-
+ .../net/ethernet/netronome/nfp/nfp_netvf_main.c    |   33 +-
+ drivers/net/ethernet/qlogic/qede/qede_main.c       |   20 +-
+ drivers/net/ethernet/qualcomm/Kconfig              |    4 +-
+ drivers/net/ethernet/realtek/8139too.c             |   52 +-
+ drivers/net/ethernet/realtek/r8169_main.c          |    8 +-
+ drivers/net/ethernet/realtek/rtase/rtase.h         |    4 +
+ drivers/net/ethernet/realtek/rtase/rtase_main.c    |    3 +
+ drivers/net/ethernet/rocker/rocker_main.c          |    4 +-
+ drivers/net/ethernet/sis/sis190.c                  |    6 +-
+ drivers/net/ethernet/sis/sis900.c                  |   10 +-
+ drivers/net/ethernet/smsc/epic100.c                |   18 +-
+ drivers/net/ethernet/sun/cassini.c                 |    8 +-
+ drivers/net/ethernet/sun/sungem.c                  |   26 +-
+ drivers/net/ethernet/ti/am65-cpsw-nuss.c           |    6 +-
+ drivers/net/ethernet/ti/tlan.c                     |   41 +-
+ drivers/net/ethernet/wangxun/ngbe/ngbe_main.c      |   26 +-
+ drivers/net/ethernet/wangxun/ngbevf/ngbevf_main.c  |   26 +-
+ drivers/net/ethernet/wangxun/txgbe/txgbe_main.c    |   18 +-
+ .../net/ethernet/wangxun/txgbevf/txgbevf_main.c    |   18 +-
+ drivers/net/ethernet/wiznet/Kconfig                |   59 +-
+ drivers/net/ethernet/wiznet/Makefile               |    4 +-
+ drivers/net/ethernet/wiznet/w5100-spi.c            |    2 +-
+ drivers/net/ethernet/wiznet/w5100.c                |  412 +-----
+ drivers/net/ethernet/wiznet/w5100.h                |    3 +-
+ drivers/net/ethernet/wiznet/w5300.c                |  687 ----------
+ drivers/net/geneve.c                               |   56 +-
+ drivers/net/gtp.c                                  |   10 +-
+ drivers/net/ipvlan/ipvlan_core.c                   |    2 +-
+ drivers/net/mctp/mctp-i2c.c                        |    4 +-
+ drivers/net/mdio/Kconfig                           |    2 +-
+ drivers/net/mdio/mdio-mscc-miim.c                  |    2 +-
+ drivers/net/netconsole.c                           |  175 +++
+ drivers/net/netdevsim/psp.c                        |   21 +-
+ drivers/net/netkit.c                               |    1 +
+ drivers/net/ovpn/udp.c                             |    2 +-
+ drivers/net/pfcp.c                                 |   17 +-
+ drivers/net/phy/Kconfig                            |    2 +-
+ drivers/net/phy/aquantia/aquantia_main.c           |    6 +-
+ drivers/net/phy/dp83867.c                          |   60 +
+ drivers/net/phy/intel-xway.c                       |   79 ++
+ drivers/net/phy/motorcomm.c                        |   41 +-
+ drivers/net/phy/realtek/realtek_main.c             |  326 ++++-
+ drivers/net/ppp/Kconfig                            |    6 +
+ drivers/net/ppp/Makefile                           |    6 +-
+ drivers/net/ppp/pppoe.c                            |  173 ++-
+ drivers/net/pse-pd/pd692x0.c                       |    2 +-
+ drivers/net/pse-pd/si3474.c                        |    4 +-
+ drivers/net/pse-pd/tps23881.c                      |    4 +-
+ drivers/net/tun.c                                  |  109 +-
+ drivers/net/usb/pegasus.c                          |   16 +-
+ drivers/net/usb/qmi_wwan.c                         |    2 +
+ drivers/net/usb/r8152.c                            |  368 +++++-
+ drivers/net/usb/usbnet.c                           |   10 +-
+ drivers/net/vxlan/vxlan_core.c                     |   63 +-
+ drivers/net/vxlan/vxlan_multicast.c                |    8 +-
+ drivers/net/wan/farsync.c                          |   24 +-
+ drivers/net/wan/pc300too.c                         |   14 +-
+ drivers/net/wan/pci200syn.c                        |    6 +-
+ drivers/net/wan/wanxl.c                            |   11 +-
+ drivers/net/wireguard/socket.c                     |    8 +-
+ drivers/net/wireless/ath/ath12k/ahb.c              |   25 +-
+ drivers/net/wireless/ath/ath12k/core.c             |    6 +
+ drivers/net/wireless/ath/ath12k/dp_htt.c           |    2 +-
+ drivers/net/wireless/ath/ath12k/dp_htt.h           |    2 +-
+ drivers/net/wireless/ath/ath12k/htc.h              |    8 +-
+ drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h     |    1 -
+ drivers/net/wireless/ath/ath12k/wmi.c              |  217 ++--
+ drivers/net/wireless/ath/ath12k/wmi.h              |    3 +
+ drivers/net/wireless/ath/ath6kl/cfg80211.c         |    3 +-
+ drivers/net/wireless/ath/ath9k/hw.c                |   33 +-
+ drivers/net/wireless/ath/ath9k/hw.h                |    3 +-
+ drivers/net/wireless/ath/wil6210/cfg80211.c        |    3 +-
+ .../net/wireless/broadcom/brcm80211/brcmfmac/p2p.c |    4 +-
+ .../net/wireless/broadcom/brcm80211/brcmfmac/p2p.h |    3 +-
+ .../wireless/broadcom/brcm80211/brcmfmac/pcie.c    |   17 +-
+ .../broadcom/brcm80211/brcmsmac/phy/phy_lcn.c      |   13 +-
+ drivers/net/wireless/intel/ipw2x00/ipw2200.c       |   52 +-
+ drivers/net/wireless/intel/iwlwifi/mld/iface.c     |    2 +-
+ drivers/net/wireless/intel/iwlwifi/mld/mac80211.c  |    6 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c  |   10 +-
+ .../net/wireless/intel/iwlwifi/mvm/mld-mac80211.c  |    4 +-
+ drivers/net/wireless/marvell/mwifiex/cfg80211.c    |    3 +-
+ drivers/net/wireless/mediatek/mt76/mt7925/mcu.c    |    4 +-
+ drivers/net/wireless/microchip/wilc1000/cfg80211.c |    3 +-
+ drivers/net/wireless/quantenna/qtnfmac/commands.c  |    4 +-
+ drivers/net/wireless/realtek/rtw89/fw.c            |    2 +-
+ drivers/net/wireless/virtual/Makefile              |    2 +
+ drivers/net/wireless/virtual/mac80211_hwsim_i.h    |  168 +++
+ .../{mac80211_hwsim.c => mac80211_hwsim_main.c}    |  974 +++++++-------
+ drivers/net/wireless/virtual/mac80211_hwsim_nan.c  | 1346 ++++++++++++++++++++
+ drivers/net/wireless/virtual/mac80211_hwsim_nan.h  |  102 ++
+ drivers/net/wwan/Kconfig                           |    4 +-
+ drivers/ptp/ptp_vmw.c                              |    3 -
+ drivers/vhost/net.c                                |   21 +-
+ include/linux/dpll.h                               |   22 +-
+ include/linux/ieee80211-eht.h                      |  129 +-
+ include/linux/ieee80211-uhr.h                      |   55 +-
+ include/linux/ieee80211.h                          |    1 +
+ include/linux/if_bridge.h                          |    1 +
+ include/linux/if_tun.h                             |    3 +
+ include/linux/igmp.h                               |  167 ++-
+ include/linux/ks8851_mll.h                         |   21 -
+ include/linux/mlx5/driver.h                        |   19 +-
+ include/linux/mlx5/eswitch.h                       |    8 +-
+ include/linux/mlx5/mlx5_ifc.h                      |   73 +-
+ include/linux/mlx5/vport.h                         |    4 +-
+ include/linux/net/intel/libie/adminq.h             |    1 +
+ include/linux/netdevice.h                          |   17 +-
+ include/linux/netpoll.h                            |    2 +-
+ include/linux/platform_data/wiznet.h               |   23 -
+ include/linux/ptr_ring.h                           |   20 +-
+ include/linux/skbuff.h                             |    3 +-
+ include/linux/socket.h                             |    2 +-
+ include/linux/tcp.h                                |   40 +-
+ include/net/Space.h                                |    7 -
+ include/net/act_api.h                              |    4 +-
+ include/net/amt.h                                  |    2 +-
+ include/net/cfg80211.h                             |  276 +++-
+ include/net/dsa.h                                  |    3 +-
+ include/net/gen_stats.h                            |   16 +-
+ include/net/mac80211.h                             |   74 +-
+ include/net/mana/mana.h                            |    2 +-
+ include/net/mld.h                                  |  187 ++-
+ include/net/netlink.h                              |    5 +-
+ include/net/sch_generic.h                          |   85 +-
+ include/net/smc.h                                  |    2 +-
+ include/net/sock.h                                 |   11 +-
+ include/net/tcp.h                                  |   53 +-
+ include/net/tcp_ao.h                               |   74 +-
+ include/net/udp_tunnel.h                           |   14 +-
+ include/net/vxlan.h                                |    5 +-
+ include/trace/events/skb.h                         |    4 +-
+ include/uapi/linux/dpll.h                          |   23 +
+ include/uapi/linux/if_bridge.h                     |    1 +
+ include/uapi/linux/if_link.h                       |   17 +
+ include/uapi/linux/mdio.h                          |   10 +
+ include/uapi/linux/nl80211.h                       |  303 ++++-
+ include/uapi/linux/pkt_sched.h                     |   10 +
+ include/uapi/linux/pps.h                           |    2 +-
+ include/uapi/linux/rtnetlink.h                     |    1 +
+ net/bridge/br_arp_nd_proxy.c                       |   38 +-
+ net/bridge/br_device.c                             |    2 +-
+ net/bridge/br_fdb.c                                |    7 +-
+ net/bridge/br_forward.c                            |   15 +-
+ net/bridge/br_input.c                              |    2 +-
+ net/bridge/br_multicast.c                          |   24 +-
+ net/bridge/br_netlink.c                            |    8 +-
+ net/bridge/br_private.h                            |    3 +
+ net/bridge/br_vlan.c                               |    1 +
+ net/bridge/br_vlan_options.c                       |   24 +-
+ net/core/dev.c                                     |   28 +-
+ net/core/devmem.c                                  |    6 +-
+ net/core/devmem.h                                  |   10 +-
+ net/core/drop_monitor.c                            |    6 +-
+ net/core/gen_stats.c                               |   51 +-
+ net/core/netdev-genl-gen.c                         |   38 +-
+ net/core/netdev-genl.c                             |   65 +-
+ net/core/netpoll.c                                 |  182 +--
+ net/core/page_pool_user.c                          |   47 +-
+ net/core/rtnetlink.c                               |   33 +-
+ net/core/skbuff.c                                  |    7 +-
+ net/core/sock.c                                    |    3 +
+ net/core/sock_destructor.h                         |   12 -
+ net/dsa/user.c                                     |    2 +-
+ net/ethtool/rings.c                                |    4 +-
+ net/ethtool/tsconfig.c                             |    4 -
+ net/hsr/hsr_netlink.c                              |    7 +-
+ net/ipv4/Kconfig                                   |    8 +-
+ net/ipv4/Makefile                                  |    1 -
+ net/ipv4/af_inet.c                                 |    2 +
+ net/ipv4/fou_core.c                                |   21 +-
+ net/ipv4/igmp.c                                    |    6 +-
+ net/ipv4/inet_fragment.c                           |    2 -
+ net/ipv4/ipmr_base.c                               |    2 +-
+ net/ipv4/nexthop.c                                 |   88 +-
+ net/ipv4/tcp.c                                     |   22 +-
+ net/ipv4/tcp_ao.c                                  |  669 +++++-----
+ net/ipv4/tcp_input.c                               |    2 +-
+ net/ipv4/tcp_output.c                              |   23 +-
+ net/ipv4/tcp_sigpool.c                             |  366 ------
+ net/ipv4/udp_tunnel_core.c                         |   23 +-
+ net/ipv6/addrconf.c                                |   13 +-
+ net/ipv6/inet6_connection_sock.c                   |    3 +-
+ net/ipv6/ioam6.c                                   |    4 +-
+ net/ipv6/ip6_offload.c                             |    2 +
+ net/ipv6/ip6_udp_tunnel.c                          |    6 +-
+ net/ipv6/ip6mr.c                                   |   22 +-
+ net/ipv6/mcast.c                                   |   19 +-
+ net/ipv6/tcp_ao.c                                  |  131 +-
+ net/ipv6/tcp_ipv6.c                                |    2 +-
+ net/kcm/kcmsock.c                                  |   16 +-
+ net/l2tp/l2tp_core.c                               |    2 +-
+ net/llc/af_llc.c                                   |   18 +-
+ net/llc/llc_c_ac.c                                 |    8 +-
+ net/mac80211/Makefile                              |    2 +-
+ net/mac80211/ap.c                                  |  207 +++
+ net/mac80211/cfg.c                                 |  213 +++-
+ net/mac80211/chan.c                                |  168 ++-
+ net/mac80211/debugfs.c                             |   47 +-
+ net/mac80211/eht.c                                 |  179 +--
+ net/mac80211/he.c                                  |    9 +-
+ net/mac80211/ht.c                                  |   59 +-
+ net/mac80211/ibss.c                                |    3 +
+ net/mac80211/ieee80211_i.h                         |   56 +-
+ net/mac80211/iface.c                               |   62 +-
+ net/mac80211/main.c                                |    7 +-
+ net/mac80211/mesh_plink.c                          |    3 +
+ net/mac80211/mlme.c                                |  604 +++++----
+ net/mac80211/nan.c                                 |  126 ++
+ net/mac80211/ocb.c                                 |    5 +-
+ net/mac80211/offchannel.c                          |   13 +-
+ net/mac80211/parse.c                               |    3 +
+ net/mac80211/rate.c                                |    4 +-
+ net/mac80211/rx.c                                  |   33 +-
+ net/mac80211/sta_info.c                            |  294 ++++-
+ net/mac80211/sta_info.h                            |   16 +-
+ net/mac80211/status.c                              |    9 +-
+ net/mac80211/tdls.c                                |   24 +-
+ net/mac80211/trace.h                               |   14 +-
+ net/mac80211/tx.c                                  |   20 +-
+ net/mac80211/uhr.c                                 |    5 +-
+ net/mac80211/util.c                                |   34 +-
+ net/mac80211/vht.c                                 |  350 +----
+ net/mctp/af_mctp.c                                 |   10 +-
+ net/mctp/test/route-test.c                         |    6 -
+ net/mptcp/pm_kernel.c                              |   77 +-
+ net/netlink/af_netlink.c                           |   21 +-
+ net/psp/psp_main.c                                 |   21 +-
+ net/rds/ib_cm.c                                    |   25 +-
+ net/rds/tcp_listen.c                               |    4 +-
+ net/rxrpc/local_object.c                           |    2 +-
+ net/sched/act_api.c                                |    2 +-
+ net/sched/act_bpf.c                                |    2 +-
+ net/sched/act_ife.c                                |   12 +-
+ net/sched/act_mpls.c                               |    2 +-
+ net/sched/act_police.c                             |    4 +-
+ net/sched/act_skbedit.c                            |    2 +-
+ net/sched/act_skbmod.c                             |    2 +-
+ net/sched/sch_api.c                                |  152 ++-
+ net/sched/sch_cake.c                               |   20 +-
+ net/sched/sch_cbs.c                                |    6 +-
+ net/sched/sch_choke.c                              |    8 +-
+ net/sched/sch_codel.c                              |    2 +-
+ net/sched/sch_drr.c                                |   20 +-
+ net/sched/sch_dualpi2.c                            |   74 +-
+ net/sched/sch_etf.c                                |    8 +-
+ net/sched/sch_ets.c                                |    6 +-
+ net/sched/sch_fq.c                                 |    8 +-
+ net/sched/sch_fq_codel.c                           |   15 +-
+ net/sched/sch_fq_pie.c                             |    8 +-
+ net/sched/sch_generic.c                            |   15 +-
+ net/sched/sch_gred.c                               |    4 +-
+ net/sched/sch_hfsc.c                               |   28 +-
+ net/sched/sch_hhf.c                                |    7 +-
+ net/sched/sch_htb.c                                |   82 +-
+ net/sched/sch_mq.c                                 |   37 +-
+ net/sched/sch_mqprio.c                             |   93 +-
+ net/sched/sch_multiq.c                             |    4 +-
+ net/sched/sch_netem.c                              |  235 ++--
+ net/sched/sch_prio.c                               |    6 +-
+ net/sched/sch_qfq.c                                |    8 +-
+ net/sched/sch_red.c                                |    6 +-
+ net/sched/sch_sfb.c                                |    8 +-
+ net/sched/sch_sfq.c                                |   11 +-
+ net/sched/sch_skbprio.c                            |    4 +-
+ net/sched/sch_taprio.c                             |   46 +-
+ net/sched/sch_tbf.c                                |   10 +-
+ net/sched/sch_teql.c                               |    2 +-
+ net/sctp/endpointola.c                             |    2 +-
+ net/sctp/protocol.c                                |   10 +-
+ net/smc/smc_core.c                                 |    4 +
+ net/smc/smc_ism.c                                  |   10 +-
+ net/socket.c                                       |    2 +-
+ net/tipc/udp_media.c                               |   32 +-
+ net/tls/tls_device.c                               |   11 +-
+ net/vmw_vsock/af_vsock.c                           |   16 +-
+ net/wireless/chan.c                                |  124 +-
+ net/wireless/core.c                                |   57 +-
+ net/wireless/core.h                                |    5 +-
+ net/wireless/mlme.c                                |   11 +-
+ net/wireless/nl80211.c                             |  626 +++++++--
+ net/wireless/nl80211.h                             |    5 +-
+ net/wireless/pmsr.c                                |  190 ++-
+ net/wireless/rdev-ops.h                            |   26 +-
+ net/wireless/reg.c                                 |    3 +
+ net/wireless/sme.c                                 |    1 +
+ net/wireless/trace.h                               |   38 +-
+ net/wireless/util.c                                |   29 +-
+ net/wireless/wext-compat.c                         |    3 +-
+ net/x25/af_x25.c                                   |   16 +-
+ tools/net/ynl/ynltool/page-pool.c                  |    6 +-
+ tools/testing/selftests/drivers/net/.gitignore     |    1 +
+ tools/testing/selftests/drivers/net/Makefile       |    2 +
+ tools/testing/selftests/drivers/net/config         |    4 +
+ tools/testing/selftests/drivers/net/gro.py         |   21 +
+ tools/testing/selftests/drivers/net/hw/Makefile    |    2 +
+ tools/testing/selftests/drivers/net/hw/devmem.py   |   77 +-
+ .../testing/selftests/drivers/net/hw/devmem_lib.py |  222 ++++
+ tools/testing/selftests/drivers/net/hw/gro_hw.py   |   10 +
+ tools/testing/selftests/drivers/net/hw/iou-zcrx.py |   12 +
+ tools/testing/selftests/drivers/net/hw/ncdevmem.c  |   58 +-
+ .../testing/selftests/drivers/net/hw/nk_devmem.py  |   46 +
+ .../drivers/net/hw/nk_primary_rx_redirect.bpf.c    |   39 +
+ .../testing/selftests/drivers/net/hw/nk_qlease.py  |    8 +-
+ tools/testing/selftests/drivers/net/hw/ntuple.py   |    5 +-
+ tools/testing/selftests/drivers/net/hw/rss_ctx.py  |   98 +-
+ .../selftests/drivers/net/lib/py/__init__.py       |    5 +-
+ tools/testing/selftests/drivers/net/lib/py/env.py  |  110 +-
+ .../selftests/{ => drivers}/net/so_txtime.c        |   33 +-
+ tools/testing/selftests/drivers/net/so_txtime.py   |  103 ++
+ tools/testing/selftests/net/.gitignore             |    1 -
+ tools/testing/selftests/net/Makefile               |    4 +-
+ tools/testing/selftests/net/config                 |    1 +
+ .../selftests/net/forwarding/bridge_vlan_mcast.sh  |  140 +-
+ tools/testing/selftests/net/getsockopt_iter.c      |  300 +++++
+ tools/testing/selftests/net/lib/gro.c              |  104 +-
+ tools/testing/selftests/net/lib/py/__init__.py     |    4 +-
+ tools/testing/selftests/net/lib/py/utils.py        |   51 +-
+ tools/testing/selftests/net/mptcp/mptcp_join.sh    |   33 +-
+ tools/testing/selftests/net/mptcp/pm_netlink.sh    |   56 +-
+ tools/testing/selftests/net/mptcp/pm_nl_ctl.c      |    8 +-
+ tools/testing/selftests/net/nl_netdev.py           |  119 +-
+ .../selftests/net/openvswitch/openvswitch.sh       |   78 ++
+ .../testing/selftests/net/openvswitch/ovs-dpctl.py |  324 ++++-
+ .../net/packetdrill/tcp_syncookies_ip4_9k.pkt      |   37 +
+ .../net/packetdrill/tcp_syncookies_ip6_9k.pkt      |   36 +
+ tools/testing/selftests/net/ppp/Makefile           |    1 +
+ tools/testing/selftests/net/ppp/config             |    2 +
+ tools/testing/selftests/net/ppp/pppol2tp.sh        |   95 ++
+ tools/testing/selftests/net/protodown.sh           |  182 +++
+ tools/testing/selftests/net/rds/README.txt         |   57 +-
+ tools/testing/selftests/net/rds/config.sh          |   15 +-
+ tools/testing/selftests/net/rds/run.sh             |  141 +-
+ tools/testing/selftests/net/rds/settings           |    2 +-
+ tools/testing/selftests/net/rds/test.py            |  641 +++++++---
+ tools/testing/selftests/net/so_txtime.sh           |  110 --
+ tools/testing/selftests/net/tcp_ao/config          |    4 -
+ .../testing/selftests/net/tcp_ao/key-management.c  |   41 +-
+ .../selftests/net/test_bridge_neigh_suppress.sh    |  468 ++++++-
+ 540 files changed, 23053 insertions(+), 10709 deletions(-)
+ create mode 100644 Documentation/firmware-guide/acpi/dsd/motorcomm-yt8xxx-phy.rst
+ delete mode 100644 Documentation/networking/device_drivers/ethernet/cirrus/cs89x0.rst
+ delete mode 100644 drivers/net/Space.c
+ create mode 100644 drivers/net/ethernet/alibaba/Kconfig
+ create mode 100644 drivers/net/ethernet/alibaba/Makefile
+ create mode 100644 drivers/net/ethernet/alibaba/eea/Makefile
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_adminq.c
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_adminq.h
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_desc.h
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_ethtool.c
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_ethtool.h
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_net.c
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_net.h
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_pci.c
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_pci.h
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_ring.c
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_ring.h
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_rx.c
+ create mode 100644 drivers/net/ethernet/alibaba/eea/eea_tx.c
+ delete mode 100644 drivers/net/ethernet/wiznet/w5300.c
+ create mode 100644 drivers/net/wireless/virtual/mac80211_hwsim_i.h
+ rename drivers/net/wireless/virtual/{mac80211_hwsim.c => mac80211_hwsim_main.c} (91%)
+ create mode 100644 drivers/net/wireless/virtual/mac80211_hwsim_nan.c
+ create mode 100644 drivers/net/wireless/virtual/mac80211_hwsim_nan.h
+ delete mode 100644 include/linux/ks8851_mll.h
+ delete mode 100644 include/linux/platform_data/wiznet.h
+ delete mode 100644 include/net/Space.h
+ delete mode 100644 net/core/sock_destructor.h
+ delete mode 100644 net/ipv4/tcp_sigpool.c
+ create mode 100644 net/mac80211/ap.c
+ create mode 100644 tools/testing/selftests/drivers/net/hw/devmem_lib.py
+ create mode 100755 tools/testing/selftests/drivers/net/hw/nk_devmem.py
+ create mode 100644 tools/testing/selftests/drivers/net/hw/nk_primary_rx_redirect.bpf.c
+ rename tools/testing/selftests/{ => drivers}/net/so_txtime.c (95%)
+ create mode 100755 tools/testing/selftests/drivers/net/so_txtime.py
+ create mode 100644 tools/testing/selftests/net/getsockopt_iter.c
+ create mode 100644 tools/testing/selftests/net/packetdrill/tcp_syncookies_ip4_9k.pkt
+ create mode 100644 tools/testing/selftests/net/packetdrill/tcp_syncookies_ip6_9k.pkt
+ create mode 100755 tools/testing/selftests/net/ppp/pppol2tp.sh
+ create mode 100755 tools/testing/selftests/net/protodown.sh
+ delete mode 100755 tools/testing/selftests/net/so_txtime.sh
+$ git am -3 ../patches/0001-net-mlx5e-Fix-up-merge.patch
+Applying: net/mlx5e: Fix up merge
+Using index info to reconstruct a base tree...
+M	drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+$ git am -3 ../patches/0001-net-Fix-up-mismerge-due-to-missing-cast-enum-skb_dro.patch
+Applying: net: Fix up mismerge due to missing cast (enum skb_drop_reason)
+Using index info to reconstruct a base tree...
+M	include/net/sch_generic.h
+Falling back to patching base and 3-way merge...
+Auto-merging include/net/sch_generic.h
+No changes -- Patch already applied.
+$ git am -3 ../patches/0001-io_uring-Fix-up-build-for-net-Slightly-simplify-net_.patch
+Applying: io_uring: Fix up build for "net: Slightly simplify net_mp_{open,close}_rxq"
+Using index info to reconstruct a base tree...
+M	io_uring/zcrx.c
+Falling back to patching base and 3-way merge...
+Auto-merging io_uring/zcrx.c
+No changes -- Patch already applied.
+Merging bpf-next/for-next (3db0419c0147c Merge branch 'selftests-bpf-xdp-lb-benchmark-fixes')
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git bpf-next/for-next
+Auto-merging MAINTAINERS
+Auto-merging fs/bpf_fs_kfuncs.c
+Auto-merging include/linux/bpf.h
+Auto-merging include/linux/bpf_verifier.h
+Auto-merging include/linux/filter.h
+Auto-merging include/linux/syscalls.h
+Auto-merging kernel/bpf/core.c
+Auto-merging kernel/bpf/fixups.c
+Auto-merging kernel/bpf/helpers.c
+Auto-merging kernel/bpf/syscall.c
+Auto-merging kernel/bpf/verifier.c
+Auto-merging kernel/trace/bpf_trace.c
+Auto-merging tools/testing/selftests/bpf/prog_tests/verifier.c
+Merge made by the 'ort' strategy.
+ Documentation/bpf/btf.rst                          |    6 +-
+ MAINTAINERS                                        |    1 +
+ arch/arm64/net/bpf_jit_comp.c                      |   92 +-
+ arch/arm64/net/bpf_timed_may_goto.S                |    8 +-
+ arch/s390/net/bpf_jit_comp.c                       |   37 +
+ arch/x86/net/bpf_jit_comp.c                        |  151 +-
+ drivers/base/power/power.h                         |    7 +
+ drivers/base/power/wakeup.c                        |   71 +-
+ drivers/net/ethernet/netronome/nfp/bpf/verifier.c  |    8 +-
+ fs/bpf_fs_kfuncs.c                                 |    2 +-
+ fs/verity/measure.c                                |    4 +-
+ include/linux/bpf.h                                |   69 +-
+ include/linux/bpf_verifier.h                       |  189 +-
+ include/linux/btf.h                                |    7 +-
+ include/linux/cnum.h                               |   82 +
+ include/linux/filter.h                             |   27 +-
+ include/linux/syscalls.h                           |    3 +-
+ include/linux/trace_events.h                       |    6 +
+ include/trace/bpf_probe.h                          |    2 -
+ include/uapi/linux/bpf.h                           |    8 +
+ include/uapi/linux/btf.h                           |   26 +-
+ kernel/bpf/Makefile                                |    2 +-
+ kernel/bpf/backtrack.c                             |   82 +-
+ kernel/bpf/btf.c                                   |   79 +-
+ kernel/bpf/cnum.c                                  |  120 +
+ kernel/bpf/cnum_defs.h                             |  247 ++
+ kernel/bpf/const_fold.c                            |    8 +
+ kernel/bpf/core.c                                  |   19 +-
+ kernel/bpf/fixups.c                                |   23 +-
+ kernel/bpf/helpers.c                               |   38 +-
+ kernel/bpf/liveness.c                              |  181 +-
+ kernel/bpf/log.c                                   |  114 +-
+ kernel/bpf/states.c                                |   45 +-
+ kernel/bpf/syscall.c                               |  122 +-
+ kernel/bpf/trampoline.c                            |    1 -
+ kernel/bpf/verifier.c                              | 2756 +++++++++-----------
+ kernel/events/core.c                               |    9 +
+ kernel/trace/bpf_trace.c                           |   70 +-
+ kernel/trace/trace_syscalls.c                      |  108 +-
+ lib/test_bpf.c                                     |  363 ++-
+ net/bpf/test_run.c                                 |   65 +-
+ tools/bpf/bpftool/btf.c                            |   17 +-
+ tools/bpf/bpftool/btf_dumper.c                     |    4 +-
+ tools/bpf/bpftool/gen.c                            |   16 +-
+ tools/include/uapi/linux/bpf.h                     |    8 +
+ tools/include/uapi/linux/btf.h                     |   26 +-
+ tools/lib/bpf/bpf.c                                |   55 +-
+ tools/lib/bpf/bpf.h                                |   17 +-
+ tools/lib/bpf/btf.c                                |   54 +-
+ tools/lib/bpf/btf.h                                |    2 +-
+ tools/lib/bpf/btf_dump.c                           |   24 +-
+ tools/lib/bpf/features.c                           |    8 +
+ tools/lib/bpf/libbpf.c                             |   97 +-
+ tools/lib/bpf/libbpf_internal.h                    |    3 +
+ tools/lib/bpf/relo_core.c                          |   16 +-
+ tools/lib/bpf/skel_internal.h                      |    5 +
+ tools/testing/selftests/bpf/.gitignore             |    1 -
+ tools/testing/selftests/bpf/Makefile               |   62 +-
+ tools/testing/selftests/bpf/README.rst             |    2 +-
+ tools/testing/selftests/bpf/bench.c                |   20 +-
+ tools/testing/selftests/bpf/bench.h                |    1 +
+ tools/testing/selftests/bpf/bench_bpf_timing.h     |   50 +
+ tools/testing/selftests/bpf/benchs/bench_bpf_nop.c |   84 +
+ .../selftests/bpf/benchs/bench_bpf_timing.c        |  298 +++
+ tools/testing/selftests/bpf/benchs/bench_xdp_lb.c  | 1124 ++++++++
+ .../selftests/bpf/benchs/run_bench_xdp_lb.sh       |   79 +
+ tools/testing/selftests/bpf/bpf_arena_alloc.h      |    2 +-
+ tools/testing/selftests/bpf/bpf_arena_list.h       |    2 +-
+ tools/testing/selftests/bpf/bpf_arena_strsearch.h  |    2 +-
+ tools/testing/selftests/bpf/bpf_experimental.h     |   84 +-
+ tools/testing/selftests/bpf/bpf_kfuncs.h           |    8 +-
+ tools/testing/selftests/bpf/config                 |    3 +-
+ tools/testing/selftests/bpf/default.profraw        |  Bin 0 -> 160 bytes
+ tools/testing/selftests/bpf/libarena/Makefile      |   92 +
+ .../bpf/{ => libarena/include}/bpf_arena_common.h  |    0
+ .../include}/bpf_arena_spin_lock.h                 |   11 +-
+ .../bpf/{ => libarena/include}/bpf_atomic.h        |    4 +-
+ .../selftests/bpf/libarena/include/bpf_may_goto.h  |   84 +
+ .../selftests/bpf/libarena/include/libarena/asan.h |  103 +
+ .../bpf/libarena/include/libarena/buddy.h          |   92 +
+ .../bpf/libarena/include/libarena/common.h         |   94 +
+ .../bpf/libarena/include/libarena/userspace.h      |  132 +
+ .../bpf/libarena/selftests/st_asan_buddy.bpf.c     |  258 ++
+ .../bpf/libarena/selftests/st_asan_common.h        |   52 +
+ .../bpf/libarena/selftests/st_buddy.bpf.c          |  209 ++
+ .../bpf/libarena/selftests/test_progs_compat.h     |   15 +
+ .../testing/selftests/bpf/libarena/src/asan.bpf.c  |  553 ++++
+ .../testing/selftests/bpf/libarena/src/buddy.bpf.c |  903 +++++++
+ .../selftests/bpf/libarena/src/common.bpf.c        |   52 +
+ .../selftests/bpf/prog_tests/arena_spin_lock.c     |    7 -
+ tools/testing/selftests/bpf/prog_tests/bpf_nf.c    |   22 +-
+ tools/testing/selftests/bpf/prog_tests/btf.c       |    8 +-
+ .../selftests/bpf/prog_tests/btf_dedup_split.c     |   51 +-
+ tools/testing/selftests/bpf/prog_tests/btf_dump.c  |    4 +-
+ tools/testing/selftests/bpf/prog_tests/cb_refs.c   |    2 +-
+ .../selftests/bpf/prog_tests/cgrp_local_storage.c  |   15 +-
+ .../testing/selftests/bpf/prog_tests/ctx_rewrite.c |   17 +-
+ .../testing/selftests/bpf/prog_tests/exceptions.c  |    7 +
+ .../testing/selftests/bpf/prog_tests/file_reader.c |   22 +-
+ .../testing/selftests/bpf/prog_tests/kfunc_call.c  |    2 +-
+ tools/testing/selftests/bpf/prog_tests/libarena.c  |   66 +
+ .../selftests/bpf/prog_tests/libarena_asan.c       |   93 +
+ .../testing/selftests/bpf/prog_tests/linked_list.c |    4 +-
+ tools/testing/selftests/bpf/prog_tests/map_init.c  |  192 ++
+ .../testing/selftests/bpf/prog_tests/reg_bounds.c  |   90 +-
+ .../bpf/prog_tests/sleepable_tracepoints.c         |  142 +
+ tools/testing/selftests/bpf/prog_tests/stack_arg.c |  139 +
+ .../selftests/bpf/prog_tests/stack_arg_fail.c      |   10 +
+ .../selftests/bpf/prog_tests/stack_arg_precision.c |   10 +
+ .../selftests/bpf/prog_tests/task_local_storage.c  |    1 +
+ tools/testing/selftests/bpf/prog_tests/verifier.c  |    4 +
+ tools/testing/selftests/bpf/prog_tests/vmlinux.c   |   45 +-
+ .../selftests/bpf/prog_tests/wakeup_source.c       |  118 +
+ tools/testing/selftests/bpf/progs/arena_atomics.c  |    2 +-
+ .../testing/selftests/bpf/progs/arena_spin_lock.c  |    2 +-
+ .../selftests/bpf/progs/bench_bpf_timing.bpf.h     |   69 +
+ tools/testing/selftests/bpf/progs/bpf_misc.h       |    1 +
+ tools/testing/selftests/bpf/progs/bpf_nop_bench.c  |   14 +
+ .../selftests/bpf/progs/btf__stack_arg_precision.c |   24 +
+ .../bpf/progs/btf__verifier_stack_arg_order.c      |   49 +
+ .../selftests/bpf/progs/cgrp_kfunc_failure.c       |   14 +-
+ .../selftests/bpf/progs/cgrp_ls_sleepable.c        |   18 +
+ .../selftests/bpf/progs/compute_live_registers.c   |    2 +-
+ .../testing/selftests/bpf/progs/cpumask_failure.c  |   10 +-
+ .../testing/selftests/bpf/progs/cpumask_success.c  |    2 +-
+ tools/testing/selftests/bpf/progs/dynptr_fail.c    |   22 +-
+ tools/testing/selftests/bpf/progs/dynptr_success.c |    6 +-
+ tools/testing/selftests/bpf/progs/exceptions.c     |  114 +
+ .../testing/selftests/bpf/progs/file_reader_fail.c |    4 +-
+ tools/testing/selftests/bpf/progs/irq.c            |    4 +-
+ tools/testing/selftests/bpf/progs/iters.c          |    6 +-
+ .../selftests/bpf/progs/iters_state_safety.c       |   14 +-
+ tools/testing/selftests/bpf/progs/iters_testmod.c  |    4 +-
+ .../selftests/bpf/progs/iters_testmod_seq.c        |    4 +-
+ tools/testing/selftests/bpf/progs/lpm_trie_bench.c |    2 +-
+ tools/testing/selftests/bpf/progs/map_kptr_fail.c  |    2 +-
+ .../selftests/bpf/progs/percpu_alloc_fail.c        |    4 +-
+ tools/testing/selftests/bpf/progs/rbtree_fail.c    |    6 +-
+ .../selftests/bpf/progs/refcounted_kptr_fail.c     |    2 +-
+ tools/testing/selftests/bpf/progs/stack_arg.c      |  253 ++
+ tools/testing/selftests/bpf/progs/stack_arg_fail.c |  114 +
+ .../testing/selftests/bpf/progs/stack_arg_kfunc.c  |  164 ++
+ .../selftests/bpf/progs/stack_arg_precision.c      |  135 +
+ tools/testing/selftests/bpf/progs/stream.c         |    2 +-
+ tools/testing/selftests/bpf/progs/stream_fail.c    |    2 +-
+ .../selftests/bpf/progs/task_kfunc_failure.c       |   18 +-
+ .../selftests/bpf/progs/task_local_storage.c       |   19 +
+ tools/testing/selftests/bpf/progs/task_work_fail.c |    6 +-
+ .../testing/selftests/bpf/progs/test_bpf_nf_fail.c |    8 +-
+ .../selftests/bpf/progs/test_kfunc_dynptr_param.c  |   11 +-
+ .../bpf/progs/test_kfunc_param_nullable.c          |    2 +-
+ .../bpf/progs/test_sleepable_tracepoints.c         |  112 +
+ .../bpf/progs/test_sleepable_tracepoints_fail.c    |   18 +
+ .../testing/selftests/bpf/progs/test_tunnel_kern.c |   13 +-
+ tools/testing/selftests/bpf/progs/test_vmlinux.c   |    4 +-
+ .../selftests/bpf/progs/test_wakeup_source.c       |   92 +
+ tools/testing/selftests/bpf/progs/verifier_arena.c |    2 +-
+ .../selftests/bpf/progs/verifier_arena_globals1.c  |    2 +-
+ .../selftests/bpf/progs/verifier_arena_globals2.c  |    2 +-
+ .../selftests/bpf/progs/verifier_arena_large.c     |    2 +-
+ .../selftests/bpf/progs/verifier_bits_iter.c       |    4 +-
+ .../testing/selftests/bpf/progs/verifier_bounds.c  |  130 +-
+ .../selftests/bpf/progs/verifier_bpf_fastcall.c    |   27 +-
+ .../selftests/bpf/progs/verifier_global_subprogs.c |    3 +-
+ .../selftests/bpf/progs/verifier_jit_inline.c      |    2 +-
+ tools/testing/selftests/bpf/progs/verifier_ldsx.c  |    8 +-
+ .../selftests/bpf/progs/verifier_may_goto_1.c      |   12 +-
+ .../selftests/bpf/progs/verifier_private_stack.c   |   25 +-
+ .../selftests/bpf/progs/verifier_ref_tracking.c    |    6 +-
+ tools/testing/selftests/bpf/progs/verifier_sdiv.c  |   64 +-
+ .../selftests/bpf/progs/verifier_stack_arg.c       |  447 ++++
+ .../selftests/bpf/progs/verifier_stack_arg_order.c |  185 ++
+ .../testing/selftests/bpf/progs/verifier_subreg.c  |    6 +-
+ .../selftests/bpf/progs/verifier_vfs_reject.c      |    8 +-
+ tools/testing/selftests/bpf/progs/wakeup_source.h  |   22 +
+ .../selftests/bpf/progs/wakeup_source_fail.c       |   76 +
+ tools/testing/selftests/bpf/progs/wq_failures.c    |    2 +-
+ tools/testing/selftests/bpf/progs/xdp_flowtable.c  |    7 +-
+ tools/testing/selftests/bpf/progs/xdp_lb_bench.c   |  647 +++++
+ tools/testing/selftests/bpf/progs/xdping_kern.c    |  183 --
+ .../testing/selftests/bpf/test_kmods/bpf_testmod.c |   72 +
+ .../selftests/bpf/test_kmods/bpf_testmod_kfunc.h   |   26 +
+ tools/testing/selftests/bpf/test_loader.c          |  187 +-
+ tools/testing/selftests/bpf/test_progs.c           |    2 +-
+ tools/testing/selftests/bpf/test_progs.h           |    2 +
+ tools/testing/selftests/bpf/test_xdping.sh         |  103 -
+ tools/testing/selftests/bpf/uprobe_multi.c         |    4 +-
+ tools/testing/selftests/bpf/verifier/calls.c       |   14 +-
+ tools/testing/selftests/bpf/verifier/sleepable.c   |   17 +-
+ tools/testing/selftests/bpf/veristat.c             |   13 +-
+ tools/testing/selftests/bpf/vmtest.sh              |    2 +-
+ tools/testing/selftests/bpf/xdp_lb_bench_common.h  |  112 +
+ tools/testing/selftests/bpf/xdping.c               |  254 --
+ tools/testing/selftests/bpf/xdping.h               |   13 -
+ 194 files changed, 12171 insertions(+), 3053 deletions(-)
+ create mode 100644 include/linux/cnum.h
+ create mode 100644 kernel/bpf/cnum.c
+ create mode 100644 kernel/bpf/cnum_defs.h
+ create mode 100644 tools/testing/selftests/bpf/bench_bpf_timing.h
+ create mode 100644 tools/testing/selftests/bpf/benchs/bench_bpf_nop.c
+ create mode 100644 tools/testing/selftests/bpf/benchs/bench_bpf_timing.c
+ create mode 100644 tools/testing/selftests/bpf/benchs/bench_xdp_lb.c
+ create mode 100755 tools/testing/selftests/bpf/benchs/run_bench_xdp_lb.sh
+ create mode 100644 tools/testing/selftests/bpf/default.profraw
+ create mode 100644 tools/testing/selftests/bpf/libarena/Makefile
+ rename tools/testing/selftests/bpf/{ => libarena/include}/bpf_arena_common.h (100%)
+ rename tools/testing/selftests/bpf/{progs => libarena/include}/bpf_arena_spin_lock.h (98%)
+ rename tools/testing/selftests/bpf/{ => libarena/include}/bpf_atomic.h (98%)
+ create mode 100644 tools/testing/selftests/bpf/libarena/include/bpf_may_goto.h
+ create mode 100644 tools/testing/selftests/bpf/libarena/include/libarena/asan.h
+ create mode 100644 tools/testing/selftests/bpf/libarena/include/libarena/buddy.h
+ create mode 100644 tools/testing/selftests/bpf/libarena/include/libarena/common.h
+ create mode 100644 tools/testing/selftests/bpf/libarena/include/libarena/userspace.h
+ create mode 100644 tools/testing/selftests/bpf/libarena/selftests/st_asan_buddy.bpf.c
+ create mode 100644 tools/testing/selftests/bpf/libarena/selftests/st_asan_common.h
+ create mode 100644 tools/testing/selftests/bpf/libarena/selftests/st_buddy.bpf.c
+ create mode 100644 tools/testing/selftests/bpf/libarena/selftests/test_progs_compat.h
+ create mode 100644 tools/testing/selftests/bpf/libarena/src/asan.bpf.c
+ create mode 100644 tools/testing/selftests/bpf/libarena/src/buddy.bpf.c
+ create mode 100644 tools/testing/selftests/bpf/libarena/src/common.bpf.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/libarena.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/libarena_asan.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/sleepable_tracepoints.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/stack_arg.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/stack_arg_fail.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/stack_arg_precision.c
+ create mode 100644 tools/testing/selftests/bpf/prog_tests/wakeup_source.c
+ create mode 100644 tools/testing/selftests/bpf/progs/bench_bpf_timing.bpf.h
+ create mode 100644 tools/testing/selftests/bpf/progs/bpf_nop_bench.c
+ create mode 100644 tools/testing/selftests/bpf/progs/btf__stack_arg_precision.c
+ create mode 100644 tools/testing/selftests/bpf/progs/btf__verifier_stack_arg_order.c
+ create mode 100644 tools/testing/selftests/bpf/progs/stack_arg.c
+ create mode 100644 tools/testing/selftests/bpf/progs/stack_arg_fail.c
+ create mode 100644 tools/testing/selftests/bpf/progs/stack_arg_kfunc.c
+ create mode 100644 tools/testing/selftests/bpf/progs/stack_arg_precision.c
+ create mode 100644 tools/testing/selftests/bpf/progs/test_sleepable_tracepoints.c
+ create mode 100644 tools/testing/selftests/bpf/progs/test_sleepable_tracepoints_fail.c
+ create mode 100644 tools/testing/selftests/bpf/progs/test_wakeup_source.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_stack_arg.c
+ create mode 100644 tools/testing/selftests/bpf/progs/verifier_stack_arg_order.c
+ create mode 100644 tools/testing/selftests/bpf/progs/wakeup_source.h
+ create mode 100644 tools/testing/selftests/bpf/progs/wakeup_source_fail.c
+ create mode 100644 tools/testing/selftests/bpf/progs/xdp_lb_bench.c
+ delete mode 100644 tools/testing/selftests/bpf/progs/xdping_kern.c
+ delete mode 100755 tools/testing/selftests/bpf/test_xdping.sh
+ create mode 100644 tools/testing/selftests/bpf/xdp_lb_bench_common.h
+ delete mode 100644 tools/testing/selftests/bpf/xdping.c
+ delete mode 100644 tools/testing/selftests/bpf/xdping.h
+$ git am -3 ../patches/0001-perf-Fixup-for-btf_vlan-API-change.patch
+Applying: perf: Fixup for btf_vlan() API change
+$ git reset HEAD^
+Unstaged changes after reset:
+M	tools/perf/builtin-trace.c
+M	tools/perf/util/btf.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master f76892bef031d] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git
+ Date: Thu May 21 11:20:59 2026 +0100
+Merging ipsec-next/master (a77d172177f37 xfrm: cleanup error path in xfrm_add_policy())
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/klassert/ipsec-next.git ipsec-next/master
+Auto-merging net/xfrm/xfrm_user.c
+Merge made by the 'ort' strategy.
+ net/xfrm/xfrm_user.c | 5 ++---
+ 1 file changed, 2 insertions(+), 3 deletions(-)
+Merging mlx5-next/mlx5-next (02c54621e81cc net/mlx5: Extend query_esw_functions output for multi-function support)
+$ git merge -m Merge branch 'mlx5-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mellanox/linux.git mlx5-next/mlx5-next
+Already up to date.
+Merging netfilter-next/main (8b2feced65cd3 Merge branch 'net-dsa-microchip-remove-one-indirection-layer')
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf-next.git netfilter-next/main
+Already up to date.
+Merging ipvs-next/main (8b2feced65cd3 Merge branch 'net-dsa-microchip-remove-one-indirection-layer')
+$ git merge -m Merge branch 'main' of https://git.kernel.org/pub/scm/linux/kernel/git/horms/ipvs-next.git ipvs-next/main
+Already up to date.
+Merging bluetooth/master (0b580042a1a54 Bluetooth: fix UAF in l2cap_sock_cleanup_listen() vs l2cap_conn_del())
+  085d13bf86124 ("Bluetooth: hci_sync: Fix not setting mask for HCI_EVT_LE_ALL_REMOTE_FEATURES_COMPLETE")
+  0b580042a1a54 ("Bluetooth: fix UAF in l2cap_sock_cleanup_listen() vs l2cap_conn_del()")
+  5daf96ab8398e ("Bluetooth: btmtk: fix urb->setup_packet leak in error paths")
+  663fc68494ce6 ("Bluetooth: btintel_pcie: Fix incorrect MAC access programming")
+  6aba94a49bc9c ("Bluetooth: ISO: drop ISO_END frames received without prior ISO_START")
+  7db62a762f613 ("Bluetooth: hci_uart: fix UAFs and race conditions in close and init paths")
+  8f5b6b4b198ed ("Bluetooth: MGMT: validate Add Extended Advertising Data length")
+  ffeee619a13bf ("Bluetooth: bnep: Fix UAF read of dev->name")
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/bluetooth/bluetooth-next.git bluetooth/master
+Auto-merging arch/arm/configs/pxa_defconfig
+Auto-merging drivers/bluetooth/btintel_pcie.c
+Auto-merging drivers/bluetooth/btintel_pcie.h
+Auto-merging drivers/bluetooth/btmtk.c
+Auto-merging net/bluetooth/iso.c
+Auto-merging net/bluetooth/l2cap_sock.c
+Auto-merging net/bluetooth/rfcomm/sock.c
+Auto-merging net/bluetooth/sco.c
+Merge made by the 'ort' strategy.
+ arch/arm/configs/pxa_defconfig              |   3 -
+ arch/arm/configs/spitz_defconfig            |   3 -
+ arch/loongarch/configs/loongson32_defconfig |   3 -
+ arch/loongarch/configs/loongson64_defconfig |   3 -
+ arch/mips/configs/mtx1_defconfig            |   3 -
+ arch/powerpc/configs/ppc6xx_defconfig       |   3 -
+ drivers/bluetooth/Kconfig                   |  42 +-
+ drivers/bluetooth/Makefile                  |   3 -
+ drivers/bluetooth/bluecard_cs.c             | 908 ----------------------------
+ drivers/bluetooth/bt3c_cs.c                 | 749 -----------------------
+ drivers/bluetooth/btintel.c                 |  10 +-
+ drivers/bluetooth/btintel.h                 |   7 +
+ drivers/bluetooth/btintel_pcie.c            | 203 ++++++-
+ drivers/bluetooth/btintel_pcie.h            |   7 +
+ drivers/bluetooth/btmtk.c                   |  23 +
+ drivers/bluetooth/btmtk.h                   |   7 +
+ drivers/bluetooth/btrtl.c                   |  13 +
+ drivers/bluetooth/btusb.c                   |  14 +
+ drivers/bluetooth/dtl1_cs.c                 | 614 -------------------
+ net/bluetooth/6lowpan.c                     |  25 +-
+ net/bluetooth/Kconfig                       |   6 +-
+ net/bluetooth/hci_sock.c                    |  26 +-
+ net/bluetooth/iso.c                         |  27 +-
+ net/bluetooth/l2cap_sock.c                  |  61 +-
+ net/bluetooth/rfcomm/sock.c                 |  30 +-
+ net/bluetooth/sco.c                         |  59 +-
+ net/bluetooth/smp.c                         | 180 ++----
+ 27 files changed, 452 insertions(+), 2580 deletions(-)
+ delete mode 100644 drivers/bluetooth/bluecard_cs.c
+ delete mode 100644 drivers/bluetooth/bt3c_cs.c
+ delete mode 100644 drivers/bluetooth/dtl1_cs.c
+Merging wireless-next/for-next (1d174fec87850 ARM: dts: omap2: add stlc4560 spi-wireless node)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wireless/wireless-next.git wireless-next/for-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/net/wireless/st,stlc4560.yaml         |  61 +++++++
+ MAINTAINERS                                        |   1 +
+ arch/arm/boot/dts/ti/omap/omap2.dtsi               |   4 +
+ .../arm/boot/dts/ti/omap/omap2420-n8x0-common.dtsi |  12 ++
+ arch/arm/mach-omap2/board-n8x0.c                   |  18 --
+ drivers/net/wireless/intersil/p54/p54spi.c         |  67 +++----
+ drivers/net/wireless/intersil/p54/p54spi.h         |   3 +
+ drivers/net/wireless/purelifi/plfxlc/usb.c         |  25 +--
+ drivers/net/wireless/ralink/rt2x00/rt2x00.h        |   3 +-
+ drivers/net/wireless/ralink/rt2x00/rt2x00usb.c     |  11 +-
+ drivers/net/wireless/virtual/mac80211_hwsim_main.c |  31 +++-
+ include/net/cfg80211.h                             |  10 ++
+ net/mac80211/chan.c                                |  10 +-
+ net/mac80211/key.c                                 |  22 ++-
+ net/mac80211/link.c                                |   5 +-
+ net/mac80211/sta_info.c                            |  17 ++
+ net/mac80211/util.c                                |   6 +-
+ net/wireless/chan.c                                | 130 ++++++++++++++
+ net/wireless/tests/chan.c                          | 192 ++++++++++++++++++++-
+ 19 files changed, 508 insertions(+), 120 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/net/wireless/st,stlc4560.yaml
+Merging ath-next/for-next (30d516006fa1f wifi: ath10k: update outdated comment for renamed ieee80211_tx_status())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ath/ath.git ath-next/for-next
+Auto-merging drivers/net/wireless/ath/ath12k/core.c
+Auto-merging drivers/net/wireless/ath/ath12k/dp_rx.c
+Auto-merging drivers/net/wireless/ath/ath12k/mac.c
+Auto-merging drivers/net/wireless/ath/ath12k/wmi.c
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/ath/ath10k/core.h             |   1 -
+ drivers/net/wireless/ath/ath10k/htt_tx.c           |   2 +-
+ drivers/net/wireless/ath/ath10k/leds.c             |   8 +-
+ drivers/net/wireless/ath/ath10k/qmi.c              |   4 +-
+ drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h     |   1 -
+ drivers/net/wireless/ath/ath11k/Kconfig            |   2 +-
+ drivers/net/wireless/ath/ath11k/dp.c               |   1 +
+ drivers/net/wireless/ath/ath11k/mhi.c              |   4 +-
+ drivers/net/wireless/ath/ath11k/pci.c              |   8 +
+ drivers/net/wireless/ath/ath11k/qmi.c              |   3 +-
+ drivers/net/wireless/ath/ath11k/qmi.h              |   1 -
+ drivers/net/wireless/ath/ath12k/Kconfig            |   6 +-
+ drivers/net/wireless/ath/ath12k/core.c             |  50 +-
+ drivers/net/wireless/ath/ath12k/core.h             |  12 +-
+ drivers/net/wireless/ath/ath12k/debugfs.c          |  46 ++
+ drivers/net/wireless/ath/ath12k/dp_rx.c            |  68 ++-
+ drivers/net/wireless/ath/ath12k/mac.c              | 110 +++-
+ drivers/net/wireless/ath/ath12k/qmi.c              |   2 +-
+ drivers/net/wireless/ath/ath12k/qmi.h              |   1 -
+ drivers/net/wireless/ath/ath12k/thermal.c          | 252 +++++++--
+ drivers/net/wireless/ath/ath12k/thermal.h          |  35 ++
+ drivers/net/wireless/ath/ath12k/wmi.c              | 565 ++++++++++++++++++++-
+ drivers/net/wireless/ath/ath12k/wmi.h              | 125 ++++-
+ drivers/net/wireless/ath/ath9k/ar9002_hw.c         |   6 +-
+ .../net/wireless/ath/ath9k/ath9k_pci_owl_loader.c  |  31 +-
+ drivers/net/wireless/ath/ath9k/common-init.c       |   8 +-
+ drivers/net/wireless/ath/ath9k/init.c              |  11 +-
+ drivers/net/wireless/ath/ath9k/recv.c              |   4 +-
+ drivers/net/wireless/ath/ath9k/xmit.c              |   7 +-
+ 29 files changed, 1179 insertions(+), 195 deletions(-)
+Merging iwlwifi-next/next (05cfe4277e92b wifi: iwlwifi: mld: fix indentation in iwl_mld_fill_supp_rates())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iwlwifi/iwlwifi-next.git iwlwifi-next/next
+Auto-merging drivers/net/wireless/intel/iwlwifi/mld/d3.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/mld/link.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/mld/tx.c
+Auto-merging drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c
+Merge made by the 'ort' strategy.
+ drivers/net/wireless/intel/iwlwifi/Makefile        |    2 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/22000.c     |   23 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/7000.c      |    5 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/8000.c      |    5 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/9000.c      |    5 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/ax210.c     |   38 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/bz.c        |   19 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/dr.c        |   19 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c     |    8 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/rf-gf.c     |   17 -
+ drivers/net/wireless/intel/iwlwifi/cfg/rf-hr.c     |   30 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c     |   22 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/rf-wh.c     |    8 +-
+ drivers/net/wireless/intel/iwlwifi/cfg/sc.c        |   22 +-
+ .../net/wireless/intel/iwlwifi/fw/api/commands.h   |   13 +-
+ .../net/wireless/intel/iwlwifi/fw/api/datapath.h   |    9 +-
+ .../net/wireless/intel/iwlwifi/fw/api/location.h   |  107 +-
+ .../net/wireless/intel/iwlwifi/fw/api/mac-cfg.h    |  184 ++-
+ drivers/net/wireless/intel/iwlwifi/fw/api/power.h  |   48 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/rx.h     |   40 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/sta.h    |    3 +-
+ drivers/net/wireless/intel/iwlwifi/fw/api/stats.h  |   90 +-
+ drivers/net/wireless/intel/iwlwifi/fw/dbg-old.c    | 1022 +++++++++++++++
+ drivers/net/wireless/intel/iwlwifi/fw/dbg.c        | 1298 +++-----------------
+ drivers/net/wireless/intel/iwlwifi/fw/dbg.h        |    7 +-
+ drivers/net/wireless/intel/iwlwifi/fw/debugfs.c    |   15 +-
+ drivers/net/wireless/intel/iwlwifi/fw/error-dump.h |   14 +-
+ drivers/net/wireless/intel/iwlwifi/fw/file.h       |   10 +-
+ drivers/net/wireless/intel/iwlwifi/fw/img.h        |    3 +-
+ drivers/net/wireless/intel/iwlwifi/fw/regulatory.c |    7 +-
+ drivers/net/wireless/intel/iwlwifi/fw/rs.c         |    5 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-config.h    |    9 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-csr.h       |    3 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-drv.c       |   14 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-io.c        |   25 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-io.h        |    6 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c |  150 ++-
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h |    2 +
+ drivers/net/wireless/intel/iwlwifi/iwl-nvm-utils.h |    9 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-prph.h      |    7 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-trans.c     |   17 +-
+ drivers/net/wireless/intel/iwlwifi/iwl-trans.h     |  123 +-
+ drivers/net/wireless/intel/iwlwifi/mld/agg.c       |    9 +
+ drivers/net/wireless/intel/iwlwifi/mld/ap.c        |   58 +-
+ drivers/net/wireless/intel/iwlwifi/mld/ap.h        |    8 +-
+ drivers/net/wireless/intel/iwlwifi/mld/d3.c        |  166 ++-
+ drivers/net/wireless/intel/iwlwifi/mld/d3.h        |    6 +-
+ drivers/net/wireless/intel/iwlwifi/mld/debugfs.c   |   74 +-
+ .../net/wireless/intel/iwlwifi/mld/ftm-initiator.c |   30 +-
+ drivers/net/wireless/intel/iwlwifi/mld/iface.c     |  187 ++-
+ drivers/net/wireless/intel/iwlwifi/mld/iface.h     |   62 +-
+ drivers/net/wireless/intel/iwlwifi/mld/key.c       |  168 ++-
+ drivers/net/wireless/intel/iwlwifi/mld/link.c      |  569 ++++++++-
+ drivers/net/wireless/intel/iwlwifi/mld/link.h      |   37 +-
+ drivers/net/wireless/intel/iwlwifi/mld/mac80211.c  |  362 ++++--
+ drivers/net/wireless/intel/iwlwifi/mld/mcc.c       |   13 +-
+ drivers/net/wireless/intel/iwlwifi/mld/mld.c       |   20 +-
+ drivers/net/wireless/intel/iwlwifi/mld/mld.h       |   16 +-
+ drivers/net/wireless/intel/iwlwifi/mld/mlo.c       |   36 +-
+ drivers/net/wireless/intel/iwlwifi/mld/nan.c       |  748 ++++++++++-
+ drivers/net/wireless/intel/iwlwifi/mld/nan.h       |   41 +-
+ drivers/net/wireless/intel/iwlwifi/mld/notif.c     |   35 +-
+ drivers/net/wireless/intel/iwlwifi/mld/phy.c       |   24 +-
+ drivers/net/wireless/intel/iwlwifi/mld/power.c     |  210 +++-
+ drivers/net/wireless/intel/iwlwifi/mld/ptp.c       |    2 +-
+ drivers/net/wireless/intel/iwlwifi/mld/rx.c        |   44 +-
+ drivers/net/wireless/intel/iwlwifi/mld/rx.h        |    7 +-
+ drivers/net/wireless/intel/iwlwifi/mld/sta.c       |  247 +++-
+ drivers/net/wireless/intel/iwlwifi/mld/sta.h       |   32 +-
+ drivers/net/wireless/intel/iwlwifi/mld/stats.c     |  108 +-
+ .../net/wireless/intel/iwlwifi/mld/tests/Makefile  |    1 +
+ .../intel/iwlwifi/mld/tests/chan_load_thresh.c     |  139 +++
+ .../intel/iwlwifi/mld/tests/link-selection.c       |    6 +-
+ .../net/wireless/intel/iwlwifi/mld/tests/utils.c   |    8 +-
+ drivers/net/wireless/intel/iwlwifi/mld/tlc.c       |  410 ++++---
+ drivers/net/wireless/intel/iwlwifi/mld/tx.c        |   51 +-
+ .../net/wireless/intel/iwlwifi/mvm/ftm-initiator.c |   30 +-
+ .../net/wireless/intel/iwlwifi/mvm/ftm-responder.c |   32 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/mvm.h       |    4 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/ops.c       |    4 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/power.c     |   14 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/ptp.c       |    2 +-
+ drivers/net/wireless/intel/iwlwifi/mvm/rx.c        |    9 +-
+ drivers/net/wireless/intel/iwlwifi/pcie/drv.c      |   14 +-
+ .../wireless/intel/iwlwifi/pcie/gen1_2/internal.h  |  107 +-
+ .../intel/iwlwifi/pcie/gen1_2/trans-gen2.c         |    9 +-
+ .../net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c |   93 +-
+ 87 files changed, 5449 insertions(+), 2266 deletions(-)
+ create mode 100644 drivers/net/wireless/intel/iwlwifi/fw/dbg-old.c
+ create mode 100644 drivers/net/wireless/intel/iwlwifi/mld/tests/chan_load_thresh.c
+Merging wpan-next/master (1dd9291eb9037 ieee802154: Remove WARN_ON() in cfg802154_pernet_exit())
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-next/master
+Merge made by the 'ort' strategy.
+ net/ieee802154/core.c | 49 ++++++++++++++++++++++++++++---------------------
+ 1 file changed, 28 insertions(+), 21 deletions(-)
+Merging wpan-staging/staging (1dd9291eb9037 ieee802154: Remove WARN_ON() in cfg802154_pernet_exit())
+$ git merge -m Merge branch 'staging' of https://git.kernel.org/pub/scm/linux/kernel/git/wpan/wpan-next.git wpan-staging/staging
+Already up to date.
+Merging mtd/mtd/next (87eee18f2974c mtd: sm_ftl: allocate cis_buffer with main struct)
+$ git merge -m Merge branch 'mtd/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git mtd/mtd/next
+Merge made by the 'ort' strategy.
+ drivers/mtd/mtdoops.c        |  7 +++----
+ drivers/mtd/mtdsuper.c       |  8 ++++----
+ drivers/mtd/nand/raw/Kconfig |  2 +-
+ drivers/mtd/sm_ftl.c         | 30 +++++++++++-------------------
+ drivers/mtd/sm_ftl.h         |  2 +-
+ 5 files changed, 20 insertions(+), 29 deletions(-)
+Merging nand/nand/next (9e3997d3ab661 mtd: spinand: Prevent continuous reads on some controllers)
+$ git merge -m Merge branch 'nand/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git nand/nand/next
+Merge made by the 'ort' strategy.
+ drivers/mtd/nand/spi/core.c    | 147 ++++++++++++-------
+ drivers/mtd/nand/spi/winbond.c | 317 ++++++++++++++++++++++++++++++++---------
+ drivers/mtd/spi-nor/core.c     |  22 +--
+ drivers/spi/spi-airoha-snfi.c  |   6 +-
+ drivers/spi/spi-aspeed-smc.c   |   4 +-
+ drivers/spi/spi-intel.c        |   6 +-
+ drivers/spi/spi-mem.c          |  32 ++++-
+ drivers/spi/spi-mxic.c         |  18 +--
+ drivers/spi/spi-npcm-fiu.c     |  16 +--
+ drivers/spi/spi-rpc-if.c       |   8 +-
+ drivers/spi/spi-stm32-ospi.c   |   6 +-
+ drivers/spi/spi-stm32-qspi.c   |   6 +-
+ drivers/spi/spi-wpcm-fiu.c     |   2 +-
+ include/linux/mtd/spinand.h    |  16 ++-
+ include/linux/spi/spi-mem.h    |  12 +-
+ 15 files changed, 446 insertions(+), 172 deletions(-)
+Merging spi-nor/spi-nor/next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'spi-nor/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git spi-nor/spi-nor/next
+Already up to date.
+Merging crypto/master (6c9dddeb582fd crypto: krb5 - filter out async aead implementations at alloc)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/herbert/cryptodev-2.6.git crypto/master
+Auto-merging Documentation/devicetree/bindings/trivial-devices.yaml
+Auto-merging crypto/krb5/krb5_api.c
+Merge made by the 'ort' strategy.
+ Documentation/crypto/api-samples.rst               |    2 +-
+ Documentation/crypto/userspace-if.rst              |  107 +-
+ .../devicetree/bindings/crypto/qcom,prng.yaml      |    2 +
+ .../devicetree/bindings/crypto/qcom-qce.yaml       |    4 +
+ Documentation/devicetree/bindings/rng/mtk-rng.yaml |   43 +-
+ .../devicetree/bindings/trivial-devices.yaml       |    4 +-
+ arch/m68k/configs/amiga_defconfig                  |    2 -
+ arch/m68k/configs/apollo_defconfig                 |    2 -
+ arch/m68k/configs/atari_defconfig                  |    2 -
+ arch/m68k/configs/bvme6000_defconfig               |    2 -
+ arch/m68k/configs/hp300_defconfig                  |    2 -
+ arch/m68k/configs/mac_defconfig                    |    2 -
+ arch/m68k/configs/multi_defconfig                  |    2 -
+ arch/m68k/configs/mvme147_defconfig                |    2 -
+ arch/m68k/configs/mvme16x_defconfig                |    2 -
+ arch/m68k/configs/q40_defconfig                    |    2 -
+ arch/m68k/configs/sun3_defconfig                   |    2 -
+ arch/m68k/configs/sun3x_defconfig                  |    2 -
+ arch/mips/configs/decstation_64_defconfig          |    2 -
+ arch/mips/configs/decstation_defconfig             |    2 -
+ arch/mips/configs/decstation_r4k_defconfig         |    2 -
+ crypto/Kconfig                                     |  109 +-
+ crypto/Makefile                                    |    5 +
+ crypto/acompress.c                                 |    8 +-
+ crypto/aead.c                                      |   10 +-
+ crypto/af_alg.c                                    |   71 +-
+ crypto/ahash.c                                     |    8 +-
+ crypto/akcipher.c                                  |    8 +-
+ crypto/algif_aead.c                                |    8 +-
+ crypto/asymmetric_keys/Kconfig                     |    1 -
+ crypto/asymmetric_keys/verify_pefile.c             |    2 +
+ crypto/authencesn.c                                |   33 +-
+ crypto/crypto_null.c                               |   35 +-
+ crypto/crypto_user.c                               |   14 +-
+ crypto/df_sp80090a.c                               |    8 +-
+ crypto/drbg.c                                      | 1819 +++-----------------
+ crypto/ecrdsa.c                                    |    2 +-
+ crypto/jitterentropy.c                             |    6 +-
+ crypto/kpp.c                                       |    8 +-
+ crypto/krb5/krb5_api.c                             |    2 +-
+ crypto/lskcipher.c                                 |   10 +-
+ crypto/rng.c                                       |    8 +-
+ crypto/scompress.c                                 |    8 +-
+ crypto/shash.c                                     |    8 +-
+ crypto/sig.c                                       |    6 +-
+ crypto/skcipher.c                                  |   10 +-
+ crypto/testmgr.c                                   |  160 +-
+ crypto/testmgr.h                                   | 1080 +++---------
+ drivers/char/hw_random/amd-rng.c                   |    6 +-
+ drivers/char/hw_random/cavium-rng.c                |    4 +-
+ drivers/char/hw_random/core.c                      |   30 +-
+ drivers/char/hw_random/geode-rng.c                 |    4 +-
+ drivers/char/hw_random/mtk-rng.c                   |  125 +-
+ drivers/crypto/allwinner/Kconfig                   |    2 -
+ drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c  |    8 +
+ drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c  |   12 +
+ drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c  |   12 +
+ drivers/crypto/amlogic/amlogic-gxl-core.c          |    2 +-
+ drivers/crypto/atmel-ecc.c                         |   17 +-
+ drivers/crypto/atmel-sha204a.c                     |   35 +-
+ drivers/crypto/axis/artpec6_crypto.c               |   21 +-
+ drivers/crypto/caam/caamalg.c                      |   12 +-
+ drivers/crypto/caam/caamalg_qi.c                   |   12 +-
+ drivers/crypto/caam/caamalg_qi2.c                  |   12 +-
+ drivers/crypto/caam/caamhash.c                     |    4 +-
+ drivers/crypto/caam/key_gen.c                      |    4 +-
+ drivers/crypto/cavium/cpt/cptpf_main.c             |    2 +-
+ drivers/crypto/cavium/cpt/cptvf_main.c             |    6 +-
+ drivers/crypto/cavium/nitrox/nitrox_main.c         |    4 +-
+ drivers/crypto/ccp/psp-dev.c                       |   12 +-
+ drivers/crypto/ccp/sev-dev-tsm.c                   |   13 +-
+ drivers/crypto/ccp/sev-dev.c                       |   96 +-
+ drivers/crypto/ccp/sp-pci.c                        |   28 +-
+ drivers/crypto/ccree/cc_aead.c                     |    9 +-
+ drivers/crypto/ccree/cc_cipher.c                   |    7 +-
+ drivers/crypto/ccree/cc_hash.c                     |   13 +-
+ drivers/crypto/inside-secure/safexcel.c            |    4 +-
+ drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c        |   25 +-
+ .../crypto/intel/qat/qat_common/adf_common_drv.h   |    1 -
+ .../intel/qat/qat_common/adf_heartbeat_inject.c    |    6 +-
+ .../crypto/intel/qat/qat_common/adf_hw_arbiter.c   |   25 -
+ drivers/crypto/marvell/cesa/cesa.c                 |   11 +-
+ drivers/crypto/marvell/cesa/cesa.h                 |   42 +-
+ drivers/crypto/marvell/octeontx/otx_cptpf_main.c   |    2 +-
+ drivers/crypto/marvell/octeontx/otx_cptvf_main.c   |    6 +-
+ drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c |    2 +-
+ drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c |    8 +-
+ drivers/crypto/omap-aes.c                          |   43 +-
+ drivers/crypto/omap-des.c                          |   26 +-
+ drivers/crypto/omap-sham.c                         |   27 +-
+ drivers/crypto/starfive/jh7110-cryp.c              |   17 +-
+ drivers/crypto/talitos.c                           |  592 ++++---
+ drivers/crypto/talitos.h                           |   17 +-
+ drivers/crypto/xilinx/xilinx-trng.c                |    1 -
+ include/crypto/df_sp80090a.h                       |   25 +
+ include/crypto/drbg.h                              |  263 ---
+ include/crypto/internal/drbg.h                     |   54 -
+ tools/testing/crypto/chacha20-s390/test-cipher.c   |    1 -
+ 98 files changed, 1497 insertions(+), 3852 deletions(-)
+ delete mode 100644 include/crypto/drbg.h
+ delete mode 100644 include/crypto/internal/drbg.h
+Merging libcrypto/libcrypto-next (cf52058dcdd96 lib/crypto: powerpc/md5: Drop powerpc optimized MD5 code)
+$ git merge -m Merge branch 'libcrypto-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git libcrypto/libcrypto-next
+Merge made by the 'ort' strategy.
+ lib/crypto/Kconfig           |   5 -
+ lib/crypto/Makefile          |   4 -
+ lib/crypto/md5.c             |  20 ++--
+ lib/crypto/powerpc/md5-asm.S | 235 -------------------------------------------
+ lib/crypto/powerpc/md5.h     |  12 ---
+ 5 files changed, 7 insertions(+), 269 deletions(-)
+ delete mode 100644 lib/crypto/powerpc/md5-asm.S
+ delete mode 100644 lib/crypto/powerpc/md5.h
+Merging drm/drm-next (4b18a9d44e38c Merge tag 'amd-drm-next-7.2-2026-05-20' of https://gitlab.freedesktop.org/agd5f/linux into drm-next)
+  014f329074f68 ("drm/amd/pm: fix memleak of dpm_policies on smu v15")
+  15c369257bd85 ("drm/amdgpu/vce3: Fix VCE 3 firmware size and offsets")
+  160164609f71f ("drm/amdgpu/userq: cancel reset work while tear down in progress")
+  1f087bb8cf9e8 ("drm/amdkfd: Check bounds on allocate_doorbell")
+  33d8951405e2d ("drm/amdgpu/vce1: Stop using amdgpu_vce_resume")
+  3aab50410653f ("drm/amdgpu/userq: use drm_exec in amdgpu_userq_fence_read_wptr")
+  4d35a45c9b4c1 ("drm/amdgpu: remove va cursors for all mappings")
+  4d50a14d34614 ("drm/amdgpu: use atomic operation to achieve lockless serialization")
+  530411b465ef0 ("drm/amdgpu: Align amdgpu_gtt_mgr entries to TLB size on Tahiti (v2)")
+  5c02889ea2257 ("drm/amdgpu: userq_va_mapped should remain true once done")
+  70cadefcc6160 ("drm/amdgpu: unmap all user mappings of framebuffer and doorbell before mode1 reset")
+  7a076df36397d ("drm/amdgpu: reject non-user addresses early in GEM_USERPTR ioctl")
+  7ef144458f48d ("drm/amdkfd: Fix OOB memory exposure in get_wave_state()")
+  89e50de5654db ("drm/amdgpu: fix handling in amdgpu_userq_create")
+  8d2a20c1721cb ("drm/amdgpu/vce1: Don't repeat GTT MGR node allocation")
+  911b1bdd22c37 ("drm/amd/ras: Fix UMC error address allocation leak")
+  a20d21df62554 ("drm/amdgpu/vce2: Fix VCE 2 firmware size and offsets")
+  a3bbf32a33693 ("drm/amdgpu/userq: pin mqd and fw object bo to avoid eviction")
+  a8b749c5c5afb ("drm/amdgpu/vpe: Force collaborate sync after TRAP")
+  a9b14d88b4d83 ("drm/amdgpu/userq: update the vm task info during signal ioctl")
+  ba4caa9fecdf7 ("drm/amd/display: Validate payload length and link_index in dc_process_dmub_aux_transfer_async")
+  ba5e95b43b773 ("drm/amd/display: Validate GPIO pin LUT table size before iterating")
+  bfe9a7545b2a7 ("drm/amdkfd: Check bounds for allocate_sdma_queue restore_sdma_id")
+  c16fe59f622a0 ("drm/amdgpu/vce1: Fix VCE 1 firmware size and offsets")
+  c1dc555e760db ("drm/amdgpu/vce1: Remove superfluous address check")
+  cc768f4dd0bb9 ("drm/amdgpu: avoid integer overflow in VA range check")
+  d3f5bbd007133 ("drm/amdgpu: Fix discovery offset check under VF")
+  d40fb392af659 ("drm/amd/display: Fix integer overflow in bios_get_image()")
+  e20ea411c99f6 ("drm/radeon/evergreen_cs: Add missing NULL prefix check in surface check")
+  e729ae5f3ac73 ("drm/amdgpu/vce1: Check that the GPU address is < 128 MiB")
+  f370ec9b16469 ("drm/amdgpu/vce1: Check if VRAM address is lower than GART.")
+  fd9200ccefab9 ("drm/amdgpu: rework userq reset work handling")
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git drm/drm-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Auto-merging Documentation/gpu/drm-uapi.rst
+Auto-merging MAINTAINERS
+Auto-merging drivers/accel/amdxdna/amdxdna_gem.c
+Auto-merging drivers/accel/amdxdna/amdxdna_ubuf.c
+Auto-merging drivers/accel/ivpu/ivpu_drv.c
+Auto-merging drivers/accel/qaic/qaic_data.c
+Auto-merging drivers/accel/qaic/qaic_ras.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu.h
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+Auto-merging drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_device.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+Auto-merging drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+Auto-merging drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
+Auto-merging drivers/gpu/drm/amd/display/dc/core/dc.c
+Auto-merging drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
+Auto-merging drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
+Auto-merging drivers/gpu/drm/bridge/chipone-icn6211.c
+Auto-merging drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
+Auto-merging drivers/gpu/drm/bridge/ite-it66121.c
+Auto-merging drivers/gpu/drm/drm_fb_helper.c
+Auto-merging drivers/gpu/drm/drm_gem.c
+Auto-merging drivers/gpu/drm/etnaviv/etnaviv_sched.c
+Auto-merging drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
+Auto-merging drivers/gpu/drm/i915/display/intel_display_types.h
+Auto-merging drivers/gpu/drm/i915/display/intel_dp.c
+Auto-merging drivers/gpu/drm/i915/display/intel_plane.c
+Auto-merging drivers/gpu/drm/i915/display/intel_psr.c
+Auto-merging drivers/gpu/drm/i915/gt/intel_reset.c
+Auto-merging drivers/gpu/drm/i915/i915_driver.c
+Auto-merging drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c
+Auto-merging drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
+Auto-merging drivers/gpu/drm/msm/msm_drv.h
+Auto-merging drivers/gpu/drm/msm/msm_gem_vma.c
+Auto-merging drivers/gpu/drm/msm/msm_ringbuffer.c
+Auto-merging drivers/gpu/drm/panel/Kconfig
+Auto-merging drivers/gpu/drm/panel/panel-himax-hx83102.c
+Auto-merging drivers/gpu/drm/panfrost/panfrost_drv.c
+Auto-merging drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c
+Auto-merging drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
+Auto-merging drivers/gpu/drm/tiny/bochs.c
+Auto-merging drivers/gpu/drm/v3d/v3d_sched.c
+Auto-merging drivers/gpu/drm/v3d/v3d_submit.c
+Auto-merging drivers/gpu/drm/virtio/virtgpu_drv.h
+Auto-merging drivers/gpu/drm/virtio/virtgpu_plane.c
+Auto-merging drivers/gpu/drm/xe/xe_bo.c
+Auto-merging drivers/gpu/drm/xe/xe_bo_types.h
+Auto-merging drivers/gpu/drm/xe/xe_gt_types.h
+Auto-merging drivers/gpu/drm/xe/xe_guc_ads.c
+Auto-merging drivers/gpu/drm/xe/xe_lrc.c
+Auto-merging drivers/gpu/drm/xe/xe_lrc.h
+Auto-merging drivers/gpu/drm/xe/xe_migrate.c
+CONFLICT (content): Merge conflict in drivers/gpu/drm/xe/xe_migrate.c
+Auto-merging drivers/gpu/drm/xe/xe_pci.c
+Auto-merging drivers/gpu/drm/xe/xe_vm.c
+Resolved 'drivers/gpu/drm/xe/xe_migrate.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 14737e54abdd9] Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/obsolete/sysfs-driver-ivpu       |    30 +
+ Documentation/ABI/testing/sysfs-driver-ivpu        |    65 +
+ Documentation/accel/amdxdna/amdnpu.rst             |    25 +
+ .../bindings/display/bridge/lontium,lt9211.yaml    |    43 +-
+ .../bindings/display/bridge/lvds-codec.yaml        |     1 +
+ .../bindings/display/bridge/simple-bridge.yaml     |     1 +
+ .../bindings/display/bridge/waveshare,dsi2dpi.yaml |     9 +-
+ .../display/panel/chipwealth,ch13726a.yaml         |    67 +
+ .../bindings/display/panel/focaltech,ota7290b.yaml |    70 +
+ .../bindings/display/panel/himax,hx83102.yaml      |     2 +
+ .../bindings/display/panel/himax,hx8394.yaml       |     2 +
+ .../bindings/display/panel/ilitek,ili9881c.yaml    |     2 +
+ .../bindings/display/panel/jadard,jd9365da-h3.yaml |     6 +
+ .../bindings/display/panel/novatek,nt35532.yaml    |    80 +
+ .../bindings/display/panel/panel-simple-dsi.yaml   |     2 +
+ .../bindings/display/panel/panel-simple.yaml       |    32 +
+ .../devicetree/bindings/gpu/arm,mali-bifrost.yaml  |     2 +
+ .../devicetree/bindings/vendor-prefixes.yaml       |     2 +
+ Documentation/gpu/amdgpu/amdgpu-glossary.rst       |     9 +-
+ Documentation/gpu/amdgpu/index.rst                 |     1 +
+ Documentation/gpu/amdgpu/ptl.rst                   |    94 +
+ Documentation/gpu/driver-uapi.rst                  |     2 +
+ Documentation/gpu/drivers.rst                      |     1 +
+ Documentation/gpu/drm-internals.rst                |     2 +
+ Documentation/gpu/drm-kms-helpers.rst              |     2 +
+ Documentation/gpu/drm-kms.rst                      |    19 +-
+ Documentation/gpu/drm-mm.rst                       |     2 +
+ Documentation/gpu/drm-ras.rst                      |    10 +
+ Documentation/gpu/drm-uapi.rst                     |     8 +-
+ Documentation/gpu/drm-usage-stats.rst              |     3 +
+ Documentation/gpu/i915.rst                         |   202 +-
+ Documentation/gpu/index.rst                        |     1 +
+ Documentation/gpu/intel-display/async-flip.rst     |     8 +
+ Documentation/gpu/intel-display/atomic.rst         |    11 +
+ Documentation/gpu/intel-display/audio.rst          |    23 +
+ Documentation/gpu/intel-display/casf.rst           |     8 +
+ Documentation/gpu/intel-display/cdclk.rst          |    11 +
+ Documentation/gpu/intel-display/cmtg.rst           |     8 +
+ Documentation/gpu/intel-display/dmc.rst            |    26 +
+ Documentation/gpu/intel-display/dpio.rst           |     8 +
+ Documentation/gpu/intel-display/dpll.rst           |    14 +
+ Documentation/gpu/intel-display/drrs.rst           |    11 +
+ Documentation/gpu/intel-display/dsb.rst            |    11 +
+ Documentation/gpu/intel-display/fbc.rst            |    11 +
+ Documentation/gpu/intel-display/fifo-underrun.rst  |    11 +
+ Documentation/gpu/intel-display/frontbuffer.rst    |    14 +
+ Documentation/gpu/intel-display/hotplug.rst        |    11 +
+ Documentation/gpu/intel-display/index.rst          |    44 +
+ Documentation/gpu/intel-display/plane.rst          |    11 +
+ Documentation/gpu/intel-display/psr.rst            |    11 +
+ Documentation/gpu/intel-display/snps-phy.rst       |     8 +
+ Documentation/gpu/intel-display/vbt.rst            |    14 +
+ Documentation/gpu/introduction.rst                 |     2 +
+ Documentation/gpu/komeda-kms.rst                   |     8 +-
+ Documentation/gpu/rfc/index.rst                    |    26 +-
+ Documentation/gpu/todo.rst                         |    23 -
+ Documentation/gpu/xe/index.rst                     |     6 +
+ Documentation/gpu/xe/xe_gt_stats.rst               |    11 +
+ Documentation/netlink/specs/drm_ras.yaml           |    13 +-
+ MAINTAINERS                                        |    13 +-
+ drivers/accel/amdxdna/Makefile                     |    13 +-
+ drivers/accel/amdxdna/aie.c                        |   167 +
+ drivers/accel/amdxdna/aie.h                        |   120 +
+ drivers/accel/amdxdna/aie2_ctx.c                   |    90 +-
+ drivers/accel/amdxdna/aie2_error.c                 |    24 +-
+ drivers/accel/amdxdna/aie2_message.c               |   288 +-
+ drivers/accel/amdxdna/aie2_msg_priv.h              |    41 +-
+ drivers/accel/amdxdna/aie2_pci.c                   |   282 +-
+ drivers/accel/amdxdna/aie2_pci.h                   |   153 +-
+ drivers/accel/amdxdna/aie2_pm.c                    |    22 +-
+ drivers/accel/amdxdna/aie2_psp.c                   |   161 -
+ drivers/accel/amdxdna/aie2_smu.c                   |   156 -
+ drivers/accel/amdxdna/aie2_solver.c                |    10 +-
+ drivers/accel/amdxdna/aie4_ctx.c                   |   333 +
+ drivers/accel/amdxdna/aie4_host_queue.h            |    24 +
+ drivers/accel/amdxdna/aie4_message.c               |    83 +
+ drivers/accel/amdxdna/aie4_msg_priv.h              |   147 +
+ drivers/accel/amdxdna/aie4_pci.c                   |   667 +
+ drivers/accel/amdxdna/aie4_pci.h                   |    89 +
+ drivers/accel/amdxdna/aie4_sriov.c                 |    88 +
+ drivers/accel/amdxdna/aie_psp.c                    |   235 +
+ drivers/accel/amdxdna/aie_smu.c                    |   153 +
+ drivers/accel/amdxdna/amdxdna_cbuf.c               |   280 +
+ drivers/accel/amdxdna/amdxdna_cbuf.h               |    18 +
+ drivers/accel/amdxdna/amdxdna_ctx.c                |    43 +-
+ drivers/accel/amdxdna/amdxdna_ctx.h                |     8 +-
+ drivers/accel/amdxdna/amdxdna_debugfs.c            |   129 +
+ drivers/accel/amdxdna/amdxdna_debugfs.h            |    18 +
+ drivers/accel/amdxdna/amdxdna_gem.c                |   100 +-
+ drivers/accel/amdxdna/amdxdna_iommu.c              |    76 +-
+ drivers/accel/amdxdna/amdxdna_mailbox.c            |    30 +-
+ drivers/accel/amdxdna/amdxdna_mailbox.h            |     8 +-
+ drivers/accel/amdxdna/amdxdna_pci_drv.c            |   157 +-
+ drivers/accel/amdxdna/amdxdna_pci_drv.h            |    28 +-
+ drivers/accel/amdxdna/amdxdna_sysfs.c              |     5 +-
+ drivers/accel/amdxdna/amdxdna_ubuf.c               |    41 +-
+ drivers/accel/amdxdna/npu1_regs.c                  |    30 +-
+ drivers/accel/amdxdna/npu3_regs.c                  |   100 +
+ drivers/accel/amdxdna/npu4_regs.c                  |    71 +-
+ drivers/accel/amdxdna/npu5_regs.c                  |    10 +-
+ drivers/accel/amdxdna/npu6_regs.c                  |    10 +-
+ drivers/accel/ethosu/ethosu_job.c                  |     1 -
+ drivers/accel/ivpu/ivpu_drv.c                      |     8 +-
+ drivers/accel/ivpu/ivpu_hw.h                       |    16 +-
+ drivers/accel/ivpu/ivpu_hw_btrs.c                  |   112 +-
+ drivers/accel/ivpu/ivpu_hw_btrs.h                  |    10 +-
+ drivers/accel/ivpu/ivpu_hw_ip.c                    |    24 +-
+ drivers/accel/ivpu/ivpu_jsm_msg.c                  |    18 +-
+ drivers/accel/ivpu/ivpu_jsm_msg.h                  |     3 +-
+ drivers/accel/ivpu/ivpu_sysfs.c                    |   190 +-
+ drivers/accel/ivpu/vpu_jsm_api.h                   |    34 +-
+ drivers/accel/qaic/mhi_controller.c                |     2 +-
+ drivers/accel/qaic/mhi_controller.h                |     9 +-
+ drivers/accel/qaic/qaic.h                          |    13 +-
+ drivers/accel/qaic/qaic_control.c                  |     2 +-
+ drivers/accel/qaic/qaic_data.c                     |     2 +-
+ drivers/accel/qaic/qaic_debugfs.c                  |    30 +-
+ drivers/accel/qaic/qaic_debugfs.h                  |     2 +-
+ drivers/accel/qaic/qaic_drv.c                      |    10 +-
+ drivers/accel/qaic/qaic_ras.c                      |     1 -
+ drivers/accel/qaic/qaic_ras.h                      |     1 +
+ drivers/accel/qaic/qaic_ssr.c                      |     2 +-
+ drivers/accel/qaic/qaic_ssr.h                      |     9 +-
+ drivers/accel/qaic/qaic_timesync.c                 |     3 +-
+ drivers/accel/qaic/qaic_timesync.h                 |     7 +-
+ drivers/accel/qaic/sahara.c                        |     2 +-
+ drivers/accel/qaic/sahara.h                        |     2 +-
+ drivers/accel/rocket/rocket_job.c                  |     1 -
+ drivers/dma-buf/.kunitconfig                       |     2 +
+ drivers/dma-buf/Kconfig                            |    11 +-
+ drivers/dma-buf/Makefile                           |     5 +-
+ drivers/dma-buf/dma-fence-array.c                  |    13 +-
+ drivers/dma-buf/dma-fence-unwrap.c                 |     3 +-
+ drivers/dma-buf/dma-fence.c                        |     3 +-
+ drivers/dma-buf/dma-resv.c                         |     3 +-
+ drivers/dma-buf/selftest.c                         |   167 -
+ drivers/dma-buf/selftest.h                         |    30 -
+ drivers/dma-buf/selftests.h                        |    16 -
+ drivers/dma-buf/st-dma-fence-chain.c               |   217 +-
+ drivers/dma-buf/st-dma-fence-unwrap.c              |   292 +-
+ drivers/dma-buf/st-dma-fence.c                     |   200 +-
+ drivers/dma-buf/st-dma-resv.c                      |   145 +-
+ drivers/dma-buf/udmabuf.c                          |    58 +-
+ drivers/gpu/buddy.c                                |    11 +
+ drivers/gpu/drm/adp/adp_drv.c                      |    14 +-
+ drivers/gpu/drm/amd/amdgpu/Kconfig                 |     1 -
+ drivers/gpu/drm/amd/amdgpu/Makefile                |     5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu.h                |    46 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c            |    76 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h            |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h         |    14 +
+ .../gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c    |    13 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c       |    24 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c           |     4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c           |    30 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c             |     6 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c        |     5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_device.c         |    31 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c      |    32 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h      |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c            |    63 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c          |    26 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c            |    25 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c            |    51 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h            |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c            |    16 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h            |     4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h             |     6 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h            |     1 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_job.c            |    27 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_job.h            |     5 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c            |    87 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h            |    24 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h           |     4 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c           |    10 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h           |     2 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_object.c         |    18 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c            |   442 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h            |    29 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c            |    27 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c     |    16 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c     |    18 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h     |     3 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c          |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h          |    69 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h           |     3 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c           |     4 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h          |     8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c            |   328 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h            |    20 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c          |    22 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c            |     2 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c            |    52 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c           |    40 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h           |     4 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c           |   127 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c             |   552 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h             |    75 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c         |    11 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c          |    13 -
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c        |    19 +-
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c            |    25 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.h            |     1 +
+ drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c            |     8 +-
+ drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h        |    15 +-
+ drivers/gpu/drm/amd/amdgpu/atom.c                  |    11 +
+ drivers/gpu/drm/amd/amdgpu/atom.h                  |     3 +
+ drivers/gpu/drm/amd/amdgpu/cik_sdma.c              |    13 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c             |     5 +
+ drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c             |    59 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c             |    57 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c             |    99 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h         |   415 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c              |     7 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c              |     6 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c              |     6 +-
+ drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c            |    90 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c             |    24 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c             |     5 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c             |    11 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c             |    13 +
+ drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c              |    28 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c              |    30 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c              |    34 +-
+ drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c              |    75 +-
+ drivers/gpu/drm/amd/amdgpu/ih_v7_0.c               |    40 +-
+ drivers/gpu/drm/amd/amdgpu/imu_v12_1.c             |    12 +-
+ drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c             |     2 +-
+ drivers/gpu/drm/amd/amdgpu/mes_userqueue.c         |     2 +-
+ drivers/gpu/drm/amd/amdgpu/mes_v12_1.c             |   150 +-
+ drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c           |    58 +-
+ drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c           |   369 +
+ drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.h           |    31 +
+ drivers/gpu/drm/amd/amdgpu/nv.c                    |    31 +-
+ drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h            |    17 +
+ drivers/gpu/drm/amd/amdgpu/psp_v11_0.c             |     8 +-
+ drivers/gpu/drm/amd/amdgpu/psp_v12_0.c             |     8 +-
+ drivers/gpu/drm/amd/amdgpu/psp_v13_0.c             |    14 +-
+ drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c           |    14 +-
+ drivers/gpu/drm/amd/amdgpu/psp_v14_0.c             |    14 +-
+ drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c           |     1 +
+ drivers/gpu/drm/amd/amdgpu/psp_v3_1.c              |     8 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c             |     8 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c             |     8 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c             |    17 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c           |    12 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c             |    11 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c             |    14 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c             |     5 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c             |     5 +-
+ drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c             |    12 +-
+ drivers/gpu/drm/amd/amdgpu/si_dma.c                |    12 +-
+ drivers/gpu/drm/amd/amdgpu/soc15.c                 |    30 +-
+ drivers/gpu/drm/amd/amdgpu/soc21.c                 |    30 +-
+ drivers/gpu/drm/amd/amdgpu/soc24.c                 |    21 +-
+ drivers/gpu/drm/amd/amdgpu/soc_v1_0.c              |    33 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c            |     2 +-
+ drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c            |     2 +-
+ drivers/gpu/drm/amd/amdgpu/vpe_v2_0.c              |   350 +
+ drivers/gpu/drm/amd/amdgpu/vpe_v2_0.h              |    29 +
+ drivers/gpu/drm/amd/amdkfd/kfd_chardev.c           |   221 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_device.c            |    20 +
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c  |   179 +-
+ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.h  |     6 +-
+ .../amd/amdkfd/kfd_device_queue_manager_v12_1.c    |    19 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_events.c            |     5 +
+ drivers/gpu/drm/amd/amdkfd/kfd_migrate.c           |     3 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c   |    16 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c   |    14 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c   |     8 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c |     6 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c    |    15 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c    |    11 +
+ drivers/gpu/drm/amd/amdkfd/kfd_priv.h              |    27 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_process.c           |    91 +-
+ drivers/gpu/drm/amd/amdkfd/kfd_topology.c          |     4 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c  |   453 +-
+ drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h  |    16 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_color.c    |    12 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c  |    43 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c |    74 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h |     5 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c  |   137 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c  |    86 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h  |     1 +
+ .../amd/display/amdgpu_dm/amdgpu_dm_mst_types.c    |    22 +-
+ .../amd/display/amdgpu_dm/amdgpu_dm_mst_types.h    |     4 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c    |     6 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c  |   244 +-
+ .../gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h  |    13 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c   |   143 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_replay.h   |    28 +-
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_services.c |    27 +
+ .../drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h    |    20 +-
+ drivers/gpu/drm/amd/display/dc/basics/conversion.c |     2 +-
+ .../gpu/drm/amd/display/dc/basics/custom_float.c   |     2 +-
+ drivers/gpu/drm/amd/display/dc/basics/dce_calcs.c  |     6 +-
+ drivers/gpu/drm/amd/display/dc/bios/bios_parser.c  |     6 +-
+ drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c |    21 +-
+ .../gpu/drm/amd/display/dc/bios/command_table.c    |    12 +-
+ .../gpu/drm/amd/display/dc/bios/command_table2.c   |     4 +-
+ .../amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c    |     6 +-
+ .../amd/display/dc/clk_mgr/dce100/dce_clk_mgr.h    |     2 +-
+ .../amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c |    12 +-
+ .../drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c |     2 +-
+ .../amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c   |    19 +-
+ .../drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c  |    42 +-
+ .../amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c   |    56 +-
+ .../dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.c       |     4 +-
+ .../dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.h       |     2 +-
+ .../drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c |    43 +-
+ .../amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c   |    55 +-
+ .../amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c |    69 +-
+ .../amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c |    68 +-
+ .../amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c |    58 +-
+ .../amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c   |    66 +-
+ .../amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c   |   127 +-
+ .../amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c |    61 +-
+ .../amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h |     4 +-
+ .../dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.c     |     4 +-
+ .../dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.h     |     2 +-
+ .../amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c   |    61 +-
+ .../drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c   |     1 -
+ drivers/gpu/drm/amd/display/dc/core/dc.c           |   881 +-
+ .../gpu/drm/amd/display/dc/core/dc_hw_sequencer.c  |  1090 +-
+ .../gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c  |    11 +-
+ .../gpu/drm/amd/display/dc/core/dc_link_exports.c  |     2 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_resource.c  |   151 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_state.c     |    10 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_stream.c    |    44 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_surface.c   |     8 +-
+ drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c |     4 +-
+ drivers/gpu/drm/amd/display/dc/dc.h                |   161 +-
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c       |   151 +-
+ drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h       |     9 +-
+ drivers/gpu/drm/amd/display/dc/dc_dsc.h            |     2 +-
+ drivers/gpu/drm/amd/display/dc/dc_fused_io.c       |     6 +-
+ drivers/gpu/drm/amd/display/dc/dc_helper.c         |    16 +-
+ drivers/gpu/drm/amd/display/dc/dc_hw_types.h       |     4 +-
+ drivers/gpu/drm/amd/display/dc/dc_stream.h         |    43 +-
+ drivers/gpu/drm/amd/display/dc/dc_types.h          |    56 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c |     4 +-
+ .../drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c   |    20 +-
+ .../gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c |     2 +
+ drivers/gpu/drm/amd/display/dc/dce/dce_aux.c       |    24 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_aux.h       |     4 +
+ .../gpu/drm/amd/display/dc/dce/dce_clock_source.c  |    24 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c    |     2 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c    |    16 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c       |     2 +-
+ .../gpu/drm/amd/display/dc/dce/dce_link_encoder.c  |     3 +-
+ .../gpu/drm/amd/display/dc/dce/dce_panel_cntl.c    |     4 +-
+ drivers/gpu/drm/amd/display/dc/dce/dce_transform.c |    12 +-
+ drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c  |    16 +-
+ drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c      |    12 +-
+ drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c   |    23 +-
+ .../amd/display/dc/dce80/dce80_timing_generator.c  |     2 +-
+ .../gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c |    16 +-
+ .../display/dc/dcn10/dcn10_hw_sequencer_debug.c    |    22 +-
+ .../gpu/drm/amd/display/dc/dcn20/dcn20_dwb_scl.c   |     2 +-
+ .../drm/amd/display/dc/dcn21/dcn21_link_encoder.c  |     2 +-
+ .../gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c |     8 +-
+ .../amd/display/dc/dio/dcn10/dcn10_link_encoder.c  |    13 +-
+ .../dc/dio/dcn30/dcn30_dio_stream_encoder.c        |     6 +-
+ .../dc/dio/dcn401/dcn401_dio_stream_encoder.c      |     2 +-
+ .../display/dc/dio/dcn42/dcn42_dio_link_encoder.h  |     2 -
+ .../dc/dio/dcn42/dcn42_dio_stream_encoder.c        |     4 +-
+ drivers/gpu/drm/amd/display/dc/dm_helpers.h        |     1 +
+ .../drm/amd/display/dc/dml/calcs/dcn_calc_auto.c   |  1824 +-
+ .../drm/amd/display/dc/dml/calcs/dcn_calc_math.c   |     6 +-
+ .../gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c   |   294 +-
+ .../gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c   |   159 +-
+ .../amd/display/dc/dml/dcn20/display_mode_vba_20.c |   328 +-
+ .../display/dc/dml/dcn20/display_mode_vba_20v2.c   |   326 +-
+ .../display/dc/dml/dcn20/display_rq_dlg_calc_20.c  |   164 +-
+ .../dc/dml/dcn20/display_rq_dlg_calc_20v2.c        |   164 +-
+ .../amd/display/dc/dml/dcn21/display_mode_vba_21.c |   368 +-
+ .../display/dc/dml/dcn21/display_rq_dlg_calc_21.c  |    80 +-
+ .../gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c   |    94 +-
+ .../amd/display/dc/dml/dcn30/display_mode_vba_30.c |   753 +-
+ .../display/dc/dml/dcn30/display_rq_dlg_calc_30.c  |    82 +-
+ .../gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c |    27 +-
+ .../gpu/drm/amd/display/dc/dml/dcn302/dcn302_fpu.c |    18 +-
+ .../gpu/drm/amd/display/dc/dml/dcn303/dcn303_fpu.c |    18 +-
+ .../gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c   |    41 +-
+ .../amd/display/dc/dml/dcn31/display_mode_vba_31.c |   881 +-
+ .../display/dc/dml/dcn31/display_rq_dlg_calc_31.c  |    76 +-
+ .../gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c |    23 +-
+ .../display/dc/dml/dcn314/display_mode_vba_314.c   |   885 +-
+ .../dc/dml/dcn314/display_rq_dlg_calc_314.c        |    80 +-
+ .../gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c   |   416 +-
+ .../amd/display/dc/dml/dcn32/display_mode_vba_32.c |   195 +-
+ .../dc/dml/dcn32/display_mode_vba_util_32.c        |   334 +-
+ .../dc/dml/dcn32/display_mode_vba_util_32.h        |     8 +-
+ .../display/dc/dml/dcn32/display_rq_dlg_calc_32.c  |   138 +-
+ .../gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c |    95 +-
+ .../gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c   |    37 +-
+ .../gpu/drm/amd/display/dc/dml/dcn351/dcn351_fpu.c |    37 +-
+ .../gpu/drm/amd/display/dc/dml/display_mode_vba.c  |     8 +-
+ .../gpu/drm/amd/display/dc/dml/display_mode_vba.h  |    12 +-
+ .../amd/display/dc/dml/dml1_display_rq_dlg_calc.c  |    32 +-
+ .../gpu/drm/amd/display/dc/dml/dml_inline_defs.h   |    18 +-
+ .../gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c   |    82 +-
+ drivers/gpu/drm/amd/display/dc/dml2_0/README.md    |    31 +
+ .../drm/amd/display/dc/dml2_0/display_mode_core.c  |    20 +-
+ .../drm/amd/display/dc/dml2_0/display_mode_util.c  |    20 +-
+ .../dc/dml2_0/dml21/dml21_translation_helper.c     |    48 +-
+ .../drm/amd/display/dc/dml2_0/dml21/dml21_utils.c  |     2 +-
+ .../display/dc/dml2_0/dml21/inc/dml_top_types.h    |     1 -
+ .../dml2_0/dml21/src/dml2_core/dml2_core_factory.c |     1 -
+ .../dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c |     1 -
+ .../dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c  |     2 +-
+ .../dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c   |     1 -
+ .../dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c   |     3 +-
+ .../dml21/src/dml2_top/dml2_top_interfaces.c       |     1 -
+ .../dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.h |     9 -
+ .../drm/amd/display/dc/dml2_0/dml2_mall_phantom.c  |   105 +-
+ .../gpu/drm/amd/display/dc/dml2_0/dml2_policy.c    |     6 +-
+ .../display/dc/dml2_0/dml2_translation_helper.c    |     4 +-
+ drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.c |    42 +-
+ .../drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c    |     4 +-
+ .../drm/amd/display/dc/dpp/dcn10/dcn10_dpp_dscl.c  |     2 +-
+ .../gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c   |    18 +-
+ .../drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c    |     5 +-
+ .../drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c  |     2 +-
+ .../amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c    |     3 +-
+ .../gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c   |    15 +-
+ drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c        |    15 +-
+ .../gpu/drm/amd/display/dc/dsc/dcn20/dcn20_dsc.c   |    27 +-
+ .../gpu/drm/amd/display/dc/dsc/dcn35/dcn35_dsc.c   |     8 +-
+ .../gpu/drm/amd/display/dc/dsc/dcn401/dcn401_dsc.c |    13 +-
+ drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c   |    33 +-
+ drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c       |     2 +-
+ drivers/gpu/drm/amd/display/dc/gpio/hw_generic.c   |     2 +-
+ drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.c      |     2 +-
+ drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.h      |     9 +
+ drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c       |     2 +-
+ .../dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c     |     2 +-
+ .../drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c |    16 +-
+ .../drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c |    28 +-
+ .../drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.h |     3 +
+ .../drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c |    12 +-
+ .../drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c |    12 +-
+ .../drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c |    12 +-
+ .../amd/display/dc/hubbub/dcn401/dcn401_hubbub.c   |     9 +-
+ .../drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c |     5 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c |     6 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c |     4 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c |     4 +-
+ .../drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c   |   233 +-
+ .../drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h   |    25 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c |    78 +-
+ .../gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h |    10 +-
+ .../drm/amd/display/dc/hwss/dce110/dce110_hwseq.c  |    71 +-
+ .../drm/amd/display/dc/hwss/dce120/dce120_hwseq.c  |    12 +-
+ .../drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c    |    85 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c |     2 +
+ .../drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c    |    47 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c |     2 +
+ .../drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c    |    10 +-
+ .../drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c    |    39 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c |     2 +
+ .../drm/amd/display/dc/hwss/dcn301/dcn301_init.c   |     2 +
+ .../drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c    |    12 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c |     2 +
+ .../drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c  |    88 +-
+ .../drm/amd/display/dc/hwss/dcn314/dcn314_init.c   |     2 +
+ .../drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c    |    38 +-
+ .../gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c |     2 +
+ .../drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c    |    83 +-
+ .../drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.c  |     2 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c  |   493 +-
+ .../drm/amd/display/dc/hwss/dcn401/dcn401_init.c   |     2 +
+ .../drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c    |   518 +-
+ .../drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h    |     9 +-
+ drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h |   393 +-
+ drivers/gpu/drm/amd/display/dc/inc/bw_fixed.h      |     4 +-
+ drivers/gpu/drm/amd/display/dc/inc/core_types.h    |    10 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h    |    22 +
+ drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h       |    39 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h        |    79 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/opp.h        |     4 +-
+ drivers/gpu/drm/amd/display/dc/inc/hw/optc.h       |     1 -
+ .../gpu/drm/amd/display/dc/inc/hw/stream_encoder.h |     1 +
+ .../drm/amd/display/dc/inc/hw/timing_generator.h   |     8 +-
+ drivers/gpu/drm/amd/display/dc/inc/link_service.h  |     2 +-
+ drivers/gpu/drm/amd/display/dc/inc/reg_helper.h    |    72 +-
+ .../drm/amd/display/dc/inc/soc_and_ip_translator.h |    19 +-
+ .../amd/display/dc/irq/dce110/irq_service_dce110.c |     2 +-
+ .../amd/display/dc/link/accessories/link_dp_cts.c  |    37 +-
+ .../amd/display/dc/link/hwss/link_hwss_hpo_dp.c    |     4 +-
+ .../gpu/drm/amd/display/dc/link/link_detection.c   |    12 +-
+ .../gpu/drm/amd/display/dc/link/link_detection.h   |     2 +-
+ drivers/gpu/drm/amd/display/dc/link/link_dpms.c    |    65 +-
+ drivers/gpu/drm/amd/display/dc/link/link_factory.c |     8 +-
+ .../gpu/drm/amd/display/dc/link/link_validation.c  |     4 +-
+ .../drm/amd/display/dc/link/protocols/link_ddc.c   |    12 +-
+ .../display/dc/link/protocols/link_dp_capability.c |     6 +-
+ .../amd/display/dc/link/protocols/link_dp_dpia.c   |     2 +-
+ .../display/dc/link/protocols/link_dp_dpia_bw.c    |    12 +-
+ .../dc/link/protocols/link_dp_irq_handler.c        |    55 +-
+ .../dc/link/protocols/link_dp_panel_replay.c       |    40 +-
+ .../display/dc/link/protocols/link_dp_training.c   |     4 +-
+ .../dc/link/protocols/link_dp_training_8b_10b.c    |    10 +-
+ .../dc/link/protocols/link_edp_panel_control.c     |    53 +-
+ .../amd/display/dc/mmhubbub/dcn20/dcn20_mmhubbub.c |     4 +-
+ .../amd/display/dc/mmhubbub/dcn32/dcn32_mmhubbub.c |     4 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c   |     4 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c   |     6 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c |   177 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h |    25 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c   |   392 +-
+ .../gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h   |    49 +-
+ .../gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c   |     2 +-
+ .../gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.h   |     2 +-
+ .../gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c |     2 +-
+ .../gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c |     4 +-
+ .../gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c |     2 +-
+ .../gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c |     1 -
+ .../gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c |     2 -
+ .../gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h |     1 -
+ .../drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c    |     6 +-
+ .../drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c    |    47 +-
+ .../display/dc/resource/dce100/dce100_resource.c   |    10 +-
+ .../display/dc/resource/dce110/dce110_resource.c   |    12 +-
+ .../display/dc/resource/dce112/dce112_resource.c   |     4 +-
+ .../display/dc/resource/dce120/dce120_resource.c   |     6 +-
+ .../amd/display/dc/resource/dce80/dce80_resource.c |     8 +-
+ .../amd/display/dc/resource/dcn10/dcn10_resource.c |    19 +-
+ .../amd/display/dc/resource/dcn20/dcn20_resource.c |    80 +-
+ .../amd/display/dc/resource/dcn21/dcn21_resource.c |    28 +-
+ .../amd/display/dc/resource/dcn30/dcn30_resource.c |    77 +-
+ .../display/dc/resource/dcn301/dcn301_resource.c   |    39 +-
+ .../display/dc/resource/dcn302/dcn302_resource.c   |    26 +-
+ .../display/dc/resource/dcn303/dcn303_resource.c   |    26 +-
+ .../amd/display/dc/resource/dcn31/dcn31_resource.c |    34 +-
+ .../display/dc/resource/dcn314/dcn314_resource.c   |    30 +-
+ .../display/dc/resource/dcn315/dcn315_resource.c   |    41 +-
+ .../display/dc/resource/dcn316/dcn316_resource.c   |    33 +-
+ .../amd/display/dc/resource/dcn32/dcn32_resource.c |    45 +-
+ .../dc/resource/dcn32/dcn32_resource_helpers.c     |     8 +-
+ .../display/dc/resource/dcn321/dcn321_resource.c   |    26 +-
+ .../amd/display/dc/resource/dcn35/dcn35_resource.c |    30 +-
+ .../display/dc/resource/dcn351/dcn351_resource.c   |    30 +-
+ .../amd/display/dc/resource/dcn36/dcn36_resource.c |    30 +-
+ .../display/dc/resource/dcn401/dcn401_resource.c   |    36 +-
+ .../amd/display/dc/resource/dcn42/dcn42_resource.c |   106 +-
+ .../amd/display/dc/resource/dcn42/dcn42_resource.h |     7 +
+ .../display/dc/resource/dcn42/dcn42_resource_fpu.c |    22 +
+ .../display/dc/resource/dcn42/dcn42_resource_fpu.h |     2 +-
+ .../dcn401/dcn401_soc_and_ip_translator.c          |    42 +-
+ .../dcn42/dcn42_soc_and_ip_translator.c            |    20 +-
+ drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h    |   140 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c |     2 +-
+ drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c  |     2 +-
+ drivers/gpu/drm/amd/display/include/fixed31_32.h   |     6 +-
+ .../drm/amd/display/modules/color/color_gamma.c    |    40 +-
+ .../gpu/drm/amd/display/modules/hdcp/hdcp_log.c    |     2 +-
+ .../drm/amd/display/modules/inc/mod_color_types.h  |    47 +
+ .../drm/amd/display/modules/inc/mod_info_packet.h  |     9 +-
+ .../display/modules/inc/mod_info_packet_types.h    |    37 +
+ .../gpu/drm/amd/display/modules/inc/mod_power.h    |   417 +
+ .../amd/display/modules/info_packet/info_packet.c  |     2 +-
+ drivers/gpu/drm/amd/display/modules/power/Makefile |     4 +-
+ drivers/gpu/drm/amd/display/modules/power/power.c  |   498 +
+ .../gpu/drm/amd/display/modules/power/power_abm.c  |  2062 +
+ .../drm/amd/display/modules/power/power_helpers.c  |  1065 -
+ .../drm/amd/display/modules/power/power_helpers.h  |   122 +
+ .../gpu/drm/amd/display/modules/power/power_psr.c  |   665 +
+ .../drm/amd/display/modules/power/power_replay.c   |   911 +
+ drivers/gpu/drm/amd/display/modules/vmid/vmid.c    |     6 +-
+ drivers/gpu/drm/amd/include/amd_cper.h             |     2 +-
+ drivers/gpu/drm/amd/include/amdgpu_ptl.h           |    64 +
+ .../amd/include/asic_reg/dcn/dcn_3_0_0_offset.h    |   129 +
+ .../amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h   |   600 +
+ .../amd/include/asic_reg/dcn/dcn_3_0_2_offset.h    |   112 +
+ .../amd/include/asic_reg/dcn/dcn_3_0_2_sh_mask.h   |  1907 +
+ .../amd/include/asic_reg/dcn/dcn_3_0_3_offset.h    |   111 +
+ .../amd/include/asic_reg/dcn/dcn_3_0_3_sh_mask.h   |   941 +
+ .../amd/include/asic_reg/dcn/dcn_3_1_2_offset.h    |   109 +
+ .../amd/include/asic_reg/dcn/dcn_3_1_2_sh_mask.h   |   548 +
+ .../amd/include/asic_reg/dcn/dcn_3_1_4_offset.h    |   108 +
+ .../amd/include/asic_reg/dcn/dcn_3_1_4_sh_mask.h   |   544 +
+ .../amd/include/asic_reg/dcn/dcn_3_1_5_offset.h    |   111 +
+ .../amd/include/asic_reg/dcn/dcn_3_1_5_sh_mask.h   |   548 +
+ .../amd/include/asic_reg/dcn/dcn_3_1_6_offset.h    |   111 +
+ .../amd/include/asic_reg/dcn/dcn_3_1_6_sh_mask.h   |   546 +
+ .../amd/include/asic_reg/dcn/dcn_3_2_0_offset.h    |   109 +
+ .../amd/include/asic_reg/dcn/dcn_3_2_0_sh_mask.h   |   544 +
+ .../amd/include/asic_reg/dcn/dcn_3_2_1_offset.h    |   124 +-
+ .../amd/include/asic_reg/dcn/dcn_3_2_1_sh_mask.h   |   548 +
+ .../drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h |   110 +
+ .../amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h    |   455 +
+ .../drm/amd/include/asic_reg/gc/gc_12_1_1_offset.h |   149 +
+ .../amd/include/asic_reg/gc/gc_12_1_1_sh_mask.h    |   377 +
+ .../amd/include/asic_reg/nbio/nbio_6_3_2_offset.h  | 13149 +++++
+ .../amd/include/asic_reg/nbio/nbio_6_3_2_sh_mask.h | 49475 +++++++++++++++++++
+ .../amd/include/asic_reg/vpe/vpe_2_0_0_offset.h    |  1041 +
+ .../amd/include/asic_reg/vpe/vpe_2_0_0_sh_mask.h   |  3162 ++
+ drivers/gpu/drm/amd/include/kgd_kfd_interface.h    |     6 +
+ drivers/gpu/drm/amd/pm/amdgpu_pm.c                 |    25 +-
+ .../gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c   |     8 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c     |    45 +-
+ .../gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c   |    38 +-
+ .../gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c   |   229 +-
+ .../gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h   |    34 +
+ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c             |    96 +-
+ drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h             |    22 +
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c   |     8 +-
+ drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c   |    26 +-
+ .../gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c  |    55 +-
+ drivers/gpu/drm/amd/ras/rascore/ras.h              |     1 -
+ drivers/gpu/drm/amd/ras/rascore/ras_cmd.c          |   108 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_cmd.h          |    12 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_core.c         |     6 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_cper.c         |    20 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_cper.h         |     2 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c       |     7 +
+ drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c    |     8 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_log_ring.c     |    23 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_log_ring.h     |     2 +-
+ drivers/gpu/drm/amd/ras/rascore/ras_umc.c          |     5 +-
+ drivers/gpu/drm/arm/display/komeda/komeda_crtc.c   |     8 +-
+ drivers/gpu/drm/arm/display/komeda/komeda_kms.c    |    10 +-
+ .../gpu/drm/arm/display/komeda/komeda_pipeline.h   |    14 +-
+ .../drm/arm/display/komeda/komeda_pipeline_state.c |    28 +-
+ drivers/gpu/drm/arm/display/komeda/komeda_plane.c  |     4 +-
+ drivers/gpu/drm/arm/hdlcd_crtc.c                   |    10 +-
+ drivers/gpu/drm/arm/malidp_crtc.c                  |     6 +-
+ drivers/gpu/drm/arm/malidp_drv.c                   |     4 +-
+ drivers/gpu/drm/arm/malidp_mw.c                    |     2 +-
+ drivers/gpu/drm/arm/malidp_mw.h                    |     2 +-
+ drivers/gpu/drm/arm/malidp_planes.c                |     6 +-
+ drivers/gpu/drm/armada/armada_crtc.c               |    10 +-
+ drivers/gpu/drm/armada/armada_overlay.c            |    10 +-
+ drivers/gpu/drm/armada/armada_plane.c              |     6 +-
+ drivers/gpu/drm/armada/armada_plane.h              |     2 +-
+ drivers/gpu/drm/ast/ast_2000.c                     |    61 +-
+ drivers/gpu/drm/ast/ast_2100.c                     |   292 +-
+ drivers/gpu/drm/ast/ast_2300.c                     |   392 +-
+ drivers/gpu/drm/ast/ast_2500.c                     |   327 +-
+ drivers/gpu/drm/ast/ast_cursor.c                   |     6 +-
+ drivers/gpu/drm/ast/ast_dp.c                       |     4 +-
+ drivers/gpu/drm/ast/ast_dp501.c                    |   134 +-
+ drivers/gpu/drm/ast/ast_drv.c                      |    84 +-
+ drivers/gpu/drm/ast/ast_drv.h                      |    60 +-
+ drivers/gpu/drm/ast/ast_mode.c                     |   104 +-
+ drivers/gpu/drm/ast/ast_post.c                     |    48 +-
+ drivers/gpu/drm/ast/ast_post.h                     |    20 +-
+ drivers/gpu/drm/ast/ast_reg.h                      |   211 +-
+ drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c     |    10 +-
+ drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c       |     2 +-
+ drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h       |     2 +-
+ drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c    |     6 +-
+ drivers/gpu/drm/bridge/Kconfig                     |    10 +
+ drivers/gpu/drm/bridge/Makefile                    |     1 +
+ drivers/gpu/drm/bridge/adv7511/adv7511_drv.c       |     6 +-
+ drivers/gpu/drm/bridge/analogix/Kconfig            |     3 +
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.c |   247 +-
+ drivers/gpu/drm/bridge/analogix/analogix_dp_core.h |     1 -
+ drivers/gpu/drm/bridge/analogix/anx7625.c          |     4 +-
+ drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c     |    15 +-
+ .../gpu/drm/bridge/cadence/cdns-mhdp8546-core.c    |     6 +-
+ drivers/gpu/drm/bridge/chipone-icn6211.c           |     6 +-
+ drivers/gpu/drm/bridge/fsl-ldb.c                   |     4 +-
+ drivers/gpu/drm/bridge/imx/Kconfig                 |    28 +-
+ drivers/gpu/drm/bridge/imx/Makefile                |     3 +-
+ drivers/gpu/drm/bridge/imx/imx-legacy-bridge.c     |    91 -
+ drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c       |     4 +-
+ .../bridge/imx/imx8mp-hdmi-tx-connector-fixup.c    |    75 +
+ .../bridge/imx/imx8mp-hdmi-tx-connector-fixup.dtso |    30 +
+ drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx.c        |     1 +
+ drivers/gpu/drm/bridge/imx/imx8qm-ldb.c            |     4 +-
+ drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c           |     6 +-
+ .../gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c    |     2 +-
+ drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c    |     4 +-
+ drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c       |     2 +-
+ drivers/gpu/drm/bridge/inno-hdmi.c                 |     6 +-
+ drivers/gpu/drm/bridge/ite-it6263.c                |    33 +-
+ drivers/gpu/drm/bridge/ite-it6505.c                |    10 +-
+ drivers/gpu/drm/bridge/ite-it66121.c               |     4 +-
+ drivers/gpu/drm/bridge/lontium-lt8912b.c           |     1 +
+ drivers/gpu/drm/bridge/lontium-lt9211.c            |     4 +-
+ drivers/gpu/drm/bridge/lontium-lt9611.c            |     8 +-
+ drivers/gpu/drm/bridge/microchip-lvds.c            |    63 +-
+ drivers/gpu/drm/bridge/nwl-dsi.c                   |     4 +-
+ drivers/gpu/drm/bridge/of-display-mode-bridge.c    |    93 +
+ drivers/gpu/drm/bridge/panel.c                     |     8 +-
+ drivers/gpu/drm/bridge/parade-ps8640.c             |     4 +-
+ drivers/gpu/drm/bridge/samsung-dsim.c              |     8 +-
+ drivers/gpu/drm/bridge/sii902x.c                   |     4 +-
+ drivers/gpu/drm/bridge/simple-bridge.c             |     5 +
+ drivers/gpu/drm/bridge/ssd2825.c                   |     6 +-
+ drivers/gpu/drm/bridge/synopsys/dw-dp.c            |     8 +-
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c       |     4 +-
+ drivers/gpu/drm/bridge/synopsys/dw-hdmi.c          |    55 +-
+ drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c      |     6 +-
+ drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c     |     6 +-
+ drivers/gpu/drm/bridge/tc358762.c                  |     6 +-
+ drivers/gpu/drm/bridge/tc358767.c                  |     8 +-
+ drivers/gpu/drm/bridge/tc358768.c                  |   210 +-
+ drivers/gpu/drm/bridge/tc358775.c                  |     6 +-
+ drivers/gpu/drm/bridge/ti-dlpc3433.c               |     6 +-
+ drivers/gpu/drm/bridge/ti-sn65dsi83.c              |    20 +-
+ drivers/gpu/drm/bridge/ti-sn65dsi86.c              |    24 +-
+ drivers/gpu/drm/bridge/ti-tdp158.c                 |     4 +-
+ drivers/gpu/drm/bridge/ti-tfp410.c                 |     2 +-
+ drivers/gpu/drm/bridge/waveshare-dsi.c             |     5 +-
+ drivers/gpu/drm/ci/xfails/vkms-none-skips.txt      |    20 +-
+ drivers/gpu/drm/display/drm_bridge_connector.c     |    13 +-
+ drivers/gpu/drm/display/drm_dp_helper.c            |    71 +-
+ drivers/gpu/drm/display/drm_dp_mst_topology.c      |    35 +-
+ drivers/gpu/drm/display/drm_dp_tunnel.c            |    31 +-
+ drivers/gpu/drm/display/drm_hdmi_state_helper.c    |     6 +-
+ drivers/gpu/drm/drm_atomic.c                       |   238 +-
+ drivers/gpu/drm/drm_atomic_helper.c                |   172 +-
+ drivers/gpu/drm/drm_atomic_state_helper.c          |     2 +-
+ drivers/gpu/drm/drm_atomic_uapi.c                  |    28 +-
+ drivers/gpu/drm/drm_auth.c                         |     2 +-
+ drivers/gpu/drm/drm_blend.c                        |     8 +-
+ drivers/gpu/drm/drm_bridge.c                       |   110 +-
+ drivers/gpu/drm/drm_buddy.c                        |     1 +
+ drivers/gpu/drm/drm_client_modeset.c               |     8 +-
+ drivers/gpu/drm/drm_color_mgmt.c                   |     6 +-
+ drivers/gpu/drm/drm_colorop.c                      |     8 +-
+ drivers/gpu/drm/drm_connector.c                    |     9 +-
+ drivers/gpu/drm/drm_crtc_helper.c                  |     2 +-
+ drivers/gpu/drm/drm_crtc_internal.h                |    10 +-
+ drivers/gpu/drm/drm_damage_helper.c                |    10 +-
+ drivers/gpu/drm/drm_encoder.c                      |    18 +-
+ drivers/gpu/drm/drm_fb_helper.c                    |     8 +-
+ drivers/gpu/drm/drm_fourcc.c                       |    25 +-
+ drivers/gpu/drm/drm_framebuffer.c                  |     8 +-
+ drivers/gpu/drm/drm_gem.c                          |    10 +
+ drivers/gpu/drm/drm_gpusvm.c                       |    55 +-
+ drivers/gpu/drm/drm_gpuvm.c                        |     3 +
+ drivers/gpu/drm/drm_mipi_dbi.c                     |     8 +-
+ drivers/gpu/drm/drm_mipi_dsi.c                     |    28 +
+ drivers/gpu/drm/drm_mode_object.c                  |     8 +-
+ drivers/gpu/drm/drm_modeset_helper.c               |     2 +-
+ drivers/gpu/drm/drm_of.c                           |    34 +
+ drivers/gpu/drm/drm_pagemap.c                      |   226 +-
+ drivers/gpu/drm/drm_panel.c                        |    30 +-
+ drivers/gpu/drm/drm_ras.c                          |    43 +-
+ drivers/gpu/drm/drm_ras_nl.c                       |    13 +
+ drivers/gpu/drm/drm_ras_nl.h                       |     2 +
+ drivers/gpu/drm/drm_self_refresh_helper.c          |    12 +-
+ drivers/gpu/drm/drm_simple_kms_helper.c            |    10 +-
+ drivers/gpu/drm/drm_vblank.c                       |    36 +-
+ drivers/gpu/drm/drm_vblank_helper.c                |     6 +-
+ drivers/gpu/drm/etnaviv/etnaviv_drv.h              |    12 -
+ drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c       |     3 +-
+ drivers/gpu/drm/etnaviv/etnaviv_sched.c            |     1 -
+ drivers/gpu/drm/exynos/Kconfig                     |     3 +
+ drivers/gpu/drm/exynos/exynos_dp.c                 |   110 +-
+ drivers/gpu/drm/exynos/exynos_drm_crtc.c           |    10 +-
+ drivers/gpu/drm/exynos/exynos_drm_plane.c          |     6 +-
+ drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c         |     6 +-
+ drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c        |     6 +-
+ drivers/gpu/drm/gma500/psb_drv.c                   |     1 -
+ drivers/gpu/drm/gud/gud_connector.c                |    10 +-
+ drivers/gpu/drm/gud/gud_drv.c                      |     1 +
+ drivers/gpu/drm/gud/gud_internal.h                 |     8 +-
+ drivers/gpu/drm/gud/gud_pipe.c                     |     8 +-
+ drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h       |     1 +
+ drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h         |     1 +
+ drivers/gpu/drm/hisilicon/hibmc/dp/dp_link.c       |     2 +-
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c     |    90 +-
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c     |    39 +-
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c    |    14 +
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h    |     1 +
+ drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c   |    75 +-
+ drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c    |    14 +-
+ drivers/gpu/drm/hyperv/hyperv_drm_modeset.c        |     6 +-
+ drivers/gpu/drm/hyperv/hyperv_drm_proto.c          |     2 +-
+ drivers/gpu/drm/i915/Kconfig.debug                 |     2 +-
+ drivers/gpu/drm/i915/Makefile                      |     7 +-
+ drivers/gpu/drm/i915/display/g4x_dp.c              |     4 +-
+ drivers/gpu/drm/i915/display/g4x_dp.h              |    10 +-
+ drivers/gpu/drm/i915/display/g4x_hdmi.c            |     4 +-
+ drivers/gpu/drm/i915/display/g4x_hdmi.h            |    12 +-
+ drivers/gpu/drm/i915/display/i9xx_display_sr.c     |     2 +-
+ drivers/gpu/drm/i915/display/i9xx_plane.c          |     2 +-
+ drivers/gpu/drm/i915/display/i9xx_wm.c             |    75 +-
+ drivers/gpu/drm/i915/display/icl_dsi.c             |     2 +-
+ drivers/gpu/drm/i915/display/intel_atomic.c        |    16 +-
+ drivers/gpu/drm/i915/display/intel_atomic.h        |    12 +-
+ drivers/gpu/drm/i915/display/intel_audio.c         |    32 +-
+ drivers/gpu/drm/i915/display/intel_backlight.c     |     2 +-
+ drivers/gpu/drm/i915/display/intel_bo.c            |    27 +
+ drivers/gpu/drm/i915/display/intel_bo.h            |     8 +
+ drivers/gpu/drm/i915/display/intel_bw.c            |    58 +-
+ drivers/gpu/drm/i915/display/intel_casf.c          |   102 +-
+ drivers/gpu/drm/i915/display/intel_casf.h          |     6 +-
+ drivers/gpu/drm/i915/display/intel_cdclk.c         |   142 +-
+ drivers/gpu/drm/i915/display/intel_color.c         |    62 +-
+ drivers/gpu/drm/i915/display/intel_combo_phy.c     |     2 +-
+ drivers/gpu/drm/i915/display/intel_crt.c           |     8 +-
+ drivers/gpu/drm/i915/display/intel_crt.h           |     6 +-
+ .../gpu/drm/i915/display/intel_crtc_state_dump.c   |    26 +-
+ drivers/gpu/drm/i915/display/intel_cursor.c        |     1 -
+ drivers/gpu/drm/i915/display/intel_cx0_phy.c       |     4 +-
+ drivers/gpu/drm/i915/display/intel_ddi.c           |    69 +-
+ drivers/gpu/drm/i915/display/intel_ddi.h           |     6 +-
+ drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c |    11 +-
+ drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h |     3 +
+ drivers/gpu/drm/i915/display/intel_de.c            |    36 +-
+ drivers/gpu/drm/i915/display/intel_de.h            |    57 +-
+ drivers/gpu/drm/i915/display/intel_display.c       |   108 +-
+ drivers/gpu/drm/i915/display/intel_display.h       |    16 +-
+ .../drm/i915/display/intel_display_clock_gating.c  |   270 +
+ .../drm/i915/display/intel_display_clock_gating.h  |    27 +
+ drivers/gpu/drm/i915/display/intel_display_core.h  |    64 +-
+ .../gpu/drm/i915/display/intel_display_debugfs.c   |     7 +-
+ .../gpu/drm/i915/display/intel_display_device.c    |     4 +-
+ .../gpu/drm/i915/display/intel_display_device.h    |     1 +
+ .../gpu/drm/i915/display/intel_display_driver.c    |     8 +-
+ .../gpu/drm/i915/display/intel_display_driver.h    |     4 +-
+ drivers/gpu/drm/i915/display/intel_display_irq.c   |    24 +-
+ drivers/gpu/drm/i915/display/intel_display_power.c |    18 +-
+ .../gpu/drm/i915/display/intel_display_power_map.c |     2 +-
+ .../drm/i915/display/intel_display_power_well.c    |    73 +-
+ .../gpu/drm/i915/display/intel_display_reg_defs.h  |    35 +
+ drivers/gpu/drm/i915/display/intel_display_regs.h  |    82 +-
+ drivers/gpu/drm/i915/display/intel_display_reset.c |    50 +-
+ drivers/gpu/drm/i915/display/intel_display_reset.h |     8 +-
+ drivers/gpu/drm/i915/display/intel_display_types.h |    28 +-
+ drivers/gpu/drm/i915/display/intel_display_wa.c    |     2 +-
+ drivers/gpu/drm/i915/display/intel_dmc.c           |    62 +-
+ drivers/gpu/drm/i915/display/intel_dmc_wl.c        |    16 +-
+ drivers/gpu/drm/i915/display/intel_dmc_wl.h        |     6 +-
+ drivers/gpu/drm/i915/display/intel_dp.c            |   312 +-
+ drivers/gpu/drm/i915/display/intel_dp.h            |     3 +
+ drivers/gpu/drm/i915/display/intel_dp_aux.c        |    28 +-
+ .../gpu/drm/i915/display/intel_dp_link_training.c  |   124 +-
+ .../gpu/drm/i915/display/intel_dp_link_training.h  |     2 +-
+ drivers/gpu/drm/i915/display/intel_dp_mst.c        |    15 +-
+ drivers/gpu/drm/i915/display/intel_dp_tunnel.c     |    18 +
+ drivers/gpu/drm/i915/display/intel_dp_tunnel.h     |     6 +
+ drivers/gpu/drm/i915/display/intel_dpio_phy.c      |   184 +-
+ drivers/gpu/drm/i915/display/intel_dpll.c          |   144 +-
+ drivers/gpu/drm/i915/display/intel_dpll_mgr.c      |    36 +-
+ drivers/gpu/drm/i915/display/intel_dram.c          |    48 +-
+ drivers/gpu/drm/i915/display/intel_dsb.c           |    22 +-
+ drivers/gpu/drm/i915/display/intel_dsb.h           |    10 +-
+ drivers/gpu/drm/i915/display/intel_dsi.c           |     7 +-
+ drivers/gpu/drm/i915/display/intel_dvo.c           |    29 +-
+ drivers/gpu/drm/i915/display/intel_dvo_dev.h       |     2 +-
+ drivers/gpu/drm/i915/display/intel_fb.c            |    49 +-
+ drivers/gpu/drm/i915/display/intel_fb.h            |     3 +-
+ drivers/gpu/drm/i915/display/intel_fb_pin.c        |   357 -
+ drivers/gpu/drm/i915/display/intel_fb_pin.h        |    33 -
+ drivers/gpu/drm/i915/display/intel_fbc.c           |    13 +-
+ drivers/gpu/drm/i915/display/intel_fbdev.c         |    59 +-
+ drivers/gpu/drm/i915/display/intel_fbdev.h         |    11 +-
+ drivers/gpu/drm/i915/display/intel_fbdev_fb.c      |   118 -
+ drivers/gpu/drm/i915/display/intel_fbdev_fb.h      |    24 -
+ drivers/gpu/drm/i915/display/intel_fdi.c           |    25 +-
+ drivers/gpu/drm/i915/display/intel_fifo_underrun.c |     4 +-
+ drivers/gpu/drm/i915/display/intel_flipq.c         |     2 +-
+ drivers/gpu/drm/i915/display/intel_gmbus.c         |     4 +-
+ drivers/gpu/drm/i915/display/intel_hdcp.c          |    27 +-
+ drivers/gpu/drm/i915/display/intel_hdmi.c          |   195 +-
+ drivers/gpu/drm/i915/display/intel_hotplug_irq.c   |    30 +-
+ drivers/gpu/drm/i915/display/intel_initial_plane.c |    83 +-
+ drivers/gpu/drm/i915/display/intel_load_detect.c   |    20 +-
+ drivers/gpu/drm/i915/display/intel_load_detect.h   |     6 +-
+ drivers/gpu/drm/i915/display/intel_lpe_audio.c     |     2 +-
+ drivers/gpu/drm/i915/display/intel_lt_phy.c        |     8 +-
+ drivers/gpu/drm/i915/display/intel_lvds.c          |    13 +-
+ drivers/gpu/drm/i915/display/intel_lvds.h          |     6 +-
+ drivers/gpu/drm/i915/display/intel_mchbar.c        |    78 +
+ drivers/gpu/drm/i915/display/intel_mchbar.h        |    21 +
+ drivers/gpu/drm/i915/display/intel_modeset_lock.c  |     2 +-
+ drivers/gpu/drm/i915/display/intel_modeset_setup.c |     9 +-
+ drivers/gpu/drm/i915/display/intel_opregion.c      |     2 +-
+ drivers/gpu/drm/i915/display/intel_overlay.c       |    10 +-
+ drivers/gpu/drm/i915/display/intel_panel.c         |     9 +-
+ drivers/gpu/drm/i915/display/intel_panel.h         |     3 +-
+ drivers/gpu/drm/i915/display/intel_parent.c        |    95 +-
+ drivers/gpu/drm/i915/display/intel_parent.h        |    42 +-
+ drivers/gpu/drm/i915/display/intel_pch_display.c   |    14 +-
+ drivers/gpu/drm/i915/display/intel_pfit.c          |    24 +-
+ drivers/gpu/drm/i915/display/intel_pipe_crc.c      |     8 +-
+ drivers/gpu/drm/i915/display/intel_plane.c         |   149 +-
+ drivers/gpu/drm/i915/display/intel_plane.h         |     6 +
+ drivers/gpu/drm/i915/display/intel_pmdemand.c      |     2 +-
+ drivers/gpu/drm/i915/display/intel_pps.c           |    38 +-
+ drivers/gpu/drm/i915/display/intel_psr.c           |    74 +-
+ drivers/gpu/drm/i915/display/intel_quirks.c        |    17 +
+ drivers/gpu/drm/i915/display/intel_quirks.h        |     1 +
+ drivers/gpu/drm/i915/display/intel_rom.c           |    19 +-
+ drivers/gpu/drm/i915/display/intel_sdvo.c          |    25 +-
+ drivers/gpu/drm/i915/display/intel_sdvo.h          |    10 +-
+ drivers/gpu/drm/i915/display/intel_snps_phy.c      |     6 +-
+ drivers/gpu/drm/i915/display/intel_sprite_uapi.c   |     8 +-
+ drivers/gpu/drm/i915/display/intel_tc.c            |    16 +-
+ drivers/gpu/drm/i915/display/intel_tv.c            |     4 +-
+ drivers/gpu/drm/i915/display/intel_vblank.c        |     6 +-
+ drivers/gpu/drm/i915/display/intel_vdsc.c          |    10 +-
+ drivers/gpu/drm/i915/display/intel_vga.c           |     4 +-
+ drivers/gpu/drm/i915/display/intel_vrr.c           |    16 +-
+ drivers/gpu/drm/i915/display/intel_wm.c            |    32 +-
+ drivers/gpu/drm/i915/display/skl_scaler.c          |   110 +-
+ drivers/gpu/drm/i915/display/skl_scaler.h          |     2 -
+ drivers/gpu/drm/i915/display/skl_universal_plane.c |    37 +-
+ drivers/gpu/drm/i915/display/skl_watermark.c       |   296 +-
+ drivers/gpu/drm/i915/display/vlv_clock.c           |    13 +-
+ drivers/gpu/drm/i915/display/vlv_dsi.c             |    36 +-
+ drivers/gpu/drm/i915/display/vlv_dsi_pll.c         |    38 +-
+ drivers/gpu/drm/i915/display/vlv_sideband.c        |   136 +-
+ drivers/gpu/drm/i915/display/vlv_sideband.h        |   166 +-
+ drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c     |     3 +-
+ drivers/gpu/drm/i915/gem/i915_gem_stolen.c         |     4 +-
+ drivers/gpu/drm/i915/gt/intel_ggtt.c               |     2 +-
+ drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c       |     2 +-
+ drivers/gpu/drm/i915/gt/intel_gt.c                 |     2 +-
+ drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c      |     1 -
+ drivers/gpu/drm/i915/gt/intel_llc.c                |     2 +-
+ drivers/gpu/drm/i915/gt/intel_region_lmem.c        |     2 +-
+ drivers/gpu/drm/i915/gt/intel_reset.c              |    40 +-
+ drivers/gpu/drm/i915/gt/intel_rps.c                |     4 +-
+ drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c        |     1 -
+ drivers/gpu/drm/i915/gvt/cfg_space.c               |     2 +-
+ drivers/gpu/drm/i915/gvt/handlers.c                |     1 -
+ drivers/gpu/drm/i915/i915_bo.c                     |   114 +
+ drivers/gpu/drm/i915/i915_bo.h                     |     6 +
+ drivers/gpu/drm/i915/i915_debugfs.c                |     2 +-
+ drivers/gpu/drm/i915/i915_dpt.c                    |     5 +-
+ drivers/gpu/drm/i915/i915_driver.c                 |     6 +-
+ drivers/gpu/drm/i915/i915_drv.h                    |     2 +
+ drivers/gpu/drm/i915/i915_fb_pin.c                 |   311 +
+ drivers/gpu/drm/i915/i915_fb_pin.h                 |     9 +
+ drivers/gpu/drm/i915/i915_freq.c                   |     2 +-
+ drivers/gpu/drm/i915/i915_gmch.c                   |     2 +-
+ drivers/gpu/drm/i915/i915_gtt_view_types.h         |    15 +
+ drivers/gpu/drm/i915/i915_hwmon.c                  |     2 +-
+ drivers/gpu/drm/i915/i915_initial_plane.c          |    33 +-
+ drivers/gpu/drm/i915/i915_irq.h                    |     7 -
+ drivers/gpu/drm/i915/i915_overlay.c                |     8 +-
+ drivers/gpu/drm/i915/i915_pci.c                    |     2 +-
+ drivers/gpu/drm/i915/i915_reg.h                    |    31 -
+ drivers/gpu/drm/i915/i915_vma.c                    |    10 -
+ drivers/gpu/drm/i915/i915_vma.h                    |     2 -
+ drivers/gpu/drm/i915/intel_clock_gating.c          |   230 +-
+ drivers/gpu/drm/i915/intel_gvt_mmio_table.c        |     2 +-
+ drivers/gpu/drm/i915/selftests/mock_gem_device.c   |     1 +
+ drivers/gpu/drm/i915/vlv_iosf_sb.c                 |     8 +
+ drivers/gpu/drm/i915/vlv_iosf_sb.h                 |    16 +-
+ drivers/gpu/drm/imagination/pvr_fw_trace.c         |     2 +-
+ drivers/gpu/drm/imagination/pvr_job.c              |     8 +-
+ drivers/gpu/drm/imagination/pvr_queue.c            |   155 +-
+ drivers/gpu/drm/imagination/pvr_queue.h            |     2 +-
+ .../gpu/drm/imagination/pvr_rogue_fwif_shared.h    |    10 +-
+ drivers/gpu/drm/imagination/pvr_sync.c             |     8 +-
+ drivers/gpu/drm/imagination/pvr_sync.h             |     2 +-
+ drivers/gpu/drm/imx/dc/dc-crtc.c                   |    10 +-
+ drivers/gpu/drm/imx/dc/dc-kms.c                    |     8 +-
+ drivers/gpu/drm/imx/dc/dc-plane.c                  |     6 +-
+ drivers/gpu/drm/imx/dcss/dcss-crtc.c               |     8 +-
+ drivers/gpu/drm/imx/dcss/dcss-kms.c                |     2 -
+ drivers/gpu/drm/imx/dcss/dcss-plane.c              |     6 +-
+ drivers/gpu/drm/imx/ipuv3/Kconfig                  |     6 +-
+ drivers/gpu/drm/imx/ipuv3/imx-drm-core.c           |     4 +-
+ drivers/gpu/drm/imx/ipuv3/imx-drm.h                |     2 +-
+ drivers/gpu/drm/imx/ipuv3/imx-ldb.c                |     8 +-
+ drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c             |    10 +-
+ drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c            |     8 +-
+ drivers/gpu/drm/imx/ipuv3/parallel-display.c       |     7 +-
+ drivers/gpu/drm/imx/lcdc/imx-lcdc.c                |     2 -
+ drivers/gpu/drm/ingenic/ingenic-drm-drv.c          |    26 +-
+ drivers/gpu/drm/ingenic/ingenic-ipu.c              |    10 +-
+ drivers/gpu/drm/kmb/kmb_crtc.c                     |    10 +-
+ drivers/gpu/drm/kmb/kmb_drv.h                      |     2 +-
+ drivers/gpu/drm/kmb/kmb_dsi.c                      |     8 +-
+ drivers/gpu/drm/kmb/kmb_dsi.h                      |     2 +-
+ drivers/gpu/drm/kmb/kmb_plane.c                    |     6 +-
+ drivers/gpu/drm/lima/lima_sched.c                  |     1 -
+ drivers/gpu/drm/logicvc/logicvc_crtc.c             |     6 +-
+ drivers/gpu/drm/logicvc/logicvc_layer.c            |     6 +-
+ drivers/gpu/drm/loongson/lsdc_crtc.c               |     8 +-
+ drivers/gpu/drm/loongson/lsdc_output_7a1000.c      |     2 +-
+ drivers/gpu/drm/loongson/lsdc_output_7a2000.c      |     6 +-
+ drivers/gpu/drm/loongson/lsdc_plane.c              |    22 +-
+ drivers/gpu/drm/mediatek/mtk_crtc.c                |    12 +-
+ drivers/gpu/drm/mediatek/mtk_crtc.h                |     2 +-
+ drivers/gpu/drm/mediatek/mtk_dp.c                  |     4 +-
+ drivers/gpu/drm/mediatek/mtk_dpi.c                 |     1 -
+ drivers/gpu/drm/mediatek/mtk_drm_drv.h             |     2 +-
+ drivers/gpu/drm/mediatek/mtk_dsi.c                 |     9 +-
+ drivers/gpu/drm/mediatek/mtk_hdmi.c                |     8 +-
+ drivers/gpu/drm/mediatek/mtk_hdmi_v2.c             |     8 +-
+ drivers/gpu/drm/mediatek/mtk_plane.c               |    10 +-
+ drivers/gpu/drm/meson/meson_crtc.c                 |    12 +-
+ drivers/gpu/drm/meson/meson_encoder_cvbs.c         |     6 +-
+ drivers/gpu/drm/meson/meson_encoder_dsi.c          |     4 +-
+ drivers/gpu/drm/meson/meson_encoder_hdmi.c         |     6 +-
+ drivers/gpu/drm/meson/meson_overlay.c              |     6 +-
+ drivers/gpu/drm/meson/meson_plane.c                |     6 +-
+ drivers/gpu/drm/mgag200/mgag200_drv.c              |     1 -
+ drivers/gpu/drm/mgag200/mgag200_drv.h              |    26 +-
+ drivers/gpu/drm/mgag200/mgag200_g200.c             |     4 +-
+ drivers/gpu/drm/mgag200/mgag200_g200eh.c           |     4 +-
+ drivers/gpu/drm/mgag200/mgag200_g200eh3.c          |     2 +-
+ drivers/gpu/drm/mgag200/mgag200_g200eh5.c          |     2 +-
+ drivers/gpu/drm/mgag200/mgag200_g200er.c           |     6 +-
+ drivers/gpu/drm/mgag200/mgag200_g200ev.c           |     6 +-
+ drivers/gpu/drm/mgag200/mgag200_g200ew3.c          |     2 +-
+ drivers/gpu/drm/mgag200/mgag200_g200se.c           |    23 +-
+ drivers/gpu/drm/mgag200/mgag200_g200wb.c           |     4 +-
+ drivers/gpu/drm/mgag200/mgag200_mode.c             |   127 +-
+ drivers/gpu/drm/mgag200/mgag200_reg.h              |     1 +
+ drivers/gpu/drm/mgag200/mgag200_vga_bmc.c          |     4 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c           |    12 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c        |     8 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h        |     4 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c            |     4 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h            |     2 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c          |    12 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h          |     2 +-
+ drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c      |     2 +-
+ drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c          |    10 +-
+ drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c           |     7 -
+ drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c         |     4 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c          |    10 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c           |     4 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h           |     4 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c         |     4 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.h         |     4 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c          |     4 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.h          |     4 +-
+ drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c         |     8 +-
+ drivers/gpu/drm/msm/disp/msm_disp_snapshot.h       |     2 +-
+ drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c  |     2 +-
+ drivers/gpu/drm/msm/dp/dp_display.c                |     6 +-
+ drivers/gpu/drm/msm/dp/dp_drm.c                    |     8 +-
+ drivers/gpu/drm/msm/dp/dp_drm.h                    |     6 +-
+ drivers/gpu/drm/msm/dsi/dsi_manager.c              |     4 -
+ drivers/gpu/drm/msm/hdmi/hdmi.c                    |     2 -
+ drivers/gpu/drm/msm/hdmi/hdmi_bridge.c             |     4 +-
+ drivers/gpu/drm/msm/msm_atomic.c                   |     8 +-
+ drivers/gpu/drm/msm/msm_drv.h                      |     6 +-
+ drivers/gpu/drm/msm/msm_gem_vma.c                  |     1 -
+ drivers/gpu/drm/msm/msm_kms.h                      |     4 +-
+ drivers/gpu/drm/msm/msm_ringbuffer.c               |     1 -
+ drivers/gpu/drm/mxsfb/Kconfig                      |     2 +
+ drivers/gpu/drm/mxsfb/lcdif_drv.c                  |    61 +-
+ drivers/gpu/drm/mxsfb/lcdif_kms.c                  |    12 +-
+ drivers/gpu/drm/mxsfb/mxsfb_kms.c                  |    16 +-
+ drivers/gpu/drm/nouveau/dispnv50/atom.h            |     6 +-
+ drivers/gpu/drm/nouveau/dispnv50/crc.c             |    17 +-
+ drivers/gpu/drm/nouveau/dispnv50/crc.h             |    16 +-
+ drivers/gpu/drm/nouveau/dispnv50/disp.c            |    70 +-
+ drivers/gpu/drm/nouveau/dispnv50/head.c            |     2 +-
+ drivers/gpu/drm/nouveau/dispnv50/wndw.c            |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_abi16.c            |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_bios.c             |    21 +-
+ drivers/gpu/drm/nouveau/nouveau_connector.c        |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_display.h          |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_drm.c              |     2 +-
+ drivers/gpu/drm/nouveau/nouveau_encoder.h          |     4 +-
+ drivers/gpu/drm/nouveau/nouveau_sched.c            |     4 +-
+ .../gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c |     3 +-
+ drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c      |     3 +-
+ drivers/gpu/drm/omapdrm/dss/hdmi4.c                |     4 +-
+ drivers/gpu/drm/omapdrm/dss/hdmi5.c                |     4 +-
+ drivers/gpu/drm/omapdrm/dss/output.c               |    11 +-
+ drivers/gpu/drm/omapdrm/omap_crtc.c                |    10 +-
+ drivers/gpu/drm/omapdrm/omap_drv.c                 |    16 +-
+ drivers/gpu/drm/omapdrm/omap_drv.h                 |     2 +-
+ drivers/gpu/drm/omapdrm/omap_overlay.c             |     4 +-
+ drivers/gpu/drm/omapdrm/omap_overlay.h             |     4 +-
+ drivers/gpu/drm/omapdrm/omap_plane.c               |     6 +-
+ drivers/gpu/drm/panel/Kconfig                      |    58 +-
+ drivers/gpu/drm/panel/Makefile                     |     3 +
+ drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c  |   333 +
+ drivers/gpu/drm/panel/panel-edp.c                  |    18 +-
+ drivers/gpu/drm/panel/panel-focaltech-ota7290b.c   |   225 +
+ drivers/gpu/drm/panel/panel-himax-hx83102.c        |   144 +-
+ drivers/gpu/drm/panel/panel-himax-hx8394.c         |   279 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c |    10 +-
+ drivers/gpu/drm/panel/panel-ilitek-ili9881c.c      |   251 +-
+ drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c   |  1423 +-
+ drivers/gpu/drm/panel/panel-lxd-m9189a.c           |    11 +-
+ drivers/gpu/drm/panel/panel-novatek-nt35532.c      |   796 +
+ drivers/gpu/drm/panel/panel-novatek-nt37700f.c     |    11 +-
+ .../gpu/drm/panel/panel-panasonic-vvx10f034n00.c   |    24 +-
+ drivers/gpu/drm/panel/panel-samsung-s6e63m0.c      |    14 +-
+ drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c    |    11 +-
+ drivers/gpu/drm/panel/panel-simple.c               |   482 +
+ .../gpu/drm/panel/panel-startek-kd070fhfid015.c    |    12 +-
+ drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c       |    14 +-
+ drivers/gpu/drm/panel/panel-truly-nt35597.c        |    10 +-
+ drivers/gpu/drm/panel/panel-visionox-g2647fb105.c  |    13 +-
+ drivers/gpu/drm/panfrost/panfrost_device.c         |    34 +-
+ drivers/gpu/drm/panfrost/panfrost_device.h         |     1 +
+ drivers/gpu/drm/panfrost/panfrost_drv.c            |     1 +
+ drivers/gpu/drm/panfrost/panfrost_job.c            |     1 -
+ drivers/gpu/drm/panthor/Kconfig                    |     1 -
+ drivers/gpu/drm/panthor/panthor_device.c           |    39 +-
+ drivers/gpu/drm/panthor/panthor_device.h           |   165 +-
+ drivers/gpu/drm/panthor/panthor_drv.c              |    41 +-
+ drivers/gpu/drm/panthor/panthor_drv.h              |     2 +
+ drivers/gpu/drm/panthor/panthor_fw.c               |    56 +-
+ drivers/gpu/drm/panthor/panthor_fw.h               |     1 +
+ drivers/gpu/drm/panthor/panthor_fw_regs.h          |    29 +
+ drivers/gpu/drm/panthor/panthor_gem.c              |  1453 +-
+ drivers/gpu/drm/panthor/panthor_gem.h              |   136 +-
+ drivers/gpu/drm/panthor/panthor_gpu.c              |   101 +-
+ drivers/gpu/drm/panthor/panthor_gpu.h              |     6 +
+ drivers/gpu/drm/panthor/panthor_gpu_regs.h         |   119 +
+ drivers/gpu/drm/panthor/panthor_heap.c             |     2 +-
+ drivers/gpu/drm/panthor/panthor_hw.c               |    56 +-
+ drivers/gpu/drm/panthor/panthor_hw.h               |     2 +-
+ drivers/gpu/drm/panthor/panthor_mmu.c              |   565 +-
+ drivers/gpu/drm/panthor/panthor_mmu.h              |     8 +
+ drivers/gpu/drm/panthor/panthor_mmu_regs.h         |    69 +
+ drivers/gpu/drm/panthor/panthor_pwr.c              |    95 +-
+ drivers/gpu/drm/panthor/panthor_pwr_regs.h         |    79 +
+ drivers/gpu/drm/panthor/panthor_regs.h             |   291 -
+ drivers/gpu/drm/panthor/panthor_sched.c            |    90 +-
+ drivers/gpu/drm/qxl/qxl_display.c                  |    16 +-
+ drivers/gpu/drm/radeon/cik.c                       |     2 +-
+ drivers/gpu/drm/radeon/radeon_gem.c                |    13 +-
+ drivers/gpu/drm/radeon/radeon_ring.c               |     4 +-
+ drivers/gpu/drm/radeon/si.c                        |     2 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c     |    18 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c  |    31 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.h  |     1 +
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_kms.c      |     6 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c    |     6 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.h    |     2 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c      |     4 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c        |     6 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c    |     6 +-
+ drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.h    |     6 +-
+ drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c      |     6 +-
+ drivers/gpu/drm/renesas/rz-du/rzg2l_du_encoder.c   |    15 +-
+ drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c       |    20 +-
+ drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.h       |     2 +
+ drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c     |    70 +-
+ drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c  |    22 +-
+ drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c |     6 +-
+ drivers/gpu/drm/rockchip/Kconfig                   |     1 +
+ drivers/gpu/drm/rockchip/analogix_dp-rockchip.c    |    73 +-
+ drivers/gpu/drm/rockchip/cdn-dp-core.c             |     6 +-
+ drivers/gpu/drm/rockchip/cdn-dp-reg.c              |     2 +
+ drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c   |     2 +-
+ drivers/gpu/drm/rockchip/dw_dp-rockchip.c          |    35 +-
+ drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c     |    15 +-
+ drivers/gpu/drm/rockchip/inno_hdmi-rockchip.c      |     5 +-
+ drivers/gpu/drm/rockchip/rk3066_hdmi.c             |     8 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_gem.c        |     2 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_vop.c        |    22 +-
+ drivers/gpu/drm/rockchip/rockchip_drm_vop2.c       |    18 +-
+ drivers/gpu/drm/rockchip/rockchip_lvds.c           |     6 -
+ drivers/gpu/drm/rockchip/rockchip_rgb.c            |     9 -
+ drivers/gpu/drm/scheduler/Makefile                 |     2 +-
+ drivers/gpu/drm/scheduler/sched_entity.c           |   175 +-
+ drivers/gpu/drm/scheduler/sched_fence.c            |     2 +-
+ drivers/gpu/drm/scheduler/sched_internal.h         |    70 +-
+ drivers/gpu/drm/scheduler/sched_main.c             |   351 +-
+ drivers/gpu/drm/scheduler/sched_rq.c               |   383 +
+ drivers/gpu/drm/scheduler/tests/Makefile           |     3 +-
+ drivers/gpu/drm/scheduler/tests/mock_scheduler.c   |     1 -
+ drivers/gpu/drm/scheduler/tests/tests_scheduler.c  |   882 +
+ drivers/gpu/drm/sitronix/st7571.c                  |    10 +-
+ drivers/gpu/drm/sitronix/st7586.c                  |     6 +-
+ drivers/gpu/drm/sitronix/st7735r.c                 |     2 +-
+ drivers/gpu/drm/sitronix/st7920.c                  |    12 +-
+ drivers/gpu/drm/solomon/ssd130x.c                  |    32 +-
+ drivers/gpu/drm/sprd/sprd_dpu.c                    |    12 +-
+ drivers/gpu/drm/sti/sti_crtc.c                     |     6 +-
+ drivers/gpu/drm/sti/sti_cursor.c                   |     6 +-
+ drivers/gpu/drm/sti/sti_gdp.c                      |     6 +-
+ drivers/gpu/drm/sti/sti_hqvdp.c                    |     6 +-
+ drivers/gpu/drm/stm/drv.c                          |     2 +-
+ drivers/gpu/drm/stm/ltdc.c                         |    12 +-
+ drivers/gpu/drm/stm/ltdc.h                         |     2 +-
+ drivers/gpu/drm/stm/lvds.c                         |     6 +-
+ drivers/gpu/drm/sun4i/sun4i_backend.c              |     4 +-
+ drivers/gpu/drm/sun4i/sun4i_crtc.c                 |    10 +-
+ drivers/gpu/drm/sun4i/sun4i_framebuffer.c          |     2 +-
+ drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c             |     4 +-
+ drivers/gpu/drm/sun4i/sun4i_layer.c                |     4 +-
+ drivers/gpu/drm/sun4i/sun4i_tv.c                   |     4 +-
+ drivers/gpu/drm/sun4i/sun8i_mixer.c                |     2 +-
+ drivers/gpu/drm/sun4i/sun8i_ui_layer.c             |     4 +-
+ drivers/gpu/drm/sun4i/sun8i_vi_layer.c             |     4 +-
+ drivers/gpu/drm/sun4i/sunxi_engine.h               |     4 +-
+ drivers/gpu/drm/sysfb/corebootdrm.c                |    21 +
+ drivers/gpu/drm/sysfb/drm_sysfb_helper.h           |     8 +-
+ drivers/gpu/drm/sysfb/drm_sysfb_modeset.c          |     8 +-
+ drivers/gpu/drm/sysfb/efidrm.c                     |    19 +
+ drivers/gpu/drm/sysfb/ofdrm.c                      |    21 +-
+ drivers/gpu/drm/sysfb/simpledrm.c                  |    21 +
+ drivers/gpu/drm/sysfb/vesadrm.c                    |    23 +-
+ drivers/gpu/drm/tegra/dc.c                         |    32 +-
+ drivers/gpu/drm/tegra/dc.h                         |     2 +-
+ drivers/gpu/drm/tegra/drm.c                        |     6 +-
+ drivers/gpu/drm/tegra/hdmi.c                       |     2 -
+ drivers/gpu/drm/tegra/hub.c                        |    12 +-
+ drivers/gpu/drm/tegra/hub.h                        |     4 +-
+ drivers/gpu/drm/tegra/rgb.c                        |     2 -
+ drivers/gpu/drm/tests/Makefile                     |     2 +-
+ ...tomic_state_test.c => drm_atomic_commit_test.c} |    12 +-
+ drivers/gpu/drm/tests/drm_bridge_test.c            |    10 +-
+ drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c |    88 +-
+ drivers/gpu/drm/tests/drm_kunit_helpers.c          |    18 +-
+ drivers/gpu/drm/tidss/tidss_crtc.c                 |    10 +-
+ drivers/gpu/drm/tidss/tidss_encoder.c              |     8 +-
+ drivers/gpu/drm/tidss/tidss_kms.c                  |     6 +-
+ drivers/gpu/drm/tidss/tidss_oldi.c                 |     4 +-
+ drivers/gpu/drm/tidss/tidss_plane.c                |     8 +-
+ drivers/gpu/drm/tilcdc/tilcdc_crtc.c               |    10 +-
+ drivers/gpu/drm/tilcdc/tilcdc_drv.c                |     2 +-
+ drivers/gpu/drm/tilcdc/tilcdc_encoder.c            |     6 -
+ drivers/gpu/drm/tilcdc/tilcdc_plane.c              |     4 +-
+ drivers/gpu/drm/tiny/appletbdrm.c                  |     6 +-
+ drivers/gpu/drm/tiny/bochs.c                       |    10 +-
+ drivers/gpu/drm/tiny/cirrus-qemu.c                 |     8 +-
+ drivers/gpu/drm/tiny/hx8357d.c                     |     2 +-
+ drivers/gpu/drm/tiny/ili9163.c                     |     2 +-
+ drivers/gpu/drm/tiny/ili9225.c                     |     6 +-
+ drivers/gpu/drm/tiny/ili9341.c                     |     2 +-
+ drivers/gpu/drm/tiny/ili9486.c                     |     2 +-
+ drivers/gpu/drm/tiny/mi0283qt.c                    |     2 +-
+ drivers/gpu/drm/tiny/panel-mipi-dbi.c              |     2 +-
+ drivers/gpu/drm/tiny/pixpaper.c                    |    10 +-
+ drivers/gpu/drm/tiny/sharp-memory.c                |    10 +-
+ drivers/gpu/drm/tve200/tve200_drv.c                |     6 +-
+ drivers/gpu/drm/udl/udl_modeset.c                  |     8 +-
+ drivers/gpu/drm/v3d/Makefile                       |     1 +
+ drivers/gpu/drm/v3d/v3d_debugfs.c                  |    23 +-
+ drivers/gpu/drm/v3d/v3d_drv.c                      |   160 +-
+ drivers/gpu/drm/v3d/v3d_drv.h                      |    18 +
+ drivers/gpu/drm/v3d/v3d_gem.c                      |    17 +-
+ drivers/gpu/drm/v3d/v3d_irq.c                      |    15 +-
+ drivers/gpu/drm/v3d/v3d_mmu.c                      |    10 +-
+ drivers/gpu/drm/v3d/v3d_perfmon.c                  |    18 +-
+ drivers/gpu/drm/v3d/v3d_power.c                    |    87 +
+ drivers/gpu/drm/v3d/v3d_sched.c                    |     1 -
+ drivers/gpu/drm/v3d/v3d_submit.c                   |    19 +-
+ drivers/gpu/drm/vboxvideo/vbox_mode.c              |    18 +-
+ drivers/gpu/drm/vc4/tests/vc4_mock.h               |     4 +-
+ drivers/gpu/drm/vc4/tests/vc4_mock_output.c        |     4 +-
+ drivers/gpu/drm/vc4/tests/vc4_test_pv_muxing.c     |    50 +-
+ drivers/gpu/drm/vc4/vc4_bo.c                       |     1 -
+ drivers/gpu/drm/vc4/vc4_crtc.c                     |    10 +-
+ drivers/gpu/drm/vc4/vc4_drv.c                      |     2 -
+ drivers/gpu/drm/vc4/vc4_drv.h                      |    28 +-
+ drivers/gpu/drm/vc4/vc4_dsi.c                      |     8 +-
+ drivers/gpu/drm/vc4/vc4_gem.c                      |     1 -
+ drivers/gpu/drm/vc4/vc4_hdmi.c                     |    12 +-
+ drivers/gpu/drm/vc4/vc4_hvs.c                      |    10 +-
+ drivers/gpu/drm/vc4/vc4_irq.c                      |    33 +-
+ drivers/gpu/drm/vc4/vc4_kms.c                      |    30 +-
+ drivers/gpu/drm/vc4/vc4_plane.c                    |    10 +-
+ drivers/gpu/drm/vc4/vc4_render_cl.c                |     1 -
+ drivers/gpu/drm/vc4/vc4_txp.c                      |    10 +-
+ drivers/gpu/drm/vc4/vc4_validate.c                 |     1 -
+ drivers/gpu/drm/vc4/vc4_vec.c                      |     4 +-
+ drivers/gpu/drm/verisilicon/vs_bridge.c            |     7 +-
+ drivers/gpu/drm/verisilicon/vs_crtc.c              |     4 +-
+ drivers/gpu/drm/verisilicon/vs_plane.c             |    54 +-
+ drivers/gpu/drm/verisilicon/vs_plane.h             |    18 +-
+ drivers/gpu/drm/verisilicon/vs_primary_plane.c     |    56 +-
+ drivers/gpu/drm/virtio/virtgpu_display.c           |     8 +-
+ drivers/gpu/drm/virtio/virtgpu_drv.h               |     2 +
+ drivers/gpu/drm/virtio/virtgpu_ioctl.c             |     1 +
+ drivers/gpu/drm/virtio/virtgpu_plane.c             |     6 +-
+ drivers/gpu/drm/virtio/virtgpu_vram.c              |    30 +-
+ drivers/gpu/drm/vkms/vkms_crtc.c                   |     6 +-
+ drivers/gpu/drm/vkms/vkms_drv.c                    |     4 +-
+ drivers/gpu/drm/vkms/vkms_plane.c                  |     4 +-
+ drivers/gpu/drm/vkms/vkms_writeback.c              |     4 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c       |     4 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.h       |     4 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_drv.h                |     2 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_kms.c                |    18 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_kms.h                |     8 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c                |     3 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c               |     5 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c               |     9 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c               |    78 +-
+ drivers/gpu/drm/vmwgfx/vmwgfx_vkms.h               |    12 +-
+ drivers/gpu/drm/xe/Makefile                        |    10 +-
+ drivers/gpu/drm/xe/abi/guc_klvs_abi.h              |     8 +
+ drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h            |    65 +
+ .../xe/compat-i915-headers/intel_clock_gating.h    |    10 +-
+ .../drm/xe/compat-i915-headers/intel_mchbar_regs.h |     6 -
+ .../drm/xe/compat-i915-headers/intel_pci_config.h  |     6 -
+ .../gpu/drm/xe/compat-i915-headers/intel_step.h    |    13 -
+ .../gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h   |    42 -
+ .../drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h   |     6 -
+ drivers/gpu/drm/xe/display/intel_fbdev_fb.c        |   120 -
+ drivers/gpu/drm/xe/display/xe_display.c            |    13 +-
+ drivers/gpu/drm/xe/display/xe_display_bo.c         |   120 +
+ drivers/gpu/drm/xe/display/xe_display_bo.h         |     6 +
+ drivers/gpu/drm/xe/display/xe_fb_pin.c             |   233 +-
+ drivers/gpu/drm/xe/display/xe_fb_pin.h             |    21 +
+ drivers/gpu/drm/xe/display/xe_initial_plane.c      |    48 +-
+ drivers/gpu/drm/xe/instructions/xe_gpu_commands.h  |     4 +-
+ drivers/gpu/drm/xe/regs/xe_engine_regs.h           |     9 +-
+ drivers/gpu/drm/xe/regs/xe_gt_regs.h               |     5 +
+ drivers/gpu/drm/xe/regs/xe_irq_regs.h              |     1 +
+ drivers/gpu/drm/xe/regs/xe_lrc_layout.h            |     3 +
+ drivers/gpu/drm/xe/regs/xe_oa_regs.h               |     3 +
+ drivers/gpu/drm/xe/regs/xe_pcode_regs.h            |     3 +
+ drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h          |    36 +
+ .../gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c |    21 +-
+ drivers/gpu/drm/xe/tests/xe_migrate.c              |     5 +-
+ drivers/gpu/drm/xe/tests/xe_pci.c                  |     4 +-
+ drivers/gpu/drm/xe/xe_bo.c                         |    37 +-
+ drivers/gpu/drm/xe/xe_bo_types.h                   |     2 +-
+ drivers/gpu/drm/xe/xe_debugfs.c                    |    43 +-
+ drivers/gpu/drm/xe/xe_dep_scheduler.c              |     1 -
+ drivers/gpu/drm/xe/xe_dep_scheduler.h              |     5 +
+ drivers/gpu/drm/xe/xe_devcoredump.c                |    10 +-
+ drivers/gpu/drm/xe/xe_device.c                     |    64 +-
+ drivers/gpu/drm/xe/xe_device.h                     |    23 +-
+ drivers/gpu/drm/xe/xe_device_types.h               |    25 +-
+ drivers/gpu/drm/xe/xe_device_wa_oob.rules          |     3 +
+ drivers/gpu/drm/xe/xe_drm_ras.c                    |    35 +-
+ drivers/gpu/drm/xe/xe_drm_ras.h                    |     4 +-
+ drivers/gpu/drm/xe/xe_eu_stall.c                   |    40 +-
+ drivers/gpu/drm/xe/xe_eu_stall.h                   |     4 +-
+ drivers/gpu/drm/xe/xe_exec_queue.c                 |    14 +-
+ drivers/gpu/drm/xe/xe_exec_queue_types.h           |     2 +-
+ drivers/gpu/drm/xe/xe_execlist.c                   |    10 -
+ drivers/gpu/drm/xe/xe_ggtt.c                       |     8 +-
+ drivers/gpu/drm/xe/xe_gpu_scheduler.c              |     1 -
+ drivers/gpu/drm/xe/xe_gt.c                         |     1 -
+ drivers/gpu/drm/xe/xe_gt.h                         |    15 +
+ drivers/gpu/drm/xe/xe_gt_mcr.c                     |    23 +-
+ drivers/gpu/drm/xe/xe_gt_mcr.h                     |     1 -
+ drivers/gpu/drm/xe/xe_gt_sriov_pf.c                |     1 +
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c         |   262 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h         |     2 +
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c         |   124 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h         |     4 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h          |     1 -
+ drivers/gpu/drm/xe/xe_gt_stats.c                   |    41 +
+ drivers/gpu/drm/xe/xe_gt_stats_types.h             |   118 +
+ drivers/gpu/drm/xe/xe_gt_types.h                   |     4 +-
+ drivers/gpu/drm/xe/xe_guc.c                        |    17 +-
+ drivers/gpu/drm/xe/xe_guc_ads.c                    |     5 +-
+ drivers/gpu/drm/xe/xe_guc_capture.c                |     8 +-
+ drivers/gpu/drm/xe/xe_guc_capture.h                |     4 +-
+ drivers/gpu/drm/xe/xe_guc_capture_types.h          |     6 +-
+ drivers/gpu/drm/xe/xe_guc_ct.c                     |    38 +-
+ drivers/gpu/drm/xe/xe_guc_ct_types.h               |     8 +-
+ drivers/gpu/drm/xe/xe_guc_fwif.h                   |     4 +-
+ drivers/gpu/drm/xe/xe_guc_pc.c                     |    42 +-
+ drivers/gpu/drm/xe/xe_guc_pc.h                     |     2 +-
+ drivers/gpu/drm/xe/xe_guc_relay_types.h            |     2 +-
+ drivers/gpu/drm/xe/xe_guc_submit.c                 |    76 +-
+ drivers/gpu/drm/xe/xe_guc_submit_types.h           |     2 +-
+ drivers/gpu/drm/xe/xe_hw_engine.c                  |    86 +-
+ drivers/gpu/drm/xe/xe_hw_engine.h                  |     1 -
+ drivers/gpu/drm/xe/xe_hw_engine_types.h            |     2 +-
+ drivers/gpu/drm/xe/xe_hw_error.c                   |    32 +-
+ drivers/gpu/drm/xe/xe_hw_error.h                   |     4 +-
+ drivers/gpu/drm/xe/xe_hwmon.c                      |    28 +-
+ drivers/gpu/drm/xe/xe_irq.c                        |     2 +
+ drivers/gpu/drm/xe/xe_lrc.c                        |   262 +-
+ drivers/gpu/drm/xe/xe_lrc.h                        |    10 +-
+ drivers/gpu/drm/xe/xe_lrc_types.h                  |    11 +
+ drivers/gpu/drm/xe/xe_memirq.c                     |    32 +-
+ drivers/gpu/drm/xe/xe_migrate.c                    |    72 +-
+ drivers/gpu/drm/xe/xe_migrate.h                    |     4 +-
+ drivers/gpu/drm/xe/xe_nvm.h                        |     4 +-
+ drivers/gpu/drm/xe/xe_oa.c                         |   176 +-
+ drivers/gpu/drm/xe/xe_oa_types.h                   |    12 +-
+ drivers/gpu/drm/xe/xe_pagefault_types.h            |     4 +-
+ drivers/gpu/drm/xe/xe_pat.c                        |     9 +
+ drivers/gpu/drm/xe/xe_pat.h                        |     8 +
+ drivers/gpu/drm/xe/xe_pci.c                        |    12 +-
+ drivers/gpu/drm/xe/xe_pci_sriov.c                  |     2 +
+ drivers/gpu/drm/xe/xe_pci_types.h                  |     2 +-
+ drivers/gpu/drm/xe/xe_pcode_api.h                  |     8 +-
+ drivers/gpu/drm/xe/xe_pm.c                         |     6 +
+ drivers/gpu/drm/xe/xe_pt.c                         |     3 +-
+ drivers/gpu/drm/xe/xe_pt_types.h                   |     2 +-
+ drivers/gpu/drm/xe/xe_pt_walk.h                    |    14 +-
+ drivers/gpu/drm/xe/xe_pxp.c                        |     6 +
+ drivers/gpu/drm/xe/xe_pxp.h                        |     6 +-
+ drivers/gpu/drm/xe/xe_pxp_debugfs.h                |     6 +-
+ drivers/gpu/drm/xe/xe_pxp_submit.h                 |     6 +-
+ drivers/gpu/drm/xe/xe_pxp_types.h                  |     6 +-
+ drivers/gpu/drm/xe/xe_query.c                      |     7 +
+ drivers/gpu/drm/xe/xe_ras.c                        |    93 +
+ drivers/gpu/drm/xe/xe_ras.h                        |    15 +
+ drivers/gpu/drm/xe/xe_ras_types.h                  |    73 +
+ drivers/gpu/drm/xe/xe_reg_sr.h                     |     4 +-
+ drivers/gpu/drm/xe/xe_reg_sr_types.h               |     4 +-
+ drivers/gpu/drm/xe/xe_reg_whitelist.c              |    14 +
+ drivers/gpu/drm/xe/xe_reg_whitelist.h              |     4 +-
+ drivers/gpu/drm/xe/xe_res_cursor.h                 |    14 +-
+ drivers/gpu/drm/xe/xe_ring_ops.c                   |     8 +-
+ drivers/gpu/drm/xe/xe_rtp.c                        |    15 +
+ drivers/gpu/drm/xe/xe_rtp.h                        |    28 +-
+ drivers/gpu/drm/xe/xe_rtp_helpers.h                |     6 +-
+ drivers/gpu/drm/xe/xe_rtp_types.h                  |     4 +-
+ drivers/gpu/drm/xe/xe_sriov_pf.c                   |     6 -
+ drivers/gpu/drm/xe/xe_sriov_pf_helpers.h           |     3 +-
+ drivers/gpu/drm/xe/xe_sriov_pf_provision.c         |    41 +
+ drivers/gpu/drm/xe/xe_sriov_pf_provision.h         |     1 +
+ drivers/gpu/drm/xe/xe_sriov_pf_types.h             |     3 -
+ drivers/gpu/drm/xe/xe_step.c                       |     2 +-
+ drivers/gpu/drm/xe/xe_step.h                       |     4 +-
+ drivers/gpu/drm/xe/xe_step_types.h                 |    61 +-
+ drivers/gpu/drm/xe/xe_survivability_mode.c         |    14 +-
+ drivers/gpu/drm/xe/xe_survivability_mode.h         |     2 +-
+ drivers/gpu/drm/xe/xe_svm.c                        |    11 +-
+ drivers/gpu/drm/xe/xe_sync.c                       |     2 +-
+ drivers/gpu/drm/xe/xe_sysctrl.c                    |   132 +
+ drivers/gpu/drm/xe/xe_sysctrl.h                    |    24 +
+ drivers/gpu/drm/xe/xe_sysctrl_event.c              |    88 +
+ drivers/gpu/drm/xe/xe_sysctrl_event_types.h        |    57 +
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox.c            |   371 +
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox.h            |    31 +
+ drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h      |    58 +
+ drivers/gpu/drm/xe/xe_sysctrl_types.h              |    39 +
+ drivers/gpu/drm/xe/xe_tlb_inval.c                  |     2 +-
+ drivers/gpu/drm/xe/xe_trace_lrc.h                  |    27 +
+ drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c             |    64 +-
+ drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h             |    12 +
+ drivers/gpu/drm/xe/xe_ttm_vram_mgr.c               |    18 +-
+ drivers/gpu/drm/xe/xe_tuning.c                     |     7 +-
+ drivers/gpu/drm/xe/xe_tuning.h                     |     4 +-
+ drivers/gpu/drm/xe/xe_uc_fw.c                      |    11 +
+ drivers/gpu/drm/xe/xe_uc_fw.h                      |     4 +
+ drivers/gpu/drm/xe/xe_uc_fw_abi.h                  |     4 +-
+ drivers/gpu/drm/xe/xe_vm.c                         |    14 +-
+ drivers/gpu/drm/xe/xe_vm_types.h                   |     3 +-
+ drivers/gpu/drm/xe/xe_vram.c                       |    22 +
+ drivers/gpu/drm/xe/xe_vram.h                       |     2 +
+ drivers/gpu/drm/xe/xe_wa.c                         |    13 +
+ drivers/gpu/drm/xe/xe_wa.h                         |     4 +-
+ drivers/gpu/drm/xe/xe_wa_oob.rules                 |     7 +
+ drivers/gpu/drm/xlnx/Kconfig                       |     4 +-
+ drivers/gpu/drm/xlnx/zynqmp_disp.c                 |    56 +-
+ drivers/gpu/drm/xlnx/zynqmp_dp.c                   |     6 +-
+ drivers/gpu/drm/xlnx/zynqmp_kms.c                  |    22 +-
+ include/drm/bridge/analogix_dp.h                   |     8 +-
+ include/drm/bridge/dw_hdmi.h                       |     6 +
+ include/drm/bridge/imx.h                           |    17 -
+ include/drm/bridge/of-display-mode-bridge.h        |    17 +
+ include/drm/display/drm_dp.h                       |    20 +-
+ include/drm/display/drm_dp_helper.h                |     1 +
+ include/drm/display/drm_dp_mst_helper.h            |    32 +-
+ include/drm/display/drm_dp_tunnel.h                |    30 +-
+ include/drm/display/drm_hdmi_state_helper.h        |     6 +-
+ include/drm/drm_atomic.h                           |   158 +-
+ include/drm/drm_atomic_helper.h                    |    78 +-
+ include/drm/drm_atomic_state_helper.h              |     4 +-
+ include/drm/drm_blend.h                            |     4 +-
+ include/drm/drm_bridge.h                           |    89 +-
+ include/drm/drm_colorop.h                          |     4 +-
+ include/drm/drm_connector.h                        |     8 +-
+ include/drm/drm_crtc.h                             |    10 +-
+ include/drm/drm_crtc_helper.h                      |     4 +-
+ include/drm/drm_damage_helper.h                    |     2 +-
+ include/drm/drm_debugfs_crc.h                      |     2 +-
+ include/drm/drm_encoder.h                          |     4 +
+ include/drm/drm_gpusvm.h                           |    12 +-
+ include/drm/drm_kunit_helpers.h                    |     2 +-
+ include/drm/drm_mipi_dbi.h                         |     8 +-
+ include/drm/drm_mipi_dsi.h                         |     1 +
+ include/drm/drm_mode_config.h                      |    38 +-
+ include/drm/drm_modeset_helper_vtables.h           |    44 +-
+ include/drm/drm_of.h                               |    13 +
+ include/drm/drm_pagemap.h                          |     9 +-
+ include/drm/drm_panel.h                            |     5 +-
+ include/drm/drm_plane.h                            |     6 +-
+ include/drm/drm_ras.h                              |    11 +
+ include/drm/drm_self_refresh_helper.h              |     6 +-
+ include/drm/drm_vblank_helper.h                    |     8 +-
+ include/drm/gpu_scheduler.h                        |    45 +-
+ include/drm/intel/display_parent_interface.h       |    65 +-
+ .../drm/intel/mchbar_regs.h                        |     7 +-
+ .../drm/intel/pci_config.h                         |     0
+ include/drm/intel/pciids.h                         |     6 +-
+ .../drm/intel/vlv_iosf_sb_regs.h                   |    18 +-
+ include/drm/ttm/ttm_placement.h                    |     4 +-
+ include/linux/dma-fence-array.h                    |     6 +-
+ include/linux/gpu_buddy.h                          |    41 +
+ include/trace/events/amdxdna.h                     |    42 +-
+ include/trace/events/dma_fence.h                   |    40 +-
+ include/uapi/drm/amdxdna_accel.h                   |    27 +-
+ include/uapi/drm/drm.h                             |    27 +-
+ include/uapi/drm/drm_fourcc.h                      |    54 +-
+ include/uapi/drm/drm_mode.h                        |    19 +-
+ include/uapi/drm/drm_ras.h                         |     1 +
+ include/uapi/drm/drm_sarea.h                       |    20 +-
+ include/uapi/drm/virtgpu_drm.h                     |     4 +
+ include/uapi/drm/xe_drm.h                          |   206 +-
+ include/uapi/linux/kfd_ioctl.h                     |    38 +-
+ 1499 files changed, 121165 insertions(+), 22075 deletions(-)
+ create mode 100644 Documentation/ABI/obsolete/sysfs-driver-ivpu
+ create mode 100644 Documentation/ABI/testing/sysfs-driver-ivpu
+ create mode 100644 Documentation/devicetree/bindings/display/panel/chipwealth,ch13726a.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/focaltech,ota7290b.yaml
+ create mode 100644 Documentation/devicetree/bindings/display/panel/novatek,nt35532.yaml
+ create mode 100644 Documentation/gpu/amdgpu/ptl.rst
+ create mode 100644 Documentation/gpu/intel-display/async-flip.rst
+ create mode 100644 Documentation/gpu/intel-display/atomic.rst
+ create mode 100644 Documentation/gpu/intel-display/audio.rst
+ create mode 100644 Documentation/gpu/intel-display/casf.rst
+ create mode 100644 Documentation/gpu/intel-display/cdclk.rst
+ create mode 100644 Documentation/gpu/intel-display/cmtg.rst
+ create mode 100644 Documentation/gpu/intel-display/dmc.rst
+ create mode 100644 Documentation/gpu/intel-display/dpio.rst
+ create mode 100644 Documentation/gpu/intel-display/dpll.rst
+ create mode 100644 Documentation/gpu/intel-display/drrs.rst
+ create mode 100644 Documentation/gpu/intel-display/dsb.rst
+ create mode 100644 Documentation/gpu/intel-display/fbc.rst
+ create mode 100644 Documentation/gpu/intel-display/fifo-underrun.rst
+ create mode 100644 Documentation/gpu/intel-display/frontbuffer.rst
+ create mode 100644 Documentation/gpu/intel-display/hotplug.rst
+ create mode 100644 Documentation/gpu/intel-display/index.rst
+ create mode 100644 Documentation/gpu/intel-display/plane.rst
+ create mode 100644 Documentation/gpu/intel-display/psr.rst
+ create mode 100644 Documentation/gpu/intel-display/snps-phy.rst
+ create mode 100644 Documentation/gpu/intel-display/vbt.rst
+ create mode 100644 Documentation/gpu/xe/xe_gt_stats.rst
+ create mode 100644 drivers/accel/amdxdna/aie.c
+ create mode 100644 drivers/accel/amdxdna/aie.h
+ delete mode 100644 drivers/accel/amdxdna/aie2_psp.c
+ delete mode 100644 drivers/accel/amdxdna/aie2_smu.c
+ create mode 100644 drivers/accel/amdxdna/aie4_ctx.c
+ create mode 100644 drivers/accel/amdxdna/aie4_host_queue.h
+ create mode 100644 drivers/accel/amdxdna/aie4_message.c
+ create mode 100644 drivers/accel/amdxdna/aie4_msg_priv.h
+ create mode 100644 drivers/accel/amdxdna/aie4_pci.c
+ create mode 100644 drivers/accel/amdxdna/aie4_pci.h
+ create mode 100644 drivers/accel/amdxdna/aie4_sriov.c
+ create mode 100644 drivers/accel/amdxdna/aie_psp.c
+ create mode 100644 drivers/accel/amdxdna/aie_smu.c
+ create mode 100644 drivers/accel/amdxdna/amdxdna_cbuf.c
+ create mode 100644 drivers/accel/amdxdna/amdxdna_cbuf.h
+ create mode 100644 drivers/accel/amdxdna/amdxdna_debugfs.c
+ create mode 100644 drivers/accel/amdxdna/amdxdna_debugfs.h
+ create mode 100644 drivers/accel/amdxdna/npu3_regs.c
+ create mode 100644 drivers/dma-buf/.kunitconfig
+ delete mode 100644 drivers/dma-buf/selftest.c
+ delete mode 100644 drivers/dma-buf/selftest.h
+ delete mode 100644 drivers/dma-buf/selftests.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.h
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/vpe_v2_0.c
+ create mode 100644 drivers/gpu/drm/amd/amdgpu/vpe_v2_0.h
+ create mode 100644 drivers/gpu/drm/amd/display/dc/dml2_0/README.md
+ delete mode 100644 drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.h
+ create mode 100644 drivers/gpu/drm/amd/display/modules/inc/mod_color_types.h
+ create mode 100644 drivers/gpu/drm/amd/display/modules/inc/mod_info_packet_types.h
+ create mode 100644 drivers/gpu/drm/amd/display/modules/inc/mod_power.h
+ create mode 100644 drivers/gpu/drm/amd/display/modules/power/power.c
+ create mode 100644 drivers/gpu/drm/amd/display/modules/power/power_abm.c
+ create mode 100644 drivers/gpu/drm/amd/display/modules/power/power_psr.c
+ create mode 100644 drivers/gpu/drm/amd/display/modules/power/power_replay.c
+ create mode 100644 drivers/gpu/drm/amd/include/amdgpu_ptl.h
+ create mode 100755 drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_1_offset.h
+ create mode 100755 drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_1_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_3_2_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_3_2_sh_mask.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/vpe/vpe_2_0_0_offset.h
+ create mode 100644 drivers/gpu/drm/amd/include/asic_reg/vpe/vpe_2_0_0_sh_mask.h
+ delete mode 100644 drivers/gpu/drm/bridge/imx/imx-legacy-bridge.c
+ create mode 100644 drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx-connector-fixup.c
+ create mode 100644 drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx-connector-fixup.dtso
+ create mode 100644 drivers/gpu/drm/bridge/of-display-mode-bridge.c
+ create mode 100644 drivers/gpu/drm/i915/display/intel_display_clock_gating.c
+ create mode 100644 drivers/gpu/drm/i915/display/intel_display_clock_gating.h
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_fb_pin.c
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_fb_pin.h
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_fbdev_fb.c
+ delete mode 100644 drivers/gpu/drm/i915/display/intel_fbdev_fb.h
+ create mode 100644 drivers/gpu/drm/i915/display/intel_mchbar.c
+ create mode 100644 drivers/gpu/drm/i915/display/intel_mchbar.h
+ create mode 100644 drivers/gpu/drm/i915/i915_fb_pin.c
+ create mode 100644 drivers/gpu/drm/i915/i915_fb_pin.h
+ create mode 100644 drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c
+ create mode 100644 drivers/gpu/drm/panel/panel-focaltech-ota7290b.c
+ create mode 100644 drivers/gpu/drm/panel/panel-novatek-nt35532.c
+ create mode 100644 drivers/gpu/drm/panthor/panthor_fw_regs.h
+ create mode 100644 drivers/gpu/drm/panthor/panthor_gpu_regs.h
+ create mode 100644 drivers/gpu/drm/panthor/panthor_mmu_regs.h
+ create mode 100644 drivers/gpu/drm/panthor/panthor_pwr_regs.h
+ delete mode 100644 drivers/gpu/drm/panthor/panthor_regs.h
+ create mode 100644 drivers/gpu/drm/scheduler/sched_rq.c
+ create mode 100644 drivers/gpu/drm/scheduler/tests/tests_scheduler.c
+ rename drivers/gpu/drm/tests/{drm_atomic_state_test.c => drm_atomic_commit_test.c} (98%)
+ create mode 100644 drivers/gpu/drm/v3d/v3d_power.c
+ create mode 100644 drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/intel_step.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h
+ delete mode 100644 drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h
+ delete mode 100644 drivers/gpu/drm/xe/display/intel_fbdev_fb.c
+ create mode 100644 drivers/gpu/drm/xe/display/xe_fb_pin.h
+ create mode 100644 drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h
+ create mode 100644 drivers/gpu/drm/xe/xe_ras.c
+ create mode 100644 drivers/gpu/drm/xe/xe_ras.h
+ create mode 100644 drivers/gpu/drm/xe/xe_ras_types.h
+ create mode 100644 drivers/gpu/drm/xe/xe_sysctrl.c
+ create mode 100644 drivers/gpu/drm/xe/xe_sysctrl.h
+ create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_event.c
+ create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_event_types.h
+ create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
+ create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_mailbox.h
+ create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
+ create mode 100644 drivers/gpu/drm/xe/xe_sysctrl_types.h
+ delete mode 100644 include/drm/bridge/imx.h
+ create mode 100644 include/drm/bridge/of-display-mode-bridge.h
+ rename drivers/gpu/drm/i915/intel_mchbar_regs.h => include/drm/intel/mchbar_regs.h (98%)
+ rename drivers/gpu/drm/i915/intel_pci_config.h => include/drm/intel/pci_config.h (100%)
+ rename drivers/gpu/drm/i915/vlv_iosf_sb_reg.h => include/drm/intel/vlv_iosf_sb_regs.h (95%)
+$ git am -3 ../patches/0001-drm-rdma-Fix-up-merge.patch
+Applying: drm/rdma: Fix up merge
+Using index info to reconstruct a base tree...
+M	drivers/infiniband/core/umem_dmabuf.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+$ git am -3 ../patches/drm-misc-what-is-a-merge
+Applying: drm/gem: Make the GEM LRU lock part of drm_device
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/gpu/drm/panthor/panthor_device.h
+M	drivers/gpu/drm/panthor/panthor_gem.c
+M	drivers/gpu/drm/panthor/panthor_mmu.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 8db5d0bdea8e6] Merge branch 'drm-next' of https://gitlab.freedesktop.org/drm/kernel.git
+ Date: Thu May 21 11:35:37 2026 +0100
+Merging drm-exynos/for-linux-next (3a8660878839f Linux 6.18-rc1)
+$ git merge -m Merge branch 'for-linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/daeinki/drm-exynos.git drm-exynos/for-linux-next
+Already up to date.
+Merging drm-misc/for-linux-next (a48bbcc7ac739 drm/virtio: use uninterruptible resv lock for plane updates)
+  a48bbcc7ac739 ("drm/virtio: use uninterruptible resv lock for plane updates")
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/misc/kernel.git drm-misc/for-linux-next
+Auto-merging drivers/accel/amdxdna/amdxdna_gem.c
+Auto-merging drivers/accel/amdxdna/amdxdna_gem.h
+Auto-merging drivers/gpu/drm/bridge/chipone-icn6211.c
+Auto-merging drivers/gpu/drm/virtio/virtgpu_drv.h
+Merge made by the 'ort' strategy.
+ drivers/accel/amdxdna/aie2_ctx.c               |  45 +++-
+ drivers/accel/amdxdna/aie2_message.c           |  52 +++-
+ drivers/accel/amdxdna/aie2_msg_priv.h          |   1 +
+ drivers/accel/amdxdna/aie2_pci.c               |   1 +
+ drivers/accel/amdxdna/aie2_pci.h               |   8 +-
+ drivers/accel/amdxdna/amdxdna_ctx.c            |  82 +++++-
+ drivers/accel/amdxdna/amdxdna_ctx.h            |   2 +
+ drivers/accel/amdxdna/amdxdna_gem.c            | 330 +++++++++++++++++++------
+ drivers/accel/amdxdna/amdxdna_gem.h            |  16 +-
+ drivers/accel/amdxdna/amdxdna_pci_drv.c        |  12 +-
+ drivers/accel/amdxdna/amdxdna_pci_drv.h        |   8 +-
+ drivers/accel/amdxdna/npu1_regs.c              |   1 +
+ drivers/accel/amdxdna/npu4_regs.c              |   2 +
+ drivers/accel/amdxdna/npu5_regs.c              |   1 +
+ drivers/accel/amdxdna/npu6_regs.c              |   1 +
+ drivers/accel/ethosu/ethosu_job.c              |  18 +-
+ drivers/gpu/drm/bridge/adv7511/adv7511.h       |   1 -
+ drivers/gpu/drm/bridge/adv7511/adv7511_drv.c   |  15 +-
+ drivers/gpu/drm/bridge/chipone-icn6211.c       |  11 +-
+ drivers/gpu/drm/bridge/chrontel-ch7033.c       |  28 +--
+ drivers/gpu/drm/bridge/lontium-lt8713sx.c      |  10 +-
+ drivers/gpu/drm/bridge/lontium-lt9611.c        |   9 +-
+ drivers/gpu/drm/bridge/lontium-lt9611uxc.c     |   9 +-
+ drivers/gpu/drm/display/drm_dp_mst_topology.c  |  23 +-
+ drivers/gpu/drm/drm_bridge.c                   |  45 +++-
+ drivers/gpu/drm/drm_of.c                       |  26 +-
+ drivers/gpu/drm/drm_syncobj.c                  |  10 +-
+ drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c   |   9 +-
+ drivers/gpu/drm/imagination/Makefile           |   3 +
+ drivers/gpu/drm/imagination/pvr_context.c      |  32 +--
+ drivers/gpu/drm/imagination/pvr_device.c       |   2 +-
+ drivers/gpu/drm/imagination/pvr_drv.c          |   3 +
+ drivers/gpu/drm/imagination/pvr_fw.c           |   4 +-
+ drivers/gpu/drm/imagination/pvr_fw.h           |   7 +-
+ drivers/gpu/drm/imagination/pvr_job.c          |   3 +
+ drivers/gpu/drm/imagination/pvr_mmu.c          |   4 +-
+ drivers/gpu/drm/imagination/pvr_power.c        |  10 +-
+ drivers/gpu/drm/imagination/pvr_queue.c        |  12 +-
+ drivers/gpu/drm/imagination/pvr_trace.h        | 113 +++++++++
+ drivers/gpu/drm/imagination/pvr_trace_points.c |   7 +
+ drivers/gpu/drm/msm/hdmi/hdmi.c                |  70 ++++--
+ drivers/gpu/drm/panel/panel-edp.c              |   1 +
+ drivers/gpu/drm/virtio/virtgpu_drv.c           |   1 +
+ drivers/gpu/drm/virtio/virtgpu_drv.h           |   2 +
+ drivers/gpu/drm/virtio/virtgpu_ioctl.c         |  10 +
+ drivers/gpu/drm/virtio/virtgpu_kms.c           |  14 +-
+ drivers/gpu/drm/xlnx/zynqmp_dp.c               |  19 +-
+ include/drm/drm_bridge.h                       |   7 +
+ include/uapi/drm/virtgpu_drm.h                 |   1 +
+ include/uapi/linux/virtio_gpu.h                |   9 +
+ 50 files changed, 864 insertions(+), 246 deletions(-)
+ create mode 100644 drivers/gpu/drm/imagination/pvr_trace.h
+ create mode 100644 drivers/gpu/drm/imagination/pvr_trace_points.c
+$ git am -3 ../patches/0001-drm-bridge-microchip-lvds-Rename-drm_atomic_state-to.patch
+Applying: drm/bridge: microchip-lvds: Rename drm_atomic_state to drm_atomic_commit
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/bridge/microchip-lvds.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging amdgpu/drm-next (4cdbba5a16aaf drm/amdgpu: restructure VM state machine v4)
+$ git merge -m Merge branch 'drm-next' of https://gitlab.freedesktop.org/agd5f/linux.git amdgpu/drm-next
+Already up to date.
+$ git am -3 ../patches/0001-amdgpu-Fix-up-drm_atomic_commit-rename.patch
+Applying: amdgpu: Fix up drm_atomic_commit rename
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+No changes -- Patch already applied.
+Merging drm-intel/for-linux-next (c987f8c20946d drm/i915/scaler: eliminate dead code)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/i915/kernel.git drm-intel/for-linux-next
+Auto-merging drivers/gpu/drm/i915/display/intel_dp.c
+Auto-merging drivers/gpu/drm/i915/display/intel_psr.c
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/i915/display/intel_bw.c            | 209 +++++++++++-------
+ drivers/gpu/drm/i915/display/intel_casf.c          |  32 +--
+ drivers/gpu/drm/i915/display/intel_display.c       |   6 +-
+ drivers/gpu/drm/i915/display/intel_display_core.h  |   3 +
+ .../gpu/drm/i915/display/intel_display_device.h    |   1 -
+ drivers/gpu/drm/i915/display/intel_display_irq.c   | 244 +++++++++++++++++----
+ drivers/gpu/drm/i915/display/intel_display_irq.h   |  37 ++--
+ drivers/gpu/drm/i915/display/intel_display_wa.c    |   2 +
+ drivers/gpu/drm/i915/display/intel_display_wa.h    |   1 +
+ drivers/gpu/drm/i915/display/intel_dmc_wl.c        |   9 +-
+ drivers/gpu/drm/i915/display/intel_dp.c            |   8 +-
+ drivers/gpu/drm/i915/display/intel_dp_aux.c        |  20 +-
+ .../gpu/drm/i915/display/intel_dp_aux_backlight.c  |  19 +-
+ drivers/gpu/drm/i915/display/intel_psr.c           |   5 +
+ drivers/gpu/drm/i915/display/skl_scaler.c          |  76 ++++---
+ drivers/gpu/drm/i915/i915_irq.c                    | 182 ++++++---------
+ drivers/gpu/drm/xe/display/xe_display.c            |  17 +-
+ 17 files changed, 529 insertions(+), 342 deletions(-)
+Merging drm-msm/msm-next (db339b6bc9f23 dt-bindings: display/msm: Fix typo in clock-names property)
+$ git merge -m Merge branch 'msm-next' of https://gitlab.freedesktop.org/drm/msm.git drm-msm/msm-next
+Already up to date.
+Merging drm-msm-lumag/msm-next-lumag (df7cfac4b2057 drm/msm/dp: clear EDID on display unplug)
+$ git merge -m Merge branch 'msm-next-lumag' of https://gitlab.freedesktop.org/lumag/msm.git drm-msm-lumag/msm-next-lumag
+Auto-merging drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
+Auto-merging drivers/gpu/drm/msm/dp/dp_ctrl.c
+Auto-merging drivers/gpu/drm/msm/dp/dp_display.c
+Auto-merging drivers/gpu/drm/msm/dp/dp_drm.c
+Auto-merging drivers/gpu/drm/msm/dp/dp_drm.h
+Merge made by the 'ort' strategy.
+ .../bindings/display/msm/dsi-controller-main.yaml  |   2 +
+ .../bindings/display/msm/qcom,milos-mdss.yaml      | 286 ++++++++
+ .../bindings/display/msm/qcom,sm8650-dpu.yaml      |   1 +
+ .../devicetree/bindings/phy/qcom,dsi-phy-7nm.yaml  |   1 +
+ .../gpu/drm/msm/disp/dpu1/catalog/dpu_10_2_milos.h | 279 ++++++++
+ drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c     |  29 +
+ drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h     |   1 +
+ drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c            |   1 +
+ drivers/gpu/drm/msm/dp/dp_ctrl.c                   |  16 -
+ drivers/gpu/drm/msm/dp/dp_ctrl.h                   |   1 -
+ drivers/gpu/drm/msm/dp/dp_display.c                | 730 +++++++--------------
+ drivers/gpu/drm/msm/dp/dp_display.h                |   3 +-
+ drivers/gpu/drm/msm/dp/dp_drm.c                    |  63 +-
+ drivers/gpu/drm/msm/dp/dp_drm.h                    |   2 +
+ drivers/gpu/drm/msm/dp/dp_panel.c                  |   8 +
+ drivers/gpu/drm/msm/dp/dp_panel.h                  |   2 +
+ drivers/gpu/drm/msm/dp/dp_reg.h                    |   4 +-
+ drivers/gpu/drm/msm/dsi/phy/dsi_phy.c              |   2 +
+ drivers/gpu/drm/msm/dsi/phy/dsi_phy.h              |   1 +
+ drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c          |  23 +
+ drivers/gpu/drm/msm/msm_mdss.c                     | 118 ++--
+ 21 files changed, 928 insertions(+), 645 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/display/msm/qcom,milos-mdss.yaml
+ create mode 100644 drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_10_2_milos.h
+Merging drm-xe/drm-xe-next (339fa0be9e4a5 drm/xe/oa: Fix exec_queue leak on width check in stream open)
+$ git merge -m Merge branch 'drm-xe-next' of https://gitlab.freedesktop.org/drm/xe/kernel.git drm-xe/drm-xe-next
+Auto-merging drivers/gpu/drm/xe/display/xe_display.c
+Merge made by the 'ort' strategy.
+ drivers/gpu/drm/xe/display/xe_display.c     |  33 +++-----
+ drivers/gpu/drm/xe/display/xe_display.h     |  21 ++++-
+ drivers/gpu/drm/xe/regs/xe_gt_regs.h        |   4 +-
+ drivers/gpu/drm/xe/tests/xe_rtp_test.c      |  71 +++++++++++++++--
+ drivers/gpu/drm/xe/tests/xe_wa_test.c       |  14 +++-
+ drivers/gpu/drm/xe/xe_device.c              |  13 +--
+ drivers/gpu/drm/xe/xe_gsc.c                 |   5 +-
+ drivers/gpu/drm/xe/xe_gt.c                  |   8 +-
+ drivers/gpu/drm/xe/xe_gt_idle.c             |   4 +-
+ drivers/gpu/drm/xe/xe_gt_mcr.c              |  65 ++++++++++-----
+ drivers/gpu/drm/xe/xe_gt_mcr.h              |   1 +
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c |   6 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h |   2 +-
+ drivers/gpu/drm/xe/xe_gt_sriov_vf.c         |  24 ++++--
+ drivers/gpu/drm/xe/xe_gt_sriov_vf.h         |   6 +-
+ drivers/gpu/drm/xe/xe_guc.c                 |   2 +-
+ drivers/gpu/drm/xe/xe_guc_submit.c          |  16 ++--
+ drivers/gpu/drm/xe/xe_hw_engine.c           |  26 ++++--
+ drivers/gpu/drm/xe/xe_hw_engine.h           |   2 +-
+ drivers/gpu/drm/xe/xe_hw_engine_types.h     |   2 +
+ drivers/gpu/drm/xe/xe_memirq.c              | 118 +++++++++++++++++-----------
+ drivers/gpu/drm/xe/xe_memirq_types.h        |  13 +--
+ drivers/gpu/drm/xe/xe_oa.c                  |   6 +-
+ drivers/gpu/drm/xe/xe_pci.c                 |   2 +
+ drivers/gpu/drm/xe/xe_reg_sr.c              |  36 +++++++++
+ drivers/gpu/drm/xe/xe_tuning.c              |   2 +-
+ drivers/gpu/drm/xe/xe_wa.c                  |   6 +-
+ 27 files changed, 348 insertions(+), 160 deletions(-)
+$ git am -3 ../patches/0001-drm-xe-Fix-up-merge-issue.patch
+Applying: drm: xe: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+No changes -- Patch already applied.
+Merging drm-rust/for-linux-next (8bfe9d72cf206 gpu: nova: Use module names consistently)
+$ git merge -m Merge branch 'for-linux-next' of https://gitlab.freedesktop.org/drm/rust/kernel.git drm-rust/for-linux-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/gpu/drm/drm_gpuvm.c
+Merge made by the 'ort' strategy.
+ Documentation/gpu/nova/core/vbios.rst     |   63 +-
+ MAINTAINERS                               |    2 +
+ drivers/gpu/drm/Kconfig                   |    7 +
+ drivers/gpu/drm/drm_gpuvm.c               |    6 +-
+ drivers/gpu/drm/nova/Kconfig              |    3 +-
+ drivers/gpu/drm/nova/Makefile             |    3 +-
+ drivers/gpu/drm/nova/driver.rs            |    6 +-
+ drivers/gpu/drm/nova/nova.rs              |    2 +-
+ drivers/gpu/drm/tyr/Kconfig               |    1 +
+ drivers/gpu/drm/tyr/driver.rs             |   58 +-
+ drivers/gpu/drm/tyr/gem.rs                |   24 +-
+ drivers/gpu/drm/tyr/gpu.rs                |  175 ++-
+ drivers/gpu/drm/tyr/regs.rs               | 1725 +++++++++++++++++++++++++++--
+ drivers/gpu/nova-core/Kconfig             |    3 +-
+ drivers/gpu/nova-core/Makefile            |    3 +-
+ drivers/gpu/nova-core/falcon.rs           |    4 +-
+ drivers/gpu/nova-core/falcon/hal.rs       |   24 +-
+ drivers/gpu/nova-core/falcon/hal/ga102.rs |    8 +-
+ drivers/gpu/nova-core/falcon/hal/tu102.rs |    4 +-
+ drivers/gpu/nova-core/fb.rs               |   27 +-
+ drivers/gpu/nova-core/fb/hal.rs           |   24 +-
+ drivers/gpu/nova-core/fb/hal/ga100.rs     |    6 +
+ drivers/gpu/nova-core/fb/hal/ga102.rs     |    4 +
+ drivers/gpu/nova-core/fb/hal/tu102.rs     |   11 +-
+ drivers/gpu/nova-core/firmware.rs         |    8 +-
+ drivers/gpu/nova-core/firmware/gsp.rs     |   28 +-
+ drivers/gpu/nova-core/gfw.rs              |   76 --
+ drivers/gpu/nova-core/gpu.rs              |   26 +-
+ drivers/gpu/nova-core/gpu/hal.rs          |   28 +
+ drivers/gpu/nova-core/gpu/hal/gh100.rs    |   18 +
+ drivers/gpu/nova-core/gpu/hal/tu102.rs    |   86 ++
+ drivers/gpu/nova-core/gsp/boot.rs         |   18 +-
+ drivers/gpu/nova-core/gsp/fw.rs           |   31 +-
+ drivers/gpu/nova-core/nova_core.rs        |    5 +-
+ drivers/gpu/nova-core/regs.rs             |    7 +-
+ drivers/gpu/nova-core/vbios.rs            |   87 +-
+ rust/bindings/bindings_helper.h           |    1 +
+ rust/helpers/drm_gpuvm.c                  |   26 +
+ rust/helpers/helpers.c                    |    1 +
+ rust/kernel/drm/gpuvm/mod.rs              |  328 ++++++
+ rust/kernel/drm/gpuvm/sm_ops.rs           |  429 +++++++
+ rust/kernel/drm/gpuvm/va.rs               |  168 +++
+ rust/kernel/drm/gpuvm/vm_bo.rs            |  249 +++++
+ rust/kernel/drm/mod.rs                    |    1 +
+ 44 files changed, 3410 insertions(+), 404 deletions(-)
+ delete mode 100644 drivers/gpu/nova-core/gfw.rs
+ create mode 100644 drivers/gpu/nova-core/gpu/hal.rs
+ create mode 100644 drivers/gpu/nova-core/gpu/hal/gh100.rs
+ create mode 100644 drivers/gpu/nova-core/gpu/hal/tu102.rs
+ create mode 100644 rust/helpers/drm_gpuvm.c
+ create mode 100644 rust/kernel/drm/gpuvm/mod.rs
+ create mode 100644 rust/kernel/drm/gpuvm/sm_ops.rs
+ create mode 100644 rust/kernel/drm/gpuvm/va.rs
+ create mode 100644 rust/kernel/drm/gpuvm/vm_bo.rs
+$ git am -3 ../patches/0001-drm-nova-Fix-up-merge-issue.patch
+Applying: drm: nova: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	drivers/gpu/nova-core/gsp/cmdq.rs
+Falling back to patching base and 3-way merge...
+Auto-merging drivers/gpu/nova-core/gsp/cmdq.rs
+No changes -- Patch already applied.
+Merging drm-nova/nova-next (93296e9d9528f gpu: nova-core: vbios: store reference to Device where relevant)
+$ git merge -m Merge branch 'nova-next' of https://gitlab.freedesktop.org/drm/nova.git drm-nova/nova-next
+Already up to date.
+Merging etnaviv/etnaviv/next (6bde14ba5f7ef drm/etnaviv: add optional reset support)
+$ git merge -m Merge branch 'etnaviv/next' of https://git.pengutronix.de/git/lst/linux etnaviv/etnaviv/next
+Already up to date.
+Merging fbdev/for-next (63a347282ebea fbdev: sunxvr2500: replace printk with device-aware logging functions)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git fbdev/for-next
+Auto-merging CREDITS
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ CREDITS                                         |   3 +
+ Documentation/admin-guide/kernel-parameters.txt |   5 -
+ MAINTAINERS                                     |   7 -
+ arch/alpha/kernel/io.c                          |   2 +-
+ arch/powerpc/include/asm/vga.h                  |   4 +-
+ drivers/tty/vt/vt.c                             |   3 -
+ drivers/video/console/Kconfig                   |  15 -
+ drivers/video/console/Makefile                  |   1 -
+ drivers/video/console/mdacon.c                  | 566 --------------------
+ drivers/video/fbdev/Kconfig                     |  13 -
+ drivers/video/fbdev/Makefile                    |   1 -
+ drivers/video/fbdev/arkfb.c                     |   4 +-
+ drivers/video/fbdev/atmel_lcdfb.c               |   6 +-
+ drivers/video/fbdev/aty/aty128fb.c              | 143 ++---
+ drivers/video/fbdev/aty/radeon_base.c           |   3 +-
+ drivers/video/fbdev/broadsheetfb.c              |   8 +-
+ drivers/video/fbdev/carminefb.c                 |   6 +-
+ drivers/video/fbdev/chipsfb.c                   |   1 +
+ drivers/video/fbdev/cirrusfb.c                  |   2 +-
+ drivers/video/fbdev/core/fbcon.c                |  12 +-
+ drivers/video/fbdev/cyber2000fb.c               |  24 +-
+ drivers/video/fbdev/efifb.c                     |   1 +
+ drivers/video/fbdev/geode/gx1fb_core.c          |  10 +-
+ drivers/video/fbdev/hecubafb.c                  |   6 +-
+ drivers/video/fbdev/hgafb.c                     | 685 ------------------------
+ drivers/video/fbdev/i740fb.c                    |   1 +
+ drivers/video/fbdev/imxfb.c                     |   6 +-
+ drivers/video/fbdev/kyro/fbdev.c                |   5 +-
+ drivers/video/fbdev/matrox/matroxfb_base.c      |  40 +-
+ drivers/video/fbdev/matrox/matroxfb_maven.c     |   2 +-
+ drivers/video/fbdev/metronomefb.c               |   8 +-
+ drivers/video/fbdev/neofb.c                     |  29 +-
+ drivers/video/fbdev/nvidia/nvidia.c             |  10 +-
+ drivers/video/fbdev/pm2fb.c                     |  11 +-
+ drivers/video/fbdev/pm3fb.c                     |   5 +-
+ drivers/video/fbdev/pvr2fb.c                    |   5 +-
+ drivers/video/fbdev/riva/fbdev.c                | 128 ++---
+ drivers/video/fbdev/s3fb.c                      |  33 +-
+ drivers/video/fbdev/savage/savagefb_driver.c    | 140 ++---
+ drivers/video/fbdev/sis/sis_main.h              |  26 +-
+ drivers/video/fbdev/sm501fb.c                   |   3 +
+ drivers/video/fbdev/sm712.h                     |   2 +-
+ drivers/video/fbdev/ssd1307fb.c                 |   8 +-
+ drivers/video/fbdev/sunxvr2500.c                |  19 +-
+ drivers/video/fbdev/tdfxfb.c                    |  22 +-
+ drivers/video/fbdev/tridentfb.c                 |  45 +-
+ drivers/video/fbdev/uvesafb.c                   |   4 +-
+ drivers/video/fbdev/vesafb.c                    |   1 +
+ drivers/video/fbdev/vt8623fb.c                  |   4 +-
+ include/linux/console.h                         |   2 -
+ include/linux/vt_buffer.h                       |   2 +-
+ 51 files changed, 355 insertions(+), 1737 deletions(-)
+ delete mode 100644 drivers/video/console/mdacon.c
+ delete mode 100644 drivers/video/fbdev/hgafb.c
+$ git am -3 ../patches/0001-fix-up-for-drm-hyperv-Remove-reference-to-hyperv_fb-.patch
+Applying: fix up for "drm/hyperv: Remove reference to hyperv_fb driver"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/gpu/drm/hyperv/Kconfig
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master db92dc753798f] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/deller/linux-fbdev.git
+ Date: Thu May 21 11:36:06 2026 +0100
+Merging regmap/for-next (4b126e54759a2 Merge remote-tracking branch 'regmap/for-7.2' into regmap-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regmap.git regmap/for-next
+Merge made by the 'ort' strategy.
+ drivers/base/regmap/regmap-i2c.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging sound/for-next (2ee646353cd5f ALSA: seq: Register kernel port with full information)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tiwai/sound.git sound/for-next
+Auto-merging sound/hda/codecs/realtek/alc269.c
+Auto-merging sound/usb/mixer_scarlett2.c
+Merge made by the 'ort' strategy.
+ drivers/base/firmware_loader/main.c            |  68 +++-
+ include/linux/firmware.h                       |  10 +
+ include/sound/emux_synth.h                     |   2 +-
+ include/sound/seq_midi_emul.h                  |   6 +-
+ include/sound/snd_wavefront.h                  |   7 +
+ sound/ac97_bus.c                               |   4 +-
+ sound/core/jack.c                              |  13 +-
+ sound/core/oss/pcm_plugin.c                    |   2 +-
+ sound/core/pcm_native.c                        |  14 +-
+ sound/core/seq/oss/seq_oss_readq.c             |  13 +-
+ sound/core/seq/oss/seq_oss_readq.h             |   3 +-
+ sound/core/seq/seq_clientmgr.c                 |   7 +-
+ sound/core/seq/seq_midi_emul.c                 |  38 +-
+ sound/core/seq/seq_ports.c                     |  24 +-
+ sound/core/seq/seq_ports.h                     |   8 +-
+ sound/hda/codecs/Kconfig                       |   1 +
+ sound/hda/codecs/ca0132.c                      | 111 +++++-
+ sound/hda/codecs/realtek/alc269.c              |  23 +-
+ sound/hda/codecs/side-codecs/cs35l41_hda.c     |  77 ++--
+ sound/hda/codecs/side-codecs/cs35l41_hda.h     |   5 +
+ sound/hda/codecs/side-codecs/tas2781_hda_i2c.c |   3 +
+ sound/hda/codecs/side-codecs/tas2781_hda_spi.c |   3 +
+ sound/hda/core/i915.c                          |  30 +-
+ sound/isa/opti9xx/miro.c                       | 268 ++++++++++++-
+ sound/isa/wavefront/wavefront.c                |  61 ++-
+ sound/isa/wavefront/wavefront_midi.c           |  47 ++-
+ sound/isa/wavefront/wavefront_synth.c          |  96 +++++
+ sound/oss/dmasound/dmasound_core.c             |  10 +-
+ sound/pci/ali5451/ali5451.c                    |   4 +-
+ sound/pci/als300.c                             |   6 +-
+ sound/pci/als4000.c                            |   4 +-
+ sound/pci/asihpi/asihpi.c                      |  17 +-
+ sound/pci/asihpi/hpicmn.c                      |  16 +-
+ sound/pci/asihpi/hpicmn.h                      |   4 +-
+ sound/pci/asihpi/hpipcida.h                    |  18 +-
+ sound/pci/atiixp.c                             |  10 +-
+ sound/pci/atiixp_modem.c                       |   6 +-
+ sound/pci/au88x0/au8810.c                      |   4 +-
+ sound/pci/au88x0/au8820.c                      |   4 +-
+ sound/pci/au88x0/au8830.c                      |   4 +-
+ sound/pci/aw2/aw2-alsa.c                       |   5 +-
+ sound/pci/azt3328.c                            |   6 +-
+ sound/pci/ca0106/ca0106_main.c                 |   4 +-
+ sound/pci/cmipci.c                             |  12 +-
+ sound/pci/cs4281.c                             |   4 +-
+ sound/pci/cs46xx/cs46xx.c                      |   8 +-
+ sound/pci/ctxfi/ctmixer.c                      |  44 +--
+ sound/pci/ctxfi/ctmixer.h                      |   7 +-
+ sound/pci/ctxfi/ctsrc.c                        |  15 +-
+ sound/pci/ctxfi/ctsrc.h                        |   2 +-
+ sound/pci/echoaudio/darla20.c                  |   4 +-
+ sound/pci/echoaudio/darla24.c                  |   6 +-
+ sound/pci/echoaudio/echo3g.c                   |   4 +-
+ sound/pci/echoaudio/gina20.c                   |   4 +-
+ sound/pci/echoaudio/gina24.c                   |  10 +-
+ sound/pci/echoaudio/indigo.c                   |   4 +-
+ sound/pci/echoaudio/indigodj.c                 |   4 +-
+ sound/pci/echoaudio/indigodjx.c                |   4 +-
+ sound/pci/echoaudio/indigoio.c                 |   4 +-
+ sound/pci/echoaudio/indigoiox.c                |   4 +-
+ sound/pci/echoaudio/layla20.c                  |   6 +-
+ sound/pci/echoaudio/layla24.c                  |   4 +-
+ sound/pci/echoaudio/mia.c                      |   6 +-
+ sound/pci/echoaudio/mona.c                     |  20 +-
+ sound/pci/emu10k1/emu10k1.c                    |   8 +-
+ sound/pci/emu10k1/emu10k1x.c                   |   4 +-
+ sound/pci/ens1370.c                            |  10 +-
+ sound/pci/es1938.c                             |   4 +-
+ sound/pci/es1968.c                             |  27 +-
+ sound/pci/fm801.c                              |  15 +-
+ sound/pci/ice1712/ice1712.c                    |   4 +-
+ sound/pci/ice1712/ice1724.c                    |  23 +-
+ sound/pci/intel8x0.c                           | 118 ++++--
+ sound/pci/intel8x0m.c                          |  88 ++++-
+ sound/pci/maestro3.c                           |  51 ++-
+ sound/pci/mixart/mixart.c                      |   4 +-
+ sound/pci/nm256/nm256.c                        |   8 +-
+ sound/pci/pcxhr/pcxhr.c                        |  70 ++--
+ sound/pci/rme32.c                              |   8 +-
+ sound/pci/rme96.c                              |  10 +-
+ sound/pci/sonicvibes.c                         |   4 +-
+ sound/pci/trident/trident.c                    |  16 +-
+ sound/pci/via82xx.c                            |   6 +-
+ sound/pci/via82xx_modem.c                      |   4 +-
+ sound/pci/vx222/vx222.c                        |   6 +-
+ sound/pci/ymfpci/ymfpci.c                      |  14 +-
+ sound/synth/emux/emux_seq.c                    |  11 +-
+ sound/usb/fcp.c                                |   4 +-
+ sound/usb/mixer.c                              |  58 +--
+ sound/usb/mixer_quirks.c                       | 107 ++++-
+ sound/usb/mixer_scarlett.c                     |   4 +-
+ sound/usb/mixer_scarlett2.c                    | 525 +++++++++++++++++++++++--
+ sound/usb/mixer_us16x08.c                      | 127 +++---
+ sound/usb/usx2y/usbusx2y.c                     |  39 +-
+ sound/virtio/virtio_kctl.c                     |  50 +++
+ sound/virtio/virtio_pcm.c                      |   3 +-
+ sound/virtio/virtio_pcm_ops.c                  |   3 +-
+ 97 files changed, 2040 insertions(+), 635 deletions(-)
+Merging ieee1394/for-next (8208d94f149a5 ALSA: firewire: Make use of ieee1394's .driver_data_ptr)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ieee1394/linux1394.git ieee1394/for-next
+Merge made by the 'ort' strategy.
+ drivers/firewire/core-cdev.c    | 376 ++++++++++++++++++++++++----------------
+ include/linux/mod_devicetable.h |   5 +-
+ sound/firewire/dice/dice.c      |  34 ++--
+ sound/firewire/fireface/ff.c    |  12 +-
+ sound/firewire/motu/motu.c      |   6 +-
+ sound/firewire/oxfw/oxfw.c      |   4 +-
+ 6 files changed, 260 insertions(+), 177 deletions(-)
+Merging sound-asoc/for-next (5ce08d8617b6f Merge remote-tracking branch 'asoc/for-7.2' into asoc-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound.git sound-asoc/for-next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm/mach-pxa/pxa27x.c
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/sound/fsl,sai.yaml         |     7 +
+ .../devicetree/bindings/sound/gpio-audio-amp.yaml  |   270 +
+ .../bindings/sound/mediatek,mt2701-audio.yaml      |    23 +
+ .../bindings/sound/mediatek,mt2701-hdmi-audio.yaml |    48 +
+ .../bindings/sound/mediatek,mt6351-sound.yaml      |    37 +
+ .../bindings/sound/mediatek,mt8196-afe.yaml        |   113 +
+ .../bindings/sound/mediatek,mt8196-nau8825.yaml    |   100 +
+ Documentation/devicetree/bindings/sound/mt6351.txt |    16 -
+ .../devicetree/bindings/sound/ti,tas67524.yaml     |   280 +
+ Documentation/sound/alsa-configuration.rst         |     4 +-
+ Documentation/sound/codecs/index.rst               |     1 +
+ Documentation/sound/codecs/tas675x.rst             |   820 ++
+ Documentation/sound/soc/index.rst                  |     2 +-
+ Documentation/sound/soc/platform.rst               |     2 +-
+ Documentation/sound/soc/usb.rst                    |     4 +-
+ MAINTAINERS                                        |    11 +
+ arch/arm/mach-pxa/pxa27x.c                         |     9 +-
+ drivers/gpio/gpiolib-of.c                          |     8 +
+ drivers/soundwire/bus.c                            |    31 +
+ include/linux/of.h                                 |     7 +
+ include/linux/platform_data/asoc-pxa.h             |    23 +-
+ include/linux/soundwire/sdw.h                      |     8 +
+ include/sound/ac97/codec.h                         |     5 +-
+ include/sound/ac97/controller.h                    |     5 +-
+ include/sound/pxa2xx-lib.h                         |    50 +-
+ include/sound/sdca.h                               |     4 +
+ include/sound/sdca_asoc.h                          |     6 +
+ include/sound/sdca_function.h                      |     1 -
+ include/sound/sdca_jack.h                          |     3 +
+ include/sound/soc-dapm.h                           |     1 +
+ include/sound/soc.h                                |     4 +-
+ include/sound/soc_sdw_utils.h                      |    18 +
+ sound/ac97/bus.c                                   |    26 +-
+ sound/arm/Kconfig                                  |     7 -
+ sound/arm/Makefile                                 |     4 -
+ sound/soc/apple/mca.c                              |    14 +-
+ sound/soc/atmel/atmel-classd.c                     |    41 +-
+ sound/soc/atmel/mchp-spdifrx.c                     |    33 +-
+ sound/soc/codecs/Kconfig                           |    32 +-
+ sound/soc/codecs/Makefile                          |     6 +
+ sound/soc/codecs/cs35l56-sdw.c                     |   147 +-
+ sound/soc/codecs/cs35l56-shared-test.c             |     6 +
+ sound/soc/codecs/cs35l56-shared.c                  |     2 +
+ sound/soc/codecs/cs35l56.c                         |     2 +
+ sound/soc/codecs/cs35l56.h                         |     1 +
+ sound/soc/codecs/cs42l42-sdw.c                     |    11 +-
+ sound/soc/codecs/cs42l43-jack.c                    |     5 +-
+ sound/soc/codecs/cx20442.c                         |    11 +-
+ sound/soc/codecs/cx20442.h                         |     5 +
+ sound/soc/codecs/es9356.c                          |  1151 ++
+ sound/soc/codecs/es9356.h                          |   208 +
+ sound/soc/codecs/max98363.c                        |    17 +-
+ sound/soc/codecs/max98373-sdw.c                    |    18 +-
+ sound/soc/codecs/nau8825.c                         |     8 +-
+ sound/soc/codecs/pcm6240.c                         |    36 +-
+ sound/soc/codecs/pcm6240.h                         |     2 +-
+ sound/soc/codecs/rt1017-sdca-sdw.c                 |    18 +-
+ sound/soc/codecs/rt1308-sdw.c                      |    18 +-
+ sound/soc/codecs/rt1316-sdw.c                      |    18 +-
+ sound/soc/codecs/rt1318-sdw.c                      |    16 +-
+ sound/soc/codecs/rt1320-sdw.c                      |    16 +-
+ sound/soc/codecs/rt5640.c                          |    37 +-
+ sound/soc/codecs/rt5682-sdw.c                      |    16 +-
+ sound/soc/codecs/rt700-sdw.c                       |    18 +-
+ sound/soc/codecs/rt711-sdca-sdw.c                  |    16 +-
+ sound/soc/codecs/rt711-sdw.c                       |    14 +-
+ sound/soc/codecs/rt712-sdca-dmic.c                 |    19 +-
+ sound/soc/codecs/rt712-sdca-sdw.c                  |    16 +-
+ sound/soc/codecs/rt715-sdca-sdw.c                  |    18 +-
+ sound/soc/codecs/rt715-sdw.c                       |    18 +-
+ sound/soc/codecs/rt721-sdca-sdw.c                  |    16 +-
+ sound/soc/codecs/rt722-sdca-sdw.c                  |    55 +-
+ sound/soc/codecs/rt722-sdca.c                      |   369 +-
+ sound/soc/codecs/rt722-sdca.h                      |    27 +
+ sound/soc/codecs/sigmadsp.c                        |     4 +-
+ sound/soc/codecs/simple-amplifier.c                |   926 +-
+ sound/soc/codecs/tac5xx2-sdw.c                     |  2037 ++++
+ sound/soc/codecs/tac5xx2.h                         |   259 +
+ sound/soc/codecs/tas2783-sdw.c                     |    24 +-
+ sound/soc/codecs/tas675x.c                         |  2194 ++++
+ sound/soc/codecs/tas675x.h                         |   367 +
+ sound/soc/codecs/tlv320aic3x.c                     |     7 +
+ sound/soc/codecs/uda1380.c                         |    13 +-
+ sound/soc/codecs/wsa881x.c                         |    35 +-
+ sound/soc/fsl/Kconfig                              |     6 +-
+ sound/soc/fsl/eukrea-tlv320.c                      |    11 +-
+ sound/soc/fsl/fsl-asoc-card.c                      |   111 +-
+ sound/soc/fsl/fsl_micfil.c                         |    79 +
+ sound/soc/fsl/fsl_micfil.h                         |     1 +
+ sound/soc/fsl/fsl_sai.c                            |     7 +-
+ sound/soc/fsl/fsl_sai.h                            |     2 +
+ sound/soc/intel/boards/Kconfig                     |     1 +
+ sound/soc/intel/boards/cht_bsw_rt5672.c            |    73 +-
+ sound/soc/intel/boards/sof_sdw.c                   |    23 +-
+ sound/soc/intel/common/soc-acpi-intel-arl-match.c  |    52 +
+ sound/soc/intel/common/soc-acpi-intel-mtl-match.c  |   132 +
+ sound/soc/intel/common/sof-function-topology-lib.c |    11 +-
+ sound/soc/mediatek/Kconfig                         |    40 +
+ sound/soc/mediatek/Makefile                        |     1 +
+ .../soc/mediatek/common/mtk-afe-platform-driver.c  |    54 +-
+ .../soc/mediatek/common/mtk-afe-platform-driver.h  |     2 +
+ sound/soc/mediatek/mt2701/Makefile                 |     1 +
+ sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.c  |    22 +
+ sound/soc/mediatek/mt2701/mt2701-afe-common.h      |     6 +
+ sound/soc/mediatek/mt2701/mt2701-afe-pcm.c         |   284 +-
+ sound/soc/mediatek/mt2701/mt2701-hdmi.c            |   114 +
+ sound/soc/mediatek/mt2701/mt2701-reg.h             |    29 +
+ sound/soc/mediatek/mt8196/Makefile                 |    14 +
+ sound/soc/mediatek/mt8196/mt8196-afe-clk.c         |   581 +
+ sound/soc/mediatek/mt8196/mt8196-afe-clk.h         |    64 +
+ sound/soc/mediatek/mt8196/mt8196-afe-common.h      |   205 +
+ sound/soc/mediatek/mt8196/mt8196-afe-pcm.c         |  2519 ++++
+ sound/soc/mediatek/mt8196/mt8196-dai-adda.c        |   845 ++
+ sound/soc/mediatek/mt8196/mt8196-dai-i2s.c         |  2613 ++++
+ sound/soc/mediatek/mt8196/mt8196-dai-tdm.c         |   675 ++
+ sound/soc/mediatek/mt8196/mt8196-interconnection.h |   121 +
+ sound/soc/mediatek/mt8196/mt8196-nau8825.c         |   870 ++
+ sound/soc/mediatek/mt8196/mt8196-reg.h             | 12068 +++++++++++++++++++
+ sound/soc/pxa/Kconfig                              |     6 +
+ sound/soc/pxa/Makefile                             |     4 +-
+ sound/soc/pxa/mmp-sspa.c                           |     2 +-
+ sound/soc/pxa/pxa-ssp.c                            |     2 +-
+ sound/{arm => soc/pxa}/pxa2xx-ac97-lib.c           |    41 +-
+ sound/{arm => soc/pxa}/pxa2xx-ac97-regs.h          |     0
+ sound/soc/pxa/pxa2xx-ac97.c                        |     6 +-
+ sound/soc/pxa/pxa2xx-i2s.c                         |     2 +-
+ sound/soc/pxa/pxa2xx-lib.h                         |    45 +
+ sound/{arm => soc/pxa}/pxa2xx-pcm-lib.c            |    26 +-
+ sound/soc/pxa/pxa2xx-pcm.c                         |     3 +-
+ sound/soc/rockchip/rk3399_gru_sound.c              |     2 +-
+ sound/soc/rockchip/rockchip_i2s.c                  |     2 +-
+ sound/soc/rockchip/rockchip_i2s_tdm.c              |     2 +-
+ sound/soc/samsung/i2s.c                            |   118 +-
+ sound/soc/samsung/idma.c                           |    26 +-
+ sound/soc/samsung/odroid.c                         |    21 +-
+ sound/soc/samsung/pcm.c                            |    99 +-
+ sound/soc/samsung/spdif.c                          |    25 +-
+ sound/soc/sdca/sdca_asoc.c                         |   106 +-
+ sound/soc/sdca/sdca_class.c                        |     7 -
+ sound/soc/sdca/sdca_class.h                        |     1 -
+ sound/soc/sdca/sdca_class_function.c               |    23 +-
+ sound/soc/sdca/sdca_functions.c                    |    51 +-
+ sound/soc/sdca/sdca_interrupts.c                   |     3 +-
+ sound/soc/sdca/sdca_jack.c                         |    76 +-
+ sound/soc/sdw_utils/Makefile                       |     1 +
+ sound/soc/sdw_utils/soc_sdw_es9356.c               |   230 +
+ sound/soc/sdw_utils/soc_sdw_ti_amp.c               |   134 +-
+ sound/soc/sdw_utils/soc_sdw_utils.c                |   287 +-
+ sound/soc/soc-core.c                               |    49 +-
+ sound/soc/soc-dapm.c                               |    30 +-
+ sound/soc/soc-devres.c                             |    28 +-
+ sound/soc/soc-ops.c                                |     4 +
+ sound/soc/sof/topology.c                           |    55 +
+ sound/soc/tegra/tegra20_ac97.c                     |     2 +-
+ sound/soc/tegra/tegra20_das.c                      |     2 +-
+ sound/soc/tegra/tegra20_i2s.c                      |     2 +-
+ sound/soc/tegra/tegra210_admaif.c                  |    28 +-
+ sound/soc/tegra/tegra210_admaif.h                  |     2 +-
+ sound/soc/tegra/tegra210_adx.c                     |    85 +-
+ sound/soc/tegra/tegra210_adx.h                     |     5 +-
+ sound/soc/tegra/tegra210_amx.c                     |    82 +-
+ sound/soc/tegra/tegra210_amx.h                     |     5 +-
+ sound/soc/tegra/tegra210_mixer.c                   |   288 +-
+ sound/soc/tegra/tegra210_mixer.h                   |    21 +-
+ sound/soc/tegra/tegra210_mvc.c                     |     3 +
+ sound/soc/tegra/tegra30_ahub.c                     |     2 +-
+ sound/soc/tegra/tegra30_i2s.c                      |     2 +-
+ sound/soc/tegra/tegra_isomgr_bw.c                  |    20 +-
+ sound/soc/ti/ams-delta.c                           |    47 +-
+ sound/soc/ti/j721e-evm.c                           |    47 +-
+ sound/soc/ti/omap-dmic.c                           |    44 +-
+ sound/soc/ti/omap-hdmi.c                           |    18 +-
+ sound/soc/ti/omap-mcbsp-st.c                       |    26 +-
+ sound/soc/ti/omap-mcbsp.c                          |    96 +-
+ sound/soc/ti/omap-mcpdm.c                          |     8 +-
+ sound/soc/ti/omap3pandora.c                        |     5 +-
+ sound/soc/ti/rx51.c                                |     4 +-
+ sound/soc/uniphier/aio-compress.c                  |    24 +-
+ sound/soc/uniphier/aio-dma.c                       |    50 +-
+ sound/soc/ux500/mop500_ab8500.c                    |    24 +-
+ 180 files changed, 32888 insertions(+), 1513 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/sound/gpio-audio-amp.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/mediatek,mt2701-hdmi-audio.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/mediatek,mt6351-sound.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/mediatek,mt8196-afe.yaml
+ create mode 100644 Documentation/devicetree/bindings/sound/mediatek,mt8196-nau8825.yaml
+ delete mode 100644 Documentation/devicetree/bindings/sound/mt6351.txt
+ create mode 100644 Documentation/devicetree/bindings/sound/ti,tas67524.yaml
+ create mode 100644 Documentation/sound/codecs/tas675x.rst
+ create mode 100644 sound/soc/codecs/es9356.c
+ create mode 100644 sound/soc/codecs/es9356.h
+ create mode 100644 sound/soc/codecs/tac5xx2-sdw.c
+ create mode 100644 sound/soc/codecs/tac5xx2.h
+ create mode 100644 sound/soc/codecs/tas675x.c
+ create mode 100644 sound/soc/codecs/tas675x.h
+ create mode 100644 sound/soc/mediatek/mt2701/mt2701-hdmi.c
+ create mode 100644 sound/soc/mediatek/mt8196/Makefile
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-afe-clk.c
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-afe-clk.h
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-afe-common.h
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-afe-pcm.c
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-dai-adda.c
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-dai-i2s.c
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-dai-tdm.c
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-interconnection.h
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-nau8825.c
+ create mode 100644 sound/soc/mediatek/mt8196/mt8196-reg.h
+ rename sound/{arm => soc/pxa}/pxa2xx-ac97-lib.c (91%)
+ rename sound/{arm => soc/pxa}/pxa2xx-ac97-regs.h (100%)
+ create mode 100644 sound/soc/pxa/pxa2xx-lib.h
+ rename sound/{arm => soc/pxa}/pxa2xx-pcm-lib.c (86%)
+ create mode 100644 sound/soc/sdw_utils/soc_sdw_es9356.c
+Merging modules/modules-next (663385f9155f2 module: Simplify warning on positive returns from module_init())
+$ git merge -m Merge branch 'modules-next' of https://git.kernel.org/pub/scm/linux/kernel/git/modules/linux.git modules/modules-next
+Already up to date.
+Merging input/next (b7710233c16c1 Input: atmel_mxt_ts - use __free() for obuf in mxt_object_show)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/dtor/input.git input/next
+Auto-merging drivers/input/touchscreen/atmel_mxt_ts.c
+Merge made by the 'ort' strategy.
+ .../bindings/input/touchscreen/st,stmfts.yaml      |   4 +
+ drivers/input/keyboard/lm8323.c                    |   2 +-
+ drivers/input/misc/Kconfig                         |  10 -
+ drivers/input/misc/Makefile                        |   1 -
+ drivers/input/misc/pcap_keys.c                     | 125 ----------
+ drivers/input/touchscreen/Kconfig                  |  10 -
+ drivers/input/touchscreen/Makefile                 |   1 -
+ drivers/input/touchscreen/atmel_mxt_ts.c           |  13 +-
+ drivers/input/touchscreen/pcap_ts.c                | 252 ---------------------
+ drivers/input/touchscreen/stmfts.c                 | 160 ++++++++-----
+ drivers/input/touchscreen/tsc2007_core.c           |   6 +-
+ 11 files changed, 118 insertions(+), 466 deletions(-)
+ delete mode 100644 drivers/input/misc/pcap_keys.c
+ delete mode 100644 drivers/input/touchscreen/pcap_ts.c
+Merging block/for-next (c16700adbbe93 Merge branch 'io_uring-7.1' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git block/for-next
+Merge made by the 'ort' strategy.
+ block/bio-integrity-fs.c                           |    6 +-
+ block/bio.c                                        |   44 +-
+ block/blk-core.c                                   |    2 -
+ block/blk.h                                        |    2 +
+ drivers/block/drbd/Makefile                        |    1 +
+ drivers/block/drbd/drbd_buildtag.c                 |    2 +-
+ .../linux => drivers/block/drbd}/drbd_config.h     |    0
+ drivers/block/drbd/drbd_debugfs.c                  |    2 +-
+ drivers/block/drbd/drbd_int.h                      |    6 +-
+ drivers/block/drbd/drbd_main.c                     |    6 +-
+ drivers/block/drbd/drbd_nl.c                       |  414 ++--
+ drivers/block/drbd/drbd_nl_gen.c                   | 2606 ++++++++++++++++++++
+ drivers/block/drbd/drbd_nl_gen.h                   |  395 +++
+ drivers/block/drbd/drbd_proc.c                     |    2 +-
+ fs/eventpoll.c                                     |   94 +-
+ include/linux/bio.h                                |    9 +-
+ include/linux/blkdev.h                             |    1 -
+ include/linux/drbd_genl.h                          |  536 ----
+ include/linux/drbd_genl_api.h                      |   56 -
+ include/linux/eventpoll.h                          |    8 +
+ include/linux/genl_magic_func.h                    |  413 ----
+ include/linux/genl_magic_struct.h                  |  272 --
+ include/linux/io_uring_types.h                     |    3 +
+ include/{ => uapi}/linux/drbd.h                    |   73 +-
+ include/uapi/linux/drbd_genl.h                     |  359 +++
+ include/{ => uapi}/linux/drbd_limits.h             |   10 +-
+ include/uapi/linux/io_uring/bpf_filter.h           |   16 +
+ io_uring/epoll.c                                   |   18 +-
+ io_uring/fdinfo.c                                  |    2 +-
+ io_uring/io_uring.c                                |   12 +-
+ io_uring/io_uring.h                                |    2 +-
+ io_uring/kbuf.c                                    |    2 +-
+ io_uring/net.c                                     |   67 +-
+ io_uring/net.h                                     |    7 +
+ io_uring/opdef.c                                   |    2 +
+ io_uring/rsrc.c                                    |  273 +-
+ io_uring/rsrc.h                                    |    7 +-
+ io_uring/timeout.c                                 |    4 +-
+ io_uring/waitid.c                                  |    1 +
+ 39 files changed, 4072 insertions(+), 1663 deletions(-)
+ rename {include/linux => drivers/block/drbd}/drbd_config.h (100%)
+ create mode 100644 drivers/block/drbd/drbd_nl_gen.c
+ create mode 100644 drivers/block/drbd/drbd_nl_gen.h
+ delete mode 100644 include/linux/drbd_genl.h
+ delete mode 100644 include/linux/drbd_genl_api.h
+ delete mode 100644 include/linux/genl_magic_func.h
+ delete mode 100644 include/linux/genl_magic_struct.h
+ rename include/{ => uapi}/linux/drbd.h (85%)
+ create mode 100644 include/uapi/linux/drbd_genl.h
+ rename include/{ => uapi}/linux/drbd_limits.h (97%)
+$ git am -3 ../patches/0001-Revert-block-remove-bio_last_bvec_all.patch
+Applying: Revert "block: remove bio_last_bvec_all"
+$ git reset HEAD^
+Unstaged changes after reset:
+M	Documentation/block/biovecs.rst
+M	include/linux/bio.h
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master 09e33901a7915] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git
+ Date: Thu May 21 11:36:22 2026 +0100
+Merging device-mapper/for-next (5aa0f9231cbac dm: limit target bio polling to one shot)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git device-mapper/for-next
+Auto-merging drivers/md/raid5.c
+Merge made by the 'ort' strategy.
+ .../admin-guide/device-mapper/dm-inlinecrypt.rst   | 123 +++++
+ Documentation/admin-guide/device-mapper/index.rst  |   1 +
+ block/blk-crypto.c                                 |   3 +
+ drivers/md/Kconfig                                 |  11 +
+ drivers/md/Makefile                                |   1 +
+ drivers/md/dm-ima.c                                | 500 ++++++++---------
+ drivers/md/dm-ima.h                                |  68 ++-
+ drivers/md/dm-inlinecrypt.c                        | 599 +++++++++++++++++++++
+ drivers/md/dm-ioctl.c                              | 151 +++++-
+ drivers/md/dm-raid.c                               |   6 +
+ drivers/md/dm-vdo/indexer/index-layout.c           |   2 +-
+ drivers/md/dm.c                                    |  15 +-
+ drivers/md/md.h                                    |   2 +
+ drivers/md/raid5.c                                 |   7 +-
+ 14 files changed, 1181 insertions(+), 308 deletions(-)
+ create mode 100644 Documentation/admin-guide/device-mapper/dm-inlinecrypt.rst
+ create mode 100644 drivers/md/dm-inlinecrypt.c
+Merging libata/for-next (0926c1b731b73 ata: pata_arasan_cf: simplify ioremap)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/libata/linux libata/for-next
+Auto-merging drivers/ata/libata-eh.c
+Auto-merging drivers/ata/libata-scsi.c
+Merge made by the 'ort' strategy.
+ drivers/ata/ahci.c           | 22 ++++++++++++++++++++++
+ drivers/ata/libahci.c        |  4 ++--
+ drivers/ata/libata-eh.c      |  2 +-
+ drivers/ata/libata-scsi.c    |  2 +-
+ drivers/ata/pata_arasan_cf.c | 24 ++++++------------------
+ 5 files changed, 32 insertions(+), 22 deletions(-)
+Merging pcmcia/pcmcia-next (b3c26ea81ccc5 pcmcia: remove obsolete host controller drivers)
+$ git merge -m Merge branch 'pcmcia-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brodo/linux.git pcmcia/pcmcia-next
+Already up to date.
+Merging mmc/next (ee06a067864b0 mmc: Merge branch fixes into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/ulfh/mmc.git mmc/next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/mmc/renesas,sdhi.yaml      |   1 +
+ .../devicetree/bindings/mmc/spacemit,sdhci.yaml    |  15 ++
+ drivers/memstick/host/tifm_ms.c                    |   2 +-
+ drivers/misc/tifm_core.c                           |   4 +-
+ drivers/mmc/core/queue.c                           |   8 +-
+ drivers/mmc/host/cavium-thunderx.c                 |   2 +-
+ drivers/mmc/host/davinci_mmc.c                     |   4 +-
+ drivers/mmc/host/dw_mmc-exynos.c                   |   1 +
+ drivers/mmc/host/dw_mmc.c                          |   4 +-
+ drivers/mmc/host/dw_mmc.h                          |   4 +-
+ drivers/mmc/host/renesas_sdhi_internal_dmac.c      |   1 +
+ drivers/mmc/host/sdhci-of-k1.c                     | 257 +++++++++++++++++++++
+ drivers/mmc/host/tifm_sd.c                         |   4 +-
+ drivers/mmc/host/via-sdmmc.c                       |   5 +-
+ drivers/mmc/host/wmt-sdmmc.c                       |   2 +-
+ include/linux/tifm.h                               |   2 +-
+ 16 files changed, 299 insertions(+), 17 deletions(-)
+Merging mfd/for-mfd-next (29305fb8c8ac7 mfd: si476x-i2c: Fix spelling mistakes in comments)
+$ git merge -m Merge branch 'for-mfd-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/mfd.git mfd/for-mfd-next
+Auto-merging CREDITS
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ CREDITS                                            |   1 +
+ .../bindings/mfd/aspeed,ast2x00-scu.yaml           | 114 +++
+ .../bindings/mfd/hisilicon,hi655x-pmic.yaml        |  80 +++
+ .../devicetree/bindings/mfd/hisilicon,hi655x.txt   |  33 -
+ .../devicetree/bindings/mfd/khadas,mcu.yaml        |  18 +
+ .../devicetree/bindings/mfd/mediatek,mt6397.yaml   |  64 ++
+ .../devicetree/bindings/mfd/qcom,tcsr.yaml         |   1 +
+ Documentation/devicetree/bindings/mfd/syscon.yaml  |   1 +
+ .../devicetree/bindings/mfd/ti,bq25703a.yaml       |  73 +-
+ MAINTAINERS                                        |   3 +-
+ drivers/gpio/gpio-timberdale.c                     |  26 +-
+ drivers/mfd/Kconfig                                |   9 +-
+ drivers/mfd/Makefile                               |   1 -
+ drivers/mfd/bq257xx.c                              |  54 +-
+ drivers/mfd/cros_ec_dev.c                          |  26 +-
+ drivers/mfd/cs42l43.c                              |   2 +-
+ drivers/mfd/cs5535-mfd.c                           |   9 +
+ drivers/mfd/ezx-pcap.c                             | 491 -------------
+ drivers/mfd/intel-lpss-pci.c                       | 786 ++++++++++-----------
+ drivers/mfd/lpc_ich.c                              | 476 ++++++-------
+ drivers/mfd/lpc_sch.c                              |  10 +-
+ drivers/mfd/max77620.c                             |  36 +-
+ drivers/mfd/max77759.c                             |  48 +-
+ drivers/mfd/menf21bmc.c                            |   8 +-
+ drivers/mfd/rsmu_i2c.c                             |   8 +-
+ drivers/mfd/rsmu_spi.c                             |   7 +-
+ drivers/mfd/si476x-i2c.c                           |   8 +-
+ drivers/mfd/simple-mfd-i2c.c                       |   1 +
+ drivers/mfd/sm501.c                                |   8 +-
+ drivers/mfd/sprd-sc27xx-spi.c                      |  62 +-
+ drivers/mfd/timberdale.c                           |  33 +-
+ drivers/mfd/timberdale.h                           |   1 -
+ drivers/mfd/tps65219.c                             |  14 +-
+ drivers/mfd/tps65910.c                             |  16 +-
+ drivers/mfd/twl-core.c                             |   2 +-
+ drivers/mfd/wm8994-irq.c                           |  94 +--
+ include/linux/mfd/bq257xx.h                        | 412 +++++++++++
+ include/linux/mfd/cs5535.h                         |   8 +
+ include/linux/mfd/ezx-pcap.h                       | 253 -------
+ include/linux/mfd/max77759.h                       |   6 +-
+ include/linux/mfd/rohm-bd72720.h                   |   1 -
+ include/linux/mfd/wm8994/pdata.h                   |   5 -
+ include/linux/timb_gpio.h                          |  25 -
+ 43 files changed, 1658 insertions(+), 1676 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/mfd/hisilicon,hi655x-pmic.yaml
+ delete mode 100644 Documentation/devicetree/bindings/mfd/hisilicon,hi655x.txt
+ delete mode 100644 drivers/mfd/ezx-pcap.c
+ create mode 100644 include/linux/mfd/cs5535.h
+ delete mode 100644 include/linux/mfd/ezx-pcap.h
+ delete mode 100644 include/linux/timb_gpio.h
+Merging backlight/for-backlight-next (acea35b567d0f MAINTAINERS: Add cgbc backlight driver)
+$ git merge -m Merge branch 'for-backlight-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/backlight.git backlight/for-backlight-next
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/leds/backlight/maxim,max25014.yaml    |  83 +++++
+ MAINTAINERS                                        |   7 +
+ drivers/video/backlight/Kconfig                    |   9 +-
+ drivers/video/backlight/Makefile                   |   1 +
+ drivers/video/backlight/ktd2801-backlight.c        |   1 +
+ drivers/video/backlight/max25014.c                 | 377 +++++++++++++++++++++
+ 6 files changed, 477 insertions(+), 1 deletion(-)
+ create mode 100644 Documentation/devicetree/bindings/leds/backlight/maxim,max25014.yaml
+ create mode 100644 drivers/video/backlight/max25014.c
+Merging battery/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-power-supply.git battery/for-next
+Already up to date.
+Merging regulator/for-next (0e8ef95038ef2 Merge remote-tracking branch 'regulator/for-7.2' into regulator-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/regulator.git regulator/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/regulator/mt6359-regulator.yaml       |  70 +---
+ .../bindings/regulator/mt6360-regulator.yaml       |   4 +
+ .../bindings/regulator/qcom,rpmh-regulator.yaml    |   3 +
+ .../bindings/regulator/qcom,smd-rpm-regulator.yaml |   4 +
+ drivers/regulator/mt6359-regulator.c               | 405 ++++++++++++++-------
+ drivers/regulator/palmas-regulator.c               |   2 +-
+ drivers/regulator/qcom-rpmh-regulator.c            |  19 +
+ drivers/regulator/qcom_smd-regulator.c             |  33 ++
+ include/linux/regulator/mt6359-regulator.h         |  10 +-
+ 9 files changed, 362 insertions(+), 188 deletions(-)
+Merging security/next (1abe66291664c Automated merge of 'dev' into 'next')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/lsm.git security/next
+Auto-merging MAINTAINERS
+Auto-merging fs/nfs/nfs4proc.c
+Auto-merging security/selinux/hooks.c
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/LSM/Hornet.rst     | 323 +++++++++++++++++++++
+ Documentation/admin-guide/LSM/index.rst      |   1 +
+ Documentation/admin-guide/LSM/ipe.rst        | 162 ++++++++++-
+ Documentation/security/ipe.rst               |  68 +++++
+ MAINTAINERS                                  |   9 +
+ certs/system_keyring.c                       |   1 +
+ crypto/asymmetric_keys/Makefile              |   4 +-
+ crypto/asymmetric_keys/pkcs7_aa.asn1         |  18 ++
+ crypto/asymmetric_keys/pkcs7_key_type.c      |  44 ++-
+ crypto/asymmetric_keys/pkcs7_parser.c        |  81 ++++++
+ crypto/asymmetric_keys/pkcs7_parser.h        |   1 +
+ crypto/asymmetric_keys/pkcs7_trust.c         |   1 +
+ fs/nfs/nfs4proc.c                            |   7 +-
+ fs/xattr.c                                   |  11 +-
+ include/crypto/pkcs7.h                       |   4 +
+ include/linux/lsm_hook_defs.h                |   9 +-
+ include/linux/oid_registry.h                 |   3 +
+ include/linux/security.h                     |  33 ++-
+ include/uapi/linux/lsm.h                     |   1 +
+ scripts/Makefile                             |   1 +
+ scripts/hornet/.gitignore                    |   1 +
+ scripts/hornet/Makefile                      |   5 +
+ scripts/hornet/extract-insn.sh               |  27 ++
+ scripts/hornet/extract-map.sh                |  27 ++
+ scripts/hornet/extract-skel.sh               |  27 ++
+ scripts/hornet/gen_sig.c                     | 402 +++++++++++++++++++++++++++
+ scripts/hornet/write-sig.sh                  |  27 ++
+ security/Kconfig                             |   3 +-
+ security/Makefile                            |   1 +
+ security/hornet/Kconfig                      |  14 +
+ security/hornet/Makefile                     |   7 +
+ security/hornet/hornet.asn1                  |  12 +
+ security/hornet/hornet_lsm.c                 | 352 +++++++++++++++++++++++
+ security/ipe/Kconfig                         |  15 +
+ security/ipe/audit.c                         |  15 +
+ security/ipe/eval.c                          |  93 +++++++
+ security/ipe/eval.h                          |  11 +
+ security/ipe/hooks.c                         |  63 +++++
+ security/ipe/hooks.h                         |  15 +
+ security/ipe/ipe.c                           |  14 +
+ security/ipe/ipe.h                           |   3 +
+ security/ipe/policy.h                        |  14 +
+ security/ipe/policy_parser.c                 |  27 ++
+ security/security.c                          |  91 +++++-
+ security/selinux/hooks.c                     |  10 +-
+ security/smack/smack_lsm.c                   |  13 +-
+ tools/testing/selftests/Makefile             |   1 +
+ tools/testing/selftests/hornet/Makefile      |  63 +++++
+ tools/testing/selftests/hornet/loader.c      |  21 ++
+ tools/testing/selftests/hornet/trivial.bpf.c |  33 +++
+ 50 files changed, 2145 insertions(+), 44 deletions(-)
+ create mode 100644 Documentation/admin-guide/LSM/Hornet.rst
+ create mode 100644 crypto/asymmetric_keys/pkcs7_aa.asn1
+ create mode 100644 scripts/hornet/.gitignore
+ create mode 100644 scripts/hornet/Makefile
+ create mode 100755 scripts/hornet/extract-insn.sh
+ create mode 100755 scripts/hornet/extract-map.sh
+ create mode 100755 scripts/hornet/extract-skel.sh
+ create mode 100644 scripts/hornet/gen_sig.c
+ create mode 100755 scripts/hornet/write-sig.sh
+ create mode 100644 security/hornet/Kconfig
+ create mode 100644 security/hornet/Makefile
+ create mode 100644 security/hornet/hornet.asn1
+ create mode 100644 security/hornet/hornet_lsm.c
+ create mode 100644 tools/testing/selftests/hornet/Makefile
+ create mode 100644 tools/testing/selftests/hornet/loader.c
+ create mode 100644 tools/testing/selftests/hornet/trivial.bpf.c
+Merging apparmor/apparmor-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'apparmor-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jj/linux-apparmor apparmor/apparmor-next
+Already up to date.
+Merging integrity/next-integrity (11143a19f5b8d evm: terminate and bound the evm_xattrs read buffer)
+$ git merge -m Merge branch 'next-integrity' of https://git.kernel.org/pub/scm/linux/kernel/git/zohar/linux-integrity integrity/next-integrity
+Merge made by the 'ort' strategy.
+ security/integrity/digsig_asymmetric.c | 152 ++++++++++++++++++++++++++++-----
+ security/integrity/evm/evm_secfs.c     |  16 ++--
+ security/integrity/ima/ima_appraise.c  |  10 ++-
+ security/integrity/ima/ima_policy.c    |   3 +-
+ 4 files changed, 152 insertions(+), 29 deletions(-)
+Merging selinux/next (ad5a51dd2f48e Automated merge of 'dev' into 'next')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/selinux.git selinux/next
+Merge made by the 'ort' strategy.
+ security/selinux/include/security.h |   1 +
+ security/selinux/selinuxfs.c        |  13 +-
+ security/selinux/ss/avtab.c         |  49 ++++-
+ security/selinux/ss/avtab.h         |  13 ++
+ security/selinux/ss/conditional.c   |  39 ++--
+ security/selinux/ss/constraint.h    |   1 +
+ security/selinux/ss/ebitmap.c       |  27 +++
+ security/selinux/ss/ebitmap.h       |   1 +
+ security/selinux/ss/hashtab.h       |   4 +-
+ security/selinux/ss/mls.c           |  66 ++++---
+ security/selinux/ss/mls.h           |   6 +-
+ security/selinux/ss/policydb.c      | 358 +++++++++++++++++++++++++++++-------
+ security/selinux/ss/policydb.h      |  56 ++++--
+ security/selinux/ss/services.c      |  13 +-
+ security/selinux/ss/symtab.c        |   2 +-
+ security/selinux/ss/symtab.h        |   2 +-
+ 16 files changed, 504 insertions(+), 147 deletions(-)
+Merging smack/next (b78fede1c69a0 smack: simplify write handlers of sysfs entries)
+$ git merge -m Merge branch 'next' of https://github.com/cschaufler/smack-next smack/next
+Auto-merging security/smack/smack_lsm.c
+Merge made by the 'ort' strategy.
+ security/smack/smack_lsm.c |  2 +-
+ security/smack/smackfs.c   | 81 +++++++++++++---------------------------------
+ 2 files changed, 23 insertions(+), 60 deletions(-)
+Merging tomoyo/master (5e6978414efa5 loop: Fix NULL pointer dereference in lo_rw_aio())
+$ git merge -m Merge branch 'master' of git://git.code.sf.net/p/tomoyo/tomoyo.git tomoyo/master
+Auto-merging include/linux/netdevice.h
+Auto-merging include/net/sock.h
+Auto-merging lib/Kconfig.debug
+Auto-merging net/core/dev.c
+Auto-merging net/socket.c
+Merge made by the 'ort' strategy.
+ drivers/block/loop.c      |  16 ++++
+ include/linux/netdevice.h |  15 ++++
+ include/net/dst.h         |  22 +++++-
+ include/net/sock.h        |   8 +-
+ kernel/locking/lockdep.c  |   2 +
+ kernel/softirq.c          |   4 +
+ kernel/workqueue.c        |   4 +
+ lib/Kconfig.debug         |   7 ++
+ net/bridge/br_nf_core.c   |   1 +
+ net/core/dev.c            | 188 ++++++++++++++++++++++++++++++++++++++++++++++
+ net/core/dst.c            |  85 +++++++++++++++++++++
+ net/core/lock_debug.c     |   1 +
+ net/socket.c              |  32 ++++++--
+ 13 files changed, 374 insertions(+), 11 deletions(-)
+Merging tpmdd-tpm/for-next-tpm (fd1ac8d2bae68 tpm_crb: Check ACPI_COMPANION() against NULL during probe)
+$ git merge -m Merge branch 'for-next-tpm' of https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git tpmdd-tpm/for-next-tpm
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt | 16 +++++++++
+ drivers/char/tpm/tpm2-cmd.c                     |  6 ++--
+ drivers/char/tpm/tpm2-sessions.c                | 11 ++++---
+ drivers/char/tpm/tpm_crb.c                      |  6 +++-
+ drivers/char/tpm/tpm_svsm.c                     |  2 +-
+ drivers/char/tpm/tpm_tis_core.c                 | 35 ++++++++++++--------
+ include/keys/trusted-type.h                     | 21 +++++++-----
+ security/keys/Kconfig                           |  1 -
+ security/keys/trusted-keys/Kconfig              | 23 +++++++++++++
+ security/keys/trusted-keys/trusted_caam.c       |  7 ++--
+ security/keys/trusted-keys/trusted_core.c       |  6 ++++
+ security/keys/trusted-keys/trusted_tpm1.c       | 44 ++++++++++++++-----------
+ security/keys/user_defined.c                    |  2 +-
+ 13 files changed, 125 insertions(+), 55 deletions(-)
+Merging tpmdd-keys/for-next-keys (26cb5b4b32ff1 keys: use kmalloc_flex in user_preparse)
+$ git merge -m Merge branch 'for-next-keys' of https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git tpmdd-keys/for-next-keys
+Already up to date.
+Merging watchdog/watchdog-next (c574bdb524095 watchdog: ziirave_wdt: Use named initializers for struct i2c_device_id)
+$ git merge -m Merge branch 'watchdog-next' of https://git.kernel.org/pub/scm/linux/kernel/git/groeck/linux-staging.git watchdog/watchdog-next
+Auto-merging Documentation/watchdog/watchdog-api.rst
+Auto-merging Documentation/watchdog/watchdog-kernel-api.rst
+Auto-merging Documentation/watchdog/watchdog-parameters.rst
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ .../bindings/watchdog/andestech,ae350-wdt.yaml     |  53 ++
+ .../devicetree/bindings/watchdog/apple,wdt.yaml    |   4 +-
+ .../bindings/watchdog/kontron,sl28cpld-wdt.yaml    |   7 +-
+ .../devicetree/bindings/watchdog/qcom-wdt.yaml     |  10 +
+ .../bindings/watchdog/renesas,r9a09g057-wdt.yaml   |   8 +-
+ .../bindings/watchdog/renesas,rzg2l-wdt.yaml       |   1 +
+ .../bindings/watchdog/renesas,rzn1-wdt.yaml        |   1 -
+ .../devicetree/bindings/watchdog/snps,dw-wdt.yaml  |   2 +
+ .../bindings/watchdog/technologic,ts4800-wdt.yaml  |  40 ++
+ .../devicetree/bindings/watchdog/ti,rti-wdt.yaml   |   1 +
+ .../devicetree/bindings/watchdog/ts4800-wdt.txt    |  25 -
+ .../watchdog/convert_drivers_to_kernel_api.rst     |  28 +-
+ Documentation/watchdog/{wdt.rst => ics-wdt.rst}    |   8 +-
+ Documentation/watchdog/index.rst                   |  20 +-
+ Documentation/watchdog/watchdog-api.rst            |  15 +-
+ Documentation/watchdog/watchdog-kernel-api.rst     |   2 +-
+ Documentation/watchdog/watchdog-parameters.rst     |  19 -
+ MAINTAINERS                                        |   7 +
+ drivers/watchdog/Kconfig                           |  68 ++-
+ drivers/watchdog/Makefile                          |   4 +-
+ drivers/watchdog/apple_wdt.c                       |   1 +
+ drivers/watchdog/at91sam9_wdt.h                    |   6 +-
+ drivers/watchdog/atcwdt200_wdt.c                   | 580 +++++++++++++++++++++
+ drivers/watchdog/bcm2835_wdt.c                     |  28 +-
+ drivers/watchdog/gunyah_wdt.c                      | 261 ++++++++++
+ drivers/watchdog/hpwdt.c                           |  22 +-
+ drivers/watchdog/imx7ulp_wdt.c                     |  10 +-
+ drivers/watchdog/lenovo_se10_wdt.c                 |  82 ++-
+ drivers/watchdog/machzwd.c                         | 452 ----------------
+ drivers/watchdog/menz69_wdt.c                      |   1 -
+ drivers/watchdog/qcom-wdt.c                        |  42 +-
+ drivers/watchdog/realtek_otto_wdt.c                |  20 +-
+ drivers/watchdog/rti_wdt.c                         |  39 +-
+ drivers/watchdog/rzn1_wdt.c                        |  35 +-
+ drivers/watchdog/sama5d4_wdt.c                     |  48 +-
+ drivers/watchdog/sc520_wdt.c                       | 430 ---------------
+ drivers/watchdog/smsc37b787_wdt.c                  |   2 +-
+ drivers/watchdog/sp5100_tco.c                      |   3 +-
+ drivers/watchdog/sprd_wdt.c                        |   5 +-
+ drivers/watchdog/ziirave_wdt.c                     |   2 +-
+ 40 files changed, 1269 insertions(+), 1123 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/watchdog/andestech,ae350-wdt.yaml
+ create mode 100644 Documentation/devicetree/bindings/watchdog/technologic,ts4800-wdt.yaml
+ delete mode 100644 Documentation/devicetree/bindings/watchdog/ts4800-wdt.txt
+ rename Documentation/watchdog/{wdt.rst => ics-wdt.rst} (91%)
+ create mode 100644 drivers/watchdog/atcwdt200_wdt.c
+ create mode 100644 drivers/watchdog/gunyah_wdt.c
+ delete mode 100644 drivers/watchdog/machzwd.c
+ delete mode 100644 drivers/watchdog/sc520_wdt.c
+Merging iommu/next (f4f5d162619bd Merge branches 'fixes', 'verisilicon', 'riscv', 'amd/amd-vi' and 'core' into next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git iommu/next
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
+Merge made by the 'ort' strategy.
+ .../bindings/iommu/verisilicon,iommu.yaml          |  71 ++
+ MAINTAINERS                                        |   8 +
+ arch/arm64/boot/dts/rockchip/rk3588-base.dtsi      |  11 +
+ drivers/iommu/Kconfig                              |  11 +
+ drivers/iommu/Makefile                             |   1 +
+ drivers/iommu/amd/amd_iommu.h                      |   2 +-
+ drivers/iommu/amd/amd_iommu_types.h                |  27 +-
+ drivers/iommu/amd/init.c                           |  56 +-
+ drivers/iommu/amd/iommu.c                          |   6 +-
+ drivers/iommu/generic_pt/fmt/amdv1.h               |   1 +
+ drivers/iommu/generic_pt/fmt/iommu_riscv64.c       |   4 +-
+ drivers/iommu/generic_pt/fmt/riscv.h               |  11 +-
+ drivers/iommu/generic_pt/iommu_pt.h                | 110 ++-
+ drivers/iommu/generic_pt/kunit_generic_pt.h        |  10 +-
+ drivers/iommu/generic_pt/kunit_iommu_pt.h          |   5 +-
+ drivers/iommu/intel/cache.c                        |  49 +-
+ drivers/iommu/io-pgtable-arm.c                     |  88 ++-
+ drivers/iommu/riscv/iommu-bits.h                   | 123 ++--
+ drivers/iommu/riscv/iommu.c                        | 162 ++++-
+ drivers/iommu/vsi-iommu.c                          | 796 +++++++++++++++++++++
+ include/linux/generic_pt/common.h                  |   9 +
+ include/linux/iommu.h                              |  42 +-
+ 22 files changed, 1352 insertions(+), 251 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/iommu/verisilicon,iommu.yaml
+ create mode 100644 drivers/iommu/vsi-iommu.c
+Merging audit/next (5200f5f493f79 Linux 7.1-rc4)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git audit/next
+Already up to date.
+Merging devicetree/for-next (98e2a77b5d2c1 dt-bindings: Consolidate "sram" property definition)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/robh/linux.git devicetree/for-next
+Auto-merging Documentation/devicetree/bindings/sram/sram.yaml
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ Documentation/devicetree/bindings/arm/cpus.yaml    |  1 +
+ .../devicetree/bindings/arm/st,nomadik.yaml        | 23 ++++++++++++
+ .../devicetree/bindings/arm/ste-nomadik.txt        | 38 -------------------
+ .../bindings/display/bridge/solomon,ssd2825.yaml   |  3 +-
+ .../display/imx/fsl,imx-parallel-display.yaml      | 11 ++++++
+ .../imx/fsl,imx8qxp-dc-command-sequencer.yaml      |  2 +-
+ .../bindings/display/mediatek/mediatek,ovl-2l.yaml |  5 +--
+ .../bindings/display/mediatek/mediatek,ovl.yaml    |  5 +--
+ .../bindings/display/mediatek/mediatek,rdma.yaml   |  4 +-
+ .../bindings/display/mediatek/mediatek,wdma.yaml   |  4 +-
+ .../devicetree/bindings/display/msm/gpu.yaml       |  6 +--
+ .../display/rockchip/rockchip,rk3399-cdn-dp.yaml   |  2 +
+ .../devicetree/bindings/dma/stericsson,dma40.yaml  |  8 +---
+ .../devicetree/bindings/i2c/i2c-demux-pinctrl.yaml |  1 +
+ .../bindings/i2c/opencores,i2c-ocores.yaml         |  8 ++--
+ .../bindings/iio/adc/st,spear600-adc.yaml          |  2 +-
+ .../devicetree/bindings/input/atmel,maxtouch.yaml  |  3 +-
+ .../devicetree/bindings/input/imx-keypad.yaml      | 32 ++++++++--------
+ .../bindings/input/microchip,cap11xx.yaml          | 12 +++---
+ .../bindings/input/touchscreen/ti,ads7843.yaml     |  4 +-
+ .../bindings/interrupt-controller/qcom,pdc.yaml    |  1 +
+ .../devicetree/bindings/leds/leds-lp55xx.yaml      |  2 +-
+ .../devicetree/bindings/media/cnm,wave521c.yaml    |  2 +-
+ .../bindings/media/mediatek,vcodec-encoder.yaml    |  2 +-
+ .../media/mediatek,vcodec-subdev-decoder.yaml      |  2 +-
+ .../devicetree/bindings/media/nxp,imx8-jpeg.yaml   |  6 +--
+ .../devicetree/bindings/media/rockchip,vdec.yaml   |  5 +--
+ .../devicetree/bindings/media/st,stm32-dcmi.yaml   |  6 +--
+ .../memory-controllers/nvidia,tegra20-mc.yaml      |  4 +-
+ .../memory-controllers/samsung,exynos5422-dmc.yaml |  2 +-
+ .../devicetree/bindings/mfd/rockchip,rk816.yaml    |  3 +-
+ .../devicetree/bindings/misc/fsl,qoriq-mc.yaml     |  2 +-
+ .../devicetree/bindings/misc/qcom,fastrpc.yaml     |  1 +
+ .../mmc/hisilicon,hi3798cv200-dw-mshc.yaml         |  7 ++--
+ .../devicetree/bindings/net/mediatek,net.yaml      |  3 +-
+ .../devicetree/bindings/net/qcom,bam-dmux.yaml     | 12 ++++++
+ .../devicetree/bindings/net/qcom,ipa.yaml          | 12 ++++++
+ .../devicetree/bindings/net/ti,icssg-prueth.yaml   |  2 +-
+ .../devicetree/bindings/net/ti,icssm-prueth.yaml   |  2 +-
+ .../bindings/net/wireless/qcom,ath10k.yaml         |  8 +++-
+ .../bindings/net/wireless/qcom,ath11k.yaml         |  8 +++-
+ .../bindings/net/wireless/qcom,ipq5332-wifi.yaml   | 18 +++++++++
+ .../bindings/pci/mediatek-pcie-mt7623.yaml         |  4 +-
+ .../bindings/pci/samsung,exynos-pcie.yaml          |  2 +-
+ .../devicetree/bindings/pci/toshiba,tc9563.yaml    |  5 ++-
+ .../remoteproc/amlogic,meson-mx-ao-arc.yaml        |  7 +---
+ .../bindings/remoteproc/qcom,msm8916-mss-pil.yaml  |  6 +++
+ .../bindings/remoteproc/qcom,msm8996-mss-pil.yaml  |  7 ++++
+ .../bindings/remoteproc/qcom,pas-common.yaml       |  6 +++
+ .../bindings/remoteproc/qcom,qcs404-cdsp-pil.yaml  |  6 +++
+ .../bindings/remoteproc/qcom,sc7180-mss-pil.yaml   |  6 +++
+ .../bindings/remoteproc/qcom,sc7280-adsp-pil.yaml  |  6 +++
+ .../bindings/remoteproc/qcom,sc7280-mss-pil.yaml   |  6 +++
+ .../bindings/remoteproc/qcom,sc7280-wpss-pil.yaml  |  6 +++
+ .../bindings/remoteproc/qcom,sdm845-adsp-pil.yaml  |  6 +++
+ .../bindings/remoteproc/qcom,wcnss-pil.yaml        |  6 +++
+ .../bindings/remoteproc/ti,k3-dsp-rproc.yaml       |  8 ----
+ .../bindings/remoteproc/ti,k3-r5f-rproc.yaml       |  8 ----
+ .../bindings/remoteproc/xlnx,zynqmp-r5fss.yaml     |  9 +----
+ .../bindings/sound/davinci-mcasp-audio.yaml        |  4 +-
+ .../devicetree/bindings/sound/samsung,tm2.yaml     |  8 +++-
+ .../devicetree/bindings/sound/simple-card.yaml     |  6 +--
+ .../devicetree/bindings/spi/st,stm32-spi.yaml      | 10 ++---
+ .../devicetree/bindings/spi/st,stm32mp25-ospi.yaml |  5 ++-
+ .../devicetree/bindings/sram/sram-consumer.yaml    | 26 +++++++++++++
+ Documentation/devicetree/bindings/sram/sram.yaml   |  1 +
+ .../devicetree/bindings/submitting-patches.rst     |  7 ++--
+ .../bindings/usb/chipidea,usb2-common.yaml         |  2 +
+ .../devicetree/bindings/usb/ci-hdrc-usb2.yaml      |  7 ++--
+ .../devicetree/bindings/usb/richtek,rt1711h.yaml   |  3 +-
+ .../devicetree/bindings/usb/ti,j721e-usb.yaml      |  8 ++--
+ .../devicetree/bindings/writing-bindings.rst       | 12 +++++-
+ MAINTAINERS                                        |  2 +-
+ drivers/of/base.c                                  | 43 +++++++++++++---------
+ drivers/of/dynamic.c                               | 20 ++++++++--
+ drivers/of/fdt.c                                   |  2 -
+ drivers/of/property.c                              | 27 +++++++++++---
+ drivers/of/unittest.c                              | 32 ++++++++++++++++
+ 78 files changed, 398 insertions(+), 215 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/arm/st,nomadik.yaml
+ delete mode 100644 Documentation/devicetree/bindings/arm/ste-nomadik.txt
+ create mode 100644 Documentation/devicetree/bindings/sram/sram-consumer.yaml
+Merging dt-krzk/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-dt.git dt-krzk/for-next
+Already up to date.
+Merging mailbox/for-next (5e4907c4908bf mailbox: add list of used channels to debugfs)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jassibrar/mailbox.git mailbox/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/mailbox/qcom,apcs-kpss-global.yaml    |   1 +
+ .../bindings/mailbox/qcom,cpucp-mbox.yaml          |   2 +
+ .../devicetree/bindings/mailbox/qcom-ipcc.yaml     |   1 +
+ drivers/mailbox/Kconfig                            |   4 +-
+ drivers/mailbox/exynos-mailbox.c                   |   7 --
+ drivers/mailbox/mailbox-mpfs.c                     |   2 +-
+ drivers/mailbox/mailbox.c                          | 101 ++++++++++++++++++---
+ drivers/mailbox/mtk-adsp-mailbox.c                 |   9 +-
+ drivers/mailbox/qcom-cpucp-mbox.c                  |  35 ++++++-
+ include/linux/mailbox_controller.h                 |   2 +
+ 10 files changed, 136 insertions(+), 28 deletions(-)
+Merging spi/for-next (0c5b5c40dc310 spi: cadence-xspi: Add COMPILE_TEST support)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git spi/for-next
+Auto-merging drivers/spi/spi-stm32-ospi.c
+Auto-merging drivers/spi/spi-stm32-qspi.c
+Merge made by the 'ort' strategy.
+ .../bindings/spi/qcom,spi-qcom-qspi.yaml           |  21 +-
+ .../devicetree/bindings/spi/spacemit,k1-spi.yaml   |  84 +++
+ drivers/spi/Kconfig                                |  12 +-
+ drivers/spi/Makefile                               |   1 +
+ drivers/spi/spi-altera-platform.c                  |  27 +-
+ drivers/spi/spi-amlogic-spifc-a1.c                 |   5 +-
+ drivers/spi/spi-amlogic-spisg.c                    |  16 +-
+ drivers/spi/spi-armada-3700.c                      |  44 +-
+ drivers/spi/spi-at91-usart.c                       |  63 +-
+ drivers/spi/spi-atcspi200.c                        |  28 +-
+ drivers/spi/spi-atmel.c                            |  34 +-
+ drivers/spi/spi-bcm63xx-hsspi.c                    |  17 +-
+ drivers/spi/spi-bcm63xx.c                          |  23 +-
+ drivers/spi/spi-cadence-quadspi.c                  |  52 +-
+ drivers/spi/spi-cadence-xspi.c                     |   7 +-
+ drivers/spi/spi-cadence.c                          |  40 +-
+ drivers/spi/spi-cavium-octeon.c                    |  17 +-
+ drivers/spi/spi-cavium-thunderx.c                  |  32 +-
+ drivers/spi/spi-clps711x.c                         |  31 +-
+ drivers/spi/spi-coldfire-qspi.c                    |  28 +-
+ drivers/spi/spi-dln2.c                             |  26 +-
+ drivers/spi/spi-dw-pci.c                           |  14 +-
+ drivers/spi/spi-ep93xx.c                           |  22 +-
+ drivers/spi/spi-falcon.c                           |   8 +-
+ drivers/spi/spi-fsi.c                              |   8 +-
+ drivers/spi/spi-fsl-espi.c                         |  20 +-
+ drivers/spi/spi-fsl-spi.c                          |  14 +-
+ drivers/spi/spi-geni-qcom.c                        |  13 +
+ drivers/spi/spi-hisi-kunpeng.c                     |   6 +-
+ drivers/spi/spi-hisi-sfc-v3xx.c                    |  20 +-
+ drivers/spi/spi-img-spfi.c                         |  40 +-
+ drivers/spi/spi-intel-pci.c                        |  66 +-
+ drivers/spi/spi-jcore.c                            |  23 +-
+ drivers/spi/spi-lantiq-ssc.c                       |  28 +-
+ drivers/spi/spi-lp8841-rtc.c                       |  16 +-
+ drivers/spi/spi-meson-spicc.c                      |  48 +-
+ drivers/spi/spi-meson-spifc.c                      |  22 +-
+ drivers/spi/spi-microchip-core-qspi.c              |   3 +-
+ drivers/spi/spi-mpc52xx.c                          |  26 +-
+ drivers/spi/spi-mux.c                              |  19 +-
+ drivers/spi/spi-mxs.c                              |  14 +-
+ drivers/spi/spi-npcm-pspi.c                        |  19 +-
+ drivers/spi/spi-nxp-xspi.c                         |  12 +-
+ drivers/spi/spi-omap2-mcspi.c                      |  42 +-
+ drivers/spi/spi-orion.c                            |  26 +-
+ drivers/spi/spi-pci1xxxx.c                         |  42 +-
+ drivers/spi/spi-pic32-sqi.c                        |  28 +-
+ drivers/spi/spi-pic32.c                            |  11 +-
+ drivers/spi/spi-pl022.c                            |  11 +-
+ drivers/spi/spi-qcom-qspi.c                        |  80 ++-
+ drivers/spi/spi-qup.c                              |  15 +-
+ drivers/spi/spi-rspi.c                             |  40 +-
+ drivers/spi/spi-sh-hspi.c                          |  11 +-
+ drivers/spi/spi-sh-msiof.c                         |  28 +-
+ drivers/spi/spi-sifive.c                           |  39 +-
+ drivers/spi/spi-slave-mt27xx.c                     |  33 +-
+ drivers/spi/spi-sn-f-ospi.c                        |   5 +-
+ drivers/spi/spi-spacemit-k1.c                      | 789 +++++++++++++++++++++
+ drivers/spi/spi-sprd.c                             |  20 +-
+ drivers/spi/spi-st-ssc4.c                          |  14 +-
+ drivers/spi/spi-stm32-ospi.c                       |   7 +-
+ drivers/spi/spi-stm32-qspi.c                       |   5 +-
+ drivers/spi/spi-sun4i.c                            |  35 +-
+ drivers/spi/spi-sun6i.c                            |  38 +-
+ drivers/spi/spi-sunplus-sp7021.c                   |   3 +-
+ drivers/spi/spi-synquacer.c                        |  24 +-
+ drivers/spi/spi-tegra114.c                         |  34 +-
+ drivers/spi/spi-tegra20-sflash.c                   |  24 +-
+ drivers/spi/spi-ti-qspi.c                          |  37 +-
+ drivers/spi/spi-topcliff-pch.c                     |   8 +-
+ drivers/spi/spi-uniphier.c                         |  46 +-
+ drivers/spi/spi-xlp.c                              |   3 +-
+ drivers/spi/spi-zynq-qspi.c                        |  40 +-
+ drivers/spi/spi.c                                  |   4 +-
+ include/linux/platform_data/spi-s3c64xx.h          |   3 +-
+ include/trace/events/qcom_geni_spi.h               | 103 +++
+ 76 files changed, 1634 insertions(+), 1083 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/spi/spacemit,k1-spi.yaml
+ create mode 100644 drivers/spi/spi-spacemit-k1.c
+ create mode 100644 include/trace/events/qcom_geni_spi.h
+Merging tip/master (564844fbd167a Merge branch into tip/master: 'x86/tdx')
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git tip/master
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Auto-merging arch/x86/kernel/smpboot.c
+Auto-merging drivers/cpufreq/Kconfig.x86
+Auto-merging drivers/crypto/ccp/sev-dev.c
+Auto-merging drivers/irqchip/Kconfig
+Auto-merging drivers/irqchip/Makefile
+Auto-merging drivers/watchdog/Kconfig
+Auto-merging include/linux/sched.h
+Auto-merging init/Kconfig
+Auto-merging kernel/fork.c
+Auto-merging kernel/workqueue.c
+Auto-merging lib/Kconfig.debug
+Merge made by the 'ort' strategy.
+ Documentation/admin-guide/kernel-parameters.txt    |   16 +-
+ Documentation/arch/x86/tdx.rst                     |    7 -
+ Documentation/dev-tools/context-analysis.rst       |    2 +-
+ .../amlogic,meson-gpio-intc.yaml                   |   21 +-
+ .../interrupt-controller/aspeed,ast2700-intc.yaml  |   90 -
+ .../aspeed,ast2700-interrupt.yaml                  |  188 ++
+ .../interrupt-controller/econet,en751221-intc.yaml |   20 +
+ ...,jh8100-intc.yaml => starfive,jhb100-intc.yaml} |   20 +-
+ .../devicetree/bindings/timer/fsl,imxgpt.yaml      |    1 +
+ Documentation/filesystems/resctrl.rst              |   22 +-
+ MAINTAINERS                                        |   13 +-
+ arch/arm/include/asm/arch_gicv3.h                  |    4 +-
+ arch/x86/Kconfig                                   |   62 +-
+ arch/x86/Kconfig.cpu                               |   54 +-
+ arch/x86/Kconfig.cpufeatures                       |    1 -
+ arch/x86/Makefile                                  |    1 -
+ arch/x86/Makefile_32.cpu                           |    6 -
+ arch/x86/boot/compressed/kaslr.c                   |    3 +-
+ arch/x86/boot/compressed/pgtable_64.c              |    1 +
+ arch/x86/boot/startup/sme.c                        |    1 +
+ arch/x86/boot/string.c                             |   30 +-
+ arch/x86/boot/string.h                             |    1 -
+ arch/x86/coco/sev/core.c                           |    8 +-
+ arch/x86/coco/sev/internal.h                       |    3 +
+ arch/x86/coco/sev/noinstr.c                        |    4 +-
+ arch/x86/coco/sev/svsm.c                           |   10 +-
+ arch/x86/coco/sev/vc-handle.c                      |   12 +-
+ arch/x86/coco/tdx/tdx.c                            |    1 +
+ arch/x86/entry/entry_64.S                          |    4 +-
+ arch/x86/entry/entry_fred.c                        |   10 -
+ arch/x86/events/amd/core.c                         |    2 +
+ arch/x86/events/amd/ibs.c                          |    1 +
+ arch/x86/events/amd/lbr.c                          |    2 +
+ arch/x86/events/amd/power.c                        |    3 +
+ arch/x86/events/amd/uncore.c                       |    1 +
+ arch/x86/events/intel/core.c                       |  490 +++-
+ arch/x86/events/intel/ds.c                         |   23 +-
+ arch/x86/events/intel/lbr.c                        |    1 +
+ arch/x86/events/perf_event.h                       |    8 +-
+ arch/x86/events/zhaoxin/core.c                     |    1 +
+ arch/x86/include/asm/acrn.h                        |    2 +
+ arch/x86/include/asm/bug.h                         |    4 +-
+ arch/x86/include/asm/cpuid/api.h                   |  253 ++-
+ arch/x86/include/asm/cpuid/leaf_types.h            | 2350 ++++++++++++++++++++
+ arch/x86/include/asm/cpuid/types.h                 |   98 +
+ arch/x86/include/asm/device.h                      |   11 -
+ arch/x86/include/asm/fpu/api.h                     |    6 -
+ arch/x86/include/asm/idtentry.h                    |   29 +-
+ arch/x86/include/asm/microcode.h                   |    1 +
+ arch/x86/include/asm/processor.h                   |    8 +-
+ arch/x86/include/asm/purgatory.h                   |    2 +-
+ arch/x86/include/asm/sev.h                         |    4 +-
+ arch/x86/include/asm/shared/tdx.h                  |    1 +
+ .../{kvm/vmx => include/asm/shared}/tdx_errno.h    |    8 +-
+ arch/x86/include/asm/tdx.h                         |    9 +-
+ arch/x86/include/asm/vermagic.h                    |    6 -
+ arch/x86/include/asm/xen/hypervisor.h              |    1 +
+ arch/x86/kernel/apic/apic.c                        |    1 +
+ arch/x86/kernel/cpu/Makefile                       |    1 +
+ arch/x86/kernel/cpu/amd.c                          |    1 +
+ arch/x86/kernel/cpu/centaur.c                      |    1 +
+ arch/x86/kernel/cpu/common.c                       |   12 +-
+ arch/x86/kernel/cpu/cpuid_parser.c                 |  182 ++
+ arch/x86/kernel/cpu/cpuid_parser.h                 |  120 +
+ arch/x86/kernel/cpu/hygon.c                        |    1 +
+ arch/x86/kernel/cpu/mce/core.c                     |    1 +
+ arch/x86/kernel/cpu/mce/inject.c                   |    1 +
+ arch/x86/kernel/cpu/microcode/amd.c                |    1 +
+ arch/x86/kernel/cpu/microcode/core.c               |    3 +-
+ arch/x86/kernel/cpu/microcode/intel-ucode-defs.h   |  479 ++--
+ arch/x86/kernel/cpu/microcode/intel.c              |    1 +
+ arch/x86/kernel/cpu/mshyperv.c                     |    1 +
+ arch/x86/kernel/cpu/resctrl/core.c                 |    1 +
+ arch/x86/kernel/cpu/resctrl/monitor.c              |    2 +
+ arch/x86/kernel/cpu/scattered.c                    |    1 +
+ arch/x86/kernel/cpu/sgx/driver.c                   |    3 +
+ arch/x86/kernel/cpu/sgx/main.c                     |    3 +
+ arch/x86/kernel/cpu/topology_amd.c                 |    3 +-
+ arch/x86/kernel/cpu/topology_common.c              |    1 +
+ arch/x86/kernel/cpu/topology_ext.c                 |    3 +-
+ arch/x86/kernel/cpu/transmeta.c                    |    3 +
+ arch/x86/kernel/cpu/umc.c                          |   26 -
+ arch/x86/kernel/cpu/vmware.c                       |    1 +
+ arch/x86/kernel/cpu/zhaoxin.c                      |    1 +
+ arch/x86/kernel/cpuid.c                            |    6 +-
+ arch/x86/kernel/crash.c                            |    2 +
+ arch/x86/kernel/fpu/core.c                         |    5 -
+ arch/x86/kernel/fpu/init.c                         |    9 +-
+ arch/x86/kernel/jailhouse.c                        |    1 +
+ arch/x86/kernel/kvm.c                              |    1 +
+ arch/x86/kernel/machine_kexec_64.c                 |   16 -
+ arch/x86/kernel/paravirt.c                         |    1 +
+ arch/x86/kernel/smpboot.c                          |   20 +
+ arch/x86/kernel/traps.c                            |   21 -
+ arch/x86/kvm/mmu/mmu.c                             |    1 +
+ arch/x86/kvm/mmu/spte.c                            |    1 +
+ arch/x86/kvm/reverse_cpuid.h                       |    2 +
+ arch/x86/kvm/svm/sev.c                             |    1 +
+ arch/x86/kvm/svm/svm.c                             |    1 +
+ arch/x86/kvm/vmx/pmu_intel.c                       |    1 +
+ arch/x86/kvm/vmx/sgx.c                             |    1 +
+ arch/x86/kvm/vmx/tdx.c                             |   10 -
+ arch/x86/kvm/vmx/tdx.h                             |    1 -
+ arch/x86/kvm/vmx/vmx.c                             |    1 +
+ arch/x86/math-emu/Makefile                         |   30 -
+ arch/x86/math-emu/README                           |  427 ----
+ arch/x86/math-emu/control_w.h                      |   46 -
+ arch/x86/math-emu/div_Xsig.S                       |  367 ---
+ arch/x86/math-emu/div_small.S                      |   48 -
+ arch/x86/math-emu/errors.c                         |  686 ------
+ arch/x86/math-emu/exception.h                      |   51 -
+ arch/x86/math-emu/fpu_arith.c                      |  153 --
+ arch/x86/math-emu/fpu_asm.h                        |   32 -
+ arch/x86/math-emu/fpu_aux.c                        |  267 ---
+ arch/x86/math-emu/fpu_emu.h                        |  218 --
+ arch/x86/math-emu/fpu_entry.c                      |  718 ------
+ arch/x86/math-emu/fpu_etc.c                        |  136 --
+ arch/x86/math-emu/fpu_proto.h                      |  157 --
+ arch/x86/math-emu/fpu_system.h                     |  130 --
+ arch/x86/math-emu/fpu_tags.c                       |  116 -
+ arch/x86/math-emu/fpu_trig.c                       | 1649 --------------
+ arch/x86/math-emu/get_address.c                    |  401 ----
+ arch/x86/math-emu/load_store.c                     |  322 ---
+ arch/x86/math-emu/mul_Xsig.S                       |  179 --
+ arch/x86/math-emu/poly.h                           |  115 -
+ arch/x86/math-emu/poly_2xm1.c                      |  146 --
+ arch/x86/math-emu/poly_atan.c                      |  209 --
+ arch/x86/math-emu/poly_l2.c                        |  245 --
+ arch/x86/math-emu/poly_sin.c                       |  379 ----
+ arch/x86/math-emu/poly_tan.c                       |  213 --
+ arch/x86/math-emu/polynom_Xsig.S                   |  137 --
+ arch/x86/math-emu/reg_add_sub.c                    |  334 ---
+ arch/x86/math-emu/reg_compare.c                    |  479 ----
+ arch/x86/math-emu/reg_constant.c                   |  123 -
+ arch/x86/math-emu/reg_constant.h                   |   26 -
+ arch/x86/math-emu/reg_convert.c                    |   47 -
+ arch/x86/math-emu/reg_divide.c                     |  183 --
+ arch/x86/math-emu/reg_ld_str.c                     | 1220 ----------
+ arch/x86/math-emu/reg_mul.c                        |  116 -
+ arch/x86/math-emu/reg_norm.S                       |  150 --
+ arch/x86/math-emu/reg_round.S                      |  711 ------
+ arch/x86/math-emu/reg_u_add.S                      |  169 --
+ arch/x86/math-emu/reg_u_div.S                      |  474 ----
+ arch/x86/math-emu/reg_u_mul.S                      |  150 --
+ arch/x86/math-emu/reg_u_sub.S                      |  274 ---
+ arch/x86/math-emu/round_Xsig.S                     |  142 --
+ arch/x86/math-emu/shr_Xsig.S                       |   89 -
+ arch/x86/math-emu/status_w.h                       |   68 -
+ arch/x86/math-emu/version.h                        |   12 -
+ arch/x86/math-emu/wm_shrx.S                        |  207 --
+ arch/x86/math-emu/wm_sqrt.S                        |  472 ----
+ arch/x86/mm/fault.c                                |    5 +
+ arch/x86/mm/pgtable.c                              |   11 -
+ arch/x86/mm/pti.c                                  |    1 +
+ arch/x86/pci/xen.c                                 |    1 +
+ arch/x86/virt/svm/sev.c                            |   18 +-
+ arch/x86/virt/vmx/tdx/tdx.c                        |   66 +-
+ arch/x86/virt/vmx/tdx/tdx.h                        |    1 +
+ arch/x86/xen/enlighten.c                           |    3 +-
+ arch/x86/xen/enlighten_hvm.c                       |    1 +
+ arch/x86/xen/enlighten_pv.c                        |    1 +
+ arch/x86/xen/pmu.c                                 |    1 +
+ arch/x86/xen/time.c                                |    1 +
+ drivers/base/cacheinfo.c                           |   23 +
+ drivers/char/agp/efficeon-agp.c                    |    1 +
+ drivers/clocksource/mmio.c                         |   11 +-
+ drivers/clocksource/timer-of.c                     |   24 +-
+ drivers/clocksource/timer-of.h                     |    5 +-
+ drivers/clocksource/timer-rtl-otto.c               |    2 +-
+ drivers/clocksource/timer-sun5i.c                  |    3 +
+ drivers/cpufreq/Kconfig.x86                        |   26 -
+ drivers/cpufreq/Makefile                           |    2 -
+ drivers/cpufreq/elanfreq.c                         |  226 --
+ drivers/cpufreq/longrun.c                          |    1 +
+ drivers/cpufreq/powernow-k7.c                      |    1 +
+ drivers/cpufreq/powernow-k8.c                      |    1 +
+ drivers/cpufreq/sc520_freq.c                       |  136 --
+ drivers/cpufreq/speedstep-lib.c                    |    1 +
+ drivers/crypto/ccp/sev-dev.c                       |    4 +-
+ drivers/firmware/efi/libstub/x86-5lvl.c            |    1 +
+ drivers/gpu/drm/gma500/mmu.c                       |    2 +
+ drivers/hwmon/fam15h_power.c                       |    1 +
+ drivers/hwmon/k10temp.c                            |    2 +
+ drivers/hwmon/k8temp.c                             |    1 +
+ drivers/irqchip/.kunitconfig                       |    5 +
+ drivers/irqchip/Kconfig                            |   35 +-
+ drivers/irqchip/Makefile                           |    5 +-
+ drivers/irqchip/irq-aspeed-intc.c                  |  139 --
+ drivers/irqchip/irq-ast2700-intc0-test.c           |  473 ++++
+ drivers/irqchip/irq-ast2700-intc0.c                |  582 +++++
+ drivers/irqchip/irq-ast2700-intc1.c                |  280 +++
+ drivers/irqchip/irq-ast2700.c                      |  107 +
+ drivers/irqchip/irq-ast2700.h                      |   48 +
+ drivers/irqchip/irq-econet-en751221.c              |  186 +-
+ drivers/irqchip/irq-gic-v3-its.c                   |   12 +-
+ drivers/irqchip/irq-meson-gpio.c                   |   77 +
+ drivers/irqchip/irq-starfive-jh8100-intc.c         |  207 --
+ drivers/irqchip/irq-starfive-jhb100-intc.c         |  254 +++
+ drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c  |    1 +
+ drivers/power/supply/charger-manager.c             |   16 +-
+ drivers/ras/amd/fmpm.c                             |    1 +
+ drivers/thermal/intel/intel_hfi.c                  |    1 +
+ drivers/thermal/intel/x86_pkg_temp_thermal.c       |    1 +
+ drivers/virt/acrn/hsm.c                            |    1 +
+ drivers/watchdog/Kconfig                           |    2 +-
+ drivers/xen/events/events_base.c                   |    1 +
+ drivers/xen/grant-table.c                          |    1 +
+ drivers/xen/xenbus/xenbus_xs.c                     |    3 +
+ fs/resctrl/internal.h                              |    2 +
+ fs/resctrl/monitor.c                               |   30 +-
+ fs/resctrl/rdtgroup.c                              |   40 +-
+ fs/timerfd.c                                       |  117 +-
+ include/asm-generic/barrier.h                      |    4 +-
+ include/linux/alarmtimer.h                         |    9 +-
+ include/linux/cacheinfo.h                          |    1 +
+ include/linux/clocksource.h                        |   15 +-
+ include/linux/compiler-context-analysis.h          |   30 +-
+ include/linux/delay.h                              |    2 +-
+ include/linux/hrtimer.h                            |   24 +-
+ include/linux/irqchip/arm-gic-v3.h                 |    2 +-
+ include/linux/irqchip/arm-gic.h                    |    4 +-
+ include/linux/mm_types.h                           |   32 +
+ include/linux/msi.h                                |    2 +-
+ include/linux/resctrl.h                            |   18 +-
+ include/linux/rtmutex.h                            |   22 +-
+ include/linux/sched.h                              |   30 +
+ include/linux/sched/clock.h                        |    5 +
+ include/linux/sched/smt.h                          |    4 -
+ include/linux/sched/topology.h                     |   32 +-
+ include/linux/seqlock.h                            |    7 +-
+ include/linux/topology.h                           |   15 +-
+ include/trace/events/timer.h                       |   13 +
+ include/trace/events/timer_migration.h             |   24 +-
+ init/Kconfig                                       |   11 +
+ init/init_task.c                                   |    4 +
+ kernel/exit.c                                      |   29 +
+ kernel/fork.c                                      |    6 +
+ kernel/irq/proc.c                                  |    7 +-
+ kernel/locking/rtmutex.c                           |    4 +
+ kernel/locking/rtmutex_api.c                       |   31 +-
+ kernel/sched/core.c                                |   34 +-
+ kernel/sched/core_sched.c                          |    2 +-
+ kernel/sched/cputime.c                             |    6 -
+ kernel/sched/deadline.c                            |    8 +-
+ kernel/sched/debug.c                               |   60 +-
+ kernel/sched/ext_idle.c                            |    6 -
+ kernel/sched/fair.c                                | 1589 +++++++++++--
+ kernel/sched/features.h                            |    8 +
+ kernel/sched/membarrier.c                          |   98 +-
+ kernel/sched/rt.c                                  |    4 +-
+ kernel/sched/sched.h                               |   59 +-
+ kernel/sched/topology.c                            |  517 ++++-
+ kernel/stop_machine.c                              |    5 +
+ kernel/time/alarmtimer.c                           |   72 +-
+ kernel/time/clocksource.c                          |   29 +-
+ kernel/time/hrtimer.c                              |  154 +-
+ kernel/time/posix-cpu-timers.c                     |   18 +-
+ kernel/time/posix-timers.c                         |   35 +-
+ kernel/time/posix-timers.h                         |    4 +-
+ kernel/time/tick-sched.c                           |    3 +-
+ kernel/time/timer.c                                |    2 +-
+ kernel/time/timer_migration.c                      |  205 +-
+ kernel/time/timer_migration.h                      |   19 +
+ kernel/trace/ring_buffer.c                         |    7 -
+ kernel/workqueue.c                                 |    4 -
+ lib/Kconfig.debug                                  |    6 +-
+ lib/test_context-analysis.c                        |   24 +
+ lib/vdso/datastore.c                               |    6 +-
+ net/netfilter/xt_IDLETIMER.c                       |   24 +-
+ scripts/Makefile.lib                               |    7 +-
+ scripts/livepatch/klp-build                        |  250 ++-
+ scripts/timer_migration_tree.py                    |  122 +
+ scripts/update-intel-ucode-defs.py                 |  130 ++
+ tools/objtool/Build                                |    2 +-
+ tools/objtool/Makefile                             |    4 +-
+ tools/objtool/arch/x86/decode.c                    |   17 +-
+ tools/objtool/builtin-check.c                      |   26 +-
+ tools/objtool/builtin-klp.c                        |    1 +
+ tools/objtool/check.c                              |  412 +---
+ tools/objtool/disas.c                              |   22 +-
+ tools/objtool/elf.c                                |  124 +-
+ tools/objtool/include/objtool/arch.h               |    3 +
+ tools/objtool/include/objtool/builtin.h            |    7 +-
+ tools/objtool/include/objtool/check.h              |   34 +-
+ tools/objtool/include/objtool/checksum.h           |   53 +-
+ tools/objtool/include/objtool/elf.h                |   59 +-
+ tools/objtool/include/objtool/klp.h                |    1 +
+ tools/objtool/include/objtool/warn.h               |   57 +-
+ tools/objtool/klp-checksum.c                       |  347 +++
+ tools/objtool/klp-diff.c                           |  826 +++++--
+ tools/objtool/objtool.c                            |    3 -
+ tools/objtool/trace.c                              |    8 +-
+ tools/testing/selftests/membarrier/Makefile        |    5 +-
+ .../selftests/membarrier/membarrier_rseq_stress.c  |  951 ++++++++
+ tools/testing/selftests/timers/posix_timers.c      |   55 +-
+ 295 files changed, 12001 insertions(+), 16051 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-intc.yaml
+ create mode 100644 Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2700-interrupt.yaml
+ rename Documentation/devicetree/bindings/interrupt-controller/{starfive,jh8100-intc.yaml => starfive,jhb100-intc.yaml} (68%)
+ create mode 100644 arch/x86/include/asm/cpuid/leaf_types.h
+ delete mode 100644 arch/x86/include/asm/device.h
+ rename arch/x86/{kvm/vmx => include/asm/shared}/tdx_errno.h (90%)
+ create mode 100644 arch/x86/kernel/cpu/cpuid_parser.c
+ create mode 100644 arch/x86/kernel/cpu/cpuid_parser.h
+ delete mode 100644 arch/x86/kernel/cpu/umc.c
+ delete mode 100644 arch/x86/math-emu/Makefile
+ delete mode 100644 arch/x86/math-emu/README
+ delete mode 100644 arch/x86/math-emu/control_w.h
+ delete mode 100644 arch/x86/math-emu/div_Xsig.S
+ delete mode 100644 arch/x86/math-emu/div_small.S
+ delete mode 100644 arch/x86/math-emu/errors.c
+ delete mode 100644 arch/x86/math-emu/exception.h
+ delete mode 100644 arch/x86/math-emu/fpu_arith.c
+ delete mode 100644 arch/x86/math-emu/fpu_asm.h
+ delete mode 100644 arch/x86/math-emu/fpu_aux.c
+ delete mode 100644 arch/x86/math-emu/fpu_emu.h
+ delete mode 100644 arch/x86/math-emu/fpu_entry.c
+ delete mode 100644 arch/x86/math-emu/fpu_etc.c
+ delete mode 100644 arch/x86/math-emu/fpu_proto.h
+ delete mode 100644 arch/x86/math-emu/fpu_system.h
+ delete mode 100644 arch/x86/math-emu/fpu_tags.c
+ delete mode 100644 arch/x86/math-emu/fpu_trig.c
+ delete mode 100644 arch/x86/math-emu/get_address.c
+ delete mode 100644 arch/x86/math-emu/load_store.c
+ delete mode 100644 arch/x86/math-emu/mul_Xsig.S
+ delete mode 100644 arch/x86/math-emu/poly.h
+ delete mode 100644 arch/x86/math-emu/poly_2xm1.c
+ delete mode 100644 arch/x86/math-emu/poly_atan.c
+ delete mode 100644 arch/x86/math-emu/poly_l2.c
+ delete mode 100644 arch/x86/math-emu/poly_sin.c
+ delete mode 100644 arch/x86/math-emu/poly_tan.c
+ delete mode 100644 arch/x86/math-emu/polynom_Xsig.S
+ delete mode 100644 arch/x86/math-emu/reg_add_sub.c
+ delete mode 100644 arch/x86/math-emu/reg_compare.c
+ delete mode 100644 arch/x86/math-emu/reg_constant.c
+ delete mode 100644 arch/x86/math-emu/reg_constant.h
+ delete mode 100644 arch/x86/math-emu/reg_convert.c
+ delete mode 100644 arch/x86/math-emu/reg_divide.c
+ delete mode 100644 arch/x86/math-emu/reg_ld_str.c
+ delete mode 100644 arch/x86/math-emu/reg_mul.c
+ delete mode 100644 arch/x86/math-emu/reg_norm.S
+ delete mode 100644 arch/x86/math-emu/reg_round.S
+ delete mode 100644 arch/x86/math-emu/reg_u_add.S
+ delete mode 100644 arch/x86/math-emu/reg_u_div.S
+ delete mode 100644 arch/x86/math-emu/reg_u_mul.S
+ delete mode 100644 arch/x86/math-emu/reg_u_sub.S
+ delete mode 100644 arch/x86/math-emu/round_Xsig.S
+ delete mode 100644 arch/x86/math-emu/shr_Xsig.S
+ delete mode 100644 arch/x86/math-emu/status_w.h
+ delete mode 100644 arch/x86/math-emu/version.h
+ delete mode 100644 arch/x86/math-emu/wm_shrx.S
+ delete mode 100644 arch/x86/math-emu/wm_sqrt.S
+ delete mode 100644 drivers/cpufreq/elanfreq.c
+ delete mode 100644 drivers/cpufreq/sc520_freq.c
+ create mode 100644 drivers/irqchip/.kunitconfig
+ delete mode 100644 drivers/irqchip/irq-aspeed-intc.c
+ create mode 100644 drivers/irqchip/irq-ast2700-intc0-test.c
+ create mode 100644 drivers/irqchip/irq-ast2700-intc0.c
+ create mode 100644 drivers/irqchip/irq-ast2700-intc1.c
+ create mode 100644 drivers/irqchip/irq-ast2700.c
+ create mode 100644 drivers/irqchip/irq-ast2700.h
+ delete mode 100644 drivers/irqchip/irq-starfive-jh8100-intc.c
+ create mode 100644 drivers/irqchip/irq-starfive-jhb100-intc.c
+ create mode 100755 scripts/timer_migration_tree.py
+ create mode 100755 scripts/update-intel-ucode-defs.py
+ create mode 100644 tools/objtool/klp-checksum.c
+ create mode 100644 tools/testing/selftests/membarrier/membarrier_rseq_stress.c
+$ git am -3 ../patches/0001-vdso-Merge-fixup-with-mm.patch
+Applying: vdso: Merge fixup with mm
+Using index info to reconstruct a base tree...
+M	kernel/time/namespace_vdso.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging kexec/kexec-next (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'kexec-next' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git kexec/kexec-next
+Already up to date.
+Merging liveupdate/next (34e8f02817e31 memblock tests: define MIGRATE_CMA)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/liveupdate/linux.git liveupdate/next
+Auto-merging kernel/liveupdate/kexec_handover.c
+Auto-merging kernel/liveupdate/luo_session.c
+Auto-merging mm/mm_init.c
+Merge made by the 'ort' strategy.
+ include/linux/liveupdate.h                      |   9 +-
+ include/linux/memblock.h                        |  21 ++++-
+ include/uapi/linux/liveupdate.h                 |  21 +++++
+ kernel/liveupdate/Kconfig                       |   2 -
+ kernel/liveupdate/kexec_handover.c              |  52 +++++------
+ kernel/liveupdate/luo_flb.c                     |  52 +++++------
+ kernel/liveupdate/luo_session.c                 |  20 +++++
+ lib/tests/liveupdate.c                          |   3 +
+ mm/memblock.c                                   |  56 +++++-------
+ mm/mm_init.c                                    |  30 ++++---
+ tools/testing/memblock/linux/mmzone.h           |   4 +
+ tools/testing/selftests/kho/vmtest.sh           |   4 +
+ tools/testing/selftests/liveupdate/liveupdate.c | 113 ++++++++++++++++++++++++
+ 13 files changed, 283 insertions(+), 104 deletions(-)
+Merging clockevents/timers/drivers/next (f182fa740218d clocksource/drivers/sun5i: Add D1 hstimer support)
+$ git merge -m Merge branch 'timers/drivers/next' of https://git.kernel.org/pub/scm/linux/kernel/git/daniel.lezcano/linux.git clockevents/timers/drivers/next
+Auto-merging Documentation/devicetree/bindings/timer/sifive,clint.yaml
+Merge made by the 'ort' strategy.
+ .../timer/allwinner,sun5i-a13-hstimer.yaml         |  9 ++-
+ .../devicetree/bindings/timer/renesas,rz-mtu3.yaml | 26 +++++--
+ .../devicetree/bindings/timer/sifive,clint.yaml    | 17 +----
+ drivers/clocksource/timer-sun5i.c                  | 84 +++++++++++++++++-----
+ 4 files changed, 94 insertions(+), 42 deletions(-)
+Merging edac/edac-for-next (4b049cd5e5194 Merge branch 'edac-drivers' into edac-for-next)
+$ git merge -m Merge branch 'edac-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ras/ras.git edac/edac-for-next
+Merge made by the 'ort' strategy.
+ drivers/edac/i10nm_base.c   |  14 +-
+ drivers/edac/i5000_edac.c   |  33 +++-
+ drivers/edac/i5100_edac.c   |  26 ++-
+ drivers/edac/i5400_edac.c   |  33 +++-
+ drivers/edac/i7300_edac.c   |  33 +++-
+ drivers/edac/igen6_edac.c   | 407 +++++++++++++++++++++++++++++++++++---------
+ drivers/edac/sb_edac.c      |   2 +-
+ drivers/edac/skx_common.c   |   3 +
+ drivers/ras/amd/atl/Kconfig |   1 -
+ include/linux/edac.h        |   3 +
+ 10 files changed, 436 insertions(+), 119 deletions(-)
+Merging ftrace/for-next (4df143bbee63a Merge probes/for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git ftrace/for-next
+Merge made by the 'ort' strategy.
+ include/linux/bootconfig.h  |  3 +++
+ init/main.c                 | 45 ----------------------------------
+ kernel/trace/trace_uprobe.c |  1 +
+ lib/bootconfig.c            | 56 ++++++++++++++++++++++++++++++++++++++++++
+ tools/bootconfig/main.c     | 60 +++++++++++++++++++++++++++++++++++++++------
+ 5 files changed, 112 insertions(+), 53 deletions(-)
+$ git am -3 ../patches/0001-ftrace-Fix-semantic-conflict-with-mm-tree.patch
+Applying: ftrace: Fix semantic conflict with mm tree
+Using index info to reconstruct a base tree...
+M	kernel/trace/trace_printk.c
+Falling back to patching base and 3-way merge...
+Auto-merging kernel/trace/trace_printk.c
+No changes -- Patch already applied.
+Merging rcu/next (ed062c41dfda2 Merge branch 'rcu-nocb.20260123a')
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/rcu/linux rcu/next
+Already up to date.
+Merging paulmck/non-rcu/next (48f7a50c027dd stop_machine: Fix the documentation for a NULL cpus argument)
+$ git merge -m Merge branch 'non-rcu/next' of https://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu.git paulmck/non-rcu/next
+Already up to date.
+Merging kvm/next (b7fbe9a1bf9ee Merge branch 'kvm-apx-prepare' into HEAD)
+$ git merge -m Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git kvm/next
+Auto-merging arch/x86/kvm/mmu/mmu.c
+Auto-merging arch/x86/kvm/mmu/spte.c
+Auto-merging arch/x86/kvm/svm/nested.c
+Auto-merging arch/x86/kvm/svm/sev.c
+Auto-merging arch/x86/kvm/svm/svm.c
+Auto-merging arch/x86/kvm/vmx/capabilities.h
+Auto-merging arch/x86/kvm/vmx/tdx.c
+Auto-merging arch/x86/kvm/vmx/vmenter.S
+Auto-merging arch/x86/kvm/vmx/vmx.c
+Merge made by the 'ort' strategy.
+ Documentation/virt/kvm/x86/mmu.rst |  10 +-
+ arch/x86/include/asm/cpufeatures.h |   1 +
+ arch/x86/include/asm/kvm-x86-ops.h |   1 +
+ arch/x86/include/asm/kvm_host.h    |  80 ++++++++++------
+ arch/x86/include/asm/svm.h         |   1 +
+ arch/x86/include/asm/vmx.h         |  14 ++-
+ arch/x86/kvm/hyperv.c              |   4 +-
+ arch/x86/kvm/kvm-asm-offsets.c     |   1 +
+ arch/x86/kvm/kvm_cache_regs.h      |  62 +++++++++----
+ arch/x86/kvm/mmu.h                 |  30 ++++--
+ arch/x86/kvm/mmu/mmu.c             | 182 +++++++++++++++++++++++++-----------
+ arch/x86/kvm/mmu/mmutrace.h        |  19 ++--
+ arch/x86/kvm/mmu/paging_tmpl.h     |  73 ++++++++++-----
+ arch/x86/kvm/mmu/spte.c            |  92 ++++++++++++-------
+ arch/x86/kvm/mmu/spte.h            |  70 ++++++++------
+ arch/x86/kvm/mmu/tdp_mmu.c         |   6 +-
+ arch/x86/kvm/svm/nested.c          |  38 ++++++--
+ arch/x86/kvm/svm/sev.c             |   2 +-
+ arch/x86/kvm/svm/svm.c             |  70 ++++++++++----
+ arch/x86/kvm/svm/svm.h             |   7 +-
+ arch/x86/kvm/svm/vmenter.S         | 103 +++++----------------
+ arch/x86/kvm/vmenter.h             |  80 ++++++++++++++++
+ arch/x86/kvm/vmx/capabilities.h    |  12 ++-
+ arch/x86/kvm/vmx/common.h          |  26 ++++--
+ arch/x86/kvm/vmx/hyperv_evmcs.h    |   1 +
+ arch/x86/kvm/vmx/main.c            |   9 ++
+ arch/x86/kvm/vmx/nested.c          |  56 +++++++++--
+ arch/x86/kvm/vmx/run_flags.h       |   9 --
+ arch/x86/kvm/vmx/tdx.c             |  38 ++++----
+ arch/x86/kvm/vmx/vmenter.S         | 184 +++++++++++++++++--------------------
+ arch/x86/kvm/vmx/vmx.c             | 113 +++++++++++------------
+ arch/x86/kvm/vmx/vmx.h             |  32 +++----
+ arch/x86/kvm/vmx/x86_ops.h         |   6 ++
+ arch/x86/kvm/x86.c                 |  38 +++-----
+ 34 files changed, 891 insertions(+), 579 deletions(-)
+ create mode 100644 arch/x86/kvm/vmenter.h
+ delete mode 100644 arch/x86/kvm/vmx/run_flags.h
+Merging kvm-arm/next (9b46f887829f7 Merge branch kvm-arm64/no-lazy-vgic-init into kvmarm-master/next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git kvm-arm/next
+Auto-merging arch/arm64/kvm/arm.c
+Merge made by the 'ort' strategy.
+ arch/arm64/include/asm/kvm_host.h |   3 +-
+ arch/arm64/kvm/arch_timer.c       | 106 +++++++++++++++++---------------------
+ arch/arm64/kvm/arm.c              |  39 +++++++++-----
+ arch/arm64/kvm/emulate-nested.c   |   4 ++
+ arch/arm64/kvm/fpsimd.c           |  26 ++++++++++
+ arch/arm64/kvm/pmu-emul.c         |  31 ++++-------
+ arch/arm64/kvm/vgic/vgic-irqfd.c  |   6 +++
+ arch/arm64/kvm/vgic/vgic.c        |   6 +--
+ include/kvm/arm_arch_timer.h      |   7 +--
+ include/kvm/arm_pmu.h             |   5 +-
+ 10 files changed, 126 insertions(+), 107 deletions(-)
+Merging kvms390/next (9b8e8aad5896d KVM: s390: ucontrol: Fix memslot handling)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux.git kvms390/next
+Already up to date.
+Merging kvm-ppc/topic/ppc-kvm (5200f5f493f79 Linux 7.1-rc4)
+$ git merge -m Merge branch 'topic/ppc-kvm' of https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git kvm-ppc/topic/ppc-kvm
+Already up to date.
+Merging kvm-riscv/riscv_kvm_next (ddbf9c76c4020 RISC-V: KVM: Fix shift-out-of-bounds in make_xfence_request())
+$ git merge -m Merge branch 'riscv_kvm_next' of https://github.com/kvm-riscv/linux.git kvm-riscv/riscv_kvm_next
+Already up to date.
+Merging kvm-x86/next (a9512a611bd03 Merge branches 'fixes', 'generic', 'misc', 'mmu', 'selftests', 'sev', 'svm', 'vfio' and 'vmx')
+$ git merge -m Merge branch 'next' of https://github.com/kvm-x86/linux.git kvm-x86/next
+Auto-merging arch/x86/include/asm/tdx.h
+CONFLICT (content): Merge conflict in arch/x86/include/asm/tdx.h
+Auto-merging arch/x86/kvm/mmu/mmu.c
+Auto-merging arch/x86/kvm/svm/sev.c
+Auto-merging arch/x86/kvm/svm/svm.c
+Auto-merging arch/x86/kvm/vmx/tdx.c
+Auto-merging arch/x86/kvm/vmx/vmx.c
+Auto-merging arch/x86/virt/vmx/tdx/tdx.c
+Auto-merging drivers/crypto/ccp/sev-dev.c
+Resolved 'arch/x86/include/asm/tdx.h' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 4be77c91cd8a4] Merge branch 'next' of https://github.com/kvm-x86/linux.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/virt/kvm/api.rst                     |  26 ++
+ arch/x86/include/asm/kvm-x86-ops.h                 |   6 +-
+ arch/x86/include/asm/kvm_host.h                    |   6 +-
+ arch/x86/include/asm/tdx.h                         |  14 +-
+ arch/x86/include/uapi/asm/kvm.h                    |   2 +
+ arch/x86/kvm/mmu/mmu.c                             |   2 +-
+ arch/x86/kvm/mmu/tdp_mmu.c                         |  10 +-
+ arch/x86/kvm/svm/nested.c                          |  64 ++-
+ arch/x86/kvm/svm/sev.c                             | 477 ++++++++++-----------
+ arch/x86/kvm/svm/svm.c                             |  52 ++-
+ arch/x86/kvm/svm/svm.h                             |  19 +-
+ arch/x86/kvm/vmx/tdx.c                             |  63 +--
+ arch/x86/kvm/vmx/vmx.c                             |   3 +
+ arch/x86/kvm/x86.c                                 |  51 ++-
+ arch/x86/virt/hw.c                                 |  15 +-
+ arch/x86/virt/vmx/tdx/tdx.c                        |  26 +-
+ drivers/crypto/ccp/sev-dev.c                       | 101 ++++-
+ include/linux/kvm_host.h                           |  12 +-
+ include/linux/psp-sev.h                            |  37 ++
+ tools/testing/selftests/kvm/Makefile.kvm           |   1 +
+ .../selftests/kvm/access_tracking_perf_test.c      |   2 +-
+ tools/testing/selftests/kvm/guest_memfd_test.c     |   2 +-
+ tools/testing/selftests/kvm/include/kvm_syscalls.h |  10 +
+ tools/testing/selftests/kvm/include/test_util.h    |   2 +-
+ tools/testing/selftests/kvm/include/x86/sev.h      |  24 ++
+ tools/testing/selftests/kvm/lib/assert.c           |   8 +-
+ tools/testing/selftests/kvm/lib/kvm_util.c         |   2 +-
+ tools/testing/selftests/kvm/memslot_perf_test.c    |  14 +-
+ .../selftests/kvm/s390/shared_zeropage_test.c      |   3 +-
+ tools/testing/selftests/kvm/s390/tprot.c           |   2 +-
+ .../testing/selftests/kvm/set_memory_region_test.c |   2 +-
+ tools/testing/selftests/kvm/x86/hyperv_features.c  |   2 +-
+ tools/testing/selftests/kvm/x86/hyperv_tlb_flush.c |  15 +-
+ tools/testing/selftests/kvm/x86/sev_dbg_test.c     | 118 +++++
+ tools/testing/selftests/kvm/x86/sev_init2_tests.c  |  14 +-
+ .../testing/selftests/kvm/x86/sev_migrate_tests.c  |   2 +-
+ tools/testing/selftests/kvm/x86/sev_smoke_test.c   |   4 +-
+ tools/testing/selftests/kvm/x86/sync_regs_test.c   |   1 -
+ virt/kvm/guest_memfd.c                             |  14 +-
+ virt/kvm/kvm_main.c                                |   6 +-
+ virt/kvm/vfio.c                                    |  98 ++---
+ 41 files changed, 852 insertions(+), 480 deletions(-)
+ create mode 100644 tools/testing/selftests/kvm/x86/sev_dbg_test.c
+$ git am -3 ../patches/0001-KVM-selftests-Fix-up-semantic-changes.patch
+Applying: KVM: selftests: Fix up semantic changes
+Using index info to reconstruct a base tree...
+M	tools/testing/selftests/kvm/lib/kvm_util.c
+Falling back to patching base and 3-way merge...
+Auto-merging tools/testing/selftests/kvm/lib/kvm_util.c
+No changes -- Patch already applied.
+$ git am -3 ../patches/kvm-x86-init
+Applying: Fix up merge
+Using index info to reconstruct a base tree...
+M	arch/x86/virt/vmx/tdx/tdx_global_metadata.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging xen-tip/linux-next (4594437880ce3 x86/xen: Tolerate nested XEN_LAZY_MMU entering/leaving)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/xen/tip.git xen-tip/linux-next
+Already up to date.
+Merging percpu/for-next (8f0b4cce4481f Linux 6.19-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dennis/percpu.git percpu/for-next
+Already up to date.
+Merging workqueues/for-next (2ccd533ef9153 Merge branch 'for-7.2' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/wq.git workqueues/for-next
+Auto-merging drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
+Auto-merging kernel/workqueue.c
+Auto-merging lib/Kconfig.debug
+Merge made by the 'ort' strategy.
+ drivers/media/pci/ddbridge/ddbridge-core.c         |  2 +-
+ .../media/platform/synopsys/hdmirx/snps_hdmirx.c   |  8 ++---
+ drivers/platform/cznic/turris-omnia-mcu-gpio.c     |  2 +-
+ drivers/rapidio/rio.c                              |  2 +-
+ drivers/virt/acrn/irqfd.c                          |  2 +-
+ kernel/umh.c                                       |  2 +-
+ kernel/workqueue.c                                 | 38 ++++++++--------------
+ lib/Kconfig.debug                                  |  5 +++
+ 8 files changed, 28 insertions(+), 33 deletions(-)
+Merging sched-ext/for-next (8ad190500ea54 Merge branch 'for-7.2' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git sched-ext/for-next
+Auto-merging kernel/sched/ext.c
+Auto-merging kernel/sched/ext_idle.c
+Merge made by the 'ort' strategy.
+ Documentation/scheduler/sched-ext.rst              |    7 +-
+ include/linux/sched/ext.h                          |    9 +
+ kernel/sched/build_policy.c                        |    5 +
+ kernel/sched/ext.c                                 | 1150 ++++++++++++++++----
+ kernel/sched/ext_cid.c                             |  725 ++++++++++++
+ kernel/sched/ext_cid.h                             |  228 ++++
+ kernel/sched/ext_idle.c                            |    9 +-
+ kernel/sched/ext_internal.h                        |  235 +++-
+ kernel/sched/ext_types.h                           |  144 +++
+ tools/sched_ext/README.md                          |    6 +-
+ tools/sched_ext/include/scx/cid.bpf.h              |  730 +++++++++++++
+ tools/sched_ext/include/scx/common.bpf.h           |   28 +
+ tools/sched_ext/include/scx/compat.bpf.h           |   30 +-
+ tools/sched_ext/include/scx/compat.h               |   23 +-
+ tools/sched_ext/include/scx/user_exit_info.bpf.h   |    3 +
+ tools/sched_ext/include/scx/user_exit_info.h       |    2 +
+ .../sched_ext/include/scx/user_exit_info_common.h  |    5 +
+ tools/sched_ext/scx_central.bpf.c                  |   10 +-
+ tools/sched_ext/scx_cpu0.bpf.c                     |    2 -
+ tools/sched_ext/scx_cpu0.c                         |    2 -
+ tools/sched_ext/scx_flatcg.c                       |   12 +-
+ tools/sched_ext/scx_qmap.bpf.c                     |  893 +++++++++------
+ tools/sched_ext/scx_qmap.c                         |  109 +-
+ tools/sched_ext/scx_qmap.h                         |   73 ++
+ .../selftests/sched_ext/non_scx_kfunc_deny.bpf.c   |    7 +-
+ tools/testing/selftests/sched_ext/select_cpu_dfl.c |   54 +-
+ 26 files changed, 3803 insertions(+), 698 deletions(-)
+ create mode 100644 kernel/sched/ext_cid.c
+ create mode 100644 kernel/sched/ext_cid.h
+ create mode 100644 kernel/sched/ext_types.h
+ create mode 100644 tools/sched_ext/include/scx/cid.bpf.h
+ create mode 100644 tools/sched_ext/scx_qmap.h
+Merging drivers-x86/for-next (5fdac9983681f platform/x86: classmate-laptop: Convert keys driver to a platform one)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git drivers-x86/for-next
+Auto-merging MAINTAINERS
+Auto-merging drivers/platform/x86/Kconfig
+Auto-merging drivers/platform/x86/intel/vsec.c
+Auto-merging drivers/platform/x86/panasonic-laptop.c
+Merge made by the 'ort' strategy.
+ .../pm/intel_uncore_frequency_scaling.rst          |    7 +
+ Documentation/wmi/devices/uniwill-laptop.rst       |    2 +-
+ MAINTAINERS                                        |    8 +
+ arch/x86/platform/olpc/olpc-xo15-sci.c             |   33 +-
+ drivers/acpi/bus.c                                 |   15 +
+ drivers/char/sonypi.c                              |    5 +-
+ drivers/platform/arm64/Kconfig                     |   13 +
+ drivers/platform/arm64/Makefile                    |    1 +
+ drivers/platform/arm64/qcom-hamoa-ec.c             |  451 ++++++
+ drivers/platform/x86/Kconfig                       |    1 +
+ drivers/platform/x86/amd/hfi/hfi.c                 |    6 +-
+ drivers/platform/x86/barco-p50-gpio.c              |   10 +-
+ drivers/platform/x86/classmate-laptop.c            |  343 +++--
+ drivers/platform/x86/dell/alienware-wmi-base.c     |   31 +-
+ .../x86/dell/dell-wmi-sysman/biosattr-interface.c  |    8 +-
+ .../x86/dell/dell-wmi-sysman/dell-wmi-sysman.h     |    4 +-
+ .../dell/dell-wmi-sysman/passwordattr-interface.c  |    4 +-
+ drivers/platform/x86/dell/dell-wmi-sysman/sysman.c |    6 +-
+ drivers/platform/x86/dell/dell_rbu.c               |    4 +-
+ drivers/platform/x86/intel/pmc/Makefile            |    3 +-
+ drivers/platform/x86/intel/pmc/arl.c               |   13 +-
+ drivers/platform/x86/intel/pmc/core.c              |  137 +-
+ drivers/platform/x86/intel/pmc/core.h              |   68 +-
+ drivers/platform/x86/intel/pmc/lnl.c               |    6 +-
+ drivers/platform/x86/intel/pmc/mtl.c               |    7 +-
+ drivers/platform/x86/intel/pmc/nvl.c               | 1539 ++++++++++++++++++++
+ drivers/platform/x86/intel/pmc/ptl.c               |    8 +-
+ drivers/platform/x86/intel/pmc/wcl.c               |    6 +-
+ .../uncore-frequency/uncore-frequency-common.c     |   16 +-
+ .../uncore-frequency/uncore-frequency-common.h     |    8 +-
+ .../intel/uncore-frequency/uncore-frequency-tpmi.c |   15 +-
+ drivers/platform/x86/intel/vsec.c                  |   14 +-
+ drivers/platform/x86/intel/vsec_tpmi.c             |   13 +-
+ drivers/platform/x86/lenovo/thinkpad_acpi.c        |   42 +-
+ drivers/platform/x86/meraki-mx100.c                |   41 +-
+ drivers/platform/x86/panasonic-laptop.c            |   19 +-
+ drivers/platform/x86/pcengines-apuv2.c             |    2 +-
+ drivers/platform/x86/pmc_atom.c                    |    4 +-
+ drivers/platform/x86/sel3350-platform.c            |  136 +-
+ drivers/platform/x86/x86-android-tablets/core.c    |   64 +-
+ drivers/platform/x86/xo15-ebook.c                  |   91 +-
+ include/linux/acpi.h                               |    7 +
+ include/linux/intel_vsec.h                         |    6 +-
+ 43 files changed, 2838 insertions(+), 379 deletions(-)
+ create mode 100644 drivers/platform/arm64/qcom-hamoa-ec.c
+ create mode 100644 drivers/platform/x86/intel/pmc/nvl.c
+$ git am -3 ../patches/0001-hid-Fix-up-mismerge.patch
+Applying: hid: Fix up mismerge
+$ git reset HEAD^
+Unstaged changes after reset:
+M	drivers/hid/hid-asus.c
+$ git add -A .
+$ git commit -v -a --amend
+warning: notes ref refs/notes/commits is invalid
+[master ec6ed6c6b46a8] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git
+ Date: Thu May 21 11:37:39 2026 +0100
+Merging chrome-platform/for-next (ace8ee71e8ef4 platform/chrome: Use named initializers for struct i2c_device_id)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform/for-next
+Merge made by the 'ort' strategy.
+ drivers/platform/chrome/chromeos_privacy_screen.c |  3 +++
+ drivers/platform/chrome/chromeos_tbmc.c           |  6 ++++-
+ drivers/platform/chrome/cros_ec_i2c.c             |  2 +-
+ drivers/platform/chrome/cros_ec_sensorhub.c       |  6 ++++-
+ drivers/platform/chrome/cros_ec_sysfs.c           |  5 ++--
+ drivers/platform/chrome/cros_hps_i2c.c            |  2 +-
+ drivers/platform/chrome/cros_kbd_led_backlight.c  | 28 ++++++-----------------
+ drivers/platform/chrome/wilco_ec/event.c          |  8 +++++--
+ include/linux/platform_data/cros_ec_proto.h       |  2 ++
+ 9 files changed, 33 insertions(+), 29 deletions(-)
+Merging chrome-platform-firmware/for-firmware-next (86e3bbc716332 firmware: google: Skip failing entries instead of aborting populate)
+$ git merge -m Merge branch 'for-firmware-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chrome-platform/linux.git chrome-platform-firmware/for-firmware-next
+Merge made by the 'ort' strategy.
+ drivers/firmware/google/coreboot_table.c | 17 ++++++++++++-----
+ 1 file changed, 12 insertions(+), 5 deletions(-)
+Merging hsi/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sre/linux-hsi.git hsi/for-next
+Already up to date.
+Merging leds-lj/for-leds-next (25025253476a6 leds: Adjust documentation of brightness sysfs node)
+$ git merge -m Merge branch 'for-leds-next' of https://git.kernel.org/pub/scm/linux/kernel/git/lee/leds.git leds-lj/for-leds-next
+Merge made by the 'ort' strategy.
+ Documentation/ABI/testing/sysfs-class-led          |   4 +-
+ .../ABI/testing/sysfs-class-led-multicolor         |  19 +-
+ .../devicetree/bindings/leds/irled/ir-spi-led.yaml |   2 +-
+ .../devicetree/bindings/leds/ti,lm3560.yaml        | 163 +++++++++++++
+ Documentation/leds/leds-class-multicolor.rst       |  21 +-
+ Documentation/leds/leds-class.rst                  |  63 +++++
+ Documentation/leds/leds-lp5812.rst                 |   2 +-
+ drivers/leds/Makefile                              |   2 +-
+ drivers/leds/blink/leds-bcm63138.c                 |   2 +-
+ drivers/leds/flash/Kconfig                         |   2 +-
+ drivers/leds/led-class-multicolor.c                |  47 +++-
+ drivers/leds/leds-blinkm.c                         |   3 +-
+ drivers/leds/leds-cros_ec.c                        |   2 +-
+ drivers/leds/leds-dac124s085.c                     |   2 +-
+ drivers/leds/leds-lp50xx.c                         |   1 +
+ drivers/leds/leds-pca9532.c                        |   8 +-
+ drivers/leds/leds-st1202.c                         |   1 -
+ drivers/leds/rgb/Kconfig                           |  25 ++
+ drivers/leds/rgb/Makefile                          |   2 +
+ drivers/leds/rgb/leds-lp5860-core.c                | 234 ++++++++++++++++++
+ drivers/leds/rgb/leds-lp5860-spi.c                 |  98 ++++++++
+ drivers/leds/rgb/leds-lp5860.h                     | 268 +++++++++++++++++++++
+ drivers/leds/rgb/leds-ncp5623.c                    |   4 +-
+ drivers/leds/rgb/leds-qcom-lpg.c                   | 120 +++++----
+ drivers/leds/trigger/ledtrig-gpio.c                |   3 +-
+ include/dt-bindings/leds/common.h                  |   2 +-
+ include/linux/led-class-multicolor.h               |  30 ++-
+ 27 files changed, 1043 insertions(+), 87 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/leds/ti,lm3560.yaml
+ create mode 100644 drivers/leds/rgb/leds-lp5860-core.c
+ create mode 100644 drivers/leds/rgb/leds-lp5860-spi.c
+ create mode 100644 drivers/leds/rgb/leds-lp5860.h
+Merging ipmi/for-next (8a8ba84886c90 ipmi: Use named initializers for struct i2c_device_id)
+$ git merge -m Merge branch 'for-next' of https://github.com/cminyard/linux-ipmi.git ipmi/for-next
+Merge made by the 'ort' strategy.
+ drivers/char/ipmi/ipmb_dev_int.c    |  4 ++--
+ drivers/char/ipmi/ipmi_ipmb.c       |  4 ++--
+ drivers/char/ipmi/ipmi_kcs_sm.c     |  2 +-
+ drivers/char/ipmi/ipmi_msghandler.c | 14 ++++----------
+ drivers/char/ipmi/ipmi_ssif.c       |  2 +-
+ drivers/char/ipmi/ssif_bmc.c        |  2 +-
+ 6 files changed, 11 insertions(+), 17 deletions(-)
+Merging driver-core/driver-core-next (454257f6d124a sysfs: clamp show() return value in sysfs_kf_read())
+$ git merge -m Merge branch 'driver-core-next' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git driver-core/driver-core-next
+Auto-merging Documentation/admin-guide/kernel-parameters.txt
+Auto-merging MAINTAINERS
+Auto-merging arch/arm64/kernel/cpufeature.c
+Auto-merging arch/powerpc/platforms/pseries/hotplug-memory.c
+Auto-merging drivers/acpi/bus.c
+Auto-merging drivers/base/memory.c
+Auto-merging drivers/base/platform.c
+Auto-merging drivers/iommu/iommu.c
+Auto-merging drivers/pci/pwrctrl/core.c
+Auto-merging include/acpi/acpi_bus.h
+Auto-merging kernel/dma/mapping.c
+Auto-merging rust/helpers/helpers.c
+Auto-merging rust/kernel/alloc/kbox.rs
+CONFLICT (content): Merge conflict in rust/kernel/alloc/kbox.rs
+Recorded preimage for 'rust/kernel/alloc/kbox.rs'
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+Recorded resolution for 'rust/kernel/alloc/kbox.rs'.
+[master 2a37f52111d5f] Merge branch 'driver-core-next' of https://git.kernel.org/pub/scm/linux/kernel/git/driver-core/driver-core.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/kernel-parameters.txt |   5 +-
+ MAINTAINERS                                     |   1 +
+ arch/arc/mm/dma.c                               |   4 +-
+ arch/arm/mach-highbank/highbank.c               |   2 +-
+ arch/arm/mach-mvebu/coherency.c                 |   2 +-
+ arch/arm/mm/dma-mapping-nommu.c                 |   4 +-
+ arch/arm/mm/dma-mapping.c                       |  28 ++--
+ arch/arm64/kernel/cpufeature.c                  |   2 +-
+ arch/arm64/mm/dma-mapping.c                     |   2 +-
+ arch/mips/mm/dma-noncoherent.c                  |   2 +-
+ arch/powerpc/kernel/dma-iommu.c                 |   8 +-
+ arch/powerpc/platforms/pseries/hotplug-memory.c |   4 +-
+ arch/riscv/mm/dma-noncoherent.c                 |   2 +-
+ drivers/acpi/bus.c                              |   6 +-
+ drivers/acpi/scan.c                             |   2 +-
+ drivers/base/base.h                             |  30 ++--
+ drivers/base/bus.c                              |  11 +-
+ drivers/base/core.c                             |  44 +++---
+ drivers/base/cpu.c                              |   4 +-
+ drivers/base/dd.c                               |  18 +--
+ drivers/base/faux.c                             |  32 ++--
+ drivers/base/firmware_loader/sysfs_upload.c     |   8 +-
+ drivers/base/isa.c                              |  12 +-
+ drivers/base/memory.c                           |   2 +-
+ drivers/base/pinctrl.c                          |   2 +-
+ drivers/base/platform.c                         |   2 +-
+ drivers/base/swnode.c                           |  40 +++--
+ drivers/dma/ti/k3-udma-glue.c                   |   6 +-
+ drivers/dma/ti/k3-udma.c                        |   6 +-
+ drivers/gpu/nova-core/driver.rs                 |  10 +-
+ drivers/iommu/dma-iommu.c                       |   9 +-
+ drivers/iommu/iommu.c                           |   5 +-
+ drivers/net/pcs/pcs-xpcs-plat.c                 |   2 +-
+ drivers/of/device.c                             |   6 +-
+ drivers/pci/of.c                                |   2 +-
+ drivers/pci/pwrctrl/core.c                      |   2 +-
+ drivers/tty/serial/serial_base_bus.c            |   2 +-
+ drivers/usb/gadget/udc/aspeed-vhub/dev.c        |   2 +-
+ fs/sysfs/file.c                                 |   9 +-
+ include/acpi/acpi_bus.h                         |  11 ++
+ include/linux/auxiliary_bus.h                   |   4 +
+ include/linux/device.h                          |  91 +++++------
+ include/linux/device/class.h                    |   2 +-
+ include/linux/dma-map-ops.h                     |   6 +-
+ include/linux/dma-mapping.h                     |   2 +-
+ include/linux/fwnode.h                          |   3 +
+ include/linux/iommu-dma.h                       |   3 +-
+ kernel/cpu.c                                    |   4 +-
+ kernel/dma/mapping.c                            |  12 +-
+ mm/hmm.c                                        |   2 +-
+ rust/helpers/acpi.c                             |  16 ++
+ rust/helpers/helpers.c                          |   1 +
+ rust/kernel/alloc/kbox.rs                       |  21 +++
+ rust/kernel/auxiliary.rs                        | 194 +++++++++++++++++-------
+ rust/kernel/device.rs                           |  60 --------
+ rust/kernel/driver.rs                           |  74 +++++++--
+ samples/rust/rust_driver_auxiliary.rs           |  40 +++--
+ 57 files changed, 510 insertions(+), 376 deletions(-)
+ create mode 100644 rust/helpers/acpi.c
+Merging usb/usb-next (e18b0dbb6ec60 Merge tag 'v7.1-rc3' into usb-next)
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/usb.git usb/usb-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/usb/cdns,usb3.yaml         |  10 +-
+ .../devicetree/bindings/usb/genesys,gl850g.yaml    |  18 ++
+ arch/arm/mach-pxa/devices.c                        |   7 -
+ arch/arm/mach-pxa/gumstix.c                        |   1 -
+ arch/arm/mach-pxa/udc.h                            |   8 -
+ drivers/usb/Makefile                               |   2 -
+ drivers/usb/cdns3/Kconfig                          | 112 +++----
+ drivers/usb/cdns3/Makefile                         |  46 +--
+ drivers/usb/cdns3/cdns3-gadget.c                   |   1 +
+ drivers/usb/cdns3/cdns3-plat.c                     |  27 +-
+ drivers/usb/cdns3/cdnsp-gadget.c                   |  50 ++-
+ drivers/usb/cdns3/cdnsp-gadget.h                   |   1 +
+ drivers/usb/cdns3/cdnsp-mem.c                      |  73 +++--
+ drivers/usb/cdns3/cdnsp-pci.c                      | 209 ++++++------
+ drivers/usb/cdns3/cdnsp-ring.c                     |   9 +-
+ drivers/usb/cdns3/core.c                           |  45 +--
+ drivers/usb/cdns3/core.h                           |   5 +-
+ drivers/usb/cdns3/gadget-export.h                  |  10 +-
+ drivers/usb/cdns3/host-export.h                    |   4 +-
+ drivers/usb/gadget/udc/core.c                      |   7 +-
+ drivers/usb/gadget/udc/pxa25x_udc.c                |  41 +--
+ drivers/usb/gadget/udc/pxa25x_udc.h                |   2 +-
+ drivers/usb/gadget/udc/pxa27x_udc.c                |  35 +--
+ drivers/usb/gadget/udc/pxa27x_udc.h                |   2 -
+ drivers/usb/misc/Kconfig                           |   2 +-
+ drivers/usb/misc/onboard_usb_dev.c                 |   1 +
+ drivers/usb/misc/onboard_usb_dev.h                 |   8 +
+ drivers/usb/typec/mux/intel_pmc_mux.c              |  19 +-
+ drivers/usb/typec/mux/ps883x.c                     | 350 +++++++++++----------
+ include/linux/platform_data/pxa2xx_udc.h           |  15 -
+ 30 files changed, 550 insertions(+), 570 deletions(-)
+ delete mode 100644 arch/arm/mach-pxa/udc.h
+Merging thunderbolt/next (e24f3c0df4837 thunderbolt: Increase Notification Timeout to 255 ms for USB4 routers)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git thunderbolt/next
+Auto-merging drivers/thunderbolt/property.c
+CONFLICT (content): Merge conflict in drivers/thunderbolt/property.c
+Resolved 'drivers/thunderbolt/property.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 72543cf21a44d] Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/westeri/thunderbolt.git
+$ git diff -M --stat --summary HEAD^..
+ .../ABI/testing/configfs-thunderbolt_stream        |   83 +
+ Documentation/admin-guide/thunderbolt.rst          |   61 +
+ drivers/net/thunderbolt/main.c                     |   20 +-
+ drivers/thunderbolt/Kconfig                        |   15 +
+ drivers/thunderbolt/Makefile                       |    4 +
+ drivers/thunderbolt/configfs.c                     |   61 +
+ drivers/thunderbolt/debugfs.c                      |    8 +-
+ drivers/thunderbolt/dma_test.c                     |   25 +-
+ drivers/thunderbolt/domain.c                       |   35 +
+ drivers/thunderbolt/icm.c                          |   15 +
+ drivers/thunderbolt/nhi.c                          |   90 +-
+ drivers/thunderbolt/nhi_regs.h                     |    3 +-
+ drivers/thunderbolt/path.c                         |   37 +-
+ drivers/thunderbolt/property.c                     |  164 +-
+ drivers/thunderbolt/stream.c                       | 1698 ++++++++++++++++++++
+ drivers/thunderbolt/switch.c                       |   36 +-
+ drivers/thunderbolt/tb.c                           |   68 +-
+ drivers/thunderbolt/tb.h                           |   11 +
+ drivers/thunderbolt/tb_regs.h                      |   19 +-
+ drivers/thunderbolt/test.c                         |  208 +++
+ drivers/thunderbolt/tunnel.c                       |   35 +
+ drivers/thunderbolt/usb4.c                         |   35 +-
+ drivers/thunderbolt/xdomain.c                      |  280 +++-
+ include/linux/thunderbolt.h                        |   48 +-
+ 24 files changed, 2834 insertions(+), 225 deletions(-)
+ create mode 100644 Documentation/ABI/testing/configfs-thunderbolt_stream
+ create mode 100644 drivers/thunderbolt/configfs.c
+ create mode 100644 drivers/thunderbolt/stream.c
+Merging usb-serial/usb-next (e690592f98c81 USB: serial: whiteheat: drop termbits include)
+$ git merge -m Merge branch 'usb-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/usb-serial.git usb-serial/usb-next
+Auto-merging drivers/usb/serial/belkin_sa.c
+Auto-merging drivers/usb/serial/digi_acceleport.c
+Auto-merging drivers/usb/serial/keyspan.c
+Auto-merging drivers/usb/serial/mct_u232.c
+Auto-merging drivers/usb/serial/option.c
+Merge made by the 'ort' strategy.
+ drivers/usb/serial/ark3116.c          |  1 -
+ drivers/usb/serial/belkin_sa.c        |  2 --
+ drivers/usb/serial/cyberjack.c        |  2 --
+ drivers/usb/serial/cypress_m8.c       |  2 --
+ drivers/usb/serial/digi_acceleport.c  |  2 --
+ drivers/usb/serial/empeg.c            |  3 ---
+ drivers/usb/serial/f81232.c           |  3 ---
+ drivers/usb/serial/f81534.c           |  1 -
+ drivers/usb/serial/ftdi_sio.c         |  1 -
+ drivers/usb/serial/garmin_gps.c       |  3 ---
+ drivers/usb/serial/generic.c          |  1 -
+ drivers/usb/serial/io_edgeport.c      |  2 +-
+ drivers/usb/serial/io_ti.c            |  2 --
+ drivers/usb/serial/ipaq.c             |  3 ---
+ drivers/usb/serial/ipw.c              |  2 --
+ drivers/usb/serial/ir-usb.c           |  2 --
+ drivers/usb/serial/iuu_phoenix.c      |  2 --
+ drivers/usb/serial/keyspan.c          |  2 --
+ drivers/usb/serial/keyspan_pda.c      |  2 --
+ drivers/usb/serial/kl5kusb105.c       |  2 --
+ drivers/usb/serial/kobil_sct.c        |  2 --
+ drivers/usb/serial/mct_u232.c         |  2 --
+ drivers/usb/serial/metro-usb.c        |  3 ---
+ drivers/usb/serial/mos7720.c          |  3 ++-
+ drivers/usb/serial/mos7840.c          |  1 -
+ drivers/usb/serial/mxuport.c          | 40 ++++++++++++++---------------------
+ drivers/usb/serial/omninet.c          |  2 --
+ drivers/usb/serial/opticon.c          |  2 --
+ drivers/usb/serial/option.c           |  1 -
+ drivers/usb/serial/oti6858.c          |  3 ---
+ drivers/usb/serial/pl2303.c           |  3 ---
+ drivers/usb/serial/qcserial.c         |  1 -
+ drivers/usb/serial/quatech2.c         |  2 --
+ drivers/usb/serial/safe_serial.c      |  2 --
+ drivers/usb/serial/spcp8x5.c          |  2 --
+ drivers/usb/serial/ssu100.c           |  2 --
+ drivers/usb/serial/symbolserial.c     |  2 --
+ drivers/usb/serial/ti_usb_3410_5052.c |  2 --
+ drivers/usb/serial/usb-serial.c       |  3 ---
+ drivers/usb/serial/usb_wwan.c         |  1 -
+ drivers/usb/serial/visor.c            |  4 ----
+ drivers/usb/serial/whiteheat.c        |  4 ----
+ drivers/usb/serial/wishbone-serial.c  |  1 -
+ drivers/usb/serial/xr_serial.c        |  1 +
+ drivers/usb/serial/xsens_mt.c         |  1 -
+ 45 files changed, 20 insertions(+), 110 deletions(-)
+Merging tty/tty-next (16e95bfb79b5d serial: qcom-geni: Avoid probing debug console UART without console support)
+$ git merge -m Merge branch 'tty-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/tty.git tty/tty-next
+Auto-merging arch/powerpc/configs/ppc6xx_defconfig
+Auto-merging drivers/net/ppp/Kconfig
+Auto-merging drivers/tty/serial/pch_uart.c
+Auto-merging drivers/tty/serial/qcom_geni_serial.c
+Auto-merging drivers/tty/serial/sh-sci.c
+Merge made by the 'ort' strategy.
+ .../bindings/serial/qcom,sa8255p-geni-uart.yaml    |   13 +-
+ .../bindings/serial/snps-dw-apb-uart.yaml          |    1 +
+ Documentation/userspace-api/ioctl/ioctl-number.rst |    1 -
+ arch/powerpc/configs/ppc6xx_defconfig              |    1 -
+ drivers/net/ppp/Kconfig                            |    4 +-
+ drivers/tty/Kconfig                                |   11 +-
+ drivers/tty/Makefile                               |    1 -
+ drivers/tty/n_hdlc.c                               |    7 -
+ drivers/tty/serial/8250/8250_dw.c                  |   31 +-
+ drivers/tty/serial/8250/8250_dwlib.c               |   49 -
+ drivers/tty/serial/8250/8250_dwlib.h               |   73 +
+ drivers/tty/serial/8250/8250_mtk.c                 |   23 +-
+ drivers/tty/serial/8250/8250_pci.c                 | 3120 ++++++------
+ drivers/tty/serial/8250/8250_port.c                |    6 +-
+ drivers/tty/serial/Kconfig                         |    6 +-
+ drivers/tty/serial/apbuart.c                       |    2 -
+ drivers/tty/serial/bcm63xx_uart.c                  |    2 +-
+ drivers/tty/serial/icom.c                          |    2 -
+ drivers/tty/serial/jsm/jsm_driver.c                |   38 +-
+ drivers/tty/serial/max310x.c                       |  179 +-
+ drivers/tty/serial/mxs-auart.c                     |    2 +-
+ drivers/tty/serial/omap-serial.c                   |    2 +-
+ drivers/tty/serial/pch_uart.c                      |    2 +-
+ drivers/tty/serial/pxa.c                           |    2 +-
+ drivers/tty/serial/qcom_geni_serial.c              |   20 +-
+ drivers/tty/serial/samsung_tty.c                   |    8 +-
+ drivers/tty/serial/serial_core.c                   |   34 +-
+ drivers/tty/serial/serial_txx9.c                   |    4 +-
+ drivers/tty/serial/sh-sci.c                        |    2 +-
+ drivers/tty/serial/sunsu.c                         |    2 +-
+ drivers/tty/synclink_gt.c                          | 5038 --------------------
+ include/linux/serial_core.h                        |   12 +
+ include/linux/serial_sci.h                         |    1 -
+ include/linux/synclink.h                           |   37 -
+ include/uapi/linux/synclink.h                      |  301 --
+ 35 files changed, 1868 insertions(+), 7169 deletions(-)
+ delete mode 100644 drivers/tty/synclink_gt.c
+ delete mode 100644 include/linux/synclink.h
+ delete mode 100644 include/uapi/linux/synclink.h
+Merging char-misc/char-misc-next (5d6919055dec1 Linux 7.1-rc3)
+$ git merge -m Merge branch 'char-misc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git char-misc/char-misc-next
+Already up to date.
+$ git am -3 ../patches/0001-bus-mhi-Fix-up-interaction-with-net-next-tree.patch
+Applying: bus: mhi: Fix up interaction with net-next tree
+Using index info to reconstruct a base tree...
+M	drivers/net/wireless/ath/ath12k/wifi7/mhi.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging coresight/next (a5dd853fb7774 coresight: sysfs: Validate CPU online status for per-CPU sources)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/coresight/linux.git coresight/next
+Merge made by the 'ort' strategy.
+ drivers/hwtracing/coresight/coresight-catu.c       |   2 +-
+ drivers/hwtracing/coresight/coresight-core.c       | 574 ++++++++++++++++++---
+ drivers/hwtracing/coresight/coresight-cti-core.c   |   9 +-
+ .../hwtracing/coresight/coresight-cti-platform.c   |   1 +
+ drivers/hwtracing/coresight/coresight-etm-perf.c   | 289 ++++++-----
+ drivers/hwtracing/coresight/coresight-etm3x-core.c |  73 +--
+ drivers/hwtracing/coresight/coresight-etm4x-core.c | 214 +++-----
+ drivers/hwtracing/coresight/coresight-priv.h       |   8 +-
+ drivers/hwtracing/coresight/coresight-syscfg.c     |  38 +-
+ drivers/hwtracing/coresight/coresight-syscfg.h     |   2 +
+ drivers/hwtracing/coresight/coresight-sysfs.c      | 135 ++---
+ drivers/hwtracing/coresight/coresight-tmc-etr.c    |   4 +-
+ drivers/hwtracing/coresight/coresight-trbe.c       |  61 ++-
+ include/linux/coresight.h                          |  27 +-
+ include/linux/cpuhotplug.h                         |   2 +-
+ 15 files changed, 928 insertions(+), 511 deletions(-)
+Merging fastrpc/for-next (8d712bcdd1ef7 misc: fastrpc: Move prints outside spinlock in fastrpc_cb_probe)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/fastrpc.git fastrpc/for-next
+Merge made by the 'ort' strategy.
+ drivers/misc/fastrpc.c | 93 ++++++++++++++++++++++++++++++++++----------------
+ 1 file changed, 63 insertions(+), 30 deletions(-)
+Merging fpga/for-next (3c310d945b949 dt-bindings: fpga: altr,socfpga-fpga-mgr: convert to DT schema)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fpga/linux-fpga.git fpga/for-next
+Auto-merging Documentation/devicetree/bindings/vendor-prefixes.yaml
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/fpga/altera-pr-ip.txt      |  12 -
+ .../bindings/fpga/altera-socfpga-fpga-mgr.txt      |  17 --
+ .../devicetree/bindings/fpga/altr,a10-pr-ip.yaml   |  34 +++
+ .../bindings/fpga/altr,socfpga-fpga-mgr.yaml       |  38 +++
+ .../bindings/fpga/efinix,trion-config.yaml         |  98 ++++++++
+ .../bindings/fpga/technologic,ts7300-fpga.yaml     |  36 +++
+ .../devicetree/bindings/vendor-prefixes.yaml       |   2 +
+ drivers/fpga/Kconfig                               |   7 +
+ drivers/fpga/Makefile                              |   1 +
+ drivers/fpga/efinix-spi.c                          | 260 +++++++++++++++++++++
+ drivers/fpga/lattice-sysconfig-spi.c               |  29 ++-
+ drivers/fpga/of-fpga-region.c                      |   3 +-
+ drivers/fpga/ts73xx-fpga.c                         |   9 +
+ 13 files changed, 499 insertions(+), 47 deletions(-)
+ delete mode 100644 Documentation/devicetree/bindings/fpga/altera-pr-ip.txt
+ delete mode 100644 Documentation/devicetree/bindings/fpga/altera-socfpga-fpga-mgr.txt
+ create mode 100644 Documentation/devicetree/bindings/fpga/altr,a10-pr-ip.yaml
+ create mode 100644 Documentation/devicetree/bindings/fpga/altr,socfpga-fpga-mgr.yaml
+ create mode 100644 Documentation/devicetree/bindings/fpga/efinix,trion-config.yaml
+ create mode 100644 Documentation/devicetree/bindings/fpga/technologic,ts7300-fpga.yaml
+ create mode 100644 drivers/fpga/efinix-spi.c
+Merging icc/icc-next (a24b55c5c084f Merge branch 'icc-eliza' into icc-next)
+$ git merge -m Merge branch 'icc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/djakov/icc.git icc/icc-next
+Merge made by the 'ort' strategy.
+ .../bindings/interconnect/qcom,hawi-rpmh.yaml      |  131 +
+ .../bindings/interconnect/qcom,msm8998-bwmon.yaml  |    1 +
+ .../bindings/interconnect/qcom,nord-rpmh.yaml      |  131 +
+ .../bindings/interconnect/qcom,sdm660.yaml         |   13 +
+ .../bindings/interconnect/qcom,shikra.yaml         |  134 +
+ .../bindings/interconnect/qcom,sm6115.yaml         |   33 +-
+ .../bindings/interconnect/qcom,x1e80100-rpmh.yaml  |   62 +
+ drivers/interconnect/core.c                        |    2 +-
+ drivers/interconnect/qcom/Kconfig                  |   95 +-
+ drivers/interconnect/qcom/Makefile                 |    6 +
+ drivers/interconnect/qcom/eliza.c                  |   15 +-
+ drivers/interconnect/qcom/hawi.c                   | 2028 +++++++++++++++
+ drivers/interconnect/qcom/msm8953.c                |    2 +-
+ drivers/interconnect/qcom/nord.c                   | 2682 ++++++++++++++++++++
+ drivers/interconnect/qcom/shikra.c                 | 1837 ++++++++++++++
+ drivers/interconnect/qcom/x1e80100.c               |  485 ++++
+ include/dt-bindings/interconnect/qcom,eliza-rpmh.h |    1 +
+ include/dt-bindings/interconnect/qcom,hawi-rpmh.h  |  165 ++
+ include/dt-bindings/interconnect/qcom,nord-rpmh.h  |  217 ++
+ include/dt-bindings/interconnect/qcom,shikra.h     |  121 +
+ 20 files changed, 8123 insertions(+), 38 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/interconnect/qcom,hawi-rpmh.yaml
+ create mode 100644 Documentation/devicetree/bindings/interconnect/qcom,nord-rpmh.yaml
+ create mode 100644 Documentation/devicetree/bindings/interconnect/qcom,shikra.yaml
+ create mode 100644 drivers/interconnect/qcom/hawi.c
+ create mode 100644 drivers/interconnect/qcom/nord.c
+ create mode 100644 drivers/interconnect/qcom/shikra.c
+ create mode 100644 include/dt-bindings/interconnect/qcom,hawi-rpmh.h
+ create mode 100644 include/dt-bindings/interconnect/qcom,nord-rpmh.h
+ create mode 100644 include/dt-bindings/interconnect/qcom,shikra.h
+Merging iio/togreg (8678fb5495889 iio: chemical: bme680: use BME680_NUM_CHANNELS for scan buffer)
+$ git merge -m Merge branch 'togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git iio/togreg
+Auto-merging MAINTAINERS
+Auto-merging drivers/iio/adc/Kconfig
+Auto-merging drivers/iio/adc/nxp-sar-adc.c
+Auto-merging drivers/iio/adc/ti-ads1298.c
+Auto-merging drivers/iio/common/ssp_sensors/ssp_dev.c
+Auto-merging drivers/iio/dac/ad5686.c
+Auto-merging drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+CONFLICT (content): Merge conflict in drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+Auto-merging drivers/iio/industrialio-buffer.c
+Auto-merging drivers/iio/light/veml6030.c
+Resolved 'drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master f1a716d8bc69c] Merge branch 'togreg' of https://git.kernel.org/pub/scm/linux/kernel/git/jic23/iio.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/ABI/testing/sysfs-bus-iio            |   51 +-
+ .../devicetree/bindings/iio/adc/adi,ad4080.yaml    |   53 +-
+ .../devicetree/bindings/iio/adc/adi,ad4130.yaml    |   19 +-
+ .../devicetree/bindings/iio/adc/adi,ad4691.yaml    |  180 ++
+ .../iio/adc/allwinner,sun20i-d1-gpadc.yaml         |   32 +-
+ .../devicetree/bindings/iio/adc/avia-hx711.yaml    |   37 +-
+ .../bindings/iio/adc/qcom,pm8018-adc.yaml          |   15 +
+ .../bindings/iio/chemical/sensirion,scd30.yaml     |    2 +-
+ .../devicetree/bindings/iio/dac/adi,ad5706r.yaml   |  105 ++
+ .../bindings/iio/dac/microchip,mcp47feb02.yaml     |   36 +-
+ .../bindings/iio/gyroscope/bosch,bmg160.yaml       |    6 +
+ .../devicetree/bindings/iio/light/tsl2772.yaml     |    2 +
+ .../bindings/iio/magnetometer/memsic,mmc5983.yaml  |   63 +
+ Documentation/driver-api/iio/buffers.rst           |    7 +-
+ Documentation/driver-api/iio/triggered-buffers.rst |    2 +-
+ Documentation/iio/ad4691.rst                       |  204 +++
+ Documentation/iio/adxl313.rst                      |   10 +-
+ Documentation/iio/adxl345.rst                      |    2 +-
+ Documentation/iio/adxl380.rst                      |    2 +-
+ Documentation/iio/iio_devbuf.rst                   |    3 +-
+ Documentation/iio/index.rst                        |    1 +
+ Documentation/misc-devices/apds990x.rst            |  128 --
+ Documentation/misc-devices/index.rst               |    1 -
+ MAINTAINERS                                        |   39 +-
+ drivers/iio/accel/adxl313_core.c                   |   10 +-
+ drivers/iio/accel/adxl313_i2c.c                    |   10 +-
+ drivers/iio/accel/adxl313_spi.c                    |   11 +-
+ drivers/iio/accel/adxl355_core.c                   |   30 +-
+ drivers/iio/accel/adxl355_i2c.c                    |   11 +-
+ drivers/iio/accel/adxl355_spi.c                    |   11 +-
+ drivers/iio/accel/adxl367.c                        |    4 +-
+ drivers/iio/accel/adxl372.c                        |   10 +-
+ drivers/iio/accel/adxl380.c                        |    4 +-
+ drivers/iio/adc/Kconfig                            |   19 +-
+ drivers/iio/adc/Makefile                           |    1 +
+ drivers/iio/adc/ad4062.c                           |    7 +-
+ drivers/iio/adc/ad4080.c                           |  257 ++-
+ drivers/iio/adc/ad4130.c                           |  493 +++++-
+ drivers/iio/adc/ad4170-4.c                         |   47 +-
+ drivers/iio/adc/ad4691.c                           | 1792 ++++++++++++++++++++
+ drivers/iio/adc/ad7191.c                           |   63 +-
+ drivers/iio/adc/ad7192.c                           |    9 +-
+ drivers/iio/adc/ad7280a.c                          |   75 +-
+ drivers/iio/adc/ad7292.c                           |   16 +-
+ drivers/iio/adc/ad7625.c                           |    4 +-
+ drivers/iio/adc/ad7768-1.c                         |    6 +-
+ drivers/iio/adc/ad7780.c                           |   46 +-
+ drivers/iio/adc/ad7791.c                           |   19 +-
+ drivers/iio/adc/ad7793.c                           |   37 +-
+ drivers/iio/adc/ad7949.c                           |   25 +-
+ drivers/iio/adc/ad799x.c                           |  135 +-
+ drivers/iio/adc/ad9467.c                           |    9 +-
+ drivers/iio/adc/hx711.c                            |   31 +-
+ drivers/iio/adc/ingenic-adc.c                      |   68 +-
+ drivers/iio/adc/ltc2309.c                          |   16 +-
+ drivers/iio/adc/mcp3422.c                          |   65 +-
+ drivers/iio/adc/nxp-sar-adc.c                      |    6 +-
+ drivers/iio/adc/pac1921.c                          |    2 +-
+ drivers/iio/adc/qcom-pm8xxx-xoadc.c                |   39 +-
+ drivers/iio/adc/rtq6056.c                          |   10 +-
+ drivers/iio/adc/sun20i-gpadc-iio.c                 |   28 +-
+ drivers/iio/adc/ti-ads1298.c                       |   19 +-
+ drivers/iio/adc/ti-ads7924.c                       |    5 +-
+ drivers/iio/adc/ti-ads7950.c                       |  256 ++-
+ drivers/iio/adc/ti-ads8688.c                       |   70 +-
+ drivers/iio/adc/xilinx-ams.c                       |  190 +--
+ drivers/iio/addac/ad74115.c                        |    5 +-
+ drivers/iio/chemical/bme680_core.c                 |    2 +-
+ drivers/iio/chemical/scd30_i2c.c                   |    5 +-
+ drivers/iio/chemical/sgp30.c                       |    3 +-
+ drivers/iio/common/ssp_sensors/ssp_dev.c           |    2 +-
+ drivers/iio/common/ssp_sensors/ssp_spi.c           |    8 +-
+ drivers/iio/dac/Kconfig                            |   11 +
+ drivers/iio/dac/Makefile                           |    1 +
+ drivers/iio/dac/ad3552r.c                          |    7 +-
+ drivers/iio/dac/ad5686.c                           |    4 +-
+ drivers/iio/dac/ad5706r.c                          |  253 +++
+ drivers/iio/dac/ad5755.c                           |    5 +-
+ drivers/iio/dac/ad5758.c                           |    4 +-
+ drivers/iio/dac/ad7303.c                           |    4 +-
+ drivers/iio/dac/ltc2664.c                          |    4 +-
+ drivers/iio/dac/mcp4821.c                          |  137 +-
+ drivers/iio/frequency/adrf6780.c                   |    7 +-
+ drivers/iio/gyro/adxrs290.c                        |   75 +-
+ drivers/iio/imu/bno055/bno055.c                    |   27 +-
+ drivers/iio/imu/st_lsm6dsx/Makefile                |    2 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h            |   28 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c     |   43 +-
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c       |   58 +
+ drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_fusion.c     |  241 +++
+ drivers/iio/industrialio-backend.c                 |   53 +-
+ drivers/iio/industrialio-buffer.c                  |   48 +-
+ drivers/iio/industrialio-core.c                    |    1 +
+ drivers/iio/light/apds9160.c                       |    2 +-
+ drivers/iio/light/cm3232.c                         |    3 +-
+ drivers/iio/light/iqs621-als.c                     |  111 +-
+ drivers/iio/light/opt3001.c                        |   20 +-
+ drivers/iio/light/si1133.c                         |   70 +-
+ drivers/iio/light/stk3310.c                        |  105 +-
+ drivers/iio/light/tsl2563.c                        |    8 +-
+ drivers/iio/light/tsl2583.c                        |    6 +-
+ drivers/iio/light/tsl2772.c                        |   16 +
+ drivers/iio/light/vcnl4000.c                       |  428 ++---
+ drivers/iio/light/veml6030.c                       |    6 +-
+ drivers/iio/magnetometer/Kconfig                   |   11 +
+ drivers/iio/magnetometer/Makefile                  |    1 +
+ drivers/iio/magnetometer/ak8975.c                  |  259 +--
+ drivers/iio/magnetometer/hid-sensor-magn-3d.c      |    6 +-
+ drivers/iio/magnetometer/mmc5983.c                 |  348 ++++
+ drivers/iio/magnetometer/rm3100-core.c             |   94 +-
+ drivers/iio/magnetometer/yamaha-yas530.c           |   22 +-
+ drivers/iio/orientation/hid-sensor-rotation.c      |   71 +-
+ drivers/iio/proximity/srf08.c                      |   17 +-
+ drivers/iio/temperature/maxim_thermocouple.c       |    6 +-
+ drivers/misc/Kconfig                               |   10 -
+ drivers/misc/Makefile                              |    1 -
+ drivers/misc/apds990x.c                            | 1284 --------------
+ drivers/staging/iio/addac/adt7316-spi.c            |   13 +-
+ drivers/staging/iio/frequency/ad9832.c             |   20 +-
+ drivers/staging/iio/frequency/ad9834.c             |   45 +-
+ include/linux/iio/backend.h                        |    1 +
+ include/linux/iio/buffer.h                         |   14 +-
+ include/linux/iio/iio.h                            |   33 +-
+ include/linux/platform_data/apds990x.h             |   65 -
+ include/uapi/linux/iio/types.h                     |    1 +
+ tools/iio/iio_event_monitor.c                      |    1 +
+ tools/iio/iio_generic_buffer.c                     |   70 +-
+ tools/iio/iio_utils.c                              |   12 +-
+ tools/iio/iio_utils.h                              |    4 +-
+ 129 files changed, 5913 insertions(+), 3275 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/iio/adc/adi,ad4691.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/dac/adi,ad5706r.yaml
+ create mode 100644 Documentation/devicetree/bindings/iio/magnetometer/memsic,mmc5983.yaml
+ create mode 100644 Documentation/iio/ad4691.rst
+ delete mode 100644 Documentation/misc-devices/apds990x.rst
+ create mode 100644 drivers/iio/adc/ad4691.c
+ create mode 100644 drivers/iio/dac/ad5706r.c
+ create mode 100644 drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_fusion.c
+ create mode 100644 drivers/iio/magnetometer/mmc5983.c
+ delete mode 100644 drivers/misc/apds990x.c
+ delete mode 100644 include/linux/platform_data/apds990x.h
+Merging phy-next/next (293e19f416fa3 MAINTAINERS: Hand over phy-zynqmp to Tomi Valkeinen)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/phy/linux-phy.git phy-next/next
+Auto-merging MAINTAINERS
+Auto-merging drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
+Auto-merging drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
+Auto-merging drivers/phy/spacemit/phy-k1-usb2.c
+Merge made by the 'ort' strategy.
+ .../bindings/display/msm/qcom,sa8775p-mdss.yaml    |   6 +-
+ .../bindings/phy/axiado,ax3000-emmc-phy.yaml       |  37 ++
+ .../bindings/phy/econet,en751221-pcie-phy.yaml     |  50 +++
+ .../devicetree/bindings/phy/fsl,lynx-28g.yaml      |  48 ++-
+ .../devicetree/bindings/phy/qcom,edp-phy.yaml      |   1 +
+ .../bindings/phy/qcom,sc8280xp-qmp-ufs-phy.yaml    |   1 +
+ .../phy/qcom,sc8280xp-qmp-usb43dp-phy.yaml         |   4 +
+ .../bindings/phy/qcom,snps-eusb2-phy.yaml          |   1 +
+ .../bindings/phy/rockchip,inno-usb2phy.yaml        |  44 ++-
+ .../devicetree/bindings/phy/spacemit,usb2-phy.yaml |   6 +-
+ .../devicetree/bindings/phy/ti,ds125df111.yaml     |  46 +++
+ .../devicetree/bindings/phy/ti,phy-gmii-sel.yaml   |  23 +-
+ .../devicetree/bindings/phy/ti,phy-j721e-wiz.yaml  |  19 +-
+ MAINTAINERS                                        |  27 +-
+ drivers/phy/Kconfig                                |  26 ++
+ drivers/phy/Makefile                               |   3 +
+ drivers/phy/axiado/Kconfig                         |  11 +
+ drivers/phy/axiado/Makefile                        |   1 +
+ drivers/phy/axiado/phy-axiado-emmc.c               | 217 +++++++++++
+ drivers/phy/freescale/phy-fsl-imx8qm-hsio.c        |  23 +-
+ drivers/phy/freescale/phy-fsl-lynx-28g.c           | 411 ++++++++++++++++++---
+ drivers/phy/mediatek/phy-mtk-xsphy.c               |  15 +-
+ drivers/phy/phy-airoha-pcie-regs.h                 |   2 +-
+ drivers/phy/phy-can-transceiver.c                  |  93 +++--
+ drivers/phy/phy-econet-pcie.c                      | 182 +++++++++
+ drivers/phy/phy-eyeq5-eth.c                        | 280 ++++++++++++++
+ drivers/phy/qualcomm/phy-qcom-m31-eusb2.c          |   2 +-
+ drivers/phy/qualcomm/phy-qcom-qmp-combo.c          |  18 +-
+ drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.c   | 110 +++---
+ drivers/phy/qualcomm/phy-qcom-qmp-pcie.c           | 212 +++++------
+ drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com-v2.h |   3 +
+ drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com.h    | 140 -------
+ .../phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v2.h    | 247 ++++++++++---
+ drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx.h   | 205 ----------
+ drivers/phy/qualcomm/phy-qcom-qmp-ufs.c            | 254 ++++++-------
+ drivers/phy/qualcomm/phy-qcom-qmp-usb.c            | 262 ++++++-------
+ drivers/phy/qualcomm/phy-qcom-qmp-usbc.c           | 310 ++++++++--------
+ drivers/phy/qualcomm/phy-qcom-qmp.h                |   3 -
+ drivers/phy/qualcomm/phy-qcom-usb-hs.c             |  15 +-
+ drivers/phy/renesas/phy-rcar-gen3-usb2.c           |  10 +-
+ drivers/phy/renesas/phy-rzg3e-usb3.c               |   8 +-
+ drivers/phy/rockchip/phy-rockchip-inno-usb2.c      | 191 +++++++---
+ drivers/phy/rockchip/phy-rockchip-naneng-combphy.c | 173 ++++-----
+ drivers/phy/spacemit/phy-k1-usb2.c                 |  34 +-
+ drivers/phy/st/Kconfig                             |   2 +-
+ drivers/phy/st/phy-miphy28lp.c                     |  12 +-
+ drivers/phy/tegra/Kconfig                          |   2 +-
+ drivers/phy/ti/Kconfig                             | 134 +++----
+ drivers/phy/ti/Makefile                            |  13 +-
+ drivers/phy/ti/phy-ds125df111.c                    | 294 +++++++++++++++
+ drivers/phy/ti/phy-gmii-sel.c                      |  13 +
+ drivers/phy/ti/phy-j721e-wiz.c                     |  24 ++
+ 52 files changed, 2897 insertions(+), 1371 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/phy/axiado,ax3000-emmc-phy.yaml
+ create mode 100644 Documentation/devicetree/bindings/phy/econet,en751221-pcie-phy.yaml
+ create mode 100644 Documentation/devicetree/bindings/phy/ti,ds125df111.yaml
+ create mode 100644 drivers/phy/axiado/Kconfig
+ create mode 100644 drivers/phy/axiado/Makefile
+ create mode 100644 drivers/phy/axiado/phy-axiado-emmc.c
+ create mode 100644 drivers/phy/phy-econet-pcie.c
+ create mode 100644 drivers/phy/phy-eyeq5-eth.c
+ delete mode 100644 drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com.h
+ delete mode 100644 drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx.h
+ create mode 100644 drivers/phy/ti/phy-ds125df111.c
+Merging soundwire/next (8a7fe10eec64b soundwire: intel_ace2x: release bpt_stream when close it)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/soundwire.git soundwire/next
+Merge made by the 'ort' strategy.
+ drivers/soundwire/generic_bandwidth_allocation.c | 56 +++++++++++-------------
+ drivers/soundwire/intel.c                        | 31 +++----------
+ drivers/soundwire/intel_ace2x.c                  | 40 +++++++++++------
+ drivers/soundwire/intel_auxdevice.c              |  3 ++
+ drivers/soundwire/slave.c                        |  4 +-
+ drivers/soundwire/stream.c                       | 13 +++++-
+ 6 files changed, 77 insertions(+), 70 deletions(-)
+Merging extcon/extcon-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'extcon-next' of https://git.kernel.org/pub/scm/linux/kernel/git/chanwoo/extcon.git extcon/extcon-next
+Already up to date.
+Merging gnss/gnss-next (5d6919055dec1 Linux 7.1-rc3)
+$ git merge -m Merge branch 'gnss-next' of https://git.kernel.org/pub/scm/linux/kernel/git/johan/gnss.git gnss/gnss-next
+Already up to date.
+Merging vfio/next (b9285405c5f61 vfio/xe: avoid duplicate reset in xe_vfio_pci_reset_done)
+$ git merge -m Merge branch 'next' of https://github.com/awilliam/linux-vfio.git vfio/next
+Merge made by the 'ort' strategy.
+ drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c     |  39 +---
+ drivers/vfio/pci/nvgrace-gpu/main.c                |  20 +-
+ drivers/vfio/pci/vfio_pci_core.c                   |  11 +-
+ drivers/vfio/pci/vfio_pci_dmabuf.c                 |   2 +-
+ drivers/vfio/pci/vfio_pci_rdwr.c                   |  30 +--
+ drivers/vfio/pci/virtio/legacy_io.c                |  13 +-
+ drivers/vfio/pci/xe/main.c                         |   2 -
+ include/linux/vfio_pci_core.h                      |  20 +-
+ tools/testing/selftests/vfio/Makefile              |  11 +-
+ tools/testing/selftests/vfio/lib/include/libvfio.h |   1 +
+ .../selftests/vfio/lib/include/libvfio/assert.h    |   5 +
+ .../selftests/vfio/lib/include/libvfio/sysfs.h     |  12 ++
+ .../vfio/lib/include/libvfio/vfio_pci_device.h     |  11 ++
+ tools/testing/selftests/vfio/lib/libvfio.mk        |   1 +
+ tools/testing/selftests/vfio/lib/sysfs.c           | 150 ++++++++++++++
+ tools/testing/selftests/vfio/lib/vfio_pci_device.c | 157 +++++++++++----
+ .../testing/selftests/vfio/vfio_dma_mapping_test.c |   6 +-
+ .../testing/selftests/vfio/vfio_pci_device_test.c  |  21 +-
+ .../selftests/vfio/vfio_pci_sriov_uapi_test.c      | 217 +++++++++++++++++++++
+ 19 files changed, 594 insertions(+), 135 deletions(-)
+ create mode 100644 tools/testing/selftests/vfio/lib/include/libvfio/sysfs.h
+ create mode 100644 tools/testing/selftests/vfio/lib/sysfs.c
+ create mode 100644 tools/testing/selftests/vfio/vfio_pci_sriov_uapi_test.c
+Merging w1/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krzk/linux-w1.git w1/for-next
+Already up to date.
+Merging spmi/spmi-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'spmi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sboyd/spmi.git spmi/spmi-next
+Already up to date.
+Merging staging/staging-next (6abf0b2df0b1c staging: rtl8723bs: fix type issue in DYNAMIC_BB_DYNAMIC_TXPWR)
+$ git merge -m Merge branch 'staging-next' of https://git.kernel.org/pub/scm/linux/kernel/git/gregkh/staging.git staging/staging-next
+Auto-merging drivers/staging/rtl8723bs/core/rtw_mlme.c
+Merge made by the 'ort' strategy.
+ .../cavium-octeon/executive/cvmx-helper-util.c     |   2 +-
+ arch/mips/cavium-octeon/executive/cvmx-helper.c    |   8 +-
+ arch/mips/cavium-octeon/executive/cvmx-pko.c       |   6 +-
+ arch/mips/cavium-octeon/executive/cvmx-spi.c       |  16 +-
+ arch/mips/include/asm/octeon/cvmx-helper-util.h    |   2 +-
+ arch/mips/include/asm/octeon/cvmx-helper.h         |   6 +-
+ arch/mips/include/asm/octeon/cvmx-pip.h            |   6 +-
+ arch/mips/include/asm/octeon/cvmx-pko.h            |  26 +--
+ arch/mips/include/asm/octeon/cvmx-pow.h            |  12 +-
+ arch/mips/include/asm/octeon/cvmx-spi.h            |  38 ++--
+ drivers/staging/fbtft/fb_ssd1351.c                 |   3 +-
+ drivers/staging/fbtft/fbtft-core.c                 |   3 +-
+ drivers/staging/fbtft/fbtft-io.c                   |   7 -
+ drivers/staging/fbtft/fbtft.h                      |   1 -
+ .../greybus/Documentation/firmware/authenticate.c  |   2 +-
+ .../greybus/Documentation/sysfs-bus-greybus        |   2 +-
+ drivers/staging/most/dim2/dim2.c                   |   6 +-
+ drivers/staging/most/dim2/errors.h                 |   2 +-
+ drivers/staging/most/dim2/hal.c                    |   2 +-
+ drivers/staging/most/dim2/hal.h                    |   2 +-
+ drivers/staging/most/dim2/reg.h                    |   2 +-
+ drivers/staging/most/dim2/sysfs.h                  |   2 +-
+ drivers/staging/most/net/net.c                     |  18 +-
+ drivers/staging/most/video/video.c                 |  23 +--
+ drivers/staging/nvec/nvec.c                        |   8 +-
+ drivers/staging/octeon/ethernet.c                  |   6 +-
+ drivers/staging/octeon/octeon-stubs.h              |  48 ++---
+ drivers/staging/rtl8723bs/core/rtw_ap.c            |  90 ++-------
+ drivers/staging/rtl8723bs/core/rtw_btcoex.c        |  15 +-
+ drivers/staging/rtl8723bs/core/rtw_cmd.c           |  12 +-
+ drivers/staging/rtl8723bs/core/rtw_efuse.c         |  25 ++-
+ drivers/staging/rtl8723bs/core/rtw_ieee80211.c     |   3 +-
+ drivers/staging/rtl8723bs/core/rtw_io.c            |  20 +-
+ drivers/staging/rtl8723bs/core/rtw_ioctl_set.c     |  58 +++---
+ drivers/staging/rtl8723bs/core/rtw_mlme.c          |  46 ++---
+ drivers/staging/rtl8723bs/core/rtw_mlme_ext.c      | 118 ++---------
+ drivers/staging/rtl8723bs/core/rtw_pwrctrl.c       |   2 +-
+ drivers/staging/rtl8723bs/core/rtw_recv.c          | 156 ++++-----------
+ drivers/staging/rtl8723bs/core/rtw_security.c      | 221 ++++++++++-----------
+ drivers/staging/rtl8723bs/core/rtw_sta_mgt.c       |   3 +-
+ drivers/staging/rtl8723bs/core/rtw_wlan_util.c     |  57 ++----
+ drivers/staging/rtl8723bs/core/rtw_xmit.c          | 183 +++++++++--------
+ drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c    |  58 +++---
+ drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.h    |  18 +-
+ drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c    |  27 ++-
+ drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.h    |  16 +-
+ drivers/staging/rtl8723bs/hal/HalBtcOutSrc.h       |  10 +-
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_BB.c   |  30 +--
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.c  |  24 +--
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.h  |  12 +-
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.c   |  24 +--
+ drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.h   |  20 +-
+ drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c     |  32 ++-
+ drivers/staging/rtl8723bs/hal/hal_btcoex.c         |  25 +--
+ drivers/staging/rtl8723bs/hal/hal_com.c            |  83 ++------
+ drivers/staging/rtl8723bs/hal/hal_com_phycfg.c     |  21 +-
+ drivers/staging/rtl8723bs/hal/hal_intf.c           |  45 +----
+ drivers/staging/rtl8723bs/hal/hal_pwr_seq.c        |  18 +-
+ drivers/staging/rtl8723bs/hal/odm.c                |  72 ++-----
+ drivers/staging/rtl8723bs/hal/odm.h                |  54 +++--
+ drivers/staging/rtl8723bs/hal/odm_CfoTracking.c    |   2 +-
+ drivers/staging/rtl8723bs/hal/odm_DIG.c            |  54 +++--
+ drivers/staging/rtl8723bs/hal/odm_DIG.h            |   4 +-
+ .../rtl8723bs/hal/odm_DynamicBBPowerSaving.c       |  12 +-
+ drivers/staging/rtl8723bs/hal/odm_HWConfig.c       |   4 +-
+ drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h   |   4 +-
+ drivers/staging/rtl8723bs/hal/odm_interface.h      |   8 -
+ drivers/staging/rtl8723bs/hal/odm_types.h          |   6 -
+ drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c       |  34 ++--
+ drivers/staging/rtl8723bs/hal/rtl8723b_dm.c        |  14 +-
+ drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c  | 105 +++++-----
+ drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c    |  67 +++----
+ drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c    |   9 +-
+ drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c     |   2 +-
+ drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c     |  14 +-
+ drivers/staging/rtl8723bs/hal/sdio_halinit.c       |  79 +++-----
+ drivers/staging/rtl8723bs/hal/sdio_ops.c           | 152 ++------------
+ drivers/staging/rtl8723bs/include/HalVerDef.h      |  85 --------
+ drivers/staging/rtl8723bs/include/cmd_osdep.h      |   1 -
+ drivers/staging/rtl8723bs/include/drv_types.h      |   9 +-
+ drivers/staging/rtl8723bs/include/hal_com.h        |   3 -
+ drivers/staging/rtl8723bs/include/hal_com_reg.h    | 104 +++++-----
+ drivers/staging/rtl8723bs/include/hal_data.h       |   6 +-
+ drivers/staging/rtl8723bs/include/hal_intf.h       |  29 +--
+ drivers/staging/rtl8723bs/include/hal_phy.h        |   4 +-
+ drivers/staging/rtl8723bs/include/hal_pwr_seq.h    | 182 ++++++++---------
+ drivers/staging/rtl8723bs/include/ieee80211.h      |  54 ++---
+ drivers/staging/rtl8723bs/include/osdep_service.h  |  56 ------
+ drivers/staging/rtl8723bs/include/rtl8192c_recv.h  |   1 -
+ drivers/staging/rtl8723bs/include/rtl8723b_dm.h    |   1 -
+ drivers/staging/rtl8723bs/include/rtl8723b_hal.h   |   3 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_recv.h  |   2 +-
+ drivers/staging/rtl8723bs/include/rtl8723b_spec.h  |  88 ++++----
+ drivers/staging/rtl8723bs/include/rtw_btcoex.h     |  11 +-
+ drivers/staging/rtl8723bs/include/rtw_cmd.h        |   6 +-
+ drivers/staging/rtl8723bs/include/rtw_efuse.h      |  14 +-
+ drivers/staging/rtl8723bs/include/rtw_ht.h         |  16 +-
+ drivers/staging/rtl8723bs/include/rtw_io.h         |  16 --
+ drivers/staging/rtl8723bs/include/rtw_ioctl_set.h  |   1 -
+ drivers/staging/rtl8723bs/include/rtw_mlme.h       |   6 +-
+ drivers/staging/rtl8723bs/include/rtw_mlme_ext.h   |  17 +-
+ drivers/staging/rtl8723bs/include/rtw_pwrctrl.h    |   4 +-
+ drivers/staging/rtl8723bs/include/rtw_recv.h       |   8 +-
+ drivers/staging/rtl8723bs/include/rtw_wifi_regd.h  |   1 -
+ drivers/staging/rtl8723bs/include/rtw_xmit.h       |   1 -
+ drivers/staging/rtl8723bs/include/sdio_ops.h       |   4 +-
+ drivers/staging/rtl8723bs/include/sdio_ops_linux.h |   1 -
+ drivers/staging/rtl8723bs/include/wifi.h           |  28 +--
+ drivers/staging/rtl8723bs/include/xmit_osdep.h     |   1 -
+ drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c  |  63 +++---
+ drivers/staging/rtl8723bs/os_dep/os_intfs.c        |  27 +--
+ drivers/staging/rtl8723bs/os_dep/sdio_intf.c       |  20 +-
+ drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c  |   4 +-
+ drivers/staging/rtl8723bs/os_dep/wifi_regd.c       |  18 +-
+ drivers/staging/rtl8723bs/os_dep/xmit_linux.c      |  15 +-
+ drivers/staging/sm750fb/ddk750_power.c             |  17 --
+ drivers/staging/sm750fb/ddk750_power.h             |   5 -
+ drivers/staging/sm750fb/sm750.c                    |  42 ++--
+ drivers/staging/sm750fb/sm750.h                    |  12 +-
+ drivers/staging/sm750fb/sm750_accel.c              |  21 +-
+ drivers/staging/sm750fb/sm750_cursor.c             |  43 ----
+ drivers/staging/sm750fb/sm750_cursor.h             |   2 -
+ drivers/staging/sm750fb/sm750_hw.c                 |  21 +-
+ drivers/staging/vme_user/vme.c                     |  24 +--
+ 124 files changed, 1293 insertions(+), 2231 deletions(-)
+ delete mode 100644 drivers/staging/rtl8723bs/include/HalVerDef.h
+Merging counter-next/counter-next (ee4133b921a17 counter: intel-qep: Replace manual mutex logic with lock guards)
+$ git merge -m Merge branch 'counter-next' of https://git.kernel.org/pub/scm/linux/kernel/git/wbg/counter.git counter-next/counter-next
+Merge made by the 'ort' strategy.
+ drivers/counter/intel-qep.c | 50 ++++++++++++++++-----------------------------
+ 1 file changed, 18 insertions(+), 32 deletions(-)
+Merging mux/for-next (59b723cd2adba Linux 6.12-rc6)
+$ git merge -m Merge branch 'for-next' of https://gitlab.com/peda-linux/mux.git mux/for-next
+Already up to date.
+Merging dmaengine/next (362ee0c0dc522 dmaengine: Move MODULE_DEVICE_TABLE next to the table itself)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine.git dmaengine/next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/dma/qcom,bam-dma.yaml      | 21 ++++++++++++
+ .../devicetree/bindings/dma/qcom,gpi.yaml          |  2 ++
+ .../devicetree/bindings/dma/spacemit,k1-pdma.yaml  |  4 ++-
+ drivers/dma/hisi_dma.c                             |  2 +-
+ drivers/dma/mmp_pdma.c                             | 37 ++++++++++++++++++++--
+ drivers/dma/pch_dma.c                              |  2 +-
+ drivers/dma/qcom/bam_dma.c                         | 30 ++++++++++++++++++
+ 7 files changed, 92 insertions(+), 6 deletions(-)
+Merging cgroup/for-next (936f0880adaf8 Merge branch 'for-7.1-fixes' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git cgroup/for-next
+Auto-merging include/linux/cgroup.h
+Auto-merging kernel/cgroup/cgroup.c
+Auto-merging mm/memcontrol.c
+Auto-merging tools/testing/selftests/cgroup/lib/cgroup_util.c
+Auto-merging tools/testing/selftests/cgroup/test_freezer.c
+Auto-merging tools/testing/selftests/cgroup/test_memcontrol.c
+CONFLICT (content): Merge conflict in tools/testing/selftests/cgroup/test_memcontrol.c
+Resolved 'tools/testing/selftests/cgroup/test_memcontrol.c' using previous resolution.
+Automatic merge failed; fix conflicts and then commit the result.
+$ git commit --no-edit -v -a
+[master 9fdfe54ab18bf] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/tj/cgroup.git
+$ git diff -M --stat --summary HEAD^..
+ Documentation/admin-guide/cgroup-v2.rst           |  53 ++++
+ block/blk-cgroup.c                                |   2 +-
+ include/linux/cgroup-defs.h                       |  30 ++-
+ include/linux/cgroup.h                            |  28 +-
+ include/linux/cgroup_rdma.h                       |   4 +
+ kernel/cgroup/cgroup.c                            | 179 ++++++------
+ kernel/cgroup/cpuset-v1.c                         |   2 +-
+ kernel/cgroup/cpuset.c                            |   5 +-
+ kernel/cgroup/rdma.c                              | 315 +++++++++++++++++-----
+ kernel/cgroup/rstat.c                             |  37 ++-
+ mm/memcontrol.c                                   |   6 +-
+ tools/testing/selftests/cgroup/lib/cgroup_util.c  |   9 +-
+ tools/testing/selftests/cgroup/test_cpu.c         |   2 +-
+ tools/testing/selftests/cgroup/test_cpuset_prs.sh |   2 +-
+ tools/testing/selftests/cgroup/test_freezer.c     |   2 +-
+ tools/testing/selftests/cgroup/test_memcontrol.c  |  53 ++--
+ 16 files changed, 511 insertions(+), 218 deletions(-)
+Merging scsi/for-next (b71cb088b2e34 scsi: target: tcm_loop: Fix NULL ptr dereference)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi.git scsi/for-next
+Already up to date.
+Merging scsi-mkp/for-next (016d484531e31 scsi: isci: Remove unused macro scu_get_command_request_logical_port())
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mkp/scsi.git scsi-mkp/for-next
+Auto-merging drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+Auto-merging drivers/ufs/core/ufshcd.c
+Auto-merging drivers/ufs/host/ufs-qcom.c
+Merge made by the 'ort' strategy.
+ .../bindings/ufs/qcom,sa8255p-ufshc.yaml           |   6 +-
+ .../devicetree/bindings/ufs/qcom,sm8650-ufshc.yaml |   3 +
+ .../bindings/ufs/samsung,exynos-ufs.yaml           |   1 +
+ Documentation/scsi/st.rst                          |   2 +-
+ drivers/scsi/Kconfig                               |   1 -
+ drivers/scsi/Makefile                              |   4 +-
+ drivers/scsi/advansys.c                            |   1 -
+ drivers/scsi/hisi_sas/hisi_sas_v3_hw.c             |  12 +-
+ drivers/scsi/isci/scu_task_context.h               |   2 -
+ drivers/scsi/libiscsi.c                            |   4 +-
+ drivers/scsi/mvsas/mv_sas.h                        |   6 +-
+ drivers/scsi/ncr53c8xx.c                           |   2 +-
+ drivers/scsi/ncr53c8xx.h                           |   4 +-
+ drivers/scsi/pm8001/pm8001_ctl.c                   |   8 +
+ drivers/scsi/pm8001/pm80xx_hwi.c                   |   7 +
+ drivers/scsi/scsi_scan.c                           |   2 +-
+ drivers/scsi/scsi_transport_srp.c                  |  10 +-
+ drivers/scsi/snic/vnic_dev.c                       |   1 -
+ drivers/scsi/storvsc_drv.c                         |   8 +-
+ drivers/ufs/core/ufs-sysfs.c                       |  30 ++-
+ drivers/ufs/core/ufs-txeq.c                        | 287 +++++++++++++++++++++
+ drivers/ufs/core/ufshcd-priv.h                     |   5 +-
+ drivers/ufs/core/ufshcd.c                          | 164 +++++++-----
+ drivers/ufs/host/Kconfig                           |   2 +-
+ drivers/ufs/host/tc-dwc-g210-pci.c                 |   4 +-
+ drivers/ufs/host/ufs-exynos.c                      | 110 ++++++++
+ drivers/ufs/host/ufs-qcom.c                        |  10 +
+ drivers/ufs/host/ufs-qcom.h                        |  11 +
+ drivers/ufs/host/ufshcd-pci.c                      |  29 +--
+ include/ufs/ufs.h                                  |   2 +
+ include/ufs/ufshcd.h                               |   2 +
+ 31 files changed, 629 insertions(+), 111 deletions(-)
+Merging vhost/linux-next (ebcff9dacaf2c vduse: avoid adding implicit padding)
+$ git merge -m Merge branch 'linux-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git vhost/linux-next
+Already up to date.
+Merging rpmsg/for-next (54dacf6efe719 Merge branch 'rproc-next' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/remoteproc/linux.git rpmsg/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/remoteproc/fsl,imx-rproc.yaml         |  3 +
+ drivers/firmware/xilinx/zynqmp.c                   | 28 +++++++
+ drivers/remoteproc/Kconfig                         |  1 -
+ drivers/remoteproc/imx_rproc.c                     | 92 +++++++++++++++++++++-
+ drivers/remoteproc/imx_rproc.h                     |  2 +
+ drivers/remoteproc/xlnx_r5_remoteproc.c            | 46 ++++++++---
+ include/linux/firmware/xlnx-zynqmp.h               | 21 +++++
+ 7 files changed, 180 insertions(+), 13 deletions(-)
+Merging gpio-brgl/gpio/for-next (c010a78304a64 gpio: Initialize all i2c_device_id arrays using member names)
+$ git merge -m Merge branch 'gpio/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git gpio-brgl/gpio/for-next
+Auto-merging drivers/gpio/gpio-pca953x.c
+Merge made by the 'ort' strategy.
+ .../bindings/gpio/fairchild,74hc595.yaml           |  13 ++
+ .../devicetree/bindings/gpio/gpio-zynq.yaml        |  17 +-
+ .../bindings/gpio/nvidia,tegra186-gpio.yaml        |   6 +
+ .../devicetree/bindings/gpio/snps,dw-apb-gpio.yaml |   6 +
+ .../bindings/gpio/waveshare,dsi-touch-gpio.yaml    |  73 ++++++++
+ drivers/gpio/Kconfig                               |  22 ++-
+ drivers/gpio/Makefile                              |   1 +
+ drivers/gpio/gpio-74x164.c                         |  17 +-
+ drivers/gpio/gpio-adnp.c                           |   4 +-
+ drivers/gpio/gpio-altera.c                         | 124 ++++--------
+ drivers/gpio/gpio-amd8111.c                        |   4 +-
+ drivers/gpio/gpio-ds4520.c                         |   2 +-
+ drivers/gpio/gpio-ep93xx.c                         |   3 +-
+ drivers/gpio/gpio-fxl6408.c                        |   2 +-
+ drivers/gpio/gpio-gw-pld.c                         |   2 +-
+ drivers/gpio/gpio-ixp4xx.c                         |   7 +-
+ drivers/gpio/gpio-max7300.c                        |   2 +-
+ drivers/gpio/gpio-max732x.c                        |  20 +-
+ drivers/gpio/gpio-pca953x.c                        |  87 +++++----
+ drivers/gpio/gpio-pca9570.c                        |   6 +-
+ drivers/gpio/gpio-pcf857x.c                        |  26 +--
+ drivers/gpio/gpio-regmap.c                         |  74 +++++++-
+ drivers/gpio/gpio-sim.c                            |  24 +--
+ drivers/gpio/gpio-tegra186.c                       |  68 +++++++
+ drivers/gpio/gpio-tpic2810.c                       |   2 +-
+ drivers/gpio/gpio-ts4900.c                         |   2 +-
+ drivers/gpio/gpio-usbio.c                          |   1 +
+ drivers/gpio/gpio-waveshare-dsi.c                  | 208 +++++++++++++++++++++
+ drivers/gpio/gpio-xilinx.c                         |   6 +-
+ drivers/gpio/gpio-zevio.c                          |   4 +-
+ drivers/gpio/gpio-zynq.c                           |  12 ++
+ drivers/gpio/gpiolib-devres.c                      |   7 +
+ drivers/gpio/gpiolib.c                             |  22 +++
+ include/dt-bindings/gpio/nvidia,tegra238-gpio.h    |  58 ++++++
+ include/linux/gpio.h                               | 162 +---------------
+ include/linux/gpio/consumer.h                      |  14 ++
+ include/linux/gpio/legacy.h                        | 173 +++++++++++++++++
+ include/linux/gpio/regmap.h                        |   7 +
+ 38 files changed, 922 insertions(+), 366 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/gpio/waveshare,dsi-touch-gpio.yaml
+ create mode 100644 drivers/gpio/gpio-waveshare-dsi.c
+ create mode 100644 include/dt-bindings/gpio/nvidia,tegra238-gpio.h
+ create mode 100644 include/linux/gpio/legacy.h
+Merging gpio-intel/for-next (3bb62e3f99a55 gpiolib: acpi: Only trigger ActiveBoth interrupts on boot)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-gpio-intel.git gpio-intel/for-next
+Already up to date.
+Merging pinctrl/for-next (73d4991a6949e Merge branch 'devel' into for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl.git pinctrl/for-next
+Merge made by the 'ort' strategy.
+ .../pinctrl/aspeed,ast2700-soc0-pinctrl.yaml       |  188 +++
+ .../bindings/pinctrl/mediatek,mt65xx-pinctrl.yaml  |    1 +
+ .../bindings/pinctrl/mediatek,mt8188-pinctrl.yaml  |    5 +
+ .../pinctrl/nvidia,tegra234-pinmux-aon.yaml        |    6 +-
+ .../bindings/pinctrl/nvidia,tegra234-pinmux.yaml   |    6 +-
+ .../bindings/pinctrl/pinctrl-multiplexer.yaml      |   57 +
+ .../devicetree/bindings/pinctrl/pinctrl.yaml       |    2 +-
+ .../bindings/pinctrl/qcom,eliza-tlmm.yaml          |   26 +-
+ .../bindings/pinctrl/qcom,ipq4019-pinctrl.yaml     |    5 -
+ .../bindings/pinctrl/qcom,ipq9650-tlmm.yaml        |  118 ++
+ .../bindings/pinctrl/qcom,nord-tlmm.yaml           |  141 ++
+ .../bindings/pinctrl/qcom,sdm845-pinctrl.yaml      |    5 -
+ .../pinctrl/qcom,sm6350-lpass-lpi-pinctrl.yaml     |  124 ++
+ .../bindings/pinctrl/qcom,tlmm-common.yaml         |    6 +
+ drivers/mux/core.c                                 |   43 +-
+ drivers/pinctrl/Kconfig                            |    9 +
+ drivers/pinctrl/Makefile                           |    9 +-
+ drivers/pinctrl/aspeed/Kconfig                     |    9 +
+ drivers/pinctrl/aspeed/Makefile                    |    1 +
+ drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc0.c    |  749 +++++++++
+ drivers/pinctrl/bcm/pinctrl-bcm4908.c              |    2 +-
+ drivers/pinctrl/bcm/pinctrl-ns.c                   |    2 +-
+ drivers/pinctrl/freescale/pinctrl-imx1-core.c      |   48 +-
+ drivers/pinctrl/mediatek/mtk-eint.c                |    6 +-
+ drivers/pinctrl/mediatek/pinctrl-airoha.c          |    2 +-
+ drivers/pinctrl/mediatek/pinctrl-moore.c           |   21 +-
+ drivers/pinctrl/mediatek/pinctrl-mtk-common.c      |   13 +-
+ drivers/pinctrl/mediatek/pinctrl-paris.c           |    8 +-
+ drivers/pinctrl/meson/pinctrl-amlogic-a4.c         |    6 +-
+ drivers/pinctrl/pinconf-generic.c                  |    7 +-
+ drivers/pinctrl/pinconf.h                          |   18 +
+ drivers/pinctrl/pinctrl-amd.c                      |   35 +
+ drivers/pinctrl/pinctrl-equilibrium.c              |    3 +-
+ drivers/pinctrl/pinctrl-generic-mux.c              |  184 ++
+ drivers/pinctrl/pinctrl-generic.c                  |   95 +-
+ drivers/pinctrl/pinctrl-ingenic.c                  |    3 +-
+ drivers/pinctrl/pinctrl-rockchip.c                 |    2 +-
+ drivers/pinctrl/pinctrl-single.c                   |    3 +-
+ drivers/pinctrl/pinmux.c                           |    5 +
+ drivers/pinctrl/qcom/Kconfig                       |   34 +-
+ drivers/pinctrl/qcom/Kconfig.msm                   |  164 +-
+ drivers/pinctrl/qcom/Makefile                      |    3 +
+ drivers/pinctrl/qcom/pinctrl-apq8064.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-apq8084.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-eliza.c               |  194 ++-
+ drivers/pinctrl/qcom/pinctrl-glymur.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-hawi.c                |    2 +-
+ drivers/pinctrl/qcom/pinctrl-ipq4019.c             |    4 +-
+ drivers/pinctrl/qcom/pinctrl-ipq6018.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-ipq8064.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-ipq8074.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-ipq9650.c             |  762 +++++++++
+ drivers/pinctrl/qcom/pinctrl-kaanapali.c           |    2 +-
+ drivers/pinctrl/qcom/pinctrl-lpass-lpi.c           |    2 +
+ drivers/pinctrl/qcom/pinctrl-lpass-lpi.h           |   20 +
+ drivers/pinctrl/qcom/pinctrl-mdm9607.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-mdm9615.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-milos.c               |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm.h                 |    5 +
+ drivers/pinctrl/qcom/pinctrl-msm8226.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm8660.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm8916.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm8953.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm8960.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm8976.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm8994.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm8996.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm8998.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-msm8x74.c             |    2 +-
+ drivers/pinctrl/qcom/pinctrl-nord.c                | 1770 ++++++++++++++++++++
+ drivers/pinctrl/qcom/pinctrl-qcm2290.c             |   25 +-
+ drivers/pinctrl/qcom/pinctrl-qcs404.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-qcs615.c              |    8 +-
+ drivers/pinctrl/qcom/pinctrl-qdu1000.c             |    6 -
+ drivers/pinctrl/qcom/pinctrl-sc7180.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sc7280.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sdm660.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sdm845.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sdx55.c               |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sdx65.c               |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm4450.c              |    7 -
+ drivers/pinctrl/qcom/pinctrl-sm6115.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm6125.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm6350-lpass-lpi.c    |  149 ++
+ drivers/pinctrl/qcom/pinctrl-sm6350.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm6375.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm8150.c              |   10 +-
+ drivers/pinctrl/qcom/pinctrl-sm8250.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm8350.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm8450.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm8550.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm8650.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-sm8750.c              |    2 +-
+ drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c           |   10 +-
+ drivers/pinctrl/qcom/pinctrl-x1e80100.c            |    2 +-
+ drivers/pinctrl/realtek/Kconfig                    |   12 +-
+ drivers/pinctrl/realtek/pinctrl-rtd.c              |   12 +-
+ drivers/pinctrl/sophgo/pinctrl-cv18xx.c            |   14 +-
+ drivers/pinctrl/starfive/pinctrl-starfive-jh7110.c |   12 +-
+ drivers/pinctrl/starfive/pinctrl-starfive-jh7110.h |    2 +-
+ drivers/pinctrl/sunxi/pinctrl-sunxi.c              |    2 +-
+ drivers/pinctrl/tegra/Kconfig                      |   22 +-
+ drivers/pinctrl/vt8500/Kconfig                     |   13 +-
+ .../dt-bindings/pinctrl/mediatek,mt6392-pinfunc.h  |   39 +
+ include/linux/mux/consumer.h                       |    8 +-
+ include/linux/pinctrl/pinmux.h                     |    5 +
+ 106 files changed, 5006 insertions(+), 359 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/aspeed,ast2700-soc0-pinctrl.yaml
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/pinctrl-multiplexer.yaml
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,ipq9650-tlmm.yaml
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,nord-tlmm.yaml
+ create mode 100644 Documentation/devicetree/bindings/pinctrl/qcom,sm6350-lpass-lpi-pinctrl.yaml
+ create mode 100644 drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc0.c
+ create mode 100644 drivers/pinctrl/pinctrl-generic-mux.c
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-ipq9650.c
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-nord.c
+ create mode 100644 drivers/pinctrl/qcom/pinctrl-sm6350-lpass-lpi.c
+ create mode 100644 include/dt-bindings/pinctrl/mediatek,mt6392-pinfunc.h
+Merging pinctrl-intel/for-next (2b9ef70d3abe1 pinctrl: intel: move PWM base computation past feature check)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/intel.git pinctrl-intel/for-next
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/intel/pinctrl-intel.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+Merging pinctrl-renesas/renesas-pinctrl (c7c8352fe569d pinctrl: renesas: sh-pfc: Implement .pin_config_group_get() callback)
+$ git merge -m Merge branch 'renesas-pinctrl' of https://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-drivers.git pinctrl-renesas/renesas-pinctrl
+Merge made by the 'ort' strategy.
+ drivers/pinctrl/renesas/pinctrl-rzg2l.c | 577 ++++++++++++++++++++++++++++++--
+ drivers/pinctrl/renesas/pinctrl-rzv2m.c |   3 +-
+ drivers/pinctrl/renesas/pinctrl.c       |  25 ++
+ 3 files changed, 583 insertions(+), 22 deletions(-)
+Merging pinctrl-samsung/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pinctrl/samsung.git pinctrl-samsung/for-next
+Already up to date.
+Merging pwm/pwm/for-next (3936b25815ee6 Merge branch 'pwm/fixes' of https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux)
+$ git merge -m Merge branch 'pwm/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ukleinek/linux.git pwm/pwm/for-next
+Merge made by the 'ort' strategy.
+ .../devicetree/bindings/pwm/marvell,pxa-pwm.yaml   |  41 +++-
+ .../devicetree/bindings/pwm/qcom,ipq6018-pwm.yaml  |  51 ++++
+ drivers/pwm/Kconfig                                |  12 +
+ drivers/pwm/Makefile                               |   1 +
+ drivers/pwm/pwm-atmel-tcb.c                        |   2 +-
+ drivers/pwm/pwm-dwc.c                              |   2 +-
+ drivers/pwm/pwm-ipq.c                              | 264 +++++++++++++++++++++
+ drivers/pwm/pwm-lpss-pci.c                         |  18 +-
+ drivers/pwm/pwm-pca9685.c                          |   2 +-
+ drivers/pwm/pwm-pxa.c                              |   8 +-
+ drivers/pwm/pwm-stm32.c                            |  27 +--
+ 11 files changed, 390 insertions(+), 38 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/pwm/qcom,ipq6018-pwm.yaml
+ create mode 100644 drivers/pwm/pwm-ipq.c
+Merging ktest/for-next (932cdaf3e273a ktest: Add logfile to failure directory)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rostedt/linux-ktest.git ktest/for-next
+Already up to date.
+Merging kselftest/next (5f4974231fb36 kselftest: fix doc for ksft_test_result_report())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kselftest/next
+Auto-merging tools/testing/selftests/kselftest.h
+Merge made by the 'ort' strategy.
+ Documentation/dev-tools/kselftest.rst         |  12 +++
+ tools/testing/selftests/kselftest.h           |   4 +-
+ tools/testing/selftests/resctrl/cache.c       |  30 +++---
+ tools/testing/selftests/resctrl/cat_test.c    |  41 ++------
+ tools/testing/selftests/resctrl/cmt_test.c    |  36 ++++++-
+ tools/testing/selftests/resctrl/fill_buf.c    |   4 +-
+ tools/testing/selftests/resctrl/mba_test.c    |   6 +-
+ tools/testing/selftests/resctrl/mbm_test.c    |   6 +-
+ tools/testing/selftests/resctrl/resctrl.h     |  20 ++--
+ tools/testing/selftests/resctrl/resctrl_val.c | 141 +++++++++++++++++++-------
+ 10 files changed, 196 insertions(+), 104 deletions(-)
+Merging kunit/test (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'test' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit/test
+Already up to date.
+Merging kunit-next/kunit (5c1553dd5db30 kunit: Add documentation for warning backtrace suppression API)
+$ git merge -m Merge branch 'kunit' of https://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest.git kunit-next/kunit
+Auto-merging lib/bug.c
+Merge made by the 'ort' strategy.
+ Documentation/dev-tools/kunit/usage.rst     |  46 ++++++-
+ drivers/gpu/drm/tests/drm_rect_test.c       |  46 ++++++-
+ include/kunit/test-bug.h                    |  26 ++++
+ include/kunit/test.h                        |  98 ++++++++++++++
+ kernel/panic.c                              |  11 ++
+ lib/bug.c                                   |  14 +-
+ lib/kunit/Makefile                          |   4 +-
+ lib/kunit/backtrace-suppression-test.c      | 198 ++++++++++++++++++++++++++++
+ lib/kunit/bug.c                             | 120 +++++++++++++++++
+ lib/kunit/hooks-impl.h                      |   2 +
+ scripts/clang-tools/gen_compile_commands.py |   2 +
+ tools/testing/kunit/kunit_kernel.py         |   2 +-
+ tools/testing/kunit/kunit_parser.py         |   2 +-
+ tools/testing/kunit/qemu_configs/or1k.py    |  18 +++
+ 14 files changed, 577 insertions(+), 12 deletions(-)
+ create mode 100644 lib/kunit/backtrace-suppression-test.c
+ create mode 100644 lib/kunit/bug.c
+ create mode 100644 tools/testing/kunit/qemu_configs/or1k.py
+Merging livepatching/for-next (8f7168335cb2e selftests: livepatch: Check if stack_order sysfs attribute exists)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/livepatching/livepatching.git livepatching/for-next
+Merge made by the 'ort' strategy.
+ tools/testing/selftests/livepatch/functions.sh     |  10 +
+ tools/testing/selftests/livepatch/test-kprobe.sh   |   8 +-
+ tools/testing/selftests/livepatch/test-sysfs.sh    | 213 +++++++++++----------
+ .../livepatch/test_modules/test_klp_syscall.c      |  27 ++-
+ 4 files changed, 150 insertions(+), 108 deletions(-)
+Merging rtc/rtc-next (b72386864481c rtc: ab8500: replace sprintf() with sysfs_emit())
+$ git merge -m Merge branch 'rtc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/abelloni/linux.git rtc/rtc-next
+Merge made by the 'ort' strategy.
+ drivers/rtc/rtc-ab8500.c | 3 +--
+ 1 file changed, 1 insertion(+), 2 deletions(-)
+Merging nvdimm/libnvdimm-for-next (45df9111692c6 dax/fsdev: fix uninitialized kaddr in fsdev_dax_zero_page_range())
+$ git merge -m Merge branch 'libnvdimm-for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nvdimm/nvdimm.git nvdimm/libnvdimm-for-next
+Already up to date.
+Merging at24/at24/for-next (b4c0149768852 eeprom: at24: Use named initializers for arrays of i2c_device_data)
+$ git merge -m Merge branch 'at24/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git at24/at24/for-next
+Merge made by the 'ort' strategy.
+ drivers/misc/eeprom/at24.c | 62 +++++++++++++++++++++++-----------------------
+ 1 file changed, 31 insertions(+), 31 deletions(-)
+Merging ntb/ntb-next (7b3302c687cae ntb_hw_amd: Fix incorrect debug message in link disable path)
+$ git merge -m Merge branch 'ntb-next' of https://github.com/jonmason/ntb.git ntb/ntb-next
+Merge made by the 'ort' strategy.
+ drivers/ntb/hw/amd/ntb_hw_amd.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging seccomp/for-next/seccomp (41fa043273841 selftests/seccomp: Add hard-coded __NR_uprobe for x86_64)
+$ git merge -m Merge branch 'for-next/seccomp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git seccomp/for-next/seccomp
+Already up to date.
+Merging slimbus/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/slimbus.git slimbus/for-next
+Already up to date.
+Merging nvmem/for-next (69679101cc893 Merge branches 'nvmem-fixes' and 'nvmem-for-7.2' into nvmem-for-next)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/srini/nvmem.git nvmem/for-next
+Merge made by the 'ort' strategy.
+ .../bindings/nvmem/airoha,smc-efuses.yaml          |  67 +++++++++++
+ .../bindings/nvmem/microchip,lan9662-otpc.yaml     |   1 +
+ .../devicetree/bindings/nvmem/qcom,qfprom.yaml     |   4 +
+ drivers/nvmem/Kconfig                              |  22 +++-
+ drivers/nvmem/Makefile                             |   2 +
+ drivers/nvmem/airoha-smc-efuses.c                  | 125 +++++++++++++++++++++
+ drivers/nvmem/core.c                               |   4 +
+ drivers/nvmem/lan9662-otpc.c                       |  12 +-
+ drivers/nvmem/layouts/onie-tlv.c                   |   3 +-
+ drivers/nvmem/layouts/u-boot-env.c                 |   5 +-
+ drivers/nvmem/nintendo-otp.c                       |   9 +-
+ drivers/nvmem/rockchip-otp.c                       |   9 +-
+ 12 files changed, 238 insertions(+), 25 deletions(-)
+ create mode 100644 Documentation/devicetree/bindings/nvmem/airoha,smc-efuses.yaml
+ create mode 100644 drivers/nvmem/airoha-smc-efuses.c
+Merging hyperv/hyperv-next (5170a82e89211 x86/hyperv: Skip LP/VP creation on kexec)
+$ git merge -m Merge branch 'hyperv-next' of https://git.kernel.org/pub/scm/linux/kernel/git/hyperv/linux.git hyperv/hyperv-next
+Already up to date.
+Merging auxdisplay/for-next (4dc76c305a73b auxdisplay: Kconfig: drop unneeded quotes in PANEL_BOOT_MESSAGE dep)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/andy/linux-auxdisplay.git auxdisplay/for-next
+Merge made by the 'ort' strategy.
+ drivers/auxdisplay/Kconfig        | 2 +-
+ drivers/auxdisplay/line-display.c | 2 +-
+ drivers/auxdisplay/max6959.c      | 5 +----
+ 3 files changed, 3 insertions(+), 6 deletions(-)
+Merging kgdb/kgdb/for-next (fdbdd0ccb30af kdb: remove redundant check for scancode 0xe0)
+$ git merge -m Merge branch 'kgdb/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/danielt/linux.git kgdb/kgdb/for-next
+Already up to date.
+Merging hmm/hmm (19272b37aa4f8 Linux 6.16-rc1)
+$ git merge -m Merge branch 'hmm' of https://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git hmm/hmm
+Already up to date.
+Merging cfi/cfi/next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'cfi/next' of https://git.kernel.org/pub/scm/linux/kernel/git/mtd/linux.git cfi/cfi/next
+Already up to date.
+Merging mhi/mhi-next (519ddf194b158 bus: mhi: ep: Protect mhi_ep_handle_syserr() in the error path)
+$ git merge -m Merge branch 'mhi-next' of https://git.kernel.org/pub/scm/linux/kernel/git/mani/mhi.git mhi/mhi-next
+Merge made by the 'ort' strategy.
+ drivers/bus/mhi/ep/main.c          | 11 +++++++----
+ drivers/bus/mhi/host/pci_generic.c | 16 +++++++++++++++-
+ 2 files changed, 22 insertions(+), 5 deletions(-)
+$ git am -3 ../patches/0001-fix-up-for-net-qrtr-Drop-the-MHI-auto_queue-feature-.patch
+Applying: fix up for "net: qrtr: Drop the MHI auto_queue feature for IPCR DL channels"
+Using index info to reconstruct a base tree...
+M	drivers/net/wireless/ath/ath12k/wifi7/mhi.c
+Falling back to patching base and 3-way merge...
+No changes -- Patch already applied.
+Merging memblock/for-next (d5759519805c5 x86/alternative: delay freeing of smp_locks section)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/rppt/memblock.git memblock/for-next
+Already up to date.
+Merging cxl/next (6c724ce0ec6ed Merge branch 'for-7.1/cxl-misc' into cxl-for-next)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl.git cxl/next
+Already up to date.
+Merging zstd/zstd-next (65d1f5507ed2c zstd: Import upstream v1.5.7)
+$ git merge -m Merge branch 'zstd-next' of https://github.com/terrelln/linux.git zstd/zstd-next
+Already up to date.
+Merging efi/next (48a4282157823 efi/capsule-loader: fix incorrect sizeof in phys array reallocation)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/efi/efi.git efi/next
+Already up to date.
+Merging unicode/for-next (bcfee135d5847 utf8: Remove unused utf8_normalize)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/krisman/unicode.git unicode/for-next
+Auto-merging fs/unicode/utf8-core.c
+Merge made by the 'ort' strategy.
+ fs/unicode/utf8-core.c  | 22 ----------------------
+ include/linux/unicode.h |  3 ---
+ 2 files changed, 25 deletions(-)
+Merging slab/slab/for-next (7140d88bb8233 Merge branch 'slab/for-7.2/alloc_token' into slab/for-next)
+$ git merge -m Merge branch 'slab/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/vbabka/slab.git slab/slab/for-next
+Auto-merging Makefile
+Auto-merging include/linux/slab.h
+Auto-merging init/Kconfig
+Auto-merging mm/Kconfig
+Auto-merging mm/kfence/kfence_test.c
+Auto-merging mm/slab_common.c
+Auto-merging mm/slub.c
+Merge made by the 'ort' strategy.
+ Makefile                            |   5 +
+ include/linux/instruction_pointer.h |  24 ++
+ include/linux/percpu.h              |   2 +-
+ include/linux/slab.h                | 318 +++++++++++++++++------
+ init/Kconfig                        |   3 +
+ kernel/configs/hardening.config     |   2 +-
+ mm/Kconfig                          |  73 +++++-
+ mm/kfence/kfence_test.c             |   4 +-
+ mm/slab.h                           |   4 +-
+ mm/slab_common.c                    |  52 ++--
+ mm/slub.c                           | 505 ++++++++++++++++--------------------
+ 11 files changed, 596 insertions(+), 396 deletions(-)
+Merging random/master (bb9ff576fdff4 virt: vmgenid: remap memory as decrypted)
+$ git merge -m Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/crng/random.git random/master
+Merge made by the 'ort' strategy.
+ drivers/virt/vmgenid.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+Merging landlock/next (9c5b83756e7b7 landlock: Demonstrate best-effort allowed_access filtering)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/mic/linux.git landlock/next
+Merge made by the 'ort' strategy.
+ Documentation/userspace-api/landlock.rst           |  48 +++++----
+ security/landlock/domain.c                         |   9 +-
+ security/landlock/domain.h                         |   5 +-
+ tools/testing/selftests/landlock/audit.h           | 119 +++++++++++++++------
+ tools/testing/selftests/landlock/audit_test.c      |   2 +
+ tools/testing/selftests/landlock/ptrace_test.c     |   1 +
+ .../selftests/landlock/scoped_abstract_unix_test.c |   1 +
+ 7 files changed, 126 insertions(+), 59 deletions(-)
+Merging sysctl/sysctl-next (f63a9df7e3f9f sysctl: fix uninitialized variable in proc_do_large_bitmap)
+$ git merge -m Merge branch 'sysctl-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sysctl/sysctl.git sysctl/sysctl-next
+Already up to date.
+Merging execve/for-next/execve (9bf092c97b86a sched: update task_struct->comm comment)
+$ git merge -m Merge branch 'for-next/execve' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git execve/for-next/execve
+Already up to date.
+Merging bitmap/bitmap-for-next (09472f591aa0b bitfield: wire __bf_shf to __builtin_ctzll)
+$ git merge -m Merge branch 'bitmap-for-next' of https://github.com/norov/linux.git bitmap/bitmap-for-next
+Auto-merging drivers/iio/magnetometer/yamaha-yas530.c
+Auto-merging drivers/iio/pressure/bmp280-core.c
+Merge made by the 'ort' strategy.
+ arch/x86/include/asm/extable_fixup_types.h         | 13 ++++---------
+ arch/x86/mm/extable.c                              |  2 +-
+ drivers/iio/adc/intel_dc_ti_adc.c                  |  4 ++--
+ drivers/iio/magnetometer/yamaha-yas530.c           | 12 ++++++------
+ drivers/iio/pressure/bmp280-core.c                 |  2 +-
+ drivers/iio/temperature/mcp9600.c                  |  2 +-
+ drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c  |  4 ++--
+ .../net/wireless/realtek/rtw89/rtw8852b_common.c   |  4 ++--
+ drivers/net/wireless/realtek/rtw89/rtw8852b_rfk.c  |  4 ++--
+ drivers/net/wireless/realtek/rtw89/rtw8852c.c      |  4 ++--
+ drivers/ptp/ptp_fc3.c                              |  4 ++--
+ drivers/rtc/rtc-rv3032.c                           |  2 +-
+ include/asm-generic/bitops/lock.h                  | 22 +++++++++++-----------
+ include/linux/bitfield.h                           | 18 +++++++++++++++++-
+ 14 files changed, 54 insertions(+), 43 deletions(-)
+Merging hte/for-next (005b25ad11717 hte: tegra194: Add Tegra264 GTE support)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pateldipen1984/linux.git hte/for-next
+Already up to date.
+Merging kspp/for-next/kspp (9a21dcb870e13 Merge branches 'for-linus/hardening' and 'for-next/hardening' into for-next/kspp)
+$ git merge -m Merge branch 'for-next/kspp' of https://git.kernel.org/pub/scm/linux/kernel/git/kees/linux.git kspp/for-next/kspp
+Auto-merging MAINTAINERS
+Merge made by the 'ort' strategy.
+ MAINTAINERS            |  2 +-
+ include/linux/stddef.h | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++
+ lib/string_helpers.c   |  8 +++----
+ 3 files changed, 70 insertions(+), 5 deletions(-)
+Merging nolibc/for-next (136ca91411b0b tools/nolibc: getopt: Fix potential out of bounds access)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/nolibc/linux-nolibc.git nolibc/for-next
+Merge made by the 'ort' strategy.
+ tools/include/nolibc/Makefile                  |   4 +-
+ tools/include/nolibc/alloca.h                  |  15 ++
+ tools/include/nolibc/arch-arm.h                |   2 +-
+ tools/include/nolibc/arch-arm64.h              |   2 +-
+ tools/include/nolibc/arch-loongarch.h          |   2 +-
+ tools/include/nolibc/arch-m68k.h               |   2 +-
+ tools/include/nolibc/arch-mips.h               |  96 +++++++------
+ tools/include/nolibc/arch-openrisc.h           | 160 +++++++++++++++++++++
+ tools/include/nolibc/arch-parisc.h             | 185 +++++++++++++++++++++++++
+ tools/include/nolibc/arch-powerpc.h            |   8 +-
+ tools/include/nolibc/arch-riscv.h              |   2 +-
+ tools/include/nolibc/arch-s390.h               |   2 +-
+ tools/include/nolibc/arch-sh.h                 |   2 +-
+ tools/include/nolibc/arch-sparc.h              |   2 +-
+ tools/include/nolibc/arch-x86.h                |  74 +++++-----
+ tools/include/nolibc/arch.h                    |   4 +
+ tools/include/nolibc/assert.h                  |  36 +++++
+ tools/include/nolibc/compiler.h                |   4 +-
+ tools/include/nolibc/crt.h                     |   8 +-
+ tools/include/nolibc/fcntl.h                   |  45 +++---
+ tools/include/nolibc/getopt.h                  |   2 +-
+ tools/include/nolibc/nolibc.h                  |   2 +
+ tools/include/nolibc/stackprotector.h          |   2 +-
+ tools/include/nolibc/sys.h                     |  32 ++++-
+ tools/include/nolibc/sys/mman.h                |   6 +-
+ tools/testing/selftests/nolibc/Makefile.nolibc |  50 ++-----
+ tools/testing/selftests/nolibc/nolibc-test.c   | 118 +++++++++++++++-
+ tools/testing/selftests/nolibc/run-tests.sh    |  10 +-
+ 28 files changed, 703 insertions(+), 174 deletions(-)
+ create mode 100644 tools/include/nolibc/alloca.h
+ create mode 100644 tools/include/nolibc/arch-openrisc.h
+ create mode 100644 tools/include/nolibc/arch-parisc.h
+ create mode 100644 tools/include/nolibc/assert.h
+Merging iommufd/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/jgg/iommufd.git iommufd/for-next
+Already up to date.
+Merging turbostat/next (d052a81a8a23c tools/power turbostat: Cleanup: bool force_load)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/lenb/linux.git turbostat/next
+Merge made by the 'ort' strategy.
+ tools/power/x86/turbostat/turbostat.8 |   6 +-
+ tools/power/x86/turbostat/turbostat.c | 627 ++++++++++++++++++----------------
+ 2 files changed, 329 insertions(+), 304 deletions(-)
+Merging pwrseq/pwrseq/for-next (513f49c33e91e power: sequencing: print power sequencing device parent in debugfs)
+$ git merge -m Merge branch 'pwrseq/for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux.git pwrseq/pwrseq/for-next
+Merge made by the 'ort' strategy.
+ drivers/power/sequencing/core.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+Merging capabilities-next/caps-next (0715881360074 ipc: don't audit capability check in ipc_permissions())
+$ git merge -m Merge branch 'caps-next' of https://git.kernel.org/pub/scm/linux/kernel/git/sergeh/linux.git capabilities-next/caps-next
+Already up to date.
+$ git am -3 ../patches/0001-sign-file-Fix-up-merge-issue.patch
+Applying: sign-file: Fix up merge issue
+Using index info to reconstruct a base tree...
+M	scripts/sign-file.c
+Falling back to patching base and 3-way merge...
+Auto-merging scripts/sign-file.c
+No changes -- Patch already applied.
+Merging ipe/next (028ef9c96e961 Linux 7.0)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/wufan/ipe.git ipe/next
+Already up to date.
+Merging kcsan/next (07a1a6562ce29 kcsan: Silence -Wmaybe-uninitialized when calling __kcsan_check_access())
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/melver/linux.git kcsan/next
+Merge made by the 'ort' strategy.
+ include/linux/kcsan-checks.h | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+Merging crc/crc-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'crc-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git crc/crc-next
+Already up to date.
+Merging keys-next/keys-next (965e9a2cf23b0 pkcs7: Change a pr_warn() to pr_warn_once())
+$ git merge -m Merge branch 'keys-next' of https://git.kernel.org/pub/scm/linux/kernel/git/dhowells/linux-fs.git keys-next/keys-next
+Already up to date.
+Merging fwctl/for-next (254f49634ee16 Linux 7.1-rc1)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/fwctl/fwctl.git fwctl/for-next
+Already up to date.
+Merging devsec-tsm/next (3177779ae17db virt: coco: change tsm_class to a const struct)
+$ git merge -m Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/devsec/tsm.git devsec-tsm/next
+Already up to date.
+Merging hisilicon/for-next (b419e155e31d9 Merge branch 'next/dt64' into for-next)
+$ git merge -m Merge branch 'for-next' of https://github.com/hisilicon/linux-hisi.git hisilicon/for-next
+Merge made by the 'ort' strategy.
+Merging kthread/for-next (fa39ec4f89f26 doc: Add housekeeping documentation)
+$ git merge -m Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/frederic/linux-dynticks.git kthread/for-next
+Already up to date.
+Merging nfc/for-next (9705c9f36439f nfc: nxp-nci: i2c: use rising-edge IRQ on ACPI systems)
+$ git merge -m Merge branch 'for-next' of https://codeberg.org/linux-nfc/linux.git nfc/for-next
+Merge made by the 'ort' strategy.
+ drivers/nfc/nxp-nci/core.c | 1 +
+ drivers/nfc/trf7970a.c     | 6 +++---
+ net/nfc/nci/uart.c         | 2 +-
+ 3 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/arch/alpha/include/asm/pci.h b/arch/alpha/include/asm/pci.h
index 6c04fcbdc8ed2..ad5d1391e1fa8 100644
--- a/arch/alpha/include/asm/pci.h
+++ b/arch/alpha/include/asm/pci.h
@@ -84,8 +84,17 @@ extern int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val,
 extern int pci_mmap_legacy_page_range(struct pci_bus *bus,
 				      struct vm_area_struct *vma,
 				      enum pci_mmap_state mmap_state);
-extern void pci_adjust_legacy_attr(struct pci_bus *bus,
-				   enum pci_mmap_state mmap_type);
+extern bool pci_legacy_has_sparse(struct pci_bus *bus,
+				  enum pci_mmap_state type);
 #define HAVE_PCI_LEGACY	1
 
+extern const struct attribute_group pci_dev_resource_attr_group;
+extern const struct attribute_group pci_dev_resource_sparse_attr_group;
+extern const struct attribute_group pci_dev_resource_dense_attr_group;
+
+#define ARCH_PCI_DEV_GROUPS		\
+	&pci_dev_resource_attr_group,	\
+	&pci_dev_resource_sparse_attr_group,	\
+	&pci_dev_resource_dense_attr_group,
+
 #endif /* __ALPHA_PCI_H */
diff --git a/arch/alpha/include/uapi/asm/errno.h b/arch/alpha/include/uapi/asm/errno.h
index 6791f6508632e..1a99f38813c79 100644
--- a/arch/alpha/include/uapi/asm/errno.h
+++ b/arch/alpha/include/uapi/asm/errno.h
@@ -127,4 +127,6 @@
 
 #define EHWPOISON	139	/* Memory page has hardware error */
 
+#define EFTYPE		140	/* Wrong file type for the intended operation */
+
 #endif
diff --git a/arch/alpha/include/uapi/asm/fcntl.h b/arch/alpha/include/uapi/asm/fcntl.h
index 50bdc8e8a271b..fe488bf7c18e5 100644
--- a/arch/alpha/include/uapi/asm/fcntl.h
+++ b/arch/alpha/include/uapi/asm/fcntl.h
@@ -34,6 +34,7 @@
 
 #define O_PATH		040000000
 #define __O_TMPFILE	0100000000
+#define OPENAT2_REGULAR	0200000000
 
 #define F_GETLK		7
 #define F_SETLK		8
diff --git a/arch/alpha/kernel/io.c b/arch/alpha/kernel/io.c
index c28035d6d1e65..2bad1b4fb2409 100644
--- a/arch/alpha/kernel/io.c
+++ b/arch/alpha/kernel/io.c
@@ -647,7 +647,7 @@ void _memset_c_io(volatile void __iomem *to, unsigned long c, long count)
 
 EXPORT_SYMBOL(_memset_c_io);
 
-#if IS_ENABLED(CONFIG_VGA_CONSOLE) || IS_ENABLED(CONFIG_MDA_CONSOLE)
+#if IS_ENABLED(CONFIG_VGA_CONSOLE)
 
 #include <asm/vga.h>
 
diff --git a/arch/alpha/kernel/pci-sysfs.c b/arch/alpha/kernel/pci-sysfs.c
index 3048758304b57..94dbc470cd6c0 100644
--- a/arch/alpha/kernel/pci-sysfs.c
+++ b/arch/alpha/kernel/pci-sysfs.c
@@ -11,8 +11,7 @@
  */
 
 #include <linux/sched.h>
-#include <linux/stat.h>
-#include <linux/slab.h>
+#include <linux/security.h>
 #include <linux/pci.h>
 
 static int hose_mmap_page_range(struct pci_controller *hose,
@@ -36,20 +35,18 @@ static int hose_mmap_page_range(struct pci_controller *hose,
 static int __pci_mmap_fits(struct pci_dev *pdev, int num,
 			   struct vm_area_struct *vma, int sparse)
 {
+	resource_size_t len = pci_resource_len(pdev, num);
 	unsigned long nr, start, size;
 	int shift = sparse ? 5 : 0;
 
+	if (!len)
+		return 0;
+
 	nr = vma_pages(vma);
 	start = vma->vm_pgoff;
-	size = ((pci_resource_len(pdev, num) - 1) >> (PAGE_SHIFT - shift)) + 1;
+	size = ((len - 1) >> (PAGE_SHIFT - shift)) + 1;
 
-	if (start < size && size - start >= nr)
-		return 1;
-	WARN(1, "process \"%s\" tried to map%s 0x%08lx-0x%08lx on %s BAR %d "
-		"(size 0x%08lx)\n",
-		current->comm, sparse ? " sparse" : "", start, start + nr,
-		pci_name(pdev), num, size);
-	return 0;
+	return start < size && size - start >= nr;
 }
 
 /**
@@ -68,26 +65,25 @@ static int pci_mmap_resource(struct kobject *kobj,
 			     struct vm_area_struct *vma, int sparse)
 {
 	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
-	struct resource *res = attr->private;
+	int barno = (unsigned long)attr->private;
 	enum pci_mmap_state mmap_type;
 	struct pci_bus_region bar;
-	int i;
+	int ret;
 
-	for (i = 0; i < PCI_STD_NUM_BARS; i++)
-		if (res == &pdev->resource[i])
-			break;
-	if (i >= PCI_STD_NUM_BARS)
-		return -ENODEV;
+	ret = security_locked_down(LOCKDOWN_PCI_ACCESS);
+	if (ret)
+		return ret;
 
-	if (res->flags & IORESOURCE_MEM && iomem_is_exclusive(res->start))
+	if (pci_resource_is_mem(pdev, barno) &&
+	    iomem_is_exclusive(pci_resource_start(pdev, barno)))
 		return -EINVAL;
 
-	if (!__pci_mmap_fits(pdev, i, vma, sparse))
+	if (!__pci_mmap_fits(pdev, barno, vma, sparse))
 		return -EINVAL;
 
-	pcibios_resource_to_bus(pdev->bus, &bar, res);
+	pcibios_resource_to_bus(pdev->bus, &bar, pci_resource_n(pdev, barno));
 	vma->vm_pgoff += bar.start >> (PAGE_SHIFT - (sparse ? 5 : 0));
-	mmap_type = res->flags & IORESOURCE_MEM ? pci_mmap_mem : pci_mmap_io;
+	mmap_type = pci_resource_is_mem(pdev, barno) ? pci_mmap_mem : pci_mmap_io;
 
 	return hose_mmap_page_range(pdev->sysdata, vma, mmap_type, sparse);
 }
@@ -106,34 +102,26 @@ static int pci_mmap_resource_dense(struct file *filp, struct kobject *kobj,
 	return pci_mmap_resource(kobj, attr, vma, 0);
 }
 
-/**
- * pci_remove_resource_files - cleanup resource files
- * @pdev: pci_dev to cleanup
- *
- * If we created resource files for @dev, remove them from sysfs and
- * free their resources.
- */
-void pci_remove_resource_files(struct pci_dev *pdev)
-{
-	int i;
-
-	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
-		struct bin_attribute *res_attr;
-
-		res_attr = pdev->res_attr[i];
-		if (res_attr) {
-			sysfs_remove_bin_file(&pdev->dev.kobj, res_attr);
-			kfree(res_attr);
-		}
-
-		res_attr = pdev->res_attr_wc[i];
-		if (res_attr) {
-			sysfs_remove_bin_file(&pdev->dev.kobj, res_attr);
-			kfree(res_attr);
-		}
-	}
+#define __pci_dev_resource_attr(_bar, _name, _suffix, _mmap)		\
+static const struct bin_attribute					\
+pci_dev_resource##_bar##_suffix##_attr = {				\
+	.attr = { .name = __stringify(_name), .mode = 0600 },		\
+	.private = (void *)(unsigned long)(_bar),			\
+	.mmap = (_mmap),						\
 }
 
+#define pci_dev_resource_attr(_bar)					\
+	__pci_dev_resource_attr(_bar, resource##_bar,,			\
+			    pci_mmap_resource_dense)
+
+#define pci_dev_resource_sparse_attr(_bar)				\
+	__pci_dev_resource_attr(_bar, resource##_bar##_sparse, _sparse,	\
+			    pci_mmap_resource_sparse)
+
+#define pci_dev_resource_dense_attr(_bar)				\
+	__pci_dev_resource_attr(_bar, resource##_bar##_dense, _dense,	\
+			    pci_mmap_resource_dense)
+
 static int sparse_mem_mmap_fits(struct pci_dev *pdev, int num)
 {
 	struct pci_bus_region bar;
@@ -141,7 +129,7 @@ static int sparse_mem_mmap_fits(struct pci_dev *pdev, int num)
 	long dense_offset;
 	unsigned long sparse_size;
 
-	pcibios_resource_to_bus(pdev->bus, &bar, &pdev->resource[num]);
+	pcibios_resource_to_bus(pdev->bus, &bar, pci_resource_n(pdev, num));
 
 	/* All core logic chips have 4G sparse address space, except
 	   CIA which has 16G (see xxx_SPARSE_MEM and xxx_DENSE_MEM
@@ -153,109 +141,10 @@ static int sparse_mem_mmap_fits(struct pci_dev *pdev, int num)
 	return bar.end < sparse_size;
 }
 
-static int pci_create_one_attr(struct pci_dev *pdev, int num, char *name,
-			       char *suffix, struct bin_attribute *res_attr,
-			       unsigned long sparse)
-{
-	size_t size = pci_resource_len(pdev, num);
-
-	sprintf(name, "resource%d%s", num, suffix);
-	res_attr->mmap = sparse ? pci_mmap_resource_sparse :
-				  pci_mmap_resource_dense;
-	res_attr->attr.name = name;
-	res_attr->attr.mode = S_IRUSR | S_IWUSR;
-	res_attr->size = sparse ? size << 5 : size;
-	res_attr->private = &pdev->resource[num];
-	return sysfs_create_bin_file(&pdev->dev.kobj, res_attr);
-}
-
-static int pci_create_attr(struct pci_dev *pdev, int num)
-{
-	/* allocate attribute structure, piggyback attribute name */
-	int retval, nlen1, nlen2 = 0, res_count = 1;
-	unsigned long sparse_base, dense_base;
-	struct bin_attribute *attr;
-	struct pci_controller *hose = pdev->sysdata;
-	char *suffix, *attr_name;
-
-	suffix = "";	/* Assume bwx machine, normal resourceN files. */
-	nlen1 = 10;
-
-	if (pdev->resource[num].flags & IORESOURCE_MEM) {
-		sparse_base = hose->sparse_mem_base;
-		dense_base = hose->dense_mem_base;
-		if (sparse_base && !sparse_mem_mmap_fits(pdev, num)) {
-			sparse_base = 0;
-			suffix = "_dense";
-			nlen1 = 16;	/* resourceN_dense */
-		}
-	} else {
-		sparse_base = hose->sparse_io_base;
-		dense_base = hose->dense_io_base;
-	}
-
-	if (sparse_base) {
-		suffix = "_sparse";
-		nlen1 = 17;
-		if (dense_base) {
-			nlen2 = 16;	/* resourceN_dense */
-			res_count = 2;
-		}
-	}
-
-	attr = kzalloc(sizeof(*attr) * res_count + nlen1 + nlen2, GFP_ATOMIC);
-	if (!attr)
-		return -ENOMEM;
-
-	/* Create bwx, sparse or single dense file */
-	attr_name = (char *)(attr + res_count);
-	pdev->res_attr[num] = attr;
-	retval = pci_create_one_attr(pdev, num, attr_name, suffix, attr,
-				     sparse_base);
-	if (retval || res_count == 1)
-		return retval;
-
-	/* Create dense file */
-	attr_name += nlen1;
-	attr++;
-	pdev->res_attr_wc[num] = attr;
-	return pci_create_one_attr(pdev, num, attr_name, "_dense", attr, 0);
-}
-
-/**
- * pci_create_resource_files - create resource files in sysfs for @pdev
- * @pdev: pci_dev in question
- *
- * Walk the resources in @dev creating files for each resource available.
- *
- * Return: %0 on success, or negative error code
- */
-int pci_create_resource_files(struct pci_dev *pdev)
-{
-	int i;
-	int retval;
-
-	/* Expose the PCI resources from this device as files */
-	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
-
-		/* skip empty resources */
-		if (!pci_resource_len(pdev, i))
-			continue;
-
-		retval = pci_create_attr(pdev, i);
-		if (retval) {
-			pci_remove_resource_files(pdev);
-			return retval;
-		}
-	}
-	return 0;
-}
-
 /* Legacy I/O bus mapping stuff. */
 
-static int __legacy_mmap_fits(struct pci_controller *hose,
-			      struct vm_area_struct *vma,
-			      unsigned long res_size, int sparse)
+static int __legacy_mmap_fits(struct vm_area_struct *vma,
+			      unsigned long res_size)
 {
 	unsigned long nr, start, size;
 
@@ -263,13 +152,7 @@ static int __legacy_mmap_fits(struct pci_controller *hose,
 	start = vma->vm_pgoff;
 	size = ((res_size - 1) >> PAGE_SHIFT) + 1;
 
-	if (start < size && size - start >= nr)
-		return 1;
-	WARN(1, "process \"%s\" tried to map%s 0x%08lx-0x%08lx on hose %d "
-		"(size 0x%08lx)\n",
-		current->comm, sparse ? " sparse" : "", start, start + nr,
-		hose->index, size);
-	return 0;
+	return start < size && size - start >= nr;
 }
 
 static inline int has_sparse(struct pci_controller *hose,
@@ -290,36 +173,22 @@ int pci_mmap_legacy_page_range(struct pci_bus *bus, struct vm_area_struct *vma,
 	int sparse = has_sparse(hose, mmap_type);
 	unsigned long res_size;
 
-	res_size = (mmap_type == pci_mmap_mem) ? bus->legacy_mem->size :
-						 bus->legacy_io->size;
-	if (!__legacy_mmap_fits(hose, vma, res_size, sparse))
+	res_size = (mmap_type == pci_mmap_mem) ? PCI_LEGACY_MEM_SIZE :
+						 PCI_LEGACY_IO_SIZE;
+	if (sparse)
+		res_size <<= 5;
+
+	if (!__legacy_mmap_fits(vma, res_size))
 		return -EINVAL;
 
 	return hose_mmap_page_range(hose, vma, mmap_type, sparse);
 }
 
-/**
- * pci_adjust_legacy_attr - adjustment of legacy file attributes
- * @bus: bus to create files under
- * @mmap_type: I/O port or memory
- *
- * Adjust file name and size for sparse mappings.
- */
-void pci_adjust_legacy_attr(struct pci_bus *bus, enum pci_mmap_state mmap_type)
+bool pci_legacy_has_sparse(struct pci_bus *bus, enum pci_mmap_state type)
 {
 	struct pci_controller *hose = bus->sysdata;
 
-	if (!has_sparse(hose, mmap_type))
-		return;
-
-	if (mmap_type == pci_mmap_mem) {
-		bus->legacy_mem->attr.name = "legacy_mem_sparse";
-		bus->legacy_mem->size <<= 5;
-	} else {
-		bus->legacy_io->attr.name = "legacy_io_sparse";
-		bus->legacy_io->size <<= 5;
-	}
-	return;
+	return has_sparse(hose, type);
 }
 
 /* Legacy I/O bus read/write functions */
@@ -370,3 +239,166 @@ int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val, size_t size)
 	}
 	return -EINVAL;
 }
+
+pci_dev_resource_attr(0);
+pci_dev_resource_attr(1);
+pci_dev_resource_attr(2);
+pci_dev_resource_attr(3);
+pci_dev_resource_attr(4);
+pci_dev_resource_attr(5);
+
+pci_dev_resource_sparse_attr(0);
+pci_dev_resource_sparse_attr(1);
+pci_dev_resource_sparse_attr(2);
+pci_dev_resource_sparse_attr(3);
+pci_dev_resource_sparse_attr(4);
+pci_dev_resource_sparse_attr(5);
+
+pci_dev_resource_dense_attr(0);
+pci_dev_resource_dense_attr(1);
+pci_dev_resource_dense_attr(2);
+pci_dev_resource_dense_attr(3);
+pci_dev_resource_dense_attr(4);
+pci_dev_resource_dense_attr(5);
+
+static inline enum pci_mmap_state pci_bar_mmap_type(struct pci_dev *pdev,
+						    int bar)
+{
+	return pci_resource_is_mem(pdev, bar) ? pci_mmap_mem : pci_mmap_io;
+}
+
+static inline umode_t __pci_resource_attr_is_visible(struct kobject *kobj,
+						     const struct bin_attribute *a,
+						     int bar)
+{
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
+
+	if (!pci_resource_len(pdev, bar))
+		return 0;
+
+	return a->attr.mode;
+}
+
+static umode_t pci_dev_resource_is_visible(struct kobject *kobj,
+					   const struct bin_attribute *a,
+					   int bar)
+{
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
+	struct pci_controller *hose = pdev->sysdata;
+
+	if (has_sparse(hose, pci_bar_mmap_type(pdev, bar)))
+		return 0;
+
+	return __pci_resource_attr_is_visible(kobj, a, bar);
+}
+
+static umode_t pci_dev_resource_sparse_is_visible(struct kobject *kobj,
+						  const struct bin_attribute *a,
+						  int bar)
+{
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
+	struct pci_controller *hose = pdev->sysdata;
+	enum pci_mmap_state type = pci_bar_mmap_type(pdev, bar);
+
+	if (!has_sparse(hose, type))
+		return 0;
+
+	if (type == pci_mmap_mem && !sparse_mem_mmap_fits(pdev, bar))
+		return 0;
+
+	return __pci_resource_attr_is_visible(kobj, a, bar);
+}
+
+static umode_t pci_dev_resource_dense_is_visible(struct kobject *kobj,
+						 const struct bin_attribute *a,
+						 int bar)
+{
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
+	struct pci_controller *hose = pdev->sysdata;
+	enum pci_mmap_state type = pci_bar_mmap_type(pdev, bar);
+	unsigned long dense_base;
+
+	if (!has_sparse(hose, type))
+		return 0;
+
+	if (type == pci_mmap_mem && !sparse_mem_mmap_fits(pdev, bar))
+		return __pci_resource_attr_is_visible(kobj, a, bar);
+
+	dense_base = (type == pci_mmap_mem) ? hose->dense_mem_base :
+					      hose->dense_io_base;
+	if (!dense_base)
+		return 0;
+
+	return __pci_resource_attr_is_visible(kobj, a, bar);
+}
+
+static inline size_t __pci_dev_resource_bin_size(struct kobject *kobj,
+						 int bar, bool sparse)
+{
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
+	size_t size = pci_resource_len(pdev, bar);
+
+	return sparse ? size << 5 : size;
+}
+
+static size_t pci_dev_resource_bin_size(struct kobject *kobj,
+					const struct bin_attribute *a,
+					int bar)
+{
+	return __pci_dev_resource_bin_size(kobj, bar, false);
+}
+
+static size_t pci_dev_resource_sparse_bin_size(struct kobject *kobj,
+					       const struct bin_attribute *a,
+					       int bar)
+{
+	return __pci_dev_resource_bin_size(kobj, bar, true);
+}
+
+static const struct bin_attribute *const pci_dev_resource_attrs[] = {
+	&pci_dev_resource0_attr,
+	&pci_dev_resource1_attr,
+	&pci_dev_resource2_attr,
+	&pci_dev_resource3_attr,
+	&pci_dev_resource4_attr,
+	&pci_dev_resource5_attr,
+	NULL,
+};
+
+static const struct bin_attribute *const pci_dev_resource_sparse_attrs[] = {
+	&pci_dev_resource0_sparse_attr,
+	&pci_dev_resource1_sparse_attr,
+	&pci_dev_resource2_sparse_attr,
+	&pci_dev_resource3_sparse_attr,
+	&pci_dev_resource4_sparse_attr,
+	&pci_dev_resource5_sparse_attr,
+	NULL,
+};
+
+static const struct bin_attribute *const pci_dev_resource_dense_attrs[] = {
+	&pci_dev_resource0_dense_attr,
+	&pci_dev_resource1_dense_attr,
+	&pci_dev_resource2_dense_attr,
+	&pci_dev_resource3_dense_attr,
+	&pci_dev_resource4_dense_attr,
+	&pci_dev_resource5_dense_attr,
+	NULL,
+};
+
+const struct attribute_group pci_dev_resource_attr_group = {
+	.bin_attrs = pci_dev_resource_attrs,
+	.is_bin_visible = pci_dev_resource_is_visible,
+	.bin_size = pci_dev_resource_bin_size,
+};
+
+const struct attribute_group pci_dev_resource_sparse_attr_group = {
+	.bin_attrs = pci_dev_resource_sparse_attrs,
+	.is_bin_visible = pci_dev_resource_sparse_is_visible,
+	.bin_size = pci_dev_resource_sparse_bin_size,
+};
+
+const struct attribute_group pci_dev_resource_dense_attr_group = {
+	.bin_attrs = pci_dev_resource_dense_attrs,
+	.is_bin_visible = pci_dev_resource_dense_is_visible,
+	.bin_size = pci_dev_resource_bin_size,
+};
diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h
index 1e8809ea000a3..6df2209541ac0 100644
--- a/arch/arc/include/asm/uaccess.h
+++ b/arch/arc/include/asm/uaccess.h
@@ -628,8 +628,7 @@ static inline unsigned long __clear_user(void __user *to, unsigned long n)
 	return res;
 }
 
-#define INLINE_COPY_TO_USER
-#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_USER
 
 #define __clear_user			__clear_user
 
diff --git a/arch/arc/mm/dma.c b/arch/arc/mm/dma.c
index 6b85e94f32759..9b9adb02b4c5a 100644
--- a/arch/arc/mm/dma.c
+++ b/arch/arc/mm/dma.c
@@ -98,8 +98,8 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
 	 * DMA buffers.
 	 */
 	if (is_isa_arcv2() && ioc_enable && coherent)
-		dev->dma_coherent = true;
+		dev_set_dma_coherent(dev);
 
 	dev_info(dev, "use %scoherent DMA ops\n",
-		 dev->dma_coherent ? "" : "non");
+		 dev_dma_coherent(dev) ? "" : "non");
 }
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index 71fc5dd4123f3..fc398730336ea 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -136,7 +136,7 @@ config ARM
 	select MMU_GATHER_RCU_TABLE_FREE if SMP && ARM_LPAE
 	select HAVE_REGS_AND_STACK_ACCESS_API
 	select HAVE_RSEQ
-	select HAVE_RUST if CPU_LITTLE_ENDIAN && CPU_32v7
+	select HAVE_RUST if CPU_LITTLE_ENDIAN && CPU_32v7 && !KASAN
 	select HAVE_STACKPROTECTOR
 	select HAVE_SYSCALL_TRACEPOINTS
 	select HAVE_UID16
@@ -465,6 +465,8 @@ source "arch/arm/mach-versatile/Kconfig"
 
 source "arch/arm/mach-vt8500/Kconfig"
 
+source "arch/arm/mach-zte/Kconfig"
+
 source "arch/arm/mach-zynq/Kconfig"
 
 # ARMv7-M architecture
diff --git a/arch/arm/Makefile b/arch/arm/Makefile
index b7de4b6b284ca..573813ef5e77a 100644
--- a/arch/arm/Makefile
+++ b/arch/arm/Makefile
@@ -223,6 +223,7 @@ machine-$(CONFIG_ARCH_SUNXI)		+= sunxi
 machine-$(CONFIG_ARCH_TEGRA)		+= tegra
 machine-$(CONFIG_ARCH_U8500)		+= ux500
 machine-$(CONFIG_ARCH_VT8500)		+= vt8500
+machine-$(CONFIG_ARCH_ZTE)		+= zte
 machine-$(CONFIG_ARCH_ZYNQ)		+= zynq
 machine-$(CONFIG_PLAT_VERSATILE)	+= versatile
 machine-$(CONFIG_PLAT_SPEAR)		+= spear
diff --git a/arch/arm/arm-soc-for-next-contents.txt b/arch/arm/arm-soc-for-next-contents.txt
new file mode 100644
index 0000000000000..6ed7086a7faf3
--- /dev/null
+++ b/arch/arm/arm-soc-for-next-contents.txt
@@ -0,0 +1,62 @@
+soc/arm
+	patch
+		ARM: select legacy gpiolib interfaces where used
+		arm64: Kconfig: drop unneeded dependency on OF_GPIO for ARCH_MVEBU
+	pxa/gpio
+		git://git.kernel.org/pub/scm/linux/kernel/git/brgl/linux tags/soc-pxa-gpio-for-v7.2
+	zx/soc
+		https://gitlab.com/stefandoesinger/zx297520-kernel tags/zx29-plat-for-7.2
+
+soc/dt
+	patch
+		arm64: dts: realtek: Add pinctrl support for RTD1625
+		ARM: dts: ixp4xx: use phandle-based GPIOs in mi424wr
+	pxa1908/dt
+		https://codeberg.org/pxa1908-mainline/linux tags/pxa1908-dt-for-7.2
+	renesas/dt
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-dts-for-v7.2-tag1
+		contains renesas/fixes-1
+	xz/dt
+		https://gitlab.com/stefandoesinger/zx297520-kernel tags/zx29-dts-for-7.2
+
+soc/drivers
+	fsl/soc-driver
+		https://git.kernel.org/pub/scm/linux/kernel/git/chleroy/linux tags/soc_fsl-7.1-2
+	renesas/drivers
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-drivers-for-v7.2-tag1
+
+soc/defconfig
+	patch
+		arm64: defconfig: Move entries to match savedefconfig
+		arm64: defconfig: Drop unused legacy netfilter options
+		arm64: defconfig: Drop default or selected drivers
+		arm64: defconfig: Drop unused Ethernet vendors
+		arm64: defconfig: Switch Ethernet drivers to modules
+	defconfig/pinctrl
+		git://git.kernel.org/pub/scm/linux/kernel/git/linusw/linux-pinctrl tags/v7.2-qcom-pinctrl-defconfigs
+	patch
+		ARM: multi_v7_defconfig: Move entries to match savedefconfig
+		ARM: configs: Drop redundant I2C_DESIGNWARE_PLATFORM
+		ARM: configs: Drop redundant SND_ATMEL_SOC
+		ARM: multi_v7_defconfig: Cleanup redundant options
+		ARM: multi_v7_defconfig: Correct QCOM_RPMH and QCOM_RPMHPD
+		arm64: defconfig: Enable PCI M.2 power sequencing driver
+	cix/defconfig
+		git://git.kernel.org/pub/scm/linux/kernel/git/peter.chen/cix tags/cix-defconfig-v7.2-rc1
+
+soc/late
+
+arm/fixes
+	renesas/fixes-1
+		git://git.kernel.org/pub/scm/linux/kernel/git/geert/renesas-devel tags/renesas-fixes-for-v7.1-tag1
+	patch
+		ARM: integrator: Fix early initialization
+		MAINTAINERS: Add maintainers for ARM/REALTEK ARCHITECTURE
+		ARM: realtek: MAINTAINERS: Include pin controller drivers
+	ffa/fixes-2
+		git://git.kernel.org/pub/scm/linux/kernel/git/sudeep.holla/linux tags/ffa-fixes-7.1-2
+	patch
+		firmware: psci: Set pm_set_resume/suspend_via_firmware() for SYSTEM_SUSPEND
+	riscv/dt-fixes
+		https://git.kernel.org/pub/scm/linux/kernel/git/conor/linux tags/riscv-dt-fixes-for-v7.1-rc3
+
diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile
index efe38eb253016..28fba538d5520 100644
--- a/arch/arm/boot/dts/Makefile
+++ b/arch/arm/boot/dts/Makefile
@@ -39,3 +39,4 @@ subdir-y += unisoc
 subdir-y += vt8500
 subdir-y += xen
 subdir-y += xilinx
+subdir-y += zte
diff --git a/arch/arm/boot/dts/allwinner/sun8i-t113s.dtsi b/arch/arm/boot/dts/allwinner/sun8i-t113s.dtsi
index 424f4a2487e2f..3ca0897353bec 100644
--- a/arch/arm/boot/dts/allwinner/sun8i-t113s.dtsi
+++ b/arch/arm/boot/dts/allwinner/sun8i-t113s.dtsi
@@ -90,3 +90,7 @@ cpu-crit {
 		};
 	};
 };
+
+&wdt {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/aspeed/Makefile b/arch/arm/boot/dts/aspeed/Makefile
index 767f7c7652d55..7d70f08e436c6 100644
--- a/arch/arm/boot/dts/aspeed/Makefile
+++ b/arch/arm/boot/dts/aspeed/Makefile
@@ -34,6 +34,7 @@ dtb-$(CONFIG_ARCH_ASPEED) += \
 	aspeed-bmc-facebook-harma.dtb \
 	aspeed-bmc-facebook-minerva.dtb \
 	aspeed-bmc-facebook-minipack.dtb \
+	aspeed-bmc-facebook-sanmiguel.dtb \
 	aspeed-bmc-facebook-santabarbara.dtb \
 	aspeed-bmc-facebook-tiogapass.dtb \
 	aspeed-bmc-facebook-wedge40.dtb \
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-anacapa.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-anacapa.dts
index 2cb7bd128d24e..1fddf3a5d1383 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-anacapa.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-anacapa.dts
@@ -481,6 +481,9 @@ gpio@22 {
 				gpio-controller;
 				#gpio-cells = <2>;
 
+				interrupt-parent = <&sgpiom0>;
+				interrupts = <174 IRQ_TYPE_LEVEL_LOW>;
+
 				gpio-line-names =
 					"RPDB_FAN_FULL_SPEED_R_N", "RPDB_I2C_TEMP75_U8_ALERT_R_N",
 					"RPDB_I2C_TMP432_U29_ALERT_R_N", "RPDB_GLOBAL_WP",
@@ -500,6 +503,9 @@ gpio@24 {
 				gpio-controller;
 				#gpio-cells = <2>;
 
+				interrupt-parent = <&sgpiom0>;
+				interrupts = <174 IRQ_TYPE_LEVEL_LOW>;
+
 				gpio-line-names =
 					"RPDB_EAM2_PRSNT_MOS_N_R", "RPDB_EAM3_PRSNT_MOS_N_R",
 					"RPDB_PWRGD_P50V_HSC4_SYS_R",
@@ -529,6 +535,9 @@ gpio@22 {
 				gpio-controller;
 				#gpio-cells = <2>;
 
+				interrupt-parent = <&sgpiom0>;
+				interrupts = <174 IRQ_TYPE_LEVEL_LOW>;
+
 				gpio-line-names =
 					"LPDB_FAN_FULL_SPEED_R_N","LPDB_I2C_TEMP75_U8_ALERT_R_N",
 					"LPDB_I2C_TMP432_U29_ALERT_R_N","LPDB_GLOBAL_WP",
@@ -546,6 +555,9 @@ gpio@24 {
 				gpio-controller;
 				#gpio-cells = <2>;
 
+				interrupt-parent = <&sgpiom0>;
+				interrupts = <174 IRQ_TYPE_LEVEL_LOW>;
+
 				gpio-line-names =
 					"LPDB_P50V_FAN1_R2_PG","LPDB_P50V_FAN2_R2_PG",
 					"LPDB_P50V_FAN3_R2_PG","LPDB_P50V_FAN4_R2_PG",
@@ -882,7 +894,7 @@ &sgpiom0 {
 	/* C0-C7 line 32-47 */
 	"RSVD_RMC_GPIO3", "", "", "",
 	"", "", "", "",
-	"LEAK_DETECT_RMC_N", "", "", "",
+	"LEAK_DETECT_RMC_N", "JTAG_CPLD_TRST_R_N", "", "",
 	"", "", "", "",
 
 	/* D0-D7 line 48-63 */
@@ -908,11 +920,11 @@ &sgpiom0 {
 	"", "", "", "",
 
 	/* G0-G7 line 96-111 */
-	"PWRGD_PVDDCR_SOC_P0", "",
-	"PWRGD_PVDDIO_P0", "",
-	"PWRGD_PVDDIO_MEM_S3_P0", "",
-	"PWRGD_CHMP_CPU0_FPGA", "",
-	"PWRGD_CHIL_CPU0_FPGA", "",
+	"L_PRSNT_EDSFF0_N", "",
+	"L_PRSNT_EDSFF1_N", "",
+	"R_PRSNT_EDSFF2_N", "",
+	"R_PRSNT_EDSFF3_N", "",
+	"HPM_EDSFF_PG", "",
 	"PWRGD_CHEH_CPU0_FPGA", "",
 	"PWRGD_CHAD_CPU0_FPGA", "FM_BMC_READY_PLD",
 	"", "",
@@ -934,7 +946,7 @@ &sgpiom0 {
 	"EAM3_BRD_PRSNT_R_L", "",
 	"EAM0_CPU_MOD_PWR_GD_R", "",
 	"EAM1_CPU_MOD_PWR_GD_R", "",
-	"EAM2_CPU_MOD_PWR_GD_R", "",
+	"PRSNT_EDSFF_BOOT_N", "",
 	"EAM3_CPU_MOD_PWR_GD_R", "",
 
 	/* J0-J7 line 144-159 */
@@ -957,8 +969,8 @@ &sgpiom0 {
 	"PDB_ALERT_R_N", "",
 
 	/* L0-L7 line 176-191 */
-	"CPU0_SP7R1", "", "CPU0_SP7R2", "",
-	"CPU0_SP7R3", "", "CPU0_SP7R4", "",
+	"L_EDSFF0_PG", "", "L_EDSFF1_PG", "",
+	"R_EDSFF2_PG", "", "R_EDSFF3_PG", "",
 	"CPU0_CORETYPE0", "", "CPU0_CORETYPE1", "",
 	"CPU0_CORETYPE2", "", "FM_BIOS_POST_CMPLT_R_N", "",
 
@@ -984,8 +996,8 @@ &sgpiom0 {
 	"HPM_PWR_FAIL", "Port80_b0",
 	"FM_DIMM_IP_FAIL", "Port80_b1",
 	"FM_DIMM_AH_FAIL", "Port80_b2",
-	"HPM_AMC_THERMTRIP_R_L", "Port80_b3",
-	"FM_CPU0_THERMTRIP_N", "Port80_b4",
+	"AMC_THERMTRIP_ASSERT", "Port80_b3",
+	"CPU_THERMTRIP_ASSERT", "Port80_b4",
 	"PVDDCR_SOC_P0_OCP_L", "Port80_b5",
 	"CPLD_SGPIO_RDY", "Port80_b6",
 	"", "Port80_b7",
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-sanmiguel.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-sanmiguel.dts
new file mode 100644
index 0000000000000..74c8c17721098
--- /dev/null
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-sanmiguel.dts
@@ -0,0 +1,668 @@
+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2025 Facebook Inc.
+
+/dts-v1/;
+#include "aspeed-g6.dtsi"
+#include <dt-bindings/gpio/aspeed-gpio.h>
+#include <dt-bindings/i2c/i2c.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/leds/leds-pca955x.h>
+#include <dt-bindings/usb/pd.h>
+
+/ {
+	model = "Facebook SanMiguel BMC";
+	compatible = "facebook,sanmiguel-bmc", "aspeed,ast2600";
+
+	aliases {
+		serial0 = &uart1;
+		serial1 = &uart2;
+		serial2 = &uart3;
+		serial3 = &uart4;
+		serial4 = &uart5;
+		i2c16	= &imux16;
+		i2c17	= &imux17;
+		i2c18	= &imux18;
+		i2c19	= &imux19;
+	};
+
+	chosen {
+		stdout-path = "serial4:57600n8";
+	};
+
+	iio-hwmon {
+		compatible = "iio-hwmon";
+		io-channels = <&adc0 0>, <&adc0 1>, <&adc0 2>, <&adc0 3>,
+			      <&adc0 4>, <&adc0 5>, <&adc0 6>;
+	};
+
+	leds-fio {
+		compatible = "gpio-leds";
+		led-0 {
+			label = "power_blue";
+			gpios = <&fio_ioexp 4 GPIO_ACTIVE_HIGH>;
+		};
+		led-1 {
+			label = "power_amber";
+			gpios = <&fio_ioexp 5 GPIO_ACTIVE_LOW>;
+		};
+		led-2 {
+			label = "id_blue";
+			gpios = <&fio_ioexp 6 GPIO_ACTIVE_HIGH>;
+		};
+		led-3 {
+			label = "id_amber";
+			gpios = <&fio_ioexp 7 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	leds-smm {
+		compatible = "gpio-leds";
+		led-0 {
+			label = "bmc_heartbeat_amber";
+			gpios = <&gpio0 ASPEED_GPIO(P, 7) GPIO_ACTIVE_LOW>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x80000000>;
+	};
+
+	spi2_gpio: spi {
+		compatible = "spi-gpio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		sck-gpios = <&gpio0 ASPEED_GPIO(X, 3) GPIO_ACTIVE_HIGH>;
+		mosi-gpios = <&gpio0 ASPEED_GPIO(X, 4) GPIO_ACTIVE_HIGH>;
+		miso-gpios = <&gpio0 ASPEED_GPIO(X, 5) GPIO_ACTIVE_HIGH>;
+		cs-gpios = <&gpio0 ASPEED_GPIO(X, 2) GPIO_ACTIVE_LOW>;
+		num-chipselects = <1>;
+
+		tpm@0 {
+			compatible = "infineon,slb9670", "tcg,tpm_tis-spi";
+			spi-max-frequency = <33000000>;
+			reg = <0>;
+		};
+	};
+
+	standby_power_regulator: standby-power-regulator {
+		compatible = "regulator-fixed";
+		gpio = <&gpio0 ASPEED_GPIO(M, 3) GPIO_ACTIVE_HIGH>;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-name = "standby_power";
+		enable-active-high;
+		regulator-always-on;
+		regulator-boot-on;
+
+		status = "okay";
+	};
+};
+
+&adc0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_adc0_default &pinctrl_adc1_default
+		&pinctrl_adc2_default &pinctrl_adc3_default
+		&pinctrl_adc4_default &pinctrl_adc5_default
+		&pinctrl_adc6_default>;
+	aspeed,int-vref-microvolt = <2500000>;
+
+	status = "okay";
+};
+
+&fmc {
+	status = "okay";
+
+	flash@0 {
+		label = "bmc";
+		spi-max-frequency = <50000000>;
+		m25p,fast-read;
+
+		status = "okay";
+#include "openbmc-flash-layout-128.dtsi"
+	};
+
+	flash@1 {
+		label = "alt-bmc";
+		spi-max-frequency = <50000000>;
+		m25p,fast-read;
+
+		status = "okay";
+#include "openbmc-flash-layout-128-alt.dtsi"
+	};
+};
+
+&gpio0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpiom4_unbiased_default
+		     &pinctrl_gpiom5_unbiased_default
+		     &pinctrl_gpiov4_unbiased_default
+		     &pinctrl_gpiov7_unbiased_default>;
+
+	gpio-line-names =
+	/*A0-A7*/	"", "", "", "", "", "", "", "",
+	/*B0-B7*/	"", "", "", "", "", "", "", "",
+	/*C0-C7*/	"", "", "", "", "", "", "", "",
+	/*D0-D7*/	"", "", "", "", "", "FPGA_PEX_RST_L", "", "",
+	/*E0-E7*/	"RTL8221_PHY_RST_L-O", "RTL8211_PHY_INT_L-I", "", "",
+			"", "", "", "MUX_SGPIO_SEL-O",
+	/*F0-F7*/	"", "", "", "", "", "", "", "",
+	/*G0-G7*/	"", "", "", "", "", "", "", "",
+	/*H0-H7*/	"", "", "", "", "", "", "", "",
+	/*I0-I7*/	"", "", "", "",
+			"", "QSPI2_RST_L-O",
+			"GLOBAL_WP_BMC-I", "BMC_DDR4_TEN-O",
+	/*J0-J7*/	"", "", "", "", "", "", "", "",
+	/*K0-K7*/	"", "", "", "", "", "", "", "",
+	/*L0-L7*/	"", "", "", "", "", "", "", "",
+	/*M0-M7*/	"USB_HUB_RST_N-O", "BMC_FRU_WP-O",
+			"", "HMC_STBY_POWER_EN-O",
+			"STBY_POWER_PG-I", "PCIE_EP_RST_L-O", "", "",
+	/*N0-N7*/	"", "", "", "", "", "", "", "",
+	/*O0-O7*/	"", "", "", "", "", "", "", "",
+	/*P0-P7*/	"", "", "", "", "", "", "", "",
+	/*Q0-Q7*/	"", "", "", "", "", "", "", "",
+	/*R0-R7*/	"", "SP0_AP_INTR_N-I", "", "", "", "", "", "",
+	/*S0-S7*/	"", "", "", "", "", "", "", "",
+	/*T0-T7*/	"", "", "", "", "", "", "", "",
+	/*U0-U7*/	"", "", "", "", "", "", "", "",
+	/*V0-V7*/	"", "", "", "", "PCB_TEMP_ALERT-I", "",	"", "",
+	/*W0-W7*/	"", "", "", "", "", "", "", "CPU_RST_L-I",
+	/*X0-X7*/	"", "", "", "", "", "", "", "",
+	/*Y0-Y7*/	"", "", "", "EMMC_RST-O", "", "", "", "",
+	/*Z0-Z7*/	"HMC_EROT_SPI_INT_L-I", "", "", "", "", "", "", "";
+};
+
+&gpio1 {
+	/* 36 1.8V GPIOs */
+	gpio-line-names =
+	/*A0-A7*/	"", "", "", "", "", "", "", "",
+	/*B0-B7*/	"", "", "", "",
+			"AP_EROT_REQ-O", "EROT_AP_GNT-I",
+			"IO_EXPANDER_INT_L-I", "",
+	/*C0-C7*/	"", "", "", "", "", "", "", "",
+	/*D0-D7*/	"", "", "", "", "", "", "", "I2C_SSIF_ALERT_L-I",
+	/*E0-E7*/	"", "", "", "", "", "", "", "";
+};
+
+&i2c0 {
+	status = "okay";
+
+	ssif-bmc@10 {
+		compatible = "ssif-bmc";
+		reg = <0x10>;
+	};
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&i2c2 {
+	status = "okay";
+
+	hpm0_ioexp_20: gpio@20 {
+		compatible = "nxp,pca9555";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <ASPEED_GPIO(B, 6) IRQ_TYPE_LEVEL_LOW>;
+		vcc-supply = <&standby_power_regulator>;
+
+		gpio-line-names =
+			"B0_M0_SHDN_FORCE_L-O",
+			"B0_M0_STBY_POWER_PG-I",
+			"B0_M0_THERM_OVERT_L-I",
+			"B0_M0_THERM_WARN_L-I",
+			"B0_M0_GLOBAL_WP-O",
+			"B0_M0_USB_HUB0_RST_L-O",
+			"B0_M0_PRE_SYS_RST_L-O",
+			"B0_M0_LEAK_DETECT_ALERT_L-I",
+			"B0_M0_RUN_POWER_EN-O",
+			"B0_M0_RUN_POWER_PG-I",
+			"B0_M0_CPU_CHIPTHROT_L-O",
+			"B0_M0_SHDN_REQ_L-O",
+			"B0_M0_CPU_SHDN_OK_L-I",
+			"B0_M0_CPLD_READY-I",
+			"B0_M0_PWR_BRAKE_L-O",
+			"B0_M0_PWR_BRAKE_STATUS_L-I";
+	};
+
+	hpm0_ioexp_21: gpio@21 {
+		compatible = "nxp,pca9555";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <ASPEED_GPIO(B, 6) IRQ_TYPE_LEVEL_LOW>;
+		vcc-supply = <&standby_power_regulator>;
+
+		gpio-line-names =
+			"B0_M0_I2C_BUS_MUX_RESET_L-O",
+			"B0_M0_HPM_MCU_OK-I",
+			"B0_M0_CPU_L0_RST_IND_L-O",
+			"B0_M0_C0_SOCAMM_DAC_SEL0-O",
+			"B0_M0_C1_SOCAMM_DAC_SEL1-O",
+			"B0_M0_C0_SOCAMM_I2C_SEL_R-O",
+			"B0_M0_C1_SOCAMM_I2C_SEL_R-O",
+			"B0_M0_EEPROM_POWER_DISABLE-O",
+			"B0_M0_L0L1_RST_L-I",
+			"B0_M0_L2_RST_L-I",
+			"B0_M0_BRD_ID_0-I",
+			"B0_M0_BRD_ID_1-I",
+			"B0_M0_BMC_LEAK_TEST_L-O",
+			"B0_M0_MCU_BMC_ALERT_L-I",
+			"B0_M0_CPU_BOOT_COMPLETE_3V3-I",
+			"B0_M0_BMC_TO_GPU_MCU_I2C_EN-O";
+	};
+};
+
+&i2c3 {
+	status = "okay";
+};
+
+&i2c4 {
+	status = "okay";
+};
+
+&i2c5 {
+	status = "okay";
+
+	smm_ioexp_20: gpio@20 {
+		compatible = "nxp,pca9555";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <ASPEED_GPIO(B, 6) IRQ_TYPE_LEVEL_LOW>;
+
+		gpio-line-names =
+			"HMC_RST_R_L-O", "HMC_RECOVERY_R-O",
+			"HMC_SPI_MUX_R_SEL-O", "GLOBAL_WP-O",
+			"HMC_READY-I", "HMC_PRSNT_R-I",
+			"BMC_SELF_PWR_CYCLE-O", "EEDO_LED2-O",
+			"PWR_LED_L-O", "PWR_BTN_L-I",
+			"UID_LED_L-O", "UID_BTN_L-I",
+			"FAULT_LED_L-O", "USB2_HUB_RST_L-O",
+			"USB2_BMC_HUB2_RST_L-O", "WARN_LED_L-O";
+	};
+
+	smm_ioexp_21: gpio@21 {
+		compatible = "nxp,pca9555";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <ASPEED_GPIO(B, 6) IRQ_TYPE_LEVEL_LOW>;
+
+		gpio-line-names =
+			"BMC_HMC_MUX_SEL-O", "BMC_TPM_MUX_SEL-O",
+			"BMC_USB2_MUX_SEL-O", "HMC_PGOOD_3V3-I",
+			"", "HDR_SPI_PRSNT_L-I",
+			"SW_NRESET_R_L-O", "MUX_I2C_ESPI_SEL-O",
+			"I2C_BUS_MUX_RESET_L-O", "USB_MUX_EN-O",
+			"PDB_TRAY_RST-O", "USB_PWR_EN-O",
+			"RTC_CLR_L-O", "I2C_RTC_ALERT_L-I",
+			"X86_TPM_RST_SEL_L-O", "BMC_MUX_PI3DP_SEL-O";
+	};
+
+	smm_temp: temperature-sensor@48 {
+		compatible = "national,lm75";
+		reg = <0x48>;
+	};
+
+	smm_fru: eeprom@50 {
+		compatible = "atmel,24c128";
+		reg = <0x50>;
+	};
+
+	rtc@6f {
+		compatible = "nuvoton,nct3018y";
+		reg = <0x6f>;
+	};
+};
+
+&i2c6 {
+	status = "okay";
+
+	hmc_ioexp: gpio@20 {
+		compatible = "ti,tca6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		gpio-line-names =
+			"", "", "HMC_EROT_FATAL_ERROR_L-I", "",
+			"", "HMC_EROT_RECOVERY_L-O", "HMC_EROT_RESET_L-O", "";
+	};
+
+	i2c-mux@70 {
+		compatible = "nxp,pca9546";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		reg = <0x70>;
+		i2c-mux-idle-disconnect;
+
+		imux16: i2c@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+		};
+
+		imux17: i2c@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+		};
+
+		imux18: i2c@2 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <2>;
+		};
+
+		imux19: i2c@3 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <3>;
+		};
+	};
+};
+
+&i2c7 {
+	status = "okay";
+
+	hpm1_ioexp_20: gpio@20 {
+		compatible = "nxp,pca9555";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <ASPEED_GPIO(B, 6) IRQ_TYPE_LEVEL_LOW>;
+		vcc-supply = <&standby_power_regulator>;
+
+		gpio-line-names =
+			"B1_M0_SHDN_FORCE_L-O",
+			"B1_M0_STBY_POWER_PG-I",
+			"B1_M0_THERM_OVERT_L-I",
+			"B1_M0_THERM_WARN_L-I",
+			"B1_M0_GLOBAL_WP-O",
+			"B1_M0_USB_HUB0_RST_L-O",
+			"B1_M0_PRE_SYS_RST_L-O",
+			"B1_M0_LEAK_DETECT_ALERT_L-I",
+			"B1_M0_RUN_POWER_EN-O",
+			"B1_M0_RUN_POWER_PG-I",
+			"B1_M0_CPU_CHIPTHROT_L-O",
+			"B1_M0_SHDN_REQ_L-O",
+			"B1_M0_CPU_SHDN_OK_L-I",
+			"B1_M0_CPLD_READY-I",
+			"B1_M0_PWR_BRAKE_L-O",
+			"B1_M0_PWR_BRAKE_STATUS_L-I";
+	};
+
+	hpm1_ioexp_21: gpio@21 {
+		compatible = "nxp,pca9555";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <ASPEED_GPIO(B, 6) IRQ_TYPE_LEVEL_LOW>;
+		vcc-supply = <&standby_power_regulator>;
+
+		gpio-line-names =
+			"B1_M0_I2C_BUS_MUX_RESET_L-O",
+			"B1_M0_HPM_MCU_OK-I",
+			"B1_M0_CPU_L0_RST_IND_L-O",
+			"B1_M0_C0_SOCAMM_DAC_SEL0-O",
+			"B1_M0_C1_SOCAMM_DAC_SEL1-O",
+			"B1_M0_C0_SOCAMM_I2C_SEL_R-O",
+			"B1_M0_C1_SOCAMM_I2C_SEL_R-O",
+			"B1_M0_EEPROM_POWER_DISABLE-O",
+			"B1_M0_L0L1_RST_L-I",
+			"B1_M0_L2_RST_L-I",
+			"B1_M0_BRD_ID_0-I",
+			"B1_M0_BRD_ID_1-I",
+			"B1_M0_BMC_LEAK_TEST_L-O",
+			"B1_M0_MCU_BMC_ALERT_L-I",
+			"B1_M0_CPU_BOOT_COMPLETE_3V3-I",
+			"B1_M0_BMC_TO_GPU_MCU_I2C_EN-O";
+	};
+};
+
+&i2c8 {
+	status = "okay";
+};
+
+&i2c9 {
+	status = "okay";
+
+	pdb_mps_hsc1: power-monitor@10 {
+		compatible = "mps,mp5926";
+		reg = <0x10>;
+	};
+
+	pdb_ti_hsc1: power-monitor@11 {
+		compatible = "ti,lm5066i";
+		reg = <0x11>;
+		shunt-resistor-micro-ohms = <763>;
+	};
+
+	pdb_mps_hsc2: power-monitor@12 {
+		compatible = "mps,mp5926";
+		reg = <0x12>;
+	};
+
+	pdb_ti_hsc2: power-monitor@13 {
+		compatible = "ti,lm5066i";
+		reg = <0x13>;
+		shunt-resistor-micro-ohms = <294>;
+	};
+
+	pdb_mps_hsc3: power-monitor@14 {
+		compatible = "mps,mp5926";
+		reg = <0x14>;
+	};
+
+	pdb_ti_hsc3: power-monitor@15 {
+		compatible = "ti,lm5066i";
+		reg = <0x15>;
+		shunt-resistor-micro-ohms = <294>;
+	};
+
+	pdb_mps_hsc4: power-monitor@16 {
+		compatible = "mps,mp5926";
+		reg = <0x16>;
+	};
+
+	pdb_ti_hsc4: power-monitor@17 {
+		compatible = "ti,lm5066i";
+		reg = <0x17>;
+		shunt-resistor-micro-ohms = <381>;
+	};
+
+	pdb_ioexp_20: gpio@20 {
+		compatible = "nxp,pca9555";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <ASPEED_GPIO(B, 6) IRQ_TYPE_LEVEL_LOW>;
+
+		gpio-line-names =
+			"PDB_STRAP-I", "STBY_POWER_EN-O", "STBY_PWR_OK-I", "",
+			"", "MAIN_PWR_EN-O", "MAIN_PWR_OK-I", "GLOBAL_WP-O",
+			"PDB_PWR_BRK_L-I", "FAN_PRSNT-I", "", "",
+			"HSC4_S5_OR_S0_N-O", "", "", "";
+	};
+
+	pdb_temp: temperature-sensor@4e {
+		compatible = "national,lm75";
+		reg = <0x4e>;
+	};
+
+	pdb_fru: eeprom@50 {
+		compatible = "atmel,24c128";
+		reg = <0x50>;
+	};
+};
+
+&i2c10 {
+	status = "okay";
+
+	scm_temp: temperature-sensor@48 {
+		compatible = "national,lm75";
+		reg = <0x48>;
+	};
+
+	scm_fru: eeprom@50 {
+		compatible = "atmel,24c128";
+		reg = <0x50>;
+	};
+};
+
+&i2c11 {
+	status = "okay";
+
+	sw_config: eeprom@50 {
+		compatible = "atmel,24c64";
+		reg = <0x50>;
+	};
+};
+
+&i2c12 {
+	status = "okay";
+};
+
+&i2c13 {
+	status = "okay";
+	mctp-controller;
+	multi-master;
+
+	mctp@10 {
+		compatible = "mctp-i2c-controller";
+		reg = <(0x10 | I2C_OWN_SLAVE_ADDRESS)>;
+	};
+
+	smm_ext_ioexp: gpio@38 {
+		compatible = "nxp,pca9554";
+		reg = <0x38>;
+		gpio-controller;
+		#gpio-cells = <2>;
+
+		gpio-line-names =
+			"SSD0_PRSNT_L-I", "E1S_PWR_EN-O",
+			"SSD0_PWRDIS-O", "I2C_PDB_ALERT_L-I",
+			"BMC_SSD0_RST_L-O", "GLOBAL_WP_E1S-O",
+			"12V_SSD0_PGD-I", "SSD0_LED-O";
+	};
+
+	smm_ext_fru: eeprom@55 {
+		compatible = "atmel,24c128";
+		reg = <0x55>;
+	};
+};
+
+&i2c14 {
+	status = "okay";
+
+	fio_ioexp: gpio@20 {
+		compatible = "nxp,pca9555";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <ASPEED_GPIO(B, 6) IRQ_TYPE_LEVEL_LOW>;
+
+		gpio-line-names =
+			"", "", "", "",
+			"PWR_LED_BLUE", "PWR_LED_AMBER_R_N",
+			"BEEP_ID_LED_BLUE", "BEEP_ID_LED_AMBER_R_N",
+			"", "", "", "",
+			"FM_MAIN_PWREN_RMC_ISO_EN_N", "LEAK_DETECT_RMC_N",
+			"PWRGD_RMC_N", "SMALL_LEAK_RMC_N";
+
+		pwrgd-rmc-hog {
+			gpio-hog;
+			gpios = <14 GPIO_ACTIVE_LOW>;
+			output-low;
+		};
+	};
+
+	fio_fru: eeprom@50 {
+		compatible = "atmel,24c64";
+		reg = <0x50>;
+	};
+};
+
+&i2c15 {
+	status = "okay";
+};
+
+&mdio0 {
+	status = "okay";
+
+	ethphy0: ethernet-phy@0 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0>;
+	};
+};
+
+&mac0 {
+	pinctrl-names = "default";
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy0>;
+	pinctrl-0 = <&pinctrl_rgmii1_default>;
+
+	status = "okay";
+};
+
+&pinctrl {
+	pinctrl_gpiom4_unbiased_default: gpiom4 {
+		pins = "B12";
+		bias-disable;
+	};
+	pinctrl_gpiom5_unbiased_default: gpiom5 {
+		pins = "C12";
+		bias-disable;
+	};
+	pinctrl_gpiov4_unbiased_default: gpiov4 {
+		pins = "AE15";
+		bias-disable;
+	};
+	pinctrl_gpiov7_unbiased_default: gpiov7 {
+		pins = "AF15";
+		bias-disable;
+	};
+};
+
+&uart1 {
+	status = "okay";
+};
+
+&uart3 {
+	status = "okay";
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&uart_routing {
+	status = "okay";
+};
+
+&wdt1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdtrst1_default>;
+	aspeed,reset-type = "soc";
+	aspeed,external-signal;
+	aspeed,ext-push-pull;
+	aspeed,ext-active-high;
+	aspeed,ext-pulse-duration = <256>;
+
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-yosemite5.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-yosemite5.dts
index 2486981f3d6bd..6287f80d70d97 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-yosemite5.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-facebook-yosemite5.dts
@@ -422,6 +422,11 @@ power-sensor@45 {
 		shunt-resistor = <1000>;
 	};
 
+	power-monitor@46 {
+		compatible = "mps,mp5998";
+		reg = <0x46>;
+	};
+
 	power-monitor@47 {
 		compatible = "ti,tps25990";
 		reg = <0x47>;
diff --git a/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts b/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts
index a12d4164de4ac..29f56176e2cae 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts
+++ b/arch/arm/boot/dts/aspeed/aspeed-bmc-nvidia-msx4-bmc.dts
@@ -208,6 +208,21 @@ &lpc_reset {
 	status = "okay";
 };
 
+&mac0 {
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy3>;
+	status = "okay";
+};
+
+&mdio3 {
+	status = "okay";
+
+	ethphy3: ethernet-phy@2 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <2>;
+	};
+};
+
 &rtc {
 	status = "okay";
 };
diff --git a/arch/arm/boot/dts/aspeed/aspeed-g6.dtsi b/arch/arm/boot/dts/aspeed/aspeed-g6.dtsi
index 189bc3bbb47c9..56bb3b0444f7e 100644
--- a/arch/arm/boot/dts/aspeed/aspeed-g6.dtsi
+++ b/arch/arm/boot/dts/aspeed/aspeed-g6.dtsi
@@ -102,6 +102,15 @@ ahbc: bus@1e600000 {
 			reg = <0x1e600000 0x100>;
 		};
 
+		pwm_tach: pwm-tach-controller@1e610000 {
+			compatible = "aspeed,ast2600-pwm-tach";
+			reg = <0x1e610000 0x100>;
+			clocks = <&syscon ASPEED_CLK_AHB>;
+			resets = <&syscon ASPEED_RESET_PWM>;
+			#pwm-cells = <3>;
+			status = "disabled";
+		};
+
 		fmc: spi@1e620000 {
 			reg = <0x1e620000 0xc4>, <0x20000000 0x10000000>;
 			#address-cells = <1>;
@@ -835,11 +844,326 @@ uart9: serial@1e790300 {
 				status = "disabled";
 			};
 
-			i2c: bus@1e78a000 {
+			bus@1e78a000 {
 				compatible = "simple-bus";
 				#address-cells = <1>;
 				#size-cells = <1>;
 				ranges = <0 0x1e78a000 0x1000>;
+
+				i2c0: i2c@80 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x80 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c1_default>;
+					status = "disabled";
+				};
+
+				i2c1: i2c@100 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x100 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c2_default>;
+					status = "disabled";
+				};
+
+				i2c2: i2c@180 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x180 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c3_default>;
+					status = "disabled";
+				};
+
+				i2c3: i2c@200 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x200 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c4_default>;
+					status = "disabled";
+				};
+
+				i2c4: i2c@280 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x280 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c5_default>;
+					status = "disabled";
+				};
+
+				i2c5: i2c@300 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x300 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c6_default>;
+					status = "disabled";
+				};
+
+				i2c6: i2c@380 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x380 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c7_default>;
+					status = "disabled";
+				};
+
+				i2c7: i2c@400 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x400 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c8_default>;
+					status = "disabled";
+				};
+
+				i2c8: i2c@480 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x480 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c9_default>;
+					status = "disabled";
+				};
+
+				i2c9: i2c@500 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x500 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c10_default>;
+					status = "disabled";
+				};
+
+				i2c10: i2c@580 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x580 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c11_default>;
+					status = "disabled";
+				};
+
+				i2c11: i2c@600 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x600 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c12_default>;
+					status = "disabled";
+				};
+
+				i2c12: i2c@680 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x680 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c13_default>;
+					status = "disabled";
+				};
+
+				i2c13: i2c@700 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x700 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c14_default>;
+					status = "disabled";
+				};
+
+				i2c14: i2c@780 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x780 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c15_default>;
+					status = "disabled";
+				};
+
+				i2c15: i2c@800 {
+					#address-cells = <1>;
+					#size-cells = <0>;
+					reg = <0x800 0x80>;
+					compatible = "aspeed,ast2600-i2c-bus";
+					clocks = <&syscon ASPEED_CLK_APB2>;
+					resets = <&syscon ASPEED_RESET_I2C>;
+					interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>;
+					bus-frequency = <100000>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i2c16_default>;
+					status = "disabled";
+				};
+			};
+
+			bus@1e7a0000 {
+				compatible = "simple-bus";
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x1e7a0000 0x8000>;
+
+				i3c_global: syscon@0 {
+					compatible = "aspeed,ast2600-i3c-global", "syscon";
+					reg = <0x0 0x1000>;
+					resets = <&syscon ASPEED_RESET_I3C_DMA>;
+				};
+
+				i3c0: i3c@2000 {
+					compatible = "aspeed,ast2600-i3c";
+					reg = <0x2000 0x1000>;
+					#address-cells = <3>;
+					#size-cells = <0>;
+					clocks = <&syscon ASPEED_CLK_GATE_I3C0CLK>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i3c1_default>;
+					interrupts = <GIC_SPI 102 IRQ_TYPE_LEVEL_HIGH>;
+					aspeed,global-regs = <&i3c_global 0>;
+					status = "disabled";
+				};
+
+				i3c1: i3c@3000 {
+					compatible = "aspeed,ast2600-i3c";
+					reg = <0x3000 0x1000>;
+					#address-cells = <3>;
+					#size-cells = <0>;
+					clocks = <&syscon ASPEED_CLK_GATE_I3C1CLK>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i3c2_default>;
+					interrupts = <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>;
+					aspeed,global-regs = <&i3c_global 1>;
+					status = "disabled";
+				};
+
+				i3c2: i3c@4000 {
+					compatible = "aspeed,ast2600-i3c";
+					reg = <0x4000 0x1000>;
+					#address-cells = <3>;
+					#size-cells = <0>;
+					clocks = <&syscon ASPEED_CLK_GATE_I3C2CLK>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i3c3_default>;
+					interrupts = <GIC_SPI 104 IRQ_TYPE_LEVEL_HIGH>;
+					aspeed,global-regs = <&i3c_global 2>;
+					status = "disabled";
+				};
+
+				i3c3: i3c@5000 {
+					compatible = "aspeed,ast2600-i3c";
+					reg = <0x5000 0x1000>;
+					#address-cells = <3>;
+					#size-cells = <0>;
+					clocks = <&syscon ASPEED_CLK_GATE_I3C3CLK>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i3c4_default>;
+					interrupts = <GIC_SPI 105 IRQ_TYPE_LEVEL_HIGH>;
+					aspeed,global-regs = <&i3c_global 3>;
+					status = "disabled";
+				};
+
+				i3c4: i3c@6000 {
+					compatible = "aspeed,ast2600-i3c";
+					reg = <0x6000 0x1000>;
+					#address-cells = <3>;
+					#size-cells = <0>;
+					clocks = <&syscon ASPEED_CLK_GATE_I3C4CLK>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i3c5_default>;
+					interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>;
+					aspeed,global-regs = <&i3c_global 4>;
+					status = "disabled";
+				};
+
+				i3c5: i3c@7000 {
+					compatible = "aspeed,ast2600-i3c";
+					reg = <0x7000 0x1000>;
+					#address-cells = <3>;
+					#size-cells = <0>;
+					clocks = <&syscon ASPEED_CLK_GATE_I3C5CLK>;
+					pinctrl-names = "default";
+					pinctrl-0 = <&pinctrl_i3c6_default>;
+					interrupts = <GIC_SPI 107 IRQ_TYPE_LEVEL_HIGH>;
+					aspeed,global-regs = <&i3c_global 5>;
+					status = "disabled";
+				};
 			};
 
 			fsim0: fsi@1e79b000 {
@@ -870,229 +1194,3 @@ fsim1: fsi@1e79b100 {
 };
 
 #include "aspeed-g6-pinctrl.dtsi"
-
-&i2c {
-	i2c0: i2c@80 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x80 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 110 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c1_default>;
-		status = "disabled";
-	};
-
-	i2c1: i2c@100 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x100 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 111 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c2_default>;
-		status = "disabled";
-	};
-
-	i2c2: i2c@180 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x180 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 112 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c3_default>;
-		status = "disabled";
-	};
-
-	i2c3: i2c@200 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x200 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 113 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c4_default>;
-		status = "disabled";
-	};
-
-	i2c4: i2c@280 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x280 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 114 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c5_default>;
-		status = "disabled";
-	};
-
-	i2c5: i2c@300 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x300 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 115 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c6_default>;
-		status = "disabled";
-	};
-
-	i2c6: i2c@380 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x380 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c7_default>;
-		status = "disabled";
-	};
-
-	i2c7: i2c@400 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x400 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c8_default>;
-		status = "disabled";
-	};
-
-	i2c8: i2c@480 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x480 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c9_default>;
-		status = "disabled";
-	};
-
-	i2c9: i2c@500 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x500 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c10_default>;
-		status = "disabled";
-	};
-
-	i2c10: i2c@580 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x580 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 120 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c11_default>;
-		status = "disabled";
-	};
-
-	i2c11: i2c@600 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x600 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 121 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c12_default>;
-		status = "disabled";
-	};
-
-	i2c12: i2c@680 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x680 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 122 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c13_default>;
-		status = "disabled";
-	};
-
-	i2c13: i2c@700 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x700 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 123 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c14_default>;
-		status = "disabled";
-	};
-
-	i2c14: i2c@780 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x780 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 124 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c15_default>;
-		status = "disabled";
-	};
-
-	i2c15: i2c@800 {
-		#address-cells = <1>;
-		#size-cells = <0>;
-		reg = <0x800 0x80>;
-		compatible = "aspeed,ast2600-i2c-bus";
-		clocks = <&syscon ASPEED_CLK_APB2>;
-		resets = <&syscon ASPEED_RESET_I2C>;
-		interrupts = <GIC_SPI 125 IRQ_TYPE_LEVEL_HIGH>;
-		bus-frequency = <100000>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_i2c16_default>;
-		status = "disabled";
-	};
-};
diff --git a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi
index 9b54e3c01a345..3043ae7232dd0 100644
--- a/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi
+++ b/arch/arm/boot/dts/intel/ixp/intel-ixp42x-actiontec-mi424wr.dtsi
@@ -195,19 +195,19 @@ gpio1: gpio@1,0 {
 
 				pci-reset-hog {
 					gpio-hog;
-					gpios = <7 GPIO_ACTIVE_HIGH>;
+					gpios = <&gpio0 7 GPIO_ACTIVE_HIGH>;
 					output-high;
 					line-name = "PCI reset";
 				};
 				pstn-relay-hog-1 {
 					gpio-hog;
-					gpios = <11 GPIO_ACTIVE_HIGH>;
+					gpios = <&gpio0 11 GPIO_ACTIVE_HIGH>;
 					output-low;
 					line-name = "PSTN relay control 1";
 				};
 				pstn-relay-hog-2 {
 					gpio-hog;
-					gpios = <12 GPIO_ACTIVE_HIGH>;
+					gpios = <&gpio0 12 GPIO_ACTIVE_HIGH>;
 					output-low;
 					line-name = "PSTN relay control 2";
 				};
diff --git a/arch/arm/boot/dts/intel/socfpga/socfpga.dtsi b/arch/arm/boot/dts/intel/socfpga/socfpga.dtsi
index 5dc8d33e8ad73..98cb4cad2a100 100644
--- a/arch/arm/boot/dts/intel/socfpga/socfpga.dtsi
+++ b/arch/arm/boot/dts/intel/socfpga/socfpga.dtsi
@@ -37,13 +37,11 @@ cpu1: cpu@1 {
 		};
 	};
 
-	pmu: pmu@ff111000 {
+	pmu: pmu {
 		compatible = "arm,cortex-a9-pmu";
 		interrupt-parent = <&intc>;
 		interrupts = <0 176 4>, <0 177 4>;
 		interrupt-affinity = <&cpu0>, <&cpu1>;
-		reg = <0xff111000 0x1000>,
-		      <0xff113000 0x1000>;
 	};
 
 	intc: interrupt-controller@fffed000 {
diff --git a/arch/arm/boot/dts/intel/socfpga/socfpga_arria10.dtsi b/arch/arm/boot/dts/intel/socfpga/socfpga_arria10.dtsi
index a53a94678df2e..ba4c0ca897883 100644
--- a/arch/arm/boot/dts/intel/socfpga/socfpga_arria10.dtsi
+++ b/arch/arm/boot/dts/intel/socfpga/socfpga_arria10.dtsi
@@ -29,13 +29,11 @@ cpu1: cpu@1 {
 		};
 	};
 
-	pmu: pmu@ff111000 {
+	pmu: pmu {
 		compatible = "arm,cortex-a9-pmu";
 		interrupt-parent = <&intc>;
 		interrupts = <0 124 4>, <0 125 4>;
 		interrupt-affinity = <&cpu0>, <&cpu1>;
-		reg = <0xff111000 0x1000>,
-		      <0xff113000 0x1000>;
 	};
 
 	intc: interrupt-controller@ffffd000 {
diff --git a/arch/arm/boot/dts/intel/socfpga/socfpga_arria10_socdk_nand.dts b/arch/arm/boot/dts/intel/socfpga/socfpga_arria10_socdk_nand.dts
index a662df319a840..5f9b98b0b393a 100644
--- a/arch/arm/boot/dts/intel/socfpga/socfpga_arria10_socdk_nand.dts
+++ b/arch/arm/boot/dts/intel/socfpga/socfpga_arria10_socdk_nand.dts
@@ -18,9 +18,9 @@ partition@0 {
 			label = "Boot and fpga data";
 			reg = <0x0 0x02500000>;
 		};
-		partition@1c00000 {
+		partition@2500000 {
 			label = "Root Filesystem - JFFS2";
-			reg = <0x02500000 0x05500000>;
+			reg = <0x02500000 0x3db00000>;
 		};
 	};
 };
diff --git a/arch/arm/boot/dts/mediatek/mt6589.dtsi b/arch/arm/boot/dts/mediatek/mt6589.dtsi
index c6babc8ad2ba6..46dea445742b2 100644
--- a/arch/arm/boot/dts/mediatek/mt6589.dtsi
+++ b/arch/arm/boot/dts/mediatek/mt6589.dtsi
@@ -42,6 +42,17 @@ cpu@3 {
 
 	};
 
+	timer {
+		compatible = "arm,armv7-timer";
+		interrupt-parent = <&gic>;
+		interrupts = <GIC_PPI 13 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>,
+			     <GIC_PPI 14 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>,
+			     <GIC_PPI 11 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>,
+			     <GIC_PPI 10 (GIC_CPU_MASK_SIMPLE(4) | IRQ_TYPE_LEVEL_HIGH)>;
+		clock-frequency = <13000000>;
+		arm,cpu-registers-not-fw-configured;
+	};
+
 	clocks {
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/mediatek/mt7623.dtsi b/arch/arm/boot/dts/mediatek/mt7623.dtsi
index 71ac2b94c6ba3..aa42377b47da6 100644
--- a/arch/arm/boot/dts/mediatek/mt7623.dtsi
+++ b/arch/arm/boot/dts/mediatek/mt7623.dtsi
@@ -249,7 +249,7 @@ pericfg: syscon@10003000 {
 		#reset-cells = <1>;
 	};
 
-	pio: pinctrl@10005000 {
+	pio: pinctrl@1000b000 {
 		compatible = "mediatek,mt7623-pinctrl";
 		reg = <0 0x1000b000 0 0x1000>;
 		mediatek,pctl-regmap = <&syscfg_pctl_a>;
@@ -1017,14 +1017,14 @@ bdpsys: syscon@1c000000 {
 };
 
 &pio {
-	cir_pins_a:cir-default {
+	cir_pins_a: cir-default-pins {
 		pins-cir {
 			pinmux = <MT7623_PIN_46_IR_FUNC_IR>;
 			bias-disable;
 		};
 	};
 
-	i2c0_pins_a: i2c0-default {
+	i2c0_pins_a: i2c0-default-pins {
 		pins-i2c0 {
 			pinmux = <MT7623_PIN_75_SDA0_FUNC_SDA0>,
 				 <MT7623_PIN_76_SCL0_FUNC_SCL0>;
@@ -1032,40 +1032,40 @@ pins-i2c0 {
 		};
 	};
 
-	i2c1_pins_a: i2c1-default {
-		pin-i2c1 {
+	i2c1_pins_a: i2c1-default-pins {
+		pins-i2c1 {
 			pinmux = <MT7623_PIN_57_SDA1_FUNC_SDA1>,
 				 <MT7623_PIN_58_SCL1_FUNC_SCL1>;
 			bias-disable;
 		};
 	};
 
-	i2c1_pins_b: i2c1-alt {
-		pin-i2c1 {
+	i2c1_pins_b: i2c1-alt-pins {
+		pins-i2c1 {
 			pinmux = <MT7623_PIN_242_URTS2_FUNC_SCL1>,
 				 <MT7623_PIN_243_UCTS2_FUNC_SDA1>;
 			bias-disable;
 		};
 	};
 
-	i2c2_pins_a: i2c2-default {
-		pin-i2c2 {
+	i2c2_pins_a: i2c2-default-pins {
+		pins-i2c2 {
 			pinmux = <MT7623_PIN_77_SDA2_FUNC_SDA2>,
 				 <MT7623_PIN_78_SCL2_FUNC_SCL2>;
 			bias-disable;
 		};
 	};
 
-	i2c2_pins_b: i2c2-alt {
-		pin-i2c2 {
+	i2c2_pins_b: i2c2-alt-pins {
+		pins-i2c2 {
 			pinmux = <MT7623_PIN_122_GPIO122_FUNC_SDA2>,
 				 <MT7623_PIN_123_HTPLG_FUNC_SCL2>;
 			bias-disable;
 		};
 	};
 
-	i2s0_pins_a: i2s0-default {
-		pin-i2s0 {
+	i2s0_pins_a: i2s0-default-pins {
+		pins-i2s0 {
 			pinmux = <MT7623_PIN_49_I2S0_DATA_FUNC_I2S0_DATA>,
 				 <MT7623_PIN_72_I2S0_DATA_IN_FUNC_I2S0_DATA_IN>,
 				 <MT7623_PIN_73_I2S0_LRCK_FUNC_I2S0_LRCK>,
@@ -1076,8 +1076,8 @@ pin-i2s0 {
 		};
 	};
 
-	i2s1_pins_a: i2s1-default {
-		pin-i2s1 {
+	i2s1_pins_a: i2s1-default-pins {
+		pins-i2s1 {
 			pinmux = <MT7623_PIN_33_I2S1_DATA_FUNC_I2S1_DATA>,
 				 <MT7623_PIN_34_I2S1_DATA_IN_FUNC_I2S1_DATA_IN>,
 				 <MT7623_PIN_35_I2S1_BCK_FUNC_I2S1_BCK>,
@@ -1088,7 +1088,7 @@ pin-i2s1 {
 		};
 	};
 
-	key_pins_a: keys-alt {
+	key_pins_a: keys-alt-pins {
 		pins-keys {
 			pinmux = <MT7623_PIN_256_GPIO256_FUNC_GPIO256>,
 				 <MT7623_PIN_257_GPIO257_FUNC_GPIO257> ;
@@ -1096,7 +1096,7 @@ pins-keys {
 		};
 	};
 
-	led_pins_a: leds-alt {
+	led_pins_a: leds-alt-pins {
 		pins-leds {
 			pinmux = <MT7623_PIN_239_EXT_SDIO0_FUNC_GPIO239>,
 				 <MT7623_PIN_240_EXT_XCS_FUNC_GPIO240>,
@@ -1104,7 +1104,7 @@ pins-leds {
 		};
 	};
 
-	mmc0_pins_default: mmc0default {
+	mmc0_pins_default: mmc0-default-pins {
 		pins-cmd-dat {
 			pinmux = <MT7623_PIN_111_MSDC0_DAT7_FUNC_MSDC0_DAT7>,
 				 <MT7623_PIN_112_MSDC0_DAT6_FUNC_MSDC0_DAT6>,
@@ -1130,7 +1130,7 @@ pins-rst {
 		};
 	};
 
-	mmc0_pins_uhs: mmc0 {
+	mmc0_pins_uhs: mmc0-uhs-pins {
 		pins-cmd-dat {
 			pinmux = <MT7623_PIN_111_MSDC0_DAT7_FUNC_MSDC0_DAT7>,
 				 <MT7623_PIN_112_MSDC0_DAT6_FUNC_MSDC0_DAT6>,
@@ -1158,7 +1158,7 @@ pins-rst {
 		};
 	};
 
-	mmc1_pins_default: mmc1default {
+	mmc1_pins_default: mmc1-default-pins {
 		pins-cmd-dat {
 			pinmux = <MT7623_PIN_107_MSDC1_DAT0_FUNC_MSDC1_DAT0>,
 				 <MT7623_PIN_108_MSDC1_DAT1_FUNC_MSDC1_DAT1>,
@@ -1188,7 +1188,7 @@ pins-insert {
 		};
 	};
 
-	mmc1_pins_uhs: mmc1 {
+	mmc1_pins_uhs: mmc1-uhs-pins {
 		pins-cmd-dat {
 			pinmux = <MT7623_PIN_107_MSDC1_DAT0_FUNC_MSDC1_DAT0>,
 				 <MT7623_PIN_108_MSDC1_DAT1_FUNC_MSDC1_DAT1>,
@@ -1207,7 +1207,7 @@ pins-clk {
 		};
 	};
 
-	nand_pins_default: nanddefault {
+	nand_pins_default: nand-default-pins {
 		pins-ale {
 			pinmux = <MT7623_PIN_116_MSDC0_CMD_FUNC_NALE>;
 			drive-strength = <8>;
@@ -1236,15 +1236,15 @@ pins-we {
 		};
 	};
 
-	pcie_default: pcie_pin_default {
-		pins_cmd_dat {
+	pcie_default: pcie-default-pins {
+		pins-cmd-dat {
 			pinmux = <MT7623_PIN_208_AUD_EXT_CK1_FUNC_PCIE0_PERST_N>,
 				 <MT7623_PIN_209_AUD_EXT_CK2_FUNC_PCIE1_PERST_N>;
 			bias-disable;
 		};
 	};
 
-	pwm_pins_a: pwm-default {
+	pwm_pins_a: pwm-default-pins {
 		pins-pwm {
 			pinmux = <MT7623_PIN_203_PWM0_FUNC_PWM0>,
 				 <MT7623_PIN_204_PWM1_FUNC_PWM1>,
@@ -1254,7 +1254,7 @@ pins-pwm {
 		};
 	};
 
-	spi0_pins_a: spi0-default {
+	spi0_pins_a: spi0-default-pins {
 		pins-spi {
 			pinmux = <MT7623_PIN_53_SPI0_CSN_FUNC_SPI0_CS>,
 				<MT7623_PIN_54_SPI0_CK_FUNC_SPI0_CK>,
@@ -1264,7 +1264,7 @@ pins-spi {
 		};
 	};
 
-	spi1_pins_a: spi1-default {
+	spi1_pins_a: spi1-default-pins {
 		pins-spi {
 			pinmux = <MT7623_PIN_7_SPI1_CSN_FUNC_SPI1_CS>,
 				<MT7623_PIN_199_SPI1_CK_FUNC_SPI1_CK>,
@@ -1273,7 +1273,7 @@ pins-spi {
 		};
 	};
 
-	spi2_pins_a: spi2-default {
+	spi2_pins_a: spi2-default-pins {
 		pins-spi {
 			pinmux = <MT7623_PIN_101_SPI2_CSN_FUNC_SPI2_CS>,
 				 <MT7623_PIN_104_SPI2_CK_FUNC_SPI2_CK>,
@@ -1282,28 +1282,28 @@ pins-spi {
 		};
 	};
 
-	uart0_pins_a: uart0-default {
+	uart0_pins_a: uart0-default-pins {
 		pins-dat {
 			pinmux = <MT7623_PIN_79_URXD0_FUNC_URXD0>,
 				 <MT7623_PIN_80_UTXD0_FUNC_UTXD0>;
 		};
 	};
 
-	uart1_pins_a: uart1-default {
+	uart1_pins_a: uart1-default-pins {
 		pins-dat {
 			pinmux = <MT7623_PIN_81_URXD1_FUNC_URXD1>,
 				 <MT7623_PIN_82_UTXD1_FUNC_UTXD1>;
 		};
 	};
 
-	uart2_pins_a: uart2-default {
+	uart2_pins_a: uart2-default-pins {
 		pins-dat {
 			pinmux = <MT7623_PIN_14_GPIO14_FUNC_URXD2>,
 				 <MT7623_PIN_15_GPIO15_FUNC_UTXD2>;
 		};
 	};
 
-	uart2_pins_b: uart2-alt {
+	uart2_pins_b: uart2-alt-pins {
 		pins-dat {
 			pinmux = <MT7623_PIN_200_URXD2_FUNC_URXD2>,
 				 <MT7623_PIN_201_UTXD2_FUNC_UTXD2>;
diff --git a/arch/arm/boot/dts/mediatek/mt8135.dtsi b/arch/arm/boot/dts/mediatek/mt8135.dtsi
index 0f291ad22d3af..b03bd980ec375 100644
--- a/arch/arm/boot/dts/mediatek/mt8135.dtsi
+++ b/arch/arm/boot/dts/mediatek/mt8135.dtsi
@@ -148,7 +148,7 @@ pericfg: pericfg@10003000 {
 		 * Pinctrl access register at 0x10005000 and 0x1020c000 through
 		 * regmap. Register 0x1000b000 is used by EINT.
 		 */
-		pio: pinctrl@10005000 {
+		pio: pinctrl@1000b000 {
 			compatible = "mediatek,mt8135-pinctrl";
 			reg = <0 0x1000b000 0 0x1000>;
 			mediatek,pctl-regmap = <&syscfg_pctl_a &syscfg_pctl_b>;
diff --git a/arch/arm/boot/dts/microchip/sam9x7.dtsi b/arch/arm/boot/dts/microchip/sam9x7.dtsi
index d242d7a934d0f..c680a5033b6b4 100644
--- a/arch/arm/boot/dts/microchip/sam9x7.dtsi
+++ b/arch/arm/boot/dts/microchip/sam9x7.dtsi
@@ -990,9 +990,9 @@ gmac: ethernet@f802c000 {
 				     <62 IRQ_TYPE_LEVEL_HIGH 3>,	/* Queue 3 */
 				     <63 IRQ_TYPE_LEVEL_HIGH 3>,	/* Queue 4 */
 				     <64 IRQ_TYPE_LEVEL_HIGH 3>;	/* Queue 5 */
-			clocks = <&pmc PMC_TYPE_PERIPHERAL 24>, <&pmc PMC_TYPE_PERIPHERAL 24>, <&pmc PMC_TYPE_GCK 24>, <&pmc PMC_TYPE_GCK 67>;
-			clock-names = "hclk", "pclk", "tx_clk", "tsu_clk";
-			assigned-clocks = <&pmc PMC_TYPE_GCK 67>;
+			clocks = <&pmc PMC_TYPE_PERIPHERAL 24>, <&pmc PMC_TYPE_PERIPHERAL 24>, <&pmc PMC_TYPE_GCK 24>;
+			clock-names = "hclk", "pclk", "tsu_clk";
+			assigned-clocks = <&pmc PMC_TYPE_GCK 24>;
 			assigned-clock-rates = <266666666>;
 			status = "disabled";
 		};
diff --git a/arch/arm/boot/dts/nxp/imx/Makefile b/arch/arm/boot/dts/nxp/imx/Makefile
index 856c9f21bd703..1a2539fa19b44 100644
--- a/arch/arm/boot/dts/nxp/imx/Makefile
+++ b/arch/arm/boot/dts/nxp/imx/Makefile
@@ -376,6 +376,7 @@ dtb-$(CONFIG_SOC_IMX6UL) += \
 	imx6ul-tx6ul-0011.dtb \
 	imx6ul-tx6ul-mainboard.dtb \
 	imx6ul-var-som-concerto.dtb \
+	imx6ul-var-som-concerto-full.dtb \
 	imx6ull-14x14-evk.dtb \
 	imx6ull-colibri-aster.dtb \
 	imx6ull-colibri-emmc-aster.dtb \
@@ -414,6 +415,8 @@ dtb-$(CONFIG_SOC_IMX6UL) += \
 	imx6ull-tqma6ull2-mba6ulx.dtb \
 	imx6ull-tqma6ull2l-mba6ulx.dtb \
 	imx6ull-uti260b.dtb \
+	imx6ull-var-som-concerto.dtb \
+	imx6ull-var-som-concerto-full.dtb \
 	imx6ulz-14x14-evk.dtb \
 	imx6ulz-bsh-smm-m2.dtb
 dtb-$(CONFIG_SOC_IMX7D) += \
diff --git a/arch/arm/boot/dts/nxp/imx/imx25.dtsi b/arch/arm/boot/dts/nxp/imx/imx25.dtsi
index 94dbcef63b8cd..160533b037940 100644
--- a/arch/arm/boot/dts/nxp/imx/imx25.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx25.dtsi
@@ -618,7 +618,6 @@ nfc: nand-controller@bb000000 {
 				compatible = "fsl,imx25-nand";
 				reg = <0xbb000000 0x2000>;
 				clocks = <&clks 50>;
-				clock-names = "";
 				interrupts = <33>;
 				status = "disabled";
 			};
diff --git a/arch/arm/boot/dts/nxp/imx/imx35.dtsi b/arch/arm/boot/dts/nxp/imx/imx35.dtsi
index ab7b646399894..314c4f4845288 100644
--- a/arch/arm/boot/dts/nxp/imx/imx35.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx35.dtsi
@@ -369,7 +369,6 @@ nfc: nand-controller@bb000000 {
 				compatible = "fsl,imx35-nand", "fsl,imx25-nand";
 				reg = <0xbb000000 0x2000>;
 				clocks = <&clks 29>;
-				clock-names = "";
 				interrupts = <33>;
 				status = "disabled";
 			};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-audio.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-audio.dtsi
new file mode 100644
index 0000000000000..3c480bc7a6ad8
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-audio.dtsi
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Audio support for Variscite VAR-SOM-6UL module.
+ *
+ * Copyright 2019-2024 Variscite Ltd.
+ * Copyright 2026 Dimonoff
+ */
+
+&iomuxc {
+	pinctrl_sai2: sai2grp {
+		fsl,pins = <
+			MX6UL_PAD_JTAG_TDI__SAI2_TX_BCLK	0x17088
+			MX6UL_PAD_JTAG_TDO__SAI2_TX_SYNC	0x17088
+			MX6UL_PAD_JTAG_TRST_B__SAI2_TX_DATA	0x11088
+			MX6UL_PAD_JTAG_TCK__SAI2_RX_DATA	0x11088
+			MX6UL_PAD_JTAG_TMS__SAI2_MCLK		0x17088
+		>;
+	};
+};
+
+&sai2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai2>;
+	assigned-clocks = <&clks IMX6UL_CLK_SAI2_SEL>,
+			  <&clks IMX6UL_CLK_SAI2>;
+	assigned-clock-parents = <&clks IMX6UL_CLK_PLL4_AUDIO_DIV>;
+	assigned-clock-rates = <0>, <12288000>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-common.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-common.dtsi
new file mode 100644
index 0000000000000..5600eeaa5854d
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-common.dtsi
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Support for the common parts shared by all the different CPU options on
+ * Variscite VAR-SOM-6UL Module
+ *
+ * Copyright 2019 Variscite Ltd.
+ * Copyright 2025 Bootlin
+ */
+
+#include <dt-bindings/clock/imx6ul-clock.h>
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	model = "Variscite VAR-SOM-6UL module";
+	compatible = "variscite,var-som-imx6ul", "fsl,imx6ul";
+
+	memory@80000000 {
+		device_type = "memory";
+		reg = <0x80000000 0x20000000>;
+	};
+
+	reg_3p3v: regulator-3p3v {
+		compatible = "regulator-fixed";
+		regulator-name = "3P3V";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_gpio_dvfs: reg-gpio-dvfs {
+		compatible = "regulator-gpio";
+		regulator-min-microvolt = <1300000>;
+		regulator-max-microvolt = <1400000>;
+		regulator-name = "gpio_dvfs";
+		regulator-type = "voltage";
+		gpios = <&gpio4 13 GPIO_ACTIVE_HIGH>;
+		states = <1300000 0x1
+			  1400000 0x0>;
+	};
+};
+
+&clks {
+	assigned-clocks = <&clks IMX6UL_CLK_PLL4_AUDIO_DIV>;
+	assigned-clock-rates = <786432000>;
+};
+
+&iomuxc {
+	pinctrl_i2c1: i2c1grp {
+		fsl,pins = <
+			MX6UL_PAD_CSI_PIXCLK__I2C1_SCL		0x4001b8b0
+			MX6UL_PAD_CSI_MCLK__I2C1_SDA		0x4001b8b0
+		>;
+	};
+
+	pinctrl_tsc: tscgrp {
+		fsl,pins = <
+			MX6UL_PAD_GPIO1_IO01__GPIO1_IO01	0xb0
+			MX6UL_PAD_GPIO1_IO02__GPIO1_IO02	0xb0
+			MX6UL_PAD_GPIO1_IO03__GPIO1_IO03	0xb0
+			MX6UL_PAD_GPIO1_IO04__GPIO1_IO04	0xb0
+		>;
+	};
+
+	pinctrl_uart2: uart2grp {
+		fsl,pins = <
+			MX6UL_PAD_UART2_TX_DATA__UART2_DCE_TX	0x1b0b1
+			MX6UL_PAD_UART2_RX_DATA__UART2_DCE_RX	0x1b0b1
+			MX6UL_PAD_UART2_CTS_B__UART2_DCE_CTS	0x1b0b1
+			MX6UL_PAD_UART2_RTS_B__UART2_DCE_RTS	0x1b0b1
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX6UL_PAD_SD1_CMD__USDHC1_CMD		0x17059
+			MX6UL_PAD_SD1_CLK__USDHC1_CLK		0x17059
+			MX6UL_PAD_SD1_DATA0__USDHC1_DATA0	0x17059
+			MX6UL_PAD_SD1_DATA1__USDHC1_DATA1	0x17059
+			MX6UL_PAD_SD1_DATA2__USDHC1_DATA2	0x17059
+			MX6UL_PAD_SD1_DATA3__USDHC1_DATA3	0x17059
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX6UL_PAD_SD1_CMD__USDHC1_CMD		0x170b9
+			MX6UL_PAD_SD1_CLK__USDHC1_CLK		0x100b9
+			MX6UL_PAD_SD1_DATA0__USDHC1_DATA0	0x170b9
+			MX6UL_PAD_SD1_DATA1__USDHC1_DATA1	0x170b9
+			MX6UL_PAD_SD1_DATA2__USDHC1_DATA2	0x170b9
+			MX6UL_PAD_SD1_DATA3__USDHC1_DATA3	0x170b9
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX6UL_PAD_SD1_CMD__USDHC1_CMD		0x170f9
+			MX6UL_PAD_SD1_CLK__USDHC1_CLK		0x100f9
+			MX6UL_PAD_SD1_DATA0__USDHC1_DATA0	0x170f9
+			MX6UL_PAD_SD1_DATA1__USDHC1_DATA1	0x170f9
+			MX6UL_PAD_SD1_DATA2__USDHC1_DATA2	0x170f9
+			MX6UL_PAD_SD1_DATA3__USDHC1_DATA3	0x170f9
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX6UL_PAD_NAND_RE_B__USDHC2_CLK		0x10069
+			MX6UL_PAD_NAND_WE_B__USDHC2_CMD		0x17059
+			MX6UL_PAD_NAND_DATA00__USDHC2_DATA0	0x17059
+			MX6UL_PAD_NAND_DATA01__USDHC2_DATA1	0x17059
+			MX6UL_PAD_NAND_DATA02__USDHC2_DATA2	0x17059
+			MX6UL_PAD_NAND_DATA03__USDHC2_DATA3	0x17059
+			MX6UL_PAD_NAND_DATA04__USDHC2_DATA4	0x17059
+			MX6UL_PAD_NAND_DATA05__USDHC2_DATA5	0x17059
+			MX6UL_PAD_NAND_DATA06__USDHC2_DATA6	0x17059
+			MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x17059
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX6UL_PAD_NAND_RE_B__USDHC2_CLK		0x100b9
+			MX6UL_PAD_NAND_WE_B__USDHC2_CMD		0x170b9
+			MX6UL_PAD_NAND_DATA00__USDHC2_DATA0	0x170b9
+			MX6UL_PAD_NAND_DATA01__USDHC2_DATA1	0x170b9
+			MX6UL_PAD_NAND_DATA02__USDHC2_DATA2	0x170b9
+			MX6UL_PAD_NAND_DATA03__USDHC2_DATA3	0x170b9
+			MX6UL_PAD_NAND_DATA04__USDHC2_DATA4	0x170b9
+			MX6UL_PAD_NAND_DATA05__USDHC2_DATA5	0x170b9
+			MX6UL_PAD_NAND_DATA06__USDHC2_DATA6	0x170b9
+			MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x170b9
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX6UL_PAD_NAND_RE_B__USDHC2_CLK		0x100f9
+			MX6UL_PAD_NAND_WE_B__USDHC2_CMD		0x170f9
+			MX6UL_PAD_NAND_DATA00__USDHC2_DATA0	0x170f9
+			MX6UL_PAD_NAND_DATA01__USDHC2_DATA1	0x170f9
+			MX6UL_PAD_NAND_DATA02__USDHC2_DATA2	0x170f9
+			MX6UL_PAD_NAND_DATA03__USDHC2_DATA3	0x170f9
+			MX6UL_PAD_NAND_DATA04__USDHC2_DATA4	0x170f9
+			MX6UL_PAD_NAND_DATA05__USDHC2_DATA5	0x170f9
+			MX6UL_PAD_NAND_DATA06__USDHC2_DATA6	0x170f9
+			MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x170f9
+		>;
+	};
+};
+
+&pxp {
+	status = "okay";
+};
+
+&snvs_poweroff {
+	status = "okay";
+};
+
+&tsc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_tsc>;
+	xnur-gpios = <&gpio1 3 GPIO_ACTIVE_LOW>;
+	measure-delay-time = <0xffff>;
+	pre-charge-time = <0xfff>;
+	status = "okay";
+};
+
+&uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart2>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc2>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>;
+	bus-width = <8>;
+	no-1-8-v;
+	non-removable;
+	keep-power-in-suspend;
+	wakeup-source;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto-common.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto-common.dtsi
new file mode 100644
index 0000000000000..e5637310ba632
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto-common.dtsi
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Support for Variscite Concerto Carrier board with the VAR-SOM-6UL
+ * Variscite SoM mounted on it (all CPU variants)
+ *
+ * Copyright 2019 Variscite Ltd.
+ * Copyright 2025 Bootlin
+ */
+
+#include <dt-bindings/leds/common.h>
+
+/ {
+	chosen {
+		stdout-path = &uart1;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_key_back>, <&pinctrl_gpio_key_wakeup>;
+
+		key-back {
+			gpios = <&gpio4 14 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_BACK>;
+		};
+
+		key-wakeup {
+			gpios = <&gpio5 8 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_WAKEUP>;
+			wakeup-source;
+		};
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpio_leds>;
+
+		led-0 {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			label = "gpled2";
+			gpios = <&gpio1 25 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	lvds_panel: lvds-panel {
+		compatible = "sgd,gktw70sdae4se", "panel-lvds";
+		data-mapping = "jeida-18";
+		width-mm = <153>;
+		height-mm = <86>;
+
+		panel-timing {
+			clock-frequency = <35000000>;
+			hactive = <800>;
+			vactive = <480>;
+			hback-porch = <40>;
+			hfront-porch = <40>;
+			vback-porch = <29>;
+			vfront-porch = <13>;
+			hsync-len = <48>;
+			vsync-len = <3>;
+			hsync-active = <0>;
+			vsync-active = <0>;
+			de-active = <1>;
+			pixelclk-active = <0>;
+		};
+	};
+};
+
+&can1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	status = "okay";
+};
+
+&i2c1 {
+	clock-frequency = <100000>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_i2c1>;
+	status = "okay";
+
+	rtc@68 {
+		/*
+		 * To actually use this interrupt
+		 * connect pins J14.8 & J14.10 on the Concerto-Board.
+		 */
+		compatible = "dallas,ds1337";
+		reg = <0x68>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_rtc>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <10 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&iomuxc {
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <
+			MX6UL_PAD_UART3_RTS_B__FLEXCAN1_RX	0x1b020
+			MX6UL_PAD_UART3_CTS_B__FLEXCAN1_TX	0x1b020
+		>;
+	};
+
+	pinctrl_gpio_key_back: gpio-key-backgrp {
+		fsl,pins = <
+			MX6UL_PAD_NAND_CE1_B__GPIO4_IO14	0x17059
+		>;
+	};
+
+	pinctrl_gpio_key_wakeup: gpio-keys-wakeupgrp {
+		fsl,pins = <
+			MX6UL_PAD_SNVS_TAMPER8__GPIO5_IO08	0x17059
+		>;
+	};
+
+	pinctrl_gpio_leds: gpio-ledsgrp {
+		fsl,pins = <
+			MX6UL_PAD_UART3_RX_DATA__GPIO1_IO25	0x1b0b0	/* GPLED2 */
+		>;
+	};
+
+	pinctrl_rtc: rtcgrp {
+		fsl,pins = <
+			MX6UL_PAD_JTAG_MOD__GPIO1_IO10		0x1b0b0 /* RTC alarm IRQ */
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX6UL_PAD_UART1_TX_DATA__UART1_DCE_TX	0x1b0b1
+			MX6UL_PAD_UART1_RX_DATA__UART1_DCE_RX	0x1b0b1
+		>;
+	};
+
+	pinctrl_uart5: uart5grp {
+		fsl,pins = <
+			MX6UL_PAD_CSI_DATA00__UART5_DCE_TX	0x1b0b1
+			MX6UL_PAD_CSI_DATA01__UART5_DCE_RX	0x1b0b1
+			MX6UL_PAD_GPIO1_IO09__UART5_DCE_CTS	0x1b0b1
+			MX6UL_PAD_GPIO1_IO08__UART5_DCE_RTS	0x1b0b1
+		>;
+	};
+
+	pinctrl_usb_otg1_id: usbotg1idgrp {
+		fsl,pins = <
+			MX6UL_PAD_UART3_TX_DATA__ANATOP_OTG1_ID	0x17059
+		>;
+	};
+
+	pinctrl_wdog: wdoggrp {
+		fsl,pins = <
+			MX6UL_PAD_GPIO1_IO01__WDOG1_WDOG_B	0x78b0
+		>;
+	};
+};
+
+&snvs_pwrkey {
+	status = "disabled";
+};
+
+&snvs_rtc {
+	status = "disabled";
+};
+
+&tsc {
+	/*
+	 * Conflics with wdog1 ext-reset-output & SD CD pins,
+	 * so we keep it disabled by default.
+	 */
+	status = "disabled";
+};
+
+/* Console UART */
+&uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+/* ttymxc4 UART */
+&uart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart5>;
+	uart-has-rtscts;
+	status = "okay";
+};
+
+&usbotg1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usb_otg1_id>;
+	dr_mode = "otg";
+	disable-over-current;
+	srp-disable;
+	hnp-disable;
+	adp-disable;
+	status = "okay";
+};
+
+&usbotg2 {
+	dr_mode = "host";
+	disable-over-current;
+	status = "okay";
+};
+
+&wdog1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_wdog>;
+	/*
+	 * To actually use ext-reset-output
+	 * connect pins J17.3 & J17.8 on the Concerto-Board
+	 */
+	fsl,ext-reset-output;
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto-full.dts b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto-full.dts
new file mode 100644
index 0000000000000..725f34d6b7ee9
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto-full.dts
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Support for Variscite MX6 Concerto Carrier board with the VAR-SOM-6UL
+ * Variscite SoM mounted on it (6UL CPU variant).
+ *
+ * Copyright 2026 Dimonoff
+ */
+
+/dts-v1/;
+
+#include "imx6ul-var-som.dtsi"
+#include "imx6ul-var-som-concerto-common.dtsi"
+#include "imx6ul-var-som-wifi.dtsi"
+#include "imx6ul-var-som-enet2.dtsi"
+#include "imx6ul-var-som-enet1.dtsi"
+#include "imx6ul-var-som-audio.dtsi"
+#include "imx6ul-var-som-lvds-panel.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-6UL Concerto Board (6UL CPU)";
+	compatible = "variscite,mx6ulconcerto", "variscite,var-som-imx6ul", "fsl,imx6ul";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto.dts b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto.dts
index 9ff3b374a2b31..c249e15772b82 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto.dts
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-concerto.dts
@@ -1,320 +1,22 @@
 // SPDX-License-Identifier: GPL-2.0+
 /*
- * Support for Variscite MX6 Concerto Carrier board with the VAR-SOM-MX6UL
- * Variscite SoM mounted on it
+ * Support for Variscite Concerto Carrier board with the VAR-SOM-6UL
+ * Variscite SoM mounted on it (6UL CPU variant)
  *
  * Copyright 2019 Variscite Ltd.
  * Copyright 2025 Bootlin
  */
 
+/dts-v1/;
+
 #include "imx6ul-var-som.dtsi"
-#include <dt-bindings/leds/common.h>
+#include "imx6ul-var-som-concerto-common.dtsi"
+#include "imx6ul-var-som-sd.dtsi"
+#include "imx6ul-var-som-enet2.dtsi"
+#include "imx6ul-var-som-audio.dtsi"
+#include "imx6ul-var-som-lvds-panel.dtsi"
 
 / {
-	model = "Variscite VAR-SOM-MX6UL Concerto Board";
+	model = "Variscite VAR-SOM-6UL Concerto Board (6UL CPU)";
 	compatible = "variscite,mx6ulconcerto", "variscite,var-som-imx6ul", "fsl,imx6ul";
-
-	chosen {
-		stdout-path = &uart1;
-	};
-
-	gpio-keys {
-		compatible = "gpio-keys";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_gpio_key_back>, <&pinctrl_gpio_key_wakeup>;
-
-		key-back {
-			gpios = <&gpio4 14 GPIO_ACTIVE_LOW>;
-			linux,code = <KEY_BACK>;
-		};
-
-		key-wakeup {
-			gpios = <&gpio5 8 GPIO_ACTIVE_LOW>;
-			linux,code = <KEY_WAKEUP>;
-			wakeup-source;
-		};
-	};
-
-	leds {
-		compatible = "gpio-leds";
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_gpio_leds>;
-
-		led-0 {
-			function = LED_FUNCTION_STATUS;
-			color = <LED_COLOR_ID_GREEN>;
-			label = "gpled2";
-			gpios = <&gpio1 25 GPIO_ACTIVE_HIGH>;
-			linux,default-trigger = "heartbeat";
-		};
-	};
-};
-
-&can1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_flexcan1>;
-	status = "okay";
-};
-
-&fec1 {
-	status = "disabled";
-};
-
-&fec2 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_enet2>, <&pinctrl_enet2_gpio>, <&pinctrl_enet2_mdio>;
-	phy-mode = "rmii";
-	phy-handle = <&ethphy1>;
-	status = "okay";
-
-	mdio {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		ethphy1: ethernet-phy@3 {
-			compatible = "ethernet-phy-ieee802.3-c22";
-			reg = <3>;
-			clocks = <&rmii_ref_clk>;
-			clock-names = "rmii-ref";
-			reset-gpios = <&gpio5 5 GPIO_ACTIVE_LOW>;
-			reset-assert-us = <100000>;
-			micrel,led-mode = <0>;
-			micrel,rmii-reference-clock-select-25-mhz = <1>;
-		};
-	};
-};
-
-&i2c1 {
-	clock-frequency = <100000>;
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_i2c1>;
-	status = "okay";
-
-	rtc@68 {
-		/*
-		 * To actually use this interrupt
-		 * connect pins J14.8 & J14.10 on the Concerto-Board.
-		 */
-		compatible = "dallas,ds1337";
-		reg = <0x68>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&pinctrl_rtc>;
-		interrupt-parent = <&gpio1>;
-		interrupts = <10 IRQ_TYPE_EDGE_FALLING>;
-	};
-};
-
-&iomuxc {
-	pinctrl_enet2: enet2grp {
-		fsl,pins = <
-			MX6UL_PAD_ENET2_RX_EN__ENET2_RX_EN	0x1b0b0
-			MX6UL_PAD_ENET2_RX_ER__ENET2_RX_ER	0x1b0b0
-			MX6UL_PAD_ENET2_RX_DATA0__ENET2_RDATA00	0x1b0b0
-			MX6UL_PAD_ENET2_RX_DATA1__ENET2_RDATA01	0x1b0b0
-			MX6UL_PAD_ENET2_TX_EN__ENET2_TX_EN	0x1b0b0
-			MX6UL_PAD_ENET2_TX_DATA0__ENET2_TDATA00	0x1b0b0
-			MX6UL_PAD_ENET2_TX_DATA1__ENET2_TDATA01	0x1b0b0
-			MX6UL_PAD_ENET2_TX_CLK__ENET2_REF_CLK2	0x4001b031
-		>;
-	};
-
-	pinctrl_enet2_gpio: enet2-gpiogrp {
-		fsl,pins = <
-			MX6UL_PAD_SNVS_TAMPER5__GPIO5_IO05	0x1b0b0 /* fec2 reset */
-		>;
-	};
-
-	pinctrl_enet2_mdio: enet2-mdiogrp {
-		fsl,pins = <
-			MX6UL_PAD_GPIO1_IO06__ENET2_MDIO	0x1b0b0
-			MX6UL_PAD_GPIO1_IO07__ENET2_MDC		0x1b0b0
-		>;
-	};
-
-	pinctrl_flexcan1: flexcan1grp {
-		fsl,pins = <
-			MX6UL_PAD_UART3_RTS_B__FLEXCAN1_RX	0x1b020
-			MX6UL_PAD_UART3_CTS_B__FLEXCAN1_TX	0x1b020
-		>;
-	};
-
-	pinctrl_gpio_key_back: gpio-key-backgrp {
-		fsl,pins = <
-			MX6UL_PAD_NAND_CE1_B__GPIO4_IO14	0x17059
-		>;
-	};
-
-	pinctrl_gpio_leds: gpio-ledsgrp {
-		fsl,pins = <
-			MX6UL_PAD_UART3_RX_DATA__GPIO1_IO25	0x1b0b0	/* GPLED2 */
-		>;
-	};
-
-	pinctrl_gpio_key_wakeup: gpio-keys-wakeupgrp {
-		fsl,pins = <
-			MX6UL_PAD_SNVS_TAMPER8__GPIO5_IO08	0x17059
-		>;
-	};
-
-	pinctrl_i2c1: i2c1grp {
-		fsl,pins = <
-			MX6UL_PAD_CSI_PIXCLK__I2C1_SCL		0x4001b8b0
-			MX6UL_PAD_CSI_MCLK__I2C1_SDA		0x4001b8b0
-		>;
-	};
-
-	pinctrl_pwm4: pwm4grp {
-		fsl,pins = <
-			MX6UL_PAD_GPIO1_IO05__PWM4_OUT		0x110b0
-		>;
-	};
-
-	pinctrl_rtc: rtcgrp {
-		fsl,pins = <
-			MX6UL_PAD_JTAG_MOD__GPIO1_IO10		0x1b0b0 /* RTC alarm IRQ */
-		>;
-	};
-
-	pinctrl_uart1: uart1grp {
-		fsl,pins = <
-			MX6UL_PAD_UART1_TX_DATA__UART1_DCE_TX	0x1b0b1
-			MX6UL_PAD_UART1_RX_DATA__UART1_DCE_RX	0x1b0b1
-		>;
-	};
-
-	pinctrl_uart5: uart5grp {
-		fsl,pins = <
-			MX6UL_PAD_CSI_DATA00__UART5_DCE_TX	0x1b0b1
-			MX6UL_PAD_CSI_DATA01__UART5_DCE_RX	0x1b0b1
-			MX6UL_PAD_GPIO1_IO09__UART5_DCE_CTS	0x1b0b1
-			MX6UL_PAD_GPIO1_IO08__UART5_DCE_RTS	0x1b0b1
-		>;
-	};
-
-	pinctrl_usb_otg1_id: usbotg1idgrp {
-		fsl,pins = <
-			MX6UL_PAD_UART3_TX_DATA__ANATOP_OTG1_ID	0x17059
-		>;
-	};
-
-	pinctrl_usdhc1: usdhc1grp {
-		fsl,pins = <
-			MX6UL_PAD_SD1_CMD__USDHC1_CMD		0x17059
-			MX6UL_PAD_SD1_CLK__USDHC1_CLK		0x17059
-			MX6UL_PAD_SD1_DATA0__USDHC1_DATA0	0x17059
-			MX6UL_PAD_SD1_DATA1__USDHC1_DATA1	0x17059
-			MX6UL_PAD_SD1_DATA2__USDHC1_DATA2	0x17059
-			MX6UL_PAD_SD1_DATA3__USDHC1_DATA3	0x17059
-		>;
-	};
-
-	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
-		fsl,pins = <
-			MX6UL_PAD_SD1_CMD__USDHC1_CMD		0x170b9
-			MX6UL_PAD_SD1_CLK__USDHC1_CLK		0x100b9
-			MX6UL_PAD_SD1_DATA0__USDHC1_DATA0	0x170b9
-			MX6UL_PAD_SD1_DATA1__USDHC1_DATA1	0x170b9
-			MX6UL_PAD_SD1_DATA2__USDHC1_DATA2	0x170b9
-			MX6UL_PAD_SD1_DATA3__USDHC1_DATA3	0x170b9
-		>;
-	};
-
-	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
-		fsl,pins = <
-			MX6UL_PAD_SD1_CMD__USDHC1_CMD		0x170f9
-			MX6UL_PAD_SD1_CLK__USDHC1_CLK		0x100f9
-			MX6UL_PAD_SD1_DATA0__USDHC1_DATA0	0x170f9
-			MX6UL_PAD_SD1_DATA1__USDHC1_DATA1	0x170f9
-			MX6UL_PAD_SD1_DATA2__USDHC1_DATA2	0x170f9
-			MX6UL_PAD_SD1_DATA3__USDHC1_DATA3	0x170f9
-		>;
-	};
-
-	pinctrl_usdhc1_gpio: usdhc1-gpiogrp {
-		fsl,pins = <
-			MX6UL_PAD_GPIO1_IO00__GPIO1_IO00	0x1b0b1 /* CD */
-		>;
-	};
-
-	pinctrl_wdog: wdoggrp {
-		fsl,pins = <
-			MX6UL_PAD_GPIO1_IO01__WDOG1_WDOG_B	0x78b0
-		>;
-	};
-};
-
-&pwm4 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_pwm4>;
-	status = "okay";
-};
-
-&snvs_pwrkey {
-	status = "disabled";
-};
-
-&snvs_rtc {
-	status = "disabled";
-};
-
-&tsc {
-	/*
-	 * Conflics with wdog1 ext-reset-output & SD CD pins,
-	 * so we keep it disabled by default.
-	 */
-	status = "disabled";
-};
-
-/* Console UART */
-&uart1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart1>;
-	status = "okay";
-};
-
-/* ttymxc4 UART */
-&uart5 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart5>;
-	uart-has-rtscts;
-	status = "okay";
-};
-
-&usbotg1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_usb_otg1_id>;
-	dr_mode = "otg";
-	disable-over-current;
-	srp-disable;
-	hnp-disable;
-	adp-disable;
-	status = "okay";
-};
-
-&usbotg2 {
-	dr_mode = "host";
-	disable-over-current;
-	status = "okay";
-};
-
-&usdhc1 {
-	pinctrl-names = "default", "state_100mhz", "state_200mhz";
-	pinctrl-0 = <&pinctrl_usdhc1>, <&pinctrl_usdhc1_gpio>;
-	pinctrl-1 = <&pinctrl_usdhc1_100mhz>, <&pinctrl_usdhc1_gpio>;
-	pinctrl-2 = <&pinctrl_usdhc1_200mhz>, <&pinctrl_usdhc1_gpio>;
-	cd-gpios = <&gpio1 0 GPIO_ACTIVE_LOW>;
-	no-1-8-v;
-	keep-power-in-suspend;
-	wakeup-source;
-	status = "okay";
-};
-
-&wdog1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_wdog>;
-	/*
-	 * To actually use ext-reset-output
-	 * connect pins J17.3 & J17.8 on the Concerto-Board
-	 */
-	fsl,ext-reset-output;
 };
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-enet1.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-enet1.dtsi
new file mode 100644
index 0000000000000..6b1e34347bec7
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-enet1.dtsi
@@ -0,0 +1,44 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Ethernet ENET1 support for Variscite VAR-SOM-6UL module with
+ * the EC configuration option ((ethernet PHY assembled on SOM).
+ *
+ * Copyright 2019-2024 Variscite Ltd.
+ * Copyright 2026 Dimonoff
+ */
+
+&fec1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet1>, <&pinctrl_enet1_gpio>;
+	phy-mode = "rmii";
+	phy-handle = <&ethphy0>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_enet1: enet1grp {
+		fsl,pins = <
+			MX6UL_PAD_ENET1_RX_EN__ENET1_RX_EN	0x1b0b0
+			MX6UL_PAD_ENET1_RX_ER__ENET1_RX_ER	0x1b0b0
+			MX6UL_PAD_ENET1_RX_DATA0__ENET1_RDATA00	0x1b0b0
+			MX6UL_PAD_ENET1_RX_DATA1__ENET1_RDATA01	0x1b0b0
+			MX6UL_PAD_ENET1_TX_EN__ENET1_TX_EN	0x1b0b0
+			MX6UL_PAD_ENET1_TX_DATA0__ENET1_TDATA00	0x1b0b0
+			MX6UL_PAD_ENET1_TX_DATA1__ENET1_TDATA01	0x1b0b0
+			MX6UL_PAD_ENET1_TX_CLK__ENET1_REF_CLK1	0x4001b031
+		>;
+	};
+};
+
+&mdio_enet2 {
+	ethphy0: ethernet-phy@1 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <1>;
+		clocks = <&rmii_ref_clk>;
+		clock-names = "rmii-ref";
+		reset-gpios = <&gpio5 0 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <100000>;
+		micrel,led-mode = <1>;
+		micrel,rmii-reference-clock-select-25-mhz;
+	};
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-enet2.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-enet2.dtsi
new file mode 100644
index 0000000000000..b29fcdc079e37
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-enet2.dtsi
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Ethernet ENET2 support for Variscite VAR-SOM-6UL module.
+ *
+ * Copyright 2019-2024 Variscite Ltd.
+ * Copyright 2026 Dimonoff
+ */
+
+/ {
+	rmii_ref_clk: rmii-ref-clk {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+		clock-output-names = "rmii-ref";
+	};
+};
+
+&fec2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_enet2>, <&pinctrl_enet2_gpio>, <&pinctrl_enet2_mdio>;
+	phy-mode = "rmii";
+	phy-handle = <&ethphy1>;
+	status = "okay";
+
+	mdio_enet2: mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			clocks = <&rmii_ref_clk>;
+			clock-names = "rmii-ref";
+			reset-gpios = <&gpio5 0 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <100000>;
+			micrel,led-mode = <1>;
+			micrel,rmii-reference-clock-select-25-mhz;
+		};
+
+		ethphy1: ethernet-phy@3 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <3>;
+			clocks = <&rmii_ref_clk>;
+			clock-names = "rmii-ref";
+			reset-gpios = <&gpio5 5 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <100000>;
+			micrel,led-mode = <0>;
+			micrel,rmii-reference-clock-select-25-mhz;
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_enet2: enet2grp {
+		fsl,pins = <
+			MX6UL_PAD_ENET2_RX_EN__ENET2_RX_EN	0x1b0b0
+			MX6UL_PAD_ENET2_RX_ER__ENET2_RX_ER	0x1b0b0
+			MX6UL_PAD_ENET2_RX_DATA0__ENET2_RDATA00	0x1b0b0
+			MX6UL_PAD_ENET2_RX_DATA1__ENET2_RDATA01	0x1b0b0
+			MX6UL_PAD_ENET2_TX_EN__ENET2_TX_EN	0x1b0b0
+			MX6UL_PAD_ENET2_TX_DATA0__ENET2_TDATA00	0x1b0b0
+			MX6UL_PAD_ENET2_TX_DATA1__ENET2_TDATA01	0x1b0b0
+			MX6UL_PAD_ENET2_TX_CLK__ENET2_REF_CLK2	0x4001b031
+		>;
+	};
+
+	pinctrl_enet2_gpio: enet2-gpiogrp {
+		fsl,pins = <
+			MX6UL_PAD_SNVS_TAMPER5__GPIO5_IO05	0x1b0b0 /* fec2 reset */
+		>;
+	};
+
+	pinctrl_enet2_mdio: enet2-mdiogrp {
+		fsl,pins = <
+			MX6UL_PAD_GPIO1_IO06__ENET2_MDIO	0x1b0b0
+			MX6UL_PAD_GPIO1_IO07__ENET2_MDC		0x1b0b0
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-lvds-panel.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-lvds-panel.dtsi
new file mode 100644
index 0000000000000..996b37d35d6e0
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-lvds-panel.dtsi
@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * LVDS panel support for Variscite VAR-SOM-6UL module.
+ *
+ * Copyright 2019-2024 Variscite Ltd.
+ * Copyright 2026 Dimonoff
+ */
+
+/ {
+	lcd_backlight: lcd-backlight {
+		compatible = "pwm-backlight";
+		pwms = <&pwm4 0 2000000 0>;
+		pwm-names = "LCD_BKLT_PWM";
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <6>;
+		status = "okay";
+	};
+
+	lvds_encoder: lvds-encoder {
+		compatible = "ti,sn75lvds93", "lvds-encoder";
+		power-supply = <&reg_3p3v>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				lvds_enc_in: endpoint {
+					remote-endpoint = <&lcdif_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				lvds_enc_out: endpoint {
+					remote-endpoint = <&lvds_panel_in>;
+				};
+			};
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_lcdif_ctrl: lcdif-ctrl-grp {
+		fsl,pins = <
+			MX6UL_PAD_LCD_CLK__LCDIF_CLK		0x79
+			MX6UL_PAD_LCD_ENABLE__LCDIF_ENABLE	0x79
+		>;
+	};
+
+	pinctrl_lcdif_dat: lcdif-dat-grp {
+		fsl,pins = <
+			MX6UL_PAD_LCD_DATA02__LCDIF_DATA02	0x79
+			MX6UL_PAD_LCD_DATA03__LCDIF_DATA03	0x79
+			MX6UL_PAD_LCD_DATA04__LCDIF_DATA04	0x79
+			MX6UL_PAD_LCD_DATA05__LCDIF_DATA05	0x79
+			MX6UL_PAD_LCD_DATA06__LCDIF_DATA06	0x79
+			MX6UL_PAD_LCD_DATA07__LCDIF_DATA07	0x79
+			MX6UL_PAD_LCD_DATA10__LCDIF_DATA10	0x79
+			MX6UL_PAD_LCD_DATA11__LCDIF_DATA11	0x79
+			MX6UL_PAD_LCD_DATA12__LCDIF_DATA12	0x79
+			MX6UL_PAD_LCD_DATA13__LCDIF_DATA13	0x79
+			MX6UL_PAD_LCD_DATA14__LCDIF_DATA14	0x79
+			MX6UL_PAD_LCD_DATA15__LCDIF_DATA15	0x79
+			MX6UL_PAD_LCD_DATA18__LCDIF_DATA18	0x79
+			MX6UL_PAD_LCD_DATA19__LCDIF_DATA19	0x79
+			MX6UL_PAD_LCD_DATA20__LCDIF_DATA20	0x79
+			MX6UL_PAD_LCD_DATA21__LCDIF_DATA21	0x79
+			MX6UL_PAD_LCD_DATA22__LCDIF_DATA22	0x79
+			MX6UL_PAD_LCD_DATA23__LCDIF_DATA23	0x79
+		>;
+	};
+
+	pinctrl_pwm4: pwm4-grp {
+		fsl,pins = <
+			MX6UL_PAD_GPIO1_IO05__PWM4_OUT		0x110b0 /* LCD BACKLIGHT */
+		>;
+	};
+};
+
+&lcdif {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lcdif_dat
+		     &pinctrl_lcdif_ctrl>;
+	status = "okay";
+
+	port {
+		lcdif_out: endpoint {
+			remote-endpoint = <&lvds_enc_in>;
+		};
+	};
+};
+
+&lvds_panel {
+	status = "okay";
+
+	port {
+		lvds_panel_in: endpoint {
+			remote-endpoint = <&lvds_enc_out>;
+		};
+	};
+};
+
+/* PWM LCD */
+&pwm4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pwm4>;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-sd.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-sd.dtsi
new file mode 100644
index 0000000000000..0e6d9b945eb4a
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-sd.dtsi
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Support optional SD card interface on Variscite VAR-SOM-6UL module.
+ *
+ * Copyright 2019-2024 Variscite Ltd.
+ * Copyright 2026 Dimonoff
+ */
+
+&iomuxc {
+	pinctrl_usdhc1_gpio: usdhc1-gpiogrp {
+		fsl,pins = <
+			MX6UL_PAD_GPIO1_IO00__GPIO1_IO00	0x1b0b1 /* CD */
+		>;
+	};
+};
+
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>, <&pinctrl_usdhc1_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>, <&pinctrl_usdhc1_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>, <&pinctrl_usdhc1_gpio>;
+	cd-gpios = <&gpio1 0 GPIO_ACTIVE_LOW>;
+	no-1-8-v;
+	keep-power-in-suspend;
+	wakeup-source;
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-wifi.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-wifi.dtsi
new file mode 100644
index 0000000000000..6d16ff7909dab
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som-wifi.dtsi
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Support optional Wifi/Bluetooth on Variscite VAR-SOM-6UL module.
+ *
+ * Copyright 2019-2024 Variscite Ltd.
+ * Copyright 2026 Dimonoff
+ */
+
+/ {
+	reg_sd1_vmmc: regulator_sd1_vmmc {
+		compatible = "regulator-fixed";
+		regulator-name = "VMMC1";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&gpio5 2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		startup-delay-us = <10000>;
+	};
+
+	usdhc1_pwrseq: usdhc1-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_brcm_wifi>;
+		reset-gpios = <&gpio5 6 GPIO_ACTIVE_LOW>;
+	};
+};
+
+&iomuxc {
+	pinctrl_32k_clk: 32kclkgrp {
+		/*
+		 * For TP option, an additional oscillator is assembled on the
+		 * SOM to provide 32 kHz to the WiFi module. Without TP option,
+		 * this pin is configured to provide the 32 KHz clock to the
+		 * WiFi module.
+		 */
+		fsl,pins = <
+			MX6UL_PAD_GPIO1_IO03__OSC32K_32K_OUT	0x03029
+		>;
+	};
+};
+
+&tsc {
+	status = "disabled";
+};
+
+/* Bluetooth UART */
+&uart2 {
+	bluetooth {
+		compatible = "brcm,bcm43438-bt";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_brcm_bt>;
+		shutdown-gpios = <&gpio5 4 GPIO_ACTIVE_HIGH>;
+		vbat-supply = <&reg_3p3v>;
+		vddio-supply = <&reg_3p3v>;
+	};
+};
+
+&usdhc1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>, <&pinctrl_32k_clk>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>, <&pinctrl_32k_clk>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>, <&pinctrl_32k_clk>;
+	no-1-8-v;
+	non-removable;
+	mmc-pwrseq = <&usdhc1_pwrseq>;
+	vmmc-supply = <&reg_sd1_vmmc>;
+	status = "okay";
+
+	brcmf: wifi@1 {
+		compatible = "brcm,bcm4329-fmac"; /* LWB option: Sterling LWB5 */
+		reg = <1>;
+	};
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som.dtsi
index 4e536e0252def..feea24c0e0683 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul-var-som.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul-var-som.dtsi
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0+
 /*
- * Support for Variscite VAR-SOM-MX6UL Module
+ * Support for Variscite VAR-SOM-6UL Module
  *
  * Copyright 2019 Variscite Ltd.
  * Copyright 2025 Bootlin
@@ -9,225 +9,30 @@
 /dts-v1/;
 
 #include "imx6ul.dtsi"
-#include <dt-bindings/clock/imx6ul-clock.h>
-#include <dt-bindings/gpio/gpio.h>
+#include "imx6ul-var-som-common.dtsi"
 
 / {
-	model = "Variscite VAR-SOM-MX6UL module";
+	model = "Variscite VAR-SOM-6UL module";
 	compatible = "variscite,var-som-imx6ul", "fsl,imx6ul";
-
-	memory@80000000 {
-		device_type = "memory";
-		reg = <0x80000000 0x20000000>;
-	};
-
-	reg_gpio_dvfs: reg-gpio-dvfs {
-		compatible = "regulator-gpio";
-		regulator-min-microvolt = <1300000>;
-		regulator-max-microvolt = <1400000>;
-		regulator-name = "gpio_dvfs";
-		regulator-type = "voltage";
-		gpios = <&gpio4 13 GPIO_ACTIVE_HIGH>;
-		states = <1300000 0x1
-			  1400000 0x0>;
-	};
-
-	rmii_ref_clk: rmii-ref-clk {
-		compatible = "fixed-clock";
-		#clock-cells = <0>;
-		clock-frequency = <25000000>;
-		clock-output-names = "rmii-ref";
-	};
-};
-
-&clks {
-	assigned-clocks = <&clks IMX6UL_CLK_PLL4_AUDIO_DIV>;
-	assigned-clock-rates = <786432000>;
-};
-
-&cpu0 {
-	dc-supply = <&reg_gpio_dvfs>;
-};
-
-&fec1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_enet1>, <&pinctrl_enet1_gpio>, <&pinctrl_enet1_mdio>;
-	phy-mode = "rmii";
-	phy-handle = <&ethphy0>;
-	status = "okay";
-
-	mdio {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		ethphy0: ethernet-phy@1 {
-			compatible = "ethernet-phy-ieee802.3-c22";
-			reg = <1>;
-			clocks = <&rmii_ref_clk>;
-			clock-names = "rmii-ref";
-			reset-gpios = <&gpio5 0 GPIO_ACTIVE_LOW>;
-			reset-assert-us = <100000>;
-			micrel,led-mode = <1>;
-			micrel,rmii-reference-clock-select-25-mhz = <1>;
-		};
-	};
 };
 
 &iomuxc {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_hog>;
-
-	pinctrl_enet1: enet1grp {
-		fsl,pins = <
-			MX6UL_PAD_ENET1_RX_EN__ENET1_RX_EN	0x1b0b0
-			MX6UL_PAD_ENET1_RX_ER__ENET1_RX_ER	0x1b0b0
-			MX6UL_PAD_ENET1_RX_DATA0__ENET1_RDATA00	0x1b0b0
-			MX6UL_PAD_ENET1_RX_DATA1__ENET1_RDATA01	0x1b0b0
-			MX6UL_PAD_ENET1_TX_EN__ENET1_TX_EN	0x1b0b0
-			MX6UL_PAD_ENET1_TX_DATA0__ENET1_TDATA00	0x1b0b0
-			MX6UL_PAD_ENET1_TX_DATA1__ENET1_TDATA01	0x1b0b0
-			MX6UL_PAD_ENET1_TX_CLK__ENET1_REF_CLK1	0x4001b031
-		>;
-	};
-
-	pinctrl_enet1_gpio: enet1-gpiogrp {
-		fsl,pins = <
-			MX6UL_PAD_SNVS_TAMPER0__GPIO5_IO00	0x1b0b0 /* fec1 reset */
-		>;
-	};
-
-	pinctrl_enet1_mdio: enet1-mdiogrp {
-		fsl,pins = <
-			MX6UL_PAD_GPIO1_IO06__ENET1_MDIO	0x1b0b0
-			MX6UL_PAD_GPIO1_IO07__ENET1_MDC		0x1b0b0
-		>;
-	};
-
-	pinctrl_hog: hoggrp {
+	pinctrl_brcm_bt: brcm-bt-grp {
 		fsl,pins = <
-			MX6UL_PAD_SNVS_TAMPER4__GPIO5_IO04	0x1b0b0	/* BT Enable */
-			MX6UL_PAD_SNVS_TAMPER6__GPIO5_IO06	0x03029	/* WLAN Enable */
+			MX6UL_PAD_SNVS_TAMPER4__GPIO5_IO04	0x1b0b0	/* BT_REG_ON (BT_EN) */
 		>;
 	};
 
-	pinctrl_sai2: sai2grp {
+	pinctrl_brcm_wifi: brcm-wifi-grp {
 		fsl,pins = <
-			MX6UL_PAD_JTAG_TDI__SAI2_TX_BCLK	0x17088
-			MX6UL_PAD_JTAG_TDO__SAI2_TX_SYNC	0x17088
-			MX6UL_PAD_JTAG_TRST_B__SAI2_TX_DATA	0x11088
-			MX6UL_PAD_JTAG_TCK__SAI2_RX_DATA	0x11088
-			MX6UL_PAD_JTAG_TMS__SAI2_MCLK		0x17088
+			MX6UL_PAD_SNVS_TAMPER2__GPIO5_IO02	0x1b0b0	/* WL_PWR (WIFI_PWR 5G) */
+			MX6UL_PAD_SNVS_TAMPER6__GPIO5_IO06	0x1b0b0	/* WL_REG_ON (WIFI_EN) */
 		>;
 	};
 
-	pinctrl_tsc: tscgrp {
-		fsl,pins = <
-			MX6UL_PAD_GPIO1_IO01__GPIO1_IO01	0xb0
-			MX6UL_PAD_GPIO1_IO02__GPIO1_IO02	0xb0
-			MX6UL_PAD_GPIO1_IO03__GPIO1_IO03	0xb0
-			MX6UL_PAD_GPIO1_IO04__GPIO1_IO04	0xb0
-		>;
-	};
-
-	pinctrl_uart2: uart2grp {
-		fsl,pins = <
-			MX6UL_PAD_UART2_TX_DATA__UART2_DCE_TX	0x1b0b1
-			MX6UL_PAD_UART2_RX_DATA__UART2_DCE_RX	0x1b0b1
-			MX6UL_PAD_UART2_CTS_B__UART2_DCE_CTS	0x1b0b1
-			MX6UL_PAD_UART2_RTS_B__UART2_DCE_RTS	0x1b0b1
-		>;
-	};
-
-	pinctrl_usdhc2: usdhc2grp {
-		fsl,pins = <
-			MX6UL_PAD_NAND_RE_B__USDHC2_CLK		0x10069
-			MX6UL_PAD_NAND_WE_B__USDHC2_CMD		0x17059
-			MX6UL_PAD_NAND_DATA00__USDHC2_DATA0	0x17059
-			MX6UL_PAD_NAND_DATA01__USDHC2_DATA1	0x17059
-			MX6UL_PAD_NAND_DATA02__USDHC2_DATA2	0x17059
-			MX6UL_PAD_NAND_DATA03__USDHC2_DATA3	0x17059
-			MX6UL_PAD_NAND_DATA04__USDHC2_DATA4	0x17059
-			MX6UL_PAD_NAND_DATA05__USDHC2_DATA5	0x17059
-			MX6UL_PAD_NAND_DATA06__USDHC2_DATA6	0x17059
-			MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x17059
-		>;
-	};
-
-	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
-		fsl,pins = <
-			MX6UL_PAD_NAND_RE_B__USDHC2_CLK		0x100b9
-			MX6UL_PAD_NAND_WE_B__USDHC2_CMD		0x170b9
-			MX6UL_PAD_NAND_DATA00__USDHC2_DATA0	0x170b9
-			MX6UL_PAD_NAND_DATA01__USDHC2_DATA1	0x170b9
-			MX6UL_PAD_NAND_DATA02__USDHC2_DATA2	0x170b9
-			MX6UL_PAD_NAND_DATA03__USDHC2_DATA3	0x170b9
-			MX6UL_PAD_NAND_DATA04__USDHC2_DATA4	0x170b9
-			MX6UL_PAD_NAND_DATA05__USDHC2_DATA5	0x170b9
-			MX6UL_PAD_NAND_DATA06__USDHC2_DATA6	0x170b9
-			MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x170b9
-		>;
-	};
-
-	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+	pinctrl_enet1_gpio: enet1-gpiogrp {
 		fsl,pins = <
-			MX6UL_PAD_NAND_RE_B__USDHC2_CLK		0x100f9
-			MX6UL_PAD_NAND_WE_B__USDHC2_CMD		0x170f9
-			MX6UL_PAD_NAND_DATA00__USDHC2_DATA0	0x170f9
-			MX6UL_PAD_NAND_DATA01__USDHC2_DATA1	0x170f9
-			MX6UL_PAD_NAND_DATA02__USDHC2_DATA2	0x170f9
-			MX6UL_PAD_NAND_DATA03__USDHC2_DATA3	0x170f9
-			MX6UL_PAD_NAND_DATA04__USDHC2_DATA4	0x170f9
-			MX6UL_PAD_NAND_DATA05__USDHC2_DATA5	0x170f9
-			MX6UL_PAD_NAND_DATA06__USDHC2_DATA6	0x170f9
-			MX6UL_PAD_NAND_DATA07__USDHC2_DATA7	0x170f9
+			MX6UL_PAD_SNVS_TAMPER0__GPIO5_IO00	0x1b0b0 /* fec1 reset */
 		>;
 	};
 };
-
-&pxp {
-	status = "okay";
-};
-
-&sai2 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_sai2>;
-	assigned-clocks = <&clks IMX6UL_CLK_SAI2_SEL>,
-			  <&clks IMX6UL_CLK_SAI2>;
-	assigned-clock-parents = <&clks IMX6UL_CLK_PLL4_AUDIO_DIV>;
-	assigned-clock-rates = <0>, <12288000>;
-	fsl,sai-mclk-direction-output;
-	status = "okay";
-};
-
-&snvs_poweroff {
-	status = "okay";
-};
-
-&tsc {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_tsc>;
-	xnur-gpios = <&gpio1 3 GPIO_ACTIVE_LOW>;
-	measure-delay-time = <0xffff>;
-	pre-charge-time = <0xfff>;
-	status = "okay";
-};
-
-&uart2 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart2>;
-	uart-has-rtscts;
-	status = "okay";
-};
-
-&usdhc2 {
-	pinctrl-names = "default", "state_100mhz", "state_200mhz";
-	pinctrl-0 = <&pinctrl_usdhc2>;
-	pinctrl-1 = <&pinctrl_usdhc2_100mhz>;
-	pinctrl-2 = <&pinctrl_usdhc2_200mhz>;
-	bus-width = <8>;
-	no-1-8-v;
-	non-removable;
-	keep-power-in-suspend;
-	wakeup-source;
-	status = "okay";
-};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ul.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ul.dtsi
index 24541fdf49ceb..d2bfa08b5e767 100644
--- a/arch/arm/boot/dts/nxp/imx/imx6ul.dtsi
+++ b/arch/arm/boot/dts/nxp/imx/imx6ul.dtsi
@@ -951,6 +951,7 @@ adc1: adc@2198000 {
 				interrupts = <GIC_SPI 100 IRQ_TYPE_LEVEL_HIGH>;
 				clocks = <&clks IMX6UL_CLK_ADC1>;
 				clock-names = "adc";
+				#io-channel-cells = <1>;
 				fsl,adck-max-frequency = <30000000>, <40000000>,
 							 <20000000>;
 				status = "disabled";
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-var-som-concerto-full.dts b/arch/arm/boot/dts/nxp/imx/imx6ull-var-som-concerto-full.dts
new file mode 100644
index 0000000000000..1b7c1a3383eec
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-var-som-concerto-full.dts
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Support for Variscite MX6 Concerto Carrier board with the VAR-SOM-6UL
+ * Variscite SoM mounted on it (6ULL CPU variant).
+ *
+ * Copyright 2026 Dimonoff
+ */
+
+/dts-v1/;
+
+#include "imx6ull-var-som.dtsi"
+#include "imx6ul-var-som-concerto-common.dtsi"
+#include "imx6ul-var-som-wifi.dtsi"
+#include "imx6ul-var-som-enet2.dtsi"
+#include "imx6ul-var-som-enet1.dtsi"
+#include "imx6ul-var-som-audio.dtsi"
+#include "imx6ul-var-som-lvds-panel.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-6UL Concerto Board (6ULL CPU)";
+	compatible = "variscite,mx6ullconcerto", "variscite,var-som-imx6ull", "fsl,imx6ull";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-var-som-concerto.dts b/arch/arm/boot/dts/nxp/imx/imx6ull-var-som-concerto.dts
new file mode 100644
index 0000000000000..9c9d16eb1a11e
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-var-som-concerto.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Support for Variscite Concerto Carrier board with the VAR-SOM-6UL
+ * Variscite SoM mounted on it (6ULL CPU variant)
+ *
+ * Copyright 2026 Dimonoff
+ */
+
+/dts-v1/;
+
+#include "imx6ull-var-som.dtsi"
+#include "imx6ul-var-som-concerto-common.dtsi"
+#include "imx6ul-var-som-sd.dtsi"
+#include "imx6ul-var-som-enet2.dtsi"
+#include "imx6ul-var-som-audio.dtsi"
+#include "imx6ul-var-som-lvds-panel.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-6UL Concerto Board (6ULL CPU)";
+	compatible = "variscite,mx6ullconcerto", "variscite,var-som-imx6ull", "fsl,imx6ull";
+};
diff --git a/arch/arm/boot/dts/nxp/imx/imx6ull-var-som.dtsi b/arch/arm/boot/dts/nxp/imx/imx6ull-var-som.dtsi
new file mode 100644
index 0000000000000..f120b1dca75ce
--- /dev/null
+++ b/arch/arm/boot/dts/nxp/imx/imx6ull-var-som.dtsi
@@ -0,0 +1,36 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Support for Variscite VAR-SOM-6UL module with imx6ull CPU
+ *
+ * Copyright 2019-2024 Variscite Ltd.
+ * Copyright 2026 Dimonoff
+ */
+
+#include "imx6ull.dtsi"
+#include "imx6ul-var-som-common.dtsi"
+
+/ {
+	model = "Variscite VAR-SOM-6UL module";
+	compatible = "variscite,var-som-imx6ull", "fsl,imx6ull";
+};
+
+&iomuxc {
+	pinctrl_brcm_bt: brcm-bt-grp {
+		fsl,pins = <
+			MX6ULL_PAD_SNVS_TAMPER4__GPIO5_IO04	0x1b0b0	/* BT_REG_ON (BT_EN) */
+		>;
+	};
+
+	pinctrl_brcm_wifi: brcm-wifi-grp {
+		fsl,pins = <
+			MX6ULL_PAD_SNVS_TAMPER2__GPIO5_IO02	0x1b0b0	/* WL_PWR (WIFI_PWR 5G) */
+			MX6ULL_PAD_SNVS_TAMPER6__GPIO5_IO06	0x1b0b0	/* WL_REG_ON (WIFI_EN) */
+		>;
+	};
+
+	pinctrl_enet1_gpio: enet1-gpiogrp {
+		fsl,pins = <
+			MX6ULL_PAD_SNVS_TAMPER0__GPIO5_IO00	0x1b0b0 /* fec1 reset */
+		>;
+	};
+};
diff --git a/arch/arm/boot/dts/renesas/r7s72100-genmai.dts b/arch/arm/boot/dts/renesas/r7s72100-genmai.dts
index 3c37565097145..da552a66615e0 100644
--- a/arch/arm/boot/dts/renesas/r7s72100-genmai.dts
+++ b/arch/arm/boot/dts/renesas/r7s72100-genmai.dts
@@ -34,9 +34,6 @@ flash@18000000 {
 		clocks = <&mstp9_clks R7S72100_CLK_SPIBSC0>;
 		power-domains = <&cpg_clocks>;
 
-		#address-cells = <1>;
-		#size-cells = <1>;
-
 		partitions {
 			compatible = "fixed-partitions";
 			#address-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r7s72100-rskrza1.dts b/arch/arm/boot/dts/renesas/r7s72100-rskrza1.dts
index 91178fb9e7210..3306bc9b7bc37 100644
--- a/arch/arm/boot/dts/renesas/r7s72100-rskrza1.dts
+++ b/arch/arm/boot/dts/renesas/r7s72100-rskrza1.dts
@@ -36,8 +36,6 @@ flash@18000000 {
 		power-domains = <&cpg_clocks>;
 		bank-width = <4>;
 		device-width = <1>;
-		#address-cells = <1>;
-		#size-cells = <1>;
 
 		partitions {
 			compatible = "fixed-partitions";
diff --git a/arch/arm/boot/dts/renesas/r7s72100.dtsi b/arch/arm/boot/dts/renesas/r7s72100.dtsi
index 245c26bb8e037..6ec57ffa72e87 100644
--- a/arch/arm/boot/dts/renesas/r7s72100.dtsi
+++ b/arch/arm/boot/dts/renesas/r7s72100.dtsi
@@ -37,7 +37,7 @@ b_clk: b {
 		clock-div = <3>;
 	};
 
-	bsc: bus {
+	bsc: bus@0 {
 		compatible = "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a73a4.dtsi b/arch/arm/boot/dts/renesas/r8a73a4.dtsi
index 2e19ebf9e2ba8..c3427dc7cf7dd 100644
--- a/arch/arm/boot/dts/renesas/r8a73a4.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a73a4.dtsi
@@ -47,9 +47,117 @@ L2_CA7: cache-controller-1 {
 		};
 	};
 
-	ptm {
-		compatible = "arm,coresight-etm3x";
+	replicator {
+		compatible = "arm,coresight-static-replicator";
+		clocks = <&cpg_clocks R8A73A4_CLK_ZTR>;
+		clock-names = "atclk";
 		power-domains = <&pd_d4>;
+
+		out-ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/* replicator output ports */
+			port@0 {
+				reg = <0>;
+
+				replicator_out_port0: endpoint {
+					remote-endpoint = <&tpiu_in_port>;
+				};
+			};
+			port@1 {
+				reg = <1>;
+
+				replicator_out_port1: endpoint {
+					remote-endpoint = <&etb_in_port>;
+				};
+			};
+		};
+
+		in-ports {
+			/* replicator input port */
+			port {
+				replicator_in_port0: endpoint {
+					remote-endpoint = <&funnel_out_port>;
+				};
+			};
+		};
+	};
+
+	etb@e6f81000 {
+		compatible = "arm,coresight-etb10", "arm,primecell";
+		reg = <0 0xe6f81000 0 0x1000>;
+		clocks = <&cpg_clocks R8A73A4_CLK_ZT>, <&cpg_clocks R8A73A4_CLK_ZTR>;
+		clock-names = "apb_pclk", "atclk";
+		power-domains = <&pd_d4>;
+
+		in-ports {
+			port {
+				etb_in_port: endpoint {
+					remote-endpoint = <&replicator_out_port1>;
+				};
+			};
+		};
+	};
+
+	tpiu@e6f83000 {
+		compatible = "arm,coresight-tpiu", "arm,primecell";
+		reg = <0 0xe6f83000 0 0x1000>;
+		clocks = <&cpg_clocks R8A73A4_CLK_ZT>, <&cpg_clocks R8A73A4_CLK_ZTR>;
+		clock-names = "apb_pclk", "atclk";
+		power-domains = <&pd_d4>;
+
+		in-ports {
+			port {
+				tpiu_in_port: endpoint {
+					remote-endpoint = <&replicator_out_port0>;
+				};
+			};
+		};
+	};
+
+	funnel {
+		compatible = "arm,coresight-static-funnel";
+
+		/* funnel output ports */
+		out-ports {
+			port {
+				funnel_out_port: endpoint {
+					remote-endpoint =
+						<&replicator_in_port0>;
+				};
+			};
+		};
+
+		in-ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/* funnel input ports */
+			port@0 {
+				reg = <0>;
+				funnel0_in_port0: endpoint {
+					remote-endpoint = <&ptm0_out_port>;
+				};
+			};
+		};
+	};
+
+	ptm@e6fbc000 {
+		compatible = "arm,coresight-etm3x", "arm,primecell";
+		reg = <0 0xe6fbc000 0 0x1000>;
+		clocks = <&cpg_clocks R8A73A4_CLK_ZT>, <&cpg_clocks R8A73A4_CLK_ZTR>;
+		clock-names = "apb_pclk", "atclk";
+		cpu = <&cpu0>;
+		power-domains = <&pd_d4>;
+
+		out-ports {
+			port {
+				ptm0_out_port: endpoint {
+					remote-endpoint = <&funnel0_in_port0>;
+				};
+			};
+		};
 	};
 
 	timer {
@@ -512,7 +620,7 @@ cpg_clocks: cpg_clocks@e6150000 {
 			clock-output-names = "main", "pll0", "pll1", "pll2",
 					     "pll2s", "pll2h", "z", "z2",
 					     "i", "m3", "b", "m1", "m2",
-					     "zx", "zs", "hp";
+					     "zx", "zs", "hp", "ztr", "zt";
 		};
 
 		/* Variable factor clocks (DIV6) */
diff --git a/arch/arm/boot/dts/renesas/r8a7740.dtsi b/arch/arm/boot/dts/renesas/r8a7740.dtsi
index d13ab86c3ab47..c7056b96ec0b7 100644
--- a/arch/arm/boot/dts/renesas/r8a7740.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7740.dtsi
@@ -18,7 +18,7 @@ / {
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
-		cpu@0 {
+		cpu0: cpu@0 {
 			compatible = "arm,cortex-a9";
 			device_type = "cpu";
 			reg = <0x0>;
@@ -59,9 +59,117 @@ pmu {
 		interrupts = <GIC_SPI 83 IRQ_TYPE_LEVEL_HIGH>;
 	};
 
-	ptm {
-		compatible = "arm,coresight-etm3x";
+	replicator {
+		compatible = "arm,coresight-static-replicator";
+		clocks = <&cpg_clocks R8A7740_CLK_ZTR>;
+		clock-names = "atclk";
 		power-domains = <&pd_d4>;
+
+		out-ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/* replicator output ports */
+			port@0 {
+				reg = <0>;
+
+				replicator_out_port0: endpoint {
+					remote-endpoint = <&tpiu_in_port>;
+				};
+			};
+			port@1 {
+				reg = <1>;
+
+				replicator_out_port1: endpoint {
+					remote-endpoint = <&etb_in_port>;
+				};
+			};
+		};
+
+		in-ports {
+			/* replicator input port */
+			port {
+				replicator_in_port0: endpoint {
+					remote-endpoint = <&funnel_out_port>;
+				};
+			};
+		};
+	};
+
+	etb@e6fa1000 {
+		compatible = "arm,coresight-etb10", "arm,primecell";
+		reg = <0xe6fa1000 0x1000>;
+		clocks = <&cpg_clocks R8A7740_CLK_ZT>, <&cpg_clocks R8A7740_CLK_ZTR>;
+		clock-names = "apb_pclk", "atclk";
+		power-domains = <&pd_d4>;
+
+		in-ports {
+			port {
+				etb_in_port: endpoint {
+					remote-endpoint = <&replicator_out_port1>;
+				};
+			};
+		};
+	};
+
+	tpiu@e6fa3000 {
+		compatible = "arm,coresight-tpiu", "arm,primecell";
+		reg = <0xe6fa3000 0x1000>;
+		clocks = <&cpg_clocks R8A7740_CLK_ZT>, <&cpg_clocks R8A7740_CLK_ZTR>;
+		clock-names = "apb_pclk", "atclk";
+		power-domains = <&pd_d4>;
+
+		in-ports {
+			port {
+				tpiu_in_port: endpoint {
+					remote-endpoint = <&replicator_out_port0>;
+				};
+			};
+		};
+	};
+
+	funnel {
+		compatible = "arm,coresight-static-funnel";
+
+		/* funnel output ports */
+		out-ports {
+			port {
+				funnel_out_port: endpoint {
+					remote-endpoint =
+						<&replicator_in_port0>;
+				};
+			};
+		};
+
+		in-ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			/* funnel input ports */
+			port@0 {
+				reg = <0>;
+				funnel0_in_port0: endpoint {
+					remote-endpoint = <&ptm0_out_port>;
+				};
+			};
+		};
+	};
+
+	ptm@e6fbc000 {
+		compatible = "arm,coresight-etm3x", "arm,primecell";
+		reg = <0xe6fbc000 0x1000>;
+		clocks = <&cpg_clocks R8A7740_CLK_ZT>, <&cpg_clocks R8A7740_CLK_ZTR>;
+		clock-names = "apb_pclk", "atclk";
+		cpu = <&cpu0>;
+		power-domains = <&pd_d4>;
+
+		out-ports {
+			port {
+				ptm0_out_port: endpoint {
+					remote-endpoint = <&funnel0_in_port0>;
+				};
+			};
+		};
 	};
 
 	ceu0: ceu@fe910000 {
@@ -553,7 +661,7 @@ cpg_clocks: cpg_clocks@e6150000 {
 					     "usb24s",
 					     "i", "zg", "b", "m1", "hp",
 					     "hpp", "usbp", "s", "zb", "m3",
-					     "cp";
+					     "cp", "ztr", "zt";
 		};
 
 		/* Variable factor clocks (DIV6) */
diff --git a/arch/arm/boot/dts/renesas/r8a7778.dtsi b/arch/arm/boot/dts/renesas/r8a7778.dtsi
index 859dd29dfce3b..7db456b19795d 100644
--- a/arch/arm/boot/dts/renesas/r8a7778.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7778.dtsi
@@ -40,7 +40,7 @@ aliases {
 		spi2 = &hspi2;
 	};
 
-	lbsc: bus {
+	lbsc: bus@0 {
 		compatible = "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7779.dtsi b/arch/arm/boot/dts/renesas/r8a7779.dtsi
index e437c22f452db..9e8a7e190c89a 100644
--- a/arch/arm/boot/dts/renesas/r8a7779.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7779.dtsi
@@ -704,7 +704,7 @@ R8A7779_CLK_MMC1 R8A7779_CLK_MMC0
 		};
 	};
 
-	lbsc: bus {
+	lbsc: bus@0 {
 		compatible = "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/renesas/r8a7792.dtsi b/arch/arm/boot/dts/renesas/r8a7792.dtsi
index 9e0de69ac3a3a..fbdbcff1cbed4 100644
--- a/arch/arm/boot/dts/renesas/r8a7792.dtsi
+++ b/arch/arm/boot/dts/renesas/r8a7792.dtsi
@@ -86,7 +86,7 @@ extal_clk: extal {
 		bootph-all;
 	};
 
-	lbsc: bus {
+	lbsc: bus@0 {
 		compatible = "simple-bus";
 		#address-cells = <1>;
 		#size-cells = <1>;
diff --git a/arch/arm/boot/dts/rockchip/rk3288-veyron-analog-audio.dtsi b/arch/arm/boot/dts/rockchip/rk3288-veyron-analog-audio.dtsi
index 51208d161d653..25c7c06678562 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-veyron-analog-audio.dtsi
+++ b/arch/arm/boot/dts/rockchip/rk3288-veyron-analog-audio.dtsi
@@ -14,8 +14,6 @@ sound {
 		rockchip,model = "VEYRON-I2S";
 		rockchip,i2s-controller = <&i2s>;
 		rockchip,audio-codec = <&max98090>;
-		rockchip,hp-det-gpios = <&gpio6 RK_PA5 GPIO_ACTIVE_HIGH>;
-		rockchip,mic-det-gpios = <&gpio6 RK_PB3 GPIO_ACTIVE_LOW>;
 		rockchip,headset-codec = <&headsetcodec>;
 		rockchip,hdmi-codec = <&hdmi>;
 	};
diff --git a/arch/arm/boot/dts/rockchip/rk3288-veyron.dtsi b/arch/arm/boot/dts/rockchip/rk3288-veyron.dtsi
index 2d6cf08d00f90..ca8e8e7350781 100644
--- a/arch/arm/boot/dts/rockchip/rk3288-veyron.dtsi
+++ b/arch/arm/boot/dts/rockchip/rk3288-veyron.dtsi
@@ -18,6 +18,11 @@ chosen {
 		stdout-path = "serial2:115200n8";
 	};
 
+	firmware {
+		#address-cells = <1>;
+		#size-cells = <1>;
+	};
+
 	/*
 	 * The default coreboot on veyron devices ignores memory@0 nodes
 	 * and would instead create another memory node.
diff --git a/arch/arm/boot/dts/st/stm32mp135f-dhcor-dhsbc.dts b/arch/arm/boot/dts/st/stm32mp135f-dhcor-dhsbc.dts
index 9902849ed0406..70d85af467353 100644
--- a/arch/arm/boot/dts/st/stm32mp135f-dhcor-dhsbc.dts
+++ b/arch/arm/boot/dts/st/stm32mp135f-dhcor-dhsbc.dts
@@ -97,6 +97,8 @@ ethphy1: ethernet-phy@1 {
 			interrupts = <12 IRQ_TYPE_LEVEL_LOW>;
 			reg = <1>;
 			realtek,clkout-disable;
+			realtek,rxc-ssc-enable;
+			realtek,sysclk-ssc-enable;
 			reset-assert-us = <15000>;
 			reset-deassert-us = <55000>;
 			reset-gpios = <&gpioa 11 GPIO_ACTIVE_LOW>;
@@ -146,6 +148,8 @@ ethphy2: ethernet-phy@1 {
 			interrupts = <15 IRQ_TYPE_LEVEL_LOW>;
 			reg = <1>;
 			realtek,clkout-disable;
+			realtek,rxc-ssc-enable;
+			realtek,sysclk-ssc-enable;
 			reset-assert-us = <15000>;
 			reset-deassert-us = <55000>;
 			reset-gpios = <&gpiog 8 GPIO_ACTIVE_LOW>;
diff --git a/arch/arm/boot/dts/st/stm32mp135f-dk.dts b/arch/arm/boot/dts/st/stm32mp135f-dk.dts
index 4d4cec8b86aca..6022e73f58afd 100644
--- a/arch/arm/boot/dts/st/stm32mp135f-dk.dts
+++ b/arch/arm/boot/dts/st/stm32mp135f-dk.dts
@@ -64,6 +64,7 @@ gpio-keys {
 		compatible = "gpio-keys";
 
 		button-user {
+			/* GPIO on PA13 "User button 2 (B2)" */
 			label = "User-PA13";
 			linux,code = <BTN_1>;
 			gpios = <&gpioa 13 (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
@@ -74,6 +75,7 @@ leds {
 		compatible = "gpio-leds";
 
 		led_blue: led-blue {
+			/* GPIO on PA14 "User LED (LD3)" */
 			function = LED_FUNCTION_HEARTBEAT;
 			color = <LED_COLOR_ID_BLUE>;
 			gpios = <&gpioa 14 GPIO_ACTIVE_LOW>;
@@ -82,6 +84,7 @@ led_blue: led-blue {
 		};
 
 		led-red {
+			/* GPIO on PA13 "User LED (LD4)" */
 			function = LED_FUNCTION_STATUS;
 			color = <LED_COLOR_ID_RED>;
 			gpios = <&gpioa 13 GPIO_ACTIVE_LOW>;
@@ -252,6 +255,7 @@ phy0_eth1: ethernet-phy@0 {
 
 &i2c1 {
 	pinctrl-names = "default", "sleep";
+	/* SDA on PE8 = CN8.27, SCL on PD12 = CN8.28 */
 	pinctrl-0 = <&i2c1_pins_a>;
 	pinctrl-1 = <&i2c1_sleep_pins_a>;
 	i2c-scl-rising-time-ns = <96>;
@@ -486,7 +490,10 @@ counter {
 		status = "okay";
 	};
 	pwm {
-		/* PWM output on pin 7 of the expansion connector (CN8.7) using TIM3_CH4 func */
+		/*
+		 * CH4 on PB1 = CN8.7;
+		 * CH3 on PB0 = CN8.10 is possible (conflicting with usart1)
+		 */
 		pinctrl-0 = <&pwm3_pins_a>;
 		pinctrl-1 = <&pwm3_sleep_pins_a>;
 		pinctrl-names = "default", "sleep";
@@ -505,7 +512,10 @@ counter {
 		status = "okay";
 	};
 	pwm {
-		/* PWM output on pin 31 of the expansion connector (CN8.31) using TIM4_CH2 func */
+		/*
+		 * CH2 on PD13 = CN8.31;
+		 * CH1 on PD12 = CN8.28 is possible (conflicting with &i2c1)
+		 */
 		pinctrl-0 = <&pwm4_pins_a>;
 		pinctrl-1 = <&pwm4_sleep_pins_a>;
 		pinctrl-names = "default", "sleep";
@@ -524,7 +534,12 @@ counter {
 		status = "okay";
 	};
 	pwm {
-		/* PWM output on pin 32 of the expansion connector (CN8.32) using TIM8_CH3 func */
+		/*
+		 * CH3 on PE5 = CN8.32
+		 * CH1N on PA7 = CN8.36 is possible (conflicting with &usart1)
+		 * CH2N on PB0 = CN8.10 is possible (conflicting with &usart1)
+		 * CH3N on PB1 = CN8.7 is possible (conflicting with &timers3)
+		 */
 		pinctrl-0 = <&pwm8_pins_a>;
 		pinctrl-1 = <&pwm8_sleep_pins_a>;
 		pinctrl-names = "default", "sleep";
@@ -541,7 +556,7 @@ counter {
 		status = "okay";
 	};
 	pwm {
-		/* PWM output on pin 33 of the expansion connector (CN8.33) using TIM14_CH1 func */
+		/* CH1 on PF9 = CN8.33 (alternatively on PA7 = CN8.36 (conflicting with &usart1)) */
 		pinctrl-0 = <&pwm14_pins_a>;
 		pinctrl-1 = <&pwm14_sleep_pins_a>;
 		pinctrl-names = "default", "sleep";
@@ -553,6 +568,7 @@ timer@13 {
 };
 
 &uart4 {
+	/* Accessible via micro USB ST-LINK USB (CN10) */
 	pinctrl-names = "default", "sleep", "idle";
 	pinctrl-0 = <&uart4_pins_a>;
 	pinctrl-1 = <&uart4_sleep_pins_a>;
@@ -564,6 +580,7 @@ &uart4 {
 
 &uart8 {
 	pinctrl-names = "default", "sleep", "idle";
+	/* TX on PE1 = CN8.37, RX on PF9 = CN8.33 */
 	pinctrl-0 = <&uart8_pins_a>;
 	pinctrl-1 = <&uart8_sleep_pins_a>;
 	pinctrl-2 = <&uart8_idle_pins_a>;
@@ -574,6 +591,7 @@ &uart8 {
 
 &usart1 {
 	pinctrl-names = "default", "sleep", "idle";
+	/* TX on PC0 = CN8.8, RX on PB0 = CN8.10, RTS on PC2 = CN8.11, CTS on PA7 = CN8.36 */
 	pinctrl-0 = <&usart1_pins_a>;
 	pinctrl-1 = <&usart1_sleep_pins_a>;
 	pinctrl-2 = <&usart1_idle_pins_a>;
@@ -584,6 +602,7 @@ &usart1 {
 /* Bluetooth */
 &usart2 {
 	pinctrl-names = "default", "sleep", "idle";
+	/* TX on PH12, RX on PD15, RTS on PD4, CTS on PE11 */
 	pinctrl-0 = <&usart2_pins_a>;
 	pinctrl-1 = <&usart2_sleep_pins_a>;
 	pinctrl-2 = <&usart2_idle_pins_a>;
@@ -613,6 +632,7 @@ hub@1 {
 };
 
 &usbotg_hs {
+	/* USB Type-C DRP (CN7) */
 	phys = <&usbphyc_port1 0>;
 	phy-names = "usb2-phy";
 	usb-role-switch;
diff --git a/arch/arm/boot/dts/st/stm32mp151c-mecio1r0.dts b/arch/arm/boot/dts/st/stm32mp151c-mecio1r0.dts
index a5ea1431c3991..862782d20d10e 100644
--- a/arch/arm/boot/dts/st/stm32mp151c-mecio1r0.dts
+++ b/arch/arm/boot/dts/st/stm32mp151c-mecio1r0.dts
@@ -36,6 +36,56 @@ led-1 {
 	};
 };
 
+&adc1 {
+	channel@0 {
+		reg = <0>;
+		st,min-sample-time-ns = <20000>;
+		label = "ain_aux0";
+	};
+
+	channel@1 {
+		reg = <1>;
+		st,min-sample-time-ns = <20000>;
+		label = "ain_aux1";
+	};
+
+	channel@3 {
+		reg = <3>;
+		st,min-sample-time-ns = <20000>;
+		label = "hpdcm1_i2";
+	};
+
+	channel@5 {
+		reg = <5>;
+		st,min-sample-time-ns = <20000>;
+		label = "pout1_i";
+	};
+
+	channel@9 {
+		reg = <9>;
+		st,min-sample-time-ns = <20000>;
+		label = "pout0_i";
+	};
+
+	channel@13 {
+		reg = <13>;
+		st,min-sample-time-ns = <200000>;
+		label = "phint1_ain";
+	};
+
+	channel@15 {
+		reg = <15>;
+		st,min-sample-time-ns = <20000>;
+		label = "hpdcm0_i1";
+	};
+
+	channel@18 {
+		reg = <18>;
+		st,min-sample-time-ns = <20000>;
+		label = "hpdcm0_i2";
+	};
+};
+
 &clk_hse {
 	clock-frequency = <25000000>;
 };
@@ -46,3 +96,81 @@ &ethernet0 {
 	assigned-clock-rates = <125000000>; /* Clock PLL3 to 625Mhz in tf-a. */
 	st,eth-clk-sel;
 };
+
+&gpio0 {
+	gpio-line-names = "HSIN0_BIAS", "HSIN1_BIAS", "HSIN2_BIAS", "HSIN3_BIAS",
+			  "HSIN4_BIAS", "", "STP_VREF0_LVL", "HSIN_VREF0_LVL",
+			  "STP0_FB_BIAS", "STP1_FB_BIAS", "STP2_FB_BIAS", "STP3_FB_BIAS",
+			  "", "", "", "";
+};
+
+&gpio1 {
+	gpio-line-names = "HSIN5_BIAS", "HSIN6_BIAS", "HSIN7_BIAS", "HSIN8_BIAS",
+			  "HSIN9_BIAS", "", "STP_VREF1_LVL", "HSIN_VREF1_LVL",
+			  "STP4_FB_BIAS", "STP5_FB_BIAS", "STP6_FB_BIAS", "",
+			  "", "", "LSIN8_BIAS", "LSIN9_BIAS";
+};
+
+&gpiod {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "STP_RESETN", "STP_ENABLEN", "HPOUT0", "HPOUT0_ALERTN";
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hog_d_mecsbc>;
+};
+
+&gpioe {
+	gpio-line-names = "HPOUT0_RESETN", "HPOUT1", "HPOUT1_ALERTN", "",
+			  "", "", "HPOUT1_RESETN",
+			  "LPOUT0", "LPOUT0_ALERTN", "LPOUT0_RESETN",
+			  "LPOUT1", "LPOUT1_ALERTN", "LPOUT1_RESETN",
+			  "LPOUT2", "LPOUT2_ALERTN", "LPOUT2_RESETN";
+};
+
+&gpiof {
+	gpio-line-names = "LPOUT3", "LPOUT3_ALERTN", "LPOUT3_RESETN",
+			  "LPOUT4", "LPOUT4_ALERTN", "LPOUT4_RESETN",
+			  "", "",
+			  "", "", "", "",
+			  "", "", "", "";
+};
+
+&gpiog {
+	gpio-line-names = "LPOUT5", "LPOUT5_ALERTN", "", "LPOUT5_RESETN",
+			  "", "", "", "",
+			  "", "", "", "",
+			  "", "", "", "";
+};
+
+&gpioh {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "GPIO0_RESETN", "", "", "",
+			  "", "", "", "";
+};
+
+&gpioi {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "HPDCM0_SLEEPN", "HPDCM1_SLEEPN", "GPIO1_RESETN", "",
+			  "", "", "", "";
+};
+
+&gpioj {
+	gpio-line-names = "HSIN10", "HSIN11", "HSIN12", "HSIN13",
+			  "HSIN14", "HSIN15", "", "",
+			  "", "", "", "",
+			  "", "RTD_RESETN", "", "";
+};
+
+&gpiok {
+	gpio-line-names = "", "", "HSIN0", "HSIN1",
+			  "HSIN2", "HSIN3", "HSIN4", "HSIN5";
+};
+
+&gpioz {
+	gpio-line-names = "", "", "", "HSIN6",
+			  "HSIN7", "HSIN8", "HSIN9", "";
+};
+
diff --git a/arch/arm/boot/dts/st/stm32mp153c-mecio1r1.dts b/arch/arm/boot/dts/st/stm32mp153c-mecio1r1.dts
index 16b814c19350c..739cc18c3d3af 100644
--- a/arch/arm/boot/dts/st/stm32mp153c-mecio1r1.dts
+++ b/arch/arm/boot/dts/st/stm32mp153c-mecio1r1.dts
@@ -36,10 +36,154 @@ led-1 {
 	};
 };
 
+&adc1 {
+	channel@0 {
+		reg = <0>;
+		st,min-sample-time-ns = <20000>;
+		label = "p24v_hpdcm";
+	};
+
+	channel@1 {
+		reg = <1>;
+		st,min-sample-time-ns = <20000>;
+		label = "p24v_stp";
+	};
+
+	channel@3 {
+		reg = <3>;
+		st,min-sample-time-ns = <200000>;
+		label = "phint1_ain";
+	};
+
+	channel@5 {
+		reg = <5>;
+		st,min-sample-time-ns = <20000>;
+		label = "hpout1_i";
+	};
+
+	channel@9 {
+		reg = <9>;
+		st,min-sample-time-ns = <20000>;
+		label = "hpout0_i";
+	};
+
+	channel@13 {
+		reg = <13>;
+		st,min-sample-time-ns = <20000>;
+		label = "hpdcm0_i2";
+	};
+
+	channel@15 {
+		reg = <15>;
+		st,min-sample-time-ns = <20000>;
+		label = "hpdcm1_i2";
+	};
+
+	channel@18 {
+		reg = <18>;
+		st,min-sample-time-ns = <20000>;
+		label = "hpdcm0_i1";
+	};
+};
+
 &clk_hse {
 	clock-frequency = <24000000>;
 };
 
+&gpio0 {
+	gpio-line-names = "HSIN0_BIAS", "HSIN1_BIAS", "HSIN2_BIAS", "HSIN3_BIAS",
+			  "", "", "HSIN_VREF0_LVL", "HSIN_VREF1_LVL",
+			  "HSIN4_BIAS", "HSIN5_BIAS", "HSIN6_BIAS", "HSIN7_BIAS",
+			  "", "", "", "";
+};
+
+&gpio1 {
+	gpio-line-names = "HSIN8_BIAS", "HSIN9_BIAS", "HSIN10_BIAS", "HSIN11_BIAS",
+			  "", "", "HSIN_VREF2_LVL", "HSIN_VREF3_LVL",
+			  "HSIN12_BIAS", "HSIN13_BIAS", "HSIN14_BIAS", "HSIN15_BIAS",
+			  "", "", "LSIN8_BIAS", "LSIN9_BIAS";
+};
+
+&gpioa {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "", "GPIO1_RESETN", "",
+			  "", "", "", "LPOUT5";
+};
+
+&gpiob {
+	gpio-line-names = "", "", "", "",
+			  "LPOUT4_RESETN", "", "", "",
+			  "", "LPOUT4_ALERTN", "", "",
+			  "", "", "", "";
+};
+
+&gpioc {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "LPOUT4", "", "",
+			  "", "", "", "";
+};
+
+&gpiod {
+	gpio-line-names = "LPOUT2", "", "LPOUT3_RESETN", "",
+			  "LPOUT2_ALERTN", "", "MECIO_ADDR0", "",
+			  "HPOUT1_ALERTN", "HPOUT1_RESETN", "", "",
+			  "", "", "HPOUT0", "HPOUT1";
+};
+
+&gpioe {
+	gpio-line-names = "LPOUT0_RESETN", "", "", "",
+			  "", "LPOUT3", "LPOUT5_ALERTN", "",
+			  "", "", "", "",
+			  "", "", "", "HSIN_RESETN";
+};
+
+&gpiof {
+	gpio-line-names = "LPOUT5_RESETN", "", "", "HPOUT0_ALERTN",
+			  "", "LPOUT1", "", "",
+			  "", "", "", "",
+			  "", "", "", "";
+};
+
+&gpiog {
+	gpio-line-names = "", "", "", "HPOUT0_RESETN",
+			  "", "", "LPOUT3_ALERTN", "",
+			  "", "", "GPIO0_RESETN", "",
+			  "", "", "", "LPOUT2_RESETN";
+};
+
+&gpioh {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "LPOUT0", "", "",
+			  "", "LPOUT0_ALERTN", "STP_ENABLEN", "STP_RESETN";
+};
+
+&gpioi {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "",
+			  "", "", "SPE_RESETN", "",
+			  "HPDCM0_SLEEPN", "", "", "";
+};
+
+&gpioj {
+	gpio-line-names = "", "", "", "",
+			  "", "", "", "MECIO_ADDR1",
+			  "", "", "", "",
+			  "", "", "", "LPOUT1_RESETN";
+};
+
+&gpiok {
+	gpio-line-names = "", "", "RTD_RESETN", "",
+			  "", "LPOUT1_ALERTN", "", "";
+};
+
+&gpioz {
+	gpio-line-names = "", "", "", "",
+			  "HPDCM1_SLEEPN", "", "", "";
+};
+
 &m_can1 {
 	pinctrl-names = "default", "sleep";
 	pinctrl-0 = <&m_can1_pins_b>;
diff --git a/arch/arm/boot/dts/st/stm32mp15x-mecio1-io.dtsi b/arch/arm/boot/dts/st/stm32mp15x-mecio1-io.dtsi
index 915ba2526f451..1a4f5a523eb33 100644
--- a/arch/arm/boot/dts/st/stm32mp15x-mecio1-io.dtsi
+++ b/arch/arm/boot/dts/st/stm32mp15x-mecio1-io.dtsi
@@ -90,94 +90,45 @@ v5v: regulator-v5v {
 };
 
 &adc {
-	/* ANA0, ANA1 are dedicated pins and don't need pinctrl: only in6. */
+	/* ANA0, ANA1 are dedicated pins and don't need pinctrl. */
 	pinctrl-0 = <&adc12_pins_mecsbc>;
 	pinctrl-names = "default";
 	vdd-supply = <&v3v3>;
 	vdda-supply = <&v3v3>;
-	vref-supply = <&v3v3>;
+	vref-supply = <&vrefbuf>;
 	status = "okay";
 };
 
 &adc1 {
 	status = "okay";
 
-	channel@0 {
-		reg = <0>;
-		/* 16.5 ck_cycles sampling time */
-		st,min-sample-time-ns = <5000>;
-		label = "p24v_stp";
-	};
-
-	channel@1 {
-		reg = <1>;
-		st,min-sample-time-ns = <5000>;
-		label = "p24v_hpdcm";
-	};
-
 	channel@2 {
 		reg = <2>;
-		st,min-sample-time-ns = <5000>;
+		st,min-sample-time-ns = <20000>;
 		label = "ain0";
 	};
 
-	channel@3 {
-		reg = <3>;
-		st,min-sample-time-ns = <5000>;
-		label = "hpdcm1_i2";
-	};
-
-	channel@5 {
-		reg = <5>;
-		st,min-sample-time-ns = <5000>;
-		label = "hpout1_i";
-	};
-
 	channel@6 {
 		reg = <6>;
-		st,min-sample-time-ns = <5000>;
+		st,min-sample-time-ns = <20000>;
 		label = "ain1";
 	};
 
-	channel@9 {
-		reg = <9>;
-		st,min-sample-time-ns = <5000>;
-		label = "hpout0_i";
-	};
-
 	channel@10 {
 		reg = <10>;
-		st,min-sample-time-ns = <5000>;
+		st,min-sample-time-ns = <200000>;
 		label = "phint0_ain";
 	};
 
-	channel@13 {
-		reg = <13>;
-		st,min-sample-time-ns = <5000>;
-		label = "phint1_ain";
-	};
-
-	channel@15 {
-		reg = <15>;
-		st,min-sample-time-ns = <5000>;
-		label = "hpdcm0_i1";
-	};
-
 	channel@16 {
 		reg = <16>;
-		st,min-sample-time-ns = <5000>;
+		st,min-sample-time-ns = <20000>;
 		label = "lsin";
 	};
 
-	channel@18 {
-		reg = <18>;
-		st,min-sample-time-ns = <5000>;
-		label = "hpdcm0_i2";
-	};
-
 	channel@19 {
 		reg = <19>;
-		st,min-sample-time-ns = <5000>;
+		st,min-sample-time-ns = <20000>;
 		label = "hpdcm1_i1";
 	};
 };
@@ -187,14 +138,13 @@ &adc2 {
 
 	channel@2 {
 		reg = <2>;
-		/* 16.5 ck_cycles sampling time */
-		st,min-sample-time-ns = <5000>;
+		st,min-sample-time-ns = <20000>;
 		label = "ain2";
 	};
 
 	channel@6 {
 		reg = <6>;
-		st,min-sample-time-ns = <5000>;
+		st,min-sample-time-ns = <20000>;
 		label = "ain3";
 	};
 };
@@ -223,69 +173,6 @@ phy0: ethernet-phy@8 {
 	};
 };
 
-&gpiod {
-	gpio-line-names = "", "", "", "",
-			  "", "", "", "",
-			  "", "", "", "",
-			  "STP_RESETN", "STP_ENABLEN", "HPOUT0", "HPOUT0_ALERTN";
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_hog_d_mecsbc>;
-};
-
-&gpioe {
-	gpio-line-names = "HPOUT0_RESETN", "HPOUT1", "HPOUT1_ALERTN", "",
-			  "", "", "HPOUT1_RESETN",
-			  "LPOUT0", "LPOUT0_ALERTN", "GPOUT0_RESETN",
-			  "LPOUT1", "LPOUT1_ALERTN", "GPOUT1_RESETN",
-			  "LPOUT2", "LPOUT2_ALERTN", "GPOUT2_RESETN";
-};
-
-&gpiof {
-	gpio-line-names = "LPOUT3", "LPOUT3_ALERTN", "GPOUT3_RESETN",
-			  "LPOUT4", "LPOUT4_ALERTN", "GPOUT4_RESETN",
-			  "", "",
-			  "", "", "", "",
-			  "", "", "", "";
-};
-
-&gpiog {
-	gpio-line-names = "LPOUT5", "LPOUT5_ALERTN", "", "LPOUT5_RESETN",
-			  "", "", "", "",
-			  "", "", "", "",
-			  "", "", "", "";
-};
-
-&gpioh {
-	gpio-line-names = "", "", "", "",
-			  "", "", "", "",
-			  "GPIO0_RESETN", "", "", "",
-			  "", "", "", "";
-};
-
-&gpioi {
-	gpio-line-names = "", "", "", "",
-			  "", "", "", "",
-			  "HPDCM0_SLEEPN", "HPDCM1_SLEEPN", "GPIO1_RESETN", "",
-			  "", "", "", "";
-};
-
-&gpioj {
-	gpio-line-names = "HSIN10", "HSIN11", "HSIN12", "HSIN13",
-			  "HSIN14", "HSIN15", "", "",
-			  "", "", "", "",
-			  "", "RTD_RESETN", "", "";
-};
-
-&gpiok {
-	gpio-line-names = "", "", "HSIN0", "HSIN1",
-			  "HSIN2", "HSIN3", "HSIN4", "HSIN5";
-};
-
-&gpioz {
-	gpio-line-names = "", "", "", "HSIN6",
-			  "HSIN7", "HSIN8", "HSIN9", "";
-};
-
 &i2c2 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c2_pins_a>;
@@ -297,10 +184,6 @@ gpio0: gpio@20 {
 		reg = <0x20>;
 		gpio-controller;
 		#gpio-cells = <2>;
-		gpio-line-names = "HSIN0_BIAS", "HSIN1_BIAS", "HSIN2_BIAS", "HSIN3_BIAS",
-				  "", "", "HSIN_VREF0_LVL", "HSIN_VREF1_LVL",
-				  "HSIN4_BIAS", "HSIN5_BIAS", "HSIN6_BIAS", "HSIN9_BIAS",
-				  "", "", "", "";
 	};
 
 	gpio1: gpio@21 {
@@ -308,10 +191,6 @@ gpio1: gpio@21 {
 		reg = <0x21>;
 		gpio-controller;
 		#gpio-cells = <2>;
-		gpio-line-names = "HSIN8_BIAS", "HSIN9_BIAS", "HSIN10_BIAS", "HSIN11_BIAS",
-				  "", "", "HSIN_VREF2_LVL", "HSIN_VREF3_LVL",
-				  "HSIN12_BIAS", "HSIN13_BIAS", "HSIN14_BIAS", "HSIN15_BIAS",
-				  "", "", "LSIN8_BIAS", "LSIN9_BIAS";
 	};
 };
 
@@ -412,6 +291,13 @@ &usbphyc_port1 {
 	phy-supply = <&v3v3>;
 };
 
+&vrefbuf {
+	regulator-min-microvolt = <2500000>;
+	regulator-max-microvolt = <2500000>;
+	vdda-supply = <&v3v3>;
+	status = "okay";
+};
+
 &pinctrl {
 	adc12_pins_mecsbc: adc12-ain-mecsbc-0 {
 		pins {
diff --git a/arch/arm/boot/dts/ti/omap/am335x-sl50.dts b/arch/arm/boot/dts/ti/omap/am335x-sl50.dts
index 1dc4e344efd63..c5259eb7d21c7 100644
--- a/arch/arm/boot/dts/ti/omap/am335x-sl50.dts
+++ b/arch/arm/boot/dts/ti/omap/am335x-sl50.dts
@@ -558,8 +558,8 @@ cpu_endpoint: endpoint {
 			remote-endpoint = <&codec_endpoint>;
 
 			dai-format = "dsp_b";
-			bitclock-master = <&codec_port>;
-			frame-master = <&codec_port>;
+			bitclock-master = <&codec_endpoint>;
+			frame-master = <&codec_endpoint>;
 			bitclock-inversion;
 			clocks = <&audio_mclk>;
 		};
diff --git a/arch/arm/boot/dts/ti/omap/dm8168-evm.dts b/arch/arm/boot/dts/ti/omap/dm8168-evm.dts
index 6130b9a5f6606..81450e15b889c 100644
--- a/arch/arm/boot/dts/ti/omap/dm8168-evm.dts
+++ b/arch/arm/boot/dts/ti/omap/dm8168-evm.dts
@@ -8,6 +8,10 @@ / {
 	model = "DM8168 EVM";
 	compatible = "ti,dm8168-evm", "ti,dm8168", "ti,dm816";
 
+	chosen {
+		stdout-path = &uart3;
+	};
+
 	memory@80000000 {
 		device_type = "memory";
 		reg = <0x80000000 0x40000000	/* 1 GB */
diff --git a/arch/arm/boot/dts/ti/omap/motorola-cpcap-mapphone.dtsi b/arch/arm/boot/dts/ti/omap/motorola-cpcap-mapphone.dtsi
index ea02fd403a9b4..73e27b95105e1 100644
--- a/arch/arm/boot/dts/ti/omap/motorola-cpcap-mapphone.dtsi
+++ b/arch/arm/boot/dts/ti/omap/motorola-cpcap-mapphone.dtsi
@@ -69,6 +69,8 @@ cpcap_regulators: regulators {
 
 		cpcap_audio: audio-codec {
 			#sound-dai-cells = <1>;
+			interrupts-extended = <&cpcap 9 0>, <&cpcap 10 0>;
+			interrupt-names = "hs", "mb2";
 
 			port@0 {
 				cpcap_audio_codec0: endpoint {
@@ -267,6 +269,8 @@ vaudio: VAUDIO {
 		regulator-min-microvolt = <2775000>;
 		regulator-max-microvolt = <2775000>;
 		regulator-enable-ramp-delay = <1000>;
-		regulator-initial-mode = <0x00>; /* NORMAL */
+		regulator-allowed-modes = <0x00>, <0x40>; /* ON, LOW_PWR */
+		regulator-initial-mode = <0x00>; /* ON */
+		regulator-always-on;
 	};
 };
diff --git a/arch/arm/boot/dts/ti/omap/omap2.dtsi b/arch/arm/boot/dts/ti/omap/omap2.dtsi
index afabb36a8ac13..fdc1790adf437 100644
--- a/arch/arm/boot/dts/ti/omap/omap2.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap2.dtsi
@@ -129,6 +129,8 @@ i2c2: i2c@48072000 {
 		};
 
 		mcspi1: spi@48098000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
 			compatible = "ti,omap2-mcspi";
 			ti,hwmods = "mcspi1";
 			reg = <0x48098000 0x100>;
@@ -140,6 +142,8 @@ mcspi1: spi@48098000 {
 		};
 
 		mcspi2: spi@4809a000 {
+			#address-cells = <1>;
+			#size-cells = <0>;
 			compatible = "ti,omap2-mcspi";
 			ti,hwmods = "mcspi2";
 			reg = <0x4809a000 0x100>;
diff --git a/arch/arm/boot/dts/ti/omap/omap2420-n8x0-common.dtsi b/arch/arm/boot/dts/ti/omap/omap2420-n8x0-common.dtsi
index 63b0b4921e4e4..fe9dd8bbfc858 100644
--- a/arch/arm/boot/dts/ti/omap/omap2420-n8x0-common.dtsi
+++ b/arch/arm/boot/dts/ti/omap/omap2420-n8x0-common.dtsi
@@ -109,3 +109,15 @@ partition@5 {
 		};
 	};
 };
+
+&mcspi2 {
+	status = "okay";
+
+	wifi@0 {
+		reg = <0>;
+		compatible = "st,stlc4560";
+		spi-max-frequency = <48000000>;
+		interrupts-extended = <&gpio3 23 IRQ_TYPE_EDGE_RISING>;
+		powerdown-gpios = <&gpio4 1 GPIO_ACTIVE_LOW>; /* gpio 97 */
+	};
+};
diff --git a/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts b/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts
index 673df1b693f28..e253e0775ea95 100644
--- a/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts
+++ b/arch/arm/boot/dts/ti/omap/omap4-epson-embt2ws.dts
@@ -351,7 +351,7 @@ mpu9150h: imu@68 {
 		pinctrl-names = "default";
 		pinctrl-0 = <&mpu9150h_pins>;
 		interrupt-parent = <&gpio2>;
-		interrupt = <19 IRQ_TYPE_LEVEL_HIGH>;
+		interrupts = <19 IRQ_TYPE_LEVEL_HIGH>;
 	};
 };
 
@@ -413,7 +413,7 @@ mpu9150: imu@68 {
 		pinctrl-names = "default";
 		pinctrl-0 = <&mpu9150_pins>;
 		interrupt-parent = <&gpio2>;
-		interrupt = <7 IRQ_TYPE_LEVEL_HIGH>;
+		interrupts = <7 IRQ_TYPE_LEVEL_HIGH>;
 		vddio-supply = <&cb_v18>;
 		vdd-supply = <&cb_v33>;
 		invensense,level-shifter;
diff --git a/arch/arm/boot/dts/zte/Makefile b/arch/arm/boot/dts/zte/Makefile
new file mode 100644
index 0000000000000..f052cfbd636cd
--- /dev/null
+++ b/arch/arm/boot/dts/zte/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0-only
+dtb-$(CONFIG_SOC_ZX297520V3) += \
+	zx297520v3-dlink-dwr932m.dtb
diff --git a/arch/arm/boot/dts/zte/zx297520v3-dlink-dwr932m.dts b/arch/arm/boot/dts/zte/zx297520v3-dlink-dwr932m.dts
new file mode 100644
index 0000000000000..1700f46aba862
--- /dev/null
+++ b/arch/arm/boot/dts/zte/zx297520v3-dlink-dwr932m.dts
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2026 Stefan Dösinger <stefandoesinger@gmail.com>
+ */
+
+/dts-v1/;
+
+#include "zx297520v3.dtsi"
+
+/ {
+	model = "D-Link DWR-932M";
+	compatible = "dlink,dwr932m", "zte,zx297520v3";
+
+	memory@20000000 {
+		device_type = "memory";
+		reg = <0x20000000 0x04000000>;
+	};
+};
+
+&uart1 {
+	status = "okay";
+};
diff --git a/arch/arm/boot/dts/zte/zx297520v3.dtsi b/arch/arm/boot/dts/zte/zx297520v3.dtsi
new file mode 100644
index 0000000000000..a16c30a164bb9
--- /dev/null
+++ b/arch/arm/boot/dts/zte/zx297520v3.dtsi
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2026 Stefan Dösinger <stefandoesinger@gmail.com>
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+
+/ {
+	#address-cells = <1>;
+	#size-cells = <1>;
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a53";
+			reg = <0>;
+		};
+	};
+
+	/* Base bus clock and default for the UART. It will be replaced once a clock driver has
+	 * been added.
+	 */
+	uartclk: uartclk-26000000 {
+		#clock-cells = <0>;
+		compatible = "fixed-clock";
+		clock-frequency = <26000000>;
+	};
+
+	timer {
+		compatible = "arm,armv7-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW>;
+		clock-frequency = <26000000>;
+		interrupt-parent = <&gic>;
+		/* I don't think uboot sets CNTVOFF and the stock kernel doesn't use the
+		 * arm timer at all. Since this is a single CPU system I don't think it
+		 * really matters that the offset is random though.
+		 */
+		arm,cpu-registers-not-fw-configured;
+	};
+
+	soc {
+		#address-cells = <1>;
+		#size-cells = <1>;
+		compatible = "simple-bus";
+		interrupt-parent = <&gic>;
+		ranges;
+
+		/* The GIC has a non-standard way of configuring ints between level-low/level
+		 * high or rising edge/falling edge at 0xf2202070 and onwards. See AP_INT_MODE_BASE
+		 * and AP_PPI_MODE_REG in the ZTE kernel, although the offsets in the kernel source
+		 * seem wrong.
+		 *
+		 * Everything defaults to active-high/rising edge, but the timer is active-low. We
+		 * currently rely on the boot loader to change timer IRQs to active-low for us for
+		 * now.
+		 */
+		gic: interrupt-controller@f2000000 {
+			compatible = "arm,gic-v3";
+			interrupt-controller;
+			#interrupt-cells = <3>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+			reg = <0xf2000000 0x10000>,
+			      <0xf2040000 0x20000>;
+		};
+
+		uart0: serial@131000 {
+			compatible = "arm,pl011", "arm,primecell";
+			arm,primecell-periphid = <0x0018c011>;
+			reg = <0x00131000 0x1000>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&uartclk>, <&uartclk>;
+			clock-names = "uartclk", "apb_pclk";
+			status = "disabled";
+		};
+
+		uart1: serial@1408000 {
+			compatible = "arm,pl011", "arm,primecell";
+			arm,primecell-periphid = <0x0018c011>;
+			reg = <0x01408000 0x1000>;
+			interrupts = <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&uartclk>, <&uartclk>;
+			clock-names = "uartclk", "apb_pclk";
+			status = "disabled";
+		};
+
+		uart2: serial@140d000 {
+			compatible = "arm,pl011", "arm,primecell";
+			arm,primecell-periphid = <0x0018c011>;
+			reg = <0x0140d000 0x1000>;
+			interrupts = <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&uartclk>, <&uartclk>;
+			clock-names = "uartclk", "apb_pclk";
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/arm/configs/at91_dt_defconfig b/arch/arm/configs/at91_dt_defconfig
index 4f1153098b16f..e331242dece71 100644
--- a/arch/arm/configs/at91_dt_defconfig
+++ b/arch/arm/configs/at91_dt_defconfig
@@ -155,7 +155,6 @@ CONFIG_LOGO=y
 CONFIG_SOUND=y
 CONFIG_SND=y
 CONFIG_SND_SOC=y
-CONFIG_SND_ATMEL_SOC=y
 CONFIG_SND_AT91_SOC_SAM9G20_WM8731=y
 CONFIG_SND_ATMEL_SOC_WM8904=y
 CONFIG_SND_AT91_SOC_SAM9X5_WM8731=y
diff --git a/arch/arm/configs/hisi_defconfig b/arch/arm/configs/hisi_defconfig
index 384aade1a48bb..dde9cff951d42 100644
--- a/arch/arm/configs/hisi_defconfig
+++ b/arch/arm/configs/hisi_defconfig
@@ -43,7 +43,6 @@ CONFIG_SERIAL_OF_PLATFORM=y
 CONFIG_SERIAL_AMBA_PL011=y
 CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 CONFIG_SPI=y
 CONFIG_SPI_PL022=y
 CONFIG_PINCTRL_SINGLE=y
diff --git a/arch/arm/configs/multi_v5_defconfig b/arch/arm/configs/multi_v5_defconfig
index 59b020e66a0b5..d0aea907124d5 100644
--- a/arch/arm/configs/multi_v5_defconfig
+++ b/arch/arm/configs/multi_v5_defconfig
@@ -186,7 +186,6 @@ CONFIG_BACKLIGHT_ATMEL_LCDC=y
 CONFIG_SOUND=y
 CONFIG_SND=y
 CONFIG_SND_SOC=y
-CONFIG_SND_ATMEL_SOC=y
 CONFIG_SND_AT91_SOC_SAM9G20_WM8731=m
 CONFIG_SND_ATMEL_SOC_WM8904=m
 CONFIG_SND_AT91_SOC_SAM9X5_WM8731=m
diff --git a/arch/arm/configs/multi_v7_defconfig b/arch/arm/configs/multi_v7_defconfig
index bcc9aabc12028..2891eeba90324 100644
--- a/arch/arm/configs/multi_v7_defconfig
+++ b/arch/arm/configs/multi_v7_defconfig
@@ -360,9 +360,9 @@ CONFIG_SERIAL_8250_CONSOLE=y
 CONFIG_SERIAL_8250_NR_UARTS=5
 CONFIG_SERIAL_8250_RUNTIME_UARTS=5
 CONFIG_SERIAL_8250_EXTENDED=y
+CONFIG_SERIAL_8250_SHARE_IRQ=y
 CONFIG_SERIAL_8250_MANY_PORTS=y
 CONFIG_SERIAL_8250_ASPEED_VUART=m
-CONFIG_SERIAL_8250_SHARE_IRQ=y
 CONFIG_SERIAL_8250_BCM2835AUX=y
 CONFIG_SERIAL_8250_DW=y
 CONFIG_SERIAL_8250_EM=y
@@ -387,7 +387,6 @@ CONFIG_SERIAL_MSM=y
 CONFIG_SERIAL_MSM_CONSOLE=y
 CONFIG_SERIAL_VT8500=y
 CONFIG_SERIAL_VT8500_CONSOLE=y
-CONFIG_SERIAL_BCM63XX=y
 CONFIG_SERIAL_BCM63XX_CONSOLE=y
 CONFIG_SERIAL_XILINX_PS_UART=y
 CONFIG_SERIAL_XILINX_PS_UART_CONSOLE=y
@@ -420,7 +419,6 @@ CONFIG_I2C_BCM2835=y
 CONFIG_I2C_CADENCE=y
 CONFIG_I2C_DAVINCI=y
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 CONFIG_I2C_DIGICOLOR=m
 CONFIG_I2C_EMEV2=m
 CONFIG_I2C_IMX=y
@@ -483,14 +481,6 @@ CONFIG_PINCTRL_PALMAS=y
 CONFIG_PINCTRL_STMFX=y
 CONFIG_PINCTRL_OWL=y
 CONFIG_PINCTRL_S500=y
-CONFIG_PINCTRL_MSM=y
-CONFIG_PINCTRL_APQ8064=y
-CONFIG_PINCTRL_APQ8084=y
-CONFIG_PINCTRL_IPQ8064=y
-CONFIG_PINCTRL_MSM8660=y
-CONFIG_PINCTRL_MSM8960=y
-CONFIG_PINCTRL_MSM8X74=y
-CONFIG_PINCTRL_MSM8916=y
 CONFIG_PINCTRL_QCOM_SPMI_PMIC=y
 CONFIG_PINCTRL_QCOM_SSBI_PMIC=y
 CONFIG_PINCTRL_RZA2=y
@@ -731,40 +721,8 @@ CONFIG_VIDEO_ADV7604_CEC=y
 CONFIG_VIDEO_ML86V7667=m
 CONFIG_IMX_IPUV3_CORE=m
 CONFIG_DRM=y
-CONFIG_DRM_NOUVEAU=m
-# CONFIG_DRM_NOUVEAU_CH7006 is not set
-# CONFIG_DRM_NOUVEAU_SIL164 is not set
-CONFIG_DRM_EXYNOS=m
-CONFIG_DRM_EXYNOS_FIMD=y
-CONFIG_DRM_EXYNOS_MIXER=y
-CONFIG_DRM_EXYNOS_DPI=y
-CONFIG_DRM_EXYNOS_DSI=y
-CONFIG_DRM_EXYNOS_HDMI=y
-CONFIG_DRM_ROCKCHIP=m
-CONFIG_ROCKCHIP_ANALOGIX_DP=y
-CONFIG_ROCKCHIP_DW_HDMI=y
-CONFIG_ROCKCHIP_DW_MIPI_DSI=y
-CONFIG_ROCKCHIP_INNO_HDMI=y
+CONFIG_DRM_ASPEED_GFX=m
 CONFIG_DRM_ATMEL_HLCDC=m
-CONFIG_DRM_RCAR_DU=m
-CONFIG_DRM_SUN4I=m
-CONFIG_DRM_OMAP=m
-CONFIG_OMAP5_DSS_HDMI=y
-CONFIG_DRM_TILCDC=m
-CONFIG_DRM_MSM=m
-CONFIG_DRM_FSL_DCU=m
-CONFIG_DRM_TEGRA=y
-CONFIG_DRM_STM=m
-CONFIG_DRM_STM_DSI=m
-CONFIG_DRM_PANEL_LVDS=m
-CONFIG_DRM_PANEL_ORISETECH_OTM8009A=m
-CONFIG_DRM_PANEL_RAYDIUM_RM68200=m
-CONFIG_DRM_PANEL_SAMSUNG_LD9040=m
-CONFIG_DRM_PANEL_SAMSUNG_S6E63J0X03=m
-CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=m
-CONFIG_DRM_PANEL_SHARP_LQ101R1SX01=m
-CONFIG_DRM_PANEL_EDP=y
-CONFIG_DRM_PANEL_SIMPLE=y
 CONFIG_DRM_DISPLAY_CONNECTOR=m
 CONFIG_DRM_I2C_NXP_TDA998X=m
 CONFIG_DRM_ITE_IT66121=m
@@ -780,20 +738,52 @@ CONFIG_DRM_TI_TFP410=m
 CONFIG_DRM_TI_TPD12S015=m
 CONFIG_DRM_I2C_ADV7511=m
 CONFIG_DRM_I2C_ADV7511_AUDIO=y
-CONFIG_DRM_STI=m
+CONFIG_DRM_ETNAVIV=m
+CONFIG_DRM_EXYNOS=m
+CONFIG_DRM_EXYNOS_FIMD=y
+CONFIG_DRM_EXYNOS_MIXER=y
+CONFIG_DRM_EXYNOS_DPI=y
+CONFIG_DRM_EXYNOS_DSI=y
+CONFIG_DRM_EXYNOS_HDMI=y
+CONFIG_DRM_FSL_DCU=m
 CONFIG_DRM_IMX=m
 CONFIG_DRM_IMX_PARALLEL_DISPLAY=m
 CONFIG_DRM_IMX_TVE=m
 CONFIG_DRM_IMX_LDB=m
 CONFIG_DRM_IMX_HDMI=m
-CONFIG_DRM_V3D=m
-CONFIG_DRM_VC4=m
-CONFIG_DRM_ETNAVIV=m
-CONFIG_DRM_MXSFB=m
-CONFIG_DRM_PL111=m
 CONFIG_DRM_LIMA=m
+CONFIG_DRM_MSM=m
+CONFIG_DRM_MXSFB=m
+CONFIG_DRM_NOUVEAU=m
+# CONFIG_DRM_NOUVEAU_CH7006 is not set
+# CONFIG_DRM_NOUVEAU_SIL164 is not set
+CONFIG_DRM_OMAP=m
+CONFIG_OMAP5_DSS_HDMI=y
+CONFIG_DRM_PANEL_LVDS=m
+CONFIG_DRM_PANEL_ORISETECH_OTM8009A=m
+CONFIG_DRM_PANEL_RAYDIUM_RM68200=m
+CONFIG_DRM_PANEL_SAMSUNG_LD9040=m
+CONFIG_DRM_PANEL_SAMSUNG_S6E63J0X03=m
+CONFIG_DRM_PANEL_SAMSUNG_S6E8AA0=m
+CONFIG_DRM_PANEL_SHARP_LQ101R1SX01=m
+CONFIG_DRM_PANEL_EDP=y
+CONFIG_DRM_PANEL_SIMPLE=y
 CONFIG_DRM_PANFROST=m
-CONFIG_DRM_ASPEED_GFX=m
+CONFIG_DRM_PL111=m
+CONFIG_DRM_RCAR_DU=m
+CONFIG_DRM_ROCKCHIP=m
+CONFIG_ROCKCHIP_ANALOGIX_DP=y
+CONFIG_ROCKCHIP_DW_HDMI=y
+CONFIG_ROCKCHIP_DW_MIPI_DSI=y
+CONFIG_ROCKCHIP_INNO_HDMI=y
+CONFIG_DRM_STI=m
+CONFIG_DRM_STM=m
+CONFIG_DRM_STM_DSI=m
+CONFIG_DRM_SUN4I=m
+CONFIG_DRM_TEGRA=y
+CONFIG_DRM_TILCDC=m
+CONFIG_DRM_V3D=m
+CONFIG_DRM_VC4=m
 CONFIG_FB=y
 CONFIG_FB_EFI=y
 CONFIG_FB_WM8505=y
@@ -811,27 +801,22 @@ CONFIG_SND_HDA_INPUT_BEEP=y
 CONFIG_SND_HDA_PATCH_LOADER=y
 CONFIG_SND_HDA_TEGRA=m
 CONFIG_SND_HDA_CODEC_REALTEK=m
-CONFIG_SND_HDA_CODEC_REALTEK_LIB=m
-CONFIG_SND_HDA_CODEC_ALC269=m
 CONFIG_SND_HDA_CODEC_HDMI=m
 CONFIG_SND_HDA_CODEC_HDMI_GENERIC=m
 CONFIG_SND_HDA_CODEC_HDMI_NVIDIA=m
 CONFIG_SND_HDA_CODEC_HDMI_TEGRA=m
 CONFIG_SND_USB_AUDIO=m
 CONFIG_SND_SOC=m
-CONFIG_SND_ATMEL_SOC=m
 CONFIG_SND_ATMEL_SOC_WM8904=m
 CONFIG_SND_ATMEL_SOC_PDMIC=m
 CONFIG_SND_ATMEL_SOC_I2S=m
 CONFIG_SND_BCM2835_SOC_I2S=m
 CONFIG_SND_IMX_SOC=m
 CONFIG_SND_SOC_FSL_ASOC_CARD=m
-CONFIG_SND_PXA_SOC_SSP=m
 CONFIG_SND_MMP_SOC_SSPA=m
 CONFIG_SND_PXA910_SOC=m
 CONFIG_SND_SOC_QCOM=m
 CONFIG_SND_SOC_APQ8016_SBC=m
-CONFIG_SND_SOC_ROCKCHIP=m
 CONFIG_SND_SOC_SH4_FSI=m
 CONFIG_SND_SOC_RCAR=m
 CONFIG_SND_SOC_ROCKCHIP_SPDIF=m
@@ -873,7 +858,6 @@ CONFIG_SND_SOC_MSM8916_WCD_ANALOG=m
 CONFIG_SND_SOC_MSM8916_WCD_DIGITAL=m
 CONFIG_SND_SOC_SGTL5000=m
 CONFIG_SND_SOC_STI_SAS=m
-CONFIG_SND_SOC_TLV320AIC32X4=m
 CONFIG_SND_SOC_TLV320AIC32X4_I2C=m
 CONFIG_SND_SOC_TLV320AIC3X_I2C=m
 CONFIG_SND_SOC_WM8960=m
@@ -933,7 +917,6 @@ CONFIG_KEYSTONE_USB_PHY=m
 CONFIG_NOP_USB_XCEIV=y
 CONFIG_AM335X_PHY_USB=m
 CONFIG_TWL6030_USB=m
-CONFIG_USB_GPIO_VBUS=y
 CONFIG_USB_ISP1301=y
 CONFIG_USB_MXS_PHY=y
 CONFIG_USB_GADGET=y
@@ -1155,7 +1138,7 @@ CONFIG_QCOM_COMMAND_DB=m
 CONFIG_QCOM_GSBI=y
 CONFIG_QCOM_OCMEM=m
 CONFIG_QCOM_RMTFS_MEM=m
-CONFIG_QCOM_RPMH=y
+CONFIG_QCOM_RPMH=m
 CONFIG_QCOM_SMEM=y
 CONFIG_QCOM_SMD_RPM=y
 CONFIG_QCOM_SMP2P=y
@@ -1170,7 +1153,7 @@ CONFIG_KEYSTONE_NAVIGATOR_DMA=y
 CONFIG_TI_PRUSS=m
 CONFIG_RASPBERRYPI_POWER=y
 CONFIG_QCOM_CPR=y
-CONFIG_QCOM_RPMHPD=y
+CONFIG_QCOM_RPMHPD=m
 CONFIG_QCOM_RPMPD=y
 CONFIG_ROCKCHIP_PM_DOMAINS=y
 CONFIG_TI_SCI_PM_DOMAINS=y
diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig
index c51ae373ca888..89187768b1329 100644
--- a/arch/arm/configs/pxa_defconfig
+++ b/arch/arm/configs/pxa_defconfig
@@ -77,9 +77,6 @@ CONFIG_BT_HCIUART_BCSP=y
 CONFIG_BT_HCIBCM203X=m
 CONFIG_BT_HCIBPA10X=m
 CONFIG_BT_HCIBFUSB=m
-CONFIG_BT_HCIDTL1=m
-CONFIG_BT_HCIBT3C=m
-CONFIG_BT_HCIBLUECARD=m
 CONFIG_BT_HCIBTUART=m
 CONFIG_BT_HCIVHCI=m
 CONFIG_BT_MRVL=m
@@ -278,7 +275,6 @@ CONFIG_I2C_CHARDEV=m
 CONFIG_I2C_MUX_PCA954x=m
 CONFIG_I2C_MUX_PINCTRL=m
 CONFIG_I2C_DESIGNWARE_CORE=m
-CONFIG_I2C_DESIGNWARE_PLATFORM=m
 CONFIG_I2C_GPIO=y
 CONFIG_I2C_PXA_SLAVE=y
 CONFIG_I2C_XILINX=m
@@ -404,7 +400,6 @@ CONFIG_SND_DEBUG=y
 CONFIG_SND_SEQUENCER=m
 CONFIG_SND_USB_AUDIO=m
 CONFIG_SND_SOC=m
-CONFIG_SND_ATMEL_SOC=m
 CONFIG_SND_PXA2XX_SOC=m
 CONFIG_SND_PXA_SOC_SSP=m
 CONFIG_SND_PXA2XX_SOC_SPITZ=m
diff --git a/arch/arm/configs/qcom_defconfig b/arch/arm/configs/qcom_defconfig
index 29a1dea500f08..03309b89ea4cf 100644
--- a/arch/arm/configs/qcom_defconfig
+++ b/arch/arm/configs/qcom_defconfig
@@ -123,22 +123,7 @@ CONFIG_I2C_QUP=y
 CONFIG_SPI=y
 CONFIG_SPI_QUP=y
 CONFIG_SPMI=y
-CONFIG_PINCTRL_MSM=y
-CONFIG_PINCTRL_APQ8064=y
-CONFIG_PINCTRL_APQ8084=y
-CONFIG_PINCTRL_IPQ4019=y
-CONFIG_PINCTRL_IPQ8064=y
-CONFIG_PINCTRL_MSM8226=y
-CONFIG_PINCTRL_MSM8660=y
-CONFIG_PINCTRL_MSM8960=y
-CONFIG_PINCTRL_MDM9607=y
-CONFIG_PINCTRL_MDM9615=y
-CONFIG_PINCTRL_MSM8X74=y
-CONFIG_PINCTRL_MSM8909=y
-CONFIG_PINCTRL_MSM8916=y
 CONFIG_GPIOLIB=y
-CONFIG_PINCTRL_SDX55=y
-CONFIG_PINCTRL_SDX65=y
 CONFIG_PINCTRL_QCOM_SPMI_PMIC=y
 CONFIG_PINCTRL_QCOM_SSBI_PMIC=y
 CONFIG_GPIO_SYSFS=y
diff --git a/arch/arm/configs/sama5_defconfig b/arch/arm/configs/sama5_defconfig
index 2cad045e1d8d7..bd7f0b5f7d660 100644
--- a/arch/arm/configs/sama5_defconfig
+++ b/arch/arm/configs/sama5_defconfig
@@ -168,7 +168,6 @@ CONFIG_FRAMEBUFFER_CONSOLE=y
 CONFIG_SOUND=y
 CONFIG_SND=y
 CONFIG_SND_SOC=y
-CONFIG_SND_ATMEL_SOC=y
 CONFIG_SND_ATMEL_SOC_WM8904=y
 CONFIG_SND_ATMEL_SOC_CLASSD=y
 CONFIG_SND_ATMEL_SOC_PDMIC=y
diff --git a/arch/arm/configs/sama7_defconfig b/arch/arm/configs/sama7_defconfig
index e52f671ccec4e..47abc56c40f59 100644
--- a/arch/arm/configs/sama7_defconfig
+++ b/arch/arm/configs/sama7_defconfig
@@ -159,7 +159,6 @@ CONFIG_BACKLIGHT_PWM=y
 CONFIG_SOUND=y
 CONFIG_SND=y
 CONFIG_SND_SOC=y
-CONFIG_SND_ATMEL_SOC=y
 CONFIG_SND_SOC_MIKROE_PROTO=m
 CONFIG_SND_MCHP_SOC_I2S_MCC=y
 CONFIG_SND_MCHP_SOC_SPDIFTX=y
diff --git a/arch/arm/configs/socfpga_defconfig b/arch/arm/configs/socfpga_defconfig
index f2e42846b1169..19a8e30a7b226 100644
--- a/arch/arm/configs/socfpga_defconfig
+++ b/arch/arm/configs/socfpga_defconfig
@@ -84,7 +84,6 @@ CONFIG_SERIAL_8250_DW=y
 CONFIG_I2C=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 CONFIG_SPI=y
 CONFIG_SPI_CADENCE_QUADSPI=y
 CONFIG_SPI_DESIGNWARE=y
diff --git a/arch/arm/configs/spear13xx_defconfig b/arch/arm/configs/spear13xx_defconfig
index 8b19af1ea67c7..6712ae741c196 100644
--- a/arch/arm/configs/spear13xx_defconfig
+++ b/arch/arm/configs/spear13xx_defconfig
@@ -63,7 +63,6 @@ CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
 CONFIG_RAW_DRIVER=y
 CONFIG_I2C=y
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 CONFIG_SPI=y
 CONFIG_SPI_PL022=y
 CONFIG_GPIO_SYSFS=y
diff --git a/arch/arm/configs/spear3xx_defconfig b/arch/arm/configs/spear3xx_defconfig
index b4e4b96a98afa..b96baa63fd582 100644
--- a/arch/arm/configs/spear3xx_defconfig
+++ b/arch/arm/configs/spear3xx_defconfig
@@ -43,7 +43,6 @@ CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
 CONFIG_RAW_DRIVER=y
 CONFIG_I2C=y
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 CONFIG_SPI=y
 CONFIG_SPI_PL022=y
 CONFIG_GPIO_SYSFS=y
diff --git a/arch/arm/configs/spear6xx_defconfig b/arch/arm/configs/spear6xx_defconfig
index 7083b1bd85739..dba3bf3fa19aa 100644
--- a/arch/arm/configs/spear6xx_defconfig
+++ b/arch/arm/configs/spear6xx_defconfig
@@ -34,7 +34,6 @@ CONFIG_SERIAL_AMBA_PL011=y
 CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
 CONFIG_I2C=y
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 CONFIG_SPI=y
 CONFIG_SPI_PL022=y
 CONFIG_GPIO_PL061=y
diff --git a/arch/arm/configs/spitz_defconfig b/arch/arm/configs/spitz_defconfig
index f116a01c3f5f1..8f8a058294fb8 100644
--- a/arch/arm/configs/spitz_defconfig
+++ b/arch/arm/configs/spitz_defconfig
@@ -62,9 +62,6 @@ CONFIG_BT_HCIUART_BCSP=y
 CONFIG_BT_HCIBCM203X=m
 CONFIG_BT_HCIBPA10X=m
 CONFIG_BT_HCIBFUSB=m
-CONFIG_BT_HCIDTL1=m
-CONFIG_BT_HCIBT3C=m
-CONFIG_BT_HCIBLUECARD=m
 CONFIG_BT_HCIBTUART=m
 CONFIG_BT_HCIVHCI=m
 CONFIG_PCCARD=y
diff --git a/arch/arm/include/asm/arch_gicv3.h b/arch/arm/include/asm/arch_gicv3.h
index 311e83038bdb3..847590df75511 100644
--- a/arch/arm/include/asm/arch_gicv3.h
+++ b/arch/arm/include/asm/arch_gicv3.h
@@ -7,7 +7,7 @@
 #ifndef __ASM_ARCH_GICV3_H
 #define __ASM_ARCH_GICV3_H
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/io.h>
 #include <linux/io-64-nonatomic-lo-hi.h>
@@ -257,5 +257,5 @@ static inline bool gic_has_relaxed_pmr_sync(void)
 	return false;
 }
 
-#endif /* !__ASSEMBLY__ */
+#endif /* !__ASSEMBLER__ */
 #endif /* !__ASM_ARCH_GICV3_H */
diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h
index d6ae80b5df36f..1593cf3b98008 100644
--- a/arch/arm/include/asm/uaccess.h
+++ b/arch/arm/include/asm/uaccess.h
@@ -616,8 +616,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 }
 #define __clear_user(addr, n)		(memset((void __force *)addr, 0, n), 0)
 #endif
-#define INLINE_COPY_TO_USER
-#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_USER
 
 static inline unsigned long __must_check clear_user(void __user *to, unsigned long n)
 {
diff --git a/arch/arm/mach-highbank/highbank.c b/arch/arm/mach-highbank/highbank.c
index 47335c7dadf8d..8b7d0929dac47 100644
--- a/arch/arm/mach-highbank/highbank.c
+++ b/arch/arm/mach-highbank/highbank.c
@@ -98,7 +98,7 @@ static int highbank_platform_notifier(struct notifier_block *nb,
 	if (of_property_read_bool(dev->of_node, "dma-coherent")) {
 		val = readl(sregs_base + reg);
 		writel(val | 0xff01, sregs_base + reg);
-		dev->dma_coherent = true;
+		dev_set_dma_coherent(dev);
 	}
 
 	return NOTIFY_OK;
diff --git a/arch/arm/mach-mv78xx0/Kconfig b/arch/arm/mach-mv78xx0/Kconfig
index 9de3bbc09c3a0..670e6587827e4 100644
--- a/arch/arm/mach-mv78xx0/Kconfig
+++ b/arch/arm/mach-mv78xx0/Kconfig
@@ -6,6 +6,7 @@ menuconfig ARCH_MV78XX0
 	depends on ATAGS
 	select CPU_FEROCEON
 	select GPIOLIB
+	select GPIOLIB_LEGACY
 	select MVEBU_MBUS
 	select FORCE_PCI
 	select PLAT_ORION_LEGACY
diff --git a/arch/arm/mach-mvebu/coherency.c b/arch/arm/mach-mvebu/coherency.c
index fa2c1e1aeb96c..7234d487ff395 100644
--- a/arch/arm/mach-mvebu/coherency.c
+++ b/arch/arm/mach-mvebu/coherency.c
@@ -95,7 +95,7 @@ static int mvebu_hwcc_notifier(struct notifier_block *nb,
 
 	if (event != BUS_NOTIFY_ADD_DEVICE)
 		return NOTIFY_DONE;
-	dev->dma_coherent = true;
+	dev_set_dma_coherent(dev);
 
 	return NOTIFY_OK;
 }
diff --git a/arch/arm/mach-omap2/board-n8x0.c b/arch/arm/mach-omap2/board-n8x0.c
index 969265d5d5c6e..d9acd32c5457c 100644
--- a/arch/arm/mach-omap2/board-n8x0.c
+++ b/arch/arm/mach-omap2/board-n8x0.c
@@ -20,7 +20,6 @@
 #include <linux/spi/spi.h>
 #include <linux/usb/musb.h>
 #include <linux/mmc/host.h>
-#include <linux/platform_data/spi-omap2-mcspi.h>
 #include <linux/platform_data/mmc-omap.h>
 #include <linux/mfd/menelaus.h>
 
@@ -106,21 +105,6 @@ static void __init n8x0_usb_init(void) {}
 
 #endif /*CONFIG_USB_MUSB_TUSB6010 */
 
-
-static struct omap2_mcspi_device_config p54spi_mcspi_config = {
-	.turbo_mode	= 0,
-};
-
-static struct spi_board_info n800_spi_board_info[] __initdata = {
-	{
-		.modalias	= "p54spi",
-		.bus_num	= 2,
-		.chip_select	= 0,
-		.max_speed_hz   = 48000000,
-		.controller_data = &p54spi_mcspi_config,
-	},
-};
-
 #if defined(CONFIG_MENELAUS) && IS_ENABLED(CONFIG_MMC_OMAP)
 
 /*
@@ -524,7 +508,5 @@ omap_late_initcall(n8x0_late_initcall);
 void * __init n8x0_legacy_init(void)
 {
 	board_check_revision();
-	spi_register_board_info(n800_spi_board_info,
-				ARRAY_SIZE(n800_spi_board_info));
 	return &mmc1_data;
 }
diff --git a/arch/arm/mach-omap2/omap_device.c b/arch/arm/mach-omap2/omap_device.c
index 79db4c49ffc9c..77a75b0b9ae69 100644
--- a/arch/arm/mach-omap2/omap_device.c
+++ b/arch/arm/mach-omap2/omap_device.c
@@ -307,35 +307,27 @@ static struct omap_device *omap_device_alloc(struct platform_device *pdev,
 	int ret = -ENOMEM;
 	struct omap_device *od;
 	int i;
-	struct omap_hwmod **hwmods;
+	struct omap_hwmod *hwmod;
 
-	od = kzalloc_obj(struct omap_device);
-	if (!od)
-		goto oda_exit1;
+	od = kzalloc_flex(*od, hwmods, oh_cnt);
+	if (!od) {
+		dev_err(&pdev->dev, "omap_device: build failed (%d)\n", ret);
+		return ERR_PTR(ret);
+	}
 
 	od->hwmods_cnt = oh_cnt;
+	memcpy(od->hwmods, ohs, oh_cnt * sizeof(*od->hwmods));
 
-	hwmods = kmemdup_array(ohs, oh_cnt, sizeof(*hwmods), GFP_KERNEL);
-	if (!hwmods)
-		goto oda_exit2;
-
-	od->hwmods = hwmods;
 	od->pdev = pdev;
 	pdev->archdata.od = od;
 
 	for (i = 0; i < oh_cnt; i++) {
-		hwmods[i]->od = od;
-		_add_hwmod_clocks_clkdev(od, hwmods[i]);
+		hwmod = od->hwmods[i];
+		hwmod->od = od;
+		_add_hwmod_clocks_clkdev(od, hwmod);
 	}
 
 	return od;
-
-oda_exit2:
-	kfree(od);
-oda_exit1:
-	dev_err(&pdev->dev, "omap_device: build failed (%d)\n", ret);
-
-	return ERR_PTR(ret);
 }
 
 static void omap_device_delete(struct omap_device *od)
@@ -344,7 +336,6 @@ static void omap_device_delete(struct omap_device *od)
 		return;
 
 	od->pdev->archdata.od = NULL;
-	kfree(od->hwmods);
 	kfree(od);
 }
 
diff --git a/arch/arm/mach-omap2/omap_device.h b/arch/arm/mach-omap2/omap_device.h
index aa8096ecb23ca..fae09cfce137a 100644
--- a/arch/arm/mach-omap2/omap_device.h
+++ b/arch/arm/mach-omap2/omap_device.h
@@ -37,11 +37,11 @@
 /**
  * struct omap_device - omap_device wrapper for platform_devices
  * @pdev: platform_device
- * @hwmods: (one .. many per omap_device)
  * @hwmods_cnt: ARRAY_SIZE() of @hwmods
  * @_state: one of OMAP_DEVICE_STATE_* (see above)
  * @flags: device flags
  * @_driver_status: one of BUS_NOTIFY_*_DRIVER from <linux/device.h>
+ * @hwmods: (one .. many per omap_device)
  *
  * Integrates omap_hwmod data into Linux platform_device.
  *
@@ -51,11 +51,11 @@
  */
 struct omap_device {
 	struct platform_device		*pdev;
-	struct omap_hwmod		**hwmods;
 	unsigned long			_driver_status;
 	u8				hwmods_cnt;
 	u8				_state;
 	u8                              flags;
+	struct omap_hwmod		*hwmods[] __counted_by(hwmods_cnt);
 };
 
 /* Device driver interface (call via platform_data fn ptrs) */
diff --git a/arch/arm/mach-orion5x/Kconfig b/arch/arm/mach-orion5x/Kconfig
index ee449ca032d21..cef19bea61641 100644
--- a/arch/arm/mach-orion5x/Kconfig
+++ b/arch/arm/mach-orion5x/Kconfig
@@ -5,6 +5,7 @@ menuconfig ARCH_ORION5X
 	depends on CPU_LITTLE_ENDIAN
 	select CPU_FEROCEON
 	select GPIOLIB
+	select GPIOLIB_LEGACY
 	select MVEBU_MBUS
 	select FORCE_PCI
 	select PCI_QUIRKS
diff --git a/arch/arm/mach-pxa/Kconfig b/arch/arm/mach-pxa/Kconfig
index 10e472f4fa434..66e26990e2c8d 100644
--- a/arch/arm/mach-pxa/Kconfig
+++ b/arch/arm/mach-pxa/Kconfig
@@ -10,6 +10,7 @@ menuconfig ARCH_PXA
 	select CPU_XSCALE if !CPU_XSC3
 	select GPIO_PXA
 	select GPIOLIB
+	select GPIOLIB_LEGACY
 	select PLAT_PXA
 	help
 	  Support for Intel/Marvell's PXA2xx/PXA3xx processor line.
diff --git a/arch/arm/mach-pxa/devices.c b/arch/arm/mach-pxa/devices.c
index 7695cfce01a16..edad956a1483c 100644
--- a/arch/arm/mach-pxa/devices.c
+++ b/arch/arm/mach-pxa/devices.c
@@ -11,7 +11,6 @@
 #include <linux/platform_data/i2c-pxa.h>
 #include <linux/soc/pxa/cpu.h>
 
-#include "udc.h"
 #include <linux/platform_data/video-pxafb.h>
 #include <linux/platform_data/mmc-pxamci.h>
 #include "irqs.h"
@@ -83,10 +82,6 @@ void __init pxa_set_mci_info(const struct pxamci_platform_data *info,
 		pr_err("Unable to create mci device: %d\n", err);
 }
 
-static struct pxa2xx_udc_mach_info pxa_udc_info = {
-	.gpio_pullup = -1,
-};
-
 static struct resource pxa2xx_udc_resources[] = {
 	[0] = {
 		.start	= 0x40600000,
@@ -108,7 +103,6 @@ struct platform_device pxa25x_device_udc = {
 	.resource	= pxa2xx_udc_resources,
 	.num_resources	= ARRAY_SIZE(pxa2xx_udc_resources),
 	.dev		=  {
-		.platform_data	= &pxa_udc_info,
 		.dma_mask	= &udc_dma_mask,
 	}
 };
@@ -119,7 +113,6 @@ struct platform_device pxa27x_device_udc = {
 	.resource	= pxa2xx_udc_resources,
 	.num_resources	= ARRAY_SIZE(pxa2xx_udc_resources),
 	.dev		=  {
-		.platform_data	= &pxa_udc_info,
 		.dma_mask	= &udc_dma_mask,
 	}
 };
diff --git a/arch/arm/mach-pxa/gumstix.c b/arch/arm/mach-pxa/gumstix.c
index 1713bdf3b71e4..6074815a4bcae 100644
--- a/arch/arm/mach-pxa/gumstix.c
+++ b/arch/arm/mach-pxa/gumstix.c
@@ -39,7 +39,6 @@
 
 #include "pxa25x.h"
 #include <linux/platform_data/mmc-pxamci.h>
-#include "udc.h"
 #include "gumstix.h"
 #include "devices.h"
 
diff --git a/arch/arm/mach-pxa/pxa25x.c b/arch/arm/mach-pxa/pxa25x.c
index 70509a5998142..a4e878be004a0 100644
--- a/arch/arm/mach-pxa/pxa25x.c
+++ b/arch/arm/mach-pxa/pxa25x.c
@@ -240,6 +240,9 @@ static int __init pxa25x_init(void)
 
 		if (!of_have_populated_dt()) {
 			software_node_register(&pxa2xx_gpiochip_node);
+			pxa25x_device_gpio.dev.fwnode = software_node_fwnode(
+								&pxa2xx_gpiochip_node);
+
 			pxa2xx_set_dmac_info(&pxa25x_dma_pdata);
 			ret = platform_add_devices(pxa25x_devices,
 						   ARRAY_SIZE(pxa25x_devices));
diff --git a/arch/arm/mach-pxa/pxa27x.c b/arch/arm/mach-pxa/pxa27x.c
index ff63619790383..1d34de7e5fbef 100644
--- a/arch/arm/mach-pxa/pxa27x.c
+++ b/arch/arm/mach-pxa/pxa27x.c
@@ -58,8 +58,15 @@ static unsigned long ac97_reset_config[] = {
 	GPIO95_AC97_nRESET,
 };
 
-void pxa27x_configure_ac97reset(int reset_gpio, bool to_gpio)
+void pxa27x_configure_ac97reset(struct gpio_desc *gpiod, bool to_gpio)
 {
+	int reset_gpio;
+
+	if (!gpiod)
+		return;
+
+	reset_gpio = desc_to_gpio(gpiod);
+
 	/*
 	 * This helper function is used to work around a bug in the pxa27x's
 	 * ac97 controller during a warm reset.  The configuration of the
@@ -342,6 +349,9 @@ static int __init pxa27x_init(void)
 
 		if (!of_have_populated_dt()) {
 			software_node_register(&pxa2xx_gpiochip_node);
+			pxa27x_device_gpio.dev.fwnode = software_node_fwnode(
+								&pxa2xx_gpiochip_node);
+
 			pxa2xx_set_dmac_info(&pxa27x_dma_pdata);
 			ret = platform_add_devices(devices,
 						   ARRAY_SIZE(devices));
diff --git a/arch/arm/mach-pxa/spitz.c b/arch/arm/mach-pxa/spitz.c
index c0b1f7e6be874..5091b601c4e1b 100644
--- a/arch/arm/mach-pxa/spitz.c
+++ b/arch/arm/mach-pxa/spitz.c
@@ -165,7 +165,7 @@ static struct scoop_config spitz_scoop_1_setup = {
 	.gpio_base	= SPITZ_SCP_GPIO_BASE,
 };
 
-struct platform_device spitz_scoop_1_device = {
+static struct platform_device spitz_scoop_1_device = {
 	.name		= "sharp-scoop",
 	.id		= 0,
 	.dev		= {
@@ -192,7 +192,7 @@ static struct scoop_config spitz_scoop_2_setup = {
 	.gpio_base	= SPITZ_SCP2_GPIO_BASE,
 };
 
-struct platform_device spitz_scoop_2_device = {
+static struct platform_device spitz_scoop_2_device = {
 	.name		= "sharp-scoop",
 	.id		= 1,
 	.dev		= {
@@ -204,11 +204,15 @@ struct platform_device spitz_scoop_2_device = {
 
 static void __init spitz_scoop_init(void)
 {
+	spitz_scoop_1_device.dev.fwnode = software_node_fwnode(&spitz_scoop_1_gpiochip_node);
 	platform_device_register(&spitz_scoop_1_device);
 
 	/* Akita doesn't have the second SCOOP chip */
-	if (!machine_is_akita())
+	if (!machine_is_akita()) {
+		spitz_scoop_2_device.dev.fwnode = software_node_fwnode(
+							&spitz_scoop_2_gpiochip_node);
 		platform_device_register(&spitz_scoop_2_device);
+	}
 }
 
 /* Power control is shared with between one of the CF slots and SD */
@@ -988,6 +992,7 @@ static struct i2c_board_info spitz_i2c_devs[] = {
 		.type		= "max7310",
 		.addr		= 0x18,
 		.platform_data	= &akita_pca953x_pdata,
+		.swnode		= &akita_max7310_gpiochip_node,
 	},
 };
 
diff --git a/arch/arm/mach-pxa/udc.h b/arch/arm/mach-pxa/udc.h
deleted file mode 100644
index 9a827e32db98b..0000000000000
--- a/arch/arm/mach-pxa/udc.h
+++ /dev/null
@@ -1,8 +0,0 @@
-/*
- * arch/arm/mach-pxa/include/mach/udc.h
- *
- */
-#include <linux/platform_data/pxa2xx_udc.h>
-
-extern void pxa_set_udc_info(struct pxa2xx_udc_mach_info *info);
-
diff --git a/arch/arm/mach-s3c/Kconfig.s3c64xx b/arch/arm/mach-s3c/Kconfig.s3c64xx
index 8f40af063ad6f..3f97fba8e4f52 100644
--- a/arch/arm/mach-s3c/Kconfig.s3c64xx
+++ b/arch/arm/mach-s3c/Kconfig.s3c64xx
@@ -101,6 +101,7 @@ config MACH_WLF_CRAGG_6410
 	depends on ATAGS
 	depends on I2C=y
 	select CPU_S3C6410
+	select GPIOLIB_LEGACY
 	select LEDS_GPIO_REGISTER
 	select S3C64XX_DEV_SPI0
 	select S3C64XX_SETUP_FB_24BPP
diff --git a/arch/arm/mach-s3c/mach-crag6410.c b/arch/arm/mach-s3c/mach-crag6410.c
index 028169c7debf3..c4040aad1ed30 100644
--- a/arch/arm/mach-s3c/mach-crag6410.c
+++ b/arch/arm/mach-s3c/mach-crag6410.c
@@ -779,46 +779,53 @@ static struct s3c_sdhci_platdata crag6410_hsmmc0_pdata = {
 static const struct gpio_led gpio_leds[] = {
 	{
 		.name = "d13:green:",
-		.gpio = MMGPIO_GPIO_BASE + 0,
 		.default_state = LEDS_GPIO_DEFSTATE_ON,
 	},
 	{
 		.name = "d14:green:",
-		.gpio = MMGPIO_GPIO_BASE + 1,
 		.default_state = LEDS_GPIO_DEFSTATE_ON,
 	},
 	{
 		.name = "d15:green:",
-		.gpio = MMGPIO_GPIO_BASE + 2,
 		.default_state = LEDS_GPIO_DEFSTATE_ON,
 	},
 	{
 		.name = "d16:green:",
-		.gpio = MMGPIO_GPIO_BASE + 3,
 		.default_state = LEDS_GPIO_DEFSTATE_ON,
 	},
 	{
 		.name = "d17:green:",
-		.gpio = MMGPIO_GPIO_BASE + 4,
 		.default_state = LEDS_GPIO_DEFSTATE_ON,
 	},
 	{
 		.name = "d18:green:",
-		.gpio = MMGPIO_GPIO_BASE + 5,
 		.default_state = LEDS_GPIO_DEFSTATE_ON,
 	},
 	{
 		.name = "d19:green:",
-		.gpio = MMGPIO_GPIO_BASE + 6,
 		.default_state = LEDS_GPIO_DEFSTATE_ON,
 	},
 	{
 		.name = "d20:green:",
-		.gpio = MMGPIO_GPIO_BASE + 7,
 		.default_state = LEDS_GPIO_DEFSTATE_ON,
 	},
 };
 
+static struct gpiod_lookup_table crag_leds_table = {
+	.dev_id = "leds-gpio",
+	.table = {
+		GPIO_LOOKUP_IDX("basic-mmio-gpio", 0, "cs", 0, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("basic-mmio-gpio", 1, "cs", 1, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("basic-mmio-gpio", 2, "cs", 2, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("basic-mmio-gpio", 3, "cs", 3, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("basic-mmio-gpio", 4, "cs", 4, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("basic-mmio-gpio", 5, "cs", 5, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("basic-mmio-gpio", 6, "cs", 6, GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP_IDX("basic-mmio-gpio", 7, "cs", 7, GPIO_ACTIVE_LOW),
+		{ },
+	},
+};
+
 static const struct gpio_led_platform_data gpio_leds_pdata = {
 	.leds = gpio_leds,
 	.num_leds = ARRAY_SIZE(gpio_leds),
@@ -875,6 +882,7 @@ static void __init crag6410_machine_init(void)
 	platform_add_devices(crag6410_devices, ARRAY_SIZE(crag6410_devices));
 	platform_device_register_full(&crag6410_mmgpio_devinfo);
 
+	gpiod_add_lookup_table(&crag_leds_table);
 	gpio_led_register_device(-1, &gpio_leds_pdata);
 
 	regulator_has_full_constraints();
diff --git a/arch/arm/mach-sa1100/Kconfig b/arch/arm/mach-sa1100/Kconfig
index 0fb4c24cfad54..e23700e0d6c88 100644
--- a/arch/arm/mach-sa1100/Kconfig
+++ b/arch/arm/mach-sa1100/Kconfig
@@ -13,6 +13,7 @@ menuconfig ARCH_SA1100
 	select CPU_FREQ
 	select CPU_SA1100
 	select GPIOLIB
+	select GPIOLIB_LEGACY
 	select IRQ_DOMAIN
 	select ISA
 	select NEED_MACH_MEMORY_H
diff --git a/arch/arm/mach-versatile/integrator_cp.c b/arch/arm/mach-versatile/integrator_cp.c
index 2ed4ded56b3fe..03dfb5f720b7b 100644
--- a/arch/arm/mach-versatile/integrator_cp.c
+++ b/arch/arm/mach-versatile/integrator_cp.c
@@ -86,14 +86,6 @@ static u64 notrace intcp_read_sched_clock(void)
 	return val;
 }
 
-static void __init intcp_init_early(void)
-{
-	cm_map = syscon_regmap_lookup_by_compatible("arm,core-module-integrator");
-	if (IS_ERR(cm_map))
-		return;
-	sched_clock_register(intcp_read_sched_clock, 32, 24000000);
-}
-
 static void __init intcp_init_irq_of(void)
 {
 	cm_init();
@@ -119,6 +111,10 @@ static void __init intcp_init_of(void)
 {
 	struct device_node *cpcon;
 
+	cm_map = syscon_regmap_lookup_by_compatible("arm,core-module-integrator");
+	if (!IS_ERR(cm_map))
+		sched_clock_register(intcp_read_sched_clock, 32, 24000000);
+
 	cpcon = of_find_matching_node(NULL, intcp_syscon_match);
 	if (!cpcon)
 		return;
@@ -138,7 +134,6 @@ static const char * intcp_dt_board_compat[] = {
 DT_MACHINE_START(INTEGRATOR_CP_DT, "ARM Integrator/CP (Device Tree)")
 	.reserve	= integrator_reserve,
 	.map_io		= intcp_map_io,
-	.init_early	= intcp_init_early,
 	.init_irq	= intcp_init_irq_of,
 	.init_machine	= intcp_init_of,
 	.dt_compat      = intcp_dt_board_compat,
diff --git a/arch/arm/mach-zte/Kconfig b/arch/arm/mach-zte/Kconfig
new file mode 100644
index 0000000000000..d3b404ca488d4
--- /dev/null
+++ b/arch/arm/mach-zte/Kconfig
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: GPL-2.0-only
+menuconfig ARCH_ZTE
+	bool "ZTE zx family"
+	depends on ARCH_MULTI_V7
+	help
+	  Support for ZTE zx-based family of processors.
+
+if ARCH_ZTE
+
+config SOC_ZX297520V3
+	bool "zx297520v3 SoC"
+	default y
+	select ARM_GIC_V3
+	# This board does not have PSCI firmware, but ARM_GIC_V3 depends on
+	# ARM_PSCI_FW being enabled.
+	select ARM_PSCI_FW
+	select ARM_AMBA
+	select HAVE_ARM_ARCH_TIMER
+	select PM_GENERIC_DOMAINS if PM
+	help
+	  Support for ZTE zx297520v3 SoC. It is a single core SoC used in cheap
+	  LTE to WiFi routers. These devices can be identified by the occurrence
+	  of the string "zx297520v3" in the boot output and /proc/cpuinfo of
+	  their stock firmware.
+
+	  Please read Documentation/arch/arm/zte/zx297520v3.rst on how to boot
+	  the kernel.
+
+endif
diff --git a/arch/arm/mach-zte/Makefile b/arch/arm/mach-zte/Makefile
new file mode 100644
index 0000000000000..1bfe4fddd6af7
--- /dev/null
+++ b/arch/arm/mach-zte/Makefile
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_SOC_ZX297520V3) += zx297520v3.o
diff --git a/arch/arm/mach-zte/zx297520v3.c b/arch/arm/mach-zte/zx297520v3.c
new file mode 100644
index 0000000000000..06f71348459e2
--- /dev/null
+++ b/arch/arm/mach-zte/zx297520v3.c
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2026 Stefan Dösinger
+ */
+
+#include <asm/mach/arch.h>
+#include <linux/init.h>
+
+static const char *const zx297520v3_dt_compat[] __initconst = {
+	"zte,zx297520v3",
+	NULL,
+};
+
+DT_MACHINE_START(ZX, "ZTE zx297520v3 (Device Tree)")
+	.dt_compat	= zx297520v3_dt_compat,
+MACHINE_END
diff --git a/arch/arm/mm/dma-mapping-nommu.c b/arch/arm/mm/dma-mapping-nommu.c
index fecac107fd0d1..c6a70686507b3 100644
--- a/arch/arm/mm/dma-mapping-nommu.c
+++ b/arch/arm/mm/dma-mapping-nommu.c
@@ -42,11 +42,11 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
 		 * enough to check if MPU is in use or not since in absence of
 		 * MPU system memory map is used.
 		 */
-		dev->dma_coherent = cacheid ? coherent : true;
+		dev_assign_dma_coherent(dev, cacheid ? coherent : true);
 	} else {
 		/*
 		 * Assume coherent DMA in case MMU/MPU has not been set up.
 		 */
-		dev->dma_coherent = (get_cr() & CR_M) ? coherent : true;
+		dev_assign_dma_coherent(dev, (get_cr() & CR_M) ? coherent : true);
 	}
 }
diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c
index f304037d1c34c..f9bc53b60f997 100644
--- a/arch/arm/mm/dma-mapping.c
+++ b/arch/arm/mm/dma-mapping.c
@@ -1076,7 +1076,7 @@ static void *arm_iommu_alloc_attrs(struct device *dev, size_t size,
 	pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);
 	struct page **pages;
 	void *addr = NULL;
-	int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL;
+	int coherent_flag = dev_dma_coherent(dev) ? COHERENT : NORMAL;
 
 	*handle = DMA_MAPPING_ERROR;
 	size = PAGE_ALIGN(size);
@@ -1124,7 +1124,7 @@ static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
 	if (vma->vm_pgoff >= nr_pages)
 		return -ENXIO;
 
-	if (!dev->dma_coherent)
+	if (!dev_dma_coherent(dev))
 		vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
 
 	err = vm_map_pages(vma, pages, nr_pages);
@@ -1141,7 +1141,7 @@ static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
 static void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr,
 	dma_addr_t handle, unsigned long attrs)
 {
-	int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL;
+	int coherent_flag = dev_dma_coherent(dev) ? COHERENT : NORMAL;
 	struct page **pages;
 	size = PAGE_ALIGN(size);
 
@@ -1202,7 +1202,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg,
 		phys_addr_t phys = page_to_phys(sg_page(s));
 		unsigned int len = PAGE_ALIGN(s->offset + s->length);
 
-		if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		if (!dev_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 			arch_sync_dma_for_device(sg_phys(s), s->length, dir);
 
 		prot = __dma_info_to_prot(dir, attrs);
@@ -1304,7 +1304,7 @@ static void arm_iommu_unmap_sg(struct device *dev,
 		if (sg_dma_len(s))
 			__iommu_remove_mapping(dev, sg_dma_address(s),
 					       sg_dma_len(s));
-		if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
+		if (!dev_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 			arch_sync_dma_for_cpu(sg_phys(s), s->length, dir);
 	}
 }
@@ -1323,7 +1323,7 @@ static void arm_iommu_sync_sg_for_cpu(struct device *dev,
 	struct scatterlist *s;
 	int i;
 
-	if (dev->dma_coherent)
+	if (dev_dma_coherent(dev))
 		return;
 
 	for_each_sg(sg, s, nents, i)
@@ -1345,7 +1345,7 @@ static void arm_iommu_sync_sg_for_device(struct device *dev,
 	struct scatterlist *s;
 	int i;
 
-	if (dev->dma_coherent)
+	if (dev_dma_coherent(dev))
 		return;
 
 	for_each_sg(sg, s, nents, i)
@@ -1371,7 +1371,7 @@ static dma_addr_t arm_iommu_map_phys(struct device *dev, phys_addr_t phys,
 	dma_addr_t dma_addr;
 	int ret, prot;
 
-	if (!dev->dma_coherent &&
+	if (!dev_dma_coherent(dev) &&
 	    !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO)))
 		arch_sync_dma_for_device(phys, size, dir);
 
@@ -1412,7 +1412,7 @@ static void arm_iommu_unmap_phys(struct device *dev, dma_addr_t handle,
 	if (!iova)
 		return;
 
-	if (!dev->dma_coherent &&
+	if (!dev_dma_coherent(dev) &&
 	    !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) {
 		phys_addr_t phys = iommu_iova_to_phys(mapping->domain, iova);
 
@@ -1431,7 +1431,7 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev,
 	unsigned int offset = handle & ~PAGE_MASK;
 	phys_addr_t phys;
 
-	if (dev->dma_coherent || !iova)
+	if (dev_dma_coherent(dev) || !iova)
 		return;
 
 	phys = iommu_iova_to_phys(mapping->domain, iova);
@@ -1446,7 +1446,7 @@ static void arm_iommu_sync_single_for_device(struct device *dev,
 	unsigned int offset = handle & ~PAGE_MASK;
 	phys_addr_t phys;
 
-	if (dev->dma_coherent || !iova)
+	if (dev_dma_coherent(dev) || !iova)
 		return;
 
 	phys = iommu_iova_to_phys(mapping->domain, iova);
@@ -1701,13 +1701,13 @@ static void arm_teardown_iommu_dma_ops(struct device *dev) { }
 void arch_setup_dma_ops(struct device *dev, bool coherent)
 {
 	/*
-	 * Due to legacy code that sets the ->dma_coherent flag from a bus
-	 * notifier we can't just assign coherent to the ->dma_coherent flag
+	 * Due to legacy code that sets the dma_coherent flag from a bus
+	 * notifier we can't just assign coherent to the dma_coherent flag
 	 * here, but instead have to make sure we only set but never clear it
 	 * for now.
 	 */
 	if (coherent)
-		dev->dma_coherent = true;
+		dev_set_dma_coherent(dev);
 
 	/*
 	 * Don't override the dma_ops if they have already been set. Ideally
diff --git a/arch/arm64/Kconfig.platforms b/arch/arm64/Kconfig.platforms
index 72c812e76b0b1..dc995a7321174 100644
--- a/arch/arm64/Kconfig.platforms
+++ b/arch/arm64/Kconfig.platforms
@@ -243,7 +243,6 @@ config ARCH_MVEBU
 	select MVEBU_ODMI
 	select MVEBU_PIC
 	select MVEBU_SEI
-	select OF_GPIO
 	select PINCTRL
 	select PINCTRL_ARMADA_37XX
 	select PINCTRL_ARMADA_AP806
diff --git a/arch/arm64/boot/dts/allwinner/sun50i-h616.dtsi b/arch/arm64/boot/dts/allwinner/sun50i-h616.dtsi
index bf054869e78be..1598e86259ab5 100644
--- a/arch/arm64/boot/dts/allwinner/sun50i-h616.dtsi
+++ b/arch/arm64/boot/dts/allwinner/sun50i-h616.dtsi
@@ -182,12 +182,38 @@ syscon: syscon@3000000 {
 			#size-cells = <1>;
 			ranges;
 
-			sram_c: sram@28000 {
+			/* SRAM C */
+			sram@28000 {
 				compatible = "mmio-sram";
 				reg = <0x00028000 0x30000>;
 				#address-cells = <1>;
 				#size-cells = <1>;
 				ranges = <0 0x00028000 0x30000>;
+
+				/*
+				 * 0x0 ~ 0x20000 is partial alias of VE SRAM below.
+				 * 0x20000 ~ 0x2ffff is (partial?) alias of DE SRAM.
+				 * However the whole region is toggled together.
+				 */
+				sram_c: sram-section@0 {
+					compatible = "allwinner,sun50i-h616-sram-c",
+						     "allwinner,sun50i-a64-sram-c";
+					reg = <0x00000 0x30000>;
+				};
+			};
+
+			/* VE SRAM */
+			sram@1a00000 {
+				compatible = "mmio-sram";
+				reg = <0x01a00000 0x200000>;
+				#address-cells = <1>;
+				#size-cells = <1>;
+				ranges = <0 0x01a00000 0x200000>;
+
+				ve_sram: sram-section@0 {
+					compatible = "allwinner,sun50i-h616-ve-sram";
+					reg = <0x000000 0x200000>;
+				};
 			};
 		};
 
@@ -237,6 +263,16 @@ timer0: timer@3009000 {
 			clocks = <&osc24M>;
 		};
 
+		hstimer@3005000 {
+			compatible = "allwinner,sun50i-h616-hstimer",
+				     "allwinner,sun20i-d1-hstimer";
+			reg = <0x03005000 0x1000>;
+			interrupts = <GIC_SPI 45 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 46 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_HSTIMER>;
+			resets = <&ccu RST_BUS_HSTIMER>;
+		};
+
 		watchdog: watchdog@30090a0 {
 			compatible = "allwinner,sun50i-h616-wdt",
 				     "allwinner,sun6i-a31-wdt";
diff --git a/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-khadas-vim4.dts b/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-khadas-vim4.dts
index f4c953034be39..c41525a34b721 100644
--- a/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-khadas-vim4.dts
+++ b/arch/arm64/boot/dts/amlogic/amlogic-t7-a311d2-khadas-vim4.dts
@@ -6,6 +6,8 @@
 /dts-v1/;
 
 #include "amlogic-t7.dtsi"
+#include <dt-bindings/gpio/amlogic,t7-periphs-pinctrl.h>
+#include <dt-bindings/gpio/gpio.h>
 
 / {
 	model = "Khadas VIM4";
@@ -13,6 +15,9 @@ / {
 
 	aliases {
 		serial0 = &uart_a;
+		mmc0 = &sd_emmc_c;
+		mmc1 = &sd_emmc_b;
+		mmc2 = &sd_emmc_a;
 	};
 
 	memory@0 {
@@ -40,13 +45,223 @@ secmon_reserved_bl32: secmon@5300000 {
 		};
 	};
 
+	dc_in: regulator-dc-in {
+		compatible = "regulator-fixed";
+		regulator-name = "DC_IN";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+	};
+
+	pwm-leds {
+		compatible = "pwm-leds";
+
+		status {
+			linux,default-trigger="heartbeat";
+			max-brightness = <255>;
+			pwms = <&pwm_ao_cd 0 30040 0>;
+		};
+	};
+
+	sd_3v3: regulator-sdcard-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "SD_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vddao_3v3>;
+		gpio = <&gpio GPIOD_11 GPIO_ACTIVE_LOW>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	sdio_pwrseq: sdio-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		reset-gpios = <&gpio GPIOX_6 GPIO_ACTIVE_LOW>;
+		post-power-on-delay-ms = <500>;
+		power-off-delay-us = <200000>;
+		clocks = <&wifi32k>;
+		clock-names = "ext_clock";
+	};
+
+	vcc5v: regulator-vcc-5v {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC5V";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&dc_in>;
+
+		gpio = <&gpio GPIOH_4 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vcc5v0_usb: regulator-vcc-usb {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC5V0_USB";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc5v>;
+
+		gpio = <&gpio GPIOY_5 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	vddao_1v8: regulator-vddao-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDAO_1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vddao_3v3>;
+		regulator-always-on;
+	};
+
+	vddao_3v3: regulator-vddao-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDAO_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&dc_in>;
+		regulator-always-on;
+	};
+
+	vddio_1v8: regulator-vddio-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vddio_3v3>;
+		regulator-always-on;
+	};
+
+	vddio_3v3: regulator-vddio-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDDIO_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vddao_3v3>;
+		regulator-always-on;
+	};
+
+	vddio_c: regulator-gpio-c {
+		compatible = "regulator-gpio";
+		regulator-name = "VDDIO_C";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vddio_3v3>;
+		gpios = <&gpio GPIOD_9 GPIO_ACTIVE_HIGH>;
+		states = <1800000 1
+			  3300000 0>;
+	};
+
+	wifi32k: wifi32k {
+		compatible = "pwm-clock";
+		#clock-cells = <0>;
+		clock-frequency = <32768>;
+			pwms = <&pwm_ab 0 30518 0>;
+	};
+
 	xtal: xtal-clk {
 		compatible = "fixed-clock";
 		clock-frequency = <24000000>;
 		clock-output-names = "xtal";
 		#clock-cells = <0>;
 	};
+};
+
+&pwm_ab {
+	status = "okay";
+	pinctrl-0 = <&pwm_a_pins>;
+	pinctrl-names = "default";
+};
+
+&pwm_ao_cd {
+	status = "okay";
+	pinctrl-0 = <&pwm_ao_c_d_pins>;
+	pinctrl-names = "default";
+};
+
+/* SDIO */
+&sd_emmc_a {
+	status = "okay";
+	pinctrl-0 = <&sdio_pins>;
+	pinctrl-1 = <&sdio_clk_gate_pins>;
+	pinctrl-names = "default", "clk-gate";
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	sd-uhs-sdr12;
+	sd-uhs-sdr25;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
+	cap-sdio-irq;
+	max-frequency = <200000000>;
+	non-removable;
+	disable-wp;
+	no-mmc;
+	no-sd;
+
+	power-domains = <&pwrc PWRC_T7_SDIO_A_ID>;
+
+	keep-power-in-suspend;
+
+	mmc-pwrseq = <&sdio_pwrseq>;
+
+	vmmc-supply = <&vddao_3v3>;
+	vqmmc-supply = <&vddao_1v8>;
+
+	brcmf: wifi@1 {
+		reg = <1>;
+		compatible = "brcm,bcm43752-fmac", "brcm,bcm4329-fmac";
+	};
+};
+
+/* SD card */
+&sd_emmc_b {
+	status = "okay";
+	pinctrl-0 = <&sdcard_pins>;
+	pinctrl-1 = <&sdcard_clk_gate_pins>;
+	pinctrl-names = "default", "clk-gate";
+
+	bus-width = <4>;
+	cap-sd-highspeed;
+	sd-uhs-sdr12;
+	sd-uhs-sdr25;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
+	max-frequency = <200000000>;
+	disable-wp;
+	no-sdio;
+	no-mmc;
+
+	power-domains = <&pwrc PWRC_T7_SDIO_B_ID>;
+
+	cd-gpios = <&gpio GPIOC_6 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&sd_3v3>;
+	vqmmc-supply = <&vddio_c>;
+};
+
+/* eMMC */
+&sd_emmc_c {
+	status = "okay";
+	pinctrl-0 = <&emmc_ctrl_pins>, <&emmc_data_8b_pins>, <&emmc_ds_pins>;
+	pinctrl-1 = <&emmc_clk_gate_pins>;
+	pinctrl-names = "default", "clk-gate";
+
+	bus-width = <8>;
+	cap-mmc-highspeed;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+	max-frequency = <200000000>;
+	disable-wp;
+	non-removable;
+	no-sdio;
+	no-sd;
+
+	power-domains = <&pwrc PWRC_T7_EMMC_ID>;
 
+	vmmc-supply = <&vddio_3v3>;
+	vqmmc-supply = <&vddio_1v8>;
 };
 
 &uart_a {
diff --git a/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi b/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi
index d523cbc0ed22a..cc371fcd18967 100644
--- a/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi
+++ b/arch/arm64/boot/dts/amlogic/amlogic-t7.dtsi
@@ -6,6 +6,9 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/power/amlogic,t7-pwrc.h>
 #include "amlogic-t7-reset.h"
+#include <dt-bindings/clock/amlogic,t7-scmi.h>
+#include <dt-bindings/clock/amlogic,t7-pll-clkc.h>
+#include <dt-bindings/clock/amlogic,t7-peripherals-clkc.h>
 
 / {
 	interrupt-parent = <&gic>;
@@ -201,6 +204,34 @@ pwrc: power-controller {
 		};
 	};
 
+	sram@f7042000 {
+		compatible = "mmio-sram";
+		reg = <0x0 0xf7042000 0x0 0x100>;
+		#address-cells = <1>;
+		#size-cells = <1>;
+		ranges = <0 0x0 0xf7042000 0x100>;
+
+		scmi_shmem: sram@0 {
+			compatible = "arm,scmi-shmem";
+			reg = <0x0 0x100>;
+		};
+	};
+
+	firmware {
+		scmi: scmi {
+			compatible = "arm,scmi-smc";
+			arm,smc-id = <0x820000c1>;
+			shmem = <&scmi_shmem>;
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			scmi_clk: protocol@14 {
+				reg = <0x14>;
+				#clock-cells = <1>;
+			};
+		};
+	};
+
 	soc {
 		compatible = "simple-bus";
 		#address-cells = <2>;
@@ -226,6 +257,42 @@ apb4: bus@fe000000 {
 			#size-cells = <2>;
 			ranges = <0x0 0x0 0x0 0xfe000000 0x0 0x480000>;
 
+			clkc_periphs: clock-controller@0 {
+				compatible = "amlogic,t7-peripherals-clkc";
+				reg = <0x0 0x0 0x0 0x1c8>;
+				#clock-cells = <1>;
+				clocks = <&xtal>,
+					 <&scmi_clk CLKID_SYS_CLK>,
+					 <&scmi_clk CLKID_FIXED_PLL>,
+					 <&scmi_clk CLKID_FCLK_DIV2>,
+					 <&scmi_clk CLKID_FCLK_DIV2P5>,
+					 <&scmi_clk CLKID_FCLK_DIV3>,
+					 <&scmi_clk CLKID_FCLK_DIV4>,
+					 <&scmi_clk CLKID_FCLK_DIV5>,
+					 <&scmi_clk CLKID_FCLK_DIV7>,
+					 <&hifi CLKID_HIFI_PLL>,
+					 <&gp0 CLKID_GP0_PLL>,
+					 <&gp1 CLKID_GP1_PLL>,
+					 <&mpll CLKID_MPLL1>,
+					 <&mpll CLKID_MPLL2>,
+					 <&mpll CLKID_MPLL3>;
+				clock-names = "xtal",
+					      "sys",
+					      "fix",
+					      "fdiv2",
+					      "fdiv2p5",
+					      "fdiv3",
+					      "fdiv4",
+					      "fdiv5",
+					      "fdiv7",
+					      "hifi",
+					      "gp0",
+					      "gp1",
+					      "mpll1",
+					      "mpll2",
+					      "mpll3";
+			};
+
 			reset: reset-controller@2000 {
 				compatible = "amlogic,t7-reset";
 				reg = <0x0 0x2000 0x0 0x98>;
@@ -252,6 +319,258 @@ gpio: bank@4000 {
 					#gpio-cells = <2>;
 					gpio-ranges = <&periphs_pinctrl 0 0 157>;
 				};
+
+				emmc_ctrl_pins: emmc-ctrl {
+					mux-0 {
+						groups = "emmc_cmd";
+						function = "emmc";
+						bias-pull-up;
+					};
+
+					mux-1 {
+						groups = "emmc_clk";
+						function = "emmc";
+						bias-disable;
+					};
+				};
+
+				emmc_data_4b_pins: emmc-data-4b {
+					mux {
+						groups = "emmc_nand_d0",
+							 "emmc_nand_d1",
+							 "emmc_nand_d2",
+							 "emmc_nand_d3";
+						function = "emmc";
+						bias-pull-up;
+					};
+				};
+
+				emmc_data_8b_pins: emmc-data-8b {
+					mux {
+						groups = "emmc_nand_d0",
+							 "emmc_nand_d1",
+							 "emmc_nand_d2",
+							 "emmc_nand_d3",
+							 "emmc_nand_d4",
+							 "emmc_nand_d5",
+							 "emmc_nand_d6",
+							 "emmc_nand_d7";
+						function = "emmc";
+						bias-pull-up;
+					};
+				};
+
+				emmc_ds_pins: emmc-ds {
+					mux {
+						groups = "emmc_nand_ds";
+						function = "emmc";
+						bias-pull-down;
+					};
+				};
+
+				emmc_clk_gate_pins: emmc-clk-gate {
+					mux {
+						groups = "GPIOB_8";
+						function = "gpio_periphs";
+						bias-pull-down;
+					};
+				};
+
+				i2c0_ao_d_pins: i2c0-ao-d {
+					mux {
+						groups = "i2c0_ao_sck_d",
+							 "i2c0_ao_sda_d";
+						function = "i2c0_ao";
+						bias-disable;
+						drive-strength-microamp = <3000>;
+					};
+				};
+
+				pwm_a_pins: pwm-a {
+					mux {
+						groups = "pwm_a";
+						function = "pwm_a";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_a_pins: pwm-ao-a {
+					mux {
+						groups = "pwm_ao_a";
+						function = "pwm_ao_a";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_b_pins: pwm-ao-b {
+					mux {
+						groups = "pwm_ao_b";
+						function = "pwm_ao_b";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_c_d_pins: pwm-ao-c-d {
+					mux {
+						groups = "pwm_ao_c_d";
+						function = "pwm_ao_c";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_c_e_pins: pwm-ao-c-e {
+					mux {
+						groups = "pwm_ao_c_e";
+						function = "pwm_ao_c";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_c_hiz_pins: pwm-ao-c-hiz {
+					mux {
+						groups = "pwm_ao_c_hiz";
+						function = "pwm_ao_c_hiz";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_d_pins: pwm-ao-d {
+					mux {
+						groups = "pwm_ao_d";
+						function = "pwm_ao_d";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_e_pins: pwm-ao-e {
+					mux {
+						groups = "pwm_ao_e";
+						function = "pwm_ao_e";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_f_pins: pwm-ao-f {
+					mux {
+						groups = "pwm_ao_f";
+						function = "pwm_ao_f";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_g_pins: pwm-ao-g {
+					mux {
+						groups = "pwm_ao_g";
+						function = "pwm_ao_g";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_g_hiz_pins: pwm-ao-g-hiz {
+					mux {
+						groups = "pwm_ao_g_hiz";
+						function = "pwm_ao_g_hiz";
+						bias-disable;
+					};
+				};
+
+				pwm_ao_h_pins: pwm-ao-h {
+					mux {
+						groups = "pwm_ao_h";
+						function = "pwm_ao_h";
+						bias-disable;
+					};
+				};
+
+				pwm_b_pins: pwm-b {
+					mux {
+						groups = "pwm_b";
+						function = "pwm_b";
+						bias-disable;
+					};
+				};
+
+				pwm_c_pins: pwm-c {
+					mux {
+						groups = "pwm_c";
+						function = "pwm_c";
+						bias-disable;
+					};
+				};
+
+				pwm_d_pins: pwm-d {
+					mux {
+						groups = "pwm_d";
+						function = "pwm_d";
+						bias-disable;
+					};
+				};
+
+				pwm_e_pins: pwm-e {
+					mux {
+						groups = "pwm_e";
+						function = "pwm_e";
+						bias-disable;
+					};
+				};
+
+				pwm_f_pins: pwm-f {
+					mux {
+						groups = "pwm_f";
+						function = "pwm_f";
+						bias-disable;
+					};
+				};
+
+				pwm_vs_pins: pwm-vs {
+					mux {
+						groups = "pwm_vs";
+						function = "pwm_vs";
+						bias-disable;
+					};
+				};
+
+				sdcard_pins: sdcard {
+					mux {
+						groups = "sdcard_d0",
+							 "sdcard_d1",
+							 "sdcard_d2",
+							 "sdcard_d3",
+							 "sdcard_clk",
+							 "sdcard_cmd";
+						function = "sdcard";
+						bias-pull-up;
+					};
+				};
+
+				sdcard_clk_gate_pins: sdcard-clk-gate {
+					mux {
+						groups = "GPIOC_4";
+						function = "gpio_periphs";
+						bias-pull-down;
+					};
+				};
+
+				sdio_pins: sdio {
+					mux {
+						groups = "sdio_d0",
+							 "sdio_d1",
+							 "sdio_d2",
+							 "sdio_d3",
+							 "sdio_clk",
+							 "sdio_cmd";
+						function = "sdio";
+						bias-pull-up;
+					};
+				};
+
+				sdio_clk_gate_pins: sdio-clk-gate {
+					mux {
+						groups = "GPIOX_4";
+						function = "gpio_periphs";
+						bias-pull-up;
+					};
+				};
 			};
 
 			gpio_intc: interrupt-controller@4080 {
@@ -271,6 +590,64 @@ uart_a: serial@78000 {
 				status = "disabled";
 			};
 
+			gp0: clock-controller@8080 {
+				compatible = "amlogic,t7-gp0-pll";
+				reg = <0x0 0x8080 0x0 0x20>;
+				clocks = <&scmi_clk CLKID_TOP_PLL_OSC>;
+				clock-names = "in0";
+				#clock-cells = <1>;
+			};
+
+			gp1: clock-controller@80c0 {
+				compatible = "amlogic,t7-gp1-pll";
+				reg = <0x0 0x80c0 0x0 0x14>;
+				clocks = <&scmi_clk CLKID_TOP_PLL_OSC>;
+				clock-names = "in0";
+				#clock-cells = <1>;
+			};
+
+			hifi: clock-controller@8100 {
+				compatible = "amlogic,t7-hifi-pll";
+				reg = <0x0 0x8100 0x0 0x20>;
+				clocks = <&scmi_clk CLKID_TOP_PLL_OSC>;
+				clock-names = "in0";
+				#clock-cells = <1>;
+			};
+
+			pcie: clock-controller@8140 {
+				compatible = "amlogic,t7-pcie-pll";
+				reg = <0x0 0x8140 0x0 0x1c>;
+				clocks = <&scmi_clk CLKID_PCIE_OSC>;
+				clock-names = "in0";
+				#clock-cells = <1>;
+			};
+
+			mpll: clock-controller@8180 {
+				compatible = "amlogic,t7-mpll";
+				reg = <0x0 0x8180 0x0 0x28>;
+				clocks = <&scmi_clk CLKID_FIXED_PLL_DCO>;
+				clock-names = "in0";
+				#clock-cells = <1>;
+			};
+
+			hdmi: clock-controller@81c0 {
+				compatible = "amlogic,t7-hdmi-pll";
+				reg = <0x0 0x81c0 0x0 0x20>;
+				clocks = <&scmi_clk CLKID_HDMI_PLL_OSC>;
+				clock-names = "in0";
+				#clock-cells = <1>;
+			};
+
+			mclk: clock-controller@8300 {
+				compatible = "amlogic,t7-mclk-pll";
+				reg = <0x0 0x8300 0x0 0x18>;
+				clocks = <&scmi_clk CLKID_MCLK_PLL_OSC>,
+					 <&xtal>,
+					 <&scmi_clk CLKID_FCLK_50M>;
+				clock-names = "in0", "in1", "in2";
+				#clock-cells = <1>;
+			};
+
 			sec_ao: ao-secure@10220 {
 				compatible = "amlogic,t7-ao-secure",
 					     "amlogic,meson-gx-ao-secure",
@@ -278,6 +655,111 @@ sec_ao: ao-secure@10220 {
 				reg = <0x0 0x10220 0x0 0x140>;
 				amlogic,has-chip-id;
 			};
+
+			pwm_ao_ef: pwm@30000 {
+				compatible = "amlogic,t7-pwm", "amlogic,meson-s4-pwm";
+				reg = <0x0 0x30000 0x0 0x24>;
+				clocks = <&clkc_periphs CLKID_PWM_AO_E>,
+					 <&clkc_periphs CLKID_PWM_AO_F>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_ao_gh: pwm@32000 {
+				compatible = "amlogic,t7-pwm", "amlogic,meson-s4-pwm";
+				reg = <0x0 0x32000 0x0 0x24>;
+				clocks = <&clkc_periphs CLKID_PWM_AO_G>,
+					 <&clkc_periphs CLKID_PWM_AO_H>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_ab: pwm@58000 {
+				compatible = "amlogic,t7-pwm", "amlogic,meson-s4-pwm";
+				reg = <0x0 0x58000 0x0 0x24>;
+				clocks = <&clkc_periphs CLKID_PWM_A>,
+					 <&clkc_periphs CLKID_PWM_B>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_cd: pwm@5a000 {
+				compatible = "amlogic,t7-pwm", "amlogic,meson-s4-pwm";
+				reg = <0x0 0x5a000 0x0 0x24>;
+				clocks = <&clkc_periphs CLKID_PWM_C>,
+					 <&clkc_periphs CLKID_PWM_D>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_ef: pwm@5c000 {
+				compatible = "amlogic,t7-pwm", "amlogic,meson-s4-pwm";
+				reg = <0x0 0x5c000 0x0 0x24>;
+				clocks = <&clkc_periphs CLKID_PWM_E>,
+					 <&clkc_periphs CLKID_PWM_F>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_ao_ab: pwm@5e000 {
+				compatible = "amlogic,t7-pwm", "amlogic,meson-s4-pwm";
+				reg = <0x0 0x5e000 0x0 0x24>;
+				clocks = <&clkc_periphs CLKID_PWM_AO_A>,
+					 <&clkc_periphs CLKID_PWM_AO_B>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			pwm_ao_cd: pwm@60000 {
+				compatible = "amlogic,t7-pwm", "amlogic,meson-s4-pwm";
+				reg = <0x0 0x60000 0x0 0x24>;
+				clocks = <&clkc_periphs CLKID_PWM_AO_C>,
+					 <&clkc_periphs CLKID_PWM_AO_D>;
+				#pwm-cells = <3>;
+				status = "disabled";
+			};
+
+			sd_emmc_a: mmc@88000 {
+				compatible = "amlogic,t7-mmc", "amlogic,meson-axg-mmc";
+				reg = <0x0 0x88000 0x0 0x800>;
+				interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&clkc_periphs CLKID_SYS_SD_EMMC_A>,
+					 <&clkc_periphs CLKID_SD_EMMC_A>,
+					 <&scmi_clk CLKID_FCLK_DIV2>;
+				clock-names = "core", "clkin0", "clkin1";
+				resets = <&reset RESET_SD_EMMC_A>;
+				assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_A_SEL>;
+				assigned-clock-parents = <&xtal>;
+				status = "disabled";
+			};
+
+			sd_emmc_b: mmc@8a000 {
+				compatible = "amlogic,t7-mmc", "amlogic,meson-axg-mmc";
+				reg = <0x0 0x8a000 0x0 0x800>;
+				interrupts = <GIC_SPI 177 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&clkc_periphs CLKID_SYS_SD_EMMC_B>,
+					 <&clkc_periphs CLKID_SD_EMMC_B>,
+					 <&scmi_clk CLKID_FCLK_DIV2>;
+				clock-names = "core", "clkin0", "clkin1";
+				resets = <&reset RESET_SD_EMMC_B>;
+				assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_B_SEL>;
+				assigned-clock-parents = <&xtal>;
+				status = "disabled";
+			};
+
+			sd_emmc_c: mmc@8c000 {
+				compatible = "amlogic,t7-mmc", "amlogic,meson-axg-mmc";
+				reg = <0x0 0x8c000 0x0 0x800>;
+				interrupts = <GIC_SPI 178 IRQ_TYPE_EDGE_RISING>;
+				clocks = <&clkc_periphs CLKID_SYS_SD_EMMC_C>,
+					 <&clkc_periphs CLKID_SD_EMMC_C>,
+					 <&scmi_clk CLKID_FCLK_DIV2>;
+				clock-names = "core", "clkin0", "clkin1";
+				resets = <&reset RESET_SD_EMMC_C>;
+				assigned-clocks = <&clkc_periphs CLKID_SD_EMMC_C_SEL>;
+				assigned-clock-parents = <&xtal>;
+				status = "disabled";
+			};
 		};
 
 	};
diff --git a/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-phicomm-n1.dts b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-phicomm-n1.dts
index 393d3cb33b9ee..dea3a60e8527d 100644
--- a/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-phicomm-n1.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-gxl-s905d-phicomm-n1.dts
@@ -30,6 +30,21 @@ &cvbs_vdac_port {
 	status = "disabled";
 };
 
+&uart_A {
+	status = "okay";
+	pinctrl-0 = <&uart_a_pins>, <&uart_a_cts_rts_pins>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+
+	bluetooth {
+		compatible = "brcm,bcm43438-bt";
+		shutdown-gpios = <&gpio GPIOX_17 GPIO_ACTIVE_HIGH>;
+		max-speed = <2000000>;
+		clocks = <&wifi32k>;
+		clock-names = "lpo";
+	};
+};
+
 &usb {
 	dr_mode = "host";
 };
diff --git a/arch/arm64/boot/dts/amlogic/meson-s4-s905y4-khadas-vim1s.dts b/arch/arm64/boot/dts/amlogic/meson-s4-s905y4-khadas-vim1s.dts
index 27d0f6134ea9d..99d5df71b9cd4 100644
--- a/arch/arm64/boot/dts/amlogic/meson-s4-s905y4-khadas-vim1s.dts
+++ b/arch/arm64/boot/dts/amlogic/meson-s4-s905y4-khadas-vim1s.dts
@@ -6,6 +6,8 @@
 /dts-v1/;
 
 #include "meson-s4.dtsi"
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
 
 / {
 	model = "Khadas VIM1S";
@@ -18,6 +20,8 @@ aliases {
 		mmc0 = &emmc; /* eMMC */
 		mmc1 = &sd; /* SD card */
 		mmc2 = &sdio; /* SDIO */
+		rtc0 = &rtc;
+		rtc1 = &vrtc;
 		serial0 = &uart_b;
 	};
 
@@ -38,11 +42,48 @@ secmon_reserved: secmon@5000000 {
 		};
 	};
 
+	adc-keys {
+		compatible = "adc-keys";
+		io-channels = <&saradc 0>;
+		io-channel-names = "buttons";
+		keyup-threshold-microvolt = <1710000>;
+		poll-interval = <100>;
+
+		button-function {
+			label = "Function";
+			linux,code = <KEY_FN>;
+			press-threshold-microvolt = <10000>;
+		};
+	};
+
 	emmc_pwrseq: emmc-pwrseq {
 		compatible = "mmc-pwrseq-emmc";
 		reset-gpios = <&gpio GPIOB_9 GPIO_ACTIVE_LOW>;
 	};
 
+	gpio-keys-polled {
+		compatible = "gpio-keys-polled";
+		poll-interval = <100>;
+
+		power-button {
+			label = "power";
+			linux,code = <KEY_POWER>;
+			gpios = <&gpio GPIOD_8 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	pwm-leds {
+		compatible = "pwm-leds";
+
+		led-1 {
+			color = <LED_COLOR_ID_WHITE>;
+			function = LED_FUNCTION_STATUS;
+			linux,default-trigger = "heartbeat";
+			max-brightness = <255>;
+			pwms = <&pwm_gh 0 30040 0>;
+		};
+	};
+
 	sdio_32k: sdio-32k {
 		compatible = "pwm-clock";
 		#clock-cells = <0>;
@@ -184,10 +225,24 @@ &ethmac {
 	phy-mode = "rmii";
 };
 
+&i2c1 {
+	status = "okay";
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_pins2>;
+	clock-frequency = <100000>;
+
+	rtc: rtc@51 {
+		compatible = "haoyu,hym8563";
+		reg = <0x51>;
+		#clock-cells = <0>;
+	};
+};
+
 &ir {
 	status = "okay";
 	pinctrl-0 = <&remote_pins>;
 	pinctrl-names = "default";
+	linux,rc-map-name = "rc-khadas";
 };
 
 &pwm_ef {
@@ -196,6 +251,12 @@ &pwm_ef {
 	pinctrl-names = "default";
 };
 
+&pwm_gh {
+	status = "okay";
+	pinctrl-0 = <&pwm_g_pins1>;
+	pinctrl-names = "default";
+};
+
 &pwm_ij {
 	status = "okay";
 };
@@ -247,6 +308,11 @@ brcmf: wifi@1 {
 	};
 };
 
+&saradc {
+	status = "okay";
+	vref-supply = <&vddio_ao1v8>;
+};
+
 &spicc0 {
 	status = "okay";
 	pinctrl-names = "default";
@@ -254,6 +320,21 @@ &spicc0 {
 	cs-gpios = <&gpio GPIOX_10 GPIO_ACTIVE_LOW>;
 };
 
+&uart_a {
+	status = "okay";
+	pinctrl-0 = <&uart_a_pins>, <&uart_a_cts_rts_pins>;
+	pinctrl-names = "default";
+	uart-has-rtscts;
+
+	bluetooth {
+		compatible = "brcm,bcm4345c5";
+		shutdown-gpios = <&gpio GPIOX_17 GPIO_ACTIVE_HIGH>;
+		max-speed = <2000000>;
+		clocks = <&sdio_32k>;
+		clock-names = "lpo";
+	};
+};
+
 &uart_b {
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi
index dfc0a30a6e61b..2a6fbd5308362 100644
--- a/arch/arm64/boot/dts/amlogic/meson-s4.dtsi
+++ b/arch/arm64/boot/dts/amlogic/meson-s4.dtsi
@@ -59,6 +59,11 @@ psci {
 		method = "smc";
 	};
 
+	vrtc: rtc@fe010288 {
+		compatible = "amlogic,meson-vrtc";
+		reg = <0x0 0xfe010288 0x0 0x4>;
+	};
+
 	xtal: xtal-clk {
 		compatible = "fixed-clock";
 		clock-frequency = <24000000>;
@@ -77,6 +82,20 @@ pwrc: power-controller {
 		};
 	};
 
+	saradc: adc@fe026000 {
+		compatible = "amlogic,meson-s4-saradc",
+			     "amlogic,meson-g12a-saradc";
+		reg = <0x0 0xfe026000 0x0 0x48>;
+		#io-channel-cells = <1>;
+		interrupts = <GIC_SPI 181 IRQ_TYPE_EDGE_RISING>;
+		clocks = <&xtal>,
+			 <&clkc_periphs CLKID_SAR_ADC>,
+			 <&clkc_periphs CLKID_SARADC>,
+			 <&clkc_periphs CLKID_SARADC_SEL>;
+		clock-names = "clkin", "core", "adc_clk", "adc_sel";
+		status = "disabled";
+	};
+
 	soc {
 		compatible = "simple-bus";
 		#address-cells = <2>;
@@ -163,6 +182,22 @@ mux {
 					};
 				};
 
+				uart_a_pins: uart-a-pins {
+					mux {
+						groups = "uart_a_tx",
+						       "uart_a_rx";
+						function = "uart_a";
+					};
+				};
+
+				uart_a_cts_rts_pins: uart-a-cts-rts-pins {
+					mux {
+						groups = "uart_a_cts",
+						       "uart_a_rts";
+						function = "uart_a";
+					};
+				};
+
 				i2c0_pins1: i2c0-pins1 {
 					mux {
 						groups = "i2c0_sda",
@@ -814,6 +849,16 @@ mdio0: mdio {
 			};
 		};
 
+		uart_a: serial@fe078000 {
+			compatible = "amlogic,meson-s4-uart",
+				     "amlogic,meson-ao-uart";
+			reg = <0x0 0xfe078000 0x0 0x18>;
+			interrupts = <GIC_SPI 168 IRQ_TYPE_EDGE_RISING>;
+			clocks = <&xtal>, <&clkc_periphs CLKID_UART_A>, <&xtal>;
+			clock-names = "xtal", "pclk", "baud";
+			status = "disabled";
+		};
+
 		sdio: mmc@fe088000 {
 			compatible = "amlogic,meson-axg-mmc";
 			reg = <0x0 0xfe088000 0x0 0x800>;
diff --git a/arch/arm64/boot/dts/exynos/exynos850.dtsi b/arch/arm64/boot/dts/exynos/exynos850.dtsi
index cb55015c8dceb..3881f573ec08d 100644
--- a/arch/arm64/boot/dts/exynos/exynos850.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynos850.dtsi
@@ -215,6 +215,13 @@ pmu_system_controller: system-controller@11860000 {
 			compatible = "samsung,exynos850-pmu", "syscon";
 			reg = <0x11860000 0x10000>;
 
+			poweroff: syscon-poweroff {
+				compatible = "syscon-poweroff";
+				offset = <0x30c>; /* PS_HOLD_CONTROL */
+				mask = <0x00000100>;
+				value = <0x0>;
+			};
+
 			reboot: syscon-reboot {
 				compatible = "syscon-reboot";
 				regmap = <&pmu_system_controller>;
diff --git a/arch/arm64/boot/dts/exynos/exynosautov920-sadk.dts b/arch/arm64/boot/dts/exynos/exynosautov920-sadk.dts
index a397f068ed535..5873720c213e2 100644
--- a/arch/arm64/boot/dts/exynos/exynosautov920-sadk.dts
+++ b/arch/arm64/boot/dts/exynos/exynosautov920-sadk.dts
@@ -83,6 +83,14 @@ &usi_0 {
 	status = "okay";
 };
 
+&ufs_0 {
+	status = "okay";
+};
+
+&ufs_0_phy {
+	status = "okay";
+};
+
 &xtcxo {
 	clock-frequency = <38400000>;
 };
diff --git a/arch/arm64/boot/dts/exynos/exynosautov920.dtsi b/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
index 0bf7c4cb98461..f1f5efcdb91ea 100644
--- a/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
+++ b/arch/arm64/boot/dts/exynos/exynosautov920.dtsi
@@ -1426,6 +1426,12 @@ cmu_hsi2: clock-controller@16b00000 {
 				      "ethernet";
 		};
 
+		syscon_hsi2: syscon@16c00000 {
+			compatible = "samsung,exynosautov920-hsi2-sysreg",
+				     "syscon";
+			reg = <0x16c00000 0x800>;
+		};
+
 		pinctrl_hsi2: pinctrl@16c10000 {
 			compatible = "samsung,exynosautov920-pinctrl";
 			reg = <0x16c10000 0x10000>;
@@ -1438,6 +1444,27 @@ pinctrl_hsi2ufs: pinctrl@16d20000 {
 			interrupts = <GIC_SPI 603 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
+		ufs_0: ufs@16e00000 {
+			compatible = "samsung,exynosautov920-ufs";
+			reg = <0x16e00000 0x100>,
+			      <0x16e01100 0x400>,
+			      <0x16e80000 0x8000>,
+			      <0x16d08000 0x800>;
+			reg-names = "hci", "vs_hci", "unipro", "ufsp";
+			interrupts = <GIC_SPI 613 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&cmu_hsi2 CLK_MOUT_HSI2_UFS_EMBD_USER>,
+				 <&cmu_hsi2 CLK_MOUT_HSI2_NOC_UFS_USER>;
+			clock-names = "core_clk", "sclk_unipro_main";
+			freq-table-hz = <0 0>, <0 0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&ufs_rst_n &ufs_refclk_out>;
+			phys = <&ufs_0_phy>;
+			phy-names = "ufs-phy";
+			samsung,sysreg = <&syscon_hsi2 0x710>;
+			dma-coherent;
+			status = "disabled";
+		};
+
 		ufs_0_phy: phy@16e04000 {
 			compatible = "samsung,exynosautov920-ufs-phy";
 			reg = <0x16e04000 0x4000>;
diff --git a/arch/arm64/boot/dts/freescale/Makefile b/arch/arm64/boot/dts/freescale/Makefile
index 711e36cc2c990..513f61eb27b85 100644
--- a/arch/arm64/boot/dts/freescale/Makefile
+++ b/arch/arm64/boot/dts/freescale/Makefile
@@ -177,11 +177,13 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-nonwifi-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-nonwifi-ivy.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-nonwifi-mallow.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-nonwifi-yavia.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-nonwifi-zinnia.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-wifi-dahlia.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-wifi-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-wifi-ivy.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-wifi-mallow.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-wifi-yavia.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mm-verdin-wifi-zinnia.dtb
 
 imx8mm-tqma8mqml-mba8mx-lvds-g133han01-dtbs += imx8mm-tqma8mqml-mba8mx.dtb imx8mm-tqma8mqml-mba8mx-lvds-g133han01.dtbo
 imx8mm-tqma8mqml-mba8mx-lvds-tm070jvhg33-dtbs += imx8mm-tqma8mqml-mba8mx.dtb imx8mm-tqma8mqml-mba8mx-lvds-tm070jvhg33.dtbo
@@ -206,10 +208,10 @@ imx8mn-vhip4-evalboard-v1-overlay-ksz8794-dtbs := imx8mn-vhip4-evalboard-v1.dtb
 						  imx8mn-vhip4-evalboard-v1-overlay-ksz8794.dtbo
 imx8mn-vhip4-evalboard-v1-overlay-ksz9031-dtbs := imx8mn-vhip4-evalboard-v1.dtb \
 						  imx8mn-vhip4-evalboard-v1-overlay-ksz9031.dtbo
-imx8mn-vhip4-evalboard-v2-overlay-ksz8794-dtbs := imx8mn-vhip4-evalboard-v2.dtb \
-						  imx8mn-vhip4-evalboard-v2-overlay-ksz8794.dtbo
 imx8mn-vhip4-evalboard-v2-overlay-adin1300-dtbs := imx8mn-vhip4-evalboard-v2.dtb \
 						   imx8mn-vhip4-evalboard-v2-overlay-adin1300.dtbo
+imx8mn-vhip4-evalboard-v2-overlay-ksz8794-dtbs := imx8mn-vhip4-evalboard-v2.dtb \
+						  imx8mn-vhip4-evalboard-v2-overlay-ksz8794.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx8mn-vhip4-evalboard-v1.dtb \
 			  imx8mn-vhip4-evalboard-v1-overlay-ksz8794.dtb \
 			  imx8mn-vhip4-evalboard-v1-overlay-ksz8794.dtbo \
@@ -345,23 +347,27 @@ dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-nonwifi-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-nonwifi-ivy.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-nonwifi-mallow.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-nonwifi-yavia.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-nonwifi-zinnia.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-wifi-dahlia.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-wifi-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-wifi-ivy.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-wifi-mallow.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-wifi-yavia.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-verdin-wifi-zinnia.dtb
 
 imx8mp-evk-lvds0-imx-dlvds-hdmi-channel0-dtbs += imx8mp-evk.dtb imx8mp-evk-lvds0-imx-dlvds-hdmi-channel0.dtbo
 imx8mp-evk-lvds0-imx-lvds-hdmi-dtbs += imx8mp-evk.dtb imx8mp-evk-lvds0-imx-lvds-hdmi.dtbo
 imx8mp-evk-lvds1-imx-dlvds-hdmi-channel0-dtbs += imx8mp-evk.dtb imx8mp-evk-lvds1-imx-dlvds-hdmi-channel0.dtbo
 imx8mp-evk-lvds1-imx-lvds-hdmi-dtbs += imx8mp-evk.dtb imx8mp-evk-lvds1-imx-lvds-hdmi.dtbo
 imx8mp-evk-mx8-dlvds-lcd1-dtbs += imx8mp-evk.dtb imx8mp-evk-mx8-dlvds-lcd1.dtbo
-imx8mp-evk-pcie-ep-dtbs += imx8mp-evk.dtb imx-pcie0-ep.dtbo
+imx8mp-evk-pcie-dtbs += imx8mp-evk.dtb imx-m2-pcie.dtbo
+imx8mp-evk-pcie-ep-dtbs += imx8mp-evk-pcie.dtb imx-pcie0-ep.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk-lvds0-imx-dlvds-hdmi-channel0.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk-lvds0-imx-lvds-hdmi.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk-lvds1-imx-dlvds-hdmi-channel0.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk-lvds1-imx-lvds-hdmi.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk-mx8-dlvds-lcd1.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk-pcie.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8mp-evk-pcie-ep.dtb
 
 imx8mp-tqma8mpql-mba8mpxl-lvds-tm070jvhg33-dtbs += imx8mp-tqma8mpql-mba8mpxl.dtb imx8mp-tqma8mpql-mba8mpxl-lvds-tm070jvhg33.dtbo
@@ -408,6 +414,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8qm-apalis-v1.1-eval-v1.2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8qm-apalis-v1.1-ixora-v1.1.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8qm-apalis-v1.1-ixora-v1.2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8qm-mek.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx8qm-tqma8qm-mba8x.dtb
 
 imx8qm-mek-ov5640-csi0-dtbs := imx8qm-mek.dtb imx8qm-mek-ov5640-csi0.dtbo
 dtb-${CONFIG_ARCH_MXC} += imx8qm-mek-ov5640-csi0.dtb
@@ -438,6 +445,7 @@ dtb-$(CONFIG_ARCH_MXC) += imx8qxp-tqma8xqp-mba8xx.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8qxp-tqma8xqps-mb-smarc-2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8ulp-9x9-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx8ulp-evk.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx91-9x9-qsb.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-frdm.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-11x11-frdm-s.dtb
@@ -449,15 +457,18 @@ imx91-tqma9131-mba91xxca-rgb-cdtech-dc44-dtbs := imx91-tqma9131-mba91xxca.dtb im
 dtb-$(CONFIG_ARCH_MXC) += imx91-tqma9131-mba91xxca-lvds-tm070jvhg33.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-tqma9131-mba91xxca-rgb-cdtech-dc44.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx91-var-dart-sonata.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx91-var-som-symphony.dtb
 
 dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb.dtb
 
 imx93-9x9-qsb-can1-dtbs += imx93-9x9-qsb.dtb imx93-9x9-qsb-can1.dtbo
 imx93-9x9-qsb-i3c-dtbs += imx93-9x9-qsb.dtb imx93-9x9-qsb-i3c.dtbo
 imx93-9x9-qsb-ontat-kd50g21-40nt-a1-dtbs += imx93-9x9-qsb.dtb imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtbo
+imx93-9x9-qsb-tianma-tm050rdh03-dtbs += imx93-9x9-qsb.dtb imx93-9x9-qsb-tianma-tm050rdh03.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb-can1.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb-i3c.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx93-9x9-qsb-tianma-tm050rdh03.dtb
 
 dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-11x11-frdm.dtb
@@ -498,10 +509,16 @@ dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba91xxca-rgb-cdtech-dc44.dtb
 imx93-tqma9352-mba93xxla-mini-ezurio-wlan-dtbs += imx93-tqma9352-mba93xxla-mini.dtb imx93-tqma9352-mba93xxla-mini-ezurio-wlan.dtbo
 
 dtb-$(CONFIG_ARCH_MXC) += imx93-tqma9352-mba93xxla-mini-ezurio-wlan.dtb
-
+dtb-$(CONFIG_ARCH_MXC) += imx93-var-dart-sonata.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93-var-som-symphony.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx93w-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx943-evk.dtb
+
+imx943-evk-pcie0-ep-dtbs += imx943-evk.dtb imx-pcie0-ep.dtbo
+imx943-evk-pcie1-ep-dtbs += imx943-evk.dtb imx-pcie1-ep.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx943-evk-pcie0-ep.dtb imx943-evk-pcie1-ep.dtb
+
+dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-ab2.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-evk.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-frdm.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-19x19-evk.dtb
@@ -514,13 +531,18 @@ dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-nonwifi-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-nonwifi-ivy.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-nonwifi-mallow.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-nonwifi-yavia.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-nonwifi-zinnia.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-wifi-dahlia.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-wifi-dev.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-wifi-ivy.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-wifi-mallow.dtb
 dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-wifi-yavia.dtb
+dtb-$(CONFIG_ARCH_MXC) += imx95-verdin-wifi-zinnia.dtb
+
+imx95-15x15-evk-pcie-dtbs += imx95-15x15-evk.dtb imx-m2-pcie.dtbo
+dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-evk-pcie.dtb
 
-imx95-15x15-evk-pcie0-ep-dtbs = imx95-15x15-evk.dtb imx-pcie0-ep.dtbo
+imx95-15x15-evk-pcie0-ep-dtbs = imx95-15x15-evk-pcie.dtb imx-pcie0-ep.dtbo
 dtb-$(CONFIG_ARCH_MXC) += imx95-15x15-evk-pcie0-ep.dtb
 imx95-19x19-evk-pcie0-ep-dtbs += imx95-19x19-evk.dtb imx-pcie0-ep.dtbo
 imx95-19x19-evk-pcie1-ep-dtbs += imx95-19x19-evk.dtb imx-pcie1-ep.dtbo
diff --git a/arch/arm64/boot/dts/freescale/imx-m2-pcie.dtso b/arch/arm64/boot/dts/freescale/imx-m2-pcie.dtso
new file mode 100644
index 0000000000000..1930de058a08b
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx-m2-pcie.dtso
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+/plugin/;
+
+&pcie0 {
+	status = "okay";
+};
+
+&m2_usdhc {
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
index bc62ae5ca812d..441e009030299 100644
--- a/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8dxl-evk.dts
@@ -686,8 +686,6 @@ &pcie0_ep {
 	phy-names = "pcie-phy";
 	pinctrl-0 = <&pinctrl_pcieb>;
 	pinctrl-names = "default";
-	reset-gpio = <&lsio_gpio4 0 GPIO_ACTIVE_LOW>;
-	vpcie-supply = <&reg_pcieb>;
 	status = "disabled";
 };
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts
index e756fe5db56b6..dd59af0ebaae5 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-bl.dts
@@ -254,10 +254,6 @@ &pwm2 {
 	status = "okay";
 };
 
-&reg_nvcc_sd {
-	sd-vsel-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>;
-};
-
 &uart1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pinctrl_uart1>;
@@ -466,7 +462,7 @@ MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d0
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d0
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d0
 			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x19
-			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x40000d0
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0xd0
 		>;
 	};
 
@@ -479,7 +475,7 @@ MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d4
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d4
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d4
 			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x19
-			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x40000d0
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0xd0
 		>;
 	};
 
@@ -492,7 +488,7 @@ MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d6
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d6
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d6
 			MX8MM_IOMUXC_SD2_CD_B_GPIO2_IO12		0x19
-			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x40000d0
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0xd0
 		>;
 	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi
index 96987910609f1..4fb13d8ecfd45 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-kontron-osm-s.dtsi
@@ -342,7 +342,6 @@ reg_nvcc_sd: LDO5 {
 				regulator-name = "NVCC_SD (LDO5)";
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <3300000>;
-				sd-vsel-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>;
 			};
 		};
 	};
@@ -795,7 +794,7 @@ MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d0 /* SDIO_A_D1 */
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d0 /* SDIO_A_D2 */
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d0 /* SDIO_A_D3 */
 			MX8MM_IOMUXC_SD2_WP_USDHC2_WP			0x400000d6 /* SDIO_A_WP */
-			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x40000090
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x90
 		>;
 	};
 
@@ -808,7 +807,7 @@ MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d4 /* SDIO_A_D1 */
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d4 /* SDIO_A_D2 */
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d4 /* SDIO_A_D3 */
 			MX8MM_IOMUXC_SD2_WP_USDHC2_WP			0x400000d6 /* SDIO_A_WP */
-			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x40000090
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x90
 		>;
 	};
 
@@ -821,7 +820,7 @@ MX8MM_IOMUXC_SD2_DATA1_USDHC2_DATA1		0x1d6 /* SDIO_A_D1 */
 			MX8MM_IOMUXC_SD2_DATA2_USDHC2_DATA2		0x1d6 /* SDIO_A_D2 */
 			MX8MM_IOMUXC_SD2_DATA3_USDHC2_DATA3		0x1d6 /* SDIO_A_D3 */
 			MX8MM_IOMUXC_SD2_WP_USDHC2_WP			0x400000d6 /* SDIO_A_WP */
-			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x40000090
+			MX8MM_IOMUXC_GPIO1_IO04_USDHC2_VSELECT		0x90
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-verdin-nonwifi-zinnia.dts b/arch/arm64/boot/dts/freescale/imx8mm-verdin-nonwifi-zinnia.dts
new file mode 100644
index 0000000000000..07b4daf916c2a
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-verdin-nonwifi-zinnia.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx-8m-mini-nano
+ * https://www.toradex.com/products/carrier-board/zinnia-carrier-board
+ */
+
+/dts-v1/;
+
+#include "imx8mm-verdin.dtsi"
+#include "imx8mm-verdin-nonwifi.dtsi"
+#include "imx8mm-verdin-zinnia.dtsi"
+
+/ {
+	model = "Toradex Verdin iMX8M Mini on Zinnia";
+	compatible = "toradex,verdin-imx8mm-nonwifi-zinnia",
+		     "toradex,verdin-imx8mm-nonwifi",
+		     "toradex,verdin-imx8mm",
+		     "fsl,imx8mm";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-verdin-wifi-zinnia.dts b/arch/arm64/boot/dts/freescale/imx8mm-verdin-wifi-zinnia.dts
new file mode 100644
index 0000000000000..01a254dc1e6c6
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-verdin-wifi-zinnia.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx-8m-mini-nano
+ * https://www.toradex.com/products/carrier-board/zinnia-carrier-board
+ */
+
+/dts-v1/;
+
+#include "imx8mm-verdin.dtsi"
+#include "imx8mm-verdin-wifi.dtsi"
+#include "imx8mm-verdin-zinnia.dtsi"
+
+/ {
+	model = "Toradex Verdin iMX8M Mini WB on Zinnia";
+	compatible = "toradex,verdin-imx8mm-wifi-zinnia",
+		     "toradex,verdin-imx8mm-wifi",
+		     "toradex,verdin-imx8mm",
+		     "fsl,imx8mm";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-verdin-zinnia.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-verdin-zinnia.dtsi
new file mode 100644
index 0000000000000..686486e031788
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mm-verdin-zinnia.dtsi
@@ -0,0 +1,383 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Common dtsi for Verdin IMX8MM SoM on Zinnia carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx-8m-mini-nano
+ * https://www.toradex.com/products/carrier-board/zinnia-carrier-board
+ */
+
+#include <dt-bindings/leds/common.h>
+
+/ {
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_zinnia_leds>;
+
+		/* LED1 Red - SODIMM 48 - LED1_R */
+		led-0 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
+			gpios = <&gpio3 21 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED1 Blue - SODIMM 46 - LED1_B */
+		led-1 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
+			gpios = <&gpio3 24 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED3 Red - SODIMM 44 - LED3_R */
+		led-2 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <3>;
+			gpios = <&gpio3 22 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED3 Green - SODIMM 54 - LED3_G */
+		led-3 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <3>;
+			gpios = <&gpio3 1 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED3 Blue - SODIMM 36 - LED3_B */
+		led-4 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <3>;
+			gpios = <&gpio4 23 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED4 Red - SODIMM 34 - LED4_R */
+		led-5 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <4>;
+			gpios = <&gpio4 26 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED4 Green - SODIMM 32 - LED4_G */
+		led-6 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <4>;
+			gpios = <&gpio4 24 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED4 Blue - SODIMM 30 - LED4_B */
+		led-7 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <4>;
+			gpios = <&gpio4 25 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	zinnia-1v8-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <39000>; /* 12k + 27k */
+		/* Verdin ADC_4 */
+		io-channels = <&verdin_som_adc 4>;
+		output-ohms = <27000>;
+	};
+
+	zinnia-3v3-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <54000>; /* 27k + 27k */
+		/* Verdin ADC_3 */
+		io-channels = <&verdin_som_adc 5>;
+		output-ohms = <27000>;
+	};
+
+	zinnia-5v-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <39000>; /* 27k + 12k */
+		/* Verdin ADC_2 */
+		io-channels = <&verdin_som_adc 6>;
+		output-ohms = <12000>;
+	};
+
+	/* Zinnia Power Supply Input Voltage */
+	zinnia-input-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <204700>; /* 200k + 4.7k */
+		/* Verdin ADC_1 */
+		io-channels = <&verdin_som_adc 7>;
+		output-ohms = <4700>;
+	};
+};
+
+/* Verdin SPI_1 */
+&ecspi2 {
+	pinctrl-0 = <&pinctrl_ecspi2>, <&pinctrl_uart3_cts_gpio>;
+	cs-gpios = <&gpio5 9 GPIO_ACTIVE_LOW>, <&gpio5 13 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+
+	tpm@1 {
+		compatible = "infineon,slb9670", "tcg,tpm_tis-spi";
+		reg = <1>;
+		spi-max-frequency = <18500000>;
+	};
+};
+
+/* EEPROM on Zinnia */
+&eeprom_carrier_board {
+	status = "okay";
+};
+
+/* Verdin ETH_1 */
+&fec1 {
+	status = "okay";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"DI2_RB", /* SODIMM 216 */ /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"DO3_EN", /* SODIMM 220 */
+		"DI3_EN", /* SODIMM 222 */
+		"", /* 10 */
+		"DI2_EN", /* SODIMM 218 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		""; /* 20 */
+};
+
+&gpio3 {
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"DO1_EN", /* SODIMM 206 */
+		"",
+		"DI3_RB", /* SODIMM 56 */
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"";
+};
+
+&gpio4 {
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"";
+};
+
+&gpio5 {
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"DI1_EN", /* SODIMM 208 */
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"DI1_RB", /* SODIMM 210 */
+		"DO2_EN", /* SODIMM 212 */
+		"",
+		"";
+};
+
+/* Temperature sensor on Zinnia */
+&hwmon_temp {
+	compatible = "ti,tmp1075";
+
+	status = "okay";
+};
+
+/* Verdin I2C_1 */
+&i2c4 {
+	status = "okay";
+};
+
+/* Verdin UART_3 */
+&uart1 {
+	status = "okay";
+};
+
+/* Verdin UART_1 */
+&uart2 {
+	status = "okay";
+};
+
+/* Verdin UART_2 */
+&uart3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart3>, <&pinctrl_uart3_rts>;
+	rs485-rx-during-tx;
+	linux,rs485-enabled-at-boot-time;
+
+	status = "okay";
+};
+
+/* Verdin USB_1 */
+&usbotg1 {
+	status = "okay";
+};
+
+/* Verdin USB_2 */
+&usbotg2 {
+	status = "okay";
+};
+
+/* Verdin SD_1 */
+&usdhc2 {
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio1>,
+		    <&pinctrl_gpio2>,
+		    <&pinctrl_gpio3>,
+		    <&pinctrl_gpio4>,
+		    <&pinctrl_gpio5>,
+		    <&pinctrl_gpio6>,
+		    <&pinctrl_gpio7>,
+		    <&pinctrl_gpio8>,
+		    <&pinctrl_qspi1_io0_gpio>;
+
+	pinctrl_qspi1_io0_gpio: gpio3io6grp {
+		fsl,pins = <MX8MM_IOMUXC_NAND_DATA00_GPIO3_IO6	0x184>;	/* SODIMM 56 */
+	};
+
+	pinctrl_uart3_cts_gpio: gpio5io9grp {
+		fsl,pins = <MX8MM_IOMUXC_ECSPI1_SS0_GPIO5_IO9	0x184>;	/* SODIMM 143 */
+	};
+
+	pinctrl_zinnia_leds: zinnialedsgrp {
+		fsl,pins =
+			<MX8MM_IOMUXC_SAI2_TXC_GPIO4_IO25	0x16>, /* SODIMM 30 */
+			<MX8MM_IOMUXC_SAI2_TXFS_GPIO4_IO24	0x16>, /* SODIMM 32 */
+			<MX8MM_IOMUXC_SAI2_TXD0_GPIO4_IO26	0x16>, /* SODIMM 34 */
+			<MX8MM_IOMUXC_SAI2_RXD0_GPIO4_IO23	0x16>, /* SODIMM 36 */
+			<MX8MM_IOMUXC_SAI5_RXD1_GPIO3_IO22	0x16>, /* SODIMM 44 */
+			<MX8MM_IOMUXC_SAI5_RXD3_GPIO3_IO24	0x16>, /* SODIMM 46 */
+			<MX8MM_IOMUXC_SAI5_RXD0_GPIO3_IO21	0x16>, /* SODIMM 48 */
+			<MX8MM_IOMUXC_NAND_CE0_B_GPIO3_IO1	0x16>; /* SODIMM 54 */
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi b/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi
index 1594ce9182a58..5fc177f589cb2 100644
--- a/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mm-verdin.dtsi
@@ -735,7 +735,7 @@ &uart2 {
 /* Verdin UART_2 */
 &uart3 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart3>;
+	pinctrl-0 = <&pinctrl_uart3>, <&pinctrl_uart3_cts>, <&pinctrl_uart3_rts>;
 	uart-has-rtscts;
 };
 
@@ -1144,12 +1144,20 @@ pinctrl_uart2: uart2grp {
 			<MX8MM_IOMUXC_SAI3_TXFS_UART2_DCE_RX		0x146>;	/* SODIMM 129 */
 	};
 
+	pinctrl_uart3_cts: uart3ctsgrp {
+		fsl,pins =
+			<MX8MM_IOMUXC_ECSPI1_SS0_UART3_DCE_RTS_B	0x146>;	/* SODIMM 143 */
+	};
+
+	pinctrl_uart3_rts: uart3rtsgrp {
+		fsl,pins =
+			<MX8MM_IOMUXC_ECSPI1_MISO_UART3_DCE_CTS_B	0x146>;	/* SODIMM 141 */
+	};
+
 	pinctrl_uart3: uart3grp {
 		fsl,pins =
-			<MX8MM_IOMUXC_ECSPI1_MISO_UART3_DCE_CTS_B	0x146>,	/* SODIMM 141 */
 			<MX8MM_IOMUXC_ECSPI1_MOSI_UART3_DCE_TX		0x146>,	/* SODIMM 139 */
-			<MX8MM_IOMUXC_ECSPI1_SCLK_UART3_DCE_RX		0x146>,	/* SODIMM 137 */
-			<MX8MM_IOMUXC_ECSPI1_SS0_UART3_DCE_RTS_B	0x146>;	/* SODIMM 143 */
+			<MX8MM_IOMUXC_ECSPI1_SCLK_UART3_DCE_RX		0x146>;	/* SODIMM 137 */
 	};
 
 	pinctrl_uart4: uart4grp {
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts
index 5f37065bf43f3..a8f7c226a61f8 100644
--- a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v1.dts
@@ -112,7 +112,7 @@ &i2c3 {
 
 &ifm_pmic {
 	interrupt-parent = <&gpio2>;
-	interrupts = <0 GPIO_ACTIVE_LOW>;
+	interrupts = <0 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &iomuxc {
diff --git a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts
index 4dadfb7f78de2..43fd4d0041ef3 100644
--- a/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mn-vhip4-evalboard-v2.dts
@@ -99,7 +99,7 @@ &i2c3 {
 
 &ifm_pmic {
 	interrupt-parent = <&gpio5>;
-	interrupts = <17 GPIO_ACTIVE_LOW>;
+	interrupts = <17 IRQ_TYPE_LEVEL_LOW>;
 };
 
 &iomuxc {
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts
index dbbc0df0e3d1c..443e4fd5b9bfc 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-ab2.dts
@@ -281,7 +281,7 @@ pca9450: pmic@25 {
 		compatible = "nxp,pca9450c";
 		reg = <0x25>;
 		interrupt-parent = <&gpio1>;
-		interrupts = <3 GPIO_ACTIVE_LOW>;
+		interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
 		pinctrl-0 = <&pinctrl_pmic>;
 
 		regulators {
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
index d0a2bd975a181..2feb5b18645cd 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-evk.dts
@@ -774,7 +774,7 @@ &pcie0 {
 	vpcie-supply = <&reg_pcie0>;
 	vpcie3v3aux-supply = <&reg_pcie0>;
 	supports-clkreq;
-	status = "okay";
+	status = "disabled";
 };
 
 &pcie0_ep {
@@ -870,7 +870,7 @@ &uart3 {
 	status = "okay";
 };
 
-&usdhc1 {
+m2_usdhc: &usdhc1 {
 	pinctrl-names = "default", "state_100mhz", "state_200mhz";
 	pinctrl-0 = <&pinctrl_usdhc1>;
 	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi
index bc1a261bb000e..ea69c639b30b8 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-kontron-osm-s.dtsi
@@ -311,7 +311,6 @@ reg_nvcc_sd: LDO5 {
 				regulator-name = "NVCC_SD (LDO5)";
 				regulator-min-microvolt = <1800000>;
 				regulator-max-microvolt = <3300000>;
-				sd-vsel-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>;
 			};
 		};
 	};
@@ -815,7 +814,7 @@ MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0		0x1d0 /* SDIO_A_D0 */
 			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1		0x1d0 /* SDIO_A_D1 */
 			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2		0x1d0 /* SDIO_A_D2 */
 			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3		0x1d0 /* SDIO_A_D3 */
-			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT		0x400001d0
+			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT		0x1d0
 		>;
 	};
 
@@ -827,7 +826,7 @@ MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0		0x1d4 /* SDIO_A_D0 */
 			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1		0x1d4 /* SDIO_A_D1 */
 			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2		0x1d4 /* SDIO_A_D2 */
 			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3		0x1d4 /* SDIO_A_D3 */
-			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT		0x400001d0
+			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT		0x1d0
 		>;
 	};
 
@@ -839,7 +838,7 @@ MX8MP_IOMUXC_SD2_DATA0__USDHC2_DATA0		0x1d6 /* SDIO_A_D0 */
 			MX8MP_IOMUXC_SD2_DATA1__USDHC2_DATA1		0x1d6 /* SDIO_A_D1 */
 			MX8MP_IOMUXC_SD2_DATA2__USDHC2_DATA2		0x1d6 /* SDIO_A_D2 */
 			MX8MP_IOMUXC_SD2_DATA3__USDHC2_DATA3		0x1d6 /* SDIO_A_D3 */
-			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT		0x400001d0
+			MX8MP_IOMUXC_GPIO1_IO04__USDHC2_VSELECT		0x1d0
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-rdk.dts b/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-rdk.dts
index 0fe52c73fc8fa..4efdc6bdfe127 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-rdk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8mp-phyboard-pollux-rdk.dts
@@ -38,6 +38,18 @@ fan0: fan {
 		#cooling-cells = <2>;
 	};
 
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&hdmi_tx_out>;
+			};
+		};
+	};
+
 	panel_lvds1: panel-lvds1 {
 		/* compatible panel in overlay */
 		backlight = <&backlight_lvds1>;
@@ -126,6 +138,13 @@ reg_vcc_1v8_exp_con: regulator-vcc-1v8 {
 		regulator-name = "VCC_1V8_EXP_CON";
 	};
 
+	sound-hdmi {
+		compatible = "fsl,imx-audio-hdmi";
+		model = "audio-hdmi";
+		audio-cpu = <&aud2htx>;
+		hdmi-out;
+	};
+
 	thermal-zones {
 		soc-thermal {
 			trips {
@@ -146,6 +165,10 @@ map1 {
 	};
 };
 
+&aud2htx {
+	status = "okay";
+};
+
 /* TPM */
 &ecspi1 {
 	#address-cells = <1>;
@@ -201,6 +224,32 @@ &flexcan2 {
 	status = "okay";
 };
 
+&hdmi_pai {
+	status = "okay";
+};
+
+&hdmi_pvi {
+	status = "okay";
+};
+
+&hdmi_tx {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hdmi>;
+	status = "okay";
+
+	ports {
+		port@1 {
+			hdmi_tx_out: endpoint {
+				remote-endpoint = <&hdmi_connector_in>;
+			};
+		};
+	};
+};
+
+&hdmi_tx_phy {
+	status = "okay";
+};
+
 &i2c2 {
 	clock-frequency = <400000>;
 	pinctrl-names = "default", "gpio";
@@ -244,6 +293,10 @@ &i2c3 {
 	scl-gpios = <&gpio5 19 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
 };
 
+&lcdif3 {
+	status = "okay";
+};
+
 &ldb_lvds_ch1 {
 	remote-endpoint = <&panel1_in>;
 };
@@ -444,6 +497,15 @@ MX8MP_IOMUXC_SAI5_RXD0__GPIO3_IO21	0x154
 		>;
 	};
 
+	pinctrl_hdmi: hdmigrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_HDMI_DDC_SCL__HDMIMIX_HDMI_SCL			0x1c3
+			MX8MP_IOMUXC_HDMI_DDC_SDA__HDMIMIX_HDMI_SDA			0x1c3
+			MX8MP_IOMUXC_HDMI_HPD__HDMIMIX_HDMI_HPD				0x19
+			MX8MP_IOMUXC_HDMI_CEC__HDMIMIX_HDMI_CEC				0x19
+		>;
+	};
+
 	pinctrl_i2c2: i2c2grp {
 		fsl,pins = <
 			MX8MP_IOMUXC_I2C2_SCL__I2C2_SCL		0x400001c2
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106-av101hdt-a10.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106-av101hdt-a10.dtso
index c6fc5d5b1e5fa..23e4bf64b28b5 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106-av101hdt-a10.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106-av101hdt-a10.dtso
@@ -17,6 +17,7 @@
 
 	panel {
 		compatible = "boe,av101hdt-a10";
+		backlight = <&backlight>;
 		enable-gpios = <&gpio1 7 GPIO_ACTIVE_HIGH>;
 		pinctrl-0 = <&pinctrl_panel>;
 		pinctrl-names = "default";
@@ -40,7 +41,30 @@ reg_vbus: regulator-vbus {
 	};
 };
 
+&i2c4 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	backlight: backlight@6f {
+		compatible = "maxim,max25014";
+		reg = <0x6f>;
+		default-brightness = <50>;
+		enable-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_backlight>;
+		maxim,iset = <7>;
+		maxim,strings = <1 1 1 0>;
+	};
+};
+
 &iomuxc {
+	pinctrl_backlight: backlightgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO04__GPIO1_IO04
+				(MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
+		>;
+	};
+
 	pinctrl_panel: panelgrp {
 		fsl,pins = <
 			MX8MP_IOMUXC_GPIO1_IO07__GPIO1_IO07
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106-av123z7m-n17.dtso b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106-av123z7m-n17.dtso
index 3eb665ce9d5d2..0b969c8c04db1 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106-av123z7m-n17.dtso
+++ b/arch/arm64/boot/dts/freescale/imx8mp-tx8p-ml81-moduline-display-106-av123z7m-n17.dtso
@@ -16,6 +16,7 @@
 
 	panel {
 		compatible = "boe,av123z7m-n17";
+		backlight = <&backlight>;
 		enable-gpios = <&gpio1 7 GPIO_ACTIVE_HIGH>;
 		pinctrl-0 = <&pinctrl_panel>;
 		pinctrl-names = "default";
@@ -91,10 +92,26 @@ lvds1_out: endpoint {
 		};
 	};
 
-	/* max25014 @ 0x6f */
+	backlight: backlight@6f {
+		compatible = "maxim,max25014";
+		reg = <0x6f>;
+		default-brightness = <50>;
+		enable-gpios = <&gpio1 4 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_backlight>;
+		maxim,iset = <7>;
+		maxim,strings = <1 1 1 1>;
+	};
 };
 
 &iomuxc {
+	pinctrl_backlight: backlightgrp {
+		fsl,pins = <
+			MX8MP_IOMUXC_GPIO1_IO04__GPIO1_IO04
+				(MX8MP_PULL_UP | MX8MP_PULL_ENABLE)
+		>;
+	};
+
 	pinctrl_lvds_bridge: lvdsbridgegrp {
 		fsl,pins = <
 			MX8MP_IOMUXC_SAI1_TXD2__GPIO4_IO14
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-verdin-nonwifi-zinnia.dts b/arch/arm64/boot/dts/freescale/imx8mp-verdin-nonwifi-zinnia.dts
new file mode 100644
index 0000000000000..e78b25d65c94e
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-verdin-nonwifi-zinnia.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx-8m-plus
+ * https://www.toradex.com/products/carrier-board/zinnia-carrier-board
+ */
+
+/dts-v1/;
+
+#include "imx8mp-verdin.dtsi"
+#include "imx8mp-verdin-nonwifi.dtsi"
+#include "imx8mp-verdin-zinnia.dtsi"
+
+/ {
+	model = "Toradex Verdin iMX8M Plus on Zinnia";
+	compatible = "toradex,verdin-imx8mp-nonwifi-zinnia",
+		     "toradex,verdin-imx8mp-nonwifi",
+		     "toradex,verdin-imx8mp",
+		     "fsl,imx8mp";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-verdin-wifi-zinnia.dts b/arch/arm64/boot/dts/freescale/imx8mp-verdin-wifi-zinnia.dts
new file mode 100644
index 0000000000000..85eb565243142
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-verdin-wifi-zinnia.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx-8m-plus
+ * https://www.toradex.com/products/carrier-board/zinnia-carrier-board
+ */
+
+/dts-v1/;
+
+#include "imx8mp-verdin.dtsi"
+#include "imx8mp-verdin-wifi.dtsi"
+#include "imx8mp-verdin-zinnia.dtsi"
+
+/ {
+	model = "Toradex Verdin iMX8M Plus WB on Zinnia";
+	compatible = "toradex,verdin-imx8mp-wifi-zinnia",
+		     "toradex,verdin-imx8mp-wifi",
+		     "toradex,verdin-imx8mp",
+		     "fsl,imx8mp";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-verdin-zinnia.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-verdin-zinnia.dtsi
new file mode 100644
index 0000000000000..6beb2f2f4548c
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8mp-verdin-zinnia.dtsi
@@ -0,0 +1,422 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Common dtsi for Verdin IMX8MP SoM on Zinnia carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx-8m-plus
+ * https://www.toradex.com/products/carrier-board/zinnia-carrier-board
+ */
+
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/net/ti-dp83867.h>
+
+/ {
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_zinnia_leds>;
+
+		/* LED1 Red - SODIMM 48 - LED1_R */
+		led-1 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
+			gpios = <&gpio4 30 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED1 Blue - SODIMM 46 - LED1_B */
+		led-2 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
+			gpios = <&gpio5 01 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED3 Red - SODIMM 44 - LED3_R */
+		led-3 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <3>;
+			gpios = <&gpio4 31 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED3 Green - SODIMM 54 - LED3_G */
+		led-4 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <3>;
+			gpios = <&gpio3 1 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED3 Blue - SODIMM 36 - LED3_B */
+		led-5 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <3>;
+			gpios = <&gpio4 2 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED4 Red - SODIMM 34 - LED4_R */
+		led-6 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <4>;
+			gpios = <&gpio3 19 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED4 Green - SODIMM 32 - LED4_G */
+		led-7 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <4>;
+			gpios = <&gpio3 22 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED4 Blue - SODIMM 30 - LED4_B */
+		led-8 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <4>;
+			gpios = <&gpio3 25 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	zinnia-1v8-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <39000>; /* 12k + 27k */
+		/* Verdin ADC_4 */
+		io-channels = <&verdin_som_adc 4>;
+		output-ohms = <27000>;
+	};
+
+	zinnia-3v3-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <54000>; /* 27k + 27k */
+		/* Verdin ADC_3 */
+		io-channels = <&verdin_som_adc 5>;
+		output-ohms = <27000>;
+	};
+
+	zinnia-5v-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <39000>; /* 27k + 12k */
+		/* Verdin ADC_2 */
+		io-channels = <&verdin_som_adc 6>;
+		output-ohms = <12000>;
+	};
+
+	/* Zinnia Power Supply Input Voltage */
+	zinnia-input-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <204700>; /* 200k + 4.7k */
+		/* Verdin ADC_1 */
+		io-channels = <&verdin_som_adc 7>;
+		output-ohms = <4700>;
+	};
+};
+
+/* Verdin SPI_1 */
+&ecspi1 {
+	pinctrl-0 = <&pinctrl_ecspi1>, <&pinctrl_uart2_cts_gpio>;
+	cs-gpios = <&gpio2 6 GPIO_ACTIVE_LOW>, <&gpio5 9 GPIO_ACTIVE_LOW>;
+
+	status = "okay";
+
+	tpm@1 {
+		compatible = "infineon,slb9670", "tcg,tpm_tis-spi";
+		reg = <1>;
+		spi-max-frequency = <18500000>;
+	};
+};
+
+/* EEPROM on Zinnia */
+&eeprom_carrier_board {
+	status = "okay";
+};
+
+/* Verdin ETH_1 */
+&eqos {
+	status = "okay";
+};
+
+/* Verdin ETH_2 */
+&fec {
+	phy-handle = <&ethphy2>;
+	phy-mode = "rgmii-id";
+
+	status = "okay";
+};
+
+&verdin_eth2_mdio {
+	ethphy2: ethernet-phy@2 {
+		reg = <2>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <18 IRQ_TYPE_LEVEL_LOW>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+	};
+};
+
+/* Verdin CAN_1 */
+&flexcan1 {
+	status = "okay";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"DO1_EN", /* 0 */ /* SODIMM 206 */
+		"DI1_EN", /* SODIMM 208 */
+		"",
+		"",
+		"",
+		"DI1_RB", /* SODIMM 210 */
+		"DO2_EN", /* SODIMM 212 */
+		"DI2_RB", /* SODIMM 216 */
+		"DI2_EN", /* SODIMM 218 */
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"";
+};
+
+&gpio2 {
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		""; /* 20 */
+};
+
+&gpio3 {
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"DI3_RB", /* SODIMM 56 */
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"";
+};
+
+&gpio4 {
+	gpio-line-names =
+		"", /* 0 */
+		"DI3_EN", /* SODIMM 222 */
+		"",
+		"DO3_EN", /* SODIMM 220 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 30 */
+		"";
+};
+
+&gpio5 {
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"";
+};
+
+/* Temperature sensor on Zinnia */
+&hwmon_temp {
+	compatible = "ti,tmp1075";
+
+	status = "okay";
+};
+
+/* Verdin I2C_1 */
+&i2c4 {
+	status = "okay";
+};
+
+/* Verdin UART_1 */
+&uart1 {
+	status = "okay";
+};
+
+/* Verdin UART_2 */
+&uart2 {
+	pinctrl-0 = <&pinctrl_uart2>, <&pinctrl_uart2_rts>;
+	linux,rs485-enabled-at-boot-time;
+	rs485-rx-during-tx;
+
+	status = "okay";
+};
+
+/* Verdin UART_3 */
+&uart3 {
+	status = "okay";
+};
+
+/* Verdin USB_1 */
+&usb3_0 {
+	status = "okay";
+};
+
+&usb3_phy0 {
+	status = "okay";
+};
+
+/* Verdin USB_2 */
+&usb3_1 {
+	status = "okay";
+};
+
+&usb3_phy1 {
+	status = "okay";
+};
+
+/* Verdin SD_1 */
+&usdhc2 {
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio1>,
+		    <&pinctrl_gpio2>,
+		    <&pinctrl_gpio3>,
+		    <&pinctrl_gpio4>,
+		    <&pinctrl_gpio5>,
+		    <&pinctrl_gpio6>,
+		    <&pinctrl_gpio7>,
+		    <&pinctrl_gpio8>,
+		    <&pinctrl_qspi1_io0_gpio>;
+
+	pinctrl_uart2_cts_gpio: gpio2io6grp {
+		fsl,pins = <MX8MP_IOMUXC_SD1_DATA4__GPIO2_IO06	0x184>;	/* SODIMM 143 */
+	};
+
+	pinctrl_qspi1_io0_gpio: gpio3io6grp {
+		fsl,pins = <MX8MP_IOMUXC_NAND_DATA00__GPIO3_IO06	0x184>;	/* SODIMM 56 */
+	};
+
+	pinctrl_zinnia_leds: zinnialedsgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SAI5_MCLK__GPIO3_IO25	0x16>, /* SODIMM 30 */
+			<MX8MP_IOMUXC_SAI5_RXD1__GPIO3_IO22	0x16>, /* SODIMM 32 */
+			<MX8MP_IOMUXC_SAI5_RXFS__GPIO3_IO19	0x16>, /* SODIMM 34 */
+			<MX8MP_IOMUXC_SAI1_RXD0__GPIO4_IO02	0x16>, /* SODIMM 36 */
+			<MX8MP_IOMUXC_SAI3_TXFS__GPIO4_IO31	0x16>, /* SODIMM 44 */
+			<MX8MP_IOMUXC_SAI3_TXD__GPIO5_IO01	0x16>, /* SODIMM 46 */
+			<MX8MP_IOMUXC_SAI3_RXD__GPIO4_IO30	0x16>, /* SODIMM 48 */
+			<MX8MP_IOMUXC_NAND_CE0_B__GPIO3_IO01	0x16>; /* SODIMM 54 */
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi b/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi
index d31f8082394fd..9fee2cf9ef546 100644
--- a/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8mp-verdin.dtsi
@@ -846,7 +846,7 @@ &uart1 {
 /* Verdin UART_2 */
 &uart2 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart2>;
+	pinctrl-0 = <&pinctrl_uart2>, <&pinctrl_uart2_cts>, <&pinctrl_uart2_rts>;
 	uart-has-rtscts;
 };
 
@@ -1277,10 +1277,18 @@ pinctrl_uart1: uart1grp {
 			<MX8MP_IOMUXC_UART1_TXD__UART1_DCE_TX		0x1c4>;	/* SODIMM 131 */
 	};
 
+	pinctrl_uart2_cts: uart2ctsgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SD1_DATA4__UART2_DCE_RTS		0x1c4>;	/* SODIMM 143 */
+	};
+
+	pinctrl_uart2_rts: uart2rtsgrp {
+		fsl,pins =
+			<MX8MP_IOMUXC_SD1_DATA5__UART2_DCE_CTS		0x1c4>;	/* SODIMM 141 */
+	};
+
 	pinctrl_uart2: uart2grp {
 		fsl,pins =
-			<MX8MP_IOMUXC_SD1_DATA4__UART2_DCE_RTS		0x1c4>,	/* SODIMM 143 */
-			<MX8MP_IOMUXC_SD1_DATA5__UART2_DCE_CTS		0x1c4>,	/* SODIMM 141 */
 			<MX8MP_IOMUXC_UART2_RXD__UART2_DCE_RX		0x1c4>,	/* SODIMM 137 */
 			<MX8MP_IOMUXC_UART2_TXD__UART2_DCE_TX		0x1c4>; /* SODIMM 139 */
 	};
diff --git a/arch/arm64/boot/dts/freescale/imx8qm-tqma8qm-mba8x.dts b/arch/arm64/boot/dts/freescale/imx8qm-tqma8qm-mba8x.dts
new file mode 100644
index 0000000000000..ab3b244b684fd
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8qm-tqma8qm-mba8x.dts
@@ -0,0 +1,871 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Copyright (c) 2019-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * D-82229 Seefeld, Germany.
+ * Author: Alexander Stein
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/input/linux-event-codes.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/pwm/pwm.h>
+
+#include "imx8qm-tqma8qm.dtsi"
+
+/ {
+	model = "TQ-Systems i.MX8QM TQMa8QM on MBa8x";
+	compatible = "tq,imx8qm-tqma8qm-mba8x", "tq,imx8qm-tqma8qm", "fsl,imx8qm";
+
+	adc {
+		compatible = "iio-hwmon";
+		io-channels = <&adc0 0>, <&adc0 1>, <&adc0 2>;
+	};
+
+	aliases {
+		rtc0 = &pcf85063;
+		rtc1 = &rtc;
+	};
+
+	chosen {
+		stdout-path = &lpuart0;
+	};
+
+
+	clk_xtal25: clk-xtal25 {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpiokeys>;
+		autorepeat;
+
+		switch-1 {
+			label = "SWITCH_A";
+			linux,code = <BTN_0>;
+			gpios = <&lsio_gpio2 11 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		switch-2 {
+			label = "SWITCH_B";
+			linux,code = <BTN_1>;
+			gpios = <&lsio_gpio1 0 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_gpioled>;
+
+		user-led0 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_HEARTBEAT;
+			gpios = <&lsio_gpio5 20 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+
+		user-led1 {
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_STATUS;
+			gpios = <&lsio_gpio5 19 GPIO_ACTIVE_HIGH>;
+			default-state = "on";
+		};
+	};
+
+	fan0: pwm-fan {
+		compatible = "pwm-fan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pwmfan>;
+		fan-supply = <&reg_pwm_fan>;
+		#cooling-cells = <2>;
+		/* typical 25 kHz -> 40.000 nsec */
+		pwms = <&lsio_pwm3 0 40000 PWM_POLARITY_INVERTED>;
+		cooling-levels = <0 32 64 128 196 240>;
+		pulses-per-revolution = <2>;
+		interrupt-parent = <&lsio_gpio2>;
+		interrupts = <20 IRQ_TYPE_EDGE_FALLING>;
+		status = "disabled";
+	};
+
+	reg_mba8x_12v: regulator-mba8x-12v {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_mba8x_12v>;
+		regulator-name = "MBa8x-V12";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		enable-active-high;
+		gpio = <&lsio_gpio1 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	reg_mba8x_v3v3: regulator-mba8x-v3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "V_3V3_MB";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_pwm_fan: regulator-pwm-fan {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_regpwmfan>;
+		regulator-name = "FAN_PWR";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		gpio = <&lsio_gpio2 12 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		vin-supply = <&reg_mba8x_12v>;
+	};
+
+	reg_usb_phy: regulator-usb-phy {
+		compatible = "regulator-fixed";
+		regulator-max-microvolt = <3000000>;
+		regulator-min-microvolt = <3000000>;
+		regulator-name = "usb-phy-dummy";
+	};
+
+	reg_v1v5_pcie: regulator-v1v5-pcie {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_pcie_v1v5>;
+		regulator-name = "V_1V5_MPCIE";
+		regulator-min-microvolt = <1500000>;
+		regulator-max-microvolt = <1500000>;
+		vin-supply = <&reg_mba8x_v3v3>;
+		enable-active-high;
+		gpio = <&lsio_gpio0 31 GPIO_ACTIVE_HIGH>;
+		regulator-always-on;
+	};
+
+	reg_v1v8: regulator-v1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "MBa8x-V1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	reg_v3v3_pcie: regulator-v3v3-pcie {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_pcie_v3v3>;
+		regulator-name = "V_3V3_MPCIE";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&reg_mba8x_v3v3>;
+		enable-active-high;
+		gpio = <&lsio_gpio1 1 GPIO_ACTIVE_HIGH>;
+		regulator-always-on;
+	};
+
+	reg_v3v3_sd: regulator-v3v3-sd {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_reg_v3v3_sd>;
+		regulator-name = "V3V3_SD";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&reg_mba8x_v3v3>;
+		gpio = <&lsio_gpio4 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-boot-on;
+		off-on-delay-us = <200000>;
+	};
+
+	reg_vref_v1v8: regulator-vref-v1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "VREF_V1V8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		/*
+		 * global autoconfigured region for contiguous allocations
+		 * must not exceed memory size and region
+		 */
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0 0x3c000000>;
+			alloc-ranges = <0 0x96000000 0 0x3c000000>;
+			linux,cma-default;
+		};
+
+		decoder_boot: decoder-boot@84000000 {
+			reg = <0 0x84000000 0 0x2000000>;
+			no-map;
+		};
+
+		encoder1_boot: encoder1-boot@86000000 {
+			reg = <0 0x86000000 0 0x200000>;
+			no-map;
+		};
+
+		encoder2_boot: encoder2-boot@86200000 {
+			reg = <0 0x86200000 0 0x200000>;
+			no-map;
+		};
+
+		decoder_rpc: decoder-rpc@92000000 {
+			reg = <0 0x92000000 0 0x100000>;
+			no-map;
+		};
+
+		encoder1_rpc: encoder1-rpc@92100000 {
+			reg = <0 0x92100000 0 0x700000>;
+			no-map;
+		};
+
+		encoder2_rpc: encoder1-rpc@92800000 {
+			reg = <0 0x92800000 0 0x700000>;
+			no-map;
+		};
+	};
+
+	sound {
+		compatible = "fsl,imx-audio-tlv320aic32x4";
+		model = "tqm-tlv320aic32";
+		ssi-controller = <&sai1>;
+		audio-codec = <&tlv320aic3x04>;
+		audio-routing =
+			"IN3_L", "Mic Jack",
+			"Mic Jack", "Mic Bias",
+			"IN1_L", "Line In Jack",
+			"IN1_R", "Line In Jack",
+			"Line Out Jack", "LOL",
+			"Line Out Jack", "LOR";
+	};
+
+	sound-hdmi {
+		compatible = "fsl,imx-audio-hdmi";
+		model = "imx-audio-dp";
+		audio-cpu = <&sai5>;
+		hdmi-out;
+	};
+
+	thermal-zones {
+		cpu0-thermal {
+			trips {
+				soc_active0_0: trip-active0 {
+					temperature = <40000>;
+					hysteresis = <5000>;
+					type = "active";
+				};
+
+				soc_active0_1: trip-active1 {
+					temperature = <48000>;
+					hysteresis = <3000>;
+					type = "active";
+				};
+
+				soc_active0_2: trip-active2 {
+					temperature = <60000>;
+					hysteresis = <10000>;
+					type = "active";
+				};
+			};
+
+			cooling-maps {
+				map1 {
+					trip = <&soc_active0_0>;
+					cooling-device = <&fan0 1 1>;
+				};
+
+				map2 {
+					trip = <&soc_active0_1>;
+					cooling-device = <&fan0 2 2>;
+				};
+
+				map3 {
+					trip = <&soc_active0_2>;
+					cooling-device = <&fan0 3 3>;
+				};
+			};
+		};
+	};
+};
+
+&fec1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_fec1>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy0>;
+	nvmem-cells = <&fec_mac0>;
+	nvmem-cell-names = "mac-address";
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_ethphy0>;
+			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+			ti,dp83867-rxctrl-strap-quirk;
+			ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
+			reset-gpios = <&lsio_gpio2 6 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <500000>;
+			reset-deassert-us = <50000>;
+			enet-phy-lane-no-swap;
+			interrupt-parent = <&lsio_gpio2>;
+			interrupts = <8 IRQ_TYPE_LEVEL_LOW>;
+		};
+	};
+};
+
+&fec2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_fec2>;
+	phy-mode = "rgmii-id";
+	phy-handle = <&ethphy3>;
+	nvmem-cells = <&fec_mac1>;
+	nvmem-cell-names = "mac-address";
+	status = "okay";
+
+	mdio {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy3: ethernet-phy@3 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <3>;
+			pinctrl-names = "default";
+			pinctrl-0 = <&pinctrl_ethphy3>;
+			ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_25_NS>;
+			ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+			ti,dp83867-rxctrl-strap-quirk;
+			ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
+			reset-gpios = <&lsio_gpio2 4 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <500000>;
+			reset-deassert-us = <50000>;
+			enet-phy-lane-no-swap;
+		};
+	};
+};
+
+&flexcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	status = "okay";
+};
+
+&flexcan2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan2>;
+	status = "okay";
+};
+
+&hsio_phy {
+	fsl,hsio-cfg = "pciea-pcieb-sata";
+	fsl,refclk-pad-mode = "input";
+	status = "okay";
+};
+
+/* no refclock gating */
+&hsio_refa_clk {
+	compatible = "fixed-factor-clock";
+	clocks = <&pcieclk 0>;
+	clock-div = <1>;
+	clock-mult = <1>;
+	/delete-property/ enable-gpios;
+};
+
+&hsio_refb_clk {
+	compatible = "fixed-factor-clock";
+	clocks = <&pcieclk 0>;
+	clock-div = <1>;
+	clock-mult = <1>;
+	/delete-property/ enable-gpios;
+};
+
+&i2c1 {
+	tlv320aic3x04: audio-codec@18 {
+		compatible = "ti,tlv320aic32x4";
+		reg = <0x18>;
+		clocks = <&mclkout0_lpcg IMX_LPCG_CLK_0>;
+		clock-names = "mclk";
+		assigned-clocks = <&clk IMX_SC_R_AUDIO_PLL_0 IMX_SC_PM_CLK_PLL>,
+				<&clk IMX_SC_R_AUDIO_PLL_0 IMX_SC_PM_CLK_SLV_BUS>,
+				<&clk IMX_SC_R_AUDIO_PLL_0 IMX_SC_PM_CLK_MST_BUS>,
+				<&mclkout0_lpcg IMX_LPCG_CLK_0>;
+		assigned-clock-rates = <786432000>, <49152000>, <12288000>, <12288000>;
+		ldoin-supply = <&reg_mba8x_v3v3>;
+		iov-supply = <&reg_v1v8>;
+	};
+
+	sensor1: temperature-sensor@1c {
+		compatible = "nxp,se97b", "jedec,jc-42.4-temp";
+		reg = <0x1c>;
+	};
+
+	eeprom2: eeprom@54 {
+		compatible = "nxp,se97b", "atmel,24c02";
+		reg = <0x54>;
+		pagesize = <16>;
+		vcc-supply = <&reg_mba8x_v3v3>;
+	};
+
+	pcieclk: clock-generator@68 {
+		compatible = "renesas,9fgv0441";
+		reg = <0x68>;
+		clocks = <&clk_xtal25>;
+		#clock-cells = <1>;
+	};
+};
+
+&lpspi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpspi0 &pinctrl_lpspi0_cs>;
+	cs-gpios = <&lsio_gpio3 5 GPIO_ACTIVE_LOW>, <&lsio_gpio3 6 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&lpspi1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpspi1 &pinctrl_lpspi1_cs>;
+	cs-gpios = <&lsio_gpio3 24 GPIO_ACTIVE_LOW>, <&lsio_gpio3 25 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&lpspi2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpspi2 &pinctrl_lpspi2_cs>;
+	cs-gpios = <&lsio_gpio3 10 GPIO_ACTIVE_LOW>, <&lsio_gpio3 11 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&lpuart0 {
+	/* console */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpuart0>;
+	status = "okay";
+};
+
+&lpuart1 {
+	/* X62 pin header */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpuart1>;
+	status = "okay";
+};
+
+&lpuart2 {
+	/* mikroBUS */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpuart2>;
+	status = "okay";
+};
+
+&lsio_gpio2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio2>;
+	gpio-line-names = "", "", "", "",
+			  "", "PCIE0_DISABLE#", "", "";
+
+	pcie0-wdisable1-hog {
+		gpio-hog;
+		gpios = <5 0>;
+		output-high;
+		line-name = "PCIE0_DISABLE#";
+	};
+
+	pcie-clk-pd-hog {
+		gpio-hog;
+		gpios = <10 0>;
+		output-high;
+		line-name = "PCIE_CLK_PD#";
+	};
+};
+
+&lsio_pwm3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lsio_pwm3>;
+	status = "okay";
+};
+
+&pciea {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pciea>;
+	phys = <&hsio_phy 0 PHY_TYPE_PCIE 0>;
+	phy-names = "pcie-phy";
+	reset-gpio = <&lsio_gpio4 29 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&pcieb {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_pcieb>;
+	phys = <&hsio_phy 1 PHY_TYPE_PCIE 1>;
+	phy-names = "pcie-phy";
+	reset-gpio = <&lsio_gpio5 0 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+&sai1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_sai1>;
+	assigned-clocks = <&clk IMX_SC_R_AUDIO_PLL_0 IMX_SC_PM_CLK_PLL>,
+			  <&clk IMX_SC_R_AUDIO_PLL_0 IMX_SC_PM_CLK_SLV_BUS>,
+			  <&clk IMX_SC_R_AUDIO_PLL_0 IMX_SC_PM_CLK_MST_BUS>,
+			  <&sai1_lpcg IMX_LPCG_CLK_0>;
+	assigned-clock-rates = <786432000>, <49152000>, <12288000>, <49152000>;
+	status = "okay";
+};
+
+&sai5 {
+	status = "okay";
+};
+
+&sai5_lpcg {
+	status = "okay";
+};
+
+&sata {
+	status = "okay";
+};
+
+&usbphy1 {
+	phy-3p0-supply = <&reg_usb_phy>;
+	status = "okay";
+};
+
+&usbotg1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usbotg1>;
+	srp-disable;
+	hnp-disable;
+	adp-disable;
+	power-active-high;
+	over-current-active-low;
+	dr_mode = "otg";
+	status = "okay";
+};
+
+&usb3_phy {
+	status = "okay";
+};
+
+&usbotg3 {
+	/* over-current disabled by default */
+	status = "okay";
+};
+
+&usbotg3_cdns3 {
+	dr_mode = "host";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_usbhub>;
+	status = "okay";
+
+	hub_2_0: hub@1 {
+		compatible = "usb451,8142";
+		reg = <1>;
+		peer-hub = <&hub_3_0>;
+		reset-gpios = <&lsio_gpio2 7 GPIO_ACTIVE_LOW>;
+		vdd-supply = <&reg_mba8x_v3v3>;
+	};
+
+	hub_3_0: hub@2 {
+		compatible = "usb451,8140";
+		reg = <2>;
+		peer-hub = <&hub_2_0>;
+		reset-gpios = <&lsio_gpio2 7 GPIO_ACTIVE_LOW>;
+		vdd-supply = <&reg_mba8x_v3v3>;
+	};
+};
+
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	bus-width = <4>;
+	cd-gpios = <&lsio_gpio5 22 GPIO_ACTIVE_LOW>;
+	wp-gpios = <&lsio_gpio5 21 GPIO_ACTIVE_HIGH>;
+	vmmc-supply = <&reg_v3v3_sd>;
+	no-mmc;
+	no-sdio;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio>;
+
+	pinctrl_adc0: adc0grp {
+		fsl,pins = <IMX8QM_ADC_IN1_DMA_ADC0_IN1		0x02000060>,
+			   <IMX8QM_ADC_IN2_DMA_ADC0_IN2		0x02000060>;
+	};
+
+	pinctrl_ethphy0: ethphy0grp {
+		fsl,pins = <IMX8QM_ESAI1_SCKR_LSIO_GPIO2_IO06	0x00000041>,
+			   <IMX8QM_ESAI1_TX0_LSIO_GPIO2_IO08	0x00000021>;
+	};
+
+	pinctrl_ethphy3: ethphy3grp {
+		fsl,pins = <IMX8QM_ESAI1_FSR_LSIO_GPIO2_IO04	0x00000041>;
+	};
+
+	pinctrl_fec1: fec1grp {
+		fsl,pins = <IMX8QM_COMP_CTL_GPIO_1V8_3V3_ENET_ENETB_PAD		0x000014a0>,
+			   <IMX8QM_ENET0_MDC_CONN_ENET0_MDC			0x06000041>,
+			   <IMX8QM_ENET0_MDIO_CONN_ENET0_MDIO			0x06000041>,
+			   <IMX8QM_ENET0_RGMII_TX_CTL_CONN_ENET0_RGMII_TX_CTL	0x00000041>,
+			   <IMX8QM_ENET0_RGMII_TXC_CONN_ENET0_RGMII_TXC		0x00000041>,
+			   <IMX8QM_ENET0_RGMII_TXD0_CONN_ENET0_RGMII_TXD0	0x00000041>,
+			   <IMX8QM_ENET0_RGMII_TXD1_CONN_ENET0_RGMII_TXD1	0x00000041>,
+			   <IMX8QM_ENET0_RGMII_TXD2_CONN_ENET0_RGMII_TXD2	0x00000041>,
+			   <IMX8QM_ENET0_RGMII_TXD3_CONN_ENET0_RGMII_TXD3	0x00000041>,
+			   <IMX8QM_ENET0_RGMII_RXC_CONN_ENET0_RGMII_RXC		0x00000040>,
+			   <IMX8QM_ENET0_RGMII_RX_CTL_CONN_ENET0_RGMII_RX_CTL	0x00000040>,
+			   <IMX8QM_ENET0_RGMII_RXD0_CONN_ENET0_RGMII_RXD0	0x00000040>,
+			   <IMX8QM_ENET0_RGMII_RXD1_CONN_ENET0_RGMII_RXD1	0x00000040>,
+			   <IMX8QM_ENET0_RGMII_RXD2_CONN_ENET0_RGMII_RXD2	0x00000040>,
+			   <IMX8QM_ENET0_RGMII_RXD3_CONN_ENET0_RGMII_RXD3	0x00000040>;
+	};
+
+	pinctrl_fec2: fec2grp {
+		fsl,pins = <IMX8QM_COMP_CTL_GPIO_1V8_3V3_ENET_ENETA_PAD		0x000014a0>,
+			   <IMX8QM_ENET1_MDC_CONN_ENET1_MDC			0x06000041>,
+			   <IMX8QM_ENET1_MDIO_CONN_ENET1_MDIO			0x06000041>,
+			   <IMX8QM_ENET1_RGMII_TX_CTL_CONN_ENET1_RGMII_TX_CTL	0x00000041>,
+			   <IMX8QM_ENET1_RGMII_TXC_CONN_ENET1_RGMII_TXC		0x00000041>,
+			   <IMX8QM_ENET1_RGMII_TXD0_CONN_ENET1_RGMII_TXD0	0x00000041>,
+			   <IMX8QM_ENET1_RGMII_TXD1_CONN_ENET1_RGMII_TXD1	0x00000041>,
+			   <IMX8QM_ENET1_RGMII_TXD2_CONN_ENET1_RGMII_TXD2	0x00000041>,
+			   <IMX8QM_ENET1_RGMII_TXD3_CONN_ENET1_RGMII_TXD3	0x00000041>,
+			   <IMX8QM_ENET1_RGMII_RXC_CONN_ENET1_RGMII_RXC		0x00000040>,
+			   <IMX8QM_ENET1_RGMII_RX_CTL_CONN_ENET1_RGMII_RX_CTL	0x00000040>,
+			   <IMX8QM_ENET1_RGMII_RXD0_CONN_ENET1_RGMII_RXD0	0x00000040>,
+			   <IMX8QM_ENET1_RGMII_RXD1_CONN_ENET1_RGMII_RXD1	0x00000040>,
+			   <IMX8QM_ENET1_RGMII_RXD2_CONN_ENET1_RGMII_RXD2	0x00000040>,
+			   <IMX8QM_ENET1_RGMII_RXD3_CONN_ENET1_RGMII_RXD3	0x00000040>;
+	};
+
+	pinctrl_flexcan1: flexcan0grp {
+		fsl,pins = <IMX8QM_FLEXCAN0_TX_DMA_FLEXCAN0_TX		0x00000021>,
+			   <IMX8QM_FLEXCAN0_RX_DMA_FLEXCAN0_RX		0x00000021>;
+	};
+
+	pinctrl_flexcan2: flexcan1grp {
+		fsl,pins = <IMX8QM_FLEXCAN1_TX_DMA_FLEXCAN1_TX		0x00000021>,
+			   <IMX8QM_FLEXCAN1_RX_DMA_FLEXCAN1_RX		0x00000021>;
+	};
+
+	pinctrl_gpio2: gpio2grp {
+		fsl,pins = <IMX8QM_ESAI1_FST_LSIO_GPIO2_IO05		0x00000021>,
+			   <IMX8QM_ESAI1_TX2_RX3_LSIO_GPIO2_IO10	0x00000021>;
+	};
+
+	pinctrl_gpio: gpiogrp {
+		fsl,pins = /* GPIO0_05 on X62:26 */
+			   <IMX8QM_SIM0_GPIO0_00_LSIO_GPIO0_IO05	0x00000021>,
+			   /* GPIO1_14 on X64:21 */
+			   <IMX8QM_LVDS1_I2C1_SCL_LSIO_GPIO1_IO14	0x00000021>,
+			   /* GPIO1_15 on X64:23 */
+			   <IMX8QM_LVDS1_I2C1_SDA_LSIO_GPIO1_IO15	0x00000021>,
+			   /* GPIO2_17 on X63:37 */
+			   <IMX8QM_SPI3_SCK_LSIO_GPIO2_IO17		0x00000021>,
+			   /* GPIO2_21 on X63:39 */
+			   <IMX8QM_SPI3_CS1_LSIO_GPIO2_IO21		0x00000021>,
+			   /* GPIO4_12 on X61:24 */
+			   <IMX8QM_USDHC2_CD_B_LSIO_GPIO4_IO12		0x00000021>,
+			   /* GPIO4_11 on X61:26 */
+			   <IMX8QM_USDHC2_WP_LSIO_GPIO4_IO11		0x00000021>,
+			   /* GPIO4_10 on X61:28 */
+			   <IMX8QM_USDHC2_VSELECT_LSIO_GPIO4_IO10	0x00000021>,
+			   /* GPIO4_09 on X61:30 */
+			   <IMX8QM_USDHC2_RESET_B_LSIO_GPIO4_IO09	0x00000021>,
+			   /* GPIO5_23 on X62:24 */
+			   <IMX8QM_USDHC1_STROBE_LSIO_GPIO5_IO23	0x00000021>,
+			   /* GPIO5_24 on X61:15 */
+			   <IMX8QM_USDHC2_CLK_LSIO_GPIO5_IO24		0x00000021>,
+			   /* GPIO5_25 on X61:17 */
+			   <IMX8QM_USDHC2_CMD_LSIO_GPIO5_IO25		0x00000021>,
+			   /* GPIO5_26 on X61:19 */
+			   <IMX8QM_USDHC2_DATA0_LSIO_GPIO5_IO26		0x00000021>,
+			   /* GPIO5_27 on X61:21 */
+			   <IMX8QM_USDHC2_DATA1_LSIO_GPIO5_IO27		0x00000021>,
+			   /* GPIO5_28 on X61:23 */
+			   <IMX8QM_USDHC2_DATA2_LSIO_GPIO5_IO28		0x00000021>,
+			   /* GPIO5_29 on X61:25 */
+			   <IMX8QM_USDHC2_DATA3_LSIO_GPIO5_IO29		0x00000021>;
+	};
+
+	pinctrl_gpiokeys: gpiokeysgrp {
+		fsl,pins = <IMX8QM_ESAI1_TX3_RX2_LSIO_GPIO2_IO11	0x00000021>,
+			   <IMX8QM_SCU_GPIO0_04_LSIO_GPIO1_IO00		0x00000021>;
+	};
+
+	pinctrl_gpioled: gpioledgrp {
+		fsl,pins = <IMX8QM_USDHC1_DATA4_LSIO_GPIO5_IO19		0x00000021>,
+			   <IMX8QM_USDHC1_DATA5_LSIO_GPIO5_IO20		0x00000021>;
+	};
+
+	pinctrl_lpspi0_cs: lpspi0csgrp {
+		fsl,pins = <IMX8QM_SPI0_CS0_LSIO_GPIO3_IO05		0x00000021>,
+			   <IMX8QM_SPI0_CS1_LSIO_GPIO3_IO06		0x00000021>;
+	};
+
+	pinctrl_lpspi0: lpspi0grp {
+		fsl,pins = <IMX8QM_SPI0_SCK_DMA_SPI0_SCK		0x0600004d>,
+			   <IMX8QM_SPI0_SDO_DMA_SPI0_SDO		0x0600004d>,
+			   <IMX8QM_SPI0_SDI_DMA_SPI0_SDI		0x0600004d>;
+	};
+
+	pinctrl_lpspi1_cs: lpspi1csgrp {
+		fsl,pins = <IMX8QM_ADC_IN6_LSIO_GPIO3_IO24		0x00000021>,
+			   <IMX8QM_ADC_IN7_LSIO_GPIO3_IO25		0x00000021>;
+	};
+
+	pinctrl_lpspi1: lpspi1grp {
+		fsl,pins = <IMX8QM_ADC_IN3_DMA_SPI1_SCK			0x0600004d>,
+			   <IMX8QM_ADC_IN4_DMA_SPI1_SDO			0x0600004d>,
+			   <IMX8QM_ADC_IN5_DMA_SPI1_SDI			0x0600004d>;
+	};
+
+	pinctrl_lpspi2: lpspi2grp {
+		fsl,pins = <IMX8QM_SPI2_SCK_DMA_SPI2_SCK		0x0600004d>,
+			   <IMX8QM_SPI2_SDO_DMA_SPI2_SDO		0x0600004d>,
+			   <IMX8QM_SPI2_SDI_DMA_SPI2_SDI		0x0600004d>;
+	};
+
+	pinctrl_lpspi2_cs: lpspi2sgrp {
+		fsl,pins = <IMX8QM_SPI2_CS0_LSIO_GPIO3_IO10		0x00000021>,
+			   <IMX8QM_SPI2_CS1_LSIO_GPIO3_IO11		0x00000021>;
+	};
+
+	pinctrl_lpuart0: lpuart0grp {
+		fsl,pins = <IMX8QM_UART0_RX_DMA_UART0_RX		0x06000021>,
+			   <IMX8QM_UART0_TX_DMA_UART0_TX		0x06000021>,
+			   <IMX8QM_UART0_CTS_B_DMA_UART0_CTS_B		0x00000021>,
+			   <IMX8QM_UART0_RTS_B_DMA_UART0_RTS_B		0x00000021>;
+	};
+
+	pinctrl_lpuart1: lpuart1grp {
+		fsl,pins = <IMX8QM_UART1_RX_DMA_UART1_RX		0x06000021>,
+			   <IMX8QM_UART1_TX_DMA_UART1_TX		0x06000021>,
+			   <IMX8QM_UART1_CTS_B_DMA_UART1_CTS_B		0x00000021>,
+			   <IMX8QM_UART1_RTS_B_DMA_UART1_RTS_B		0x00000021>;
+	};
+
+	pinctrl_lpuart2: lpuart2grp {
+		fsl,pins = <IMX8QM_LVDS0_I2C1_SDA_DMA_UART2_RX		0x06000021>,
+			   <IMX8QM_LVDS0_I2C1_SCL_DMA_UART2_TX		0x06000021>;
+	};
+
+	pinctrl_lsio_pwm3: lsiopwm3grp {
+		fsl,pins = <IMX8QM_GPT0_COMPARE_LSIO_PWM3_OUT		0x00000021>;
+	};
+
+	pinctrl_pciea: pcieagrp {
+		fsl,pins = <IMX8QM_PCIE_CTRL0_PERST_B_LSIO_GPIO4_IO29		0x06000021>,
+			   <IMX8QM_PCIE_CTRL0_CLKREQ_B_HSIO_PCIE0_CLKREQ_B	0x06000021>,
+			   <IMX8QM_PCIE_CTRL0_WAKE_B_LSIO_GPIO4_IO28		0x04000021>;
+	};
+
+	pinctrl_pcieb: pciebgrp {
+		fsl,pins = <IMX8QM_PCIE_CTRL1_PERST_B_LSIO_GPIO5_IO00		0x06000021>,
+			   <IMX8QM_PCIE_CTRL1_CLKREQ_B_HSIO_PCIE1_CLKREQ_B	0x06000021>,
+			   <IMX8QM_PCIE_CTRL1_WAKE_B_LSIO_GPIO4_IO31		0x04000021>;
+	};
+
+	pinctrl_pwmfan: pwmfangrp {
+		fsl,pins = <IMX8QM_SPI3_CS0_LSIO_GPIO2_IO20		0x30>;
+	};
+
+	pinctrl_reg_mba8x_12v: regmba12vgrp {
+		fsl,pins = <IMX8QM_SCU_GPIO0_06_LSIO_GPIO1_IO02		0x00000021>;
+	};
+
+	pinctrl_reg_pcie_v1v5: regpcie1v5grp {
+		fsl,pins = <IMX8QM_SCU_GPIO0_03_LSIO_GPIO0_IO31		0x00000021>;
+	};
+
+	pinctrl_reg_pcie_v3v3: regpcie3v3grp {
+		fsl,pins = <IMX8QM_SCU_GPIO0_05_LSIO_GPIO1_IO01		0x00000021>;
+	};
+
+	pinctrl_regpwmfan: regpwmfangrp {
+		fsl,pins = <IMX8QM_ESAI1_TX4_RX1_LSIO_GPIO2_IO12	0x00000021>;
+	};
+
+	pinctrl_reg_v3v3_sd: regv3v3sdgrp {
+		fsl,pins = <IMX8QM_USDHC1_RESET_B_LSIO_GPIO4_IO07	0x00000021>;
+	};
+
+	pinctrl_sai1: sai1grp {
+		fsl,pins = <IMX8QM_SAI1_RXD_AUD_SAI1_RXD		0x06000041>,
+			   <IMX8QM_SAI1_RXC_AUD_SAI1_RXC		0x06000041>,
+			   <IMX8QM_SAI1_RXFS_AUD_SAI1_RXFS		0x06000041>,
+			   <IMX8QM_SAI1_TXD_AUD_SAI1_TXD		0x06000061>,
+			   <IMX8QM_SAI1_TXC_AUD_SAI1_TXC		0x06000041>,
+			   <IMX8QM_MCLK_OUT0_AUD_ACM_MCLK_OUT0		0x0600004d>;
+	};
+
+	pinctrl_usbhub: usbhubgrp {
+		fsl,pins = <IMX8QM_ESAI1_SCKT_LSIO_GPIO2_IO07		0x00000021>;
+	};
+
+	pinctrl_usbotg1: usbotg1grp {
+		fsl,pins = <IMX8QM_USB_SS3_TC2_CONN_USB_OTG1_OC		0x00000021>,
+			   <IMX8QM_USB_SS3_TC0_CONN_USB_OTG1_PWR	0x00000021>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <IMX8QM_USDHC1_CLK_CONN_USDHC1_CLK		0x06000041>,
+			   <IMX8QM_USDHC1_CMD_CONN_USDHC1_CMD		0x00000021>,
+			   <IMX8QM_USDHC1_DATA0_CONN_USDHC1_DATA0	0x00000021>,
+			   <IMX8QM_USDHC1_DATA1_CONN_USDHC1_DATA1	0x00000021>,
+			   <IMX8QM_USDHC1_DATA2_CONN_USDHC1_DATA2	0x00000021>,
+			   <IMX8QM_USDHC1_DATA3_CONN_USDHC1_DATA3	0x00000021>,
+			   <IMX8QM_USDHC1_VSELECT_CONN_USDHC1_VSELECT	0x00000021>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <IMX8QM_USDHC1_CLK_CONN_USDHC1_CLK		0x06000041>,
+			   <IMX8QM_USDHC1_CMD_CONN_USDHC1_CMD		0x00000021>,
+			   <IMX8QM_USDHC1_DATA0_CONN_USDHC1_DATA0	0x00000021>,
+			   <IMX8QM_USDHC1_DATA1_CONN_USDHC1_DATA1	0x00000021>,
+			   <IMX8QM_USDHC1_DATA2_CONN_USDHC1_DATA2	0x00000021>,
+			   <IMX8QM_USDHC1_DATA3_CONN_USDHC1_DATA3	0x00000021>,
+			   <IMX8QM_USDHC1_VSELECT_CONN_USDHC1_VSELECT	0x00000021>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <IMX8QM_USDHC1_CLK_CONN_USDHC1_CLK		0x06000041>,
+			   <IMX8QM_USDHC1_CMD_CONN_USDHC1_CMD		0x00000021>,
+			   <IMX8QM_USDHC1_DATA0_CONN_USDHC1_DATA0	0x00000021>,
+			   <IMX8QM_USDHC1_DATA1_CONN_USDHC1_DATA1	0x00000021>,
+			   <IMX8QM_USDHC1_DATA2_CONN_USDHC1_DATA2	0x00000021>,
+			   <IMX8QM_USDHC1_DATA3_CONN_USDHC1_DATA3	0x00000021>,
+			   <IMX8QM_USDHC1_VSELECT_CONN_USDHC1_VSELECT	0x00000021>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2-gpiogrp {
+		fsl,pins = <IMX8QM_USDHC1_DATA6_LSIO_GPIO5_IO21		0x00000021>,
+			   <IMX8QM_USDHC1_DATA7_LSIO_GPIO5_IO22		0x00000021>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8qm-tqma8qm.dtsi b/arch/arm64/boot/dts/freescale/imx8qm-tqma8qm.dtsi
new file mode 100644
index 0000000000000..f0e398eb2aad7
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx8qm-tqma8qm.dtsi
@@ -0,0 +1,325 @@
+// SPDX-License-Identifier: (GPL-2.0-or-later OR MIT)
+/*
+ * Copyright (c) 2019-2026 TQ-Systems GmbH <linux@ew.tq-group.com>,
+ * D-82229 Seefeld, Germany.
+ * Author: Alexander Stein
+ */
+
+#include <dt-bindings/firmware/imx/rsrc.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/thermal/thermal.h>
+
+#include "imx8qm.dtsi"
+
+/ {
+	model = "TQ-Systems i.MX8QM TQMa8QM";
+	compatible = "tq,imx8qm-tqma8qm", "fsl,imx8qm";
+
+	/* Due to missing workaround for ERR050104 */
+	cpus {
+		/delete-node/ cpu-map;
+		/delete-node/ cpu@100;
+		/delete-node/ cpu@101;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/*
+		 * DRAM base addr, size : 1024 MiB DRAM
+		 * should be corrected by bootloader
+		 */
+		reg = <0x00000000 0x80000000 0 0x40000000>;
+	};
+
+	reg_tqma8x_v3v3: regulator-tqma8x-v3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "V_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	/* SW7 controlled by SCU */
+	reg_1v8_io1: regulator-v1v8-io1 {
+		compatible = "regulator-fixed";
+		regulator-name = "V_1V8_IO1";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+	};
+
+	/* LDO4 controlled by SCU */
+	reg_3v3_emmc: regulator-v3v3-emmc {
+		compatible = "regulator-fixed";
+		regulator-name = "V_3V3_EMMC";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+};
+
+&acm {
+	status = "okay";
+};
+
+&adc0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_adc0>;
+	vref-supply = <&reg_vref_v1v8>;
+	status = "okay";
+};
+
+/* TQMa8QM only uses industrial grade, reduce trip points accordingly */
+&cpu_alert0 {
+	temperature = <95000>;
+};
+
+&cpu_crit0 {
+	temperature = <100000>;
+};
+
+&cpu_alert1 {
+	temperature = <95000>;
+};
+
+&cpu_crit1 {
+	temperature = <100000>;
+};
+
+&gpu_alert0 {
+	temperature = <95000>;
+};
+
+&gpu_crit0 {
+	temperature = <100000>;
+};
+
+&gpu_alert1 {
+	temperature = <95000>;
+};
+
+&gpu_crit1 {
+	temperature = <100000>;
+};
+
+&drc_alert0 {
+	temperature = <95000>;
+};
+
+&drc_crit0 {
+	temperature = <100000>;
+};
+/* end of temperature grade adjustments */
+
+&flexspi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexspi0>;
+	status = "okay";
+
+	flash0: flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <66000000>;
+		spi-tx-bus-width = <4>;
+		spi-rx-bus-width = <4>;
+		vcc-supply = <&reg_1v8_io1>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+		};
+	};
+};
+
+&i2c1 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "gpio";
+	pinctrl-0 = <&pinctrl_lpi2c1>;
+	pinctrl-1 = <&pinctrl_lpi2c1gpio>;
+	scl-gpios = <&lsio_gpio0 14 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&lsio_gpio0 15 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	se97: temperature-sensor@1b {
+		compatible = "nxp,se97b", "jedec,jc-42.4-temp";
+		reg = <0x1b>;
+	};
+
+	pcf85063: rtc@51 {
+		compatible = "nxp,pcf85063a";
+		reg = <0x51>;
+		quartz-load-femtofarads = <7000>;
+	};
+
+	at24c02: eeprom@53 {
+		compatible = "nxp,se97b", "atmel,24c02";
+		reg = <0x53>;
+		pagesize = <16>;
+		read-only;
+		vcc-supply = <&reg_tqma8x_v3v3>;
+	};
+
+	m24c64: eeprom@57 {
+		compatible = "atmel,24c64";
+		reg = <0x57>;
+		pagesize = <32>;
+		vcc-supply = <&reg_tqma8x_v3v3>;
+	};
+};
+
+&mu_m0 {
+	status = "okay";
+};
+
+&mu1_m0 {
+	status = "okay";
+};
+
+&mu2_m0 {
+	status = "okay";
+};
+
+&thermal_zones {
+	/delete-node/ cpu1-thermal;
+
+	pmic0-thermal {
+		polling-delay-passive = <250>;
+		polling-delay = <2000>;
+		thermal-sensors = <&tsens IMX_SC_R_PMIC_0>;
+
+		trips {
+			pmic_alert0: trip0 {
+				temperature = <110000>;
+				hysteresis = <2000>;
+				type = "passive";
+			};
+			pmic_crit0: trip1 {
+				temperature = <125000>;
+				hysteresis = <2000>;
+				type = "critical";
+			};
+		};
+
+		cooling-maps {
+			map0 {
+				trip = <&pmic_alert0>;
+				cooling-device =
+					<&A53_0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+					<&A53_1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+					<&A53_2 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>,
+					<&A53_3 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>;
+			};
+		};
+	};
+};
+
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	bus-width = <8>;
+	vmmc-supply = <&reg_3v3_emmc>;
+	vqmmc-supply = <&reg_1v8_io1>;
+	no-sd;
+	no-sdio;
+	non-removable;
+	status = "okay";
+};
+
+&vpu {
+	compatible = "nxp,imx8qm-vpu";
+	status = "okay";
+};
+
+&vpu_core0 {
+	memory-region = <&decoder_boot>, <&decoder_rpc>;
+	status = "okay";
+};
+
+&vpu_core1 {
+	memory-region = <&encoder1_boot>, <&encoder1_rpc>;
+	status = "okay";
+};
+
+&vpu_core2 {
+	memory-region = <&encoder2_boot>, <&encoder2_rpc>;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_lpi2c1: lpi2c1grp {
+		fsl,pins = <IMX8QM_GPT0_CAPTURE_DMA_I2C1_SDA		0x0600004d>,
+			   <IMX8QM_GPT0_CLK_DMA_I2C1_SCL		0x0600004d>;
+	};
+
+	pinctrl_lpi2c1gpio: lpi2c1gpiogrp {
+		fsl,pins = <IMX8QM_GPT0_CAPTURE_LSIO_GPIO0_IO15		0x0600004d>,
+			   <IMX8QM_GPT0_CLK_LSIO_GPIO0_IO14		0x0600004d>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <IMX8QM_EMMC0_CLK_CONN_EMMC0_CLK		0x06000041>,
+			   <IMX8QM_EMMC0_CMD_CONN_EMMC0_CMD		0x00000021>,
+			   <IMX8QM_EMMC0_DATA0_CONN_EMMC0_DATA0		0x00000021>,
+			   <IMX8QM_EMMC0_DATA1_CONN_EMMC0_DATA1		0x00000021>,
+			   <IMX8QM_EMMC0_DATA2_CONN_EMMC0_DATA2		0x00000021>,
+			   <IMX8QM_EMMC0_DATA3_CONN_EMMC0_DATA3		0x00000021>,
+			   <IMX8QM_EMMC0_DATA4_CONN_EMMC0_DATA4		0x00000021>,
+			   <IMX8QM_EMMC0_DATA5_CONN_EMMC0_DATA5		0x00000021>,
+			   <IMX8QM_EMMC0_DATA6_CONN_EMMC0_DATA6		0x00000021>,
+			   <IMX8QM_EMMC0_DATA7_CONN_EMMC0_DATA7		0x00000021>,
+			   <IMX8QM_EMMC0_STROBE_CONN_EMMC0_STROBE	0x00000041>,
+			   <IMX8QM_EMMC0_RESET_B_CONN_EMMC0_RESET_B	0x00000021>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <IMX8QM_EMMC0_CLK_CONN_EMMC0_CLK		0x06000040>,
+			   <IMX8QM_EMMC0_CMD_CONN_EMMC0_CMD		0x00000021>,
+			   <IMX8QM_EMMC0_DATA0_CONN_EMMC0_DATA0		0x00000021>,
+			   <IMX8QM_EMMC0_DATA1_CONN_EMMC0_DATA1		0x00000021>,
+			   <IMX8QM_EMMC0_DATA2_CONN_EMMC0_DATA2		0x00000021>,
+			   <IMX8QM_EMMC0_DATA3_CONN_EMMC0_DATA3		0x00000021>,
+			   <IMX8QM_EMMC0_DATA4_CONN_EMMC0_DATA4		0x00000021>,
+			   <IMX8QM_EMMC0_DATA5_CONN_EMMC0_DATA5		0x00000021>,
+			   <IMX8QM_EMMC0_DATA6_CONN_EMMC0_DATA6		0x00000021>,
+			   <IMX8QM_EMMC0_DATA7_CONN_EMMC0_DATA7		0x00000021>,
+			   <IMX8QM_EMMC0_STROBE_CONN_EMMC0_STROBE	0x00000041>,
+			   <IMX8QM_EMMC0_RESET_B_CONN_EMMC0_RESET_B	0x00000021>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <IMX8QM_EMMC0_CLK_CONN_EMMC0_CLK		0x06000040>,
+			   <IMX8QM_EMMC0_CMD_CONN_EMMC0_CMD		0x00000021>,
+			   <IMX8QM_EMMC0_DATA0_CONN_EMMC0_DATA0		0x00000021>,
+			   <IMX8QM_EMMC0_DATA1_CONN_EMMC0_DATA1		0x00000021>,
+			   <IMX8QM_EMMC0_DATA2_CONN_EMMC0_DATA2		0x00000021>,
+			   <IMX8QM_EMMC0_DATA3_CONN_EMMC0_DATA3		0x00000021>,
+			   <IMX8QM_EMMC0_DATA4_CONN_EMMC0_DATA4		0x00000021>,
+			   <IMX8QM_EMMC0_DATA5_CONN_EMMC0_DATA5		0x00000021>,
+			   <IMX8QM_EMMC0_DATA6_CONN_EMMC0_DATA6		0x00000021>,
+			   <IMX8QM_EMMC0_DATA7_CONN_EMMC0_DATA7		0x00000021>,
+			   <IMX8QM_EMMC0_STROBE_CONN_EMMC0_STROBE	0x00000041>,
+			   <IMX8QM_EMMC0_RESET_B_CONN_EMMC0_RESET_B	0x00000021>;
+	};
+
+	pinctrl_flexspi0: flexspi0grp {
+		fsl,pins = <IMX8QM_QSPI0A_DATA0_LSIO_QSPI0A_DATA0	0x0000004d>,
+			   <IMX8QM_QSPI0A_DATA1_LSIO_QSPI0A_DATA1	0x0000004d>,
+			   <IMX8QM_QSPI0A_DATA2_LSIO_QSPI0A_DATA2	0x0000004d>,
+			   <IMX8QM_QSPI0A_DATA3_LSIO_QSPI0A_DATA3	0x0000004d>,
+			   <IMX8QM_QSPI0A_DQS_LSIO_QSPI0A_DQS		0x0000004d>,
+			   <IMX8QM_QSPI0A_SS0_B_LSIO_QSPI0A_SS0_B	0x0000004d>,
+			   <IMX8QM_QSPI0A_SS1_B_LSIO_QSPI0A_SS1_B	0x0000004d>,
+			   <IMX8QM_QSPI0A_SCLK_LSIO_QSPI0A_SCLK		0x0000004d>,
+			   <IMX8QM_QSPI0B_SCLK_LSIO_QSPI0B_SCLK		0x0000004d>,
+			   <IMX8QM_QSPI0B_DATA0_LSIO_QSPI0B_DATA0	0x0000004d>,
+			   <IMX8QM_QSPI0B_DATA1_LSIO_QSPI0B_DATA1	0x0000004d>,
+			   <IMX8QM_QSPI0B_DATA2_LSIO_QSPI0B_DATA2	0x0000004d>,
+			   <IMX8QM_QSPI0B_DATA3_LSIO_QSPI0B_DATA3	0x0000004d>,
+			   <IMX8QM_QSPI0B_DQS_LSIO_QSPI0B_DQS		0x0000004d>,
+			   <IMX8QM_QSPI0B_SS0_B_LSIO_QSPI0B_SS0_B	0x0000004d>,
+			   <IMX8QM_QSPI0B_SS1_B_LSIO_QSPI0B_SS1_B	0x0000004d>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts
index 623169f7ddb5f..c07138055229c 100644
--- a/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts
+++ b/arch/arm64/boot/dts/freescale/imx8qxp-mek.dts
@@ -742,7 +742,6 @@ &pcie0_ep {
 	phy-names = "pcie-phy";
 	pinctrl-0 = <&pinctrl_pcieb>;
 	pinctrl-names = "default";
-	vpcie-supply = <&reg_pcieb>;
 	status = "disabled";
 };
 
diff --git a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts
index 290a49bea2f76..5dea66c1e7aa0 100644
--- a/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx8ulp-evk.dts
@@ -166,7 +166,7 @@ &lpi2c7 {
 	ptn5150_1: typec@1d {
 		compatible = "nxp,ptn5150";
 		reg = <0x1d>;
-		int-gpios = <&gpiof 3 IRQ_TYPE_EDGE_FALLING>;
+		int-gpios = <&gpiof 3 GPIO_ACTIVE_LOW>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_typec1>;
 		status = "disabled";
@@ -182,7 +182,7 @@ pcal6408: gpio@21 {
 	ptn5150_2: typec@3d {
 		compatible = "nxp,ptn5150";
 		reg = <0x3d>;
-		int-gpios = <&gpiof 5 IRQ_TYPE_EDGE_FALLING>;
+		int-gpios = <&gpiof 5 GPIO_ACTIVE_LOW>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_typec2>;
 		status = "disabled";
diff --git a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi
index 1de3ad60c6aa7..17caf3703fb17 100644
--- a/arch/arm64/boot/dts/freescale/imx8ulp.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8ulp.dtsi
@@ -858,6 +858,72 @@ spdif: spdif@2dab0000 {
 				dma-names = "rx", "tx";
 				status = "disabled";
 			};
+
+			isi: isi@2dac0000 {
+				compatible = "fsl,imx8ulp-isi";
+				reg = <0x2dac0000 0x10000>;
+				interrupts = <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
+				clocks = <&pcc5 IMX8ULP_CLK_ISI>,
+					 <&cgc2 IMX8ULP_CLK_LPAV_AXI_DIV>;
+				clock-names = "axi", "apb";
+				power-domains = <&scmi_devpd IMX8ULP_PD_ISI>;
+				status = "disabled";
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+						isi_in: endpoint {
+							remote-endpoint = <&mipi_csi_out>;
+						};
+					};
+				};
+			};
+
+			mipi_csi: csi@2daf0000 {
+				compatible = "fsl,imx8ulp-mipi-csi2";
+				reg = <0x2daf0000 0x10000>,
+				      <0x2dad0000 0x10000>;
+				clocks = <&pcc5 IMX8ULP_CLK_CSI>,
+					 <&pcc5 IMX8ULP_CLK_CSI_CLK_ESC>,
+					 <&pcc5 IMX8ULP_CLK_CSI_CLK_UI>,
+					 <&pcc5 IMX8ULP_CLK_CSI_REGS>;
+				clock-names = "core", "esc", "ui", "pclk";
+				assigned-clocks = <&pcc5 IMX8ULP_CLK_CSI>,
+						  <&pcc5 IMX8ULP_CLK_CSI_CLK_ESC>,
+						  <&pcc5 IMX8ULP_CLK_CSI_CLK_UI>,
+						  <&pcc5 IMX8ULP_CLK_CSI_REGS>;
+				assigned-clock-parents = <&cgc2 IMX8ULP_CLK_PLL4_PFD1_DIV1>,
+							 <&cgc2 IMX8ULP_CLK_PLL4_PFD1_DIV2>,
+							 <&cgc2 IMX8ULP_CLK_PLL4_PFD0_DIV1>;
+				assigned-clock-rates = <200000000>,
+						       <80000000>,
+						       <100000000>,
+						       <79200000>;
+				power-domains = <&scmi_devpd IMX8ULP_PD_MIPI_CSI>;
+				resets = <&pcc5 5>,	/* PCC5_CSI_REGS_SWRST */
+					 <&pcc5 6>;	/* PCC5_CSI_SWRST> */
+				status = "disabled";
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+					};
+
+					port@1 {
+						reg = <1>;
+
+						mipi_csi_out: endpoint {
+							remote-endpoint = <&isi_in>;
+						};
+					};
+				};
+			};
 		};
 
 		gpiod: gpio@2e200000 {
diff --git a/arch/arm64/boot/dts/freescale/imx8x-colibri.dtsi b/arch/arm64/boot/dts/freescale/imx8x-colibri.dtsi
index 47895ff8cb244..2415487d3a5de 100644
--- a/arch/arm64/boot/dts/freescale/imx8x-colibri.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx8x-colibri.dtsi
@@ -631,12 +631,12 @@ pinctrl_adc0: adc0grp {
 	 */
 	pinctrl_atmel_adap: atmeladaptergrp {
 		fsl,pins = <IMX8QXP_UART1_RX_LSIO_GPIO0_IO22			0x21>,		/* SODIMM  30 */
-			   <IMX8QXP_UART1_TX_LSIO_GPIO0_IO21			0x4000021>;	/* SODIMM  28 */
+			   <IMX8QXP_UART1_TX_LSIO_GPIO0_IO21			0x40000021>;	/* SODIMM  28 */
 	};
 
 	/* Atmel MXT touchsceen + boards with built-in Capacitive Touch Connector */
 	pinctrl_atmel_conn: atmelconnectorgrp {
-		fsl,pins = <IMX8QXP_QSPI0B_DATA2_LSIO_GPIO3_IO20		0x4000021>,	/* SODIMM 107 */
+		fsl,pins = <IMX8QXP_QSPI0B_DATA2_LSIO_GPIO3_IO20		0x40000021>,	/* SODIMM 107 */
 			   <IMX8QXP_QSPI0B_SS1_B_LSIO_GPIO3_IO24		0x21>;		/* SODIMM 106 */
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts b/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts
index c083b97476a51..de524f1921de8 100644
--- a/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-11x11-evk.dts
@@ -195,7 +195,6 @@ mdio {
 		compatible = "snps,dwmac-mdio";
 		#address-cells = <1>;
 		#size-cells = <0>;
-		clock-frequency = <5000000>;
 
 		ethphy1: ethernet-phy@1 {
 			reg = <1>;
diff --git a/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts b/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts
new file mode 100644
index 0000000000000..1239bbf579da4
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-9x9-qsb.dts
@@ -0,0 +1,425 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/usb/pd.h>
+#include "imx91.dtsi"
+
+/ {
+	compatible = "fsl,imx91-9x9-qsb", "fsl,imx91";
+	model = "NXP i.MX91 9x9 Quick Start Board";
+
+	aliases {
+		ethernet0 = &fec;
+		ethernet1 = &eqos;
+		gpio0 = &gpio1;
+		gpio1 = &gpio2;
+		gpio2 = &gpio3;
+		i2c0 = &lpi2c1;
+		i2c1 = &lpi2c2;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		rtc0 = &bbnsm_rtc;
+		serial0 = &lpuart1;
+	};
+
+	chosen {
+		stdout-path = &lpuart1;
+	};
+
+	reg_vref_1v8: regulator-adc-vref {
+		compatible = "regulator-fixed";
+		regulator-name = "vref_1v8";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2 {
+		compatible = "regulator-fixed";
+		regulator-name = "VSD_3V3";
+		off-on-delay-us = <12000>;
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0 0x10000000>;
+			linux,cma-default;
+		};
+	};
+};
+
+&adc1 {
+	vref-supply = <&reg_vref_1v8>;
+	status = "okay";
+};
+
+&eqos {
+	phy-handle = <&ethphy1>;
+	phy-mode = "rgmii-id";
+	pinctrl-0 = <&pinctrl_eqos>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		clock-frequency = <5000000>;
+
+		ethphy1: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			realtek,clkout-disable;
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <
+			MX91_PAD_ENET1_MDC__ENET1_MDC                           0x57e
+			MX91_PAD_ENET1_MDIO__ENET_QOS_MDIO                      0x57e
+			MX91_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0                  0x57e
+			MX91_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1                  0x57e
+			MX91_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2                  0x57e
+			MX91_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3                  0x57e
+			MX91_PAD_ENET1_RXC__ENET_QOS_RGMII_RXC                  0x5fe
+			MX91_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL            0x57e
+			MX91_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0                  0x57e
+			MX91_PAD_ENET1_TD1__ENET1_RGMII_TD1                     0x57e
+			MX91_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2                  0x57e
+			MX91_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3                  0x57e
+			MX91_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK  0x5fe
+			MX91_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL            0x57e
+		>;
+	};
+
+	pinctrl_lpi2c1: lpi2c1grp {
+		fsl,pins = <
+			MX91_PAD_I2C1_SCL__LPI2C1_SCL                           0x40000b9e
+			MX91_PAD_I2C1_SDA__LPI2C1_SDA                           0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c2: lpi2c2grp {
+		fsl,pins = <
+			MX91_PAD_I2C2_SCL__LPI2C2_SCL                           0x40000b9e
+			MX91_PAD_I2C2_SDA__LPI2C2_SDA                           0x40000b9e
+		>;
+	};
+
+	pinctrl_pcal6524: pcal6524grp {
+		fsl,pins = <
+			MX91_PAD_CCM_CLKO1__GPIO3_IO26                          0x31e
+		>;
+	};
+
+	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_RESET_B__GPIO3_IO7                         0x31e
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX91_PAD_UART1_RXD__LPUART1_RX                          0x31e
+			MX91_PAD_UART1_TXD__LPUART1_TX                          0x31e
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX91_PAD_SD1_CLK__USDHC1_CLK                            0x1582
+			MX91_PAD_SD1_CMD__USDHC1_CMD                            0x1382
+			MX91_PAD_SD1_DATA0__USDHC1_DATA0                        0x1382
+			MX91_PAD_SD1_DATA1__USDHC1_DATA1                        0x1382
+			MX91_PAD_SD1_DATA2__USDHC1_DATA2                        0x1382
+			MX91_PAD_SD1_DATA3__USDHC1_DATA3                        0x1382
+			MX91_PAD_SD1_DATA4__USDHC1_DATA4                        0x1382
+			MX91_PAD_SD1_DATA5__USDHC1_DATA5                        0x1382
+			MX91_PAD_SD1_DATA6__USDHC1_DATA6                        0x1382
+			MX91_PAD_SD1_DATA7__USDHC1_DATA7                        0x1382
+			MX91_PAD_SD1_STROBE__USDHC1_STROBE                      0x1582
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD1_CLK__USDHC1_CLK                            0x158e
+			MX91_PAD_SD1_CMD__USDHC1_CMD                            0x138e
+			MX91_PAD_SD1_DATA0__USDHC1_DATA0                        0x138e
+			MX91_PAD_SD1_DATA1__USDHC1_DATA1                        0x138e
+			MX91_PAD_SD1_DATA2__USDHC1_DATA2                        0x138e
+			MX91_PAD_SD1_DATA3__USDHC1_DATA3                        0x138e
+			MX91_PAD_SD1_DATA4__USDHC1_DATA4                        0x138e
+			MX91_PAD_SD1_DATA5__USDHC1_DATA5                        0x138e
+			MX91_PAD_SD1_DATA6__USDHC1_DATA6                        0x138e
+			MX91_PAD_SD1_DATA7__USDHC1_DATA7                        0x138e
+			MX91_PAD_SD1_STROBE__USDHC1_STROBE                      0x158e
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD1_CLK__USDHC1_CLK                            0x15fe
+			MX91_PAD_SD1_CMD__USDHC1_CMD                            0x13fe
+			MX91_PAD_SD1_DATA0__USDHC1_DATA0                        0x13fe
+			MX91_PAD_SD1_DATA1__USDHC1_DATA1                        0x13fe
+			MX91_PAD_SD1_DATA2__USDHC1_DATA2                        0x13fe
+			MX91_PAD_SD1_DATA3__USDHC1_DATA3                        0x13fe
+			MX91_PAD_SD1_DATA4__USDHC1_DATA4                        0x13fe
+			MX91_PAD_SD1_DATA5__USDHC1_DATA5                        0x13fe
+			MX91_PAD_SD1_DATA6__USDHC1_DATA6                        0x13fe
+			MX91_PAD_SD1_DATA7__USDHC1_DATA7                        0x13fe
+			MX91_PAD_SD1_STROBE__USDHC1_STROBE                      0x15fe
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CD_B__GPIO3_IO0                            0x31e
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__USDHC2_CLK                            0x1582
+			MX91_PAD_SD2_CMD__USDHC2_CMD                            0x1382
+			MX91_PAD_SD2_DATA0__USDHC2_DATA0                        0x1382
+			MX91_PAD_SD2_DATA1__USDHC2_DATA1                        0x1382
+			MX91_PAD_SD2_DATA2__USDHC2_DATA2                        0x1382
+			MX91_PAD_SD2_DATA3__USDHC2_DATA3                        0x1382
+			MX91_PAD_SD2_VSELECT__USDHC2_VSELECT                    0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__USDHC2_CLK                            0x158e
+			MX91_PAD_SD2_CMD__USDHC2_CMD                            0x138e
+			MX91_PAD_SD2_DATA0__USDHC2_DATA0                        0x138e
+			MX91_PAD_SD2_DATA1__USDHC2_DATA1                        0x138e
+			MX91_PAD_SD2_DATA2__USDHC2_DATA2                        0x138e
+			MX91_PAD_SD2_DATA3__USDHC2_DATA3                        0x138e
+			MX91_PAD_SD2_VSELECT__USDHC2_VSELECT                    0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__USDHC2_CLK                            0x15fe
+			MX91_PAD_SD2_CMD__USDHC2_CMD                            0x13fe
+			MX91_PAD_SD2_DATA0__USDHC2_DATA0                        0x13fe
+			MX91_PAD_SD2_DATA1__USDHC2_DATA1                        0x13fe
+			MX91_PAD_SD2_DATA2__USDHC2_DATA2                        0x13fe
+			MX91_PAD_SD2_DATA3__USDHC2_DATA3                        0x13fe
+			MX91_PAD_SD2_VSELECT__USDHC2_VSELECT                    0x51e
+		>;
+	};
+};
+
+&lpi2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c1>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	p3t1085: temperature-sensor@48 {
+		compatible = "nxp,p3t1085";
+		reg = <0x48>;
+	};
+
+	ptn5110: tcpc@50 {
+		compatible = "nxp,ptn5110", "tcpci";
+		reg = <0x50>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <26 IRQ_TYPE_LEVEL_LOW>;
+
+		typec1_con: connector {
+			compatible = "usb-c-connector";
+			data-role = "dual";
+			label = "USB-C";
+			op-sink-microwatt = <15000000>;
+			power-role = "dual";
+			self-powered;
+			sink-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)
+				     PDO_VAR(5000, 20000, 3000)>;
+			source-pdos = <PDO_FIXED(5000, 3000, PDO_FIXED_USB_COMM)>;
+			try-power-role = "sink";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+					typec1_dr_sw: endpoint {
+						remote-endpoint = <&usb1_drd_sw>;
+					};
+				};
+			};
+		};
+	};
+
+	pcf2131: rtc@53 {
+		compatible = "nxp,pcf2131";
+		reg = <0x53>;
+		interrupt-parent = <&pcal6524>;
+		interrupts = <1 IRQ_TYPE_EDGE_FALLING>;
+	};
+};
+
+&lpi2c2 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c2>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pcal6524: gpio@22 {
+		compatible = "nxp,pcal6524";
+		reg = <0x22>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio3>;
+		interrupts = <26 IRQ_TYPE_LEVEL_LOW>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		pinctrl-0 = <&pinctrl_pcal6524>;
+		pinctrl-names = "default";
+	};
+
+	pmic@32 {
+		compatible = "nxp,pf9453";
+		reg = <0x32>;
+		interrupt-parent = <&pcal6524>;
+		interrupts = <11 IRQ_TYPE_EDGE_FALLING>;
+
+		regulators {
+			buck1: BUCK1 {
+				regulator-name = "BUCK1";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1140000>;
+				regulator-min-microvolt = <1060000>;
+			};
+
+			buck2: BUCK2 {
+				regulator-name = "BUCK2";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <900000>;
+				regulator-min-microvolt = <610000>;
+				regulator-ramp-delay = <12500>;
+			};
+
+			buck3: BUCK3 {
+				regulator-name = "BUCK3";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3450000>;
+				regulator-min-microvolt = <1650000>;
+			};
+
+			buck4: BUCK4 {
+				regulator-name = "BUCK4";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3450000>;
+				regulator-min-microvolt = <1650000>;
+			};
+
+			ldo1: LDO1 {
+				regulator-name = "LDO1";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <3300000>;
+				regulator-min-microvolt = <1650000>;
+			};
+
+			ldo2: LDO2 {
+				regulator-name = "LDO2";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1890000>;
+				regulator-min-microvolt = <760000>;
+			};
+
+			ldo_snvs: LDO-SNVS {
+				regulator-name = "LDO-SNVS";
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-max-microvolt = <1950000>;
+				regulator-min-microvolt = <1650000>;
+			};
+		};
+	};
+};
+
+&lpuart1 {
+	pinctrl-0 = <&pinctrl_uart1>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&usbotg1 {
+	adp-disable;
+	disable-over-current;
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	usb-role-switch;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	status = "okay";
+
+	port {
+		usb1_drd_sw: endpoint {
+			remote-endpoint = <&typec1_dr_sw>;
+		};
+	};
+};
+
+&usdhc1 {
+	bus-width = <8>;
+	non-removable;
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	fsl,tuning-step = <1>;
+	status = "okay";
+};
+
+&usdhc2 {
+	bus-width = <4>;
+	cd-gpios = <&gpio3 0 GPIO_ACTIVE_LOW>;
+	no-mmc;
+	no-sdio;
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	fsl,tuning-step = <1>;
+	status = "okay";
+};
+
+&wdog3 {
+	fsl,ext-reset-output;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91-phyboard-segin.dts b/arch/arm64/boot/dts/freescale/imx91-phyboard-segin.dts
index 7b18a58024f5c..022e9c6841efd 100644
--- a/arch/arm64/boot/dts/freescale/imx91-phyboard-segin.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-phyboard-segin.dts
@@ -137,6 +137,24 @@ ethphy2: ethernet-phy@2 {
 	};
 };
 
+&gpio1 {
+	gpio-line-names = "", "USER_LED", "I2C1_SCL", "I2C1_SDA";
+};
+
+&gpio3 {
+	gpio-line-names = "SD1_nCD", "", "", "", "",
+			  "", "", "SD1_nRESET";
+};
+
+&gpio4 {
+	gpio-line-names = "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "CAN_EN", "", "", "",
+			  "", "", "", "RESET_PHY", "",
+			  "", "RTC_nINT", "PMIC_IRQ_B";
+};
+
 /* CAN */
 &flexcan1 {
 	pinctrl-names = "default";
@@ -309,7 +327,7 @@ MX91_PAD_SD2_CD_B__GPIO3_IO0		0x31e
 
 	pinctrl_usdhc2_default: usdhc2grp {
 		fsl,pins = <
-			MX91_PAD_SD2_CLK__USDHC2_CLK		0x158e
+			MX91_PAD_SD2_CLK__USDHC2_CLK		0x118e
 			MX91_PAD_SD2_CMD__USDHC2_CMD		0x1382
 			MX91_PAD_SD2_DATA0__USDHC2_DATA0	0x1386
 			MX91_PAD_SD2_DATA1__USDHC2_DATA1	0x138e
@@ -321,7 +339,7 @@ MX91_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
 
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
 		fsl,pins = <
-			MX91_PAD_SD2_CLK__USDHC2_CLK		0x159e
+			MX91_PAD_SD2_CLK__USDHC2_CLK		0x119e
 			MX91_PAD_SD2_CMD__USDHC2_CMD		0x139e
 			MX91_PAD_SD2_DATA0__USDHC2_DATA0	0x138e
 			MX91_PAD_SD2_DATA1__USDHC2_DATA1	0x138e
@@ -333,7 +351,7 @@ MX91_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
 
 	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
 		fsl,pins = <
-			MX91_PAD_SD2_CLK__USDHC2_CLK		0x158e
+			MX91_PAD_SD2_CLK__USDHC2_CLK		0x118e
 			MX91_PAD_SD2_CMD__USDHC2_CMD		0x138e
 			MX91_PAD_SD2_DATA0__USDHC2_DATA0	0x139e
 			MX91_PAD_SD2_DATA1__USDHC2_DATA1	0x139e
diff --git a/arch/arm64/boot/dts/freescale/imx91-phycore-som.dtsi b/arch/arm64/boot/dts/freescale/imx91-phycore-som.dtsi
index 29a428a052b07..8b8cb3daecbb5 100644
--- a/arch/arm64/boot/dts/freescale/imx91-phycore-som.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx91-phycore-som.dtsi
@@ -8,6 +8,7 @@
  */
 
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/regulator/nxp,pca9450-regulator.h>
 
 #include "imx91.dtsi"
 
@@ -88,6 +89,28 @@ ethphy1: ethernet-phy@1 {
 	};
 };
 
+&gpio1 {
+	gpio-line-names = "", "USER_LED";
+};
+
+&gpio2 {
+	gpio-line-names = "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "I2C3_SDA", "I2C3_SCL";
+};
+
+&gpio4 {
+	gpio-line-names = "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "RESET_PHY", "",
+			  "", "", "PMIC_IRQ_B";
+};
+
 /* I2C3 */
 &lpi2c3 {
 	clock-frequency = <400000>;
@@ -138,6 +161,7 @@ buck5: BUCK5 {
 				regulator-min-microvolt = <1800000>;
 				regulator-boot-on;
 				regulator-always-on;
+				regulator-initial-mode = <PCA9450_BUCK_MODE_FORCE_PWM>;
 			};
 
 			buck6: BUCK6 {
@@ -192,6 +216,7 @@ &usdhc1 {
 	bus-width = <8>;
 	non-removable;
 	no-1-8-v;
+	fsl,strobe-dll-delay-target = <1>;
 	status = "okay";
 };
 
@@ -218,7 +243,7 @@ MX91_PAD_ENET2_TD0__ENET2_RGMII_TD0		0x50e
 			MX91_PAD_ENET2_TD1__ENET2_RGMII_TD1		0x50e
 			MX91_PAD_ENET2_TX_CTL__ENET2_RGMII_TX_CTL	0x50e
 			MX91_PAD_ENET2_TD2__ENET2_TX_CLK2		0x4000050e
-			MX91_PAD_ENET2_RXC__GPIO4_IO23			0x51e
+			MX91_PAD_ENET2_RXC__GPIO4_IO23			0x502
 		>;
 	};
 
@@ -250,7 +275,7 @@ MX91_PAD_ENET2_RD3__GPIO4_IO27		0x31e
 
 	pinctrl_usdhc1: usdhc1grp {
 		fsl,pins = <
-			MX91_PAD_SD1_CLK__USDHC1_CLK		0x179e
+			MX91_PAD_SD1_CLK__USDHC1_CLK		0x119e
 			MX91_PAD_SD1_CMD__USDHC1_CMD		0x1386
 			MX91_PAD_SD1_DATA0__USDHC1_DATA0	0x138e
 			MX91_PAD_SD1_DATA1__USDHC1_DATA1	0x1386
@@ -260,13 +285,13 @@ MX91_PAD_SD1_DATA4__USDHC1_DATA4	0x1386
 			MX91_PAD_SD1_DATA5__USDHC1_DATA5	0x1386
 			MX91_PAD_SD1_DATA6__USDHC1_DATA6	0x1386
 			MX91_PAD_SD1_DATA7__USDHC1_DATA7	0x1386
-			MX91_PAD_SD1_STROBE__USDHC1_STROBE	0x179e
+			MX91_PAD_SD1_STROBE__USDHC1_STROBE	0x159e
 		>;
 	};
 
 	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
 		fsl,pins = <
-			MX91_PAD_SD1_CLK__USDHC1_CLK		0x17be
+			MX91_PAD_SD1_CLK__USDHC1_CLK		0x11be
 			MX91_PAD_SD1_CMD__USDHC1_CMD		0x139e
 			MX91_PAD_SD1_DATA0__USDHC1_DATA0	0x138e
 			MX91_PAD_SD1_DATA1__USDHC1_DATA1	0x139e
@@ -276,13 +301,13 @@ MX91_PAD_SD1_DATA4__USDHC1_DATA4	0x139e
 			MX91_PAD_SD1_DATA5__USDHC1_DATA5	0x139e
 			MX91_PAD_SD1_DATA6__USDHC1_DATA6	0x139e
 			MX91_PAD_SD1_DATA7__USDHC1_DATA7	0x139e
-			MX91_PAD_SD1_STROBE__USDHC1_STROBE	0x179e
+			MX91_PAD_SD1_STROBE__USDHC1_STROBE	0x159e
 		>;
 	};
 
 	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
 		fsl,pins = <
-			MX91_PAD_SD1_CLK__USDHC1_CLK		0x17be
+			MX91_PAD_SD1_CLK__USDHC1_CLK		0x11be
 			MX91_PAD_SD1_CMD__USDHC1_CMD		0x139e
 			MX91_PAD_SD1_DATA0__USDHC1_DATA0	0x139e
 			MX91_PAD_SD1_DATA1__USDHC1_DATA1	0x13be
@@ -292,7 +317,7 @@ MX91_PAD_SD1_DATA4__USDHC1_DATA4	0x13be
 			MX91_PAD_SD1_DATA5__USDHC1_DATA5	0x13be
 			MX91_PAD_SD1_DATA6__USDHC1_DATA6	0x13be
 			MX91_PAD_SD1_DATA7__USDHC1_DATA7	0x13be
-			MX91_PAD_SD1_STROBE__USDHC1_STROBE	0x179e
+			MX91_PAD_SD1_STROBE__USDHC1_STROBE	0x159e
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx91-var-dart-sonata.dts b/arch/arm64/boot/dts/freescale/imx91-var-dart-sonata.dts
index afa39dab240a7..3f0fd321d95f0 100644
--- a/arch/arm64/boot/dts/freescale/imx91-var-dart-sonata.dts
+++ b/arch/arm64/boot/dts/freescale/imx91-var-dart-sonata.dts
@@ -90,6 +90,13 @@ reg_vref_1v8: regulator-adc-vref {
 		regulator-max-microvolt = <1800000>;
 	};
 
+	reg_rgb_sel: regulator-rgb-enable {
+		compatible = "regulator-fixed";
+		regulator-name = "RGBSEL";
+		gpio = <&pca9534 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
 	reg_usdhc2_vmmc: regulator-vmmc-usdhc2 {
 		compatible = "regulator-fixed";
 		pinctrl-names = "default";
@@ -195,6 +202,7 @@ pca6408_1: gpio@20 {
 		#gpio-cells = <2>;
 		interrupt-parent = <&gpio1>;
 		interrupts = <10 IRQ_TYPE_LEVEL_LOW>;
+		vcc-supply = <&reg_rgb_sel>;
 	};
 
 	pca6408_2: gpio@21 {
@@ -204,6 +212,7 @@ pca6408_2: gpio@21 {
 		#gpio-cells = <2>;
 		interrupt-parent = <&gpio1>;
 		interrupts = <10 IRQ_TYPE_LEVEL_LOW>;
+		vcc-supply = <&reg_rgb_sel>;
 	};
 
 	pca9534: gpio@22 {
diff --git a/arch/arm64/boot/dts/freescale/imx91-var-som-symphony.dts b/arch/arm64/boot/dts/freescale/imx91-var-som-symphony.dts
new file mode 100644
index 0000000000000..ac9fed58357e6
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-var-som-symphony.dts
@@ -0,0 +1,527 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Variscite Symphony carrier board for VAR-SOM-MX91
+ *
+ * Link: https://variscite.com/carrier-boards/symphony-board/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+#include "imx91-var-som.dtsi"
+
+/{
+	model = "Variscite VAR-SOM-MX91 on Symphony evaluation board";
+	compatible = "variscite,var-som-mx91-symphony",
+		     "variscite,var-som-mx91", "fsl,imx91";
+
+	aliases {
+		ethernet0 = &eqos;
+		ethernet1 = &fec;
+		gpio0 = &gpio1;
+		gpio1 = &gpio2;
+		gpio2 = &gpio3;
+		gpio3 = &gpio4;
+		gpio4 = &pca9534;
+		gpio5 = &pca6408;
+		i2c0 = &lpi2c1;
+		i2c1 = &lpi2c2;
+		i2c2 = &lpi2c3;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		serial0 = &lpuart1;
+		serial1 = &lpuart2;
+		serial2 = &lpuart3;
+		serial3 = &lpuart4;
+		serial4 = &lpuart5;
+	};
+
+	chosen {
+		stdout-path = &lpuart1;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		key-back {
+			label = "Back";
+			gpios = <&pca9534 1 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_BACK>;
+		};
+
+		key-home {
+			label = "Home";
+			gpios = <&pca9534 2 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_HOME>;
+		};
+
+		key-menu {
+			label = "Menu";
+			gpios = <&pca9534 3 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_MENU>;
+		};
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+
+		led-hearthbeat {
+			function = LED_FUNCTION_STATUS;
+			color = <LED_COLOR_ID_GREEN>;
+			gpios = <&pca9534 0 GPIO_ACTIVE_HIGH>;
+			linux,default-trigger = "heartbeat";
+		};
+	};
+
+	reg_vref_1v8: regulator-adc-vref {
+		compatible = "regulator-fixed";
+		regulator-name = "vref_1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+	};
+
+	/*
+	 * Needed only for Symphony <= v1.5
+	 */
+	reg_fec_phy: regulator-fec-phy {
+		compatible = "regulator-fixed";
+		regulator-name = "fec-phy";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-enable-ramp-delay = <20000>;
+		gpio = <&pca9534 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+		regulator-always-on;
+	};
+
+	reg_rgb_sel: regulator-rgb-enable {
+		compatible = "regulator-fixed";
+		regulator-name = "RGBSEL";
+		gpio = <&pca9534 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2 {
+		compatible = "regulator-fixed";
+		regulator-name = "VSD_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&pca6408 6 GPIO_ACTIVE_HIGH>;
+		off-on-delay-us = <20000>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		ele_reserved: ele-reserved@87de0000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x87de0000 0 0x100000>;
+			no-map;
+		};
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0 0x80000000 0 0x40000000>;
+			reusable;
+			size = <0 0x10000000>;
+			linux,cma-default;
+		};
+	};
+};
+
+&adc1 {
+	vref-supply = <&reg_vref_1v8>;
+	status = "okay";
+};
+
+/* Use external instead of internal RTC*/
+&bbnsm_rtc {
+	status = "disabled";
+};
+
+&eqos {
+	mdio {
+		ethphy1: ethernet-phy@5 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <5>;
+			eee-broken-1000t;
+			reset-gpios = <&pca9534 5 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <20000>;
+			vddio-supply = <&vddio1>;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@0 {
+					reg = <0>;
+					color = <LED_COLOR_ID_YELLOW>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+
+				led@1 {
+					reg = <1>;
+					color = <LED_COLOR_ID_GREEN>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+			};
+
+			vddio1: vddio-regulator {
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+			};
+		};
+	};
+};
+
+&ethphy0 {
+	leds {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@0 {
+			reg = <0>;
+			color = <LED_COLOR_ID_YELLOW>;
+			function = LED_FUNCTION_LAN;
+			linux,default-trigger = "netdev";
+		};
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			linux,default-trigger = "netdev";
+		};
+	};
+};
+
+&fec {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_fec>;
+	pinctrl-1 = <&pinctrl_fec_sleep>;
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy1>;
+	phy-supply = <&reg_fec_phy>;
+	status = "okay";
+};
+
+&flexcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	status = "okay";
+};
+
+&lpi2c1 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "sleep", "gpio";
+	pinctrl-0 = <&pinctrl_lpi2c1>;
+	pinctrl-1 = <&pinctrl_lpi2c1_gpio>;
+	pinctrl-2 = <&pinctrl_lpi2c1_gpio>;
+	scl-gpios = <&gpio1 0 GPIO_ACTIVE_HIGH>;
+	sda-gpios = <&gpio1 1 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+
+	pca9534: gpio@20 {
+		compatible = "nxp,pca9534";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pca9534>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <26 IRQ_TYPE_EDGE_FALLING>;
+		wakeup-source;
+	};
+
+	pca6408: gpio@21 {
+		compatible = "nxp,pcal6408";
+		reg = <0x21>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pca6408>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-controller;
+		#interrupt-cells = <2>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <4 IRQ_TYPE_EDGE_FALLING>;
+		vcc-supply = <&reg_rgb_sel>;
+		wakeup-source;
+
+		tpm-enable-hog {
+			gpio-hog;
+			gpios = <4 GPIO_ACTIVE_HIGH>;
+			output-high;
+			line-name = "tpm_en";
+		};
+	};
+
+	/* USB Type-C Controller */
+	ptn5150: typec@3d {
+		compatible = "nxp,ptn5150";
+		reg = <0x3d>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_ptn5150>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <10 IRQ_TYPE_NONE>;
+
+		port {
+			typec1_dr_sw: endpoint {
+				remote-endpoint = <&usb1_drd_sw>;
+			};
+		};
+	};
+
+	st33ktpm2xi2c: tpm@2e {
+		compatible = "st,st33ktpm2xi2c", "tcg,tpm-tis-i2c";
+		reg = <0x2e>;
+	};
+
+	/* Capacitive touch controller */
+	ft5x06_ts: touchscreen@38 {
+		compatible = "edt,edt-ft5206";
+		reg = <0x38>;
+		interrupt-parent = <&pca6408>;
+		interrupts = <3 IRQ_TYPE_EDGE_FALLING>;
+		touchscreen-size-x = <800>;
+		touchscreen-size-y = <480>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+		wakeup-source;
+	};
+
+	/* DS1337 RTC module */
+	rtc@68 {
+		compatible = "dallas,ds1337";
+		reg = <0x68>;
+	};
+};
+
+/* pins conflict */
+&lpspi8 {
+	status = "disabled";
+};
+
+/* Console */
+&lpuart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+&tpm4 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_tpm4>;
+	pinctrl-1 = <&pinctrl_tpm4_sleep>;
+	status = "okay";
+};
+
+&usbotg1 {
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	usb-role-switch;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	status = "okay";
+
+	port {
+		usb1_drd_sw: endpoint {
+			remote-endpoint = <&typec1_dr_sw>;
+		};
+	};
+};
+
+&usbotg2 {
+	dr_mode = "host";
+	disable-over-current;
+	status = "okay";
+};
+
+/* SD */
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-3 = <&pinctrl_usdhc2_sleep>, <&pinctrl_usdhc2_gpio_sleep>;
+	bus-width = <4>;
+	cd-gpios = <&gpio3 00 GPIO_ACTIVE_LOW>;
+	no-mmc;
+	no-sdio;
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	status = "okay";
+};
+
+/* Watchdog */
+&wdog3 {
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX91_PAD_ENET2_RD0__ENET2_RGMII_RD0		0x57e
+			MX91_PAD_ENET2_RD1__ENET2_RGMII_RD1		0x57e
+			MX91_PAD_ENET2_RD2__ENET2_RGMII_RD2		0x57e
+			MX91_PAD_ENET2_RD3__ENET2_RGMII_RD3		0x57e
+			MX91_PAD_ENET2_RXC__ENET2_RGMII_RXC		0x5fe
+			MX91_PAD_ENET2_RX_CTL__ENET2_RGMII_RX_CTL	0x57e
+			MX91_PAD_ENET2_TD0__ENET2_RGMII_TD0		0x57e
+			MX91_PAD_ENET2_TD1__ENET2_RGMII_TD1		0x57e
+			MX91_PAD_ENET2_TD2__ENET2_RGMII_TD2		0x57e
+			MX91_PAD_ENET2_TD3__ENET2_RGMII_TD3		0x57e
+			MX91_PAD_ENET2_TXC__ENET2_RGMII_TXC		0x5fe
+			MX91_PAD_ENET2_TX_CTL__ENET2_RGMII_TX_CTL	0x57e
+		>;
+	};
+
+	pinctrl_fec_sleep: fecsleepgrp {
+		fsl,pins = <
+			MX91_PAD_ENET2_RD0__GPIO4_IO24			0x51e
+			MX91_PAD_ENET2_RD1__GPIO4_IO25			0x51e
+			MX91_PAD_ENET2_RD2__GPIO4_IO26			0x51e
+			MX91_PAD_ENET2_RD3__GPIO4_IO27			0x51e
+			MX91_PAD_ENET2_RXC__GPIO4_IO23                  0x51e
+			MX91_PAD_ENET2_RX_CTL__GPIO4_IO22		0x51e
+			MX91_PAD_ENET2_TD0__GPIO4_IO19			0x51e
+			MX91_PAD_ENET2_TD1__GPIO4_IO18			0x51e
+			MX91_PAD_ENET2_TD2__GPIO4_IO17			0x51e
+			MX91_PAD_ENET2_TD3__GPIO4_IO16			0x51e
+			MX91_PAD_ENET2_TXC__GPIO4_IO21			0x51e
+			MX91_PAD_ENET2_TX_CTL__GPIO4_IO20		0x51e
+		>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <
+			MX91_PAD_PDM_CLK__CAN1_TX			0x139e
+			MX91_PAD_PDM_BIT_STREAM0__CAN1_RX		0x139e
+		>;
+	};
+
+	pinctrl_lpi2c1: lpi2c1grp {
+		fsl,pins = <
+			MX91_PAD_I2C1_SCL__LPI2C1_SCL			0x40000b9e
+			MX91_PAD_I2C1_SDA__LPI2C1_SDA			0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c1_gpio: lpi2c1gpiogrp {
+		fsl,pins = <
+			MX91_PAD_I2C1_SCL__GPIO1_IO0			0x31e
+			MX91_PAD_I2C1_SDA__GPIO1_IO1			0x31e
+		>;
+	};
+
+	pinctrl_pca6408: pca6408grp {
+		fsl,pins = <
+			MX91_PAD_GPIO_IO04__GPIO2_IO4			0x31e
+		>;
+	};
+
+	pinctrl_pca9534: pca9534grp {
+		fsl,pins = <
+			MX91_PAD_CCM_CLKO1__GPIO3_IO26			0x31e
+		>;
+	};
+
+	pinctrl_ptn5150: ptn5150grp {
+		fsl,pins = <
+			MX91_PAD_PDM_BIT_STREAM1__GPIO1_IO10            0x31e
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX91_PAD_UART1_RXD__LPUART1_RX			0x31e
+			MX91_PAD_UART1_TXD__LPUART1_TX			0x31e
+		>;
+	};
+
+	pinctrl_tpm4: tpm4grp {
+		fsl,pins = <
+			MX91_PAD_GPIO_IO05__TPM4_CH0			0x51e
+		>;
+	};
+
+	pinctrl_tpm4_sleep: tpm4sleepgrp {
+		fsl,pins = <
+			MX91_PAD_GPIO_IO05__GPIO2_IO5			0x51e
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__USDHC2_CLK			0x1582
+			MX91_PAD_SD2_CMD__USDHC2_CMD			0x1382
+			MX91_PAD_SD2_DATA0__USDHC2_DATA0		0x1382
+			MX91_PAD_SD2_DATA1__USDHC2_DATA1		0x1382
+			MX91_PAD_SD2_DATA2__USDHC2_DATA2		0x1382
+			MX91_PAD_SD2_DATA3__USDHC2_DATA3		0x1382
+			MX91_PAD_SD2_VSELECT__USDHC2_VSELECT		0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__USDHC2_CLK			0x158e
+			MX91_PAD_SD2_CMD__USDHC2_CMD			0x138e
+			MX91_PAD_SD2_DATA0__USDHC2_DATA0		0x138e
+			MX91_PAD_SD2_DATA1__USDHC2_DATA1		0x138e
+			MX91_PAD_SD2_DATA2__USDHC2_DATA2		0x138e
+			MX91_PAD_SD2_DATA3__USDHC2_DATA3		0x138e
+			MX91_PAD_SD2_VSELECT__USDHC2_VSELECT		0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__USDHC2_CLK			0x15fe
+			MX91_PAD_SD2_CMD__USDHC2_CMD			0x13fe
+			MX91_PAD_SD2_DATA0__USDHC2_DATA0		0x13fe
+			MX91_PAD_SD2_DATA1__USDHC2_DATA1		0x13fe
+			MX91_PAD_SD2_DATA2__USDHC2_DATA2		0x13fe
+			MX91_PAD_SD2_DATA3__USDHC2_DATA3		0x13fe
+			MX91_PAD_SD2_VSELECT__USDHC2_VSELECT		0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_sleep: usdhc2sleep-grp {
+		fsl,pins = <
+			MX91_PAD_SD2_CLK__GPIO3_IO1			0x51e
+			MX91_PAD_SD2_CMD__GPIO3_IO2			0x51e
+			MX91_PAD_SD2_DATA0__GPIO3_IO3			0x51e
+			MX91_PAD_SD2_DATA1__GPIO3_IO4			0x51e
+			MX91_PAD_SD2_DATA2__GPIO3_IO5			0x51e
+			MX91_PAD_SD2_DATA3__GPIO3_IO6			0x51e
+			MX91_PAD_SD2_VSELECT__GPIO3_IO19		0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			MX91_PAD_SD2_CD_B__GPIO3_IO0			0x31e
+		>;
+	};
+
+	pinctrl_usdhc2_gpio_sleep: usdhc2gpiosleep-grp {
+		fsl,pins = <
+			MX91_PAD_SD2_CD_B__GPIO3_IO0			0x51e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx91-var-som.dtsi b/arch/arm64/boot/dts/freescale/imx91-var-som.dtsi
new file mode 100644
index 0000000000000..b30a0d8a81ba0
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx91-var-som.dtsi
@@ -0,0 +1,456 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Common dtsi for Variscite VAR-SOM-MX91
+ *
+ * Link: https://variscite.com/system-on-module-som/i-mx-9/i-mx-91/var-som-mx91/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include "imx91.dtsi"
+
+/{
+	model = "Variscite VAR-SOM-MX91 module";
+	compatible = "variscite,var-som-mx91", "fsl,imx91";
+
+	usdhc3_pwrseq: mmc-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		post-power-on-delay-ms = <100>;
+		power-off-delay-us = <10000>;
+		reset-gpios = <&gpio4 14 GPIO_ACTIVE_LOW>, /* WIFI_RESET */
+			      <&gpio3 7 GPIO_ACTIVE_LOW>; /* WIFI_PWR_EN */
+	};
+
+	sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&codec_dai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&codec_dai>;
+		simple-audio-card,name = "wm8904-audio";
+		simple-audio-card,routing =
+			"Headphone Jack", "HPOUTL",
+			"Headphone Jack", "HPOUTR",
+			"IN2L", "Line In Jack",
+			"IN2R", "Line In Jack",
+			"IN1L", "Microphone Jack",
+			"IN1R", "Microphone Jack";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Headphone", "Headphone Jack",
+			"Line", "Line In Jack";
+		simple-audio-card,mclk-fs = <256>;
+
+		codec_dai: simple-audio-card,codec {
+			sound-dai = <&wm8904>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai1>;
+		};
+	};
+};
+
+&eqos {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_eqos>;
+	pinctrl-1 = <&pinctrl_eqos_sleep>;
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy0>;
+	snps,clk-csr = <5>;
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+			eee-broken-1000t;
+			reset-gpios = <&gpio1 7 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <15000>;
+			reset-deassert-us = <100000>;
+		};
+	};
+};
+
+&lpi2c3 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "sleep", "gpio";
+	pinctrl-0 = <&pinctrl_lpi2c3>;
+	pinctrl-1 = <&pinctrl_lpi2c3_gpio>;
+	pinctrl-2 = <&pinctrl_lpi2c3_gpio>;
+	scl-gpios = <&gpio2 29 GPIO_ACTIVE_HIGH>;
+	sda-gpios = <&gpio2 28 GPIO_ACTIVE_HIGH>;
+	status = "okay";
+
+	pmic@25 {
+		compatible = "nxp,pca9451a";
+		reg = <0x25>;
+
+		regulators {
+			buck1: BUCK1 {
+				regulator-name = "BUCK1";
+				regulator-min-microvolt = <650000>;
+				regulator-max-microvolt = <2237500>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck2: BUCK2 {
+				regulator-name = "BUCK2";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <2187500>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck4: BUCK4 {
+				regulator-name = "BUCK4";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck5: BUCK5 {
+				regulator-name = "BUCK5";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck6: BUCK6 {
+				regulator-name = "BUCK6";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo1: LDO1 {
+				regulator-name = "LDO1";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo4: LDO4 {
+				regulator-name = "LDO4";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo5: LDO5 {
+				regulator-name = "LDO5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+
+	wm8904: audio-codec@1a {
+		compatible = "wlf,wm8904";
+		reg = <0x1a>;
+		#sound-dai-cells = <0>;
+		clocks = <&clk IMX93_CLK_SAI1_GATE>;
+		clock-names = "mclk";
+		AVDD-supply = <&buck5>;
+		CPVDD-supply = <&buck5>;
+		DBVDD-supply = <&buck4>;
+		DCVDD-supply = <&buck5>;
+		MICVDD-supply = <&buck5>;
+		wlf,drc-cfg-names = "default", "peaklimiter", "tradition",
+				    "soft", "music";
+		/*
+		 * Config registers per name, respectively:
+		 * KNEE_IP = 0,   KNEE_OP = 0,     HI_COMP = 1,   LO_COMP = 1
+		 * KNEE_IP = -24, KNEE_OP = -6,    HI_COMP = 1/4, LO_COMP = 1
+		 * KNEE_IP = -42, KNEE_OP = -3,    HI_COMP = 0,   LO_COMP = 1
+		 * KNEE_IP = -45, KNEE_OP = -9,    HI_COMP = 1/8, LO_COMP = 1
+		 * KNEE_IP = -30, KNEE_OP = -10.5, HI_COMP = 1/4, LO_COMP = 1
+		 */
+		wlf,drc-cfg-regs = /bits/ 16 <0x01af 0x3248 0x0000 0x0000>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x0408>,
+				   /bits/ 16 <0x04af 0x324b 0x0028 0x0704>,
+				   /bits/ 16 <0x04af 0x324b 0x0018 0x078c>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x050e>;
+		/* GPIO1 = DMIC_CLK, don't touch others */
+		wlf,gpio-cfg = <0x0018>, <0xffff>, <0xffff>, <0xffff>;
+	};
+};
+
+&lpspi8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpspi8>;
+	cs-gpios = <&gpio2 12 GPIO_ACTIVE_LOW>;
+	status = "okay";
+};
+
+/* BT module */
+&lpuart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpuart5>, <&pinctrl_bluetooth>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+&sai1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_sai1>;
+	pinctrl-1 = <&pinctrl_sai1_sleep>;
+	assigned-clocks = <&clk IMX93_CLK_SAI1>;
+	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
+	assigned-clock-rates = <12288000>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+/* eMMC */
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+/* WiFi */
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	pinctrl-0 = <&pinctrl_usdhc3>, <&pinctrl_usdhc3_wlan>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>, <&pinctrl_usdhc3_wlan>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>, <&pinctrl_usdhc3_wlan>;
+	pinctrl-3 = <&pinctrl_usdhc3_sleep>, <&pinctrl_usdhc3_wlan>;
+	bus-width = <4>;
+	keep-power-in-suspend;
+	mmc-pwrseq = <&usdhc3_pwrseq>;
+	non-removable;
+	wakeup-source;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_bluetooth: bluetoothgrp {
+		fsl,pins = <
+			MX91_PAD_ENET2_MDIO__GPIO4_IO15				0x51e
+		>;
+	};
+
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <
+			MX91_PAD_ENET1_MDC__ENET1_MDC				0x57e
+			MX91_PAD_ENET1_MDIO__ENET_QOS_MDIO			0x57e
+			MX91_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x57e
+			MX91_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x57e
+			MX91_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x57e
+			MX91_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x57e
+			MX91_PAD_ENET1_RXC__ENET_QOS_RGMII_RXC			0x5fe
+			MX91_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x57e
+			MX91_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x57e
+			MX91_PAD_ENET1_TD1__ENET1_RGMII_TD1			0x57e
+			MX91_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x57e
+			MX91_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x57e
+			MX91_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x5fe
+			MX91_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x57e
+			MX91_PAD_UART2_TXD__GPIO1_IO7				0x51e
+		>;
+	};
+
+	pinctrl_eqos_sleep: eqos-sleepgrp {
+		fsl,pins = <
+			MX91_PAD_ENET1_MDC__GPIO4_IO0				0x31e
+			MX91_PAD_ENET1_MDIO__GPIO4_IO1				0x31e
+			MX91_PAD_ENET1_RD0__GPIO4_IO10				0x31e
+			MX91_PAD_ENET1_RD1__GPIO4_IO11				0x31e
+			MX91_PAD_ENET1_RD2__GPIO4_IO12				0x31e
+			MX91_PAD_ENET1_RD3__GPIO4_IO13				0x31e
+			MX91_PAD_ENET1_RXC__GPIO4_IO9				0x31e
+			MX91_PAD_ENET1_RX_CTL__GPIO4_IO8			0x31e
+			MX91_PAD_ENET1_TD0__GPIO4_IO5				0x31e
+			MX91_PAD_ENET1_TD1__GPIO4_IO4				0x31e
+			MX91_PAD_ENET1_TD2__GPIO4_IO3				0x31e
+			MX91_PAD_ENET1_TD3__GPIO4_IO2				0x31e
+			MX91_PAD_ENET1_TXC__GPIO4_IO7				0x31e
+			MX91_PAD_ENET1_TX_CTL__GPIO4_IO6			0x31e
+		>;
+	};
+
+	pinctrl_lpi2c3: lpi2c3grp {
+		fsl,pins = <
+			MX91_PAD_GPIO_IO28__LPI2C3_SDA				0x40000b9e
+			MX91_PAD_GPIO_IO29__LPI2C3_SCL				0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c3_gpio: lpi2c3-gpiogrp {
+		fsl,pins = <
+			MX91_PAD_GPIO_IO28__GPIO2_IO28				0x40000b9e
+			MX91_PAD_GPIO_IO29__GPIO2_IO29				0x40000b9e
+		>;
+	};
+
+	pinctrl_lpspi8: lpspi8grp {
+		fsl,pins = <
+			MX91_PAD_GPIO_IO12__GPIO2_IO12				0x31e
+			MX91_PAD_GPIO_IO13__LPSPI8_SIN				0x31e
+			MX91_PAD_GPIO_IO14__LPSPI8_SOUT				0x31e
+			MX91_PAD_GPIO_IO15__LPSPI8_SCK				0x31e
+		>;
+	};
+
+	pinctrl_lpuart5: lpuart5grp {
+		fsl,pins = <
+			MX91_PAD_DAP_TDO_TRACESWO__LPUART5_TX			0x31e
+			MX91_PAD_DAP_TDI__LPUART5_RX				0x31e
+			MX91_PAD_DAP_TMS_SWDIO__LPUART5_RTS_B			0x31e
+			MX91_PAD_DAP_TCLK_SWCLK__LPUART5_CTS_B			0x31e
+		>;
+	};
+
+	pinctrl_sai1: sai1grp {
+		fsl,pins = <
+			MX91_PAD_SAI1_TXC__SAI1_TX_BCLK				0x31e
+			MX91_PAD_SAI1_TXFS__SAI1_TX_SYNC			0x31e
+			MX91_PAD_SAI1_TXD0__SAI1_TX_DATA0			0x31e
+			MX91_PAD_SAI1_RXD0__SAI1_RX_DATA0			0x31e
+			MX91_PAD_I2C2_SDA__SAI1_RX_BCLK				0x31e
+			MX91_PAD_I2C2_SCL__SAI1_RX_SYNC				0x31e
+			MX91_PAD_UART2_RXD__SAI1_MCLK				0x31e
+		>;
+	};
+
+	pinctrl_sai1_sleep: sai1-sleepgrp {
+		fsl,pins = <
+			MX91_PAD_SAI1_TXC__GPIO1_IO12				0x31e
+			MX91_PAD_SAI1_TXFS__GPIO1_IO11				0x31e
+			MX91_PAD_SAI1_TXD0__GPIO1_IO13				0x31e
+			MX91_PAD_SAI1_RXD0__GPIO1_IO14				0x31e
+			MX91_PAD_UART2_RXD__GPIO1_IO6				0x31e
+			MX91_PAD_I2C2_SDA__GPIO1_IO3				0x31e
+			MX91_PAD_I2C2_SCL__GPIO1_IO2				0x31e
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX91_PAD_SD1_CLK__USDHC1_CLK				0x1582
+			MX91_PAD_SD1_CMD__USDHC1_CMD				0x1382
+			MX91_PAD_SD1_DATA0__USDHC1_DATA0			0x1382
+			MX91_PAD_SD1_DATA1__USDHC1_DATA1			0x1382
+			MX91_PAD_SD1_DATA2__USDHC1_DATA2			0x1382
+			MX91_PAD_SD1_DATA3__USDHC1_DATA3			0x1382
+			MX91_PAD_SD1_DATA4__USDHC1_DATA4			0x1382
+			MX91_PAD_SD1_DATA5__USDHC1_DATA5			0x1382
+			MX91_PAD_SD1_DATA6__USDHC1_DATA6			0x1382
+			MX91_PAD_SD1_DATA7__USDHC1_DATA7			0x1382
+			MX91_PAD_SD1_STROBE__USDHC1_STROBE			0x1582
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD1_CLK__USDHC1_CLK				0x158e
+			MX91_PAD_SD1_CMD__USDHC1_CMD				0x138e
+			MX91_PAD_SD1_DATA0__USDHC1_DATA0			0x138e
+			MX91_PAD_SD1_DATA1__USDHC1_DATA1			0x138e
+			MX91_PAD_SD1_DATA2__USDHC1_DATA2			0x138e
+			MX91_PAD_SD1_DATA3__USDHC1_DATA3			0x138e
+			MX91_PAD_SD1_DATA4__USDHC1_DATA4			0x138e
+			MX91_PAD_SD1_DATA5__USDHC1_DATA5			0x138e
+			MX91_PAD_SD1_DATA6__USDHC1_DATA6			0x138e
+			MX91_PAD_SD1_DATA7__USDHC1_DATA7			0x138e
+			MX91_PAD_SD1_STROBE__USDHC1_STROBE			0x158e
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD1_CLK__USDHC1_CLK				0x15fe
+			MX91_PAD_SD1_CMD__USDHC1_CMD				0x13fe
+			MX91_PAD_SD1_DATA0__USDHC1_DATA0			0x13fe
+			MX91_PAD_SD1_DATA1__USDHC1_DATA1			0x13fe
+			MX91_PAD_SD1_DATA2__USDHC1_DATA2			0x13fe
+			MX91_PAD_SD1_DATA3__USDHC1_DATA3			0x13fe
+			MX91_PAD_SD1_DATA4__USDHC1_DATA4			0x13fe
+			MX91_PAD_SD1_DATA5__USDHC1_DATA5			0x13fe
+			MX91_PAD_SD1_DATA6__USDHC1_DATA6			0x13fe
+			MX91_PAD_SD1_DATA7__USDHC1_DATA7			0x13fe
+			MX91_PAD_SD1_STROBE__USDHC1_STROBE			0x15fe
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX91_PAD_SD3_CLK__USDHC3_CLK				0x1582
+			MX91_PAD_SD3_CMD__USDHC3_CMD				0x1382
+			MX91_PAD_SD3_DATA0__USDHC3_DATA0			0x1382
+			MX91_PAD_SD3_DATA1__USDHC3_DATA1			0x1382
+			MX91_PAD_SD3_DATA2__USDHC3_DATA2			0x1382
+			MX91_PAD_SD3_DATA3__USDHC3_DATA3			0x1382
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD3_CLK__USDHC3_CLK				0x158e
+			MX91_PAD_SD3_CMD__USDHC3_CMD				0x138e
+			MX91_PAD_SD3_DATA0__USDHC3_DATA0			0x138e
+			MX91_PAD_SD3_DATA1__USDHC3_DATA1			0x138e
+			MX91_PAD_SD3_DATA2__USDHC3_DATA2			0x138e
+			MX91_PAD_SD3_DATA3__USDHC3_DATA3			0x138e
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <
+			MX91_PAD_SD3_CLK__USDHC3_CLK				0x15fe
+			MX91_PAD_SD3_CMD__USDHC3_CMD				0x13fe
+			MX91_PAD_SD3_DATA0__USDHC3_DATA0			0x13fe
+			MX91_PAD_SD3_DATA1__USDHC3_DATA1			0x13fe
+			MX91_PAD_SD3_DATA2__USDHC3_DATA2			0x13fe
+			MX91_PAD_SD3_DATA3__USDHC3_DATA3			0x13fe
+		>;
+	};
+
+	pinctrl_usdhc3_sleep: usdhc3-sleepgrp {
+		fsl,pins = <
+			MX91_PAD_SD3_CLK__GPIO3_IO20				0x31e
+			MX91_PAD_SD3_CMD__GPIO3_IO21				0x31e
+			MX91_PAD_SD3_DATA0__GPIO3_IO22				0x31e
+			MX91_PAD_SD3_DATA1__GPIO3_IO23				0x31e
+			MX91_PAD_SD3_DATA2__GPIO3_IO24				0x31e
+			MX91_PAD_SD3_DATA3__GPIO3_IO25				0x31e
+		>;
+	};
+
+	pinctrl_usdhc3_wlan: usdhc3-wlangrp {
+		fsl,pins = <
+			MX91_PAD_ENET2_MDC__GPIO4_IO14				0x51e
+			MX91_PAD_SD2_RESET_B__GPIO3_IO7				0x51e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi b/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi
index 7d3fc4ad7b8b7..f48e7947cf647 100644
--- a/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93-11x11-evk-common.dtsi
@@ -187,7 +187,6 @@ mdio {
 		compatible = "snps,dwmac-mdio";
 		#address-cells = <1>;
 		#size-cells = <0>;
-		clock-frequency = <5000000>;
 
 		ethphy1: ethernet-phy@1 {
 			reg = <1>;
diff --git a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtsi b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtsi
new file mode 100644
index 0000000000000..d167c9fc3b8f8
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtsi
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include "imx93-pinfunc.h"
+
+&{/} {
+	backlight: backlight {
+		compatible = "gpio-backlight";
+		gpios = <&pcal6524 2 GPIO_ACTIVE_HIGH>;
+	};
+
+	panel {
+		compatible = "ontat,kd50g21-40nt-a1";
+		backlight = <&backlight>;
+		power-supply = <&reg_rpi_3v3>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&dpi_to_panel>;
+			};
+		};
+	};
+};
+
+&dpi_bridge {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lcdif>;
+	status = "okay";
+
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@1 {
+			reg = <1>;
+
+			dpi_to_panel: endpoint {
+				remote-endpoint = <&panel_in>;
+				bus-width = <18>;
+			};
+		};
+	};
+};
+
+&iomuxc {
+	pinctrl_lcdif: lcdifgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO00__MEDIAMIX_DISP_CLK		0x31e
+			MX93_PAD_GPIO_IO01__MEDIAMIX_DISP_DE		0x31e
+			MX93_PAD_GPIO_IO02__MEDIAMIX_DISP_VSYNC		0x31e
+			MX93_PAD_GPIO_IO03__MEDIAMIX_DISP_HSYNC		0x31e
+			MX93_PAD_GPIO_IO04__MEDIAMIX_DISP_DATA00	0x31e
+			MX93_PAD_GPIO_IO05__MEDIAMIX_DISP_DATA01	0x31e
+			MX93_PAD_GPIO_IO06__MEDIAMIX_DISP_DATA02	0x31e
+			MX93_PAD_GPIO_IO07__MEDIAMIX_DISP_DATA03	0x31e
+			MX93_PAD_GPIO_IO08__MEDIAMIX_DISP_DATA04	0x31e
+			MX93_PAD_GPIO_IO09__MEDIAMIX_DISP_DATA05	0x31e
+			MX93_PAD_GPIO_IO10__MEDIAMIX_DISP_DATA06	0x31e
+			MX93_PAD_GPIO_IO11__MEDIAMIX_DISP_DATA07	0x31e
+			MX93_PAD_GPIO_IO12__MEDIAMIX_DISP_DATA08	0x31e
+			MX93_PAD_GPIO_IO13__MEDIAMIX_DISP_DATA09	0x31e
+			MX93_PAD_GPIO_IO14__MEDIAMIX_DISP_DATA10	0x31e
+			MX93_PAD_GPIO_IO15__MEDIAMIX_DISP_DATA11	0x31e
+			MX93_PAD_GPIO_IO16__MEDIAMIX_DISP_DATA12	0x31e
+			MX93_PAD_GPIO_IO17__MEDIAMIX_DISP_DATA13	0x31e
+			MX93_PAD_GPIO_IO18__MEDIAMIX_DISP_DATA14	0x31e
+			MX93_PAD_GPIO_IO19__MEDIAMIX_DISP_DATA15	0x31e
+			MX93_PAD_GPIO_IO20__MEDIAMIX_DISP_DATA16	0x31e
+			MX93_PAD_GPIO_IO21__MEDIAMIX_DISP_DATA17	0x31e
+		>;
+	};
+};
+
+&lcdif {
+	status = "okay";
+};
+
+&media_blk_ctrl {
+	status = "okay";
+};
+
+&pcal6524 {
+	/*
+	 * exp-sel-hog has property 'output-low' while DT overlay doesn't
+	 * support /delete-property/. Both 'output-low' and 'output-high'
+	 * will exist under hog nodes if DT overlay file sets 'output-high'.
+	 * Workaround is to disable this hog and create new hog with
+	 * 'output-high'.
+	 */
+	exp-sel-hog {
+		status = "disabled";
+	};
+
+	exp-high-sel-hog {
+		gpio-hog;
+		gpios = <22 GPIO_ACTIVE_HIGH>;
+		output-high;
+	};
+};
+
+&sai3 {
+	/* disable due to GPIO12 and GPIO17~20 pin conflicts with LCDIF */
+	status = "disabled";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso
index d167c9fc3b8f8..356533a7b513d 100644
--- a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso
+++ b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtso
@@ -3,108 +3,4 @@
  * Copyright 2026 NXP
  */
 
-/dts-v1/;
-/plugin/;
-
-#include <dt-bindings/gpio/gpio.h>
-#include "imx93-pinfunc.h"
-
-&{/} {
-	backlight: backlight {
-		compatible = "gpio-backlight";
-		gpios = <&pcal6524 2 GPIO_ACTIVE_HIGH>;
-	};
-
-	panel {
-		compatible = "ontat,kd50g21-40nt-a1";
-		backlight = <&backlight>;
-		power-supply = <&reg_rpi_3v3>;
-
-		port {
-			panel_in: endpoint {
-				remote-endpoint = <&dpi_to_panel>;
-			};
-		};
-	};
-};
-
-&dpi_bridge {
-	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_lcdif>;
-	status = "okay";
-
-	ports {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		port@1 {
-			reg = <1>;
-
-			dpi_to_panel: endpoint {
-				remote-endpoint = <&panel_in>;
-				bus-width = <18>;
-			};
-		};
-	};
-};
-
-&iomuxc {
-	pinctrl_lcdif: lcdifgrp {
-		fsl,pins = <
-			MX93_PAD_GPIO_IO00__MEDIAMIX_DISP_CLK		0x31e
-			MX93_PAD_GPIO_IO01__MEDIAMIX_DISP_DE		0x31e
-			MX93_PAD_GPIO_IO02__MEDIAMIX_DISP_VSYNC		0x31e
-			MX93_PAD_GPIO_IO03__MEDIAMIX_DISP_HSYNC		0x31e
-			MX93_PAD_GPIO_IO04__MEDIAMIX_DISP_DATA00	0x31e
-			MX93_PAD_GPIO_IO05__MEDIAMIX_DISP_DATA01	0x31e
-			MX93_PAD_GPIO_IO06__MEDIAMIX_DISP_DATA02	0x31e
-			MX93_PAD_GPIO_IO07__MEDIAMIX_DISP_DATA03	0x31e
-			MX93_PAD_GPIO_IO08__MEDIAMIX_DISP_DATA04	0x31e
-			MX93_PAD_GPIO_IO09__MEDIAMIX_DISP_DATA05	0x31e
-			MX93_PAD_GPIO_IO10__MEDIAMIX_DISP_DATA06	0x31e
-			MX93_PAD_GPIO_IO11__MEDIAMIX_DISP_DATA07	0x31e
-			MX93_PAD_GPIO_IO12__MEDIAMIX_DISP_DATA08	0x31e
-			MX93_PAD_GPIO_IO13__MEDIAMIX_DISP_DATA09	0x31e
-			MX93_PAD_GPIO_IO14__MEDIAMIX_DISP_DATA10	0x31e
-			MX93_PAD_GPIO_IO15__MEDIAMIX_DISP_DATA11	0x31e
-			MX93_PAD_GPIO_IO16__MEDIAMIX_DISP_DATA12	0x31e
-			MX93_PAD_GPIO_IO17__MEDIAMIX_DISP_DATA13	0x31e
-			MX93_PAD_GPIO_IO18__MEDIAMIX_DISP_DATA14	0x31e
-			MX93_PAD_GPIO_IO19__MEDIAMIX_DISP_DATA15	0x31e
-			MX93_PAD_GPIO_IO20__MEDIAMIX_DISP_DATA16	0x31e
-			MX93_PAD_GPIO_IO21__MEDIAMIX_DISP_DATA17	0x31e
-		>;
-	};
-};
-
-&lcdif {
-	status = "okay";
-};
-
-&media_blk_ctrl {
-	status = "okay";
-};
-
-&pcal6524 {
-	/*
-	 * exp-sel-hog has property 'output-low' while DT overlay doesn't
-	 * support /delete-property/. Both 'output-low' and 'output-high'
-	 * will exist under hog nodes if DT overlay file sets 'output-high'.
-	 * Workaround is to disable this hog and create new hog with
-	 * 'output-high'.
-	 */
-	exp-sel-hog {
-		status = "disabled";
-	};
-
-	exp-high-sel-hog {
-		gpio-hog;
-		gpios = <22 GPIO_ACTIVE_HIGH>;
-		output-high;
-	};
-};
-
-&sai3 {
-	/* disable due to GPIO12 and GPIO17~20 pin conflicts with LCDIF */
-	status = "disabled";
-};
+#include "imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtsi"
diff --git a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-tianma-tm050rdh03.dtso b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-tianma-tm050rdh03.dtso
new file mode 100644
index 0000000000000..c233797ec28c9
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb-tianma-tm050rdh03.dtso
@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2026 NXP
+ */
+
+#include <dt-bindings/gpio/gpio.h>
+#include "imx93-9x9-qsb-ontat-kd50g21-40nt-a1.dtsi"
+
+&{/} {
+	panel {
+		compatible = "tianma,tm050rdh03";
+		enable-gpios = <&pcal6524 8 GPIO_ACTIVE_HIGH>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts
index 7bcebd7021063..01c11c517986a 100644
--- a/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-9x9-qsb.dts
@@ -228,7 +228,6 @@ mdio {
 		compatible = "snps,dwmac-mdio";
 		#address-cells = <1>;
 		#size-cells = <0>;
-		clock-frequency = <5000000>;
 
 		ethphy1: ethernet-phy@1 {
 			compatible = "ethernet-phy-ieee802.3-c22";
diff --git a/arch/arm64/boot/dts/freescale/imx93-phyboard-nash.dts b/arch/arm64/boot/dts/freescale/imx93-phyboard-nash.dts
index eac389ed30f36..f868ed5c2c291 100644
--- a/arch/arm64/boot/dts/freescale/imx93-phyboard-nash.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-phyboard-nash.dts
@@ -141,6 +141,37 @@ &flexcan1 {
 	status = "okay";
 };
 
+&gpio1 {
+	gpio-line-names = "", "USER_LED", "I2C2_SCL", "I2C2_SDA";
+};
+
+&gpio2 {
+	gpio-line-names = "SPI6_CS0", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "TPM_nIRQ", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "I2C3_SDA", "I2C3_SCL";
+};
+
+&gpio3 {
+	gpio-line-names = "SD2_nCD", "", "", "", "",
+			  "", "", "SD2_nRESET", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "nENET1_INT";
+};
+
+&gpio4 {
+	gpio-line-names = "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "nCAN_EN", "", "", "",
+			  "", "", "", "RESET_PHY", "",
+			  "", "RTC_nINT", "PMIC_IRQ_B";
+};
+
 /* I2C2 */
 &lpi2c2 {
 	clock-frequency = <400000>;
@@ -339,8 +370,8 @@ MX93_PAD_SD2_CD_B__GPIO3_IO00		0x31e
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc2_default: usdhc2grp {
 		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x159e
-			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000178e
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x119e
+			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000138e
 			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x40001386
 			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x40001386
 			MX93_PAD_SD2_DATA2__USDHC2_DATA2	0x40001386
@@ -352,7 +383,7 @@ MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
 		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x159e
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x119e
 			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000139e
 			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x4000139e
 			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x4000139e
@@ -365,7 +396,7 @@ MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
 		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x159e
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x119e
 			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000139e
 			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x4000139e
 			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x4000139e
diff --git a/arch/arm64/boot/dts/freescale/imx93-phyboard-segin.dts b/arch/arm64/boot/dts/freescale/imx93-phyboard-segin.dts
index a982606de1ee4..d929aa9ff255a 100644
--- a/arch/arm64/boot/dts/freescale/imx93-phyboard-segin.dts
+++ b/arch/arm64/boot/dts/freescale/imx93-phyboard-segin.dts
@@ -145,6 +145,24 @@ &flexcan1 {
 	status = "okay";
 };
 
+&gpio1 {
+	gpio-line-names = "", "USER_LED", "I2C1_SCL", "I2C1_SDA";
+};
+
+&gpio3 {
+	gpio-line-names = "SD1_nCD", "", "", "", "",
+			  "", "", "SD1_nRESET";
+};
+
+&gpio4 {
+	gpio-line-names = "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "CAN_EN", "", "", "",
+			  "", "", "", "RESET_PHY", "",
+			  "", "RTC_nINT", "PMIC_IRQ_B";
+};
+
 /* I2C2 */
 &lpi2c2 {
 	clock-frequency = <400000>;
@@ -310,7 +328,7 @@ MX93_PAD_SD2_CD_B__GPIO3_IO00		0x31e
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc2_default: usdhc2grp {
 		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x159e
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x119e
 			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000139e
 			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x4000138e
 			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x4000138e
@@ -323,7 +341,7 @@ MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
 		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x159e
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x119e
 			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000139e
 			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x4000138e
 			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x4000138e
@@ -336,7 +354,7 @@ MX93_PAD_SD2_VSELECT__USDHC2_VSELECT	0x51e
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
 		fsl,pins = <
-			MX93_PAD_SD2_CLK__USDHC2_CLK		0x158e
+			MX93_PAD_SD2_CLK__USDHC2_CLK		0x118e
 			MX93_PAD_SD2_CMD__USDHC2_CMD		0x4000139e
 			MX93_PAD_SD2_DATA0__USDHC2_DATA0	0x4000139e
 			MX93_PAD_SD2_DATA1__USDHC2_DATA1	0x4000139e
diff --git a/arch/arm64/boot/dts/freescale/imx93-phycore-som.dtsi b/arch/arm64/boot/dts/freescale/imx93-phycore-som.dtsi
index ebc57841f27f5..325e465d00568 100644
--- a/arch/arm64/boot/dts/freescale/imx93-phycore-som.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx93-phycore-som.dtsi
@@ -9,6 +9,7 @@
  */
 
 #include <dt-bindings/leds/common.h>
+#include <dt-bindings/regulator/nxp,pca9450-regulator.h>
 
 #include "imx93.dtsi"
 
@@ -90,6 +91,28 @@ ethphy1: ethernet-phy@1 {
 	};
 };
 
+&gpio1 {
+	gpio-line-names = "", "USER_LED";
+};
+
+&gpio2 {
+	gpio-line-names = "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "I2C3_SDA", "I2C3_SCL";
+};
+
+&gpio4 {
+	gpio-line-names = "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "", "",
+			  "", "", "", "RESET_PHY", "",
+			  "", "", "PMIC_IRQ_B";
+};
+
 /* I2C3 */
 &lpi2c3 {
 	clock-frequency = <400000>;
@@ -140,6 +163,7 @@ buck5: BUCK5 {
 				regulator-max-microvolt = <1800000>;
 				regulator-boot-on;
 				regulator-always-on;
+				regulator-initial-mode = <PCA9450_BUCK_MODE_FORCE_PWM>;
 			};
 
 			buck6: BUCK6 {
@@ -194,6 +218,7 @@ &usdhc1 {
 	bus-width = <8>;
 	non-removable;
 	no-1-8-v;
+	fsl,strobe-dll-delay-target = <1>;
 	status = "okay";
 };
 
@@ -220,7 +245,7 @@ MX93_PAD_ENET2_TD0__ENET1_RGMII_TD0		0x50e
 			MX93_PAD_ENET2_TD1__ENET1_RGMII_TD1		0x50e
 			MX93_PAD_ENET2_TX_CTL__ENET1_RGMII_TX_CTL	0x50e
 			MX93_PAD_ENET2_TD2__ENET1_TX_CLK		0x4000050e
-			MX93_PAD_ENET2_RXC__GPIO4_IO23			0x51e
+			MX93_PAD_ENET2_RXC__GPIO4_IO23			0x502
 		>;
 	};
 
@@ -253,7 +278,7 @@ MX93_PAD_ENET2_RD3__GPIO4_IO27		0x31e
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc1: usdhc1grp {
 		fsl,pins = <
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x179e
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x119e
 			MX93_PAD_SD1_CMD__USDHC1_CMD		0x40001386
 			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x4000138e
 			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x40001386
@@ -263,14 +288,14 @@ MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x40001386
 			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x40001386
 			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x40001386
 			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x40001386
-			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x179e
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x159e
 		>;
 	};
 
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
 		fsl,pins = <
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x17be
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x11be
 			MX93_PAD_SD1_CMD__USDHC1_CMD		0x4000139e
 			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x4000138e
 			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x4000139e
@@ -280,14 +305,14 @@ MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x4000139e
 			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x4000139e
 			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x4000139e
 			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x4000139e
-			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x179e
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x159e
 		>;
 	};
 
 	/* need to config the SION for data and cmd pad, refer to ERR052021 */
 	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
 		fsl,pins = <
-			MX93_PAD_SD1_CLK__USDHC1_CLK		0x17be
+			MX93_PAD_SD1_CLK__USDHC1_CLK		0x11be
 			MX93_PAD_SD1_CMD__USDHC1_CMD		0x4000139e
 			MX93_PAD_SD1_DATA0__USDHC1_DATA0	0x4000139e
 			MX93_PAD_SD1_DATA1__USDHC1_DATA1	0x400013be
@@ -297,7 +322,7 @@ MX93_PAD_SD1_DATA4__USDHC1_DATA4	0x400013be
 			MX93_PAD_SD1_DATA5__USDHC1_DATA5	0x400013be
 			MX93_PAD_SD1_DATA6__USDHC1_DATA6	0x400013be
 			MX93_PAD_SD1_DATA7__USDHC1_DATA7	0x400013be
-			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x179e
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE	0x159e
 		>;
 	};
 
diff --git a/arch/arm64/boot/dts/freescale/imx93-var-dart-sonata.dts b/arch/arm64/boot/dts/freescale/imx93-var-dart-sonata.dts
new file mode 100644
index 0000000000000..5513d3b148a2f
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-var-dart-sonata.dts
@@ -0,0 +1,654 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Variscite Sonata carrier board for DART-MX93
+ *
+ * Link: https://variscite.com/carrier-boards/sonata-board/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/leds/common.h>
+#include "imx93-var-dart.dtsi"
+
+/ {
+	model = "Variscite DART-MX93 on Sonata-Board";
+	compatible = "variscite,var-dart-mx93-sonata",
+		     "variscite,var-dart-mx93",
+		     "fsl,imx93";
+
+	aliases {
+		ethernet0 = &eqos;
+		ethernet1 = &fec;
+		gpio0 = &gpio1;
+		gpio1 = &gpio2;
+		gpio2 = &gpio3;
+		i2c0 = &lpi2c1;
+		i2c1 = &lpi2c2;
+		i2c2 = &lpi2c3;
+		i2c3 = &lpi2c4;
+		i2c4 = &lpi2c5;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		serial0 = &lpuart1;
+		serial1 = &lpuart2;
+		serial2 = &lpuart3;
+		serial3 = &lpuart4;
+		serial4 = &lpuart5;
+		serial5 = &lpuart6;
+		serial6 = &lpuart7;
+	};
+
+	chosen {
+		stdout-path = &lpuart1;
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		button-home {
+			label = "Home";
+			linux,code = <KEY_HOME>;
+			gpios = <&pca6408_1 4 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-up {
+			label = "Up";
+			linux,code = <KEY_UP>;
+			gpios = <&pca6408_1 5 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-down {
+			label = "Down";
+			linux,code = <KEY_DOWN>;
+			gpios = <&pca6408_1 6 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+
+		button-back {
+			label = "Back";
+			linux,code = <KEY_BACK>;
+			gpios = <&pca6408_1 7 GPIO_ACTIVE_LOW>;
+			wakeup-source;
+		};
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_leds_gpio>;
+
+		led-emmc {
+			gpios = <&gpio2 11 GPIO_ACTIVE_HIGH>;
+			label = "eMMC";
+			linux,default-trigger = "mmc0";
+		};
+	};
+
+	clk40m: oscillator {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <40000000>;
+		clock-output-names = "can_osc";
+	};
+
+	reg_vref_1v8: regulator-adc-vref {
+		compatible = "regulator-fixed";
+		regulator-name = "vref_1v8";
+		regulator-max-microvolt = <1800000>;
+		regulator-min-microvolt = <1800000>;
+	};
+
+	reg_usdhc2_vmmc: regulator-vmmc-usdhc2 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_SD2_3V3";
+		off-on-delay-us = <20000>;
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&gpio2 18 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ethosu_mem: ethosu-region@88000000 {
+			compatible = "shared-dma-pool";
+			reusable;
+			reg = <0x0 0x88000000 0x0 0x8000000>;
+		};
+
+		vdev0vring0: vdev0vring0@87ee0000 {
+			reg = <0 0x87ee0000 0 0x8000>;
+			no-map;
+		};
+
+		vdev0vring1: vdev0vring1@87ee8000 {
+			reg = <0 0x87ee8000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring0: vdev1vring0@87ef0000 {
+			reg = <0 0x87ef0000 0 0x8000>;
+			no-map;
+		};
+
+		vdev1vring1: vdev1vring1@87ef8000 {
+			reg = <0 0x87ef8000 0 0x8000>;
+			no-map;
+		};
+
+		rsc_table: rsc-table@2021e000 {
+			reg = <0 0x2021e000 0 0x1000>;
+			no-map;
+		};
+
+		vdevbuffer: vdevbuffer@87f00000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x87f00000 0 0x100000>;
+			no-map;
+		};
+
+		ele_reserved: ele-reserved@87de0000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x87de0000 0 0x100000>;
+			no-map;
+		};
+	};
+};
+
+&adc1 {
+	vref-supply = <&reg_vref_1v8>;
+	status = "okay";
+};
+
+/* Use external instead of internal RTC */
+&bbnsm_rtc {
+	status = "disabled";
+};
+
+&eqos {
+	mdio {
+		ethphy1: ethernet-phy@1 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <1>;
+			reset-assert-us = <15000>;
+			reset-deassert-us = <100000>;
+			reset-gpios = <&pca6408_2 0 GPIO_ACTIVE_LOW>;
+
+			leds {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				led@0 {
+					reg = <0>;
+					color = <LED_COLOR_ID_YELLOW>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+
+				led@1 {
+					reg = <1>;
+					color = <LED_COLOR_ID_GREEN>;
+					function = LED_FUNCTION_LAN;
+					linux,default-trigger = "netdev";
+				};
+			};
+		};
+	};
+};
+
+&ethphy0 {
+	leds {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		led@0 {
+			reg = <0>;
+			color = <LED_COLOR_ID_YELLOW>;
+			function = LED_FUNCTION_LAN;
+			linux,default-trigger = "netdev";
+		};
+
+		led@1 {
+			reg = <1>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_LAN;
+			linux,default-trigger = "netdev";
+		};
+	};
+};
+
+&fec {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_fec>;
+	pinctrl-1 = <&pinctrl_fec_sleep>;
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy1>;
+	status = "okay";
+};
+
+&flexcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_flexcan1>;
+	status = "okay";
+};
+
+&lpi2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c1>;
+	pinctrl-1 = <&pinctrl_lpi2c1_gpio>;
+	pinctrl-2 = <&pinctrl_lpi2c1_gpio>;
+	pinctrl-names = "default", "sleep", "gpio";
+	scl-gpios = <&gpio1 0 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio1 1 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pca9534: gpio@22 {
+		compatible = "nxp,pca9534";
+		reg = <0x22>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <10 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	/* Capacitive touch controller */
+	ft5x06_ts: touchscreen@38 {
+		compatible = "edt,edt-ft5206";
+		reg = <0x38>;
+		interrupt-parent = <&gpio3>;
+		interrupts = <27 IRQ_TYPE_EDGE_FALLING>;
+		pinctrl-0 = <&pinctrl_captouch>;
+		pinctrl-names = "default";
+		reset-gpios = <&pca6408_2 4 GPIO_ACTIVE_LOW>;
+		touchscreen-inverted-x;
+		touchscreen-inverted-y;
+		touchscreen-size-x = <800>;
+		touchscreen-size-y = <480>;
+		wakeup-source;
+	};
+
+	/* USB Type-C Controller */
+	typec@3d {
+		compatible = "nxp,ptn5150";
+		reg = <0x3d>;
+		interrupt-parent = <&gpio4>;
+		interrupts = <29 IRQ_TYPE_LEVEL_HIGH>;
+		pinctrl-0 = <&pinctrl_extcon>;
+		pinctrl-names = "default";
+
+		port {
+			typec1_dr_sw: endpoint {
+				remote-endpoint = <&usb1_drd_sw>;
+			};
+		};
+	};
+
+	rtc@68 {
+		compatible = "dallas,ds1337";
+		reg = <0x68>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <2 IRQ_TYPE_EDGE_FALLING>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_rtc>;
+		wakeup-source;
+	};
+};
+
+&lpi2c5 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "sleep", "gpio";
+	pinctrl-0 = <&pinctrl_lpi2c5>;
+	pinctrl-1 = <&pinctrl_lpi2c5_gpio>;
+	pinctrl-2 = <&pinctrl_lpi2c5_gpio>;
+	scl-gpios = <&gpio2 23 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 22 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+};
+
+&lpi2c7 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "sleep", "gpio";
+	pinctrl-0 = <&pinctrl_lpi2c7>;
+	pinctrl-1 = <&pinctrl_lpi2c7_gpio>;
+	pinctrl-2 = <&pinctrl_lpi2c7_gpio>;
+	scl-gpios = <&gpio2 7 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 6 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	pca6408_1: gpio@20 {
+		compatible = "nxp,pcal6408";
+		reg = <0x20>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <10 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	pca6408_2: gpio@21 {
+		compatible = "nxp,pcal6408";
+		reg = <0x21>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <10 IRQ_TYPE_LEVEL_LOW>;
+	};
+
+	st33ktpm2xi2c: tpm@2e {
+		compatible = "st,st33ktpm2xi2c", "tcg,tpm-tis-i2c";
+		reg = <0x2e>;
+	};
+};
+
+&lpspi8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpspi8>;
+	cs-gpios = <&gpio2 1 GPIO_ACTIVE_LOW>;
+	status = "okay";
+
+	/* CAN controller */
+	can0: can@0 {
+		compatible = "microchip,mcp251xfd";
+		reg = <0>;
+		clocks = <&clk40m>;
+		interrupt-parent = <&gpio2>;
+		interrupts = <3 IRQ_TYPE_LEVEL_LOW>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_can>;
+		spi-max-frequency = <1000000>;
+	};
+};
+
+/* Console (J10) */
+&lpuart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+/* Header (J12.4, J12.6) */
+&lpuart6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart6>;
+	status = "okay";
+};
+
+/* Header (J12.11, J12.13) */
+&lpuart7 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart7>;
+	status = "okay";
+};
+
+&tpm3 {
+	pinctrl-0 = <&pinctrl_tpm3>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&usbotg1 {
+	dr_mode = "otg";
+	hnp-disable;
+	srp-disable;
+	adp-disable;
+	usb-role-switch;
+	disable-over-current;
+	samsung,picophy-pre-emp-curr-control = <3>;
+	samsung,picophy-dc-vol-level-adjust = <7>;
+	status = "okay";
+
+	port {
+		usb1_drd_sw: endpoint {
+			remote-endpoint = <&typec1_dr_sw>;
+		};
+	};
+};
+
+&usbotg2 {
+	disable-over-current;
+	dr_mode = "host";
+	status = "okay";
+};
+
+/* SD */
+&usdhc2 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	cd-gpios = <&gpio3 0 GPIO_ACTIVE_LOW>;
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	bus-width = <4>;
+	no-sdio;
+	no-mmc;
+	status = "okay";
+};
+
+&wdog3 {
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_hog>;
+
+	pinctrl_can: cangrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO03__GPIO2_IO03			0x31e
+		>;
+	};
+
+	pinctrl_captouch: captouchgrp {
+		fsl,pins = <
+			MX93_PAD_CCM_CLKO2__GPIO3_IO27			0x31e
+		>;
+	};
+
+	pinctrl_extcon: extcongrp {
+		fsl,pins = <
+			MX93_PAD_CCM_CLKO4__GPIO4_IO29			0x31e
+		>;
+	};
+
+	pinctrl_fec: fecgrp {
+		fsl,pins = <
+			MX93_PAD_ENET2_RD0__ENET1_RGMII_RD0		0x57e
+			MX93_PAD_ENET2_RD1__ENET1_RGMII_RD1		0x57e
+			MX93_PAD_ENET2_RD2__ENET1_RGMII_RD2		0x57e
+			MX93_PAD_ENET2_RD3__ENET1_RGMII_RD3		0x37e
+			MX93_PAD_ENET2_RXC__ENET1_RGMII_RXC		0x58e
+			MX93_PAD_ENET2_RX_CTL__ENET1_RGMII_RX_CTL	0x57e
+			MX93_PAD_ENET2_TD0__ENET1_RGMII_TD0		0x57e
+			MX93_PAD_ENET2_TD1__ENET1_RGMII_TD1		0x57e
+			MX93_PAD_ENET2_TD2__ENET1_RGMII_TD2		0x57e
+			MX93_PAD_ENET2_TD3__ENET1_RGMII_TD3		0x57e
+			MX93_PAD_ENET2_TXC__ENET1_RGMII_TXC		0x58e
+			MX93_PAD_ENET2_TX_CTL__ENET1_RGMII_TX_CTL	0x57e
+		>;
+	};
+
+	pinctrl_fec_sleep: fecsleepgrp {
+		fsl,pins = <
+			MX93_PAD_ENET2_RD0__GPIO4_IO24			0x51e
+			MX93_PAD_ENET2_RD1__GPIO4_IO25			0x51e
+			MX93_PAD_ENET2_RD2__GPIO4_IO26			0x51e
+			MX93_PAD_ENET2_RD3__GPIO4_IO27			0x51e
+			MX93_PAD_ENET2_RXC__GPIO4_IO23			0x51e
+			MX93_PAD_ENET2_RX_CTL__GPIO4_IO22		0x51e
+			MX93_PAD_ENET2_TD0__GPIO4_IO19			0x51e
+			MX93_PAD_ENET2_TD1__GPIO4_IO18			0x51e
+			MX93_PAD_ENET2_TD2__GPIO4_IO17			0x51e
+			MX93_PAD_ENET2_TD3__GPIO4_IO16			0x51e
+			MX93_PAD_ENET2_TXC__GPIO4_IO21			0x51e
+			MX93_PAD_ENET2_TX_CTL__GPIO4_IO20		0x51e
+		>;
+	};
+
+	pinctrl_flexcan1: flexcan1grp {
+		fsl,pins = <
+			MX93_PAD_PDM_CLK__CAN1_TX			0x139e
+			MX93_PAD_PDM_BIT_STREAM0__CAN1_RX		0x139e
+		>;
+	};
+
+	pinctrl_hog: hoggrp {
+		fsl,pins = <
+			/* GPIO Expanders shared IRQ */
+			MX93_PAD_PDM_BIT_STREAM1__GPIO1_IO10            0x31e
+		>;
+	};
+
+	pinctrl_leds_gpio: ledgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO11__GPIO2_IO11			0x31e
+		>;
+	};
+
+	pinctrl_lpi2c1: lpi2c1grp {
+		fsl,pins = <
+			MX93_PAD_I2C1_SCL__LPI2C1_SCL			0x40000b9e
+			MX93_PAD_I2C1_SDA__LPI2C1_SDA			0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c1_gpio: lpi2c1-gpiogrp {
+		fsl,pins = <
+			MX93_PAD_I2C1_SCL__GPIO1_IO00			0x31e
+			MX93_PAD_I2C1_SDA__GPIO1_IO01			0x31e
+		>;
+	};
+
+	pinctrl_lpi2c5: lpi2c5grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO22__LPI2C5_SDA			0x40000b9e
+			MX93_PAD_GPIO_IO23__LPI2C5_SCL			0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c5_gpio: lpi2c5-gpiogrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO22__GPIO2_IO22			0x31e
+			MX93_PAD_GPIO_IO23__GPIO2_IO23			0x31e
+		>;
+	};
+
+	pinctrl_lpi2c7: lpi2c7grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO07__LPI2C7_SCL			0x40000b9e
+			MX93_PAD_GPIO_IO06__LPI2C7_SDA			0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c7_gpio: lpi2c7-gpiogrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO07__GPIO2_IO07			0x31e
+			MX93_PAD_GPIO_IO06__GPIO2_IO06			0x31e
+		>;
+	};
+
+	pinctrl_lpspi8: lpspi8grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO00__GPIO2_IO00			0x31e
+			MX93_PAD_GPIO_IO01__GPIO2_IO01			0x31e
+			MX93_PAD_GPIO_IO12__GPIO2_IO12			0x31e
+			MX93_PAD_GPIO_IO13__LPSPI8_SIN			0x31e
+			MX93_PAD_GPIO_IO14__LPSPI8_SOUT			0x31e
+			MX93_PAD_GPIO_IO15__LPSPI8_SCK			0x31e
+		>;
+	};
+
+	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
+		fsl,pins = <
+			MX93_PAD_CCM_CLKO3__GPIO4_IO28			0x31e
+		>;
+	};
+
+	pinctrl_rtc: rtcgrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO02__GPIO2_IO02			0x31e
+		>;
+	};
+
+	pinctrl_tpm3: tpm3grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO24__TPM3_CH3			0x51e
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			MX93_PAD_UART1_RXD__LPUART1_RX			0x31e
+			MX93_PAD_UART1_TXD__LPUART1_TX			0x31e
+		>;
+	};
+
+	pinctrl_uart6: uart6grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO05__LPUART6_RX			0x31e
+			MX93_PAD_GPIO_IO04__LPUART6_TX			0x31e
+		>;
+	};
+
+	pinctrl_uart7: uart7grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO09__LPUART7_RX			0x31e
+			MX93_PAD_GPIO_IO08__LPUART7_TX			0x31e
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__USDHC2_CLK			0x1582
+			MX93_PAD_SD2_CMD__USDHC2_CMD			0x40001382
+			MX93_PAD_SD2_DATA0__USDHC2_DATA0		0x40001382
+			MX93_PAD_SD2_DATA1__USDHC2_DATA1		0x40001382
+			MX93_PAD_SD2_DATA2__USDHC2_DATA2		0x40001382
+			MX93_PAD_SD2_DATA3__USDHC2_DATA3		0x40001382
+			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT		0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__USDHC2_CLK			0x158e
+			MX93_PAD_SD2_CMD__USDHC2_CMD			0x4000138e
+			MX93_PAD_SD2_DATA0__USDHC2_DATA0		0x4000138e
+			MX93_PAD_SD2_DATA1__USDHC2_DATA1		0x4000138e
+			MX93_PAD_SD2_DATA2__USDHC2_DATA2		0x4000138e
+			MX93_PAD_SD2_DATA3__USDHC2_DATA3		0x4000138e
+			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT		0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CLK__USDHC2_CLK			0x15fe
+			MX93_PAD_SD2_CMD__USDHC2_CMD			0x400013fe
+			MX93_PAD_SD2_DATA0__USDHC2_DATA0		0x400013fe
+			MX93_PAD_SD2_DATA1__USDHC2_DATA1		0x400013fe
+			MX93_PAD_SD2_DATA2__USDHC2_DATA2		0x400013fe
+			MX93_PAD_SD2_DATA3__USDHC2_DATA3		0x400013fe
+			MX93_PAD_SD2_VSELECT__USDHC2_VSELECT		0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			MX93_PAD_SD2_CD_B__GPIO3_IO00			0x31e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx93-var-dart.dtsi b/arch/arm64/boot/dts/freescale/imx93-var-dart.dtsi
new file mode 100644
index 0000000000000..69495bb7fc9f5
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx93-var-dart.dtsi
@@ -0,0 +1,461 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Common dtsi for Variscite DART-MX93
+ *
+ * Link: https://variscite.com/system-on-module-som/i-mx-9/i-mx-93/dart-mx93/
+ *
+ * Copyright (C) 2026 Variscite Ltd. - https://www.variscite.com/
+ *
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/usb/pd.h>
+#include "imx93.dtsi"
+
+/ {
+	model = "Variscite DART-MX93 Module";
+	compatible = "variscite,var-dart-mx93", "fsl,imx93";
+
+	sound-wm8904 {
+		compatible = "simple-audio-card";
+		simple-audio-card,bitclock-master = <&codec_dai>;
+		simple-audio-card,format = "i2s";
+		simple-audio-card,frame-master = <&codec_dai>;
+		simple-audio-card,mclk-fs = <256>;
+		simple-audio-card,name = "wm8904-audio";
+		simple-audio-card,routing =
+			"Headphone Jack", "HPOUTL",
+			"Headphone Jack", "HPOUTR",
+			"IN2L", "Line In Jack",
+			"IN2R", "Line In Jack",
+			"IN1L", "Microphone Jack",
+			"IN1R", "Microphone Jack";
+		simple-audio-card,widgets =
+			"Microphone", "Microphone Jack",
+			"Headphone", "Headphone Jack",
+			"Line", "Line In Jack";
+
+		codec_dai: simple-audio-card,codec {
+			sound-dai = <&wm8904>;
+		};
+
+		simple-audio-card,cpu {
+			sound-dai = <&sai1>;
+		};
+	};
+
+	wifi_pwrseq: wifi-pwrseq {
+		compatible = "mmc-pwrseq-simple";
+		post-power-on-delay-ms = <100>;
+		power-off-delay-us = <10000>;
+		reset-gpios = <&gpio4 14 GPIO_ACTIVE_LOW>, /* WIFI_RESET */
+			      <&gpio3 7 GPIO_ACTIVE_LOW>; /* WIFI_PWR_EN */
+	};
+};
+
+&cm33 {
+	mbox-names = "tx", "rx", "rxdb";
+	mboxes = <&mu1 0 1>,
+		 <&mu1 1 1>,
+		 <&mu1 3 1>;
+	memory-region = <&vdevbuffer>, <&vdev0vring0>, <&vdev0vring1>,
+			<&vdev1vring0>, <&vdev1vring1>, <&rsc_table>;
+	status = "okay";
+};
+
+&eqos {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_eqos>;
+	pinctrl-1 = <&pinctrl_eqos_sleep>;
+	/*
+	 * The required RGMII TX and RX 2ns delays are implemented directly
+	 * in hardware via passive delay elements on the SOM PCB.
+	 * No delay configuration is needed in software via PHY driver.
+	 */
+	phy-mode = "rgmii";
+	phy-handle = <&ethphy0>;
+	snps,clk-csr = <5>;
+	status = "okay";
+
+	mdio {
+		compatible = "snps,dwmac-mdio";
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		ethphy0: ethernet-phy@0 {
+			compatible = "ethernet-phy-ieee802.3-c22";
+			reg = <0>;
+			reset-gpios = <&gpio1 7 GPIO_ACTIVE_LOW>;
+			reset-assert-us = <10000>;
+			reset-deassert-us = <100000>;
+		};
+	};
+};
+
+&lpi2c3 {
+	clock-frequency = <400000>;
+	pinctrl-names = "default", "sleep", "gpio";
+	pinctrl-0 = <&pinctrl_lpi2c3>;
+	pinctrl-1 = <&pinctrl_lpi2c3_gpio>;
+	pinctrl-2 = <&pinctrl_lpi2c3_gpio>;
+	scl-gpios = <&gpio2 29 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	sda-gpios = <&gpio2 28 (GPIO_ACTIVE_HIGH | GPIO_OPEN_DRAIN)>;
+	status = "okay";
+
+	wm8904: audio-codec@1a {
+		compatible = "wlf,wm8904";
+		reg = <0x1a>;
+		#sound-dai-cells = <0>;
+		clocks = <&clk IMX93_CLK_SAI1_GATE>;
+		clock-names = "mclk";
+		AVDD-supply = <&buck5>;
+		CPVDD-supply = <&buck5>;
+		DBVDD-supply = <&buck4>;
+		DCVDD-supply = <&buck5>;
+		MICVDD-supply = <&buck5>;
+		wlf,drc-cfg-names = "default", "peaklimiter", "tradition",
+				    "soft", "music";
+		/*
+		 * Config registers per name, respectively:
+		 * KNEE_IP = 0,   KNEE_OP = 0,     HI_COMP = 1,   LO_COMP = 1
+		 * KNEE_IP = -24, KNEE_OP = -6,    HI_COMP = 1/4, LO_COMP = 1
+		 * KNEE_IP = -42, KNEE_OP = -3,    HI_COMP = 0,   LO_COMP = 1
+		 * KNEE_IP = -45, KNEE_OP = -9,    HI_COMP = 1/8, LO_COMP = 1
+		 * KNEE_IP = -30, KNEE_OP = -10.5, HI_COMP = 1/4, LO_COMP = 1
+		 */
+		wlf,drc-cfg-regs = /bits/ 16 <0x01af 0x3248 0x0000 0x0000>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x0408>,
+				   /bits/ 16 <0x04af 0x324b 0x0028 0x0704>,
+				   /bits/ 16 <0x04af 0x324b 0x0018 0x078c>,
+				   /bits/ 16 <0x04af 0x324b 0x0010 0x050e>;
+		/* GPIO1 = DMIC_CLK, don't touch others */
+		wlf,gpio-cfg = <0x0018>, <0xffff>, <0xffff>, <0xffff>;
+		/* DMIC is connected to IN1L */
+		wlf,in1l-as-dmicdat1;
+	};
+
+	pmic@25 {
+		compatible = "nxp,pca9451a";
+		reg = <0x25>;
+
+		regulators {
+			buck1: BUCK1 {
+				regulator-name = "BUCK1";
+				regulator-min-microvolt = <650000>;
+				regulator-max-microvolt = <2237500>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck2: BUCK2 {
+				regulator-name = "BUCK2";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <2187500>;
+				regulator-boot-on;
+				regulator-always-on;
+				regulator-ramp-delay = <3125>;
+			};
+
+			buck4: BUCK4 {
+				regulator-name = "BUCK4";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck5: BUCK5 {
+				regulator-name = "BUCK5";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			buck6: BUCK6 {
+				regulator-name = "BUCK6";
+				regulator-min-microvolt = <600000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo1: LDO1 {
+				regulator-name = "LDO1";
+				regulator-min-microvolt = <1600000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo4: LDO4 {
+				regulator-name = "LDO4";
+				regulator-min-microvolt = <800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			ldo5: LDO5 {
+				regulator-name = "LDO5";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
+/* BT module */
+&lpuart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart5>, <&pinctrl_bt>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+	};
+};
+
+&mu1 {
+	status = "okay";
+};
+
+&mu2 {
+	status = "okay";
+};
+
+&sai1 {
+	pinctrl-names = "default", "sleep";
+	pinctrl-0 = <&pinctrl_sai1>;
+	pinctrl-1 = <&pinctrl_sai1_sleep>;
+	assigned-clocks = <&clk IMX93_CLK_SAI1>;
+	assigned-clock-parents = <&clk IMX93_CLK_AUDIO_PLL>;
+	assigned-clock-rates = <12288000>;
+	#sound-dai-cells = <0>;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+/* eMMC */
+&usdhc1 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz";
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	bus-width = <8>;
+	non-removable;
+	status = "okay";
+};
+
+/* WiFi */
+&usdhc3 {
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	pinctrl-0 = <&pinctrl_usdhc3>, <&pinctrl_usdhc3_wlan>;
+	pinctrl-1 = <&pinctrl_usdhc3_100mhz>, <&pinctrl_usdhc3_wlan>;
+	pinctrl-2 = <&pinctrl_usdhc3_200mhz>, <&pinctrl_usdhc3_wlan>;
+	pinctrl-3 = <&pinctrl_usdhc3_sleep>, <&pinctrl_usdhc3_wlan>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	keep-power-in-suspend;
+	bus-width = <4>;
+	non-removable;
+	wakeup-source;
+	status = "okay";
+};
+
+&iomuxc {
+	pinctrl_bt: btgrp {
+		fsl,pins = <
+			MX93_PAD_ENET2_MDIO__GPIO4_IO15				0x51e
+		>;
+	};
+
+	pinctrl_eqos: eqosgrp {
+		fsl,pins = <
+			MX93_PAD_ENET1_MDC__ENET_QOS_MDC			0x57e
+			MX93_PAD_ENET1_MDIO__ENET_QOS_MDIO			0x57e
+			MX93_PAD_ENET1_RD0__ENET_QOS_RGMII_RD0			0x57e
+			MX93_PAD_ENET1_RD1__ENET_QOS_RGMII_RD1			0x57e
+			MX93_PAD_ENET1_RD2__ENET_QOS_RGMII_RD2			0x57e
+			MX93_PAD_ENET1_RD3__ENET_QOS_RGMII_RD3			0x57e
+			MX93_PAD_ENET1_RXC__CCM_ENET_QOS_CLOCK_GENERATE_RX_CLK	0x58e
+			MX93_PAD_ENET1_RX_CTL__ENET_QOS_RGMII_RX_CTL		0x57e
+			MX93_PAD_ENET1_TD0__ENET_QOS_RGMII_TD0			0x57e
+			MX93_PAD_ENET1_TD1__ENET_QOS_RGMII_TD1			0x57e
+			MX93_PAD_ENET1_TD2__ENET_QOS_RGMII_TD2			0x57e
+			MX93_PAD_ENET1_TD3__ENET_QOS_RGMII_TD3			0x57e
+			MX93_PAD_ENET1_TXC__CCM_ENET_QOS_CLOCK_GENERATE_TX_CLK	0x58e
+			MX93_PAD_ENET1_TX_CTL__ENET_QOS_RGMII_TX_CTL		0x57e
+			MX93_PAD_UART2_TXD__GPIO1_IO07				0x51e
+		>;
+	};
+
+	pinctrl_eqos_sleep: eqos-sleepgrp {
+		fsl,pins = <
+			MX93_PAD_ENET1_MDC__GPIO4_IO00				0x31e
+			MX93_PAD_ENET1_MDIO__GPIO4_IO01				0x31e
+			MX93_PAD_ENET1_RD0__GPIO4_IO10				0x31e
+			MX93_PAD_ENET1_RD1__GPIO4_IO11				0x31e
+			MX93_PAD_ENET1_RD2__GPIO4_IO12				0x31e
+			MX93_PAD_ENET1_RD3__GPIO4_IO13				0x31e
+			MX93_PAD_ENET1_RXC__GPIO4_IO09				0x31e
+			MX93_PAD_ENET1_RX_CTL__GPIO4_IO08			0x31e
+			MX93_PAD_ENET1_TD0__GPIO4_IO05				0x31e
+			MX93_PAD_ENET1_TD1__GPIO4_IO04				0x31e
+			MX93_PAD_ENET1_TD2__GPIO4_IO03				0x31e
+			MX93_PAD_ENET1_TD3__GPIO4_IO02				0x31e
+			MX93_PAD_ENET1_TXC__GPIO4_IO07				0x31e
+			MX93_PAD_ENET1_TX_CTL__GPIO4_IO06			0x31e
+		>;
+	};
+
+	pinctrl_lpi2c3: lpi2c3grp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO28__LPI2C3_SDA				0x40000b9e
+			MX93_PAD_GPIO_IO29__LPI2C3_SCL				0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c3_gpio: lpi2c3gpiogrp {
+		fsl,pins = <
+			MX93_PAD_GPIO_IO28__GPIO2_IO28				0x40000b9e
+			MX93_PAD_GPIO_IO29__GPIO2_IO29				0x40000b9e
+		>;
+	};
+
+	pinctrl_sai1: sai1grp {
+		fsl,pins = <
+			MX93_PAD_SAI1_TXC__SAI1_TX_BCLK				0x31e
+			MX93_PAD_SAI1_TXFS__SAI1_TX_SYNC			0x31e
+			MX93_PAD_SAI1_TXD0__SAI1_TX_DATA00			0x31e
+			MX93_PAD_SAI1_RXD0__SAI1_RX_DATA00			0x31e
+			MX93_PAD_I2C2_SDA__SAI1_RX_BCLK				0x31e
+			MX93_PAD_I2C2_SCL__SAI1_RX_SYNC				0x31e
+			MX93_PAD_UART2_RXD__SAI1_MCLK				0x31e
+		>;
+	};
+
+	pinctrl_sai1_sleep: sai1-sleepgrp {
+		fsl,pins = <
+			MX93_PAD_SAI1_TXC__GPIO1_IO12				0x31e
+			MX93_PAD_SAI1_TXFS__GPIO1_IO11				0x31e
+			MX93_PAD_SAI1_TXD0__GPIO1_IO13				0x31e
+			MX93_PAD_SAI1_RXD0__GPIO1_IO14				0x31e
+			MX93_PAD_UART2_RXD__GPIO1_IO06				0x31e
+			MX93_PAD_I2C2_SDA__GPIO1_IO03				0x31e
+			MX93_PAD_I2C2_SCL__GPIO1_IO02				0x31e
+		>;
+	};
+
+	pinctrl_uart5: uart5grp {
+		fsl,pins = <
+			MX93_PAD_DAP_TDO_TRACESWO__LPUART5_TX			0x31e
+			MX93_PAD_DAP_TDI__LPUART5_RX				0x31e
+			MX93_PAD_DAP_TMS_SWDIO__LPUART5_RTS_B			0x31e
+			MX93_PAD_DAP_TCLK_SWCLK__LPUART5_CTS_B			0x31e
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			MX93_PAD_SD1_CLK__USDHC1_CLK				0x1582
+			MX93_PAD_SD1_CMD__USDHC1_CMD				0x40001382
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0			0x40001382
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1			0x40001382
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2			0x40001382
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3			0x40001382
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4			0x40001382
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5			0x40001382
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6			0x40001382
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7			0x40001382
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE			0x1582
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD1_CLK__USDHC1_CLK				0x158e
+			MX93_PAD_SD1_CMD__USDHC1_CMD				0x4000138e
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0			0x4000138e
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1			0x4000138e
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2			0x4000138e
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3			0x4000138e
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4			0x4000138e
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5			0x4000138e
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6			0x4000138e
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7			0x4000138e
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE			0x158e
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD1_CLK__USDHC1_CLK				0x15fe
+			MX93_PAD_SD1_CMD__USDHC1_CMD				0x400013fe
+			MX93_PAD_SD1_DATA0__USDHC1_DATA0			0x400013fe
+			MX93_PAD_SD1_DATA1__USDHC1_DATA1			0x400013fe
+			MX93_PAD_SD1_DATA2__USDHC1_DATA2			0x400013fe
+			MX93_PAD_SD1_DATA3__USDHC1_DATA3			0x400013fe
+			MX93_PAD_SD1_DATA4__USDHC1_DATA4			0x400013fe
+			MX93_PAD_SD1_DATA5__USDHC1_DATA5			0x400013fe
+			MX93_PAD_SD1_DATA6__USDHC1_DATA6			0x400013fe
+			MX93_PAD_SD1_DATA7__USDHC1_DATA7			0x400013fe
+			MX93_PAD_SD1_STROBE__USDHC1_STROBE			0x15fe
+		>;
+	};
+
+	pinctrl_usdhc3: usdhc3grp {
+		fsl,pins = <
+			MX93_PAD_SD3_CLK__USDHC3_CLK				0x1582
+			MX93_PAD_SD3_CMD__USDHC3_CMD				0x40001382
+			MX93_PAD_SD3_DATA0__USDHC3_DATA0			0x40001382
+			MX93_PAD_SD3_DATA1__USDHC3_DATA1			0x40001382
+			MX93_PAD_SD3_DATA2__USDHC3_DATA2			0x40001382
+			MX93_PAD_SD3_DATA3__USDHC3_DATA3			0x40001382
+		>;
+	};
+
+	pinctrl_usdhc3_100mhz: usdhc3-100mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD3_CLK__USDHC3_CLK				0x158e
+			MX93_PAD_SD3_CMD__USDHC3_CMD				0x4000138e
+			MX93_PAD_SD3_DATA0__USDHC3_DATA0			0x4000138e
+			MX93_PAD_SD3_DATA1__USDHC3_DATA1			0x4000138e
+			MX93_PAD_SD3_DATA2__USDHC3_DATA2			0x4000138e
+			MX93_PAD_SD3_DATA3__USDHC3_DATA3			0x4000138e
+		>;
+	};
+
+	pinctrl_usdhc3_200mhz: usdhc3-200mhzgrp {
+		fsl,pins = <
+			MX93_PAD_SD3_CLK__USDHC3_CLK				0x15fe
+			MX93_PAD_SD3_CMD__USDHC3_CMD				0x400013fe
+			MX93_PAD_SD3_DATA0__USDHC3_DATA0			0x400013fe
+			MX93_PAD_SD3_DATA1__USDHC3_DATA1			0x400013fe
+			MX93_PAD_SD3_DATA2__USDHC3_DATA2			0x400013fe
+			MX93_PAD_SD3_DATA3__USDHC3_DATA3			0x400013fe
+		>;
+	};
+
+	pinctrl_usdhc3_sleep: usdhc3-sleepgrp {
+		fsl,pins = <
+			MX93_PAD_SD3_CLK__GPIO3_IO20				0x400
+			MX93_PAD_SD3_CMD__GPIO3_IO21				0x400
+			MX93_PAD_SD3_DATA0__GPIO3_IO22				0x400
+			MX93_PAD_SD3_DATA1__GPIO3_IO23				0x400
+			MX93_PAD_SD3_DATA2__GPIO3_IO24				0x400
+			MX93_PAD_SD3_DATA3__GPIO3_IO25				0x400
+		>;
+	};
+
+	pinctrl_usdhc3_wlan: usdhc3wlangrp {
+		fsl,pins = <
+			MX93_PAD_ENET2_MDC__GPIO4_IO14				0x51e
+			MX93_PAD_SD2_RESET_B__GPIO3_IO07			0x51e
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx94.dtsi b/arch/arm64/boot/dts/freescale/imx94.dtsi
index c460ece6070f8..589f4aadc742f 100644
--- a/arch/arm64/boot/dts/freescale/imx94.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx94.dtsi
@@ -67,6 +67,13 @@ sai4_mclk: clock-sai4-mclk1 {
 		clock-output-names = "sai4_mclk";
 	};
 
+	clk_sys100m: clock-sys100m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+		clock-output-names = "clk_sys100m";
+	};
+
 	firmware {
 		scmi {
 			compatible = "arm,scmi";
@@ -1286,6 +1293,14 @@ wdog3: watchdog@49220000 {
 			};
 		};
 
+		hsio_blk_ctl: syscon@4c0100c0 {
+			compatible = "nxp,imx95-hsio-blk-ctl", "syscon";
+			reg = <0x0 0x4c0100c0 0x0 0x1>;
+			#clock-cells = <1>;
+			clocks = <&clk_sys100m>;
+			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
+		};
+
 		usb3: usb@4c100000 {
 			compatible = "nxp,imx94-dwc3", "nxp,imx8mp-dwc3";
 			reg = <0x0 0x4c100000 0x0 0x10000>,
@@ -1340,6 +1355,79 @@ usbmisc: usbmisc@4c200200 {
 			#index-cells = <1>;
 		};
 
+		pcie0: pcie@4c300000 {
+			compatible = "fsl,imx94-pcie", "fsl,imx95-pcie";
+			reg = <0 0x4c300000 0 0x10000>,
+			      <0 0x60100000 0 0xfe00000>,
+			      <0 0x4c360000 0 0x10000>,
+			      <0 0x4c340000 0 0x4000>;
+			reg-names = "dbi", "config", "atu", "app";
+			ranges = <0x81000000 0x0 0x00000000 0x0 0x6ff00000 0 0x00100000>,
+				 <0x82000000 0x0 0x10000000 0x9 0x10000000 0 0x80000000>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			linux,pci-domain = <2>;
+			msi-map = <0x0 &its 0x10 0x1>,
+				  <0x100 &its 0x11 0x7>;
+			msi-map-mask = <0x1ff>;
+			bus-range = <0x00 0xff>;
+			num-lanes = <1>;
+			num-viewport = <8>;
+			interrupts = <GIC_SPI 364 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 365 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi", "dma";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &gic 0 0 GIC_SPI 360 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &gic 0 0 GIC_SPI 361 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gic 0 0 GIC_SPI 362 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gic 0 0 GIC_SPI 363 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&scmi_clk IMX94_CLK_HSIO>,
+				 <&scmi_clk IMX94_CLK_HSIOPLL>,
+				 <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+				 <&scmi_clk IMX94_CLK_HSIOPCIEAUX>,
+				 <&hsio_blk_ctl 0>;
+			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux", "ref";
+			assigned-clocks = <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+					  <&scmi_clk IMX94_CLK_HSIOPLL>,
+					  <&scmi_clk IMX94_CLK_HSIOPCIEAUX>;
+			assigned-clock-rates = <3600000000>, <100000000>, <10000000>;
+			assigned-clock-parents = <0>, <0>,
+						 <&scmi_clk IMX94_CLK_SYSPLL1_PFD1_DIV2>;
+			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
+			fsl,max-link-speed = <3>;
+			status = "disabled";
+		};
+
+		pcie0_ep: pcie-ep@4c300000 {
+			compatible = "fsl,imx94-pcie-ep", "fsl,imx95-pcie-ep";
+			reg = <0 0x4c300000 0 0x10000>,
+			      <0 0x4c360000 0 0x1000>,
+			      <0 0x4c320000 0 0x1000>,
+			      <0 0x4c340000 0 0x4000>,
+			      <0 0x4c370000 0 0x10000>,
+			      <0x9 0 1 0>;
+			reg-names = "dbi", "atu", "dbi2", "app", "dma", "addr_space";
+			num-lanes = <1>;
+			interrupts = <GIC_SPI 365 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dma";
+			clocks = <&scmi_clk IMX94_CLK_HSIO>,
+				 <&scmi_clk IMX94_CLK_HSIOPLL>,
+				 <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+				 <&scmi_clk IMX94_CLK_HSIOPCIEAUX>;
+			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux";
+			assigned-clocks = <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+					  <&scmi_clk IMX94_CLK_HSIOPLL>,
+					  <&scmi_clk IMX94_CLK_HSIOPCIEAUX>;
+			assigned-clock-rates = <3600000000>, <100000000>, <10000000>;
+			assigned-clock-parents = <0>, <0>,
+						 <&scmi_clk IMX94_CLK_SYSPLL1_PFD1_DIV2>;
+			msi-map = <0x0 &its 0x10 0x1>;
+			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
+			status = "disabled";
+		};
+
 		netc_blk_ctrl: system-controller@4ceb0000 {
 			compatible = "nxp,imx94-netc-blk-ctrl";
 			reg = <0x0 0x4ceb0000 0x0 0x10000>,
diff --git a/arch/arm64/boot/dts/freescale/imx943-evk.dts b/arch/arm64/boot/dts/freescale/imx943-evk.dts
index 52f7ef7dbf272..fe4fc512d95d4 100644
--- a/arch/arm64/boot/dts/freescale/imx943-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx943-evk.dts
@@ -58,6 +58,20 @@ chosen {
 		stdout-path = &lpuart1;
 	};
 
+	pcie_ref_clk: clock-pcie-ref {
+		compatible = "gpio-gate-clock";
+		clocks = <&xtal25m>;
+		#clock-cells = <0>;
+		enable-gpios = <&pca9670_i2c3 7 GPIO_ACTIVE_LOW>;
+	};
+
+	xtal25m: clock-xtal25m {
+		compatible = "fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <25000000>;
+		clock-output-names = "xtal_25MHz";
+	};
+
 	dmic: dmic {
 		compatible = "dmic-codec";
 		#sound-dai-cells = <0>;
@@ -79,6 +93,15 @@ reg_m2_pwr: regulator-m2-pwr {
 		startup-delay-us = <5000>;
 	};
 
+	reg_slot_pwr: regulator-slot-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "PCIe slot-power";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		gpio = <&pcal6416_i2c3_u46 0 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
 	reg_m2_wlan: regulator-wlan {
 		compatible = "regulator-fixed";
 		regulator-name = "WLAN_EN";
@@ -725,6 +748,18 @@ IMX94_PAD_GPIO_IO28__LPI2C6_SCL		0x40000b9e
 		>;
 	};
 
+	pinctrl_pcie0: pcie0grp {
+		fsl,pins = <
+			IMX94_PAD_GPIO_IO20__PCIE1_CLKREQ_B	0x4000031e
+		>;
+	};
+
+	pinctrl_pcie1: pcie1grp {
+		fsl,pins = <
+			IMX94_PAD_GPIO_IO23__PCIE2_CLKREQ_B	0x4000031e
+		>;
+	};
+
 	pinctrl_pdm: pdmgrp {
 		fsl,pins = <
 			IMX94_PAD_PDM_CLK__PDM_CLK			0x31e
@@ -988,6 +1023,54 @@ pf5302_crit: trip1 {
 	};
 };
 
+&pcie0 {
+	pinctrl-0 = <&pinctrl_pcie0>;
+	pinctrl-names = "default";
+	clocks = <&scmi_clk IMX94_CLK_HSIO>,
+		 <&scmi_clk IMX94_CLK_HSIOPLL>,
+		 <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+		 <&scmi_clk IMX94_CLK_HSIOPCIEAUX>,
+		 <&hsio_blk_ctl 0>,
+		 <&pcie_ref_clk>;
+	clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux",
+		      "ref", "extref";
+	reset-gpio = <&pcal6416_i2c3_u46 3 GPIO_ACTIVE_LOW>;
+	vpcie3v3aux-supply = <&reg_m2_wlan>;
+	supports-clkreq;
+	status = "okay";
+};
+
+&pcie0_ep {
+	pinctrl-0 = <&pinctrl_pcie0>;
+	pinctrl-names = "default";
+	vpcie-supply = <&reg_m2_wlan>;
+	status = "disabled";
+};
+
+&pcie1 {
+	pinctrl-0 = <&pinctrl_pcie1>;
+	pinctrl-names = "default";
+	clocks = <&scmi_clk IMX94_CLK_HSIO>,
+		 <&scmi_clk IMX94_CLK_HSIOPLL>,
+		 <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+		 <&scmi_clk IMX94_CLK_HSIOPCIEAUX>,
+		 <&hsio_blk_ctl 0>,
+		 <&pcie_ref_clk>;
+	clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux",
+		      "ref", "extref";
+	reset-gpio = <&pcal6416_i2c3_u46 1 GPIO_ACTIVE_LOW>;
+	vpcie3v3aux-supply = <&reg_slot_pwr>;
+	supports-clkreq;
+	status = "okay";
+};
+
+&pcie1_ep {
+	pinctrl-0 = <&pinctrl_pcie1>;
+	pinctrl-names = "default";
+	vpcie-supply = <&reg_slot_pwr>;
+	status = "disabled";
+};
+
 &usb2 {
 	dr_mode = "otg";
 	disable-over-current;
diff --git a/arch/arm64/boot/dts/freescale/imx943.dtsi b/arch/arm64/boot/dts/freescale/imx943.dtsi
index dfd956ece2e38..ed030d4bc7bd9 100644
--- a/arch/arm64/boot/dts/freescale/imx943.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx943.dtsi
@@ -209,4 +209,79 @@ map0 {
 			};
 		};
 	};
+
+	soc {
+		pcie1: pcie@4c380000 {
+			compatible = "fsl,imx943-pcie", "fsl,imx95-pcie";
+			reg = <0 0x4c380000 0 0x10000>,
+			      <8 0x80100000 0 0xfe00000>,
+			      <0 0x4c3e0000 0 0x10000>,
+			      <0 0x4c3c0000 0 0x4000>;
+			reg-names = "dbi", "config", "atu", "app";
+			ranges = <0x81000000 0 0x00000000 0x8 0x8ff00000 0 0x00100000>,
+				 <0x82000000 0 0x10000000 0xa 0x10000000 0 0x80000000>;
+			#address-cells = <3>;
+			#size-cells = <2>;
+			device_type = "pci";
+			linux,pci-domain = <3>;
+			msi-map = <0x0 &its 0x98 0x1>,
+				  <0x100 &its 0x99 0x7>;
+			msi-map-mask = <0x1ff>;
+			bus-range = <0x00 0xff>;
+			num-lanes = <1>;
+			num-viewport = <8>;
+			interrupts = <GIC_SPI 370 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 371 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi", "dma";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &gic 0 0 GIC_SPI 366 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &gic 0 0 GIC_SPI 367 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &gic 0 0 GIC_SPI 368 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &gic 0 0 GIC_SPI 369 IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&scmi_clk IMX94_CLK_HSIO>,
+				 <&scmi_clk IMX94_CLK_HSIOPLL>,
+				 <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+				 <&scmi_clk IMX94_CLK_HSIOPCIEAUX>,
+				 <&hsio_blk_ctl 0>;
+			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux", "ref";
+			assigned-clocks = <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+					  <&scmi_clk IMX94_CLK_HSIOPLL>,
+					  <&scmi_clk IMX94_CLK_HSIOPCIEAUX>;
+			assigned-clock-rates = <3600000000>, <100000000>, <10000000>;
+			assigned-clock-parents = <0>, <0>,
+						 <&scmi_clk IMX94_CLK_SYSPLL1_PFD1_DIV2>;
+			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
+			fsl,max-link-speed = <3>;
+			status = "disabled";
+		};
+
+		pcie1_ep: pcie-ep@4c380000 {
+			compatible = "fsl,imx943-pcie-ep", "fsl,imx95-pcie-ep";
+			reg = <0 0x4c380000 0 0x10000>,
+			      <0 0x4c3e0000 0 0x1000>,
+			      <0 0x4c3a0000 0 0x1000>,
+			      <0 0x4c3c0000 0 0x4000>,
+			      <0 0x4c3f0000 0 0x10000>,
+			      <0xa 0 1 0>;
+			reg-names = "dbi", "atu", "dbi2", "app", "dma", "addr_space";
+			num-lanes = <1>;
+			interrupts = <GIC_SPI 371 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dma";
+			clocks = <&scmi_clk IMX94_CLK_HSIO>,
+				 <&scmi_clk IMX94_CLK_HSIOPLL>,
+				 <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+				 <&scmi_clk IMX94_CLK_HSIOPCIEAUX>;
+			clock-names = "pcie", "pcie_bus", "pcie_phy", "pcie_aux";
+			assigned-clocks = <&scmi_clk IMX94_CLK_HSIOPLL_VCO>,
+					  <&scmi_clk IMX94_CLK_HSIOPLL>,
+					  <&scmi_clk IMX94_CLK_HSIOPCIEAUX>;
+			assigned-clock-rates = <3600000000>, <100000000>, <10000000>;
+			assigned-clock-parents = <0>, <0>,
+						 <&scmi_clk IMX94_CLK_SYSPLL1_PFD1_DIV2>;
+			msi-map = <0x0 &its 0x98 0x1>;
+			power-domains = <&scmi_devpd IMX94_PD_HSIO_TOP>;
+			status = "disabled";
+		};
+	};
 };
diff --git a/arch/arm64/boot/dts/freescale/imx95-15x15-ab2.dts b/arch/arm64/boot/dts/freescale/imx95-15x15-ab2.dts
new file mode 100644
index 0000000000000..f6829145f396d
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx95-15x15-ab2.dts
@@ -0,0 +1,669 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
+/*
+ * Copyright 2024 NXP
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/usb/pd.h>
+#include "imx95.dtsi"
+
+#define BRD_SM_CTRL_SD3_WAKE		0x8000
+#define BRD_SM_CTRL_PCIE1_WAKE		0x8001
+#define BRD_SM_CTRL_BT_WAKE		0x8002
+#define BRD_SM_CTRL_PCIE2_WAKE		0x8003
+#define BRD_SM_CTRL_BUTTON		0x8004
+
+/ {
+	compatible = "fsl,imx95-15x15-ab2", "fsl,imx95";
+	model = "NXP i.MX95 CPU on AB2";
+
+	aliases {
+		ethernet0 = &enetc_port0;
+		gpio0 = &gpio1;
+		gpio1 = &gpio2;
+		gpio2 = &gpio3;
+		gpio3 = &gpio4;
+		gpio4 = &gpio5;
+		i2c0 = &lpi2c1;
+		i2c1 = &lpi2c2;
+		i2c2 = &lpi2c3;
+		i2c3 = &lpi2c4;
+		mmc0 = &usdhc1;
+		mmc1 = &usdhc2;
+		serial0 = &lpuart1;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		stdout-path = &lpuart1;
+	};
+
+	reg_ab2_ana_pwr: regulator-ab2-ana-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "ab2_ana_pwr";
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&pcal6524 1 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_ab2_vdd_pwr_5v0: regulator-ab2-vdd-pwr-5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "ab2_vdd_pwr_5v0";
+		regulator-always-on;
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&pcal6524 2 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reg_usdhc2_vmmc: regulator-usdhc2 {
+		compatible = "regulator-fixed";
+		regulator-name = "VDD_SD2_3V3";
+		off-on-delay-us = <12000>;
+		pinctrl-0 = <&pinctrl_reg_usdhc2_vmmc>;
+		pinctrl-names = "default";
+		regulator-max-microvolt = <3300000>;
+		regulator-min-microvolt = <3300000>;
+		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	reserved-memory {
+		ranges;
+		#address-cells = <2>;
+		#size-cells = <2>;
+
+		linux_cma: linux,cma {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0 0x80000000 0 0x7f000000>;
+			reusable;
+			size = <0 0x3c000000>;
+			linux,cma-default;
+		};
+
+		vpu_boot: vpu-boot@a0000000 {
+			reg = <0 0xa0000000 0 0x100000>;
+			no-map;
+		};
+	};
+
+	sound-ak4458 {
+		compatible = "fsl,imx-audio-card";
+		model = "ak4458-audio";
+
+		pri-dai-link {
+			format = "i2s";
+			link-name = "akcodec";
+			fsl,mclk-equal-bclk;
+
+			codec {
+				sound-dai = <&ak4458_1>, <&ak4458_2>;
+			};
+
+			cpu {
+				sound-dai = <&sai2>;
+			};
+		};
+	};
+
+	sound-ak5552 {
+		compatible = "fsl,imx-audio-card";
+		model = "ak5552-audio";
+
+		pri-dai-link {
+			format = "i2s";
+			link-name = "akcodec";
+			fsl,mclk-equal-bclk;
+
+			codec {
+				sound-dai = <&ak5552>;
+			};
+
+			cpu {
+				sound-dai = <&sai3>;
+			};
+		};
+	};
+
+	sound-micfil {
+		compatible = "fsl,imx-audio-card";
+		model = "micfil-audio";
+
+		pri-dai-link {
+			format = "i2s";
+			link-name = "micfil hifi";
+
+			cpu {
+				sound-dai = <&micfil>;
+			};
+		};
+	};
+
+	sound-xcvr {
+		compatible = "fsl,imx-audio-card";
+		model = "imx-audio-xcvr";
+
+		pri-dai-link {
+			link-name = "XCVR PCM";
+
+			cpu {
+				sound-dai = <&xcvr>;
+			};
+		};
+	};
+
+	memory@80000000 {
+		reg = <0x0 0x80000000 0 0x80000000>;
+		device_type = "memory";
+	};
+};
+
+&enetc_port0 {
+	phy-handle = <&ethphy0>;
+	phy-mode = "rgmii-id";
+	pinctrl-0 = <&pinctrl_enetc0>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&lpi2c1 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c1>;
+	pinctrl-names = "default";
+	status = "disabled";
+};
+
+&lpi2c2 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c2>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pcal6524: gpio@22 {
+		compatible = "nxp,pcal6524";
+		reg = <0x22>;
+		#interrupt-cells = <2>;
+		interrupt-controller;
+		interrupt-parent = <&gpio5>;
+		interrupts = <14 IRQ_TYPE_LEVEL_LOW>;
+		#gpio-cells = <2>;
+		gpio-controller;
+		pinctrl-0 = <&pinctrl_pcal6524>;
+		pinctrl-names = "default";
+	};
+};
+
+&lpi2c3 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c3>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	ak4458_1: audio-codec@10 {
+		compatible = "asahi-kasei,ak4458";
+		reg = <0x10>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_ab2_ana_pwr>;
+		DVDD-supply = <&reg_ab2_ana_pwr>;
+		reset-gpios = <&pca6416 4 GPIO_ACTIVE_LOW>;
+		sound-name-prefix = "0";
+	};
+
+	ak4458_3: audio-codec@11 {
+		compatible = "asahi-kasei,ak4458";
+		reg = <0x11>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_ab2_ana_pwr>;
+		DVDD-supply = <&reg_ab2_ana_pwr>;
+		reset-gpios = <&pca6416 4 GPIO_ACTIVE_LOW>;
+		status = "disabled";
+	};
+
+	ak4458_2: audio-codec@12 {
+		compatible = "asahi-kasei,ak4458";
+		reg = <0x12>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_ab2_ana_pwr>;
+		DVDD-supply = <&reg_ab2_ana_pwr>;
+		reset-gpios = <&pca6416 4 GPIO_ACTIVE_LOW>;
+		sound-name-prefix = "1";
+	};
+
+	ak5552: audio-codec@13 {
+		compatible = "asahi-kasei,ak5552";
+		reg = <0x13>;
+		#sound-dai-cells = <0>;
+		AVDD-supply = <&reg_ab2_ana_pwr>;
+		DVDD-supply = <&reg_ab2_ana_pwr>;
+		reset-gpios = <&pca6416 3 GPIO_ACTIVE_LOW>;
+	};
+
+	pca6416: gpio@20 {
+		compatible = "ti,tca6416";
+		reg = <0x20>;
+		#gpio-cells = <2>;
+		gpio-controller;
+	};
+};
+
+&lpi2c4 {
+	clock-frequency = <400000>;
+	pinctrl-0 = <&pinctrl_lpi2c4>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&lpuart1 {
+	/* console */
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart1>;
+	status = "okay";
+};
+
+&micfil {
+	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX95_CLK_PDM>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				 <&scmi_clk IMX95_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>,
+			       <3612672000>, <393216000>,
+			       <361267200>, <49152000>;
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_pdm>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&netc_blk_ctrl {
+	status = "okay";
+};
+
+&netc_emdio {
+	pinctrl-0 = <&pinctrl_emdio>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	ethphy0: ethernet-phy@1 {
+		reg = <1>;
+		reset-assert-us = <10000>;
+		reset-deassert-us = <80000>;
+		reset-gpios = <&pcal6524 4 GPIO_ACTIVE_LOW>;
+		realtek,clkout-disable;
+	};
+};
+
+&netcmix_blk_ctrl {
+	status = "okay";
+};
+
+&sai2 {
+	clocks = <&scmi_clk IMX95_CLK_BUSNETCMIX>, <&dummy>,
+		 <&scmi_clk IMX95_CLK_SAI2>, <&dummy>,
+		 <&dummy>, <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+		 <&scmi_clk IMX95_CLK_AUDIOPLL2>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k", "pll11k";
+	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX95_CLK_SAI2>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				 <&scmi_clk IMX95_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>,
+			       <3612672000>, <393216000>,
+			       <361267200>, <12288000>;
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_sai2>;
+	pinctrl-names = "default";
+	fsl,sai-asynchronous;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&sai3 {
+	clocks = <&scmi_clk IMX95_CLK_BUSNETCMIX>, <&dummy>,
+		 <&scmi_clk IMX95_CLK_SAI3>, <&dummy>,
+		 <&dummy>, <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+		 <&scmi_clk IMX95_CLK_AUDIOPLL2>;
+	clock-names = "bus", "mclk0", "mclk1", "mclk2", "mclk3", "pll8k", "pll11k";
+	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX95_CLK_SAI3>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				 <&scmi_clk IMX95_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>,
+			       <3612672000>, <393216000>,
+			       <361267200>, <12288000>;
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_sai3>;
+	pinctrl-names = "default";
+	fsl,sai-asynchronous;
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&sai5 {
+	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX95_CLK_SAI5>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				 <&scmi_clk IMX95_CLK_AUDIOPLL1>;
+	assigned-clock-rates = <3932160000>,
+			       <3612672000>, <393216000>,
+			       <361267200>, <12288000>;
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_sai5>;
+	pinctrl-names = "default";
+	fsl,sai-mclk-direction-output;
+	status = "okay";
+};
+
+&scmi_iomuxc {
+	pinctrl_emdio: emdiogrp {
+		fsl,pins = <
+			IMX95_PAD_ENET1_MDC__NETCMIX_TOP_NETC_MDC		0x50e
+			IMX95_PAD_ENET1_MDIO__NETCMIX_TOP_NETC_MDIO		0x90e
+		>;
+	};
+
+	pinctrl_enetc0: enetc0grp {
+		fsl,pins = <
+			IMX95_PAD_ENET1_TD3__NETCMIX_TOP_ETH0_RGMII_TD3		0x50e
+			IMX95_PAD_ENET1_TD2__NETCMIX_TOP_ETH0_RGMII_TD2		0x50e
+			IMX95_PAD_ENET1_TD1__NETCMIX_TOP_ETH0_RGMII_TD1		0x50e
+			IMX95_PAD_ENET1_TD0__NETCMIX_TOP_ETH0_RGMII_TD0		0x50e
+			IMX95_PAD_ENET1_TX_CTL__NETCMIX_TOP_ETH0_RGMII_TX_CTL	0x57e
+			IMX95_PAD_ENET1_TXC__NETCMIX_TOP_ETH0_RGMII_TX_CLK	0x58e
+			IMX95_PAD_ENET1_RX_CTL__NETCMIX_TOP_ETH0_RGMII_RX_CTL	0x57e
+			IMX95_PAD_ENET1_RXC__NETCMIX_TOP_ETH0_RGMII_RX_CLK	0x58e
+			IMX95_PAD_ENET1_RD0__NETCMIX_TOP_ETH0_RGMII_RD0		0x57e
+			IMX95_PAD_ENET1_RD1__NETCMIX_TOP_ETH0_RGMII_RD1		0x57e
+			IMX95_PAD_ENET1_RD2__NETCMIX_TOP_ETH0_RGMII_RD2		0x57e
+			IMX95_PAD_ENET1_RD3__NETCMIX_TOP_ETH0_RGMII_RD3		0x57e
+		>;
+	};
+
+	pinctrl_lpi2c1: lpi2c1grp {
+		fsl,pins = <
+			IMX95_PAD_I2C1_SCL__AONMIX_TOP_LPI2C1_SCL	0x40000b9e
+			IMX95_PAD_I2C1_SDA__AONMIX_TOP_LPI2C1_SDA	0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c2: lpi2c2grp {
+		fsl,pins = <
+			IMX95_PAD_I2C2_SCL__AONMIX_TOP_LPI2C2_SCL	0x40000b9e
+			IMX95_PAD_I2C2_SDA__AONMIX_TOP_LPI2C2_SDA	0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c3: lpi2c3grp {
+		fsl,pins = <
+			IMX95_PAD_GPIO_IO28__LPI2C3_SDA			0x40000b9e
+			IMX95_PAD_GPIO_IO29__LPI2C3_SCL			0x40000b9e
+		>;
+	};
+
+	pinctrl_lpi2c4: lpi2c4grp {
+		fsl,pins = <
+			IMX95_PAD_GPIO_IO30__LPI2C4_SDA			0x40000b9e
+			IMX95_PAD_GPIO_IO31__LPI2C4_SCL			0x40000b9e
+		>;
+	};
+
+	pinctrl_pcal6524: pcal6524grp {
+		fsl,pins = <
+			IMX95_PAD_GPIO_IO34__GPIO5_IO_BIT14		0x31e
+		>;
+	};
+
+	pinctrl_pdm: pdmgrp {
+		fsl,pins = <
+			IMX95_PAD_PDM_CLK__AONMIX_TOP_PDM_CLK				0x31e
+			IMX95_PAD_PDM_BIT_STREAM0__AONMIX_TOP_PDM_BIT_STREAM_BIT0	0x31e
+			IMX95_PAD_GPIO_IO06__AONMIX_TOP_PDM_BIT_STREAM_BIT1		0x31e
+			IMX95_PAD_GPIO_IO12__AONMIX_TOP_PDM_BIT_STREAM_BIT2		0x31e
+			IMX95_PAD_GPIO_IO13__AONMIX_TOP_PDM_BIT_STREAM_BIT3		0x31e
+		>;
+	};
+
+	pinctrl_reg_usdhc2_vmmc: regusdhc2vmmcgrp {
+		fsl,pins = <
+			IMX95_PAD_SD2_RESET_B__GPIO3_IO_BIT7			0x31e
+		>;
+	};
+
+	pinctrl_sai2: sai2grp {
+		fsl,pins = <
+			IMX95_PAD_ENET2_RD2__NETCMIX_TOP_SAI2_MCLK			0x31e
+			IMX95_PAD_ENET2_MDIO__NETCMIX_TOP_SAI2_RX_BCLK			0x31e
+			IMX95_PAD_ENET2_MDC__NETCMIX_TOP_SAI2_RX_SYNC			0x31e
+			IMX95_PAD_ENET2_TD3__NETCMIX_TOP_SAI2_RX_DATA_BIT0		0x31e
+			IMX95_PAD_ENET2_TD2__NETCMIX_TOP_SAI2_RX_DATA_BIT1		0x31e
+			IMX95_PAD_ENET2_TD1__NETCMIX_TOP_SAI2_RX_DATA_BIT2		0x31e
+			IMX95_PAD_ENET2_TD0__NETCMIX_TOP_SAI2_RX_DATA_BIT3		0x31e
+			IMX95_PAD_ENET2_TXC__NETCMIX_TOP_SAI2_TX_BCLK			0x31e
+			IMX95_PAD_ENET2_TX_CTL__NETCMIX_TOP_SAI2_TX_SYNC		0x31e
+			IMX95_PAD_ENET2_RX_CTL__NETCMIX_TOP_SAI2_TX_DATA_BIT0		0x31e
+			IMX95_PAD_ENET2_RXC__NETCMIX_TOP_SAI2_TX_DATA_BIT1		0x31e
+			IMX95_PAD_ENET2_RD0__NETCMIX_TOP_SAI2_TX_DATA_BIT2		0x31e
+			IMX95_PAD_ENET2_RD1__NETCMIX_TOP_SAI2_TX_DATA_BIT3		0x31e
+			IMX95_PAD_XSPI1_DATA0__NETCMIX_TOP_SAI2_TX_DATA_BIT4		0x31e
+			IMX95_PAD_XSPI1_DATA1__NETCMIX_TOP_SAI2_TX_DATA_BIT5		0x31e
+			IMX95_PAD_XSPI1_DATA2__NETCMIX_TOP_SAI2_TX_DATA_BIT6		0x31e
+			IMX95_PAD_XSPI1_DATA3__NETCMIX_TOP_SAI2_TX_DATA_BIT7		0x31e
+		>;
+	};
+
+	pinctrl_sai3: sai3grp {
+		fsl,pins = <
+			IMX95_PAD_GPIO_IO17__SAI3_MCLK				0x31e
+			IMX95_PAD_GPIO_IO18__SAI3_RX_BCLK			0x31e
+			IMX95_PAD_GPIO_IO19__SAI3_RX_SYNC			0x31e
+			IMX95_PAD_GPIO_IO20__SAI3_RX_DATA_BIT0			0x31e
+		>;
+	};
+
+	pinctrl_sai5: sai5grp {
+		fsl,pins = <
+			IMX95_PAD_XSPI1_DATA4__SAI5_TX_DATA_BIT0		0x31e
+			IMX95_PAD_XSPI1_DATA7__SAI5_RX_DATA_BIT0		0x31e
+			IMX95_PAD_XSPI1_SS1_B__SAI5_RX_BCLK			0x31e
+			IMX95_PAD_XSPI1_DQS__SAI5_RX_SYNC			0x31e
+
+		>;
+	};
+
+	pinctrl_spdif: spdifgrp {
+		fsl,pins = <
+			IMX95_PAD_GPIO_IO22__SPDIF_IN				0x31e
+			IMX95_PAD_GPIO_IO23__SPDIF_OUT				0x31e
+		>;
+	};
+
+	pinctrl_uart1: uart1grp {
+		fsl,pins = <
+			IMX95_PAD_UART1_RXD__AONMIX_TOP_LPUART1_RX      0x31e
+			IMX95_PAD_UART1_TXD__AONMIX_TOP_LPUART1_TX      0x31e
+		>;
+	};
+
+	pinctrl_usdhc1: usdhc1grp {
+		fsl,pins = <
+			IMX95_PAD_SD1_CLK__USDHC1_CLK				0x158e
+			IMX95_PAD_SD1_CMD__USDHC1_CMD				0x138e
+			IMX95_PAD_SD1_DATA0__USDHC1_DATA0			0x138e
+			IMX95_PAD_SD1_DATA1__USDHC1_DATA1			0x138e
+			IMX95_PAD_SD1_DATA2__USDHC1_DATA2			0x138e
+			IMX95_PAD_SD1_DATA3__USDHC1_DATA3			0x138e
+			IMX95_PAD_SD1_DATA4__USDHC1_DATA4			0x138e
+			IMX95_PAD_SD1_DATA5__USDHC1_DATA5			0x138e
+			IMX95_PAD_SD1_DATA6__USDHC1_DATA6			0x138e
+			IMX95_PAD_SD1_DATA7__USDHC1_DATA7			0x138e
+			IMX95_PAD_SD1_STROBE__USDHC1_STROBE			0x158e
+		>;
+	};
+
+	pinctrl_usdhc1_100mhz: usdhc1-100mhzgrp {
+		fsl,pins = <
+			IMX95_PAD_SD1_CLK__USDHC1_CLK				0x158e
+			IMX95_PAD_SD1_CMD__USDHC1_CMD				0x138e
+			IMX95_PAD_SD1_DATA0__USDHC1_DATA0			0x138e
+			IMX95_PAD_SD1_DATA1__USDHC1_DATA1			0x138e
+			IMX95_PAD_SD1_DATA2__USDHC1_DATA2			0x138e
+			IMX95_PAD_SD1_DATA3__USDHC1_DATA3			0x138e
+			IMX95_PAD_SD1_DATA4__USDHC1_DATA4			0x138e
+			IMX95_PAD_SD1_DATA5__USDHC1_DATA5			0x138e
+			IMX95_PAD_SD1_DATA6__USDHC1_DATA6			0x138e
+			IMX95_PAD_SD1_DATA7__USDHC1_DATA7			0x138e
+			IMX95_PAD_SD1_STROBE__USDHC1_STROBE			0x158e
+		>;
+	};
+
+	pinctrl_usdhc1_200mhz: usdhc1-200mhzgrp {
+		fsl,pins = <
+			IMX95_PAD_SD1_CLK__USDHC1_CLK				0x15fe
+			IMX95_PAD_SD1_CMD__USDHC1_CMD				0x13fe
+			IMX95_PAD_SD1_DATA0__USDHC1_DATA0			0x13fe
+			IMX95_PAD_SD1_DATA1__USDHC1_DATA1			0x13fe
+			IMX95_PAD_SD1_DATA2__USDHC1_DATA2			0x13fe
+			IMX95_PAD_SD1_DATA3__USDHC1_DATA3			0x13fe
+			IMX95_PAD_SD1_DATA4__USDHC1_DATA4			0x13fe
+			IMX95_PAD_SD1_DATA5__USDHC1_DATA5			0x13fe
+			IMX95_PAD_SD1_DATA6__USDHC1_DATA6			0x13fe
+			IMX95_PAD_SD1_DATA7__USDHC1_DATA7			0x13fe
+			IMX95_PAD_SD1_STROBE__USDHC1_STROBE			0x15fe
+		>;
+	};
+
+	pinctrl_usdhc2_gpio: usdhc2gpiogrp {
+		fsl,pins = <
+			IMX95_PAD_SD2_CD_B__GPIO3_IO_BIT0			0x31e
+		>;
+	};
+
+	pinctrl_usdhc2: usdhc2grp {
+		fsl,pins = <
+			IMX95_PAD_SD2_CLK__USDHC2_CLK				0x158e
+			IMX95_PAD_SD2_CMD__USDHC2_CMD				0x138e
+			IMX95_PAD_SD2_DATA0__USDHC2_DATA0			0x138e
+			IMX95_PAD_SD2_DATA1__USDHC2_DATA1			0x138e
+			IMX95_PAD_SD2_DATA2__USDHC2_DATA2			0x138e
+			IMX95_PAD_SD2_DATA3__USDHC2_DATA3			0x138e
+			IMX95_PAD_SD2_VSELECT__USDHC2_VSELECT			0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_100mhz: usdhc2-100mhzgrp {
+		fsl,pins = <
+			IMX95_PAD_SD2_CLK__USDHC2_CLK				0x158e
+			IMX95_PAD_SD2_CMD__USDHC2_CMD				0x138e
+			IMX95_PAD_SD2_DATA0__USDHC2_DATA0			0x138e
+			IMX95_PAD_SD2_DATA1__USDHC2_DATA1			0x138e
+			IMX95_PAD_SD2_DATA2__USDHC2_DATA2			0x138e
+			IMX95_PAD_SD2_DATA3__USDHC2_DATA3			0x138e
+			IMX95_PAD_SD2_VSELECT__USDHC2_VSELECT			0x51e
+		>;
+	};
+
+	pinctrl_usdhc2_200mhz: usdhc2-200mhzgrp {
+		fsl,pins = <
+			IMX95_PAD_SD2_CLK__USDHC2_CLK				0x15fe
+			IMX95_PAD_SD2_CMD__USDHC2_CMD				0x13fe
+			IMX95_PAD_SD2_DATA0__USDHC2_DATA0			0x13fe
+			IMX95_PAD_SD2_DATA1__USDHC2_DATA1			0x13fe
+			IMX95_PAD_SD2_DATA2__USDHC2_DATA2			0x13fe
+			IMX95_PAD_SD2_DATA3__USDHC2_DATA3			0x13fe
+			IMX95_PAD_SD2_VSELECT__USDHC2_VSELECT			0x51e
+		>;
+	};
+};
+
+&scmi_misc {
+	nxp,ctrl-ids = <BRD_SM_CTRL_SD3_WAKE		1>,
+		       <BRD_SM_CTRL_PCIE1_WAKE		1>,
+		       <BRD_SM_CTRL_BT_WAKE		1>,
+		       <BRD_SM_CTRL_PCIE2_WAKE		1>,
+		       <BRD_SM_CTRL_BUTTON		1>;
+};
+
+&usb3 {
+	status = "okay";
+};
+
+&usb3_dwc3 {
+	adp-disable;
+	dr_mode = "host";
+	hnp-disable;
+	srp-disable;
+	snps,dis-u1-entry-quirk;
+	snps,dis-u2-entry-quirk;
+	status = "okay";
+};
+
+&usb3_phy {
+	status = "okay";
+};
+
+&usdhc1 {
+	bus-width = <8>;
+	non-removable;
+	no-sd;
+	no-sdio;
+	pinctrl-0 = <&pinctrl_usdhc1>;
+	pinctrl-1 = <&pinctrl_usdhc1_100mhz>;
+	pinctrl-2 = <&pinctrl_usdhc1_200mhz>;
+	pinctrl-3 = <&pinctrl_usdhc1>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	fsl,tuning-step = <1>;
+	status = "okay";
+};
+
+&usdhc2 {
+	bus-width = <4>;
+	cd-gpios = <&gpio3 00 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-1 = <&pinctrl_usdhc2_100mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-2 = <&pinctrl_usdhc2_200mhz>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-3 = <&pinctrl_usdhc2>, <&pinctrl_usdhc2_gpio>;
+	pinctrl-names = "default", "state_100mhz", "state_200mhz", "sleep";
+	vmmc-supply = <&reg_usdhc2_vmmc>;
+	status = "okay";
+};
+
+&xcvr {
+	clocks = <&scmi_clk IMX95_CLK_BUSWAKEUP>,
+		 <&scmi_clk IMX95_CLK_SPDIF>,
+		 <&dummy>,
+		 <&scmi_clk IMX95_CLK_AUDIOXCVR>,
+		 <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+		 <&scmi_clk IMX95_CLK_AUDIOPLL2>;
+	clock-names = "ipg", "phy", "spba", "pll_ipg", "pll8k", "pll11k";
+	assigned-clocks = <&scmi_clk IMX95_CLK_AUDIOPLL1_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2_VCO>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+			  <&scmi_clk IMX95_CLK_AUDIOPLL2>,
+			  <&scmi_clk IMX95_CLK_SPDIF>,
+			  <&scmi_clk IMX95_CLK_AUDIOXCVR>;
+	assigned-clock-parents = <0>, <0>, <0>, <0>,
+				 <&scmi_clk IMX95_CLK_AUDIOPLL1>,
+				 <&scmi_clk IMX95_CLK_SYSPLL1_PFD1_DIV2>;
+	assigned-clock-rates = <3932160000>, <3612672000>,
+			       <393216000>, <361267200>,
+			       <12288000>, <0>;
+	#sound-dai-cells = <0>;
+	pinctrl-0 = <&pinctrl_spdif>;
+	pinctrl-names = "default";
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts b/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts
index 7eb12e7d50143..e4649d7f9122a 100644
--- a/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-15x15-evk.dts
@@ -557,7 +557,7 @@ &pcie0 {
 	vpcie-supply = <&reg_m2_pwr>;
 	vpcie3v3aux-supply = <&reg_m2_pwr>;
 	supports-clkreq;
-	status = "okay";
+	status = "disabled";
 };
 
 &pcie0_ep {
@@ -1137,7 +1137,7 @@ &usdhc2 {
 	status = "okay";
 };
 
-&usdhc3 {
+m2_usdhc: &usdhc3 {
 	bus-width = <4>;
 	keep-power-in-suspend;
 	mmc-pwrseq = <&usdhc3_pwrseq>;
diff --git a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
index 041fd838fabba..49400bc9ba919 100644
--- a/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-19x19-evk.dts
@@ -550,7 +550,7 @@ &pcie0 {
 &pcie0_ep {
 	pinctrl-0 = <&pinctrl_pcie0>;
 	pinctrl-names = "default";
-	vpcie-supply = <&reg_pcie0>;
+	vpcie-supply = <&reg_m2_pwr>;
 	status = "disabled";
 };
 
diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc-dev.dts b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc-dev.dts
index 5b05f256fd52e..7437e523ff639 100644
--- a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc-dev.dts
+++ b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc-dev.dts
@@ -210,6 +210,11 @@ &lpuart3 {
 	status = "okay";
 };
 
+/* SMARC SER2 */
+&lpuart6 {
+	status = "okay";
+};
+
 /* SMARC MDIO, shared between all ethernet ports */
 &netc_emdio {
 	status = "okay";
diff --git a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
index 7a73958f6eecc..7d760470201fa 100644
--- a/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95-toradex-smarc.dtsi
@@ -22,6 +22,7 @@ aliases {
 		rtc1 = &scmi_bbm;
 		serial0 = &lpuart2;
 		serial1 = &lpuart1;
+		serial2 = &lpuart6;
 		serial3 = &lpuart3;
 	};
 
@@ -112,7 +113,7 @@ reg_usdhc2_vmmc: regulator-vmmc-usdhc2 {
 		compatible = "regulator-fixed";
 		pinctrl-names = "default";
 		pinctrl-0 = <&pinctrl_usdhc2_pwr_en>;
-		gpio = <&gpio3 7 GPIO_ACTIVE_HIGH>;
+		gpios = <&gpio3 7 GPIO_ACTIVE_HIGH>;
 		enable-active-high;
 		off-on-delay-us = <100000>;
 		regulator-max-microvolt = <3300000>;
@@ -450,6 +451,13 @@ som_gpio_expander_1: gpio@21 {
 			"",
 			"",
 			"SMARC_SDIO_WP";
+
+		wifi-uart-en-hog {
+			gpio-hog;
+			gpios = <12 GPIO_ACTIVE_HIGH>;
+			line-name = "WIFI_UART_EN";
+			output-high;
+		};
 	};
 
 	embedded-controller@28 {
@@ -615,6 +623,26 @@ &lpuart3 {
 	pinctrl-0 = <&pinctrl_uart3>;
 };
 
+/* On-module Bluetooth */
+&lpuart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_bt_uart>;
+	uart-has-rtscts;
+	status = "okay";
+
+	bluetooth {
+		compatible = "nxp,88w8987-bt";
+		fw-init-baudrate = <3000000>;
+	};
+};
+
+/* SMARC SER2 */
+&lpuart6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart6>;
+	uart-has-rtscts;
+};
+
 &mu7 {
 	status = "okay";
 };
@@ -822,6 +850,14 @@ &wdog3 {
 };
 
 &scmi_iomuxc {
+	/* On-module Bluetooth, UART pins shared with JTAG */
+	pinctrl_bt_uart: btuartgrp {
+		fsl,pins = <IMX95_PAD_DAP_TDO_TRACESWO__LPUART5_TX	0x31e>, /* WiFI_UART_RXD */
+			   <IMX95_PAD_DAP_TDI__LPUART5_RX		0x31e>, /* WiFI_UART_TXD */
+			   <IMX95_PAD_DAP_TCLK_SWCLK__LPUART5_CTS_B	0x31e>, /* WiFI_UART_RTS# */
+			   <IMX95_PAD_DAP_TMS_SWDIO__LPUART5_RTS_B	0x31e>; /* WiFI_UART_CTS# */
+	};
+
 	/* SMARC CAM_MCK */
 	pinctrl_cam_mck: cammckgrp {
 		fsl,pins = <IMX95_PAD_CCM_CLKO1__CCMSRCGPCMIX_TOP_CLKO_1	0x51e>; /* SMARC S6 - CAM_MCK */
@@ -1105,6 +1141,14 @@ pinctrl_uart3: uart3grp {
 			   <IMX95_PAD_GPIO_IO15__LPUART3_RX	0x31e>; /* SMARC P141 - SER3_RX */
 	};
 
+	/* SMARC SER2 */
+	pinctrl_uart6: uart6grp {
+		fsl,pins = <IMX95_PAD_GPIO_IO34__LPUART6_CTS_B	0x31e>, /* SMARC P139 - SER2_CTS# */
+			   <IMX95_PAD_GPIO_IO07__LPUART6_RTS_B	0x31e>, /* SMARC P138 - SER2_RTS# */
+			   <IMX95_PAD_GPIO_IO05__LPUART6_RX	0x31e>, /* SMARC P137 - SER2_RX   */
+			   <IMX95_PAD_GPIO_IO04__LPUART6_TX	0x31e>; /* SMARC P136 - SER2_TX   */
+	};
+
 	/* On-module eMMC */
 	pinctrl_usdhc1: usdhc1grp {
 		fsl,pins = <IMX95_PAD_SD1_CLK__USDHC1_CLK	0x158e>, /* SD1_CLK    */
diff --git a/arch/arm64/boot/dts/freescale/imx95-verdin-ivy.dtsi b/arch/arm64/boot/dts/freescale/imx95-verdin-ivy.dtsi
index 8337c8b25f050..ff31f7c48cfb4 100644
--- a/arch/arm64/boot/dts/freescale/imx95-verdin-ivy.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95-verdin-ivy.dtsi
@@ -452,7 +452,6 @@ &lpuart7 {
 
 /* Verdin UART_2, through RS485 transceiver */
 &lpuart8 {
-	rs485-rts-active-low;
 	rs485-rx-during-tx;
 	linux,rs485-enabled-at-boot-time;
 
diff --git a/arch/arm64/boot/dts/freescale/imx95-verdin-nonwifi-zinnia.dts b/arch/arm64/boot/dts/freescale/imx95-verdin-nonwifi-zinnia.dts
new file mode 100644
index 0000000000000..b9fa18f34e9a6
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx95-verdin-nonwifi-zinnia.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx95
+ * https://www.toradex.com/products/carrier-board/zinnia-carrier-board
+ */
+
+/dts-v1/;
+
+#include "imx95-verdin.dtsi"
+#include "imx95-verdin-nonwifi.dtsi"
+#include "imx95-verdin-zinnia.dtsi"
+
+/ {
+	model = "Toradex Verdin iMX95 on Zinnia Board";
+	compatible = "toradex,verdin-imx95-nonwifi-zinnia",
+		     "toradex,verdin-imx95-nonwifi",
+		     "toradex,verdin-imx95",
+		     "fsl,imx95";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx95-verdin-wifi-zinnia.dts b/arch/arm64/boot/dts/freescale/imx95-verdin-wifi-zinnia.dts
new file mode 100644
index 0000000000000..7896147963703
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx95-verdin-wifi-zinnia.dts
@@ -0,0 +1,21 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx95
+ * https://www.toradex.com/products/carrier-board/zinnia-carrier-board
+ */
+
+/dts-v1/;
+
+#include "imx95-verdin.dtsi"
+#include "imx95-verdin-wifi.dtsi"
+#include "imx95-verdin-zinnia.dtsi"
+
+/ {
+	model = "Toradex Verdin iMX95 WB on Zinnia Board";
+	compatible = "toradex,verdin-imx95-wifi-zinnia",
+		     "toradex,verdin-imx95-wifi",
+		     "toradex,verdin-imx95",
+		     "fsl,imx95";
+};
diff --git a/arch/arm64/boot/dts/freescale/imx95-verdin-zinnia.dtsi b/arch/arm64/boot/dts/freescale/imx95-verdin-zinnia.dtsi
new file mode 100644
index 0000000000000..6409a6ead7137
--- /dev/null
+++ b/arch/arm64/boot/dts/freescale/imx95-verdin-zinnia.dtsi
@@ -0,0 +1,429 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Common dtsi for Verdin iMX95 SoM on Zinnia carrier board
+ *
+ * https://www.toradex.com/computer-on-modules/verdin-arm-family/nxp-imx95
+ * https://www.toradex.com/products/carrier-board/zinnia-carrier-board
+ */
+
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/net/ti-dp83867.h>
+
+/ {
+	aliases {
+		eeprom1 = &carrier_eeprom;
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_zinnia_leds>;
+
+		/* LED1 Red - SODIMM 48 - LED1_R */
+		led-1 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
+			gpios = <&gpio5 7 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED1 Blue - SODIMM 46 - LED1_B */
+		led-2 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <1>;
+			gpios = <&gpio5 4 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED3 Red - SODIMM 44 - LED3_R */
+		led-3 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <3>;
+			gpios = <&gpio5 5 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED3 Green - SODIMM 54 - LED3_G */
+		led-4 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <3>;
+			gpios = <&gpio5 10 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED3 Blue - SODIMM 36 - LED3_B */
+		led-5 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <3>;
+			gpios = <&gpio2 20 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED4 Red - SODIMM 34 - LED4_R */
+		led-6 {
+			color = <LED_COLOR_ID_RED>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <4>;
+			gpios = <&gpio2 21 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED4 Green - SODIMM 32 - LED4_G */
+		led-7 {
+			color = <LED_COLOR_ID_GREEN>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <4>;
+			gpios = <&gpio2 26 GPIO_ACTIVE_HIGH>;
+		};
+
+		/* LED4 Blue - SODIMM 30 - LED4_B */
+		led-8 {
+			color = <LED_COLOR_ID_BLUE>;
+			default-state = "off";
+			function = LED_FUNCTION_STATUS;
+			function-enumerator = <4>;
+			gpios = <&gpio2 16 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	/* Zinnia Power Supply Input Voltage */
+	zinnia-1v8-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <39000>; /* 12k + 27k */
+		/* Verdin ADC_4 */
+		io-channels = <&adc1 3>;
+		output-ohms = <27000>;
+	};
+
+	zinnia-3v3-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <54000>; /* 27k + 27k */
+		/* Verdin ADC_3 */
+		io-channels = <&adc1 2>;
+		output-ohms = <27000>;
+	};
+
+	zinnia-5v-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <39000>; /* 27k + 12k */
+		/* Verdin ADC_2 */
+		io-channels = <&adc1 1>;
+		output-ohms = <12000>;
+	};
+
+	zinnia-input-voltage {
+		compatible = "voltage-divider";
+		full-ohms = <204700>; /* 200k + 4.7k */
+		/* Verdin ADC_1 */
+		io-channels = <&adc1 0>;
+		output-ohms = <4700>;
+	};
+};
+
+/* Verdin ADC_1, ADC_2, ADC_3 and ADC_4 */
+&adc1 {
+	status = "okay";
+};
+
+/* Verdin ETH_1 (On-module PHY) */
+&enetc_port0 {
+	status = "okay";
+};
+
+/* Verdin ETH_2_RGMII */
+&enetc_port1 {
+	phy-handle = <&ethphy2>;
+	phy-mode = "rgmii-id";
+
+	status = "okay";
+};
+
+/* Verdin CAN_1 */
+&flexcan1 {
+	status = "okay";
+};
+
+&gpio1 {
+	gpio-line-names =
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"";
+};
+
+&gpio2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio1>, <&pinctrl_gpio2>, <&pinctrl_gpio3>;
+	gpio-line-names =
+		"DO1_EN", /* SODIMM 206 */ /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"DI1_EN", /* SODIMM 208 */
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"DI1_RB", /* SODIMM 210 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 30 */
+		"";
+};
+
+&gpio3 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio6>;
+	gpio-line-names =
+		"", /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"DI2_EN", /* SODIMM 218 */
+		"",
+		"",
+		"", /* 30 */
+		"";
+};
+
+&gpio4 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio5>;
+	gpio-line-names =
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 20 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"DI2_RB", /* SODIMM 216 */
+		"";
+};
+
+&gpio5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_gpio4>, <&pinctrl_qspi1_io0_gpio>;
+	gpio-line-names =
+		"DI3_RB", /* SODIMM 56 */ /* 0 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"DO2_EN", /* SODIMM 212 */
+		"",
+		"",
+		"",
+		"",
+		"";
+};
+
+/* Verdin I2C_1 */
+&lpi2c4 {
+	status = "okay";
+
+	temperature-sensor@4f {
+		compatible = "ti,tmp1075";
+		reg = <0x4f>;
+	};
+
+	carrier_eeprom: eeprom@57 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x57>;
+		pagesize = <16>;
+	};
+};
+
+/* Verdin SPI_1 */
+&lpspi6 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_lpspi6>,
+		    <&pinctrl_spi1_cs>,
+		    <&pinctrl_uart8_cts_gpio>;
+	cs-gpios = <&gpio2 14 GPIO_ACTIVE_LOW>,
+		   <&som_gpio_expander 13 GPIO_ACTIVE_LOW>,
+		   <&gpio4 29 GPIO_ACTIVE_LOW>;
+
+	tpm@2 {
+		compatible = "infineon,slb9670", "tcg,tpm_tis-spi";
+		reg = <2>;
+		spi-max-frequency = <18500000>;
+	};
+};
+
+/* Verdin UART_3, used as the Linux console */
+&lpuart1 {
+	status = "okay";
+};
+
+/* Verdin UART_1 */
+&lpuart7 {
+	status = "okay";
+};
+
+/* Verdin UART_2, through RS485 transceiver */
+&lpuart8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pinctrl_uart8>, <&pinctrl_uart8_rts>;
+	rs485-rx-during-tx;
+	linux,rs485-enabled-at-boot-time;
+
+	status = "okay";
+};
+
+&netc_emdio {
+	ethphy2: ethernet-phy@2 {
+		reg = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_eth2_rgmii_int>;
+		interrupt-parent = <&gpio1>;
+		interrupts = <12 IRQ_TYPE_LEVEL_LOW>;
+		ti,rx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+		ti,tx-internal-delay = <DP83867_RGMIIDCTL_2_00_NS>;
+	};
+};
+
+&som_gpio_expander {
+	gpio-line-names =
+		"DO3_EN", /* SODIMM 220 */
+		"DI3_EN", /* SODIMM 222 */
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"",
+		"", /* 10 */
+		"",
+		"",
+		"",
+		"",
+		"";
+};
+
+/* Verdin USB_1 */
+&usb2 {
+	status = "okay";
+};
+
+/* Verdin USB_2 */
+&usb3 {
+	status = "okay";
+};
+
+&usb3_phy {
+	status = "okay";
+};
+
+/* Verdin SD_1 */
+&usdhc2 {
+	status = "okay";
+};
+
+&scmi_iomuxc {
+	pinctrl_uart8_cts_gpio: uart8ctsgrp {
+		fsl,pins = <IMX95_PAD_GPIO_IO14__GPIO2_IO_BIT14	0x51e>; /* SODIMM 143 */
+	};
+
+	pinctrl_zinnia_leds: zinnialedsgrp {
+		fsl,pins = <IMX95_PAD_GPIO_IO16__GPIO2_IO_BIT16		0x11e>, /* SODIMM 30 */
+			   <IMX95_PAD_GPIO_IO26__GPIO2_IO_BIT26		0x11e>, /* SODIMM 32 */
+			   <IMX95_PAD_GPIO_IO21__GPIO2_IO_BIT21		0x11e>, /* SODIMM 34 */
+			   <IMX95_PAD_GPIO_IO20__GPIO2_IO_BIT20		0x11e>, /* SODIMM 36 */
+			   <IMX95_PAD_XSPI1_DATA5__GPIO5_IO_BIT5	0x11e>, /* SODIMM 44 */
+			   <IMX95_PAD_XSPI1_DATA4__GPIO5_IO_BIT4	0x11e>, /* SODIMM 46 */
+			   <IMX95_PAD_XSPI1_DATA7__GPIO5_IO_BIT7	0x11e>, /* SODIMM 48 */
+			   <IMX95_PAD_XSPI1_SS0_B__GPIO5_IO_BIT10	0x11e>; /* SODIMM 54 */
+	};
+};
diff --git a/arch/arm64/boot/dts/freescale/imx95-verdin.dtsi b/arch/arm64/boot/dts/freescale/imx95-verdin.dtsi
index d3737956e2f9b..72e7f1e884098 100644
--- a/arch/arm64/boot/dts/freescale/imx95-verdin.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95-verdin.dtsi
@@ -541,7 +541,7 @@ &lpuart7 {
 /* Verdin UART_2 */
 &lpuart8 {
 	pinctrl-names = "default";
-	pinctrl-0 = <&pinctrl_uart8>;
+	pinctrl-0 = <&pinctrl_uart8>, <&pinctrl_uart8_cts>, <&pinctrl_uart8_rts>;
 	uart-has-rtscts;
 };
 
@@ -1058,12 +1058,20 @@ pinctrl_uart7: uart7grp {
 			   <IMX95_PAD_GPIO_IO11__LPUART7_RTS_B	0x31e>; /* SODIMM 133 */
 	};
 
-	/* Verdin UART_2 */
+	/* Verdin UART_2 CTS */
+	pinctrl_uart8_cts: uart8ctsgrp {
+		fsl,pins = <IMX95_PAD_GPIO_IO14__LPUART8_CTS_B	0x31e>; /* SODIMM 143 */
+	};
+
+	/* Verdin UART_2 RTS */
+	pinctrl_uart8_rts: uart8rtsgrp {
+		fsl,pins = <IMX95_PAD_GPIO_IO15__LPUART8_RTS_B	0x31e>; /* SODIMM 141 */
+	};
+
+	/* Verdin UART_2 RX/TX */
 	pinctrl_uart8: uart8grp {
 		fsl,pins = <IMX95_PAD_GPIO_IO12__LPUART8_TX	0x31e>, /* SODIMM 139 */
-			   <IMX95_PAD_GPIO_IO13__LPUART8_RX	0x31e>, /* SODIMM 137 */
-			   <IMX95_PAD_GPIO_IO14__LPUART8_CTS_B	0x31e>, /* SODIMM 143 */
-			   <IMX95_PAD_GPIO_IO15__LPUART8_RTS_B	0x31e>; /* SODIMM 141 */
+			   <IMX95_PAD_GPIO_IO13__LPUART8_RX	0x31e>; /* SODIMM 137 */
 	};
 
 	/* On-module eMMC */
diff --git a/arch/arm64/boot/dts/freescale/imx95.dtsi b/arch/arm64/boot/dts/freescale/imx95.dtsi
index 71394871d8dd0..3e35c956a4d7a 100644
--- a/arch/arm64/boot/dts/freescale/imx95.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx95.dtsi
@@ -1761,7 +1761,7 @@ aips4: bus@49000000 {
 
 			smmu: iommu@490d0000 {
 				compatible = "arm,smmu-v3";
-				reg = <0x490d0000 0x100000>;
+				reg = <0x490d0000 0x40000>;
 				interrupts = <GIC_SPI 325 IRQ_TYPE_EDGE_RISING>,
 					     <GIC_SPI 328 IRQ_TYPE_EDGE_RISING>,
 					     <GIC_SPI 334 IRQ_TYPE_EDGE_RISING>,
@@ -1770,6 +1770,90 @@ smmu: iommu@490d0000 {
 				#iommu-cells = <1>;
 				status = "disabled";
 			};
+
+			pmu@490d2000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x490d2000 0x1000>,
+				      <0x490f2000 0x1000>;
+				interrupts = <GIC_SPI 333 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@49112000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x49112000 0x1000>,
+				      <0x49122000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@49132000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x49132000 0x1000>,
+				      <0x49142000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@49152000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x49152000 0x1000>,
+				      <0x49162000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@49172000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x49172000 0x1000>,
+				      <0x49182000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@49192000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x49192000 0x1000>,
+				      <0x491a2000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@491b2000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x491b2000 0x1000>,
+				      <0x491c2000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@491d2000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x491d2000 0x1000>,
+				      <0x491e2000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@491f2000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x491f2000 0x1000>,
+				      <0x49202000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@49212000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x49212000 0x1000>,
+				      <0x49222000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@49232000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x49232000 0x1000>,
+				      <0x49242000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
+
+			pmu@49252000 {
+				compatible = "arm,smmu-v3-pmcg";
+				reg = <0x49252000 0x1000>,
+				      <0x49262000 0x1000>;
+				interrupts = <GIC_SPI 323 IRQ_TYPE_EDGE_RISING>;
+			};
 		};
 
 		usb3: usb@4c010010 {
diff --git a/arch/arm64/boot/dts/freescale/imx952.dtsi b/arch/arm64/boot/dts/freescale/imx952.dtsi
index b30707837f353..8d65c380e1c43 100644
--- a/arch/arm64/boot/dts/freescale/imx952.dtsi
+++ b/arch/arm64/boot/dts/freescale/imx952.dtsi
@@ -318,6 +318,28 @@ usbphynop2: usbphynop2 {
 		clock-names = "main_clk";
 	};
 
+	gpu_opp_table: opp-table {
+		compatible = "operating-points-v2";
+
+		opp-500000000 {
+			opp-hz = /bits/ 64 <500000000>;
+			opp-hz-real = /bits/ 64 <500000000>;
+			opp-microvolt = <920000>;
+		};
+
+		opp-800000000 {
+			opp-hz = /bits/ 64 <800000000>;
+			opp-hz-real = /bits/ 64 <800000000>;
+			opp-microvolt = <920000>;
+		};
+
+		opp-1000000000 {
+			opp-hz = /bits/ 64 <1000000000>;
+			opp-hz-real = /bits/ 64 <1000000000>;
+			opp-microvolt = <920000>;
+		};
+	};
+
 	soc {
 		compatible = "simple-bus";
 		#address-cells = <2>;
@@ -1484,5 +1506,19 @@ sai2: sai@4c880000 {
 			power-domains = <&scmi_devpd IMX952_PD_NETC>;
 			status = "disabled";
 		};
+
+		gpu: gpu@4d900000 {
+			compatible = "nxp,imx95-mali", "arm,mali-valhall-csf";
+			reg = <0 0x4d900000 0 0x480000>;
+			interrupts = <GIC_SPI 289 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 290 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 288 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "job", "mmu", "gpu";
+			clocks = <&scmi_clk IMX952_CLK_GPU>;
+			clock-names = "core";
+			power-domains = <&scmi_devpd IMX952_PD_GPU>;
+			operating-points-v2 = <&gpu_opp_table>;
+			dynamic-power-coefficient = <1013>;
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/intel/Makefile b/arch/arm64/boot/dts/intel/Makefile
index 33fcc55d0cb96..088a03b89c995 100644
--- a/arch/arm64/boot/dts/intel/Makefile
+++ b/arch/arm64/boot/dts/intel/Makefile
@@ -8,5 +8,6 @@ dtb-$(CONFIG_ARCH_INTEL_SOCFPGA) += socfpga_agilex_n6000.dtb \
 				socfpga_agilex5_socdk_013b.dtb \
 				socfpga_agilex5_socdk_modular.dtb \
 				socfpga_agilex5_socdk_nand.dtb \
+				socfpga_agilex7m_socdk.dtb \
 				socfpga_n5x_socdk.dtb
 dtb-$(CONFIG_ARCH_KEEMBAY) += keembay-evm.dtb
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex3_socdk.dts b/arch/arm64/boot/dts/intel/socfpga_agilex3_socdk.dts
index 14b299f19f3a5..25e17df0cbdbd 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex3_socdk.dts
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex3_socdk.dts
@@ -12,6 +12,8 @@ / {
 	aliases {
 		serial0 = &uart0;
 		ethernet2 = &gmac2;
+		i3c0 = &i3c0;
+		i3c1 = &i3c1;
 	};
 
 	chosen {
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi b/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
index 02e62d954e949..b06c6d5d60eed 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5.dtsi
@@ -343,7 +343,7 @@ dmac0: dma-controller@0 {
 				#dma-cells = <1>;
 				dma-channels = <4>;
 				snps,dma-masters = <1>;
-				snps,data-width = <2>;
+				snps,data-width = <3>;
 				snps,block-size = <32767 32767 32767 32767>;
 				snps,priority = <0 1 2 3>;
 				snps,axi-max-burst-len = <8>;
@@ -362,7 +362,7 @@ dmac1: dma-controller@10000 {
 				#dma-cells = <1>;
 				dma-channels = <4>;
 				snps,dma-masters = <1>;
-				snps,data-width = <2>;
+				snps,data-width = <3>;
 				snps,block-size = <32767 32767 32767 32767>;
 				snps,priority = <0 1 2 3>;
 				snps,axi-max-burst-len = <8>;
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk.dts b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk.dts
index 262bb3e8e5c72..57d3c5807c65c 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk.dts
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk.dts
@@ -13,6 +13,8 @@ aliases {
 		ethernet0 = &gmac0;
 		ethernet1 = &gmac1;
 		ethernet2 = &gmac2;
+		i3c0 = &i3c0;
+		i3c1 = &i3c1;
 	};
 
 	chosen {
@@ -42,7 +44,7 @@ &gpio0 {
 
 &gmac2 {
 	status = "okay";
-	phy-mode = "rgmii-id";
+	phy-mode = "rgmii"; /* TX/RX clock delays provided by Agilex5 I/O hardware */
 	phy-handle = <&emac2_phy0>;
 	max-frame-size = <9000>;
 
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_013b.dts b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_013b.dts
index f71e1280c7788..82cd4115746ec 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_013b.dts
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_013b.dts
@@ -11,6 +11,8 @@ / {
 	aliases {
 		serial0 = &uart0;
 		ethernet2 = &gmac2;
+		i3c0 = &i3c0;
+		i3c1 = &i3c1;
 	};
 
 	chosen {
@@ -41,7 +43,7 @@ memory@80000000 {
 
 &gmac2 {
 	status = "okay";
-	phy-mode = "rgmii-id";
+	phy-mode = "rgmii"; /* TX/RX clock delays provided by Agilex5 I/O hardware */
 	phy-handle = <&emac2_phy0>;
 	max-frame-size = <9000>;
 
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_modular.dts b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_modular.dts
index 1831402d88082..4d32b6928ce17 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_modular.dts
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_modular.dts
@@ -11,6 +11,8 @@ / {
 	aliases {
 		serial0 = &uart0;
 		ethernet2 = &gmac2;
+		i3c0 = &i3c0;
+		i3c1 = &i3c1;
 	};
 
 	chosen {
@@ -44,7 +46,7 @@ &gpio1 {
 
 &gmac2 {
 	status = "okay";
-	phy-mode = "rgmii-id";
+	phy-mode = "rgmii"; /* TX/RX clock delays provided by Agilex5 I/O hardware */
 	phy-handle = <&emac2_phy0>;
 	max-frame-size = <9000>;
 
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_nand.dts b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_nand.dts
index ec4541d44c9bc..81443096cf991 100644
--- a/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_nand.dts
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex5_socdk_nand.dts
@@ -11,6 +11,8 @@ / {
 	aliases {
 		serial0 = &uart0;
 		ethernet0 = &gmac0;
+		i3c0 = &i3c0;
+		i3c1 = &i3c1;
 	};
 
 	chosen {
@@ -39,7 +41,7 @@ memory@80000000 {
 
 &gmac0 {
 	status = "okay";
-	phy-mode = "rgmii-id";
+	phy-mode = "rgmii"; /* TX/RX clock delays provided by Agilex5 I/O hardware */
 	phy-handle = <&emac0_phy0>;
 	max-frame-size = <9000>;
 
diff --git a/arch/arm64/boot/dts/intel/socfpga_agilex7m_socdk.dts b/arch/arm64/boot/dts/intel/socfpga_agilex7m_socdk.dts
new file mode 100644
index 0000000000000..c1f270b9334cd
--- /dev/null
+++ b/arch/arm64/boot/dts/intel/socfpga_agilex7m_socdk.dts
@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: GPL-2.0
+// Copyright (c) 2026 Altera Corporation
+
+#include "socfpga_agilex.dtsi"
+
+/ {
+	model = "Altera SoCFPGA Agilex7-M SoCDK";
+	compatible = "altr,socfpga-agilex7m-socdk",
+		     "altr,socfpga-agilex7m";
+
+	aliases {
+		serial0 = &uart0;
+		ethernet0 = &gmac0;
+		ethernet1 = &gmac1;
+		ethernet2 = &gmac2;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	leds {
+		compatible = "gpio-leds";
+		led-0 {
+			label = "hps_led0";
+			gpios = <&portb 20 GPIO_ACTIVE_HIGH>;
+		};
+
+		led-1 {
+			label = "hps_led1";
+			gpios = <&portb 19 GPIO_ACTIVE_HIGH>;
+		};
+
+		led-2 {
+			label = "hps_led2";
+			gpios = <&portb 21 GPIO_ACTIVE_HIGH>;
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/* We expect the bootloader to fill in the reg */
+		reg = <0 0x80000000 0 0>;
+	};
+};
+
+&gpio1 {
+	status = "okay";
+};
+
+&gmac0 {
+	status = "okay";
+	phy-mode = "rgmii-id";
+	phy-handle = <&phy0>;
+
+	max-frame-size = <9000>;
+
+	mdio0 {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		compatible = "snps,dwmac-mdio";
+		phy0: ethernet-phy@4 {
+			reg = <4>;
+
+			txd0-skew-ps = <0>; /* -420ps */
+			txd1-skew-ps = <0>; /* -420ps */
+			txd2-skew-ps = <0>; /* -420ps */
+			txd3-skew-ps = <0>; /* -420ps */
+			rxd0-skew-ps = <420>; /* 0ps */
+			rxd1-skew-ps = <420>; /* 0ps */
+			rxd2-skew-ps = <420>; /* 0ps */
+			rxd3-skew-ps = <420>; /* 0ps */
+			txen-skew-ps = <0>; /* -420ps */
+			txc-skew-ps = <900>; /* 0ps */
+			rxdv-skew-ps = <420>; /* 0ps */
+			rxc-skew-ps = <1680>; /* 780ps */
+		};
+	};
+};
+
+&mmc {
+	status = "okay";
+	cap-sd-highspeed;
+	broken-cd;
+	bus-width = <4>;
+	clk-phase-sd-hs = <0>, <135>;
+};
+
+&osc1 {
+	clock-frequency = <25000000>;
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&usb0 {
+	status = "okay";
+};
+
+&watchdog0 {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/marvell/mmp/pxa1908-samsung-coreprimevelte.dts b/arch/arm64/boot/dts/marvell/mmp/pxa1908-samsung-coreprimevelte.dts
index b2ce5edd9c6ac..6ec899c427e14 100644
--- a/arch/arm64/boot/dts/marvell/mmp/pxa1908-samsung-coreprimevelte.dts
+++ b/arch/arm64/boot/dts/marvell/mmp/pxa1908-samsung-coreprimevelte.dts
@@ -23,7 +23,7 @@ chosen {
 
 		fb0: framebuffer@17177000 {
 			compatible = "simple-framebuffer";
-			reg = <0 0x17177000 0 (480 * 800 * 4)>;
+			memory-region = <&fb_mem>;
 			power-domains = <&apmu PXA1908_POWER_DOMAIN_DSI>;
 			width = <480>;
 			height = <800>;
@@ -48,8 +48,9 @@ secure-region@0 {
 			reg = <0 0 0 0x1000000>;
 		};
 
-		framebuffer@17000000 {
-			reg = <0 0x17000000 0 0x1800000>;
+		/* The "active buffer" is at 0x17000000 + (size of one buffer). */
+		fb_mem: framebuffer@17177000 {
+			reg = <0 0x17177000 0 (480 * 800 * 4)>;
 			no-map;
 		};
 	};
@@ -460,7 +461,7 @@ pmic@30 {
 
 		regulators {
 			ldo2: ldo2 {
-				regulator-min-microvolt = <1900000>;
+				regulator-min-microvolt = <2500000>;
 				regulator-max-microvolt = <3100000>;
 			};
 
@@ -523,6 +524,8 @@ &sdh1 {
 	pinctrl-1 = <&sdh1_fast_pins_0 &sdh1_fast_pins_1 &sdh1_pins_2>;
 	bus-width = <4>;
 	non-removable;
+	keep-power-in-suspend;
+	wakeup-source;
 };
 
 &pwm3 {
diff --git a/arch/arm64/boot/dts/marvell/mmp/pxa1908.dtsi b/arch/arm64/boot/dts/marvell/mmp/pxa1908.dtsi
index 5778bfdb85675..91022b62a39b1 100644
--- a/arch/arm64/boot/dts/marvell/mmp/pxa1908.dtsi
+++ b/arch/arm64/boot/dts/marvell/mmp/pxa1908.dtsi
@@ -55,8 +55,11 @@ pmu {
 	};
 
 	psci {
-		compatible = "arm,psci-0.2";
+		compatible = "arm,psci-0.2", "arm,psci";
 		method = "smc";
+
+		cpu_off = <0x85000001>;
+		cpu_on = <0x85000002>;
 	};
 
 	reserved-memory {
diff --git a/arch/arm64/boot/dts/mediatek/Makefile b/arch/arm64/boot/dts/mediatek/Makefile
index 387faa9c2a09b..a86fb313b1a9d 100644
--- a/arch/arm64/boot/dts/mediatek/Makefile
+++ b/arch/arm64/boot/dts/mediatek/Makefile
@@ -47,6 +47,8 @@ dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-2g5.dtb
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-emmc.dtbo
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-4e.dtb
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x.dtb
+dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-cn13.dtbo
+dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-cn14.dtbo
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-cn15.dtbo
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-cn18.dtbo
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-emmc.dtbo
@@ -70,18 +72,24 @@ mt7988a-bananapi-bpi-r4-2g5-sd-dtbs := \
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-2g5-sd.dtb
 mt7988a-bananapi-bpi-r4-pro-8x-emmc-dtbs := \
 	mt7988a-bananapi-bpi-r4-pro-8x.dtb \
+	mt7988a-bananapi-bpi-r4-pro-cn13.dtbo \
+	mt7988a-bananapi-bpi-r4-pro-cn14.dtbo \
 	mt7988a-bananapi-bpi-r4-pro-emmc.dtbo
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x-emmc.dtb
 mt7988a-bananapi-bpi-r4-pro-8x-sd-dtbs := \
 	mt7988a-bananapi-bpi-r4-pro-8x.dtb \
+	mt7988a-bananapi-bpi-r4-pro-cn13.dtbo \
+	mt7988a-bananapi-bpi-r4-pro-cn14.dtbo \
 	mt7988a-bananapi-bpi-r4-pro-sd.dtbo
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x-sd.dtb
 mt7988a-bananapi-bpi-r4-pro-8x-sd-cn15-dtbs := \
 	mt7988a-bananapi-bpi-r4-pro-8x-sd.dtb \
+	mt7988a-bananapi-bpi-r4-pro-cn14.dtbo \
 	mt7988a-bananapi-bpi-r4-pro-cn15.dtbo
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x-sd-cn15.dtb
 mt7988a-bananapi-bpi-r4-pro-8x-sd-cn18-dtbs := \
 	mt7988a-bananapi-bpi-r4-pro-8x-sd.dtb \
+	mt7988a-bananapi-bpi-r4-pro-cn13.dtbo \
 	mt7988a-bananapi-bpi-r4-pro-cn18.dtbo
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt7988a-bananapi-bpi-r4-pro-8x-sd-cn18.dtb
 dtb-$(CONFIG_ARCH_MEDIATEK) += mt8167-pumpkin.dtb
diff --git a/arch/arm64/boot/dts/mediatek/mt6359.dtsi b/arch/arm64/boot/dts/mediatek/mt6359.dtsi
index 45ad69ee49edd..a953fb527b69b 100644
--- a/arch/arm64/boot/dts/mediatek/mt6359.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt6359.dtsi
@@ -164,15 +164,8 @@ mt6359_vcn13_ldo_reg: ldo_vcn13 {
 				regulator-min-microvolt = <900000>;
 				regulator-max-microvolt = <1300000>;
 			};
-			mt6359_vcn33_1_bt_ldo_reg: ldo_vcn33_1_bt {
-				regulator-name = "vcn33_1_bt";
-				regulator-min-microvolt = <2800000>;
-				regulator-max-microvolt = <3500000>;
-			};
-			mt6359_vcn33_1_wifi_ldo_reg: ldo_vcn33_1_wifi {
-				regulator-name = "vcn33_1_wifi";
-				regulator-min-microvolt = <2800000>;
-				regulator-max-microvolt = <3500000>;
+			mt6359_vcn33_1_ldo_reg: ldo_vcn33_1 {
+				regulator-name = "vcn33_1";
 			};
 			mt6359_vaux18_ldo_reg: ldo_vaux18 {
 				regulator-name = "vaux18";
@@ -231,15 +224,8 @@ mt6359_vemc_1_ldo_reg: ldo_vemc_1 {
 				regulator-min-microvolt = <2500000>;
 				regulator-max-microvolt = <3300000>;
 			};
-			mt6359_vcn33_2_bt_ldo_reg: ldo_vcn33_2_bt {
-				regulator-name = "vcn33_2_bt";
-				regulator-min-microvolt = <2800000>;
-				regulator-max-microvolt = <3500000>;
-			};
-			mt6359_vcn33_2_wifi_ldo_reg: ldo_vcn33_2_wifi {
-				regulator-name = "vcn33_2_wifi";
-				regulator-min-microvolt = <2800000>;
-				regulator-max-microvolt = <3500000>;
+			mt6359_vcn33_2_ldo_reg: ldo_vcn33_2 {
+				regulator-name = "vcn33_2";
 			};
 			mt6359_va12_ldo_reg: ldo_va12 {
 				regulator-name = "va12";
diff --git a/arch/arm64/boot/dts/mediatek/mt6365.dtsi b/arch/arm64/boot/dts/mediatek/mt6365.dtsi
new file mode 100644
index 0000000000000..8d16f53c8c6bb
--- /dev/null
+++ b/arch/arm64/boot/dts/mediatek/mt6365.dtsi
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2026 MediaTek Inc.
+ */
+
+#include "mt6359.dtsi"
+
+&pmic {
+	compatible = "mediatek,mt6365", "mediatek,mt6359";
+
+	mt6365regulators: regulators {
+		compatible = "mediatek,mt6365-regulator", "mediatek,mt6359-regulator";
+	};
+};
+
+&pmic_adc {
+	compatible = "mediatek,mt6365-auxadc", "mediatek,mt6359-auxadc";
+};
+
+mt6365codec: &mt6359codec {
+	compatible = "mediatek,mt6365-codec", "mediatek,mt6359-codec";
+};
+
+mt6365rtc: &mt6359rtc {
+	compatible = "mediatek,mt6365-rtc", "mediatek,mt6358-rtc";
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt7981b.dtsi b/arch/arm64/boot/dts/mediatek/mt7981b.dtsi
index 1bbe219380f99..e63ba3ae395e9 100644
--- a/arch/arm64/boot/dts/mediatek/mt7981b.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt7981b.dtsi
@@ -150,6 +150,21 @@ sgmiisys1: syscon@10070000 {
 			#clock-cells = <1>;
 		};
 
+		crypto@10320000 {
+			compatible = "mediatek,mt7981-crypto",
+				"inside-secure,safexcel-eip97ies";
+			reg = <0 0x10320000 0 0x40000>;
+			interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 117 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 118 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "ring0", "ring1", "ring2", "ring3";
+			clocks = <&topckgen CLK_TOP_EIP97B>;
+			clock-names = "core";
+			assigned-clocks = <&topckgen CLK_TOP_EIP97B_SEL>;
+			assigned-clock-parents = <&topckgen CLK_TOP_CB_NET1_D5>;
+		};
+
 		uart0: serial@11002000 {
 			compatible = "mediatek,mt7981-uart", "mediatek,mt6577-uart";
 			reg = <0 0x11002000 0 0x100>;
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso
new file mode 100644
index 0000000000000..973b76ba0cbfb
--- /dev/null
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn13.dtso
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2025 MediaTek Inc.
+ * Author: Frank Wunderlich <frank-w@public-files.de>
+ */
+
+/* This enables key-m slot CN13 on pcie2(11280000 1L0) on BPI-R4-Pro */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	compatible = "bananapi,bpi-r4-pro", "mediatek,mt7988a";
+};
+
+&{/soc/pinctrl@1001f000/pcie-2-hog} {
+	output-high;
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso
new file mode 100644
index 0000000000000..90b2a64459c31
--- /dev/null
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro-cn14.dtso
@@ -0,0 +1,20 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2025 MediaTek Inc.
+ * Author: Frank Wunderlich <frank-w@public-files.de>
+ */
+
+/* This enables key-m slot CN14 on pcie3(11290000 1L1) on BPI-R4-Pro */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+
+/ {
+	compatible = "bananapi,bpi-r4-pro", "mediatek,mt7988a";
+};
+
+&{/soc/pinctrl@1001f000/pcie-3-hog} {
+	output-high;
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi
index a48132f094114..1eeb72108b9b7 100644
--- a/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt7988a-bananapi-bpi-r4-pro.dtsi
@@ -61,14 +61,14 @@ gpio-leds {
 
 		led_red: sys-led-red {
 			color = <LED_COLOR_ID_RED>;
+			function = LED_FUNCTION_FAULT;
 			gpios = <&pca9555 15 GPIO_ACTIVE_HIGH>;
-			default-state = "on";
 		};
 
 		led_blue: sys-led-blue {
 			color = <LED_COLOR_ID_BLUE>;
+			function = LED_FUNCTION_ACTIVITY;
 			gpios = <&pca9555 14 GPIO_ACTIVE_HIGH>;
-			default-state = "on";
 		};
 	};
 
@@ -185,13 +185,6 @@ &eth {
 	status = "okay";
 };
 
-&fan {
-	pinctrl-0 = <&pwm0_pins>;
-	pinctrl-names = "default";
-	pwms = <&pwm 0 50000>;
-	status = "okay";
-};
-
 &gmac0 {
 	status = "okay";
 };
@@ -207,7 +200,7 @@ &gsw_phy0_led0 {
 };
 
 &gsw_port0 {
-	label = "mgmt";
+	label = "lan5";
 };
 
 /* R4Pro has only port 0 connected, so disable the others */
@@ -437,14 +430,12 @@ mux {
 	pcie-2-hog {
 		gpio-hog;
 		gpios = <79 GPIO_ACTIVE_HIGH>;
-		output-high;
 	};
 
 	/* 1L1 0=key-b (CN18), 1=key-m (CN14) */
 	pcie-3-hog {
 		gpio-hog;
 		gpios = <63 GPIO_ACTIVE_HIGH>;
-		output-high;
 	};
 
 	pwm0_pins: pwm0-pins {
diff --git a/arch/arm64/boot/dts/mediatek/mt8186-corsola-voltorb.dtsi b/arch/arm64/boot/dts/mediatek/mt8186-corsola-voltorb.dtsi
index b495a241b4432..0250b4eeadd45 100644
--- a/arch/arm64/boot/dts/mediatek/mt8186-corsola-voltorb.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8186-corsola-voltorb.dtsi
@@ -73,6 +73,10 @@ &spmi {
 	pmic@6 {
 		compatible = "mediatek,mt6319-regulator", "mediatek,mt6315-regulator";
 		reg = <0x6 SPMI_USID>;
+		pvdd1-supply = <&pp4200_z2>;
+		pvdd2-supply = <&pp4200_z2>;
+		pvdd3-supply = <&pp4200_z2>;
+		pvdd4-supply = <&pp4200_z2>;
 
 		regulators {
 			mt6319_buck1: vbuck1 {
diff --git a/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi b/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi
index 8e423504ec052..4cb23595d17b4 100644
--- a/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8188-geralt.dtsi
@@ -217,6 +217,30 @@ &cam_vcore {
 	domain-supply = <&mt6359_vproc1_buck_reg>;
 };
 
+&cpu0 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu1 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu2 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu3 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu4 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu5 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
 /*
  * Geralt is the reference design and doesn't have target TDP.
  * Ciri is (currently) the only device following Geralt, and its
@@ -1149,6 +1173,23 @@ pins-en-pp3300-wlan {
 
 &pmic {
 	interrupts-extended = <&pio 222 IRQ_TYPE_LEVEL_HIGH>;
+	vsys-smps-supply = <&pp4200_s5>;
+	vsys-vcore-supply = <&pp4200_s5>;
+	vsys-vgpu11-supply = <&pp4200_s5>;
+	vsys-vgpu12-supply = <&pp4200_s5>;
+	vsys-vpa-supply = <&pp4200_s5>;
+	vsys-vproc1-supply = <&pp4200_s5>;
+	vsys-vproc2-supply = <&pp4200_s5>;
+	vsys-vpu-supply = <&pp4200_s5>;
+	vsys-vs1-supply = <&pp4200_s5>;
+	vsys-vs2-supply = <&pp4200_s5>;
+	vsys-vmodem-supply = <&pp4200_s5>;
+	vsys-ldo1-supply = <&pp4200_s5>;
+	vsys-ldo2-supply = <&pp4200_s5>;
+	vs1-ldo1-supply = <&mt6359_vs1_buck_reg>;
+	vs1-ldo2-supply = <&mt6359_vs1_buck_reg>;
+	vs2-ldo1-supply = <&mt6359_vs2_buck_reg>;
+	vs2-ldo2-supply = <&mt6359_vs2_buck_reg>;
 };
 
 &postmask0_in {
diff --git a/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi b/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi
index eadf1b2d156f2..40d34b16dc807 100644
--- a/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8192-asurada.dtsi
@@ -167,12 +167,22 @@ pp3300_wlan: regulator-3v3-wlan {
 		regulator-name = "pp3300_wlan";
 		regulator-always-on;
 		regulator-boot-on;
-		regulator-min-microvolt = <3300000>;
-		regulator-max-microvolt = <3300000>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&pp3300_wlan_pins>;
 		enable-active-high;
 		gpio = <&pio 143 GPIO_ACTIVE_HIGH>;
+		vin-supply = <&pp3300_g>;
+	};
+
+	/* system wide switching 4.2V power rail */
+	pp4200_g: regulator-4v2-g {
+		compatible = "regulator-fixed";
+		regulator-name = "pp4200_g";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <4200000>;
+		regulator-max-microvolt = <4200000>;
+		vin-supply = <&ppvar_sys>;
 	};
 
 	/* system wide switching 5.0V power rail */
@@ -287,6 +297,38 @@ &afe {
 	memory-region = <&afe_dma_mem>;
 };
 
+&cpu0 {
+	cpu-supply = <&mt6315_6_vbuck3>;
+};
+
+&cpu1 {
+	cpu-supply = <&mt6315_6_vbuck3>;
+};
+
+&cpu2 {
+	cpu-supply = <&mt6315_6_vbuck3>;
+};
+
+&cpu3 {
+	cpu-supply = <&mt6315_6_vbuck3>;
+};
+
+&cpu4 {
+	cpu-supply = <&mt6315_6_vbuck1>;
+};
+
+&cpu5 {
+	cpu-supply = <&mt6315_6_vbuck1>;
+};
+
+&cpu6 {
+	cpu-supply = <&mt6315_6_vbuck1>;
+};
+
+&cpu7 {
+	cpu-supply = <&mt6315_6_vbuck1>;
+};
+
 &dsi0 {
 	status = "okay";
 };
@@ -514,17 +556,19 @@ &nor_flash {
 	assigned-clock-parents = <&topckgen CLK_TOP_UNIVPLL_D6_D8>;
 
 	flash@0 {
-		compatible = "winbond,w25q64jwm", "jedec,spi-nor";
+		compatible = "jedec,spi-nor";
 		reg = <0>;
 		spi-max-frequency = <52000000>;
 		spi-rx-bus-width = <2>;
 		spi-tx-bus-width = <2>;
+		vcc-supply = <&mt6359_vio18_ldo_reg>;
 	};
 };
 
 &pcie {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie_pins>;
+	memory-region = <&wifi_restricted_dma_region>;
 
 	pcie0: pcie@0,0 {
 		device_type = "pci";
@@ -539,7 +583,6 @@ pcie0: pcie@0,0 {
 		wifi: wifi@0,0 {
 			reg = <0x10000 0 0 0 0x100000>,
 			      <0x10000 0 0x100000 0 0x100000>;
-			memory-region = <&wifi_restricted_dma_region>;
 		};
 	};
 };
@@ -609,9 +652,9 @@ &pio {
 			  "SD_DATA1",
 			  "",
 			  "",
+			  "BT_KILL_1V8_L",
 			  "",
-			  "",
-			  "",
+			  "WIFI_KILL_1V8_L",
 			  "",
 			  "PCIE_WAKE_ODL",
 			  "PCIE_RST_L",
@@ -1306,6 +1349,23 @@ pins-miso-on {
 
 &pmic {
 	interrupts-extended = <&pio 214 IRQ_TYPE_LEVEL_HIGH>;
+	vsys-smps-supply = <&pp4200_g>;
+	vsys-vcore-supply = <&pp4200_g>;
+	vsys-vgpu11-supply = <&pp4200_g>;
+	vsys-vgpu12-supply = <&pp4200_g>;
+	vsys-vpa-supply = <&pp4200_g>;
+	vsys-vproc1-supply = <&pp4200_g>;
+	vsys-vproc2-supply = <&pp4200_g>;
+	vsys-vpu-supply = <&pp4200_g>;
+	vsys-vs1-supply = <&pp4200_g>;
+	vsys-vs2-supply = <&pp4200_g>;
+	vsys-vmodem-supply = <&pp4200_g>;
+	vsys-ldo1-supply = <&pp4200_g>;
+	vsys-ldo2-supply = <&pp4200_g>;
+	vs1-ldo1-supply = <&mt6359_vs1_buck_reg>;
+	vs1-ldo2-supply = <&mt6359_vs1_buck_reg>;
+	vs2-ldo1-supply = <&mt6359_vs2_buck_reg>;
+	vs2-ldo2-supply = <&mt6359_vs2_buck_reg>;
 };
 
 &pwm0 {
@@ -1367,6 +1427,7 @@ mt6360_ldo3_reg: regulator@0 {
 			reg = <0>;
 			regulator-min-microvolt = <1800000>;
 			regulator-max-microvolt = <3300000>;
+			vin-supply = <&pp4200_g>;
 		};
 
 		mt6360_ldo5_reg: regulator@1 {
@@ -1374,6 +1435,7 @@ mt6360_ldo5_reg: regulator@1 {
 			reg = <1>;
 			regulator-min-microvolt = <3300000>;
 			regulator-max-microvolt = <3300000>;
+			vin-supply = <&pp4200_g>;
 		};
 
 		typec {
@@ -1427,6 +1489,10 @@ &spmi {
 	mt6315_6: pmic@6 {
 		compatible = "mediatek,mt6315-regulator";
 		reg = <0x6 SPMI_USID>;
+		pvdd1-supply = <&pp4200_g>;
+		pvdd2-supply = <&pp4200_g>;
+		pvdd3-supply = <&pp4200_g>;
+		pvdd4-supply = <&pp4200_g>;
 
 		regulators {
 			mt6315_6_vbuck1: vbuck1 {
@@ -1452,6 +1518,10 @@ mt6315_6_vbuck3: vbuck3 {
 	mt6315_7: pmic@7 {
 		compatible = "mediatek,mt6315-regulator";
 		reg = <0x7 SPMI_USID>;
+		pvdd1-supply = <&pp4200_g>;
+		pvdd2-supply = <&pp4200_g>;
+		pvdd3-supply = <&pp4200_g>;
+		pvdd4-supply = <&pp4200_g>;
 
 		regulators {
 			mt6315_7_vbuck1: vbuck1 {
diff --git a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
index f1ff64a84267c..ef7afc436aefc 100644
--- a/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8195-cherry.dtsi
@@ -47,6 +47,17 @@ memory@40000000 {
 		reg = <0 0x40000000 0 0x80000000>;
 	};
 
+	/* system critical LDO 1.8V power rail */
+	pp1800_ldo_z2: regulator-pp1800-ldo-z2 {
+		compatible = "regulator-fixed";
+		regulator-name = "pp1800_ldo_z2";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&pp3300_z2>;
+	};
+
 	pp3300_disp_x: regulator-pp3300-disp-x {
 		compatible = "regulator-fixed";
 		regulator-name = "pp3300_disp_x";
@@ -138,6 +149,53 @@ ppvar_sys: regulator-ppvar-sys {
 		regulator-boot-on;
 	};
 
+	usb_vbus: regulator-5v0-usb-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "usb-vbus";
+		enable-active-high;
+		regulator-always-on;
+		vin-supply = <&pp5000_s5>;
+	};
+
+	reserved_memory: reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		scp_mem: memory@50000000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x50000000 0 0x2900000>;
+			no-map;
+		};
+
+		adsp_mem: memory@60000000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x60000000 0 0xd80000>;
+			no-map;
+		};
+
+		afe_mem: memory@60d80000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x60d80000 0 0x100000>;
+			no-map;
+		};
+
+		adsp_device_mem: memory@60e80000 {
+			compatible = "shared-dma-pool";
+			reg = <0 0x60e80000 0 0x280000>;
+			no-map;
+		};
+	};
+
+	spk_amplifier: rt1019p {
+		compatible = "realtek,rt1019p";
+		label = "rt1019p";
+		#sound-dai-cells = <0>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&rt1019p_pins_default>;
+		sdb-gpios = <&pio 100 GPIO_ACTIVE_HIGH>;
+	};
+
 	/* Murata NCP03WF104F05RL */
 	tboard_thermistor1: thermal-sensor-t1 {
 		compatible = "generic-adc-thermal";
@@ -208,54 +266,6 @@ tboard_thermistor2: thermal-sensor-t2 {
 						120000 51
 						125000 44>;
 	};
-
-	usb_vbus: regulator-5v0-usb-vbus {
-		compatible = "regulator-fixed";
-		regulator-name = "usb-vbus";
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		enable-active-high;
-		regulator-always-on;
-	};
-
-	reserved_memory: reserved-memory {
-		#address-cells = <2>;
-		#size-cells = <2>;
-		ranges;
-
-		scp_mem: memory@50000000 {
-			compatible = "shared-dma-pool";
-			reg = <0 0x50000000 0 0x2900000>;
-			no-map;
-		};
-
-		adsp_mem: memory@60000000 {
-			compatible = "shared-dma-pool";
-			reg = <0 0x60000000 0 0xd80000>;
-			no-map;
-		};
-
-		afe_mem: memory@60d80000 {
-			compatible = "shared-dma-pool";
-			reg = <0 0x60d80000 0 0x100000>;
-			no-map;
-		};
-
-		adsp_device_mem: memory@60e80000 {
-			compatible = "shared-dma-pool";
-			reg = <0 0x60e80000 0 0x280000>;
-			no-map;
-		};
-	};
-
-	spk_amplifier: rt1019p {
-		compatible = "realtek,rt1019p";
-		label = "rt1019p";
-		#sound-dai-cells = <0>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&rt1019p_pins_default>;
-		sdb-gpios = <&pio 100 GPIO_ACTIVE_HIGH>;
-	};
 };
 
 &adsp {
@@ -707,8 +717,8 @@ &mmc1 {
 	pinctrl-1 = <&mmc1_pins_default>;
 	sd-uhs-sdr50;
 	sd-uhs-sdr104;
-	vmmc-supply = <&mt_pmic_vmch_ldo_reg>;
-	vqmmc-supply = <&mt_pmic_vmc_ldo_reg>;
+	vmmc-supply = <&pp3000_sd>;
+	vqmmc-supply = <&pp3000_vmc_pmu>;
 };
 
 &mt6359codec {
@@ -763,6 +773,7 @@ flash@0 {
 		spi-max-frequency = <52000000>;
 		spi-rx-bus-width = <2>;
 		spi-tx-bus-width = <2>;
+		vcc-supply = <&pp1800_ldo_z2>;
 	};
 };
 
@@ -1312,6 +1323,23 @@ pins-report-sw {
 
 &pmic {
 	interrupts-extended = <&pio 222 IRQ_TYPE_LEVEL_HIGH>;
+	vsys-smps-supply = <&pp4200_z2>;
+	vsys-vcore-supply = <&pp4200_z2>;
+	vsys-vgpu11-supply = <&pp4200_z2>;
+	vsys-vgpu12-supply = <&pp4200_z2>;
+	vsys-vpa-supply = <&pp4200_z2>;
+	vsys-vproc1-supply = <&pp4200_z2>;
+	vsys-vproc2-supply = <&pp4200_z2>;
+	vsys-vpu-supply = <&pp4200_z2>;
+	vsys-vs1-supply = <&pp4200_z2>;
+	vsys-vs2-supply = <&pp4200_z2>;
+	vsys-vmodem-supply = <&pp4200_z2>;
+	vsys-ldo1-supply = <&pp4200_z2>;
+	vsys-ldo2-supply = <&pp4200_z2>;
+	vs1-ldo1-supply = <&mt6359_vs1_buck_reg>;
+	vs1-ldo2-supply = <&mt6359_vs1_buck_reg>;
+	vs2-ldo1-supply = <&mt6359_vs2_buck_reg>;
+	vs2-ldo2-supply = <&mt6359_vs2_buck_reg>;
 };
 
 &scp {
@@ -1408,20 +1436,22 @@ i2c_tunnel: i2c-tunnel {
 			#size-cells = <0>;
 		};
 
-		mt_pmic_vmc_ldo_reg: regulator@0 {
+		pp3000_vmc_pmu: regulator@0 {
 			compatible = "google,cros-ec-regulator";
 			reg = <0>;
-			regulator-name = "mt_pmic_vmc_ldo";
+			regulator-name = "pp3000_vmc_pmu";
 			regulator-min-microvolt = <1200000>;
 			regulator-max-microvolt = <3600000>;
+			vin-supply = <&pp4200_z2>;
 		};
 
-		mt_pmic_vmch_ldo_reg: regulator@1 {
+		pp3000_sd: regulator@1 {
 			compatible = "google,cros-ec-regulator";
 			reg = <1>;
-			regulator-name = "mt_pmic_vmch_ldo";
+			regulator-name = "pp3000_sd";
 			regulator-min-microvolt = <2700000>;
 			regulator-max-microvolt = <3600000>;
+			vin-supply = <&pp4200_z2>;
 		};
 
 		typec {
@@ -1455,6 +1485,10 @@ &spmi {
 	mt6315@6 {
 		compatible = "mediatek,mt6315-regulator";
 		reg = <0x6 SPMI_USID>;
+		pvdd1-supply = <&pp4200_z2>;
+		pvdd2-supply = <&pp4200_z2>;
+		pvdd3-supply = <&pp4200_z2>;
+		pvdd4-supply = <&pp4200_z2>;
 
 		regulators {
 			mt6315_6_vbuck1: vbuck1 {
@@ -1472,6 +1506,10 @@ mt6315_6_vbuck1: vbuck1 {
 	mt6315@7 {
 		compatible = "mediatek,mt6315-regulator";
 		reg = <0x7 SPMI_USID>;
+		pvdd1-supply = <&pp4200_z2>;
+		pvdd2-supply = <&pp4200_z2>;
+		pvdd3-supply = <&pp4200_z2>;
+		pvdd4-supply = <&pp4200_z2>;
 
 		regulators {
 			mt6315_7_vbuck1: vbuck1 {
@@ -1584,21 +1622,23 @@ vdosys1_ep_ext: endpoint@1 {
 };
 
 &xhci0 {
-	status = "okay";
-
 	rx-fifo-depth = <3072>;
 	vbus-supply = <&usb_vbus>;
+	vusb33-supply = <&mt6359_vusb_ldo_reg>;
+	status = "okay";
 };
 
 &xhci2 {
-	status = "okay";
 	vbus-supply = <&usb_vbus>;
+	vusb33-supply = <&mt6359_vusb_ldo_reg>;
+	status = "okay";
 };
 
 &xhci3 {
 	/* MT7921's USB Bluetooth has issues with USB2 LPM */
 	usb2-lpm-disable;
 	vbus-supply = <&pp3300_wlan>;
+	vusb33-supply = <&mt6359_vusb_ldo_reg>;
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/mediatek/mt8390-genio-700-evk.dts b/arch/arm64/boot/dts/mediatek/mt8390-genio-700-evk.dts
index 612336713a64e..0d5a75efb2eed 100644
--- a/arch/arm64/boot/dts/mediatek/mt8390-genio-700-evk.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8390-genio-700-evk.dts
@@ -21,3 +21,10 @@ memory@40000000 {
 	};
 };
 
+&cpu4 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu5 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
diff --git a/arch/arm64/boot/dts/mediatek/mt8390-genio-common.dtsi b/arch/arm64/boot/dts/mediatek/mt8390-genio-common.dtsi
index 2062506f6cc52..b0c97930a0e6c 100644
--- a/arch/arm64/boot/dts/mediatek/mt8390-genio-common.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8390-genio-common.dtsi
@@ -10,7 +10,7 @@
  *                    AngeloGioacchino Del Regno <angelogioacchino.delregno@collabora.com>   
  */
 
-#include "mt6359.dtsi"
+#include "mt6365.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -285,6 +285,30 @@ &afe {
 	status = "okay";
 };
 
+&cpu0 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu1 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu2 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu3 {
+	cpu-supply = <&mt6359_vcore_buck_reg>;
+};
+
+&cpu6 {
+	cpu-supply = <&mt6319_vbuck1>;
+};
+
+&cpu7 {
+	cpu-supply = <&mt6319_vbuck1>;
+};
+
 &disp_dsi0 {
 	#address-cells = <1>;
 	#size-cells = <0>;
@@ -445,7 +469,7 @@ typec-mux@48 {
 		mode-switch;
 		orientation-switch;
 
-		vcc-supply = <&mt6359_vcn33_1_bt_ldo_reg>;
+		vcc-supply = <&mt6359_vcn33_1_ldo_reg>;
 
 		port {
 			it5205_sbu_mux: endpoint {
@@ -637,7 +661,7 @@ &mt6359_vcn18_ldo_reg {
 	regulator-always-on;
 };
 
-&mt6359_vcn33_2_bt_ldo_reg {
+&mt6359_vcn33_2_ldo_reg {
 	regulator-name = "vcn33_2_pmu";
 	regulator-always-on;
 };
@@ -695,7 +719,7 @@ &mt6359_vufs_ldo_reg {
 	regulator-always-on;
 };
 
-&mt6359codec {
+&mt6365codec {
 	mediatek,mic-type-0 = <1>; /* ACC */
 	mediatek,mic-type-1 = <3>; /* DCC */
 };
@@ -1281,8 +1305,8 @@ &pmic {
 	interrupt-parent = <&pio>;
 	interrupts = <222 IRQ_TYPE_LEVEL_HIGH>;
 
-	mt6359keys: keys {
-		compatible = "mediatek,mt6359-keys";
+	mt6365keys: keys {
+		compatible = "mediatek,mt6365-keys", "mediatek,mt6359-keys";
 		mediatek,long-press-mode = <1>;
 		power-off-time-sec = <0>;
 
@@ -1364,6 +1388,50 @@ &spi2 {
 	status = "okay";
 };
 
+&spmi {
+	#address-cells = <2>;
+	#size-cells = <0>;
+
+	pmic@6 {
+		compatible = "mediatek,mt6319-regulator", "mediatek,mt6315-regulator";
+		reg = <0x6 SPMI_USID>;
+
+		pvdd1-supply = <&reg_vsys>;
+		pvdd2-supply = <&reg_vsys>;
+		pvdd3-supply = <&reg_vsys>;
+		pvdd4-supply = <&reg_vsys>;
+
+		regulators {
+			mt6319_vbuck1: vbuck1 {
+				regulator-name = "dvdd_proc_b";
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1193750>;
+				regulator-enable-ramp-delay = <256>;
+				regulator-allowed-modes = <0 1 2>;
+				regulator-always-on;
+			};
+
+			vbuck3 {
+				regulator-name = "avdd2_emi";
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1193750>;
+				regulator-enable-ramp-delay = <256>;
+				regulator-allowed-modes = <0 1 2>;
+				regulator-always-on;
+			};
+
+			vbuck4 {
+				regulator-name = "avddq_emi";
+				regulator-min-microvolt = <300000>;
+				regulator-max-microvolt = <1193750>;
+				regulator-enable-ramp-delay = <256>;
+				regulator-allowed-modes = <0 1 2>;
+				regulator-always-on;
+			};
+		};
+	};
+};
+
 &uart0 {
 	pinctrl-0 = <&uart0_pins>;
 	pinctrl-names = "default";
diff --git a/arch/arm64/boot/dts/mediatek/mt8390-grinn-genio-som.dtsi b/arch/arm64/boot/dts/mediatek/mt8390-grinn-genio-som.dtsi
index 8da47c9163138..84064511fa0d3 100644
--- a/arch/arm64/boot/dts/mediatek/mt8390-grinn-genio-som.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8390-grinn-genio-som.dtsi
@@ -58,7 +58,7 @@ &mt6359_vcn18_ldo_reg {
 	regulator-always-on;
 };
 
-&mt6359_vcn33_2_bt_ldo_reg {
+&mt6359_vcn33_2_ldo_reg {
 	regulator-name = "vcn33_2_pmu";
 	regulator-always-on;
 };
diff --git a/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi b/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi
index 40b381d4cc35b..9f5a0ec563e8c 100644
--- a/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8390-tungsten-smarc.dtsi
@@ -46,6 +46,20 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	connector {
+		compatible = "hdmi-connector";
+		label = "hdmi";
+		type = "a";
+		ddc-i2c-bus = <&hdmi_ddc>;
+		hdmi-pwr-supply = <&hdmi_phy>;
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&hdmi0_out>;
+			};
+		};
+	};
+
 	firmware {
 		optee {
 			compatible = "linaro,optee-tz";
@@ -283,6 +297,18 @@ &dither0_out {
 	remote-endpoint = <&dsi0_in>;
 };
 
+&dpi1 {
+	status = "okay";
+};
+
+&dpi1_in {
+	remote-endpoint = <&merge5_out>;
+};
+
+&dpi1_out {
+	remote-endpoint = <&hdmi0_in>;
+};
+
 &eth {
 	phy-mode ="rgmii-id";
 	phy-handle = <&ethernet_phy0>;
@@ -304,6 +330,35 @@ ethernet_phy0: ethernet-phy@7 {
 	};
 };
 
+&ethdr0 {
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+
+			ethdr0_in: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&vdosys1_ep_ext>;
+			};
+		};
+
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+
+			ethdr0_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&merge5_in>;
+			};
+		};
+	};
+};
+
 &gamma0_out {
 	remote-endpoint = <&postmask0_in>;
 };
@@ -313,6 +368,26 @@ &gpu {
 	status = "okay";
 };
 
+&hdmi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&hdmi_pins>;
+	status = "okay";
+};
+
+&hdmi0_in {
+	remote-endpoint = <&dpi1_out>;
+};
+
+&hdmi0_out {
+	remote-endpoint = <&hdmi_connector_in>;
+};
+
+&hdmi_phy {
+	pinctrl-names = "default";
+	pinctrl-0 = <&hdmi_vreg_pins>;
+	status = "okay";
+};
+
 &i2c0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c0_pins>;
@@ -531,6 +606,35 @@ touchscren@5d {
 	};
 };
 
+&merge5 {
+	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		port@0 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <0>;
+
+			merge5_in: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&ethdr0_out>;
+			};
+		};
+
+		port@1 {
+			#address-cells = <1>;
+			#size-cells = <0>;
+			reg = <1>;
+
+			merge5_out: endpoint@1 {
+				reg = <1>;
+				remote-endpoint = <&dpi1_in>;
+			};
+		};
+	};
+};
+
 &mfg0 {
 	domain-supply = <&mt6359_vproc2_buck_reg>;
 };
@@ -592,7 +696,7 @@ &mmc2 {
 	interrupt-names = "msdc", "sdio_wakeup";
 	interrupts-extended = <&gic GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH 0>,
 			      <&pio 172 IRQ_TYPE_LEVEL_LOW>;
-	vmmc-supply = <&mt6359_vcn33_2_bt_ldo_reg>;
+	vmmc-supply = <&mt6359_vcn33_2_ldo_reg>;
 	vqmmc-supply = <&mt6359_vcn18_ldo_reg>;
 	mmc-pwrseq = <&wifi_pwrseq>;
 	status = "okay";
@@ -617,12 +721,12 @@ &mt6359_vcn18_ldo_reg {
 	regulator-boot-on;
 };
 
-&mt6359_vcn33_1_bt_ldo_reg {
+&mt6359_vcn33_1_ldo_reg {
 	regulator-name = "vcn33_1_pmu";
 	regulator-always-on;
 };
 
-&mt6359_vcn33_2_bt_ldo_reg {
+&mt6359_vcn33_2_ldo_reg {
 	regulator-name = "vcn33_2_pmu";
 	regulator-min-microvolt = <3300000>;
 	regulator-max-microvolt = <3300000>;
@@ -977,6 +1081,18 @@ vdosys0_ep_main: endpoint@0 {
 	};
 };
 
+&vdosys1 {
+	port {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		vdosys1_ep_ext: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&ethdr0_in>;
+		};
+	};
+};
+
 &watchdog {
 	pinctrl-names = "default";
 	pinctrl-0 = <&watchdog_pins>;
diff --git a/arch/arm64/boot/dts/mediatek/mt8395-genio-common.dtsi b/arch/arm64/boot/dts/mediatek/mt8395-genio-common.dtsi
index 62c336e215009..edc5539bebde1 100644
--- a/arch/arm64/boot/dts/mediatek/mt8395-genio-common.dtsi
+++ b/arch/arm64/boot/dts/mediatek/mt8395-genio-common.dtsi
@@ -7,7 +7,7 @@
 /dts-v1/;
 
 #include "mt8195.dtsi"
-#include "mt6359.dtsi"
+#include "mt6365.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -201,6 +201,22 @@ wifi_fixed_3v3: regulator-2 {
 		enable-active-high;
 		regulator-always-on;
 	};
+
+	/* system wide 4.2V power rail */
+	reg_vsys: regulator-vsys {
+		compatible = "regulator-fixed";
+		regulator-name = "vsys";
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	reg_vsys_buck: regulator-vsys-buck {
+		compatible = "regulator-fixed";
+		regulator-name = "vsys_buck";
+		regulator-always-on;
+		regulator-boot-on;
+		vin-supply = <&reg_vsys>;
+	};
 };
 
 &adsp {
@@ -478,7 +494,11 @@ otg_vbus_regulator: usb-otg-vbus-regulator {
 
 		regulator {
 			compatible = "mediatek,mt6360-regulator";
-			LDO_VIN3-supply = <&mt6360_buck2>;
+			BUCK1_VIN-supply = <&reg_vsys>;
+			BUCK2_VIN-supply = <&reg_vsys>;
+			LDO_VIN1-supply = <&reg_vsys_buck>;
+			LDO_VIN2-supply = <&reg_vsys_buck>;
+			LDO_VIN3-supply = <&reg_vsys>;
 
 			mt6360_buck1: buck1 {
 				regulator-name = "emi_vdd2";
@@ -701,7 +721,7 @@ &mt6359_vcamio_ldo_reg {
 	regulator-always-on;
 };
 
-&mt6359_vcn33_2_bt_ldo_reg {
+&mt6359_vcn33_2_ldo_reg {
 	regulator-min-microvolt = <3300000>;
 	regulator-max-microvolt = <3300000>;
 };
@@ -728,7 +748,7 @@ &mt6359_vsram_others_ldo_reg {
 	regulator-max-microvolt = <750000>;
 };
 
-&mt6359codec {
+&mt6365codec {
 	mediatek,mic-type-0 = <1>; /* ACC */
 	mediatek,mic-type-1 = <3>; /* DCC */
 	mediatek,mic-type-2 = <1>; /* ACC */
@@ -1151,8 +1171,8 @@ pins {
 &pmic {
 	interrupts-extended = <&pio 222 IRQ_TYPE_LEVEL_HIGH>;
 
-	mt6359keys: keys {
-		compatible = "mediatek,mt6359-keys";
+	mt6365keys: keys {
+		compatible = "mediatek,mt6365-keys", "mediatek,mt6359-keys";
 		mediatek,long-press-mode = <1>;
 		power-off-time-sec = <0>;
 
@@ -1216,8 +1236,8 @@ can0: can@0 {
 		clocks = <&can_clk>;
 		spi-max-frequency = <20000000>;
 		interrupts-extended = <&pio 16 IRQ_TYPE_LEVEL_LOW>;
-		vdd-supply = <&mt6359_vcn33_2_bt_ldo_reg>;
-		xceiver-supply = <&mt6359_vcn33_2_bt_ldo_reg>;
+		vdd-supply = <&mt6359_vcn33_2_ldo_reg>;
+		xceiver-supply = <&mt6359_vcn33_2_ldo_reg>;
 	};
 };
 
@@ -1238,6 +1258,11 @@ mt6315_6: pmic@6 {
 		compatible = "mediatek,mt6315-regulator";
 		reg = <0x6 SPMI_USID>;
 
+		pvdd1-supply = <&reg_vsys>;
+		pvdd2-supply = <&reg_vsys>;
+		pvdd3-supply = <&reg_vsys>;
+		pvdd4-supply = <&reg_vsys>;
+
 		regulators {
 			mt6315_6_vbuck1: vbuck1 {
 				regulator-name = "Vbcpu";
@@ -1254,6 +1279,11 @@ mt6315_7: pmic@7 {
 		compatible = "mediatek,mt6315-regulator";
 		reg = <0x7 SPMI_USID>;
 
+		pvdd1-supply = <&reg_vsys>;
+		pvdd2-supply = <&reg_vsys>;
+		pvdd3-supply = <&reg_vsys>;
+		pvdd4-supply = <&reg_vsys>;
+
 		regulators {
 			mt6315_7_vbuck1: vbuck1 {
 				regulator-name = "Vgpu";
diff --git a/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l-8-hd-panel.dtso b/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l-8-hd-panel.dtso
index 0389c9cb8581c..ecc9fd27b82d0 100644
--- a/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l-8-hd-panel.dtso
+++ b/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l-8-hd-panel.dtso
@@ -29,7 +29,7 @@ panel@0 {
 		compatible = "radxa,display-8hd-ad002", "jadard,jd9365da-h3";
 		reg = <0>;
 		backlight = <&backlight>;
-		vdd-supply = <&mt6359_vcn33_2_bt_ldo_reg>;
+		vdd-supply = <&mt6359_vcn33_2_ldo_reg>;
 		vccio-supply = <&mt6360_ldo2>;
 		reset-gpios = <&pio 108 GPIO_ACTIVE_LOW>;
 		pinctrl-names = "default";
@@ -57,7 +57,7 @@ touchscreen@14 {
 		interrupts-extended = <&pio 132 IRQ_TYPE_EDGE_RISING>;
 		irq-gpios = <&pio 132 GPIO_ACTIVE_HIGH>;
 		reset-gpios = <&pio 133 GPIO_ACTIVE_HIGH>;
-		VDDIO-supply = <&mt6359_vcn33_2_bt_ldo_reg>;
+		VDDIO-supply = <&mt6359_vcn33_2_ldo_reg>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&touch_pins>;
 	};
diff --git a/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l.dts b/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l.dts
index bf91305e8e4a5..b279eed1bfc3a 100644
--- a/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l.dts
+++ b/arch/arm64/boot/dts/mediatek/mt8395-radxa-nio-12l.dts
@@ -6,7 +6,7 @@
  */
 
 #include "mt8195.dtsi"
-#include "mt6359.dtsi"
+#include "mt6365.dtsi"
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
 #include <dt-bindings/interrupt-controller/irq.h>
@@ -614,7 +614,7 @@ &mt6359_vcamio_ldo_reg {
 	regulator-always-on;
 };
 
-&mt6359_vcn33_2_bt_ldo_reg {
+&mt6359_vcn33_2_ldo_reg {
 	regulator-min-microvolt = <3300000>;
 	regulator-max-microvolt = <3300000>;
 };
@@ -1034,8 +1034,8 @@ &pciephy {
 &pmic {
 	interrupts-extended = <&pio 222 IRQ_TYPE_LEVEL_HIGH>;
 
-	mt6359keys: keys {
-		compatible = "mediatek,mt6359-keys";
+	mt6365keys: keys {
+		compatible = "mediatek,mt6365-keys", "mediatek,mt6359-keys";
 		mediatek,long-press-mode = <1>;
 		power-off-time-sec = <0>;
 
diff --git a/arch/arm64/boot/dts/nuvoton/ma35d1-iot-512m.dts b/arch/arm64/boot/dts/nuvoton/ma35d1-iot-512m.dts
index 9482bec1aa573..78534d613486a 100644
--- a/arch/arm64/boot/dts/nuvoton/ma35d1-iot-512m.dts
+++ b/arch/arm64/boot/dts/nuvoton/ma35d1-iot-512m.dts
@@ -18,6 +18,8 @@ aliases {
 		serial12 = &uart12;
 		serial13 = &uart13;
 		serial14 = &uart14;
+		ethernet0 = &gmac0;
+		ethernet1 = &gmac1;
 	};
 
 	chosen {
@@ -126,3 +128,26 @@ &uart14 {
 	pinctrl-0 = <&pinctrl_uart14>;
 	status = "okay";
 };
+
+&gmac0 {
+	status = "okay";
+	phy-handle = <&eth_phy0>;
+};
+
+&mdio0 {
+	eth_phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+};
+
+&gmac1 {
+	status = "okay";
+	phy-mode = "rmii";
+	phy-handle = <&eth_phy1>;
+};
+
+&mdio1 {
+	eth_phy1: ethernet-phy@1 {
+		reg = <1>;
+	};
+};
diff --git a/arch/arm64/boot/dts/nuvoton/ma35d1-som-256m.dts b/arch/arm64/boot/dts/nuvoton/ma35d1-som-256m.dts
index f6f20a17e5018..a029b660e8dc3 100644
--- a/arch/arm64/boot/dts/nuvoton/ma35d1-som-256m.dts
+++ b/arch/arm64/boot/dts/nuvoton/ma35d1-som-256m.dts
@@ -18,6 +18,8 @@ aliases {
 		serial12 = &uart12;
 		serial14 = &uart14;
 		serial16 = &uart16;
+		ethernet0 = &gmac0;
+		ethernet1 = &gmac1;
 	};
 
 	chosen {
@@ -129,3 +131,25 @@ &uart16 {
 	pinctrl-0 = <&pinctrl_uart16>;
 	status = "okay";
 };
+
+&gmac0 {
+	status = "okay";
+	phy-handle = <&eth_phy0>;
+};
+
+&mdio0 {
+	eth_phy0: ethernet-phy@0 {
+		reg = <0>;
+	};
+};
+
+&gmac1 {
+	status = "okay";
+	phy-handle = <&eth_phy1>;
+};
+
+&mdio1 {
+	eth_phy1: ethernet-phy@1 {
+		reg = <1>;
+	};
+};
diff --git a/arch/arm64/boot/dts/nuvoton/ma35d1.dtsi b/arch/arm64/boot/dts/nuvoton/ma35d1.dtsi
index e51b98f5bdce4..7228ad4735b5c 100644
--- a/arch/arm64/boot/dts/nuvoton/ma35d1.dtsi
+++ b/arch/arm64/boot/dts/nuvoton/ma35d1.dtsi
@@ -379,5 +379,57 @@ uart16: serial@40880000 {
 			clocks = <&clk UART16_GATE>;
 			status = "disabled";
 		};
+
+		gmac0: ethernet@40120000 {
+			compatible = "nuvoton,ma35d1-dwmac", "snps,dwmac-3.70a";
+			reg = <0x0 0x40120000 0x0 0x10000>;
+			interrupts = <GIC_SPI 23 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq";
+			clocks = <&clk EMAC0_GATE>, <&clk EPLL_DIV8>;
+			clock-names = "stmmaceth", "ptp_ref";
+
+			nuvoton,sys = <&sys 0>;
+			resets = <&sys MA35D1_RESET_GMAC0>;
+			reset-names = "stmmaceth";
+			snps,multicast-filter-bins = <0>;
+			snps,perfect-filter-entries = <8>;
+			rx-fifo-depth = <4096>;
+			tx-fifo-depth = <2048>;
+
+			phy-mode = "rgmii-id";
+			status = "disabled";
+
+			mdio0: mdio {
+				compatible = "snps,dwmac-mdio";
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
+
+		gmac1: ethernet@40130000 {
+			compatible = "nuvoton,ma35d1-dwmac", "snps,dwmac-3.70a";
+			reg = <0x0 0x40130000 0x0 0x10000>;
+			interrupts = <GIC_SPI 24 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "macirq";
+			clocks = <&clk EMAC1_GATE>, <&clk EPLL_DIV8>;
+			clock-names = "stmmaceth", "ptp_ref";
+
+			nuvoton,sys = <&sys 1>;
+			resets = <&sys MA35D1_RESET_GMAC1>;
+			reset-names = "stmmaceth";
+			snps,multicast-filter-bins = <0>;
+			snps,perfect-filter-entries = <8>;
+			rx-fifo-depth = <4096>;
+			tx-fifo-depth = <2048>;
+
+			phy-mode = "rgmii-id";
+			status = "disabled";
+
+			mdio1: mdio {
+				compatible = "snps,dwmac-mdio";
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/nvidia/tegra194.dtsi b/arch/arm64/boot/dts/nvidia/tegra194.dtsi
index 849694f751d90..1d659454a6f9f 100644
--- a/arch/arm64/boot/dts/nvidia/tegra194.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra194.dtsi
@@ -2382,6 +2382,7 @@ pcie@14100000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <4000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -2434,6 +2435,7 @@ pcie@14120000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <4000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -2486,6 +2488,7 @@ pcie@14140000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <4000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -2538,6 +2541,7 @@ pcie@14160000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <4000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -2583,6 +2587,7 @@ pcie-ep@14160000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <4000>;
 
 			interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE4R &emc>,
 					<&mc TEGRA194_MEMORY_CLIENT_PCIE4W &emc>;
@@ -2629,6 +2634,7 @@ pcie@14180000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <4000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -2674,6 +2680,7 @@ pcie-ep@14180000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <4000>;
 
 			interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE0R &emc>,
 					<&mc TEGRA194_MEMORY_CLIENT_PCIE0W &emc>;
@@ -2723,6 +2730,7 @@ pcie@141a0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <4000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -2771,6 +2779,7 @@ pcie-ep@141a0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <4000>;
 
 			interconnects = <&mc TEGRA194_MEMORY_CLIENT_PCIE5R &emc>,
 					<&mc TEGRA194_MEMORY_CLIENT_PCIE5W &emc>;
diff --git a/arch/arm64/boot/dts/nvidia/tegra234.dtsi b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
index 04a95b6658caa..75bb9a0ad0279 100644
--- a/arch/arm64/boot/dts/nvidia/tegra234.dtsi
+++ b/arch/arm64/boot/dts/nvidia/tegra234.dtsi
@@ -3605,7 +3605,7 @@ ethernet@6800000 {
 				 <&bpmp TEGRA234_CLK_MGBE0_RX_PCS_M>,
 				 <&bpmp TEGRA234_CLK_MGBE0_RX_PCS>,
 				 <&bpmp TEGRA234_CLK_MGBE0_TX_PCS>;
-			clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m",
+			clock-names = "mgbe", "mac", "mac-divider", "ptp_ref", "rx-input-m",
 				      "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m",
 				      "rx-pcs", "tx-pcs";
 			resets = <&bpmp TEGRA234_RESET_MGBE0_MAC>,
@@ -3647,7 +3647,7 @@ ethernet@6900000 {
 				 <&bpmp TEGRA234_CLK_MGBE1_RX_PCS_M>,
 				 <&bpmp TEGRA234_CLK_MGBE1_RX_PCS>,
 				 <&bpmp TEGRA234_CLK_MGBE1_TX_PCS>;
-			clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m",
+			clock-names = "mgbe", "mac", "mac-divider", "ptp_ref", "rx-input-m",
 				      "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m",
 				      "rx-pcs", "tx-pcs";
 			resets = <&bpmp TEGRA234_RESET_MGBE1_MAC>,
@@ -3689,7 +3689,7 @@ ethernet@6a00000 {
 				 <&bpmp TEGRA234_CLK_MGBE2_RX_PCS_M>,
 				 <&bpmp TEGRA234_CLK_MGBE2_RX_PCS>,
 				 <&bpmp TEGRA234_CLK_MGBE2_TX_PCS>;
-			clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m",
+			clock-names = "mgbe", "mac", "mac-divider", "ptp_ref", "rx-input-m",
 				      "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m",
 				      "rx-pcs", "tx-pcs";
 			resets = <&bpmp TEGRA234_RESET_MGBE2_MAC>,
@@ -3731,7 +3731,7 @@ ethernet@6b00000 {
 				 <&bpmp TEGRA234_CLK_MGBE3_RX_PCS_M>,
 				 <&bpmp TEGRA234_CLK_MGBE3_RX_PCS>,
 				 <&bpmp TEGRA234_CLK_MGBE3_TX_PCS>;
-			clock-names = "mgbe", "mac", "mac-divider", "ptp-ref", "rx-input-m",
+			clock-names = "mgbe", "mac", "mac-divider", "ptp_ref", "rx-input-m",
 				      "rx-input", "tx", "eee-pcs", "rx-pcs-input", "rx-pcs-m",
 				      "rx-pcs", "tx-pcs";
 			resets = <&bpmp TEGRA234_RESET_MGBE3_MAC>,
@@ -4532,6 +4532,7 @@ pcie@140a0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -4586,6 +4587,7 @@ pcie@140c0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -4640,6 +4642,7 @@ pcie@140e0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -4686,6 +4689,7 @@ pcie-ep@140e0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <16000>;
 
 			interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE10AR &emc>,
 					<&mc TEGRA234_MEMORY_CLIENT_PCIE10AW &emc>;
@@ -4734,6 +4738,7 @@ pcie@14100000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -4788,6 +4793,7 @@ pcie@14120000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -4842,6 +4848,7 @@ pcie@14140000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -4896,6 +4903,7 @@ pcie@14160000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -4937,6 +4945,7 @@ pcie-ep@14160000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <16000>;
 
 			interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE4R &emc>,
 				      <&mc TEGRA234_MEMORY_CLIENT_PCIE4W &emc>;
@@ -4983,6 +4992,7 @@ pcie@14180000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -5037,6 +5047,7 @@ pcie@141a0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -5083,6 +5094,7 @@ pcie-ep@141a0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <16000>;
 
 			interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE5R &emc>,
 					<&mc TEGRA234_MEMORY_CLIENT_PCIE5W &emc>;
@@ -5131,6 +5143,7 @@ pcie@141c0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -5177,6 +5190,7 @@ pcie-ep@141c0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <16000>;
 
 			interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE6AR &emc>,
 					<&mc TEGRA234_MEMORY_CLIENT_PCIE6AW &emc>;
@@ -5225,6 +5239,7 @@ pcie@141e0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <8000>;
 
 			bus-range = <0x0 0xff>;
 
@@ -5271,6 +5286,7 @@ pcie-ep@141e0000 {
 			nvidia,aspm-cmrt-us = <60>;
 			nvidia,aspm-pwr-on-t-us = <20>;
 			nvidia,aspm-l0s-entrance-latency-us = <3>;
+			aspm-l1-entry-delay-ns = <16000>;
 
 			interconnects = <&mc TEGRA234_MEMORY_CLIENT_PCIE7AR &emc>,
 					<&mc TEGRA234_MEMORY_CLIENT_PCIE7AW &emc>;
diff --git a/arch/arm64/boot/dts/qcom/Makefile b/arch/arm64/boot/dts/qcom/Makefile
index 4ba8e73064194..795cee4757abe 100644
--- a/arch/arm64/boot/dts/qcom/Makefile
+++ b/arch/arm64/boot/dts/qcom/Makefile
@@ -39,6 +39,7 @@ dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp433.dtb ipq9574-rdp433-emmc.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp449.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp453.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= ipq9574-rdp454.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= ipq9650-rdp488.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= kaanapali-mtp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= kaanapali-qrd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= lemans-evk.dtb
@@ -56,6 +57,7 @@ lemans-evk-ifp-mezzanine-dtbs	:= lemans-evk.dtb lemans-evk-ifp-mezzanine.dtbo
 dtb-$(CONFIG_ARCH_QCOM)	+= lemans-evk-ifp-mezzanine.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= mahua-crd.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= milos-fairphone-fp6.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= milos-nothing-asteroids.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= monaco-arduino-monza.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= monaco-evk.dtb
 
@@ -157,6 +159,10 @@ dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-sony-xperia-yoshino-maple.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-sony-xperia-yoshino-poplar.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= msm8998-xiaomi-sagit.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= purwa-iot-evk.dtb
+
+purwa-iot-evk-el2-dtbs	:= purwa-iot-evk.dtb x1-el2.dtbo
+
+dtb-$(CONFIG_ARCH_QCOM)	+= purwa-iot-evk-el2.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcm6490-fairphone-fp5.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcm6490-idp.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= qcm6490-particle-tachyon.dtb
@@ -340,6 +346,7 @@ dtb-$(CONFIG_ARCH_QCOM)	+= sm6375-sony-xperia-murray-pdx225.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7125-xiaomi-curtana.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7125-xiaomi-joyeuse.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7225-fairphone-fp4.dtb
+dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-motorola-dubai.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm7325-nothing-spacewar.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8150-hdk.dtb
 dtb-$(CONFIG_ARCH_QCOM)	+= sm8150-microsoft-surface-duo.dtb
diff --git a/arch/arm64/boot/dts/qcom/agatti.dtsi b/arch/arm64/boot/dts/qcom/agatti.dtsi
index 8a7337239b1eb..f0b6ae9b81528 100644
--- a/arch/arm64/boot/dts/qcom/agatti.dtsi
+++ b/arch/arm64/boot/dts/qcom/agatti.dtsi
@@ -433,6 +433,18 @@ wlan_smp2p_in: wlan-wpss-to-ap {
 			interrupt-controller;
 			#interrupt-cells = <2>;
 		};
+
+		ipa_smp2p_out: ipa-ap-to-modem {
+			qcom,entry-name = "ipa";
+			#qcom,smem-state-cells = <1>;
+		};
+
+		ipa_smp2p_in: ipa-modem-to-ap {
+			qcom,entry-name = "ipa";
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
 	};
 
 	soc: soc@0 {
@@ -1641,6 +1653,47 @@ usb_dwc3_ss: endpoint {
 			};
 		};
 
+		ipa: ipa@5840000 {
+			compatible = "qcom,qcm2290-ipa", "qcom,sc7180-ipa";
+
+			iommus = <&apps_smmu 0x140 0x0>;
+			reg = <0x0 0x05840000 0x0 0x7000>,
+			      <0x0 0x05847000 0x0 0x2000>,
+			      <0x0 0x05804000 0x0 0x2c000>;
+			reg-names = "ipa-reg",
+				    "ipa-shared",
+				    "gsi";
+
+			interrupts-extended = <&intc GIC_SPI 257 IRQ_TYPE_EDGE_RISING>,
+					      <&intc GIC_SPI 259 IRQ_TYPE_LEVEL_HIGH>,
+					      <&ipa_smp2p_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&ipa_smp2p_in 1 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "ipa",
+					  "gsi",
+					  "ipa-clock-query",
+					  "ipa-setup-ready";
+
+			clocks = <&rpmcc RPM_SMD_IPA_CLK>;
+			clock-names = "core";
+
+			interconnects = <&system_noc MASTER_IPA RPM_ALWAYS_TAG
+					 &bimc SLAVE_EBI1 RPM_ALWAYS_TAG>,
+					<&system_noc MASTER_IPA RPM_ALWAYS_TAG
+					 &system_noc SLAVE_IMEM RPM_ALWAYS_TAG>,
+					<&bimc MASTER_APPSS_PROC RPM_ACTIVE_TAG
+					 &config_noc SLAVE_IPA_CFG RPM_ACTIVE_TAG>;
+			interconnect-names = "memory",
+					     "imem",
+					     "config";
+
+			qcom,smem-states = <&ipa_smp2p_out 0>,
+					   <&ipa_smp2p_out 1>;
+			qcom,smem-state-names = "ipa-clock-enabled-valid",
+						"ipa-clock-enabled";
+
+			status = "disabled";
+		};
+
 		gpu: gpu@5900000 {
 			compatible = "qcom,adreno-07000200", "qcom,adreno";
 			reg = <0x0 0x05900000 0x0 0x40000>,
@@ -2455,6 +2508,16 @@ opp-240000000 {
 					opp-hz = /bits/ 64 <240000000>;
 					required-opps = <&rpmpd_opp_svs>;
 				};
+
+				opp-300000000 {
+					opp-hz = /bits/ 64 <300000000>;
+					required-opps = <&rpmpd_opp_svs_plus>;
+				};
+
+				opp-384000000 {
+					opp-hz = /bits/ 64 <384000000>;
+					required-opps = <&rpmpd_opp_nom>;
+				};
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/eliza.dtsi b/arch/arm64/boot/dts/qcom/eliza.dtsi
index 4a7a0ac40ce62..24c6807954812 100644
--- a/arch/arm64/boot/dts/qcom/eliza.dtsi
+++ b/arch/arm64/boot/dts/qcom/eliza.dtsi
@@ -846,6 +846,38 @@ ice: crypto@1d88000 {
 			clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
 		};
 
+		cryptobam: dma-controller@1dc4000 {
+			compatible = "qcom,bam-v1.7.4", "qcom,bam-v1.7.0";
+			reg = <0x0 0x01dc4000 0x0 0x28000>;
+
+			interrupts = <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+
+			#dma-cells = <1>;
+
+			iommus = <&apps_smmu 0x480 0>,
+				 <&apps_smmu 0x481 0>;
+
+			qcom,ee = <0>;
+			qcom,num-ees = <4>;
+			num-channels = <20>;
+			qcom,controlled-remotely;
+		};
+
+		crypto: crypto@1dfa000 {
+			compatible = "qcom,eliza-qce", "qcom,sm8150-qce", "qcom,qce";
+			reg = <0x0 0x01dfa000 0x0 0x6000>;
+
+			interconnects = <&aggre2_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "memory";
+
+			dmas = <&cryptobam 4>, <&cryptobam 5>;
+			dma-names = "rx", "tx";
+
+			iommus = <&apps_smmu 0x480 0>,
+				 <&apps_smmu 0x481 0>;
+		};
+
 		tcsr_mutex: hwlock@1f40000 {
 			compatible = "qcom,tcsr-mutex";
 			reg = <0x0 0x01f40000 0x0 0x20000>;
diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dts b/arch/arm64/boot/dts/qcom/glymur-crd.dts
index 35aaf09e4e2b4..c98dfb3941fa3 100644
--- a/arch/arm64/boot/dts/qcom/glymur-crd.dts
+++ b/arch/arm64/boot/dts/qcom/glymur-crd.dts
@@ -322,8 +322,6 @@ reset-n-pins {
 };
 
 &usb_0 {
-	dr_mode = "host";
-
 	status = "okay";
 };
 
@@ -353,8 +351,6 @@ &usb_0_qmpphy_out {
 };
 
 &usb_1 {
-	dr_mode = "host";
-
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
index 2852d257ac8c0..e784b538f42e1 100644
--- a/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
+++ b/arch/arm64/boot/dts/qcom/glymur-crd.dtsi
@@ -10,6 +10,8 @@
 #include "pmk8850.dtsi"         /* SPMI0: SID-0                  */
 #include "smb2370.dtsi"         /* SPMI2: SID-9/10/11            */
 
+#include <dt-bindings/input/gpio-keys.h>
+
 / {
 	model = "Qualcomm Technologies, Inc. Glymur CRD";
 	compatible = "qcom,glymur-crd", "qcom,glymur";
@@ -44,7 +46,7 @@ sleep_clk: sleep-clk {
 	gpio-keys {
 		compatible = "gpio-keys";
 
-		pinctrl-0 = <&key_vol_up_default>;
+		pinctrl-0 = <&key_vol_up_default>, <&hall_int_n_default>;
 		pinctrl-names = "default";
 
 		key-volume-up {
@@ -55,6 +57,15 @@ key-volume-up {
 			linux,can-disable;
 			wakeup-source;
 		};
+
+		switch-lid {
+			label = "lid";
+			gpios = <&tlmm 92 GPIO_ACTIVE_LOW>;
+			linux,input-type = <EV_SW>;
+			linux,code = <SW_LID>;
+			wakeup-source;
+			wakeup-event-action = <EV_ACT_DEASSERTED>;
+		};
 	};
 
 	vreg_nvme: regulator-nvme {
@@ -560,11 +571,31 @@ &pon_resin {
 	status = "okay";
 };
 
+&remoteproc_adsp {
+	firmware-name = "qcom/glymur/adsp.mbn",
+			"qcom/glymur/adsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/glymur/cdsp.mbn",
+			"qcom/glymur/cdsp_dtb.mbn";
+
+	status = "okay";
+};
+
 &tlmm {
 	gpio-reserved-ranges = <4 4>, /* EC TZ Secure I3C */
 			       <10 2>, /* OOB UART */
 			       <44 4>; /* Security SPI (TPM) */
 
+	hall_int_n_default: hall-int-n-state {
+		pins = "gpio92";
+		function = "gpio";
+		bias-disable;
+	};
+
 	pcie4_default: pcie4-default-state {
 		clkreq-n-pins {
 			pins = "gpio147";
diff --git a/arch/arm64/boot/dts/qcom/glymur.dtsi b/arch/arm64/boot/dts/qcom/glymur.dtsi
index f23cf81ddb77a..72c7dc6e4f098 100644
--- a/arch/arm64/boot/dts/qcom/glymur.dtsi
+++ b/arch/arm64/boot/dts/qcom/glymur.dtsi
@@ -5,7 +5,10 @@
 
 #include <dt-bindings/clock/qcom,glymur-dispcc.h>
 #include <dt-bindings/clock/qcom,glymur-gcc.h>
+#include <dt-bindings/clock/qcom,glymur-gpucc.h>
 #include <dt-bindings/clock/qcom,glymur-tcsr.h>
+#include <dt-bindings/clock/qcom,glymur-videocc.h>
+#include <dt-bindings/clock/qcom,kaanapali-gxclkctl.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
 #include <dt-bindings/dma/qcom-gpi.h>
 #include <dt-bindings/gpio/gpio.h>
@@ -39,6 +42,7 @@ cpu0: cpu@0 {
 			power-domains = <&cpu_pd0>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
 
 			l2_0: l2-cache {
 				compatible = "cache";
@@ -55,6 +59,7 @@ cpu1: cpu@100 {
 			power-domains = <&cpu_pd1>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
 		};
 
 		cpu2: cpu@200 {
@@ -65,6 +70,7 @@ cpu2: cpu@200 {
 			power-domains = <&cpu_pd2>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
 		};
 
 		cpu3: cpu@300 {
@@ -75,6 +81,7 @@ cpu3: cpu@300 {
 			power-domains = <&cpu_pd3>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
 		};
 
 		cpu4: cpu@400 {
@@ -85,6 +92,7 @@ cpu4: cpu@400 {
 			power-domains = <&cpu_pd4>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
 		};
 
 		cpu5: cpu@500 {
@@ -95,6 +103,7 @@ cpu5: cpu@500 {
 			power-domains = <&cpu_pd5>, <&scmi_perf 0>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_0>;
+			#cooling-cells = <2>;
 		};
 
 		cpu6: cpu@10000 {
@@ -105,6 +114,7 @@ cpu6: cpu@10000 {
 			power-domains = <&cpu_pd6>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
+			#cooling-cells = <2>;
 
 			l2_1: l2-cache {
 				compatible = "cache";
@@ -121,6 +131,7 @@ cpu7: cpu@10100 {
 			power-domains = <&cpu_pd7>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
+			#cooling-cells = <2>;
 		};
 
 		cpu8: cpu@10200 {
@@ -131,6 +142,7 @@ cpu8: cpu@10200 {
 			power-domains = <&cpu_pd8>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
+			#cooling-cells = <2>;
 		};
 
 		cpu9: cpu@10300 {
@@ -141,6 +153,7 @@ cpu9: cpu@10300 {
 			power-domains = <&cpu_pd9>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
+			#cooling-cells = <2>;
 		};
 
 		cpu10: cpu@10400 {
@@ -151,6 +164,7 @@ cpu10: cpu@10400 {
 			power-domains = <&cpu_pd10>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
+			#cooling-cells = <2>;
 		};
 
 		cpu11: cpu@10500 {
@@ -161,6 +175,7 @@ cpu11: cpu@10500 {
 			power-domains = <&cpu_pd11>, <&scmi_perf 1>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_1>;
+			#cooling-cells = <2>;
 		};
 
 		cpu12: cpu@20000 {
@@ -171,6 +186,7 @@ cpu12: cpu@20000 {
 			power-domains = <&cpu_pd12>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
+			#cooling-cells = <2>;
 
 			l2_2: l2-cache {
 				compatible = "cache";
@@ -187,6 +203,7 @@ cpu13: cpu@20100 {
 			power-domains = <&cpu_pd13>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
+			#cooling-cells = <2>;
 		};
 
 		cpu14: cpu@20200 {
@@ -197,6 +214,7 @@ cpu14: cpu@20200 {
 			power-domains = <&cpu_pd14>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
+			#cooling-cells = <2>;
 		};
 
 		cpu15: cpu@20300 {
@@ -207,6 +225,7 @@ cpu15: cpu@20300 {
 			power-domains = <&cpu_pd15>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
+			#cooling-cells = <2>;
 		};
 
 		cpu16: cpu@20400 {
@@ -217,6 +236,7 @@ cpu16: cpu@20400 {
 			power-domains = <&cpu_pd16>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
+			#cooling-cells = <2>;
 		};
 
 		cpu17: cpu@20500 {
@@ -227,6 +247,7 @@ cpu17: cpu@20500 {
 			power-domains = <&cpu_pd17>, <&scmi_perf 2>;
 			power-domain-names = "psci", "perf";
 			next-level-cache = <&l2_2>;
+			#cooling-cells = <2>;
 		};
 
 		cpu-map {
@@ -2314,11 +2335,9 @@ usb_mp_qmpphy0: phy@fa3000 {
 
 			clocks = <&gcc GCC_USB3_MP_PHY_AUX_CLK>,
 				 <&tcsr TCSR_USB3_0_CLKREF_EN>,
-				 <&rpmhcc RPMH_CXO_CLK>,
 				 <&gcc GCC_USB3_MP_PHY_COM_AUX_CLK>,
 				 <&gcc GCC_USB3_MP_PHY_PIPE_0_CLK>;
 			clock-names = "aux",
-				      "clkref",
 				      "ref",
 				      "com_aux",
 				      "pipe";
@@ -2343,11 +2362,9 @@ usb_mp_qmpphy1: phy@fa5000 {
 
 			clocks = <&gcc GCC_USB3_MP_PHY_AUX_CLK>,
 				 <&tcsr TCSR_USB3_1_CLKREF_EN>,
-				 <&rpmhcc RPMH_CXO_CLK>,
 				 <&gcc GCC_USB3_MP_PHY_COM_AUX_CLK>,
 				 <&gcc GCC_USB3_MP_PHY_PIPE_1_CLK>;
 			clock-names = "aux",
-				      "clkref",
 				      "ref",
 				      "com_aux",
 				      "pipe";
@@ -2482,15 +2499,13 @@ usb_1_qmpphy: phy@fde000 {
 			reg = <0x0 0x00fde000 0x0 0x8000>;
 
 			clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK>,
-				 <&rpmhcc RPMH_CXO_CLK>,
+				 <&tcsr TCSR_USB4_1_CLKREF_EN>,
 				 <&gcc GCC_USB3_SEC_PHY_COM_AUX_CLK>,
-				 <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>,
-				 <&tcsr TCSR_USB4_1_CLKREF_EN>;
+				 <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>;
 			clock-names = "aux",
 				      "ref",
 				      "com_aux",
-				      "usb3_pipe",
-				      "clkref";
+				      "usb3_pipe";
 
 			power-domains = <&gcc GCC_USB_1_PHY_GDSC>;
 
@@ -2776,10 +2791,10 @@ pcie4: pci@1bf0000 {
 
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
-			interrupt-map = <0 0 0 1 &intc 0 0 0 513 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &intc 0 0 0 514 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &intc 0 0 0 515 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &intc 0 0 0 516 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 513 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 GIC_SPI 514 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 GIC_SPI 515 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 GIC_SPI 516 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks = <&gcc GCC_PCIE_4_AUX_CLK>,
 				 <&gcc GCC_PCIE_4_CFG_AHB_CLK>,
@@ -2986,10 +3001,10 @@ pcie5: pci@1b40000 {
 
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
-			interrupt-map = <0 0 0 1 &intc 0 0 0 526 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &intc 0 0 0 428 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &intc 0 0 0 429 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &intc 0 0 0 435 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 526 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 GIC_SPI 428 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 GIC_SPI 429 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks = <&gcc GCC_PCIE_5_AUX_CLK>,
 				 <&gcc GCC_PCIE_5_CFG_AHB_CLK>,
@@ -3252,10 +3267,10 @@ pcie6: pci@1c00000 {
 
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
-			interrupt-map = <0 0 0 1 &intc 0 0 0 472 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &intc 0 0 0 473 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &intc 0 0 0 474 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &intc 0 0 0 475 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 472 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 GIC_SPI 473 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 GIC_SPI 474 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 GIC_SPI 475 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks = <&gcc GCC_PCIE_6_AUX_CLK>,
 				 <&gcc GCC_PCIE_6_CFG_AHB_CLK>,
@@ -3462,10 +3477,10 @@ pcie3b: pci@1b80000 {
 
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
-			interrupt-map = <0 0 0 1 &intc 0 0 0 831 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &intc 0 0 0 832 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &intc 0 0 0 833 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &intc 0 0 0 834 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 831 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 GIC_SPI 832 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 GIC_SPI 833 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 GIC_SPI 834 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks = <&gcc GCC_PCIE_3B_AUX_CLK>,
 				 <&gcc GCC_PCIE_3B_CFG_AHB_CLK>,
@@ -3675,6 +3690,32 @@ pcie3b_phy: phy@f10000 {
 			status = "disabled";
 		};
 
+		cryptobam: dma-controller@1dc4000 {
+			compatible = "qcom,bam-v1.7.4", "qcom,bam-v1.7.0";
+			reg = <0x0 0x01dc4000 0x0 0x28000>;
+			interrupts = <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>;
+			#dma-cells = <1>;
+			iommus = <&apps_smmu 0x80 0x0>,
+				 <&apps_smmu 0x81 0x0>;
+			qcom,ee = <0>;
+			qcom,controlled-remotely;
+			num-channels = <20>;
+			qcom,num-ees = <4>;
+		};
+
+		crypto: crypto@1dfa000 {
+			compatible = "qcom,glymur-qce", "qcom,sm8150-qce", "qcom,qce";
+			reg = <0x0 0x01dfa000 0x0 0x6000>;
+			dmas = <&cryptobam 4>, <&cryptobam 5>;
+			dma-names = "rx",
+				    "tx";
+			iommus = <&apps_smmu 0x80 0x0>,
+				 <&apps_smmu 0x81 0x0>;
+			interconnects = <&aggre1_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "memory";
+		};
+
 		tcsr_mutex: hwlock@1f40000 {
 			compatible = "qcom,tcsr-mutex";
 			reg = <0x0 0x01f40000 0x0 0x20000>;
@@ -3698,6 +3739,34 @@ hsc_noc: interconnect@2000000 {
 			#interconnect-cells = <2>;
 		};
 
+		gxclkctl: clock-controller@3d64000 {
+			compatible = "qcom,glymur-gxclkctl";
+			reg = <0x0 0x03d64000 0x0 0x6000>;
+
+			power-domains = <&rpmhpd RPMHPD_GFX>,
+					<&rpmhpd RPMHPD_GMXC>,
+					<&gpucc GPU_CC_CX_GDSC>;
+
+			#power-domain-cells = <1>;
+		};
+
+		gpucc: clock-controller@3d90000 {
+			compatible = "qcom,glymur-gpucc";
+			reg = <0x0 0x03d90000 0x0 0x9800>;
+			clocks = <&rpmhcc RPMH_CXO_CLK>,
+				<&gcc GCC_GPU_GPLL0_CLK_SRC>,
+				<&gcc GCC_GPU_GPLL0_DIV_CLK_SRC>;
+
+			power-domains = <&rpmhpd RPMHPD_MX>,
+					<&rpmhpd RPMHPD_CX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		ipcc: mailbox@3e04000 {
 			compatible = "qcom,glymur-ipcc", "qcom,ipcc";
 			reg = <0x0 0x03e04000 0x0 0x1000>;
@@ -3709,6 +3778,122 @@ ipcc: mailbox@3e04000 {
 			#mbox-cells = <2>;
 		};
 
+		remoteproc_adsp: remoteproc@6800000 {
+			compatible = "qcom,glymur-adsp-pas", "qcom,sm8550-adsp-pas";
+			reg = <0x0 0x06800000 0x0 0x10000>;
+
+			iommus = <&apps_smmu 0x1000 0x0>;
+
+			interrupts-extended = <&pdc 6 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 1 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 2 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 3 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_adsp_in 7 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog",
+					  "fatal",
+					  "ready",
+					  "handover",
+					  "stop-ack",
+					  "shutdown-ack";
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "xo";
+
+			interconnects = <&lpass_lpicx_noc MASTER_LPASS_PROC QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+
+			power-domains = <&rpmhpd RPMHPD_LCX>,
+					<&rpmhpd RPMHPD_LMX>;
+			power-domain-names = "lcx",
+					     "lmx";
+
+			memory-region = <&adspslpi_mem>, <&q6_adsp_dtb_mem>;
+
+			qcom,qmp = <&aoss_qmp>;
+
+			qcom,smem-states = <&smp2p_adsp_out 0>;
+			qcom,smem-state-names = "stop";
+
+			status = "disabled";
+
+			remoteproc_adsp_glink: glink-edge {
+				interrupts-extended = <&ipcc IPCC_MPROC_LPASS
+							     IPCC_MPROC_SIGNAL_GLINK_QMP
+							     IRQ_TYPE_EDGE_RISING>;
+
+				mboxes = <&ipcc IPCC_MPROC_LPASS
+						IPCC_MPROC_SIGNAL_GLINK_QMP>;
+
+				qcom,remote-pid = <2>;
+
+				label = "lpass";
+
+				fastrpc {
+					compatible = "qcom,glymur-fastrpc", "qcom,kaanapali-fastrpc";
+					qcom,glink-channels = "fastrpcglink-apps-dsp";
+					label = "adsp";
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					compute-cb@3 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <3>;
+
+						iommus = <&apps_smmu 0x1003 0x80>,
+							 <&apps_smmu 0x1063 0x20>;
+						dma-coherent;
+					};
+
+					compute-cb@4 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <4>;
+
+						iommus = <&apps_smmu 0x1004 0x80>,
+							 <&apps_smmu 0x1064 0x20>;
+						dma-coherent;
+					};
+
+					compute-cb@5 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <5>;
+
+						iommus = <&apps_smmu 0x1005 0x80>,
+							 <&apps_smmu 0x1065 0x20>;
+						dma-coherent;
+					};
+
+					compute-cb@6 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <6>;
+
+						iommus = <&apps_smmu 0x1006 0x80>,
+							 <&apps_smmu 0x1066 0x20>;
+						dma-coherent;
+					};
+
+					compute-cb@7 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <7>;
+
+						iommus = <&apps_smmu 0x1007 0x40>,
+							 <&apps_smmu 0x1067 0x0>,
+							 <&apps_smmu 0x1087 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@8 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <8>;
+
+						iommus = <&apps_smmu 0x1008 0x80>,
+							 <&apps_smmu 0x1068 0x20>;
+						dma-coherent;
+					};
+				};
+			};
+		};
+
 		lpass_lpiaon_noc: interconnect@7400000 {
 			compatible = "qcom,glymur-lpass-lpiaon-noc";
 			reg = <0x0 0x07400000 0x0 0x19080>;
@@ -3750,15 +3935,13 @@ usb_2_qmpphy: phy@88e1000 {
 			reg = <0x0 0x088e1000 0x0 0x8000>;
 
 			clocks = <&gcc GCC_USB3_TERT_PHY_AUX_CLK>,
-				 <&rpmhcc RPMH_CXO_CLK>,
+				 <&tcsr TCSR_USB4_2_CLKREF_EN>,
 				 <&gcc GCC_USB3_TERT_PHY_COM_AUX_CLK>,
-				 <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>,
-				 <&tcsr TCSR_USB4_2_CLKREF_EN>;
+				 <&gcc GCC_USB3_TERT_PHY_PIPE_CLK>;
 			clock-names = "aux",
 				      "ref",
 				      "com_aux",
-				      "usb3_pipe",
-				      "clkref";
+				      "usb3_pipe";
 
 			power-domains = <&gcc GCC_USB_2_PHY_GDSC>;
 
@@ -3928,6 +4111,8 @@ usb_1: usb@a800000 {
 			snps,dis_u2_susphy_quirk;
 			snps,dis_enblslpm_quirk;
 
+			usb-role-switch;
+
 			status = "disabled";
 
 			ports {
@@ -4001,6 +4186,8 @@ usb_2: usb@a000000 {
 			snps,dis_u2_susphy_quirk;
 			snps,dis_enblslpm_quirk;
 
+			usb-role-switch;
+
 			status = "disabled";
 
 			ports {
@@ -4024,7 +4211,7 @@ usb_2_dwc3_ss: endpoint {
 			};
 		};
 
-		usb_hs: usb@a2f8800 {
+		usb_hs: usb@a200000 {
 			compatible = "qcom,glymur-dwc3", "qcom,snps-dwc3";
 			reg = <0x0 0x0a200000 0x0 0xfc100>;
 
@@ -4589,6 +4776,22 @@ mdss_dp3_out: endpoint {
 			};
 		};
 
+		videocc: clock-controller@aaf0000 {
+			compatible = "qcom,glymur-videocc";
+			reg = <0x0 0x0aaf0000 0x0 0x10000>;
+			clocks = <&rpmhcc RPMH_CXO_CLK>,
+				 <&rpmhcc RPMH_CXO_CLK_A>;
+
+			power-domains = <&rpmhpd RPMHPD_MMCX>,
+					<&rpmhpd RPMHPD_MXC>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		dispcc: clock-controller@af00000 {
 			compatible = "qcom,glymur-dispcc";
 			reg = <0x0 0x0af00000 0x0 0x20000>;
@@ -5873,7 +6076,7 @@ oobm_ss_noc: interconnect@1f300000 {
 			#interconnect-cells = <2>;
 		};
 
-		system-cache-controller@20400000 {
+		system-cache-controller@21800000 {
 			compatible = "qcom,glymur-llcc";
 			reg = <0x0 0x21800000 0x0 0x100000>,
 			      <0x0 0x21a00000 0x0 0x100000>,
@@ -5907,6 +6110,176 @@ system-cache-controller@20400000 {
 			interrupts = <GIC_SPI 266 IRQ_TYPE_LEVEL_HIGH>;
 		};
 
+		remoteproc_cdsp: remoteproc@32300000 {
+			compatible = "qcom,glymur-cdsp-pas", "qcom,sm8550-cdsp-pas";
+			reg = <0x0 0x32300000 0x0 0x10000>;
+
+			iommus = <&apps_smmu 0x2400 0x400>;
+
+			interrupts-extended = <&intc GIC_SPI 578 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 0 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 1 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 2 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 3 IRQ_TYPE_EDGE_RISING>,
+					      <&smp2p_cdsp_in 7 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "wdog",
+					  "fatal",
+					  "ready",
+					  "handover",
+					  "stop-ack",
+					  "shutdown-ack";
+
+			clocks = <&rpmhcc RPMH_CXO_CLK>;
+			clock-names = "xo";
+
+			interconnects = <&nsp_noc MASTER_CDSP_PROC QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+
+			power-domains = <&rpmhpd RPMHPD_CX>,
+					<&rpmhpd RPMHPD_MXC>,
+					<&rpmhpd RPMHPD_NSP>;
+			power-domain-names = "cx",
+					     "mxc",
+					     "nsp";
+
+			memory-region = <&cdsp_mem>, <&q6_cdsp_dtb_mem>;
+			qcom,qmp = <&aoss_qmp>;
+			qcom,smem-states = <&smp2p_cdsp_out 0>;
+			qcom,smem-state-names = "stop";
+
+			status = "disabled";
+
+			glink-edge {
+				interrupts-extended = <&ipcc IPCC_MPROC_CDSP
+							     IPCC_MPROC_SIGNAL_GLINK_QMP
+							     IRQ_TYPE_EDGE_RISING>;
+				mboxes = <&ipcc IPCC_MPROC_CDSP
+						IPCC_MPROC_SIGNAL_GLINK_QMP>;
+				qcom,remote-pid = <5>;
+				label = "cdsp";
+
+				fastrpc {
+					compatible = "qcom,glymur-fastrpc", "qcom,kaanapali-fastrpc";
+					qcom,glink-channels = "fastrpcglink-apps-dsp";
+					label = "cdsp";
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					compute-cb@1 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <1>;
+
+						iommus = <&apps_smmu 0x2401 0x440>,
+							 <&apps_smmu 0x1961 0x0>,
+							 <&apps_smmu 0x19c1 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@2 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <2>;
+
+						iommus = <&apps_smmu 0x2402 0x440>,
+							 <&apps_smmu 0x1962 0x0>,
+							 <&apps_smmu 0x19c2 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@3 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <3>;
+
+						iommus = <&apps_smmu 0x2403 0x440>,
+							 <&apps_smmu 0x1963 0x0>,
+							 <&apps_smmu 0x19c3 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@4 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <4>;
+
+						iommus = <&apps_smmu 0x2404 0x440>,
+							 <&apps_smmu 0x1964 0x0>,
+							 <&apps_smmu 0x19c4 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@5 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <5>;
+
+						iommus = <&apps_smmu 0x2405 0x440>,
+							 <&apps_smmu 0x1965 0x0>,
+							 <&apps_smmu 0x19c5 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@6 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <6>;
+
+						iommus = <&apps_smmu 0x2406 0x440>,
+							 <&apps_smmu 0x1966 0x0>,
+							 <&apps_smmu 0x19c6 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@7 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <7>;
+
+						iommus = <&apps_smmu 0x2407 0x440>,
+							 <&apps_smmu 0x1967 0x0>,
+							 <&apps_smmu 0x19c7 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@8 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <8>;
+
+						iommus = <&apps_smmu 0x2408 0x440>,
+							 <&apps_smmu 0x1968 0x0>,
+							 <&apps_smmu 0x19c8 0x0>;
+						dma-coherent;
+					};
+
+					/* note: compute-cb@9 is secure */
+
+					compute-cb@10 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <10>;
+
+						iommus = <&apps_smmu 0x240c 0x440>,
+							 <&apps_smmu 0x196c 0x0>,
+							 <&apps_smmu 0x19cc 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@11 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <11>;
+
+						iommus = <&apps_smmu 0x240d 0x440>,
+							 <&apps_smmu 0x196d 0x0>,
+							 <&apps_smmu 0x19cd 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@12 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <12>;
+
+						iommus = <&apps_smmu 0x240e 0x440>,
+							 <&apps_smmu 0x196e 0x0>,
+							 <&apps_smmu 0x19ce 0x0>;
+						dma-coherent;
+					};
+				};
+			};
+		};
+
 		nsp_noc: interconnect@320c0000 {
 			compatible = "qcom,glymur-nsp-noc";
 			reg = <0x0 0x320c0000 0x0 0x21280>;
@@ -5914,7 +6287,7 @@ nsp_noc: interconnect@320c0000 {
 			#interconnect-cells = <2>;
 		};
 
-		imem: sram@81e08000 {
+		imem: sram@81e08600 {
 			compatible = "mmio-sram";
 			reg = <0x0 0x81e08600 0x0 0x300>;
 
diff --git a/arch/arm64/boot/dts/qcom/hamoa.dtsi b/arch/arm64/boot/dts/qcom/hamoa.dtsi
index 051dee0764167..4ba751a65142b 100644
--- a/arch/arm64/boot/dts/qcom/hamoa.dtsi
+++ b/arch/arm64/boot/dts/qcom/hamoa.dtsi
@@ -5744,18 +5744,18 @@ mdss_dp0_out: endpoint {
 				mdss_dp0_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-162000000 {
-						opp-hz = /bits/ 64 <162000000>;
-						required-opps = <&rpmhpd_opp_low_svs>;
-					};
-
 					opp-270000000 {
 						opp-hz = /bits/ 64 <270000000>;
-						required-opps = <&rpmhpd_opp_svs>;
+						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
 					opp-540000000 {
 						opp-hz = /bits/ 64 <540000000>;
+						required-opps = <&rpmhpd_opp_svs>;
+					};
+
+					opp-594000000 {
+						opp-hz = /bits/ 64 <594000000>;
 						required-opps = <&rpmhpd_opp_svs_l1>;
 					};
 
@@ -5796,7 +5796,7 @@ mdss_dp1: displayport-controller@ae98000 {
 							 <&usb_1_ss1_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>,
 							 <&usb_1_ss1_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>;
 
-				operating-points-v2 = <&mdss_dp1_opp_table>;
+				operating-points-v2 = <&mdss_dp0_opp_table>;
 
 				power-domains = <&rpmhpd RPMHPD_MMCX>;
 
@@ -5829,30 +5829,6 @@ mdss_dp1_out: endpoint {
 						};
 					};
 				};
-
-				mdss_dp1_opp_table: opp-table {
-					compatible = "operating-points-v2";
-
-					opp-162000000 {
-						opp-hz = /bits/ 64 <162000000>;
-						required-opps = <&rpmhpd_opp_low_svs>;
-					};
-
-					opp-270000000 {
-						opp-hz = /bits/ 64 <270000000>;
-						required-opps = <&rpmhpd_opp_svs>;
-					};
-
-					opp-540000000 {
-						opp-hz = /bits/ 64 <540000000>;
-						required-opps = <&rpmhpd_opp_svs_l1>;
-					};
-
-					opp-810000000 {
-						opp-hz = /bits/ 64 <810000000>;
-						required-opps = <&rpmhpd_opp_nom>;
-					};
-				};
 			};
 
 			mdss_dp2: displayport-controller@ae9a000 {
@@ -5885,7 +5861,7 @@ mdss_dp2: displayport-controller@ae9a000 {
 							 <&usb_1_ss2_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>,
 							 <&usb_1_ss2_qmpphy QMP_USB43DP_DP_VCO_DIV_CLK>;
 
-				operating-points-v2 = <&mdss_dp2_opp_table>;
+				operating-points-v2 = <&mdss_dp0_opp_table>;
 
 				power-domains = <&rpmhpd RPMHPD_MMCX>;
 
@@ -5917,30 +5893,6 @@ mdss_dp2_out: endpoint {
 						};
 					};
 				};
-
-				mdss_dp2_opp_table: opp-table {
-					compatible = "operating-points-v2";
-
-					opp-162000000 {
-						opp-hz = /bits/ 64 <162000000>;
-						required-opps = <&rpmhpd_opp_low_svs>;
-					};
-
-					opp-270000000 {
-						opp-hz = /bits/ 64 <270000000>;
-						required-opps = <&rpmhpd_opp_svs>;
-					};
-
-					opp-540000000 {
-						opp-hz = /bits/ 64 <540000000>;
-						required-opps = <&rpmhpd_opp_svs_l1>;
-					};
-
-					opp-810000000 {
-						opp-hz = /bits/ 64 <810000000>;
-						required-opps = <&rpmhpd_opp_nom>;
-					};
-				};
 			};
 
 			mdss_dp3: displayport-controller@aea0000 {
@@ -6004,19 +5956,14 @@ mdss_dp3_out: endpoint {
 				mdss_dp3_opp_table: opp-table {
 					compatible = "operating-points-v2";
 
-					opp-162000000 {
-						opp-hz = /bits/ 64 <162000000>;
-						required-opps = <&rpmhpd_opp_low_svs>;
-					};
-
 					opp-270000000 {
 						opp-hz = /bits/ 64 <270000000>;
-						required-opps = <&rpmhpd_opp_svs>;
+						required-opps = <&rpmhpd_opp_low_svs>;
 					};
 
-					opp-540000000 {
-						opp-hz = /bits/ 64 <540000000>;
-						required-opps = <&rpmhpd_opp_svs_l1>;
+					opp-594000000 {
+						opp-hz = /bits/ 64 <594000000>;
+						required-opps = <&rpmhpd_opp_svs>;
 					};
 
 					opp-810000000 {
diff --git a/arch/arm64/boot/dts/qcom/ipq5424.dtsi b/arch/arm64/boot/dts/qcom/ipq5424.dtsi
index f20cda4290949..876bf6a8b8ff0 100644
--- a/arch/arm64/boot/dts/qcom/ipq5424.dtsi
+++ b/arch/arm64/boot/dts/qcom/ipq5424.dtsi
@@ -639,7 +639,7 @@ qusb_phy_1: phy@71000 {
 			status = "disabled";
 		};
 
-		usb2: usb2@1e00000 {
+		usb2: usb2@1ef8800 {
 			compatible = "qcom,ipq5424-dwc3", "qcom,dwc3";
 			reg = <0 0x01ef8800 0 0x400>;
 			#address-cells = <2>;
@@ -730,7 +730,7 @@ ssphy_0: phy@7d000 {
 			status = "disabled";
 		};
 
-		usb3: usb3@8a00000 {
+		usb3: usb3@8af8800 {
 			compatible = "qcom,ipq5424-dwc3", "qcom,dwc3";
 			reg = <0 0x08af8800 0 0x400>;
 
diff --git a/arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts b/arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts
new file mode 100644
index 0000000000000..6871f3dc4eaf0
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/ipq9650-rdp488.dts
@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+/dts-v1/;
+
+#include "ipq9650.dtsi"
+
+/ {
+	model = "Qualcomm Technologies, Inc. IPQ9650 RDP488";
+	compatible = "qcom,ipq9650-rdp488", "qcom,ipq9650";
+
+	aliases {
+		serial0 = &uart1;
+	};
+
+	chosen {
+		stdout-path = "serial0";
+	};
+};
+
+&sdhc {
+	max-frequency = <192000000>;
+	bus-width = <4>;
+	mmc-ddr-1_8v;
+	mmc-hs200-1_8v;
+	pinctrl-0 = <&sdhc_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&sleep_clk {
+	clock-frequency = <32000>;
+};
+
+&tlmm {
+	qup_uart1_default_state: qup-uart1-default-state {
+		pins = "gpio43", "gpio44";
+		function = "qup_se6";
+		drive-strength = <8>;
+		bias-pull-down;
+	};
+
+	sdhc_default_state: sdhc-default-state {
+		clk-pins {
+			pins = "gpio5";
+			function = "sdc_clk";
+			drive-strength = <8>;
+			bias-disable;
+		};
+
+		cmd-pins {
+			pins = "gpio4";
+			function = "sdc_cmd";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+
+		data-pins {
+			pins = "gpio0", "gpio1", "gpio2", "gpio3";
+			function = "sdc_data";
+			drive-strength = <8>;
+			bias-pull-up;
+		};
+	};
+};
+
+&uart1 {
+	pinctrl-0 = <&qup_uart1_default_state>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&xo_board {
+	clock-frequency = <24000000>;
+};
diff --git a/arch/arm64/boot/dts/qcom/ipq9650.dtsi b/arch/arm64/boot/dts/qcom/ipq9650.dtsi
new file mode 100644
index 0000000000000..3d3b317c6a3cc
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/ipq9650.dtsi
@@ -0,0 +1,384 @@
+// SPDX-License-Identifier: BSD-3-Clause
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <dt-bindings/interrupt-controller/arm-gic.h>
+#include <dt-bindings/clock/qcom,ipq9650-gcc.h>
+#include <dt-bindings/reset/qcom,ipq9650-gcc.h>
+
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	interrupt-parent = <&intc>;
+
+	clocks {
+		sleep_clk: sleep-clk {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+		};
+
+		xo_board: xo-board-clk {
+			compatible = "fixed-clock";
+			#clock-cells = <0>;
+		};
+	};
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		cpu0: cpu@0 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x0>;
+			enable-method = "psci";
+			next-level-cache = <&l2_0>;
+
+			l2_0: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
+
+				l3_0: l3-cache {
+					compatible = "cache";
+					cache-level = <3>;
+					cache-unified;
+				};
+			};
+		};
+
+		cpu1: cpu@100 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x100>;
+			enable-method = "psci";
+			next-level-cache = <&l2_100>;
+
+			l2_100: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
+			};
+		};
+
+		cpu2: cpu@200 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x200>;
+			enable-method = "psci";
+
+			l2_200: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
+			};
+		};
+
+		cpu3: cpu@300 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a55";
+			reg = <0x300>;
+			enable-method = "psci";
+
+			l2_300: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
+			};
+		};
+
+		cpu4: cpu@400 {
+			device_type = "cpu";
+			compatible = "arm,cortex-a78";
+			reg = <0x400>;
+			enable-method = "psci";
+			next-level-cache = <&l2_400>;
+
+			l2_400: l2-cache {
+				compatible = "cache";
+				cache-level = <2>;
+				cache-unified;
+				next-level-cache = <&l3_0>;
+			};
+		};
+
+		cpu-map {
+			cluster0 {
+				core0 {
+					cpu = <&cpu0>;
+				};
+
+				core1 {
+					cpu = <&cpu1>;
+				};
+
+				core2 {
+					cpu = <&cpu2>;
+				};
+
+				core3 {
+					cpu = <&cpu3>;
+				};
+
+				core4 {
+					cpu = <&cpu4>;
+				};
+			};
+		};
+	};
+
+	firmware {
+		optee {
+			compatible = "linaro,optee-tz";
+			method = "smc";
+		};
+
+		scm {
+			compatible = "qcom,scm-ipq9650", "qcom,scm";
+		};
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/* We expect the bootloader to fill in the size */
+		reg = <0x0 0x80000000 0x0 0x0>;
+	};
+
+	pmu-a55 {
+		compatible = "arm,cortex-a55-pmu";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW &ppi_cluster0>;
+	};
+
+	pmu-a78 {
+		compatible = "arm,cortex-a78-pmu";
+		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW &ppi_cluster1>;
+	};
+
+	psci {
+		compatible = "arm,psci-1.0";
+		method = "smc";
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		bootloader@8a100000 {
+			reg = <0x0 0x8a100000 0x0 0x400000>;
+			no-map;
+		};
+
+		smem@8a500000 {
+			compatible = "qcom,smem";
+			reg = <0x0 0x8a500000 0x0 0x40000>;
+			no-map;
+
+			hwlocks = <&tcsr_mutex 3>;
+		};
+
+		tfa@8a600000 {
+			reg = <0x0 0x8a600000 0x0 0x80000>;
+			no-map;
+		};
+
+		optee@8a680000 {
+			reg = <0x0 0x8a680000 0x0 0x280000>;
+			no-map;
+		};
+	};
+
+	soc@0 {
+		compatible = "simple-bus";
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-ranges = <0 0 0 0 0x10 0>;
+		ranges = <0 0 0 0 0x10 0>;
+
+		tlmm: pinctrl@1000000 {
+			compatible = "qcom,ipq9650-tlmm";
+			reg = <0x0 0x01000000 0x0 0x300000>;
+			interrupts = <GIC_SPI 84 IRQ_TYPE_LEVEL_HIGH 0>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&tlmm 0 0 54>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+		};
+
+		gcc: clock-controller@1800000 {
+			compatible = "qcom,ipq9650-gcc";
+			reg = <0x0 0x01800000 0x0 0x40000>;
+			clocks = <&xo_board>,
+				 <&sleep_clk>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>,
+				 <0>;
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+		};
+
+		tcsr_mutex: hwlock@1917000 {
+			compatible = "qcom,tcsr-mutex";
+			reg = <0x0 0x01917000 0x0 0x57000>;
+			#hwlock-cells = <1>;
+		};
+
+		qupv3: geniqup@1ac0000 {
+			compatible = "qcom,geni-se-qup";
+			reg = <0x0 0x01ac0000 0x0 0x2000>;
+			clocks = <&gcc GCC_QUPV3_AHB_MST_CLK>,
+				 <&gcc GCC_QUPV3_AHB_SLV_CLK>;
+			clock-names = "m-ahb", "s-ahb";
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			uart1: serial@1a98000 {
+				compatible = "qcom,geni-debug-uart";
+				reg = <0x0 0x01a98000 0x0 0x4000>;
+				clocks = <&gcc GCC_QUPV3_WRAP_SE6_CLK>;
+				clock-names = "se";
+				interrupts = <GIC_SPI 380 IRQ_TYPE_LEVEL_HIGH 0>;
+
+				status = "disabled";
+			};
+		};
+
+		sdhc: mmc@7804000 {
+			compatible = "qcom,ipq9650-sdhci", "qcom,sdhci-msm-v5";
+			reg = <0x0 0x07804000 0x0 0x1000>,
+			      <0x0 0x07805000 0x0 0x1000>;
+			reg-names = "hc",
+				    "cqhci";
+
+			interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH 0>,
+				     <GIC_SPI 119 IRQ_TYPE_LEVEL_HIGH 0>;
+			interrupt-names = "hc_irq",
+					  "pwr_irq";
+
+			clocks = <&gcc GCC_SDCC1_AHB_CLK>,
+				 <&gcc GCC_SDCC1_APPS_CLK>,
+				 <&xo_board>;
+			clock-names = "iface",
+				      "core",
+				      "xo";
+			non-removable;
+
+			status = "disabled";
+		};
+
+		intc: interrupt-controller@f200000 {
+			compatible = "arm,gic-v3";
+			reg = <0x0 0x0f200000 0x0 0x10000>,
+			      <0x0 0x0f240000 0x0 0xa0000>;
+			#interrupt-cells = <0x4>;
+			interrupt-controller;
+			#redistributor-regions = <1>;
+			redistributor-stride = <0x0 0x20000>;
+			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_HIGH 0>;
+			mbi-ranges = <800 160>;
+			msi-controller;
+
+			ppi-partitions {
+				ppi_cluster0: interrupt-partition-0 {
+					affinity = <&cpu0 &cpu1 &cpu2 &cpu3>;
+				};
+
+				ppi_cluster1: interrupt-partition-1 {
+					affinity = <&cpu4>;
+				};
+			};
+		};
+
+		watchdog@f410000 {
+			compatible = "qcom,apss-wdt-ipq9650", "qcom,kpss-wdt";
+			reg = <0x0 0x0f410000 0x0 0x1000>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_EDGE_RISING 0>;
+			clocks = <&sleep_clk>;
+		};
+
+		timer@f420000 {
+			compatible = "arm,armv7-timer-mem";
+			reg = <0x0 0x0f420000 0x0 0x1000>;
+			ranges = <0 0 0 0x10000000>;
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			frame@f421000 {
+				reg = <0x0f421000 0x1000>,
+				      <0x0f422000 0x1000>;
+				interrupts = <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH 0>,
+					     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH 0>;
+				frame-number = <0>;
+			};
+
+			frame@f423000 {
+				reg = <0x0f423000 0x1000>;
+				interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH 0>;
+				frame-number = <1>;
+
+				status = "disabled";
+			};
+
+			frame@f425000 {
+				reg = <0x0f425000 0x1000>,
+				      <0x0f426000 0x1000>;
+				interrupts = <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH 0>;
+				frame-number = <2>;
+
+				status = "disabled";
+			};
+
+			frame@f427000 {
+				reg = <0x0f427000 0x1000>;
+				interrupts = <GIC_SPI 11 IRQ_TYPE_LEVEL_HIGH 0>;
+				frame-number = <3>;
+
+				status = "disabled";
+			};
+
+			frame@f429000 {
+				reg = <0x0f429000 0x1000>;
+				interrupts = <GIC_SPI 12 IRQ_TYPE_LEVEL_HIGH 0>;
+				frame-number = <4>;
+
+				status = "disabled";
+			};
+
+			frame@f42b000 {
+				reg = <0x0f42b000 0x1000>;
+				interrupts = <GIC_SPI 13 IRQ_TYPE_LEVEL_HIGH 0>;
+				frame-number = <5>;
+
+				status = "disabled";
+			};
+
+			frame@f42d000 {
+				reg = <0x0f42d000 0x1000>;
+				interrupts = <GIC_SPI 14 IRQ_TYPE_LEVEL_HIGH 0>;
+				frame-number = <6>;
+
+				status = "disabled";
+			};
+		};
+	};
+
+	timer {
+		compatible = "arm,armv8-timer";
+		interrupts = <GIC_PPI 13 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 14 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 11 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 10 IRQ_TYPE_LEVEL_LOW 0>,
+			     <GIC_PPI 12 IRQ_TYPE_LEVEL_LOW 0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
index 07247dc98b701..f9b5b5718b904 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
+++ b/arch/arm64/boot/dts/qcom/kaanapali-mtp.dts
@@ -952,6 +952,11 @@ wifi@0 {
 	};
 };
 
+&pmih0108_e1_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_1p8>;
+	vdd3-supply = <&vreg_l5b_3p1>;
+};
+
 &pmh0101_flash {
 	status = "okay";
 
@@ -1321,3 +1326,25 @@ &ufs_mem_phy {
 
 	status = "okay";
 };
+
+&usb {
+	dr_mode = "peripheral";
+
+	status = "okay";
+};
+
+&usb_hsphy {
+	vdd-supply = <&vreg_l4f_0p8>;
+	vdda12-supply = <&vreg_l1d_1p2>;
+
+	phys = <&pmih0108_e1_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy {
+	vdda-phy-supply = <&vreg_l1d_1p2>;
+	vdda-pll-supply = <&vreg_l4f_0p8>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
index da0e8f9091c36..55d02219ef4e9 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
+++ b/arch/arm64/boot/dts/qcom/kaanapali-qrd.dts
@@ -744,6 +744,11 @@ led@3 {
 	};
 };
 
+&pmih0108_e1_eusb2_repeater {
+	vdd18-supply = <&vreg_l15b_1p8>;
+	vdd3-supply = <&vreg_l5b_3p1>;
+};
+
 &pon_resin {
 	linux,code = <KEY_VOLUMEDOWN>;
 
@@ -816,3 +821,25 @@ &ufs_mem_phy {
 
 	status = "okay";
 };
+
+&usb {
+	dr_mode = "peripheral";
+
+	status = "okay";
+};
+
+&usb_hsphy {
+	vdd-supply = <&vreg_l4f_0p8>;
+	vdda12-supply = <&vreg_l1d_1p2>;
+
+	phys = <&pmih0108_e1_eusb2_repeater>;
+
+	status = "okay";
+};
+
+&usb_dp_qmpphy {
+	vdda-phy-supply = <&vreg_l1d_1p2>;
+	vdda-pll-supply = <&vreg_l4f_0p8>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/kaanapali.dtsi b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
index 7cc326aa1a1aa..bcd1cee31356c 100644
--- a/arch/arm64/boot/dts/qcom/kaanapali.dtsi
+++ b/arch/arm64/boot/dts/qcom/kaanapali.dtsi
@@ -2313,10 +2313,10 @@ &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
 			iommu-map = <0 &apps_smmu 0x1400 0x1>,
 				    <0x100 &apps_smmu 0x1401 0x1>;
 
-			interrupt-map = <0 0 0 1 &intc 0 0 0 149 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &intc 0 0 0 150 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &intc 0 0 0 151 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &intc 0 0 0 152 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
 			interrupt-map-mask = <0 0 0 0x7>;
 			#interrupt-cells = <1>;
 
@@ -3138,6 +3138,77 @@ opp-202000000 {
 			};
 		};
 
+		usb_hsphy: phy@88e3000 {
+			compatible = "qcom,kaanapali-m31-eusb2-phy",
+				     "qcom,sm8750-m31-eusb2-phy";
+			reg = <0x0 0x88e3000 0x0 0x29c>;
+
+			clocks = <&tcsr TCSR_USB2_CLKREF_EN>;
+			clock-names = "ref";
+
+			resets = <&gcc GCC_QUSB2PHY_PRIM_BCR>;
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		usb_dp_qmpphy: phy@88e8000 {
+			compatible = "qcom,kaanapali-qmp-usb3-dp-phy",
+				     "qcom,sm8750-qmp-usb3-dp-phy";
+			reg = <0x0 0x088e8000 0x0 0x4000>;
+
+			clocks = <&gcc GCC_USB3_PRIM_PHY_AUX_CLK>,
+				 <&tcsr TCSR_USB3_CLKREF_EN>,
+				 <&gcc GCC_USB3_PRIM_PHY_COM_AUX_CLK>,
+				 <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>;
+			clock-names = "aux",
+				      "ref",
+				      "com_aux",
+				      "usb3_pipe";
+
+			resets = <&gcc GCC_USB3_PHY_PRIM_BCR>,
+				 <&gcc GCC_USB3_DP_PHY_PRIM_BCR>;
+			reset-names = "phy",
+				      "common";
+
+			power-domains = <&gcc GCC_USB3_PHY_GDSC>;
+
+			#clock-cells = <1>;
+			#phy-cells = <1>;
+
+			orientation-switch;
+
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_dp_qmpphy_out: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usb_dp_qmpphy_usb_ss_in: endpoint {
+						remote-endpoint = <&usb_dwc3_ss>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					usb_dp_qmpphy_dp_in: endpoint {
+					};
+				};
+			};
+		};
+
 		camcc: clock-controller@956d000 {
 			compatible = "qcom,kaanapali-camcc";
 			reg = <0x0 0x0956d000 0x0 0x80000>;
@@ -3422,6 +3493,89 @@ dispcc: clock-controller@9ba2000 {
 			#reset-cells = <1>;
 		};
 
+		usb: usb@a600000 {
+			compatible = "qcom,kaanapali-dwc3", "qcom,snps-dwc3";
+			reg = <0x0 0x0a600000 0x0 0xfc100>;
+
+			clocks = <&gcc GCC_CFG_NOC_USB3_PRIM_AXI_CLK>,
+				 <&gcc GCC_USB30_PRIM_MASTER_CLK>,
+				 <&gcc GCC_AGGRE_USB3_PRIM_AXI_CLK>,
+				 <&gcc GCC_USB30_PRIM_SLEEP_CLK>,
+				 <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>;
+			clock-names = "cfg_noc",
+				      "core",
+				      "iface",
+				      "sleep",
+				      "mock_utmi";
+
+			assigned-clocks = <&gcc GCC_USB30_PRIM_MOCK_UTMI_CLK>,
+					  <&gcc GCC_USB30_PRIM_MASTER_CLK>;
+			assigned-clock-rates = <19200000>, <200000000>;
+
+			interrupts-extended = <&intc GIC_SPI 133 IRQ_TYPE_LEVEL_HIGH>,
+					      <&intc GIC_SPI 130 IRQ_TYPE_LEVEL_HIGH>,
+					      <&intc GIC_SPI 131 IRQ_TYPE_LEVEL_HIGH>,
+					      <&pdc 14 IRQ_TYPE_EDGE_BOTH>,
+					      <&pdc 15 IRQ_TYPE_EDGE_BOTH>,
+					      <&pdc 17 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "dwc_usb3",
+					  "pwr_event",
+					  "hs_phy_irq",
+					  "dp_hs_phy_irq",
+					  "dm_hs_phy_irq",
+					  "ss_phy_irq";
+
+			power-domains = <&gcc GCC_USB30_PRIM_GDSC>;
+			required-opps = <&rpmhpd_opp_nom>;
+
+			resets = <&gcc GCC_USB30_PRIM_BCR>;
+
+			interconnects = <&aggre_noc MASTER_USB3 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_USB3 QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "usb-ddr", "apps-usb";
+			iommus = <&apps_smmu 0x40 0x0>;
+
+			phys = <&usb_hsphy>, <&usb_dp_qmpphy QMP_USB43DP_USB3_PHY>;
+			phy-names = "usb2-phy", "usb3-phy";
+
+			snps,hird-threshold = /bits/ 8 <0x0>;
+			snps,usb2-gadget-lpm-disable;
+			snps,dis_u2_susphy_quirk;
+			snps,dis_enblslpm_quirk;
+			snps,dis-u1-entry-quirk;
+			snps,dis-u2-entry-quirk;
+			snps,is-utmi-l1-suspend;
+			snps,usb3_lpm_capable;
+			snps,usb2-lpm-disable;
+			snps,has-lpm-erratum;
+			tx-fifo-resize;
+			dma-coherent;
+
+			status = "disabled";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					usb_dwc3_hs: endpoint {
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					usb_dwc3_ss: endpoint {
+						remote-endpoint = <&usb_dp_qmpphy_usb_ss_in>;
+					};
+				};
+			};
+		};
+
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,kaanapali-pdc", "qcom,pdc";
 			reg = <0x0 0x0b220000 0x0 0x10000>,
diff --git a/arch/arm64/boot/dts/qcom/kodiak.dtsi b/arch/arm64/boot/dts/qcom/kodiak.dtsi
index 988ca5f7c8a0e..c51beada8c7de 100644
--- a/arch/arm64/boot/dts/qcom/kodiak.dtsi
+++ b/arch/arm64/boot/dts/qcom/kodiak.dtsi
@@ -2578,7 +2578,7 @@ ufs_mem_phy: phy@1d87000 {
 		ice: crypto@1d88000 {
 			compatible = "qcom,sc7280-inline-crypto-engine",
 				     "qcom,inline-crypto-engine";
-			reg = <0 0x01d88000 0 0x8000>;
+			reg = <0 0x01d88000 0 0x18000>;
 			clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
 		};
 
@@ -4418,9 +4418,12 @@ qspi: spi@88dc000 {
 			clocks = <&gcc GCC_QSPI_CNOC_PERIPH_AHB_CLK>,
 				 <&gcc GCC_QSPI_CORE_CLK>;
 			clock-names = "iface", "core";
-			interconnects = <&gem_noc MASTER_APPSS_PROC 0
-					&cnoc2 SLAVE_QSPI_0 0>;
-			interconnect-names = "qspi-config";
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &cnoc2 SLAVE_QSPI_0 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&aggre1_noc MASTER_QSPI_0 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "qspi-config",
+					     "qspi-memory";
 			power-domains = <&rpmhpd SC7280_CX>;
 			operating-points-v2 = <&qspi_opp_table>;
 			status = "disabled";
diff --git a/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso
index 268fc6b05d4b4..44bd9b1a17652 100644
--- a/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso
+++ b/arch/arm64/boot/dts/qcom/lemans-evk-ifp-mezzanine.dtso
@@ -11,6 +11,30 @@
 &{/} {
 	model = "Qualcomm Technologies, Inc. Lemans-evk IFP Mezzanine";
 
+	dp2-connector {
+		compatible = "dp-connector";
+		label = "eDP2";
+		type = "full-size";
+
+		port {
+			dp2_connector_in: endpoint {
+				remote-endpoint = <&mdss1_dp0_out>;
+			};
+		};
+	};
+
+	dp3-connector {
+		compatible = "dp-connector";
+		label = "eDP3";
+		type = "full-size";
+
+		port {
+			dp3_connector_in: endpoint {
+				remote-endpoint = <&mdss1_dp1_out>;
+			};
+		};
+	};
+
 	vreg_0p9: regulator-0v9 {
 		compatible = "regulator-fixed";
 		regulator-name = "VREG_0P9";
@@ -141,6 +165,44 @@ mac_addr1: mac-addr@0 {
 	};
 };
 
+&mdss1 {
+	status = "okay";
+};
+
+&mdss1_dp0 {
+	pinctrl-0 = <&dp2_hot_plug_det>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&mdss1_dp1 {
+	pinctrl-0 = <&dp3_hot_plug_det>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&mdss1_dp0_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+	remote-endpoint = <&dp2_connector_in>;
+};
+
+&mdss1_dp1_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+	remote-endpoint = <&dp3_connector_in>;
+};
+
+&mdss1_dp0_phy {
+	status = "okay";
+};
+
+&mdss1_dp1_phy {
+	status = "okay";
+};
+
 &pcie0 {
 	iommu-map = <0x0 &pcie_smmu 0x0 0x1>,
 		    <0x100 &pcie_smmu 0x1 0x1>,
@@ -235,6 +297,18 @@ &serdes1 {
 };
 
 &tlmm {
+	dp2_hot_plug_det: dp2-hot-plug-det-state {
+		pins = "gpio104";
+		function = "edp2_hot";
+		bias-disable;
+	};
+
+	dp3_hot_plug_det: dp3-hot-plug-det-state {
+		pins = "gpio103";
+		function = "edp3_hot";
+		bias-disable;
+	};
+
 	ethernet1_default: ethernet1-default-state {
 		ethernet1-mdc-pins {
 			pins = "gpio20";
diff --git a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi
index 31bd00546d55f..40f88498999bd 100644
--- a/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/lemans-ride-common.dtsi
@@ -53,6 +53,16 @@ vreg_3p0: vreg-3p0-regulator {
 		regulator-max-microvolt = <3000000>;
 	};
 
+	vreg_3p3: vreg-3p3-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "VREG_3P3";
+
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+	};
+
 	vreg_conn_1p8: vreg_conn_1p8 {
 		compatible = "regulator-fixed";
 		regulator-name = "vreg_conn_1p8";
@@ -69,6 +79,22 @@ vreg_conn_pa: vreg_conn_pa {
 		gpio = <&pmm8654au_1_gpios 6 GPIO_ACTIVE_HIGH>;
 	};
 
+	/*
+	 * TODO: These two regulators are actually part of the removable M.2
+	 * card and not the mainboard. Need to describe this differently.
+	 * Functionally it works correctly, because all we need to do is to
+	 * turn on the actual 3.3V supply above.
+	 */
+	vreg_wcn_0p95: vreg-wcn-0p95-regulator {
+		compatible = "regulator-fixed";
+
+		regulator-name = "VREG_WCN_0P95";
+		regulator-min-microvolt = <950000>;
+		regulator-max-microvolt = <950000>;
+
+		vin-supply = <&vreg_3p3>;
+	};
+
 	wcn6855-pmu {
 		compatible = "qcom,wcn6855-pmu";
 
@@ -78,6 +104,8 @@ wcn6855-pmu {
 		vddio-supply = <&vreg_conn_pa>;
 		vddaon-supply = <&vreg_l2c>;
 		vddpmu-supply = <&vreg_conn_1p8>;
+		vddpmumx-supply = <&vreg_wcn_0p95>;
+		vddpmucx-supply = <&vreg_wcn_0p95>;
 		vddrfa0p95-supply = <&vreg_l2c>;
 		vddrfa1p3-supply = <&vreg_l6e>;
 		vddrfa1p9-supply = <&vreg_s5a>;
@@ -154,6 +182,30 @@ dp1_connector_in: endpoint {
 		};
 	};
 
+	dp2-connector {
+		compatible = "dp-connector";
+		label = "eDP2";
+		type = "full-size";
+
+		port {
+			dp2_connector_in: endpoint {
+				remote-endpoint = <&mdss1_dp0_out>;
+			};
+		};
+	};
+
+	dp3-connector {
+		compatible = "dp-connector";
+		label = "eDP3";
+		type = "full-size";
+
+		port {
+			dp3_connector_in: endpoint {
+				remote-endpoint = <&mdss1_dp1_out>;
+			};
+		};
+	};
+
 	dp-dsi0-connector {
 		compatible = "dp-connector";
 		label = "DSI0";
@@ -613,6 +665,50 @@ &mdss0_dsi1_phy {
 	status = "okay";
 };
 
+&mdss1 {
+	status = "okay";
+};
+
+&mdss1_dp0 {
+	pinctrl-0 = <&dp2_hot_plug_det>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&mdss1_dp0_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+	remote-endpoint = <&dp2_connector_in>;
+};
+
+&mdss1_dp0_phy {
+	vdda-phy-supply = <&vreg_l1c>;
+	vdda-pll-supply = <&vreg_l4a>;
+
+	status = "okay";
+};
+
+&mdss1_dp1 {
+	pinctrl-0 = <&dp3_hot_plug_det>;
+	pinctrl-names = "default";
+
+	status = "okay";
+};
+
+&mdss1_dp1_out {
+	data-lanes = <0 1 2 3>;
+	link-frequencies = /bits/ 64 <1620000000 2700000000 5400000000 8100000000>;
+	remote-endpoint = <&dp3_connector_in>;
+};
+
+&mdss1_dp1_phy {
+	vdda-phy-supply = <&vreg_l1c>;
+	vdda-pll-supply = <&vreg_l4a>;
+
+	status = "okay";
+};
+
 &pmm8654au_0_gpios {
 	gpio-line-names = "DS_EN",
 			  "POFF_COMPLETE",
@@ -790,6 +886,18 @@ dp1_hot_plug_det: dp1-hot-plug-det-state {
 		bias-disable;
 	};
 
+	dp2_hot_plug_det: dp2-hot-plug-det-state {
+		pins = "gpio104";
+		function = "edp2_hot";
+		bias-disable;
+	};
+
+	dp3_hot_plug_det: dp3-hot-plug-det-state {
+		pins = "gpio103";
+		function = "edp3_hot";
+		bias-disable;
+	};
+
 	io_expander_intr_active: io-expander-intr-active-state {
 		pins = "gpio98";
 		function = "gpio";
@@ -956,6 +1064,8 @@ bluetooth {
 		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
 		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
 		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
+		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
+		vddwlmx-supply = <&vreg_pmu_wlmx_0p85>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/qcom/lemans.dtsi b/arch/arm64/boot/dts/qcom/lemans.dtsi
index fe6e763518230..bc7b4f65ad5ee 100644
--- a/arch/arm64/boot/dts/qcom/lemans.dtsi
+++ b/arch/arm64/boot/dts/qcom/lemans.dtsi
@@ -2694,6 +2694,352 @@ mmss_noc: interconnect@17a0000 {
 			qcom,bcm-voters = <&apps_bcm_voter>;
 		};
 
+		pcie0: pcie@1c00000 {
+			compatible = "qcom,pcie-sa8775p";
+			reg = <0x0 0x01c00000 0x0 0x3000>,
+			      <0x0 0x40000000 0x0 0xf20>,
+			      <0x0 0x40000f20 0x0 0xa8>,
+			      <0x0 0x40001000 0x0 0x4000>,
+			      <0x0 0x40100000 0x0 0x100000>,
+			      <0x0 0x01c03000 0x0 0x1000>;
+			reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi";
+			device_type = "pci";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x1fd00000>;
+			bus-range = <0x00 0xff>;
+
+			dma-coherent;
+
+			linux,pci-domain = <0>;
+			num-lanes = <2>;
+
+			interrupts = <GIC_SPI 307 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 374 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 375 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 306 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_PCIE_0_AUX_CLK>,
+				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>;
+
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a";
+
+			assigned-clocks = <&gcc GCC_PCIE_0_AUX_CLK>;
+			assigned-clock-rates = <19200000>;
+
+			interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>,
+					<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>;
+			interconnect-names = "pcie-mem", "cpu-pcie";
+
+			iommu-map = <0x0 &pcie_smmu 0x0000 0x1>,
+				    <0x100 &pcie_smmu 0x0001 0x1>;
+
+			resets = <&gcc GCC_PCIE_0_BCR>,
+				 <&gcc GCC_PCIE_0_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			power-domains = <&gcc PCIE_0_GDSC>;
+
+			phys = <&pcie0_phy>;
+			phy-names = "pciephy";
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555>;
+			eq-presets-16gts = /bits/ 8 <0x55 0x55>;
+
+			status = "disabled";
+
+			pcieport0: pcie@0 {
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie0_ep: pcie-ep@1c00000 {
+			compatible = "qcom,sa8775p-pcie-ep";
+			reg = <0x0 0x01c00000 0x0 0x3000>,
+			      <0x0 0x40000000 0x0 0xf20>,
+			      <0x0 0x40000f20 0x0 0xa8>,
+			      <0x0 0x40001000 0x0 0x4000>,
+			      <0x0 0x40200000 0x0 0x1fe00000>,
+			      <0x0 0x01c03000 0x0 0x1000>,
+			      <0x0 0x40005000 0x0 0x2000>;
+			reg-names = "parf", "dbi", "elbi", "atu", "addr_space",
+				    "mmio", "dma";
+
+			clocks = <&gcc GCC_PCIE_0_AUX_CLK>,
+				<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				<&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
+				<&gcc GCC_PCIE_0_SLV_AXI_CLK>,
+				<&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>;
+
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a";
+
+			interrupts = <GIC_SPI 306 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 630 IRQ_TYPE_LEVEL_HIGH>;
+
+			interrupt-names = "global", "doorbell", "dma";
+
+			interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>,
+					<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>;
+			interconnect-names = "pcie-mem", "cpu-pcie";
+
+			dma-coherent;
+			iommus = <&pcie_smmu 0x0000 0x7f>;
+			resets = <&gcc GCC_PCIE_0_BCR>;
+			reset-names = "core";
+			power-domains = <&gcc PCIE_0_GDSC>;
+			phys = <&pcie0_phy>;
+			phy-names = "pciephy";
+			num-lanes = <2>;
+			linux,pci-domain = <0>;
+
+			status = "disabled";
+		};
+
+		pcie0_phy: phy@1c04000 {
+			compatible = "qcom,sa8775p-qmp-gen4x2-pcie-phy";
+			reg = <0x0 0x1c04000 0x0 0x2000>;
+
+			clocks = <&gcc GCC_PCIE_0_PHY_AUX_CLK>,
+				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_CLKREF_EN>,
+				 <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_0_PIPE_CLK>,
+				 <&gcc GCC_PCIE_0_PIPEDIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			resets = <&gcc GCC_PCIE_0_PHY_BCR>;
+			reset-names = "phy";
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie_0_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
+		pcie1: pcie@1c10000 {
+			compatible = "qcom,pcie-sa8775p";
+			reg = <0x0 0x01c10000 0x0 0x3000>,
+			      <0x0 0x60000000 0x0 0xf20>,
+			      <0x0 0x60000f20 0x0 0xa8>,
+			      <0x0 0x60001000 0x0 0x4000>,
+			      <0x0 0x60100000 0x0 0x100000>,
+			      <0x0 0x01c13000 0x0 0x1000>;
+			reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi";
+			device_type = "pci";
+
+			#address-cells = <3>;
+			#size-cells = <2>;
+			ranges = <0x01000000 0x0 0x00000000 0x0 0x60200000 0x0 0x100000>,
+				 <0x02000000 0x0 0x60300000 0x0 0x60300000 0x0 0x1fd00000>;
+			bus-range = <0x00 0xff>;
+
+			dma-coherent;
+
+			linux,pci-domain = <1>;
+			num-lanes = <4>;
+
+			interrupts = <GIC_SPI 519 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 518 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "msi0",
+					  "msi1",
+					  "msi2",
+					  "msi3",
+					  "msi4",
+					  "msi5",
+					  "msi6",
+					  "msi7",
+					  "global";
+			#interrupt-cells = <1>;
+			interrupt-map-mask = <0 0 0 0x7>;
+			interrupt-map = <0 0 0 1 &intc GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
+
+			clocks = <&gcc GCC_PCIE_1_AUX_CLK>,
+				 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_1_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_1_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>;
+
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a";
+
+			assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>;
+			assigned-clock-rates = <19200000>;
+
+			interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>,
+					<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>;
+			interconnect-names = "pcie-mem", "cpu-pcie";
+
+			iommu-map = <0x0 &pcie_smmu 0x0080 0x1>,
+				    <0x100 &pcie_smmu 0x0081 0x1>;
+
+			resets = <&gcc GCC_PCIE_1_BCR>,
+				 <&gcc GCC_PCIE_1_LINK_DOWN_BCR>;
+			reset-names = "pci",
+				      "link_down";
+
+			power-domains = <&gcc PCIE_1_GDSC>;
+
+			phys = <&pcie1_phy>;
+			phy-names = "pciephy";
+
+			eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>;
+			eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>;
+
+			status = "disabled";
+
+			pcie@0 {
+				device_type = "pci";
+				reg = <0x0 0x0 0x0 0x0 0x0>;
+				bus-range = <0x01 0xff>;
+
+				#address-cells = <3>;
+				#size-cells = <2>;
+				ranges;
+			};
+		};
+
+		pcie1_ep: pcie-ep@1c10000 {
+			compatible = "qcom,sa8775p-pcie-ep";
+			reg = <0x0 0x01c10000 0x0 0x3000>,
+			      <0x0 0x60000000 0x0 0xf20>,
+			      <0x0 0x60000f20 0x0 0xa8>,
+			      <0x0 0x60001000 0x0 0x4000>,
+			      <0x0 0x60200000 0x0 0x1fe00000>,
+			      <0x0 0x01c13000 0x0 0x1000>,
+			      <0x0 0x60005000 0x0 0x2000>;
+			reg-names = "parf", "dbi", "elbi", "atu", "addr_space",
+				    "mmio", "dma";
+
+			clocks = <&gcc GCC_PCIE_1_AUX_CLK>,
+				 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_1_MSTR_AXI_CLK>,
+				 <&gcc GCC_PCIE_1_SLV_AXI_CLK>,
+				 <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>;
+
+			clock-names = "aux",
+				      "cfg",
+				      "bus_master",
+				      "bus_slave",
+				      "slave_q2a";
+
+			interrupts = <GIC_SPI 518 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 474 IRQ_TYPE_LEVEL_HIGH>;
+
+			interrupt-names = "global", "doorbell", "dma";
+
+			interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>,
+					<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>;
+			interconnect-names = "pcie-mem", "cpu-pcie";
+
+			dma-coherent;
+			iommus = <&pcie_smmu 0x80 0x7f>;
+			resets = <&gcc GCC_PCIE_1_BCR>;
+			reset-names = "core";
+			power-domains = <&gcc PCIE_1_GDSC>;
+			phys = <&pcie1_phy>;
+			phy-names = "pciephy";
+			num-lanes = <4>;
+			linux,pci-domain = <1>;
+
+			status = "disabled";
+		};
+
+		pcie1_phy: phy@1c14000 {
+			compatible = "qcom,sa8775p-qmp-gen4x4-pcie-phy";
+			reg = <0x0 0x1c14000 0x0 0x4000>;
+
+			clocks = <&gcc GCC_PCIE_1_PHY_AUX_CLK>,
+				 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
+				 <&gcc GCC_PCIE_CLKREF_EN>,
+				 <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>,
+				 <&gcc GCC_PCIE_1_PIPE_CLK>,
+				 <&gcc GCC_PCIE_1_PIPEDIV2_CLK>;
+			clock-names = "aux",
+				      "cfg_ahb",
+				      "ref",
+				      "rchng",
+				      "pipe",
+				      "pipediv2";
+
+			assigned-clocks = <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>;
+			assigned-clock-rates = <100000000>;
+
+			resets = <&gcc GCC_PCIE_1_PHY_BCR>;
+			reset-names = "phy";
+
+			#clock-cells = <0>;
+			clock-output-names = "pcie_1_pipe_clk";
+
+			#phy-cells = <0>;
+
+			status = "disabled";
+		};
+
 		ufs_mem_hc: ufshc@1d84000 {
 			compatible = "qcom,sa8775p-ufshc", "qcom,ufshc", "jedec,ufs-2.0";
 			reg = <0x0 0x01d84000 0x0 0x3000>;
@@ -5470,7 +5816,7 @@ mdss0_dp1: displayport-controller@af5c000 {
 				phys = <&mdss0_dp1_phy>;
 				phy-names = "dp";
 
-				operating-points-v2 = <&dp1_opp_table>;
+				operating-points-v2 = <&dp_opp_table>;
 				power-domains = <&rpmhpd SA8775P_MMCX>;
 
 				#sound-dai-cells = <0>;
@@ -5495,30 +5841,6 @@ port@1 {
 						mdss0_dp1_out: endpoint { };
 					};
 				};
-
-				dp1_opp_table: opp-table {
-					compatible = "operating-points-v2";
-
-					opp-162000000 {
-						opp-hz = /bits/ 64 <162000000>;
-						required-opps = <&rpmhpd_opp_low_svs>;
-					};
-
-					opp-270000000 {
-						opp-hz = /bits/ 64 <270000000>;
-						required-opps = <&rpmhpd_opp_svs>;
-					};
-
-					opp-540000000 {
-						opp-hz = /bits/ 64 <540000000>;
-						required-opps = <&rpmhpd_opp_svs_l1>;
-					};
-
-					opp-810000000 {
-						opp-hz = /bits/ 64 <810000000>;
-						required-opps = <&rpmhpd_opp_nom>;
-					};
-				};
 			};
 		};
 
@@ -7051,6 +7373,265 @@ compute-cb@3 {
 			};
 		};
 
+		mdss1: display-subsystem@22000000 {
+			compatible = "qcom,sa8775p-mdss";
+			reg = <0x0 0x22000000 0x0 0x1000>;
+			reg-names = "mdss";
+
+			interconnects = <&mmss_noc MASTER_MDP_CORE1_0 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&mmss_noc MASTER_MDP_CORE1_1 QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>,
+					<&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_DISPLAY_CFG QCOM_ICC_TAG_ACTIVE_ONLY>;
+			interconnect-names = "mdp0-mem",
+					     "mdp1-mem",
+					     "cpu-cfg";
+
+			resets = <&dispcc1 MDSS_DISP_CC_MDSS_CORE_BCR>;
+
+			power-domains = <&dispcc1 MDSS_DISP_CC_MDSS_CORE_GDSC>;
+
+			clocks = <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>,
+				 <&gcc GCC_DISP1_HF_AXI_CLK>,
+				 <&dispcc1 MDSS_DISP_CC_MDSS_MDP_CLK>;
+
+			interrupts = <GIC_SPI 865 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-controller;
+			#interrupt-cells = <1>;
+
+			iommus = <&apps_smmu 0x1800 0x402>;
+
+			#address-cells = <2>;
+			#size-cells = <2>;
+			ranges;
+
+			status = "disabled";
+
+			display-controller@22001000 {
+				compatible = "qcom,sa8775p-dpu";
+				reg = <0x0 0x22001000 0x0 0x8f000>,
+				      <0x0 0x220b0000 0x0 0x3000>;
+				reg-names = "mdp", "vbif";
+
+				clocks = <&gcc GCC_DISP1_HF_AXI_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_MDP_LUT_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_MDP_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_VSYNC_CLK>;
+				clock-names = "nrt_bus",
+					      "iface",
+					      "lut",
+					      "core",
+					      "vsync";
+
+				assigned-clocks = <&dispcc1 MDSS_DISP_CC_MDSS_VSYNC_CLK>;
+				assigned-clock-rates = <19200000>;
+
+				operating-points-v2 = <&mdss0_mdp_opp_table>;
+				power-domains = <&rpmhpd SA8775P_MMCX>;
+
+				interrupt-parent = <&mdss1>;
+				interrupts = <0>;
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						dpu1_intf0_out: endpoint {
+							remote-endpoint = <&mdss1_dp0_in>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						dpu1_intf4_out: endpoint {
+							remote-endpoint = <&mdss1_dp1_in>;
+						};
+					};
+				};
+			};
+
+			mdss1_dp0_phy: phy@220c2a00 {
+				compatible = "qcom,sa8775p-edp-phy";
+				reg = <0x0 0x220c2a00 0x0 0x200>,
+				      <0x0 0x220c2200 0x0 0xd0>,
+				      <0x0 0x220c2600 0x0 0xd0>,
+				      <0x0 0x220c2000 0x0 0x1c8>;
+
+				clocks = <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>,
+					 <&gcc GCC_EDP_REF_CLKREF_EN>;
+				clock-names = "aux",
+					      "cfg_ahb",
+					      "ref";
+
+				#clock-cells = <1>;
+				#phy-cells = <0>;
+
+				status = "disabled";
+			};
+
+			mdss1_dp1_phy: phy@220c5a00 {
+				compatible = "qcom,sa8775p-edp-phy";
+				reg = <0x0 0x220c5a00 0x0 0x200>,
+				      <0x0 0x220c5200 0x0 0xd0>,
+				      <0x0 0x220c5600 0x0 0xd0>,
+				      <0x0 0x220c5000 0x0 0x1c8>;
+
+				clocks = <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>,
+					 <&gcc GCC_EDP_REF_CLKREF_EN>;
+				clock-names = "aux",
+					      "cfg_ahb",
+					      "ref";
+
+				#clock-cells = <1>;
+				#phy-cells = <0>;
+
+				status = "disabled";
+			};
+
+			mdss1_dp0: displayport-controller@22154000 {
+				compatible = "qcom,sa8775p-dp";
+				reg = <0x0 0x22154000 0x0 0x104>,
+				      <0x0 0x22154200 0x0 0x0c0>,
+				      <0x0 0x22155000 0x0 0x770>,
+				      <0x0 0x22156000 0x0 0x09c>,
+				      <0x0 0x22157000 0x0 0x09c>,
+				      <0x0 0x22158000 0x0 0x09c>,
+				      <0x0 0x22159000 0x0 0x09c>,
+				      <0x0 0x2215a000 0x0 0x23c>,
+				      <0x0 0x2215b000 0x0 0x23c>;
+
+				interrupt-parent = <&mdss1>;
+				interrupts = <12>;
+
+				clocks = <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_AUX_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_LINK_INTF_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK>;
+				clock-names = "core_iface",
+					      "core_aux",
+					      "ctrl_link",
+					      "ctrl_link_iface",
+					      "stream_pixel",
+					      "stream_1_pixel",
+					      "stream_2_pixel",
+					      "stream_3_pixel";
+				assigned-clocks = <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_LINK_CLK_SRC>,
+						  <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL0_CLK_SRC>,
+						  <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL1_CLK_SRC>,
+						  <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL2_CLK_SRC>,
+						  <&dispcc1 MDSS_DISP_CC_MDSS_DPTX0_PIXEL3_CLK_SRC>;
+				assigned-clock-parents = <&mdss1_dp0_phy 0>,
+							 <&mdss1_dp0_phy 1>,
+							 <&mdss1_dp0_phy 1>,
+							 <&mdss1_dp0_phy 1>,
+							 <&mdss1_dp0_phy 1>;
+				phys = <&mdss1_dp0_phy>;
+				phy-names = "dp";
+
+				operating-points-v2 = <&dp_opp_table>;
+				power-domains = <&rpmhpd SA8775P_MMCX>;
+
+				#sound-dai-cells = <0>;
+
+				status = "disabled";
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						mdss1_dp0_in: endpoint {
+							remote-endpoint = <&dpu1_intf0_out>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						mdss1_dp0_out: endpoint { };
+					};
+				};
+			};
+
+			mdss1_dp1: displayport-controller@2215c000 {
+				compatible = "qcom,sa8775p-dp";
+				reg = <0x0 0x2215c000 0x0 0x104>,
+				      <0x0 0x2215c200 0x0 0x0c0>,
+				      <0x0 0x2215d000 0x0 0x770>,
+				      <0x0 0x2215e000 0x0 0x09c>,
+				      <0x0 0x2215f000 0x0 0x09c>,
+				      <0x0 0x22160000 0x0 0x09c>,
+				      <0x0 0x22161000 0x0 0x09c>,
+				      <0x0 0x22162000 0x0 0x23c>,
+				      <0x0 0x22163000 0x0 0x23c>;
+
+				interrupt-parent = <&mdss1>;
+				interrupts = <13>;
+
+				clocks = <&dispcc1 MDSS_DISP_CC_MDSS_AHB_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_AUX_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_LINK_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_LINK_INTF_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_PIXEL0_CLK>,
+					 <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_PIXEL1_CLK>;
+				clock-names = "core_iface",
+					      "core_aux",
+					      "ctrl_link",
+					      "ctrl_link_iface",
+					      "stream_pixel",
+					      "stream_1_pixel";
+				assigned-clocks = <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_LINK_CLK_SRC>,
+						  <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_PIXEL0_CLK_SRC>,
+						  <&dispcc1 MDSS_DISP_CC_MDSS_DPTX1_PIXEL1_CLK_SRC>;
+				assigned-clock-parents = <&mdss1_dp1_phy 0>,
+							 <&mdss1_dp1_phy 1>,
+							 <&mdss1_dp1_phy 1>;
+				phys = <&mdss1_dp1_phy>;
+				phy-names = "dp";
+
+				operating-points-v2 = <&dp_opp_table>;
+				power-domains = <&rpmhpd SA8775P_MMCX>;
+
+				#sound-dai-cells = <0>;
+
+				status = "disabled";
+
+				ports {
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					port@0 {
+						reg = <0>;
+
+						mdss1_dp1_in: endpoint {
+							remote-endpoint = <&dpu1_intf4_out>;
+						};
+					};
+
+					port@1 {
+						reg = <1>;
+
+						mdss1_dp1_out: endpoint { };
+					};
+				};
+
+			};
+		};
+
 		dispcc1: clock-controller@22100000 {
 			compatible = "qcom,sa8775p-dispcc1";
 			reg = <0x0 0x22100000 0x0 0x20000>;
@@ -7058,13 +7639,13 @@ dispcc1: clock-controller@22100000 {
 				 <&rpmhcc RPMH_CXO_CLK>,
 				 <&rpmhcc RPMH_CXO_CLK_A>,
 				 <&sleep_clk>,
-				 <0>, <0>, <0>, <0>,
+				 <&mdss1_dp0_phy 0>, <&mdss1_dp0_phy 1>,
+				 <&mdss1_dp1_phy 0>, <&mdss1_dp1_phy 1>,
 				 <0>, <0>, <0>, <0>;
 			power-domains = <&rpmhpd SA8775P_MMCX>;
 			#clock-cells = <1>;
 			#reset-cells = <1>;
 			#power-domain-cells = <1>;
-			status = "disabled";
 		};
 
 		ethernet1: ethernet@23000000 {
@@ -8601,350 +9182,4 @@ turing_llm_tpdm_out: endpoint {
 			};
 		};
 	};
-
-	pcie0: pcie@1c00000 {
-		compatible = "qcom,pcie-sa8775p";
-		reg = <0x0 0x01c00000 0x0 0x3000>,
-		      <0x0 0x40000000 0x0 0xf20>,
-		      <0x0 0x40000f20 0x0 0xa8>,
-		      <0x0 0x40001000 0x0 0x4000>,
-		      <0x0 0x40100000 0x0 0x100000>,
-		      <0x0 0x01c03000 0x0 0x1000>;
-		reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi";
-		device_type = "pci";
-
-		#address-cells = <3>;
-		#size-cells = <2>;
-		ranges = <0x01000000 0x0 0x00000000 0x0 0x40200000 0x0 0x100000>,
-			 <0x02000000 0x0 0x40300000 0x0 0x40300000 0x0 0x1fd00000>;
-		bus-range = <0x00 0xff>;
-
-		dma-coherent;
-
-		linux,pci-domain = <0>;
-		num-lanes = <2>;
-
-		interrupts = <GIC_SPI 307 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 309 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 312 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 313 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 314 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 374 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 375 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 306 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "msi0",
-				  "msi1",
-				  "msi2",
-				  "msi3",
-				  "msi4",
-				  "msi5",
-				  "msi6",
-				  "msi7",
-				  "global";
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0 0 0 0x7>;
-		interrupt-map = <0 0 0 1 &intc GIC_SPI 434 IRQ_TYPE_LEVEL_HIGH>,
-				<0 0 0 2 &intc GIC_SPI 435 IRQ_TYPE_LEVEL_HIGH>,
-				<0 0 0 3 &intc GIC_SPI 438 IRQ_TYPE_LEVEL_HIGH>,
-				<0 0 0 4 &intc GIC_SPI 439 IRQ_TYPE_LEVEL_HIGH>;
-
-		clocks = <&gcc GCC_PCIE_0_AUX_CLK>,
-			 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
-			 <&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
-			 <&gcc GCC_PCIE_0_SLV_AXI_CLK>,
-			 <&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>;
-
-		clock-names = "aux",
-			      "cfg",
-			      "bus_master",
-			      "bus_slave",
-			      "slave_q2a";
-
-		assigned-clocks = <&gcc GCC_PCIE_0_AUX_CLK>;
-		assigned-clock-rates = <19200000>;
-
-		interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>,
-				<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>;
-		interconnect-names = "pcie-mem", "cpu-pcie";
-
-		iommu-map = <0x0 &pcie_smmu 0x0000 0x1>,
-			    <0x100 &pcie_smmu 0x0001 0x1>;
-
-		resets = <&gcc GCC_PCIE_0_BCR>,
-			 <&gcc GCC_PCIE_0_LINK_DOWN_BCR>;
-		reset-names = "pci",
-			      "link_down";
-
-		power-domains = <&gcc PCIE_0_GDSC>;
-
-		phys = <&pcie0_phy>;
-		phy-names = "pciephy";
-
-		eq-presets-8gts = /bits/ 16 <0x5555 0x5555>;
-		eq-presets-16gts = /bits/ 8 <0x55 0x55>;
-
-		status = "disabled";
-
-		pcieport0: pcie@0 {
-			device_type = "pci";
-			reg = <0x0 0x0 0x0 0x0 0x0>;
-			bus-range = <0x01 0xff>;
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-		};
-	};
-
-	pcie0_ep: pcie-ep@1c00000 {
-		compatible = "qcom,sa8775p-pcie-ep";
-		reg = <0x0 0x01c00000 0x0 0x3000>,
-		      <0x0 0x40000000 0x0 0xf20>,
-		      <0x0 0x40000f20 0x0 0xa8>,
-		      <0x0 0x40001000 0x0 0x4000>,
-		      <0x0 0x40200000 0x0 0x1fe00000>,
-		      <0x0 0x01c03000 0x0 0x1000>,
-		      <0x0 0x40005000 0x0 0x2000>;
-		reg-names = "parf", "dbi", "elbi", "atu", "addr_space",
-			    "mmio", "dma";
-
-		clocks = <&gcc GCC_PCIE_0_AUX_CLK>,
-			<&gcc GCC_PCIE_0_CFG_AHB_CLK>,
-			<&gcc GCC_PCIE_0_MSTR_AXI_CLK>,
-			<&gcc GCC_PCIE_0_SLV_AXI_CLK>,
-			<&gcc GCC_PCIE_0_SLV_Q2A_AXI_CLK>;
-
-		clock-names = "aux",
-			      "cfg",
-			      "bus_master",
-			      "bus_slave",
-			      "slave_q2a";
-
-		interrupts = <GIC_SPI 306 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 630 IRQ_TYPE_LEVEL_HIGH>;
-
-		interrupt-names = "global", "doorbell", "dma";
-
-		interconnects = <&pcie_anoc MASTER_PCIE_0 0 &mc_virt SLAVE_EBI1 0>,
-				<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_0 0>;
-		interconnect-names = "pcie-mem", "cpu-pcie";
-
-		dma-coherent;
-		iommus = <&pcie_smmu 0x0000 0x7f>;
-		resets = <&gcc GCC_PCIE_0_BCR>;
-		reset-names = "core";
-		power-domains = <&gcc PCIE_0_GDSC>;
-		phys = <&pcie0_phy>;
-		phy-names = "pciephy";
-		num-lanes = <2>;
-		linux,pci-domain = <0>;
-
-		status = "disabled";
-	};
-
-	pcie0_phy: phy@1c04000 {
-		compatible = "qcom,sa8775p-qmp-gen4x2-pcie-phy";
-		reg = <0x0 0x1c04000 0x0 0x2000>;
-
-		clocks = <&gcc GCC_PCIE_0_PHY_AUX_CLK>,
-			 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
-			 <&gcc GCC_PCIE_CLKREF_EN>,
-			 <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>,
-			 <&gcc GCC_PCIE_0_PIPE_CLK>,
-			 <&gcc GCC_PCIE_0_PIPEDIV2_CLK>;
-		clock-names = "aux",
-			      "cfg_ahb",
-			      "ref",
-			      "rchng",
-			      "pipe",
-			      "pipediv2";
-
-		assigned-clocks = <&gcc GCC_PCIE_0_PHY_RCHNG_CLK>;
-		assigned-clock-rates = <100000000>;
-
-		resets = <&gcc GCC_PCIE_0_PHY_BCR>;
-		reset-names = "phy";
-
-		#clock-cells = <0>;
-		clock-output-names = "pcie_0_pipe_clk";
-
-		#phy-cells = <0>;
-
-		status = "disabled";
-	};
-
-	pcie1: pcie@1c10000 {
-		compatible = "qcom,pcie-sa8775p";
-		reg = <0x0 0x01c10000 0x0 0x3000>,
-		      <0x0 0x60000000 0x0 0xf20>,
-		      <0x0 0x60000f20 0x0 0xa8>,
-		      <0x0 0x60001000 0x0 0x4000>,
-		      <0x0 0x60100000 0x0 0x100000>,
-		      <0x0 0x01c13000 0x0 0x1000>;
-		reg-names = "parf", "dbi", "elbi", "atu", "config", "mhi";
-		device_type = "pci";
-
-		#address-cells = <3>;
-		#size-cells = <2>;
-		ranges = <0x01000000 0x0 0x00000000 0x0 0x60200000 0x0 0x100000>,
-			 <0x02000000 0x0 0x60300000 0x0 0x60300000 0x0 0x1fd00000>;
-		bus-range = <0x00 0xff>;
-
-		dma-coherent;
-
-		linux,pci-domain = <1>;
-		num-lanes = <4>;
-
-		interrupts = <GIC_SPI 519 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 140 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 141 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 142 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 143 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 144 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 145 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 146 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 518 IRQ_TYPE_LEVEL_HIGH>;
-		interrupt-names = "msi0",
-				  "msi1",
-				  "msi2",
-				  "msi3",
-				  "msi4",
-				  "msi5",
-				  "msi6",
-				  "msi7",
-				  "global";
-		#interrupt-cells = <1>;
-		interrupt-map-mask = <0 0 0 0x7>;
-		interrupt-map = <0 0 0 1 &intc GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH>,
-				<0 0 0 2 &intc GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>,
-				<0 0 0 3 &intc GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>,
-				<0 0 0 4 &intc GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>;
-
-		clocks = <&gcc GCC_PCIE_1_AUX_CLK>,
-			 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
-			 <&gcc GCC_PCIE_1_MSTR_AXI_CLK>,
-			 <&gcc GCC_PCIE_1_SLV_AXI_CLK>,
-			 <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>;
-
-		clock-names = "aux",
-			      "cfg",
-			      "bus_master",
-			      "bus_slave",
-			      "slave_q2a";
-
-		assigned-clocks = <&gcc GCC_PCIE_1_AUX_CLK>;
-		assigned-clock-rates = <19200000>;
-
-		interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>,
-				<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>;
-		interconnect-names = "pcie-mem", "cpu-pcie";
-
-		iommu-map = <0x0 &pcie_smmu 0x0080 0x1>,
-			    <0x100 &pcie_smmu 0x0081 0x1>;
-
-		resets = <&gcc GCC_PCIE_1_BCR>,
-			 <&gcc GCC_PCIE_1_LINK_DOWN_BCR>;
-		reset-names = "pci",
-			      "link_down";
-
-		power-domains = <&gcc PCIE_1_GDSC>;
-
-		phys = <&pcie1_phy>;
-		phy-names = "pciephy";
-
-		eq-presets-8gts = /bits/ 16 <0x5555 0x5555 0x5555 0x5555>;
-		eq-presets-16gts = /bits/ 8 <0x55 0x55 0x55 0x55>;
-
-		status = "disabled";
-
-		pcie@0 {
-			device_type = "pci";
-			reg = <0x0 0x0 0x0 0x0 0x0>;
-			bus-range = <0x01 0xff>;
-
-			#address-cells = <3>;
-			#size-cells = <2>;
-			ranges;
-		};
-	};
-
-	pcie1_ep: pcie-ep@1c10000 {
-		compatible = "qcom,sa8775p-pcie-ep";
-		reg = <0x0 0x01c10000 0x0 0x3000>,
-		      <0x0 0x60000000 0x0 0xf20>,
-		      <0x0 0x60000f20 0x0 0xa8>,
-		      <0x0 0x60001000 0x0 0x4000>,
-		      <0x0 0x60200000 0x0 0x1fe00000>,
-		      <0x0 0x01c13000 0x0 0x1000>,
-		      <0x0 0x60005000 0x0 0x2000>;
-		reg-names = "parf", "dbi", "elbi", "atu", "addr_space",
-			    "mmio", "dma";
-
-		clocks = <&gcc GCC_PCIE_1_AUX_CLK>,
-			 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
-			 <&gcc GCC_PCIE_1_MSTR_AXI_CLK>,
-			 <&gcc GCC_PCIE_1_SLV_AXI_CLK>,
-			 <&gcc GCC_PCIE_1_SLV_Q2A_AXI_CLK>;
-
-		clock-names = "aux",
-			      "cfg",
-			      "bus_master",
-			      "bus_slave",
-			      "slave_q2a";
-
-		interrupts = <GIC_SPI 518 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>,
-			     <GIC_SPI 474 IRQ_TYPE_LEVEL_HIGH>;
-
-		interrupt-names = "global", "doorbell", "dma";
-
-		interconnects = <&pcie_anoc MASTER_PCIE_1 0 &mc_virt SLAVE_EBI1 0>,
-				<&gem_noc MASTER_APPSS_PROC 0 &config_noc SLAVE_PCIE_1 0>;
-		interconnect-names = "pcie-mem", "cpu-pcie";
-
-		dma-coherent;
-		iommus = <&pcie_smmu 0x80 0x7f>;
-		resets = <&gcc GCC_PCIE_1_BCR>;
-		reset-names = "core";
-		power-domains = <&gcc PCIE_1_GDSC>;
-		phys = <&pcie1_phy>;
-		phy-names = "pciephy";
-		num-lanes = <4>;
-		linux,pci-domain = <1>;
-
-		status = "disabled";
-	};
-
-	pcie1_phy: phy@1c14000 {
-		compatible = "qcom,sa8775p-qmp-gen4x4-pcie-phy";
-		reg = <0x0 0x1c14000 0x0 0x4000>;
-
-		clocks = <&gcc GCC_PCIE_1_PHY_AUX_CLK>,
-			 <&gcc GCC_PCIE_1_CFG_AHB_CLK>,
-			 <&gcc GCC_PCIE_CLKREF_EN>,
-			 <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>,
-			 <&gcc GCC_PCIE_1_PIPE_CLK>,
-			 <&gcc GCC_PCIE_1_PIPEDIV2_CLK>;
-		clock-names = "aux",
-			      "cfg_ahb",
-			      "ref",
-			      "rchng",
-			      "pipe",
-			      "pipediv2";
-
-		assigned-clocks = <&gcc GCC_PCIE_1_PHY_RCHNG_CLK>;
-		assigned-clock-rates = <100000000>;
-
-		resets = <&gcc GCC_PCIE_1_PHY_BCR>;
-		reset-names = "phy";
-
-		#clock-cells = <0>;
-		clock-output-names = "pcie_1_pipe_clk";
-
-		#phy-cells = <0>;
-
-		status = "disabled";
-	};
 };
diff --git a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
index c1899db46e714..ebdca0e2d2f3a 100644
--- a/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
+++ b/arch/arm64/boot/dts/qcom/milos-fairphone-fp6.dts
@@ -687,7 +687,17 @@ vreg_l7p: ldo7 {
 	};
 
 	/* VL53L3 ToF @ 0x29 */
-	/* AW86938FCR vibrator @ 0x5a */
+
+	vibrator@5a {
+		compatible = "awinic,aw86938", "awinic,aw86927";
+		reg = <0x5a>;
+
+		interrupts-extended = <&tlmm 80 IRQ_TYPE_EDGE_FALLING>;
+		reset-gpios = <&tlmm 78 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&aw86938_int_default>, <&aw86938_reset_default>;
+		pinctrl-names = "default";
+	};
 };
 
 &pm8550vs_c {
@@ -786,6 +796,11 @@ &remoteproc_wpss {
 	status = "okay";
 };
 
+&rmtfs_mem {
+	/* Increase the size from 2.5 MiB to 6 MiB */
+	reg = <0x0 0xe1f00000 0x0 0x600000>;
+};
+
 &sdhc_2 {
 	cd-gpios = <&tlmm 65 GPIO_ACTIVE_HIGH>;
 
@@ -824,6 +839,20 @@ hall_sensor_default: hall-sensor-default-state {
 		bias-disable;
 	};
 
+	aw86938_reset_default: aw86938-reset-default-state {
+		pins = "gpio78";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	aw86938_int_default: aw86938-int-default-state {
+		pins = "gpio80";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
 	pm8008_int_default: pm8008-int-default-state {
 		pins = "gpio125";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/milos-nothing-asteroids.dts b/arch/arm64/boot/dts/qcom/milos-nothing-asteroids.dts
new file mode 100644
index 0000000000000..7393978926e44
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/milos-nothing-asteroids.dts
@@ -0,0 +1,964 @@
+// SPDX-License-Identifier: BSD-3-Clause
+
+/dts-v1/;
+
+#define PMIV0104_SID 7
+
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "milos.dtsi"
+#include "pm7550.dtsi"
+#include "pm8550vs.dtsi"
+#include "pmiv0104.dtsi" /* PMIV0102 */
+#include "pmk8550.dtsi" /* PMK7635 */
+#include "pmr735b.dtsi"
+
+/ {
+	model = "Nothing Phone (3a)";
+	compatible = "nothing,asteroids", "qcom,milos";
+	chassis-type = "handset";
+
+	aliases {
+		serial0 = &uart5;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		stdout-path = "serial0:115200n8";
+
+		framebuffer0: framebuffer@e3940000 {
+			compatible = "simple-framebuffer";
+			reg = <0x0 0xe3940000 0x0 (2392 * 1080 * 4)>;
+			width = <1080>;
+			height = <2392>;
+			stride = <(1080 * 4)>;
+			format = "a8r8g8b8";
+			panel = <&panel>;
+			interconnects = <&mmss_noc MASTER_MDP QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			clocks = <&gcc GCC_DISP_HF_AXI_CLK>;
+
+			/* Dummy panel for simple-framebuffer dimension info */
+			panel: dummy-panel {
+				compatible = "boe,bf068mwm-td0";
+				height-mm = <157>;
+				width-mm = <71>;
+			};
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&assist_default>, <&volume_up_default>;
+		pinctrl-names = "default";
+
+		key-assist {
+			gpios = <&tlmm 71 GPIO_ACTIVE_LOW>;
+			label = "Assistant";
+			linux,code = <KEY_LEFTMETA>;
+		};
+
+		key-volume-up {
+			gpios = <&pm7550_gpios 6 GPIO_ACTIVE_LOW>;
+			label = "Volume Up";
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,milos-pmic-glink",
+			     "qcom,sm8550-pmic-glink",
+			     "qcom,pmic-glink";
+		#address-cells = <1>;
+		#size-cells = <0>;
+		orientation-gpios = <&tlmm 131 GPIO_ACTIVE_HIGH>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+
+			data-role = "dual";
+			power-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@1 {
+					reg = <1>;
+
+					pmic_glink_sbu: endpoint {
+						remote-endpoint = <&fsa4480_sbu_mux>;
+					};
+				};
+			};
+		};
+	};
+
+	vph_pwr: regulator-vph-pwr {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ramoops@81f20000 {
+			compatible = "ramoops";
+			reg = <0x0 0x81f20000 0x0 0x400000>;
+			console-size = <0x200000>;
+			mem-type = <2>;
+			pmsg-size = <0x200000>;
+		};
+
+		/*
+		 * ABL is powering down display and controller if this node is
+		 * not named exactly "splash_region".
+		 */
+		splash_region@e3940000 {
+			reg = <0x0 0xe3940000 0x0 0x2b00000>;
+			no-map;
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm7550-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s1b>;
+		vdd-l2-l3-supply = <&vreg_s3b>;
+		vdd-l4-l5-supply = <&vreg_s2b>;
+		vdd-l7-supply = <&vreg_s1b>;
+		vdd-l8-supply = <&vreg_s1b>;
+		vdd-l9-l10-supply = <&vreg_s1b>;
+		vdd-l11-supply = <&vreg_s1b>;
+		vdd-l12-l14-supply = <&vreg_bob>;
+		vdd-l13-l16-supply = <&vreg_bob>;
+		vdd-l15-l17-l18-l19-l20-l21-l22-l23-supply = <&vreg_bob>;
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+		vdd-bob-supply = <&vph_pwr>;
+
+		qcom,pmic-id = "b";
+
+		vreg_s1b: smps1 {
+			regulator-name = "vreg_s1b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2080000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s2b: smps2 {
+			regulator-name = "vreg_s2b";
+			regulator-min-microvolt = <1256000>;
+			regulator-max-microvolt = <1408000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s3b: smps3 {
+			regulator-name = "vreg_s3b";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <1040000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4b: smps4 {
+			regulator-name = "vreg_s4b";
+			regulator-min-microvolt = <300000>;
+			regulator-max-microvolt = <1004000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1b: ldo1 {
+			regulator-name = "vreg_l1b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2b: ldo2 {
+			regulator-name = "vreg_l2b";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3b: ldo3 {
+			regulator-name = "vreg_l3b";
+			regulator-min-microvolt = <880000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4b: ldo4 {
+			regulator-name = "vreg_l4b";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5b: ldo5 {
+			regulator-name = "vreg_l5b";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b: ldo7 {
+			regulator-name = "vreg_l7b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b: ldo8 {
+			regulator-name = "vreg_l8b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b: ldo9 {
+			regulator-name = "vreg_l9b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10b: ldo10 {
+			regulator-name = "vreg_l10b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b: ldo11 {
+			regulator-name = "vreg_l11b";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b: ldo12 {
+			regulator-name = "vreg_l12b";
+			regulator-min-microvolt = <2952000>;
+			regulator-max-microvolt = <2952000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b: ldo13 {
+			regulator-name = "vreg_l13b";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b: ldo14 {
+			regulator-name = "vreg_l14b";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b: ldo15 {
+			regulator-name = "vreg_l15b";
+			regulator-min-microvolt = <3300000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b: ldo16 {
+			regulator-name = "vreg_l16b";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3008000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b: ldo17 {
+			regulator-name = "vreg_l17b";
+			regulator-min-microvolt = <3104000>;
+			regulator-max-microvolt = <3104000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b: ldo18 {
+			regulator-name = "vreg_l18b";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <2800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l19b: ldo19 {
+			regulator-name = "vreg_l19b";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l20b: ldo20 {
+			regulator-name = "vreg_l20b";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l21b: ldo21 {
+			regulator-name = "vreg_l21b";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l22b: ldo22 {
+			regulator-name = "vreg_l22b";
+			regulator-min-microvolt = <3200000>;
+			regulator-max-microvolt = <3200000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l23b: ldo23 {
+			regulator-name = "vreg_l23b";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_bob: bob {
+			regulator-name = "vreg_bob";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8550vs-rpmh-regulators";
+
+		vdd-l1-supply = <&vreg_s3b>;
+		vdd-l2-supply = <&vreg_s3b>;
+		vdd-l3-supply = <&vreg_s3b>;
+		vdd-s3-supply = <&vph_pwr>;
+		vdd-s4-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s6-supply = <&vph_pwr>;
+
+		qcom,pmic-id = "c";
+
+		vreg_s3c: smps3 {
+			regulator-name = "vreg_s3c";
+			regulator-min-microvolt = <1010000>;
+			regulator-max-microvolt = <1120000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s4c: smps4 {
+			regulator-name = "vreg_s4c";
+			regulator-min-microvolt = <352000>;
+			regulator-max-microvolt = <812000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s5c: smps5 {
+			regulator-name = "vreg_s5c";
+			regulator-min-microvolt = <540000>;
+			regulator-max-microvolt = <884000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_s6c: smps6 {
+			regulator-name = "vreg_s6c";
+			regulator-min-microvolt = <352000>;
+			regulator-max-microvolt = <904000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l1c: ldo1 {
+			regulator-name = "vreg_l1c";
+			regulator-min-microvolt = <320000>;
+			regulator-max-microvolt = <1004000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2c: ldo2 {
+			regulator-name = "vreg_l2c";
+			regulator-min-microvolt = <320000>;
+			regulator-max-microvolt = <650000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3c: ldo3 {
+			regulator-name = "vreg_l3c";
+			regulator-min-microvolt = <320000>;
+			regulator-max-microvolt = <912000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-2 {
+		compatible = "qcom,pmr735b-rpmh-regulators";
+
+		vdd-l1-l2-supply = <&vreg_s3b>;
+		vdd-l3-supply = <&vreg_s3b>;
+		vdd-l4-supply = <&vreg_s1b>;
+		vdd-l5-supply = <&vreg_s2b>;
+		vdd-l7-l8-supply = <&vreg_s2b>;
+		vdd-l9-supply = <&vreg_s3b>;
+		vdd-l10-supply = <&vreg_s1b>;
+		vdd-l11-supply = <&vreg_s3b>;
+		vdd-l12-supply = <&vreg_s3b>;
+
+		qcom,pmic-id = "f";
+
+		vreg_l1f: ldo1 {
+			regulator-name = "vreg_l1f";
+			regulator-min-microvolt = <852000>;
+			regulator-max-microvolt = <950000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l2f: ldo2 {
+			regulator-name = "vreg_l2f";
+			regulator-min-microvolt = <751000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3f: ldo3 {
+			regulator-name = "vreg_l3f";
+			regulator-min-microvolt = <650000>;
+			regulator-max-microvolt = <880000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4f: ldo4 {
+			regulator-name = "vreg_l4f";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <1950000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5f: ldo5 {
+			regulator-name = "vreg_l5f";
+			regulator-min-microvolt = <1140000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7f: ldo7 {
+			regulator-name = "vreg_l7f";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1350000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9f: ldo9 {
+			regulator-name = "vreg_l9f";
+			regulator-min-microvolt = <870000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l10f: ldo10 {
+			regulator-name = "vreg_l10f";
+			regulator-min-microvolt = <1500000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11f: ldo11 {
+			regulator-name = "vreg_l11f";
+			regulator-min-microvolt = <320000>;
+			regulator-max-microvolt = <864000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12f: ldo12 {
+			regulator-name = "vreg_l12f";
+			regulator-min-microvolt = <480000>;
+			regulator-max-microvolt = <884000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+};
+
+&dispcc {
+	/* Disable for now so simple-framebuffer continues working */
+	status = "disabled";
+};
+
+&gcc {
+	protected-clocks = <GCC_PCIE_1_AUX_CLK>, <GCC_PCIE_1_AUX_CLK_SRC>,
+			   <GCC_PCIE_1_CFG_AHB_CLK>, <GCC_PCIE_1_MSTR_AXI_CLK>,
+			   <GCC_PCIE_1_PHY_RCHNG_CLK>, <GCC_PCIE_1_PHY_RCHNG_CLK_SRC>,
+			   <GCC_PCIE_1_PIPE_CLK>, <GCC_PCIE_1_PIPE_CLK_SRC>,
+			   <GCC_PCIE_1_PIPE_DIV2_CLK>, <GCC_PCIE_1_PIPE_DIV2_CLK_SRC>,
+			   <GCC_PCIE_1_SLV_AXI_CLK>, <GCC_PCIE_1_SLV_Q2A_AXI_CLK>;
+};
+
+&i2c7 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	led-controller@3a {
+		compatible = "awinic,aw20036";
+		reg = <0x3a>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		enable-gpios = <&tlmm 54 GPIO_ACTIVE_HIGH>;
+
+		led@0 {
+			reg = <0x0>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <0>;
+			led-max-microamp = <6560>;
+		};
+
+		led@1 {
+			reg = <0x1>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <1>;
+			led-max-microamp = <6560>;
+		};
+
+		led@2 {
+			reg = <0x2>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <2>;
+			led-max-microamp = <6560>;
+		};
+
+		led@3 {
+			reg = <0x3>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <3>;
+			led-max-microamp = <6560>;
+		};
+
+		led@4 {
+			reg = <0x4>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <4>;
+			led-max-microamp = <6560>;
+		};
+
+		led@5 {
+			reg = <0x5>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <5>;
+			led-max-microamp = <6560>;
+		};
+
+		led@6 {
+			reg = <0x6>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <6>;
+			led-max-microamp = <6560>;
+		};
+
+		led@7 {
+			reg = <0x7>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-2";
+			function-enumerator = <0>;
+			led-max-microamp = <6560>;
+		};
+
+		led@8 {
+			reg = <0x8>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <0>;
+			led-max-microamp = <6560>;
+		};
+
+		led@9 {
+			reg = <0x9>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <1>;
+			led-max-microamp = <6560>;
+		};
+
+		led@a {
+			reg = <0xa>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <2>;
+			led-max-microamp = <6560>;
+		};
+
+		led@b {
+			reg = <0xb>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <3>;
+			led-max-microamp = <6560>;
+		};
+
+		led@c {
+			reg = <0xc>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <7>;
+			led-max-microamp = <6560>;
+		};
+
+		led@d {
+			reg = <0xd>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <8>;
+			led-max-microamp = <6560>;
+		};
+
+		led@e {
+			reg = <0xe>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <9>;
+			led-max-microamp = <6560>;
+		};
+
+		led@f {
+			reg = <0xf>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <10>;
+			led-max-microamp = <6560>;
+		};
+
+		led@10 {
+			reg = <0x10>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <11>;
+			led-max-microamp = <6560>;
+		};
+
+		led@11 {
+			reg = <0x11>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <12>;
+			led-max-microamp = <6560>;
+		};
+
+		led@12 {
+			reg = <0x12>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <13>;
+			led-max-microamp = <6560>;
+		};
+
+		led@13 {
+			reg = <0x13>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-2";
+			function-enumerator = <1>;
+			led-max-microamp = <6560>;
+		};
+
+		led@14 {
+			reg = <0x14>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-2";
+			function-enumerator = <2>;
+			led-max-microamp = <6560>;
+		};
+
+		led@15 {
+			reg = <0x15>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <4>;
+			led-max-microamp = <6560>;
+		};
+
+		led@16 {
+			reg = <0x16>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <5>;
+			led-max-microamp = <6560>;
+		};
+
+		led@17 {
+			reg = <0x17>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <6>;
+			led-max-microamp = <6560>;
+		};
+
+		led@18 {
+			reg = <0x18>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <14>;
+			led-max-microamp = <6560>;
+		};
+
+		led@19 {
+			reg = <0x19>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <15>;
+			led-max-microamp = <6560>;
+		};
+
+		led@1a {
+			reg = <0x1a>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <16>;
+			led-max-microamp = <6560>;
+		};
+
+		led@1b {
+			reg = <0x1b>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <17>;
+			led-max-microamp = <6560>;
+		};
+
+		led@1c {
+			reg = <0x1c>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <18>;
+			led-max-microamp = <6560>;
+		};
+
+		led@1d {
+			reg = <0x1d>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-1";
+			function-enumerator = <19>;
+			led-max-microamp = <6560>;
+		};
+
+		led@1e {
+			reg = <0x1e>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-2";
+			function-enumerator = <3>;
+			led-max-microamp = <6560>;
+		};
+
+		led@1f {
+			reg = <0x1f>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-2";
+			function-enumerator = <4>;
+			led-max-microamp = <6560>;
+		};
+
+		led@20 {
+			reg = <0x20>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <7>;
+			led-max-microamp = <6560>;
+		};
+
+		led@21 {
+			reg = <0x21>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <8>;
+			led-max-microamp = <6560>;
+		};
+
+		led@22 {
+			reg = <0x22>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <9>;
+			led-max-microamp = <6560>;
+		};
+
+		led@23 {
+			reg = <0x23>;
+			color = <LED_COLOR_ID_WHITE>;
+			function = "glyph-3";
+			function-enumerator = <10>;
+			led-max-microamp = <6560>;
+		};
+	};
+
+	typec-mux@42 {
+		compatible = "fcs,fsa4480";
+		reg = <0x42>;
+
+		vcc-supply = <&vph_pwr>;
+
+		mode-switch;
+		orientation-switch;
+
+		port {
+			fsa4480_sbu_mux: endpoint {
+				remote-endpoint = <&pmic_glink_sbu>;
+			};
+		};
+	};
+};
+
+&pm7550_flash {
+	status = "okay";
+
+	led-0 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <1>, <4>;
+		led-max-microamp = <350000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <400000>;
+	};
+};
+
+&pm7550_gpios {
+	volume_up_default: volume-up-default-state {
+		pins = "gpio6";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		power-source = <1>;
+		bias-pull-up;
+	};
+};
+
+&pm8550vs_c {
+	status = "okay";
+};
+
+&pmiv0104_eusb2_repeater {
+	vdd18-supply = <&vreg_l7b>;
+	vdd3-supply = <&vreg_l17b>;
+
+	qcom,tune-res-fsdif = /bits/ 8 <0x5>;
+	qcom,tune-usb2-amplitude = /bits/ 8 <0x8>;
+	qcom,tune-usb2-disc-thres = /bits/ 8 <0x7>;
+	qcom,tune-usb2-preem = /bits/ 8 <0x6>;
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/milos/nothing/asteroids/adsp.mbn",
+			"qcom/milos/nothing/asteroids/adsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/milos/nothing/asteroids/cdsp.mbn",
+			"qcom/milos/nothing/asteroids/cdsp_dtb.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_mpss {
+	firmware-name = "qcom/milos/nothing/asteroids/modem.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_wpss {
+	firmware-name = "qcom/milos/nothing/asteroids/wpss.mbn";
+
+	status = "okay";
+};
+
+&tlmm {
+	gpio-reserved-ranges = <13 1>, /* Display */
+			       <63 2>; /* WLAN UART */
+
+	assist_default: assist-default-state {
+		pins = "gpio71";
+		function = "gpio";
+		bias-pull-up;
+	};
+};
+
+&trusted_apps_mem {
+	reg = <0x0 0xe9500000 0x0 0x5500000>;
+};
+
+/*
+ * UART_RX and UART_TX are routed to test points on board, not
+ * available over USB.
+ */
+&uart5 {
+	status = "okay";
+};
+
+&usb_1 {
+	/* USB 2.0 only, HW does not support USB 3.x */
+	qcom,select-utmi-as-pipe-clk;
+
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_1_hsphy {
+	vdd-supply = <&vreg_l2b>;
+	vdda12-supply = <&vreg_l4b>;
+
+	phys = <&pmiv0104_eusb2_repeater>;
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/milos.dtsi b/arch/arm64/boot/dts/qcom/milos.dtsi
index 4a64a98a434b6..f09b30ea96ee3 100644
--- a/arch/arm64/boot/dts/qcom/milos.dtsi
+++ b/arch/arm64/boot/dts/qcom/milos.dtsi
@@ -642,7 +642,7 @@ cpusys_vm_mem: cpusys-vm-region@e0600000 {
 
 		rmtfs_mem: rmtfs@e1f00000 {
 			compatible = "qcom,rmtfs-mem";
-			reg = <0x0 0xe1f00000 0x0 0x600000>;
+			reg = <0x0 0xe1f00000 0x0 0x280000>;
 			no-map;
 
 			qcom,client-id = <1>;
@@ -1278,6 +1278,38 @@ ice: crypto@1d88000 {
 			clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
 		};
 
+		cryptobam: dma-controller@1dc4000 {
+			compatible = "qcom,bam-v1.7.4", "qcom,bam-v1.7.0";
+			reg = <0x0 0x01dc4000 0x0 0x28000>;
+
+			interrupts = <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			#dma-cells = <1>;
+
+			iommus = <&apps_smmu 0x480 0>,
+				 <&apps_smmu 0x481 0>;
+
+			qcom,ee = <0>;
+			qcom,num-ees = <4>;
+			num-channels = <20>;
+			qcom,controlled-remotely;
+		};
+
+		crypto: crypto@1dfa000 {
+			compatible = "qcom,milos-qce", "qcom,sm8150-qce", "qcom,qce";
+			reg = <0x0 0x01dfa000 0x0 0x6000>;
+
+			interconnects = <&aggre2_noc MASTER_CRYPTO QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "memory";
+
+			dmas = <&cryptobam 4>, <&cryptobam 5>;
+			dma-names = "rx", "tx";
+
+			iommus = <&apps_smmu 0x480 0>,
+				 <&apps_smmu 0x481 0>;
+		};
+
 		tcsr_mutex: hwlock@1f40000 {
 			compatible = "qcom,tcsr-mutex";
 			reg = <0x0 0x01f40000 0x0 0x20000>;
diff --git a/arch/arm64/boot/dts/qcom/monaco.dtsi b/arch/arm64/boot/dts/qcom/monaco.dtsi
index 7b1d57460f1e6..ce6ff259cb4a0 100644
--- a/arch/arm64/boot/dts/qcom/monaco.dtsi
+++ b/arch/arm64/boot/dts/qcom/monaco.dtsi
@@ -7615,6 +7615,35 @@ IPCC_MPROC_SIGNAL_GLINK_QMP
 
 				label = "gpdsp";
 				qcom,remote-pid = <17>;
+
+				fastrpc {
+					compatible = "qcom,fastrpc";
+					qcom,glink-channels = "fastrpcglink-apps-dsp";
+					label = "gdsp0";
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					compute-cb@1 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <1>;
+						iommus = <&apps_smmu 0x28a1 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@2 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <2>;
+						iommus = <&apps_smmu 0x28a2 0x0>;
+						dma-coherent;
+					};
+
+					compute-cb@3 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <3>;
+						iommus = <&apps_smmu 0x28a3 0x0>;
+						dma-coherent;
+					};
+				};
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/msm8998.dtsi b/arch/arm64/boot/dts/qcom/msm8998.dtsi
index d41b5c470c485..3477060116372 100644
--- a/arch/arm64/boot/dts/qcom/msm8998.dtsi
+++ b/arch/arm64/boot/dts/qcom/msm8998.dtsi
@@ -955,10 +955,10 @@ pcie0: pcie@1c00000 {
 					  "msi7",
 					  "global";
 			interrupt-map-mask = <0 0 0 0x7>;
-			interrupt-map = <0 0 0 1 &intc 0 0 135 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &intc 0 0 136 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &intc 0 0 138 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &intc 0 0 139 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &intc 0 GIC_SPI 135 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 GIC_SPI 136 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 GIC_SPI 138 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 GIC_SPI 139 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks = <&gcc GCC_PCIE_0_PIPE_CLK>,
 				 <&gcc GCC_PCIE_0_AUX_CLK>,
diff --git a/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi b/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi
index 7a1e5f355c175..6b4747025b9f8 100644
--- a/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi
+++ b/arch/arm64/boot/dts/qcom/pmh0104-glymur.dtsi
@@ -7,7 +7,7 @@
 #include <dt-bindings/spmi/spmi.h>
 
 /{
-	thermal_zones {
+	thermal-zones {
 		pmh0104_i0_thermal: pmh0104-i0-thermal {
 			polling-delay-passive = <100>;
 			thermal-sensors = <&pmh0104_i_e0_temp_alarm>;
diff --git a/arch/arm64/boot/dts/qcom/qcs615-ride.dts b/arch/arm64/boot/dts/qcom/qcs615-ride.dts
index 7e05f873194a0..51b13b492472f 100644
--- a/arch/arm64/boot/dts/qcom/qcs615-ride.dts
+++ b/arch/arm64/boot/dts/qcom/qcs615-ride.dts
@@ -498,6 +498,18 @@ &pon_resin {
 	status = "okay";
 };
 
+&qspi {
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <25000000>;
+		spi-tx-bus-width = <2>;
+		spi-rx-bus-width = <2>;
+	};
+};
+
 &qupv3_id_0 {
 	status = "okay";
 };
@@ -610,7 +622,7 @@ bluetooth {
 		vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>;
 		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
 		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
index e393ccf1884af..ceb68a890bf40 100644
--- a/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
+++ b/arch/arm64/boot/dts/qcom/qcs6490-rb3gen2.dts
@@ -44,6 +44,14 @@ chosen {
 		stdout-path = "serial0:115200n8";
 	};
 
+	clocks {
+		mcp2518fd_osc: can-clk {
+			compatible = "fixed-clock";
+			clock-frequency = <40000000>;
+			#clock-cells = <0>;
+		};
+	};
+
 	dp-connector {
 		compatible = "dp-connector";
 		label = "DP";
@@ -1208,6 +1216,20 @@ platform {
 	};
 };
 
+&spi3 {
+	status = "okay";
+
+	can@0 {
+		compatible = "microchip,mcp2518fd";
+		reg = <0>;
+		interrupts-extended = <&tlmm 7 IRQ_TYPE_LEVEL_LOW>;
+		clocks = <&mcp2518fd_osc>;
+		spi-max-frequency = <14000000>;
+		vdd-supply = <&vreg_l11c_2p8>;
+		microchip,xstbyen;
+	};
+};
+
 &swr2 {
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/qcom/sc7180.dtsi b/arch/arm64/boot/dts/qcom/sc7180.dtsi
index a4b17564469ee..8341a7c4a4c6b 100644
--- a/arch/arm64/boot/dts/qcom/sc7180.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc7180.dtsi
@@ -2866,9 +2866,12 @@ qspi: spi@88dc000 {
 			clocks = <&gcc GCC_QSPI_CNOC_PERIPH_AHB_CLK>,
 				 <&gcc GCC_QSPI_CORE_CLK>;
 			clock-names = "iface", "core";
-			interconnects = <&gem_noc MASTER_APPSS_PROC 0
-					&config_noc SLAVE_QSPI_0 0>;
-			interconnect-names = "qspi-config";
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_QSPI_0 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&aggre1_noc MASTER_QSPI QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "qspi-config",
+					     "qspi-memory";
 			power-domains = <&rpmhpd SC7180_CX>;
 			operating-points-v2 = <&qspi_opp_table>;
 			status = "disabled";
diff --git a/arch/arm64/boot/dts/qcom/sc8180x.dtsi b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
index f45deb188c6c0..e87e82fa73e9c 100644
--- a/arch/arm64/boot/dts/qcom/sc8180x.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8180x.dtsi
@@ -2647,7 +2647,7 @@ usb_mp_qmpphy1: phy@88ec000 {
 			status = "disabled";
 		};
 
-		usb_sec_qmpphy: phy@88ee000 {
+		usb_sec_qmpphy: phy@88ed000 {
 			compatible = "qcom,sc8180x-qmp-usb3-dp-phy";
 			reg = <0 0x088ed000 0 0x3000>;
 
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
index c53e00cae465a..dcdeefd287283 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-crd.dts
@@ -787,7 +787,7 @@ bluetooth {
 		vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>;
 		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
 		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts b/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts
index 9819454abe131..f3c00be67081a 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-huawei-gaokun3.dts
@@ -1122,7 +1122,7 @@ bluetooth {
 		vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>;
 		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
 		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
 
 		max-speed = <3200000>;
 	};
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
index d84ca010ab9dc..abd9c5a67b9ff 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-lenovo-thinkpad-x13s.dts
@@ -1331,7 +1331,7 @@ bluetooth {
 		vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>;
 		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
 		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
 
 		max-speed = <3200000>;
 	};
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts
index 00bbeeef6f14d..125af356e24b9 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts
+++ b/arch/arm64/boot/dts/qcom/sc8280xp-microsoft-blackrock.dts
@@ -976,7 +976,7 @@ bluetooth {
 		vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>;
 		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
 		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
 
 		max-speed = <3200000>;
 	};
diff --git a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
index 761f229e8f472..ee02acd188561 100644
--- a/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
+++ b/arch/arm64/boot/dts/qcom/sc8280xp.dtsi
@@ -2966,6 +2966,33 @@ IPCC_MPROC_SIGNAL_GLINK_QMP
 				label = "lpass";
 				qcom,remote-pid = <2>;
 
+				fastrpc {
+					compatible = "qcom,fastrpc";
+					qcom,glink-channels = "fastrpcglink-apps-dsp";
+					label = "adsp";
+					qcom,non-secure-domain;
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					compute-cb@3 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <3>;
+						iommus = <&apps_smmu 0x0c03 0x0>;
+					};
+
+					compute-cb@4 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <4>;
+						iommus = <&apps_smmu 0x0c04 0x0>;
+					};
+
+					compute-cb@5 {
+						compatible = "qcom,fastrpc-compute-cb";
+						reg = <5>;
+						iommus = <&apps_smmu 0x0c05 0x0>;
+					};
+				};
+
 				gpr {
 					compatible = "qcom,gpr";
 					qcom,glink-channels = "adsp_apps";
diff --git a/arch/arm64/boot/dts/qcom/sdm630.dtsi b/arch/arm64/boot/dts/qcom/sdm630.dtsi
index bef3213165d66..a41ba0da380bb 100644
--- a/arch/arm64/boot/dts/qcom/sdm630.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm630.dtsi
@@ -494,9 +494,12 @@ venus_region: venus@9f800000 {
 			no-map;
 		};
 
-		adsp_mem: adsp-region@f6000000 {
-			reg = <0x0 0xf6000000 0x0 0x800000>;
-			no-map;
+		adsp_mem: adsp-region {
+			compatible = "shared-dma-pool";
+			alloc-ranges = <0x0 0x80000000 0x0 0x80000000>;
+			alignment = <0x0 0x400000>;
+			size = <0x0 0x800000>;
+			reusable;
 		};
 
 		qseecom_mem: qseecom-region@f6800000 {
@@ -2455,31 +2458,34 @@ fastrpc {
 					compatible = "qcom,fastrpc";
 					qcom,glink-channels = "fastrpcglink-apps-dsp";
 					label = "adsp";
+					memory-region = <&adsp_mem>;
+					qcom,vmids = <QCOM_SCM_VMID_HLOS
+						      QCOM_SCM_VMID_CP_ADSP_SHARED>;
 					qcom,non-secure-domain;
 					#address-cells = <1>;
 					#size-cells = <0>;
 
-					compute-cb@1 {
+					compute-cb@3 {
 						compatible = "qcom,fastrpc-compute-cb";
-						reg = <1>;
+						reg = <3>;
 						iommus = <&lpass_smmu 3>;
 					};
 
-					compute-cb@2 {
+					compute-cb@7 {
 						compatible = "qcom,fastrpc-compute-cb";
-						reg = <2>;
+						reg = <7>;
 						iommus = <&lpass_smmu 7>;
 					};
 
-					compute-cb@3 {
+					compute-cb@8 {
 						compatible = "qcom,fastrpc-compute-cb";
-						reg = <3>;
+						reg = <8>;
 						iommus = <&lpass_smmu 8>;
 					};
 
-					compute-cb@4 {
+					compute-cb@9 {
 						compatible = "qcom,fastrpc-compute-cb";
-						reg = <4>;
+						reg = <9>;
 						iommus = <&lpass_smmu 9>;
 					};
 				};
diff --git a/arch/arm64/boot/dts/qcom/sdm660.dtsi b/arch/arm64/boot/dts/qcom/sdm660.dtsi
index 3fd6dd82a9927..0fca9662c64a1 100644
--- a/arch/arm64/boot/dts/qcom/sdm660.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm660.dtsi
@@ -350,57 +350,57 @@ fastrpc {
 				#address-cells = <1>;
 				#size-cells = <0>;
 
-				compute-cb@5 {
+				compute-cb@3 {
 					compatible = "qcom,fastrpc-compute-cb";
-					reg = <5>;
+					reg = <3>;
 					iommus = <&cdsp_smmu 3>;
 				};
 
-				compute-cb@6 {
+				compute-cb@4 {
 					compatible = "qcom,fastrpc-compute-cb";
-					reg = <6>;
+					reg = <4>;
 					iommus = <&cdsp_smmu 4>;
 				};
 
-				compute-cb@7 {
+				compute-cb@5 {
 					compatible = "qcom,fastrpc-compute-cb";
-					reg = <7>;
+					reg = <5>;
 					iommus = <&cdsp_smmu 5>;
 				};
 
-				compute-cb@8 {
+				compute-cb@6 {
 					compatible = "qcom,fastrpc-compute-cb";
-					reg = <8>;
+					reg = <6>;
 					iommus = <&cdsp_smmu 6>;
 				};
 
-				compute-cb@9 {
+				compute-cb@7 {
 					compatible = "qcom,fastrpc-compute-cb";
-					reg = <9>;
+					reg = <7>;
 					iommus = <&cdsp_smmu 7>;
 				};
 
-				compute-cb@10 {
+				compute-cb@8 {
 					compatible = "qcom,fastrpc-compute-cb";
-					reg = <10>;
+					reg = <8>;
 					iommus = <&cdsp_smmu 8>;
 				};
 
-				compute-cb@11 {
+				compute-cb@9 {
 					compatible = "qcom,fastrpc-compute-cb";
-					reg = <11>;
+					reg = <9>;
 					iommus = <&cdsp_smmu 9>;
 				};
 
-				compute-cb@12 {
+				compute-cb@10 {
 					compatible = "qcom,fastrpc-compute-cb";
-					reg = <12>;
+					reg = <10>;
 					iommus = <&cdsp_smmu 10>;
 				};
 
-				compute-cb@13 {
+				compute-cb@11 {
 					compatible = "qcom,fastrpc-compute-cb";
-					reg = <13>;
+					reg = <11>;
 					iommus = <&cdsp_smmu 11>;
 				};
 			};
diff --git a/arch/arm64/boot/dts/qcom/sdm670-google-sargo.dts b/arch/arm64/boot/dts/qcom/sdm670-google-sargo.dts
index ed55646ca419d..eba2f3792dd05 100644
--- a/arch/arm64/boot/dts/qcom/sdm670-google-sargo.dts
+++ b/arch/arm64/boot/dts/qcom/sdm670-google-sargo.dts
@@ -172,6 +172,34 @@ vreg_s2b_1p05: vreg-s2b-regulator {
 		regulator-min-microvolt = <1050000>;
 		regulator-max-microvolt = <1050000>;
 	};
+
+	cam_front_ldo: cam-front-ldo-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "cam_front_ldo";
+		regulator-min-microvolt = <1352000>;
+		regulator-max-microvolt = <1352000>;
+		regulator-enable-ramp-delay = <135>;
+
+		gpios = <&pm660l_gpios 4 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam_front_ldo_pin>;
+		pinctrl-names = "default";
+	};
+
+	cam_vio_ldo: cam-vio-ldo-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "cam_vio_ldo";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-enable-ramp-delay = <233>;
+
+		gpios = <&pm660_gpios 13 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+
+		pinctrl-0 = <&cam_vio_pin>;
+		pinctrl-names = "default";
+	};
 };
 
 &apps_rsc {
@@ -392,6 +420,64 @@ vreg_bob: bob {
 	};
 };
 
+&camss {
+	vdda-phy-supply = <&vreg_l1a_1p225>;
+	vdda-pll-supply = <&vreg_s6a_0p87>;
+
+	status = "okay";
+};
+
+&camss_port1 {
+	camss_endpoint1: endpoint {
+		data-lanes = <0 1 2 3>;
+		remote-endpoint = <&cam_front_endpoint>;
+	};
+};
+
+&cci {
+	pinctrl-0 = <&cci1_default>;
+	pinctrl-1 = <&cci1_sleep>;
+	pinctrl-names = "default", "sleep";
+
+	status = "okay";
+};
+
+&cci_i2c1 {
+	camera@1a {
+		compatible = "sony,imx355";
+		reg = <0x1a>;
+
+		clocks = <&camcc CAM_CC_MCLK2_CLK>;
+
+		/*
+		 * The sensor can accept a 24 MHz clock, but 19.2 MHz has
+		 * better driver compatibility.
+		 */
+		assigned-clocks = <&camcc CAM_CC_MCLK2_CLK>;
+		assigned-clock-rates = <19200000>;
+
+		reset-gpios = <&tlmm 9 GPIO_ACTIVE_LOW>;
+
+		avdd-supply = <&cam_front_ldo>;
+		dvdd-supply = <&cam_front_ldo>;
+		dovdd-supply = <&cam_vio_ldo>;
+
+		pinctrl-0 = <&cam_mclk2_default>;
+		pinctrl-names = "default";
+
+		rotation = <270>;
+		orientation = <0>;
+
+		port {
+			cam_front_endpoint: endpoint {
+				data-lanes = <1 2 3 4>;
+				link-frequencies = /bits/ 64 <360000000>;
+				remote-endpoint = <&camss_endpoint1>;
+			};
+		};
+	};
+};
+
 &gcc {
 	protected-clocks = <GCC_QSPI_CORE_CLK>,
 			   <GCC_QSPI_CORE_CLK_SRC>,
@@ -490,6 +576,14 @@ &pm660_charger {
 	status = "okay";
 };
 
+&pm660_gpios {
+	cam_vio_pin: cam-vio-state {
+		pins = "gpio13";
+		function = "normal";
+		power-source = <0>;
+	};
+};
+
 &pm660_rradc {
 	status = "okay";
 };
@@ -508,6 +602,12 @@ led-0 {
 };
 
 &pm660l_gpios {
+	cam_front_ldo_pin: cam-front-state {
+		pins = "gpio4";
+		function = "normal";
+		power-source = <0>;
+	};
+
 	vol_up_pin: vol-up-state {
 		pins = "gpio7";
 		function = "normal";
diff --git a/arch/arm64/boot/dts/qcom/sdm670.dtsi b/arch/arm64/boot/dts/qcom/sdm670.dtsi
index c195c79c1c85b..30844b150c801 100644
--- a/arch/arm64/boot/dts/qcom/sdm670.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm670.dtsi
@@ -1207,6 +1207,34 @@ tlmm: pinctrl@3400000 {
 			gpio-ranges = <&tlmm 0 0 151>;
 			wakeup-parent = <&pdc>;
 
+			cam_mclk0_default: cam-mclk0-default-state {
+				pins = "gpio13";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk1_default: cam-mclk1-default-state {
+				pins = "gpio14";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk2_default: cam-mclk2-default-state {
+				pins = "gpio15";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
+			cam_mclk3_default: cam-mclk3-default-state {
+				pins = "gpio16";
+				function = "cam_mclk";
+				drive-strength = <2>;
+				bias-disable;
+			};
+
 			cci0_default: cci0-default-state {
 				pins = "gpio17", "gpio18";
 				function = "cci_i2c";
@@ -1787,28 +1815,16 @@ ports {
 				#address-cells = <1>;
 				#size-cells = <0>;
 
-				port@0 {
+				camss_port0: port@0 {
 					reg = <0>;
-
-					camss_endpoint0: endpoint {
-						status = "disabled";
-					};
 				};
 
-				port@1 {
+				camss_port1: port@1 {
 					reg = <1>;
-
-					camss_endpoint1: endpoint {
-						status = "disabled";
-					};
 				};
 
-				port@2 {
+				camss_port2: port@2 {
 					reg = <2>;
-
-					camss_endpoint2: endpoint {
-						status = "disabled";
-					};
 				};
 			};
 		};
@@ -2284,5 +2300,78 @@ cpufreq_hw: cpufreq@17d43000 {
 
 			#freq-domain-cells = <1>;
 		};
+
+		lpi_tlmm: pinctrl@62b40000 {
+			compatible = "qcom,sdm670-lpass-lpi-pinctrl";
+			reg = <0 0x62b40000 0 0x20000>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&lpi_tlmm 0 0 32>;
+
+			cdc_pdm_default: cdc-pdm-default-state {
+				clk-pins {
+					pins = "gpio18";
+					function = "slimbus_clk";
+					drive-strength = <4>;
+					output-low;
+				};
+
+				sync-pins {
+					pins = "gpio19";
+					function = "pdm_sync";
+					drive-strength = <4>;
+					output-low;
+				};
+
+				tx-pins {
+					pins = "gpio20";
+					function = "pdm_tx";
+					drive-strength = <8>;
+				};
+
+				rx-pins {
+					pins = "gpio21", "gpio23", "gpio25";
+					function = "pdm_rx";
+					drive-strength = <4>;
+					output-low;
+				};
+			};
+
+			cdc_comp_default: cdc-comp-default-state {
+				pins = "gpio22", "gpio24";
+				function = "comp_rx";
+				drive-strength = <4>;
+			};
+
+			cdc_dmic_default: cdc-dmic-default-state {
+				clk1-pins {
+					pins = "gpio26";
+					function = "dmic1_clk";
+					drive-strength = <8>;
+					output-high;
+				};
+
+				clk2-pins {
+					pins = "gpio28";
+					function = "dmic2_clk";
+					drive-strength = <8>;
+					output-high;
+				};
+
+				data1-pins {
+					pins = "gpio27";
+					function = "dmic1_data";
+					drive-strength = <8>;
+					input-enable;
+				};
+
+				data2-pins {
+					pins = "gpio29";
+					function = "dmic2_data";
+					drive-strength = <8>;
+					input-enable;
+				};
+			};
+		};
 	};
 };
diff --git a/arch/arm64/boot/dts/qcom/sdm845-db845c-navigation-mezzanine.dtso b/arch/arm64/boot/dts/qcom/sdm845-db845c-navigation-mezzanine.dtso
index dbe1911d8e470..678a17c805f74 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-db845c-navigation-mezzanine.dtso
+++ b/arch/arm64/boot/dts/qcom/sdm845-db845c-navigation-mezzanine.dtso
@@ -16,7 +16,12 @@ &camss {
 	status = "okay";
 
 	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
 		port@0 {
+			reg = <0>;
+
 			csiphy0_ep: endpoint {
 				data-lanes = <0 1 2 3>;
 				remote-endpoint = <&ov8856_ep>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi
index 6930066857768..b2dc0327dff85 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-google-common.dtsi
@@ -391,6 +391,23 @@ &gpu_zap_shader {
 	firmware-name = "qcom/sdm845/Google/blueline/a630_zap.mbn";
 };
 
+&i2c3 {
+	status = "okay";
+
+	nfc@28 {
+		compatible = "nxp,pn557", "nxp,nxp-nci-i2c";
+		reg = <0x28>;
+
+		interrupts-extended = <&tlmm 63 IRQ_TYPE_EDGE_RISING>;
+
+		enable-gpios = <&tlmm 25 GPIO_ACTIVE_HIGH>;
+		firmware-gpios = <&tlmm 79 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&nfc_int_default &nfc_enable_default>;
+		pinctrl-names = "default";
+	};
+};
+
 &i2c12 {
 	/* Bottom spkr (right) CS35L36 @ 40 */
 
@@ -459,6 +476,20 @@ &tlmm {
 	gpio-reserved-ranges = < 0 4>, /* SPI (Intel MNH Pixel Visual Core) */
 			       <81 4>; /* SPI (most likely Fingerprint Cards FPC1075) */
 
+	nfc_int_default: nfc-int-default-state {
+		pins = "gpio63";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	nfc_enable_default: nfc-enable-default-state {
+		pins = "gpio25", "gpio79";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
 	touchscreen_reset: ts-reset-state {
 		pins = "gpio99";
 		function = "gpio";
diff --git a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi
index 71d070619ad73..e0c3566761bf8 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-lg-common.dtsi
@@ -98,8 +98,7 @@ spss_mem: memory@99000000 {
 			no-map;
 		};
 
-		/* Framebuffer region */
-		memory@9d400000 {
+		framebuffer_mem: memory@9d400000 {
 			reg = <0x0 0x9d400000 0x0 0x2400000>;
 			no-map;
 		};
@@ -473,7 +472,9 @@ &gpu {
 };
 
 &ipa {
-	qcom,gsi-loader = "modem";
+	qcom,gsi-loader = "self";
+	memory-region = <&ipa_fw_mem>;
+
 	status = "okay";
 };
 
@@ -496,6 +497,19 @@ &pm8998_resin {
 	status = "okay";
 };
 
+&pmi8998_flash {
+	status = "okay";
+
+	led-0 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <1>, <2>;
+		led-max-microamp = <100000>;
+		flash-max-microamp = <500000>;
+		flash-max-timeout-us = <500000>;
+	};
+};
+
 &pmi8998_lpg {
 	status = "okay";
 
@@ -680,5 +694,7 @@ &wifi {
 	vdd-3.3-ch0-supply = <&vreg_l25a_3p3>;
 	vdd-3.3-ch1-supply = <&vreg_l23a_3p3>;
 
+	qcom,snoc-host-cap-skip-quirk;
+
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sdm845-lg-judyln.dts b/arch/arm64/boot/dts/qcom/sdm845-lg-judyln.dts
index adf41aa0146a2..349faa123ff15 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-lg-judyln.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-lg-judyln.dts
@@ -14,9 +14,9 @@ / {
 	compatible = "lg,judyln", "qcom,sdm845";
 
 	chosen {
-		framebuffer@9d400000 {
+		framebuffer {
 			compatible = "simple-framebuffer";
-			reg = <0x0 0x9d400000 0x0 (1440 * 3120 * 4)>;
+			memory-region = <&framebuffer_mem>;
 			width = <1440>;
 			height = <3120>;
 			stride = <(1440 * 4)>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845-lg-judyp.dts b/arch/arm64/boot/dts/qcom/sdm845-lg-judyp.dts
index d244ebdd17beb..44e762f78e958 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-lg-judyp.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-lg-judyp.dts
@@ -14,9 +14,9 @@ / {
 	compatible = "lg,judyp", "qcom,sdm845";
 
 	chosen {
-		framebuffer@9d400000 {
+		framebuffer {
 			compatible = "simple-framebuffer";
-			reg = <0x0 0x9d400000 0x0 (1440 * 2880 * 4)>;
+			memory-region = <&framebuffer_mem>;
 			width = <1440>;
 			height = <2880>;
 			stride = <(1440 * 4)>;
diff --git a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
index 6b7378cf4d493..a6c2519a418d9 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-oneplus-common.dtsi
@@ -72,7 +72,7 @@ chosen {
 
 		stdout-path = "serial0:115200n8";
 
-		framebuffer: framebuffer@9d400000 {
+		framebuffer: framebuffer {
 			compatible = "simple-framebuffer";
 			memory-region = <&cont_splash_mem>;
 
@@ -442,6 +442,26 @@ vreg_s3c_0p6: smps3 {
 	};
 };
 
+&cci {
+	status = "okay";
+};
+
+&cci_i2c0 {
+	/* front sony,imx371 @10 */
+
+	/* rear sony,imx519 @1a */
+
+	/* rear onnn,lc898217xc @72 */
+};
+
+&cci_i2c1 {
+	/* sa,sa3103 @0d */
+
+	/* rear sony,imx376k @10 */
+
+	/* onnn,lc898217xc @74 */
+};
+
 &cdsp_pas {
 	firmware-name = "qcom/sdm845/OnePlus/enchilada/cdsp.mbn";
 
@@ -464,6 +484,26 @@ &gpu_zap_shader {
 	firmware-name = "qcom/sdm845/OnePlus/enchilada/a630_zap.mbn";
 };
 
+&i2c3 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	nfc@28 {
+		compatible = "nxp,pn553", "nxp,nxp-nci-i2c";
+		reg = <0x28>;
+
+		interrupts-extended = <&tlmm 63 IRQ_TYPE_EDGE_RISING>;
+
+		enable-gpios = <&tlmm 12 GPIO_ACTIVE_HIGH>;
+		firmware-gpios = <&tlmm 62 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&nfc_default>;
+		pinctrl-1 = <&nfc_sleep>;
+		pinctrl-names = "default", "sleep";
+	};
+};
+
 &i2c10 {
 	status = "okay";
 	clock-frequency = <100000>;
@@ -674,6 +714,11 @@ &qupv3_id_0 {
 	status = "okay";
 };
 
+&qup_i2c3_default {
+	drive-strength = <2>;
+	bias-disable;
+};
+
 &qup_i2c10_default {
 	drive-strength = <2>;
 	bias-disable;
@@ -1007,6 +1052,38 @@ speaker_default: speaker-default-state {
 		bias-pull-up;
 		output-high;
 	};
+
+	nfc_default: nfc-default-state {
+		enable-pins {
+			pins = "gpio12", "gpio62";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		int-pins {
+			pins = "gpio63";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	nfc_sleep: nfc-sleep-state {
+		enable-pins {
+			pins = "gpio12", "gpio62";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		int-pins {
+			pins = "gpio63";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
 };
 
 &venus {
diff --git a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts
index 77f5872de6f19..0d915ecb0444f 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-samsung-starqltechn.dts
@@ -43,9 +43,10 @@ chosen {
 		#address-cells = <2>;
 		#size-cells = <2>;
 		ranges;
-		framebuffer: framebuffer@9d400000 {
+
+		framebuffer {
 			compatible = "simple-framebuffer";
-			reg = <0 0x9d400000 0 (2960 * 1440 * 4)>;//2400000
+			memory-region = <&framebuffer_mem>;
 			width = <1440>;
 			height = <2960>;
 			stride = <(1440 * 4)>;
@@ -109,7 +110,7 @@ vreg_s4a_1p8: pm8998-smps4 {
 	};
 
 	reserved-memory {
-		memory@9d400000 {
+		framebuffer_mem: memory@9d400000 {
 			reg = <0x0 0x9d400000 0x0 0x02400000>;
 			no-map;
 		};
diff --git a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts
index 7d81198bc499c..f21b914f22b51 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-shift-axolotl.dts
@@ -23,7 +23,7 @@ / {
 	qcom,board-id = <11 0>;
 
 	aliases {
-		display0 = &framebuffer0;
+		display0 = &framebuffer;
 		serial0 = &uart9;
 		serial1 = &uart6;
 	};
@@ -36,9 +36,9 @@ chosen {
 		stdout-path = "serial0";
 
 		/* Use framebuffer setup by the bootloader. */
-		framebuffer0: framebuffer@9d400000 {
+		framebuffer: framebuffer {
 			compatible = "simple-framebuffer";
-			reg = <0x0 0x9d400000 0x0 (1080 * 2160 * 4)>;
+			memory-region = <&framebuffer_mem>;
 			width = <1080>;
 			height = <2160>;
 			stride = <(1080 * 4)>;
@@ -62,7 +62,7 @@ key-vol-up {
 	};
 
 	reserved-memory {
-		framebuffer@9d400000 {
+		framebuffer_mem: framebuffer@9d400000 {
 			reg = <0x0 0x9d400000 0x0 (1080 * 2160 * 4)>;
 			no-map;
 		};
@@ -105,6 +105,43 @@ vreg_s4a_1p8: pm8998-smps4 {
 
 		vin-supply = <&vph_pwr>;
 	};
+
+	wcn3990-pmu {
+		compatible = "qcom,wcn3990-pmu";
+
+		pinctrl-0 = <&sw_ctrl_default>;
+		pinctrl-names = "default";
+
+		vddio-supply = <&vreg_s4a_1p8>;
+		vddxo-supply = <&vreg_l7a_1p8>;
+		vddrf-supply = <&vreg_l17a_1p3>;
+		vddch0-supply = <&vreg_l25a_3p3>;
+		vddch1-supply = <&vreg_l23a_3p3>;
+
+		swctrl-gpios = <&pm8998_gpios 3 GPIO_ACTIVE_HIGH>;
+
+		regulators {
+			vreg_pmu_io: ldo0 {
+				regulator-name = "vreg_pmu_io";
+			};
+
+			vreg_pmu_xo: ldo1 {
+				regulator-name = "vreg_pmu_xo";
+			};
+
+			vreg_pmu_rf: ldo2 {
+				regulator-name = "vreg_pmu_rf";
+			};
+
+			vreg_pmu_ch0: ldo3 {
+				regulator-name = "vreg_pmu_ch0";
+			};
+
+			vreg_pmu_ch1: ldo4 {
+				regulator-name = "vreg_pmu_ch1";
+			};
+		};
+	};
 };
 
 &adsp_pas {
@@ -431,6 +468,26 @@ &gpu_zap_shader {
 	firmware-name = "qcom/sdm845/SHIFT/axolotl/a630_zap.mbn";
 };
 
+&i2c3 {
+	clock-frequency = <400000>;
+
+	status = "okay";
+
+	nfc@28 {
+		compatible = "nxp,pn553", "nxp,nxp-nci-i2c";
+		reg = <0x28>;
+
+		interrupts-extended = <&tlmm 63 IRQ_TYPE_EDGE_RISING>;
+
+		enable-gpios = <&tlmm 12 GPIO_ACTIVE_HIGH>;
+		firmware-gpios = <&tlmm 62 GPIO_ACTIVE_HIGH>;
+
+		pinctrl-0 = <&nfc_default>;
+		pinctrl-1 = <&nfc_sleep>;
+		pinctrl-names = "default", "sleep";
+	};
+};
+
 &i2c5 {
 	clock-frequency = <400000>;
 
@@ -448,7 +505,7 @@ touchscreen@38 {
 
 		pinctrl-0 = <&ts_int_active &ts_reset_active>;
 		pinctrl-1 = <&ts_int_suspend &ts_reset_suspend>;
-		pinctrl-names = "default", "suspend";
+		pinctrl-names = "default", "sleep";
 
 		touchscreen-size-x = <1080>;
 		touchscreen-size-y = <2160>;
@@ -526,6 +583,13 @@ &mss_pil {
 };
 
 &pm8998_gpios {
+	sw_ctrl_default: sw-ctrl-default-state {
+		pins = "gpio3";
+		function = "normal";
+		input-enable;
+		bias-pull-down;
+	};
+
 	volume_up_gpio: pm8998-gpio6-state {
 		pinconf {
 			pins = "gpio6";
@@ -668,6 +732,38 @@ sdc2_card_det_n: sd-card-det-n-state {
 		bias-disable;
 	};
 
+	nfc_default: nfc-default-state {
+		enable-pins {
+			pins = "gpio12", "gpio62";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+
+		int-pins {
+			pins = "gpio63";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
+	nfc_sleep: nfc-sleep-state {
+		enable-pins {
+			pins = "gpio12", "gpio62";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-disable;
+		};
+
+		int-pins {
+			pins = "gpio63";
+			function = "gpio";
+			drive-strength = <2>;
+			bias-pull-up;
+		};
+	};
+
 	sde_dsi_active: sde-dsi-active-state {
 		pins = "gpio6", "gpio11";
 		function = "gpio";
@@ -732,10 +828,11 @@ bluetooth {
 		 */
 		firmware-name = "SHIFT/axolotl/crnv21.bin";
 
-		vddio-supply = <&vreg_s4a_1p8>;
-		vddxo-supply = <&vreg_l7a_1p8>;
-		vddrf-supply = <&vreg_l17a_1p3>;
-		vddch0-supply = <&vreg_l25a_3p3>;
+		vddio-supply = <&vreg_pmu_io>;
+		vddxo-supply = <&vreg_pmu_xo>;
+		vddrf-supply = <&vreg_pmu_rf>;
+		vddch0-supply = <&vreg_pmu_ch0>;
+
 		max-speed = <3200000>;
 	};
 };
@@ -790,14 +887,14 @@ &venus {
 };
 
 &wifi {
-	status = "okay";
-
 	vdd-0.8-cx-mx-supply = <&vreg_l5a_0p8>;
-	vdd-1.3-rfa-supply = <&vreg_l17a_1p3>;
-	vdd-1.8-xo-supply = <&vreg_l7a_1p8>;
-	vdd-3.3-ch0-supply = <&vreg_l25a_3p3>;
-	vdd-3.3-ch1-supply = <&vreg_l23a_3p3>;
+	vdd-1.8-xo-supply = <&vreg_pmu_xo>;
+	vdd-1.3-rfa-supply = <&vreg_pmu_rf>;
+	vdd-3.3-ch0-supply = <&vreg_pmu_ch0>;
+	vdd-3.3-ch1-supply = <&vreg_pmu_ch1>;
 
 	qcom,calibration-variant = "shift_axolotl";
 	qcom,snoc-host-cap-8bit-quirk;
+
+	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
index 1298485c42142..9b7fdbca9e1c6 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-common.dtsi
@@ -39,6 +39,22 @@ aliases {
 		serial1 = &uart6;
 	};
 
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		framebuffer {
+			compatible = "simple-framebuffer";
+			memory-region = <&cont_splash_mem>;
+
+			format = "a8r8g8b8";
+			stride = <(1080 * 4)>;
+			height = <2246>;
+			width = <1080>;
+		};
+	};
+
 	gpio-keys {
 		compatible = "gpio-keys";
 		autorepeat;
@@ -661,5 +677,6 @@ &wifi {
 	vdd-3.3-ch1-supply = <&vreg_l23a_3p3>;
 
 	qcom,calibration-variant = "xiaomi_beryllium";
+	qcom,snoc-host-cap-skip-quirk;
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-ebbg.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-ebbg.dts
index 2d6f0e382a6cb..d157622f84d13 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-ebbg.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-ebbg.dts
@@ -6,7 +6,7 @@
 
 / {
 	model = "Xiaomi Pocophone F1 (EBBG)";
-	compatible = "xiaomi,beryllium-ebbg", "qcom,sdm845";
+	compatible = "xiaomi,beryllium-ebbg", "xiaomi,beryllium", "qcom,sdm845";
 };
 
 &display_panel {
diff --git a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-tianma.dts b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-tianma.dts
index b58964cde8342..71816a9f33b48 100644
--- a/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-tianma.dts
+++ b/arch/arm64/boot/dts/qcom/sdm845-xiaomi-beryllium-tianma.dts
@@ -6,7 +6,7 @@
 
 / {
 	model = "Xiaomi Pocophone F1 (Tianma)";
-	compatible = "xiaomi,beryllium", "qcom,sdm845";
+	compatible = "xiaomi,beryllium-tianma", "xiaomi,beryllium", "qcom,sdm845";
 };
 
 &display_panel {
diff --git a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts
index 97f64cb5d570e..b9d9439e230b4 100644
--- a/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts
+++ b/arch/arm64/boot/dts/qcom/sm6125-xiaomi-laurel-sprout.dts
@@ -95,6 +95,33 @@ ts_vdd_supply: regulator-ts-vdd {
 		pinctrl-names = "default";
 	};
 
+	panel_vdd_1p8: regulator-panel-vdd {
+		compatible = "regulator-fixed";
+		regulator-name = "panel_vdd_1p8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+		vin-supply = <&vreg_l9a>;
+
+		enable-active-high;
+		gpio = <&tlmm 26 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&panel_vdd_1p8_en>;
+		pinctrl-names = "default";
+	};
+
+	panel_vci_3p0: regulator-panel-vci {
+		compatible = "regulator-fixed";
+		regulator-name = "panel_vci_3p0";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-boot-on;
+
+		enable-active-high;
+		gpio = <&tlmm 124 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&panel_vci_3p0_en>;
+		pinctrl-names = "default";
+	};
+
 	thermal-zones {
 		rf-pa0-thermal {
 			thermal-sensors = <&pm6125_adc_tm 0>;
@@ -162,6 +189,45 @@ touchscreen@38 {
 	};
 };
 
+&mdss {
+	status = "okay";
+};
+
+&mdss_dsi0 {
+	vdda-supply = <&vreg_l18a>;
+	status = "okay";
+
+	panel@0 {
+		compatible = "samsung,s6e8fco-m1906f9";
+		reg = <0>;
+
+		reset-gpios = <&tlmm 90 GPIO_ACTIVE_LOW>;
+
+		pinctrl-0 = <&mdss_default>;
+		pinctrl-1 = <&mdss_sleep>;
+		pinctrl-names = "default", "sleep";
+
+		vdd-supply = <&panel_vdd_1p8>;
+		vci-supply = <&panel_vci_3p0>;
+
+		port {
+			panel_in: endpoint {
+				remote-endpoint = <&mdss_dsi0_out>;
+			};
+		};
+	};
+};
+
+&mdss_dsi0_out {
+	data-lanes = <0 1 2 3>;
+	remote-endpoint = <&panel_in>;
+};
+
+
+&mdss_dsi0_phy {
+	status = "okay";
+};
+
 &pm6125_adc {
 	pinctrl-names = "default";
 	pinctrl-0 = <&camera_flash_therm &emmc_ufs_therm>;
@@ -426,6 +492,13 @@ &sdhc_2 {
 &tlmm {
 	gpio-reserved-ranges = <22 2>, <28 6>;
 
+	panel_vdd_1p8_en: panel-vdd-default-state {
+		pins = "gpio26";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
 	ts_vdd_en: ts-vdd-default-state {
 		pins = "gpio83";
 		function = "gpio";
@@ -460,6 +533,27 @@ ts_int_suspend: pmx-ts-int-suspend-state {
 		drive-strength = <2>;
 		bias-pull-down;
 	};
+
+	mdss_default: mdss-default-state {
+		pins = "gpio90";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	mdss_sleep: mdss-sleep-state {
+		pins = "gpio90";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	panel_vci_3p0_en: panel-vci-default-state {
+		pins = "gpio124";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
 };
 
 &ufs_mem_hc {
diff --git a/arch/arm64/boot/dts/qcom/sm6350.dtsi b/arch/arm64/boot/dts/qcom/sm6350.dtsi
index 034545d2af2d1..d6adf68563cb0 100644
--- a/arch/arm64/boot/dts/qcom/sm6350.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm6350.dtsi
@@ -1450,6 +1450,72 @@ compute-cb@5 {
 			};
 		};
 
+		lpass_tlmm: pinctrl@33c0000 {
+			compatible = "qcom,sm6350-lpass-lpi-pinctrl";
+			reg = <0x0 0x033c0000 0x0 0x20000>,
+			      <0x0 0x03550000 0x0 0x10000>;
+			gpio-controller;
+			#gpio-cells = <2>;
+			gpio-ranges = <&lpass_tlmm 0 0 15>;
+
+			clocks = <&q6afecc LPASS_HW_MACRO_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>,
+				 <&q6afecc LPASS_HW_DCODEC_VOTE LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+			clock-names = "core",
+				      "audio";
+
+			i2s1_active: i2s1-active-state {
+				clk-pins {
+					pins = "gpio6";
+					function = "i2s1_clk";
+					drive-strength = <8>;
+					bias-disable;
+					output-high;
+				};
+
+				ws-pins {
+					pins = "gpio7";
+					function = "i2s1_ws";
+					drive-strength = <8>;
+					bias-disable;
+					output-high;
+				};
+
+				data-pins {
+					pins = "gpio8", "gpio9";
+					function = "i2s1_data";
+					drive-strength = <8>;
+					bias-disable;
+					output-high;
+				};
+			};
+
+			i2s1_sleep: i2s1-sleep-state {
+				clk-pins {
+					pins = "gpio6";
+					function = "i2s1_clk";
+					drive-strength = <2>;
+					bias-pull-down;
+					input-enable;
+				};
+
+				ws-pins {
+					pins = "gpio7";
+					function = "i2s1_ws";
+					drive-strength = <2>;
+					bias-pull-down;
+					input-enable;
+				};
+
+				data-pins {
+					pins = "gpio8", "gpio9";
+					function = "i2s1_data";
+					drive-strength = <2>;
+					bias-pull-down;
+					input-enable;
+				};
+			};
+		};
+
 		gpu: gpu@3d00000 {
 			compatible = "qcom,adreno-619.0", "qcom,adreno";
 			reg = <0x0 0x03d00000 0x0 0x40000>,
diff --git a/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts
new file mode 100644
index 0000000000000..3c836e196b19c
--- /dev/null
+++ b/arch/arm64/boot/dts/qcom/sm7325-motorola-dubai.dts
@@ -0,0 +1,1456 @@
+// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
+/*
+ * Copyright (c) 2026, Val Packett <val@packett.cool>
+ */
+
+/dts-v1/;
+
+/* PM7250B is configured to use SID8/9 */
+#define PM7250B_SID 8
+#define PM7250B_SID1 9
+
+#include <dt-bindings/arm/qcom,ids.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pm7325.h>
+#include <dt-bindings/iio/qcom,spmi-adc7-pmk8350.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/pinctrl/qcom,pmic-gpio.h>
+#include <dt-bindings/regulator/qcom,rpmh-regulator.h>
+
+#include "sm7325.dtsi"
+#include "pm7325.dtsi"
+#include "pm7250b.dtsi"
+#include "pm8350c.dtsi" /* PM7350C */
+#include "pmk8350.dtsi" /* PMK7325 */
+
+/ {
+	model = "Motorola Edge 30";
+	compatible = "motorola,dubai", "qcom,sm7325";
+	chassis-type = "handset";
+
+	aliases {
+		bluetooth0 = &bluetooth;
+		serial0 = &uart5;
+		serial1 = &uart7;
+		wifi0 = &wifi;
+	};
+
+	chosen {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		stdout-path = "serial0:115200n8";
+
+		framebuffer {
+			compatible = "simple-framebuffer";
+			memory-region = <&framebuffer_mem>;
+			width = <1080>;
+			height = <2400>;
+			stride = <(1080 * 4)>;
+			format = "a8r8g8b8";
+
+			clocks = <&gcc GCC_DISP_HF_AXI_CLK>,
+				 <&dispcc DISP_CC_MDSS_MDP_CLK>,
+				 <&dispcc DISP_CC_MDSS_BYTE0_CLK>,
+				 <&dispcc DISP_CC_MDSS_BYTE0_INTF_CLK>,
+				 <&dispcc DISP_CC_MDSS_PCLK0_CLK>,
+				 <&dispcc DISP_CC_MDSS_VSYNC_CLK>;
+			power-domains = <&dispcc DISP_CC_MDSS_CORE_GDSC>;
+		};
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+
+		pinctrl-0 = <&kypd_vol_up_n>;
+		pinctrl-names = "default";
+
+		key-volume-up {
+			label = "Volume Up";
+			gpios = <&pm7325_gpios 6 GPIO_ACTIVE_LOW>;
+			linux,code = <KEY_VOLUMEUP>;
+		};
+	};
+
+	pmic-glink {
+		compatible = "qcom,sm7325-pmic-glink",
+			     "qcom,qcm6490-pmic-glink",
+			     "qcom,pmic-glink";
+
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		orientation-gpios = <&tlmm 140 GPIO_ACTIVE_HIGH>;
+
+		connector@0 {
+			compatible = "usb-c-connector";
+			reg = <0>;
+
+			power-role = "dual";
+			data-role = "dual";
+
+			ports {
+				#address-cells = <1>;
+				#size-cells = <0>;
+
+				port@0 {
+					reg = <0>;
+
+					pmic_glink_hs_in: endpoint {
+						remote-endpoint = <&usb_1_dwc3_hs>;
+					};
+				};
+
+				port@2 {
+					reg = <2>;
+
+					pmic_glink_sbu: endpoint {
+						remote-endpoint = <&fsa4480_sbu_mux>;
+					};
+				};
+			};
+		};
+	};
+
+	reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ramoops@ae000000 {
+			compatible = "ramoops";
+			reg = <0x0 0xae000000 0x0 0xc0000>;
+			console-size = <0x40000>;
+			mem-type = <2>;
+			pmsg-size = <0x40000>;
+			record-size = <0x3f800>;
+		};
+
+		removed@c0000000 {
+			reg = <0x0 0xc0000000 0x0 0x5100000>;
+			no-map;
+		};
+
+		framebuffer_mem: framebuffer@e1000000 {
+			reg = <0x0 0xe1000000 0x0 (1080 * 2400 * 4)>;
+			no-map;
+		};
+
+		linux,cma {
+			compatible = "shared-dma-pool";
+			size = <0x0 0x8000000>;
+			reusable;
+			linux,cma-default;
+		};
+	};
+
+	thermal-zones {
+		cam-flash-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 2>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		chg-skin-thermal {
+			thermal-sensors = <&pm7250b_adc_tm 0>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		chg-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 4>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		conn-thermal {
+			thermal-sensors = <&pm7250b_adc_tm 1>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pa-1-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 5>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		pa-2-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 6>;
+
+			/* Reports negative temperature. */
+			status = "disabled";
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		quiet-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 1>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		sdm-skin-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 3>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+
+		xo-thermal {
+			thermal-sensors = <&pmk8350_adc_tm 0>;
+
+			trips {
+				active-config0 {
+					temperature = <125000>;
+					hysteresis = <1000>;
+					type = "passive";
+				};
+			};
+		};
+	};
+
+	// S2B is really ebi.lvl but it's there for supply map completeness sake.
+	vreg_s2b_0p7: smpb2-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vreg_s2b_0p7";
+
+		regulator-min-microvolt = <65535>;
+		regulator-max-microvolt = <65535>;
+		regulator-always-on;
+		vin-supply = <&vph_pwr>;
+	};
+
+	vph_pwr: vph-pwr-regulator {
+		compatible = "regulator-fixed";
+		regulator-name = "vph_pwr";
+		regulator-min-microvolt = <3700000>;
+		regulator-max-microvolt = <3700000>;
+	};
+
+	wcn6750-pmu {
+		compatible = "qcom,wcn6750-pmu";
+		pinctrl-0 = <&bt_en>;
+		pinctrl-names = "default";
+		vddaon-supply = <&mos_0p95_aon_s10c>;
+		vddasd-supply = <&vreg_l11c_2p8>;
+		vddpmu-supply = <&mos_0p95_dig_s10c>;
+		vddio-supply = <&vdd18_io>;
+		vddrfa0p8-supply = <&vdd09_pmu_rfa_i>;
+		vddrfa1p2-supply = <&vdd13_pmu_rfa_i>;
+		vddrfa1p7-supply = <&vdd19_pmu_rfa_i>;
+		vddrfa2p2-supply = <&vdd22_wlpa_s1c>;
+
+		bt-enable-gpios = <&tlmm 85 GPIO_ACTIVE_HIGH>;
+
+		regulators {
+			vreg_pmu_rfa_cmn: ldo0 {
+				regulator-name = "vreg_pmu_rfa_cmn";
+			};
+
+			vreg_pmu_aon_0p59: ldo1 {
+				regulator-name = "vreg_pmu_aon_0p59";
+			};
+
+			vreg_pmu_wlcx_0p8: ldo2 {
+				regulator-name = "vreg_pmu_wlcx_0p8";
+			};
+
+			vreg_pmu_wlmx_0p85: ldo3 {
+				regulator-name = "vreg_pmu_wlmx_0p85";
+			};
+
+			vreg_pmu_btcmx_0p85: ldo4 {
+				regulator-name = "vreg_pmu_btcmx_0p85";
+			};
+
+			vreg_pmu_rfa_0p8: ldo5 {
+				regulator-name = "vreg_pmu_rfa_0p8";
+			};
+
+			vreg_pmu_rfa_1p2: ldo6 {
+				regulator-name = "vreg_pmu_rfa_1p2";
+			};
+
+			vreg_pmu_rfa_1p7: ldo7 {
+				regulator-name = "vreg_pmu_rfa_1p7";
+			};
+
+			vreg_pmu_pcie_0p9: ldo8 {
+				regulator-name = "vreg_pmu_pcie_0p9";
+			};
+
+			vreg_pmu_pcie_1p8: ldo9 {
+				regulator-name = "vreg_pmu_pcie_1p8";
+			};
+		};
+	};
+};
+
+&apps_rsc {
+	regulators-0 {
+		compatible = "qcom,pm7325-rpmh-regulators";
+		qcom,pmic-id = "b";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s8-supply = <&vph_pwr>;
+
+		vdd-l1-l4-l12-l15-supply = <&vreg_s7b_0p952>;
+		vdd-l2-l7-supply = <&vreg_bob>;
+		vdd-l3-supply = <&vreg_s2b_0p7>;
+		vdd-l5-supply = <&vreg_s2b_0p7>;
+		vdd-l6-l9-l10-supply = <&vreg_s8b_1p256>;
+		vdd-l8-supply = <&vreg_s7b_0p952>;
+		vdd-l11-l17-l18-l19-supply = <&vreg_s1b_1p856>;
+		vdd-l13-supply = <&vreg_s7b_0p952>;
+		vdd-l14-l16-supply = <&vreg_s8b_1p256>;
+
+		/*
+		 * S2, L4-L5 are ARCs:
+		 * S2 - ebi.lvl,
+		 * L4 - lmx.lvl,
+		 * l5 - lcx.lvl.
+		 *
+		 * L10 are unused.
+		 */
+
+		vdd19_pmu_rfa_i:
+		vreg_s1b_1p856: smps1 {
+			regulator-name = "vreg_s1b_1p856";
+			regulator-min-microvolt = <1840000>;
+			regulator-max-microvolt = <2040000>;
+		};
+
+		mos_0p95_aon_s10c:
+		mos_0p95_dig_s10c:
+		vdd09_pmu_rfa_i:
+		vreg_s7b_0p952: smps7 {
+			regulator-name = "vreg_s7b_0p952";
+			regulator-min-microvolt = <535000>;
+			regulator-max-microvolt = <1120000>;
+		};
+
+		vdd13_pmu_rfa_i:
+		vreg_s8b_1p256: smps8 {
+			regulator-name = "vreg_s8b_1p256";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1500000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_RET>;
+		};
+
+		vreg_l1b_0p912: ldo1 {
+			regulator-name = "vreg_l1b_0p912";
+			regulator-min-microvolt = <825000>;
+			regulator-max-microvolt = <925000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_a_usbhs_3p1:
+		vreg_l2b_3p072: ldo2 {
+			regulator-name = "vreg_l2b_3p072";
+			regulator-min-microvolt = <2700000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l3b_0p6: ldo3 {
+			regulator-name = "vreg_l3b_0p6";
+			regulator-min-microvolt = <312000>;
+			regulator-max-microvolt = <910000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_a_dsi_0_1p2:
+		vdd_a_ufs_0_1p2:
+		vreg_l6b_1p2: ldo6 {
+			regulator-name = "vreg_l6b_1p2";
+			regulator-min-microvolt = <1140000>;
+			regulator-max-microvolt = <1260000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7b_2p96: ldo7 {
+			regulator-name = "vreg_l7b_2p96";
+			regulator-min-microvolt = <2400000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8b_0p904: ldo8 {
+			regulator-name = "vreg_l8b_0p904";
+			regulator-min-microvolt = <870000>;
+			regulator-max-microvolt = <970000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9b_1p2: ldo9 {
+			regulator-name = "vreg_l9b_1p2";
+			regulator-min-microvolt = <1200000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11b_1p776: ldo11 {
+			regulator-name = "vreg_l11b_1p776";
+			regulator-min-microvolt = <1504000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l12b_0p8: ldo12 {
+			regulator-name = "vreg_l12b_0p8";
+			regulator-min-microvolt = <751000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l13b_0p8: ldo13 {
+			regulator-name = "vreg_l13b_0p8";
+			regulator-min-microvolt = <530000>;
+			regulator-max-microvolt = <824000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l14b_1p2: ldo14 {
+			regulator-name = "vreg_l14b_1p2";
+			regulator-min-microvolt = <1080000>;
+			regulator-max-microvolt = <1304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l15b_0p88: ldo15 {
+			regulator-name = "vreg_l15b_0p88";
+			regulator-min-microvolt = <765000>;
+			regulator-max-microvolt = <1020000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l16b_1p2: ldo16 {
+			regulator-name = "vreg_l16b_1p2";
+			regulator-min-microvolt = <1100000>;
+			regulator-max-microvolt = <1300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l17b_1p8: ldo17 {
+			regulator-name = "vreg_l17b_1p8";
+			regulator-min-microvolt = <1700000>;
+			regulator-max-microvolt = <1900000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l18b_1p8: ldo18 {
+			regulator-name = "vreg_l18b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-always-on;
+			regulator-boot-on;
+		};
+
+		vdd18_io:
+		vreg_l19b_1p8: ldo19 {
+			regulator-name = "vreg_l19b_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+	};
+
+	regulators-1 {
+		compatible = "qcom,pm8350c-rpmh-regulators";
+		qcom,pmic-id = "c";
+
+		vdd-s1-supply = <&vph_pwr>;
+		vdd-s2-supply = <&vph_pwr>;
+		vdd-s5-supply = <&vph_pwr>;
+		vdd-s7-supply = <&vph_pwr>;
+		vdd-s9-supply = <&vph_pwr>;
+		vdd-s10-supply = <&vph_pwr>;
+
+		vdd-l1-l12-supply = <&vreg_s1b_1p856>;
+		vdd-l2-l8-supply = <&vreg_s1b_1p856>;
+		vdd-l3-l4-l5-l7-l13-supply = <&vreg_bob>;
+		vdd-l6-l9-l11-supply = <&vreg_bob>;
+		vdd-l10-supply = <&vreg_s7b_0p952>;
+
+		vdd-bob-supply = <&vph_pwr>;
+
+		/*
+		 * S2, S5, S7, S10 are ARCs:
+		 * S2 - cx.lvl,
+		 * S5 - mss.lvl,
+		 * S7 - gfx.lvl,
+		 * S10 - mx.lvl.
+		 */
+
+		vdd22_wlpa_s1c:
+		vreg_s1c_2p2: smps1 {
+			regulator-name = "vreg_s1c_2p2";
+			regulator-min-microvolt = <2190000>;
+			regulator-max-microvolt = <2210000>;
+		};
+
+		vreg_s9c_0p676: smps9 {
+			regulator-name = "vreg_s9c_0p676";
+			regulator-min-microvolt = <1010000>;
+			regulator-max-microvolt = <1170000>;
+		};
+
+		vdd_a_usbhs_1p8:
+		vdd_qfprom:
+		vreg_l1c_1p8: ldo1 {
+			regulator-name = "vreg_l1c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1980000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		iovdd_ts:
+		vreg_l2c_1p8: ldo2 {
+			regulator-name = "vreg_l2c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_ts:
+		vreg_l3c_3p0: ldo3 {
+			regulator-name = "vreg_l3c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3304000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l4c_1p8_3p0: ldo4 {
+			regulator-name = "vreg_l4c_1p8_3p0";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l5c_1p8_3p0: ldo5 {
+			regulator-name = "vreg_l5c_1p8_3p0";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l6c_2p96: ldo6 {
+			regulator-name = "vreg_l6c_2p96";
+			regulator-min-microvolt = <1650000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l7c_3p0: ldo7 {
+			regulator-name = "vreg_l7c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l8c_1p8: ldo8 {
+			regulator-name = "vreg_l8c_1p8";
+			regulator-min-microvolt = <1620000>;
+			regulator-max-microvolt = <2000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l9c_3p3: ldo9 {
+			regulator-name = "vreg_l9c_2p96";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3300000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vdd_a_dsi_0_0p9:
+		vdd_a_ufs_0_core:
+		vdd_a_usbhs_core:
+		vreg_l10c_0p88: ldo10 {
+			regulator-name = "vreg_l10c_0p88";
+			regulator-min-microvolt = <720000>;
+			regulator-max-microvolt = <1050000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+			regulator-allow-set-load;
+			regulator-allowed-modes = <RPMH_REGULATOR_MODE_LPM
+						   RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		vreg_l11c_2p8: ldo11 {
+			regulator-name = "vreg_l11c_2p8";
+			regulator-min-microvolt = <2800000>;
+			regulator-max-microvolt = <3544000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		disp_iovcc_1p8:
+		vreg_l12c_1p8: ldo12 {
+			regulator-name = "vreg_l12c_1p8";
+			regulator-min-microvolt = <1800000>;
+			regulator-max-microvolt = <1800000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		disp_vci_3p0:
+		vreg_l13c_3p0: ldo13 {
+			regulator-name = "vreg_l13c_3p0";
+			regulator-min-microvolt = <3000000>;
+			regulator-max-microvolt = <3000000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_HPM>;
+		};
+
+		audio_sw_vcc:
+		vreg_bob: bob {
+			regulator-name = "vreg_bob";
+			regulator-min-microvolt = <3008000>;
+			regulator-max-microvolt = <3960000>;
+			regulator-initial-mode = <RPMH_REGULATOR_MODE_AUTO>;
+		};
+	};
+};
+
+&gcc {
+	protected-clocks = <GCC_CFG_NOC_LPASS_CLK>,
+			   <GCC_MSS_CFG_AHB_CLK>,
+			   <GCC_MSS_OFFLINE_AXI_CLK>,
+			   <GCC_MSS_Q6SS_BOOT_CLK_SRC>,
+			   <GCC_MSS_Q6_MEMNOC_AXI_CLK>,
+			   <GCC_MSS_SNOC_AXI_CLK>,
+			   <GCC_QSPI_CNOC_PERIPH_AHB_CLK>,
+			   <GCC_QSPI_CORE_CLK>,
+			   <GCC_QSPI_CORE_CLK_SRC>,
+			   <GCC_SEC_CTRL_CLK_SRC>,
+			   <GCC_WPSS_AHB_BDG_MST_CLK>,
+			   <GCC_WPSS_AHB_CLK>,
+			   <GCC_WPSS_RSCP_CLK>;
+};
+
+&gpi_dma0 {
+	status = "okay";
+};
+
+&gpi_dma1 {
+	status = "okay";
+};
+
+&gpu {
+	status = "okay";
+};
+
+&gpu_zap_shader {
+	firmware-name = "qcom/sm7325/motorola/dubai/a660_zap.mbn";
+};
+
+&i2c4 {
+	clock-frequency = <100000>;
+
+	status = "okay";
+
+	typec-mux@42 {
+		compatible = "fcs,fsa4480";
+		reg = <0x42>;
+
+		interrupts-extended = <&tlmm 6 IRQ_TYPE_LEVEL_LOW>;
+
+		vcc-supply = <&audio_sw_vcc>;
+
+		mode-switch;
+		orientation-switch;
+
+		port {
+			fsa4480_sbu_mux: endpoint {
+				remote-endpoint = <&pmic_glink_sbu>;
+			};
+		};
+	};
+};
+
+&ipa {
+	firmware-name = "qcom/sm7325/motorola/dubai/yupik_ipa_fws.mbn";
+	memory-region = <&ipa_fw_mem>;
+
+	qcom,gsi-loader = "self";
+
+	status = "okay";
+};
+
+&pm7250b_adc {
+	channel@4e {
+		reg = <ADC5_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "chg_skin_therm";
+	};
+
+	channel@4f {
+		reg = <ADC5_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "conn_therm";
+	};
+};
+
+&pm7250b_adc_tm {
+	status = "okay";
+
+	chg-skin-therm@0 {
+		reg = <0>;
+		io-channels = <&pm7250b_adc ADC5_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	conn-therm@1 {
+		reg = <1>;
+		io-channels = <&pm7250b_adc ADC5_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+};
+
+&pm7250b_gpios {
+	gpio-line-names = "FG_ALERT_N", /* GPIO_1 */
+			  "SLAVECP_INT",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "DOUBLER_STAT",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC", /* GPIO_10 */
+			  "NC",
+			  "NC";
+};
+
+&pm7325_gpios {
+	gpio-line-names = "PA_THERM1", /* GPIO_1 */
+			  "NC",
+			  "NC",
+			  "PA_THERM2",
+			  "CBL_PWR_N",
+			  "KYPD_VOL_UP_N",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC"; /* GPIO_10 */
+
+	kypd_vol_up_n: kypd-volp-n-state {
+		pins = "gpio6";
+		function = PMIC_GPIO_FUNC_NORMAL;
+		bias-pull-up;
+		input-enable;
+		power-source = <1>;
+	};
+};
+
+&pm8350c_gpios {
+	gpio-line-names = "NC", /* GPIO_1 */
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC"; /* GPIO_9 */
+};
+
+&pm8350c_flash {
+	status = "okay";
+
+	led-0 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <1>, <4>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <400000>;
+	};
+
+	led-1 {
+		function = LED_FUNCTION_FLASH;
+		color = <LED_COLOR_ID_WHITE>;
+		led-sources = <2>, <3>;
+		led-max-microamp = <500000>;
+		flash-max-microamp = <1500000>;
+		flash-max-timeout-us = <400000>;
+	};
+};
+
+&pmk8350_adc_tm {
+	status = "okay";
+
+	xo-therm@0 {
+		reg = <0>;
+		io-channels = <&pmk8350_vadc PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	quiet-therm@1 {
+		reg = <1>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	cam-flash-therm@2 {
+		reg = <2>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	sdm-skin-therm@3 {
+		reg = <3>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	chg-therm@4 {
+		reg = <4>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_AMUX_THM4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	pa-therm-1@5 {
+		reg = <5>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_GPIO1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+
+	pa-therm-2@6 {
+		reg = <6>;
+		io-channels = <&pmk8350_vadc PM7325_ADC7_GPIO4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time-us = <200>;
+	};
+};
+
+&pmk8350_gpios {
+	gpio-line-names = "NC", /* GPIO_0 */
+			  "NC",
+			  "TP_PMK_GPIO_3",
+			  "PMK_OPTION"; /* GPIO_4 */
+};
+
+&pmk8350_rtc {
+	status = "okay";
+};
+
+&pmk8350_vadc {
+	status = "okay";
+
+	channel@44 {
+		reg = <PMK8350_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pmk8350_xo_therm";
+	};
+
+	channel@144 {
+		reg = <PM7325_ADC7_AMUX_THM1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_quiet_therm";
+	};
+
+	channel@145 {
+		reg = <PM7325_ADC7_AMUX_THM2_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_cam_flash_therm";
+	};
+
+	channel@146 {
+		reg = <PM7325_ADC7_AMUX_THM3_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_sdm_skin_therm";
+	};
+
+	channel@147 {
+		reg = <PM7325_ADC7_AMUX_THM4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_chg_therm";
+	};
+
+	channel@14a {
+		reg = <PM7325_ADC7_GPIO1_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_pa_therm1";
+	};
+
+	channel@14d {
+		reg = <PM7325_ADC7_GPIO4_100K_PU>;
+		qcom,ratiometric;
+		qcom,hw-settle-time = <200>;
+		qcom,pre-scaling = <1 1>;
+		label = "pm7325_pa_therm2";
+	};
+};
+
+&pon_pwrkey {
+	status = "okay";
+};
+
+&pon_resin {
+	linux,code = <KEY_VOLUMEDOWN>;
+
+	status = "okay";
+};
+
+&qfprom {
+	vcc-supply = <&vdd_qfprom>;
+};
+
+&qup_spi13_cs {
+	drive-strength = <6>;
+	bias-pull-down;
+};
+
+&qup_spi13_data_clk {
+	drive-strength = <6>;
+	bias-pull-down;
+};
+
+&qup_uart5_rx {
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qup_uart5_tx {
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qup_uart7_cts {
+	/*
+	 * Configure a bias-bus-hold on CTS to lower power
+	 * usage when Bluetooth is turned off. Bus hold will
+	 * maintain a low power state regardless of whether
+	 * the Bluetooth module drives the pin in either
+	 * direction or leaves the pin fully unpowered.
+	 */
+	bias-bus-hold;
+};
+
+&qup_uart7_rts {
+	/* We'll drive RTS, so no pull */
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qup_uart7_rx {
+	/*
+	 * Configure a pull-up on RX. This is needed to avoid
+	 * garbage data when the TX pin of the Bluetooth module is
+	 * in tri-state (module powered off or not driving the
+	 * signal yet).
+	 */
+	bias-pull-up;
+};
+
+&qup_uart7_tx {
+	/* We'll drive TX, so no pull */
+	drive-strength = <2>;
+	bias-disable;
+};
+
+&qupv3_id_0 {
+	status = "okay";
+};
+
+&qupv3_id_1 {
+	status = "okay";
+};
+
+&remoteproc_adsp {
+	firmware-name = "qcom/sm7325/motorola/dubai/adsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_cdsp {
+	firmware-name = "qcom/sm7325/motorola/dubai/cdsp.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_mpss {
+	firmware-name = "qcom/sm7325/motorola/dubai/modem.mbn";
+
+	status = "okay";
+};
+
+&remoteproc_wpss {
+	firmware-name = "qcom/sm7325/motorola/dubai/wpss.mbn";
+
+	status = "okay";
+};
+
+&rmtfs_mem {
+	qcom,vmid = <QCOM_SCM_VMID_MSS_MSA>,
+		    <QCOM_SCM_VMID_NAV>;
+};
+
+&spi13 {
+	status = "okay";
+
+	touchscreen@0 {
+		compatible = "goodix,gt9916";
+		reg = <0>;
+
+		interrupts-extended = <&tlmm 81 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&tlmm 105 GPIO_ACTIVE_LOW>;
+
+		avdd-supply = <&vdd_ts>;
+		vddio-supply = <&iovdd_ts>;
+
+		spi-max-frequency = <1000000>;
+
+		touchscreen-size-x = <1080>;
+		touchscreen-size-y = <2400>;
+
+		pinctrl-0 = <&ts_int_n>, <&ts_reset_n>;
+		pinctrl-names = "default";
+	};
+};
+
+&tlmm {
+	gpio-line-names = "NC", /* GPIO_0 */
+			  "NC",
+			  "NC",
+			  "NC",
+			  "SMARTPA_I2C_SDA",
+			  "SMARTPA_I2C_SCK",
+			  "AUD_4480_DET",
+			  "AUD_4480_INT",
+			  "FASTCHARGE_I2C_SDA",
+			  "FASTCHARGE_I2C_SCL",
+			  "SM_GPSLNA_EN_GPIO10", /* GPIO_10 */
+			  "NC",
+			  "PERI_I2C_SDA",
+			  "PERI_I2C_SCL",
+			  "NC",
+			  "NC",
+			  "APPS_I2C_SDA",
+			  "APPS_I2C_SCL",
+			  "NC",
+			  "DISP_LDO_EN",
+			  "CAM_FW_RST_N", /* GPIO_20 */
+			  "CAM_RM_RST_N",
+			  "DBG_UART_TXD",
+			  "DBG_UART_RXD",
+			  "TOPSB_INT_N",
+			  "MCAM_LDO_EN",
+			  "HOST2WLAN_SOL",
+			  "WLAN2HOST_SOL",
+			  "UART_BT_RTS_AP_CTS",
+			  "UART_BT_CTS_AP_RTS",
+			  "UART_BT_RX_AP_TX", /* GPIO_30 */
+			  "UART_BT_TX_AP_RX",
+			  "FRONT_60M_INT",
+			  "FRONT_60M_RST",
+			  "FP_INT",
+			  "FP_RST",
+			  "NFC_I2C_SDA",
+			  "NFC_I2C_SCL",
+			  "NFC_EN",
+			  "NFC_CLK_REQ",
+			  "NFC_SE_SPI_NREST", /* GPIO_40 */
+			  "NFC_IRQ",
+			  "RF_LDO_GPIO42_EN",
+			  "GOOGLE_KEY_INT", /* no such key exists */
+			  "DISP_RST_N",
+			  "SB_INT_N",
+			  "HAP_RST_N",
+			  "HAP_INT",
+			  "ESE_SPI_MISO",
+			  "ESE_SPI_MOSI",
+			  "ESE_SPI_CLK", /* GPIO_50 */
+			  "ESE_SPI_CS",
+			  "TP_SPI_MISO",
+			  "TP_SPI_MOSI",
+			  "TP_SPI_CLK",
+			  "TP_SPI_CS_N",
+			  "FP_SPI_MISO",
+			  "FP_SPI_MOSI",
+			  "FP_SPI_CLK",
+			  "FP_SPI_CS_N",
+			  "HW_ID_1", /* GPIO_60 */
+			  "HW_ID_2",
+			  "CAM_PMU_EN",
+			  "WIDE_DVDD_LDO_EN",
+			  "CAM_MCLK0",
+			  "CAM_MCLK1",
+			  "CAM_MCLK2",
+			  "CAM_MCLK3",
+			  "NC",
+			  "CCI_I2C_SDA0",
+			  "CCI_I2C_SCL0", /* GPIO_70 */
+			  "CCI_I2C_SDA1",
+			  "CCI_I2C_SCL1",
+			  "CCI_I2C_SDA2",
+			  "CCI_I2C_SCL2",
+			  "CCI_I2C_SDA3",
+			  "CCI_I2C_SCL3",
+			  "CAM_RT_RST_N",
+			  "CAM_RU_RST_N",
+			  "FCAM_LDO_EN",
+			  "DISP_TE", /* GPIO_80 */
+			  "TP_INT_N",
+			  "FORCED_USB_BOOT",
+			  "SM_CDC_RST_N",
+			  "WLAN_EN",
+			  "BT_EN",
+			  "WCN_SW_CTRL",
+			  "PCIE0_RESET_N",
+			  "PCIE0_CLK_REQ_N",
+			  "PCIE0_WAKE_N",
+			  "MOS_AS_EN", /* GPIO_90 */
+			  "NC",
+			  "",
+			  "NC",
+			  "BT_FM_SLIMBUS_CLK",
+			  "BT_FM_SLIMBUS_DATA",
+			  "NC",
+			  "RFCONN_DET_1",
+			  "RF_CON_DET_2",
+			  "RF_CON_DET_3",
+			  "NC", /* GPIO_100 */
+			  "NC",
+			  "NC",
+			  "ACCEL_INT",
+			  "NC",
+			  "TP_RST_N",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "UIM2_DATA",
+			  "UIM2_CLK", /* GPIO_110 */
+			  "UIM2_RESET",
+			  "UIM2_PRESENT",
+			  "UIM1_DATA",
+			  "UIM1_CLK",
+			  "UIM1_RESET",
+			  "UIM1_PRESENT",
+			  "SM_RFFE0_CLK",
+			  "SM_RFFE0_DATA",
+			  "SM_RFFE1_CLK",
+			  "SM_RFFE1_DATA", /* GPIO_120 */
+			  "PA_MUTING",
+			  "SM_GRFC5",
+			  "LAA_RX",
+			  "SM_GRFC7",
+			  "SM_RFFE4_CLK",
+			  "SM_RFFE4_DATA",
+			  "WLAN_COEX_UART_RX",
+			  "WLAN_COEX_UART_TX",
+			  "NC",
+			  "NC", /* GPIO_130 */
+			  "SM_GRFC12",
+			  "NC",
+			  "QLINK0_REQUEST",
+			  "QLINK0_ENABLE",
+			  "QLINK0_WMSS_RESET_N",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "NC",
+			  "USB_CC_DIR", /* GPIO_140 */
+			  "SAR_INT_N",
+			  "PROX_INT_N",
+			  "",
+			  "SM_SWR_TX_CLK",
+			  "SM_SWR_TX_DATA0",
+			  "SM_SWR_TX_DATA1",
+			  "SM_SWR_RX_CLK",
+			  "SM_SWR_RX_DATA0",
+			  "SM_SWR_RX_DATA1",
+			  "NC", /* GPIO_150 */
+			  "NC",
+			  "NC",
+			  "NC",
+			  "SB_MI2S_SCK",
+			  "SB_MI2S_WS",
+			  "SB_MI2S_RXDAT_AP_TX",
+			  "SB_MI2S_TXDAT_AP_RX",
+			  "NC",
+			  "SNS_I3C0_SDA",
+			  "SNS_I3C0_SCL", /* GPIO_160 */
+			  "SSC_I2C4_SDA",
+			  "SSC_I2C4_SCL",
+			  "MAG_I2C_SDA",
+			  "MAG_I2C_SCL",
+			  "NC",
+			  "NC",
+			  "",
+			  "",
+			  "",
+			  "", /* GPIO_170 */
+			  "SSC_BT_UART4_TX",
+			  "SSC_BT_UART4_RX",
+			  "NC",
+			  "NC";
+	gpio-reserved-ranges = <48 4>, /* SPI (eSE - embedded Secure Element) */
+			       <56 4>; /* SPI (fingerprint reader) */
+
+	qup_uart7_sleep_cts: qup-uart7-sleep-cts-state {
+		pins = "gpio28";
+		function = "gpio";
+		/*
+		 * Configure a bias-bus-hold on CTS to lower power
+		 * usage when Bluetooth is turned off. Bus hold will
+		 * maintain a low power state regardless of whether
+		 * the Bluetooth module drives the pin in either
+		 * direction or leaves the pin fully unpowered.
+		 */
+		bias-bus-hold;
+	};
+
+	qup_uart7_sleep_rts: qup-uart7-sleep-rts-state {
+		pins = "gpio29";
+		function = "gpio";
+		/*
+		 * Configure pull-down on RTS. As RTS is active low
+		 * signal, pull it low to indicate the BT SoC that it
+		 * can wakeup the system anytime from suspend state by
+		 * pulling RX low (by sending wakeup bytes).
+		 */
+		bias-pull-down;
+	};
+
+	qup_uart7_sleep_tx: qup-uart7-sleep-tx-state {
+		pins = "gpio30";
+		function = "gpio";
+		/*
+		 * Configure pull-up on TX when it isn't actively driven
+		 * to prevent BT SoC from receiving garbage during sleep.
+		 */
+		bias-pull-up;
+	};
+
+	qup_uart7_sleep_rx: qup-uart7-sleep-rx-state {
+		pins = "gpio31";
+		function = "gpio";
+		/*
+		 * Configure a pull-up on RX. This is needed to avoid
+		 * garbage data when the TX pin of the Bluetooth module
+		 * is floating which may cause spurious wakeups.
+		 */
+		bias-pull-up;
+	};
+
+	oled_reset_n: oled-reset-n-state {
+		pins = "gpio44";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-disable;
+	};
+
+	aw86224_reset_default: aw86224-reset-default-state {
+		pins = "gpio46";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	aw86224_int_default: aw86224-int-default-state {
+		pins = "gpio47";
+		function = "gpio";
+		drive-strength = <2>;
+		bias-pull-up;
+	};
+
+	mdp_vsync_p: mdp-vsync-p-state {
+		pins = "gpio80";
+		function = "mdp_vsync";
+		drive-strength = <2>;
+		bias-pull-down;
+	};
+
+	ts_int_n: ts-int-n-state {
+		pins = "gpio81";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+
+	bt_en: bt-en-state {
+		pins = "gpio85";
+		function = "gpio";
+		output-low;
+		bias-disable;
+	};
+
+	ts_reset_n: ts-int-n-state {
+		pins = "gpio105";
+		function = "gpio";
+		drive-strength = <8>;
+		bias-pull-up;
+	};
+};
+
+&uart5 {
+	status = "okay";
+};
+
+&uart7 {
+	/delete-property/ interrupts;
+	interrupts-extended = <&intc GIC_SPI 608 IRQ_TYPE_LEVEL_HIGH>,
+			      <&tlmm 31 IRQ_TYPE_EDGE_FALLING>;
+	pinctrl-1 = <&qup_uart7_sleep_cts>,
+		    <&qup_uart7_sleep_rts>,
+		    <&qup_uart7_sleep_tx>,
+		    <&qup_uart7_sleep_rx>;
+	pinctrl-names = "default",
+			"sleep";
+
+	status = "okay";
+
+	bluetooth: bluetooth {
+		compatible = "qcom,wcn6750-bt";
+
+		vddrfacmn-supply = <&vreg_pmu_rfa_cmn>;
+		vddaon-supply = <&vreg_pmu_aon_0p59>;
+		vddbtcmx-supply = <&vreg_pmu_btcmx_0p85>;
+		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
+
+		max-speed = <3200000>;
+
+		qcom,local-bd-address-broken;
+	};
+};
+
+&ufs_mem_hc {
+	reset-gpios = <&tlmm 175 GPIO_ACTIVE_LOW>;
+
+	vcc-supply = <&vreg_l7b_2p96>;
+	vcc-max-microamp = <800000>;
+
+	vccq-supply = <&vreg_l9b_1p2>;
+	vccq-max-microamp = <900000>;
+
+	status = "okay";
+};
+
+&ufs_mem_phy {
+	vdda-phy-supply = <&vdd_a_ufs_0_core>;
+	vdda-pll-supply = <&vdd_a_ufs_0_1p2>;
+
+	status = "okay";
+};
+
+&usb_1 {
+	/* USB 2.0 only */
+	qcom,select-utmi-as-pipe-clk;
+	maximum-speed = "high-speed";
+
+	/* Remove USB3 phy */
+	phys = <&usb_1_hsphy>;
+	phy-names = "usb2-phy";
+
+	status = "okay";
+};
+
+&usb_1_dwc3_hs {
+	remote-endpoint = <&pmic_glink_hs_in>;
+};
+
+&usb_1_hsphy {
+	vdda-pll-supply = <&vdd_a_usbhs_core>;
+	vdda18-supply = <&vdd_a_usbhs_1p8>;
+	vdda33-supply = <&vdd_a_usbhs_3p1>;
+
+	status = "okay";
+};
+
+&venus {
+	firmware-name = "qcom/sm7325/motorola/dubai/vpu20_1v.mbn";
+
+	status = "okay";
+};
+
+&wifi {
+	qcom,calibration-variant = "Motorola_dubai";
+
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/qcom/sm8450-hdk.dts b/arch/arm64/boot/dts/qcom/sm8450-hdk.dts
index 268ae0cd642a7..b37998cd9a2c3 100644
--- a/arch/arm64/boot/dts/qcom/sm8450-hdk.dts
+++ b/arch/arm64/boot/dts/qcom/sm8450-hdk.dts
@@ -1172,7 +1172,7 @@ bluetooth {
 		vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>;
 		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
 		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/qcom/sm8450.dtsi b/arch/arm64/boot/dts/qcom/sm8450.dtsi
index 03bf30b53f289..e0c37ce3042a2 100644
--- a/arch/arm64/boot/dts/qcom/sm8450.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8450.dtsi
@@ -5373,7 +5373,7 @@ ufs_mem_phy: phy@1d87000 {
 		ice: crypto@1d88000 {
 			compatible = "qcom,sm8450-inline-crypto-engine",
 				     "qcom,inline-crypto-engine";
-			reg = <0 0x01d88000 0 0x8000>;
+			reg = <0 0x01d88000 0 0x18000>;
 			clocks = <&gcc GCC_UFS_PHY_ICE_CORE_CLK>;
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/sm8550.dtsi b/arch/arm64/boot/dts/qcom/sm8550.dtsi
index 912525e9bca6f..b7a7c49db077b 100644
--- a/arch/arm64/boot/dts/qcom/sm8550.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8550.dtsi
@@ -17,6 +17,7 @@
 #include <dt-bindings/interrupt-controller/arm-gic.h>
 #include <dt-bindings/interconnect/qcom,icc.h>
 #include <dt-bindings/interconnect/qcom,sm8550-rpmh.h>
+#include <dt-bindings/interconnect/qcom,osm-l3.h>
 #include <dt-bindings/mailbox/qcom-ipcc.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
@@ -78,6 +79,13 @@ cpu0: cpu@0 {
 			qcom,freq-domain = <&cpufreq_hw 0>;
 			capacity-dmips-mhz = <326>;
 			dynamic-power-coefficient = <251>;
+			operating-points-v2 = <&cpu0_opp_table>;
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
 			#cooling-cells = <2>;
 			l2_0: l2-cache {
 				compatible = "cache";
@@ -104,6 +112,13 @@ cpu1: cpu@100 {
 			qcom,freq-domain = <&cpufreq_hw 0>;
 			capacity-dmips-mhz = <326>;
 			dynamic-power-coefficient = <251>;
+			operating-points-v2 = <&cpu0_opp_table>;
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
 			#cooling-cells = <2>;
 			l2_100: l2-cache {
 				compatible = "cache";
@@ -125,6 +140,13 @@ cpu2: cpu@200 {
 			qcom,freq-domain = <&cpufreq_hw 0>;
 			capacity-dmips-mhz = <326>;
 			dynamic-power-coefficient = <251>;
+			operating-points-v2 = <&cpu0_opp_table>;
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
 			#cooling-cells = <2>;
 			l2_200: l2-cache {
 				compatible = "cache";
@@ -146,6 +168,13 @@ cpu3: cpu@300 {
 			qcom,freq-domain = <&cpufreq_hw 1>;
 			capacity-dmips-mhz = <693>;
 			dynamic-power-coefficient = <447>;
+			operating-points-v2 = <&cpu3_opp_table>;
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
 			#cooling-cells = <2>;
 			l2_300: l2-cache {
 				compatible = "cache";
@@ -167,6 +196,13 @@ cpu4: cpu@400 {
 			qcom,freq-domain = <&cpufreq_hw 1>;
 			capacity-dmips-mhz = <693>;
 			dynamic-power-coefficient = <447>;
+			operating-points-v2 = <&cpu3_opp_table>;
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
 			#cooling-cells = <2>;
 			l2_400: l2-cache {
 				compatible = "cache";
@@ -188,6 +224,13 @@ cpu5: cpu@500 {
 			qcom,freq-domain = <&cpufreq_hw 1>;
 			capacity-dmips-mhz = <693>;
 			dynamic-power-coefficient = <447>;
+			operating-points-v2 = <&cpu3_opp_table>;
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
 			#cooling-cells = <2>;
 			l2_500: l2-cache {
 				compatible = "cache";
@@ -209,6 +252,13 @@ cpu6: cpu@600 {
 			qcom,freq-domain = <&cpufreq_hw 1>;
 			capacity-dmips-mhz = <693>;
 			dynamic-power-coefficient = <447>;
+			operating-points-v2 = <&cpu3_opp_table>;
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
 			#cooling-cells = <2>;
 			l2_600: l2-cache {
 				compatible = "cache";
@@ -230,6 +280,13 @@ cpu7: cpu@700 {
 			qcom,freq-domain = <&cpufreq_hw 2>;
 			capacity-dmips-mhz = <1024>;
 			dynamic-power-coefficient = <1057>;
+			operating-points-v2 = <&cpu7_opp_table>;
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &gem_noc SLAVE_LLCC QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&mc_virt MASTER_LLCC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&epss_l3 MASTER_EPSS_L3_APPS
+					 &epss_l3 SLAVE_EPSS_L3_SHARED>;
 			#cooling-cells = <2>;
 			l2_700: l2-cache {
 				compatible = "cache";
@@ -397,6 +454,306 @@ memory@a0000000 {
 		reg = <0 0xa0000000 0 0>;
 	};
 
+	cpu0_opp_table: opp-table-cpu0 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-307200000 {
+			opp-hz = /bits/ 64 <307200000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (307200 * 32)>;
+		};
+
+		opp-441600000 {
+			opp-hz = /bits/ 64 <441600000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (384000 * 32)>;
+		};
+
+		opp-556800000 {
+			opp-hz = /bits/ 64 <556800000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (499200 * 32)>;
+		};
+
+		opp-672000000 {
+			opp-hz = /bits/ 64 <672000000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (499200 * 32)>;
+		};
+
+		opp-787200000 {
+			opp-hz = /bits/ 64 <787200000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (729600 * 32)>;
+		};
+
+		opp-902400000 {
+			opp-hz = /bits/ 64 <902400000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (844800 * 32)>;
+		};
+
+		opp-1017600000 {
+			opp-hz = /bits/ 64 <1017600000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (940800 * 32)>;
+		};
+
+		opp-1113600000 {
+			opp-hz = /bits/ 64 <1113600000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (1056000 * 32)>;
+		};
+
+		opp-1228800000 {
+			opp-hz = /bits/ 64 <1228800000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (1152000 * 32)>;
+		};
+
+		opp-1344000000 {
+			opp-hz = /bits/ 64 <1344000000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (1267200 * 32)>;
+		};
+
+		opp-1459200000 {
+			opp-hz = /bits/ 64 <1459200000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (1267200 * 32)>;
+		};
+
+		opp-1555200000 {
+			opp-hz = /bits/ 64 <1555200000>;
+			opp-peak-kBps = <(466000 * 16) (768000 * 4) (1478400 * 32)>;
+		};
+
+		opp-1670400000 {
+			opp-hz = /bits/ 64 <1670400000>;
+			opp-peak-kBps = <(466000 * 16) (768000 * 4) (1478400 * 32)>;
+		};
+
+		opp-1785600000 {
+			opp-hz = /bits/ 64 <1785600000>;
+			opp-peak-kBps = <(466000 * 16) (768000 * 4) (1478400 * 32)>;
+		};
+
+		opp-1900800000 {
+			opp-hz = /bits/ 64 <1900800000>;
+			opp-peak-kBps = <(466000 * 16) (768000 * 4) (1689600 * 32)>;
+		};
+
+		opp-2016000000 {
+			opp-hz = /bits/ 64 <2016000000>;
+			opp-peak-kBps = <(600000 * 16) (1555000 * 4) (1804800 * 32)>;
+		};
+	};
+
+	cpu3_opp_table: opp-table-cpu3 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-499200000 {
+			opp-hz = /bits/ 64 <499200000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (307200 * 32)>;
+		};
+
+		opp-614400000 {
+			opp-hz = /bits/ 64 <614400000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (499200 * 32)>;
+		};
+
+		opp-729600000 {
+			opp-hz = /bits/ 64 <729600000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (499200 * 32)>;
+		};
+
+		opp-844800000 {
+			opp-hz = /bits/ 64 <844800000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (499200 * 32)>;
+		};
+
+		opp-940800000 {
+			opp-hz = /bits/ 64 <940800000>;
+			opp-peak-kBps = <(300000 * 16) (768000 * 4) (729600 * 32)>;
+		};
+
+		opp-1056000000 {
+			opp-hz = /bits/ 64 <1056000000>;
+			opp-peak-kBps = <(300000 * 16) (768000 * 4) (729600 * 32)>;
+		};
+
+		opp-1171200000 {
+			opp-hz = /bits/ 64 <1171200000>;
+			opp-peak-kBps = <(466000 * 16) (1555000 * 4) (940800 * 32)>;
+		};
+
+		opp-1286400000 {
+			opp-hz = /bits/ 64 <1286400000>;
+			opp-peak-kBps = <(466000 * 16) (1555000 * 4) (940800 * 32)>;
+		};
+
+		opp-1401600000 {
+			opp-hz = /bits/ 64 <1401600000>;
+			opp-peak-kBps = <(600000 * 16) (1708000 * 4) (1056000 * 32)>;
+		};
+
+		opp-1536000000 {
+			opp-hz = /bits/ 64 <1536000000>;
+			opp-peak-kBps = <(600000 * 16) (1708000 * 4) (1056000 * 32)>;
+		};
+
+		opp-1651200000 {
+			opp-hz = /bits/ 64 <1651200000>;
+			opp-peak-kBps = <(600000 * 16) (1708000 * 4) (1267200 * 32)>;
+		};
+
+		opp-1785600000 {
+			opp-hz = /bits/ 64 <1785600000>;
+			opp-peak-kBps = <(600000 * 16) (1708000 * 4) (1267200 * 32)>;
+		};
+
+		opp-1920000000 {
+			opp-hz = /bits/ 64 <1920000000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1267200 * 32)>;
+		};
+
+		opp-2054400000 {
+			opp-hz = /bits/ 64 <2054400000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2188800000 {
+			opp-hz = /bits/ 64 <2188800000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2323200000 {
+			opp-hz = /bits/ 64 <2323200000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2457600000 {
+			opp-hz = /bits/ 64 <2457600000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2592000000 {
+			opp-hz = /bits/ 64 <2592000000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2707200000 {
+			opp-hz = /bits/ 64 <2707200000>;
+			opp-peak-kBps = <(933000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2803200000 {
+			opp-hz = /bits/ 64 <2803200000>;
+			opp-peak-kBps = <(933000 * 16) (3686000 * 4) (1689600 * 32)>;
+		};
+	};
+
+	cpu7_opp_table: opp-table-cpu7 {
+		compatible = "operating-points-v2";
+		opp-shared;
+
+		opp-595200000 {
+			opp-hz = /bits/ 64 <595200000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (307200 * 32)>;
+		};
+
+		opp-729600000 {
+			opp-hz = /bits/ 64 <729600000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (499200 * 32)>;
+		};
+
+		opp-864000000 {
+			opp-hz = /bits/ 64 <864000000>;
+			opp-peak-kBps = <(300000 * 16) (547000 * 4) (499200 * 32)>;
+		};
+
+		opp-998400000 {
+			opp-hz = /bits/ 64 <998400000>;
+			opp-peak-kBps = <(300000 * 16) (768000 * 4) (729600 * 32)>;
+		};
+
+		opp-1132800000 {
+			opp-hz = /bits/ 64 <1132800000>;
+			opp-peak-kBps = <(300000 * 16) (768000 * 4) (729600 * 32)>;
+		};
+
+		opp-1248000000 {
+			opp-hz = /bits/ 64 <1248000000>;
+			opp-peak-kBps = <(466000 * 16) (1555000 * 4) (940800 * 32)>;
+		};
+
+		opp-1363200000 {
+			opp-hz = /bits/ 64 <1363200000>;
+			opp-peak-kBps = <(466000 * 16) (1555000 * 4) (940800 * 32)>;
+		};
+
+		opp-1478400000 {
+			opp-hz = /bits/ 64 <1478400000>;
+			opp-peak-kBps = <(600000 * 16) (1708000 * 4) (1056000 * 32)>;
+		};
+
+		opp-1593600000 {
+			opp-hz = /bits/ 64 <1593600000>;
+			opp-peak-kBps = <(600000 * 16) (1708000 * 4) (1056000 * 32)>;
+		};
+
+		opp-1708800000 {
+			opp-hz = /bits/ 64 <1708800000>;
+			opp-peak-kBps = <(600000 * 16) (1708000 * 4) (1267200 * 32)>;
+		};
+
+		opp-1843200000 {
+			opp-hz = /bits/ 64 <1843200000>;
+			opp-peak-kBps = <(600000 * 16) (1708000 * 4) (1267200 * 32)>;
+		};
+
+		opp-1977600000 {
+			opp-hz = /bits/ 64 <1977600000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1267200 * 32)>;
+		};
+
+		opp-2092800000 {
+			opp-hz = /bits/ 64 <2092800000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2227200000 {
+			opp-hz = /bits/ 64 <2227200000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2342400000 {
+			opp-hz = /bits/ 64 <2342400000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2476800000 {
+			opp-hz = /bits/ 64 <2476800000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2592000000 {
+			opp-hz = /bits/ 64 <2592000000>;
+			opp-peak-kBps = <(806000 * 16) (2736000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2726400000 {
+			opp-hz = /bits/ 64 <2726400000>;
+			opp-peak-kBps = <(933000 * 16) (3686000 * 4) (1478400 * 32)>;
+		};
+
+		opp-2841600000 {
+			opp-hz = /bits/ 64 <2841600000>;
+			opp-peak-kBps = <(933000 * 16) (3686000 * 4) (1689600 * 32)>;
+		};
+
+		opp-2956800000 {
+			opp-hz = /bits/ 64 <2956800000>;
+			opp-peak-kBps = <(933000 * 16) (3686000 * 4) (1689600 * 32)>;
+		};
+
+		opp-3187200000 {
+			opp-hz = /bits/ 64 <3187200000>;
+			opp-peak-kBps = <(933000 * 16) (3686000 * 4) (1689600 * 32)>;
+		};
+	};
+
 	pmu-a510 {
 		compatible = "arm,cortex-a510-pmu";
 		interrupts = <GIC_PPI 7 IRQ_TYPE_LEVEL_LOW &ppi_cluster0>;
@@ -5482,6 +5839,16 @@ rpmhpd_opp_turbo_l1: opp-416 {
 			};
 		};
 
+		epss_l3: interconnect@17d90000 {
+			compatible = "qcom,sm8550-epss-l3", "qcom,epss-l3";
+			reg = <0 0x17d90000 0 0x1000>;
+
+			clocks = <&bi_tcxo_div2>, <&gcc GCC_GPLL0>;
+			clock-names = "xo", "alternate";
+
+			#interconnect-cells = <1>;
+		};
+
 		cpufreq_hw: cpufreq@17d91000 {
 			compatible = "qcom,sm8550-cpufreq-epss", "qcom,cpufreq-epss";
 			reg = <0 0x17d91000 0 0x1000>,
diff --git a/arch/arm64/boot/dts/qcom/sm8750.dtsi b/arch/arm64/boot/dts/qcom/sm8750.dtsi
index 18fb52c14acd7..70830cb49e73a 100644
--- a/arch/arm64/boot/dts/qcom/sm8750.dtsi
+++ b/arch/arm64/boot/dts/qcom/sm8750.dtsi
@@ -5,6 +5,8 @@
 
 #include <dt-bindings/clock/qcom,dsi-phy-28nm.h>
 #include <dt-bindings/clock/qcom,rpmh.h>
+#include <dt-bindings/clock/qcom,sm8750-cambistmclkcc.h>
+#include <dt-bindings/clock/qcom,sm8750-camcc.h>
 #include <dt-bindings/clock/qcom,sm8750-dispcc.h>
 #include <dt-bindings/clock/qcom,sm8750-gcc.h>
 #include <dt-bindings/clock/qcom,sm8750-tcsr.h>
@@ -42,6 +44,7 @@ cpu0: cpu@0 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd0>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 
 			l2_0: l2-cache {
 				compatible = "cache";
@@ -58,6 +61,7 @@ cpu1: cpu@100 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd1>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu2: cpu@200 {
@@ -68,6 +72,7 @@ cpu2: cpu@200 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd2>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu3: cpu@300 {
@@ -78,6 +83,7 @@ cpu3: cpu@300 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd3>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu4: cpu@400 {
@@ -88,6 +94,7 @@ cpu4: cpu@400 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd4>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu5: cpu@500 {
@@ -98,6 +105,7 @@ cpu5: cpu@500 {
 			next-level-cache = <&l2_0>;
 			power-domains = <&cpu_pd5>, <&scmi_dvfs 0>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu6: cpu@10000 {
@@ -108,6 +116,7 @@ cpu6: cpu@10000 {
 			next-level-cache = <&l2_1>;
 			power-domains = <&cpu_pd6>, <&scmi_dvfs 1>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 
 			l2_1: l2-cache {
 				compatible = "cache";
@@ -124,6 +133,7 @@ cpu7: cpu@10100 {
 			next-level-cache = <&l2_1>;
 			power-domains = <&cpu_pd7>, <&scmi_dvfs 1>;
 			power-domain-names = "psci", "perf";
+			#cooling-cells = <2>;
 		};
 
 		cpu-map {
@@ -2074,6 +2084,23 @@ aggre2_noc: interconnect@1700000 {
 			clocks = <&rpmhcc RPMH_IPA_CLK>;
 		};
 
+		cambistmclkcc: clock-controller@1760000 {
+		       compatible = "qcom,sm8750-cambistmclkcc";
+		       reg = <0x0 0x1760000 0x0 0x6000>;
+		       clocks = <&gcc GCC_CAM_BIST_MCLK_AHB_CLK>,
+				<&bi_tcxo_div2>,
+				<&bi_tcxo_ao_div2>,
+				<&sleep_clk>;
+			power-domains = <&rpmhpd RPMHPD_MMCX>,
+					<&rpmhpd RPMHPD_MX>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		mmss_noc: interconnect@1780000 {
 			compatible = "qcom,sm8750-mmss-noc";
 			reg = <0x0 0x01780000 0x0 0x5b800>;
@@ -3313,7 +3340,7 @@ mdss_dsi1_phy: phy@ae97000 {
 					    "dsi_pll";
 
 				clocks = <&dispcc DISP_CC_MDSS_AHB_CLK>,
-					 <&rpmhcc RPMH_CXO_CLK>;
+					 <&bi_tcxo_div2>;
 				clock-names = "iface",
 					      "ref";
 
@@ -3436,6 +3463,23 @@ dispcc: clock-controller@af00000 {
 			#power-domain-cells = <1>;
 		};
 
+		camcc: clock-controller@ade0000 {
+			compatible = "qcom,sm8750-camcc";
+			reg = <0x0 0xade0000 0x0 0x20000>;
+			clocks = <&gcc GCC_CAMERA_AHB_CLK>,
+				 <&bi_tcxo_div2>,
+				 <&bi_tcxo_ao_div2>,
+				 <&sleep_clk>;
+			power-domains = <&rpmhpd RPMHPD_MMCX>,
+					<&rpmhpd RPMHPD_MXC>;
+			required-opps = <&rpmhpd_opp_low_svs>,
+					<&rpmhpd_opp_low_svs>;
+
+			#clock-cells = <1>;
+			#reset-cells = <1>;
+			#power-domain-cells = <1>;
+		};
+
 		pdc: interrupt-controller@b220000 {
 			compatible = "qcom,sm8750-pdc", "qcom,pdc";
 			reg = <0x0 0x0b220000 0x0 0x10000>, <0x0 0x164400f0 0x0 0x64>;
@@ -5202,10 +5246,10 @@ pcie0: pcie@1c00000 {
 
 			#interrupt-cells = <1>;
 			interrupt-map-mask = <0 0 0 0x7>;
-			interrupt-map = <0 0 0 1 &intc 0 0 0 149 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 2 &intc 0 0 0 150 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 3 &intc 0 0 0 151 IRQ_TYPE_LEVEL_HIGH>,
-					<0 0 0 4 &intc 0 0 0 152 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-map = <0 0 0 1 &intc 0 0 GIC_SPI 149 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 2 &intc 0 0 GIC_SPI 150 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 3 &intc 0 0 GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH>,
+					<0 0 0 4 &intc 0 0 GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH>;
 
 			clocks = <&gcc GCC_PCIE_0_AUX_CLK>,
 				 <&gcc GCC_PCIE_0_CFG_AHB_CLK>,
diff --git a/arch/arm64/boot/dts/qcom/talos-evk.dts b/arch/arm64/boot/dts/qcom/talos-evk.dts
index af100e22beeec..b7f514fbc7b2d 100644
--- a/arch/arm64/boot/dts/qcom/talos-evk.dts
+++ b/arch/arm64/boot/dts/qcom/talos-evk.dts
@@ -5,6 +5,7 @@
 /dts-v1/;
 
 #include "talos-evk-som.dtsi"
+#include <dt-bindings/sound/qcom,q6afe.h>
 
 / {
 	model = "Qualcomm QCS615 IQ 615 EVK";
@@ -40,6 +41,46 @@ hdmi_con_out: endpoint {
 		};
 	};
 
+	sound {
+		compatible = "qcom,qcs615-sndcard";
+		model = "TALOS-EVK";
+
+		pinctrl-0 = <&mi2s1_pins>, <&mi2s_mclk>;
+		pinctrl-names = "default";
+
+		pri-mi2s-capture-dai-link {
+			link-name = "Primary MI2S Capture";
+
+			codec {
+				sound-dai = <&codec_da7212>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai PRIMARY_MI2S_TX>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+
+		pri-mi2s-playback-dai-link {
+			link-name = "Primary MI2S Playback";
+
+			codec {
+				sound-dai = <&codec_da7212>;
+			};
+
+			cpu {
+				sound-dai = <&q6apmbedai PRIMARY_MI2S_RX>;
+			};
+
+			platform {
+				sound-dai = <&q6apm>;
+			};
+		};
+	};
+
 	vreg_v1p8_out: regulator-v1p8-out {
 		compatible = "regulator-fixed";
 		regulator-name = "vreg-v1p8-out";
@@ -109,6 +150,21 @@ adv7535_out: endpoint {
 	};
 };
 
+&i2c5 {
+	status = "okay";
+
+	codec_da7212: codec@1a {
+		compatible = "dlg,da7212";
+		reg = <0x1a>;
+		#sound-dai-cells = <0>;
+		clocks = <&q6prmcc LPASS_CLK_ID_MCLK_2 LPASS_CLK_ATTRIBUTE_COUPLE_NO>;
+		clock-names = "mclk";
+		VDDA-supply = <&vreg_v1p8_out>;
+		VDDIO-supply = <&vreg_v1p8_out>;
+		VDDMIC-supply = <&vreg_v3p3_out>;
+	};
+};
+
 &mdss_dsi0_out {
 	remote-endpoint = <&adv7535_in>;
 	data-lanes = <0 1 2 3>;
diff --git a/arch/arm64/boot/dts/qcom/talos.dtsi b/arch/arm64/boot/dts/qcom/talos.dtsi
index ff5afbfce2a47..fb1bbc51bb8a4 100644
--- a/arch/arm64/boot/dts/qcom/talos.dtsi
+++ b/arch/arm64/boot/dts/qcom/talos.dtsi
@@ -18,6 +18,7 @@
 #include <dt-bindings/phy/phy-qcom-qmp.h>
 #include <dt-bindings/power/qcom-rpmpd.h>
 #include <dt-bindings/power/qcom,rpmhpd.h>
+#include <dt-bindings/soc/qcom,gpr.h>
 #include <dt-bindings/soc/qcom,rpmh-rsc.h>
 #include <dt-bindings/thermal/thermal.h>
 
@@ -531,6 +532,25 @@ cdsp_smp2p_in: slave-kernel {
 
 	};
 
+	qspi_opp_table: opp-table-qspi {
+		compatible = "operating-points-v2";
+
+		opp-60000000 {
+			opp-hz = /bits/ 64 <60000000>;
+			required-opps = <&rpmhpd_opp_low_svs>;
+		};
+
+		opp-133250000 {
+			opp-hz = /bits/ 64 <133250000>;
+			required-opps = <&rpmhpd_opp_svs>;
+		};
+
+		opp-266500000 {
+			opp-hz = /bits/ 64 <266500000>;
+			required-opps = <&rpmhpd_opp_nom>;
+		};
+	};
+
 	qup_opp_table: opp-table-qup {
 		compatible = "operating-points-v2";
 
@@ -1610,6 +1630,48 @@ cci_i2c1_default: cci-i2c1-default-state {
 				bias-pull-up;
 			};
 
+			qspi_cs0: qspi-cs0-state {
+				pins = "gpio44";
+				function = "qspi";
+				bias-disable;
+				drive-strength = <6>;
+			};
+
+			qspi_data0123: qspi-data0123-state {
+				pins = "gpio45", "gpio46", "gpio47", "gpio49";
+				function = "qspi";
+				bias-pull-down;
+				drive-strength = <6>;
+			};
+
+			qspi_clk: qspi-clk-state {
+				pins = "gpio48";
+				function = "qspi";
+				bias-pull-down;
+				drive-strength = <6>;
+			};
+
+			qspi_cs1: qspi-cs1-state {
+				pins = "gpio50";
+				function = "qspi";
+				bias-pull-down;
+				drive-strength = <6>;
+			};
+
+			mi2s1_pins: mi2s1-state {
+				pins = "gpio108", "gpio109", "gpio110", "gpio111";
+				function = "mi2s_1";
+				drive-strength = <8>;
+				bias-disable;
+			};
+
+			mi2s_mclk: mi2s-mclk-state {
+				pins = "gpio122";
+				function = "mclk2";
+				drive-strength = <8>;
+				bias-disable;
+			};
+
 			qup_i2c1_data_clk: qup-i2c1-data-clk-state {
 				pins = "gpio4", "gpio5";
 				function = "qup0";
@@ -3896,6 +3958,39 @@ opp-202000000 {
 			};
 		};
 
+		qspi: spi@88df000 {
+			compatible = "qcom,qcs615-qspi",
+				     "qcom,qspi-v1";
+			reg = <0x0 0x088df000 0x0 0x1000>;
+
+			interrupts = <GIC_SPI 82 IRQ_TYPE_LEVEL_HIGH 0>;
+
+			clocks = <&gcc GCC_QSPI_CNOC_PERIPH_AHB_CLK>,
+				 <&gcc GCC_QSPI_CORE_CLK>;
+			clock-names = "iface",
+				      "core";
+
+			interconnects = <&gem_noc MASTER_APPSS_PROC QCOM_ICC_TAG_ACTIVE_ONLY
+					 &config_noc SLAVE_QSPI QCOM_ICC_TAG_ACTIVE_ONLY>,
+					<&aggre1_noc MASTER_QSPI QCOM_ICC_TAG_ALWAYS
+					 &mc_virt SLAVE_EBI1 QCOM_ICC_TAG_ALWAYS>;
+			interconnect-names = "qspi-config",
+					     "qspi-memory";
+
+			power-domains = <&rpmhpd RPMHPD_CX>;
+			operating-points-v2 = <&qspi_opp_table>;
+
+			iommus = <&apps_smmu 0x160 0x0>;
+
+			pinctrl-0 = <&qspi_clk>, <&qspi_cs0>, <&qspi_data0123>;
+			pinctrl-names = "default";
+
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			status = "disabled";
+		};
+
 		dc_noc: interconnect@9160000 {
 			reg = <0x0 0x09160000 0x0 0x3200>;
 			compatible = "qcom,qcs615-dc-noc";
@@ -5132,6 +5227,45 @@ compute-cb@6 {
 						dma-coherent;
 					};
 				};
+
+				gpr: gpr {
+					compatible = "qcom,gpr";
+					qcom,glink-channels = "adsp_apps";
+					qcom,domain = <GPR_DOMAIN_ID_ADSP>;
+					qcom,intents = <512 20>;
+					#address-cells = <1>;
+					#size-cells = <0>;
+
+					q6apm: service@1 {
+						compatible = "qcom,q6apm";
+						reg = <GPR_APM_MODULE_IID>;
+						#sound-dai-cells = <0>;
+						qcom,protection-domain = "avs/audio",
+									 "msm/adsp/audio_pd";
+
+						q6apmbedai: bedais {
+							compatible = "qcom,q6apm-lpass-dais";
+							#sound-dai-cells = <1>;
+						};
+
+						q6apmdai: dais {
+							compatible = "qcom,q6apm-dais";
+							iommus = <&apps_smmu 0x1721 0x0>;
+						};
+					};
+
+					q6prm: service@2 {
+						compatible = "qcom,q6prm";
+						reg = <GPR_PRM_MODULE_IID>;
+						qcom,protection-domain = "avs/audio",
+									 "msm/adsp/audio_pd";
+
+						q6prmcc: clock-controller {
+							compatible = "qcom,q6prm-lpass-clocks";
+							#clock-cells = <2>;
+						};
+					};
+				};
 			};
 		};
 
diff --git a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
index 0d9a324cc6cc3..db291730130c7 100644
--- a/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-dell-thena.dtsi
@@ -982,12 +982,6 @@ &i2c8 {
 	status = "okay";
 };
 
-&i2c20 {
-	clock-frequency = <400000>;
-
-	status = "okay";
-};
-
 &lpass_tlmm {
 	spkr_01_sd_n_active: spkr-01-sd-n-active-state {
 		pins = "gpio12";
@@ -1308,6 +1302,7 @@ right_tweeter: speaker@0,1 {
 &tlmm {
 	gpio-reserved-ranges = <44 4>,  /* SPI11 (TPM) */
 			       <76 4>,  /* SPI19 (TZ Protected) */
+			       <80 2>,  /* I2C20 (Battery SMBus) */
 			       <238 1>; /* UFS Reset */
 
 	cam_rgb_default: cam-rgb-default-state {
diff --git a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi
index b773a4976d1b7..02708f23a8657 100644
--- a/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1-hp-omnibook-x14.dtsi
@@ -1452,7 +1452,7 @@ bluetooth {
 		vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>;
 		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
 		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
 	};
 };
 
diff --git a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi
index 22470a97e1e39..7f5860d2b6cc1 100644
--- a/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi
+++ b/arch/arm64/boot/dts/qcom/x1p42100-asus-zenbook-a14.dtsi
@@ -128,7 +128,7 @@ bluetooth {
 		vddbtcmx-supply = <&vreg_pmu_btcmx_0p8>;
 		vddrfa0p8-supply = <&vreg_pmu_rfa_0p8>;
 		vddrfa1p2-supply = <&vreg_pmu_rfa_1p2>;
-		vddrfa1p8-supply = <&vreg_pmu_rfa_1p7>;
+		vddrfa1p7-supply = <&vreg_pmu_rfa_1p7>;
 		vddrfacmn-supply = <&vreg_pmu_rfa_cmn_0p8>;
 		vddwlcx-supply = <&vreg_pmu_wlcx_0p8>;
 		vddwlmx-supply = <&vreg_pmu_wlmx_0p8>;
diff --git a/arch/arm64/boot/dts/realtek/kent.dtsi b/arch/arm64/boot/dts/realtek/kent.dtsi
index ae006ce244205..8d4293cd4c036 100644
--- a/arch/arm64/boot/dts/realtek/kent.dtsi
+++ b/arch/arm64/boot/dts/realtek/kent.dtsi
@@ -150,6 +150,26 @@ uart0: serial@7800 {
 				reg-shift = <2>;
 				status = "disabled";
 			};
+
+			iso_pinctrl: pinctrl@4e000 {
+				compatible = "realtek,rtd1625-iso-pinctrl";
+				reg = <0x4e000 0x1a4>;
+			};
+
+			main2_pinctrl: pinctrl@4f200 {
+				compatible = "realtek,rtd1625-main2-pinctrl";
+				reg = <0x4f200 0x50>;
+			};
+
+			isom_pinctrl: pinctrl@146200 {
+				compatible = "realtek,rtd1625-isom-pinctrl";
+				reg = <0x146200 0x34>;
+			};
+
+			ve4_pinctrl: pinctrl@14e000 {
+				compatible = "realtek,rtd1625-ve4-pinctrl";
+				reg = <0x14e000 0x84>;
+			};
 		};
 
 		gic: interrupt-controller@ff100000 {
diff --git a/arch/arm64/boot/dts/renesas/draak-ebisu-panel-aa104xd12.dtso b/arch/arm64/boot/dts/renesas/draak-ebisu-panel-aa104xd12.dtso
index 258f8668ca361..90767d74e21b2 100644
--- a/arch/arm64/boot/dts/renesas/draak-ebisu-panel-aa104xd12.dtso
+++ b/arch/arm64/boot/dts/renesas/draak-ebisu-panel-aa104xd12.dtso
@@ -27,7 +27,12 @@ &lvds1 {
 	status = "okay";
 
 	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
 		port@1 {
+			reg = <1>;
+
 			lvds1_out: endpoint {
 				remote-endpoint = <&panel_in>;
 			};
diff --git a/arch/arm64/boot/dts/renesas/ebisu.dtsi b/arch/arm64/boot/dts/renesas/ebisu.dtsi
index aaedb1fb51aed..ba564aa098661 100644
--- a/arch/arm64/boot/dts/renesas/ebisu.dtsi
+++ b/arch/arm64/boot/dts/renesas/ebisu.dtsi
@@ -186,23 +186,6 @@ reg_12p0v: regulator-12p0v {
 		regulator-always-on;
 	};
 
-	rsnd_ak4613: sound {
-		compatible = "simple-audio-card";
-
-		simple-audio-card,name = "rsnd-ak4613";
-		simple-audio-card,format = "left_j";
-		simple-audio-card,bitclock-master = <&sndcpu>;
-		simple-audio-card,frame-master = <&sndcpu>;
-
-		sndcodec: simple-audio-card,codec {
-			sound-dai = <&ak4613>;
-		};
-
-		sndcpu: simple-audio-card,cpu {
-			sound-dai = <&rcar_sound>;
-		};
-	};
-
 	vbus0_usb2: regulator-vbus0-usb2 {
 		compatible = "regulator-fixed";
 
@@ -260,6 +243,23 @@ vccq_sdhi1: regulator-vccq-sdhi1 {
 		states = <3300000 1>, <1800000 0>;
 	};
 
+	rsnd_ak4613: sound {
+		compatible = "simple-audio-card";
+
+		simple-audio-card,name = "rsnd-ak4613";
+		simple-audio-card,format = "left_j";
+		simple-audio-card,bitclock-master = <&sndcpu>;
+		simple-audio-card,frame-master = <&sndcpu>;
+
+		sndcodec: simple-audio-card,codec {
+			sound-dai = <&ak4613>;
+		};
+
+		sndcpu: simple-audio-card,cpu {
+			sound-dai = <&rcar_sound>;
+		};
+	};
+
 	vga {
 		compatible = "vga-connector";
 
diff --git a/arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi b/arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi
index 2edb5cb3407b4..792a4aa8f4a9d 100644
--- a/arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi
+++ b/arch/arm64/boot/dts/renesas/gray-hawk-single.dtsi
@@ -208,11 +208,11 @@ mdio {
 
 		avb0_phy: ethernet-phy@0 {
 			compatible = "ethernet-phy-id0022.1622",
-				"ethernet-phy-ieee802.3-c22";
-				rxc-skew-ps = <1500>;
-				reg = <0>;
-				interrupts-extended = <&gpio7 5 IRQ_TYPE_LEVEL_LOW>;
-				reset-gpios = <&gpio7 10 GPIO_ACTIVE_LOW>;
+				     "ethernet-phy-ieee802.3-c22";
+			rxc-skew-ps = <1500>;
+			reg = <0>;
+			interrupts-extended = <&gpio7 5 IRQ_TYPE_LEVEL_LOW>;
+			reset-gpios = <&gpio7 10 GPIO_ACTIVE_LOW>;
 		};
 	};
 };
diff --git a/arch/arm64/boot/dts/renesas/r8a77960-salvator-x.dts b/arch/arm64/boot/dts/renesas/r8a77960-salvator-x.dts
index d5543f26c4720..0d5c754a7f0ec 100644
--- a/arch/arm64/boot/dts/renesas/r8a77960-salvator-x.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77960-salvator-x.dts
@@ -35,3 +35,7 @@ &du {
 	clock-names = "du.0", "du.1", "du.2",
 		      "dclkin.0", "dclkin.1", "dclkin.2";
 };
+
+&gpu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77960-salvator-xs.dts b/arch/arm64/boot/dts/renesas/r8a77960-salvator-xs.dts
index 9ebb47b6bf2dc..115cc47bb0724 100644
--- a/arch/arm64/boot/dts/renesas/r8a77960-salvator-xs.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77960-salvator-xs.dts
@@ -35,3 +35,7 @@ &du {
 	clock-names = "du.0", "du.1", "du.2",
 		      "dclkin.0", "dclkin.1", "dclkin.2";
 };
+
+&gpu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77960-ulcb.dts b/arch/arm64/boot/dts/renesas/r8a77960-ulcb.dts
index 4bfeb1df0488d..c3282593346bc 100644
--- a/arch/arm64/boot/dts/renesas/r8a77960-ulcb.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77960-ulcb.dts
@@ -36,3 +36,7 @@ &du {
 	clock-names = "du.0", "du.1", "du.2",
 		      "dclkin.0", "dclkin.1", "dclkin.2";
 };
+
+&gpu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77961-salvator-xs.dts b/arch/arm64/boot/dts/renesas/r8a77961-salvator-xs.dts
index c7f14177f7b95..b35de49406a01 100644
--- a/arch/arm64/boot/dts/renesas/r8a77961-salvator-xs.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77961-salvator-xs.dts
@@ -40,3 +40,7 @@ &du {
 	clock-names = "du.0", "du.1", "du.2",
 		      "dclkin.0", "dclkin.1", "dclkin.2";
 };
+
+&gpu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a77961-ulcb.dts b/arch/arm64/boot/dts/renesas/r8a77961-ulcb.dts
index 70cf926667a6e..0c0806cec6989 100644
--- a/arch/arm64/boot/dts/renesas/r8a77961-ulcb.dts
+++ b/arch/arm64/boot/dts/renesas/r8a77961-ulcb.dts
@@ -40,3 +40,7 @@ &du {
 	clock-names = "du.0", "du.1", "du.2",
 		      "dclkin.0", "dclkin.1", "dclkin.2";
 };
+
+&gpu {
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/r8a78000.dtsi b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
index 3e1c98903cea0..3ec1b53d27828 100644
--- a/arch/arm64/boot/dts/renesas/r8a78000.dtsi
+++ b/arch/arm64/boot/dts/renesas/r8a78000.dtsi
@@ -699,7 +699,7 @@ scif0: serial@c0700000 {
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc0700000 0 0x40>;
 			interrupts = <GIC_ESPI 10 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
+			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
 		};
@@ -709,7 +709,7 @@ scif1: serial@c0704000 {
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc0704000 0 0x40>;
 			interrupts = <GIC_ESPI 11 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
+			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
 		};
@@ -719,7 +719,7 @@ scif3: serial@c0708000 {
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc0708000 0 0x40>;
 			interrupts = <GIC_ESPI 12 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
+			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
 		};
@@ -729,7 +729,7 @@ scif4: serial@c070c000 {
 				     "renesas,rcar-gen5-scif", "renesas,scif";
 			reg = <0 0xc070c000 0 0x40>;
 			interrupts = <GIC_ESPI 13 IRQ_TYPE_LEVEL_HIGH>;
-			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd16>, <&scif_clk>;
+			clocks = <&dummy_clk_sgasyncd16>, <&dummy_clk_sgasyncd4>, <&scif_clk>;
 			clock-names = "fck", "brg_int", "scif_clk";
 			status = "disabled";
 		};
diff --git a/arch/arm64/boot/dts/renesas/r9a07g043.dtsi b/arch/arm64/boot/dts/renesas/r9a07g043.dtsi
index 593c66b27ad12..ded4f1f11d605 100644
--- a/arch/arm64/boot/dts/renesas/r9a07g043.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a07g043.dtsi
@@ -604,6 +604,7 @@ pinctrl: pinctrl@11030000 {
 			resets = <&cpg R9A07G043_GPIO_RSTN>,
 				 <&cpg R9A07G043_GPIO_PORT_RESETN>,
 				 <&cpg R9A07G043_GPIO_SPARE_RESETN>;
+			reset-names = "rstn", "port", "spare";
 		};
 
 		dmac: dma-controller@11820000 {
diff --git a/arch/arm64/boot/dts/renesas/r9a07g044.dtsi b/arch/arm64/boot/dts/renesas/r9a07g044.dtsi
index 29273da819951..cb0c9550aa033 100644
--- a/arch/arm64/boot/dts/renesas/r9a07g044.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a07g044.dtsi
@@ -1017,6 +1017,7 @@ pinctrl: pinctrl@11030000 {
 			resets = <&cpg R9A07G044_GPIO_RSTN>,
 				 <&cpg R9A07G044_GPIO_PORT_RESETN>,
 				 <&cpg R9A07G044_GPIO_SPARE_RESETN>;
+			reset-names = "rstn", "port", "spare";
 		};
 
 		irqc: interrupt-controller@110a0000 {
diff --git a/arch/arm64/boot/dts/renesas/r9a07g054.dtsi b/arch/arm64/boot/dts/renesas/r9a07g054.dtsi
index 0dee48c4f1e44..7a3e5b6a685f5 100644
--- a/arch/arm64/boot/dts/renesas/r9a07g054.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a07g054.dtsi
@@ -1025,6 +1025,7 @@ pinctrl: pinctrl@11030000 {
 			resets = <&cpg R9A07G054_GPIO_RSTN>,
 				 <&cpg R9A07G054_GPIO_PORT_RESETN>,
 				 <&cpg R9A07G054_GPIO_SPARE_RESETN>;
+			reset-names = "rstn", "port", "spare";
 		};
 
 		irqc: interrupt-controller@110a0000 {
diff --git a/arch/arm64/boot/dts/renesas/r9a08g045.dtsi b/arch/arm64/boot/dts/renesas/r9a08g045.dtsi
index 997e6cf0bb824..3a69bb246babd 100644
--- a/arch/arm64/boot/dts/renesas/r9a08g045.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a08g045.dtsi
@@ -533,6 +533,7 @@ pinctrl: pinctrl@11030000 {
 			resets = <&cpg R9A08G045_GPIO_RSTN>,
 				 <&cpg R9A08G045_GPIO_PORT_RESETN>,
 				 <&cpg R9A08G045_GPIO_SPARE_RESETN>;
+			reset-names = "rstn", "port", "spare";
 		};
 
 		irqc: interrupt-controller@11050000 {
diff --git a/arch/arm64/boot/dts/renesas/r9a08g046.dtsi b/arch/arm64/boot/dts/renesas/r9a08g046.dtsi
index 28b0c75587483..02a3029c058e2 100644
--- a/arch/arm64/boot/dts/renesas/r9a08g046.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a08g046.dtsi
@@ -14,6 +14,42 @@ / {
 	#size-cells = <2>;
 	interrupt-parent = <&gic>;
 
+	cluster0_opp: opp-table-0 {
+		compatible = "operating-points-v2";
+
+		opp-37500000 {
+			opp-hz = /bits/ 64 <37500000>;
+			opp-microvolt = <1000000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-75000000 {
+			opp-hz = /bits/ 64 <75000000>;
+			opp-microvolt = <1000000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-150000000 {
+			opp-hz = /bits/ 64 <150000000>;
+			opp-microvolt = <1000000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-300000000 {
+			opp-hz = /bits/ 64 <300000000>;
+			opp-microvolt = <1000000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-600000000 {
+			opp-hz = /bits/ 64 <600000000>;
+			opp-microvolt = <1000000>;
+			clock-latency-ns = <300000>;
+		};
+		opp-1200000000 {
+			opp-hz = /bits/ 64 <1200000000>;
+			opp-microvolt = <1000000>;
+			clock-latency-ns = <300000>;
+			opp-suspend;
+		};
+	};
+
 	cpus {
 		#address-cells = <1>;
 		#size-cells = <0>;
@@ -24,6 +60,8 @@ cpu0: cpu@0 {
 			device_type = "cpu";
 			next-level-cache = <&L3_CA55>;
 			enable-method = "psci";
+			clocks = <&cpg CPG_CORE R9A08G046_CLK_IC0>;
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu1: cpu@100 {
@@ -32,6 +70,8 @@ cpu1: cpu@100 {
 			device_type = "cpu";
 			next-level-cache = <&L3_CA55>;
 			enable-method = "psci";
+			clocks = <&cpg CPG_CORE R9A08G046_CLK_IC1>;
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu2: cpu@200 {
@@ -40,6 +80,8 @@ cpu2: cpu@200 {
 			device_type = "cpu";
 			next-level-cache = <&L3_CA55>;
 			enable-method = "psci";
+			clocks = <&cpg CPG_CORE R9A08G046_CLK_IC2>;
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		cpu3: cpu@300 {
@@ -48,6 +90,8 @@ cpu3: cpu@300 {
 			device_type = "cpu";
 			next-level-cache = <&L3_CA55>;
 			enable-method = "psci";
+			clocks = <&cpg CPG_CORE R9A08G046_CLK_IC3>;
+			operating-points-v2 = <&cluster0_opp>;
 		};
 
 		L3_CA55: cache-controller-0 {
@@ -160,10 +204,111 @@ sysc: system-controller@11020000 {
 		};
 
 		pinctrl: pinctrl@11030000 {
+			compatible = "renesas,r9a08g046-pinctrl";
 			reg = <0 0x11030000 0 0x10000>;
 			gpio-controller;
 			#gpio-cells = <2>;
-			/* placeholder */
+			gpio-ranges = <&pinctrl 0 0 232>;
+			interrupt-controller;
+			#interrupt-cells = <2>;
+			interrupt-parent = <&icu>;
+			clocks = <&cpg CPG_MOD R9A08G046_GPIO_HCLK>;
+			power-domains = <&cpg>;
+			resets = <&cpg R9A08G046_GPIO_RSTN>,
+				 <&cpg R9A08G046_GPIO_PORT_RESETN>,
+				 <&cpg R9A08G046_GPIO_SPARE_RESETN>;
+			reset-names = "rstn", "port", "spare";
+			renesas,clonech = <&sysc 0xe2c>;
+		};
+
+		icu: interrupt-controller@11050000 {
+			compatible = "renesas,r9a08g046-irqc";
+			#interrupt-cells = <2>;
+			#address-cells = <0>;
+			interrupt-controller;
+			reg = <0 0x11050000 0 0x10000>;
+			interrupts = <GIC_SPI 0 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 1 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 2 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 3 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 4 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 5 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 6 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 7 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 8 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 529 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 530 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 531 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 532 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 533 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 534 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 535 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 536 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 505 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 506 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 507 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 508 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 509 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 510 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 511 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 512 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 513 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 514 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 515 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 516 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 517 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 518 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 519 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 520 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 521 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 522 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 523 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 524 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 525 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 526 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 527 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 528 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 529 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 530 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 531 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 532 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 533 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 534 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 535 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 536 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 33 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 35 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 36 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 37 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 212 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 236 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 248 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 260 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 272 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 284 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 296 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "nmi",
+					  "irq0", "irq1", "irq2", "irq3",
+					  "irq4", "irq5", "irq6", "irq7",
+					  "irq8", "irq9", "irq10", "irq11",
+					  "irq12", "irq13", "irq14", "irq15",
+					  "tint0", "tint1", "tint2", "tint3",
+					  "tint4", "tint5", "tint6", "tint7",
+					  "tint8", "tint9", "tint10", "tint11",
+					  "tint12", "tint13", "tint14", "tint15",
+					  "tint16", "tint17", "tint18", "tint19",
+					  "tint20", "tint21", "tint22", "tint23",
+					  "tint24", "tint25", "tint26", "tint27",
+					  "tint28", "tint29", "tint30", "tint31",
+					  "bus-err", "ec7tie1-0", "ec7tie2-0", "ec7tiovf-0",
+					  "ovfunf0", "ovfunf1", "ovfunf2", "ovfunf3",
+					  "ovfunf4", "ovfunf5", "ovfunf6", "ovfunf7";
+			clocks = <&cpg CPG_MOD R9A08G046_IA55_CLK>,
+				 <&cpg CPG_MOD R9A08G046_IA55_PCLK>;
+			clock-names = "clk", "pclk";
+			power-domains = <&cpg>;
+			resets = <&cpg R9A08G046_IA55_RESETN>;
 		};
 
 		sdhi1: mmc@11c10000 {
@@ -171,6 +316,240 @@ sdhi1: mmc@11c10000 {
 			/* placeholder */
 		};
 
+		eth0: ethernet@11c30000 {
+			compatible = "renesas,r9a08g046-gbeth", "snps,dwmac-5.30a";
+			reg = <0 0x11c30000 0 0x10000>;
+			interrupts = <GIC_SPI 78 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 80 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 79 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 85 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 86 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 87 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 88 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 81 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 82 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 83 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 84 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 89 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 90 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 91 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 92 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "macirq", "eth_wake_irq", "eth_lpi",
+					  "rx-queue-0", "rx-queue-1", "rx-queue-2",
+					  "rx-queue-3", "tx-queue-0", "tx-queue-1",
+					  "tx-queue-2", "tx-queue-3", "ptp-pps-0",
+					  "ptp-pps-1", "ptp-pps-2", "ptp-pps-3";
+			clocks =  <&cpg CPG_MOD R9A08G046_ETH0_CLK_AXI>,
+				  <&cpg CPG_MOD R9A08G046_ETH0_CLK_CHI>,
+				  <&cpg CPG_MOD R9A08G046_ETH0_CLK_PTP_REF_I>,
+				  <&cpg CPG_MOD R9A08G046_ETH0_CLK_TX_I>,
+				  <&cpg CPG_MOD R9A08G046_ETH0_CLK_RX_I>,
+				  <&cpg CPG_MOD R9A08G046_ETH0_CLK_TX_180_I>,
+				  <&cpg CPG_MOD R9A08G046_ETH0_CLK_RX_180_I>,
+				  <&cpg CPG_MOD R9A08G046_ETH0_CLK_RMII_I>,
+				  <&cpg CPG_MOD R9A08G046_ETH0_CLK_TX_I_RMII>,
+				  <&cpg CPG_MOD R9A08G046_ETH0_CLK_RX_I_RMII>;
+			clock-names = "stmmaceth", "pclk", "ptp_ref",
+				      "tx", "rx", "tx-180", "rx-180",
+				      "rmii", "rmii_tx", "rmii_rx";
+			resets = <&cpg R9A08G046_ETH0_ARESET_N>;
+			power-domains = <&cpg>;
+			snps,multicast-filter-bins = <256>;
+			snps,perfect-filter-entries = <128>;
+			rx-fifo-depth = <8192>;
+			tx-fifo-depth = <8192>;
+			snps,fixed-burst;
+			snps,no-pbl-x8;
+			snps,force_thresh_dma_mode;
+			snps,axi-config = <&stmmac_axi_setup>;
+			snps,mtl-rx-config = <&mtl_rx_setup0>;
+			snps,mtl-tx-config = <&mtl_tx_setup0>;
+			snps,txpbl = <32>;
+			snps,rxpbl = <32>;
+			status = "disabled";
+
+			mdio0: mdio {
+				compatible = "snps,dwmac-mdio";
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			mtl_rx_setup0: rx-queues-config {
+				snps,rx-queues-to-use = <4>;
+				snps,rx-sched-sp;
+
+				queue0 {
+					snps,dcb-algorithm;
+					snps,priority = <0x1>;
+					snps,map-to-dma-channel = <0>;
+				};
+
+				queue1 {
+					snps,dcb-algorithm;
+					snps,priority = <0x2>;
+					snps,map-to-dma-channel = <1>;
+				};
+
+				queue2 {
+					snps,dcb-algorithm;
+					snps,priority = <0x4>;
+					snps,map-to-dma-channel = <2>;
+				};
+
+				queue3 {
+					snps,dcb-algorithm;
+					snps,priority = <0x8>;
+					snps,map-to-dma-channel = <3>;
+				};
+			};
+
+			mtl_tx_setup0: tx-queues-config {
+				snps,tx-queues-to-use = <4>;
+				snps,tx-sched-wrr;
+
+				queue0 {
+					snps,weight = <0x10>;
+					snps,dcb-algorithm;
+					snps,priority = <0x1>;
+				};
+
+				queue1 {
+					snps,weight = <0x12>;
+					snps,dcb-algorithm;
+					snps,priority = <0x2>;
+				};
+
+				queue2 {
+					snps,weight = <0x14>;
+					snps,dcb-algorithm;
+					snps,priority = <0x4>;
+				};
+
+				queue3 {
+					snps,weight = <0x18>;
+					snps,dcb-algorithm;
+					snps,priority = <0x8>;
+				};
+			};
+		};
+
+		eth1: ethernet@11c40000 {
+			compatible = "renesas,r9a08g046-gbeth", "snps,dwmac-5.30a";
+			reg = <0 0x11c40000 0 0x10000>;
+			interrupts = <GIC_SPI 93 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 94 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 100 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 101 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 102 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 103 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 96 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 97 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 98 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 99 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 104 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 105 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 106 IRQ_TYPE_EDGE_RISING>,
+				     <GIC_SPI 107 IRQ_TYPE_EDGE_RISING>;
+			interrupt-names = "macirq", "eth_wake_irq", "eth_lpi",
+					  "rx-queue-0", "rx-queue-1", "rx-queue-2",
+					  "rx-queue-3", "tx-queue-0", "tx-queue-1",
+					  "tx-queue-2", "tx-queue-3", "ptp-pps-0",
+					  "ptp-pps-1", "ptp-pps-2", "ptp-pps-3";
+			clocks = <&cpg CPG_MOD R9A08G046_ETH1_CLK_AXI>,
+				 <&cpg CPG_MOD R9A08G046_ETH1_CLK_CHI>,
+				 <&cpg CPG_MOD R9A08G046_ETH1_CLK_PTP_REF_I>,
+				 <&cpg CPG_MOD R9A08G046_ETH1_CLK_TX_I>,
+				 <&cpg CPG_MOD R9A08G046_ETH1_CLK_RX_I>,
+				 <&cpg CPG_MOD R9A08G046_ETH1_CLK_TX_180_I>,
+				 <&cpg CPG_MOD R9A08G046_ETH1_CLK_RX_180_I>,
+				 <&cpg CPG_MOD R9A08G046_ETH1_CLK_RMII_I>,
+				 <&cpg CPG_MOD R9A08G046_ETH1_CLK_TX_I_RMII>,
+				 <&cpg CPG_MOD R9A08G046_ETH1_CLK_RX_I_RMII>;
+			clock-names = "stmmaceth", "pclk", "ptp_ref",
+				      "tx", "rx", "tx-180", "rx-180",
+				      "rmii", "rmii_tx", "rmii_rx";
+			resets = <&cpg R9A08G046_ETH1_ARESET_N>;
+			power-domains = <&cpg>;
+			snps,multicast-filter-bins = <256>;
+			snps,perfect-filter-entries = <128>;
+			rx-fifo-depth = <8192>;
+			tx-fifo-depth = <8192>;
+			snps,fixed-burst;
+			snps,no-pbl-x8;
+			snps,force_thresh_dma_mode;
+			snps,axi-config = <&stmmac_axi_setup>;
+			snps,mtl-rx-config = <&mtl_rx_setup1>;
+			snps,mtl-tx-config = <&mtl_tx_setup1>;
+			snps,txpbl = <32>;
+			snps,rxpbl = <32>;
+			status = "disabled";
+
+			mdio1: mdio {
+				compatible = "snps,dwmac-mdio";
+				#address-cells = <1>;
+				#size-cells = <0>;
+			};
+
+			mtl_rx_setup1: rx-queues-config {
+				snps,rx-queues-to-use = <4>;
+				snps,rx-sched-sp;
+
+				queue0 {
+					snps,dcb-algorithm;
+					snps,priority = <0x1>;
+					snps,map-to-dma-channel = <0>;
+				};
+
+				queue1 {
+					snps,dcb-algorithm;
+					snps,priority = <0x2>;
+					snps,map-to-dma-channel = <1>;
+				};
+
+				queue2 {
+					snps,dcb-algorithm;
+					snps,priority = <0x4>;
+					snps,map-to-dma-channel = <2>;
+				};
+
+				queue3 {
+					snps,dcb-algorithm;
+					snps,priority = <0x8>;
+					snps,map-to-dma-channel = <3>;
+				};
+			};
+
+			mtl_tx_setup1: tx-queues-config {
+				snps,tx-queues-to-use = <4>;
+				snps,tx-sched-wrr;
+
+				queue0 {
+					snps,weight = <0x10>;
+					snps,dcb-algorithm;
+					snps,priority = <0x1>;
+				};
+
+				queue1 {
+					snps,weight = <0x12>;
+					snps,dcb-algorithm;
+					snps,priority = <0x2>;
+				};
+
+				queue2 {
+					snps,weight = <0x14>;
+					snps,dcb-algorithm;
+					snps,priority = <0x4>;
+				};
+
+				queue3 {
+					snps,weight = <0x18>;
+					snps,dcb-algorithm;
+					snps,priority = <0x8>;
+				};
+			};
+		};
+
 		pcie: pcie@11e40000 {
 			reg = <0 0x11e40000 0 0x10000>;
 			ranges = <0x02000000 0 0x30000000 0 0x30000000 0 0x08000000>;
@@ -198,6 +577,27 @@ gic: interrupt-controller@12400000 {
 			interrupt-controller;
 			interrupts = <GIC_PPI 9 IRQ_TYPE_LEVEL_LOW>;
 		};
+
+		wdt0: watchdog@12800800 {
+			compatible = "renesas,r9a08g046-wdt", "renesas,rzg2l-wdt";
+			reg = <0 0x12800800 0 0x400>;
+			clocks = <&cpg CPG_MOD R9A08G046_WDT0_PCLK>,
+				 <&cpg CPG_MOD R9A08G046_WDT0_CLK>;
+			clock-names = "pclk", "oscclk";
+			interrupts = <GIC_SPI 53 IRQ_TYPE_LEVEL_HIGH>,
+				     <GIC_SPI 52 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-names = "wdt", "perrout";
+			resets = <&cpg R9A08G046_WDT0_PRESETN>;
+			power-domains = <&cpg>;
+			status = "disabled";
+		};
+	};
+
+	stmmac_axi_setup: stmmac-axi-config {
+		snps,lpi_en;
+		snps,wr_osr_lmt = <0xf>;
+		snps,rd_osr_lmt = <0xf>;
+		snps,blen = <16 8 4 0 0 0 0>;
 	};
 
 	timer {
diff --git a/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts b/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts
index 86db86335d5e0..0ae052238b3b5 100644
--- a/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts
+++ b/arch/arm64/boot/dts/renesas/r9a08g046l48-smarc.dts
@@ -14,6 +14,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/input/input.h>
+#include <dt-bindings/pinctrl/renesas,r9a08g046-pinctrl.h>
 #include "r9a08g046l48.dtsi"
 #include "rzg3l-smarc-som.dtsi"
 #include "renesas-smarc2.dtsi"
@@ -35,3 +36,15 @@ &keys {
 	/delete-node/ key-2;
 	/delete-node/ key-3;
 };
+
+&pinctrl {
+	scif0_pins: scif0 {
+		pins = "SCIF0_TXD", "SCIF0_RXD";
+		power-source = <1800>;
+	};
+};
+
+&scif0 {
+	pinctrl-0 = <&scif0_pins>;
+	pinctrl-names = "default";
+};
diff --git a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
index 95a4e30a064d1..4267b10937f3f 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g047.dtsi
@@ -265,6 +265,7 @@ pinctrl: pinctrl@10410000 {
 			interrupt-parent = <&icu>;
 			power-domains = <&cpg>;
 			resets = <&cpg 0xa5>, <&cpg 0xa6>;
+			reset-names = "main", "error";
 		};
 
 		cpg: clock-controller@10420000 {
diff --git a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
index 40525470194ec..dc5b116679c0c 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g056.dtsi
@@ -288,6 +288,7 @@ pinctrl: pinctrl@10410000 {
 			gpio-ranges = <&pinctrl 0 0 96>;
 			power-domains = <&cpg>;
 			resets = <&cpg 0xa5>, <&cpg 0xa6>;
+			reset-names = "main", "error";
 		};
 
 		cpg: clock-controller@10420000 {
@@ -1327,6 +1328,7 @@ usb20phyrst: usb20phy-reset@15830000 {
 			resets = <&cpg 0xaf>;
 			power-domains = <&cpg>;
 			#reset-cells = <0>;
+			#mux-state-cells = <1>;
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
index 9581af58024ed..1e94366bb7eee 100644
--- a/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
+++ b/arch/arm64/boot/dts/renesas/r9a09g057.dtsi
@@ -270,6 +270,7 @@ pinctrl: pinctrl@10410000 {
 			interrupt-parent = <&icu>;
 			power-domains = <&cpg>;
 			resets = <&cpg 0xa5>, <&cpg 0xa6>;
+			reset-names = "main", "error";
 		};
 
 		cpg: clock-controller@10420000 {
@@ -1345,6 +1346,7 @@ usb20phyrst: usb20phy-reset@15830000 {
 			resets = <&cpg 0xaf>;
 			power-domains = <&cpg>;
 			#reset-cells = <0>;
+			#mux-state-cells = <1>;
 			status = "disabled";
 		};
 
@@ -1355,6 +1357,7 @@ usb21phyrst: usb21phy-reset@15840000 {
 			resets = <&cpg 0xaf>;
 			power-domains = <&cpg>;
 			#reset-cells = <0>;
+			#mux-state-cells = <1>;
 			status = "disabled";
 		};
 
diff --git a/arch/arm64/boot/dts/renesas/rz-smarc-cru-csi-ov5645.dtsi b/arch/arm64/boot/dts/renesas/rz-smarc-cru-csi-ov5645.dtsi
index 4d2b0655859ab..3feffa4f16a9a 100644
--- a/arch/arm64/boot/dts/renesas/rz-smarc-cru-csi-ov5645.dtsi
+++ b/arch/arm64/boot/dts/renesas/rz-smarc-cru-csi-ov5645.dtsi
@@ -46,7 +46,12 @@ &csi2 {
 	status = "okay";
 
 	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
 		port@0 {
+			reg = <0>;
+
 			csi2_in: endpoint {
 				clock-lanes = <0>;
 				data-lanes = <1 2>;
diff --git a/arch/arm64/boot/dts/renesas/rz-smarc-du-adv7513.dtsi b/arch/arm64/boot/dts/renesas/rz-smarc-du-adv7513.dtsi
index 36707576030d4..f5412578ee65f 100644
--- a/arch/arm64/boot/dts/renesas/rz-smarc-du-adv7513.dtsi
+++ b/arch/arm64/boot/dts/renesas/rz-smarc-du-adv7513.dtsi
@@ -26,7 +26,12 @@ &du {
 	status = "okay";
 
 	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
 		port@0 {
+			reg = <0>;
+
 			du_out_rgb: endpoint {
 				remote-endpoint = <&adv7513_in>;
 			};
diff --git a/arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi b/arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi
index ab4950671c7c8..b28e59a652599 100644
--- a/arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi
+++ b/arch/arm64/boot/dts/renesas/rzg3l-smarc-som.dtsi
@@ -8,6 +8,11 @@
 / {
 	compatible = "renesas,rzg3l-smarcm", "renesas,r9a08g046l48", "renesas,r9a08g046";
 
+	aliases {
+		ethernet0 = &eth0;
+		ethernet1 = &eth1;
+	};
+
 	memory@48000000 {
 		device_type = "memory";
 		/* First 128MiB is reserved for secure area. */
@@ -15,6 +20,133 @@ memory@48000000 {
 	};
 };
 
+&eth0 {
+	phy-handle = <&phy0>;
+	phy-mode = "rgmii-id";
+
+	pinctrl-0 = <&eth0_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&eth0_rxc_rx_clk {
+	clock-frequency = <125000000>;
+};
+
+&eth1 {
+	phy-handle = <&phy1>;
+	phy-mode = "rgmii-id";
+
+	pinctrl-0 = <&eth1_pins>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
+&eth1_rxc_rx_clk {
+	clock-frequency = <125000000>;
+};
+
 &extal_clk {
 	clock-frequency = <24000000>;
 };
+
+&mdio0 {
+	phy0: ethernet-phy@7 {
+		compatible = "ethernet-phy-id0022.1640";
+		reg = <7>;
+		interrupts-extended = <&icu 3 IRQ_TYPE_LEVEL_LOW>;
+		rxc-skew-psec = <1400>;
+		txc-skew-psec = <1400>;
+		rxdv-skew-psec = <0>;
+		txen-skew-psec = <0>;
+		rxd0-skew-psec = <0>;
+		rxd1-skew-psec = <0>;
+		rxd2-skew-psec = <0>;
+		rxd3-skew-psec = <0>;
+		txd0-skew-psec = <0>;
+		txd1-skew-psec = <0>;
+		txd2-skew-psec = <0>;
+		txd3-skew-psec = <0>;
+	};
+};
+
+&mdio1 {
+	phy1: ethernet-phy@7 {
+		compatible = "ethernet-phy-id0022.1640";
+		reg = <7>;
+		interrupts-extended = <&icu 16 IRQ_TYPE_LEVEL_LOW>;
+		rxc-skew-psec = <1400>;
+		txc-skew-psec = <1400>;
+		rxdv-skew-psec = <0>;
+		txen-skew-psec = <0>;
+		rxd0-skew-psec = <0>;
+		rxd1-skew-psec = <0>;
+		rxd2-skew-psec = <0>;
+		rxd3-skew-psec = <0>;
+		txd0-skew-psec = <0>;
+		txd1-skew-psec = <0>;
+		txd2-skew-psec = <0>;
+		txd3-skew-psec = <0>;
+	};
+};
+
+&pinctrl {
+	eth0_pins: eth0 {
+		txc {
+			pinmux = <RZG3L_PORT_PINMUX(B, 1, 1)>;  /* ETH0_TXC_REF_CLK */
+			power-source = <1800>;
+			output-enable;
+			drive-strength-microamp = <5200>;
+		};
+
+		ctrl {
+			pinmux = <RZG3L_PORT_PINMUX(A, 0, 1)>, /* MDIO */
+				 <RZG3L_PORT_PINMUX(A, 1, 1)>, /* MDC */
+				 <RZG3L_PORT_PINMUX(A, 2, 1)>, /* RX_CTL */
+				 <RZG3L_PORT_PINMUX(A, 3, 1)>, /* TX_CTL */
+				 <RZG3L_PORT_PINMUX(B, 0, 1)>, /* RXC */
+				 <RZG3L_PORT_PINMUX(B, 2, 1)>, /* TXD0 */
+				 <RZG3L_PORT_PINMUX(B, 3, 1)>, /* TXD1 */
+				 <RZG3L_PORT_PINMUX(B, 4, 1)>, /* TXD2 */
+				 <RZG3L_PORT_PINMUX(B, 5, 1)>, /* TXD3 */
+				 <RZG3L_PORT_PINMUX(B, 6, 1)>, /* RXD0 */
+				 <RZG3L_PORT_PINMUX(B, 7, 1)>, /* RXD1 */
+				 <RZG3L_PORT_PINMUX(C, 0, 1)>, /* RXD2 */
+				 <RZG3L_PORT_PINMUX(C, 1, 1)>, /* RXD3 */
+				 <RZG3L_PORT_PINMUX(C, 2, 15)>; /* PHY_INTR */
+				 power-source = <1800>;
+		};
+	};
+
+	eth1_pins: eth1 {
+		txc {
+			pinmux = <RZG3L_PORT_PINMUX(E, 1, 1)>;  /* ETH1_TXC_REF_CLK */
+			power-source = <1800>;
+			output-enable;
+			drive-strength-microamp = <5200>;
+		};
+
+		ctrl {
+			pinmux = <RZG3L_PORT_PINMUX(D, 0, 1)>, /* MDIO */
+				 <RZG3L_PORT_PINMUX(D, 1, 1)>, /* MDC */
+				 <RZG3L_PORT_PINMUX(D, 2, 1)>, /* RX_CTL */
+				 <RZG3L_PORT_PINMUX(D, 3, 1)>, /* TX_CTL */
+				 <RZG3L_PORT_PINMUX(E, 0, 1)>, /* RXC */
+				 <RZG3L_PORT_PINMUX(E, 2, 1)>, /* TXD0 */
+				 <RZG3L_PORT_PINMUX(E, 3, 1)>, /* TXD1 */
+				 <RZG3L_PORT_PINMUX(E, 4, 1)>, /* TXD2 */
+				 <RZG3L_PORT_PINMUX(E, 5, 1)>, /* TXD3 */
+				 <RZG3L_PORT_PINMUX(E, 6, 1)>, /* RXD0 */
+				 <RZG3L_PORT_PINMUX(E, 7, 1)>, /* RXD1 */
+				 <RZG3L_PORT_PINMUX(F, 0, 1)>, /* RXD2 */
+				 <RZG3L_PORT_PINMUX(F, 1, 1)>, /* RXD3 */
+				 <RZG3L_PORT_PINMUX(F, 2, 15)>; /* PHY_INTR */
+				 power-source = <1800>;
+		};
+	};
+};
+
+&wdt0 {
+	timeout-sec = <60>;
+	status = "okay";
+};
diff --git a/arch/arm64/boot/dts/renesas/salvator-common.dtsi b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
index e505161caa675..0b29bf9564eb1 100644
--- a/arch/arm64/boot/dts/renesas/salvator-common.dtsi
+++ b/arch/arm64/boot/dts/renesas/salvator-common.dtsi
@@ -211,19 +211,6 @@ reg_12v: regulator-12v {
 		regulator-always-on;
 	};
 
-	sound_card: sound {
-		compatible = "audio-graph-card";
-
-		label = "rcar-sound";
-
-		dais = <&rsnd_port0	/* ak4613 */
-			&rsnd_port1	/* HDMI0  */
-#ifdef SOC_HAS_HDMI1
-			&rsnd_port2	/* HDMI1  */
-#endif
-			>;
-	};
-
 	vbus0_usb2: regulator-vbus0-usb2 {
 		compatible = "regulator-fixed";
 
@@ -281,6 +268,19 @@ vccq_sdhi3: regulator-vccq-sdhi3 {
 		states = <3300000 1>, <1800000 0>;
 	};
 
+	sound_card: sound {
+		compatible = "audio-graph-card";
+
+		label = "rcar-sound";
+
+		dais = <&rsnd_port0	/* ak4613 */
+			&rsnd_port1	/* HDMI0  */
+#ifdef SOC_HAS_HDMI1
+			&rsnd_port2	/* HDMI1  */
+#endif
+			>;
+	};
+
 	vga {
 		compatible = "vga-connector";
 
diff --git a/arch/arm64/boot/dts/renesas/salvator-panel-aa104xd12.dtso b/arch/arm64/boot/dts/renesas/salvator-panel-aa104xd12.dtso
index c83a30adc6adf..7807c3f80409a 100644
--- a/arch/arm64/boot/dts/renesas/salvator-panel-aa104xd12.dtso
+++ b/arch/arm64/boot/dts/renesas/salvator-panel-aa104xd12.dtso
@@ -27,7 +27,12 @@ &lvds0 {
 	status = "okay";
 
 	ports {
+		#address-cells = <1>;
+		#size-cells = <0>;
+
 		port@1 {
+			reg = <1>;
+
 			lvds0_out: endpoint {
 				remote-endpoint = <&panel_in>;
 			};
diff --git a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi
index b7e669d8ba4d1..add917af5de78 100644
--- a/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi
+++ b/arch/arm64/boot/dts/rockchip/px30-cobra.dtsi
@@ -397,7 +397,7 @@ &io_domains {
 
 &mdio {
 	dp83825: ethernet-phy@0 {
-		compatible = "ethernet-phy-ieee802.3-c22";
+		compatible = "ethernet-phy-id2000.a140";
 		reg = <0x0>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&phy_rst>;
diff --git a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi
index 4203b335a2633..973b4c5880e24 100644
--- a/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi
+++ b/arch/arm64/boot/dts/rockchip/px30-ringneck.dtsi
@@ -344,7 +344,7 @@ &io_domains {
 
 &mdio {
 	dp83825: ethernet-phy@0 {
-		compatible = "ethernet-phy-ieee802.3-c22";
+		compatible = "ethernet-phy-id2000.a140";
 		reg = <0x0>;
 		pinctrl-names = "default";
 		pinctrl-0 = <&phy_rst>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
index 7eca1da78cffa..2f9e39671efc0 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399-gru.dtsi
@@ -18,6 +18,11 @@ chosen {
 		stdout-path = "serial2:115200n8";
 	};
 
+	firmware {
+		#address-cells = <2>;
+		#size-cells = <2>;
+	};
+
 	/*
 	 * Power Tree
 	 *
diff --git a/arch/arm64/boot/dts/rockchip/rk3528.dtsi b/arch/arm64/boot/dts/rockchip/rk3528.dtsi
index 806b8109f67df..77d314716b43b 100644
--- a/arch/arm64/boot/dts/rockchip/rk3528.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3528.dtsi
@@ -983,6 +983,14 @@ pwm7: pwm@ffa98030 {
 			status = "disabled";
 		};
 
+		wdt: watchdog@ffac0000 {
+			compatible = "rockchip,rk3528-wdt", "snps,dw-wdt";
+			reg = <0x0 0xffac0000 0x0 0x100>;
+			clocks = <&cru TCLK_WDT_NS>, <&cru PCLK_WDT_NS>;
+			clock-names = "tclk", "pclk";
+			interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>;
+		};
+
 		saradc: adc@ffae0000 {
 			compatible = "rockchip,rk3528-saradc";
 			reg = <0x0 0xffae0000 0x0 0x10000>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts b/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts
index 7e499064e0357..985770e3a5e2c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3566-roc-pc.dts
@@ -245,7 +245,7 @@ rk809: pmic@20 {
 		compatible = "rockchip,rk809";
 		reg = <0x20>;
 		interrupt-parent = <&gpio0>;
-		interrupts = <RK_PA7 IRQ_TYPE_LEVEL_LOW>;
+		interrupts = <RK_PA3 IRQ_TYPE_LEVEL_LOW>;
 		clock-output-names = "rk808-clkout1", "rk808-clkout2";
 		assigned-clocks = <&cru I2S1_MCLKOUT_TX>;
 		assigned-clock-parents = <&cru CLK_I2S1_8CH_TX>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dts b/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dts
index 718d1a2da8e56..90ce6f0e1dcff 100644
--- a/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3568-nanopi-r5s.dts
@@ -98,7 +98,7 @@ &mdio0 {
 	rgmii_phy0: ethernet-phy@1 {
 		compatible = "ethernet-phy-ieee802.3-c22";
 		reg = <1>;
-		pinctrl-0 = <&eth_phy0_reset_pin>;
+		pinctrl-0 = <&gmac0_rstn_gpio0_c5_pin>;
 		pinctrl-names = "default";
 	};
 };
@@ -132,8 +132,8 @@ &pcie3x2 {
 
 &pinctrl {
 	gmac0 {
-		eth_phy0_reset_pin: eth-phy0-reset-pin {
-			rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_up>;
+		gmac0_rstn_gpio0_c5_pin: gmac0-rstn-gpio0-c5-pin {
+			rockchip,pins = <0 RK_PC5 RK_FUNC_GPIO &pcfg_pull_up>;
 		};
 	};
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3568-rock-3b.dts b/arch/arm64/boot/dts/rockchip/rk3568-rock-3b.dts
index 69001e453732e..149a5306bb032 100644
--- a/arch/arm64/boot/dts/rockchip/rk3568-rock-3b.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3568-rock-3b.dts
@@ -56,7 +56,15 @@ led-0 {
 		};
 	};
 
-	/* pi6c pcie clock generator */
+	/* PI6C20100 PCIe reference clock buffer */
+	pcie30_refclk: pcie-clock-generator {
+		compatible = "gated-fixed-clock";
+		#clock-cells = <0>;
+		clock-frequency = <100000000>;
+		clock-output-names = "pcie30_refclk";
+		vdd-supply = <&vcc3v3_pi6c_03>;
+	};
+
 	vcc3v3_pi6c_03: regulator-3v3-vcc-pi6c-03 {
 		compatible = "regulator-fixed";
 		enable-active-high;
@@ -64,8 +72,6 @@ vcc3v3_pi6c_03: regulator-3v3-vcc-pi6c-03 {
 		pinctrl-names = "default";
 		pinctrl-0 = <&pcie_pwren_h>;
 		regulator-name = "vcc3v3_pi6c_03";
-		regulator-always-on;
-		regulator-boot-on;
 		regulator-min-microvolt = <3300000>;
 		regulator-max-microvolt = <3300000>;
 		startup-delay-us = <10000>;
@@ -553,6 +559,14 @@ &pcie30phy {
 };
 
 &pcie3x2 {
+	clocks = <&cru ACLK_PCIE30X2_MST>, <&cru ACLK_PCIE30X2_SLV>,
+		 <&cru ACLK_PCIE30X2_DBI>, <&cru PCLK_PCIE30X2>,
+		 <&cru CLK_PCIE30X2_AUX_NDFT>,
+		 <&cru CLK_PCIE30X2_PIPE_DFT>,
+		 <&pcie30_refclk>;
+	clock-names = "aclk_mst", "aclk_slv",
+		      "aclk_dbi", "pclk", "aux",
+		      "pipe", "ref";
 	pinctrl-names = "default";
 	pinctrl-0 = <&pcie30x2m1_pins>;
 	reset-gpios = <&gpio2 RK_PD6 GPIO_ACTIVE_HIGH>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-100ask-dshanpi-a1.dts b/arch/arm64/boot/dts/rockchip/rk3576-100ask-dshanpi-a1.dts
index b19f9b6be6bf8..05fb4a7f9a167 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-100ask-dshanpi-a1.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-100ask-dshanpi-a1.dts
@@ -278,6 +278,9 @@ &gpu {
 };
 
 &hdmi {
+	frl-enable-gpios = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmi_txm0_pins &hdmi_tx_scl &hdmi_tx_sda &hdmi_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -738,6 +741,12 @@ hp_det: hp-det {
 		};
 	};
 
+	hdmi {
+		hdmi_tx_on_h: hdmi-tx-on-h {
+			rockchip,pins = <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	pcie {
 		pcie_reset: pcie-reset {
 			rockchip,pins = <1 RK_PB7 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -806,6 +815,7 @@ &u2phy1_otg {
 
 &uart0 {
 	pinctrl-0 = <&uart0m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-armsom-sige5.dts b/arch/arm64/boot/dts/rockchip/rk3576-armsom-sige5.dts
index 1c100ffd15186..4ac4465e39a51 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-armsom-sige5.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-armsom-sige5.dts
@@ -304,6 +304,9 @@ &gpu {
 };
 
 &hdmi {
+	frl-enable-gpios = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmi_txm0_pins &hdmi_tx_scl &hdmi_tx_sda &hdmi_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -819,6 +822,12 @@ hp_det_l: hp-det-l {
 		};
 	};
 
+	hdmi {
+		hdmi_tx_on_h: hdmi-tx-on-h {
+			rockchip,pins = <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -966,6 +975,7 @@ &u2phy1_otg {
 
 &uart0 {
 	pinctrl-0 = <&uart0m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts
index fb0dd1bc51482..4c82980a9f63a 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-evb1-v10.dts
@@ -370,6 +370,9 @@ &gpu {
 };
 
 &hdmi {
+	frl-enable-gpios = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmi_txm0_pins &hdmi_tx_scl &hdmi_tx_sda &hdmi_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -895,6 +898,12 @@ host_wake_bt: host-wake-bt {
 		};
 	};
 
+	hdmi {
+		hdmi_tx_on_h: hdmi-tx-on-h {
+			rockchip,pins = <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		rtc_int: rtc-int {
 			rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts b/arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts
index 98d5d00d63b57..dfc756d5ec5e1 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-evb2-v10.dts
@@ -375,6 +375,9 @@ &gpu {
 };
 
 &hdmi {
+	frl-enable-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmi_txm0_pins &hdmi_tx_scl &hdmi_tx_sda &hdmi_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -832,6 +835,12 @@ image_pwren: image-pwren {
 		};
 	};
 
+	hdmi {
+		hdmi_tx_on_h: hdmi-tx-on-h {
+			rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		rtc_int: rtc-int {
 			rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-khadas-edge-2l.dts b/arch/arm64/boot/dts/rockchip/rk3576-khadas-edge-2l.dts
index 68630379af630..e7fe22eb31a3c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-khadas-edge-2l.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-khadas-edge-2l.dts
@@ -4,6 +4,7 @@
 
 #include <dt-bindings/gpio/gpio.h>
 #include <dt-bindings/pinctrl/rockchip.h>
+#include <dt-bindings/soc/rockchip,vop2.h>
 #include "rk3576.dtsi"
 
 / {
@@ -17,6 +18,528 @@ aliases {
 	chosen {
 		stdout-path = "serial0:1500000n8";
 	};
+
+	hdmi-con {
+		compatible = "hdmi-connector";
+		type = "a";
+
+		port {
+			hdmi_con_in: endpoint {
+				remote-endpoint = <&hdmi_out_con>;
+			};
+		};
+	};
+
+	vcc_sys: regulator-vcc5v0-sys {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_sys";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	vcc_1v1_nldo_s3: regulator-vcc-1v1-nldo-s3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_1v1_nldo_s3";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <1100000>;
+		regulator-max-microvolt = <1100000>;
+		vin-supply = <&vcc_sys>;
+	};
+
+	vcc_2v0_pldo_s3: regulator-vcc-2v0-pldo-s3 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_2v0_pldo_s3";
+		regulator-boot-on;
+		regulator-always-on;
+		regulator-min-microvolt = <2000000>;
+		regulator-max-microvolt = <2000000>;
+		vin-supply = <&vcc_sys>;
+	};
+
+	vcc_wifi_reg_on: regulator-wifi-reg-on {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio4 RK_PA2 GPIO_ACTIVE_HIGH>;
+		pinctrl-0 = <&wifi_reg_on>;
+		pinctrl-names = "default";
+		regulator-name = "wifi_reg_on";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		vin-supply = <&vcc_1v8_s3>;
+	};
+};
+
+&combphy0_ps {
+	status = "okay";
+};
+
+&cpu_l0 {
+	cpu-supply = <&vdd_cpu_lit_s0>;
+};
+
+&cpu_b0 {
+	cpu-supply = <&vdd_cpu_big_s0>;
+};
+
+&gpu {
+	mali-supply = <&vdd_gpu_s0>;
+	status = "okay";
+};
+
+&hdmi {
+	status = "okay";
+};
+
+&hdmi_in {
+	hdmi_in_vp0: endpoint {
+		remote-endpoint = <&vp0_out_hdmi>;
+	};
+};
+
+&hdmi_out {
+	hdmi_out_con: endpoint {
+		remote-endpoint = <&hdmi_con_in>;
+	};
+};
+
+&hdmi_sound {
+	status = "okay";
+};
+
+&hdptxphy {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+
+	rk806: pmic@23 {
+		compatible = "rockchip,rk806";
+		reg = <0x23>;
+		interrupt-parent = <&gpio0>;
+		interrupts = <6 IRQ_TYPE_LEVEL_LOW>;
+		gpio-controller;
+		#gpio-cells = <2>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_pins>, <&rk806_dvs1_null>,
+			    <&rk806_dvs2_null>, <&rk806_dvs3_null>;
+		system-power-controller;
+
+		vcc1-supply = <&vcc_sys>;
+		vcc2-supply = <&vcc_sys>;
+		vcc3-supply = <&vcc_sys>;
+		vcc4-supply = <&vcc_sys>;
+		vcc5-supply = <&vcc_sys>;
+		vcc6-supply = <&vcc_sys>;
+		vcc7-supply = <&vcc_sys>;
+		vcc8-supply = <&vcc_sys>;
+		vcc9-supply = <&vcc_sys>;
+		vcc10-supply = <&vcc_sys>;
+		vcc11-supply = <&vcc_2v0_pldo_s3>;
+		vcc12-supply = <&vcc_sys>;
+		vcc13-supply = <&vcc_1v1_nldo_s3>;
+		vcc14-supply = <&vcc_1v1_nldo_s3>;
+		vcca-supply = <&vcc_sys>;
+
+		rk806_dvs1_null: dvs1-null-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs2_null: dvs2-null-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs3_null: dvs3-null-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun0";
+		};
+
+		rk806_dvs1_slp: dvs1-slp-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun1";
+		};
+
+		rk806_dvs1_pwrdn: dvs1-pwrdn-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun2";
+		};
+
+		rk806_dvs1_rst: dvs1-rst-pins {
+			pins = "gpio_pwrctrl1";
+			function = "pin_fun3";
+		};
+
+		rk806_dvs2_slp: dvs2-slp-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun1";
+		};
+
+		rk806_dvs2_pwrdn: dvs2-pwrdn-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun2";
+		};
+
+		rk806_dvs2_rst: dvs2-rst-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun3";
+		};
+
+		rk806_dvs2_dvs: dvs2-dvs-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun4";
+		};
+
+		rk806_dvs2_gpio: dvs2-gpio-pins {
+			pins = "gpio_pwrctrl2";
+			function = "pin_fun5";
+		};
+
+		rk806_dvs3_slp: dvs3-slp-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun1";
+		};
+
+		rk806_dvs3_pwrdn: dvs3-pwrdn-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun2";
+		};
+
+		rk806_dvs3_rst: dvs3-rst-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun3";
+		};
+
+		rk806_dvs3_dvs: dvs3-dvs-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun4";
+		};
+
+		rk806_dvs3_gpio: dvs3-gpio-pins {
+			pins = "gpio_pwrctrl3";
+			function = "pin_fun5";
+		};
+
+		regulators {
+			vdd_cpu_big_s0: dcdc-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_cpu_big_s0";
+				regulator-enable-ramp-delay = <400>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_npu_s0: dcdc-reg2 {
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_npu_s0";
+				regulator-enable-ramp-delay = <400>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_cpu_lit_s0: dcdc-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <950000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_cpu_lit_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			vcc_3v3_s3: dcdc-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcc_3v3_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <3300000>;
+				};
+			};
+
+			vdd_gpu_s0: dcdc-reg5 {
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <900000>;
+				regulator-ramp-delay = <12500>;
+				regulator-name = "vdd_gpu_s0";
+				regulator-enable-ramp-delay = <400>;
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+					regulator-suspend-microvolt = <850000>;
+				};
+			};
+
+			vddq_ddr_s0: dcdc-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vddq_ddr_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdd_logic_s0: dcdc-reg7 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <800000>;
+				regulator-name = "vdd_logic_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcc_1v8_s3: dcdc-reg8 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcc_1v8_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdd2_ddr_s3: dcdc-reg9 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-name = "vdd2_ddr_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+				};
+			};
+
+			vdd_ddr_s0: dcdc-reg10 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-name = "vdd_ddr_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca_1v8_s0: pldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca_1v8_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca1v8_pldo2_s0: pldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca1v8_pldo2_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_1v2_s0: pldo-reg3 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1200000>;
+				regulator-max-microvolt = <1200000>;
+				regulator-name = "vdda_1v2_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca_3v3_s0: pldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vcca_3v3_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vccio_sd_s0: pldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-name = "vccio_sd_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vcca1v8_pldo6_s3: pldo-reg6 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-name = "vcca1v8_pldo6_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <1800000>;
+				};
+			};
+
+			vdd_0v75_s3: nldo-reg1 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <550000>;
+				regulator-max-microvolt = <750000>;
+				regulator-name = "vdd_0v75_s3";
+
+				regulator-state-mem {
+					regulator-on-in-suspend;
+					regulator-suspend-microvolt = <750000>;
+				};
+			};
+
+			vdda_ddr_pll_s0: nldo-reg2 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <850000>;
+				regulator-name = "vdda_ddr_pll_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda0v75_hdmi_s0: nldo-reg3 {
+				regulator-boot-on;
+				regulator-min-microvolt = <837500>;
+				regulator-max-microvolt = <837500>;
+				regulator-name = "vdda0v75_hdmi_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_0v85_s0: nldo-reg4 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <850000>;
+				regulator-name = "vdda_0v85_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+
+			vdda_0v75_s0: nldo-reg5 {
+				regulator-always-on;
+				regulator-boot-on;
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <750000>;
+				regulator-name = "vdda_0v75_s0";
+
+				regulator-state-mem {
+					regulator-off-in-suspend;
+				};
+			};
+		};
+	};
+};
+
+&i2c2 {
+	status = "okay";
+
+	hym8563: rtc@51 {
+		compatible = "haoyu,hym8563";
+		reg = <0x51>;
+		#clock-cells = <0>;
+		clock-output-names = "hym8563";
+		wakeup-source;
+	};
+};
+
+&pcie0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie0_rst>;
+	reset-gpios = <&gpio4 RK_PB4 GPIO_ACTIVE_HIGH>;
+	vpcie3v3-supply = <&vcc_3v3_s3>;
+	status = "okay";
+
+	pcie@0,0 {
+		reg = <0x0 0 0 0 0>;
+		bus-range = <0x0 0xf>;
+		device_type = "pci";
+		ranges;
+		#address-cells = <3>;
+		#size-cells = <2>;
+
+		wifi: wifi@0,0 {
+			compatible = "pci14e4,449d";
+			reg = <0x10000 0 0 0 0>;
+			clocks = <&hym8563>;
+			clock-names = "lpo";
+		};
+	};
+};
+
+&pinctrl {
+	pcie0 {
+		pcie0_rst: pcie0-rst {
+			rockchip,pins = <4 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
+	wifi {
+		wifi_reg_on: wifi-reg-on {
+			rockchip,pins = <4 RK_PA2 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+
+		wifi_wake_host: wifi-wake-host {
+			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_down>;
+		};
+	};
 };
 
 &sdhci {
@@ -32,3 +555,18 @@ &sdhci {
 &uart0 {
 	status = "okay";
 };
+
+&vop {
+	status = "okay";
+};
+
+&vop_mmu {
+	status = "okay";
+};
+
+&vp0 {
+	vp0_out_hdmi: endpoint@ROCKCHIP_VOP2_EP_HDMI0 {
+		reg = <ROCKCHIP_VOP2_EP_HDMI0>;
+		remote-endpoint = <&hdmi_in_vp0>;
+	};
+};
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-luckfox-core3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576-luckfox-core3576.dtsi
index 749f0a54b478e..4fc8496828f80 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-luckfox-core3576.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3576-luckfox-core3576.dtsi
@@ -26,7 +26,6 @@ chosen {
 
 	hdmi-con {
 		compatible = "hdmi-connector";
-		hdmi-pwr-supply = <&vcc_5v0_hdmi>;
 		type = "a";
 
 		port {
@@ -138,22 +137,6 @@ regulator-state-mem {
 		};
 	};
 
-	vcc_5v0_hdmi: regulator-vcc-5v0-hdmi {
-		compatible = "regulator-fixed";
-		enable-active-high;
-		gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&hdmi_con_en>;
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		regulator-name = "vcc_5v0_hdmi";
-		vin-supply = <&vcc_5v0_sys>;
-
-		regulator-state-mem {
-			regulator-off-in-suspend;
-		};
-	};
-
 	vcc_5v0_host: regulator-vcc-5v0-host {
 		compatible = "regulator-fixed";
 		enable-active-high;
@@ -231,6 +214,9 @@ &gpu {
 };
 
 &hdmi {
+	frl-enable-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmi_txm0_pins &hdmi_tx_scl &hdmi_tx_sda &hdmi_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -655,7 +641,7 @@ &pcie0 {
 
 &pinctrl {
 	hdmi {
-		hdmi_con_en: hdmi-con-en {
+		hdmi_tx_on_h: hdmi-tx-on-h {
 			rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-nanopi-m5.dts b/arch/arm64/boot/dts/rockchip/rk3576-nanopi-m5.dts
index 7406a4adf8105..227fe6f3ec7c7 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-nanopi-m5.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-nanopi-m5.dts
@@ -327,6 +327,9 @@ &gpu {
 };
 
 &hdmi {
+	frl-enable-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmi_txm0_pins &hdmi_tx_scl &hdmi_tx_sda &hdmi_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -813,6 +816,12 @@ gmac1_rst: gmac1-rst {
 		};
 	};
 
+	hdmi {
+		hdmi_tx_on_h: hdmi-tx-on-h {
+			rockchip,pins = <4 RK_PC6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PA5 RK_FUNC_GPIO &pcfg_pull_up>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-nanopi-r76s.dts b/arch/arm64/boot/dts/rockchip/rk3576-nanopi-r76s.dts
index 7ec27b05ff10e..0a5cd5f6fd334 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-nanopi-r76s.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-nanopi-r76s.dts
@@ -70,7 +70,6 @@ led-2 {
 
 	hdmi-con {
 		compatible = "hdmi-connector";
-		hdmi-pwr-supply = <&vcc5v_hdmi_tx>;
 		type = "a";
 
 		port {
@@ -109,18 +108,6 @@ vcc5v_dcin: regulator-vcc5v-dcin {
 		regulator-name = "vcc5v_dcin";
 	};
 
-	vcc5v_hdmi_tx: regulator-vcc5v-hdmi-tx {
-		compatible = "regulator-fixed";
-		enable-active-high;
-		gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_HIGH>;
-		pinctrl-names = "default";
-		pinctrl-0 = <&hdmi_tx_on_h>;
-		regulator-min-microvolt = <5000000>;
-		regulator-max-microvolt = <5000000>;
-		regulator-name = "vcc5v_hdmi_tx";
-		vin-supply = <&vcc5v0_sys_s5>;
-	};
-
 	vcc5v0_device_s0: regulator-vcc5v0-device-s0 {
 		compatible = "regulator-fixed";
 		regulator-always-on;
@@ -252,6 +239,9 @@ &gpu {
 };
 
 &hdmi {
+	frl-enable-gpios = <&gpio4 RK_PC6 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmi_txm0_pins &hdmi_tx_scl &hdmi_tx_sda &hdmi_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-roc-pc.dts b/arch/arm64/boot/dts/rockchip/rk3576-roc-pc.dts
index d0ab1d1e0e119..8411c16fa4716 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-roc-pc.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-roc-pc.dts
@@ -283,6 +283,9 @@ &eth0m0_rgmii_bus
 };
 
 &hdmi {
+	frl-enable-gpios = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmi_txm0_pins &hdmi_tx_scl &hdmi_tx_sda &hdmi_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -720,6 +723,12 @@ &sdmmc {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi_tx_on_h: hdmi-tx-on-h {
+			rockchip,pins = <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		rtc_int_l: rtc-int-l {
 			rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -767,6 +776,7 @@ wd_en: wd-en {
 
 &uart0 {
 	pinctrl-0 = <&uart0m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3576-rock-4d.dts b/arch/arm64/boot/dts/rockchip/rk3576-rock-4d.dts
index 899a84b1fbf9e..272af1012ab03 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576-rock-4d.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3576-rock-4d.dts
@@ -289,6 +289,9 @@ &gpu {
 };
 
 &hdmi {
+	frl-enable-gpios = <&gpio2 RK_PB0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmi_txm0_pins &hdmi_tx_scl &hdmi_tx_sda &hdmi_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -720,6 +723,12 @@ &pcie0 {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi_tx_on_h: hdmi-tx-on-h {
+			rockchip,pins = <2 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PA0 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -824,6 +833,7 @@ &u2phy1_otg {
 
 &uart0 {
 	pinctrl-0 = <&uart0m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3576.dtsi b/arch/arm64/boot/dts/rockchip/rk3576.dtsi
index 28175d8200d57..e12a2a0cfb891 100644
--- a/arch/arm64/boot/dts/rockchip/rk3576.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3576.dtsi
@@ -1281,10 +1281,10 @@ gpu: gpu@27800000 {
 
 		vdec: video-codec@27b00000 {
 			compatible = "rockchip,rk3576-vdec";
-			reg = <0x0 0x27b00100 0x0 0x500>,
-			      <0x0 0x27b00000 0x0 0x100>,
+			reg = <0x0 0x27b00000 0x0 0x100>,
+			      <0x0 0x27b00100 0x0 0x500>,
 			      <0x0 0x27b00600 0x0 0x100>;
-			reg-names = "function", "link", "cache";
+			reg-names = "link", "function", "cache";
 			interrupts = <GIC_SPI 308 IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&cru ACLK_RKVDEC_ROOT>, <&cru HCLK_RKVDEC>,
 				 <&cru ACLK_RKVDEC_ROOT_BAK>, <&cru CLK_RKVDEC_CORE>,
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-armsom-sige7.dts b/arch/arm64/boot/dts/rockchip/rk3588-armsom-sige7.dts
index 39197ee198370..1005b15d708a9 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-armsom-sige7.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-armsom-sige7.dts
@@ -177,6 +177,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -347,6 +351,12 @@ &pd_gpu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -778,6 +788,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-armsom-w3.dts b/arch/arm64/boot/dts/rockchip/rk3588-armsom-w3.dts
index 6ad2759ddccaf..faeddcfdeb32f 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-armsom-w3.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-armsom-w3.dts
@@ -162,6 +162,10 @@ &combphy2_psu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -182,6 +186,10 @@ &hdmi0_sound {
 };
 
 &hdmi1 {
+	frl-enable-gpios = <&gpio4 RK_PA1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim2_tx1_cec &hdmim0_tx1_hpd
+		     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -330,6 +338,16 @@ &pcie3x4 {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -421,6 +439,7 @@ &sdio {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
index 4fb8888c281c8..546e19487219e 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-base.dtsi
@@ -1355,10 +1355,10 @@ vepu121_3_mmu: iommu@fdbac800 {
 
 	vdec0: video-codec@fdc38000 {
 		compatible = "rockchip,rk3588-vdec";
-		reg = <0x0 0xfdc38100 0x0 0x500>,
-		      <0x0 0xfdc38000 0x0 0x100>,
+		reg = <0x0 0xfdc38000 0x0 0x100>,
+		      <0x0 0xfdc38100 0x0 0x500>,
 		      <0x0 0xfdc38600 0x0 0x100>;
-		reg-names = "function", "link", "cache";
+		reg-names = "link", "function", "cache";
 		interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH 0>;
 		clocks = <&cru ACLK_RKVDEC0>, <&cru HCLK_RKVDEC0>, <&cru CLK_RKVDEC0_CA>,
 			 <&cru CLK_RKVDEC0_CORE>, <&cru CLK_RKVDEC0_HEVC_CA>;
@@ -1387,10 +1387,10 @@ vdec0_mmu: iommu@fdc38700 {
 
 	vdec1: video-codec@fdc40000 {
 		compatible = "rockchip,rk3588-vdec";
-		reg = <0x0 0xfdc40100 0x0 0x500>,
-		      <0x0 0xfdc40000 0x0 0x100>,
+		reg = <0x0 0xfdc40000 0x0 0x100>,
+		      <0x0 0xfdc40100 0x0 0x500>,
 		      <0x0 0xfdc40600 0x0 0x100>;
-		reg-names = "function", "link", "cache";
+		reg-names = "link", "function", "cache";
 		interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH 0>;
 		clocks = <&cru ACLK_RKVDEC1>, <&cru HCLK_RKVDEC1>, <&cru CLK_RKVDEC1_CA>,
 			 <&cru CLK_RKVDEC1_CORE>, <&cru CLK_RKVDEC1_HEVC_CA>;
@@ -1428,6 +1428,69 @@ av1d: video-codec@fdc70000 {
 		clock-names = "aclk", "hclk";
 		power-domains = <&power RK3588_PD_AV1>;
 		resets = <&cru SRST_A_AV1>, <&cru SRST_P_AV1>, <&cru SRST_A_AV1_BIU>, <&cru SRST_P_AV1_BIU>;
+		iommus = <&av1d_mmu>;
+	};
+
+	av1d_mmu: iommu@fdca0000 {
+		compatible = "rockchip,rk3588-av1-iommu", "verisilicon,iommu-1.2";
+		reg = <0x0 0xfdca0000 0x0 0x600>;
+		interrupts = <GIC_SPI 109 IRQ_TYPE_LEVEL_HIGH 0>;
+		clocks = <&cru ACLK_AV1>, <&cru PCLK_AV1>;
+		clock-names = "core", "iface";
+		#iommu-cells = <0>;
+		power-domains = <&power RK3588_PD_AV1>;
+	};
+
+	csi2: csi@fdd30000 {
+		compatible = "rockchip,rk3588-mipi-csi2", "rockchip,rk3568-mipi-csi2";
+		reg = <0x0 0xfdd30000 0x0 0x10000>;
+		interrupts = <GIC_SPI 147 IRQ_TYPE_LEVEL_HIGH 0>,
+			     <GIC_SPI 148 IRQ_TYPE_LEVEL_HIGH 0>;
+		interrupt-names = "err1", "err2";
+		clocks = <&cru PCLK_CSI_HOST_2>;
+		phys = <&csi_dphy0>;
+		power-domains = <&power RK3588_PD_VI>;
+		resets = <&cru SRST_P_CSI_HOST_2>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csi2_in: port@0 {
+				reg = <0>;
+			};
+
+			csi2_out: port@1 {
+				reg = <1>;
+			};
+		};
+	};
+
+	csi4: csi@fdd50000 {
+		compatible = "rockchip,rk3588-mipi-csi2", "rockchip,rk3568-mipi-csi2";
+		reg = <0x0 0xfdd50000 0x0 0x10000>;
+		interrupts = <GIC_SPI 151 IRQ_TYPE_LEVEL_HIGH 0>,
+			     <GIC_SPI 152 IRQ_TYPE_LEVEL_HIGH 0>;
+		interrupt-names = "err1", "err2";
+		clocks = <&cru PCLK_CSI_HOST_4>;
+		phys = <&csi_dphy1>;
+		power-domains = <&power RK3588_PD_VI>;
+		resets = <&cru SRST_P_CSI_HOST_4>;
+		status = "disabled";
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			csi4_in: port@0 {
+				reg = <0>;
+			};
+
+			csi4_out: port@1 {
+				reg = <1>;
+			};
+		};
 	};
 
 	vop: vop@fdd90000 {
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5-evb.dts b/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5-evb.dts
index 3d5c8b753208a..66e3c20d7b4f1 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5-evb.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5-evb.dts
@@ -125,6 +125,10 @@ vcc5v0_usb30_otg: regulator-vcc5v0-usb30-otg {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -145,8 +149,10 @@ hdmi0_out_con: endpoint {
 };
 
 &hdmi1 {
+	frl-enable-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim2_tx1_cec &hdmim0_tx1_hpd &hdmim1_tx1_scl &hdmim1_tx1_sda
+		     &hdmi1_tx_on_h>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&hdmim2_tx1_cec &hdmim0_tx1_hpd &hdmim1_tx1_scl &hdmim1_tx1_sda>;
 	status = "okay";
 };
 
@@ -211,6 +217,16 @@ &pcie3x4 {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	lcd {
 		lcdpwr_en: lcdpwr-en {
 			rockchip,pins = <1 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5-genbook.dts b/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5-genbook.dts
index 738637ecaf557..37d7b82ad553a 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5-genbook.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5-genbook.dts
@@ -184,7 +184,10 @@ edp_out_panel: endpoint {
 
 /* HDMI CEC is not used */
 &hdmi0 {
-	pinctrl-0 = <&hdmim0_tx0_hpd &hdmim0_tx0_scl &hdmim0_tx0_sda>;
+	frl-enable-gpios = <&gpio4 RK_PB0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_hpd &hdmim0_tx0_scl &hdmim0_tx0_sda
+		     &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -277,6 +280,12 @@ &pcie3x4 {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	lcd {
 		lcdpwr_en: lcdpwr-en {
 			rockchip,pins = <0 RK_PC4 RK_FUNC_GPIO &pcfg_pull_down>;
@@ -326,6 +335,7 @@ pcie_wake: pcie-wake {
 
 &pwm6 {
 	pinctrl-0 = <&pwm6m1_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5.dtsi
index b07543315f878..33dacaa6af4da 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-coolpi-cm5.dtsi
@@ -143,6 +143,7 @@ &gpu {
 
 &i2c0 {
 	pinctrl-0 = <&i2c0m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	vdd_cpu_big0_s0: regulator@42 {
@@ -219,6 +220,7 @@ hym8563: rtc@51 {
 
 &i2c7 {
 	pinctrl-0 = <&i2c7m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	es8316: audio-codec@10 {
@@ -244,6 +246,7 @@ &i2s0_mclk
 		     &i2s0_sclk
 		     &i2s0_sdi0
 		     &i2s0_sdo0>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	i2s0_8ch_p0: port {
@@ -656,5 +659,6 @@ &tsadc {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-edgeble-neu6a-io.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-edgeble-neu6a-io.dtsi
index 08920344a4b8d..2751f84a4b8b7 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-edgeble-neu6a-io.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-edgeble-neu6a-io.dtsi
@@ -252,6 +252,7 @@ &sdmmc {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts
index 09bc7b68dcc05..8969b56f3063e 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-evb1-v10.dts
@@ -360,6 +360,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -376,6 +380,10 @@ hdmi0_out_con: endpoint {
 };
 
 &hdmi1 {
+	frl-enable-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim2_tx1_cec &hdmim0_tx1_hpd
+		     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -502,6 +510,7 @@ &i2s0_mclk
 		     &i2s0_sclk
 		     &i2s0_sdi0
 		     &i2s0_sdo0>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -612,6 +621,16 @@ rtl8211f_rst: rtl8211f-rst {
 
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hdmirx {
 		hdmirx_hpd: hdmirx-5v-detection {
 			rockchip,pins = <2 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -1368,6 +1387,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-evb2-v10.dts b/arch/arm64/boot/dts/rockchip/rk3588-evb2-v10.dts
index 60ba6ac55b239..059ab02411d91 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-evb2-v10.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-evb2-v10.dts
@@ -143,6 +143,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio3 RK_PC5 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -183,6 +187,12 @@ &pd_gpu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <3 RK_PC5 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -911,6 +921,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-fet3588-c.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-fet3588-c.dtsi
index 4331cdc70f973..d3fb2677a8559 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-fet3588-c.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-fet3588-c.dtsi
@@ -558,5 +558,6 @@ &tsadc {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-firefly-core-3588j.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-firefly-core-3588j.dtsi
index 80e16ea4154c0..ecb27b4a29bd3 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-firefly-core-3588j.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-firefly-core-3588j.dtsi
@@ -443,5 +443,6 @@ regulator-state-mem {
 /* rk3588 preferred debug out */
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-firefly-icore-3588q.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-firefly-icore-3588q.dtsi
index 6726eeb492556..80ca7e78354cb 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-firefly-icore-3588q.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-firefly-icore-3588q.dtsi
@@ -439,5 +439,6 @@ regulator-state-mem {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-firefly-itx-3588j.dts b/arch/arm64/boot/dts/rockchip/rk3588-firefly-itx-3588j.dts
index e086114c76348..8dac069f8351b 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-firefly-itx-3588j.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-firefly-itx-3588j.dts
@@ -322,6 +322,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PA0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -487,6 +491,12 @@ dp1_hpd: dp1-hpd {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -558,11 +568,13 @@ &sata2 {
 /* uart/232/485 */
 &uart0 {
 	pinctrl-0 = <&uart0m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
 &uart1 {
 	pinctrl-0 = <&uart1m1_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-friendlyelec-cm3588-nas.dts b/arch/arm64/boot/dts/rockchip/rk3588-friendlyelec-cm3588-nas.dts
index 10a7d3691a26f..0dce96ca8c288 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-friendlyelec-cm3588-nas.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-friendlyelec-cm3588-nas.dts
@@ -331,6 +331,10 @@ &gpio4 {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -347,6 +351,10 @@ hdmi0_out_con: endpoint {
 };
 
 &hdmi1 {
+	frl-enable-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim2_tx1_cec &hdmim0_tx1_hpd
+		     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -520,6 +528,16 @@ key1_pin: key1-pin {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hdmirx {
 		hdmirx_hpd: hdmirx-5v-detection {
 			rockchip,pins = <3 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-h96-max-v58.dts b/arch/arm64/boot/dts/rockchip/rk3588-h96-max-v58.dts
index 73d8ce4fde2b8..3811f9a73bb6c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-h96-max-v58.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-h96-max-v58.dts
@@ -210,6 +210,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -339,6 +343,12 @@ &pd_gpu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -774,6 +784,7 @@ &u2phy1_otg {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts b/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts
index 5f5d89a33a4ae..41758fe7d3689 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-jaguar.dts
@@ -278,8 +278,10 @@ &gpu {
 
 &hdmi0 {
 	/* No CEC on Jaguar */
+	frl-enable-gpios = <&gpio0 RK_PD3 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_hpd &hdmim0_tx0_scl &hdmim0_tx0_sda
+		     &hdmi0_tx_on_h>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&hdmim0_tx0_hpd &hdmim0_tx0_scl &hdmim0_tx0_sda>;
 	status = "okay";
 };
 
@@ -305,6 +307,7 @@ &hdptxphy0 {
 
 &i2c0 {
 	pinctrl-0 = <&i2c0m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	fan@18 {
@@ -415,10 +418,12 @@ regulator-state-mem {
 
 &i2c1 {
 	pinctrl-0 = <&i2c1m4_xfer>;
+	pinctrl-names = "default";
 };
 
 &i2c6 {
 	pinctrl-0 = <&i2c6m4_xfer>;
+	pinctrl-names = "default";
 };
 
 &i2c7 {
@@ -437,6 +442,7 @@ eeprom@54 {
 
 &i2c8 {
 	pinctrl-0 = <&i2c8m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	typec-portc@22 {
@@ -571,6 +577,12 @@ eth_reset: eth-reset {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	leds {
 		led1_pin: led1-pin {
 			rockchip,pins = <1 RK_PD4 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -1046,18 +1058,21 @@ &u2phy3_host {
 /* Mule-ATtiny debug UART; typically baudrate 9600 */
 &uart0 {
 	pinctrl-0 = <&uart0m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
 /* Main debug interface on P20 micro-USB B port and P21 header */
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
 /* RS485 on P19 */
 &uart3 {
 	pinctrl-0 = <&uart3m2_xfer &uart3_rtsn>;
+	pinctrl-names = "default";
 	linux,rs485-enabled-at-boot-time;
 	status = "okay";
 };
@@ -1065,6 +1080,7 @@ &uart3 {
 /* Mule-ATtiny UPDI flashing UART */
 &uart7 {
 	pinctrl-0 = <&uart7m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -1134,9 +1150,6 @@ &usb_host0_xhci {
 	status = "okay";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		usb_host0_xhci_drd_sw: endpoint {
 			remote-endpoint = <&usbc0_hs>;
 		};
@@ -1149,9 +1162,6 @@ &usb_host1_xhci {
 	status = "okay";
 
 	port {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
 		usb_host1_xhci_drd_sw: endpoint {
 			remote-endpoint = <&usbc1_hs>;
 		};
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-mnt-reform2.dts b/arch/arm64/boot/dts/rockchip/rk3588-mnt-reform2.dts
index 78a4e896f665b..9052e0d5494f5 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-mnt-reform2.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-mnt-reform2.dts
@@ -148,6 +148,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PA0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -181,6 +185,7 @@ rgmii_phy: ethernet-phy@0 {
 
 &pcie2x1l2 {
 	pinctrl-0 = <&pcie2_0_rst>;
+	pinctrl-names = "default";
 	reset-gpios = <&gpio3 RK_PD1 GPIO_ACTIVE_HIGH>;
 	status = "okay";
 };
@@ -205,6 +210,12 @@ dp1_hpd: dp1-hpd {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	pcie2 {
 		pcie2_0_rst: pcie2-0-rst {
 			rockchip,pins = <3 RK_PD1 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -226,6 +237,7 @@ eth_phy_reset: eth-phy-reset {
 
 &pwm8 {
 	pinctrl-0 = <&pwm8m2_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
index 84b6b53f016ab..7495af5bd9731 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-nanopc-t6.dtsi
@@ -361,6 +361,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -381,6 +385,10 @@ &hdmi0_sound {
 };
 
 &hdmi1 {
+	frl-enable-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim2_tx1_cec &hdmim0_tx1_hpd
+		     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -571,6 +579,7 @@ rt5616_p0_0: endpoint {
 
 &i2c8 {
 	pinctrl-0 = <&i2c8m2_xfer>;
+	pinctrl-names = "default";
 };
 
 &i2s0_8ch {
@@ -682,6 +691,16 @@ usr_led_pin: usr-led-pin {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hdmirx {
 		hdmirx_hpd: hdmirx-5v-detection {
 			rockchip,pins = <1 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -745,6 +764,7 @@ usbc0_int: usbc0-int {
 
 &pwm1 {
 	pinctrl-0 = <&pwm1m1_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -1156,6 +1176,7 @@ &tsadc {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-max.dts b/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-max.dts
index 8b1d35760c3bc..762f9fd966a42 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-max.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-max.dts
@@ -35,6 +35,10 @@ hdmi1_con_in: endpoint {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -55,9 +59,10 @@ &hdmi0_sound {
 };
 
 &hdmi1 {
-	pinctrl-names = "default";
+	frl-enable-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim0_tx1_cec &hdmim0_tx1_hpd
-			     &hdmim1_tx1_scl &hdmim1_tx1_sda>;
+			     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -102,6 +107,15 @@ &led_green_pwm {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
 
 	usb {
 		usb_otg_pwren: usb-otg-pwren {
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-plus.dts b/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-plus.dts
index 9950d1147e129..145986695720a 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-plus.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-plus.dts
@@ -121,6 +121,10 @@ &fan {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -141,6 +145,10 @@ hdmi0_out_con: endpoint {
 };
 
 &hdmi1 {
+	frl-enable-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim2_tx1_cec &hdmim0_tx1_hpd
+		     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -278,6 +286,16 @@ &pcie2x1l2 {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -333,6 +351,7 @@ &pwm2 {
 
 &pwm3 {
 	pinctrl-0 = <&pwm3m1_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -351,6 +370,7 @@ &u2phy1_otg {
 
 &uart9 {
 	pinctrl-0 = <&uart9m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-ultra.dts b/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-ultra.dts
index f8c6c080e418d..2b693dfb434c4 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-ultra.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5-ultra.dts
@@ -25,9 +25,10 @@ hdmi1_con_in: endpoint {
 };
 
 &hdmi1 {
-	pinctrl-names = "default";
+	frl-enable-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim0_tx1_cec &hdmim0_tx1_hpd
-			     &hdmim1_tx1_scl &hdmim1_tx1_sda>;
+			     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -64,6 +65,12 @@ &led_green_pwm {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	usb {
 		usb_otg_pwren: usb-otg-pwren {
 			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5.dtsi
index 3bceee9484582..4e9fe872b5811 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-orangepi-5.dtsi
@@ -822,6 +822,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-roc-rt.dts b/arch/arm64/boot/dts/rockchip/rk3588-roc-rt.dts
index 2d6fed2a84a38..02d532c77a266 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-roc-rt.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-roc-rt.dts
@@ -325,6 +325,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB0 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -345,6 +349,10 @@ &hdmi0_sound {
 };
 
 &hdmi1 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim2_tx1_cec &hdmim0_tx1_hpd
+		     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -583,6 +591,16 @@ led_pins: led-pins {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -1041,6 +1059,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts b/arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts
index f7dd01d6fa0ab..6f25e088f3189 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-rock-5-itx.dts
@@ -346,8 +346,10 @@ &gpu {
 };
 
 &hdmi1 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim0_tx1_cec &hdmim0_tx1_hpd
-		     &hdmim1_tx1_scl &hdmim1_tx1_sda>;
+		     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -433,7 +435,6 @@ vdd_npu_s0: regulator@42 {
 		reg = <0x42>;
 		fcs,suspend-voltage-selector = <1>;
 		regulator-name = "vdd_npu_s0";
-		regulator-always-on;
 		regulator-boot-on;
 		regulator-min-microvolt = <550000>;
 		regulator-max-microvolt = <950000>;
@@ -695,7 +696,17 @@ &pd_gpu {
 	domain-supply = <&vdd_gpu_s0>;
 };
 
+&pd_npu {
+	domain-supply = <&vdd_npu_s0>;
+};
+
 &pinctrl {
+	hdmi {
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		rtc_int: rtc-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -814,6 +825,36 @@ dp1_hpd: dp1-hpd {
 	};
 };
 
+&rknn_core_0 {
+	npu-supply = <&vdd_npu_s0>;
+	sram-supply = <&vdd_npu_s0>;
+	status = "okay";
+};
+
+&rknn_core_1 {
+	npu-supply = <&vdd_npu_s0>;
+	sram-supply = <&vdd_npu_s0>;
+	status = "okay";
+};
+
+&rknn_core_2 {
+	npu-supply = <&vdd_npu_s0>;
+	sram-supply = <&vdd_npu_s0>;
+	status = "okay";
+};
+
+&rknn_mmu_0 {
+	status = "okay";
+};
+
+&rknn_mmu_1 {
+	status = "okay";
+};
+
+&rknn_mmu_2 {
+	status = "okay";
+};
+
 &pwm14 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&pwm14m1_pins>;
@@ -886,6 +927,7 @@ spi_flash: flash@0 {
 
 &spdif_tx1 {
 	pinctrl-0 = <&spdif1m2_tx>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -1220,6 +1262,7 @@ &tsadc {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-5bp-5t.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-5bp-5t.dtsi
index bf4a1d2e55ca3..13aaf63ad0936 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-5bp-5t.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-rock-5b-5bp-5t.dtsi
@@ -183,6 +183,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -203,8 +207,10 @@ &hdmi0_sound {
 };
 
 &hdmi1 {
+	frl-enable-gpios = <&gpio4 RK_PA1 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim0_tx1_cec &hdmim0_tx1_hpd
-		     &hdmim1_tx1_scl &hdmim1_tx1_sda>;
+		     &hdmim1_tx1_scl &hdmim1_tx1_sda &hdmi1_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -505,6 +511,16 @@ &pd_npu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+
+		hdmi1_tx_on_h: hdmi1-tx-on-h {
+			rockchip,pins = <4 RK_PA1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -959,6 +975,7 @@ &tsadc {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-tiger-haikou.dts b/arch/arm64/boot/dts/rockchip/rk3588-tiger-haikou.dts
index caa43d1abf179..873fbeb8daa16 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-tiger-haikou.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-tiger-haikou.dts
@@ -172,8 +172,9 @@ &hdmi0 {
 	 * While HDMI-CEC is present on the Q7 connector, it is not
 	 * connected on Haikou itself.
 	 */
+	pinctrl-0 = <&hdmim0_tx0_hpd &hdmim1_tx0_scl &hdmim1_tx0_sda
+		     &hdmi0_tx_on_h>;
 	pinctrl-names = "default";
-	pinctrl-0 = <&hdmim0_tx0_hpd &hdmim1_tx0_scl &hdmim1_tx0_sda>;
 	status = "okay";
 };
 
@@ -273,6 +274,7 @@ &sdmmc {
 	cd-gpios = <&gpio0 RK_PA4 GPIO_ACTIVE_LOW>;
 	disable-wp;
 	pinctrl-0 = <&sdmmc_bus4 &sdmmc_cmd &sdmmc_clk>;
+	pinctrl-names = "default";
 	sd-uhs-sdr12;
 	sd-uhs-sdr25;
 	sd-uhs-sdr50;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi
index a0e97481afb7d..139f70e7bbd90 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-tiger.dtsi
@@ -148,13 +148,15 @@ &gpu {
 };
 
 &hdmi0 {
-	pinctrl-names = "default";
+	frl-enable-gpios = <&gpio0 RK_PD3 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim1_tx0_cec &hdmim0_tx0_hpd &hdmim1_tx0_scl
-		     &hdmim1_tx0_sda>;
+		     &hdmim1_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 };
 
 &i2c1 {
 	pinctrl-0 = <&i2c1m0_xfer>;
+	pinctrl-names = "default";
 };
 
 &i2c1m0_xfer {
@@ -167,6 +169,7 @@ &i2c1m0_xfer {
 
 &i2c2 {
 	pinctrl-0 = <&i2c2m3_xfer>;
+	pinctrl-names = "default";
 };
 
 &i2c2m3_xfer {
@@ -179,10 +182,12 @@ &i2c2m3_xfer {
 
 &i2c3 {
 	pinctrl-0 = <&i2c3m0_xfer>;
+	pinctrl-names = "default";
 };
 
 &i2c4 {
 	pinctrl-0 = <&i2c4m4_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	vdd_npu_s0: regulator@42 {
@@ -204,6 +209,7 @@ regulator-state-mem {
 
 &i2c5 {
 	pinctrl-0 = <&i2c5m1_xfer>;
+	pinctrl-names = "default";
 };
 
 &i2c5m1_xfer {
@@ -301,6 +307,7 @@ &i2c7m0_xfer {
 
 &i2c8 {
 	pinctrl-0 = <&i2c8m2_xfer>;
+	pinctrl-names = "default";
 };
 
 &mdio0 {
@@ -349,6 +356,12 @@ eth_reset: eth-reset {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <0 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	leds {
 		module_led_pin: module-led-pin {
 			rockchip,pins = <1 RK_PD3 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -434,6 +447,7 @@ &sdmmc {
 
 &spi0 {
 	pinctrl-0 = <&spi0m1_cs0 &spi0m1_cs1 &spi0m3_pins>;
+	pinctrl-names = "default";
 };
 
 &spi2 {
@@ -764,10 +778,12 @@ &tsadc {
 /* Routed to UART0 on the Q7 connector */
 &uart2 {
 	pinctrl-0 = <&uart2m2_xfer>;
+	pinctrl-names = "default";
 };
 
 /* Mule-ATtiny UPDI */
 &uart4 {
 	pinctrl-0 = <&uart4m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-toybrick-x0.dts b/arch/arm64/boot/dts/rockchip/rk3588-toybrick-x0.dts
index 5a428e00ab938..85ac072f64b9b 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-toybrick-x0.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588-toybrick-x0.dts
@@ -675,6 +675,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588-turing-rk1.dtsi b/arch/arm64/boot/dts/rockchip/rk3588-turing-rk1.dtsi
index b11d24dcc1806..9d5d4cfe01702 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588-turing-rk1.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588-turing-rk1.dtsi
@@ -704,11 +704,13 @@ &tsadc {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
 &uart9 {
 	pinctrl-0 = <&uart9m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-coolpi-4b.dts b/arch/arm64/boot/dts/rockchip/rk3588s-coolpi-4b.dts
index 189444d207797..2e8661ac7fcf4 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-coolpi-4b.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-coolpi-4b.dts
@@ -251,6 +251,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -272,6 +276,7 @@ &hdptxphy0 {
 
 &i2c0 {
 	pinctrl-0 = <&i2c0m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	vdd_cpu_big0_s0: regulator@42 {
@@ -332,6 +337,7 @@ regulator-state-mem {
 
 &i2c6 {
 	pinctrl-0 = <&i2c6m3_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	hym8563: rtc@51 {
@@ -348,6 +354,7 @@ hym8563: rtc@51 {
 
 &i2c7 {
 	pinctrl-0 = <&i2c7m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	es8316: audio-codec@10 {
@@ -373,6 +380,7 @@ &i2s0_mclk
 		     &i2s0_sclk
 		     &i2s0_sdi0
 		     &i2s0_sdo0>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	i2s0_8ch_p0: port {
@@ -396,6 +404,12 @@ &pd_gpu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -459,6 +473,7 @@ wifi_poweren_pin: wifi-poweren-pin {
 
 &pwm2 {
 	pinctrl-0 = <&pwm2m1_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -865,6 +880,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts b/arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts
index d6b62cd1b90b5..c4ddb78f9bb2f 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-evb1-v10.dts
@@ -457,6 +457,7 @@ usbc0_int: usbc0-int {
 
 &pwm12 {
 	pinctrl-0 = <&pwm12m1_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts b/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts
index 89618394c0bfb..230aac005e8f6 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-gameforce-ace.dts
@@ -302,9 +302,6 @@ amp_headphone: headphone-amplifier {
 
 	hdmi0-con {
 		compatible = "hdmi-connector";
-		ddc-en-gpios = <&gpio4 RK_PB3 GPIO_ACTIVE_HIGH>;
-		pinctrl-0 = <&hdmi0_en>;
-		pinctrl-names = "default";
 		type = "d";
 
 		port {
@@ -514,8 +511,9 @@ &gpu {
 
 &hdmi0 {
 	no-hpd;
+	frl-enable-gpios = <&gpio4 RK_PB3 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim0_tx0_cec>, <&hdmim0_tx0_scl>,
-		    <&hdmim0_tx0_sda>;
+		    <&hdmim0_tx0_sda>, <&hdmi0_tx_on_h>;
 	pinctrl-names = "default";
 	status = "okay";
 };
@@ -616,6 +614,7 @@ touchscreen@14 {
 
 &i2c4 {
 	pinctrl-0 = <&i2c4m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	ti_adc: adc@48 {
@@ -652,6 +651,7 @@ imu@68 {
 
 &i2c6 {
 	pinctrl-0 = <&i2c6m3_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	fusb302: typec@22 {
@@ -785,6 +785,7 @@ &i2s0_mclk
 		     &i2s0_sclk
 		     &i2s0_sdi0
 		     &i2s0_sdo0>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -893,7 +894,7 @@ charger_int_h: charger-int-h {
 	};
 
 	hdmi {
-		hdmi0_en: hdmi0-en {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
 			rockchip,pins =
 				<4 RK_PB3 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
@@ -1028,11 +1029,13 @@ wifi_host_wake_irq: wifi-host-wake-irq {
 
 &pwm12 {
 	pinctrl-0 = <&pwm12m1_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
 &pwm13 {
 	pinctrl-0 = <&pwm13m1_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -1434,11 +1437,13 @@ &u2phy0_otg {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
 &uart9 {
 	pinctrl-0 = <&uart9m2_xfer>, <&uart9m2_ctsn>, <&uart9m2_rtsn>;
+	pinctrl-names = "default";
 	uart-has-rtscts;
 	status = "okay";
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-indiedroid-nova.dts b/arch/arm64/boot/dts/rockchip/rk3588s-indiedroid-nova.dts
index 174d299cc6bb9..ed36c27c2320d 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-indiedroid-nova.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-indiedroid-nova.dts
@@ -277,8 +277,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim0_tx0_scl>, <&hdmim0_tx0_sda>,
-		    <&hdmim0_tx0_hpd>, <&hdmim0_tx0_cec>;
+		    <&hdmim0_tx0_hpd>, <&hdmim0_tx0_cec>,
+		    <&hdmi0_tx_on_h>;
 	pinctrl-names = "default";
 	status = "okay";
 };
@@ -366,6 +368,7 @@ regulator-state-mem {
 
 &i2c6 {
 	pinctrl-0 = <&i2c6m3_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	fusb302: typec-portc@22 {
@@ -438,6 +441,7 @@ rtc_hym8563: rtc@51 {
 
 &i2c7 {
 	pinctrl-0 = <&i2c7m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	es8388: audio-codec@11 {
@@ -517,6 +521,12 @@ bt_wake_host: bt-wake-host {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 
 		hym8563_int: hym8563-int {
@@ -965,6 +975,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-khadas-edge2.dts b/arch/arm64/boot/dts/rockchip/rk3588s-khadas-edge2.dts
index 2c22abaf40a82..833f27fdd1c6c 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-khadas-edge2.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-khadas-edge2.dts
@@ -194,6 +194,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB1 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -217,10 +221,6 @@ &hdptxphy0 {
 	status = "okay";
 };
 
-&hdmi0_sound {
-	status = "okay";
-};
-
 &i2c0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&i2c0m2_xfer>;
@@ -282,6 +282,12 @@ &pd_gpu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB1 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	vdd_sd {
 		vdd_sd_en: vdd-sd-en {
 			rockchip,pins = <1 RK_PB6 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -736,6 +742,7 @@ &tsadc {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-nanopi-r6.dtsi b/arch/arm64/boot/dts/rockchip/rk3588s-nanopi-r6.dtsi
index 1b6a59f7cabce..91b6eefd7abf5 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-nanopi-r6.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-nanopi-r6.dtsi
@@ -236,6 +236,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -404,6 +408,12 @@ lan2_led_pin: lan2-led-pin {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		rtc_int: rtc-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
@@ -841,6 +851,7 @@ &u2phy2_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-odroid-m2.dts b/arch/arm64/boot/dts/rockchip/rk3588s-odroid-m2.dts
index a72063c551401..4e98fa33492c6 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-odroid-m2.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-odroid-m2.dts
@@ -249,6 +249,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -438,6 +442,12 @@ &pd_gpu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	lcd {
 		lcd_pwren: lcd-pwren {
 			rockchip,pins = <4 RK_PA3 RK_FUNC_GPIO &pcfg_pull_none>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dtsi b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dtsi
index dafad29f98544..9da13f96f13a1 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-5.dtsi
@@ -181,6 +181,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -381,6 +385,12 @@ &pd_npu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	hym8563 {
 		hym8563_int: hym8563-int {
 			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -837,6 +847,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-cm5-base.dts b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-cm5-base.dts
index 06120b2db690a..0145e194841c8 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-cm5-base.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-cm5-base.dts
@@ -143,10 +143,11 @@ &gmac1_rgmii_clk
 };
 
 &hdmi0 {
-	pinctrl-names = "default";
+	frl-enable-gpios = <&gpio4 RK_PB5 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
 		     &hdmim0_tx0_scl &hdmim0_tx0_sda
-		     &hdmi_frl_pin>;
+		     &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -172,6 +173,7 @@ &hdptxphy0 {
 
 &i2c1 {
 	pinctrl-0 = <&i2c1m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	rtc@51 {
@@ -245,8 +247,8 @@ key1_pin: key1-pin {
 	};
 
 	hdmi {
-		hdmi_frl_pin: hdmi-frl-pin {
-			rockchip,pins = <4 RK_PB5 RK_FUNC_GPIO &pcfg_pull_up>;
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB5 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
 
@@ -279,6 +281,7 @@ &pwm4 {
 
 &pwm5 {
 	pinctrl-0 = <&pwm5m1_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -319,6 +322,7 @@ &u2phy2_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-cm5.dtsi b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-cm5.dtsi
index 32357eba4b789..5d3dc21c4d61d 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-cm5.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-orangepi-cm5.dtsi
@@ -64,6 +64,7 @@ &gpu {
 
 &i2c0 {
 	pinctrl-0 = <&i2c0m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	vdd_cpu_big0_s0: regulator@42 {
@@ -142,6 +143,7 @@ &spi2 {
 	assigned-clock-rates = <200000000>;
 	num-cs = <1>;
 	pinctrl-0 = <&spi2m2_cs0 &spi2m2_pins>;
+	pinctrl-names = "default";
 	status = "okay";
 
 	pmic@0 {
@@ -468,5 +470,6 @@ &tsadc {
 	 * through the CRU.
 	 */
 	pinctrl-0 = <&tsadc_gpio_func>;
+	pinctrl-names = "default";
 	status = "okay";
 };
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-radxa-cm5-io.dts b/arch/arm64/boot/dts/rockchip/rk3588s-radxa-cm5-io.dts
index f80d5a00a4bdd..af4a9bc015e16 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-radxa-cm5-io.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-radxa-cm5-io.dts
@@ -120,6 +120,10 @@ &gmac1 {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -217,6 +221,12 @@ usbc0_int: usbc0-int {
 		};
 	};
 
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	usb {
 		vcc5v0_host_en: vcc5v0-host-en {
 			rockchip,pins = <1 RK_PA0 RK_FUNC_GPIO &pcfg_pull_none>;
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts b/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts
index 7e179862da6e5..d534d662c40f1 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-roc-pc.dts
@@ -224,6 +224,10 @@ &gpu {
 };
 
 &hdmi0 {
+	frl-enable-gpios = <&gpio4 RK_PB2 GPIO_ACTIVE_LOW>;
+	pinctrl-0 = <&hdmim0_tx0_cec &hdmim0_tx0_hpd
+		     &hdmim0_tx0_scl &hdmim0_tx0_sda &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -367,9 +371,9 @@ &pd_gpu {
 };
 
 &pinctrl {
-	hym8563 {
-		hym8563_int: hym8563-int {
-			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB2 RK_FUNC_GPIO &pcfg_pull_none>;
 		};
 	};
 
@@ -379,6 +383,12 @@ hp_detect: hp-detect {
 		};
 	};
 
+	hym8563 {
+		hym8563_int: hym8563-int {
+			rockchip,pins = <0 RK_PB0 RK_FUNC_GPIO &pcfg_pull_up>;
+		};
+	};
+
 	leds {
 		led_pins: led-pins {
 			rockchip,pins = <1 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>,
@@ -800,6 +810,7 @@ &uart2 {
 
 &uart7 {
 	pinctrl-0 = <&uart7m2_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-rock-5a.dts b/arch/arm64/boot/dts/rockchip/rk3588s-rock-5a.dts
index 0991f6a211900..a76734da982f9 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-rock-5a.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-rock-5a.dts
@@ -323,11 +323,13 @@ &gmac1_rgmii_clk
 };
 
 &hdmi0 {
-	pinctrl-names = "default";
+	frl-enable-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim0_tx0_cec
 		     &hdmim1_tx0_hpd
 		     &hdmim0_tx0_scl
-		     &hdmim0_tx0_sda>;
+		     &hdmim0_tx0_sda
+		     &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -373,6 +375,12 @@ &pd_gpu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	leds {
 		io_led: io-led {
 			rockchip,pins = <3 RK_PD5 RK_FUNC_GPIO &pcfg_pull_none>;
@@ -819,6 +827,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/rockchip/rk3588s-rock-5c.dts b/arch/arm64/boot/dts/rockchip/rk3588s-rock-5c.dts
index 7fe42f4ff8279..e4784885c2678 100644
--- a/arch/arm64/boot/dts/rockchip/rk3588s-rock-5c.dts
+++ b/arch/arm64/boot/dts/rockchip/rk3588s-rock-5c.dts
@@ -258,11 +258,13 @@ &gpu {
 };
 
 &hdmi0 {
-	pinctrl-names = "default";
+	frl-enable-gpios = <&gpio4 RK_PB6 GPIO_ACTIVE_LOW>;
 	pinctrl-0 = <&hdmim0_tx0_cec
 		     &hdmim1_tx0_hpd
 		     &hdmim0_tx0_scl
-		     &hdmim0_tx0_sda>;
+		     &hdmim0_tx0_sda
+		     &hdmi0_tx_on_h>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
@@ -461,6 +463,12 @@ &pd_gpu {
 };
 
 &pinctrl {
+	hdmi {
+		hdmi0_tx_on_h: hdmi0-tx-on-h {
+			rockchip,pins = <4 RK_PB6 RK_FUNC_GPIO &pcfg_pull_none>;
+		};
+	};
+
 	leds {
 		led_pins: led-pins {
 			rockchip,pins = <3 RK_PC4 RK_FUNC_GPIO &pcfg_pull_none>,
@@ -922,6 +930,7 @@ &u2phy3_host {
 
 &uart2 {
 	pinctrl-0 = <&uart2m0_xfer>;
+	pinctrl-names = "default";
 	status = "okay";
 };
 
diff --git a/arch/arm64/boot/dts/ti/Makefile b/arch/arm64/boot/dts/ti/Makefile
index 5269c9619b65c..21db60cd19de6 100644
--- a/arch/arm64/boot/dts/ti/Makefile
+++ b/arch/arm64/boot/dts/ti/Makefile
@@ -9,18 +9,32 @@
 # alphabetically.
 
 # Boards with AM62x SoC
+k3-am625-verdin-nonwifi-dahlia-dsi-to-hdmi-dtbs := k3-am625-verdin-nonwifi-dahlia.dtb \
+	k3-am625-verdin-dsi-to-hdmi.dtbo
+k3-am625-verdin-nonwifi-dev-dsi-to-hdmi-dtbs := k3-am625-verdin-nonwifi-dev.dtb \
+	k3-am625-verdin-dsi-to-hdmi.dtbo
+k3-am625-verdin-wifi-dahlia-dsi-to-hdmi-dtbs := k3-am625-verdin-wifi-dahlia.dtb \
+	k3-am625-verdin-dsi-to-hdmi.dtbo
+k3-am625-verdin-wifi-dev-dsi-to-hdmi-dtbs := k3-am625-verdin-wifi-dev.dtb \
+	k3-am625-verdin-dsi-to-hdmi.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am625-beagleplay.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-beagleplay-csi2-ov5640.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am625-beagleplay-csi2-tevi-ov5640.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am625-phyboard-lyra-rdk.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-sk.dtb
+dtb-$(CONFIG_ARCH_K3) += k3-am625-tqma62xx-mba62xx.dtb
+dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-dsi-to-hdmi.dtbo
+dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-nonwifi-dahlia-dsi-to-hdmi.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-nonwifi-dahlia.dtb
+dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-nonwifi-dev-dsi-to-hdmi.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-nonwifi-dev.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-nonwifi-ivy.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-nonwifi-mallow.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-nonwifi-yavia.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-nonwifi-zinnia.dtb
+dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-wifi-dahlia-dsi-to-hdmi.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-wifi-dahlia.dtb
+dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-wifi-dev-dsi-to-hdmi.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-wifi-dev.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-wifi-ivy.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-wifi-mallow.dtb
@@ -29,6 +43,7 @@ dtb-$(CONFIG_ARCH_K3) += k3-am625-verdin-wifi-zinnia.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am62x-phyboard-lyra-gpio-fan.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am62-lp-sk.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am62-lp-sk-nand.dtbo
+dtb-$(CONFIG_ARCH_K3) += k3-am62-phyboard-lyra-oldi-lcd185.dtbo
 dtb-$(CONFIG_ARCH_K3) += k3-am62-pocketbeagle2.dtb
 dtb-$(CONFIG_ARCH_K3) += k3-am6254atl-sk.dtb
 
@@ -183,6 +198,8 @@ k3-am625-phyboard-lyra-disable-spi-nor-dtbs := k3-am625-phyboard-lyra-rdk.dtb \
 	k3-am6xx-phycore-disable-spi-nor.dtbo
 k3-am625-phyboard-lyra-gpio-fan-dtbs := k3-am625-phyboard-lyra-rdk.dtb \
 	k3-am62x-phyboard-lyra-gpio-fan.dtbo
+k3-am625-phyboard-lyra-oldi-lcd185-dtbs := k3-am625-phyboard-lyra-rdk.dtb \
+	k3-am62-phyboard-lyra-oldi-lcd185.dtbo
 k3-am625-phyboard-lyra-qspi-nor-dtbs := k3-am625-phyboard-lyra-rdk.dtb \
 	k3-am6xx-phycore-qspi-nor.dtbo
 k3-am625-sk-csi2-imx219-dtbs := k3-am625-sk.dtb \
@@ -287,6 +304,7 @@ dtb- += k3-am625-beagleplay-csi2-ov5640.dtb \
 	k3-am625-phyboard-lyra-disable-rtc.dtb \
 	k3-am625-phyboard-lyra-disable-spi-nor.dtb \
 	k3-am625-phyboard-lyra-gpio-fan.dtb \
+	k3-am625-phyboard-lyra-oldi-lcd185.dtb \
 	k3-am625-phyboard-lyra-qspi-nor.dtb \
 	k3-am625-sk-csi2-imx219.dtb \
 	k3-am625-sk-csi2-ov5640.dtb \
diff --git a/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts b/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts
index 8a556fbbe08b7..dfd65c00f5b95 100644
--- a/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts
+++ b/arch/arm64/boot/dts/ti/k3-am62-lp-sk.dts
@@ -206,6 +206,7 @@ tps65219: pmic@30 {
 
 		interrupt-parent = <&gic500>;
 		interrupts = <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>;
+		system-power-controller;
 
 		regulators {
 			buck1_reg: buck1 {
diff --git a/arch/arm64/boot/dts/ti/k3-am62-phyboard-lyra-oldi-lcd185.dtso b/arch/arm64/boot/dts/ti/k3-am62-phyboard-lyra-oldi-lcd185.dtso
new file mode 100644
index 0000000000000..d9f75ff28165e
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am62-phyboard-lyra-oldi-lcd185.dtso
@@ -0,0 +1,185 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (C) 2022-2026 PHYTEC Messtechnik GmbH
+ * Author: Wadim Egorov <w.egorov@phytec.de>
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/pwm/pwm.h>
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
+#include "k3-pinctrl.h"
+
+&{/} {
+	display {
+		compatible = "lincolntech,lcd185-101ct";
+		backlight = <&backlight>;
+		power-supply = <&vdd_usb_5v0>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+				dual-lvds-odd-pixels;
+				lcd_in0: endpoint {
+					remote-endpoint = <&oldi_0_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+				dual-lvds-even-pixels;
+				lcd_in1: endpoint {
+					remote-endpoint = <&oldi_1_out>;
+				};
+			};
+		};
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		brightness-levels = <0 4 8 16 32 64 128 255>;
+		default-brightness-level = <6>;
+		enable-gpios = <&gpio_exp 5 GPIO_ACTIVE_HIGH>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&bl_pwm_pins_default>;
+		pwms = <&epwm0 1 50000 0>;
+	};
+
+	vdd_usb_5v0: regulator-vdd-usb5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vdd-usb5v0";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+};
+
+&dss {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_oldi0_pins_default>, <&main_dss0_pins_default>;
+};
+
+&dss_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	/* VP1: Output to OLDI */
+	port@0 {
+		reg = <0>;
+		#address-cells = <1>;
+		#size-cells = <0>;
+
+		dpi0_out0: endpoint@0 {
+			reg = <0>;
+			remote-endpoint = <&oldi_0_in>;
+		};
+
+		dpi0_out1: endpoint@1 {
+			reg = <1>;
+			remote-endpoint = <&oldi_1_in>;
+		};
+	};
+};
+
+&epwm0 {
+	status = "okay";
+};
+
+&main_i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	touchscreen@5d {
+		compatible = "goodix,gt928";
+		reg = <0x5d>;
+
+		pinctrl-names = "default";
+		pinctrl-0 = <&touch_screen_pins_default>;
+
+		interrupt-parent = <&main_gpio0>;
+		interrupts = <19 IRQ_TYPE_LEVEL_LOW>;
+
+		reset-gpios = <&main_gpio0 18 GPIO_ACTIVE_HIGH>;
+		irq-gpios = <&main_gpio0 19 GPIO_ACTIVE_HIGH>;
+	};
+};
+
+&main_pmx0 {
+	bl_pwm_pins_default: bl-pwm-default-pins {
+		pinctrl-single,pins = <
+			AM62X_IOPAD(0x01b8, PIN_INPUT, 2) /* (C13) SPI0_CS1.EHRPWM0_B */
+		>;
+	};
+
+	touch_screen_pins_default: touch-screen-default-pins {
+		pinctrl-single,pins = <
+			AM62X_IOPAD(0x048, PIN_OUTPUT, 7) /* (N25) GPMC0_AD3.GPIO0_18 - RST */
+			AM62X_IOPAD(0x04c, PIN_INPUT, 7) /* (P24) GPMC0_AD4.GPIO0_19 - INT */
+		>;
+	};
+
+	main_oldi0_pins_default: main-oldi0-default-pins {
+		pinctrl-single,pins = <
+			AM62X_IOPAD(0x0260, PIN_OUTPUT, 0) /* (AA5) OLDI0_A0N */
+			AM62X_IOPAD(0x025c, PIN_OUTPUT, 0) /* (Y6) OLDI0_A0P */
+			AM62X_IOPAD(0x0268, PIN_OUTPUT, 0) /* (AD3) OLDI0_A1N */
+			AM62X_IOPAD(0x0264, PIN_OUTPUT, 0) /* (AB4) OLDI0_A1P */
+			AM62X_IOPAD(0x0270, PIN_OUTPUT, 0) /* (Y8) OLDI0_A2N */
+			AM62X_IOPAD(0x026c, PIN_OUTPUT, 0) /* (AA8) OLDI0_A2P */
+			AM62X_IOPAD(0x0278, PIN_OUTPUT, 0) /* (AB6) OLDI0_A3N */
+			AM62X_IOPAD(0x0274, PIN_OUTPUT, 0) /* (AA7) OLDI0_A3P */
+			AM62X_IOPAD(0x0280, PIN_OUTPUT, 0) /* (AC6) OLDI0_A4N */
+			AM62X_IOPAD(0x027c, PIN_OUTPUT, 0) /* (AC5) OLDI0_A4P */
+			AM62X_IOPAD(0x0288, PIN_OUTPUT, 0) /* (AE5) OLDI0_A5N */
+			AM62X_IOPAD(0x0284, PIN_OUTPUT, 0) /* (AD6) OLDI0_A5P */
+			AM62X_IOPAD(0x0290, PIN_OUTPUT, 0) /* (AE6) OLDI0_A6N */
+			AM62X_IOPAD(0x028c, PIN_OUTPUT, 0) /* (AD7) OLDI0_A6P */
+			AM62X_IOPAD(0x0298, PIN_OUTPUT, 0) /* (AD8) OLDI0_A7N */
+			AM62X_IOPAD(0x0294, PIN_OUTPUT, 0) /* (AE7) OLDI0_A7P */
+			AM62X_IOPAD(0x02a0, PIN_OUTPUT, 0) /* (AD4) OLDI0_CLK0N */
+			AM62X_IOPAD(0x029c, PIN_OUTPUT, 0) /* (AE3) OLDI0_CLK0P */
+			AM62X_IOPAD(0x02a8, PIN_OUTPUT, 0) /* (AE4) OLDI0_CLK1N */
+			AM62X_IOPAD(0x02a4, PIN_OUTPUT, 0) /* (AD5) OLDI0_CLK1P */
+		>;
+	};
+};
+
+&oldi0 {
+	ti,companion-oldi = <&oldi1>;
+	status = "okay";
+};
+
+&oldi0_port0 {
+	oldi_0_in: endpoint {
+		remote-endpoint = <&dpi0_out0>;
+	};
+};
+
+&oldi0_port1 {
+	oldi_0_out: endpoint {
+		remote-endpoint = <&lcd_in0>;
+	};
+};
+
+&oldi1 {
+	ti,secondary-oldi;
+	status = "okay";
+};
+
+&oldi1_port0 {
+	oldi_1_in: endpoint {
+		remote-endpoint = <&dpi0_out1>;
+	};
+};
+
+&oldi1_port1 {
+	oldi_1_out: endpoint {
+		remote-endpoint = <&lcd_in1>;
+	};
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts b/arch/arm64/boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts
new file mode 100644
index 0000000000000..efc00d2a860a7
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am625-tqma62xx-mba62xx.dts
@@ -0,0 +1,1033 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (C) 2021-2022 Texas Instruments Incorporated - https://www.ti.com/
+ * Copyright (c) 2023-2026 TQ-Systems GmbH <linux@ew.tq-group.com>, D-82229 Seefeld, Germany.
+ * Author: Nora Schiffer
+ */
+
+/dts-v1/;
+
+#include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/input/input.h>
+#include <dt-bindings/leds/common.h>
+#include <dt-bindings/net/ti-dp83867.h>
+#include <dt-bindings/pwm/pwm.h>
+#include "k3-am625-tqma62xx.dtsi"
+
+/ {
+	compatible = "tq,am625-tqma6254-mba62xx", "tq,am625-tqma6254",
+		     "ti,am625";
+	model = "TQ-Systems TQMa62xx SoM on MBa62xx carrier board";
+	chassis-type = "embedded";
+
+	aliases {
+		can0 = &mcu_mcan0;
+		can1 = &mcu_mcan1;
+		ethernet0 = &cpsw_port1;
+		ethernet1 = &cpsw_port2;
+		i2c1 = &main_i2c1;
+		mmc1 = &sdhci1;
+		mmc2 = &sdhci2;
+		serial0 = &main_uart0;
+		serial1 = &mcu_uart0;
+		spi1 = &main_spi0;
+		usb0 = &usb0;
+		usb1 = &usb1;
+	};
+
+	chosen {
+		stdout-path = &main_uart0;
+	};
+
+	backlight: backlight {
+		compatible = "pwm-backlight";
+		pinctrl-names = "default";
+		pinctrl-0 = <&backlight_pins>;
+		enable-gpios = <&main_gpio0 38 GPIO_ACTIVE_HIGH>;
+		/* Enabled by display overlays */
+		status = "disabled";
+	};
+
+	gpio-keys {
+		compatible = "gpio-keys";
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_key_pins>;
+
+		user-button {
+			label = "USER_BUTTON";
+			linux,code = <BTN_0>;
+			gpios = <&main_gpio0 40 GPIO_ACTIVE_LOW>;
+		};
+	};
+
+	gpio-leds {
+		compatible = "gpio-leds";
+		pinctrl-names = "default";
+		pinctrl-0 = <&gpio_led_pins>;
+
+		led-1 {
+			gpios = <&main_gpio0 41 GPIO_ACTIVE_HIGH>;
+			color = <LED_COLOR_ID_GREEN>;
+			function = LED_FUNCTION_INDICATOR;
+		};
+
+		led-2 {
+			gpios = <&main_gpio0 42 GPIO_ACTIVE_HIGH>;
+			color = <LED_COLOR_ID_YELLOW>;
+			function = LED_FUNCTION_INDICATOR;
+		};
+	};
+
+	panel: panel {
+		pinctrl-names = "default";
+		pinctrl-0 = <&lvds_panel_pins>;
+		enable-gpios = <&main_gpio0 36 GPIO_ACTIVE_HIGH>;
+		power-supply = <&reg_lvds_pwr>;
+		/* Completed and enabled by display overlays */
+		status = "disabled";
+	};
+
+	fan0: pwm-fan {
+		compatible = "pwm-fan";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pwm_fan_pins>;
+		fan-supply = <&reg_pwm_fan>;
+		#cooling-cells = <2>;
+		/* typical 25 kHz -> 40.000 nsec */
+		pwms = <&epwm0 1 40000 PWM_POLARITY_INVERTED>;
+		cooling-levels = <0 32 64 128 196 240>;
+		pulses-per-revolution = <2>;
+		interrupt-parent = <&main_gpio1>;
+		interrupts = <30 IRQ_TYPE_EDGE_FALLING>;
+		/*
+		 * 4-pin connector for optional fan - disabled by default, as
+		 * PWM period may need to be adjusted for the used fan
+		 */
+		status = "disabled";
+	};
+
+	wifi_pwrseq: pwrseq-wifi {
+		compatible = "mmc-pwrseq-simple";
+		pinctrl-names = "default";
+		pinctrl-0 = <&main_mmc2_pwrseq_pins>;
+		reset-gpios = <&main_gpio0 44 GPIO_ACTIVE_HIGH>;
+	};
+
+	reg_1v8: regulator-1v8 {
+		compatible = "regulator-fixed";
+		regulator-name = "V_1V8_MBA";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	reg_3v3_sd: regulator-3v3-sd {
+		/* TPS22963CYZTP */
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&reg_3v3_sd_pins>;
+		regulator-name = "V_3V3_SD";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		enable-active-high;
+		vin-supply = <&reg_3v3>;
+		gpio = <&main_gpio1 8 GPIO_ACTIVE_HIGH>;
+		bootph-all;
+	};
+
+	reg_lvds_pwr: regulator-lvds-pwr {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&reg_lvds_pwr_pins>;
+		regulator-name = "LVDS0_PWR";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		enable-active-high;
+		vin-supply = <&reg_3v3>;
+		gpio = <&main_gpio0 61 GPIO_ACTIVE_HIGH>;
+	};
+
+	reg_pwm_fan: regulator-pwm-fan {
+		compatible = "regulator-fixed";
+		pinctrl-names = "default";
+		pinctrl-0 = <&reg_pwm_fan_pins>;
+		regulator-name = "FAN_PWR";
+		regulator-min-microvolt = <12000000>;
+		regulator-max-microvolt = <12000000>;
+		gpio = <&main_gpio0 62 GPIO_ACTIVE_HIGH>;
+		enable-active-high;
+	};
+
+	sound: sound {
+		compatible = "simple-audio-card";
+		simple-audio-card,name = "tqm-tlv320aic32";
+		simple-audio-card,widgets =
+			"Line Out",	"Line Out",
+			"Line In",	"Line In",
+			"Microphone",	"Microphone Jack";
+		simple-audio-card,routing =
+			"Line Out",		"LOL",
+			"Line Out",		"LOR",
+			"IN1_L",		"Line In",
+			"IN1_R",		"Line In",
+			"IN3_L",		"Microphone Jack",
+			"Microphone Jack",	"Mic Bias";
+		simple-audio-card,format = "i2s";
+		simple-audio-card,bitclock-master = <&sound_master>;
+		simple-audio-card,frame-master = <&sound_master>;
+
+		simple-audio-card,cpu {
+			sound-dai = <&mcasp1>;
+		};
+
+		sound_master: simple-audio-card,codec {
+			sound-dai = <&tlv320aic32x4>;
+			clocks = <&audio_refclk0>;
+		};
+	};
+};
+
+&audio_refclk0 {
+	/* Set parent to POSTDIV1_16FFT_MAIN_1_HSDIVOUT6, for 96MHz clock output */
+	assigned-clock-parents = <&k3_clks 157 7>;
+};
+
+&cpsw3g {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_rgmii1_pins>, <&main_rgmii2_pins>;
+	status = "okay";
+};
+
+&cpsw_port1 {
+	phy-mode = "rgmii-id";
+	phy-handle = <&cpsw3g_phy0>;
+};
+
+&cpsw_port2 {
+	phy-mode = "rgmii-id";
+	phy-handle = <&cpsw3g_phy3>;
+};
+
+&cpsw3g_mdio {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mdio1_pins>;
+	status = "okay";
+
+	cpsw3g_phy0: ethernet-phy@0 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x0>;
+		reset-gpios = <&main_gpio1 11 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <1000>;
+		reset-deassert-us = <1000>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+		ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
+	};
+
+	cpsw3g_phy3: ethernet-phy@3 {
+		compatible = "ethernet-phy-ieee802.3-c22";
+		reg = <0x3>;
+		reset-gpios = <&main_gpio1 12 GPIO_ACTIVE_LOW>;
+		reset-assert-us = <1000>;
+		reset-deassert-us = <1000>;
+		ti,fifo-depth = <DP83867_PHYCR_FIFO_DEPTH_4_B_NIB>;
+		ti,clk-output-sel = <DP83867_CLK_O_SEL_OFF>;
+	};
+};
+
+&dss {
+	pinctrl-names = "default";
+	pinctrl-0 = <&dss_pins>;
+};
+
+&epwm0 {
+	status = "okay";
+};
+
+&epwm1 {
+	status = "okay";
+};
+
+&main_gpio0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_gpio0_pins>, <&main_gpio0_pr0_pins>;
+	gpio-line-names =
+		"", "", "", "", /* 0-3 */
+		"", "", "", "", /* 4-7 */
+		"", "", "", "", /* 8-11 */
+		"ADC_SYNC", "ADC_RST#", "ADC_DATA_RDY", "", /* 12-15 */
+		"", "", "", "", /* 16-19 */
+		"", "", "", "", /* 20-23 */
+		"", "", "", "", /* 24-27 */
+		"", "", "", "", /* 28-31 */
+		"", "", "", "", /* 32-35 */
+		"", "", "", "BG95_PWRKEY", /* 36-39 */
+		"", "", "", "BG95_RESET", /* 40-43 */
+		"", "", "", "", /* 44-47 */
+		"", "", "", "", /* 48-51 */
+		"", "", "", "", /* 52-55 */
+		"", "", "", "", /* 56-59 */
+		"", "", "", "", /* 60-63 */
+		"", "", "", "", /* 64-67 */
+		"", "", "", "", /* 68-71 */
+		"ADC_INT"; /* 72- */
+	bootph-all;
+};
+
+&main_gpio1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_gpio1_pins>, <&main_gpio1_pr0_pins>;
+	bootph-all;
+
+	/* No overcurrent handling in USB host driver - pin is hogged for now */
+	line7-hog {
+		gpio-hog;
+		gpios = <7 GPIO_ACTIVE_HIGH>;
+		line-name = "USB0_VBUS_OC#";
+		input;
+	};
+};
+
+&main_i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c1_pins>;
+	clock-frequency = <400000>;
+	status = "okay";
+
+	tlv320aic32x4: audio-codec@18 {
+		compatible = "ti,tlv320aic32x4";
+		reg = <0x18>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&audio_codec_pins>;
+		#sound-dai-cells = <0>;
+		clock-names = "mclk";
+		clocks = <&audio_refclk0>;
+		reset-gpios = <&main_gpio0 33 GPIO_ACTIVE_LOW>;
+		iov-supply = <&reg_1v8>;
+		ldoin-supply = <&reg_3v3>;
+	};
+};
+
+&main_spi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_spi0_pins>;
+	ti,pindir-d0-out-d1-in;
+	status = "okay";
+
+	/* adc@0: NXP NAFE13388 */
+};
+
+&main0_thermal {
+	trips {
+		main0_active0: trip-active0 {
+			temperature = <40000>;
+			hysteresis = <5000>;
+			type = "active";
+		};
+
+		main0_active1: trip-active1 {
+			temperature = <48000>;
+			hysteresis = <3000>;
+			type = "active";
+		};
+
+		main0_active2: trip-active2 {
+			temperature = <60000>;
+			hysteresis = <10000>;
+			type = "active";
+		};
+	};
+
+	cooling-maps {
+		map1 {
+			trip = <&main0_active0>;
+			cooling-device = <&fan0 1 1>;
+		};
+
+		map2 {
+			trip = <&main0_active1>;
+			cooling-device = <&fan0 2 2>;
+		};
+
+		map3 {
+			trip = <&main0_active2>;
+			cooling-device = <&fan0 3 3>;
+		};
+	};
+};
+
+&main1_thermal {
+	trips {
+		main1_active0: trip-active0 {
+			temperature = <40000>;
+			hysteresis = <5000>;
+			type = "active";
+		};
+
+		main1_active1: trip-active1 {
+			temperature = <48000>;
+			hysteresis = <3000>;
+			type = "active";
+		};
+
+		main1_active2: trip-active2 {
+			temperature = <60000>;
+			hysteresis = <10000>;
+			type = "active";
+		};
+	};
+
+	cooling-maps {
+		map1 {
+			trip = <&main1_active0>;
+			cooling-device = <&fan0 1 1>;
+		};
+
+		map2 {
+			trip = <&main1_active1>;
+			cooling-device = <&fan0 2 2>;
+		};
+
+		map3 {
+			trip = <&main1_active2>;
+			cooling-device = <&fan0 3 3>;
+		};
+	};
+};
+
+/* Main console */
+&main_uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_uart0_pins>;
+	bootph-all;
+	status = "okay";
+};
+
+/*
+ * IOT module - GNSS UART
+ *
+ * Board configuration must not enable UART trace output for TIFS firmware
+ */
+&main_uart1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_uart1_pins>;
+	/* IOT module uses USB by default, UART can be enabled as fallback */
+	status = "disabled";
+};
+
+/* Bluetooth module */
+&main_uart2 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_uart2_pins>;
+	/* Bluetooth module uses SDIO by default, UART can be enabled as fallback */
+	status = "disabled";
+};
+
+/* IOT module - main UART */
+&main_uart5 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_uart5_pins>;
+	/* IOT module uses USB by default, UART can be enabled as fallback */
+	status = "disabled";
+};
+
+&mcasp1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mcasp1_pins>;
+	#sound-dai-cells = <0>;
+	op-mode = <0>; /* MCASP_IIS_MODE */
+	tdm-slots = <2>;
+	serial-dir = < /* 0: INACTIVE, 1: TX, 2: RX */
+	       1 0 2 0
+	       0 0 0 0
+	       0 0 0 0
+	       0 0 0 0
+	>;
+	tx-num-evt = <0>;
+	rx-num-evt = <0>;
+	status = "okay";
+};
+
+&mcu_gpio0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_gpio0_pins>;
+	gpio-line-names =
+		"EN_DIG_OUT_1", "EN_DIG_OUT_2", "STATUS_OUT_1", "STATUS_OUT_2", /* 0-3 */
+		"EN_DIG_OUT_3", "", "", "V_VPP_EN", /* 4-7 */
+		"", "", "", "EN_DIG_OUT_4", /* 8-11 */
+		"STATUS_OUT_3", "", "", "", /* 12-15 */
+		"", "STATUS_OUT_4", "DIG_IN_1", "DIG_IN_2", /* 16-19 */
+		"DIG_IN_3", "", "", "DIG_IN_4"; /* 20-23 */
+	bootph-all;
+};
+
+&mcu_mcan0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_mcan0_pins>;
+	status = "okay";
+};
+
+&mcu_mcan1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_mcan1_pins>;
+	status = "okay";
+};
+
+&mcu_uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&mcu_uart0_pins>;
+	rs485-rts-active-low;
+	linux,rs485-enabled-at-boot-time;
+	status = "okay";
+};
+
+&reg_sd {
+	pinctrl-names = "default";
+	pinctrl-0 = <&reg_sd_pins>;
+	gpios = <&main_gpio0 31 GPIO_ACTIVE_LOW>;
+	bootph-all;
+	status = "okay";
+};
+
+&sdhci1 {
+	/* SD-card */
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mmc1_pins>;
+	vmmc-supply = <&reg_3v3_sd>;
+	vqmmc-supply = <&reg_sd>;
+	bus-width = <4>;
+	disable-wp;
+	no-mmc;
+	no-sdio;
+	bootph-all;
+	ti,driver-strength-ohm = <50>;
+	status = "okay";
+};
+
+&sdhci2 {
+	/* WLAN */
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mmc2_pins>;
+	mmc-pwrseq = <&wifi_pwrseq>;
+	keep-power-in-suspend;
+	cap-power-off-card;
+	bus-width = <4>;
+	non-removable;
+	no-mmc;
+	no-sd;
+	ti,driver-strength-ohm = <50>;
+	ti,fails-without-test-cd;
+	status = "okay";
+};
+
+&usbss0 {
+	bootph-all;
+	ti,vbus-divider;
+	status = "okay";
+};
+
+&usbss1 {
+	ti,vbus-divider;
+	status = "okay";
+};
+
+&usb0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_usb0_pins>;
+	dr_mode = "otg";
+	usb-role-switch;
+	bootph-all;
+
+	connector {
+		compatible = "gpio-usb-b-connector", "usb-b-connector";
+		id-gpios = <&main_gpio0 71 GPIO_ACTIVE_HIGH>;
+		type = "micro";
+	};
+};
+
+&usb1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_usb1_hub_pins>;
+	#address-cells = <1>;
+	#size-cells = <0>;
+	dr_mode = "host";
+
+	usb1_hub_2_0: hub@1 {
+		compatible = "usb424,2514";
+		reg = <1>;
+		reset-gpios = <&main_gpio1 23 GPIO_ACTIVE_HIGH>;
+		vdd-supply = <&reg_3v3>;
+		vdda-supply = <&reg_3v3>;
+	};
+};
+
+&wkup_uart0 {
+	/* WKUP UART0 is used by DM firmware */
+	status = "reserved";
+};
+
+&main_pmx0 {
+	audio_codec_pins: audio-codec-pins {
+		pinctrl-single,pins = <
+			/* (L24) GPMC0_OEn_REn.GPIO0_33 */
+			AM62X_IOPAD(0x088, PIN_OUTPUT, 7)
+			/* (A15) UART0_CTSn.AUDIO_EXT_REFCLK0 */
+			AM62X_IOPAD(0x1d0, PIN_OUTPUT, 5)
+		>;
+	};
+
+	backlight_pins: backlight-pins {
+		pinctrl-single,pins = <
+			/* (V25) GPMC0_WAIT1.GPIO0_38 */
+			AM62X_IOPAD(0x009c, PIN_OUTPUT, 7)
+			/* (E18) MCASP0_AXR0.EHRPWM1_B */
+			AM62X_IOPAD(0x01a0, PIN_OUTPUT, 6)
+		>;
+	};
+
+	dss_pins: dss-pins {
+		pinctrl-single,pins = <
+			/* (AA5) OLDI0_A0N */
+			AM62X_IOPAD(0x0260, PIN_OUTPUT, 0)
+			/* (Y6) OLDI0_A0P */
+			AM62X_IOPAD(0x025c, PIN_OUTPUT, 0)
+			/* (AD3) OLDI0_A1N */
+			AM62X_IOPAD(0x0268, PIN_OUTPUT, 0)
+			/* (AB4) OLDI0_A1P */
+			AM62X_IOPAD(0x0264, PIN_OUTPUT, 0)
+			/* (Y8) OLDI0_A2N */
+			AM62X_IOPAD(0x0270, PIN_OUTPUT, 0)
+			/* (AA8) OLDI0_A2P */
+			AM62X_IOPAD(0x026c, PIN_OUTPUT, 0)
+			/* (AB6) OLDI0_A3N */
+			AM62X_IOPAD(0x0278, PIN_OUTPUT, 0)
+			/* (AA7) OLDI0_A3P */
+			AM62X_IOPAD(0x0274, PIN_OUTPUT, 0)
+			/* (AC6) OLDI0_A4N */
+			AM62X_IOPAD(0x0280, PIN_OUTPUT, 0)
+			/* (AC5) OLDI0_A4P */
+			AM62X_IOPAD(0x027c, PIN_OUTPUT, 0)
+			/* (AE5) OLDI0_A5N */
+			AM62X_IOPAD(0x0288, PIN_OUTPUT, 0)
+			/* (AD6) OLDI0_A5P */
+			AM62X_IOPAD(0x0284, PIN_OUTPUT, 0)
+			/* (AE6) OLDI0_A6N */
+			AM62X_IOPAD(0x0290, PIN_OUTPUT, 0)
+			/* (AD7) OLDI0_A6P */
+			AM62X_IOPAD(0x028c, PIN_OUTPUT, 0)
+			/* (AD8) OLDI0_A7N */
+			AM62X_IOPAD(0x0298, PIN_OUTPUT, 0)
+			/* (AE7) OLDI0_A7P */
+			AM62X_IOPAD(0x0294, PIN_OUTPUT, 0)
+			/* (AD4) OLDI0_CLK0N */
+			AM62X_IOPAD(0x02a0, PIN_OUTPUT, 0)
+			/* (AE3) OLDI0_CLK0P */
+			AM62X_IOPAD(0x029c, PIN_OUTPUT, 0)
+			/* (AE4) OLDI0_CLK1N */
+			AM62X_IOPAD(0x02a8, PIN_OUTPUT, 0)
+			/* (AD5) OLDI0_CLK1P */
+			AM62X_IOPAD(0x02a4, PIN_OUTPUT, 0)
+		>;
+	};
+
+	csi_clk_pins: csi-clk-pins {
+		pinctrl-single,pins = <
+			/* (B18) MCASP0_AXR1.EHRPWM1_A */
+			AM62X_IOPAD(0x019c, PIN_OUTPUT, 6)
+		>;
+	};
+
+	gpio_key_pins: gpio-key-pins {
+		pinctrl-single,pins = <
+			/* (M22) GPMC0_DIR.GPIO0_40 */
+			AM62X_IOPAD(0x0a4, PIN_INPUT, 7)
+		>;
+	};
+
+	gpio_led_pins: gpio-led-pins {
+		pinctrl-single,pins = <
+			/* (M21) GPMC0_CSn0.GPIO0_41 */
+			AM62X_IOPAD(0x0a8, PIN_OUTPUT, 7)
+			/* (L21) GPMC0_CSn1.GPIO0_42 */
+			AM62X_IOPAD(0x0ac, PIN_OUTPUT, 7)
+		>;
+	};
+
+	lvds_panel_pins: lvds-panel-pins {
+		pinctrl-single,pins = <
+			/* (N20) GPMC0_BE1n.GPIO0_36 - LVDS0_RESET# */
+			AM62X_IOPAD(0x0094, PIN_OUTPUT, 7)
+		>;
+	};
+
+	main_gpio0_pins: main-gpio0-pins {
+		pinctrl-single,pins = <
+			/* Control GPIOs for IOT Module */
+			/* (K25) GPMC0_WPn.GPIO0_39 - BG95_PWRKEY */
+			AM62X_IOPAD(0x0a0, PIN_OUTPUT, 7)
+			/* (K22) GPMC0_CSn2.GPIO0_43 - BG95_RESET */
+			AM62X_IOPAD(0x0b0, PIN_OUTPUT, 7)
+		>;
+	};
+
+	main_gpio0_pr0_pins: main-gpio0-pr0-pins {
+		pinctrl-single,pins = <
+			/* (N24) GPMC0_AD2.GPIO0_17  */
+			AM62X_IOPAD(0x0044, PIN_INPUT, 7)
+			/* (N25) GPMC0_AD3.GPIO0_18  */
+			AM62X_IOPAD(0x0048, PIN_INPUT, 7)
+			/* (P24) GPMC0_AD4.GPIO0_19  */
+			AM62X_IOPAD(0x004c, PIN_INPUT, 7)
+			/* (P22) GPMC0_AD5.GPIO0_20  */
+			AM62X_IOPAD(0x0050, PIN_INPUT, 7)
+			/* (W25) VOUT0_DATA2.GPIO0_47  */
+			AM62X_IOPAD(0x00c0, PIN_INPUT, 7)
+			/* (W24) VOUT0_DATA3.GPIO0_48  */
+			AM62X_IOPAD(0x00c4, PIN_INPUT, 7)
+			/* (Y25) VOUT0_DATA4.GPIO0_49  */
+			AM62X_IOPAD(0x00c8, PIN_INPUT, 7)
+			/* (Y24) VOUT0_DATA5.GPIO0_50  */
+			AM62X_IOPAD(0x00cc, PIN_INPUT, 7)
+		>;
+	};
+
+	main_gpio1_pins: main-gpio1-pins {
+		pinctrl-single,pins = <
+			/* (B19) MCASP0_AXR3.GPIO1_7 - USB0_VBUS_OC# */
+			AM62X_IOPAD(0x0194, PIN_INPUT, 7)
+		>;
+	};
+
+	main_gpio1_pr0_pins: main-gpio1-pr0-pins {
+		pinctrl-single,pins = <
+			/* (C15) MCAN0_TX.GPIO1_24 */
+			AM62X_IOPAD(0x01d8, PIN_INPUT, 7)
+			/* (E15) MCAN0_RX.GPIO1_25 */
+			AM62X_IOPAD(0x01dc, PIN_INPUT, 7)
+		>;
+	};
+
+	main_i2c1_pins: main-i2c1-pins {
+		pinctrl-single,pins = <
+			/* (B17) I2C1_SCL */
+			AM62X_IOPAD(0x1e8, PIN_INPUT, 0)
+			/* (A17) I2C1_SDA */
+			AM62X_IOPAD(0x1ec, PIN_INPUT, 0)
+		>;
+	};
+
+	main_mcasp1_pins: main-mcasp1-pins {
+		pinctrl-single,pins = <
+			/* (M24) GPMC0_BE0N_CLE.MCASP1_ACLKX */
+			AM62X_IOPAD(0x090, PIN_INPUT, 2)
+			/* (U23) GPMC0_WAIT0.MCASP1_AFSX */
+			AM62X_IOPAD(0x098, PIN_INPUT, 2)
+			/* (L25) GPMC0_WEN.MCASP1_AXR0 */
+			AM62X_IOPAD(0x08c, PIN_OUTPUT, 2)
+			/* (L23) GPMC0_ADVN_ALE.MCASP1_AXR2 */
+			AM62X_IOPAD(0x084, PIN_INPUT, 2)
+		>;
+	};
+
+	main_mmc1_pins: main-mmc1-pins {
+		pinctrl-single,pins = <
+			/* (A21) MMC1_CMD */
+			AM62X_IOPAD(0x23c, PIN_INPUT, 0)
+			/* (B22) MMC1_CLK */
+			AM62X_IOPAD(0x234, PIN_INPUT, 0)
+			/* (A22) MMC1_DAT0 */
+			AM62X_IOPAD(0x230, PIN_INPUT, 0)
+			/* (B21) MMC1_DAT1 */
+			AM62X_IOPAD(0x22c, PIN_INPUT, 0)
+			/* (C21) MMC1_DAT2 */
+			AM62X_IOPAD(0x228, PIN_INPUT, 0)
+			/* (D22) MMC1_DAT3 */
+			AM62X_IOPAD(0x224, PIN_INPUT, 0)
+			/* (D17) MMC1_SDCD */
+			AM62X_IOPAD(0x240, PIN_INPUT, 0)
+		>;
+		bootph-all;
+	};
+
+	main_mmc2_pins: main-mmc2-pins {
+		pinctrl-single,pins = <
+			/* (C24) MMC2_CMD */
+			AM62X_IOPAD(0x120, PIN_INPUT, 0)
+			/* (D25) MMC2_CLK */
+			AM62X_IOPAD(0x118, PIN_INPUT, 0)
+			/* (B24) MMC2_DAT0 */
+			AM62X_IOPAD(0x114, PIN_INPUT, 0)
+			/* (C25) MMC2_DAT1 */
+			AM62X_IOPAD(0x110, PIN_INPUT, 0)
+			/* (E23) MMC2_DAT2 */
+			AM62X_IOPAD(0x10c, PIN_INPUT, 0)
+			/* (D24) MMC2_DAT3 */
+			AM62X_IOPAD(0x108, PIN_INPUT, 0)
+			/* (#N/A) MMC2_CLKB */
+			AM62X_IOPAD(0x11c, PIN_INPUT, 0)
+		>;
+	};
+
+	main_mmc2_pwrseq_pins: main-mmc2-pwrseq-pins {
+		pinctrl-single,pins = <
+			/* (K24) GPMC0_CSn3.GPIO0_44 - WIFI-BT_EN */
+			AM62X_IOPAD(0x00b4, PIN_OUTPUT, 7)
+		>;
+	};
+
+	main_mdio1_pins: main-mdio1-pins {
+		pinctrl-single,pins = <
+			/* (B20) MCASP0_ACLKX.GPIO1_11 - RESET_RGMII1# */
+			AM62X_IOPAD(0x1a4, PIN_OUTPUT, 7)
+			/* (D20) MCASP0_AFSX.GPIO1_12 - RESET_RGMII2# */
+			AM62X_IOPAD(0x1a8, PIN_OUTPUT, 7)
+
+			/* (AD24) MDIO0_MDC */
+			AM62X_IOPAD(0x160, PIN_OUTPUT, 0)
+			/* (AB22) MDIO0_MDIO */
+			AM62X_IOPAD(0x15c, PIN_INPUT, 0)
+		>;
+	};
+
+	main_rgmii1_pins: main-rgmii1-pins {
+		pinctrl-single,pins = <
+			/* (AB17) RGMII1_RD0 */
+			AM62X_IOPAD(0x14c, PIN_INPUT, 0)
+			/* (AC17) RGMII1_RD1 */
+			AM62X_IOPAD(0x150, PIN_INPUT, 0)
+			/* (AB16) RGMII1_RD2 */
+			AM62X_IOPAD(0x154, PIN_INPUT, 0)
+			/* (AA15) RGMII1_RD3 */
+			AM62X_IOPAD(0x158, PIN_INPUT, 0)
+			/* (AD17) RGMII1_RXC */
+			AM62X_IOPAD(0x148, PIN_INPUT, 0)
+			/* (AE17) RGMII1_RX_CTL */
+			AM62X_IOPAD(0x144, PIN_INPUT, 0)
+			/* (AE20) RGMII1_TD0 */
+			AM62X_IOPAD(0x134, PIN_OUTPUT, 0)
+			/* (AD20) RGMII1_TD1 */
+			AM62X_IOPAD(0x138, PIN_OUTPUT, 0)
+			/* (AE18) RGMII1_TD2 */
+			AM62X_IOPAD(0x13c, PIN_OUTPUT, 0)
+			/* (AD18) RGMII1_TD3 */
+			AM62X_IOPAD(0x140, PIN_OUTPUT, 0)
+			/* (AE19) RGMII1_TXC */
+			AM62X_IOPAD(0x130, PIN_OUTPUT, 0)
+			/* (AD19) RGMII1_TX_CTL */
+			AM62X_IOPAD(0x12c, PIN_OUTPUT, 0)
+		>;
+	};
+
+	main_rgmii2_pins: main-rgmii2-pins {
+		pinctrl-single,pins = <
+			/* (AE23) RGMII2_RD0 */
+			AM62X_IOPAD(0x184, PIN_INPUT, 0)
+			/* (AB20) RGMII2_RD1 */
+			AM62X_IOPAD(0x188, PIN_INPUT, 0)
+			/* (AC21) RGMII2_RD2 */
+			AM62X_IOPAD(0x18c, PIN_INPUT, 0)
+			/* (AE22) RGMII2_RD3 */
+			AM62X_IOPAD(0x190, PIN_INPUT, 0)
+			/* (AD23) RGMII2_RXC */
+			AM62X_IOPAD(0x180, PIN_INPUT, 0)
+			/* (AD22) RGMII2_RX_CTL */
+			AM62X_IOPAD(0x17c, PIN_INPUT, 0)
+			/* (Y18) RGMII2_TD0 */
+			AM62X_IOPAD(0x16c, PIN_OUTPUT, 0)
+			/* (AA18) RGMII2_TD1 */
+			AM62X_IOPAD(0x170, PIN_OUTPUT, 0)
+			/* (AD21) RGMII2_TD2 */
+			AM62X_IOPAD(0x174, PIN_OUTPUT, 0)
+			/* (AC20) RGMII2_TD3 */
+			AM62X_IOPAD(0x178, PIN_OUTPUT, 0)
+			/* (AE21) RGMII2_TXC */
+			AM62X_IOPAD(0x168, PIN_OUTPUT, 0)
+			/* (AA19) RGMII2_TX_CTL */
+			AM62X_IOPAD(0x164, PIN_OUTPUT, 0)
+		>;
+	};
+
+	main_spi0_pins: main-spi0-pins {
+		pinctrl-single,pins = <
+			/* (A14) SPI0_CLK */
+			AM62X_IOPAD(0x1bc, PIN_OUTPUT, 0)
+			/* (A13) SPI0_CS0 */
+			AM62X_IOPAD(0x1b4, PIN_OUTPUT, 0)
+			/* (B13) SPI0_D0 */
+			AM62X_IOPAD(0x1c0, PIN_OUTPUT, 0)
+			/* (B14) SPI0_D1 */
+			AM62X_IOPAD(0x1c4, PIN_INPUT, 0)
+		>;
+	};
+
+	main_spi0_adc_pins: main-spi0-adc-pins {
+		pinctrl-single,pins = <
+			/* (G21) OSPI0_CSn1.GPIO0_12 - ADC_SYNC */
+			AM62X_IOPAD(0x030, PIN_INPUT, 7)
+			/* (H21) OSPI0_CSn2.GPIO0_13 - ADC_RST# */
+			AM62X_IOPAD(0x034, PIN_OUTPUT, 7)
+			/* (E24) OSPI0_CSn3.GPIO0_14 - ADC_DATA_RDY */
+			AM62X_IOPAD(0x038, PIN_INPUT, 7)
+			/* (B23) MMC2_SDWP.GPIO0_72 - ADC_INT# */
+			AM62X_IOPAD(0x128, PIN_INPUT, 7)
+		>;
+	};
+
+	main_uart0_pins: main-uart0-pins {
+		pinctrl-single,pins = <
+			/* (D14) UART0_RXD */
+			AM62X_IOPAD(0x1c8, PIN_INPUT, 0)
+			/* (E14) UART0_TXD */
+			AM62X_IOPAD(0x1cc, PIN_OUTPUT, 0)
+		>;
+		bootph-all;
+	};
+
+	main_uart1_pins: main-uart1-pins {
+		pinctrl-single,pins = <
+			/* (E19) MCASP0_AFSR.UART1_RXD */
+			AM62X_IOPAD(0x1ac, PIN_INPUT, 2)
+			/* (A20) MCASP0_ACLKR.UART1_TXD */
+			AM62X_IOPAD(0x1b0, PIN_OUTPUT, 2)
+		>;
+	};
+
+	main_uart2_pins: main-uart2-pins {
+		pinctrl-single,pins = <
+			/* (U22) VOUT0_DATA0.UART2_RXD */
+			AM62X_IOPAD(0x0b8, PIN_INPUT, 4)
+			/* (V24) VOUT0_DATA1.UART2_TXD */
+			AM62X_IOPAD(0x0bc, PIN_OUTPUT, 4)
+			/* (AC24) VOUT0_PCLK.UART2_CTS# */
+			AM62X_IOPAD(0x104, PIN_INPUT, 4)
+			/* (AC25) VOUT0_VSYNC.UART2_RTS# */
+			AM62X_IOPAD(0x100, PIN_OUTPUT, 4)
+		>;
+	};
+
+	main_uart5_pins: main-uart5-pins {
+		pinctrl-single,pins = <
+			/* (Y23) VOUT0_DATA6.UART5_RXD */
+			AM62X_IOPAD(0x0d0, PIN_INPUT, 4)
+			/* (AA25) VOUT0_DATA7.UART5_TXD */
+			AM62X_IOPAD(0x0d4, PIN_OUTPUT, 4)
+		>;
+	};
+
+	main_usb0_pins: main-usb0-pins {
+		pinctrl-single,pins = <
+			/* (C20) USB0_DRVVBUS */
+			AM62X_IOPAD(0x254, PIN_OUTPUT, 0)
+			/* (A23) MMC2_SDCD.GPIO0_71 */
+			AM62X_IOPAD(0x124, PIN_INPUT, 7)
+		>;
+	};
+
+	main_usb1_hub_pins: main-usb1-hub-pins {
+		pinctrl-single,pins = <
+			/* (B15) UART0_RTSn.GPIO1_23 - USB_HUB_RESET */
+			AM62X_IOPAD(0x01d4, PIN_OUTPUT, 7)
+		>;
+	};
+
+	pwm_fan_pins: pwm-fan-pins {
+		pinctrl-single,pins = <
+			/* (C13) SPI0_CS1.EHRPWM0_B - FAN_PWM */
+			AM62X_IOPAD(0x01b8, PIN_OUTPUT, 2)
+			/* (A18) EXT_REFCLK1.GPIO1_30 - FAN_RPM */
+			AM62X_IOPAD(0x01f0, PIN_INPUT, 7)
+		>;
+	};
+
+	reg_3v3_sd_pins: reg-3v3-sd-pins {
+		pinctrl-single,pins = <
+			/* (A19) MCASP0_AXR2.GPIO1_8 */
+			AM62X_IOPAD(0x198, PIN_OUTPUT, 7)
+		>;
+		bootph-all;
+	};
+
+	reg_lvds_pwr_pins: reg-lvds-pwr-pins {
+		pinctrl-single,pins = <
+			/* (AB24) VOUT0_HSYNC.GPIO0_61 */
+			AM62X_IOPAD(0x00f8, PIN_OUTPUT, 7)
+		>;
+	};
+
+	reg_pwm_fan_pins: reg-pwm-fan-pins {
+		pinctrl-single,pins = <
+			/* (Y20) VOUT0_DE.GPIO0_62 */
+			AM62X_IOPAD(0x00fc, PIN_OUTPUT, 7)
+		>;
+	};
+
+	reg_sd_pins: reg-sd-pins {
+		pinctrl-single,pins = <
+			/* (P25) GPMC0_CLK.GPIO0_31 */
+			AM62X_IOPAD(0x07c, PIN_OUTPUT, 7)
+		>;
+		bootph-all;
+	};
+};
+
+&mcu_pmx0 {
+	mcu_gpio0_pins: mcu-gpio0-pins {
+		pinctrl-single,pins = <
+			/* (E8) MCU_SPI0_CS0.MCU_GPIO0_0 - EN_DIG_OUT1 */
+			AM62X_MCU_IOPAD(0x000, PIN_OUTPUT, 7)
+			/* (B8) MCU_SPI0_CS1.MCU_GPIO0_1 - EN_DIG_OUT2 */
+			AM62X_MCU_IOPAD(0x004, PIN_OUTPUT, 7)
+			/* (A7) MCU_SPI0_CLK.MCU_GPIO0_2 - STATUS_OUT1 */
+			AM62X_MCU_IOPAD(0x008, PIN_INPUT, 7)
+			/* (D9) MCU_SPI0_D0.MCU_GPIO0_3 - STATUS_OUT2 */
+			AM62X_MCU_IOPAD(0x00c, PIN_INPUT, 7)
+			/* (C9) MCU_SPI0_D1.MCU_GPIO0_4 - EN_DIG_OUT3 */
+			AM62X_MCU_IOPAD(0x010, PIN_OUTPUT, 7)
+			/* (C6) WKUP_UART0_CTSn.MCU_GPIO0_11 - EN_DIG_OUT4 */
+			AM62X_MCU_IOPAD(0x02c, PIN_OUTPUT, 7)
+			/* (A4) WKUP_UART0_RTSn.MCU_GPIO0_12 - STATUS_OUT3 */
+			AM62X_MCU_IOPAD(0x030, PIN_INPUT, 7)
+			/* (A8) MCU_I2C0_SCL.MCU_GPIO0_17 - STATUS_OUT4 */
+			AM62X_MCU_IOPAD(0x044, PIN_INPUT, 7)
+			/* (D10) MCU_I2C0_SDA.MCU_GPIO0_18 - DIG_IN_1 */
+			AM62X_MCU_IOPAD(0x048, PIN_INPUT, 7)
+			/* (B9) WKUP_I2C0_SCL.MCU_GPIO0_19 - DIG_IN_2 */
+			AM62X_MCU_IOPAD(0x04c, PIN_INPUT, 7)
+			/* (A9) WKUP_I2C0_SDA.MCU_GPIO0_20 - DIG_IN_3 */
+			AM62X_MCU_IOPAD(0x050, PIN_INPUT, 7)
+			/* (A12) WKUP_CLKOUT0.MCU_GPIO0_23 - DIG_IN_4 */
+			AM62X_MCU_IOPAD(0x084, PIN_INPUT, 7)
+			/* (A6) MCU_UART0_CTSn.MCU_GPIO0_7 - V_VPP_EN */
+			AM62X_MCU_IOPAD(0x01c, PIN_OUTPUT, 7)
+		>;
+	};
+	mcu_mcan0_pins: mcu-mcan0-pins {
+		pinctrl-single,pins = <
+			/* (B3) MCU_MCAN0_RX */
+			AM62X_MCU_IOPAD(0x038, PIN_INPUT, 0)
+			/* (D6) MCU_MCAN0_TX */
+			AM62X_MCU_IOPAD(0x034, PIN_OUTPUT, 0)
+		>;
+	};
+
+	mcu_mcan1_pins: mcu-mcan1-pins {
+		pinctrl-single,pins = <
+			/* (D4) MCU_MCAN1_RX */
+			AM62X_MCU_IOPAD(0x040, PIN_INPUT, 0)
+			/* (E5) MCU_MCAN1_TX */
+			AM62X_MCU_IOPAD(0x03c, PIN_OUTPUT, 0)
+		>;
+	};
+
+	mcu_uart0_pins: mcu-uart0-pins {
+		pinctrl-single,pins = <
+			/* (B5) MCU_UART0_RXD */
+			AM62X_MCU_IOPAD(0x014, PIN_INPUT, 0)
+			/* (A5) MCU_UART0_TXD */
+			AM62X_MCU_IOPAD(0x018, PIN_OUTPUT, 0)
+			/* (B6) MCU_UART0_RTS# */
+			AM62X_MCU_IOPAD(0x020, PIN_OUTPUT, 0)
+		>;
+	};
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi b/arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi
new file mode 100644
index 0000000000000..72288678cd01f
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am625-tqma62xx.dtsi
@@ -0,0 +1,360 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Copyright (C) 2021-2022 Texas Instruments Incorporated - https://www.ti.com/
+ * Copyright (c) 2023-2026 TQ-Systems GmbH <linux@ew.tq-group.com>, D-82229 Seefeld, Germany.
+ * Author: Nora Schiffer
+ */
+
+#include "k3-am625.dtsi"
+
+/ {
+	aliases {
+		i2c0 = &main_i2c0;
+		mmc0 = &sdhci0;
+		spi0 = &ospi0;
+	};
+
+	memory@80000000 {
+		device_type = "memory";
+		/* 1G RAM */
+		reg = <0x00000000 0x80000000 0x00000000 0x40000000>;
+		bootph-pre-ram;
+	};
+
+	reserved_memory: reserved-memory {
+		#address-cells = <2>;
+		#size-cells = <2>;
+		ranges;
+
+		ramoops@9c700000 {
+			compatible = "ramoops";
+			reg = <0x00 0x9c700000 0x00 0x00100000>;
+			record-size = <0x8000>;
+			console-size = <0x8000>;
+			ftrace-size = <0x00>;
+			pmsg-size = <0x8000>;
+		};
+
+		/* global cma region */
+		linux,cma {
+			compatible = "shared-dma-pool";
+			reusable;
+			size = <0x00 0x8000000>;
+			linux,cma-default;
+		};
+
+		rtos_ipc_memory_region: memory@9c800000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0x9c800000 0x00 0x00300000>;
+			no-map;
+		};
+
+		wkup_r5fss0_core0_dma_memory_region: memory@9da00000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0x9da00000 0x00 0x100000>;
+			no-map;
+		};
+
+		wkup_r5fss0_core0_memory_region: memory@9db00000 {
+			compatible = "shared-dma-pool";
+			reg = <0x00 0x9db00000 0x00 0xc00000>;
+			no-map;
+		};
+
+		secure_tfa_ddr: tfa@9e780000 {
+			reg = <0x00 0x9e780000 0x00 0x80000>;
+			alignment = <0x1000>;
+			no-map;
+		};
+
+		secure_ddr: optee@9e800000 {
+			reg = <0x00 0x9e800000 0x00 0x01800000>; /* for OP-TEE */
+			alignment = <0x1000>;
+			no-map;
+		};
+	};
+
+	reg_3v3: regulator-3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "V_3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+		regulator-boot-on;
+	};
+
+	reg_sd: regulator-sd {
+		/* Output of TPS6521902RSM */
+		compatible = "regulator-gpio";
+		regulator-name = "V_VDDSHV5";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-boot-on;
+		vin-supply = <&reg_ldo1>;
+		states = <1800000 0x0>,
+			 <3300000 0x1>;
+		/* Controlling GPIO set by base board */
+		status = "disabled";
+	};
+};
+
+&main_i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_i2c0_pins>;
+	clock-frequency = <400000>;
+	bootph-pre-ram;
+	status = "okay";
+
+	tps65219: pmic@30 {
+		compatible = "ti,tps65219";
+		reg = <0x30>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pmic_irq_pins>;
+		interrupt-parent = <&gic500>;
+		interrupts = <GIC_SPI 224 IRQ_TYPE_LEVEL_HIGH>;
+		interrupt-controller;
+		#interrupt-cells = <1>;
+		buck1-supply = <&reg_3v3>;
+		buck2-supply = <&reg_3v3>;
+		buck3-supply = <&reg_3v3>;
+		ldo1-supply = <&reg_3v3>;
+		ldo2-supply = <&reg_buck2>;
+		ldo3-supply = <&reg_3v3>;
+		ldo4-supply = <&reg_3v3>;
+		system-power-controller;
+		ti,power-button;
+
+		regulators {
+			reg_buck1: buck1 {
+				regulator-name = "V_VDD_CORE";
+				regulator-min-microvolt = <750000>;
+				regulator-max-microvolt = <750000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_buck2: buck2 {
+				regulator-name = "V_1V8";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_buck3: buck3 {
+				regulator-name = "V_1V1";
+				regulator-min-microvolt = <1100000>;
+				regulator-max-microvolt = <1100000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_ldo1: ldo1 {
+				/* Actual voltage of LDO1 is controlled by GPIO, see reg_sd */
+				regulator-name = "V_VDDSHV5_3V3";
+				regulator-min-microvolt = <3300000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-allow-bypass;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_ldo2: ldo2 {
+				regulator-name = "V_0V85";
+				regulator-min-microvolt = <850000>;
+				regulator-max-microvolt = <850000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_ldo3: ldo3 {
+				regulator-name = "V_1V8A";
+				regulator-min-microvolt = <1800000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-boot-on;
+				regulator-always-on;
+			};
+
+			reg_ldo4: ldo4 {
+				/* Unused */
+				regulator-name = "V_VLDO4";
+				regulator-boot-on;
+			};
+		};
+	};
+
+	tmp1075: temperature-sensor@4a {
+		compatible = "ti,tmp1075";
+		reg = <0x4a>;
+		vs-supply = <&reg_buck2>;
+	};
+
+	eeprom0: eeprom@50 {
+		compatible = "st,24c02", "atmel,24c02";
+		reg = <0x50>;
+		vcc-supply = <&reg_buck2>;
+		pagesize = <16>;
+		read-only;
+		bootph-pre-ram;
+	};
+
+	pcf85063: rtc@51 {
+		compatible = "nxp,pcf85063a";
+		reg = <0x51>;
+		quartz-load-femtofarads = <12500>;
+	};
+
+	eeprom1: eeprom@54 {
+		compatible = "st,24c64", "atmel,24c64";
+		reg = <0x54>;
+		vcc-supply = <&reg_buck2>;
+		pagesize = <32>;
+	};
+};
+
+&a53_opp_table {
+	opp-1400000000 {
+		opp-hz = /bits/ 64 <1400000000>;
+		opp-supported-hw = <0x01 0x0004>;
+		opp-suspend;
+		clock-latency-ns = <6000000>;
+		/*
+		 * Enabled by bootloader if supported. The PMIC configuration
+		 * will be adjusted accordingly, as a higher core voltage is
+		 * required for 1400MHz operation.
+		 */
+		status = "disabled";
+	};
+};
+
+&ospi0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&ospi0_pins>;
+	bootph-all;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0x0>;
+		vcc-supply = <&reg_buck2>;
+		spi-tx-bus-width = <8>;
+		spi-rx-bus-width = <8>;
+		spi-max-frequency = <84000000>;
+		bootph-all;
+		cdns,tshsl-ns = <60>;
+		cdns,tsd2d-ns = <60>;
+		cdns,tchsh-ns = <60>;
+		cdns,tslch-ns = <60>;
+		cdns,read-delay = <2>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			/* Filled by bootloader */
+		};
+	};
+};
+
+&sdhci0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&main_mmc0_pins>;
+	non-removable;
+	disable-wp;
+	no-sd;
+	no-sdio;
+	bootph-all;
+	ti,driver-strength-ohm = <50>;
+	status = "okay";
+};
+
+&wkup_rtc0 {
+	/*
+	 * Erratum i2327: We can't guarantee that the TQMa62xx will boot fast
+	 * enough for U-Boot to apply the workaround within one second after
+	 * power-on. Keep the RTC disabled to avoid RTC interrupt issues.
+	 *
+	 * The external RTC of the TQMa62xx should be used instead.
+	 *
+	 * If needed, the RTC can be enabled in a baseboard DTS, as long as
+	 * boot is fast enough on all relevant boot media.
+	 */
+	status = "disabled";
+};
+
+&main_pmx0 {
+	main_i2c0_pins: main-i2c0-pins {
+		pinctrl-single,pins = <
+			/* (B16) I2C0_SCL */
+			AM62X_IOPAD(0x1e0, PIN_INPUT, 0)
+			/* (A16) I2C0_SDA */
+			AM62X_IOPAD(0x1e4, PIN_INPUT, 0)
+		>;
+		bootph-pre-ram;
+	};
+
+	main_mmc0_pins: main-mmc0-pins {
+		pinctrl-single,pins = <
+			/* (Y3) MMC0_CMD */
+			AM62X_IOPAD(0x220, PIN_INPUT, 0)
+			/* (AB1) MMC0_CLK */
+			AM62X_IOPAD(0x218, PIN_INPUT, 0)
+			/* (AA2) MMC0_DAT0 */
+			AM62X_IOPAD(0x214, PIN_INPUT, 0)
+			/* (AA1) MMC0_DAT1 */
+			AM62X_IOPAD(0x210, PIN_INPUT, 0)
+			/* (AA3) MMC0_DAT2 */
+			AM62X_IOPAD(0x20c, PIN_INPUT, 0)
+			/* (Y4) MMC0_DAT3 */
+			AM62X_IOPAD(0x208, PIN_INPUT, 0)
+			/* (AB2) MMC0_DAT4 */
+			AM62X_IOPAD(0x204, PIN_INPUT, 0)
+			/* (AC1) MMC0_DAT5 */
+			AM62X_IOPAD(0x200, PIN_INPUT, 0)
+			/* (AD2) MMC0_DAT6 */
+			AM62X_IOPAD(0x1fc, PIN_INPUT, 0)
+			/* (AC2) MMC0_DAT7 */
+			AM62X_IOPAD(0x1f8, PIN_INPUT, 0)
+		>;
+		bootph-all;
+	};
+
+	ospi0_pins: ospi0-pins {
+		pinctrl-single,pins = <
+			/* (H24) OSPI0_CLK */
+			AM62X_IOPAD(0x000, PIN_OUTPUT, 0)
+			/* (F23) OSPI0_CSn0 */
+			AM62X_IOPAD(0x02c, PIN_OUTPUT, 0)
+			/* (E25) OSPI0_D0 */
+			AM62X_IOPAD(0x00c, PIN_INPUT, 0)
+			/* (G24) OSPI0_D1 */
+			AM62X_IOPAD(0x010, PIN_INPUT, 0)
+			/* (F25) OSPI0_D2 */
+			AM62X_IOPAD(0x014, PIN_INPUT, 0)
+			/* (F24) OSPI0_D3 */
+			AM62X_IOPAD(0x018, PIN_INPUT, 0)
+			/* (J23) OSPI0_D4 */
+			AM62X_IOPAD(0x01c, PIN_INPUT, 0)
+			/* (J25) OSPI0_D5 */
+			AM62X_IOPAD(0x020, PIN_INPUT, 0)
+			/* (H25) OSPI0_D6 */
+			AM62X_IOPAD(0x024, PIN_INPUT, 0)
+			/* (J22) OSPI0_D7 */
+			AM62X_IOPAD(0x028, PIN_INPUT, 0)
+			/* (J24) OSPI0_DQS */
+			AM62X_IOPAD(0x008, PIN_INPUT, 0)
+			/* (G25) OSPI0_LBCLKO */
+			AM62X_IOPAD(0x004, PIN_INPUT, 0)
+		>;
+	};
+
+	pmic_irq_pins: pmic-irq-pins {
+		pinctrl-single,pins = <
+			/* (D16) EXTINTn */
+			AM62X_IOPAD(0x01f4, PIN_INPUT_PULLUP, 0)
+		>;
+	};
+};
+
+#include "k3-am62-ti-ipc-firmware.dtsi"
diff --git a/arch/arm64/boot/dts/ti/k3-am625-verdin-dsi-to-hdmi.dtso b/arch/arm64/boot/dts/ti/k3-am625-verdin-dsi-to-hdmi.dtso
new file mode 100644
index 0000000000000..a219faf0dcc63
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-am625-verdin-dsi-to-hdmi.dtso
@@ -0,0 +1,94 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR MIT
+/*
+ * Copyright (c) Toradex
+ *
+ * Toradex DSI to HDMI Adapter on Verdin DSI_1
+ *
+ * https://developer.toradex.com/hardware/accessories/add-ons/dsi-hdmi-adapter
+ */
+
+/dts-v1/;
+/plugin/;
+
+#include <dt-bindings/gpio/gpio.h>
+
+&{/} {
+	hdmi-connector {
+		compatible = "hdmi-connector";
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_pwm3_dsi_gpio>;
+		ddc-i2c-bus = <&main_i2c2>;
+		/* Verdin PWM_3_DSI (SODIMM 19) */
+		hpd-gpios = <&main_gpio1 17 GPIO_ACTIVE_HIGH>;
+		type = "a";
+
+		port {
+			hdmi_connector_in: endpoint {
+				remote-endpoint = <&dsi_hdmi_bridge_out>;
+			};
+		};
+	};
+};
+
+&dsi_bridge {
+	status = "okay";
+};
+
+&dsi_bridge_ports {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	port@1 {
+		reg = <1>;
+
+		dsi_bridge_out: endpoint {
+			remote-endpoint = <&dsi_hdmi_bridge_in>;
+		};
+	};
+};
+
+&dss {
+	status = "okay";
+};
+
+/* Verdin PWM_3_DSI (SODIMM 19) repurposed as HDMI HPD GPIO */
+&epwm1 {
+	status = "disabled";
+};
+
+/* Verdin I2C_1 */
+&main_i2c1 {
+	#address-cells = <1>;
+	#size-cells = <0>;
+
+	hdmi@48 {
+		compatible = "lontium,lt8912b";
+		reg = <0x48>;
+		pinctrl-names = "default";
+		pinctrl-0 = <&pinctrl_dsi1_bkl_en>;
+		/* Verdin GPIO_10_DSI (SODIMM 21) */
+		reset-gpios = <&main_gpio0 30 GPIO_ACTIVE_LOW>;
+
+		ports {
+			#address-cells = <1>;
+			#size-cells = <0>;
+
+			port@0 {
+				reg = <0>;
+
+				dsi_hdmi_bridge_in: endpoint {
+					data-lanes = <1 2 3 4>;
+					remote-endpoint = <&dsi_bridge_out>;
+				};
+			};
+
+			port@1 {
+				reg = <1>;
+
+				dsi_hdmi_bridge_out: endpoint {
+					remote-endpoint = <&hdmi_connector_in>;
+				};
+			};
+		};
+	};
+};
diff --git a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
index c1e9067b3bdd5..821a9705bb7d4 100644
--- a/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
+++ b/arch/arm64/boot/dts/ti/k3-am62a7-sk.dts
@@ -154,6 +154,7 @@ vddshv_sdio: regulator-5 {
 		gpios = <&main_gpio0 31 GPIO_ACTIVE_HIGH>;
 		states = <1800000 0x0>,
 			 <3300000 0x1>;
+		bootph-all;
 	};
 
 	leds {
@@ -407,6 +408,7 @@ vddshv_sdio_pins_default: vddshv-sdio-default-pins {
 		pinctrl-single,pins = <
 			AM62AX_IOPAD(0x07c, PIN_OUTPUT, 7) /* (N22) GPMC0_CLK.GPIO0_31 */
 		>;
+		bootph-all;
 	};
 
 	main_ecap0_pins_default: main-ecap0-default-pins {
diff --git a/arch/arm64/boot/dts/ti/k3-j721s2-som-p0.dtsi b/arch/arm64/boot/dts/ti/k3-j721s2-som-p0.dtsi
index 12a38dd1514bb..a19e535f49464 100644
--- a/arch/arm64/boot/dts/ti/k3-j721s2-som-p0.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-j721s2-som-p0.dtsi
@@ -250,6 +250,7 @@ buckb1: buck1 {
 				regulator-max-microvolt = <1800000>;
 				regulator-always-on;
 				regulator-boot-on;
+				bootph-pre-ram;
 			};
 
 			buckb2: buck2 {
diff --git a/arch/arm64/boot/dts/ti/k3-j722s-main.dtsi b/arch/arm64/boot/dts/ti/k3-j722s-main.dtsi
index ddf20e44f0eaf..d1dbf1e24fbf5 100644
--- a/arch/arm64/boot/dts/ti/k3-j722s-main.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-j722s-main.dtsi
@@ -430,19 +430,6 @@ &main_bcdma_csi {
 	ti,sci-rm-range-tchan = <0x22>;
 };
 
-/* MCU domain overrides */
-
-&mcu_r5fss0_core0 {
-	firmware-name = "j722s-mcu-r5f0_0-fw";
-};
-
-/* Wakeup domain overrides */
-
-&wkup_r5fss0_core0 {
-	firmware-name = "j722s-wkup-r5f0_0-fw";
-};
-
-/* MAIN domain overrides */
 &hsm {
 	firmware-name = "j722s-hsm-m4f-fw";
 };
@@ -457,13 +444,6 @@ serdes_ln_ctrl: mux-controller@4080 {
 	};
 };
 
-&wkup_conf {
-	pcie0_ctrl: pcie0-ctrl@4070 {
-		compatible = "ti,j784s4-pcie-ctrl", "syscon";
-		reg = <0x4070 0x4>;
-	};
-};
-
 &oc_sram {
 	reg = <0x00 0x70000000 0x00 0x40000>;
 	ranges = <0x00 0x00 0x70000000 0x40000>;
@@ -473,6 +453,10 @@ &inta_main_dmss {
 	ti,interrupt-ranges = <7 71 21>;
 };
 
+&main_pmx0 {
+	compatible = "ti,j7200-padconf", "pinctrl-single";
+};
+
 &main_gpio0 {
 	gpio-ranges = <&main_pmx0 0 0 32>, <&main_pmx0 32 33 38>,
 			<&main_pmx0 70 72 17>;
diff --git a/arch/arm64/boot/dts/ti/k3-j722s-mcu.dtsi b/arch/arm64/boot/dts/ti/k3-j722s-mcu.dtsi
new file mode 100644
index 0000000000000..ab43a7e49f373
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-j722s-mcu.dtsi
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Device Tree file for the J722S MCU domain peripherals
+ *
+ * Copyright (C) 2026 Texas Instruments Incorporated - https://www.ti.com/
+ * Copyright (C) 2026 Bootlin
+ */
+
+&mcu_pmx0 {
+	compatible = "ti,j7200-padconf", "pinctrl-single";
+};
+
+&mcu_r5fss0_core0 {
+	firmware-name = "j722s-mcu-r5f0_0-fw";
+};
diff --git a/arch/arm64/boot/dts/ti/k3-j722s-wakeup.dtsi b/arch/arm64/boot/dts/ti/k3-j722s-wakeup.dtsi
new file mode 100644
index 0000000000000..1297813f48298
--- /dev/null
+++ b/arch/arm64/boot/dts/ti/k3-j722s-wakeup.dtsi
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0-only OR MIT
+/*
+ * Device Tree file for the J722S WAKEUP domain peripherals
+ *
+ * Copyright (C) 2026 Texas Instruments Incorporated - https://www.ti.com/
+ * Copyright (C) 2026 Bootlin
+ */
+
+&wkup_conf {
+	pcie0_ctrl: pcie0-ctrl@4070 {
+		compatible = "ti,j784s4-pcie-ctrl", "syscon";
+		reg = <0x4070 0x4>;
+	};
+};
+
+&wkup_r5fss0_core0 {
+	firmware-name = "j722s-wkup-r5f0_0-fw";
+};
diff --git a/arch/arm64/boot/dts/ti/k3-j722s.dtsi b/arch/arm64/boot/dts/ti/k3-j722s.dtsi
index 1b36dcf37925e..ca10c6904b69f 100644
--- a/arch/arm64/boot/dts/ti/k3-j722s.dtsi
+++ b/arch/arm64/boot/dts/ti/k3-j722s.dtsi
@@ -238,3 +238,5 @@ cbass_wakeup: bus@b00000 {
 
 /* Include J722S specific peripherals */
 #include "k3-j722s-main.dtsi"
+#include "k3-j722s-mcu.dtsi"
+#include "k3-j722s-wakeup.dtsi"
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index d905a0777f939..cb68a70832379 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -50,7 +50,6 @@ CONFIG_ARCH_BLAIZE=y
 CONFIG_ARCH_BST=y
 CONFIG_ARCH_CIX=y
 CONFIG_ARCH_EXYNOS=y
-CONFIG_ARCH_SPARX5=y
 CONFIG_ARCH_K3=y
 CONFIG_ARCH_LG1K=y
 CONFIG_ARCH_HISI=y
@@ -58,6 +57,7 @@ CONFIG_ARCH_KEEMBAY=y
 CONFIG_ARCH_MEDIATEK=y
 CONFIG_ARCH_MESON=y
 CONFIG_ARCH_MICROCHIP=y
+CONFIG_ARCH_SPARX5=y
 CONFIG_ARCH_MVEBU=y
 CONFIG_ARCH_NXP=y
 CONFIG_ARCH_LAYERSCAPE=y
@@ -99,7 +99,6 @@ CONFIG_CPU_FREQ_GOV_USERSPACE=y
 CONFIG_CPU_FREQ_GOV_ONDEMAND=y
 CONFIG_CPU_FREQ_GOV_CONSERVATIVE=m
 CONFIG_CPUFREQ_DT=y
-CONFIG_ACPI_CPPC_CPUFREQ=m
 CONFIG_ARM_ALLWINNER_SUN50I_CPUFREQ_NVMEM=m
 CONFIG_ARM_APPLE_SOC_CPUFREQ=m
 CONFIG_ARM_ARMADA_37XX_CPUFREQ=y
@@ -112,6 +111,7 @@ CONFIG_ARM_RASPBERRYPI_CPUFREQ=m
 CONFIG_ARM_SCMI_CPUFREQ=y
 CONFIG_ARM_TEGRA186_CPUFREQ=y
 CONFIG_QORIQ_CPUFREQ=y
+CONFIG_ACPI_CPPC_CPUFREQ=m
 CONFIG_ACPI=y
 CONFIG_ACPI_HOTPLUG_MEMORY=y
 CONFIG_ACPI_HMAT=y
@@ -126,6 +126,7 @@ CONFIG_JUMP_LABEL=y
 CONFIG_MODULES=y
 CONFIG_MODULE_UNLOAD=y
 CONFIG_IOSCHED_BFQ=y
+CONFIG_BLK_INLINE_ENCRYPTION=y
 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
 # CONFIG_COMPAT_BRK is not set
 CONFIG_MEMORY_HOTPLUG=y
@@ -154,17 +155,7 @@ CONFIG_NETFILTER_XT_MATCH_CONNTRACK=m
 CONFIG_NETFILTER_XT_MATCH_IPVS=m
 CONFIG_IP_VS=m
 CONFIG_IP_NF_IPTABLES=m
-CONFIG_IP_NF_FILTER=m
-CONFIG_IP_NF_TARGET_REJECT=m
-CONFIG_IP_NF_NAT=m
-CONFIG_IP_NF_TARGET_MASQUERADE=m
-CONFIG_IP_NF_MANGLE=m
 CONFIG_IP6_NF_IPTABLES=m
-CONFIG_IP6_NF_FILTER=m
-CONFIG_IP6_NF_TARGET_REJECT=m
-CONFIG_IP6_NF_MANGLE=m
-CONFIG_IP6_NF_NAT=m
-CONFIG_IP6_NF_TARGET_MASQUERADE=m
 CONFIG_BRIDGE=m
 CONFIG_BRIDGE_VLAN_FILTERING=y
 CONFIG_NET_DSA=m
@@ -182,8 +173,8 @@ CONFIG_NET_CLS_FLOWER=m
 CONFIG_NET_CLS_ACT=y
 CONFIG_NET_ACT_GACT=m
 CONFIG_NET_ACT_MIRRED=m
-CONFIG_HSR=m
 CONFIG_NET_ACT_GATE=m
+CONFIG_HSR=m
 CONFIG_QRTR_SMD=m
 CONFIG_QRTR_TUN=m
 CONFIG_CAN=m
@@ -201,6 +192,7 @@ CONFIG_BT_HCIUART_QCA=y
 CONFIG_BT_HCIUART_MRVL=y
 CONFIG_BT_MRVL=m
 CONFIG_BT_MRVL_SDIO=m
+CONFIG_BT_MTKSDIO=m
 CONFIG_BT_QCOMSMD=m
 CONFIG_BT_NXPUART=m
 CONFIG_CFG80211=m
@@ -208,8 +200,8 @@ CONFIG_MAC80211=m
 CONFIG_MAC80211_LEDS=y
 CONFIG_RFKILL=m
 CONFIG_RFKILL_GPIO=m
-CONFIG_NET_9P=y
-CONFIG_NET_9P_VIRTIO=y
+CONFIG_NET_9P=m
+CONFIG_NET_9P_VIRTIO=m
 CONFIG_NFC=m
 CONFIG_NFC_NCI=m
 CONFIG_NFC_NXP_NCI=m
@@ -241,7 +233,9 @@ CONFIG_PCIE_XILINX=y
 CONFIG_PCIE_XILINX_DMA_PL=y
 CONFIG_PCIE_XILINX_NWL=y
 CONFIG_PCIE_XILINX_CPM=y
+CONFIG_PCI_SKY1_HOST=m
 CONFIG_PCI_J721E_HOST=m
+CONFIG_PCI_SKY1_HOST=m
 CONFIG_PCI_IMX6_HOST=y
 CONFIG_PCI_LAYERSCAPE=y
 CONFIG_PCI_HISI=y
@@ -260,10 +254,12 @@ CONFIG_PCI_ENDPOINT=y
 CONFIG_PCI_ENDPOINT_CONFIGFS=y
 CONFIG_PCI_EPF_TEST=m
 CONFIG_PCI_PWRCTRL_GENERIC=m
+CONFIG_POWER_SEQUENCING_PCIE_M2=m
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 CONFIG_FW_LOADER_USER_HELPER=y
 CONFIG_HISILICON_LPC=y
+CONFIG_IMX_AIPSTZ=m
 CONFIG_TEGRA_ACONNECT=m
 CONFIG_MHI_BUS_PCI_GENERIC=m
 CONFIG_ARM_SCMI_PROTOCOL=y
@@ -276,7 +272,6 @@ CONFIG_GOOGLE_FIRMWARE=y
 CONFIG_GOOGLE_CBMEM=m
 CONFIG_GOOGLE_COREBOOT_TABLE=m
 CONFIG_EFI_CAPSULE_LOADER=y
-CONFIG_IMX_AIPSTZ=m
 CONFIG_IMX_SCU=y
 CONFIG_QCOM_TZMEM_MODE_SHMBRIDGE=y
 CONFIG_QCOM_QSEECOM=y
@@ -303,11 +298,12 @@ CONFIG_MTD_NAND_MARVELL=y
 CONFIG_MTD_NAND_BRCMNAND=m
 CONFIG_MTD_NAND_FSL_IFC=y
 CONFIG_MTD_NAND_QCOM=y
-CONFIG_MTD_SPI_NOR=y
 CONFIG_MTD_SPI_NAND=m
+CONFIG_MTD_SPI_NOR=y
 CONFIG_MTD_UBI=m
 CONFIG_MTD_HYPERBUS=m
 CONFIG_HBMC_AM654=m
+CONFIG_OF_OVERLAY=y
 CONFIG_BLK_DEV_LOOP=y
 CONFIG_BLK_DEV_NBD=m
 CONFIG_VIRTIO_BLK=y
@@ -321,6 +317,7 @@ CONFIG_XILINX_SDFEC=m
 CONFIG_EEPROM_AT24=m
 CONFIG_EEPROM_AT25=m
 CONFIG_UACCE=m
+CONFIG_MISC_RP1=m
 # CONFIG_SCSI_PROC_FS is not set
 CONFIG_BLK_DEV_SD=y
 CONFIG_SCSI_SAS_ATA=y
@@ -355,58 +352,110 @@ CONFIG_VIRTIO_NET=y
 CONFIG_MHI_NET=m
 CONFIG_NET_DSA_BCM_SF2=m
 CONFIG_NET_DSA_MSCC_FELIX=m
+# CONFIG_NET_VENDOR_3COM is not set
+# CONFIG_NET_VENDOR_ACTIONS is not set
+# CONFIG_NET_VENDOR_ADAPTEC is not set
+# CONFIG_NET_VENDOR_AGERE is not set
+# CONFIG_NET_VENDOR_ALACRITECH is not set
+# CONFIG_NET_VENDOR_ALLWINNER is not set
 CONFIG_ENA_ETHERNET=m
-CONFIG_AMD_XGBE=y
+CONFIG_AMD_XGBE=m
 CONFIG_NET_XGENE=y
+# CONFIG_NET_VENDOR_AQUANTIA is not set
+# CONFIG_NET_VENDOR_ARC is not set
+# CONFIG_NET_VENDOR_ASIX is not set
 CONFIG_ATL1C=m
+CONFIG_BCM4908_ENET=m
 CONFIG_BCMGENET=m
 CONFIG_BNX2X=m
+CONFIG_BGMAC_PLATFORM=m
 CONFIG_SYSTEMPORT=m
-CONFIG_MACB=y
-CONFIG_THUNDER_NIC_PF=y
-CONFIG_FEC=y
-CONFIG_FSL_FMAN=y
-CONFIG_FSL_DPAA_ETH=y
-CONFIG_FSL_DPAA2_ETH=y
-CONFIG_FSL_ENETC=y
+CONFIG_BCMASP=m
+CONFIG_MACB=m
+CONFIG_THUNDER_NIC_PF=m
+# CONFIG_NET_VENDOR_CHELSIO is not set
+# CONFIG_NET_VENDOR_CISCO is not set
+# CONFIG_NET_VENDOR_CORTINA is not set
+# CONFIG_NET_VENDOR_DAVICOM is not set
+# CONFIG_NET_VENDOR_DEC is not set
+# CONFIG_NET_VENDOR_DLINK is not set
+# CONFIG_NET_VENDOR_EMULEX is not set
+# CONFIG_NET_VENDOR_ENGLEDER is not set
+# CONFIG_NET_VENDOR_EZCHIP is not set
+CONFIG_FEC=m
+CONFIG_FSL_FMAN=m
+CONFIG_FSL_DPAA_ETH=m
+CONFIG_FSL_DPAA2_ETH=m
+CONFIG_FSL_ENETC=m
 CONFIG_NXP_ENETC4=m
-CONFIG_FSL_ENETC_VF=y
+CONFIG_FSL_ENETC_VF=m
 CONFIG_FSL_ENETC_QOS=y
 CONFIG_NXP_NETC_BLK_CTRL=m
-CONFIG_HIX5HD2_GMAC=y
-CONFIG_HNS_DSAF=y
-CONFIG_HNS_ENET=y
-CONFIG_HNS3=y
-CONFIG_HNS3_HCLGE=y
-CONFIG_HNS3_ENET=y
-CONFIG_E1000=y
-CONFIG_E1000E=y
-CONFIG_IGB=y
-CONFIG_IGBVF=y
-CONFIG_MVNETA=y
-CONFIG_MVPP2=y
-CONFIG_SKY2=y
+# CONFIG_NET_VENDOR_FUNGIBLE is not set
+# CONFIG_NET_VENDOR_GOOGLE is not set
+CONFIG_HIX5HD2_GMAC=m
+CONFIG_HNS_DSAF=m
+CONFIG_HNS_ENET=m
+CONFIG_HNS3=m
+# CONFIG_NET_VENDOR_HUAWEI is not set
+CONFIG_E1000=m
+CONFIG_E1000E=m
+CONFIG_IGB=m
+CONFIG_IGBVF=m
+# CONFIG_NET_VENDOR_ADI is not set
+# CONFIG_NET_VENDOR_LITEX is not set
+CONFIG_MVNETA=m
+CONFIG_MVPP2=m
+CONFIG_SKY2=m
 CONFIG_NET_VENDOR_MEDIATEK=y
 CONFIG_NET_MEDIATEK_STAR_EMAC=m
 CONFIG_MLX4_EN=m
 CONFIG_MLX5_CORE=m
 CONFIG_MLX5_CORE_EN=y
+# CONFIG_NET_VENDOR_META is not set
+# CONFIG_NET_VENDOR_MICREL is not set
+# CONFIG_NET_VENDOR_MICROCHIP is not set
+# CONFIG_NET_VENDOR_MICROSOFT is not set
+# CONFIG_NET_VENDOR_MUCSE is not set
+# CONFIG_NET_VENDOR_MYRI is not set
+# CONFIG_NET_VENDOR_NI is not set
+# CONFIG_NET_VENDOR_NATSEMI is not set
+# CONFIG_NET_VENDOR_NETRONOME is not set
+# CONFIG_NET_VENDOR_NVIDIA is not set
+# CONFIG_NET_VENDOR_OKI is not set
+# CONFIG_NET_VENDOR_PACKET_ENGINES is not set
+# CONFIG_NET_VENDOR_QLOGIC is not set
+# CONFIG_NET_VENDOR_BROCADE is not set
 CONFIG_QCOM_EMAC=m
 CONFIG_RMNET=m
+# CONFIG_NET_VENDOR_RDC is not set
 CONFIG_R8169=m
-CONFIG_SH_ETH=y
-CONFIG_RAVB=y
-CONFIG_RENESAS_ETHER_SWITCH=y
-CONFIG_RTSN=y
-CONFIG_SMC91X=y
-CONFIG_SMSC911X=y
-CONFIG_SNI_AVE=y
-CONFIG_SNI_NETSEC=y
+CONFIG_SH_ETH=m
+CONFIG_RAVB=m
+CONFIG_RENESAS_ETHER_SWITCH=m
+CONFIG_RTSN=m
+# CONFIG_NET_VENDOR_ROCKER is not set
+# CONFIG_NET_VENDOR_SAMSUNG is not set
+# CONFIG_NET_VENDOR_SEEQ is not set
+# CONFIG_NET_VENDOR_SILAN is not set
+# CONFIG_NET_VENDOR_SIS is not set
+# CONFIG_NET_VENDOR_SOLARFLARE is not set
+CONFIG_SMC91X=m
+CONFIG_SMSC911X=m
+CONFIG_SNI_AVE=m
+CONFIG_SNI_NETSEC=m
 CONFIG_STMMAC_ETH=m
 CONFIG_DWMAC_MEDIATEK=m
 CONFIG_DWMAC_TEGRA=m
-CONFIG_TI_K3_AM65_CPSW_NUSS=y
+# CONFIG_NET_VENDOR_SUN is not set
+# CONFIG_NET_VENDOR_SYNOPSYS is not set
+# CONFIG_NET_VENDOR_TEHUTI is not set
+CONFIG_TI_K3_AM65_CPSW_NUSS=m
 CONFIG_TI_ICSSG_PRUETH=m
+# CONFIG_NET_VENDOR_VERTEXCOM is not set
+# CONFIG_NET_VENDOR_VIA is not set
+# CONFIG_NET_VENDOR_WANGXUN is not set
+# CONFIG_NET_VENDOR_WIZNET is not set
 CONFIG_XILINX_AXI_EMAC=m
 CONFIG_QCOM_IPA=m
 CONFIG_MESON_GXL_PHY=m
@@ -423,8 +472,8 @@ CONFIG_REALTEK_PHY=y
 CONFIG_ROCKCHIP_PHY=y
 CONFIG_DP83867_PHY=y
 CONFIG_DP83869_PHY=m
-CONFIG_DP83TG720_PHY=m
 CONFIG_DP83TD510_PHY=y
+CONFIG_DP83TG720_PHY=m
 CONFIG_VITESSE_PHY=y
 CONFIG_XILINX_GMII2RGMII=m
 CONFIG_CAN_FLEXCAN=m
@@ -434,7 +483,7 @@ CONFIG_CAN_M_CAN_PLATFORM=m
 CONFIG_CAN_RCAR=m
 CONFIG_CAN_RCAR_CANFD=m
 CONFIG_CAN_MCP251XFD=m
-CONFIG_MDIO_GPIO=y
+CONFIG_MDIO_GPIO=m
 CONFIG_MDIO_BUS_MUX_MULTIPLEXER=y
 CONFIG_MDIO_BUS_MUX_MMIOREG=y
 CONFIG_USB_PEGASUS=m
@@ -466,6 +515,7 @@ CONFIG_MWIFIEX_SDIO=m
 CONFIG_MWIFIEX_PCIE=m
 CONFIG_MWIFIEX_USB=m
 CONFIG_MT7921E=m
+CONFIG_MT7921S=m
 CONFIG_RSI_91X=m
 CONFIG_WL18XX=m
 CONFIG_WLCORE_SDIO=m
@@ -556,6 +606,7 @@ CONFIG_TCG_TIS_SPI=m
 CONFIG_TCG_TIS_SPI_CR50=y
 CONFIG_TCG_TIS_I2C_CR50=m
 CONFIG_TCG_TIS_I2C_INFINEON=y
+CONFIG_I2C=y
 CONFIG_I2C_CHARDEV=y
 CONFIG_I2C_MUX=y
 CONFIG_I2C_MUX_PCA954x=y
@@ -563,7 +614,6 @@ CONFIG_I2C_MUX_PINCTRL=m
 CONFIG_I2C_BCM2835=m
 CONFIG_I2C_CADENCE=m
 CONFIG_I2C_DESIGNWARE_CORE=y
-CONFIG_I2C_DESIGNWARE_PLATFORM=y
 CONFIG_I2C_GPIO=m
 CONFIG_I2C_IMX=y
 CONFIG_I2C_IMX_LPI2C=y
@@ -630,8 +680,6 @@ CONFIG_SPMI=y
 CONFIG_SPMI_APPLE=m
 CONFIG_SPMI_MTK_PMIF=m
 CONFIG_PINCTRL_APPLE_GPIO=m
-CONFIG_PINCTRL_BRCMSTB=y
-CONFIG_PINCTRL_BCM2712=y
 CONFIG_PINCTRL_DA9062=m
 CONFIG_PINCTRL_MAX77620=y
 CONFIG_PINCTRL_RK805=m
@@ -640,69 +688,17 @@ CONFIG_PINCTRL_SX150X=m
 CONFIG_PINCTRL_OWL=y
 CONFIG_PINCTRL_S700=y
 CONFIG_PINCTRL_S900=y
-CONFIG_PINCTRL_IMX8MM=y
-CONFIG_PINCTRL_IMX8MN=y
-CONFIG_PINCTRL_IMX8MP=y
-CONFIG_PINCTRL_IMX8MQ=y
-CONFIG_PINCTRL_IMX8QM=y
-CONFIG_PINCTRL_IMX8QXP=y
-CONFIG_PINCTRL_IMX8DXL=y
-CONFIG_PINCTRL_IMX8ULP=y
-CONFIG_PINCTRL_IMX91=y
-CONFIG_PINCTRL_IMX93=y
+CONFIG_PINCTRL_BRCMSTB=y
+CONFIG_PINCTRL_BCM2712=y
+CONFIG_PINCTRL_SKY1=y
 CONFIG_PINCTRL_IMX_SCMI=y
-CONFIG_PINCTRL_MSM=y
-CONFIG_PINCTRL_ELIZA=y
-CONFIG_PINCTRL_GLYMUR=y
-CONFIG_PINCTRL_IPQ5018=y
-CONFIG_PINCTRL_IPQ5210=y
-CONFIG_PINCTRL_IPQ5332=y
-CONFIG_PINCTRL_IPQ5424=y
-CONFIG_PINCTRL_IPQ8074=y
-CONFIG_PINCTRL_IPQ6018=y
-CONFIG_PINCTRL_IPQ9574=y
-CONFIG_PINCTRL_KAANAPALI=y
-CONFIG_PINCTRL_MSM8916=y
-CONFIG_PINCTRL_MSM8953=y
-CONFIG_PINCTRL_MSM8976=y
-CONFIG_PINCTRL_MSM8994=y
-CONFIG_PINCTRL_MSM8996=y
-CONFIG_PINCTRL_MSM8998=y
-CONFIG_PINCTRL_QCM2290=y
-CONFIG_PINCTRL_QCS404=y
-CONFIG_PINCTRL_QCS615=y
-CONFIG_PINCTRL_QCS8300=y
-CONFIG_PINCTRL_QDF2XXX=y
-CONFIG_PINCTRL_QDU1000=y
-CONFIG_PINCTRL_RP1=m
-CONFIG_PINCTRL_SA8775P=y
-CONFIG_PINCTRL_SC7180=y
-CONFIG_PINCTRL_SC7280=y
-CONFIG_PINCTRL_SC8180X=y
-CONFIG_PINCTRL_SC8280XP=y
-CONFIG_PINCTRL_SDM660=y
-CONFIG_PINCTRL_SDM670=y
-CONFIG_PINCTRL_SDM845=y
-CONFIG_PINCTRL_SDX75=y
-CONFIG_PINCTRL_SM4450=y
-CONFIG_PINCTRL_SM6115=y
-CONFIG_PINCTRL_SM6125=y
-CONFIG_PINCTRL_SM6350=y
-CONFIG_PINCTRL_SM6375=y
-CONFIG_PINCTRL_MILOS=y
-CONFIG_PINCTRL_SM8150=y
-CONFIG_PINCTRL_SM8250=y
-CONFIG_PINCTRL_SM8350=y
-CONFIG_PINCTRL_SM8450=y
-CONFIG_PINCTRL_SM8550=y
-CONFIG_PINCTRL_SM8650=y
-CONFIG_PINCTRL_SM8750=y
-CONFIG_PINCTRL_X1E80100=y
+CONFIG_PINCTRL_IMX91=y
 CONFIG_PINCTRL_QCOM_SPMI_PMIC=y
 CONFIG_PINCTRL_LPASS_LPI=m
 CONFIG_PINCTRL_MILOS_LPASS_LPI=m
 CONFIG_PINCTRL_SC7280_LPASS_LPI=m
 CONFIG_PINCTRL_SM6115_LPASS_LPI=m
+CONFIG_PINCTRL_SM6350_LPASS_LPI=m
 CONFIG_PINCTRL_SM8250_LPASS_LPI=m
 CONFIG_PINCTRL_SM8450_LPASS_LPI=m
 CONFIG_PINCTRL_SC8280XP_LPASS_LPI=m
@@ -710,6 +706,7 @@ CONFIG_PINCTRL_SM8550_LPASS_LPI=m
 CONFIG_PINCTRL_SM8650_LPASS_LPI=m
 CONFIG_PINCTRL_SOPHGO_SG2000=y
 CONFIG_GPIO_ALTERA=m
+CONFIG_GPIO_CADENCE=m
 CONFIG_GPIO_DAVINCI=y
 CONFIG_GPIO_DWAPB=y
 CONFIG_GPIO_MB86S7X=y
@@ -719,9 +716,9 @@ CONFIG_GPIO_PL061=y
 CONFIG_GPIO_RCAR=y
 CONFIG_GPIO_SYSCON=y
 CONFIG_GPIO_UNIPHIER=y
+CONFIG_GPIO_VF610=y
 CONFIG_GPIO_VISCONTI=y
 CONFIG_GPIO_WCD934X=m
-CONFIG_GPIO_VF610=y
 CONFIG_GPIO_XGENE=y
 CONFIG_GPIO_XGENE_SB=y
 CONFIG_GPIO_XILINX=m
@@ -729,9 +726,9 @@ CONFIG_GPIO_ZYNQ=m
 CONFIG_GPIO_MAX732X=y
 CONFIG_GPIO_PCA953X=y
 CONFIG_GPIO_PCA953X_IRQ=y
-CONFIG_GPIO_ADP5585=m
 CONFIG_GPIO_PCF857X=m
 CONFIG_GPIO_TPIC2810=m
+CONFIG_GPIO_ADP5585=m
 CONFIG_GPIO_BD9571MWV=m
 CONFIG_GPIO_MACSMC=m
 CONFIG_GPIO_MAX77620=y
@@ -770,7 +767,6 @@ CONFIG_SENSORS_AMC6821=m
 CONFIG_SENSORS_INA2XX=m
 CONFIG_SENSORS_INA3221=m
 CONFIG_SENSORS_TMP102=m
-CONFIG_MISC_RP1=m
 CONFIG_THERMAL_GOV_POWER_ALLOCATOR=y
 CONFIG_CPU_THERMAL=y
 CONFIG_DEVFREQ_THERMAL=y
@@ -781,11 +777,6 @@ CONFIG_K3_THERMAL=m
 CONFIG_QORIQ_THERMAL=m
 CONFIG_SUN8I_THERMAL=y
 CONFIG_ROCKCHIP_THERMAL=m
-CONFIG_RCAR_THERMAL=y
-CONFIG_RCAR_GEN3_THERMAL=y
-CONFIG_RZG2L_THERMAL=y
-CONFIG_RZG3E_THERMAL=y
-CONFIG_RZG3S_THERMAL=m
 CONFIG_ARMADA_THERMAL=y
 CONFIG_MTK_THERMAL=m
 CONFIG_MTK_LVTS_THERMAL=m
@@ -793,6 +784,11 @@ CONFIG_BCM2711_THERMAL=m
 CONFIG_BCM2835_THERMAL=m
 CONFIG_BRCMSTB_THERMAL=m
 CONFIG_EXYNOS_THERMAL=y
+CONFIG_RCAR_THERMAL=y
+CONFIG_RCAR_GEN3_THERMAL=y
+CONFIG_RZG2L_THERMAL=y
+CONFIG_RZG3E_THERMAL=y
+CONFIG_RZG3S_THERMAL=m
 CONFIG_TEGRA_SOCTHERM=m
 CONFIG_TEGRA_BPMP_THERMAL=m
 CONFIG_GENERIC_ADC_THERMAL=m
@@ -918,10 +914,10 @@ CONFIG_SDR_PLATFORM_DRIVERS=y
 CONFIG_V4L_MEM2MEM_DRIVERS=y
 CONFIG_VIDEO_AMPHION_VPU=m
 CONFIG_VIDEO_CADENCE_CSI2RX=m
-CONFIG_VIDEO_MEDIATEK_JPEG=m
-CONFIG_VIDEO_MEDIATEK_VCODEC=m
 CONFIG_VIDEO_WAVE_VPU=m
 CONFIG_VIDEO_E5010_JPEG_ENC=m
+CONFIG_VIDEO_MEDIATEK_JPEG=m
+CONFIG_VIDEO_MEDIATEK_VCODEC=m
 CONFIG_VIDEO_MEDIATEK_MDP3=m
 CONFIG_VIDEO_IMX7_CSI=m
 CONFIG_VIDEO_IMX_MIPI_CSIS=m
@@ -931,8 +927,8 @@ CONFIG_VIDEO_IMX8_JPEG=m
 CONFIG_VIDEO_QCOM_CAMSS=m
 CONFIG_VIDEO_QCOM_IRIS=m
 CONFIG_VIDEO_QCOM_VENUS=m
-CONFIG_VIDEO_RCAR_ISP=m
 CONFIG_VIDEO_RCAR_CSI2=m
+CONFIG_VIDEO_RCAR_ISP=m
 CONFIG_VIDEO_RCAR_VIN=m
 CONFIG_VIDEO_RZG2L_CSI2=m
 CONFIG_VIDEO_RZG2L_CRU=m
@@ -940,8 +936,8 @@ CONFIG_VIDEO_RENESAS_FCP=m
 CONFIG_VIDEO_RENESAS_FDP1=m
 CONFIG_VIDEO_RENESAS_VSP1=m
 CONFIG_VIDEO_RCAR_DRIF=m
-CONFIG_VIDEO_ROCKCHIP_CIF=m
 CONFIG_VIDEO_ROCKCHIP_RGA=m
+CONFIG_VIDEO_ROCKCHIP_CIF=m
 CONFIG_VIDEO_SAMSUNG_EXYNOS_GSC=m
 CONFIG_VIDEO_SAMSUNG_S5P_JPEG=m
 CONFIG_VIDEO_SAMSUNG_S5P_MFC=m
@@ -956,64 +952,12 @@ CONFIG_VIDEO_OV5640=m
 CONFIG_VIDEO_OV5645=m
 CONFIG_VIDEO_S5KJN1=m
 CONFIG_DRM=m
-CONFIG_DRM_I2C_NXP_TDA998X=m
 CONFIG_DRM_HDLCD=m
 CONFIG_DRM_MALI_DISPLAY=m
 CONFIG_DRM_KOMEDA=m
-CONFIG_DRM_NOUVEAU=m
-CONFIG_DRM_EXYNOS=m
-CONFIG_DRM_EXYNOS5433_DECON=y
-CONFIG_DRM_EXYNOS7_DECON=y
-CONFIG_DRM_EXYNOS_DSI=y
-# CONFIG_DRM_EXYNOS_DP is not set
-CONFIG_DRM_EXYNOS_HDMI=y
-CONFIG_DRM_EXYNOS_MIC=y
-CONFIG_DRM_ROCKCHIP=m
-CONFIG_ROCKCHIP_VOP2=y
-CONFIG_ROCKCHIP_ANALOGIX_DP=y
-CONFIG_ROCKCHIP_CDN_DP=y
-CONFIG_ROCKCHIP_DW_DP=y
-CONFIG_ROCKCHIP_DW_HDMI=y
-CONFIG_ROCKCHIP_DW_HDMI_QP=y
-CONFIG_ROCKCHIP_DW_MIPI_DSI=y
-CONFIG_ROCKCHIP_INNO_HDMI=y
-CONFIG_ROCKCHIP_LVDS=y
-CONFIG_DRM_RCAR_DU=m
-CONFIG_DRM_RCAR_DW_HDMI=m
-CONFIG_DRM_RCAR_MIPI_DSI=m
-CONFIG_DRM_RZG2L_MIPI_DSI=m
-CONFIG_DRM_RZG2L_DU=m
-CONFIG_DRM_SUN4I=m
-CONFIG_DRM_SUN6I_DSI=m
-CONFIG_DRM_SUN8I_DW_HDMI=m
-CONFIG_DRM_SUN8I_MIXER=m
-CONFIG_DRM_MSM=m
-CONFIG_DRM_TEGRA=m
-CONFIG_DRM_STM=m
-CONFIG_DRM_STM_LVDS=m
-CONFIG_DRM_PANEL_BOE_TV101WUM_NL6=m
-CONFIG_DRM_PANEL_LVDS=m
-CONFIG_DRM_PANEL_SIMPLE=m
-CONFIG_DRM_PANEL_SUMMIT=m
-CONFIG_DRM_PANEL_EDP=m
-CONFIG_DRM_PANEL_HIMAX_HX8279=m
-CONFIG_DRM_PANEL_HIMAX_HX83112A=m
-CONFIG_DRM_PANEL_HIMAX_HX83112B=m
-CONFIG_DRM_PANEL_ILITEK_ILI9882T=m
-CONFIG_DRM_PANEL_KHADAS_TS050=m
-CONFIG_DRM_PANEL_MANTIX_MLAF057WE51=m
-CONFIG_DRM_PANEL_NOVATEK_NT36672A=m
-CONFIG_DRM_PANEL_NOVATEK_NT36672E=m
-CONFIG_DRM_PANEL_NOVATEK_NT37801=m
-CONFIG_DRM_PANEL_RAYDIUM_RM67191=m
-CONFIG_DRM_PANEL_RAYDIUM_RM692E5=m
-CONFIG_DRM_PANEL_SAMSUNG_ATNA33XC20=m
-CONFIG_DRM_PANEL_SITRONIX_ST7703=m
-CONFIG_DRM_PANEL_STARTEK_KD070FHFID015=m
-CONFIG_DRM_PANEL_TRULY_NT35597_WQXGA=m
-CONFIG_DRM_PANEL_VISIONOX_VTDR6130=m
 CONFIG_DRM_DISPLAY_CONNECTOR=m
 CONFIG_DRM_FSL_LDB=m
+CONFIG_DRM_I2C_NXP_TDA998X=m
 CONFIG_DRM_ITE_IT6263=m
 CONFIG_DRM_LONTIUM_LT8912B=m
 CONFIG_DRM_LONTIUM_LT9611=m
@@ -1022,7 +966,6 @@ CONFIG_DRM_LONTIUM_LT8713SX=m
 CONFIG_DRM_ITE_IT66121=m
 CONFIG_DRM_NWL_MIPI_DSI=m
 CONFIG_DRM_PARADE_PS8640=m
-CONFIG_DRM_SAMSUNG_DSIM=m
 CONFIG_DRM_SII902X=m
 CONFIG_DRM_SIMPLE_BRIDGE=m
 CONFIG_DRM_THINE_THC63LVD1024=m
@@ -1040,27 +983,74 @@ CONFIG_DRM_IMX8MP_DW_HDMI_BRIDGE=m
 CONFIG_DRM_DW_HDMI_AHB_AUDIO=m
 CONFIG_DRM_DW_HDMI_CEC=m
 CONFIG_DRM_DW_HDMI_QP_CEC=y
-CONFIG_DRM_IMX_DCSS=m
-CONFIG_DRM_V3D=m
-CONFIG_DRM_VC4=m
 CONFIG_DRM_ETNAVIV=m
+CONFIG_DRM_EXYNOS=m
+CONFIG_DRM_EXYNOS5433_DECON=y
+CONFIG_DRM_EXYNOS7_DECON=y
+CONFIG_DRM_EXYNOS_DSI=y
+# CONFIG_DRM_EXYNOS_DP is not set
+CONFIG_DRM_EXYNOS_HDMI=y
+CONFIG_DRM_EXYNOS_MIC=y
 CONFIG_DRM_HISI_HIBMC=m
 CONFIG_DRM_HISI_KIRIN=m
+CONFIG_DRM_POWERVR=m
+CONFIG_DRM_IMX_DCSS=m
+CONFIG_DRM_LIMA=m
 CONFIG_DRM_MEDIATEK=m
 CONFIG_DRM_MEDIATEK_DP=m
 CONFIG_DRM_MEDIATEK_HDMI=m
 CONFIG_DRM_MEDIATEK_HDMI_V2=m
+CONFIG_DRM_MESON=m
+CONFIG_DRM_MSM=m
 CONFIG_DRM_MXSFB=m
 CONFIG_DRM_IMX_LCDIF=m
-CONFIG_DRM_MESON=m
-CONFIG_DRM_PL111=m
-CONFIG_DRM_LIMA=m
+CONFIG_DRM_NOUVEAU=m
+CONFIG_DRM_PANEL_BOE_TV101WUM_NL6=m
+CONFIG_DRM_PANEL_LVDS=m
+CONFIG_DRM_PANEL_HIMAX_HX8279=m
+CONFIG_DRM_PANEL_HIMAX_HX83112A=m
+CONFIG_DRM_PANEL_HIMAX_HX83112B=m
+CONFIG_DRM_PANEL_ILITEK_ILI9882T=m
+CONFIG_DRM_PANEL_KHADAS_TS050=m
+CONFIG_DRM_PANEL_MANTIX_MLAF057WE51=m
+CONFIG_DRM_PANEL_NOVATEK_NT36672A=m
+CONFIG_DRM_PANEL_NOVATEK_NT36672E=m
+CONFIG_DRM_PANEL_NOVATEK_NT37801=m
+CONFIG_DRM_PANEL_RAYDIUM_RM67191=m
+CONFIG_DRM_PANEL_RAYDIUM_RM692E5=m
+CONFIG_DRM_PANEL_SAMSUNG_ATNA33XC20=m
+CONFIG_DRM_PANEL_SITRONIX_ST7703=m
+CONFIG_DRM_PANEL_STARTEK_KD070FHFID015=m
+CONFIG_DRM_PANEL_EDP=m
+CONFIG_DRM_PANEL_SIMPLE=m
+CONFIG_DRM_PANEL_SUMMIT=m
+CONFIG_DRM_PANEL_TRULY_NT35597_WQXGA=m
+CONFIG_DRM_PANEL_VISIONOX_VTDR6130=m
 CONFIG_DRM_PANFROST=m
 CONFIG_DRM_PANTHOR=m
+CONFIG_DRM_PL111=m
+CONFIG_DRM_RCAR_DU=m
+CONFIG_DRM_RCAR_DW_HDMI=m
+CONFIG_DRM_RZG2L_DU=m
+CONFIG_DRM_ROCKCHIP=m
+CONFIG_ROCKCHIP_VOP2=y
+CONFIG_ROCKCHIP_ANALOGIX_DP=y
+CONFIG_ROCKCHIP_CDN_DP=y
+CONFIG_ROCKCHIP_DW_DP=y
+CONFIG_ROCKCHIP_DW_HDMI=y
+CONFIG_ROCKCHIP_DW_HDMI_QP=y
+CONFIG_ROCKCHIP_DW_MIPI_DSI=y
+CONFIG_ROCKCHIP_INNO_HDMI=y
+CONFIG_ROCKCHIP_LVDS=y
+CONFIG_DRM_STM=m
+CONFIG_DRM_STM_LVDS=m
+CONFIG_DRM_SUN4I=m
+CONFIG_DRM_TEGRA=m
 CONFIG_DRM_TIDSS=m
+CONFIG_DRM_V3D=m
+CONFIG_DRM_VC4=m
 CONFIG_DRM_ZYNQMP_DPSUB=m
 CONFIG_DRM_ZYNQMP_DPSUB_AUDIO=y
-CONFIG_DRM_POWERVR=m
 CONFIG_FB=y
 CONFIG_FB_EFI=y
 CONFIG_FB_MODE_HELPERS=y
@@ -1113,21 +1103,15 @@ CONFIG_SND_SOC_SC8280XP=m
 CONFIG_SND_SOC_SC7180=m
 CONFIG_SND_SOC_SC7280=m
 CONFIG_SND_SOC_X1E80100=m
-CONFIG_SND_SOC_ROCKCHIP=m
+CONFIG_SND_SOC_RCAR=m
+CONFIG_SND_SOC_MSIOF=m
+CONFIG_SND_SOC_RZ=m
 CONFIG_SND_SOC_ROCKCHIP_I2S_TDM=m
 CONFIG_SND_SOC_ROCKCHIP_SAI=m
 CONFIG_SND_SOC_ROCKCHIP_SPDIF=m
 CONFIG_SND_SOC_ROCKCHIP_RT5645=m
 CONFIG_SND_SOC_RK3399_GRU_SOUND=m
-CONFIG_SND_SOC_RCAR=m
-CONFIG_SND_SOC_MSIOF=m
-CONFIG_SND_SOC_RZ=m
 CONFIG_SND_SOC_SAMSUNG=m
-CONFIG_SND_SOC_SOF_TOPLEVEL=y
-CONFIG_SND_SOC_SOF_OF=m
-CONFIG_SND_SOC_SOF_MTK_TOPLEVEL=y
-CONFIG_SND_SOC_SOF_MT8186=m
-CONFIG_SND_SOC_SOF_MT8195=m
 CONFIG_SND_SUN8I_CODEC=m
 CONFIG_SND_SUN8I_CODEC_ANALOG=m
 CONFIG_SND_SUN50I_CODEC_ANALOG=m
@@ -1147,11 +1131,15 @@ CONFIG_SND_SOC_TEGRA210_AMX=m
 CONFIG_SND_SOC_TEGRA210_ADX=m
 CONFIG_SND_SOC_TEGRA210_MIXER=m
 CONFIG_SND_SOC_TEGRA_AUDIO_GRAPH_CARD=m
-CONFIG_SND_SOC_DAVINCI_MCASP=m
 CONFIG_SND_SOC_J721E_EVM=m
 CONFIG_SND_SOC_XILINX_I2S=m
 CONFIG_SND_SOC_XILINX_AUDIO_FORMATTER=m
 CONFIG_SND_SOC_XILINX_SPDIF=m
+CONFIG_SND_SOC_SOF_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_OF=m
+CONFIG_SND_SOC_SOF_MTK_TOPLEVEL=y
+CONFIG_SND_SOC_SOF_MT8186=m
+CONFIG_SND_SOC_SOF_MT8195=m
 CONFIG_SND_SOC_AK4613=m
 CONFIG_SND_SOC_AK4619=m
 CONFIG_SND_SOC_DA7213=m
@@ -1163,7 +1151,6 @@ CONFIG_SND_SOC_GTM601=m
 CONFIG_SND_SOC_MAX98090=m
 CONFIG_SND_SOC_MSM8916_WCD_ANALOG=m
 CONFIG_SND_SOC_MSM8916_WCD_DIGITAL=m
-CONFIG_SND_SOC_PCM3168A_I2C=m
 CONFIG_SND_SOC_RK3308=m
 CONFIG_SND_SOC_RK817=m
 CONFIG_SND_SOC_RT5640=m
@@ -1220,12 +1207,10 @@ CONFIG_USB_CDNS_SUPPORT=m
 CONFIG_USB_CDNS3=m
 CONFIG_USB_CDNS3_GADGET=y
 CONFIG_USB_CDNS3_HOST=y
-CONFIG_USB_CDNS3_IMX=m
 CONFIG_USB_MTU3=y
 CONFIG_USB_MUSB_HDRC=y
 CONFIG_USB_MUSB_SUNXI=y
 CONFIG_USB_DWC3=y
-CONFIG_OMAP_USB2=m
 CONFIG_USB_DWC2=y
 CONFIG_USB_CHIPIDEA=y
 CONFIG_USB_CHIPIDEA_UDC=y
@@ -1311,6 +1296,7 @@ CONFIG_MMC_SDHCI_AM654=y
 CONFIG_MMC_OWL=y
 CONFIG_SCSI_UFSHCD=y
 CONFIG_SCSI_UFS_BSG=y
+CONFIG_SCSI_UFS_CRYPTO=y
 CONFIG_SCSI_UFSHCD_PLATFORM=y
 CONFIG_SCSI_UFS_CDNS_PLATFORM=m
 CONFIG_SCSI_UFS_QCOM=m
@@ -1320,8 +1306,6 @@ CONFIG_SCSI_UFS_RENESAS=m
 CONFIG_SCSI_UFS_TI_J721E=m
 CONFIG_SCSI_UFS_EXYNOS=y
 CONFIG_SCSI_UFS_ROCKCHIP=y
-CONFIG_BLK_INLINE_ENCRYPTION=y
-CONFIG_SCSI_UFS_CRYPTO=y
 CONFIG_NEW_LEDS=y
 CONFIG_LEDS_CLASS=y
 CONFIG_LEDS_CLASS_FLASH=m
@@ -1348,9 +1332,9 @@ CONFIG_RTC_CLASS=y
 CONFIG_RTC_DRV_DS1307=m
 CONFIG_RTC_DRV_HYM8563=m
 CONFIG_RTC_DRV_MAX77686=y
+CONFIG_RTC_DRV_NVIDIA_VRS10=m
 CONFIG_RTC_DRV_RK808=m
 CONFIG_RTC_DRV_ISL1208=m
-CONFIG_RTC_DRV_PCF85063=m
 CONFIG_RTC_DRV_PCF85363=m
 CONFIG_RTC_DRV_PCF8563=m
 CONFIG_RTC_DRV_M41T80=m
@@ -1358,10 +1342,10 @@ CONFIG_RTC_DRV_BQ32K=m
 CONFIG_RTC_DRV_RX8581=m
 CONFIG_RTC_DRV_RV3028=m
 CONFIG_RTC_DRV_RV8803=m
-CONFIG_RTC_DRV_S32G=m
 CONFIG_RTC_DRV_S5M=y
 CONFIG_RTC_DRV_DS3232=y
 CONFIG_RTC_DRV_PCF2127=m
+CONFIG_RTC_DRV_PCF85063=m
 CONFIG_RTC_DRV_DA9063=m
 CONFIG_RTC_DRV_EFI=y
 CONFIG_RTC_DRV_ZYNQMP=m
@@ -1380,8 +1364,8 @@ CONFIG_RTC_DRV_MT6397=m
 CONFIG_RTC_DRV_XGENE=y
 CONFIG_RTC_DRV_TI_K3=m
 CONFIG_RTC_DRV_RENESAS_RTCA3=m
-CONFIG_RTC_DRV_NVIDIA_VRS10=m
 CONFIG_RTC_DRV_MACSMC=m
+CONFIG_RTC_DRV_S32G=m
 CONFIG_DMADEVICES=y
 CONFIG_APPLE_ADMAC=m
 CONFIG_DMA_BCM2835=y
@@ -1432,11 +1416,11 @@ CONFIG_CROS_EC_RPMSG=m
 CONFIG_CROS_EC_SPI=y
 CONFIG_CROS_KBD_LED_BACKLIGHT=m
 CONFIG_CROS_EC_CHARDEV=m
-CONFIG_COMMON_CLK_APPLE_NCO=m
 CONFIG_EC_ACER_ASPIRE1=m
 CONFIG_EC_HUAWEI_GAOKUN=m
 CONFIG_EC_LENOVO_YOGA_C630=m
 CONFIG_EC_LENOVO_THINKPAD_T14S=m
+CONFIG_COMMON_CLK_APPLE_NCO=m
 CONFIG_COMMON_CLK_RK808=y
 CONFIG_COMMON_CLK_SCMI=y
 CONFIG_COMMON_CLK_SCPI=y
@@ -1444,7 +1428,6 @@ CONFIG_COMMON_CLK_CS2000_CP=y
 CONFIG_COMMON_CLK_FSL_SAI=y
 CONFIG_COMMON_CLK_S2MPS11=y
 CONFIG_COMMON_CLK_PWM=y
-CONFIG_COMMON_CLK_RP1=m
 CONFIG_COMMON_CLK_RS9_PCIE=y
 CONFIG_COMMON_CLK_VC3=y
 CONFIG_COMMON_CLK_VC5=y
@@ -1497,7 +1480,6 @@ CONFIG_QCOM_CLK_APCC_MSM8996=y
 CONFIG_QCOM_CLK_SMD_RPM=y
 CONFIG_QCOM_CLK_RPMH=y
 CONFIG_IPQ_APSS_6018=y
-CONFIG_IPQ_APSS_5018=y
 CONFIG_IPQ_CMN_PLL=m
 CONFIG_IPQ_GCC_5018=y
 CONFIG_IPQ_GCC_5210=y
@@ -1521,17 +1503,16 @@ CONFIG_QCM_DISPCC_2290=m
 CONFIG_QCS_DISPCC_615=m
 CONFIG_QCS_CAMCC_615=m
 CONFIG_QCS_GCC_404=y
-CONFIG_QCS_GCC_615=y
-CONFIG_QCS_GCC_8300=y
-CONFIG_SC_CAMCC_7280=m
 CONFIG_SA_CAMCC_8775P=m
+CONFIG_QCS_GCC_8300=y
+CONFIG_QCS_GCC_615=y
 CONFIG_QCS_GPUCC_615=m
 CONFIG_QCS_VIDEOCC_615=m
-CONFIG_QDU_GCC_1000=y
+CONFIG_SC_CAMCC_7280=m
 CONFIG_SC_CAMCC_8280XP=m
+CONFIG_SA_DISPCC_8775P=m
 CONFIG_SC_DISPCC_7280=m
 CONFIG_SC_DISPCC_8280XP=m
-CONFIG_SA_DISPCC_8775P=m
 CONFIG_SA_GCC_8775P=y
 CONFIG_SA_GPUCC_8775P=m
 CONFIG_SC_GCC_7180=y
@@ -1544,6 +1525,7 @@ CONFIG_SC_LPASSCC_8280XP=m
 CONFIG_SC_LPASS_CORECC_7280=m
 CONFIG_SC_VIDEOCC_7280=m
 CONFIG_SDM_CAMCC_845=m
+CONFIG_QDU_GCC_1000=y
 CONFIG_SDM_GPUCC_845=y
 CONFIG_SDM_VIDEOCC_845=y
 CONFIG_SDM_DISPCC_845=y
@@ -1608,22 +1590,22 @@ CONFIG_RENESAS_OSTM=y
 CONFIG_ARM_MHU=y
 CONFIG_EXYNOS_MBOX=m
 CONFIG_IMX_MBOX=y
-CONFIG_OMAP2PLUS_MBOX=m
 CONFIG_PLATFORM_MHU=y
+CONFIG_OMAP2PLUS_MBOX=m
 CONFIG_BCM2835_MBOX=y
 CONFIG_QCOM_APCS_IPC=y
+CONFIG_TEGRA_HSP_MBOX=y
 CONFIG_MTK_ADSP_MBOX=m
 CONFIG_QCOM_CPUCP_MBOX=m
-CONFIG_TEGRA_HSP_MBOX=y
 CONFIG_QCOM_IPCC=y
 CONFIG_CIX_MBOX=y
-CONFIG_ROCKCHIP_IOMMU=y
-CONFIG_TEGRA_IOMMU_SMMU=y
 CONFIG_ARM_SMMU=y
 CONFIG_ARM_SMMU_V3=y
-CONFIG_MTK_IOMMU=y
 CONFIG_QCOM_IOMMU=y
+CONFIG_ROCKCHIP_IOMMU=y
+CONFIG_TEGRA_IOMMU_SMMU=y
 CONFIG_APPLE_DART=m
+CONFIG_MTK_IOMMU=y
 CONFIG_REMOTEPROC=y
 CONFIG_IMX_REMOTEPROC=y
 CONFIG_MTK_SCP=m
@@ -1727,9 +1709,9 @@ CONFIG_PWM_BCM2835=m
 CONFIG_PWM_BRCMSTB=m
 CONFIG_PWM_CROS_EC=m
 CONFIG_PWM_IMX27=m
+CONFIG_PWM_MEDIATEK=m
 CONFIG_PWM_MESON=m
 CONFIG_PWM_MTK_DISP=m
-CONFIG_PWM_MEDIATEK=m
 CONFIG_PWM_RENESAS_RCAR=m
 CONFIG_PWM_RENESAS_RZG2L_GPT=m
 CONFIG_PWM_RENESAS_RZ_MTU3=m
@@ -1757,10 +1739,10 @@ CONFIG_RESET_QCOM_PDC=m
 CONFIG_RESET_RZG2L_USBPHY_CTRL=y
 CONFIG_RESET_RZV2H_USB2PHY=m
 CONFIG_RESET_TI_SCI=y
-CONFIG_PHY_SNPS_EUSB2=m
-CONFIG_PHY_XGENE=y
 CONFIG_PHY_CAN_TRANSCEIVER=m
 CONFIG_PHY_NXP_PTN3222=m
+CONFIG_PHY_SNPS_EUSB2=m
+CONFIG_PHY_XGENE=y
 CONFIG_PHY_SUN4I_USB=y
 CONFIG_PHY_CADENCE_TORRENT=m
 CONFIG_PHY_CADENCE_DPHY=m
@@ -1811,6 +1793,7 @@ CONFIG_PHY_UNIPHIER_USB3=y
 CONFIG_PHY_TEGRA_XUSB=y
 CONFIG_PHY_AM654_SERDES=m
 CONFIG_PHY_J721E_WIZ=m
+CONFIG_OMAP_USB2=m
 CONFIG_PHY_XILINX_ZYNQMP=m
 CONFIG_ARM_CCI_PMU=m
 CONFIG_ARM_CCN=m
@@ -1854,7 +1837,6 @@ CONFIG_ALTERA_FREEZE_BRIDGE=m
 CONFIG_XILINX_PR_DECOUPLER=m
 CONFIG_FPGA_REGION=m
 CONFIG_OF_FPGA_REGION=m
-CONFIG_OF_OVERLAY=y
 CONFIG_FPGA_MGR_ZYNQMP_FPGA=m
 CONFIG_FPGA_MGR_VERSAL_FPGA=m
 CONFIG_TEE=y
@@ -1903,9 +1885,9 @@ CONFIG_INTERCONNECT_QCOM_SM8650=y
 CONFIG_INTERCONNECT_QCOM_SM8750=y
 CONFIG_INTERCONNECT_QCOM_X1E80100=y
 CONFIG_COUNTER=m
-CONFIG_TI_EQEP=m
 CONFIG_RZ_MTU3_CNT=m
 CONFIG_STM32_TIMER_CNT=m
+CONFIG_TI_EQEP=m
 CONFIG_HTE=y
 CONFIG_HTE_TEGRA194=y
 CONFIG_HTE_TEGRA194_TEST=m
@@ -1934,13 +1916,13 @@ CONFIG_NFS_V4=y
 CONFIG_NFS_V4_1=y
 CONFIG_NFS_V4_2=y
 CONFIG_ROOT_NFS=y
-CONFIG_9P_FS=y
+CONFIG_9P_FS=m
 CONFIG_NLS_CODEPAGE_437=y
 CONFIG_NLS_ISO8859_1=y
 CONFIG_SECURITY=y
 CONFIG_CRYPTO_USER=y
-CONFIG_CRYPTO_CHACHA20=m
 CONFIG_CRYPTO_BENCHMARK=m
+CONFIG_CRYPTO_CHACHA20=m
 CONFIG_CRYPTO_ECHAINIV=y
 CONFIG_CRYPTO_SHA3=m
 CONFIG_CRYPTO_SM3=m
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 4de51f8d92cba..a57870fa96db5 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -620,6 +620,13 @@ static inline bool id_aa64pfr0_mpam(u64 pfr0)
 	return val > 0;
 }
 
+static inline bool id_aa64pfr1_mpamfrac(u64 pfr1)
+{
+	u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT);
+
+	return val > 0;
+}
+
 static inline bool id_aa64pfr1_mte(u64 pfr1)
 {
 	u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_EL1_MTE_SHIFT);
diff --git a/arch/arm64/include/asm/daifflags.h b/arch/arm64/include/asm/daifflags.h
index 5fca480090434..795b351284673 100644
--- a/arch/arm64/include/asm/daifflags.h
+++ b/arch/arm64/include/asm/daifflags.h
@@ -19,7 +19,7 @@
 
 
 /* mask/save/unmask/restore all exceptions, including interrupts. */
-static inline void local_daif_mask(void)
+static __always_inline void local_daif_mask(void)
 {
 	WARN_ON(system_has_prio_mask_debugging() &&
 		(read_sysreg_s(SYS_ICC_PMR_EL1) == (GIC_PRIO_IRQOFF |
@@ -38,7 +38,7 @@ static inline void local_daif_mask(void)
 	trace_hardirqs_off();
 }
 
-static inline unsigned long local_daif_save_flags(void)
+static __always_inline unsigned long local_daif_save_flags(void)
 {
 	unsigned long flags;
 
@@ -53,7 +53,7 @@ static inline unsigned long local_daif_save_flags(void)
 	return flags;
 }
 
-static inline unsigned long local_daif_save(void)
+static __always_inline unsigned long local_daif_save(void)
 {
 	unsigned long flags;
 
@@ -64,7 +64,7 @@ static inline unsigned long local_daif_save(void)
 	return flags;
 }
 
-static inline void local_daif_restore(unsigned long flags)
+static __always_inline void local_daif_restore(unsigned long flags)
 {
 	bool irq_disabled = flags & PSR_I_BIT;
 
@@ -124,7 +124,7 @@ static inline void local_daif_restore(unsigned long flags)
  * Called by synchronous exception handlers to restore the DAIF bits that were
  * modified by taking an exception.
  */
-static inline void local_daif_inherit(struct pt_regs *regs)
+static __always_inline void local_daif_inherit(struct pt_regs *regs)
 {
 	unsigned long flags = regs->pstate & DAIF_MASK;
 
diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h
index 587507a9980ec..aa8ec9df80243 100644
--- a/arch/arm64/include/asm/el2_setup.h
+++ b/arch/arm64/include/asm/el2_setup.h
@@ -510,7 +510,9 @@
 #endif
 
 .macro finalise_el2_state
-	check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2
+	check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lmpam_minor_\@, x1, x2
+.Lmpam_minor_\@:
+	check_override id_aa64pfr1, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2
 
 .Linit_mpam_\@:
 	mov	x0, #MPAM2_EL2_EnMPAMSM_MASK
diff --git a/arch/arm64/include/asm/esr.h b/arch/arm64/include/asm/esr.h
index 7e86d400864e0..81c17320a588d 100644
--- a/arch/arm64/include/asm/esr.h
+++ b/arch/arm64/include/asm/esr.h
@@ -160,6 +160,8 @@
 #define ESR_ELx_CM 		(UL(1) << ESR_ELx_CM_SHIFT)
 
 /* ISS2 field definitions for Data Aborts */
+#define ESR_ELx_HDBSSF_SHIFT	(11)
+#define ESR_ELx_HDBSSF		(UL(1) << ESR_ELx_HDBSSF_SHIFT)
 #define ESR_ELx_TnD_SHIFT	(10)
 #define ESR_ELx_TnD 		(UL(1) << ESR_ELx_TnD_SHIFT)
 #define ESR_ELx_TagAccess_SHIFT	(9)
diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
index f463a654a2bbd..cc0702fa64a79 100644
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -409,7 +409,7 @@ __AARCH64_INSN_FUNCS(cbz,	0x7F000000, 0x34000000)
 __AARCH64_INSN_FUNCS(cbnz,	0x7F000000, 0x35000000)
 __AARCH64_INSN_FUNCS(tbz,	0x7F000000, 0x36000000)
 __AARCH64_INSN_FUNCS(tbnz,	0x7F000000, 0x37000000)
-__AARCH64_INSN_FUNCS(bcond,	0xFF000010, 0x54000000)
+__AARCH64_INSN_FUNCS(bcond,	0xFF000000, 0x54000000)
 __AARCH64_INSN_FUNCS(svc,	0xFFE0001F, 0xD4000001)
 __AARCH64_INSN_FUNCS(hvc,	0xFFE0001F, 0xD4000002)
 __AARCH64_INSN_FUNCS(smc,	0xFFE0001F, 0xD4000003)
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 65eead8362e0b..c79747d5f4dd1 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -1112,7 +1112,8 @@ struct kvm_vcpu_arch {
 #define IN_NESTED_ERET		__vcpu_single_flag(sflags, BIT(7))
 /* SError pending for nested guest */
 #define NESTED_SERROR_PENDING	__vcpu_single_flag(sflags, BIT(8))
-
+/* KVM is currently emulating an L2 to L1 exception */
+#define IN_NESTED_EXCEPTION	__vcpu_single_flag(sflags, BIT(9))
 
 /* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
 #define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) +	\
diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h
index 40bd17add5397..73eabc82a6bb2 100644
--- a/arch/arm64/include/asm/linkage.h
+++ b/arch/arm64/include/asm/linkage.h
@@ -43,4 +43,6 @@
 	SYM_TYPED_START(name, SYM_L_GLOBAL, SYM_A_ALIGN)	\
 	bti c ;
 
+#define _THIS_IP_ ({ unsigned long __ip; asm volatile("adr %0, ." : "=r" (__ip)); __ip; })
+
 #endif
diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h
index e25d0d18f6d77..58200de8a221a 100644
--- a/arch/arm64/include/asm/page.h
+++ b/arch/arm64/include/asm/page.h
@@ -33,7 +33,7 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,
 						unsigned long vaddr);
 #define vma_alloc_zeroed_movable_folio vma_alloc_zeroed_movable_folio
 
-bool tag_clear_highpages(struct page *to, int numpages);
+bool tag_clear_highpages(struct page *to, int numpages, bool clear_pages);
 #define __HAVE_ARCH_TAG_CLEAR_HIGHPAGES
 
 #define copy_user_page(to, from, vaddr, pg)	copy_page(to, from)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 4dfa42b7d0535..c9e4e00a9af27 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -1007,7 +1007,7 @@ static inline pud_t *p4d_pgtable(p4d_t p4d)
 
 static inline phys_addr_t pud_offset_phys(p4d_t *p4dp, unsigned long addr)
 {
-	BUG_ON(!pgtable_l4_enabled());
+	VM_WARN_ON_ONCE(!pgtable_l4_enabled());
 
 	return p4d_page_paddr(READ_ONCE(*p4dp)) + pud_index(addr) * sizeof(pud_t);
 }
@@ -1130,7 +1130,7 @@ static inline p4d_t *pgd_to_folded_p4d(pgd_t *pgdp, unsigned long addr)
 
 static inline phys_addr_t p4d_offset_phys(pgd_t *pgdp, unsigned long addr)
 {
-	BUG_ON(!pgtable_l5_enabled());
+	VM_WARN_ON_ONCE(!pgtable_l5_enabled());
 
 	return pgd_page_paddr(READ_ONCE(*pgdp)) + p4d_index(addr) * sizeof(p4d_t);
 }
diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h
index b0c83a08dda97..9f5bd9c69c249 100644
--- a/arch/arm64/include/asm/uaccess.h
+++ b/arch/arm64/include/asm/uaccess.h
@@ -456,8 +456,7 @@ do {									\
 	unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u8, label);	\
 } while (0)
 
-#define INLINE_COPY_TO_USER
-#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_USER
 
 extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n);
 static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n)
diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h
index 06f83ca8de562..10272ddb4d6f8 100644
--- a/arch/arm64/include/uapi/asm/hwcap.h
+++ b/arch/arm64/include/uapi/asm/hwcap.h
@@ -147,5 +147,13 @@
 #define HWCAP3_MTE_STORE_ONLY		(1UL << 1)
 #define HWCAP3_LSFE		(1UL << 2)
 #define HWCAP3_LS64		(1UL << 3)
+#define HWCAP3_SVE_B16MM	(1UL << 4)
+#define HWCAP3_SVE2P3		(1UL << 5)
+#define HWCAP3_SME_LUT6		(1UL << 6)
+#define HWCAP3_SME2P3		(1UL << 7)
+#define HWCAP3_F16MM		(1UL << 8)
+#define HWCAP3_F16F32DOT	(1UL << 9)
+#define HWCAP3_F16F32MM		(1UL << 10)
+#define HWCAP3_SVE_LUT6		(1UL << 11)
 
 #endif /* _UAPI__ASM_HWCAP_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 6d53bb15cf7bb..bad7f0330b130 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -365,6 +365,8 @@ static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = {
 static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = {
 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
+		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUT6_SHIFT, 1, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUTv2_SHIFT, 1, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
@@ -419,6 +421,7 @@ static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = {
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM8_SHIFT, 1, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM4_SHIFT, 1, 0),
+	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F16MM2_SHIFT, 1, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0),
 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0),
 	ARM64_FTR_END,
@@ -1164,6 +1167,14 @@ static __init void detect_system_supports_pseudo_nmi(void)
 static inline void detect_system_supports_pseudo_nmi(void) { }
 #endif
 
+static bool detect_ftr_has_mpam(void)
+{
+	u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
+	u64 pfr1 = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1);
+
+	return id_aa64pfr0_mpam(pfr0) || id_aa64pfr1_mpamfrac(pfr1);
+}
+
 void __init init_cpu_features(struct cpuinfo_arm64 *info)
 {
 	/* Before we start using the tables, make sure it is sorted */
@@ -1211,7 +1222,7 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info)
 		cpacr_restore(cpacr);
 	}
 
-	if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
+	if (detect_ftr_has_mpam()) {
 		info->reg_mpamidr = read_cpuid(MPAMIDR_EL1);
 		init_cpu_ftr_reg(SYS_MPAMIDR_EL1, info->reg_mpamidr);
 	}
@@ -1467,7 +1478,7 @@ void update_cpu_features(int cpu,
 		cpacr_restore(cpacr);
 	}
 
-	if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
+	if (detect_ftr_has_mpam()) {
 		info->reg_mpamidr = read_cpuid(MPAMIDR_EL1);
 		taint |= check_update_ftr_reg(SYS_MPAMIDR_EL1, cpu,
 					info->reg_mpamidr, boot->reg_mpamidr);
@@ -2486,7 +2497,7 @@ cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap)
 static bool
 test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope)
 {
-	if (!has_cpuid_feature(entry, scope))
+	if (!detect_ftr_has_mpam())
 		return false;
 
 	/* Check firmware actually enabled MPAM on this cpu. */
@@ -3093,7 +3104,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
 		.capability = ARM64_MPAM,
 		.matches = test_has_mpam,
 		.cpu_enable = cpu_enable_mpam,
-		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, MPAM, 1)
 	},
 	{
 		.desc = "Memory Partitioning And Monitoring Virtualisation",
@@ -3284,6 +3294,8 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 	HWCAP_CAP(ID_AA64ISAR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SM4),
 	HWCAP_CAP(ID_AA64ISAR0_EL1, DP, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP),
 	HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM),
+	HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32DOT, CAP_HWCAP, KERNEL_HWCAP_F16F32DOT),
+	HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, F16F32MM, CAP_HWCAP, KERNEL_HWCAP_F16F32MM),
 	HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM, CAP_HWCAP, KERNEL_HWCAP_FLAGM),
 	HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2),
 	HWCAP_CAP(ID_AA64ISAR0_EL1, RNDR, IMP, CAP_HWCAP, KERNEL_HWCAP_RNG),
@@ -3313,7 +3325,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 	HWCAP_CAP(ID_AA64ISAR3_EL1, LSFE, IMP, CAP_HWCAP, KERNEL_HWCAP_LSFE),
 	HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT),
 #ifdef CONFIG_ARM64_SVE
+	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ISAR2_EL1, LUT, LUT6, CAP_HWCAP, KERNEL_HWCAP_SVE_LUT6),
 	HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE),
+	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p3, CAP_HWCAP, KERNEL_HWCAP_SVE2P3),
 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p2, CAP_HWCAP, KERNEL_HWCAP_SVE2P2),
 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1),
 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2),
@@ -3323,6 +3337,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM),
 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16),
 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, BFSCALE, CAP_HWCAP, KERNEL_HWCAP_SVE_BFSCALE),
+	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, B16MM, CAP_HWCAP, KERNEL_HWCAP_SVE_B16MM),
 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16),
 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16),
 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3),
@@ -3362,7 +3377,9 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 #ifdef CONFIG_ARM64_SME
 	HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME),
 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64),
+	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUT6, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUT6),
 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2),
+	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p3, CAP_HWCAP, KERNEL_HWCAP_SME2P3),
 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p2, CAP_HWCAP, KERNEL_HWCAP_SME2P2),
 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1),
 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2),
@@ -3393,6 +3410,7 @@ static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2),
 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM8),
 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM4),
+	HWCAP_CAP(ID_AA64FPFR0_EL1, F16MM2, IMP, CAP_HWCAP, KERNEL_HWCAP_F16MM),
 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3),
 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2),
 #ifdef CONFIG_ARM64_POE
@@ -4052,7 +4070,7 @@ static int enable_mismatched_32bit_el0(unsigned int cpu)
 	 */
 	lucky_winner = cpu_32bit ? cpu : cpumask_any_and(cpu_32bit_el0_mask,
 							 cpu_active_mask);
-	get_cpu_device(lucky_winner)->offline_disabled = true;
+	dev_set_offline_disabled(get_cpu_device(lucky_winner));
 	setup_elf_hwcaps(compat_elf_hwcaps);
 	elf_hwcap_fixup();
 	pr_info("Asymmetric 32-bit EL0 support detected on CPU %u; CPU hot-unplug disabled on CPU %u\n",
diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c
index 6149bc91251d1..d50e2a9b066b3 100644
--- a/arch/arm64/kernel/cpuinfo.c
+++ b/arch/arm64/kernel/cpuinfo.c
@@ -164,6 +164,14 @@ static const char *const hwcap_str[] = {
 	[KERNEL_HWCAP_MTE_FAR]		= "mtefar",
 	[KERNEL_HWCAP_MTE_STORE_ONLY]	= "mtestoreonly",
 	[KERNEL_HWCAP_LSFE]		= "lsfe",
+	[KERNEL_HWCAP_SVE_B16MM]	= "sveb16mm",
+	[KERNEL_HWCAP_SVE2P3]		= "sve2p3",
+	[KERNEL_HWCAP_SME_LUT6]		= "smelut6",
+	[KERNEL_HWCAP_SME2P3]		= "sme2p3",
+	[KERNEL_HWCAP_F16MM]		= "f16mm",
+	[KERNEL_HWCAP_F16F32DOT]	= "f16f32dot",
+	[KERNEL_HWCAP_F16F32MM]		= "f16f32mm",
+	[KERNEL_HWCAP_SVE_LUT6]		= "svelut6",
 };
 
 #ifdef CONFIG_COMPAT
diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 15dedb385b9e4..9fafd826002b7 100644
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -10,6 +10,7 @@
  * Copyright (C) 2012 ARM Ltd.
  */
 
+#include <linux/errno.h>
 #include <linux/hardirq.h>
 #include <linux/init.h>
 #include <linux/irq.h>
@@ -32,34 +33,43 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts);
 
 DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
 
-
 DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
 
 #ifdef CONFIG_SHADOW_CALL_STACK
 DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
 #endif
 
-static void init_irq_scs(void)
+static int __init init_irq_scs(void)
 {
 	int cpu;
+	void *s;
 
 	if (!scs_is_enabled())
-		return;
+		return 0;
+
+	for_each_possible_cpu(cpu) {
+		s = scs_alloc(early_cpu_to_node(cpu));
+		if (!s)
+			return -ENOMEM;
+		per_cpu(irq_shadow_call_stack_ptr, cpu) = s;
+	}
 
-	for_each_possible_cpu(cpu)
-		per_cpu(irq_shadow_call_stack_ptr, cpu) =
-			scs_alloc(early_cpu_to_node(cpu));
+	return 0;
 }
 
-static void __init init_irq_stacks(void)
+static int __init init_irq_stacks(void)
 {
 	int cpu;
 	unsigned long *p;
 
 	for_each_possible_cpu(cpu) {
 		p = arch_alloc_vmap_stack(IRQ_STACK_SIZE, early_cpu_to_node(cpu));
+		if (!p)
+			return -ENOMEM;
 		per_cpu(irq_stack_ptr, cpu) = p;
 	}
+
+	return 0;
 }
 
 #ifdef CONFIG_SOFTIRQ_ON_OWN_STACK
@@ -109,8 +119,9 @@ int __init set_handle_fiq(void (*handle_fiq)(struct pt_regs *))
 
 void __init init_IRQ(void)
 {
-	init_irq_stacks();
-	init_irq_scs();
+	if (init_irq_stacks() || init_irq_scs())
+		panic("Failed to allocate IRQ stack resources\n");
+
 	irqchip_init();
 
 	if (system_uses_irq_prio_masking()) {
diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index b3801f532b10b..7bb6553fec087 100644
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -24,6 +24,7 @@
 #include <linux/nospec.h>
 #include <linux/prctl.h>
 #include <linux/sched/task_stack.h>
+#include <linux/sysfs.h>
 
 #include <asm/debug-monitors.h>
 #include <asm/insn.h>
@@ -61,7 +62,7 @@ static void update_mitigation_state(enum mitigation_state *oldp,
 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
 			    char *buf)
 {
-	return sprintf(buf, "Mitigation: __user pointer sanitization\n");
+	return sysfs_emit(buf, "Mitigation: __user pointer sanitization\n");
 }
 
 /*
@@ -126,7 +127,7 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
 	switch (spectre_v2_state) {
 	case SPECTRE_UNAFFECTED:
 		if (bhb_state == SPECTRE_UNAFFECTED)
-			return sprintf(buf, "Not affected\n");
+			return sysfs_emit(buf, "Not affected\n");
 
 		/*
 		 * Platforms affected by Spectre-BHB can't report
@@ -136,13 +137,13 @@ ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
 		fallthrough;
 	case SPECTRE_MITIGATED:
 		if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled())
-			return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n");
+			return sysfs_emit(buf, "Vulnerable: Unprivileged eBPF enabled\n");
 
-		return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
+		return sysfs_emit(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
 	case SPECTRE_VULNERABLE:
 		fallthrough;
 	default:
-		return sprintf(buf, "Vulnerable\n");
+		return sysfs_emit(buf, "Vulnerable\n");
 	}
 }
 
@@ -438,13 +439,13 @@ ssize_t cpu_show_spec_store_bypass(struct device *dev,
 {
 	switch (spectre_v4_state) {
 	case SPECTRE_UNAFFECTED:
-		return sprintf(buf, "Not affected\n");
+		return sysfs_emit(buf, "Not affected\n");
 	case SPECTRE_MITIGATED:
-		return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n");
+		return sysfs_emit(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n");
 	case SPECTRE_VULNERABLE:
 		fallthrough;
 	default:
-		return sprintf(buf, "Vulnerable\n");
+		return sysfs_emit(buf, "Vulnerable\n");
 	}
 }
 
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 1aa324104afb4..1b63846f646a4 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -745,15 +745,21 @@ void __init smp_init_cpus(void)
 	else
 		acpi_parse_and_init_cpus();
 
-	if (cpu_count > nr_cpu_ids)
-		pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n",
-			cpu_count, nr_cpu_ids);
-
 	if (!bootcpu_valid) {
 		pr_err("missing boot CPU MPIDR, not enabling secondaries\n");
 		return;
 	}
 
+	/*
+	 * For the nosmp/maxcpus=0 case, do not mark the secondary CPUs
+	 * possible.
+	 */
+	if (!setup_max_cpus)
+		return;
+
+	if (cpu_count > nr_cpu_ids)
+		pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n",
+			cpu_count, nr_cpu_ids);
 	/*
 	 * We need to set the cpu_logical_map entries before enabling
 	 * the cpus so that cpu processor description entries (DT cpu nodes
diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c
index cbea4d9ee9552..c3b8257888e89 100644
--- a/arch/arm64/kvm/arch_timer.c
+++ b/arch/arm64/kvm/arch_timer.c
@@ -39,10 +39,9 @@ static const u8 default_ppi[] = {
 	[TIMER_HVTIMER] = 28,
 };
 
-static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 				 struct arch_timer_context *timer_ctx);
-static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
+static bool kvm_timer_pending(struct arch_timer_context *timer_ctx);
 static void kvm_arm_timer_write(struct kvm_vcpu *vcpu,
 				struct arch_timer_context *timer,
 				enum kvm_arch_timer_regs treg,
@@ -224,7 +223,7 @@ static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
 	else
 		ctx = map.direct_ptimer;
 
-	if (kvm_timer_should_fire(ctx))
+	if (kvm_timer_pending(ctx))
 		kvm_timer_update_irq(vcpu, true, ctx);
 
 	if (userspace_irqchip(vcpu->kvm) &&
@@ -257,7 +256,7 @@ static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
 	return kvm_counter_compute_delta(timer_ctx, timer_get_cval(timer_ctx));
 }
 
-static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
+static bool kvm_timer_enabled(struct arch_timer_context *timer_ctx)
 {
 	WARN_ON(timer_ctx && timer_ctx->loaded);
 	return timer_ctx &&
@@ -294,7 +293,7 @@ static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
 		struct arch_timer_context *ctx = &vcpu->arch.timer_cpu.timers[i];
 
 		WARN(ctx->loaded, "timer %d loaded\n", i);
-		if (kvm_timer_irq_can_fire(ctx))
+		if (kvm_timer_enabled(ctx))
 			min_delta = min(min_delta, kvm_timer_compute_delta(ctx));
 	}
 
@@ -358,7 +357,7 @@ static enum hrtimer_restart kvm_hrtimer_expire(struct hrtimer *hrt)
 	return HRTIMER_NORESTART;
 }
 
-static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
+static bool kvm_timer_pending(struct arch_timer_context *timer_ctx)
 {
 	enum kvm_arch_timers index;
 	u64 cval, now;
@@ -391,7 +390,7 @@ static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
 		       !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
 	}
 
-	if (!kvm_timer_irq_can_fire(timer_ctx))
+	if (!kvm_timer_enabled(timer_ctx))
 		return false;
 
 	cval = timer_get_cval(timer_ctx);
@@ -405,22 +404,30 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 	return vcpu_has_wfit_active(vcpu) && wfit_delay_ns(vcpu) == 0;
 }
 
+static u64 kvm_timer_needs_notify(struct kvm_vcpu *vcpu)
+{
+	u64 v = vcpu->run->s.regs.device_irq_level;
+
+	v ^= kvm_timer_pending(vcpu_vtimer(vcpu)) ? KVM_ARM_DEV_EL1_VTIMER : 0;
+	v ^= kvm_timer_pending(vcpu_ptimer(vcpu)) ? KVM_ARM_DEV_EL1_PTIMER : 0;
+
+	return v & (KVM_ARM_DEV_EL1_VTIMER | KVM_ARM_DEV_EL1_PTIMER);
+}
+
+bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
+{
+	return !!kvm_timer_needs_notify(vcpu);
+}
+
 /*
  * Reflect the timer output level into the kvm_run structure
  */
-void kvm_timer_update_run(struct kvm_vcpu *vcpu)
+bool kvm_timer_update_run(struct kvm_vcpu *vcpu)
 {
-	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
-	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
-	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
-
-	/* Populate the device bitmap with the timer states */
-	regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
-				    KVM_ARM_DEV_EL1_PTIMER);
-	if (kvm_timer_should_fire(vtimer))
-		regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
-	if (kvm_timer_should_fire(ptimer))
-		regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
+	u64 mask = kvm_timer_needs_notify(vcpu);
+	if (mask)
+		vcpu->run->s.regs.device_irq_level ^= mask;
+	return !!mask;
 }
 
 static void kvm_timer_update_status(struct arch_timer_context *ctx, bool level)
@@ -446,9 +453,8 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 {
 	kvm_timer_update_status(timer_ctx, new_level);
 
-	timer_ctx->irq.level = new_level;
 	trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_irq(timer_ctx),
-				   timer_ctx->irq.level);
+				   new_level);
 
 	if (userspace_irqchip(vcpu->kvm))
 		return;
@@ -466,28 +472,25 @@ static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 
 	kvm_vgic_inject_irq(vcpu->kvm, vcpu,
 			    timer_irq(timer_ctx),
-			    timer_ctx->irq.level,
+			    new_level,
 			    timer_ctx);
 }
 
 /* Only called for a fully emulated timer */
 static void timer_emulate(struct arch_timer_context *ctx)
 {
-	bool should_fire = kvm_timer_should_fire(ctx);
+	bool pending = kvm_timer_pending(ctx);
 
-	trace_kvm_timer_emulate(ctx, should_fire);
+	trace_kvm_timer_emulate(ctx, pending);
 
-	if (should_fire != ctx->irq.level)
-		kvm_timer_update_irq(timer_context_to_vcpu(ctx), should_fire, ctx);
-
-	kvm_timer_update_status(ctx, should_fire);
+	kvm_timer_update_irq(timer_context_to_vcpu(ctx), pending, ctx);
 
 	/*
-	 * If the timer can fire now, we don't need to have a soft timer
-	 * scheduled for the future.  If the timer cannot fire at all,
-	 * then we also don't need a soft timer.
+	 * If the timer is pending, we don't need to have a soft timer
+	 * scheduled for the future.  If the timer is disabled, then
+	 * we don't need a soft timer either.
 	 */
-	if (should_fire || !kvm_timer_irq_can_fire(ctx))
+	if (pending || !kvm_timer_enabled(ctx))
 		return;
 
 	soft_timer_start(&ctx->hrtimer, kvm_timer_compute_delta(ctx));
@@ -594,10 +597,10 @@ static void kvm_timer_blocking(struct kvm_vcpu *vcpu)
 	 * If no timers are capable of raising interrupts (disabled or
 	 * masked), then there's no more work for us to do.
 	 */
-	if (!kvm_timer_irq_can_fire(map.direct_vtimer) &&
-	    !kvm_timer_irq_can_fire(map.direct_ptimer) &&
-	    !kvm_timer_irq_can_fire(map.emul_vtimer) &&
-	    !kvm_timer_irq_can_fire(map.emul_ptimer) &&
+	if (!kvm_timer_enabled(map.direct_vtimer) &&
+	    !kvm_timer_enabled(map.direct_ptimer) &&
+	    !kvm_timer_enabled(map.emul_vtimer) &&
+	    !kvm_timer_enabled(map.emul_ptimer) &&
 	    !vcpu_has_wfit_active(vcpu))
 		return;
 
@@ -677,6 +680,7 @@ static inline void set_timer_irq_phys_active(struct arch_timer_context *ctx, boo
 static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
 {
 	struct kvm_vcpu *vcpu = timer_context_to_vcpu(ctx);
+	bool pending = kvm_timer_pending(ctx);
 	bool phys_active = false;
 
 	/*
@@ -685,12 +689,12 @@ static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
 	 * this point and the register restoration, we'll take the
 	 * interrupt anyway.
 	 */
-	kvm_timer_update_irq(vcpu, kvm_timer_should_fire(ctx), ctx);
+	kvm_timer_update_irq(vcpu, pending, ctx);
 
 	if (irqchip_in_kernel(vcpu->kvm))
 		phys_active = kvm_vgic_map_is_active(vcpu, timer_irq(ctx));
 
-	phys_active |= ctx->irq.level;
+	phys_active |= pending;
 	phys_active |= vgic_is_v5(vcpu->kvm);
 
 	set_timer_irq_phys_active(ctx, phys_active);
@@ -699,6 +703,7 @@ static void kvm_timer_vcpu_load_gic(struct arch_timer_context *ctx)
 static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
 {
 	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
+	bool pending = kvm_timer_pending(vtimer);
 
 	/*
 	 * Update the timer output so that it is likely to match the
@@ -706,7 +711,7 @@ static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
 	 * this point and the register restoration, we'll take the
 	 * interrupt anyway.
 	 */
-	kvm_timer_update_irq(vcpu, kvm_timer_should_fire(vtimer), vtimer);
+	kvm_timer_update_irq(vcpu, pending, vtimer);
 
 	/*
 	 * When using a userspace irqchip with the architected timers and a
@@ -718,7 +723,7 @@ static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
 	 * being de-asserted, we unmask the interrupt again so that we exit
 	 * from the guest when the timer fires.
 	 */
-	if (vtimer->irq.level)
+	if (pending)
 		disable_percpu_irq(host_vtimer_irq);
 	else
 		enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
@@ -904,23 +909,6 @@ void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
 	timer_set_traps(vcpu, &map);
 }
 
-bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
-{
-	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
-	struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
-	struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
-	bool vlevel, plevel;
-
-	if (likely(irqchip_in_kernel(vcpu->kvm)))
-		return false;
-
-	vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
-	plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
-
-	return kvm_timer_should_fire(vtimer) != vlevel ||
-	       kvm_timer_should_fire(ptimer) != plevel;
-}
-
 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
 {
 	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
@@ -1006,7 +994,7 @@ static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
 {
 	struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 
-	if (!kvm_timer_should_fire(vtimer)) {
+	if (!kvm_timer_pending(vtimer)) {
 		kvm_timer_update_irq(vcpu, false, vtimer);
 		if (static_branch_likely(&has_gic_active_state))
 			set_timer_irq_phys_active(vtimer, false);
@@ -1579,7 +1567,7 @@ static bool kvm_arch_timer_get_input_level(int vintid)
 
 		ctx = vcpu_get_timer(vcpu, i);
 		if (timer_irq(ctx) == vintid)
-			return kvm_timer_should_fire(ctx);
+			return kvm_timer_pending(ctx);
 	}
 
 	/* A timer IRQ has fired, but no matching timer was found? */
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 34c9950884d5e..a988fee6b6d58 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -52,6 +52,7 @@
 
 #include <linux/irqchip/arm-gic-v5.h>
 
+#include "vgic/vgic.h"
 #include "sys_regs.h"
 
 static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
@@ -555,8 +556,10 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 	kvm_destroy_mpidr_data(vcpu->kvm);
 
 	err = kvm_vgic_vcpu_init(vcpu);
-	if (err)
+	if (err) {
+		kvm_vgic_vcpu_destroy(vcpu);
 		return err;
+	}
 
 	err = kvm_share_hyp(vcpu, vcpu + 1);
 	if (err)
@@ -1164,6 +1167,15 @@ static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
 	return !kvm_supports_32bit_el0();
 }
 
+static bool kvm_irq_update_run(struct kvm_vcpu *vcpu)
+{
+	bool r;
+
+	r  = kvm_timer_update_run(vcpu);
+	r |= kvm_pmu_update_run(vcpu);
+	return r;
+}
+
 /**
  * kvm_vcpu_exit_request - returns true if the VCPU should *not* enter the guest
  * @vcpu:	The VCPU pointer
@@ -1185,13 +1197,11 @@ static bool kvm_vcpu_exit_request(struct kvm_vcpu *vcpu, int *ret)
 	/*
 	 * If we're using a userspace irqchip, then check if we need
 	 * to tell a userspace irqchip about timer or PMU level
-	 * changes and if so, exit to userspace (the actual level
-	 * state gets updated in kvm_timer_update_run and
-	 * kvm_pmu_update_run below).
+	 * changes and if so, exit to userspace while updating the run
+	 * state.
 	 */
 	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
-		if (kvm_timer_should_notify_user(vcpu) ||
-		    kvm_pmu_should_notify_user(vcpu)) {
+		if (unlikely(kvm_irq_update_run(vcpu))) {
 			*ret = -EINTR;
 			run->exit_reason = KVM_EXIT_INTR;
 			return true;
@@ -1406,11 +1416,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
 		ret = handle_exit(vcpu, ret);
 	}
 
-	/* Tell userspace about in-kernel device output levels */
-	if (unlikely(!irqchip_in_kernel(vcpu->kvm))) {
-		kvm_timer_update_run(vcpu);
-		kvm_pmu_update_run(vcpu);
-	}
+	if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
+		kvm_irq_update_run(vcpu);
 
 	kvm_sigset_deactivate(vcpu);
 
@@ -1494,8 +1501,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
 
 		return vcpu_interrupt_line(vcpu, irq_num, level);
 	case KVM_ARM_IRQ_TYPE_PPI:
-		if (!irqchip_in_kernel(kvm))
+		if (irqchip_in_kernel(kvm)) {
+			int ret = vgic_lazy_init(kvm);
+			if (ret)
+				return ret;
+		} else {
 			return -ENXIO;
+		}
 
 		vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
 		if (!vcpu)
@@ -1522,8 +1534,13 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level,
 
 		return kvm_vgic_inject_irq(kvm, vcpu, irq_num, level, NULL);
 	case KVM_ARM_IRQ_TYPE_SPI:
-		if (!irqchip_in_kernel(kvm))
+		if (irqchip_in_kernel(kvm)) {
+			int ret = vgic_lazy_init(kvm);
+			if (ret)
+				return ret;
+		} else {
 			return -ENXIO;
+		}
 
 		if (vgic_is_v5(kvm)) {
 			/* Build a GICv5-style IntID here */
diff --git a/arch/arm64/kvm/emulate-nested.c b/arch/arm64/kvm/emulate-nested.c
index dba7ced74ca5e..15c691a6266d5 100644
--- a/arch/arm64/kvm/emulate-nested.c
+++ b/arch/arm64/kvm/emulate-nested.c
@@ -2862,6 +2862,8 @@ static int kvm_inject_nested(struct kvm_vcpu *vcpu, u64 esr_el2,
 
 	preempt_disable();
 
+	vcpu_set_flag(vcpu, IN_NESTED_EXCEPTION);
+
 	/*
 	 * We may have an exception or PC update in the EL0/EL1 context.
 	 * Commit it before entering EL2.
@@ -2884,6 +2886,8 @@ static int kvm_inject_nested(struct kvm_vcpu *vcpu, u64 esr_el2,
 	__kvm_adjust_pc(vcpu);
 
 	kvm_arch_vcpu_load(vcpu, smp_processor_id());
+	vcpu_clear_flag(vcpu, IN_NESTED_EXCEPTION);
+
 	preempt_enable();
 
 	if (kvm_vcpu_has_pmu(vcpu))
diff --git a/arch/arm64/kvm/fpsimd.c b/arch/arm64/kvm/fpsimd.c
index 15e17aca1dec0..3f6b1e29cd6b9 100644
--- a/arch/arm64/kvm/fpsimd.c
+++ b/arch/arm64/kvm/fpsimd.c
@@ -28,6 +28,20 @@ void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu)
 	if (!system_supports_fpsimd())
 		return;
 
+	/*
+	 * Avoid needless save/restore of the guest's common
+	 * FPSIMD/SVE/SME regs during transitions between L1/L2.
+	 *
+	 * These transitions only happens in a non-preemptible context
+	 * where the host regs have already been saved and unbound. The
+	 * live registers are either free or owned by the guest.
+	 */
+	if (vcpu_get_flag(vcpu, IN_NESTED_ERET) ||
+	    vcpu_get_flag(vcpu, IN_NESTED_EXCEPTION)) {
+		WARN_ON_ONCE(host_owns_fp_regs());
+		return;
+	}
+
 	/*
 	 * Ensure that any host FPSIMD/SVE/SME state is saved and unbound such
 	 * that the host kernel is responsible for restoring this state upon
@@ -102,6 +116,18 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
 {
 	unsigned long flags;
 
+	/*
+	 * See comment in kvm_arch_vcpu_load_fp(). Note that we also rely on
+	 * the guest's max VL to have been set by fpsimd_lazy_switch_to_host()
+	 * so that any intervening kernel-mode SIMD (NEON or otherwise)
+	 * operation sees the full guest state that needs saving.
+	 */
+	if (vcpu_get_flag(vcpu, IN_NESTED_ERET) ||
+	    vcpu_get_flag(vcpu, IN_NESTED_EXCEPTION)) {
+		WARN_ON_ONCE(host_owns_fp_regs());
+		return;
+	}
+
 	local_irq_save(flags);
 
 	if (guest_owns_fp_regs()) {
diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c
index a6ca27b18e154..e7e150ab265ff 100644
--- a/arch/arm64/kvm/hyp/nvhe/trace.c
+++ b/arch/arm64/kvm/hyp/nvhe/trace.c
@@ -164,13 +164,16 @@ static int hyp_trace_buffer_load(struct hyp_trace_buffer *trace_buffer,
 	return ret;
 }
 
-static bool hyp_trace_desc_validate(struct hyp_trace_desc *desc, size_t desc_size)
+static bool hyp_trace_desc_is_valid(struct hyp_trace_desc *desc, size_t desc_size)
 {
 	struct ring_buffer_desc *rb_desc;
 	unsigned int cpu;
 	size_t nr_bpages;
 	void *desc_end;
 
+	if (!is_protected_kvm_enabled())
+		return true;
+
 	/*
 	 * Both desc_size and bpages_backing_size are untrusted host-provided
 	 * values. We rely on __pkvm_host_donate_hyp() to enforce their validity.
@@ -212,8 +215,10 @@ int __tracing_load(unsigned long desc_hva, size_t desc_size)
 	if (ret)
 		return ret;
 
-	if (!hyp_trace_desc_validate(desc, desc_size))
+	if (!hyp_trace_desc_is_valid(desc, desc_size)) {
+		ret = -EINVAL;
 		goto err_release_desc;
+	}
 
 	hyp_spin_lock(&trace_buffer.lock);
 
diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c
index e1860acae641f..edb21239478a9 100644
--- a/arch/arm64/kvm/pmu-emul.c
+++ b/arch/arm64/kvm/pmu-emul.c
@@ -396,44 +396,31 @@ static bool kvm_pmu_overflow_status(struct kvm_vcpu *vcpu)
 static void kvm_pmu_update_state(struct kvm_vcpu *vcpu)
 {
 	struct kvm_pmu *pmu = &vcpu->arch.pmu;
-	bool overflow;
 
-	overflow = kvm_pmu_overflow_status(vcpu);
-	if (pmu->irq_level == overflow)
+	if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
 		return;
 
-	pmu->irq_level = overflow;
-
-	if (likely(irqchip_in_kernel(vcpu->kvm))) {
-		int ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu,
-					      pmu->irq_num, overflow, pmu);
-		WARN_ON(ret);
-	}
+	WARN_ON(kvm_vgic_inject_irq(vcpu->kvm, vcpu, pmu->irq_num,
+				    kvm_pmu_overflow_status(vcpu), pmu));
 }
 
 bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
 {
-	struct kvm_pmu *pmu = &vcpu->arch.pmu;
 	struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
 	bool run_level = sregs->device_irq_level & KVM_ARM_DEV_PMU;
 
-	if (likely(irqchip_in_kernel(vcpu->kvm)))
-		return false;
-
-	return pmu->irq_level != run_level;
+	return kvm_pmu_overflow_status(vcpu) != run_level;
 }
 
 /*
  * Reflect the PMU overflow interrupt output level into the kvm_run structure
  */
-void kvm_pmu_update_run(struct kvm_vcpu *vcpu)
+bool kvm_pmu_update_run(struct kvm_vcpu *vcpu)
 {
-	struct kvm_sync_regs *regs = &vcpu->run->s.regs;
-
-	/* Populate the timer bitmap for user space */
-	regs->device_irq_level &= ~KVM_ARM_DEV_PMU;
-	if (vcpu->arch.pmu.irq_level)
-		regs->device_irq_level |= KVM_ARM_DEV_PMU;
+	bool update = kvm_pmu_should_notify_user(vcpu);
+	if (update)
+		vcpu->run->s.regs.device_irq_level ^= KVM_ARM_DEV_PMU;
+	return update;
 }
 
 /**
diff --git a/arch/arm64/kvm/vgic/vgic-irqfd.c b/arch/arm64/kvm/vgic/vgic-irqfd.c
index b9b86e3a6c862..19a1094536e6a 100644
--- a/arch/arm64/kvm/vgic/vgic-irqfd.c
+++ b/arch/arm64/kvm/vgic/vgic-irqfd.c
@@ -20,9 +20,15 @@ static int vgic_irqfd_set_irq(struct kvm_kernel_irq_routing_entry *e,
 			int level, bool line_status)
 {
 	unsigned int spi_id = e->irqchip.pin + VGIC_NR_PRIVATE_IRQS;
+	int ret;
 
 	if (!vgic_valid_spi(kvm, spi_id))
 		return -EINVAL;
+
+	ret = vgic_lazy_init(kvm);
+	if (ret)
+		return ret;
+
 	return kvm_vgic_inject_irq(kvm, NULL, spi_id, level, NULL);
 }
 
diff --git a/arch/arm64/kvm/vgic/vgic-its.c b/arch/arm64/kvm/vgic/vgic-its.c
index 2ea9f1c7ebcd0..1d7e5d560af4c 100644
--- a/arch/arm64/kvm/vgic/vgic-its.c
+++ b/arch/arm64/kvm/vgic/vgic-its.c
@@ -2307,6 +2307,10 @@ static int vgic_its_restore_dte(struct vgic_its *its, u32 id,
 	/* dte entry is valid */
 	offset = (entry & KVM_ITS_DTE_NEXT_MASK) >> KVM_ITS_DTE_NEXT_SHIFT;
 
+	/* Mimic the MAPD behaviour and reject invalid EID bits. */
+	if (num_eventid_bits > VITS_TYPER_IDBITS)
+		return -EINVAL;
+
 	if (!vgic_its_check_id(its, baser, id, NULL))
 		return -EINVAL;
 
diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c
index 1e9fe8764584d..9e29f03d3463c 100644
--- a/arch/arm64/kvm/vgic/vgic.c
+++ b/arch/arm64/kvm/vgic/vgic.c
@@ -534,11 +534,9 @@ int kvm_vgic_inject_irq(struct kvm *kvm, struct kvm_vcpu *vcpu,
 {
 	struct vgic_irq *irq;
 	unsigned long flags;
-	int ret;
 
-	ret = vgic_lazy_init(kvm);
-	if (ret)
-		return ret;
+	if (unlikely(!vgic_initialized(kvm)))
+		return 0;
 
 	if (!vcpu && irq_is_private(kvm, intid))
 		return -EINVAL;
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index ae1ae0280eef7..994b7b36e2b92 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -48,7 +48,7 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
 		   dev_driver_string(dev), dev_name(dev),
 		   ARCH_DMA_MINALIGN, cls);
 
-	dev->dma_coherent = coherent;
+	dev_assign_dma_coherent(dev, coherent);
 
 	xen_setup_dma_ops(dev);
 }
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 0f3c5c7ca0547..739800835920c 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -1018,7 +1018,7 @@ struct folio *vma_alloc_zeroed_movable_folio(struct vm_area_struct *vma,
 	return vma_alloc_folio(flags, 0, vma, vaddr);
 }
 
-bool tag_clear_highpages(struct page *page, int numpages)
+bool tag_clear_highpages(struct page *page, int numpages, bool clear_pages)
 {
 	/*
 	 * Check if MTE is supported and fall back to clear_highpage().
@@ -1026,13 +1026,16 @@ bool tag_clear_highpages(struct page *page, int numpages)
 	 * post_alloc_hook() will invoke tag_clear_highpages().
 	 */
 	if (!system_supports_mte())
-		return false;
+		return clear_pages;
 
 	/* Newly allocated pages, shouldn't have been tagged yet */
 	for (int i = 0; i < numpages; i++, page++) {
 		WARN_ON_ONCE(!try_page_mte_tagging(page));
-		mte_zero_clear_page_tags(page_address(page));
+		if (clear_pages)
+			mte_zero_clear_page_tags(page_address(page));
+		else
+			mte_clear_page_tags(page_address(page));
 		set_page_mte_tagged(page);
 	}
-	return true;
+	return false;
 }
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index dd85e093ffdb1..0e5b7df87e676 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -273,8 +273,8 @@ static int init_pmd(pmd_t *pmdp, unsigned long addr, unsigned long end,
 			if (ret)
 				return ret;
 
-			BUG_ON(pmd_val(old_pmd) != 0 &&
-			       pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp)));
+			VM_WARN_ON_ONCE(pmd_val(old_pmd) != 0 &&
+					pmd_val(old_pmd) != READ_ONCE(pmd_val(*pmdp)));
 		}
 		phys += next - addr;
 	} while (pmdp++, addr = next, addr != end);
@@ -394,8 +394,8 @@ static int alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
 			if (ret)
 				goto out;
 
-			BUG_ON(pud_val(old_pud) != 0 &&
-			       pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
+			VM_WARN_ON_ONCE(pud_val(old_pud) != 0 &&
+					pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
 		}
 		phys += next - addr;
 	} while (pudp++, addr = next, addr != end);
@@ -445,8 +445,8 @@ static int alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
 		if (ret)
 			goto out;
 
-		BUG_ON(p4d_val(old_p4d) != 0 &&
-		       p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp)));
+		VM_WARN_ON_ONCE(p4d_val(old_p4d) != 0 &&
+				p4d_val(old_p4d) != READ_ONCE(p4d_val(*p4dp)));
 
 		phys += next - addr;
 	} while (p4dp++, addr = next, addr != end);
@@ -2024,12 +2024,13 @@ int arch_add_memory(int nid, u64 start, u64 size,
 	return ret;
 }
 
-void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+			struct dev_pagemap *pgmap)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 
-	__remove_pages(start_pfn, nr_pages, altmap);
+	__remove_pages(start_pfn, nr_pages, altmap, pgmap);
 	__remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size);
 }
 
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 0816c40fc7af9..e3bbeaa94590c 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -47,7 +47,7 @@
 /* Map BPF registers to A64 registers */
 static const int bpf2a64[] = {
 	/* return value from in-kernel function, and exit value from eBPF */
-	[BPF_REG_0] = A64_R(7),
+	[BPF_REG_0] = A64_R(8),
 	/* arguments from eBPF program to in-kernel function */
 	[BPF_REG_1] = A64_R(0),
 	[BPF_REG_2] = A64_R(1),
@@ -86,6 +86,7 @@ struct jit_ctx {
 	__le32 *image;
 	__le32 *ro_image;
 	u32 stack_size;
+	u16 stack_arg_size;
 	u64 user_vm_start;
 	u64 arena_vm_start;
 	bool fp_used;
@@ -533,13 +534,19 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
 	 *                        |     |
 	 *                        +-----+ <= (BPF_FP - prog->aux->stack_depth)
 	 *                        |RSVD | padding
-	 * current A64_SP =>      +-----+ <= (BPF_FP - ctx->stack_size)
+	 *                        +-----+ <= (BPF_FP - ctx->stack_size)
+	 *                        |     |
+	 *                        | ... | outgoing stack args (9+, if any)
+	 *                        |     |
+	 * current A64_SP =>      +-----+
 	 *                        |     |
 	 *                        | ... | Function call stack
 	 *                        |     |
 	 *                        +-----+
 	 *                          low
 	 *
+	 * Stack args 6-8 are passed in x5-x7, args 9+ at [SP].
+	 * Incoming args 9+ are at [FP + 16], [FP + 24], ...
 	 */
 
 	emit_kcfi(is_main_prog ? cfi_bpf_hash : cfi_bpf_subprog_hash, ctx);
@@ -613,6 +620,9 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
 	if (ctx->stack_size && !ctx->priv_sp_used)
 		emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
 
+	if (ctx->stack_arg_size)
+		emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_arg_size), ctx);
+
 	if (ctx->arena_vm_start)
 		emit_a64_mov_i64(arena_vm_base, ctx->arena_vm_start, ctx);
 
@@ -673,6 +683,9 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx)
 	/* Update tail_call_cnt if the slot is populated. */
 	emit(A64_STR64I(tcc, ptr, 0), ctx);
 
+	if (ctx->stack_arg_size)
+		emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_arg_size), ctx);
+
 	/* restore SP */
 	if (ctx->stack_size && !ctx->priv_sp_used)
 		emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
@@ -1034,6 +1047,9 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic)
 	const u8 r0 = bpf2a64[BPF_REG_0];
 	const u8 ptr = bpf2a64[TCCNT_PTR];
 
+	if (ctx->stack_arg_size)
+		emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_arg_size), ctx);
+
 	/* We're done with BPF stack */
 	if (ctx->stack_size && !ctx->priv_sp_used)
 		emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
@@ -1048,7 +1064,7 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic)
 	/* Restore FP/LR registers */
 	emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
 
-	/* Move the return value from bpf:r0 (aka x7) to x0 */
+	/* Move the return value from bpf:r0 (aka x8) to x0 */
 	emit(A64_MOV(1, A64_R(0), r0), ctx);
 
 	/* Authenticate lr */
@@ -1191,6 +1207,41 @@ static int add_exception_handler(const struct bpf_insn *insn,
 	return 0;
 }
 
+static const u8 stack_arg_reg[] = { A64_R(5), A64_R(6), A64_R(7) };
+
+#define NR_STACK_ARG_REGS	ARRAY_SIZE(stack_arg_reg)
+
+static void emit_stack_arg_load(u8 dst, s16 bpf_off, struct jit_ctx *ctx)
+{
+	int idx = bpf_off / sizeof(u64) - 1;
+
+	if (idx < NR_STACK_ARG_REGS)
+		emit(A64_MOV(1, dst, stack_arg_reg[idx]), ctx);
+	else
+		emit(A64_LDR64I(dst, A64_FP, (idx - NR_STACK_ARG_REGS) * sizeof(u64) + 16), ctx);
+}
+
+static void emit_stack_arg_store(u8 src_a64, s16 bpf_off, struct jit_ctx *ctx)
+{
+	int idx = -bpf_off / sizeof(u64) - 1;
+
+	if (idx < NR_STACK_ARG_REGS)
+		emit(A64_MOV(1, stack_arg_reg[idx], src_a64), ctx);
+	else
+		emit(A64_STR64I(src_a64, A64_SP, (idx - NR_STACK_ARG_REGS) * sizeof(u64)), ctx);
+}
+
+static void emit_stack_arg_store_imm(s32 imm, s16 bpf_off, const u8 tmp, struct jit_ctx *ctx)
+{
+	int idx = -bpf_off / sizeof(u64) - 1;
+
+	emit_a64_mov_i(1, tmp, imm, ctx);
+	if (idx < NR_STACK_ARG_REGS)
+		emit(A64_MOV(1, stack_arg_reg[idx], tmp), ctx);
+	else
+		emit(A64_STR64I(tmp, A64_SP, (idx - NR_STACK_ARG_REGS) * sizeof(u64)), ctx);
+}
+
 /* JITs an eBPF instruction.
  * Returns:
  * 0  - successfully JITed an 8-byte eBPF instruction.
@@ -1646,6 +1697,11 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 	case BPF_LDX | BPF_MEM | BPF_H:
 	case BPF_LDX | BPF_MEM | BPF_B:
 	case BPF_LDX | BPF_MEM | BPF_DW:
+		if (insn->src_reg == BPF_REG_PARAMS) {
+			emit_stack_arg_load(dst, off, ctx);
+			break;
+		}
+		fallthrough;
 	case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
 	case BPF_LDX | BPF_PROBE_MEM | BPF_W:
 	case BPF_LDX | BPF_PROBE_MEM | BPF_H:
@@ -1672,6 +1728,8 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 		if (src == fp) {
 			src_adj = ctx->priv_sp_used ? priv_sp : A64_SP;
 			off_adj = off + ctx->stack_size;
+			if (!ctx->priv_sp_used)
+				off_adj += ctx->stack_arg_size;
 		} else {
 			src_adj = src;
 			off_adj = off;
@@ -1752,6 +1810,11 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 	case BPF_ST | BPF_MEM | BPF_H:
 	case BPF_ST | BPF_MEM | BPF_B:
 	case BPF_ST | BPF_MEM | BPF_DW:
+		if (insn->dst_reg == BPF_REG_PARAMS) {
+			emit_stack_arg_store_imm(imm, off, tmp, ctx);
+			break;
+		}
+		fallthrough;
 	case BPF_ST | BPF_PROBE_MEM32 | BPF_B:
 	case BPF_ST | BPF_PROBE_MEM32 | BPF_H:
 	case BPF_ST | BPF_PROBE_MEM32 | BPF_W:
@@ -1763,6 +1826,8 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 		if (dst == fp) {
 			dst_adj = ctx->priv_sp_used ? priv_sp : A64_SP;
 			off_adj = off + ctx->stack_size;
+			if (!ctx->priv_sp_used)
+				off_adj += ctx->stack_arg_size;
 		} else {
 			dst_adj = dst;
 			off_adj = off;
@@ -1814,6 +1879,11 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 	case BPF_STX | BPF_MEM | BPF_H:
 	case BPF_STX | BPF_MEM | BPF_B:
 	case BPF_STX | BPF_MEM | BPF_DW:
+		if (insn->dst_reg == BPF_REG_PARAMS) {
+			emit_stack_arg_store(src, off, ctx);
+			break;
+		}
+		fallthrough;
 	case BPF_STX | BPF_PROBE_MEM32 | BPF_B:
 	case BPF_STX | BPF_PROBE_MEM32 | BPF_H:
 	case BPF_STX | BPF_PROBE_MEM32 | BPF_W:
@@ -1825,6 +1895,8 @@ static int build_insn(const struct bpf_verifier_env *env, const struct bpf_insn
 		if (dst == fp) {
 			dst_adj = ctx->priv_sp_used ? priv_sp : A64_SP;
 			off_adj = off + ctx->stack_size;
+			if (!ctx->priv_sp_used)
+				off_adj += ctx->stack_arg_size;
 		} else {
 			dst_adj = dst;
 			off_adj = off;
@@ -2018,6 +2090,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 	u8 *ro_image_ptr;
 	int body_idx;
 	int exentry_idx;
+	int out_cnt;
 
 	if (!prog->jit_requested)
 		return prog;
@@ -2065,6 +2138,14 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 	ctx.user_vm_start = bpf_arena_get_user_vm_start(prog->aux->arena);
 	ctx.arena_vm_start = bpf_arena_get_kern_vm_start(prog->aux->arena);
 
+	out_cnt = bpf_out_stack_arg_cnt(env, prog);
+	if (out_cnt) {
+		int nr_on_stack = out_cnt - NR_STACK_ARG_REGS;
+
+		if (nr_on_stack > 0)
+			ctx.stack_arg_size = round_up(nr_on_stack * sizeof(u64), 16);
+	}
+
 	if (priv_stack_ptr)
 		ctx.priv_sp_used = true;
 
@@ -2229,6 +2310,11 @@ bool bpf_jit_supports_kfunc_call(void)
 	return true;
 }
 
+bool bpf_jit_supports_stack_args(void)
+{
+	return true;
+}
+
 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
 {
 	if (!aarch64_insn_copy(dst, src, len))
diff --git a/arch/arm64/net/bpf_timed_may_goto.S b/arch/arm64/net/bpf_timed_may_goto.S
index 894cfcd7b2416..a9a802711a7f9 100644
--- a/arch/arm64/net/bpf_timed_may_goto.S
+++ b/arch/arm64/net/bpf_timed_may_goto.S
@@ -8,8 +8,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
 	stp     x29, x30, [sp, #-64]!
 	mov     x29, sp
 
-	/* Save BPF registers R0 - R5 (x7, x0-x4)*/
-	stp	x7, x0, [sp, #16]
+	/* Save BPF registers R0 - R5 (x8, x0-x4)*/
+	stp	x8, x0, [sp, #16]
 	stp	x1, x2, [sp, #32]
 	stp	x3, x4, [sp, #48]
 
@@ -28,8 +28,8 @@ SYM_FUNC_START(arch_bpf_timed_may_goto)
 	/* BPF_REG_AX(x9) will be stored into count, so move return value to it. */
 	mov	x9, x0
 
-	/* Restore BPF registers R0 - R5 (x7, x0-x4) */
-	ldp	x7, x0, [sp, #16]
+	/* Restore BPF registers R0 - R5 (x8, x0-x4) */
+	ldp	x8, x0, [sp, #16]
 	ldp	x1, x2, [sp, #32]
 	ldp	x3, x4, [sp, #48]
 
diff --git a/arch/arm64/tools/sysreg b/arch/arm64/tools/sysreg
index 6c3ff14e561e6..4516971cea06e 100644
--- a/arch/arm64/tools/sysreg
+++ b/arch/arm64/tools/sysreg
@@ -4597,6 +4597,35 @@ Sysreg	GCSPR_EL2	3	4	2	5	1
 Fields	GCSPR_ELx
 EndSysreg
 
+Sysreg	HDBSSBR_EL2	3	4	2	3	2
+Res0	63:56
+Field	55:12	BADDR
+Res0	11:4
+Enum	3:0	SZ
+	0b0000	4KB
+	0b0001	8KB
+	0b0010	16KB
+	0b0011	32KB
+	0b0100	64KB
+	0b0101	128KB
+	0b0110	256KB
+	0b0111	512KB
+	0b1000	1MB
+	0b1001	2MB
+EndEnum
+EndSysreg
+
+Sysreg	HDBSSPROD_EL2	3	4	2	3	3
+Res0	63:32
+Enum	31:26	FSC
+	0b000000	OK
+	0b010000	ExternalAbort
+	0b101000	GPF
+EndEnum
+Res0	25:19
+Field	18:0	INDEX
+EndSysreg
+
 Sysreg	DACR32_EL2	3	4	3	0	0
 Res0	63:32
 Field	31:30	D15
diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h
index bff77efc0d9a9..1aecf60ec4f5a 100644
--- a/arch/hexagon/include/asm/uaccess.h
+++ b/arch/hexagon/include/asm/uaccess.h
@@ -26,8 +26,7 @@ unsigned long raw_copy_from_user(void *to, const void __user *from,
 				     unsigned long n);
 unsigned long raw_copy_to_user(void __user *to, const void *from,
 				   unsigned long n);
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 __kernel_size_t __clear_user_hexagon(void __user *dest, unsigned long count);
 #define __clear_user(a, s) __clear_user_hexagon((a), (s))
diff --git a/arch/loongarch/configs/loongson32_defconfig b/arch/loongarch/configs/loongson32_defconfig
index d5ef396dffe30..f319c880127c2 100644
--- a/arch/loongarch/configs/loongson32_defconfig
+++ b/arch/loongarch/configs/loongson32_defconfig
@@ -383,9 +383,6 @@ CONFIG_BT_HCIUART_AG6XX=y
 CONFIG_BT_HCIBCM203X=m
 CONFIG_BT_HCIBPA10X=m
 CONFIG_BT_HCIBFUSB=m
-CONFIG_BT_HCIDTL1=m
-CONFIG_BT_HCIBT3C=m
-CONFIG_BT_HCIBLUECARD=m
 CONFIG_BT_HCIVHCI=m
 CONFIG_BT_MRVL=m
 CONFIG_BT_ATH3K=m
diff --git a/arch/loongarch/configs/loongson64_defconfig b/arch/loongarch/configs/loongson64_defconfig
index cba4cdff5acd8..ffa8ae26d7138 100644
--- a/arch/loongarch/configs/loongson64_defconfig
+++ b/arch/loongarch/configs/loongson64_defconfig
@@ -399,9 +399,6 @@ CONFIG_BT_HCIUART_AG6XX=y
 CONFIG_BT_HCIBCM203X=m
 CONFIG_BT_HCIBPA10X=m
 CONFIG_BT_HCIBFUSB=m
-CONFIG_BT_HCIDTL1=m
-CONFIG_BT_HCIBT3C=m
-CONFIG_BT_HCIBLUECARD=m
 CONFIG_BT_HCIVHCI=m
 CONFIG_BT_MRVL=m
 CONFIG_BT_ATH3K=m
diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h
index 438269313e78c..428f373feabf1 100644
--- a/arch/loongarch/include/asm/uaccess.h
+++ b/arch/loongarch/include/asm/uaccess.h
@@ -292,8 +292,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 	return __copy_user((__force void *)to, from, n);
 }
 
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 /*
  * __clear_user: - Zero a block of memory in user space, with less checking.
diff --git a/arch/loongarch/mm/init.c b/arch/loongarch/mm/init.c
index 3f9ab54114c51..055ecd2c8fd94 100644
--- a/arch/loongarch/mm/init.c
+++ b/arch/loongarch/mm/init.c
@@ -119,7 +119,8 @@ int arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *params)
 	return ret;
 }
 
-void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+			struct dev_pagemap *pgmap)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
@@ -128,7 +129,7 @@ void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
 	/* With altmap the first mapped page is offset from @start */
 	if (altmap)
 		page += vmem_altmap_offset(altmap);
-	__remove_pages(start_pfn, nr_pages, altmap);
+	__remove_pages(start_pfn, nr_pages, altmap, pgmap);
 }
 #endif
 
diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu
index c9a7e602d8a4d..148f8a79d2068 100644
--- a/arch/m68k/Kconfig.cpu
+++ b/arch/m68k/Kconfig.cpu
@@ -30,6 +30,7 @@ config COLDFIRE
 	select CPU_HAS_NO_MULDIV64
 	select GENERIC_CSUM
 	select GPIOLIB
+	select GPIOLIB_LEGACY if SPI_COLDFIRE_QSPI
 	select HAVE_LEGACY_CLK
 	select HAVE_PAGE_SIZE_8KB if !MMU
 
diff --git a/arch/m68k/Kconfig.machine b/arch/m68k/Kconfig.machine
index 624e6b27f3944..cff990c319cb6 100644
--- a/arch/m68k/Kconfig.machine
+++ b/arch/m68k/Kconfig.machine
@@ -277,16 +277,16 @@ config M5407C3
 	  Support for the Motorola M5407C3 board.
 
 config AMCORE
-	bool "Sysam AMCORE board support"
+	bool "Kernelspace AMCORE board support"
 	depends on M5307
 	help
-	  Support for the Sysam AMCORE open-hardware generic board.
+	  Support for the Kernelspace AMCORE open-hardware generic board.
 
 config STMARK2
-	bool "Sysam stmark2 board support"
+	bool "Kernelspace stmark2 board support"
 	depends on M5441x
 	help
-	  Support for the Sysam stmark2 open-hardware generic board.
+	  Support for the Kernelspace stmark2 open-hardware generic board.
 
 config FIREBEE
 	bool "FireBee board support"
diff --git a/arch/m68k/coldfire/amcore.c b/arch/m68k/coldfire/amcore.c
index c6cb1a5cc1a5e..6531ef62bc013 100644
--- a/arch/m68k/coldfire/amcore.c
+++ b/arch/m68k/coldfire/amcore.c
@@ -1,7 +1,7 @@
 /*
- * amcore.c -- Support for Sysam AMCORE open board
+ * amcore.c -- Support for Kernelspace AMCORE open board
  *
- * (C) Copyright 2016, Angelo Dureghello <angelo@sysam.it>
+ * (C) Copyright 2026, Angelo Dureghello <angelo@kernel-space.org>
  *
  * This file is subject to the terms and conditions of the GNU General Public
  * License.  See the file COPYING in the main directory of this archive
diff --git a/arch/m68k/coldfire/clk.c b/arch/m68k/coldfire/clk.c
index d03b6c4aa86b4..c120bbca6c1b3 100644
--- a/arch/m68k/coldfire/clk.c
+++ b/arch/m68k/coldfire/clk.c
@@ -42,12 +42,12 @@ void __clk_init_disabled(struct clk *clk)
 
 static void __clk_enable0(struct clk *clk)
 {
-	__raw_writeb(clk->slot, MCFPM_PPMCR0);
+	mcf_write8(clk->slot, MCFPM_PPMCR0);
 }
 
 static void __clk_disable0(struct clk *clk)
 {
-	__raw_writeb(clk->slot, MCFPM_PPMSR0);
+	mcf_write8(clk->slot, MCFPM_PPMSR0);
 }
 
 struct clk_ops clk_ops0 = {
@@ -58,12 +58,12 @@ struct clk_ops clk_ops0 = {
 #ifdef MCFPM_PPMCR1
 static void __clk_enable1(struct clk *clk)
 {
-	__raw_writeb(clk->slot, MCFPM_PPMCR1);
+	mcf_write8(clk->slot, MCFPM_PPMCR1);
 }
 
 static void __clk_disable1(struct clk *clk)
 {
-	__raw_writeb(clk->slot, MCFPM_PPMSR1);
+	mcf_write8(clk->slot, MCFPM_PPMSR1);
 }
 
 struct clk_ops clk_ops1 = {
diff --git a/arch/m68k/coldfire/device.c b/arch/m68k/coldfire/device.c
index b6958ec2a220c..1420bae0964f3 100644
--- a/arch/m68k/coldfire/device.c
+++ b/arch/m68k/coldfire/device.c
@@ -669,13 +669,13 @@ static void __init mcf_uart_set_irq(void)
 {
 #ifdef MCFUART_UIVR
 	/* UART0 interrupt setup */
-	writeb(MCFSIM_ICR_LEVEL6 | MCFSIM_ICR_PRI1, MCFSIM_UART1ICR);
-	writeb(MCF_IRQ_UART0, MCFUART_BASE0 + MCFUART_UIVR);
+	mcf_write8(MCFSIM_ICR_LEVEL6 | MCFSIM_ICR_PRI1, MCFSIM_UART1ICR);
+	mcf_write8(MCF_IRQ_UART0, MCFUART_BASE0 + MCFUART_UIVR);
 	mcf_mapirq2imr(MCF_IRQ_UART0, MCFINTC_UART0);
 
 	/* UART1 interrupt setup */
-	writeb(MCFSIM_ICR_LEVEL6 | MCFSIM_ICR_PRI2, MCFSIM_UART2ICR);
-	writeb(MCF_IRQ_UART1, MCFUART_BASE1 + MCFUART_UIVR);
+	mcf_write8(MCFSIM_ICR_LEVEL6 | MCFSIM_ICR_PRI2, MCFSIM_UART2ICR);
+	mcf_write8(MCF_IRQ_UART1, MCFUART_BASE1 + MCFUART_UIVR);
 	mcf_mapirq2imr(MCF_IRQ_UART1, MCFINTC_UART1);
 #endif
 }
diff --git a/arch/m68k/coldfire/dma_timer.c b/arch/m68k/coldfire/dma_timer.c
index 91e6728f51ed6..0605050d83c2b 100644
--- a/arch/m68k/coldfire/dma_timer.c
+++ b/arch/m68k/coldfire/dma_timer.c
@@ -37,7 +37,7 @@
 
 static u64 cf_dt_get_cycles(struct clocksource *cs)
 {
-	return __raw_readl(DTCN0);
+	return mcf_read32(DTCN0);
 }
 
 static struct clocksource clocksource_cf_dt = {
@@ -56,10 +56,10 @@ static int __init init_cf_dt_clocksource(void)
 	 * get a ~213 ns resolution and the 32bit register will overflow almost
 	 * every 15 minutes.
 	 */
-	__raw_writeb(0x00, DTXMR0);
-	__raw_writeb(0x00, DTER0);
-	__raw_writel(0x00000000, DTRR0);
-	__raw_writew(DMA_DTMR_CLK_DIV_16 | DMA_DTMR_ENABLE, DTMR0);
+	mcf_write8(0x00, DTXMR0);
+	mcf_write8(0x00, DTER0);
+	mcf_write32(0x00000000, DTRR0);
+	mcf_write16(DMA_DTMR_CLK_DIV_16 | DMA_DTMR_ENABLE, DTMR0);
 	return clocksource_register_hz(&clocksource_cf_dt, DMA_FREQ);
 }
 
@@ -76,7 +76,7 @@ static unsigned long long cycles2ns(unsigned long cycl)
 
 unsigned long long sched_clock(void)
 {
-	unsigned long cycl = __raw_readl(DTCN0);
+	unsigned long cycl = mcf_read32(DTCN0);
 
 	return cycles2ns(cycl);
 }
diff --git a/arch/m68k/coldfire/intc-2.c b/arch/m68k/coldfire/intc-2.c
index f74f0e4731193..c5e8e0ccded49 100644
--- a/arch/m68k/coldfire/intc-2.c
+++ b/arch/m68k/coldfire/intc-2.c
@@ -61,8 +61,8 @@ static void intc_irq_mask(struct irq_data *d)
 	imraddr += (irq & 0x20) ? MCFINTC_IMRH : MCFINTC_IMRL;
 	imrbit = 0x1 << (irq & 0x1f);
 
-	val = __raw_readl(imraddr);
-	__raw_writel(val | imrbit, imraddr);
+	val = mcf_read32(imraddr);
+	mcf_write32(val | imrbit, imraddr);
 }
 
 static void intc_irq_unmask(struct irq_data *d)
@@ -83,8 +83,8 @@ static void intc_irq_unmask(struct irq_data *d)
 	if ((irq & 0x20) == 0)
 		imrbit |= 0x1;
 
-	val = __raw_readl(imraddr);
-	__raw_writel(val & ~imrbit, imraddr);
+	val = mcf_read32(imraddr);
+	mcf_write32(val & ~imrbit, imraddr);
 }
 
 /*
@@ -97,7 +97,7 @@ static void intc_irq_ack(struct irq_data *d)
 {
 	unsigned int irq = d->irq;
 
-	__raw_writeb(0x1 << (irq - EINT0), MCFEPORT_EPFR);
+	mcf_write8(0x1 << (irq - EINT0), MCFEPORT_EPFR);
 }
 
 /*
@@ -120,8 +120,8 @@ static unsigned int intc_irq_startup(struct irq_data *d)
 	icraddr = MCFICM_INTC0;
 #endif
 	icraddr += MCFINTC_ICR0 + (irq & 0x3f);
-	if (__raw_readb(icraddr) == 0)
-		__raw_writeb(intc_intpri--, icraddr);
+	if (mcf_read8(icraddr) == 0)
+		mcf_write8(intc_intpri--, icraddr);
 
 	irq = d->irq;
 	if ((irq >= EINT1) && (irq <= EINT7)) {
@@ -130,12 +130,12 @@ static unsigned int intc_irq_startup(struct irq_data *d)
 		irq -= EINT0;
 
 		/* Set EPORT line as input */
-		v = __raw_readb(MCFEPORT_EPDDR);
-		__raw_writeb(v & ~(0x1 << irq), MCFEPORT_EPDDR);
+		v = mcf_read8(MCFEPORT_EPDDR);
+		mcf_write8(v & ~(0x1 << irq), MCFEPORT_EPDDR);
 
 		/* Set EPORT line as interrupt source */
-		v = __raw_readb(MCFEPORT_EPIER);
-		__raw_writeb(v | (0x1 << irq), MCFEPORT_EPIER);
+		v = mcf_read8(MCFEPORT_EPIER);
+		mcf_write8(v | (0x1 << irq), MCFEPORT_EPIER);
 	}
 
 	intc_irq_unmask(d);
@@ -167,9 +167,9 @@ static int intc_irq_set_type(struct irq_data *d, unsigned int type)
 		irq_set_handler(irq, handle_edge_irq);
 
 	irq -= EINT0;
-	pa = __raw_readw(MCFEPORT_EPPAR);
+	pa = mcf_read16(MCFEPORT_EPPAR);
 	pa = (pa & ~(0x3 << (irq * 2))) | (tb << (irq * 2));
-	__raw_writew(pa, MCFEPORT_EPPAR);
+	mcf_write16(pa, MCFEPORT_EPPAR);
 	
 	return 0;
 }
@@ -195,9 +195,9 @@ void __init init_IRQ(void)
 	int irq;
 
 	/* Mask all interrupt sources */
-	__raw_writel(0x1, MCFICM_INTC0 + MCFINTC_IMRL);
+	mcf_write32(0x1, MCFICM_INTC0 + MCFINTC_IMRL);
 #ifdef MCFICM_INTC1
-	__raw_writel(0x1, MCFICM_INTC1 + MCFINTC_IMRL);
+	mcf_write32(0x1, MCFICM_INTC1 + MCFINTC_IMRL);
 #endif
 
 	for (irq = MCFINT_VECBASE; (irq < MCFINT_VECBASE + NR_VECS); irq++) {
diff --git a/arch/m68k/coldfire/intc-5249.c b/arch/m68k/coldfire/intc-5249.c
index b0d1641053e4e..f1636a9c96c84 100644
--- a/arch/m68k/coldfire/intc-5249.c
+++ b/arch/m68k/coldfire/intc-5249.c
@@ -20,22 +20,22 @@
 static void intc2_irq_gpio_mask(struct irq_data *d)
 {
 	u32 imr;
-	imr = readl(MCFSIM2_GPIOINTENABLE);
+	imr = mcf_read32(MCFSIM2_GPIOINTENABLE);
 	imr &= ~(0x1 << (d->irq - MCF_IRQ_GPIO0));
-	writel(imr, MCFSIM2_GPIOINTENABLE);
+	mcf_write32(imr, MCFSIM2_GPIOINTENABLE);
 }
 
 static void intc2_irq_gpio_unmask(struct irq_data *d)
 {
 	u32 imr;
-	imr = readl(MCFSIM2_GPIOINTENABLE);
+	imr = mcf_read32(MCFSIM2_GPIOINTENABLE);
 	imr |= (0x1 << (d->irq - MCF_IRQ_GPIO0));
-	writel(imr, MCFSIM2_GPIOINTENABLE);
+	mcf_write32(imr, MCFSIM2_GPIOINTENABLE);
 }
 
 static void intc2_irq_gpio_ack(struct irq_data *d)
 {
-	writel(0x1 << (d->irq - MCF_IRQ_GPIO0), MCFSIM2_GPIOINTCLEAR);
+	mcf_write32(0x1 << (d->irq - MCF_IRQ_GPIO0), MCFSIM2_GPIOINTCLEAR);
 }
 
 static struct irq_chip intc2_irq_gpio_chip = {
diff --git a/arch/m68k/coldfire/intc-525x.c b/arch/m68k/coldfire/intc-525x.c
index b23204d059acd..09a8752c96e70 100644
--- a/arch/m68k/coldfire/intc-525x.c
+++ b/arch/m68k/coldfire/intc-525x.c
@@ -20,7 +20,7 @@
 
 static void intc2_irq_gpio_mask(struct irq_data *d)
 {
-	u32 imr = readl(MCFSIM2_GPIOINTENABLE);
+	u32 imr = mcf_read32(MCFSIM2_GPIOINTENABLE);
 	u32 type = irqd_get_trigger_type(d);
 	int irq = d->irq - MCF_IRQ_GPIO0;
 
@@ -28,12 +28,12 @@ static void intc2_irq_gpio_mask(struct irq_data *d)
 		imr &= ~(0x001 << irq);
 	if (type & IRQ_TYPE_EDGE_FALLING)
 		imr &= ~(0x100 << irq);
-	writel(imr, MCFSIM2_GPIOINTENABLE);
+	mcf_write32(imr, MCFSIM2_GPIOINTENABLE);
 }
 
 static void intc2_irq_gpio_unmask(struct irq_data *d)
 {
-	u32 imr = readl(MCFSIM2_GPIOINTENABLE);
+	u32 imr = mcf_read32(MCFSIM2_GPIOINTENABLE);
 	u32 type = irqd_get_trigger_type(d);
 	int irq = d->irq - MCF_IRQ_GPIO0;
 
@@ -41,7 +41,7 @@ static void intc2_irq_gpio_unmask(struct irq_data *d)
 		imr |= (0x001 << irq);
 	if (type & IRQ_TYPE_EDGE_FALLING)
 		imr |= (0x100 << irq);
-	writel(imr, MCFSIM2_GPIOINTENABLE);
+	mcf_write32(imr, MCFSIM2_GPIOINTENABLE);
 }
 
 static void intc2_irq_gpio_ack(struct irq_data *d)
@@ -54,7 +54,7 @@ static void intc2_irq_gpio_ack(struct irq_data *d)
 		imr |= (0x001 << irq);
 	if (type & IRQ_TYPE_EDGE_FALLING)
 		imr |= (0x100 << irq);
-	writel(imr, MCFSIM2_GPIOINTCLEAR);
+	mcf_write32(imr, MCFSIM2_GPIOINTCLEAR);
 }
 
 static int intc2_irq_gpio_set_type(struct irq_data *d, unsigned int f)
@@ -77,7 +77,7 @@ static int __init mcf_intc2_init(void)
 	int irq;
 
 	/* set the interrupt base for the second interrupt controller */
-	writel(MCFINTC2_VECBASE, MCFINTC2_INTBASE);
+	mcf_write32(MCFINTC2_VECBASE, MCFINTC2_INTBASE);
 
 	/* GPIO interrupt sources */
 	for (irq = MCF_IRQ_GPIO0; (irq <= MCF_IRQ_GPIO6); irq++) {
diff --git a/arch/m68k/coldfire/intc-5272.c b/arch/m68k/coldfire/intc-5272.c
index b0a19e207a636..8d7e188cdb44f 100644
--- a/arch/m68k/coldfire/intc-5272.c
+++ b/arch/m68k/coldfire/intc-5272.c
@@ -86,7 +86,7 @@ static void intc_irq_mask(struct irq_data *d)
 		u32 v;
 		irq -= MCFINT_VECBASE;
 		v = 0x8 << intc_irqmap[irq].index;
-		writel(v, intc_irqmap[irq].icr);
+		mcf_write32(v, intc_irqmap[irq].icr);
 	}
 }
 
@@ -98,7 +98,7 @@ static void intc_irq_unmask(struct irq_data *d)
 		u32 v;
 		irq -= MCFINT_VECBASE;
 		v = 0xd << intc_irqmap[irq].index;
-		writel(v, intc_irqmap[irq].icr);
+		mcf_write32(v, intc_irqmap[irq].icr);
 	}
 }
 
@@ -111,10 +111,10 @@ static void intc_irq_ack(struct irq_data *d)
 		irq -= MCFINT_VECBASE;
 		if (intc_irqmap[irq].ack) {
 			u32 v;
-			v = readl(intc_irqmap[irq].icr);
+			v = mcf_read32(intc_irqmap[irq].icr);
 			v &= (0x7 << intc_irqmap[irq].index);
 			v |= (0x8 << intc_irqmap[irq].index);
-			writel(v, intc_irqmap[irq].icr);
+			mcf_write32(v, intc_irqmap[irq].icr);
 		}
 	}
 }
@@ -127,12 +127,12 @@ static int intc_irq_set_type(struct irq_data *d, unsigned int type)
 		irq -= MCFINT_VECBASE;
 		if (intc_irqmap[irq].ack) {
 			u32 v;
-			v = readl(MCFSIM_PITR);
+			v = mcf_read32(MCFSIM_PITR);
 			if (type == IRQ_TYPE_EDGE_FALLING)
 				v &= ~(0x1 << (32 - irq));
 			else
 				v |= (0x1 << (32 - irq));
-			writel(v, MCFSIM_PITR);
+			mcf_write32(v, MCFSIM_PITR);
 		}
 	}
 	return 0;
@@ -163,10 +163,10 @@ void __init init_IRQ(void)
 	int irq, edge;
 
 	/* Mask all interrupt sources */
-	writel(0x88888888, MCFSIM_ICR1);
-	writel(0x88888888, MCFSIM_ICR2);
-	writel(0x88888888, MCFSIM_ICR3);
-	writel(0x88888888, MCFSIM_ICR4);
+	mcf_write32(0x88888888, MCFSIM_ICR1);
+	mcf_write32(0x88888888, MCFSIM_ICR2);
+	mcf_write32(0x88888888, MCFSIM_ICR3);
+	mcf_write32(0x88888888, MCFSIM_ICR4);
 
 	for (irq = 0; (irq < NR_IRQS); irq++) {
 		irq_set_chip(irq, &intc_irq_chip);
diff --git a/arch/m68k/coldfire/intc-simr.c b/arch/m68k/coldfire/intc-simr.c
index f7c2c41b31564..2cbda945f0e4d 100644
--- a/arch/m68k/coldfire/intc-simr.c
+++ b/arch/m68k/coldfire/intc-simr.c
@@ -69,11 +69,11 @@ static void intc_irq_mask(struct irq_data *d)
 	unsigned int irq = d->irq - MCFINT_VECBASE;
 
 	if (MCFINTC2_SIMR && (irq > 127))
-		__raw_writeb(irq - 128, MCFINTC2_SIMR);
+		mcf_write8(irq - 128, MCFINTC2_SIMR);
 	else if (MCFINTC1_SIMR && (irq > 63))
-		__raw_writeb(irq - 64, MCFINTC1_SIMR);
+		mcf_write8(irq - 64, MCFINTC1_SIMR);
 	else
-		__raw_writeb(irq, MCFINTC0_SIMR);
+		mcf_write8(irq, MCFINTC0_SIMR);
 }
 
 static void intc_irq_unmask(struct irq_data *d)
@@ -81,18 +81,18 @@ static void intc_irq_unmask(struct irq_data *d)
 	unsigned int irq = d->irq - MCFINT_VECBASE;
 
 	if (MCFINTC2_CIMR && (irq > 127))
-		__raw_writeb(irq - 128, MCFINTC2_CIMR);
+		mcf_write8(irq - 128, MCFINTC2_CIMR);
 	else if (MCFINTC1_CIMR && (irq > 63))
-		__raw_writeb(irq - 64, MCFINTC1_CIMR);
+		mcf_write8(irq - 64, MCFINTC1_CIMR);
 	else
-		__raw_writeb(irq, MCFINTC0_CIMR);
+		mcf_write8(irq, MCFINTC0_CIMR);
 }
 
 static void intc_irq_ack(struct irq_data *d)
 {
 	unsigned int ebit = irq2ebit(d->irq);
 
-	__raw_writeb(0x1 << ebit, MCFEPORT_EPFR);
+	mcf_write8(0x1 << ebit, MCFEPORT_EPFR);
 }
 
 static unsigned int intc_irq_startup(struct irq_data *d)
@@ -105,22 +105,22 @@ static unsigned int intc_irq_startup(struct irq_data *d)
 
 #if defined(MCFEPORT_EPDDR)
 		/* Set EPORT line as input */
-		v = __raw_readb(MCFEPORT_EPDDR);
-		__raw_writeb(v & ~(0x1 << ebit), MCFEPORT_EPDDR);
+		v = mcf_read8(MCFEPORT_EPDDR);
+		mcf_write8(v & ~(0x1 << ebit), MCFEPORT_EPDDR);
 #endif
 
 		/* Set EPORT line as interrupt source */
-		v = __raw_readb(MCFEPORT_EPIER);
-		__raw_writeb(v | (0x1 << ebit), MCFEPORT_EPIER);
+		v = mcf_read8(MCFEPORT_EPIER);
+		mcf_write8(v | (0x1 << ebit), MCFEPORT_EPIER);
 	}
 
 	irq -= MCFINT_VECBASE;
 	if (MCFINTC2_ICR0 && (irq > 127))
-		__raw_writeb(5, MCFINTC2_ICR0 + irq - 128);
+		mcf_write8(5, MCFINTC2_ICR0 + irq - 128);
 	else if (MCFINTC1_ICR0 && (irq > 63))
-		__raw_writeb(5, MCFINTC1_ICR0 + irq - 64);
+		mcf_write8(5, MCFINTC1_ICR0 + irq - 64);
 	else
-		__raw_writeb(5, MCFINTC0_ICR0 + irq);
+		mcf_write8(5, MCFINTC0_ICR0 + irq);
 
 	intc_irq_unmask(d);
 	return 0;
@@ -151,9 +151,9 @@ static int intc_irq_set_type(struct irq_data *d, unsigned int type)
 		irq_set_handler(irq, handle_edge_irq);
 
 	ebit = irq2ebit(irq) * 2;
-	pa = __raw_readw(MCFEPORT_EPPAR);
+	pa = mcf_read16(MCFEPORT_EPPAR);
 	pa = (pa & ~(0x3 << ebit)) | (tb << ebit);
-	__raw_writew(pa, MCFEPORT_EPPAR);
+	mcf_write16(pa, MCFEPORT_EPPAR);
 	
 	return 0;
 }
@@ -179,11 +179,11 @@ void __init init_IRQ(void)
 	int irq, eirq;
 
 	/* Mask all interrupt sources */
-	__raw_writeb(0xff, MCFINTC0_SIMR);
+	mcf_write8(0xff, MCFINTC0_SIMR);
 	if (MCFINTC1_SIMR)
-		__raw_writeb(0xff, MCFINTC1_SIMR);
+		mcf_write8(0xff, MCFINTC1_SIMR);
 	if (MCFINTC2_SIMR)
-		__raw_writeb(0xff, MCFINTC2_SIMR);
+		mcf_write8(0xff, MCFINTC2_SIMR);
 
 	eirq = MCFINT_VECBASE + 64 + (MCFINTC1_ICR0 ? 64 : 0) +
 						(MCFINTC2_ICR0 ? 64 : 0);
diff --git a/arch/m68k/coldfire/intc.c b/arch/m68k/coldfire/intc.c
index b434371e2b99e..f441ecf7c17c7 100644
--- a/arch/m68k/coldfire/intc.c
+++ b/arch/m68k/coldfire/intc.c
@@ -45,23 +45,23 @@ unsigned char mcf_irq2imr[NR_IRQS];
 void mcf_setimr(int index)
 {
 	u16 imr;
-	imr = __raw_readw(MCFSIM_IMR);
-	__raw_writew(imr | (0x1 << index), MCFSIM_IMR);
+	imr = mcf_read16(MCFSIM_IMR);
+	mcf_write16(imr | (0x1 << index), MCFSIM_IMR);
 }
 
 void mcf_clrimr(int index)
 {
 	u16 imr;
-	imr = __raw_readw(MCFSIM_IMR);
-	__raw_writew(imr & ~(0x1 << index), MCFSIM_IMR);
+	imr = mcf_read16(MCFSIM_IMR);
+	mcf_write16(imr & ~(0x1 << index), MCFSIM_IMR);
 }
 
 static void mcf_maskimr(unsigned int mask)
 {
 	u16 imr;
-	imr = __raw_readw(MCFSIM_IMR);
+	imr = mcf_read16(MCFSIM_IMR);
 	imr |= mask;
-	__raw_writew(imr, MCFSIM_IMR);
+	mcf_write16(imr, MCFSIM_IMR);
 }
 
 #else
@@ -69,23 +69,23 @@ static void mcf_maskimr(unsigned int mask)
 void mcf_setimr(int index)
 {
 	u32 imr;
-	imr = __raw_readl(MCFSIM_IMR);
-	__raw_writel(imr | (0x1 << index), MCFSIM_IMR);
+	imr = mcf_read32(MCFSIM_IMR);
+	mcf_write32(imr | (0x1 << index), MCFSIM_IMR);
 }
 
 void mcf_clrimr(int index)
 {
 	u32 imr;
-	imr = __raw_readl(MCFSIM_IMR);
-	__raw_writel(imr & ~(0x1 << index), MCFSIM_IMR);
+	imr = mcf_read32(MCFSIM_IMR);
+	mcf_write32(imr & ~(0x1 << index), MCFSIM_IMR);
 }
 
 static void mcf_maskimr(unsigned int mask)
 {
 	u32 imr;
-	imr = __raw_readl(MCFSIM_IMR);
+	imr = mcf_read32(MCFSIM_IMR);
 	imr |= mask;
-	__raw_writel(imr, MCFSIM_IMR);
+	mcf_write32(imr, MCFSIM_IMR);
 }
 
 #endif
@@ -104,9 +104,9 @@ void mcf_autovector(int irq)
 #ifdef MCFSIM_AVR
 	if ((irq >= EIRQ1) && (irq <= EIRQ7)) {
 		u8 avec;
-		avec = __raw_readb(MCFSIM_AVR);
+		avec = mcf_read8(MCFSIM_AVR);
 		avec |= (0x1 << (irq - EIRQ1 + 1));
-		__raw_writeb(avec, MCFSIM_AVR);
+		mcf_write8(avec, MCFSIM_AVR);
 	}
 #endif
 }
diff --git a/arch/m68k/coldfire/m5206.c b/arch/m68k/coldfire/m5206.c
index 5e726e94b5ab4..eff7d56bbb94f 100644
--- a/arch/m68k/coldfire/m5206.c
+++ b/arch/m68k/coldfire/m5206.c
@@ -40,7 +40,7 @@ static struct clk_lookup m5206_clk_lookup[] = {
 static void __init m5206_i2c_init(void)
 {
 #if IS_ENABLED(CONFIG_I2C_IMX)
-	writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
+	mcf_write8(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
 	       MCFSIM_I2CICR);
 	mcf_mapirq2imr(MCF_IRQ_I2C0, MCFINTC_I2C);
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
diff --git a/arch/m68k/coldfire/m520x.c b/arch/m68k/coldfire/m520x.c
index d2f96b40aee17..08644be99837f 100644
--- a/arch/m68k/coldfire/m520x.c
+++ b/arch/m68k/coldfire/m520x.c
@@ -125,11 +125,11 @@ static void __init m520x_qspi_init(void)
 #if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
 	u16 par;
 	/* setup Port QS for QSPI with gpio CS control */
-	writeb(0x3f, MCF_GPIO_PAR_QSPI);
+	mcf_write8(0x3f, MCF_GPIO_PAR_QSPI);
 	/* make U1CTS and U2RTS gpio for cs_control */
-	par = readw(MCF_GPIO_PAR_UART);
+	par = mcf_read16(MCF_GPIO_PAR_UART);
 	par &= 0x00ff;
-	writew(par, MCF_GPIO_PAR_UART);
+	mcf_write16(par, MCF_GPIO_PAR_UART);
 #endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */
 }
 
@@ -142,9 +142,9 @@ static void __init m520x_i2c_init(void)
 
 	/* setup Port FECI2C Pin Assignment Register for I2C */
 	/*  set PAR_SCL to SCL and PAR_SDA to SDA */
-	par = readb(MCF_GPIO_PAR_FECI2C);
+	par = mcf_read8(MCF_GPIO_PAR_FECI2C);
 	par |= 0x0f;
-	writeb(par, MCF_GPIO_PAR_FECI2C);
+	mcf_write8(par, MCF_GPIO_PAR_FECI2C);
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
 }
 
@@ -156,17 +156,17 @@ static void __init m520x_uarts_init(void)
 	u8 par2;
 
 	/* UART0 and UART1 GPIO pin setup */
-	par = readw(MCF_GPIO_PAR_UART);
+	par = mcf_read16(MCF_GPIO_PAR_UART);
 	par |= MCF_GPIO_PAR_UART_PAR_UTXD0 | MCF_GPIO_PAR_UART_PAR_URXD0;
 	par |= MCF_GPIO_PAR_UART_PAR_UTXD1 | MCF_GPIO_PAR_UART_PAR_URXD1;
-	writew(par, MCF_GPIO_PAR_UART);
+	mcf_write16(par, MCF_GPIO_PAR_UART);
 
 	/* UART1 GPIO pin setup */
-	par2 = readb(MCF_GPIO_PAR_FECI2C);
+	par2 = mcf_read8(MCF_GPIO_PAR_FECI2C);
 	par2 &= ~0x0F;
 	par2 |= MCF_GPIO_PAR_FECI2C_PAR_SCL_UTXD2 |
 		MCF_GPIO_PAR_FECI2C_PAR_SDA_URXD2;
-	writeb(par2, MCF_GPIO_PAR_FECI2C);
+	mcf_write8(par2, MCF_GPIO_PAR_FECI2C);
 }
 
 /***************************************************************************/
@@ -176,11 +176,11 @@ static void __init m520x_fec_init(void)
 	u8 v;
 
 	/* Set multi-function pins to ethernet mode */
-	v = readb(MCF_GPIO_PAR_FEC);
-	writeb(v | 0xf0, MCF_GPIO_PAR_FEC);
+	v = mcf_read8(MCF_GPIO_PAR_FEC);
+	mcf_write8(v | 0xf0, MCF_GPIO_PAR_FEC);
 
-	v = readb(MCF_GPIO_PAR_FECI2C);
-	writeb(v | 0x0f, MCF_GPIO_PAR_FECI2C);
+	v = mcf_read8(MCF_GPIO_PAR_FECI2C);
+	mcf_write8(v | 0x0f, MCF_GPIO_PAR_FECI2C);
 }
 
 /***************************************************************************/
diff --git a/arch/m68k/coldfire/m523x.c b/arch/m68k/coldfire/m523x.c
index 83a997313393c..11d7423ef6463 100644
--- a/arch/m68k/coldfire/m523x.c
+++ b/arch/m68k/coldfire/m523x.c
@@ -51,11 +51,11 @@ static void __init m523x_qspi_init(void)
 	u16 par;
 
 	/* setup QSPS pins for QSPI with gpio CS control */
-	writeb(0x1f, MCFGPIO_PAR_QSPI);
+	mcf_write8(0x1f, MCFGPIO_PAR_QSPI);
 	/* and CS2 & CS3 as gpio */
-	par = readw(MCFGPIO_PAR_TIMER);
+	par = mcf_read16(MCFGPIO_PAR_TIMER);
 	par &= 0x3f3f;
-	writew(par, MCFGPIO_PAR_TIMER);
+	mcf_write16(par, MCFGPIO_PAR_TIMER);
 #endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */
 }
 
@@ -68,9 +68,9 @@ static void __init m523x_i2c_init(void)
 
 	/* setup Port AS Pin Assignment Register for I2C */
 	/*  set PASPA0 to SCL and PASPA1 to SDA */
-	par = readb(MCFGPIO_PAR_FECI2C);
+	par = mcf_read8(MCFGPIO_PAR_FECI2C);
 	par |= 0x0f;
-	writeb(par, MCFGPIO_PAR_FECI2C);
+	mcf_write8(par, MCFGPIO_PAR_FECI2C);
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
 }
 
@@ -79,7 +79,7 @@ static void __init m523x_i2c_init(void)
 static void __init m523x_fec_init(void)
 {
 	/* Set multi-function pins to ethernet use */
-	writeb(readb(MCFGPIO_PAR_FECI2C) | 0xf0, MCFGPIO_PAR_FECI2C);
+	mcf_write8(read8(MCFGPIO_PAR_FECI2C) | 0xf0, MCFGPIO_PAR_FECI2C);
 }
 
 /***************************************************************************/
diff --git a/arch/m68k/coldfire/m5249.c b/arch/m68k/coldfire/m5249.c
index 6d66972de2144..5a0495a2b6fe0 100644
--- a/arch/m68k/coldfire/m5249.c
+++ b/arch/m68k/coldfire/m5249.c
@@ -75,7 +75,7 @@ static void __init m5249_qspi_init(void)
 {
 #if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
 	/* QSPI irq setup */
-	writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL4 | MCFSIM_ICR_PRI0,
+	mcf_write8(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL4 | MCFSIM_ICR_PRI0,
 	       MCFSIM_QSPIICR);
 	mcf_mapirq2imr(MCF_IRQ_QSPI, MCFINTC_QSPI);
 #endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */
@@ -89,15 +89,15 @@ static void __init m5249_i2c_init(void)
 	u32 r;
 
 	/* first I2C controller uses regular irq setup */
-	writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
+	mcf_write8(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
 	       MCFSIM_I2CICR);
 	mcf_mapirq2imr(MCF_IRQ_I2C0, MCFINTC_I2C);
 
 	/* second I2C controller is completely different */
-	r = readl(MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
+	r = mcf_read32(MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
 	r &= ~MCFINTC2_INTPRI_BITS(0xf, MCF_IRQ_I2C1);
 	r |= MCFINTC2_INTPRI_BITS(0x5, MCF_IRQ_I2C1);
-	writel(r, MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
+	mcf_write32(r, MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
 #endif /* CONFIG_I2C_IMX */
 }
 
@@ -110,11 +110,11 @@ static void __init m5249_smc91x_init(void)
 	u32  gpio;
 
 	/* Set the GPIO line as interrupt source for smc91x device */
-	gpio = readl(MCFSIM2_GPIOINTENABLE);
-	writel(gpio | 0x40, MCFSIM2_GPIOINTENABLE);
+	gpio = mcf_read32(MCFSIM2_GPIOINTENABLE);
+	mcf_write32(gpio | 0x40, MCFSIM2_GPIOINTENABLE);
 
-	gpio = readl(MCFINTC2_INTPRI5);
-	writel(gpio | 0x04000000, MCFINTC2_INTPRI5);
+	gpio = mcf_read32(MCFINTC2_INTPRI5);
+	mcf_write32(gpio | 0x04000000, MCFINTC2_INTPRI5);
 }
 
 #endif /* CONFIG_M5249C3 */
diff --git a/arch/m68k/coldfire/m525x.c b/arch/m68k/coldfire/m525x.c
index 485375112e28a..8061ca7af99f1 100644
--- a/arch/m68k/coldfire/m525x.c
+++ b/arch/m68k/coldfire/m525x.c
@@ -44,12 +44,12 @@ static void __init m525x_qspi_init(void)
 #if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
 	/* set the GPIO function for the qspi cs gpios */
 	/* FIXME: replace with pinmux/pinctl support */
-	u32 f = readl(MCFSIM2_GPIOFUNC);
+	u32 f = mcf_read32(MCFSIM2_GPIOFUNC);
 	f |= (1 << MCFQSPI_CS2) | (1 << MCFQSPI_CS1) | (1 << MCFQSPI_CS0);
-	writel(f, MCFSIM2_GPIOFUNC);
+	mcf_write32(f, MCFSIM2_GPIOFUNC);
 
 	/* QSPI irq setup */
-	writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL4 | MCFSIM_ICR_PRI0,
+	mcf_write8(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL4 | MCFSIM_ICR_PRI0,
 	       MCFSIM_QSPIICR);
 	mcf_mapirq2imr(MCF_IRQ_QSPI, MCFINTC_QSPI);
 #endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */
@@ -61,15 +61,15 @@ static void __init m525x_i2c_init(void)
 	u32 r;
 
 	/* first I2C controller uses regular irq setup */
-	writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
+	mcf_write8(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
 	       MCFSIM_I2CICR);
 	mcf_mapirq2imr(MCF_IRQ_I2C0, MCFINTC_I2C);
 
 	/* second I2C controller is completely different */
-	r = readl(MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
+	r = mcf_read32(MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
 	r &= ~MCFINTC2_INTPRI_BITS(0xf, MCF_IRQ_I2C1);
 	r |= MCFINTC2_INTPRI_BITS(0x5, MCF_IRQ_I2C1);
-	writel(r, MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
+	mcf_write32(r, MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
 }
 
diff --git a/arch/m68k/coldfire/m5272.c b/arch/m68k/coldfire/m5272.c
index 28b3ffa25ba0b..bf38cf1a73577 100644
--- a/arch/m68k/coldfire/m5272.c
+++ b/arch/m68k/coldfire/m5272.c
@@ -55,13 +55,13 @@ static void __init m5272_uarts_init(void)
 	u32 v;
 
 	/* Enable the output lines for the serial ports */
-	v = readl(MCFSIM_PBCNT);
+	v = mcf_read32(MCFSIM_PBCNT);
 	v = (v & ~0x000000ff) | 0x00000055;
-	writel(v, MCFSIM_PBCNT);
+	mcf_write32(v, MCFSIM_PBCNT);
 
-	v = readl(MCFSIM_PDCNT);
+	v = mcf_read32(MCFSIM_PDCNT);
 	v = (v & ~0x000003fc) | 0x000002a8;
-	writel(v, MCFSIM_PDCNT);
+	mcf_write32(v, MCFSIM_PDCNT);
 }
 
 /***************************************************************************/
@@ -70,9 +70,9 @@ static void m5272_cpu_reset(void)
 {
 	local_irq_disable();
 	/* Set watchdog to reset, and enabled */
-	__raw_writew(0, MCFSIM_WIRR);
-	__raw_writew(1, MCFSIM_WRRR);
-	__raw_writew(0, MCFSIM_WCR);
+	mcf_write16(0, MCFSIM_WIRR);
+	mcf_write16(1, MCFSIM_WRRR);
+	mcf_write16(0, MCFSIM_WCR);
 	for (;;)
 		/* wait for watchdog to timeout */;
 }
@@ -83,7 +83,7 @@ void __init config_BSP(char *commandp, int size)
 {
 #if defined (CONFIG_MOD5272)
 	/* Set base of device vectors to be 64 */
-	writeb(0x40, MCFSIM_PIVR);
+	mcf_write8(0x40, MCFSIM_PIVR);
 #endif
 
 #if defined(CONFIG_NETtel) || defined(CONFIG_SCALES)
diff --git a/arch/m68k/coldfire/m527x.c b/arch/m68k/coldfire/m527x.c
index 037f3e520acc8..648f4d3dd3580 100644
--- a/arch/m68k/coldfire/m527x.c
+++ b/arch/m68k/coldfire/m527x.c
@@ -54,14 +54,14 @@ static void __init m527x_qspi_init(void)
 	u16 par;
 
 	/* setup QSPS pins for QSPI with gpio CS control */
-	writeb(0x1f, MCFGPIO_PAR_QSPI);
+	mcf_write8(0x1f, MCFGPIO_PAR_QSPI);
 	/* and CS2 & CS3 as gpio */
-	par = readw(MCFGPIO_PAR_TIMER);
+	par = mcf_read16(MCFGPIO_PAR_TIMER);
 	par &= 0x3f3f;
-	writew(par, MCFGPIO_PAR_TIMER);
+	mcf_write16(par, MCFGPIO_PAR_TIMER);
 #elif defined(CONFIG_M5275)
 	/* setup QSPS pins for QSPI with gpio CS control */
-	writew(0x003e, MCFGPIO_PAR_QSPI);
+	mcf_write16(0x003e, MCFGPIO_PAR_QSPI);
 #endif
 #endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */
 }
@@ -76,17 +76,17 @@ static void __init m527x_i2c_init(void)
 
 	/* setup Port FECI2C Pin Assignment Register for I2C */
 	/*  set PAR_SCL to SCL and PAR_SDA to SDA */
-	par = readb(MCFGPIO_PAR_FECI2C);
+	par = mcf_read8(MCFGPIO_PAR_FECI2C);
 	par |= 0x0f;
-	writeb(par, MCFGPIO_PAR_FECI2C);
+	mcf_write8(par, MCFGPIO_PAR_FECI2C);
 #elif defined(CONFIG_M5275)
 	u16 par;
 
 	/* setup Port FECI2C Pin Assignment Register for I2C */
 	/*  set PAR_SCL to SCL and PAR_SDA to SDA */
-	par = readw(MCFGPIO_PAR_FECI2C);
+	par = mcf_read16(MCFGPIO_PAR_FECI2C);
 	par |= 0x0f;
-	writew(par, MCFGPIO_PAR_FECI2C);
+	mcf_write16(par, MCFGPIO_PAR_FECI2C);
 #endif
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
 }
@@ -100,9 +100,9 @@ static void __init m527x_uarts_init(void)
 	/*
 	 * External Pin Mask Setting & Enable External Pin for Interface
 	 */
-	sepmask = readw(MCFGPIO_PAR_UART);
+	sepmask = mcf_read16(MCFGPIO_PAR_UART);
 	sepmask |= UART0_ENABLE_MASK | UART1_ENABLE_MASK | UART2_ENABLE_MASK;
-	writew(sepmask, MCFGPIO_PAR_UART);
+	mcf_write16(sepmask, MCFGPIO_PAR_UART);
 }
 
 /***************************************************************************/
@@ -113,21 +113,21 @@ static void __init m527x_fec_init(void)
 
 	/* Set multi-function pins to ethernet mode for fec0 */
 #if defined(CONFIG_M5271)
-	v = readb(MCFGPIO_PAR_FECI2C);
-	writeb(v | 0xf0, MCFGPIO_PAR_FECI2C);
+	v = mcf_read8(MCFGPIO_PAR_FECI2C);
+	mcf_write8(v | 0xf0, MCFGPIO_PAR_FECI2C);
 #else
 	u16 par;
 
-	par = readw(MCFGPIO_PAR_FECI2C);
-	writew(par | 0xf00, MCFGPIO_PAR_FECI2C);
-	v = readb(MCFGPIO_PAR_FEC0HL);
-	writeb(v | 0xc0, MCFGPIO_PAR_FEC0HL);
+	par = mcf_read16(MCFGPIO_PAR_FECI2C);
+	mcf_write16(par | 0xf00, MCFGPIO_PAR_FECI2C);
+	v = mcf_read8(MCFGPIO_PAR_FEC0HL);
+	mcf_write8(v | 0xc0, MCFGPIO_PAR_FEC0HL);
 
 	/* Set multi-function pins to ethernet mode for fec1 */
-	par = readw(MCFGPIO_PAR_FECI2C);
-	writew(par | 0xa0, MCFGPIO_PAR_FECI2C);
-	v = readb(MCFGPIO_PAR_FEC1HL);
-	writeb(v | 0xc0, MCFGPIO_PAR_FEC1HL);
+	par = mcf_read16(MCFGPIO_PAR_FECI2C);
+	mcf_write16(par | 0xa0, MCFGPIO_PAR_FECI2C);
+	v = mcf_read8(MCFGPIO_PAR_FEC1HL);
+	mcf_write8(v | 0xc0, MCFGPIO_PAR_FEC1HL);
 #endif
 }
 
diff --git a/arch/m68k/coldfire/m528x.c b/arch/m68k/coldfire/m528x.c
index 51a6a6236e126..b244c9ba40a70 100644
--- a/arch/m68k/coldfire/m528x.c
+++ b/arch/m68k/coldfire/m528x.c
@@ -51,7 +51,7 @@ static void __init m528x_qspi_init(void)
 {
 #if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
 	/* setup Port QS for QSPI with gpio CS control */
-	__raw_writeb(0x07, MCFGPIO_PQSPAR);
+	mcf_write8(0x07, MCFGPIO_PQSPAR);
 #endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */
 }
 
@@ -64,9 +64,9 @@ static void __init m528x_i2c_init(void)
 
 	/* setup Port AS Pin Assignment Register for I2C */
 	/*  set PASPA0 to SCL and PASPA1 to SDA */
-	paspar = readw(MCFGPIO_PASPAR);
+	paspar = mcf_read16(MCFGPIO_PASPAR);
 	paspar |= 0xF;
-	writew(paspar, MCFGPIO_PASPAR);
+	mcf_write16(paspar, MCFGPIO_PASPAR);
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
 }
 
@@ -77,9 +77,9 @@ static void __init m528x_uarts_init(void)
 	u8 port;
 
 	/* make sure PUAPAR is set for UART0 and UART1 */
-	port = readb(MCFGPIO_PUAPAR);
+	port = mcf_read8(MCFGPIO_PUAPAR);
 	port |= 0x03 | (0x03 << 2);
-	writeb(port, MCFGPIO_PUAPAR);
+	mcf_write8(port, MCFGPIO_PUAPAR);
 }
 
 /***************************************************************************/
@@ -89,9 +89,9 @@ static void __init m528x_fec_init(void)
 	u16 v16;
 
 	/* Set multi-function pins to ethernet mode for fec0 */
-	v16 = readw(MCFGPIO_PASPAR);
-	writew(v16 | 0xf00, MCFGPIO_PASPAR);
-	writeb(0xc0, MCFGPIO_PEHLPAR);
+	v16 = mcf_read16(MCFGPIO_PASPAR);
+	mcf_write16(v16 | 0xf00, MCFGPIO_PASPAR);
+	mcf_write8(0xc0, MCFGPIO_PEHLPAR);
 }
 
 /***************************************************************************/
@@ -99,8 +99,8 @@ static void __init m528x_fec_init(void)
 #ifdef CONFIG_WILDFIRE
 void wildfire_halt(void)
 {
-	writeb(0, 0x30000007);
-	writeb(0x2, 0x30000007);
+	mcf_write8(0, 0x30000007);
+	mcf_write8(0x2, 0x30000007);
 }
 #endif
 
@@ -110,14 +110,14 @@ void wildfiremod_halt(void)
 	printk(KERN_INFO "WildFireMod hibernating...\n");
 
 	/* Set portE.5 to Digital IO */
-	writew(readw(MCFGPIO_PEPAR) & ~(1 << (5 * 2)), MCFGPIO_PEPAR);
+	mcf_write16(read16(MCFGPIO_PEPAR) & ~(1 << (5 * 2)), MCFGPIO_PEPAR);
 
 	/* Make portE.5 an output */
-	writeb(readb(MCFGPIO_PDDR_E) | (1 << 5), MCFGPIO_PDDR_E);
+	mcf_write8(read8(MCFGPIO_PDDR_E) | (1 << 5), MCFGPIO_PDDR_E);
 
 	/* Now toggle portE.5 from low to high */
-	writeb(readb(MCFGPIO_PODR_E) & ~(1 << 5), MCFGPIO_PODR_E);
-	writeb(readb(MCFGPIO_PODR_E) | (1 << 5), MCFGPIO_PODR_E);
+	mcf_write8(read8(MCFGPIO_PODR_E) & ~(1 << 5), MCFGPIO_PODR_E);
+	mcf_write8(read8(MCFGPIO_PODR_E) | (1 << 5), MCFGPIO_PODR_E);
 
 	printk(KERN_EMERG "Failed to hibernate. Halting!\n");
 }
diff --git a/arch/m68k/coldfire/m5307.c b/arch/m68k/coldfire/m5307.c
index 4ed2e43ab3ad8..fc8f543b97918 100644
--- a/arch/m68k/coldfire/m5307.c
+++ b/arch/m68k/coldfire/m5307.c
@@ -49,7 +49,7 @@ static struct clk_lookup m5307_clk_lookup[] = {
 static void __init m5307_i2c_init(void)
 {
 #if IS_ENABLED(CONFIG_I2C_IMX)
-	writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
+	mcf_write8(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
 	       MCFSIM_I2CICR);
 	mcf_mapirq2imr(MCF_IRQ_I2C0, MCFINTC_I2C);
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
diff --git a/arch/m68k/coldfire/m53xx.c b/arch/m68k/coldfire/m53xx.c
index 17af5f6737969..88d4f8da055c0 100644
--- a/arch/m68k/coldfire/m53xx.c
+++ b/arch/m68k/coldfire/m53xx.c
@@ -166,7 +166,7 @@ static void __init m53xx_qspi_init(void)
 {
 #if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
 	/* setup QSPS pins for QSPI with gpio CS control */
-	writew(0x01f0, MCFGPIO_PAR_QSPI);
+	mcf_write16(0x01f0, MCFGPIO_PAR_QSPI);
 #endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */
 }
 
@@ -177,9 +177,9 @@ static void __init m53xx_i2c_init(void)
 #if IS_ENABLED(CONFIG_I2C_IMX)
 	/* setup Port AS Pin Assignment Register for I2C */
 	/*  set PASPA0 to SCL and PASPA1 to SDA */
-	u8 r = readb(MCFGPIO_PAR_FECI2C);
+	u8 r = mcf_read8(MCFGPIO_PAR_FECI2C);
 	r |= 0x0f;
-	writeb(r, MCFGPIO_PAR_FECI2C);
+	mcf_write8(r, MCFGPIO_PAR_FECI2C);
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
 }
 
@@ -188,7 +188,7 @@ static void __init m53xx_i2c_init(void)
 static void __init m53xx_uarts_init(void)
 {
 	/* UART GPIO initialization */
-	writew(readw(MCFGPIO_PAR_UART) | 0x0FFF, MCFGPIO_PAR_UART);
+	mcf_write16(mcf_read16(MCFGPIO_PAR_UART) | 0x0FFF, MCFGPIO_PAR_UART);
 }
 
 /***************************************************************************/
@@ -198,14 +198,14 @@ static void __init m53xx_fec_init(void)
 	u8 v;
 
 	/* Set multi-function pins to ethernet mode for fec0 */
-	v = readb(MCFGPIO_PAR_FECI2C);
+	v = mcf_read8(MCFGPIO_PAR_FECI2C);
 	v |= MCF_GPIO_PAR_FECI2C_PAR_MDC_EMDC |
 		MCF_GPIO_PAR_FECI2C_PAR_MDIO_EMDIO;
-	writeb(v, MCFGPIO_PAR_FECI2C);
+	mcf_write8(v, MCFGPIO_PAR_FECI2C);
 
-	v = readb(MCFGPIO_PAR_FEC);
+	v = mcf_read8(MCFGPIO_PAR_FEC);
 	v = MCF_GPIO_PAR_FEC_PAR_FEC_7W_FEC | MCF_GPIO_PAR_FEC_PAR_FEC_MII_FEC;
-	writeb(v, MCFGPIO_PAR_FEC);
+	mcf_write8(v, MCFGPIO_PAR_FEC);
 }
 
 /***************************************************************************/
@@ -305,7 +305,7 @@ asmlinkage void __init sysinit(void)
 void wtm_init(void)
 {
 	/* Disable watchdog timer */
-	writew(0, MCF_WTM_WCR);
+	mcf_write16(0, MCF_WTM_WCR);
 }
 
 #define MCF_SCM_BCR_GBW		(0x00000100)
@@ -314,53 +314,53 @@ void wtm_init(void)
 void scm_init(void)
 {
 	/* All masters are trusted */
-	writel(0x77777777, MCF_SCM_MPR);
+	mcf_write32(0x77777777, MCF_SCM_MPR);
     
 	/* Allow supervisor/user, read/write, and trusted/untrusted
 	   access to all slaves */
-	writel(0, MCF_SCM_PACRA);
-	writel(0, MCF_SCM_PACRB);
-	writel(0, MCF_SCM_PACRC);
-	writel(0, MCF_SCM_PACRD);
-	writel(0, MCF_SCM_PACRE);
-	writel(0, MCF_SCM_PACRF);
+	mcf_write32(0, MCF_SCM_PACRA);
+	mcf_write32(0, MCF_SCM_PACRB);
+	mcf_write32(0, MCF_SCM_PACRC);
+	mcf_write32(0, MCF_SCM_PACRD);
+	mcf_write32(0, MCF_SCM_PACRE);
+	mcf_write32(0, MCF_SCM_PACRF);
 
 	/* Enable bursts */
-	writel(MCF_SCM_BCR_GBR | MCF_SCM_BCR_GBW, MCF_SCM_BCR);
+	mcf_write32(MCF_SCM_BCR_GBR | MCF_SCM_BCR_GBW, MCF_SCM_BCR);
 }
 
 
 void fbcs_init(void)
 {
-	writeb(0x3E, MCFGPIO_PAR_CS);
+	mcf_write8(0x3E, MCFGPIO_PAR_CS);
 
 	/* Latch chip select */
-	writel(0x10080000, MCF_FBCS1_CSAR);
+	mcf_write32(0x10080000, MCF_FBCS1_CSAR);
 
-	writel(0x002A3780, MCF_FBCS1_CSCR);
-	writel(MCF_FBCS_CSMR_BAM_2M | MCF_FBCS_CSMR_V, MCF_FBCS1_CSMR);
+	mcf_write32(0x002A3780, MCF_FBCS1_CSCR);
+	mcf_write32(MCF_FBCS_CSMR_BAM_2M | MCF_FBCS_CSMR_V, MCF_FBCS1_CSMR);
 
 	/* Initialize latch to drive signals to inactive states */
-	writew(0xffff, 0x10080000);
+	mcf_write16(0xffff, 0x10080000);
 
 	/* External SRAM */
-	writel(EXT_SRAM_ADDRESS, MCF_FBCS1_CSAR);
-	writel(MCF_FBCS_CSCR_PS_16 |
+	mcf_write32(EXT_SRAM_ADDRESS, MCF_FBCS1_CSAR);
+	mcf_write32(MCF_FBCS_CSCR_PS_16 |
 		MCF_FBCS_CSCR_AA |
 		MCF_FBCS_CSCR_SBM |
 		MCF_FBCS_CSCR_WS(1),
 		MCF_FBCS1_CSCR);
-	writel(MCF_FBCS_CSMR_BAM_512K | MCF_FBCS_CSMR_V, MCF_FBCS1_CSMR);
+	mcf_write32(MCF_FBCS_CSMR_BAM_512K | MCF_FBCS_CSMR_V, MCF_FBCS1_CSMR);
 
 	/* Boot Flash connected to FBCS0 */
-	writel(FLASH_ADDRESS, MCF_FBCS0_CSAR);
-	writel(MCF_FBCS_CSCR_PS_16 |
+	mcf_write32(FLASH_ADDRESS, MCF_FBCS0_CSAR);
+	mcf_write32(MCF_FBCS_CSCR_PS_16 |
 		MCF_FBCS_CSCR_BEM |
 		MCF_FBCS_CSCR_AA |
 		MCF_FBCS_CSCR_SBM |
 		MCF_FBCS_CSCR_WS(7),
 		MCF_FBCS0_CSCR);
-	writel(MCF_FBCS_CSMR_BAM_32M | MCF_FBCS_CSMR_V, MCF_FBCS0_CSMR);
+	mcf_write32(MCF_FBCS_CSMR_BAM_32M | MCF_FBCS_CSMR_V, MCF_FBCS0_CSMR);
 }
 
 void sdramc_init(void)
@@ -369,18 +369,18 @@ void sdramc_init(void)
 	 * Check to see if the SDRAM has already been initialized
 	 * by a run control tool
 	 */
-	if (!(readl(MCF_SDRAMC_SDCR) & MCF_SDRAMC_SDCR_REF)) {
+	if (!(mcf_read32(MCF_SDRAMC_SDCR) & MCF_SDRAMC_SDCR_REF)) {
 		/* SDRAM chip select initialization */
 		
 		/* Initialize SDRAM chip select */
-		writel(MCF_SDRAMC_SDCS_BA(SDRAM_ADDRESS) |
+		mcf_write32(MCF_SDRAMC_SDCS_BA(SDRAM_ADDRESS) |
 			MCF_SDRAMC_SDCS_CSSZ(MCF_SDRAMC_SDCS_CSSZ_32MBYTE),
 			MCF_SDRAMC_SDCS0);
 
 	/*
 	 * Basic configuration and initialization
 	 */
-	writel(MCF_SDRAMC_SDCFG1_SRD2RW((int)((SDRAM_CASL + 2) + 0.5)) |
+	mcf_write32(MCF_SDRAMC_SDCFG1_SRD2RW((int)((SDRAM_CASL + 2) + 0.5)) |
 		MCF_SDRAMC_SDCFG1_SWT2RD(SDRAM_TWR + 1) |
 		MCF_SDRAMC_SDCFG1_RDLAT((int)((SDRAM_CASL * 2) + 2)) |
 		MCF_SDRAMC_SDCFG1_ACT2RW((int)(SDRAM_TRCD + 0.5)) |
@@ -388,7 +388,7 @@ void sdramc_init(void)
 		MCF_SDRAMC_SDCFG1_REF2ACT((int)(SDRAM_TRFC + 0.5)) |
 		MCF_SDRAMC_SDCFG1_WTLAT(3),
 		MCF_SDRAMC_SDCFG1);
-	writel(MCF_SDRAMC_SDCFG2_BRD2PRE(SDRAM_BL / 2 + 1) |
+	mcf_write32(MCF_SDRAMC_SDCFG2_BRD2PRE(SDRAM_BL / 2 + 1) |
 		MCF_SDRAMC_SDCFG2_BWT2RW(SDRAM_BL / 2 + SDRAM_TWR) |
 		MCF_SDRAMC_SDCFG2_BRD2WT((int)((SDRAM_CASL + SDRAM_BL / 2 - 1.0) + 0.5)) |
 		MCF_SDRAMC_SDCFG2_BL(SDRAM_BL - 1),
@@ -398,7 +398,7 @@ void sdramc_init(void)
 	/*
 	 * Precharge and enable write to SDMR
 	 */
-	writel(MCF_SDRAMC_SDCR_MODE_EN |
+	mcf_write32(MCF_SDRAMC_SDCR_MODE_EN |
 		MCF_SDRAMC_SDCR_CKE |
 		MCF_SDRAMC_SDCR_DDR |
 		MCF_SDRAMC_SDCR_MUX(1) |
@@ -410,7 +410,7 @@ void sdramc_init(void)
 	/*
 	 * Write extended mode register
 	 */
-	writel(MCF_SDRAMC_SDMR_BNKAD_LEMR |
+	mcf_write32(MCF_SDRAMC_SDMR_BNKAD_LEMR |
 		MCF_SDRAMC_SDMR_AD(0x0) |
 		MCF_SDRAMC_SDMR_CMD,
 		MCF_SDRAMC_SDMR);
@@ -418,7 +418,7 @@ void sdramc_init(void)
 	/*
 	 * Write mode register and reset DLL
 	 */
-	writel(MCF_SDRAMC_SDMR_BNKAD_LMR |
+	mcf_write32(MCF_SDRAMC_SDMR_BNKAD_LMR |
 		MCF_SDRAMC_SDMR_AD(0x163) |
 		MCF_SDRAMC_SDMR_CMD,
 		MCF_SDRAMC_SDMR);
@@ -426,18 +426,18 @@ void sdramc_init(void)
 	/*
 	 * Execute a PALL command
 	 */
-	writel(readl(MCF_SDRAMC_SDCR) | MCF_SDRAMC_SDCR_IPALL, MCF_SDRAMC_SDCR);
+	mcf_write32(mcf_read32(MCF_SDRAMC_SDCR) | MCF_SDRAMC_SDCR_IPALL, MCF_SDRAMC_SDCR);
 
 	/*
 	 * Perform two REF cycles
 	 */
-	writel(readl(MCF_SDRAMC_SDCR) | MCF_SDRAMC_SDCR_IREF, MCF_SDRAMC_SDCR);
-	writel(readl(MCF_SDRAMC_SDCR) | MCF_SDRAMC_SDCR_IREF, MCF_SDRAMC_SDCR);
+	mcf_write32(mcf_read32(MCF_SDRAMC_SDCR) | MCF_SDRAMC_SDCR_IREF, MCF_SDRAMC_SDCR);
+	mcf_write32(mcf_read32(MCF_SDRAMC_SDCR) | MCF_SDRAMC_SDCR_IREF, MCF_SDRAMC_SDCR);
 
 	/*
 	 * Write mode register and clear reset DLL
 	 */
-	writel(MCF_SDRAMC_SDMR_BNKAD_LMR |
+	mcf_write32(MCF_SDRAMC_SDMR_BNKAD_LMR |
 		MCF_SDRAMC_SDMR_AD(0x063) |
 		MCF_SDRAMC_SDMR_CMD,
 		MCF_SDRAMC_SDMR);
@@ -445,9 +445,9 @@ void sdramc_init(void)
 	/*
 	 * Enable auto refresh and lock SDMR
 	 */
-	writel(readl(MCF_SDRAMC_SDCR) & ~MCF_SDRAMC_SDCR_MODE_EN,
+	mcf_write32(mcf_read32(MCF_SDRAMC_SDCR) & ~MCF_SDRAMC_SDCR_MODE_EN,
 		MCF_SDRAMC_SDCR);
-	writel(MCF_SDRAMC_SDCR_REF | MCF_SDRAMC_SDCR_DQS_OE(0xC),
+	mcf_write32(MCF_SDRAMC_SDCR_REF | MCF_SDRAMC_SDCR_DQS_OE(0xC),
 		MCF_SDRAMC_SDCR);
 	}
 }
@@ -455,16 +455,16 @@ void sdramc_init(void)
 void gpio_init(void)
 {
 	/* Enable UART0 pins */
-	writew(MCF_GPIO_PAR_UART_PAR_URXD0 | MCF_GPIO_PAR_UART_PAR_UTXD0,
+	mcf_write16(MCF_GPIO_PAR_UART_PAR_URXD0 | MCF_GPIO_PAR_UART_PAR_UTXD0,
 		MCFGPIO_PAR_UART);
 
 	/*
 	 * Initialize TIN3 as a GPIO output to enable the write
 	 * half of the latch.
 	 */
-	writeb(0x00, MCFGPIO_PAR_TIMER);
-	writeb(0x08, MCFGPIO_PDDR_TIMER);
-	writeb(0x00, MCFGPIO_PCLRR_TIMER);
+	mcf_write8(0x00, MCFGPIO_PAR_TIMER);
+	mcf_write8(0x08, MCFGPIO_PDDR_TIMER);
+	mcf_write8(0x00, MCFGPIO_PCLRR_TIMER);
 }
 
 int clock_pll(int fsys, int flags)
@@ -476,7 +476,7 @@ int clock_pll(int fsys, int flags)
         
 	if (fsys == 0) {
 		/* Return current PLL output */
-		mfd = readb(MCF_PLL_PFDR);
+		mfd = mcf_read8(MCF_PLL_PFDR);
 
 		return (fref * mfd / (BUSDIV * 4));
 	}
@@ -502,9 +502,9 @@ int clock_pll(int fsys, int flags)
 	 * If it has then the SDRAM needs to be put into self refresh
 	 * mode before reprogramming the PLL.
 	 */
-	if (readl(MCF_SDRAMC_SDCR) & MCF_SDRAMC_SDCR_REF)
+	if (mcf_read32(MCF_SDRAMC_SDCR) & MCF_SDRAMC_SDCR_REF)
 		/* Put SDRAM into self refresh mode */
-		writel(readl(MCF_SDRAMC_SDCR) & ~MCF_SDRAMC_SDCR_CKE,
+		mcf_write32(mcf_read32(MCF_SDRAMC_SDCR) & ~MCF_SDRAMC_SDCR_CKE,
 			MCF_SDRAMC_SDCR);
 
 	/*
@@ -516,10 +516,10 @@ int clock_pll(int fsys, int flags)
 	clock_limp(DEFAULT_LPD);
      					
 	/* Reprogram PLL for desired fsys */
-	writeb(MCF_PLL_PODR_CPUDIV(BUSDIV/3) | MCF_PLL_PODR_BUSDIV(BUSDIV),
+	mcf_write8(MCF_PLL_PODR_CPUDIV(BUSDIV/3) | MCF_PLL_PODR_BUSDIV(BUSDIV),
 		MCF_PLL_PODR);
 						
-	writeb(mfd, MCF_PLL_PFDR);
+	mcf_write8(mfd, MCF_PLL_PFDR);
 		
 	/* Exit LIMP mode */
 	clock_exit_limp();
@@ -527,13 +527,13 @@ int clock_pll(int fsys, int flags)
 	/*
 	 * Return the SDRAM to normal operation if it is in use.
 	 */
-	if (readl(MCF_SDRAMC_SDCR) & MCF_SDRAMC_SDCR_REF)
+	if (mcf_read32(MCF_SDRAMC_SDCR) & MCF_SDRAMC_SDCR_REF)
 		/* Exit self refresh mode */
-		writel(readl(MCF_SDRAMC_SDCR) | MCF_SDRAMC_SDCR_CKE,
+		mcf_write32(mcf_read32(MCF_SDRAMC_SDCR) | MCF_SDRAMC_SDCR_CKE,
 			MCF_SDRAMC_SDCR);
 
 	/* Errata - workaround for SDRAM operation after exiting LIMP mode */
-	writel(MCF_SDRAMC_REFRESH, MCF_SDRAMC_LIMP_FIX);
+	mcf_write32(MCF_SDRAMC_REFRESH, MCF_SDRAMC_LIMP_FIX);
 
 	/* wait for DQS logic to relock */
 	for (i = 0; i < 0x200; i++)
@@ -554,12 +554,12 @@ int clock_limp(int div)
     
 	/* Save of the current value of the SSIDIV so we don't
 	   overwrite the value*/
-	temp = readw(MCF_CCM_CDR) & MCF_CCM_CDR_SSIDIV(0xF);
+	temp = mcf_read16(MCF_CCM_CDR) & MCF_CCM_CDR_SSIDIV(0xF);
       
 	/* Apply the divider to the system clock */
-	writew(MCF_CCM_CDR_LPDIV(div) | MCF_CCM_CDR_SSIDIV(temp), MCF_CCM_CDR);
+	mcf_write16(MCF_CCM_CDR_LPDIV(div) | MCF_CCM_CDR_SSIDIV(temp), MCF_CCM_CDR);
     
-	writew(readw(MCF_CCM_MISCCR) | MCF_CCM_MISCCR_LIMP, MCF_CCM_MISCCR);
+	mcf_write16(mcf_read16(MCF_CCM_MISCCR) | MCF_CCM_MISCCR_LIMP, MCF_CCM_MISCCR);
     
 	return (FREF/(3*(1 << div)));
 }
@@ -569,10 +569,10 @@ int clock_exit_limp(void)
 	int fout;
 	
 	/* Exit LIMP mode */
-	writew(readw(MCF_CCM_MISCCR) & ~MCF_CCM_MISCCR_LIMP, MCF_CCM_MISCCR);
+	mcf_write16(mcf_read16(MCF_CCM_MISCCR) & ~MCF_CCM_MISCCR_LIMP, MCF_CCM_MISCCR);
 
 	/* Wait for PLL to lock */
-	while (!(readw(MCF_CCM_MISCCR) & MCF_CCM_MISCCR_PLL_LOCK))
+	while (!(mcf_read16(MCF_CCM_MISCCR) & MCF_CCM_MISCCR_PLL_LOCK))
 		;
 	
 	fout = get_sys_clock();
@@ -585,10 +585,10 @@ int get_sys_clock(void)
 	int divider;
 	
 	/* Test to see if device is in LIMP mode */
-	if (readw(MCF_CCM_MISCCR) & MCF_CCM_MISCCR_LIMP) {
-		divider = readw(MCF_CCM_CDR) & MCF_CCM_CDR_LPDIV(0xF);
+	if (mcf_read16(MCF_CCM_MISCCR) & MCF_CCM_MISCCR_LIMP) {
+		divider = mcf_read16(MCF_CCM_CDR) & MCF_CCM_CDR_LPDIV(0xF);
 		return (FREF/(2 << divider));
 	}
 	else
-		return (FREF * readb(MCF_PLL_PFDR)) / (BUSDIV * 4);
+		return (FREF * mcf_read8(MCF_PLL_PFDR)) / (BUSDIV * 4);
 }
diff --git a/arch/m68k/coldfire/m5407.c b/arch/m68k/coldfire/m5407.c
index b32efb3042a23..23a14416241c8 100644
--- a/arch/m68k/coldfire/m5407.c
+++ b/arch/m68k/coldfire/m5407.c
@@ -40,7 +40,7 @@ static struct clk_lookup m5407_clk_lookup[] = {
 static void __init m5407_i2c_init(void)
 {
 #if IS_ENABLED(CONFIG_I2C_IMX)
-	writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
+	mcf_write8(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
 	       MCFSIM_I2CICR);
 	mcf_mapirq2imr(MCF_IRQ_I2C0, MCFINTC_I2C);
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
diff --git a/arch/m68k/coldfire/m5441x.c b/arch/m68k/coldfire/m5441x.c
index 7a25cfc7ac075..6ce730098ff60 100644
--- a/arch/m68k/coldfire/m5441x.c
+++ b/arch/m68k/coldfire/m5441x.c
@@ -201,12 +201,12 @@ static struct clk * const disable_clks[] __initconst = {
 
 static void __clk_enable2(struct clk *clk)
 {
-	__raw_writel(__raw_readl(MCFSDHC_CLK) | (1 << clk->slot), MCFSDHC_CLK);
+	mcf_write32(mcf_read32(MCFSDHC_CLK) | (1 << clk->slot), MCFSDHC_CLK);
 }
 
 static void __clk_disable2(struct clk *clk)
 {
-	__raw_writel(__raw_readl(MCFSDHC_CLK) & ~(1 << clk->slot), MCFSDHC_CLK);
+	mcf_write32(mcf_read32(MCFSDHC_CLK) & ~(1 << clk->slot), MCFSDHC_CLK);
 }
 
 struct clk_ops clk_ops2 = {
@@ -229,14 +229,14 @@ static void __init m5441x_clk_init(void)
 
 static void __init m5441x_uarts_init(void)
 {
-	__raw_writeb(0x0f, MCFGPIO_PAR_UART0);
-	__raw_writeb(0x00, MCFGPIO_PAR_UART1);
-	__raw_writeb(0x00, MCFGPIO_PAR_UART2);
+	mcf_write8(0x0f, MCFGPIO_PAR_UART0);
+	mcf_write8(0x00, MCFGPIO_PAR_UART1);
+	mcf_write8(0x00, MCFGPIO_PAR_UART2);
 }
 
 static void __init m5441x_fec_init(void)
 {
-	__raw_writeb(0x03, MCFGPIO_PAR_FEC);
+	mcf_write8(0x03, MCFGPIO_PAR_FEC);
 }
 
 void __init config_BSP(char *commandp, int size)
diff --git a/arch/m68k/coldfire/m54xx.c b/arch/m68k/coldfire/m54xx.c
index 8e3c8fee83271..90bb209b278d8 100644
--- a/arch/m68k/coldfire/m54xx.c
+++ b/arch/m68k/coldfire/m54xx.c
@@ -51,12 +51,12 @@ static struct clk_lookup m54xx_clk_lookup[] = {
 static void __init m54xx_uarts_init(void)
 {
 	/* enable io pins */
-	__raw_writeb(MCF_PAR_PSC_TXD | MCF_PAR_PSC_RXD, MCFGPIO_PAR_PSC0);
-	__raw_writeb(MCF_PAR_PSC_TXD | MCF_PAR_PSC_RXD | MCF_PAR_PSC_RTS_RTS,
+	mcf_write8(MCF_PAR_PSC_TXD | MCF_PAR_PSC_RXD, MCFGPIO_PAR_PSC0);
+	mcf_write8(MCF_PAR_PSC_TXD | MCF_PAR_PSC_RXD | MCF_PAR_PSC_RTS_RTS,
 		MCFGPIO_PAR_PSC1);
-	__raw_writeb(MCF_PAR_PSC_TXD | MCF_PAR_PSC_RXD | MCF_PAR_PSC_RTS_RTS |
+	mcf_write8(MCF_PAR_PSC_TXD | MCF_PAR_PSC_RXD | MCF_PAR_PSC_RTS_RTS |
 		MCF_PAR_PSC_CTS_CTS, MCFGPIO_PAR_PSC2);
-	__raw_writeb(MCF_PAR_PSC_TXD | MCF_PAR_PSC_RXD, MCFGPIO_PAR_PSC3);
+	mcf_write8(MCF_PAR_PSC_TXD | MCF_PAR_PSC_RXD, MCFGPIO_PAR_PSC3);
 }
 
 /***************************************************************************/
@@ -67,9 +67,9 @@ static void __init m54xx_i2c_init(void)
 	u32 r;
 
 	/* set the fec/i2c/irq pin assignment register for i2c */
-	r = readl(MCF_PAR_FECI2CIRQ);
+	r = mcf_read32(MCF_PAR_FECI2CIRQ);
 	r |= MCF_PAR_FECI2CIRQ_SDA | MCF_PAR_FECI2CIRQ_SCL;
-	writel(r, MCF_PAR_FECI2CIRQ);
+	mcf_write32(r, MCF_PAR_FECI2CIRQ);
 #endif /* IS_ENABLED(CONFIG_I2C_IMX) */
 }
 
@@ -79,9 +79,9 @@ static void mcf54xx_reset(void)
 {
 	/* disable interrupts and enable the watchdog */
 	asm("movew #0x2700, %sr\n");
-	__raw_writel(0, MCF_GPT_GMS0);
-	__raw_writel(MCF_GPT_GCIR_CNT(1), MCF_GPT_GCIR0);
-	__raw_writel(MCF_GPT_GMS_WDEN | MCF_GPT_GMS_CE | MCF_GPT_GMS_TMS(4),
+	mcf_write32(0, MCF_GPT_GMS0);
+	mcf_write32(MCF_GPT_GCIR_CNT(1), MCF_GPT_GCIR0);
+	mcf_write32(MCF_GPT_GMS_WDEN | MCF_GPT_GMS_CE | MCF_GPT_GMS_TMS(4),
 		MCF_GPT_GMS0);
 }
 
diff --git a/arch/m68k/coldfire/nettel.c b/arch/m68k/coldfire/nettel.c
index ea8df6e7a6cc6..e304571e03ea8 100644
--- a/arch/m68k/coldfire/nettel.c
+++ b/arch/m68k/coldfire/nettel.c
@@ -106,10 +106,10 @@ static void __init nettel_smc91x_setmac(unsigned int ioaddr, unsigned int flasha
 	if ((macp[0] == 0xffff) && (macp[1] == 0xffff) && (macp[2] == 0xffff))
 		macp = (u16 *) &nettel_macdefault[0];
 
-	writew(1, NETTEL_SMC0_ADDR + SMC91xx_BANKSELECT);
-	writew(macp[0], ioaddr + SMC91xx_BASEMAC);
-	writew(macp[1], ioaddr + SMC91xx_BASEMAC + 2);
-	writew(macp[2], ioaddr + SMC91xx_BASEMAC + 4);
+	mcf_write16(1, NETTEL_SMC0_ADDR + SMC91xx_BANKSELECT);
+	mcf_write16(macp[0], ioaddr + SMC91xx_BASEMAC);
+	mcf_write16(macp[1], ioaddr + SMC91xx_BASEMAC + 2);
+	mcf_write16(macp[2], ioaddr + SMC91xx_BASEMAC + 4);
 }
 
 /***************************************************************************/
@@ -122,14 +122,14 @@ static void __init nettel_smc91x_setmac(unsigned int ioaddr, unsigned int flasha
 
 static void __init nettel_smc91x_init(void)
 {
-	writew(0x00ec, MCFSIM_PADDR);
+	mcf_write16(0x00ec, MCFSIM_PADDR);
 	mcf_setppdata(0, 0x0080);
-	writew(1, NETTEL_SMC0_ADDR + SMC91xx_BANKSELECT);
-	writew(0x0067, NETTEL_SMC0_ADDR + SMC91xx_BASEADDR);
+	mcf_write16(1, NETTEL_SMC0_ADDR + SMC91xx_BANKSELECT);
+	mcf_write16(0x0067, NETTEL_SMC0_ADDR + SMC91xx_BASEADDR);
 	mcf_setppdata(0x0080, 0);
 
 	/* Set correct chip select timing for SMC9196 accesses */
-	writew(0x1180, MCFSIM_CSCR3);
+	mcf_write16(0x1180, MCFSIM_CSCR3);
 
 	/* Set the SMC interrupts to be auto-vectored */
 	mcf_autovector(NETTEL_SMC0_IRQ);
diff --git a/arch/m68k/coldfire/pci.c b/arch/m68k/coldfire/pci.c
index ceb5775b8d239..5afb207ccc510 100644
--- a/arch/m68k/coldfire/pci.c
+++ b/arch/m68k/coldfire/pci.c
@@ -68,24 +68,24 @@ static int mcf_pci_readconfig(struct pci_bus *bus, unsigned int devfn,
 	}
 
 	addr = mcf_mk_pcicar(bus->number, devfn, where);
-	__raw_writel(PCICAR_E | addr, PCICAR);
-	__raw_readl(PCICAR);
+	mcf_write32(PCICAR_E | addr, PCICAR);
+	mcf_read32(PCICAR);
 	addr = iospace + (where & 0x3);
 
 	switch (size) {
 	case 1:
-		*value = __raw_readb(addr);
+		*value = mcf_read8(addr);
 		break;
 	case 2:
-		*value = le16_to_cpu(__raw_readw(addr));
+		*value = le16_to_cpu(mcf_read16(addr));
 		break;
 	default:
-		*value = le32_to_cpu(__raw_readl(addr));
+		*value = le32_to_cpu(mcf_read32(addr));
 		break;
 	}
 
-	__raw_writel(0, PCICAR);
-	__raw_readl(PCICAR);
+	mcf_write32(0, PCICAR);
+	mcf_read32(PCICAR);
 	return PCIBIOS_SUCCESSFUL;
 }
 
@@ -100,24 +100,24 @@ static int mcf_pci_writeconfig(struct pci_bus *bus, unsigned int devfn,
 	}
 
 	addr = mcf_mk_pcicar(bus->number, devfn, where);
-	__raw_writel(PCICAR_E | addr, PCICAR);
-	__raw_readl(PCICAR);
+	mcf_write32(PCICAR_E | addr, PCICAR);
+	mcf_read32(PCICAR);
 	addr = iospace + (where & 0x3);
 
 	switch (size) {
 	case 1:
-		 __raw_writeb(value, addr);
+		 mcf_write8(value, addr);
 		break;
 	case 2:
-		__raw_writew(cpu_to_le16(value), addr);
+		mcf_write16(cpu_to_le16(value), addr);
 		break;
 	default:
-		__raw_writel(cpu_to_le32(value), addr);
+		mcf_write32(cpu_to_le32(value), addr);
 		break;
 	}
 
-	__raw_writel(0, PCICAR);
-	__raw_readl(PCICAR);
+	mcf_write32(0, PCICAR);
+	mcf_read32(PCICAR);
 	return PCIBIOS_SUCCESSFUL;
 }
 
@@ -175,44 +175,44 @@ static int __init mcf_pci_init(void)
 	pr_info("ColdFire: PCI bus initialization...\n");
 
 	/* Reset the external PCI bus */
-	__raw_writel(PCIGSCR_RESET, PCIGSCR);
-	__raw_writel(0, PCITCR);
+	mcf_write32(PCIGSCR_RESET, PCIGSCR);
+	mcf_write32(0, PCITCR);
 
 	request_resource(&iomem_resource, &mcf_pci_mem);
 	request_resource(&iomem_resource, &mcf_pci_io);
 
 	/* Configure PCI arbiter */
-	__raw_writel(PACR_INTMPRI | PACR_INTMINTE | PACR_EXTMPRI(0x1f) |
+	mcf_write32(PACR_INTMPRI | PACR_INTMINTE | PACR_EXTMPRI(0x1f) |
 		PACR_EXTMINTE(0x1f), PACR);
 
 	/* Set required multi-function pins for PCI bus use */
-	__raw_writew(0x3ff, MCFGPIO_PAR_PCIBG);
-	__raw_writew(0x3ff, MCFGPIO_PAR_PCIBR);
+	mcf_write16(0x3ff, MCFGPIO_PAR_PCIBG);
+	mcf_write16(0x3ff, MCFGPIO_PAR_PCIBR);
 
 	/* Set up config space for local host bus controller */
-	__raw_writel(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+	mcf_write32(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
 		PCI_COMMAND_INVALIDATE, PCISCR);
-	__raw_writel(PCICR1_LT(32) | PCICR1_CL(8), PCICR1);
-	__raw_writel(0, PCICR2);
+	mcf_write32(PCICR1_LT(32) | PCICR1_CL(8), PCICR1);
+	mcf_write32(0, PCICR2);
 
 	/*
 	 * Set up the initiator windows for memory and IO mapping.
 	 * These give the CPU bus access onto the PCI bus. One for each of
 	 * PCI memory and IO address spaces.
 	 */
-	__raw_writel(WXBTAR(PCI_MEM_PA, PCI_MEM_BA, PCI_MEM_SIZE),
+	mcf_write32(WXBTAR(PCI_MEM_PA, PCI_MEM_BA, PCI_MEM_SIZE),
 		PCIIW0BTAR);
-	__raw_writel(WXBTAR(PCI_IO_PA, PCI_IO_BA, PCI_IO_SIZE),
+	mcf_write32(WXBTAR(PCI_IO_PA, PCI_IO_BA, PCI_IO_SIZE),
 		PCIIW1BTAR);
-	__raw_writel(PCIIWCR_W0_MEM /*| PCIIWCR_W0_MRDL*/ | PCIIWCR_W0_E |
+	mcf_write32(PCIIWCR_W0_MEM /*| PCIIWCR_W0_MRDL*/ | PCIIWCR_W0_E |
 		PCIIWCR_W1_IO | PCIIWCR_W1_E, PCIIWCR);
 
 	/*
 	 * Set up the target windows for access from the PCI bus back to the
 	 * CPU bus. All we need is access to system RAM (for mastering).
 	 */
-	__raw_writel(CONFIG_RAMBASE, PCIBAR1);
-	__raw_writel(CONFIG_RAMBASE | PCITBATR1_E, PCITBATR1);
+	mcf_write32(CONFIG_RAMBASE, PCIBAR1);
+	mcf_write32(CONFIG_RAMBASE | PCITBATR1_E, PCITBATR1);
 
 	/* Keep a virtual mapping to IO/config space active */
 	iospace = (unsigned long) ioremap(PCI_IO_PA, PCI_IO_SIZE);
@@ -224,7 +224,7 @@ static int __init mcf_pci_init(void)
 		(u32) iospace);
 
 	/* Turn of PCI reset, and wait for devices to settle */
-	__raw_writel(0, PCIGSCR);
+	mcf_write32(0, PCIGSCR);
 	set_current_state(TASK_UNINTERRUPTIBLE);
 	schedule_timeout(msecs_to_jiffies(200));
 
diff --git a/arch/m68k/coldfire/pit.c b/arch/m68k/coldfire/pit.c
index 855d0af470973..de4eaf8db22cd 100644
--- a/arch/m68k/coldfire/pit.c
+++ b/arch/m68k/coldfire/pit.c
@@ -45,9 +45,9 @@ static u32 pit_cnt;
 
 static int cf_pit_set_periodic(struct clock_event_device *evt)
 {
-	__raw_writew(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR));
-	__raw_writew(PIT_CYCLES_PER_JIFFY, TA(MCFPIT_PMR));
-	__raw_writew(MCFPIT_PCSR_EN | MCFPIT_PCSR_PIE |
+	mcf_write16(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR));
+	mcf_write16(PIT_CYCLES_PER_JIFFY, TA(MCFPIT_PMR));
+	mcf_write16(MCFPIT_PCSR_EN | MCFPIT_PCSR_PIE |
 		     MCFPIT_PCSR_OVW | MCFPIT_PCSR_RLD |
 		     MCFPIT_PCSR_CLK64, TA(MCFPIT_PCSR));
 	return 0;
@@ -55,15 +55,15 @@ static int cf_pit_set_periodic(struct clock_event_device *evt)
 
 static int cf_pit_set_oneshot(struct clock_event_device *evt)
 {
-	__raw_writew(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR));
-	__raw_writew(MCFPIT_PCSR_EN | MCFPIT_PCSR_PIE |
+	mcf_write16(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR));
+	mcf_write16(MCFPIT_PCSR_EN | MCFPIT_PCSR_PIE |
 		     MCFPIT_PCSR_OVW | MCFPIT_PCSR_CLK64, TA(MCFPIT_PCSR));
 	return 0;
 }
 
 static int cf_pit_shutdown(struct clock_event_device *evt)
 {
-	__raw_writew(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR));
+	mcf_write16(MCFPIT_PCSR_DISABLE, TA(MCFPIT_PCSR));
 	return 0;
 }
 
@@ -75,7 +75,7 @@ static int cf_pit_shutdown(struct clock_event_device *evt)
 static int cf_pit_next_event(unsigned long delta,
 		struct clock_event_device *evt)
 {
-	__raw_writew(delta, TA(MCFPIT_PMR));
+	mcf_write16(delta, TA(MCFPIT_PMR));
 	return 0;
 }
 
@@ -101,8 +101,8 @@ static irqreturn_t pit_tick(int irq, void *dummy)
 	u16 pcsr;
 
 	/* Reset the ColdFire timer */
-	pcsr = __raw_readw(TA(MCFPIT_PCSR));
-	__raw_writew(pcsr | MCFPIT_PCSR_PIF, TA(MCFPIT_PCSR));
+	pcsr = mcf_read16(TA(MCFPIT_PCSR));
+	mcf_write16(pcsr | MCFPIT_PCSR_PIF, TA(MCFPIT_PCSR));
 
 	pit_cnt += PIT_CYCLES_PER_JIFFY;
 	evt->event_handler(evt);
@@ -118,7 +118,7 @@ static u64 pit_read_clk(struct clocksource *cs)
 	u16 pcntr;
 
 	local_irq_save(flags);
-	pcntr = __raw_readw(TA(MCFPIT_PCNTR));
+	pcntr = mcf_read16(TA(MCFPIT_PCNTR));
 	cycles = pit_cnt;
 	local_irq_restore(flags);
 
diff --git a/arch/m68k/coldfire/reset.c b/arch/m68k/coldfire/reset.c
index f30952f0cbe6f..6e5f8ab39f324 100644
--- a/arch/m68k/coldfire/reset.c
+++ b/arch/m68k/coldfire/reset.c
@@ -27,7 +27,7 @@ static void mcf_cpu_reset(void)
 {
 	local_irq_disable();
 	/* Set watchdog to soft reset, and enabled */
-	__raw_writeb(0xc0, MCFSIM_SYPCR);
+	mcf_write8(0xc0, MCFSIM_SYPCR);
 	for (;;)
 		/* wait for watchdog to timeout */;
 }
@@ -37,7 +37,7 @@ static void mcf_cpu_reset(void)
 static void mcf_cpu_reset(void)
 {
 	local_irq_disable();
-	__raw_writeb(MCF_RCR_SWRESET, MCF_RCR);
+	mcf_write8(MCF_RCR_SWRESET, MCF_RCR);
 }
 #endif
 
diff --git a/arch/m68k/coldfire/sltimers.c b/arch/m68k/coldfire/sltimers.c
index f9d572ee63dbc..e683f29df8063 100644
--- a/arch/m68k/coldfire/sltimers.c
+++ b/arch/m68k/coldfire/sltimers.c
@@ -44,7 +44,7 @@
 irqreturn_t mcfslt_profile_tick(int irq, void *dummy)
 {
 	/* Reset Slice Timer 1 */
-	__raw_writel(MCFSLT_SSR_BE | MCFSLT_SSR_TE, PA(MCFSLT_SSR));
+	mcf_write32(MCFSLT_SSR_BE | MCFSLT_SSR_TE, PA(MCFSLT_SSR));
 	if (current->pid)
 		profile_tick(CPU_PROFILING);
 	return IRQ_HANDLED;
@@ -65,8 +65,8 @@ void mcfslt_profile_init(void)
 	}
 
 	/* Set up TIMER 2 as high speed profile clock */
-	__raw_writel(MCF_BUSCLK / PROFILEHZ - 1, PA(MCFSLT_STCNT));
-	__raw_writel(MCFSLT_SCR_RUN | MCFSLT_SCR_IEN | MCFSLT_SCR_TEN,
+	mcf_write32(MCF_BUSCLK / PROFILEHZ - 1, PA(MCFSLT_STCNT));
+	mcf_write32(MCFSLT_SCR_RUN | MCFSLT_SCR_IEN | MCFSLT_SCR_TEN,
 								PA(MCFSLT_SCR));
 
 }
@@ -86,7 +86,7 @@ static u32 mcfslt_cnt;
 static irqreturn_t mcfslt_tick(int irq, void *dummy)
 {
 	/* Reset Slice Timer 0 */
-	__raw_writel(MCFSLT_SSR_BE | MCFSLT_SSR_TE, TA(MCFSLT_SSR));
+	mcf_write32(MCFSLT_SSR_BE | MCFSLT_SSR_TE, TA(MCFSLT_SSR));
 	mcfslt_cnt += mcfslt_cycles_per_jiffy;
 	legacy_timer_tick(1);
 	return IRQ_HANDLED;
@@ -98,11 +98,11 @@ static u64 mcfslt_read_clk(struct clocksource *cs)
 	u32 cycles, scnt;
 
 	local_irq_save(flags);
-	scnt = __raw_readl(TA(MCFSLT_SCNT));
+	scnt = mcf_read32(TA(MCFSLT_SCNT));
 	cycles = mcfslt_cnt;
-	if (__raw_readl(TA(MCFSLT_SSR)) & MCFSLT_SSR_TE) {
+	if (mcf_read32(TA(MCFSLT_SSR)) & MCFSLT_SSR_TE) {
 		cycles += mcfslt_cycles_per_jiffy;
-		scnt = __raw_readl(TA(MCFSLT_SCNT));
+		scnt = mcf_read32(TA(MCFSLT_SCNT));
 	}
 	local_irq_restore(flags);
 
@@ -129,8 +129,8 @@ void hw_timer_init(void)
 	 *	STCNT + 1 steps for 1 tick, not STCNT.  So if you want
 	 *	n cycles, initialize STCNT with n - 1.
 	 */
-	__raw_writel(mcfslt_cycles_per_jiffy - 1, TA(MCFSLT_STCNT));
-	__raw_writel(MCFSLT_SCR_RUN | MCFSLT_SCR_IEN | MCFSLT_SCR_TEN,
+	mcf_write32(mcfslt_cycles_per_jiffy - 1, TA(MCFSLT_STCNT));
+	mcf_write32(MCFSLT_SCR_RUN | MCFSLT_SCR_IEN | MCFSLT_SCR_TEN,
 								TA(MCFSLT_SCR));
 	/* initialize mcfslt_cnt knowing that slice timers count down */
 	mcfslt_cnt = mcfslt_cycles_per_jiffy;
diff --git a/arch/m68k/coldfire/stmark2.c b/arch/m68k/coldfire/stmark2.c
index 036a6ae5f5992..9263b77bd09a9 100644
--- a/arch/m68k/coldfire/stmark2.c
+++ b/arch/m68k/coldfire/stmark2.c
@@ -1,7 +1,7 @@
 /*
- * stmark2.c -- Support for Sysam AMCORE open board
+ * stmark2.c -- Support for Kernelspace AMCORE open board
  *
- * (C) Copyright 2017, Angelo Dureghello <angelo@sysam.it>
+ * (C) Copyright 2026, Angelo Dureghello <angelo@kernel-space.org>
  *
  * This file is subject to the terms and conditions of the GNU General Public
  * License.  See the file COPYING in the main directory of this archive
@@ -104,16 +104,16 @@ static struct platform_device *stmark2_devices[] __initdata = {
 static int __init init_stmark2(void)
 {
 	/* DSPI0, all pins as DSPI, and using CS1 */
-	__raw_writeb(0x80, MCFGPIO_PAR_DSPIOWL);
-	__raw_writeb(0xfc, MCFGPIO_PAR_DSPIOWH);
+	mcf_write8(0x80, MCFGPIO_PAR_DSPIOWL);
+	mcf_write8(0xfc, MCFGPIO_PAR_DSPIOWH);
 
 	/* Board gpio setup */
-	__raw_writeb(0x00, MCFGPIO_PAR_BE);
-	__raw_writeb(0x00, MCFGPIO_PAR_FBCTL);
-	__raw_writeb(0x00, MCFGPIO_PAR_CS);
+	mcf_write8(0x00, MCFGPIO_PAR_BE);
+	mcf_write8(0x00, MCFGPIO_PAR_FBCTL);
+	mcf_write8(0x00, MCFGPIO_PAR_CS);
 
 	/* CAN pads */
-	__raw_writeb(0x50, MCFGPIO_PAR_CANI2C);
+	mcf_write8(0x50, MCFGPIO_PAR_CANI2C);
 
 	platform_add_devices(stmark2_devices, ARRAY_SIZE(stmark2_devices));
 
diff --git a/arch/m68k/coldfire/timers.c b/arch/m68k/coldfire/timers.c
index 05a42d8e0a59d..9db8e7e6d797b 100644
--- a/arch/m68k/coldfire/timers.c
+++ b/arch/m68k/coldfire/timers.c
@@ -38,11 +38,11 @@
 void coldfire_profile_init(void);
 
 #if defined(CONFIG_M53xx) || defined(CONFIG_M5441x)
-#define	__raw_readtrr	__raw_readl
-#define	__raw_writetrr	__raw_writel
+#define	mcf_readtrr	mcf_read32
+#define	mcf_writetrr	mcf_write32
 #else
-#define	__raw_readtrr	__raw_readw
-#define	__raw_writetrr	__raw_writew
+#define	mcf_readtrr	mcf_read16
+#define	mcf_writetrr	mcf_write16
 #endif
 
 static u32 mcftmr_cycles_per_jiffy;
@@ -54,13 +54,13 @@ static void init_timer_irq(void)
 {
 #ifdef MCFSIM_ICR_AUTOVEC
 	/* Timer1 is always used as system timer */
-	writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL6 | MCFSIM_ICR_PRI3,
+	mcf_write8(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL6 | MCFSIM_ICR_PRI3,
 		MCFSIM_TIMER1ICR);
 	mcf_mapirq2imr(MCF_IRQ_TIMER, MCFINTC_TIMER1);
 
 #ifdef CONFIG_HIGHPROFILE
 	/* Timer2 is to be used as a high speed profile timer  */
-	writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL7 | MCFSIM_ICR_PRI3,
+	mcf_write8(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL7 | MCFSIM_ICR_PRI3,
 		MCFSIM_TIMER2ICR);
 	mcf_mapirq2imr(MCF_IRQ_PROFILER, MCFINTC_TIMER2);
 #endif
@@ -72,7 +72,7 @@ static void init_timer_irq(void)
 static irqreturn_t mcftmr_tick(int irq, void *dummy)
 {
 	/* Reset the ColdFire timer */
-	__raw_writeb(MCFTIMER_TER_CAP | MCFTIMER_TER_REF, TA(MCFTIMER_TER));
+	mcf_write8(MCFTIMER_TER_CAP | MCFTIMER_TER_REF, TA(MCFTIMER_TER));
 
 	mcftmr_cnt += mcftmr_cycles_per_jiffy;
 	legacy_timer_tick(1);
@@ -88,7 +88,7 @@ static u64 mcftmr_read_clk(struct clocksource *cs)
 	u16 tcn;
 
 	local_irq_save(flags);
-	tcn = __raw_readw(TA(MCFTIMER_TCN));
+	tcn = mcf_read16(TA(MCFTIMER_TCN));
 	cycles = mcftmr_cnt;
 	local_irq_restore(flags);
 
@@ -111,7 +111,7 @@ void hw_timer_init(void)
 {
 	int r;
 
-	__raw_writew(MCFTIMER_TMR_DISABLE, TA(MCFTIMER_TMR));
+	mcf_write16(MCFTIMER_TMR_DISABLE, TA(MCFTIMER_TMR));
 	mcftmr_cycles_per_jiffy = FREQ / HZ;
 	/*
 	 *	The coldfire timer runs from 0 to TRR included, then 0
@@ -119,8 +119,8 @@ void hw_timer_init(void)
 	 *	for 1 tick, not TRR.  So if you want n cycles,
 	 *	initialize TRR with n - 1.
 	 */
-	__raw_writetrr(mcftmr_cycles_per_jiffy - 1, TA(MCFTIMER_TRR));
-	__raw_writew(MCFTIMER_TMR_ENORI | MCFTIMER_TMR_CLK16 |
+	mcf_writetrr(mcftmr_cycles_per_jiffy - 1, TA(MCFTIMER_TRR));
+	mcf_write16(MCFTIMER_TMR_ENORI | MCFTIMER_TMR_CLK16 |
 		MCFTIMER_TMR_RESTART | MCFTIMER_TMR_ENABLE, TA(MCFTIMER_TMR));
 
 	clocksource_register_hz(&mcftmr_clk, FREQ);
@@ -158,7 +158,7 @@ void hw_timer_init(void)
 irqreturn_t coldfire_profile_tick(int irq, void *dummy)
 {
 	/* Reset ColdFire timer2 */
-	__raw_writeb(MCFTIMER_TER_CAP | MCFTIMER_TER_REF, PA(MCFTIMER_TER));
+	mcf_write8(MCFTIMER_TER_CAP | MCFTIMER_TER_REF, PA(MCFTIMER_TER));
 	if (current->pid)
 		profile_tick(CPU_PROFILING);
 	return IRQ_HANDLED;
@@ -174,10 +174,10 @@ void coldfire_profile_init(void)
 	       PROFILEHZ);
 
 	/* Set up TIMER 2 as high speed profile clock */
-	__raw_writew(MCFTIMER_TMR_DISABLE, PA(MCFTIMER_TMR));
+	mcf_write16(MCFTIMER_TMR_DISABLE, PA(MCFTIMER_TMR));
 
-	__raw_writetrr(((MCF_BUSCLK / 16) / PROFILEHZ), PA(MCFTIMER_TRR));
-	__raw_writew(MCFTIMER_TMR_ENORI | MCFTIMER_TMR_CLK16 |
+	mcf_writetrr(((MCF_BUSCLK / 16) / PROFILEHZ), PA(MCFTIMER_TRR));
+	mcf_write16(MCFTIMER_TMR_ENORI | MCFTIMER_TMR_CLK16 |
 		MCFTIMER_TMR_RESTART | MCFTIMER_TMR_ENABLE, PA(MCFTIMER_TMR));
 
 	ret = request_irq(MCF_IRQ_PROFILER, coldfire_profile_tick, IRQF_TIMER,
diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig
index 47e48c18e55c7..a886494c5a30b 100644
--- a/arch/m68k/configs/amiga_defconfig
+++ b/arch/m68k/configs/amiga_defconfig
@@ -550,8 +550,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig
index 161586d611ab0..14c6298b1175e 100644
--- a/arch/m68k/configs/apollo_defconfig
+++ b/arch/m68k/configs/apollo_defconfig
@@ -505,8 +505,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig
index c13c6deeac224..86a5c69b3655b 100644
--- a/arch/m68k/configs/atari_defconfig
+++ b/arch/m68k/configs/atari_defconfig
@@ -527,8 +527,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig
index d4f3f94b61ff3..b3828dc6ee250 100644
--- a/arch/m68k/configs/bvme6000_defconfig
+++ b/arch/m68k/configs/bvme6000_defconfig
@@ -497,8 +497,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig
index 58288f83349d2..6b890c6bcf35c 100644
--- a/arch/m68k/configs/hp300_defconfig
+++ b/arch/m68k/configs/hp300_defconfig
@@ -507,8 +507,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/m5208evb_defconfig b/arch/m68k/configs/m5208evb_defconfig
index 31035a0b92473..cc17d3cdcab15 100644
--- a/arch/m68k/configs/m5208evb_defconfig
+++ b/arch/m68k/configs/m5208evb_defconfig
@@ -1,22 +1,23 @@
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_EXPERT=y
-# CONFIG_KALLSYMS is not set
 # CONFIG_FUTEX is not set
 # CONFIG_EPOLL is not set
 # CONFIG_SIGNALFD is not set
 # CONFIG_TIMERFD is not set
 # CONFIG_EVENTFD is not set
 # CONFIG_AIO is not set
-# CONFIG_VM_EVENT_COUNTERS is not set
-# CONFIG_COMPAT_BRK is not set
-# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_KALLSYMS is not set
 # CONFIG_MMU is not set
 # CONFIG_4KSTACKS is not set
 CONFIG_RAMBASE=0x40000000
 CONFIG_RAMSIZE=0x2000000
 CONFIG_VECTORBASE=0x40000000
 CONFIG_KERNELBASE=0x40020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
 CONFIG_BINFMT_FLAT=y
+# CONFIG_COMPAT_BRK is not set
+# CONFIG_VM_EVENT_COUNTERS is not set
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
diff --git a/arch/m68k/configs/m5249evb_defconfig b/arch/m68k/configs/m5249evb_defconfig
index 5706d7a1daba3..65d006c4d29f9 100644
--- a/arch/m68k/configs/m5249evb_defconfig
+++ b/arch/m68k/configs/m5249evb_defconfig
@@ -1,15 +1,12 @@
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_EXPERT=y
-# CONFIG_KALLSYMS is not set
 # CONFIG_FUTEX is not set
 # CONFIG_EPOLL is not set
 # CONFIG_SIGNALFD is not set
 # CONFIG_TIMERFD is not set
 # CONFIG_EVENTFD is not set
 # CONFIG_AIO is not set
-# CONFIG_VM_EVENT_COUNTERS is not set
-# CONFIG_SLUB_DEBUG is not set
-# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_KALLSYMS is not set
 # CONFIG_MMU is not set
 CONFIG_M5249=y
 CONFIG_M5249C3=y
@@ -17,7 +14,10 @@ CONFIG_RAMBASE=0x00000000
 CONFIG_RAMSIZE=0x00800000
 CONFIG_VECTORBASE=0x00000000
 CONFIG_KERNELBASE=0x00020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
 CONFIG_BINFMT_FLAT=y
+# CONFIG_VM_EVENT_COUNTERS is not set
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -45,5 +45,6 @@ CONFIG_EXT2_FS=y
 CONFIG_ROMFS_FS=y
 CONFIG_ROMFS_BACKED_BY_MTD=y
 # CONFIG_NETWORK_FILESYSTEMS is not set
+# CONFIG_SLUB_DEBUG is not set
 CONFIG_BOOTPARAM=y
 CONFIG_BOOTPARAM_STRING="root=/dev/mtdblock0"
diff --git a/arch/m68k/configs/m5272c3_defconfig b/arch/m68k/configs/m5272c3_defconfig
index f02fe144f4adc..40572e719b275 100644
--- a/arch/m68k/configs/m5272c3_defconfig
+++ b/arch/m68k/configs/m5272c3_defconfig
@@ -1,15 +1,12 @@
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_EXPERT=y
-# CONFIG_KALLSYMS is not set
 # CONFIG_FUTEX is not set
 # CONFIG_EPOLL is not set
 # CONFIG_SIGNALFD is not set
 # CONFIG_TIMERFD is not set
 # CONFIG_EVENTFD is not set
 # CONFIG_AIO is not set
-# CONFIG_VM_EVENT_COUNTERS is not set
-# CONFIG_SLUB_DEBUG is not set
-# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_KALLSYMS is not set
 # CONFIG_MMU is not set
 CONFIG_M5272=y
 CONFIG_M5272C3=y
@@ -17,7 +14,10 @@ CONFIG_RAMBASE=0x00000000
 CONFIG_RAMSIZE=0x00800000
 CONFIG_VECTORBASE=0x00000000
 CONFIG_KERNELBASE=0x00020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
 CONFIG_BINFMT_FLAT=y
+# CONFIG_VM_EVENT_COUNTERS is not set
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -46,5 +46,6 @@ CONFIG_EXT2_FS=y
 CONFIG_ROMFS_FS=y
 CONFIG_ROMFS_BACKED_BY_MTD=y
 # CONFIG_NETWORK_FILESYSTEMS is not set
+# CONFIG_SLUB_DEBUG is not set
 CONFIG_BOOTPARAM=y
 CONFIG_BOOTPARAM_STRING="root=/dev/mtdblock0"
diff --git a/arch/m68k/configs/m5275evb_defconfig b/arch/m68k/configs/m5275evb_defconfig
index 781f307ff330d..6856848b3fbb6 100644
--- a/arch/m68k/configs/m5275evb_defconfig
+++ b/arch/m68k/configs/m5275evb_defconfig
@@ -1,15 +1,12 @@
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_EXPERT=y
-# CONFIG_KALLSYMS is not set
 # CONFIG_FUTEX is not set
 # CONFIG_EPOLL is not set
 # CONFIG_SIGNALFD is not set
 # CONFIG_TIMERFD is not set
 # CONFIG_EVENTFD is not set
 # CONFIG_AIO is not set
-# CONFIG_VM_EVENT_COUNTERS is not set
-# CONFIG_SLUB_DEBUG is not set
-# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_KALLSYMS is not set
 # CONFIG_MMU is not set
 CONFIG_M5275=y
 # CONFIG_4KSTACKS is not set
@@ -17,7 +14,10 @@ CONFIG_RAMBASE=0x00000000
 CONFIG_RAMSIZE=0x00000000
 CONFIG_VECTORBASE=0x00000000
 CONFIG_KERNELBASE=0x00020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
 CONFIG_BINFMT_FLAT=y
+# CONFIG_VM_EVENT_COUNTERS is not set
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -47,5 +47,6 @@ CONFIG_EXT2_FS=y
 CONFIG_ROMFS_FS=y
 CONFIG_ROMFS_BACKED_BY_MTD=y
 # CONFIG_NETWORK_FILESYSTEMS is not set
+# CONFIG_SLUB_DEBUG is not set
 CONFIG_BOOTPARAM=y
 CONFIG_BOOTPARAM_STRING="root=/dev/mtdblock0"
diff --git a/arch/m68k/configs/m5307c3_defconfig b/arch/m68k/configs/m5307c3_defconfig
index 6eac482356ca7..4e67e48c85a4e 100644
--- a/arch/m68k/configs/m5307c3_defconfig
+++ b/arch/m68k/configs/m5307c3_defconfig
@@ -1,15 +1,12 @@
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_EXPERT=y
-# CONFIG_KALLSYMS is not set
 # CONFIG_FUTEX is not set
 # CONFIG_EPOLL is not set
 # CONFIG_SIGNALFD is not set
 # CONFIG_TIMERFD is not set
 # CONFIG_EVENTFD is not set
 # CONFIG_AIO is not set
-# CONFIG_VM_EVENT_COUNTERS is not set
-# CONFIG_SLUB_DEBUG is not set
-# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_KALLSYMS is not set
 # CONFIG_MMU is not set
 CONFIG_M5307=y
 CONFIG_M5307C3=y
@@ -17,7 +14,10 @@ CONFIG_RAMBASE=0x00000000
 CONFIG_RAMSIZE=0x00800000
 CONFIG_VECTORBASE=0x00000000
 CONFIG_KERNELBASE=0x00020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
 CONFIG_BINFMT_FLAT=y
+# CONFIG_VM_EVENT_COUNTERS is not set
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
@@ -34,7 +34,6 @@ CONFIG_NETDEVICES=y
 CONFIG_PPP=y
 CONFIG_SLIP=y
 CONFIG_SLIP_COMPRESSED=y
-# CONFIG_INPUT_MOUSEDEV is not set
 # CONFIG_INPUT_KEYBOARD is not set
 # CONFIG_INPUT_MOUSE is not set
 # CONFIG_SERIO is not set
@@ -50,6 +49,7 @@ CONFIG_EXT2_FS=y
 CONFIG_ROMFS_FS=y
 CONFIG_ROMFS_BACKED_BY_MTD=y
 # CONFIG_NETWORK_FILESYSTEMS is not set
+# CONFIG_SLUB_DEBUG is not set
 CONFIG_BOOTPARAM=y
 CONFIG_BOOTPARAM_STRING="root=/dev/mtdblock0"
 CONFIG_FULLDEBUG=y
diff --git a/arch/m68k/configs/m5329evb_defconfig b/arch/m68k/configs/m5329evb_defconfig
new file mode 100644
index 0000000000000..69b490e66a1a9
--- /dev/null
+++ b/arch/m68k/configs/m5329evb_defconfig
@@ -0,0 +1,61 @@
+CONFIG_LOG_BUF_SHIFT=14
+CONFIG_SYSFS_SYSCALL=y
+CONFIG_EXPERT=y
+# CONFIG_FHANDLE is not set
+# CONFIG_FUTEX is not set
+# CONFIG_EPOLL is not set
+# CONFIG_SIGNALFD is not set
+# CONFIG_TIMERFD is not set
+# CONFIG_EVENTFD is not set
+# CONFIG_AIO is not set
+# CONFIG_KALLSYMS is not set
+# CONFIG_MMU is not set
+CONFIG_M532x=y
+CONFIG_CLOCK_FREQ=240000000
+CONFIG_RAMBASE=0x40000000
+CONFIG_RAMSIZE=0x02000000
+CONFIG_VECTORBASE=0x40000000
+CONFIG_KERNELBASE=0x40020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BINFMT_FLAT=y
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_INET=y
+# CONFIG_IPV6 is not set
+CONFIG_UEVENT_HELPER=y
+# CONFIG_FW_LOADER is not set
+CONFIG_MTD=y
+CONFIG_MTD_BLOCK=y
+CONFIG_MTD_RAM=y
+CONFIG_MTD_UCLINUX=y
+CONFIG_BLK_DEV_LOOP=y
+CONFIG_BLK_DEV_NBD=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_NETDEVICES=y
+CONFIG_FEC=y
+# CONFIG_WLAN is not set
+CONFIG_INPUT_EVDEV=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+# CONFIG_SERIO is not set
+# CONFIG_VT_CONSOLE is not set
+# CONFIG_UNIX98_PTYS is not set
+CONFIG_SERIAL_MCF=y
+CONFIG_SERIAL_MCF_BAUDRATE=115200
+CONFIG_SERIAL_MCF_CONSOLE=y
+# CONFIG_HW_RANDOM is not set
+CONFIG_SPI=y
+CONFIG_SPI_COLDFIRE_QSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+# CONFIG_HWMON is not set
+CONFIG_EXT2_FS=y
+# CONFIG_FILE_LOCKING is not set
+# CONFIG_DNOTIFY is not set
+# CONFIG_INOTIFY_USER is not set
+CONFIG_JFFS2_FS=m
+CONFIG_ROMFS_FS=y
+CONFIG_ROMFS_BACKED_BY_MTD=y
+# CONFIG_NETWORK_FILESYSTEMS is not set
+CONFIG_FULLDEBUG=y
diff --git a/arch/m68k/configs/m5407c3_defconfig b/arch/m68k/configs/m5407c3_defconfig
index 496dcccb1c182..47712498d7dc4 100644
--- a/arch/m68k/configs/m5407c3_defconfig
+++ b/arch/m68k/configs/m5407c3_defconfig
@@ -1,16 +1,12 @@
 CONFIG_LOG_BUF_SHIFT=14
 CONFIG_EXPERT=y
-# CONFIG_KALLSYMS is not set
 # CONFIG_FUTEX is not set
 # CONFIG_EPOLL is not set
 # CONFIG_SIGNALFD is not set
 # CONFIG_TIMERFD is not set
 # CONFIG_EVENTFD is not set
 # CONFIG_AIO is not set
-# CONFIG_VM_EVENT_COUNTERS is not set
-CONFIG_MODULES=y
-CONFIG_MODULE_UNLOAD=y
-# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_KALLSYMS is not set
 # CONFIG_MMU is not set
 CONFIG_M5407=y
 CONFIG_M5407C3=y
@@ -18,7 +14,10 @@ CONFIG_RAMBASE=0x00000000
 CONFIG_RAMSIZE=0x00000000
 CONFIG_VECTORBASE=0x00000000
 CONFIG_KERNELBASE=0x00020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
 CONFIG_BINFMT_FLAT=y
+# CONFIG_VM_EVENT_COUNTERS is not set
 CONFIG_NET=y
 CONFIG_PACKET=y
 CONFIG_UNIX=y
diff --git a/arch/m68k/configs/m54418evb_defconfig b/arch/m68k/configs/m54418evb_defconfig
new file mode 100644
index 0000000000000..be1bc29ad90eb
--- /dev/null
+++ b/arch/m68k/configs/m54418evb_defconfig
@@ -0,0 +1,70 @@
+CONFIG_LOG_BUF_SHIFT=14
+CONFIG_SYSFS_SYSCALL=y
+CONFIG_EXPERT=y
+# CONFIG_FHANDLE is not set
+# CONFIG_FUTEX is not set
+# CONFIG_EPOLL is not set
+# CONFIG_SIGNALFD is not set
+# CONFIG_TIMERFD is not set
+# CONFIG_EVENTFD is not set
+# CONFIG_AIO is not set
+# CONFIG_KALLSYMS is not set
+CONFIG_COLDFIRE=y
+CONFIG_M5441x=y
+CONFIG_CLOCK_FREQ=250000000
+# CONFIG_4KSTACKS is not set
+CONFIG_RAMBASE=0x40000000
+CONFIG_RAMSIZE=0x02000000
+CONFIG_VECTORBASE=0x40000000
+CONFIG_KERNELBASE=0x40020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+# CONFIG_IOSCHED_BFQ is not set
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_INET=y
+# CONFIG_INET_DIAG is not set
+# CONFIG_IPV6 is not set
+CONFIG_UEVENT_HELPER=y
+# CONFIG_FW_LOADER is not set
+CONFIG_MTD=y
+CONFIG_MTD_BLOCK=y
+CONFIG_MTD_RAM=y
+CONFIG_MTD_UCLINUX=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_NETDEVICES=y
+CONFIG_PPP=y
+# CONFIG_WLAN is not set
+# CONFIG_INPUT is not set
+# CONFIG_VT is not set
+# CONFIG_UNIX98_PTYS is not set
+CONFIG_SERIAL_MCF=y
+CONFIG_SERIAL_MCF_CONSOLE=y
+# CONFIG_HW_RANDOM is not set
+CONFIG_SPI=y
+CONFIG_SPI_FSL_DSPI=y
+# CONFIG_PTP_1588_CLOCK is not set
+# CONFIG_HWMON is not set
+# CONFIG_USB_SUPPORT is not set
+CONFIG_MMC=y
+CONFIG_MMC_SDHCI=y
+CONFIG_MMC_SDHCI_PLTFM=y
+CONFIG_MMC_SDHCI_ESDHC_MCF=y
+CONFIG_DMADEVICES=y
+CONFIG_MCF_EDMA=y
+# CONFIG_NVMEM is not set
+CONFIG_VALIDATE_FS_PARSER=y
+CONFIG_EXT2_FS=y
+# CONFIG_FILE_LOCKING is not set
+# CONFIG_DNOTIFY is not set
+# CONFIG_INOTIFY_USER is not set
+CONFIG_ROMFS_FS=y
+CONFIG_ROMFS_BACKED_BY_MTD=y
+# CONFIG_NETWORK_FILESYSTEMS is not set
+CONFIG_CRYPTO=y
+CONFIG_CRYPTO_NULL=y
+CONFIG_CRYPTO_AES=y
+CONFIG_CRYPTO_HMAC=y
+CONFIG_CRYPTO_SHA256=y
+CONFIG_FRAME_WARN=1024
diff --git a/arch/m68k/configs/m5475evb_defconfig b/arch/m68k/configs/m5475evb_defconfig
index 9be4dae84ebfd..818cbdf65ce34 100644
--- a/arch/m68k/configs/m5475evb_defconfig
+++ b/arch/m68k/configs/m5475evb_defconfig
@@ -1,6 +1,5 @@
-# CONFIG_SWAP is not set
 CONFIG_LOG_BUF_SHIFT=14
-# CONFIG_KALLSYMS is not set
+CONFIG_EXPERT=y
 # CONFIG_FUTEX is not set
 # CONFIG_EPOLL is not set
 # CONFIG_SIGNALFD is not set
@@ -8,9 +7,7 @@ CONFIG_LOG_BUF_SHIFT=14
 # CONFIG_EVENTFD is not set
 # CONFIG_SHMEM is not set
 # CONFIG_AIO is not set
-CONFIG_EXPERT=y
-CONFIG_MODULES=y
-# CONFIG_BLK_DEV_BSG is not set
+# CONFIG_KALLSYMS is not set
 CONFIG_COLDFIRE=y
 # CONFIG_4KSTACKS is not set
 CONFIG_RAMBASE=0x0
@@ -18,8 +15,11 @@ CONFIG_RAMSIZE=0x2000000
 CONFIG_VECTORBASE=0x0
 CONFIG_MBAR=0xff000000
 CONFIG_KERNELBASE=0x20000
-CONFIG_PCI=y
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
 # CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
+# CONFIG_SWAP is not set
+CONFIG_PCI=y
 # CONFIG_FW_LOADER is not set
 CONFIG_MTD=y
 CONFIG_MTD_BLOCK=y
diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig
index abb369fd1f556..1a7694486c08e 100644
--- a/arch/m68k/configs/mac_defconfig
+++ b/arch/m68k/configs/mac_defconfig
@@ -526,8 +526,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig
index cb8de979700fd..468f0dab58ad4 100644
--- a/arch/m68k/configs/multi_defconfig
+++ b/arch/m68k/configs/multi_defconfig
@@ -613,8 +613,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig
index 176540bd50741..ef6e94e5abcdb 100644
--- a/arch/m68k/configs/mvme147_defconfig
+++ b/arch/m68k/configs/mvme147_defconfig
@@ -497,8 +497,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig
index 8b2e5cf4d2f21..cdb8cb030c337 100644
--- a/arch/m68k/configs/mvme16x_defconfig
+++ b/arch/m68k/configs/mvme16x_defconfig
@@ -498,8 +498,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/nettel_defconfig b/arch/m68k/configs/nettel_defconfig
new file mode 100644
index 0000000000000..ee787b5729e4d
--- /dev/null
+++ b/arch/m68k/configs/nettel_defconfig
@@ -0,0 +1,56 @@
+CONFIG_LOG_BUF_SHIFT=14
+CONFIG_EXPERT=y
+# CONFIG_FUTEX is not set
+# CONFIG_EPOLL is not set
+# CONFIG_SIGNALFD is not set
+# CONFIG_TIMERFD is not set
+# CONFIG_EVENTFD is not set
+# CONFIG_AIO is not set
+# CONFIG_KALLSYMS is not set
+# CONFIG_MMU is not set
+CONFIG_M5307=y
+CONFIG_NETtel=y
+CONFIG_RAMBASE=0x00000000
+CONFIG_RAMSIZE=0x00800000
+CONFIG_VECTORBASE=0x00000000
+CONFIG_KERNELBASE=0x00020000
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+CONFIG_BINFMT_FLAT=y
+# CONFIG_VM_EVENT_COUNTERS is not set
+CONFIG_NET=y
+CONFIG_PACKET=y
+CONFIG_UNIX=y
+CONFIG_INET=y
+# CONFIG_INET_DIAG is not set
+# CONFIG_IPV6 is not set
+# CONFIG_FW_LOADER is not set
+CONFIG_MTD=y
+CONFIG_MTD_BLOCK=y
+CONFIG_MTD_RAM=y
+CONFIG_MTD_UCLINUX=y
+CONFIG_BLK_DEV_RAM=y
+CONFIG_NETDEVICES=y
+CONFIG_SMC91X=y
+CONFIG_PPP=y
+CONFIG_SLIP=y
+CONFIG_SLIP_COMPRESSED=y
+# CONFIG_INPUT_KEYBOARD is not set
+# CONFIG_INPUT_MOUSE is not set
+# CONFIG_SERIO is not set
+# CONFIG_VT is not set
+# CONFIG_LEGACY_PTYS is not set
+CONFIG_SERIAL_MCF=y
+CONFIG_SERIAL_MCF_CONSOLE=y
+# CONFIG_HW_RANDOM is not set
+# CONFIG_HWMON is not set
+# CONFIG_USB_SUPPORT is not set
+CONFIG_EXT2_FS=y
+# CONFIG_DNOTIFY is not set
+CONFIG_ROMFS_FS=y
+CONFIG_ROMFS_BACKED_BY_MTD=y
+# CONFIG_NETWORK_FILESYSTEMS is not set
+# CONFIG_SLUB_DEBUG is not set
+CONFIG_BOOTPARAM=y
+CONFIG_BOOTPARAM_STRING="root=/dev/mtdblock0"
+CONFIG_FULLDEBUG=y
diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig
index d48f3cf5285bc..a6af67707e9ed 100644
--- a/arch/m68k/configs/q40_defconfig
+++ b/arch/m68k/configs/q40_defconfig
@@ -516,8 +516,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/stmark2_defconfig b/arch/m68k/configs/stmark2_defconfig
index 515d9b208b106..b3fb95f73a95c 100644
--- a/arch/m68k/configs/stmark2_defconfig
+++ b/arch/m68k/configs/stmark2_defconfig
@@ -2,16 +2,20 @@ CONFIG_LOCALVERSION="stmark2-001"
 CONFIG_DEFAULT_HOSTNAME="stmark2"
 CONFIG_SYSVIPC=y
 CONFIG_LOG_BUF_SHIFT=14
-CONFIG_NAMESPACES=y
 CONFIG_BLK_DEV_INITRD=y
+# CONFIG_RD_BZIP2 is not set
+# CONFIG_RD_LZMA is not set
+# CONFIG_RD_XZ is not set
+# CONFIG_RD_LZO is not set
+# CONFIG_RD_LZ4 is not set
+# CONFIG_RD_ZSTD is not set
 CONFIG_CC_OPTIMIZE_FOR_SIZE=y
+CONFIG_EXPERT=y
 # CONFIG_FHANDLE is not set
 # CONFIG_AIO is not set
 # CONFIG_ADVISE_SYSCALLS is not set
 # CONFIG_MEMBARRIER is not set
-CONFIG_EXPERT=y
-# CONFIG_VM_EVENT_COUNTERS is not set
-# CONFIG_COMPAT_BRK is not set
+# CONFIG_KALLSYMS is not set
 CONFIG_COLDFIRE=y
 CONFIG_M5441x=y
 CONFIG_CLOCK_FREQ=240000000
@@ -21,10 +25,12 @@ CONFIG_RAMBASE=0x40000000
 CONFIG_RAMSIZE=0x8000000
 CONFIG_VECTORBASE=0x40000000
 CONFIG_KERNELBASE=0x40001000
-# CONFIG_BLK_DEV_BSG is not set
 CONFIG_BINFMT_FLAT=y
 CONFIG_BINFMT_ZFLAT=y
 CONFIG_BINFMT_MISC=y
+# CONFIG_COREDUMP is not set
+# CONFIG_COMPAT_BRK is not set
+# CONFIG_VM_EVENT_COUNTERS is not set
 CONFIG_DEVTMPFS=y
 CONFIG_DEVTMPFS_MOUNT=y
 # CONFIG_ALLOW_DEV_COREDUMP is not set
@@ -47,11 +53,11 @@ CONFIG_MTD_SPI_NOR=y
 # CONFIG_INPUT_MOUSE is not set
 CONFIG_SERIO_LIBPS2=y
 # CONFIG_UNIX98_PTYS is not set
-# CONFIG_DEVMEM is not set
 CONFIG_SERIAL_MCF=y
 CONFIG_SERIAL_MCF_BAUDRATE=115200
 CONFIG_SERIAL_MCF_CONSOLE=y
 # CONFIG_HW_RANDOM is not set
+# CONFIG_DEVMEM is not set
 CONFIG_SPI=y
 CONFIG_SPI_DEBUG=y
 CONFIG_SPI_FSL_DSPI=y
@@ -59,7 +65,7 @@ CONFIG_DEBUG_GPIO=y
 CONFIG_GPIO_SYSFS=y
 CONFIG_GPIO_GENERIC_PLATFORM=y
 # CONFIG_HWMON is not set
-# CONFIG_HID is not set
+# CONFIG_HID_SUPPORT is not set
 # CONFIG_USB_SUPPORT is not set
 CONFIG_MMC=y
 CONFIG_MMC_DEBUG=y
@@ -68,6 +74,8 @@ CONFIG_MMC_SDHCI_PLTFM=y
 CONFIG_MMC_SDHCI_ESDHC_MCF=y
 CONFIG_DMADEVICES=y
 CONFIG_MCF_EDMA=y
+# CONFIG_VIRTIO_MENU is not set
+# CONFIG_VHOST_MENU is not set
 CONFIG_EXT2_FS=y
 CONFIG_EXT2_FS_XATTR=y
 CONFIG_EXT2_FS_POSIX_ACL=y
@@ -79,14 +87,13 @@ CONFIG_EXT4_FS_SECURITY=y
 # CONFIG_DNOTIFY is not set
 # CONFIG_INOTIFY_USER is not set
 CONFIG_OVERLAY_FS=y
-CONFIG_FSCACHE=y
 # CONFIG_PROC_SYSCTL is not set
-CONFIG_CRAMFS=y
 CONFIG_SQUASHFS=y
-CONFIG_ROMFS_FS=y
-# CONFIG_CRYPTO_HW is not set
+CONFIG_XZ_DEC=y
 CONFIG_PRINTK_TIME=y
-# CONFIG_DEBUG_BUGVERBOSE is not set
+CONFIG_DYNAMIC_DEBUG_CORE=y
+# CONFIG_DEBUG_MISC is not set
 # CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
-CONFIG_SLUB_DEBUG_ON=y
+# CONFIG_SLUB_DEBUG is not set
 CONFIG_PANIC_ON_OOPS=y
+# CONFIG_RUNTIME_TESTING_MENU is not set
diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig
index 0b96428f25d40..1d8233ebb4547 100644
--- a/arch/m68k/configs/sun3_defconfig
+++ b/arch/m68k/configs/sun3_defconfig
@@ -495,8 +495,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig
index 6140e18244a16..8366a1d850b04 100644
--- a/arch/m68k/configs/sun3x_defconfig
+++ b/arch/m68k/configs/sun3x_defconfig
@@ -495,8 +495,6 @@ CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
 CONFIG_CRYPTO_ZSTD=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 CONFIG_CRYPTO_USER_API_HASH=m
 CONFIG_CRYPTO_USER_API_SKCIPHER=m
 CONFIG_CRYPTO_USER_API_RNG=m
diff --git a/arch/m68k/include/asm/io_no.h b/arch/m68k/include/asm/io_no.h
index 516371d5587aa..4f0f34b06e37c 100644
--- a/arch/m68k/include/asm/io_no.h
+++ b/arch/m68k/include/asm/io_no.h
@@ -107,6 +107,22 @@ static inline void writel(u32 value, volatile void __iomem *addr)
 
 #endif /* IOMEMBASE */
 
+#if defined(CONFIG_COLDFIRE)
+/*
+ * The ColdFire internal peripheral registers are big-endian, so you
+ * cannot use the conventional little-endian readb/readw/readl and
+ * writeb/writew/writel access functions. Define a family of access
+ * functions to give correct endian access that can be used by all
+ * architecture code.
+ */
+#define mcf_read8	__raw_readb
+#define mcf_read16	__raw_readw
+#define mcf_read32	__raw_readl
+#define mcf_write8	__raw_writeb
+#define mcf_write16	__raw_writew
+#define mcf_write32	__raw_writel
+#endif /* CONFIG_COLDFIRE */
+
 #if defined(CONFIG_PCI)
 /*
  * Support for PCI bus access uses the asm-generic access functions.
diff --git a/arch/m68k/include/asm/mcfgpio.h b/arch/m68k/include/asm/mcfgpio.h
index 9c91ecdafc453..7103cfa4edb6c 100644
--- a/arch/m68k/include/asm/mcfgpio.h
+++ b/arch/m68k/include/asm/mcfgpio.h
@@ -95,8 +95,8 @@ static inline void gpio_free(unsigned gpio)
 
 #define MCFGPIO_PORTTYPE		u8
 #define MCFGPIO_PORTSIZE		8
-#define mcfgpio_read(port)		__raw_readb(port)
-#define mcfgpio_write(data, port)	__raw_writeb(data, port)
+#define mcfgpio_read(port)		mcf_read8(port)
+#define mcfgpio_write(data, port)	mcf_write8(data, port)
 
 #elif defined(CONFIG_M5307) || defined(CONFIG_M5407) || defined(CONFIG_M5272)
 
@@ -104,8 +104,8 @@ static inline void gpio_free(unsigned gpio)
 
 #define MCFGPIO_PORTTYPE		u16
 #define MCFGPIO_PORTSIZE		16
-#define mcfgpio_read(port)		__raw_readw(port)
-#define mcfgpio_write(data, port)	__raw_writew(data, port)
+#define mcfgpio_read(port)		mcf_read16(port)
+#define mcfgpio_write(data, port)	mcf_write16(data, port)
 
 #elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
 
@@ -113,8 +113,8 @@ static inline void gpio_free(unsigned gpio)
 
 #define MCFGPIO_PORTTYPE		u32
 #define MCFGPIO_PORTSIZE		32
-#define mcfgpio_read(port)		__raw_readl(port)
-#define mcfgpio_write(data, port)	__raw_writel(data, port)
+#define mcfgpio_read(port)		mcf_read32(port)
+#define mcfgpio_write(data, port)	mcf_write32(data, port)
 
 #endif
 
diff --git a/arch/m68k/include/asm/nettel.h b/arch/m68k/include/asm/nettel.h
index 9bf55cef119e2..30bc915c3d979 100644
--- a/arch/m68k/include/asm/nettel.h
+++ b/arch/m68k/include/asm/nettel.h
@@ -87,12 +87,12 @@ static __inline__ void mcf_setppdata(unsigned int mask, unsigned int bits)
  */
 static __inline__ unsigned int mcf_getppdata(void)
 {
-	return readw(MCFSIM_PBDAT);
+	return mcf_read16(MCFSIM_PBDAT);
 }
 
 static __inline__ void mcf_setppdata(unsigned int mask, unsigned int bits)
 {
-	writew((readw(MCFSIM_PBDAT) & ~mask) | bits, MCFSIM_PBDAT);
+	mcf_write16((mcf_read16(MCFSIM_PBDAT) & ~mask) | bits, MCFSIM_PBDAT);
 }
 #endif
 
diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h
index 64914872a5c98..31d133faa45ef 100644
--- a/arch/m68k/include/asm/uaccess.h
+++ b/arch/m68k/include/asm/uaccess.h
@@ -377,8 +377,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 		return __constant_copy_to_user(to, from, n);
 	return __generic_copy_to_user(to, from, n);
 }
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 #define __get_kernel_nofault(dst, src, type, err_label)			\
 do {									\
diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h
index 3aab2f17e0462..afa0dd8d013fb 100644
--- a/arch/microblaze/include/asm/uaccess.h
+++ b/arch/microblaze/include/asm/uaccess.h
@@ -250,8 +250,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 {
 	return __copy_tofrom_user(to, (__force const void __user *)from, n);
 }
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 /*
  * Copy a null terminated string from userspace.
diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper-util.c b/arch/mips/cavium-octeon/executive/cvmx-helper-util.c
index 53b912745dbdb..abaf91f6ae7c5 100644
--- a/arch/mips/cavium-octeon/executive/cvmx-helper-util.c
+++ b/arch/mips/cavium-octeon/executive/cvmx-helper-util.c
@@ -52,7 +52,7 @@
  *
  * Returns String
  */
-const char *cvmx_helper_interface_mode_to_string(cvmx_helper_interface_mode_t
+const char *cvmx_helper_interface_mode_to_string(enum cvmx_helper_interface_mode
 						 mode)
 {
 	switch (mode) {
diff --git a/arch/mips/cavium-octeon/executive/cvmx-helper.c b/arch/mips/cavium-octeon/executive/cvmx-helper.c
index 9abfc4bf9bd83..1985cd66806a8 100644
--- a/arch/mips/cavium-octeon/executive/cvmx-helper.c
+++ b/arch/mips/cavium-octeon/executive/cvmx-helper.c
@@ -95,7 +95,7 @@ EXPORT_SYMBOL_GPL(cvmx_helper_ports_on_interface);
  * @INTERNAL
  * Return interface mode for CN68xx.
  */
-static cvmx_helper_interface_mode_t __cvmx_get_mode_cn68xx(int interface)
+static enum cvmx_helper_interface_mode __cvmx_get_mode_cn68xx(int interface)
 {
 	union cvmx_mio_qlmx_cfg qlm_cfg;
 	switch (interface) {
@@ -147,7 +147,7 @@ static cvmx_helper_interface_mode_t __cvmx_get_mode_cn68xx(int interface)
  * @INTERNAL
  * Return interface mode for an Octeon II
  */
-static cvmx_helper_interface_mode_t __cvmx_get_mode_octeon2(int interface)
+static enum cvmx_helper_interface_mode __cvmx_get_mode_octeon2(int interface)
 {
 	union cvmx_gmxx_inf_mode mode;
 
@@ -247,7 +247,7 @@ static cvmx_helper_interface_mode_t __cvmx_get_mode_octeon2(int interface)
  * @INTERNAL
  * Return interface mode for CN7XXX.
  */
-static cvmx_helper_interface_mode_t __cvmx_get_mode_cn7xxx(int interface)
+static enum cvmx_helper_interface_mode __cvmx_get_mode_cn7xxx(int interface)
 {
 	union cvmx_gmxx_inf_mode mode;
 
@@ -289,7 +289,7 @@ static cvmx_helper_interface_mode_t __cvmx_get_mode_cn7xxx(int interface)
  * Returns Mode of the interface. Unknown or unsupported interfaces return
  *	   DISABLED.
  */
-cvmx_helper_interface_mode_t cvmx_helper_interface_get_mode(int interface)
+enum cvmx_helper_interface_mode cvmx_helper_interface_get_mode(int interface)
 {
 	union cvmx_gmxx_inf_mode mode;
 
diff --git a/arch/mips/cavium-octeon/executive/cvmx-pko.c b/arch/mips/cavium-octeon/executive/cvmx-pko.c
index 6e70b859a0ac6..b0199d5cb551f 100644
--- a/arch/mips/cavium-octeon/executive/cvmx-pko.c
+++ b/arch/mips/cavium-octeon/executive/cvmx-pko.c
@@ -120,7 +120,7 @@ static void __cvmx_pko_port_map_o68(void)
 {
 	int port;
 	int interface, index;
-	cvmx_helper_interface_mode_t mode;
+	enum cvmx_helper_interface_mode mode;
 	union cvmx_pko_mem_iport_ptrs config;
 
 	/*
@@ -323,11 +323,11 @@ EXPORT_SYMBOL_GPL(cvmx_pko_shutdown);
  *		     queues have higher priority than higher numbered queues.
  *		     There must be num_queues elements in the array.
  */
-cvmx_pko_status_t cvmx_pko_config_port(uint64_t port, uint64_t base_queue,
+enum cvmx_pko_status cvmx_pko_config_port(uint64_t port, uint64_t base_queue,
 				       uint64_t num_queues,
 				       const uint64_t priority[])
 {
-	cvmx_pko_status_t result_code;
+	enum cvmx_pko_status result_code;
 	uint64_t queue;
 	union cvmx_pko_mem_queue_ptrs config;
 	union cvmx_pko_reg_queue_ptrs1 config1;
diff --git a/arch/mips/cavium-octeon/executive/cvmx-spi.c b/arch/mips/cavium-octeon/executive/cvmx-spi.c
index eb9333e84a6be..b6c0b3fa73ad8 100644
--- a/arch/mips/cavium-octeon/executive/cvmx-spi.c
+++ b/arch/mips/cavium-octeon/executive/cvmx-spi.c
@@ -102,7 +102,7 @@ void cvmx_spi_set_callbacks(cvmx_spi_callbacks_t *new_callbacks)
  *
  * Returns Zero on success, negative of failure.
  */
-int cvmx_spi_start_interface(int interface, cvmx_spi_mode_t mode, int timeout,
+int cvmx_spi_start_interface(int interface, enum cvmx_spi_mode mode, int timeout,
 			     int num_ports)
 {
 	int res = -1;
@@ -147,7 +147,7 @@ int cvmx_spi_start_interface(int interface, cvmx_spi_mode_t mode, int timeout,
  *
  * Returns Zero on success, negative of failure.
  */
-int cvmx_spi_restart_interface(int interface, cvmx_spi_mode_t mode, int timeout)
+int cvmx_spi_restart_interface(int interface, enum cvmx_spi_mode mode, int timeout)
 {
 	int res = -1;
 
@@ -192,7 +192,7 @@ EXPORT_SYMBOL_GPL(cvmx_spi_restart_interface);
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-int cvmx_spi_reset_cb(int interface, cvmx_spi_mode_t mode)
+int cvmx_spi_reset_cb(int interface, enum cvmx_spi_mode mode)
 {
 	union cvmx_spxx_dbg_deskew_ctl spxx_dbg_deskew_ctl;
 	union cvmx_spxx_clk_ctl spxx_clk_ctl;
@@ -308,7 +308,7 @@ int cvmx_spi_reset_cb(int interface, cvmx_spi_mode_t mode)
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-int cvmx_spi_calendar_setup_cb(int interface, cvmx_spi_mode_t mode,
+int cvmx_spi_calendar_setup_cb(int interface, enum cvmx_spi_mode mode,
 			       int num_ports)
 {
 	int port;
@@ -427,7 +427,7 @@ int cvmx_spi_calendar_setup_cb(int interface, cvmx_spi_mode_t mode,
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-int cvmx_spi_clock_detect_cb(int interface, cvmx_spi_mode_t mode, int timeout)
+int cvmx_spi_clock_detect_cb(int interface, enum cvmx_spi_mode mode, int timeout)
 {
 	int clock_transitions;
 	union cvmx_spxx_clk_stat stat;
@@ -505,7 +505,7 @@ int cvmx_spi_clock_detect_cb(int interface, cvmx_spi_mode_t mode, int timeout)
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-int cvmx_spi_training_cb(int interface, cvmx_spi_mode_t mode, int timeout)
+int cvmx_spi_training_cb(int interface, enum cvmx_spi_mode mode, int timeout)
 {
 	union cvmx_spxx_trn4_ctl spxx_trn4_ctl;
 	union cvmx_spxx_clk_stat stat;
@@ -574,7 +574,7 @@ int cvmx_spi_training_cb(int interface, cvmx_spi_mode_t mode, int timeout)
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-int cvmx_spi_calendar_sync_cb(int interface, cvmx_spi_mode_t mode, int timeout)
+int cvmx_spi_calendar_sync_cb(int interface, enum cvmx_spi_mode mode, int timeout)
 {
 	uint64_t MS = cvmx_sysinfo_get()->cpu_clock_hz / 1000;
 	if (mode & CVMX_SPI_MODE_RX_HALFPLEX) {
@@ -630,7 +630,7 @@ int cvmx_spi_calendar_sync_cb(int interface, cvmx_spi_mode_t mode, int timeout)
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-int cvmx_spi_interface_up_cb(int interface, cvmx_spi_mode_t mode)
+int cvmx_spi_interface_up_cb(int interface, enum cvmx_spi_mode mode)
 {
 	union cvmx_gmxx_rxx_frm_min gmxx_rxx_frm_min;
 	union cvmx_gmxx_rxx_frm_max gmxx_rxx_frm_max;
diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig
index 7c43352fac6bd..509515cf4751f 100644
--- a/arch/mips/configs/decstation_64_defconfig
+++ b/arch/mips/configs/decstation_64_defconfig
@@ -199,8 +199,6 @@ CONFIG_CRYPTO_LZO=m
 CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 # CONFIG_CRYPTO_HW is not set
 CONFIG_MAGIC_SYSRQ=y
 # CONFIG_FTRACE is not set
diff --git a/arch/mips/configs/decstation_defconfig b/arch/mips/configs/decstation_defconfig
index aee10274f0488..3a17c080292fd 100644
--- a/arch/mips/configs/decstation_defconfig
+++ b/arch/mips/configs/decstation_defconfig
@@ -194,8 +194,6 @@ CONFIG_CRYPTO_LZO=m
 CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 # CONFIG_CRYPTO_HW is not set
 CONFIG_FRAME_WARN=2048
 CONFIG_MAGIC_SYSRQ=y
diff --git a/arch/mips/configs/decstation_r4k_defconfig b/arch/mips/configs/decstation_r4k_defconfig
index a1698049aa7a7..7f5823663702a 100644
--- a/arch/mips/configs/decstation_r4k_defconfig
+++ b/arch/mips/configs/decstation_r4k_defconfig
@@ -194,8 +194,6 @@ CONFIG_CRYPTO_LZO=m
 CONFIG_CRYPTO_842=m
 CONFIG_CRYPTO_LZ4=m
 CONFIG_CRYPTO_LZ4HC=m
-CONFIG_CRYPTO_DRBG_HASH=y
-CONFIG_CRYPTO_DRBG_CTR=y
 # CONFIG_CRYPTO_HW is not set
 CONFIG_FRAME_WARN=2048
 CONFIG_MAGIC_SYSRQ=y
diff --git a/arch/mips/configs/mtx1_defconfig b/arch/mips/configs/mtx1_defconfig
index 3629afbe5d75c..aec719b1b1930 100644
--- a/arch/mips/configs/mtx1_defconfig
+++ b/arch/mips/configs/mtx1_defconfig
@@ -184,9 +184,6 @@ CONFIG_BT_HCIUART_BCSP=y
 CONFIG_BT_HCIBCM203X=m
 CONFIG_BT_HCIBPA10X=m
 CONFIG_BT_HCIBFUSB=m
-CONFIG_BT_HCIDTL1=m
-CONFIG_BT_HCIBT3C=m
-CONFIG_BT_HCIBLUECARD=m
 CONFIG_BT_HCIVHCI=m
 CONFIG_CONNECTOR=m
 CONFIG_MTD=y
diff --git a/arch/mips/include/asm/octeon/cvmx-helper-util.h b/arch/mips/include/asm/octeon/cvmx-helper-util.h
index 97b27a07cfb0c..103bb5b3142b1 100644
--- a/arch/mips/include/asm/octeon/cvmx-helper-util.h
+++ b/arch/mips/include/asm/octeon/cvmx-helper-util.h
@@ -42,7 +42,7 @@
  * Returns String
  */
 extern const char
-    *cvmx_helper_interface_mode_to_string(cvmx_helper_interface_mode_t mode);
+    *cvmx_helper_interface_mode_to_string(enum cvmx_helper_interface_mode mode);
 
 /**
  * Setup Random Early Drop to automatically begin dropping packets.
diff --git a/arch/mips/include/asm/octeon/cvmx-helper.h b/arch/mips/include/asm/octeon/cvmx-helper.h
index 0cddce35291bd..98824ff6314c7 100644
--- a/arch/mips/include/asm/octeon/cvmx-helper.h
+++ b/arch/mips/include/asm/octeon/cvmx-helper.h
@@ -38,7 +38,7 @@
 #include <asm/octeon/cvmx-fpa.h>
 #include <asm/octeon/cvmx-wqe.h>
 
-typedef enum {
+enum cvmx_helper_interface_mode {
 	CVMX_HELPER_INTERFACE_MODE_DISABLED,
 	CVMX_HELPER_INTERFACE_MODE_RGMII,
 	CVMX_HELPER_INTERFACE_MODE_GMII,
@@ -49,7 +49,7 @@ typedef enum {
 	CVMX_HELPER_INTERFACE_MODE_PICMG,
 	CVMX_HELPER_INTERFACE_MODE_NPI,
 	CVMX_HELPER_INTERFACE_MODE_LOOP,
-} cvmx_helper_interface_mode_t;
+};
 
 union cvmx_helper_link_info {
 	uint64_t u64;
@@ -125,7 +125,7 @@ extern int cvmx_helper_get_number_of_interfaces(void);
  * Returns Mode of the interface. Unknown or unsupported interfaces return
  *	   DISABLED.
  */
-extern cvmx_helper_interface_mode_t cvmx_helper_interface_get_mode(int
+extern enum cvmx_helper_interface_mode cvmx_helper_interface_get_mode(int
 								   interface);
 
 /**
diff --git a/arch/mips/include/asm/octeon/cvmx-pip.h b/arch/mips/include/asm/octeon/cvmx-pip.h
index 01ca7267a2bac..911276ee90c22 100644
--- a/arch/mips/include/asm/octeon/cvmx-pip.h
+++ b/arch/mips/include/asm/octeon/cvmx-pip.h
@@ -180,7 +180,7 @@ typedef union {
 /**
  * Status statistics for a port
  */
-typedef struct {
+struct cvmx_pip_port_status {
 	/* Inbound octets marked to be dropped by the IPD */
 	uint32_t dropped_octets;
 	/* Inbound packets marked to be dropped by the IPD */
@@ -236,7 +236,7 @@ typedef struct {
 	uint64_t inb_octets;
 	/* Number of packets with GMX/SPX/PCI errors received by PIP */
 	uint16_t inb_errors;
-} cvmx_pip_port_status_t;
+};
 
 /**
  * Definition of the PIP custom header that can be prepended
@@ -365,7 +365,7 @@ static inline void cvmx_pip_config_diffserv_qos(uint64_t diffserv, uint64_t qos)
  * @status:   Where to put the results.
  */
 static inline void cvmx_pip_get_port_status(uint64_t port_num, uint64_t clear,
-					    cvmx_pip_port_status_t *status)
+					    struct cvmx_pip_port_status *status)
 {
 	union cvmx_pip_stat_ctl pip_stat_ctl;
 	union cvmx_pip_stat0_prtx stat0;
diff --git a/arch/mips/include/asm/octeon/cvmx-pko.h b/arch/mips/include/asm/octeon/cvmx-pko.h
index f18a7f24daf82..26cb26a7ff2b4 100644
--- a/arch/mips/include/asm/octeon/cvmx-pko.h
+++ b/arch/mips/include/asm/octeon/cvmx-pko.h
@@ -80,7 +80,7 @@
 #define CVMX_PKO_ILLEGAL_QUEUE	0xFFFF
 #define CVMX_PKO_MAX_QUEUE_DEPTH 0
 
-typedef enum {
+enum cvmx_pko_status {
 	CVMX_PKO_SUCCESS,
 	CVMX_PKO_INVALID_PORT,
 	CVMX_PKO_INVALID_QUEUE,
@@ -88,12 +88,12 @@ typedef enum {
 	CVMX_PKO_NO_MEMORY,
 	CVMX_PKO_PORT_ALREADY_SETUP,
 	CVMX_PKO_CMD_QUEUE_INIT_ERROR
-} cvmx_pko_status_t;
+};
 
 /**
  * This enumeration represents the different locking modes supported by PKO.
  */
-typedef enum {
+enum cvmx_pko_lock {
 	/*
 	 * PKO doesn't do any locking. It is the responsibility of the
 	 * application to make sure that no other core is accessing
@@ -112,13 +112,13 @@ typedef enum {
 	 * ll/sc. This is the most portable locking mechanism.
 	 */
 	CVMX_PKO_LOCK_CMD_QUEUE = 2,
-} cvmx_pko_lock_t;
+};
 
-typedef struct {
+struct cvmx_pko_port_status {
 	uint32_t packets;
 	uint64_t octets;
 	uint64_t doorbell;
-} cvmx_pko_port_status_t;
+};
 
 /**
  * This structure defines the address to use on a packet enqueue
@@ -306,7 +306,7 @@ extern void cvmx_pko_shutdown(void);
  *		     of a value of 1. There must be num_queues elements in the
  *		     array.
  */
-extern cvmx_pko_status_t cvmx_pko_config_port(uint64_t port,
+extern enum cvmx_pko_status cvmx_pko_config_port(uint64_t port,
 					      uint64_t base_queue,
 					      uint64_t num_queues,
 					      const uint64_t priority[]);
@@ -374,7 +374,7 @@ static inline void cvmx_pko_doorbell(uint64_t port, uint64_t queue,
  */
 
 static inline void cvmx_pko_send_packet_prepare(uint64_t port, uint64_t queue,
-						cvmx_pko_lock_t use_locking)
+						enum cvmx_pko_lock use_locking)
 {
 	if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG) {
 		/*
@@ -414,12 +414,12 @@ static inline void cvmx_pko_send_packet_prepare(uint64_t port, uint64_t queue,
  * Returns: CVMX_PKO_SUCCESS on success, or error code on
  * failure of output
  */
-static inline cvmx_pko_status_t cvmx_pko_send_packet_finish(
+static inline enum cvmx_pko_status cvmx_pko_send_packet_finish(
 	uint64_t port,
 	uint64_t queue,
 	union cvmx_pko_command_word0 pko_command,
 	union cvmx_buf_ptr packet,
-	cvmx_pko_lock_t use_locking)
+	enum cvmx_pko_lock use_locking)
 {
 	cvmx_cmd_queue_result_t result;
 	if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG)
@@ -457,13 +457,13 @@ static inline cvmx_pko_status_t cvmx_pko_send_packet_finish(
  * Returns: CVMX_PKO_SUCCESS on success, or error code on
  * failure of output
  */
-static inline cvmx_pko_status_t cvmx_pko_send_packet_finish3(
+static inline enum cvmx_pko_status cvmx_pko_send_packet_finish3(
 	uint64_t port,
 	uint64_t queue,
 	union cvmx_pko_command_word0 pko_command,
 	union cvmx_buf_ptr packet,
 	uint64_t addr,
-	cvmx_pko_lock_t use_locking)
+	enum cvmx_pko_lock use_locking)
 {
 	cvmx_cmd_queue_result_t result;
 	if (use_locking == CVMX_PKO_LOCK_ATOMIC_TAG)
@@ -574,7 +574,7 @@ static inline int cvmx_pko_get_num_queues(int port)
  * @status:   Where to put the results.
  */
 static inline void cvmx_pko_get_port_status(uint64_t port_num, uint64_t clear,
-					    cvmx_pko_port_status_t *status)
+					    struct cvmx_pko_port_status *status)
 {
 	union cvmx_pko_reg_read_idx pko_reg_read_idx;
 	union cvmx_pko_mem_count0 pko_mem_count0;
diff --git a/arch/mips/include/asm/octeon/cvmx-pow.h b/arch/mips/include/asm/octeon/cvmx-pow.h
index 21b4378244faf..409029809639f 100644
--- a/arch/mips/include/asm/octeon/cvmx-pow.h
+++ b/arch/mips/include/asm/octeon/cvmx-pow.h
@@ -84,10 +84,10 @@ enum cvmx_pow_tag_type {
 /**
  * Wait flag values for pow functions.
  */
-typedef enum {
+enum cvmx_pow_wait {
 	CVMX_POW_WAIT = 1,
 	CVMX_POW_NO_WAIT = 0,
-} cvmx_pow_wait_t;
+};
 
 /**
  *  POW tag operations.	 These are used in the data stored to the POW.
@@ -1348,7 +1348,7 @@ static inline void cvmx_pow_tag_sw_wait(void)
  * Returns: the WQE pointer from POW. Returns NULL if no work
  * was available.
  */
-static inline struct cvmx_wqe *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_t
+static inline struct cvmx_wqe *cvmx_pow_work_request_sync_nocheck(enum cvmx_pow_wait
 							     wait)
 {
 	cvmx_pow_load_addr_t ptr;
@@ -1382,7 +1382,7 @@ static inline struct cvmx_wqe *cvmx_pow_work_request_sync_nocheck(cvmx_pow_wait_
  * Returns: the WQE pointer from POW. Returns NULL if no work
  * was available.
  */
-static inline struct cvmx_wqe *cvmx_pow_work_request_sync(cvmx_pow_wait_t wait)
+static inline struct cvmx_wqe *cvmx_pow_work_request_sync(enum cvmx_pow_wait wait)
 {
 	if (CVMX_ENABLE_POW_CHECKS)
 		__cvmx_pow_warn_if_pending_switch(__func__);
@@ -1436,7 +1436,7 @@ static inline enum cvmx_pow_tag_type cvmx_pow_work_request_null_rd(void)
  *	  timeout), 0 to cause response to return immediately
  */
 static inline void cvmx_pow_work_request_async_nocheck(int scr_addr,
-						       cvmx_pow_wait_t wait)
+						       enum cvmx_pow_wait wait)
 {
 	cvmx_pow_iobdma_store_t data;
 
@@ -1465,7 +1465,7 @@ static inline void cvmx_pow_work_request_async_nocheck(int scr_addr,
  *		    timeout), 0 to cause response to return immediately
  */
 static inline void cvmx_pow_work_request_async(int scr_addr,
-					       cvmx_pow_wait_t wait)
+					       enum cvmx_pow_wait wait)
 {
 	if (CVMX_ENABLE_POW_CHECKS)
 		__cvmx_pow_warn_if_pending_switch(__func__);
diff --git a/arch/mips/include/asm/octeon/cvmx-spi.h b/arch/mips/include/asm/octeon/cvmx-spi.h
index d5038cc4b475a..88f7e59a396b4 100644
--- a/arch/mips/include/asm/octeon/cvmx-spi.h
+++ b/arch/mips/include/asm/octeon/cvmx-spi.h
@@ -36,35 +36,35 @@
 
 /* CSR typedefs have been moved to cvmx-csr-*.h */
 
-typedef enum {
+enum cvmx_spi_mode {
 	CVMX_SPI_MODE_UNKNOWN = 0,
 	CVMX_SPI_MODE_TX_HALFPLEX = 1,
 	CVMX_SPI_MODE_RX_HALFPLEX = 2,
 	CVMX_SPI_MODE_DUPLEX = 3
-} cvmx_spi_mode_t;
+};
 
 /** Callbacks structure to customize SPI4 initialization sequence */
 typedef struct {
     /** Called to reset SPI4 DLL */
-	int (*reset_cb) (int interface, cvmx_spi_mode_t mode);
+	int (*reset_cb)(int interface, enum cvmx_spi_mode mode);
 
     /** Called to setup calendar */
-	int (*calendar_setup_cb) (int interface, cvmx_spi_mode_t mode,
-				  int num_ports);
+	int (*calendar_setup_cb)(int interface, enum cvmx_spi_mode mode,
+				 int num_ports);
 
     /** Called for Tx and Rx clock detection */
-	int (*clock_detect_cb) (int interface, cvmx_spi_mode_t mode,
-				int timeout);
+	int (*clock_detect_cb)(int interface, enum cvmx_spi_mode mode,
+			       int timeout);
 
     /** Called to perform link training */
-	int (*training_cb) (int interface, cvmx_spi_mode_t mode, int timeout);
+	int (*training_cb)(int interface, enum cvmx_spi_mode mode, int timeout);
 
     /** Called for calendar data synchronization */
-	int (*calendar_sync_cb) (int interface, cvmx_spi_mode_t mode,
-				 int timeout);
+	int (*calendar_sync_cb)(int interface, enum cvmx_spi_mode mode,
+				int timeout);
 
     /** Called when interface is up */
-	int (*interface_up_cb) (int interface, cvmx_spi_mode_t mode);
+	int (*interface_up_cb)(int interface, enum cvmx_spi_mode mode);
 
 } cvmx_spi_callbacks_t;
 
@@ -94,7 +94,7 @@ static inline int cvmx_spi_is_spi_interface(int interface)
  *
  * Returns Zero on success, negative of failure.
  */
-extern int cvmx_spi_start_interface(int interface, cvmx_spi_mode_t mode,
+extern int cvmx_spi_start_interface(int interface, enum cvmx_spi_mode mode,
 				    int timeout, int num_ports);
 
 /**
@@ -110,7 +110,7 @@ extern int cvmx_spi_start_interface(int interface, cvmx_spi_mode_t mode,
  * @timeout:   Timeout to wait for clock synchronization in seconds
  * Returns Zero on success, negative of failure.
  */
-extern int cvmx_spi_restart_interface(int interface, cvmx_spi_mode_t mode,
+extern int cvmx_spi_restart_interface(int interface, enum cvmx_spi_mode mode,
 				      int timeout);
 
 /**
@@ -180,7 +180,7 @@ extern void cvmx_spi_set_callbacks(cvmx_spi_callbacks_t *new_callbacks);
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-extern int cvmx_spi_reset_cb(int interface, cvmx_spi_mode_t mode);
+extern int cvmx_spi_reset_cb(int interface, enum cvmx_spi_mode mode);
 
 /**
  * Callback to setup calendar and miscellaneous settings before clock
@@ -197,7 +197,7 @@ extern int cvmx_spi_reset_cb(int interface, cvmx_spi_mode_t mode);
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-extern int cvmx_spi_calendar_setup_cb(int interface, cvmx_spi_mode_t mode,
+extern int cvmx_spi_calendar_setup_cb(int interface, enum cvmx_spi_mode mode,
 				      int num_ports);
 
 /**
@@ -214,7 +214,7 @@ extern int cvmx_spi_calendar_setup_cb(int interface, cvmx_spi_mode_t mode,
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-extern int cvmx_spi_clock_detect_cb(int interface, cvmx_spi_mode_t mode,
+extern int cvmx_spi_clock_detect_cb(int interface, enum cvmx_spi_mode mode,
 				    int timeout);
 
 /**
@@ -231,7 +231,7 @@ extern int cvmx_spi_clock_detect_cb(int interface, cvmx_spi_mode_t mode,
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-extern int cvmx_spi_training_cb(int interface, cvmx_spi_mode_t mode,
+extern int cvmx_spi_training_cb(int interface, enum cvmx_spi_mode mode,
 				int timeout);
 
 /**
@@ -248,7 +248,7 @@ extern int cvmx_spi_training_cb(int interface, cvmx_spi_mode_t mode,
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-extern int cvmx_spi_calendar_sync_cb(int interface, cvmx_spi_mode_t mode,
+extern int cvmx_spi_calendar_sync_cb(int interface, enum cvmx_spi_mode mode,
 				     int timeout);
 
 /**
@@ -264,6 +264,6 @@ extern int cvmx_spi_calendar_sync_cb(int interface, cvmx_spi_mode_t mode,
  * Returns Zero on success, non-zero error code on failure (will cause
  * SPI initialization to abort)
  */
-extern int cvmx_spi_interface_up_cb(int interface, cvmx_spi_mode_t mode);
+extern int cvmx_spi_interface_up_cb(int interface, enum cvmx_spi_mode mode);
 
 #endif /* __CVMX_SPI_H__ */
diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h
index c0cede273c7c0..f00c36676b737 100644
--- a/arch/mips/include/asm/uaccess.h
+++ b/arch/mips/include/asm/uaccess.h
@@ -433,8 +433,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 	return __cu_len_r;
 }
 
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 extern __kernel_size_t __bzero(void __user *addr, __kernel_size_t size);
 
diff --git a/arch/mips/include/uapi/asm/errno.h b/arch/mips/include/uapi/asm/errno.h
index c01ed91b1ef44..1835a50b69cef 100644
--- a/arch/mips/include/uapi/asm/errno.h
+++ b/arch/mips/include/uapi/asm/errno.h
@@ -126,6 +126,8 @@
 
 #define EHWPOISON	168	/* Memory page has hardware error */
 
+#define EFTYPE		169	/* Wrong file type for the intended operation */
+
 #define EDQUOT		1133	/* Quota exceeded */
 
 
diff --git a/arch/mips/mm/dma-noncoherent.c b/arch/mips/mm/dma-noncoherent.c
index ab4f2a75a7d01..30ef3e247eb7a 100644
--- a/arch/mips/mm/dma-noncoherent.c
+++ b/arch/mips/mm/dma-noncoherent.c
@@ -139,6 +139,6 @@ void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
 #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS
 void arch_setup_dma_ops(struct device *dev, bool coherent)
 {
-	dev->dma_coherent = coherent;
+	dev_assign_dma_coherent(dev, coherent);
 }
 #endif
diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h
index 6ccc9a232c239..5e6e05cc6efc7 100644
--- a/arch/nios2/include/asm/uaccess.h
+++ b/arch/nios2/include/asm/uaccess.h
@@ -57,8 +57,7 @@ extern unsigned long
 raw_copy_from_user(void *to, const void __user *from, unsigned long n);
 extern unsigned long
 raw_copy_to_user(void __user *to, const void *from, unsigned long n);
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 extern long strncpy_from_user(char *__to, const char __user *__from,
 			      long __len);
diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h
index d6500a374e183..db934ebc0069f 100644
--- a/arch/openrisc/include/asm/uaccess.h
+++ b/arch/openrisc/include/asm/uaccess.h
@@ -218,8 +218,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long size)
 {
 	return __copy_tofrom_user((__force void *)to, from, size);
 }
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 extern unsigned long __clear_user(void __user *addr, unsigned long size);
 
diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h
index 6c531d2c847eb..0d17f81c8b270 100644
--- a/arch/parisc/include/asm/uaccess.h
+++ b/arch/parisc/include/asm/uaccess.h
@@ -197,7 +197,6 @@ unsigned long __must_check raw_copy_to_user(void __user *dst, const void *src,
 					    unsigned long len);
 unsigned long __must_check raw_copy_from_user(void *dst, const void __user *src,
 					    unsigned long len);
-#define INLINE_COPY_TO_USER
-#define INLINE_COPY_FROM_USER
+#define INLINE_COPY_USER
 
 #endif /* __PARISC_UACCESS_H */
diff --git a/arch/parisc/include/uapi/asm/errno.h b/arch/parisc/include/uapi/asm/errno.h
index 8cbc07c1903e4..93194fbb0a80d 100644
--- a/arch/parisc/include/uapi/asm/errno.h
+++ b/arch/parisc/include/uapi/asm/errno.h
@@ -124,4 +124,6 @@
 
 #define EHWPOISON	257	/* Memory page has hardware error */
 
+#define EFTYPE		258	/* Wrong file type for the intended operation */
+
 #endif
diff --git a/arch/parisc/include/uapi/asm/fcntl.h b/arch/parisc/include/uapi/asm/fcntl.h
index 03dee816cb136..d46812f2f0f49 100644
--- a/arch/parisc/include/uapi/asm/fcntl.h
+++ b/arch/parisc/include/uapi/asm/fcntl.h
@@ -19,6 +19,7 @@
 
 #define O_PATH		020000000
 #define __O_TMPFILE	040000000
+#define OPENAT2_REGULAR	0100000000
 
 #define F_GETLK64	8
 #define F_SETLK64	9
diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig
index ccabc6e17168e..00f280e38004e 100644
--- a/arch/powerpc/configs/ppc6xx_defconfig
+++ b/arch/powerpc/configs/ppc6xx_defconfig
@@ -282,9 +282,6 @@ CONFIG_BT_HCIUART_BCSP=y
 CONFIG_BT_HCIBCM203X=m
 CONFIG_BT_HCIBPA10X=m
 CONFIG_BT_HCIBFUSB=m
-CONFIG_BT_HCIDTL1=m
-CONFIG_BT_HCIBT3C=m
-CONFIG_BT_HCIBLUECARD=m
 CONFIG_BT_HCIVHCI=m
 CONFIG_CFG80211=m
 CONFIG_MAC80211=m
@@ -551,7 +548,6 @@ CONFIG_GAMEPORT_EMU10K1=m
 CONFIG_GAMEPORT_FM801=m
 # CONFIG_LEGACY_PTYS is not set
 CONFIG_SERIAL_NONSTANDARD=y
-CONFIG_SYNCLINK_GT=m
 CONFIG_NOZOMI=m
 CONFIG_N_HDLC=m
 CONFIG_SERIAL_8250=y
diff --git a/arch/powerpc/include/asm/machdep.h b/arch/powerpc/include/asm/machdep.h
index 3298eec123a32..256f9309bf4fa 100644
--- a/arch/powerpc/include/asm/machdep.h
+++ b/arch/powerpc/include/asm/machdep.h
@@ -169,7 +169,7 @@ struct machdep_calls {
 
 #ifdef CONFIG_PCI_IOV
 	void (*pcibios_fixup_sriov)(struct pci_dev *pdev);
-	resource_size_t (*pcibios_iov_resource_alignment)(struct pci_dev *, int resno);
+	resource_size_t (*pcibios_iov_resource_alignment)(const struct pci_dev *, int resno);
 	int (*pcibios_sriov_enable)(struct pci_dev *pdev, u16 num_vfs);
 	int (*pcibios_sriov_disable)(struct pci_dev *pdev);
 #endif /* CONFIG_PCI_IOV */
diff --git a/arch/powerpc/include/asm/pci.h b/arch/powerpc/include/asm/pci.h
index 46a9c4491ed0c..72f286e747863 100644
--- a/arch/powerpc/include/asm/pci.h
+++ b/arch/powerpc/include/asm/pci.h
@@ -82,8 +82,6 @@ extern int pci_legacy_write(struct pci_bus *bus, loff_t port, u32 val,
 extern int pci_mmap_legacy_page_range(struct pci_bus *bus,
 				      struct vm_area_struct *vma,
 				      enum pci_mmap_state mmap_state);
-extern void pci_adjust_legacy_attr(struct pci_bus *bus,
-				   enum pci_mmap_state mmap_type);
 #define HAVE_PCI_LEGACY	1
 
 extern void pcibios_claim_one_bus(struct pci_bus *b);
diff --git a/arch/powerpc/include/asm/vga.h b/arch/powerpc/include/asm/vga.h
index f2dc40e1c52ae..e45063b02b458 100644
--- a/arch/powerpc/include/asm/vga.h
+++ b/arch/powerpc/include/asm/vga.h
@@ -14,7 +14,7 @@
 #include <asm/io.h>
 
 
-#if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_MDA_CONSOLE)
+#ifdef CONFIG_VGA_CONSOLE
 
 #define VT_BUF_HAVE_RW
 /*
@@ -40,7 +40,7 @@ static inline void scr_memsetw(u16 *s, u16 v, unsigned int n)
 	memset16(s, cpu_to_le16(v), n / 2);
 }
 
-#endif /* !CONFIG_VGA_CONSOLE && !CONFIG_MDA_CONSOLE */
+#endif /* !CONFIG_VGA_CONSOLE */
 
 #ifdef __powerpc64__
 #define VGA_MAP_MEM(x,s) ((unsigned long) ioremap((x), s))
diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c
index 8b4de508d2eb1..1f8b5a0f69be2 100644
--- a/arch/powerpc/kernel/dma-iommu.c
+++ b/arch/powerpc/kernel/dma-iommu.c
@@ -67,7 +67,7 @@ bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg,
 }
 bool arch_dma_alloc_direct(struct device *dev)
 {
-	if (dev->dma_ops_bypass && dev->bus_dma_limit)
+	if (dev_dma_ops_bypass(dev) && dev->bus_dma_limit)
 		return true;
 
 	return false;
@@ -75,7 +75,7 @@ bool arch_dma_alloc_direct(struct device *dev)
 
 bool arch_dma_free_direct(struct device *dev, dma_addr_t dma_handle)
 {
-	if (!dev->dma_ops_bypass || !dev->bus_dma_limit)
+	if (!dev_dma_ops_bypass(dev) || !dev->bus_dma_limit)
 		return false;
 
 	return is_direct_handle(dev, dma_handle);
@@ -164,7 +164,7 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask)
 		 * fixed ops will be used for RAM. This is limited by
 		 * bus_dma_limit which is set when RAM is pre-mapped.
 		 */
-		dev->dma_ops_bypass = true;
+		dev_set_dma_ops_bypass(dev);
 		dev_info(dev, "iommu: 64-bit OK but direct DMA is limited by %llx\n",
 			 dev->bus_dma_limit);
 		return 1;
@@ -185,7 +185,7 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask)
 	}
 
 	dev_dbg(dev, "iommu: not 64-bit, using default ops\n");
-	dev->dma_ops_bypass = false;
+	dev_clear_dma_ops_bypass(dev);
 	return 1;
 }
 
diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c
index 8efe95a0c4ffd..3c4ca90e2ab77 100644
--- a/arch/powerpc/kernel/pci-common.c
+++ b/arch/powerpc/kernel/pci-common.c
@@ -254,7 +254,7 @@ resource_size_t pcibios_default_alignment(void)
 }
 
 #ifdef CONFIG_PCI_IOV
-resource_size_t pcibios_iov_resource_alignment(struct pci_dev *pdev, int resno)
+resource_size_t pcibios_iov_resource_alignment(const struct pci_dev *pdev, int resno)
 {
 	if (ppc_md.pcibios_iov_resource_alignment)
 		return ppc_md.pcibios_iov_resource_alignment(pdev, resno);
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index b6f3ae03ca9e6..64f0df5bb5cdc 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -41,7 +41,6 @@
 #include <linux/libfdt.h>
 #include <linux/memremap.h>
 #include <linux/memory.h>
-#include <linux/bootmem_info.h>
 
 #include <asm/pgalloc.h>
 #include <asm/page.h>
@@ -388,13 +387,6 @@ void __ref vmemmap_free(unsigned long start, unsigned long end,
 
 #endif
 
-#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
-void register_page_bootmem_memmap(unsigned long section_nr,
-				  struct page *start_page, unsigned long size)
-{
-}
-#endif /* CONFIG_HAVE_BOOTMEM_INFO_NODE */
-
 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
 
 #ifdef CONFIG_PPC_BOOK3S_64
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 648d0c5602ec8..4c1afab919963 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -158,12 +158,13 @@ int __ref arch_add_memory(int nid, u64 start, u64 size,
 	return rc;
 }
 
-void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+			      struct dev_pagemap *pgmap)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 
-	__remove_pages(start_pfn, nr_pages, altmap);
+	__remove_pages(start_pfn, nr_pages, altmap, pgmap);
 	arch_remove_linear_mapping(start, size);
 }
 #endif
diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c
index 10fa9b844fccd..f6de8c1169d5b 100644
--- a/arch/powerpc/platforms/cell/spufs/file.c
+++ b/arch/powerpc/platforms/cell/spufs/file.c
@@ -1430,7 +1430,7 @@ static int spufs_mfc_open(struct inode *inode, struct file *file)
 	if (ctx->owner != current->mm)
 		return -EINVAL;
 
-	if (icount_read(inode) != 1)
+	if (icount_read_once(inode) != 1)
 		return -EBUSY;
 
 	mutex_lock(&ctx->mapping_lock);
diff --git a/arch/powerpc/platforms/powernv/pci-sriov.c b/arch/powerpc/platforms/powernv/pci-sriov.c
index 7105a573aec42..8652078801f2a 100644
--- a/arch/powerpc/platforms/powernv/pci-sriov.c
+++ b/arch/powerpc/platforms/powernv/pci-sriov.c
@@ -244,8 +244,8 @@ void pnv_pci_ioda_fixup_iov(struct pci_dev *pdev)
 	}
 }
 
-resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
-						      int resno)
+resource_size_t pnv_pci_iov_resource_alignment(const struct pci_dev *pdev,
+					       int resno)
 {
 	resource_size_t align = pci_iov_resource_size(pdev, resno);
 	struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h
index 42075501663b7..3ac718d471c20 100644
--- a/arch/powerpc/platforms/powernv/pci.h
+++ b/arch/powerpc/platforms/powernv/pci.h
@@ -251,13 +251,14 @@ struct pnv_iov_data {
 	struct resource holes[PCI_SRIOV_NUM_BARS];
 };
 
-static inline struct pnv_iov_data *pnv_iov_get(struct pci_dev *pdev)
+static inline struct pnv_iov_data *pnv_iov_get(const struct pci_dev *pdev)
 {
 	return pdev->dev.archdata.iov_data;
 }
 
 void pnv_pci_ioda_fixup_iov(struct pci_dev *pdev);
-resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev, int resno);
+resource_size_t pnv_pci_iov_resource_alignment(const struct pci_dev *pdev,
+					       int resno);
 
 int pnv_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs);
 int pnv_pcibios_sriov_disable(struct pci_dev *pdev);
diff --git a/arch/powerpc/platforms/pseries/hotplug-memory.c b/arch/powerpc/platforms/pseries/hotplug-memory.c
index b2f14db59034c..94f3b57054b60 100644
--- a/arch/powerpc/platforms/pseries/hotplug-memory.c
+++ b/arch/powerpc/platforms/pseries/hotplug-memory.c
@@ -164,13 +164,7 @@ static int update_lmb_associativity_index(struct drmem_lmb *lmb)
 
 static struct memory_block *lmb_to_memblock(struct drmem_lmb *lmb)
 {
-	unsigned long section_nr;
-	struct memory_block *mem_block;
-
-	section_nr = pfn_to_section_nr(PFN_DOWN(lmb->base_addr));
-
-	mem_block = find_memory_block(section_nr);
-	return mem_block;
+	return memory_block_get(phys_to_block_id(lmb->base_addr));
 }
 
 static int get_lmb_range(u32 drc_index, int n_lmbs,
@@ -213,14 +207,14 @@ static int dlpar_change_lmb_state(struct drmem_lmb *lmb, bool online)
 		return -EINVAL;
 	}
 
-	if (online && mem_block->dev.offline)
+	if (online && dev_offline(&mem_block->dev))
 		rc = device_online(&mem_block->dev);
-	else if (!online && !mem_block->dev.offline)
+	else if (!online && !dev_offline(&mem_block->dev))
 		rc = device_offline(&mem_block->dev);
 	else
 		rc = 0;
 
-	put_device(&mem_block->dev);
+	memory_block_put(mem_block);
 
 	return rc;
 }
@@ -319,12 +313,12 @@ static int dlpar_remove_lmb(struct drmem_lmb *lmb)
 
 	rc = dlpar_offline_lmb(lmb);
 	if (rc) {
-		put_device(&mem_block->dev);
+		memory_block_put(mem_block);
 		return rc;
 	}
 
 	__remove_memory(lmb->base_addr, memory_block_size);
-	put_device(&mem_block->dev);
+	memory_block_put(mem_block);
 
 	/* Update memory regions for memory remove */
 	memblock_remove(lmb->base_addr, memory_block_size);
diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c
index 50b26ed8432d5..1223dc9612429 100644
--- a/arch/powerpc/platforms/pseries/setup.c
+++ b/arch/powerpc/platforms/pseries/setup.c
@@ -658,7 +658,8 @@ enum get_iov_fw_value_index {
 	WDW_SIZE      = 3     /*  Get Window Size */
 };
 
-static resource_size_t pseries_get_iov_fw_value(struct pci_dev *dev, int resno,
+static resource_size_t pseries_get_iov_fw_value(const struct pci_dev *dev,
+						int resno,
 						enum get_iov_fw_value_index value)
 {
 	const int *indexes;
@@ -788,7 +789,7 @@ static void pseries_pci_fixup_iov_resources(struct pci_dev *pdev)
 		pseries_disable_sriov_resources(pdev);
 }
 
-static resource_size_t pseries_pci_iov_resource_alignment(struct pci_dev *pdev,
+static resource_size_t pseries_pci_iov_resource_alignment(const struct pci_dev *pdev,
 							  int resno)
 {
 	const __be32 *reg;
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index c5754942cf85a..0d10b299bad83 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -404,9 +404,6 @@ config ARCH_RV32I
 	bool "RV32I"
 	depends on NONPORTABLE
 	select 32BIT
-	select GENERIC_LIB_ASHLDI3
-	select GENERIC_LIB_ASHRDI3
-	select GENERIC_LIB_LSHRDI3
 	select GENERIC_LIB_UCMPDI2
 
 config ARCH_RV64I
diff --git a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi
index 82cc85acccb13..a810ad3eb2a2b 100644
--- a/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi
+++ b/arch/riscv/boot/dts/allwinner/sunxi-d1s-t113.dtsi
@@ -295,6 +295,15 @@ timer: timer@2050000 {
 			clocks = <&dcxo>;
 		};
 
+		hstimer@3008000 {
+			compatible = "allwinner,sun20i-d1-hstimer";
+			reg = <0x03008000 0x1000>;
+			interrupts = <SOC_PERIPHERAL_IRQ(55) IRQ_TYPE_LEVEL_HIGH>,
+				     <SOC_PERIPHERAL_IRQ(56) IRQ_TYPE_LEVEL_HIGH>;
+			clocks = <&ccu CLK_BUS_HSTIMER>;
+			resets = <&ccu RST_BUS_HSTIMER>;
+		};
+
 		wdt: watchdog@20500a0 {
 			compatible = "allwinner,sun20i-d1-wdt-reset",
 				     "allwinner,sun20i-d1-wdt";
diff --git a/arch/riscv/boot/dts/canaan/Makefile b/arch/riscv/boot/dts/canaan/Makefile
index 987d1f0c41f0e..7d54ea5c6f3d2 100644
--- a/arch/riscv/boot/dts/canaan/Makefile
+++ b/arch/riscv/boot/dts/canaan/Makefile
@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: GPL-2.0
 dtb-$(CONFIG_ARCH_CANAAN) += canaan_kd233.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += k210_generic.dtb
+dtb-$(CONFIG_ARCH_CANAAN) += k230-canmv.dtb
+dtb-$(CONFIG_ARCH_CANAAN) += k230-evb.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_bit.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_dock.dtb
 dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_go.dtb
diff --git a/arch/riscv/boot/dts/canaan/k230-canmv.dts b/arch/riscv/boot/dts/canaan/k230-canmv.dts
new file mode 100644
index 0000000000000..32ba53d17c000
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230-canmv.dts
@@ -0,0 +1,332 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name>
+ */
+
+#include "k230.dtsi"
+
+/ {
+	model = "Canaan CanMV-K230";
+	compatible = "canaan,canmv-k230", "canaan,kendryte-k230";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	ddr: memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x20000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
+
+&pinctrl {
+	jtag_pins: jtag-pins {
+		jtag-tck-cfg {
+			pinmux = <K230_PINMUX(2, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-pull-down;
+			input-schmitt-enable;
+		};
+
+		jtag-tdi-cfg {
+			pinmux = <K230_PINMUX(3, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+		};
+
+		jtag-tdo-cfg {
+			pinmux = <K230_PINMUX(4, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+
+		jtag-tms-cfg {
+			pinmux = <K230_PINMUX(5, 1)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-pull-up;
+		};
+	};
+
+	uart2_pins: uart2-pins {
+		uart2-pins-cfg {
+			pinmux = <K230_PINMUX(5, 3)>, /* uart2 txd */
+				 <K230_PINMUX(6, 3)>; /* uart2 rxd */
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+		};
+	};
+
+	pwm2_pins: pwm2-pins {
+		pwm2-pin-cfg {
+			pinmux = <K230_PINMUX(7, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	pwm3_pins: pwm3-pins {
+		pwm3-pin-cfg {
+			pinmux = <K230_PINMUX(8, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	pwm4_pins: pwm4-pins {
+		pwm4-pin-cfg {
+			pinmux = <K230_PINMUX(9, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	iis_pins: iis-pins {
+		iis-clk-cfg {
+			pinmux = <K230_PINMUX(32, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+
+		iis-ws-cfg {
+			pinmux = <K230_PINMUX(33, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+
+		iis-din0-cfg {
+			pinmux = <K230_PINMUX(34, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+		};
+
+		iis-dout0-cfg {
+			pinmux = <K230_PINMUX(35, 2)>;
+			slew-rate = <0>;
+			drive-strength = <4>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+		};
+	};
+
+	uart4_pins: uart4-pins {
+		uart4-txd-cfg {
+			pinmux = <K230_PINMUX(36, 4)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		uart4-rxd-cfg {
+			pinmux = <K230_PINMUX(37, 4)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	uart0_pins: uart0-pins {
+		uart0-txd-cfg {
+			pinmux = <K230_PINMUX(38, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		uart0-rxd-cfg {
+			pinmux = <K230_PINMUX(39, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	iic1_pins: iic1-pins {
+		iic1-pins-cfg {
+			pinmux = <K230_PINMUX(40, 2)>, /* iic1 scl */
+				 <K230_PINMUX(41, 2)>; /* iic1 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	iic3_pins: iic3-pins {
+		iic3-pins-cfg {
+			pinmux = <K230_PINMUX(44, 2)>, /* iic3 scl */
+				 <K230_PINMUX(45, 2)>; /* iic3 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	iic4_pins: iic4-pins {
+		iic4-pins-cfg {
+			pinmux = <K230_PINMUX(46, 3)>, /* iic4 scl */
+				 <K230_PINMUX(47, 3)>; /* iic4 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	iic0_pins: iic0-pins {
+		iic0-pins-cfg {
+			pinmux = <K230_PINMUX(48, 3)>, /* iic0 scl */
+				 <K230_PINMUX(49, 3)>; /* iic0 sda */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_1V8>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+
+	uart3_pins: uart3-pins {
+		uart3-txd-cfg {
+			pinmux = <K230_PINMUX(50, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		uart3-rxd-cfg {
+			pinmux = <K230_PINMUX(51, 1)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	key_pins: key-pins {
+		key-pins-cfg {
+			pinmux = <K230_PINMUX(52, 0)>, /* key0 */
+				 <K230_PINMUX(53, 0)>; /* key1 */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+	};
+
+	mmc1_pins: mmc1-pins {
+		mmc1-cmd-cfg {
+			pinmux = <K230_PINMUX(54, 2)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+
+		mmc1-clk-cfg {
+			pinmux = <K230_PINMUX(55, 2)>;
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			output-enable;
+			bias-disable;
+			input-schmitt-enable;
+		};
+
+		mmc1-data-cfg {
+			pinmux = <K230_PINMUX(56, 2)>, /* mmc1 data0 */
+				 <K230_PINMUX(57, 2)>, /* mmc1 data1 */
+				 <K230_PINMUX(58, 2)>, /* mmc1 data2 */
+				 <K230_PINMUX(59, 2)>; /* mmc1 data3 */
+			slew-rate = <0>;
+			drive-strength = <7>;
+			power-source = <K230_MSC_3V3>;
+			input-enable;
+			output-enable;
+			bias-pull-up;
+			input-schmitt-enable;
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/canaan/k230-evb.dts b/arch/riscv/boot/dts/canaan/k230-evb.dts
new file mode 100644
index 0000000000000..bfa53f2e24033
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230-evb.dts
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name>
+ */
+
+#include "k230.dtsi"
+
+/ {
+	model = "Kendryte K230 EVB";
+	compatible = "canaan,k230-usip-lp3-evb", "canaan,kendryte-k230";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0:115200n8";
+	};
+
+	ddr: memory@0 {
+		device_type = "memory";
+		reg = <0x0 0x0 0x0 0x20000000>;
+	};
+};
+
+&uart0 {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/canaan/k230-pinctrl.h b/arch/riscv/boot/dts/canaan/k230-pinctrl.h
new file mode 100644
index 0000000000000..63dd999ca55b6
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230-pinctrl.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
+/*
+ * Copyright (C) 2024 Canaan Bright Sight Co. Ltd
+ * Copyright (C) 2024 Ze Huang <18771902331@163.com>
+ */
+
+#ifndef _K230_PINCTRL_H
+#define _K230_PINCTRL_H
+
+#define K230_MSC_3V3 0
+#define K230_MSC_1V8 1
+
+#define BANK_VOLTAGE_DEFAULT       K230_MSC_1V8
+#define BANK_VOLTAGE_IO50_IO61     K230_MSC_3V3
+
+#define K230_PINMUX(pin, mode) (((pin) << 8) | (mode))
+
+#endif /* _K230_PINCTRL_H */
diff --git a/arch/riscv/boot/dts/canaan/k230.dtsi b/arch/riscv/boot/dts/canaan/k230.dtsi
new file mode 100644
index 0000000000000..a73b1926ab934
--- /dev/null
+++ b/arch/riscv/boot/dts/canaan/k230.dtsi
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/*
+ * Copyright (C) 2024 Yangyu Chen <cyy@cyyself.name>
+ */
+
+#include <dt-bindings/interrupt-controller/irq.h>
+#include <dt-bindings/reset/canaan,k230-rst.h>
+#include "k230-pinctrl.h"
+
+/dts-v1/;
+/ {
+	#address-cells = <2>;
+	#size-cells = <2>;
+	compatible = "canaan,kendryte-k230";
+
+	cpus {
+		#address-cells = <1>;
+		#size-cells = <0>;
+		timebase-frequency = <27000000>;
+
+		cpu@0 {
+			compatible = "thead,c908", "riscv";
+			device_type = "cpu";
+			reg = <0>;
+			riscv,isa = "rv64imafdcbv_zicbom_zicbop_zicboz_zicntr_zicsr_zifencei_zihpm_zfh_zba_zbb_zbc_zbs_zvfh_svpbmt";
+			riscv,isa-base = "rv64i";
+			riscv,isa-extensions = "i", "m", "a", "f", "d", "c", "b", "v", "zicbom",
+					       "zicbop", "zicboz", "zicntr", "zicsr", "zifencei",
+					       "zihpm", "zfh", "zba", "zbb", "zbc", "zbs", "zvfh",
+					       "svpbmt";
+			riscv,cbom-block-size = <64>;
+			riscv,cbop-block-size = <64>;
+			riscv,cboz-block-size = <64>;
+			d-cache-block-size = <64>;
+			d-cache-sets = <128>;
+			d-cache-size = <32768>;
+			i-cache-block-size = <64>;
+			i-cache-sets = <128>;
+			i-cache-size = <32768>;
+			next-level-cache = <&l2_cache>;
+			mmu-type = "riscv,sv39";
+
+			cpu0_intc: interrupt-controller {
+				compatible = "riscv,cpu-intc";
+				interrupt-controller;
+				#interrupt-cells = <1>;
+			};
+		};
+
+		l2_cache: l2-cache {
+			compatible = "cache";
+			cache-block-size = <64>;
+			cache-level = <2>;
+			cache-size = <262144>;
+			cache-sets = <256>;
+			cache-unified;
+		};
+	};
+
+	apb_clk: apb-clk-clock {
+		compatible = "fixed-clock";
+		clock-frequency = <50000000>;
+		clock-output-names = "apb_clk";
+		#clock-cells = <0>;
+	};
+
+	soc {
+		compatible = "simple-bus";
+		interrupt-parent = <&plic>;
+		#address-cells = <2>;
+		#size-cells = <2>;
+		dma-noncoherent;
+		ranges;
+
+		plic: interrupt-controller@f00000000 {
+			compatible = "canaan,k230-plic", "thead,c900-plic";
+			reg = <0xf 0x00000000 0x0 0x04000000>;
+			interrupts-extended = <&cpu0_intc 11>, <&cpu0_intc 9>;
+			interrupt-controller;
+			#address-cells = <0>;
+			#interrupt-cells = <2>;
+			riscv,ndev = <208>;
+		};
+
+		clint: timer@f04000000 {
+			compatible = "canaan,k230-clint", "thead,c900-clint";
+			reg = <0xf 0x04000000 0x0 0x00010000>;
+			interrupts-extended = <&cpu0_intc 3>, <&cpu0_intc 7>;
+		};
+
+		rst: reset-controller@91101000 {
+			compatible = "canaan,k230-rst";
+			reg = <0x0 0x91101000 0x0 0x1000>;
+			#reset-cells = <1>;
+		};
+
+		pinctrl: pinctrl@91105000 {
+			compatible = "canaan,k230-pinctrl";
+			reg = <0x0 0x91105000 0x0 0x100>;
+		};
+
+		uart0: serial@91400000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91400000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <16 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART0>;
+			status = "disabled";
+		};
+
+		uart1: serial@91401000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91401000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <17 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART1>;
+			status = "disabled";
+		};
+
+		uart2: serial@91402000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91402000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <18 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART2>;
+			status = "disabled";
+		};
+
+		uart3: serial@91403000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91403000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <19 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART3>;
+			status = "disabled";
+		};
+
+		uart4: serial@91404000 {
+			compatible = "snps,dw-apb-uart";
+			reg = <0x0 0x91404000 0x0 0x1000>;
+			clocks = <&apb_clk>;
+			interrupts = <20 IRQ_TYPE_LEVEL_HIGH>;
+			reg-io-width = <4>;
+			reg-shift = <2>;
+			resets = <&rst RST_UART4>;
+			status = "disabled";
+		};
+	};
+};
diff --git a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
index 0e1b0b8d394b9..6d7037c9528dd 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-beaglev-fire.dts
@@ -4,6 +4,7 @@
 /dts-v1/;
 
 #include <dt-bindings/gpio/gpio.h>
+#include <dt-bindings/interrupt-controller/irq.h>
 #include "mpfs.dtsi"
 #include "mpfs-beaglev-fire-fabric.dtsi"
 
@@ -63,20 +64,29 @@ imx219_vana: fixedregulator-0 {
 		regulator-max-microvolt = <2800000>;
 	};
 
-	imx219_vdig: fixedregulator-1 {
+	imx219_vdig: regulator-1v8 {
 		compatible = "regulator-fixed";
 		regulator-name = "imx219_vdig";
 		regulator-min-microvolt = <1800000>;
 		regulator-max-microvolt = <1800000>;
 	};
 
-	imx219_vddl: fixedregulator-2 {
+	imx219_vddl: regulator-1v2 {
 		compatible = "regulator-fixed";
 		regulator-name = "imx219_vddl";
 		regulator-min-microvolt = <1200000>;
 		regulator-max-microvolt = <1200000>;
 	};
 
+	regulator-1v8-syzygy {
+		compatible = "regulator-fixed";
+		gpios = <&gpio2 30 GPIO_ACTIVE_HIGH>;
+		regulator-name = "syzygy_1v8";
+		regulator-min-microvolt = <1800000>;
+		regulator-max-microvolt = <1800000>;
+		regulator-always-on;
+		enable-active-high;
+	};
 };
 
 &gpio0 {
@@ -85,6 +95,8 @@ &gpio0 {
 		     <21>, <22>, <23>, <24>,
 		     <25>, <26>;
 	ngpios = <14>;
+	gpio-line-names = "", "", "", "", "", "", "",
+			  "", "", "", "", "", "SD_CARD_CS", "USER_BUTTON";
 	status = "okay";
 };
 
@@ -96,6 +108,9 @@ &gpio1 {
 		     <43>, <44>, <45>, <46>,
 		     <47>, <48>, <49>, <50>;
 	ngpios = <24>;
+	gpio-line-names = "", "", "", "", "", "", "", "", "", "",
+			  "", "", "", "", "", "", "", "", "", "",
+			  "ADC_IRQn", "", "", "USB_OCn";
 	status = "okay";
 };
 
@@ -118,20 +133,6 @@ &gpio2 {
 			  "P8_PIN27", "P8_PIN28", "P8_PIN29", "P8_PIN30", "M2_W_DISABLE1",
 			  "M2_W_DISABLE2", "VIO_ENABLE", "SD_DET";
 	status = "okay";
-
-	vio-enable-hog {
-		gpio-hog;
-		gpios = <30 30>;
-		output-high;
-		line-name = "VIO_ENABLE";
-	};
-
-	sd-det-hog {
-		gpio-hog;
-		gpios = <31 31>;
-		input;
-		line-name = "SD_DET";
-	};
 };
 
 &i2c0 {
@@ -260,6 +261,7 @@ adc@0 {
 		spi-cpha;
 		spi-max-frequency = <5000000>;
 		microchip,hw-device-address = <1>;
+		interrupts-extended = <&gpio1 20 IRQ_TYPE_LEVEL_LOW>;
 		#address-cells = <1>;
 		#size-cells = <0>;
 		status = "okay";
@@ -316,7 +318,7 @@ channel@7 {
 	mmc@1 {
 		compatible = "mmc-spi-slot";
 		reg = <1>;
-		gpios = <&gpio2 31 1>;
+		cd-gpios = <&gpio2 31 GPIO_ACTIVE_LOW>;
 		voltage-ranges = <3300 3300>;
 		spi-max-frequency = <5000000>;
 		disable-wp;
@@ -326,7 +328,6 @@ mmc@1 {
 
 &syscontroller {
 	microchip,bitstream-flash = <&sys_ctrl_flash>;
-	status = "okay";
 };
 
 &syscontroller_qspi {
diff --git a/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts
index f769c9d5d7b47..da2fcde263e3c 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-disco-kit.dts
@@ -199,7 +199,3 @@ &spi0 {
 &spi1 {
 	status = "okay";
 };
-
-&syscontroller {
-	status = "okay";
-};
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi
index 7816408343a32..a11e3103b8f25 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-common.dtsi
@@ -238,10 +238,6 @@ &spi1 {
 	status = "okay";
 };
 
-&syscontroller {
-	status = "okay";
-};
-
 &syscontroller_qspi {
 	/*
 	 * The flash *is* there, but Icicle kits that have engineering sample
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
index 2d14e92f068d5..9078e5b1e49c1 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-fabric.dtsi
@@ -101,16 +101,6 @@ &ccc_nw {
 	status = "okay";
 };
 
-&i2c0 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&i2c0_fabric>;
-};
-
-&i2c1 {
-	pinctrl-names = "default";
-	pinctrl-0 = <&i2c1_mssio>;
-};
-
 &mmuart1 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart1_fabric>;
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts
index 8afedece89d1f..636493f6584d2 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit-prod.dts
@@ -14,6 +14,16 @@ / {
 		     "microchip,mpfs";
 };
 
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_fabric>;
+};
+
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_mssio>;
+};
+
 &syscontroller {
 	microchip,bitstream-flash = <&sys_ctrl_flash>;
 };
diff --git a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts
index 556aa9638282e..6fadce815c9a2 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-icicle-kit.dts
@@ -11,3 +11,22 @@ / {
 		     "microchip,mpfs-icicle-kit",
 		     "microchip,mpfs";
 };
+
+&i2c0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c0_fabric>;
+};
+
+/*
+ * Due to silicon errata, routing via MSS IOs doesn't work on ES devices.
+ * Instead, i2c1, appearing on B1/C1, which are normally MSS IOs, is routed
+ * via the fabric and back to B1/C1 via "fabric-test" functionality.
+ * This is done silently by Libero, so the iomux0 setting for i2c1 has to
+ * be fabric IO, despite tooling etc saying that MSS IOs are used.
+ *
+ * See Section 3.3 of https://ww1.microchip.com/downloads/aemDocuments/documents/FPGA/ProductDocuments/Errata/polarfiresoc/microsemi_polarfire_soc_fpga_egineering_samples_errata_er0219_v1.pdf
+ */
+&i2c1 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c1_fabric>;
+};
diff --git a/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts b/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts
index 86234968df486..973e34b74be44 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-m100pfsevp.dts
@@ -179,10 +179,6 @@ &spi1 {
 	status = "okay";
 };
 
-&syscontroller {
-	status = "okay";
-};
-
 &usb {
 	status = "okay";
 	dr_mode = "host";
diff --git a/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts b/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts
index 510d59153cd07..cad0871a62a5d 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-polarberry.dts
@@ -112,7 +112,3 @@ &refclk {
 &rtc {
 	status = "okay";
 };
-
-&syscontroller {
-	status = "okay";
-};
diff --git a/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts b/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts
index 8f1908a105671..46ded8a4fc363 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-sev-kit.dts
@@ -149,10 +149,6 @@ &rtc {
 	status = "okay";
 };
 
-&syscontroller {
-	status = "okay";
-};
-
 &usb {
 	status = "okay";
 	dr_mode = "otg";
diff --git a/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts b/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts
index bc15530a2979b..c6950410e918a 100644
--- a/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts
+++ b/arch/riscv/boot/dts/microchip/mpfs-tysom-m.dts
@@ -171,10 +171,6 @@ flash@0 {
 	};
 };
 
-&syscontroller {
-	status = "okay";
-};
-
 &usb {
 	status = "okay";
 	dr_mode = "host";
diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi
index d535d4c72763c..85d8df6437f23 100644
--- a/arch/riscv/boot/dts/microchip/mpfs.dtsi
+++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi
@@ -499,7 +499,7 @@ gpio0: gpio@20120000 {
 			reg = <0x0 0x20120000 0x0 0x1000>;
 			interrupt-parent = <&irqmux>;
 			interrupt-controller;
-			#interrupt-cells = <1>;
+			#interrupt-cells = <2>;
 			interrupts = <0>, <1>, <2>, <3>,
 				     <4>, <5>, <6>, <7>,
 				     <8>, <9>, <10>, <11>,
@@ -516,7 +516,7 @@ gpio1: gpio@20121000 {
 			reg = <0x0 0x20121000 0x0 0x1000>;
 			interrupt-parent = <&irqmux>;
 			interrupt-controller;
-			#interrupt-cells = <1>;
+			#interrupt-cells = <2>;
 			interrupts = <32>, <33>, <34>, <35>,
 				     <36>, <37>, <38>, <39>,
 				     <40>, <41>, <42>, <43>,
@@ -535,7 +535,7 @@ gpio2: gpio@20122000 {
 			reg = <0x0 0x20122000 0x0 0x1000>;
 			interrupt-parent = <&irqmux>;
 			interrupt-controller;
-			#interrupt-cells = <1>;
+			#interrupt-cells = <2>;
 			interrupts = <64>, <65>, <66>, <67>,
 				     <68>, <69>, <70>, <71>,
 				     <72>, <73>, <74>, <75>,
diff --git a/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts
index 2f2ccd77af30a..180fd8891903c 100644
--- a/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts
+++ b/arch/riscv/boot/dts/microchip/pic64gx-curiosity-kit.dts
@@ -63,10 +63,6 @@ hss: hss-buffer@bfc00000 {
 };
 
 &gpio0 {
-	interrupts = <13>, <14>, <15>, <16>,
-		     <17>, <18>, <19>, <20>,
-		     <21>, <22>, <23>, <24>,
-		     <25>, <26>;
 	status ="okay";
 	gpio-line-names =
 		"", "", "", "", "", "", "", "",
@@ -74,12 +70,6 @@ &gpio0 {
 };
 
 &gpio1 {
-	interrupts = <27>, <28>, <29>, <30>,
-		     <31>, <32>, <33>, <34>,
-		     <35>, <36>, <37>, <38>,
-		     <39>, <40>, <41>, <42>,
-		     <43>, <44>, <45>, <46>,
-		     <47>, <48>, <49>, <50>;
 	status ="okay";
 	gpio-line-names =
 		"", "", "LED1", "LED2", "LED3", "LED4", "LED5", "LED6",
@@ -88,14 +78,6 @@ &gpio1 {
 };
 
 &gpio2 {
-	interrupts = <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>,
-		     <53>, <53>, <53>, <53>;
 	pinctrl-names = "default";
 	pinctrl-0 = <&mdio1_gpio>, <&spi0_gpio>, <&can0_gpio>, <&pcie_gpio>,
 		    <&qspi_gpio>, <&uart3_gpio>, <&uart4_gpio>, <&can1_gpio>;
@@ -107,6 +89,43 @@ &gpio2 {
 		"DIP4", "USR_IO11", "", "", "SWITCH1", "", "", "";
 };
 
+&i2c0 {
+	status = "okay";
+};
+
+&i2c1 {
+	status = "okay";
+};
+
+&irqmux {
+	interrupt-map = <0 &plic 13>, <1 &plic 14>, <2 &plic 15>,
+			<3 &plic 16>, <4 &plic 17>, <5 &plic 18>,
+			<6 &plic 19>, <7 &plic 20>, <8 &plic 21>,
+			<9 &plic 22>, <10 &plic 23>, <11 &plic 24>,
+			<12 &plic 25>, <13 &plic 26>,
+
+			<32 &plic 27>, <33 &plic 28>, <34 &plic 29>,
+			<35 &plic 30>, <36 &plic 31>, <37 &plic 32>,
+			<38 &plic 33>, <39 &plic 34>, <40 &plic 35>,
+			<41 &plic 36>, <42 &plic 37>, <43 &plic 38>,
+			<44 &plic 39>, <45 &plic 40>, <46 &plic 41>,
+			<47 &plic 42>, <48 &plic 43>, <49 &plic 44>,
+			<50 &plic 45>, <51 &plic 46>, <52 &plic 47>,
+			<53 &plic 48>, <54 &plic 49>, <55 &plic 50>,
+
+			<64 &plic 53>, <65 &plic 53>, <66 &plic 53>,
+			<67 &plic 53>, <68 &plic 53>, <69 &plic 53>,
+			<70 &plic 53>, <71 &plic 53>, <72 &plic 53>,
+			<73 &plic 53>, <74 &plic 53>, <75 &plic 53>,
+			<76 &plic 53>, <77 &plic 53>, <78 &plic 53>,
+			<79 &plic 53>, <80 &plic 53>, <81 &plic 53>,
+			<82 &plic 53>, <83 &plic 53>, <84 &plic 53>,
+			<85 &plic 53>, <86 &plic 53>, <87 &plic 53>,
+			<88 &plic 53>, <89 &plic 53>, <90 &plic 53>,
+			<91 &plic 53>, <92 &plic 53>, <93 &plic 53>,
+			<94 &plic 53>, <95 &plic 53>;
+};
+
 &mac0 {
 	status = "okay";
 	phy-mode = "sgmii";
@@ -123,14 +142,6 @@ &mbox {
 	status = "okay";
 };
 
-&i2c0 {
-	status = "okay";
-};
-
-&i2c1 {
-	status = "okay";
-};
-
 &mmc {
 	bus-width = <4>;
 	disable-wp;
@@ -159,7 +170,3 @@ &refclk {
 &rtc {
 	status = "okay";
 };
-
-&syscontroller {
-	status = "okay";
-};
diff --git a/arch/riscv/boot/dts/microchip/pic64gx.dtsi b/arch/riscv/boot/dts/microchip/pic64gx.dtsi
index c164d7bc270a2..5addfd4357117 100644
--- a/arch/riscv/boot/dts/microchip/pic64gx.dtsi
+++ b/arch/riscv/boot/dts/microchip/pic64gx.dtsi
@@ -295,6 +295,14 @@ mss_top_sysreg: syscon@20002000 {
 			#size-cells = <1>;
 			#reset-cells = <1>;
 
+			irqmux: interrupt-controller@54 {
+				compatible = "microchip,pic64gx-irqmux", "microchip,mpfs-irqmux";
+				reg = <0x54 0x4>;
+				#address-cells = <0>;
+				#interrupt-cells = <1>;
+				interrupt-map-mask = <0x7f>;
+			};
+
 			iomux0: pinctrl@200 {
 				compatible = "microchip,pic64gx-pinctrl-iomux0",
 					     "microchip,mpfs-pinctrl-iomux0";
@@ -459,8 +467,8 @@ mac0: ethernet@20110000 {
 			interrupts = <64>, <65>, <66>, <67>, <68>, <69>;
 			/* Filled in by a bootloader */
 			local-mac-address = [00 00 00 00 00 00];
-			clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>;
-			clock-names = "pclk", "hclk";
+			clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>, <&refclk>;
+			clock-names = "pclk", "hclk", "tsu_clk";
 			resets = <&mss_top_sysreg CLK_MAC0>;
 			status = "disabled";
 		};
@@ -475,8 +483,8 @@ mac1: ethernet@20112000 {
 			interrupts = <70>, <71>, <72>, <73>, <74>, <75>;
 			/* Filled in by a bootloader */
 			local-mac-address = [00 00 00 00 00 00];
-			clocks = <&clkcfg CLK_MAC1>, <&clkcfg CLK_AHB>;
-			clock-names = "pclk", "hclk";
+			clocks = <&clkcfg CLK_MAC0>, <&clkcfg CLK_AHB>, <&refclk>;
+			clock-names = "pclk", "hclk", "tsu_clk";
 			resets = <&mss_top_sysreg CLK_MAC1>;
 			status = "disabled";
 		};
@@ -484,9 +492,13 @@ mac1: ethernet@20112000 {
 		gpio0: gpio@20120000 {
 			compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio";
 			reg = <0x0 0x20120000 0x0 0x1000>;
-			interrupt-parent = <&plic>;
+			interrupt-parent = <&irqmux>;
 			interrupt-controller;
-			#interrupt-cells = <1>;
+			#interrupt-cells = <2>;
+			interrupts = <0>, <1>, <2>, <3>,
+				     <4>, <5>, <6>, <7>,
+				     <8>, <9>, <10>, <11>,
+				     <12>, <13>;
 			clocks = <&clkcfg CLK_GPIO0>;
 			gpio-controller;
 			#gpio-cells = <2>;
@@ -497,9 +509,15 @@ gpio0: gpio@20120000 {
 		gpio1: gpio@20121000 {
 			compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio";
 			reg = <0x0 0x20121000 0x0 0x1000>;
-			interrupt-parent = <&plic>;
+			interrupt-parent = <&irqmux>;
 			interrupt-controller;
-			#interrupt-cells = <1>;
+			#interrupt-cells = <2>;
+			interrupts = <32>, <33>, <34>, <35>,
+				     <36>, <37>, <38>, <39>,
+				     <40>, <41>, <42>, <43>,
+				     <44>, <45>, <46>, <47>,
+				     <48>, <49>, <50>, <51>,
+				     <52>, <53>, <54>, <55>;
 			clocks = <&clkcfg CLK_GPIO1>;
 			gpio-controller;
 			#gpio-cells = <2>;
@@ -510,9 +528,17 @@ gpio1: gpio@20121000 {
 		gpio2: gpio@20122000 {
 			compatible = "microchip,pic64gx-gpio", "microchip,mpfs-gpio";
 			reg = <0x0 0x20122000 0x0 0x1000>;
-			interrupt-parent = <&plic>;
+			interrupt-parent = <&irqmux>;
 			interrupt-controller;
-			#interrupt-cells = <1>;
+			#interrupt-cells = <2>;
+			interrupts = <64>, <65>, <66>, <67>,
+				     <68>, <69>, <70>, <71>,
+				     <72>, <73>, <74>, <75>,
+				     <76>, <77>, <78>, <79>,
+				     <80>, <81>, <82>, <83>,
+				     <84>, <85>, <86>, <87>,
+				     <88>, <89>, <90>, <91>,
+				     <92>, <93>, <94>, <95>;
 			clocks = <&clkcfg CLK_GPIO2>;
 			gpio-controller;
 			#gpio-cells = <2>;
diff --git a/arch/riscv/boot/dts/sophgo/cv180x.dtsi b/arch/riscv/boot/dts/sophgo/cv180x.dtsi
index 06b0ce5a2db7a..8446b4dfe1a1d 100644
--- a/arch/riscv/boot/dts/sophgo/cv180x.dtsi
+++ b/arch/riscv/boot/dts/sophgo/cv180x.dtsi
@@ -417,7 +417,7 @@ sdhci1: mmc@4320000 {
 		};
 
 		dmac: dma-controller@4330000 {
-			compatible = "snps,axi-dma-1.01a";
+			compatible = "sophgo,cv1800b-axi-dma";
 			reg = <0x04330000 0x1000>;
 			interrupts = <SOC_PERIPHERAL_IRQ(13) IRQ_TYPE_LEVEL_HIGH>;
 			clocks = <&clk CLK_SDMA_AXI>, <&clk CLK_SDMA_AXI>;
diff --git a/arch/riscv/boot/dts/spacemit/Makefile b/arch/riscv/boot/dts/spacemit/Makefile
index 7e2b877025718..acb993c452ba2 100644
--- a/arch/riscv/boot/dts/spacemit/Makefile
+++ b/arch/riscv/boot/dts/spacemit/Makefile
@@ -4,4 +4,5 @@ dtb-$(CONFIG_ARCH_SPACEMIT) += k1-milkv-jupiter.dtb
 dtb-$(CONFIG_ARCH_SPACEMIT) += k1-musepi-pro.dtb
 dtb-$(CONFIG_ARCH_SPACEMIT) += k1-orangepi-r2s.dtb
 dtb-$(CONFIG_ARCH_SPACEMIT) += k1-orangepi-rv2.dtb
+dtb-$(CONFIG_ARCH_SPACEMIT) += k3-deepcomputing-fml13v05.dtb
 dtb-$(CONFIG_ARCH_SPACEMIT) += k3-pico-itx.dtb
diff --git a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
index 5790d927b93db..72f77e9edd63d 100644
--- a/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-bananapi-f3.dts
@@ -14,12 +14,15 @@ aliases {
 		ethernet0 = &eth0;
 		ethernet1 = &eth1;
 		serial0 = &uart0;
+		spi3 = &spi3;
 		i2c2 = &i2c2;
 		i2c8 = &i2c8;
+		mmc0 = &emmc;
+		mmc1 = &sdhci0;
 	};
 
 	chosen {
-		stdout-path = "serial0";
+		stdout-path = "serial0:115200n8";
 	};
 
 	leds {
@@ -220,7 +223,7 @@ buck3_1v8: buck3 {
 				regulator-always-on;
 			};
 
-			buck4 {
+			buck4: buck4 {
 				regulator-min-microvolt = <500000>;
 				regulator-max-microvolt = <3300000>;
 				regulator-ramp-delay = <5000>;
@@ -241,7 +244,7 @@ buck6 {
 				regulator-always-on;
 			};
 
-			aldo1 {
+			aldo1: aldo1 {
 				regulator-min-microvolt = <500000>;
 				regulator-max-microvolt = <3400000>;
 				regulator-boot-on;
@@ -335,6 +338,12 @@ &pcie2 {
 	status = "okay";
 };
 
+&spi3 {
+	pinctrl-0 = <&ssp3_0_cfg>;
+	pinctrl-names = "default";
+	status = "okay";
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_2_cfg>;
@@ -367,3 +376,21 @@ hub_3_0: hub@2 {
 		reset-gpios = <&gpio K1_GPIO(124) GPIO_ACTIVE_LOW>;
 	};
 };
+
+&sdhci0 {
+	pinctrl-names = "default", "uhs";
+	pinctrl-0 = <&mmc1_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg>;
+	bus-width = <4>;
+	cd-gpios = <&gpio K1_GPIO(80) (GPIO_ACTIVE_LOW | GPIO_PULL_UP)>;
+	no-mmc;
+	no-sdio;
+	disable-wp;
+	cap-sd-highspeed;
+	vmmc-supply = <&buck4>;
+	vqmmc-supply = <&aldo1>;
+	sd-uhs-sdr25;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts
index 29e333b670cf0..4aef287353cab 100644
--- a/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-musepi-pro.dts
@@ -17,10 +17,12 @@ / {
 	aliases {
 		ethernet0 = &eth0;
 		serial0 = &uart0;
+		i2c2 = &i2c2;
+		i2c8 = &i2c8;
 	};
 
 	chosen {
-		stdout-path = "serial0";
+		stdout-path = "serial0:115200n8";
 	};
 
 	leds {
@@ -33,6 +35,52 @@ led1 {
 			default-state = "on";
 		};
 	};
+
+	reg_pcie_vcc_3v3: regulator-pcie-vcc3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "PCIE_VCC3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
+	reg_vcc_4v: regulator-vcc-4v {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC4V0";
+		regulator-min-microvolt = <4000000>;
+		regulator-max-microvolt = <4000000>;
+		regulator-boot-on;
+		regulator-always-on;
+	};
+
+	reg_qspi_vcc1v833: regulator-qspi-vcc1v833 {
+		compatible = "regulator-fixed";
+		regulator-name = "QSPI_VCC1833";
+		regulator-min-microvolt = <1833000>;
+		regulator-max-microvolt = <1833000>;
+		regulator-always-on;
+		vin-supply = <&buck4_3v3>;
+	};
+
+	reg_5v_vbus: regulator-5v-vbus {
+		compatible = "regulator-fixed";
+		regulator-name = "5V_VBUS";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		gpio = <&gpio K1_GPIO(79) GPIO_ACTIVE_HIGH>; /* USB3_PWREN */
+		enable-active-high;
+	};
+
+	reg_vcc5v_hub: regulator-vcc5v-hub {
+		compatible = "regulator-fixed";
+		regulator-name = "VCC5V0_HUB";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		regulator-always-on;
+		gpio = <&gpio K1_GPIO(127) GPIO_ACTIVE_HIGH>; /* HUB_PWREN */
+		enable-active-high;
+	};
 };
 
 &emmc {
@@ -45,6 +93,10 @@ &emmc {
 	status = "okay";
 };
 
+&combo_phy {
+	status = "okay";
+};
+
 &eth0 {
 	phy-handle = <&rgmii0>;
 	phy-mode = "rgmii-id";
@@ -72,8 +124,263 @@ &pdma {
 	status = "okay";
 };
 
+&i2c2 {
+	pinctrl-0 = <&i2c2_0_cfg>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	eeprom@50 {
+		compatible = "atmel,24c02";
+		reg = <0x50>;
+		vcc-supply = <&buck3_1v8>; /* EEPROM_VCC1V8 */
+		pagesize = <8>;
+		read-only;
+		size = <256>;
+
+		nvmem-layout {
+			compatible = "onie,tlv-layout";
+
+			product-name {
+			};
+
+			serial-number {
+			};
+
+			mac-address {
+				#nvmem-cell-cells = <1>;
+			};
+
+			num-macs {
+			};
+		};
+	};
+};
+
+&i2c8 {
+	pinctrl-0 = <&i2c8_cfg>;
+	pinctrl-names = "default";
+	status = "okay";
+
+	pmic@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		interrupts = <64>;
+		vin1-supply = <&reg_vcc_4v>;
+		vin2-supply = <&reg_vcc_4v>;
+		vin3-supply = <&reg_vcc_4v>;
+		vin4-supply = <&reg_vcc_4v>;
+		vin5-supply = <&reg_vcc_4v>;
+		vin6-supply = <&reg_vcc_4v>;
+		aldoin-supply = <&reg_vcc_4v>;
+		dldoin1-supply = <&buck5>;
+		dldoin2-supply = <&buck5>;
+
+		regulators {
+			buck1 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck2 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck3_1v8: buck3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck4_3v3: buck4 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			aldo1 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+			};
+
+			aldo2 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			aldo4 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo1 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+			};
+
+			dldo2 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo4 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-always-on;
+			};
+
+			dldo5 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-always-on;
+			};
+
+			dldo7 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+		};
+	};
+};
+
+&pcie1_phy {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie1_3_cfg>;
+	status = "okay";
+};
+
+&pcie1_port {
+	phys = <&pcie1_phy>;
+	vpcie3v3-supply = <&reg_pcie_vcc_3v3>;
+};
+
+&pcie1 {
+	status = "okay";
+};
+
+&pcie2_phy {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_4_cfg>;
+	status = "okay";
+};
+
+&pcie2_port {
+	phys = <&pcie2_phy>;
+	vpcie3v3-supply = <&reg_pcie_vcc_3v3>;
+};
+
+&pcie2 {
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
+		vcc-supply = <&reg_qspi_vcc1v833>;
+		m25p,fast-read;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			bootinfo@0 {
+				reg = <0x0 0x10000>;
+			};
+			private@10000 {
+				reg = <0x10000 0x10000>;
+			};
+			fsbl@20000 {
+				reg = <0x20000 0x40000>;
+			};
+			env@60000 {
+				reg = <0x60000 0x10000>;
+			};
+			opensbi@70000 {
+				reg = <0x70000 0x30000>;
+			};
+			uboot@a0000 {
+				reg = <0xa0000 0x760000>;
+			};
+		};
+	};
+};
+
 &uart0 {
 	pinctrl-0 = <&uart0_2_cfg>;
 	pinctrl-names = "default";
 	status = "okay";
 };
+
+&usbphy2 {
+	status = "okay";
+};
+
+&usb_dwc3 {
+	dr_mode = "host";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	vbus-supply = <&reg_5v_vbus>;
+	status = "okay";
+
+	hub_2_0: hub@1 {
+		compatible = "usb2109,2817";
+		reg = <0x1>;
+		vdd-supply = <&reg_vcc5v_hub>;
+		peer-hub = <&hub_3_0>;
+		reset-gpios = <&gpio K1_GPIO(123) GPIO_ACTIVE_LOW>; /* HUB_RST */
+	};
+
+	hub_3_0: hub@2 {
+		compatible = "usb2109,817";
+		reg = <0x2>;
+		vdd-supply = <&reg_vcc5v_hub>;
+		peer-hub = <&hub_2_0>;
+		reset-gpios = <&gpio K1_GPIO(123) GPIO_ACTIVE_LOW>; /* HUB_RST */
+	};
+};
diff --git a/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts
index 7b7331cb3c726..7c49bce427f30 100644
--- a/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts
+++ b/arch/riscv/boot/dts/spacemit/k1-orangepi-rv2.dts
@@ -17,12 +17,53 @@ aliases {
 		serial0 = &uart0;
 		ethernet0 = &eth0;
 		ethernet1 = &eth1;
+		mmc0 = &emmc;
+		mmc1 = &sdhci0;
 	};
 
 	chosen {
 		stdout-path = "serial0";
 	};
 
+	pcie_vcc3v3: regulator-pcie-vcc3v3 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio K1_GPIO(116) GPIO_ACTIVE_HIGH>;
+		regulator-name = "pcie_vcc3v3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		vin-supply = <&vcc_5v0>;
+	};
+
+	vcc_5v0: regulator-vcc-5v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc_5v0";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+	};
+
+	vcc4v0: regulator-vcc4v0 {
+		compatible = "regulator-fixed";
+		regulator-name = "vcc4v0";
+		regulator-always-on;
+		regulator-boot-on;
+		regulator-min-microvolt = <4000000>;
+		regulator-max-microvolt = <4000000>;
+		vin-supply = <&vcc_5v0>;
+	};
+
+	vcc5v0_usb30: regulator-vcc5v0-usb30 {
+		compatible = "regulator-fixed";
+		enable-active-high;
+		gpios = <&gpio K1_GPIO(123) GPIO_ACTIVE_HIGH>;
+		regulator-name = "vcc5v0_usb30";
+		regulator-min-microvolt = <5000000>;
+		regulator-max-microvolt = <5000000>;
+		vin-supply = <&vcc_5v0>;
+	};
+
 	leds {
 		compatible = "gpio-leds";
 
@@ -35,6 +76,20 @@ led1 {
 	};
 };
 
+&combo_phy {
+	status = "okay";
+};
+
+&emmc {
+	bus-width = <8>;
+	mmc-hs400-1_8v;
+	mmc-hs400-enhanced-strobe;
+	no-sd;
+	no-sdio;
+	non-removable;
+	status = "okay";
+};
+
 &eth0 {
 	phy-handle = <&rgmii0>;
 	phy-mode = "rgmii-id";
@@ -87,8 +142,245 @@ &pdma {
 	status = "okay";
 };
 
+&i2c8 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&i2c8_cfg>;
+	status = "okay";
+
+	pmic@41 {
+		compatible = "spacemit,p1";
+		reg = <0x41>;
+		interrupts = <64>;
+		vin1-supply = <&vcc4v0>;
+		vin2-supply = <&vcc4v0>;
+		vin3-supply = <&vcc4v0>;
+		vin4-supply = <&vcc4v0>;
+		vin5-supply = <&vcc4v0>;
+		vin6-supply = <&vcc4v0>;
+		aldoin-supply = <&vcc4v0>;
+		dldoin1-supply = <&buck5>;
+		dldoin2-supply = <&buck5>;
+
+		regulators {
+			buck1 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck2 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck3_1v8: buck3 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <1800000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck4_3v3: buck4 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3300000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck5: buck5 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			buck6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3450000>;
+				regulator-ramp-delay = <5000>;
+				regulator-always-on;
+			};
+
+			aldo1: aldo1 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+			};
+
+			aldo2 {
+				/* not connected */
+			};
+
+			aldo3 {
+				/* not connected */
+			};
+
+			aldo4 {
+				/* not connected */
+			};
+
+			dldo1 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-boot-on;
+			};
+
+			dldo2 {
+				/* not connected */
+			};
+
+			dldo3 {
+				/* not connected */
+			};
+
+			dldo4 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-always-on;
+			};
+
+			dldo5 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+			};
+
+			dldo6 {
+				regulator-min-microvolt = <500000>;
+				regulator-max-microvolt = <3400000>;
+				regulator-always-on;
+			};
+
+			dldo7 {
+				/* not connected */
+			};
+		};
+	};
+};
+
+&pcie1_phy {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie1_3_cfg>;
+	status = "okay";
+};
+
+&pcie1_port {
+	phys = <&pcie1_phy>;
+	vpcie3v3-supply = <&pcie_vcc3v3>;
+};
+
+&pcie1 {
+	status = "okay";
+};
+
+&pcie2_phy {
+	pinctrl-names = "default";
+	pinctrl-0 = <&pcie2_4_cfg>;
+	status = "okay";
+};
+
+&pcie2_port {
+	phys = <&pcie2_phy>;
+	vpcie3v3-supply = <&pcie_vcc3v3>;
+};
+
+&pcie2 {
+	status = "okay";
+};
+
+&qspi {
+	pinctrl-names = "default";
+	pinctrl-0 = <&qspi_cfg>;
+	status = "okay";
+
+	flash@0 {
+		compatible = "jedec,spi-nor";
+		reg = <0>;
+		spi-max-frequency = <26500000>;
+		spi-rx-bus-width = <4>;
+		spi-tx-bus-width = <4>;
+		vcc-supply = <&buck3_1v8>;
+
+		partitions {
+			compatible = "fixed-partitions";
+			#address-cells = <1>;
+			#size-cells = <1>;
+
+			bootinfo@0 {
+				reg = <0x00000 0x010000>;
+			};
+
+			private@10000 {
+				reg = <0x10000 0x010000>;
+			};
+
+			fsbl@20000 {
+				reg = <0x20000 0x040000>;
+			};
+
+			env@60000 {
+				reg = <0x60000 0x010000>;
+			};
+
+			opensbi@70000 {
+				reg = <0x70000 0x030000>;
+			};
+
+			uboot@a0000 {
+				reg = <0xa0000 0x760000>;
+			};
+		};
+	};
+};
+
 &uart0 {
 	pinctrl-names = "default";
 	pinctrl-0 = <&uart0_2_cfg>;
 	status = "okay";
 };
+
+&usbphy2 {
+	status = "okay";
+};
+
+&usb_dwc3 {
+	dr_mode = "host";
+	#address-cells = <1>;
+	#size-cells = <0>;
+	vbus-supply = <&vcc5v0_usb30>;
+	status = "okay";
+
+	hub_2_0: hub@1 {
+		compatible = "usb5e3,610";
+		reg = <0x1>;
+		peer-hub = <&hub_3_0>;
+		vdd-supply = <&vcc_5v0>;
+	};
+
+	hub_3_0: hub@2 {
+		compatible = "usb5e3,620";
+		reg = <0x2>;
+		peer-hub = <&hub_2_0>;
+		vdd-supply = <&vcc_5v0>;
+	};
+};
+
+&sdhci0 {
+	pinctrl-names = "default", "uhs";
+	pinctrl-0 = <&mmc1_cfg>;
+	pinctrl-1 = <&mmc1_uhs_cfg>;
+	bus-width = <4>;
+	cd-gpios = <&gpio K1_GPIO(80) GPIO_ACTIVE_LOW>;
+	no-mmc;
+	no-sdio;
+	disable-wp;
+	cap-sd-highspeed;
+	vmmc-supply = <&buck4_3v3>;
+	vqmmc-supply = <&aldo1>;
+	sd-uhs-sdr25;
+	sd-uhs-sdr50;
+	sd-uhs-sdr104;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi
index b13dcb10f4d66..4e9a62d0e85b5 100644
--- a/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k1-pinctrl.dtsi
@@ -570,4 +570,64 @@ pwm14-1-pins {
 			drive-strength = <32>;
 		};
 	};
+
+	ssp3_0_cfg: ssp3-0-cfg {
+		ssp3-0-pins {
+			pinmux = <K1_PADCONF(75, 2)>,	/* SCLK */
+				 <K1_PADCONF(77, 2)>,	/* MOSI  */
+				 <K1_PADCONF(78, 2)>;	/* MISO */
+
+			bias-disable;
+			drive-strength = <19>;
+			power-source = <3300>;
+		};
+
+		ssp3-0-frm-pins {
+			pinmux = <K1_PADCONF(76, 2)>;	/* FRM (frame) */
+
+			bias-pull-up = <0>;
+			drive-strength = <19>;
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_cfg: mmc1-cfg {
+		mmc1-data-cmd-pins {
+			pinmux = <K1_PADCONF(104, 0)>,   /* mmc1_d3 */
+				 <K1_PADCONF(105, 0)>,   /* mmc1_d2 */
+				 <K1_PADCONF(106, 0)>,   /* mmc1_d1 */
+				 <K1_PADCONF(107, 0)>,   /* mmc1_d0 */
+				 <K1_PADCONF(108, 0)>;   /* mmc1_cmd */
+			bias-pull-up = <1>;
+			drive-strength = <19>;
+			power-source = <3300>;
+		};
+
+		mmc1-clk-pins {
+			pinmux = <K1_PADCONF(109, 0)>;   /* mmc1_clk */
+			bias-pull-down = <1>;
+			drive-strength = <19>;
+			power-source = <3300>;
+		};
+	};
+
+	mmc1_uhs_cfg: mmc1-uhs-cfg {
+		mmc1-data-cmd-pins {
+			pinmux = <K1_PADCONF(104, 0)>,   /* mmc1_d3 */
+				 <K1_PADCONF(105, 0)>,   /* mmc1_d2 */
+				 <K1_PADCONF(106, 0)>,   /* mmc1_d1 */
+				 <K1_PADCONF(107, 0)>,   /* mmc1_d0 */
+				 <K1_PADCONF(108, 0)>;   /* mmc1_cmd */
+			bias-pull-up = <1>;
+			drive-strength = <42>;
+			power-source = <1800>;
+		};
+
+		mmc1-clk-pins {
+			pinmux = <K1_PADCONF(109, 0)>;   /* mmc1_clk */
+			bias-pull-down = <1>;
+			drive-strength = <42>;
+			power-source = <1800>;
+		};
+	};
 };
diff --git a/arch/riscv/boot/dts/spacemit/k1.dtsi b/arch/riscv/boot/dts/spacemit/k1.dtsi
index f0bad6855c970..06469c364eefd 100644
--- a/arch/riscv/boot/dts/spacemit/k1.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k1.dtsi
@@ -333,7 +333,7 @@ vctcxo_3m: clock-3m {
 
 		osc_32k: clock-32k {
 			compatible = "fixed-clock";
-			clock-frequency = <32000>;
+			clock-frequency = <32768>;
 			clock-output-names = "osc_32k";
 			#clock-cells = <0>;
 		};
@@ -983,6 +983,21 @@ qspi: spi@d420c000 {
 				status = "disabled";
 			};
 
+			spi3: spi@d401c000 {
+				compatible = "spacemit,k1-spi";
+				reg = <0x0 0xd401c000 0x0 0x30>;
+				#address-cells = <1>;
+				#size-cells = <0>;
+				clocks = <&syscon_apbc CLK_SSP3>,
+					 <&syscon_apbc CLK_SSP3_BUS>;
+				clock-names = "core", "bus";
+				resets = <&syscon_apbc RESET_SSP3>;
+				interrupts = <55>;
+				dmas = <&pdma 20>, <&pdma 19>;
+				dma-names = "rx", "tx";
+				status = "disabled";
+			};
+
 			/* sec_uart1: 0xf0612000, not available from Linux */
 		};
 
@@ -1199,6 +1214,19 @@ usb_dwc3: usb@c0a00000 {
 				status = "disabled";
 			};
 
+			sdhci0: mmc@d4280000 {
+				compatible = "spacemit,k1-sdhci";
+				reg = <0x0 0xd4280000 0x0 0x200>;
+				clocks = <&syscon_apmu CLK_SDH_AXI>,
+					 <&syscon_apmu CLK_SDH0>;
+				clock-names = "core", "io";
+				resets = <&syscon_apmu RESET_SDH_AXI>,
+					 <&syscon_apmu RESET_SDH0>;
+				reset-names = "axi", "sdh";
+				interrupts = <99>;
+				status = "disabled";
+			};
+
 			emmc: mmc@d4281000 {
 				compatible = "spacemit,k1-sdhci";
 				reg = <0x0 0xd4281000 0x0 0x200>;
diff --git a/arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts b/arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts
new file mode 100644
index 0000000000000..b5d5112de3581
--- /dev/null
+++ b/arch/riscv/boot/dts/spacemit/k3-deepcomputing-fml13v05.dts
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: (GPL-2.0 OR MIT)
+/*
+ * Copyright (C) 2026 DeepComputing (HK) Limited
+ */
+
+#include "k3.dtsi"
+#include "k3-pinctrl.dtsi"
+
+/ {
+	model = "DeepComputing FML13V05";
+	compatible = "deepcomputing,fml13v05", "spacemit,k3";
+
+	aliases {
+		serial0 = &uart0;
+	};
+
+	chosen {
+		stdout-path = "serial0";
+	};
+
+	memory@100000000 {
+		device_type = "memory";
+		reg = <0x1 0x00000000 0x4 0x00000000>;
+	};
+};
+
+&uart0 {
+	pinctrl-names = "default";
+	pinctrl-0 = <&uart0_0_cfg>;
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
index 4486dc1fe114d..b89c1521e6649 100644
--- a/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
+++ b/arch/riscv/boot/dts/spacemit/k3-pico-itx.dts
@@ -26,6 +26,14 @@ memory@100000000 {
 		reg = <0x1 0x00000000 0x4 0x00000000>;
 	};
 
+	reg_aux_vcc3v3: regulator-aux-vcc3v3 {
+		compatible = "regulator-fixed";
+		regulator-name = "AUX_VCC3V3";
+		regulator-min-microvolt = <3300000>;
+		regulator-max-microvolt = <3300000>;
+		regulator-always-on;
+	};
+
 	reg_aux_vcc5v: regulator-aux-vcc5v {
 		compatible = "regulator-fixed";
 		regulator-name = "AUX_VCC5V";
@@ -197,3 +205,19 @@ &uart0 {
 	pinctrl-0 = <&uart0_0_cfg>;
 	status = "okay";
 };
+
+&usb2_host {
+	#address-cells = <1>;
+	#size-cells = <0>;
+	status = "okay";
+
+	hub@1 {
+		compatible = "usb1a40,0101";
+		reg = <1>;
+		vdd-supply = <&reg_aux_vcc3v3>;
+	};
+};
+
+&usb2_phy {
+	status = "okay";
+};
diff --git a/arch/riscv/boot/dts/spacemit/k3.dtsi b/arch/riscv/boot/dts/spacemit/k3.dtsi
index 815debd16409b..23cca70f796be 100644
--- a/arch/riscv/boot/dts/spacemit/k3.dtsi
+++ b/arch/riscv/boot/dts/spacemit/k3.dtsi
@@ -424,7 +424,7 @@ vctcxo_3m: clock-3m {
 
 		osc_32k: clock-32k {
 			compatible = "fixed-clock";
-			clock-frequency = <32000>;
+			clock-frequency = <32768>;
 			clock-output-names = "osc_32k";
 			#clock-cells = <0>;
 		};
@@ -438,6 +438,37 @@ soc: soc {
 		dma-noncoherent;
 		ranges;
 
+		usb2_host: usb@c0a00000 {
+			compatible = "spacemit,k3-dwc3";
+			reg = <0x0 0xc0a00000 0x0 0x10000>;
+			clocks = <&syscon_apmu CLK_APMU_USB2_BUS>;
+			clock-names = "usbdrd30";
+			resets = <&syscon_apmu RESET_APMU_USB2_AHB>,
+				 <&syscon_apmu RESET_APMU_USB2_VCC>,
+				 <&syscon_apmu RESET_APMU_USB2_PHY>;
+			reset-names = "ahb", "vcc", "phy";
+			interrupts = <105 IRQ_TYPE_LEVEL_HIGH>;
+			interrupt-parent = <&saplic>;
+			phys = <&usb2_phy>;
+			phy-names = "usb2-phy";
+			phy_type = "utmi";
+			snps,dis_enblslpm_quirk;
+			snps,dis_u2_susphy_quirk;
+			snps,dis-del-phy-power-chg-quirk;
+			snps,dis-tx-ipgap-linecheck-quirk;
+			dr_mode = "host";
+			maximum-speed = "high-speed";
+			status = "disabled";
+		};
+
+		usb2_phy: phy@c0a20000 {
+			compatible = "spacemit,k3-usb2-phy";
+			reg = <0x0 0xc0a20000 0x0 0x200>;
+			clocks = <&syscon_apmu CLK_APMU_USB2_BUS>;
+			#phy-cells = <0>;
+			status = "disabled";
+		};
+
 		eth0: ethernet@cac80000 {
 			compatible = "spacemit,k3-dwmac", "snps,dwmac-5.40a";
 			reg = <0x0 0xcac80000 0x0 0x2000>;
diff --git a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi
index 8cfe8033305d8..a7a1c09a2c907 100644
--- a/arch/riscv/boot/dts/starfive/jh7110-common.dtsi
+++ b/arch/riscv/boot/dts/starfive/jh7110-common.dtsi
@@ -135,29 +135,6 @@ &tdm_ext {
 	clock-frequency = <49152000>;
 };
 
-&camss {
-	assigned-clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>,
-			  <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>;
-	assigned-clock-rates = <49500000>, <198000000>;
-
-	ports {
-		#address-cells = <1>;
-		#size-cells = <0>;
-
-		port@0 {
-			reg = <0>;
-		};
-
-		port@1 {
-			reg = <1>;
-
-			camss_from_csi2rx: endpoint {
-				remote-endpoint = <&csi2rx_to_camss>;
-			};
-		};
-	};
-};
-
 &csi2rx {
 	assigned-clocks = <&ispcrg JH7110_ISPCLK_VIN_SYS>;
 	assigned-clock-rates = <297000000>;
@@ -175,9 +152,7 @@ port@0 {
 		port@1 {
 			reg = <1>;
 
-			csi2rx_to_camss: endpoint {
-				remote-endpoint = <&camss_from_csi2rx>;
-			};
+			/* remote CAMSS endpoint */
 		};
 	};
 };
diff --git a/arch/riscv/boot/dts/starfive/jh7110.dtsi b/arch/riscv/boot/dts/starfive/jh7110.dtsi
index 6e56e9d20bb06..9c3e4598747eb 100644
--- a/arch/riscv/boot/dts/starfive/jh7110.dtsi
+++ b/arch/riscv/boot/dts/starfive/jh7110.dtsi
@@ -1199,34 +1199,6 @@ csi_phy: phy@19820000 {
 			#phy-cells = <0>;
 		};
 
-		camss: isp@19840000 {
-			compatible = "starfive,jh7110-camss";
-			reg = <0x0 0x19840000 0x0 0x10000>,
-			      <0x0 0x19870000 0x0 0x30000>;
-			reg-names = "syscon", "isp";
-			clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>,
-				 <&ispcrg JH7110_ISPCLK_ISPV2_TOP_WRAPPER_C>,
-				 <&ispcrg JH7110_ISPCLK_DVP_INV>,
-				 <&ispcrg JH7110_ISPCLK_VIN_P_AXI_WR>,
-				 <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>,
-				 <&syscrg JH7110_SYSCLK_ISP_TOP_CORE>,
-				 <&syscrg JH7110_SYSCLK_ISP_TOP_AXI>;
-			clock-names = "apb_func", "wrapper_clk_c", "dvp_inv",
-				      "axiwr", "mipi_rx0_pxl", "ispcore_2x",
-				      "isp_axi";
-			resets = <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_P>,
-				 <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_C>,
-				 <&ispcrg JH7110_ISPRST_VIN_P_AXI_RD>,
-				 <&ispcrg JH7110_ISPRST_VIN_P_AXI_WR>,
-				 <&syscrg JH7110_SYSRST_ISP_TOP>,
-				 <&syscrg JH7110_SYSRST_ISP_TOP_AXI>;
-			reset-names = "wrapper_p", "wrapper_c", "axird",
-				      "axiwr", "isp_top_n", "isp_top_axi";
-			power-domains = <&pwrc JH7110_PD_ISP>;
-			interrupts = <92>, <87>, <90>, <88>;
-			status = "disabled";
-		};
-
 		voutcrg: clock-controller@295c0000 {
 			compatible = "starfive,jh7110-voutcrg";
 			reg = <0x0 0x295c0000 0x0 0x10000>;
diff --git a/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi b/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi
index 6408810d8d80d..5f709e45d9b28 100644
--- a/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi
+++ b/arch/riscv/boot/dts/tenstorrent/blackhole.dtsi
@@ -77,6 +77,54 @@ cpu3_intc: interrupt-controller {
 		};
 	};
 
+	pmu {
+		compatible = "riscv,pmu";
+		riscv,event-to-mhpmevent =
+			/* SBI_PMU_HW_CPU_CYCLES -> CPU cycles */
+			<0x00001 0x00000000 0x0001>,
+			/* SBI_PMU_HW_INSTRUCTIONS -> Instructions executed */
+			<0x00002 0x00000000 0x0004>,
+			/* SBI_PMU_HW_CACHE_REFERENCES -> I-cache/ITIM busy | D-cache/DTIM busy */
+			<0x00003 0x00000000 0x1801>,
+			/* SBI_PMU_HW_CACHE_MISSES -> I-cache miss | D-cache miss */
+			<0x00004 0x00000000 0x0302>,
+			/* SBI_PMU_HW_BRANCH_INSTRUCTIONS -> Conditional branch retired */
+			<0x00005 0x00000000 0x4000>,
+			/*
+			 * SBI_PMU_HW_BRANCH_MISSES ->
+			 * Branch direction misprediction | Branch/jump target misprediction
+			 */
+			<0x00006 0x00000000 0x6001>,
+			/* L1D_READ_MISS -> Data cache miss or MMIO access */
+			<0x10001 0x00000000 0x0202>,
+			/* L1D_WRITE_ACCESS -> Data cache write-back */
+			<0x10002 0x00000000 0x0402>,
+			/* L1I_READ_MISS -> Instruction cache miss */
+			<0x10009 0x00000000 0x0102>,
+			/* LL_READ_MISS -> UTLB miss */
+			<0x10011 0x00000000 0x2002>,
+			/* DTLB_READ_MISS -> Data TLB miss */
+			<0x10019 0x00000000 0x1002>,
+			/* ITLB_READ_MISS -> Instruction TLB miss */
+			<0x10021 0x00000000 0x0802>;
+		riscv,event-to-mhpmcounters =
+			<0x00001 0x00001 0x01>,
+			<0x00002 0x00002 0x04>,
+			<0x00003 0x00006 0x78>,
+			<0x10001 0x10002 0x78>,
+			<0x10009 0x10009 0x78>,
+			<0x10011 0x10011 0x78>,
+			<0x10019 0x10019 0x78>,
+			<0x10021 0x10021 0x78>;
+		riscv,raw-event-to-mhpmcounters =
+			/* Class 0: Instruction Commit Events, bits 8-25 variant */
+			<0x0 0x0 0xffffffff 0xfc0000ff 0x78>,
+			/* Class 1: Microarchitectural Events, bits 8-18 variant */
+			<0x0 0x1 0xffffffff 0xfff800ff 0x78>,
+			/* Class 2: Memory System Events, bits 8-13 variant */
+			<0x0 0x2 0xffffffff 0xffffc0ff 0x78>;
+	};
+
 	soc {
 		#address-cells = <2>;
 		#size-cells = <2>;
diff --git a/arch/riscv/include/asm/asm-prototypes.h b/arch/riscv/include/asm/asm-prototypes.h
index 5b90ba5314ee9..a0ca9efff267e 100644
--- a/arch/riscv/include/asm/asm-prototypes.h
+++ b/arch/riscv/include/asm/asm-prototypes.h
@@ -5,6 +5,10 @@
 #include <linux/ftrace.h>
 #include <asm-generic/asm-prototypes.h>
 
+long long __lshrdi3(long long a, int b);
+long long __ashrdi3(long long a, int b);
+long long __ashldi3(long long a, int b);
+
 long long __lshrti3(long long a, int b);
 long long __ashrti3(long long a, int b);
 long long __ashlti3(long long a, int b);
diff --git a/arch/riscv/include/asm/syscall_wrapper.h b/arch/riscv/include/asm/syscall_wrapper.h
index ac80216549ffa..366a5354caa57 100644
--- a/arch/riscv/include/asm/syscall_wrapper.h
+++ b/arch/riscv/include/asm/syscall_wrapper.h
@@ -32,6 +32,8 @@ asmlinkage long __riscv_sys_ni_syscall(const struct pt_regs *);
 	__diag_push();									\
 	__diag_ignore(GCC, 8, "-Wattribute-alias",					\
 			"Type aliasing is used to sanitize syscall arguments");		\
+	__diag_ignore(clang, 23, "-Wattribute-alias",					\
+			"Type aliasing is used to sanitize syscall arguments");		\
 	static long __se_##prefix##name(ulong, ulong, ulong, ulong, ulong, ulong, 	\
 					ulong)						\
 			__attribute__((alias(__stringify(___se_##prefix##name))));	\
diff --git a/arch/riscv/kernel/image-vars.h b/arch/riscv/kernel/image-vars.h
index 3bd9d06a8b8ff..7b44b94f1283b 100644
--- a/arch/riscv/kernel/image-vars.h
+++ b/arch/riscv/kernel/image-vars.h
@@ -32,6 +32,15 @@ __efistub___init_text_end	= __init_text_end;
 __efistub_sysfb_primary_display	= sysfb_primary_display;
 #endif
 
+/*
+ * These double-word integer shifts are used by the library code, and
+ * the first two of them are required to link EFI stub. Note __ashrdi3()
+ * is not actually used by the stub but this may change in the future.
+ */
+PROVIDE(__efistub___lshrdi3	= __lshrdi3);
+PROVIDE(__efistub___ashldi3	= __ashldi3);
+PROVIDE(__efistub___ashrdi3	= __ashrdi3);
+
 #endif
 
 #endif /* __RISCV_KERNEL_IMAGE_VARS_H */
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index 6f767b2a349d7..f668b98970bda 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -16,6 +16,7 @@ ifeq ($(CONFIG_MMU), y)
 lib-$(CONFIG_RISCV_ISA_V)	+= uaccess_vector.o
 endif
 lib-$(CONFIG_MMU)	+= uaccess.o
+lib-$(CONFIG_32BIT)	+= ashldi3.o ashrdi3.o lshrdi3.o
 lib-$(CONFIG_64BIT)	+= tishift.o
 lib-$(CONFIG_RISCV_ISA_ZICBOZ)	+= clear_page.o
 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/arch/riscv/lib/ashldi3.S b/arch/riscv/lib/ashldi3.S
new file mode 100644
index 0000000000000..c3408862e2f6a
--- /dev/null
+++ b/arch/riscv/lib/ashldi3.S
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/**
+ * Adopted for the Linux kernel from IPXE project, see
+ * https://github.com/ipxe/ipxe/blob/master/src/arch/riscv32/libgcc/llshift.S
+ */
+
+#include <linux/linkage.h>
+#include <asm/asm.h>
+
+/**
+ * Shift left
+ *
+ * @v a1:a0             Value to shift
+ * @v a2                Shift amount
+ * @ret a1:a0           Shifted value
+ */
+
+SYM_FUNC_START(__ashldi3)
+
+        /* Perform shift by 32 bits, if applicable */
+        li      t0, 32
+        sub     t1, t0, a2
+        bgtz    t1, 1f
+        mv      a1, a0
+        mv      a0, zero
+1:      /* Perform shift by modulo-32 bits, if applicable */
+        andi    a2, a2, 0x1f
+        beqz    a2, 2f
+        srl     t2, a0, t1
+        sll     a0, a0, a2
+        sll     a1, a1, a2
+        or      a1, a1, t2
+2:      ret
+
+SYM_FUNC_END(__ashldi3)
+EXPORT_SYMBOL(__ashldi3)
diff --git a/arch/riscv/lib/ashrdi3.S b/arch/riscv/lib/ashrdi3.S
new file mode 100644
index 0000000000000..426de09466064
--- /dev/null
+++ b/arch/riscv/lib/ashrdi3.S
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/**
+ * Adopted for the Linux kernel from IPXE project, see
+ * https://github.com/ipxe/ipxe/blob/master/src/arch/riscv32/libgcc/llshift.S
+ */
+
+#include <linux/linkage.h>
+#include <asm/asm.h>
+
+/**
+ * Arithmetic shift right
+ *
+ * @v a1:a0             Value to shift
+ * @v a2                Shift amount
+ * @ret a1:a0           Shifted value
+ */
+
+SYM_FUNC_START(__ashrdi3)
+
+        /* Perform shift by 32 bits, if applicable */
+        li      t0, 32
+        sub     t1, t0, a2
+        bgtz    t1, 1f
+        mv      a0, a1
+        srai    a1, a1, 31
+1:      /* Perform shift by modulo-32 bits, if applicable */
+        andi    a2, a2, 0x1f
+        beqz    a2, 2f
+        sll     t2, a1, t1
+        sra     a1, a1, a2
+        srl     a0, a0, a2
+        or      a0, a0, t2
+2:      ret
+
+SYM_FUNC_END(__ashrdi3)
+EXPORT_SYMBOL(__ashrdi3)
diff --git a/arch/riscv/lib/lshrdi3.S b/arch/riscv/lib/lshrdi3.S
new file mode 100644
index 0000000000000..1af03985ccb72
--- /dev/null
+++ b/arch/riscv/lib/lshrdi3.S
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/**
+ * Adopted for the Linux kernel from IPXE project, see
+ * https://github.com/ipxe/ipxe/blob/master/src/arch/riscv32/libgcc/llshift.S
+ */
+
+#include <linux/linkage.h>
+#include <asm/asm.h>
+
+/**
+ * Logical shift right
+ *
+ * @v a1:a0             Value to shift
+ * @v a2                Shift amount
+ * @ret a1:a0           Shifted value
+ */
+
+SYM_FUNC_START(__lshrdi3)
+
+        /* Perform shift by 32 bits, if applicable */
+        li      t0, 32
+        sub     t1, t0, a2
+        bgtz    t1, 1f
+        mv      a0, a1
+        mv      a1, zero
+1:      /* Perform shift by modulo-32 bits, if applicable */
+        andi    a2, a2, 0x1f
+        beqz    a2, 2f
+        sll     t2, a1, t1
+        srl     a1, a1, a2
+        srl     a0, a0, a2
+        or      a0, a0, t2
+2:      ret
+
+SYM_FUNC_END(__lshrdi3)
+EXPORT_SYMBOL(__lshrdi3)
diff --git a/arch/riscv/mm/dma-noncoherent.c b/arch/riscv/mm/dma-noncoherent.c
index cb89d7e0ba88a..a1ec2d71d1c92 100644
--- a/arch/riscv/mm/dma-noncoherent.c
+++ b/arch/riscv/mm/dma-noncoherent.c
@@ -140,7 +140,7 @@ void arch_setup_dma_ops(struct device *dev, bool coherent)
 		   "%s %s: device non-coherent but no non-coherent operations supported",
 		   dev_driver_string(dev), dev_name(dev));
 
-	dev->dma_coherent = coherent;
+	dev_assign_dma_coherent(dev, coherent);
 }
 
 void riscv_noncoherent_supported(void)
diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c
index fa8d2f6f554b5..885f1db4e9bfd 100644
--- a/arch/riscv/mm/init.c
+++ b/arch/riscv/mm/init.c
@@ -1742,9 +1742,10 @@ int __ref arch_add_memory(int nid, u64 start, u64 size, struct mhp_params *param
 	return ret;
 }
 
-void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+			      struct dev_pagemap *pgmap)
 {
-	__remove_pages(start >> PAGE_SHIFT, size >> PAGE_SHIFT, altmap);
+	__remove_pages(start >> PAGE_SHIFT, size >> PAGE_SHIFT, altmap, pgmap);
 	remove_linear_mapping(start, size);
 	flush_tlb_all();
 }
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index ecbcbb781e40a..301cdc9f73aff 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -179,6 +179,7 @@ config S390
 	select GENERIC_SMP_IDLE_THREAD
 	select GENERIC_TIME_VSYSCALL
 	select GENERIC_IOREMAP if PCI
+	select GLOB
 	select HAVE_ALIGNED_STRUCT_PAGE
 	select HAVE_ARCH_AUDITSYSCALL
 	select HAVE_ARCH_GET_SECUREBOOT
diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h
index dad02f5b3c8d3..d1e4201f38506 100644
--- a/arch/s390/include/asm/barrier.h
+++ b/arch/s390/include/asm/barrier.h
@@ -8,6 +8,7 @@
 #ifndef __ASM_BARRIER_H
 #define __ASM_BARRIER_H
 
+#include <asm/alternative.h>
 #include <asm/march.h>
 
 /*
@@ -16,16 +17,11 @@
  * to devices.
  */
 
-#ifdef MARCH_HAS_Z196_FEATURES
-/* Fast-BCR without checkpoint synchronization */
-#define __ASM_BCR_SERIALIZE "bcr 14,0"
-#else
-#define __ASM_BCR_SERIALIZE "bcr 15,0"
-#endif
-
 static __always_inline void bcr_serialize(void)
 {
-	asm volatile(__ASM_BCR_SERIALIZE : : : "memory");
+	asm_inline volatile(
+		ALTERNATIVE("bcr 15,0", "bcr 14,0", ALT_FACILITY(45))
+		: : : "memory");
 }
 
 #define __mb()		bcr_serialize()
diff --git a/arch/s390/include/asm/debug.h b/arch/s390/include/asm/debug.h
index c5440b3ee53d7..39d484c597748 100644
--- a/arch/s390/include/asm/debug.h
+++ b/arch/s390/include/asm/debug.h
@@ -490,6 +490,7 @@ arch_initcall(VNAME(var, reg))
 __DEFINE_STATIC_AREA(var);						\
 static debug_info_t __refdata var =					\
 	__DEBUG_INFO_INIT(var, (name), (buf_size));			\
+static debug_info_t __used __section(".s390dbf_info") *VNAME(var, info) = &var; \
 __REGISTER_STATIC_DEBUG_INFO(var, name, pages, nr_areas, view)
 
 void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas);
diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h
index 78195ee5e99f6..ecd3341686eb6 100644
--- a/arch/s390/include/asm/processor.h
+++ b/arch/s390/include/asm/processor.h
@@ -33,6 +33,7 @@
 #include <linux/irqflags.h>
 #include <linux/instruction_pointer.h>
 #include <linux/bitops.h>
+#include <asm/vdso/processor.h>
 #include <asm/fpu-types.h>
 #include <asm/cpu.h>
 #include <asm/page.h>
@@ -282,8 +283,6 @@ static __always_inline unsigned short stap(void)
 	return cpu_address;
 }
 
-#define cpu_relax() barrier()
-
 #define ECAG_CACHE_ATTRIBUTE	0
 #define ECAG_CPU_ATTRIBUTE	1
 
diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h
index 0f184dbdbe5e0..d928a9ddfe402 100644
--- a/arch/s390/include/asm/sclp.h
+++ b/arch/s390/include/asm/sclp.h
@@ -20,6 +20,7 @@
 #define SCLP_ERRNOTIFY_AQ_REPAIR		1
 #define SCLP_ERRNOTIFY_AQ_INFO_LOG		2
 #define SCLP_ERRNOTIFY_AQ_OPTICS_DATA		3
+#define SCLP_ERRNOTIFY_AQ_NVME_SMART_LOG	4
 
 #ifndef __ASSEMBLER__
 #include <linux/uio.h>
diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h
index dff035372601e..a9f32c53f699b 100644
--- a/arch/s390/include/asm/uaccess.h
+++ b/arch/s390/include/asm/uaccess.h
@@ -30,8 +30,7 @@ void debug_user_asce(int exit);
 #define uaccess_kmsan_or_inline __always_inline
 #endif
 
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 static uaccess_kmsan_or_inline __must_check unsigned long
 raw_copy_from_user(void *to, const void __user *from, unsigned long size)
diff --git a/arch/s390/include/asm/vdso/processor.h b/arch/s390/include/asm/vdso/processor.h
index cfcc3e117c4c9..6775621e5a5a3 100644
--- a/arch/s390/include/asm/vdso/processor.h
+++ b/arch/s390/include/asm/vdso/processor.h
@@ -2,6 +2,8 @@
 #ifndef __ASM_VDSO_PROCESSOR_H
 #define __ASM_VDSO_PROCESSOR_H
 
-#define cpu_relax() barrier()
+#include <asm/barrier.h>
+
+#define cpu_relax()	bcr_serialize()
 
 #endif /* __ASM_VDSO_PROCESSOR_H */
diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c
index 7650f2adb5cf8..7bc403d081fac 100644
--- a/arch/s390/kernel/debug.c
+++ b/arch/s390/kernel/debug.c
@@ -26,6 +26,8 @@
 #include <linux/math.h>
 #include <linux/minmax.h>
 #include <linux/debugfs.h>
+#include <linux/glob.h>
+#include <linux/stringify.h>
 
 #include <asm/debug.h>
 
@@ -154,6 +156,7 @@ static unsigned int __used debug_feature_version = __DEBUG_FEATURE_VERSION;
 static debug_info_t *debug_area_first;
 static debug_info_t *debug_area_last;
 static DEFINE_MUTEX(debug_mutex);
+extern debug_info_t *__s390dbf_info[], *__s390dbf_info_end[];
 
 static int initialized;
 static int debug_critical;
@@ -168,7 +171,91 @@ static const struct file_operations debug_file_ops = {
 
 static struct dentry *debug_debugfs_root_entry;
 
+/* List of debug area parameters to override */
+#define PARAM_UNSET	-2
+#define PARAM_NUM	16
+static struct debug_param_t {
+	char name[DEBUG_MAX_NAME_LEN + 1];
+	int level;
+	int pages;
+} debug_param[PARAM_NUM];
+static int debug_param_num;
+
 /* functions */
+static void debug_get_param(const char *name, int *level, int *pages)
+{
+	struct debug_param_t *p;
+	int i;
+
+	for (i = 0; i < debug_param_num; i++) {
+		p = &debug_param[i];
+		if (!glob_match(p->name, name))
+			continue;
+		if (level && p->level != PARAM_UNSET) {
+			pr_info("%s: override level to %d\n", name, p->level);
+			*level = p->level;
+		}
+		if (pages && p->pages != PARAM_UNSET) {
+			pr_info("%s: override pages to %d\n", name, p->pages);
+			*pages = p->pages;
+		}
+	}
+}
+
+#define LVL_LEN		10
+#define LVL_FMT		"%" __stringify(LVL_LEN) "[^:]"
+#define NAME_FMT	"%" __stringify(DEBUG_MAX_NAME_LEN) "[^:]"
+
+static bool __init s390dbf_parse_one(const char *arg, struct debug_param_t *param)
+{
+	struct debug_param_t p = { { 0 }, PARAM_UNSET, PARAM_UNSET };
+	char level[LVL_LEN + 1] = { 0 };
+
+	/* arg: <name|pattern>:[<level>|-]:[<pages>] */
+	if (sscanf(arg, NAME_FMT ":" LVL_FMT ":%d", p.name, level, &p.pages) > 1) {
+		if (strcmp(level, "-") == 0)
+			p.level = DEBUG_OFF_LEVEL;
+		else if (kstrtoint(level, 0, &p.level) != 0)
+			return false;
+	} else if (sscanf(arg, NAME_FMT "::%d", p.name, &p.pages) != 2) {
+		return false;
+	}
+
+	if (p.level != PARAM_UNSET && p.level != DEBUG_OFF_LEVEL &&
+	    (p.level < 0 || p.level > DEBUG_MAX_LEVEL))
+		return false;
+	if (p.pages != PARAM_UNSET && p.pages < 0)
+		return false;
+	*param = p;
+
+	return true;
+}
+
+static int __init s390dbf_parse(char *arg)
+{
+	debug_info_t **id;
+	int i, rc = 0;
+
+	while (arg && debug_param_num < PARAM_NUM) {
+		if (s390dbf_parse_one(arg, &debug_param[debug_param_num]))
+			debug_param_num++;
+		else
+			rc = -EINVAL;
+		arg = strchr(arg, ',');
+		if (arg)
+			arg++;
+	}
+
+	/*
+	 * Apply level to static debug areas, delay buffer size changes until
+	 * regular memory allocations are possible.
+	 */
+	for (i = 0, id = __s390dbf_info; &id[i] < __s390dbf_info_end; i++)
+		debug_get_param(id[i]->name, &id[i]->level, NULL);
+
+	return rc;
+}
+early_param("s390dbf", s390dbf_parse);
 
 /*
  * debug_areas_alloc
@@ -305,10 +392,11 @@ static void debug_info_free(debug_info_t *db_info)
 static debug_info_t *debug_info_create(const char *name, int pages_per_area,
 				       int nr_areas, int buf_size, umode_t mode)
 {
+	int level = DEBUG_DEFAULT_LEVEL;
 	debug_info_t *rc;
 
-	rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size,
-			      DEBUG_DEFAULT_LEVEL, ALL_AREAS);
+	debug_get_param(name, &level, &pages_per_area);
+	rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, level, ALL_AREAS);
 	if (!rc)
 		goto out;
 
@@ -872,6 +960,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas)
 		return;
 	}
 
+	debug_get_param(id->name, &id->level, &pages_per_area);
 	copy = debug_info_alloc("", pages_per_area, nr_areas, id->buf_size,
 				id->level, ALL_AREAS);
 	if (!copy) {
@@ -975,16 +1064,7 @@ static int debug_set_size(debug_info_t *id, int nr_areas, int pages_per_area)
 	return 0;
 }
 
-/**
- * debug_set_level() - Sets new actual debug level if new_level is valid.
- *
- * @id:		handle for debug log
- * @new_level:	new debug level
- *
- * Return:
- *    none
- */
-void debug_set_level(debug_info_t *id, int new_level)
+static void _debug_set_level(debug_info_t *id, int new_level)
 {
 	unsigned long flags;
 
@@ -1003,6 +1083,23 @@ void debug_set_level(debug_info_t *id, int new_level)
 	id->level = new_level;
 	raw_spin_unlock_irqrestore(&id->lock, flags);
 }
+
+/**
+ * debug_set_level() - Sets new actual debug level if new_level is valid.
+ *
+ * @id:		handle for debug log
+ * @new_level:	new debug level
+ *
+ * Return:
+ *    none
+ */
+void debug_set_level(debug_info_t *id, int new_level)
+{
+	/* Level specified via kernel parameter takes precedence */
+	debug_get_param(id->name, &new_level, NULL);
+
+	_debug_set_level(id, new_level);
+}
 EXPORT_SYMBOL(debug_set_level);
 
 /*
@@ -1529,7 +1626,7 @@ static int debug_input_level_fn(debug_info_t *id, struct debug_view *view,
 		goto out;
 	}
 	if (str[0] == '-') {
-		debug_set_level(id, DEBUG_OFF_LEVEL);
+		_debug_set_level(id, DEBUG_OFF_LEVEL);
 		rc = user_len;
 		goto free_str;
 	} else {
@@ -1539,7 +1636,7 @@ static int debug_input_level_fn(debug_info_t *id, struct debug_view *view,
 		pr_warn("%s is not a valid level for a debug feature\n", str);
 		rc = -EINVAL;
 	} else {
-		debug_set_level(id, new_level);
+		_debug_set_level(id, new_level);
 		rc = user_len;
 	}
 free_str:
diff --git a/arch/s390/kernel/perf_pai.c b/arch/s390/kernel/perf_pai.c
index 86f71a3d1ef2d..cdb8006220ca0 100644
--- a/arch/s390/kernel/perf_pai.c
+++ b/arch/s390/kernel/perf_pai.c
@@ -186,6 +186,13 @@ static u64 pai_getctr(unsigned long *page, int nr, unsigned long offset)
 	return page[nr];
 }
 
+static void pai_setctr(unsigned long *page, int nr, unsigned long offset, u64 v)
+{
+	if (offset)
+		nr += offset / sizeof(*page);
+	page[nr] = v;
+}
+
 /* Read the counter values. Return value from location in CMP. For base
  * event xxx_ALL sum up all events. Returns counter value.
  */
@@ -551,6 +558,8 @@ static void paicrypt_del(struct perf_event *event, int flags)
 /* Create raw data and save it in buffer. Calculate the delta for each
  * counter between this invocation and the last invocation.
  * Returns number of bytes copied.
+ * After reading from PAI counter page, save the read value to the old
+ * page to calculate PAI counter deltas.
  * Saves only entries with positive counter difference of the form
  * 2 bytes: Number of counter
  * 8 bytes: Value of counter
@@ -562,16 +571,22 @@ static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page,
 	int i, outidx = 0;
 
 	for (i = 1; i <= pp->num_avail; i++) {
-		u64 val = 0, val_old = 0;
+		u64 val = 0, val_old = 0, val_k = 0, val_old_k = 0;
 
 		if (!exclude_kernel) {
-			val += pai_getctr(page, i, pp->kernel_offset);
-			val_old += pai_getctr(page_old, i, pp->kernel_offset);
+			val_k = pai_getctr(page, i, pp->kernel_offset);
+			val_old_k = pai_getctr(page_old, i, pp->kernel_offset);
+			if (val_k != val_old_k)
+				pai_setctr(page_old, i, pp->kernel_offset, val_k);
 		}
 		if (!exclude_user) {
-			val += pai_getctr(page, i, 0);
-			val_old += pai_getctr(page_old, i, 0);
+			val = pai_getctr(page, i, 0);
+			val_old = pai_getctr(page_old, i, 0);
+			if (val != val_old)
+				pai_setctr(page_old, i, 0, val);
 		}
+		val += val_k;
+		val_old += val_old_k;
 		if (val >= val_old)
 			val -= val_old;
 		else
@@ -602,8 +617,6 @@ static size_t pai_copy(struct pai_userdata *userdata, unsigned long *page,
 static int pai_push_sample(size_t rawsize, struct pai_map *cpump,
 			   struct perf_event *event)
 {
-	int idx = PAI_PMU_IDX(event);
-	struct pai_pmu *pp = &pai_pmu[idx];
 	struct perf_sample_data data;
 	struct perf_raw_record raw;
 	struct pt_regs regs;
@@ -634,8 +647,6 @@ static int pai_push_sample(size_t rawsize, struct pai_map *cpump,
 
 	overflow = perf_event_overflow(event, &data, &regs);
 	perf_event_update_userpage(event);
-	/* Save crypto counter lowcore page after reading event data. */
-	memcpy((void *)PAI_SAVE_AREA(event), cpump->area, pp->area_size);
 	return overflow;
 }
 
@@ -651,7 +662,7 @@ static void pai_have_sample(struct perf_event *event, struct pai_map *cpump)
 	rawsize = pai_copy(cpump->save, cpump->area, pp,
 			   (unsigned long *)PAI_SAVE_AREA(event),
 			   event->attr.exclude_user,
-			   event->attr.exclude_kernel);
+			   !pp->kernel_offset ? true : event->attr.exclude_kernel);
 	if (rawsize)			/* No incremented counters */
 		pai_push_sample(rawsize, cpump, event);
 }
diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c
index 0df95dcb21012..dabd569dbba8d 100644
--- a/arch/s390/kernel/process.c
+++ b/arch/s390/kernel/process.c
@@ -203,9 +203,6 @@ unsigned long __get_wchan(struct task_struct *p)
 	struct unwind_state state;
 	unsigned long ip = 0;
 
-	if (!task_stack_page(p))
-		return 0;
-
 	if (!try_get_task_stack(p))
 		return 0;
 
diff --git a/arch/s390/kernel/topology.c b/arch/s390/kernel/topology.c
index 1913a5566ac2b..1377c6f3f6709 100644
--- a/arch/s390/kernel/topology.c
+++ b/arch/s390/kernel/topology.c
@@ -192,17 +192,21 @@ static void tl_to_masks(struct sysinfo_15_1_x *info)
 	end = (union topology_entry *)((unsigned long)info + info->length);
 	while (tle < end) {
 		switch (tle->nl) {
+		/*
+		 * Adjust drawer_id, book_id, and socked_id so they match the
+		 * numbering scheme of e.g. the hardware management console.
+		 */
 		case 3:
 			drawer = drawer->next;
-			drawer->id = tle->container.id;
+			drawer->id = tle->container.id - 1;
 			break;
 		case 2:
 			book = book->next;
-			book->id = tle->container.id;
+			book->id = tle->container.id - 1;
 			break;
 		case 1:
 			socket = socket->next;
-			socket->id = tle->container.id;
+			socket->id = tle->container.id - 1;
 			break;
 		case 0:
 			add_cpus_to_mask(&tle->cpu, drawer, book, socket);
diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S
index 2b62395e35bfb..1f0e71c58eb9c 100644
--- a/arch/s390/kernel/vmlinux.lds.S
+++ b/arch/s390/kernel/vmlinux.lds.S
@@ -159,6 +159,13 @@ SECTIONS
 	}
 #endif
 
+	. = ALIGN(8);
+	.s390dbf_info : {
+		__s390dbf_info = .;
+		*(.s390dbf_info)
+		__s390dbf_info_end = .;
+	}
+
 	/*
 	 * Table with the patch locations to undo expolines
 	*/
diff --git a/arch/s390/kvm/dat.c b/arch/s390/kvm/dat.c
index 7b8d70fe406de..4a41c0247ffa2 100644
--- a/arch/s390/kvm/dat.c
+++ b/arch/s390/kvm/dat.c
@@ -267,6 +267,7 @@ static int dat_split_ste(struct kvm_s390_mmu_cache *mc, union pmd *pmdp, gfn_t g
 			/* No need to take locks as the page table is not installed yet. */
 			pgste_init.prefix_notif = old.s.fc1.prefix_notif;
 			pgste_init.vsie_notif = old.s.fc1.vsie_notif;
+			pgste_init.vsie_gmem = old.s.fc1.vsie_notif;
 			pgste_init.pcl = uses_skeys && init.h.i;
 			dat_init_pgstes(pt, pgste_init.val);
 		} else {
diff --git a/arch/s390/kvm/dat.h b/arch/s390/kvm/dat.h
index 8f8278c448794..873e13ac5a27f 100644
--- a/arch/s390/kvm/dat.h
+++ b/arch/s390/kvm/dat.h
@@ -145,7 +145,8 @@ union pgste {
 		unsigned long cmma_d       : 1; /* Dirty flag for CMMA bits */
 		unsigned long prefix_notif : 1; /* Guest prefix invalidation notification */
 		unsigned long vsie_notif   : 1; /* Referenced in a shadow table */
-		unsigned long              : 5;
+		unsigned long vsie_gmem    : 1; /* Contains nested guest memory */
+		unsigned long              : 4;
 		unsigned long              : 8;
 	};
 	struct {
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index b07accd196182..4f8d5592c9a92 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -1445,6 +1445,7 @@ static int _do_shadow_pte(struct gmap *sg, gpa_t raddr, union pte *ptep_h, union
 	} else {
 		pgste = _gmap_ptep_xchg(sg->parent, ptep_h, newpte, pgste, f->gfn, false);
 		pgste.vsie_notif = 1;
+		pgste.vsie_gmem = 1;
 	}
 	pgste_set_unlock(ptep_h, pgste);
 	if (rc)
diff --git a/arch/s390/kvm/gmap.c b/arch/s390/kvm/gmap.c
index 3c26e35af0ef7..957126ab991ca 100644
--- a/arch/s390/kvm/gmap.c
+++ b/arch/s390/kvm/gmap.c
@@ -125,7 +125,7 @@ struct gmap *gmap_new_child(struct gmap *parent, gfn_t limit)
 
 int gmap_set_limit(struct gmap *gmap, gfn_t limit)
 {
-	struct kvm_s390_mmu_cache *mc;
+	struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
 	int rc, type;
 
 	type = gmap_limit_to_type(limit);
@@ -142,7 +142,6 @@ int gmap_set_limit(struct gmap *gmap, gfn_t limit)
 			rc = dat_set_asce_limit(mc, &gmap->asce, type);
 	} while (rc == -ENOMEM);
 
-	kvm_s390_free_mmu_cache(mc);
 	return 0;
 }
 
@@ -822,8 +821,8 @@ int gmap_ucas_translate(struct kvm_s390_mmu_cache *mc, struct gmap *gmap, gpa_t
 
 int gmap_ucas_map(struct gmap *gmap, gfn_t p_gfn, gfn_t c_gfn, unsigned long count)
 {
-	struct kvm_s390_mmu_cache *mc;
-	int rc;
+	struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
+	int rc = 0;
 
 	mc = kvm_s390_new_mmu_cache();
 	if (!mc)
@@ -1026,13 +1025,15 @@ int gmap_insert_rmap(struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn, int level)
 int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gfn, gfn_t r_gfn,
 		      kvm_pfn_t pfn, int level, bool wr)
 {
+	unsigned long bitmask;
 	union crste *crstep;
 	union pgste pgste;
 	union pte *ptep;
 	union pte pte;
 	int flags, rc;
 
-	KVM_BUG_ON(!is_shadow(sg), sg->kvm);
+	if (KVM_BUG_ON(!is_shadow(sg) || level <= TABLE_TYPE_PAGE_TABLE, sg->kvm))
+		return -EINVAL;
 	lockdep_assert_held(&sg->parent->children_lock);
 
 	flags = DAT_WALK_SPLIT_ALLOC | (uses_skeys(sg->parent) ? DAT_WALK_USES_SKEYS : 0);
@@ -1041,8 +1042,9 @@ int gmap_protect_rmap(struct kvm_s390_mmu_cache *mc, struct gmap *sg, gfn_t p_gf
 	if (rc)
 		return rc;
 	if (level <= TABLE_TYPE_REGION1) {
+		bitmask = -1UL << (8 + 11 * level);
 		scoped_guard(spinlock, &sg->host_to_rmap_lock)
-			rc = gmap_insert_rmap(sg, p_gfn, r_gfn, level);
+			rc = gmap_insert_rmap(sg, p_gfn, r_gfn & bitmask, level);
 	}
 	if (rc)
 		return rc;
@@ -1143,8 +1145,10 @@ void _gmap_handle_vsie_unshadow_event(struct gmap *parent, gfn_t gfn)
 		}
 		scoped_guard(spinlock, &sg->host_to_rmap_lock)
 			head = radix_tree_delete(&sg->host_to_rmap, gfn);
-		gmap_for_each_rmap_safe(rmap, rnext, head)
+		gmap_for_each_rmap_safe(rmap, rnext, head) {
 			gmap_unshadow_level(sg, rmap->r_gfn, rmap->level);
+			kfree(rmap);
+		}
 	}
 }
 
diff --git a/arch/s390/kvm/gmap.h b/arch/s390/kvm/gmap.h
index 96ee1395a5925..742e42a317445 100644
--- a/arch/s390/kvm/gmap.h
+++ b/arch/s390/kvm/gmap.h
@@ -167,6 +167,36 @@ static inline bool gmap_unmap_prefix(struct gmap *gmap, gfn_t gfn, gfn_t end)
 	return _gmap_unmap_prefix(gmap, gfn, end, false);
 }
 
+/**
+ * pte_needs_unshadow() -- Check if the pte operations triggers unshadowing.
+ * @oldpte: the previous value for the guest pte.
+ * @newpte: the new pte being set.
+ * @pgste: the pgste for the pte entry.
+ *
+ * If the pgste.vsie_notif bit is not set, return false: the page is not
+ * involved in vsie and thus should not trigger an unshadow operation.
+ *
+ * If the pgste.vsie_gmem bit is set, this pte represents shadowed guest
+ * memory. The access rights on g3's memory should be synchronized with g1's
+ * and g2's. Therefore unshadowing is triggered if the new and old pte
+ * differ in protection, or if the new pte is invalid.
+ *
+ * If the pgste.vsie_gmem bit is not set, this pte maps the g2 dat tables
+ * for g3. If the entry becomes writable or absent, it becomes impossible to
+ * guarantee that the shadow mapping will match g2's mapping. In that case,
+ * trigger an unshadow event.
+ *
+ * Return: true if an unshadow event should be triggered, otherwise false.
+ */
+static inline bool pte_needs_unshadow(union pte oldpte, union pte newpte, union pgste pgste)
+{
+	if (!pgste.vsie_notif)
+		return false;
+	if (pgste.vsie_gmem)
+		return (oldpte.h.p != newpte.h.p) || newpte.h.i;
+	return !newpte.h.p || !newpte.s.pr;
+}
+
 static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, union pte newpte,
 					  union pgste pgste, gfn_t gfn, bool needs_lock)
 {
@@ -180,8 +210,9 @@ static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, un
 		pgste.prefix_notif = 0;
 		gmap_unmap_prefix(gmap, gfn, gfn + 1);
 	}
-	if (pgste.vsie_notif && (ptep->h.p != newpte.h.p || newpte.h.i)) {
+	if (pte_needs_unshadow(*ptep, newpte, pgste)) {
 		pgste.vsie_notif = 0;
+		pgste.vsie_gmem = 0;
 		if (needs_lock)
 			gmap_handle_vsie_unshadow_event(gmap, gfn);
 		else
@@ -189,6 +220,7 @@ static inline union pgste _gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, un
 	}
 	if (!ptep->s.d && newpte.s.d && !newpte.s.s)
 		SetPageDirty(pfn_to_page(newpte.h.pfra));
+	pgste.zero = 0;
 	return __dat_ptep_xchg(ptep, pgste, newpte, gfn, gmap->asce, uses_skeys(gmap));
 }
 
@@ -198,6 +230,30 @@ static inline union pgste gmap_ptep_xchg(struct gmap *gmap, union pte *ptep, uni
 	return _gmap_ptep_xchg(gmap, ptep, newpte, pgste, gfn, true);
 }
 
+/**
+ * crste_needs_unshadow() -- Check if the crste operations triggers unshadowing.
+ * @oldcrste: the previous value for the crste.
+ * @newcrste: the new value for the crste.
+ *
+ * If the old crste did not have the vsie_notif bit set, return false: the
+ * page is not involved in vsie and thus should not trigger an unshadow
+ * operation. Conversely, if the bit is set, it can only be g3 memory, since
+ * dat tables are never mapped using large pages.
+ *
+ * Similar to the pgste.vsie_gmem case of pte_needs_unshadow(), if the
+ * protection bit is changing or the new page is invalid, trigger an
+ * unshadow event. Also trigger an unshadow event if the new crste does not
+ * have the vsie_notif bit set.
+ *
+ * Return: true if an unshadow event should be triggered, otherwise false.
+ */
+static inline bool crste_needs_unshadow(union crste oldcrste, union crste newcrste)
+{
+	if (!oldcrste.s.fc1.vsie_notif)
+		return false;
+	return (newcrste.h.p != oldcrste.h.p) || newcrste.h.i || !newcrste.s.fc1.vsie_notif;
+}
+
 static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, union crste *crstep,
 							 union crste oldcrste, union crste newcrste,
 							 gfn_t gfn, bool needs_lock)
@@ -216,8 +272,7 @@ static inline bool __must_check _gmap_crstep_xchg_atomic(struct gmap *gmap, unio
 		newcrste.s.fc1.prefix_notif = 0;
 		gmap_unmap_prefix(gmap, gfn, gfn + align);
 	}
-	if (crste_leaf(oldcrste) && oldcrste.s.fc1.vsie_notif &&
-	    (newcrste.h.p || newcrste.h.i || !newcrste.s.fc1.vsie_notif)) {
+	if (crste_leaf(oldcrste) && crste_needs_unshadow(oldcrste, newcrste)) {
 		newcrste.s.fc1.vsie_notif = 0;
 		if (needs_lock)
 			gmap_handle_vsie_unshadow_event(gmap, gfn);
diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index 1f72efc2a579f..f07168a0d3dd3 100644
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -84,6 +84,8 @@ void __init arch_setup_zero_pages(void)
 	empty_zero_page = (unsigned long)memblock_alloc_or_panic(PAGE_SIZE << order, PAGE_SIZE);
 
 	zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK;
+
+	set_memory_ro(empty_zero_page, 1UL << order);
 }
 
 void __init arch_zone_limits_init(unsigned long *max_zone_pfns)
@@ -276,12 +278,13 @@ int arch_add_memory(int nid, u64 start, u64 size,
 	return rc;
 }
 
-void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+			struct dev_pagemap *pgmap)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 
-	__remove_pages(start_pfn, nr_pages, altmap);
+	__remove_pages(start_pfn, nr_pages, altmap, pgmap);
 	vmem_remove_mapping(start, size);
 }
 #endif /* CONFIG_MEMORY_HOTPLUG */
diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c
index eeadff45e0e10..d8b2a60e0c33b 100644
--- a/arch/s390/mm/vmem.c
+++ b/arch/s390/mm/vmem.c
@@ -4,7 +4,6 @@
  */
 
 #include <linux/memory_hotplug.h>
-#include <linux/bootmem_info.h>
 #include <linux/cpufeature.h>
 #include <linux/memblock.h>
 #include <linux/pfn.h>
@@ -51,7 +50,7 @@ static void vmem_free_pages(unsigned long addr, int order, struct vmem_altmap *a
 	if (PageReserved(page)) {
 		/* allocated from memblock */
 		while (nr_pages--)
-			free_bootmem_page(page++);
+			free_reserved_page(page++);
 	} else {
 		free_pages(addr, order);
 	}
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index 94128fe6be233..14eaaa5b21851 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -27,6 +27,7 @@
 #include <asm/extable.h>
 #include <asm/dis.h>
 #include <asm/facility.h>
+#include <asm/lowcore.h>
 #include <asm/nospec-branch.h>
 #include <asm/set_memory.h>
 #include <asm/text-patching.h>
@@ -1777,6 +1778,30 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
 		int j, ret;
 		u64 func;
 
+		/* Implement helper call to bpf_get_smp_processor_id() inline */
+		if (insn->src_reg == 0 &&
+		    insn->imm == BPF_FUNC_get_smp_processor_id) {
+			const u32 *cpu_nr = &get_lowcore()->cpu_nr;
+
+			/* ly %b0, cpu_nr */
+			EMIT6_DISP_LH(0xe3000000, 0x0058, BPF_REG_0, REG_0, REG_0,
+				      (unsigned long)cpu_nr);
+			break;
+		}
+
+		/* Implement helper call to bpf_get_current_task/_btf() inline */
+		if (insn->src_reg == 0 &&
+		    (insn->imm == BPF_FUNC_get_current_task ||
+		     insn->imm == BPF_FUNC_get_current_task_btf)) {
+			const u64 *current_task =
+				&get_lowcore()->current_task;
+
+			/* lg %b0, current_task */
+			EMIT6_DISP_LH(0xe3000000, 0x0004, BPF_REG_0, REG_0, REG_0,
+				      (unsigned long)current_task);
+			break;
+		}
+
 		ret = bpf_jit_get_func_addr(fp, insn, extra_pass,
 					    &func, &func_addr_fixed);
 		if (ret < 0)
@@ -3057,3 +3082,15 @@ bool bpf_jit_supports_timed_may_goto(void)
 {
 	return true;
 }
+
+bool bpf_jit_inlines_helper_call(s32 imm)
+{
+	switch (imm) {
+	case BPF_FUNC_get_smp_processor_id:
+	case BPF_FUNC_get_current_task:
+	case BPF_FUNC_get_current_task_btf:
+		return true;
+	default:
+		return false;
+	}
+}
diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h
index a79609eb14be4..02e7a066538ec 100644
--- a/arch/sh/include/asm/uaccess.h
+++ b/arch/sh/include/asm/uaccess.h
@@ -95,8 +95,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 {
 	return __copy_user((__force void *)to, from, n);
 }
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 /*
  * Clear the area and return remaining number of bytes
diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c
index 83fb34b39ca7c..8bc9ce541c141 100644
--- a/arch/sh/mm/cache-sh4.c
+++ b/arch/sh/mm/cache-sh4.c
@@ -248,7 +248,7 @@ static void sh4_flush_cache_page(void *args)
 		 */
 		map_coherent = (current_cpu_data.dcache.n_aliases &&
 			test_bit(PG_dcache_clean, folio_flags(folio, 0)) &&
-			page_mapped(page));
+			folio_mapped(folio));
 		if (map_coherent)
 			vaddr = kmap_coherent(page, address);
 		else
diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h
index 43284b6ec46a6..5542d5b32994f 100644
--- a/arch/sparc/include/asm/uaccess_32.h
+++ b/arch/sparc/include/asm/uaccess_32.h
@@ -190,8 +190,7 @@ static inline unsigned long raw_copy_from_user(void *to, const void __user *from
 	return __copy_user((__force void __user *) to, from, n);
 }
 
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 static inline unsigned long __clear_user(void __user *addr, unsigned long size)
 {
diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h
index b825a5dd0210e..e2989cfba626d 100644
--- a/arch/sparc/include/asm/uaccess_64.h
+++ b/arch/sparc/include/asm/uaccess_64.h
@@ -231,8 +231,7 @@ unsigned long __must_check raw_copy_from_user(void *to,
 unsigned long __must_check raw_copy_to_user(void __user *to,
 					   const void *from,
 					   unsigned long size);
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 unsigned long __must_check raw_copy_in_user(void __user *to,
 					   const void __user *from,
diff --git a/arch/sparc/include/uapi/asm/errno.h b/arch/sparc/include/uapi/asm/errno.h
index 4a41e7835fd5b..71940ec9130b4 100644
--- a/arch/sparc/include/uapi/asm/errno.h
+++ b/arch/sparc/include/uapi/asm/errno.h
@@ -117,4 +117,6 @@
 
 #define EHWPOISON	135	/* Memory page has hardware error */
 
+#define EFTYPE		136	/* Wrong file type for the intended operation */
+
 #endif
diff --git a/arch/sparc/include/uapi/asm/fcntl.h b/arch/sparc/include/uapi/asm/fcntl.h
index 67dae75e5274d..2fe7a042bea96 100644
--- a/arch/sparc/include/uapi/asm/fcntl.h
+++ b/arch/sparc/include/uapi/asm/fcntl.h
@@ -37,6 +37,7 @@
 
 #define O_PATH		0x1000000
 #define __O_TMPFILE	0x2000000
+#define OPENAT2_REGULAR	0x8000000
 
 #define F_GETOWN	5	/*  for sockets. */
 #define F_SETOWN	6	/*  for sockets. */
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 367c269305e51..3b679b1d1d72d 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -27,7 +27,6 @@
 #include <linux/percpu.h>
 #include <linux/mmzone.h>
 #include <linux/gfp.h>
-#include <linux/bootmem_info.h>
 
 #include <asm/head.h>
 #include <asm/page.h>
@@ -2477,17 +2476,6 @@ int page_in_phys_avail(unsigned long paddr)
 	return 0;
 }
 
-static void __init register_page_bootmem_info(void)
-{
-#ifdef CONFIG_NUMA
-	int i;
-
-	for_each_online_node(i)
-		if (NODE_DATA(i)->node_spanned_pages)
-			register_page_bootmem_info_node(NODE_DATA(i));
-#endif
-}
-
 void __init arch_setup_zero_pages(void)
 {
 	phys_addr_t zero_page_pa = kern_base +
@@ -2498,14 +2486,6 @@ void __init arch_setup_zero_pages(void)
 
 void __init mem_init(void)
 {
-	/*
-	 * Must be done after boot memory is put on freelist, because here we
-	 * might set fields in deferred struct pages that have not yet been
-	 * initialized, and memblock_free_all() initializes all the reserved
-	 * deferred pages for us.
-	 */
-	register_page_bootmem_info();
-
 	if (tlb_type == cheetah || tlb_type == cheetah_plus)
 		cheetah_ecache_flush_init();
 }
diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h
index 0df9ea4abda83..4417c8b1d37a6 100644
--- a/arch/um/include/asm/uaccess.h
+++ b/arch/um/include/asm/uaccess.h
@@ -27,8 +27,7 @@ static inline int __access_ok(const void __user *ptr, unsigned long size);
 #define __access_ok __access_ok
 #define __clear_user __clear_user
 
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 #include <asm-generic/uaccess.h>
 
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index f3f7cb01d69d0..f24810015234e 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -695,17 +695,6 @@ config X86_INTEL_QUARK
 	  Say Y here if you have a Quark based system such as the Arduino
 	  compatible Intel Galileo.
 
-config X86_RDC321X
-	bool "RDC R-321x SoC"
-	depends on X86_32
-	depends on X86_EXTENDED_PLATFORM
-	select M486
-	select X86_REBOOTFIXUPS
-	help
-	  This option is needed for RDC R-321x system-on-chip, also known
-	  as R-8610-(G).
-	  If you don't have one of these chips, you should say N here.
-
 config X86_INTEL_LPSS
 	bool "Intel Low Power Subsystem Support"
 	depends on X86 && ACPI && PCI
@@ -1644,33 +1633,6 @@ config X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK
 	  Set whether the default state of memory_corruption_check is
 	  on or off.
 
-config MATH_EMULATION
-	bool
-	depends on MODIFY_LDT_SYSCALL
-	prompt "Math emulation" if X86_32 && (M486SX || MELAN)
-	help
-	  Linux can emulate a math coprocessor (used for floating point
-	  operations) if you don't have one. 486DX and Pentium processors have
-	  a math coprocessor built in, 486SX and 386 do not, unless you added
-	  a 487DX or 387, respectively. (The messages during boot time can
-	  give you some hints here ["man dmesg"].) Everyone needs either a
-	  coprocessor or this emulation.
-
-	  If you don't have a math coprocessor, you need to say Y here; if you
-	  say Y here even though you have a coprocessor, the coprocessor will
-	  be used nevertheless. (This behavior can be changed with the kernel
-	  command line option "no387", which comes handy if your coprocessor
-	  is broken. Try "man bootparam" or see the documentation of your boot
-	  loader (lilo or loadlin) about how to pass options to the kernel at
-	  boot time.) This means that it is a good idea to say Y here if you
-	  intend to use this kernel on different machines.
-
-	  More information about the internals of the Linux math coprocessor
-	  emulation can be found in <file:arch/x86/math-emu/README>.
-
-	  If you are not sure, say Y; apart from resulting in a 66 KB bigger
-	  kernel, it won't hurt.
-
 config MTRR
 	def_bool y
 	prompt "MTRR (Memory Type Range Register) support" if EXPERT
@@ -2437,10 +2399,6 @@ config CALL_THUNKS
 	def_bool n
 	select CALL_PADDING
 
-config PREFIX_SYMBOLS
-	def_bool y
-	depends on CALL_PADDING && !CFI
-
 menuconfig CPU_MITIGATIONS
 	bool "Mitigations for CPU vulnerabilities"
 	default y
@@ -2780,18 +2738,16 @@ menuconfig APM
 	  1) make sure that you have enough swap space and that it is
 	  enabled.
 	  2) pass the "idle=poll" option to the kernel
-	  3) switch on floating point emulation in the kernel and pass
-	  the "no387" option to the kernel
-	  4) pass the "floppy=nodma" option to the kernel
-	  5) pass the "mem=4M" option to the kernel (thereby disabling
+	  3) pass the "floppy=nodma" option to the kernel
+	  4) pass the "mem=4M" option to the kernel (thereby disabling
 	  all but the first 4 MB of RAM)
-	  6) make sure that the CPU is not over clocked.
-	  7) read the sig11 FAQ at <http://www.bitwizard.nl/sig11/>
-	  8) disable the cache from your BIOS settings
-	  9) install a fan for the video card or exchange video RAM
-	  10) install a better fan for the CPU
-	  11) exchange RAM chips
-	  12) exchange the motherboard.
+	  5) make sure that the CPU is not over clocked.
+	  6) read the sig11 FAQ at <http://www.bitwizard.nl/sig11/>
+	  7) disable the cache from your BIOS settings
+	  8) install a fan for the video card or exchange video RAM
+	  9) install a better fan for the CPU
+	  10) exchange RAM chips
+	  11) exchange the motherboard.
 
 	  To compile this driver as a module, choose M here: the
 	  module will be called apm.
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index d7ba9219cb47d..1377edd9a9974 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -32,8 +32,6 @@ choice
 	  - "Athlon" for the AMD K7 family (Athlon/Duron/Thunderbird).
 	  - "Crusoe" for the Transmeta Crusoe series.
 	  - "Efficeon" for the Transmeta Efficeon series.
-	  - "Winchip-C6" for original IDT Winchip.
-	  - "Winchip-2" for IDT Winchips with 3dNow! capabilities.
 	  - "AMD Elan" for the 32-bit AMD Elan embedded CPU.
 	  - "GeodeGX1" for Geode GX1 (Cyrix MediaGX).
 	  - "Geode GX/LX" For AMD Geode GX and LX processors.
@@ -45,14 +43,6 @@ choice
 	  See each option's help text for additional details. If you don't know
 	  what to do, choose "Pentium-Pro".
 
-config M586
-	bool "586/K5/5x86/6x86/6x86MX"
-	depends on X86_32
-	help
-	  Select this for an 586 or 686 series processor such as the AMD K5,
-	  the Cyrix 5x86, 6x86 and 6x86MX.  This choice does not
-	  assume the RDTSC (Read Time Stamp Counter) instruction.
-
 config M586TSC
 	bool "Pentium-Classic"
 	depends on X86_32
@@ -155,24 +145,6 @@ config MEFFICEON
 	help
 	  Select this for a Transmeta Efficeon processor.
 
-config MWINCHIPC6
-	bool "Winchip-C6"
-	depends on X86_32
-	help
-	  Select this for an IDT Winchip C6 chip.  Linux and GCC
-	  treat this chip as a 586TSC with some extended instructions
-	  and alignment requirements.
-
-config MWINCHIP3D
-	bool "Winchip-2/Winchip-2A/Winchip-3"
-	depends on X86_32
-	help
-	  Select this for an IDT Winchip-2, 2A or 3.  Linux and GCC
-	  treat this chip as a 586TSC with some extended instructions
-	  and alignment requirements.  Also enable out of order memory
-	  stores for this CPU, which can increase performance of some
-	  operations.
-
 config MGEODEGX1
 	bool "GeodeGX1"
 	depends on X86_32
@@ -270,19 +242,15 @@ config X86_L1_CACHE_SHIFT
 	default "7" if MPENTIUM4
 	default "6" if MK7 || MPENTIUMM || MATOM || MVIAC7 || X86_GENERIC || X86_64
 	default "4" if MGEODEGX1
-	default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX
+	default "5" if MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MVIAC3_2 || MGEODE_LX
 
 config X86_F00F_BUG
 	def_bool y
 	depends on M586MMX || M586TSC || M586
 
-config X86_INVD_BUG
-	def_bool y
-	depends on M486SX || M486
-
 config X86_ALIGNMENT_16
 	def_bool y
-	depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK6 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODEGX1
+	depends on MCYRIXIII || MK6 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODEGX1
 
 config X86_INTEL_USERCOPY
 	def_bool y
@@ -290,7 +258,7 @@ config X86_INTEL_USERCOPY
 
 config X86_USE_PPRO_CHECKSUM
 	def_bool y
-	depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MATOM
+	depends on MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MATOM
 
 config X86_TSC
 	def_bool y
@@ -318,7 +286,7 @@ config X86_MINIMUM_CPU_FAMILY
 
 config X86_DEBUGCTLMSR
 	def_bool y
-	depends on !(MK6 || MWINCHIPC6 || MWINCHIP3D || MCYRIXIII || M586MMX || M586TSC || M586) && !UML
+	depends on !(MK6 || MCYRIXIII || M586MMX || M586TSC || M586) && !UML
 
 config IA32_FEAT_CTL
 	def_bool y
@@ -419,20 +387,6 @@ config CPU_SUP_TRANSMETA_32
 
 	  If unsure, say N.
 
-config CPU_SUP_UMC_32
-	default y
-	bool "Support UMC processors" if PROCESSOR_SELECT
-	depends on M486SX || M486 || (EXPERT && !64BIT)
-	help
-	  This enables detection, tunings and quirks for UMC processors
-
-	  You need this enabled if you want your kernel to run on a
-	  UMC CPU. Disabling this option on other types of CPUs
-	  makes the kernel a tiny bit smaller. Disabling it on a UMC
-	  CPU might render the kernel unbootable.
-
-	  If unsure, say N.
-
 config CPU_SUP_ZHAOXIN
 	default y
 	bool "Support Zhaoxin processors" if PROCESSOR_SELECT
diff --git a/arch/x86/Kconfig.cpufeatures b/arch/x86/Kconfig.cpufeatures
index 532cbc276b1e3..89cbf8f572aee 100644
--- a/arch/x86/Kconfig.cpufeatures
+++ b/arch/x86/Kconfig.cpufeatures
@@ -70,7 +70,6 @@ config X86_REQUIRED_FEATURE_UP
 
 config X86_REQUIRED_FEATURE_FPU
 	def_bool y
-	depends on !MATH_EMULATION
 
 config X86_REQUIRED_FEATURE_PAE
 	def_bool y
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 46fec0b08487a..96bfa137a82b9 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -276,7 +276,6 @@ archprepare: $(cpufeaturemasks.hdr)
 libs-y  += arch/x86/lib/
 
 # drivers-y are linked after core-y
-drivers-$(CONFIG_MATH_EMULATION) += arch/x86/math-emu/
 drivers-$(CONFIG_PCI)            += arch/x86/pci/
 
 # suspend and hibernation support
diff --git a/arch/x86/Makefile_32.cpu b/arch/x86/Makefile_32.cpu
index 7c9898c153761..c5aa169b596d3 100644
--- a/arch/x86/Makefile_32.cpu
+++ b/arch/x86/Makefile_32.cpu
@@ -10,7 +10,6 @@ else
 align		:= -falign-functions=0 -falign-jumps=0 -falign-loops=0
 endif
 
-cflags-$(CONFIG_M586)		+= -march=i586
 cflags-$(CONFIG_M586TSC)	+= -march=i586
 cflags-$(CONFIG_M586MMX)	+= -march=pentium-mmx
 cflags-$(CONFIG_M686)		+= -march=i686
@@ -24,16 +23,11 @@ cflags-$(CONFIG_MK6)		+= -march=k6
 cflags-$(CONFIG_MK7)		+= -march=athlon
 cflags-$(CONFIG_MCRUSOE)	+= -march=i686 $(align)
 cflags-$(CONFIG_MEFFICEON)	+= -march=i686 $(call tune,pentium3) $(align)
-cflags-$(CONFIG_MWINCHIPC6)	+= $(call cc-option,-march=winchip-c6,-march=i586)
-cflags-$(CONFIG_MWINCHIP3D)	+= $(call cc-option,-march=winchip2,-march=i586)
 cflags-$(CONFIG_MCYRIXIII)	+= $(call cc-option,-march=c3,-march=i486) $(align)
 cflags-$(CONFIG_MVIAC3_2)	+= $(call cc-option,-march=c3-2,-march=i686)
 cflags-$(CONFIG_MVIAC7)		+= -march=i686
 cflags-$(CONFIG_MATOM)		+= -march=atom
 
-# AMD Elan support
-cflags-$(CONFIG_MELAN)		+= -march=i486
-
 # Geode GX1 support
 cflags-$(CONFIG_MGEODEGX1)	+= -march=pentium-mmx
 cflags-$(CONFIG_MGEODE_LX)	+= $(call cc-option,-march=geode,-march=pentium-mmx)
diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c
index 3b0948ad449f9..8e4bf5365ac63 100644
--- a/arch/x86/boot/compressed/kaslr.c
+++ b/arch/x86/boot/compressed/kaslr.c
@@ -219,7 +219,8 @@ static void parse_gb_huge_pages(char *param, char *val)
 
 	if (!strcmp(param, "hugepages") && gbpage_sz) {
 		p = val;
-		max_gb_huge_pages = simple_strtoull(p, &p, 0);
+		if (boot_kstrtoul(p, 0, &max_gb_huge_pages))
+			warn("Failed to parse hugepages= boot parameter\n");
 		return;
 	}
 }
diff --git a/arch/x86/boot/compressed/pgtable_64.c b/arch/x86/boot/compressed/pgtable_64.c
index 0e89e197e1126..1b2fb35704f9b 100644
--- a/arch/x86/boot/compressed/pgtable_64.c
+++ b/arch/x86/boot/compressed/pgtable_64.c
@@ -2,6 +2,7 @@
 #include "misc.h"
 #include <asm/bootparam.h>
 #include <asm/bootparam_utils.h>
+#include <asm/cpuid/api.h>
 #include <asm/e820/types.h>
 #include <asm/pgtable.h>
 #include <asm/processor.h>
diff --git a/arch/x86/boot/startup/sme.c b/arch/x86/boot/startup/sme.c
index b76a7c95dfe1b..c07a2c381ed1f 100644
--- a/arch/x86/boot/startup/sme.c
+++ b/arch/x86/boot/startup/sme.c
@@ -43,6 +43,7 @@
 #include <asm/init.h>
 #include <asm/setup.h>
 #include <asm/sections.h>
+#include <asm/cpuid/api.h>
 #include <asm/coco.h>
 #include <asm/sev.h>
 
diff --git a/arch/x86/boot/string.c b/arch/x86/boot/string.c
index b25c6a9303b73..ac0f900ebc47e 100644
--- a/arch/x86/boot/string.c
+++ b/arch/x86/boot/string.c
@@ -289,6 +289,9 @@ static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res)
 	unsigned long long _res;
 	unsigned int rv;
 
+	if (s[0] == '+')
+		s++;
+
 	s = _parse_integer_fixup_radix(s, &base);
 	rv = _parse_integer(s, base, &_res);
 	if (rv & KSTRTOX_OVERFLOW)
@@ -304,35 +307,12 @@ static int _kstrtoull(const char *s, unsigned int base, unsigned long long *res)
 	return 0;
 }
 
-/**
- * kstrtoull - convert a string to an unsigned long long
- * @s: The start of the string. The string must be null-terminated, and may also
- *  include a single newline before its terminating null. The first character
- *  may also be a plus sign, but not a minus sign.
- * @base: The number base to use. The maximum supported base is 16. If base is
- *  given as 0, then the base of the string is automatically detected with the
- *  conventional semantics - If it begins with 0x the number will be parsed as a
- *  hexadecimal (case insensitive), if it otherwise begins with 0, it will be
- *  parsed as an octal number. Otherwise it will be parsed as a decimal.
- * @res: Where to write the result of the conversion on success.
- *
- * Returns 0 on success, -ERANGE on overflow and -EINVAL on parsing error.
- * Used as a replacement for the obsolete simple_strtoull. Return code must
- * be checked.
- */
-int kstrtoull(const char *s, unsigned int base, unsigned long long *res)
-{
-	if (s[0] == '+')
-		s++;
-	return _kstrtoull(s, base, res);
-}
-
 static int _kstrtoul(const char *s, unsigned int base, unsigned long *res)
 {
 	unsigned long long tmp;
 	int rv;
 
-	rv = kstrtoull(s, base, &tmp);
+	rv = _kstrtoull(s, base, &tmp);
 	if (rv < 0)
 		return rv;
 	if (tmp != (unsigned long)tmp)
@@ -364,7 +344,7 @@ int boot_kstrtoul(const char *s, unsigned int base, unsigned long *res)
 	 */
 	if (sizeof(unsigned long) == sizeof(unsigned long long) &&
 	    __alignof__(unsigned long) == __alignof__(unsigned long long))
-		return kstrtoull(s, base, (unsigned long long *)res);
+		return _kstrtoull(s, base, (unsigned long long *)res);
 	else
 		return _kstrtoul(s, base, res);
 }
diff --git a/arch/x86/boot/string.h b/arch/x86/boot/string.h
index a5b05ebc037de..4092bf2ed1eec 100644
--- a/arch/x86/boot/string.h
+++ b/arch/x86/boot/string.h
@@ -28,6 +28,5 @@ extern unsigned long long simple_strtoull(const char *cp, char **endp,
 					  unsigned int base);
 long simple_strtol(const char *cp, char **endp, unsigned int base);
 
-int kstrtoull(const char *s, unsigned int base, unsigned long long *res);
 int boot_kstrtoul(const char *s, unsigned int base, unsigned long *res);
 #endif /* BOOT_STRING_H */
diff --git a/arch/x86/coco/sev/core.c b/arch/x86/coco/sev/core.c
index 7ed3da998489d..ecd77d3217f3c 100644
--- a/arch/x86/coco/sev/core.c
+++ b/arch/x86/coco/sev/core.c
@@ -367,17 +367,13 @@ static unsigned long __set_pages_state(struct snp_psc_desc *data, unsigned long
 
 	local_irq_save(flags);
 
-	if (sev_cfg.ghcbs_initialized)
-		ghcb = __sev_get_ghcb(&state);
-	else
-		ghcb = boot_ghcb;
+	ghcb = __sev_get_ghcb(&state);
 
 	/* Invoke the hypervisor to perform the page state changes */
 	if (!ghcb || vmgexit_psc(ghcb, data))
 		sev_es_terminate(SEV_TERM_SET_LINUX, GHCB_TERM_PSC);
 
-	if (sev_cfg.ghcbs_initialized)
-		__sev_put_ghcb(&state);
+	__sev_put_ghcb(&state);
 
 	local_irq_restore(flags);
 
diff --git a/arch/x86/coco/sev/internal.h b/arch/x86/coco/sev/internal.h
index b1d0c66a651a6..b9632c0fc3916 100644
--- a/arch/x86/coco/sev/internal.h
+++ b/arch/x86/coco/sev/internal.h
@@ -70,6 +70,9 @@ void svsm_pval_pages(struct snp_psc_desc *desc);
 int svsm_perform_call_protocol(struct svsm_call *call);
 bool snp_svsm_vtpm_probe(void);
 
+noinstr void kernel_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code);
+noinstr void user_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code);
+
 static inline u64 sev_es_rd_ghcb_msr(void)
 {
 	return native_rdmsrq(MSR_AMD64_SEV_ES_GHCB);
diff --git a/arch/x86/coco/sev/noinstr.c b/arch/x86/coco/sev/noinstr.c
index 5afd663a1c217..e1e03f12fc7bb 100644
--- a/arch/x86/coco/sev/noinstr.c
+++ b/arch/x86/coco/sev/noinstr.c
@@ -121,8 +121,10 @@ noinstr struct ghcb *__sev_get_ghcb(struct ghcb_state *state)
 
 	WARN_ON(!irqs_disabled());
 
-	if (!sev_cfg.ghcbs_initialized)
+	if (!sev_cfg.ghcbs_initialized) {
+		state->ghcb = NULL;
 		return boot_ghcb;
+	}
 
 	data = this_cpu_read(runtime_data);
 	ghcb = &data->ghcb_page;
diff --git a/arch/x86/coco/sev/svsm.c b/arch/x86/coco/sev/svsm.c
index 2acf4a76afe7a..916d62cd17dc7 100644
--- a/arch/x86/coco/sev/svsm.c
+++ b/arch/x86/coco/sev/svsm.c
@@ -74,20 +74,14 @@ int svsm_perform_call_protocol(struct svsm_call *call)
 
 	flags = native_local_irq_save();
 
-	if (sev_cfg.ghcbs_initialized)
-		ghcb = __sev_get_ghcb(&state);
-	else if (boot_ghcb)
-		ghcb = boot_ghcb;
-	else
-		ghcb = NULL;
+	ghcb = __sev_get_ghcb(&state);
 
 	do {
 		ret = ghcb ? svsm_perform_ghcb_protocol(ghcb, call)
 			   : __pi_svsm_perform_msr_protocol(call);
 	} while (ret == -EAGAIN);
 
-	if (sev_cfg.ghcbs_initialized)
-		__sev_put_ghcb(&state);
+	__sev_put_ghcb(&state);
 
 	native_local_irq_restore(flags);
 
diff --git a/arch/x86/coco/sev/vc-handle.c b/arch/x86/coco/sev/vc-handle.c
index d98b5c08ef00d..96b62b49b2b53 100644
--- a/arch/x86/coco/sev/vc-handle.c
+++ b/arch/x86/coco/sev/vc-handle.c
@@ -954,7 +954,7 @@ static __always_inline bool vc_is_db(unsigned long error_code)
  * Runtime #VC exception handler when raised from kernel mode. Runs in NMI mode
  * and will panic when an error happens.
  */
-DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
+noinstr void kernel_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code)
 {
 	irqentry_state_t irq_state;
 
@@ -1006,7 +1006,7 @@ DEFINE_IDTENTRY_VC_KERNEL(exc_vmm_communication)
  * Runtime #VC exception handler when raised from user mode. Runs in IRQ mode
  * and will kill the current task with SIGBUS when an error happens.
  */
-DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
+noinstr void user_exc_vmm_communication(struct pt_regs *regs, unsigned long error_code)
 {
 	/*
 	 * Handle #DB before calling into !noinstr code to avoid recursive #DB.
@@ -1032,6 +1032,14 @@ DEFINE_IDTENTRY_VC_USER(exc_vmm_communication)
 	irqentry_exit_to_user_mode(regs);
 }
 
+DEFINE_IDTENTRY_RAW_ERRORCODE(exc_vmm_communication)
+{
+	if (user_mode(regs))
+		return user_exc_vmm_communication(regs, error_code);
+	else
+		return kernel_exc_vmm_communication(regs, error_code);
+}
+
 bool __init handle_vc_boot_ghcb(struct pt_regs *regs)
 {
 	unsigned long exit_code = regs->orig_ax;
diff --git a/arch/x86/coco/tdx/tdx.c b/arch/x86/coco/tdx/tdx.c
index 186915a17c50f..29b6f1ed59ec5 100644
--- a/arch/x86/coco/tdx/tdx.c
+++ b/arch/x86/coco/tdx/tdx.c
@@ -14,6 +14,7 @@
 #include <asm/ia32.h>
 #include <asm/insn.h>
 #include <asm/insn-eval.h>
+#include <asm/cpuid/api.h>
 #include <asm/paravirt_types.h>
 #include <asm/pgtable.h>
 #include <asm/set_memory.h>
diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile
index 72cae8e0ce856..83b4762d6ecba 100644
--- a/arch/x86/entry/Makefile
+++ b/arch/x86/entry/Makefile
@@ -13,7 +13,7 @@ CFLAGS_REMOVE_syscall_64.o	= $(CC_FLAGS_FTRACE)
 CFLAGS_syscall_32.o		+= -fno-stack-protector
 CFLAGS_syscall_64.o		+= -fno-stack-protector
 
-obj-y				:= entry.o entry_$(BITS).o syscall_$(BITS).o
+obj-y				:= entry.o entry_$(BITS).o syscall_$(BITS).o common.o
 
 obj-y				+= vdso/
 obj-y				+= vsyscall/
diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c
new file mode 100644
index 0000000000000..06c7c6ebd6f94
--- /dev/null
+++ b/arch/x86/entry/common.c
@@ -0,0 +1,61 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#include <linux/entry-common.h>
+#include <linux/kvm_types.h>
+#include <linux/hrtimer_rearm.h>
+#include <asm/fred.h>
+#include <asm/desc.h>
+
+#if IS_ENABLED(CONFIG_KVM_INTEL)
+/*
+ * On VMX, NMIs and IRQs (as configured by KVM) are acknowledged by hardware as
+ * part of the VM-Exit, i.e. the event itself is consumed as part the VM-Exit.
+ * x86_entry_from_kvm() is invoked by KVM to effectively forward NMIs and IRQs
+ * to the kernel for servicing.  On SVM, a.k.a. AMD, the NMI/IRQ VM-Exit is
+ * purely a signal that an NMI/IRQ is pending, i.e. the event that triggered
+ * the VM-Exit is held pending until it's unblocked in the host.
+ */
+noinstr void x86_entry_from_kvm(unsigned int event_type, unsigned int vector)
+{
+	if (event_type == EVENT_TYPE_EXTINT) {
+#ifdef CONFIG_X86_64
+		/*
+		 * Use FRED dispatch, even when running IDT. The dispatch
+		 * tables are kept in sync between FRED and IDT, and the FRED
+		 * dispatch works well with CFI.
+		 */
+		fred_entry_from_kvm(event_type, vector);
+#else
+		idt_entry_from_kvm(vector);
+#endif
+		/*
+		 * Strictly speaking, only the NMI path requires noinstr.
+		 */
+		instrumentation_begin();
+		/*
+		 * KVM/VMX will dispatch from IRQ-disabled but for a context
+		 * that will have IRQs-enabled. This confuses the entry code
+		 * and it will not have reprogrammed the timer. Do so now.
+		 */
+		hrtimer_rearm_deferred();
+		instrumentation_end();
+
+		return;
+	}
+
+	WARN_ON_ONCE(event_type != EVENT_TYPE_NMI);
+
+#ifdef CONFIG_X86_64
+	if (cpu_feature_enabled(X86_FEATURE_FRED))
+		return fred_entry_from_kvm(event_type, vector);
+#endif
+
+	/*
+	 * Notably, we must use IDT dispatch for NMI when running in IDT mode.
+	 * The FRED NMI context is significantly different and will not work
+	 * right (specifically FRED fixed the NMI recursion issue).
+	 */
+	idt_entry_from_kvm(vector);
+}
+EXPORT_SYMBOL_FOR_KVM(x86_entry_from_kvm);
+#endif
diff --git a/arch/x86/entry/entry.S b/arch/x86/entry/entry.S
index 6ba2b3adcef0f..a56e043b266dc 100644
--- a/arch/x86/entry/entry.S
+++ b/arch/x86/entry/entry.S
@@ -75,3 +75,49 @@ THUNK warn_thunk_thunk, __warn_thunk
 #if defined(CONFIG_STACKPROTECTOR) && defined(CONFIG_SMP)
 EXPORT_SYMBOL(__ref_stack_chk_guard);
 #endif
+
+#if IS_ENABLED(CONFIG_KVM_INTEL)
+.macro IDT_DO_EVENT_IRQOFF call_insn call_target
+	/*
+	 * Unconditionally create a stack frame, getting the correct RSP on the
+	 * stack (for x86-64) would take two instructions anyways, and RBP can
+	 * be used to restore RSP to make objtool happy (see below).
+	 */
+	push %_ASM_BP
+	mov %_ASM_SP, %_ASM_BP
+
+#ifdef CONFIG_X86_64
+	/*
+	 * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
+	 * creating the synthetic interrupt stack frame for the IRQ/NMI.
+	 */
+	and  $-16, %rsp
+	push $__KERNEL_DS
+	push %rbp
+#endif
+	pushf
+	push $__KERNEL_CS
+	\call_insn \call_target
+
+	/*
+	 * "Restore" RSP from RBP, even though IRET has already unwound RSP to
+	 * the correct value.  objtool doesn't know the callee will IRET and,
+	 * without the explicit restore, thinks the stack is getting walloped.
+	 * Using an unwind hint is problematic due to x86-64's dynamic alignment.
+	 */
+	leave
+	RET
+.endm
+
+.pushsection .text, "ax"
+SYM_FUNC_START(idt_do_interrupt_irqoff)
+	IDT_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
+SYM_FUNC_END(idt_do_interrupt_irqoff)
+.popsection
+
+.pushsection .noinstr.text, "ax"
+SYM_FUNC_START(idt_do_nmi_irqoff)
+	IDT_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
+SYM_FUNC_END(idt_do_nmi_irqoff)
+.popsection
+#endif
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 42447b1e1dffa..c6d996593f329 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -492,7 +492,7 @@ SYM_CODE_START(\asmsym)
 
 	movq	%rsp, %rdi		/* pt_regs pointer */
 
-	call	kernel_\cfunc
+	call	\cfunc
 
 	/*
 	 * No need to switch back to the IST stack. The current stack is either
@@ -503,7 +503,7 @@ SYM_CODE_START(\asmsym)
 
 	/* Switch to the regular task stack */
 .Lfrom_usermode_switch_stack_\@:
-	idtentry_body user_\cfunc, has_error_code=1
+	idtentry_body \cfunc, has_error_code=1
 
 _ASM_NOKPROBE(\asmsym)
 SYM_CODE_END(\asmsym)
diff --git a/arch/x86/entry/entry_64_fred.S b/arch/x86/entry/entry_64_fred.S
index 894f7f16eb80a..0d2768ab836c4 100644
--- a/arch/x86/entry/entry_64_fred.S
+++ b/arch/x86/entry/entry_64_fred.S
@@ -147,5 +147,4 @@ SYM_FUNC_START(asm_fred_entry_from_kvm)
 	RET
 
 SYM_FUNC_END(asm_fred_entry_from_kvm)
-EXPORT_SYMBOL_FOR_KVM(asm_fred_entry_from_kvm);
 #endif
diff --git a/arch/x86/entry/entry_fred.c b/arch/x86/entry/entry_fred.c
index fbe2d10dd737d..fb3594ddf731f 100644
--- a/arch/x86/entry/entry_fred.c
+++ b/arch/x86/entry/entry_fred.c
@@ -177,16 +177,6 @@ static noinstr void fred_extint(struct pt_regs *regs)
 	}
 }
 
-#ifdef CONFIG_AMD_MEM_ENCRYPT
-noinstr void exc_vmm_communication(struct pt_regs *regs, unsigned long error_code)
-{
-	if (user_mode(regs))
-		return user_exc_vmm_communication(regs, error_code);
-	else
-		return kernel_exc_vmm_communication(regs, error_code);
-}
-#endif
-
 static noinstr void fred_hwexc(struct pt_regs *regs, unsigned long error_code)
 {
 	/* Optimize for #PF. That's the only exception which matters performance wise */
diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c
index a6bfcc8243cd9..d903bce24f15d 100644
--- a/arch/x86/entry/vdso/vma.c
+++ b/arch/x86/entry/vdso/vma.c
@@ -178,7 +178,7 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
 		do_munmap(mm, text_start, image->size, NULL);
-		do_munmap(mm, addr, image->size, NULL);
+		do_munmap(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, NULL);
 		goto up_fail;
 	}
 
diff --git a/arch/x86/events/amd/core.c b/arch/x86/events/amd/core.c
index 0c92ed5f464b1..d66a357f219d3 100644
--- a/arch/x86/events/amd/core.c
+++ b/arch/x86/events/amd/core.c
@@ -8,8 +8,10 @@
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/jiffies.h>
+
 #include <asm/apicdef.h>
 #include <asm/apic.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 #include <asm/nmi.h>
 
diff --git a/arch/x86/events/amd/ibs.c b/arch/x86/events/amd/ibs.c
index e0bd5051db2a1..20c2de5c697b9 100644
--- a/arch/x86/events/amd/ibs.c
+++ b/arch/x86/events/amd/ibs.c
@@ -15,6 +15,7 @@
 #include <linux/sched/clock.h>
 
 #include <asm/apic.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 
 #include "../perf_event.h"
diff --git a/arch/x86/events/amd/lbr.c b/arch/x86/events/amd/lbr.c
index d24da377df776..5b437dc8e4ce2 100644
--- a/arch/x86/events/amd/lbr.c
+++ b/arch/x86/events/amd/lbr.c
@@ -1,5 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0
 #include <linux/perf_event.h>
+
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 #include <asm/perf_event.h>
 
diff --git a/arch/x86/events/amd/power.c b/arch/x86/events/amd/power.c
index dad42790cf7d5..744dffa42deea 100644
--- a/arch/x86/events/amd/power.c
+++ b/arch/x86/events/amd/power.c
@@ -10,8 +10,11 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/perf_event.h>
+
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
+
 #include "../perf_event.h"
 
 /* Event code: LSB 8 bits, passed in attr->config any other bit is reserved. */
diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c
index dd956cfcadefa..05cff39968ec9 100644
--- a/arch/x86/events/amd/uncore.c
+++ b/arch/x86/events/amd/uncore.c
@@ -16,6 +16,7 @@
 #include <linux/smp.h>
 
 #include <asm/perf_event.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 
 #define NUM_COUNTERS_NB		4
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index dd1e3aa75ee9b..af7261db1d1f6 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -17,6 +17,7 @@
 #include <linux/kvm_host.h>
 
 #include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
 #include <asm/debugreg.h>
 #include <asm/hardirq.h>
 #include <asm/intel-family.h>
@@ -215,39 +216,47 @@ static struct event_constraint intel_slm_event_constraints[] __read_mostly =
 
 static struct event_constraint intel_grt_event_constraints[] __read_mostly = {
 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+	FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
-	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
+	FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
 	FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
 	EVENT_CONSTRAINT_END
 };
 
 static struct event_constraint intel_skt_event_constraints[] __read_mostly = {
 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+	FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
-	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
+	FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
 	FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
 	FIXED_EVENT_CONSTRAINT(0x0073, 4), /* TOPDOWN_BAD_SPECULATION.ALL */
+	FIXED_EVENT_CONSTRAINT(0x0500, 4), /* pseudo TOPDOWN_BAD_SPECULATION.ALL */
 	FIXED_EVENT_CONSTRAINT(0x019c, 5), /* TOPDOWN_FE_BOUND.ALL */
+	FIXED_EVENT_CONSTRAINT(0x0600, 5), /* pseudo TOPDOWN_FE_BOUND.ALL */
 	FIXED_EVENT_CONSTRAINT(0x02c2, 6), /* TOPDOWN_RETIRING.ALL */
+	FIXED_EVENT_CONSTRAINT(0x0700, 6), /* pseudo TOPDOWN_RETIRING.ALL */
 	EVENT_CONSTRAINT_END
 };
 
 static struct event_constraint intel_arw_event_constraints[] __read_mostly = {
 	FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
+	FIXED_EVENT_CONSTRAINT(0x0100, 0), /* pseudo INST_RETIRED.ANY */
 	FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
-	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF */
+	FIXED_EVENT_CONSTRAINT(0x0200, 1), /* pseudo CPU_CLK_UNHALTED.THREAD */
+	FIXED_EVENT_CONSTRAINT(0x0300, 2), /* pseudo CPU_CLK_UNHALTED.REF_TSC */
 	FIXED_EVENT_CONSTRAINT(0x013c, 2), /* CPU_CLK_UNHALTED.REF_TSC_P */
 	FIXED_EVENT_CONSTRAINT(0x0073, 4), /* TOPDOWN_BAD_SPECULATION.ALL */
+	FIXED_EVENT_CONSTRAINT(0x0500, 4), /* pseudo TOPDOWN_BAD_SPECULATION.ALL */
 	FIXED_EVENT_CONSTRAINT(0x019c, 5), /* TOPDOWN_FE_BOUND.ALL */
+	FIXED_EVENT_CONSTRAINT(0x0600, 5), /* pseudo TOPDOWN_FE_BOUND.ALL */
 	FIXED_EVENT_CONSTRAINT(0x02c2, 6), /* TOPDOWN_RETIRING.ALL */
+	FIXED_EVENT_CONSTRAINT(0x0700, 6), /* pseudo TOPDOWN_RETIRING.ALL */
 	INTEL_UEVENT_CONSTRAINT(0x01b7, 0x1),
 	INTEL_UEVENT_CONSTRAINT(0x02b7, 0x2),
 	INTEL_UEVENT_CONSTRAINT(0x04b7, 0x4),
 	INTEL_UEVENT_CONSTRAINT(0x08b7, 0x8),
-	INTEL_UEVENT_CONSTRAINT(0x01d4, 0x1),
-	INTEL_UEVENT_CONSTRAINT(0x02d4, 0x2),
-	INTEL_UEVENT_CONSTRAINT(0x04d4, 0x4),
-	INTEL_UEVENT_CONSTRAINT(0x08d4, 0x8),
 	INTEL_UEVENT_CONSTRAINT(0x0175, 0x1),
 	INTEL_UEVENT_CONSTRAINT(0x0275, 0x2),
 	INTEL_UEVENT_CONSTRAINT(0x21d3, 0x1),
@@ -310,10 +319,11 @@ static struct extra_reg intel_skl_extra_regs[] __read_mostly = {
 static struct event_constraint intel_icl_event_constraints[] = {
 	FIXED_EVENT_CONSTRAINT(0x00c0, 0),	/* INST_RETIRED.ANY */
 	FIXED_EVENT_CONSTRAINT(0x01c0, 0),	/* old INST_RETIRED.PREC_DIST */
-	FIXED_EVENT_CONSTRAINT(0x0100, 0),	/* INST_RETIRED.PREC_DIST */
+	FIXED_EVENT_CONSTRAINT(0x0100, 0),	/* pseudo INST_RETIRED.ANY */
 	FIXED_EVENT_CONSTRAINT(0x003c, 1),	/* CPU_CLK_UNHALTED.CORE */
-	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* CPU_CLK_UNHALTED.REF */
-	FIXED_EVENT_CONSTRAINT(0x0400, 3),	/* SLOTS */
+	FIXED_EVENT_CONSTRAINT(0x0200, 1),	/* pseudo CPU_CLK_UNHALTED.THREAD */
+	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* pseudo CPU_CLK_UNHALTED.REF_TSC */
+	FIXED_EVENT_CONSTRAINT(0x0400, 3),	/* pseudo TOPDOWN.SLOTS */
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
@@ -355,11 +365,12 @@ static struct extra_reg intel_glc_extra_regs[] __read_mostly = {
 
 static struct event_constraint intel_glc_event_constraints[] = {
 	FIXED_EVENT_CONSTRAINT(0x00c0, 0),	/* INST_RETIRED.ANY */
-	FIXED_EVENT_CONSTRAINT(0x0100, 0),	/* INST_RETIRED.PREC_DIST */
+	FIXED_EVENT_CONSTRAINT(0x0100, 0),	/* pseudo INST_RETIRED.ANY */
 	FIXED_EVENT_CONSTRAINT(0x003c, 1),	/* CPU_CLK_UNHALTED.CORE */
-	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* CPU_CLK_UNHALTED.REF */
+	FIXED_EVENT_CONSTRAINT(0x0200, 1),	/* pseudo CPU_CLK_UNHALTED.THREAD */
+	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* pseudo CPU_CLK_UNHALTED.REF_TSC */
 	FIXED_EVENT_CONSTRAINT(0x013c, 2),	/* CPU_CLK_UNHALTED.REF_TSC_P */
-	FIXED_EVENT_CONSTRAINT(0x0400, 3),	/* SLOTS */
+	FIXED_EVENT_CONSTRAINT(0x0400, 3),	/* pseudo TOPDOWN.SLOTS */
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
@@ -379,9 +390,13 @@ static struct event_constraint intel_glc_event_constraints[] = {
 
 	INTEL_UEVENT_CONSTRAINT(0x01a3, 0xf),
 	INTEL_UEVENT_CONSTRAINT(0x02a3, 0xf),
+	INTEL_UEVENT_CONSTRAINT(0x05a3, 0xf),
+	INTEL_UEVENT_CONSTRAINT(0x06a3, 0xf),
 	INTEL_UEVENT_CONSTRAINT(0x08a3, 0xf),
+	INTEL_UEVENT_CONSTRAINT(0x0ca3, 0xf),
 	INTEL_UEVENT_CONSTRAINT(0x04a4, 0x1),
 	INTEL_UEVENT_CONSTRAINT(0x08a4, 0x1),
+	INTEL_UEVENT_CONSTRAINT(0x01cd, 0xfe),
 	INTEL_UEVENT_CONSTRAINT(0x02cd, 0x1),
 	INTEL_EVENT_CONSTRAINT(0xce, 0x1),
 	INTEL_EVENT_CONSTRAINT_RANGE(0xd0, 0xdf, 0xf),
@@ -407,11 +422,12 @@ static struct extra_reg intel_rwc_extra_regs[] __read_mostly = {
 
 static struct event_constraint intel_lnc_event_constraints[] = {
 	FIXED_EVENT_CONSTRAINT(0x00c0, 0),	/* INST_RETIRED.ANY */
-	FIXED_EVENT_CONSTRAINT(0x0100, 0),	/* INST_RETIRED.PREC_DIST */
+	FIXED_EVENT_CONSTRAINT(0x0100, 0),	/* pseudo INST_RETIRED.ANY */
 	FIXED_EVENT_CONSTRAINT(0x003c, 1),	/* CPU_CLK_UNHALTED.CORE */
-	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* CPU_CLK_UNHALTED.REF */
+	FIXED_EVENT_CONSTRAINT(0x0200, 1),	/* pseudo CPU_CLK_UNHALTED.THREAD */
+	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* pseudo CPU_CLK_UNHALTED.REF_TSC */
 	FIXED_EVENT_CONSTRAINT(0x013c, 2),	/* CPU_CLK_UNHALTED.REF_TSC_P */
-	FIXED_EVENT_CONSTRAINT(0x0400, 3),	/* SLOTS */
+	FIXED_EVENT_CONSTRAINT(0x0400, 3),	/* pseudo TOPDOWN.SLOTS */
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
@@ -423,8 +439,6 @@ static struct event_constraint intel_lnc_event_constraints[] = {
 
 	INTEL_EVENT_CONSTRAINT(0x20, 0xf),
 
-	INTEL_UEVENT_CONSTRAINT(0x012a, 0xf),
-	INTEL_UEVENT_CONSTRAINT(0x012b, 0xf),
 	INTEL_UEVENT_CONSTRAINT(0x0148, 0x4),
 	INTEL_UEVENT_CONSTRAINT(0x0175, 0x4),
 
@@ -435,15 +449,14 @@ static struct event_constraint intel_lnc_event_constraints[] = {
 	INTEL_UEVENT_CONSTRAINT(0x0ca3, 0x4),
 	INTEL_UEVENT_CONSTRAINT(0x04a4, 0x1),
 	INTEL_UEVENT_CONSTRAINT(0x08a4, 0x1),
-	INTEL_UEVENT_CONSTRAINT(0x10a4, 0x1),
+	INTEL_UEVENT_CONSTRAINT(0x10a4, 0x8),
 	INTEL_UEVENT_CONSTRAINT(0x01b1, 0x8),
 	INTEL_UEVENT_CONSTRAINT(0x01cd, 0x3fc),
 	INTEL_UEVENT_CONSTRAINT(0x02cd, 0x3),
 
+	INTEL_UEVENT_CONSTRAINT(0x87d0, 0x3ff),
 	INTEL_EVENT_CONSTRAINT_RANGE(0xd0, 0xdf, 0xf),
 
-	INTEL_UEVENT_CONSTRAINT(0x00e0, 0xf),
-
 	EVENT_CONSTRAINT_END
 };
 
@@ -460,11 +473,12 @@ static struct extra_reg intel_lnc_extra_regs[] __read_mostly = {
 
 static struct event_constraint intel_pnc_event_constraints[] = {
 	FIXED_EVENT_CONSTRAINT(0x00c0, 0),	/* INST_RETIRED.ANY */
-	FIXED_EVENT_CONSTRAINT(0x0100, 0),	/* INST_RETIRED.PREC_DIST */
+	FIXED_EVENT_CONSTRAINT(0x0100, 0),	/* pseudo INST_RETIRED.ANY */
 	FIXED_EVENT_CONSTRAINT(0x003c, 1),	/* CPU_CLK_UNHALTED.CORE */
-	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* CPU_CLK_UNHALTED.REF */
+	FIXED_EVENT_CONSTRAINT(0x0200, 1),      /* pseudo CPU_CLK_UNHALTED.THREAD */
+	FIXED_EVENT_CONSTRAINT(0x0300, 2),	/* pseudo CPU_CLK_UNHALTED.REF_TSC */
 	FIXED_EVENT_CONSTRAINT(0x013c, 2),	/* CPU_CLK_UNHALTED.REF_TSC_P */
-	FIXED_EVENT_CONSTRAINT(0x0400, 3),	/* SLOTS */
+	FIXED_EVENT_CONSTRAINT(0x0400, 3),	/* pseudo TOPDOWN.SLOTS */
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
 	METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
@@ -706,6 +720,80 @@ static __initconst const u64 glc_hw_cache_event_ids
  },
 };
 
+/* ADL P-core (Golden cove) specific event code. */
+static __initconst const u64 adl_glc_hw_cache_event_ids
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(L1D ) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x81d0,
+		[ C(RESULT_MISS)   ] = 0xe124,
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = 0x82d0,
+	},
+ },
+ [ C(L1I ) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_MISS)   ] = 0xe424,
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = -1,
+		[ C(RESULT_MISS)   ] = -1,
+	},
+ },
+ [ C(LL  ) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x12a,
+		[ C(RESULT_MISS)   ] = 0x12a,
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = 0x12a,
+		[ C(RESULT_MISS)   ] = 0x12a,
+	},
+ },
+ [ C(DTLB) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x81d0,
+		[ C(RESULT_MISS)   ] = 0xe12,
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = 0x82d0,
+		[ C(RESULT_MISS)   ] = 0xe13,
+	},
+ },
+ [ C(ITLB) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = -1,
+		[ C(RESULT_MISS)   ] = 0xe11,
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = -1,
+		[ C(RESULT_MISS)   ] = -1,
+	},
+	[ C(OP_PREFETCH) ] = {
+		[ C(RESULT_ACCESS) ] = -1,
+		[ C(RESULT_MISS)   ] = -1,
+	},
+ },
+ [ C(BPU ) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x4c4,
+		[ C(RESULT_MISS)   ] = 0x4c5,
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = -1,
+		[ C(RESULT_MISS)   ] = -1,
+	},
+	[ C(OP_PREFETCH) ] = {
+		[ C(RESULT_ACCESS) ] = -1,
+		[ C(RESULT_MISS)   ] = -1,
+	},
+ },
+};
+
 static __initconst const u64 glc_hw_cache_extra_regs
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
@@ -713,18 +801,71 @@ static __initconst const u64 glc_hw_cache_extra_regs
 {
  [ C(LL  ) ] = {
 	[ C(OP_READ) ] = {
-		[ C(RESULT_ACCESS) ] = 0x10001,
-		[ C(RESULT_MISS)   ] = 0x3fbfc00001,
+		[ C(RESULT_ACCESS) ] = 0x10001,		/* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0x3fbfc00001,	/* OCR.DEMAND_DATA_RD.L3_MISS */
 	},
 	[ C(OP_WRITE) ] = {
-		[ C(RESULT_ACCESS) ] = 0x3f3ffc0002,
-		[ C(RESULT_MISS)   ] = 0x3f3fc00002,
+		[ C(RESULT_ACCESS) ] = 0x3f3ffc0002,	/* OCR.DEMAND_RFO.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0x3f3fc00002,	/* OCR.DEMAND_RFO.L3_MISS */
 	},
  },
  [ C(NODE) ] = {
 	[ C(OP_READ) ] = {
-		[ C(RESULT_ACCESS) ] = 0x10c000001,
-		[ C(RESULT_MISS)   ] = 0x3fb3000001,
+		[ C(RESULT_ACCESS) ] = 0x104000001,	/* OCR.DEMAND_DATA_RD.LOCAL_DRAM */
+		[ C(RESULT_MISS)   ] = 0x730000001,	/* OCR.DEMAND_DATA_RD.REMOTE_DRAM */
+	},
+ },
+};
+
+/* ADL P-core (Golden cove) specific extra regs value. */
+static __initconst const u64 adl_glc_hw_cache_extra_regs
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL  ) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x10001,		/* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0x3fbfc00001,	/* OCR.DEMAND_DATA_RD.L3_MISS */
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = 0x10002,		/* OCR.DEMAND_RFO.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0x3fbfc00002,	/* OCR.DEMAND_RFO.L3_MISS */
+	},
+ },
+};
+
+static __initconst const u64 lnc_hw_cache_extra_regs
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL  ) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x10001,		/* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0x9E7FA000001,	/* OCR.DEMAND_DATA_RD.L3_MISS */
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = 0x10002,		/* OCR.DEMAND_RFO.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0x9E7FA000002,	/* OCR.DEMAND_RFO.L3_MISS */
+	},
+ },
+};
+
+/* ARL specific lioncove hw_cache_extra_regs[] variant. */
+static __initconst const u64 arl_lnc_hw_cache_extra_regs
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL  ) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x10001,		/* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0xFE7F8000001,	/* OCR.DEMAND_DATA_RD.L3_MISS */
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = 0x10002,		/* OCR.DEMAND_RFO.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0xFE7F8000002,	/* OCR.DEMAND_RFO.L3_MISS */
 	},
  },
 };
@@ -815,12 +956,29 @@ static __initconst const u64 pnc_hw_cache_extra_regs
 {
  [ C(LL  ) ] = {
 	[ C(OP_READ) ] = {
-		[ C(RESULT_ACCESS) ] = 0x4000000000000001,
-		[ C(RESULT_MISS)   ] = 0xFFFFF000000001,
+		[ C(RESULT_ACCESS) ] = 0x4000000000000001,	/* OMR.DEMAND_DATA_RD.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0xFFFFF000000001,	/* OMR.DEMAND_DATA_RD.L3_MISS */
 	},
 	[ C(OP_WRITE) ] = {
-		[ C(RESULT_ACCESS) ] = 0x4000000000000002,
-		[ C(RESULT_MISS)   ] = 0xFFFFF000000002,
+		[ C(RESULT_ACCESS) ] = 0x4000000000000002, 	/* OMR.DEMAND_RFO.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0xFFFFF000000002,	/* OMR.DEMAND_RFO.L3_MISS */
+	},
+ },
+};
+
+static __initconst const u64 cyc_hw_cache_extra_regs
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL  ) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x4000000000000001,	/* OMR.DEMAND_DATA_RD.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0xFF03F000000001,	/* OMR.DEMAND_DATA_RD.L3_MISS */
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = 0x4000000000000002, 	/* OMR.DEMAND_RFO.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0xFF03F000000002,	/* OMR.DEMAND_RFO.L3_MISS */
 	},
  },
 };
@@ -1019,6 +1177,41 @@ static __initconst const u64 skl_hw_cache_extra_regs
  },
 };
 
+static __initconst const u64 snc_hw_cache_extra_regs
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] =
+{
+ [ C(LL  ) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x10001,		/* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0x3FBFC00001,	/* OCR.DEMAND_DATA_RD.L3_MISS */
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = 0x3F3FFC0002,	/* OCR.DEMAND_RFO.ANY_RESPONSE */
+		[ C(RESULT_MISS)   ] = 0x3F3FC00002,	/* OCR.DEMAND_RFO.L3_MISS */
+	},
+	[ C(OP_PREFETCH) ] = {
+		[ C(RESULT_ACCESS) ] = 0x0,
+		[ C(RESULT_MISS)   ] = 0x0,
+	},
+ },
+ [ C(NODE) ] = {
+	[ C(OP_READ) ] = {
+		[ C(RESULT_ACCESS) ] = 0x104000001,	/* OCR.DEMAND_DATA_RD.LOCAL_DRAM */
+		[ C(RESULT_MISS)   ] = 0x730000001,	/* OCR.DEMAND_DATA_RD.REMOTE_DRAM */
+	},
+	[ C(OP_WRITE) ] = {
+		[ C(RESULT_ACCESS) ] = 0x104000002,	/* OCR.DEMAND_RFO.LOCAL_DRAM */
+		[ C(RESULT_MISS)   ] = 0x730000002,	/* OCR.DEMAND_RFO.REMOTE_DRAM */
+	},
+	[ C(OP_PREFETCH) ] = {
+		[ C(RESULT_ACCESS) ] = 0x0,
+		[ C(RESULT_MISS)   ] = 0x0,
+	},
+ },
+};
+
 #define SNB_DMND_DATA_RD	(1ULL << 0)
 #define SNB_DMND_RFO		(1ULL << 1)
 #define SNB_DMND_IFETCH		(1ULL << 2)
@@ -2342,22 +2535,82 @@ static __initconst const u64 tnt_hw_cache_extra_regs
 	},
 };
 
-static __initconst const u64 arw_hw_cache_extra_regs
+static __initconst const u64 grt_hw_cache_extra_regs
 				[PERF_COUNT_HW_CACHE_MAX]
 				[PERF_COUNT_HW_CACHE_OP_MAX]
 				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 	[C(LL)] = {
 		[C(OP_READ)] = {
-			[C(RESULT_ACCESS)]	= 0x4000000000000001,
-			[C(RESULT_MISS)]	= 0xFFFFF000000001,
+			[C(RESULT_ACCESS)]	= 0x10001,	/* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0x3F84400001,	/* OCR.DEMAND_DATA_RD.L3_MISS */
 		},
 		[C(OP_WRITE)] = {
-			[C(RESULT_ACCESS)]	= 0x4000000000000002,
-			[C(RESULT_MISS)]	= 0xFFFFF000000002,
+			[C(RESULT_ACCESS)]	= 0x10002,	/* OCR.DEMAND_RFO.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0x3F84400002,	/* OCR.DEMAND_RFO.L3_MISS */
 		},
-		[C(OP_PREFETCH)] = {
-			[C(RESULT_ACCESS)]	= 0x0,
-			[C(RESULT_MISS)]	= 0x0,
+	},
+};
+
+static __initconst const u64 cmt_hw_cache_extra_regs
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	[C(LL)] = {
+		[C(OP_READ)] = {
+			[C(RESULT_ACCESS)]	= 0x10001,	/* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0x3fbfc00001,	/* OCR.DEMAND_DATA_RD.L3_MISS */
+		},
+		[C(OP_WRITE)] = {
+			[C(RESULT_ACCESS)]	= 0x10002,	/* OCR.DEMAND_RFO.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0x3fbfc00002,	/* OCR.DEMAND_RFO.L3_MISS */
+		},
+	},
+};
+
+static __initconst const u64 skt_hw_cache_extra_regs
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	[C(LL)] = {
+		[C(OP_READ)] = {
+			[C(RESULT_ACCESS)]	= 0x10001,		/* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0x13FBFC00001,	/* OCR.DEMAND_DATA_RD.L3_MISS */
+		},
+		[C(OP_WRITE)] = {
+			[C(RESULT_ACCESS)]	= 0x10002,		/* OCR.DEMAND_RFO.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0x13FBFC00002,	/* OCR.DEMAND_RFO.L3_MISS */
+		},
+	},
+};
+
+static __initconst const u64 dkt_hw_cache_extra_regs
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	[C(LL)] = {
+		[C(OP_READ)] = {
+			[C(RESULT_ACCESS)]	= 0x10001,		/* OCR.DEMAND_DATA_RD.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0x33FBFC00001,	/* OCR.DEMAND_DATA_RD.L3_MISS */
+		},
+		[C(OP_WRITE)] = {
+			[C(RESULT_ACCESS)]	= 0x10002,		/* OCR.DEMAND_RFO.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0x33FBFC00002,	/* OCR.DEMAND_RFO.L3_MISS */
+		},
+	},
+};
+
+static __initconst const u64 arw_hw_cache_extra_regs
+				[PERF_COUNT_HW_CACHE_MAX]
+				[PERF_COUNT_HW_CACHE_OP_MAX]
+				[PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+	[C(LL)] = {
+		[C(OP_READ)] = {
+			[C(RESULT_ACCESS)]	= 0x4000000000000009,	/* OMR.DEMAND_DATA_RD.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0xFF03F000000009,	/* OMR.DEMAND_DATA_RD.L3_MISS */
+		},
+		[C(OP_WRITE)] = {
+			[C(RESULT_ACCESS)]	= 0x400000000000000A,	/* OMR.DEMAND_RFO.ANY_RESPONSE */
+			[C(RESULT_MISS)]	= 0xFF03F00000000A,	/* OMR.DEMAND_RFO.L3_MISS */
 		},
 	},
 };
@@ -2392,9 +2645,12 @@ static struct attribute *grt_mem_attrs[] = {
 };
 
 static struct extra_reg intel_grt_extra_regs[] __read_mostly = {
-	/* must define OFFCORE_RSP_X first, see intel_fixup_er() */
-	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffffffffull, RSP_0),
-	INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x3fffffffffull, RSP_1),
+	/*
+	 * Must define OFFCORE_RSP_X first, see intel_fixup_er().
+	 * Bit 63 only valid on OFFCORE_RSP_0 MSR.
+	 */
+	INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x8003f03fffffffffull, RSP_0),
+	INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0x3f03fffffffffull, RSP_1),
 	INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x5d0),
 	EVENT_EXTRA_END
 };
@@ -2426,10 +2682,6 @@ static struct extra_reg intel_arw_extra_regs[] __read_mostly = {
 	INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OMR_1, 0xc0ffffffffffffffull, OMR_1),
 	INTEL_UEVENT_EXTRA_REG(0x04b7, MSR_OMR_2, 0xc0ffffffffffffffull, OMR_2),
 	INTEL_UEVENT_EXTRA_REG(0x08b7, MSR_OMR_3, 0xc0ffffffffffffffull, OMR_3),
-	INTEL_UEVENT_EXTRA_REG(0x01d4, MSR_OMR_0, 0xc0ffffffffffffffull, OMR_0),
-	INTEL_UEVENT_EXTRA_REG(0x02d4, MSR_OMR_1, 0xc0ffffffffffffffull, OMR_1),
-	INTEL_UEVENT_EXTRA_REG(0x04d4, MSR_OMR_2, 0xc0ffffffffffffffull, OMR_2),
-	INTEL_UEVENT_EXTRA_REG(0x08d4, MSR_OMR_3, 0xc0ffffffffffffffull, OMR_3),
 	INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x5d0),
 	INTEL_UEVENT_EXTRA_REG(0x0127, MSR_SNOOP_RSP_0, 0xffffffffffffffffull, SNOOP_0),
 	INTEL_UEVENT_EXTRA_REG(0x0227, MSR_SNOOP_RSP_1, 0xffffffffffffffffull, SNOOP_1),
@@ -3169,10 +3421,10 @@ static void intel_pmu_config_acr(int idx, u64 mask, u32 reload)
 		wrmsrl(msr_b + msr_offset, mask);
 		cpuc->acr_cfg_b[idx] = mask;
 	}
-	/* Only need to update the reload value when there is a valid config value. */
-	if (mask && cpuc->acr_cfg_c[idx] != reload) {
+	/* Only update CFG_C reload when ACR is actively enabled (mask != 0) */
+	if (mask && ((cpuc->cfg_c_val[idx] & ARCH_PEBS_RELOAD) != reload)) {
 		wrmsrl(msr_c + msr_offset, reload);
-		cpuc->acr_cfg_c[idx] = reload;
+		cpuc->cfg_c_val[idx] = reload;
 	}
 }
 
@@ -3198,14 +3450,15 @@ static void intel_pmu_enable_event_ext(struct perf_event *event)
 {
 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 	struct hw_perf_event *hwc = &event->hw;
-	union arch_pebs_index old, new;
-	struct arch_pebs_cap cap;
 	u64 ext = 0;
 
-	cap = hybrid(cpuc->pmu, arch_pebs_cap);
+	if (is_acr_event_group(event))
+		ext |= (-hwc->sample_period) & ARCH_PEBS_RELOAD;
 
 	if (event->attr.precise_ip) {
 		u64 pebs_data_cfg = intel_get_arch_pebs_data_config(event);
+		struct arch_pebs_cap cap = hybrid(cpuc->pmu, arch_pebs_cap);
+		union arch_pebs_index old, new;
 
 		ext |= ARCH_PEBS_EN;
 		if (hwc->flags & PERF_X86_EVENT_AUTO_RELOAD)
@@ -7490,6 +7743,15 @@ static __always_inline void intel_pmu_init_glc(struct pmu *pmu)
 	intel_pmu_ref_cycles_ext();
 }
 
+static __always_inline void intel_pmu_init_glc_hybrid(struct pmu *pmu)
+{
+	intel_pmu_init_glc(pmu);
+
+	/* ADL has different extra MSR values from Server for the L3 or node OCR/OMR events. */
+	memcpy(hybrid_var(pmu, hw_cache_event_ids), adl_glc_hw_cache_event_ids, sizeof(hw_cache_event_ids));
+	memcpy(hybrid_var(pmu, hw_cache_extra_regs), adl_glc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+}
+
 static __always_inline void intel_pmu_init_grt(struct pmu *pmu)
 {
 	x86_pmu.mid_ack = true;
@@ -7502,7 +7764,7 @@ static __always_inline void intel_pmu_init_grt(struct pmu *pmu)
 	x86_pmu.flags |= PMU_FL_INSTR_LATENCY;
 
 	memcpy(hybrid_var(pmu, hw_cache_event_ids), glp_hw_cache_event_ids, sizeof(hw_cache_event_ids));
-	memcpy(hybrid_var(pmu, hw_cache_extra_regs), tnt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+	memcpy(hybrid_var(pmu, hw_cache_extra_regs), grt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
 	hybrid_var(pmu, hw_cache_event_ids)[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
 	hybrid(pmu, event_constraints) = intel_grt_event_constraints;
 	hybrid(pmu, pebs_constraints) = intel_grt_pebs_event_constraints;
@@ -7511,12 +7773,24 @@ static __always_inline void intel_pmu_init_grt(struct pmu *pmu)
 	intel_pmu_ref_cycles_ext();
 }
 
+static __always_inline void intel_pmu_init_cmt(struct pmu *pmu)
+{
+	intel_pmu_init_grt(pmu);
+	memcpy(hybrid_var(pmu, hw_cache_extra_regs),
+	       cmt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+	hybrid(pmu, pebs_constraints) = intel_cmt_pebs_event_constraints;
+	hybrid(pmu, extra_regs) = intel_cmt_extra_regs;
+}
+
 static __always_inline void intel_pmu_init_lnc(struct pmu *pmu)
 {
 	intel_pmu_init_glc(pmu);
 	hybrid(pmu, event_constraints) = intel_lnc_event_constraints;
 	hybrid(pmu, pebs_constraints) = intel_lnc_pebs_event_constraints;
 	hybrid(pmu, extra_regs) = intel_lnc_extra_regs;
+
+	memcpy(hybrid_var(pmu, hw_cache_event_ids), adl_glc_hw_cache_event_ids, sizeof(hw_cache_event_ids));
+	memcpy(hybrid_var(pmu, hw_cache_extra_regs), lnc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
 }
 
 static __always_inline void intel_pmu_init_pnc(struct pmu *pmu)
@@ -7534,14 +7808,41 @@ static __always_inline void intel_pmu_init_pnc(struct pmu *pmu)
 	static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
 }
 
+static __always_inline void intel_pmu_init_cyc(struct pmu *pmu)
+{
+	intel_pmu_init_pnc(pmu);
+	memcpy(hybrid_var(pmu, hw_cache_extra_regs),
+	       cyc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+}
+
 static __always_inline void intel_pmu_init_skt(struct pmu *pmu)
 {
-	intel_pmu_init_grt(pmu);
+	intel_pmu_init_cmt(pmu);
 	hybrid(pmu, event_constraints) = intel_skt_event_constraints;
-	hybrid(pmu, extra_regs) = intel_cmt_extra_regs;
+	memcpy(hybrid_var(pmu, hw_cache_extra_regs),
+	       skt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
 	static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
 }
 
+/* Hybrid client variant. */
+static __always_inline void intel_pmu_init_dkt_hybrid(struct pmu *pmu)
+{
+	intel_pmu_init_skt(pmu);
+	hybrid(pmu, pebs_constraints) = intel_dkt_pebs_event_constraints;
+}
+
+/*
+ * Darkmont is used by the CWF and PTL E-cores, but their L3 OCR
+ * events require different extra MSR values. Keep a separate init
+ * function for the non-hybrid server variant.
+ */
+static __always_inline void intel_pmu_init_dkt(struct pmu *pmu)
+{
+	intel_pmu_init_dkt_hybrid(pmu);
+	memcpy(hybrid_var(pmu, hw_cache_extra_regs),
+	       dkt_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+}
+
 static __always_inline void intel_pmu_init_arw(struct pmu *pmu)
 {
 	intel_pmu_init_grt(pmu);
@@ -7550,7 +7851,7 @@ static __always_inline void intel_pmu_init_arw(struct pmu *pmu)
 	memcpy(hybrid_var(pmu, hw_cache_extra_regs),
 	       arw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
 	hybrid(pmu, event_constraints) = intel_arw_event_constraints;
-	hybrid(pmu, pebs_constraints) = intel_arw_pebs_event_constraints;
+	hybrid(pmu, pebs_constraints) = intel_dkt_pebs_event_constraints;
 	hybrid(pmu, extra_regs) = intel_arw_extra_regs;
 	static_call_update(intel_pmu_enable_acr_event, intel_pmu_enable_acr);
 }
@@ -7864,8 +8165,7 @@ __init int intel_pmu_init(void)
 
 	case INTEL_ATOM_CRESTMONT:
 	case INTEL_ATOM_CRESTMONT_X:
-		intel_pmu_init_grt(NULL);
-		x86_pmu.extra_regs = intel_cmt_extra_regs;
+		intel_pmu_init_cmt(NULL);
 		intel_pmu_pebs_data_source_cmt();
 		x86_pmu.pebs_latency_data = cmt_latency_data;
 		x86_pmu.get_event_constraints = cmt_get_event_constraints;
@@ -7877,7 +8177,7 @@ __init int intel_pmu_init(void)
 		break;
 
 	case INTEL_ATOM_DARKMONT_X:
-		intel_pmu_init_skt(NULL);
+		intel_pmu_init_dkt(NULL);
 		intel_pmu_pebs_data_source_cmt();
 		x86_pmu.pebs_latency_data = cmt_latency_data;
 		x86_pmu.get_event_constraints = cmt_get_event_constraints;
@@ -8153,17 +8453,19 @@ __init int intel_pmu_init(void)
 
 	case INTEL_ICELAKE_X:
 	case INTEL_ICELAKE_D:
+		memcpy(hw_cache_extra_regs, snc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
 		x86_pmu.pebs_ept = 1;
 		pmem = true;
-		fallthrough;
+		goto snc_common;
 	case INTEL_ICELAKE_L:
 	case INTEL_ICELAKE:
 	case INTEL_TIGERLAKE_L:
 	case INTEL_TIGERLAKE:
 	case INTEL_ROCKETLAKE:
+		memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+	snc_common:
 		x86_pmu.late_ack = true;
 		memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids));
-		memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
 		hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
 		intel_pmu_lbr_init_skl();
 
@@ -8259,7 +8561,7 @@ __init int intel_pmu_init(void)
 
 		/* Initialize big core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
-		intel_pmu_init_glc(&pmu->pmu);
+		intel_pmu_init_glc_hybrid(&pmu->pmu);
 		if (cpu_feature_enabled(X86_FEATURE_HYBRID_CPU)) {
 			pmu->cntr_mask64 <<= 2;
 			pmu->cntr_mask64 |= 0x3;
@@ -8316,13 +8618,12 @@ __init int intel_pmu_init(void)
 
 		/* Initialize big core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
-		intel_pmu_init_glc(&pmu->pmu);
+		intel_pmu_init_glc_hybrid(&pmu->pmu);
 		pmu->extra_regs = intel_rwc_extra_regs;
 
 		/* Initialize Atom core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
-		intel_pmu_init_grt(&pmu->pmu);
-		pmu->extra_regs = intel_cmt_extra_regs;
+		intel_pmu_init_cmt(&pmu->pmu);
 
 		intel_pmu_pebs_data_source_mtl();
 		pr_cont("Meteorlake Hybrid events, ");
@@ -8333,16 +8634,50 @@ __init int intel_pmu_init(void)
 	case INTEL_WILDCATLAKE_L:
 		pr_cont("Pantherlake Hybrid events, ");
 		name = "pantherlake_hybrid";
+
+		intel_pmu_init_hybrid(hybrid_big_small);
+
+		/* Initialize big core specific PerfMon capabilities.*/
+		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
+		intel_pmu_init_lnc(&pmu->pmu);
+		/* Initialize Atom core specific PerfMon capabilities.*/
+		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
+		intel_pmu_init_dkt_hybrid(&pmu->pmu);
+
 		goto lnl_common;
 
-	case INTEL_LUNARLAKE_M:
 	case INTEL_ARROWLAKE:
+		pr_cont("Arrowlake Hybrid events, ");
+		name = "arrowlake_hybrid";
+
+		intel_pmu_init_hybrid(hybrid_big_small);
+
+		/* Initialize big core specific PerfMon capabilities.*/
+		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
+		intel_pmu_init_lnc(&pmu->pmu);
+		memcpy(hybrid_var(&pmu->pmu, hw_cache_extra_regs),
+		       arl_lnc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
+		/* Initialize Atom core specific PerfMon capabilities.*/
+		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
+		intel_pmu_init_skt(&pmu->pmu);
+
+		goto lnl_common;
+
+	case INTEL_LUNARLAKE_M:
 		pr_cont("Lunarlake Hybrid events, ");
 		name = "lunarlake_hybrid";
 
-	lnl_common:
 		intel_pmu_init_hybrid(hybrid_big_small);
 
+		/* Initialize big core specific PerfMon capabilities.*/
+		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
+		intel_pmu_init_lnc(&pmu->pmu);
+		/* Initialize Atom core specific PerfMon capabilities.*/
+		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
+		intel_pmu_init_skt(&pmu->pmu);
+
+	lnl_common:
+
 		x86_pmu.pebs_latency_data = lnl_latency_data;
 		x86_pmu.get_event_constraints = mtl_get_event_constraints;
 		x86_pmu.hw_config = adl_hw_config;
@@ -8353,14 +8688,6 @@ __init int intel_pmu_init(void)
 		extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
 			mtl_hybrid_extra_attr_rtm : mtl_hybrid_extra_attr;
 
-		/* Initialize big core specific PerfMon capabilities.*/
-		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
-		intel_pmu_init_lnc(&pmu->pmu);
-
-		/* Initialize Atom core specific PerfMon capabilities.*/
-		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
-		intel_pmu_init_skt(&pmu->pmu);
-
 		intel_pmu_pebs_data_source_lnl();
 		break;
 
@@ -8380,6 +8707,8 @@ __init int intel_pmu_init(void)
 		/* Initialize big core specific PerfMon capabilities. */
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
 		intel_pmu_init_lnc(&pmu->pmu);
+		memcpy(hybrid_var(&pmu->pmu, hw_cache_extra_regs),
+		       arl_lnc_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
 
 		/* Initialize Atom core specific PerfMon capabilities. */
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
@@ -8387,8 +8716,7 @@ __init int intel_pmu_init(void)
 
 		/* Initialize Lower Power Atom specific PerfMon capabilities. */
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_TINY_IDX];
-		intel_pmu_init_grt(&pmu->pmu);
-		pmu->extra_regs = intel_cmt_extra_regs;
+		intel_pmu_init_cmt(&pmu->pmu);
 
 		intel_pmu_pebs_data_source_arl_h();
 		pr_cont("ArrowLake-H Hybrid events, ");
@@ -8413,7 +8741,7 @@ __init int intel_pmu_init(void)
 
 		/* Initialize big core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX];
-		intel_pmu_init_pnc(&pmu->pmu);
+		intel_pmu_init_cyc(&pmu->pmu);
 
 		/* Initialize Atom core specific PerfMon capabilities.*/
 		pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_ATOM_IDX];
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index 7f0d515c07c54..cb72af9b61cee 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -1292,18 +1292,21 @@ struct event_constraint intel_glm_pebs_event_constraints[] = {
 struct event_constraint intel_grt_pebs_event_constraints[] = {
 	/* Allow all events as PEBS with no flags */
 	INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3),
-	INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xf),
+	INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0x3f),
 	EVENT_CONSTRAINT_END
 };
 
-struct event_constraint intel_arw_pebs_event_constraints[] = {
+struct event_constraint intel_cmt_pebs_event_constraints[] = {
+	/* Allow all events as PEBS with no flags */
+	INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0x3),
+	INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff),
+	EVENT_CONSTRAINT_END
+};
+
+struct event_constraint intel_dkt_pebs_event_constraints[] = {
 	/* Allow all events as PEBS with no flags */
 	INTEL_HYBRID_LAT_CONSTRAINT(0x5d0, 0xff),
 	INTEL_HYBRID_LAT_CONSTRAINT(0x6d0, 0xff),
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x01d4, 0x1),
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x02d4, 0x2),
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x04d4, 0x4),
-	INTEL_FLAGS_UEVENT_CONSTRAINT(0x08d4, 0x8),
 	EVENT_CONSTRAINT_END
 };
 
@@ -1500,6 +1503,13 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = {
 	INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL),	/* INST_RETIRED.PREC_DIST */
 	INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL),
 
+	INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1),		/* OCR.* events */
+	INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1),		/* OCR.* events */
+
+	INTEL_FLAGS_UEVENT_CONSTRAINT(0x04a4, 0x1),		/* TOPDOWN.BAD_SPEC_SLOTS */
+	INTEL_FLAGS_UEVENT_CONSTRAINT(0x08a4, 0x1),		/* TOPDOWN.BR_MISPREDICT_SLOTS */
+	INTEL_FLAGS_UEVENT_CONSTRAINT(0x10a4, 0x8),		/* TOPDOWN.MEMORY_BOUND_SLOTS */
+
 	INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0x3fc),
 	INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3),
 	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf),	/* MEM_INST_RETIRED.STLB_MISS_LOADS */
@@ -1509,6 +1519,7 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = {
 	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x42d0, 0xf),	/* MEM_INST_RETIRED.SPLIT_STORES */
 	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x81d0, 0xf),	/* MEM_INST_RETIRED.ALL_LOADS */
 	INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x82d0, 0xf),	/* MEM_INST_RETIRED.ALL_STORES */
+	INTEL_FLAGS_UEVENT_CONSTRAINT(0x87d0, 0x3ff),		/* MEM_INST_RETIRED.ANY */
 
 	INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD_RANGE(0xd1, 0xd4, 0xf),
 
diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index 72f2adcda7c68..cae2e02fe6cc4 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -4,6 +4,7 @@
 #include <linux/types.h>
 
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/perf_event.h>
 #include <asm/msr.h>
 
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index 524668dcf4cc1..eae24bb35dc14 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -322,10 +322,8 @@ struct cpu_hw_events {
 	u64			fixed_ctrl_val;
 	u64			active_fixed_ctrl_val;
 
-	/* Intel ACR configuration */
+	/* Intel ACR/arch-PEBS configuration */
 	u64			acr_cfg_b[X86_PMC_IDX_MAX];
-	u64			acr_cfg_c[X86_PMC_IDX_MAX];
-	/* Cached CFG_C values */
 	u64			cfg_c_val[X86_PMC_IDX_MAX];
 
 	/*
@@ -1712,7 +1710,9 @@ extern struct event_constraint intel_glp_pebs_event_constraints[];
 
 extern struct event_constraint intel_grt_pebs_event_constraints[];
 
-extern struct event_constraint intel_arw_pebs_event_constraints[];
+extern struct event_constraint intel_cmt_pebs_event_constraints[];
+
+extern struct event_constraint intel_dkt_pebs_event_constraints[];
 
 extern struct event_constraint intel_nehalem_pebs_event_constraints[];
 
diff --git a/arch/x86/events/zhaoxin/core.c b/arch/x86/events/zhaoxin/core.c
index 4bdfcf0912001..6ed644fe89aae 100644
--- a/arch/x86/events/zhaoxin/core.c
+++ b/arch/x86/events/zhaoxin/core.c
@@ -13,6 +13,7 @@
 #include <linux/nmi.h>
 
 #include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
 #include <asm/hardirq.h>
 #include <asm/apic.h>
 #include <asm/msr.h>
diff --git a/arch/x86/hyperv/mmu.c b/arch/x86/hyperv/mmu.c
index cfcb60468b01c..2cf1eeaffd6fe 100644
--- a/arch/x86/hyperv/mmu.c
+++ b/arch/x86/hyperv/mmu.c
@@ -63,7 +63,7 @@ static void hyperv_flush_tlb_multi(const struct cpumask *cpus,
 	struct hv_tlb_flush *flush;
 	u64 status;
 	unsigned long flags;
-	bool do_lazy = !info->freed_tables;
+	bool do_lazy = !info->wake_lazy_cpus;
 
 	trace_hyperv_mmu_flush_tlb_multi(cpus, info);
 
@@ -198,7 +198,7 @@ static u64 hyperv_flush_tlb_others_ex(const struct cpumask *cpus,
 
 	flush->hv_vp_set.format = HV_GENERIC_SET_SPARSE_4K;
 	nr_bank = cpumask_to_vpset_skip(&flush->hv_vp_set, cpus,
-			info->freed_tables ? NULL : cpu_is_lazy);
+			info->wake_lazy_cpus ? NULL : cpu_is_lazy);
 	if (nr_bank < 0)
 		return HV_STATUS_INVALID_PARAMETER;
 
diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h
index fab11192c60a6..db42b477c41d3 100644
--- a/arch/x86/include/asm/acrn.h
+++ b/arch/x86/include/asm/acrn.h
@@ -2,6 +2,8 @@
 #ifndef _ASM_X86_ACRN_H
 #define _ASM_X86_ACRN_H
 
+#include <asm/cpuid/api.h>
+
 /*
  * This CPUID returns feature bitmaps in EAX.
  * Guest VM uses this to detect the appropriate feature bit.
diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h
index 80c1696d8d597..23ab054382695 100644
--- a/arch/x86/include/asm/bug.h
+++ b/arch/x86/include/asm/bug.h
@@ -153,7 +153,7 @@ struct arch_va_list {
 	struct sysv_va_list args;
 };
 extern void *__warn_args(struct arch_va_list *args, struct pt_regs *regs);
-#endif /* __ASSEMBLER__ */
+static __always_inline __printf(1, 2) void __WARN_validate_printf(const char *fmt, ...) { }
 
 #define __WARN_bug_entry(flags, format) ({				\
 	struct bug_entry *bug;						\
@@ -172,6 +172,7 @@ extern void *__warn_args(struct arch_va_list *args, struct pt_regs *regs);
 #define __WARN_print_arg(flags, format, arg...)				\
 do {									\
 	int __flags = (flags) | BUGFLAG_WARNING | BUGFLAG_ARGS ;	\
+	__WARN_validate_printf(format, ## arg);				\
 	static_call_mod(WARN_trap)(__WARN_bug_entry(__flags, format), ## arg); \
 	asm (""); /* inhibit tail-call optimization */			\
 } while (0)
@@ -187,6 +188,7 @@ do {									\
 	}								\
 	__ret_warn_on;							\
 })
+#endif /* __ASSEMBLER__ */
 
 #endif /* HAVE_ARCH_BUG_FORMAT_ARGS */
 
diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h
index 1d506e5d6f46a..35a2a0f9ab32e 100644
--- a/arch/x86/include/asm/cpufeatures.h
+++ b/arch/x86/include/asm/cpufeatures.h
@@ -379,6 +379,7 @@
 #define X86_FEATURE_AVIC		(15*32+13) /* "avic" Virtual Interrupt Controller */
 #define X86_FEATURE_V_VMSAVE_VMLOAD	(15*32+15) /* "v_vmsave_vmload" Virtual VMSAVE VMLOAD */
 #define X86_FEATURE_VGIF		(15*32+16) /* "vgif" Virtual GIF */
+#define X86_FEATURE_GMET		(15*32+17) /* Guest Mode Execution Trap */
 #define X86_FEATURE_X2AVIC		(15*32+18) /* "x2avic" Virtual x2apic */
 #define X86_FEATURE_V_SPEC_CTRL		(15*32+20) /* "v_spec_ctrl" Virtual SPEC_CTRL */
 #define X86_FEATURE_VNMI		(15*32+25) /* "vnmi" Virtual NMI */
diff --git a/arch/x86/include/asm/cpuid/api.h b/arch/x86/include/asm/cpuid/api.h
index 44fa82e1267ce..82eddfa2347b3 100644
--- a/arch/x86/include/asm/cpuid/api.h
+++ b/arch/x86/include/asm/cpuid/api.h
@@ -7,6 +7,7 @@
 #include <linux/build_bug.h>
 #include <linux/types.h>
 
+#include <asm/processor.h>
 #include <asm/string.h>
 
 /*
@@ -131,12 +132,12 @@ static inline void __cpuid_read(u32 leaf, u32 subleaf, u32 *regs)
 	__cpuid(regs + CPUID_EAX, regs + CPUID_EBX, regs + CPUID_ECX, regs + CPUID_EDX);
 }
 
-#define cpuid_subleaf(leaf, subleaf, regs) {		\
+#define cpuid_read_subleaf(leaf, subleaf, regs) {	\
 	static_assert(sizeof(*(regs)) == 16);		\
 	__cpuid_read(leaf, subleaf, (u32 *)(regs));	\
 }
 
-#define cpuid_leaf(leaf, regs) {			\
+#define cpuid_read(leaf, regs) {			\
 	static_assert(sizeof(*(regs)) == 16);		\
 	__cpuid_read(leaf, 0, (u32 *)(regs));		\
 }
@@ -228,7 +229,7 @@ static inline u32 cpuid_base_hypervisor(const char *sig, u32 leaves)
  */
 static inline void cpuid_leaf_0x2(union leaf_0x2_regs *regs)
 {
-	cpuid_leaf(0x2, regs);
+	cpuid_read(0x2, regs);
 
 	/*
 	 * All Intel CPUs must report an iteration count of 1.	In case
@@ -289,4 +290,250 @@ static inline bool cpuid_amd_hygon_has_l3_cache(void)
 	return cpuid_edx(0x80000006);
 }
 
+/*
+ * 'struct cpuid_leaves' accessors (without sanity checks):
+ *
+ * For internal use by the CPUID parser.
+ */
+
+/* Return constified pointers for all call-site APIs */
+#define __const_ptr(_ptr)							\
+	((const __typeof__(*(_ptr)) *)(_ptr))
+
+#define __cpuid_leaves_subleaf(_leaves, _leaf, _subleaf)			\
+	__const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## _subleaf)[0])
+
+#define __cpuid_leaves_subleaf_n(_leaves, _leaf, _index)			\
+	__const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## n)[_index])
+
+#define __cpuid_leaves_subleaf_info(_leaves, _leaf, _subleaf)			\
+	__const_ptr(&((_leaves)->leaf_ ## _leaf ## _ ## _subleaf ## _ ## info))
+
+/*
+ * 'struct cpuid_table' accessors (with sanity checks):
+ *
+ * For internal use by the CPUID parser.
+ */
+
+#define __cpuid_table_nr_filled_subleaves(_table, _leaf, _subleaf)		\
+	__cpuid_leaves_subleaf_info(&((_table)->leaves), _leaf, _subleaf)->nr_entries
+
+#define __cpuid_table_subleaf_range_size(_table, _leaf)				\
+	ARRAY_SIZE((_table)->leaves.leaf_ ## _leaf ## _n)
+
+#define __cpuid_table_invalid_subleaf(_table, _leaf, _subleaf)			\
+	(((_subleaf) < (__cpuid_leaf_first_subleaf(_leaf))) ||			\
+	 ((_subleaf) > (__cpuid_leaf_first_subleaf(_leaf) +			\
+			__cpuid_table_subleaf_range_size(_table, _leaf) - 1)))
+
+/* Return NULL if the parser did not fill that leaf.  Check cpuid_subleaf(). */
+#define __cpuid_table_subleaf(_table, _leaf, _subleaf)						\
+({												\
+	unsigned int ____f = __cpuid_table_nr_filled_subleaves(_table, _leaf, _subleaf);	\
+												\
+	(____f != 1) ? NULL : __cpuid_leaves_subleaf(&((_table)->leaves), _leaf, _subleaf);	\
+})
+
+/*
+ * Return NULL if the CPUID parser did not fill this leaf, or if the given
+ * dynamic subleaf value is out of range.  Check cpuid_subleaf_n().
+ */
+#define __cpuid_table_subleaf_n(_table, _leaf, _subleaf)					\
+({												\
+	unsigned int ____i = (_subleaf) - __cpuid_leaf_first_subleaf(_leaf);			\
+	unsigned int ____f = __cpuid_table_nr_filled_subleaves(_table, _leaf, n);		\
+												\
+	/* CPUID parser might not have filled the entire subleaf range */			\
+	((____i >= ____f) || __cpuid_table_invalid_subleaf(_table, _leaf, _subleaf)) ?		\
+		NULL : __cpuid_leaves_subleaf_n(&((_table)->leaves), _leaf, ____i);		\
+})
+
+/*
+ * Compile-time checks for leaves with a subleaf range:
+ */
+
+#define __cpuid_assert_subleaf_range(_cpuinfo, _leaf)						\
+	static_assert(__cpuid_table_subleaf_range_size(&(_cpuinfo)->cpuid, _leaf) > 1)
+
+#define __cpuid_assert_subleaf_within_range(_cpuinfo, _leaf, _subleaf)				\
+	BUILD_BUG_ON(__builtin_constant_p(_subleaf) &&						\
+		     __cpuid_table_invalid_subleaf(&(_cpuinfo)->cpuid, _leaf, _subleaf))
+
+/*
+ *                     CPUID Parser Call-site APIs
+ *
+ * Call sites should use below APIs instead of invoking direct CPUID queries.
+ *
+ * Benefits include:
+ *
+ * - Return CPUID output as typed C structures that are auto-generated from a
+ *   centralized database (see <asm/cpuid/leaf_types.h).  Such data types have a
+ *   full C99 bitfield layout per CPUID leaf/subleaf combination.  Call sites
+ *   can thus avoid doing ugly and cryptic bitwise operations on raw CPUID data.
+ *
+ * - Return cached, per-CPU, CPUID output.  Below APIs do not invoke any CPUID
+ *   queries, thus avoiding their side effects like serialization and VM exits.
+ *   Call-site-specific hard coded constants and macros for caching CPUID query
+ *   outputs can also be avoided.
+ *
+ * - Return sanitized CPUID data.  Below APIs return NULL if the given CPUID
+ *   leaf/subleaf input is not supported by hardware, or if the hardware CPUID
+ *   output was deemed invalid by the CPUID parser.  This centralizes all CPUID
+ *   data sanitization in one place (the kernel's CPUID parser.)
+ *
+ * - A centralized global view of system CPUID data.  Below APIs will reflect
+ *   any kernel-enforced feature masking or overrides, unlike ad hoc parsing of
+ *   raw CPUID output by drivers and individual call sites.
+ */
+
+/*
+ * Call-site APIs for CPUID leaves with a single subleaf:
+ */
+
+/**
+ * cpuid_subleaf() - Access parsed CPUID
+ * @_cpuinfo:	CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf:	CPUID leaf, in compile-time 0xN format; e.g. 0x7, 0xf
+ * @_subleaf:	CPUID subleaf, in compile-time decimal format; e.g. 0, 1, 3
+ *
+ * Returns a pointer to parsed CPUID output, from the CPUID table inside
+ * @_cpuinfo, as a <cpuid/leaf_types.h> data type: 'struct leaf_0xM_N', where
+ * 0xM is the token provided at @_leaf, and N is the token provided at
+ * @_subleaf; e.g. struct leaf_0x7_0.
+ *
+ * Returns NULL if the requested CPUID @_leaf/@_subleaf query output is not
+ * present at the parsed CPUID table inside @_cpuinfo.  This can happen if:
+ *
+ * - The CPUID table inside @_cpuinfo has not yet been populated.
+ * - The CPUID table inside @_cpuinfo was populated, but the CPU does not
+ *   implement the requested CPUID @_leaf/@_subleaf combination.
+ * - The CPUID table inside @_cpuinfo was populated, but the kernel's CPUID
+ *   parser has predetermined that the requested CPUID @_leaf/@_subleaf
+ *   hardware output is invalid or unsupported.
+ *
+ * Example usage::
+ *
+ *	const struct leaf_0x7_0 *l7_0 = cpuid_subleaf(c, 0x7, 0);
+ *	if (!l7_0) {
+ *		// Handle error
+ *	}
+ *
+ *	const struct leaf_0x7_1 *l7_1 = cpuid_subleaf(c, 0x7, 1);
+ *	if (!l7_1) {
+ *		// Handle error
+ *	}
+ */
+#define cpuid_subleaf(_cpuinfo, _leaf, _subleaf)				\
+	__cpuid_table_subleaf(&(_cpuinfo)->cpuid, _leaf, _subleaf)		\
+
+/**
+ * cpuid_leaf() - Access parsed CPUID data
+ * @_cpuinfo:	CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf:	CPUID leaf, in compile-time 0xN format; e.g. 0x0, 0x2, 0x80000000
+ *
+ * Similar to cpuid_subleaf(), but with a CPUID subleaf = 0.
+ *
+ * Example usage::
+ *
+ *	const struct leaf_0x0_0 *l0 = cpuid_leaf(c, 0x0);
+ *	if (!l0) {
+ *		// Handle error
+ *	}
+ *
+ *	const struct leaf_0x80000000_0 *el0 = cpuid_leaf(c, 0x80000000);
+ *	if (!el0) {
+ *		// Handle error
+ *	}
+ */
+#define cpuid_leaf(_cpuinfo, _leaf)						\
+	cpuid_subleaf(_cpuinfo, _leaf, 0)
+
+/**
+ * cpuid_leaf_raw() - Access parsed CPUID data in raw format
+ * @_cpuinfo:	CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf:	CPUID leaf, in compile-time 0xN format
+ *
+ * Similar to cpuid_leaf(), but returns a raw 'struct cpuid_regs' pointer to
+ * the parsed CPUID data instead of a "typed" <asm/cpuid/leaf_types.h> pointer.
+ */
+#define cpuid_leaf_raw(_cpuinfo, _leaf)						\
+	((const struct cpuid_regs *)(cpuid_leaf(_cpuinfo, _leaf)))
+
+/*
+ * Call-site APIs for CPUID leaves with a subleaf range:
+ */
+
+/**
+ * cpuid_subleaf_n() - Access parsed CPUID data for leaf with a subleaf range
+ * @_cpuinfo:	CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf:	CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d
+ * @_subleaf:	Subleaf number, which can be passed dynamically.  It must be smaller
+ *		than cpuid_subleaf_count(@_cpuinfo, @_leaf).
+ *
+ * Build-time errors will be emitted in the following cases:
+ *
+ * - @_leaf has no subleaf range.  Leaves with a subleaf range have an '_n' type
+ *   suffix and are listed at <asm/cpuid/types.h> using the CPUID_LEAF_N() macro.
+ *
+ * - @_subleaf is known at compile-time but is out of range.
+ *
+ * Example usage::
+ *
+ *	const struct leaf_0x4_n *l4;
+ *
+ *	for (int i = 0; i < cpuid_subleaf_count(c, 0x4); i++) {
+ *		l4 = cpuid_subleaf_n(c, 0x4, i);
+ *		if (!l4) {
+ *			// Handle error
+ *		}
+ *		...
+ *	}
+ *
+ * Beside the standard error situations detailed at cpuid_subleaf(), this
+ * macro will also return NULL if @_subleaf is out of the leaf's subleaf range.
+ */
+#define cpuid_subleaf_n(_cpuinfo, _leaf, _subleaf)				\
+({										\
+	__cpuid_assert_subleaf_range(_cpuinfo, _leaf);				\
+	__cpuid_assert_subleaf_within_range(_cpuinfo, _leaf, _subleaf);		\
+	__cpuid_table_subleaf_n(&(_cpuinfo)->cpuid, _leaf, _subleaf);		\
+})
+
+/**
+ * cpuid_subleaf_n_raw() - Access parsed CPUID data for leaf with subleaf range
+ * @_cpuinfo:	CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf:	CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d
+ * @_subleaf:	Subleaf number, which can be passed dynamically.  It must be smaller
+ *		than cpuid_subleaf_count(@_cpuinfo, @_leaf).
+ *
+ * Similar to cpuid_subleaf_n(), but returns a raw 'struct cpuid_regs' pointer to
+ * the parsed CPUID data instead of a "typed" <asm/cpuid/leaf_types.h> pointer.
+ */
+#define cpuid_subleaf_n_raw(_cpuinfo, _leaf, _subleaf)				\
+	((const struct cpuid_regs *)cpuid_subleaf_n(_cpuinfo, _leaf, _subleaf))
+
+/**
+ * cpuid_subleaf_count() - Number of filled subleaves for @_leaf
+ * @_cpuinfo:	CPU capability structure reference ('struct cpuinfo_x86')
+ * @_leaf:	CPUID leaf, in compile-time 0xN format; e.g. 0x4, 0x8000001d
+ *
+ * Return the number of subleaves filled by the CPUID parser for @_leaf.
+ *
+ * @_leaf must have subleaf range.  Leaves with a subleaf range have an '_n' type
+ * suffix and are listed at <asm/cpuid/types.h> using the CPUID_LEAF_N() macro.
+ */
+#define cpuid_subleaf_count(_cpuinfo, _leaf)					\
+({										\
+	__cpuid_assert_subleaf_range(_cpuinfo, _leaf);				\
+	__cpuid_table_nr_filled_subleaves(&(_cpuinfo)->cpuid, _leaf, n);	\
+})
+
+/*
+ * CPUID parser exported APIs:
+ */
+
+void cpuid_scan_cpu(struct cpuinfo_x86 *c);
+void cpuid_refresh_leaf(struct cpuinfo_x86 *c, u32 leaf);
+void cpuid_refresh_range(struct cpuinfo_x86 *c, u32 start, u32 end);
+
 #endif /* _ASM_X86_CPUID_API_H */
diff --git a/arch/x86/include/asm/cpuid/leaf_types.h b/arch/x86/include/asm/cpuid/leaf_types.h
new file mode 100644
index 0000000000000..5b0008e455e20
--- /dev/null
+++ b/arch/x86/include/asm/cpuid/leaf_types.h
@@ -0,0 +1,2350 @@
+/* SPDX-License-Identifier: MIT */
+/* Generator: x86-cpuid-db v3.0 */
+
+/*
+ * Auto-generated file.
+ * Please submit all updates and bugfixes to https://x86-cpuid.org
+ */
+
+#ifndef _ASM_X86_CPUID_LEAF_TYPES
+#define _ASM_X86_CPUID_LEAF_TYPES
+
+#include <linux/types.h>
+
+/*
+ * Leaf 0x0
+ * Maximum standard leaf + CPU vendor string
+ */
+
+struct leaf_0x0_0 {
+	// eax
+	u32	max_std_leaf			: 32; // Highest standard CPUID leaf
+	// ebx
+	u32	cpu_vendorid_0			: 32; // CPU vendor ID string bytes 0 - 3
+	// ecx
+	u32	cpu_vendorid_2			: 32; // CPU vendor ID string bytes 8 - 11
+	// edx
+	u32	cpu_vendorid_1			: 32; // CPU vendor ID string bytes 4 - 7
+};
+
+/*
+ * Leaf 0x1
+ * CPU FMS (Family/Model/Stepping) + standard feature flags
+ */
+
+struct leaf_0x1_0 {
+	// eax
+	u32	stepping			:  4, // Stepping ID
+		base_model			:  4, // Base CPU model ID
+		base_family_id			:  4, // Base CPU family ID
+		cpu_type			:  2, // CPU type
+						:  2, // Reserved
+		ext_model			:  4, // Extended CPU model ID
+		ext_family			:  8, // Extended CPU family ID
+						:  4; // Reserved
+	// ebx
+	u32	brand_id			:  8, // Brand index
+		clflush_size			:  8, // CLFLUSH instruction cache line size
+		n_logical_cpu			:  8, // Logical CPU count
+		local_apic_id			:  8; // Initial local APIC physical ID
+	// ecx
+	u32	sse3				:  1, // Streaming SIMD Extensions 3 (SSE3)
+		pclmulqdq			:  1, // PCLMULQDQ instruction support
+		dtes64				:  1, // 64-bit DS save area
+		monitor				:  1, // MONITOR/MWAIT support
+		dscpl				:  1, // CPL Qualified Debug Store
+		vmx				:  1, // Virtual Machine Extensions
+		smx				:  1, // Safer Mode Extensions
+		est				:  1, // Enhanced Intel SpeedStep
+		tm2				:  1, // Thermal Monitor 2
+		ssse3				:  1, // Supplemental SSE3
+		cntxt_id			:  1, // L1 Context ID
+		sdbg				:  1, // Silicon Debug
+		fma				:  1, // FMA extensions using YMM state
+		cx16				:  1, // CMPXCHG16B instruction support
+		xtpr_update			:  1, // xTPR Update Control
+		pdcm				:  1, // Perfmon and Debug Capability
+						:  1, // Reserved
+		pcid				:  1, // Process-context identifiers
+		dca				:  1, // Direct Cache Access
+		sse4_1				:  1, // SSE4.1
+		sse4_2				:  1, // SSE4.2
+		x2apic				:  1, // X2APIC support
+		movbe				:  1, // MOVBE instruction support
+		popcnt				:  1, // POPCNT instruction support
+		tsc_deadline_timer		:  1, // APIC timer one-shot operation
+		aes				:  1, // AES instructions
+		xsave				:  1, // XSAVE (and related instructions) support
+		osxsave				:  1, // XSAVE (and related instructions) are enabled by OS
+		avx				:  1, // AVX instructions support
+		f16c				:  1, // Half-precision floating-point conversion support
+		rdrand				:  1, // RDRAND instruction support
+		guest_status			:  1; // System is running as guest; (para-)virtualized system
+	// edx
+	u32	fpu				:  1, // Floating-Point Unit on-chip (x87)
+		vme				:  1, // Virtual-8086 Mode Extensions
+		de				:  1, // Debugging Extensions
+		pse				:  1, // Page Size Extension
+		tsc				:  1, // Time Stamp Counter
+		msr				:  1, // Model-Specific Registers (RDMSR and WRMSR support)
+		pae				:  1, // Physical Address Extensions
+		mce				:  1, // Machine Check Exception
+		cx8				:  1, // CMPXCHG8B instruction
+		apic				:  1, // APIC on-chip
+						:  1, // Reserved
+		sep				:  1, // SYSENTER, SYSEXIT, and associated MSRs
+		mtrr				:  1, // Memory Type Range Registers
+		pge				:  1, // Page Global Extensions
+		mca				:  1, // Machine Check Architecture
+		cmov				:  1, // Conditional Move Instruction
+		pat				:  1, // Page Attribute Table
+		pse36				:  1, // Page Size Extension (36-bit)
+		psn				:  1, // Processor Serial Number
+		clflush				:  1, // CLFLUSH instruction
+						:  1, // Reserved
+		ds				:  1, // Debug Store
+		acpi				:  1, // Thermal monitor and clock control
+		mmx				:  1, // MMX instructions
+		fxsr				:  1, // FXSAVE and FXRSTOR instructions
+		sse				:  1, // SSE instructions
+		sse2				:  1, // SSE2 instructions
+		selfsnoop			:  1, // Self Snoop
+		htt				:  1, // Hyper-threading
+		tm				:  1, // Thermal Monitor
+		ia64				:  1, // Legacy IA-64 (Itanium) support bit, now reserved
+		pbe				:  1; // Pending Break Enable
+};
+
+/*
+ * Leaf 0x2
+ * Intel cache and TLB information one-byte descriptors
+ */
+
+struct leaf_0x2_0 {
+	// eax
+	u32	iteration_count			:  8, // Number of times this leaf must be queried
+		desc1				:  8, // Descriptor #1
+		desc2				:  8, // Descriptor #2
+		desc3				:  7, // Descriptor #3
+		eax_invalid			:  1; // Descriptors 1-3 are invalid if set
+	// ebx
+	u32	desc4				:  8, // Descriptor #4
+		desc5				:  8, // Descriptor #5
+		desc6				:  8, // Descriptor #6
+		desc7				:  7, // Descriptor #7
+		ebx_invalid			:  1; // Descriptors 4-7 are invalid if set
+	// ecx
+	u32	desc8				:  8, // Descriptor #8
+		desc9				:  8, // Descriptor #9
+		desc10				:  8, // Descriptor #10
+		desc11				:  7, // Descriptor #11
+		ecx_invalid			:  1; // Descriptors 8-11 are invalid if set
+	// edx
+	u32	desc12				:  8, // Descriptor #12
+		desc13				:  8, // Descriptor #13
+		desc14				:  8, // Descriptor #14
+		desc15				:  7, // Descriptor #15
+		edx_invalid			:  1; // Descriptors 12-15 are invalid if set
+};
+
+/*
+ * Leaf 0x4
+ * Intel deterministic cache parameters
+ */
+
+struct leaf_0x4_n {
+	// eax
+	u32	cache_type			:  5, // Cache type field
+		cache_level			:  3, // Cache level (1-based)
+		cache_self_init			:  1, // Self-initializing cache level
+		fully_associative		:  1, // Fully-associative cache
+						:  4, // Reserved
+		num_threads_sharing		: 12, // Number logical CPUs sharing this cache
+		num_cores_on_die		:  6; // Number of cores in the physical package
+	// ebx
+	u32	cache_linesize			: 12, // System coherency line size (0-based)
+		cache_npartitions		: 10, // Physical line partitions (0-based)
+		cache_nways			: 10; // Ways of associativity (0-based)
+	// ecx
+	u32	cache_nsets			: 31, // Cache number of sets (0-based)
+						:  1; // Reserved
+	// edx
+	u32	wbinvd_rll_no_guarantee		:  1, // WBINVD/INVD not guaranteed for Remote Lower-Level caches
+		ll_inclusive			:  1, // Cache is inclusive of Lower-Level caches
+		complex_indexing		:  1, // Not a direct-mapped cache (complex function)
+						: 29; // Reserved
+};
+
+#define LEAF_0x4_SUBLEAF_N_FIRST		0
+#define LEAF_0x4_SUBLEAF_N_LAST			31
+
+/*
+ * Leaf 0x5
+ * MONITOR/MWAIT instructions
+ */
+
+struct leaf_0x5_0 {
+	// eax
+	u32	min_mon_size			: 16, // Smallest monitor-line size, in bytes
+						: 16; // Reserved
+	// ebx
+	u32	max_mon_size			: 16, // Largest monitor-line size, in bytes
+						: 16; // Reserved
+	// ecx
+	u32	mwait_ext			:  1, // MONITOR/MWAIT extensions
+		mwait_irq_break			:  1, // Interrupts as a break event for MWAIT
+						: 30; // Reserved
+	// edx
+	u32	n_c0_substates			:  4, // Number of C0 sub C-states
+		n_c1_substates			:  4, // Number of C1 sub C-states
+		n_c2_substates			:  4, // Number of C2 sub C-states
+		n_c3_substates			:  4, // Number of C3 sub C-states
+		n_c4_substates			:  4, // Number of C4 sub C-states
+		n_c5_substates			:  4, // Number of C5 sub C-states
+		n_c6_substates			:  4, // Number of C6 sub C-states
+		n_c7_substates			:  4; // Number of C7 sub C-states
+};
+
+/*
+ * Leaf 0x6
+ * Thermal and power management
+ */
+
+struct leaf_0x6_0 {
+	// eax
+	u32	digital_temp			:  1, // Digital temperature sensor
+		turbo_boost			:  1, // Intel Turbo Boost
+		lapic_timer_always_on		:  1, // Always-Running APIC Timer (not affected by p-state)
+						:  1, // Reserved
+		power_limit_event		:  1, // Power Limit Notification (PLN) event
+		ecmd				:  1, // Clock modulation duty cycle extension
+		package_thermal			:  1, // Package thermal management
+		hwp_base_regs			:  1, // HWP (Hardware P-states) base registers
+		hwp_notify			:  1, // HWP notification (IA32_HWP_INTERRUPT MSR)
+		hwp_activity_window		:  1, // HWP activity window (IA32_HWP_REQUEST[bits 41:32])
+		hwp_energy_perf_pr		:  1, // HWP Energy Performance Preference
+		hwp_package_req			:  1, // HWP Package Level Request
+						:  1, // Reserved
+		hdc_base_regs			:  1, // HDC base registers
+		turbo_boost_3_0			:  1, // Intel Turbo Boost Max 3.0
+		hwp_capabilities		:  1, // HWP Highest Performance change
+		hwp_peci_override		:  1, // HWP PECI override
+		hwp_flexible			:  1, // Flexible HWP
+		hwp_fast			:  1, // IA32_HWP_REQUEST MSR fast access mode
+		hw_feedback			:  1, // HW_FEEDBACK MSRs
+		hwp_ignore_idle			:  1, // Ignoring idle logical CPU HWP request is supported
+						:  1, // Reserved
+		hwp_ctl				:  1, // IA32_HWP_CTL MSR
+		thread_director			:  1, // Intel thread director
+		therm_interrupt_bit25		:  1, // IA32_THERM_INTERRUPT MSR bit 25
+						:  7; // Reserved
+	// ebx
+	u32	n_therm_thresholds		:  4, // Digital thermometer thresholds
+						: 28; // Reserved
+	// ecx
+	u32	aperf_mperf			:  1, // MPERF/APERF MSRs (effective frequency interface)
+						:  2, // Reserved
+		energy_perf_bias		:  1, // IA32_ENERGY_PERF_BIAS MSR
+						:  4, // Reserved
+		thrd_director_nclasses		:  8, // Number of classes, Intel thread director
+						: 16; // Reserved
+	// edx
+	u32	perfcap_reporting		:  1, // Performance capability reporting
+		encap_reporting			:  1, // Energy efficiency capability reporting
+						:  6, // Reserved
+		feedback_sz			:  4, // Feedback interface structure size, in 4K pages
+						:  4, // Reserved
+		this_lcpu_hwfdbk_idx		: 16; // This logical CPU hardware feedback interface index
+};
+
+/*
+ * Leaf 0x7
+ * Extended CPU features
+ */
+
+struct leaf_0x7_0 {
+	// eax
+	u32	leaf7_n_subleaves		: 32; // Number of leaf 0x7 subleaves
+	// ebx
+	u32	fsgsbase			:  1, // FSBASE/GSBASE read/write
+		tsc_adjust			:  1, // IA32_TSC_ADJUST MSR
+		sgx				:  1, // Intel SGX (Software Guard Extensions)
+		bmi1				:  1, // Bit manipulation extensions group 1
+		hle				:  1, // Hardware Lock Elision
+		avx2				:  1, // AVX2 instruction set
+		fdp_excptn_only			:  1, // FPU Data Pointer updated only on x87 exceptions
+		smep				:  1, // Supervisor Mode Execution Protection
+		bmi2				:  1, // Bit manipulation extensions group 2
+		erms				:  1, // Enhanced REP MOVSB/STOSB
+		invpcid				:  1, // INVPCID instruction (Invalidate Processor Context ID)
+		rtm				:  1, // Intel restricted transactional memory
+		pqm				:  1, // Intel RDT-CMT / AMD Platform-QoS cache monitoring
+		zero_fcs_fds			:  1, // Deprecated FPU CS/DS (stored as zero)
+		mpx				:  1, // Intel memory protection extensions
+		rdt_a				:  1, // Intel RDT / AMD Platform-QoS Enforcement
+		avx512f				:  1, // AVX-512 foundation instructions
+		avx512dq			:  1, // AVX-512 double/quadword instructions
+		rdseed				:  1, // RDSEED instruction
+		adx				:  1, // ADCX/ADOX instructions
+		smap				:  1, // Supervisor mode access prevention
+		avx512ifma			:  1, // AVX-512 integer fused multiply add
+						:  1, // Reserved
+		clflushopt			:  1, // CLFLUSHOPT instruction
+		clwb				:  1, // CLWB instruction
+		intel_pt			:  1, // Intel processor trace
+		avx512pf			:  1, // AVX-512 prefetch instructions
+		avx512er			:  1, // AVX-512 exponent/reciprocal instructions
+		avx512cd			:  1, // AVX-512 conflict detection instructions
+		sha				:  1, // SHA/SHA256 instructions
+		avx512bw			:  1, // AVX-512 byte/word instructions
+		avx512vl			:  1; // AVX-512 VL (128/256 vector length) extensions
+	// ecx
+	u32	prefetchwt1			:  1, // PREFETCHWT1 (Intel Xeon Phi only)
+		avx512vbmi			:  1, // AVX-512 Vector byte manipulation instructions
+		umip				:  1, // User mode instruction protection
+		pku				:  1, // Protection keys for user-space
+		ospke				:  1, // OS protection keys enable
+		waitpkg				:  1, // WAITPKG instructions
+		avx512_vbmi2			:  1, // AVX-512 vector byte manipulation instructions group 2
+		cet_ss				:  1, // CET shadow stack features
+		gfni				:  1, // Galois field new instructions
+		vaes				:  1, // Vector AES instructions
+		vpclmulqdq			:  1, // VPCLMULQDQ 256-bit instruction
+		avx512_vnni			:  1, // Vector neural network instructions
+		avx512_bitalg			:  1, // AVX-512 bitwise algorithms
+		tme				:  1, // Intel total memory encryption
+		avx512_vpopcntdq		:  1, // AVX-512: POPCNT for vectors of DWORD/QWORD
+						:  1, // Reserved
+		la57				:  1, // 57-bit linear addresses (five-level paging)
+		mawau_val_lm			:  5, // BNDLDX/BNDSTX MAWAU value in 64-bit mode
+		rdpid				:  1, // RDPID instruction
+		key_locker			:  1, // Intel key locker
+		bus_lock_detect			:  1, // OS bus-lock detection
+		cldemote			:  1, // CLDEMOTE instruction
+						:  1, // Reserved
+		movdiri				:  1, // MOVDIRI instruction
+		movdir64b			:  1, // MOVDIR64B instruction
+		enqcmd				:  1, // Enqueue stores (ENQCMD{,S})
+		sgx_lc				:  1, // Intel SGX launch configuration
+		pks				:  1; // Protection keys for supervisor-mode pages
+	// edx
+	u32					:  1, // Reserved
+		sgx_keys			:  1, // Intel SGX attestation services
+		avx512_4vnniw			:  1, // AVX-512 neural network instructions
+		avx512_4fmaps			:  1, // AVX-512 multiply accumulation single precision
+		fsrm				:  1, // Fast short REP MOV
+		uintr				:  1, // User interrupts
+						:  2, // Reserved
+		avx512_vp2intersect		:  1, // VP2INTERSECT{D,Q} instructions
+		srdbs_ctrl			:  1, // SRBDS mitigation MSR
+		md_clear			:  1, // VERW MD_CLEAR microcode
+		rtm_always_abort		:  1, // XBEGIN (RTM transaction) always aborts
+						:  1, // Reserved
+		tsx_force_abort			:  1, // MSR TSX_FORCE_ABORT, RTM_ABORT bit
+		serialize			:  1, // SERIALIZE instruction
+		hybrid_cpu			:  1, // The CPU is identified as a 'hybrid part'
+		tsxldtrk			:  1, // TSX suspend/resume load address tracking
+						:  1, // Reserved
+		pconfig				:  1, // PCONFIG instruction
+		arch_lbr			:  1, // Intel architectural LBRs
+		cet_ibt				:  1, // CET indirect branch tracking
+						:  1, // Reserved
+		amx_bf16			:  1, // AMX-BF16: tile bfloat16
+		avx512_fp16			:  1, // AVX-512 FP16 instructions
+		amx_tile			:  1, // AMX-TILE: tile architecture
+		amx_int8			:  1, // AMX-INT8: tile 8-bit integer
+		spec_ctrl			:  1, // Speculation Control (IBRS/IBPB: indirect branch restrictions)
+		intel_stibp			:  1, // Single thread indirect branch predictors
+		flush_l1d			:  1, // FLUSH L1D cache: IA32_FLUSH_CMD MSR
+		arch_capabilities		:  1, // Intel IA32_ARCH_CAPABILITIES MSR
+		core_capabilities		:  1, // IA32_CORE_CAPABILITIES MSR
+		spec_ctrl_ssbd			:  1; // Speculative store bypass disable
+};
+
+struct leaf_0x7_1 {
+	// eax
+	u32					:  4, // Reserved
+		avx_vnni			:  1, // AVX-VNNI instructions
+		avx512_bf16			:  1, // AVX-512 bfloat16 instructions
+		lass				:  1, // Linear address space separation
+		cmpccxadd			:  1, // CMPccXADD instructions
+		arch_perfmon_ext		:  1, // ArchPerfmonExt: leaf 0x23
+						:  1, // Reserved
+		fzrm				:  1, // Fast zero-length REP MOVSB
+		fsrs				:  1, // Fast short REP STOSB
+		fsrc				:  1, // Fast Short REP CMPSB/SCASB
+						:  4, // Reserved
+		fred				:  1, // FRED: Flexible return and event delivery transitions
+		lkgs				:  1, // LKGS: Load 'kernel' (userspace) GS
+		wrmsrns				:  1, // WRMSRNS instruction (WRMSR-non-serializing)
+		nmi_src				:  1, // NMI-source reporting with FRED event data
+		amx_fp16			:  1, // AMX-FP16: FP16 tile operations
+		hreset 				:  1, // HRESET (Thread director history reset)
+		avx_ifma			:  1, // Integer fused multiply add
+						:  2, // Reserved
+		lam				:  1, // Linear address masking
+		rd_wr_msrlist			:  1, // RDMSRLIST/WRMSRLIST instructions
+						:  4; // Reserved
+	// ebx
+	u32	intel_ppin			:  1, // Protected processor inventory number (PPIN{,_CTL} MSRs)
+						: 31; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					:  4, // Reserved
+		avx_vnni_int8			:  1, // AVX-VNNI-INT8 instructions
+		avx_ne_convert			:  1, // AVX-NE-CONVERT instructions
+						:  2, // Reserved
+		amx_complex			:  1, // AMX-COMPLEX instructions (starting from Granite Rapids)
+						:  5, // Reserved
+		prefetchit_0_1			:  1, // PREFETCHIT0/1 instructions
+						:  3, // Reserved
+		cet_sss				:  1, // CET supervisor shadow stacks safe to use
+						: 13; // Reserved
+};
+
+struct leaf_0x7_2 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32	intel_psfd			:  1, // Intel predictive store forward disable
+		ipred_ctrl			:  1, // MSR bits IA32_SPEC_CTRL.IPRED_DIS_{U,S}
+		rrsba_ctrl			:  1, // MSR bits IA32_SPEC_CTRL.RRSBA_DIS_{U,S}
+		ddp_ctrl			:  1, // MSR bit  IA32_SPEC_CTRL.DDPD_U
+		bhi_ctrl			:  1, // MSR bit  IA32_SPEC_CTRL.BHI_DIS_S
+		mcdt_no				:  1, // MCDT mitigation not needed
+		uclock_disable			:  1, // UC-lock disable
+						: 25; // Reserved
+};
+
+/*
+ * Leaf 0x9
+ * Intel DCA (Direct Cache Access)
+ */
+
+struct leaf_0x9_0 {
+	// eax
+	u32	dca_enabled_in_bios		:  1, // DCA is enabled in BIOS
+						: 31; // Reserved
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0xa
+ * Intel PMU (Performance Monitoring Unit)
+ */
+
+struct leaf_0xa_0 {
+	// eax
+	u32	pmu_version			:  8, // Performance monitoring unit version ID
+		num_counters_gp			:  8, // Number of general-purpose PMU counters per logical CPU
+		bit_width_gp			:  8, // Bitwidth of PMU general-purpose counters
+		events_mask_len			:  8; // Length of CPUID(0xa).EBX bit vector
+	// ebx
+	u32	no_core_cycle			:  1, // Core cycle event not available
+		no_instruction_retired		:  1, // Instruction retired event not available
+		no_reference_cycles		:  1, // Reference cycles event not available
+		no_llc_reference		:  1, // LLC-reference event not available
+		no_llc_misses			:  1, // LLC-misses event not available
+		no_br_insn_retired		:  1, // Branch instruction retired event not available
+		no_br_misses_retired		:  1, // Branch mispredict retired event not available
+		no_topdown_slots		:  1, // Topdown slots event not available
+		no_backend_bound		:  1, // Topdown backend bound not available
+		no_bad_speculation		:  1, // Topdown bad speculation not available
+		no_frontend_bound		:  1, // Topdown frontend bound not available
+		no_retiring			:  1, // Topdown retiring not available
+		no_lbr_inserts			:  1, // LBR inserts not available
+						: 19; // Reserved
+	// ecx
+	u32	pmu_fcounters_bitmap		: 32; // Fixed-function PMU counters support bitmap
+	// edx
+	u32	num_counters_fixed		:  5, // Number of fixed PMU counters
+		bitwidth_fixed			:  8, // Bitwidth of PMU fixed counters
+						:  2, // Reserved
+		anythread_deprecation		:  1, // AnyThread mode deprecation
+						: 16; // Reserved
+};
+
+/*
+ * Leaf 0xb
+ * CPU extended topology v1
+ */
+
+struct leaf_0xb_n {
+	// eax
+	u32	x2apic_id_shift			:  5, // Bit width of this level (previous levels inclusive)
+						: 27; // Reserved
+	// ebx
+	u32	domain_lcpus_count		: 16, // Logical CPUs count across all instances of this domain
+						: 16; // Reserved
+	// ecx
+	u32	domain_nr			:  8, // This domain level (subleaf ID)
+		domain_type			:  8, // This domain type
+						: 16; // Reserved
+	// edx
+	u32	x2apic_id			: 32; // x2APIC ID of current logical CPU
+};
+
+#define LEAF_0xb_SUBLEAF_N_FIRST		0
+#define LEAF_0xb_SUBLEAF_N_LAST			1
+
+/*
+ * Leaf 0xd
+ * CPU extended state
+ */
+
+struct leaf_0xd_0 {
+	// eax
+	u32	xcr0_x87			:  1, // XCR0.X87
+		xcr0_sse			:  1, // XCR0.SSE
+		xcr0_avx			:  1, // XCR0.AVX
+		xcr0_mpx_bndregs		:  1, // XCR0.BNDREGS: MPX BND0-BND3 registers
+		xcr0_mpx_bndcsr			:  1, // XCR0.BNDCSR: MPX BNDCFGU/BNDSTATUS registers
+		xcr0_avx512_opmask		:  1, // XCR0.OPMASK: AVX-512 k0-k7 registers
+		xcr0_avx512_zmm_hi256		:  1, // XCR0.ZMM_Hi256: AVX-512 ZMM0->ZMM7/15 registers
+		xcr0_avx512_hi16_zmm		:  1, // XCR0.HI16_ZMM: AVX-512 ZMM16->ZMM31 registers
+						:  1, // Reserved
+		xcr0_pkru			:  1, // XCR0.PKRU: XSAVE PKRU registers
+						:  1, // Reserved
+		xcr0_cet_u			:  1, // XCR0.CET_U: CET user state
+		xcr0_cet_s			:  1, // XCR0.CET_S: CET supervisor state
+						:  4, // Reserved
+		xcr0_tileconfig			:  1, // XCR0.TILECONFIG: AMX can manage TILECONFIG
+		xcr0_tiledata			:  1, // XCR0.TILEDATA: AMX can manage TILEDATA
+						: 13; // Reserved
+	// ebx
+	u32	xsave_sz_xcr0			: 32; // XSAVE/XRSTOR area byte size, for XCR0 enabled features
+	// ecx
+	u32	xsave_sz_max			: 32; // XSAVE/XRSTOR area max byte size, all CPU features
+	// edx
+	u32					: 30, // Reserved
+		xcr0_lwp			:  1, // AMD XCR0.LWP: Light-weight Profiling
+						:  1; // Reserved
+};
+
+struct leaf_0xd_1 {
+	// eax
+	u32	xsaveopt			:  1, // XSAVEOPT instruction
+		xsavec				:  1, // XSAVEC instruction
+		xgetbv1				:  1, // XGETBV instruction with ECX = 1
+		xsaves				:  1, // XSAVES/XRSTORS instructions (and XSS MSR)
+		xfd				:  1, // Extended feature disable
+						: 27; // Reserved
+	// ebx
+	u32	xsave_sz_xcr0_xss		: 32; // XSAVES/XSAVEC area byte size, for XCR0|XSS enabled features
+	// ecx
+	u32					:  8, // Reserved
+		xss_pt				:  1, // PT state
+						:  1, // Reserved
+		xss_pasid			:  1, // PASID state
+		xss_cet_u			:  1, // CET user state
+		xss_cet_p			:  1, // CET supervisor state
+		xss_hdc				:  1, // HDC state
+		xss_uintr			:  1, // UINTR state
+		xss_lbr				:  1, // LBR state
+		xss_hwp				:  1, // HWP state
+						: 15; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0xd_n {
+	// eax
+	u32	xsave_sz			: 32; // Subleaf-N feature save area size, in bytes
+	// ebx
+	u32	xsave_offset			: 32; // Subleaf-N feature save area offset, in bytes
+	// ecx
+	u32	is_xss_bit			:  1, // Subleaf N describes an XSS bit (otherwise XCR0)
+		compacted_xsave_64byte_aligned	:  1, // When compacted, subleaf-N XSAVE area is 64-byte aligned
+						: 30; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+#define LEAF_0xd_SUBLEAF_N_FIRST		2
+#define LEAF_0xd_SUBLEAF_N_LAST			63
+
+/*
+ * Leaf 0xf
+ * Intel RDT / AMD PQoS resource monitoring
+ */
+
+struct leaf_0xf_0 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32	core_rmid_max			: 32; // RMID max within this core (0-based)
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					:  1, // Reserved
+		llc_qos_mon			:  1, // LLC QoS-monitoring
+						: 30; // Reserved
+};
+
+struct leaf_0xf_1 {
+	// eax
+	u32	l3c_qm_bitwidth			:  8, // L3 QoS-monitoring counter bitwidth (24-based)
+		l3c_qm_overflow_bit		:  1, // QM_CTR MSR bit 61 is an overflow bit
+		io_rdt_cmt			:  1, // non-CPU agent supporting Intel RDT CMT present
+		io_rdt_mbm			:  1, // non-CPU agent supporting Intel RDT MBM present
+						: 21; // Reserved
+	// ebx
+	u32	l3c_qm_conver_factor		: 32; // QM_CTR MSR conversion factor to bytes
+	// ecx
+	u32	l3c_qm_rmid_max			: 32; // L3 QoS-monitoring max RMID
+	// edx
+	u32	l3c_qm_occupancy		:  1, // L3 QoS occupancy monitoring
+		l3c_qm_mbm_total		:  1, // L3 QoS total bandwidth monitoring
+		l3c_qm_mbm_local		:  1, // L3 QoS local bandwidth monitoring
+						: 29; // Reserved
+};
+
+/*
+ * Leaf 0x10
+ * Intel RDT / AMD PQoS allocation
+ */
+
+struct leaf_0x10_0 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32					:  1, // Reserved
+		cat_l3				:  1, // L3 Cache Allocation Technology
+		cat_l2				:  1, // L2 Cache Allocation Technology
+		mba				:  1, // Memory Bandwidth Allocation
+						: 28; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0x10_n {
+	// eax
+	u32	cat_cbm_len			:  5, // L3/L2_CAT capacity bitmask length, minus-one notation
+						: 27; // Reserved
+	// ebx
+	u32	cat_units_bitmap		: 32; // L3/L2_CAT allocation units bitmap
+	// ecx
+	u32					:  1, // Reserved
+		l3_cat_cos_infreq_updates	:  1, // L3_CAT COS updates should be infrequent
+		cat_cdp_supported		:  1, // L3/L2_CAT Code and Data Prioritization
+		cat_sparse_1s			:  1, // L3/L2_CAT non-contiguous 1s value
+						: 28; // Reserved
+	// edx
+	u32	cat_cos_max			: 16, // L3/L2_CAT max Class of Service
+						: 16; // Reserved
+};
+
+#define LEAF_0x10_SUBLEAF_N_FIRST		1
+#define LEAF_0x10_SUBLEAF_N_LAST		2
+
+struct leaf_0x10_3 {
+	// eax
+	u32	mba_max_delay			: 12, // Max MBA throttling value; minus-one notation
+						: 20; // Reserved
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32	mba_per_thread			:  1, // Per-thread MBA controls
+						:  1, // Reserved
+		mba_delay_linear		:  1, // Delay values are linear
+						: 29; // Reserved
+	// edx
+	u32	mba_cos_max			: 16, // MBA max Class of Service
+						: 16; // Reserved
+};
+
+/*
+ * Leaf 0x12
+ * Intel SGX (Software Guard Extensions)
+ */
+
+struct leaf_0x12_0 {
+	// eax
+	u32	sgx1				:  1, // SGX1 leaf functions
+		sgx2				:  1, // SGX2 leaf functions
+						:  3, // Reserved
+		enclv_leaves			:  1, // ENCLV leaves
+		encls_leaves			:  1, // ENCLS leaves
+		enclu_everifyreport2		:  1, // ENCLU leaf EVERIFYREPORT2
+						:  2, // Reserved
+		encls_eupdatesvn		:  1, // ENCLS leaf EUPDATESVN
+		enclu_edeccssa			:  1, // ENCLU leaf EDECCSSA
+						: 20; // Reserved
+	// ebx
+	u32	miscselect_exinfo		:  1, // SSA.MISC frame: Enclave #PF and #GP reporting
+		miscselect_cpinfo		:  1, // SSA.MISC frame: Enclave #CP reporting
+						: 30; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32	max_enclave_sz_not64		:  8, // Maximum enclave size in non-64-bit mode (log2)
+		max_enclave_sz_64		:  8, // Maximum enclave size in 64-bit mode (log2)
+						: 16; // Reserved
+};
+
+struct leaf_0x12_1 {
+	// eax
+	u32	secs_attr_init			:  1, // Enclave initialized by EINIT
+		secs_attr_debug			:  1, // Enclave permits debugger read/write
+		secs_attr_mode64bit		:  1, // Enclave runs in 64-bit mode
+						:  1, // Reserved
+		secs_attr_provisionkey		:  1, // Provisioning key
+		secs_attr_einittoken_key	:  1, // EINIT token key
+		secs_attr_cet			:  1, // CET attributes
+		secs_attr_kss			:  1, // Key Separation and Sharing
+						:  2, // Reserved
+		secs_attr_aexnotify		:  1, // Enclave threads: AEX notifications
+						: 21; // Reserved
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32	xfrm_x87			:  1, // Enclave XFRM.X87
+		xfrm_sse			:  1, // Enclave XFRM.SEE
+		xfrm_avx			:  1, // Enclave XFRM.AVX
+		xfrm_mpx_bndregs		:  1, // Enclave XFRM.BNDREGS (MPX BND0-BND3 registers)
+		xfrm_mpx_bndcsr			:  1, // Enclave XFRM.BNDCSR (MPX BNDCFGU/BNDSTATUS registers)
+		xfrm_avx512_opmask		:  1, // Enclave XFRM.OPMASK (AVX-512 k0-k7 registers)
+		xfrm_avx512_zmm_hi256		:  1, // Enclave XFRM.ZMM_Hi256 (AVX-512 ZMM0->ZMM7/15 registers)
+		xfrm_avx512_hi16_zmm		:  1, // Enclave XFRM.HI16_ZMM (AVX-512 ZMM16->ZMM31 registers)
+						:  1, // Reserved
+		xfrm_pkru			:  1, // Enclave XFRM.PKRU (XSAVE PKRU registers)
+						:  7, // Reserved
+		xfrm_tileconfig			:  1, // Enclave XFRM.TILECONFIG (AMX can manage TILECONFIG)
+		xfrm_tiledata			:  1, // Enclave XFRM.TILEDATA (AMX can manage TILEDATA)
+						: 13; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0x12_n {
+	// eax
+	u32	subleaf_type			:  4, // Subleaf type
+						:  8, // Reserved
+		epc_sec_base_addr_0		: 20; // EPC section base address, bits[12:31]
+	// ebx
+	u32	epc_sec_base_addr_1		: 20, // EPC section base address, bits[32:51]
+						: 12; // Reserved
+	// ecx
+	u32	epc_sec_type			:  4, // EPC section type / property encoding
+						:  8, // Reserved
+		epc_sec_size_0			: 20; // EPC section size, bits[12:31]
+	// edx
+	u32	epc_sec_size_1			: 20, // EPC section size, bits[32:51]
+						: 12; // Reserved
+};
+
+#define LEAF_0x12_SUBLEAF_N_FIRST		2
+#define LEAF_0x12_SUBLEAF_N_LAST		31
+
+/*
+ * Leaf 0x14
+ * Intel Processor Trace
+ */
+
+struct leaf_0x14_0 {
+	// eax
+	u32	pt_max_subleaf			: 32; // Maximum leaf 0x14 subleaf
+	// ebx
+	u32	cr3_filtering			:  1, // IA32_RTIT_CR3_MATCH is accessible
+		psb_cyc				:  1, // Configurable PSB and cycle-accurate mode
+		ip_filtering			:  1, // IP/TraceStop filtering; Warm-reset PT MSRs preservation
+		mtc_timing			:  1, // MTC timing packet; COFI-based packets suppression
+		ptwrite				:  1, // PTWRITE instruction
+		power_event_trace		:  1, // Power Event Trace
+		psb_pmi_preserve		:  1, // PSB and PMI preservation
+		event_trace			:  1, // Event Trace packet generation
+		tnt_disable			:  1, // TNT packet generation disable
+						: 23; // Reserved
+	// ecx
+	u32	topa_output			:  1, // ToPA output scheme
+		topa_multiple_entries		:  1, // ToPA tables can hold multiple entries
+		single_range_output		:  1, // Single-range output
+		trance_transport_output		:  1, // Trace Transport subsystem output
+						: 27, // Reserved
+		ip_payloads_lip			:  1; // IP payloads have LIP values (CS base included)
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0x14_1 {
+	// eax
+	u32	num_address_ranges		:  3, // Number of configurable Address Ranges
+						: 13, // Reserved
+		mtc_periods_bmp			: 16; // MTC period encodings bitmap
+	// ebx
+	u32	cycle_thresholds_bmp		: 16, // Cycle Threshold encodings bitmap
+		psb_periods_bmp			: 16; // Configurable PSB frequency encodings bitmap
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x15
+ * Intel TSC (Time Stamp Counter)
+ */
+
+struct leaf_0x15_0 {
+	// eax
+	u32	tsc_denominator			: 32; // Denominator of the TSC/'core crystal clock' ratio
+	// ebx
+	u32	tsc_numerator			: 32; // Numerator of the TSC/'core crystal clock' ratio
+	// ecx
+	u32	cpu_crystal_hz			: 32; // Core crystal clock nominal frequency, in Hz
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x16
+ * Intel processor frequency
+ */
+
+struct leaf_0x16_0 {
+	// eax
+	u32	cpu_base_mhz			: 16, // Processor base frequency, in MHz
+						: 16; // Reserved
+	// ebx
+	u32	cpu_max_mhz			: 16, // Processor max frequency, in MHz
+						: 16; // Reserved
+	// ecx
+	u32	bus_mhz				: 16, // Bus reference frequency, in MHz
+						: 16; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x17
+ * Intel SoC vendor attributes
+ */
+
+struct leaf_0x17_0 {
+	// eax
+	u32	soc_max_subleaf			: 32; // Maximum leaf 0x17 subleaf
+	// ebx
+	u32	soc_vendor_id			: 16, // SoC vendor ID
+		is_vendor_scheme		:  1, // Assigned by industry enumeration scheme (not Intel)
+						: 15; // Reserved
+	// ecx
+	u32	soc_proj_id			: 32; // SoC project ID, assigned by vendor
+	// edx
+	u32	soc_stepping_id			: 32; // Soc project stepping ID, assigned by vendor
+};
+
+struct leaf_0x17_n {
+	// eax
+	u32	vendor_brand_a			: 32; // Vendor Brand ID string, bytes subleaf_nr * (0 -> 3)
+	// ebx
+	u32	vendor_brand_b			: 32; // Vendor Brand ID string, bytes subleaf_nr * (4 -> 7)
+	// ecx
+	u32	vendor_brand_c			: 32; // Vendor Brand ID string, bytes subleaf_nr * (8 -> 11)
+	// edx
+	u32	vendor_brand_d			: 32; // Vendor Brand ID string, bytes subleaf_nr * (12 -> 15)
+};
+
+#define LEAF_0x17_SUBLEAF_N_FIRST		1
+#define LEAF_0x17_SUBLEAF_N_LAST		3
+
+/*
+ * Leaf 0x18
+ * Intel deterministic address translation (TLB) parameters
+ */
+
+struct leaf_0x18_n {
+	// eax
+	u32	tlb_max_subleaf			: 32; // Maximum leaf 0x18 subleaf
+	// ebx
+	u32	tlb_4k_page			:  1, // TLB supports 4KB-page entries
+		tlb_2m_page			:  1, // TLB supports 2MB-page entries
+		tlb_4m_page			:  1, // TLB supports 4MB-page entries
+		tlb_1g_page			:  1, // TLB supports 1GB-page entries
+						:  4, // Reserved
+		hard_partitioning		:  3, // Partitioning between logical CPUs
+						:  5, // Reserved
+		n_way_associative		: 16; // Ways of associativity
+	// ecx
+	u32	n_sets				: 32; // Number of sets
+	// edx
+	u32	tlb_type			:  5, // Translation cache type (TLB type)
+		tlb_cache_level			:  3, // Translation cache level (1-based)
+		is_fully_associative		:  1, // Fully-associative
+						:  5, // Reserved
+		tlb_max_addressible_ids		: 12, // Max number of addressable IDs - 1
+						:  6; // Reserved
+};
+
+#define LEAF_0x18_SUBLEAF_N_FIRST		0
+#define LEAF_0x18_SUBLEAF_N_LAST		31
+
+/*
+ * Leaf 0x19
+ * Intel key locker
+ */
+
+struct leaf_0x19_0 {
+	// eax
+	u32	kl_cpl0_only			:  1, // CPL0-only key Locker restriction
+		kl_no_encrypt			:  1, // No-encrypt key locker restriction
+		kl_no_decrypt			:  1, // No-decrypt key locker restriction
+						: 29; // Reserved
+	// ebx
+	u32	aes_keylocker			:  1, // AES key locker instructions
+						:  1, // Reserved
+		aes_keylocker_wide		:  1, // AES wide key locker instructions
+						:  1, // Reserved
+		kl_msr_iwkey			:  1, // Key locker MSRs and IWKEY backups
+						: 27; // Reserved
+	// ecx
+	u32	loadiwkey_no_backup		:  1, // LOADIWKEY NoBackup parameter
+		iwkey_rand			:  1, // IWKEY randomization
+						: 30; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x1a
+ * Intel hybrid CPUs identification (e.g. Atom, Core)
+ */
+
+struct leaf_0x1a_0 {
+	// eax
+	u32	core_native_model		: 24, // This core's native model ID
+		core_type			:  8; // This core's type
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x1b
+ * Intel PCONFIG (Platform configuration)
+ */
+
+struct leaf_0x1b_n {
+	// eax
+	u32	pconfig_subleaf_type		: 12, // CPUID 0x1b subleaf type
+						: 20; // Reserved
+	// ebx
+	u32	pconfig_target_id_x		: 32; // A supported PCONFIG target ID
+	// ecx
+	u32	pconfig_target_id_y		: 32; // A supported PCONFIG target ID
+	// edx
+	u32	pconfig_target_id_z		: 32; // A supported PCONFIG target ID
+};
+
+#define LEAF_0x1b_SUBLEAF_N_FIRST		0
+#define LEAF_0x1b_SUBLEAF_N_LAST		31
+
+/*
+ * Leaf 0x1c
+ * Intel LBR (Last Branch Record)
+ */
+
+struct leaf_0x1c_0 {
+	// eax
+	u32	lbr_depth_mask			:  8, // Max LBR stack depth bitmask
+						: 22, // Reserved
+		lbr_deep_c_reset		:  1, // LBRs maybe cleared on MWAIT C-state > C1
+		lbr_ip_is_lip			:  1; // LBR IP contain Last IP (otherwise effective IP)
+	// ebx
+	u32	lbr_cpl				:  1, // CPL filtering
+		lbr_branch_filter		:  1, // Branch filtering
+		lbr_call_stack			:  1, // Call-stack mode
+						: 29; // Reserved
+	// ecx
+	u32	lbr_mispredict			:  1, // Branch misprediction bit
+		lbr_timed_lbr			:  1, // Timed LBRs (CPU cycles since last LBR entry)
+		lbr_branch_type			:  1, // Branch type field
+						: 13, // Reserved
+		lbr_events_gpc_bmp		:  4, // PMU-events logging support
+						: 12; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x1d
+ * Intel AMX (Advanced Matrix Extensions) tile information
+ */
+
+struct leaf_0x1d_0 {
+	// eax
+	u32	amx_max_palette			: 32; // Highest palette ID / subleaf ID
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0x1d_1 {
+	// eax
+	u32	amx_palette_size		: 16, // AMX palette total tiles size, in bytes
+		amx_tile_size			: 16; // AMX single tile's size, in bytes
+	// ebx
+	u32	amx_tile_row_size		: 16, // AMX tile single row's size, in bytes
+		amx_palette_nr_tiles		: 16; // AMX palette number of tiles
+	// ecx
+	u32	amx_tile_nr_rows		: 16, // AMX tile max number of rows
+						: 16; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x1e
+ * Intel TMUL (Tile-matrix Multiply)
+ */
+
+struct leaf_0x1e_0 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32	tmul_maxk			:  8, // TMUL unit maximum height, K (rows or columns)
+		tmul_maxn			: 16, // TMUL unit maximum SIMD dimension, N (column bytes)
+						:  8; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x1f
+ * Intel extended topology v2
+ */
+
+struct leaf_0x1f_n {
+	// eax
+	u32	x2apic_id_shift			:  5, // Bit width of this level (previous levels inclusive)
+						: 27; // Reserved
+	// ebx
+	u32	domain_lcpus_count		: 16, // Logical CPUs count across all instances of this domain
+						: 16; // Reserved
+	// ecx
+	u32	domain_level			:  8, // This domain level (subleaf ID)
+		domain_type			:  8, // This domain type
+						: 16; // Reserved
+	// edx
+	u32	x2apic_id			: 32; // x2APIC ID of current logical CPU
+};
+
+#define LEAF_0x1f_SUBLEAF_N_FIRST		0
+#define LEAF_0x1f_SUBLEAF_N_LAST		5
+
+/*
+ * Leaf 0x20
+ * Intel HRESET (History Reset)
+ */
+
+struct leaf_0x20_0 {
+	// eax
+	u32	hreset_nr_subleaves		: 32; // CPUID 0x20 max subleaf + 1
+	// ebx
+	u32	hreset_thread_director		:  1, // Intel thread director HRESET
+						: 31; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x21
+ * Intel TD (Trust Domain)
+ */
+
+struct leaf_0x21_0 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32	tdx_vendorid_0			: 32; // TDX vendor ID string bytes 0 - 3
+	// ecx
+	u32	tdx_vendorid_2			: 32; // CPU vendor ID string bytes 8 - 11
+	// edx
+	u32	tdx_vendorid_1			: 32; // CPU vendor ID string bytes 4 - 7
+};
+
+/*
+ * Leaf 0x23
+ * Intel Architectural Performance Monitoring Extended (ArchPerfmonExt)
+ */
+
+struct leaf_0x23_0 {
+	// eax
+	u32	subleaf_0			:  1, // Subleaf 0, this subleaf
+		counters_subleaf		:  1, // Subleaf 1, PMU counter bitmaps
+		acr_subleaf			:  1, // Subleaf 2, Auto Counter Reload bitmaps
+		events_subleaf			:  1, // Subleaf 3, PMU event bitmaps
+		pebs_caps_subleaf		:  1, // Subleaf 4, PEBS capabilities
+		pebs_subleaf			:  1, // Subleaf 5, Arch PEBS bitmaps
+						: 26; // Reserved
+	// ebx
+	u32	unitmask2			:  1, // IA32_PERFEVTSELx MSRs UnitMask2 bit
+		eq				:  1, // IA32_PERFEVTSELx MSRs EQ bit
+		rdpmc_user_disable		:  1, // RDPMC userspace disable
+						: 29; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0x23_1 {
+	// eax
+	u32	gp_counters			: 32; // Bitmap of general-purpose PMU counters
+	// ebx
+	u32	fixed_counters			: 32; // Bitmap of fixed PMU counters
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0x23_2 {
+	// eax
+	u32	acr_gp_reload			: 32; // Bitmap of general-purpose counters that can be reloaded
+	// ebx
+	u32	acr_fixed_reload		: 32; // Bitmap of fixed counters that can be reloaded
+	// ecx
+	u32	acr_gp_trigger			: 32; // Bitmap of general-purpose counters that can trigger reloads
+	// edx
+	u32	acr_fixed_trigger		: 32; // Bitmap of fixed counters that can trigger reloads
+};
+
+struct leaf_0x23_3 {
+	// eax
+	u32	core_cycles_evt			:  1, // Core cycles event
+		insn_retired_evt		:  1, // Instructions retired event
+		ref_cycles_evt			:  1, // Reference cycles event
+		llc_refs_evt			:  1, // Last-level cache references event
+		llc_misses_evt			:  1, // Last-level cache misses event
+		br_insn_ret_evt			:  1, // Branch instruction retired event
+		br_mispr_evt			:  1, // Branch mispredict retired event
+		td_slots_evt			:  1, // Topdown slots event
+		td_backend_bound_evt		:  1, // Topdown backend bound event
+		td_bad_spec_evt			:  1, // Topdown bad speculation event
+		td_frontend_bound_evt		:  1, // Topdown frontend bound event
+		td_retiring_evt			:  1, // Topdown retiring event
+						: 20; // Reserved
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0x23_4 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32					:  3, // Reserved
+		allow_in_record			:  1, // ALLOW_IN_RECORD bit in MSRs
+		counters_gp			:  1, // Counters group sub-group general-purpose counters
+		counters_fixed			:  1, // Counters group sub-group fixed-function counters
+		counters_metrics		:  1, // Counters group sub-group performance metrics
+						:  1, // Reserved
+		lbr				:  2, // LBR group
+						:  6, // Reserved
+		xer				:  8, // XER group
+						:  5, // Reserved
+		gpr				:  1, // GPR group
+		aux				:  1, // AUX group
+						:  1; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0x23_5 {
+	// eax
+	u32	pebs_gp				: 32; // Architectural PEBS general-purpose counters
+	// ebx
+	u32	pebs_pdist_gp			: 32; // Architectural PEBS PDIST general-purpose counters
+	// ecx
+	u32	pebs_fixed			: 32; // Architectural PEBS fixed counters
+	// edx
+	u32	pebs_pdist_fixed		: 32; // Architectural PEBS PDIST fixed counters
+};
+
+/*
+ * Leaf 0x40000000
+ * Maximum hypervisor leaf + hypervisor vendor string
+ */
+
+struct leaf_0x40000000_0 {
+	// eax
+	u32	max_hyp_leaf			: 32; // Maximum hypervisor leaf
+	// ebx
+	u32	hypervisor_id_0			: 32; // Hypervisor ID string bytes 0 - 3
+	// ecx
+	u32	hypervisor_id_1			: 32; // Hypervisor ID string bytes 4 - 7
+	// edx
+	u32	hypervisor_id_2			: 32; // Hypervisor ID string bytes 8 - 11
+};
+
+/*
+ * Leaf 0x4c780001
+ * Linux-defined synthetic feature flags
+ */
+
+struct leaf_0x4c780001_0 {
+	// eax
+	u32	cxmmx				:  1, // Cyrix MMX extensions
+		k6_mtrr				:  1, // AMD K6 nonstandard MTRRs
+		cyrix_arr			:  1, // Cyrix ARRs (= MTRRs)
+		centaur_mcr			:  1, // Centaur MCRs (= MTRRs)
+		k8				:  1, // Opteron, Athlon64
+		zen5				:  1, // CPU based on Zen5 micro-architecture
+		zen6				:  1, // CPU based on Zen6 micro-architecture
+						:  1, // Reserved
+		constant_tsc			:  1, // TSC ticks at a constant rate
+		up				:  1, // SMP kernel running on UP
+		art				:  1, // Always running timer (ART)
+		arch_perfmon			:  1, // Intel Architectural PerfMon
+		pebs				:  1, // Precise-Event Based Sampling
+		bts				:  1, // Branch Trace Store
+		syscall32			:  1, // SYSCALL in IA32 userspace
+		sysenter32			:  1, // SYSENTER in IA32 userspace
+		rep_good			:  1, // REP microcode works well
+		amd_lbr_v2			:  1, // AMD Last Branch Record Extension version 2
+		clear_cpu_buf			:  1, // Clear CPU buffers using VERW
+		acc_power			:  1, // AMD Accumulated Power Mechanism
+		nopl				:  1, // The NOPL instructions
+		always				:  1, // Always-present feature
+		xtopology			:  1, // CPU topology enumeration extensions
+		tsc_reliable			:  1, // TSC is known to be reliable
+		nonstop_tsc			:  1, // TSC does not stop in C states
+		cpuid				:  1, // CPU has the CPUID instruction
+		extd_apicid			:  1, // Extended APIC ID (8 bits)
+		amd_dcm				:  1, // AMD multi-node processor
+		aperfmperf			:  1, // APERF/MPERF MSRs: P-State hardware coordination feedback
+		rapl				:  1, // AMD/Hygon RAPL interface
+		nonstop_tsc_s3			:  1, // TSC does not stop in S3 state
+		tsc_known_freq			:  1; // TSC has known frequency
+	// ebx
+	u32	ring3mwait			:  1, // Ring 3 MONITOR/MWAIT instructions
+		cpuid_fault			:  1, // Intel CPUID faulting
+		cpb				:  1, // AMD Core Performance Boost
+		epb				:  1, // IA32_ENERGY_PERF_BIAS support
+		cat_l3				:  1, // Cache Allocation Technology L3
+		cat_l2				:  1, // Cache Allocation Technology L2
+		cdp_l3				:  1, // Code and Data Prioritization L3
+		tdx_host_platform		:  1, // Platform supports being a TDX host
+		hw_pstate			:  1, // AMD Hardware P-state control
+		proc_feedback			:  1, // AMD Processor Feedback Interface
+		xcompacted			:  1, // Use compacted XSTATE (XSAVES or XSAVEC)
+		pti				:  1, // Kernel Page Table Isolation enabled
+		kernel_ibrs			:  1, // Set/clear IBRS on kernel entry/exit
+		rsb_vmexit			:  1, // Fill RSB on VM-Exit
+		intel_ppin			:  1, // Intel Processor Inventory Number
+		cdp_l2				:  1, // Code and Data Prioritization L2
+		msr_spec_ctrl			:  1, // MSR SPEC_CTRL is implemented
+		ssbd				:  1, // Speculative Store Bypass Disable
+		mba				:  1, // Memory Bandwidth Allocation
+		rsb_ctxsw			:  1, // Fill RSB on context switches
+		perfmon_v2			:  1, // AMD Performance Monitoring Version 2
+						:  1, // Reserved
+		use_ibrs_fw			:  1, // Use IBRS during runtime firmware calls
+		ss_bypass_disable		:  1, // Disable Speculative Store Bypass
+		ls_cfg_ssbd			:  1, // AMD SSBD implementation via LS_CFG MSR
+		ibrs				:  1, // Indirect Branch Restricted Speculation
+		ibpb				:  1, // Indirect Branch Prediction Barrier (without RSB flush guarantee)
+		stibp				:  1, // Single Thread Indirect Branch Predictors
+		zen				:  1, // Generic flag for all Zen and newer
+		l1tf_pteinv			:  1, // L1TF workaround PTE inversion
+		ibrs_enhanced			:  1, // Enhanced IBRS
+		msr_ia32_feat_ctl		:  1; // MSR IA32_FEAT_CTL configured
+	// ecx
+	u32	tpr_shadow			:  1, // Intel TPR Shadow
+		flexpriority			:  1, // Intel FlexPriority
+		ept				:  1, // Intel Extended Page Table
+		vpid				:  1, // Intel Virtual Processor ID
+		coherency_sfw_no		:  1, // SNP cache coherency software work around not needed
+						: 10, // Reserved
+		vmmcall				:  1, // Prefer VMMCALL to VMCALL
+		xenpv				:  1, // Xen paravirtual guest
+		ept_ad				:  1, // Intel Extended Page Table access-dirty bit
+		VMCALL				:  1, // Hypervisor supports the VMCALL instruction
+		vmw_vmmcall			:  1, // VMware prefers the VMMCALL instruction
+		pvunlock			:  1, // PV unlock function
+		vcpupreempt			:  1, // PV vcpu_is_preempted function
+		tdx_guest			:  1, // Intel Trust Domain Extensions Guest
+						:  9; // Reserved
+	// edx
+	u32	cqm_llc				:  1, // LLC QoS
+		cqm_occup_llc			:  1, // LLC occupancy monitoring
+		cqm_mbm_total			:  1, // LLC Total MBM monitoring
+		cqm_mbm_local			:  1, // LLC Local MBM monitoring
+		fence_swapgs_user		:  1, // LFENCE in user entry SWAPGS path
+		fence_swapgs_kernel		:  1, // LFENCE in kernel entry SWAPGS path
+		split_lock_detect		:  1, // #AC for split lock
+		per_thread_mba			:  1, // Per-thread Memory Bandwidth Allocation
+		sgx1				:  1, // SGX Basic
+		sgx2				:  1, // SGX Enclave Dynamic Memory Management (EDMM)
+		entry_ibpb			:  1, // Issue an IBPB on kernel entry
+		rrsba_ctrl			:  1, // RET prediction control
+		retpoline			:  1, // Generic Retpoline mitigation for Spectre variant 2
+		retpoline_lfence		:  1, // Use LFENCE for Spectre variant 2
+		rethunk				:  1, // Use Return THUNK
+		unret				:  1, // AMD BTB untrain return
+		use_ibpb_fw			:  1, // Use IBPB during runtime firmware calls
+		rsb_vmexit_lite			:  1, // Fill RSB on VM exit when EIBRS is enabled
+		sgx_edeccssa			:  1, // SGX EDECCSSA user leaf function
+		call_depth			:  1, // Call depth tracking for RSB stuffing
+		msr_tsx_ctrl			:  1, // MSR IA32_TSX_CTRL (Intel) implemented
+		smba				:  1, // Slow Memory Bandwidth Allocation
+		bmec				:  1, // Bandwidth Monitoring Event Configuration
+		user_shstk			:  1, // Shadow stack support for user mode applications
+		srso				:  1, // AMD BTB untrain RETs
+		srso_alias			:  1, // AMD BTB untrain RETs through aliasing
+		ibpb_on_vmexit			:  1, // Issue an IBPB only on VMEXIT
+		apic_msrs_fence			:  1, // IA32_TSC_DEADLINE and X2APIC MSRs need fencing
+		zen2				:  1, // CPU based on Zen2 microarchitecture
+		zen3				:  1, // CPU based on Zen3 microarchitecture
+		zen4				:  1, // CPU based on Zen4 microarchitecture
+		zen1				:  1; // CPU based on Zen1 microarchitecture
+};
+
+struct leaf_0x4c780001_1 {
+	// eax
+	u32	overflow_recov			:  1, // MCA overflow recovery support
+		succor				:  1, // Uncorrectable error containment and recovery
+						:  1, // Reserved
+		smca				:  1, // Scalable MCA
+						: 28; // Reserved
+	// ebx
+	u32	amd_lbr_pmc_freeze		:  1, // AMD LBR and PMC Freeze
+		clear_bhb_loop			:  1, // Clear branch history at SYSCALL entry using SW loop
+		bhi_ctrl			:  1, // BHI_DIS_S HW control available
+		clear_bhb_hw			:  1, // BHI_DIS_S HW control enabled
+		clear_bhb_vmexit		:  1, // Clear branch history at VMEXIT using SW loop
+		amd_fast_cppc			:  1, // AMD fast Collaborative Processor Performance Control
+		amd_htr_cores			:  1, // Heterogeneous Core Topology
+		amd_workload_class		:  1, // Workload Classification
+		prefer_ymm			:  1, // Avoid ZMM registers due to downclocking
+		apx				:  1, // Advanced Performance Extensions
+		indirect_thunk_its		:  1, // Use thunk for indirect branches in lower half of cache line
+		tsa_sq_no			:  1, // AMD CPU not vulnerable to TSA-SQ
+		tsa_l1_no			:  1, // AMD CPU not vulnerable to TSA-L1
+		clear_cpu_buf_vm		:  1, // Clear CPU buffers using VERW before VMRUN
+		ibpb_exit_to_user		:  1, // Use IBPB on exit-to-userspace, see VMSCAPE bug
+						: 17; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x4c780002
+ * Linux-defined synthetic CPU bug flags
+ */
+
+struct leaf_0x4c780002_0 {
+	// eax
+	u32	f00f				:  1, // Intel F00F
+		fdiv				:  1, // FPU FDIV
+		coma				:  1, // Cyrix 6x86 coma
+		amd_tlb_mmatch			:  1, // AMD Erratum 383
+		amd_apic_c1e			:  1, // AMD Erratum 400
+		bug_11ap			:  1, // Bad local APIC aka 11AP
+		fxsave_leak			:  1, // FXSAVE leaks FOP/FIP/FOP
+		clflush_monitor			:  1, // AAI65, CLFLUSH required before MONITOR
+		sysret_ss_attrs			:  1, // SYSRET does not fix up SS attributes
+		espfix				:  1, // IRET to 16-bit SS corrupts ESP/RSP high bits (x86-32)
+		null_seg			:  1, // Setting a selector to NULL preserves the base
+		swapgs_fence			:  1, // SWAPGS without input dep on GS
+		monitor				:  1, // IPI required to wake up remote CPU
+		amd_e400			:  1, // CPU is among the affected by Erratum 400
+		cpu_meltdown			:  1, // CPU affected by meltdown; needs kernel page table isolation
+		spectre_v1			:  1, // CPU affected by Spectre variant 1 with conditional branches
+		specture_v2			:  1, // CPU affected by Spectre variant 2 with indirect branches
+		spec_store_bypass		:  1, // CPU affected by speculative store bypass attack
+		l1tf				:  1, // CPU affected by L1 Terminal Fault
+		mds				:  1, // CPU affected by Microarchitectural data sampling
+		msbds_only			:  1, // Microarchitectural data sampling: CPU only affected by the MSDBS variant
+		swapgs				:  1, // CPU affected by speculation through SWAPGS
+		taa				:  1, // CPU is affected by TSX Async Abort (TAA)
+		itlb_multihit			:  1, // CPU may incur MCE during certain page attribute changes
+		srbds				:  1, // CPU may leak RNG bits if not mitigated
+		mmio_stale_data			:  1, // CPU affected by Processor MMIO Stale Data vulnerabilities
+						:  1, // Reserved
+		retbleed			:  1, // CPU affected by Retbleed
+		eibrs_pbrsb			:  1, // EIBRS is vulnerable to Post Barrier RSB Predictions
+		smt_rsb				:  1, // CPU vulnerable to Cross-Thread Return Address Predictions
+		gds				:  1, // CPU affected by Gather Data Sampling
+		tdx_pw_mce			:  1; // CPU may incur #MC if non-TD software does partial write to TDX private memory
+	// ebx
+	u32	srso				:  1, // AMD SRSO bug
+		div0				:  1, // AMD DIV0 speculation bug
+		rfds				:  1, // CPU vulnerable to Register File Data Sampling
+		bhi				:  1, // CPU affected by Branch History Injection
+		ibpb_no_ret			:  1, // IBPB omits return target predictions
+		spectre_v2_user			:  1, // CPU affected by Spectre variant 2 between user processes
+		old_microcode			:  1, // CPU has old microcode; it must be vulnerable to something
+		its				:  1, // CPU affected by Indirect Target Selection
+		its_native_only			:  1, // CPU affected by ITS; VMX is not affected
+		tsa				:  1, // CPU affected by Transient Scheduler Attacks
+		vmscape				:  1, // CPU affected by VMSCAPE attacks from guests
+						: 21; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000000
+ * Maximum extended leaf + CPU vendor string
+ */
+
+struct leaf_0x80000000_0 {
+	// eax
+	u32	max_ext_leaf			: 32; // Maximum extended CPUID leaf
+	// ebx
+	u32	cpu_vendorid_0			: 32; // Vendor ID string bytes 0 - 3
+	// ecx
+	u32	cpu_vendorid_2			: 32; // Vendor ID string bytes 8 - 11
+	// edx
+	u32	cpu_vendorid_1			: 32; // Vendor ID string bytes 4 - 7
+};
+
+/*
+ * Leaf 0x80000001
+ * Extended CPU features
+ */
+
+struct leaf_0x80000001_0 {
+	// eax
+	u32	e_stepping_id			:  4, // Stepping ID
+		e_base_model			:  4, // Base processor model
+		e_base_family			:  4, // Base processor family
+		e_base_type			:  2, // Base processor type (Transmeta)
+						:  2, // Reserved
+		e_ext_model			:  4, // Extended processor model
+		e_ext_family			:  8, // Extended processor family
+						:  4; // Reserved
+	// ebx
+	u32	brand_id			: 16, // Brand ID
+						: 12, // Reserved
+		pkg_type			:  4; // Package type
+	// ecx
+	u32	lahf_lm				:  1, // LAHF and SAHF in 64-bit mode
+		cmp_legacy			:  1, // Multi-processing legacy mode (No HT)
+		svm				:  1, // Secure Virtual Machine
+		extapic				:  1, // Extended APIC space
+		cr8_legacy			:  1, // LOCK MOV CR0 means MOV CR8
+		lzcnt_abm			:  1, // LZCNT advanced bit manipulation
+		sse4a				:  1, // SSE4A support
+		misaligned_sse			:  1, // Misaligned SSE mode
+		_3dnow_prefetch			:  1, // 3DNow PREFETCH/PREFETCHW support
+		osvw				:  1, // OS visible workaround
+		ibs				:  1, // Instruction based sampling
+		xop				:  1, // XOP: extended operation (AVX instructions)
+		skinit				:  1, // SKINIT/STGI support
+		wdt				:  1, // Watchdog timer support
+						:  1, // Reserved
+		lwp				:  1, // Lightweight profiling
+		fma4				:  1, // 4-operand FMA instruction
+		tce				:  1, // Translation cache extension
+						:  1, // Reserved
+		nodeid_msr			:  1, // NodeId MSR (0xc001100c)
+						:  1, // Reserved
+		tbm				:  1, // Trailing bit manipulations
+		topoext				:  1, // Topology Extensions (leaf 0x8000001d)
+		perfctr_core			:  1, // Core performance counter extensions
+		perfctr_nb			:  1, // NB/DF performance counter extensions
+						:  1, // Reserved
+		data_bp_ext			:  1, // Data access breakpoint extension
+		perf_tsc			:  1, // Performance time-stamp counter
+		perfctr_llc			:  1, // LLC (L3) performance counter extensions
+		mwaitx				:  1, // MWAITX/MONITORX support
+		addr_mask_ext			:  1, // Breakpoint address mask extension (to bit 31)
+						:  1; // Reserved
+	// edx
+	u32	e_fpu				:  1, // Floating-Point Unit on-chip (x87)
+		e_vme				:  1, // Virtual-8086 Mode Extensions
+		e_de				:  1, // Debugging Extensions
+		e_pse				:  1, // Page Size Extension
+		e_tsc				:  1, // Time Stamp Counter
+		e_msr				:  1, // Model-Specific Registers (RDMSR and WRMSR support)
+		pae				:  1, // Physical Address Extensions
+		mce				:  1, // Machine Check Exception
+		cx8				:  1, // CMPXCHG8B instruction
+		apic				:  1, // APIC on-chip
+						:  1, // Reserved
+		syscall				:  1, // SYSCALL and SYSRET instructions
+		mtrr				:  1, // Memory Type Range Registers
+		pge				:  1, // Page Global Extensions
+		mca				:  1, // Machine Check Architecture
+		cmov				:  1, // Conditional Move Instruction
+		pat				:  1, // Page Attribute Table
+		pse36				:  1, // Page Size Extension (36-bit)
+						:  1, // Reserved
+		obsolete_mp_bit			:  1, // Out-of-spec AMD Multiprocessing bit
+		nx				:  1, // No-execute page protection
+						:  1, // Reserved
+		mmxext				:  1, // AMD MMX extensions
+		e_mmx				:  1, // MMX instructions
+		e_fxsr				:  1, // FXSAVE and FXRSTOR instructions
+		fxsr_opt			:  1, // FXSAVE and FXRSTOR optimizations
+		page1gb				:  1, // 1-GB large page support
+		rdtscp				:  1, // RDTSCP instruction
+						:  1, // Reserved
+		lm				:  1, // Long mode (x86-64, 64-bit support)
+		_3dnowext			:  1, // AMD 3DNow extensions
+		_3dnow				:  1; // 3DNow instructions
+};
+
+/*
+ * Leaf 0x80000002
+ * CPU brand ID string, bytes 0 - 15
+ */
+
+struct leaf_0x80000002_0 {
+	// eax
+	u32	cpu_brandid_0			: 32; // CPU brand ID string, bytes 0 - 3
+	// ebx
+	u32	cpu_brandid_1			: 32; // CPU brand ID string, bytes 4 - 7
+	// ecx
+	u32	cpu_brandid_2			: 32; // CPU brand ID string, bytes 8 - 11
+	// edx
+	u32	cpu_brandid_3			: 32; // CPU brand ID string, bytes 12 - 15
+};
+
+/*
+ * Leaf 0x80000003
+ * CPU brand ID string, bytes 16 - 31
+ */
+
+struct leaf_0x80000003_0 {
+	// eax
+	u32	cpu_brandid_4			: 32; // CPU brand ID string bytes, 16 - 19
+	// ebx
+	u32	cpu_brandid_5			: 32; // CPU brand ID string bytes, 20 - 23
+	// ecx
+	u32	cpu_brandid_6			: 32; // CPU brand ID string bytes, 24 - 27
+	// edx
+	u32	cpu_brandid_7			: 32; // CPU brand ID string bytes, 28 - 31
+};
+
+/*
+ * Leaf 0x80000004
+ * CPU brand ID string, bytes 32 - 47
+ */
+
+struct leaf_0x80000004_0 {
+	// eax
+	u32	cpu_brandid_8			: 32; // CPU brand ID string, bytes 32 - 35
+	// ebx
+	u32	cpu_brandid_9			: 32; // CPU brand ID string, bytes 36 - 39
+	// ecx
+	u32	cpu_brandid_10			: 32; // CPU brand ID string, bytes 40 - 43
+	// edx
+	u32	cpu_brandid_11			: 32; // CPU brand ID string, bytes 44 - 47
+};
+
+/*
+ * Leaf 0x80000005
+ * AMD/Transmeta L1 cache and TLB
+ */
+
+struct leaf_0x80000005_0 {
+	// eax
+	u32	l1_itlb_2m_4m_nentries		:  8, // L1 ITLB #entries, 2M and 4M pages
+		l1_itlb_2m_4m_assoc		:  8, // L1 ITLB associativity, 2M and 4M pages
+		l1_dtlb_2m_4m_nentries		:  8, // L1 DTLB #entries, 2M and 4M pages
+		l1_dtlb_2m_4m_assoc		:  8; // L1 DTLB associativity, 2M and 4M pages
+	// ebx
+	u32	l1_itlb_4k_nentries		:  8, // L1 ITLB #entries, 4K pages
+		l1_itlb_4k_assoc		:  8, // L1 ITLB associativity, 4K pages
+		l1_dtlb_4k_nentries		:  8, // L1 DTLB #entries, 4K pages
+		l1_dtlb_4k_assoc		:  8; // L1 DTLB associativity, 4K pages
+	// ecx
+	u32	l1_dcache_line_size		:  8, // L1 dcache line size, in bytes
+		l1_dcache_nlines		:  8, // L1 dcache lines per tag
+		l1_dcache_assoc			:  8, // L1 dcache associativity
+		l1_dcache_size_kb		:  8; // L1 dcache size, in KB
+	// edx
+	u32	l1_icache_line_size		:  8, // L1 icache line size, in bytes
+		l1_icache_nlines		:  8, // L1 icache lines per tag
+		l1_icache_assoc			:  8, // L1 icache associativity
+		l1_icache_size_kb		:  8; // L1 icache size, in KB
+};
+
+/*
+ * Leaf 0x80000006
+ * (Mostly AMD) L2/L3 cache and TLB
+ */
+
+struct leaf_0x80000006_0 {
+	// eax
+	u32	l2_itlb_2m_4m_nentries		: 12, // L2 iTLB #entries, 2M and 4M pages
+		l2_itlb_2m_4m_assoc		:  4, // L2 iTLB associativity, 2M and 4M pages
+		l2_dtlb_2m_4m_nentries		: 12, // L2 dTLB #entries, 2M and 4M pages
+		l2_dtlb_2m_4m_assoc		:  4; // L2 dTLB associativity, 2M and 4M pages
+	// ebx
+	u32	l2_itlb_4k_nentries		: 12, // L2 iTLB #entries, 4K pages
+		l2_itlb_4k_assoc		:  4, // L2 iTLB associativity, 4K pages
+		l2_dtlb_4k_nentries		: 12, // L2 dTLB #entries, 4K pages
+		l2_dtlb_4k_assoc		:  4; // L2 dTLB associativity, 4K pages
+	// ecx
+	u32	l2_line_size			:  8, // L2 cache line size, in bytes
+		l2_nlines			:  4, // L2 cache number of lines per tag
+		l2_assoc			:  4, // L2 cache associativity
+		l2_size_kb			: 16; // L2 cache size, in KB
+	// edx
+	u32	l3_line_size			:  8, // L3 cache line size, in bytes
+		l3_nlines			:  4, // L3 cache number of lines per tag
+		l3_assoc			:  4, // L3 cache associativity
+						:  2, // Reserved
+		l3_size_range			: 14; // L3 cache size range
+};
+
+/*
+ * Leaf 0x80000007
+ * CPU power management (mostly AMD) and AMD RAS
+ */
+
+struct leaf_0x80000007_0 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32	mca_overflow_recovery		:  1, // MCA overflow conditions not fatal
+		succor				:  1, // Software containment of uncorrectable errors
+		hw_assert			:  1, // Hardware assert MSRs
+		scalable_mca			:  1, // Scalable MCA (MCAX MSRs)
+						: 28; // Reserved
+	// ecx
+	u32	cpu_pwr_sample_ratio		: 32; // CPU power sample time ratio
+	// edx
+	u32	digital_temp			:  1, // Digital temperature sensor
+		powernow_freq_id		:  1, // PowerNOW! frequency scaling
+		powernow_volt_id		:  1, // PowerNOW! voltage scaling
+		thermal_trip			:  1, // THERMTRIP (Thermal Trip)
+		hw_thermal_control		:  1, // Hardware thermal control
+		sw_thermal_control		:  1, // Software thermal control
+		_100mhz_steps			:  1, // 100 MHz multiplier control
+		hw_pstate			:  1, // Hardware P-state control
+		constant_tsc			:  1, // TSC ticks at constant rate across all P and C states
+		core_perf_boost			:  1, // Core performance boost
+		eff_freq_ro			:  1, // Read-only effective frequency interface
+		proc_feedback			:  1, // Processor feedback interface (deprecated)
+		proc_power_reporting		:  1, // Processor power reporting interface
+		connected_standby		:  1, // CPU Connected Standby support
+		rapl_interface			:  1, // Runtime Average Power Limit interface
+						: 17; // Reserved
+};
+
+/*
+ * Leaf 0x80000008
+ * CPU capacity parameters and extended feature flags (mostly AMD)
+ */
+
+struct leaf_0x80000008_0 {
+	// eax
+	u32	phys_addr_bits			:  8, // Max physical address bits
+		virt_addr_bits			:  8, // Max virtual address bits
+		guest_phys_addr_bits		:  8, // Max nested-paging guest physical address bits
+						:  8; // Reserved
+	// ebx
+	u32	clzero				:  1, // CLZERO instruction
+		insn_retired_perf		:  1, // Instruction retired counter MSR
+		xsave_err_ptr			:  1, // XSAVE/XRSTOR always saves/restores FPU error pointers
+		invlpgb				:  1, // INVLPGB broadcasts a TLB invalidate
+		rdpru				:  1, // RDPRU (Read Processor Register at User level)
+						:  1, // Reserved
+		mba				:  1, // Memory Bandwidth Allocation (AMD bit)
+						:  1, // Reserved
+		mcommit				:  1, // MCOMMIT instruction
+		wbnoinvd			:  1, // WBNOINVD instruction
+						:  2, // Reserved
+		ibpb				:  1, // Indirect Branch Prediction Barrier
+		wbinvd_int			:  1, // Interruptible WBINVD/WBNOINVD
+		ibrs				:  1, // Indirect Branch Restricted Speculation
+		stibp				:  1, // Single Thread Indirect Branch Prediction mode
+		ibrs_always_on			:  1, // IBRS always-on preferred
+		stibp_always_on			:  1, // STIBP always-on preferred
+		ibrs_fast			:  1, // IBRS is preferred over software solution
+		ibrs_same_mode			:  1, // IBRS provides same mode protection
+		no_efer_lmsle			:  1, // Long-Mode Segment Limit Enable unsupported
+		tlb_flush_nested		:  1, // INVLPGB RAX[5] bit can be set
+						:  1, // Reserved
+		amd_ppin			:  1, // Protected Processor Inventory Number
+		amd_ssbd			:  1, // Speculative Store Bypass Disable
+		virt_ssbd			:  1, // virtualized SSBD (Speculative Store Bypass Disable)
+		amd_ssb_no			:  1, // SSBD is not needed (fixed in hardware)
+		cppc				:  1, // Collaborative Processor Performance Control
+		amd_psfd			:  1, // Predictive Store Forward Disable
+		btc_no				:  1, // CPU not affected by Branch Type Confusion
+		ibpb_ret			:  1, // IBPB clears RSB/RAS too
+		branch_sampling			:  1; // Branch Sampling
+	// ecx
+	u32	cpu_nthreads			:  8, // Number of physical threads - 1
+						:  4, // Reserved
+		apicid_coreid_len		:  4, // Number of thread core ID bits (shift) in APIC ID
+		perf_tsc_len			:  2, // Performance time-stamp counter size
+						: 14; // Reserved
+	// edx
+	u32	invlpgb_max_pages		: 16, // INVLPGB maximum page count
+		rdpru_max_reg_id		: 16; // RDPRU max register ID (ECX input)
+};
+
+/*
+ * Leaf 0x8000000a
+ * AMD SVM (Secure Virtual Machine)
+ */
+
+struct leaf_0x8000000a_0 {
+	// eax
+	u32	svm_version			:  8, // SVM revision number
+						: 24; // Reserved
+	// ebx
+	u32	svm_nasid			: 32; // Number of address space identifiers (ASID)
+	// ecx
+	u32					:  4, // Reserved
+		pml				:  1, // Page Modification Logging (PML)
+						: 27; // Reserved
+	// edx
+	u32	nested_pt			:  1, // Nested paging
+		lbr_virt			:  1, // LBR virtualization
+		svm_lock			:  1, // SVM lock
+		nrip_save			:  1, // NRIP save support on #VMEXIT
+		tsc_rate_msr			:  1, // MSR based TSC rate control
+		vmcb_clean			:  1, // VMCB clean bits support
+		flush_by_asid			:  1, // Flush by ASID + Extended VMCB TLB_Control
+		decode_assists			:  1, // Decode Assists support
+						:  2, // Reserved
+		pause_filter			:  1, // Pause intercept filter
+						:  1, // Reserved
+		pf_threshold			:  1, // Pause filter threshold
+		avic				:  1, // Advanced virtual interrupt controller
+						:  1, // Reserved
+		v_vmsave_vmload			:  1, // Virtual VMSAVE/VMLOAD (nested virtualization)
+		v_gif				:  1, // Virtualize the Global Interrupt Flag
+		gmet				:  1, // Guest mode execution trap
+		x2avic				:  1, // Virtual x2APIC
+		sss_check			:  1, // Supervisor Shadow Stack restrictions
+		v_spec_ctrl			:  1, // Virtual SPEC_CTRL
+		ro_gpt				:  1, // Read-Only guest page table support
+						:  1, // Reserved
+		h_mce_override			:  1, // Host MCE override
+		tlbsync_int			:  1, // TLBSYNC intercept + INVLPGB/TLBSYNC in VMCB
+		nmi_virt			:  1, // NMI virtualization
+		ibs_virt			:  1, // IBS Virtualization
+		ext_lvt_off_chg			:  1, // Extended LVT offset fault change
+		svme_addr_chk			:  1, // Guest SVME address check
+						:  3; // Reserved
+};
+
+/*
+ * Leaf 0x80000019
+ * AMD TLB characteristics for 1GB pages
+ */
+
+struct leaf_0x80000019_0 {
+	// eax
+	u32	l1_itlb_1g_nentries		: 12, // L1 iTLB #entries, 1G pages
+		l1_itlb_1g_assoc		:  4, // L1 iTLB associativity, 1G pages
+		l1_dtlb_1g_nentries		: 12, // L1 dTLB #entries, 1G pages
+		l1_dtlb_1g_assoc		:  4; // L1 dTLB associativity, 1G pages
+	// ebx
+	u32	l2_itlb_1g_nentries		: 12, // L2 iTLB #entries, 1G pages
+		l2_itlb_1g_assoc		:  4, // L2 iTLB associativity, 1G pages
+		l2_dtlb_1g_nentries		: 12, // L2 dTLB #entries, 1G pages
+		l2_dtlb_1g_assoc		:  4; // L2 dTLB associativity, 1G pages
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x8000001a
+ * AMD instruction optimizations
+ */
+
+struct leaf_0x8000001a_0 {
+	// eax
+	u32	fp_128				:  1, // Internal FP/SIMD exec data path is 128-bits wide
+		movu_preferred			:  1, // SSE: MOVU* better than MOVL*/MOVH*
+		fp_256				:  1, // internal FP/SSE exec data path is 256-bits wide
+						: 29; // Reserved
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x8000001b
+ * AMD IBS (Instruction-Based Sampling)
+ */
+
+struct leaf_0x8000001b_0 {
+	// eax
+	u32	ibs_flags			:  1, // IBS feature flags
+		ibs_fetch_sampling		:  1, // IBS fetch sampling
+		ibs_op_sampling			:  1, // IBS execution sampling
+		ibs_rdwr_op_counter		:  1, // IBS read/write of op counter
+		ibs_op_count			:  1, // IBS OP counting mode
+		ibs_branch_target		:  1, // IBS branch target address reporting
+		ibs_op_counters_ext		:  1, // IBS IbsOpCurCnt/IbsOpMaxCnt extend by 7 bits
+		ibs_rip_invalid_chk		:  1, // IBS invalid RIP indication
+		ibs_op_branch_fuse		:  1, // IBS fused branch micro-op indication
+		ibs_fetch_ctl_ext		:  1, // IBS Fetch Control Extended MSR
+		ibs_op_data_4			:  1, // IBS op data 4 MSR
+		ibs_l3_miss_filter		:  1, // IBS L3-miss filtering (Zen4+)
+						: 20; // Reserved
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x8000001c
+ * AMD LWP (Lightweight Profiling)
+ */
+
+struct leaf_0x8000001c_0 {
+	// eax
+	u32	os_lwp_avail			:  1, // OS: LWP is available to application programs
+		os_lpwval			:  1, // OS: LWPVAL instruction
+		os_lwp_ire			:  1, // OS: Instructions Retired Event
+		os_lwp_bre			:  1, // OS: Branch Retired Event
+		os_lwp_dme			:  1, // OS: Dcache Miss Event
+		os_lwp_cnh			:  1, // OS: CPU Clocks Not Halted event
+		os_lwp_rnh			:  1, // OS: CPU Reference clocks Not Halted event
+						: 22, // Reserved
+		os_lwp_cont			:  1, // OS: LWP sampling in continuous mode
+		os_lwp_ptsc			:  1, // OS: Performance Time Stamp Counter in event records
+		os_lwp_int			:  1; // OS: Interrupt on threshold overflow
+	// ebx
+	u32	lwp_lwpcb_sz			:  8, // Control Block size, in quadwords
+		lwp_event_sz			:  8, // Event record size, in bytes
+		lwp_max_events			:  8, // Max EventID supported
+		lwp_event_offset		:  8; // Control Block events area offset
+	// ecx
+	u32	lwp_latency_max			:  5, // Cache latency counters number of bits
+		lwp_data_addr			:  1, // Cache miss events report data cache address
+		lwp_latency_rnd			:  3, // Cache latency rounding amount
+		lwp_version			:  7, // LWP version
+		lwp_buf_min_sz			:  8, // LWP event ring buffer min size, 32 event records units
+						:  4, // Reserved
+		lwp_branch_predict		:  1, // Branches Retired events can be filtered
+		lwp_ip_filtering		:  1, // IP filtering (IPI, IPF, BaseIP, and LimitIP @ LWPCP)
+		lwp_cache_levels		:  1, // Cache-related events: filter by cache level
+		lwp_cache_latency		:  1; // Cache-related events: filter by latency
+	// edx
+	u32	hw_lwp_avail			:  1, // HW: LWP available
+		hw_lpwval			:  1, // HW: LWPVAL available
+		hw_lwp_ire			:  1, // HW: Instructions Retired Event
+		hw_lwp_bre			:  1, // HW: Branch Retired Event
+		hw_lwp_dme			:  1, // HW: Dcache Miss Event
+		hw_lwp_cnh			:  1, // HW: Clocks Not Halted event
+		hw_lwp_rnh			:  1, // HW: Reference clocks Not Halted event
+						: 22, // Reserved
+		hw_lwp_cont			:  1, // HW: LWP sampling in continuous mode
+		hw_lwp_ptsc			:  1, // HW: Performance Time Stamp Counter in event records
+		hw_lwp_int			:  1; // HW: Interrupt on threshold overflow
+};
+
+/*
+ * Leaf 0x8000001d
+ * AMD deterministic cache parameters
+ */
+
+struct leaf_0x8000001d_n {
+	// eax
+	u32	cache_type			:  5, // Cache type field
+		cache_level			:  3, // Cache level (1-based)
+		cache_self_init			:  1, // Self-initializing cache level
+		fully_associative		:  1, // Fully-associative cache
+						:  4, // Reserved
+		num_threads_sharing		: 12, // Number of logical CPUs sharing cache
+						:  6; // Reserved
+	// ebx
+	u32	cache_linesize			: 12, // System coherency line size (0-based)
+		cache_npartitions		: 10, // Physical line partitions (0-based)
+		cache_nways			: 10; // Ways of associativity (0-based)
+	// ecx
+	u32	cache_nsets			: 31, // Cache number of sets (0-based)
+						:  1; // Reserved
+	// edx
+	u32	wbinvd_rll_no_guarantee		:  1, // WBINVD/INVD not guaranteed for Remote Lower-Level caches
+		ll_inclusive			:  1, // Cache is inclusive of Lower-Level caches
+						: 30; // Reserved
+};
+
+#define LEAF_0x8000001d_SUBLEAF_N_FIRST		0
+#define LEAF_0x8000001d_SUBLEAF_N_LAST		31
+
+/*
+ * Leaf 0x8000001e
+ * AMD CPU topology
+ */
+
+struct leaf_0x8000001e_0 {
+	// eax
+	u32	ext_apic_id			: 32; // Extended APIC ID
+	// ebx
+	u32	core_id				:  8, // Unique per-socket logical core unit ID
+		core_nthreads			:  8, // #Threads per core (zero-based)
+						: 16; // Reserved
+	// ecx
+	u32	node_id				:  8, // Node (die) ID of invoking logical CPU
+		nnodes_per_socket		:  3, // #nodes in invoking logical CPU's package/socket
+						: 21; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x8000001f
+ * AMD encrypted memory capabilities (SME/SEV)
+ */
+
+struct leaf_0x8000001f_0 {
+	// eax
+	u32	sme				:  1, // Secure Memory Encryption
+		sev				:  1, // Secure Encrypted Virtualization
+		vm_page_flush			:  1, // VM Page Flush MSR
+		sev_encrypted_state		:  1, // SEV Encrypted State
+		sev_nested_paging		:  1, // SEV secure nested paging
+		vm_permission_levels		:  1, // VMPL
+		rpmquery			:  1, // RPMQUERY instruction
+		vmpl_sss			:  1, // VMPL supervisor shadow stack
+		secure_tsc			:  1, // Secure TSC
+		virt_tsc_aux			:  1, // Hardware virtualizes TSC_AUX
+		sme_coherent			:  1, // Cache coherency enforcement across encryption domains
+		req_64bit_hypervisor		:  1, // SEV guest mandates 64-bit hypervisor
+		restricted_injection		:  1, // Restricted Injection supported
+		alternate_injection		:  1, // Alternate Injection supported
+		debug_swap			:  1, // SEV-ES: Full debug state swap
+		disallow_host_ibs		:  1, // SEV-ES: Disallowing IBS use by the host
+		virt_transparent_enc		:  1, // Virtual Transparent Encryption
+		vmgexit_parameter		:  1, // SEV_FEATURES: VmgexitParameter
+		virt_tom_msr			:  1, // Virtual TOM MSR
+		virt_ibs			:  1, // SEV-ES guests: IBS state virtualization
+						:  4, // Reserved
+		vmsa_reg_protection		:  1, // VMSA register protection
+		smt_protection			:  1, // SMT protection
+						:  2, // Reserved
+		svsm_page_msr			:  1, // SVSM communication page MSR
+		nested_virt_snp_msr		:  1, // VIRT_RMPUPDATE/VIRT_PSMASH MSRs
+						:  2; // Reserved
+	// ebx
+	u32	pte_cbit_pos			:  6, // PTE bit number to enable memory encryption
+		phys_addr_reduction_nbits	:  6, // Reduction of phys address space in bits
+		vmpl_count			:  4, // Number of VM permission levels (VMPL)
+						: 16; // Reserved
+	// ecx
+	u32	enc_guests_max			: 32; // Max number of simultaneous encrypted guests
+	// edx
+	u32	min_sev_asid_no_sev_es		: 32; // Minimum ASID for SEV-enabled SEV-ES-disabled guest
+};
+
+/*
+ * Leaf 0x80000020
+ * AMD PQoS (Platform QoS) extended features
+ */
+
+struct leaf_0x80000020_0 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32					:  1, // Reserved
+		mba				:  1, // Memory Bandwidth Allocation support
+		smba				:  1, // Slow Memory Bandwidth Allocation support
+		bmec				:  1, // Bandwidth Monitoring Event Configuration support
+		l3rr				:  1, // L3 Range Reservation support
+		abmc				:  1, // Assignable Bandwidth Monitoring Counters
+		sdciae				:  1, // Smart Data Cache Injection (SDCI) Allocation Enforcement
+						: 25; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+struct leaf_0x80000020_1 {
+	// eax
+	u32	mba_limit_len			: 32; // MBA enforcement limit size
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32	mba_cos_max			: 32; // MBA max Class of Service number (zero-based)
+};
+
+struct leaf_0x80000020_2 {
+	// eax
+	u32	smba_limit_len			: 32; // SMBA enforcement limit size
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32	smba_cos_max			: 32; // SMBA max Class of Service number (zero-based)
+};
+
+struct leaf_0x80000020_3 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32	bmec_num_events			:  8, // BMEC number of bandwidth events available
+						: 24; // Reserved
+	// ecx
+	u32	bmec_local_reads		:  1, // Local NUMA reads can be tracked
+		bmec_remote_reads		:  1, // Remote NUMA reads can be tracked
+		bmec_local_nontemp_wr		:  1, // Local NUMA non-temporal writes can be tracked
+		bmec_remote_nontemp_wr		:  1, // Remote NUMA non-temporal writes can be tracked
+		bmec_local_slow_mem_rd		:  1, // Local NUMA slow-memory reads can be tracked
+		bmec_remote_slow_mem_rd		:  1, // Remote NUMA slow-memory reads can be tracked
+		bmec_all_dirty_victims		:  1, // Dirty QoS victims to all types of memory can be tracked
+						: 25; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000021
+ * AMD extended CPU features 2
+ */
+
+struct leaf_0x80000021_0 {
+	// eax
+	u32	no_nested_data_bp		:  1, // No nested data breakpoints
+		fsgs_non_serializing		:  1, // WRMSR to {FS,GS,KERNEL_GS}_BASE is non-serializing
+		lfence_serializing		:  1, // LFENCE always serializing / synchronizes RDTSC
+		smm_page_cfg_lock		:  1, // SMM paging configuration lock
+						:  2, // Reserved
+		null_sel_clr_base		:  1, // Null selector clears base
+		upper_addr_ignore		:  1, // EFER MSR Upper Address Ignore
+		auto_ibrs			:  1, // EFER MSR Automatic IBRS
+		no_smm_ctl_msr			:  1, // SMM_CTL MSR not available
+		fsrs				:  1, // Fast Short Rep STOSB
+		fsrc				:  1, // Fast Short Rep CMPSB
+						:  1, // Reserved
+		prefetch_ctl_msr		:  1, // Prefetch control MSR
+						:  2, // Reserved
+		opcode_reclaim			:  1, // Reserves opcode space
+		user_cpuid_disable		:  1, // #GP when executing CPUID at CPL > 0
+		epsf				:  1, // Enhanced Predictive Store Forwarding
+						:  3, // Reserved
+		wl_feedback			:  1, // Workload-based heuristic feedback to OS
+						:  1, // Reserved
+		eraps				:  1, // Enhanced Return Address Predictor Security
+						:  2, // Reserved
+		sbpb				:  1, // Selective Branch Predictor Barrier
+		ibpb_brtype			:  1, // Branch predictions flushed from CPU branch predictor
+		srso_no				:  1, // No SRSO vulnerability
+		srso_uk_no			:  1, // No SRSO at user-kernel boundary
+		srso_msr_fix			:  1; // MSR BP_CFG[BpSpecReduce] SRSO mitigation
+	// ebx
+	u32	microcode_patch_size		: 16, // Microcode patch size, in 16-byte units
+		rap_size			:  8, // Return Address Predictor size
+						:  8; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000022
+ * AMD extended performance monitoring
+ */
+
+struct leaf_0x80000022_0 {
+	// eax
+	u32	perfmon_v2			:  1, // Performance monitoring v2
+		lbr_v2				:  1, // Last Branch Record v2 extensions (LBR Stack)
+		lbr_pmc_freeze			:  1, // Freezing core performance counters / LBR Stack
+						: 29; // Reserved
+	// ebx
+	u32	n_pmc_core			:  4, // Number of core performance counters
+		lbr_v2_stack_size		:  6, // Number of LBR stack entries
+		n_pmc_northbridge		:  6, // Number of northbridge performance counters
+		n_pmc_umc			:  6, // Number of UMC performance counters
+						: 10; // Reserved
+	// ecx
+	u32	active_umc_bitmask		: 32; // Active UMCs bitmask
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000023
+ * AMD multi-key encrypted memory
+ */
+
+struct leaf_0x80000023_0 {
+	// eax
+	u32	mem_hmk_mode			:  1, // MEM-HMK encryption mode
+						: 31; // Reserved
+	// ebx
+	u32	mem_hmk_avail_keys		: 16, // Total number of available encryption keys
+						: 16; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x80000026
+ * AMD extended CPU topology
+ */
+
+struct leaf_0x80000026_n {
+	// eax
+	u32	x2apic_id_shift			:  5, // Bit width of this level (previous levels inclusive)
+						: 24, // Reserved
+		core_has_pwreff_ranking		:  1, // This core has a power efficiency ranking
+		domain_has_hybrid_cores		:  1, // This domain level has hybrid (E, P) cores
+		domain_core_count_asymm		:  1; // The 'Core' domain has asymmetric cores count
+	// ebx
+	u32	domain_lcpus_count		: 16, // Number of logical CPUs at this domain instance
+		core_pwreff_ranking		:  8, // This core's static power efficiency ranking
+		core_native_model_id		:  4, // This core's native model ID
+		core_type			:  4; // This core's type
+	// ecx
+	u32	domain_level			:  8, // This domain level (subleaf ID)
+		domain_type			:  8, // This domain type
+						: 16; // Reserved
+	// edx
+	u32	x2apic_id			: 32; // x2APIC ID of current logical CPU
+};
+
+#define LEAF_0x80000026_SUBLEAF_N_FIRST		0
+#define LEAF_0x80000026_SUBLEAF_N_LAST		3
+
+/*
+ * Leaf 0x80860000
+ * Maximum Transmeta leaf + CPU vendor string
+ */
+
+struct leaf_0x80860000_0 {
+	// eax
+	u32	max_tra_leaf			: 32; // Maximum Transmeta leaf
+	// ebx
+	u32	cpu_vendorid_0			: 32; // Transmeta Vendor ID string bytes 0 - 3
+	// ecx
+	u32	cpu_vendorid_2			: 32; // Transmeta Vendor ID string bytes 8 - 11
+	// edx
+	u32	cpu_vendorid_1			: 32; // Transmeta Vendor ID string bytes 4 - 7
+};
+
+/*
+ * Leaf 0x80860001
+ * Transmeta extended CPU features
+ */
+
+struct leaf_0x80860001_0 {
+	// eax
+	u32	stepping			:  4, // Stepping ID
+		base_model			:  4, // Base CPU model ID
+		base_family_id			:  4, // Base CPU family ID
+		cpu_type			:  2, // CPU type
+						: 18; // Reserved
+	// ebx
+	u32	cpu_rev_mask_minor		:  8, // CPU revision ID, mask minor
+		cpu_rev_mask_major		:  8, // CPU revision ID, mask major
+		cpu_rev_minor			:  8, // CPU revision ID, minor
+		cpu_rev_major			:  8; // CPU revision ID, major
+	// ecx
+	u32	cpu_base_mhz			: 32; // CPU nominal frequency, in MHz
+	// edx
+	u32	recovery			:  1, // Recovery CMS is active (after bad flush)
+		longrun				:  1, // LongRun power management capabilities
+						:  1, // Reserved
+		lrti				:  1, // LongRun Table Interface
+						: 28; // Reserved
+};
+
+/*
+ * Leaf 0x80860002
+ * Transmeta CMS (Code Morphing Software)
+ */
+
+struct leaf_0x80860002_0 {
+	// eax
+	u32	cpu_rev_id			: 32; // CPU revision ID
+	// ebx
+	u32	cms_rev_mask_2			:  8, // CMS revision ID, mask component 2
+		cms_rev_mask_1			:  8, // CMS revision ID, mask component 1
+		cms_rev_minor			:  8, // CMS revision ID, minor
+		cms_rev_major			:  8; // CMS revision ID, major
+	// ecx
+	u32	cms_rev_mask_3			: 32; // CMS revision ID, mask component 3
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0x80860003
+ * Transmeta CPU information string, bytes 0 - 15
+ */
+
+struct leaf_0x80860003_0 {
+	// eax
+	u32	cpu_info_0			: 32; // CPU info string bytes 0 - 3
+	// ebx
+	u32	cpu_info_1			: 32; // CPU info string bytes 4 - 7
+	// ecx
+	u32	cpu_info_2			: 32; // CPU info string bytes 8 - 11
+	// edx
+	u32	cpu_info_3			: 32; // CPU info string bytes 12 - 15
+};
+
+/*
+ * Leaf 0x80860004
+ * Transmeta CPU information string, bytes 16 - 31
+ */
+
+struct leaf_0x80860004_0 {
+	// eax
+	u32	cpu_info_4			: 32; // CPU info string bytes 16 - 19
+	// ebx
+	u32	cpu_info_5			: 32; // CPU info string bytes 20 - 23
+	// ecx
+	u32	cpu_info_6			: 32; // CPU info string bytes 24 - 27
+	// edx
+	u32	cpu_info_7			: 32; // CPU info string bytes 28 - 31
+};
+
+/*
+ * Leaf 0x80860005
+ * Transmeta CPU information string, bytes 32 - 47
+ */
+
+struct leaf_0x80860005_0 {
+	// eax
+	u32	cpu_info_8			: 32; // CPU info string bytes 32 - 35
+	// ebx
+	u32	cpu_info_9			: 32; // CPU info string bytes 36 - 39
+	// ecx
+	u32	cpu_info_10			: 32; // CPU info string bytes 40 - 43
+	// edx
+	u32	cpu_info_11			: 32; // CPU info string bytes 44 - 47
+};
+
+/*
+ * Leaf 0x80860006
+ * Transmeta CPU information string, bytes 48 - 63
+ */
+
+struct leaf_0x80860006_0 {
+	// eax
+	u32	cpu_info_12			: 32; // CPU info string bytes 48 - 51
+	// ebx
+	u32	cpu_info_13			: 32; // CPU info string bytes 52 - 55
+	// ecx
+	u32	cpu_info_14			: 32; // CPU info string bytes 56 - 59
+	// edx
+	u32	cpu_info_15			: 32; // CPU info string bytes 60 - 63
+};
+
+/*
+ * Leaf 0x80860007
+ * Transmeta live CPU information
+ */
+
+struct leaf_0x80860007_0 {
+	// eax
+	u32	cpu_cur_mhz			: 32; // Current CPU frequency, in MHz
+	// ebx
+	u32	cpu_cur_voltage			: 32; // Current CPU voltage, in millivolts
+	// ecx
+	u32	cpu_cur_perf_pctg		: 32; // Current CPU performance percentage, 0 - 100
+	// edx
+	u32	cpu_cur_gate_delay		: 32; // Current CPU gate delay, in femtoseconds
+};
+
+/*
+ * Leaf 0xc0000000
+ * Maximum Centaur/Zhaoxin leaf
+ */
+
+struct leaf_0xc0000000_0 {
+	// eax
+	u32	max_cntr_leaf			: 32; // Maximum Centaur/Zhaoxin leaf
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32					: 32; // Reserved
+};
+
+/*
+ * Leaf 0xc0000001
+ * Centaur/Zhaoxin extended CPU features
+ */
+
+struct leaf_0xc0000001_0 {
+	// eax
+	u32					: 32; // Reserved
+	// ebx
+	u32					: 32; // Reserved
+	// ecx
+	u32					: 32; // Reserved
+	// edx
+	u32	ccs_sm2				:  1, // CCS SM2 instructions
+		ccs_sm2_en			:  1, // CCS SM2 enabled
+		rng				:  1, // Random Number Generator
+		rng_en				:  1, // RNG enabled
+		ccs_sm3_sm4			:  1, // CCS SM3 and SM4 instructions
+		ccs_sm3_sm4_en			:  1, // CCS SM3/SM4 enabled
+		ace				:  1, // Advanced Cryptography Engine
+		ace_en				:  1, // ACE enabled
+		ace2				:  1, // Advanced Cryptography Engine v2
+		ace2_en				:  1, // ACE v2 enabled
+		phe				:  1, // PadLock Hash Engine
+		phe_en				:  1, // PHE enabled
+		pmm				:  1, // PadLock Montgomery Multiplier
+		pmm_en				:  1, // PMM enabled
+						:  2, // Reserved
+		parallax			:  1, // Parallax auto adjust processor voltage
+		parallax_en			:  1, // Parallax enabled
+						:  2, // Reserved
+		tm3				:  1, // Thermal Monitor v3
+		tm3_en				:  1, // TM v3 enabled
+						:  3, // Reserved
+		phe2				:  1, // PadLock Hash Engine v2 (SHA384/SHA512)
+		phe2_en				:  1, // PHE v2 enabled
+		rsa				:  1, // RSA instructions (XMODEXP/MONTMUL2)
+		rsa_en				:  1, // RSA instructions enabled
+						:  3; // Reserved
+};
+
+#endif /* _ASM_X86_CPUID_LEAF_TYPES */
diff --git a/arch/x86/include/asm/cpuid/types.h b/arch/x86/include/asm/cpuid/types.h
index 8a00364b79dee..3d0e611c97ba5 100644
--- a/arch/x86/include/asm/cpuid/types.h
+++ b/arch/x86/include/asm/cpuid/types.h
@@ -5,6 +5,8 @@
 #include <linux/build_bug.h>
 #include <linux/types.h>
 
+#include <asm/cpuid/leaf_types.h>
+
 /*
  * Types for raw CPUID access:
  */
@@ -30,6 +32,12 @@ enum cpuid_regs_idx {
 #define CPUID_LEAF_FREQ		0x16
 #define CPUID_LEAF_TILE		0x1d
 
+#define CPUID_RANGE(idx)	((idx) & 0xffff0000)
+#define CPUID_RANGE_MAX(idx)	(CPUID_RANGE(idx) + 0xffff)
+
+#define CPUID_BASE_START	0x00000000
+#define CPUID_BASE_END		CPUID_RANGE_MAX(CPUID_BASE_START)
+
 /*
  * Types for CPUID(0x2) parsing:
  */
@@ -124,4 +132,94 @@ extern const struct leaf_0x2_table cpuid_0x2_table[256];
  */
 #define TLB_0x63_2M_4M_ENTRIES		32
 
+/*
+ * Types for centralized CPUID tables:
+ *
+ * For internal use by the CPUID parser.
+ */
+
+/**
+ * struct leaf_parse_info - CPUID query parse info
+ * @nr_entries:	Number of valid entries filled by the CPUID parser
+ */
+struct leaf_parse_info {
+	unsigned int		nr_entries;
+};
+
+/**
+ * __CPUID_LEAF() - Define a CPUID output and parse info entry
+ * @_name:	Struct type name of the CPUID leaf/subleaf (e.g. 'leaf_0x7_0'). Such
+ *		types are defined at <cpuid/leaf_types.h> and follow the leaf_0xM_N
+ *		format, where 0xM is the leaf and N is the subleaf.
+ * @_count:	Number of storage entries to allocate for this leaf/subleaf.
+ *
+ * For a given leaf/subleaf, define an array of CPUID storage entries and an associated
+ * query info structure.
+ *
+ * Use an array of storage entries to accommodate CPUID leaves with multiple subleaves
+ * having the same output format.  This is common for hierarchical enumeration; e.g.,
+ * CPUID(0x4), CPUID(0x12), and CPUID(0x8000001d).
+ */
+#define __CPUID_LEAF(_name, _count)				\
+	struct _name		_name[_count];			\
+	struct leaf_parse_info	_name##_info
+
+/**
+ * CPUID_LEAF() - Define a 'struct cpuid_leaves' storage entry
+ * @_leaf:	Leaf number, in compile-time 0xN format
+ * @_subleaf:	Subleaf number, in compile-time decimal format
+ *
+ * Convenience wrapper around __CPUID_LEAF().
+ */
+#define CPUID_LEAF(_leaf, _subleaf)				\
+	__CPUID_LEAF(leaf_ ## _leaf ## _ ## _subleaf, 1)
+
+#define __cpuid_leaf_first_subleaf(_l)				\
+	LEAF_ ## _l ## _ ## SUBLEAF_N_FIRST
+#define __cpuid_leaf_last_subleaf(_l)				\
+	LEAF_ ## _l ## _ ## SUBLEAF_N_LAST
+
+#define __cpuid_leaf_subleaf_count_min(_l)			2
+#define __cpuid_leaf_subleaf_count_max(_l)			\
+	(__cpuid_leaf_last_subleaf(_l) - __cpuid_leaf_first_subleaf(_l) + 1)
+
+/**
+ * CPUID_LEAF_N() - Define a 'struct cpuid_leaves' storage entry
+ * @_leaf:	Leaf number, in compile-time 0xN format
+ * @_count:	Number of storage entries to allocate for that leaf. It must not exceed
+ *		the limits defined at <cpuid/leaf_types.h>.
+ *
+ * Convenience wrapper around __CPUID_LEAF().
+ */
+#define CPUID_LEAF_N(_leaf, _count)					\
+	static_assert(_count >= __cpuid_leaf_subleaf_count_min(_leaf));	\
+	static_assert(_count <= __cpuid_leaf_subleaf_count_max(_leaf));	\
+	__CPUID_LEAF(leaf_ ## _leaf ## _ ## n, _count)
+
+/*
+ * struct cpuid_leaves - Parsed CPUID data
+ */
+struct cpuid_leaves {
+	/*		Leaf		Subleaf number (or max number of subleaves) */
+	CPUID_LEAF   (	0x0,		0  );
+	CPUID_LEAF   (	0x1,		0  );
+};
+
+/*
+ * Types for centralized CPUID tables:
+ *
+ * For external use.
+ */
+
+/**
+ * struct cpuid_table - Per-CPU CPUID data repository
+ * @leaves:	Parsed CPUID queries output and their metadata
+ *
+ * This is to be embedded inside 'struct cpuinfo_x86' to provide parsed and
+ * sanitized CPUID data per CPU.
+ */
+struct cpuid_table {
+	struct cpuid_leaves	leaves;
+};
+
 #endif /* _ASM_X86_CPUID_TYPES_H */
diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h
index ec95fe44fa3a0..00aeae8435297 100644
--- a/arch/x86/include/asm/desc.h
+++ b/arch/x86/include/asm/desc.h
@@ -438,6 +438,10 @@ extern void idt_setup_traps(void);
 extern void idt_setup_apic_and_irq_gates(void);
 extern bool idt_is_f00f_address(unsigned long address);
 
+extern void idt_do_interrupt_irqoff(unsigned long address);
+extern void idt_do_nmi_irqoff(void);
+extern void idt_entry_from_kvm(unsigned int vector);
+
 #ifdef CONFIG_X86_64
 extern void idt_setup_early_pf(void);
 #else
diff --git a/arch/x86/include/asm/desc_defs.h b/arch/x86/include/asm/desc_defs.h
index 7e6b9314758a1..2f2ce8aadf076 100644
--- a/arch/x86/include/asm/desc_defs.h
+++ b/arch/x86/include/asm/desc_defs.h
@@ -145,7 +145,7 @@ struct gate_struct {
 typedef struct gate_struct gate_desc;
 
 #ifndef _SETUP
-static inline unsigned long gate_offset(const gate_desc *g)
+static __always_inline unsigned long gate_offset(const gate_desc *g)
 {
 #ifdef CONFIG_X86_64
 	return g->offset_low | ((unsigned long)g->offset_middle << 16) |
diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h
deleted file mode 100644
index 7c0a52ca2f4dd..0000000000000
--- a/arch/x86/include/asm/device.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_X86_DEVICE_H
-#define _ASM_X86_DEVICE_H
-
-struct dev_archdata {
-};
-
-struct pdev_archdata {
-};
-
-#endif /* _ASM_X86_DEVICE_H */
diff --git a/arch/x86/include/asm/entry-common.h b/arch/x86/include/asm/entry-common.h
index 7535131c711bb..eca24b5e07f4c 100644
--- a/arch/x86/include/asm/entry-common.h
+++ b/arch/x86/include/asm/entry-common.h
@@ -97,4 +97,6 @@ static __always_inline void arch_exit_to_user_mode(void)
 }
 #define arch_exit_to_user_mode arch_exit_to_user_mode
 
+extern void x86_entry_from_kvm(unsigned int entry_type, unsigned int vector);
+
 #endif
diff --git a/arch/x86/include/asm/extable_fixup_types.h b/arch/x86/include/asm/extable_fixup_types.h
index 906b0d5541e89..fd0cfb4721039 100644
--- a/arch/x86/include/asm/extable_fixup_types.h
+++ b/arch/x86/include/asm/extable_fixup_types.h
@@ -2,15 +2,10 @@
 #ifndef _ASM_X86_EXTABLE_FIXUP_TYPES_H
 #define _ASM_X86_EXTABLE_FIXUP_TYPES_H
 
-/*
- * Our IMM is signed, as such it must live at the top end of the word. Also,
- * since C99 hex constants are of ambiguous type, force cast the mask to 'int'
- * so that FIELD_GET() will DTRT and sign extend the value when it extracts it.
- */
-#define EX_DATA_TYPE_MASK		((int)0x000000FF)
-#define EX_DATA_REG_MASK		((int)0x00000F00)
-#define EX_DATA_FLAG_MASK		((int)0x0000F000)
-#define EX_DATA_IMM_MASK		((int)0xFFFF0000)
+#define EX_DATA_TYPE_MASK		(0x000000FF)
+#define EX_DATA_REG_MASK		(0x00000F00)
+#define EX_DATA_FLAG_MASK		(0x0000F000)
+#define EX_DATA_IMM_MASK		(0xFFFF0000)
 
 #define EX_DATA_REG_SHIFT		8
 #define EX_DATA_FLAG_SHIFT		12
diff --git a/arch/x86/include/asm/fpu/api.h b/arch/x86/include/asm/fpu/api.h
index cd6f194a912bf..90c63fe19c0fb 100644
--- a/arch/x86/include/asm/fpu/api.h
+++ b/arch/x86/include/asm/fpu/api.h
@@ -119,12 +119,6 @@ extern void fpu__init_system(void);
 extern void fpu__init_check_bugs(void);
 extern void fpu__resume_cpu(void);
 
-#ifdef CONFIG_MATH_EMULATION
-extern void fpstate_init_soft(struct swregs_state *soft);
-#else
-static inline void fpstate_init_soft(struct swregs_state *soft) {}
-#endif
-
 /* State tracking */
 DECLARE_PER_CPU(bool, kernel_fpu_allowed);
 DECLARE_PER_CPU(struct fpu *, fpu_fpregs_owner_ctx);
diff --git a/arch/x86/include/asm/fred.h b/arch/x86/include/asm/fred.h
index 2bb65677c079b..18a2f811c3580 100644
--- a/arch/x86/include/asm/fred.h
+++ b/arch/x86/include/asm/fred.h
@@ -110,7 +110,6 @@ static __always_inline unsigned long fred_event_data(struct pt_regs *regs) { ret
 static inline void cpu_init_fred_exceptions(void) { }
 static inline void cpu_init_fred_rsps(void) { }
 static inline void fred_complete_exception_setup(void) { }
-static inline void fred_entry_from_kvm(unsigned int type, unsigned int vector) { }
 static inline void fred_sync_rsp0(unsigned long rsp0) { }
 static inline void fred_update_rsp0(void) { }
 #endif /* CONFIG_X86_FRED */
diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
index 42bf6a58ec368..20f5487024042 100644
--- a/arch/x86/include/asm/idtentry.h
+++ b/arch/x86/include/asm/idtentry.h
@@ -340,17 +340,14 @@ static __always_inline void __##func(struct pt_regs *regs)
 	__visible void noist_##func(struct pt_regs *regs)
 
 /**
- * DECLARE_IDTENTRY_VC - Declare functions for the VC entry point
+ * DECLARE_IDTENTRY_VC - Declare a function for the VC entry point
  * @vector:	Vector number (ignored for C)
  * @func:	Function name of the entry point
  *
- * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE, but declares also the
- * safe_stack C handler.
+ * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE.
  */
 #define DECLARE_IDTENTRY_VC(vector, func)				\
-	DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func);			\
-	__visible noinstr void kernel_##func(struct pt_regs *regs, unsigned long error_code);	\
-	__visible noinstr void   user_##func(struct pt_regs *regs, unsigned long error_code)
+	DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func);
 
 /**
  * DEFINE_IDTENTRY_IST - Emit code for IST entry points
@@ -391,26 +388,6 @@ static __always_inline void __##func(struct pt_regs *regs)
 #define DEFINE_IDTENTRY_DF(func)					\
 	DEFINE_IDTENTRY_RAW_ERRORCODE(func)
 
-/**
- * DEFINE_IDTENTRY_VC_KERNEL - Emit code for VMM communication handler
- *			       when raised from kernel mode
- * @func:	Function name of the entry point
- *
- * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
- */
-#define DEFINE_IDTENTRY_VC_KERNEL(func)				\
-	DEFINE_IDTENTRY_RAW_ERRORCODE(kernel_##func)
-
-/**
- * DEFINE_IDTENTRY_VC_USER - Emit code for VMM communication handler
- *			     when raised from user mode
- * @func:	Function name of the entry point
- *
- * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE
- */
-#define DEFINE_IDTENTRY_VC_USER(func)				\
-	DEFINE_IDTENTRY_RAW_ERRORCODE(user_##func)
-
 #else	/* CONFIG_X86_64 */
 
 /**
diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h
index 3776cf5382a26..b0269325646cb 100644
--- a/arch/x86/include/asm/kvm-x86-ops.h
+++ b/arch/x86/include/asm/kvm-x86-ops.h
@@ -94,10 +94,11 @@ KVM_X86_OP_OPTIONAL(sync_pir_to_irr)
 KVM_X86_OP_OPTIONAL_RET0(set_tss_addr)
 KVM_X86_OP_OPTIONAL_RET0(set_identity_map_addr)
 KVM_X86_OP_OPTIONAL_RET0(get_mt_mask)
+KVM_X86_OP_OPTIONAL_RET0(tdp_has_smep)
 KVM_X86_OP(load_mmu_pgd)
-KVM_X86_OP_OPTIONAL(link_external_spt)
-KVM_X86_OP_OPTIONAL(set_external_spte)
-KVM_X86_OP_OPTIONAL(free_external_spt)
+KVM_X86_OP_OPTIONAL_RET0(link_external_spt)
+KVM_X86_OP_OPTIONAL_RET0(set_external_spte)
+KVM_X86_OP_OPTIONAL_RET0(free_external_spt)
 KVM_X86_OP_OPTIONAL(remove_external_spte)
 KVM_X86_OP(has_wbinvd_exit)
 KVM_X86_OP(get_l2_tsc_offset)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index c470e40a00aa4..271bdd109a98e 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -191,11 +191,12 @@ enum kvm_reg {
 	VCPU_REGS_R14 = __VCPU_REGS_R14,
 	VCPU_REGS_R15 = __VCPU_REGS_R15,
 #endif
-	VCPU_REGS_RIP,
-	NR_VCPU_REGS,
+	NR_VCPU_GENERAL_PURPOSE_REGS,
 
-	VCPU_EXREG_PDPTR = NR_VCPU_REGS,
-	VCPU_EXREG_CR0,
+	VCPU_REG_RIP = NR_VCPU_GENERAL_PURPOSE_REGS,
+
+	VCPU_REG_PDPTR,
+	VCPU_REG_CR0,
 	/*
 	 * Alias AMD's ERAPS (not a real register) to CR3 so that common code
 	 * can trigger emulation of the RAP (Return Address Predictor) with
@@ -203,13 +204,15 @@ enum kvm_reg {
 	 * is cleared on writes to CR3, i.e. marking CR3 dirty will naturally
 	 * mark ERAPS dirty as well.
 	 */
-	VCPU_EXREG_CR3,
-	VCPU_EXREG_ERAPS = VCPU_EXREG_CR3,
-	VCPU_EXREG_CR4,
-	VCPU_EXREG_RFLAGS,
-	VCPU_EXREG_SEGMENTS,
-	VCPU_EXREG_EXIT_INFO_1,
-	VCPU_EXREG_EXIT_INFO_2,
+	VCPU_REG_CR3,
+	VCPU_REG_ERAPS = VCPU_REG_CR3,
+	VCPU_REG_CR4,
+	VCPU_REG_RFLAGS,
+	VCPU_REG_SEGMENTS,
+	VCPU_REG_EXIT_INFO_1,
+	VCPU_REG_EXIT_INFO_2,
+
+	NR_VCPU_TOTAL_REGS,
 };
 
 enum {
@@ -328,11 +331,11 @@ struct kvm_kernel_irq_routing_entry;
  * the number of unique SPs that can theoretically be created is 2^n, where n
  * is the number of bits that are used to compute the role.
  *
- * But, even though there are 20 bits in the mask below, not all combinations
+ * But, even though there are 21 bits in the mask below, not all combinations
  * of modes and flags are possible:
  *
  *   - invalid shadow pages are not accounted, mirror pages are not shadowed,
- *     so the bits are effectively 18.
+ *     so the bits are effectively 19.
  *
  *   - quadrant will only be used if has_4_byte_gpte=1 (non-PAE paging);
  *     execonly and ad_disabled are only used for nested EPT which has
@@ -343,11 +346,11 @@ struct kvm_kernel_irq_routing_entry;
  *     paging has exactly one upper level, making level completely redundant
  *     when has_4_byte_gpte=1.
  *
- *   - on top of this, smep_andnot_wp and smap_andnot_wp are only set if
- *     cr0_wp=0, therefore these three bits only give rise to 5 possibilities.
+ *   - on top of this, smap_andnot_wp is only set if cr0_wp=0,
+ *     therefore these two bits only give rise to 3 possibilities.
  *
  * Therefore, the maximum number of possible upper-level shadow pages for a
- * single gfn is a bit less than 2^13.
+ * single gfn is a bit less than 2^14.
  */
 union kvm_mmu_page_role {
 	u32 word;
@@ -356,17 +359,26 @@ union kvm_mmu_page_role {
 		unsigned has_4_byte_gpte:1;
 		unsigned quadrant:2;
 		unsigned direct:1;
-		unsigned access:3;
+		unsigned access:4;
 		unsigned invalid:1;
 		unsigned efer_nx:1;
 		unsigned cr0_wp:1;
-		unsigned smep_andnot_wp:1;
 		unsigned smap_andnot_wp:1;
 		unsigned ad_disabled:1;
 		unsigned guest_mode:1;
 		unsigned passthrough:1;
 		unsigned is_mirror:1;
-		unsigned :4;
+
+		/*
+		 * cr4_smep is also set for EPT MBEC.  Because it affects
+		 * which pages are considered non-present (bit 10 additionally
+		 * must be zero if MBEC is on) it has to be in the base role.
+		 * It also has to be in the base role for AMD GMET because
+		 * kernel-executable pages need to have U=0 with GMET enabled.
+		 */
+		unsigned cr4_smep:1;
+
+		unsigned:3;
 
 		/*
 		 * This is left at the top of the word so that
@@ -392,10 +404,10 @@ union kvm_mmu_page_role {
  * tables (because KVM doesn't support Protection Keys with shadow paging), and
  * CR0.PG, CR4.PAE, and CR4.PSE are indirectly reflected in role.level.
  *
- * Note, SMEP and SMAP are not redundant with sm*p_andnot_wp in the page role.
- * If CR0.WP=1, KVM can reuse shadow pages for the guest regardless of SMEP and
- * SMAP, but the MMU's permission checks for software walks need to be SMEP and
- * SMAP aware regardless of CR0.WP.
+ * Note, SMAP is not redundant with smap_andnot_wp in the page role.  If
+ * CR0.WP=1, KVM can reuse shadow pages for the guest regardless of SMAP,
+ * but the MMU's permission checks for software walks need to be SMAP
+ * aware regardless of CR0.WP.
  */
 union kvm_mmu_extended_role {
 	u32 word;
@@ -405,9 +417,15 @@ union kvm_mmu_extended_role {
 		unsigned int cr4_pse:1;
 		unsigned int cr4_pke:1;
 		unsigned int cr4_smap:1;
-		unsigned int cr4_smep:1;
 		unsigned int cr4_la57:1;
 		unsigned int efer_lma:1;
+
+		/*
+		 * True if either CR4.SMEP or EFER.NXE are set.  For AMD NPT
+		 * this is the "real" host CR4.SMEP whereas cr4_smep is
+		 * actually GMET.
+		 */
+		unsigned int has_pferr_fetch:1;
 	};
 };
 
@@ -492,7 +510,7 @@ struct kvm_mmu {
 	 * Byte index: page fault error code [4:1]
 	 * Bit index: pte permissions in ACC_* format
 	 */
-	u8 permissions[16];
+	u16 permissions[16];
 
 	u64 *pae_root;
 	u64 *pml4_root;
@@ -799,9 +817,10 @@ struct kvm_vcpu_arch {
 	 * rip and regs accesses must go through
 	 * kvm_{register,rip}_{read,write} functions.
 	 */
-	unsigned long regs[NR_VCPU_REGS];
-	u32 regs_avail;
-	u32 regs_dirty;
+	unsigned long regs[NR_VCPU_GENERAL_PURPOSE_REGS];
+	unsigned long rip;
+	DECLARE_BITMAP(regs_avail, NR_VCPU_TOTAL_REGS);
+	DECLARE_BITMAP(regs_dirty, NR_VCPU_TOTAL_REGS);
 
 	unsigned long cr0;
 	unsigned long cr0_guest_owned_bits;
@@ -1038,8 +1057,6 @@ struct kvm_vcpu_arch {
 		u16 vec;
 		u32 id;
 		u32 host_apf_flags;
-		bool send_always;
-		bool delivery_as_pf_vmexit;
 		bool pageready_pending;
 	} apf;
 
@@ -1504,6 +1521,7 @@ struct kvm_arch {
 	bool use_master_clock;
 	u64 master_kernel_ns;
 	u64 master_cycle_now;
+	struct ratelimit_state kvmclock_update_rs;
 
 #ifdef CONFIG_KVM_HYPERV
 	struct kvm_hv hyperv;
@@ -1887,6 +1905,7 @@ struct kvm_x86_ops {
 	int (*set_tss_addr)(struct kvm *kvm, unsigned int addr);
 	int (*set_identity_map_addr)(struct kvm *kvm, u64 ident_addr);
 	u8 (*get_mt_mask)(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
+	bool (*tdp_has_smep)(struct kvm *kvm);
 
 	void (*load_mmu_pgd)(struct kvm_vcpu *vcpu, hpa_t root_hpa,
 			     int root_level);
@@ -2010,6 +2029,10 @@ struct kvm_x86_nested_ops {
 			 struct kvm_nested_state *kvm_state);
 	bool (*get_nested_state_pages)(struct kvm_vcpu *vcpu);
 	int (*write_log_dirty)(struct kvm_vcpu *vcpu, gpa_t l2_gpa);
+	gpa_t (*translate_nested_gpa)(struct kvm_vcpu *vcpu, gpa_t gpa,
+				      u64 access,
+				      struct x86_exception *exception,
+				      u64 pte_access);
 
 	int (*enable_evmcs)(struct kvm_vcpu *vcpu,
 			    uint16_t *vmcs_version);
@@ -2526,7 +2549,8 @@ int memslot_rmap_alloc(struct kvm_memory_slot *slot, unsigned long npages);
 	 KVM_X86_QUIRK_SLOT_ZAP_ALL |		\
 	 KVM_X86_QUIRK_STUFF_FEATURE_MSRS |	\
 	 KVM_X86_QUIRK_IGNORE_GUEST_PAT |	\
-	 KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM)
+	 KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM |	\
+	 KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT)
 
 #define KVM_X86_CONDITIONAL_QUIRKS		\
 	(KVM_X86_QUIRK_CD_NW_CLEARED |		\
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index 3c317d1557716..9cd136d4515c4 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -3,6 +3,7 @@
 #define _ASM_X86_MICROCODE_H
 
 #include <asm/msr.h>
+#include <asm/cpuid/api.h>
 
 struct cpu_signature {
 	unsigned int sig;
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 10b5355b323e2..55e15110e4dea 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -16,7 +16,7 @@ struct vm86;
 #include <uapi/asm/sigcontext.h>
 #include <asm/current.h>
 #include <asm/cpufeatures.h>
-#include <asm/cpuid/api.h>
+#include <asm/cpuid/types.h>
 #include <asm/page.h>
 #include <asm/pgtable_types.h>
 #include <asm/percpu.h>
@@ -170,6 +170,7 @@ struct cpuinfo_x86 {
 	char			x86_vendor_id[16];
 	char			x86_model_id[64];
 	struct cpuinfo_topology	topo;
+	struct cpuid_table	cpuid;
 	/* in KB - valid for CPUS which support this call: */
 	unsigned int		x86_cache_size;
 	int			x86_cache_alignment;	/* In bytes */
@@ -704,6 +705,11 @@ static inline u32 per_cpu_l2c_id(unsigned int cpu)
 	return per_cpu(cpu_info.topo.l2c_id, cpu);
 }
 
+static inline u32 per_cpu_core_id(unsigned int cpu)
+{
+	return per_cpu(cpu_info.topo.core_id, cpu);
+}
+
 #ifdef CONFIG_CPU_SUP_AMD
 /*
  * Issue a DIV 0/1 insn to clear any division data from previous DIV
diff --git a/arch/x86/include/asm/purgatory.h b/arch/x86/include/asm/purgatory.h
index 2fee5e9f1ccc3..56a9e81edb151 100644
--- a/arch/x86/include/asm/purgatory.h
+++ b/arch/x86/include/asm/purgatory.h
@@ -8,4 +8,4 @@
 extern void purgatory(void);
 #endif	/* __ASSEMBLER__ */
 
-#endif /* _ASM_PURGATORY_H */
+#endif /* _ASM_X86_PURGATORY_H */
diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h
index 09e605c85de4c..594cfa19cbd4b 100644
--- a/arch/x86/include/asm/sev.h
+++ b/arch/x86/include/asm/sev.h
@@ -661,7 +661,7 @@ static inline void snp_leak_pages(u64 pfn, unsigned int pages)
 {
 	__snp_leak_pages(pfn, pages, true);
 }
-void snp_prepare(void);
+int snp_prepare(void);
 void snp_shutdown(void);
 #else
 static inline bool snp_probe_rmptable_info(void) { return false; }
@@ -679,7 +679,7 @@ static inline void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp)
 static inline void snp_leak_pages(u64 pfn, unsigned int npages) {}
 static inline void kdump_sev_callback(void) { }
 static inline void snp_fixup_e820_tables(void) {}
-static inline void snp_prepare(void) {}
+static inline int snp_prepare(void) { return -ENODEV; }
 static inline void snp_shutdown(void) {}
 #endif
 
diff --git a/arch/x86/include/asm/shared/tdx.h b/arch/x86/include/asm/shared/tdx.h
index 049638e3da743..f20e91d7ac35b 100644
--- a/arch/x86/include/asm/shared/tdx.h
+++ b/arch/x86/include/asm/shared/tdx.h
@@ -4,6 +4,7 @@
 
 #include <linux/bits.h>
 #include <linux/types.h>
+#include <asm/shared/tdx_errno.h>
 
 #define TDX_HYPERCALL_STANDARD  0
 
diff --git a/arch/x86/include/asm/shared/tdx_errno.h b/arch/x86/include/asm/shared/tdx_errno.h
new file mode 100644
index 0000000000000..ee411b360e20d
--- /dev/null
+++ b/arch/x86/include/asm/shared/tdx_errno.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* architectural status code for SEAMCALL */
+#ifndef _ASM_X86_SHARED_TDX_ERRNO_H
+#define _ASM_X86_SHARED_TDX_ERRNO_H
+
+#define TDX_SEAMCALL_STATUS_MASK		0xFFFFFFFF00000000ULL
+
+/*
+ * TDX SEAMCALL Status Codes (returned in RAX)
+ */
+#define TDX_NON_RECOVERABLE_VCPU		0x4000000100000000ULL
+#define TDX_NON_RECOVERABLE_TD			0x4000000200000000ULL
+#define TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE	0x6000000500000000ULL
+#define TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE	0x6000000700000000ULL
+#define TDX_INTERRUPTED_RESUMABLE		0x8000000300000000ULL
+#define TDX_SYS_BUSY				0x8000020200000000ULL
+#define TDX_OPERAND_INVALID			0xC000010000000000ULL
+#define TDX_OPERAND_BUSY			0x8000020000000000ULL
+#define TDX_PREVIOUS_TLB_EPOCH_BUSY		0x8000020100000000ULL
+#define TDX_PAGE_METADATA_INCORRECT		0xC000030000000000ULL
+#define TDX_VCPU_NOT_ASSOCIATED			0x8000070200000000ULL
+#define TDX_KEY_GENERATION_FAILED		0x8000080000000000ULL
+#define TDX_KEY_STATE_INCORRECT			0xC000081100000000ULL
+#define TDX_KEY_CONFIGURED			0x0000081500000000ULL
+#define TDX_NO_HKID_READY_TO_WBCACHE		0x0000082100000000ULL
+#define TDX_FLUSHVP_NOT_DONE			0x8000082400000000ULL
+#define TDX_EPT_WALK_FAILED			0xC0000B0000000000ULL
+#define TDX_EPT_ENTRY_STATE_INCORRECT		0xC0000B0D00000000ULL
+#define TDX_METADATA_FIELD_NOT_READABLE		0xC0000C0200000000ULL
+
+/*
+ * TDX module operand ID, appears in 31:0 part of error code as
+ * detail information
+ */
+#define TDX_OPERAND_ID_RCX			0x01
+#define TDX_OPERAND_ID_TDR			0x80
+#define TDX_OPERAND_ID_SEPT			0x92
+#define TDX_OPERAND_ID_TD_EPOCH			0xa9
+
+#endif /* _ASM_X86_SHARED_TDX_ERRNO_H */
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index bcfeb5e7c0edf..aa63431ba92c3 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -243,6 +243,7 @@ struct __attribute__ ((__packed__)) vmcb_control_area {
 #define SVM_MISC_ENABLE_NP		BIT(0)
 #define SVM_MISC_ENABLE_SEV		BIT(1)
 #define SVM_MISC_ENABLE_SEV_ES	BIT(2)
+#define SVM_MISC_ENABLE_GMET	BIT(3)
 
 #define SVM_MISC2_ENABLE_V_LBR	BIT_ULL(0)
 #define SVM_MISC2_ENABLE_V_VMLOAD_VMSAVE	BIT_ULL(1)
diff --git a/arch/x86/include/asm/tdx.h b/arch/x86/include/asm/tdx.h
index a149740b24e8b..9f4f7c974781d 100644
--- a/arch/x86/include/asm/tdx.h
+++ b/arch/x86/include/asm/tdx.h
@@ -187,19 +187,15 @@ static inline u64 mk_keyed_paddr(u16 hkid, struct page *page)
 	return ret;
 }
 
-static inline int pg_level_to_tdx_sept_level(enum pg_level level)
-{
-        WARN_ON_ONCE(level == PG_LEVEL_NONE);
-        return level - 1;
-}
+void tdx_sys_disable(void);
 
 u64 tdh_vp_enter(struct tdx_vp *vp, struct tdx_module_args *args);
 u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page);
 u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page *source, u64 *ext_err1, u64 *ext_err2);
-u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, enum pg_level level, struct page *page, u64 *ext_err1, u64 *ext_err2);
 u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page);
-u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2);
-u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, int level, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, enum pg_level level, struct page *page, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, enum pg_level level, u64 *ext_err1, u64 *ext_err2);
 u64 tdh_mng_key_config(struct tdx_td *td);
 u64 tdh_mng_create(struct tdx_td *td, u16 hkid);
 u64 tdh_vp_create(struct tdx_td *td, struct tdx_vp *vp);
@@ -215,7 +211,7 @@ u64 tdh_vp_rd(struct tdx_vp *vp, u64 field, u64 *data);
 u64 tdh_vp_wr(struct tdx_vp *vp, u64 field, u64 data, u64 mask);
 u64 tdh_phymem_page_reclaim(struct page *page, u64 *tdx_pt, u64 *tdx_owner, u64 *tdx_size);
 u64 tdh_mem_track(struct tdx_td *tdr);
-u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, u64 level, u64 *ext_err1, u64 *ext_err2);
+u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, enum pg_level level, u64 *ext_err1, u64 *ext_err2);
 u64 tdh_phymem_cache_wb(bool resume);
 u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td);
 u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page);
@@ -224,13 +220,8 @@ static inline void tdx_init(void) { }
 static inline u32 tdx_get_nr_guest_keyids(void) { return 0; }
 static inline const char *tdx_dump_mce_info(struct mce *m) { return NULL; }
 static inline const struct tdx_sys_info *tdx_get_sysinfo(void) { return NULL; }
+static inline void tdx_sys_disable(void) { }
 #endif	/* CONFIG_INTEL_TDX_HOST */
 
-#ifdef CONFIG_KEXEC_CORE
-void tdx_cpu_flush_cache_for_kexec(void);
-#else
-static inline void tdx_cpu_flush_cache_for_kexec(void) { }
-#endif
-
 #endif /* !__ASSEMBLER__ */
 #endif /* _ASM_X86_TDX_H */
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 866ea78ba156a..fb256fd95f951 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -5,22 +5,39 @@
 #define tlb_flush tlb_flush
 static inline void tlb_flush(struct mmu_gather *tlb);
 
+#define tlb_table_flush_implies_ipi_broadcast tlb_table_flush_implies_ipi_broadcast
+static inline bool tlb_table_flush_implies_ipi_broadcast(void);
+
 #include <asm-generic/tlb.h>
 #include <linux/kernel.h>
 #include <vdso/bits.h>
 #include <vdso/page.h>
 
+DECLARE_STATIC_KEY_FALSE(tlb_ipi_broadcast_key);
+
+static inline bool tlb_table_flush_implies_ipi_broadcast(void)
+{
+	return static_branch_likely(&tlb_ipi_broadcast_key);
+}
+
 static inline void tlb_flush(struct mmu_gather *tlb)
 {
 	unsigned long start = 0UL, end = TLB_FLUSH_ALL;
 	unsigned int stride_shift = tlb_get_unmap_shift(tlb);
 
+	/*
+	 * Both freed_tables and unshared_tables must wake lazy-TLB CPUs, so
+	 * they receive IPIs before reusing or freeing page tables, allowing
+	 * us to safely implement tlb_table_flush_implies_ipi_broadcast().
+	 */
+	bool wake_lazy_cpus = tlb->freed_tables || tlb->unshared_tables;
+
 	if (!tlb->fullmm && !tlb->need_flush_all) {
 		start = tlb->start;
 		end = tlb->end;
 	}
 
-	flush_tlb_mm_range(tlb->mm, start, end, stride_shift, tlb->freed_tables);
+	flush_tlb_mm_range(tlb->mm, start, end, stride_shift, wake_lazy_cpus);
 }
 
 static inline void invlpg(unsigned long addr)
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 0545fe75c3fa1..938d2bb995011 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -18,6 +18,8 @@
 
 DECLARE_PER_CPU(u64, tlbstate_untag_mask);
 
+void __init native_pv_tlb_init(void);
+
 void __flush_tlb_all(void);
 
 #define TLB_FLUSH_ALL	-1UL
@@ -247,7 +249,7 @@ struct flush_tlb_info {
 	u64			new_tlb_gen;
 	unsigned int		initiating_cpu;
 	u8			stride_shift;
-	u8			freed_tables;
+	u8			wake_lazy_cpus;
 	u8			trim_cpumask;
 };
 
@@ -337,7 +339,7 @@ static inline bool mm_in_asid_transition(struct mm_struct *mm) { return false; }
 extern void flush_tlb_all(void);
 extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
 				unsigned long end, unsigned int stride_shift,
-				bool freed_tables);
+				bool wake_lazy_cpus);
 extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
 
 static inline void flush_tlb_page(struct vm_area_struct *vma, unsigned long a)
diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h
index eda233a90ea86..e26061df0c9b1 100644
--- a/arch/x86/include/asm/vermagic.h
+++ b/arch/x86/include/asm/vermagic.h
@@ -5,8 +5,6 @@
 
 #ifdef CONFIG_X86_64
 /* X86_64 does not define MODULE_PROC_FAMILY */
-#elif defined CONFIG_M586
-#define MODULE_PROC_FAMILY "586 "
 #elif defined CONFIG_M586TSC
 #define MODULE_PROC_FAMILY "586TSC "
 #elif defined CONFIG_M586MMX
@@ -31,10 +29,6 @@
 #define MODULE_PROC_FAMILY "CRUSOE "
 #elif defined CONFIG_MEFFICEON
 #define MODULE_PROC_FAMILY "EFFICEON "
-#elif defined CONFIG_MWINCHIPC6
-#define MODULE_PROC_FAMILY "WINCHIPC6 "
-#elif defined CONFIG_MWINCHIP3D
-#define MODULE_PROC_FAMILY "WINCHIP3D "
 #elif defined CONFIG_MCYRIXIII
 #define MODULE_PROC_FAMILY "CYRIXIII "
 #elif defined CONFIG_MVIAC3_2
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 37080382df548..ed2ded531e554 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -535,6 +535,7 @@ enum vmcs_field {
 #define VMX_EPT_1GB_PAGE_BIT			(1ull << 17)
 #define VMX_EPT_INVEPT_BIT			(1ull << 20)
 #define VMX_EPT_AD_BIT				    (1ull << 21)
+#define VMX_EPT_ADVANCED_VMEXIT_INFO_BIT	(1ull << 22)
 #define VMX_EPT_EXTENT_CONTEXT_BIT		(1ull << 25)
 #define VMX_EPT_EXTENT_GLOBAL_BIT		(1ull << 26)
 
@@ -560,10 +561,12 @@ enum vmcs_field {
 #define VMX_EPT_ACCESS_BIT			(1ull << 8)
 #define VMX_EPT_DIRTY_BIT			(1ull << 9)
 #define VMX_EPT_SUPPRESS_VE_BIT			(1ull << 63)
+
 #define VMX_EPT_RWX_MASK                        (VMX_EPT_READABLE_MASK |       \
 						 VMX_EPT_WRITABLE_MASK |       \
 						 VMX_EPT_EXECUTABLE_MASK)
 #define VMX_EPT_MT_MASK				(7ull << VMX_EPT_MT_EPTE_SHIFT)
+#define VMX_EPT_USER_EXECUTABLE_MASK		(1ull << 10)
 
 static inline u8 vmx_eptp_page_walk_level(u64 eptp)
 {
@@ -608,17 +611,24 @@ enum vm_entry_failure_code {
 #define EPT_VIOLATION_PROT_READ		BIT(3)
 #define EPT_VIOLATION_PROT_WRITE	BIT(4)
 #define EPT_VIOLATION_PROT_EXEC		BIT(5)
-#define EPT_VIOLATION_EXEC_FOR_RING3_LIN BIT(6)
+#define EPT_VIOLATION_PROT_USER_EXEC	BIT(6)
 #define EPT_VIOLATION_PROT_MASK		(EPT_VIOLATION_PROT_READ  | \
 					 EPT_VIOLATION_PROT_WRITE | \
-					 EPT_VIOLATION_PROT_EXEC)
+					 EPT_VIOLATION_PROT_EXEC  | \
+					 EPT_VIOLATION_PROT_USER_EXEC)
 #define EPT_VIOLATION_GVA_IS_VALID	BIT(7)
 #define EPT_VIOLATION_GVA_TRANSLATED	BIT(8)
+#define EPT_VIOLATION_GVA_USER		BIT(9)
+#define EPT_VIOLATION_GVA_WRITABLE	BIT(10)
+#define EPT_VIOLATION_GVA_NX		BIT(11)
 
 #define EPT_VIOLATION_RWX_TO_PROT(__epte) (((__epte) & VMX_EPT_RWX_MASK) << 3)
+#define EPT_VIOLATION_USER_EXEC_TO_PROT(__epte) (((__epte) & VMX_EPT_USER_EXECUTABLE_MASK) >> 4)
 
 static_assert(EPT_VIOLATION_RWX_TO_PROT(VMX_EPT_RWX_MASK) ==
 	      (EPT_VIOLATION_PROT_READ | EPT_VIOLATION_PROT_WRITE | EPT_VIOLATION_PROT_EXEC));
+static_assert(EPT_VIOLATION_USER_EXEC_TO_PROT(VMX_EPT_USER_EXECUTABLE_MASK) ==
+	      (EPT_VIOLATION_PROT_USER_EXEC));
 
 /*
  * Exit Qualifications for NOTIFY VM EXIT
diff --git a/arch/x86/include/asm/xen/hypervisor.h b/arch/x86/include/asm/xen/hypervisor.h
index c2fc7869b996a..7c596cebfb787 100644
--- a/arch/x86/include/asm/xen/hypervisor.h
+++ b/arch/x86/include/asm/xen/hypervisor.h
@@ -37,6 +37,7 @@ extern struct shared_info *HYPERVISOR_shared_info;
 extern struct start_info *xen_start_info;
 
 #include <asm/bug.h>
+#include <asm/cpuid/api.h>
 #include <asm/processor.h>
 
 #define XEN_SIGNATURE "XenVMMXenVMM"
diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
index 5f2b30d0405c8..1585ec8040666 100644
--- a/arch/x86/include/uapi/asm/kvm.h
+++ b/arch/x86/include/uapi/asm/kvm.h
@@ -477,6 +477,7 @@ struct kvm_sync_regs {
 #define KVM_X86_QUIRK_STUFF_FEATURE_MSRS	(1 << 8)
 #define KVM_X86_QUIRK_IGNORE_GUEST_PAT		(1 << 9)
 #define KVM_X86_QUIRK_VMCS12_ALLOW_FREEZE_IN_SMM (1 << 10)
+#define KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT	(1 << 11)
 
 #define KVM_STATE_NESTED_FORMAT_VMX	0
 #define KVM_STATE_NESTED_FORMAT_SVM	1
@@ -532,6 +533,7 @@ struct kvm_svm_nested_state_data {
 
 struct kvm_svm_nested_state_hdr {
 	__u64 vmcb_pa;
+	__u64 gpat;
 };
 
 /* for KVM_CAP_NESTED_STATE */
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 6399049114442..8c614750a19b2 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -64,6 +64,7 @@
 #include <asm/tsc.h>
 #include <asm/hypervisor.h>
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/intel-family.h>
 #include <asm/irq_regs.h>
 #include <asm/cpu.h>
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index 2f8a58ef690ec..d2e8a849f1805 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -19,6 +19,7 @@ KCSAN_SANITIZE_common.o := n
 
 obj-y			:= cacheinfo.o scattered.o
 obj-y			+= topology_common.o topology_ext.o topology_amd.o
+obj-y			+= cpuid_parser.o
 obj-y			+= common.o
 obj-y			+= rdrand.o
 obj-y			+= match.o
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 2f8e8ff2d000a..62f74a7f2f8de 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -16,6 +16,7 @@
 #include <asm/cacheinfo.h>
 #include <asm/cpu.h>
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/spec-ctrl.h>
 #include <asm/smp.h>
 #include <asm/numa.h>
diff --git a/arch/x86/kernel/cpu/centaur.c b/arch/x86/kernel/cpu/centaur.c
index 81695da9c5242..681d2da493412 100644
--- a/arch/x86/kernel/cpu/centaur.c
+++ b/arch/x86/kernel/cpu/centaur.c
@@ -5,6 +5,7 @@
 
 #include <asm/cpu.h>
 #include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
 #include <asm/e820/api.h>
 #include <asm/mtrr.h>
 #include <asm/msr.h>
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index a4268c47f2bc3..a3df21d264606 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1737,13 +1737,6 @@ static void __init cpu_parse_early_param(void)
 	int arglen;
 
 #ifdef CONFIG_X86_32
-	if (cmdline_find_option_bool(boot_command_line, "no387"))
-#ifdef CONFIG_MATH_EMULATION
-		setup_clear_cpu_cap(X86_FEATURE_FPU);
-#else
-		pr_err("Option 'no387' required CONFIG_MATH_EMULATION enabled.\n");
-#endif
-
 	if (cmdline_find_option_bool(boot_command_line, "nofxsr"))
 		setup_clear_cpu_cap(X86_FEATURE_FXSR);
 #endif
@@ -1791,6 +1784,7 @@ static void __init cpu_parse_early_param(void)
 static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 {
 	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
+	memset(&c->cpuid, 0, sizeof(c->cpuid));
 	c->extended_cpuid_level = 0;
 
 	if (!cpuid_feature())
@@ -1798,6 +1792,7 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
 
 	/* cyrix could have cpuid enabled via c_identify()*/
 	if (cpuid_feature()) {
+		cpuid_scan_cpu(c);
 		cpu_detect(c);
 		get_cpu_vendor(c);
 		intel_unlock_cpuid_leafs(c);
@@ -1970,8 +1965,8 @@ static void generic_identify(struct cpuinfo_x86 *c)
 	if (!cpuid_feature())
 		return;
 
+	cpuid_scan_cpu(c);
 	cpu_detect(c);
-
 	get_cpu_vendor(c);
 	intel_unlock_cpuid_leafs(c);
 	get_cpu_cap(c);
@@ -2023,6 +2018,7 @@ static void identify_cpu(struct cpuinfo_x86 *c)
 #endif
 	c->x86_cache_alignment = c->x86_clflush_size;
 	memset(&c->x86_capability, 0, sizeof(c->x86_capability));
+	memset(&c->cpuid, 0, sizeof(c->cpuid));
 #ifdef CONFIG_X86_VMX_FEATURE_NAMES
 	memset(&c->vmx_capability, 0, sizeof(c->vmx_capability));
 #endif
diff --git a/arch/x86/kernel/cpu/cpuid-deps.c b/arch/x86/kernel/cpu/cpuid-deps.c
index 146f6f8b0650b..99801e844b305 100644
--- a/arch/x86/kernel/cpu/cpuid-deps.c
+++ b/arch/x86/kernel/cpu/cpuid-deps.c
@@ -92,6 +92,7 @@ static const struct cpuid_dep cpuid_deps[] = {
 	{ X86_FEATURE_FRED,			X86_FEATURE_LKGS      },
 	{ X86_FEATURE_SPEC_CTRL_SSBD,		X86_FEATURE_SPEC_CTRL },
 	{ X86_FEATURE_LASS,			X86_FEATURE_SMAP      },
+	{ X86_FEATURE_INVLPGB,			X86_FEATURE_PCID      },
 	{}
 };
 
diff --git a/arch/x86/kernel/cpu/cpuid_parser.c b/arch/x86/kernel/cpu/cpuid_parser.c
new file mode 100644
index 0000000000000..898b0c4414316
--- /dev/null
+++ b/arch/x86/kernel/cpu/cpuid_parser.c
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * CPUID parser; for populating the system's CPUID tables.
+ */
+
+#include <linux/kernel.h>
+
+#include <asm/cpuid/api.h>
+#include <asm/processor.h>
+
+#include "cpuid_parser.h"
+
+/* Clear a single CPUID table entry */
+static void cpuid_clear(const struct cpuid_parse_entry *e, const struct cpuid_read_output *output)
+{
+	struct cpuid_regs *regs = output->regs;
+
+	for (int i = 0; i < e->maxcnt; i++, regs++)
+		memset(regs, 0, sizeof(*regs));
+
+	memset(output->info, 0, sizeof(*output->info));
+}
+
+/*
+ * Leaf read functions:
+ */
+
+/*
+ * Default CPUID read function
+ * Satisfies the requirements stated at 'struct cpuid_parse_entry'->read().
+ */
+static void
+cpuid_read_generic(const struct cpuid_parse_entry *e, const struct cpuid_read_output *output)
+{
+	struct cpuid_regs *regs = output->regs;
+
+	for (int i = 0; i < e->maxcnt; i++, regs++, output->info->nr_entries++)
+		cpuid_read_subleaf(e->leaf, e->subleaf + i, regs);
+}
+
+/*
+ * CPUID parser table:
+ */
+
+static const struct cpuid_parse_entry cpuid_parse_entries[] = {
+	CPUID_PARSE_ENTRIES
+};
+
+/*
+ * Leaf-independent parser code:
+ */
+
+static unsigned int cpuid_range_max_leaf(const struct cpuid_table *t, unsigned int range)
+{
+	const struct leaf_0x0_0 *l0 = __cpuid_table_subleaf(t, 0x0, 0);
+
+	switch (range) {
+	case CPUID_BASE_START:	return l0  ?  l0->max_std_leaf : 0;
+	default:		return 0;
+	}
+}
+
+static void
+__cpuid_reset_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[],
+		    unsigned int nr_entries, unsigned int start, unsigned int end, bool fill)
+{
+	const struct cpuid_parse_entry *entry = entries;
+	unsigned int range = CPUID_RANGE(start);
+
+	for (unsigned int i = 0; i < nr_entries; i++, entry++) {
+		struct cpuid_read_output output = {
+			.regs = cpuid_table_regs_p(t, entry->regs_offs),
+			.info = cpuid_table_info_p(t, entry->info_offs),
+		};
+
+		if (entry->leaf < start || entry->leaf > end)
+			continue;
+
+		cpuid_clear(entry, &output);
+
+		/*
+		 * Read the range's anchor leaf unconditionally so that the cached
+		 * maximum valid leaf value is available for the remaining entries.
+		 */
+		if (fill && (entry->leaf == range || entry->leaf <= cpuid_range_max_leaf(t, range)))
+			entry->read(entry, &output);
+	}
+}
+
+/*
+ * Zero all cached CPUID entries within [@start-@end] range.  This is needed when
+ * certain operations like MSR writes induce changes to the CPU's CPUID layout.
+ */
+static void
+__cpuid_zero_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[],
+		   unsigned int nr_entries, unsigned int start, unsigned int end)
+{
+	__cpuid_reset_table(t, entries, nr_entries, start, end, false);
+}
+
+static void
+__cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[],
+		   unsigned int nr_entries, unsigned int start, unsigned int end)
+{
+	__cpuid_reset_table(t, entries, nr_entries, start, end, true);
+}
+
+static void
+cpuid_fill_table(struct cpuid_table *t, const struct cpuid_parse_entry entries[], unsigned int nr_entries)
+{
+	static const struct {
+		unsigned int start;
+		unsigned int end;
+	} ranges[] = {
+		{ CPUID_BASE_START, CPUID_BASE_END },
+	};
+
+	for (unsigned int i = 0; i < ARRAY_SIZE(ranges); i++)
+		__cpuid_fill_table(t, entries, nr_entries, ranges[i].start, ranges[i].end);
+}
+
+static void __cpuid_scan_cpu_full(struct cpuinfo_x86 *c)
+{
+	unsigned int nr_entries = ARRAY_SIZE(cpuid_parse_entries);
+	struct cpuid_table *table = &c->cpuid;
+
+	cpuid_fill_table(table, cpuid_parse_entries, nr_entries);
+}
+
+static void
+__cpuid_scan_cpu_partial(struct cpuinfo_x86 *c, unsigned int start_leaf, unsigned int end_leaf)
+{
+	unsigned int nr_entries = ARRAY_SIZE(cpuid_parse_entries);
+	struct cpuid_table *table = &c->cpuid;
+
+	__cpuid_zero_table(table, cpuid_parse_entries, nr_entries, start_leaf, end_leaf);
+	__cpuid_fill_table(table, cpuid_parse_entries, nr_entries, start_leaf, end_leaf);
+}
+
+/*
+ * Call-site APIs:
+ */
+
+/**
+ * cpuid_scan_cpu() - Populate current CPU's CPUID table
+ * @c:		CPU capability structure associated with the current CPU
+ *
+ * Populate the CPUID table embedded within @c with parsed CPUID data.  All CPUID
+ * instructions are invoked locally, so this must be called on the CPU associated
+ * with @c.
+ */
+void cpuid_scan_cpu(struct cpuinfo_x86 *c)
+{
+	__cpuid_scan_cpu_full(c);
+}
+
+/**
+ * cpuid_refresh_range() - Rescan a CPUID table's leaf range
+ * @c:		CPU capability structure associated with the current CPU
+ * @start:	Start of leaf range to be re-scanned
+ * @end:	End of leaf range
+ */
+void cpuid_refresh_range(struct cpuinfo_x86 *c, u32 start, u32 end)
+{
+	if (WARN_ON_ONCE(start > end))
+		return;
+
+	if (WARN_ON_ONCE(CPUID_RANGE(start) != CPUID_RANGE(end)))
+		return;
+
+	__cpuid_scan_cpu_partial(c, start, end);
+}
+
+/**
+ * cpuid_refresh_leaf() - Rescan a CPUID table's leaf
+ * @c:		CPU capability structure associated with the current CPU
+ * @leaf:	Leaf to be re-scanned
+ */
+void cpuid_refresh_leaf(struct cpuinfo_x86 *c, u32 leaf)
+{
+	cpuid_refresh_range(c, leaf, leaf);
+}
diff --git a/arch/x86/kernel/cpu/cpuid_parser.h b/arch/x86/kernel/cpu/cpuid_parser.h
new file mode 100644
index 0000000000000..df627306cc8c4
--- /dev/null
+++ b/arch/x86/kernel/cpu/cpuid_parser.h
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ARCH_X86_CPUID_PARSER_H
+#define _ARCH_X86_CPUID_PARSER_H
+
+#include <asm/cpuid/types.h>
+
+/*
+ * Since accessing the CPUID leaves at 'struct cpuid_leaves' require compile time
+ * tokenization, split the CPUID parser into two stages: compile time macros for
+ * tokenizing the leaf/subleaf output offsets within the table, and generic runtime
+ * code to write to the relevant CPUID leaves using such offsets.
+ *
+ * The output of the compile time macros is cached by a compile time "parse entry"
+ * table (see 'struct cpuid_parse_entry').  The runtime parser code will utilize
+ * such offsets by passing them to the cpuid_table_*_p() functions.
+ */
+
+/*
+ * Compile time CPUID table offset calculations:
+ *
+ * @_leaf:	CPUID leaf, in 0xN format
+ * @_subleaf:	CPUID subleaf, in decimal format
+ */
+
+#define __cpuid_leaves_regs_offset(_leaf, _subleaf)			\
+	offsetof(struct cpuid_leaves, leaf_ ## _leaf ## _ ## _subleaf)
+
+#define __cpuid_leaves_info_offset(_leaf, _subleaf)			\
+	offsetof(struct cpuid_leaves, leaf_ ## _leaf ## _ ## _subleaf ## _ ## info)
+
+#define __cpuid_leaves_regs_maxcnt(_leaf, _subleaf)			\
+	ARRAY_SIZE(((struct cpuid_leaves *)NULL)->leaf_ ## _leaf ## _ ## _subleaf)
+
+/*
+ * Translation of compile time offsets to generic runtime pointers:
+ */
+
+static inline struct cpuid_regs *
+cpuid_table_regs_p(const struct cpuid_table *t, unsigned long regs_offset)
+{
+	return (struct cpuid_regs *)((unsigned long)(&t->leaves) + regs_offset);
+}
+
+static inline struct leaf_parse_info *
+cpuid_table_info_p(const struct cpuid_table *t, unsigned long info_offset)
+{
+	return (struct leaf_parse_info *)((unsigned long)(&t->leaves) + info_offset);
+}
+
+/**
+ * struct cpuid_read_output - Output of a CPUID read operation
+ * @regs:	Pointer to an array of CPUID outputs, where each array element covers the
+ *		full EAX->EDX output range.
+ * @info:	Pointer to query info; for saving the number of filled elements at @regs.
+ *
+ * A CPUID parser read function like cpuid_read_generic() or cpuid_read_0xN() uses this
+ * structure to save the CPUID query outputs.  Actual storage for @regs and @info is
+ * provided by the read function caller, and is typically within the CPU's CPUID table.
+ *
+ * See struct cpuid_parse_entry.read().
+ */
+struct cpuid_read_output {
+	struct cpuid_regs	*regs;
+	struct leaf_parse_info	*info;
+};
+
+/**
+ * struct cpuid_parse_entry - CPUID parse table entry
+ * @leaf:	Leaf number to be parsed
+ * @subleaf:	Subleaf number to be parsed
+ * @regs_offs:	Offset within 'struct cpuid_leaves' for saving the CPUID query output; to be
+ *		passed to cpuid_table_regs_p().
+ * @info_offs:	Offset within 'struct cpuid_leaves' for saving the CPUID query parse info; to be
+ *		passed to cpuid_table_info_p().
+ * @maxcnt:	Maximum number of output storage entries available for the CPUID query.
+ * @read:	Read function for this entry.  It must save the parsed CPUID output to the passed
+ *		'struct cpuid_read_output'->regs array of size >= @maxcnt.  It must set
+ *		'struct cpuid_read_output'->info.nr_entries to the number of CPUID output entries
+ *		parsed and filled.  A generic implementation is provided at cpuid_read_generic().
+ */
+struct cpuid_parse_entry {
+	unsigned int	leaf;
+	unsigned int	subleaf;
+	unsigned int	regs_offs;
+	unsigned int	info_offs;
+	unsigned int	maxcnt;
+	void		(*read)(const struct cpuid_parse_entry *e, const struct cpuid_read_output *o);
+};
+
+#define __CPUID_PARSE_ENTRY(_leaf, _subleaf, _suffix, _reader_fn)		\
+	{									\
+		.leaf		= _leaf,					\
+		.subleaf	= _subleaf,					\
+		.regs_offs	= __cpuid_leaves_regs_offset(_leaf, _suffix),	\
+		.info_offs	= __cpuid_leaves_info_offset(_leaf, _suffix),	\
+		.maxcnt		= __cpuid_leaves_regs_maxcnt(_leaf, _suffix),	\
+		.read		= cpuid_read_ ## _reader_fn,			\
+	}
+
+/*
+ * CPUID_PARSE_ENTRY_N() is for parsing CPUID leaves with a subleaf range.
+ * Check <asm/cpuid/types.h> __CPUID_LEAF() vs. CPUID_LEAF_N().
+ */
+
+#define CPUID_PARSE_ENTRY(_leaf, _subleaf, _reader_fn)				\
+	__CPUID_PARSE_ENTRY(_leaf, _subleaf, _subleaf, _reader_fn)
+
+#define CPUID_PARSE_ENTRY_N(_leaf, _reader_fn)					\
+	__CPUID_PARSE_ENTRY(_leaf, __cpuid_leaf_first_subleaf(_leaf), n, _reader_fn)
+
+/*
+ * CPUID parser table:
+ */
+
+#define CPUID_PARSE_ENTRIES									\
+	/*			Leaf		Subleaf		Reader function */		\
+	CPUID_PARSE_ENTRY   (	0x0,		0,		generic			),	\
+	CPUID_PARSE_ENTRY   (	0x1,		0,		generic			),	\
+
+#endif /* _ARCH_X86_CPUID_PARSER_H */
diff --git a/arch/x86/kernel/cpu/hygon.c b/arch/x86/kernel/cpu/hygon.c
index 7f95a74e4c657..3e8891a9caf27 100644
--- a/arch/x86/kernel/cpu/hygon.c
+++ b/arch/x86/kernel/cpu/hygon.c
@@ -10,6 +10,7 @@
 
 #include <asm/apic.h>
 #include <asm/cpu.h>
+#include <asm/cpuid/api.h>
 #include <asm/smp.h>
 #include <asm/numa.h>
 #include <asm/cacheinfo.h>
diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c
index f3a793e3a6c8f..eebc60342cd66 100644
--- a/arch/x86/kernel/cpu/mce/core.c
+++ b/arch/x86/kernel/cpu/mce/core.c
@@ -49,6 +49,7 @@
 
 #include <asm/fred.h>
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/processor.h>
 #include <asm/traps.h>
 #include <asm/tlbflush.h>
diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c
index d02c4f556cd05..42c82c14c48a0 100644
--- a/arch/x86/kernel/cpu/mce/inject.c
+++ b/arch/x86/kernel/cpu/mce/inject.c
@@ -26,6 +26,7 @@
 
 #include <asm/amd/nb.h>
 #include <asm/apic.h>
+#include <asm/cpuid/api.h>
 #include <asm/irq_vectors.h>
 #include <asm/mce.h>
 #include <asm/msr.h>
diff --git a/arch/x86/kernel/cpu/microcode/amd.c b/arch/x86/kernel/cpu/microcode/amd.c
index e533881284a12..874b5b70c0d27 100644
--- a/arch/x86/kernel/cpu/microcode/amd.c
+++ b/arch/x86/kernel/cpu/microcode/amd.c
@@ -34,6 +34,7 @@
 
 #include <asm/microcode.h>
 #include <asm/processor.h>
+#include <asm/cpuid/api.h>
 #include <asm/cmdline.h>
 #include <asm/setup.h>
 #include <asm/cpu.h>
diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c
index 651202e6fefbe..d822a673c80a1 100644
--- a/arch/x86/kernel/cpu/microcode/core.c
+++ b/arch/x86/kernel/cpu/microcode/core.c
@@ -34,6 +34,7 @@
 
 #include <asm/apic.h>
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/perf_event.h>
 #include <asm/processor.h>
 #include <asm/cmdline.h>
@@ -126,7 +127,7 @@ bool __init microcode_loader_disabled(void)
 	}
 
 	/*
-	 * 2) Bit 31 in CPUID[1]:ECX is clear
+	 * 2) Bit 31 in CPUID[1]:ECX is set
 	 *    The bit is reserved for hypervisor use. This is still not
 	 *    completely accurate as XEN PV guests don't see that CPUID bit
 	 *    set, but that's good enough as they don't land on the BSP
diff --git a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
index 72c8809f88e52..af8b1d8b26f68 100644
--- a/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
+++ b/arch/x86/kernel/cpu/microcode/intel-ucode-defs.h
@@ -1,238 +1,241 @@
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x03, .steppings = 0x0004, .platform_mask = 0x00, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x40 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0001, .platform_mask = 0x02, .driver_data = 0x41 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0001, .platform_mask = 0x08, .driver_data = 0x45 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x40 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x2a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0004, .platform_mask = 0x02, .driver_data = 0x2c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x10 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0008, .platform_mask = 0x02, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0xb },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x05, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0xb },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x2000, .platform_mask = 0x02, .driver_data = 0x5 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x2000, .platform_mask = 0x08, .driver_data = 0x6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x06, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x07, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x14 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x07, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x38 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x07, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0x2e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x10 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0xf },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x11 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xe },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0008, .platform_mask = 0x20, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0x4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x08, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x5 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x09, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x09, .steppings = 0x0020, .platform_mask = 0x20, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x09, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0x47 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0a, .steppings = 0x0001, .platform_mask = 0x04, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0a, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x1c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0x1d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x20, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0d, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0x18 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0e, .steppings = 0x0100, .platform_mask = 0x20, .driver_data = 0x39 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x20, .driver_data = 0x54 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x80, .driver_data = 0x59 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x5d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x5c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0xd0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0xd1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x6a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x40, .driver_data = 0x6b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x95 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x01, .driver_data = 0xba },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x04, .driver_data = 0xbc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x08, .driver_data = 0xbb },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x10, .driver_data = 0xba },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x20, .driver_data = 0xba },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x40, .driver_data = 0xbc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x80, .driver_data = 0xba },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x01, .driver_data = 0xa4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0xa4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x80, .driver_data = 0xa4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x16, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x43 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x16, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x42 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x16, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x44 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x40, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0x60f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x70a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0400, .platform_mask = 0x11, .driver_data = 0xa0b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0400, .platform_mask = 0x44, .driver_data = 0xa0b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x17, .steppings = 0x0400, .platform_mask = 0xa0, .driver_data = 0xa0b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1a, .steppings = 0x0010, .platform_mask = 0x03, .driver_data = 0x12 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1a, .steppings = 0x0020, .platform_mask = 0x03, .driver_data = 0x1d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x217 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x218 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x08, .driver_data = 0x219 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x107 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x107 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0x107 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x107 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1d, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0x29 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x1e, .steppings = 0x0020, .platform_mask = 0x13, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x25, .steppings = 0x0004, .platform_mask = 0x12, .driver_data = 0x11 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x25, .steppings = 0x0020, .platform_mask = 0x92, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x26, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x104 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x26, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x105 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2a, .steppings = 0x0080, .platform_mask = 0x12, .driver_data = 0x2f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2c, .steppings = 0x0004, .platform_mask = 0x03, .driver_data = 0x1f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2d, .steppings = 0x0040, .platform_mask = 0x6d, .driver_data = 0x621 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2d, .steppings = 0x0080, .platform_mask = 0x6d, .driver_data = 0x71a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2e, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x2f, .steppings = 0x0004, .platform_mask = 0x05, .driver_data = 0x3b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x37, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0x838 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x37, .steppings = 0x0100, .platform_mask = 0x0c, .driver_data = 0x838 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x37, .steppings = 0x0200, .platform_mask = 0x0f, .driver_data = 0x90d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3a, .steppings = 0x0200, .platform_mask = 0x12, .driver_data = 0x21 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3c, .steppings = 0x0008, .platform_mask = 0x32, .driver_data = 0x28 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3d, .steppings = 0x0010, .platform_mask = 0xc0, .driver_data = 0x2f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3e, .steppings = 0x0010, .platform_mask = 0xed, .driver_data = 0x42e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3e, .steppings = 0x0040, .platform_mask = 0xed, .driver_data = 0x600 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3e, .steppings = 0x0080, .platform_mask = 0xed, .driver_data = 0x715 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3f, .steppings = 0x0004, .platform_mask = 0x6f, .driver_data = 0x49 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x3f, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x1a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x45, .steppings = 0x0002, .platform_mask = 0x72, .driver_data = 0x26 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x46, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x1c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x47, .steppings = 0x0002, .platform_mask = 0x22, .driver_data = 0x22 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4c, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x368 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4c, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x411 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4d, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x12d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x4e, .steppings = 0x0008, .platform_mask = 0xc0, .driver_data = 0xf0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0008, .platform_mask = 0x97, .driver_data = 0x1000191 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0010, .platform_mask = 0xb7, .driver_data = 0x2007006 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0020, .platform_mask = 0xb7, .driver_data = 0x3000010 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0080, .platform_mask = 0xbf, .driver_data = 0x5003901 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x55, .steppings = 0x0800, .platform_mask = 0xbf, .driver_data = 0x7002b01 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0004, .platform_mask = 0x10, .driver_data = 0x1c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0008, .platform_mask = 0x10, .driver_data = 0x700001c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0xf00001a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x56, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0xe000015 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x14 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5c, .steppings = 0x0200, .platform_mask = 0x03, .driver_data = 0x48 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5c, .steppings = 0x0400, .platform_mask = 0x03, .driver_data = 0x28 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5e, .steppings = 0x0008, .platform_mask = 0x36, .driver_data = 0xf0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x5f, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x3e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x66, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x2a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x6a, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0xc0002f0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x6a, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0xd000404 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x6c, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x10002d0 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x7a, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x42 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x7a, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x26 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x7e, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0xca },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8a, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x33 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8c, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0xbc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8c, .steppings = 0x0004, .platform_mask = 0xc2, .driver_data = 0x3c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8d, .steppings = 0x0002, .platform_mask = 0xc2, .driver_data = 0x56 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0200, .platform_mask = 0x10, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0200, .platform_mask = 0xc0, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0400, .platform_mask = 0xc0, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x0800, .platform_mask = 0xd0, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8e, .steppings = 0x1000, .platform_mask = 0x94, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0080, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x10, .driver_data = 0x2c0003f7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x87, .driver_data = 0x2b000639 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x96, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x1a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x97, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x97, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9a, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x437 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x40, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x437 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9c, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x24000026 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x0200, .platform_mask = 0x2a, .driver_data = 0xf8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x0400, .platform_mask = 0x22, .driver_data = 0xfa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x0800, .platform_mask = 0x02, .driver_data = 0xf6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x1000, .platform_mask = 0x22, .driver_data = 0xf8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0x9e, .steppings = 0x2000, .platform_mask = 0x22, .driver_data = 0x104 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa5, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa5, .steppings = 0x0008, .platform_mask = 0x22, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa5, .steppings = 0x0020, .platform_mask = 0x22, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa6, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0x102 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa6, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x100 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xa7, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x64 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xaa, .steppings = 0x0010, .platform_mask = 0xe6, .driver_data = 0x24 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xad, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xa0000d1 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xad, .steppings = 0x0002, .platform_mask = 0x95, .driver_data = 0x10003a2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xaf, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x3000341 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xb5, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xb7, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x12f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xb7, .steppings = 0x0010, .platform_mask = 0x32, .driver_data = 0x12f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xba, .steppings = 0x0004, .platform_mask = 0xe0, .driver_data = 0x4128 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xba, .steppings = 0x0008, .platform_mask = 0xe0, .driver_data = 0x4128 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xba, .steppings = 0x0100, .platform_mask = 0xe0, .driver_data = 0x4128 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbd, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x11f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbe, .steppings = 0x0001, .platform_mask = 0x19, .driver_data = 0x1d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0040, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xbf, .steppings = 0x0080, .platform_mask = 0x07, .driver_data = 0x3a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xc5, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xc6, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xc6, .steppings = 0x0010, .platform_mask = 0x82, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xca, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x118 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xcf, .steppings = 0x0002, .platform_mask = 0x87, .driver_data = 0x210002a9 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6,  .model = 0xcf, .steppings = 0x0004, .platform_mask = 0x87, .driver_data = 0x210002a9 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0080, .platform_mask = 0x01, .driver_data = 0x12 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x13 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0x15 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x00, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x14 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x01, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0010, .platform_mask = 0x02, .driver_data = 0x1f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0010, .platform_mask = 0x04, .driver_data = 0x1e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x21 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x29 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0020, .platform_mask = 0x02, .driver_data = 0x2a },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0020, .platform_mask = 0x04, .driver_data = 0x2b },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0x10 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x38 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0080, .platform_mask = 0x04, .driver_data = 0x37 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0080, .platform_mask = 0x08, .driver_data = 0x39 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0200, .platform_mask = 0x02, .driver_data = 0x2d },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0200, .platform_mask = 0x04, .driver_data = 0x2e },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x02, .steppings = 0x0200, .platform_mask = 0x08, .driver_data = 0x2f },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x03, .steppings = 0x0004, .platform_mask = 0x0d, .driver_data = 0xa },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x03, .steppings = 0x0008, .platform_mask = 0x0d, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x03, .steppings = 0x0010, .platform_mask = 0x1d, .driver_data = 0x17 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x16 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0002, .platform_mask = 0xbd, .driver_data = 0x17 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0008, .platform_mask = 0x9d, .driver_data = 0x5 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0010, .platform_mask = 0x9d, .driver_data = 0x6 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0080, .platform_mask = 0x9d, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0xc },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0xe },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0100, .platform_mask = 0x5f, .driver_data = 0x7 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0200, .platform_mask = 0xbd, .driver_data = 0x3 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5c, .driver_data = 0x4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5d, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0xf },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x2 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0010, .platform_mask = 0x34, .driver_data = 0x4 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x8 },
-{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf,  .model = 0x06, .steppings = 0x0100, .platform_mask = 0x22, .driver_data = 0x9 },
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Auto-generated by scripts/update-intel-ucode-defs.py */
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x00, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x40 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x02, .driver_data = 0x41 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0001, .platform_mask = 0x08, .driver_data = 0x45 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x40 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x2a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x02, .driver_data = 0x2c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x10 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x02, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0xb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x05, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0xb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x02, .driver_data = 0x5 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x08, .driver_data = 0x6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x06, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x14 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x38 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x07, .steppings = 0x0008, .platform_mask = 0x04, .driver_data = 0x2e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x10 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0xf },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x11 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xe },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x08, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0008, .platform_mask = 0x20, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x20, .driver_data = 0x4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x08, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x5 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x20, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x09, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0x47 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0001, .platform_mask = 0x04, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0a, .steppings = 0x0002, .platform_mask = 0x04, .driver_data = 0x1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x1c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0x1d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0b, .steppings = 0x0010, .platform_mask = 0x20, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0d, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0x18 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x0100, .platform_mask = 0x20, .driver_data = 0x39 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x20, .driver_data = 0x54 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0e, .steppings = 0x1000, .platform_mask = 0x80, .driver_data = 0x59 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x5d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x5c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0xd0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0040, .platform_mask = 0x20, .driver_data = 0xd1 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x6a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0080, .platform_mask = 0x40, .driver_data = 0x6b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0400, .platform_mask = 0x80, .driver_data = 0x95 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x01, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x04, .driver_data = 0xbc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x08, .driver_data = 0xbb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x10, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x20, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x40, .driver_data = 0xbc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x0800, .platform_mask = 0x80, .driver_data = 0xba },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x01, .driver_data = 0xa4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x20, .driver_data = 0xa4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x0f, .steppings = 0x2000, .platform_mask = 0x80, .driver_data = 0xa4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x43 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x42 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x16, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x44 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x01, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x40, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0040, .platform_mask = 0x80, .driver_data = 0x60f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0080, .platform_mask = 0x10, .driver_data = 0x70a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x11, .driver_data = 0xa0b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0x44, .driver_data = 0xa0b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x17, .steppings = 0x0400, .platform_mask = 0xa0, .driver_data = 0xa0b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0010, .platform_mask = 0x03, .driver_data = 0x12 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1a, .steppings = 0x0020, .platform_mask = 0x03, .driver_data = 0x1d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x217 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x218 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0004, .platform_mask = 0x08, .driver_data = 0x219 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x08, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1c, .steppings = 0x0400, .platform_mask = 0x10, .driver_data = 0x107 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1d, .steppings = 0x0002, .platform_mask = 0x08, .driver_data = 0x29 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x1e, .steppings = 0x0020, .platform_mask = 0x13, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0004, .platform_mask = 0x12, .driver_data = 0x11 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x25, .steppings = 0x0020, .platform_mask = 0x92, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x104 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x26, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x105 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2a, .steppings = 0x0080, .platform_mask = 0x12, .driver_data = 0x2f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2c, .steppings = 0x0004, .platform_mask = 0x03, .driver_data = 0x1f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0040, .platform_mask = 0x6d, .driver_data = 0x621 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2d, .steppings = 0x0080, .platform_mask = 0x6d, .driver_data = 0x71a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2e, .steppings = 0x0040, .platform_mask = 0x04, .driver_data = 0xd },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x2f, .steppings = 0x0004, .platform_mask = 0x05, .driver_data = 0x3b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0x838 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0100, .platform_mask = 0x0c, .driver_data = 0x838 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x37, .steppings = 0x0200, .platform_mask = 0x0f, .driver_data = 0x90d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3a, .steppings = 0x0200, .platform_mask = 0x12, .driver_data = 0x21 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3c, .steppings = 0x0008, .platform_mask = 0x32, .driver_data = 0x28 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3d, .steppings = 0x0010, .platform_mask = 0xc0, .driver_data = 0x2f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0010, .platform_mask = 0xed, .driver_data = 0x42e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0040, .platform_mask = 0xed, .driver_data = 0x600 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3e, .steppings = 0x0080, .platform_mask = 0xed, .driver_data = 0x715 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0004, .platform_mask = 0x6f, .driver_data = 0x49 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x3f, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x1a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x45, .steppings = 0x0002, .platform_mask = 0x72, .driver_data = 0x26 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x46, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x1c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x47, .steppings = 0x0002, .platform_mask = 0x22, .driver_data = 0x22 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x368 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4c, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x411 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4d, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x12d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x4e, .steppings = 0x0008, .platform_mask = 0xc0, .driver_data = 0xf0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0008, .platform_mask = 0x97, .driver_data = 0x1000191 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0010, .platform_mask = 0xb7, .driver_data = 0x2007006 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0020, .platform_mask = 0xb7, .driver_data = 0x3000010 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0080, .platform_mask = 0xbf, .driver_data = 0x5003901 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x55, .steppings = 0x0800, .platform_mask = 0xbf, .driver_data = 0x7002b01 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0004, .platform_mask = 0x10, .driver_data = 0x1c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0008, .platform_mask = 0x10, .driver_data = 0x700001c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0xf00001a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x56, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0xe000015 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0004, .platform_mask = 0x01, .driver_data = 0x14 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0200, .platform_mask = 0x03, .driver_data = 0x48 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5c, .steppings = 0x0400, .platform_mask = 0x03, .driver_data = 0x28 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5e, .steppings = 0x0008, .platform_mask = 0x36, .driver_data = 0xf0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x5f, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x3e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x66, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x2a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0xc0002f0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6a, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0xd000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x6c, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x10002e0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x42 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7a, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0x26 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x7e, .steppings = 0x0020, .platform_mask = 0x80, .driver_data = 0xca },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8a, .steppings = 0x0002, .platform_mask = 0x10, .driver_data = 0x33 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0xbc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8c, .steppings = 0x0004, .platform_mask = 0xc2, .driver_data = 0x3c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8d, .steppings = 0x0002, .platform_mask = 0xc2, .driver_data = 0x56 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0x10, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0200, .platform_mask = 0xc0, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0400, .platform_mask = 0xc0, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x0800, .platform_mask = 0xd0, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8e, .steppings = 0x1000, .platform_mask = 0x94, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x2c000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0010, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0020, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x10, .driver_data = 0x2c000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0040, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0080, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x10, .driver_data = 0x2c000410 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x8f, .steppings = 0x0100, .platform_mask = 0x87, .driver_data = 0x2b000650 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x96, .steppings = 0x0002, .platform_mask = 0x01, .driver_data = 0x1a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x97, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0008, .platform_mask = 0x80, .driver_data = 0x43a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x40, .driver_data = 0xb },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9a, .steppings = 0x0010, .platform_mask = 0x80, .driver_data = 0x43a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9c, .steppings = 0x0001, .platform_mask = 0x01, .driver_data = 0x24000026 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0200, .platform_mask = 0x2a, .driver_data = 0xf8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0400, .platform_mask = 0x22, .driver_data = 0xfa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x0800, .platform_mask = 0x02, .driver_data = 0xf6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x1000, .platform_mask = 0x22, .driver_data = 0xf8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0x9e, .steppings = 0x2000, .platform_mask = 0x22, .driver_data = 0x104 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0004, .platform_mask = 0x20, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0008, .platform_mask = 0x22, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa5, .steppings = 0x0020, .platform_mask = 0x22, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0x102 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa6, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x100 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xa7, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x64 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaa, .steppings = 0x0010, .platform_mask = 0xe6, .driver_data = 0x25 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x20, .driver_data = 0xa000124 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xad, .steppings = 0x0002, .platform_mask = 0x95, .driver_data = 0x10003f0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xae, .steppings = 0x0002, .platform_mask = 0x97, .driver_data = 0x1000273 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xaf, .steppings = 0x0008, .platform_mask = 0x01, .driver_data = 0x3000382 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb5, .steppings = 0x0001, .platform_mask = 0x80, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0002, .platform_mask = 0x32, .driver_data = 0x132 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xb7, .steppings = 0x0010, .platform_mask = 0x32, .driver_data = 0x132 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0004, .platform_mask = 0xe0, .driver_data = 0x6133 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0008, .platform_mask = 0xe0, .driver_data = 0x6133 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xba, .steppings = 0x0100, .platform_mask = 0xe0, .driver_data = 0x6133 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbd, .steppings = 0x0002, .platform_mask = 0x80, .driver_data = 0x125 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbe, .steppings = 0x0001, .platform_mask = 0x19, .driver_data = 0x1e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0004, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0020, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0040, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xbf, .steppings = 0x0080, .platform_mask = 0x07, .driver_data = 0x3d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc5, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xc6, .steppings = 0x0010, .platform_mask = 0x82, .driver_data = 0x11a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xca, .steppings = 0x0004, .platform_mask = 0x82, .driver_data = 0x11a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0002, .platform_mask = 0x87, .driver_data = 0x210002c0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x6, .model = 0xcf, .steppings = 0x0004, .platform_mask = 0x87, .driver_data = 0x210002c0 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x01, .driver_data = 0x12 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x01, .driver_data = 0x13 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x02, .driver_data = 0x15 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x00, .steppings = 0x0400, .platform_mask = 0x04, .driver_data = 0x14 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x01, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0x2e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x02, .driver_data = 0x1f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x04, .driver_data = 0x1e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0010, .platform_mask = 0x10, .driver_data = 0x21 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x29 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x02, .driver_data = 0x2a },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x04, .driver_data = 0x2b },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0020, .platform_mask = 0x10, .driver_data = 0x2c },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0040, .platform_mask = 0x02, .driver_data = 0x10 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x02, .driver_data = 0x38 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x04, .driver_data = 0x37 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0080, .platform_mask = 0x08, .driver_data = 0x39 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x02, .driver_data = 0x2d },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x04, .driver_data = 0x2e },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x02, .steppings = 0x0200, .platform_mask = 0x08, .driver_data = 0x2f },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0004, .platform_mask = 0x0d, .driver_data = 0xa },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0008, .platform_mask = 0x0d, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x03, .steppings = 0x0010, .platform_mask = 0x1d, .driver_data = 0x17 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0x02, .driver_data = 0x16 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0002, .platform_mask = 0xbd, .driver_data = 0x17 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0008, .platform_mask = 0x9d, .driver_data = 0x5 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0010, .platform_mask = 0x9d, .driver_data = 0x6 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0080, .platform_mask = 0x9d, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x01, .driver_data = 0xc },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x02, .driver_data = 0xe },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0100, .platform_mask = 0x5f, .driver_data = 0x7 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0200, .platform_mask = 0xbd, .driver_data = 0x3 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5c, .driver_data = 0x4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x04, .steppings = 0x0400, .platform_mask = 0x5d, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0004, .platform_mask = 0x04, .driver_data = 0xf },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x01, .driver_data = 0x2 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0010, .platform_mask = 0x34, .driver_data = 0x4 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0020, .platform_mask = 0x01, .driver_data = 0x8 },
+{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0xf, .model = 0x06, .steppings = 0x0100, .platform_mask = 0x22, .driver_data = 0x9 },
diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
index 37ac4afe09724..18d2eff7a4b73 100644
--- a/arch/x86/kernel/cpu/microcode/intel.c
+++ b/arch/x86/kernel/cpu/microcode/intel.c
@@ -25,6 +25,7 @@
 #include <linux/mm.h>
 
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/processor.h>
 #include <asm/tlbflush.h>
 #include <asm/setup.h>
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index b5b6a58b67b0c..640e6b223c2d4 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -19,6 +19,7 @@
 #include <linux/random.h>
 #include <asm/processor.h>
 #include <asm/hypervisor.h>
+#include <asm/cpuid/api.h>
 #include <hyperv/hvhdk.h>
 #include <asm/mshyperv.h>
 #include <asm/desc.h>
diff --git a/arch/x86/kernel/cpu/resctrl/core.c b/arch/x86/kernel/cpu/resctrl/core.c
index 7667cf7c4e945..9c01d2562b7a8 100644
--- a/arch/x86/kernel/cpu/resctrl/core.c
+++ b/arch/x86/kernel/cpu/resctrl/core.c
@@ -22,6 +22,7 @@
 #include <linux/cpuhotplug.h>
 
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 #include <asm/resctrl.h>
 #include "internal.h"
diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c
index 9bd87bae49834..9bf9d7e201aa5 100644
--- a/arch/x86/kernel/cpu/resctrl/monitor.c
+++ b/arch/x86/kernel/cpu/resctrl/monitor.c
@@ -21,6 +21,7 @@
 #include <linux/resctrl.h>
 
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 
 #include "internal.h"
@@ -454,6 +455,7 @@ int __init rdt_get_l3_mon_config(struct rdt_resource *r)
 	    (rdt_cpu_has(X86_FEATURE_CQM_MBM_TOTAL) ||
 	     rdt_cpu_has(X86_FEATURE_CQM_MBM_LOCAL))) {
 		r->mon.mbm_cntr_assignable = true;
+		r->mon.mbm_cntr_configurable = true;
 		cpuid_count(0x80000020, 5, &eax, &ebx, &ecx, &edx);
 		r->mon.num_mbm_cntrs = (ebx & GENMASK(15, 0)) + 1;
 		hw_res->mbm_cntr_assign_enabled = true;
diff --git a/arch/x86/kernel/cpu/scattered.c b/arch/x86/kernel/cpu/scattered.c
index 837d6a4b0c282..937129ce6a964 100644
--- a/arch/x86/kernel/cpu/scattered.c
+++ b/arch/x86/kernel/cpu/scattered.c
@@ -6,6 +6,7 @@
 
 #include <asm/memtype.h>
 #include <asm/apic.h>
+#include <asm/cpuid/api.h>
 #include <asm/processor.h>
 
 #include "cpu.h"
diff --git a/arch/x86/kernel/cpu/sgx/driver.c b/arch/x86/kernel/cpu/sgx/driver.c
index 473619741bc44..9268289cd9f92 100644
--- a/arch/x86/kernel/cpu/sgx/driver.c
+++ b/arch/x86/kernel/cpu/sgx/driver.c
@@ -6,7 +6,10 @@
 #include <linux/mman.h>
 #include <linux/security.h>
 #include <linux/suspend.h>
+
+#include <asm/cpuid/api.h>
 #include <asm/traps.h>
+
 #include "driver.h"
 #include "encl.h"
 
diff --git a/arch/x86/kernel/cpu/sgx/main.c b/arch/x86/kernel/cpu/sgx/main.c
index 38b7fd2f63be1..4505f808af5e5 100644
--- a/arch/x86/kernel/cpu/sgx/main.c
+++ b/arch/x86/kernel/cpu/sgx/main.c
@@ -15,9 +15,12 @@
 #include <linux/slab.h>
 #include <linux/sysfs.h>
 #include <linux/vmalloc.h>
+
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 #include <asm/sgx.h>
 #include <asm/archrandom.h>
+
 #include "driver.h"
 #include "encl.h"
 #include "encls.h"
diff --git a/arch/x86/kernel/cpu/topology_amd.c b/arch/x86/kernel/cpu/topology_amd.c
index 6ac097e131065..da080d732e107 100644
--- a/arch/x86/kernel/cpu/topology_amd.c
+++ b/arch/x86/kernel/cpu/topology_amd.c
@@ -2,6 +2,7 @@
 #include <linux/cpu.h>
 
 #include <asm/apic.h>
+#include <asm/cpuid/api.h>
 #include <asm/memtype.h>
 #include <asm/msr.h>
 #include <asm/processor.h>
@@ -79,7 +80,7 @@ static bool parse_8000_001e(struct topo_scan *tscan)
 	if (!boot_cpu_has(X86_FEATURE_TOPOEXT))
 		return false;
 
-	cpuid_leaf(0x8000001e, &leaf);
+	cpuid_read(0x8000001e, &leaf);
 
 	/*
 	 * If leaf 0xb/0x26 is available, then the APIC ID and the domain
diff --git a/arch/x86/kernel/cpu/topology_common.c b/arch/x86/kernel/cpu/topology_common.c
index d0d79d5b8eb9d..cf7513416b702 100644
--- a/arch/x86/kernel/cpu/topology_common.c
+++ b/arch/x86/kernel/cpu/topology_common.c
@@ -6,6 +6,7 @@
 #include <asm/intel-family.h>
 #include <asm/apic.h>
 #include <asm/processor.h>
+#include <asm/cpuid/api.h>
 #include <asm/smp.h>
 
 #include "cpu.h"
diff --git a/arch/x86/kernel/cpu/topology_ext.c b/arch/x86/kernel/cpu/topology_ext.c
index 467b0326bf1a1..60dfaa02ffd0d 100644
--- a/arch/x86/kernel/cpu/topology_ext.c
+++ b/arch/x86/kernel/cpu/topology_ext.c
@@ -2,6 +2,7 @@
 #include <linux/cpu.h>
 
 #include <asm/apic.h>
+#include <asm/cpuid/api.h>
 #include <asm/memtype.h>
 #include <asm/processor.h>
 
@@ -70,7 +71,7 @@ static inline bool topo_subleaf(struct topo_scan *tscan, u32 leaf, u32 subleaf,
 	default: return false;
 	}
 
-	cpuid_subleaf(leaf, subleaf, &sl);
+	cpuid_read_subleaf(leaf, subleaf, &sl);
 
 	if (!sl.num_processors || sl.type == INVALID_TYPE)
 		return false;
diff --git a/arch/x86/kernel/cpu/transmeta.c b/arch/x86/kernel/cpu/transmeta.c
index 42c939827621a..1fdcd69c625c2 100644
--- a/arch/x86/kernel/cpu/transmeta.c
+++ b/arch/x86/kernel/cpu/transmeta.c
@@ -3,8 +3,11 @@
 #include <linux/sched.h>
 #include <linux/sched/clock.h>
 #include <linux/mm.h>
+
 #include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
+
 #include "cpu.h"
 
 static void early_init_transmeta(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/umc.c b/arch/x86/kernel/cpu/umc.c
deleted file mode 100644
index 65a58a390fc31..0000000000000
--- a/arch/x86/kernel/cpu/umc.c
+++ /dev/null
@@ -1,26 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-#include <linux/kernel.h>
-#include <asm/processor.h>
-#include "cpu.h"
-
-/*
- * UMC chips appear to be only either 386 or 486,
- * so no special init takes place.
- */
-
-static const struct cpu_dev umc_cpu_dev = {
-	.c_vendor	= "UMC",
-	.c_ident	= { "UMC UMC UMC" },
-	.legacy_models	= {
-		{ .family = 4, .model_names =
-		  {
-			  [1] = "U5D",
-			  [2] = "U5S",
-		  }
-		},
-	},
-	.c_x86_vendor	= X86_VENDOR_UMC,
-};
-
-cpu_dev_register(umc_cpu_dev);
-
diff --git a/arch/x86/kernel/cpu/vmware.c b/arch/x86/kernel/cpu/vmware.c
index eee0d1a488029..34b73573b108d 100644
--- a/arch/x86/kernel/cpu/vmware.c
+++ b/arch/x86/kernel/cpu/vmware.c
@@ -33,6 +33,7 @@
 #include <asm/div64.h>
 #include <asm/x86_init.h>
 #include <asm/hypervisor.h>
+#include <asm/cpuid/api.h>
 #include <asm/timer.h>
 #include <asm/apic.h>
 #include <asm/vmware.h>
diff --git a/arch/x86/kernel/cpu/zhaoxin.c b/arch/x86/kernel/cpu/zhaoxin.c
index 031379b7d4faf..761aef5590ac4 100644
--- a/arch/x86/kernel/cpu/zhaoxin.c
+++ b/arch/x86/kernel/cpu/zhaoxin.c
@@ -4,6 +4,7 @@
 
 #include <asm/cpu.h>
 #include <asm/cpufeature.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 
 #include "cpu.h"
diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index dae436253de43..b55fe9c7359ad 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -37,6 +37,7 @@
 #include <linux/gfp.h>
 #include <linux/completion.h>
 
+#include <asm/cpuid/api.h>
 #include <asm/processor.h>
 #include <asm/msr.h>
 
@@ -58,8 +59,7 @@ static void cpuid_smp_cpuid(void *cmd_block)
 	complete(&cmd->done);
 }
 
-static ssize_t cpuid_read(struct file *file, char __user *buf,
-			  size_t count, loff_t *ppos)
+static ssize_t cpuid_read_f(struct file *file, char __user *buf, size_t count, loff_t *ppos)
 {
 	char __user *tmp = buf;
 	struct cpuid_regs_done cmd;
@@ -119,7 +119,7 @@ static int cpuid_open(struct inode *inode, struct file *file)
 static const struct file_operations cpuid_fops = {
 	.owner = THIS_MODULE,
 	.llseek = no_seek_end_llseek,
-	.read = cpuid_read,
+	.read = cpuid_read_f,
 	.open = cpuid_open,
 };
 
diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index cd796818d94d9..623d4474631a6 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -38,6 +38,7 @@
 #include <linux/kdebug.h>
 #include <asm/cpu.h>
 #include <asm/reboot.h>
+#include <asm/tdx.h>
 #include <asm/intel_pt.h>
 #include <asm/crash.h>
 #include <asm/cmdline.h>
@@ -112,6 +113,7 @@ void native_machine_crash_shutdown(struct pt_regs *regs)
 
 	crash_smp_send_stop();
 
+	tdx_sys_disable();
 	x86_virt_emergency_disable_virtualization_cpu();
 
 	/*
diff --git a/arch/x86/kernel/fpu/core.c b/arch/x86/kernel/fpu/core.c
index 608983806fd7f..584fb9913be42 100644
--- a/arch/x86/kernel/fpu/core.c
+++ b/arch/x86/kernel/fpu/core.c
@@ -558,11 +558,6 @@ static inline void fpstate_init_fstate(struct fpstate *fpstate)
  */
 void fpstate_init_user(struct fpstate *fpstate)
 {
-	if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
-		fpstate_init_soft(&fpstate->regs.soft);
-		return;
-	}
-
 	xstate_init_xcomp_bv(&fpstate->regs.xsave, fpstate->xfeatures);
 
 	if (cpu_feature_enabled(X86_FEATURE_FXSR))
diff --git a/arch/x86/kernel/fpu/init.c b/arch/x86/kernel/fpu/init.c
index ff988b9ea39f0..0d33c217b71c1 100644
--- a/arch/x86/kernel/fpu/init.c
+++ b/arch/x86/kernel/fpu/init.c
@@ -36,12 +36,7 @@ static void fpu__init_cpu_generic(void)
 	write_cr0(cr0);
 
 	/* Flush out any pending x87 state: */
-#ifdef CONFIG_MATH_EMULATION
-	if (!boot_cpu_has(X86_FEATURE_FPU))
-		;
-	else
-#endif
-		asm volatile ("fninit");
+	asm volatile ("fninit");
 }
 
 /*
@@ -86,13 +81,11 @@ static void __init fpu__init_system_early_generic(void)
 			setup_clear_cpu_cap(X86_FEATURE_FPU);
 	}
 
-#ifndef CONFIG_MATH_EMULATION
 	if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
 		pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
 		for (;;)
 			asm volatile("hlt");
 	}
-#endif
 }
 
 /*
diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c
index 2604565887564..7bcf1decc0349 100644
--- a/arch/x86/kernel/idt.c
+++ b/arch/x86/kernel/idt.c
@@ -268,6 +268,21 @@ void __init idt_setup_early_pf(void)
 }
 #endif
 
+#if IS_ENABLED(CONFIG_KVM_INTEL)
+noinstr void idt_entry_from_kvm(unsigned int vector)
+{
+	if (vector == NMI_VECTOR)
+		return idt_do_nmi_irqoff();
+
+	/*
+	 * Only the NMI path requires noinstr.
+	 */
+	instrumentation_begin();
+	idt_do_interrupt_irqoff(gate_offset(idt_table + vector));
+	instrumentation_end();
+}
+#endif
+
 static void __init idt_map_in_cea(void)
 {
 	/*
diff --git a/arch/x86/kernel/jailhouse.c b/arch/x86/kernel/jailhouse.c
index 9e9a591a5fecb..f58ce9220e0f3 100644
--- a/arch/x86/kernel/jailhouse.c
+++ b/arch/x86/kernel/jailhouse.c
@@ -17,6 +17,7 @@
 #include <asm/io_apic.h>
 #include <asm/acpi.h>
 #include <asm/cpu.h>
+#include <asm/cpuid/api.h>
 #include <asm/hypervisor.h>
 #include <asm/i8259.h>
 #include <asm/irqdomain.h>
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 29226d112029e..06534e16cfb56 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -41,6 +41,7 @@
 #include <asm/hypervisor.h>
 #include <asm/mtrr.h>
 #include <asm/tlb.h>
+#include <asm/cpuid/api.h>
 #include <asm/cpuidle_haltpoll.h>
 #include <asm/msr.h>
 #include <asm/ptrace.h>
diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
index 0590d399d4f1f..c3f4a389992da 100644
--- a/arch/x86/kernel/machine_kexec_64.c
+++ b/arch/x86/kernel/machine_kexec_64.c
@@ -347,22 +347,6 @@ int machine_kexec_prepare(struct kimage *image)
 	unsigned long reloc_end = (unsigned long)__relocate_kernel_end;
 	int result;
 
-	/*
-	 * Some early TDX-capable platforms have an erratum.  A kernel
-	 * partial write (a write transaction of less than cacheline
-	 * lands at memory controller) to TDX private memory poisons that
-	 * memory, and a subsequent read triggers a machine check.
-	 *
-	 * On those platforms the old kernel must reset TDX private
-	 * memory before jumping to the new kernel otherwise the new
-	 * kernel may see unexpected machine check.  For simplicity
-	 * just fail kexec/kdump on those platforms.
-	 */
-	if (boot_cpu_has_bug(X86_BUG_TDX_PW_MCE)) {
-		pr_info_once("Not allowed on platform with tdx_pw_mce bug\n");
-		return -EOPNOTSUPP;
-	}
-
 	/* Setup the identity mapped 64bit page table */
 	result = init_pgtable(image, __pa(control_page));
 	if (result)
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 3d239ed127441..52a3afb1b79e9 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -614,7 +614,6 @@ DEFINE_IDTENTRY_RAW(exc_nmi_kvm_vmx)
 {
 	exc_nmi(regs);
 }
-EXPORT_SYMBOL_FOR_KVM(asm_exc_nmi_kvm_vmx);
 #endif
 
 #ifdef CONFIG_NMI_CHECK_CPU
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 792fa96b32330..44f29fc05b3df 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -24,6 +24,7 @@
 #include <asm/time.h>
 #include <asm/pgalloc.h>
 #include <asm/irq.h>
+#include <asm/cpuid/api.h>
 #include <asm/delay.h>
 #include <asm/fixmap.h>
 #include <asm/apic.h>
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 294a8ea602986..8b6af1623040a 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -424,6 +424,21 @@ static const struct x86_cpu_id intel_cod_cpu[] = {
 	{}
 };
 
+/*
+ * Allows splitting the LLC by matching 'core_id % split_llc'.
+ *
+ * This is mostly a debug hack to emulate systems with multiple LLCs per node
+ * on systems that do not naturally have this.
+ */
+static unsigned int split_llc = 0;
+
+static int __init split_llc_setup(char *str)
+{
+	get_option(&str, &split_llc);
+	return 0;
+}
+early_param("split_llc", split_llc_setup);
+
 static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 {
 	const struct x86_cpu_id *id = x86_match_cpu(intel_cod_cpu);
@@ -438,6 +453,11 @@ static bool match_llc(struct cpuinfo_x86 *c, struct cpuinfo_x86 *o)
 	if (per_cpu_llc_id(cpu1) != per_cpu_llc_id(cpu2))
 		return false;
 
+	if (split_llc &&
+	    (per_cpu_core_id(cpu1) % split_llc) !=
+	    (per_cpu_core_id(cpu2) % split_llc))
+		return false;
+
 	/*
 	 * Allow the SNC topology without warning. Return of false
 	 * means 'c' does not share the LLC of 'o'. This will be
@@ -1256,6 +1276,7 @@ void __init native_smp_prepare_boot_cpu(void)
 		switch_gdt_and_percpu_base(me);
 
 	native_pv_lock_init();
+	native_pv_tlb_init();
 }
 
 void __init native_smp_cpus_done(unsigned int max_cpus)
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 0ca3912ecb7f4..30aa8369957e9 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -1475,13 +1475,6 @@ DEFINE_IDTENTRY(exc_coprocessor_error)
 
 DEFINE_IDTENTRY(exc_simd_coprocessor_error)
 {
-	if (IS_ENABLED(CONFIG_X86_INVD_BUG)) {
-		/* AMD 486 bug: INVD in CPL 0 raises #XF instead of #GP */
-		if (!static_cpu_has(X86_FEATURE_XMM)) {
-			__exc_general_protection(regs, 0);
-			return;
-		}
-	}
 	math_error(regs, X86_TRAP_XF);
 }
 
@@ -1550,20 +1543,6 @@ DEFINE_IDTENTRY(exc_device_not_available)
 	if (handle_xfd_event(regs))
 		return;
 
-#ifdef CONFIG_MATH_EMULATION
-	if (!boot_cpu_has(X86_FEATURE_FPU) && (cr0 & X86_CR0_EM)) {
-		struct math_emu_info info = { };
-
-		cond_local_irq_enable(regs);
-
-		info.regs = regs;
-		math_emulate(&info);
-
-		cond_local_irq_disable(regs);
-		return;
-	}
-#endif
-
 	/* This should not happen. */
 	if (WARN(cr0 & X86_CR0_TS, "CR0.TS was set")) {
 		/* Try to fix it up and carry on. */
diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c
index 4438ecac9a89b..015c6947b462e 100644
--- a/arch/x86/kvm/hyperv.c
+++ b/arch/x86/kvm/hyperv.c
@@ -2041,7 +2041,9 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
 	 * read with kvm_read_guest().
 	 */
 	if (!hc->fast && mmu_is_nested(vcpu)) {
-		hc->ingpa = translate_nested_gpa(vcpu, hc->ingpa, 0, NULL);
+		hc->ingpa = kvm_x86_ops.nested_ops->translate_nested_gpa(
+					vcpu, hc->ingpa,
+					PFERR_GUEST_FINAL_MASK, NULL, 0);
 		if (unlikely(hc->ingpa == INVALID_GPA))
 			return HV_STATUS_INVALID_HYPERCALL_INPUT;
 	}
diff --git a/arch/x86/kvm/kvm-asm-offsets.c b/arch/x86/kvm/kvm-asm-offsets.c
index 24a710d373238..36ac61724dd7f 100644
--- a/arch/x86/kvm/kvm-asm-offsets.c
+++ b/arch/x86/kvm/kvm-asm-offsets.c
@@ -24,6 +24,7 @@ static void __used common(void)
 
 	if (IS_ENABLED(CONFIG_KVM_INTEL)) {
 		BLANK();
+		OFFSET(VMX_vcpu_arch_regs, vcpu_vmx, vcpu.arch.regs);
 		OFFSET(VMX_spec_ctrl, vcpu_vmx, spec_ctrl);
 	}
 }
diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h
index 8ddb01191d6f6..2ae492ad6412b 100644
--- a/arch/x86/kvm/kvm_cache_regs.h
+++ b/arch/x86/kvm/kvm_cache_regs.h
@@ -67,29 +67,29 @@ static inline bool kvm_register_is_available(struct kvm_vcpu *vcpu,
 					     enum kvm_reg reg)
 {
 	kvm_assert_register_caching_allowed(vcpu);
-	return test_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
+	return test_bit(reg, vcpu->arch.regs_avail);
 }
 
 static inline bool kvm_register_is_dirty(struct kvm_vcpu *vcpu,
 					 enum kvm_reg reg)
 {
 	kvm_assert_register_caching_allowed(vcpu);
-	return test_bit(reg, (unsigned long *)&vcpu->arch.regs_dirty);
+	return test_bit(reg, vcpu->arch.regs_dirty);
 }
 
 static inline void kvm_register_mark_available(struct kvm_vcpu *vcpu,
 					       enum kvm_reg reg)
 {
 	kvm_assert_register_caching_allowed(vcpu);
-	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
+	__set_bit(reg, vcpu->arch.regs_avail);
 }
 
 static inline void kvm_register_mark_dirty(struct kvm_vcpu *vcpu,
 					   enum kvm_reg reg)
 {
 	kvm_assert_register_caching_allowed(vcpu);
-	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
-	__set_bit(reg, (unsigned long *)&vcpu->arch.regs_dirty);
+	__set_bit(reg, vcpu->arch.regs_avail);
+	__set_bit(reg, vcpu->arch.regs_dirty);
 }
 
 /*
@@ -102,7 +102,29 @@ static __always_inline bool kvm_register_test_and_mark_available(struct kvm_vcpu
 								 enum kvm_reg reg)
 {
 	kvm_assert_register_caching_allowed(vcpu);
-	return arch___test_and_set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail);
+	return arch___test_and_set_bit(reg, vcpu->arch.regs_avail);
+}
+
+static __always_inline void kvm_clear_available_registers(struct kvm_vcpu *vcpu,
+							  unsigned long clear_mask)
+{
+	BUILD_BUG_ON(sizeof(clear_mask) != sizeof(vcpu->arch.regs_avail[0]));
+	BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.regs_avail) != 1);
+
+	/*
+	 * Note the bitwise-AND!  In practice, a straight write would also work
+	 * as KVM initializes the mask to all ones and never clears registers
+	 * that are eagerly synchronized.  Using a bitwise-AND adds a bit of
+	 * sanity checking as incorrectly marking an eagerly sync'd register
+	 * unavailable will generate a WARN due to an unexpected cache request.
+	 */
+	vcpu->arch.regs_avail[0] &= ~clear_mask;
+}
+
+static __always_inline void kvm_reset_dirty_registers(struct kvm_vcpu *vcpu)
+{
+	BUILD_BUG_ON(ARRAY_SIZE(vcpu->arch.regs_dirty) != 1);
+	vcpu->arch.regs_dirty[0] = 0;
 }
 
 /*
@@ -112,7 +134,7 @@ static __always_inline bool kvm_register_test_and_mark_available(struct kvm_vcpu
  */
 static inline unsigned long kvm_register_read_raw(struct kvm_vcpu *vcpu, int reg)
 {
-	if (WARN_ON_ONCE((unsigned int)reg >= NR_VCPU_REGS))
+	if (WARN_ON_ONCE((unsigned int)reg >= NR_VCPU_GENERAL_PURPOSE_REGS))
 		return 0;
 
 	if (!kvm_register_is_available(vcpu, reg))
@@ -124,7 +146,7 @@ static inline unsigned long kvm_register_read_raw(struct kvm_vcpu *vcpu, int reg
 static inline void kvm_register_write_raw(struct kvm_vcpu *vcpu, int reg,
 					  unsigned long val)
 {
-	if (WARN_ON_ONCE((unsigned int)reg >= NR_VCPU_REGS))
+	if (WARN_ON_ONCE((unsigned int)reg >= NR_VCPU_GENERAL_PURPOSE_REGS))
 		return;
 
 	vcpu->arch.regs[reg] = val;
@@ -133,12 +155,16 @@ static inline void kvm_register_write_raw(struct kvm_vcpu *vcpu, int reg,
 
 static inline unsigned long kvm_rip_read(struct kvm_vcpu *vcpu)
 {
-	return kvm_register_read_raw(vcpu, VCPU_REGS_RIP);
+	if (!kvm_register_is_available(vcpu, VCPU_REG_RIP))
+		kvm_x86_call(cache_reg)(vcpu, VCPU_REG_RIP);
+
+	return vcpu->arch.rip;
 }
 
 static inline void kvm_rip_write(struct kvm_vcpu *vcpu, unsigned long val)
 {
-	kvm_register_write_raw(vcpu, VCPU_REGS_RIP, val);
+	vcpu->arch.rip = val;
+	kvm_register_mark_dirty(vcpu, VCPU_REG_RIP);
 }
 
 static inline unsigned long kvm_rsp_read(struct kvm_vcpu *vcpu)
@@ -155,8 +181,8 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index)
 {
 	might_sleep();  /* on svm */
 
-	if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR))
-		kvm_x86_call(cache_reg)(vcpu, VCPU_EXREG_PDPTR);
+	if (!kvm_register_is_available(vcpu, VCPU_REG_PDPTR))
+		kvm_x86_call(cache_reg)(vcpu, VCPU_REG_PDPTR);
 
 	return vcpu->arch.walk_mmu->pdptrs[index];
 }
@@ -170,8 +196,8 @@ static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask)
 {
 	ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS;
 	if ((tmask & vcpu->arch.cr0_guest_owned_bits) &&
-	    !kvm_register_is_available(vcpu, VCPU_EXREG_CR0))
-		kvm_x86_call(cache_reg)(vcpu, VCPU_EXREG_CR0);
+	    !kvm_register_is_available(vcpu, VCPU_REG_CR0))
+		kvm_x86_call(cache_reg)(vcpu, VCPU_REG_CR0);
 	return vcpu->arch.cr0 & mask;
 }
 
@@ -192,8 +218,8 @@ static inline ulong kvm_read_cr4_bits(struct kvm_vcpu *vcpu, ulong mask)
 {
 	ulong tmask = mask & KVM_POSSIBLE_CR4_GUEST_BITS;
 	if ((tmask & vcpu->arch.cr4_guest_owned_bits) &&
-	    !kvm_register_is_available(vcpu, VCPU_EXREG_CR4))
-		kvm_x86_call(cache_reg)(vcpu, VCPU_EXREG_CR4);
+	    !kvm_register_is_available(vcpu, VCPU_REG_CR4))
+		kvm_x86_call(cache_reg)(vcpu, VCPU_REG_CR4);
 	return vcpu->arch.cr4 & mask;
 }
 
@@ -207,8 +233,8 @@ static __always_inline bool kvm_is_cr4_bit_set(struct kvm_vcpu *vcpu,
 
 static inline ulong kvm_read_cr3(struct kvm_vcpu *vcpu)
 {
-	if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
-		kvm_x86_call(cache_reg)(vcpu, VCPU_EXREG_CR3);
+	if (!kvm_register_is_available(vcpu, VCPU_REG_CR3))
+		kvm_x86_call(cache_reg)(vcpu, VCPU_REG_CR3);
 	return vcpu->arch.cr3;
 }
 
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 830f46145692a..ddf4e467c071e 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -37,6 +37,13 @@ extern bool __read_mostly enable_mmio_caching;
 #define PT32_ROOT_LEVEL 2
 #define PT32E_ROOT_LEVEL 3
 
+#define ACC_READ_MASK    PT_PRESENT_MASK
+#define ACC_WRITE_MASK   PT_WRITABLE_MASK
+#define ACC_USER_MASK    PT_USER_MASK   /* non EPT */
+#define ACC_USER_EXEC_MASK ACC_USER_MASK /* EPT only */
+#define ACC_EXEC_MASK    8
+#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK | ACC_READ_MASK)
+
 #define KVM_MMU_CR4_ROLE_BITS (X86_CR4_PSE | X86_CR4_PAE | X86_CR4_LA57 | \
 			       X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE)
 
@@ -76,19 +83,24 @@ static inline gfn_t kvm_mmu_max_gfn(void)
 	return (1ULL << (max_gpa_bits - PAGE_SHIFT)) - 1;
 }
 
+static inline bool mmu_has_mbec(struct kvm_mmu *mmu)
+{
+	return mmu->root_role.cr4_smep;
+}
+
 u8 kvm_mmu_get_max_tdp_level(void);
 
 void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask);
 void kvm_mmu_set_mmio_spte_value(struct kvm *kvm, u64 mmio_value);
 void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask);
-void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only);
+void kvm_mmu_set_ept_masks(bool has_ad_bits);
 
 void kvm_init_mmu(struct kvm_vcpu *vcpu);
-void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
-			     unsigned long cr4, u64 efer, gpa_t nested_cr3);
+void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
+			     u64 efer, gpa_t nested_cr3, u64 misc_ctl);
 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
 			     int huge_page_level, bool accessed_dirty,
-			     gpa_t new_eptp);
+			     bool mbec, gpa_t new_eptp);
 bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu);
 int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
 				u64 fault_address, char *insn, int insn_len);
@@ -288,17 +300,17 @@ static inline void kvm_update_page_stats(struct kvm *kvm, int level, int count)
 	atomic64_add(count, &kvm->stat.pages[level - 1]);
 }
 
-gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,
-			   struct x86_exception *exception);
-
 static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu,
 				      struct kvm_mmu *mmu,
 				      gpa_t gpa, u64 access,
-				      struct x86_exception *exception)
+				      struct x86_exception *exception,
+				      u64 pte_access)
 {
 	if (mmu != &vcpu->arch.nested_mmu)
 		return gpa;
-	return translate_nested_gpa(vcpu, gpa, access, exception);
+	return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access,
+							    exception,
+							    pte_access);
 }
 
 static inline bool kvm_has_mirrored_tdp(const struct kvm *kvm)
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index f0144ae8d891d..a01a1faab2eef 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -52,9 +52,11 @@
 #include <asm/page.h>
 #include <asm/memtype.h>
 #include <asm/cmpxchg.h>
+#include <asm/cpuid/api.h>
 #include <asm/io.h>
 #include <asm/set_memory.h>
 #include <asm/spec-ctrl.h>
+#include <asm/svm.h>
 #include <asm/vmx.h>
 
 #include "trace.h"
@@ -229,13 +231,18 @@ static inline bool __maybe_unused is_##reg##_##name(struct kvm_mmu *mmu)	\
 }
 BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp);
 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pse);
-BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smep);
+BUILD_MMU_ROLE_ACCESSOR(base, cr4, smep);
 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, smap);
 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, pke);
 BUILD_MMU_ROLE_ACCESSOR(ext,  cr4, la57);
 BUILD_MMU_ROLE_ACCESSOR(base, efer, nx);
 BUILD_MMU_ROLE_ACCESSOR(ext,  efer, lma);
 
+static inline bool has_pferr_fetch(struct kvm_mmu *mmu)
+{
+	return mmu->cpu_role.ext.has_pferr_fetch;
+}
+
 static inline bool is_cr0_pg(struct kvm_mmu *mmu)
 {
         return mmu->cpu_role.base.level > 0;
@@ -2022,7 +2029,7 @@ static bool kvm_sync_page_check(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
 	 */
 	const union kvm_mmu_page_role sync_role_ign = {
 		.level = 0xf,
-		.access = 0x7,
+		.access = ACC_ALL,
 		.quadrant = 0x3,
 		.passthrough = 0x1,
 	};
@@ -3458,12 +3465,13 @@ static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 {
 	struct kvm_shadow_walk_iterator it;
 	struct kvm_mmu_page *sp;
-	int ret;
+	int ret, access;
 	gfn_t base_gfn = fault->gfn;
 
 	kvm_mmu_hugepage_adjust(vcpu, fault);
 
-	trace_kvm_mmu_spte_requested(fault);
+	access = vcpu->arch.mmu->root_role.access;
+	trace_kvm_mmu_spte_requested(fault, access);
 	for_each_shadow_entry(vcpu, fault->addr, it) {
 		/*
 		 * We cannot overwrite existing page tables with an NX
@@ -3476,7 +3484,7 @@ static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 		if (it.level == fault->goal_level)
 			break;
 
-		sp = kvm_mmu_get_child_sp(vcpu, it.sptep, base_gfn, true, ACC_ALL);
+		sp = kvm_mmu_get_child_sp(vcpu, it.sptep, base_gfn, true, access);
 		if (sp == ERR_PTR(-EEXIST))
 			continue;
 
@@ -3489,7 +3497,7 @@ static int direct_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 	if (WARN_ON_ONCE(it.level != fault->goal_level))
 		return -EFAULT;
 
-	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, ACC_ALL,
+	ret = mmu_set_spte(vcpu, fault->slot, it.sptep, access,
 			   base_gfn, fault->pfn, fault);
 	if (ret == RET_PF_SPURIOUS)
 		return ret;
@@ -4360,7 +4368,14 @@ static gpa_t nonpaging_gva_to_gpa(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
 {
 	if (exception)
 		exception->error_code = 0;
-	return kvm_translate_gpa(vcpu, mmu, vaddr, access, exception);
+	/*
+	 * EPT MBEC uses the effective access bits from the PTE to distinguish
+	 * user and supervisor accesses, and treats every linear address as a
+	 * user-mode address if CR0.PG=0.  Therefore *include* ACC_USER_MASK in
+	 * the last argument to kvm_translate_gpa (which NPT does not use).
+	 */
+	return kvm_translate_gpa(vcpu, mmu, vaddr, access | PFERR_GUEST_FINAL_MASK,
+				 exception, ACC_ALL);
 }
 
 static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct)
@@ -5496,7 +5511,7 @@ static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu,
 static inline bool boot_cpu_is_amd(void)
 {
 	WARN_ON_ONCE(!tdp_enabled);
-	return shadow_x_mask == 0;
+	return shadow_xs_mask == 0;
 }
 
 /*
@@ -5541,55 +5556,106 @@ reset_ept_shadow_zero_bits_mask(struct kvm_mmu *context, bool execonly)
 				    max_huge_page_level);
 }
 
-#define BYTE_MASK(access) \
-	((1 & (access) ? 2 : 0) | \
-	 (2 & (access) ? 4 : 0) | \
-	 (3 & (access) ? 8 : 0) | \
-	 (4 & (access) ? 16 : 0) | \
-	 (5 & (access) ? 32 : 0) | \
-	 (6 & (access) ? 64 : 0) | \
-	 (7 & (access) ? 128 : 0))
-
-
-static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept)
+/*
+ * Build a mask with all combinations of PTE access rights that
+ * include the given access bit.  The mask can be queried with
+ * "mask & (1 << access)", where access is a combination of
+ * ACC_* bits.
+ *
+ * By mixing and matching multiple masks returned by ACC_BITS_MASK,
+ * update_permission_bitmask() builds what is effectively a
+ * two-dimensional array of bools.  The second dimension is
+ * provided by individual bits of permissions[pfec >> 1], and
+ * logical &, | and ~ operations operate on all the 16 possible
+ * combinations of ACC_* bits.
+ */
+#define ACC_BITS_MASK(access) \
+	((1 & (access) ? 1 << 1 : 0) | \
+	 (2 & (access) ? 1 << 2 : 0) | \
+	 (3 & (access) ? 1 << 3 : 0) | \
+	 (4 & (access) ? 1 << 4 : 0) | \
+	 (5 & (access) ? 1 << 5 : 0) | \
+	 (6 & (access) ? 1 << 6 : 0) | \
+	 (7 & (access) ? 1 << 7 : 0) | \
+	 (8 & (access) ? 1 << 8 : 0) | \
+	 (9 & (access) ? 1 << 9 : 0) | \
+	 (10 & (access) ? 1 << 10 : 0) | \
+	 (11 & (access) ? 1 << 11 : 0) | \
+	 (12 & (access) ? 1 << 12 : 0) | \
+	 (13 & (access) ? 1 << 13 : 0) | \
+	 (14 & (access) ? 1 << 14 : 0) | \
+	 (15 & (access) ? 1 << 15 : 0))
+
+static void update_permission_bitmask(struct kvm_mmu *mmu, bool tdp, bool ept)
 {
-	unsigned byte;
+	unsigned index;
 
-	const u8 x = BYTE_MASK(ACC_EXEC_MASK);
-	const u8 w = BYTE_MASK(ACC_WRITE_MASK);
-	const u8 u = BYTE_MASK(ACC_USER_MASK);
+	const u16 w = ACC_BITS_MASK(ACC_WRITE_MASK);
+	const u16 r = ACC_BITS_MASK(ACC_READ_MASK);
 
 	bool cr4_smep = is_cr4_smep(mmu);
 	bool cr4_smap = is_cr4_smap(mmu);
 	bool cr0_wp = is_cr0_wp(mmu);
 	bool efer_nx = is_efer_nx(mmu);
 
-	for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
-		unsigned pfec = byte << 1;
+	/*
+	 * In hardware, page fault error codes are generated (as the name
+	 * suggests) on any kind of page fault.  permission_fault() and
+	 * paging_tmpl.h already use the same bits after a successful page
+	 * table walk, to indicate the kind of access being performed.
+	 *
+	 * However, PFERR_PRESENT_MASK and PFERR_RSVD_MASK are never set here,
+	 * exactly because the page walk is successful.  PFERR_PRESENT_MASK is
+	 * removed by the shift, while PFERR_RSVD_MASK is repurposed in
+	 * permission_fault() to indicate accesses that are *not* subject to
+	 * SMAP restrictions.
+	 */
+	for (index = 0; index < ARRAY_SIZE(mmu->permissions); ++index) {
+		unsigned pfec = index << 1;
 
 		/*
-		 * Each "*f" variable has a 1 bit for each UWX value
+		 * Each "*f" variable has a 1 bit for each ACC_* combo
 		 * that causes a fault with the given PFEC.
 		 */
 
+		/* Faults from reads to non-readable pages */
+		u16 rf = (pfec & (PFERR_WRITE_MASK|PFERR_FETCH_MASK)) ? 0 : (u16)~r;
 		/* Faults from writes to non-writable pages */
-		u8 wf = (pfec & PFERR_WRITE_MASK) ? (u8)~w : 0;
+		u16 wf = (pfec & PFERR_WRITE_MASK) ? (u16)~w : 0;
 		/* Faults from user mode accesses to supervisor pages */
-		u8 uf = (pfec & PFERR_USER_MASK) ? (u8)~u : 0;
-		/* Faults from fetches of non-executable pages*/
-		u8 ff = (pfec & PFERR_FETCH_MASK) ? (u8)~x : 0;
-		/* Faults from kernel mode fetches of user pages */
-		u8 smepf = 0;
+		u16 uf = 0;
+		/* Faults from fetches of non-executable pages */
+		u16 ff = 0;
 		/* Faults from kernel mode accesses of user pages */
-		u8 smapf = 0;
+		u16 smapf = 0;
+
+		if (ept) {
+			const u16 xs = ACC_BITS_MASK(ACC_EXEC_MASK);
+			const u16 xu = ACC_BITS_MASK(ACC_USER_EXEC_MASK);
+
+			if (pfec & PFERR_FETCH_MASK) {
+				/* Ignore XU unless MBEC is enabled.  */
+				if (cr4_smep)
+					ff = pfec & PFERR_USER_MASK ? (u16)~xu : (u16)~xs;
+				else
+					ff = (u16)~xs;
+			}
+		} else {
+			const u16 x = ACC_BITS_MASK(ACC_EXEC_MASK);
+			const u16 u = ACC_BITS_MASK(ACC_USER_MASK);
 
-		if (!ept) {
 			/* Faults from kernel mode accesses to user pages */
-			u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u;
+			u16 kf = (pfec & PFERR_USER_MASK) ? 0 : u;
 
-			/* Not really needed: !nx will cause pte.nx to fault */
-			if (!efer_nx)
-				ff = 0;
+			/*
+			 * For NPT GMET, U=0 does not affect reads and writes.  Fetches
+			 * are handled below via cr4_smep.
+			 */
+			if (!(tdp && cr4_smep))
+				uf = (pfec & PFERR_USER_MASK) ? (u16)~u : 0;
+
+			if (efer_nx)
+				ff |= (pfec & PFERR_FETCH_MASK) ? (u16)~x : 0;
 
 			/* Allow supervisor writes if !cr0.wp */
 			if (!cr0_wp)
@@ -5597,7 +5663,7 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept)
 
 			/* Disallow supervisor fetches of user code if cr4.smep */
 			if (cr4_smep)
-				smepf = (pfec & PFERR_FETCH_MASK) ? kf : 0;
+				ff |= (pfec & PFERR_FETCH_MASK) ? kf : 0;
 
 			/*
 			 * SMAP:kernel-mode data accesses from user-mode
@@ -5610,16 +5676,15 @@ static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept)
 			 *   - The access is supervisor mode
 			 *   - If implicit supervisor access or X86_EFLAGS_AC is clear
 			 *
-			 * Here, we cover the first four conditions.
-			 * The fifth is computed dynamically in permission_fault();
-			 * PFERR_RSVD_MASK bit will be set in PFEC if the access is
-			 * *not* subject to SMAP restrictions.
+			 * Here, we cover the first four conditions.  The fifth
+			 * is computed dynamically in permission_fault() and
+			 * communicated by setting PFERR_RSVD_MASK.
 			 */
 			if (cr4_smap)
 				smapf = (pfec & (PFERR_RSVD_MASK|PFERR_FETCH_MASK)) ? 0 : kf;
 		}
 
-		mmu->permissions[byte] = ff | uf | wf | smepf | smapf;
+		mmu->permissions[index] = ff | uf | wf | rf | smapf;
 	}
 }
 
@@ -5698,7 +5763,7 @@ static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu,
 		return;
 
 	reset_guest_rsvds_bits_mask(vcpu, mmu);
-	update_permission_bitmask(mmu, false);
+	update_permission_bitmask(mmu, mmu == &vcpu->arch.guest_mmu, false);
 	update_pkru_bitmask(mmu);
 }
 
@@ -5733,7 +5798,7 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu,
 
 	role.base.efer_nx = ____is_efer_nx(regs);
 	role.base.cr0_wp = ____is_cr0_wp(regs);
-	role.base.smep_andnot_wp = ____is_cr4_smep(regs) && !____is_cr0_wp(regs);
+	role.base.cr4_smep = ____is_cr4_smep(regs);
 	role.base.smap_andnot_wp = ____is_cr4_smap(regs) && !____is_cr0_wp(regs);
 	role.base.has_4_byte_gpte = !____is_cr4_pae(regs);
 
@@ -5745,7 +5810,6 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu,
 	else
 		role.base.level = PT32_ROOT_LEVEL;
 
-	role.ext.cr4_smep = ____is_cr4_smep(regs);
 	role.ext.cr4_smap = ____is_cr4_smap(regs);
 	role.ext.cr4_pse = ____is_cr4_pse(regs);
 
@@ -5753,6 +5817,8 @@ static union kvm_cpu_role kvm_calc_cpu_role(struct kvm_vcpu *vcpu,
 	role.ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs);
 	role.ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs);
 	role.ext.efer_lma = ____is_efer_lma(regs);
+
+	role.ext.has_pferr_fetch = role.base.efer_nx | role.base.cr4_smep;
 	return role;
 }
 
@@ -5802,8 +5868,8 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu,
 {
 	union kvm_mmu_page_role role = {0};
 
-	role.access = ACC_ALL;
 	role.cr0_wp = true;
+	role.cr4_smep = kvm_x86_call(tdp_has_smep)(vcpu->kvm);
 	role.efer_nx = true;
 	role.smm = cpu_role.base.smm;
 	role.guest_mode = cpu_role.base.guest_mode;
@@ -5812,6 +5878,11 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu,
 	role.direct = true;
 	role.has_4_byte_gpte = false;
 
+	/* All TDP pages are supervisor-executable */
+	role.access = ACC_ALL;
+	if (role.cr4_smep && shadow_user_mask)
+		role.access &= ~ACC_USER_MASK;
+
 	return role;
 }
 
@@ -5891,13 +5962,13 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
 	shadow_mmu_init_context(vcpu, context, cpu_role, root_role);
 }
 
-void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
-			     unsigned long cr4, u64 efer, gpa_t nested_cr3)
+void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
+			     u64 efer, gpa_t nested_cr3, u64 misc_ctl)
 {
 	struct kvm_mmu *context = &vcpu->arch.guest_mmu;
 	struct kvm_mmu_role_regs regs = {
-		.cr0 = cr0,
-		.cr4 = cr4 & ~X86_CR4_PKE,
+		.cr0 = X86_CR0_PG | X86_CR0_WP,
+		.cr4 = cr4 & ~(X86_CR4_PKE | X86_CR4_SMAP),
 		.efer = efer,
 	};
 	union kvm_cpu_role cpu_role = kvm_calc_cpu_role(vcpu, &regs);
@@ -5905,6 +5976,7 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0,
 
 	/* NPT requires CR0.PG=1. */
 	WARN_ON_ONCE(cpu_role.base.direct || !cpu_role.base.guest_mode);
+	cpu_role.base.cr4_smep = (misc_ctl & SVM_MISC_ENABLE_GMET) != 0;
 
 	root_role = cpu_role.base;
 	root_role.level = kvm_mmu_get_tdp_level(vcpu);
@@ -5919,7 +5991,7 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_shadow_npt_mmu);
 
 static union kvm_cpu_role
 kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty,
-				   bool execonly, u8 level)
+				   bool execonly, u8 level, bool mbec)
 {
 	union kvm_cpu_role role = {0};
 
@@ -5929,6 +6001,7 @@ kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty,
 	 */
 	WARN_ON_ONCE(is_smm(vcpu));
 	role.base.level = level;
+	role.base.cr4_smep = mbec;
 	role.base.has_4_byte_gpte = false;
 	role.base.direct = false;
 	role.base.ad_disabled = !accessed_dirty;
@@ -5944,13 +6017,13 @@ kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty,
 
 void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
 			     int huge_page_level, bool accessed_dirty,
-			     gpa_t new_eptp)
+			     bool mbec, gpa_t new_eptp)
 {
 	struct kvm_mmu *context = &vcpu->arch.guest_mmu;
 	u8 level = vmx_eptp_page_walk_level(new_eptp);
 	union kvm_cpu_role new_mode =
 		kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty,
-						   execonly, level);
+						   execonly, level, mbec);
 
 	if (new_mode.as_u64 != context->cpu_role.as_u64) {
 		/* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */
@@ -5961,7 +6034,7 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
 		context->gva_to_gpa = ept_gva_to_gpa;
 		context->sync_spte = ept_sync_spte;
 
-		update_permission_bitmask(context, true);
+		update_permission_bitmask(context, true, true);
 		context->pkru_mask = 0;
 		reset_rsvds_bits_mask_ept(vcpu, context, execonly, huge_page_level);
 		reset_ept_shadow_zero_bits_mask(context, execonly);
@@ -6949,7 +7022,7 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end)
 
 	write_lock(&kvm->mmu_lock);
 
-	kvm_mmu_invalidate_begin(kvm);
+	kvm_mmu_invalidate_start(kvm);
 
 	kvm_mmu_invalidate_range_add(kvm, gfn_start, gfn_end);
 
diff --git a/arch/x86/kvm/mmu/mmutrace.h b/arch/x86/kvm/mmu/mmutrace.h
index 764e3015d021b..fa01719baf8d4 100644
--- a/arch/x86/kvm/mmu/mmutrace.h
+++ b/arch/x86/kvm/mmu/mmutrace.h
@@ -25,7 +25,8 @@
 #define KVM_MMU_PAGE_PRINTK() ({				        \
 	const char *saved_ptr = trace_seq_buffer_ptr(p);		\
 	static const char *access_str[] = {			        \
-		"---", "--x", "w--", "w-x", "-u-", "-ux", "wu-", "wux"  \
+		"----", "r---", "-w--", "rw--", "--u-", "r-u-", "-wu-", "rwu-", \
+		"---x", "r--x", "-w-x", "rw-x", "--ux", "r-ux", "-wux", "rwux"	\
 	};							        \
 	union kvm_mmu_page_role role;				        \
 								        \
@@ -356,8 +357,8 @@ TRACE_EVENT(
 		__entry->sptep = virt_to_phys(sptep);
 		__entry->level = level;
 		__entry->r = shadow_present_mask || (__entry->spte & PT_PRESENT_MASK);
-		__entry->x = is_executable_pte(__entry->spte);
-		__entry->u = shadow_user_mask ? !!(__entry->spte & shadow_user_mask) : -1;
+		__entry->x = (__entry->spte & (shadow_xs_mask | shadow_nx_mask)) == shadow_xs_mask;
+		__entry->u = !!(__entry->spte & (shadow_xu_mask | shadow_user_mask));
 	),
 
 	TP_printk("gfn %llx spte %llx (%s%s%s%s) level %d at %llx",
@@ -365,30 +366,32 @@ TRACE_EVENT(
 		  __entry->r ? "r" : "-",
 		  __entry->spte & PT_WRITABLE_MASK ? "w" : "-",
 		  __entry->x ? "x" : "-",
-		  __entry->u == -1 ? "" : (__entry->u ? "u" : "-"),
+		  __entry->u ? "u" : "-",
 		  __entry->level, __entry->sptep
 	)
 );
 
 TRACE_EVENT(
 	kvm_mmu_spte_requested,
-	TP_PROTO(struct kvm_page_fault *fault),
-	TP_ARGS(fault),
+	TP_PROTO(struct kvm_page_fault *fault, u8 access),
+	TP_ARGS(fault, access),
 
 	TP_STRUCT__entry(
 		__field(u64, gfn)
 		__field(u64, pfn)
 		__field(u8, level)
+		__field(u8, access)
 	),
 
 	TP_fast_assign(
 		__entry->gfn = fault->gfn;
 		__entry->pfn = fault->pfn | (fault->gfn & (KVM_PAGES_PER_HPAGE(fault->goal_level) - 1));
 		__entry->level = fault->goal_level;
+		__entry->access = access;
 	),
 
-	TP_printk("gfn %llx pfn %llx level %d",
-		  __entry->gfn, __entry->pfn, __entry->level
+	TP_printk("gfn %llx pfn %llx level %d access %x",
+		  __entry->gfn, __entry->pfn, __entry->level, __entry->access
 	)
 );
 
diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h
index 901cd2bd40b84..07100bbfc2701 100644
--- a/arch/x86/kvm/mmu/paging_tmpl.h
+++ b/arch/x86/kvm/mmu/paging_tmpl.h
@@ -124,12 +124,17 @@ static inline void FNAME(protect_clean_gpte)(struct kvm_mmu *mmu, unsigned *acce
 	*access &= mask;
 }
 
-static inline int FNAME(is_present_gpte)(unsigned long pte)
+static inline int FNAME(is_present_gpte)(struct kvm_mmu *mmu,
+					 unsigned long pte)
 {
 #if PTTYPE != PTTYPE_EPT
 	return pte & PT_PRESENT_MASK;
 #else
-	return pte & 7;
+	/*
+	 * For EPT, an entry is present if any of bits 2:0 are set.
+	 * With mode-based execute control, bit 10 also indicates presence.
+	 */
+	return pte & (7 | (mmu_has_mbec(mmu) ? VMX_EPT_USER_EXECUTABLE_MASK : 0));
 #endif
 }
 
@@ -152,7 +157,7 @@ static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
 				  struct kvm_mmu_page *sp, u64 *spte,
 				  u64 gpte)
 {
-	if (!FNAME(is_present_gpte)(gpte))
+	if (!FNAME(is_present_gpte)(vcpu->arch.mmu, gpte))
 		goto no_present;
 
 	/* Prefetch only accessed entries (unless A/D bits are disabled). */
@@ -170,25 +175,31 @@ static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
 	return true;
 }
 
-/*
- * For PTTYPE_EPT, a page table can be executable but not readable
- * on supported processors. Therefore, set_spte does not automatically
- * set bit 0 if execute only is supported. Here, we repurpose ACC_USER_MASK
- * to signify readability since it isn't used in the EPT case
- */
 static inline unsigned FNAME(gpte_access)(u64 gpte)
 {
 	unsigned access;
+	/*
+	 * Set bits in ACC_*_MASK even if they might not be used in the
+	 * actual checks.  For example, if EFER.NX is clear permission_fault()
+	 * will ignore ACC_EXEC_MASK, and if MBEC is disabled it will
+	 * ignore ACC_USER_EXEC_MASK.
+	 */
 #if PTTYPE == PTTYPE_EPT
 	access = ((gpte & VMX_EPT_WRITABLE_MASK) ? ACC_WRITE_MASK : 0) |
 		((gpte & VMX_EPT_EXECUTABLE_MASK) ? ACC_EXEC_MASK : 0) |
-		((gpte & VMX_EPT_READABLE_MASK) ? ACC_USER_MASK : 0);
+		((gpte & VMX_EPT_READABLE_MASK) ? ACC_READ_MASK : 0) |
+		((gpte & VMX_EPT_USER_EXECUTABLE_MASK) ? ACC_USER_EXEC_MASK : 0);
 #else
-	BUILD_BUG_ON(ACC_EXEC_MASK != PT_PRESENT_MASK);
-	BUILD_BUG_ON(ACC_EXEC_MASK != 1);
+	/*
+	 * P is set here, so the page is always readable and W/U/!NX represent
+	 * allowed accesses.
+	 */
+	BUILD_BUG_ON(ACC_READ_MASK != PT_PRESENT_MASK);
+	BUILD_BUG_ON(ACC_WRITE_MASK != PT_WRITABLE_MASK);
+	BUILD_BUG_ON(ACC_USER_MASK != PT_USER_MASK);
+	BUILD_BUG_ON(ACC_EXEC_MASK & (PT_WRITABLE_MASK | PT_USER_MASK | PT_PRESENT_MASK));
 	access = gpte & (PT_WRITABLE_MASK | PT_USER_MASK | PT_PRESENT_MASK);
-	/* Combine NX with P (which is set here) to get ACC_EXEC_MASK.  */
-	access ^= (gpte >> PT64_NX_SHIFT);
+	access |= gpte & PT64_NX_MASK ? 0 : ACC_EXEC_MASK;
 #endif
 
 	return access;
@@ -332,7 +343,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 	if (walker->level == PT32E_ROOT_LEVEL) {
 		pte = mmu->get_pdptr(vcpu, (addr >> 30) & 3);
 		trace_kvm_mmu_paging_element(pte, walker->level);
-		if (!FNAME(is_present_gpte)(pte))
+		if (!FNAME(is_present_gpte)(mmu, pte))
 			goto error;
 		--walker->level;
 	}
@@ -377,7 +388,8 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 		walker->pte_gpa[walker->level - 1] = pte_gpa;
 
 		real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(table_gfn),
-					     nested_access, &walker->fault);
+					     nested_access | PFERR_GUEST_PAGE_MASK,
+					     &walker->fault, 0);
 
 		/*
 		 * FIXME: This can happen if emulation (for of an INS/OUTS
@@ -414,7 +426,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 		 */
 		pte_access = pt_access & (pte ^ walk_nx_mask);
 
-		if (unlikely(!FNAME(is_present_gpte)(pte)))
+		if (unlikely(!FNAME(is_present_gpte)(mmu, pte)))
 			goto error;
 
 		if (unlikely(FNAME(is_rsvd_bits_set)(mmu, pte, walker->level))) {
@@ -445,7 +457,9 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 		gfn += pse36_gfn_delta(pte);
 #endif
 
-	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(gfn), access, &walker->fault);
+	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(gfn),
+				     access | PFERR_GUEST_FINAL_MASK,
+				     &walker->fault, walker->pte_access);
 	if (real_gpa == INVALID_GPA)
 		return 0;
 
@@ -475,7 +489,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 
 error:
 	errcode |= write_fault | user_fault;
-	if (fetch_fault && (is_efer_nx(mmu) || is_cr4_smep(mmu)))
+	if (fetch_fault && has_pferr_fetch(mmu))
 		errcode |= PFERR_FETCH_MASK;
 
 	walker->fault.vector = PF_VECTOR;
@@ -492,7 +506,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 	 * [2:0] - Derive from the access bits. The exit_qualification might be
 	 *         out of date if it is serving an EPT misconfiguration.
 	 * [5:3] - Calculated by the page walk of the guest EPT page tables
-	 * [7:8] - Derived from [7:8] of real exit_qualification
+	 * [7:11] - Derived from [7:11] of real exit_qualification
 	 *
 	 * The other bits are set to 0.
 	 */
@@ -501,16 +515,27 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
 
 		if (write_fault)
 			walker->fault.exit_qualification |= EPT_VIOLATION_ACC_WRITE;
-		if (user_fault)
-			walker->fault.exit_qualification |= EPT_VIOLATION_ACC_READ;
-		if (fetch_fault)
+		else if (fetch_fault)
 			walker->fault.exit_qualification |= EPT_VIOLATION_ACC_INSTR;
+		else
+			walker->fault.exit_qualification |= EPT_VIOLATION_ACC_READ;
+
+		/*
+		 * Accesses to guest paging structures are either "reads" or
+		 * "read+write" accesses, so consider them the latter if write_fault
+		 * is true.
+		 */
+		if (access & PFERR_GUEST_PAGE_MASK)
+			walker->fault.exit_qualification |= EPT_VIOLATION_ACC_READ;
 
 		/*
 		 * Note, pte_access holds the raw RWX bits from the EPTE, not
 		 * ACC_*_MASK flags!
 		 */
 		walker->fault.exit_qualification |= EPT_VIOLATION_RWX_TO_PROT(pte_access);
+		if (mmu_has_mbec(mmu))
+			walker->fault.exit_qualification |=
+				EPT_VIOLATION_USER_EXEC_TO_PROT(pte_access);
 	}
 #endif
 	walker->fault.address = addr;
@@ -709,7 +734,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 	 */
 	kvm_mmu_hugepage_adjust(vcpu, fault);
 
-	trace_kvm_mmu_spte_requested(fault);
+	trace_kvm_mmu_spte_requested(fault, gw->pte_access);
 
 	for (; shadow_walk_okay(&it); shadow_walk_next(&it)) {
 		/*
diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c
index 85a0473809b0e..72d2394e089c9 100644
--- a/arch/x86/kvm/mmu/spte.c
+++ b/arch/x86/kvm/mmu/spte.c
@@ -15,6 +15,7 @@
 #include "x86.h"
 #include "spte.h"
 
+#include <asm/cpuid/api.h>
 #include <asm/e820/api.h>
 #include <asm/memtype.h>
 #include <asm/vmx.h>
@@ -29,8 +30,9 @@ bool __read_mostly kvm_ad_enabled;
 u64 __read_mostly shadow_host_writable_mask;
 u64 __read_mostly shadow_mmu_writable_mask;
 u64 __read_mostly shadow_nx_mask;
-u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
 u64 __read_mostly shadow_user_mask;
+u64 __read_mostly shadow_xs_mask; /* mutual exclusive with nx_mask and user_mask */
+u64 __read_mostly shadow_xu_mask; /* mutual exclusive with nx_mask and user_mask */
 u64 __read_mostly shadow_accessed_mask;
 u64 __read_mostly shadow_dirty_mask;
 u64 __read_mostly shadow_mmio_value;
@@ -194,12 +196,6 @@ bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 	int is_host_mmio = -1;
 	bool wrprot = false;
 
-	/*
-	 * For the EPT case, shadow_present_mask has no RWX bits set if
-	 * exec-only page table entries are supported.  In that case,
-	 * ACC_USER_MASK and shadow_user_mask are used to represent
-	 * read access.  See FNAME(gpte_access) in paging_tmpl.h.
-	 */
 	WARN_ON_ONCE((pte_access | shadow_present_mask) == SHADOW_NONPRESENT_VALUE);
 
 	if (sp->role.ad_disabled)
@@ -223,18 +219,26 @@ bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
 	 * would tie make_spte() further to vCPU/MMU state, and add complexity
 	 * just to optimize a mode that is anything but performance critical.
 	 */
-	if (level > PG_LEVEL_4K && (pte_access & ACC_EXEC_MASK) &&
-	    is_nx_huge_page_enabled(vcpu->kvm)) {
+	if (level > PG_LEVEL_4K && is_nx_huge_page_enabled(vcpu->kvm)) {
 		pte_access &= ~ACC_EXEC_MASK;
+		if (shadow_xu_mask)
+			pte_access &= ~ACC_USER_EXEC_MASK;
 	}
 
-	if (pte_access & ACC_EXEC_MASK)
-		spte |= shadow_x_mask;
-	else
-		spte |= shadow_nx_mask;
-
-	if (pte_access & ACC_USER_MASK)
-		spte |= shadow_user_mask;
+	if (pte_access & ACC_READ_MASK)
+		spte |= PT_PRESENT_MASK; /* or VMX_EPT_READABLE_MASK */
+
+	if (shadow_nx_mask) {
+		if (!(pte_access & ACC_EXEC_MASK))
+			spte |= shadow_nx_mask;
+		if (pte_access & ACC_USER_MASK)
+			spte |= shadow_user_mask;
+	} else {
+		if (pte_access & ACC_EXEC_MASK)
+			spte |= shadow_xs_mask;
+		if (pte_access & ACC_USER_EXEC_MASK)
+			spte |= shadow_xu_mask;
+	}
 
 	if (level > PG_LEVEL_4K)
 		spte |= PT_PAGE_SIZE_MASK;
@@ -317,14 +321,18 @@ static u64 modify_spte_protections(u64 spte, u64 set, u64 clear)
 	return spte;
 }
 
-static u64 make_spte_executable(u64 spte)
+static u64 change_spte_executable(u64 spte, u8 access)
 {
-	return modify_spte_protections(spte, shadow_x_mask, shadow_nx_mask);
-}
+	u64 set, clear;
 
-static u64 make_spte_nonexecutable(u64 spte)
-{
-	return modify_spte_protections(spte, shadow_nx_mask, shadow_x_mask);
+	if (shadow_nx_mask)
+		set = (access & ACC_EXEC_MASK) ? 0 : shadow_nx_mask;
+	else
+		set =
+			(access & ACC_EXEC_MASK ? shadow_xs_mask : 0) |
+			(access & ACC_USER_EXEC_MASK ? shadow_xu_mask : 0);
+	clear = set ^ (shadow_nx_mask | shadow_xs_mask | shadow_xu_mask);
+	return modify_spte_protections(spte, set, clear);
 }
 
 /*
@@ -356,8 +364,8 @@ u64 make_small_spte(struct kvm *kvm, u64 huge_spte,
 		 * the page executable as the NX hugepage mitigation no longer
 		 * applies.
 		 */
-		if ((role.access & ACC_EXEC_MASK) && is_nx_huge_page_enabled(kvm))
-			child_spte = make_spte_executable(child_spte);
+		if (is_nx_huge_page_enabled(kvm))
+			child_spte = change_spte_executable(child_spte, role.access);
 	}
 
 	return child_spte;
@@ -379,7 +387,7 @@ u64 make_huge_spte(struct kvm *kvm, u64 small_spte, int level)
 	huge_spte &= KVM_HPAGE_MASK(level) | ~PAGE_MASK;
 
 	if (is_nx_huge_page_enabled(kvm))
-		huge_spte = make_spte_nonexecutable(huge_spte);
+		huge_spte = change_spte_executable(huge_spte, 0);
 
 	return huge_spte;
 }
@@ -389,7 +397,8 @@ u64 make_nonleaf_spte(u64 *child_pt, bool ad_disabled)
 	u64 spte = SPTE_MMU_PRESENT_MASK;
 
 	spte |= __pa(child_pt) | shadow_present_mask | PT_WRITABLE_MASK |
-		shadow_user_mask | shadow_x_mask | shadow_me_value;
+		PT_PRESENT_MASK /* or VMX_EPT_READABLE_MASK */ |
+		shadow_user_mask | shadow_xs_mask | shadow_xu_mask | shadow_me_value;
 
 	if (ad_disabled)
 		spte |= SPTE_TDP_AD_DISABLED;
@@ -489,20 +498,37 @@ void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask)
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_mmu_set_me_spte_mask);
 
-void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only)
+void kvm_mmu_set_ept_masks(bool has_ad_bits)
 {
 	kvm_ad_enabled		= has_ad_bits;
 
-	shadow_user_mask	= VMX_EPT_READABLE_MASK;
+	shadow_user_mask	= 0;
 	shadow_accessed_mask	= VMX_EPT_ACCESS_BIT;
 	shadow_dirty_mask	= VMX_EPT_DIRTY_BIT;
 	shadow_nx_mask		= 0ull;
-	shadow_x_mask		= VMX_EPT_EXECUTABLE_MASK;
-	/* VMX_EPT_SUPPRESS_VE_BIT is needed for W or X violation. */
-	shadow_present_mask	=
-		(has_exec_only ? 0ull : VMX_EPT_READABLE_MASK) | VMX_EPT_SUPPRESS_VE_BIT;
+	shadow_xs_mask		= VMX_EPT_EXECUTABLE_MASK;
+
+	/*
+	 * The MMU always maps ACC_EXEC_MASK and ACC_USER_EXEC_MASK to the
+	 * XS and XU bits of shadow EPT entries, regardless of whether MBEC
+	 * is available on the host or enabled in the VMCS.
+	 *
+	 * For the non-nested case, pages are mapped with ACC_EXEC_MASK
+	 * and ACC_USER_EXEC_MASK set in tandem, so XS == XU and the
+	 * host's MBEC setting does not matter.  On hardware without MBEC
+	 * the XU bit is reserved-as-ignored, and setting it does no harm.
+	 *
+	 * For nested EPT, when MBEC is disabled by L1, correctness relies
+	 * on (a) ignoring bit 10 of the gPTE in is_present_gpte(), rather
+	 * than treating it as a present bit, and (b) permission_fault()
+	 * using an mmu->permissions[] array that effectively ignores
+	 * ACC_USER_EXEC_MASK.  Bit 10 of the gPTE does end up mirrored
+	 * in the sPTEs but is ignored because L2 runs with MBEC disabled.
+	 */
+	shadow_xu_mask		= VMX_EPT_USER_EXECUTABLE_MASK;
+	shadow_present_mask	= VMX_EPT_SUPPRESS_VE_BIT;
 
-	shadow_acc_track_mask	= VMX_EPT_RWX_MASK;
+	shadow_acc_track_mask	= VMX_EPT_RWX_MASK | VMX_EPT_USER_EXECUTABLE_MASK;
 	shadow_host_writable_mask = EPT_SPTE_HOST_WRITABLE;
 	shadow_mmu_writable_mask  = EPT_SPTE_MMU_WRITABLE;
 
@@ -550,7 +576,8 @@ void kvm_mmu_reset_all_pte_masks(void)
 	shadow_accessed_mask	= PT_ACCESSED_MASK;
 	shadow_dirty_mask	= PT_DIRTY_MASK;
 	shadow_nx_mask		= PT64_NX_MASK;
-	shadow_x_mask		= 0;
+	shadow_xs_mask		= 0;
+	shadow_xu_mask		= 0;
 	shadow_present_mask	= PT_PRESENT_MASK;
 
 	shadow_acc_track_mask	= 0;
diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h
index 8c0ffa2cded69..13eea94dd2128 100644
--- a/arch/x86/kvm/mmu/spte.h
+++ b/arch/x86/kvm/mmu/spte.h
@@ -17,10 +17,20 @@
  */
 #define SPTE_MMU_PRESENT_MASK		BIT_ULL(11)
 
+/*
+ * The ignored high bits are allocated as follows:
+ * - bits 52, 54: saved X-R bits for access tracking when EPT does not have A/D
+ * - bits 53 (EPT only): host writable
+ * - bits 55 (EPT only): MMU-writable
+ * - bits 56-59: unused
+ * - bits 60-61: type of A/D tracking
+ * - bits 62 (EPT only): saved XU bit for disabled AD
+ */
+
 /*
  * TDP SPTES (more specifically, EPT SPTEs) may not have A/D bits, and may also
  * be restricted to using write-protection (for L2 when CPU dirty logging, i.e.
- * PML, is enabled).  Use bits 52 and 53 to hold the type of A/D tracking that
+ * PML, is enabled).  Use bits 60 and 61 to hold the type of A/D tracking that
  * is must be employed for a given TDP SPTE.
  *
  * Note, the "enabled" mask must be '0', as bits 62:52 are _reserved_ for PAE
@@ -29,7 +39,7 @@
  * TDP with CPU dirty logging (PML).  If NPT ever gains PML-like support, it
  * must be restricted to 64-bit KVM.
  */
-#define SPTE_TDP_AD_SHIFT		52
+#define SPTE_TDP_AD_SHIFT		60
 #define SPTE_TDP_AD_MASK		(3ULL << SPTE_TDP_AD_SHIFT)
 #define SPTE_TDP_AD_ENABLED		(0ULL << SPTE_TDP_AD_SHIFT)
 #define SPTE_TDP_AD_DISABLED		(1ULL << SPTE_TDP_AD_SHIFT)
@@ -42,18 +52,6 @@ static_assert(SPTE_TDP_AD_ENABLED == 0);
 #define SPTE_BASE_ADDR_MASK (((1ULL << 52) - 1) & ~(u64)(PAGE_SIZE-1))
 #endif
 
-#define SPTE_PERM_MASK (PT_PRESENT_MASK | PT_WRITABLE_MASK | shadow_user_mask \
-			| shadow_x_mask | shadow_nx_mask | shadow_me_mask)
-
-#define ACC_EXEC_MASK    1
-#define ACC_WRITE_MASK   PT_WRITABLE_MASK
-#define ACC_USER_MASK    PT_USER_MASK
-#define ACC_ALL          (ACC_EXEC_MASK | ACC_WRITE_MASK | ACC_USER_MASK)
-
-/* The mask for the R/X bits in EPT PTEs */
-#define SPTE_EPT_READABLE_MASK			0x1ull
-#define SPTE_EPT_EXECUTABLE_MASK		0x4ull
-
 #define SPTE_LEVEL_BITS			9
 #define SPTE_LEVEL_SHIFT(level)		__PT_LEVEL_SHIFT(level, SPTE_LEVEL_BITS)
 #define SPTE_INDEX(address, level)	__PT_INDEX(address, level, SPTE_LEVEL_BITS)
@@ -66,9 +64,10 @@ static_assert(SPTE_TDP_AD_ENABLED == 0);
  * restored only when a write is attempted to the page.  This mask obviously
  * must not overlap the A/D type mask.
  */
-#define SHADOW_ACC_TRACK_SAVED_BITS_MASK (SPTE_EPT_READABLE_MASK | \
-					  SPTE_EPT_EXECUTABLE_MASK)
-#define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT 54
+#define SHADOW_ACC_TRACK_SAVED_BITS_MASK (VMX_EPT_READABLE_MASK | \
+					  VMX_EPT_EXECUTABLE_MASK | \
+					  VMX_EPT_USER_EXECUTABLE_MASK)
+#define SHADOW_ACC_TRACK_SAVED_BITS_SHIFT 52
 #define SHADOW_ACC_TRACK_SAVED_MASK	(SHADOW_ACC_TRACK_SAVED_BITS_MASK << \
 					 SHADOW_ACC_TRACK_SAVED_BITS_SHIFT)
 static_assert(!(SPTE_TDP_AD_MASK & SHADOW_ACC_TRACK_SAVED_MASK));
@@ -87,8 +86,8 @@ static_assert(!(SPTE_TDP_AD_MASK & SHADOW_ACC_TRACK_SAVED_MASK));
  * to not overlap the A/D type mask or the saved access bits of access-tracked
  * SPTEs when A/D bits are disabled.
  */
-#define EPT_SPTE_HOST_WRITABLE		BIT_ULL(57)
-#define EPT_SPTE_MMU_WRITABLE		BIT_ULL(58)
+#define EPT_SPTE_HOST_WRITABLE		BIT_ULL(53)
+#define EPT_SPTE_MMU_WRITABLE		BIT_ULL(55)
 
 static_assert(!(EPT_SPTE_HOST_WRITABLE & SPTE_TDP_AD_MASK));
 static_assert(!(EPT_SPTE_MMU_WRITABLE & SPTE_TDP_AD_MASK));
@@ -99,11 +98,11 @@ static_assert(!(EPT_SPTE_MMU_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
 #undef SHADOW_ACC_TRACK_SAVED_MASK
 
 /*
- * Due to limited space in PTEs, the MMIO generation is a 19 bit subset of
+ * Due to limited space in PTEs, the MMIO generation is an 18 bit subset of
  * the memslots generation and is derived as follows:
  *
- * Bits 0-7 of the MMIO generation are propagated to spte bits 3-10
- * Bits 8-18 of the MMIO generation are propagated to spte bits 52-62
+ * Bits 0-6 of the MMIO generation are propagated to spte bits 3-9
+ * Bits 7-17 of the MMIO generation are propagated to spte bits 52-62
  *
  * The KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS flag is intentionally not included in
  * the MMIO generation number, as doing so would require stealing a bit from
@@ -114,7 +113,7 @@ static_assert(!(EPT_SPTE_MMU_WRITABLE & SHADOW_ACC_TRACK_SAVED_MASK));
  */
 
 #define MMIO_SPTE_GEN_LOW_START		3
-#define MMIO_SPTE_GEN_LOW_END		10
+#define MMIO_SPTE_GEN_LOW_END		9
 
 #define MMIO_SPTE_GEN_HIGH_START	52
 #define MMIO_SPTE_GEN_HIGH_END		62
@@ -136,7 +135,8 @@ static_assert(!(SPTE_MMU_PRESENT_MASK &
  * and so they're off-limits for generation; additional checks ensure the mask
  * doesn't overlap legal PA bits), and bit 63 (carved out for future usage).
  */
-#define SPTE_MMIO_ALLOWED_MASK (BIT_ULL(63) | GENMASK_ULL(51, 12) | GENMASK_ULL(2, 0))
+#define SPTE_MMIO_ALLOWED_MASK (BIT_ULL(63) | GENMASK_ULL(51, 12) | \
+				BIT_ULL(10) | GENMASK_ULL(2, 0))
 static_assert(!(SPTE_MMIO_ALLOWED_MASK &
 		(SPTE_MMU_PRESENT_MASK | MMIO_SPTE_GEN_LOW_MASK | MMIO_SPTE_GEN_HIGH_MASK)));
 
@@ -144,7 +144,7 @@ static_assert(!(SPTE_MMIO_ALLOWED_MASK &
 #define MMIO_SPTE_GEN_HIGH_BITS		(MMIO_SPTE_GEN_HIGH_END - MMIO_SPTE_GEN_HIGH_START + 1)
 
 /* remember to adjust the comment above as well if you change these */
-static_assert(MMIO_SPTE_GEN_LOW_BITS == 8 && MMIO_SPTE_GEN_HIGH_BITS == 11);
+static_assert(MMIO_SPTE_GEN_LOW_BITS == 7 && MMIO_SPTE_GEN_HIGH_BITS == 11);
 
 #define MMIO_SPTE_GEN_LOW_SHIFT		(MMIO_SPTE_GEN_LOW_START - 0)
 #define MMIO_SPTE_GEN_HIGH_SHIFT	(MMIO_SPTE_GEN_HIGH_START - MMIO_SPTE_GEN_LOW_BITS)
@@ -179,8 +179,9 @@ extern bool __read_mostly kvm_ad_enabled;
 extern u64 __read_mostly shadow_host_writable_mask;
 extern u64 __read_mostly shadow_mmu_writable_mask;
 extern u64 __read_mostly shadow_nx_mask;
-extern u64 __read_mostly shadow_x_mask; /* mutual exclusive with nx_mask */
 extern u64 __read_mostly shadow_user_mask;
+extern u64 __read_mostly shadow_xs_mask; /* mutual exclusive with nx_mask and user_mask */
+extern u64 __read_mostly shadow_xu_mask; /* mutual exclusive with nx_mask and user_mask */
 extern u64 __read_mostly shadow_accessed_mask;
 extern u64 __read_mostly shadow_dirty_mask;
 extern u64 __read_mostly shadow_mmio_value;
@@ -220,10 +221,11 @@ extern u64 __read_mostly shadow_nonpresent_or_rsvd_mask;
  *
  * Only used by the TDP MMU.
  */
-#define FROZEN_SPTE	(SHADOW_NONPRESENT_VALUE | 0x5a0ULL)
+#define FROZEN_SPTE	(SHADOW_NONPRESENT_VALUE | 0x1a0ULL)
 
-/* Frozen SPTEs must not be misconstrued as shadow present PTEs. */
-static_assert(!(FROZEN_SPTE & SPTE_MMU_PRESENT_MASK));
+/* Frozen SPTEs must not be misconstrued as shadow or MMU present PTEs. */
+static_assert(!(FROZEN_SPTE & (SPTE_MMU_PRESENT_MASK |
+			       VMX_EPT_RWX_MASK | VMX_EPT_USER_EXECUTABLE_MASK)));
 
 static inline bool is_frozen_spte(u64 spte)
 {
@@ -357,7 +359,13 @@ static inline bool is_last_spte(u64 pte, int level)
 
 static inline bool is_executable_pte(u64 spte)
 {
-	return (spte & (shadow_x_mask | shadow_nx_mask)) == shadow_x_mask;
+	/*
+	 * For now, return true if either the XS or XU bit is set
+	 * This function is only used for fast_page_fault,
+	 * which never processes shadow EPT, and regular page
+	 * tables always have XS==XU.
+	 */
+	return (spte & (shadow_xs_mask | shadow_xu_mask | shadow_nx_mask)) != shadow_nx_mask;
 }
 
 static inline kvm_pfn_t spte_to_pfn(u64 pte)
@@ -387,6 +395,8 @@ static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check,
 static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check,
 				   u64 pte)
 {
+	if (pte & VMX_EPT_USER_EXECUTABLE_MASK)
+		pte |= VMX_EPT_EXECUTABLE_MASK;
 	return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f);
 }
 
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 7b1102d26f9cd..f98afc3422ce5 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -509,10 +509,10 @@ static void *get_external_spt(gfn_t gfn, u64 new_spte, int level)
 }
 
 static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sptep,
-						 gfn_t gfn, u64 old_spte,
+						 gfn_t gfn, u64 *old_spte,
 						 u64 new_spte, int level)
 {
-	bool was_present = is_shadow_present_pte(old_spte);
+	bool was_present = is_shadow_present_pte(*old_spte);
 	bool is_present = is_shadow_present_pte(new_spte);
 	bool is_leaf = is_present && is_last_spte(new_spte, level);
 	int ret = 0;
@@ -525,7 +525,7 @@ static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sp
 	 * page table has been modified. Use FROZEN_SPTE similar to
 	 * the zapping case.
 	 */
-	if (!try_cmpxchg64(rcu_dereference(sptep), &old_spte, FROZEN_SPTE))
+	if (!try_cmpxchg64(rcu_dereference(sptep), old_spte, FROZEN_SPTE))
 		return -EBUSY;
 
 	/*
@@ -541,7 +541,7 @@ static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sp
 		ret = kvm_x86_call(link_external_spt)(kvm, gfn, level, external_spt);
 	}
 	if (ret)
-		__kvm_tdp_mmu_write_spte(sptep, old_spte);
+		__kvm_tdp_mmu_write_spte(sptep, *old_spte);
 	else
 		__kvm_tdp_mmu_write_spte(sptep, new_spte);
 	return ret;
@@ -670,7 +670,7 @@ static inline int __must_check __tdp_mmu_set_spte_atomic(struct kvm *kvm,
 			return -EBUSY;
 
 		ret = set_external_spte_present(kvm, iter->sptep, iter->gfn,
-						iter->old_spte, new_spte, iter->level);
+						&iter->old_spte, new_spte, iter->level);
 		if (ret)
 			return ret;
 	} else {
@@ -1185,9 +1185,9 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu,
 	}
 
 	if (unlikely(!fault->slot))
-		new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL);
+		new_spte = make_mmio_spte(vcpu, iter->gfn, sp->role.access);
 	else
-		wrprot = make_spte(vcpu, sp, fault->slot, ACC_ALL, iter->gfn,
+		wrprot = make_spte(vcpu, sp, fault->slot, sp->role.access, iter->gfn,
 				   fault->pfn, iter->old_spte, fault->prefetch,
 				   false, fault->map_writable, &new_spte);
 
@@ -1272,7 +1272,7 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
 
 	kvm_mmu_hugepage_adjust(vcpu, fault);
 
-	trace_kvm_mmu_spte_requested(fault);
+	trace_kvm_mmu_spte_requested(fault, root->role.access);
 
 	rcu_read_lock();
 
diff --git a/arch/x86/kvm/reverse_cpuid.h b/arch/x86/kvm/reverse_cpuid.h
index 657f5f743ed97..2ad25781cefb0 100644
--- a/arch/x86/kvm/reverse_cpuid.h
+++ b/arch/x86/kvm/reverse_cpuid.h
@@ -3,8 +3,10 @@
 #define ARCH_X86_KVM_REVERSE_CPUID_H
 
 #include <uapi/asm/kvm.h>
+
 #include <asm/cpufeature.h>
 #include <asm/cpufeatures.h>
+#include <asm/cpuid/types.h>
 
 /*
  * Define a KVM-only feature flag.
diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c
index b340dc9991adb..4ef9bc6a553f3 100644
--- a/arch/x86/kvm/svm/nested.c
+++ b/arch/x86/kvm/svm/nested.c
@@ -93,9 +93,10 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
 	 * when called via KVM_SET_NESTED_STATE, that state may _not_ match current
 	 * vCPU state.  CR0.WP is explicitly ignored, while CR0.PG is required.
 	 */
-	kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4,
+	kvm_init_shadow_npt_mmu(vcpu, svm->vmcb01.ptr->save.cr4,
 				svm->vmcb01.ptr->save.efer,
-				svm->nested.ctl.nested_cr3);
+				svm->nested.ctl.nested_cr3,
+				svm->nested.ctl.misc_ctl);
 	vcpu->arch.mmu->get_guest_pgd     = nested_svm_get_tdp_cr3;
 	vcpu->arch.mmu->get_pdptr         = nested_svm_get_tdp_pdptr;
 	vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
@@ -420,7 +421,8 @@ static bool nested_vmcb_check_controls(struct kvm_vcpu *vcpu,
 
 /* Common checks that apply to both L1 and L2 state.  */
 static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu,
-				   struct vmcb_save_area_cached *save)
+				   struct vmcb_save_area_cached *save,
+				   bool check_gpat)
 {
 	if (CC(!(save->efer & EFER_SVME)))
 		return false;
@@ -455,6 +457,15 @@ static bool nested_vmcb_check_save(struct kvm_vcpu *vcpu,
 	if (CC(!kvm_valid_efer(vcpu, save->efer)))
 		return false;
 
+	/*
+	 * If userspace contrives to get an invalid g_pat into vmcb02 by
+	 * disabling KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT in a race with
+	 * this check, it should be prepared for the KVM_EXIT_FAIL_ENTRY
+	 * that will follow.
+	 */
+	if (check_gpat && CC(!kvm_pat_valid(save->g_pat)))
+		return false;
+
 	return true;
 }
 
@@ -462,7 +473,8 @@ int nested_svm_check_cached_vmcb12(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_svm *svm = to_svm(vcpu);
 
-	if (!nested_vmcb_check_save(vcpu, &svm->nested.save) ||
+	if (!nested_vmcb_check_save(vcpu, &svm->nested.save,
+				    l2_has_separate_pat(vcpu)) ||
 	    !nested_vmcb_check_controls(vcpu, &svm->nested.ctl))
 		return -EINVAL;
 
@@ -498,11 +510,14 @@ void __nested_copy_vmcb_control_to_cache(struct kvm_vcpu *vcpu,
 	nested_svm_sanitize_intercept(vcpu, to, SKINIT);
 	nested_svm_sanitize_intercept(vcpu, to, RDPRU);
 
-	/* Always clear SVM_MISC_ENABLE_NP if the guest cannot use NPTs */
+	/* Always clear misc_ctl bits that the guest cannot use */
 	to->misc_ctl = from->misc_ctl;
 	if (!guest_cpu_cap_has(vcpu, X86_FEATURE_NPT))
 		to->misc_ctl &= ~SVM_MISC_ENABLE_NP;
 
+	if (!gmet_enabled || !guest_cpu_cap_has(vcpu, X86_FEATURE_GMET))
+		to->misc_ctl &= ~SVM_MISC_ENABLE_GMET;
+
 	to->iopm_base_pa        = from->iopm_base_pa & PAGE_MASK;
 	to->msrpm_base_pa       = from->msrpm_base_pa & PAGE_MASK;
 	to->tsc_offset          = from->tsc_offset;
@@ -572,6 +587,7 @@ static void __nested_copy_vmcb_save_to_cache(struct vmcb_save_area_cached *to,
 
 	to->rax = from->rax;
 	to->cr2 = from->cr2;
+	to->g_pat = from->g_pat;
 
 	svm_copy_lbrs(to, from);
 }
@@ -701,15 +717,6 @@ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
 	return 0;
 }
 
-void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm)
-{
-	if (!svm->nested.vmcb02.ptr)
-		return;
-
-	/* FIXME: merge g_pat from vmcb01 and vmcb12.  */
-	svm->nested.vmcb02.ptr->save.g_pat = svm->vmcb01.ptr->save.g_pat;
-}
-
 static bool nested_vmcb12_has_lbrv(struct kvm_vcpu *vcpu)
 {
 	return guest_cpu_cap_has(vcpu, X86_FEATURE_LBRV) &&
@@ -725,9 +732,6 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm)
 	struct vmcb *vmcb02 = svm->nested.vmcb02.ptr;
 	struct kvm_vcpu *vcpu = &svm->vcpu;
 
-	nested_vmcb02_compute_g_pat(svm);
-	vmcb_mark_dirty(vmcb02, VMCB_NPT);
-
 	/* Load the nested guest state */
 	if (svm->nested.vmcb12_gpa != svm->nested.last_vmcb12_gpa) {
 		new_vmcb12 = true;
@@ -758,6 +762,13 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm)
 		vmcb_mark_dirty(vmcb02, VMCB_CET);
 	}
 
+	if (l2_has_separate_pat(vcpu)) {
+		if (unlikely(new_vmcb12 || vmcb12_is_dirty(control, VMCB_NPT)))
+			vmcb_set_gpat(vmcb02, svm->nested.save.g_pat);
+	} else if (npt_enabled) {
+		vmcb_set_gpat(vmcb02, vcpu->arch.pat);
+	}
+
 	kvm_set_rflags(vcpu, save->rflags | X86_EFLAGS_FIXED);
 
 	svm_set_efer(vcpu, svm->nested.save.efer);
@@ -866,7 +877,7 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)
 	 * the latter, L1 runs L2 with shadow page tables that translate L2 GVAs
 	 * to L1 GPAs, so the same NPTs can be used for L1 and L2.
 	 */
-	vmcb02->control.misc_ctl = vmcb01->control.misc_ctl & SVM_MISC_ENABLE_NP;
+	vmcb02->control.misc_ctl = vmcb01->control.misc_ctl & (SVM_MISC_ENABLE_NP | SVM_MISC_ENABLE_GMET);
 	vmcb02->control.iopm_base_pa = vmcb01->control.iopm_base_pa;
 	vmcb02->control.msrpm_base_pa = vmcb01->control.msrpm_base_pa;
 	vmcb_mark_dirty(vmcb02, VMCB_PERM_MAP);
@@ -903,9 +914,13 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm)
 	/* Also overwritten later if necessary.  */
 	vmcb02->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
 
-	/* nested_cr3.  */
-	if (nested_npt_enabled(svm))
+	/* Use vmcb01 MMU and format if guest does not use nNPT */
+	if (nested_npt_enabled(svm)) {
+		vmcb02->control.misc_ctl &= ~SVM_MISC_ENABLE_GMET;
+		vmcb02->control.misc_ctl |= (svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET);
+
 		nested_svm_init_mmu_context(vcpu);
+	}
 
 	vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset(vcpu->arch.l1_tsc_offset,
 							   vmcb12_ctrl->tsc_offset,
@@ -1234,6 +1249,9 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu)
 	vmcb12->save.dr6    = svm->vcpu.arch.dr6;
 	vmcb12->save.cpl    = vmcb02->save.cpl;
 
+	if (l2_has_separate_pat(vcpu))
+		vmcb12->save.g_pat = vmcb02->save.g_pat;
+
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK)) {
 		vmcb12->save.s_cet	= vmcb02->save.s_cet;
 		vmcb12->save.isst_addr	= vmcb02->save.isst_addr;
@@ -1843,6 +1861,9 @@ static int svm_get_nested_state(struct kvm_vcpu *vcpu,
 	/* First fill in the header and copy it out.  */
 	if (is_guest_mode(vcpu)) {
 		kvm_state.hdr.svm.vmcb_pa = svm->nested.vmcb12_gpa;
+		kvm_state.hdr.svm.gpat = 0;
+		if (l2_has_separate_pat(vcpu))
+			kvm_state.hdr.svm.gpat = svm->vmcb->save.g_pat;
 		kvm_state.size += KVM_STATE_NESTED_SVM_VMCB_SIZE;
 		kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
 
@@ -1895,6 +1916,7 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 	struct vmcb_save_area *save;
 	struct vmcb_save_area_cached save_cached;
 	struct vmcb_ctrl_area_cached ctl_cached;
+	bool use_separate_l2_pat;
 	unsigned long cr0;
 	int ret;
 
@@ -1959,15 +1981,29 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 
 	/*
 	 * Validate host state saved from before VMRUN (see
-	 * nested_svm_check_permissions).
+	 * nested_svm_check_permissions). Note that the g_pat field is not
+	 * validated, because (a) it may have been clobbered by SMM before
+	 * KVM_GET_NESTED_STATE, and (b) it is not loaded at emulated
+	 * #VMEXIT.
 	 */
 	__nested_copy_vmcb_save_to_cache(&save_cached, save);
 	if (!(save->cr0 & X86_CR0_PG) ||
 	    !(save->cr0 & X86_CR0_PE) ||
 	    (save->rflags & X86_EFLAGS_VM) ||
-	    !nested_vmcb_check_save(vcpu, &save_cached))
+	    !nested_vmcb_check_save(vcpu, &save_cached, false))
 		goto out_free;
 
+	/*
+	 * Validate gPAT when the shared PAT quirk is disabled (i.e. L2
+	 * has its own gPAT). This is done separately from the
+	 * vmcb_save_area_cached validation above, because gPAT is L2
+	 * state, but the vmcb_save_area_cached is populated with L1 state.
+	 */
+	use_separate_l2_pat = (ctl_cached.misc_ctl & SVM_MISC_ENABLE_NP) &&
+			      !kvm_check_has_quirk(vcpu->kvm,
+						   KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT);
+	if (use_separate_l2_pat && !kvm_pat_valid(kvm_state->hdr.svm.gpat))
+		goto out_free;
 
 	/*
 	 * All checks done, we can enter guest mode. Userspace provides
@@ -1994,6 +2030,10 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu,
 	nested_copy_vmcb_control_to_cache(svm, ctl);
 
 	svm_switch_vmcb(svm, &svm->nested.vmcb02);
+
+	if (use_separate_l2_pat)
+		vmcb_set_gpat(svm->vmcb, kvm_state->hdr.svm.gpat);
+
 	nested_vmcb02_prepare_control(svm);
 
 	/*
@@ -2056,8 +2096,26 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu)
 	return true;
 }
 
+static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
+				      u64 access,
+				      struct x86_exception *exception,
+				      u64 pte_access)
+{
+	struct vcpu_svm *svm = to_svm(vcpu);
+	struct kvm_mmu *mmu = vcpu->arch.mmu;
+
+	BUG_ON(!mmu_is_nested(vcpu));
+
+	/* Non-GMET walks are always user-walks */
+	if (!(svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_GMET))
+		access |= PFERR_USER_MASK;
+
+	return mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
+}
+
 struct kvm_x86_nested_ops svm_nested_ops = {
 	.leave_nested = svm_leave_nested,
+	.translate_nested_gpa = svm_translate_nested_gpa,
 	.is_exception_vmexit = nested_svm_is_exception_vmexit,
 	.check_events = svm_check_nested_events,
 	.triple_fault = nested_svm_triple_fault,
diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c
index c2126b3c30724..3314223d4a373 100644
--- a/arch/x86/kvm/svm/sev.c
+++ b/arch/x86/kvm/svm/sev.c
@@ -23,6 +23,7 @@
 
 #include <asm/pkru.h>
 #include <asm/trapnr.h>
+#include <asm/cpuid/api.h>
 #include <asm/fpu/xcr.h>
 #include <asm/fpu/xstate.h>
 #include <asm/debugreg.h>
@@ -720,14 +721,50 @@ static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	return ret;
 }
 
+static int sev_check_pin_count(struct kvm *kvm, unsigned long npages)
+{
+	unsigned long total_npages, lock_limit;
+
+	total_npages = to_kvm_sev_info(kvm)->pages_locked + npages;
+	if (total_npages > totalram_pages())
+		return -EINVAL;
+
+	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
+	if (total_npages > lock_limit && !capable(CAP_IPC_LOCK)) {
+		pr_err_ratelimited("SEV: %lu total pages would exceed the lock limit of %lu.\n",
+				   total_npages, lock_limit);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int sev_pin_user_pages(struct kvm *kvm, unsigned long addr, int npages,
+			      unsigned int gup_flags, struct page **pages)
+{
+	int npinned;
+
+	lockdep_assert_held(&kvm->lock);
+
+	npinned = pin_user_pages_fast(addr, npages, gup_flags, pages);
+	if (npinned != npages) {
+		if (npinned > 0)
+			unpin_user_pages(pages, npinned);
+		pr_err_ratelimited("SEV: Failure locking %u pages.\n", npages);
+		return -ENOMEM;
+	}
+
+	to_kvm_sev_info(kvm)->pages_locked += npages;
+	return 0;
+}
+
 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
 				    unsigned long ulen, unsigned long *n,
 				    unsigned int flags)
 {
-	struct kvm_sev_info *sev = to_kvm_sev_info(kvm);
-	unsigned long npages, total_npages, lock_limit;
+	unsigned long npages;
 	struct page **pages;
-	int npinned, ret;
+	int ret;
 
 	lockdep_assert_held(&kvm->lock);
 
@@ -743,16 +780,9 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
 	if (npages > INT_MAX)
 		return ERR_PTR(-EINVAL);
 
-	total_npages = sev->pages_locked + npages;
-	if (total_npages > totalram_pages())
-		return ERR_PTR(-EINVAL);
-
-	lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-	if (total_npages > lock_limit && !capable(CAP_IPC_LOCK)) {
-		pr_err("SEV: %lu total pages would exceed the lock limit of %lu.\n",
-		       total_npages, lock_limit);
-		return ERR_PTR(-ENOMEM);
-	}
+	ret = sev_check_pin_count(kvm, npages);
+	if (ret)
+		return ERR_PTR(ret);
 
 	/*
 	 * Don't WARN if the kernel (rightly) thinks the total size is absurd,
@@ -764,25 +794,14 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
 	if (!pages)
 		return ERR_PTR(-ENOMEM);
 
-	/* Pin the user virtual address. */
-	npinned = pin_user_pages_fast(uaddr, npages, flags, pages);
-	if (npinned != npages) {
-		pr_err("SEV: Failure locking %lu pages.\n", npages);
-		ret = -ENOMEM;
-		goto err;
+	ret = sev_pin_user_pages(kvm, uaddr, npages, flags, pages);
+	if (ret) {
+		kvfree(pages);
+		return ERR_PTR(ret);
 	}
 
 	*n = npages;
-	sev->pages_locked = total_npages;
-
 	return pages;
-
-err:
-	if (npinned > 0)
-		unpin_user_pages(pages, npinned);
-
-	kvfree(pages);
-	return ERR_PTR(ret);
 }
 
 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
@@ -793,6 +812,29 @@ static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
 	to_kvm_sev_info(kvm)->pages_locked -= npages;
 }
 
+static struct page *sev_pin_page(struct kvm *kvm, unsigned long addr,
+				 unsigned int flags)
+{
+	struct page *page;
+	int r;
+
+	r = sev_check_pin_count(kvm, 1);
+	if (r)
+		return ERR_PTR(r);
+
+	r = sev_pin_user_pages(kvm, addr, 1, flags, &page);
+	if (r)
+		return ERR_PTR(r);
+
+	return page;
+}
+
+static void sev_unpin_page(struct kvm *kvm, struct page *page)
+{
+	unpin_user_pages(&page, 1);
+	to_kvm_sev_info(kvm)->pages_locked -= 1;
+}
+
 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
 {
 	uint8_t *page_virtual;
@@ -967,7 +1009,7 @@ static int sev_es_sync_vmsa(struct vcpu_svm *svm)
 	save->r14 = svm->vcpu.arch.regs[VCPU_REGS_R14];
 	save->r15 = svm->vcpu.arch.regs[VCPU_REGS_R15];
 #endif
-	save->rip = svm->vcpu.arch.regs[VCPU_REGS_RIP];
+	save->rip = svm->vcpu.arch.rip;
 
 	/* Sync some non-GPR registers before encrypting */
 	save->xcr0 = svm->vcpu.arch.xcr0;
@@ -1196,160 +1238,115 @@ static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	return ret;
 }
 
-static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
-			       unsigned long dst, int size,
-			       int *error, bool enc)
+static int sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src_pa,
+			     unsigned long dst_pa, unsigned int size,
+			     unsigned int ioctl, int *error)
 {
-	struct sev_data_dbg data;
-
-	data.reserved = 0;
-	data.handle = to_kvm_sev_info(kvm)->handle;
-	data.dst_addr = dst;
-	data.src_addr = src;
-	data.len = size;
+	int cmd = ioctl == KVM_SEV_DBG_DECRYPT ? SEV_CMD_DBG_DECRYPT :
+						 SEV_CMD_DBG_ENCRYPT;
+	struct sev_data_dbg data = {
+		.handle = to_kvm_sev_info(kvm)->handle,
+		.dst_addr = dst_pa,
+		.src_addr = src_pa,
+		.len = size,
+	};
 
-	return sev_issue_cmd(kvm,
-			     enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
-			     &data, error);
+	return sev_issue_cmd(kvm, cmd, &data, error);
 }
 
-static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
-			     unsigned long dst_paddr, int sz, int *err)
+static void *sev_dbg_crypt_slow_alloc(struct page *page, unsigned long __va,
+				      unsigned int len, unsigned long *pa,
+				      unsigned int *nr_bytes)
 {
-	int offset;
+	unsigned long va = ALIGN_DOWN(__va, 16);
+
+	/* The number of bytes to {de,en}crypt must be 16-byte aligned. */
+	*nr_bytes = round_up(len, 16);
 
 	/*
-	 * Its safe to read more than we are asked, caller should ensure that
-	 * destination has enough space.
+	 * Increase the number of bytes to {de,en}crypt by one chunk (16 bytes)
+	 * if the aligned address and length doesn't cover the unaligned range,
+	 * e.g. if the address is unaligned _and_ the access will split a chunk
+	 * at the tail.
 	 */
-	offset = src_paddr & 15;
-	src_paddr = round_down(src_paddr, 16);
-	sz = round_up(sz + offset, 16);
+	if (va + *nr_bytes < __va + len)
+		*nr_bytes += 16;
 
-	return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
-}
+	*pa = __sme_page_pa(page) + (va & ~PAGE_MASK);
 
-static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
-				  void __user *dst_uaddr,
-				  unsigned long dst_paddr,
-				  int size, int *err)
-{
-	struct page *tpage = NULL;
-	int ret, offset;
-
-	/* if inputs are not 16-byte then use intermediate buffer */
-	if (!IS_ALIGNED(dst_paddr, 16) ||
-	    !IS_ALIGNED(paddr,     16) ||
-	    !IS_ALIGNED(size,      16)) {
-		tpage = (void *)alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO);
-		if (!tpage)
-			return -ENOMEM;
+	/*
+	 * Sanity check that the new access won't split a page.  This should
+	 * never happen; just pretend the allocation failed.
+	 */
+	if (WARN_ON_ONCE((*pa & PAGE_MASK) != ((*pa + *nr_bytes - 1) & PAGE_MASK)))
+		return NULL;
 
-		dst_paddr = __sme_page_pa(tpage);
-	}
+	return kmalloc(*nr_bytes, GFP_KERNEL);
+}
 
-	ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
-	if (ret)
-		goto e_free;
+static int sev_dbg_decrypt_slow(struct kvm *kvm, unsigned long src,
+				struct page *src_p, unsigned long dst,
+				unsigned int len, int *err)
+{
+	unsigned int nr_bytes;
+	unsigned long src_pa;
+	void *buf;
+	int r;
 
-	if (tpage) {
-		offset = paddr & 15;
-		if (copy_to_user(dst_uaddr, page_address(tpage) + offset, size))
-			ret = -EFAULT;
-	}
+	buf = sev_dbg_crypt_slow_alloc(src_p, src, len, &src_pa, &nr_bytes);
+	if (!buf)
+		return -ENOMEM;
 
-e_free:
-	if (tpage)
-		__free_page(tpage);
+	r = sev_issue_dbg_cmd(kvm, src_pa, __sme_set(__pa(buf)),
+			      nr_bytes, KVM_SEV_DBG_DECRYPT, err);
+	if (r)
+		goto out;
 
-	return ret;
+	if (copy_to_user((void __user *)dst, buf + (src & 15), len))
+		r = -EFAULT;
+out:
+	kfree(buf);
+	return r;
 }
 
-static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
-				  void __user *vaddr,
-				  unsigned long dst_paddr,
-				  void __user *dst_vaddr,
-				  int size, int *error)
+static int sev_dbg_encrypt_slow(struct kvm *kvm, unsigned long src,
+				unsigned long dst, struct page *dst_p,
+				unsigned int len, int *err)
 {
-	struct page *src_tpage = NULL;
-	struct page *dst_tpage = NULL;
-	int ret, len = size;
-
-	/* If source buffer is not aligned then use an intermediate buffer */
-	if (!IS_ALIGNED((unsigned long)vaddr, 16)) {
-		src_tpage = alloc_page(GFP_KERNEL_ACCOUNT);
-		if (!src_tpage)
-			return -ENOMEM;
+	unsigned int nr_bytes;
+	unsigned long dst_pa;
+	void *buf;
+	int r;
 
-		if (copy_from_user(page_address(src_tpage), vaddr, size)) {
-			__free_page(src_tpage);
-			return -EFAULT;
-		}
+	/* Decrypt the _destination_ to do a RMW on plaintext. */
+	buf = sev_dbg_crypt_slow_alloc(dst_p, dst, len, &dst_pa, &nr_bytes);
+	if (!buf)
+		return -ENOMEM;
 
-		paddr = __sme_page_pa(src_tpage);
-	}
+	r = sev_issue_dbg_cmd(kvm, dst_pa, __sme_set(__pa(buf)),
+			      nr_bytes, KVM_SEV_DBG_DECRYPT, err);
+	if (r)
+		goto out;
 
 	/*
-	 *  If destination buffer or length is not aligned then do read-modify-write:
-	 *   - decrypt destination in an intermediate buffer
-	 *   - copy the source buffer in an intermediate buffer
-	 *   - use the intermediate buffer as source buffer
+	 * Copy from the source into the intermediate buffer, and then
+	 * re-encrypt the buffer into the destination.
 	 */
-	if (!IS_ALIGNED((unsigned long)dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
-		int dst_offset;
-
-		dst_tpage = alloc_page(GFP_KERNEL_ACCOUNT);
-		if (!dst_tpage) {
-			ret = -ENOMEM;
-			goto e_free;
-		}
-
-		ret = __sev_dbg_decrypt(kvm, dst_paddr,
-					__sme_page_pa(dst_tpage), size, error);
-		if (ret)
-			goto e_free;
-
-		/*
-		 *  If source is kernel buffer then use memcpy() otherwise
-		 *  copy_from_user().
-		 */
-		dst_offset = dst_paddr & 15;
-
-		if (src_tpage)
-			memcpy(page_address(dst_tpage) + dst_offset,
-			       page_address(src_tpage), size);
-		else {
-			if (copy_from_user(page_address(dst_tpage) + dst_offset,
-					   vaddr, size)) {
-				ret = -EFAULT;
-				goto e_free;
-			}
-		}
-
-		paddr = __sme_page_pa(dst_tpage);
-		dst_paddr = round_down(dst_paddr, 16);
-		len = round_up(size, 16);
-	}
-
-	ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
-
-e_free:
-	if (src_tpage)
-		__free_page(src_tpage);
-	if (dst_tpage)
-		__free_page(dst_tpage);
-	return ret;
+	if (copy_from_user(buf + (dst & 15), (void __user *)src, len))
+		r = -EFAULT;
+	else
+		r = sev_issue_dbg_cmd(kvm, __sme_set(__pa(buf)), dst_pa,
+				      nr_bytes, KVM_SEV_DBG_ENCRYPT, err);
+out:
+	kfree(buf);
+	return r;
 }
 
-static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
+static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp,
+			 unsigned int cmd)
 {
-	unsigned long vaddr, vaddr_end, next_vaddr;
-	unsigned long dst_vaddr;
-	struct page **src_p, **dst_p;
 	struct kvm_sev_dbg debug;
-	unsigned long n;
-	unsigned int size;
-	int ret;
+	unsigned int i, len;
 
 	if (!sev_guest(kvm))
 		return -ENOTTY;
@@ -1357,27 +1354,38 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
 	if (copy_from_user(&debug, u64_to_user_ptr(argp->data), sizeof(debug)))
 		return -EFAULT;
 
-	if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
+	if (!debug.len || !debug.src_uaddr || !debug.dst_uaddr)
 		return -EINVAL;
-	if (!debug.dst_uaddr)
+
+	if (debug.src_uaddr + debug.len < debug.src_uaddr ||
+	    debug.dst_uaddr + debug.len < debug.dst_uaddr)
 		return -EINVAL;
 
-	vaddr = debug.src_uaddr;
-	size = debug.len;
-	vaddr_end = vaddr + size;
-	dst_vaddr = debug.dst_uaddr;
+	for (i = 0; i < debug.len; i += len) {
+		unsigned long src = debug.src_uaddr + i;
+		unsigned long dst = debug.dst_uaddr + i;
+		unsigned long s_off = src & ~PAGE_MASK;
+		unsigned long d_off = dst & ~PAGE_MASK;
+		struct page *src_p, *dst_p;
+		int ret;
 
-	for (; vaddr < vaddr_end; vaddr = next_vaddr) {
-		int len, s_off, d_off;
+		/*
+		 * Copy as many remaining bytes as possible while staying in a
+		 * single page for both the source and destination.
+		 */
+		len = min3(debug.len - i, PAGE_SIZE - s_off, PAGE_SIZE - d_off);
 
-		/* lock userspace source and destination page */
-		src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
+		/*
+		 * Pin the source and destination pages; firmware operates on
+		 * physical addresses.
+		 */
+		src_p = sev_pin_page(kvm, src & PAGE_MASK, 0);
 		if (IS_ERR(src_p))
 			return PTR_ERR(src_p);
 
-		dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, FOLL_WRITE);
+		dst_p = sev_pin_page(kvm, dst & PAGE_MASK, FOLL_WRITE);
 		if (IS_ERR(dst_p)) {
-			sev_unpin_memory(kvm, src_p, n);
+			sev_unpin_page(kvm, src_p);
 			return PTR_ERR(dst_p);
 		}
 
@@ -1386,43 +1394,28 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
 		 * the pages; flush the destination too so that future accesses do not
 		 * see stale data.
 		 */
-		sev_clflush_pages(src_p, 1);
-		sev_clflush_pages(dst_p, 1);
-
-		/*
-		 * Since user buffer may not be page aligned, calculate the
-		 * offset within the page.
-		 */
-		s_off = vaddr & ~PAGE_MASK;
-		d_off = dst_vaddr & ~PAGE_MASK;
-		len = min_t(size_t, (PAGE_SIZE - s_off), size);
-
-		if (dec)
-			ret = __sev_dbg_decrypt_user(kvm,
-						     __sme_page_pa(src_p[0]) + s_off,
-						     (void __user *)dst_vaddr,
-						     __sme_page_pa(dst_p[0]) + d_off,
-						     len, &argp->error);
+		sev_clflush_pages(&src_p, 1);
+		sev_clflush_pages(&dst_p, 1);
+
+		if (IS_ALIGNED(src, 16) && IS_ALIGNED(dst, 16) && IS_ALIGNED(len, 16))
+			ret = sev_issue_dbg_cmd(kvm,
+						__sme_page_pa(src_p) + s_off,
+						__sme_page_pa(dst_p) + d_off,
+						len, cmd, &argp->error);
+		else if (cmd == KVM_SEV_DBG_DECRYPT)
+			ret = sev_dbg_decrypt_slow(kvm, src, src_p, dst,
+						   len, &argp->error);
 		else
-			ret = __sev_dbg_encrypt_user(kvm,
-						     __sme_page_pa(src_p[0]) + s_off,
-						     (void __user *)vaddr,
-						     __sme_page_pa(dst_p[0]) + d_off,
-						     (void __user *)dst_vaddr,
-						     len, &argp->error);
+			ret = sev_dbg_encrypt_slow(kvm, src, dst, dst_p,
+						   len, &argp->error);
 
-		sev_unpin_memory(kvm, src_p, n);
-		sev_unpin_memory(kvm, dst_p, n);
+		sev_unpin_page(kvm, src_p);
+		sev_unpin_page(kvm, dst_p);
 
 		if (ret)
-			goto err;
-
-		next_vaddr = vaddr + len;
-		dst_vaddr = dst_vaddr + len;
-		size -= len;
+			return ret;
 	}
-err:
-	return ret;
+	return 0;
 }
 
 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
@@ -1695,8 +1688,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	struct sev_data_send_update_data data;
 	struct kvm_sev_send_update_data params;
 	void *hdr, *trans_data;
-	struct page **guest_page;
-	unsigned long n;
+	struct page *guest_page;
 	int ret, offset;
 
 	if (!sev_guest(kvm))
@@ -1720,8 +1712,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 		return -EINVAL;
 
 	/* Pin guest memory */
-	guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK,
-				    PAGE_SIZE, &n, 0);
+	guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, 0);
 	if (IS_ERR(guest_page))
 		return PTR_ERR(guest_page);
 
@@ -1742,7 +1733,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	data.trans_len = params.trans_len;
 
 	/* The SEND_UPDATE_DATA command requires C-bit to be always set. */
-	data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset;
+	data.guest_address = page_to_phys(guest_page) + offset;
 	data.guest_address |= sev_me_mask;
 	data.guest_len = params.guest_len;
 	data.handle = to_kvm_sev_info(kvm)->handle;
@@ -1769,8 +1760,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 e_free_hdr:
 	kfree(hdr);
 e_unpin:
-	sev_unpin_memory(kvm, guest_page, n);
-
+	sev_unpin_page(kvm, guest_page);
 	return ret;
 }
 
@@ -1875,8 +1865,7 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	struct kvm_sev_receive_update_data params;
 	struct sev_data_receive_update_data data;
 	void *hdr = NULL, *trans = NULL;
-	struct page **guest_page;
-	unsigned long n;
+	struct page *guest_page;
 	int ret, offset;
 
 	if (!sev_guest(kvm))
@@ -1913,8 +1902,7 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	data.trans_len = params.trans_len;
 
 	/* Pin guest memory */
-	guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK,
-				    PAGE_SIZE, &n, FOLL_WRITE);
+	guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, FOLL_WRITE);
 	if (IS_ERR(guest_page)) {
 		ret = PTR_ERR(guest_page);
 		goto e_free_trans;
@@ -1925,10 +1913,10 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	 * encrypts the written data with the guest's key, and the cache may
 	 * contain dirty, unencrypted data.
 	 */
-	sev_clflush_pages(guest_page, n);
+	sev_clflush_pages(&guest_page, 1);
 
 	/* The RECEIVE_UPDATE_DATA command requires C-bit to be always set. */
-	data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset;
+	data.guest_address = page_to_phys(guest_page) + offset;
 	data.guest_address |= sev_me_mask;
 	data.guest_len = params.guest_len;
 	data.handle = to_kvm_sev_info(kvm)->handle;
@@ -1936,7 +1924,7 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
 	ret = sev_issue_cmd(kvm, SEV_CMD_RECEIVE_UPDATE_DATA, &data,
 				&argp->error);
 
-	sev_unpin_memory(kvm, guest_page, n);
+	sev_unpin_page(kvm, guest_page);
 
 e_free_trans:
 	kfree(trans);
@@ -2689,10 +2677,8 @@ int sev_mem_enc_ioctl(struct kvm *kvm, void __user *argp)
 		r = sev_guest_status(kvm, &sev_cmd);
 		break;
 	case KVM_SEV_DBG_DECRYPT:
-		r = sev_dbg_crypt(kvm, &sev_cmd, true);
-		break;
 	case KVM_SEV_DBG_ENCRYPT:
-		r = sev_dbg_crypt(kvm, &sev_cmd, false);
+		r = sev_dbg_crypt(kvm, &sev_cmd, sev_cmd.id);
 		break;
 	case KVM_SEV_LAUNCH_SECRET:
 		r = sev_launch_secret(kvm, &sev_cmd);
@@ -3013,18 +2999,14 @@ void sev_vm_destroy(struct kvm *kvm)
 
 void __init sev_set_cpu_caps(void)
 {
-	if (sev_enabled) {
+	if (sev_enabled)
 		kvm_cpu_cap_set(X86_FEATURE_SEV);
-		kvm_caps.supported_vm_types |= BIT(KVM_X86_SEV_VM);
-	}
-	if (sev_es_enabled) {
+
+	if (sev_es_enabled)
 		kvm_cpu_cap_set(X86_FEATURE_SEV_ES);
-		kvm_caps.supported_vm_types |= BIT(KVM_X86_SEV_ES_VM);
-	}
-	if (sev_snp_enabled) {
+
+	if (sev_snp_enabled)
 		kvm_cpu_cap_set(X86_FEATURE_SEV_SNP);
-		kvm_caps.supported_vm_types |= BIT(KVM_X86_SNP_VM);
-	}
 }
 
 static bool is_sev_snp_initialized(void)
@@ -3054,6 +3036,11 @@ static bool is_sev_snp_initialized(void)
 	return initialized;
 }
 
+static const char * __init sev_str_feature_state(bool is_supported, bool is_usable)
+{
+	return is_supported ? is_usable ? "enabled" : "unusable" : "disabled";
+}
+
 void __init sev_hardware_setup(void)
 {
 	unsigned int eax, ebx, ecx, edx, sev_asid_count, sev_es_asid_count;
@@ -3061,6 +3048,7 @@ void __init sev_hardware_setup(void)
 	bool sev_snp_supported = false;
 	bool sev_es_supported = false;
 	bool sev_supported = false;
+	u32 vm_types = 0;
 
 	if (!sev_enabled || !npt_enabled || !nrips)
 		goto out;
@@ -3194,21 +3182,27 @@ void __init sev_hardware_setup(void)
 		}
 	}
 
+	if (sev_supported && min_sev_asid <= max_sev_asid)
+		vm_types |= BIT(KVM_X86_SEV_VM);
+	if (sev_es_supported && min_sev_es_asid <= max_sev_es_asid)
+		vm_types |= BIT(KVM_X86_SEV_ES_VM);
+	if (sev_snp_supported)
+		vm_types |= BIT(KVM_X86_SNP_VM);
+	vm_types &= sev_firmware_supported_vm_types();
+
+	kvm_caps.supported_vm_types |= vm_types;
+
 	if (boot_cpu_has(X86_FEATURE_SEV))
 		pr_info("SEV %s (ASIDs %u - %u)\n",
-			sev_supported ? min_sev_asid <= max_sev_asid ? "enabled" :
-								       "unusable" :
-								       "disabled",
+			sev_str_feature_state(sev_supported, vm_types & BIT(KVM_X86_SEV_VM)),
 			min_sev_asid, max_sev_asid);
 	if (boot_cpu_has(X86_FEATURE_SEV_ES))
 		pr_info("SEV-ES %s (ASIDs %u - %u)\n",
-			sev_es_supported ? min_sev_es_asid <= max_sev_es_asid ? "enabled" :
-										"unusable" :
-										"disabled",
+			sev_str_feature_state(sev_es_supported, vm_types & BIT(KVM_X86_SEV_ES_VM)),
 			min_sev_es_asid, max_sev_es_asid);
 	if (boot_cpu_has(X86_FEATURE_SEV_SNP))
 		pr_info("SEV-SNP %s (ASIDs %u - %u)\n",
-			str_enabled_disabled(sev_snp_supported),
+			sev_str_feature_state(sev_snp_supported, vm_types & BIT(KVM_X86_SNP_VM)),
 			min_snp_asid, max_snp_asid);
 
 	sev_enabled = sev_supported;
diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c
index e02a38da5296e..aa58c79b31bce 100644
--- a/arch/x86/kvm/svm/svm.c
+++ b/arch/x86/kvm/svm/svm.c
@@ -41,6 +41,7 @@
 #include <asm/irq_remapping.h>
 #include <asm/spec-ctrl.h>
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/traps.h>
 #include <asm/reboot.h>
 #include <asm/fpu/api.h>
@@ -50,6 +51,7 @@
 
 #include "trace.h"
 
+#include "vmenter.h"
 #include "svm.h"
 #include "svm_ops.h"
 
@@ -138,6 +140,9 @@ module_param(pause_filter_count_max, ushort, 0444);
 bool __ro_after_init npt_enabled = true;
 module_param_named(npt, npt_enabled, bool, 0444);
 
+bool gmet_enabled = true;
+module_param_named(gmet, gmet_enabled, bool, 0444);
+
 /* allow nested virtualization in KVM/SVM */
 static int __ro_after_init nested = true;
 module_param(nested, int, 0444);
@@ -661,7 +666,7 @@ static void clr_dr_intercepts(struct vcpu_svm *svm)
 	svm_mark_intercepts_dirty(svm);
 }
 
-static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
+static bool msr_write_intercepted(struct vcpu_svm *svm, u32 msr)
 {
 	/*
 	 * For non-nested case:
@@ -672,8 +677,7 @@ static bool msr_write_intercepted(struct kvm_vcpu *vcpu, u32 msr)
 	 * If the L02 MSR bitmap does not intercept the MSR, then we need to
 	 * save it.
 	 */
-	void *msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm :
-					    to_svm(vcpu)->msrpm;
+	void *msrpm = is_guest_mode(&svm->vcpu) ? svm->nested.msrpm : svm->msrpm;
 
 	return svm_test_msr_bitmap_write(msrpm, msr);
 }
@@ -1220,6 +1224,10 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event)
 		save->g_pat = vcpu->arch.pat;
 		save->cr3 = 0;
 	}
+
+	if (gmet_enabled)
+		control->misc_ctl |= SVM_MISC_ENABLE_GMET;
+
 	svm->current_vmcb->asid_generation = 0;
 	svm->asid = 0;
 
@@ -1528,7 +1536,7 @@ static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
 	kvm_register_mark_available(vcpu, reg);
 
 	switch (reg) {
-	case VCPU_EXREG_PDPTR:
+	case VCPU_REG_PDPTR:
 		/*
 		 * When !npt_enabled, mmu->pdptrs[] is already available since
 		 * it is always updated per SDM when moving to CRs.
@@ -1997,6 +2005,18 @@ static int npf_interception(struct kvm_vcpu *vcpu)
 		}
 	}
 
+	if (!is_sev_es_guest(vcpu) &&
+	    (svm->vmcb->control.misc_ctl & SVM_MISC_ENABLE_GMET) &&
+	    (error_code & PFERR_FETCH_MASK)) {
+		/*
+		 * Work around errata 1218: EXITINFO1[2] May Be Incorrectly Set
+		 * When GMET (Guest Mode Execute Trap extension) is Enabled
+		 */
+		error_code |= PFERR_USER_MASK;
+		if (svm_get_cpl(vcpu) != 3)
+			error_code &= ~PFERR_USER_MASK;
+	}
+
 	if (is_sev_snp_guest(vcpu) && (error_code & PFERR_GUEST_ENC_MASK))
 		error_code |= PFERR_PRIVATE_ACCESS;
 
@@ -2775,7 +2795,21 @@ static bool sev_es_prevent_msr_access(struct kvm_vcpu *vcpu,
 {
 	return is_sev_es_guest(vcpu) && vcpu->arch.guest_state_protected &&
 	       msr_info->index != MSR_IA32_XSS &&
-	       !msr_write_intercepted(vcpu, msr_info->index);
+	       !msr_write_intercepted(to_svm(vcpu), msr_info->index);
+}
+
+static bool svm_pat_accesses_gpat(struct kvm_vcpu *vcpu, bool from_host)
+{
+	/*
+	 * When KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT is disabled and nested
+	 * NPT is enabled, L2 has a separate PAT from L1.  Guest accesses
+	 * to IA32_PAT while running L2 target L2's gPAT; host-initiated
+	 * accesses always target L1's hPAT so that KVM_GET/SET_MSRS and
+	 * KVM_GET/SET_NESTED_STATE are independent of each other and can
+	 * be ordered arbitrarily during save and restore.
+	 */
+	WARN_ON_ONCE(from_host && vcpu->wants_to_run);
+	return !from_host && is_guest_mode(vcpu) && l2_has_separate_pat(vcpu);
 }
 
 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
@@ -2894,6 +2928,12 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_AMD64_DE_CFG:
 		msr_info->data = svm->msr_decfg;
 		break;
+	case MSR_IA32_CR_PAT:
+		if (svm_pat_accesses_gpat(vcpu, msr_info->host_initiated)) {
+			msr_info->data = svm->vmcb->save.g_pat;
+			break;
+		}
+		return kvm_get_msr_common(vcpu, msr_info);
 	default:
 		return kvm_get_msr_common(vcpu, msr_info);
 	}
@@ -2977,14 +3017,23 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
 
 		break;
 	case MSR_IA32_CR_PAT:
+		if (svm_pat_accesses_gpat(vcpu, msr->host_initiated)) {
+			if (!kvm_pat_valid(data))
+				return 1;
+
+			vmcb_set_gpat(svm->vmcb, data);
+			break;
+		}
+
 		ret = kvm_set_msr_common(vcpu, msr);
 		if (ret)
 			break;
 
-		svm->vmcb01.ptr->save.g_pat = data;
-		if (is_guest_mode(vcpu))
-			nested_vmcb02_compute_g_pat(svm);
-		vmcb_mark_dirty(svm->vmcb, VMCB_NPT);
+		if (npt_enabled) {
+			vmcb_set_gpat(svm->vmcb01.ptr, data);
+			if (is_guest_mode(vcpu) && !l2_has_separate_pat(vcpu))
+				vmcb_set_gpat(svm->vmcb, data);
+		}
 		break;
 	case MSR_IA32_SPEC_CTRL:
 		if (!msr->host_initiated &&
@@ -3655,13 +3704,8 @@ static int svm_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
 	struct vcpu_svm *svm = to_svm(vcpu);
 	struct kvm_run *kvm_run = vcpu->run;
 
-	/* SEV-ES guests must use the CR write traps to track CR registers. */
-	if (!is_sev_es_guest(vcpu)) {
-		if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
-			vcpu->arch.cr0 = svm->vmcb->save.cr0;
-		if (npt_enabled)
-			vcpu->arch.cr3 = svm->vmcb->save.cr3;
-	}
+	if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE))
+		return 0;
 
 	if (is_guest_mode(vcpu)) {
 		int vmexit;
@@ -4190,7 +4234,7 @@ static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
 
 static void svm_flush_tlb_guest(struct kvm_vcpu *vcpu)
 {
-	kvm_register_mark_dirty(vcpu, VCPU_EXREG_ERAPS);
+	kvm_register_mark_dirty(vcpu, VCPU_REG_ERAPS);
 
 	svm_flush_tlb_asid(vcpu);
 }
@@ -4389,7 +4433,7 @@ static fastpath_t svm_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
 	return EXIT_FASTPATH_NONE;
 }
 
-static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_intercepted)
+static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, unsigned enter_flags)
 {
 	struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu);
 	struct vcpu_svm *svm = to_svm(vcpu);
@@ -4411,10 +4455,10 @@ static noinstr void svm_vcpu_enter_exit(struct kvm_vcpu *vcpu, bool spec_ctrl_in
 	amd_clear_divider();
 
 	if (is_sev_es_guest(vcpu))
-		__svm_sev_es_vcpu_run(svm, spec_ctrl_intercepted,
+		__svm_sev_es_vcpu_run(svm, enter_flags,
 				      sev_es_host_save_area(sd));
 	else
-		__svm_vcpu_run(svm, spec_ctrl_intercepted);
+		__svm_vcpu_run(svm, enter_flags);
 
 	raw_local_irq_disable();
 
@@ -4425,13 +4469,16 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 {
 	bool force_immediate_exit = run_flags & KVM_RUN_FORCE_IMMEDIATE_EXIT;
 	struct vcpu_svm *svm = to_svm(vcpu);
-	bool spec_ctrl_intercepted = msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL);
+	unsigned enter_flags = 0;
+
+	if (!msr_write_intercepted(svm, MSR_IA32_SPEC_CTRL))
+		enter_flags |= KVM_ENTER_SAVE_SPEC_CTRL;
 
 	trace_kvm_entry(vcpu, force_immediate_exit);
 
 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
-	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
+	svm->vmcb->save.rip = vcpu->arch.rip;
 
 	/*
 	 * Disable singlestep if we're injecting an interrupt/exception.
@@ -4468,7 +4515,7 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 	svm->vmcb->save.cr2 = vcpu->arch.cr2;
 
 	if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS) &&
-	    kvm_register_is_dirty(vcpu, VCPU_EXREG_ERAPS))
+	    kvm_register_is_dirty(vcpu, VCPU_REG_ERAPS))
 		svm->vmcb->control.erap_ctl |= ERAP_CONTROL_CLEAR_RAP;
 
 	svm_fixup_nested_rips(vcpu);
@@ -4508,18 +4555,24 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
 		x86_spec_ctrl_set_guest(svm->virt_spec_ctrl);
 
-	svm_vcpu_enter_exit(vcpu, spec_ctrl_intercepted);
+	svm_vcpu_enter_exit(vcpu, enter_flags);
 
 	if (!static_cpu_has(X86_FEATURE_V_SPEC_CTRL))
 		x86_spec_ctrl_restore_host(svm->virt_spec_ctrl);
 
+	/* SEV-ES guests must use the CR write traps to track CR registers. */
 	if (!is_sev_es_guest(vcpu)) {
 		vcpu->arch.cr2 = svm->vmcb->save.cr2;
 		vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
 		vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
-		vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
+		vcpu->arch.rip = svm->vmcb->save.rip;
+
+		if (!svm_is_intercept(svm, INTERCEPT_CR0_WRITE))
+			vcpu->arch.cr0 = svm->vmcb->save.cr0;
+		if (npt_enabled)
+			vcpu->arch.cr3 = svm->vmcb->save.cr3;
 	}
-	vcpu->arch.regs_dirty = 0;
+	kvm_reset_dirty_registers(vcpu);
 
 	if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
 		kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
@@ -4565,9 +4618,9 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 		vcpu->arch.apf.host_apf_flags =
 			kvm_read_and_reset_apf_flags();
 
-	vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET;
+	kvm_clear_available_registers(vcpu, SVM_REGS_LAZY_LOAD_SET);
 
-	if (!msr_write_intercepted(vcpu, MSR_AMD64_PERF_CNTR_GLOBAL_CTL))
+	if (!msr_write_intercepted(svm, MSR_AMD64_PERF_CNTR_GLOBAL_CTL))
 		rdmsrq(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, vcpu_to_pmu(vcpu)->global_ctrl);
 
 	trace_kvm_exit(vcpu, KVM_ISA_SVM);
@@ -4623,6 +4676,11 @@ svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
 	hypercall[2] = 0xd9;
 }
 
+static bool svm_tdp_has_smep(struct kvm *kvm)
+{
+	return gmet_enabled;
+}
+
 /*
  * The kvm parameter can be NULL (module initialization, or invocation before
  * VM creation). Be sure to check the kvm parameter before using it.
@@ -4957,7 +5015,7 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram)
 
 	svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
 	svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
-	svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
+	svm->vmcb->save.rip = vcpu->arch.rip;
 
 	nested_svm_simple_vmexit(svm, SVM_EXIT_SW);
 
@@ -5366,6 +5424,7 @@ struct kvm_x86_ops svm_x86_ops __initdata = {
 	.write_tsc_multiplier = svm_write_tsc_multiplier,
 
 	.load_mmu_pgd = svm_load_mmu_pgd,
+	.tdp_has_smep = svm_tdp_has_smep,
 
 	.check_intercept = svm_check_intercept,
 	.handle_exit_irqoff = svm_handle_exit_irqoff,
@@ -5490,6 +5549,9 @@ static __init void svm_set_cpu_caps(void)
 		if (boot_cpu_has(X86_FEATURE_PFTHRESHOLD))
 			kvm_cpu_cap_set(X86_FEATURE_PFTHRESHOLD);
 
+		if (gmet_enabled)
+			kvm_cpu_cap_set(X86_FEATURE_GMET);
+
 		if (vgif)
 			kvm_cpu_cap_set(X86_FEATURE_VGIF);
 
@@ -5599,6 +5661,9 @@ static __init int svm_hardware_setup(void)
 	if (!boot_cpu_has(X86_FEATURE_NPT))
 		npt_enabled = false;
 
+	if (!npt_enabled || !boot_cpu_has(X86_FEATURE_GMET))
+		gmet_enabled = false;
+
 	/* Force VM NPT level equal to the host's paging level */
 	kvm_configure_mmu(npt_enabled, get_npt_level(),
 			  get_npt_level(), PG_LEVEL_1G);
diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h
index a10668d17a16a..2b6733dffd76f 100644
--- a/arch/x86/kvm/svm/svm.h
+++ b/arch/x86/kvm/svm/svm.h
@@ -24,6 +24,7 @@
 
 #include "cpuid.h"
 #include "kvm_cache_regs.h"
+#include "x86.h"
 
 /*
  * Helpers to convert to/from physical addresses for pages whose address is
@@ -44,6 +45,7 @@ static inline struct page *__sme_pa_to_page(unsigned long pa)
 #define	IOPM_SIZE PAGE_SIZE * 3
 #define	MSRPM_SIZE PAGE_SIZE * 2
 
+extern bool gmet_enabled;
 extern bool npt_enabled;
 extern int nrips;
 extern int vgif;
@@ -165,6 +167,7 @@ struct vmcb_save_area_cached {
 	u64 isst_addr;
 	u64 rax;
 	u64 cr2;
+	u64 g_pat;
 	u64 dbgctl;
 	u64 br_from;
 	u64 br_to;
@@ -463,6 +466,12 @@ static inline bool vmcb12_is_dirty(struct vmcb_ctrl_area_cached *control, int bi
 	return !test_bit(bit, (unsigned long *)&control->clean);
 }
 
+static inline void vmcb_set_gpat(struct vmcb *vmcb, u64 data)
+{
+	vmcb->save.g_pat = data;
+	vmcb_mark_dirty(vmcb, VMCB_NPT);
+}
+
 static __always_inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 {
 	return container_of(vcpu, struct vcpu_svm, vcpu);
@@ -484,7 +493,7 @@ static inline bool svm_is_vmrun_failure(u64 exit_code)
  * KVM_REQ_LOAD_MMU_PGD is always requested when the cached vcpu->arch.cr3
  * is changed.  svm_load_mmu_pgd() then syncs the new CR3 value into the VMCB.
  */
-#define SVM_REGS_LAZY_LOAD_SET	(1 << VCPU_EXREG_PDPTR)
+#define SVM_REGS_LAZY_LOAD_SET	(BIT(VCPU_REG_PDPTR))
 
 static inline void __vmcb_set_intercept(unsigned long *intercepts, u32 bit)
 {
@@ -640,6 +649,16 @@ static inline bool nested_npt_enabled(struct vcpu_svm *svm)
 	return svm->nested.ctl.misc_ctl & SVM_MISC_ENABLE_NP;
 }
 
+static inline bool l2_has_separate_pat(struct kvm_vcpu *vcpu)
+{
+	/*
+	 * If KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT is disabled while a vCPU
+	 * is running, the L2 IA32_PAT semantics for that vCPU are undefined.
+	 */
+	return nested_npt_enabled(to_svm(vcpu)) &&
+	       !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_NESTED_SVM_SHARED_PAT);
+}
+
 static inline bool nested_vnmi_enabled(struct vcpu_svm *svm)
 {
 	return guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_VNMI) &&
@@ -874,7 +893,6 @@ void nested_copy_vmcb_control_to_cache(struct vcpu_svm *svm,
 void nested_copy_vmcb_save_to_cache(struct vcpu_svm *svm,
 				    struct vmcb_save_area *save);
 void nested_sync_control_from_vmcb02(struct vcpu_svm *svm);
-void nested_vmcb02_compute_g_pat(struct vcpu_svm *svm);
 void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb);
 
 extern struct kvm_x86_nested_ops svm_nested_ops;
@@ -1007,9 +1025,9 @@ static inline void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_sa
 
 /* vmenter.S */
 
-void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted,
+void __svm_sev_es_vcpu_run(struct vcpu_svm *svm, unsigned int flags,
 			   struct sev_es_save_area *hostsa);
-void __svm_vcpu_run(struct vcpu_svm *svm, bool spec_ctrl_intercepted);
+void __svm_vcpu_run(struct vcpu_svm *svm, unsigned int flags);
 
 #define DEFINE_KVM_GHCB_ACCESSORS(field)						\
 static __always_inline u64 kvm_ghcb_get_##field(struct vcpu_svm *svm)			\
diff --git a/arch/x86/kvm/svm/vmenter.S b/arch/x86/kvm/svm/vmenter.S
index d47c5c93c9913..f523d9e498398 100644
--- a/arch/x86/kvm/svm/vmenter.S
+++ b/arch/x86/kvm/svm/vmenter.S
@@ -7,6 +7,7 @@
 #include <asm/kvm_vcpu_regs.h>
 #include <asm/nospec-branch.h>
 #include "kvm-asm-offsets.h"
+#include "vmenter.h"
 
 #define WORD_SIZE (BITS_PER_LONG / 8)
 
@@ -39,38 +40,6 @@
 	ALTERNATIVE_2 "", \
 		"jmp 800f", X86_FEATURE_MSR_SPEC_CTRL, \
 		"", X86_FEATURE_V_SPEC_CTRL
-801:
-.endm
-.macro RESTORE_GUEST_SPEC_CTRL_BODY
-800:
-	/*
-	 * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the
-	 * host's, write the MSR.  This is kept out-of-line so that the common
-	 * case does not have to jump.
-	 *
-	 * IMPORTANT: To avoid RSB underflow attacks and any other nastiness,
-	 * there must not be any returns or indirect branches between this code
-	 * and vmentry.
-	 */
-#ifdef CONFIG_X86_64
-	mov SVM_spec_ctrl(%rdi), %rdx
-	cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx
-	je 801b
-	movl %edx, %eax
-	shr $32, %rdx
-#else
-	mov SVM_spec_ctrl(%edi), %eax
-	mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx
-	xor %eax, %ecx
-	mov SVM_spec_ctrl + 4(%edi), %edx
-	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %esi
-	xor %edx, %esi
-	or %esi, %ecx
-	je 801b
-#endif
-	mov $MSR_IA32_SPEC_CTRL, %ecx
-	wrmsr
-	jmp 801b
 .endm
 
 .macro RESTORE_HOST_SPEC_CTRL
@@ -78,42 +47,6 @@
 	ALTERNATIVE_2 "", \
 		"jmp 900f", X86_FEATURE_MSR_SPEC_CTRL, \
 		"", X86_FEATURE_V_SPEC_CTRL
-901:
-.endm
-.macro RESTORE_HOST_SPEC_CTRL_BODY spec_ctrl_intercepted:req
-900:
-	/* Same for after vmexit.  */
-	mov $MSR_IA32_SPEC_CTRL, %ecx
-
-	/*
-	 * Load the value that the guest had written into MSR_IA32_SPEC_CTRL,
-	 * if it was not intercepted during guest execution.
-	 */
-	cmpb $0, \spec_ctrl_intercepted
-	jnz 998f
-	rdmsr
-	movl %eax, SVM_spec_ctrl(%_ASM_DI)
-	movl %edx, SVM_spec_ctrl + 4(%_ASM_DI)
-998:
-	/* Now restore the host value of the MSR if different from the guest's.  */
-#ifdef CONFIG_X86_64
-	mov PER_CPU_VAR(x86_spec_ctrl_current), %rdx
-	cmp SVM_spec_ctrl(%rdi), %rdx
-	je 901b
-	movl %edx, %eax
-	shr $32, %rdx
-#else
-	mov PER_CPU_VAR(x86_spec_ctrl_current), %eax
-	mov SVM_spec_ctrl(%edi), %esi
-	xor %eax, %esi
-	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edx
-	mov SVM_spec_ctrl + 4(%edi), %edi
-	xor %edx, %edi
-	or %edi, %esi
-	je 901b
-#endif
-	wrmsr
-	jmp 901b
 .endm
 
 #define SVM_CLEAR_CPU_BUFFERS \
@@ -121,8 +54,8 @@
 
 /**
  * __svm_vcpu_run - Run a vCPU via a transition to SVM guest mode
- * @svm:	struct vcpu_svm *
- * @spec_ctrl_intercepted: bool
+ * @svm:	 struct vcpu_svm *
+ * @enter_flags: u32
  */
 SYM_FUNC_START(__svm_vcpu_run)
 	push %_ASM_BP
@@ -162,6 +95,7 @@ SYM_FUNC_START(__svm_vcpu_run)
 
 	/* Clobbers RAX, RCX, RDX (and ESI on 32-bit), consumes RDI (@svm). */
 	RESTORE_GUEST_SPEC_CTRL
+801:
 
 	/*
 	 * Use a single vmcb (vmcb01 because it's always valid) for
@@ -242,6 +176,7 @@ SYM_FUNC_START(__svm_vcpu_run)
 	 * and RSP (pointer to @spec_ctrl_intercepted).
 	 */
 	RESTORE_HOST_SPEC_CTRL
+901:
 
 	/*
 	 * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
@@ -278,7 +213,7 @@ SYM_FUNC_START(__svm_vcpu_run)
 	xor %r15d, %r15d
 #endif
 
-	/* "Pop" @spec_ctrl_intercepted.  */
+	/* "Pop" @enter_flags.  */
 	pop %_ASM_BX
 
 	pop %_ASM_BX
@@ -295,8 +230,12 @@ SYM_FUNC_START(__svm_vcpu_run)
 	pop %_ASM_BP
 	RET
 
-	RESTORE_GUEST_SPEC_CTRL_BODY
-	RESTORE_HOST_SPEC_CTRL_BODY (%_ASM_SP)
+800:
+	RESTORE_GUEST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), 801b
+	jmp 801b
+900:
+	RESTORE_HOST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), (%_ASM_SP), 901b
+	jmp 901b
 
 10:	cmpb $0, _ASM_RIP(virt_rebooting)
 	jne 2b
@@ -335,8 +274,8 @@ SYM_FUNC_END(__svm_vcpu_run)
 
 /**
  * __svm_sev_es_vcpu_run - Run a SEV-ES vCPU via a transition to SVM guest mode
- * @svm:	struct vcpu_svm *
- * @spec_ctrl_intercepted: bool
+ * @svm:	 struct vcpu_svm *
+ * @enter_flags: u32
  */
 SYM_FUNC_START(__svm_sev_es_vcpu_run)
 	FRAME_BEGIN
@@ -355,13 +294,14 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
 
 	/*
 	 * Save volatile registers that hold arguments that are needed after
-	 * #VMEXIT (RDI=@svm and RSI=@spec_ctrl_intercepted).
+	 * #VMEXIT (RDI=@svm and RSI=@enter_flags).
 	 */
 	mov %rdi, SEV_ES_RDI (%rdx)
 	mov %rsi, SEV_ES_RSI (%rdx)
 
 	/* Clobbers RAX, RCX, and RDX (@hostsa), consumes RDI (@svm). */
 	RESTORE_GUEST_SPEC_CTRL
+801:
 
 	/* Get svm->current_vmcb->pa into RAX. */
 	mov SVM_current_vmcb(%rdi), %rax
@@ -376,8 +316,9 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
 	/* IMPORTANT: Stuff the RSB immediately after VM-Exit, before RET! */
 	FILL_RETURN_BUFFER %rax, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT
 
-	/* Clobbers RAX, RCX, RDX, consumes RDI (@svm) and RSI (@spec_ctrl_intercepted). */
+	/* Clobbers RAX, RCX, RDX, consumes RDI (@svm) and RSI (@enter_flags). */
 	RESTORE_HOST_SPEC_CTRL
+901:
 
 	/*
 	 * Mitigate RETBleed for AMD/Hygon Zen uarch. RET should be
@@ -391,8 +332,12 @@ SYM_FUNC_START(__svm_sev_es_vcpu_run)
 	FRAME_END
 	RET
 
-	RESTORE_GUEST_SPEC_CTRL_BODY
-	RESTORE_HOST_SPEC_CTRL_BODY %sil
+800:
+	RESTORE_GUEST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), 801b
+	jmp 801b
+900:
+	RESTORE_HOST_SPEC_CTRL_BODY SVM_spec_ctrl(%_ASM_DI), %esi, 901b
+	jmp 901b
 
 3:	cmpb $0, virt_rebooting(%rip)
 	jne 2b
diff --git a/arch/x86/kvm/vmenter.h b/arch/x86/kvm/vmenter.h
new file mode 100644
index 0000000000000..ba3f71449c62c
--- /dev/null
+++ b/arch/x86/kvm/vmenter.h
@@ -0,0 +1,80 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __KVM_X86_VMENTER_H
+#define __KVM_X86_VMENTER_H
+
+#define KVM_ENTER_VMRESUME			BIT(0)
+#define KVM_ENTER_SAVE_SPEC_CTRL		BIT(1)
+#define KVM_ENTER_CLEAR_CPU_BUFFERS_FOR_MMIO	BIT(2)
+
+#ifdef __ASSEMBLER__
+.macro RESTORE_GUEST_SPEC_CTRL_BODY guest_spec_ctrl:req, label:req
+	/*
+	 * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the
+	 * host's, write the MSR.  This is kept out-of-line so that the common
+	 * case does not have to jump.
+	 *
+	 * IMPORTANT: To avoid RSB underflow attacks and any other nastiness,
+	 * there must not be any returns or indirect branches between this code
+	 * and vmentry.
+	 */
+#ifdef CONFIG_X86_64
+	mov \guest_spec_ctrl, %rdx
+	cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx
+	je \label
+	movl %edx, %eax
+	shr $32, %rdx
+#else
+	mov \guest_spec_ctrl, %eax
+	mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx
+	xor %eax, %ecx
+	mov 4 + \guest_spec_ctrl, %edx
+	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %esi
+	xor %edx, %esi
+	or %esi, %ecx
+	je \label
+#endif
+	mov $MSR_IA32_SPEC_CTRL, %ecx
+	wrmsr
+.endm
+
+.macro RESTORE_HOST_SPEC_CTRL_BODY guest_spec_ctrl:req, enter_flags:req, label:req
+	/* Same for after vmexit.  */
+	mov $MSR_IA32_SPEC_CTRL, %ecx
+
+	/*
+	 * Load the value that the guest had written into MSR_IA32_SPEC_CTRL,
+	 * if it was not intercepted during guest execution.
+	 */
+	testl $KVM_ENTER_SAVE_SPEC_CTRL, \enter_flags
+	jz 998f
+	rdmsr
+	movl %eax, \guest_spec_ctrl
+	movl %edx, 4 + \guest_spec_ctrl
+998:
+	/* Now restore the host value of the MSR if different from the guest's.  */
+#ifdef CONFIG_X86_64
+	mov PER_CPU_VAR(x86_spec_ctrl_current), %rdx
+	cmp \guest_spec_ctrl, %rdx
+	/*
+	 * For legacy IBRS, the IBRS bit always needs to be written after
+	 * transitioning from a less privileged predictor mode, regardless of
+	 * whether the guest/host values differ.
+	 */
+	ALTERNATIVE __stringify(je \label), "", X86_FEATURE_KERNEL_IBRS
+	movl %edx, %eax
+	shr $32, %rdx
+#else
+	mov PER_CPU_VAR(x86_spec_ctrl_current), %eax
+	mov \guest_spec_ctrl, %esi
+	xor %eax, %esi
+	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edx
+	mov 4 + \guest_spec_ctrl, %edi
+	xor %edx, %edi
+	or %edi, %esi
+	ALTERNATIVE __stringify(je \label), "", X86_FEATURE_KERNEL_IBRS
+#endif
+	wrmsr
+.endm
+
+#endif /* __ASSEMBLER__ */
+#endif /* __KVM_X86_VMENTER_H */
diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h
index 31568274d8bb0..810119167f798 100644
--- a/arch/x86/kvm/vmx/capabilities.h
+++ b/arch/x86/kvm/vmx/capabilities.h
@@ -16,6 +16,7 @@ extern bool __read_mostly enable_unrestricted_guest;
 extern bool __read_mostly enable_ept_ad_bits;
 extern bool __read_mostly enable_cet;
 extern bool __read_mostly enable_pml;
+extern bool __read_mostly enable_mbec;
 extern int __read_mostly pt_mode;
 
 #define PT_MODE_SYSTEM		0
@@ -301,11 +302,6 @@ static inline bool cpu_has_vmx_flexpriority(void)
 		cpu_has_vmx_virtualize_apic_accesses();
 }
 
-static inline bool cpu_has_vmx_ept_execute_only(void)
-{
-	return vmx_capability.ept & VMX_EPT_EXECUTE_ONLY_BIT;
-}
-
 static inline bool cpu_has_vmx_ept_4levels(void)
 {
 	return vmx_capability.ept & VMX_EPT_PAGE_WALK_4_BIT;
@@ -412,4 +408,10 @@ static inline bool cpu_has_notify_vmexit(void)
 		SECONDARY_EXEC_NOTIFY_VM_EXITING;
 }
 
+static inline bool cpu_has_ept_mbec(void)
+{
+	return vmcs_config.cpu_based_2nd_exec_ctrl &
+		SECONDARY_EXEC_MODE_BASED_EPT_EXEC;
+}
+
 #endif /* __KVM_X86_VMX_CAPS_H */
diff --git a/arch/x86/kvm/vmx/common.h b/arch/x86/kvm/vmx/common.h
index 412d0829d7a21..08005676702c2 100644
--- a/arch/x86/kvm/vmx/common.h
+++ b/arch/x86/kvm/vmx/common.h
@@ -85,22 +85,30 @@ static inline int __vmx_handle_ept_violation(struct kvm_vcpu *vcpu, gpa_t gpa,
 {
 	u64 error_code;
 
-	/* Is it a read fault? */
-	error_code = (exit_qualification & EPT_VIOLATION_ACC_READ)
-		     ? PFERR_USER_MASK : 0;
 	/* Is it a write fault? */
-	error_code |= (exit_qualification & EPT_VIOLATION_ACC_WRITE)
+	error_code = (exit_qualification & EPT_VIOLATION_ACC_WRITE)
 		      ? PFERR_WRITE_MASK : 0;
 	/* Is it a fetch fault? */
 	error_code |= (exit_qualification & EPT_VIOLATION_ACC_INSTR)
 		      ? PFERR_FETCH_MASK : 0;
-	/* ept page table entry is present? */
-	error_code |= (exit_qualification & EPT_VIOLATION_PROT_MASK)
+	/* ept page table entry is present?  */
+	error_code |= (exit_qualification &
+		       (EPT_VIOLATION_PROT_MASK & ~EPT_VIOLATION_PROT_USER_EXEC))
 		      ? PFERR_PRESENT_MASK : 0;
 
-	if (exit_qualification & EPT_VIOLATION_GVA_IS_VALID)
-		error_code |= (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) ?
-			      PFERR_GUEST_FINAL_MASK : PFERR_GUEST_PAGE_MASK;
+	if (mmu_has_mbec(vcpu->arch.mmu))
+		error_code |= (exit_qualification & EPT_VIOLATION_PROT_USER_EXEC)
+			      ? PFERR_PRESENT_MASK : 0;
+
+	if (exit_qualification & EPT_VIOLATION_GVA_IS_VALID) {
+		if (exit_qualification & EPT_VIOLATION_GVA_TRANSLATED) {
+			error_code |= PFERR_GUEST_FINAL_MASK;
+			if (exit_qualification & EPT_VIOLATION_GVA_USER)
+				error_code |= PFERR_USER_MASK;
+		} else {
+			error_code |= PFERR_GUEST_PAGE_MASK;
+		}
+	}
 
 	if (vt_is_tdx_private_gpa(vcpu->kvm, gpa))
 		error_code |= PFERR_PRIVATE_ACCESS;
diff --git a/arch/x86/kvm/vmx/hyperv_evmcs.h b/arch/x86/kvm/vmx/hyperv_evmcs.h
index fc7c4e7bd1bfb..bc08fe40590e9 100644
--- a/arch/x86/kvm/vmx/hyperv_evmcs.h
+++ b/arch/x86/kvm/vmx/hyperv_evmcs.h
@@ -87,6 +87,7 @@
 	 SECONDARY_EXEC_PT_CONCEAL_VMX |				\
 	 SECONDARY_EXEC_BUS_LOCK_DETECTION |				\
 	 SECONDARY_EXEC_NOTIFY_VM_EXITING |				\
+	 SECONDARY_EXEC_MODE_BASED_EPT_EXEC |				\
 	 SECONDARY_EXEC_ENCLS_EXITING)
 
 #define EVMCS1_SUPPORTED_3RDEXEC (0ULL)
diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c
index dbebddf648be7..83d9921277eac 100644
--- a/arch/x86/kvm/vmx/main.c
+++ b/arch/x86/kvm/vmx/main.c
@@ -755,6 +755,14 @@ static int vt_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
 	return vmx_set_identity_map_addr(kvm, ident_addr);
 }
 
+static bool vt_tdp_has_smep(struct kvm *kvm)
+{
+	if (is_td(kvm))
+		return false;
+
+	return vmx_tdp_has_smep(kvm);
+}
+
 static u64 vt_get_l2_tsc_offset(struct kvm_vcpu *vcpu)
 {
 	/* TDX doesn't support L2 guest at the moment. */
@@ -966,6 +974,7 @@ struct kvm_x86_ops vt_x86_ops __initdata = {
 	.set_tss_addr = vt_op(set_tss_addr),
 	.set_identity_map_addr = vt_op(set_identity_map_addr),
 	.get_mt_mask = vmx_get_mt_mask,
+	.tdp_has_smep = vt_op(tdp_has_smep),
 
 	.get_exit_info = vt_op(get_exit_info),
 	.get_entry_info = vt_op(get_entry_info),
diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index 3fe88f29be7a9..4690a4d23709d 100644
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -310,13 +310,13 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
 	vmx_sync_vmcs_host_state(vmx, prev);
 	put_cpu();
 
-	vcpu->arch.regs_avail = ~VMX_REGS_LAZY_LOAD_SET;
+	kvm_clear_available_registers(vcpu, VMX_REGS_LAZY_LOAD_SET);
 
 	/*
 	 * All lazily updated registers will be reloaded from VMCS12 on both
 	 * vmentry and vmexit.
 	 */
-	vcpu->arch.regs_dirty = 0;
+	kvm_reset_dirty_registers(vcpu);
 }
 
 static void nested_put_vmcs12_pages(struct kvm_vcpu *vcpu)
@@ -443,10 +443,14 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
 			vm_exit_reason = EXIT_REASON_EPT_MISCONFIG;
 			exit_qualification = 0;
 		} else {
+			u64 mask = EPT_VIOLATION_GVA_IS_VALID |
+				EPT_VIOLATION_GVA_TRANSLATED;
+			if (vmx->nested.msrs.ept_caps & VMX_EPT_ADVANCED_VMEXIT_INFO_BIT)
+				mask |= EPT_VIOLATION_GVA_USER |
+					       EPT_VIOLATION_GVA_WRITABLE |
+					       EPT_VIOLATION_GVA_NX;
 			exit_qualification = fault->exit_qualification;
-			exit_qualification |= vmx_get_exit_qual(vcpu) &
-					      (EPT_VIOLATION_GVA_IS_VALID |
-					       EPT_VIOLATION_GVA_TRANSLATED);
+			exit_qualification |= vmx_get_exit_qual(vcpu) & mask;
 			vm_exit_reason = EXIT_REASON_EPT_VIOLATION;
 		}
 
@@ -465,6 +469,13 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
 	vmcs12->guest_physical_address = fault->address;
 }
 
+static inline bool nested_ept_mbec_enabled(struct kvm_vcpu *vcpu)
+{
+	struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
+
+	return nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC);
+}
+
 static void nested_ept_new_eptp(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
@@ -473,6 +484,7 @@ static void nested_ept_new_eptp(struct kvm_vcpu *vcpu)
 
 	kvm_init_shadow_ept_mmu(vcpu, execonly, ept_lpage_level,
 				nested_ept_ad_enabled(vcpu),
+				nested_ept_mbec_enabled(vcpu),
 				nested_ept_get_eptp(vcpu));
 }
 
@@ -1189,7 +1201,7 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3,
 	}
 
 	vcpu->arch.cr3 = cr3;
-	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
+	kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
 
 	/* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */
 	kvm_init_mmu(vcpu);
@@ -2440,6 +2452,7 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct loaded_vmcs *vmcs0
 				  SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
 				  SECONDARY_EXEC_APIC_REGISTER_VIRT |
 				  SECONDARY_EXEC_ENABLE_VMFUNC |
+				  SECONDARY_EXEC_MODE_BASED_EPT_EXEC |
 				  SECONDARY_EXEC_DESC);
 
 		if (nested_cpu_has(vmcs12,
@@ -4972,7 +4985,7 @@ static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu)
 
 	nested_ept_uninit_mmu_context(vcpu);
 	vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
-	kvm_register_mark_available(vcpu, VCPU_EXREG_CR3);
+	kvm_register_mark_available(vcpu, VCPU_REG_CR3);
 
 	/*
 	 * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs
@@ -5074,7 +5087,7 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason,
 	kvm_service_local_tlb_flush_requests(vcpu);
 
 	/*
-	 * VCPU_EXREG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between
+	 * VCPU_REG_PDPTR will be clobbered in arch/x86/kvm/vmx/vmx.h between
 	 * now and the new vmentry.  Ensure that the VMCS02 PDPTR fields are
 	 * up-to-date before switching to L1.
 	 */
@@ -7239,7 +7252,8 @@ static void nested_vmx_setup_secondary_ctls(u32 ept_caps,
 			VMX_EPT_PAGE_WALK_5_BIT |
 			VMX_EPTP_WB_BIT |
 			VMX_EPT_INVEPT_BIT |
-			VMX_EPT_EXECUTE_ONLY_BIT;
+			VMX_EPT_EXECUTE_ONLY_BIT |
+			VMX_EPT_ADVANCED_VMEXIT_INFO_BIT;
 
 		msrs->ept_caps &= ept_caps;
 		msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
@@ -7251,6 +7265,9 @@ static void nested_vmx_setup_secondary_ctls(u32 ept_caps,
 			msrs->ept_caps |= VMX_EPT_AD_BIT;
 		}
 
+		if (enable_mbec)
+			msrs->secondary_ctls_high |=
+				SECONDARY_EXEC_MODE_BASED_EPT_EXEC;
 		/*
 		 * Advertise EPTP switching irrespective of hardware support,
 		 * KVM emulates it in software so long as VMFUNC is supported.
@@ -7438,8 +7455,29 @@ __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
 	return 0;
 }
 
+static gpa_t vmx_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa,
+				      u64 access,
+				      struct x86_exception *exception,
+				      u64 pte_access)
+{
+	struct kvm_mmu *mmu = vcpu->arch.mmu;
+
+	BUG_ON(!mmu_is_nested(vcpu));
+
+	/*
+	 * MBEC differentiates based on the effective U/S bit of
+	 * the guest page tables; not the processor CPL.
+	 */
+	access &= ~PFERR_USER_MASK;
+	if ((pte_access & ACC_USER_MASK) && (access & PFERR_GUEST_FINAL_MASK))
+		access |= PFERR_USER_MASK;
+
+	return mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
+}
+
 struct kvm_x86_nested_ops vmx_nested_ops = {
 	.leave_nested = vmx_leave_nested,
+	.translate_nested_gpa = vmx_translate_nested_gpa,
 	.is_exception_vmexit = nested_vmx_is_exception_vmexit,
 	.check_events = vmx_check_nested_events,
 	.has_events = vmx_has_nested_events,
diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c
index 27eb76e6b6a03..74e0b01185b80 100644
--- a/arch/x86/kvm/vmx/pmu_intel.c
+++ b/arch/x86/kvm/vmx/pmu_intel.c
@@ -15,6 +15,7 @@
 #include <linux/perf_event.h>
 #include <asm/msr.h>
 #include <asm/perf_event.h>
+#include <asm/cpuid/api.h>
 #include "x86.h"
 #include "cpuid.h"
 #include "lapic.h"
diff --git a/arch/x86/kvm/vmx/run_flags.h b/arch/x86/kvm/vmx/run_flags.h
deleted file mode 100644
index 6a87a12135fb5..0000000000000
--- a/arch/x86/kvm/vmx/run_flags.h
+++ /dev/null
@@ -1,9 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __KVM_X86_VMX_RUN_FLAGS_H
-#define __KVM_X86_VMX_RUN_FLAGS_H
-
-#define VMX_RUN_VMRESUME			BIT(0)
-#define VMX_RUN_SAVE_SPEC_CTRL			BIT(1)
-#define VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO	BIT(2)
-
-#endif /* __KVM_X86_VMX_RUN_FLAGS_H */
diff --git a/arch/x86/kvm/vmx/sgx.c b/arch/x86/kvm/vmx/sgx.c
index df1d0cf769475..29a1f8e3be605 100644
--- a/arch/x86/kvm/vmx/sgx.c
+++ b/arch/x86/kvm/vmx/sgx.c
@@ -2,6 +2,7 @@
 /*  Copyright(c) 2021 Intel Corporation. */
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 #include <asm/sgx.h>
 
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index 04ce321ebdf39..0becb7a5678b0 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -440,16 +440,6 @@ void tdx_disable_virtualization_cpu(void)
 		tdx_flush_vp(&arg);
 	}
 	local_irq_restore(flags);
-
-	/*
-	 * Flush cache now if kexec is possible: this is necessary to avoid
-	 * having dirty private memory cachelines when the new kernel boots,
-	 * but WBINVD is a relatively expensive operation and doing it during
-	 * kexec can exacerbate races in native_stop_other_cpus().  Do it
-	 * now, since this is a safe moment and there is going to be no more
-	 * TDX activity on this CPU from this point on.
-	 */
-	tdx_cpu_flush_cache_for_kexec();
 }
 
 #define TDX_SEAMCALL_RETRIES 10000
@@ -1013,23 +1003,23 @@ static fastpath_t tdx_exit_handlers_fastpath(struct kvm_vcpu *vcpu)
 	return EXIT_FASTPATH_NONE;
 }
 
-#define TDX_REGS_AVAIL_SET	(BIT_ULL(VCPU_EXREG_EXIT_INFO_1) | \
-				 BIT_ULL(VCPU_EXREG_EXIT_INFO_2) | \
-				 BIT_ULL(VCPU_REGS_RAX) | \
-				 BIT_ULL(VCPU_REGS_RBX) | \
-				 BIT_ULL(VCPU_REGS_RCX) | \
-				 BIT_ULL(VCPU_REGS_RDX) | \
-				 BIT_ULL(VCPU_REGS_RBP) | \
-				 BIT_ULL(VCPU_REGS_RSI) | \
-				 BIT_ULL(VCPU_REGS_RDI) | \
-				 BIT_ULL(VCPU_REGS_R8) | \
-				 BIT_ULL(VCPU_REGS_R9) | \
-				 BIT_ULL(VCPU_REGS_R10) | \
-				 BIT_ULL(VCPU_REGS_R11) | \
-				 BIT_ULL(VCPU_REGS_R12) | \
-				 BIT_ULL(VCPU_REGS_R13) | \
-				 BIT_ULL(VCPU_REGS_R14) | \
-				 BIT_ULL(VCPU_REGS_R15))
+#define TDX_REGS_AVAIL_SET	(BIT(VCPU_REG_EXIT_INFO_1) | \
+				 BIT(VCPU_REG_EXIT_INFO_2) | \
+				 BIT(VCPU_REGS_RAX) | \
+				 BIT(VCPU_REGS_RBX) | \
+				 BIT(VCPU_REGS_RCX) | \
+				 BIT(VCPU_REGS_RDX) | \
+				 BIT(VCPU_REGS_RBP) | \
+				 BIT(VCPU_REGS_RSI) | \
+				 BIT(VCPU_REGS_RDI) | \
+				 BIT(VCPU_REGS_R8) | \
+				 BIT(VCPU_REGS_R9) | \
+				 BIT(VCPU_REGS_R10) | \
+				 BIT(VCPU_REGS_R11) | \
+				 BIT(VCPU_REGS_R12) | \
+				 BIT(VCPU_REGS_R13) | \
+				 BIT(VCPU_REGS_R14) | \
+				 BIT(VCPU_REGS_R15))
 
 static void tdx_load_host_xsave_state(struct kvm_vcpu *vcpu)
 {
@@ -1098,7 +1088,7 @@ fastpath_t tdx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 
 	tdx_load_host_xsave_state(vcpu);
 
-	vcpu->arch.regs_avail &= TDX_REGS_AVAIL_SET;
+	kvm_clear_available_registers(vcpu, ~TDX_REGS_AVAIL_SET);
 
 	if (unlikely(tdx->vp_enter_ret == EXIT_REASON_EPT_MISCONFIG))
 		return EXIT_FASTPATH_NONE;
@@ -1638,14 +1628,13 @@ static int tdx_mem_page_add(struct kvm *kvm, gfn_t gfn, enum pg_level level,
 static int tdx_mem_page_aug(struct kvm *kvm, gfn_t gfn,
 			    enum pg_level level, kvm_pfn_t pfn)
 {
-	int tdx_level = pg_level_to_tdx_sept_level(level);
 	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
 	struct page *page = pfn_to_page(pfn);
 	gpa_t gpa = gfn_to_gpa(gfn);
 	u64 entry, level_state;
 	u64 err;
 
-	err = tdh_mem_page_aug(&kvm_tdx->td, gpa, tdx_level, page, &entry, &level_state);
+	err = tdh_mem_page_aug(&kvm_tdx->td, gpa, level, page, &entry, &level_state);
 	if (unlikely(tdx_operand_busy(err)))
 		return -EBUSY;
 
@@ -1689,12 +1678,11 @@ static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,
 static int tdx_sept_link_private_spt(struct kvm *kvm, gfn_t gfn,
 				     enum pg_level level, void *private_spt)
 {
-	int tdx_level = pg_level_to_tdx_sept_level(level);
 	gpa_t gpa = gfn_to_gpa(gfn);
 	struct page *page = virt_to_page(private_spt);
 	u64 err, entry, level_state;
 
-	err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, tdx_level, page, &entry,
+	err = tdh_mem_sept_add(&to_kvm_tdx(kvm)->td, gpa, level, page, &entry,
 			       &level_state);
 	if (unlikely(tdx_operand_busy(err)))
 		return -EBUSY;
@@ -1778,7 +1766,6 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn,
 					 enum pg_level level, u64 mirror_spte)
 {
 	struct page *page = pfn_to_page(spte_to_pfn(mirror_spte));
-	int tdx_level = pg_level_to_tdx_sept_level(level);
 	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
 	gpa_t gpa = gfn_to_gpa(gfn);
 	u64 err, entry, level_state;
@@ -1798,7 +1785,7 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn,
 		return;
 
 	err = tdh_do_no_vcpus(tdh_mem_range_block, kvm, &kvm_tdx->td, gpa,
-			      tdx_level, &entry, &level_state);
+			      level, &entry, &level_state);
 	if (TDX_BUG_ON_2(err, TDH_MEM_RANGE_BLOCK, entry, level_state, kvm))
 		return;
 
@@ -1814,7 +1801,7 @@ static void tdx_sept_remove_private_spte(struct kvm *kvm, gfn_t gfn,
 	 * Race with TDH.VP.ENTER due to (0-step mitigation) and Guest TDCALLs.
 	 */
 	err = tdh_do_no_vcpus(tdh_mem_page_remove, kvm, &kvm_tdx->td, gpa,
-			      tdx_level, &entry, &level_state);
+			      level, &entry, &level_state);
 	if (TDX_BUG_ON_2(err, TDH_MEM_PAGE_REMOVE, entry, level_state, kvm))
 		return;
 
@@ -1845,7 +1832,7 @@ static inline bool tdx_is_sept_violation_unexpected_pending(struct kvm_vcpu *vcp
 	if (eeq_type != TDX_EXT_EXIT_QUAL_TYPE_PENDING_EPT_VIOLATION)
 		return false;
 
-	return !(eq & EPT_VIOLATION_PROT_MASK) && !(eq & EPT_VIOLATION_EXEC_FOR_RING3_LIN);
+	return !(eq & EPT_VIOLATION_PROT_MASK);
 }
 
 static int tdx_handle_ept_violation(struct kvm_vcpu *vcpu)
@@ -2116,23 +2103,29 @@ bool tdx_has_emulated_msr(u32 index)
 	case MSR_IA32_MC0_CTL2 ... MSR_IA32_MCx_CTL2(KVM_MAX_MCE_BANKS) - 1:
 		/* MSR_IA32_MCx_{CTL, STATUS, ADDR, MISC, CTL2} */
 	case MSR_KVM_POLL_CONTROL:
+	/*
+	 * Except for x2APIC registers that are virtualized by the CPU, which
+	 * KVM can't emulate as KVM doesn't have access to the virtual APIC
+	 * page, KVM emulates the same set of x2APIC registers for TDX versus
+	 * non-TDX guests.
+	 */
+	case X2APIC_MSR(APIC_ID):
+	case X2APIC_MSR(APIC_LVR):
+	case X2APIC_MSR(APIC_LDR):
+	case X2APIC_MSR(APIC_SPIV):
+	case X2APIC_MSR(APIC_ESR):
+	case X2APIC_MSR(APIC_LVTCMCI):
+	case X2APIC_MSR(APIC_ICR):
+	case X2APIC_MSR(APIC_LVTT):
+	case X2APIC_MSR(APIC_LVTTHMR):
+	case X2APIC_MSR(APIC_LVTPC):
+	case X2APIC_MSR(APIC_LVT0):
+	case X2APIC_MSR(APIC_LVT1):
+	case X2APIC_MSR(APIC_LVTERR):
+	case X2APIC_MSR(APIC_TMICT):
+	case X2APIC_MSR(APIC_TMCCT):
+	case X2APIC_MSR(APIC_TDCR):
 		return true;
-	case APIC_BASE_MSR ... APIC_BASE_MSR + 0xff:
-		/*
-		 * x2APIC registers that are virtualized by the CPU can't be
-		 * emulated, KVM doesn't have access to the virtual APIC page.
-		 */
-		switch (index) {
-		case X2APIC_MSR(APIC_TASKPRI):
-		case X2APIC_MSR(APIC_PROCPRI):
-		case X2APIC_MSR(APIC_EOI):
-		case X2APIC_MSR(APIC_ISR) ... X2APIC_MSR(APIC_ISR + APIC_ISR_NR):
-		case X2APIC_MSR(APIC_TMR) ... X2APIC_MSR(APIC_TMR + APIC_ISR_NR):
-		case X2APIC_MSR(APIC_IRR) ... X2APIC_MSR(APIC_IRR + APIC_ISR_NR):
-			return false;
-		default:
-			return true;
-		}
 	default:
 		return false;
 	}
@@ -2387,20 +2380,20 @@ static int __tdx_td_init(struct kvm *kvm, struct td_params *td_params,
 
 	ret = -ENOMEM;
 
-	tdr_page = alloc_page(GFP_KERNEL);
+	tdr_page = alloc_page(GFP_KERNEL_ACCOUNT);
 	if (!tdr_page)
 		goto free_hkid;
 
 	kvm_tdx->td.tdcs_nr_pages = tdx_sysinfo->td_ctrl.tdcs_base_size / PAGE_SIZE;
 	/* TDVPS = TDVPR(4K page) + TDCX(multiple 4K pages), -1 for TDVPR. */
 	kvm_tdx->td.tdcx_nr_pages = tdx_sysinfo->td_ctrl.tdvps_base_size / PAGE_SIZE - 1;
-	tdcs_pages = kzalloc_objs(*kvm_tdx->td.tdcs_pages,
-				  kvm_tdx->td.tdcs_nr_pages);
+	tdcs_pages = kzalloc_objs(*kvm_tdx->td.tdcs_pages, kvm_tdx->td.tdcs_nr_pages,
+				  GFP_KERNEL_ACCOUNT);
 	if (!tdcs_pages)
 		goto free_tdr;
 
 	for (i = 0; i < kvm_tdx->td.tdcs_nr_pages; i++) {
-		tdcs_pages[i] = alloc_page(GFP_KERNEL);
+		tdcs_pages[i] = alloc_page(GFP_KERNEL_ACCOUNT);
 		if (!tdcs_pages[i])
 			goto free_tdcs;
 	}
@@ -2875,7 +2868,7 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 	int ret, i;
 	u64 err;
 
-	page = alloc_page(GFP_KERNEL);
+	page = alloc_page(GFP_KERNEL_ACCOUNT);
 	if (!page)
 		return -ENOMEM;
 	tdx->vp.tdvpr_page = page;
@@ -2888,14 +2881,14 @@ static int tdx_td_vcpu_init(struct kvm_vcpu *vcpu, u64 vcpu_rcx)
 	tdx->vp.tdvpr_pa = page_to_phys(tdx->vp.tdvpr_page);
 
 	tdx->vp.tdcx_pages = kcalloc(kvm_tdx->td.tdcx_nr_pages, sizeof(*tdx->vp.tdcx_pages),
-			       	     GFP_KERNEL);
+				     GFP_KERNEL_ACCOUNT);
 	if (!tdx->vp.tdcx_pages) {
 		ret = -ENOMEM;
 		goto free_tdvpr;
 	}
 
 	for (i = 0; i < kvm_tdx->td.tdcx_nr_pages; i++) {
-		page = alloc_page(GFP_KERNEL);
+		page = alloc_page(GFP_KERNEL_ACCOUNT);
 		if (!page) {
 			ret = -ENOMEM;
 			goto free_tdcx;
diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h
index b5cd2ffb303e5..ac8323a68b163 100644
--- a/arch/x86/kvm/vmx/tdx.h
+++ b/arch/x86/kvm/vmx/tdx.h
@@ -3,7 +3,6 @@
 #define __KVM_X86_VMX_TDX_H
 
 #include "tdx_arch.h"
-#include "tdx_errno.h"
 
 #ifdef CONFIG_KVM_INTEL_TDX
 #include "common.h"
diff --git a/arch/x86/kvm/vmx/tdx_errno.h b/arch/x86/kvm/vmx/tdx_errno.h
deleted file mode 100644
index 6ff4672c41810..0000000000000
--- a/arch/x86/kvm/vmx/tdx_errno.h
+++ /dev/null
@@ -1,40 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* architectural status code for SEAMCALL */
-
-#ifndef __KVM_X86_TDX_ERRNO_H
-#define __KVM_X86_TDX_ERRNO_H
-
-#define TDX_SEAMCALL_STATUS_MASK		0xFFFFFFFF00000000ULL
-
-/*
- * TDX SEAMCALL Status Codes (returned in RAX)
- */
-#define TDX_NON_RECOVERABLE_VCPU		0x4000000100000000ULL
-#define TDX_NON_RECOVERABLE_TD			0x4000000200000000ULL
-#define TDX_NON_RECOVERABLE_TD_NON_ACCESSIBLE	0x6000000500000000ULL
-#define TDX_NON_RECOVERABLE_TD_WRONG_APIC_MODE	0x6000000700000000ULL
-#define TDX_INTERRUPTED_RESUMABLE		0x8000000300000000ULL
-#define TDX_OPERAND_INVALID			0xC000010000000000ULL
-#define TDX_OPERAND_BUSY			0x8000020000000000ULL
-#define TDX_PREVIOUS_TLB_EPOCH_BUSY		0x8000020100000000ULL
-#define TDX_PAGE_METADATA_INCORRECT		0xC000030000000000ULL
-#define TDX_VCPU_NOT_ASSOCIATED			0x8000070200000000ULL
-#define TDX_KEY_GENERATION_FAILED		0x8000080000000000ULL
-#define TDX_KEY_STATE_INCORRECT			0xC000081100000000ULL
-#define TDX_KEY_CONFIGURED			0x0000081500000000ULL
-#define TDX_NO_HKID_READY_TO_WBCACHE		0x0000082100000000ULL
-#define TDX_FLUSHVP_NOT_DONE			0x8000082400000000ULL
-#define TDX_EPT_WALK_FAILED			0xC0000B0000000000ULL
-#define TDX_EPT_ENTRY_STATE_INCORRECT		0xC0000B0D00000000ULL
-#define TDX_METADATA_FIELD_NOT_READABLE		0xC0000C0200000000ULL
-
-/*
- * TDX module operand ID, appears in 31:0 part of error code as
- * detail information
- */
-#define TDX_OPERAND_ID_RCX			0x01
-#define TDX_OPERAND_ID_TDR			0x80
-#define TDX_OPERAND_ID_SEPT			0x92
-#define TDX_OPERAND_ID_TD_EPOCH			0xa9
-
-#endif /* __KVM_X86_TDX_ERRNO_H */
diff --git a/arch/x86/kvm/vmx/vmenter.S b/arch/x86/kvm/vmx/vmenter.S
index 8a481dae9cae2..196a1f481881c 100644
--- a/arch/x86/kvm/vmx/vmenter.S
+++ b/arch/x86/kvm/vmx/vmenter.S
@@ -7,71 +7,38 @@
 #include <asm/percpu.h>
 #include <asm/segment.h>
 #include "kvm-asm-offsets.h"
-#include "run_flags.h"
+#include "vmenter.h"
 
 #define WORD_SIZE (BITS_PER_LONG / 8)
 
-#define VCPU_RAX	__VCPU_REGS_RAX * WORD_SIZE
-#define VCPU_RCX	__VCPU_REGS_RCX * WORD_SIZE
-#define VCPU_RDX	__VCPU_REGS_RDX * WORD_SIZE
-#define VCPU_RBX	__VCPU_REGS_RBX * WORD_SIZE
+#define VCPU_RAX	(VMX_vcpu_arch_regs + __VCPU_REGS_RAX * WORD_SIZE)
+#define VCPU_RCX	(VMX_vcpu_arch_regs + __VCPU_REGS_RCX * WORD_SIZE)
+#define VCPU_RDX	(VMX_vcpu_arch_regs + __VCPU_REGS_RDX * WORD_SIZE)
+#define VCPU_RBX	(VMX_vcpu_arch_regs + __VCPU_REGS_RBX * WORD_SIZE)
 /* Intentionally omit RSP as it's context switched by hardware */
-#define VCPU_RBP	__VCPU_REGS_RBP * WORD_SIZE
-#define VCPU_RSI	__VCPU_REGS_RSI * WORD_SIZE
-#define VCPU_RDI	__VCPU_REGS_RDI * WORD_SIZE
+#define VCPU_RBP	(VMX_vcpu_arch_regs + __VCPU_REGS_RBP * WORD_SIZE)
+#define VCPU_RSI	(VMX_vcpu_arch_regs + __VCPU_REGS_RSI * WORD_SIZE)
+#define VCPU_RDI	(VMX_vcpu_arch_regs + __VCPU_REGS_RDI * WORD_SIZE)
 
 #ifdef CONFIG_X86_64
-#define VCPU_R8		__VCPU_REGS_R8  * WORD_SIZE
-#define VCPU_R9		__VCPU_REGS_R9  * WORD_SIZE
-#define VCPU_R10	__VCPU_REGS_R10 * WORD_SIZE
-#define VCPU_R11	__VCPU_REGS_R11 * WORD_SIZE
-#define VCPU_R12	__VCPU_REGS_R12 * WORD_SIZE
-#define VCPU_R13	__VCPU_REGS_R13 * WORD_SIZE
-#define VCPU_R14	__VCPU_REGS_R14 * WORD_SIZE
-#define VCPU_R15	__VCPU_REGS_R15 * WORD_SIZE
+#define VCPU_R8		(VMX_vcpu_arch_regs + __VCPU_REGS_R8  * WORD_SIZE)
+#define VCPU_R9		(VMX_vcpu_arch_regs + __VCPU_REGS_R9  * WORD_SIZE)
+#define VCPU_R10	(VMX_vcpu_arch_regs + __VCPU_REGS_R10 * WORD_SIZE)
+#define VCPU_R11	(VMX_vcpu_arch_regs + __VCPU_REGS_R11 * WORD_SIZE)
+#define VCPU_R12	(VMX_vcpu_arch_regs + __VCPU_REGS_R12 * WORD_SIZE)
+#define VCPU_R13	(VMX_vcpu_arch_regs + __VCPU_REGS_R13 * WORD_SIZE)
+#define VCPU_R14	(VMX_vcpu_arch_regs + __VCPU_REGS_R14 * WORD_SIZE)
+#define VCPU_R15	(VMX_vcpu_arch_regs + __VCPU_REGS_R15 * WORD_SIZE)
 #endif
 
-.macro VMX_DO_EVENT_IRQOFF call_insn call_target
-	/*
-	 * Unconditionally create a stack frame, getting the correct RSP on the
-	 * stack (for x86-64) would take two instructions anyways, and RBP can
-	 * be used to restore RSP to make objtool happy (see below).
-	 */
-	push %_ASM_BP
-	mov %_ASM_SP, %_ASM_BP
-
-#ifdef CONFIG_X86_64
-	/*
-	 * Align RSP to a 16-byte boundary (to emulate CPU behavior) before
-	 * creating the synthetic interrupt stack frame for the IRQ/NMI.
-	 */
-	and  $-16, %rsp
-	push $__KERNEL_DS
-	push %rbp
-#endif
-	pushf
-	push $__KERNEL_CS
-	\call_insn \call_target
-
-	/*
-	 * "Restore" RSP from RBP, even though IRET has already unwound RSP to
-	 * the correct value.  objtool doesn't know the callee will IRET and,
-	 * without the explicit restore, thinks the stack is getting walloped.
-	 * Using an unwind hint is problematic due to x86-64's dynamic alignment.
-	 */
-	leave
-	RET
-.endm
-
 .section .noinstr.text, "ax"
 
 /**
  * __vmx_vcpu_run - Run a vCPU via a transition to VMX guest mode
  * @vmx:	struct vcpu_vmx *
- * @regs:	unsigned long * (to guest registers)
- * @flags:	VMX_RUN_VMRESUME:	use VMRESUME instead of VMLAUNCH
- *		VMX_RUN_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl
- *		VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO: vCPU can access host MMIO
+ * @flags:	KVM_ENTER_VMRESUME:	use VMRESUME instead of VMLAUNCH
+ *		KVM_ENTER_SAVE_SPEC_CTRL: save guest SPEC_CTRL into vmx->spec_ctrl
+ *		KVM_ENTER_CLEAR_CPU_BUFFERS_FOR_MMIO: vCPU can access host MMIO
  *
  * Returns:
  *	0 on VM-Exit, 1 on VM-Fail
@@ -94,76 +61,46 @@ SYM_FUNC_START(__vmx_vcpu_run)
 	push %_ASM_ARG1
 
 	/* Save @flags (used for VMLAUNCH vs. VMRESUME and mitigations). */
-	push %_ASM_ARG3
-
-	/*
-	 * Save @regs, _ASM_ARG2 may be modified by vmx_update_host_rsp() and
-	 * @regs is needed after VM-Exit to save the guest's register values.
-	 */
 	push %_ASM_ARG2
 
 	lea (%_ASM_SP), %_ASM_ARG2
 	call vmx_update_host_rsp
 
-	ALTERNATIVE "jmp .Lspec_ctrl_done", "", X86_FEATURE_MSR_SPEC_CTRL
+	/* Reload @vmx, _ASM_ARG1 may be modified by vmx_update_host_rsp().  */
+	mov WORD_SIZE(%_ASM_SP), %_ASM_DI
 
 	/*
-	 * SPEC_CTRL handling: if the guest's SPEC_CTRL value differs from the
-	 * host's, write the MSR.
-	 *
-	 * IMPORTANT: To avoid RSB underflow attacks and any other nastiness,
-	 * there must not be any returns or indirect branches between this code
-	 * and vmentry.
+	 * Unlike AMD there's no V_SPEC_CTRL here, so do not leave the body
+	 * out of line.  Clobbers RAX, RCX, RDX, RSI.
 	 */
-	mov 2*WORD_SIZE(%_ASM_SP), %_ASM_DI
-#ifdef CONFIG_X86_64
-	mov VMX_spec_ctrl(%rdi), %rdx
-	cmp PER_CPU_VAR(x86_spec_ctrl_current), %rdx
-	je .Lspec_ctrl_done
-	movl %edx, %eax
-	shr $32, %rdx
-#else
-	mov VMX_spec_ctrl(%edi), %eax
-	mov PER_CPU_VAR(x86_spec_ctrl_current), %ecx
-	xor %eax, %ecx
-	mov VMX_spec_ctrl + 4(%edi), %edx
-	mov PER_CPU_VAR(x86_spec_ctrl_current + 4), %edi
-	xor %edx, %edi
-	or %edi, %ecx
-	je .Lspec_ctrl_done
-#endif
-	mov $MSR_IA32_SPEC_CTRL, %ecx
-	wrmsr
-
-.Lspec_ctrl_done:
+	ALTERNATIVE "jmp .Lspec_ctrl_guest_done", "", X86_FEATURE_MSR_SPEC_CTRL
+	RESTORE_GUEST_SPEC_CTRL_BODY VMX_spec_ctrl(%_ASM_DI), .Lspec_ctrl_guest_done
+.Lspec_ctrl_guest_done:
 
 	/*
 	 * Since vmentry is serializing on affected CPUs, there's no need for
 	 * an LFENCE to stop speculation from skipping the wrmsr.
 	 */
 
-	/* Load @regs to RAX. */
-	mov (%_ASM_SP), %_ASM_AX
-
 	/* Load guest registers.  Don't clobber flags. */
-	mov VCPU_RCX(%_ASM_AX), %_ASM_CX
-	mov VCPU_RDX(%_ASM_AX), %_ASM_DX
-	mov VCPU_RBX(%_ASM_AX), %_ASM_BX
-	mov VCPU_RBP(%_ASM_AX), %_ASM_BP
-	mov VCPU_RSI(%_ASM_AX), %_ASM_SI
-	mov VCPU_RDI(%_ASM_AX), %_ASM_DI
+	mov VCPU_RAX(%_ASM_DI), %_ASM_AX
+	mov VCPU_RCX(%_ASM_DI), %_ASM_CX
+	mov VCPU_RDX(%_ASM_DI), %_ASM_DX
+	mov VCPU_RBX(%_ASM_DI), %_ASM_BX
+	mov VCPU_RBP(%_ASM_DI), %_ASM_BP
+	mov VCPU_RSI(%_ASM_DI), %_ASM_SI
 #ifdef CONFIG_X86_64
-	mov VCPU_R8 (%_ASM_AX),  %r8
-	mov VCPU_R9 (%_ASM_AX),  %r9
-	mov VCPU_R10(%_ASM_AX), %r10
-	mov VCPU_R11(%_ASM_AX), %r11
-	mov VCPU_R12(%_ASM_AX), %r12
-	mov VCPU_R13(%_ASM_AX), %r13
-	mov VCPU_R14(%_ASM_AX), %r14
-	mov VCPU_R15(%_ASM_AX), %r15
+	mov VCPU_R8 (%_ASM_DI),  %r8
+	mov VCPU_R9 (%_ASM_DI),  %r9
+	mov VCPU_R10(%_ASM_DI), %r10
+	mov VCPU_R11(%_ASM_DI), %r11
+	mov VCPU_R12(%_ASM_DI), %r12
+	mov VCPU_R13(%_ASM_DI), %r13
+	mov VCPU_R14(%_ASM_DI), %r14
+	mov VCPU_R15(%_ASM_DI), %r15
 #endif
-	/* Load guest RAX.  This kills the @regs pointer! */
-	mov VCPU_RAX(%_ASM_AX), %_ASM_AX
+	/* Load guest RDI.  This kills the @vmx pointer! */
+	mov VCPU_RDI(%_ASM_DI), %_ASM_DI
 
 	/*
 	 * Note, ALTERNATIVE_2 works in reverse order.  If CLEAR_CPU_BUF_VM is
@@ -172,7 +109,7 @@ SYM_FUNC_START(__vmx_vcpu_run)
 	 * do VERW.  Else, do nothing (no mitigations needed/enabled).
 	 */
 	ALTERNATIVE_2 "",									  \
-		      __stringify(testl $VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO, WORD_SIZE(%_ASM_SP); \
+		      __stringify(testl $KVM_ENTER_CLEAR_CPU_BUFFERS_FOR_MMIO, (%_ASM_SP);	  \
 				  jz .Lskip_mmio_verw;						  \
 				  VERW;								  \
 				  .Lskip_mmio_verw:),					  	  \
@@ -180,7 +117,7 @@ SYM_FUNC_START(__vmx_vcpu_run)
 		      __stringify(VERW), X86_FEATURE_CLEAR_CPU_BUF_VM
 
 	/* Check @flags to see if VMLAUNCH or VMRESUME is needed. */
-	testl $VMX_RUN_VMRESUME, WORD_SIZE(%_ASM_SP)
+	testl $KVM_ENTER_VMRESUME, (%_ASM_SP)
 	jz .Lvmlaunch
 
 	/*
@@ -212,38 +149,35 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL)
 	UNWIND_HINT_RESTORE
 	ENDBR
 
-	/* Temporarily save guest's RAX. */
-	push %_ASM_AX
+	/* Temporarily save guest's RDI. */
+	push %_ASM_DI
+
+	/* Reload @vmx to RDI. */
+	mov 2*WORD_SIZE(%_ASM_SP), %_ASM_DI
 
-	/* Reload @regs to RAX. */
-	mov WORD_SIZE(%_ASM_SP), %_ASM_AX
-
-	/* Save all guest registers, including RAX from the stack */
-	pop           VCPU_RAX(%_ASM_AX)
-	mov %_ASM_CX, VCPU_RCX(%_ASM_AX)
-	mov %_ASM_DX, VCPU_RDX(%_ASM_AX)
-	mov %_ASM_BX, VCPU_RBX(%_ASM_AX)
-	mov %_ASM_BP, VCPU_RBP(%_ASM_AX)
-	mov %_ASM_SI, VCPU_RSI(%_ASM_AX)
-	mov %_ASM_DI, VCPU_RDI(%_ASM_AX)
+	/* Save all guest registers, including RDI from the stack */
+	mov %_ASM_AX, VCPU_RAX(%_ASM_DI)
+	mov %_ASM_CX, VCPU_RCX(%_ASM_DI)
+	mov %_ASM_DX, VCPU_RDX(%_ASM_DI)
+	mov %_ASM_BX, VCPU_RBX(%_ASM_DI)
+	mov %_ASM_BP, VCPU_RBP(%_ASM_DI)
+	mov %_ASM_SI, VCPU_RSI(%_ASM_DI)
+	pop           VCPU_RDI(%_ASM_DI)
 #ifdef CONFIG_X86_64
-	mov %r8,  VCPU_R8 (%_ASM_AX)
-	mov %r9,  VCPU_R9 (%_ASM_AX)
-	mov %r10, VCPU_R10(%_ASM_AX)
-	mov %r11, VCPU_R11(%_ASM_AX)
-	mov %r12, VCPU_R12(%_ASM_AX)
-	mov %r13, VCPU_R13(%_ASM_AX)
-	mov %r14, VCPU_R14(%_ASM_AX)
-	mov %r15, VCPU_R15(%_ASM_AX)
+	mov %r8,  VCPU_R8 (%_ASM_DI)
+	mov %r9,  VCPU_R9 (%_ASM_DI)
+	mov %r10, VCPU_R10(%_ASM_DI)
+	mov %r11, VCPU_R11(%_ASM_DI)
+	mov %r12, VCPU_R12(%_ASM_DI)
+	mov %r13, VCPU_R13(%_ASM_DI)
+	mov %r14, VCPU_R14(%_ASM_DI)
+	mov %r15, VCPU_R15(%_ASM_DI)
 #endif
 
 	/* Clear return value to indicate VM-Exit (as opposed to VM-Fail). */
 	xor %ebx, %ebx
 
 .Lclear_regs:
-	/* Discard @regs.  The register is irrelevant, it just can't be RBX. */
-	pop %_ASM_AX
-
 	/*
 	 * Clear all general purpose registers except RSP and RBX to prevent
 	 * speculative use of the guest's values, even those that are reloaded
@@ -286,16 +220,32 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL)
 	FILL_RETURN_BUFFER %_ASM_CX, RSB_CLEAR_LOOPS, X86_FEATURE_RSB_VMEXIT,\
 			   X86_FEATURE_RSB_VMEXIT_LITE
 
-	pop %_ASM_ARG2	/* @flags */
-	pop %_ASM_ARG1	/* @vmx */
+	/* Clobbers RAX, RCX, RDX, RSI.  */
+	ALTERNATIVE "jmp .Lspec_ctrl_host_done", "", X86_FEATURE_MSR_SPEC_CTRL
+	mov WORD_SIZE(%_ASM_SP), %_ASM_DI
+	RESTORE_HOST_SPEC_CTRL_BODY VMX_spec_ctrl(%_ASM_DI), (%_ASM_SP), .Lspec_ctrl_host_done
+.Lspec_ctrl_host_done:
 
-	call vmx_spec_ctrl_restore_host
+	/*
+	 * Halt speculation past a conditional wrmsr.  Intel's eIBRS
+	 * guarantees that the guest cannot control the RSB "once IBRS is
+	 * set", but in the eIBRS case speculative execution past the 'je'
+	 * can go all the way to the RET below while MSR_IA32_SPEC_CTRL
+	 * still holds the guest value.
+	 */
+	ALTERNATIVE_2 "", "lfence", X86_FEATURE_MSR_SPEC_CTRL, \
+		"", X86_FEATURE_KERNEL_IBRS
 
 	CLEAR_BRANCH_HISTORY_VMEXIT
 
 	/* Put return value in AX */
 	mov %_ASM_BX, %_ASM_AX
 
+	/* Pop our saved arguments from the stack */
+	pop %_ASM_BX
+	pop %_ASM_BX
+
+	/* ... and then the callee-save registers */
 	pop %_ASM_BX
 #ifdef CONFIG_X86_64
 	pop %r12
@@ -320,10 +270,6 @@ SYM_INNER_LABEL_ALIGN(vmx_vmexit, SYM_L_GLOBAL)
 
 SYM_FUNC_END(__vmx_vcpu_run)
 
-SYM_FUNC_START(vmx_do_nmi_irqoff)
-	VMX_DO_EVENT_IRQOFF call asm_exc_nmi_kvm_vmx
-SYM_FUNC_END(vmx_do_nmi_irqoff)
-
 #ifndef CONFIG_CC_HAS_ASM_GOTO_OUTPUT
 
 /**
@@ -375,13 +321,3 @@ SYM_FUNC_START(vmread_error_trampoline)
 	RET
 SYM_FUNC_END(vmread_error_trampoline)
 #endif
-
-.section .text, "ax"
-
-#ifndef CONFIG_X86_FRED
-
-SYM_FUNC_START(vmx_do_interrupt_irqoff)
-	VMX_DO_EVENT_IRQOFF CALL_NOSPEC _ASM_ARG1
-SYM_FUNC_END(vmx_do_interrupt_irqoff)
-
-#endif
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 49feecb286b23..33acd27dff315 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -33,6 +33,7 @@
 #include <asm/asm.h>
 #include <asm/cpu.h>
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/debugreg.h>
 #include <asm/desc.h>
 #include <asm/fpu/api.h>
@@ -73,6 +74,7 @@
 #include "x86_ops.h"
 #include "smm.h"
 #include "vmx_onhyperv.h"
+#include "vmenter.h"
 #include "posted_intr.h"
 
 #include "mmu/spte.h"
@@ -117,6 +119,9 @@ module_param(emulate_invalid_guest_state, bool, 0444);
 static bool __read_mostly fasteoi = 1;
 module_param(fasteoi, bool, 0444);
 
+bool __read_mostly enable_mbec = 1;
+module_param_named(mbec, enable_mbec, bool, 0444);
+
 module_param(enable_apicv, bool, 0444);
 module_param(enable_ipiv, bool, 0444);
 
@@ -846,8 +851,8 @@ static bool vmx_segment_cache_test_set(struct vcpu_vmx *vmx, unsigned seg,
 	bool ret;
 	u32 mask = 1 << (seg * SEG_FIELD_NR + field);
 
-	if (!kvm_register_is_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS)) {
-		kvm_register_mark_available(&vmx->vcpu, VCPU_EXREG_SEGMENTS);
+	if (!kvm_register_is_available(&vmx->vcpu, VCPU_REG_SEGMENTS)) {
+		kvm_register_mark_available(&vmx->vcpu, VCPU_REG_SEGMENTS);
 		vmx->segment_cache.bitmask = 0;
 	}
 	ret = vmx->segment_cache.bitmask & mask;
@@ -967,12 +972,12 @@ static bool msr_write_intercepted(struct vcpu_vmx *vmx, u32 msr)
 	return vmx_test_msr_bitmap_write(vmx->loaded_vmcs->msr_bitmap, msr);
 }
 
-unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx)
+unsigned int __vmx_vcpu_enter_flags(struct vcpu_vmx *vmx)
 {
 	unsigned int flags = 0;
 
 	if (vmx->loaded_vmcs->launched)
-		flags |= VMX_RUN_VMRESUME;
+		flags |= KVM_ENTER_VMRESUME;
 
 	/*
 	 * If writes to the SPEC_CTRL MSR aren't intercepted, the guest is free
@@ -980,11 +985,11 @@ unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx)
 	 * it after vmexit and store it in vmx->spec_ctrl.
 	 */
 	if (!msr_write_intercepted(vmx, MSR_IA32_SPEC_CTRL))
-		flags |= VMX_RUN_SAVE_SPEC_CTRL;
+		flags |= KVM_ENTER_SAVE_SPEC_CTRL;
 
 	if (cpu_feature_enabled(X86_FEATURE_CLEAR_CPU_BUF_VM_MMIO) &&
 	    kvm_vcpu_can_access_host_mmio(&vmx->vcpu))
-		flags |= VMX_RUN_CLEAR_CPU_BUFFERS_FOR_MMIO;
+		flags |= KVM_ENTER_CLEAR_CPU_BUFFERS_FOR_MMIO;
 
 	return flags;
 }
@@ -1612,8 +1617,8 @@ unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu)
 	struct vcpu_vmx *vmx = to_vmx(vcpu);
 	unsigned long rflags, save_rflags;
 
-	if (!kvm_register_is_available(vcpu, VCPU_EXREG_RFLAGS)) {
-		kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
+	if (!kvm_register_is_available(vcpu, VCPU_REG_RFLAGS)) {
+		kvm_register_mark_available(vcpu, VCPU_REG_RFLAGS);
 		rflags = vmcs_readl(GUEST_RFLAGS);
 		if (vmx->rmode.vm86_active) {
 			rflags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
@@ -1636,7 +1641,7 @@ void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
 	 * if L1 runs L2 as a restricted guest.
 	 */
 	if (is_unrestricted_guest(vcpu)) {
-		kvm_register_mark_available(vcpu, VCPU_EXREG_RFLAGS);
+		kvm_register_mark_available(vcpu, VCPU_REG_RFLAGS);
 		vmx->rflags = rflags;
 		vmcs_writel(GUEST_RFLAGS, rflags);
 		return;
@@ -2607,20 +2612,20 @@ void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
 	case VCPU_REGS_RSP:
 		vcpu->arch.regs[VCPU_REGS_RSP] = vmcs_readl(GUEST_RSP);
 		break;
-	case VCPU_REGS_RIP:
-		vcpu->arch.regs[VCPU_REGS_RIP] = vmcs_readl(GUEST_RIP);
+	case VCPU_REG_RIP:
+		vcpu->arch.rip = vmcs_readl(GUEST_RIP);
 		break;
-	case VCPU_EXREG_PDPTR:
+	case VCPU_REG_PDPTR:
 		if (enable_ept)
 			ept_save_pdptrs(vcpu);
 		break;
-	case VCPU_EXREG_CR0:
+	case VCPU_REG_CR0:
 		guest_owned_bits = vcpu->arch.cr0_guest_owned_bits;
 
 		vcpu->arch.cr0 &= ~guest_owned_bits;
 		vcpu->arch.cr0 |= vmcs_readl(GUEST_CR0) & guest_owned_bits;
 		break;
-	case VCPU_EXREG_CR3:
+	case VCPU_REG_CR3:
 		/*
 		 * When intercepting CR3 loads, e.g. for shadowing paging, KVM's
 		 * CR3 is loaded into hardware, not the guest's CR3.
@@ -2628,7 +2633,7 @@ void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
 		if (!(exec_controls_get(to_vmx(vcpu)) & CPU_BASED_CR3_LOAD_EXITING))
 			vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
 		break;
-	case VCPU_EXREG_CR4:
+	case VCPU_REG_CR4:
 		guest_owned_bits = vcpu->arch.cr4_guest_owned_bits;
 
 		vcpu->arch.cr4 &= ~guest_owned_bits;
@@ -2776,6 +2781,7 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf,
 			return -EIO;
 
 		vmx_cap->ept = 0;
+		_cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_MODE_BASED_EPT_EXEC;
 		_cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_EPT_VIOLATION_VE;
 	}
 	if (!(_cpu_based_2nd_exec_control & SECONDARY_EXEC_ENABLE_VPID) &&
@@ -2789,6 +2795,16 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf,
 		vmx_cap->vpid = 0;
 	}
 
+	/*
+	 * Virtualizing MBEC requires advanced vmexit information in order to
+	 * distinguish supervisor and user accesses.  For simplicity and clarity
+	 * disable MBEC entirely if advanced vmexit information is not available,
+	 * this way mbec=1 in the kvm_intel module parameters implies availability
+	 * to nested guests as well.
+	 */
+	if (!(vmx_cap->ept & VMX_EPT_ADVANCED_VMEXIT_INFO_BIT))
+		_cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_MODE_BASED_EPT_EXEC;
+
 	if (!cpu_has_sgx())
 		_cpu_based_2nd_exec_control &= ~SECONDARY_EXEC_ENCLS_EXITING;
 
@@ -3353,7 +3369,7 @@ void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu)
 {
 	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
 
-	if (!kvm_register_is_dirty(vcpu, VCPU_EXREG_PDPTR))
+	if (!kvm_register_is_dirty(vcpu, VCPU_REG_PDPTR))
 		return;
 
 	if (is_pae_paging(vcpu)) {
@@ -3376,7 +3392,7 @@ void ept_save_pdptrs(struct kvm_vcpu *vcpu)
 	mmu->pdptrs[2] = vmcs_read64(GUEST_PDPTR2);
 	mmu->pdptrs[3] = vmcs_read64(GUEST_PDPTR3);
 
-	kvm_register_mark_available(vcpu, VCPU_EXREG_PDPTR);
+	kvm_register_mark_available(vcpu, VCPU_REG_PDPTR);
 }
 
 #define CR3_EXITING_BITS (CPU_BASED_CR3_LOAD_EXITING | \
@@ -3419,7 +3435,7 @@ void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 	vmcs_writel(CR0_READ_SHADOW, cr0);
 	vmcs_writel(GUEST_CR0, hw_cr0);
 	vcpu->arch.cr0 = cr0;
-	kvm_register_mark_available(vcpu, VCPU_EXREG_CR0);
+	kvm_register_mark_available(vcpu, VCPU_REG_CR0);
 
 #ifdef CONFIG_X86_64
 	if (vcpu->arch.efer & EFER_LME) {
@@ -3437,8 +3453,8 @@ void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 		 * (correctly) stop reading vmcs.GUEST_CR3 because it thinks
 		 * KVM's CR3 is installed.
 		 */
-		if (!kvm_register_is_available(vcpu, VCPU_EXREG_CR3))
-			vmx_cache_reg(vcpu, VCPU_EXREG_CR3);
+		if (!kvm_register_is_available(vcpu, VCPU_REG_CR3))
+			vmx_cache_reg(vcpu, VCPU_REG_CR3);
 
 		/*
 		 * When running with EPT but not unrestricted guest, KVM must
@@ -3475,7 +3491,7 @@ void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
 		 * GUEST_CR3 is still vmx->ept_identity_map_addr if EPT + !URG.
 		 */
 		if (!(old_cr0_pg & X86_CR0_PG) && (cr0 & X86_CR0_PG))
-			kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
+			kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
 	}
 
 	/* depends on vcpu->arch.cr0 to be set to a new value */
@@ -3504,7 +3520,7 @@ void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level)
 
 		if (!enable_unrestricted_guest && !is_paging(vcpu))
 			guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr;
-		else if (kvm_register_is_dirty(vcpu, VCPU_EXREG_CR3))
+		else if (kvm_register_is_dirty(vcpu, VCPU_REG_CR3))
 			guest_cr3 = vcpu->arch.cr3;
 		else /* vmcs.GUEST_CR3 is already up-to-date. */
 			update_guest_cr3 = false;
@@ -3564,7 +3580,7 @@ void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
 	}
 
 	vcpu->arch.cr4 = cr4;
-	kvm_register_mark_available(vcpu, VCPU_EXREG_CR4);
+	kvm_register_mark_available(vcpu, VCPU_REG_CR4);
 
 	if (!enable_unrestricted_guest) {
 		if (enable_ept) {
@@ -4745,6 +4761,9 @@ static u32 vmx_secondary_exec_control(struct vcpu_vmx *vmx)
 	 */
 	exec_control &= ~SECONDARY_EXEC_ENABLE_VMFUNC;
 
+	if (!enable_mbec)
+		exec_control &= ~SECONDARY_EXEC_MODE_BASED_EPT_EXEC;
+
 	/* SECONDARY_EXEC_DESC is enabled/disabled on writes to CR4.UMIP,
 	 * in vmx_set_cr4.  */
 	exec_control &= ~SECONDARY_EXEC_DESC;
@@ -5031,7 +5050,7 @@ void vmx_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 	vmcs_write32(GUEST_IDTR_LIMIT, 0xffff);
 
 	vmx_segment_cache_clear(vmx);
-	kvm_register_mark_available(vcpu, VCPU_EXREG_SEGMENTS);
+	kvm_register_mark_available(vcpu, VCPU_REG_SEGMENTS);
 
 	vmcs_write32(GUEST_ACTIVITY_STATE, GUEST_ACTIVITY_ACTIVE);
 	vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0);
@@ -6697,6 +6716,9 @@ static int __vmx_handle_exit(struct kvm_vcpu *vcpu, fastpath_t exit_fastpath)
 	if (enable_pml && !is_guest_mode(vcpu))
 		vmx_flush_pml_buffer(vcpu);
 
+	if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE))
+		return 0;
+
 	/*
 	 * KVM should never reach this point with a pending nested VM-Enter.
 	 * More specifically, short-circuiting VM-Entry to emulate L2 due to
@@ -7117,9 +7139,6 @@ void vmx_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
 	vmcs_write64(EOI_EXIT_BITMAP3, eoi_exit_bitmap[3]);
 }
 
-void vmx_do_interrupt_irqoff(unsigned long entry);
-void vmx_do_nmi_irqoff(void);
-
 static void handle_nm_fault_irqoff(struct kvm_vcpu *vcpu)
 {
 	/*
@@ -7161,17 +7180,8 @@ static void handle_external_interrupt_irqoff(struct kvm_vcpu *vcpu,
 	    "unexpected VM-Exit interrupt info: 0x%x", intr_info))
 		return;
 
-	/*
-	 * Invoke the kernel's IRQ handler for the vector.  Use the FRED path
-	 * when it's available even if FRED isn't fully enabled, e.g. even if
-	 * FRED isn't supported in hardware, in order to avoid the indirect
-	 * CALL in the non-FRED path.
-	 */
 	kvm_before_interrupt(vcpu, KVM_HANDLING_IRQ);
-	if (IS_ENABLED(CONFIG_X86_FRED))
-		fred_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
-	else
-		vmx_do_interrupt_irqoff(gate_offset((gate_desc *)host_idt_base + vector));
+	x86_entry_from_kvm(EVENT_TYPE_EXTINT, vector);
 	kvm_after_interrupt(vcpu);
 
 	vcpu->arch.at_instruction_boundary = true;
@@ -7421,31 +7431,6 @@ void noinstr vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp)
 	}
 }
 
-void noinstr vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx,
-					unsigned int flags)
-{
-	u64 hostval = this_cpu_read(x86_spec_ctrl_current);
-
-	if (!cpu_feature_enabled(X86_FEATURE_MSR_SPEC_CTRL))
-		return;
-
-	if (flags & VMX_RUN_SAVE_SPEC_CTRL)
-		vmx->spec_ctrl = native_rdmsrq(MSR_IA32_SPEC_CTRL);
-
-	/*
-	 * If the guest/host SPEC_CTRL values differ, restore the host value.
-	 *
-	 * For legacy IBRS, the IBRS bit always needs to be written after
-	 * transitioning from a less privileged predictor mode, regardless of
-	 * whether the guest/host values differ.
-	 */
-	if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) ||
-	    vmx->spec_ctrl != hostval)
-		native_wrmsrq(MSR_IA32_SPEC_CTRL, hostval);
-
-	barrier_nospec();
-}
-
 static fastpath_t vmx_exit_handlers_fastpath(struct kvm_vcpu *vcpu,
 					     bool force_immediate_exit)
 {
@@ -7481,10 +7466,7 @@ noinstr void vmx_handle_nmi(struct kvm_vcpu *vcpu)
 		return;
 
 	kvm_before_interrupt(vcpu, KVM_HANDLING_NMI);
-	if (cpu_feature_enabled(X86_FEATURE_FRED))
-		fred_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
-	else
-		vmx_do_nmi_irqoff();
+	x86_entry_from_kvm(EVENT_TYPE_NMI, NMI_VECTOR);
 	kvm_after_interrupt(vcpu);
 }
 
@@ -7502,11 +7484,10 @@ static noinstr void vmx_vcpu_enter_exit(struct kvm_vcpu *vcpu,
 	if (vcpu->arch.cr2 != native_read_cr2())
 		native_write_cr2(vcpu->arch.cr2);
 
-	vmx->fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
-				   flags);
+	vmx->fail = __vmx_vcpu_run(vmx, flags);
 
 	vcpu->arch.cr2 = native_read_cr2();
-	vcpu->arch.regs_avail &= ~VMX_REGS_LAZY_LOAD_SET;
+	kvm_clear_available_registers(vcpu, VMX_REGS_LAZY_LOAD_SET);
 
 	vmx->idt_vectoring_info = 0;
 
@@ -7548,9 +7529,9 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 
 		vmx->vt.exit_reason.full = EXIT_REASON_INVALID_STATE;
 		vmx->vt.exit_reason.failed_vmentry = 1;
-		kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1);
+		kvm_register_mark_available(vcpu, VCPU_REG_EXIT_INFO_1);
 		vmx->vt.exit_qualification = ENTRY_FAIL_DEFAULT;
-		kvm_register_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2);
+		kvm_register_mark_available(vcpu, VCPU_REG_EXIT_INFO_2);
 		vmx->vt.exit_intr_info = 0;
 		return EXIT_FASTPATH_NONE;
 	}
@@ -7570,9 +7551,9 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 
 	if (kvm_register_is_dirty(vcpu, VCPU_REGS_RSP))
 		vmcs_writel(GUEST_RSP, vcpu->arch.regs[VCPU_REGS_RSP]);
-	if (kvm_register_is_dirty(vcpu, VCPU_REGS_RIP))
-		vmcs_writel(GUEST_RIP, vcpu->arch.regs[VCPU_REGS_RIP]);
-	vcpu->arch.regs_dirty = 0;
+	if (kvm_register_is_dirty(vcpu, VCPU_REG_RIP))
+		vmcs_writel(GUEST_RIP, vcpu->arch.rip);
+	kvm_reset_dirty_registers(vcpu);
 
 	if (run_flags & KVM_RUN_LOAD_GUEST_DR6)
 		set_debugreg(vcpu->arch.dr6, 6);
@@ -7621,7 +7602,7 @@ fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu, u64 run_flags)
 	kvm_wait_lapic_expire(vcpu);
 
 	/* The actual VMENTER/EXIT is in the .noinstr.text section. */
-	vmx_vcpu_enter_exit(vcpu, __vmx_vcpu_run_flags(vmx));
+	vmx_vcpu_enter_exit(vcpu, __vmx_vcpu_enter_flags(vmx));
 
 	/* All fields are clean at this point */
 	if (kvm_is_using_evmcs()) {
@@ -8659,6 +8640,8 @@ __init int vmx_hardware_setup(void)
 
 	if (!cpu_has_vmx_ept_ad_bits() || !enable_ept)
 		enable_ept_ad_bits = 0;
+	if (!cpu_has_ept_mbec() || !enable_ept)
+		enable_mbec = 0;
 
 	if (!cpu_has_vmx_unrestricted_guest() || !enable_ept)
 		enable_unrestricted_guest = 0;
@@ -8720,8 +8703,7 @@ __init int vmx_hardware_setup(void)
 	set_bit(0, vmx_vpid_bitmap); /* 0 is reserved for host */
 
 	if (enable_ept)
-		kvm_mmu_set_ept_masks(enable_ept_ad_bits,
-				      cpu_has_vmx_ept_execute_only());
+		kvm_mmu_set_ept_masks(enable_ept_ad_bits);
 	else
 		vt_x86_ops.get_mt_mask = NULL;
 
diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index db84e8001da58..daedf663c0a9c 100644
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -15,7 +15,6 @@
 #include "vmcs.h"
 #include "vmx_ops.h"
 #include "../cpuid.h"
-#include "run_flags.h"
 #include "../mmu.h"
 #include "common.h"
 
@@ -317,7 +316,7 @@ static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vt *vt = to_vt(vcpu);
 
-	if (!kvm_register_test_and_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1) &&
+	if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_1) &&
 	    !WARN_ON_ONCE(is_td_vcpu(vcpu)))
 		vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
 
@@ -328,7 +327,7 @@ static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_vt *vt = to_vt(vcpu);
 
-	if (!kvm_register_test_and_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2) &&
+	if (!kvm_register_test_and_mark_available(vcpu, VCPU_REG_EXIT_INFO_2) &&
 	    !WARN_ON_ONCE(is_td_vcpu(vcpu)))
 		vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
 
@@ -368,10 +367,8 @@ void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu);
 struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr);
 void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu);
 void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp);
-void vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, unsigned int flags);
-unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx);
-bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs,
-		    unsigned int flags);
+unsigned int __vmx_vcpu_enter_flags(struct vcpu_vmx *vmx);
+bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned int flags);
 void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu);
 
 void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool set);
@@ -567,6 +564,7 @@ static inline u8 vmx_get_rvi(void)
 	 SECONDARY_EXEC_ENABLE_VMFUNC |					\
 	 SECONDARY_EXEC_BUS_LOCK_DETECTION |				\
 	 SECONDARY_EXEC_NOTIFY_VM_EXITING |				\
+	 SECONDARY_EXEC_MODE_BASED_EPT_EXEC |				\
 	 SECONDARY_EXEC_ENCLS_EXITING |					\
 	 SECONDARY_EXEC_EPT_VIOLATION_VE)
 
@@ -620,16 +618,16 @@ BUILD_CONTROLS_SHADOW(tertiary_exec, TERTIARY_VM_EXEC_CONTROL, 64)
  * cache on demand.  Other registers not listed here are synced to
  * the cache immediately after VM-Exit.
  */
-#define VMX_REGS_LAZY_LOAD_SET	((1 << VCPU_REGS_RIP) |         \
-				(1 << VCPU_REGS_RSP) |          \
-				(1 << VCPU_EXREG_RFLAGS) |      \
-				(1 << VCPU_EXREG_PDPTR) |       \
-				(1 << VCPU_EXREG_SEGMENTS) |    \
-				(1 << VCPU_EXREG_CR0) |         \
-				(1 << VCPU_EXREG_CR3) |         \
-				(1 << VCPU_EXREG_CR4) |         \
-				(1 << VCPU_EXREG_EXIT_INFO_1) | \
-				(1 << VCPU_EXREG_EXIT_INFO_2))
+#define VMX_REGS_LAZY_LOAD_SET	(BIT(VCPU_REGS_RSP) |		\
+				 BIT(VCPU_REG_RIP) |		\
+				 BIT(VCPU_REG_RFLAGS) |		\
+				 BIT(VCPU_REG_PDPTR) |		\
+				 BIT(VCPU_REG_SEGMENTS) |	\
+				 BIT(VCPU_REG_CR0) |		\
+				 BIT(VCPU_REG_CR3) |		\
+				 BIT(VCPU_REG_CR4) |		\
+				 BIT(VCPU_REG_EXIT_INFO_1) |	\
+				 BIT(VCPU_REG_EXIT_INFO_2))
 
 static inline unsigned long vmx_l1_guest_owned_cr0_bits(void)
 {
diff --git a/arch/x86/kvm/vmx/x86_ops.h b/arch/x86/kvm/vmx/x86_ops.h
index d09abeac2b56a..4098580742462 100644
--- a/arch/x86/kvm/vmx/x86_ops.h
+++ b/arch/x86/kvm/vmx/x86_ops.h
@@ -4,6 +4,7 @@
 
 #include <linux/kvm_host.h>
 
+#include "capabilities.h"
 #include "x86.h"
 
 __init int vmx_hardware_setup(void);
@@ -104,6 +105,11 @@ int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr);
 int vmx_set_identity_map_addr(struct kvm *kvm, u64 ident_addr);
 u8 vmx_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio);
 
+static inline bool vmx_tdp_has_smep(struct kvm *kvm)
+{
+	return enable_mbec;
+}
+
 void vmx_get_exit_info(struct kvm_vcpu *vcpu, u32 *reason,
 		       u64 *info1, u64 *info2, u32 *intr_info, u32 *error_code);
 void vmx_get_entry_info(struct kvm_vcpu *vcpu, u32 *intr_info, u32 *error_code);
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 0a1b63c63d1a9..209eae67ab186 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -1043,11 +1043,16 @@ bool kvm_require_dr(struct kvm_vcpu *vcpu, int dr)
 }
 EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_require_dr);
 
-static bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
+static bool __kvm_pv_async_pf_enabled(u64 data)
 {
 	u64 mask = KVM_ASYNC_PF_ENABLED | KVM_ASYNC_PF_DELIVERY_AS_INT;
 
-	return (vcpu->arch.apf.msr_en_val & mask) == mask;
+	return (data & mask) == mask;
+}
+
+static bool kvm_pv_async_pf_enabled(struct kvm_vcpu *vcpu)
+{
+	return __kvm_pv_async_pf_enabled(vcpu->arch.apf.msr_en_val);
 }
 
 static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu)
@@ -1072,7 +1077,8 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 	 * to an L1 GPA.
 	 */
 	real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(pdpt_gfn),
-				     PFERR_USER_MASK | PFERR_WRITE_MASK, NULL);
+				     PFERR_USER_MASK | PFERR_WRITE_MASK |
+				     PFERR_GUEST_PAGE_MASK, NULL, 0);
 	if (real_gpa == INVALID_GPA)
 		return 0;
 
@@ -1090,14 +1096,14 @@ int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3)
 	}
 
 	/*
-	 * Marking VCPU_EXREG_PDPTR dirty doesn't work for !tdp_enabled.
+	 * Marking VCPU_REG_PDPTR dirty doesn't work for !tdp_enabled.
 	 * Shadow page roots need to be reconstructed instead.
 	 */
 	if (!tdp_enabled && memcmp(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)))
 		kvm_mmu_free_roots(vcpu->kvm, mmu, KVM_MMU_ROOT_CURRENT);
 
 	memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs));
-	kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
+	kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR);
 	kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu);
 	vcpu->arch.pdptrs_from_userspace = false;
 
@@ -1478,7 +1484,7 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3)
 		kvm_mmu_new_pgd(vcpu, cr3);
 
 	vcpu->arch.cr3 = cr3;
-	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
+	kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
 	/* Do not call post_set_cr3, we do not get here for confidential guests.  */
 
 handle_tlb_flush:
@@ -3647,23 +3653,19 @@ static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
 	if (!lapic_in_kernel(vcpu))
 		return data ? 1 : 0;
 
+	if (__kvm_pv_async_pf_enabled(data) &&
+	    kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
+				      sizeof(u64)))
+		return 1;
+
 	vcpu->arch.apf.msr_en_val = data;
 
-	if (!kvm_pv_async_pf_enabled(vcpu)) {
+	if (__kvm_pv_async_pf_enabled(data)) {
+		kvm_async_pf_wakeup_all(vcpu);
+	} else {
 		kvm_clear_async_pf_completion_queue(vcpu);
 		kvm_async_pf_hash_reset(vcpu);
-		return 0;
 	}
-
-	if (kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.apf.data, gpa,
-					sizeof(u64)))
-		return 1;
-
-	vcpu->arch.apf.send_always = (data & KVM_ASYNC_PF_SEND_ALWAYS);
-	vcpu->arch.apf.delivery_as_pf_vmexit = data & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
-
-	kvm_async_pf_wakeup_all(vcpu);
-
 	return 0;
 }
 
@@ -5227,8 +5229,13 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 		 * On a host with synchronized TSC, there is no need to update
 		 * kvmclock on vcpu->cpu migration
 		 */
-		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1)
-			kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
+		if (!vcpu->kvm->arch.use_master_clock || vcpu->cpu == -1) {
+			if (__ratelimit(&vcpu->kvm->arch.kvmclock_update_rs))
+				kvm_make_request(KVM_REQ_GLOBAL_CLOCK_UPDATE, vcpu);
+			else
+				kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
+		}
+
 		if (vcpu->cpu != cpu)
 			kvm_make_request(KVM_REQ_MIGRATE_TIMER, vcpu);
 		vcpu->cpu = cpu;
@@ -7847,21 +7854,6 @@ void kvm_get_segment(struct kvm_vcpu *vcpu,
 	kvm_x86_call(get_segment)(vcpu, var, seg);
 }
 
-gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u64 access,
-			   struct x86_exception *exception)
-{
-	struct kvm_mmu *mmu = vcpu->arch.mmu;
-	gpa_t t_gpa;
-
-	BUG_ON(!mmu_is_nested(vcpu));
-
-	/* NPT walks are always user-walks */
-	access |= PFERR_USER_MASK;
-	t_gpa  = mmu->gva_to_gpa(vcpu, mmu, gpa, access, exception);
-
-	return t_gpa;
-}
-
 gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
 			      struct x86_exception *exception)
 {
@@ -11602,9 +11594,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu)
 	if (vcpu->arch.apic_attention)
 		kvm_lapic_sync_from_vapic(vcpu);
 
-	if (unlikely(exit_fastpath == EXIT_FASTPATH_EXIT_USERSPACE))
-		return 0;
-
 	r = kvm_x86_call(handle_exit)(vcpu, exit_fastpath);
 	return r;
 
@@ -12473,7 +12462,7 @@ static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs,
 	vcpu->arch.cr2 = sregs->cr2;
 	*mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3;
 	vcpu->arch.cr3 = sregs->cr3;
-	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
+	kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
 	kvm_x86_call(post_set_cr3)(vcpu, sregs->cr3);
 
 	kvm_set_cr8(vcpu, sregs->cr8);
@@ -12566,7 +12555,7 @@ static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2)
 		for (i = 0; i < 4 ; i++)
 			kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]);
 
-		kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR);
+		kvm_register_mark_dirty(vcpu, VCPU_REG_PDPTR);
 		mmu_reset_needed = 1;
 		vcpu->arch.pdptrs_from_userspace = true;
 	}
@@ -12836,8 +12825,8 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 	int r;
 
 	vcpu->arch.last_vmentry_cpu = -1;
-	vcpu->arch.regs_avail = ~0;
-	vcpu->arch.regs_dirty = ~0;
+	bitmap_fill(vcpu->arch.regs_avail, NR_VCPU_TOTAL_REGS);
+	bitmap_fill(vcpu->arch.regs_dirty, NR_VCPU_TOTAL_REGS);
 
 	kvm_gpc_init(&vcpu->arch.pv_time, vcpu->kvm);
 
@@ -13111,7 +13100,7 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 	kvm_rip_write(vcpu, 0xfff0);
 
 	vcpu->arch.cr3 = 0;
-	kvm_register_mark_dirty(vcpu, VCPU_EXREG_CR3);
+	kvm_register_mark_dirty(vcpu, VCPU_REG_CR3);
 
 	/*
 	 * CR0.CD/NW are set on RESET, preserved on INIT.  Note, some versions
@@ -13366,6 +13355,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 	raw_spin_lock_init(&kvm->arch.tsc_write_lock);
 	mutex_init(&kvm->arch.apic_map_lock);
 	seqcount_raw_spinlock_init(&kvm->arch.pvclock_sc, &kvm->arch.tsc_write_lock);
+	ratelimit_state_init(&kvm->arch.kvmclock_update_rs, HZ, 10);
+	ratelimit_set_flags(&kvm->arch.kvmclock_update_rs, RATELIMIT_MSG_ON_RELEASE);
 	kvm->arch.kvmclock_offset = -get_kvmclock_base_ns();
 
 	raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags);
@@ -14020,7 +14011,7 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
 	if (!kvm_pv_async_pf_enabled(vcpu))
 		return false;
 
-	if (!vcpu->arch.apf.send_always &&
+	if (!(vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_SEND_ALWAYS) &&
 	    (vcpu->arch.guest_state_protected || !kvm_x86_call(get_cpl)(vcpu)))
 		return false;
 
@@ -14029,7 +14020,7 @@ static bool kvm_can_deliver_async_pf(struct kvm_vcpu *vcpu)
 		 * L1 needs to opt into the special #PF vmexits that are
 		 * used to deliver async page faults.
 		 */
-		return vcpu->arch.apf.delivery_as_pf_vmexit;
+		return vcpu->arch.apf.msr_en_val & KVM_ASYNC_PF_DELIVERY_AS_PF_VMEXIT;
 	} else {
 		/*
 		 * Play it safe in case the guest temporarily disables paging.
@@ -14323,7 +14314,7 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
 		 * the RAP (Return Address Predicator).
 		 */
 		if (guest_cpu_cap_has(vcpu, X86_FEATURE_ERAPS))
-			kvm_register_is_dirty(vcpu, VCPU_EXREG_ERAPS);
+			kvm_register_is_dirty(vcpu, VCPU_REG_ERAPS);
 
 		kvm_invalidate_pcid(vcpu, operand.pcid);
 		return kvm_skip_emulated_instruction(vcpu);
@@ -14339,7 +14330,7 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva)
 		fallthrough;
 	case INVPCID_TYPE_ALL_INCL_GLOBAL:
 		/*
-		 * Don't bother marking VCPU_EXREG_ERAPS dirty, SVM will take
+		 * Don't bother marking VCPU_REG_ERAPS dirty, SVM will take
 		 * care of doing so when emulating the full guest TLB flush
 		 * (the RAP is cleared on all implicit TLB flushes).
 		 */
diff --git a/arch/x86/math-emu/Makefile b/arch/x86/math-emu/Makefile
deleted file mode 100644
index 02211fc6f2ecf..0000000000000
--- a/arch/x86/math-emu/Makefile
+++ /dev/null
@@ -1,30 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-#               Makefile for wm-FPU-emu
-#
-
-#DEBUG	= -DDEBUGGING
-DEBUG	=
-PARANOID = -DPARANOID
-ccflags-y += $(PARANOID) $(DEBUG) -fno-builtin $(MATH_EMULATION)
-asflags-y += $(PARANOID)
-
-# From 'C' language sources:
-C_OBJS =fpu_entry.o errors.o \
-	fpu_arith.o fpu_aux.o fpu_etc.o fpu_tags.o fpu_trig.o \
-	load_store.o get_address.o \
-	poly_atan.o poly_l2.o poly_2xm1.o poly_sin.o poly_tan.o \
-	reg_add_sub.o reg_compare.o reg_constant.o reg_convert.o \
-	reg_ld_str.o reg_divide.o reg_mul.o
-
-# From 80x86 assembler sources:
-A_OBJS =reg_u_add.o reg_u_div.o reg_u_mul.o reg_u_sub.o \
-	div_small.o reg_norm.o reg_round.o \
-	wm_shrx.o wm_sqrt.o \
-	div_Xsig.o polynom_Xsig.o round_Xsig.o \
-	shr_Xsig.o mul_Xsig.o
-
-obj-y =$(C_OBJS) $(A_OBJS)
-
-proto:
-	cproto -e -DMAKING_PROTO *.c >fpu_proto.h
diff --git a/arch/x86/math-emu/README b/arch/x86/math-emu/README
deleted file mode 100644
index e6235491d6eb0..0000000000000
--- a/arch/x86/math-emu/README
+++ /dev/null
@@ -1,427 +0,0 @@
- +---------------------------------------------------------------------------+
- |  wm-FPU-emu   an FPU emulator for 80386 and 80486SX microprocessors.      |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1995,1996,1997,1999                          |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@melbpc.org.au              |
- |                                                                           |
- |    This program is free software; you can redistribute it and/or modify   |
- |    it under the terms of the GNU General Public License version 2 as      |
- |    published by the Free Software Foundation.                             |
- |                                                                           |
- |    This program is distributed in the hope that it will be useful,        |
- |    but WITHOUT ANY WARRANTY; without even the implied warranty of         |
- |    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          |
- |    GNU General Public License for more details.                           |
- |                                                                           |
- |    You should have received a copy of the GNU General Public License      |
- |    along with this program; if not, write to the Free Software            |
- |    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.              |
- |                                                                           |
- +---------------------------------------------------------------------------+
-
-
-
-wm-FPU-emu is an FPU emulator for Linux. It is derived from wm-emu387
-which was my 80387 emulator for early versions of djgpp (gcc under
-msdos); wm-emu387 was in turn based upon emu387 which was written by
-DJ Delorie for djgpp.  The interface to the Linux kernel is based upon
-the original Linux math emulator by Linus Torvalds.
-
-My target FPU for wm-FPU-emu is that described in the Intel486
-Programmer's Reference Manual (1992 edition). Unfortunately, numerous
-facets of the functioning of the FPU are not well covered in the
-Reference Manual. The information in the manual has been supplemented
-with measurements on real 80486's. Unfortunately, it is simply not
-possible to be sure that all of the peculiarities of the 80486 have
-been discovered, so there is always likely to be obscure differences
-in the detailed behaviour of the emulator and a real 80486.
-
-wm-FPU-emu does not implement all of the behaviour of the 80486 FPU,
-but is very close.  See "Limitations" later in this file for a list of
-some differences.
-
-Please report bugs, etc to me at:
-       billm@melbpc.org.au
-or     b.metzenthen@medoto.unimelb.edu.au
-
-For more information on the emulator and on floating point topics, see
-my web pages, currently at  http://www.suburbia.net/~billm/
-
-
---Bill Metzenthen
-  December 1999
-
-
------------------------ Internals of wm-FPU-emu -----------------------
-
-Numeric algorithms:
-(1) Add, subtract, and multiply. Nothing remarkable in these.
-(2) Divide has been tuned to get reasonable performance. The algorithm
-    is not the obvious one which most people seem to use, but is designed
-    to take advantage of the characteristics of the 80386. I expect that
-    it has been invented many times before I discovered it, but I have not
-    seen it. It is based upon one of those ideas which one carries around
-    for years without ever bothering to check it out.
-(3) The sqrt function has been tuned to get good performance. It is based
-    upon Newton's classic method. Performance was improved by capitalizing
-    upon the properties of Newton's method, and the code is once again
-    structured taking account of the 80386 characteristics.
-(4) The trig, log, and exp functions are based in each case upon quasi-
-    "optimal" polynomial approximations. My definition of "optimal" was
-    based upon getting good accuracy with reasonable speed.
-(5) The argument reducing code for the trig function effectively uses
-    a value of pi which is accurate to more than 128 bits. As a consequence,
-    the reduced argument is accurate to more than 64 bits for arguments up
-    to a few pi, and accurate to more than 64 bits for most arguments,
-    even for arguments approaching 2^63. This is far superior to an
-    80486, which uses a value of pi which is accurate to 66 bits.
-
-The code of the emulator is complicated slightly by the need to
-account for a limited form of re-entrancy. Normally, the emulator will
-emulate each FPU instruction to completion without interruption.
-However, it may happen that when the emulator is accessing the user
-memory space, swapping may be needed. In this case the emulator may be
-temporarily suspended while disk i/o takes place. During this time
-another process may use the emulator, thereby perhaps changing static
-variables. The code which accesses user memory is confined to five
-files:
-    fpu_entry.c
-    reg_ld_str.c
-    load_store.c
-    get_address.c
-    errors.c
-As from version 1.12 of the emulator, no static variables are used
-(apart from those in the kernel's per-process tables). The emulator is
-therefore now fully re-entrant, rather than having just the restricted
-form of re-entrancy which is required by the Linux kernel.
-
------------------------ Limitations of wm-FPU-emu -----------------------
-
-There are a number of differences between the current wm-FPU-emu
-(version 2.01) and the 80486 FPU (apart from bugs).  The differences
-are fewer than those which applied to the 1.xx series of the emulator.
-Some of the more important differences are listed below:
-
-The Roundup flag does not have much meaning for the transcendental
-functions and its 80486 value with these functions is likely to differ
-from its emulator value.
-
-In a few rare cases the Underflow flag obtained with the emulator will
-be different from that obtained with an 80486. This occurs when the
-following conditions apply simultaneously:
-(a) the operands have a higher precision than the current setting of the
-    precision control (PC) flags.
-(b) the underflow exception is masked.
-(c) the magnitude of the exact result (before rounding) is less than 2^-16382.
-(d) the magnitude of the final result (after rounding) is exactly 2^-16382.
-(e) the magnitude of the exact result would be exactly 2^-16382 if the
-    operands were rounded to the current precision before the arithmetic
-    operation was performed.
-If all of these apply, the emulator will set the Underflow flag but a real
-80486 will not.
-
-NOTE: Certain formats of Extended Real are UNSUPPORTED. They are
-unsupported by the 80486. They are the Pseudo-NaNs, Pseudoinfinities,
-and Unnormals. None of these will be generated by an 80486 or by the
-emulator. Do not use them. The emulator treats them differently in
-detail from the way an 80486 does.
-
-Self modifying code can cause the emulator to fail. An example of such
-code is:
-          movl %esp,[%ebx]
-	  fld1
-The FPU instruction may be (usually will be) loaded into the pre-fetch
-queue of the CPU before the mov instruction is executed. If the
-destination of the 'movl' overlaps the FPU instruction then the bytes
-in the prefetch queue and memory will be inconsistent when the FPU
-instruction is executed. The emulator will be invoked but will not be
-able to find the instruction which caused the device-not-present
-exception. For this case, the emulator cannot emulate the behaviour of
-an 80486DX.
-
-Handling of the address size override prefix byte (0x67) has not been
-extensively tested yet. A major problem exists because using it in
-vm86 mode can cause a general protection fault. Address offsets
-greater than 0xffff appear to be illegal in vm86 mode but are quite
-acceptable (and work) in real mode. A small test program developed to
-check the addressing, and which runs successfully in real mode,
-crashes dosemu under Linux and also brings Windows down with a general
-protection fault message when run under the MS-DOS prompt of Windows
-3.1. (The program simply reads data from a valid address).
-
-The emulator supports 16-bit protected mode, with one difference from
-an 80486DX.  A 80486DX will allow some floating point instructions to
-write a few bytes below the lowest address of the stack.  The emulator
-will not allow this in 16-bit protected mode: no instructions are
-allowed to write outside the bounds set by the protection.
-
------------------------ Performance of wm-FPU-emu -----------------------
-
-Speed.
------
-
-The speed of floating point computation with the emulator will depend
-upon instruction mix. Relative performance is best for the instructions
-which require most computation. The simple instructions are adversely
-affected by the FPU instruction trap overhead.
-
-
-Timing: Some simple timing tests have been made on the emulator functions.
-The times include load/store instructions. All times are in microseconds
-measured on a 33MHz 386 with 64k cache. The Turbo C tests were under
-ms-dos, the next two columns are for emulators running with the djgpp
-ms-dos extender. The final column is for wm-FPU-emu in Linux 0.97,
-using libm4.0 (hard).
-
-function      Turbo C        djgpp 1.06        WM-emu387     wm-FPU-emu
-
-   +          60.5           154.8              76.5          139.4
-   -          61.1-65.5      157.3-160.8        76.2-79.5     142.9-144.7
-   *          71.0           190.8              79.6          146.6
-   /          61.2-75.0      261.4-266.9        75.3-91.6     142.2-158.1
-
- sin()        310.8          4692.0            319.0          398.5
- cos()        284.4          4855.2            308.0          388.7
- tan()        495.0          8807.1            394.9          504.7
- atan()       328.9          4866.4            601.1          419.5-491.9
-
- sqrt()       128.7          crashed           145.2          227.0
- log()        413.1-419.1    5103.4-5354.21    254.7-282.2    409.4-437.1
- exp()        479.1          6619.2            469.1          850.8
-
-
-The performance under Linux is improved by the use of look-ahead code.
-The following results show the improvement which is obtained under
-Linux due to the look-ahead code. Also given are the times for the
-original Linux emulator with the 4.1 'soft' lib.
-
- [ Linus' note: I changed look-ahead to be the default under linux, as
-   there was no reason not to use it after I had edited it to be
-   disabled during tracing ]
-
-            wm-FPU-emu w     original w
-            look-ahead       'soft' lib
-   +         106.4             190.2
-   -         108.6-111.6      192.4-216.2
-   *         113.4             193.1
-   /         108.8-124.4      700.1-706.2
-
- sin()       390.5            2642.0
- cos()       381.5            2767.4
- tan()       496.5            3153.3
- atan()      367.2-435.5     2439.4-3396.8
-
- sqrt()      195.1            4732.5
- log()       358.0-387.5     3359.2-3390.3
- exp()       619.3            4046.4
-
-
-These figures are now somewhat out-of-date. The emulator has become
-progressively slower for most functions as more of the 80486 features
-have been implemented.
-
-
------------------------ Accuracy of wm-FPU-emu -----------------------
-
-
-The accuracy of the emulator is in almost all cases equal to or better
-than that of an Intel 80486 FPU.
-
-The results of the basic arithmetic functions (+,-,*,/), and fsqrt
-match those of an 80486 FPU. They are the best possible; the error for
-these never exceeds 1/2 an lsb. The fprem and fprem1 instructions
-return exact results; they have no error.
-
-
-The following table compares the emulator accuracy for the sqrt(),
-trig and log functions against the Turbo C "emulator". For this table,
-each function was tested at about 400 points. Ideal worst-case results
-would be 64 bits. The reduced Turbo C accuracy of cos() and tan() for
-arguments greater than pi/4 can be thought of as being related to the
-precision of the argument x; e.g. an argument of pi/2-(1e-10) which is
-accurate to 64 bits can result in a relative accuracy in cos() of
-about 64 + log2(cos(x)) = 31 bits.
-
-
-Function      Tested x range            Worst result                Turbo C
-                                        (relative bits)
-
-sqrt(x)       1 .. 2                    64.1                         63.2
-atan(x)       1e-10 .. 200              64.2                         62.8
-cos(x)        0 .. pi/2-(1e-10)         64.4 (x <= pi/4)             62.4
-                                        64.1 (x = pi/2-(1e-10))      31.9
-sin(x)        1e-10 .. pi/2             64.0                         62.8
-tan(x)        1e-10 .. pi/2-(1e-10)     64.0 (x <= pi/4)             62.1
-                                        64.1 (x = pi/2-(1e-10))      31.9
-exp(x)        0 .. 1                    63.1 **                      62.9
-log(x)        1+1e-6 .. 2               63.8 **                      62.1
-
-** The accuracy for exp() and log() is low because the FPU (emulator)
-does not compute them directly; two operations are required.
-
-
-The emulator passes the "paranoia" tests (compiled with gcc 2.3.3 or
-later) for 'float' variables (24 bit precision numbers) when precision
-control is set to 24, 53 or 64 bits, and for 'double' variables (53
-bit precision numbers) when precision control is set to 53 bits (a
-properly performing FPU cannot pass the 'paranoia' tests for 'double'
-variables when precision control is set to 64 bits).
-
-The code for reducing the argument for the trig functions (fsin, fcos,
-fptan and fsincos) has been improved and now effectively uses a value
-for pi which is accurate to more than 128 bits precision. As a
-consequence, the accuracy of these functions for large arguments has
-been dramatically improved (and is now very much better than an 80486
-FPU). There is also now no degradation of accuracy for fcos and fptan
-for operands close to pi/2. Measured results are (note that the
-definition of accuracy has changed slightly from that used for the
-above table):
-
-Function      Tested x range          Worst result
-                                     (absolute bits)
-
-cos(x)        0 .. 9.22e+18              62.0
-sin(x)        1e-16 .. 9.22e+18          62.1
-tan(x)        1e-16 .. 9.22e+18          61.8
-
-It is possible with some effort to find very large arguments which
-give much degraded precision. For example, the integer number
-           8227740058411162616.0
-is within about 10e-7 of a multiple of pi. To find the tan (for
-example) of this number to 64 bits precision it would be necessary to
-have a value of pi which had about 150 bits precision. The FPU
-emulator computes the result to about 42.6 bits precision (the correct
-result is about -9.739715e-8). On the other hand, an 80486 FPU returns
-0.01059, which in relative terms is hopelessly inaccurate.
-
-For arguments close to critical angles (which occur at multiples of
-pi/2) the emulator is more accurate than an 80486 FPU. For very large
-arguments, the emulator is far more accurate.
-
-
-Prior to version 1.20 of the emulator, the accuracy of the results for
-the transcendental functions (in their principal range) was not as
-good as the results from an 80486 FPU. From version 1.20, the accuracy
-has been considerably improved and these functions now give measured
-worst-case results which are better than the worst-case results given
-by an 80486 FPU.
-
-The following table gives the measured results for the emulator. The
-number of randomly selected arguments in each case is about half a
-million.  The group of three columns gives the frequency of the given
-accuracy in number of times per million, thus the second of these
-columns shows that an accuracy of between 63.80 and 63.89 bits was
-found at a rate of 133 times per one million measurements for fsin.
-The results show that the fsin, fcos and fptan instructions return
-results which are in error (i.e. less accurate than the best possible
-result (which is 64 bits)) for about one per cent of all arguments
-between -pi/2 and +pi/2.  The other instructions have a lower
-frequency of results which are in error.  The last two columns give
-the worst accuracy which was found (in bits) and the approximate value
-of the argument which produced it.
-
-                                frequency (per M)
-                               -------------------   ---------------
-instr   arg range    # tests   63.7   63.8    63.9   worst   at arg
-                               bits   bits    bits    bits
------  ------------  -------   ----   ----   -----   -----  --------
-fsin     (0,pi/2)     547756      0    133   10673   63.89  0.451317
-fcos     (0,pi/2)     547563      0    126   10532   63.85  0.700801
-fptan    (0,pi/2)     536274     11    267   10059   63.74  0.784876
-fpatan  4 quadrants   517087      0      8    1855   63.88  0.435121 (4q)
-fyl2x     (0,20)      541861      0      0    1323   63.94  1.40923  (x)
-fyl2xp1 (-.293,.414)  520256      0      0    5678   63.93  0.408542 (x)
-f2xm1     (-1,1)      538847      4    481    6488   63.79  0.167709
-
-
-Tests performed on an 80486 FPU showed results of lower accuracy. The
-following table gives the results which were obtained with an AMD
-486DX2/66 (other tests indicate that an Intel 486DX produces
-identical results).  The tests were basically the same as those used
-to measure the emulator (the values, being random, were in general not
-the same).  The total number of tests for each instruction are given
-at the end of the table, in case each about 100k tests were performed.
-Another line of figures at the end of the table shows that most of the
-instructions return results which are in error for more than 10
-percent of the arguments tested.
-
-The numbers in the body of the table give the approx number of times a
-result of the given accuracy in bits (given in the left-most column)
-was obtained per one million arguments. For three of the instructions,
-two columns of results are given: * The second column for f2xm1 gives
-the number cases where the results of the first column were for a
-positive argument, this shows that this instruction gives better
-results for positive arguments than it does for negative.  * In the
-cases of fcos and fptan, the first column gives the results when all
-cases where arguments greater than 1.5 were removed from the results
-given in the second column. Unlike the emulator, an 80486 FPU returns
-results of relatively poor accuracy for these instructions when the
-argument approaches pi/2. The table does not show those cases when the
-accuracy of the results were less than 62 bits, which occurs quite
-often for fsin and fptan when the argument approaches pi/2. This poor
-accuracy is discussed above in relation to the Turbo C "emulator", and
-the accuracy of the value of pi.
-
-
-bits   f2xm1  f2xm1 fpatan   fcos   fcos  fyl2x fyl2xp1  fsin  fptan  fptan
-62.0       0      0      0      0    437      0      0      0      0    925
-62.1       0      0     10      0    894      0      0      0      0   1023
-62.2      14      0      0      0   1033      0      0      0      0    945
-62.3      57      0      0      0   1202      0      0      0      0   1023
-62.4     385      0      0     10   1292      0     23      0      0   1178
-62.5    1140      0      0    119   1649      0     39      0      0   1149
-62.6    2037      0      0    189   1620      0     16      0      0   1169
-62.7    5086     14      0    646   2315     10    101     35     39   1402
-62.8    8818     86      0    984   3050     59    287    131    224   2036
-62.9   11340   1355      0   2126   4153     79    605    357    321   1948
-63.0   15557   4750      0   3319   5376    246   1281    862    808   2688
-63.1   20016   8288      0   4620   6628    511   2569   1723   1510   3302
-63.2   24945  11127     10   6588   8098   1120   4470   2968   2990   4724
-63.3   25686  12382     69   8774  10682   1906   6775   4482   5474   7236
-63.4   29219  14722     79  11109  12311   3094   9414   7259   8912  10587
-63.5   30458  14936    393  13802  15014   5874  12666   9609  13762  15262
-63.6   32439  16448   1277  17945  19028  10226  15537  14657  19158  20346
-63.7   35031  16805   4067  23003  23947  18910  20116  21333  25001  26209
-63.8   33251  15820   7673  24781  25675  24617  25354  24440  29433  30329
-63.9   33293  16833  18529  28318  29233  31267  31470  27748  29676  30601
-
-Per cent with error:
-        30.9           3.2          18.5    9.8   13.1   11.6          17.4
-Total arguments tested:
-       70194  70099 101784 100641 100641 101799 128853 114893 102675 102675
-
-
-------------------------- Contributors -------------------------------
-
-A number of people have contributed to the development of the
-emulator, often by just reporting bugs, sometimes with suggested
-fixes, and a few kind people have provided me with access in one way
-or another to an 80486 machine. Contributors include (to those people
-who I may have forgotten, please forgive me):
-
-Linus Torvalds
-Tommy.Thorn@daimi.aau.dk
-Andrew.Tridgell@anu.edu.au
-Nick Holloway, alfie@dcs.warwick.ac.uk
-Hermano Moura, moura@dcs.gla.ac.uk
-Jon Jagger, J.Jagger@scp.ac.uk
-Lennart Benschop
-Brian Gallew, geek+@CMU.EDU
-Thomas Staniszewski, ts3v+@andrew.cmu.edu
-Martin Howell, mph@plasma.apana.org.au
-M Saggaf, alsaggaf@athena.mit.edu
-Peter Barker, PETER@socpsy.sci.fau.edu
-tom@vlsivie.tuwien.ac.at
-Dan Russel, russed@rpi.edu
-Daniel Carosone, danielce@ee.mu.oz.au
-cae@jpmorgan.com
-Hamish Coleman, t933093@minyos.xx.rmit.oz.au
-Bruce Evans, bde@kralizec.zeta.org.au
-Timo Korvola, Timo.Korvola@hut.fi
-Rick Lyons, rick@razorback.brisnet.org.au
-Rick, jrs@world.std.com
- 
-...and numerous others who responded to my request for help with
-a real 80486.
-
diff --git a/arch/x86/math-emu/control_w.h b/arch/x86/math-emu/control_w.h
deleted file mode 100644
index 93cbc89b34e25..0000000000000
--- a/arch/x86/math-emu/control_w.h
+++ /dev/null
@@ -1,46 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  control_w.h                                                              |
- |                                                                           |
- | Copyright (C) 1992,1993                                                   |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@vaxc.cc.monash.edu.au    |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#ifndef _CONTROLW_H_
-#define _CONTROLW_H_
-
-#ifdef __ASSEMBLER__
-#define	_Const_(x)	$##x
-#else
-#define	_Const_(x)	x
-#endif
-
-#define CW_RC		_Const_(0x0C00)	/* rounding control */
-#define CW_PC		_Const_(0x0300)	/* precision control */
-
-#define CW_Precision	Const_(0x0020)	/* loss of precision mask */
-#define CW_Underflow	Const_(0x0010)	/* underflow mask */
-#define CW_Overflow	Const_(0x0008)	/* overflow mask */
-#define CW_ZeroDiv	Const_(0x0004)	/* divide by zero mask */
-#define CW_Denormal	Const_(0x0002)	/* denormalized operand mask */
-#define CW_Invalid	Const_(0x0001)	/* invalid operation mask */
-
-#define CW_Exceptions  	_Const_(0x003f)	/* all masks */
-
-#define RC_RND		_Const_(0x0000)
-#define RC_DOWN		_Const_(0x0400)
-#define RC_UP		_Const_(0x0800)
-#define RC_CHOP		_Const_(0x0C00)
-
-/* p 15-5: Precision control bits affect only the following:
-   ADD, SUB(R), MUL, DIV(R), and SQRT */
-#define PR_24_BITS        _Const_(0x000)
-#define PR_53_BITS        _Const_(0x200)
-#define PR_64_BITS        _Const_(0x300)
-#define PR_RESERVED_BITS  _Const_(0x100)
-/* FULL_PRECISION simulates all exceptions masked */
-#define FULL_PRECISION  (PR_64_BITS | RC_RND | 0x3f)
-
-#endif /* _CONTROLW_H_ */
diff --git a/arch/x86/math-emu/div_Xsig.S b/arch/x86/math-emu/div_Xsig.S
deleted file mode 100644
index 8c270ab415bee..0000000000000
--- a/arch/x86/math-emu/div_Xsig.S
+++ /dev/null
@@ -1,367 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file	"div_Xsig.S"
-/*---------------------------------------------------------------------------+
- |  div_Xsig.S                                                               |
- |                                                                           |
- | Division subroutine for 96 bit quantities                                 |
- |                                                                           |
- | Copyright (C) 1994,1995                                                   |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@jacobi.maths.monash.edu.au |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Divide the 96 bit quantity pointed to by a, by that pointed to by b, and  |
- | put the 96 bit result at the location d.                                  |
- |                                                                           |
- | The result may not be accurate to 96 bits. It is intended for use where   |
- | a result better than 64 bits is required. The result should usually be    |
- | good to at least 94 bits.                                                 |
- | The returned result is actually divided by one half. This is done to      |
- | prevent overflow.                                                         |
- |                                                                           |
- |  .aaaaaaaaaaaaaa / .bbbbbbbbbbbbb  ->  .dddddddddddd                      |
- |                                                                           |
- |  void div_Xsig(Xsig *a, Xsig *b, Xsig *dest)                              |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-
-
-#define	XsigLL(x)	(x)
-#define	XsigL(x)	4(x)
-#define	XsigH(x)	8(x)
-
-
-#ifndef NON_REENTRANT_FPU
-/*
-	Local storage on the stack:
-	Accumulator:	FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0
- */
-#define FPU_accum_3	-4(%ebp)
-#define FPU_accum_2	-8(%ebp)
-#define FPU_accum_1	-12(%ebp)
-#define FPU_accum_0	-16(%ebp)
-#define FPU_result_3	-20(%ebp)
-#define FPU_result_2	-24(%ebp)
-#define FPU_result_1	-28(%ebp)
-
-#else
-.data
-/*
-	Local storage in a static area:
-	Accumulator:	FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0
- */
-	.align 4,0
-FPU_accum_3:
-	.long	0
-FPU_accum_2:
-	.long	0
-FPU_accum_1:
-	.long	0
-FPU_accum_0:
-	.long	0
-FPU_result_3:
-	.long	0
-FPU_result_2:
-	.long	0
-FPU_result_1:
-	.long	0
-#endif /* NON_REENTRANT_FPU */
-
-
-.text
-SYM_FUNC_START(div_Xsig)
-	pushl	%ebp
-	movl	%esp,%ebp
-#ifndef NON_REENTRANT_FPU
-	subl	$28,%esp
-#endif /* NON_REENTRANT_FPU */ 
-
-	pushl	%esi
-	pushl	%edi
-	pushl	%ebx
-
-	movl	PARAM1,%esi	/* pointer to num */
-	movl	PARAM2,%ebx	/* pointer to denom */
-
-#ifdef PARANOID
-	testl	$0x80000000, XsigH(%ebx)	/* Divisor */
-	je	L_bugged
-#endif /* PARANOID */
-
-
-/*---------------------------------------------------------------------------+
- |  Divide:   Return  arg1/arg2 to arg3.                                     |
- |                                                                           |
- |  The maximum returned value is (ignoring exponents)                       |
- |               .ffffffff ffffffff                                          |
- |               ------------------  =  1.ffffffff fffffffe                  |
- |               .80000000 00000000                                          |
- | and the minimum is                                                        |
- |               .80000000 00000000                                          |
- |               ------------------  =  .80000000 00000001   (rounded)       |
- |               .ffffffff ffffffff                                          |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-	/* Save extended dividend in local register */
-
-	/* Divide by 2 to prevent overflow */
-	clc
-	movl	XsigH(%esi),%eax
-	rcrl	%eax
-	movl	%eax,FPU_accum_3
-	movl	XsigL(%esi),%eax
-	rcrl	%eax
-	movl	%eax,FPU_accum_2
-	movl	XsigLL(%esi),%eax
-	rcrl	%eax
-	movl	%eax,FPU_accum_1
-	movl	$0,%eax
-	rcrl	%eax
-	movl	%eax,FPU_accum_0
-
-	movl	FPU_accum_2,%eax	/* Get the current num */
-	movl	FPU_accum_3,%edx
-
-/*----------------------------------------------------------------------*/
-/* Initialization done.
-   Do the first 32 bits. */
-
-	/* We will divide by a number which is too large */
-	movl	XsigH(%ebx),%ecx
-	addl	$1,%ecx
-	jnc	LFirst_div_not_1
-
-	/* here we need to divide by 100000000h,
-	   i.e., no division at all.. */
-	mov	%edx,%eax
-	jmp	LFirst_div_done
-
-LFirst_div_not_1:
-	divl	%ecx		/* Divide the numerator by the augmented
-				   denom ms dw */
-
-LFirst_div_done:
-	movl	%eax,FPU_result_3	/* Put the result in the answer */
-
-	mull	XsigH(%ebx)	/* mul by the ms dw of the denom */
-
-	subl	%eax,FPU_accum_2	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_3
-
-	movl	FPU_result_3,%eax	/* Get the result back */
-	mull	XsigL(%ebx)	/* now mul the ls dw of the denom */
-
-	subl	%eax,FPU_accum_1	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_2
-	sbbl	$0,FPU_accum_3
-	je	LDo_2nd_32_bits		/* Must check for non-zero result here */
-
-#ifdef PARANOID
-	jb	L_bugged_1
-#endif /* PARANOID */ 
-
-	/* need to subtract another once of the denom */
-	incl	FPU_result_3	/* Correct the answer */
-
-	movl	XsigL(%ebx),%eax
-	movl	XsigH(%ebx),%edx
-	subl	%eax,FPU_accum_1	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_2
-
-#ifdef PARANOID
-	sbbl	$0,FPU_accum_3
-	jne	L_bugged_1	/* Must check for non-zero result here */
-#endif /* PARANOID */ 
-
-/*----------------------------------------------------------------------*/
-/* Half of the main problem is done, there is just a reduced numerator
-   to handle now.
-   Work with the second 32 bits, FPU_accum_0 not used from now on */
-LDo_2nd_32_bits:
-	movl	FPU_accum_2,%edx	/* get the reduced num */
-	movl	FPU_accum_1,%eax
-
-	/* need to check for possible subsequent overflow */
-	cmpl	XsigH(%ebx),%edx
-	jb	LDo_2nd_div
-	ja	LPrevent_2nd_overflow
-
-	cmpl	XsigL(%ebx),%eax
-	jb	LDo_2nd_div
-
-LPrevent_2nd_overflow:
-/* The numerator is greater or equal, would cause overflow */
-	/* prevent overflow */
-	subl	XsigL(%ebx),%eax
-	sbbl	XsigH(%ebx),%edx
-	movl	%edx,FPU_accum_2
-	movl	%eax,FPU_accum_1
-
-	incl	FPU_result_3	/* Reflect the subtraction in the answer */
-
-#ifdef PARANOID
-	je	L_bugged_2	/* Can't bump the result to 1.0 */
-#endif /* PARANOID */ 
-
-LDo_2nd_div:
-	cmpl	$0,%ecx		/* augmented denom msw */
-	jnz	LSecond_div_not_1
-
-	/* %ecx == 0, we are dividing by 1.0 */
-	mov	%edx,%eax
-	jmp	LSecond_div_done
-
-LSecond_div_not_1:
-	divl	%ecx		/* Divide the numerator by the denom ms dw */
-
-LSecond_div_done:
-	movl	%eax,FPU_result_2	/* Put the result in the answer */
-
-	mull	XsigH(%ebx)	/* mul by the ms dw of the denom */
-
-	subl	%eax,FPU_accum_1	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_2
-
-#ifdef PARANOID
-	jc	L_bugged_2
-#endif /* PARANOID */
-
-	movl	FPU_result_2,%eax	/* Get the result back */
-	mull	XsigL(%ebx)	/* now mul the ls dw of the denom */
-
-	subl	%eax,FPU_accum_0	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_1	/* Subtract from the num local reg */
-	sbbl	$0,FPU_accum_2
-
-#ifdef PARANOID
-	jc	L_bugged_2
-#endif /* PARANOID */
-
-	jz	LDo_3rd_32_bits
-
-#ifdef PARANOID
-	cmpl	$1,FPU_accum_2
-	jne	L_bugged_2
-#endif /* PARANOID */ 
-
-	/* need to subtract another once of the denom */
-	movl	XsigL(%ebx),%eax
-	movl	XsigH(%ebx),%edx
-	subl	%eax,FPU_accum_0	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_1
-	sbbl	$0,FPU_accum_2
-
-#ifdef PARANOID
-	jc	L_bugged_2
-	jne	L_bugged_2
-#endif /* PARANOID */ 
-
-	addl	$1,FPU_result_2	/* Correct the answer */
-	adcl	$0,FPU_result_3
-
-#ifdef PARANOID
-	jc	L_bugged_2	/* Must check for non-zero result here */
-#endif /* PARANOID */ 
-
-/*----------------------------------------------------------------------*/
-/* The division is essentially finished here, we just need to perform
-   tidying operations.
-   Deal with the 3rd 32 bits */
-LDo_3rd_32_bits:
-	/* We use an approximation for the third 32 bits.
-	To take account of the 3rd 32 bits of the divisor
-	(call them del), we subtract  del * (a/b) */
-
-	movl	FPU_result_3,%eax	/* a/b */
-	mull	XsigLL(%ebx)		/* del */
-
-	subl	%edx,FPU_accum_1
-
-	/* A borrow indicates that the result is negative */
-	jnb	LTest_over
-
-	movl	XsigH(%ebx),%edx
-	addl	%edx,FPU_accum_1
-
-	subl	$1,FPU_result_2		/* Adjust the answer */
-	sbbl	$0,FPU_result_3
-
-	/* The above addition might not have been enough, check again. */
-	movl	FPU_accum_1,%edx	/* get the reduced num */
-	cmpl	XsigH(%ebx),%edx	/* denom */
-	jb	LDo_3rd_div
-
-	movl	XsigH(%ebx),%edx
-	addl	%edx,FPU_accum_1
-
-	subl	$1,FPU_result_2		/* Adjust the answer */
-	sbbl	$0,FPU_result_3
-	jmp	LDo_3rd_div
-
-LTest_over:
-	movl	FPU_accum_1,%edx	/* get the reduced num */
-
-	/* need to check for possible subsequent overflow */
-	cmpl	XsigH(%ebx),%edx	/* denom */
-	jb	LDo_3rd_div
-
-	/* prevent overflow */
-	subl	XsigH(%ebx),%edx
-	movl	%edx,FPU_accum_1
-
-	addl	$1,FPU_result_2	/* Reflect the subtraction in the answer */
-	adcl	$0,FPU_result_3
-
-LDo_3rd_div:
-	movl	FPU_accum_0,%eax
-	movl	FPU_accum_1,%edx
-	divl	XsigH(%ebx)
-
-	movl    %eax,FPU_result_1       /* Rough estimate of third word */
-
-	movl	PARAM3,%esi		/* pointer to answer */
-
-	movl	FPU_result_1,%eax
-	movl	%eax,XsigLL(%esi)
-	movl	FPU_result_2,%eax
-	movl	%eax,XsigL(%esi)
-	movl	FPU_result_3,%eax
-	movl	%eax,XsigH(%esi)
-
-L_exit:
-	popl	%ebx
-	popl	%edi
-	popl	%esi
-
-	leave
-	RET
-
-
-#ifdef PARANOID
-/* The logic is wrong if we got here */
-L_bugged:
-	pushl	EX_INTERNAL|0x240
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_exit
-
-L_bugged_1:
-	pushl	EX_INTERNAL|0x241
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_exit
-
-L_bugged_2:
-	pushl	EX_INTERNAL|0x242
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_exit
-#endif /* PARANOID */ 
-SYM_FUNC_END(div_Xsig)
diff --git a/arch/x86/math-emu/div_small.S b/arch/x86/math-emu/div_small.S
deleted file mode 100644
index 637439bfefa47..0000000000000
--- a/arch/x86/math-emu/div_small.S
+++ /dev/null
@@ -1,48 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file	"div_small.S"
-/*---------------------------------------------------------------------------+
- |  div_small.S                                                              |
- |                                                                           |
- | Divide a 64 bit integer by a 32 bit integer & return remainder.           |
- |                                                                           |
- | Copyright (C) 1992,1995                                                   |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@jacobi.maths.monash.edu.au |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- |    unsigned long FPU_div_small(unsigned long long *x, unsigned long y)    |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-.text
-SYM_FUNC_START(FPU_div_small)
-	pushl	%ebp
-	movl	%esp,%ebp
-
-	pushl	%esi
-
-	movl	PARAM1,%esi	/* pointer to num */
-	movl	PARAM2,%ecx	/* The denominator */
-
-	movl	4(%esi),%eax	/* Get the current num msw */
-	xorl	%edx,%edx
-	divl	%ecx
-
-	movl	%eax,4(%esi)
-
-	movl	(%esi),%eax	/* Get the num lsw */
-	divl	%ecx
-
-	movl	%eax,(%esi)
-
-	movl	%edx,%eax	/* Return the remainder in eax */
-
-	popl	%esi
-
-	leave
-	RET
-SYM_FUNC_END(FPU_div_small)
diff --git a/arch/x86/math-emu/errors.c b/arch/x86/math-emu/errors.c
deleted file mode 100644
index ec071cbb08043..0000000000000
--- a/arch/x86/math-emu/errors.c
+++ /dev/null
@@ -1,686 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  errors.c                                                                 |
- |                                                                           |
- |  The error handling functions for wm-FPU-emu                              |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1996                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@jacobi.maths.monash.edu.au                |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note:                                                                     |
- |    The file contains code which accesses user memory.                     |
- |    Emulator static data may change when user memory is accessed, due to   |
- |    other processes using the emulator while swapping is in progress.      |
- +---------------------------------------------------------------------------*/
-
-#include <linux/signal.h>
-
-#include <linux/uaccess.h>
-
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "exception.h"
-#include "status_w.h"
-#include "control_w.h"
-#include "reg_constant.h"
-#include "version.h"
-
-/* */
-#undef PRINT_MESSAGES
-/* */
-
-#if 0
-void Un_impl(void)
-{
-	u_char byte1, FPU_modrm;
-	unsigned long address = FPU_ORIG_EIP;
-
-	RE_ENTRANT_CHECK_OFF;
-	/* No need to check access_ok(), we have previously fetched these bytes. */
-	printk("Unimplemented FPU Opcode at eip=%p : ", (void __user *)address);
-	if (FPU_CS == __USER_CS) {
-		while (1) {
-			FPU_get_user(byte1, (u_char __user *) address);
-			if ((byte1 & 0xf8) == 0xd8)
-				break;
-			printk("[%02x]", byte1);
-			address++;
-		}
-		printk("%02x ", byte1);
-		FPU_get_user(FPU_modrm, 1 + (u_char __user *) address);
-
-		if (FPU_modrm >= 0300)
-			printk("%02x (%02x+%d)\n", FPU_modrm, FPU_modrm & 0xf8,
-			       FPU_modrm & 7);
-		else
-			printk("/%d\n", (FPU_modrm >> 3) & 7);
-	} else {
-		printk("cs selector = %04x\n", FPU_CS);
-	}
-
-	RE_ENTRANT_CHECK_ON;
-
-	EXCEPTION(EX_Invalid);
-
-}
-#endif /*  0  */
-
-/*
-   Called for opcodes which are illegal and which are known to result in a
-   SIGILL with a real 80486.
-   */
-void FPU_illegal(void)
-{
-	math_abort(FPU_info, SIGILL);
-}
-
-void FPU_printall(void)
-{
-	int i;
-	static const char *tag_desc[] = { "Valid", "Zero", "ERROR", "Empty",
-		"DeNorm", "Inf", "NaN"
-	};
-	u_char byte1, FPU_modrm;
-	unsigned long address = FPU_ORIG_EIP;
-
-	RE_ENTRANT_CHECK_OFF;
-	/* No need to check access_ok(), we have previously fetched these bytes. */
-	printk("At %p:", (void *)address);
-	if (FPU_CS == __USER_CS) {
-#define MAX_PRINTED_BYTES 20
-		for (i = 0; i < MAX_PRINTED_BYTES; i++) {
-			FPU_get_user(byte1, (u_char __user *) address);
-			if ((byte1 & 0xf8) == 0xd8) {
-				printk(" %02x", byte1);
-				break;
-			}
-			printk(" [%02x]", byte1);
-			address++;
-		}
-		if (i == MAX_PRINTED_BYTES)
-			printk(" [more..]\n");
-		else {
-			FPU_get_user(FPU_modrm, 1 + (u_char __user *) address);
-
-			if (FPU_modrm >= 0300)
-				printk(" %02x (%02x+%d)\n", FPU_modrm,
-				       FPU_modrm & 0xf8, FPU_modrm & 7);
-			else
-				printk(" /%d, mod=%d rm=%d\n",
-				       (FPU_modrm >> 3) & 7,
-				       (FPU_modrm >> 6) & 3, FPU_modrm & 7);
-		}
-	} else {
-		printk("%04x\n", FPU_CS);
-	}
-
-	partial_status = status_word();
-
-#ifdef DEBUGGING
-	if (partial_status & SW_Backward)
-		printk("SW: backward compatibility\n");
-	if (partial_status & SW_C3)
-		printk("SW: condition bit 3\n");
-	if (partial_status & SW_C2)
-		printk("SW: condition bit 2\n");
-	if (partial_status & SW_C1)
-		printk("SW: condition bit 1\n");
-	if (partial_status & SW_C0)
-		printk("SW: condition bit 0\n");
-	if (partial_status & SW_Summary)
-		printk("SW: exception summary\n");
-	if (partial_status & SW_Stack_Fault)
-		printk("SW: stack fault\n");
-	if (partial_status & SW_Precision)
-		printk("SW: loss of precision\n");
-	if (partial_status & SW_Underflow)
-		printk("SW: underflow\n");
-	if (partial_status & SW_Overflow)
-		printk("SW: overflow\n");
-	if (partial_status & SW_Zero_Div)
-		printk("SW: divide by zero\n");
-	if (partial_status & SW_Denorm_Op)
-		printk("SW: denormalized operand\n");
-	if (partial_status & SW_Invalid)
-		printk("SW: invalid operation\n");
-#endif /* DEBUGGING */
-
-	printk(" SW: b=%d st=%d es=%d sf=%d cc=%d%d%d%d ef=%d%d%d%d%d%d\n", partial_status & 0x8000 ? 1 : 0,	/* busy */
-	       (partial_status & 0x3800) >> 11,	/* stack top pointer */
-	       partial_status & 0x80 ? 1 : 0,	/* Error summary status */
-	       partial_status & 0x40 ? 1 : 0,	/* Stack flag */
-	       partial_status & SW_C3 ? 1 : 0, partial_status & SW_C2 ? 1 : 0,	/* cc */
-	       partial_status & SW_C1 ? 1 : 0, partial_status & SW_C0 ? 1 : 0,	/* cc */
-	       partial_status & SW_Precision ? 1 : 0,
-	       partial_status & SW_Underflow ? 1 : 0,
-	       partial_status & SW_Overflow ? 1 : 0,
-	       partial_status & SW_Zero_Div ? 1 : 0,
-	       partial_status & SW_Denorm_Op ? 1 : 0,
-	       partial_status & SW_Invalid ? 1 : 0);
-
-	printk(" CW: ic=%d rc=%d%d pc=%d%d iem=%d     ef=%d%d%d%d%d%d\n",
-	       control_word & 0x1000 ? 1 : 0,
-	       (control_word & 0x800) >> 11, (control_word & 0x400) >> 10,
-	       (control_word & 0x200) >> 9, (control_word & 0x100) >> 8,
-	       control_word & 0x80 ? 1 : 0,
-	       control_word & SW_Precision ? 1 : 0,
-	       control_word & SW_Underflow ? 1 : 0,
-	       control_word & SW_Overflow ? 1 : 0,
-	       control_word & SW_Zero_Div ? 1 : 0,
-	       control_word & SW_Denorm_Op ? 1 : 0,
-	       control_word & SW_Invalid ? 1 : 0);
-
-	for (i = 0; i < 8; i++) {
-		FPU_REG *r = &st(i);
-		u_char tagi = FPU_gettagi(i);
-
-		switch (tagi) {
-		case TAG_Empty:
-			continue;
-		case TAG_Zero:
-		case TAG_Special:
-			/* Update tagi for the printk below */
-			tagi = FPU_Special(r);
-			fallthrough;
-		case TAG_Valid:
-			printk("st(%d)  %c .%04lx %04lx %04lx %04lx e%+-6d ", i,
-			       getsign(r) ? '-' : '+',
-			       (long)(r->sigh >> 16),
-			       (long)(r->sigh & 0xFFFF),
-			       (long)(r->sigl >> 16),
-			       (long)(r->sigl & 0xFFFF),
-			       exponent(r) - EXP_BIAS + 1);
-			break;
-		default:
-			printk("Whoops! Error in errors.c: tag%d is %d ", i,
-			       tagi);
-			continue;
-		}
-		printk("%s\n", tag_desc[(int)(unsigned)tagi]);
-	}
-
-	RE_ENTRANT_CHECK_ON;
-
-}
-
-static struct {
-	int type;
-	const char *name;
-} exception_names[] = {
-	{
-	EX_StackOver, "stack overflow"}, {
-	EX_StackUnder, "stack underflow"}, {
-	EX_Precision, "loss of precision"}, {
-	EX_Underflow, "underflow"}, {
-	EX_Overflow, "overflow"}, {
-	EX_ZeroDiv, "divide by zero"}, {
-	EX_Denormal, "denormalized operand"}, {
-	EX_Invalid, "invalid operation"}, {
-	EX_INTERNAL, "INTERNAL BUG in " FPU_VERSION}, {
-	0, NULL}
-};
-
-/*
- EX_INTERNAL is always given with a code which indicates where the
- error was detected.
-
- Internal error types:
-       0x14   in fpu_etc.c
-       0x1nn  in a *.c file:
-              0x101  in reg_add_sub.c
-              0x102  in reg_mul.c
-              0x104  in poly_atan.c
-              0x105  in reg_mul.c
-              0x107  in fpu_trig.c
-	      0x108  in reg_compare.c
-	      0x109  in reg_compare.c
-	      0x110  in reg_add_sub.c
-	      0x111  in fpe_entry.c
-	      0x112  in fpu_trig.c
-	      0x113  in errors.c
-	      0x115  in fpu_trig.c
-	      0x116  in fpu_trig.c
-	      0x117  in fpu_trig.c
-	      0x118  in fpu_trig.c
-	      0x119  in fpu_trig.c
-	      0x120  in poly_atan.c
-	      0x121  in reg_compare.c
-	      0x122  in reg_compare.c
-	      0x123  in reg_compare.c
-	      0x125  in fpu_trig.c
-	      0x126  in fpu_entry.c
-	      0x127  in poly_2xm1.c
-	      0x128  in fpu_entry.c
-	      0x129  in fpu_entry.c
-	      0x130  in get_address.c
-	      0x131  in get_address.c
-	      0x132  in get_address.c
-	      0x133  in get_address.c
-	      0x140  in load_store.c
-	      0x141  in load_store.c
-              0x150  in poly_sin.c
-              0x151  in poly_sin.c
-	      0x160  in reg_ld_str.c
-	      0x161  in reg_ld_str.c
-	      0x162  in reg_ld_str.c
-	      0x163  in reg_ld_str.c
-	      0x164  in reg_ld_str.c
-	      0x170  in fpu_tags.c
-	      0x171  in fpu_tags.c
-	      0x172  in fpu_tags.c
-	      0x180  in reg_convert.c
-       0x2nn  in an *.S file:
-              0x201  in reg_u_add.S
-              0x202  in reg_u_div.S
-              0x203  in reg_u_div.S
-              0x204  in reg_u_div.S
-              0x205  in reg_u_mul.S
-              0x206  in reg_u_sub.S
-              0x207  in wm_sqrt.S
-	      0x208  in reg_div.S
-              0x209  in reg_u_sub.S
-              0x210  in reg_u_sub.S
-              0x211  in reg_u_sub.S
-              0x212  in reg_u_sub.S
-	      0x213  in wm_sqrt.S
-	      0x214  in wm_sqrt.S
-	      0x215  in wm_sqrt.S
-	      0x220  in reg_norm.S
-	      0x221  in reg_norm.S
-	      0x230  in reg_round.S
-	      0x231  in reg_round.S
-	      0x232  in reg_round.S
-	      0x233  in reg_round.S
-	      0x234  in reg_round.S
-	      0x235  in reg_round.S
-	      0x236  in reg_round.S
-	      0x240  in div_Xsig.S
-	      0x241  in div_Xsig.S
-	      0x242  in div_Xsig.S
- */
-
-asmlinkage __visible void FPU_exception(int n)
-{
-	int i, int_type;
-
-	int_type = 0;		/* Needed only to stop compiler warnings */
-	if (n & EX_INTERNAL) {
-		int_type = n - EX_INTERNAL;
-		n = EX_INTERNAL;
-		/* Set lots of exception bits! */
-		partial_status |= (SW_Exc_Mask | SW_Summary | SW_Backward);
-	} else {
-		/* Extract only the bits which we use to set the status word */
-		n &= (SW_Exc_Mask);
-		/* Set the corresponding exception bit */
-		partial_status |= n;
-		/* Set summary bits iff exception isn't masked */
-		if (partial_status & ~control_word & CW_Exceptions)
-			partial_status |= (SW_Summary | SW_Backward);
-		if (n & (SW_Stack_Fault | EX_Precision)) {
-			if (!(n & SW_C1))
-				/* This bit distinguishes over- from underflow for a stack fault,
-				   and roundup from round-down for precision loss. */
-				partial_status &= ~SW_C1;
-		}
-	}
-
-	RE_ENTRANT_CHECK_OFF;
-	if ((~control_word & n & CW_Exceptions) || (n == EX_INTERNAL)) {
-		/* Get a name string for error reporting */
-		for (i = 0; exception_names[i].type; i++)
-			if ((exception_names[i].type & n) ==
-			    exception_names[i].type)
-				break;
-
-		if (exception_names[i].type) {
-#ifdef PRINT_MESSAGES
-			printk("FP Exception: %s!\n", exception_names[i].name);
-#endif /* PRINT_MESSAGES */
-		} else
-			printk("FPU emulator: Unknown Exception: 0x%04x!\n", n);
-
-		if (n == EX_INTERNAL) {
-			printk("FPU emulator: Internal error type 0x%04x\n",
-			       int_type);
-			FPU_printall();
-		}
-#ifdef PRINT_MESSAGES
-		else
-			FPU_printall();
-#endif /* PRINT_MESSAGES */
-
-		/*
-		 * The 80486 generates an interrupt on the next non-control FPU
-		 * instruction. So we need some means of flagging it.
-		 * We use the ES (Error Summary) bit for this.
-		 */
-	}
-	RE_ENTRANT_CHECK_ON;
-
-#ifdef __DEBUG__
-	math_abort(FPU_info, SIGFPE);
-#endif /* __DEBUG__ */
-
-}
-
-/* Real operation attempted on a NaN. */
-/* Returns < 0 if the exception is unmasked */
-int real_1op_NaN(FPU_REG *a)
-{
-	int signalling, isNaN;
-
-	isNaN = (exponent(a) == EXP_OVER) && (a->sigh & 0x80000000);
-
-	/* The default result for the case of two "equal" NaNs (signs may
-	   differ) is chosen to reproduce 80486 behaviour */
-	signalling = isNaN && !(a->sigh & 0x40000000);
-
-	if (!signalling) {
-		if (!isNaN) {	/* pseudo-NaN, or other unsupported? */
-			if (control_word & CW_Invalid) {
-				/* Masked response */
-				reg_copy(&CONST_QNaN, a);
-			}
-			EXCEPTION(EX_Invalid);
-			return (!(control_word & CW_Invalid) ? FPU_Exception :
-				0) | TAG_Special;
-		}
-		return TAG_Special;
-	}
-
-	if (control_word & CW_Invalid) {
-		/* The masked response */
-		if (!(a->sigh & 0x80000000)) {	/* pseudo-NaN ? */
-			reg_copy(&CONST_QNaN, a);
-		}
-		/* ensure a Quiet NaN */
-		a->sigh |= 0x40000000;
-	}
-
-	EXCEPTION(EX_Invalid);
-
-	return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | TAG_Special;
-}
-
-/* Real operation attempted on two operands, one a NaN. */
-/* Returns < 0 if the exception is unmasked */
-int real_2op_NaN(FPU_REG const *b, u_char tagb,
-		 int deststnr, FPU_REG const *defaultNaN)
-{
-	FPU_REG *dest = &st(deststnr);
-	FPU_REG const *a = dest;
-	u_char taga = FPU_gettagi(deststnr);
-	FPU_REG const *x;
-	int signalling, unsupported;
-
-	if (taga == TAG_Special)
-		taga = FPU_Special(a);
-	if (tagb == TAG_Special)
-		tagb = FPU_Special(b);
-
-	/* TW_NaN is also used for unsupported data types. */
-	unsupported = ((taga == TW_NaN)
-		       && !((exponent(a) == EXP_OVER)
-			    && (a->sigh & 0x80000000)))
-	    || ((tagb == TW_NaN)
-		&& !((exponent(b) == EXP_OVER) && (b->sigh & 0x80000000)));
-	if (unsupported) {
-		if (control_word & CW_Invalid) {
-			/* Masked response */
-			FPU_copy_to_regi(&CONST_QNaN, TAG_Special, deststnr);
-		}
-		EXCEPTION(EX_Invalid);
-		return (!(control_word & CW_Invalid) ? FPU_Exception : 0) |
-		    TAG_Special;
-	}
-
-	if (taga == TW_NaN) {
-		x = a;
-		if (tagb == TW_NaN) {
-			signalling = !(a->sigh & b->sigh & 0x40000000);
-			if (significand(b) > significand(a))
-				x = b;
-			else if (significand(b) == significand(a)) {
-				/* The default result for the case of two "equal" NaNs (signs may
-				   differ) is chosen to reproduce 80486 behaviour */
-				x = defaultNaN;
-			}
-		} else {
-			/* return the quiet version of the NaN in a */
-			signalling = !(a->sigh & 0x40000000);
-		}
-	} else
-#ifdef PARANOID
-	if (tagb == TW_NaN)
-#endif /* PARANOID */
-	{
-		signalling = !(b->sigh & 0x40000000);
-		x = b;
-	}
-#ifdef PARANOID
-	else {
-		signalling = 0;
-		EXCEPTION(EX_INTERNAL | 0x113);
-		x = &CONST_QNaN;
-	}
-#endif /* PARANOID */
-
-	if ((!signalling) || (control_word & CW_Invalid)) {
-		if (!x)
-			x = b;
-
-		if (!(x->sigh & 0x80000000))	/* pseudo-NaN ? */
-			x = &CONST_QNaN;
-
-		FPU_copy_to_regi(x, TAG_Special, deststnr);
-
-		if (!signalling)
-			return TAG_Special;
-
-		/* ensure a Quiet NaN */
-		dest->sigh |= 0x40000000;
-	}
-
-	EXCEPTION(EX_Invalid);
-
-	return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | TAG_Special;
-}
-
-/* Invalid arith operation on Valid registers */
-/* Returns < 0 if the exception is unmasked */
-asmlinkage __visible int arith_invalid(int deststnr)
-{
-
-	EXCEPTION(EX_Invalid);
-
-	if (control_word & CW_Invalid) {
-		/* The masked response */
-		FPU_copy_to_regi(&CONST_QNaN, TAG_Special, deststnr);
-	}
-
-	return (!(control_word & CW_Invalid) ? FPU_Exception : 0) | TAG_Valid;
-
-}
-
-/* Divide a finite number by zero */
-asmlinkage __visible int FPU_divide_by_zero(int deststnr, u_char sign)
-{
-	FPU_REG *dest = &st(deststnr);
-	int tag = TAG_Valid;
-
-	if (control_word & CW_ZeroDiv) {
-		/* The masked response */
-		FPU_copy_to_regi(&CONST_INF, TAG_Special, deststnr);
-		setsign(dest, sign);
-		tag = TAG_Special;
-	}
-
-	EXCEPTION(EX_ZeroDiv);
-
-	return (!(control_word & CW_ZeroDiv) ? FPU_Exception : 0) | tag;
-
-}
-
-/* This may be called often, so keep it lean */
-int set_precision_flag(int flags)
-{
-	if (control_word & CW_Precision) {
-		partial_status &= ~(SW_C1 & flags);
-		partial_status |= flags;	/* The masked response */
-		return 0;
-	} else {
-		EXCEPTION(flags);
-		return 1;
-	}
-}
-
-/* This may be called often, so keep it lean */
-asmlinkage __visible void set_precision_flag_up(void)
-{
-	if (control_word & CW_Precision)
-		partial_status |= (SW_Precision | SW_C1);	/* The masked response */
-	else
-		EXCEPTION(EX_Precision | SW_C1);
-}
-
-/* This may be called often, so keep it lean */
-asmlinkage __visible void set_precision_flag_down(void)
-{
-	if (control_word & CW_Precision) {	/* The masked response */
-		partial_status &= ~SW_C1;
-		partial_status |= SW_Precision;
-	} else
-		EXCEPTION(EX_Precision);
-}
-
-asmlinkage __visible int denormal_operand(void)
-{
-	if (control_word & CW_Denormal) {	/* The masked response */
-		partial_status |= SW_Denorm_Op;
-		return TAG_Special;
-	} else {
-		EXCEPTION(EX_Denormal);
-		return TAG_Special | FPU_Exception;
-	}
-}
-
-asmlinkage __visible int arith_overflow(FPU_REG *dest)
-{
-	int tag = TAG_Valid;
-
-	if (control_word & CW_Overflow) {
-		/* The masked response */
-/* ###### The response here depends upon the rounding mode */
-		reg_copy(&CONST_INF, dest);
-		tag = TAG_Special;
-	} else {
-		/* Subtract the magic number from the exponent */
-		addexponent(dest, (-3 * (1 << 13)));
-	}
-
-	EXCEPTION(EX_Overflow);
-	if (control_word & CW_Overflow) {
-		/* The overflow exception is masked. */
-		/* By definition, precision is lost.
-		   The roundup bit (C1) is also set because we have
-		   "rounded" upwards to Infinity. */
-		EXCEPTION(EX_Precision | SW_C1);
-		return tag;
-	}
-
-	return tag;
-
-}
-
-asmlinkage __visible int arith_underflow(FPU_REG *dest)
-{
-	int tag = TAG_Valid;
-
-	if (control_word & CW_Underflow) {
-		/* The masked response */
-		if (exponent16(dest) <= EXP_UNDER - 63) {
-			reg_copy(&CONST_Z, dest);
-			partial_status &= ~SW_C1;	/* Round down. */
-			tag = TAG_Zero;
-		} else {
-			stdexp(dest);
-		}
-	} else {
-		/* Add the magic number to the exponent. */
-		addexponent(dest, (3 * (1 << 13)) + EXTENDED_Ebias);
-	}
-
-	EXCEPTION(EX_Underflow);
-	if (control_word & CW_Underflow) {
-		/* The underflow exception is masked. */
-		EXCEPTION(EX_Precision);
-		return tag;
-	}
-
-	return tag;
-
-}
-
-void FPU_stack_overflow(void)
-{
-
-	if (control_word & CW_Invalid) {
-		/* The masked response */
-		top--;
-		FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
-	}
-
-	EXCEPTION(EX_StackOver);
-
-	return;
-
-}
-
-void FPU_stack_underflow(void)
-{
-
-	if (control_word & CW_Invalid) {
-		/* The masked response */
-		FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
-	}
-
-	EXCEPTION(EX_StackUnder);
-
-	return;
-
-}
-
-void FPU_stack_underflow_i(int i)
-{
-
-	if (control_word & CW_Invalid) {
-		/* The masked response */
-		FPU_copy_to_regi(&CONST_QNaN, TAG_Special, i);
-	}
-
-	EXCEPTION(EX_StackUnder);
-
-	return;
-
-}
-
-void FPU_stack_underflow_pop(int i)
-{
-
-	if (control_word & CW_Invalid) {
-		/* The masked response */
-		FPU_copy_to_regi(&CONST_QNaN, TAG_Special, i);
-		FPU_pop();
-	}
-
-	EXCEPTION(EX_StackUnder);
-
-	return;
-
-}
diff --git a/arch/x86/math-emu/exception.h b/arch/x86/math-emu/exception.h
deleted file mode 100644
index 59961d350bc4d..0000000000000
--- a/arch/x86/math-emu/exception.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  exception.h                                                              |
- |                                                                           |
- | Copyright (C) 1992    W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@vaxc.cc.monash.edu.au    |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#ifndef _EXCEPTION_H_
-#define _EXCEPTION_H_
-
-#ifdef __ASSEMBLER__
-#define	Const_(x)	$##x
-#else
-#define	Const_(x)	x
-#endif
-
-#ifndef SW_C1
-#include "fpu_emu.h"
-#endif /* SW_C1 */
-
-#define FPU_BUSY        Const_(0x8000)	/* FPU busy bit (8087 compatibility) */
-#define EX_ErrorSummary Const_(0x0080)	/* Error summary status */
-/* Special exceptions: */
-#define	EX_INTERNAL	Const_(0x8000)	/* Internal error in wm-FPU-emu */
-#define EX_StackOver	Const_(0x0041|SW_C1)	/* stack overflow */
-#define EX_StackUnder	Const_(0x0041)	/* stack underflow */
-/* Exception flags: */
-#define EX_Precision	Const_(0x0020)	/* loss of precision */
-#define EX_Underflow	Const_(0x0010)	/* underflow */
-#define EX_Overflow	Const_(0x0008)	/* overflow */
-#define EX_ZeroDiv	Const_(0x0004)	/* divide by zero */
-#define EX_Denormal	Const_(0x0002)	/* denormalized operand */
-#define EX_Invalid	Const_(0x0001)	/* invalid operation */
-
-#define PRECISION_LOST_UP    Const_((EX_Precision | SW_C1))
-#define PRECISION_LOST_DOWN  Const_(EX_Precision)
-
-#ifndef __ASSEMBLER__
-
-#ifdef DEBUG
-#define	EXCEPTION(x)	{ printk("exception in %s at line %d\n", \
-	__FILE__, __LINE__); FPU_exception(x); }
-#else
-#define	EXCEPTION(x)	FPU_exception(x)
-#endif
-
-#endif /* __ASSEMBLER__ */
-
-#endif /* _EXCEPTION_H_ */
diff --git a/arch/x86/math-emu/fpu_arith.c b/arch/x86/math-emu/fpu_arith.c
deleted file mode 100644
index 09006dc474a0a..0000000000000
--- a/arch/x86/math-emu/fpu_arith.c
+++ /dev/null
@@ -1,153 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  fpu_arith.c                                                              |
- |                                                                           |
- | Code to implement the FPU register/register arithmetic instructions       |
- |                                                                           |
- | Copyright (C) 1992,1993,1997                                              |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-#include "status_w.h"
-
-void fadd__(void)
-{
-	/* fadd st,st(i) */
-	int i = FPU_rm;
-	clear_C1();
-	FPU_add(&st(i), FPU_gettagi(i), 0, control_word);
-}
-
-void fmul__(void)
-{
-	/* fmul st,st(i) */
-	int i = FPU_rm;
-	clear_C1();
-	FPU_mul(&st(i), FPU_gettagi(i), 0, control_word);
-}
-
-void fsub__(void)
-{
-	/* fsub st,st(i) */
-	clear_C1();
-	FPU_sub(0, FPU_rm, control_word);
-}
-
-void fsubr_(void)
-{
-	/* fsubr st,st(i) */
-	clear_C1();
-	FPU_sub(REV, FPU_rm, control_word);
-}
-
-void fdiv__(void)
-{
-	/* fdiv st,st(i) */
-	clear_C1();
-	FPU_div(0, FPU_rm, control_word);
-}
-
-void fdivr_(void)
-{
-	/* fdivr st,st(i) */
-	clear_C1();
-	FPU_div(REV, FPU_rm, control_word);
-}
-
-void fadd_i(void)
-{
-	/* fadd st(i),st */
-	int i = FPU_rm;
-	clear_C1();
-	FPU_add(&st(i), FPU_gettagi(i), i, control_word);
-}
-
-void fmul_i(void)
-{
-	/* fmul st(i),st */
-	clear_C1();
-	FPU_mul(&st(0), FPU_gettag0(), FPU_rm, control_word);
-}
-
-void fsubri(void)
-{
-	/* fsubr st(i),st */
-	clear_C1();
-	FPU_sub(DEST_RM, FPU_rm, control_word);
-}
-
-void fsub_i(void)
-{
-	/* fsub st(i),st */
-	clear_C1();
-	FPU_sub(REV | DEST_RM, FPU_rm, control_word);
-}
-
-void fdivri(void)
-{
-	/* fdivr st(i),st */
-	clear_C1();
-	FPU_div(DEST_RM, FPU_rm, control_word);
-}
-
-void fdiv_i(void)
-{
-	/* fdiv st(i),st */
-	clear_C1();
-	FPU_div(REV | DEST_RM, FPU_rm, control_word);
-}
-
-void faddp_(void)
-{
-	/* faddp st(i),st */
-	int i = FPU_rm;
-	clear_C1();
-	if (FPU_add(&st(i), FPU_gettagi(i), i, control_word) >= 0)
-		FPU_pop();
-}
-
-void fmulp_(void)
-{
-	/* fmulp st(i),st */
-	clear_C1();
-	if (FPU_mul(&st(0), FPU_gettag0(), FPU_rm, control_word) >= 0)
-		FPU_pop();
-}
-
-void fsubrp(void)
-{
-	/* fsubrp st(i),st */
-	clear_C1();
-	if (FPU_sub(DEST_RM, FPU_rm, control_word) >= 0)
-		FPU_pop();
-}
-
-void fsubp_(void)
-{
-	/* fsubp st(i),st */
-	clear_C1();
-	if (FPU_sub(REV | DEST_RM, FPU_rm, control_word) >= 0)
-		FPU_pop();
-}
-
-void fdivrp(void)
-{
-	/* fdivrp st(i),st */
-	clear_C1();
-	if (FPU_div(DEST_RM, FPU_rm, control_word) >= 0)
-		FPU_pop();
-}
-
-void fdivp_(void)
-{
-	/* fdivp st(i),st */
-	clear_C1();
-	if (FPU_div(REV | DEST_RM, FPU_rm, control_word) >= 0)
-		FPU_pop();
-}
diff --git a/arch/x86/math-emu/fpu_asm.h b/arch/x86/math-emu/fpu_asm.h
deleted file mode 100644
index a83353d5271c2..0000000000000
--- a/arch/x86/math-emu/fpu_asm.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  fpu_asm.h                                                                |
- |                                                                           |
- | Copyright (C) 1992,1995,1997                                              |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@suburbia.net               |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#ifndef _FPU_ASM_H_
-#define _FPU_ASM_H_
-
-#include <linux/linkage.h>
-
-#define	EXCEPTION	FPU_exception
-
-#define PARAM1	8(%ebp)
-#define	PARAM2	12(%ebp)
-#define	PARAM3	16(%ebp)
-#define	PARAM4	20(%ebp)
-#define	PARAM5	24(%ebp)
-#define	PARAM6	28(%ebp)
-#define	PARAM7	32(%ebp)
-
-#define SIGL_OFFSET 0
-#define	EXP(x)	8(x)
-#define SIG(x)	SIGL_OFFSET##(x)
-#define	SIGL(x)	SIGL_OFFSET##(x)
-#define	SIGH(x)	4(x)
-
-#endif /* _FPU_ASM_H_ */
diff --git a/arch/x86/math-emu/fpu_aux.c b/arch/x86/math-emu/fpu_aux.c
deleted file mode 100644
index 5f253ae406b67..0000000000000
--- a/arch/x86/math-emu/fpu_aux.c
+++ /dev/null
@@ -1,267 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  fpu_aux.c                                                                |
- |                                                                           |
- | Code to implement some of the FPU auxiliary instructions.                 |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "control_w.h"
-
-static void fnop(void)
-{
-}
-
-static void fclex(void)
-{
-	partial_status &=
-	    ~(SW_Backward | SW_Summary | SW_Stack_Fault | SW_Precision |
-	      SW_Underflow | SW_Overflow | SW_Zero_Div | SW_Denorm_Op |
-	      SW_Invalid);
-	no_ip_update = 1;
-}
-
-/* Needs to be externally visible */
-void fpstate_init_soft(struct swregs_state *soft)
-{
-	struct address *oaddr, *iaddr;
-	memset(soft, 0, sizeof(*soft));
-	soft->cwd = 0x037f;
-	soft->swd = 0;
-	soft->ftop = 0;	/* We don't keep top in the status word internally. */
-	soft->twd = 0xffff;
-	/* The behaviour is different from that detailed in
-	   Section 15.1.6 of the Intel manual */
-	oaddr = (struct address *)&soft->foo;
-	oaddr->offset = 0;
-	oaddr->selector = 0;
-	iaddr = (struct address *)&soft->fip;
-	iaddr->offset = 0;
-	iaddr->selector = 0;
-	iaddr->opcode = 0;
-	soft->no_update = 1;
-}
-
-void finit(void)
-{
-	fpstate_init_soft(&x86_task_fpu(current)->fpstate->regs.soft);
-}
-
-/*
- * These are nops on the i387..
- */
-#define feni fnop
-#define fdisi fnop
-#define fsetpm fnop
-
-static FUNC const finit_table[] = {
-	feni, fdisi, fclex, finit,
-	fsetpm, FPU_illegal, FPU_illegal, FPU_illegal
-};
-
-void finit_(void)
-{
-	(finit_table[FPU_rm]) ();
-}
-
-static void fstsw_ax(void)
-{
-	*(short *)&FPU_EAX = status_word();
-	no_ip_update = 1;
-}
-
-static FUNC const fstsw_table[] = {
-	fstsw_ax, FPU_illegal, FPU_illegal, FPU_illegal,
-	FPU_illegal, FPU_illegal, FPU_illegal, FPU_illegal
-};
-
-void fstsw_(void)
-{
-	(fstsw_table[FPU_rm]) ();
-}
-
-static FUNC const fp_nop_table[] = {
-	fnop, FPU_illegal, FPU_illegal, FPU_illegal,
-	FPU_illegal, FPU_illegal, FPU_illegal, FPU_illegal
-};
-
-void fp_nop(void)
-{
-	(fp_nop_table[FPU_rm]) ();
-}
-
-void fld_i_(void)
-{
-	FPU_REG *st_new_ptr;
-	int i;
-	u_char tag;
-
-	if (STACK_OVERFLOW) {
-		FPU_stack_overflow();
-		return;
-	}
-
-	/* fld st(i) */
-	i = FPU_rm;
-	if (NOT_EMPTY(i)) {
-		reg_copy(&st(i), st_new_ptr);
-		tag = FPU_gettagi(i);
-		push();
-		FPU_settag0(tag);
-	} else {
-		if (control_word & CW_Invalid) {
-			/* The masked response */
-			FPU_stack_underflow();
-		} else
-			EXCEPTION(EX_StackUnder);
-	}
-
-}
-
-void fxch_i(void)
-{
-	/* fxch st(i) */
-	FPU_REG t;
-	int i = FPU_rm;
-	FPU_REG *st0_ptr = &st(0), *sti_ptr = &st(i);
-	long tag_word = fpu_tag_word;
-	int regnr = top & 7, regnri = ((regnr + i) & 7);
-	u_char st0_tag = (tag_word >> (regnr * 2)) & 3;
-	u_char sti_tag = (tag_word >> (regnri * 2)) & 3;
-
-	if (st0_tag == TAG_Empty) {
-		if (sti_tag == TAG_Empty) {
-			FPU_stack_underflow();
-			FPU_stack_underflow_i(i);
-			return;
-		}
-		if (control_word & CW_Invalid) {
-			/* Masked response */
-			FPU_copy_to_reg0(sti_ptr, sti_tag);
-		}
-		FPU_stack_underflow_i(i);
-		return;
-	}
-	if (sti_tag == TAG_Empty) {
-		if (control_word & CW_Invalid) {
-			/* Masked response */
-			FPU_copy_to_regi(st0_ptr, st0_tag, i);
-		}
-		FPU_stack_underflow();
-		return;
-	}
-	clear_C1();
-
-	reg_copy(st0_ptr, &t);
-	reg_copy(sti_ptr, st0_ptr);
-	reg_copy(&t, sti_ptr);
-
-	tag_word &= ~(3 << (regnr * 2)) & ~(3 << (regnri * 2));
-	tag_word |= (sti_tag << (regnr * 2)) | (st0_tag << (regnri * 2));
-	fpu_tag_word = tag_word;
-}
-
-static void fcmovCC(void)
-{
-	/* fcmovCC st(i) */
-	int i = FPU_rm;
-	FPU_REG *st0_ptr = &st(0);
-	FPU_REG *sti_ptr = &st(i);
-	long tag_word = fpu_tag_word;
-	int regnr = top & 7;
-	int regnri = (top + i) & 7;
-	u_char sti_tag = (tag_word >> (regnri * 2)) & 3;
-
-	if (sti_tag == TAG_Empty) {
-		FPU_stack_underflow();
-		clear_C1();
-		return;
-	}
-	reg_copy(sti_ptr, st0_ptr);
-	tag_word &= ~(3 << (regnr * 2));
-	tag_word |= (sti_tag << (regnr * 2));
-	fpu_tag_word = tag_word;
-}
-
-void fcmovb(void)
-{
-	if (FPU_EFLAGS & X86_EFLAGS_CF)
-		fcmovCC();
-}
-
-void fcmove(void)
-{
-	if (FPU_EFLAGS & X86_EFLAGS_ZF)
-		fcmovCC();
-}
-
-void fcmovbe(void)
-{
-	if (FPU_EFLAGS & (X86_EFLAGS_CF|X86_EFLAGS_ZF))
-		fcmovCC();
-}
-
-void fcmovu(void)
-{
-	if (FPU_EFLAGS & X86_EFLAGS_PF)
-		fcmovCC();
-}
-
-void fcmovnb(void)
-{
-	if (!(FPU_EFLAGS & X86_EFLAGS_CF))
-		fcmovCC();
-}
-
-void fcmovne(void)
-{
-	if (!(FPU_EFLAGS & X86_EFLAGS_ZF))
-		fcmovCC();
-}
-
-void fcmovnbe(void)
-{
-	if (!(FPU_EFLAGS & (X86_EFLAGS_CF|X86_EFLAGS_ZF)))
-		fcmovCC();
-}
-
-void fcmovnu(void)
-{
-	if (!(FPU_EFLAGS & X86_EFLAGS_PF))
-		fcmovCC();
-}
-
-void ffree_(void)
-{
-	/* ffree st(i) */
-	FPU_settagi(FPU_rm, TAG_Empty);
-}
-
-void ffreep(void)
-{
-	/* ffree st(i) + pop - unofficial code */
-	FPU_settagi(FPU_rm, TAG_Empty);
-	FPU_pop();
-}
-
-void fst_i_(void)
-{
-	/* fst st(i) */
-	FPU_copy_to_regi(&st(0), FPU_gettag0(), FPU_rm);
-}
-
-void fstp_i(void)
-{
-	/* fstp st(i) */
-	FPU_copy_to_regi(&st(0), FPU_gettag0(), FPU_rm);
-	FPU_pop();
-}
diff --git a/arch/x86/math-emu/fpu_emu.h b/arch/x86/math-emu/fpu_emu.h
deleted file mode 100644
index def569c50b760..0000000000000
--- a/arch/x86/math-emu/fpu_emu.h
+++ /dev/null
@@ -1,218 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  fpu_emu.h                                                                |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@suburbia.net             |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#ifndef _FPU_EMU_H_
-#define _FPU_EMU_H_
-
-/*
- * Define PECULIAR_486 to get a closer approximation to 80486 behaviour,
- * rather than behaviour which appears to be cleaner.
- * This is a matter of opinion: for all I know, the 80486 may simply
- * be complying with the IEEE spec. Maybe one day I'll get to see the
- * spec...
- */
-#define PECULIAR_486
-
-#ifdef __ASSEMBLER__
-#include "fpu_asm.h"
-#define	Const(x)	$##x
-#else
-#define	Const(x)	x
-#endif
-
-#define EXP_BIAS	Const(0)
-#define EXP_OVER	Const(0x4000)	/* smallest invalid large exponent */
-#define	EXP_UNDER	Const(-0x3fff)	/* largest invalid small exponent */
-#define EXP_WAY_UNDER   Const(-0x6000)	/* Below the smallest denormal, but
-					   still a 16 bit nr. */
-#define EXP_Infinity    EXP_OVER
-#define EXP_NaN         EXP_OVER
-
-#define EXTENDED_Ebias Const(0x3fff)
-#define EXTENDED_Emin (-0x3ffe)	/* smallest valid exponent */
-
-#define SIGN_POS	Const(0)
-#define SIGN_NEG	Const(0x80)
-
-#define SIGN_Positive	Const(0)
-#define SIGN_Negative	Const(0x8000)
-
-/* Keep the order TAG_Valid, TAG_Zero, TW_Denormal */
-/* The following fold to 2 (Special) in the Tag Word */
-#define TW_Denormal     Const(4)	/* De-normal */
-#define TW_Infinity	Const(5)	/* + or - infinity */
-#define	TW_NaN		Const(6)	/* Not a Number */
-#define	TW_Unsupported	Const(7)	/* Not supported by an 80486 */
-
-#define TAG_Valid	Const(0)	/* valid */
-#define TAG_Zero	Const(1)	/* zero */
-#define TAG_Special	Const(2)	/* De-normal, + or - infinity,
-					   or Not a Number */
-#define TAG_Empty	Const(3)	/* empty */
-#define TAG_Error	Const(0x80)	/* probably need to abort */
-
-#define LOADED_DATA	Const(10101)	/* Special st() number to identify
-					   loaded data (not on stack). */
-
-/* A few flags (must be >= 0x10). */
-#define REV             0x10
-#define DEST_RM         0x20
-#define LOADED          0x40
-
-#define FPU_Exception   Const(0x80000000)	/* Added to tag returns. */
-
-#ifndef __ASSEMBLER__
-
-#include "fpu_system.h"
-
-#include <uapi/asm/sigcontext.h>	/* for struct _fpstate */
-#include <asm/math_emu.h>
-#include <linux/linkage.h>
-
-/*
-#define RE_ENTRANT_CHECKING
- */
-
-#ifdef RE_ENTRANT_CHECKING
-extern u_char emulating;
-#  define RE_ENTRANT_CHECK_OFF emulating = 0
-#  define RE_ENTRANT_CHECK_ON emulating = 1
-#else
-#  define RE_ENTRANT_CHECK_OFF
-#  define RE_ENTRANT_CHECK_ON
-#endif /* RE_ENTRANT_CHECKING */
-
-#define FWAIT_OPCODE 0x9b
-#define OP_SIZE_PREFIX 0x66
-#define ADDR_SIZE_PREFIX 0x67
-#define PREFIX_CS 0x2e
-#define PREFIX_DS 0x3e
-#define PREFIX_ES 0x26
-#define PREFIX_SS 0x36
-#define PREFIX_FS 0x64
-#define PREFIX_GS 0x65
-#define PREFIX_REPE 0xf3
-#define PREFIX_REPNE 0xf2
-#define PREFIX_LOCK 0xf0
-#define PREFIX_CS_ 1
-#define PREFIX_DS_ 2
-#define PREFIX_ES_ 3
-#define PREFIX_FS_ 4
-#define PREFIX_GS_ 5
-#define PREFIX_SS_ 6
-#define PREFIX_DEFAULT 7
-
-struct address {
-	unsigned int offset;
-	unsigned int selector:16;
-	unsigned int opcode:11;
-	unsigned int empty:5;
-};
-struct fpu__reg {
-	unsigned sigl;
-	unsigned sigh;
-	short exp;
-};
-
-typedef void (*FUNC) (void);
-typedef struct fpu__reg FPU_REG;
-typedef void (*FUNC_ST0) (FPU_REG *st0_ptr, u_char st0_tag);
-typedef struct {
-	u_char address_size, operand_size, segment;
-} overrides;
-/* This structure is 32 bits: */
-typedef struct {
-	overrides override;
-	u_char default_mode;
-} fpu_addr_modes;
-/* PROTECTED has a restricted meaning in the emulator; it is used
-   to signal that the emulator needs to do special things to ensure
-   that protection is respected in a segmented model. */
-#define PROTECTED 4
-#define SIXTEEN   1		/* We rely upon this being 1 (true) */
-#define VM86      SIXTEEN
-#define PM16      (SIXTEEN | PROTECTED)
-#define SEG32     PROTECTED
-extern u_char const data_sizes_16[32];
-
-#define register_base ((u_char *) registers )
-#define fpu_register(x)  ( * ((FPU_REG *)( register_base + 10 * (x & 7) )) )
-#define	st(x)      ( * ((FPU_REG *)( register_base + 10 * ((top+x) & 7) )) )
-
-#define	STACK_OVERFLOW	(FPU_stackoverflow(&st_new_ptr))
-#define	NOT_EMPTY(i)	(!FPU_empty_i(i))
-
-#define	NOT_EMPTY_ST0	(st0_tag ^ TAG_Empty)
-
-#define poppop() { FPU_pop(); FPU_pop(); }
-
-/* push() does not affect the tags */
-#define push()	{ top--; }
-
-#define signbyte(a) (((u_char *)(a))[9])
-#define getsign(a) (signbyte(a) & 0x80)
-#define setsign(a,b) { if ((b) != 0) signbyte(a) |= 0x80; else signbyte(a) &= 0x7f; }
-#define copysign(a,b) { if (getsign(a)) signbyte(b) |= 0x80; \
-                        else signbyte(b) &= 0x7f; }
-#define changesign(a) { signbyte(a) ^= 0x80; }
-#define setpositive(a) { signbyte(a) &= 0x7f; }
-#define setnegative(a) { signbyte(a) |= 0x80; }
-#define signpositive(a) ( (signbyte(a) & 0x80) == 0 )
-#define signnegative(a) (signbyte(a) & 0x80)
-
-static inline void reg_copy(FPU_REG const *x, FPU_REG *y)
-{
-	*(short *)&(y->exp) = *(const short *)&(x->exp);
-	*(long long *)&(y->sigl) = *(const long long *)&(x->sigl);
-}
-
-#define exponent(x)  (((*(short *)&((x)->exp)) & 0x7fff) - EXTENDED_Ebias)
-#define setexponentpos(x,y) { (*(short *)&((x)->exp)) = \
-  ((y) + EXTENDED_Ebias) & 0x7fff; }
-#define exponent16(x)         (*(short *)&((x)->exp))
-#define setexponent16(x,y)  { (*(short *)&((x)->exp)) = (u16)(y); }
-#define addexponent(x,y)    { (*(short *)&((x)->exp)) += (y); }
-#define stdexp(x)           { (*(short *)&((x)->exp)) += EXTENDED_Ebias; }
-
-#define isdenormal(ptr)   (exponent(ptr) == EXP_BIAS+EXP_UNDER)
-
-#define significand(x) ( ((unsigned long long *)&((x)->sigl))[0] )
-
-/*----- Prototypes for functions written in assembler -----*/
-/* extern void reg_move(FPU_REG *a, FPU_REG *b); */
-
-asmlinkage int FPU_normalize(FPU_REG *x);
-asmlinkage int FPU_normalize_nuo(FPU_REG *x);
-asmlinkage int FPU_u_sub(FPU_REG const *arg1, FPU_REG const *arg2,
-			 FPU_REG * answ, unsigned int control_w, u_char sign,
-			 int expa, int expb);
-asmlinkage int FPU_u_mul(FPU_REG const *arg1, FPU_REG const *arg2,
-			 FPU_REG * answ, unsigned int control_w, u_char sign,
-			 int expon);
-asmlinkage int FPU_u_div(FPU_REG const *arg1, FPU_REG const *arg2,
-			 FPU_REG * answ, unsigned int control_w, u_char sign);
-asmlinkage int FPU_u_add(FPU_REG const *arg1, FPU_REG const *arg2,
-			 FPU_REG * answ, unsigned int control_w, u_char sign,
-			 int expa, int expb);
-asmlinkage int wm_sqrt(FPU_REG *n, int dummy1, int dummy2,
-		       unsigned int control_w, u_char sign);
-asmlinkage unsigned FPU_shrx(void *l, unsigned x);
-asmlinkage unsigned FPU_shrxs(void *v, unsigned x);
-asmlinkage unsigned long FPU_div_small(unsigned long long *x, unsigned long y);
-asmlinkage int FPU_round(FPU_REG *arg, unsigned int extent, int dummy,
-			 unsigned int control_w, u_char sign);
-
-#ifndef MAKING_PROTO
-#include "fpu_proto.h"
-#endif
-
-#endif /* __ASSEMBLER__ */
-
-#endif /* _FPU_EMU_H_ */
diff --git a/arch/x86/math-emu/fpu_entry.c b/arch/x86/math-emu/fpu_entry.c
deleted file mode 100644
index 5034df6177404..0000000000000
--- a/arch/x86/math-emu/fpu_entry.c
+++ /dev/null
@@ -1,718 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  fpu_entry.c                                                              |
- |                                                                           |
- | The entry functions for wm-FPU-emu                                        |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1996,1997                                    |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- | See the files "README" and "COPYING" for further copyright and warranty   |
- | information.                                                              |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note:                                                                     |
- |    The file contains code which accesses user memory.                     |
- |    Emulator static data may change when user memory is accessed, due to   |
- |    other processes using the emulator while swapping is in progress.      |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | math_emulate(), restore_i387_soft() and save_i387_soft() are the only     |
- | entry points for wm-FPU-emu.                                              |
- +---------------------------------------------------------------------------*/
-
-#include <linux/signal.h>
-#include <linux/regset.h>
-
-#include <linux/uaccess.h>
-#include <asm/traps.h>
-#include <asm/user.h>
-#include <asm/fpu/api.h>
-#include <asm/fpu/regset.h>
-
-#include "fpu_system.h"
-#include "fpu_emu.h"
-#include "exception.h"
-#include "control_w.h"
-#include "status_w.h"
-
-#define __BAD__ FPU_illegal	/* Illegal on an 80486, causes SIGILL */
-
-/* fcmovCC and f(u)comi(p) are enabled if CPUID(1).EDX(15) "cmov" is set */
-
-/* WARNING: "u" entries are not documented by Intel in their 80486 manual
-   and may not work on FPU clones or later Intel FPUs.
-   Changes to support them provided by Linus Torvalds. */
-
-static FUNC const st_instr_table[64] = {
-/* Opcode:	d8		d9		da		db */
-/*		dc		dd		de		df */
-/* c0..7 */	fadd__,		fld_i_,		fcmovb,		fcmovnb,
-/* c0..7 */	fadd_i,		ffree_,		faddp_,		ffreep,/*u*/
-/* c8..f */	fmul__,		fxch_i,		fcmove,		fcmovne,
-/* c8..f */	fmul_i,		fxch_i,/*u*/	fmulp_,		fxch_i,/*u*/
-/* d0..7 */	fcom_st,	fp_nop,		fcmovbe,	fcmovnbe,
-/* d0..7 */	fcom_st,/*u*/	fst_i_,		fcompst,/*u*/	fstp_i,/*u*/
-/* d8..f */	fcompst,	fstp_i,/*u*/	fcmovu,		fcmovnu,
-/* d8..f */	fcompst,/*u*/	fstp_i,		fcompp,		fstp_i,/*u*/
-/* e0..7 */	fsub__,		FPU_etc,	__BAD__,	finit_,
-/* e0..7 */	fsubri,		fucom_,		fsubrp,		fstsw_,
-/* e8..f */	fsubr_,		fconst,		fucompp,	fucomi_,
-/* e8..f */	fsub_i,		fucomp,		fsubp_,		fucomip,
-/* f0..7 */	fdiv__,		FPU_triga,	__BAD__,	fcomi_,
-/* f0..7 */	fdivri,		__BAD__,	fdivrp,		fcomip,
-/* f8..f */	fdivr_,		FPU_trigb,	__BAD__,	__BAD__,
-/* f8..f */	fdiv_i,		__BAD__,	fdivp_,		__BAD__,
-};
-
-#define _NONE_ 0		/* Take no special action */
-#define _REG0_ 1		/* Need to check for not empty st(0) */
-#define _REGI_ 2		/* Need to check for not empty st(0) and st(rm) */
-#define _REGi_ 0		/* Uses st(rm) */
-#define _PUSH_ 3		/* Need to check for space to push onto stack */
-#define _null_ 4		/* Function illegal or not implemented */
-#define _REGIi 5		/* Uses st(0) and st(rm), result to st(rm) */
-#define _REGIp 6		/* Uses st(0) and st(rm), result to st(rm) then pop */
-#define _REGIc 0		/* Compare st(0) and st(rm) */
-#define _REGIn 0		/* Uses st(0) and st(rm), but handle checks later */
-
-static u_char const type_table[64] = {
-/* Opcode:	d8	d9	da	db	dc	dd	de	df */
-/* c0..7 */	_REGI_, _NONE_, _REGIn, _REGIn, _REGIi, _REGi_, _REGIp, _REGi_,
-/* c8..f */	_REGI_, _REGIn, _REGIn, _REGIn, _REGIi, _REGI_, _REGIp, _REGI_,
-/* d0..7 */	_REGIc, _NONE_, _REGIn, _REGIn, _REGIc, _REG0_, _REGIc, _REG0_,
-/* d8..f */	_REGIc, _REG0_, _REGIn, _REGIn, _REGIc, _REG0_, _REGIc, _REG0_,
-/* e0..7 */	_REGI_, _NONE_, _null_, _NONE_, _REGIi, _REGIc, _REGIp, _NONE_,
-/* e8..f */	_REGI_, _NONE_, _REGIc, _REGIc, _REGIi, _REGIc, _REGIp, _REGIc,
-/* f0..7 */	_REGI_, _NONE_, _null_, _REGIc, _REGIi, _null_, _REGIp, _REGIc,
-/* f8..f */	_REGI_, _NONE_, _null_, _null_, _REGIi, _null_, _REGIp, _null_,
-};
-
-#ifdef RE_ENTRANT_CHECKING
-u_char emulating = 0;
-#endif /* RE_ENTRANT_CHECKING */
-
-static int valid_prefix(u_char *Byte, u_char __user ** fpu_eip,
-			overrides * override);
-
-void math_emulate(struct math_emu_info *info)
-{
-	u_char FPU_modrm, byte1;
-	unsigned short code;
-	fpu_addr_modes addr_modes;
-	int unmasked;
-	FPU_REG loaded_data;
-	FPU_REG *st0_ptr;
-	u_char loaded_tag, st0_tag;
-	void __user *data_address;
-	struct address data_sel_off;
-	struct address entry_sel_off;
-	unsigned long code_base = 0;
-	unsigned long code_limit = 0;	/* Initialized to stop compiler warnings */
-	struct desc_struct code_descriptor;
-
-#ifdef RE_ENTRANT_CHECKING
-	if (emulating) {
-		printk("ERROR: wm-FPU-emu is not RE-ENTRANT!\n");
-	}
-	RE_ENTRANT_CHECK_ON;
-#endif /* RE_ENTRANT_CHECKING */
-
-	FPU_info = info;
-
-	FPU_ORIG_EIP = FPU_EIP;
-
-	if ((FPU_EFLAGS & 0x00020000) != 0) {
-		/* Virtual 8086 mode */
-		addr_modes.default_mode = VM86;
-		FPU_EIP += code_base = FPU_CS << 4;
-		code_limit = code_base + 0xffff;	/* Assumes code_base <= 0xffff0000 */
-	} else if (FPU_CS == __USER_CS && FPU_DS == __USER_DS) {
-		addr_modes.default_mode = 0;
-	} else if (FPU_CS == __KERNEL_CS) {
-		printk("math_emulate: %04x:%08lx\n", FPU_CS, FPU_EIP);
-		panic("Math emulation needed in kernel");
-	} else {
-
-		if ((FPU_CS & 4) != 4) {	/* Must be in the LDT */
-			/* Can only handle segmented addressing via the LDT
-			   for now, and it must be 16 bit */
-			printk("FPU emulator: Unsupported addressing mode\n");
-			math_abort(FPU_info, SIGILL);
-		}
-
-		code_descriptor = FPU_get_ldt_descriptor(FPU_CS);
-		if (code_descriptor.d) {
-			/* The above test may be wrong, the book is not clear */
-			/* Segmented 32 bit protected mode */
-			addr_modes.default_mode = SEG32;
-		} else {
-			/* 16 bit protected mode */
-			addr_modes.default_mode = PM16;
-		}
-		FPU_EIP += code_base = seg_get_base(&code_descriptor);
-		code_limit = seg_get_limit(&code_descriptor) + 1;
-		code_limit *= seg_get_granularity(&code_descriptor);
-		code_limit += code_base - 1;
-		if (code_limit < code_base)
-			code_limit = 0xffffffff;
-	}
-
-	FPU_lookahead = !(FPU_EFLAGS & X86_EFLAGS_TF);
-
-	if (!valid_prefix(&byte1, (u_char __user **) & FPU_EIP,
-			  &addr_modes.override)) {
-		RE_ENTRANT_CHECK_OFF;
-		printk
-		    ("FPU emulator: Unknown prefix byte 0x%02x, probably due to\n"
-		     "FPU emulator: self-modifying code! (emulation impossible)\n",
-		     byte1);
-		RE_ENTRANT_CHECK_ON;
-		EXCEPTION(EX_INTERNAL | 0x126);
-		math_abort(FPU_info, SIGILL);
-	}
-
-      do_another_FPU_instruction:
-
-	no_ip_update = 0;
-
-	FPU_EIP++;		/* We have fetched the prefix and first code bytes. */
-
-	if (addr_modes.default_mode) {
-		/* This checks for the minimum instruction bytes.
-		   We also need to check any extra (address mode) code access. */
-		if (FPU_EIP > code_limit)
-			math_abort(FPU_info, SIGSEGV);
-	}
-
-	if ((byte1 & 0xf8) != 0xd8) {
-		if (byte1 == FWAIT_OPCODE) {
-			if (partial_status & SW_Summary)
-				goto do_the_FPU_interrupt;
-			else
-				goto FPU_fwait_done;
-		}
-#ifdef PARANOID
-		EXCEPTION(EX_INTERNAL | 0x128);
-		math_abort(FPU_info, SIGILL);
-#endif /* PARANOID */
-	}
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_code_access_ok(1);
-	FPU_get_user(FPU_modrm, (u_char __user *) FPU_EIP);
-	RE_ENTRANT_CHECK_ON;
-	FPU_EIP++;
-
-	if (partial_status & SW_Summary) {
-		/* Ignore the error for now if the current instruction is a no-wait
-		   control instruction */
-		/* The 80486 manual contradicts itself on this topic,
-		   but a real 80486 uses the following instructions:
-		   fninit, fnstenv, fnsave, fnstsw, fnstenv, fnclex.
-		 */
-		code = (FPU_modrm << 8) | byte1;
-		if (!((((code & 0xf803) == 0xe003) ||	/* fnclex, fninit, fnstsw */
-		       (((code & 0x3003) == 0x3001) &&	/* fnsave, fnstcw, fnstenv,
-							   fnstsw */
-			((code & 0xc000) != 0xc000))))) {
-			/*
-			 *  We need to simulate the action of the kernel to FPU
-			 *  interrupts here.
-			 */
-		      do_the_FPU_interrupt:
-
-			FPU_EIP = FPU_ORIG_EIP;	/* Point to current FPU instruction. */
-
-			RE_ENTRANT_CHECK_OFF;
-			current->thread.trap_nr = X86_TRAP_MF;
-			current->thread.error_code = 0;
-			send_sig(SIGFPE, current, 1);
-			return;
-		}
-	}
-
-	entry_sel_off.offset = FPU_ORIG_EIP;
-	entry_sel_off.selector = FPU_CS;
-	entry_sel_off.opcode = (byte1 << 8) | FPU_modrm;
-	entry_sel_off.empty = 0;
-
-	FPU_rm = FPU_modrm & 7;
-
-	if (FPU_modrm < 0300) {
-		/* All of these instructions use the mod/rm byte to get a data address */
-
-		if ((addr_modes.default_mode & SIXTEEN)
-		    ^ (addr_modes.override.address_size == ADDR_SIZE_PREFIX))
-			data_address =
-			    FPU_get_address_16(FPU_modrm, &FPU_EIP,
-					       &data_sel_off, addr_modes);
-		else
-			data_address =
-			    FPU_get_address(FPU_modrm, &FPU_EIP, &data_sel_off,
-					    addr_modes);
-
-		if (addr_modes.default_mode) {
-			if (FPU_EIP - 1 > code_limit)
-				math_abort(FPU_info, SIGSEGV);
-		}
-
-		if (!(byte1 & 1)) {
-			unsigned short status1 = partial_status;
-
-			st0_ptr = &st(0);
-			st0_tag = FPU_gettag0();
-
-			/* Stack underflow has priority */
-			if (NOT_EMPTY_ST0) {
-				if (addr_modes.default_mode & PROTECTED) {
-					/* This table works for 16 and 32 bit protected mode */
-					if (access_limit <
-					    data_sizes_16[(byte1 >> 1) & 3])
-						math_abort(FPU_info, SIGSEGV);
-				}
-
-				unmasked = 0;	/* Do this here to stop compiler warnings. */
-				switch ((byte1 >> 1) & 3) {
-				case 0:
-					unmasked =
-					    FPU_load_single((float __user *)
-							    data_address,
-							    &loaded_data);
-					loaded_tag = unmasked & 0xff;
-					unmasked &= ~0xff;
-					break;
-				case 1:
-					loaded_tag =
-					    FPU_load_int32((long __user *)
-							   data_address,
-							   &loaded_data);
-					break;
-				case 2:
-					unmasked =
-					    FPU_load_double((double __user *)
-							    data_address,
-							    &loaded_data);
-					loaded_tag = unmasked & 0xff;
-					unmasked &= ~0xff;
-					break;
-				case 3:
-				default:	/* Used here to suppress gcc warnings. */
-					loaded_tag =
-					    FPU_load_int16((short __user *)
-							   data_address,
-							   &loaded_data);
-					break;
-				}
-
-				/* No more access to user memory, it is safe
-				   to use static data now */
-
-				/* NaN operands have the next priority. */
-				/* We have to delay looking at st(0) until after
-				   loading the data, because that data might contain an SNaN */
-				if (((st0_tag == TAG_Special) && isNaN(st0_ptr))
-				    || ((loaded_tag == TAG_Special)
-					&& isNaN(&loaded_data))) {
-					/* Restore the status word; we might have loaded a
-					   denormal. */
-					partial_status = status1;
-					if ((FPU_modrm & 0x30) == 0x10) {
-						/* fcom or fcomp */
-						EXCEPTION(EX_Invalid);
-						setcc(SW_C3 | SW_C2 | SW_C0);
-						if ((FPU_modrm & 0x08)
-						    && (control_word &
-							CW_Invalid))
-							FPU_pop();	/* fcomp, masked, so we pop. */
-					} else {
-						if (loaded_tag == TAG_Special)
-							loaded_tag =
-							    FPU_Special
-							    (&loaded_data);
-#ifdef PECULIAR_486
-						/* This is not really needed, but gives behaviour
-						   identical to an 80486 */
-						if ((FPU_modrm & 0x28) == 0x20)
-							/* fdiv or fsub */
-							real_2op_NaN
-							    (&loaded_data,
-							     loaded_tag, 0,
-							     &loaded_data);
-						else
-#endif /* PECULIAR_486 */
-							/* fadd, fdivr, fmul, or fsubr */
-							real_2op_NaN
-							    (&loaded_data,
-							     loaded_tag, 0,
-							     st0_ptr);
-					}
-					goto reg_mem_instr_done;
-				}
-
-				if (unmasked && !((FPU_modrm & 0x30) == 0x10)) {
-					/* Is not a comparison instruction. */
-					if ((FPU_modrm & 0x38) == 0x38) {
-						/* fdivr */
-						if ((st0_tag == TAG_Zero) &&
-						    ((loaded_tag == TAG_Valid)
-						     || (loaded_tag ==
-							 TAG_Special
-							 &&
-							 isdenormal
-							 (&loaded_data)))) {
-							if (FPU_divide_by_zero
-							    (0,
-							     getsign
-							     (&loaded_data))
-							    < 0) {
-								/* We use the fact here that the unmasked
-								   exception in the loaded data was for a
-								   denormal operand */
-								/* Restore the state of the denormal op bit */
-								partial_status
-								    &=
-								    ~SW_Denorm_Op;
-								partial_status
-								    |=
-								    status1 &
-								    SW_Denorm_Op;
-							} else
-								setsign(st0_ptr,
-									getsign
-									(&loaded_data));
-						}
-					}
-					goto reg_mem_instr_done;
-				}
-
-				switch ((FPU_modrm >> 3) & 7) {
-				case 0:	/* fadd */
-					clear_C1();
-					FPU_add(&loaded_data, loaded_tag, 0,
-						control_word);
-					break;
-				case 1:	/* fmul */
-					clear_C1();
-					FPU_mul(&loaded_data, loaded_tag, 0,
-						control_word);
-					break;
-				case 2:	/* fcom */
-					FPU_compare_st_data(&loaded_data,
-							    loaded_tag);
-					break;
-				case 3:	/* fcomp */
-					if (!FPU_compare_st_data
-					    (&loaded_data, loaded_tag)
-					    && !unmasked)
-						FPU_pop();
-					break;
-				case 4:	/* fsub */
-					clear_C1();
-					FPU_sub(LOADED | loaded_tag,
-						(int)&loaded_data,
-						control_word);
-					break;
-				case 5:	/* fsubr */
-					clear_C1();
-					FPU_sub(REV | LOADED | loaded_tag,
-						(int)&loaded_data,
-						control_word);
-					break;
-				case 6:	/* fdiv */
-					clear_C1();
-					FPU_div(LOADED | loaded_tag,
-						(int)&loaded_data,
-						control_word);
-					break;
-				case 7:	/* fdivr */
-					clear_C1();
-					if (st0_tag == TAG_Zero)
-						partial_status = status1;	/* Undo any denorm tag,
-										   zero-divide has priority. */
-					FPU_div(REV | LOADED | loaded_tag,
-						(int)&loaded_data,
-						control_word);
-					break;
-				}
-			} else {
-				if ((FPU_modrm & 0x30) == 0x10) {
-					/* The instruction is fcom or fcomp */
-					EXCEPTION(EX_StackUnder);
-					setcc(SW_C3 | SW_C2 | SW_C0);
-					if ((FPU_modrm & 0x08)
-					    && (control_word & CW_Invalid))
-						FPU_pop();	/* fcomp */
-				} else
-					FPU_stack_underflow();
-			}
-		      reg_mem_instr_done:
-			operand_address = data_sel_off;
-		} else {
-			if (!(no_ip_update =
-			      FPU_load_store(((FPU_modrm & 0x38) | (byte1 & 6))
-					     >> 1, addr_modes, data_address))) {
-				operand_address = data_sel_off;
-			}
-		}
-
-	} else {
-		/* None of these instructions access user memory */
-		u_char instr_index = (FPU_modrm & 0x38) | (byte1 & 7);
-
-#ifdef PECULIAR_486
-		/* This is supposed to be undefined, but a real 80486 seems
-		   to do this: */
-		operand_address.offset = 0;
-		operand_address.selector = FPU_DS;
-#endif /* PECULIAR_486 */
-
-		st0_ptr = &st(0);
-		st0_tag = FPU_gettag0();
-		switch (type_table[(int)instr_index]) {
-		case _NONE_:	/* also _REGIc: _REGIn */
-			break;
-		case _REG0_:
-			if (!NOT_EMPTY_ST0) {
-				FPU_stack_underflow();
-				goto FPU_instruction_done;
-			}
-			break;
-		case _REGIi:
-			if (!NOT_EMPTY_ST0 || !NOT_EMPTY(FPU_rm)) {
-				FPU_stack_underflow_i(FPU_rm);
-				goto FPU_instruction_done;
-			}
-			break;
-		case _REGIp:
-			if (!NOT_EMPTY_ST0 || !NOT_EMPTY(FPU_rm)) {
-				FPU_stack_underflow_pop(FPU_rm);
-				goto FPU_instruction_done;
-			}
-			break;
-		case _REGI_:
-			if (!NOT_EMPTY_ST0 || !NOT_EMPTY(FPU_rm)) {
-				FPU_stack_underflow();
-				goto FPU_instruction_done;
-			}
-			break;
-		case _PUSH_:	/* Only used by the fld st(i) instruction */
-			break;
-		case _null_:
-			FPU_illegal();
-			goto FPU_instruction_done;
-		default:
-			EXCEPTION(EX_INTERNAL | 0x111);
-			goto FPU_instruction_done;
-		}
-		(*st_instr_table[(int)instr_index]) ();
-
-	      FPU_instruction_done:
-		;
-	}
-
-	if (!no_ip_update)
-		instruction_address = entry_sel_off;
-
-      FPU_fwait_done:
-
-#ifdef DEBUG
-	RE_ENTRANT_CHECK_OFF;
-	FPU_printall();
-	RE_ENTRANT_CHECK_ON;
-#endif /* DEBUG */
-
-	if (FPU_lookahead && !need_resched()) {
-		FPU_ORIG_EIP = FPU_EIP - code_base;
-		if (valid_prefix(&byte1, (u_char __user **) & FPU_EIP,
-				 &addr_modes.override))
-			goto do_another_FPU_instruction;
-	}
-
-	if (addr_modes.default_mode)
-		FPU_EIP -= code_base;
-
-	RE_ENTRANT_CHECK_OFF;
-}
-
-/* Support for prefix bytes is not yet complete. To properly handle
-   all prefix bytes, further changes are needed in the emulator code
-   which accesses user address space. Access to separate segments is
-   important for msdos emulation. */
-static int valid_prefix(u_char *Byte, u_char __user **fpu_eip,
-			overrides * override)
-{
-	u_char byte;
-	u_char __user *ip = *fpu_eip;
-
-	*override = (overrides) {
-	0, 0, PREFIX_DEFAULT};	/* defaults */
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_code_access_ok(1);
-	FPU_get_user(byte, ip);
-	RE_ENTRANT_CHECK_ON;
-
-	while (1) {
-		switch (byte) {
-		case ADDR_SIZE_PREFIX:
-			override->address_size = ADDR_SIZE_PREFIX;
-			goto do_next_byte;
-
-		case OP_SIZE_PREFIX:
-			override->operand_size = OP_SIZE_PREFIX;
-			goto do_next_byte;
-
-		case PREFIX_CS:
-			override->segment = PREFIX_CS_;
-			goto do_next_byte;
-		case PREFIX_ES:
-			override->segment = PREFIX_ES_;
-			goto do_next_byte;
-		case PREFIX_SS:
-			override->segment = PREFIX_SS_;
-			goto do_next_byte;
-		case PREFIX_FS:
-			override->segment = PREFIX_FS_;
-			goto do_next_byte;
-		case PREFIX_GS:
-			override->segment = PREFIX_GS_;
-			goto do_next_byte;
-		case PREFIX_DS:
-			override->segment = PREFIX_DS_;
-			goto do_next_byte;
-
-/* lock is not a valid prefix for FPU instructions,
-   let the cpu handle it to generate a SIGILL. */
-/*	case PREFIX_LOCK: */
-
-			/* rep.. prefixes have no meaning for FPU instructions */
-		case PREFIX_REPE:
-		case PREFIX_REPNE:
-
-		      do_next_byte:
-			ip++;
-			RE_ENTRANT_CHECK_OFF;
-			FPU_code_access_ok(1);
-			FPU_get_user(byte, ip);
-			RE_ENTRANT_CHECK_ON;
-			break;
-		case FWAIT_OPCODE:
-			*Byte = byte;
-			return 1;
-		default:
-			if ((byte & 0xf8) == 0xd8) {
-				*Byte = byte;
-				*fpu_eip = ip;
-				return 1;
-			} else {
-				/* Not a valid sequence of prefix bytes followed by
-				   an FPU instruction. */
-				*Byte = byte;	/* Needed for error message. */
-				return 0;
-			}
-		}
-	}
-}
-
-void math_abort(struct math_emu_info *info, unsigned int signal)
-{
-	FPU_EIP = FPU_ORIG_EIP;
-	current->thread.trap_nr = X86_TRAP_MF;
-	current->thread.error_code = 0;
-	send_sig(signal, current, 1);
-	RE_ENTRANT_CHECK_OFF;
-      __asm__("movl %0,%%esp ; ret": :"g"(((long)info) - 4));
-#ifdef PARANOID
-	printk("ERROR: wm-FPU-emu math_abort failed!\n");
-#endif /* PARANOID */
-}
-
-#define S387 ((struct swregs_state *)s387)
-#define sstatus_word() \
-  ((S387->swd & ~SW_Top & 0xffff) | ((S387->ftop << SW_Top_Shift) & SW_Top))
-
-int fpregs_soft_set(struct task_struct *target,
-		    const struct user_regset *regset,
-		    unsigned int pos, unsigned int count,
-		    const void *kbuf, const void __user *ubuf)
-{
-	struct swregs_state *s387 = &x86_task_fpu(target)->fpstate->regs.soft;
-	void *space = s387->st_space;
-	int ret;
-	int offset, other, i, tags, regnr, tag, newtop;
-
-	RE_ENTRANT_CHECK_OFF;
-	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, s387, 0,
-				 offsetof(struct swregs_state, st_space));
-	RE_ENTRANT_CHECK_ON;
-
-	if (ret)
-		return ret;
-
-	S387->ftop = (S387->swd >> SW_Top_Shift) & 7;
-	offset = (S387->ftop & 7) * 10;
-	other = 80 - offset;
-
-	RE_ENTRANT_CHECK_OFF;
-
-	/* Copy all registers in stack order. */
-	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
-				 space + offset, 0, other);
-	if (!ret && offset)
-		ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
-					 space, 0, offset);
-
-	RE_ENTRANT_CHECK_ON;
-
-	/* The tags may need to be corrected now. */
-	tags = S387->twd;
-	newtop = S387->ftop;
-	for (i = 0; i < 8; i++) {
-		regnr = (i + newtop) & 7;
-		if (((tags >> ((regnr & 7) * 2)) & 3) != TAG_Empty) {
-			/* The loaded data over-rides all other cases. */
-			tag =
-			    FPU_tagof((FPU_REG *) ((u_char *) S387->st_space +
-						   10 * regnr));
-			tags &= ~(3 << (regnr * 2));
-			tags |= (tag & 3) << (regnr * 2);
-		}
-	}
-	S387->twd = tags;
-
-	return ret;
-}
-
-int fpregs_soft_get(struct task_struct *target,
-		    const struct user_regset *regset,
-		    struct membuf to)
-{
-	struct swregs_state *s387 = &x86_task_fpu(target)->fpstate->regs.soft;
-	const void *space = s387->st_space;
-	int offset = (S387->ftop & 7) * 10, other = 80 - offset;
-
-	RE_ENTRANT_CHECK_OFF;
-
-#ifdef PECULIAR_486
-	S387->cwd &= ~0xe080;
-	/* An 80486 sets nearly all of the reserved bits to 1. */
-	S387->cwd |= 0xffff0040;
-	S387->swd = sstatus_word() | 0xffff0000;
-	S387->twd |= 0xffff0000;
-	S387->fcs &= ~0xf8000000;
-	S387->fos |= 0xffff0000;
-#endif /* PECULIAR_486 */
-
-	membuf_write(&to, s387, offsetof(struct swregs_state, st_space));
-	membuf_write(&to, space + offset, other);
-	membuf_write(&to, space, offset);
-
-	RE_ENTRANT_CHECK_ON;
-
-	return 0;
-}
diff --git a/arch/x86/math-emu/fpu_etc.c b/arch/x86/math-emu/fpu_etc.c
deleted file mode 100644
index 39423ec409e18..0000000000000
--- a/arch/x86/math-emu/fpu_etc.c
+++ /dev/null
@@ -1,136 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  fpu_etc.c                                                                |
- |                                                                           |
- | Implement a few FPU instructions.                                         |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@suburbia.net             |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "reg_constant.h"
-
-static void fchs(FPU_REG *st0_ptr, u_char st0tag)
-{
-	if (st0tag ^ TAG_Empty) {
-		signbyte(st0_ptr) ^= SIGN_NEG;
-		clear_C1();
-	} else
-		FPU_stack_underflow();
-}
-
-static void fabs(FPU_REG *st0_ptr, u_char st0tag)
-{
-	if (st0tag ^ TAG_Empty) {
-		setpositive(st0_ptr);
-		clear_C1();
-	} else
-		FPU_stack_underflow();
-}
-
-static void ftst_(FPU_REG *st0_ptr, u_char st0tag)
-{
-	switch (st0tag) {
-	case TAG_Zero:
-		setcc(SW_C3);
-		break;
-	case TAG_Valid:
-		if (getsign(st0_ptr) == SIGN_POS)
-			setcc(0);
-		else
-			setcc(SW_C0);
-		break;
-	case TAG_Special:
-		switch (FPU_Special(st0_ptr)) {
-		case TW_Denormal:
-			if (getsign(st0_ptr) == SIGN_POS)
-				setcc(0);
-			else
-				setcc(SW_C0);
-			if (denormal_operand() < 0) {
-#ifdef PECULIAR_486
-				/* This is weird! */
-				if (getsign(st0_ptr) == SIGN_POS)
-					setcc(SW_C3);
-#endif /* PECULIAR_486 */
-				return;
-			}
-			break;
-		case TW_NaN:
-			setcc(SW_C0 | SW_C2 | SW_C3);	/* Operand is not comparable */
-			EXCEPTION(EX_Invalid);
-			break;
-		case TW_Infinity:
-			if (getsign(st0_ptr) == SIGN_POS)
-				setcc(0);
-			else
-				setcc(SW_C0);
-			break;
-		default:
-			setcc(SW_C0 | SW_C2 | SW_C3);	/* Operand is not comparable */
-			EXCEPTION(EX_INTERNAL | 0x14);
-			break;
-		}
-		break;
-	case TAG_Empty:
-		setcc(SW_C0 | SW_C2 | SW_C3);
-		EXCEPTION(EX_StackUnder);
-		break;
-	}
-}
-
-static void fxam(FPU_REG *st0_ptr, u_char st0tag)
-{
-	int c = 0;
-	switch (st0tag) {
-	case TAG_Empty:
-		c = SW_C3 | SW_C0;
-		break;
-	case TAG_Zero:
-		c = SW_C3;
-		break;
-	case TAG_Valid:
-		c = SW_C2;
-		break;
-	case TAG_Special:
-		switch (FPU_Special(st0_ptr)) {
-		case TW_Denormal:
-			c = SW_C2 | SW_C3;	/* Denormal */
-			break;
-		case TW_NaN:
-			/* We also use NaN for unsupported types. */
-			if ((st0_ptr->sigh & 0x80000000)
-			    && (exponent(st0_ptr) == EXP_OVER))
-				c = SW_C0;
-			break;
-		case TW_Infinity:
-			c = SW_C2 | SW_C0;
-			break;
-		}
-	}
-	if (getsign(st0_ptr) == SIGN_NEG)
-		c |= SW_C1;
-	setcc(c);
-}
-
-static void FPU_ST0_illegal(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	FPU_illegal();
-}
-
-static FUNC_ST0 const fp_etc_table[] = {
-	fchs, fabs, FPU_ST0_illegal, FPU_ST0_illegal,
-	ftst_, fxam, FPU_ST0_illegal, FPU_ST0_illegal,
-};
-
-void FPU_etc(void)
-{
-	(fp_etc_table[FPU_rm]) (&st(0), FPU_gettag0());
-}
diff --git a/arch/x86/math-emu/fpu_proto.h b/arch/x86/math-emu/fpu_proto.h
deleted file mode 100644
index 94c4023092f3a..0000000000000
--- a/arch/x86/math-emu/fpu_proto.h
+++ /dev/null
@@ -1,157 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _FPU_PROTO_H
-#define _FPU_PROTO_H
-
-/* errors.c */
-extern void FPU_illegal(void);
-extern void FPU_printall(void);
-asmlinkage void FPU_exception(int n);
-extern int real_1op_NaN(FPU_REG *a);
-extern int real_2op_NaN(FPU_REG const *b, u_char tagb, int deststnr,
-			FPU_REG const *defaultNaN);
-asmlinkage int arith_invalid(int deststnr);
-asmlinkage int FPU_divide_by_zero(int deststnr, u_char sign);
-extern int set_precision_flag(int flags);
-asmlinkage void set_precision_flag_up(void);
-asmlinkage void set_precision_flag_down(void);
-asmlinkage int denormal_operand(void);
-asmlinkage int arith_overflow(FPU_REG *dest);
-asmlinkage int arith_underflow(FPU_REG *dest);
-extern void FPU_stack_overflow(void);
-extern void FPU_stack_underflow(void);
-extern void FPU_stack_underflow_i(int i);
-extern void FPU_stack_underflow_pop(int i);
-/* fpu_arith.c */
-extern void fadd__(void);
-extern void fmul__(void);
-extern void fsub__(void);
-extern void fsubr_(void);
-extern void fdiv__(void);
-extern void fdivr_(void);
-extern void fadd_i(void);
-extern void fmul_i(void);
-extern void fsubri(void);
-extern void fsub_i(void);
-extern void fdivri(void);
-extern void fdiv_i(void);
-extern void faddp_(void);
-extern void fmulp_(void);
-extern void fsubrp(void);
-extern void fsubp_(void);
-extern void fdivrp(void);
-extern void fdivp_(void);
-/* fpu_aux.c */
-extern void finit(void);
-extern void finit_(void);
-extern void fstsw_(void);
-extern void fp_nop(void);
-extern void fld_i_(void);
-extern void fxch_i(void);
-extern void fcmovb(void);
-extern void fcmove(void);
-extern void fcmovbe(void);
-extern void fcmovu(void);
-extern void fcmovnb(void);
-extern void fcmovne(void);
-extern void fcmovnbe(void);
-extern void fcmovnu(void);
-extern void ffree_(void);
-extern void ffreep(void);
-extern void fst_i_(void);
-extern void fstp_i(void);
-/* fpu_entry.c */
-extern void math_emulate(struct math_emu_info *info);
-extern void math_abort(struct math_emu_info *info, unsigned int signal);
-/* fpu_etc.c */
-extern void FPU_etc(void);
-/* fpu_tags.c */
-extern int FPU_gettag0(void);
-extern int FPU_gettagi(int stnr);
-extern int FPU_gettag(int regnr);
-extern void FPU_settag0(int tag);
-extern void FPU_settagi(int stnr, int tag);
-extern void FPU_settag(int regnr, int tag);
-extern int FPU_Special(FPU_REG const *ptr);
-extern int isNaN(FPU_REG const *ptr);
-extern void FPU_pop(void);
-extern int FPU_empty_i(int stnr);
-extern int FPU_stackoverflow(FPU_REG ** st_new_ptr);
-extern void FPU_copy_to_regi(FPU_REG const *r, u_char tag, int stnr);
-extern void FPU_copy_to_reg1(FPU_REG const *r, u_char tag);
-extern void FPU_copy_to_reg0(FPU_REG const *r, u_char tag);
-/* fpu_trig.c */
-extern void FPU_triga(void);
-extern void FPU_trigb(void);
-/* get_address.c */
-extern void __user *FPU_get_address(u_char FPU_modrm, unsigned long *fpu_eip,
-				    struct address *addr,
-				    fpu_addr_modes addr_modes);
-extern void __user *FPU_get_address_16(u_char FPU_modrm, unsigned long *fpu_eip,
-				       struct address *addr,
-				       fpu_addr_modes addr_modes);
-/* load_store.c */
-extern int FPU_load_store(u_char type, fpu_addr_modes addr_modes,
-			  void __user * data_address);
-/* poly_2xm1.c */
-extern int poly_2xm1(u_char sign, FPU_REG * arg, FPU_REG *result);
-/* poly_atan.c */
-extern void poly_atan(FPU_REG * st0_ptr, u_char st0_tag, FPU_REG *st1_ptr,
-		      u_char st1_tag);
-/* poly_l2.c */
-extern void poly_l2(FPU_REG *st0_ptr, FPU_REG *st1_ptr, u_char st1_sign);
-extern int poly_l2p1(u_char s0, u_char s1, FPU_REG *r0, FPU_REG *r1,
-		     FPU_REG * d);
-/* poly_sin.c */
-extern void poly_sine(FPU_REG *st0_ptr);
-extern void poly_cos(FPU_REG *st0_ptr);
-/* poly_tan.c */
-extern void poly_tan(FPU_REG *st0_ptr);
-/* reg_add_sub.c */
-extern int FPU_add(FPU_REG const *b, u_char tagb, int destrnr, int control_w);
-extern int FPU_sub(int flags, int rm, int control_w);
-/* reg_compare.c */
-extern int FPU_compare_st_data(FPU_REG const *loaded_data, u_char loaded_tag);
-extern void fcom_st(void);
-extern void fcompst(void);
-extern void fcompp(void);
-extern void fucom_(void);
-extern void fucomp(void);
-extern void fucompp(void);
-extern void fcomi_(void);
-extern void fcomip(void);
-extern void fucomi_(void);
-extern void fucomip(void);
-/* reg_constant.c */
-extern void fconst(void);
-/* reg_ld_str.c */
-extern int FPU_load_extended(long double __user *s, int stnr);
-extern int FPU_load_double(double __user *dfloat, FPU_REG *loaded_data);
-extern int FPU_load_single(float __user *single, FPU_REG *loaded_data);
-extern int FPU_load_int64(long long __user *_s);
-extern int FPU_load_int32(long __user *_s, FPU_REG *loaded_data);
-extern int FPU_load_int16(short __user *_s, FPU_REG *loaded_data);
-extern int FPU_load_bcd(u_char __user *s);
-extern int FPU_store_extended(FPU_REG *st0_ptr, u_char st0_tag,
-			      long double __user * d);
-extern int FPU_store_double(FPU_REG *st0_ptr, u_char st0_tag,
-			    double __user * dfloat);
-extern int FPU_store_single(FPU_REG *st0_ptr, u_char st0_tag,
-			    float __user * single);
-extern int FPU_store_int64(FPU_REG *st0_ptr, u_char st0_tag,
-			   long long __user * d);
-extern int FPU_store_int32(FPU_REG *st0_ptr, u_char st0_tag, long __user *d);
-extern int FPU_store_int16(FPU_REG *st0_ptr, u_char st0_tag, short __user *d);
-extern int FPU_store_bcd(FPU_REG *st0_ptr, u_char st0_tag, u_char __user *d);
-extern int FPU_round_to_int(FPU_REG *r, u_char tag);
-extern u_char __user *fldenv(fpu_addr_modes addr_modes, u_char __user *s);
-extern void FPU_frstor(fpu_addr_modes addr_modes, u_char __user *data_address);
-extern u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d);
-extern void fsave(fpu_addr_modes addr_modes, u_char __user *data_address);
-extern int FPU_tagof(FPU_REG *ptr);
-/* reg_mul.c */
-extern int FPU_mul(FPU_REG const *b, u_char tagb, int deststnr, int control_w);
-
-extern int FPU_div(int flags, int regrm, int control_w);
-/* reg_convert.c */
-extern int FPU_to_exp16(FPU_REG const *a, FPU_REG *x);
-#endif /* _FPU_PROTO_H */
diff --git a/arch/x86/math-emu/fpu_system.h b/arch/x86/math-emu/fpu_system.h
deleted file mode 100644
index 5e238e930fe30..0000000000000
--- a/arch/x86/math-emu/fpu_system.h
+++ /dev/null
@@ -1,130 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  fpu_system.h                                                             |
- |                                                                           |
- | Copyright (C) 1992,1994,1997                                              |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@suburbia.net             |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#ifndef _FPU_SYSTEM_H
-#define _FPU_SYSTEM_H
-
-/* system dependent definitions */
-
-#include <linux/sched.h>
-#include <linux/kernel.h>
-#include <linux/mm.h>
-
-#include <asm/desc.h>
-#include <asm/mmu_context.h>
-
-static inline struct desc_struct FPU_get_ldt_descriptor(unsigned seg)
-{
-	static struct desc_struct zero_desc;
-	struct desc_struct ret = zero_desc;
-
-#ifdef CONFIG_MODIFY_LDT_SYSCALL
-	seg >>= 3;
-	mutex_lock(&current->mm->context.lock);
-	if (current->mm->context.ldt && seg < current->mm->context.ldt->nr_entries)
-		ret = current->mm->context.ldt->entries[seg];
-	mutex_unlock(&current->mm->context.lock);
-#endif
-	return ret;
-}
-
-#define SEG_TYPE_WRITABLE	(1U << 1)
-#define SEG_TYPE_EXPANDS_DOWN	(1U << 2)
-#define SEG_TYPE_EXECUTE	(1U << 3)
-#define SEG_TYPE_EXPAND_MASK	(SEG_TYPE_EXPANDS_DOWN | SEG_TYPE_EXECUTE)
-#define SEG_TYPE_EXECUTE_MASK	(SEG_TYPE_WRITABLE | SEG_TYPE_EXECUTE)
-
-static inline unsigned long seg_get_base(struct desc_struct *d)
-{
-	unsigned long base = (unsigned long)d->base2 << 24;
-
-	return base | ((unsigned long)d->base1 << 16) | d->base0;
-}
-
-static inline unsigned long seg_get_limit(struct desc_struct *d)
-{
-	return ((unsigned long)d->limit1 << 16) | d->limit0;
-}
-
-static inline unsigned long seg_get_granularity(struct desc_struct *d)
-{
-	return d->g ? 4096 : 1;
-}
-
-static inline bool seg_expands_down(struct desc_struct *d)
-{
-	return (d->type & SEG_TYPE_EXPAND_MASK) == SEG_TYPE_EXPANDS_DOWN;
-}
-
-static inline bool seg_execute_only(struct desc_struct *d)
-{
-	return (d->type & SEG_TYPE_EXECUTE_MASK) == SEG_TYPE_EXECUTE;
-}
-
-static inline bool seg_writable(struct desc_struct *d)
-{
-	return (d->type & SEG_TYPE_EXECUTE_MASK) == SEG_TYPE_WRITABLE;
-}
-
-#define I387			(&x86_task_fpu(current)->fpstate->regs)
-#define FPU_info		(I387->soft.info)
-
-#define FPU_CS			(*(unsigned short *) &(FPU_info->regs->cs))
-#define FPU_SS			(*(unsigned short *) &(FPU_info->regs->ss))
-#define FPU_DS			(*(unsigned short *) &(FPU_info->regs->ds))
-#define FPU_EAX			(FPU_info->regs->ax)
-#define FPU_EFLAGS		(FPU_info->regs->flags)
-#define FPU_EIP			(FPU_info->regs->ip)
-#define FPU_ORIG_EIP		(FPU_info->___orig_eip)
-
-#define FPU_lookahead           (I387->soft.lookahead)
-
-/* nz if ip_offset and cs_selector are not to be set for the current
-   instruction. */
-#define no_ip_update		(*(u_char *)&(I387->soft.no_update))
-#define FPU_rm			(*(u_char *)&(I387->soft.rm))
-
-/* Number of bytes of data which can be legally accessed by the current
-   instruction. This only needs to hold a number <= 108, so a byte will do. */
-#define access_limit		(*(u_char *)&(I387->soft.alimit))
-
-#define partial_status		(I387->soft.swd)
-#define control_word		(I387->soft.cwd)
-#define fpu_tag_word		(I387->soft.twd)
-#define registers		(I387->soft.st_space)
-#define top			(I387->soft.ftop)
-
-#define instruction_address	(*(struct address *)&I387->soft.fip)
-#define operand_address		(*(struct address *)&I387->soft.foo)
-
-#define FPU_access_ok(y,z)	if ( !access_ok(y,z) ) \
-				math_abort(FPU_info,SIGSEGV)
-#define FPU_abort		math_abort(FPU_info, SIGSEGV)
-#define FPU_copy_from_user(to, from, n)	\
-		do { if (copy_from_user(to, from, n)) FPU_abort; } while (0)
-
-#undef FPU_IGNORE_CODE_SEGV
-#ifdef FPU_IGNORE_CODE_SEGV
-/* access_ok() is very expensive, and causes the emulator to run
-   about 20% slower if applied to the code. Anyway, errors due to bad
-   code addresses should be much rarer than errors due to bad data
-   addresses. */
-#define	FPU_code_access_ok(z)
-#else
-/* A simpler test than access_ok() can probably be done for
-   FPU_code_access_ok() because the only possible error is to step
-   past the upper boundary of a legal code area. */
-#define	FPU_code_access_ok(z) FPU_access_ok((void __user *)FPU_EIP,z)
-#endif
-
-#define FPU_get_user(x,y) do { if (get_user((x),(y))) FPU_abort; } while (0)
-#define FPU_put_user(x,y) do { if (put_user((x),(y))) FPU_abort; } while (0)
-
-#endif
diff --git a/arch/x86/math-emu/fpu_tags.c b/arch/x86/math-emu/fpu_tags.c
deleted file mode 100644
index bff95d4e7dca0..0000000000000
--- a/arch/x86/math-emu/fpu_tags.c
+++ /dev/null
@@ -1,116 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  fpu_tags.c                                                               |
- |                                                                           |
- |  Set FPU register tags.                                                   |
- |                                                                           |
- | Copyright (C) 1997                                                        |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@jacobi.maths.monash.edu.au                |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "exception.h"
-
-void FPU_pop(void)
-{
-	fpu_tag_word |= 3 << ((top & 7) * 2);
-	top++;
-}
-
-int FPU_gettag0(void)
-{
-	return (fpu_tag_word >> ((top & 7) * 2)) & 3;
-}
-
-int FPU_gettagi(int stnr)
-{
-	return (fpu_tag_word >> (((top + stnr) & 7) * 2)) & 3;
-}
-
-int FPU_gettag(int regnr)
-{
-	return (fpu_tag_word >> ((regnr & 7) * 2)) & 3;
-}
-
-void FPU_settag0(int tag)
-{
-	int regnr = top;
-	regnr &= 7;
-	fpu_tag_word &= ~(3 << (regnr * 2));
-	fpu_tag_word |= (tag & 3) << (regnr * 2);
-}
-
-void FPU_settagi(int stnr, int tag)
-{
-	int regnr = stnr + top;
-	regnr &= 7;
-	fpu_tag_word &= ~(3 << (regnr * 2));
-	fpu_tag_word |= (tag & 3) << (regnr * 2);
-}
-
-void FPU_settag(int regnr, int tag)
-{
-	regnr &= 7;
-	fpu_tag_word &= ~(3 << (regnr * 2));
-	fpu_tag_word |= (tag & 3) << (regnr * 2);
-}
-
-int FPU_Special(FPU_REG const *ptr)
-{
-	int exp = exponent(ptr);
-
-	if (exp == EXP_BIAS + EXP_UNDER)
-		return TW_Denormal;
-	else if (exp != EXP_BIAS + EXP_OVER)
-		return TW_NaN;
-	else if ((ptr->sigh == 0x80000000) && (ptr->sigl == 0))
-		return TW_Infinity;
-	return TW_NaN;
-}
-
-int isNaN(FPU_REG const *ptr)
-{
-	return ((exponent(ptr) == EXP_BIAS + EXP_OVER)
-		&& !((ptr->sigh == 0x80000000) && (ptr->sigl == 0)));
-}
-
-int FPU_empty_i(int stnr)
-{
-	int regnr = (top + stnr) & 7;
-
-	return ((fpu_tag_word >> (regnr * 2)) & 3) == TAG_Empty;
-}
-
-int FPU_stackoverflow(FPU_REG ** st_new_ptr)
-{
-	*st_new_ptr = &st(-1);
-
-	return ((fpu_tag_word >> (((top - 1) & 7) * 2)) & 3) != TAG_Empty;
-}
-
-void FPU_copy_to_regi(FPU_REG const *r, u_char tag, int stnr)
-{
-	reg_copy(r, &st(stnr));
-	FPU_settagi(stnr, tag);
-}
-
-void FPU_copy_to_reg1(FPU_REG const *r, u_char tag)
-{
-	reg_copy(r, &st(1));
-	FPU_settagi(1, tag);
-}
-
-void FPU_copy_to_reg0(FPU_REG const *r, u_char tag)
-{
-	int regnr = top;
-	regnr &= 7;
-
-	reg_copy(r, &st(0));
-
-	fpu_tag_word &= ~(3 << (regnr * 2));
-	fpu_tag_word |= (tag & 3) << (regnr * 2);
-}
diff --git a/arch/x86/math-emu/fpu_trig.c b/arch/x86/math-emu/fpu_trig.c
deleted file mode 100644
index 85daf98c81c3e..0000000000000
--- a/arch/x86/math-emu/fpu_trig.c
+++ /dev/null
@@ -1,1649 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  fpu_trig.c                                                               |
- |                                                                           |
- | Implementation of the FPU "transcendental" functions.                     |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997,1999                                    |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@melbpc.org.au            |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "control_w.h"
-#include "reg_constant.h"
-
-static void rem_kernel(unsigned long long st0, unsigned long long *y,
-		       unsigned long long st1, unsigned long long q, int n);
-
-#define BETTER_THAN_486
-
-#define FCOS  4
-
-/* Used only by fptan, fsin, fcos, and fsincos. */
-/* This routine produces very accurate results, similar to
-   using a value of pi with more than 128 bits precision. */
-/* Limited measurements show no results worse than 64 bit precision
-   except for the results for arguments close to 2^63, where the
-   precision of the result sometimes degrades to about 63.9 bits */
-static int trig_arg(FPU_REG *st0_ptr, int even)
-{
-	FPU_REG tmp;
-	u_char tmptag;
-	unsigned long long q;
-	int old_cw = control_word, saved_status = partial_status;
-	int tag, st0_tag = TAG_Valid;
-
-	if (exponent(st0_ptr) >= 63) {
-		partial_status |= SW_C2;	/* Reduction incomplete. */
-		return -1;
-	}
-
-	control_word &= ~CW_RC;
-	control_word |= RC_CHOP;
-
-	setpositive(st0_ptr);
-	tag = FPU_u_div(st0_ptr, &CONST_PI2, &tmp, PR_64_BITS | RC_CHOP | 0x3f,
-			SIGN_POS);
-
-	FPU_round_to_int(&tmp, tag);	/* Fortunately, this can't overflow
-					   to 2^64 */
-	q = significand(&tmp);
-	if (q) {
-		rem_kernel(significand(st0_ptr),
-			   &significand(&tmp),
-			   significand(&CONST_PI2),
-			   q, exponent(st0_ptr) - exponent(&CONST_PI2));
-		setexponent16(&tmp, exponent(&CONST_PI2));
-		st0_tag = FPU_normalize(&tmp);
-		FPU_copy_to_reg0(&tmp, st0_tag);
-	}
-
-	if ((even && !(q & 1)) || (!even && (q & 1))) {
-		st0_tag =
-		    FPU_sub(REV | LOADED | TAG_Valid, (int)&CONST_PI2,
-			    FULL_PRECISION);
-
-#ifdef BETTER_THAN_486
-		/* So far, the results are exact but based upon a 64 bit
-		   precision approximation to pi/2. The technique used
-		   now is equivalent to using an approximation to pi/2 which
-		   is accurate to about 128 bits. */
-		if ((exponent(st0_ptr) <= exponent(&CONST_PI2extra) + 64)
-		    || (q > 1)) {
-			/* This code gives the effect of having pi/2 to better than
-			   128 bits precision. */
-
-			significand(&tmp) = q + 1;
-			setexponent16(&tmp, 63);
-			FPU_normalize(&tmp);
-			tmptag =
-			    FPU_u_mul(&CONST_PI2extra, &tmp, &tmp,
-				      FULL_PRECISION, SIGN_POS,
-				      exponent(&CONST_PI2extra) +
-				      exponent(&tmp));
-			setsign(&tmp, getsign(&CONST_PI2extra));
-			st0_tag = FPU_add(&tmp, tmptag, 0, FULL_PRECISION);
-			if (signnegative(st0_ptr)) {
-				/* CONST_PI2extra is negative, so the result of the addition
-				   can be negative. This means that the argument is actually
-				   in a different quadrant. The correction is always < pi/2,
-				   so it can't overflow into yet another quadrant. */
-				setpositive(st0_ptr);
-				q++;
-			}
-		}
-#endif /* BETTER_THAN_486 */
-	}
-#ifdef BETTER_THAN_486
-	else {
-		/* So far, the results are exact but based upon a 64 bit
-		   precision approximation to pi/2. The technique used
-		   now is equivalent to using an approximation to pi/2 which
-		   is accurate to about 128 bits. */
-		if (((q > 0)
-		     && (exponent(st0_ptr) <= exponent(&CONST_PI2extra) + 64))
-		    || (q > 1)) {
-			/* This code gives the effect of having p/2 to better than
-			   128 bits precision. */
-
-			significand(&tmp) = q;
-			setexponent16(&tmp, 63);
-			FPU_normalize(&tmp);	/* This must return TAG_Valid */
-			tmptag =
-			    FPU_u_mul(&CONST_PI2extra, &tmp, &tmp,
-				      FULL_PRECISION, SIGN_POS,
-				      exponent(&CONST_PI2extra) +
-				      exponent(&tmp));
-			setsign(&tmp, getsign(&CONST_PI2extra));
-			st0_tag = FPU_sub(LOADED | (tmptag & 0x0f), (int)&tmp,
-					  FULL_PRECISION);
-			if ((exponent(st0_ptr) == exponent(&CONST_PI2)) &&
-			    ((st0_ptr->sigh > CONST_PI2.sigh)
-			     || ((st0_ptr->sigh == CONST_PI2.sigh)
-				 && (st0_ptr->sigl > CONST_PI2.sigl)))) {
-				/* CONST_PI2extra is negative, so the result of the
-				   subtraction can be larger than pi/2. This means
-				   that the argument is actually in a different quadrant.
-				   The correction is always < pi/2, so it can't overflow
-				   into yet another quadrant. */
-				st0_tag =
-				    FPU_sub(REV | LOADED | TAG_Valid,
-					    (int)&CONST_PI2, FULL_PRECISION);
-				q++;
-			}
-		}
-	}
-#endif /* BETTER_THAN_486 */
-
-	FPU_settag0(st0_tag);
-	control_word = old_cw;
-	partial_status = saved_status & ~SW_C2;	/* Reduction complete. */
-
-	return (q & 3) | even;
-}
-
-/* Convert a long to register */
-static void convert_l2reg(long const *arg, int deststnr)
-{
-	int tag;
-	long num = *arg;
-	u_char sign;
-	FPU_REG *dest = &st(deststnr);
-
-	if (num == 0) {
-		FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
-		return;
-	}
-
-	if (num > 0) {
-		sign = SIGN_POS;
-	} else {
-		num = -num;
-		sign = SIGN_NEG;
-	}
-
-	dest->sigh = num;
-	dest->sigl = 0;
-	setexponent16(dest, 31);
-	tag = FPU_normalize(dest);
-	FPU_settagi(deststnr, tag);
-	setsign(dest, sign);
-	return;
-}
-
-static void single_arg_error(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	if (st0_tag == TAG_Empty)
-		FPU_stack_underflow();	/* Puts a QNaN in st(0) */
-	else if (st0_tag == TW_NaN)
-		real_1op_NaN(st0_ptr);	/* return with a NaN in st(0) */
-#ifdef PARANOID
-	else
-		EXCEPTION(EX_INTERNAL | 0x0112);
-#endif /* PARANOID */
-}
-
-static void single_arg_2_error(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	int isNaN;
-
-	switch (st0_tag) {
-	case TW_NaN:
-		isNaN = (exponent(st0_ptr) == EXP_OVER)
-		    && (st0_ptr->sigh & 0x80000000);
-		if (isNaN && !(st0_ptr->sigh & 0x40000000)) {	/* Signaling ? */
-			EXCEPTION(EX_Invalid);
-			if (control_word & CW_Invalid) {
-				/* The masked response */
-				/* Convert to a QNaN */
-				st0_ptr->sigh |= 0x40000000;
-				push();
-				FPU_copy_to_reg0(st0_ptr, TAG_Special);
-			}
-		} else if (isNaN) {
-			/* A QNaN */
-			push();
-			FPU_copy_to_reg0(st0_ptr, TAG_Special);
-		} else {
-			/* pseudoNaN or other unsupported */
-			EXCEPTION(EX_Invalid);
-			if (control_word & CW_Invalid) {
-				/* The masked response */
-				FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
-				push();
-				FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
-			}
-		}
-		break;		/* return with a NaN in st(0) */
-#ifdef PARANOID
-	default:
-		EXCEPTION(EX_INTERNAL | 0x0112);
-#endif /* PARANOID */
-	}
-}
-
-/*---------------------------------------------------------------------------*/
-
-static void f2xm1(FPU_REG *st0_ptr, u_char tag)
-{
-	FPU_REG a;
-
-	clear_C1();
-
-	if (tag == TAG_Valid) {
-		/* For an 80486 FPU, the result is undefined if the arg is >= 1.0 */
-		if (exponent(st0_ptr) < 0) {
-		      denormal_arg:
-
-			FPU_to_exp16(st0_ptr, &a);
-
-			/* poly_2xm1(x) requires 0 < st(0) < 1. */
-			poly_2xm1(getsign(st0_ptr), &a, st0_ptr);
-		}
-		set_precision_flag_up();	/* 80486 appears to always do this */
-		return;
-	}
-
-	if (tag == TAG_Zero)
-		return;
-
-	if (tag == TAG_Special)
-		tag = FPU_Special(st0_ptr);
-
-	switch (tag) {
-	case TW_Denormal:
-		if (denormal_operand() < 0)
-			return;
-		goto denormal_arg;
-	case TW_Infinity:
-		if (signnegative(st0_ptr)) {
-			/* -infinity gives -1 (p16-10) */
-			FPU_copy_to_reg0(&CONST_1, TAG_Valid);
-			setnegative(st0_ptr);
-		}
-		return;
-	default:
-		single_arg_error(st0_ptr, tag);
-	}
-}
-
-static void fptan(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	FPU_REG *st_new_ptr;
-	int q;
-	u_char arg_sign = getsign(st0_ptr);
-
-	/* Stack underflow has higher priority */
-	if (st0_tag == TAG_Empty) {
-		FPU_stack_underflow();	/* Puts a QNaN in st(0) */
-		if (control_word & CW_Invalid) {
-			st_new_ptr = &st(-1);
-			push();
-			FPU_stack_underflow();	/* Puts a QNaN in the new st(0) */
-		}
-		return;
-	}
-
-	if (STACK_OVERFLOW) {
-		FPU_stack_overflow();
-		return;
-	}
-
-	if (st0_tag == TAG_Valid) {
-		if (exponent(st0_ptr) > -40) {
-			if ((q = trig_arg(st0_ptr, 0)) == -1) {
-				/* Operand is out of range */
-				return;
-			}
-
-			poly_tan(st0_ptr);
-			setsign(st0_ptr, (q & 1) ^ (arg_sign != 0));
-			set_precision_flag_up();	/* We do not really know if up or down */
-		} else {
-			/* For a small arg, the result == the argument */
-			/* Underflow may happen */
-
-		      denormal_arg:
-
-			FPU_to_exp16(st0_ptr, st0_ptr);
-
-			st0_tag =
-			    FPU_round(st0_ptr, 1, 0, FULL_PRECISION, arg_sign);
-			FPU_settag0(st0_tag);
-		}
-		push();
-		FPU_copy_to_reg0(&CONST_1, TAG_Valid);
-		return;
-	}
-
-	if (st0_tag == TAG_Zero) {
-		push();
-		FPU_copy_to_reg0(&CONST_1, TAG_Valid);
-		setcc(0);
-		return;
-	}
-
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-
-	if (st0_tag == TW_Denormal) {
-		if (denormal_operand() < 0)
-			return;
-
-		goto denormal_arg;
-	}
-
-	if (st0_tag == TW_Infinity) {
-		/* The 80486 treats infinity as an invalid operand */
-		if (arith_invalid(0) >= 0) {
-			st_new_ptr = &st(-1);
-			push();
-			arith_invalid(0);
-		}
-		return;
-	}
-
-	single_arg_2_error(st0_ptr, st0_tag);
-}
-
-static void fxtract(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	FPU_REG *st_new_ptr;
-	u_char sign;
-	register FPU_REG *st1_ptr = st0_ptr;	/* anticipate */
-
-	if (STACK_OVERFLOW) {
-		FPU_stack_overflow();
-		return;
-	}
-
-	clear_C1();
-
-	if (st0_tag == TAG_Valid) {
-		long e;
-
-		push();
-		sign = getsign(st1_ptr);
-		reg_copy(st1_ptr, st_new_ptr);
-		setexponent16(st_new_ptr, exponent(st_new_ptr));
-
-	      denormal_arg:
-
-		e = exponent16(st_new_ptr);
-		convert_l2reg(&e, 1);
-		setexponentpos(st_new_ptr, 0);
-		setsign(st_new_ptr, sign);
-		FPU_settag0(TAG_Valid);	/* Needed if arg was a denormal */
-		return;
-	} else if (st0_tag == TAG_Zero) {
-		sign = getsign(st0_ptr);
-
-		if (FPU_divide_by_zero(0, SIGN_NEG) < 0)
-			return;
-
-		push();
-		FPU_copy_to_reg0(&CONST_Z, TAG_Zero);
-		setsign(st_new_ptr, sign);
-		return;
-	}
-
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-
-	if (st0_tag == TW_Denormal) {
-		if (denormal_operand() < 0)
-			return;
-
-		push();
-		sign = getsign(st1_ptr);
-		FPU_to_exp16(st1_ptr, st_new_ptr);
-		goto denormal_arg;
-	} else if (st0_tag == TW_Infinity) {
-		sign = getsign(st0_ptr);
-		setpositive(st0_ptr);
-		push();
-		FPU_copy_to_reg0(&CONST_INF, TAG_Special);
-		setsign(st_new_ptr, sign);
-		return;
-	} else if (st0_tag == TW_NaN) {
-		if (real_1op_NaN(st0_ptr) < 0)
-			return;
-
-		push();
-		FPU_copy_to_reg0(st0_ptr, TAG_Special);
-		return;
-	} else if (st0_tag == TAG_Empty) {
-		/* Is this the correct behaviour? */
-		if (control_word & EX_Invalid) {
-			FPU_stack_underflow();
-			push();
-			FPU_stack_underflow();
-		} else
-			EXCEPTION(EX_StackUnder);
-	}
-#ifdef PARANOID
-	else
-		EXCEPTION(EX_INTERNAL | 0x119);
-#endif /* PARANOID */
-}
-
-static void fdecstp(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	clear_C1();
-	top--;
-}
-
-static void fincstp(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	clear_C1();
-	top++;
-}
-
-static void fsqrt_(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	int expon;
-
-	clear_C1();
-
-	if (st0_tag == TAG_Valid) {
-		u_char tag;
-
-		if (signnegative(st0_ptr)) {
-			arith_invalid(0);	/* sqrt(negative) is invalid */
-			return;
-		}
-
-		/* make st(0) in  [1.0 .. 4.0) */
-		expon = exponent(st0_ptr);
-
-	      denormal_arg:
-
-		setexponent16(st0_ptr, (expon & 1));
-
-		/* Do the computation, the sign of the result will be positive. */
-		tag = wm_sqrt(st0_ptr, 0, 0, control_word, SIGN_POS);
-		addexponent(st0_ptr, expon >> 1);
-		FPU_settag0(tag);
-		return;
-	}
-
-	if (st0_tag == TAG_Zero)
-		return;
-
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-
-	if (st0_tag == TW_Infinity) {
-		if (signnegative(st0_ptr))
-			arith_invalid(0);	/* sqrt(-Infinity) is invalid */
-		return;
-	} else if (st0_tag == TW_Denormal) {
-		if (signnegative(st0_ptr)) {
-			arith_invalid(0);	/* sqrt(negative) is invalid */
-			return;
-		}
-
-		if (denormal_operand() < 0)
-			return;
-
-		FPU_to_exp16(st0_ptr, st0_ptr);
-
-		expon = exponent16(st0_ptr);
-
-		goto denormal_arg;
-	}
-
-	single_arg_error(st0_ptr, st0_tag);
-
-}
-
-static void frndint_(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	int flags, tag;
-
-	if (st0_tag == TAG_Valid) {
-		u_char sign;
-
-	      denormal_arg:
-
-		sign = getsign(st0_ptr);
-
-		if (exponent(st0_ptr) > 63)
-			return;
-
-		if (st0_tag == TW_Denormal) {
-			if (denormal_operand() < 0)
-				return;
-		}
-
-		/* Fortunately, this can't overflow to 2^64 */
-		if ((flags = FPU_round_to_int(st0_ptr, st0_tag)))
-			set_precision_flag(flags);
-
-		setexponent16(st0_ptr, 63);
-		tag = FPU_normalize(st0_ptr);
-		setsign(st0_ptr, sign);
-		FPU_settag0(tag);
-		return;
-	}
-
-	if (st0_tag == TAG_Zero)
-		return;
-
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-
-	if (st0_tag == TW_Denormal)
-		goto denormal_arg;
-	else if (st0_tag == TW_Infinity)
-		return;
-	else
-		single_arg_error(st0_ptr, st0_tag);
-}
-
-static int f_sin(FPU_REG *st0_ptr, u_char tag)
-{
-	u_char arg_sign = getsign(st0_ptr);
-
-	if (tag == TAG_Valid) {
-		int q;
-
-		if (exponent(st0_ptr) > -40) {
-			if ((q = trig_arg(st0_ptr, 0)) == -1) {
-				/* Operand is out of range */
-				return 1;
-			}
-
-			poly_sine(st0_ptr);
-
-			if (q & 2)
-				changesign(st0_ptr);
-
-			setsign(st0_ptr, getsign(st0_ptr) ^ arg_sign);
-
-			/* We do not really know if up or down */
-			set_precision_flag_up();
-			return 0;
-		} else {
-			/* For a small arg, the result == the argument */
-			set_precision_flag_up();	/* Must be up. */
-			return 0;
-		}
-	}
-
-	if (tag == TAG_Zero) {
-		setcc(0);
-		return 0;
-	}
-
-	if (tag == TAG_Special)
-		tag = FPU_Special(st0_ptr);
-
-	if (tag == TW_Denormal) {
-		if (denormal_operand() < 0)
-			return 1;
-
-		/* For a small arg, the result == the argument */
-		/* Underflow may happen */
-		FPU_to_exp16(st0_ptr, st0_ptr);
-
-		tag = FPU_round(st0_ptr, 1, 0, FULL_PRECISION, arg_sign);
-
-		FPU_settag0(tag);
-
-		return 0;
-	} else if (tag == TW_Infinity) {
-		/* The 80486 treats infinity as an invalid operand */
-		arith_invalid(0);
-		return 1;
-	} else {
-		single_arg_error(st0_ptr, tag);
-		return 1;
-	}
-}
-
-static void fsin(FPU_REG *st0_ptr, u_char tag)
-{
-	f_sin(st0_ptr, tag);
-}
-
-static int f_cos(FPU_REG *st0_ptr, u_char tag)
-{
-	u_char st0_sign;
-
-	st0_sign = getsign(st0_ptr);
-
-	if (tag == TAG_Valid) {
-		int q;
-
-		if (exponent(st0_ptr) > -40) {
-			if ((exponent(st0_ptr) < 0)
-			    || ((exponent(st0_ptr) == 0)
-				&& (significand(st0_ptr) <=
-				    0xc90fdaa22168c234LL))) {
-				poly_cos(st0_ptr);
-
-				/* We do not really know if up or down */
-				set_precision_flag_down();
-
-				return 0;
-			} else if ((q = trig_arg(st0_ptr, FCOS)) != -1) {
-				poly_sine(st0_ptr);
-
-				if ((q + 1) & 2)
-					changesign(st0_ptr);
-
-				/* We do not really know if up or down */
-				set_precision_flag_down();
-
-				return 0;
-			} else {
-				/* Operand is out of range */
-				return 1;
-			}
-		} else {
-		      denormal_arg:
-
-			setcc(0);
-			FPU_copy_to_reg0(&CONST_1, TAG_Valid);
-#ifdef PECULIAR_486
-			set_precision_flag_down();	/* 80486 appears to do this. */
-#else
-			set_precision_flag_up();	/* Must be up. */
-#endif /* PECULIAR_486 */
-			return 0;
-		}
-	} else if (tag == TAG_Zero) {
-		FPU_copy_to_reg0(&CONST_1, TAG_Valid);
-		setcc(0);
-		return 0;
-	}
-
-	if (tag == TAG_Special)
-		tag = FPU_Special(st0_ptr);
-
-	if (tag == TW_Denormal) {
-		if (denormal_operand() < 0)
-			return 1;
-
-		goto denormal_arg;
-	} else if (tag == TW_Infinity) {
-		/* The 80486 treats infinity as an invalid operand */
-		arith_invalid(0);
-		return 1;
-	} else {
-		single_arg_error(st0_ptr, tag);	/* requires st0_ptr == &st(0) */
-		return 1;
-	}
-}
-
-static void fcos(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	f_cos(st0_ptr, st0_tag);
-}
-
-static void fsincos(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	FPU_REG *st_new_ptr;
-	FPU_REG arg;
-	u_char tag;
-
-	/* Stack underflow has higher priority */
-	if (st0_tag == TAG_Empty) {
-		FPU_stack_underflow();	/* Puts a QNaN in st(0) */
-		if (control_word & CW_Invalid) {
-			st_new_ptr = &st(-1);
-			push();
-			FPU_stack_underflow();	/* Puts a QNaN in the new st(0) */
-		}
-		return;
-	}
-
-	if (STACK_OVERFLOW) {
-		FPU_stack_overflow();
-		return;
-	}
-
-	if (st0_tag == TAG_Special)
-		tag = FPU_Special(st0_ptr);
-	else
-		tag = st0_tag;
-
-	if (tag == TW_NaN) {
-		single_arg_2_error(st0_ptr, TW_NaN);
-		return;
-	} else if (tag == TW_Infinity) {
-		/* The 80486 treats infinity as an invalid operand */
-		if (arith_invalid(0) >= 0) {
-			/* Masked response */
-			push();
-			arith_invalid(0);
-		}
-		return;
-	}
-
-	reg_copy(st0_ptr, &arg);
-	if (!f_sin(st0_ptr, st0_tag)) {
-		push();
-		FPU_copy_to_reg0(&arg, st0_tag);
-		f_cos(&st(0), st0_tag);
-	} else {
-		/* An error, so restore st(0) */
-		FPU_copy_to_reg0(&arg, st0_tag);
-	}
-}
-
-/*---------------------------------------------------------------------------*/
-/* The following all require two arguments: st(0) and st(1) */
-
-/* A lean, mean kernel for the fprem instructions. This relies upon
-   the division and rounding to an integer in do_fprem giving an
-   exact result. Because of this, rem_kernel() needs to deal only with
-   the least significant 64 bits, the more significant bits of the
-   result must be zero.
- */
-static void rem_kernel(unsigned long long st0, unsigned long long *y,
-		       unsigned long long st1, unsigned long long q, int n)
-{
-	int dummy;
-	unsigned long long x;
-
-	x = st0 << n;
-
-	/* Do the required multiplication and subtraction in the one operation */
-
-	/* lsw x -= lsw st1 * lsw q */
-	asm volatile ("mull %4; subl %%eax,%0; sbbl %%edx,%1":"=m"
-		      (((unsigned *)&x)[0]), "=m"(((unsigned *)&x)[1]),
-		      "=a"(dummy)
-		      :"2"(((unsigned *)&st1)[0]), "m"(((unsigned *)&q)[0])
-		      :"%dx");
-	/* msw x -= msw st1 * lsw q */
-	asm volatile ("mull %3; subl %%eax,%0":"=m" (((unsigned *)&x)[1]),
-		      "=a"(dummy)
-		      :"1"(((unsigned *)&st1)[1]), "m"(((unsigned *)&q)[0])
-		      :"%dx");
-	/* msw x -= lsw st1 * msw q */
-	asm volatile ("mull %3; subl %%eax,%0":"=m" (((unsigned *)&x)[1]),
-		      "=a"(dummy)
-		      :"1"(((unsigned *)&st1)[0]), "m"(((unsigned *)&q)[1])
-		      :"%dx");
-
-	*y = x;
-}
-
-/* Remainder of st(0) / st(1) */
-/* This routine produces exact results, i.e. there is never any
-   rounding or truncation, etc of the result. */
-static void do_fprem(FPU_REG *st0_ptr, u_char st0_tag, int round)
-{
-	FPU_REG *st1_ptr = &st(1);
-	u_char st1_tag = FPU_gettagi(1);
-
-	if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) {
-		FPU_REG tmp, st0, st1;
-		u_char st0_sign, st1_sign;
-		u_char tmptag;
-		int tag;
-		int old_cw;
-		int expdif;
-		long long q;
-		unsigned short saved_status;
-		int cc;
-
-	      fprem_valid:
-		/* Convert registers for internal use. */
-		st0_sign = FPU_to_exp16(st0_ptr, &st0);
-		st1_sign = FPU_to_exp16(st1_ptr, &st1);
-		expdif = exponent16(&st0) - exponent16(&st1);
-
-		old_cw = control_word;
-		cc = 0;
-
-		/* We want the status following the denorm tests, but don't want
-		   the status changed by the arithmetic operations. */
-		saved_status = partial_status;
-		control_word &= ~CW_RC;
-		control_word |= RC_CHOP;
-
-		if (expdif < 64) {
-			/* This should be the most common case */
-
-			if (expdif > -2) {
-				u_char sign = st0_sign ^ st1_sign;
-				tag = FPU_u_div(&st0, &st1, &tmp,
-						PR_64_BITS | RC_CHOP | 0x3f,
-						sign);
-				setsign(&tmp, sign);
-
-				if (exponent(&tmp) >= 0) {
-					FPU_round_to_int(&tmp, tag);	/* Fortunately, this can't
-									   overflow to 2^64 */
-					q = significand(&tmp);
-
-					rem_kernel(significand(&st0),
-						   &significand(&tmp),
-						   significand(&st1),
-						   q, expdif);
-
-					setexponent16(&tmp, exponent16(&st1));
-				} else {
-					reg_copy(&st0, &tmp);
-					q = 0;
-				}
-
-				if ((round == RC_RND)
-				    && (tmp.sigh & 0xc0000000)) {
-					/* We may need to subtract st(1) once more,
-					   to get a result <= 1/2 of st(1). */
-					unsigned long long x;
-					expdif =
-					    exponent16(&st1) - exponent16(&tmp);
-					if (expdif <= 1) {
-						if (expdif == 0)
-							x = significand(&st1) -
-							    significand(&tmp);
-						else	/* expdif is 1 */
-							x = (significand(&st1)
-							     << 1) -
-							    significand(&tmp);
-						if ((x < significand(&tmp)) ||
-						    /* or equi-distant (from 0 & st(1)) and q is odd */
-						    ((x == significand(&tmp))
-						     && (q & 1))) {
-							st0_sign = !st0_sign;
-							significand(&tmp) = x;
-							q++;
-						}
-					}
-				}
-
-				if (q & 4)
-					cc |= SW_C0;
-				if (q & 2)
-					cc |= SW_C3;
-				if (q & 1)
-					cc |= SW_C1;
-			} else {
-				control_word = old_cw;
-				setcc(0);
-				return;
-			}
-		} else {
-			/* There is a large exponent difference ( >= 64 ) */
-			/* To make much sense, the code in this section should
-			   be done at high precision. */
-			int exp_1, N;
-			u_char sign;
-
-			/* prevent overflow here */
-			/* N is 'a number between 32 and 63' (p26-113) */
-			reg_copy(&st0, &tmp);
-			tmptag = st0_tag;
-			N = (expdif & 0x0000001f) + 32;	/* This choice gives results
-							   identical to an AMD 486 */
-			setexponent16(&tmp, N);
-			exp_1 = exponent16(&st1);
-			setexponent16(&st1, 0);
-			expdif -= N;
-
-			sign = getsign(&tmp) ^ st1_sign;
-			tag =
-			    FPU_u_div(&tmp, &st1, &tmp,
-				      PR_64_BITS | RC_CHOP | 0x3f, sign);
-			setsign(&tmp, sign);
-
-			FPU_round_to_int(&tmp, tag);	/* Fortunately, this can't
-							   overflow to 2^64 */
-
-			rem_kernel(significand(&st0),
-				   &significand(&tmp),
-				   significand(&st1),
-				   significand(&tmp), exponent(&tmp)
-			    );
-			setexponent16(&tmp, exp_1 + expdif);
-
-			/* It is possible for the operation to be complete here.
-			   What does the IEEE standard say? The Intel 80486 manual
-			   implies that the operation will never be completed at this
-			   point, and the behaviour of a real 80486 confirms this.
-			 */
-			if (!(tmp.sigh | tmp.sigl)) {
-				/* The result is zero */
-				control_word = old_cw;
-				partial_status = saved_status;
-				FPU_copy_to_reg0(&CONST_Z, TAG_Zero);
-				setsign(&st0, st0_sign);
-#ifdef PECULIAR_486
-				setcc(SW_C2);
-#else
-				setcc(0);
-#endif /* PECULIAR_486 */
-				return;
-			}
-			cc = SW_C2;
-		}
-
-		control_word = old_cw;
-		partial_status = saved_status;
-		tag = FPU_normalize_nuo(&tmp);
-		reg_copy(&tmp, st0_ptr);
-
-		/* The only condition to be looked for is underflow,
-		   and it can occur here only if underflow is unmasked. */
-		if ((exponent16(&tmp) <= EXP_UNDER) && (tag != TAG_Zero)
-		    && !(control_word & CW_Underflow)) {
-			setcc(cc);
-			tag = arith_underflow(st0_ptr);
-			setsign(st0_ptr, st0_sign);
-			FPU_settag0(tag);
-			return;
-		} else if ((exponent16(&tmp) > EXP_UNDER) || (tag == TAG_Zero)) {
-			stdexp(st0_ptr);
-			setsign(st0_ptr, st0_sign);
-		} else {
-			tag =
-			    FPU_round(st0_ptr, 0, 0, FULL_PRECISION, st0_sign);
-		}
-		FPU_settag0(tag);
-		setcc(cc);
-
-		return;
-	}
-
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-	if (st1_tag == TAG_Special)
-		st1_tag = FPU_Special(st1_ptr);
-
-	if (((st0_tag == TAG_Valid) && (st1_tag == TW_Denormal))
-	    || ((st0_tag == TW_Denormal) && (st1_tag == TAG_Valid))
-	    || ((st0_tag == TW_Denormal) && (st1_tag == TW_Denormal))) {
-		if (denormal_operand() < 0)
-			return;
-		goto fprem_valid;
-	} else if ((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty)) {
-		FPU_stack_underflow();
-		return;
-	} else if (st0_tag == TAG_Zero) {
-		if (st1_tag == TAG_Valid) {
-			setcc(0);
-			return;
-		} else if (st1_tag == TW_Denormal) {
-			if (denormal_operand() < 0)
-				return;
-			setcc(0);
-			return;
-		} else if (st1_tag == TAG_Zero) {
-			arith_invalid(0);
-			return;
-		} /* fprem(?,0) always invalid */
-		else if (st1_tag == TW_Infinity) {
-			setcc(0);
-			return;
-		}
-	} else if ((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) {
-		if (st1_tag == TAG_Zero) {
-			arith_invalid(0);	/* fprem(Valid,Zero) is invalid */
-			return;
-		} else if (st1_tag != TW_NaN) {
-			if (((st0_tag == TW_Denormal)
-			     || (st1_tag == TW_Denormal))
-			    && (denormal_operand() < 0))
-				return;
-
-			if (st1_tag == TW_Infinity) {
-				/* fprem(Valid,Infinity) is o.k. */
-				setcc(0);
-				return;
-			}
-		}
-	} else if (st0_tag == TW_Infinity) {
-		if (st1_tag != TW_NaN) {
-			arith_invalid(0);	/* fprem(Infinity,?) is invalid */
-			return;
-		}
-	}
-
-	/* One of the registers must contain a NaN if we got here. */
-
-#ifdef PARANOID
-	if ((st0_tag != TW_NaN) && (st1_tag != TW_NaN))
-		EXCEPTION(EX_INTERNAL | 0x118);
-#endif /* PARANOID */
-
-	real_2op_NaN(st1_ptr, st1_tag, 0, st1_ptr);
-
-}
-
-/* ST(1) <- ST(1) * log ST;  pop ST */
-static void fyl2x(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	FPU_REG *st1_ptr = &st(1), exponent;
-	u_char st1_tag = FPU_gettagi(1);
-	u_char sign;
-	int e, tag;
-
-	clear_C1();
-
-	if ((st0_tag == TAG_Valid) && (st1_tag == TAG_Valid)) {
-	      both_valid:
-		/* Both regs are Valid or Denormal */
-		if (signpositive(st0_ptr)) {
-			if (st0_tag == TW_Denormal)
-				FPU_to_exp16(st0_ptr, st0_ptr);
-			else
-				/* Convert st(0) for internal use. */
-				setexponent16(st0_ptr, exponent(st0_ptr));
-
-			if ((st0_ptr->sigh == 0x80000000)
-			    && (st0_ptr->sigl == 0)) {
-				/* Special case. The result can be precise. */
-				u_char esign;
-				e = exponent16(st0_ptr);
-				if (e >= 0) {
-					exponent.sigh = e;
-					esign = SIGN_POS;
-				} else {
-					exponent.sigh = -e;
-					esign = SIGN_NEG;
-				}
-				exponent.sigl = 0;
-				setexponent16(&exponent, 31);
-				tag = FPU_normalize_nuo(&exponent);
-				stdexp(&exponent);
-				setsign(&exponent, esign);
-				tag =
-				    FPU_mul(&exponent, tag, 1, FULL_PRECISION);
-				if (tag >= 0)
-					FPU_settagi(1, tag);
-			} else {
-				/* The usual case */
-				sign = getsign(st1_ptr);
-				if (st1_tag == TW_Denormal)
-					FPU_to_exp16(st1_ptr, st1_ptr);
-				else
-					/* Convert st(1) for internal use. */
-					setexponent16(st1_ptr,
-						      exponent(st1_ptr));
-				poly_l2(st0_ptr, st1_ptr, sign);
-			}
-		} else {
-			/* negative */
-			if (arith_invalid(1) < 0)
-				return;
-		}
-
-		FPU_pop();
-
-		return;
-	}
-
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-	if (st1_tag == TAG_Special)
-		st1_tag = FPU_Special(st1_ptr);
-
-	if ((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty)) {
-		FPU_stack_underflow_pop(1);
-		return;
-	} else if ((st0_tag <= TW_Denormal) && (st1_tag <= TW_Denormal)) {
-		if (st0_tag == TAG_Zero) {
-			if (st1_tag == TAG_Zero) {
-				/* Both args zero is invalid */
-				if (arith_invalid(1) < 0)
-					return;
-			} else {
-				u_char sign;
-				sign = getsign(st1_ptr) ^ SIGN_NEG;
-				if (FPU_divide_by_zero(1, sign) < 0)
-					return;
-
-				setsign(st1_ptr, sign);
-			}
-		} else if (st1_tag == TAG_Zero) {
-			/* st(1) contains zero, st(0) valid <> 0 */
-			/* Zero is the valid answer */
-			sign = getsign(st1_ptr);
-
-			if (signnegative(st0_ptr)) {
-				/* log(negative) */
-				if (arith_invalid(1) < 0)
-					return;
-			} else if ((st0_tag == TW_Denormal)
-				   && (denormal_operand() < 0))
-				return;
-			else {
-				if (exponent(st0_ptr) < 0)
-					sign ^= SIGN_NEG;
-
-				FPU_copy_to_reg1(&CONST_Z, TAG_Zero);
-				setsign(st1_ptr, sign);
-			}
-		} else {
-			/* One or both operands are denormals. */
-			if (denormal_operand() < 0)
-				return;
-			goto both_valid;
-		}
-	} else if ((st0_tag == TW_NaN) || (st1_tag == TW_NaN)) {
-		if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
-			return;
-	}
-	/* One or both arg must be an infinity */
-	else if (st0_tag == TW_Infinity) {
-		if ((signnegative(st0_ptr)) || (st1_tag == TAG_Zero)) {
-			/* log(-infinity) or 0*log(infinity) */
-			if (arith_invalid(1) < 0)
-				return;
-		} else {
-			u_char sign = getsign(st1_ptr);
-
-			if ((st1_tag == TW_Denormal)
-			    && (denormal_operand() < 0))
-				return;
-
-			FPU_copy_to_reg1(&CONST_INF, TAG_Special);
-			setsign(st1_ptr, sign);
-		}
-	}
-	/* st(1) must be infinity here */
-	else if (((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal))
-		 && (signpositive(st0_ptr))) {
-		if (exponent(st0_ptr) >= 0) {
-			if ((exponent(st0_ptr) == 0) &&
-			    (st0_ptr->sigh == 0x80000000) &&
-			    (st0_ptr->sigl == 0)) {
-				/* st(0) holds 1.0 */
-				/* infinity*log(1) */
-				if (arith_invalid(1) < 0)
-					return;
-			}
-			/* else st(0) is positive and > 1.0 */
-		} else {
-			/* st(0) is positive and < 1.0 */
-
-			if ((st0_tag == TW_Denormal)
-			    && (denormal_operand() < 0))
-				return;
-
-			changesign(st1_ptr);
-		}
-	} else {
-		/* st(0) must be zero or negative */
-		if (st0_tag == TAG_Zero) {
-			/* This should be invalid, but a real 80486 is happy with it. */
-
-#ifndef PECULIAR_486
-			sign = getsign(st1_ptr);
-			if (FPU_divide_by_zero(1, sign) < 0)
-				return;
-#endif /* PECULIAR_486 */
-
-			changesign(st1_ptr);
-		} else if (arith_invalid(1) < 0)	/* log(negative) */
-			return;
-	}
-
-	FPU_pop();
-}
-
-static void fpatan(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	FPU_REG *st1_ptr = &st(1);
-	u_char st1_tag = FPU_gettagi(1);
-	int tag;
-
-	clear_C1();
-	if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) {
-	      valid_atan:
-
-		poly_atan(st0_ptr, st0_tag, st1_ptr, st1_tag);
-
-		FPU_pop();
-
-		return;
-	}
-
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-	if (st1_tag == TAG_Special)
-		st1_tag = FPU_Special(st1_ptr);
-
-	if (((st0_tag == TAG_Valid) && (st1_tag == TW_Denormal))
-	    || ((st0_tag == TW_Denormal) && (st1_tag == TAG_Valid))
-	    || ((st0_tag == TW_Denormal) && (st1_tag == TW_Denormal))) {
-		if (denormal_operand() < 0)
-			return;
-
-		goto valid_atan;
-	} else if ((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty)) {
-		FPU_stack_underflow_pop(1);
-		return;
-	} else if ((st0_tag == TW_NaN) || (st1_tag == TW_NaN)) {
-		if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) >= 0)
-			FPU_pop();
-		return;
-	} else if ((st0_tag == TW_Infinity) || (st1_tag == TW_Infinity)) {
-		u_char sign = getsign(st1_ptr);
-		if (st0_tag == TW_Infinity) {
-			if (st1_tag == TW_Infinity) {
-				if (signpositive(st0_ptr)) {
-					FPU_copy_to_reg1(&CONST_PI4, TAG_Valid);
-				} else {
-					setpositive(st1_ptr);
-					tag =
-					    FPU_u_add(&CONST_PI4, &CONST_PI2,
-						      st1_ptr, FULL_PRECISION,
-						      SIGN_POS,
-						      exponent(&CONST_PI4),
-						      exponent(&CONST_PI2));
-					if (tag >= 0)
-						FPU_settagi(1, tag);
-				}
-			} else {
-				if ((st1_tag == TW_Denormal)
-				    && (denormal_operand() < 0))
-					return;
-
-				if (signpositive(st0_ptr)) {
-					FPU_copy_to_reg1(&CONST_Z, TAG_Zero);
-					setsign(st1_ptr, sign);	/* An 80486 preserves the sign */
-					FPU_pop();
-					return;
-				} else {
-					FPU_copy_to_reg1(&CONST_PI, TAG_Valid);
-				}
-			}
-		} else {
-			/* st(1) is infinity, st(0) not infinity */
-			if ((st0_tag == TW_Denormal)
-			    && (denormal_operand() < 0))
-				return;
-
-			FPU_copy_to_reg1(&CONST_PI2, TAG_Valid);
-		}
-		setsign(st1_ptr, sign);
-	} else if (st1_tag == TAG_Zero) {
-		/* st(0) must be valid or zero */
-		u_char sign = getsign(st1_ptr);
-
-		if ((st0_tag == TW_Denormal) && (denormal_operand() < 0))
-			return;
-
-		if (signpositive(st0_ptr)) {
-			/* An 80486 preserves the sign */
-			FPU_pop();
-			return;
-		}
-
-		FPU_copy_to_reg1(&CONST_PI, TAG_Valid);
-		setsign(st1_ptr, sign);
-	} else if (st0_tag == TAG_Zero) {
-		/* st(1) must be TAG_Valid here */
-		u_char sign = getsign(st1_ptr);
-
-		if ((st1_tag == TW_Denormal) && (denormal_operand() < 0))
-			return;
-
-		FPU_copy_to_reg1(&CONST_PI2, TAG_Valid);
-		setsign(st1_ptr, sign);
-	}
-#ifdef PARANOID
-	else
-		EXCEPTION(EX_INTERNAL | 0x125);
-#endif /* PARANOID */
-
-	FPU_pop();
-	set_precision_flag_up();	/* We do not really know if up or down */
-}
-
-static void fprem(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	do_fprem(st0_ptr, st0_tag, RC_CHOP);
-}
-
-static void fprem1(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	do_fprem(st0_ptr, st0_tag, RC_RND);
-}
-
-static void fyl2xp1(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	u_char sign, sign1;
-	FPU_REG *st1_ptr = &st(1), a, b;
-	u_char st1_tag = FPU_gettagi(1);
-
-	clear_C1();
-	if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) {
-	      valid_yl2xp1:
-
-		sign = getsign(st0_ptr);
-		sign1 = getsign(st1_ptr);
-
-		FPU_to_exp16(st0_ptr, &a);
-		FPU_to_exp16(st1_ptr, &b);
-
-		if (poly_l2p1(sign, sign1, &a, &b, st1_ptr))
-			return;
-
-		FPU_pop();
-		return;
-	}
-
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-	if (st1_tag == TAG_Special)
-		st1_tag = FPU_Special(st1_ptr);
-
-	if (((st0_tag == TAG_Valid) && (st1_tag == TW_Denormal))
-	    || ((st0_tag == TW_Denormal) && (st1_tag == TAG_Valid))
-	    || ((st0_tag == TW_Denormal) && (st1_tag == TW_Denormal))) {
-		if (denormal_operand() < 0)
-			return;
-
-		goto valid_yl2xp1;
-	} else if ((st0_tag == TAG_Empty) | (st1_tag == TAG_Empty)) {
-		FPU_stack_underflow_pop(1);
-		return;
-	} else if (st0_tag == TAG_Zero) {
-		switch (st1_tag) {
-		case TW_Denormal:
-			if (denormal_operand() < 0)
-				return;
-			fallthrough;
-		case TAG_Zero:
-		case TAG_Valid:
-			setsign(st0_ptr, getsign(st0_ptr) ^ getsign(st1_ptr));
-			FPU_copy_to_reg1(st0_ptr, st0_tag);
-			break;
-
-		case TW_Infinity:
-			/* Infinity*log(1) */
-			if (arith_invalid(1) < 0)
-				return;
-			break;
-
-		case TW_NaN:
-			if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
-				return;
-			break;
-
-		default:
-#ifdef PARANOID
-			EXCEPTION(EX_INTERNAL | 0x116);
-			return;
-#endif /* PARANOID */
-			break;
-		}
-	} else if ((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) {
-		switch (st1_tag) {
-		case TAG_Zero:
-			if (signnegative(st0_ptr)) {
-				if (exponent(st0_ptr) >= 0) {
-					/* st(0) holds <= -1.0 */
-#ifdef PECULIAR_486		/* Stupid 80486 doesn't worry about log(negative). */
-					changesign(st1_ptr);
-#else
-					if (arith_invalid(1) < 0)
-						return;
-#endif /* PECULIAR_486 */
-				} else if ((st0_tag == TW_Denormal)
-					   && (denormal_operand() < 0))
-					return;
-				else
-					changesign(st1_ptr);
-			} else if ((st0_tag == TW_Denormal)
-				   && (denormal_operand() < 0))
-				return;
-			break;
-
-		case TW_Infinity:
-			if (signnegative(st0_ptr)) {
-				if ((exponent(st0_ptr) >= 0) &&
-				    !((st0_ptr->sigh == 0x80000000) &&
-				      (st0_ptr->sigl == 0))) {
-					/* st(0) holds < -1.0 */
-#ifdef PECULIAR_486		/* Stupid 80486 doesn't worry about log(negative). */
-					changesign(st1_ptr);
-#else
-					if (arith_invalid(1) < 0)
-						return;
-#endif /* PECULIAR_486 */
-				} else if ((st0_tag == TW_Denormal)
-					   && (denormal_operand() < 0))
-					return;
-				else
-					changesign(st1_ptr);
-			} else if ((st0_tag == TW_Denormal)
-				   && (denormal_operand() < 0))
-				return;
-			break;
-
-		case TW_NaN:
-			if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
-				return;
-		}
-
-	} else if (st0_tag == TW_NaN) {
-		if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
-			return;
-	} else if (st0_tag == TW_Infinity) {
-		if (st1_tag == TW_NaN) {
-			if (real_2op_NaN(st0_ptr, st0_tag, 1, st0_ptr) < 0)
-				return;
-		} else if (signnegative(st0_ptr)) {
-#ifndef PECULIAR_486
-			/* This should have higher priority than denormals, but... */
-			if (arith_invalid(1) < 0)	/* log(-infinity) */
-				return;
-#endif /* PECULIAR_486 */
-			if ((st1_tag == TW_Denormal)
-			    && (denormal_operand() < 0))
-				return;
-#ifdef PECULIAR_486
-			/* Denormal operands actually get higher priority */
-			if (arith_invalid(1) < 0)	/* log(-infinity) */
-				return;
-#endif /* PECULIAR_486 */
-		} else if (st1_tag == TAG_Zero) {
-			/* log(infinity) */
-			if (arith_invalid(1) < 0)
-				return;
-		}
-
-		/* st(1) must be valid here. */
-
-		else if ((st1_tag == TW_Denormal) && (denormal_operand() < 0))
-			return;
-
-		/* The Manual says that log(Infinity) is invalid, but a real
-		   80486 sensibly says that it is o.k. */
-		else {
-			u_char sign = getsign(st1_ptr);
-			FPU_copy_to_reg1(&CONST_INF, TAG_Special);
-			setsign(st1_ptr, sign);
-		}
-	}
-#ifdef PARANOID
-	else {
-		EXCEPTION(EX_INTERNAL | 0x117);
-		return;
-	}
-#endif /* PARANOID */
-
-	FPU_pop();
-	return;
-
-}
-
-static void fscale(FPU_REG *st0_ptr, u_char st0_tag)
-{
-	FPU_REG *st1_ptr = &st(1);
-	u_char st1_tag = FPU_gettagi(1);
-	int old_cw = control_word;
-	u_char sign = getsign(st0_ptr);
-
-	clear_C1();
-	if (!((st0_tag ^ TAG_Valid) | (st1_tag ^ TAG_Valid))) {
-		long scale;
-		FPU_REG tmp;
-
-		/* Convert register for internal use. */
-		setexponent16(st0_ptr, exponent(st0_ptr));
-
-	      valid_scale:
-
-		if (exponent(st1_ptr) > 30) {
-			/* 2^31 is far too large, would require 2^(2^30) or 2^(-2^30) */
-
-			if (signpositive(st1_ptr)) {
-				EXCEPTION(EX_Overflow);
-				FPU_copy_to_reg0(&CONST_INF, TAG_Special);
-			} else {
-				EXCEPTION(EX_Underflow);
-				FPU_copy_to_reg0(&CONST_Z, TAG_Zero);
-			}
-			setsign(st0_ptr, sign);
-			return;
-		}
-
-		control_word &= ~CW_RC;
-		control_word |= RC_CHOP;
-		reg_copy(st1_ptr, &tmp);
-		FPU_round_to_int(&tmp, st1_tag);	/* This can never overflow here */
-		control_word = old_cw;
-		scale = signnegative(st1_ptr) ? -tmp.sigl : tmp.sigl;
-		scale += exponent16(st0_ptr);
-
-		setexponent16(st0_ptr, scale);
-
-		/* Use FPU_round() to properly detect under/overflow etc */
-		FPU_round(st0_ptr, 0, 0, control_word, sign);
-
-		return;
-	}
-
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-	if (st1_tag == TAG_Special)
-		st1_tag = FPU_Special(st1_ptr);
-
-	if ((st0_tag == TAG_Valid) || (st0_tag == TW_Denormal)) {
-		switch (st1_tag) {
-		case TAG_Valid:
-			/* st(0) must be a denormal */
-			if ((st0_tag == TW_Denormal)
-			    && (denormal_operand() < 0))
-				return;
-
-			FPU_to_exp16(st0_ptr, st0_ptr);	/* Will not be left on stack */
-			goto valid_scale;
-
-		case TAG_Zero:
-			if (st0_tag == TW_Denormal)
-				denormal_operand();
-			return;
-
-		case TW_Denormal:
-			denormal_operand();
-			return;
-
-		case TW_Infinity:
-			if ((st0_tag == TW_Denormal)
-			    && (denormal_operand() < 0))
-				return;
-
-			if (signpositive(st1_ptr))
-				FPU_copy_to_reg0(&CONST_INF, TAG_Special);
-			else
-				FPU_copy_to_reg0(&CONST_Z, TAG_Zero);
-			setsign(st0_ptr, sign);
-			return;
-
-		case TW_NaN:
-			real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr);
-			return;
-		}
-	} else if (st0_tag == TAG_Zero) {
-		switch (st1_tag) {
-		case TAG_Valid:
-		case TAG_Zero:
-			return;
-
-		case TW_Denormal:
-			denormal_operand();
-			return;
-
-		case TW_Infinity:
-			if (signpositive(st1_ptr))
-				arith_invalid(0);	/* Zero scaled by +Infinity */
-			return;
-
-		case TW_NaN:
-			real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr);
-			return;
-		}
-	} else if (st0_tag == TW_Infinity) {
-		switch (st1_tag) {
-		case TAG_Valid:
-		case TAG_Zero:
-			return;
-
-		case TW_Denormal:
-			denormal_operand();
-			return;
-
-		case TW_Infinity:
-			if (signnegative(st1_ptr))
-				arith_invalid(0);	/* Infinity scaled by -Infinity */
-			return;
-
-		case TW_NaN:
-			real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr);
-			return;
-		}
-	} else if (st0_tag == TW_NaN) {
-		if (st1_tag != TAG_Empty) {
-			real_2op_NaN(st1_ptr, st1_tag, 0, st0_ptr);
-			return;
-		}
-	}
-#ifdef PARANOID
-	if (!((st0_tag == TAG_Empty) || (st1_tag == TAG_Empty))) {
-		EXCEPTION(EX_INTERNAL | 0x115);
-		return;
-	}
-#endif
-
-	/* At least one of st(0), st(1) must be empty */
-	FPU_stack_underflow();
-
-}
-
-/*---------------------------------------------------------------------------*/
-
-static FUNC_ST0 const trig_table_a[] = {
-	f2xm1, fyl2x, fptan, fpatan,
-	fxtract, fprem1, fdecstp, fincstp,
-};
-
-void FPU_triga(void)
-{
-	(trig_table_a[FPU_rm]) (&st(0), FPU_gettag0());
-}
-
-static FUNC_ST0 const trig_table_b[] = {
-	fprem, fyl2xp1, fsqrt_, fsincos, frndint_, fscale, fsin, fcos
-};
-
-void FPU_trigb(void)
-{
-	(trig_table_b[FPU_rm]) (&st(0), FPU_gettag0());
-}
diff --git a/arch/x86/math-emu/get_address.c b/arch/x86/math-emu/get_address.c
deleted file mode 100644
index 4a9fd9029a536..0000000000000
--- a/arch/x86/math-emu/get_address.c
+++ /dev/null
@@ -1,401 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  get_address.c                                                            |
- |                                                                           |
- | Get the effective address from an FPU instruction.                        |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@suburbia.net             |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note:                                                                     |
- |    The file contains code which accesses user memory.                     |
- |    Emulator static data may change when user memory is accessed, due to   |
- |    other processes using the emulator while swapping is in progress.      |
- +---------------------------------------------------------------------------*/
-
-#include <linux/stddef.h>
-
-#include <linux/uaccess.h>
-#include <asm/vm86.h>
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-
-#define FPU_WRITE_BIT 0x10
-
-static int reg_offset[] = {
-	offsetof(struct pt_regs, ax),
-	offsetof(struct pt_regs, cx),
-	offsetof(struct pt_regs, dx),
-	offsetof(struct pt_regs, bx),
-	offsetof(struct pt_regs, sp),
-	offsetof(struct pt_regs, bp),
-	offsetof(struct pt_regs, si),
-	offsetof(struct pt_regs, di)
-};
-
-#define REG_(x) (*(long *)(reg_offset[(x)] + (u_char *)FPU_info->regs))
-
-static int reg_offset_vm86[] = {
-	offsetof(struct pt_regs, cs),
-	offsetof(struct kernel_vm86_regs, ds),
-	offsetof(struct kernel_vm86_regs, es),
-	offsetof(struct kernel_vm86_regs, fs),
-	offsetof(struct kernel_vm86_regs, gs),
-	offsetof(struct pt_regs, ss),
-	offsetof(struct kernel_vm86_regs, ds)
-};
-
-#define VM86_REG_(x) (*(unsigned short *) \
-		(reg_offset_vm86[((unsigned)x)] + (u_char *)FPU_info->regs))
-
-static int reg_offset_pm[] = {
-	offsetof(struct pt_regs, cs),
-	offsetof(struct pt_regs, ds),
-	offsetof(struct pt_regs, es),
-	offsetof(struct pt_regs, fs),
-	offsetof(struct pt_regs, ds),	/* dummy, not saved on stack */
-	offsetof(struct pt_regs, ss),
-	offsetof(struct pt_regs, ds)
-};
-
-#define PM_REG_(x) (*(unsigned short *) \
-		(reg_offset_pm[((unsigned)x)] + (u_char *)FPU_info->regs))
-
-/* Decode the SIB byte. This function assumes mod != 0 */
-static int sib(int mod, unsigned long *fpu_eip)
-{
-	u_char ss, index, base;
-	long offset;
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_code_access_ok(1);
-	FPU_get_user(base, (u_char __user *) (*fpu_eip));	/* The SIB byte */
-	RE_ENTRANT_CHECK_ON;
-	(*fpu_eip)++;
-	ss = base >> 6;
-	index = (base >> 3) & 7;
-	base &= 7;
-
-	if ((mod == 0) && (base == 5))
-		offset = 0;	/* No base register */
-	else
-		offset = REG_(base);
-
-	if (index == 4) {
-		/* No index register */
-		/* A non-zero ss is illegal */
-		if (ss)
-			EXCEPTION(EX_Invalid);
-	} else {
-		offset += (REG_(index)) << ss;
-	}
-
-	if (mod == 1) {
-		/* 8 bit signed displacement */
-		long displacement;
-		RE_ENTRANT_CHECK_OFF;
-		FPU_code_access_ok(1);
-		FPU_get_user(displacement, (signed char __user *)(*fpu_eip));
-		offset += displacement;
-		RE_ENTRANT_CHECK_ON;
-		(*fpu_eip)++;
-	} else if (mod == 2 || base == 5) {	/* The second condition also has mod==0 */
-		/* 32 bit displacement */
-		long displacement;
-		RE_ENTRANT_CHECK_OFF;
-		FPU_code_access_ok(4);
-		FPU_get_user(displacement, (long __user *)(*fpu_eip));
-		offset += displacement;
-		RE_ENTRANT_CHECK_ON;
-		(*fpu_eip) += 4;
-	}
-
-	return offset;
-}
-
-static unsigned long vm86_segment(u_char segment, struct address *addr)
-{
-	segment--;
-#ifdef PARANOID
-	if (segment > PREFIX_SS_) {
-		EXCEPTION(EX_INTERNAL | 0x130);
-		math_abort(FPU_info, SIGSEGV);
-	}
-#endif /* PARANOID */
-	addr->selector = VM86_REG_(segment);
-	return (unsigned long)VM86_REG_(segment) << 4;
-}
-
-/* This should work for 16 and 32 bit protected mode. */
-static long pm_address(u_char FPU_modrm, u_char segment,
-		       struct address *addr, long offset)
-{
-	struct desc_struct descriptor;
-	unsigned long base_address, limit, address, seg_top;
-
-	segment--;
-
-#ifdef PARANOID
-	/* segment is unsigned, so this also detects if segment was 0: */
-	if (segment > PREFIX_SS_) {
-		EXCEPTION(EX_INTERNAL | 0x132);
-		math_abort(FPU_info, SIGSEGV);
-	}
-#endif /* PARANOID */
-
-	switch (segment) {
-	case PREFIX_GS_ - 1:
-		/* user gs handling can be lazy, use special accessors */
-		savesegment(gs, addr->selector);
-		break;
-	default:
-		addr->selector = PM_REG_(segment);
-	}
-
-	descriptor = FPU_get_ldt_descriptor(addr->selector);
-	base_address = seg_get_base(&descriptor);
-	address = base_address + offset;
-	limit = seg_get_limit(&descriptor) + 1;
-	limit *= seg_get_granularity(&descriptor);
-	limit += base_address - 1;
-	if (limit < base_address)
-		limit = 0xffffffff;
-
-	if (seg_expands_down(&descriptor)) {
-		if (descriptor.g) {
-			seg_top = 0xffffffff;
-		} else {
-			seg_top = base_address + (1 << 20);
-			if (seg_top < base_address)
-				seg_top = 0xffffffff;
-		}
-		access_limit =
-		    (address <= limit) || (address >= seg_top) ? 0 :
-		    ((seg_top - address) >= 255 ? 255 : seg_top - address);
-	} else {
-		access_limit =
-		    (address > limit) || (address < base_address) ? 0 :
-		    ((limit - address) >= 254 ? 255 : limit - address + 1);
-	}
-	if (seg_execute_only(&descriptor) ||
-	    (!seg_writable(&descriptor) && (FPU_modrm & FPU_WRITE_BIT))) {
-		access_limit = 0;
-	}
-	return address;
-}
-
-/*
-       MOD R/M byte:  MOD == 3 has a special use for the FPU
-                      SIB byte used iff R/M = 100b
-
-       7   6   5   4   3   2   1   0
-       .....   .........   .........
-        MOD    OPCODE(2)     R/M
-
-       SIB byte
-
-       7   6   5   4   3   2   1   0
-       .....   .........   .........
-        SS      INDEX        BASE
-
-*/
-
-void __user *FPU_get_address(u_char FPU_modrm, unsigned long *fpu_eip,
-			     struct address *addr, fpu_addr_modes addr_modes)
-{
-	u_char mod;
-	unsigned rm = FPU_modrm & 7;
-	long *cpu_reg_ptr;
-	int address = 0;	/* Initialized just to stop compiler warnings. */
-
-	/* Memory accessed via the cs selector is write protected
-	   in `non-segmented' 32 bit protected mode. */
-	if (!addr_modes.default_mode && (FPU_modrm & FPU_WRITE_BIT)
-	    && (addr_modes.override.segment == PREFIX_CS_)) {
-		math_abort(FPU_info, SIGSEGV);
-	}
-
-	addr->selector = FPU_DS;	/* Default, for 32 bit non-segmented mode. */
-
-	mod = (FPU_modrm >> 6) & 3;
-
-	if (rm == 4 && mod != 3) {
-		address = sib(mod, fpu_eip);
-	} else {
-		cpu_reg_ptr = &REG_(rm);
-		switch (mod) {
-		case 0:
-			if (rm == 5) {
-				/* Special case: disp32 */
-				RE_ENTRANT_CHECK_OFF;
-				FPU_code_access_ok(4);
-				FPU_get_user(address,
-					     (unsigned long __user
-					      *)(*fpu_eip));
-				(*fpu_eip) += 4;
-				RE_ENTRANT_CHECK_ON;
-				addr->offset = address;
-				return (void __user *)address;
-			} else {
-				address = *cpu_reg_ptr;	/* Just return the contents
-							   of the cpu register */
-				addr->offset = address;
-				return (void __user *)address;
-			}
-		case 1:
-			/* 8 bit signed displacement */
-			RE_ENTRANT_CHECK_OFF;
-			FPU_code_access_ok(1);
-			FPU_get_user(address, (signed char __user *)(*fpu_eip));
-			RE_ENTRANT_CHECK_ON;
-			(*fpu_eip)++;
-			break;
-		case 2:
-			/* 32 bit displacement */
-			RE_ENTRANT_CHECK_OFF;
-			FPU_code_access_ok(4);
-			FPU_get_user(address, (long __user *)(*fpu_eip));
-			(*fpu_eip) += 4;
-			RE_ENTRANT_CHECK_ON;
-			break;
-		case 3:
-			/* Not legal for the FPU */
-			EXCEPTION(EX_Invalid);
-		}
-		address += *cpu_reg_ptr;
-	}
-
-	addr->offset = address;
-
-	switch (addr_modes.default_mode) {
-	case 0:
-		break;
-	case VM86:
-		address += vm86_segment(addr_modes.override.segment, addr);
-		break;
-	case PM16:
-	case SEG32:
-		address = pm_address(FPU_modrm, addr_modes.override.segment,
-				     addr, address);
-		break;
-	default:
-		EXCEPTION(EX_INTERNAL | 0x133);
-	}
-
-	return (void __user *)address;
-}
-
-void __user *FPU_get_address_16(u_char FPU_modrm, unsigned long *fpu_eip,
-				struct address *addr, fpu_addr_modes addr_modes)
-{
-	u_char mod;
-	unsigned rm = FPU_modrm & 7;
-	int address = 0;	/* Default used for mod == 0 */
-
-	/* Memory accessed via the cs selector is write protected
-	   in `non-segmented' 32 bit protected mode. */
-	if (!addr_modes.default_mode && (FPU_modrm & FPU_WRITE_BIT)
-	    && (addr_modes.override.segment == PREFIX_CS_)) {
-		math_abort(FPU_info, SIGSEGV);
-	}
-
-	addr->selector = FPU_DS;	/* Default, for 32 bit non-segmented mode. */
-
-	mod = (FPU_modrm >> 6) & 3;
-
-	switch (mod) {
-	case 0:
-		if (rm == 6) {
-			/* Special case: disp16 */
-			RE_ENTRANT_CHECK_OFF;
-			FPU_code_access_ok(2);
-			FPU_get_user(address,
-				     (unsigned short __user *)(*fpu_eip));
-			(*fpu_eip) += 2;
-			RE_ENTRANT_CHECK_ON;
-			goto add_segment;
-		}
-		break;
-	case 1:
-		/* 8 bit signed displacement */
-		RE_ENTRANT_CHECK_OFF;
-		FPU_code_access_ok(1);
-		FPU_get_user(address, (signed char __user *)(*fpu_eip));
-		RE_ENTRANT_CHECK_ON;
-		(*fpu_eip)++;
-		break;
-	case 2:
-		/* 16 bit displacement */
-		RE_ENTRANT_CHECK_OFF;
-		FPU_code_access_ok(2);
-		FPU_get_user(address, (unsigned short __user *)(*fpu_eip));
-		(*fpu_eip) += 2;
-		RE_ENTRANT_CHECK_ON;
-		break;
-	case 3:
-		/* Not legal for the FPU */
-		EXCEPTION(EX_Invalid);
-		break;
-	}
-	switch (rm) {
-	case 0:
-		address += FPU_info->regs->bx + FPU_info->regs->si;
-		break;
-	case 1:
-		address += FPU_info->regs->bx + FPU_info->regs->di;
-		break;
-	case 2:
-		address += FPU_info->regs->bp + FPU_info->regs->si;
-		if (addr_modes.override.segment == PREFIX_DEFAULT)
-			addr_modes.override.segment = PREFIX_SS_;
-		break;
-	case 3:
-		address += FPU_info->regs->bp + FPU_info->regs->di;
-		if (addr_modes.override.segment == PREFIX_DEFAULT)
-			addr_modes.override.segment = PREFIX_SS_;
-		break;
-	case 4:
-		address += FPU_info->regs->si;
-		break;
-	case 5:
-		address += FPU_info->regs->di;
-		break;
-	case 6:
-		address += FPU_info->regs->bp;
-		if (addr_modes.override.segment == PREFIX_DEFAULT)
-			addr_modes.override.segment = PREFIX_SS_;
-		break;
-	case 7:
-		address += FPU_info->regs->bx;
-		break;
-	}
-
-      add_segment:
-	address &= 0xffff;
-
-	addr->offset = address;
-
-	switch (addr_modes.default_mode) {
-	case 0:
-		break;
-	case VM86:
-		address += vm86_segment(addr_modes.override.segment, addr);
-		break;
-	case PM16:
-	case SEG32:
-		address = pm_address(FPU_modrm, addr_modes.override.segment,
-				     addr, address);
-		break;
-	default:
-		EXCEPTION(EX_INTERNAL | 0x131);
-	}
-
-	return (void __user *)address;
-}
diff --git a/arch/x86/math-emu/load_store.c b/arch/x86/math-emu/load_store.c
deleted file mode 100644
index 4092df79de4f5..0000000000000
--- a/arch/x86/math-emu/load_store.c
+++ /dev/null
@@ -1,322 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  load_store.c                                                             |
- |                                                                           |
- | This file contains most of the code to interpret the FPU instructions     |
- | which load and store from user memory.                                    |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@suburbia.net             |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note:                                                                     |
- |    The file contains code which accesses user memory.                     |
- |    Emulator static data may change when user memory is accessed, due to   |
- |    other processes using the emulator while swapping is in progress.      |
- +---------------------------------------------------------------------------*/
-
-#include <linux/uaccess.h>
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "control_w.h"
-
-#define _NONE_ 0		/* st0_ptr etc not needed */
-#define _REG0_ 1		/* Will be storing st(0) */
-#define _PUSH_ 3		/* Need to check for space to push onto stack */
-#define _null_ 4		/* Function illegal or not implemented */
-
-#define pop_0()	{ FPU_settag0(TAG_Empty); top++; }
-
-/* index is a 5-bit value: (3-bit FPU_modrm.reg field | opcode[2,1]) */
-static u_char const type_table[32] = {
-	_PUSH_, _PUSH_, _PUSH_, _PUSH_, /* /0: d9:fld f32,  db:fild m32,  dd:fld f64,  df:fild m16 */
-	_null_, _REG0_, _REG0_, _REG0_, /* /1: d9:undef,    db,dd,df:fisttp m32/64/16 */
-	_REG0_, _REG0_, _REG0_, _REG0_, /* /2: d9:fst f32,  db:fist m32,  dd:fst f64,  df:fist m16 */
-	_REG0_, _REG0_, _REG0_, _REG0_, /* /3: d9:fstp f32, db:fistp m32, dd:fstp f64, df:fistp m16 */
-	_NONE_, _null_, _NONE_, _PUSH_,
-	_NONE_, _PUSH_, _null_, _PUSH_,
-	_NONE_, _null_, _NONE_, _REG0_,
-	_NONE_, _REG0_, _NONE_, _REG0_
-};
-
-u_char const data_sizes_16[32] = {
-	4, 4, 8, 2,
-	0, 4, 8, 2, /* /1: d9:undef, db,dd,df:fisttp */
-	4, 4, 8, 2,
-	4, 4, 8, 2,
-	14, 0, 94, 10, 2, 10, 0, 8,
-	14, 0, 94, 10, 2, 10, 2, 8
-};
-
-static u_char const data_sizes_32[32] = {
-	4, 4, 8, 2,
-	0, 4, 8, 2, /* /1: d9:undef, db,dd,df:fisttp */
-	4, 4, 8, 2,
-	4, 4, 8, 2,
-	28, 0, 108, 10, 2, 10, 0, 8,
-	28, 0, 108, 10, 2, 10, 2, 8
-};
-
-int FPU_load_store(u_char type, fpu_addr_modes addr_modes,
-		   void __user * data_address)
-{
-	FPU_REG loaded_data;
-	FPU_REG *st0_ptr;
-	u_char st0_tag = TAG_Empty;	/* This is just to stop a gcc warning. */
-	u_char loaded_tag;
-	int sv_cw;
-
-	st0_ptr = NULL;		/* Initialized just to stop compiler warnings. */
-
-	if (addr_modes.default_mode & PROTECTED) {
-		if (addr_modes.default_mode == SEG32) {
-			if (access_limit < data_sizes_32[type])
-				math_abort(FPU_info, SIGSEGV);
-		} else if (addr_modes.default_mode == PM16) {
-			if (access_limit < data_sizes_16[type])
-				math_abort(FPU_info, SIGSEGV);
-		}
-#ifdef PARANOID
-		else
-			EXCEPTION(EX_INTERNAL | 0x140);
-#endif /* PARANOID */
-	}
-
-	switch (type_table[type]) {
-	case _NONE_:
-		break;
-	case _REG0_:
-		st0_ptr = &st(0);	/* Some of these instructions pop after
-					   storing */
-		st0_tag = FPU_gettag0();
-		break;
-	case _PUSH_:
-		{
-			if (FPU_gettagi(-1) != TAG_Empty) {
-				FPU_stack_overflow();
-				return 0;
-			}
-			top--;
-			st0_ptr = &st(0);
-		}
-		break;
-	case _null_:
-		FPU_illegal();
-		return 0;
-#ifdef PARANOID
-	default:
-		EXCEPTION(EX_INTERNAL | 0x141);
-		return 0;
-#endif /* PARANOID */
-	}
-
-	switch (type) {
-	/* type is a 5-bit value: (3-bit FPU_modrm.reg field | opcode[2,1]) */
-	case 000:		/* fld m32real (d9 /0) */
-		clear_C1();
-		loaded_tag =
-		    FPU_load_single((float __user *)data_address, &loaded_data);
-		if ((loaded_tag == TAG_Special)
-		    && isNaN(&loaded_data)
-		    && (real_1op_NaN(&loaded_data) < 0)) {
-			top++;
-			break;
-		}
-		FPU_copy_to_reg0(&loaded_data, loaded_tag);
-		break;
-	case 001:		/* fild m32int (db /0) */
-		clear_C1();
-		loaded_tag =
-		    FPU_load_int32((long __user *)data_address, &loaded_data);
-		FPU_copy_to_reg0(&loaded_data, loaded_tag);
-		break;
-	case 002:		/* fld m64real (dd /0) */
-		clear_C1();
-		loaded_tag =
-		    FPU_load_double((double __user *)data_address,
-				    &loaded_data);
-		if ((loaded_tag == TAG_Special)
-		    && isNaN(&loaded_data)
-		    && (real_1op_NaN(&loaded_data) < 0)) {
-			top++;
-			break;
-		}
-		FPU_copy_to_reg0(&loaded_data, loaded_tag);
-		break;
-	case 003:		/* fild m16int (df /0) */
-		clear_C1();
-		loaded_tag =
-		    FPU_load_int16((short __user *)data_address, &loaded_data);
-		FPU_copy_to_reg0(&loaded_data, loaded_tag);
-		break;
-	/* case 004: undefined (d9 /1) */
-	/* fisttp are enabled if CPUID(1).ECX(0) "sse3" is set */
-	case 005:		/* fisttp m32int (db /1) */
-		clear_C1();
-		sv_cw = control_word;
-		control_word |= RC_CHOP;
-		if (FPU_store_int32
-		    (st0_ptr, st0_tag, (long __user *)data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		control_word = sv_cw;
-		break;
-	case 006:		/* fisttp m64int (dd /1) */
-		clear_C1();
-		sv_cw = control_word;
-		control_word |= RC_CHOP;
-		if (FPU_store_int64
-		    (st0_ptr, st0_tag, (long long __user *)data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		control_word = sv_cw;
-		break;
-	case 007:		/* fisttp m16int (df /1) */
-		clear_C1();
-		sv_cw = control_word;
-		control_word |= RC_CHOP;
-		if (FPU_store_int16
-		    (st0_ptr, st0_tag, (short __user *)data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		control_word = sv_cw;
-		break;
-	case 010:		/* fst m32real */
-		clear_C1();
-		FPU_store_single(st0_ptr, st0_tag,
-				 (float __user *)data_address);
-		break;
-	case 011:		/* fist m32int */
-		clear_C1();
-		FPU_store_int32(st0_ptr, st0_tag, (long __user *)data_address);
-		break;
-	case 012:		/* fst m64real */
-		clear_C1();
-		FPU_store_double(st0_ptr, st0_tag,
-				 (double __user *)data_address);
-		break;
-	case 013:		/* fist m16int */
-		clear_C1();
-		FPU_store_int16(st0_ptr, st0_tag, (short __user *)data_address);
-		break;
-	case 014:		/* fstp m32real */
-		clear_C1();
-		if (FPU_store_single
-		    (st0_ptr, st0_tag, (float __user *)data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		break;
-	case 015:		/* fistp m32int */
-		clear_C1();
-		if (FPU_store_int32
-		    (st0_ptr, st0_tag, (long __user *)data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		break;
-	case 016:		/* fstp m64real */
-		clear_C1();
-		if (FPU_store_double
-		    (st0_ptr, st0_tag, (double __user *)data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		break;
-	case 017:		/* fistp m16int */
-		clear_C1();
-		if (FPU_store_int16
-		    (st0_ptr, st0_tag, (short __user *)data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		break;
-	case 020:		/* fldenv  m14/28byte */
-		fldenv(addr_modes, (u_char __user *) data_address);
-		/* Ensure that the values just loaded are not changed by
-		   fix-up operations. */
-		return 1;
-	case 022:		/* frstor m94/108byte */
-		FPU_frstor(addr_modes, (u_char __user *) data_address);
-		/* Ensure that the values just loaded are not changed by
-		   fix-up operations. */
-		return 1;
-	case 023:		/* fbld m80dec */
-		clear_C1();
-		loaded_tag = FPU_load_bcd((u_char __user *) data_address);
-		FPU_settag0(loaded_tag);
-		break;
-	case 024:		/* fldcw */
-		RE_ENTRANT_CHECK_OFF;
-		FPU_access_ok(data_address, 2);
-		FPU_get_user(control_word,
-			     (unsigned short __user *)data_address);
-		RE_ENTRANT_CHECK_ON;
-		if (partial_status & ~control_word & CW_Exceptions)
-			partial_status |= (SW_Summary | SW_Backward);
-		else
-			partial_status &= ~(SW_Summary | SW_Backward);
-#ifdef PECULIAR_486
-		control_word |= 0x40;	/* An 80486 appears to always set this bit */
-#endif /* PECULIAR_486 */
-		return 1;
-	case 025:		/* fld m80real */
-		clear_C1();
-		loaded_tag =
-		    FPU_load_extended((long double __user *)data_address, 0);
-		FPU_settag0(loaded_tag);
-		break;
-	case 027:		/* fild m64int */
-		clear_C1();
-		loaded_tag = FPU_load_int64((long long __user *)data_address);
-		if (loaded_tag == TAG_Error)
-			return 0;
-		FPU_settag0(loaded_tag);
-		break;
-	case 030:		/* fstenv  m14/28byte */
-		fstenv(addr_modes, (u_char __user *) data_address);
-		return 1;
-	case 032:		/* fsave */
-		fsave(addr_modes, (u_char __user *) data_address);
-		return 1;
-	case 033:		/* fbstp m80dec */
-		clear_C1();
-		if (FPU_store_bcd
-		    (st0_ptr, st0_tag, (u_char __user *) data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		break;
-	case 034:		/* fstcw m16int */
-		RE_ENTRANT_CHECK_OFF;
-		FPU_access_ok(data_address, 2);
-		FPU_put_user(control_word,
-			     (unsigned short __user *)data_address);
-		RE_ENTRANT_CHECK_ON;
-		return 1;
-	case 035:		/* fstp m80real */
-		clear_C1();
-		if (FPU_store_extended
-		    (st0_ptr, st0_tag, (long double __user *)data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		break;
-	case 036:		/* fstsw m2byte */
-		RE_ENTRANT_CHECK_OFF;
-		FPU_access_ok(data_address, 2);
-		FPU_put_user(status_word(),
-			     (unsigned short __user *)data_address);
-		RE_ENTRANT_CHECK_ON;
-		return 1;
-	case 037:		/* fistp m64int */
-		clear_C1();
-		if (FPU_store_int64
-		    (st0_ptr, st0_tag, (long long __user *)data_address))
-			pop_0();	/* pop only if the number was actually stored
-					   (see the 80486 manual p16-28) */
-		break;
-	}
-	return 0;
-}
diff --git a/arch/x86/math-emu/mul_Xsig.S b/arch/x86/math-emu/mul_Xsig.S
deleted file mode 100644
index 54a031b661421..0000000000000
--- a/arch/x86/math-emu/mul_Xsig.S
+++ /dev/null
@@ -1,179 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  mul_Xsig.S                                                               |
- |                                                                           |
- | Multiply a 12 byte fixed point number by another fixed point number.      |
- |                                                                           |
- | Copyright (C) 1992,1994,1995                                              |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@jacobi.maths.monash.edu.au |
- |                                                                           |
- | Call from C as:                                                           |
- |   void mul32_Xsig(Xsig *x, unsigned b)                                    |
- |                                                                           |
- |   void mul64_Xsig(Xsig *x, unsigned long long *b)                         |
- |                                                                           |
- |   void mul_Xsig_Xsig(Xsig *x, unsigned *b)                                |
- |                                                                           |
- | The result is neither rounded nor normalized, and the ls bit or so may    |
- | be wrong.                                                                 |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-	.file	"mul_Xsig.S"
-
-
-#include "fpu_emu.h"
-
-.text
-SYM_FUNC_START(mul32_Xsig)
-	pushl %ebp
-	movl %esp,%ebp
-	subl $16,%esp
-	pushl %esi
-
-	movl PARAM1,%esi
-	movl PARAM2,%ecx
-
-	xor %eax,%eax
-	movl %eax,-4(%ebp)
-	movl %eax,-8(%ebp)
-
-	movl (%esi),%eax        /* lsl of Xsig */
-	mull %ecx		/* msl of b */
-	movl %edx,-12(%ebp)
-
-	movl 4(%esi),%eax	/* midl of Xsig */
-	mull %ecx		/* msl of b */
-	addl %eax,-12(%ebp)
-	adcl %edx,-8(%ebp)
-	adcl $0,-4(%ebp)
-
-	movl 8(%esi),%eax	/* msl of Xsig */
-	mull %ecx		/* msl of b */
-	addl %eax,-8(%ebp)
-	adcl %edx,-4(%ebp)
-
-	movl -12(%ebp),%eax
-	movl %eax,(%esi)
-	movl -8(%ebp),%eax
-	movl %eax,4(%esi)
-	movl -4(%ebp),%eax
-	movl %eax,8(%esi)
-
-	popl %esi
-	leave
-	RET
-SYM_FUNC_END(mul32_Xsig)
-
-
-SYM_FUNC_START(mul64_Xsig)
-	pushl %ebp
-	movl %esp,%ebp
-	subl $16,%esp
-	pushl %esi
-
-	movl PARAM1,%esi
-	movl PARAM2,%ecx
-
-	xor %eax,%eax
-	movl %eax,-4(%ebp)
-	movl %eax,-8(%ebp)
-
-	movl (%esi),%eax        /* lsl of Xsig */
-	mull 4(%ecx)		/* msl of b */
-	movl %edx,-12(%ebp)
-
-	movl 4(%esi),%eax	/* midl of Xsig */
-	mull (%ecx)		/* lsl of b */
-	addl %edx,-12(%ebp)
-	adcl $0,-8(%ebp)
-	adcl $0,-4(%ebp)
-
-	movl 4(%esi),%eax	/* midl of Xsig */
-	mull 4(%ecx)		/* msl of b */
-	addl %eax,-12(%ebp)
-	adcl %edx,-8(%ebp)
-	adcl $0,-4(%ebp)
-
-	movl 8(%esi),%eax	/* msl of Xsig */
-	mull (%ecx)		/* lsl of b */
-	addl %eax,-12(%ebp)
-	adcl %edx,-8(%ebp)
-	adcl $0,-4(%ebp)
-
-	movl 8(%esi),%eax	/* msl of Xsig */
-	mull 4(%ecx)		/* msl of b */
-	addl %eax,-8(%ebp)
-	adcl %edx,-4(%ebp)
-
-	movl -12(%ebp),%eax
-	movl %eax,(%esi)
-	movl -8(%ebp),%eax
-	movl %eax,4(%esi)
-	movl -4(%ebp),%eax
-	movl %eax,8(%esi)
-
-	popl %esi
-	leave
-	RET
-SYM_FUNC_END(mul64_Xsig)
-
-
-
-SYM_FUNC_START(mul_Xsig_Xsig)
-	pushl %ebp
-	movl %esp,%ebp
-	subl $16,%esp
-	pushl %esi
-
-	movl PARAM1,%esi
-	movl PARAM2,%ecx
-
-	xor %eax,%eax
-	movl %eax,-4(%ebp)
-	movl %eax,-8(%ebp)
-
-	movl (%esi),%eax        /* lsl of Xsig */
-	mull 8(%ecx)		/* msl of b */
-	movl %edx,-12(%ebp)
-
-	movl 4(%esi),%eax	/* midl of Xsig */
-	mull 4(%ecx)		/* midl of b */
-	addl %edx,-12(%ebp)
-	adcl $0,-8(%ebp)
-	adcl $0,-4(%ebp)
-
-	movl 8(%esi),%eax	/* msl of Xsig */
-	mull (%ecx)		/* lsl of b */
-	addl %edx,-12(%ebp)
-	adcl $0,-8(%ebp)
-	adcl $0,-4(%ebp)
-
-	movl 4(%esi),%eax	/* midl of Xsig */
-	mull 8(%ecx)		/* msl of b */
-	addl %eax,-12(%ebp)
-	adcl %edx,-8(%ebp)
-	adcl $0,-4(%ebp)
-
-	movl 8(%esi),%eax	/* msl of Xsig */
-	mull 4(%ecx)		/* midl of b */
-	addl %eax,-12(%ebp)
-	adcl %edx,-8(%ebp)
-	adcl $0,-4(%ebp)
-
-	movl 8(%esi),%eax	/* msl of Xsig */
-	mull 8(%ecx)		/* msl of b */
-	addl %eax,-8(%ebp)
-	adcl %edx,-4(%ebp)
-
-	movl -12(%ebp),%edx
-	movl %edx,(%esi)
-	movl -8(%ebp),%edx
-	movl %edx,4(%esi)
-	movl -4(%ebp),%edx
-	movl %edx,8(%esi)
-
-	popl %esi
-	leave
-	RET
-SYM_FUNC_END(mul_Xsig_Xsig)
diff --git a/arch/x86/math-emu/poly.h b/arch/x86/math-emu/poly.h
deleted file mode 100644
index 654bfe4e29a0c..0000000000000
--- a/arch/x86/math-emu/poly.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  poly.h                                                                   |
- |                                                                           |
- |  Header file for the FPU-emu poly*.c source files.                        |
- |                                                                           |
- | Copyright (C) 1994,1999                                                   |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@melbpc.org.au            |
- |                                                                           |
- | Declarations and definitions for functions operating on Xsig (12-byte     |
- | extended-significand) quantities.                                         |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#ifndef _POLY_H
-#define _POLY_H
-
-/* This 12-byte structure is used to improve the accuracy of computation
-   of transcendental functions.
-   Intended to be used to get results better than 8-byte computation
-   allows. 9-byte would probably be sufficient.
-   */
-typedef struct {
-	unsigned long lsw;
-	unsigned long midw;
-	unsigned long msw;
-} Xsig;
-
-asmlinkage void mul64(unsigned long long const *a, unsigned long long const *b,
-		      unsigned long long *result);
-asmlinkage void polynomial_Xsig(Xsig *, const unsigned long long *x,
-				const unsigned long long terms[], const int n);
-
-asmlinkage void mul32_Xsig(Xsig *, const unsigned long mult);
-asmlinkage void mul64_Xsig(Xsig *, const unsigned long long *mult);
-asmlinkage void mul_Xsig_Xsig(Xsig *dest, const Xsig *mult);
-
-asmlinkage void shr_Xsig(Xsig *, const int n);
-asmlinkage int round_Xsig(Xsig *);
-asmlinkage int norm_Xsig(Xsig *);
-asmlinkage void div_Xsig(Xsig *x1, const Xsig *x2, Xsig *dest);
-
-/* Macro to extract the most significant 32 bits from a long long */
-#define LL_MSW(x)     (((unsigned long *)&x)[1])
-
-/* Macro to initialize an Xsig struct */
-#define MK_XSIG(a,b,c)     { c, b, a }
-
-/* Macro to access the 8 ms bytes of an Xsig as a long long */
-#define XSIG_LL(x)         (*(unsigned long long *)&x.midw)
-
-/*
-   Need to run gcc with optimizations on to get these to
-   actually be in-line.
-   */
-
-/* Multiply two fixed-point 32 bit numbers, producing a 32 bit result.
-   The answer is the ms word of the product. */
-/* Some versions of gcc make it difficult to stop eax from being clobbered.
-   Merely specifying that it is used doesn't work...
- */
-static inline unsigned long mul_32_32(const unsigned long arg1,
-				      const unsigned long arg2)
-{
-	int retval;
-	asm volatile ("mull %2; movl %%edx,%%eax":"=a" (retval)
-		      :"0"(arg1), "g"(arg2)
-		      :"dx");
-	return retval;
-}
-
-/* Add the 12 byte Xsig x2 to Xsig dest, with no checks for overflow. */
-static inline void add_Xsig_Xsig(Xsig *dest, const Xsig *x2)
-{
-	asm volatile ("movl %1,%%edi; movl %2,%%esi;\n"
-		      "movl (%%esi),%%eax; addl %%eax,(%%edi);\n"
-		      "movl 4(%%esi),%%eax; adcl %%eax,4(%%edi);\n"
-		      "movl 8(%%esi),%%eax; adcl %%eax,8(%%edi);\n":"=g"
-		      (*dest):"g"(dest), "g"(x2)
-		      :"ax", "si", "di");
-}
-
-/* Add the 12 byte Xsig x2 to Xsig dest, adjust exp if overflow occurs. */
-/* Note: the constraints in the asm statement didn't always work properly
-   with gcc 2.5.8.  Changing from using edi to using ecx got around the
-   problem, but keep fingers crossed! */
-static inline void add_two_Xsig(Xsig *dest, const Xsig *x2, long int *exp)
-{
-	asm volatile ("movl %2,%%ecx; movl %3,%%esi;\n"
-		      "movl (%%esi),%%eax; addl %%eax,(%%ecx);\n"
-		      "movl 4(%%esi),%%eax; adcl %%eax,4(%%ecx);\n"
-		      "movl 8(%%esi),%%eax; adcl %%eax,8(%%ecx);\n"
-		      "jnc 0f;\n"
-		      "rcrl 8(%%ecx); rcrl 4(%%ecx); rcrl (%%ecx)\n"
-		      "movl %4,%%ecx; incl (%%ecx)\n"
-		      "movl $1,%%eax; jmp 1f;\n"
-		      "0: xorl %%eax,%%eax;\n" "1:\n":"=g" (*exp), "=g"(*dest)
-		      :"g"(dest), "g"(x2), "g"(exp)
-		      :"cx", "si", "ax");
-}
-
-/* Negate (subtract from 1.0) the 12 byte Xsig */
-/* This is faster in a loop on my 386 than using the "neg" instruction. */
-static inline void negate_Xsig(Xsig *x)
-{
-	asm volatile ("movl %1,%%esi;\n"
-		      "xorl %%ecx,%%ecx;\n"
-		      "movl %%ecx,%%eax; subl (%%esi),%%eax; movl %%eax,(%%esi);\n"
-		      "movl %%ecx,%%eax; sbbl 4(%%esi),%%eax; movl %%eax,4(%%esi);\n"
-		      "movl %%ecx,%%eax; sbbl 8(%%esi),%%eax; movl %%eax,8(%%esi);\n":"=g"
-		      (*x):"g"(x):"si", "ax", "cx");
-}
-
-#endif /* _POLY_H */
diff --git a/arch/x86/math-emu/poly_2xm1.c b/arch/x86/math-emu/poly_2xm1.c
deleted file mode 100644
index aa33006bafd5f..0000000000000
--- a/arch/x86/math-emu/poly_2xm1.c
+++ /dev/null
@@ -1,146 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  poly_2xm1.c                                                              |
- |                                                                           |
- | Function to compute 2^x-1 by a polynomial approximation.                  |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "control_w.h"
-#include "poly.h"
-
-#define	HIPOWER	11
-static const unsigned long long lterms[HIPOWER] = {
-	0x0000000000000000LL,	/* This term done separately as 12 bytes */
-	0xf5fdeffc162c7543LL,
-	0x1c6b08d704a0bfa6LL,
-	0x0276556df749cc21LL,
-	0x002bb0ffcf14f6b8LL,
-	0x0002861225ef751cLL,
-	0x00001ffcbfcd5422LL,
-	0x00000162c005d5f1LL,
-	0x0000000da96ccb1bLL,
-	0x0000000078d1b897LL,
-	0x000000000422b029LL
-};
-
-static const Xsig hiterm = MK_XSIG(0xb17217f7, 0xd1cf79ab, 0xc8a39194);
-
-/* Four slices: 0.0 : 0.25 : 0.50 : 0.75 : 1.0,
-   These numbers are 2^(1/4), 2^(1/2), and 2^(3/4)
- */
-static const Xsig shiftterm0 = MK_XSIG(0, 0, 0);
-static const Xsig shiftterm1 = MK_XSIG(0x9837f051, 0x8db8a96f, 0x46ad2318);
-static const Xsig shiftterm2 = MK_XSIG(0xb504f333, 0xf9de6484, 0x597d89b3);
-static const Xsig shiftterm3 = MK_XSIG(0xd744fcca, 0xd69d6af4, 0x39a68bb9);
-
-static const Xsig *shiftterm[] = { &shiftterm0, &shiftterm1,
-	&shiftterm2, &shiftterm3
-};
-
-/*--- poly_2xm1() -----------------------------------------------------------+
- | Requires st(0) which is TAG_Valid and < 1.                                |
- +---------------------------------------------------------------------------*/
-int poly_2xm1(u_char sign, FPU_REG *arg, FPU_REG *result)
-{
-	long int exponent, shift;
-	unsigned long long Xll;
-	Xsig accumulator, Denom, argSignif;
-	u_char tag;
-
-	exponent = exponent16(arg);
-
-#ifdef PARANOID
-	if (exponent >= 0) {	/* Don't want a |number| >= 1.0 */
-		/* Number negative, too large, or not Valid. */
-		EXCEPTION(EX_INTERNAL | 0x127);
-		return 1;
-	}
-#endif /* PARANOID */
-
-	argSignif.lsw = 0;
-	XSIG_LL(argSignif) = Xll = significand(arg);
-
-	if (exponent == -1) {
-		shift = (argSignif.msw & 0x40000000) ? 3 : 2;
-		/* subtract 0.5 or 0.75 */
-		exponent -= 2;
-		XSIG_LL(argSignif) <<= 2;
-		Xll <<= 2;
-	} else if (exponent == -2) {
-		shift = 1;
-		/* subtract 0.25 */
-		exponent--;
-		XSIG_LL(argSignif) <<= 1;
-		Xll <<= 1;
-	} else
-		shift = 0;
-
-	if (exponent < -2) {
-		/* Shift the argument right by the required places. */
-		if (FPU_shrx(&Xll, -2 - exponent) >= 0x80000000U)
-			Xll++;	/* round up */
-	}
-
-	accumulator.lsw = accumulator.midw = accumulator.msw = 0;
-	polynomial_Xsig(&accumulator, &Xll, lterms, HIPOWER - 1);
-	mul_Xsig_Xsig(&accumulator, &argSignif);
-	shr_Xsig(&accumulator, 3);
-
-	mul_Xsig_Xsig(&argSignif, &hiterm);	/* The leading term */
-	add_two_Xsig(&accumulator, &argSignif, &exponent);
-
-	if (shift) {
-		/* The argument is large, use the identity:
-		   f(x+a) = f(a) * (f(x) + 1) - 1;
-		 */
-		shr_Xsig(&accumulator, -exponent);
-		accumulator.msw |= 0x80000000;	/* add 1.0 */
-		mul_Xsig_Xsig(&accumulator, shiftterm[shift]);
-		accumulator.msw &= 0x3fffffff;	/* subtract 1.0 */
-		exponent = 1;
-	}
-
-	if (sign != SIGN_POS) {
-		/* The argument is negative, use the identity:
-		   f(-x) = -f(x) / (1 + f(x))
-		 */
-		Denom.lsw = accumulator.lsw;
-		XSIG_LL(Denom) = XSIG_LL(accumulator);
-		if (exponent < 0)
-			shr_Xsig(&Denom, -exponent);
-		else if (exponent > 0) {
-			/* exponent must be 1 here */
-			XSIG_LL(Denom) <<= 1;
-			if (Denom.lsw & 0x80000000)
-				XSIG_LL(Denom) |= 1;
-			(Denom.lsw) <<= 1;
-		}
-		Denom.msw |= 0x80000000;	/* add 1.0 */
-		div_Xsig(&accumulator, &Denom, &accumulator);
-	}
-
-	/* Convert to 64 bit signed-compatible */
-	exponent += round_Xsig(&accumulator);
-
-	result = &st(0);
-	significand(result) = XSIG_LL(accumulator);
-	setexponent16(result, exponent);
-
-	tag = FPU_round(result, 1, 0, FULL_PRECISION, sign);
-
-	setsign(result, sign);
-	FPU_settag0(tag);
-
-	return 0;
-
-}
diff --git a/arch/x86/math-emu/poly_atan.c b/arch/x86/math-emu/poly_atan.c
deleted file mode 100644
index 7e7412c5a1fef..0000000000000
--- a/arch/x86/math-emu/poly_atan.c
+++ /dev/null
@@ -1,209 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  poly_atan.c                                                              |
- |                                                                           |
- | Compute the arctan of a FPU_REG, using a polynomial approximation.        |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "status_w.h"
-#include "control_w.h"
-#include "poly.h"
-
-#define	HIPOWERon	6	/* odd poly, negative terms */
-static const unsigned long long oddnegterms[HIPOWERon] = {
-	0x0000000000000000LL,	/* Dummy (not for - 1.0) */
-	0x015328437f756467LL,
-	0x0005dda27b73dec6LL,
-	0x0000226bf2bfb91aLL,
-	0x000000ccc439c5f7LL,
-	0x0000000355438407LL
-};
-
-#define	HIPOWERop	6	/* odd poly, positive terms */
-static const unsigned long long oddplterms[HIPOWERop] = {
-/*  0xaaaaaaaaaaaaaaabLL,  transferred to fixedpterm[] */
-	0x0db55a71875c9ac2LL,
-	0x0029fce2d67880b0LL,
-	0x0000dfd3908b4596LL,
-	0x00000550fd61dab4LL,
-	0x0000001c9422b3f9LL,
-	0x000000003e3301e1LL
-};
-
-static const unsigned long long denomterm = 0xebd9b842c5c53a0eLL;
-
-static const Xsig fixedpterm = MK_XSIG(0xaaaaaaaa, 0xaaaaaaaa, 0xaaaaaaaa);
-
-static const Xsig pi_signif = MK_XSIG(0xc90fdaa2, 0x2168c234, 0xc4c6628b);
-
-/*--- poly_atan() -----------------------------------------------------------+
- |                                                                           |
- +---------------------------------------------------------------------------*/
-void poly_atan(FPU_REG *st0_ptr, u_char st0_tag,
-	       FPU_REG *st1_ptr, u_char st1_tag)
-{
-	u_char transformed, inverted, sign1, sign2;
-	int exponent;
-	long int dummy_exp;
-	Xsig accumulator, Numer, Denom, accumulatore, argSignif, argSq, argSqSq;
-	u_char tag;
-
-	sign1 = getsign(st0_ptr);
-	sign2 = getsign(st1_ptr);
-	if (st0_tag == TAG_Valid) {
-		exponent = exponent(st0_ptr);
-	} else {
-		/* This gives non-compatible stack contents... */
-		FPU_to_exp16(st0_ptr, st0_ptr);
-		exponent = exponent16(st0_ptr);
-	}
-	if (st1_tag == TAG_Valid) {
-		exponent -= exponent(st1_ptr);
-	} else {
-		/* This gives non-compatible stack contents... */
-		FPU_to_exp16(st1_ptr, st1_ptr);
-		exponent -= exponent16(st1_ptr);
-	}
-
-	if ((exponent < 0) || ((exponent == 0) &&
-			       ((st0_ptr->sigh < st1_ptr->sigh) ||
-				((st0_ptr->sigh == st1_ptr->sigh) &&
-				 (st0_ptr->sigl < st1_ptr->sigl))))) {
-		inverted = 1;
-		Numer.lsw = Denom.lsw = 0;
-		XSIG_LL(Numer) = significand(st0_ptr);
-		XSIG_LL(Denom) = significand(st1_ptr);
-	} else {
-		inverted = 0;
-		exponent = -exponent;
-		Numer.lsw = Denom.lsw = 0;
-		XSIG_LL(Numer) = significand(st1_ptr);
-		XSIG_LL(Denom) = significand(st0_ptr);
-	}
-	div_Xsig(&Numer, &Denom, &argSignif);
-	exponent += norm_Xsig(&argSignif);
-
-	if ((exponent >= -1)
-	    || ((exponent == -2) && (argSignif.msw > 0xd413ccd0))) {
-		/* The argument is greater than sqrt(2)-1 (=0.414213562...) */
-		/* Convert the argument by an identity for atan */
-		transformed = 1;
-
-		if (exponent >= 0) {
-#ifdef PARANOID
-			if (!((exponent == 0) &&
-			      (argSignif.lsw == 0) && (argSignif.midw == 0) &&
-			      (argSignif.msw == 0x80000000))) {
-				EXCEPTION(EX_INTERNAL | 0x104);	/* There must be a logic error */
-				return;
-			}
-#endif /* PARANOID */
-			argSignif.msw = 0;	/* Make the transformed arg -> 0.0 */
-		} else {
-			Numer.lsw = Denom.lsw = argSignif.lsw;
-			XSIG_LL(Numer) = XSIG_LL(Denom) = XSIG_LL(argSignif);
-
-			if (exponent < -1)
-				shr_Xsig(&Numer, -1 - exponent);
-			negate_Xsig(&Numer);
-
-			shr_Xsig(&Denom, -exponent);
-			Denom.msw |= 0x80000000;
-
-			div_Xsig(&Numer, &Denom, &argSignif);
-
-			exponent = -1 + norm_Xsig(&argSignif);
-		}
-	} else {
-		transformed = 0;
-	}
-
-	argSq.lsw = argSignif.lsw;
-	argSq.midw = argSignif.midw;
-	argSq.msw = argSignif.msw;
-	mul_Xsig_Xsig(&argSq, &argSq);
-
-	argSqSq.lsw = argSq.lsw;
-	argSqSq.midw = argSq.midw;
-	argSqSq.msw = argSq.msw;
-	mul_Xsig_Xsig(&argSqSq, &argSqSq);
-
-	accumulatore.lsw = argSq.lsw;
-	XSIG_LL(accumulatore) = XSIG_LL(argSq);
-
-	shr_Xsig(&argSq, 2 * (-1 - exponent - 1));
-	shr_Xsig(&argSqSq, 4 * (-1 - exponent - 1));
-
-	/* Now have argSq etc with binary point at the left
-	   .1xxxxxxxx */
-
-	/* Do the basic fixed point polynomial evaluation */
-	accumulator.msw = accumulator.midw = accumulator.lsw = 0;
-	polynomial_Xsig(&accumulator, &XSIG_LL(argSqSq),
-			oddplterms, HIPOWERop - 1);
-	mul64_Xsig(&accumulator, &XSIG_LL(argSq));
-	negate_Xsig(&accumulator);
-	polynomial_Xsig(&accumulator, &XSIG_LL(argSqSq), oddnegterms,
-			HIPOWERon - 1);
-	negate_Xsig(&accumulator);
-	add_two_Xsig(&accumulator, &fixedpterm, &dummy_exp);
-
-	mul64_Xsig(&accumulatore, &denomterm);
-	shr_Xsig(&accumulatore, 1 + 2 * (-1 - exponent));
-	accumulatore.msw |= 0x80000000;
-
-	div_Xsig(&accumulator, &accumulatore, &accumulator);
-
-	mul_Xsig_Xsig(&accumulator, &argSignif);
-	mul_Xsig_Xsig(&accumulator, &argSq);
-
-	shr_Xsig(&accumulator, 3);
-	negate_Xsig(&accumulator);
-	add_Xsig_Xsig(&accumulator, &argSignif);
-
-	if (transformed) {
-		/* compute pi/4 - accumulator */
-		shr_Xsig(&accumulator, -1 - exponent);
-		negate_Xsig(&accumulator);
-		add_Xsig_Xsig(&accumulator, &pi_signif);
-		exponent = -1;
-	}
-
-	if (inverted) {
-		/* compute pi/2 - accumulator */
-		shr_Xsig(&accumulator, -exponent);
-		negate_Xsig(&accumulator);
-		add_Xsig_Xsig(&accumulator, &pi_signif);
-		exponent = 0;
-	}
-
-	if (sign1) {
-		/* compute pi - accumulator */
-		shr_Xsig(&accumulator, 1 - exponent);
-		negate_Xsig(&accumulator);
-		add_Xsig_Xsig(&accumulator, &pi_signif);
-		exponent = 1;
-	}
-
-	exponent += round_Xsig(&accumulator);
-
-	significand(st1_ptr) = XSIG_LL(accumulator);
-	setexponent16(st1_ptr, exponent);
-
-	tag = FPU_round(st1_ptr, 1, 0, FULL_PRECISION, sign2);
-	FPU_settagi(1, tag);
-
-	set_precision_flag_up();	/* We do not really know if up or down,
-					   use this as the default. */
-
-}
diff --git a/arch/x86/math-emu/poly_l2.c b/arch/x86/math-emu/poly_l2.c
deleted file mode 100644
index 98b6949bb854a..0000000000000
--- a/arch/x86/math-emu/poly_l2.c
+++ /dev/null
@@ -1,245 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  poly_l2.c                                                                |
- |                                                                           |
- | Compute the base 2 log of a FPU_REG, using a polynomial approximation.    |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "control_w.h"
-#include "poly.h"
-
-static void log2_kernel(FPU_REG const *arg, u_char argsign,
-			Xsig * accum_result, long int *expon);
-
-/*--- poly_l2() -------------------------------------------------------------+
- |   Base 2 logarithm by a polynomial approximation.                         |
- +---------------------------------------------------------------------------*/
-void poly_l2(FPU_REG *st0_ptr, FPU_REG *st1_ptr, u_char st1_sign)
-{
-	long int exponent, expon, expon_expon;
-	Xsig accumulator, expon_accum, yaccum;
-	u_char sign, argsign;
-	FPU_REG x;
-	int tag;
-
-	exponent = exponent16(st0_ptr);
-
-	/* From st0_ptr, make a number > sqrt(2)/2 and < sqrt(2) */
-	if (st0_ptr->sigh > (unsigned)0xb504f334) {
-		/* Treat as  sqrt(2)/2 < st0_ptr < 1 */
-		significand(&x) = -significand(st0_ptr);
-		setexponent16(&x, -1);
-		exponent++;
-		argsign = SIGN_NEG;
-	} else {
-		/* Treat as  1 <= st0_ptr < sqrt(2) */
-		x.sigh = st0_ptr->sigh - 0x80000000;
-		x.sigl = st0_ptr->sigl;
-		setexponent16(&x, 0);
-		argsign = SIGN_POS;
-	}
-	tag = FPU_normalize_nuo(&x);
-
-	if (tag == TAG_Zero) {
-		expon = 0;
-		accumulator.msw = accumulator.midw = accumulator.lsw = 0;
-	} else {
-		log2_kernel(&x, argsign, &accumulator, &expon);
-	}
-
-	if (exponent < 0) {
-		sign = SIGN_NEG;
-		exponent = -exponent;
-	} else
-		sign = SIGN_POS;
-	expon_accum.msw = exponent;
-	expon_accum.midw = expon_accum.lsw = 0;
-	if (exponent) {
-		expon_expon = 31 + norm_Xsig(&expon_accum);
-		shr_Xsig(&accumulator, expon_expon - expon);
-
-		if (sign ^ argsign)
-			negate_Xsig(&accumulator);
-		add_Xsig_Xsig(&accumulator, &expon_accum);
-	} else {
-		expon_expon = expon;
-		sign = argsign;
-	}
-
-	yaccum.lsw = 0;
-	XSIG_LL(yaccum) = significand(st1_ptr);
-	mul_Xsig_Xsig(&accumulator, &yaccum);
-
-	expon_expon += round_Xsig(&accumulator);
-
-	if (accumulator.msw == 0) {
-		FPU_copy_to_reg1(&CONST_Z, TAG_Zero);
-		return;
-	}
-
-	significand(st1_ptr) = XSIG_LL(accumulator);
-	setexponent16(st1_ptr, expon_expon + exponent16(st1_ptr) + 1);
-
-	tag = FPU_round(st1_ptr, 1, 0, FULL_PRECISION, sign ^ st1_sign);
-	FPU_settagi(1, tag);
-
-	set_precision_flag_up();	/* 80486 appears to always do this */
-
-	return;
-
-}
-
-/*--- poly_l2p1() -----------------------------------------------------------+
- |   Base 2 logarithm by a polynomial approximation.                         |
- |   log2(x+1)                                                               |
- +---------------------------------------------------------------------------*/
-int poly_l2p1(u_char sign0, u_char sign1,
-	      FPU_REG * st0_ptr, FPU_REG * st1_ptr, FPU_REG * dest)
-{
-	u_char tag;
-	long int exponent;
-	Xsig accumulator, yaccum;
-
-	if (exponent16(st0_ptr) < 0) {
-		log2_kernel(st0_ptr, sign0, &accumulator, &exponent);
-
-		yaccum.lsw = 0;
-		XSIG_LL(yaccum) = significand(st1_ptr);
-		mul_Xsig_Xsig(&accumulator, &yaccum);
-
-		exponent += round_Xsig(&accumulator);
-
-		exponent += exponent16(st1_ptr) + 1;
-		if (exponent < EXP_WAY_UNDER)
-			exponent = EXP_WAY_UNDER;
-
-		significand(dest) = XSIG_LL(accumulator);
-		setexponent16(dest, exponent);
-
-		tag = FPU_round(dest, 1, 0, FULL_PRECISION, sign0 ^ sign1);
-		FPU_settagi(1, tag);
-
-		if (tag == TAG_Valid)
-			set_precision_flag_up();	/* 80486 appears to always do this */
-	} else {
-		/* The magnitude of st0_ptr is far too large. */
-
-		if (sign0 != SIGN_POS) {
-			/* Trying to get the log of a negative number. */
-#ifdef PECULIAR_486		/* Stupid 80486 doesn't worry about log(negative). */
-			changesign(st1_ptr);
-#else
-			if (arith_invalid(1) < 0)
-				return 1;
-#endif /* PECULIAR_486 */
-		}
-
-		/* 80486 appears to do this */
-		if (sign0 == SIGN_NEG)
-			set_precision_flag_down();
-		else
-			set_precision_flag_up();
-	}
-
-	if (exponent(dest) <= EXP_UNDER)
-		EXCEPTION(EX_Underflow);
-
-	return 0;
-
-}
-
-#undef HIPOWER
-#define	HIPOWER	10
-static const unsigned long long logterms[HIPOWER] = {
-	0x2a8eca5705fc2ef0LL,
-	0xf6384ee1d01febceLL,
-	0x093bb62877cdf642LL,
-	0x006985d8a9ec439bLL,
-	0x0005212c4f55a9c8LL,
-	0x00004326a16927f0LL,
-	0x0000038d1d80a0e7LL,
-	0x0000003141cc80c6LL,
-	0x00000002b1668c9fLL,
-	0x000000002c7a46aaLL
-};
-
-static const unsigned long leadterm = 0xb8000000;
-
-/*--- log2_kernel() ---------------------------------------------------------+
- |   Base 2 logarithm by a polynomial approximation.                         |
- |   log2(x+1)                                                               |
- +---------------------------------------------------------------------------*/
-static void log2_kernel(FPU_REG const *arg, u_char argsign, Xsig *accum_result,
-			long int *expon)
-{
-	long int exponent, adj;
-	unsigned long long Xsq;
-	Xsig accumulator, Numer, Denom, argSignif, arg_signif;
-
-	exponent = exponent16(arg);
-	Numer.lsw = Denom.lsw = 0;
-	XSIG_LL(Numer) = XSIG_LL(Denom) = significand(arg);
-	if (argsign == SIGN_POS) {
-		shr_Xsig(&Denom, 2 - (1 + exponent));
-		Denom.msw |= 0x80000000;
-		div_Xsig(&Numer, &Denom, &argSignif);
-	} else {
-		shr_Xsig(&Denom, 1 - (1 + exponent));
-		negate_Xsig(&Denom);
-		if (Denom.msw & 0x80000000) {
-			div_Xsig(&Numer, &Denom, &argSignif);
-			exponent++;
-		} else {
-			/* Denom must be 1.0 */
-			argSignif.lsw = Numer.lsw;
-			argSignif.midw = Numer.midw;
-			argSignif.msw = Numer.msw;
-		}
-	}
-
-#ifndef PECULIAR_486
-	/* Should check here that  |local_arg|  is within the valid range */
-	if (exponent >= -2) {
-		if ((exponent > -2) || (argSignif.msw > (unsigned)0xafb0ccc0)) {
-			/* The argument is too large */
-		}
-	}
-#endif /* PECULIAR_486 */
-
-	arg_signif.lsw = argSignif.lsw;
-	XSIG_LL(arg_signif) = XSIG_LL(argSignif);
-	adj = norm_Xsig(&argSignif);
-	accumulator.lsw = argSignif.lsw;
-	XSIG_LL(accumulator) = XSIG_LL(argSignif);
-	mul_Xsig_Xsig(&accumulator, &accumulator);
-	shr_Xsig(&accumulator, 2 * (-1 - (1 + exponent + adj)));
-	Xsq = XSIG_LL(accumulator);
-	if (accumulator.lsw & 0x80000000)
-		Xsq++;
-
-	accumulator.msw = accumulator.midw = accumulator.lsw = 0;
-	/* Do the basic fixed point polynomial evaluation */
-	polynomial_Xsig(&accumulator, &Xsq, logterms, HIPOWER - 1);
-
-	mul_Xsig_Xsig(&accumulator, &argSignif);
-	shr_Xsig(&accumulator, 6 - adj);
-
-	mul32_Xsig(&arg_signif, leadterm);
-	add_two_Xsig(&accumulator, &arg_signif, &exponent);
-
-	*expon = exponent + 1;
-	accum_result->lsw = accumulator.lsw;
-	accum_result->midw = accumulator.midw;
-	accum_result->msw = accumulator.msw;
-
-}
diff --git a/arch/x86/math-emu/poly_sin.c b/arch/x86/math-emu/poly_sin.c
deleted file mode 100644
index c192fba515264..0000000000000
--- a/arch/x86/math-emu/poly_sin.c
+++ /dev/null
@@ -1,379 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  poly_sin.c                                                               |
- |                                                                           |
- |  Computation of an approximation of the sin function and the cosine       |
- |  function by a polynomial.                                                |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997,1999                                    |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@melbpc.org.au                             |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "control_w.h"
-#include "poly.h"
-
-#define	N_COEFF_P	4
-#define	N_COEFF_N	4
-
-static const unsigned long long pos_terms_l[N_COEFF_P] = {
-	0xaaaaaaaaaaaaaaabLL,
-	0x00d00d00d00cf906LL,
-	0x000006b99159a8bbLL,
-	0x000000000d7392e6LL
-};
-
-static const unsigned long long neg_terms_l[N_COEFF_N] = {
-	0x2222222222222167LL,
-	0x0002e3bc74aab624LL,
-	0x0000000b09229062LL,
-	0x00000000000c7973LL
-};
-
-#define	N_COEFF_PH	4
-#define	N_COEFF_NH	4
-static const unsigned long long pos_terms_h[N_COEFF_PH] = {
-	0x0000000000000000LL,
-	0x05b05b05b05b0406LL,
-	0x000049f93edd91a9LL,
-	0x00000000c9c9ed62LL
-};
-
-static const unsigned long long neg_terms_h[N_COEFF_NH] = {
-	0xaaaaaaaaaaaaaa98LL,
-	0x001a01a01a019064LL,
-	0x0000008f76c68a77LL,
-	0x0000000000d58f5eLL
-};
-
-/*--- poly_sine() -----------------------------------------------------------+
- |                                                                           |
- +---------------------------------------------------------------------------*/
-void poly_sine(FPU_REG *st0_ptr)
-{
-	int exponent, echange;
-	Xsig accumulator, argSqrd, argTo4;
-	unsigned long fix_up, adj;
-	unsigned long long fixed_arg;
-	FPU_REG result;
-
-	exponent = exponent(st0_ptr);
-
-	accumulator.lsw = accumulator.midw = accumulator.msw = 0;
-
-	/* Split into two ranges, for arguments below and above 1.0 */
-	/* The boundary between upper and lower is approx 0.88309101259 */
-	if ((exponent < -1)
-	    || ((exponent == -1) && (st0_ptr->sigh <= 0xe21240aa))) {
-		/* The argument is <= 0.88309101259 */
-
-		argSqrd.msw = st0_ptr->sigh;
-		argSqrd.midw = st0_ptr->sigl;
-		argSqrd.lsw = 0;
-		mul64_Xsig(&argSqrd, &significand(st0_ptr));
-		shr_Xsig(&argSqrd, 2 * (-1 - exponent));
-		argTo4.msw = argSqrd.msw;
-		argTo4.midw = argSqrd.midw;
-		argTo4.lsw = argSqrd.lsw;
-		mul_Xsig_Xsig(&argTo4, &argTo4);
-
-		polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_l,
-				N_COEFF_N - 1);
-		mul_Xsig_Xsig(&accumulator, &argSqrd);
-		negate_Xsig(&accumulator);
-
-		polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_l,
-				N_COEFF_P - 1);
-
-		shr_Xsig(&accumulator, 2);	/* Divide by four */
-		accumulator.msw |= 0x80000000;	/* Add 1.0 */
-
-		mul64_Xsig(&accumulator, &significand(st0_ptr));
-		mul64_Xsig(&accumulator, &significand(st0_ptr));
-		mul64_Xsig(&accumulator, &significand(st0_ptr));
-
-		/* Divide by four, FPU_REG compatible, etc */
-		exponent = 3 * exponent;
-
-		/* The minimum exponent difference is 3 */
-		shr_Xsig(&accumulator, exponent(st0_ptr) - exponent);
-
-		negate_Xsig(&accumulator);
-		XSIG_LL(accumulator) += significand(st0_ptr);
-
-		echange = round_Xsig(&accumulator);
-
-		setexponentpos(&result, exponent(st0_ptr) + echange);
-	} else {
-		/* The argument is > 0.88309101259 */
-		/* We use sin(st(0)) = cos(pi/2-st(0)) */
-
-		fixed_arg = significand(st0_ptr);
-
-		if (exponent == 0) {
-			/* The argument is >= 1.0 */
-
-			/* Put the binary point at the left. */
-			fixed_arg <<= 1;
-		}
-		/* pi/2 in hex is: 1.921fb54442d18469 898CC51701B839A2 52049C1 */
-		fixed_arg = 0x921fb54442d18469LL - fixed_arg;
-		/* There is a special case which arises due to rounding, to fix here. */
-		if (fixed_arg == 0xffffffffffffffffLL)
-			fixed_arg = 0;
-
-		XSIG_LL(argSqrd) = fixed_arg;
-		argSqrd.lsw = 0;
-		mul64_Xsig(&argSqrd, &fixed_arg);
-
-		XSIG_LL(argTo4) = XSIG_LL(argSqrd);
-		argTo4.lsw = argSqrd.lsw;
-		mul_Xsig_Xsig(&argTo4, &argTo4);
-
-		polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_h,
-				N_COEFF_NH - 1);
-		mul_Xsig_Xsig(&accumulator, &argSqrd);
-		negate_Xsig(&accumulator);
-
-		polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_h,
-				N_COEFF_PH - 1);
-		negate_Xsig(&accumulator);
-
-		mul64_Xsig(&accumulator, &fixed_arg);
-		mul64_Xsig(&accumulator, &fixed_arg);
-
-		shr_Xsig(&accumulator, 3);
-		negate_Xsig(&accumulator);
-
-		add_Xsig_Xsig(&accumulator, &argSqrd);
-
-		shr_Xsig(&accumulator, 1);
-
-		accumulator.lsw |= 1;	/* A zero accumulator here would cause problems */
-		negate_Xsig(&accumulator);
-
-		/* The basic computation is complete. Now fix the answer to
-		   compensate for the error due to the approximation used for
-		   pi/2
-		 */
-
-		/* This has an exponent of -65 */
-		fix_up = 0x898cc517;
-		/* The fix-up needs to be improved for larger args */
-		if (argSqrd.msw & 0xffc00000) {
-			/* Get about 32 bit precision in these: */
-			fix_up -= mul_32_32(0x898cc517, argSqrd.msw) / 6;
-		}
-		fix_up = mul_32_32(fix_up, LL_MSW(fixed_arg));
-
-		adj = accumulator.lsw;	/* temp save */
-		accumulator.lsw -= fix_up;
-		if (accumulator.lsw > adj)
-			XSIG_LL(accumulator)--;
-
-		echange = round_Xsig(&accumulator);
-
-		setexponentpos(&result, echange - 1);
-	}
-
-	significand(&result) = XSIG_LL(accumulator);
-	setsign(&result, getsign(st0_ptr));
-	FPU_copy_to_reg0(&result, TAG_Valid);
-
-#ifdef PARANOID
-	if ((exponent(&result) >= 0)
-	    && (significand(&result) > 0x8000000000000000LL)) {
-		EXCEPTION(EX_INTERNAL | 0x150);
-	}
-#endif /* PARANOID */
-
-}
-
-/*--- poly_cos() ------------------------------------------------------------+
- |                                                                           |
- +---------------------------------------------------------------------------*/
-void poly_cos(FPU_REG *st0_ptr)
-{
-	FPU_REG result;
-	long int exponent, exp2, echange;
-	Xsig accumulator, argSqrd, fix_up, argTo4;
-	unsigned long long fixed_arg;
-
-#ifdef PARANOID
-	if ((exponent(st0_ptr) > 0)
-	    || ((exponent(st0_ptr) == 0)
-		&& (significand(st0_ptr) > 0xc90fdaa22168c234LL))) {
-		EXCEPTION(EX_Invalid);
-		FPU_copy_to_reg0(&CONST_QNaN, TAG_Special);
-		return;
-	}
-#endif /* PARANOID */
-
-	exponent = exponent(st0_ptr);
-
-	accumulator.lsw = accumulator.midw = accumulator.msw = 0;
-
-	if ((exponent < -1)
-	    || ((exponent == -1) && (st0_ptr->sigh <= 0xb00d6f54))) {
-		/* arg is < 0.687705 */
-
-		argSqrd.msw = st0_ptr->sigh;
-		argSqrd.midw = st0_ptr->sigl;
-		argSqrd.lsw = 0;
-		mul64_Xsig(&argSqrd, &significand(st0_ptr));
-
-		if (exponent < -1) {
-			/* shift the argument right by the required places */
-			shr_Xsig(&argSqrd, 2 * (-1 - exponent));
-		}
-
-		argTo4.msw = argSqrd.msw;
-		argTo4.midw = argSqrd.midw;
-		argTo4.lsw = argSqrd.lsw;
-		mul_Xsig_Xsig(&argTo4, &argTo4);
-
-		polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_h,
-				N_COEFF_NH - 1);
-		mul_Xsig_Xsig(&accumulator, &argSqrd);
-		negate_Xsig(&accumulator);
-
-		polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_h,
-				N_COEFF_PH - 1);
-		negate_Xsig(&accumulator);
-
-		mul64_Xsig(&accumulator, &significand(st0_ptr));
-		mul64_Xsig(&accumulator, &significand(st0_ptr));
-		shr_Xsig(&accumulator, -2 * (1 + exponent));
-
-		shr_Xsig(&accumulator, 3);
-		negate_Xsig(&accumulator);
-
-		add_Xsig_Xsig(&accumulator, &argSqrd);
-
-		shr_Xsig(&accumulator, 1);
-
-		/* It doesn't matter if accumulator is all zero here, the
-		   following code will work ok */
-		negate_Xsig(&accumulator);
-
-		if (accumulator.lsw & 0x80000000)
-			XSIG_LL(accumulator)++;
-		if (accumulator.msw == 0) {
-			/* The result is 1.0 */
-			FPU_copy_to_reg0(&CONST_1, TAG_Valid);
-			return;
-		} else {
-			significand(&result) = XSIG_LL(accumulator);
-
-			/* will be a valid positive nr with expon = -1 */
-			setexponentpos(&result, -1);
-		}
-	} else {
-		fixed_arg = significand(st0_ptr);
-
-		if (exponent == 0) {
-			/* The argument is >= 1.0 */
-
-			/* Put the binary point at the left. */
-			fixed_arg <<= 1;
-		}
-		/* pi/2 in hex is: 1.921fb54442d18469 898CC51701B839A2 52049C1 */
-		fixed_arg = 0x921fb54442d18469LL - fixed_arg;
-		/* There is a special case which arises due to rounding, to fix here. */
-		if (fixed_arg == 0xffffffffffffffffLL)
-			fixed_arg = 0;
-
-		exponent = -1;
-		exp2 = -1;
-
-		/* A shift is needed here only for a narrow range of arguments,
-		   i.e. for fixed_arg approx 2^-32, but we pick up more... */
-		if (!(LL_MSW(fixed_arg) & 0xffff0000)) {
-			fixed_arg <<= 16;
-			exponent -= 16;
-			exp2 -= 16;
-		}
-
-		XSIG_LL(argSqrd) = fixed_arg;
-		argSqrd.lsw = 0;
-		mul64_Xsig(&argSqrd, &fixed_arg);
-
-		if (exponent < -1) {
-			/* shift the argument right by the required places */
-			shr_Xsig(&argSqrd, 2 * (-1 - exponent));
-		}
-
-		argTo4.msw = argSqrd.msw;
-		argTo4.midw = argSqrd.midw;
-		argTo4.lsw = argSqrd.lsw;
-		mul_Xsig_Xsig(&argTo4, &argTo4);
-
-		polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), neg_terms_l,
-				N_COEFF_N - 1);
-		mul_Xsig_Xsig(&accumulator, &argSqrd);
-		negate_Xsig(&accumulator);
-
-		polynomial_Xsig(&accumulator, &XSIG_LL(argTo4), pos_terms_l,
-				N_COEFF_P - 1);
-
-		shr_Xsig(&accumulator, 2);	/* Divide by four */
-		accumulator.msw |= 0x80000000;	/* Add 1.0 */
-
-		mul64_Xsig(&accumulator, &fixed_arg);
-		mul64_Xsig(&accumulator, &fixed_arg);
-		mul64_Xsig(&accumulator, &fixed_arg);
-
-		/* Divide by four, FPU_REG compatible, etc */
-		exponent = 3 * exponent;
-
-		/* The minimum exponent difference is 3 */
-		shr_Xsig(&accumulator, exp2 - exponent);
-
-		negate_Xsig(&accumulator);
-		XSIG_LL(accumulator) += fixed_arg;
-
-		/* The basic computation is complete. Now fix the answer to
-		   compensate for the error due to the approximation used for
-		   pi/2
-		 */
-
-		/* This has an exponent of -65 */
-		XSIG_LL(fix_up) = 0x898cc51701b839a2ll;
-		fix_up.lsw = 0;
-
-		/* The fix-up needs to be improved for larger args */
-		if (argSqrd.msw & 0xffc00000) {
-			/* Get about 32 bit precision in these: */
-			fix_up.msw -= mul_32_32(0x898cc517, argSqrd.msw) / 2;
-			fix_up.msw += mul_32_32(0x898cc517, argTo4.msw) / 24;
-		}
-
-		exp2 += norm_Xsig(&accumulator);
-		shr_Xsig(&accumulator, 1);	/* Prevent overflow */
-		exp2++;
-		shr_Xsig(&fix_up, 65 + exp2);
-
-		add_Xsig_Xsig(&accumulator, &fix_up);
-
-		echange = round_Xsig(&accumulator);
-
-		setexponentpos(&result, exp2 + echange);
-		significand(&result) = XSIG_LL(accumulator);
-	}
-
-	FPU_copy_to_reg0(&result, TAG_Valid);
-
-#ifdef PARANOID
-	if ((exponent(&result) >= 0)
-	    && (significand(&result) > 0x8000000000000000LL)) {
-		EXCEPTION(EX_INTERNAL | 0x151);
-	}
-#endif /* PARANOID */
-
-}
diff --git a/arch/x86/math-emu/poly_tan.c b/arch/x86/math-emu/poly_tan.c
deleted file mode 100644
index 1f5b1d7123230..0000000000000
--- a/arch/x86/math-emu/poly_tan.c
+++ /dev/null
@@ -1,213 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  poly_tan.c                                                               |
- |                                                                           |
- | Compute the tan of a FPU_REG, using a polynomial approximation.           |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997,1999                                    |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@melbpc.org.au            |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-#include "control_w.h"
-#include "poly.h"
-
-#define	HiPOWERop	3	/* odd poly, positive terms */
-static const unsigned long long oddplterm[HiPOWERop] = {
-	0x0000000000000000LL,
-	0x0051a1cf08fca228LL,
-	0x0000000071284ff7LL
-};
-
-#define	HiPOWERon	2	/* odd poly, negative terms */
-static const unsigned long long oddnegterm[HiPOWERon] = {
-	0x1291a9a184244e80LL,
-	0x0000583245819c21LL
-};
-
-#define	HiPOWERep	2	/* even poly, positive terms */
-static const unsigned long long evenplterm[HiPOWERep] = {
-	0x0e848884b539e888LL,
-	0x00003c7f18b887daLL
-};
-
-#define	HiPOWERen	2	/* even poly, negative terms */
-static const unsigned long long evennegterm[HiPOWERen] = {
-	0xf1f0200fd51569ccLL,
-	0x003afb46105c4432LL
-};
-
-static const unsigned long long twothirds = 0xaaaaaaaaaaaaaaabLL;
-
-/*--- poly_tan() ------------------------------------------------------------+
- |                                                                           |
- +---------------------------------------------------------------------------*/
-void poly_tan(FPU_REG *st0_ptr)
-{
-	long int exponent;
-	int invert;
-	Xsig argSq, argSqSq, accumulatoro, accumulatore, accum,
-	    argSignif, fix_up;
-	unsigned long adj;
-
-	exponent = exponent(st0_ptr);
-
-#ifdef PARANOID
-	if (signnegative(st0_ptr)) {	/* Can't hack a number < 0.0 */
-		arith_invalid(0);
-		return;
-	}			/* Need a positive number */
-#endif /* PARANOID */
-
-	/* Split the problem into two domains, smaller and larger than pi/4 */
-	if ((exponent == 0)
-	    || ((exponent == -1) && (st0_ptr->sigh > 0xc90fdaa2))) {
-		/* The argument is greater than (approx) pi/4 */
-		invert = 1;
-		accum.lsw = 0;
-		XSIG_LL(accum) = significand(st0_ptr);
-
-		if (exponent == 0) {
-			/* The argument is >= 1.0 */
-			/* Put the binary point at the left. */
-			XSIG_LL(accum) <<= 1;
-		}
-		/* pi/2 in hex is: 1.921fb54442d18469 898CC51701B839A2 52049C1 */
-		XSIG_LL(accum) = 0x921fb54442d18469LL - XSIG_LL(accum);
-		/* This is a special case which arises due to rounding. */
-		if (XSIG_LL(accum) == 0xffffffffffffffffLL) {
-			FPU_settag0(TAG_Valid);
-			significand(st0_ptr) = 0x8a51e04daabda360LL;
-			setexponent16(st0_ptr,
-				      (0x41 + EXTENDED_Ebias) | SIGN_Negative);
-			return;
-		}
-
-		argSignif.lsw = accum.lsw;
-		XSIG_LL(argSignif) = XSIG_LL(accum);
-		exponent = -1 + norm_Xsig(&argSignif);
-	} else {
-		invert = 0;
-		argSignif.lsw = 0;
-		XSIG_LL(accum) = XSIG_LL(argSignif) = significand(st0_ptr);
-
-		if (exponent < -1) {
-			/* shift the argument right by the required places */
-			if (FPU_shrx(&XSIG_LL(accum), -1 - exponent) >=
-			    0x80000000U)
-				XSIG_LL(accum)++;	/* round up */
-		}
-	}
-
-	XSIG_LL(argSq) = XSIG_LL(accum);
-	argSq.lsw = accum.lsw;
-	mul_Xsig_Xsig(&argSq, &argSq);
-	XSIG_LL(argSqSq) = XSIG_LL(argSq);
-	argSqSq.lsw = argSq.lsw;
-	mul_Xsig_Xsig(&argSqSq, &argSqSq);
-
-	/* Compute the negative terms for the numerator polynomial */
-	accumulatoro.msw = accumulatoro.midw = accumulatoro.lsw = 0;
-	polynomial_Xsig(&accumulatoro, &XSIG_LL(argSqSq), oddnegterm,
-			HiPOWERon - 1);
-	mul_Xsig_Xsig(&accumulatoro, &argSq);
-	negate_Xsig(&accumulatoro);
-	/* Add the positive terms */
-	polynomial_Xsig(&accumulatoro, &XSIG_LL(argSqSq), oddplterm,
-			HiPOWERop - 1);
-
-	/* Compute the positive terms for the denominator polynomial */
-	accumulatore.msw = accumulatore.midw = accumulatore.lsw = 0;
-	polynomial_Xsig(&accumulatore, &XSIG_LL(argSqSq), evenplterm,
-			HiPOWERep - 1);
-	mul_Xsig_Xsig(&accumulatore, &argSq);
-	negate_Xsig(&accumulatore);
-	/* Add the negative terms */
-	polynomial_Xsig(&accumulatore, &XSIG_LL(argSqSq), evennegterm,
-			HiPOWERen - 1);
-	/* Multiply by arg^2 */
-	mul64_Xsig(&accumulatore, &XSIG_LL(argSignif));
-	mul64_Xsig(&accumulatore, &XSIG_LL(argSignif));
-	/* de-normalize and divide by 2 */
-	shr_Xsig(&accumulatore, -2 * (1 + exponent) + 1);
-	negate_Xsig(&accumulatore);	/* This does 1 - accumulator */
-
-	/* Now find the ratio. */
-	if (accumulatore.msw == 0) {
-		/* accumulatoro must contain 1.0 here, (actually, 0) but it
-		   really doesn't matter what value we use because it will
-		   have negligible effect in later calculations
-		 */
-		XSIG_LL(accum) = 0x8000000000000000LL;
-		accum.lsw = 0;
-	} else {
-		div_Xsig(&accumulatoro, &accumulatore, &accum);
-	}
-
-	/* Multiply by 1/3 * arg^3 */
-	mul64_Xsig(&accum, &XSIG_LL(argSignif));
-	mul64_Xsig(&accum, &XSIG_LL(argSignif));
-	mul64_Xsig(&accum, &XSIG_LL(argSignif));
-	mul64_Xsig(&accum, &twothirds);
-	shr_Xsig(&accum, -2 * (exponent + 1));
-
-	/* tan(arg) = arg + accum */
-	add_two_Xsig(&accum, &argSignif, &exponent);
-
-	if (invert) {
-		/* We now have the value of tan(pi_2 - arg) where pi_2 is an
-		   approximation for pi/2
-		 */
-		/* The next step is to fix the answer to compensate for the
-		   error due to the approximation used for pi/2
-		 */
-
-		/* This is (approx) delta, the error in our approx for pi/2
-		   (see above). It has an exponent of -65
-		 */
-		XSIG_LL(fix_up) = 0x898cc51701b839a2LL;
-		fix_up.lsw = 0;
-
-		if (exponent == 0)
-			adj = 0xffffffff;	/* We want approx 1.0 here, but
-						   this is close enough. */
-		else if (exponent > -30) {
-			adj = accum.msw >> -(exponent + 1);	/* tan */
-			adj = mul_32_32(adj, adj);	/* tan^2 */
-		} else
-			adj = 0;
-		adj = mul_32_32(0x898cc517, adj);	/* delta * tan^2 */
-
-		fix_up.msw += adj;
-		if (!(fix_up.msw & 0x80000000)) {	/* did fix_up overflow ? */
-			/* Yes, we need to add an msb */
-			shr_Xsig(&fix_up, 1);
-			fix_up.msw |= 0x80000000;
-			shr_Xsig(&fix_up, 64 + exponent);
-		} else
-			shr_Xsig(&fix_up, 65 + exponent);
-
-		add_two_Xsig(&accum, &fix_up, &exponent);
-
-		/* accum now contains tan(pi/2 - arg).
-		   Use tan(arg) = 1.0 / tan(pi/2 - arg)
-		 */
-		accumulatoro.lsw = accumulatoro.midw = 0;
-		accumulatoro.msw = 0x80000000;
-		div_Xsig(&accumulatoro, &accum, &accum);
-		exponent = -exponent - 1;
-	}
-
-	/* Transfer the result */
-	round_Xsig(&accum);
-	FPU_settag0(TAG_Valid);
-	significand(st0_ptr) = XSIG_LL(accum);
-	setexponent16(st0_ptr, exponent + EXTENDED_Ebias);	/* Result is positive. */
-
-}
diff --git a/arch/x86/math-emu/polynom_Xsig.S b/arch/x86/math-emu/polynom_Xsig.S
deleted file mode 100644
index 35fd723fc0df8..0000000000000
--- a/arch/x86/math-emu/polynom_Xsig.S
+++ /dev/null
@@ -1,137 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  polynomial_Xsig.S                                                        |
- |                                                                           |
- | Fixed point arithmetic polynomial evaluation.                             |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1995                                         |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@jacobi.maths.monash.edu.au |
- |                                                                           |
- | Call from C as:                                                           |
- |   void polynomial_Xsig(Xsig *accum, unsigned long long x,                 |
- |                        unsigned long long terms[], int n)                 |
- |                                                                           |
- | Computes:                                                                 |
- | terms[0] + (terms[1] + (terms[2] + ... + (terms[n-1]*x)*x)*x)*x) ... )*x  |
- | and adds the result to the 12 byte Xsig.                                  |
- | The terms[] are each 8 bytes, but all computation is performed to 12 byte |
- | precision.                                                                |
- |                                                                           |
- | This function must be used carefully: most overflow of intermediate       |
- | results is controlled, but overflow of the result is not.                 |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-	.file	"polynomial_Xsig.S"
-
-#include "fpu_emu.h"
-
-
-#define	TERM_SIZE	$8
-#define	SUM_MS		-20(%ebp)	/* sum ms long */
-#define SUM_MIDDLE	-24(%ebp)	/* sum middle long */
-#define	SUM_LS		-28(%ebp)	/* sum ls long */
-#define	ACCUM_MS	-4(%ebp)	/* accum ms long */
-#define	ACCUM_MIDDLE	-8(%ebp)	/* accum middle long */
-#define	ACCUM_LS	-12(%ebp)	/* accum ls long */
-#define OVERFLOWED      -16(%ebp)	/* addition overflow flag */
-
-.text
-SYM_FUNC_START(polynomial_Xsig)
-	pushl	%ebp
-	movl	%esp,%ebp
-	subl	$32,%esp
-	pushl	%esi
-	pushl	%edi
-	pushl	%ebx
-
-	movl	PARAM2,%esi		/* x */
-	movl	PARAM3,%edi		/* terms */
-
-	movl	TERM_SIZE,%eax
-	mull	PARAM4			/* n */
-	addl	%eax,%edi
-
-	movl	4(%edi),%edx		/* terms[n] */
-	movl	%edx,SUM_MS
-	movl	(%edi),%edx		/* terms[n] */
-	movl	%edx,SUM_MIDDLE
-	xor	%eax,%eax
-	movl	%eax,SUM_LS
-	movb	%al,OVERFLOWED
-
-	subl	TERM_SIZE,%edi
-	decl	PARAM4
-	js	L_accum_done
-
-L_accum_loop:
-	xor	%eax,%eax
-	movl	%eax,ACCUM_MS
-	movl	%eax,ACCUM_MIDDLE
-
-	movl	SUM_MIDDLE,%eax
-	mull	(%esi)			/* x ls long */
-	movl	%edx,ACCUM_LS
-
-	movl	SUM_MIDDLE,%eax
-	mull	4(%esi)			/* x ms long */
-	addl	%eax,ACCUM_LS
-	adcl	%edx,ACCUM_MIDDLE
-	adcl	$0,ACCUM_MS
-
-	movl	SUM_MS,%eax
-	mull	(%esi)			/* x ls long */
-	addl	%eax,ACCUM_LS
-	adcl	%edx,ACCUM_MIDDLE
-	adcl	$0,ACCUM_MS
-
-	movl	SUM_MS,%eax
-	mull	4(%esi)			/* x ms long */
-	addl	%eax,ACCUM_MIDDLE
-	adcl	%edx,ACCUM_MS
-
-	testb	$0xff,OVERFLOWED
-	jz	L_no_overflow
-
-	movl	(%esi),%eax
-	addl	%eax,ACCUM_MIDDLE
-	movl	4(%esi),%eax
-	adcl	%eax,ACCUM_MS		/* This could overflow too */
-
-L_no_overflow:
-
-/*
- * Now put the sum of next term and the accumulator
- * into the sum register
- */
-	movl	ACCUM_LS,%eax
-	addl	(%edi),%eax		/* term ls long */
-	movl	%eax,SUM_LS
-	movl	ACCUM_MIDDLE,%eax
-	adcl	(%edi),%eax		/* term ls long */
-	movl	%eax,SUM_MIDDLE
-	movl	ACCUM_MS,%eax
-	adcl	4(%edi),%eax		/* term ms long */
-	movl	%eax,SUM_MS
-	sbbb	%al,%al
-	movb	%al,OVERFLOWED		/* Used in the next iteration */
-
-	subl	TERM_SIZE,%edi
-	decl	PARAM4
-	jns	L_accum_loop
-
-L_accum_done:
-	movl	PARAM1,%edi		/* accum */
-	movl	SUM_LS,%eax
-	addl	%eax,(%edi)
-	movl	SUM_MIDDLE,%eax
-	adcl	%eax,4(%edi)
-	movl	SUM_MS,%eax
-	adcl	%eax,8(%edi)
-
-	popl	%ebx
-	popl	%edi
-	popl	%esi
-	leave
-	RET
-SYM_FUNC_END(polynomial_Xsig)
diff --git a/arch/x86/math-emu/reg_add_sub.c b/arch/x86/math-emu/reg_add_sub.c
deleted file mode 100644
index 29451dd075567..0000000000000
--- a/arch/x86/math-emu/reg_add_sub.c
+++ /dev/null
@@ -1,334 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  reg_add_sub.c                                                            |
- |                                                                           |
- | Functions to add or subtract two registers and put the result in a third. |
- |                                                                           |
- | Copyright (C) 1992,1993,1997                                              |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- |  For each function, the destination may be any FPU_REG, including one of  |
- | the source FPU_REGs.                                                      |
- |  Each function returns 0 if the answer is o.k., otherwise a non-zero      |
- | value is returned, indicating either an exception condition or an         |
- | internal error.                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-#include "fpu_system.h"
-
-static
-int add_sub_specials(FPU_REG const *a, u_char taga, u_char signa,
-		     FPU_REG const *b, u_char tagb, u_char signb,
-		     FPU_REG * dest, int deststnr, int control_w);
-
-/*
-  Operates on st(0) and st(n), or on st(0) and temporary data.
-  The destination must be one of the source st(x).
-  */
-int FPU_add(FPU_REG const *b, u_char tagb, int deststnr, int control_w)
-{
-	FPU_REG *a = &st(0);
-	FPU_REG *dest = &st(deststnr);
-	u_char signb = getsign(b);
-	u_char taga = FPU_gettag0();
-	u_char signa = getsign(a);
-	u_char saved_sign = getsign(dest);
-	int diff, tag, expa, expb;
-
-	if (!(taga | tagb)) {
-		expa = exponent(a);
-		expb = exponent(b);
-
-	      valid_add:
-		/* Both registers are valid */
-		if (!(signa ^ signb)) {
-			/* signs are the same */
-			tag =
-			    FPU_u_add(a, b, dest, control_w, signa, expa, expb);
-		} else {
-			/* The signs are different, so do a subtraction */
-			diff = expa - expb;
-			if (!diff) {
-				diff = a->sigh - b->sigh;	/* This works only if the ms bits
-								   are identical. */
-				if (!diff) {
-					diff = a->sigl > b->sigl;
-					if (!diff)
-						diff = -(a->sigl < b->sigl);
-				}
-			}
-
-			if (diff > 0) {
-				tag =
-				    FPU_u_sub(a, b, dest, control_w, signa,
-					      expa, expb);
-			} else if (diff < 0) {
-				tag =
-				    FPU_u_sub(b, a, dest, control_w, signb,
-					      expb, expa);
-			} else {
-				FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
-				/* sign depends upon rounding mode */
-				setsign(dest, ((control_w & CW_RC) != RC_DOWN)
-					? SIGN_POS : SIGN_NEG);
-				return TAG_Zero;
-			}
-		}
-
-		if (tag < 0) {
-			setsign(dest, saved_sign);
-			return tag;
-		}
-		FPU_settagi(deststnr, tag);
-		return tag;
-	}
-
-	if (taga == TAG_Special)
-		taga = FPU_Special(a);
-	if (tagb == TAG_Special)
-		tagb = FPU_Special(b);
-
-	if (((taga == TAG_Valid) && (tagb == TW_Denormal))
-	    || ((taga == TW_Denormal) && (tagb == TAG_Valid))
-	    || ((taga == TW_Denormal) && (tagb == TW_Denormal))) {
-		FPU_REG x, y;
-
-		if (denormal_operand() < 0)
-			return FPU_Exception;
-
-		FPU_to_exp16(a, &x);
-		FPU_to_exp16(b, &y);
-		a = &x;
-		b = &y;
-		expa = exponent16(a);
-		expb = exponent16(b);
-		goto valid_add;
-	}
-
-	if ((taga == TW_NaN) || (tagb == TW_NaN)) {
-		if (deststnr == 0)
-			return real_2op_NaN(b, tagb, deststnr, a);
-		else
-			return real_2op_NaN(a, taga, deststnr, a);
-	}
-
-	return add_sub_specials(a, taga, signa, b, tagb, signb,
-				dest, deststnr, control_w);
-}
-
-/* Subtract b from a.  (a-b) -> dest */
-int FPU_sub(int flags, int rm, int control_w)
-{
-	FPU_REG const *a, *b;
-	FPU_REG *dest;
-	u_char taga, tagb, signa, signb, saved_sign, sign;
-	int diff, tag = 0, expa, expb, deststnr;
-
-	a = &st(0);
-	taga = FPU_gettag0();
-
-	deststnr = 0;
-	if (flags & LOADED) {
-		b = (FPU_REG *) rm;
-		tagb = flags & 0x0f;
-	} else {
-		b = &st(rm);
-		tagb = FPU_gettagi(rm);
-
-		if (flags & DEST_RM)
-			deststnr = rm;
-	}
-
-	signa = getsign(a);
-	signb = getsign(b);
-
-	if (flags & REV) {
-		signa ^= SIGN_NEG;
-		signb ^= SIGN_NEG;
-	}
-
-	dest = &st(deststnr);
-	saved_sign = getsign(dest);
-
-	if (!(taga | tagb)) {
-		expa = exponent(a);
-		expb = exponent(b);
-
-	      valid_subtract:
-		/* Both registers are valid */
-
-		diff = expa - expb;
-
-		if (!diff) {
-			diff = a->sigh - b->sigh;	/* Works only if ms bits are identical */
-			if (!diff) {
-				diff = a->sigl > b->sigl;
-				if (!diff)
-					diff = -(a->sigl < b->sigl);
-			}
-		}
-
-		switch ((((int)signa) * 2 + signb) / SIGN_NEG) {
-		case 0:	/* P - P */
-		case 3:	/* N - N */
-			if (diff > 0) {
-				/* |a| > |b| */
-				tag =
-				    FPU_u_sub(a, b, dest, control_w, signa,
-					      expa, expb);
-			} else if (diff == 0) {
-				FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
-
-				/* sign depends upon rounding mode */
-				setsign(dest, ((control_w & CW_RC) != RC_DOWN)
-					? SIGN_POS : SIGN_NEG);
-				return TAG_Zero;
-			} else {
-				sign = signa ^ SIGN_NEG;
-				tag =
-				    FPU_u_sub(b, a, dest, control_w, sign, expb,
-					      expa);
-			}
-			break;
-		case 1:	/* P - N */
-			tag =
-			    FPU_u_add(a, b, dest, control_w, SIGN_POS, expa,
-				      expb);
-			break;
-		case 2:	/* N - P */
-			tag =
-			    FPU_u_add(a, b, dest, control_w, SIGN_NEG, expa,
-				      expb);
-			break;
-#ifdef PARANOID
-		default:
-			EXCEPTION(EX_INTERNAL | 0x111);
-			return -1;
-#endif
-		}
-		if (tag < 0) {
-			setsign(dest, saved_sign);
-			return tag;
-		}
-		FPU_settagi(deststnr, tag);
-		return tag;
-	}
-
-	if (taga == TAG_Special)
-		taga = FPU_Special(a);
-	if (tagb == TAG_Special)
-		tagb = FPU_Special(b);
-
-	if (((taga == TAG_Valid) && (tagb == TW_Denormal))
-	    || ((taga == TW_Denormal) && (tagb == TAG_Valid))
-	    || ((taga == TW_Denormal) && (tagb == TW_Denormal))) {
-		FPU_REG x, y;
-
-		if (denormal_operand() < 0)
-			return FPU_Exception;
-
-		FPU_to_exp16(a, &x);
-		FPU_to_exp16(b, &y);
-		a = &x;
-		b = &y;
-		expa = exponent16(a);
-		expb = exponent16(b);
-
-		goto valid_subtract;
-	}
-
-	if ((taga == TW_NaN) || (tagb == TW_NaN)) {
-		FPU_REG const *d1, *d2;
-		if (flags & REV) {
-			d1 = b;
-			d2 = a;
-		} else {
-			d1 = a;
-			d2 = b;
-		}
-		if (flags & LOADED)
-			return real_2op_NaN(b, tagb, deststnr, d1);
-		if (flags & DEST_RM)
-			return real_2op_NaN(a, taga, deststnr, d2);
-		else
-			return real_2op_NaN(b, tagb, deststnr, d2);
-	}
-
-	return add_sub_specials(a, taga, signa, b, tagb, signb ^ SIGN_NEG,
-				dest, deststnr, control_w);
-}
-
-static
-int add_sub_specials(FPU_REG const *a, u_char taga, u_char signa,
-		     FPU_REG const *b, u_char tagb, u_char signb,
-		     FPU_REG * dest, int deststnr, int control_w)
-{
-	if (((taga == TW_Denormal) || (tagb == TW_Denormal))
-	    && (denormal_operand() < 0))
-		return FPU_Exception;
-
-	if (taga == TAG_Zero) {
-		if (tagb == TAG_Zero) {
-			/* Both are zero, result will be zero. */
-			u_char different_signs = signa ^ signb;
-
-			FPU_copy_to_regi(a, TAG_Zero, deststnr);
-			if (different_signs) {
-				/* Signs are different. */
-				/* Sign of answer depends upon rounding mode. */
-				setsign(dest, ((control_w & CW_RC) != RC_DOWN)
-					? SIGN_POS : SIGN_NEG);
-			} else
-				setsign(dest, signa);	/* signa may differ from the sign of a. */
-			return TAG_Zero;
-		} else {
-			reg_copy(b, dest);
-			if ((tagb == TW_Denormal) && (b->sigh & 0x80000000)) {
-				/* A pseudoDenormal, convert it. */
-				addexponent(dest, 1);
-				tagb = TAG_Valid;
-			} else if (tagb > TAG_Empty)
-				tagb = TAG_Special;
-			setsign(dest, signb);	/* signb may differ from the sign of b. */
-			FPU_settagi(deststnr, tagb);
-			return tagb;
-		}
-	} else if (tagb == TAG_Zero) {
-		reg_copy(a, dest);
-		if ((taga == TW_Denormal) && (a->sigh & 0x80000000)) {
-			/* A pseudoDenormal */
-			addexponent(dest, 1);
-			taga = TAG_Valid;
-		} else if (taga > TAG_Empty)
-			taga = TAG_Special;
-		setsign(dest, signa);	/* signa may differ from the sign of a. */
-		FPU_settagi(deststnr, taga);
-		return taga;
-	} else if (taga == TW_Infinity) {
-		if ((tagb != TW_Infinity) || (signa == signb)) {
-			FPU_copy_to_regi(a, TAG_Special, deststnr);
-			setsign(dest, signa);	/* signa may differ from the sign of a. */
-			return taga;
-		}
-		/* Infinity-Infinity is undefined. */
-		return arith_invalid(deststnr);
-	} else if (tagb == TW_Infinity) {
-		FPU_copy_to_regi(b, TAG_Special, deststnr);
-		setsign(dest, signb);	/* signb may differ from the sign of b. */
-		return tagb;
-	}
-#ifdef PARANOID
-	EXCEPTION(EX_INTERNAL | 0x101);
-#endif
-
-	return FPU_Exception;
-}
diff --git a/arch/x86/math-emu/reg_compare.c b/arch/x86/math-emu/reg_compare.c
deleted file mode 100644
index eacb5128fc09e..0000000000000
--- a/arch/x86/math-emu/reg_compare.c
+++ /dev/null
@@ -1,479 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  reg_compare.c                                                            |
- |                                                                           |
- | Compare two floating point registers                                      |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | compare() is the core FPU_REG comparison function                         |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-#include "status_w.h"
-
-static int compare(FPU_REG const *b, int tagb)
-{
-	int diff, exp0, expb;
-	u_char st0_tag;
-	FPU_REG *st0_ptr;
-	FPU_REG x, y;
-	u_char st0_sign, signb = getsign(b);
-
-	st0_ptr = &st(0);
-	st0_tag = FPU_gettag0();
-	st0_sign = getsign(st0_ptr);
-
-	if (tagb == TAG_Special)
-		tagb = FPU_Special(b);
-	if (st0_tag == TAG_Special)
-		st0_tag = FPU_Special(st0_ptr);
-
-	if (((st0_tag != TAG_Valid) && (st0_tag != TW_Denormal))
-	    || ((tagb != TAG_Valid) && (tagb != TW_Denormal))) {
-		if (st0_tag == TAG_Zero) {
-			if (tagb == TAG_Zero)
-				return COMP_A_eq_B;
-			if (tagb == TAG_Valid)
-				return ((signb ==
-					 SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B);
-			if (tagb == TW_Denormal)
-				return ((signb ==
-					 SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B)
-				    | COMP_Denormal;
-		} else if (tagb == TAG_Zero) {
-			if (st0_tag == TAG_Valid)
-				return ((st0_sign ==
-					 SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B);
-			if (st0_tag == TW_Denormal)
-				return ((st0_sign ==
-					 SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B)
-				    | COMP_Denormal;
-		}
-
-		if (st0_tag == TW_Infinity) {
-			if ((tagb == TAG_Valid) || (tagb == TAG_Zero))
-				return ((st0_sign ==
-					 SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B);
-			else if (tagb == TW_Denormal)
-				return ((st0_sign ==
-					 SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B)
-				    | COMP_Denormal;
-			else if (tagb == TW_Infinity) {
-				/* The 80486 book says that infinities can be equal! */
-				return (st0_sign == signb) ? COMP_A_eq_B :
-				    ((st0_sign ==
-				      SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B);
-			}
-			/* Fall through to the NaN code */
-		} else if (tagb == TW_Infinity) {
-			if ((st0_tag == TAG_Valid) || (st0_tag == TAG_Zero))
-				return ((signb ==
-					 SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B);
-			if (st0_tag == TW_Denormal)
-				return ((signb ==
-					 SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B)
-				    | COMP_Denormal;
-			/* Fall through to the NaN code */
-		}
-
-		/* The only possibility now should be that one of the arguments
-		   is a NaN */
-		if ((st0_tag == TW_NaN) || (tagb == TW_NaN)) {
-			int signalling = 0, unsupported = 0;
-			if (st0_tag == TW_NaN) {
-				signalling =
-				    (st0_ptr->sigh & 0xc0000000) == 0x80000000;
-				unsupported = !((exponent(st0_ptr) == EXP_OVER)
-						&& (st0_ptr->
-						    sigh & 0x80000000));
-			}
-			if (tagb == TW_NaN) {
-				signalling |=
-				    (b->sigh & 0xc0000000) == 0x80000000;
-				unsupported |= !((exponent(b) == EXP_OVER)
-						 && (b->sigh & 0x80000000));
-			}
-			if (signalling || unsupported)
-				return COMP_No_Comp | COMP_SNaN | COMP_NaN;
-			else
-				/* Neither is a signaling NaN */
-				return COMP_No_Comp | COMP_NaN;
-		}
-
-		EXCEPTION(EX_Invalid);
-	}
-
-	if (st0_sign != signb) {
-		return ((st0_sign == SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B)
-		    | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ?
-		       COMP_Denormal : 0);
-	}
-
-	if ((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) {
-		FPU_to_exp16(st0_ptr, &x);
-		FPU_to_exp16(b, &y);
-		st0_ptr = &x;
-		b = &y;
-		exp0 = exponent16(st0_ptr);
-		expb = exponent16(b);
-	} else {
-		exp0 = exponent(st0_ptr);
-		expb = exponent(b);
-	}
-
-#ifdef PARANOID
-	if (!(st0_ptr->sigh & 0x80000000))
-		EXCEPTION(EX_Invalid);
-	if (!(b->sigh & 0x80000000))
-		EXCEPTION(EX_Invalid);
-#endif /* PARANOID */
-
-	diff = exp0 - expb;
-	if (diff == 0) {
-		diff = st0_ptr->sigh - b->sigh;	/* Works only if ms bits are
-						   identical */
-		if (diff == 0) {
-			diff = st0_ptr->sigl > b->sigl;
-			if (diff == 0)
-				diff = -(st0_ptr->sigl < b->sigl);
-		}
-	}
-
-	if (diff > 0) {
-		return ((st0_sign == SIGN_POS) ? COMP_A_gt_B : COMP_A_lt_B)
-		    | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ?
-		       COMP_Denormal : 0);
-	}
-	if (diff < 0) {
-		return ((st0_sign == SIGN_POS) ? COMP_A_lt_B : COMP_A_gt_B)
-		    | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ?
-		       COMP_Denormal : 0);
-	}
-
-	return COMP_A_eq_B
-	    | (((st0_tag == TW_Denormal) || (tagb == TW_Denormal)) ?
-	       COMP_Denormal : 0);
-
-}
-
-/* This function requires that st(0) is not empty */
-int FPU_compare_st_data(FPU_REG const *loaded_data, u_char loaded_tag)
-{
-	int f, c;
-
-	c = compare(loaded_data, loaded_tag);
-
-	if (c & COMP_NaN) {
-		EXCEPTION(EX_Invalid);
-		f = SW_C3 | SW_C2 | SW_C0;
-	} else
-		switch (c & 7) {
-		case COMP_A_lt_B:
-			f = SW_C0;
-			break;
-		case COMP_A_eq_B:
-			f = SW_C3;
-			break;
-		case COMP_A_gt_B:
-			f = 0;
-			break;
-		case COMP_No_Comp:
-			f = SW_C3 | SW_C2 | SW_C0;
-			break;
-		default:
-#ifdef PARANOID
-			EXCEPTION(EX_INTERNAL | 0x121);
-#endif /* PARANOID */
-			f = SW_C3 | SW_C2 | SW_C0;
-			break;
-		}
-	setcc(f);
-	if (c & COMP_Denormal) {
-		return denormal_operand() < 0;
-	}
-	return 0;
-}
-
-static int compare_st_st(int nr)
-{
-	int f, c;
-	FPU_REG *st_ptr;
-
-	if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) {
-		setcc(SW_C3 | SW_C2 | SW_C0);
-		/* Stack fault */
-		EXCEPTION(EX_StackUnder);
-		return !(control_word & CW_Invalid);
-	}
-
-	st_ptr = &st(nr);
-	c = compare(st_ptr, FPU_gettagi(nr));
-	if (c & COMP_NaN) {
-		setcc(SW_C3 | SW_C2 | SW_C0);
-		EXCEPTION(EX_Invalid);
-		return !(control_word & CW_Invalid);
-	} else
-		switch (c & 7) {
-		case COMP_A_lt_B:
-			f = SW_C0;
-			break;
-		case COMP_A_eq_B:
-			f = SW_C3;
-			break;
-		case COMP_A_gt_B:
-			f = 0;
-			break;
-		case COMP_No_Comp:
-			f = SW_C3 | SW_C2 | SW_C0;
-			break;
-		default:
-#ifdef PARANOID
-			EXCEPTION(EX_INTERNAL | 0x122);
-#endif /* PARANOID */
-			f = SW_C3 | SW_C2 | SW_C0;
-			break;
-		}
-	setcc(f);
-	if (c & COMP_Denormal) {
-		return denormal_operand() < 0;
-	}
-	return 0;
-}
-
-static int compare_i_st_st(int nr)
-{
-	int f, c;
-	FPU_REG *st_ptr;
-
-	if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) {
-		FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF);
-		/* Stack fault */
-		EXCEPTION(EX_StackUnder);
-		return !(control_word & CW_Invalid);
-	}
-
-	partial_status &= ~SW_C0;
-	st_ptr = &st(nr);
-	c = compare(st_ptr, FPU_gettagi(nr));
-	if (c & COMP_NaN) {
-		FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF);
-		EXCEPTION(EX_Invalid);
-		return !(control_word & CW_Invalid);
-	}
-
-	switch (c & 7) {
-	case COMP_A_lt_B:
-		f = X86_EFLAGS_CF;
-		break;
-	case COMP_A_eq_B:
-		f = X86_EFLAGS_ZF;
-		break;
-	case COMP_A_gt_B:
-		f = 0;
-		break;
-	case COMP_No_Comp:
-		f = X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF;
-		break;
-	default:
-#ifdef PARANOID
-		EXCEPTION(EX_INTERNAL | 0x122);
-#endif /* PARANOID */
-		f = 0;
-		break;
-	}
-	FPU_EFLAGS = (FPU_EFLAGS & ~(X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF)) | f;
-	if (c & COMP_Denormal) {
-		return denormal_operand() < 0;
-	}
-	return 0;
-}
-
-static int compare_u_st_st(int nr)
-{
-	int f = 0, c;
-	FPU_REG *st_ptr;
-
-	if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) {
-		setcc(SW_C3 | SW_C2 | SW_C0);
-		/* Stack fault */
-		EXCEPTION(EX_StackUnder);
-		return !(control_word & CW_Invalid);
-	}
-
-	st_ptr = &st(nr);
-	c = compare(st_ptr, FPU_gettagi(nr));
-	if (c & COMP_NaN) {
-		setcc(SW_C3 | SW_C2 | SW_C0);
-		if (c & COMP_SNaN) {	/* This is the only difference between
-					   un-ordered and ordinary comparisons */
-			EXCEPTION(EX_Invalid);
-			return !(control_word & CW_Invalid);
-		}
-		return 0;
-	} else
-		switch (c & 7) {
-		case COMP_A_lt_B:
-			f = SW_C0;
-			break;
-		case COMP_A_eq_B:
-			f = SW_C3;
-			break;
-		case COMP_A_gt_B:
-			f = 0;
-			break;
-		case COMP_No_Comp:
-			f = SW_C3 | SW_C2 | SW_C0;
-			break;
-#ifdef PARANOID
-		default:
-			EXCEPTION(EX_INTERNAL | 0x123);
-			f = SW_C3 | SW_C2 | SW_C0;
-			break;
-#endif /* PARANOID */
-		}
-	setcc(f);
-	if (c & COMP_Denormal) {
-		return denormal_operand() < 0;
-	}
-	return 0;
-}
-
-static int compare_ui_st_st(int nr)
-{
-	int f = 0, c;
-	FPU_REG *st_ptr;
-
-	if (!NOT_EMPTY(0) || !NOT_EMPTY(nr)) {
-		FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF);
-		/* Stack fault */
-		EXCEPTION(EX_StackUnder);
-		return !(control_word & CW_Invalid);
-	}
-
-	partial_status &= ~SW_C0;
-	st_ptr = &st(nr);
-	c = compare(st_ptr, FPU_gettagi(nr));
-	if (c & COMP_NaN) {
-		FPU_EFLAGS |= (X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF);
-		if (c & COMP_SNaN) {	/* This is the only difference between
-					   un-ordered and ordinary comparisons */
-			EXCEPTION(EX_Invalid);
-			return !(control_word & CW_Invalid);
-		}
-		return 0;
-	}
-
-	switch (c & 7) {
-	case COMP_A_lt_B:
-		f = X86_EFLAGS_CF;
-		break;
-	case COMP_A_eq_B:
-		f = X86_EFLAGS_ZF;
-		break;
-	case COMP_A_gt_B:
-		f = 0;
-		break;
-	case COMP_No_Comp:
-		f = X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF;
-		break;
-#ifdef PARANOID
-	default:
-		EXCEPTION(EX_INTERNAL | 0x123);
-		f = 0;
-		break;
-#endif /* PARANOID */
-	}
-	FPU_EFLAGS = (FPU_EFLAGS & ~(X86_EFLAGS_ZF | X86_EFLAGS_PF | X86_EFLAGS_CF)) | f;
-	if (c & COMP_Denormal) {
-		return denormal_operand() < 0;
-	}
-	return 0;
-}
-
-/*---------------------------------------------------------------------------*/
-
-void fcom_st(void)
-{
-	/* fcom st(i) */
-	compare_st_st(FPU_rm);
-}
-
-void fcompst(void)
-{
-	/* fcomp st(i) */
-	if (!compare_st_st(FPU_rm))
-		FPU_pop();
-}
-
-void fcompp(void)
-{
-	/* fcompp */
-	if (FPU_rm != 1) {
-		FPU_illegal();
-		return;
-	}
-	if (!compare_st_st(1))
-		poppop();
-}
-
-void fucom_(void)
-{
-	/* fucom st(i) */
-	compare_u_st_st(FPU_rm);
-
-}
-
-void fucomp(void)
-{
-	/* fucomp st(i) */
-	if (!compare_u_st_st(FPU_rm))
-		FPU_pop();
-}
-
-void fucompp(void)
-{
-	/* fucompp */
-	if (FPU_rm == 1) {
-		if (!compare_u_st_st(1))
-			poppop();
-	} else
-		FPU_illegal();
-}
-
-/* P6+ compare-to-EFLAGS ops */
-
-void fcomi_(void)
-{
-	/* fcomi st(i) */
-	compare_i_st_st(FPU_rm);
-}
-
-void fcomip(void)
-{
-	/* fcomip st(i) */
-	if (!compare_i_st_st(FPU_rm))
-		FPU_pop();
-}
-
-void fucomi_(void)
-{
-	/* fucomi st(i) */
-	compare_ui_st_st(FPU_rm);
-}
-
-void fucomip(void)
-{
-	/* fucomip st(i) */
-	if (!compare_ui_st_st(FPU_rm))
-		FPU_pop();
-}
diff --git a/arch/x86/math-emu/reg_constant.c b/arch/x86/math-emu/reg_constant.c
deleted file mode 100644
index 003a0b2753e68..0000000000000
--- a/arch/x86/math-emu/reg_constant.c
+++ /dev/null
@@ -1,123 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  reg_constant.c                                                           |
- |                                                                           |
- | All of the constant FPU_REGs                                              |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1997                                         |
- |                     W. Metzenthen, 22 Parker St, Ormond, Vic 3163,        |
- |                     Australia.  E-mail   billm@suburbia.net               |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_system.h"
-#include "fpu_emu.h"
-#include "status_w.h"
-#include "reg_constant.h"
-#include "control_w.h"
-
-#define MAKE_REG(s, e, l, h) { l, h, \
-		(u16)((EXTENDED_Ebias+(e)) | ((SIGN_##s != 0)*0x8000)) }
-
-FPU_REG const CONST_1 = MAKE_REG(POS, 0, 0x00000000, 0x80000000);
-#if 0
-FPU_REG const CONST_2 = MAKE_REG(POS, 1, 0x00000000, 0x80000000);
-FPU_REG const CONST_HALF = MAKE_REG(POS, -1, 0x00000000, 0x80000000);
-#endif /*  0  */
-static FPU_REG const CONST_L2T = MAKE_REG(POS, 1, 0xcd1b8afe, 0xd49a784b);
-static FPU_REG const CONST_L2E = MAKE_REG(POS, 0, 0x5c17f0bc, 0xb8aa3b29);
-FPU_REG const CONST_PI = MAKE_REG(POS, 1, 0x2168c235, 0xc90fdaa2);
-FPU_REG const CONST_PI2 = MAKE_REG(POS, 0, 0x2168c235, 0xc90fdaa2);
-FPU_REG const CONST_PI4 = MAKE_REG(POS, -1, 0x2168c235, 0xc90fdaa2);
-static FPU_REG const CONST_LG2 = MAKE_REG(POS, -2, 0xfbcff799, 0x9a209a84);
-static FPU_REG const CONST_LN2 = MAKE_REG(POS, -1, 0xd1cf79ac, 0xb17217f7);
-
-/* Extra bits to take pi/2 to more than 128 bits precision. */
-FPU_REG const CONST_PI2extra = MAKE_REG(NEG, -66,
-					0xfc8f8cbb, 0xece675d1);
-
-/* Only the sign (and tag) is used in internal zeroes */
-FPU_REG const CONST_Z = MAKE_REG(POS, EXP_UNDER, 0x0, 0x0);
-
-/* Only the sign and significand (and tag) are used in internal NaNs */
-/* The 80486 never generates one of these
-FPU_REG const CONST_SNAN = MAKE_REG(POS, EXP_OVER, 0x00000001, 0x80000000);
- */
-/* This is the real indefinite QNaN */
-FPU_REG const CONST_QNaN = MAKE_REG(NEG, EXP_OVER, 0x00000000, 0xC0000000);
-
-/* Only the sign (and tag) is used in internal infinities */
-FPU_REG const CONST_INF = MAKE_REG(POS, EXP_OVER, 0x00000000, 0x80000000);
-
-static void fld_const(FPU_REG const * c, int adj, u_char tag)
-{
-	FPU_REG *st_new_ptr;
-
-	if (STACK_OVERFLOW) {
-		FPU_stack_overflow();
-		return;
-	}
-	push();
-	reg_copy(c, st_new_ptr);
-	st_new_ptr->sigl += adj;	/* For all our fldxxx constants, we don't need to
-					   borrow or carry. */
-	FPU_settag0(tag);
-	clear_C1();
-}
-
-/* A fast way to find out whether x is one of RC_DOWN or RC_CHOP
-   (and not one of RC_RND or RC_UP).
-   */
-#define DOWN_OR_CHOP(x)  (x & RC_DOWN)
-
-static void fld1(int rc)
-{
-	fld_const(&CONST_1, 0, TAG_Valid);
-}
-
-static void fldl2t(int rc)
-{
-	fld_const(&CONST_L2T, (rc == RC_UP) ? 1 : 0, TAG_Valid);
-}
-
-static void fldl2e(int rc)
-{
-	fld_const(&CONST_L2E, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid);
-}
-
-static void fldpi(int rc)
-{
-	fld_const(&CONST_PI, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid);
-}
-
-static void fldlg2(int rc)
-{
-	fld_const(&CONST_LG2, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid);
-}
-
-static void fldln2(int rc)
-{
-	fld_const(&CONST_LN2, DOWN_OR_CHOP(rc) ? -1 : 0, TAG_Valid);
-}
-
-static void fldz(int rc)
-{
-	fld_const(&CONST_Z, 0, TAG_Zero);
-}
-
-typedef void (*FUNC_RC) (int);
-
-static void FPU_RC_illegal(int unused)
-{
-	FPU_illegal();
-}
-
-static FUNC_RC constants_table[] = {
-	fld1, fldl2t, fldl2e, fldpi, fldlg2, fldln2, fldz, FPU_RC_illegal
-};
-
-void fconst(void)
-{
-	(constants_table[FPU_rm]) (control_word & CW_RC);
-}
diff --git a/arch/x86/math-emu/reg_constant.h b/arch/x86/math-emu/reg_constant.h
deleted file mode 100644
index f2fdd344dcaad..0000000000000
--- a/arch/x86/math-emu/reg_constant.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  reg_constant.h                                                           |
- |                                                                           |
- | Copyright (C) 1992    W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@vaxc.cc.monash.edu.au    |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#ifndef _REG_CONSTANT_H_
-#define _REG_CONSTANT_H_
-
-#include "fpu_emu.h"
-
-extern FPU_REG const CONST_1;
-extern FPU_REG const CONST_PI;
-extern FPU_REG const CONST_PI2;
-extern FPU_REG const CONST_PI2extra;
-extern FPU_REG const CONST_PI4;
-extern FPU_REG const CONST_Z;
-extern FPU_REG const CONST_PINF;
-extern FPU_REG const CONST_INF;
-extern FPU_REG const CONST_MINF;
-extern FPU_REG const CONST_QNaN;
-
-#endif /* _REG_CONSTANT_H_ */
diff --git a/arch/x86/math-emu/reg_convert.c b/arch/x86/math-emu/reg_convert.c
deleted file mode 100644
index 251180623ce05..0000000000000
--- a/arch/x86/math-emu/reg_convert.c
+++ /dev/null
@@ -1,47 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  reg_convert.c                                                            |
- |                                                                           |
- |  Convert register representation.                                         |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1996,1997                                    |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-
-int FPU_to_exp16(FPU_REG const *a, FPU_REG *x)
-{
-	int sign = getsign(a);
-
-	*(long long *)&(x->sigl) = *(const long long *)&(a->sigl);
-
-	/* Set up the exponent as a 16 bit quantity. */
-	setexponent16(x, exponent(a));
-
-	if (exponent16(x) == EXP_UNDER) {
-		/* The number is a de-normal or pseudodenormal. */
-		/* We only deal with the significand and exponent. */
-
-		if (x->sigh & 0x80000000) {
-			/* Is a pseudodenormal. */
-			/* This is non-80486 behaviour because the number
-			   loses its 'denormal' identity. */
-			addexponent(x, 1);
-		} else {
-			/* Is a denormal. */
-			addexponent(x, 1);
-			FPU_normalize_nuo(x);
-		}
-	}
-
-	if (!(x->sigh & 0x80000000)) {
-		EXCEPTION(EX_INTERNAL | 0x180);
-	}
-
-	return sign;
-}
diff --git a/arch/x86/math-emu/reg_divide.c b/arch/x86/math-emu/reg_divide.c
deleted file mode 100644
index 08c2f6de0288b..0000000000000
--- a/arch/x86/math-emu/reg_divide.c
+++ /dev/null
@@ -1,183 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  reg_divide.c                                                             |
- |                                                                           |
- | Divide one FPU_REG by another and put the result in a destination FPU_REG.|
- |                                                                           |
- | Copyright (C) 1996                                                        |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@jacobi.maths.monash.edu.au                |
- |                                                                           |
- |    Return value is the tag of the answer, or-ed with FPU_Exception if     |
- |    one was raised, or -1 on internal error.                               |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | The destination may be any FPU_REG, including one of the source FPU_REGs. |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_emu.h"
-#include "fpu_system.h"
-
-/*
-  Divide one register by another and put the result into a third register.
-  */
-int FPU_div(int flags, int rm, int control_w)
-{
-	FPU_REG x, y;
-	FPU_REG const *a, *b, *st0_ptr, *st_ptr;
-	FPU_REG *dest;
-	u_char taga, tagb, signa, signb, sign, saved_sign;
-	int tag, deststnr;
-
-	if (flags & DEST_RM)
-		deststnr = rm;
-	else
-		deststnr = 0;
-
-	if (flags & REV) {
-		b = &st(0);
-		st0_ptr = b;
-		tagb = FPU_gettag0();
-		if (flags & LOADED) {
-			a = (FPU_REG *) rm;
-			taga = flags & 0x0f;
-		} else {
-			a = &st(rm);
-			st_ptr = a;
-			taga = FPU_gettagi(rm);
-		}
-	} else {
-		a = &st(0);
-		st0_ptr = a;
-		taga = FPU_gettag0();
-		if (flags & LOADED) {
-			b = (FPU_REG *) rm;
-			tagb = flags & 0x0f;
-		} else {
-			b = &st(rm);
-			st_ptr = b;
-			tagb = FPU_gettagi(rm);
-		}
-	}
-
-	signa = getsign(a);
-	signb = getsign(b);
-
-	sign = signa ^ signb;
-
-	dest = &st(deststnr);
-	saved_sign = getsign(dest);
-
-	if (!(taga | tagb)) {
-		/* Both regs Valid, this should be the most common case. */
-		reg_copy(a, &x);
-		reg_copy(b, &y);
-		setpositive(&x);
-		setpositive(&y);
-		tag = FPU_u_div(&x, &y, dest, control_w, sign);
-
-		if (tag < 0)
-			return tag;
-
-		FPU_settagi(deststnr, tag);
-		return tag;
-	}
-
-	if (taga == TAG_Special)
-		taga = FPU_Special(a);
-	if (tagb == TAG_Special)
-		tagb = FPU_Special(b);
-
-	if (((taga == TAG_Valid) && (tagb == TW_Denormal))
-	    || ((taga == TW_Denormal) && (tagb == TAG_Valid))
-	    || ((taga == TW_Denormal) && (tagb == TW_Denormal))) {
-		if (denormal_operand() < 0)
-			return FPU_Exception;
-
-		FPU_to_exp16(a, &x);
-		FPU_to_exp16(b, &y);
-		tag = FPU_u_div(&x, &y, dest, control_w, sign);
-		if (tag < 0)
-			return tag;
-
-		FPU_settagi(deststnr, tag);
-		return tag;
-	} else if ((taga <= TW_Denormal) && (tagb <= TW_Denormal)) {
-		if (tagb != TAG_Zero) {
-			/* Want to find Zero/Valid */
-			if (tagb == TW_Denormal) {
-				if (denormal_operand() < 0)
-					return FPU_Exception;
-			}
-
-			/* The result is zero. */
-			FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
-			setsign(dest, sign);
-			return TAG_Zero;
-		}
-		/* We have an exception condition, either 0/0 or Valid/Zero. */
-		if (taga == TAG_Zero) {
-			/* 0/0 */
-			return arith_invalid(deststnr);
-		}
-		/* Valid/Zero */
-		return FPU_divide_by_zero(deststnr, sign);
-	}
-	/* Must have infinities, NaNs, etc */
-	else if ((taga == TW_NaN) || (tagb == TW_NaN)) {
-		if (flags & LOADED)
-			return real_2op_NaN((FPU_REG *) rm, flags & 0x0f, 0,
-					    st0_ptr);
-
-		if (flags & DEST_RM) {
-			int tag;
-			tag = FPU_gettag0();
-			if (tag == TAG_Special)
-				tag = FPU_Special(st0_ptr);
-			return real_2op_NaN(st0_ptr, tag, rm,
-					    (flags & REV) ? st0_ptr : &st(rm));
-		} else {
-			int tag;
-			tag = FPU_gettagi(rm);
-			if (tag == TAG_Special)
-				tag = FPU_Special(&st(rm));
-			return real_2op_NaN(&st(rm), tag, 0,
-					    (flags & REV) ? st0_ptr : &st(rm));
-		}
-	} else if (taga == TW_Infinity) {
-		if (tagb == TW_Infinity) {
-			/* infinity/infinity */
-			return arith_invalid(deststnr);
-		} else {
-			/* tagb must be Valid or Zero */
-			if ((tagb == TW_Denormal) && (denormal_operand() < 0))
-				return FPU_Exception;
-
-			/* Infinity divided by Zero or Valid does
-			   not raise and exception, but returns Infinity */
-			FPU_copy_to_regi(a, TAG_Special, deststnr);
-			setsign(dest, sign);
-			return taga;
-		}
-	} else if (tagb == TW_Infinity) {
-		if ((taga == TW_Denormal) && (denormal_operand() < 0))
-			return FPU_Exception;
-
-		/* The result is zero. */
-		FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
-		setsign(dest, sign);
-		return TAG_Zero;
-	}
-#ifdef PARANOID
-	else {
-		EXCEPTION(EX_INTERNAL | 0x102);
-		return FPU_Exception;
-	}
-#endif /* PARANOID */
-
-	return 0;
-}
diff --git a/arch/x86/math-emu/reg_ld_str.c b/arch/x86/math-emu/reg_ld_str.c
deleted file mode 100644
index 7e4521fbe7da0..0000000000000
--- a/arch/x86/math-emu/reg_ld_str.c
+++ /dev/null
@@ -1,1220 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  reg_ld_str.c                                                             |
- |                                                                           |
- | All of the functions which transfer data between user memory and FPU_REGs.|
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1996,1997                                    |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Note:                                                                     |
- |    The file contains code which accesses user memory.                     |
- |    Emulator static data may change when user memory is accessed, due to   |
- |    other processes using the emulator while swapping is in progress.      |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-#include <linux/uaccess.h>
-
-#include "fpu_system.h"
-#include "exception.h"
-#include "reg_constant.h"
-#include "control_w.h"
-#include "status_w.h"
-
-#define DOUBLE_Emax 1023	/* largest valid exponent */
-#define DOUBLE_Ebias 1023
-#define DOUBLE_Emin (-1022)	/* smallest valid exponent */
-
-#define SINGLE_Emax 127		/* largest valid exponent */
-#define SINGLE_Ebias 127
-#define SINGLE_Emin (-126)	/* smallest valid exponent */
-
-static u_char normalize_no_excep(FPU_REG *r, int exp, int sign)
-{
-	u_char tag;
-
-	setexponent16(r, exp);
-
-	tag = FPU_normalize_nuo(r);
-	stdexp(r);
-	if (sign)
-		setnegative(r);
-
-	return tag;
-}
-
-int FPU_tagof(FPU_REG *ptr)
-{
-	int exp;
-
-	exp = exponent16(ptr) & 0x7fff;
-	if (exp == 0) {
-		if (!(ptr->sigh | ptr->sigl)) {
-			return TAG_Zero;
-		}
-		/* The number is a de-normal or pseudodenormal. */
-		return TAG_Special;
-	}
-
-	if (exp == 0x7fff) {
-		/* Is an Infinity, a NaN, or an unsupported data type. */
-		return TAG_Special;
-	}
-
-	if (!(ptr->sigh & 0x80000000)) {
-		/* Unsupported data type. */
-		/* Valid numbers have the ms bit set to 1. */
-		/* Unnormal. */
-		return TAG_Special;
-	}
-
-	return TAG_Valid;
-}
-
-/* Get a long double from user memory */
-int FPU_load_extended(long double __user *s, int stnr)
-{
-	FPU_REG *sti_ptr = &st(stnr);
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(s, 10);
-	FPU_copy_from_user(sti_ptr, s, 10);
-	RE_ENTRANT_CHECK_ON;
-
-	return FPU_tagof(sti_ptr);
-}
-
-/* Get a double from user memory */
-int FPU_load_double(double __user *dfloat, FPU_REG *loaded_data)
-{
-	int exp, tag, negative;
-	unsigned m64, l64;
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(dfloat, 8);
-	FPU_get_user(m64, 1 + (unsigned long __user *)dfloat);
-	FPU_get_user(l64, (unsigned long __user *)dfloat);
-	RE_ENTRANT_CHECK_ON;
-
-	negative = (m64 & 0x80000000) ? SIGN_Negative : SIGN_Positive;
-	exp = ((m64 & 0x7ff00000) >> 20) - DOUBLE_Ebias + EXTENDED_Ebias;
-	m64 &= 0xfffff;
-	if (exp > DOUBLE_Emax + EXTENDED_Ebias) {
-		/* Infinity or NaN */
-		if ((m64 == 0) && (l64 == 0)) {
-			/* +- infinity */
-			loaded_data->sigh = 0x80000000;
-			loaded_data->sigl = 0x00000000;
-			exp = EXP_Infinity + EXTENDED_Ebias;
-			tag = TAG_Special;
-		} else {
-			/* Must be a signaling or quiet NaN */
-			exp = EXP_NaN + EXTENDED_Ebias;
-			loaded_data->sigh = (m64 << 11) | 0x80000000;
-			loaded_data->sigh |= l64 >> 21;
-			loaded_data->sigl = l64 << 11;
-			tag = TAG_Special;	/* The calling function must look for NaNs */
-		}
-	} else if (exp < DOUBLE_Emin + EXTENDED_Ebias) {
-		/* Zero or de-normal */
-		if ((m64 == 0) && (l64 == 0)) {
-			/* Zero */
-			reg_copy(&CONST_Z, loaded_data);
-			exp = 0;
-			tag = TAG_Zero;
-		} else {
-			/* De-normal */
-			loaded_data->sigh = m64 << 11;
-			loaded_data->sigh |= l64 >> 21;
-			loaded_data->sigl = l64 << 11;
-
-			return normalize_no_excep(loaded_data, DOUBLE_Emin,
-						  negative)
-			    | (denormal_operand() < 0 ? FPU_Exception : 0);
-		}
-	} else {
-		loaded_data->sigh = (m64 << 11) | 0x80000000;
-		loaded_data->sigh |= l64 >> 21;
-		loaded_data->sigl = l64 << 11;
-
-		tag = TAG_Valid;
-	}
-
-	setexponent16(loaded_data, exp | negative);
-
-	return tag;
-}
-
-/* Get a float from user memory */
-int FPU_load_single(float __user *single, FPU_REG *loaded_data)
-{
-	unsigned m32;
-	int exp, tag, negative;
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(single, 4);
-	FPU_get_user(m32, (unsigned long __user *)single);
-	RE_ENTRANT_CHECK_ON;
-
-	negative = (m32 & 0x80000000) ? SIGN_Negative : SIGN_Positive;
-
-	if (!(m32 & 0x7fffffff)) {
-		/* Zero */
-		reg_copy(&CONST_Z, loaded_data);
-		addexponent(loaded_data, negative);
-		return TAG_Zero;
-	}
-	exp = ((m32 & 0x7f800000) >> 23) - SINGLE_Ebias + EXTENDED_Ebias;
-	m32 = (m32 & 0x7fffff) << 8;
-	if (exp < SINGLE_Emin + EXTENDED_Ebias) {
-		/* De-normals */
-		loaded_data->sigh = m32;
-		loaded_data->sigl = 0;
-
-		return normalize_no_excep(loaded_data, SINGLE_Emin, negative)
-		    | (denormal_operand() < 0 ? FPU_Exception : 0);
-	} else if (exp > SINGLE_Emax + EXTENDED_Ebias) {
-		/* Infinity or NaN */
-		if (m32 == 0) {
-			/* +- infinity */
-			loaded_data->sigh = 0x80000000;
-			loaded_data->sigl = 0x00000000;
-			exp = EXP_Infinity + EXTENDED_Ebias;
-			tag = TAG_Special;
-		} else {
-			/* Must be a signaling or quiet NaN */
-			exp = EXP_NaN + EXTENDED_Ebias;
-			loaded_data->sigh = m32 | 0x80000000;
-			loaded_data->sigl = 0;
-			tag = TAG_Special;	/* The calling function must look for NaNs */
-		}
-	} else {
-		loaded_data->sigh = m32 | 0x80000000;
-		loaded_data->sigl = 0;
-		tag = TAG_Valid;
-	}
-
-	setexponent16(loaded_data, exp | negative);	/* Set the sign. */
-
-	return tag;
-}
-
-/* Get a long long from user memory */
-int FPU_load_int64(long long __user *_s)
-{
-	long long s;
-	int sign;
-	FPU_REG *st0_ptr = &st(0);
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(_s, 8);
-	if (copy_from_user(&s, _s, 8))
-		FPU_abort;
-	RE_ENTRANT_CHECK_ON;
-
-	if (s == 0) {
-		reg_copy(&CONST_Z, st0_ptr);
-		return TAG_Zero;
-	}
-
-	if (s > 0)
-		sign = SIGN_Positive;
-	else {
-		s = -s;
-		sign = SIGN_Negative;
-	}
-
-	significand(st0_ptr) = s;
-
-	return normalize_no_excep(st0_ptr, 63, sign);
-}
-
-/* Get a long from user memory */
-int FPU_load_int32(long __user *_s, FPU_REG *loaded_data)
-{
-	long s;
-	int negative;
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(_s, 4);
-	FPU_get_user(s, _s);
-	RE_ENTRANT_CHECK_ON;
-
-	if (s == 0) {
-		reg_copy(&CONST_Z, loaded_data);
-		return TAG_Zero;
-	}
-
-	if (s > 0)
-		negative = SIGN_Positive;
-	else {
-		s = -s;
-		negative = SIGN_Negative;
-	}
-
-	loaded_data->sigh = s;
-	loaded_data->sigl = 0;
-
-	return normalize_no_excep(loaded_data, 31, negative);
-}
-
-/* Get a short from user memory */
-int FPU_load_int16(short __user *_s, FPU_REG *loaded_data)
-{
-	int s, negative;
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(_s, 2);
-	/* Cast as short to get the sign extended. */
-	FPU_get_user(s, _s);
-	RE_ENTRANT_CHECK_ON;
-
-	if (s == 0) {
-		reg_copy(&CONST_Z, loaded_data);
-		return TAG_Zero;
-	}
-
-	if (s > 0)
-		negative = SIGN_Positive;
-	else {
-		s = -s;
-		negative = SIGN_Negative;
-	}
-
-	loaded_data->sigh = s << 16;
-	loaded_data->sigl = 0;
-
-	return normalize_no_excep(loaded_data, 15, negative);
-}
-
-/* Get a packed bcd array from user memory */
-int FPU_load_bcd(u_char __user *s)
-{
-	FPU_REG *st0_ptr = &st(0);
-	int pos;
-	u_char bcd;
-	long long l = 0;
-	int sign;
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(s, 10);
-	RE_ENTRANT_CHECK_ON;
-	for (pos = 8; pos >= 0; pos--) {
-		l *= 10;
-		RE_ENTRANT_CHECK_OFF;
-		FPU_get_user(bcd, s + pos);
-		RE_ENTRANT_CHECK_ON;
-		l += bcd >> 4;
-		l *= 10;
-		l += bcd & 0x0f;
-	}
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_get_user(sign, s + 9);
-	sign = sign & 0x80 ? SIGN_Negative : SIGN_Positive;
-	RE_ENTRANT_CHECK_ON;
-
-	if (l == 0) {
-		reg_copy(&CONST_Z, st0_ptr);
-		addexponent(st0_ptr, sign);	/* Set the sign. */
-		return TAG_Zero;
-	} else {
-		significand(st0_ptr) = l;
-		return normalize_no_excep(st0_ptr, 63, sign);
-	}
-}
-
-/*===========================================================================*/
-
-/* Put a long double into user memory */
-int FPU_store_extended(FPU_REG *st0_ptr, u_char st0_tag,
-		       long double __user * d)
-{
-	/*
-	   The only exception raised by an attempt to store to an
-	   extended format is the Invalid Stack exception, i.e.
-	   attempting to store from an empty register.
-	 */
-
-	if (st0_tag != TAG_Empty) {
-		RE_ENTRANT_CHECK_OFF;
-		FPU_access_ok(d, 10);
-
-		FPU_put_user(st0_ptr->sigl, (unsigned long __user *)d);
-		FPU_put_user(st0_ptr->sigh,
-			     (unsigned long __user *)((u_char __user *) d + 4));
-		FPU_put_user(exponent16(st0_ptr),
-			     (unsigned short __user *)((u_char __user *) d +
-						       8));
-		RE_ENTRANT_CHECK_ON;
-
-		return 1;
-	}
-
-	/* Empty register (stack underflow) */
-	EXCEPTION(EX_StackUnder);
-	if (control_word & CW_Invalid) {
-		/* The masked response */
-		/* Put out the QNaN indefinite */
-		RE_ENTRANT_CHECK_OFF;
-		FPU_access_ok(d, 10);
-		FPU_put_user(0, (unsigned long __user *)d);
-		FPU_put_user(0xc0000000, 1 + (unsigned long __user *)d);
-		FPU_put_user(0xffff, 4 + (short __user *)d);
-		RE_ENTRANT_CHECK_ON;
-		return 1;
-	} else
-		return 0;
-
-}
-
-/* Put a double into user memory */
-int FPU_store_double(FPU_REG *st0_ptr, u_char st0_tag, double __user *dfloat)
-{
-	unsigned long l[2];
-	unsigned long increment = 0;	/* avoid gcc warnings */
-	int precision_loss;
-	int exp;
-	FPU_REG tmp;
-
-	l[0] = 0;
-	l[1] = 0;
-	if (st0_tag == TAG_Valid) {
-		reg_copy(st0_ptr, &tmp);
-		exp = exponent(&tmp);
-
-		if (exp < DOUBLE_Emin) {	/* It may be a denormal */
-			addexponent(&tmp, -DOUBLE_Emin + 52);	/* largest exp to be 51 */
-denormal_arg:
-			if ((precision_loss = FPU_round_to_int(&tmp, st0_tag))) {
-#ifdef PECULIAR_486
-				/* Did it round to a non-denormal ? */
-				/* This behaviour might be regarded as peculiar, it appears
-				   that the 80486 rounds to the dest precision, then
-				   converts to decide underflow. */
-				if (!
-				    ((tmp.sigh == 0x00100000) && (tmp.sigl == 0)
-				     && (st0_ptr->sigl & 0x000007ff)))
-#endif /* PECULIAR_486 */
-				{
-					EXCEPTION(EX_Underflow);
-					/* This is a special case: see sec 16.2.5.1 of
-					   the 80486 book */
-					if (!(control_word & CW_Underflow))
-						return 0;
-				}
-				EXCEPTION(precision_loss);
-				if (!(control_word & CW_Precision))
-					return 0;
-			}
-			l[0] = tmp.sigl;
-			l[1] = tmp.sigh;
-		} else {
-			if (tmp.sigl & 0x000007ff) {
-				precision_loss = 1;
-				switch (control_word & CW_RC) {
-				case RC_RND:
-					/* Rounding can get a little messy.. */
-					increment = ((tmp.sigl & 0x7ff) > 0x400) |	/* nearest */
-					    ((tmp.sigl & 0xc00) == 0xc00);	/* odd -> even */
-					break;
-				case RC_DOWN:	/* towards -infinity */
-					increment =
-					    signpositive(&tmp) ? 0 : tmp.
-					    sigl & 0x7ff;
-					break;
-				case RC_UP:	/* towards +infinity */
-					increment =
-					    signpositive(&tmp) ? tmp.
-					    sigl & 0x7ff : 0;
-					break;
-				case RC_CHOP:
-					increment = 0;
-					break;
-				}
-
-				/* Truncate the mantissa */
-				tmp.sigl &= 0xfffff800;
-
-				if (increment) {
-					if (tmp.sigl >= 0xfffff800) {
-						/* the sigl part overflows */
-						if (tmp.sigh == 0xffffffff) {
-							/* The sigh part overflows */
-							tmp.sigh = 0x80000000;
-							exp++;
-							if (exp >= EXP_OVER)
-								goto overflow;
-						} else {
-							tmp.sigh++;
-						}
-						tmp.sigl = 0x00000000;
-					} else {
-						/* We only need to increment sigl */
-						tmp.sigl += 0x00000800;
-					}
-				}
-			} else
-				precision_loss = 0;
-
-			l[0] = (tmp.sigl >> 11) | (tmp.sigh << 21);
-			l[1] = ((tmp.sigh >> 11) & 0xfffff);
-
-			if (exp > DOUBLE_Emax) {
-			      overflow:
-				EXCEPTION(EX_Overflow);
-				if (!(control_word & CW_Overflow))
-					return 0;
-				set_precision_flag_up();
-				if (!(control_word & CW_Precision))
-					return 0;
-
-				/* This is a special case: see sec 16.2.5.1 of the 80486 book */
-				/* Overflow to infinity */
-				l[1] = 0x7ff00000;	/* Set to + INF */
-			} else {
-				if (precision_loss) {
-					if (increment)
-						set_precision_flag_up();
-					else
-						set_precision_flag_down();
-				}
-				/* Add the exponent */
-				l[1] |= (((exp + DOUBLE_Ebias) & 0x7ff) << 20);
-			}
-		}
-	} else if (st0_tag == TAG_Zero) {
-		/* Number is zero */
-	} else if (st0_tag == TAG_Special) {
-		st0_tag = FPU_Special(st0_ptr);
-		if (st0_tag == TW_Denormal) {
-			/* A denormal will always underflow. */
-#ifndef PECULIAR_486
-			/* An 80486 is supposed to be able to generate
-			   a denormal exception here, but... */
-			/* Underflow has priority. */
-			if (control_word & CW_Underflow)
-				denormal_operand();
-#endif /* PECULIAR_486 */
-			reg_copy(st0_ptr, &tmp);
-			goto denormal_arg;
-		} else if (st0_tag == TW_Infinity) {
-			l[1] = 0x7ff00000;
-		} else if (st0_tag == TW_NaN) {
-			/* Is it really a NaN ? */
-			if ((exponent(st0_ptr) == EXP_OVER)
-			    && (st0_ptr->sigh & 0x80000000)) {
-				/* See if we can get a valid NaN from the FPU_REG */
-				l[0] =
-				    (st0_ptr->sigl >> 11) | (st0_ptr->
-							     sigh << 21);
-				l[1] = ((st0_ptr->sigh >> 11) & 0xfffff);
-				if (!(st0_ptr->sigh & 0x40000000)) {
-					/* It is a signalling NaN */
-					EXCEPTION(EX_Invalid);
-					if (!(control_word & CW_Invalid))
-						return 0;
-					l[1] |= (0x40000000 >> 11);
-				}
-				l[1] |= 0x7ff00000;
-			} else {
-				/* It is an unsupported data type */
-				EXCEPTION(EX_Invalid);
-				if (!(control_word & CW_Invalid))
-					return 0;
-				l[1] = 0xfff80000;
-			}
-		}
-	} else if (st0_tag == TAG_Empty) {
-		/* Empty register (stack underflow) */
-		EXCEPTION(EX_StackUnder);
-		if (control_word & CW_Invalid) {
-			/* The masked response */
-			/* Put out the QNaN indefinite */
-			RE_ENTRANT_CHECK_OFF;
-			FPU_access_ok(dfloat, 8);
-			FPU_put_user(0, (unsigned long __user *)dfloat);
-			FPU_put_user(0xfff80000,
-				     1 + (unsigned long __user *)dfloat);
-			RE_ENTRANT_CHECK_ON;
-			return 1;
-		} else
-			return 0;
-	}
-	if (getsign(st0_ptr))
-		l[1] |= 0x80000000;
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(dfloat, 8);
-	FPU_put_user(l[0], (unsigned long __user *)dfloat);
-	FPU_put_user(l[1], 1 + (unsigned long __user *)dfloat);
-	RE_ENTRANT_CHECK_ON;
-
-	return 1;
-}
-
-/* Put a float into user memory */
-int FPU_store_single(FPU_REG *st0_ptr, u_char st0_tag, float __user *single)
-{
-	long templ = 0;
-	unsigned long increment = 0;	/* avoid gcc warnings */
-	int precision_loss;
-	int exp;
-	FPU_REG tmp;
-
-	if (st0_tag == TAG_Valid) {
-
-		reg_copy(st0_ptr, &tmp);
-		exp = exponent(&tmp);
-
-		if (exp < SINGLE_Emin) {
-			addexponent(&tmp, -SINGLE_Emin + 23);	/* largest exp to be 22 */
-
-		      denormal_arg:
-
-			if ((precision_loss = FPU_round_to_int(&tmp, st0_tag))) {
-#ifdef PECULIAR_486
-				/* Did it round to a non-denormal ? */
-				/* This behaviour might be regarded as peculiar, it appears
-				   that the 80486 rounds to the dest precision, then
-				   converts to decide underflow. */
-				if (!((tmp.sigl == 0x00800000) &&
-				      ((st0_ptr->sigh & 0x000000ff)
-				       || st0_ptr->sigl)))
-#endif /* PECULIAR_486 */
-				{
-					EXCEPTION(EX_Underflow);
-					/* This is a special case: see sec 16.2.5.1 of
-					   the 80486 book */
-					if (!(control_word & CW_Underflow))
-						return 0;
-				}
-				EXCEPTION(precision_loss);
-				if (!(control_word & CW_Precision))
-					return 0;
-			}
-			templ = tmp.sigl;
-		} else {
-			if (tmp.sigl | (tmp.sigh & 0x000000ff)) {
-				unsigned long sigh = tmp.sigh;
-				unsigned long sigl = tmp.sigl;
-
-				precision_loss = 1;
-				switch (control_word & CW_RC) {
-				case RC_RND:
-					increment = ((sigh & 0xff) > 0x80)	/* more than half */
-					    ||(((sigh & 0xff) == 0x80) && sigl)	/* more than half */
-					    ||((sigh & 0x180) == 0x180);	/* round to even */
-					break;
-				case RC_DOWN:	/* towards -infinity */
-					increment = signpositive(&tmp)
-					    ? 0 : (sigl | (sigh & 0xff));
-					break;
-				case RC_UP:	/* towards +infinity */
-					increment = signpositive(&tmp)
-					    ? (sigl | (sigh & 0xff)) : 0;
-					break;
-				case RC_CHOP:
-					increment = 0;
-					break;
-				}
-
-				/* Truncate part of the mantissa */
-				tmp.sigl = 0;
-
-				if (increment) {
-					if (sigh >= 0xffffff00) {
-						/* The sigh part overflows */
-						tmp.sigh = 0x80000000;
-						exp++;
-						if (exp >= EXP_OVER)
-							goto overflow;
-					} else {
-						tmp.sigh &= 0xffffff00;
-						tmp.sigh += 0x100;
-					}
-				} else {
-					tmp.sigh &= 0xffffff00;	/* Finish the truncation */
-				}
-			} else
-				precision_loss = 0;
-
-			templ = (tmp.sigh >> 8) & 0x007fffff;
-
-			if (exp > SINGLE_Emax) {
-			      overflow:
-				EXCEPTION(EX_Overflow);
-				if (!(control_word & CW_Overflow))
-					return 0;
-				set_precision_flag_up();
-				if (!(control_word & CW_Precision))
-					return 0;
-
-				/* This is a special case: see sec 16.2.5.1 of the 80486 book. */
-				/* Masked response is overflow to infinity. */
-				templ = 0x7f800000;
-			} else {
-				if (precision_loss) {
-					if (increment)
-						set_precision_flag_up();
-					else
-						set_precision_flag_down();
-				}
-				/* Add the exponent */
-				templ |= ((exp + SINGLE_Ebias) & 0xff) << 23;
-			}
-		}
-	} else if (st0_tag == TAG_Zero) {
-		templ = 0;
-	} else if (st0_tag == TAG_Special) {
-		st0_tag = FPU_Special(st0_ptr);
-		if (st0_tag == TW_Denormal) {
-			reg_copy(st0_ptr, &tmp);
-
-			/* A denormal will always underflow. */
-#ifndef PECULIAR_486
-			/* An 80486 is supposed to be able to generate
-			   a denormal exception here, but... */
-			/* Underflow has priority. */
-			if (control_word & CW_Underflow)
-				denormal_operand();
-#endif /* PECULIAR_486 */
-			goto denormal_arg;
-		} else if (st0_tag == TW_Infinity) {
-			templ = 0x7f800000;
-		} else if (st0_tag == TW_NaN) {
-			/* Is it really a NaN ? */
-			if ((exponent(st0_ptr) == EXP_OVER)
-			    && (st0_ptr->sigh & 0x80000000)) {
-				/* See if we can get a valid NaN from the FPU_REG */
-				templ = st0_ptr->sigh >> 8;
-				if (!(st0_ptr->sigh & 0x40000000)) {
-					/* It is a signalling NaN */
-					EXCEPTION(EX_Invalid);
-					if (!(control_word & CW_Invalid))
-						return 0;
-					templ |= (0x40000000 >> 8);
-				}
-				templ |= 0x7f800000;
-			} else {
-				/* It is an unsupported data type */
-				EXCEPTION(EX_Invalid);
-				if (!(control_word & CW_Invalid))
-					return 0;
-				templ = 0xffc00000;
-			}
-		}
-#ifdef PARANOID
-		else {
-			EXCEPTION(EX_INTERNAL | 0x164);
-			return 0;
-		}
-#endif
-	} else if (st0_tag == TAG_Empty) {
-		/* Empty register (stack underflow) */
-		EXCEPTION(EX_StackUnder);
-		if (control_word & EX_Invalid) {
-			/* The masked response */
-			/* Put out the QNaN indefinite */
-			RE_ENTRANT_CHECK_OFF;
-			FPU_access_ok(single, 4);
-			FPU_put_user(0xffc00000,
-				     (unsigned long __user *)single);
-			RE_ENTRANT_CHECK_ON;
-			return 1;
-		} else
-			return 0;
-	}
-#ifdef PARANOID
-	else {
-		EXCEPTION(EX_INTERNAL | 0x163);
-		return 0;
-	}
-#endif
-	if (getsign(st0_ptr))
-		templ |= 0x80000000;
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(single, 4);
-	FPU_put_user(templ, (unsigned long __user *)single);
-	RE_ENTRANT_CHECK_ON;
-
-	return 1;
-}
-
-/* Put a long long into user memory */
-int FPU_store_int64(FPU_REG *st0_ptr, u_char st0_tag, long long __user *d)
-{
-	FPU_REG t;
-	long long tll;
-	int precision_loss;
-
-	if (st0_tag == TAG_Empty) {
-		/* Empty register (stack underflow) */
-		EXCEPTION(EX_StackUnder);
-		goto invalid_operand;
-	} else if (st0_tag == TAG_Special) {
-		st0_tag = FPU_Special(st0_ptr);
-		if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
-			EXCEPTION(EX_Invalid);
-			goto invalid_operand;
-		}
-	}
-
-	reg_copy(st0_ptr, &t);
-	precision_loss = FPU_round_to_int(&t, st0_tag);
-	((long *)&tll)[0] = t.sigl;
-	((long *)&tll)[1] = t.sigh;
-	if ((precision_loss == 1) ||
-	    ((t.sigh & 0x80000000) &&
-	     !((t.sigh == 0x80000000) && (t.sigl == 0) && signnegative(&t)))) {
-		EXCEPTION(EX_Invalid);
-		/* This is a special case: see sec 16.2.5.1 of the 80486 book */
-	      invalid_operand:
-		if (control_word & EX_Invalid) {
-			/* Produce something like QNaN "indefinite" */
-			tll = 0x8000000000000000LL;
-		} else
-			return 0;
-	} else {
-		if (precision_loss)
-			set_precision_flag(precision_loss);
-		if (signnegative(&t))
-			tll = -tll;
-	}
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(d, 8);
-	if (copy_to_user(d, &tll, 8))
-		FPU_abort;
-	RE_ENTRANT_CHECK_ON;
-
-	return 1;
-}
-
-/* Put a long into user memory */
-int FPU_store_int32(FPU_REG *st0_ptr, u_char st0_tag, long __user *d)
-{
-	FPU_REG t;
-	int precision_loss;
-
-	if (st0_tag == TAG_Empty) {
-		/* Empty register (stack underflow) */
-		EXCEPTION(EX_StackUnder);
-		goto invalid_operand;
-	} else if (st0_tag == TAG_Special) {
-		st0_tag = FPU_Special(st0_ptr);
-		if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
-			EXCEPTION(EX_Invalid);
-			goto invalid_operand;
-		}
-	}
-
-	reg_copy(st0_ptr, &t);
-	precision_loss = FPU_round_to_int(&t, st0_tag);
-	if (t.sigh ||
-	    ((t.sigl & 0x80000000) &&
-	     !((t.sigl == 0x80000000) && signnegative(&t)))) {
-		EXCEPTION(EX_Invalid);
-		/* This is a special case: see sec 16.2.5.1 of the 80486 book */
-	      invalid_operand:
-		if (control_word & EX_Invalid) {
-			/* Produce something like QNaN "indefinite" */
-			t.sigl = 0x80000000;
-		} else
-			return 0;
-	} else {
-		if (precision_loss)
-			set_precision_flag(precision_loss);
-		if (signnegative(&t))
-			t.sigl = -(long)t.sigl;
-	}
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(d, 4);
-	FPU_put_user(t.sigl, (unsigned long __user *)d);
-	RE_ENTRANT_CHECK_ON;
-
-	return 1;
-}
-
-/* Put a short into user memory */
-int FPU_store_int16(FPU_REG *st0_ptr, u_char st0_tag, short __user *d)
-{
-	FPU_REG t;
-	int precision_loss;
-
-	if (st0_tag == TAG_Empty) {
-		/* Empty register (stack underflow) */
-		EXCEPTION(EX_StackUnder);
-		goto invalid_operand;
-	} else if (st0_tag == TAG_Special) {
-		st0_tag = FPU_Special(st0_ptr);
-		if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
-			EXCEPTION(EX_Invalid);
-			goto invalid_operand;
-		}
-	}
-
-	reg_copy(st0_ptr, &t);
-	precision_loss = FPU_round_to_int(&t, st0_tag);
-	if (t.sigh ||
-	    ((t.sigl & 0xffff8000) &&
-	     !((t.sigl == 0x8000) && signnegative(&t)))) {
-		EXCEPTION(EX_Invalid);
-		/* This is a special case: see sec 16.2.5.1 of the 80486 book */
-	      invalid_operand:
-		if (control_word & EX_Invalid) {
-			/* Produce something like QNaN "indefinite" */
-			t.sigl = 0x8000;
-		} else
-			return 0;
-	} else {
-		if (precision_loss)
-			set_precision_flag(precision_loss);
-		if (signnegative(&t))
-			t.sigl = -t.sigl;
-	}
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(d, 2);
-	FPU_put_user((short)t.sigl, d);
-	RE_ENTRANT_CHECK_ON;
-
-	return 1;
-}
-
-/* Put a packed bcd array into user memory */
-int FPU_store_bcd(FPU_REG *st0_ptr, u_char st0_tag, u_char __user *d)
-{
-	FPU_REG t;
-	unsigned long long ll;
-	u_char b;
-	int i, precision_loss;
-	u_char sign = (getsign(st0_ptr) == SIGN_NEG) ? 0x80 : 0;
-
-	if (st0_tag == TAG_Empty) {
-		/* Empty register (stack underflow) */
-		EXCEPTION(EX_StackUnder);
-		goto invalid_operand;
-	} else if (st0_tag == TAG_Special) {
-		st0_tag = FPU_Special(st0_ptr);
-		if ((st0_tag == TW_Infinity) || (st0_tag == TW_NaN)) {
-			EXCEPTION(EX_Invalid);
-			goto invalid_operand;
-		}
-	}
-
-	reg_copy(st0_ptr, &t);
-	precision_loss = FPU_round_to_int(&t, st0_tag);
-	ll = significand(&t);
-
-	/* Check for overflow, by comparing with 999999999999999999 decimal. */
-	if ((t.sigh > 0x0de0b6b3) ||
-	    ((t.sigh == 0x0de0b6b3) && (t.sigl > 0xa763ffff))) {
-		EXCEPTION(EX_Invalid);
-		/* This is a special case: see sec 16.2.5.1 of the 80486 book */
-	      invalid_operand:
-		if (control_word & CW_Invalid) {
-			/* Produce the QNaN "indefinite" */
-			RE_ENTRANT_CHECK_OFF;
-			FPU_access_ok(d, 10);
-			for (i = 0; i < 7; i++)
-				FPU_put_user(0, d + i);	/* These bytes "undefined" */
-			FPU_put_user(0xc0, d + 7);	/* This byte "undefined" */
-			FPU_put_user(0xff, d + 8);
-			FPU_put_user(0xff, d + 9);
-			RE_ENTRANT_CHECK_ON;
-			return 1;
-		} else
-			return 0;
-	} else if (precision_loss) {
-		/* Precision loss doesn't stop the data transfer */
-		set_precision_flag(precision_loss);
-	}
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(d, 10);
-	RE_ENTRANT_CHECK_ON;
-	for (i = 0; i < 9; i++) {
-		b = FPU_div_small(&ll, 10);
-		b |= (FPU_div_small(&ll, 10)) << 4;
-		RE_ENTRANT_CHECK_OFF;
-		FPU_put_user(b, d + i);
-		RE_ENTRANT_CHECK_ON;
-	}
-	RE_ENTRANT_CHECK_OFF;
-	FPU_put_user(sign, d + 9);
-	RE_ENTRANT_CHECK_ON;
-
-	return 1;
-}
-
-/*===========================================================================*/
-
-/* r gets mangled such that sig is int, sign: 
-   it is NOT normalized */
-/* The return value (in eax) is zero if the result is exact,
-   if bits are changed due to rounding, truncation, etc, then
-   a non-zero value is returned */
-/* Overflow is signaled by a non-zero return value (in eax).
-   In the case of overflow, the returned significand always has the
-   largest possible value */
-int FPU_round_to_int(FPU_REG *r, u_char tag)
-{
-	u_char very_big;
-	unsigned eax;
-
-	if (tag == TAG_Zero) {
-		/* Make sure that zero is returned */
-		significand(r) = 0;
-		return 0;	/* o.k. */
-	}
-
-	if (exponent(r) > 63) {
-		r->sigl = r->sigh = ~0;	/* The largest representable number */
-		return 1;	/* overflow */
-	}
-
-	eax = FPU_shrxs(&r->sigl, 63 - exponent(r));
-	very_big = !(~(r->sigh) | ~(r->sigl));	/* test for 0xfff...fff */
-#define	half_or_more	(eax & 0x80000000)
-#define	frac_part	(eax)
-#define more_than_half  ((eax & 0x80000001) == 0x80000001)
-	switch (control_word & CW_RC) {
-	case RC_RND:
-		if (more_than_half	/* nearest */
-		    || (half_or_more && (r->sigl & 1))) {	/* odd -> even */
-			if (very_big)
-				return 1;	/* overflow */
-			significand(r)++;
-			return PRECISION_LOST_UP;
-		}
-		break;
-	case RC_DOWN:
-		if (frac_part && getsign(r)) {
-			if (very_big)
-				return 1;	/* overflow */
-			significand(r)++;
-			return PRECISION_LOST_UP;
-		}
-		break;
-	case RC_UP:
-		if (frac_part && !getsign(r)) {
-			if (very_big)
-				return 1;	/* overflow */
-			significand(r)++;
-			return PRECISION_LOST_UP;
-		}
-		break;
-	case RC_CHOP:
-		break;
-	}
-
-	return eax ? PRECISION_LOST_DOWN : 0;
-
-}
-
-/*===========================================================================*/
-
-u_char __user *fldenv(fpu_addr_modes addr_modes, u_char __user *s)
-{
-	unsigned short tag_word = 0;
-	u_char tag;
-	int i;
-
-	if ((addr_modes.default_mode == VM86) ||
-	    ((addr_modes.default_mode == PM16)
-	     ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) {
-		RE_ENTRANT_CHECK_OFF;
-		FPU_access_ok(s, 0x0e);
-		FPU_get_user(control_word, (unsigned short __user *)s);
-		FPU_get_user(partial_status, (unsigned short __user *)(s + 2));
-		FPU_get_user(tag_word, (unsigned short __user *)(s + 4));
-		FPU_get_user(instruction_address.offset,
-			     (unsigned short __user *)(s + 6));
-		FPU_get_user(instruction_address.selector,
-			     (unsigned short __user *)(s + 8));
-		FPU_get_user(operand_address.offset,
-			     (unsigned short __user *)(s + 0x0a));
-		FPU_get_user(operand_address.selector,
-			     (unsigned short __user *)(s + 0x0c));
-		RE_ENTRANT_CHECK_ON;
-		s += 0x0e;
-		if (addr_modes.default_mode == VM86) {
-			instruction_address.offset
-			    += (instruction_address.selector & 0xf000) << 4;
-			operand_address.offset +=
-			    (operand_address.selector & 0xf000) << 4;
-		}
-	} else {
-		RE_ENTRANT_CHECK_OFF;
-		FPU_access_ok(s, 0x1c);
-		FPU_get_user(control_word, (unsigned short __user *)s);
-		FPU_get_user(partial_status, (unsigned short __user *)(s + 4));
-		FPU_get_user(tag_word, (unsigned short __user *)(s + 8));
-		FPU_get_user(instruction_address.offset,
-			     (unsigned long __user *)(s + 0x0c));
-		FPU_get_user(instruction_address.selector,
-			     (unsigned short __user *)(s + 0x10));
-		FPU_get_user(instruction_address.opcode,
-			     (unsigned short __user *)(s + 0x12));
-		FPU_get_user(operand_address.offset,
-			     (unsigned long __user *)(s + 0x14));
-		FPU_get_user(operand_address.selector,
-			     (unsigned long __user *)(s + 0x18));
-		RE_ENTRANT_CHECK_ON;
-		s += 0x1c;
-	}
-
-#ifdef PECULIAR_486
-	control_word &= ~0xe080;
-#endif /* PECULIAR_486 */
-
-	top = (partial_status >> SW_Top_Shift) & 7;
-
-	if (partial_status & ~control_word & CW_Exceptions)
-		partial_status |= (SW_Summary | SW_Backward);
-	else
-		partial_status &= ~(SW_Summary | SW_Backward);
-
-	for (i = 0; i < 8; i++) {
-		tag = tag_word & 3;
-		tag_word >>= 2;
-
-		if (tag == TAG_Empty)
-			/* New tag is empty.  Accept it */
-			FPU_settag(i, TAG_Empty);
-		else if (FPU_gettag(i) == TAG_Empty) {
-			/* Old tag is empty and new tag is not empty.  New tag is determined
-			   by old reg contents */
-			if (exponent(&fpu_register(i)) == -EXTENDED_Ebias) {
-				if (!
-				    (fpu_register(i).sigl | fpu_register(i).
-				     sigh))
-					FPU_settag(i, TAG_Zero);
-				else
-					FPU_settag(i, TAG_Special);
-			} else if (exponent(&fpu_register(i)) ==
-				   0x7fff - EXTENDED_Ebias) {
-				FPU_settag(i, TAG_Special);
-			} else if (fpu_register(i).sigh & 0x80000000)
-				FPU_settag(i, TAG_Valid);
-			else
-				FPU_settag(i, TAG_Special);	/* An Un-normal */
-		}
-		/* Else old tag is not empty and new tag is not empty.  Old tag
-		   remains correct */
-	}
-
-	return s;
-}
-
-void FPU_frstor(fpu_addr_modes addr_modes, u_char __user *data_address)
-{
-	int i, regnr;
-	u_char __user *s = fldenv(addr_modes, data_address);
-	int offset = (top & 7) * 10, other = 80 - offset;
-
-	/* Copy all registers in stack order. */
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(s, 80);
-	FPU_copy_from_user(register_base + offset, s, other);
-	if (offset)
-		FPU_copy_from_user(register_base, s + other, offset);
-	RE_ENTRANT_CHECK_ON;
-
-	for (i = 0; i < 8; i++) {
-		regnr = (i + top) & 7;
-		if (FPU_gettag(regnr) != TAG_Empty)
-			/* The loaded data over-rides all other cases. */
-			FPU_settag(regnr, FPU_tagof(&st(i)));
-	}
-
-}
-
-u_char __user *fstenv(fpu_addr_modes addr_modes, u_char __user *d)
-{
-	if ((addr_modes.default_mode == VM86) ||
-	    ((addr_modes.default_mode == PM16)
-	     ^ (addr_modes.override.operand_size == OP_SIZE_PREFIX))) {
-		RE_ENTRANT_CHECK_OFF;
-		FPU_access_ok(d, 14);
-#ifdef PECULIAR_486
-		FPU_put_user(control_word & ~0xe080, (unsigned long __user *)d);
-#else
-		FPU_put_user(control_word, (unsigned short __user *)d);
-#endif /* PECULIAR_486 */
-		FPU_put_user(status_word(), (unsigned short __user *)(d + 2));
-		FPU_put_user(fpu_tag_word, (unsigned short __user *)(d + 4));
-		FPU_put_user(instruction_address.offset,
-			     (unsigned short __user *)(d + 6));
-		FPU_put_user(operand_address.offset,
-			     (unsigned short __user *)(d + 0x0a));
-		if (addr_modes.default_mode == VM86) {
-			FPU_put_user((instruction_address.
-				      offset & 0xf0000) >> 4,
-				     (unsigned short __user *)(d + 8));
-			FPU_put_user((operand_address.offset & 0xf0000) >> 4,
-				     (unsigned short __user *)(d + 0x0c));
-		} else {
-			FPU_put_user(instruction_address.selector,
-				     (unsigned short __user *)(d + 8));
-			FPU_put_user(operand_address.selector,
-				     (unsigned short __user *)(d + 0x0c));
-		}
-		RE_ENTRANT_CHECK_ON;
-		d += 0x0e;
-	} else {
-		RE_ENTRANT_CHECK_OFF;
-		FPU_access_ok(d, 7 * 4);
-#ifdef PECULIAR_486
-		control_word &= ~0xe080;
-		/* An 80486 sets nearly all of the reserved bits to 1. */
-		control_word |= 0xffff0040;
-		partial_status = status_word() | 0xffff0000;
-		fpu_tag_word |= 0xffff0000;
-		I387->soft.fcs &= ~0xf8000000;
-		I387->soft.fos |= 0xffff0000;
-#endif /* PECULIAR_486 */
-		if (__copy_to_user(d, &control_word, 7 * 4))
-			FPU_abort;
-		RE_ENTRANT_CHECK_ON;
-		d += 0x1c;
-	}
-
-	control_word |= CW_Exceptions;
-	partial_status &= ~(SW_Summary | SW_Backward);
-
-	return d;
-}
-
-void fsave(fpu_addr_modes addr_modes, u_char __user *data_address)
-{
-	u_char __user *d;
-	int offset = (top & 7) * 10, other = 80 - offset;
-
-	d = fstenv(addr_modes, data_address);
-
-	RE_ENTRANT_CHECK_OFF;
-	FPU_access_ok(d, 80);
-
-	/* Copy all registers in stack order. */
-	if (__copy_to_user(d, register_base + offset, other))
-		FPU_abort;
-	if (offset)
-		if (__copy_to_user(d + other, register_base, offset))
-			FPU_abort;
-	RE_ENTRANT_CHECK_ON;
-
-	finit();
-}
-
-/*===========================================================================*/
diff --git a/arch/x86/math-emu/reg_mul.c b/arch/x86/math-emu/reg_mul.c
deleted file mode 100644
index d69618572f14d..0000000000000
--- a/arch/x86/math-emu/reg_mul.c
+++ /dev/null
@@ -1,116 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*---------------------------------------------------------------------------+
- |  reg_mul.c                                                                |
- |                                                                           |
- | Multiply one FPU_REG by another, put the result in a destination FPU_REG. |
- |                                                                           |
- | Copyright (C) 1992,1993,1997                                              |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- | Returns the tag of the result if no exceptions or errors occurred.        |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | The destination may be any FPU_REG, including one of the source FPU_REGs. |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-#include "exception.h"
-#include "reg_constant.h"
-#include "fpu_system.h"
-
-/*
-  Multiply two registers to give a register result.
-  The sources are st(deststnr) and (b,tagb,signb).
-  The destination is st(deststnr).
-  */
-/* This routine must be called with non-empty source registers */
-int FPU_mul(FPU_REG const *b, u_char tagb, int deststnr, int control_w)
-{
-	FPU_REG *a = &st(deststnr);
-	FPU_REG *dest = a;
-	u_char taga = FPU_gettagi(deststnr);
-	u_char saved_sign = getsign(dest);
-	u_char sign = (getsign(a) ^ getsign(b));
-	int tag;
-
-	if (!(taga | tagb)) {
-		/* Both regs Valid, this should be the most common case. */
-
-		tag =
-		    FPU_u_mul(a, b, dest, control_w, sign,
-			      exponent(a) + exponent(b));
-		if (tag < 0) {
-			setsign(dest, saved_sign);
-			return tag;
-		}
-		FPU_settagi(deststnr, tag);
-		return tag;
-	}
-
-	if (taga == TAG_Special)
-		taga = FPU_Special(a);
-	if (tagb == TAG_Special)
-		tagb = FPU_Special(b);
-
-	if (((taga == TAG_Valid) && (tagb == TW_Denormal))
-	    || ((taga == TW_Denormal) && (tagb == TAG_Valid))
-	    || ((taga == TW_Denormal) && (tagb == TW_Denormal))) {
-		FPU_REG x, y;
-		if (denormal_operand() < 0)
-			return FPU_Exception;
-
-		FPU_to_exp16(a, &x);
-		FPU_to_exp16(b, &y);
-		tag = FPU_u_mul(&x, &y, dest, control_w, sign,
-				exponent16(&x) + exponent16(&y));
-		if (tag < 0) {
-			setsign(dest, saved_sign);
-			return tag;
-		}
-		FPU_settagi(deststnr, tag);
-		return tag;
-	} else if ((taga <= TW_Denormal) && (tagb <= TW_Denormal)) {
-		if (((tagb == TW_Denormal) || (taga == TW_Denormal))
-		    && (denormal_operand() < 0))
-			return FPU_Exception;
-
-		/* Must have either both arguments == zero, or
-		   one valid and the other zero.
-		   The result is therefore zero. */
-		FPU_copy_to_regi(&CONST_Z, TAG_Zero, deststnr);
-		/* The 80486 book says that the answer is +0, but a real
-		   80486 behaves this way.
-		   IEEE-754 apparently says it should be this way. */
-		setsign(dest, sign);
-		return TAG_Zero;
-	}
-	/* Must have infinities, NaNs, etc */
-	else if ((taga == TW_NaN) || (tagb == TW_NaN)) {
-		return real_2op_NaN(b, tagb, deststnr, &st(0));
-	} else if (((taga == TW_Infinity) && (tagb == TAG_Zero))
-		   || ((tagb == TW_Infinity) && (taga == TAG_Zero))) {
-		return arith_invalid(deststnr);	/* Zero*Infinity is invalid */
-	} else if (((taga == TW_Denormal) || (tagb == TW_Denormal))
-		   && (denormal_operand() < 0)) {
-		return FPU_Exception;
-	} else if (taga == TW_Infinity) {
-		FPU_copy_to_regi(a, TAG_Special, deststnr);
-		setsign(dest, sign);
-		return TAG_Special;
-	} else if (tagb == TW_Infinity) {
-		FPU_copy_to_regi(b, TAG_Special, deststnr);
-		setsign(dest, sign);
-		return TAG_Special;
-	}
-#ifdef PARANOID
-	else {
-		EXCEPTION(EX_INTERNAL | 0x102);
-		return FPU_Exception;
-	}
-#endif /* PARANOID */
-
-	return 0;
-}
diff --git a/arch/x86/math-emu/reg_norm.S b/arch/x86/math-emu/reg_norm.S
deleted file mode 100644
index 594936eeed67a..0000000000000
--- a/arch/x86/math-emu/reg_norm.S
+++ /dev/null
@@ -1,150 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  reg_norm.S                                                               |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1995,1997                                    |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@suburbia.net               |
- |                                                                           |
- | Normalize the value in a FPU_REG.                                         |
- |                                                                           |
- | Call from C as:                                                           |
- |    int FPU_normalize(FPU_REG *n)                                          |
- |                                                                           |
- |    int FPU_normalize_nuo(FPU_REG *n)                                      |
- |                                                                           |
- |    Return value is the tag of the answer, or-ed with FPU_Exception if     |
- |    one was raised, or -1 on internal error.                               |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-
-.text
-SYM_FUNC_START(FPU_normalize)
-	pushl	%ebp
-	movl	%esp,%ebp
-	pushl	%ebx
-
-	movl	PARAM1,%ebx
-
-	movl	SIGH(%ebx),%edx
-	movl	SIGL(%ebx),%eax
-
-	orl	%edx,%edx	/* ms bits */
-	js	L_done		/* Already normalized */
-	jnz	L_shift_1	/* Shift left 1 - 31 bits */
-
-	orl	%eax,%eax
-	jz	L_zero		/* The contents are zero */
-
-	movl	%eax,%edx
-	xorl	%eax,%eax
-	subw	$32,EXP(%ebx)	/* This can cause an underflow */
-
-/* We need to shift left by 1 - 31 bits */
-L_shift_1:
-	bsrl	%edx,%ecx	/* get the required shift in %ecx */
-	subl	$31,%ecx
-	negl	%ecx
-	shld	%cl,%eax,%edx
-	shl	%cl,%eax
-	subw	%cx,EXP(%ebx)	/* This can cause an underflow */
-
-	movl	%edx,SIGH(%ebx)
-	movl	%eax,SIGL(%ebx)
-
-L_done:
-	cmpw	EXP_OVER,EXP(%ebx)
-	jge	L_overflow
-
-	cmpw	EXP_UNDER,EXP(%ebx)
-	jle	L_underflow
-
-L_exit_valid:
-	movl	TAG_Valid,%eax
-
-	/* Convert the exponent to 80x87 form. */
-	addw	EXTENDED_Ebias,EXP(%ebx)
-	andw	$0x7fff,EXP(%ebx)
-
-L_exit:
-	popl	%ebx
-	leave
-	RET
-
-
-L_zero:
-	movw	$0,EXP(%ebx)
-	movl	TAG_Zero,%eax
-	jmp	L_exit
-
-L_underflow:
-	/* Convert the exponent to 80x87 form. */
-	addw	EXTENDED_Ebias,EXP(%ebx)
-	push	%ebx
-	call	arith_underflow
-	pop	%ebx
-	jmp	L_exit
-
-L_overflow:
-	/* Convert the exponent to 80x87 form. */
-	addw	EXTENDED_Ebias,EXP(%ebx)
-	push	%ebx
-	call	arith_overflow
-	pop	%ebx
-	jmp	L_exit
-SYM_FUNC_END(FPU_normalize)
-
-
-
-/* Normalise without reporting underflow or overflow */
-SYM_FUNC_START(FPU_normalize_nuo)
-	pushl	%ebp
-	movl	%esp,%ebp
-	pushl	%ebx
-
-	movl	PARAM1,%ebx
-
-	movl	SIGH(%ebx),%edx
-	movl	SIGL(%ebx),%eax
-
-	orl	%edx,%edx	/* ms bits */
-	js	L_exit_nuo_valid	/* Already normalized */
-	jnz	L_nuo_shift_1	/* Shift left 1 - 31 bits */
-
-	orl	%eax,%eax
-	jz	L_exit_nuo_zero		/* The contents are zero */
-
-	movl	%eax,%edx
-	xorl	%eax,%eax
-	subw	$32,EXP(%ebx)	/* This can cause an underflow */
-
-/* We need to shift left by 1 - 31 bits */
-L_nuo_shift_1:
-	bsrl	%edx,%ecx	/* get the required shift in %ecx */
-	subl	$31,%ecx
-	negl	%ecx
-	shld	%cl,%eax,%edx
-	shl	%cl,%eax
-	subw	%cx,EXP(%ebx)	/* This can cause an underflow */
-
-	movl	%edx,SIGH(%ebx)
-	movl	%eax,SIGL(%ebx)
-
-L_exit_nuo_valid:
-	movl	TAG_Valid,%eax
-
-	popl	%ebx
-	leave
-	RET
-
-L_exit_nuo_zero:
-	movl	TAG_Zero,%eax
-	movw	EXP_UNDER,EXP(%ebx)
-
-	popl	%ebx
-	leave
-	RET
-SYM_FUNC_END(FPU_normalize_nuo)
diff --git a/arch/x86/math-emu/reg_round.S b/arch/x86/math-emu/reg_round.S
deleted file mode 100644
index 0bb2a092161af..0000000000000
--- a/arch/x86/math-emu/reg_round.S
+++ /dev/null
@@ -1,711 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file "reg_round.S"
-/*---------------------------------------------------------------------------+
- |  reg_round.S                                                              |
- |                                                                           |
- | Rounding/truncation/etc for FPU basic arithmetic functions.               |
- |                                                                           |
- | Copyright (C) 1993,1995,1997                                              |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@suburbia.net               |
- |                                                                           |
- | This code has four possible entry points.                                 |
- | The following must be entered by a jmp instruction:                       |
- |   fpu_reg_round, fpu_reg_round_sqrt, and fpu_Arith_exit.                  |
- |                                                                           |
- | The FPU_round entry point is intended to be used by C code.               |
- | From C, call as:                                                          |
- |  int FPU_round(FPU_REG *arg, unsigned int extent, unsigned int control_w) |
- |                                                                           |
- |    Return value is the tag of the answer, or-ed with FPU_Exception if     |
- |    one was raised, or -1 on internal error.                               |
- |                                                                           |
- | For correct "up" and "down" rounding, the argument must have the correct  |
- | sign.                                                                     |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Four entry points.                                                        |
- |                                                                           |
- | Needed by both the fpu_reg_round and fpu_reg_round_sqrt entry points:     |
- |  %eax:%ebx  64 bit significand                                            |
- |  %edx       32 bit extension of the significand                           |
- |  %edi       pointer to an FPU_REG for the result to be stored             |
- |  stack      calling function must have set up a C stack frame and         |
- |             pushed %esi, %edi, and %ebx                                   |
- |                                                                           |
- | Needed just for the fpu_reg_round_sqrt entry point:                       |
- |  %cx  A control word in the same format as the FPU control word.          |
- | Otherwise, PARAM4 must give such a value.                                 |
- |                                                                           |
- |                                                                           |
- | The significand and its extension are assumed to be exact in the          |
- | following sense:                                                          |
- |   If the significand by itself is the exact result then the significand   |
- |   extension (%edx) must contain 0, otherwise the significand extension    |
- |   must be non-zero.                                                       |
- |   If the significand extension is non-zero then the significand is        |
- |   smaller than the magnitude of the correct exact result by an amount     |
- |   greater than zero and less than one ls bit of the significand.          |
- |   The significand extension is only required to have three possible       |
- |   non-zero values:                                                        |
- |       less than 0x80000000  <=> the significand is less than 1/2 an ls    |
- |                                 bit smaller than the magnitude of the     |
- |                                 true exact result.                        |
- |         exactly 0x80000000  <=> the significand is exactly 1/2 an ls bit  |
- |                                 smaller than the magnitude of the true    |
- |                                 exact result.                             |
- |    greater than 0x80000000  <=> the significand is more than 1/2 an ls    |
- |                                 bit smaller than the magnitude of the     |
- |                                 true exact result.                        |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- |  The code in this module has become quite complex, but it should handle   |
- |  all of the FPU flags which are set at this stage of the basic arithmetic |
- |  computations.                                                            |
- |  There are a few rare cases where the results are not set identically to  |
- |  a real FPU. These require a bit more thought because at this stage the   |
- |  results of the code here appear to be more consistent...                 |
- |  This may be changed in a future version.                                 |
- +---------------------------------------------------------------------------*/
-
-
-#include "fpu_emu.h"
-#include "exception.h"
-#include "control_w.h"
-
-/* Flags for FPU_bits_lost */
-#define	LOST_DOWN	$1
-#define	LOST_UP		$2
-
-/* Flags for FPU_denormal */
-#define	DENORMAL	$1
-#define	UNMASKED_UNDERFLOW $2
-
-
-#ifndef NON_REENTRANT_FPU
-/*	Make the code re-entrant by putting
-	local storage on the stack: */
-#define FPU_bits_lost	(%esp)
-#define FPU_denormal	1(%esp)
-
-#else
-/*	Not re-entrant, so we can gain speed by putting
-	local storage in a static area: */
-.data
-	.align 4,0
-FPU_bits_lost:
-	.byte	0
-FPU_denormal:
-	.byte	0
-#endif /* NON_REENTRANT_FPU */
-
-
-.text
-.globl fpu_reg_round
-.globl fpu_Arith_exit
-
-/* Entry point when called from C */
-SYM_FUNC_START(FPU_round)
-	pushl	%ebp
-	movl	%esp,%ebp
-	pushl	%esi
-	pushl	%edi
-	pushl	%ebx
-
-	movl	PARAM1,%edi
-	movl	SIGH(%edi),%eax
-	movl	SIGL(%edi),%ebx
-	movl	PARAM2,%edx
-
-fpu_reg_round:			/* Normal entry point */
-	movl	PARAM4,%ecx
-
-#ifndef NON_REENTRANT_FPU
-	pushl	%ebx		/* adjust the stack pointer */
-#endif /* NON_REENTRANT_FPU */ 
-
-#ifdef PARANOID
-/* Cannot use this here yet */
-/*	orl	%eax,%eax */
-/*	jns	L_entry_bugged */
-#endif /* PARANOID */
-
-	cmpw	EXP_UNDER,EXP(%edi)
-	jle	L_Make_denorm			/* The number is a de-normal */
-
-	movb	$0,FPU_denormal			/* 0 -> not a de-normal */
-
-Denorm_done:
-	movb	$0,FPU_bits_lost		/* No bits yet lost in rounding */
-
-	movl	%ecx,%esi
-	andl	CW_PC,%ecx
-	cmpl	PR_64_BITS,%ecx
-	je	LRound_To_64
-
-	cmpl	PR_53_BITS,%ecx
-	je	LRound_To_53
-
-	cmpl	PR_24_BITS,%ecx
-	je	LRound_To_24
-
-#ifdef PECULIAR_486
-/* With the precision control bits set to 01 "(reserved)", a real 80486
-   behaves as if the precision control bits were set to 11 "64 bits" */
-	cmpl	PR_RESERVED_BITS,%ecx
-	je	LRound_To_64
-#ifdef PARANOID
-	jmp	L_bugged_denorm_486
-#endif /* PARANOID */ 
-#else
-#ifdef PARANOID
-	jmp	L_bugged_denorm	/* There is no bug, just a bad control word */
-#endif /* PARANOID */ 
-#endif /* PECULIAR_486 */
-
-
-/* Round etc to 24 bit precision */
-LRound_To_24:
-	movl	%esi,%ecx
-	andl	CW_RC,%ecx
-	cmpl	RC_RND,%ecx
-	je	LRound_nearest_24
-
-	cmpl	RC_CHOP,%ecx
-	je	LCheck_truncate_24
-
-	cmpl	RC_UP,%ecx		/* Towards +infinity */
-	je	LUp_24
-
-	cmpl	RC_DOWN,%ecx		/* Towards -infinity */
-	je	LDown_24
-
-#ifdef PARANOID
-	jmp	L_bugged_round24
-#endif /* PARANOID */ 
-
-LUp_24:
-	cmpb	SIGN_POS,PARAM5
-	jne	LCheck_truncate_24	/* If negative then  up==truncate */
-
-	jmp	LCheck_24_round_up
-
-LDown_24:
-	cmpb	SIGN_POS,PARAM5
-	je	LCheck_truncate_24	/* If positive then  down==truncate */
-
-LCheck_24_round_up:
-	movl	%eax,%ecx
-	andl	$0x000000ff,%ecx
-	orl	%ebx,%ecx
-	orl	%edx,%ecx
-	jnz	LDo_24_round_up
-	jmp	L_Re_normalise
-
-LRound_nearest_24:
-	/* Do rounding of the 24th bit if needed (nearest or even) */
-	movl	%eax,%ecx
-	andl	$0x000000ff,%ecx
-	cmpl	$0x00000080,%ecx
-	jc	LCheck_truncate_24	/* less than half, no increment needed */
-
-	jne	LGreater_Half_24	/* greater than half, increment needed */
-
-	/* Possibly half, we need to check the ls bits */
-	orl	%ebx,%ebx
-	jnz	LGreater_Half_24	/* greater than half, increment needed */
-
-	orl	%edx,%edx
-	jnz	LGreater_Half_24	/* greater than half, increment needed */
-
-	/* Exactly half, increment only if 24th bit is 1 (round to even) */
-	testl	$0x00000100,%eax
-	jz	LDo_truncate_24
-
-LGreater_Half_24:			/* Rounding: increment at the 24th bit */
-LDo_24_round_up:
-	andl	$0xffffff00,%eax	/* Truncate to 24 bits */
-	xorl	%ebx,%ebx
-	movb	LOST_UP,FPU_bits_lost
-	addl	$0x00000100,%eax
-	jmp	LCheck_Round_Overflow
-
-LCheck_truncate_24:
-	movl	%eax,%ecx
-	andl	$0x000000ff,%ecx
-	orl	%ebx,%ecx
-	orl	%edx,%ecx
-	jz	L_Re_normalise		/* No truncation needed */
-
-LDo_truncate_24:
-	andl	$0xffffff00,%eax	/* Truncate to 24 bits */
-	xorl	%ebx,%ebx
-	movb	LOST_DOWN,FPU_bits_lost
-	jmp	L_Re_normalise
-
-
-/* Round etc to 53 bit precision */
-LRound_To_53:
-	movl	%esi,%ecx
-	andl	CW_RC,%ecx
-	cmpl	RC_RND,%ecx
-	je	LRound_nearest_53
-
-	cmpl	RC_CHOP,%ecx
-	je	LCheck_truncate_53
-
-	cmpl	RC_UP,%ecx		/* Towards +infinity */
-	je	LUp_53
-
-	cmpl	RC_DOWN,%ecx		/* Towards -infinity */
-	je	LDown_53
-
-#ifdef PARANOID
-	jmp	L_bugged_round53
-#endif /* PARANOID */ 
-
-LUp_53:
-	cmpb	SIGN_POS,PARAM5
-	jne	LCheck_truncate_53	/* If negative then  up==truncate */
-
-	jmp	LCheck_53_round_up
-
-LDown_53:
-	cmpb	SIGN_POS,PARAM5
-	je	LCheck_truncate_53	/* If positive then  down==truncate */
-
-LCheck_53_round_up:
-	movl	%ebx,%ecx
-	andl	$0x000007ff,%ecx
-	orl	%edx,%ecx
-	jnz	LDo_53_round_up
-	jmp	L_Re_normalise
-
-LRound_nearest_53:
-	/* Do rounding of the 53rd bit if needed (nearest or even) */
-	movl	%ebx,%ecx
-	andl	$0x000007ff,%ecx
-	cmpl	$0x00000400,%ecx
-	jc	LCheck_truncate_53	/* less than half, no increment needed */
-
-	jnz	LGreater_Half_53	/* greater than half, increment needed */
-
-	/* Possibly half, we need to check the ls bits */
-	orl	%edx,%edx
-	jnz	LGreater_Half_53	/* greater than half, increment needed */
-
-	/* Exactly half, increment only if 53rd bit is 1 (round to even) */
-	testl	$0x00000800,%ebx
-	jz	LTruncate_53
-
-LGreater_Half_53:			/* Rounding: increment at the 53rd bit */
-LDo_53_round_up:
-	movb	LOST_UP,FPU_bits_lost
-	andl	$0xfffff800,%ebx	/* Truncate to 53 bits */
-	addl	$0x00000800,%ebx
-	adcl	$0,%eax
-	jmp	LCheck_Round_Overflow
-
-LCheck_truncate_53:
-	movl	%ebx,%ecx
-	andl	$0x000007ff,%ecx
-	orl	%edx,%ecx
-	jz	L_Re_normalise
-
-LTruncate_53:
-	movb	LOST_DOWN,FPU_bits_lost
-	andl	$0xfffff800,%ebx	/* Truncate to 53 bits */
-	jmp	L_Re_normalise
-
-
-/* Round etc to 64 bit precision */
-LRound_To_64:
-	movl	%esi,%ecx
-	andl	CW_RC,%ecx
-	cmpl	RC_RND,%ecx
-	je	LRound_nearest_64
-
-	cmpl	RC_CHOP,%ecx
-	je	LCheck_truncate_64
-
-	cmpl	RC_UP,%ecx		/* Towards +infinity */
-	je	LUp_64
-
-	cmpl	RC_DOWN,%ecx		/* Towards -infinity */
-	je	LDown_64
-
-#ifdef PARANOID
-	jmp	L_bugged_round64
-#endif /* PARANOID */ 
-
-LUp_64:
-	cmpb	SIGN_POS,PARAM5
-	jne	LCheck_truncate_64	/* If negative then  up==truncate */
-
-	orl	%edx,%edx
-	jnz	LDo_64_round_up
-	jmp	L_Re_normalise
-
-LDown_64:
-	cmpb	SIGN_POS,PARAM5
-	je	LCheck_truncate_64	/* If positive then  down==truncate */
-
-	orl	%edx,%edx
-	jnz	LDo_64_round_up
-	jmp	L_Re_normalise
-
-LRound_nearest_64:
-	cmpl	$0x80000000,%edx
-	jc	LCheck_truncate_64
-
-	jne	LDo_64_round_up
-
-	/* Now test for round-to-even */
-	testb	$1,%bl
-	jz	LCheck_truncate_64
-
-LDo_64_round_up:
-	movb	LOST_UP,FPU_bits_lost
-	addl	$1,%ebx
-	adcl	$0,%eax
-
-LCheck_Round_Overflow:
-	jnc	L_Re_normalise
-
-	/* Overflow, adjust the result (significand to 1.0) */
-	rcrl	$1,%eax
-	rcrl	$1,%ebx
-	incw	EXP(%edi)
-	jmp	L_Re_normalise
-
-LCheck_truncate_64:
-	orl	%edx,%edx
-	jz	L_Re_normalise
-
-LTruncate_64:
-	movb	LOST_DOWN,FPU_bits_lost
-
-L_Re_normalise:
-	testb	$0xff,FPU_denormal
-	jnz	Normalise_result
-
-L_Normalised:
-	movl	TAG_Valid,%edx
-
-L_deNormalised:
-	cmpb	LOST_UP,FPU_bits_lost
-	je	L_precision_lost_up
-
-	cmpb	LOST_DOWN,FPU_bits_lost
-	je	L_precision_lost_down
-
-L_no_precision_loss:
-	/* store the result */
-
-L_Store_significand:
-	movl	%eax,SIGH(%edi)
-	movl	%ebx,SIGL(%edi)
-
-	cmpw	EXP_OVER,EXP(%edi)
-	jge	L_overflow
-
-	movl	%edx,%eax
-
-	/* Convert the exponent to 80x87 form. */
-	addw	EXTENDED_Ebias,EXP(%edi)
-	andw	$0x7fff,EXP(%edi)
-
-fpu_reg_round_signed_special_exit:
-
-	cmpb	SIGN_POS,PARAM5
-	je	fpu_reg_round_special_exit
-
-	orw	$0x8000,EXP(%edi)	/* Negative sign for the result. */
-
-fpu_reg_round_special_exit:
-
-#ifndef NON_REENTRANT_FPU
-	popl	%ebx		/* adjust the stack pointer */
-#endif /* NON_REENTRANT_FPU */ 
-
-fpu_Arith_exit:
-	popl	%ebx
-	popl	%edi
-	popl	%esi
-	leave
-	RET
-
-
-/*
- * Set the FPU status flags to represent precision loss due to
- * round-up.
- */
-L_precision_lost_up:
-	push	%edx
-	push	%eax
-	call	set_precision_flag_up
-	popl	%eax
-	popl	%edx
-	jmp	L_no_precision_loss
-
-/*
- * Set the FPU status flags to represent precision loss due to
- * truncation.
- */
-L_precision_lost_down:
-	push	%edx
-	push	%eax
-	call	set_precision_flag_down
-	popl	%eax
-	popl	%edx
-	jmp	L_no_precision_loss
-
-
-/*
- * The number is a denormal (which might get rounded up to a normal)
- * Shift the number right the required number of bits, which will
- * have to be undone later...
- */
-L_Make_denorm:
-	/* The action to be taken depends upon whether the underflow
-	   exception is masked */
-	testb	CW_Underflow,%cl		/* Underflow mask. */
-	jz	Unmasked_underflow		/* Do not make a denormal. */
-
-	movb	DENORMAL,FPU_denormal
-
-	pushl	%ecx		/* Save */
-	movw	EXP_UNDER+1,%cx
-	subw	EXP(%edi),%cx
-
-	cmpw	$64,%cx	/* shrd only works for 0..31 bits */
-	jnc	Denorm_shift_more_than_63
-
-	cmpw	$32,%cx	/* shrd only works for 0..31 bits */
-	jnc	Denorm_shift_more_than_32
-
-/*
- * We got here without jumps by assuming that the most common requirement
- *   is for a small de-normalising shift.
- * Shift by [1..31] bits
- */
-	addw	%cx,EXP(%edi)
-	orl	%edx,%edx	/* extension */
-	setne	%ch		/* Save whether %edx is non-zero */
-	xorl	%edx,%edx
-	shrd	%cl,%ebx,%edx
-	shrd	%cl,%eax,%ebx
-	shr	%cl,%eax
-	orb	%ch,%dl
-	popl	%ecx
-	jmp	Denorm_done
-
-/* Shift by [32..63] bits */
-Denorm_shift_more_than_32:
-	addw	%cx,EXP(%edi)
-	subb	$32,%cl
-	orl	%edx,%edx
-	setne	%ch
-	orb	%ch,%bl
-	xorl	%edx,%edx
-	shrd	%cl,%ebx,%edx
-	shrd	%cl,%eax,%ebx
-	shr	%cl,%eax
-	orl	%edx,%edx		/* test these 32 bits */
-	setne	%cl
-	orb	%ch,%bl
-	orb	%cl,%bl
-	movl	%ebx,%edx
-	movl	%eax,%ebx
-	xorl	%eax,%eax
-	popl	%ecx
-	jmp	Denorm_done
-
-/* Shift by [64..) bits */
-Denorm_shift_more_than_63:
-	cmpw	$64,%cx
-	jne	Denorm_shift_more_than_64
-
-/* Exactly 64 bit shift */
-	addw	%cx,EXP(%edi)
-	xorl	%ecx,%ecx
-	orl	%edx,%edx
-	setne	%cl
-	orl	%ebx,%ebx
-	setne	%ch
-	orb	%ch,%cl
-	orb	%cl,%al
-	movl	%eax,%edx
-	xorl	%eax,%eax
-	xorl	%ebx,%ebx
-	popl	%ecx
-	jmp	Denorm_done
-
-Denorm_shift_more_than_64:
-	movw	EXP_UNDER+1,EXP(%edi)
-/* This is easy, %eax must be non-zero, so.. */
-	movl	$1,%edx
-	xorl	%eax,%eax
-	xorl	%ebx,%ebx
-	popl	%ecx
-	jmp	Denorm_done
-
-
-Unmasked_underflow:
-	movb	UNMASKED_UNDERFLOW,FPU_denormal
-	jmp	Denorm_done
-
-
-/* Undo the de-normalisation. */
-Normalise_result:
-	cmpb	UNMASKED_UNDERFLOW,FPU_denormal
-	je	Signal_underflow
-
-/* The number must be a denormal if we got here. */
-#ifdef PARANOID
-	/* But check it... just in case. */
-	cmpw	EXP_UNDER+1,EXP(%edi)
-	jne	L_norm_bugged
-#endif /* PARANOID */
-
-#ifdef PECULIAR_486
-	/*
-	 * This implements a special feature of 80486 behaviour.
-	 * Underflow will be signaled even if the number is
-	 * not a denormal after rounding.
-	 * This difference occurs only for masked underflow, and not
-	 * in the unmasked case.
-	 * Actual 80486 behaviour differs from this in some circumstances.
-	 */
-	orl	%eax,%eax		/* ms bits */
-	js	LPseudoDenormal		/* Will be masked underflow */
-#else
-	orl	%eax,%eax		/* ms bits */
-	js	L_Normalised		/* No longer a denormal */
-#endif /* PECULIAR_486 */ 
-
-	jnz	LDenormal_adj_exponent
-
-	orl	%ebx,%ebx
-	jz	L_underflow_to_zero	/* The contents are zero */
-
-LDenormal_adj_exponent:
-	decw	EXP(%edi)
-
-LPseudoDenormal:
-	testb	$0xff,FPU_bits_lost	/* bits lost == underflow */
-	movl	TAG_Special,%edx
-	jz	L_deNormalised
-
-	/* There must be a masked underflow */
-	push	%eax
-	pushl	EX_Underflow
-	call	EXCEPTION
-	popl	%eax
-	popl	%eax
-	movl	TAG_Special,%edx
-	jmp	L_deNormalised
-
-
-/*
- * The operations resulted in a number too small to represent.
- * Masked response.
- */
-L_underflow_to_zero:
-	push	%eax
-	call	set_precision_flag_down
-	popl	%eax
-
-	push	%eax
-	pushl	EX_Underflow
-	call	EXCEPTION
-	popl	%eax
-	popl	%eax
-
-/* Reduce the exponent to EXP_UNDER */
-	movw	EXP_UNDER,EXP(%edi)
-	movl	TAG_Zero,%edx
-	jmp	L_Store_significand
-
-
-/* The operations resulted in a number too large to represent. */
-L_overflow:
-	addw	EXTENDED_Ebias,EXP(%edi)	/* Set for unmasked response. */
-	push	%edi
-	call	arith_overflow
-	pop	%edi
-	jmp	fpu_reg_round_signed_special_exit
-
-
-Signal_underflow:
-	/* The number may have been changed to a non-denormal */
-	/* by the rounding operations. */
-	cmpw	EXP_UNDER,EXP(%edi)
-	jle	Do_unmasked_underflow
-
-	jmp	L_Normalised
-
-Do_unmasked_underflow:
-	/* Increase the exponent by the magic number */
-	addw	$(3*(1<<13)),EXP(%edi)
-	push	%eax
-	pushl	EX_Underflow
-	call	EXCEPTION
-	popl	%eax
-	popl	%eax
-	jmp	L_Normalised
-
-
-#ifdef PARANOID
-#ifdef PECULIAR_486
-L_bugged_denorm_486:
-	pushl	EX_INTERNAL|0x236
-	call	EXCEPTION
-	popl	%ebx
-	jmp	L_exception_exit
-#else
-L_bugged_denorm:
-	pushl	EX_INTERNAL|0x230
-	call	EXCEPTION
-	popl	%ebx
-	jmp	L_exception_exit
-#endif /* PECULIAR_486 */ 
-
-L_bugged_round24:
-	pushl	EX_INTERNAL|0x231
-	call	EXCEPTION
-	popl	%ebx
-	jmp	L_exception_exit
-
-L_bugged_round53:
-	pushl	EX_INTERNAL|0x232
-	call	EXCEPTION
-	popl	%ebx
-	jmp	L_exception_exit
-
-L_bugged_round64:
-	pushl	EX_INTERNAL|0x233
-	call	EXCEPTION
-	popl	%ebx
-	jmp	L_exception_exit
-
-L_norm_bugged:
-	pushl	EX_INTERNAL|0x234
-	call	EXCEPTION
-	popl	%ebx
-	jmp	L_exception_exit
-
-L_entry_bugged:
-	pushl	EX_INTERNAL|0x235
-	call	EXCEPTION
-	popl	%ebx
-L_exception_exit:
-	mov	$-1,%eax
-	jmp	fpu_reg_round_special_exit
-#endif /* PARANOID */ 
-
-SYM_FUNC_END(FPU_round)
diff --git a/arch/x86/math-emu/reg_u_add.S b/arch/x86/math-emu/reg_u_add.S
deleted file mode 100644
index 07247287a3af7..0000000000000
--- a/arch/x86/math-emu/reg_u_add.S
+++ /dev/null
@@ -1,169 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file	"reg_u_add.S"
-/*---------------------------------------------------------------------------+
- |  reg_u_add.S                                                              |
- |                                                                           |
- | Add two valid (TAG_Valid) FPU_REG numbers, of the same sign, and put the  |
- |   result in a destination FPU_REG.                                        |
- |                                                                           |
- | Copyright (C) 1992,1993,1995,1997                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- | Call from C as:                                                           |
- |   int  FPU_u_add(FPU_REG *arg1, FPU_REG *arg2, FPU_REG *answ,             |
- |                                                int control_w)             |
- |    Return value is the tag of the answer, or-ed with FPU_Exception if     |
- |    one was raised, or -1 on internal error.                               |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*
- |    Kernel addition routine FPU_u_add(reg *arg1, reg *arg2, reg *answ).
- |    Takes two valid reg f.p. numbers (TAG_Valid), which are
- |    treated as unsigned numbers,
- |    and returns their sum as a TAG_Valid or TAG_Special f.p. number.
- |    The returned number is normalized.
- |    Basic checks are performed if PARANOID is defined.
- */
-
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-
-.text
-SYM_FUNC_START(FPU_u_add)
-	pushl	%ebp
-	movl	%esp,%ebp
-	pushl	%esi
-	pushl	%edi
-	pushl	%ebx
-
-	movl	PARAM1,%esi		/* source 1 */
-	movl	PARAM2,%edi		/* source 2 */
-
-	movl	PARAM6,%ecx
-	movl	%ecx,%edx
-	subl	PARAM7,%ecx			/* exp1 - exp2 */
-	jge	L_arg1_larger
-
-	/* num1 is smaller */
-	movl	SIGL(%esi),%ebx
-	movl	SIGH(%esi),%eax
-
-	movl	%edi,%esi
-	movl	PARAM7,%edx
-	negw	%cx
-	jmp	L_accum_loaded
-
-L_arg1_larger:
-	/* num1 has larger or equal exponent */
-	movl	SIGL(%edi),%ebx
-	movl	SIGH(%edi),%eax
-
-L_accum_loaded:
-	movl	PARAM3,%edi		/* destination */
-	movw	%dx,EXP(%edi)		/* Copy exponent to destination */
-
-	xorl	%edx,%edx		/* clear the extension */
-
-#ifdef PARANOID
-	testl	$0x80000000,%eax
-	je	L_bugged
-
-	testl	$0x80000000,SIGH(%esi)
-	je	L_bugged
-#endif /* PARANOID */
-
-/* The number to be shifted is in %eax:%ebx:%edx */
-	cmpw	$32,%cx		/* shrd only works for 0..31 bits */
-	jnc	L_more_than_31
-
-/* less than 32 bits */
-	shrd	%cl,%ebx,%edx
-	shrd	%cl,%eax,%ebx
-	shr	%cl,%eax
-	jmp	L_shift_done
-
-L_more_than_31:
-	cmpw	$64,%cx
-	jnc	L_more_than_63
-
-	subb	$32,%cl
-	jz	L_exactly_32
-
-	shrd	%cl,%eax,%edx
-	shr	%cl,%eax
-	orl	%ebx,%ebx
-	jz	L_more_31_no_low	/* none of the lowest bits is set */
-
-	orl	$1,%edx			/* record the fact in the extension */
-
-L_more_31_no_low:
-	movl	%eax,%ebx
-	xorl	%eax,%eax
-	jmp	L_shift_done
-
-L_exactly_32:
-	movl	%ebx,%edx
-	movl	%eax,%ebx
-	xorl	%eax,%eax
-	jmp	L_shift_done
-
-L_more_than_63:
-	cmpw	$65,%cx
-	jnc	L_more_than_64
-
-	movl	%eax,%edx
-	orl	%ebx,%ebx
-	jz	L_more_63_no_low
-
-	orl	$1,%edx
-	jmp	L_more_63_no_low
-
-L_more_than_64:
-	movl	$1,%edx		/* The shifted nr always at least one '1' */
-
-L_more_63_no_low:
-	xorl	%ebx,%ebx
-	xorl	%eax,%eax
-
-L_shift_done:
-	/* Now do the addition */
-	addl	SIGL(%esi),%ebx
-	adcl	SIGH(%esi),%eax
-	jnc	L_round_the_result
-
-	/* Overflow, adjust the result */
-	rcrl	$1,%eax
-	rcrl	$1,%ebx
-	rcrl	$1,%edx
-	jnc	L_no_bit_lost
-
-	orl	$1,%edx
-
-L_no_bit_lost:
-	incw	EXP(%edi)
-
-L_round_the_result:
-	jmp	fpu_reg_round	/* Round the result */
-
-
-
-#ifdef PARANOID
-/* If we ever get here then we have problems! */
-L_bugged:
-	pushl	EX_INTERNAL|0x201
-	call	EXCEPTION
-	pop	%ebx
-	movl	$-1,%eax
-	jmp	L_exit
-
-L_exit:
-	popl	%ebx
-	popl	%edi
-	popl	%esi
-	leave
-	RET
-#endif /* PARANOID */
-SYM_FUNC_END(FPU_u_add)
diff --git a/arch/x86/math-emu/reg_u_div.S b/arch/x86/math-emu/reg_u_div.S
deleted file mode 100644
index b5a41e2fc484c..0000000000000
--- a/arch/x86/math-emu/reg_u_div.S
+++ /dev/null
@@ -1,474 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file	"reg_u_div.S"
-/*---------------------------------------------------------------------------+
- |  reg_u_div.S                                                              |
- |                                                                           |
- | Divide one FPU_REG by another and put the result in a destination FPU_REG.|
- |                                                                           |
- | Copyright (C) 1992,1993,1995,1997                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- | Call from C as:                                                           |
- |    int FPU_u_div(FPU_REG *a, FPU_REG *b, FPU_REG *dest,                   |
- |                unsigned int control_word, char *sign)                     |
- |                                                                           |
- |  Does not compute the destination exponent, but does adjust it.           |
- |                                                                           |
- |    Return value is the tag of the answer, or-ed with FPU_Exception if     |
- |    one was raised, or -1 on internal error.                               |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-
-
-/* #define	dSIGL(x)	(x) */
-/* #define	dSIGH(x)	4(x) */
-
-
-#ifndef NON_REENTRANT_FPU
-/*
-	Local storage on the stack:
-	Result:		FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0
-	Overflow flag:	ovfl_flag
- */
-#define FPU_accum_3	-4(%ebp)
-#define FPU_accum_2	-8(%ebp)
-#define FPU_accum_1	-12(%ebp)
-#define FPU_accum_0	-16(%ebp)
-#define FPU_result_1	-20(%ebp)
-#define FPU_result_2	-24(%ebp)
-#define FPU_ovfl_flag	-28(%ebp)
-
-#else
-.data
-/*
-	Local storage in a static area:
-	Result:		FPU_accum_3:FPU_accum_2:FPU_accum_1:FPU_accum_0
-	Overflow flag:	ovfl_flag
- */
-	.align 4,0
-FPU_accum_3:
-	.long	0
-FPU_accum_2:
-	.long	0
-FPU_accum_1:
-	.long	0
-FPU_accum_0:
-	.long	0
-FPU_result_1:
-	.long	0
-FPU_result_2:
-	.long	0
-FPU_ovfl_flag:
-	.byte	0
-#endif /* NON_REENTRANT_FPU */
-
-#define REGA	PARAM1
-#define REGB	PARAM2
-#define DEST	PARAM3
-
-.text
-SYM_FUNC_START(FPU_u_div)
-	pushl	%ebp
-	movl	%esp,%ebp
-#ifndef NON_REENTRANT_FPU
-	subl	$28,%esp
-#endif /* NON_REENTRANT_FPU */
-
-	pushl	%esi
-	pushl	%edi
-	pushl	%ebx
-
-	movl	REGA,%esi
-	movl	REGB,%ebx
-	movl	DEST,%edi
-
-	movswl	EXP(%esi),%edx
-	movswl	EXP(%ebx),%eax
-	subl	%eax,%edx
-	addl	EXP_BIAS,%edx
-
-	/* A denormal and a large number can cause an exponent underflow */
-	cmpl	EXP_WAY_UNDER,%edx
-	jg	xExp_not_underflow
-
-	/* Set to a really low value allow correct handling */
-	movl	EXP_WAY_UNDER,%edx
-
-xExp_not_underflow:
-
-	movw    %dx,EXP(%edi)
-
-#ifdef PARANOID
-/*	testl	$0x80000000, SIGH(%esi)	// Dividend */
-/*	je	L_bugged */
-	testl	$0x80000000, SIGH(%ebx)	/* Divisor */
-	je	L_bugged
-#endif /* PARANOID */ 
-
-/* Check if the divisor can be treated as having just 32 bits */
-	cmpl	$0,SIGL(%ebx)
-	jnz	L_Full_Division	/* Can't do a quick divide */
-
-/* We should be able to zip through the division here */
-	movl	SIGH(%ebx),%ecx	/* The divisor */
-	movl	SIGH(%esi),%edx	/* Dividend */
-	movl	SIGL(%esi),%eax	/* Dividend */
-
-	cmpl	%ecx,%edx
-	setaeb	FPU_ovfl_flag	/* Keep a record */
-	jb	L_no_adjust
-
-	subl	%ecx,%edx	/* Prevent the overflow */
-
-L_no_adjust:
-	/* Divide the 64 bit number by the 32 bit denominator */
-	divl	%ecx
-	movl	%eax,FPU_result_2
-
-	/* Work on the remainder of the first division */
-	xorl	%eax,%eax
-	divl	%ecx
-	movl	%eax,FPU_result_1
-
-	/* Work on the remainder of the 64 bit division */
-	xorl	%eax,%eax
-	divl	%ecx
-
-	testb	$255,FPU_ovfl_flag	/* was the num > denom ? */
-	je	L_no_overflow
-
-	/* Do the shifting here */
-	/* increase the exponent */
-	incw	EXP(%edi)
-
-	/* shift the mantissa right one bit */
-	stc			/* To set the ms bit */
-	rcrl	FPU_result_2
-	rcrl	FPU_result_1
-	rcrl	%eax
-
-L_no_overflow:
-	jmp	LRound_precision	/* Do the rounding as required */
-
-
-/*---------------------------------------------------------------------------+
- |  Divide:   Return  arg1/arg2 to arg3.                                     |
- |                                                                           |
- |  This routine does not use the exponents of arg1 and arg2, but does       |
- |  adjust the exponent of arg3.                                             |
- |                                                                           |
- |  The maximum returned value is (ignoring exponents)                       |
- |               .ffffffff ffffffff                                          |
- |               ------------------  =  1.ffffffff fffffffe                  |
- |               .80000000 00000000                                          |
- | and the minimum is                                                        |
- |               .80000000 00000000                                          |
- |               ------------------  =  .80000000 00000001   (rounded)       |
- |               .ffffffff ffffffff                                          |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-
-L_Full_Division:
-	/* Save extended dividend in local register */
-	movl	SIGL(%esi),%eax
-	movl	%eax,FPU_accum_2
-	movl	SIGH(%esi),%eax
-	movl	%eax,FPU_accum_3
-	xorl	%eax,%eax
-	movl	%eax,FPU_accum_1	/* zero the extension */
-	movl	%eax,FPU_accum_0	/* zero the extension */
-
-	movl	SIGL(%esi),%eax	/* Get the current num */
-	movl	SIGH(%esi),%edx
-
-/*----------------------------------------------------------------------*/
-/* Initialization done.
-   Do the first 32 bits. */
-
-	movb	$0,FPU_ovfl_flag
-	cmpl	SIGH(%ebx),%edx	/* Test for imminent overflow */
-	jb	LLess_than_1
-	ja	LGreater_than_1
-
-	cmpl	SIGL(%ebx),%eax
-	jb	LLess_than_1
-
-LGreater_than_1:
-/* The dividend is greater or equal, would cause overflow */
-	setaeb	FPU_ovfl_flag		/* Keep a record */
-
-	subl	SIGL(%ebx),%eax
-	sbbl	SIGH(%ebx),%edx	/* Prevent the overflow */
-	movl	%eax,FPU_accum_2
-	movl	%edx,FPU_accum_3
-
-LLess_than_1:
-/* At this point, we have a dividend < divisor, with a record of
-   adjustment in FPU_ovfl_flag */
-
-	/* We will divide by a number which is too large */
-	movl	SIGH(%ebx),%ecx
-	addl	$1,%ecx
-	jnc	LFirst_div_not_1
-
-	/* here we need to divide by 100000000h,
-	   i.e., no division at all.. */
-	mov	%edx,%eax
-	jmp	LFirst_div_done
-
-LFirst_div_not_1:
-	divl	%ecx		/* Divide the numerator by the augmented
-				   denom ms dw */
-
-LFirst_div_done:
-	movl	%eax,FPU_result_2	/* Put the result in the answer */
-
-	mull	SIGH(%ebx)	/* mul by the ms dw of the denom */
-
-	subl	%eax,FPU_accum_2	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_3
-
-	movl	FPU_result_2,%eax	/* Get the result back */
-	mull	SIGL(%ebx)	/* now mul the ls dw of the denom */
-
-	subl	%eax,FPU_accum_1	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_2
-	sbbl	$0,FPU_accum_3
-	je	LDo_2nd_32_bits		/* Must check for non-zero result here */
-
-#ifdef PARANOID
-	jb	L_bugged_1
-#endif /* PARANOID */ 
-
-	/* need to subtract another once of the denom */
-	incl	FPU_result_2	/* Correct the answer */
-
-	movl	SIGL(%ebx),%eax
-	movl	SIGH(%ebx),%edx
-	subl	%eax,FPU_accum_1	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_2
-
-#ifdef PARANOID
-	sbbl	$0,FPU_accum_3
-	jne	L_bugged_1	/* Must check for non-zero result here */
-#endif /* PARANOID */ 
-
-/*----------------------------------------------------------------------*/
-/* Half of the main problem is done, there is just a reduced numerator
-   to handle now.
-   Work with the second 32 bits, FPU_accum_0 not used from now on */
-LDo_2nd_32_bits:
-	movl	FPU_accum_2,%edx	/* get the reduced num */
-	movl	FPU_accum_1,%eax
-
-	/* need to check for possible subsequent overflow */
-	cmpl	SIGH(%ebx),%edx
-	jb	LDo_2nd_div
-	ja	LPrevent_2nd_overflow
-
-	cmpl	SIGL(%ebx),%eax
-	jb	LDo_2nd_div
-
-LPrevent_2nd_overflow:
-/* The numerator is greater or equal, would cause overflow */
-	/* prevent overflow */
-	subl	SIGL(%ebx),%eax
-	sbbl	SIGH(%ebx),%edx
-	movl	%edx,FPU_accum_2
-	movl	%eax,FPU_accum_1
-
-	incl	FPU_result_2	/* Reflect the subtraction in the answer */
-
-#ifdef PARANOID
-	je	L_bugged_2	/* Can't bump the result to 1.0 */
-#endif /* PARANOID */ 
-
-LDo_2nd_div:
-	cmpl	$0,%ecx		/* augmented denom msw */
-	jnz	LSecond_div_not_1
-
-	/* %ecx == 0, we are dividing by 1.0 */
-	mov	%edx,%eax
-	jmp	LSecond_div_done
-
-LSecond_div_not_1:
-	divl	%ecx		/* Divide the numerator by the denom ms dw */
-
-LSecond_div_done:
-	movl	%eax,FPU_result_1	/* Put the result in the answer */
-
-	mull	SIGH(%ebx)	/* mul by the ms dw of the denom */
-
-	subl	%eax,FPU_accum_1	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_2
-
-#ifdef PARANOID
-	jc	L_bugged_2
-#endif /* PARANOID */ 
-
-	movl	FPU_result_1,%eax	/* Get the result back */
-	mull	SIGL(%ebx)	/* now mul the ls dw of the denom */
-
-	subl	%eax,FPU_accum_0	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_1	/* Subtract from the num local reg */
-	sbbl	$0,FPU_accum_2
-
-#ifdef PARANOID
-	jc	L_bugged_2
-#endif /* PARANOID */ 
-
-	jz	LDo_3rd_32_bits
-
-#ifdef PARANOID
-	cmpl	$1,FPU_accum_2
-	jne	L_bugged_2
-#endif /* PARANOID */
-
-	/* need to subtract another once of the denom */
-	movl	SIGL(%ebx),%eax
-	movl	SIGH(%ebx),%edx
-	subl	%eax,FPU_accum_0	/* Subtract from the num local reg */
-	sbbl	%edx,FPU_accum_1
-	sbbl	$0,FPU_accum_2
-
-#ifdef PARANOID
-	jc	L_bugged_2
-	jne	L_bugged_2
-#endif /* PARANOID */ 
-
-	addl	$1,FPU_result_1	/* Correct the answer */
-	adcl	$0,FPU_result_2
-
-#ifdef PARANOID
-	jc	L_bugged_2	/* Must check for non-zero result here */
-#endif /* PARANOID */
-
-/*----------------------------------------------------------------------*/
-/* The division is essentially finished here, we just need to perform
-   tidying operations.
-   Deal with the 3rd 32 bits */
-LDo_3rd_32_bits:
-	movl	FPU_accum_1,%edx		/* get the reduced num */
-	movl	FPU_accum_0,%eax
-
-	/* need to check for possible subsequent overflow */
-	cmpl	SIGH(%ebx),%edx	/* denom */
-	jb	LRound_prep
-	ja	LPrevent_3rd_overflow
-
-	cmpl	SIGL(%ebx),%eax	/* denom */
-	jb	LRound_prep
-
-LPrevent_3rd_overflow:
-	/* prevent overflow */
-	subl	SIGL(%ebx),%eax
-	sbbl	SIGH(%ebx),%edx
-	movl	%edx,FPU_accum_1
-	movl	%eax,FPU_accum_0
-
-	addl	$1,FPU_result_1	/* Reflect the subtraction in the answer */
-	adcl	$0,FPU_result_2
-	jne	LRound_prep
-	jnc	LRound_prep
-
-	/* This is a tricky spot, there is an overflow of the answer */
-	movb	$255,FPU_ovfl_flag		/* Overflow -> 1.000 */
-
-LRound_prep:
-/*
- * Prepare for rounding.
- * To test for rounding, we just need to compare 2*accum with the
- * denom.
- */
-	movl	FPU_accum_0,%ecx
-	movl	FPU_accum_1,%edx
-	movl	%ecx,%eax
-	orl	%edx,%eax
-	jz	LRound_ovfl		/* The accumulator contains zero. */
-
-	/* Multiply by 2 */
-	clc
-	rcll	$1,%ecx
-	rcll	$1,%edx
-	jc	LRound_large		/* No need to compare, denom smaller */
-
-	subl	SIGL(%ebx),%ecx
-	sbbl	SIGH(%ebx),%edx
-	jnc	LRound_not_small
-
-	movl	$0x70000000,%eax	/* Denom was larger */
-	jmp	LRound_ovfl
-
-LRound_not_small:
-	jnz	LRound_large
-
-	movl	$0x80000000,%eax	/* Remainder was exactly 1/2 denom */
-	jmp	LRound_ovfl
-
-LRound_large:
-	movl	$0xff000000,%eax	/* Denom was smaller */
-
-LRound_ovfl:
-/* We are now ready to deal with rounding, but first we must get
-   the bits properly aligned */
-	testb	$255,FPU_ovfl_flag	/* was the num > denom ? */
-	je	LRound_precision
-
-	incw	EXP(%edi)
-
-	/* shift the mantissa right one bit */
-	stc			/* Will set the ms bit */
-	rcrl	FPU_result_2
-	rcrl	FPU_result_1
-	rcrl	%eax
-
-/* Round the result as required */
-LRound_precision:
-	decw	EXP(%edi)	/* binary point between 1st & 2nd bits */
-
-	movl	%eax,%edx
-	movl	FPU_result_1,%ebx
-	movl	FPU_result_2,%eax
-	jmp	fpu_reg_round
-
-
-#ifdef PARANOID
-/* The logic is wrong if we got here */
-L_bugged:
-	pushl	EX_INTERNAL|0x202
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_exit
-
-L_bugged_1:
-	pushl	EX_INTERNAL|0x203
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_exit
-
-L_bugged_2:
-	pushl	EX_INTERNAL|0x204
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_exit
-
-L_exit:
-	movl	$-1,%eax
-	popl	%ebx
-	popl	%edi
-	popl	%esi
-
-	leave
-	RET
-#endif /* PARANOID */ 
-
-SYM_FUNC_END(FPU_u_div)
diff --git a/arch/x86/math-emu/reg_u_mul.S b/arch/x86/math-emu/reg_u_mul.S
deleted file mode 100644
index e2588b24b8c2c..0000000000000
--- a/arch/x86/math-emu/reg_u_mul.S
+++ /dev/null
@@ -1,150 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file	"reg_u_mul.S"
-/*---------------------------------------------------------------------------+
- |  reg_u_mul.S                                                              |
- |                                                                           |
- | Core multiplication routine                                               |
- |                                                                           |
- | Copyright (C) 1992,1993,1995,1997                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- |   Basic multiplication routine.                                           |
- |   Does not check the resulting exponent for overflow/underflow            |
- |                                                                           |
- |   FPU_u_mul(FPU_REG *a, FPU_REG *b, FPU_REG *c, unsigned int cw);         |
- |                                                                           |
- |   Internal working is at approx 128 bits.                                 |
- |   Result is rounded to nearest 53 or 64 bits, using "nearest or even".    |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-
-
-
-#ifndef NON_REENTRANT_FPU
-/*  Local storage on the stack: */
-#define FPU_accum_0	-4(%ebp)	/* ms word */
-#define FPU_accum_1	-8(%ebp)
-
-#else
-/*  Local storage in a static area: */
-.data
-	.align 4,0
-FPU_accum_0:
-	.long	0
-FPU_accum_1:
-	.long	0
-#endif /* NON_REENTRANT_FPU */
-
-
-.text
-SYM_FUNC_START(FPU_u_mul)
-	pushl	%ebp
-	movl	%esp,%ebp
-#ifndef NON_REENTRANT_FPU
-	subl	$8,%esp
-#endif /* NON_REENTRANT_FPU */ 
-
-	pushl	%esi
-	pushl	%edi
-	pushl	%ebx
-
-	movl	PARAM1,%esi
-	movl	PARAM2,%edi
-
-#ifdef PARANOID
-	testl	$0x80000000,SIGH(%esi)
-	jz	L_bugged
-	testl	$0x80000000,SIGH(%edi)
-	jz	L_bugged
-#endif /* PARANOID */
-
-	xorl	%ecx,%ecx
-	xorl	%ebx,%ebx
-
-	movl	SIGL(%esi),%eax
-	mull	SIGL(%edi)
-	movl	%eax,FPU_accum_0
-	movl	%edx,FPU_accum_1
-
-	movl	SIGL(%esi),%eax
-	mull	SIGH(%edi)
-	addl	%eax,FPU_accum_1
-	adcl	%edx,%ebx
-/*	adcl	$0,%ecx		// overflow here is not possible */
-
-	movl	SIGH(%esi),%eax
-	mull	SIGL(%edi)
-	addl	%eax,FPU_accum_1
-	adcl	%edx,%ebx
-	adcl	$0,%ecx
-
-	movl	SIGH(%esi),%eax
-	mull	SIGH(%edi)
-	addl	%eax,%ebx
-	adcl	%edx,%ecx
-
-	/* Get the sum of the exponents. */
-	movl	PARAM6,%eax
-	subl	EXP_BIAS-1,%eax
-
-	/* Two denormals can cause an exponent underflow */
-	cmpl	EXP_WAY_UNDER,%eax
-	jg	Exp_not_underflow
-
-	/* Set to a really low value allow correct handling */
-	movl	EXP_WAY_UNDER,%eax
-
-Exp_not_underflow:
-
-/*  Have now finished with the sources */
-	movl	PARAM3,%edi	/* Point to the destination */
-	movw	%ax,EXP(%edi)
-
-/*  Now make sure that the result is normalized */
-	testl	$0x80000000,%ecx
-	jnz	LResult_Normalised
-
-	/* Normalize by shifting left one bit */
-	shll	$1,FPU_accum_0
-	rcll	$1,FPU_accum_1
-	rcll	$1,%ebx
-	rcll	$1,%ecx
-	decw	EXP(%edi)
-
-LResult_Normalised:
-	movl	FPU_accum_0,%eax
-	movl	FPU_accum_1,%edx
-	orl	%eax,%eax
-	jz	L_extent_zero
-
-	orl	$1,%edx
-
-L_extent_zero:
-	movl	%ecx,%eax
-	jmp	fpu_reg_round
-
-
-#ifdef PARANOID
-L_bugged:
-	pushl	EX_INTERNAL|0x205
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_exit
-
-L_exit:
-	popl	%ebx
-	popl	%edi
-	popl	%esi
-	leave
-	RET
-#endif /* PARANOID */ 
-
-SYM_FUNC_END(FPU_u_mul)
diff --git a/arch/x86/math-emu/reg_u_sub.S b/arch/x86/math-emu/reg_u_sub.S
deleted file mode 100644
index 4c900c29e4ff2..0000000000000
--- a/arch/x86/math-emu/reg_u_sub.S
+++ /dev/null
@@ -1,274 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file	"reg_u_sub.S"
-/*---------------------------------------------------------------------------+
- |  reg_u_sub.S                                                              |
- |                                                                           |
- | Core floating point subtraction routine.                                  |
- |                                                                           |
- | Copyright (C) 1992,1993,1995,1997                                         |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@suburbia.net                              |
- |                                                                           |
- | Call from C as:                                                           |
- |    int FPU_u_sub(FPU_REG *arg1, FPU_REG *arg2, FPU_REG *answ,             |
- |                                                int control_w)             |
- |    Return value is the tag of the answer, or-ed with FPU_Exception if     |
- |    one was raised, or -1 on internal error.                               |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*
- |    Kernel subtraction routine FPU_u_sub(reg *arg1, reg *arg2, reg *answ).
- |    Takes two valid reg f.p. numbers (TAG_Valid), which are
- |    treated as unsigned numbers,
- |    and returns their difference as a TAG_Valid or TAG_Zero f.p.
- |    number.
- |    The first number (arg1) must be the larger.
- |    The returned number is normalized.
- |    Basic checks are performed if PARANOID is defined.
- */
-
-#include "exception.h"
-#include "fpu_emu.h"
-#include "control_w.h"
-
-.text
-SYM_FUNC_START(FPU_u_sub)
-	pushl	%ebp
-	movl	%esp,%ebp
-	pushl	%esi
-	pushl	%edi
-	pushl	%ebx
-
-	movl	PARAM1,%esi	/* source 1 */
-	movl	PARAM2,%edi	/* source 2 */
-	
-	movl	PARAM6,%ecx
-	subl	PARAM7,%ecx	/* exp1 - exp2 */
-
-#ifdef PARANOID
-	/* source 2 is always smaller than source 1 */
-	js	L_bugged_1
-
-	testl	$0x80000000,SIGH(%edi)	/* The args are assumed to be be normalized */
-	je	L_bugged_2
-
-	testl	$0x80000000,SIGH(%esi)
-	je	L_bugged_2
-#endif /* PARANOID */
-
-/*--------------------------------------+
- |	Form a register holding the     |
- |	smaller number                  |
- +--------------------------------------*/
-	movl	SIGH(%edi),%eax	/* register ms word */
-	movl	SIGL(%edi),%ebx	/* register ls word */
-
-	movl	PARAM3,%edi	/* destination */
-	movl	PARAM6,%edx
-	movw	%dx,EXP(%edi)	/* Copy exponent to destination */
-
-	xorl	%edx,%edx	/* register extension */
-
-/*--------------------------------------+
- |	Shift the temporary register	|
- |      right the required number of	|
- |	places.				|
- +--------------------------------------*/
-
-	cmpw	$32,%cx		/* shrd only works for 0..31 bits */
-	jnc	L_more_than_31
-
-/* less than 32 bits */
-	shrd	%cl,%ebx,%edx
-	shrd	%cl,%eax,%ebx
-	shr	%cl,%eax
-	jmp	L_shift_done
-
-L_more_than_31:
-	cmpw	$64,%cx
-	jnc	L_more_than_63
-
-	subb	$32,%cl
-	jz	L_exactly_32
-
-	shrd	%cl,%eax,%edx
-	shr	%cl,%eax
-	orl	%ebx,%ebx
-	jz	L_more_31_no_low	/* none of the lowest bits is set */
-
-	orl	$1,%edx			/* record the fact in the extension */
-
-L_more_31_no_low:
-	movl	%eax,%ebx
-	xorl	%eax,%eax
-	jmp	L_shift_done
-
-L_exactly_32:
-	movl	%ebx,%edx
-	movl	%eax,%ebx
-	xorl	%eax,%eax
-	jmp	L_shift_done
-
-L_more_than_63:
-	cmpw	$65,%cx
-	jnc	L_more_than_64
-
-	/* Shift right by 64 bits */
-	movl	%eax,%edx
-	orl	%ebx,%ebx
-	jz	L_more_63_no_low
-
-	orl	$1,%edx
-	jmp	L_more_63_no_low
-
-L_more_than_64:
-	jne	L_more_than_65
-
-	/* Shift right by 65 bits */
-	/* Carry is clear if we get here */
-	movl	%eax,%edx
-	rcrl	%edx
-	jnc	L_shift_65_nc
-
-	orl	$1,%edx
-	jmp	L_more_63_no_low
-
-L_shift_65_nc:
-	orl	%ebx,%ebx
-	jz	L_more_63_no_low
-
-	orl	$1,%edx
-	jmp	L_more_63_no_low
-
-L_more_than_65:
-	movl	$1,%edx		/* The shifted nr always at least one '1' */
-
-L_more_63_no_low:
-	xorl	%ebx,%ebx
-	xorl	%eax,%eax
-
-L_shift_done:
-L_subtr:
-/*------------------------------+
- |	Do the subtraction	|
- +------------------------------*/
-	xorl	%ecx,%ecx
-	subl	%edx,%ecx
-	movl	%ecx,%edx
-	movl	SIGL(%esi),%ecx
-	sbbl	%ebx,%ecx
-	movl	%ecx,%ebx
-	movl	SIGH(%esi),%ecx
-	sbbl	%eax,%ecx
-	movl	%ecx,%eax
-
-#ifdef PARANOID
-	/* We can never get a borrow */
-	jc	L_bugged
-#endif /* PARANOID */
-
-/*--------------------------------------+
- |	Normalize the result		|
- +--------------------------------------*/
-	testl	$0x80000000,%eax
-	jnz	L_round		/* no shifting needed */
-
-	orl	%eax,%eax
-	jnz	L_shift_1	/* shift left 1 - 31 bits */
-
-	orl	%ebx,%ebx
-	jnz	L_shift_32	/* shift left 32 - 63 bits */
-
-/*
- *	 A rare case, the only one which is non-zero if we got here
- *         is:           1000000 .... 0000
- *                      -0111111 .... 1111 1
- *                       -------------------- 
- *                       0000000 .... 0000 1 
- */
-
-	cmpl	$0x80000000,%edx
-	jnz	L_must_be_zero
-
-	/* Shift left 64 bits */
-	subw	$64,EXP(%edi)
-	xchg	%edx,%eax
-	jmp	fpu_reg_round
-
-L_must_be_zero:
-#ifdef PARANOID
-	orl	%edx,%edx
-	jnz	L_bugged_3
-#endif /* PARANOID */ 
-
-	/* The result is zero */
-	movw	$0,EXP(%edi)		/* exponent */
-	movl	$0,SIGL(%edi)
-	movl	$0,SIGH(%edi)
-	movl	TAG_Zero,%eax
-	jmp	L_exit
-
-L_shift_32:
-	movl	%ebx,%eax
-	movl	%edx,%ebx
-	movl	$0,%edx
-	subw	$32,EXP(%edi)	/* Can get underflow here */
-
-/* We need to shift left by 1 - 31 bits */
-L_shift_1:
-	bsrl	%eax,%ecx	/* get the required shift in %ecx */
-	subl	$31,%ecx
-	negl	%ecx
-	shld	%cl,%ebx,%eax
-	shld	%cl,%edx,%ebx
-	shl	%cl,%edx
-	subw	%cx,EXP(%edi)	/* Can get underflow here */
-
-L_round:
-	jmp	fpu_reg_round	/* Round the result */
-
-
-#ifdef PARANOID
-L_bugged_1:
-	pushl	EX_INTERNAL|0x206
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_error_exit
-
-L_bugged_2:
-	pushl	EX_INTERNAL|0x209
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_error_exit
-
-L_bugged_3:
-	pushl	EX_INTERNAL|0x210
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_error_exit
-
-L_bugged_4:
-	pushl	EX_INTERNAL|0x211
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_error_exit
-
-L_bugged:
-	pushl	EX_INTERNAL|0x212
-	call	EXCEPTION
-	pop	%ebx
-	jmp	L_error_exit
-
-L_error_exit:
-	movl	$-1,%eax
-
-#endif /* PARANOID */
-
-L_exit:
-	popl	%ebx
-	popl	%edi
-	popl	%esi
-	leave
-	RET
-SYM_FUNC_END(FPU_u_sub)
diff --git a/arch/x86/math-emu/round_Xsig.S b/arch/x86/math-emu/round_Xsig.S
deleted file mode 100644
index 126c40473badb..0000000000000
--- a/arch/x86/math-emu/round_Xsig.S
+++ /dev/null
@@ -1,142 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  round_Xsig.S                                                             |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1995                                         |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@jacobi.maths.monash.edu.au |
- |                                                                           |
- | Normalize and round a 12 byte quantity.                                   |
- | Call from C as:                                                           |
- |   int round_Xsig(Xsig *n)                                                 |
- |                                                                           |
- | Normalize a 12 byte quantity.                                             |
- | Call from C as:                                                           |
- |   int norm_Xsig(Xsig *n)                                                  |
- |                                                                           |
- | Each function returns the size of the shift (nr of bits).                 |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-	.file	"round_Xsig.S"
-
-#include "fpu_emu.h"
-
-
-.text
-SYM_FUNC_START(round_Xsig)
-	pushl	%ebp
-	movl	%esp,%ebp
-	pushl	%ebx		/* Reserve some space */
-	pushl	%ebx
-	pushl	%esi
-
-	movl	PARAM1,%esi
-
-	movl	8(%esi),%edx
-	movl	4(%esi),%ebx
-	movl	(%esi),%eax
-
-	movl	$0,-4(%ebp)
-
-	orl	%edx,%edx	/* ms bits */
-	js	L_round		/* Already normalized */
-	jnz	L_shift_1	/* Shift left 1 - 31 bits */
-
-	movl	%ebx,%edx
-	movl	%eax,%ebx
-	xorl	%eax,%eax
-	movl	$-32,-4(%ebp)
-
-/* We need to shift left by 1 - 31 bits */
-L_shift_1:
-	bsrl	%edx,%ecx	/* get the required shift in %ecx */
-	subl	$31,%ecx
-	negl	%ecx
-	subl	%ecx,-4(%ebp)
-	shld	%cl,%ebx,%edx
-	shld	%cl,%eax,%ebx
-	shl	%cl,%eax
-
-L_round:
-	testl	$0x80000000,%eax
-	jz	L_exit
-
-	addl	$1,%ebx
-	adcl	$0,%edx
-	jnz	L_exit
-
-	movl	$0x80000000,%edx
-	incl	-4(%ebp)
-
-L_exit:
-	movl	%edx,8(%esi)
-	movl	%ebx,4(%esi)
-	movl	%eax,(%esi)
-
-	movl	-4(%ebp),%eax
-
-	popl	%esi
-	popl	%ebx
-	leave
-	RET
-SYM_FUNC_END(round_Xsig)
-
-
-
-SYM_FUNC_START(norm_Xsig)
-	pushl	%ebp
-	movl	%esp,%ebp
-	pushl	%ebx		/* Reserve some space */
-	pushl	%ebx
-	pushl	%esi
-
-	movl	PARAM1,%esi
-
-	movl	8(%esi),%edx
-	movl	4(%esi),%ebx
-	movl	(%esi),%eax
-
-	movl	$0,-4(%ebp)
-
-	orl	%edx,%edx	/* ms bits */
-	js	L_n_exit		/* Already normalized */
-	jnz	L_n_shift_1	/* Shift left 1 - 31 bits */
-
-	movl	%ebx,%edx
-	movl	%eax,%ebx
-	xorl	%eax,%eax
-	movl	$-32,-4(%ebp)
-
-	orl	%edx,%edx	/* ms bits */
-	js	L_n_exit	/* Normalized now */
-	jnz	L_n_shift_1	/* Shift left 1 - 31 bits */
-
-	movl	%ebx,%edx
-	movl	%eax,%ebx
-	xorl	%eax,%eax
-	addl	$-32,-4(%ebp)
-	jmp	L_n_exit	/* Might not be normalized,
-	                           but shift no more. */
-
-/* We need to shift left by 1 - 31 bits */
-L_n_shift_1:
-	bsrl	%edx,%ecx	/* get the required shift in %ecx */
-	subl	$31,%ecx
-	negl	%ecx
-	subl	%ecx,-4(%ebp)
-	shld	%cl,%ebx,%edx
-	shld	%cl,%eax,%ebx
-	shl	%cl,%eax
-
-L_n_exit:
-	movl	%edx,8(%esi)
-	movl	%ebx,4(%esi)
-	movl	%eax,(%esi)
-
-	movl	-4(%ebp),%eax
-
-	popl	%esi
-	popl	%ebx
-	leave
-	RET
-SYM_FUNC_END(norm_Xsig)
diff --git a/arch/x86/math-emu/shr_Xsig.S b/arch/x86/math-emu/shr_Xsig.S
deleted file mode 100644
index f726bf6f6396e..0000000000000
--- a/arch/x86/math-emu/shr_Xsig.S
+++ /dev/null
@@ -1,89 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file	"shr_Xsig.S"
-/*---------------------------------------------------------------------------+
- |  shr_Xsig.S                                                               |
- |                                                                           |
- | 12 byte right shift function                                              |
- |                                                                           |
- | Copyright (C) 1992,1994,1995                                              |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@jacobi.maths.monash.edu.au |
- |                                                                           |
- | Call from C as:                                                           |
- |   void shr_Xsig(Xsig *arg, unsigned nr)                                   |
- |                                                                           |
- |   Extended shift right function.                                          |
- |   Fastest for small shifts.                                               |
- |   Shifts the 12 byte quantity pointed to by the first arg (arg)           |
- |   right by the number of bits specified by the second arg (nr).           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-.text
-SYM_FUNC_START(shr_Xsig)
-	push	%ebp
-	movl	%esp,%ebp
-	pushl	%esi
-	movl	PARAM2,%ecx
-	movl	PARAM1,%esi
-	cmpl	$32,%ecx	/* shrd only works for 0..31 bits */
-	jnc	L_more_than_31
-
-/* less than 32 bits */
-	pushl	%ebx
-	movl	(%esi),%eax	/* lsl */
-	movl	4(%esi),%ebx	/* midl */
-	movl	8(%esi),%edx	/* msl */
-	shrd	%cl,%ebx,%eax
-	shrd	%cl,%edx,%ebx
-	shr	%cl,%edx
-	movl	%eax,(%esi)
-	movl	%ebx,4(%esi)
-	movl	%edx,8(%esi)
-	popl	%ebx
-	popl	%esi
-	leave
-	RET
-
-L_more_than_31:
-	cmpl	$64,%ecx
-	jnc	L_more_than_63
-
-	subb	$32,%cl
-	movl	4(%esi),%eax	/* midl */
-	movl	8(%esi),%edx	/* msl */
-	shrd	%cl,%edx,%eax
-	shr	%cl,%edx
-	movl	%eax,(%esi)
-	movl	%edx,4(%esi)
-	movl	$0,8(%esi)
-	popl	%esi
-	leave
-	RET
-
-L_more_than_63:
-	cmpl	$96,%ecx
-	jnc	L_more_than_95
-
-	subb	$64,%cl
-	movl	8(%esi),%eax	/* msl */
-	shr	%cl,%eax
-	xorl	%edx,%edx
-	movl	%eax,(%esi)
-	movl	%edx,4(%esi)
-	movl	%edx,8(%esi)
-	popl	%esi
-	leave
-	RET
-
-L_more_than_95:
-	xorl	%eax,%eax
-	movl	%eax,(%esi)
-	movl	%eax,4(%esi)
-	movl	%eax,8(%esi)
-	popl	%esi
-	leave
-	RET
-SYM_FUNC_END(shr_Xsig)
diff --git a/arch/x86/math-emu/status_w.h b/arch/x86/math-emu/status_w.h
deleted file mode 100644
index f642957330efc..0000000000000
--- a/arch/x86/math-emu/status_w.h
+++ /dev/null
@@ -1,68 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*---------------------------------------------------------------------------+
- |  status_w.h                                                               |
- |                                                                           |
- | Copyright (C) 1992,1993                                                   |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail   billm@vaxc.cc.monash.edu.au    |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#ifndef _STATUS_H_
-#define _STATUS_H_
-
-#include "fpu_emu.h"		/* for definition of PECULIAR_486 */
-
-#ifdef __ASSEMBLER__
-#define	Const__(x)	$##x
-#else
-#define	Const__(x)	x
-#endif
-
-#define SW_Backward    	Const__(0x8000)	/* backward compatibility */
-#define SW_C3		Const__(0x4000)	/* condition bit 3 */
-#define SW_Top		Const__(0x3800)	/* top of stack */
-#define SW_Top_Shift 	Const__(11)	/* shift for top of stack bits */
-#define SW_C2		Const__(0x0400)	/* condition bit 2 */
-#define SW_C1		Const__(0x0200)	/* condition bit 1 */
-#define SW_C0		Const__(0x0100)	/* condition bit 0 */
-#define SW_Summary     	Const__(0x0080)	/* exception summary */
-#define SW_Stack_Fault	Const__(0x0040)	/* stack fault */
-#define SW_Precision   	Const__(0x0020)	/* loss of precision */
-#define SW_Underflow   	Const__(0x0010)	/* underflow */
-#define SW_Overflow    	Const__(0x0008)	/* overflow */
-#define SW_Zero_Div    	Const__(0x0004)	/* divide by zero */
-#define SW_Denorm_Op   	Const__(0x0002)	/* denormalized operand */
-#define SW_Invalid     	Const__(0x0001)	/* invalid operation */
-
-#define SW_Exc_Mask     Const__(0x27f)	/* Status word exception bit mask */
-
-#ifndef __ASSEMBLER__
-
-#define COMP_A_gt_B	1
-#define COMP_A_eq_B	2
-#define COMP_A_lt_B	3
-#define COMP_No_Comp	4
-#define COMP_Denormal   0x20
-#define COMP_NaN	0x40
-#define COMP_SNaN	0x80
-
-#define status_word() \
-  ((partial_status & ~SW_Top & 0xffff) | ((top << SW_Top_Shift) & SW_Top))
-static inline void setcc(int cc)
-{
-	partial_status &= ~(SW_C0 | SW_C1 | SW_C2 | SW_C3);
-	partial_status |= (cc) & (SW_C0 | SW_C1 | SW_C2 | SW_C3);
-}
-
-#ifdef PECULIAR_486
-   /* Default, this conveys no information, but an 80486 does it. */
-   /* Clear the SW_C1 bit, "other bits undefined". */
-#  define clear_C1()  { partial_status &= ~SW_C1; }
-# else
-#  define clear_C1()
-#endif /* PECULIAR_486 */
-
-#endif /* __ASSEMBLER__ */
-
-#endif /* _STATUS_H_ */
diff --git a/arch/x86/math-emu/version.h b/arch/x86/math-emu/version.h
deleted file mode 100644
index a0d73a1d2b679..0000000000000
--- a/arch/x86/math-emu/version.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/*---------------------------------------------------------------------------+
- |  version.h                                                                |
- |                                                                           |
- |                                                                           |
- | Copyright (C) 1992,1993,1994,1996,1997,1999                               |
- |                  W. Metzenthen, 22 Parker St, Ormond, Vic 3163, Australia |
- |                  E-mail   billm@melbpc.org.au                             |
- |                                                                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#define FPU_VERSION "wm-FPU-emu version 2.01"
diff --git a/arch/x86/math-emu/wm_shrx.S b/arch/x86/math-emu/wm_shrx.S
deleted file mode 100644
index f608a28a4c43a..0000000000000
--- a/arch/x86/math-emu/wm_shrx.S
+++ /dev/null
@@ -1,207 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file	"wm_shrx.S"
-/*---------------------------------------------------------------------------+
- |  wm_shrx.S                                                                |
- |                                                                           |
- | 64 bit right shift functions                                              |
- |                                                                           |
- | Copyright (C) 1992,1995                                                   |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@jacobi.maths.monash.edu.au |
- |                                                                           |
- | Call from C as:                                                           |
- |   unsigned FPU_shrx(void *arg1, unsigned arg2)                            |
- | and                                                                       |
- |   unsigned FPU_shrxs(void *arg1, unsigned arg2)                           |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-#include "fpu_emu.h"
-
-.text
-/*---------------------------------------------------------------------------+
- |   unsigned FPU_shrx(void *arg1, unsigned arg2)                            |
- |                                                                           |
- |   Extended shift right function.                                          |
- |   Fastest for small shifts.                                               |
- |   Shifts the 64 bit quantity pointed to by the first arg (arg1)           |
- |   right by the number of bits specified by the second arg (arg2).         |
- |   Forms a 96 bit quantity from the 64 bit arg and eax:                    |
- |                [  64 bit arg ][ eax ]                                     |
- |            shift right  --------->                                        |
- |   The eax register is initialized to 0 before the shifting.               |
- |   Results returned in the 64 bit arg and eax.                             |
- +---------------------------------------------------------------------------*/
-
-SYM_FUNC_START(FPU_shrx)
-	push	%ebp
-	movl	%esp,%ebp
-	pushl	%esi
-	movl	PARAM2,%ecx
-	movl	PARAM1,%esi
-	cmpl	$32,%ecx	/* shrd only works for 0..31 bits */
-	jnc	L_more_than_31
-
-/* less than 32 bits */
-	pushl	%ebx
-	movl	(%esi),%ebx	/* lsl */
-	movl	4(%esi),%edx	/* msl */
-	xorl	%eax,%eax	/* extension */
-	shrd	%cl,%ebx,%eax
-	shrd	%cl,%edx,%ebx
-	shr	%cl,%edx
-	movl	%ebx,(%esi)
-	movl	%edx,4(%esi)
-	popl	%ebx
-	popl	%esi
-	leave
-	RET
-
-L_more_than_31:
-	cmpl	$64,%ecx
-	jnc	L_more_than_63
-
-	subb	$32,%cl
-	movl	(%esi),%eax	/* lsl */
-	movl	4(%esi),%edx	/* msl */
-	shrd	%cl,%edx,%eax
-	shr	%cl,%edx
-	movl	%edx,(%esi)
-	movl	$0,4(%esi)
-	popl	%esi
-	leave
-	RET
-
-L_more_than_63:
-	cmpl	$96,%ecx
-	jnc	L_more_than_95
-
-	subb	$64,%cl
-	movl	4(%esi),%eax	/* msl */
-	shr	%cl,%eax
-	xorl	%edx,%edx
-	movl	%edx,(%esi)
-	movl	%edx,4(%esi)
-	popl	%esi
-	leave
-	RET
-
-L_more_than_95:
-	xorl	%eax,%eax
-	movl	%eax,(%esi)
-	movl	%eax,4(%esi)
-	popl	%esi
-	leave
-	RET
-SYM_FUNC_END(FPU_shrx)
-
-
-/*---------------------------------------------------------------------------+
- |   unsigned FPU_shrxs(void *arg1, unsigned arg2)                           |
- |                                                                           |
- |   Extended shift right function (optimized for small floating point       |
- |   integers).                                                              |
- |   Shifts the 64 bit quantity pointed to by the first arg (arg1)           |
- |   right by the number of bits specified by the second arg (arg2).         |
- |   Forms a 96 bit quantity from the 64 bit arg and eax:                    |
- |                [  64 bit arg ][ eax ]                                     |
- |            shift right  --------->                                        |
- |   The eax register is initialized to 0 before the shifting.               |
- |   The lower 8 bits of eax are lost and replaced by a flag which is        |
- |   set (to 0x01) if any bit, apart from the first one, is set in the       |
- |   part which has been shifted out of the arg.                             |
- |   Results returned in the 64 bit arg and eax.                             |
- +---------------------------------------------------------------------------*/
-SYM_FUNC_START(FPU_shrxs)
-	push	%ebp
-	movl	%esp,%ebp
-	pushl	%esi
-	pushl	%ebx
-	movl	PARAM2,%ecx
-	movl	PARAM1,%esi
-	cmpl	$64,%ecx	/* shrd only works for 0..31 bits */
-	jnc	Ls_more_than_63
-
-	cmpl	$32,%ecx	/* shrd only works for 0..31 bits */
-	jc	Ls_less_than_32
-
-/* We got here without jumps by assuming that the most common requirement
-   is for small integers */
-/* Shift by [32..63] bits */
-	subb	$32,%cl
-	movl	(%esi),%eax	/* lsl */
-	movl	4(%esi),%edx	/* msl */
-	xorl	%ebx,%ebx
-	shrd	%cl,%eax,%ebx
-	shrd	%cl,%edx,%eax
-	shr	%cl,%edx
-	orl	%ebx,%ebx		/* test these 32 bits */
-	setne	%bl
-	test	$0x7fffffff,%eax	/* and 31 bits here */
-	setne	%bh
-	orw	%bx,%bx			/* Any of the 63 bit set ? */
-	setne	%al
-	movl	%edx,(%esi)
-	movl	$0,4(%esi)
-	popl	%ebx
-	popl	%esi
-	leave
-	RET
-
-/* Shift by [0..31] bits */
-Ls_less_than_32:
-	movl	(%esi),%ebx	/* lsl */
-	movl	4(%esi),%edx	/* msl */
-	xorl	%eax,%eax	/* extension */
-	shrd	%cl,%ebx,%eax
-	shrd	%cl,%edx,%ebx
-	shr	%cl,%edx
-	test	$0x7fffffff,%eax	/* only need to look at eax here */
-	setne	%al
-	movl	%ebx,(%esi)
-	movl	%edx,4(%esi)
-	popl	%ebx
-	popl	%esi
-	leave
-	RET
-
-/* Shift by [64..95] bits */
-Ls_more_than_63:
-	cmpl	$96,%ecx
-	jnc	Ls_more_than_95
-
-	subb	$64,%cl
-	movl	(%esi),%ebx	/* lsl */
-	movl	4(%esi),%eax	/* msl */
-	xorl	%edx,%edx	/* extension */
-	shrd	%cl,%ebx,%edx
-	shrd	%cl,%eax,%ebx
-	shr	%cl,%eax
-	orl	%ebx,%edx
-	setne	%bl
-	test	$0x7fffffff,%eax	/* only need to look at eax here */
-	setne	%bh
-	orw	%bx,%bx
-	setne	%al
-	xorl	%edx,%edx
-	movl	%edx,(%esi)	/* set to zero */
-	movl	%edx,4(%esi)	/* set to zero */
-	popl	%ebx
-	popl	%esi
-	leave
-	RET
-
-Ls_more_than_95:
-/* Shift by [96..inf) bits */
-	xorl	%eax,%eax
-	movl	(%esi),%ebx
-	orl	4(%esi),%ebx
-	setne	%al
-	xorl	%ebx,%ebx
-	movl	%ebx,(%esi)
-	movl	%ebx,4(%esi)
-	popl	%ebx
-	popl	%esi
-	leave
-	RET
-SYM_FUNC_END(FPU_shrxs)
diff --git a/arch/x86/math-emu/wm_sqrt.S b/arch/x86/math-emu/wm_sqrt.S
deleted file mode 100644
index 40526dd85137b..0000000000000
--- a/arch/x86/math-emu/wm_sqrt.S
+++ /dev/null
@@ -1,472 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-	.file	"wm_sqrt.S"
-/*---------------------------------------------------------------------------+
- |  wm_sqrt.S                                                                |
- |                                                                           |
- | Fixed point arithmetic square root evaluation.                            |
- |                                                                           |
- | Copyright (C) 1992,1993,1995,1997                                         |
- |                       W. Metzenthen, 22 Parker St, Ormond, Vic 3163,      |
- |                       Australia.  E-mail billm@suburbia.net               |
- |                                                                           |
- | Call from C as:                                                           |
- |    int wm_sqrt(FPU_REG *n, unsigned int control_word)                     |
- |                                                                           |
- +---------------------------------------------------------------------------*/
-
-/*---------------------------------------------------------------------------+
- |  wm_sqrt(FPU_REG *n, unsigned int control_word)                           |
- |    returns the square root of n in n.                                     |
- |                                                                           |
- |  Use Newton's method to compute the square root of a number, which must   |
- |  be in the range  [1.0 .. 4.0),  to 64 bits accuracy.                     |
- |  Does not check the sign or tag of the argument.                          |
- |  Sets the exponent, but not the sign or tag of the result.                |
- |                                                                           |
- |  The guess is kept in %esi:%edi                                           |
- +---------------------------------------------------------------------------*/
-
-#include "exception.h"
-#include "fpu_emu.h"
-
-
-#ifndef NON_REENTRANT_FPU
-/*	Local storage on the stack: */
-#define FPU_accum_3	-4(%ebp)	/* ms word */
-#define FPU_accum_2	-8(%ebp)
-#define FPU_accum_1	-12(%ebp)
-#define FPU_accum_0	-16(%ebp)
-
-/*
- * The de-normalised argument:
- *                  sq_2                  sq_1              sq_0
- *        b b b b b b b ... b b b   b b b .... b b b   b 0 0 0 ... 0
- *           ^ binary point here
- */
-#define FPU_fsqrt_arg_2	-20(%ebp)	/* ms word */
-#define FPU_fsqrt_arg_1	-24(%ebp)
-#define FPU_fsqrt_arg_0	-28(%ebp)	/* ls word, at most the ms bit is set */
-
-#else
-/*	Local storage in a static area: */
-.data
-	.align 4,0
-FPU_accum_3:
-	.long	0		/* ms word */
-FPU_accum_2:
-	.long	0
-FPU_accum_1:
-	.long	0
-FPU_accum_0:
-	.long	0
-
-/* The de-normalised argument:
-                    sq_2                  sq_1              sq_0
-          b b b b b b b ... b b b   b b b .... b b b   b 0 0 0 ... 0
-             ^ binary point here
- */
-FPU_fsqrt_arg_2:
-	.long	0		/* ms word */
-FPU_fsqrt_arg_1:
-	.long	0
-FPU_fsqrt_arg_0:
-	.long	0		/* ls word, at most the ms bit is set */
-#endif /* NON_REENTRANT_FPU */ 
-
-
-.text
-SYM_FUNC_START(wm_sqrt)
-	pushl	%ebp
-	movl	%esp,%ebp
-#ifndef NON_REENTRANT_FPU
-	subl	$28,%esp
-#endif /* NON_REENTRANT_FPU */
-	pushl	%esi
-	pushl	%edi
-	pushl	%ebx
-
-	movl	PARAM1,%esi
-
-	movl	SIGH(%esi),%eax
-	movl	SIGL(%esi),%ecx
-	xorl	%edx,%edx
-
-/* We use a rough linear estimate for the first guess.. */
-
-	cmpw	EXP_BIAS,EXP(%esi)
-	jnz	sqrt_arg_ge_2
-
-	shrl	$1,%eax			/* arg is in the range  [1.0 .. 2.0) */
-	rcrl	$1,%ecx
-	rcrl	$1,%edx
-
-sqrt_arg_ge_2:
-/* From here on, n is never accessed directly again until it is
-   replaced by the answer. */
-
-	movl	%eax,FPU_fsqrt_arg_2		/* ms word of n */
-	movl	%ecx,FPU_fsqrt_arg_1
-	movl	%edx,FPU_fsqrt_arg_0
-
-/* Make a linear first estimate */
-	shrl	$1,%eax
-	addl	$0x40000000,%eax
-	movl	$0xaaaaaaaa,%ecx
-	mull	%ecx
-	shll	%edx			/* max result was 7fff... */
-	testl	$0x80000000,%edx	/* but min was 3fff... */
-	jnz	sqrt_prelim_no_adjust
-
-	movl	$0x80000000,%edx	/* round up */
-
-sqrt_prelim_no_adjust:
-	movl	%edx,%esi	/* Our first guess */
-
-/* We have now computed (approx)   (2 + x) / 3, which forms the basis
-   for a few iterations of Newton's method */
-
-	movl	FPU_fsqrt_arg_2,%ecx	/* ms word */
-
-/*
- * From our initial estimate, three iterations are enough to get us
- * to 30 bits or so. This will then allow two iterations at better
- * precision to complete the process.
- */
-
-/* Compute  (g + n/g)/2  at each iteration (g is the guess). */
-	shrl	%ecx		/* Doing this first will prevent a divide */
-				/* overflow later. */
-
-	movl	%ecx,%edx	/* msw of the arg / 2 */
-	divl	%esi		/* current estimate */
-	shrl	%esi		/* divide by 2 */
-	addl	%eax,%esi	/* the new estimate */
-
-	movl	%ecx,%edx
-	divl	%esi
-	shrl	%esi
-	addl	%eax,%esi
-
-	movl	%ecx,%edx
-	divl	%esi
-	shrl	%esi
-	addl	%eax,%esi
-
-/*
- * Now that an estimate accurate to about 30 bits has been obtained (in %esi),
- * we improve it to 60 bits or so.
- *
- * The strategy from now on is to compute new estimates from
- *      guess := guess + (n - guess^2) / (2 * guess)
- */
-
-/* First, find the square of the guess */
-	movl	%esi,%eax
-	mull	%esi
-/* guess^2 now in %edx:%eax */
-
-	movl	FPU_fsqrt_arg_1,%ecx
-	subl	%ecx,%eax
-	movl	FPU_fsqrt_arg_2,%ecx	/* ms word of normalized n */
-	sbbl	%ecx,%edx
-	jnc	sqrt_stage_2_positive
-
-/* Subtraction gives a negative result,
-   negate the result before division. */
-	notl	%edx
-	notl	%eax
-	addl	$1,%eax
-	adcl	$0,%edx
-
-	divl	%esi
-	movl	%eax,%ecx
-
-	movl	%edx,%eax
-	divl	%esi
-	jmp	sqrt_stage_2_finish
-
-sqrt_stage_2_positive:
-	divl	%esi
-	movl	%eax,%ecx
-
-	movl	%edx,%eax
-	divl	%esi
-
-	notl	%ecx
-	notl	%eax
-	addl	$1,%eax
-	adcl	$0,%ecx
-
-sqrt_stage_2_finish:
-	sarl	$1,%ecx		/* divide by 2 */
-	rcrl	$1,%eax
-
-	/* Form the new estimate in %esi:%edi */
-	movl	%eax,%edi
-	addl	%ecx,%esi
-
-	jnz	sqrt_stage_2_done	/* result should be [1..2) */
-
-#ifdef PARANOID
-/* It should be possible to get here only if the arg is ffff....ffff */
-	cmpl	$0xffffffff,FPU_fsqrt_arg_1
-	jnz	sqrt_stage_2_error
-#endif /* PARANOID */
-
-/* The best rounded result. */
-	xorl	%eax,%eax
-	decl	%eax
-	movl	%eax,%edi
-	movl	%eax,%esi
-	movl	$0x7fffffff,%eax
-	jmp	sqrt_round_result
-
-#ifdef PARANOID
-sqrt_stage_2_error:
-	pushl	EX_INTERNAL|0x213
-	call	EXCEPTION
-#endif /* PARANOID */ 
-
-sqrt_stage_2_done:
-
-/* Now the square root has been computed to better than 60 bits. */
-
-/* Find the square of the guess. */
-	movl	%edi,%eax		/* ls word of guess */
-	mull	%edi
-	movl	%edx,FPU_accum_1
-
-	movl	%esi,%eax
-	mull	%esi
-	movl	%edx,FPU_accum_3
-	movl	%eax,FPU_accum_2
-
-	movl	%edi,%eax
-	mull	%esi
-	addl	%eax,FPU_accum_1
-	adcl	%edx,FPU_accum_2
-	adcl	$0,FPU_accum_3
-
-/*	movl	%esi,%eax */
-/*	mull	%edi */
-	addl	%eax,FPU_accum_1
-	adcl	%edx,FPU_accum_2
-	adcl	$0,FPU_accum_3
-
-/* guess^2 now in FPU_accum_3:FPU_accum_2:FPU_accum_1 */
-
-	movl	FPU_fsqrt_arg_0,%eax		/* get normalized n */
-	subl	%eax,FPU_accum_1
-	movl	FPU_fsqrt_arg_1,%eax
-	sbbl	%eax,FPU_accum_2
-	movl	FPU_fsqrt_arg_2,%eax		/* ms word of normalized n */
-	sbbl	%eax,FPU_accum_3
-	jnc	sqrt_stage_3_positive
-
-/* Subtraction gives a negative result,
-   negate the result before division */
-	notl	FPU_accum_1
-	notl	FPU_accum_2
-	notl	FPU_accum_3
-	addl	$1,FPU_accum_1
-	adcl	$0,FPU_accum_2
-
-#ifdef PARANOID
-	adcl	$0,FPU_accum_3	/* This must be zero */
-	jz	sqrt_stage_3_no_error
-
-sqrt_stage_3_error:
-	pushl	EX_INTERNAL|0x207
-	call	EXCEPTION
-
-sqrt_stage_3_no_error:
-#endif /* PARANOID */
-
-	movl	FPU_accum_2,%edx
-	movl	FPU_accum_1,%eax
-	divl	%esi
-	movl	%eax,%ecx
-
-	movl	%edx,%eax
-	divl	%esi
-
-	sarl	$1,%ecx		/* divide by 2 */
-	rcrl	$1,%eax
-
-	/* prepare to round the result */
-
-	addl	%ecx,%edi
-	adcl	$0,%esi
-
-	jmp	sqrt_stage_3_finished
-
-sqrt_stage_3_positive:
-	movl	FPU_accum_2,%edx
-	movl	FPU_accum_1,%eax
-	divl	%esi
-	movl	%eax,%ecx
-
-	movl	%edx,%eax
-	divl	%esi
-
-	sarl	$1,%ecx		/* divide by 2 */
-	rcrl	$1,%eax
-
-	/* prepare to round the result */
-
-	notl	%eax		/* Negate the correction term */
-	notl	%ecx
-	addl	$1,%eax
-	adcl	$0,%ecx		/* carry here ==> correction == 0 */
-	adcl	$0xffffffff,%esi
-
-	addl	%ecx,%edi
-	adcl	$0,%esi
-
-sqrt_stage_3_finished:
-
-/*
- * The result in %esi:%edi:%esi should be good to about 90 bits here,
- * and the rounding information here does not have sufficient accuracy
- * in a few rare cases.
- */
-	cmpl	$0xffffffe0,%eax
-	ja	sqrt_near_exact_x
-
-	cmpl	$0x00000020,%eax
-	jb	sqrt_near_exact
-
-	cmpl	$0x7fffffe0,%eax
-	jb	sqrt_round_result
-
-	cmpl	$0x80000020,%eax
-	jb	sqrt_get_more_precision
-
-sqrt_round_result:
-/* Set up for rounding operations */
-	movl	%eax,%edx
-	movl	%esi,%eax
-	movl	%edi,%ebx
-	movl	PARAM1,%edi
-	movw	EXP_BIAS,EXP(%edi)	/* Result is in  [1.0 .. 2.0) */
-	jmp	fpu_reg_round
-
-
-sqrt_near_exact_x:
-/* First, the estimate must be rounded up. */
-	addl	$1,%edi
-	adcl	$0,%esi
-
-sqrt_near_exact:
-/*
- * This is an easy case because x^1/2 is monotonic.
- * We need just find the square of our estimate, compare it
- * with the argument, and deduce whether our estimate is
- * above, below, or exact. We use the fact that the estimate
- * is known to be accurate to about 90 bits.
- */
-	movl	%edi,%eax		/* ls word of guess */
-	mull	%edi
-	movl	%edx,%ebx		/* 2nd ls word of square */
-	movl	%eax,%ecx		/* ls word of square */
-
-	movl	%edi,%eax
-	mull	%esi
-	addl	%eax,%ebx
-	addl	%eax,%ebx
-
-#ifdef PARANOID
-	cmp	$0xffffffb0,%ebx
-	jb	sqrt_near_exact_ok
-
-	cmp	$0x00000050,%ebx
-	ja	sqrt_near_exact_ok
-
-	pushl	EX_INTERNAL|0x214
-	call	EXCEPTION
-
-sqrt_near_exact_ok:
-#endif /* PARANOID */ 
-
-	or	%ebx,%ebx
-	js	sqrt_near_exact_small
-
-	jnz	sqrt_near_exact_large
-
-	or	%ebx,%edx
-	jnz	sqrt_near_exact_large
-
-/* Our estimate is exactly the right answer */
-	xorl	%eax,%eax
-	jmp	sqrt_round_result
-
-sqrt_near_exact_small:
-/* Our estimate is too small */
-	movl	$0x000000ff,%eax
-	jmp	sqrt_round_result
-	
-sqrt_near_exact_large:
-/* Our estimate is too large, we need to decrement it */
-	subl	$1,%edi
-	sbbl	$0,%esi
-	movl	$0xffffff00,%eax
-	jmp	sqrt_round_result
-
-
-sqrt_get_more_precision:
-/* This case is almost the same as the above, except we start
-   with an extra bit of precision in the estimate. */
-	stc			/* The extra bit. */
-	rcll	$1,%edi		/* Shift the estimate left one bit */
-	rcll	$1,%esi
-
-	movl	%edi,%eax		/* ls word of guess */
-	mull	%edi
-	movl	%edx,%ebx		/* 2nd ls word of square */
-	movl	%eax,%ecx		/* ls word of square */
-
-	movl	%edi,%eax
-	mull	%esi
-	addl	%eax,%ebx
-	addl	%eax,%ebx
-
-/* Put our estimate back to its original value */
-	stc			/* The ms bit. */
-	rcrl	$1,%esi		/* Shift the estimate left one bit */
-	rcrl	$1,%edi
-
-#ifdef PARANOID
-	cmp	$0xffffff60,%ebx
-	jb	sqrt_more_prec_ok
-
-	cmp	$0x000000a0,%ebx
-	ja	sqrt_more_prec_ok
-
-	pushl	EX_INTERNAL|0x215
-	call	EXCEPTION
-
-sqrt_more_prec_ok:
-#endif /* PARANOID */ 
-
-	or	%ebx,%ebx
-	js	sqrt_more_prec_small
-
-	jnz	sqrt_more_prec_large
-
-	or	%ebx,%ecx
-	jnz	sqrt_more_prec_large
-
-/* Our estimate is exactly the right answer */
-	movl	$0x80000000,%eax
-	jmp	sqrt_round_result
-
-sqrt_more_prec_small:
-/* Our estimate is too small */
-	movl	$0x800000ff,%eax
-	jmp	sqrt_round_result
-	
-sqrt_more_prec_large:
-/* Our estimate is too large */
-	movl	$0x7fffff00,%eax
-	jmp	sqrt_round_result
-SYM_FUNC_END(wm_sqrt)
diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c
index 6b9ff1c6cafa2..ceb8d03191ab2 100644
--- a/arch/x86/mm/extable.c
+++ b/arch/x86/mm/extable.c
@@ -322,7 +322,7 @@ int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code,
 
 	type = FIELD_GET(EX_DATA_TYPE_MASK, e->data);
 	reg  = FIELD_GET(EX_DATA_REG_MASK,  e->data);
-	imm  = FIELD_GET(EX_DATA_IMM_MASK,  e->data);
+	imm  = FIELD_GET_SIGNED(EX_DATA_IMM_MASK, e->data);
 
 	switch (type) {
 	case EX_TYPE_DEFAULT:
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 63de8e8684f23..250942ef60bb3 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -169,6 +169,11 @@ is_prefetch(struct pt_regs *regs, unsigned long error_code, unsigned long addr)
 }
 
 DEFINE_SPINLOCK(pgd_lock);
+/*
+ * List of all process' PGD pages (excluding init_mm and some other special
+ * system-wide PGDs). Mainly used to keep the kernel portion of the address
+ * space in sync across mms.
+ */
 LIST_HEAD(pgd_list);
 
 #ifdef CONFIG_X86_32
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index df2261fa4f985..77b889b71cf38 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -1288,12 +1288,13 @@ kernel_physical_mapping_remove(unsigned long start, unsigned long end)
 	remove_pagetable(start, end, true, NULL);
 }
 
-void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap)
+void __ref arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+			      struct dev_pagemap *pgmap)
 {
 	unsigned long start_pfn = start >> PAGE_SHIFT;
 	unsigned long nr_pages = size >> PAGE_SHIFT;
 
-	__remove_pages(start_pfn, nr_pages, altmap);
+	__remove_pages(start_pfn, nr_pages, altmap, pgmap);
 	kernel_physical_mapping_remove(start, start + size);
 }
 #endif /* CONFIG_MEMORY_HOTPLUG */
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index da7f0a03cf90c..f32facdb30354 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -99,17 +99,6 @@ static void pgd_dtor(pgd_t *pgd)
 	spin_unlock(&pgd_lock);
 }
 
-/*
- * List of all pgd's needed for non-PAE so it can invalidate entries
- * in both cached and uncached pgd's; not needed for PAE since the
- * kernel pmd is shared. If PAE were not to share the pmd a similar
- * tactic would be needed. This is essentially codepath-based locking
- * against pageattr.c; it is the unique case in which a valid change
- * of kernel pagetables can't be lazily synchronized by vmalloc faults.
- * vmalloc faults work because attached pagetables are never freed.
- * -- nyc
- */
-
 #ifdef CONFIG_X86_PAE
 /*
  * In PAE mode, we need to do a cr3 reload (=tlb flush) when
diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c
index 631f0375bd427..598f553cc8713 100644
--- a/arch/x86/mm/pti.c
+++ b/arch/x86/mm/pti.c
@@ -31,6 +31,7 @@
 
 #include <asm/cpufeature.h>
 #include <asm/hypervisor.h>
+#include <asm/cpuid/api.h>
 #include <asm/vsyscall.h>
 #include <asm/cmdline.h>
 #include <asm/pti.h>
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index af43d177087e7..bfe286ec3ac83 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -26,6 +26,8 @@
 
 #include "mm_internal.h"
 
+DEFINE_STATIC_KEY_FALSE(tlb_ipi_broadcast_key);
+
 #ifdef CONFIG_PARAVIRT
 # define STATIC_NOPV
 #else
@@ -1339,16 +1341,16 @@ STATIC_NOPV void native_flush_tlb_multi(const struct cpumask *cpumask,
 				(info->end - info->start) >> PAGE_SHIFT);
 
 	/*
-	 * If no page tables were freed, we can skip sending IPIs to
-	 * CPUs in lazy TLB mode. They will flush the CPU themselves
-	 * at the next context switch.
+	 * If lazy-TLB CPUs do not need to be woken, we can skip sending
+	 * IPIs to them. They will flush themselves at the next context
+	 * switch.
 	 *
-	 * However, if page tables are getting freed, we need to send the
-	 * IPI everywhere, to prevent CPUs in lazy TLB mode from tripping
-	 * up on the new contents of what used to be page tables, while
-	 * doing a speculative memory access.
+	 * However, if page tables are getting freed or unshared, we need
+	 * to send the IPI everywhere, to prevent CPUs in lazy TLB mode
+	 * from tripping up on the new contents of what used to be page
+	 * tables, while doing a speculative memory access.
 	 */
-	if (info->freed_tables || mm_in_asid_transition(info->mm))
+	if (info->wake_lazy_cpus || mm_in_asid_transition(info->mm))
 		on_each_cpu_mask(cpumask, flush_tlb_func, (void *)info, true);
 	else
 		on_each_cpu_cond_mask(should_flush_tlb, flush_tlb_func,
@@ -1381,7 +1383,7 @@ static DEFINE_PER_CPU(unsigned int, flush_tlb_info_idx);
 
 static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm,
 			unsigned long start, unsigned long end,
-			unsigned int stride_shift, bool freed_tables,
+			unsigned int stride_shift, bool wake_lazy_cpus,
 			u64 new_tlb_gen)
 {
 	struct flush_tlb_info *info = this_cpu_ptr(&flush_tlb_info);
@@ -1408,7 +1410,7 @@ static struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm,
 	info->end		= end;
 	info->mm		= mm;
 	info->stride_shift	= stride_shift;
-	info->freed_tables	= freed_tables;
+	info->wake_lazy_cpus	= wake_lazy_cpus;
 	info->new_tlb_gen	= new_tlb_gen;
 	info->initiating_cpu	= smp_processor_id();
 	info->trim_cpumask	= 0;
@@ -1427,7 +1429,7 @@ static void put_flush_tlb_info(void)
 
 void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
 				unsigned long end, unsigned int stride_shift,
-				bool freed_tables)
+				bool wake_lazy_cpus)
 {
 	struct flush_tlb_info *info;
 	int cpu = get_cpu();
@@ -1436,7 +1438,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
 	/* This is also a barrier that synchronizes with switch_mm(). */
 	new_tlb_gen = inc_mm_tlb_gen(mm);
 
-	info = get_flush_tlb_info(mm, start, end, stride_shift, freed_tables,
+	info = get_flush_tlb_info(mm, start, end, stride_shift, wake_lazy_cpus,
 				  new_tlb_gen);
 
 	/*
@@ -1813,3 +1815,16 @@ static int __init create_tlb_single_page_flush_ceiling(void)
 	return 0;
 }
 late_initcall(create_tlb_single_page_flush_ceiling);
+
+void __init native_pv_tlb_init(void)
+{
+#ifdef CONFIG_PARAVIRT
+	if (pv_ops.mmu.flush_tlb_multi != native_flush_tlb_multi)
+		return;
+#endif
+
+	if (cpu_feature_enabled(X86_FEATURE_INVLPGB))
+		return;
+
+	static_branch_enable(&tlb_ipi_broadcast_key);
+}
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index ea9e707e8abff..a0c541a441cfa 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -10,6 +10,7 @@
 #include <linux/if_vlan.h>
 #include <linux/bitfield.h>
 #include <linux/bpf.h>
+#include <linux/bpf_verifier.h>
 #include <linux/memory.h>
 #include <linux/sort.h>
 #include <asm/extable.h>
@@ -390,6 +391,34 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used)
 	*pprog = prog;
 }
 
+/* add rsp, depth */
+static void emit_add_rsp(u8 **pprog, u16 depth)
+{
+	u8 *prog = *pprog;
+
+	if (!depth)
+		return;
+	if (is_imm8(depth))
+		EMIT4(0x48, 0x83, 0xC4, depth); /* add rsp, imm8 */
+	else
+		EMIT3_off32(0x48, 0x81, 0xC4, depth); /* add rsp, imm32 */
+	*pprog = prog;
+}
+
+/* sub rsp, depth */
+static void emit_sub_rsp(u8 **pprog, u16 depth)
+{
+	u8 *prog = *pprog;
+
+	if (!depth)
+		return;
+	if (is_imm8(depth))
+		EMIT4(0x48, 0x83, 0xEC, depth); /* sub rsp, imm8 */
+	else
+		EMIT3_off32(0x48, 0x81, 0xEC, depth); /* sub rsp, imm32 */
+	*pprog = prog;
+}
+
 static void emit_nops(u8 **pprog, int len)
 {
 	u8 *prog = *pprog;
@@ -1659,21 +1688,47 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 	bool seen_exit = false;
 	u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
 	void __percpu *priv_frame_ptr = NULL;
+	u16 out_stack_arg_cnt, outgoing_rsp;
 	u64 arena_vm_start, user_vm_start;
 	void __percpu *priv_stack_ptr;
 	int i, excnt = 0;
 	int ilen, proglen = 0;
 	u8 *ip, *prog = temp;
 	u32 stack_depth;
+	int callee_saved_size;
+	s32 outgoing_arg_base;
 	int err;
 
 	stack_depth = bpf_prog->aux->stack_depth;
+	out_stack_arg_cnt = bpf_out_stack_arg_cnt(env, bpf_prog);
 	priv_stack_ptr = bpf_prog->aux->priv_stack_ptr;
 	if (priv_stack_ptr) {
 		priv_frame_ptr = priv_stack_ptr + PRIV_STACK_GUARD_SZ + round_up(stack_depth, 8);
 		stack_depth = 0;
 	}
 
+	/*
+	 * Follow x86-64 calling convention for both BPF-to-BPF and
+	 * kfunc calls:
+	 *   - Arg 6 is passed in R9 register
+	 *   - Args 7+ are passed on the stack at [rsp]
+	 *
+	 * Incoming arg 6 is read from R9 (BPF r11+8 → MOV from R9).
+	 * Incoming args 7+ are read from [rbp + 16], [rbp + 24], ...
+	 * (BPF r11+16, r11+24, ... map directly with no offset change).
+	 *
+	 * tail_call_reachable is rejected by the verifier and priv_stack
+	 * is disabled by the JIT when stack args exist, so R9 is always
+	 * available.
+	 *
+	 * Stack layout (high to low):
+	 *   [rbp + 16 + ...]    incoming stack args 7+ (from caller)
+	 *   [rbp + 8]           return address
+	 *   [rbp]               saved rbp
+	 *   [rbp - prog_stack]  program stack
+	 *   [below]             callee-saved regs
+	 *   [below]             outgoing args 7+ (= rsp)
+	 */
 	arena_vm_start = bpf_arena_get_kern_vm_start(bpf_prog->aux->arena);
 	user_vm_start = bpf_arena_get_user_vm_start(bpf_prog->aux->arena);
 
@@ -1700,6 +1755,44 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 			push_r12(&prog);
 		push_callee_regs(&prog, callee_regs_used);
 	}
+
+	/* Compute callee-saved register area size. */
+	callee_saved_size = 0;
+	if (bpf_prog->aux->exception_boundary || arena_vm_start)
+		callee_saved_size += 8; /* r12 */
+	if (bpf_prog->aux->exception_boundary) {
+		callee_saved_size += 4 * 8; /* rbx, r13, r14, r15 */
+	} else {
+		int j;
+
+		for (j = 0; j < 4; j++)
+			if (callee_regs_used[j])
+				callee_saved_size += 8;
+	}
+	/*
+	 * Base offset from rbp for translating BPF outgoing args 7+
+	 * to native offsets. BPF uses negative offsets from r11
+	 * (r11-8 for arg6, r11-16 for arg7, ...) while x86 uses
+	 * positive offsets from rsp ([rsp+0] for arg7, [rsp+8] for
+	 * arg8, ...). Arg 6 goes to R9 directly.
+	 *
+	 * The translation reverses direction:
+	 *   native_off = outgoing_arg_base - outgoing_rsp - bpf_off - 16
+	 *
+	 * Note that tail_call_reachable is guaranteed to be false when
+	 * stack args exist, so tcc pushes need not be accounted for.
+	 */
+	outgoing_arg_base = -(round_up(stack_depth, 8) + callee_saved_size);
+
+	/*
+	 * Allocate outgoing stack arg area for args 7+ only.
+	 * Arg 6 goes into r9 register, not on stack.
+	 */
+	outgoing_rsp = out_stack_arg_cnt > 1 ? (out_stack_arg_cnt - 1) * 8 : 0;
+	if (bpf_prog->aux->exception_boundary)
+		bpf_prog->aux->stack_arg_sp_adjust = outgoing_rsp;
+	emit_sub_rsp(&prog, outgoing_rsp);
+
 	if (arena_vm_start)
 		emit_mov_imm64(&prog, X86_REG_R12,
 			       arena_vm_start >> 32, (u32) arena_vm_start);
@@ -1721,7 +1814,7 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 		u8 b2 = 0, b3 = 0;
 		u8 *start_of_ldx;
 		s64 jmp_offset;
-		s16 insn_off;
+		s32 insn_off;
 		u8 jmp_cond;
 		u8 *func;
 		int nops;
@@ -2134,12 +2227,27 @@ static int do_jit(struct bpf_verifier_env *env, struct bpf_prog *bpf_prog, int *
 				EMIT1(0xC7);
 			goto st;
 		case BPF_ST | BPF_MEM | BPF_DW:
+			if (dst_reg == BPF_REG_PARAMS && insn->off == -8) {
+				/* Arg 6: store immediate in r9 register */
+				emit_mov_imm64(&prog, X86_REG_R9, imm32 >> 31, (u32)imm32);
+				break;
+			}
 			EMIT2(add_1mod(0x48, dst_reg), 0xC7);
 
-st:			if (is_imm8(insn->off))
-				EMIT2(add_1reg(0x40, dst_reg), insn->off);
+st:			insn_off = insn->off;
+			if (dst_reg == BPF_REG_PARAMS) {
+				/*
+				 * Args 7+: reverse BPF negative offsets to
+				 * x86 positive rsp offsets.
+				 * BPF off=-16 → [rsp+0], off=-24 → [rsp+8], ...
+				 */
+				insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16;
+				dst_reg = BPF_REG_FP;
+			}
+			if (is_imm8(insn_off))
+				EMIT2(add_1reg(0x40, dst_reg), insn_off);
 			else
-				EMIT1_off32(add_1reg(0x80, dst_reg), insn->off);
+				EMIT1_off32(add_1reg(0x80, dst_reg), insn_off);
 
 			EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
 			break;
@@ -2149,7 +2257,17 @@ st:			if (is_imm8(insn->off))
 		case BPF_STX | BPF_MEM | BPF_H:
 		case BPF_STX | BPF_MEM | BPF_W:
 		case BPF_STX | BPF_MEM | BPF_DW:
-			emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn->off);
+			if (dst_reg == BPF_REG_PARAMS && insn->off == -8) {
+				/* Arg 6: store register value in r9 */
+				EMIT_mov(X86_REG_R9, src_reg);
+				break;
+			}
+			insn_off = insn->off;
+			if (dst_reg == BPF_REG_PARAMS) {
+				insn_off = outgoing_arg_base - outgoing_rsp - insn_off - 16;
+				dst_reg = BPF_REG_FP;
+			}
+			emit_stx(&prog, BPF_SIZE(insn->code), dst_reg, src_reg, insn_off);
 			break;
 
 		case BPF_ST | BPF_PROBE_MEM32 | BPF_B:
@@ -2248,6 +2366,19 @@ st:			if (is_imm8(insn->off))
 		case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
 		case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
 			insn_off = insn->off;
+			if (src_reg == BPF_REG_PARAMS) {
+				if (insn_off == 8) {
+					/* Incoming arg 6: read from r9 */
+					EMIT_mov(dst_reg, X86_REG_R9);
+					break;
+				}
+				src_reg = BPF_REG_FP;
+				/*
+				 * Incoming args 7+: native_off == bpf_off
+				 * (r11+16 → [rbp+16], r11+24 → [rbp+24], ...)
+				 * No offset adjustment needed.
+				 */
+			}
 
 			if (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
 			    BPF_MODE(insn->code) == BPF_PROBE_MEMSX) {
@@ -2736,6 +2867,8 @@ st:			if (is_imm8(insn->off))
 				if (emit_spectre_bhb_barrier(&prog, ip, bpf_prog))
 					return -EINVAL;
 			}
+			/* Deallocate outgoing args 7+ area. */
+			emit_add_rsp(&prog, outgoing_rsp);
 			if (bpf_prog->aux->exception_boundary) {
 				pop_callee_regs(&prog, all_callee_regs_used);
 				pop_r12(&prog);
@@ -3793,7 +3926,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_pr
 	for (pass = 0; pass < MAX_PASSES || image; pass++) {
 		if (!padding && pass >= PADDING_PASSES)
 			padding = true;
-		proglen = do_jit(env, prog, addrs, image, rw_image, oldproglen, &ctx, padding);
+		proglen = do_jit(env, prog, addrs, image, rw_image, oldproglen,
+				 &ctx, padding);
 		if (proglen <= 0) {
 out_image:
 			image = NULL;
@@ -3910,6 +4044,11 @@ bool bpf_jit_supports_kfunc_call(void)
 	return true;
 }
 
+bool bpf_jit_supports_stack_args(void)
+{
+	return true;
+}
+
 void *bpf_arch_text_copy(void *dst, void *src, size_t len)
 {
 	if (text_poke_copy(dst, src, len) == NULL)
diff --git a/arch/x86/pci/xen.c b/arch/x86/pci/xen.c
index 6818515a501b9..550c631bc77f3 100644
--- a/arch/x86/pci/xen.c
+++ b/arch/x86/pci/xen.c
@@ -18,6 +18,7 @@
 #include <linux/io.h>
 #include <asm/io_apic.h>
 #include <asm/pci_x86.h>
+#include <asm/cpuid/api.h>
 
 #include <asm/xen/hypervisor.h>
 
diff --git a/arch/x86/platform/olpc/olpc-xo15-sci.c b/arch/x86/platform/olpc/olpc-xo15-sci.c
index 82c51b6ec5288..e486109e88c87 100644
--- a/arch/x86/platform/olpc/olpc-xo15-sci.c
+++ b/arch/x86/platform/olpc/olpc-xo15-sci.c
@@ -9,6 +9,7 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/workqueue.h>
+#include <linux/platform_device.h>
 #include <linux/power_supply.h>
 #include <linux/olpc-ec.h>
 
@@ -136,14 +137,16 @@ static u32 xo15_sci_gpe_handler(acpi_handle gpe_device, u32 gpe, void *context)
 	return ACPI_INTERRUPT_HANDLED | ACPI_REENABLE_GPE;
 }
 
-static int xo15_sci_add(struct acpi_device *device)
+static int xo15_sci_probe(struct platform_device *pdev)
 {
+	struct acpi_device *device;
 	unsigned long long tmp;
 	acpi_status status;
 	int r;
 
+	device = ACPI_COMPANION(&pdev->dev);
 	if (!device)
-		return -EINVAL;
+		return -ENODEV;
 
 	strscpy(acpi_device_name(device), XO15_SCI_DEVICE_NAME);
 	strscpy(acpi_device_class(device), XO15_SCI_CLASS);
@@ -160,7 +163,7 @@ static int xo15_sci_add(struct acpi_device *device)
 	if (ACPI_FAILURE(status))
 		return -ENODEV;
 
-	dev_info(&device->dev, "Initialized, GPE = 0x%lx\n", xo15_sci_gpe);
+	dev_info(&pdev->dev, "Initialized, GPE = 0x%lx\n", xo15_sci_gpe);
 
 	r = sysfs_create_file(&device->dev.kobj, &lid_wake_on_close_attr.attr);
 	if (r)
@@ -174,7 +177,7 @@ static int xo15_sci_add(struct acpi_device *device)
 
 	/* Enable wake-on-EC */
 	if (device->wakeup.flags.valid)
-		device_init_wakeup(&device->dev, true);
+		device_init_wakeup(&pdev->dev, true);
 
 	return 0;
 
@@ -184,8 +187,11 @@ static int xo15_sci_add(struct acpi_device *device)
 	return r;
 }
 
-static void xo15_sci_remove(struct acpi_device *device)
+static void xo15_sci_remove(struct platform_device *pdev)
 {
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+
+	device_init_wakeup(&pdev->dev, false);
 	acpi_disable_gpe(NULL, xo15_sci_gpe);
 	acpi_remove_gpe_handler(NULL, xo15_sci_gpe, xo15_sci_gpe_handler);
 	cancel_work_sync(&sci_work);
@@ -213,19 +219,18 @@ static const struct acpi_device_id xo15_sci_device_ids[] = {
 	{"", 0},
 };
 
-static struct acpi_driver xo15_sci_drv = {
-	.name = DRV_NAME,
-	.class = XO15_SCI_CLASS,
-	.ids = xo15_sci_device_ids,
-	.ops = {
-		.add = xo15_sci_add,
-		.remove = xo15_sci_remove,
+static struct platform_driver xo15_sci_drv = {
+	.probe = xo15_sci_probe,
+	.remove = xo15_sci_remove,
+	.driver = {
+		.name = DRV_NAME,
+		.acpi_match_table = xo15_sci_device_ids,
+		.pm = &xo15_sci_pm,
 	},
-	.drv.pm = &xo15_sci_pm,
 };
 
 static int __init xo15_sci_init(void)
 {
-	return acpi_bus_register_driver(&xo15_sci_drv);
+	return platform_driver_register(&xo15_sci_drv);
 }
 device_initcall(xo15_sci_init);
diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c
index f647557d38ac5..7e9091c640be0 100644
--- a/arch/x86/virt/hw.c
+++ b/arch/x86/virt/hw.c
@@ -49,7 +49,20 @@ static void x86_virt_invoke_kvm_emergency_callback(void)
 {
 	cpu_emergency_virt_cb *kvm_callback;
 
-	kvm_callback = rcu_dereference(kvm_emergency_callback);
+	/*
+	 * RCU may not be watching the crashing CPU here, so rcu_dereference()
+	 * triggers a suspicious-RCU-usage splat. In principle, a concurrent
+	 * KVM module unload could race with this read; see commit 2baa33a8ddd6
+	 * ("KVM: x86: Leave user-return notifier registered on reboot/shutdown")
+	 * which notes that nothing prevents module unload during panic/reboot.
+	 *
+	 * However, taking a lock here would be riskier than the current race:
+	 * the system is going down via NMI shootdown, and any lock could be
+	 * held by an already-stopped CPU. Use rcu_dereference_raw() to silence
+	 * the lockdep splat and accept the comically small remaining race;
+	 * panic context inherently cannot guarantee complete correctness.
+	 */
+	kvm_callback = rcu_dereference_raw(kvm_emergency_callback);
 	if (kvm_callback)
 		kvm_callback();
 }
diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c
index 41f76f15caa13..8bcdce98f6dce 100644
--- a/arch/x86/virt/svm/sev.c
+++ b/arch/x86/virt/svm/sev.c
@@ -511,8 +511,9 @@ static void clear_hsave_pa(void *arg)
 	wrmsrq(MSR_VM_HSAVE_PA, 0);
 }
 
-void snp_prepare(void)
+int snp_prepare(void)
 {
+	int ret;
 	u64 val;
 
 	/*
@@ -521,12 +522,20 @@ void snp_prepare(void)
 	 */
 	rdmsrq(MSR_AMD64_SYSCFG, val);
 	if (val & MSR_AMD64_SYSCFG_SNP_EN)
-		return;
+		return 0;
 
 	clear_rmp();
 
 	cpus_read_lock();
 
+	if (!cpumask_equal(cpu_online_mask, cpu_present_mask)) {
+		ret = -EOPNOTSUPP;
+		pr_warn("SNP init failed: not all CPUs online. (%*pbl online <-> %*pbl present masks).\n",
+			cpumask_pr_args(cpu_online_mask),
+			cpumask_pr_args(cpu_present_mask));
+		goto unlock;
+	}
+
 	/*
 	 * MtrrFixDramModEn is not shared between threads on a core,
 	 * therefore it must be set on all CPUs prior to enabling SNP.
@@ -537,7 +546,12 @@ void snp_prepare(void)
 	/* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */
 	on_each_cpu(clear_hsave_pa, NULL, 1);
 
+	ret = 0;
+
+unlock:
 	cpus_read_unlock();
+
+	return ret;
 }
 EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp");
 
diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c
index cb9b3210ab710..413fa63fb5497 100644
--- a/arch/x86/virt/vmx/tdx/tdx.c
+++ b/arch/x86/virt/vmx/tdx/tdx.c
@@ -37,6 +37,7 @@
 #include <asm/msr.h>
 #include <asm/cpufeature.h>
 #include <asm/tdx.h>
+#include <asm/shared/tdx_errno.h>
 #include <asm/cpu_device_id.h>
 #include <asm/processor.h>
 #include <asm/mce.h>
@@ -184,6 +185,17 @@ static int tdx_online_cpu(unsigned int cpu)
 	return ret;
 }
 
+static void tdx_cpu_flush_cache(void)
+{
+	lockdep_assert_preemption_disabled();
+
+	if (!this_cpu_read(cache_state_incoherent))
+		return;
+
+	wbinvd();
+	this_cpu_write(cache_state_incoherent, false);
+}
+
 static int tdx_offline_cpu(unsigned int cpu)
 {
 	int i;
@@ -220,17 +232,34 @@ static int tdx_offline_cpu(unsigned int cpu)
 	return -EBUSY;
 
 done:
+	/*
+	 * Flush cache on the CPU going offline to ensure no dirty
+	 * cachelines of TDX private memory remain. This may be
+	 * redundant with WBINVD done elsewhere during CPU offline
+	 * (e.g. hlt_play_dead()), but do it explicitly for safety.
+	 */
+	tdx_cpu_flush_cache();
 	x86_virt_put_ref(X86_FEATURE_VMX);
 	return 0;
 }
 
 static void tdx_shutdown_cpu(void *ign)
 {
+	/*
+	 * Flush cache in preparation for kexec - this is necessary to avoid
+	 * having dirty private memory cachelines when the new kernel boots,
+	 * but WBINVD is a relatively expensive operation and doing it during
+	 * kexec can exacerbate races in native_stop_other_cpus().  Do it
+	 * now, since this is a safe moment and there is going to be no more
+	 * TDX activity on this CPU from this point on.
+	 */
+	tdx_cpu_flush_cache();
 	x86_virt_put_ref(X86_FEATURE_VMX);
 }
 
 static void tdx_shutdown(void *ign)
 {
+	tdx_sys_disable();
 	on_each_cpu(tdx_shutdown_cpu, NULL, 1);
 }
 
@@ -1568,6 +1597,12 @@ static void tdx_clflush_page(struct page *page)
 	clflush_cache_range(page_to_virt(page), PAGE_SIZE);
 }
 
+static int pg_level_to_tdx_sept_level(enum pg_level level)
+{
+	WARN_ON_ONCE(level == PG_LEVEL_NONE);
+	return level - 1;
+}
+
 noinstr u64 tdh_vp_enter(struct tdx_vp *td, struct tdx_module_args *args)
 {
 	args->rcx = td->tdvpr_pa;
@@ -1608,10 +1643,11 @@ u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, struct page *page, struct page
 }
 EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_add);
 
-u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2)
+u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, enum pg_level level,
+		     struct page *page, u64 *ext_err1, u64 *ext_err2)
 {
 	struct tdx_module_args args = {
-		.rcx = gpa | level,
+		.rcx = gpa | pg_level_to_tdx_sept_level(level),
 		.rdx = tdx_tdr_pa(td),
 		.r8 = page_to_phys(page),
 	};
@@ -1639,10 +1675,11 @@ u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page)
 }
 EXPORT_SYMBOL_FOR_KVM(tdh_vp_addcx);
 
-u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u64 *ext_err1, u64 *ext_err2)
+u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, enum pg_level level,
+		     struct page *page, u64 *ext_err1, u64 *ext_err2)
 {
 	struct tdx_module_args args = {
-		.rcx = gpa | level,
+		.rcx = gpa | pg_level_to_tdx_sept_level(level),
 		.rdx = tdx_tdr_pa(td),
 		.r8 = page_to_phys(page),
 	};
@@ -1658,10 +1695,11 @@ u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, int level, struct page *page, u
 }
 EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_aug);
 
-u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, int level, u64 *ext_err1, u64 *ext_err2)
+u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, enum pg_level level,
+			u64 *ext_err1, u64 *ext_err2)
 {
 	struct tdx_module_args args = {
-		.rcx = gpa | level,
+		.rcx = gpa | pg_level_to_tdx_sept_level(level),
 		.rdx = tdx_tdr_pa(td),
 	};
 	u64 ret;
@@ -1874,10 +1912,11 @@ u64 tdh_mem_track(struct tdx_td *td)
 }
 EXPORT_SYMBOL_FOR_KVM(tdh_mem_track);
 
-u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, u64 level, u64 *ext_err1, u64 *ext_err2)
+u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, enum pg_level level,
+			u64 *ext_err1, u64 *ext_err2)
 {
 	struct tdx_module_args args = {
-		.rcx = gpa | level,
+		.rcx = gpa | pg_level_to_tdx_sept_level(level),
 		.rdx = tdx_tdr_pa(td),
 	};
 	u64 ret;
@@ -1921,21 +1960,32 @@ u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, struct page *page)
 }
 EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid);
 
-#ifdef CONFIG_KEXEC_CORE
-void tdx_cpu_flush_cache_for_kexec(void)
+void tdx_sys_disable(void)
 {
-	lockdep_assert_preemption_disabled();
+	struct tdx_module_args args = {};
+	u64 ret;
 
-	if (!this_cpu_read(cache_state_incoherent))
-		return;
+	/*
+	 * Don't loop forever.
+	 *
+	 *  - TDX_INTERRUPTED_RESUMABLE guarantees forward progress between
+	 *    calls.
+	 *
+	 *  - TDX_SYS_BUSY could be returned due to contention with other
+	 *    TDH.SYS.* SEAMCALLs, but will lock out *new* TDH.SYS.* SEAMCALLs,
+	 *    so that SYS.DISABLE can eventually make progress.
+	 *
+	 * This is a 'destructive' SEAMCALL, in that no other SEAMCALL can be
+	 * run after this until a full reinitialization is done.
+	 */
+	do {
+		ret = seamcall(TDH_SYS_DISABLE, &args);
+	} while (ret == TDX_INTERRUPTED_RESUMABLE || ret == TDX_SYS_BUSY);
 
 	/*
-	 * Private memory cachelines need to be clean at the time of
-	 * kexec.  Write them back now, as the caller promises that
-	 * there should be no more SEAMCALLs on this CPU.
+	 * Print SEAMCALL failures, but not SW-defined error codes
+	 * (SEAMCALL faulted with #GP/#UD, TDX not supported).
 	 */
-	wbinvd();
-	this_cpu_write(cache_state_incoherent, false);
+	if (ret && (ret & TDX_SW_ERROR) != TDX_SW_ERROR)
+		pr_err("TDH.SYS.DISABLE failed: 0x%016llx\n", ret);
 }
-EXPORT_SYMBOL_FOR_KVM(tdx_cpu_flush_cache_for_kexec);
-#endif
diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h
index dde219c823b41..e2cf2dd487554 100644
--- a/arch/x86/virt/vmx/tdx/tdx.h
+++ b/arch/x86/virt/vmx/tdx/tdx.h
@@ -46,6 +46,7 @@
 #define TDH_PHYMEM_PAGE_WBINVD		41
 #define TDH_VP_WR			43
 #define TDH_SYS_CONFIG			45
+#define TDH_SYS_DISABLE			69
 
 /*
  * SEAMCALL leaf:
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 23b91bf9b6630..cf061ed45ce84 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -17,7 +17,7 @@
 #include <asm/xen/hypercall.h>
 #include <asm/xen/hypervisor.h>
 #include <asm/cpu.h>
-#include <asm/e820/api.h> 
+#include <asm/e820/api.h>
 #include <asm/setup.h>
 
 #include "xen-ops.h"
@@ -76,6 +76,7 @@ unsigned long xen_released_pages;
 static __ref void xen_get_vendor(void)
 {
 	init_cpu_devs();
+	cpuid_scan_cpu(&boot_cpu_data);
 	cpu_detect(&boot_cpu_data);
 	get_cpu_vendor(&boot_cpu_data);
 }
diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c
index 2f9fa27e5a3c2..2bf05bf3e17b7 100644
--- a/arch/x86/xen/enlighten_hvm.c
+++ b/arch/x86/xen/enlighten_hvm.c
@@ -20,6 +20,7 @@
 #include <asm/setup.h>
 #include <asm/idtentry.h>
 #include <asm/hypervisor.h>
+#include <asm/cpuid/api.h>
 #include <asm/e820/api.h>
 #include <asm/early_ioremap.h>
 
diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
index ed2d7a3756ce8..223e9a2eb2d57 100644
--- a/arch/x86/xen/enlighten_pv.c
+++ b/arch/x86/xen/enlighten_pv.c
@@ -1429,6 +1429,7 @@ asmlinkage __visible void __init xen_start_kernel(struct start_info *si)
 	xen_build_dynamic_phys_to_machine();
 
 	/* Work out if we support NX */
+	cpuid_scan_cpu(&boot_cpu_data);
 	get_cpu_cap(&boot_cpu_data);
 	x86_configure_nx();
 
diff --git a/arch/x86/xen/pmu.c b/arch/x86/xen/pmu.c
index 8f89ce0b67e3b..5f50a3ee08f50 100644
--- a/arch/x86/xen/pmu.c
+++ b/arch/x86/xen/pmu.c
@@ -2,6 +2,7 @@
 #include <linux/types.h>
 #include <linux/interrupt.h>
 
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 #include <asm/xen/hypercall.h>
 #include <xen/xen.h>
diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
index 6f9f665bb7aea..d62c14334b351 100644
--- a/arch/x86/xen/time.c
+++ b/arch/x86/xen/time.c
@@ -18,6 +18,7 @@
 #include <linux/timekeeper_internal.h>
 #include <linux/sched/cputime.h>
 
+#include <asm/cpuid/api.h>
 #include <asm/pvclock.h>
 #include <asm/timer.h>
 #include <asm/xen/hypervisor.h>
diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h
index 56aec6d504fee..6538a29a2bbd4 100644
--- a/arch/xtensa/include/asm/uaccess.h
+++ b/arch/xtensa/include/asm/uaccess.h
@@ -237,8 +237,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 	prefetch(from);
 	return __xtensa_copy_user((__force void *)to, from, n);
 }
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 
 /*
  * We need to return the number of bytes not cleared.  Our memset()
diff --git a/arch/xtensa/platforms/iss/console.c b/arch/xtensa/platforms/iss/console.c
index 8b95221375a89..8e54625cb2ba1 100644
--- a/arch/xtensa/platforms/iss/console.c
+++ b/arch/xtensa/platforms/iss/console.c
@@ -166,8 +166,9 @@ late_initcall(rs_init);
 
 static void iss_console_write(struct console *co, const char *s, unsigned count)
 {
-	if (s && *s != 0)
-		simc_write(1, s, min(count, strlen(s)));
+	count = s ? strnlen(s, count) : 0;
+	if (count)
+		simc_write(1, s, count);
 }
 
 static struct tty_driver* iss_console_device(struct console *c, int *index)
diff --git a/block/bio-integrity-fs.c b/block/bio-integrity-fs.c
index acb1e5f270d2b..0daa42d9ead77 100644
--- a/block/bio-integrity-fs.c
+++ b/block/bio-integrity-fs.c
@@ -55,6 +55,10 @@ int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size)
 {
 	struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
 	struct bio_integrity_payload *bip = bio_integrity(bio);
+	struct bvec_iter data_iter = {
+		.bi_sector	= sector,
+		.bi_size	= size,
+	};
 
 	/*
 	 * Reinitialize bip->bip_iter.
@@ -65,7 +69,7 @@ int fs_bio_integrity_verify(struct bio *bio, sector_t sector, unsigned int size)
 	memset(&bip->bip_iter, 0, sizeof(bip->bip_iter));
 	bip->bip_iter.bi_sector = sector;
 	bip->bip_iter.bi_size = bio_integrity_bytes(bi, size >> SECTOR_SHIFT);
-	return blk_status_to_errno(bio_integrity_verify(bio, &bip->bip_iter));
+	return blk_status_to_errno(bio_integrity_verify(bio, &data_iter));
 }
 
 static int __init fs_bio_integrity_init(void)
diff --git a/block/bio.c b/block/bio.c
index 5f10900b3f42a..499a34e9f28f5 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -635,15 +635,15 @@ struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask)
 }
 EXPORT_SYMBOL(bio_kmalloc);
 
-void zero_fill_bio_iter(struct bio *bio, struct bvec_iter start)
+void zero_fill_bio(struct bio *bio)
 {
 	struct bio_vec bv;
 	struct bvec_iter iter;
 
-	__bio_for_each_segment(bv, bio, iter, start)
+	bio_for_each_segment(bv, bio, iter)
 		memzero_bvec(&bv);
 }
-EXPORT_SYMBOL(zero_fill_bio_iter);
+EXPORT_SYMBOL(zero_fill_bio);
 
 /**
  * bio_truncate - truncate the bio to small size of @new_size
@@ -1578,26 +1578,6 @@ void __bio_advance(struct bio *bio, unsigned bytes)
 }
 EXPORT_SYMBOL(__bio_advance);
 
-void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
-			struct bio *src, struct bvec_iter *src_iter)
-{
-	while (src_iter->bi_size && dst_iter->bi_size) {
-		struct bio_vec src_bv = bio_iter_iovec(src, *src_iter);
-		struct bio_vec dst_bv = bio_iter_iovec(dst, *dst_iter);
-		unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len);
-		void *src_buf = bvec_kmap_local(&src_bv);
-		void *dst_buf = bvec_kmap_local(&dst_bv);
-
-		memcpy(dst_buf, src_buf, bytes);
-
-		kunmap_local(dst_buf);
-		kunmap_local(src_buf);
-
-		bio_advance_iter_single(src, src_iter, bytes);
-		bio_advance_iter_single(dst, dst_iter, bytes);
-	}
-}
-EXPORT_SYMBOL(bio_copy_data_iter);
 
 /**
  * bio_copy_data - copy contents of data buffers from one bio to another
@@ -1612,7 +1592,21 @@ void bio_copy_data(struct bio *dst, struct bio *src)
 	struct bvec_iter src_iter = src->bi_iter;
 	struct bvec_iter dst_iter = dst->bi_iter;
 
-	bio_copy_data_iter(dst, &dst_iter, src, &src_iter);
+	while (src_iter.bi_size && dst_iter.bi_size) {
+		struct bio_vec src_bv = bio_iter_iovec(src, src_iter);
+		struct bio_vec dst_bv = bio_iter_iovec(dst, dst_iter);
+		unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len);
+		void *src_buf = bvec_kmap_local(&src_bv);
+		void *dst_buf = bvec_kmap_local(&dst_bv);
+
+		memcpy(dst_buf, src_buf, bytes);
+
+		kunmap_local(dst_buf);
+		kunmap_local(src_buf);
+
+		bio_advance_iter_single(src, &src_iter, bytes);
+		bio_advance_iter_single(dst, &dst_iter, bytes);
+	}
 }
 EXPORT_SYMBOL(bio_copy_data);
 
@@ -1659,7 +1653,6 @@ void bio_set_pages_dirty(struct bio *bio)
 		folio_unlock(fi.folio);
 	}
 }
-EXPORT_SYMBOL_GPL(bio_set_pages_dirty);
 
 /*
  * bio_check_pages_dirty() will check that all the BIO's pages are still dirty.
@@ -1718,7 +1711,6 @@ void bio_check_pages_dirty(struct bio *bio)
 	spin_unlock_irqrestore(&bio_dirty_lock, flags);
 	schedule_work(&bio_dirty_work);
 }
-EXPORT_SYMBOL_GPL(bio_check_pages_dirty);
 
 static inline bool bio_remaining_done(struct bio *bio)
 {
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 554c87bb4a865..bc63bd220865d 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -2241,7 +2241,7 @@ void blk_cgroup_bio_start(struct bio *bio)
 	}
 
 	u64_stats_update_end_irqrestore(&bis->sync, flags);
-	css_rstat_updated(&blkcg->css, cpu);
+	__css_rstat_updated(&blkcg->css, cpu);
 	put_cpu();
 }
 
diff --git a/block/blk-core.c b/block/blk-core.c
index 17450058ea6d8..22af5dec112b5 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -197,7 +197,6 @@ const char *blk_status_to_str(blk_status_t status)
 		return "<null>";
 	return blk_errors[idx].name;
 }
-EXPORT_SYMBOL_GPL(blk_status_to_str);
 
 /**
  * blk_sync_queue - cancel any pending callbacks on a queue
@@ -1270,7 +1269,6 @@ void blk_io_schedule(void)
 	else
 		io_schedule();
 }
-EXPORT_SYMBOL_GPL(blk_io_schedule);
 
 int __init blk_dev_init(void)
 {
diff --git a/block/blk-crypto.c b/block/blk-crypto.c
index 856d3c5b1fa0d..40a99a859748a 100644
--- a/block/blk-crypto.c
+++ b/block/blk-crypto.c
@@ -116,6 +116,7 @@ void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key,
 
 	bio->bi_crypt_context = bc;
 }
+EXPORT_SYMBOL_GPL(bio_crypt_set_ctx);
 
 void __bio_crypt_free_ctx(struct bio *bio)
 {
@@ -349,6 +350,7 @@ int blk_crypto_init_key(struct blk_crypto_key *blk_key,
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(blk_crypto_init_key);
 
 bool blk_crypto_config_supported_natively(struct block_device *bdev,
 					  const struct blk_crypto_config *cfg)
@@ -399,6 +401,7 @@ int blk_crypto_start_using_key(struct block_device *bdev,
 	}
 	return blk_crypto_fallback_start_using_mode(key->crypto_cfg.crypto_mode);
 }
+EXPORT_SYMBOL_GPL(blk_crypto_start_using_key);
 
 /**
  * blk_crypto_evict_key() - Evict a blk_crypto_key from a block_device
diff --git a/block/blk.h b/block/blk.h
index b998a7761faf3..bf1a80493ff1c 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -49,6 +49,8 @@ struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
 					      gfp_t flags);
 void blk_free_flush_queue(struct blk_flush_queue *q);
 
+const char *blk_status_to_str(blk_status_t status);
+
 bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic);
 bool blk_queue_start_drain(struct request_queue *q);
 bool __blk_freeze_queue_start(struct request_queue *q,
diff --git a/certs/Kconfig b/certs/Kconfig
index 8e39a80c7abe5..9d2bf7fb5b9e4 100644
--- a/certs/Kconfig
+++ b/certs/Kconfig
@@ -6,14 +6,14 @@ config MODULE_SIG_KEY
 	default "certs/signing_key.pem"
 	depends on MODULE_SIG || (IMA_APPRAISE_MODSIG && MODULES)
 	help
-         Provide the file name of a private key/certificate in PEM format,
-         or a PKCS#11 URI according to RFC7512. The file should contain, or
-         the URI should identify, both the certificate and its corresponding
-         private key.
-
-         If this option is unchanged from its default "certs/signing_key.pem",
-         then the kernel will automatically generate the private key and
-         certificate as described in Documentation/admin-guide/module-signing.rst
+	 Provide the file name of a private key/certificate in PEM format,
+	 or a PKCS#11 URI according to RFC7512. The file should contain, or
+	 the URI should identify, both the certificate and its corresponding
+	 private key.
+
+	 If this option is unchanged from its default "certs/signing_key.pem",
+	 then the kernel will automatically generate the private key and
+	 certificate as described in Documentation/admin-guide/module-signing.rst
 
 choice
 	prompt "Type of module signing key to be generated"
diff --git a/certs/system_keyring.c b/certs/system_keyring.c
index e0761436ec7f4..9bda49295bd02 100644
--- a/certs/system_keyring.c
+++ b/certs/system_keyring.c
@@ -380,6 +380,7 @@ int verify_pkcs7_message_sig(const void *data, size_t len,
 	pr_devel("<==%s() = %d\n", __func__, ret);
 	return ret;
 }
+EXPORT_SYMBOL(verify_pkcs7_message_sig);
 
 /**
  * verify_pkcs7_signature - Verify a PKCS#7-based signature on system data.
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 103d1f58cb7c2..00b7ce4227ae1 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -1113,44 +1113,16 @@ endmenu
 
 menu "Random number generation"
 
-menuconfig CRYPTO_DRBG_MENU
+config CRYPTO_DRBG
 	tristate "NIST SP800-90A DRBG (Deterministic Random Bit Generator)"
+	select CRYPTO_JITTERENTROPY
+	select CRYPTO_LIB_SHA512
+	select CRYPTO_RNG
 	help
 	  DRBG (Deterministic Random Bit Generator) (NIST SP800-90A)
 
-	  In the following submenu, one or more of the DRBG types must be selected.
-
-if CRYPTO_DRBG_MENU
-
-config CRYPTO_DRBG_HMAC
-	bool
-	default y
-	select CRYPTO_HMAC
-	select CRYPTO_SHA512
-
-config CRYPTO_DRBG_HASH
-	bool "Hash_DRBG"
-	select CRYPTO_SHA256
-	help
-	  Hash_DRBG variant as defined in NIST SP800-90A.
-
-	  This uses the SHA-1, SHA-256, SHA-384, or SHA-512 hash algorithms.
-
-config CRYPTO_DRBG_CTR
-	bool "CTR_DRBG"
-	select CRYPTO_DF80090A
-	help
-	  CTR_DRBG variant as defined in NIST SP800-90A.
-
-	  This uses the AES cipher algorithm with the counter block mode.
-
-config CRYPTO_DRBG
-	tristate
-	default CRYPTO_DRBG_MENU
-	select CRYPTO_RNG
-	select CRYPTO_JITTERENTROPY
-
-endif	# if CRYPTO_DRBG_MENU
+	  Enable this only if you need it for a FIPS 140 certification.
+	  It's otherwise redundant with the kernel's regular RNG.
 
 config CRYPTO_JITTERENTROPY
 	tristate "CPU Jitter Non-Deterministic RNG (Random Number Generator)"
@@ -1280,44 +1252,68 @@ config CRYPTO_DF80090A
 	select CRYPTO_CTR
 
 endmenu
-menu "Userspace interface"
+menu "Userspace interface (deprecated)"
 
 config CRYPTO_USER_API
 	tristate
 
 config CRYPTO_USER_API_HASH
-	tristate "Hash algorithms"
+	tristate "Hash algorithms (deprecated)"
 	depends on NET
 	select CRYPTO_HASH
 	select CRYPTO_USER_API
 	help
-	  Enable the userspace interface for hash algorithms.
+	  Enable the AF_ALG userspace interface for hash algorithms.  This
+	  provides unprivileged userspace programs access to arbitrary hash
+	  algorithms implemented in the kernel's privileged execution context.
 
-	  See Documentation/crypto/userspace-if.rst and
-	  https://www.chronox.de/libkcapi/html/index.html
+	  This interface is deprecated and is supported only for backwards
+	  compatibility.  It regularly has vulnerabilities, and the capabilities
+	  it provides are redundant with userspace crypto libraries.
+
+	  Enable this only if needed for support for a program that hasn't yet
+	  been converted to userspace crypto, for example iwd.
+
+	  See also Documentation/crypto/userspace-if.rst
 
 config CRYPTO_USER_API_SKCIPHER
-	tristate "Symmetric key cipher algorithms"
+	tristate "Symmetric key cipher algorithms (deprecated)"
 	depends on NET
 	select CRYPTO_SKCIPHER
 	select CRYPTO_USER_API
 	help
-	  Enable the userspace interface for symmetric key cipher algorithms.
+	  Enable the AF_ALG userspace interface for symmetric key algorithms.
+	  This provides unprivileged userspace programs access to arbitrary
+	  symmetric key algorithms implemented in the kernel's privileged
+	  execution context.
+
+	  This interface is deprecated and is supported only for backwards
+	  compatibility.  It regularly has vulnerabilities, and the capabilities
+	  it provides are redundant with userspace crypto libraries.
+
+	  Enable this only if needed for support for a program that hasn't yet
+	  been converted to userspace crypto, for example iwd, or cryptsetup
+	  with certain algorithms.
 
-	  See Documentation/crypto/userspace-if.rst and
-	  https://www.chronox.de/libkcapi/html/index.html
+	  See also Documentation/crypto/userspace-if.rst
 
 config CRYPTO_USER_API_RNG
-	tristate "RNG (random number generator) algorithms"
+	tristate "Random number generation algorithms (deprecated)"
 	depends on NET
 	select CRYPTO_RNG
 	select CRYPTO_USER_API
 	help
-	  Enable the userspace interface for RNG (random number generator)
-	  algorithms.
+	  Enable the AF_ALG userspace interface for random number generation
+	  (RNG) algorithms.  This provides unprivileged userspace programs
+	  access to arbitrary RNG algorithms implemented in the kernel's
+	  privileged execution context.
 
-	  See Documentation/crypto/userspace-if.rst and
-	  https://www.chronox.de/libkcapi/html/index.html
+	  This interface is deprecated and is supported only for backwards
+	  compatibility.  It regularly has vulnerabilities, and the capabilities
+	  it provides are redundant with userspace crypto libraries as well as
+	  the normal kernel RNG (e.g., /dev/urandom and getrandom(2)).
+
+	  See also Documentation/crypto/userspace-if.rst
 
 config CRYPTO_USER_API_RNG_CAVP
 	bool "Enable CAVP testing of DRBG"
@@ -1332,16 +1328,25 @@ config CRYPTO_USER_API_RNG_CAVP
 	  no unless you know what this is.
 
 config CRYPTO_USER_API_AEAD
-	tristate "AEAD cipher algorithms"
+	tristate "AEAD cipher algorithms (deprecated)"
 	depends on NET
 	select CRYPTO_AEAD
 	select CRYPTO_SKCIPHER
 	select CRYPTO_USER_API
 	help
-	  Enable the userspace interface for AEAD cipher algorithms.
+	  Enable the AF_ALG userspace interface for authenticated encryption
+	  with associated data (AEAD) algorithms.  This provides unprivileged
+	  userspace programs access to arbitrary AEAD algorithms implemented in
+	  the kernel's privileged execution context.
+
+	  This interface is deprecated and is supported only for backwards
+	  compatibility.  It regularly has vulnerabilities, and the capabilities
+	  it provides are redundant with userspace crypto libraries.
+
+	  Enable this only if needed for support for a program that hasn't yet
+	  been converted to userspace crypto, for example iwd.
 
-	  See Documentation/crypto/userspace-if.rst and
-	  https://www.chronox.de/libkcapi/html/index.html
+	  See also Documentation/crypto/userspace-if.rst
 
 config CRYPTO_USER_API_ENABLE_OBSOLETE
 	bool "Obsolete cryptographic algorithms"
diff --git a/crypto/Makefile b/crypto/Makefile
index 162242593c7c1..c73f4d51d0368 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -178,6 +178,11 @@ obj-$(CONFIG_CRYPTO_ZSTD) += zstd.o
 obj-$(CONFIG_CRYPTO_ECC) += ecc.o
 obj-$(CONFIG_CRYPTO_ESSIV) += essiv.o
 
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=124949
+ifeq ($(CONFIG_ARM)$(CONFIG_KASAN_STACK)$(CONFIG_CC_IS_GCC),yyy)
+CFLAGS_ecc.o += $(call cc-option,-Wframe-larger-than=1536)
+endif
+
 ecdh_generic-y += ecdh.o
 ecdh_generic-y += ecdh_helper.o
 obj-$(CONFIG_CRYPTO_ECDH) += ecdh_generic.o
diff --git a/crypto/acompress.c b/crypto/acompress.c
index 6025c1acce491..032de704eb2ca 100644
--- a/crypto/acompress.c
+++ b/crypto/acompress.c
@@ -51,11 +51,9 @@ static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm)
 static int __maybe_unused crypto_acomp_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
-	struct crypto_report_acomp racomp;
-
-	memset(&racomp, 0, sizeof(racomp));
-
-	strscpy(racomp.type, "acomp", sizeof(racomp.type));
+	struct crypto_report_acomp racomp = {
+		.type = "acomp",
+	};
 
 	return nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(racomp), &racomp);
 }
diff --git a/crypto/aead.c b/crypto/aead.c
index e009937bf3a5d..045b74c3779f4 100644
--- a/crypto/aead.c
+++ b/crypto/aead.c
@@ -136,13 +136,11 @@ static int crypto_aead_init_tfm(struct crypto_tfm *tfm)
 static int __maybe_unused crypto_aead_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
-	struct crypto_report_aead raead;
 	struct aead_alg *aead = container_of(alg, struct aead_alg, base);
-
-	memset(&raead, 0, sizeof(raead));
-
-	strscpy(raead.type, "aead", sizeof(raead.type));
-	strscpy(raead.geniv, "<none>", sizeof(raead.geniv));
+	struct crypto_report_aead raead = {
+		.type = "aead",
+		.geniv = "<none>",
+	};
 
 	raead.blocksize = alg->cra_blocksize;
 	raead.maxauthsize = aead->maxauthsize;
diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index 5a00c18eb145c..48c53f488e0fd 100644
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -584,6 +584,8 @@ static int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con)
 			if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32)))
 				return -EINVAL;
 			con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg);
+			if (con->aead_assoclen >= 0x80000000u)
+				return -EINVAL;
 			break;
 
 		default:
@@ -973,7 +975,7 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
 		ssize_t plen;
 
 		/* use the existing memory in an allocated page */
-		if (ctx->merge && !(msg->msg_flags & MSG_SPLICE_PAGES)) {
+		if (ctx->merge) {
 			sgl = list_entry(ctx->tsgl_list.prev,
 					 struct af_alg_tsgl, list);
 			sg = sgl->sg + sgl->cur - 1;
@@ -1017,60 +1019,37 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
 		if (sgl->cur)
 			sg_unmark_end(sg + sgl->cur - 1);
 
-		if (msg->msg_flags & MSG_SPLICE_PAGES) {
-			struct sg_table sgtable = {
-				.sgl		= sg,
-				.nents		= sgl->cur,
-				.orig_nents	= sgl->cur,
-			};
-
-			plen = extract_iter_to_sg(&msg->msg_iter, len, &sgtable,
-						  MAX_SGL_ENTS - sgl->cur, 0);
-			if (plen < 0) {
-				err = plen;
+		do {
+			struct page *pg;
+			unsigned int i = sgl->cur;
+
+			plen = min_t(size_t, len, PAGE_SIZE);
+
+			pg = alloc_page(GFP_KERNEL);
+			if (!pg) {
+				err = -ENOMEM;
+				goto unlock;
+			}
+
+			sg_assign_page(sg + i, pg);
+
+			err = memcpy_from_msg(page_address(sg_page(sg + i)),
+					      msg, plen);
+			if (err) {
+				__free_page(sg_page(sg + i));
+				sg_assign_page(sg + i, NULL);
 				goto unlock;
 			}
 
-			for (; sgl->cur < sgtable.nents; sgl->cur++)
-				get_page(sg_page(&sg[sgl->cur]));
+			sg[i].length = plen;
 			len -= plen;
 			ctx->used += plen;
 			copied += plen;
 			size -= plen;
-		} else {
-			do {
-				struct page *pg;
-				unsigned int i = sgl->cur;
-
-				plen = min_t(size_t, len, PAGE_SIZE);
-
-				pg = alloc_page(GFP_KERNEL);
-				if (!pg) {
-					err = -ENOMEM;
-					goto unlock;
-				}
-
-				sg_assign_page(sg + i, pg);
-
-				err = memcpy_from_msg(
-					page_address(sg_page(sg + i)),
-					msg, plen);
-				if (err) {
-					__free_page(sg_page(sg + i));
-					sg_assign_page(sg + i, NULL);
-					goto unlock;
-				}
-
-				sg[i].length = plen;
-				len -= plen;
-				ctx->used += plen;
-				copied += plen;
-				size -= plen;
-				sgl->cur++;
-			} while (len && sgl->cur < MAX_SGL_ENTS);
-
-			ctx->merge = plen & (PAGE_SIZE - 1);
-		}
+			sgl->cur++;
+		} while (len && sgl->cur < MAX_SGL_ENTS);
+
+		ctx->merge = plen & (PAGE_SIZE - 1);
 
 		if (!size)
 			sg_mark_end(sg + sgl->cur - 1);
diff --git a/crypto/ahash.c b/crypto/ahash.c
index 7a730324c50e2..dd56b0e45c0db 100644
--- a/crypto/ahash.c
+++ b/crypto/ahash.c
@@ -789,11 +789,9 @@ static void crypto_ahash_free_instance(struct crypto_instance *inst)
 static int __maybe_unused crypto_ahash_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
-	struct crypto_report_hash rhash;
-
-	memset(&rhash, 0, sizeof(rhash));
-
-	strscpy(rhash.type, "ahash", sizeof(rhash.type));
+	struct crypto_report_hash rhash = {
+		.type = "ahash",
+	};
 
 	rhash.blocksize = alg->cra_blocksize;
 	rhash.digestsize = __crypto_hash_alg_common(alg)->digestsize;
diff --git a/crypto/akcipher.c b/crypto/akcipher.c
index dfe87b3ce1837..630bb19738beb 100644
--- a/crypto/akcipher.c
+++ b/crypto/akcipher.c
@@ -36,11 +36,9 @@ struct crypto_akcipher_sync_data {
 static int __maybe_unused crypto_akcipher_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
-	struct crypto_report_akcipher rakcipher;
-
-	memset(&rakcipher, 0, sizeof(rakcipher));
-
-	strscpy(rakcipher.type, "akcipher", sizeof(rakcipher.type));
+	struct crypto_report_akcipher rakcipher = {
+		.type = "akcipher",
+	};
 
 	return nla_put(skb, CRYPTOCFGA_REPORT_AKCIPHER,
 		       sizeof(rakcipher), &rakcipher);
diff --git a/crypto/algif_aead.c b/crypto/algif_aead.c
index cb651ab58d629..c6c2ce21895dd 100644
--- a/crypto/algif_aead.c
+++ b/crypto/algif_aead.c
@@ -9,10 +9,10 @@
  * The following concept of the memory management is used:
  *
  * The kernel maintains two SGLs, the TX SGL and the RX SGL. The TX SGL is
- * filled by user space with the data submitted via sendmsg (maybe with
- * MSG_SPLICE_PAGES).  Filling up the TX SGL does not cause a crypto operation
- * -- the data will only be tracked by the kernel. Upon receipt of one recvmsg
- * call, the caller must provide a buffer which is tracked with the RX SGL.
+ * filled by user space with the data submitted via sendmsg.  Filling up the TX
+ * SGL does not cause a crypto operation -- the data will only be tracked by the
+ * kernel. Upon receipt of one recvmsg call, the caller must provide a buffer
+ * which is tracked with the RX SGL.
  *
  * During the processing of the recvmsg operation, the cipher request is
  * allocated and prepared. As part of the recvmsg operation, the processed
diff --git a/crypto/asymmetric_keys/Kconfig b/crypto/asymmetric_keys/Kconfig
index e50bd9b3e27ba..6a2f664046efe 100644
--- a/crypto/asymmetric_keys/Kconfig
+++ b/crypto/asymmetric_keys/Kconfig
@@ -95,7 +95,6 @@ config FIPS_SIGNATURE_SELFTEST
 	  verification code, using some built in data.  This is required
 	  for FIPS.
 	depends on KEYS
-	depends on ASYMMETRIC_KEY_TYPE
 	depends on PKCS7_MESSAGE_PARSER=X509_CERTIFICATE_PARSER
 	depends on X509_CERTIFICATE_PARSER
 	depends on CRYPTO_RSA
diff --git a/crypto/asymmetric_keys/Makefile b/crypto/asymmetric_keys/Makefile
index bc65d3b98dcbf..f99b7169ae7cd 100644
--- a/crypto/asymmetric_keys/Makefile
+++ b/crypto/asymmetric_keys/Makefile
@@ -53,12 +53,14 @@ clean-files	+= pkcs8.asn1.c pkcs8.asn1.h
 obj-$(CONFIG_PKCS7_MESSAGE_PARSER) += pkcs7_message.o
 pkcs7_message-y := \
 	pkcs7.asn1.o \
+	pkcs7_aa.asn1.o \
 	pkcs7_parser.o \
 	pkcs7_trust.o \
 	pkcs7_verify.o
 
-$(obj)/pkcs7_parser.o: $(obj)/pkcs7.asn1.h
+$(obj)/pkcs7_parser.o: $(obj)/pkcs7.asn1.h $(obj)/pkcs7_aa.asn1.h
 $(obj)/pkcs7.asn1.o: $(obj)/pkcs7.asn1.c $(obj)/pkcs7.asn1.h
+$(obj)/pkcs7_aa.asn1.o: $(obj)/pkcs7_aa.asn1.c $(obj)/pkcs7_aa.asn1.h
 
 #
 # PKCS#7 parser testing key
diff --git a/crypto/asymmetric_keys/pkcs7_aa.asn1 b/crypto/asymmetric_keys/pkcs7_aa.asn1
new file mode 100644
index 0000000000000..7a8857bdf56e1
--- /dev/null
+++ b/crypto/asymmetric_keys/pkcs7_aa.asn1
@@ -0,0 +1,18 @@
+-- SPDX-License-Identifier: BSD-3-Clause
+--
+-- Copyright (C) 2009 IETF Trust and the persons identified as authors
+-- of the code
+--
+-- https://www.rfc-editor.org/rfc/rfc5652#section-3
+
+AA ::= 	CHOICE {
+	aaSet		[0] IMPLICIT AASet,
+	aaSequence	[2] EXPLICIT SEQUENCE OF AuthenticatedAttribute
+}
+
+AASet ::= SET OF AuthenticatedAttribute
+
+AuthenticatedAttribute ::= SEQUENCE {
+	type	OBJECT IDENTIFIER ({ pkcs7_aa_note_OID }),
+	values	SET OF ANY ({ pkcs7_aa_note_attr })
+}
diff --git a/crypto/asymmetric_keys/pkcs7_key_type.c b/crypto/asymmetric_keys/pkcs7_key_type.c
index b930d3bbf1af5..e0b1ce0202f6d 100644
--- a/crypto/asymmetric_keys/pkcs7_key_type.c
+++ b/crypto/asymmetric_keys/pkcs7_key_type.c
@@ -12,6 +12,7 @@
 #include <linux/verification.h>
 #include <linux/key-type.h>
 #include <keys/user-type.h>
+#include <crypto/pkcs7.h>
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("PKCS#7 testing key type");
@@ -51,16 +52,57 @@ static int pkcs7_view_content(void *ctx, const void *data, size_t len,
 static int pkcs7_preparse(struct key_preparsed_payload *prep)
 {
 	enum key_being_used_for usage = pkcs7_usage;
+	int ret;
+	struct pkcs7_message *pkcs7;
+	const void *data;
+	size_t len;
 
 	if (usage >= NR__KEY_BEING_USED_FOR) {
 		pr_err("Invalid usage type %d\n", usage);
 		return -EINVAL;
 	}
 
-	return verify_pkcs7_signature(NULL, 0,
+	ret = verify_pkcs7_signature(NULL, 0,
 				      prep->data, prep->datalen,
 				      VERIFY_USE_SECONDARY_KEYRING, usage,
 				      pkcs7_view_content, prep);
+	if (ret)
+		return ret;
+
+	pkcs7 = pkcs7_parse_message(prep->data, prep->datalen);
+	if (IS_ERR(pkcs7)) {
+		pr_err("pkcs7 parse error\n");
+		return PTR_ERR(pkcs7);
+	}
+
+	/*
+	 * the parsed message has no trusted signer, so nothing should
+	 * be returned here
+	 */
+	ret = pkcs7_get_authattr(pkcs7, OID_messageDigest, &data, &len);
+	if (ret == 0) {
+		pr_err("OID returned when no trust in signer\n");
+		goto out;
+	}
+	/* add trust and check again */
+	ret = verify_pkcs7_message_sig(NULL, 0, pkcs7,
+				       VERIFY_USE_SECONDARY_KEYRING, usage,
+				       NULL, NULL);
+	if (ret) {
+		pr_err("verify_pkcs7_message_sig failed!!\n");
+		goto out;
+	}
+	/* now we should find the OID */
+	ret = pkcs7_get_authattr(pkcs7, OID_messageDigest, &data, &len);
+	if (ret) {
+		pr_err("Failed to get message digest\n");
+		goto out;
+	}
+	pr_info("Correctly Got message hash, size=%zu\n", len);
+
+ out:
+	pkcs7_free_message(pkcs7);
+	return 0;
 }
 
 /*
diff --git a/crypto/asymmetric_keys/pkcs7_parser.c b/crypto/asymmetric_keys/pkcs7_parser.c
index 6e3ffdac83ace..d467866f7d930 100644
--- a/crypto/asymmetric_keys/pkcs7_parser.c
+++ b/crypto/asymmetric_keys/pkcs7_parser.c
@@ -15,6 +15,7 @@
 #include <crypto/public_key.h>
 #include "pkcs7_parser.h"
 #include "pkcs7.asn1.h"
+#include "pkcs7_aa.asn1.h"
 
 MODULE_DESCRIPTION("PKCS#7 parser");
 MODULE_AUTHOR("Red Hat, Inc.");
@@ -211,6 +212,86 @@ int pkcs7_get_content_data(const struct pkcs7_message *pkcs7,
 }
 EXPORT_SYMBOL_GPL(pkcs7_get_content_data);
 
+struct pkcs7_aa_context {
+	bool found;
+	enum OID oid_to_find;
+	const void *data;
+	size_t len;
+};
+
+int pkcs7_aa_note_OID(void *context, size_t hdrlen,
+		      unsigned char tag,
+		      const void *value, size_t vlen)
+{
+	struct pkcs7_aa_context *ctx = context;
+	enum OID oid = look_up_OID(value, vlen);
+
+	ctx->found = (oid == ctx->oid_to_find);
+
+	return 0;
+}
+
+int pkcs7_aa_note_attr(void *context, size_t hdrlen,
+		       unsigned char tag,
+		       const void *value, size_t vlen)
+{
+	struct pkcs7_aa_context *ctx = context;
+
+	if (ctx->found) {
+		ctx->data = value;
+		ctx->len = vlen;
+	}
+
+	return 0;
+}
+
+/**
+ * pkcs7_get_authattr - get authenticated attribute by OID
+ * @pkcs7: The preparsed PKCS#7 message
+ * @oid: the enum value of the OID to find
+ * @_data: Place to return a pointer to the attribute value
+ * @_len: length of the attribute value
+ *
+ * Searches the authenticated attributes until one is found with a
+ * matching OID.  Note that because the attributes are per signer
+ * there could be multiple signers with different values, but this
+ * routine will simply return the first one in parse order.
+ *
+ * Returns -ENODATA if the attribute can't be found
+ */
+int pkcs7_get_authattr(const struct pkcs7_message *pkcs7,
+		       enum OID oid,
+		       const void **_data, size_t *_len)
+{
+	struct pkcs7_signed_info *sinfo = pkcs7->signed_infos;
+	struct pkcs7_aa_context ctx;
+
+	ctx.data = NULL;
+	ctx.oid_to_find = oid;
+
+	for (; sinfo; sinfo = sinfo->next) {
+		int ret;
+
+		/* only extract OIDs from validated signers */
+		if (!sinfo->verified)
+			continue;
+
+		ret = asn1_ber_decoder(&pkcs7_aa_decoder, &ctx,
+				       sinfo->authattrs, sinfo->authattrs_len);
+		if (ret < 0 || ctx.data != NULL)
+			break;
+	}
+
+	if (!ctx.data)
+		return -ENODATA;
+
+	*_data = ctx.data;
+	*_len = ctx.len;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(pkcs7_get_authattr);
+
 /*
  * Note an OID when we find one for later processing when we know how
  * to interpret it.
diff --git a/crypto/asymmetric_keys/pkcs7_parser.h b/crypto/asymmetric_keys/pkcs7_parser.h
index 6ef9f335bb17f..203062a33def6 100644
--- a/crypto/asymmetric_keys/pkcs7_parser.h
+++ b/crypto/asymmetric_keys/pkcs7_parser.h
@@ -20,6 +20,7 @@ struct pkcs7_signed_info {
 	unsigned	index;
 	bool		unsupported_crypto;	/* T if not usable due to missing crypto */
 	bool		blacklisted;
+	bool		verified; /* T if this signer has validated trust */
 
 	/* Message digest - the digest of the Content Data (or NULL) */
 	const void	*msgdigest;
diff --git a/crypto/asymmetric_keys/pkcs7_trust.c b/crypto/asymmetric_keys/pkcs7_trust.c
index 9a87c34ed1733..78ebfb6373b61 100644
--- a/crypto/asymmetric_keys/pkcs7_trust.c
+++ b/crypto/asymmetric_keys/pkcs7_trust.c
@@ -127,6 +127,7 @@ static int pkcs7_validate_trust_one(struct pkcs7_message *pkcs7,
 		for (p = sinfo->signer; p != x509; p = p->signer)
 			p->verified = true;
 	}
+	sinfo->verified = true;
 	kleave(" = 0");
 	return 0;
 }
diff --git a/crypto/asymmetric_keys/verify_pefile.c b/crypto/asymmetric_keys/verify_pefile.c
index 1f3b227ba7f22..cec99db14129a 100644
--- a/crypto/asymmetric_keys/verify_pefile.c
+++ b/crypto/asymmetric_keys/verify_pefile.c
@@ -305,6 +305,8 @@ static int pefile_digest_pe_contents(const void *pebuf, unsigned int pelen,
 
 	if (pelen > hashed_bytes) {
 		tmp = hashed_bytes + ctx->certs_size;
+		if (tmp <= hashed_bytes || pelen < tmp)
+			return -ELIBBAD;
 		ret = crypto_shash_update(desc,
 					  pebuf + hashed_bytes,
 					  pelen - tmp);
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
index 9e4bb7fbde25e..27f99349e310b 100644
--- a/crypto/async_tx/async_pq.c
+++ b/crypto/async_tx/async_pq.c
@@ -8,6 +8,7 @@
 #include <linux/module.h>
 #include <linux/dma-mapping.h>
 #include <linux/raid/pq.h>
+#include <linux/raid/pq_tables.h>
 #include <linux/async_tx.h>
 #include <linux/gfp.h>
 
@@ -119,7 +120,7 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks,
 	for (i = 0; i < disks; i++) {
 		if (blocks[i] == NULL) {
 			BUG_ON(i > disks - 3); /* P or Q can't be zero */
-			srcs[i] = raid6_get_zero_page();
+			srcs[i] = page_address(ZERO_PAGE(0));
 		} else {
 			srcs[i] = page_address(blocks[i]) + offsets[i];
 
@@ -131,11 +132,11 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks,
 		}
 	}
 	if (submit->flags & ASYNC_TX_PQ_XOR_DST) {
-		BUG_ON(!raid6_call.xor_syndrome);
+		BUG_ON(!raid6_can_xor_syndrome());
 		if (start >= 0)
-			raid6_call.xor_syndrome(disks, start, stop, len, srcs);
+			raid6_xor_syndrome(disks, start, stop, len, srcs);
 	} else
-		raid6_call.gen_syndrome(disks, len, srcs);
+		raid6_gen_syndrome(disks, len, srcs);
 	async_tx_sync_epilog(submit);
 }
 
diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c
index 539ea5b378dcd..e53870d84bc55 100644
--- a/crypto/async_tx/async_raid6_recov.c
+++ b/crypto/async_tx/async_raid6_recov.c
@@ -11,6 +11,7 @@
 #include <linux/module.h>
 #include <linux/dma-mapping.h>
 #include <linux/raid/pq.h>
+#include <linux/raid/pq_tables.h>
 #include <linux/async_tx.h>
 #include <linux/dmaengine.h>
 
@@ -414,11 +415,11 @@ async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
 		async_tx_quiesce(&submit->depend_tx);
 		for (i = 0; i < disks; i++)
 			if (blocks[i] == NULL)
-				ptrs[i] = raid6_get_zero_page();
+				ptrs[i] = page_address(ZERO_PAGE(0));
 			else
 				ptrs[i] = page_address(blocks[i]) + offs[i];
 
-		raid6_2data_recov(disks, bytes, faila, failb, ptrs);
+		raid6_recov_2data(disks, bytes, faila, failb, ptrs);
 
 		async_tx_sync_epilog(submit);
 
@@ -497,11 +498,11 @@ async_raid6_datap_recov(int disks, size_t bytes, int faila,
 		async_tx_quiesce(&submit->depend_tx);
 		for (i = 0; i < disks; i++)
 			if (blocks[i] == NULL)
-				ptrs[i] = raid6_get_zero_page();
+				ptrs[i] = page_address(ZERO_PAGE(0));
 			else
 				ptrs[i] = page_address(blocks[i]) + offs[i];
 
-		raid6_datap_recov(disks, bytes, faila, ptrs);
+		raid6_recov_datap(disks, bytes, faila, ptrs);
 
 		async_tx_sync_epilog(submit);
 
diff --git a/crypto/authencesn.c b/crypto/authencesn.c
index 522df41365d8f..ca063576f6706 100644
--- a/crypto/authencesn.c
+++ b/crypto/authencesn.c
@@ -94,11 +94,11 @@ static int crypto_authenc_esn_genicv_tail(struct aead_request *req,
 	u32 tmp[2];
 
 	/* Move high-order bits of sequence number back. */
-	scatterwalk_map_and_copy(tmp, dst, 4, 4, 0);
-	scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0);
-	scatterwalk_map_and_copy(tmp, dst, 0, 8, 1);
+	memcpy_from_sglist(tmp, dst, 4, 4);
+	memcpy_from_sglist(tmp + 1, dst, assoclen + cryptlen, 4);
+	memcpy_to_sglist(dst, 0, tmp, 8);
 
-	scatterwalk_map_and_copy(hash, dst, assoclen + cryptlen, authsize, 1);
+	memcpy_to_sglist(dst, assoclen + cryptlen, hash, authsize);
 	return 0;
 }
 
@@ -129,9 +129,9 @@ static int crypto_authenc_esn_genicv(struct aead_request *req,
 		return 0;
 
 	/* Move high-order bits of sequence number to the end. */
-	scatterwalk_map_and_copy(tmp, dst, 0, 8, 0);
-	scatterwalk_map_and_copy(tmp, dst, 4, 4, 1);
-	scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1);
+	memcpy_from_sglist(tmp, dst, 0, 8);
+	memcpy_to_sglist(dst, 4, tmp, 4);
+	memcpy_to_sglist(dst, assoclen + cryptlen, tmp + 1, 4);
 
 	sg_init_table(areq_ctx->dst, 2);
 	dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4);
@@ -217,9 +217,9 @@ static int crypto_authenc_esn_decrypt_tail(struct aead_request *req,
 
 	if (src == dst) {
 		/* Move high-order bits of sequence number back. */
-		scatterwalk_map_and_copy(tmp, dst, 4, 4, 0);
-		scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 0);
-		scatterwalk_map_and_copy(tmp, dst, 0, 8, 1);
+		memcpy_from_sglist(tmp, dst, 4, 4);
+		memcpy_from_sglist(tmp + 1, dst, assoclen + cryptlen, 4);
+		memcpy_to_sglist(dst, 0, tmp, 8);
 	} else
 		memcpy_sglist(dst, src, assoclen);
 
@@ -274,18 +274,17 @@ static int crypto_authenc_esn_decrypt(struct aead_request *req)
 		goto tail;
 
 	cryptlen -= authsize;
-	scatterwalk_map_and_copy(ihash, req->src, assoclen + cryptlen,
-				 authsize, 0);
+	memcpy_from_sglist(ihash, req->src, assoclen + cryptlen, authsize);
 
 	/* Move high-order bits of sequence number to the end. */
-	scatterwalk_map_and_copy(tmp, src, 0, 8, 0);
+	memcpy_from_sglist(tmp, src, 0, 8);
 	if (src == dst) {
-		scatterwalk_map_and_copy(tmp, dst, 4, 4, 1);
-		scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen, 4, 1);
+		memcpy_to_sglist(dst, 4, tmp, 4);
+		memcpy_to_sglist(dst, assoclen + cryptlen, tmp + 1, 4);
 		dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4);
 	} else {
-		scatterwalk_map_and_copy(tmp, dst, 0, 4, 1);
-		scatterwalk_map_and_copy(tmp + 1, dst, assoclen + cryptlen - 4, 4, 1);
+		memcpy_to_sglist(dst, 0, tmp, 4);
+		memcpy_to_sglist(dst, assoclen + cryptlen - 4, tmp + 1, 4);
 
 		src = scatterwalk_ffwd(areq_ctx->src, src, 8);
 		dst = scatterwalk_ffwd(areq_ctx->dst, dst, 4);
diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c
index 34588f39fdfc6..fea151df1648a 100644
--- a/crypto/crypto_null.c
+++ b/crypto/crypto_null.c
@@ -50,15 +50,6 @@ static int null_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
 				unsigned int keylen)
 { return 0; }
 
-static int null_setkey(struct crypto_tfm *tfm, const u8 *key,
-		       unsigned int keylen)
-{ return 0; }
-
-static void null_crypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
-{
-	memcpy(dst, src, NULL_BLOCK_SIZE);
-}
-
 static int null_skcipher_crypt(struct skcipher_request *req)
 {
 	if (req->src != req->dst)
@@ -97,35 +88,16 @@ static struct skcipher_alg skcipher_null = {
 	.decrypt		=	null_skcipher_crypt,
 };
 
-static struct crypto_alg cipher_null = {
-	.cra_name		=	"cipher_null",
-	.cra_driver_name	=	"cipher_null-generic",
-	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
-	.cra_blocksize		=	NULL_BLOCK_SIZE,
-	.cra_ctxsize		=	0,
-	.cra_module		=	THIS_MODULE,
-	.cra_u			=	{ .cipher = {
-	.cia_min_keysize	=	NULL_KEY_SIZE,
-	.cia_max_keysize	=	NULL_KEY_SIZE,
-	.cia_setkey		= 	null_setkey,
-	.cia_encrypt		=	null_crypt,
-	.cia_decrypt		=	null_crypt } }
-};
-
 MODULE_ALIAS_CRYPTO("digest_null");
-MODULE_ALIAS_CRYPTO("cipher_null");
+MODULE_ALIAS_CRYPTO("ecb(cipher_null)");
 
 static int __init crypto_null_mod_init(void)
 {
 	int ret = 0;
 
-	ret = crypto_register_alg(&cipher_null);
-	if (ret < 0)
-		goto out;
-
 	ret = crypto_register_shash(&digest_null);
 	if (ret < 0)
-		goto out_unregister_algs;
+		goto out;
 
 	ret = crypto_register_skcipher(&skcipher_null);
 	if (ret < 0)
@@ -135,15 +107,12 @@ static int __init crypto_null_mod_init(void)
 
 out_unregister_shash:
 	crypto_unregister_shash(&digest_null);
-out_unregister_algs:
-	crypto_unregister_alg(&cipher_null);
 out:
 	return ret;
 }
 
 static void __exit crypto_null_mod_fini(void)
 {
-	crypto_unregister_alg(&cipher_null);
 	crypto_unregister_shash(&digest_null);
 	crypto_unregister_skcipher(&skcipher_null);
 }
diff --git a/crypto/crypto_user.c b/crypto/crypto_user.c
index 3187e0d276f90..e8b6ae75f31f7 100644
--- a/crypto/crypto_user.c
+++ b/crypto/crypto_user.c
@@ -70,11 +70,9 @@ static struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact)
 
 static int crypto_report_cipher(struct sk_buff *skb, struct crypto_alg *alg)
 {
-	struct crypto_report_cipher rcipher;
-
-	memset(&rcipher, 0, sizeof(rcipher));
-
-	strscpy(rcipher.type, "cipher", sizeof(rcipher.type));
+	struct crypto_report_cipher rcipher = {
+		.type = "cipher",
+	};
 
 	rcipher.blocksize = alg->cra_blocksize;
 	rcipher.min_keysize = alg->cra_cipher.cia_min_keysize;
@@ -103,10 +101,10 @@ static int crypto_report_one(struct crypto_alg *alg,
 	if (nla_put_u32(skb, CRYPTOCFGA_PRIORITY_VAL, alg->cra_priority))
 		goto nla_put_failure;
 	if (alg->cra_flags & CRYPTO_ALG_LARVAL) {
-		struct crypto_report_larval rl;
+		struct crypto_report_larval rl = {
+			.type = "larval",
+		};
 
-		memset(&rl, 0, sizeof(rl));
-		strscpy(rl.type, "larval", sizeof(rl.type));
 		if (nla_put(skb, CRYPTOCFGA_REPORT_LARVAL, sizeof(rl), &rl))
 			goto nla_put_failure;
 		goto out;
diff --git a/crypto/df_sp80090a.c b/crypto/df_sp80090a.c
index b8134be6f7ad9..90e1973ee40c7 100644
--- a/crypto/df_sp80090a.c
+++ b/crypto/df_sp80090a.c
@@ -10,9 +10,9 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/string.h>
+#include <linux/unaligned.h>
 #include <crypto/aes.h>
 #include <crypto/df_sp80090a.h>
-#include <crypto/internal/drbg.h>
 
 static void drbg_kcapi_sym(struct aes_enckey *aeskey, unsigned char *outval,
 			   const struct drbg_string *in, u8 blocklen_bytes)
@@ -141,10 +141,10 @@ int crypto_drbg_ctr_df(struct aes_enckey *aeskey,
 	/* 10.4.2 step 2 -- calculate the entire length of all input data */
 	list_for_each_entry(seed, seedlist, list)
 		inputlen += seed->len;
-	drbg_cpu_to_be32(inputlen, &L_N[0]);
+	put_unaligned_be32(inputlen, &L_N[0]);
 
 	/* 10.4.2 step 3 */
-	drbg_cpu_to_be32(bytes_to_return, &L_N[4]);
+	put_unaligned_be32(bytes_to_return, &L_N[4]);
 
 	/* 10.4.2 step 5: length is L_N, input_string, one byte, padding */
 	padlen = (inputlen + sizeof(L_N) + 1) % (blocklen_bytes);
@@ -175,7 +175,7 @@ int crypto_drbg_ctr_df(struct aes_enckey *aeskey,
 		 * holds zeros after allocation -- even the increment of i
 		 * is irrelevant as the increment remains within length of i
 		 */
-		drbg_cpu_to_be32(i, iv);
+		put_unaligned_be32(i, iv);
 		/* 10.4.2 step 9.2 -- BCC and concatenation with temp */
 		drbg_ctr_bcc(aeskey, temp + templen, K, &bcc_list,
 			     blocklen_bytes, keylen);
diff --git a/crypto/drbg.c b/crypto/drbg.c
index 9204e6edb4268..d66c7211d6bb0 100644
--- a/crypto/drbg.c
+++ b/crypto/drbg.c
@@ -1,13 +1,9 @@
 /*
  * DRBG: Deterministic Random Bits Generator
- *       Based on NIST Recommended DRBG from NIST SP800-90A with the following
- *       properties:
- *		* CTR DRBG with DF with AES-128, AES-192, AES-256 cores
- *		* Hash DRBG with DF with SHA-1, SHA-256, SHA-384, SHA-512 cores
- *		* HMAC DRBG with DF with SHA-1, SHA-256, SHA-384, SHA-512 cores
- *		* with and without prediction resistance
+ *       Implementation of the HMAC SHA-512 DRBG from NIST SP800-90A
  *
  * Copyright Stephan Mueller <smueller@chronox.de>, 2014
+ * Copyright 2026 Google LLC
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -55,7 +51,7 @@
  * char data[DATALEN];
  *
  * drng = crypto_alloc_rng(drng_name, 0, 0);
- * err = crypto_rng_get_bytes(drng, &data, DATALEN);
+ * err = crypto_rng_get_bytes(drng, data, DATALEN);
  * crypto_free_rng(drng);
  *
  *
@@ -64,15 +60,13 @@
  * struct crypto_rng *drng;
  * int err;
  * char data[DATALEN];
- * struct drbg_string pers;
  * char personalization[11] = "some-string";
  *
- * drbg_string_fill(&pers, personalization, strlen(personalization));
  * drng = crypto_alloc_rng(drng_name, 0, 0);
  * // The reset completely re-initializes the DRBG with the provided
  * // personalization string
- * err = crypto_rng_reset(drng, &personalization, strlen(personalization));
- * err = crypto_rng_get_bytes(drng, &data, DATALEN);
+ * err = crypto_rng_reset(drng, personalization, strlen(personalization));
+ * err = crypto_rng_get_bytes(drng, data, DATALEN);
  * crypto_free_rng(drng);
  *
  *
@@ -82,13 +76,10 @@
  * int err;
  * char data[DATALEN];
  * char addtl_string[11] = "some-string";
- * string drbg_string addtl;
  *
- * drbg_string_fill(&addtl, addtl_string, strlen(addtl_string));
  * drng = crypto_alloc_rng(drng_name, 0, 0);
- * // The following call is a wrapper to crypto_rng_get_bytes() and returns
- * // the same error codes.
- * err = crypto_drbg_get_bytes_addtl(drng, &data, DATALEN, &addtl);
+ * err = crypto_rng_generate(drng, addtl_string, strlen(addtl_string),
+			     data, DATALEN);
  * crypto_free_rng(drng);
  *
  *
@@ -97,803 +88,129 @@
  * Just mix both scenarios above.
  */
 
-#include <crypto/drbg.h>
-#include <crypto/df_sp80090a.h>
-#include <crypto/internal/cipher.h>
+#include <crypto/internal/rng.h>
+#include <crypto/sha2.h>
+#include <linux/fips.h>
 #include <linux/kernel.h>
-#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/string_choices.h>
+#include <linux/unaligned.h>
 
-/***************************************************************
- * Backend cipher definitions available to DRBG
- ***************************************************************/
-
-/*
- * The order of the DRBG definitions here matter: every DRBG is registered
- * as stdrng. Each DRBG receives an increasing cra_priority values the later
- * they are defined in this array (see drbg_fill_array).
- *
- * HMAC DRBGs are favored over Hash DRBGs over CTR DRBGs, and the
- * HMAC-SHA512 / SHA256 / AES 256 over other ciphers. Thus, the
- * favored DRBGs are the latest entries in this array.
- */
-static const struct drbg_core drbg_cores[] = {
-#ifdef CONFIG_CRYPTO_DRBG_CTR
-	{
-		.flags = DRBG_CTR | DRBG_STRENGTH128,
-		.statelen = 32, /* 256 bits as defined in 10.2.1 */
-		.blocklen_bytes = 16,
-		.cra_name = "ctr_aes128",
-		.backend_cra_name = "aes",
-	}, {
-		.flags = DRBG_CTR | DRBG_STRENGTH192,
-		.statelen = 40, /* 320 bits as defined in 10.2.1 */
-		.blocklen_bytes = 16,
-		.cra_name = "ctr_aes192",
-		.backend_cra_name = "aes",
-	}, {
-		.flags = DRBG_CTR | DRBG_STRENGTH256,
-		.statelen = 48, /* 384 bits as defined in 10.2.1 */
-		.blocklen_bytes = 16,
-		.cra_name = "ctr_aes256",
-		.backend_cra_name = "aes",
-	},
-#endif /* CONFIG_CRYPTO_DRBG_CTR */
-#ifdef CONFIG_CRYPTO_DRBG_HASH
-	{
-		.flags = DRBG_HASH | DRBG_STRENGTH256,
-		.statelen = 111, /* 888 bits */
-		.blocklen_bytes = 48,
-		.cra_name = "sha384",
-		.backend_cra_name = "sha384",
-	}, {
-		.flags = DRBG_HASH | DRBG_STRENGTH256,
-		.statelen = 111, /* 888 bits */
-		.blocklen_bytes = 64,
-		.cra_name = "sha512",
-		.backend_cra_name = "sha512",
-	}, {
-		.flags = DRBG_HASH | DRBG_STRENGTH256,
-		.statelen = 55, /* 440 bits */
-		.blocklen_bytes = 32,
-		.cra_name = "sha256",
-		.backend_cra_name = "sha256",
-	},
-#endif /* CONFIG_CRYPTO_DRBG_HASH */
-#ifdef CONFIG_CRYPTO_DRBG_HMAC
-	{
-		.flags = DRBG_HMAC | DRBG_STRENGTH256,
-		.statelen = 48, /* block length of cipher */
-		.blocklen_bytes = 48,
-		.cra_name = "hmac_sha384",
-		.backend_cra_name = "hmac(sha384)",
-	}, {
-		.flags = DRBG_HMAC | DRBG_STRENGTH256,
-		.statelen = 32, /* block length of cipher */
-		.blocklen_bytes = 32,
-		.cra_name = "hmac_sha256",
-		.backend_cra_name = "hmac(sha256)",
-	}, {
-		.flags = DRBG_HMAC | DRBG_STRENGTH256,
-		.statelen = 64, /* block length of cipher */
-		.blocklen_bytes = 64,
-		.cra_name = "hmac_sha512",
-		.backend_cra_name = "hmac(sha512)",
-	},
-#endif /* CONFIG_CRYPTO_DRBG_HMAC */
-};
+/* State length in bytes */
+#define DRBG_STATE_LEN		SHA512_DIGEST_SIZE
 
-static int drbg_uninstantiate(struct drbg_state *drbg);
-
-/******************************************************************
- * Generic helper functions
- ******************************************************************/
+/* Security strength in bytes */
+#define DRBG_SEC_STRENGTH	(SHA512_DIGEST_SIZE / 2)
 
 /*
- * Return strength of DRBG according to SP800-90A section 8.4
- *
- * @flags DRBG flags reference
- *
- * Return: normalized strength in *bytes* value or 32 as default
- *	   to counter programming errors
+ * Maximum number of requests before reseeding is forced.
+ * SP800-90A allows this to be up to 2**48.  We use a lower value.
  */
-static inline unsigned short drbg_sec_strength(drbg_flag_t flags)
-{
-	switch (flags & DRBG_STRENGTH_MASK) {
-	case DRBG_STRENGTH128:
-		return 16;
-	case DRBG_STRENGTH192:
-		return 24;
-	case DRBG_STRENGTH256:
-		return 32;
-	default:
-		return 32;
-	}
-}
+#define DRBG_MAX_REQUESTS	4096
 
 /*
- * FIPS 140-2 continuous self test for the noise source
- * The test is performed on the noise source input data. Thus, the function
- * implicitly knows the size of the buffer to be equal to the security
- * strength.
- *
- * Note, this function disregards the nonce trailing the entropy data during
- * initial seeding.
- *
- * drbg->drbg_mutex must have been taken.
- *
- * @drbg DRBG handle
- * @entropy buffer of seed data to be checked
- *
- * return:
- *	%true on success
- *	%false when the CTRNG is not yet primed
+ * Maximum number of random bytes that can be requested at once.
+ * SP800-90A allows up to 2**19 bits, which is 2**16 bytes.
  */
-static bool drbg_fips_continuous_test(struct drbg_state *drbg,
-				      const unsigned char *entropy)
-	__must_hold(&drbg->drbg_mutex)
-{
-	unsigned short entropylen = drbg_sec_strength(drbg->core->flags);
-
-	if (!IS_ENABLED(CONFIG_CRYPTO_FIPS))
-		return true;
-
-	/* skip test if we test the overall system */
-	if (list_empty(&drbg->test_data.list))
-		return true;
-	/* only perform test in FIPS mode */
-	if (!fips_enabled)
-		return true;
-
-	if (!drbg->fips_primed) {
-		/* Priming of FIPS test */
-		memcpy(drbg->prev, entropy, entropylen);
-		drbg->fips_primed = true;
-		/* priming: another round is needed */
-		return false;
-	}
-	if (!memcmp(drbg->prev, entropy, entropylen))
-		panic("DRBG continuous self test failed\n");
-	memcpy(drbg->prev, entropy, entropylen);
-
-	/* the test shall pass when the two values are not equal */
-	return true;
-}
-
-/******************************************************************
- * CTR DRBG callback functions
- ******************************************************************/
-
-#ifdef CONFIG_CRYPTO_DRBG_CTR
-#define CRYPTO_DRBG_CTR_STRING "CTR "
-MODULE_ALIAS_CRYPTO("drbg_pr_ctr_aes256");
-MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes256");
-MODULE_ALIAS_CRYPTO("drbg_pr_ctr_aes192");
-MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes192");
-MODULE_ALIAS_CRYPTO("drbg_pr_ctr_aes128");
-MODULE_ALIAS_CRYPTO("drbg_nopr_ctr_aes128");
-
-static int drbg_init_sym_kernel(struct drbg_state *drbg);
-static int drbg_fini_sym_kernel(struct drbg_state *drbg);
-static int drbg_kcapi_sym_ctr(struct drbg_state *drbg,
-			      u8 *inbuf, u32 inbuflen,
-			      u8 *outbuf, u32 outlen);
-#define DRBG_OUTSCRATCHLEN 256
-
-static int drbg_ctr_df(struct drbg_state *drbg,
-		       unsigned char *df_data, size_t bytes_to_return,
-		       struct list_head *seedlist)
-{
-	return crypto_drbg_ctr_df(drbg->priv_data, df_data, drbg_statelen(drbg),
-				  seedlist, drbg_blocklen(drbg), drbg_statelen(drbg));
-}
+#define DRBG_MAX_REQUEST_BYTES	(1 << 16)
 
 /*
- * update function of CTR DRBG as defined in 10.2.1.2
- *
- * The reseed variable has an enhanced meaning compared to the update
- * functions of the other DRBGs as follows:
- * 0 => initial seed from initialization
- * 1 => reseed via drbg_seed
- * 2 => first invocation from drbg_ctr_update when addtl is present. In
- *      this case, the df_data scratchpad is not deleted so that it is
- *      available for another calls to prevent calling the DF function
- *      again.
- * 3 => second invocation from drbg_ctr_update. When the update function
- *      was called with addtl, the df_data memory already contains the
- *      DFed addtl information and we do not need to call DF again.
+ * Maximum length of additional info and personalization strings, in bytes.
+ * SP800-90A allows up to 2**35 bits, i.e. 2**32 bytes.  We use 2**32 - 2 bytes
+ * so that the value never quite completely fills the range of a size_t,
+ * allowing the health check to verify that larger values are rejected.
  */
-static int drbg_ctr_update(struct drbg_state *drbg, struct list_head *seed,
-			   int reseed)
-{
-	int ret = -EFAULT;
-	/* 10.2.1.2 step 1 */
-	unsigned char *temp = drbg->scratchpad;
-	unsigned char *df_data = drbg->scratchpad + drbg_statelen(drbg) +
-				 drbg_blocklen(drbg);
-
-	if (3 > reseed)
-		memset(df_data, 0, drbg_statelen(drbg));
-
-	if (!reseed) {
-		/*
-		 * The DRBG uses the CTR mode of the underlying AES cipher. The
-		 * CTR mode increments the counter value after the AES operation
-		 * but SP800-90A requires that the counter is incremented before
-		 * the AES operation. Hence, we increment it at the time we set
-		 * it by one.
-		 */
-		crypto_inc(drbg->V, drbg_blocklen(drbg));
-
-		ret = crypto_skcipher_setkey(drbg->ctr_handle, drbg->C,
-					     drbg_keylen(drbg));
-		if (ret)
-			goto out;
-	}
-
-	/* 10.2.1.3.2 step 2 and 10.2.1.4.2 step 2 */
-	if (seed) {
-		ret = drbg_ctr_df(drbg, df_data, drbg_statelen(drbg), seed);
-		if (ret)
-			goto out;
-	}
-
-	ret = drbg_kcapi_sym_ctr(drbg, df_data, drbg_statelen(drbg),
-				 temp, drbg_statelen(drbg));
-	if (ret)
-		return ret;
-
-	/* 10.2.1.2 step 5 */
-	ret = crypto_skcipher_setkey(drbg->ctr_handle, temp,
-				     drbg_keylen(drbg));
-	if (ret)
-		goto out;
-	/* 10.2.1.2 step 6 */
-	memcpy(drbg->V, temp + drbg_keylen(drbg), drbg_blocklen(drbg));
-	/* See above: increment counter by one to compensate timing of CTR op */
-	crypto_inc(drbg->V, drbg_blocklen(drbg));
-	ret = 0;
-
-out:
-	memset(temp, 0, drbg_statelen(drbg) + drbg_blocklen(drbg));
-	if (2 != reseed)
-		memset(df_data, 0, drbg_statelen(drbg));
-	return ret;
-}
-
-/*
- * scratchpad use: drbg_ctr_update is called independently from
- * drbg_ctr_extract_bytes. Therefore, the scratchpad is reused
- */
-/* Generate function of CTR DRBG as defined in 10.2.1.5.2 */
-static int drbg_ctr_generate(struct drbg_state *drbg,
-			     unsigned char *buf, unsigned int buflen,
-			     struct list_head *addtl)
-{
-	int ret;
-	int len = min_t(int, buflen, INT_MAX);
-
-	/* 10.2.1.5.2 step 2 */
-	if (addtl && !list_empty(addtl)) {
-		ret = drbg_ctr_update(drbg, addtl, 2);
-		if (ret)
-			return 0;
-	}
-
-	/* 10.2.1.5.2 step 4.1 */
-	ret = drbg_kcapi_sym_ctr(drbg, NULL, 0, buf, len);
-	if (ret)
-		return ret;
-
-	/* 10.2.1.5.2 step 6 */
-	ret = drbg_ctr_update(drbg, NULL, 3);
-	if (ret)
-		len = ret;
-
-	return len;
-}
-
-static const struct drbg_state_ops drbg_ctr_ops = {
-	.update		= drbg_ctr_update,
-	.generate	= drbg_ctr_generate,
-	.crypto_init	= drbg_init_sym_kernel,
-	.crypto_fini	= drbg_fini_sym_kernel,
+#define DRBG_MAX_ADDTL_BYTES	(U32_MAX - 1)
+
+struct drbg_state {
+	struct mutex drbg_mutex;	/* lock around DRBG */
+	u8 V[DRBG_STATE_LEN];		/* internal state -- 10.1.2.1 1a */
+	struct hmac_sha512_key key;	/* current key -- 10.1.2.1 1b */
+	/* Number of RNG requests since last reseed -- 10.1.2.1 1c */
+	size_t reseed_ctr;
+	bool instantiated;
+	struct crypto_rng *jent;
+	const u8 *test_entropy;
+	size_t test_entropylen;
 };
-#endif /* CONFIG_CRYPTO_DRBG_CTR */
 
 /******************************************************************
- * HMAC DRBG callback functions
+ * HMAC DRBG functions
  ******************************************************************/
 
-#if defined(CONFIG_CRYPTO_DRBG_HASH) || defined(CONFIG_CRYPTO_DRBG_HMAC)
-static int drbg_kcapi_hash(struct drbg_state *drbg, unsigned char *outval,
-			   const struct list_head *in);
-static void drbg_kcapi_hmacsetkey(struct drbg_state *drbg,
-				  const unsigned char *key);
-static int drbg_init_hash_kernel(struct drbg_state *drbg);
-static int drbg_fini_hash_kernel(struct drbg_state *drbg);
-#endif /* (CONFIG_CRYPTO_DRBG_HASH || CONFIG_CRYPTO_DRBG_HMAC) */
-
-#ifdef CONFIG_CRYPTO_DRBG_HMAC
-#define CRYPTO_DRBG_HMAC_STRING "HMAC "
-MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha512");
-MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha512");
-MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha384");
-MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha384");
-MODULE_ALIAS_CRYPTO("drbg_pr_hmac_sha256");
-MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha256");
-
 /* update function of HMAC DRBG as defined in 10.1.2.2 */
-static int drbg_hmac_update(struct drbg_state *drbg, struct list_head *seed,
-			    int reseed)
+static void drbg_hmac_update(struct drbg_state *drbg,
+			     const u8 *data1, size_t data1_len,
+			     const u8 *data2, size_t data2_len)
 {
-	int ret = -EFAULT;
-	int i = 0;
-	struct drbg_string seed1, seed2, vdata;
-	LIST_HEAD(seedlist);
-	LIST_HEAD(vdatalist);
-
-	if (!reseed) {
-		/* 10.1.2.3 step 2 -- memset(0) of C is implicit with kzalloc */
-		memset(drbg->V, 1, drbg_statelen(drbg));
-		drbg_kcapi_hmacsetkey(drbg, drbg->C);
-	}
+	struct hmac_sha512_ctx hmac_ctx;
+	u8 new_key[DRBG_STATE_LEN];
 
-	drbg_string_fill(&seed1, drbg->V, drbg_statelen(drbg));
-	list_add_tail(&seed1.list, &seedlist);
-	/* buffer of seed2 will be filled in for loop below with one byte */
-	drbg_string_fill(&seed2, NULL, 1);
-	list_add_tail(&seed2.list, &seedlist);
-	/* input data of seed is allowed to be NULL at this point */
-	if (seed)
-		list_splice_tail(seed, &seedlist);
-
-	drbg_string_fill(&vdata, drbg->V, drbg_statelen(drbg));
-	list_add_tail(&vdata.list, &vdatalist);
-	for (i = 2; 0 < i; i--) {
-		/* first round uses 0x0, second 0x1 */
-		unsigned char prefix = DRBG_PREFIX0;
-		if (1 == i)
-			prefix = DRBG_PREFIX1;
+	for (u8 i = 0; i < 2; i++) {
 		/* 10.1.2.2 step 1 and 4 -- concatenation and HMAC for key */
-		seed2.buf = &prefix;
-		ret = drbg_kcapi_hash(drbg, drbg->C, &seedlist);
-		if (ret)
-			return ret;
-		drbg_kcapi_hmacsetkey(drbg, drbg->C);
+		hmac_sha512_init(&hmac_ctx, &drbg->key);
+		hmac_sha512_update(&hmac_ctx, drbg->V, DRBG_STATE_LEN);
+		hmac_sha512_update(&hmac_ctx, &i, 1);
+		hmac_sha512_update(&hmac_ctx, data1, data1_len);
+		hmac_sha512_update(&hmac_ctx, data2, data2_len);
+		hmac_sha512_final(&hmac_ctx, new_key);
+		hmac_sha512_preparekey(&drbg->key, new_key, DRBG_STATE_LEN);
 
 		/* 10.1.2.2 step 2 and 5 -- HMAC for V */
-		ret = drbg_kcapi_hash(drbg, drbg->V, &vdatalist);
-		if (ret)
-			return ret;
+		hmac_sha512(&drbg->key, drbg->V, DRBG_STATE_LEN, drbg->V);
 
 		/* 10.1.2.2 step 3 */
-		if (!seed)
-			return ret;
+		if (data1_len == 0 && data2_len == 0)
+			break;
 	}
-
-	return 0;
+	memzero_explicit(new_key, sizeof(new_key));
 }
 
 /* generate function of HMAC DRBG as defined in 10.1.2.5 */
-static int drbg_hmac_generate(struct drbg_state *drbg,
-			      unsigned char *buf,
-			      unsigned int buflen,
-			      struct list_head *addtl)
+static void drbg_hmac_generate(struct drbg_state *drbg, u8 *out, size_t outlen,
+			       const u8 *addtl1, size_t addtl1_len)
 {
-	int len = 0;
-	int ret = 0;
-	struct drbg_string data;
-	LIST_HEAD(datalist);
+	u8 addtl2[32];
+	size_t addtl2_len = 0;
 
-	/* 10.1.2.5 step 2 */
-	if (addtl && !list_empty(addtl)) {
-		ret = drbg_hmac_update(drbg, addtl, 1);
-		if (ret)
-			return ret;
+	/*
+	 * Append some bytes from get_random_bytes() to the additional input
+	 * string, except when in test mode (as it would break the tests).
+	 * Using a nonempty additional input string works around the forward
+	 * secrecy bug in HMAC_DRBG described by Woodage & Shumow (2018)
+	 * (https://eprint.iacr.org/2018/349.pdf).  Filling the string with
+	 * get_random_bytes() rather than a fixed value is safer still, and in
+	 * particular makes random.c reseeds be immediately reflected.
+	 *
+	 * Note that there's no need to pull bytes from jitterentropy here too,
+	 * since FIPS doesn't require any entropy in the additional input.
+	 */
+	if (drbg->test_entropylen == 0) {
+		get_random_bytes(addtl2, sizeof(addtl2));
+		addtl2_len = sizeof(addtl2);
 	}
 
-	drbg_string_fill(&data, drbg->V, drbg_statelen(drbg));
-	list_add_tail(&data.list, &datalist);
-	while (len < buflen) {
-		unsigned int outlen = 0;
+	/* 10.1.2.5 step 2 */
+	if (addtl1_len || addtl2_len)
+		drbg_hmac_update(drbg, addtl1, addtl1_len, addtl2, addtl2_len);
+
+	while (outlen) {
+		size_t n = min(DRBG_STATE_LEN, outlen);
+
 		/* 10.1.2.5 step 4.1 */
-		ret = drbg_kcapi_hash(drbg, drbg->V, &datalist);
-		if (ret)
-			return ret;
-		outlen = (drbg_blocklen(drbg) < (buflen - len)) ?
-			  drbg_blocklen(drbg) : (buflen - len);
+		hmac_sha512(&drbg->key, drbg->V, DRBG_STATE_LEN, drbg->V);
 
 		/* 10.1.2.5 step 4.2 */
-		memcpy(buf + len, drbg->V, outlen);
-		len += outlen;
+		memcpy(out, drbg->V, n);
+		out += n;
+		outlen -= n;
 	}
 
 	/* 10.1.2.5 step 6 */
-	if (addtl && !list_empty(addtl))
-		ret = drbg_hmac_update(drbg, addtl, 1);
-	else
-		ret = drbg_hmac_update(drbg, NULL, 1);
-	if (ret)
-		return ret;
-
-	return len;
-}
-
-static const struct drbg_state_ops drbg_hmac_ops = {
-	.update		= drbg_hmac_update,
-	.generate	= drbg_hmac_generate,
-	.crypto_init	= drbg_init_hash_kernel,
-	.crypto_fini	= drbg_fini_hash_kernel,
-};
-#endif /* CONFIG_CRYPTO_DRBG_HMAC */
-
-/******************************************************************
- * Hash DRBG callback functions
- ******************************************************************/
-
-#ifdef CONFIG_CRYPTO_DRBG_HASH
-#define CRYPTO_DRBG_HASH_STRING "HASH "
-MODULE_ALIAS_CRYPTO("drbg_pr_sha512");
-MODULE_ALIAS_CRYPTO("drbg_nopr_sha512");
-MODULE_ALIAS_CRYPTO("drbg_pr_sha384");
-MODULE_ALIAS_CRYPTO("drbg_nopr_sha384");
-MODULE_ALIAS_CRYPTO("drbg_pr_sha256");
-MODULE_ALIAS_CRYPTO("drbg_nopr_sha256");
-
-/*
- * Increment buffer
- *
- * @dst buffer to increment
- * @add value to add
- */
-static inline void drbg_add_buf(unsigned char *dst, size_t dstlen,
-				const unsigned char *add, size_t addlen)
-{
-	/* implied: dstlen > addlen */
-	unsigned char *dstptr;
-	const unsigned char *addptr;
-	unsigned int remainder = 0;
-	size_t len = addlen;
-
-	dstptr = dst + (dstlen-1);
-	addptr = add + (addlen-1);
-	while (len) {
-		remainder += *dstptr + *addptr;
-		*dstptr = remainder & 0xff;
-		remainder >>= 8;
-		len--; dstptr--; addptr--;
-	}
-	len = dstlen - addlen;
-	while (len && remainder > 0) {
-		remainder = *dstptr + 1;
-		*dstptr = remainder & 0xff;
-		remainder >>= 8;
-		len--; dstptr--;
-	}
-}
-
-/*
- * scratchpad usage: as drbg_hash_update and drbg_hash_df are used
- * interlinked, the scratchpad is used as follows:
- * drbg_hash_update
- *	start: drbg->scratchpad
- *	length: drbg_statelen(drbg)
- * drbg_hash_df:
- *	start: drbg->scratchpad + drbg_statelen(drbg)
- *	length: drbg_blocklen(drbg)
- *
- * drbg_hash_process_addtl uses the scratchpad, but fully completes
- * before either of the functions mentioned before are invoked. Therefore,
- * drbg_hash_process_addtl does not need to be specifically considered.
- */
-
-/* Derivation Function for Hash DRBG as defined in 10.4.1 */
-static int drbg_hash_df(struct drbg_state *drbg,
-			unsigned char *outval, size_t outlen,
-			struct list_head *entropylist)
-{
-	int ret = 0;
-	size_t len = 0;
-	unsigned char input[5];
-	unsigned char *tmp = drbg->scratchpad + drbg_statelen(drbg);
-	struct drbg_string data;
-
-	/* 10.4.1 step 3 */
-	input[0] = 1;
-	drbg_cpu_to_be32((outlen * 8), &input[1]);
-
-	/* 10.4.1 step 4.1 -- concatenation of data for input into hash */
-	drbg_string_fill(&data, input, 5);
-	list_add(&data.list, entropylist);
-
-	/* 10.4.1 step 4 */
-	while (len < outlen) {
-		short blocklen = 0;
-		/* 10.4.1 step 4.1 */
-		ret = drbg_kcapi_hash(drbg, tmp, entropylist);
-		if (ret)
-			goto out;
-		/* 10.4.1 step 4.2 */
-		input[0]++;
-		blocklen = (drbg_blocklen(drbg) < (outlen - len)) ?
-			    drbg_blocklen(drbg) : (outlen - len);
-		memcpy(outval + len, tmp, blocklen);
-		len += blocklen;
-	}
-
-out:
-	memset(tmp, 0, drbg_blocklen(drbg));
-	return ret;
-}
-
-/* update function for Hash DRBG as defined in 10.1.1.2 / 10.1.1.3 */
-static int drbg_hash_update(struct drbg_state *drbg, struct list_head *seed,
-			    int reseed)
-{
-	int ret = 0;
-	struct drbg_string data1, data2;
-	LIST_HEAD(datalist);
-	LIST_HEAD(datalist2);
-	unsigned char *V = drbg->scratchpad;
-	unsigned char prefix = DRBG_PREFIX1;
-
-	if (!seed)
-		return -EINVAL;
-
-	if (reseed) {
-		/* 10.1.1.3 step 1 */
-		memcpy(V, drbg->V, drbg_statelen(drbg));
-		drbg_string_fill(&data1, &prefix, 1);
-		list_add_tail(&data1.list, &datalist);
-		drbg_string_fill(&data2, V, drbg_statelen(drbg));
-		list_add_tail(&data2.list, &datalist);
-	}
-	list_splice_tail(seed, &datalist);
-
-	/* 10.1.1.2 / 10.1.1.3 step 2 and 3 */
-	ret = drbg_hash_df(drbg, drbg->V, drbg_statelen(drbg), &datalist);
-	if (ret)
-		goto out;
-
-	/* 10.1.1.2 / 10.1.1.3 step 4  */
-	prefix = DRBG_PREFIX0;
-	drbg_string_fill(&data1, &prefix, 1);
-	list_add_tail(&data1.list, &datalist2);
-	drbg_string_fill(&data2, drbg->V, drbg_statelen(drbg));
-	list_add_tail(&data2.list, &datalist2);
-	/* 10.1.1.2 / 10.1.1.3 step 4 */
-	ret = drbg_hash_df(drbg, drbg->C, drbg_statelen(drbg), &datalist2);
-
-out:
-	memset(drbg->scratchpad, 0, drbg_statelen(drbg));
-	return ret;
-}
-
-/* processing of additional information string for Hash DRBG */
-static int drbg_hash_process_addtl(struct drbg_state *drbg,
-				   struct list_head *addtl)
-{
-	int ret = 0;
-	struct drbg_string data1, data2;
-	LIST_HEAD(datalist);
-	unsigned char prefix = DRBG_PREFIX2;
-
-	/* 10.1.1.4 step 2 */
-	if (!addtl || list_empty(addtl))
-		return 0;
-
-	/* 10.1.1.4 step 2a */
-	drbg_string_fill(&data1, &prefix, 1);
-	drbg_string_fill(&data2, drbg->V, drbg_statelen(drbg));
-	list_add_tail(&data1.list, &datalist);
-	list_add_tail(&data2.list, &datalist);
-	list_splice_tail(addtl, &datalist);
-	ret = drbg_kcapi_hash(drbg, drbg->scratchpad, &datalist);
-	if (ret)
-		goto out;
-
-	/* 10.1.1.4 step 2b */
-	drbg_add_buf(drbg->V, drbg_statelen(drbg),
-		     drbg->scratchpad, drbg_blocklen(drbg));
-
-out:
-	memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
-	return ret;
-}
-
-/* Hashgen defined in 10.1.1.4 */
-static int drbg_hash_hashgen(struct drbg_state *drbg,
-			     unsigned char *buf,
-			     unsigned int buflen)
-{
-	int len = 0;
-	int ret = 0;
-	unsigned char *src = drbg->scratchpad;
-	unsigned char *dst = drbg->scratchpad + drbg_statelen(drbg);
-	struct drbg_string data;
-	LIST_HEAD(datalist);
-
-	/* 10.1.1.4 step hashgen 2 */
-	memcpy(src, drbg->V, drbg_statelen(drbg));
-
-	drbg_string_fill(&data, src, drbg_statelen(drbg));
-	list_add_tail(&data.list, &datalist);
-	while (len < buflen) {
-		unsigned int outlen = 0;
-		/* 10.1.1.4 step hashgen 4.1 */
-		ret = drbg_kcapi_hash(drbg, dst, &datalist);
-		if (ret) {
-			len = ret;
-			goto out;
-		}
-		outlen = (drbg_blocklen(drbg) < (buflen - len)) ?
-			  drbg_blocklen(drbg) : (buflen - len);
-		/* 10.1.1.4 step hashgen 4.2 */
-		memcpy(buf + len, dst, outlen);
-		len += outlen;
-		/* 10.1.1.4 hashgen step 4.3 */
-		if (len < buflen)
-			crypto_inc(src, drbg_statelen(drbg));
-	}
+	drbg_hmac_update(drbg, addtl1, addtl1_len, addtl2, addtl2_len);
 
-out:
-	memset(drbg->scratchpad, 0,
-	       (drbg_statelen(drbg) + drbg_blocklen(drbg)));
-	return len;
-}
-
-/* generate function for Hash DRBG as defined in  10.1.1.4 */
-static int drbg_hash_generate(struct drbg_state *drbg,
-			      unsigned char *buf, unsigned int buflen,
-			      struct list_head *addtl)
-{
-	int len = 0;
-	int ret = 0;
-	union {
-		unsigned char req[8];
-		__be64 req_int;
-	} u;
-	unsigned char prefix = DRBG_PREFIX3;
-	struct drbg_string data1, data2;
-	LIST_HEAD(datalist);
-
-	/* 10.1.1.4 step 2 */
-	ret = drbg_hash_process_addtl(drbg, addtl);
-	if (ret)
-		return ret;
-	/* 10.1.1.4 step 3 */
-	len = drbg_hash_hashgen(drbg, buf, buflen);
-
-	/* this is the value H as documented in 10.1.1.4 */
-	/* 10.1.1.4 step 4 */
-	drbg_string_fill(&data1, &prefix, 1);
-	list_add_tail(&data1.list, &datalist);
-	drbg_string_fill(&data2, drbg->V, drbg_statelen(drbg));
-	list_add_tail(&data2.list, &datalist);
-	ret = drbg_kcapi_hash(drbg, drbg->scratchpad, &datalist);
-	if (ret) {
-		len = ret;
-		goto out;
-	}
-
-	/* 10.1.1.4 step 5 */
-	drbg_add_buf(drbg->V, drbg_statelen(drbg),
-		     drbg->scratchpad, drbg_blocklen(drbg));
-	drbg_add_buf(drbg->V, drbg_statelen(drbg),
-		     drbg->C, drbg_statelen(drbg));
-	u.req_int = cpu_to_be64(drbg->reseed_ctr);
-	drbg_add_buf(drbg->V, drbg_statelen(drbg), u.req, 8);
-
-out:
-	memset(drbg->scratchpad, 0, drbg_blocklen(drbg));
-	return len;
-}
-
-/*
- * scratchpad usage: as update and generate are used isolated, both
- * can use the scratchpad
- */
-static const struct drbg_state_ops drbg_hash_ops = {
-	.update		= drbg_hash_update,
-	.generate	= drbg_hash_generate,
-	.crypto_init	= drbg_init_hash_kernel,
-	.crypto_fini	= drbg_fini_hash_kernel,
-};
-#endif /* CONFIG_CRYPTO_DRBG_HASH */
-
-/******************************************************************
- * Functions common for DRBG implementations
- ******************************************************************/
-
-static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed,
-			      int reseed, enum drbg_seed_state new_seed_state)
-{
-	int ret = drbg->d_ops->update(drbg, seed, reseed);
-
-	if (ret)
-		return ret;
-
-	drbg->seeded = new_seed_state;
-	drbg->last_seed_time = jiffies;
-	/* 10.1.1.2 / 10.1.1.3 step 5 */
-	drbg->reseed_ctr = 1;
-
-	switch (drbg->seeded) {
-	case DRBG_SEED_STATE_UNSEEDED:
-		/* Impossible, but handle it to silence compiler warnings. */
-		fallthrough;
-	case DRBG_SEED_STATE_PARTIAL:
-		/*
-		 * Require frequent reseeds until the seed source is
-		 * fully initialized.
-		 */
-		drbg->reseed_threshold = 50;
-		break;
-
-	case DRBG_SEED_STATE_FULL:
-		/*
-		 * Seed source has become fully initialized, frequent
-		 * reseeds no longer required.
-		 */
-		drbg->reseed_threshold = drbg_max_requests(drbg);
-		break;
-	}
-
-	return ret;
-}
-
-static inline void drbg_get_random_bytes(struct drbg_state *drbg,
-					 unsigned char *entropy,
-					 unsigned int entropylen)
-	__must_hold(&drbg->drbg_mutex)
-{
-	do
-		get_random_bytes(entropy, entropylen);
-	while (!drbg_fips_continuous_test(drbg, entropy));
-}
-
-static int drbg_seed_from_random(struct drbg_state *drbg)
-	__must_hold(&drbg->drbg_mutex)
-{
-	struct drbg_string data;
-	LIST_HEAD(seedlist);
-	unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
-	unsigned char entropy[32];
-	int ret;
-
-	BUG_ON(!entropylen);
-	BUG_ON(entropylen > sizeof(entropy));
-
-	drbg_string_fill(&data, entropy, entropylen);
-	list_add_tail(&data.list, &seedlist);
-
-	drbg_get_random_bytes(drbg, entropy, entropylen);
-
-	ret = __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL);
-
-	memzero_explicit(entropy, entropylen);
-	return ret;
-}
-
-static bool drbg_nopr_reseed_interval_elapsed(struct drbg_state *drbg)
-{
-	unsigned long next_reseed;
-
-	/* Don't ever reseed from get_random_bytes() in test mode. */
-	if (list_empty(&drbg->test_data.list))
-		return false;
-
-	/*
-	 * Obtain fresh entropy for the nopr DRBGs after 300s have
-	 * elapsed in order to still achieve sort of partial
-	 * prediction resistance over the time domain at least. Note
-	 * that the period of 300s has been chosen to match the
-	 * CRNG_RESEED_INTERVAL of the get_random_bytes()' chacha
-	 * rngs.
-	 */
-	next_reseed = drbg->last_seed_time + 300 * HZ;
-	return time_after(jiffies, next_reseed);
+	memzero_explicit(addtl2, sizeof(addtl2));
 }
 
 /*
@@ -901,33 +218,32 @@ static bool drbg_nopr_reseed_interval_elapsed(struct drbg_state *drbg)
  *
  * @drbg: DRBG state struct
  * @pers: personalization / additional information buffer
- * @reseed: 0 for initial seed process, 1 for reseeding
+ * @pers_len: length of @pers in bytes
+ * @reseed: false for initial seeding (instantiation), true for reseeding
  *
  * return:
  *	0 on success
  *	error value otherwise
  */
-static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
+static int drbg_seed(struct drbg_state *drbg, const u8 *pers, size_t pers_len,
 		     bool reseed)
 	__must_hold(&drbg->drbg_mutex)
 {
 	int ret;
-	unsigned char entropy[((32 + 16) * 2)];
-	unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
-	struct drbg_string data1;
-	LIST_HEAD(seedlist);
-	enum drbg_seed_state new_seed_state = DRBG_SEED_STATE_FULL;
+	u8 entropy_buf[(32 + 16) * 2];
+	size_t entropylen;
+	const u8 *entropy;
 
 	/* 9.1 / 9.2 / 9.3.1 step 3 */
-	if (pers && pers->len > (drbg_max_addtl(drbg))) {
+	if (pers_len > DRBG_MAX_ADDTL_BYTES) {
 		pr_devel("DRBG: personalization string too long %zu\n",
-			 pers->len);
+			 pers_len);
 		return -EINVAL;
 	}
 
-	if (list_empty(&drbg->test_data.list)) {
-		drbg_string_fill(&data1, drbg->test_data.buf,
-				 drbg->test_data.len);
+	if (drbg->test_entropylen) {
+		entropy = drbg->test_entropy;
+		entropylen = drbg->test_entropylen;
 		pr_devel("DRBG: using test entropy\n");
 	} else {
 		/*
@@ -938,20 +254,18 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
 		 * of the strength. The consideration of a nonce is only
 		 * applicable during initial seeding.
 		 */
-		BUG_ON(!entropylen);
+		entropy = entropy_buf;
 		if (!reseed)
-			entropylen = ((entropylen + 1) / 2) * 3;
-		BUG_ON((entropylen * 2) > sizeof(entropy));
+			entropylen = ((DRBG_SEC_STRENGTH + 1) / 2) * 3;
+		else
+			entropylen = DRBG_SEC_STRENGTH;
+		BUG_ON(entropylen * 2 > sizeof(entropy_buf));
 
 		/* Get seed from in-kernel /dev/urandom */
-		if (!rng_is_initialized())
-			new_seed_state = DRBG_SEED_STATE_PARTIAL;
-
-		drbg_get_random_bytes(drbg, entropy, entropylen);
+		get_random_bytes(entropy_buf, entropylen);
 
 		if (!drbg->jent) {
-			drbg_string_fill(&data1, entropy, entropylen);
-			pr_devel("DRBG: (re)seeding with %u bytes of entropy\n",
+			pr_devel("DRBG: (re)seeding with %zu bytes of entropy\n",
 				 entropylen);
 		} else {
 			/*
@@ -959,7 +273,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
 			 * fatal only in FIPS mode.
 			 */
 			ret = crypto_rng_get_bytes(drbg->jent,
-						   entropy + entropylen,
+						   &entropy_buf[entropylen],
 						   entropylen);
 			if (fips_enabled && ret) {
 				pr_devel("DRBG: jent failed with %d\n", ret);
@@ -982,238 +296,94 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
 					goto out;
 			}
 
-			drbg_string_fill(&data1, entropy, entropylen * 2);
-			pr_devel("DRBG: (re)seeding with %u bytes of entropy\n",
-				 entropylen * 2);
+			entropylen *= 2;
+			pr_devel("DRBG: (re)seeding with %zu bytes of entropy\n",
+				 entropylen);
 		}
 	}
-	list_add_tail(&data1.list, &seedlist);
 
-	/*
-	 * concatenation of entropy with personalization str / addtl input)
-	 * the variable pers is directly handed in by the caller, so check its
-	 * contents whether it is appropriate
-	 */
-	if (pers && pers->buf && 0 < pers->len) {
-		list_add_tail(&pers->list, &seedlist);
+	if (pers_len)
 		pr_devel("DRBG: using personalization string\n");
-	}
-
-	if (!reseed) {
-		memset(drbg->V, 0, drbg_statelen(drbg));
-		memset(drbg->C, 0, drbg_statelen(drbg));
-	}
-
-	ret = __drbg_seed(drbg, &seedlist, reseed, new_seed_state);
 
+	drbg_hmac_update(drbg, entropy, entropylen, pers, pers_len);
+	drbg->reseed_ctr = 1;
+	ret = 0;
 out:
-	memzero_explicit(entropy, entropylen * 2);
+	memzero_explicit(entropy_buf, sizeof(entropy_buf));
 
 	return ret;
 }
 
-/* Free all substructures in a DRBG state without the DRBG state structure */
-static inline void drbg_dealloc_state(struct drbg_state *drbg)
-{
-	if (!drbg)
-		return;
-	kfree_sensitive(drbg->Vbuf);
-	drbg->Vbuf = NULL;
-	drbg->V = NULL;
-	kfree_sensitive(drbg->Cbuf);
-	drbg->Cbuf = NULL;
-	drbg->C = NULL;
-	kfree_sensitive(drbg->scratchpadbuf);
-	drbg->scratchpadbuf = NULL;
-	drbg->reseed_ctr = 0;
-	drbg->d_ops = NULL;
-	drbg->core = NULL;
-	if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) {
-		kfree_sensitive(drbg->prev);
-		drbg->prev = NULL;
-		drbg->fips_primed = false;
-	}
-}
-
 /*
- * Allocate all sub-structures for a DRBG state.
- * The DRBG state structure must already be allocated.
- */
-static inline int drbg_alloc_state(struct drbg_state *drbg)
-{
-	int ret = -ENOMEM;
-	unsigned int sb_size = 0;
-
-	switch (drbg->core->flags & DRBG_TYPE_MASK) {
-#ifdef CONFIG_CRYPTO_DRBG_HMAC
-	case DRBG_HMAC:
-		drbg->d_ops = &drbg_hmac_ops;
-		break;
-#endif /* CONFIG_CRYPTO_DRBG_HMAC */
-#ifdef CONFIG_CRYPTO_DRBG_HASH
-	case DRBG_HASH:
-		drbg->d_ops = &drbg_hash_ops;
-		break;
-#endif /* CONFIG_CRYPTO_DRBG_HASH */
-#ifdef CONFIG_CRYPTO_DRBG_CTR
-	case DRBG_CTR:
-		drbg->d_ops = &drbg_ctr_ops;
-		break;
-#endif /* CONFIG_CRYPTO_DRBG_CTR */
-	default:
-		ret = -EOPNOTSUPP;
-		goto err;
-	}
-
-	ret = drbg->d_ops->crypto_init(drbg);
-	if (ret < 0)
-		goto err;
-
-	drbg->Vbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL);
-	if (!drbg->Vbuf) {
-		ret = -ENOMEM;
-		goto fini;
-	}
-	drbg->V = PTR_ALIGN(drbg->Vbuf, ret + 1);
-	drbg->Cbuf = kmalloc(drbg_statelen(drbg) + ret, GFP_KERNEL);
-	if (!drbg->Cbuf) {
-		ret = -ENOMEM;
-		goto fini;
-	}
-	drbg->C = PTR_ALIGN(drbg->Cbuf, ret + 1);
-	/* scratchpad is only generated for CTR and Hash */
-	if (drbg->core->flags & DRBG_HMAC)
-		sb_size = 0;
-	else if (drbg->core->flags & DRBG_CTR)
-		sb_size = drbg_statelen(drbg) + drbg_blocklen(drbg) + /* temp */
-			  crypto_drbg_ctr_df_datalen(drbg_statelen(drbg),
-						     drbg_blocklen(drbg));
-	else
-		sb_size = drbg_statelen(drbg) + drbg_blocklen(drbg);
-
-	if (0 < sb_size) {
-		drbg->scratchpadbuf = kzalloc(sb_size + ret, GFP_KERNEL);
-		if (!drbg->scratchpadbuf) {
-			ret = -ENOMEM;
-			goto fini;
-		}
-		drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1);
-	}
-
-	if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) {
-		drbg->prev = kzalloc(drbg_sec_strength(drbg->core->flags),
-				     GFP_KERNEL);
-		if (!drbg->prev) {
-			ret = -ENOMEM;
-			goto fini;
-		}
-		drbg->fips_primed = false;
-	}
-
-	return 0;
-
-fini:
-	drbg->d_ops->crypto_fini(drbg);
-err:
-	drbg_dealloc_state(drbg);
-	return ret;
-}
-
-/*************************************************************************
- * DRBG interface functions
- *************************************************************************/
-
-/*
- * DRBG generate function as required by SP800-90A - this function
- * generates random numbers
+ * Generate random bytes from an SP800-90A DRBG.
  *
  * @drbg DRBG state handle
- * @buf Buffer where to store the random numbers -- the buffer must already
- *      be pre-allocated by caller
- * @buflen Length of output buffer - this value defines the number of random
- *	   bytes pulled from DRBG
- * @addtl Additional input that is mixed into state, may be NULL -- note
- *	  the entropy is pulled by the DRBG internally unconditionally
- *	  as defined in SP800-90A. The additional input is mixed into
- *	  the state in addition to the pulled entropy.
+ * @out Buffer where to store the random bytes
+ * @outlen Number of random bytes to generate
+ * @addtl Optional additional input that is mixed into state
+ * @addtl_len Length of @addtl in bytes, may be 0
  *
  * return: 0 when all bytes are generated; < 0 in case of an error
  */
-static int drbg_generate(struct drbg_state *drbg,
-			 unsigned char *buf, unsigned int buflen,
-			 struct drbg_string *addtl)
+static int drbg_generate(struct drbg_state *drbg, u8 *out, size_t outlen,
+			 const u8 *addtl, size_t addtl_len)
 	__must_hold(&drbg->drbg_mutex)
 {
-	int len = 0;
-	LIST_HEAD(addtllist);
+	int err;
 
-	if (!drbg->core) {
-		pr_devel("DRBG: not yet seeded\n");
+	if (!drbg->instantiated) {
+		pr_devel("DRBG: not yet instantiated\n");
 		return -EINVAL;
 	}
-	if (0 == buflen || !buf) {
+	if (out == NULL || outlen == 0) {
 		pr_devel("DRBG: no output buffer provided\n");
 		return -EINVAL;
 	}
-	if (addtl && NULL == addtl->buf && 0 < addtl->len) {
+	if (addtl == NULL && addtl_len != 0) {
 		pr_devel("DRBG: wrong format of additional information\n");
 		return -EINVAL;
 	}
 
 	/* 9.3.1 step 2 */
-	len = -EINVAL;
-	if (buflen > (drbg_max_request_bytes(drbg))) {
-		pr_devel("DRBG: requested random numbers too large %u\n",
-			 buflen);
-		goto err;
+	if (outlen > DRBG_MAX_REQUEST_BYTES) {
+		pr_devel("DRBG: request length is too long %zu\n", outlen);
+		return -EINVAL;
 	}
 
 	/* 9.3.1 step 3 is implicit with the chosen DRBG */
 
 	/* 9.3.1 step 4 */
-	if (addtl && addtl->len > (drbg_max_addtl(drbg))) {
+	if (addtl_len > DRBG_MAX_ADDTL_BYTES) {
 		pr_devel("DRBG: additional information string too long %zu\n",
-			 addtl->len);
-		goto err;
+			 addtl_len);
+		return -EINVAL;
 	}
 	/* 9.3.1 step 5 is implicit with the chosen DRBG */
 
 	/*
 	 * 9.3.1 step 6 and 9 supplemented by 9.3.2 step c is implemented
 	 * here. The spec is a bit convoluted here, we make it simpler.
+	 *
+	 * We no longer try to detect when random.c has reseeded itself and call
+	 * drbg_seed() then too, since drbg_hmac_generate() adds bytes from
+	 * random.c to the additional input, which is a de facto reseed anyway.
 	 */
-	if (drbg->reseed_threshold < drbg->reseed_ctr)
-		drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
-
-	if (drbg->pr || drbg->seeded == DRBG_SEED_STATE_UNSEEDED) {
-		pr_devel("DRBG: reseeding before generation (prediction "
-			 "resistance: %s, state %s)\n",
-			 str_true_false(drbg->pr),
-			 (drbg->seeded ==  DRBG_SEED_STATE_FULL ?
-			  "seeded" : "unseeded"));
+	if (drbg->reseed_ctr > DRBG_MAX_REQUESTS) {
+		pr_devel("DRBG: reseeding before generation\n");
 		/* 9.3.1 steps 7.1 through 7.3 */
-		len = drbg_seed(drbg, addtl, true);
-		if (len)
-			goto err;
+		err = drbg_seed(drbg, addtl, addtl_len, true);
+		if (err)
+			return err;
 		/* 9.3.1 step 7.4 */
 		addtl = NULL;
-	} else if (rng_is_initialized() &&
-		   (drbg->seeded == DRBG_SEED_STATE_PARTIAL ||
-		    drbg_nopr_reseed_interval_elapsed(drbg))) {
-		len = drbg_seed_from_random(drbg);
-		if (len)
-			goto err;
+		addtl_len = 0;
 	}
 
-	if (addtl && 0 < addtl->len)
-		list_add_tail(&addtl->list, &addtllist);
 	/* 9.3.1 step 8 and 10 */
-	len = drbg->d_ops->generate(drbg, buf, buflen, &addtllist);
+	drbg_hmac_generate(drbg, out, outlen, addtl, addtl_len);
 
-	/* 10.1.1.4 step 6, 10.1.2.5 step 7, 10.2.1.5.2 step 7 */
+	/* 10.1.2.5 step 7 */
 	drbg->reseed_ctr++;
-	if (0 >= len)
-		goto err;
 
 	/*
 	 * Section 11.3.3 requires to re-perform self tests after some
@@ -1226,504 +396,126 @@ static int drbg_generate(struct drbg_state *drbg,
 	 * In this case, the entire kernel operation is questionable and it
 	 * is unlikely that the integrity violation only affects the
 	 * correct operation of the DRBG.
-	 *
-	 * Albeit the following code is commented out, it is provided in
-	 * case somebody has a need to implement the test of 11.3.3.
-	 */
-#if 0
-	if (drbg->reseed_ctr && !(drbg->reseed_ctr % 4096)) {
-		int err = 0;
-		pr_devel("DRBG: start to perform self test\n");
-		if (drbg->core->flags & DRBG_HMAC)
-			err = alg_test("drbg_pr_hmac_sha512",
-				       "drbg_pr_hmac_sha512", 0, 0);
-		else if (drbg->core->flags & DRBG_CTR)
-			err = alg_test("drbg_pr_ctr_aes256",
-				       "drbg_pr_ctr_aes256", 0, 0);
-		else
-			err = alg_test("drbg_pr_sha256",
-				       "drbg_pr_sha256", 0, 0);
-		if (err) {
-			pr_err("DRBG: periodical self test failed\n");
-			/*
-			 * uninstantiate implies that from now on, only errors
-			 * are returned when reusing this DRBG cipher handle
-			 */
-			drbg_uninstantiate(drbg);
-			return 0;
-		} else {
-			pr_devel("DRBG: self test successful\n");
-		}
-	}
-#endif
-
-	/*
-	 * All operations were successful, return 0 as mandated by
-	 * the kernel crypto API interface.
 	 */
-	len = 0;
-err:
-	return len;
-}
 
-/*
- * Wrapper around drbg_generate which can pull arbitrary long strings
- * from the DRBG without hitting the maximum request limitation.
- *
- * Parameters: see drbg_generate
- * Return codes: see drbg_generate -- if one drbg_generate request fails,
- *		 the entire drbg_generate_long request fails
- */
-static int drbg_generate_long(struct drbg_state *drbg,
-			      unsigned char *buf, unsigned int buflen,
-			      struct drbg_string *addtl)
-{
-	unsigned int len = 0;
-	unsigned int slice = 0;
-	do {
-		int err = 0;
-		unsigned int chunk = 0;
-		slice = ((buflen - len) / drbg_max_request_bytes(drbg));
-		chunk = slice ? drbg_max_request_bytes(drbg) : (buflen - len);
-		mutex_lock(&drbg->drbg_mutex);
-		err = drbg_generate(drbg, buf + len, chunk, addtl);
-		mutex_unlock(&drbg->drbg_mutex);
-		if (0 > err)
-			return err;
-		len += chunk;
-	} while (slice > 0 && (len < buflen));
 	return 0;
 }
 
-static int drbg_prepare_hrng(struct drbg_state *drbg)
-{
-	/* We do not need an HRNG in test mode. */
-	if (list_empty(&drbg->test_data.list))
-		return 0;
-
-	drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0);
-	if (IS_ERR(drbg->jent)) {
-		const int err = PTR_ERR(drbg->jent);
-
-		drbg->jent = NULL;
-		if (fips_enabled)
-			return err;
-		pr_info("DRBG: Continuing without Jitter RNG\n");
-	}
-
-	return 0;
-}
+/***************************************************************
+ * Kernel crypto API interface to DRBG
+ ***************************************************************/
 
-/*
- * DRBG instantiation function as required by SP800-90A - this function
- * sets up the DRBG handle, performs the initial seeding and all sanity
- * checks required by SP800-90A
- *
- * @drbg memory of state -- if NULL, new memory is allocated
- * @pers Personalization string that is mixed into state, may be NULL -- note
- *	 the entropy is pulled by the DRBG internally unconditionally
- *	 as defined in SP800-90A. The additional input is mixed into
- *	 the state in addition to the pulled entropy.
- * @coreref reference to core
- * @pr prediction resistance enabled
- *
- * return
- *	0 on success
- *	error value otherwise
- */
-static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers,
-			    int coreref, bool pr)
+static int drbg_kcapi_init(struct crypto_tfm *tfm)
 {
-	int ret;
-	bool reseed = true;
-
-	pr_devel("DRBG: Initializing DRBG core %d with prediction resistance "
-		 "%s\n", coreref, str_enabled_disabled(pr));
-	mutex_lock(&drbg->drbg_mutex);
-
-	/* 9.1 step 1 is implicit with the selected DRBG type */
-
-	/*
-	 * 9.1 step 2 is implicit as caller can select prediction resistance
-	 * and the flag is copied into drbg->flags --
-	 * all DRBG types support prediction resistance
-	 */
-
-	/* 9.1 step 4 is implicit in  drbg_sec_strength */
-
-	if (!drbg->core) {
-		drbg->core = &drbg_cores[coreref];
-		drbg->pr = pr;
-		drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
-		drbg->last_seed_time = 0;
-		drbg->reseed_threshold = drbg_max_requests(drbg);
-
-		ret = drbg_alloc_state(drbg);
-		if (ret)
-			goto unlock;
-
-		ret = drbg_prepare_hrng(drbg);
-		if (ret)
-			goto free_everything;
-
-		reseed = false;
-	}
-
-	ret = drbg_seed(drbg, pers, reseed);
-
-	if (ret && !reseed)
-		goto free_everything;
-
-	mutex_unlock(&drbg->drbg_mutex);
-	return ret;
-
-unlock:
-	mutex_unlock(&drbg->drbg_mutex);
-	return ret;
-
-free_everything:
-	mutex_unlock(&drbg->drbg_mutex);
-	drbg_uninstantiate(drbg);
-	return ret;
-}
+	struct drbg_state *drbg = crypto_tfm_ctx(tfm);
 
-/*
- * DRBG uninstantiate function as required by SP800-90A - this function
- * frees all buffers and the DRBG handle
- *
- * @drbg DRBG state handle
- *
- * return
- *	0 on success
- */
-static int drbg_uninstantiate(struct drbg_state *drbg)
-{
-	if (!IS_ERR_OR_NULL(drbg->jent))
-		crypto_free_rng(drbg->jent);
-	drbg->jent = NULL;
+	mutex_init(&drbg->drbg_mutex);
 
-	if (drbg->d_ops)
-		drbg->d_ops->crypto_fini(drbg);
-	drbg_dealloc_state(drbg);
-	/* no scrubbing of test_data -- this shall survive an uninstantiate */
 	return 0;
 }
 
-/*
- * Helper function for setting the test data in the DRBG
- *
- * @drbg DRBG state handle
- * @data test data
- * @len test data length
- */
+/* Set test entropy in the DRBG. */
 static void drbg_kcapi_set_entropy(struct crypto_rng *tfm,
 				   const u8 *data, unsigned int len)
 {
 	struct drbg_state *drbg = crypto_rng_ctx(tfm);
 
 	mutex_lock(&drbg->drbg_mutex);
-	drbg_string_fill(&drbg->test_data, data, len);
+	drbg->test_entropy = data;
+	drbg->test_entropylen = len;
 	mutex_unlock(&drbg->drbg_mutex);
 }
 
-/***************************************************************
- * Kernel crypto API cipher invocations requested by DRBG
- ***************************************************************/
-
-#if defined(CONFIG_CRYPTO_DRBG_HASH) || defined(CONFIG_CRYPTO_DRBG_HMAC)
-struct sdesc {
-	struct shash_desc shash;
-};
-
-static int drbg_init_hash_kernel(struct drbg_state *drbg)
-{
-	struct sdesc *sdesc;
-	struct crypto_shash *tfm;
-
-	tfm = crypto_alloc_shash(drbg->core->backend_cra_name, 0, 0);
-	if (IS_ERR(tfm)) {
-		pr_info("DRBG: could not allocate digest TFM handle: %s\n",
-				drbg->core->backend_cra_name);
-		return PTR_ERR(tfm);
-	}
-	BUG_ON(drbg_blocklen(drbg) != crypto_shash_digestsize(tfm));
-	sdesc = kzalloc(sizeof(struct shash_desc) + crypto_shash_descsize(tfm),
-			GFP_KERNEL);
-	if (!sdesc) {
-		crypto_free_shash(tfm);
-		return -ENOMEM;
-	}
-
-	sdesc->shash.tfm = tfm;
-	drbg->priv_data = sdesc;
-
-	return 0;
-}
-
-static int drbg_fini_hash_kernel(struct drbg_state *drbg)
-{
-	struct sdesc *sdesc = drbg->priv_data;
-	if (sdesc) {
-		crypto_free_shash(sdesc->shash.tfm);
-		kfree_sensitive(sdesc);
-	}
-	drbg->priv_data = NULL;
-	return 0;
-}
-
-static void drbg_kcapi_hmacsetkey(struct drbg_state *drbg,
-				  const unsigned char *key)
-{
-	struct sdesc *sdesc = drbg->priv_data;
-
-	crypto_shash_setkey(sdesc->shash.tfm, key, drbg_statelen(drbg));
-}
-
-static int drbg_kcapi_hash(struct drbg_state *drbg, unsigned char *outval,
-			   const struct list_head *in)
-{
-	struct sdesc *sdesc = drbg->priv_data;
-	struct drbg_string *input = NULL;
-
-	crypto_shash_init(&sdesc->shash);
-	list_for_each_entry(input, in, list)
-		crypto_shash_update(&sdesc->shash, input->buf, input->len);
-	return crypto_shash_final(&sdesc->shash, outval);
-}
-#endif /* (CONFIG_CRYPTO_DRBG_HASH || CONFIG_CRYPTO_DRBG_HMAC) */
-
-#ifdef CONFIG_CRYPTO_DRBG_CTR
-static int drbg_fini_sym_kernel(struct drbg_state *drbg)
-{
-	struct aes_enckey *aeskey = drbg->priv_data;
-
-	kfree(aeskey);
-	drbg->priv_data = NULL;
-
-	if (drbg->ctr_handle)
-		crypto_free_skcipher(drbg->ctr_handle);
-	drbg->ctr_handle = NULL;
-
-	if (drbg->ctr_req)
-		skcipher_request_free(drbg->ctr_req);
-	drbg->ctr_req = NULL;
-
-	kfree(drbg->outscratchpadbuf);
-	drbg->outscratchpadbuf = NULL;
-
-	return 0;
-}
-
-static int drbg_init_sym_kernel(struct drbg_state *drbg)
-{
-	struct aes_enckey *aeskey;
-	struct crypto_skcipher *sk_tfm;
-	struct skcipher_request *req;
-	unsigned int alignmask;
-	char ctr_name[CRYPTO_MAX_ALG_NAME];
-
-	aeskey = kzalloc_obj(*aeskey);
-	if (!aeskey)
-		return -ENOMEM;
-	drbg->priv_data = aeskey;
-
-	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)",
-	    drbg->core->backend_cra_name) >= CRYPTO_MAX_ALG_NAME) {
-		drbg_fini_sym_kernel(drbg);
-		return -EINVAL;
-	}
-	sk_tfm = crypto_alloc_skcipher(ctr_name, 0, 0);
-	if (IS_ERR(sk_tfm)) {
-		pr_info("DRBG: could not allocate CTR cipher TFM handle: %s\n",
-				ctr_name);
-		drbg_fini_sym_kernel(drbg);
-		return PTR_ERR(sk_tfm);
-	}
-	drbg->ctr_handle = sk_tfm;
-	crypto_init_wait(&drbg->ctr_wait);
-
-	req = skcipher_request_alloc(sk_tfm, GFP_KERNEL);
-	if (!req) {
-		pr_info("DRBG: could not allocate request queue\n");
-		drbg_fini_sym_kernel(drbg);
-		return -ENOMEM;
-	}
-	drbg->ctr_req = req;
-	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
-						CRYPTO_TFM_REQ_MAY_SLEEP,
-					crypto_req_done, &drbg->ctr_wait);
-
-	alignmask = crypto_skcipher_alignmask(sk_tfm);
-	drbg->outscratchpadbuf = kmalloc(DRBG_OUTSCRATCHLEN + alignmask,
-					 GFP_KERNEL);
-	if (!drbg->outscratchpadbuf) {
-		drbg_fini_sym_kernel(drbg);
-		return -ENOMEM;
-	}
-	drbg->outscratchpad = (u8 *)PTR_ALIGN(drbg->outscratchpadbuf,
-					      alignmask + 1);
-
-	sg_init_table(&drbg->sg_in, 1);
-	sg_init_one(&drbg->sg_out, drbg->outscratchpad, DRBG_OUTSCRATCHLEN);
-
-	return alignmask;
-}
-
-static int drbg_kcapi_sym_ctr(struct drbg_state *drbg,
-			      u8 *inbuf, u32 inlen,
-			      u8 *outbuf, u32 outlen)
+/* Seed (i.e. instantiate) or re-seed the DRBG. */
+static int drbg_kcapi_seed(struct crypto_rng *tfm,
+			   const u8 *pers, unsigned int pers_len)
 {
-	struct scatterlist *sg_in = &drbg->sg_in, *sg_out = &drbg->sg_out;
-	u32 scratchpad_use = min_t(u32, outlen, DRBG_OUTSCRATCHLEN);
+	static const u8 initial_key[DRBG_STATE_LEN]; /* all zeroes */
+	struct drbg_state *drbg = crypto_rng_ctx(tfm);
 	int ret;
 
-	if (inbuf) {
-		/* Use caller-provided input buffer */
-		sg_set_buf(sg_in, inbuf, inlen);
-	} else {
-		/* Use scratchpad for in-place operation */
-		inlen = scratchpad_use;
-		memset(drbg->outscratchpad, 0, scratchpad_use);
-		sg_set_buf(sg_in, drbg->outscratchpad, scratchpad_use);
-	}
-
-	while (outlen) {
-		u32 cryptlen = min3(inlen, outlen, (u32)DRBG_OUTSCRATCHLEN);
-
-		/* Output buffer may not be valid for SGL, use scratchpad */
-		skcipher_request_set_crypt(drbg->ctr_req, sg_in, sg_out,
-					   cryptlen, drbg->V);
-		ret = crypto_wait_req(crypto_skcipher_encrypt(drbg->ctr_req),
-					&drbg->ctr_wait);
-		if (ret)
-			goto out;
+	pr_devel("DRBG: Initializing DRBG\n");
+	guard(mutex)(&drbg->drbg_mutex);
 
-		crypto_init_wait(&drbg->ctr_wait);
+	if (drbg->instantiated)
+		return drbg_seed(drbg, pers, pers_len, /* reseed= */ true);
 
-		memcpy(outbuf, drbg->outscratchpad, cryptlen);
-		memzero_explicit(drbg->outscratchpad, cryptlen);
-
-		outlen -= cryptlen;
-		outbuf += cryptlen;
-	}
-	ret = 0;
+	/* 9.1 step 1 is implicit with the selected DRBG type */
 
-out:
-	return ret;
-}
-#endif /* CONFIG_CRYPTO_DRBG_CTR */
+	/*
+	 * 9.1 step 2 is implicit, as this implementation doesn't support
+	 * prediction resistance
+	 */
 
-/***************************************************************
- * Kernel crypto API interface to register DRBG
- ***************************************************************/
+	/* 9.1 step 4 is implicit in DRBG_SEC_STRENGTH */
 
-/*
- * Look up the DRBG flags by given kernel crypto API cra_name
- * The code uses the drbg_cores definition to do this
- *
- * @cra_name kernel crypto API cra_name
- * @coreref reference to integer which is filled with the pointer to
- *  the applicable core
- * @pr reference for setting prediction resistance
- *
- * return: flags
- */
-static inline void drbg_convert_tfm_core(const char *cra_driver_name,
-					 int *coreref, bool *pr)
-{
-	int i = 0;
-	size_t start = 0;
-	int len = 0;
-
-	*pr = true;
-	/* disassemble the names */
-	if (!memcmp(cra_driver_name, "drbg_nopr_", 10)) {
-		start = 10;
-		*pr = false;
-	} else if (!memcmp(cra_driver_name, "drbg_pr_", 8)) {
-		start = 8;
-	} else {
-		return;
-	}
+	memset(drbg->V, 1, DRBG_STATE_LEN);
+	hmac_sha512_preparekey(&drbg->key, initial_key, DRBG_STATE_LEN);
 
-	/* remove the first part */
-	len = strlen(cra_driver_name) - start;
-	for (i = 0; ARRAY_SIZE(drbg_cores) > i; i++) {
-		if (!memcmp(cra_driver_name + start, drbg_cores[i].cra_name,
-			    len)) {
-			*coreref = i;
-			return;
+	/* Allocate jitterentropy_rng if not in test mode. */
+	if (drbg->test_entropylen == 0) {
+		drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0);
+		if (IS_ERR(drbg->jent)) {
+			ret = PTR_ERR(drbg->jent);
+			drbg->jent = NULL;
+			if (fips_enabled)
+				return ret;
+			pr_info("DRBG: Continuing without Jitter RNG\n");
 		}
 	}
-}
-
-static int drbg_kcapi_init(struct crypto_tfm *tfm)
-{
-	struct drbg_state *drbg = crypto_tfm_ctx(tfm);
-
-	mutex_init(&drbg->drbg_mutex);
 
+	ret = drbg_seed(drbg, pers, pers_len, /* reseed= */ false);
+	if (ret) {
+		crypto_free_rng(drbg->jent);
+		drbg->jent = NULL;
+		return ret;
+	}
+	drbg->instantiated = true;
 	return 0;
 }
 
-static void drbg_kcapi_cleanup(struct crypto_tfm *tfm)
-{
-	drbg_uninstantiate(crypto_tfm_ctx(tfm));
-}
-
 /*
  * Generate random numbers invoked by the kernel crypto API:
- * The API of the kernel crypto API is extended as follows:
  *
  * src is additional input supplied to the RNG.
  * slen is the length of src.
  * dst is the output buffer where random data is to be stored.
  * dlen is the length of dst.
  */
-static int drbg_kcapi_random(struct crypto_rng *tfm,
-			     const u8 *src, unsigned int slen,
-			     u8 *dst, unsigned int dlen)
+static int drbg_kcapi_generate(struct crypto_rng *tfm,
+			       const u8 *src, unsigned int slen,
+			       u8 *dst, unsigned int dlen)
 {
 	struct drbg_state *drbg = crypto_rng_ctx(tfm);
-	struct drbg_string *addtl = NULL;
-	struct drbg_string string;
 
-	if (slen) {
-		/* linked list variable is now local to allow modification */
-		drbg_string_fill(&string, src, slen);
-		addtl = &string;
-	}
+	/*
+	 * Break the request into multiple requests if needed, to avoid
+	 * exceeding the maximum request length of the core algorithm.
+	 */
+	do {
+		unsigned int n = min(dlen, DRBG_MAX_REQUEST_BYTES);
+		int err;
 
-	return drbg_generate_long(drbg, dst, dlen, addtl);
+		mutex_lock(&drbg->drbg_mutex);
+		err = drbg_generate(drbg, dst, n, src, slen);
+		mutex_unlock(&drbg->drbg_mutex);
+		if (err < 0)
+			return err;
+		dst += n;
+		dlen -= n;
+	} while (dlen);
+	return 0;
 }
 
-/*
- * Seed the DRBG invoked by the kernel crypto API
- */
-static int drbg_kcapi_seed(struct crypto_rng *tfm,
-			   const u8 *seed, unsigned int slen)
+/* Uninstantiate the DRBG. */
+static void drbg_kcapi_exit(struct crypto_tfm *tfm)
 {
-	struct drbg_state *drbg = crypto_rng_ctx(tfm);
-	struct crypto_tfm *tfm_base = crypto_rng_tfm(tfm);
-	bool pr = false;
-	struct drbg_string string;
-	struct drbg_string *seed_string = NULL;
-	int coreref = 0;
-
-	drbg_convert_tfm_core(crypto_tfm_alg_driver_name(tfm_base), &coreref,
-			      &pr);
-	if (0 < slen) {
-		drbg_string_fill(&string, seed, slen);
-		seed_string = &string;
-	}
+	struct drbg_state *drbg = crypto_tfm_ctx(tfm);
 
-	return drbg_instantiate(drbg, seed_string, coreref, pr);
+	crypto_free_rng(drbg->jent);
+	memzero_explicit(drbg, sizeof(*drbg));
 }
 
-/***************************************************************
- * Kernel module: code to load the module
- ***************************************************************/
-
 /*
  * Tests as defined in 11.3.2 in addition to the cipher tests: testing
  * of the error handling.
@@ -1736,38 +528,21 @@ static int drbg_kcapi_seed(struct crypto_rng *tfm,
  */
 static inline int __init drbg_healthcheck_sanity(void)
 {
-	int len = 0;
 #define OUTBUFLEN 16
-	unsigned char buf[OUTBUFLEN];
+	u8 buf[OUTBUFLEN];
 	struct drbg_state *drbg = NULL;
 	int ret;
-	int rc = -EFAULT;
-	bool pr = false;
-	int coreref = 0;
-	struct drbg_string addtl;
-	size_t max_addtllen, max_request_bytes;
 
 	/* only perform test in FIPS mode */
 	if (!fips_enabled)
 		return 0;
 
-#ifdef CONFIG_CRYPTO_DRBG_CTR
-	drbg_convert_tfm_core("drbg_nopr_ctr_aes256", &coreref, &pr);
-#endif
-#ifdef CONFIG_CRYPTO_DRBG_HASH
-	drbg_convert_tfm_core("drbg_nopr_sha256", &coreref, &pr);
-#endif
-#ifdef CONFIG_CRYPTO_DRBG_HMAC
-	drbg_convert_tfm_core("drbg_nopr_hmac_sha512", &coreref, &pr);
-#endif
-
 	drbg = kzalloc_obj(struct drbg_state);
 	if (!drbg)
 		return -ENOMEM;
 
 	guard(mutex_init)(&drbg->drbg_mutex);
-	drbg->core = &drbg_cores[coreref];
-	drbg->reseed_threshold = drbg_max_requests(drbg);
+	drbg->instantiated = true;
 
 	/*
 	 * if the following tests fail, it is likely that there is a buffer
@@ -1777,128 +552,66 @@ static inline int __init drbg_healthcheck_sanity(void)
 	 * grave bug.
 	 */
 
-	max_addtllen = drbg_max_addtl(drbg);
-	max_request_bytes = drbg_max_request_bytes(drbg);
-	drbg_string_fill(&addtl, buf, max_addtllen + 1);
 	/* overflow addtllen with additional info string */
-	len = drbg_generate(drbg, buf, OUTBUFLEN, &addtl);
-	BUG_ON(0 < len);
+	ret = drbg_generate(drbg, buf, OUTBUFLEN, buf,
+			    DRBG_MAX_ADDTL_BYTES + 1);
+	BUG_ON(ret == 0);
 	/* overflow max_bits */
-	len = drbg_generate(drbg, buf, (max_request_bytes + 1), NULL);
-	BUG_ON(0 < len);
+	ret = drbg_generate(drbg, buf, DRBG_MAX_REQUEST_BYTES + 1, NULL, 0);
+	BUG_ON(ret == 0);
 
 	/* overflow max addtllen with personalization string */
-	ret = drbg_seed(drbg, &addtl, false);
-	BUG_ON(0 == ret);
+	ret = drbg_seed(drbg, buf, DRBG_MAX_ADDTL_BYTES + 1, false);
+	BUG_ON(ret == 0);
 	/* all tests passed */
-	rc = 0;
 
 	pr_devel("DRBG: Sanity tests for failure code paths successfully "
 		 "completed\n");
 
 	kfree(drbg);
-	return rc;
+	return 0;
 }
 
-static struct rng_alg drbg_algs[22];
-
-/*
- * Fill the array drbg_algs used to register the different DRBGs
- * with the kernel crypto API. To fill the array, the information
- * from drbg_cores[] is used.
- */
-static inline void __init drbg_fill_array(struct rng_alg *alg,
-					  const struct drbg_core *core, int pr)
-{
-	int pos = 0;
-	static int priority = 200;
-
-	memcpy(alg->base.cra_name, "stdrng", 6);
-	if (pr) {
-		memcpy(alg->base.cra_driver_name, "drbg_pr_", 8);
-		pos = 8;
-	} else {
-		memcpy(alg->base.cra_driver_name, "drbg_nopr_", 10);
-		pos = 10;
-	}
-	memcpy(alg->base.cra_driver_name + pos, core->cra_name,
-	       strlen(core->cra_name));
-
-	alg->base.cra_priority = priority;
-	priority++;
-	/*
-	 * If FIPS mode enabled, the selected DRBG shall have the
-	 * highest cra_priority over other stdrng instances to ensure
-	 * it is selected.
-	 */
-	if (fips_enabled)
-		alg->base.cra_priority += 200;
-
-	alg->base.cra_ctxsize 	= sizeof(struct drbg_state);
-	alg->base.cra_module	= THIS_MODULE;
-	alg->base.cra_init	= drbg_kcapi_init;
-	alg->base.cra_exit	= drbg_kcapi_cleanup;
-	alg->generate		= drbg_kcapi_random;
-	alg->seed		= drbg_kcapi_seed;
-	alg->set_ent		= drbg_kcapi_set_entropy;
-	alg->seedsize		= 0;
-}
+static struct rng_alg drbg_alg = {
+	.base.cra_name		= "stdrng",
+	.base.cra_driver_name	= "drbg_nopr_hmac_sha512",
+	.base.cra_priority	= 201,
+	.base.cra_ctxsize	= sizeof(struct drbg_state),
+	.base.cra_module	= THIS_MODULE,
+	.base.cra_init		= drbg_kcapi_init,
+	.set_ent		= drbg_kcapi_set_entropy,
+	.seed			= drbg_kcapi_seed,
+	.generate		= drbg_kcapi_generate,
+	.base.cra_exit		= drbg_kcapi_exit,
+};
 
 static int __init drbg_init(void)
 {
-	unsigned int i = 0; /* pointer to drbg_algs */
-	unsigned int j = 0; /* pointer to drbg_cores */
 	int ret;
 
 	ret = drbg_healthcheck_sanity();
 	if (ret)
 		return ret;
 
-	if (ARRAY_SIZE(drbg_cores) * 2 > ARRAY_SIZE(drbg_algs)) {
-		pr_info("DRBG: Cannot register all DRBG types"
-			"(slots needed: %zu, slots available: %zu)\n",
-			ARRAY_SIZE(drbg_cores) * 2, ARRAY_SIZE(drbg_algs));
-		return -EFAULT;
-	}
-
 	/*
-	 * each DRBG definition can be used with PR and without PR, thus
-	 * we instantiate each DRBG in drbg_cores[] twice.
-	 *
-	 * As the order of placing them into the drbg_algs array matters
-	 * (the later DRBGs receive a higher cra_priority) we register the
-	 * prediction resistance DRBGs first as the should not be too
-	 * interesting.
+	 * In FIPS mode, boost the algorithm priority to ensure that when users
+	 * request "stdrng", they really get the algorithm from here.
 	 */
-	for (j = 0; ARRAY_SIZE(drbg_cores) > j; j++, i++)
-		drbg_fill_array(&drbg_algs[i], &drbg_cores[j], 1);
-	for (j = 0; ARRAY_SIZE(drbg_cores) > j; j++, i++)
-		drbg_fill_array(&drbg_algs[i], &drbg_cores[j], 0);
-	return crypto_register_rngs(drbg_algs, (ARRAY_SIZE(drbg_cores) * 2));
+	if (fips_enabled)
+		drbg_alg.base.cra_priority += 2000;
+
+	return crypto_register_rng(&drbg_alg);
 }
 
 static void __exit drbg_exit(void)
 {
-	crypto_unregister_rngs(drbg_algs, (ARRAY_SIZE(drbg_cores) * 2));
+	crypto_unregister_rng(&drbg_alg);
 }
 
 module_init(drbg_init);
 module_exit(drbg_exit);
-#ifndef CRYPTO_DRBG_HASH_STRING
-#define CRYPTO_DRBG_HASH_STRING ""
-#endif
-#ifndef CRYPTO_DRBG_HMAC_STRING
-#define CRYPTO_DRBG_HMAC_STRING ""
-#endif
-#ifndef CRYPTO_DRBG_CTR_STRING
-#define CRYPTO_DRBG_CTR_STRING ""
-#endif
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
-MODULE_DESCRIPTION("NIST SP800-90A Deterministic Random Bit Generator (DRBG) "
-		   "using following cores: "
-		   CRYPTO_DRBG_HASH_STRING
-		   CRYPTO_DRBG_HMAC_STRING
-		   CRYPTO_DRBG_CTR_STRING);
+MODULE_DESCRIPTION("NIST SP800-90A Deterministic Random Bit Generator (DRBG)");
 MODULE_ALIAS_CRYPTO("stdrng");
-MODULE_IMPORT_NS("CRYPTO_INTERNAL");
+MODULE_ALIAS_CRYPTO("drbg_nopr_hmac_sha512");
diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c
index 2c0602f0cd406..0cd7eb3676041 100644
--- a/crypto/ecrdsa.c
+++ b/crypto/ecrdsa.c
@@ -145,7 +145,7 @@ int ecrdsa_param_curve(void *context, size_t hdrlen, unsigned char tag,
 	struct ecrdsa_ctx *ctx = context;
 
 	ctx->curve_oid = look_up_OID(value, vlen);
-	if (!ctx->curve_oid)
+	if (ctx->curve_oid == OID__NR)
 		return -EINVAL;
 	ctx->curve = get_curve_by_oid(ctx->curve_oid);
 	return 0;
diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c
index d5832caa8ab3c..b024bff7024a8 100644
--- a/crypto/jitterentropy.c
+++ b/crypto/jitterentropy.c
@@ -7,7 +7,7 @@
  * Design
  * ======
  *
- * See https://www.chronox.de/jent.html
+ * See https://www.chronox.de/jent/
  *
  * License
  * =======
@@ -47,7 +47,7 @@
 
 /*
  * This Jitterentropy RNG is based on the jitterentropy library
- * version 3.4.0 provided at https://www.chronox.de/jent.html
+ * version 3.4.0 provided at https://www.chronox.de/jent/
  */
 
 #ifdef __OPTIMIZE__
@@ -775,7 +775,7 @@ int jent_entropy_init(unsigned int osr, unsigned int flags,
 		 * delta even when called shortly after each other -- this
 		 * implies that we also have a high resolution timer
 		 */
-		if (!delta || (end_time == start_time)) {
+		if (!delta) {
 			ret = JENT_ECOARSETIME;
 			goto out;
 		}
diff --git a/crypto/kpp.c b/crypto/kpp.c
index 7451d39a7ad8b..522c352a03afc 100644
--- a/crypto/kpp.c
+++ b/crypto/kpp.c
@@ -20,11 +20,9 @@
 static int __maybe_unused crypto_kpp_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
-	struct crypto_report_kpp rkpp;
-
-	memset(&rkpp, 0, sizeof(rkpp));
-
-	strscpy(rkpp.type, "kpp", sizeof(rkpp.type));
+	struct crypto_report_kpp rkpp = {
+		.type = "kpp",
+	};
 
 	return nla_put(skb, CRYPTOCFGA_REPORT_KPP, sizeof(rkpp), &rkpp);
 }
diff --git a/crypto/krb5/krb5_api.c b/crypto/krb5/krb5_api.c
index 23026d4206c82..03395b89cc61e 100644
--- a/crypto/krb5/krb5_api.c
+++ b/crypto/krb5/krb5_api.c
@@ -134,27 +134,69 @@ EXPORT_SYMBOL(crypto_krb5_how_much_data);
  * Find the offset and size of the data in a secure message so that this
  * information can be used in the metadata buffer which will get added to the
  * digest by crypto_krb5_verify_mic().
+ *
+ * Return: 0 if successful, -EBADMSG if the message is too short or -EINVAL if
+ * the mode is unsupported.
  */
-void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
-				   enum krb5_crypto_mode mode,
-				   size_t *_offset, size_t *_len)
+int crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
+				  enum krb5_crypto_mode mode,
+				  size_t *_offset, size_t *_len)
 {
 	switch (mode) {
 	case KRB5_CHECKSUM_MODE:
+		if (*_len < krb5->cksum_len)
+			return -EBADMSG;
 		*_offset += krb5->cksum_len;
 		*_len -= krb5->cksum_len;
-		return;
+		return 0;
 	case KRB5_ENCRYPT_MODE:
+		if (*_len < krb5->conf_len + krb5->cksum_len)
+			return -EBADMSG;
 		*_offset += krb5->conf_len;
 		*_len -= krb5->conf_len + krb5->cksum_len;
-		return;
+		return 0;
 	default:
 		WARN_ON_ONCE(1);
-		return;
+		return -EINVAL;
 	}
 }
 EXPORT_SYMBOL(crypto_krb5_where_is_the_data);
 
+/**
+ * crypto_krb5_check_data_len - Check a message is big enough
+ * @krb5: The encoding to use.
+ * @mode: Mode of operation.
+ * @len: The length of the secure blob.
+ * @min_content: Minimum length of the content inside the blob.
+ *
+ * Check that a message is large enough to hold whatever bits the encryption
+ * type wants to glue on (nonce, checksum) plus a minimum amount of content.
+ *
+ * Return: 0 if successful, -EBADMSG if the message is too short or -EINVAL if
+ * the mode is unsupported.
+ */
+int crypto_krb5_check_data_len(const struct krb5_enctype *krb5,
+			       enum krb5_crypto_mode mode,
+			       size_t len, size_t min_content)
+{
+	switch (mode) {
+	case KRB5_CHECKSUM_MODE:
+		if (len < krb5->cksum_len ||
+		    len - krb5->cksum_len < min_content)
+			return -EBADMSG;
+		return 0;
+	case KRB5_ENCRYPT_MODE:
+		if (len < krb5->conf_len + krb5->cksum_len ||
+		    len - (krb5->conf_len + krb5->cksum_len) < min_content)
+			return -EBADMSG;
+		return 0;
+	default:
+		WARN_ON_ONCE(1);
+		return -EINVAL;
+	}
+}
+EXPORT_SYMBOL(crypto_krb5_check_data_len);
+
 /*
  * Prepare the encryption with derived key data.
  */
@@ -165,7 +207,7 @@ struct crypto_aead *krb5_prepare_encryption(const struct krb5_enctype *krb5,
 	struct crypto_aead *ci = NULL;
 	int ret = -ENOMEM;
 
-	ci = crypto_alloc_aead(krb5->encrypt_name, 0, 0);
+	ci = crypto_alloc_aead(krb5->encrypt_name, 0, CRYPTO_ALG_ASYNC);
 	if (IS_ERR(ci)) {
 		ret = PTR_ERR(ci);
 		if (ret == -ENOENT)
diff --git a/crypto/lskcipher.c b/crypto/lskcipher.c
index bb166250b7327..e4328df6e26c0 100644
--- a/crypto/lskcipher.c
+++ b/crypto/lskcipher.c
@@ -264,12 +264,10 @@ static int __maybe_unused crypto_lskcipher_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
 	struct lskcipher_alg *skcipher = __crypto_lskcipher_alg(alg);
-	struct crypto_report_blkcipher rblkcipher;
-
-	memset(&rblkcipher, 0, sizeof(rblkcipher));
-
-	strscpy(rblkcipher.type, "lskcipher", sizeof(rblkcipher.type));
-	strscpy(rblkcipher.geniv, "<none>", sizeof(rblkcipher.geniv));
+	struct crypto_report_blkcipher rblkcipher = {
+		.type = "lskcipher",
+		.geniv = "<none>",
+	};
 
 	rblkcipher.blocksize = alg->cra_blocksize;
 	rblkcipher.min_keysize = skcipher->co.min_keysize;
diff --git a/crypto/rng.c b/crypto/rng.c
index 1d4b9177bad4d..eec786c45bdd8 100644
--- a/crypto/rng.c
+++ b/crypto/rng.c
@@ -65,11 +65,9 @@ static unsigned int seedsize(struct crypto_alg *alg)
 static int __maybe_unused crypto_rng_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
-	struct crypto_report_rng rrng;
-
-	memset(&rrng, 0, sizeof(rrng));
-
-	strscpy(rrng.type, "rng", sizeof(rrng.type));
+	struct crypto_report_rng rrng = {
+		.type = "rng",
+	};
 
 	rrng.seedsize = seedsize(alg);
 
diff --git a/crypto/scompress.c b/crypto/scompress.c
index 253655ece83fd..de54227203ce9 100644
--- a/crypto/scompress.c
+++ b/crypto/scompress.c
@@ -48,11 +48,9 @@ static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn);
 static int __maybe_unused crypto_scomp_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
-	struct crypto_report_comp rscomp;
-
-	memset(&rscomp, 0, sizeof(rscomp));
-
-	strscpy(rscomp.type, "scomp", sizeof(rscomp.type));
+	struct crypto_report_comp rscomp = {
+		.type = "scomp",
+	};
 
 	return nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS,
 		       sizeof(rscomp), &rscomp);
diff --git a/crypto/shash.c b/crypto/shash.c
index 2f07d0bd1f61b..d31a65570f696 100644
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -333,12 +333,10 @@ static void crypto_shash_free_instance(struct crypto_instance *inst)
 static int __maybe_unused crypto_shash_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
-	struct crypto_report_hash rhash;
 	struct shash_alg *salg = __crypto_shash_alg(alg);
-
-	memset(&rhash, 0, sizeof(rhash));
-
-	strscpy(rhash.type, "shash", sizeof(rhash.type));
+	struct crypto_report_hash rhash = {
+		.type = "shash",
+	};
 
 	rhash.blocksize = alg->cra_blocksize;
 	rhash.digestsize = salg->digestsize;
diff --git a/crypto/sig.c b/crypto/sig.c
index beba745b64057..7d2048da5c3af 100644
--- a/crypto/sig.c
+++ b/crypto/sig.c
@@ -53,9 +53,9 @@ static void __maybe_unused crypto_sig_show(struct seq_file *m,
 static int __maybe_unused crypto_sig_report(struct sk_buff *skb,
 					    struct crypto_alg *alg)
 {
-	struct crypto_report_sig rsig = {};
-
-	strscpy(rsig.type, "sig", sizeof(rsig.type));
+	struct crypto_report_sig rsig = {
+		.type = "sig",
+	};
 
 	return nla_put(skb, CRYPTOCFGA_REPORT_SIG, sizeof(rsig), &rsig);
 }
diff --git a/crypto/skcipher.c b/crypto/skcipher.c
index 2b31d1d5d268b..617e840432b17 100644
--- a/crypto/skcipher.c
+++ b/crypto/skcipher.c
@@ -591,12 +591,10 @@ static int __maybe_unused crypto_skcipher_report(
 	struct sk_buff *skb, struct crypto_alg *alg)
 {
 	struct skcipher_alg *skcipher = __crypto_skcipher_alg(alg);
-	struct crypto_report_blkcipher rblkcipher;
-
-	memset(&rblkcipher, 0, sizeof(rblkcipher));
-
-	strscpy(rblkcipher.type, "skcipher", sizeof(rblkcipher.type));
-	strscpy(rblkcipher.geniv, "<none>", sizeof(rblkcipher.geniv));
+	struct crypto_report_blkcipher rblkcipher = {
+		.type = "skcipher",
+		.geniv = "<none>",
+	};
 
 	rblkcipher.blocksize = alg->cra_blocksize;
 	rblkcipher.min_keysize = skcipher->min_keysize;
diff --git a/crypto/testmgr.c b/crypto/testmgr.c
index 4d86efae65b21..a3e80de91d4d6 100644
--- a/crypto/testmgr.c
+++ b/crypto/testmgr.c
@@ -28,13 +28,12 @@
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/uio.h>
-#include <crypto/rng.h>
-#include <crypto/drbg.h>
 #include <crypto/akcipher.h>
 #include <crypto/kpp.h>
 #include <crypto/acompress.h>
 #include <crypto/sig.h>
 #include <crypto/internal/cipher.h>
+#include <crypto/internal/rng.h>
 #include <crypto/internal/simd.h>
 
 #include "internal.h"
@@ -3482,13 +3481,11 @@ static int alg_test_comp(const struct alg_test_desc *desc, const char *driver,
 	return err;
 }
 
-static int drbg_cavs_test(const struct drbg_testvec *test, int pr,
-			  const char *driver, u32 type, u32 mask)
+static int drbg_cavs_test(const struct drbg_testvec *test, const char *driver,
+			  u32 type, u32 mask)
 {
 	int ret = -EAGAIN;
 	struct crypto_rng *drng;
-	struct drbg_test_data test_data;
-	struct drbg_string addtl, pers, testentropy;
 	unsigned char *buf = kzalloc(test->expectedlen, GFP_KERNEL);
 
 	if (!buf)
@@ -3504,39 +3501,34 @@ static int drbg_cavs_test(const struct drbg_testvec *test, int pr,
 		return PTR_ERR(drng);
 	}
 
-	test_data.testentropy = &testentropy;
-	drbg_string_fill(&testentropy, test->entropy, test->entropylen);
-	drbg_string_fill(&pers, test->pers, test->perslen);
-	ret = crypto_drbg_reset_test(drng, &pers, &test_data);
+	crypto_rng_set_entropy(drng, test->entropy, test->entropylen);
+	ret = crypto_rng_reset(drng, test->pers, test->perslen);
 	if (ret) {
-		printk(KERN_ERR "alg: drbg: Failed to reset rng\n");
+		printk(KERN_ERR "alg: drbg: Failed to instantiate rng\n");
 		goto outbuf;
 	}
 
-	drbg_string_fill(&addtl, test->addtla, test->addtllen);
-	if (pr) {
-		drbg_string_fill(&testentropy, test->entpra, test->entprlen);
-		ret = crypto_drbg_get_bytes_addtl_test(drng,
-			buf, test->expectedlen, &addtl,	&test_data);
-	} else {
-		ret = crypto_drbg_get_bytes_addtl(drng,
-			buf, test->expectedlen, &addtl);
+	if (test->ent_reseed_len) {
+		crypto_rng_set_entropy(drng, test->ent_reseed,
+				       test->ent_reseed_len);
+		ret = crypto_rng_reset(drng, test->addtl_reseed,
+				       test->addtl_reseed_len);
+		if (ret) {
+			printk(KERN_ERR "alg: drbg: Failed to reseed rng\n");
+			goto outbuf;
+		}
 	}
+
+	ret = crypto_rng_generate(drng, test->addtla, test->addtllen,
+				  buf, test->expectedlen);
 	if (ret < 0) {
 		printk(KERN_ERR "alg: drbg: could not obtain random data for "
 		       "driver %s\n", driver);
 		goto outbuf;
 	}
 
-	drbg_string_fill(&addtl, test->addtlb, test->addtllen);
-	if (pr) {
-		drbg_string_fill(&testentropy, test->entprb, test->entprlen);
-		ret = crypto_drbg_get_bytes_addtl_test(drng,
-			buf, test->expectedlen, &addtl, &test_data);
-	} else {
-		ret = crypto_drbg_get_bytes_addtl(drng,
-			buf, test->expectedlen, &addtl);
-	}
+	ret = crypto_rng_generate(drng, test->addtlb, test->addtllen,
+				  buf, test->expectedlen);
 	if (ret < 0) {
 		printk(KERN_ERR "alg: drbg: could not obtain random data for "
 		       "driver %s\n", driver);
@@ -3556,16 +3548,12 @@ static int alg_test_drbg(const struct alg_test_desc *desc, const char *driver,
 			 u32 type, u32 mask)
 {
 	int err = 0;
-	int pr = 0;
 	int i = 0;
 	const struct drbg_testvec *template = desc->suite.drbg.vecs;
 	unsigned int tcount = desc->suite.drbg.count;
 
-	if (0 == memcmp(driver, "drbg_pr_", 8))
-		pr = 1;
-
 	for (i = 0; i < tcount; i++) {
-		err = drbg_cavs_test(&template[i], pr, driver, type, mask);
+		err = drbg_cavs_test(&template[i], driver, type, mask);
 		if (err) {
 			printk(KERN_ERR "alg: drbg: Test %d failed for %s\n",
 			       i, driver);
@@ -4653,42 +4641,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 	}, {
 		.alg = "digest_null",
 		.test = alg_test_null,
-	}, {
-		.alg = "drbg_nopr_ctr_aes128",
-		.test = alg_test_drbg,
-		.fips_allowed = 1,
-		.suite = {
-			.drbg = __VECS(drbg_nopr_ctr_aes128_tv_template)
-		}
-	}, {
-		.alg = "drbg_nopr_ctr_aes192",
-		.test = alg_test_drbg,
-		.fips_allowed = 1,
-		.suite = {
-			.drbg = __VECS(drbg_nopr_ctr_aes192_tv_template)
-		}
-	}, {
-		.alg = "drbg_nopr_ctr_aes256",
-		.test = alg_test_drbg,
-		.fips_allowed = 1,
-		.suite = {
-			.drbg = __VECS(drbg_nopr_ctr_aes256_tv_template)
-		}
-	}, {
-		.alg = "drbg_nopr_hmac_sha256",
-		.test = alg_test_drbg,
-		.fips_allowed = 1,
-		.suite = {
-			.drbg = __VECS(drbg_nopr_hmac_sha256_tv_template)
-		}
-	}, {
-		/*
-		 * There is no need to specifically test the DRBG with every
-		 * backend cipher -- covered by drbg_nopr_hmac_sha512 test
-		 */
-		.alg = "drbg_nopr_hmac_sha384",
-		.test = alg_test_null,
-		.fips_allowed = 1
 	}, {
 		.alg = "drbg_nopr_hmac_sha512",
 		.test = alg_test_drbg,
@@ -4696,70 +4648,6 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.suite = {
 			.drbg = __VECS(drbg_nopr_hmac_sha512_tv_template)
 		}
-	}, {
-		.alg = "drbg_nopr_sha256",
-		.test = alg_test_drbg,
-		.fips_allowed = 1,
-		.suite = {
-			.drbg = __VECS(drbg_nopr_sha256_tv_template)
-		}
-	}, {
-		/* covered by drbg_nopr_sha256 test */
-		.alg = "drbg_nopr_sha384",
-		.test = alg_test_null,
-		.fips_allowed = 1
-	}, {
-		.alg = "drbg_nopr_sha512",
-		.fips_allowed = 1,
-		.test = alg_test_null,
-	}, {
-		.alg = "drbg_pr_ctr_aes128",
-		.test = alg_test_drbg,
-		.fips_allowed = 1,
-		.suite = {
-			.drbg = __VECS(drbg_pr_ctr_aes128_tv_template)
-		}
-	}, {
-		/* covered by drbg_pr_ctr_aes128 test */
-		.alg = "drbg_pr_ctr_aes192",
-		.fips_allowed = 1,
-		.test = alg_test_null,
-	}, {
-		.alg = "drbg_pr_ctr_aes256",
-		.fips_allowed = 1,
-		.test = alg_test_null,
-	}, {
-		.alg = "drbg_pr_hmac_sha256",
-		.test = alg_test_drbg,
-		.fips_allowed = 1,
-		.suite = {
-			.drbg = __VECS(drbg_pr_hmac_sha256_tv_template)
-		}
-	}, {
-		/* covered by drbg_pr_hmac_sha256 test */
-		.alg = "drbg_pr_hmac_sha384",
-		.test = alg_test_null,
-		.fips_allowed = 1
-	}, {
-		.alg = "drbg_pr_hmac_sha512",
-		.test = alg_test_null,
-		.fips_allowed = 1,
-	}, {
-		.alg = "drbg_pr_sha256",
-		.test = alg_test_drbg,
-		.fips_allowed = 1,
-		.suite = {
-			.drbg = __VECS(drbg_pr_sha256_tv_template)
-		}
-	}, {
-		/* covered by drbg_pr_sha256 test */
-		.alg = "drbg_pr_sha384",
-		.test = alg_test_null,
-		.fips_allowed = 1
-	}, {
-		.alg = "drbg_pr_sha512",
-		.fips_allowed = 1,
-		.test = alg_test_null,
 	}, {
 		.alg = "ecb(aes)",
 		.generic_driver = "ecb(aes-lib)",
@@ -5295,6 +5183,9 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.suite = {
 			.sig = __VECS(pkcs1_rsa_none_tv_template)
 		}
+	}, {
+		.alg = "pkcs1(rsa,sha1)",
+		.test = alg_test_null,
 	}, {
 		.alg = "pkcs1(rsa,sha224)",
 		.test = alg_test_null,
@@ -5330,6 +5221,9 @@ static const struct alg_test_desc alg_test_descs[] = {
 		.alg = "pkcs1pad(rsa)",
 		.test = alg_test_null,
 		.fips_allowed = 1,
+	}, {
+		.alg = "pkcs1pad(rsa,sha1)",
+		.test = alg_test_null,
 	}, {
 		.alg = "rfc3686(ctr(aes))",
 		.generic_driver = "rfc3686(ctr(aes-lib))",
diff --git a/crypto/testmgr.h b/crypto/testmgr.h
index 9b4d7e11c9fd9..b7dcf40af6dbe 100644
--- a/crypto/testmgr.h
+++ b/crypto/testmgr.h
@@ -120,16 +120,24 @@ struct aead_testvec {
 };
 
 struct drbg_testvec {
+	/* Instantiate */
 	const unsigned char *entropy;
 	size_t entropylen;
-	const unsigned char *entpra;
-	const unsigned char *entprb;
-	size_t entprlen;
+	const unsigned char *pers;
+	size_t perslen;
+
+	/* Reseed (optional) */
+	const unsigned char *ent_reseed;
+	size_t ent_reseed_len;
+	const unsigned char *addtl_reseed;
+	size_t addtl_reseed_len;
+
+	/* Generate (twice) */
 	const unsigned char *addtla;
 	const unsigned char *addtlb;
 	size_t addtllen;
-	const unsigned char *pers;
-	size_t perslen;
+
+	/* Expected output from last call to Generate */
 	const unsigned char *expected;
 	size_t expectedlen;
 };
@@ -23422,881 +23430,199 @@ static const struct aead_testvec aegis128_tv_template[] = {
 	},
 };
 
-/*
- * SP800-90A DRBG Test vectors from
- * http://csrc.nist.gov/groups/STM/cavp/documents/drbg/drbgtestvectors.zip
- *
- * Test vectors for DRBG with prediction resistance. All types of DRBGs
- * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and
- * w/o personalization string, w/ and w/o additional input string).
- */
-static const struct drbg_testvec drbg_pr_sha256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\x72\x88\x4c\xcd\x6c\x85\x57\x70\xf7\x0b\x8b\x86"
-			"\xc1\xeb\xd2\x4e\x36\x14\xab\x18\xc4\x9c\xc9\xcf"
-			"\x1a\xe8\xf7\x7b\x02\x49\x73\xd7\xf1\x42\x7d\xc6"
-			"\x3f\x29\x2d\xec\xd3\x66\x51\x3f\x1d\x8d\x5b\x4e",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\x38\x9c\x91\xfa\xc2\xa3\x46\x89\x56\x08\x3f\x62"
-			"\x73\xd5\x22\xa9\x29\x63\x3a\x1d\xe5\x5d\x5e\x4f"
-			"\x67\xb0\x67\x7a\x5e\x9e\x0c\x62",
-		.entprb = (unsigned char *)
-			"\xb2\x8f\x36\xb2\xf6\x8d\x39\x13\xfa\x6c\x66\xcf"
-			"\x62\x8a\x7e\x8c\x12\x33\x71\x9c\x69\xe4\xa5\xf0"
-			"\x8c\xee\xeb\x9c\xf5\x31\x98\x31",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x52\x7b\xa3\xad\x71\x77\xa4\x49\x42\x04\x61\xc7"
-			"\xf0\xaf\xa5\xfd\xd3\xb3\x0d\x6a\x61\xba\x35\x49"
-			"\xbb\xaa\xaf\xe4\x25\x7d\xb5\x48\xaf\x5c\x18\x3d"
-			"\x33\x8d\x9d\x45\xdf\x98\xd5\x94\xa8\xda\x92\xfe"
-			"\xc4\x3c\x94\x2a\xcf\x7f\x7b\xf2\xeb\x28\xa9\xf1"
-			"\xe0\x86\x30\xa8\xfe\xf2\x48\x90\x91\x0c\x75\xb5"
-			"\x3c\x00\xf0\x4d\x09\x4f\x40\xa7\xa2\x8c\x52\xdf"
-			"\x52\xef\x17\xbf\x3d\xd1\xa2\x31\xb4\xb8\xdc\xe6"
-			"\x5b\x0d\x1f\x78\x36\xb4\xe6\x4b\xa7\x11\x25\xd5"
-			"\x94\xc6\x97\x36\xab\xf0\xe5\x31\x28\x6a\xbb\xce"
-			"\x30\x81\xa6\x8f\x27\x14\xf8\x1c",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x5d\xf2\x14\xbc\xf6\xb5\x4e\x0b\xf0\x0d\x6f\x2d"
-			"\xe2\x01\x66\x7b\xd0\xa4\x73\xa4\x21\xdd\xb0\xc0"
-			"\x51\x79\x09\xf4\xea\xa9\x08\xfa\xa6\x67\xe0\xe1"
-			"\xd1\x88\xa8\xad\xee\x69\x74\xb3\x55\x06\x9b\xf6",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xef\x48\x06\xa2\xc2\x45\xf1\x44\xfa\x34\x2c\xeb"
-			"\x8d\x78\x3c\x09\x8f\x34\x72\x20\xf2\xe7\xfd\x13"
-			"\x76\x0a\xf6\xdc\x3c\xf5\xc0\x15",
-		.entprb = (unsigned char *)
-			"\x4b\xbe\xe5\x24\xed\x6a\x2d\x0c\xdb\x73\x5e\x09"
-			"\xf9\xad\x67\x7c\x51\x47\x8b\x6b\x30\x2a\xc6\xde"
-			"\x76\xaa\x55\x04\x8b\x0a\x72\x95",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x3b\x14\x71\x99\xa1\xda\xa0\x42\xe6\xc8\x85\x32"
-			"\x70\x20\x32\x53\x9a\xbe\xd1\x1e\x15\xef\xfb\x4c"
-			"\x25\x6e\x19\x3a\xf0\xb9\xcb\xde\xf0\x3b\xc6\x18"
-			"\x4d\x85\x5a\x9b\xf1\xe3\xc2\x23\x03\x93\x08\xdb"
-			"\xa7\x07\x4b\x33\x78\x40\x4d\xeb\x24\xf5\x6e\x81"
-			"\x4a\x1b\x6e\xa3\x94\x52\x43\xb0\xaf\x2e\x21\xf4"
-			"\x42\x46\x8e\x90\xed\x34\x21\x75\xea\xda\x67\xb6"
-			"\xe4\xf6\xff\xc6\x31\x6c\x9a\x5a\xdb\xb3\x97\x13"
-			"\x09\xd3\x20\x98\x33\x2d\x6d\xd7\xb5\x6a\xa8\xa9"
-			"\x9a\x5b\xd6\x87\x52\xa1\x89\x2b\x4b\x9c\x64\x60"
-			"\x50\x47\xa3\x63\x81\x16\xaf\x19",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\xbe\x13\xdb\x2a\xe9\xa8\xfe\x09\x97\xe1\xce\x5d"
-			"\xe8\xbb\xc0\x7c\x4f\xcb\x62\x19\x3f\x0f\xd2\xad"
-			"\xa9\xd0\x1d\x59\x02\xc4\xff\x70",
-		.addtlb = (unsigned char *)
-			"\x6f\x96\x13\xe2\xa7\xf5\x6c\xfe\xdf\x66\xe3\x31"
-			"\x63\x76\xbf\x20\x27\x06\x49\xf1\xf3\x01\x77\x41"
-			"\x9f\xeb\xe4\x38\xfe\x67\x00\xcd",
-		.addtllen = 32,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\xc6\x1c\xaf\x83\xa2\x56\x38\xf9\xb0\xbc\xd9\x85"
-			"\xf5\x2e\xc4\x46\x9c\xe1\xb9\x40\x98\x70\x10\x72"
-			"\xd7\x7d\x15\x85\xa1\x83\x5a\x97\xdf\xc8\xa8\xe8"
-			"\x03\x4c\xcb\x70\x35\x8b\x90\x94\x46\x8a\x6e\xa1",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xc9\x05\xa4\xcf\x28\x80\x4b\x93\x0f\x8b\xc6\xf9"
-			"\x09\x41\x58\x74\xe9\xec\x28\xc7\x53\x0a\x73\x60"
-			"\xba\x0a\xde\x57\x5b\x4b\x9f\x29",
-		.entprb = (unsigned char *)
-			"\x4f\x31\xd2\xeb\xac\xfa\xa8\xe2\x01\x7d\xf3\xbd"
-			"\x42\xbd\x20\xa0\x30\x65\x74\xd5\x5d\xd2\xad\xa4"
-			"\xa9\xeb\x1f\x4d\xf6\xfd\xb8\x26",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\xf6\x13\x05\xcb\x83\x60\x16\x42\x49\x1d\xc6\x25"
-			"\x3b\x8c\x31\xa3\xbe\x8b\xbd\x1c\xe2\xec\x1d\xde"
-			"\xbb\xbf\xa1\xac\xa8\x9f\x50\xce\x69\xce\xef\xd5"
-			"\xd6\xf2\xef\x6a\xf7\x81\x38\xdf\xbc\xa7\x5a\xb9"
-			"\xb2\x42\x65\xab\xe4\x86\x8d\x2d\x9d\x59\x99\x2c"
-			"\x5a\x0d\x71\x55\x98\xa4\x45\xc2\x8d\xdb\x05\x5e"
-			"\x50\x21\xf7\xcd\xe8\x98\x43\xce\x57\x74\x63\x4c"
-			"\xf3\xb1\xa5\x14\x1e\x9e\x01\xeb\x54\xd9\x56\xae"
-			"\xbd\xb6\x6f\x1a\x47\x6b\x3b\x44\xe4\xa2\xe9\x3c"
-			"\x6c\x83\x12\x30\xb8\x78\x7f\x8e\x54\x82\xd4\xfe"
-			"\x90\x35\x0d\x4c\x4d\x85\xe7\x13",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xa5\xbf\xac\x4f\x71\xa1\xbb\x67\x94\xc6\x50\xc7"
-			"\x2a\x45\x9e\x10\xa8\xed\xf7\x52\x4f\xfe\x21\x90"
-			"\xa4\x1b\xe1\xe2\x53\xcc\x61\x47",
-		.perslen = 32,
-	}, {
-		.entropy = (unsigned char *)
-			"\xb6\xc1\x8d\xdf\x99\x54\xbe\x95\x10\x48\xd9\xf6"
-			"\xd7\x48\xa8\x73\x2d\x74\xde\x1e\xde\x57\x7e\xf4"
-			"\x7b\x7b\x64\xef\x88\x7a\xa8\x10\x4b\xe1\xc1\x87"
-			"\xbb\x0b\xe1\x39\x39\x50\xaf\x68\x9c\xa2\xbf\x5e",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xdc\x81\x0a\x01\x58\xa7\x2e\xce\xee\x48\x8c\x7c"
-			"\x77\x9e\x3c\xf1\x17\x24\x7a\xbb\xab\x9f\xca\x12"
-			"\x19\xaf\x97\x2d\x5f\xf9\xff\xfc",
-		.entprb = (unsigned char *)
-			"\xaf\xfc\x4f\x98\x8b\x93\x95\xc1\xb5\x8b\x7f\x73"
-			"\x6d\xa6\xbe\x6d\x33\xeb\x2c\x82\xb1\xaf\xc1\xb6"
-			"\xb6\x05\xe2\x44\xaa\xfd\xe7\xdb",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x51\x79\xde\x1c\x0f\x58\xf3\xf4\xc9\x57\x2e\x31"
-			"\xa7\x09\xa1\x53\x64\x63\xa2\xc5\x1d\x84\x88\x65"
-			"\x01\x1b\xc6\x16\x3c\x49\x5b\x42\x8e\x53\xf5\x18"
-			"\xad\x94\x12\x0d\x4f\x55\xcc\x45\x5c\x98\x0f\x42"
-			"\x28\x2f\x47\x11\xf9\xc4\x01\x97\x6b\xa0\x94\x50"
-			"\xa9\xd1\x5e\x06\x54\x3f\xdf\xbb\xc4\x98\xee\x8b"
-			"\xba\xa9\xfa\x49\xee\x1d\xdc\xfb\x50\xf6\x51\x9f"
-			"\x6c\x4a\x9a\x6f\x63\xa2\x7d\xad\xaf\x3a\x24\xa0"
-			"\xd9\x9f\x07\xeb\x15\xee\x26\xe0\xd5\x63\x39\xda"
-			"\x3c\x59\xd6\x33\x6c\x02\xe8\x05\x71\x46\x68\x44"
-			"\x63\x4a\x68\x72\xe9\xf5\x55\xfe",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x15\x20\x2f\xf6\x98\x28\x63\xa2\xc4\x4e\xbb\x6c"
-			"\xb2\x25\x92\x61\x79\xc9\x22\xc4\x61\x54\x96\xff"
-			"\x4a\x85\xca\x80\xfe\x0d\x1c\xd0",
-		.addtlb = (unsigned char *)
-			"\xde\x29\x8e\x03\x42\x61\xa3\x28\x5e\xc8\x80\xc2"
-			"\x6d\xbf\xad\x13\xe1\x8d\x2a\xc7\xe8\xc7\x18\x89"
-			"\x42\x58\x9e\xd6\xcc\xad\x7b\x1e",
-		.addtllen = 32,
-		.pers = (unsigned char *)
-			"\x84\xc3\x73\x9e\xce\xb3\xbc\x89\xf7\x62\xb3\xe1"
-			"\xd7\x48\x45\x8a\xa9\xcc\xe9\xed\xd5\x81\x84\x52"
-			"\x82\x4c\xdc\x19\xb8\xf8\x92\x5c",
-		.perslen = 32,
-	},
-};
-
-static const struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\x99\x69\xe5\x4b\x47\x03\xff\x31\x78\x5b\x87\x9a"
-			"\x7e\x5c\x0e\xae\x0d\x3e\x30\x95\x59\xe9\xfe\x96"
-			"\xb0\x67\x6d\x49\xd5\x91\xea\x4d\x07\xd2\x0d\x46"
-			"\xd0\x64\x75\x7d\x30\x23\xca\xc2\x37\x61\x27\xab",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xc6\x0f\x29\x99\x10\x0f\x73\x8c\x10\xf7\x47\x92"
-			"\x67\x6a\x3f\xc4\xa2\x62\xd1\x37\x21\x79\x80\x46"
-			"\xe2\x9a\x29\x51\x81\x56\x9f\x54",
-		.entprb = (unsigned char *)
-			"\xc1\x1d\x45\x24\xc9\x07\x1b\xd3\x09\x60\x15\xfc"
-			"\xf7\xbc\x24\xa6\x07\xf2\x2f\xa0\x65\xc9\x37\x65"
-			"\x8a\x2a\x77\xa8\x69\x90\x89\xf4",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\xab\xc0\x15\x85\x60\x94\x80\x3a\x93\x8d\xff\xd2"
-			"\x0d\xa9\x48\x43\x87\x0e\xf9\x35\xb8\x2c\xfe\xc1"
-			"\x77\x06\xb8\xf5\x51\xb8\x38\x50\x44\x23\x5d\xd4"
-			"\x4b\x59\x9f\x94\xb3\x9b\xe7\x8d\xd4\x76\xe0\xcf"
-			"\x11\x30\x9c\x99\x5a\x73\x34\xe0\xa7\x8b\x37\xbc"
-			"\x95\x86\x23\x50\x86\xfa\x3b\x63\x7b\xa9\x1c\xf8"
-			"\xfb\x65\xef\xa2\x2a\x58\x9c\x13\x75\x31\xaa\x7b"
-			"\x2d\x4e\x26\x07\xaa\xc2\x72\x92\xb0\x1c\x69\x8e"
-			"\x6e\x01\xae\x67\x9e\xb8\x7c\x01\xa8\x9c\x74\x22"
-			"\xd4\x37\x2d\x6d\x75\x4a\xba\xbb\x4b\xf8\x96\xfc"
-			"\xb1\xcd\x09\xd6\x92\xd0\x28\x3f",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\xb9\x1f\xe9\xef\xdd\x9b\x7d\x20\xb6\xec\xe0\x2f"
-			"\xdb\x76\x24\xce\x41\xc8\x3a\x4a\x12\x7f\x3e\x2f"
-			"\xae\x05\x99\xea\xb5\x06\x71\x0d\x0c\x4c\xb4\x05"
-			"\x26\xc6\xbd\xf5\x7f\x2a\x3d\xf2\xb5\x49\x7b\xda",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xef\x67\x50\x9c\xa7\x7d\xdf\xb7\x2d\x81\x01\xa4"
-			"\x62\x81\x6a\x69\x5b\xb3\x37\x45\xa7\x34\x8e\x26"
-			"\x46\xd9\x26\xa2\x19\xd4\x94\x43",
-		.entprb = (unsigned char *)
-			"\x97\x75\x53\x53\xba\xb4\xa6\xb2\x91\x60\x71\x79"
-			"\xd1\x6b\x4a\x24\x9a\x34\x66\xcc\x33\xab\x07\x98"
-			"\x51\x78\x72\xb2\x79\xfd\x2c\xff",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x9c\xdc\x63\x8a\x19\x23\x22\x66\x0c\xc5\xb9\xd7"
-			"\xfb\x2a\xb0\x31\xe3\x8a\x36\xa8\x5a\xa8\x14\xda"
-			"\x1e\xa9\xcc\xfe\xb8\x26\x44\x83\x9f\xf6\xff\xaa"
-			"\xc8\x98\xb8\x30\x35\x3b\x3d\x36\xd2\x49\xd4\x40"
-			"\x62\x0a\x65\x10\x76\x55\xef\xc0\x95\x9c\xa7\xda"
-			"\x3f\xcf\xb7\x7b\xc6\xe1\x28\x52\xfc\x0c\xe2\x37"
-			"\x0d\x83\xa7\x51\x4b\x31\x47\x3c\xe1\x3c\xae\x70"
-			"\x01\xc8\xa3\xd3\xc2\xac\x77\x9c\xd1\x68\x77\x9b"
-			"\x58\x27\x3b\xa5\x0f\xc2\x7a\x8b\x04\x65\x62\xd5"
-			"\xe8\xd6\xfe\x2a\xaf\xd3\xd3\xfe\xbd\x18\xfb\xcd"
-			"\xcd\x66\xb5\x01\x69\x66\xa0\x3c",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x17\xc1\x56\xcb\xcc\x50\xd6\x03\x7d\x45\x76\xa3"
-			"\x75\x76\xc1\x4a\x66\x1b\x2e\xdf\xb0\x2e\x7d\x56"
-			"\x6d\x99\x3b\xc6\x58\xda\x03\xf6",
-		.addtlb = (unsigned char *)
-			"\x7c\x7b\x4a\x4b\x32\x5e\x6f\x67\x34\xf5\x21\x4c"
-			"\xf9\x96\xf9\xbf\x1c\x8c\x81\xd3\x9b\x60\x6a\x44"
-			"\xc6\x03\xa2\xfb\x13\x20\x19\xb7",
-		.addtllen = 32,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x13\x54\x96\xfc\x1b\x7d\x28\xf3\x18\xc9\xa7\x89"
-			"\xb6\xb3\xc8\x72\xac\x00\xd4\x59\x36\x25\x05\xaf"
-			"\xa5\xdb\x96\xcb\x3c\x58\x46\x87\xa5\xaa\xbf\x20"
-			"\x3b\xfe\x23\x0e\xd1\xc7\x41\x0f\x3f\xc9\xb3\x67",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\xe2\xbd\xb7\x48\x08\x06\xf3\xe1\x93\x3c\xac\x79"
-			"\xa7\x2b\x11\xda\xe3\x2e\xe1\x91\xa5\x02\x19\x57"
-			"\x20\x28\xad\xf2\x60\xd7\xcd\x45",
-		.entprb = (unsigned char *)
-			"\x8b\xd4\x69\xfc\xff\x59\x95\x95\xc6\x51\xde\x71"
-			"\x68\x5f\xfc\xf9\x4a\xab\xec\x5a\xcb\xbe\xd3\x66"
-			"\x1f\xfa\x74\xd3\xac\xa6\x74\x60",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x1f\x9e\xaf\xe4\xd2\x46\xb7\x47\x41\x4c\x65\x99"
-			"\x01\xe9\x3b\xbb\x83\x0c\x0a\xb0\xc1\x3a\xe2\xb3"
-			"\x31\x4e\xeb\x93\x73\xee\x0b\x26\xc2\x63\xa5\x75"
-			"\x45\x99\xd4\x5c\x9f\xa1\xd4\x45\x87\x6b\x20\x61"
-			"\x40\xea\x78\xa5\x32\xdf\x9e\x66\x17\xaf\xb1\x88"
-			"\x9e\x2e\x23\xdd\xc1\xda\x13\x97\x88\xa5\xb6\x5e"
-			"\x90\x14\x4e\xef\x13\xab\x5c\xd9\x2c\x97\x9e\x7c"
-			"\xd7\xf8\xce\xea\x81\xf5\xcd\x71\x15\x49\x44\xce"
-			"\x83\xb6\x05\xfb\x7d\x30\xb5\x57\x2c\x31\x4f\xfc"
-			"\xfe\x80\xb6\xc0\x13\x0c\x5b\x9b\x2e\x8f\x3d\xfc"
-			"\xc2\xa3\x0c\x11\x1b\x80\x5f\xf3",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\x64\xb6\xfc\x60\xbc\x61\x76\x23\x6d\x3f\x4a\x0f"
-			"\xe1\xb4\xd5\x20\x9e\x70\xdd\x03\x53\x6d\xbf\xce"
-			"\xcd\x56\x80\xbc\xb8\x15\xc8\xaa",
-		.perslen = 32,
-	}, {
-		.entropy = (unsigned char *)
-			"\xc7\xcc\xbc\x67\x7e\x21\x66\x1e\x27\x2b\x63\xdd"
-			"\x3a\x78\xdc\xdf\x66\x6d\x3f\x24\xae\xcf\x37\x01"
-			"\xa9\x0d\x89\x8a\xa7\xdc\x81\x58\xae\xb2\x10\x15"
-			"\x7e\x18\x44\x6d\x13\xea\xdf\x37\x85\xfe\x81\xfb",
-		.entropylen = 48,
-		.entpra = (unsigned char *)
-			"\x7b\xa1\x91\x5b\x3c\x04\xc4\x1b\x1d\x19\x2f\x1a"
-			"\x18\x81\x60\x3c\x6c\x62\x91\xb7\xe9\xf5\xcb\x96"
-			"\xbb\x81\x6a\xcc\xb5\xae\x55\xb6",
-		.entprb = (unsigned char *)
-			"\x99\x2c\xc7\x78\x7e\x3b\x88\x12\xef\xbe\xd3\xd2"
-			"\x7d\x2a\xa5\x86\xda\x8d\x58\x73\x4a\x0a\xb2\x2e"
-			"\xbb\x4c\x7e\xe3\x9a\xb6\x81\xc1",
-		.entprlen = 32,
-		.expected = (unsigned char *)
-			"\x95\x6f\x95\xfc\x3b\xb7\xfe\x3e\xd0\x4e\x1a\x14"
-			"\x6c\x34\x7f\x7b\x1d\x0d\x63\x5e\x48\x9c\x69\xe6"
-			"\x46\x07\xd2\x87\xf3\x86\x52\x3d\x98\x27\x5e\xd7"
-			"\x54\xe7\x75\x50\x4f\xfb\x4d\xfd\xac\x2f\x4b\x77"
-			"\xcf\x9e\x8e\xcc\x16\xa2\x24\xcd\x53\xde\x3e\xc5"
-			"\x55\x5d\xd5\x26\x3f\x89\xdf\xca\x8b\x4e\x1e\xb6"
-			"\x88\x78\x63\x5c\xa2\x63\x98\x4e\x6f\x25\x59\xb1"
-			"\x5f\x2b\x23\xb0\x4b\xa5\x18\x5d\xc2\x15\x74\x40"
-			"\x59\x4c\xb4\x1e\xcf\x9a\x36\xfd\x43\xe2\x03\xb8"
-			"\x59\x91\x30\x89\x2a\xc8\x5a\x43\x23\x7c\x73\x72"
-			"\xda\x3f\xad\x2b\xba\x00\x6b\xd1",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x18\xe8\x17\xff\xef\x39\xc7\x41\x5c\x73\x03\x03"
-			"\xf6\x3d\xe8\x5f\xc8\xab\xe4\xab\x0f\xad\xe8\xd6"
-			"\x86\x88\x55\x28\xc1\x69\xdd\x76",
-		.addtlb = (unsigned char *)
-			"\xac\x07\xfc\xbe\x87\x0e\xd3\xea\x1f\x7e\xb8\xe7"
-			"\x9d\xec\xe8\xe7\xbc\xf3\x18\x25\x77\x35\x4a\xaa"
-			"\x00\x99\x2a\xdd\x0a\x00\x50\x82",
-		.addtllen = 32,
-		.pers = (unsigned char *)
-			"\xbc\x55\xab\x3c\xf6\x52\xb0\x11\x3d\x7b\x90\xb8"
-			"\x24\xc9\x26\x4e\x5a\x1e\x77\x0d\x3d\x58\x4a\xda"
-			"\xd1\x81\xe9\xf8\xeb\x30\x8f\x6f",
-		.perslen = 32,
-	},
-};
-
-static const struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\xd1\x44\xc6\x61\x81\x6d\xca\x9d\x15\x28\x8a\x42"
-			"\x94\xd7\x28\x9c\x43\x77\x19\x29\x1a\x6d\xc3\xa2",
-		.entropylen = 24,
-		.entpra = (unsigned char *)
-			"\x96\xd8\x9e\x45\x32\xc9\xd2\x08\x7a\x6d\x97\x15"
-			"\xb4\xec\x80\xb1",
-		.entprb = (unsigned char *)
-			"\x8b\xb6\x72\xb5\x24\x0b\x98\x65\x95\x95\xe9\xc9"
-			"\x28\x07\xeb\xc2",
-		.entprlen = 16,
-		.expected = (unsigned char *)
-			"\x70\x19\xd0\x4c\x45\x78\xd6\x68\xa9\x9a\xaa\xfe"
-			"\xc1\xdf\x27\x9a\x1c\x0d\x0d\xf7\x24\x75\x46\xcc"
-			"\x77\x6b\xdf\x89\xc6\x94\xdc\x74\x50\x10\x70\x18"
-			"\x9b\xdc\x96\xb4\x89\x23\x40\x1a\xce\x09\x87\xce"
-			"\xd2\xf3\xd5\xe4\x51\x67\x74\x11\x5a\xcc\x8b\x3b"
-			"\x8a\xf1\x23\xa8",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x8e\x83\xe0\xeb\x37\xea\x3e\x53\x5e\x17\x6e\x77"
-			"\xbd\xb1\x53\x90\xfc\xdc\xc1\x3c\x9a\x88\x22\x94",
-		.entropylen = 24,
-		.entpra = (unsigned char *)
-			"\x6a\x85\xe7\x37\xc8\xf1\x04\x31\x98\x4f\xc8\x73"
-			"\x67\xd1\x08\xf8",
-		.entprb = (unsigned char *)
-			"\xd7\xa4\x68\xe2\x12\x74\xc3\xd9\xf1\xb7\x05\xbc"
-			"\xd4\xba\x04\x58",
-		.entprlen = 16,
-		.expected = (unsigned char *)
-			"\x78\xd6\xa6\x70\xff\xd1\x82\xf5\xa2\x88\x7f\x6d"
-			"\x3d\x8c\x39\xb1\xa8\xcb\x2c\x91\xab\x14\x7e\xbc"
-			"\x95\x45\x9f\x24\xb8\x20\xac\x21\x23\xdb\x72\xd7"
-			"\x12\x8d\x48\x95\xf3\x19\x0c\x43\xc6\x19\x45\xfc"
-			"\x8b\xac\x40\x29\x73\x00\x03\x45\x5e\x12\xff\x0c"
-			"\xc1\x02\x41\x82",
-		.expectedlen = 64,
-		.addtla = (unsigned char *)
-			"\xa2\xd9\x38\xcf\x8b\x29\x67\x5b\x65\x62\x6f\xe8"
-			"\xeb\xb3\x01\x76",
-		.addtlb = (unsigned char *)
-			"\x59\x63\x1e\x81\x8a\x14\xa8\xbb\xa1\xb8\x41\x25"
-			"\xd0\x7f\xcc\x43",
-		.addtllen = 16,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x04\xd9\x49\xa6\xdc\xe8\x6e\xbb\xf1\x08\x77\x2b"
-			"\x9e\x08\xca\x92\x65\x16\xda\x99\xa2\x59\xf3\xe8",
-		.entropylen = 24,
-		.entpra = (unsigned char *)
-			"\x38\x7e\x3f\x6b\x51\x70\x7b\x20\xec\x53\xd0\x66"
-			"\xc3\x0f\xe3\xb0",
-		.entprb = (unsigned char *)
-			"\xe0\x86\xa6\xaa\x5f\x72\x2f\xad\xf7\xef\x06\xb8"
-			"\xd6\x9c\x9d\xe8",
-		.entprlen = 16,
-		.expected = (unsigned char *)
-			"\xc9\x0a\xaf\x85\x89\x71\x44\x66\x4f\x25\x0b\x2b"
-			"\xde\xd8\xfa\xff\x52\x5a\x1b\x32\x5e\x41\x7a\x10"
-			"\x1f\xef\x1e\x62\x23\xe9\x20\x30\xc9\x0d\xad\x69"
-			"\xb4\x9c\x5b\xf4\x87\x42\xd5\xae\x5e\x5e\x43\xcc"
-			"\xd9\xfd\x0b\x93\x4a\xe3\xd4\x06\x37\x36\x0f\x3f"
-			"\x72\x82\x0c\xcf",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xbf\xa4\x9a\x8f\x7b\xd8\xb1\x7a\x9d\xfa\x45\xed"
-			"\x21\x52\xb3\xad",
-		.perslen = 16,
-	}, {
-		.entropy = (unsigned char *)
-			"\x92\x89\x8f\x31\xfa\x1c\xff\x6d\x18\x2f\x26\x06"
-			"\x43\xdf\xf8\x18\xc2\xa4\xd9\x72\xc3\xb9\xb6\x97",
-		.entropylen = 24,
-		.entpra = (unsigned char *)
-			"\x20\x72\x8a\x06\xf8\x6f\x8d\xd4\x41\xe2\x72\xb7"
-			"\xc4\x2c\xe8\x10",
-		.entprb = (unsigned char *)
-			"\x3d\xb0\xf0\x94\xf3\x05\x50\x33\x17\x86\x3e\x22"
-			"\x08\xf7\xa5\x01",
-		.entprlen = 16,
-		.expected = (unsigned char *)
-			"\x5a\x35\x39\x87\x0f\x4d\x22\xa4\x09\x24\xee\x71"
-			"\xc9\x6f\xac\x72\x0a\xd6\xf0\x88\x82\xd0\x83\x28"
-			"\x73\xec\x3f\x93\xd8\xab\x45\x23\xf0\x7e\xac\x45"
-			"\x14\x5e\x93\x9f\xb1\xd6\x76\x43\x3d\xb6\xe8\x08"
-			"\x88\xf6\xda\x89\x08\x77\x42\xfe\x1a\xf4\x3f\xc4"
-			"\x23\xc5\x1f\x68",
-		.expectedlen = 64,
-		.addtla = (unsigned char *)
-			"\x1a\x40\xfa\xe3\xcc\x6c\x7c\xa0\xf8\xda\xba\x59"
-			"\x23\x6d\xad\x1d",
-		.addtlb = (unsigned char *)
-			"\x9f\x72\x76\x6c\xc7\x46\xe5\xed\x2e\x53\x20\x12"
-			"\xbc\x59\x31\x8c",
-		.addtllen = 16,
-		.pers = (unsigned char *)
-			"\xea\x65\xee\x60\x26\x4e\x7e\xb6\x0e\x82\x68\xc4"
-			"\x37\x3c\x5c\x0b",
-		.perslen = 16,
-	},
-};
-
-/*
- * SP800-90A DRBG Test vectors from
- * http://csrc.nist.gov/groups/STM/cavp/documents/drbg/drbgtestvectors.zip
- *
- * Test vectors for DRBG without prediction resistance. All types of DRBGs
- * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and
- * w/o personalization string, w/ and w/o additional input string).
- */
-static const struct drbg_testvec drbg_nopr_sha256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\xa6\x5a\xd0\xf3\x45\xdb\x4e\x0e\xff\xe8\x75\xc3"
-			"\xa2\xe7\x1f\x42\xc7\x12\x9d\x62\x0f\xf5\xc1\x19"
-			"\xa9\xef\x55\xf0\x51\x85\xe0\xfb\x85\x81\xf9\x31"
-			"\x75\x17\x27\x6e\x06\xe9\x60\x7d\xdb\xcb\xcc\x2e",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xd3\xe1\x60\xc3\x5b\x99\xf3\x40\xb2\x62\x82\x64"
-			"\xd1\x75\x10\x60\xe0\x04\x5d\xa3\x83\xff\x57\xa5"
-			"\x7d\x73\xa6\x73\xd2\xb8\xd8\x0d\xaa\xf6\xa6\xc3"
-			"\x5a\x91\xbb\x45\x79\xd7\x3f\xd0\xc8\xfe\xd1\x11"
-			"\xb0\x39\x13\x06\x82\x8a\xdf\xed\x52\x8f\x01\x81"
-			"\x21\xb3\xfe\xbd\xc3\x43\xe7\x97\xb8\x7d\xbb\x63"
-			"\xdb\x13\x33\xde\xd9\xd1\xec\xe1\x77\xcf\xa6\xb7"
-			"\x1f\xe8\xab\x1d\xa4\x66\x24\xed\x64\x15\xe5\x1c"
-			"\xcd\xe2\xc7\xca\x86\xe2\x83\x99\x0e\xea\xeb\x91"
-			"\x12\x04\x15\x52\x8b\x22\x95\x91\x02\x81\xb0\x2d"
-			"\xd4\x31\xf4\xc9\xf7\x04\x27\xdf",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x73\xd3\xfb\xa3\x94\x5f\x2b\x5f\xb9\x8f\xf6\x9c"
-			"\x8a\x93\x17\xae\x19\xc3\x4c\xc3\xd6\xca\xa3\x2d"
-			"\x16\xfc\x42\xd2\x2d\xd5\x6f\x56\xcc\x1d\x30\xff"
-			"\x9e\x06\x3e\x09\xce\x58\xe6\x9a\x35\xb3\xa6\x56",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\x71\x7b\x93\x46\x1a\x40\xaa\x35\xa4\xaa\xc5\xe7"
-			"\x6d\x5b\x5b\x8a\xa0\xdf\x39\x7d\xae\x71\x58\x5b"
-			"\x3c\x7c\xb4\xf0\x89\xfa\x4a\x8c\xa9\x5c\x54\xc0"
-			"\x40\xdf\xbc\xce\x26\x81\x34\xf8\xba\x7d\x1c\xe8"
-			"\xad\x21\xe0\x74\xcf\x48\x84\x30\x1f\xa1\xd5\x4f"
-			"\x81\x42\x2f\xf4\xdb\x0b\x23\xf8\x73\x27\xb8\x1d"
-			"\x42\xf8\x44\x58\xd8\x5b\x29\x27\x0a\xf8\x69\x59"
-			"\xb5\x78\x44\xeb\x9e\xe0\x68\x6f\x42\x9a\xb0\x5b"
-			"\xe0\x4e\xcb\x6a\xaa\xe2\xd2\xd5\x33\x25\x3e\xe0"
-			"\x6c\xc7\x6a\x07\xa5\x03\x83\x9f\xe2\x8b\xd1\x1c"
-			"\x70\xa8\x07\x59\x97\xeb\xf6\xbe",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\xf4\xd5\x98\x3d\xa8\xfc\xfa\x37\xb7\x54\x67\x73"
-			"\xc7\xc3\xdd\x47\x34\x71\x02\x5d\xc1\xa0\xd3\x10"
-			"\xc1\x8b\xbd\xf5\x66\x34\x6f\xdd",
-		.addtlb = (unsigned char *)
-			"\xf7\x9e\x6a\x56\x0e\x73\xe9\xd9\x7a\xd1\x69\xe0"
-			"\x6f\x8c\x55\x1c\x44\xd1\xce\x6f\x28\xcc\xa4\x4d"
-			"\xa8\xc0\x85\xd1\x5a\x0c\x59\x40",
-		.addtllen = 32,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x2a\x85\xa9\x8b\xd0\xda\x83\xd6\xad\xab\x9f\xbb"
-			"\x54\x31\x15\x95\x1c\x4d\x49\x9f\x6a\x15\xf6\xe4"
-			"\x15\x50\x88\x06\x29\x0d\xed\x8d\xb9\x6f\x96\xe1"
-			"\x83\x9f\xf7\x88\xda\x84\xbf\x44\x28\xd9\x1d\xaa",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\x2d\x55\xde\xc9\xed\x05\x47\x07\x3d\x04\xfc\x28"
-			"\x0f\x92\xf0\x4d\xd8\x00\x32\x47\x0a\x1b\x1c\x4b"
-			"\xef\xd9\x97\xa1\x17\x67\xda\x26\x6c\xfe\x76\x46"
-			"\x6f\xbc\x6d\x82\x4e\x83\x8a\x98\x66\x6c\x01\xb6"
-			"\xe6\x64\xe0\x08\x10\x6f\xd3\x5d\x90\xe7\x0d\x72"
-			"\xa6\xa7\xe3\xbb\x98\x11\x12\x56\x23\xc2\x6d\xd1"
-			"\xc8\xa8\x7a\x39\xf3\x34\xe3\xb8\xf8\x66\x00\x77"
-			"\x7d\xcf\x3c\x3e\xfa\xc9\x0f\xaf\xe0\x24\xfa\xe9"
-			"\x84\xf9\x6a\x01\xf6\x35\xdb\x5c\xab\x2a\xef\x4e"
-			"\xac\xab\x55\xb8\x9b\xef\x98\x68\xaf\x51\xd8\x16"
-			"\xa5\x5e\xae\xf9\x1e\xd2\xdb\xe6",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xa8\x80\xec\x98\x30\x98\x15\xd2\xc6\xc4\x68\xf1"
-			"\x3a\x1c\xbf\xce\x6a\x40\x14\xeb\x36\x99\x53\xda"
-			"\x57\x6b\xce\xa4\x1c\x66\x3d\xbc",
-		.perslen = 32,
-	}, {
-		.entropy = (unsigned char *)
-			"\x69\xed\x82\xa9\xc5\x7b\xbf\xe5\x1d\x2f\xcb\x7a"
-			"\xd3\x50\x7d\x96\xb4\xb9\x2b\x50\x77\x51\x27\x74"
-			"\x33\x74\xba\xf1\x30\xdf\x8e\xdf\x87\x1d\x87\xbc"
-			"\x96\xb2\xc3\xa7\xed\x60\x5e\x61\x4e\x51\x29\x1a",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xa5\x71\x24\x31\x11\xfe\x13\xe1\xa8\x24\x12\xfb"
-			"\x37\xa1\x27\xa5\xab\x77\xa1\x9f\xae\x8f\xaf\x13"
-			"\x93\xf7\x53\x85\x91\xb6\x1b\xab\xd4\x6b\xea\xb6"
-			"\xef\xda\x4c\x90\x6e\xef\x5f\xde\xe1\xc7\x10\x36"
-			"\xd5\x67\xbd\x14\xb6\x89\x21\x0c\xc9\x92\x65\x64"
-			"\xd0\xf3\x23\xe0\x7f\xd1\xe8\x75\xc2\x85\x06\xea"
-			"\xca\xc0\xcb\x79\x2d\x29\x82\xfc\xaa\x9a\xc6\x95"
-			"\x7e\xdc\x88\x65\xba\xec\x0e\x16\x87\xec\xa3\x9e"
-			"\xd8\x8c\x80\xab\x3a\x64\xe0\xcb\x0e\x45\x98\xdd"
-			"\x7c\x6c\x6c\x26\x11\x13\xc8\xce\xa9\x47\xa6\x06"
-			"\x57\xa2\x66\xbb\x2d\x7f\xf3\xc1",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x74\xd3\x6d\xda\xe8\xd6\x86\x5f\x63\x01\xfd\xf2"
-			"\x7d\x06\x29\x6d\x94\xd1\x66\xf0\xd2\x72\x67\x4e"
-			"\x77\xc5\x3d\x9e\x03\xe3\xa5\x78",
-		.addtlb = (unsigned char *)
-			"\xf6\xb6\x3d\xf0\x7c\x26\x04\xc5\x8b\xcd\x3e\x6a"
-			"\x9f\x9c\x3a\x2e\xdb\x47\x87\xe5\x8e\x00\x5e\x2b"
-			"\x74\x7f\xa6\xf6\x80\xcd\x9b\x21",
-		.addtllen = 32,
-		.pers = (unsigned char *)
-			"\x74\xa6\xe0\x08\xf9\x27\xee\x1d\x6e\x3c\x28\x20"
-			"\x87\xdd\xd7\x54\x31\x47\x78\x4b\xe5\x6d\xa3\x73"
-			"\xa9\x65\xb1\x10\xc1\xdc\x77\x7c",
-		.perslen = 32,
-	},
-};
-
-static const struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\xca\x85\x19\x11\x34\x93\x84\xbf\xfe\x89\xde\x1c"
-			"\xbd\xc4\x6e\x68\x31\xe4\x4d\x34\xa4\xfb\x93\x5e"
-			"\xe2\x85\xdd\x14\xb7\x1a\x74\x88\x65\x9b\xa9\x6c"
-			"\x60\x1d\xc6\x9f\xc9\x02\x94\x08\x05\xec\x0c\xa8",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xe5\x28\xe9\xab\xf2\xde\xce\x54\xd4\x7c\x7e\x75"
-			"\xe5\xfe\x30\x21\x49\xf8\x17\xea\x9f\xb4\xbe\xe6"
-			"\xf4\x19\x96\x97\xd0\x4d\x5b\x89\xd5\x4f\xbb\x97"
-			"\x8a\x15\xb5\xc4\x43\xc9\xec\x21\x03\x6d\x24\x60"
-			"\xb6\xf7\x3e\xba\xd0\xdc\x2a\xba\x6e\x62\x4a\xbf"
-			"\x07\x74\x5b\xc1\x07\x69\x4b\xb7\x54\x7b\xb0\x99"
-			"\x5f\x70\xde\x25\xd6\xb2\x9e\x2d\x30\x11\xbb\x19"
-			"\xd2\x76\x76\xc0\x71\x62\xc8\xb5\xcc\xde\x06\x68"
-			"\x96\x1d\xf8\x68\x03\x48\x2c\xb3\x7e\xd6\xd5\xc0"
-			"\xbb\x8d\x50\xcf\x1f\x50\xd4\x76\xaa\x04\x58\xbd"
-			"\xab\xa8\x06\xf4\x8b\xe9\xdc\xb8",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\xf9\x7a\x3c\xfd\x91\xfa\xa0\x46\xb9\xe6\x1b\x94"
-			"\x93\xd4\x36\xc4\x93\x1f\x60\x4b\x22\xf1\x08\x15"
-			"\x21\xb3\x41\x91\x51\xe8\xff\x06\x11\xf3\xa7\xd4"
-			"\x35\x95\x35\x7d\x58\x12\x0b\xd1\xe2\xdd\x8a\xed",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xc6\x87\x1c\xff\x08\x24\xfe\x55\xea\x76\x89\xa5"
-			"\x22\x29\x88\x67\x30\x45\x0e\x5d\x36\x2d\xa5\xbf"
-			"\x59\x0d\xcf\x9a\xcd\x67\xfe\xd4\xcb\x32\x10\x7d"
-			"\xf5\xd0\x39\x69\xa6\x6b\x1f\x64\x94\xfd\xf5\xd6"
-			"\x3d\x5b\x4d\x0d\x34\xea\x73\x99\xa0\x7d\x01\x16"
-			"\x12\x6d\x0d\x51\x8c\x7c\x55\xba\x46\xe1\x2f\x62"
-			"\xef\xc8\xfe\x28\xa5\x1c\x9d\x42\x8e\x6d\x37\x1d"
-			"\x73\x97\xab\x31\x9f\xc7\x3d\xed\x47\x22\xe5\xb4"
-			"\xf3\x00\x04\x03\x2a\x61\x28\xdf\x5e\x74\x97\xec"
-			"\xf8\x2c\xa7\xb0\xa5\x0e\x86\x7e\xf6\x72\x8a\x4f"
-			"\x50\x9a\x8c\x85\x90\x87\x03\x9c",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x51\x72\x89\xaf\xe4\x44\xa0\xfe\x5e\xd1\xa4\x1d"
-			"\xbb\xb5\xeb\x17\x15\x00\x79\xbd\xd3\x1e\x29\xcf"
-			"\x2f\xf3\x00\x34\xd8\x26\x8e\x3b",
-		.addtlb = (unsigned char *)
-			"\x88\x02\x8d\x29\xef\x80\xb4\xe6\xf0\xfe\x12\xf9"
-			"\x1d\x74\x49\xfe\x75\x06\x26\x82\xe8\x9c\x57\x14"
-			"\x40\xc0\xc9\xb5\x2c\x42\xa6\xe0",
-		.addtllen = 32,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x8d\xf0\x13\xb4\xd1\x03\x52\x30\x73\x91\x7d\xdf"
-			"\x6a\x86\x97\x93\x05\x9e\x99\x43\xfc\x86\x54\x54"
-			"\x9e\x7a\xb2\x2f\x7c\x29\xf1\x22\xda\x26\x25\xaf"
-			"\x2d\xdd\x4a\xbc\xce\x3c\xf4\xfa\x46\x59\xd8\x4e",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xb9\x1c\xba\x4c\xc8\x4f\xa2\x5d\xf8\x61\x0b\x81"
-			"\xb6\x41\x40\x27\x68\xa2\x09\x72\x34\x93\x2e\x37"
-			"\xd5\x90\xb1\x15\x4c\xbd\x23\xf9\x74\x52\xe3\x10"
-			"\xe2\x91\xc4\x51\x46\x14\x7f\x0d\xa2\xd8\x17\x61"
-			"\xfe\x90\xfb\xa6\x4f\x94\x41\x9c\x0f\x66\x2b\x28"
-			"\xc1\xed\x94\xda\x48\x7b\xb7\xe7\x3e\xec\x79\x8f"
-			"\xbc\xf9\x81\xb7\x91\xd1\xbe\x4f\x17\x7a\x89\x07"
-			"\xaa\x3c\x40\x16\x43\xa5\xb6\x2b\x87\xb8\x9d\x66"
-			"\xb3\xa6\x0e\x40\xd4\xa8\xe4\xe9\xd8\x2a\xf6\xd2"
-			"\x70\x0e\x6f\x53\x5c\xdb\x51\xf7\x5c\x32\x17\x29"
-			"\x10\x37\x41\x03\x0c\xcc\x3a\x56",
-		.expectedlen = 128,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xb5\x71\xe6\x6d\x7c\x33\x8b\xc0\x7b\x76\xad\x37"
-			"\x57\xbb\x2f\x94\x52\xbf\x7e\x07\x43\x7a\xe8\x58"
-			"\x1c\xe7\xbc\x7c\x3a\xc6\x51\xa9",
-		.perslen = 32,
-	}, {
-		.entropy = (unsigned char *)
-			"\xc2\xa5\x66\xa9\xa1\x81\x7b\x15\xc5\xc3\xb7\x78"
-			"\x17\x7a\xc8\x7c\x24\xe7\x97\xbe\x0a\x84\x5f\x11"
-			"\xc2\xfe\x39\x9d\xd3\x77\x32\xf2\xcb\x18\x94\xeb"
-			"\x2b\x97\xb3\xc5\x6e\x62\x83\x29\x51\x6f\x86\xec",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\xb3\xa3\x69\x8d\x77\x76\x99\xa0\xdd\x9f\xa3\xf0"
-			"\xa9\xfa\x57\x83\x2d\x3c\xef\xac\x5d\xf2\x44\x37"
-			"\xc6\xd7\x3a\x0f\xe4\x10\x40\xf1\x72\x90\x38\xae"
-			"\xf1\xe9\x26\x35\x2e\xa5\x9d\xe1\x20\xbf\xb7\xb0"
-			"\x73\x18\x3a\x34\x10\x6e\xfe\xd6\x27\x8f\xf8\xad"
-			"\x84\x4b\xa0\x44\x81\x15\xdf\xdd\xf3\x31\x9a\x82"
-			"\xde\x6b\xb1\x1d\x80\xbd\x87\x1a\x9a\xcd\x35\xc7"
-			"\x36\x45\xe1\x27\x0f\xb9\xfe\x4f\xa8\x8e\xc0\xe4"
-			"\x65\x40\x9e\xa0\xcb\xa8\x09\xfe\x2f\x45\xe0\x49"
-			"\x43\xa2\xe3\x96\xbb\xb7\xdd\x2f\x4e\x07\x95\x30"
-			"\x35\x24\xcc\x9c\xc5\xea\x54\xa1",
-		.expectedlen = 128,
-		.addtla = (unsigned char *)
-			"\x41\x3d\xd8\x3f\xe5\x68\x35\xab\xd4\x78\xcb\x96"
-			"\x93\xd6\x76\x35\x90\x1c\x40\x23\x9a\x26\x64\x62"
-			"\xd3\x13\x3b\x83\xe4\x9c\x82\x0b",
-		.addtlb = (unsigned char *)
-			"\xd5\xc4\xa7\x1f\x9d\x6d\x95\xa1\xbe\xdf\x0b\xd2"
-			"\x24\x7c\x27\x7d\x1f\x84\xa4\xe5\x7a\x4a\x88\x25"
-			"\xb8\x2a\x2d\x09\x7d\xe6\x3e\xf1",
-		.addtllen = 32,
-		.pers = (unsigned char *)
-			"\x13\xce\x4d\x8d\xd2\xdb\x97\x96\xf9\x41\x56\xc8"
-			"\xe8\xf0\x76\x9b\x0a\xa1\xc8\x2c\x13\x23\xb6\x15"
-			"\x36\x60\x3b\xca\x37\xc9\xee\x29",
-		.perslen = 32,
-	},
-};
-
-/* Test vector obtained during NIST ACVP testing */
 static const struct drbg_testvec drbg_nopr_hmac_sha512_tv_template[] = {
+	/*
+	 * Borrowed from the first applicable test vector from ACVP:
+	 * https://github.com/usnistgov/ACVP-Server/blob/v1.1.0.33/gen-val/json-files/hmacDRBG-1.0/prompt.json#L4596
+	 * https://github.com/usnistgov/ACVP-Server/blob/v1.1.0.33/gen-val/json-files/hmacDRBG-1.0/expectedResults.json#L986
+	 */
 	{
-		.entropy = (unsigned char *)
-			"\xDF\xB0\xF2\x18\xF0\x78\x07\x01\x29\xA4\x29\x26"
-			"\x2F\x8A\x34\xCB\x37\xEF\xEE\x41\xE6\x96\xF7\xFF"
-			"\x61\x47\xD3\xED\x41\x97\xEF\x64\x0C\x48\x56\x5A"
-			"\xE6\x40\x6E\x4A\x3B\x9E\x7F\xAC\x08\xEC\x25\xAE"
-			"\x0B\x51\x0E\x2C\x44\x2E\xBD\xDB\x57\xD0\x4A\x6D"
-			"\x80\x3E\x37\x0F",
-		.entropylen = 64,
-		.expected = (unsigned char *)
-			"\x48\xc6\xa8\xdb\x09\xae\xde\x5d\x8c\x77\xf3\x52"
-			"\x92\x71\xa7\xb9\x6d\x53\x6d\xa3\x73\xe3\x55\xb8"
-			"\x39\xd6\x44\x2b\xee\xcb\xe1\x32\x15\x30\xbe\x4e"
-			"\x9b\x1e\x06\xd1\x6b\xbf\xd5\x3e\xea\x7c\xf5\xaa"
-			"\x4b\x05\xb5\xd3\xa7\xb2\xc4\xfe\xe7\x1b\xda\x11"
-			"\x43\x98\x03\x70\x90\xbf\x6e\x43\x9b\xe4\x14\xef"
-			"\x71\xa3\x2a\xef\x9f\x0d\xb9\xe3\x52\xf2\x89\xc9"
-			"\x66\x9a\x60\x60\x99\x60\x62\x4c\xd6\x45\x52\x54"
-			"\xe6\x32\xb2\x1b\xd4\x48\xb5\xa6\xf9\xba\xd3\xff"
-			"\x29\xc5\x21\xe0\x91\x31\xe0\x38\x8c\x93\x0f\x3c"
-			"\x30\x7b\x53\xa3\xc0\x7f\x2d\xc1\x39\xec\x69\x0e"
-			"\xf2\x4a\x3c\x65\xcc\xed\x07\x2a\xf2\x33\x83\xdb"
-			"\x10\x74\x96\x40\xa7\xc5\x1b\xde\x81\xca\x0b\x8f"
-			"\x1e\x0a\x1a\x7a\xbf\x3c\x4a\xb8\x8c\xaf\x7b\x80"
-			"\xb7\xdc\x5d\x0f\xef\x1b\x97\x6e\x3d\x17\x23\x5a"
-			"\x31\xb9\x19\xcf\x5a\xc5\x00\x2a\xb6\xf3\x99\x34"
-			"\x65\xee\xe9\x1c\x55\xa0\x3b\x07\x60\xc9\xc4\xe4"
-			"\xf7\x57\x5c\x34\x9f\xc6\x31\x30\x3f\x23\xb2\x89"
-			"\xc0\xe7\x50\xf3\xde\x59\xd1\x0e\xb3\x0f\x78\xcc"
-			"\x7e\x54\x5e\x61\xf6\x86\x3d\xb3\x11\x94\x36\x3e"
-			"\x61\x5c\x48\x99\xf6\x7b\x02\x9a\xdc\x6a\x28\xe6"
-			"\xd1\xa7\xd1\xa3",
-		.expectedlen = 256,
-		.addtla = (unsigned char *)
-			"\x6B\x0F\x4A\x48\x0B\x12\x85\xE4\x72\x23\x7F\x7F"
-			"\x94\x7C\x24\x69\x14\x9F\xDC\x72\xA6\x33\xAD\x3C"
-			"\x8C\x72\xC1\x88\x49\x59\x82\xC5",
-		.addtlb = (unsigned char *)
-			"\xC4\xAF\x36\x3D\xB8\x5D\x9D\xFA\x92\xF5\xC3\x3C"
-			"\x2D\x1E\x22\x2A\xBD\x8B\x05\x6F\xA3\xFC\xBF\x16"
-			"\xED\xAA\x75\x8D\x73\x9A\xF6\xEC",
-		.addtllen = 32,
-		.pers = NULL,
-		.perslen = 0,
-	}
-};
+		/* .entropy = ACVP entropyInput || nonce */
+		.entropy = "\x47\x28\x33\x05\x82\xE8\xA7\x58\x05\xBD\x07\x9E"
+			   "\xDB\x59\x87\x08\x3B\x82\x40\x70\xA1\xB7\x33\x4F"
+			   "\x0F\x53\xBE\xF6\x98\x75\xFA\x21\x00\x04\xBC\x58"
+			   "\x9B\xB1\xB3\xBC\x2A\xF6\xA4\x2A\xE1\xB3\x65\x24"
+			   "\xA8\xEE\x56\x70\x5E\x28\x06\x04\x80\xCE\xB7\x36"
+			   "\xA4\xD1\x4E\xAA\xE7\xE8\x51\x40\x7D\x30\xF7\x91"
+			   "\xF2\x9E\x98\xA7\xBB\xB3\x1D\xC6\x8A\x62\xC7\xE7"
+			   "\x1F\xE1\x0A\x52\x45\x39\x88\x92\x43\xA6\x5B\xC3"
+			   "\x9D\x60\xDE\xFE\x79\xEF\x87\x6F\xD5\x76\x51\x69"
+			   "\x7B\x4C\x3D\x08\x60\xFA\xDC\x00\x94\xEC\x0A\xC5"
+			   "\xE7\xCB\x30\x05\x9F\xCC\x07\x44\x07\x14\xF4\xF9"
+			   "\xBC\x7D\xC0\x66\x63\x5B\xD3\x2A\xA7\xF2\x1D\x75"
+			   "\x3F\x28\x5F\x4A\xEB\x63\xD7\x92\x5A\x9A\x7C\x3A"
+			   "\x56\xEA\xCB\xA9\xA0\xF9\x5B\x99\xC2\x7D\x31\xBF"
+			   "\x44\xC7\x64\x6B\x7E\x64\x60\x64\x05\x86\xF0\x6B"
+			   "\x76\xA7\x3E\x63\xAD\xB6\x76\xED\x3D\x05\x2E\xC7"
+			   "\xC7\x52\x8F\x07\x56\x23\xFC\x2F\x93\xD5\xF2\x29"
+			   "\xC6\x84\x48\xD1\x88\xB9\x3F\x84\xB2\x93\xFC\x2F"
+			   "\x00\xB1\xBC\xBE\xAC\xC4\x94\x36\xF1\x2B\xF0\xC7"
+			   "\x36\xE8\x1D\x45\x0A\xA2\xFE\x72\xD7\x3E\x15\x6A"
+			   "\x74\x23\xB2\xD1\x3C\x20\xF2\xB7\x19\xC5\xF0\x74"
+			   "\x69\x06\xD1\x92\x47\xC9\x68\xDC\xEE\xBC\xF4\x3A"
+			   "\x8B\x7C\xA4\xA4\x4A\x15\xF4\xF0\x26\x26\xC5\x7D"
+			   "\x57\xDC\x00\x59\x07\xDF\x84\x2A\x23\xA9\xC1\xAF"
+			   "\x47\x49\x88\xCC\x2F\x3B\x90\x39\xA0\x09\xAE\x3C"
+			   "\x79\xF9\x97\x9D\xEF\x13\x1D\x2B\xB6\x18\xDB\xCF"
+			   "\x95\x98\x17\x7F\xF6\x95\x44\x3A\x40\x90\x63\x16"
+			   "\x70\x38\x22\xC5\xD8\xE4\x57\xA0\x8E\x51\x6B\xED"
+			   "\x03\x18\xC6\x74\xFA\xCA\xF2\xD5\xB2\x5A\x0C\x02"
+			   "\x48\x9F\x70\x0B\x1C\x0A\xD1\xD6\x4F\x3F\x70\x3C"
+			   "\x1A\x62\x29\x63\xD0\x92\x76\x30\x7C\xB9\x54\xEC"
+			   "\xF6\x92\xA3\x11\x66\xB2\x56\xE0\x75\x3D\x27\x01",
+		.entropylen = 384,
+		.pers = "\x03\x75\x33\x7B\x6B\xE5\x14\xAF\x0E\x43\x33\xCC\xD3"
+			"\xF4\xF0\x8B\x8A\xCC\xFE\x95\xC1\x20\xB9\x56\x3D\x2C"
+			"\x34\xC7\xA1\x4C\xBC\xDC\x0A\x8D\xBE\x68\x28\x89\x2A"
+			"\x92\x10\x05\x4E\xF9\xE2\x51\xF7\x4F\xF4\x48\x76\xC9"
+			"\xCE\xA8\x6D\xC1\xDE\x5C\x05\x95\x77\x24\xB4\x5D\x3D"
+			"\x16\x1D\xCB\x43\xE5\xD3\x01\x96\xCC\x3E\x2E\xE1\x97"
+			"\xB3\x42\xC2\xAB\xEC\x56\xA9\xF9\xF2\x28\x59\x01\x4E"
+			"\xFC\x8A\xC0\x5E\x4E\x33\x30\x91\x96\x93\xBA\xFA\xE1"
+			"\x9F\x20\xC3\xC1\xD1\xC6\xC3\x2C\x59\x8F\x56\xD5\x73"
+			"\xE8\x8D\xCF\xAC\xB1\x85\x3E\xC7\x73\x21\x59\x8D\x0F"
+			"\xF4\xA4\xC4\x0C\xD2\x18\x81\x5F\x3E\x37\xC9\x9F\xD9"
+			"\xDE\x3F\x15\x6F\xAC\x1A\xEC\x5E\x28\xD5\x56\xA2\x65"
+			"\xD2\x7C\x41\xDE\x57\x8C\x60\xAB\xA5\x03\x42\xC8\x3C"
+			"\x70\xC2\xD1\x11\x62\x25\x4E\x17\x58\xBB\x08\x51\x79"
+			"\xE6\xED\xBA\x9A\xA7\x07\xD5\x0E\x89\x43\x31\x4F\x99"
+			"\xEE\xED\xF1\x36\xFA\x1E\xC0\x25\x01\xDA\x19\xC2\x7D"
+			"\x5D\x78\x74\x5C\xC0\x72\x2E\xF1\x7B\x4E\x9C\xDB\xFE"
+			"\x60\x89\x62\x71\x9C\xEA\xD3\xDF\x82\xC7\x8B\xB4\x7B"
+			"\x7B\x7F\x6A\x62\x3C\x57\x76\xA5\x67\x5F\x70\xFB\xE0"
+			"\x1B\x23\x79\x71\xB5\x7C\xEA\x76\x99",
+		.perslen = 256,
 
-static const struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\xc3\x5c\x2f\xa2\xa8\x9d\x52\xa1\x1f\xa3\x2a\xa9"
-			"\x6c\x95\xb8\xf1\xc9\xa8\xf9\xcb\x24\x5a\x8b\x40"
-			"\xf3\xa6\xe5\xa7\xfb\xd9\xd3\xc6\x8e\x27\x7b\xa9"
-			"\xac\x9b\xbb\x00",
-		.entropylen = 40,
-		.expected = (unsigned char *)
-			"\x8c\x2e\x72\xab\xfd\x9b\xb8\x28\x4d\xb7\x9e\x17"
-			"\xa4\x3a\x31\x46\xcd\x76\x94\xe3\x52\x49\xfc\x33"
-			"\x83\x91\x4a\x71\x17\xf4\x13\x68\xe6\xd4\xf1\x48"
-			"\xff\x49\xbf\x29\x07\x6b\x50\x15\xc5\x9f\x45\x79"
-			"\x45\x66\x2e\x3d\x35\x03\x84\x3f\x4a\xa5\xa3\xdf"
-			"\x9a\x9d\xf1\x0d",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	},
-};
+		.ent_reseed = "\xE0\x5B\x69\xC5\xF9\xD4\x5A\x1F\xC1\x71\x0C"
+			      "\x37\x6F\xFE\x31\xE0\xEF\x3E\x9B\xD5\x46\x43"
+			      "\xA2\x15\xFA\x86\x75\x1C\x95\x9F\xE6\xD6\x41"
+			      "\x50\x36\x0F\x67\x68\x4A\x72\x32\x85\xB0\xCB"
+			      "\xA4\x0D\x1F\xA2\x43\xCA\x30\xF1\x42\xF8\xC4"
+			      "\x17\x6A\xD1\x2D\x43\x46\x8C\xA5\x72\x3C\xB1"
+			      "\x44\x8E\xA1\x75\xAA\x20\x30\xDD\xCD\x0B\x5A"
+			      "\x0D\xE6\x28\xCA\xBC\xA9\x3B\x11\x23\x88\x85"
+			      "\xDF\xD8\x84\x11\x89\x7A\x4F\x00\x90\x0B\xD8"
+			      "\x8A\x35\xE9\x5F\xEC\x05\x42\x15\xCA\x7E\xA3"
+			      "\xEF\xEF\xE3\xB7\xBE\x96\x8C\x36\x7D\x15\x3C"
+			      "\x7A\x78\x00\x61\xAF\x62\x45\xA7\x4B\xAE\x4A"
+			      "\xFD\xC9\xE0\x89\x53\x36\x71\x8F\x38\xC4\x00"
+			      "\x57\xD9\xE6\xC1\xEC\xBF\xF7\x4B\x3F\x55\xD6"
+			      "\x54\x15\x72\xB2\x23\x76\xD1\x4E\x8E\x7E\xFB"
+			      "\xC8\xD6\xA4\xA4\x7E\x8E\xD8\x76\xD3\xEB\x90"
+			      "\x90\xEB\xB3\x84\x26\x34\x93\x36\xCA\xA9\x12"
+			      "\xD3\x6D\x84\x07\xE1\x0E\xE7\x63\x24\x54\x90"
+			      "\x42\x8C\x6B\xE7\x21\x53\x2D\x16\xE9\x68\xA9"
+			      "\xC2\x9A\x7A\x93\x31\x5E\x35\x72\xE2\x0F\x3D"
+			      "\x55\x41\x8C\xC5\xD1\xBB\xEE\x5F\x3C\x95\x52"
+			      "\x2A\x65\x47\xB6\x84\x6A\x05\x19\xD9\xE7\x68"
+			      "\x15\xFA\x49\xD9\x9D\xFD\x47\x9F\xB9\xA6\xBB"
+			      "\xA8\xD4\x22\x6C\x8B\x59\x47\xEF\xD1\xD0\xBF"
+			      "\xBC\xE4\x53\x31\x2F\xA5\xD4\x2C\x89\xCA\xE5"
+			      "\x29\x64\xF8\x51\x2A\x02\xDE\x31\x0D\x54\x0E"
+			      "\x0E\xB8\x78\xE6\x2A\x3E\x31\xA2\xA4\xCD\x91"
+			      "\x79\xF2\x9B\x04\xA0\xDB\x78\xAC\x0D\xB5\xCA"
+			      "\x60\x4C\x08\x96\xCA\x8A\xE9\x93\x9F\x62\x38"
+			      "\xAE",
+		.ent_reseed_len = 320,
+		.addtl_reseed = "\xD5\x46\x71\x38\x46\xE0\xC1\x27\x2C\xF1\x07"
+				"\xDD\x29\x73\xD7\xF4\x5A\x3A\xE0\x99\x98\x0A"
+				"\xEB\xA5\x9E\x2E\x49\x21\x7B\x18\xC0\xD8\x3D"
+				"\xEE\x49\xF9\x52\x7C\x55\x5E\x5E\x06\xC2\x7E"
+				"\x34\x3A\xA5\xB7\x78\x21\xC1\xB8\x8B\x4E\x07"
+				"\x1F\x3F\xC4\xB4\x9B\x3F\x20\x2D\xEE\x70\x3B"
+				"\x18\xCF\x2B\x50\x15\x1B\xF3\x6C\xE7\x9F\x2A"
+				"\x2B\x69\x53\xCA\x88\x41\xA9\x07\x31\x14\x55"
+				"\xBD\x75\x49\x68\x98\x48\xCA\x46\xA5\xEC\xAA"
+				"\x9D\x8E\x6F\x40\xBA\xCA\xC8\xE9\x9F\xA4\xC5"
+				"\x94\x89\x2C\x4D\xE4\x0A\x66\xD3\xDB\x1A\xD9"
+				"\xA7\x3F\x6B\x0A\xE0\xE4\xDC\x0B\x8B\x64\xBF"
+				"\xB4\x95\xA1\xAE\x92\xA8\x7C\xE2\x4B\xED\xFD"
+				"\xFD\xE0\x54\x48\x7C\x25\xD6\xD1\x0A\xAB\x5E"
+				"\x33\x2F\x1D\xD3\x62\x47\xAB\x35\x8C\xA1\xC2"
+				"\x75\xDD\x75\xCA\x09\xE1\x38\xDE\x87\x79\x0C"
+				"\xD7\x5D\x3E\x6D\x28\x8F\x12\x98\x2C\x26\x34"
+				"\x91\x0B\x6B\xB7\x73",
+		.addtl_reseed_len = 192,
 
-static const struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\x36\x40\x19\x40\xfa\x8b\x1f\xba\x91\xa1\x66\x1f"
-			"\x21\x1d\x78\xa0\xb9\x38\x9a\x74\xe5\xbc\xcf\xec"
-			"\xe8\xd7\x66\xaf\x1a\x6d\x3b\x14\x49\x6f\x25\xb0"
-			"\xf1\x30\x1b\x4f\x50\x1b\xe3\x03\x80\xa1\x37\xeb",
-		.entropylen = 48,
-		.expected = (unsigned char *)
-			"\x58\x62\xeb\x38\xbd\x55\x8d\xd9\x78\xa6\x96\xe6"
-			"\xdf\x16\x47\x82\xdd\xd8\x87\xe7\xe9\xa6\xc9\xf3"
-			"\xf1\xfb\xaf\xb7\x89\x41\xb5\x35\xa6\x49\x12\xdf"
-			"\xd2\x24\xc6\xdc\x74\x54\xe5\x25\x0b\x3d\x97\x16"
-			"\x5e\x16\x26\x0c\x2f\xaf\x1c\xc7\x73\x5c\xb7\x5f"
-			"\xb4\xf0\x7e\x1d",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	},
-};
+		.addtla = "\xC8\xE1\x03\x59\xEA\x9F\xD3\x1F\xCC\xCB\x5A\x3E"
+			  "\x23\xDB\x47\x75\x1B\x3A\x6C\x0B\xED\xE6\xC0\xB6"
+			  "\x4B\x82\x53\x5B\x6D\xEE\x2B\x97\x0F\x59\x07\xA4"
+			  "\xFE\xE2\x26\x4F\xFF\xF7\x46\xAF\x62\xC4\x42\xE8"
+			  "\x06\xFA\x1F\xAA\x66\xF3\xA4\xCF\x85\x02\x3C\x11"
+			  "\xEF\xD2\x16\x13\x7A\xBD\x03\x66\xC3\x4D\x6F\x3A"
+			  "\xE5\xD3\xFF\x92\xB6\x45\xB8\x88\x94\x42\x2E\x84"
+			  "\x02\xA5\x54\x12\xBF\xCB\xF1\x6E\x1C\x0C\x23\xFE"
+			  "\x8A\xFD\xF5\x1A\x4C\xD6\x2A\x34\x7F\x21\x50\xBB"
+			  "\x06\x64\x9A\x77\x70\x59\x1D\x12\xD1\xA0\x10\x70"
+			  "\xD7\x4F\x3C\xEE\x82\xEF\x5B\x35\x8F\x05\xF6\x51"
+			  "\xBC\x28\xE1\xB1\x54\xFF\x25\x12\x16\x2C\x62\xFC"
+			  "\xB9\x74\x5E\x78\x41\xB2\x04\x80\x4E\xB1\x43\x2E"
+			  "\x88\x7F\x34\xA4\x33\x66\xE8\x68\x60\x54\xEB\xCE"
+			  "\xD1\x5F\xC4\xD6\x21\x5E\x22\x1A\x7E\xC5\x6D\xB7"
+			  "\x93\x8F\x9C\x77\x6E\x56\xA5\xA8\xF7\xB2\x10\x19",
+		.addtlb = "\xB8\x1E\x9E\xE2\xB3\xE4\x04\x43\x51\x7B\x96\x32"
+			  "\xAC\x95\xF2\x1E\x5E\x7A\xBF\x68\xE3\x7A\xCE\xE5"
+			  "\x2B\x3B\xB0\x34\x7F\xE1\xB5\x37\x3C\x5B\xD2\x6B"
+			  "\x17\xF6\x18\xAF\x7C\x01\xA5\x84\xFE\x07\x0B\x46"
+			  "\xCC\x96\x61\xAB\xCD\x81\x95\x8A\x8A\xB6\xD8\xCC"
+			  "\x94\x80\x45\xF6\xFF\x95\x3A\xB8\xDC\xBF\xB4\xBF"
+			  "\x26\x8C\xE9\x79\x05\x58\x9C\x03\x08\x45\xB5\xC4"
+			  "\xD8\xB4\xCD\x76\x13\xAE\x2B\xCC\xB5\x3E\x3C\x65"
+			  "\x7A\xAA\xDC\x9B\x0B\x13\x8B\x82\x5A\x39\x60\x9A"
+			  "\xF7\xA8\x38\xB5\x83\x8E\x0B\xD1\xE6\xBF\xA1\xDC"
+			  "\x30\x45\xCC\x67\x41\x0F\xF9\x86\x27\xE0\x22\x3E"
+			  "\x0D\x1F\x92\x13\x16\x9B\x84\x79\x91\xFE\xBC\xDF"
+			  "\x15\x6C\xDC\xAE\xC5\x64\x46\x9C\xE8\xEA\xBB\x9D"
+			  "\x27\xA2\x1E\xDF\xB2\xF2\x98\xB4\x88\x4E\x22\x5F"
+			  "\x3C\xD0\x35\x2C\x5C\x89\xC7\xEC\xE3\xF3\x09\x60"
+			  "\xEF\xAC\x78\xBF\x94\xAA\x8C\x30\x7F\xCF\xE4\x4B",
+		.addtllen = 192,
 
-static const struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = {
-	{
-		.entropy = (unsigned char *)
-			"\x87\xe1\xc5\x32\x99\x7f\x57\xa3\x5c\x28\x6d\xe8"
-			"\x64\xbf\xf2\x64\xa3\x9e\x98\xdb\x6c\x10\x78\x7f",
-		.entropylen = 24,
-		.expected = (unsigned char *)
-			"\x2c\x14\x7e\x24\x11\x9a\xd8\xd4\xb2\xed\x61\xc1"
-			"\x53\xd0\x50\xc9\x24\xff\x59\x75\x15\xf1\x17\x3a"
-			"\x3d\xf4\x4b\x2c\x84\x28\xef\x89\x0e\xb9\xde\xf3"
-			"\xe4\x78\x04\xb2\xfd\x9b\x35\x7f\xe1\x3f\x8a\x3e"
-			"\x10\xc8\x67\x0a\xf9\xdf\x2d\x6c\x96\xfb\xb2\xb8"
-			"\xcb\x2d\xd6\xb0",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\x71\xbd\xce\x35\x42\x7d\x20\xbf\x58\xcf\x17\x74"
-			"\xce\x72\xd8\x33\x34\x50\x2d\x8f\x5b\x14\xc4\xdd",
-		.entropylen = 24,
-		.expected = (unsigned char *)
-			"\x97\x33\xe8\x20\x12\xe2\x7b\xa1\x46\x8f\xf2\x34"
-			"\xb3\xc9\xb6\x6b\x20\xb2\x4f\xee\x27\xd8\x0b\x21"
-			"\x8c\xff\x63\x73\x69\x29\xfb\xf3\x85\xcd\x88\x8e"
-			"\x43\x2c\x71\x8b\xa2\x55\xd2\x0f\x1d\x7f\xe3\xe1"
-			"\x2a\xa3\xe9\x2c\x25\x89\xc7\x14\x52\x99\x56\xcc"
-			"\xc3\xdf\xb3\x81",
-		.expectedlen = 64,
-		.addtla = (unsigned char *)
-			"\x66\xef\x42\xd6\x9a\x8c\x3d\x6d\x4a\x9e\x95\xa6"
-			"\x91\x4d\x81\x56",
-		.addtlb = (unsigned char *)
-			"\xe3\x18\x83\xd9\x4b\x5e\xc4\xcc\xaa\x61\x2f\xbb"
-			"\x4a\x55\xd1\xc6",
-		.addtllen = 16,
-		.pers = NULL,
-		.perslen = 0,
-	}, {
-		.entropy = (unsigned char *)
-			"\xca\x4b\x1e\xfa\x75\xbd\x69\x36\x38\x73\xb8\xf9"
-			"\xdb\x4d\x35\x0e\x47\xbf\x6c\x37\x72\xfd\xf7\xa9",
-		.entropylen = 24,
-		.expected = (unsigned char *)
-			"\x59\xc3\x19\x79\x1b\xb1\xf3\x0e\xe9\x34\xae\x6e"
-			"\x8b\x1f\xad\x1f\x74\xca\x25\x45\x68\xb8\x7f\x75"
-			"\x12\xf8\xf2\xab\x4c\x23\x01\x03\x05\xe1\x70\xee"
-			"\x75\xd8\xcb\xeb\x23\x4c\x7a\x23\x6e\x12\x27\xdb"
-			"\x6f\x7a\xac\x3c\x44\xb7\x87\x4b\x65\x56\x74\x45"
-			"\x34\x30\x0c\x3d",
-		.expectedlen = 64,
-		.addtla = NULL,
-		.addtlb = NULL,
-		.addtllen = 0,
-		.pers = (unsigned char *)
-			"\xeb\xaa\x60\x2c\x4d\xbe\x33\xff\x1b\xef\xbf\x0a"
-			"\x0b\xc6\x97\x54",
-		.perslen = 16,
-	}, {
-		.entropy = (unsigned char *)
-			"\xc0\x70\x1f\x92\x50\x75\x8f\xcd\xf2\xbe\x73\x98"
-			"\x80\xdb\x66\xeb\x14\x68\xb4\xa5\x87\x9c\x2d\xa6",
-		.entropylen = 24,
-		.expected = (unsigned char *)
-			"\x97\xc0\xc0\xe5\xa0\xcc\xf2\x4f\x33\x63\x48\x8a"
-			"\xdb\x13\x0a\x35\x89\xbf\x80\x65\x62\xee\x13\x95"
-			"\x7c\x33\xd3\x7d\xf4\x07\x77\x7a\x2b\x65\x0b\x5f"
-			"\x45\x5c\x13\xf1\x90\x77\x7f\xc5\x04\x3f\xcc\x1a"
-			"\x38\xf8\xcd\x1b\xbb\xd5\x57\xd1\x4a\x4c\x2e\x8a"
-			"\x2b\x49\x1e\x5c",
-		.expectedlen = 64,
-		.addtla = (unsigned char *)
-			"\xf9\x01\xf8\x16\x7a\x1d\xff\xde\x8e\x3c\x83\xe2"
-			"\x44\x85\xe7\xfe",
-		.addtlb = (unsigned char *)
-			"\x17\x1c\x09\x38\xc2\x38\x9f\x97\x87\x60\x55\xb4"
-			"\x82\x16\x62\x7f",
-		.addtllen = 16,
-		.pers = (unsigned char *)
-			"\x80\x08\xae\xe8\xe9\x69\x40\xc5\x08\x73\xc7\x9f"
-			"\x8e\xcf\xe0\x02",
-		.perslen = 16,
-	},
+		.expected = "\x93\xFF\x4F\x4A\xA7\xE0\xFC\x3B\xCF\xCE\x57\xCD"
+			    "\xFD\xE9\x94\x6D\x0D\x45\x7E\xA3\x98\xBB\xE6\x54"
+			    "\x36\xF9\x92\xE6\xBB\xC5\x93\x16\x69\x35\x01\xF0"
+			    "\x5A\xE2\x43\xFC\x73\x7C\xA3\x5C\x5B\x49\x72\x69"
+			    "\x6A\xC4\xB6\xD4\xB2\x48\x27\x72\xCA\xE3\xF3\xB8"
+			    "\xD6\x95\x9F\x39\xC8\xD7\x72\xAF\xC2\x7C\x75\xE5"
+			    "\xA4\xF7\x52\x4F\xDF\x80\x75\xCA\x4B\xFA\x75\x9A"
+			    "\x7A\x8D\x48\x94\x30\x1C\x82\x5E\xD6\x3F\xF1\x78"
+			    "\x84\xAA\x8D\x8B\x4F\x6A\x20\x12\xA2\x65\xC5\x02"
+			    "\x32\xA1\xE4\x6B\x86\x2F\xCF\x92\xCD\x01\x55\xF7"
+			    "\xB8\xAB\xC3\x2C\x42\xC3\xFD\x9B\x59\x2D\x47\x8E"
+			    "\xB6\x8D\xE7\x04\x2E\x67\xD9\x07\x14\xA3\xF2\x74"
+			    "\x79\x9C\xAB\xC6\xD6\x52\x3E\x91\x81\xA4\x3C\x9A"
+			    "\x3E\x10\x1E\xD0\x28\xBC\x1C\x50\xF0\xB6\xEC\x01"
+			    "\xE5\x40\x16\x08\xB5\x58\xE0\xC4\xC2\x9A\xBB\x13"
+			    "\xAB\x10\x4B\x07\x24\x09\xEB\xF9\x5E\xAA\x3E\x3A"
+			    "\x25\x34\x5B\x97\x92\xF9\x46\x8A\x8E\x5A\x06\x9C"
+			    "\x2B\x1C\x8E\x47\x19\xC3\x3B\x14\x8A\xC1\xEB\xF7"
+			    "\xB0\xA2\x05\xE0\x70\x12\x88\x21\xFF\xC0\x35\x6D"
+			    "\xE9\x08\x7E\x11\x6E\xD7\x16\x48\x24\x43\xAF\x33"
+			    "\xA3\x48\x44\x91\xC5\x32\x1D\x87\xF6\x9B\x10\x1C"
+			    "\xA4\x7A\x74\x23\xB3\xFC\xB9\x87\x08\xE6\x9A\xEF"
+			    "\xD5\x93\x82\x17\xD4\x78\xDA\x46\x34\xC2\x61\x8C"
+			    "\xB7\x53\x89\xC4\x8A\xD8\xA0\x2E\xCE\xCA\x24\x16"
+			    "\xF0\x18\x7F\xC3\x12\x3F\xD0\xB1\xEF\x37\x99\xA8"
+			    "\x04\x1F\x76\x50\x81\x08\xCD\x4D\x43\xEF\x64\xB6"
+			    "\x03\x57\xB7\x5F\xDD\xF6\xBA\x34\xE2\x88\xE7\x1F"
+			    "\x62\xA5\xA4\x05\xEB\x1A\xFE\x0B\xFE\x74\x7D\x64"
+			    "\x29\x83\xE8\x0B\x97\x7D\x94\x5B\xF9\x3C\x8E\xC8"
+			    "\x6B\xCD\xAD\x63\x32\x6E\x00\xBF\x8E\x57\xF2\x32"
+			    "\x88\x53\x17\x40\xE3\xF8\xF8\xD9\xFD\xCD\x76\x84"
+			    "\xC6\xFD\xB1\x10\x40\xED\x75\xDD\xBD\x78\x8B\x0A"
+			    "\x61\x1E\xCC\xC4\x4F\x95\xF1\x2F\x8F\x5A\xF0\x12"
+			    "\x00\xCE\x75\xB8\xE9\xAB\x63\xC0\x7A\xF6\x9A\x47"
+			    "\xD2\x3C\xB0\xAD\xAF\x03\xDF\x25\xDD\x0C\x0F\xB6"
+			    "\x29\x09\x0F\x28\x5E\xA8\x67\xBB\xC9\x78\xBE\x12"
+			    "\xD8\xD6\xD0\x98\x82\x29\xA3\x82\xFB\x8E\x1C\x79"
+			    "\x28\x52\x22\x73\x3C\x6F\xB5\x33\xFF\x40\x26\x4C"
+			    "\x30\xE4\xE0\x39\x59\x72\x65\x79\x9F\xB7\x89\x87"
+			    "\x64\xAC\x45\x11\x0D\xDD\xC0\x44\x5E\x0D\xDF\xDB"
+			    "\x91\x42\x5F\x52\x7F\xBB\x66\xEE\x32\x90\x8F\x6F"
+			    "\xF8\xB9\xFE\xEA\x42\xFB\x60\x2B\xE3\xD9\xB2\xD1"
+			    "\x74\x37\x14\xC6\xDA\x68\xE6\x09",
+		.expectedlen = 512,
+	}
 };
 
 /* Cast5 test vectors from RFC 2144 */
diff --git a/drivers/accel/amdxdna/Makefile b/drivers/accel/amdxdna/Makefile
index cf9bf19dedb9e..05cce0a38692b 100644
--- a/drivers/accel/amdxdna/Makefile
+++ b/drivers/accel/amdxdna/Makefile
@@ -1,14 +1,19 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
 amdxdna-y := \
+	aie.o \
+	aie_psp.o \
+	aie_smu.o \
 	aie2_ctx.o \
 	aie2_error.o \
 	aie2_message.o \
 	aie2_pci.o \
 	aie2_pm.o \
-	aie2_psp.o \
-	aie2_smu.o \
 	aie2_solver.o \
+	aie4_ctx.o \
+	aie4_message.o \
+	aie4_pci.o \
+	amdxdna_cbuf.o \
 	amdxdna_ctx.o \
 	amdxdna_gem.o \
 	amdxdna_iommu.o \
@@ -19,7 +24,11 @@ amdxdna-y := \
 	amdxdna_sysfs.o \
 	amdxdna_ubuf.o \
 	npu1_regs.o \
+	npu3_regs.o \
 	npu4_regs.o \
 	npu5_regs.o \
 	npu6_regs.o
+
+amdxdna-$(CONFIG_PCI_IOV) += aie4_sriov.o
+amdxdna-$(CONFIG_DEBUG_FS) += amdxdna_debugfs.o
 obj-$(CONFIG_DRM_ACCEL_AMDXDNA) = amdxdna.o
diff --git a/drivers/accel/amdxdna/aie.c b/drivers/accel/amdxdna/aie.c
new file mode 100644
index 0000000000000..dd6f36f222c78
--- /dev/null
+++ b/drivers/accel/amdxdna/aie.c
@@ -0,0 +1,167 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <linux/errno.h>
+
+#include "aie.h"
+#include "amdxdna_mailbox_helper.h"
+#include "amdxdna_mailbox.h"
+#include "amdxdna_pci_drv.h"
+
+void aie_dump_mgmt_chann_debug(struct aie_device *aie)
+{
+	struct amdxdna_dev *xdna = aie->xdna;
+
+	XDNA_DBG(xdna, "i2x tail    0x%x", aie->mgmt_i2x.mb_tail_ptr_reg);
+	XDNA_DBG(xdna, "i2x head    0x%x", aie->mgmt_i2x.mb_head_ptr_reg);
+	XDNA_DBG(xdna, "i2x ringbuf 0x%x", aie->mgmt_i2x.rb_start_addr);
+	XDNA_DBG(xdna, "i2x rsize   0x%x", aie->mgmt_i2x.rb_size);
+	XDNA_DBG(xdna, "x2i tail    0x%x", aie->mgmt_x2i.mb_tail_ptr_reg);
+	XDNA_DBG(xdna, "x2i head    0x%x", aie->mgmt_x2i.mb_head_ptr_reg);
+	XDNA_DBG(xdna, "x2i ringbuf 0x%x", aie->mgmt_x2i.rb_start_addr);
+	XDNA_DBG(xdna, "x2i rsize   0x%x", aie->mgmt_x2i.rb_size);
+	XDNA_DBG(xdna, "x2i chann index 0x%x", aie->mgmt_chan_idx);
+	XDNA_DBG(xdna, "mailbox protocol major 0x%x", aie->mgmt_prot_major);
+	XDNA_DBG(xdna, "mailbox protocol minor 0x%x", aie->mgmt_prot_minor);
+}
+
+void aie_destroy_chann(struct aie_device *aie, struct mailbox_channel **chann)
+{
+	struct amdxdna_dev *xdna = aie->xdna;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+
+	if (!*chann)
+		return;
+
+	xdna_mailbox_stop_channel(*chann);
+	xdna_mailbox_free_channel(*chann);
+	*chann = NULL;
+}
+
+int aie_send_mgmt_msg_wait(struct aie_device *aie, struct xdna_mailbox_msg *msg)
+{
+	struct amdxdna_dev *xdna = aie->xdna;
+	struct xdna_notify *hdl = msg->handle;
+	int ret;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+
+	if (!aie->mgmt_chann)
+		return -ENODEV;
+
+	ret = xdna_send_msg_wait(xdna, aie->mgmt_chann, msg);
+	if (ret == -ETIME)
+		aie_destroy_chann(aie, &aie->mgmt_chann);
+
+	if (!ret && *hdl->status) {
+		XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x",
+			 msg->opcode, *hdl->data);
+		ret = -EINVAL;
+	}
+
+	return ret;
+}
+
+int aie_check_protocol(struct aie_device *aie, u32 fw_major, u32 fw_minor)
+{
+	const struct amdxdna_fw_feature_tbl *feature;
+	bool found = false;
+
+	for (feature = aie->xdna->dev_info->fw_feature_tbl;
+	     feature->major; feature++) {
+		if (feature->major != fw_major)
+			continue;
+		if (fw_minor < feature->min_minor)
+			continue;
+		if (feature->max_minor > 0 && fw_minor > feature->max_minor)
+			continue;
+
+		aie->feature_mask |= feature->features;
+
+		/* firmware version matches one of the driver support entry */
+		found = true;
+	}
+
+	return found ? 0 : -EOPNOTSUPP;
+}
+
+static void amdxdna_update_vbnv(struct amdxdna_dev *xdna,
+				const struct amdxdna_rev_vbnv *tbl,
+				u32 rev)
+{
+	int i;
+
+	for (i = 0; tbl[i].vbnv; i++) {
+		if (tbl[i].revision == rev) {
+			xdna->vbnv = tbl[i].vbnv;
+			break;
+		}
+	}
+}
+
+void amdxdna_vbnv_init(struct amdxdna_dev *xdna)
+{
+	const struct amdxdna_dev_info *info = xdna->dev_info;
+	u32 rev;
+
+	xdna->vbnv = info->default_vbnv;
+
+	if (!info->ops->get_dev_revision || !info->rev_vbnv_tbl)
+		return;
+
+	if (info->ops->get_dev_revision(xdna, &rev))
+		return;
+
+	amdxdna_update_vbnv(xdna, info->rev_vbnv_tbl, rev);
+}
+
+int amdxdna_get_metadata(struct aie_device *aie,
+			 struct amdxdna_client *client,
+			 struct amdxdna_drm_get_info *args)
+{
+	int ret = 0;
+	u32 buf_sz;
+
+	buf_sz = min(args->buffer_size, sizeof(aie->metadata));
+	if (copy_to_user(u64_to_user_ptr(args->buffer), &aie->metadata, buf_sz))
+		ret = -EFAULT;
+
+	return ret;
+}
+
+void *amdxdna_alloc_msg_buffer(struct amdxdna_dev *xdna, u32 *size,
+			       dma_addr_t *dma_addr)
+{
+	void *vaddr;
+	int order;
+
+	*size = max_t(u32, *size, SZ_8K);
+	order = get_order(*size);
+	if (order > MAX_PAGE_ORDER)
+		return ERR_PTR(-EINVAL);
+	*size = PAGE_SIZE << order;
+
+	if (amdxdna_iova_on(xdna))
+		return amdxdna_iommu_alloc(xdna, *size, dma_addr);
+
+	vaddr = dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr,
+				      DMA_FROM_DEVICE, GFP_KERNEL);
+	if (!vaddr)
+		return ERR_PTR(-ENOMEM);
+
+	return vaddr;
+}
+
+void amdxdna_free_msg_buffer(struct amdxdna_dev *xdna, size_t size,
+			     void *cpu_addr, dma_addr_t dma_addr)
+{
+	if (amdxdna_iova_on(xdna)) {
+		amdxdna_iommu_free(xdna, size, cpu_addr, dma_addr);
+		return;
+	}
+
+	dma_free_noncoherent(xdna->ddev.dev, size, cpu_addr, dma_addr, DMA_FROM_DEVICE);
+}
diff --git a/drivers/accel/amdxdna/aie.h b/drivers/accel/amdxdna/aie.h
new file mode 100644
index 0000000000000..0483d582b7f8d
--- /dev/null
+++ b/drivers/accel/amdxdna/aie.h
@@ -0,0 +1,120 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+#ifndef _AIE_H_
+#define _AIE_H_
+
+#include <drm/amdxdna_accel.h>
+#include "amdxdna_pci_drv.h"
+#include "amdxdna_mailbox.h"
+
+#define AIE_INTERVAL	20000	/* us */
+#define AIE_TIMEOUT	1000000	/* us */
+
+struct psp_device;
+struct smu_device;
+
+struct aie_device {
+	struct amdxdna_dev *xdna;
+	struct mailbox_channel *mgmt_chann;
+	struct xdna_mailbox_chann_res mgmt_x2i;
+	struct xdna_mailbox_chann_res mgmt_i2x;
+	u32 mgmt_chan_idx;
+	u32 mgmt_prot_major;
+	u32 mgmt_prot_minor;
+	unsigned long feature_mask;
+
+	struct psp_device *psp_hdl;
+	struct smu_device *smu_hdl;
+
+	struct amdxdna_drm_query_aie_metadata metadata;
+};
+
+#define DECLARE_AIE_MSG(name, op) \
+	DECLARE_XDNA_MSG_COMMON(name, op, -1)
+#define AIE_FEATURE_ON(aie, feature) test_bit(feature, &(aie)->feature_mask)
+
+#define PSP_REG_BAR(ndev, idx) ((ndev)->priv->psp_regs_off[(idx)].bar_idx)
+#define PSP_REG_OFF(ndev, idx) ((ndev)->priv->psp_regs_off[(idx)].offset)
+
+#define SMU_REG_BAR(ndev, idx) ((ndev)->priv->smu_regs_off[(idx)].bar_idx)
+#define SMU_REG_OFF(ndev, idx) ((ndev)->priv->smu_regs_off[(idx)].offset)
+
+#define DEFINE_BAR_OFFSET(reg_name, bar, reg_addr) \
+	[reg_name] = {bar##_BAR_INDEX, (reg_addr) - bar##_BAR_BASE}
+
+enum smu_reg_idx {
+	SMU_CMD_REG = 0,
+	SMU_ARG_REG,
+	SMU_INTR_REG,
+	SMU_RESP_REG,
+	SMU_OUT_REG,
+	SMU_MAX_REGS /* Keep this at the end */
+};
+
+enum psp_reg_idx {
+	PSP_CMD_REG = 0,
+	PSP_ARG0_REG,
+	PSP_ARG1_REG,
+	PSP_ARG2_REG,
+	PSP_NUM_IN_REGS, /* number of input registers */
+	PSP_INTR_REG = PSP_NUM_IN_REGS,
+	PSP_STATUS_REG,
+	PSP_RESP_REG,
+	PSP_PWAITMODE_REG,
+	PSP_MAX_REGS /* Keep this at the end */
+};
+
+struct aie_bar_off_pair {
+	int	bar_idx;
+	u32	offset;
+};
+
+struct smu_config {
+	void __iomem    *smu_regs[SMU_MAX_REGS];
+};
+
+struct psp_config {
+	const void		*fw_buf;
+	u32			fw_size;
+	const void              *certfw_buf;
+	u32                     certfw_size;
+	void __iomem		*psp_regs[PSP_MAX_REGS];
+	u32			arg2_mask;
+	u32			notify_val;
+};
+
+/* Device revision to VBNV string mapping table entry */
+struct amdxdna_rev_vbnv {
+	u32		revision;
+	const char	*vbnv;
+};
+
+/* aie.c */
+void aie_dump_mgmt_chann_debug(struct aie_device *aie);
+void aie_destroy_chann(struct aie_device *aie, struct mailbox_channel **chann);
+int aie_send_mgmt_msg_wait(struct aie_device *aie, struct xdna_mailbox_msg *msg);
+int aie_check_protocol(struct aie_device *aie, u32 fw_major, u32 fw_minor);
+void amdxdna_vbnv_init(struct amdxdna_dev *xdna);
+int amdxdna_get_metadata(struct aie_device *aie, struct amdxdna_client *client,
+			 struct amdxdna_drm_get_info *args);
+void *amdxdna_alloc_msg_buffer(struct amdxdna_dev *xdna, u32 *size,
+			       dma_addr_t *dma_addr);
+void amdxdna_free_msg_buffer(struct amdxdna_dev *xdna, size_t size,
+			     void *cpu_addr, dma_addr_t dma_addr);
+
+/* aie_psp.c */
+struct psp_device *aiem_psp_create(struct drm_device *ddev, struct psp_config *conf);
+int aie_psp_start(struct psp_device *psp);
+void aie_psp_stop(struct psp_device *psp);
+int aie_psp_waitmode_poll(struct psp_device *psp);
+
+/* aie_smu.c */
+struct smu_device *aiem_smu_create(struct drm_device *ddev, struct smu_config *conf);
+int aie_smu_init(struct smu_device *smu);
+void aie_smu_fini(struct smu_device *smu);
+int aie_smu_set_clocks(struct smu_device *smu, u32 *npuclk, u32 *hclk);
+int aie_smu_set_dpm(struct smu_device *smu, u32 dpm_level);
+
+#endif /* _AIE_H_ */
diff --git a/drivers/accel/amdxdna/aie2_ctx.c b/drivers/accel/amdxdna/aie2_ctx.c
index 286379d9511d4..658a5fb1fda69 100644
--- a/drivers/accel/amdxdna/aie2_ctx.c
+++ b/drivers/accel/amdxdna/aie2_ctx.c
@@ -27,7 +27,9 @@ static bool force_cmdlist = true;
 module_param(force_cmdlist, bool, 0600);
 MODULE_PARM_DESC(force_cmdlist, "Force use command list (Default true)");
 
-#define HWCTX_MAX_TIMEOUT	60000 /* milliseconds */
+uint tdr_timeout_ms = 2000;
+module_param(tdr_timeout_ms, int, 0400);
+MODULE_PARM_DESC(tdr_timeout_ms, "TDR (Timeout Detection and Recovery) timeout in milliseconds (0 = disable)");
 
 struct aie2_ctx_health {
 	struct amdxdna_ctx_health header;
@@ -39,11 +41,30 @@ struct aie2_ctx_health {
 	u32 fatal_error_app_module;
 };
 
+static inline void aie2_tdr_signal(struct amdxdna_dev *xdna)
+{
+	WRITE_ONCE(xdna->dev_handle->tdr_status, AIE2_TDR_SIGNALED);
+}
+
+static bool aie2_tdr_detect(struct amdxdna_dev *xdna)
+{
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+
+	if (READ_ONCE(ndev->tdr_status) == AIE2_TDR_WAIT) {
+		XDNA_ERR(xdna, "TDR timeout detected");
+		return true;
+	}
+
+	WRITE_ONCE(ndev->tdr_status, AIE2_TDR_WAIT);
+	return false;
+}
+
 static void aie2_job_release(struct kref *ref)
 {
 	struct amdxdna_sched_job *job;
 
 	job = container_of(ref, struct amdxdna_sched_job, refcnt);
+
 	amdxdna_sched_job_cleanup(job);
 	atomic64_inc(&job->hwctx->job_free_cnt);
 	wake_up(&job->hwctx->priv->job_free_wq);
@@ -70,6 +91,7 @@ static void aie2_hwctx_stop(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwct
 static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hwctx)
 {
 	struct amdxdna_gem_obj *heap = hwctx->priv->heap;
+	unsigned long heap_id;
 	int ret;
 
 	ret = aie2_create_context(xdna->dev_handle, hwctx);
@@ -86,6 +108,17 @@ static int aie2_hwctx_restart(struct amdxdna_dev *xdna, struct amdxdna_hwctx *hw
 		goto out;
 	}
 
+	xa_for_each_range(&hwctx->client->dev_heap_xa, heap_id, heap, 1,
+			  hwctx->last_attached_heap) {
+		ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id,
+					amdxdna_obj_dma_addr(heap),
+					heap->mem.size);
+		if (ret) {
+			XDNA_ERR(xdna, "Add heap %ld failed ret %d", heap_id, ret);
+			goto out;
+		}
+	}
+
 	ret = aie2_config_cu(hwctx, NULL);
 	if (ret) {
 		XDNA_ERR(xdna, "Config cu failed, ret %d", ret);
@@ -175,8 +208,10 @@ aie2_sched_notify(struct amdxdna_sched_job *job)
 {
 	struct dma_fence *fence = job->fence;
 
-	trace_xdna_job(&job->base, job->hwctx->name, "signaled fence", job->seq);
+	trace_xdna_job(&job->base, job->hwctx->name, "signaling fence",
+		       job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO);
 
+	aie2_tdr_signal(job->hwctx->client->xdna);
 	job->hwctx->priv->completed++;
 	dma_fence_signal(fence);
 
@@ -345,6 +380,9 @@ aie2_sched_job_run(struct drm_sched_job *sched_job)
 	struct dma_fence *fence;
 	int ret;
 
+	trace_xdna_job(sched_job, hwctx->name, "job run",
+		       job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO);
+
 	if (!hwctx->priv->mbox_chann)
 		return NULL;
 
@@ -360,6 +398,7 @@ aie2_sched_job_run(struct drm_sched_job *sched_job)
 			ret = aie2_sync_bo(hwctx, job, aie2_sched_drvcmd_resp_handler);
 			break;
 		case ATTACH_DEBUG_BO:
+		case DETACH_DEBUG_BO:
 			ret = aie2_config_debug_bo(hwctx, job, aie2_sched_drvcmd_resp_handler);
 			break;
 		default:
@@ -384,8 +423,11 @@ aie2_sched_job_run(struct drm_sched_job *sched_job)
 		aie2_job_put(job);
 		mmput(job->mm);
 		fence = ERR_PTR(ret);
+	} else {
+		aie2_tdr_signal(hwctx->client->xdna);
 	}
-	trace_xdna_job(sched_job, hwctx->name, "sent to device", job->seq);
+	trace_xdna_job(sched_job, hwctx->name, "sent to device",
+		       job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO);
 
 	return fence;
 }
@@ -395,7 +437,8 @@ static void aie2_sched_job_free(struct drm_sched_job *sched_job)
 	struct amdxdna_sched_job *job = drm_job_to_xdna_job(sched_job);
 	struct amdxdna_hwctx *hwctx = job->hwctx;
 
-	trace_xdna_job(sched_job, hwctx->name, "job free", job->seq);
+	trace_xdna_job(sched_job, hwctx->name, "job free",
+		       job->seq, job->drv_cmd ? job->drv_cmd->opcode : DEFAULT_IO);
 	if (!job->job_done)
 		up(&hwctx->priv->job_sem);
 
@@ -413,10 +456,12 @@ aie2_sched_job_timedout(struct drm_sched_job *sched_job)
 	int ret;
 
 	xdna = hwctx->client->xdna;
-	trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq);
-	job->job_timeout = true;
 
-	mutex_lock(&xdna->dev_lock);
+	guard(mutex)(&xdna->dev_lock);
+
+	if (!aie2_tdr_detect(xdna))
+		return DRM_GPU_SCHED_STAT_NO_HANG;
+
 	report = kzalloc_obj(*report);
 	if (!report)
 		goto reset_hwctx;
@@ -428,10 +473,10 @@ aie2_sched_job_timedout(struct drm_sched_job *sched_job)
 		job->aie2_job_health = report;
 
 reset_hwctx:
+	job->job_timeout = true;
 	aie2_hwctx_stop(xdna, hwctx, sched_job);
 
 	aie2_hwctx_restart(xdna, hwctx);
-	mutex_unlock(&xdna->dev_lock);
 
 	return DRM_GPU_SCHED_STAT_RESET;
 }
@@ -456,12 +501,12 @@ static int aie2_hwctx_col_list(struct amdxdna_hwctx *hwctx)
 	}
 
 	ndev = xdna->dev_handle;
-	if (unlikely(!ndev->metadata.core.row_count)) {
+	if (unlikely(!ndev->aie.metadata.core.row_count)) {
 		XDNA_WARN(xdna, "Core tile row count is zero");
 		return -EINVAL;
 	}
 
-	hwctx->num_col = hwctx->num_tiles / ndev->metadata.core.row_count;
+	hwctx->num_col = hwctx->num_tiles / ndev->aie.metadata.core.row_count;
 	if (!hwctx->num_col || hwctx->num_col > ndev->total_col) {
 		XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col);
 		return -EINVAL;
@@ -513,22 +558,24 @@ static int aie2_alloc_resource(struct amdxdna_hwctx *hwctx)
 {
 	struct amdxdna_dev *xdna = hwctx->client->xdna;
 	struct alloc_requests *xrs_req;
+	u32 temporal_only_col = 0;
 	int ret;
 
-	if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) {
-		hwctx->num_unused_col = xdna->dev_handle->total_col - hwctx->num_col;
-		hwctx->num_col = xdna->dev_handle->total_col;
-		return aie2_create_context(xdna->dev_handle, hwctx);
-	}
-
 	xrs_req = kzalloc_obj(*xrs_req);
 	if (!xrs_req)
 		return -ENOMEM;
 
-	xrs_req->cdo.start_cols = hwctx->col_list;
-	xrs_req->cdo.cols_len = hwctx->col_list_len;
-	xrs_req->cdo.ncols = hwctx->num_col;
-	xrs_req->cdo.qos_cap.opc = hwctx->max_opc;
+	if (AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_TEMPORAL_ONLY)) {
+		xrs_req->cdo.start_cols = &temporal_only_col;
+		xrs_req->cdo.cols_len = 1;
+		xrs_req->cdo.ncols = xdna->dev_handle->total_col;
+	} else {
+		xrs_req->cdo.start_cols = hwctx->col_list;
+		xrs_req->cdo.cols_len = hwctx->col_list_len;
+		xrs_req->cdo.ncols = hwctx->num_col;
+	}
+	/* Use platform opc */
+	xrs_req->cdo.qos_cap.opc = xdna->dev_handle->priv->col_opc * hwctx->num_col;
 
 	xrs_req->rqos.gops = hwctx->qos.gops;
 	xrs_req->rqos.fps = hwctx->qos.fps;
@@ -552,15 +599,9 @@ static void aie2_release_resource(struct amdxdna_hwctx *hwctx)
 	struct amdxdna_dev *xdna = hwctx->client->xdna;
 	int ret;
 
-	if (AIE2_FEATURE_ON(xdna->dev_handle, AIE2_TEMPORAL_ONLY)) {
-		ret = aie2_destroy_context(xdna->dev_handle, hwctx);
-		if (ret && ret != -ENODEV)
-			XDNA_ERR(xdna, "Destroy temporal only context failed, ret %d", ret);
-	} else {
-		ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx);
-		if (ret)
-			XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret);
-	}
+	ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx);
+	if (ret)
+		XDNA_ERR(xdna, "Release AIE resource failed, ret %d", ret);
 }
 
 static int aie2_ctx_syncobj_create(struct amdxdna_hwctx *hwctx)
@@ -605,9 +646,8 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
 	struct amdxdna_dev *xdna = client->xdna;
 	const struct drm_sched_init_args args = {
 		.ops = &sched_ops,
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit = HWCTX_MAX_CMDS,
-		.timeout = msecs_to_jiffies(HWCTX_MAX_TIMEOUT),
+		.timeout = msecs_to_jiffies(tdr_timeout_ms),
 		.name = "amdxdna_js",
 		.dev = xdna->ddev.dev,
 	};
@@ -622,7 +662,7 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
 	hwctx->priv = priv;
 
 	mutex_lock(&client->mm_lock);
-	heap = client->dev_heap;
+	heap = xa_load(&client->dev_heap_xa, 0);
 	if (!heap) {
 		XDNA_ERR(xdna, "The client dev heap object not exist");
 		mutex_unlock(&client->mm_lock);
@@ -704,6 +744,12 @@ int aie2_hwctx_init(struct amdxdna_hwctx *hwctx)
 		goto release_resource;
 	}
 
+	ret = amdxdna_update_heap(client, hwctx);
+	if (ret) {
+		XDNA_ERR(xdna, "Update heap failed, ret %d", ret);
+		goto release_resource;
+	}
+
 	ret = aie2_ctx_syncobj_create(hwctx);
 	if (ret) {
 		XDNA_ERR(xdna, "Create syncobj failed, ret %d", ret);
@@ -1133,3 +1179,28 @@ void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo,
 	else if (ret == -ERESTARTSYS)
 		XDNA_DBG(xdna, "Wait for bo interrupted by signal");
 }
+
+int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx,
+			   struct amdxdna_gem_obj *heap)
+{
+	struct amdxdna_client *client = hwctx->client;
+	struct amdxdna_dev *xdna = client->xdna;
+	u64 addr;
+	int ret;
+
+	ret = amdxdna_pm_resume_get_locked(xdna);
+	if (ret)
+		return ret;
+
+	addr = amdxdna_obj_dma_addr(heap);
+	ret = aie2_add_host_buf(xdna->dev_handle, hwctx->fw_ctx_id,
+				addr, heap->mem.size);
+	if (ret) {
+		XDNA_ERR(xdna, "Add heap failed hwctx %s 0x%lx ret %d",
+			 hwctx->name, heap->mem.size, ret);
+	}
+
+	amdxdna_pm_suspend_put(xdna);
+
+	return ret;
+}
diff --git a/drivers/accel/amdxdna/aie2_error.c b/drivers/accel/amdxdna/aie2_error.c
index 58abb59b61535..babdac0157ab8 100644
--- a/drivers/accel/amdxdna/aie2_error.c
+++ b/drivers/accel/amdxdna/aie2_error.c
@@ -11,6 +11,7 @@
 #include <linux/kthread.h>
 #include <linux/kernel.h>
 
+#include "aie.h"
 #include "aie2_msg_priv.h"
 #include "aie2_pci.h"
 #include "amdxdna_error.h"
@@ -249,12 +250,12 @@ static u32 aie2_error_backtrack(struct amdxdna_dev_hdl *ndev, void *err_info, u3
 		enum aie_error_category cat;
 
 		cat = aie_get_error_category(err->row, err->event_id, err->mod_type);
-		XDNA_ERR(ndev->xdna, "Row: %d, Col: %d, module %d, event ID %d, category %d",
+		XDNA_ERR(ndev->aie.xdna, "Row: %d, Col: %d, module %d, event ID %d, category %d",
 			 err->row, err->col, err->mod_type,
 			 err->event_id, cat);
 
 		if (err->col >= 32) {
-			XDNA_WARN(ndev->xdna, "Invalid column number");
+			XDNA_WARN(ndev->aie.xdna, "Invalid column number");
 			break;
 		}
 
@@ -294,7 +295,7 @@ static void aie2_error_worker(struct work_struct *err_work)
 
 	e = container_of(err_work, struct async_event, work);
 
-	xdna = e->ndev->xdna;
+	xdna = e->ndev->aie.xdna;
 
 	if (e->resp.status == MAX_AIE2_STATUS_CODE)
 		return;
@@ -329,7 +330,7 @@ static void aie2_error_worker(struct work_struct *err_work)
 
 void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev)
 {
-	struct amdxdna_dev *xdna = ndev->xdna;
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	struct async_events *events;
 
 	events = ndev->async_events;
@@ -338,13 +339,13 @@ void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev)
 	destroy_workqueue(events->wq);
 	mutex_lock(&xdna->dev_lock);
 
-	aie2_free_msg_buffer(ndev, events->size, events->buf, events->addr);
+	amdxdna_free_msg_buffer(xdna, events->size, events->buf, events->addr);
 	kfree(events);
 }
 
 int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev)
 {
-	struct amdxdna_dev *xdna = ndev->xdna;
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	u32 total_col = ndev->total_col;
 	u32 total_size = ASYNC_BUF_SIZE * total_col;
 	struct async_events *events;
@@ -354,7 +355,7 @@ int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev)
 	if (!events)
 		return -ENOMEM;
 
-	events->buf = aie2_alloc_msg_buffer(ndev, &total_size, &events->addr);
+	events->buf = amdxdna_alloc_msg_buffer(xdna, &total_size, &events->addr);
 	if (IS_ERR(events->buf)) {
 		ret = PTR_ERR(events->buf);
 		goto free_events;
@@ -394,7 +395,7 @@ int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev)
 free_wq:
 	destroy_workqueue(events->wq);
 free_buf:
-	aie2_free_msg_buffer(ndev, events->size, events->buf, events->addr);
+	amdxdna_free_msg_buffer(xdna, events->size, events->buf, events->addr);
 free_events:
 	kfree(events);
 	return ret;
@@ -402,12 +403,15 @@ int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev)
 
 int aie2_get_array_async_error(struct amdxdna_dev_hdl *ndev, struct amdxdna_drm_get_array *args)
 {
-	struct amdxdna_dev *xdna = ndev->xdna;
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 
 	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
 
+	if (!args->num_element)
+		return -EINVAL;
+
 	args->num_element = 1;
-	args->element_size = sizeof(ndev->last_async_err);
+	args->element_size = min(args->element_size, sizeof(ndev->last_async_err));
 	if (copy_to_user(u64_to_user_ptr(args->buffer),
 			 &ndev->last_async_err, args->element_size))
 		return -EFAULT;
diff --git a/drivers/accel/amdxdna/aie2_message.c b/drivers/accel/amdxdna/aie2_message.c
index a1c546c3e81c3..c4b364801cc04 100644
--- a/drivers/accel/amdxdna/aie2_message.c
+++ b/drivers/accel/amdxdna/aie2_message.c
@@ -16,6 +16,7 @@
 #include <linux/types.h>
 #include <linux/xarray.h>
 
+#include "aie.h"
 #include "aie2_msg_priv.h"
 #include "aie2_pci.h"
 #include "amdxdna_ctx.h"
@@ -24,103 +25,40 @@
 #include "amdxdna_mailbox_helper.h"
 #include "amdxdna_pci_drv.h"
 
-#define DECLARE_AIE2_MSG(name, op) \
-	DECLARE_XDNA_MSG_COMMON(name, op, MAX_AIE2_STATUS_CODE)
-
 #define EXEC_MSG_OPS(xdna)	((xdna)->dev_handle->exec_msg_ops)
 
-static int aie2_send_mgmt_msg_wait(struct amdxdna_dev_hdl *ndev,
-				   struct xdna_mailbox_msg *msg)
-{
-	struct amdxdna_dev *xdna = ndev->xdna;
-	struct xdna_notify *hdl = msg->handle;
-	int ret;
-
-	if (!ndev->mgmt_chann)
-		return -ENODEV;
-
-	ret = xdna_send_msg_wait(xdna, ndev->mgmt_chann, msg);
-	if (ret == -ETIME)
-		aie2_destroy_mgmt_chann(ndev);
-
-	if (!ret && *hdl->status != AIE2_STATUS_SUCCESS) {
-		XDNA_ERR(xdna, "command opcode 0x%x failed, status 0x%x",
-			 msg->opcode, *hdl->data);
-		ret = -EINVAL;
-	}
-
-	return ret;
-}
-
-void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size,
-			    dma_addr_t *dma_addr)
-{
-	struct amdxdna_dev *xdna = ndev->xdna;
-	void *vaddr;
-	int order;
-
-	*size = max(*size, SZ_8K);
-	order = get_order(*size);
-	if (order > MAX_PAGE_ORDER)
-		return ERR_PTR(-EINVAL);
-	*size = PAGE_SIZE << order;
-
-	if (amdxdna_iova_on(xdna))
-		return amdxdna_iommu_alloc(xdna, *size, dma_addr);
-
-	vaddr = dma_alloc_noncoherent(xdna->ddev.dev, *size, dma_addr,
-				      DMA_FROM_DEVICE, GFP_KERNEL);
-	if (!vaddr)
-		return ERR_PTR(-ENOMEM);
-
-	return vaddr;
-}
-
-void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size,
-			  void *cpu_addr, dma_addr_t dma_addr)
-{
-	struct amdxdna_dev *xdna = ndev->xdna;
-
-	if (amdxdna_iova_on(xdna)) {
-		amdxdna_iommu_free(xdna, size, cpu_addr, dma_addr);
-		return;
-	}
-
-	dma_free_noncoherent(xdna->ddev.dev, size, cpu_addr, dma_addr, DMA_FROM_DEVICE);
-}
-
 int aie2_suspend_fw(struct amdxdna_dev_hdl *ndev)
 {
-	DECLARE_AIE2_MSG(suspend, MSG_OP_SUSPEND);
+	DECLARE_AIE_MSG(suspend, MSG_OP_SUSPEND);
 	int ret;
 
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Failed to suspend fw, ret %d", ret);
+		XDNA_ERR(ndev->aie.xdna, "Failed to suspend fw, ret %d", ret);
 		return ret;
 	}
 
-	return aie2_psp_waitmode_poll(ndev->psp_hdl);
+	return aie_psp_waitmode_poll(ndev->aie.psp_hdl);
 }
 
 int aie2_resume_fw(struct amdxdna_dev_hdl *ndev)
 {
-	DECLARE_AIE2_MSG(suspend, MSG_OP_RESUME);
+	DECLARE_AIE_MSG(suspend, MSG_OP_RESUME);
 
-	return aie2_send_mgmt_msg_wait(ndev, &msg);
+	return aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 }
 
 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value)
 {
-	DECLARE_AIE2_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG);
+	DECLARE_AIE_MSG(set_runtime_cfg, MSG_OP_SET_RUNTIME_CONFIG);
 	int ret;
 
 	req.type = type;
 	req.value = value;
 
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Failed to set runtime config, ret %d", ret);
+		XDNA_ERR(ndev->aie.xdna, "Failed to set runtime config, ret %d", ret);
 		return ret;
 	}
 
@@ -129,13 +67,13 @@ int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value)
 
 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value)
 {
-	DECLARE_AIE2_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG);
+	DECLARE_AIE_MSG(get_runtime_cfg, MSG_OP_GET_RUNTIME_CONFIG);
 	int ret;
 
 	req.type = type;
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Failed to get runtime config, ret %d", ret);
+		XDNA_ERR(ndev->aie.xdna, "Failed to get runtime config, ret %d", ret);
 		return ret;
 	}
 
@@ -145,20 +83,21 @@ int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value)
 
 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid)
 {
-	DECLARE_AIE2_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID);
+	DECLARE_AIE_MSG(assign_mgmt_pasid, MSG_OP_ASSIGN_MGMT_PASID);
 
 	req.pasid = pasid;
 
-	return aie2_send_mgmt_msg_wait(ndev, &msg);
+	return aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 }
 
-int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version)
+int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev,
+			   struct amdxdna_drm_query_aie_version *version)
 {
-	DECLARE_AIE2_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION);
-	struct amdxdna_dev *xdna = ndev->xdna;
+	DECLARE_AIE_MSG(aie_version_info, MSG_OP_QUERY_AIE_VERSION);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	int ret;
 
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret)
 		return ret;
 
@@ -171,16 +110,17 @@ int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *ver
 	return 0;
 }
 
-int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata)
+int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev,
+			    struct amdxdna_drm_query_aie_metadata *metadata)
 {
-	DECLARE_AIE2_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO);
+	DECLARE_AIE_MSG(aie_tile_info, MSG_OP_QUERY_AIE_TILE_INFO);
 	int ret;
 
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret)
 		return ret;
 
-	metadata->size = resp.info.size;
+	metadata->col_size = resp.info.size;
 	metadata->cols = resp.info.cols;
 	metadata->rows = resp.info.rows;
 
@@ -211,10 +151,10 @@ int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *m
 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
 				struct amdxdna_fw_ver *fw_ver)
 {
-	DECLARE_AIE2_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION);
+	DECLARE_AIE_MSG(firmware_version, MSG_OP_GET_FIRMWARE_VERSION);
 	int ret;
 
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret)
 		return ret;
 
@@ -228,12 +168,12 @@ int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
 
 static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id)
 {
-	DECLARE_AIE2_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT);
-	struct amdxdna_dev *xdna = ndev->xdna;
+	DECLARE_AIE_MSG(destroy_ctx, MSG_OP_DESTROY_CONTEXT);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	int ret;
 
 	req.context_id = id;
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret && ret != -ENODEV)
 		XDNA_WARN(xdna, "Destroy context failed, ret %d", ret);
 	else if (ret == -ENODEV)
@@ -245,7 +185,7 @@ static int aie2_destroy_context_req(struct amdxdna_dev_hdl *ndev, u32 id)
 static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev,
 				     struct amdxdna_hwctx *hwctx)
 {
-	if (!AIE2_FEATURE_ON(ndev, AIE2_PREEMPT))
+	if (!AIE_FEATURE_ON(&ndev->aie, AIE2_PREEMPT))
 		return PRIORITY_HIGH;
 
 	switch (hwctx->qos.priority) {
@@ -264,8 +204,8 @@ static u32 aie2_get_context_priority(struct amdxdna_dev_hdl *ndev,
 
 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
 {
-	DECLARE_AIE2_MSG(create_ctx, MSG_OP_CREATE_CONTEXT);
-	struct amdxdna_dev *xdna = ndev->xdna;
+	DECLARE_AIE_MSG(create_ctx, MSG_OP_CREATE_CONTEXT);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	struct xdna_mailbox_chann_res x2i;
 	struct xdna_mailbox_chann_res i2x;
 	struct cq_pair *cq_pair;
@@ -280,7 +220,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
 	req.pasid = amdxdna_pasid_on(hwctx->client) ? hwctx->client->pasid : 0;
 	req.context_priority = aie2_get_context_priority(ndev, hwctx);
 
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret)
 		return ret;
 
@@ -344,7 +284,7 @@ int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwct
 
 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx)
 {
-	struct amdxdna_dev *xdna = ndev->xdna;
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	int ret;
 
 	if (!hwctx->priv->mbox_chann)
@@ -361,25 +301,59 @@ int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwc
 	return ret;
 }
 
-int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size)
+static int aie2_send_host_buf_msgs(struct amdxdna_dev_hdl *ndev, u32 context_id,
+				   u64 addr, u64 size, u32 initial_opcode)
 {
-	DECLARE_AIE2_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER);
-	struct amdxdna_dev *xdna = ndev->xdna;
+	DECLARE_AIE_MSG(map_host_buffer, MSG_OP_MAP_HOST_BUFFER);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	size_t chunk_size;
 	int ret;
 
-	req.context_id = context_id;
-	req.buf_addr = addr;
-	req.buf_size = size;
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
-	if (ret)
-		return ret;
+	chunk_size = xdna->dev_info->dev_mem_size;
+	if (!size || !IS_ALIGNED(size, chunk_size)) {
+		XDNA_ERR(xdna, "Invalid size 0x%llx for chunk 0x%lx",
+			 size, chunk_size);
+		return -EINVAL;
+	}
 
-	XDNA_DBG(xdna, "fw ctx %d map host buf addr 0x%llx size 0x%llx",
-		 context_id, addr, size);
+	msg.opcode = initial_opcode;
+	do {
+		req.context_id = context_id;
+		req.buf_addr = addr;
+		req.buf_size = chunk_size;
+		ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+		if (ret) {
+			XDNA_ERR(xdna, "fw ctx %d addr 0x%llx size 0x%lx",
+				 context_id, addr, chunk_size);
+			return ret;
+		}
+
+		XDNA_DBG(xdna, "fw ctx %d host buf op 0x%x addr 0x%llx size 0x%lx",
+			 context_id, msg.opcode, addr, chunk_size);
+
+		addr += chunk_size;
+		size -= chunk_size;
+		msg.opcode = MSG_OP_ADD_HOST_BUFFER;
+	} while (size);
 
 	return 0;
 }
 
+int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size)
+{
+	return aie2_send_host_buf_msgs(ndev, context_id, addr, size,
+				       MSG_OP_MAP_HOST_BUFFER);
+}
+
+int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size)
+{
+	if (!AIE_FEATURE_ON(&ndev->aie, AIE2_ADD_HOST_BUFFER))
+		return -EOPNOTSUPP;
+
+	return aie2_send_host_buf_msgs(ndev, context_id, addr, size,
+				       MSG_OP_ADD_HOST_BUFFER);
+}
+
 static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg)
 {
 	u32 *bitmap = arg;
@@ -392,15 +366,16 @@ static int amdxdna_hwctx_col_map(struct amdxdna_hwctx *hwctx, void *arg)
 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
 		      u32 size, u32 *cols_filled)
 {
-	DECLARE_AIE2_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS);
-	struct amdxdna_dev *xdna = ndev->xdna;
-	u32 buf_sz = size, aie_bitmap = 0;
+	DECLARE_AIE_MSG(aie_column_info, MSG_OP_QUERY_COL_STATUS);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	u32 buf_sz, aie_bitmap = 0;
 	struct amdxdna_client *client;
 	dma_addr_t dma_addr;
 	u8 *buff_addr;
 	int ret;
 
-	buff_addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
+	buf_sz = ndev->aie.metadata.cols * ndev->aie.metadata.col_size;
+	buff_addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr);
 	if (IS_ERR(buff_addr))
 		return PTR_ERR(buff_addr);
 
@@ -414,8 +389,8 @@ int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
 	req.num_cols = hweight32(aie_bitmap);
 	req.aie_bitmap = aie_bitmap;
 
-	drm_clflush_virt_range(buff_addr, size); /* device can access */
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	drm_clflush_virt_range(buff_addr, req.dump_buff_size); /* device can access */
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret) {
 		XDNA_ERR(xdna, "Error during NPU query, status %d", ret);
 		goto fail;
@@ -423,13 +398,14 @@ int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
 
 	XDNA_DBG(xdna, "Query NPU status completed");
 
-	if (size < resp.size) {
+	if (buf_sz < resp.size) {
 		ret = -EINVAL;
-		XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size);
+		XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size);
 		goto fail;
 	}
 
-	if (copy_to_user(buf, buff_addr, resp.size)) {
+	size = min(size, resp.size);
+	if (copy_to_user(buf, buff_addr, size)) {
 		ret = -EFAULT;
 		XDNA_ERR(xdna, "Failed to copy NPU status to user space");
 		goto fail;
@@ -438,7 +414,7 @@ int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf,
 	*cols_filled = aie_bitmap;
 
 fail:
-	aie2_free_msg_buffer(ndev, buf_sz, buff_addr, dma_addr);
+	amdxdna_free_msg_buffer(xdna, buf_sz, buff_addr, dma_addr);
 	return ret;
 }
 
@@ -446,17 +422,18 @@ int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
 			 char __user *buf, u32 size,
 			 struct amdxdna_drm_query_telemetry_header *header)
 {
-	DECLARE_AIE2_MSG(get_telemetry, MSG_OP_GET_TELEMETRY);
-	struct amdxdna_dev *xdna = ndev->xdna;
+	DECLARE_AIE_MSG(get_telemetry, MSG_OP_GET_TELEMETRY);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	dma_addr_t dma_addr;
-	u32 buf_sz = size;
+	u32 buf_sz;
 	u8 *addr;
 	int ret;
 
 	if (header->type >= MAX_TELEMETRY_TYPE)
 		return -EINVAL;
 
-	addr = aie2_alloc_msg_buffer(ndev, &buf_sz, &dma_addr);
+	buf_sz = min(size, SZ_4M);
+	addr = amdxdna_alloc_msg_buffer(xdna, &buf_sz, &dma_addr);
 	if (IS_ERR(addr))
 		return PTR_ERR(addr);
 
@@ -464,20 +441,21 @@ int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
 	req.buf_size = buf_sz;
 	req.type = header->type;
 
-	drm_clflush_virt_range(addr, size); /* device can access */
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	drm_clflush_virt_range(addr, req.buf_size); /* device can access */
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret) {
 		XDNA_ERR(xdna, "Query telemetry failed, status %d", ret);
 		goto free_buf;
 	}
 
-	if (size < resp.size) {
+	if (buf_sz < resp.size) {
 		ret = -EINVAL;
-		XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", size, resp.size);
+		XDNA_ERR(xdna, "Bad buffer size. Available: %u. Needs: %u", buf_sz, resp.size);
 		goto free_buf;
 	}
 
-	if (copy_to_user(buf, addr, resp.size)) {
+	size = min(size, resp.size);
+	if (copy_to_user(buf, addr, size)) {
 		ret = -EFAULT;
 		XDNA_ERR(xdna, "Failed to copy telemetry to user space");
 		goto free_buf;
@@ -487,7 +465,7 @@ int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
 	header->minor = resp.minor;
 
 free_buf:
-	aie2_free_msg_buffer(ndev, buf_sz, addr, dma_addr);
+	amdxdna_free_msg_buffer(xdna, buf_sz, addr, dma_addr);
 	return ret;
 }
 
@@ -506,8 +484,8 @@ int aie2_register_asyn_event_msg(struct amdxdna_dev_hdl *ndev, dma_addr_t addr,
 	req.buf_addr = addr;
 	req.buf_size = size;
 
-	XDNA_DBG(ndev->xdna, "Register addr 0x%llx size 0x%x", addr, size);
-	return xdna_mailbox_send_msg(ndev->mgmt_chann, &msg, TX_TIMEOUT);
+	XDNA_DBG(ndev->aie.xdna, "Register addr 0x%llx size 0x%x", addr, size);
+	return xdna_mailbox_send_msg(ndev->aie.mgmt_chann, &msg, TX_TIMEOUT);
 }
 
 int aie2_config_cu(struct amdxdna_hwctx *hwctx,
@@ -866,7 +844,6 @@ static int aie2_init_exec_req(void *req, struct amdxdna_gem_obj *cmd_abo,
 	int ret;
 	u32 op;
 
-
 	op = amdxdna_cmd_get_op(cmd_abo);
 	switch (op) {
 	case ERT_START_CU:
@@ -915,12 +892,12 @@ aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo,
 		ret = EXEC_MSG_OPS(xdna)->fill_dpu_slot(cmd_abo, slot, size);
 		break;
 	case ERT_START_NPU_PREEMPT:
-		if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT))
+		if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT))
 			return -EOPNOTSUPP;
 		ret = EXEC_MSG_OPS(xdna)->fill_preempt_slot(cmd_abo, slot, size);
 		break;
 	case ERT_START_NPU_PREEMPT_ELF:
-		if (!AIE2_FEATURE_ON(xdna->dev_handle, AIE2_PREEMPT))
+		if (!AIE_FEATURE_ON(&xdna->dev_handle->aie, AIE2_PREEMPT))
 			return -EOPNOTSUPP;
 		ret = EXEC_MSG_OPS(xdna)->fill_elf_slot(cmd_abo, slot, size);
 		break;
@@ -935,26 +912,12 @@ aie2_cmdlist_fill_slot(void *slot, struct amdxdna_gem_obj *cmd_abo,
 
 void aie2_msg_init(struct amdxdna_dev_hdl *ndev)
 {
-	if (AIE2_FEATURE_ON(ndev, AIE2_NPU_COMMAND))
+	if (AIE_FEATURE_ON(&ndev->aie, AIE2_NPU_COMMAND))
 		ndev->exec_msg_ops = &npu_exec_message_ops;
 	else
 		ndev->exec_msg_ops = &legacy_exec_message_ops;
 }
 
-void aie2_destroy_mgmt_chann(struct amdxdna_dev_hdl *ndev)
-{
-	struct amdxdna_dev *xdna = ndev->xdna;
-
-	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
-
-	if (!ndev->mgmt_chann)
-		return;
-
-	xdna_mailbox_stop_channel(ndev->mgmt_chann);
-	xdna_mailbox_free_channel(ndev->mgmt_chann);
-	ndev->mgmt_chann = NULL;
-}
-
 static inline struct amdxdna_gem_obj *
 aie2_cmdlist_get_cmd_buf(struct amdxdna_sched_job *job)
 {
@@ -1199,20 +1162,20 @@ int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *
 int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
 			  struct app_health_report *report)
 {
-	DECLARE_AIE2_MSG(get_app_health, MSG_OP_GET_APP_HEALTH);
-	struct amdxdna_dev *xdna = ndev->xdna;
+	DECLARE_AIE_MSG(get_app_health, MSG_OP_GET_APP_HEALTH);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	struct app_health_report *buf;
 	dma_addr_t dma_addr;
 	u32 buf_size;
 	int ret;
 
-	if (!AIE2_FEATURE_ON(ndev, AIE2_APP_HEALTH)) {
+	if (!AIE_FEATURE_ON(&ndev->aie, AIE2_APP_HEALTH)) {
 		XDNA_DBG(xdna, "App health feature not supported");
 		return -EOPNOTSUPP;
 	}
 
 	buf_size = sizeof(*report);
-	buf = aie2_alloc_msg_buffer(ndev, &buf_size, &dma_addr);
+	buf = amdxdna_alloc_msg_buffer(xdna, &buf_size, &dma_addr);
 	if (IS_ERR(buf)) {
 		XDNA_ERR(xdna, "Failed to allocate buffer for app health");
 		return PTR_ERR(buf);
@@ -1222,8 +1185,8 @@ int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
 	req.context_id = context_id;
 	req.buf_size = buf_size;
 
-	drm_clflush_virt_range(buf, sizeof(*report));
-	ret = aie2_send_mgmt_msg_wait(ndev, &msg);
+	drm_clflush_virt_range(buf, req.buf_size);
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
 	if (ret) {
 		XDNA_ERR(xdna, "Get app health failed, ret %d status 0x%x", ret, resp.status);
 		goto free_buf;
@@ -1233,6 +1196,75 @@ int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
 	memcpy(report, buf, sizeof(*report));
 
 free_buf:
-	aie2_free_msg_buffer(ndev, buf_size, buf, dma_addr);
+	amdxdna_free_msg_buffer(xdna, buf_size, buf, dma_addr);
 	return ret;
 }
+
+static int
+aie2_runtime_update_ctx_prop(struct amdxdna_dev_hdl *ndev,
+			     struct amdxdna_hwctx *ctx, u32 type, u32 value)
+{
+	DECLARE_AIE_MSG(update_property, MSG_OP_UPDATE_PROPERTY);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	int ret;
+
+	if (!AIE_FEATURE_ON(&ndev->aie, AIE2_UPDATE_PROPERTY))
+		return -EOPNOTSUPP;
+
+	if (ctx)
+		req.context_id = ctx->fw_ctx_id;
+	else
+		req.context_id = AIE2_UPDATE_PROPERTY_ALL_CTX;
+
+	req.time_quota_us = value;
+	req.type = type;
+
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret) {
+		XDNA_ERR(xdna, "%s update property failed, type %d ret %d",
+			 ctx ? ctx->name : "ctx.all", type, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	int ret;
+
+	ret = aie2_runtime_update_ctx_prop(ndev, NULL, UPDATE_PROPERTY_TIME_QUOTA, us);
+	if (ret == -EOPNOTSUPP) {
+		XDNA_DBG(xdna, "update time quota not support, skipped");
+		ret = 0;
+	} else if (!ret) {
+		XDNA_DBG(xdna, "Ctx exec time quantum updated to %u us", us);
+	}
+	return ret;
+}
+
+int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev)
+{
+	DECLARE_AIE_MSG(get_dev_revision, MSG_OP_GET_DEV_REVISION);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	int ret;
+
+	if (!AIE_FEATURE_ON(&ndev->aie, AIE2_GET_DEV_REVISION))
+		return -EOPNOTSUPP;
+
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret)
+		return ret;
+
+	*rev = resp.rev;
+
+	if (*rev < AIE2_DEV_REVISION_STXA || *rev >= AIE2_DEV_REVISION_UNKN) {
+		XDNA_ERR(xdna, "Unknown device revision: %d (raw fuse: 0x%x)",
+			 *rev, resp.raw_fuse_data);
+		return -EINVAL;
+	}
+
+	XDNA_DBG(xdna, "Device revision: %d (raw fuse: 0x%x)", *rev, resp.raw_fuse_data);
+	return 0;
+}
diff --git a/drivers/accel/amdxdna/aie2_msg_priv.h b/drivers/accel/amdxdna/aie2_msg_priv.h
index f18e89a39e357..fd65a4236d491 100644
--- a/drivers/accel/amdxdna/aie2_msg_priv.h
+++ b/drivers/accel/amdxdna/aie2_msg_priv.h
@@ -31,7 +31,10 @@ enum aie2_msg_opcode {
 	MSG_OP_SET_RUNTIME_CONFIG          = 0x10A,
 	MSG_OP_GET_RUNTIME_CONFIG          = 0x10B,
 	MSG_OP_REGISTER_ASYNC_EVENT_MSG    = 0x10C,
+	MSG_OP_UPDATE_PROPERTY             = 0x113,
 	MSG_OP_GET_APP_HEALTH              = 0x114,
+	MSG_OP_ADD_HOST_BUFFER             = 0x115,
+	MSG_OP_GET_DEV_REVISION            = 0x117,
 	MSG_OP_MAX_DRV_OPCODE,
 	MSG_OP_GET_PROTOCOL_VERSION        = 0x301,
 	MSG_OP_MAX_OPCODE
@@ -460,7 +463,7 @@ struct fatal_error_info {
 	__u32 exception_pc;       /* Program Counter at the time of the exception */
 	__u32 app_module;         /* Error module name */
 	__u32 task_index;         /* Index of the task in which the error occurred */
-	__u32 reserved[128];
+	__u32 reserved[127];
 };
 
 struct app_health_report {
@@ -503,4 +506,41 @@ struct get_app_health_resp {
 	__u32 required_buffer_size;
 	__u32 reserved[7];
 } __packed;
+
+struct update_property_req {
+#define UPDATE_PROPERTY_TIME_QUOTA 0
+	__u32 type;
+#define AIE2_UPDATE_PROPERTY_ALL_CTX	0xFF
+	__u8 context_id;
+	__u8 reserved[7];
+	__u32 time_quota_us;
+	__u32 reserved1;
+} __packed;
+
+struct update_property_resp {
+	enum aie2_msg_status status;
+} __packed;
+
+enum aie2_dev_revision {
+	AIE2_DEV_REVISION_STXA = 1,
+	AIE2_DEV_REVISION_STXB,
+	AIE2_DEV_REVISION_KRK1,
+	AIE2_DEV_REVISION_KRK2,
+	AIE2_DEV_REVISION_HALO,
+	AIE2_DEV_REVISION_GPT1,
+	AIE2_DEV_REVISION_GPT2,
+	AIE2_DEV_REVISION_GPT3,
+	AIE2_DEV_REVISION_UNKN,
+};
+
+struct get_dev_revision_req {
+	__u32			place_holder;
+} __packed;
+
+struct get_dev_revision_resp {
+	enum aie2_msg_status	status;
+	enum aie2_dev_revision	rev;
+	__u32			raw_fuse_data;
+} __packed;
+
 #endif /* _AIE2_MSG_PRIV_H_ */
diff --git a/drivers/accel/amdxdna/aie2_pci.c b/drivers/accel/amdxdna/aie2_pci.c
index f1ac4e00bd9f4..c4d345d4c76b2 100644
--- a/drivers/accel/amdxdna/aie2_pci.c
+++ b/drivers/accel/amdxdna/aie2_pci.c
@@ -33,6 +33,8 @@ static int aie2_max_col = XRS_MAX_COL;
 module_param(aie2_max_col, uint, 0600);
 MODULE_PARM_DESC(aie2_max_col, "Maximum column could be used");
 
+#define DEFAULT_TIME_QUANTUM 30000 /* microseconds */
+
 static char *npu_fw[] = {
 	"npu_7.sbin",
 	"npu.sbin"
@@ -60,45 +62,6 @@ struct mgmt_mbox_chann_info {
 	__u32	rsvd[4];
 };
 
-static int aie2_check_protocol(struct amdxdna_dev_hdl *ndev, u32 fw_major, u32 fw_minor)
-{
-	const struct aie2_fw_feature_tbl *feature;
-	bool found = false;
-
-	for (feature = ndev->priv->fw_feature_tbl; feature->major; feature++) {
-		if (feature->major != fw_major)
-			continue;
-		if (fw_minor < feature->min_minor)
-			continue;
-		if (feature->max_minor > 0 && fw_minor > feature->max_minor)
-			continue;
-
-		ndev->feature_mask |= feature->features;
-
-		/* firmware version matches one of the driver support entry */
-		found = true;
-	}
-
-	return found ? 0 : -EOPNOTSUPP;
-}
-
-static void aie2_dump_chann_info_debug(struct amdxdna_dev_hdl *ndev)
-{
-	struct amdxdna_dev *xdna = ndev->xdna;
-
-	XDNA_DBG(xdna, "i2x tail    0x%x", ndev->mgmt_i2x.mb_tail_ptr_reg);
-	XDNA_DBG(xdna, "i2x head    0x%x", ndev->mgmt_i2x.mb_head_ptr_reg);
-	XDNA_DBG(xdna, "i2x ringbuf 0x%x", ndev->mgmt_i2x.rb_start_addr);
-	XDNA_DBG(xdna, "i2x rsize   0x%x", ndev->mgmt_i2x.rb_size);
-	XDNA_DBG(xdna, "x2i tail    0x%x", ndev->mgmt_x2i.mb_tail_ptr_reg);
-	XDNA_DBG(xdna, "x2i head    0x%x", ndev->mgmt_x2i.mb_head_ptr_reg);
-	XDNA_DBG(xdna, "x2i ringbuf 0x%x", ndev->mgmt_x2i.rb_start_addr);
-	XDNA_DBG(xdna, "x2i rsize   0x%x", ndev->mgmt_x2i.rb_size);
-	XDNA_DBG(xdna, "x2i chann index 0x%x", ndev->mgmt_chan_idx);
-	XDNA_DBG(xdna, "mailbox protocol major 0x%x", ndev->mgmt_prot_major);
-	XDNA_DBG(xdna, "mailbox protocol minor 0x%x", ndev->mgmt_prot_minor);
-}
-
 static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
 {
 	struct mgmt_mbox_chann_info info_regs;
@@ -118,7 +81,7 @@ static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
 	 * is alive.
 	 */
 	ret = readx_poll_timeout(readl, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF),
-				 addr, addr, AIE2_INTERVAL, AIE2_TIMEOUT);
+				 addr, addr, AIE_INTERVAL, AIE_TIMEOUT);
 	if (ret || !addr)
 		return -ETIME;
 
@@ -128,13 +91,13 @@ static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
 		reg[i] = readl(ndev->sram_base + off + i * sizeof(u32));
 
 	if (info_regs.magic != MGMT_MBOX_MAGIC) {
-		XDNA_ERR(ndev->xdna, "Invalid mbox magic 0x%x", info_regs.magic);
+		XDNA_ERR(ndev->aie.xdna, "Invalid mbox magic 0x%x", info_regs.magic);
 		ret = -EINVAL;
 		goto done;
 	}
 
-	i2x = &ndev->mgmt_i2x;
-	x2i = &ndev->mgmt_x2i;
+	i2x = &ndev->aie.mgmt_i2x;
+	x2i = &ndev->aie.mgmt_x2i;
 
 	i2x->mb_head_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_head);
 	i2x->mb_tail_ptr_reg = AIE2_MBOX_OFF(ndev, info_regs.i2x_tail);
@@ -146,14 +109,15 @@ static int aie2_get_mgmt_chann_info(struct amdxdna_dev_hdl *ndev)
 	x2i->rb_start_addr   = AIE2_SRAM_OFF(ndev, info_regs.x2i_buf);
 	x2i->rb_size         = info_regs.x2i_buf_sz;
 
-	ndev->mgmt_chan_idx  = info_regs.msi_id;
-	ndev->mgmt_prot_major = info_regs.prot_major;
-	ndev->mgmt_prot_minor = info_regs.prot_minor;
+	ndev->aie.mgmt_chan_idx  = info_regs.msi_id;
+	ndev->aie.mgmt_prot_major = info_regs.prot_major;
+	ndev->aie.mgmt_prot_minor = info_regs.prot_minor;
 
-	ret = aie2_check_protocol(ndev, ndev->mgmt_prot_major, ndev->mgmt_prot_minor);
+	ret = aie_check_protocol(&ndev->aie, ndev->aie.mgmt_prot_major,
+				 ndev->aie.mgmt_prot_minor);
 
 done:
-	aie2_dump_chann_info_debug(ndev);
+	aie_dump_mgmt_chann_debug(&ndev->aie);
 
 	/* Must clear address at FW_ALIVE_OFF */
 	writel(0, SRAM_GET_ADDR(ndev, FW_ALIVE_OFF));
@@ -173,13 +137,14 @@ int aie2_runtime_cfg(struct amdxdna_dev_hdl *ndev,
 			continue;
 
 		if (cfg->feature_mask &&
-		    bitmap_subset(&cfg->feature_mask, &ndev->feature_mask, AIE2_FEATURE_MAX))
+		    bitmap_subset(&cfg->feature_mask, &ndev->aie.feature_mask,
+				  AIE2_FEATURE_MAX))
 			continue;
 
 		value = val ? *val : cfg->value;
 		ret = aie2_set_runtime_cfg(ndev, cfg->type, value);
 		if (ret) {
-			XDNA_ERR(ndev->xdna, "Set type %d value %d failed",
+			XDNA_ERR(ndev->aie.xdna, "Set type %d value %d failed",
 				 cfg->type, value);
 			return ret;
 		}
@@ -194,13 +159,13 @@ static int aie2_xdna_reset(struct amdxdna_dev_hdl *ndev)
 
 	ret = aie2_suspend_fw(ndev);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Suspend firmware failed");
+		XDNA_ERR(ndev->aie.xdna, "Suspend firmware failed");
 		return ret;
 	}
 
 	ret = aie2_resume_fw(ndev);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Resume firmware failed");
+		XDNA_ERR(ndev->aie.xdna, "Resume firmware failed");
 		return ret;
 	}
 
@@ -213,19 +178,25 @@ static int aie2_mgmt_fw_init(struct amdxdna_dev_hdl *ndev)
 
 	ret = aie2_runtime_cfg(ndev, AIE2_RT_CFG_INIT, NULL);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Runtime config failed");
+		XDNA_ERR(ndev->aie.xdna, "Runtime config failed");
 		return ret;
 	}
 
 	ret = aie2_assign_mgmt_pasid(ndev, 0);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Can not assign PASID");
+		XDNA_ERR(ndev->aie.xdna, "Can not assign PASID");
+		return ret;
+	}
+
+	ret = aie2_update_prop_time_quota(ndev, DEFAULT_TIME_QUANTUM);
+	if (ret) {
+		XDNA_ERR(ndev->aie.xdna, "Failed to update execution time quantum");
 		return ret;
 	}
 
 	ret = aie2_xdna_reset(ndev);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Reset firmware failed");
+		XDNA_ERR(ndev->aie.xdna, "Reset firmware failed");
 		return ret;
 	}
 
@@ -236,25 +207,25 @@ static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev)
 {
 	int ret;
 
-	ret = aie2_query_firmware_version(ndev, &ndev->xdna->fw_ver);
+	ret = aie2_query_firmware_version(ndev, &ndev->aie.xdna->fw_ver);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "query firmware version failed");
+		XDNA_ERR(ndev->aie.xdna, "query firmware version failed");
 		return ret;
 	}
 
 	ret = aie2_query_aie_version(ndev, &ndev->version);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Query AIE version failed");
+		XDNA_ERR(ndev->aie.xdna, "Query AIE version failed");
 		return ret;
 	}
 
-	ret = aie2_query_aie_metadata(ndev, &ndev->metadata);
+	ret = aie2_query_aie_metadata(ndev, &ndev->aie.metadata);
 	if (ret) {
-		XDNA_ERR(ndev->xdna, "Query AIE metadata failed");
+		XDNA_ERR(ndev->aie.xdna, "Query AIE metadata failed");
 		return ret;
 	}
 
-	ndev->total_col = min(aie2_max_col, ndev->metadata.cols);
+	ndev->total_col = min(aie2_max_col, ndev->aie.metadata.cols);
 
 	return 0;
 }
@@ -262,8 +233,8 @@ static int aie2_mgmt_fw_query(struct amdxdna_dev_hdl *ndev)
 static void aie2_mgmt_fw_fini(struct amdxdna_dev_hdl *ndev)
 {
 	if (aie2_suspend_fw(ndev))
-		XDNA_ERR(ndev->xdna, "Suspend_fw failed");
-	XDNA_DBG(ndev->xdna, "Firmware suspended");
+		XDNA_ERR(ndev->aie.xdna, "Suspend_fw failed");
+	XDNA_DBG(ndev->aie.xdna, "Firmware suspended");
 }
 
 static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action)
@@ -275,6 +246,7 @@ static int aie2_xrs_load(void *cb_arg, struct xrs_action_load *action)
 	xdna = hwctx->client->xdna;
 
 	hwctx->start_col = action->part.start_col;
+	hwctx->num_unused_col = action->part.ncols - hwctx->num_col;
 	hwctx->num_col = action->part.ncols;
 	ret = aie2_create_context(xdna->dev_handle, hwctx);
 	if (ret)
@@ -319,6 +291,12 @@ static struct xrs_action_ops aie2_xrs_actions = {
 	.set_dft_dpm_level = aie2_xrs_set_dft_dpm_level,
 };
 
+static void aie2_smu_fini(struct amdxdna_dev_hdl *ndev)
+{
+	ndev->priv->hw_ops->set_dpm(ndev, 0);
+	aie_smu_fini(ndev->aie.smu_hdl);
+}
+
 static void aie2_hw_stop(struct amdxdna_dev *xdna)
 {
 	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
@@ -331,10 +309,10 @@ static void aie2_hw_stop(struct amdxdna_dev *xdna)
 
 	aie2_runtime_cfg(ndev, AIE2_RT_CFG_CLK_GATING, NULL);
 	aie2_mgmt_fw_fini(ndev);
-	aie2_destroy_mgmt_chann(ndev);
+	aie_destroy_chann(&ndev->aie, &ndev->aie.mgmt_chann);
 	drmm_kfree(&xdna->ddev, ndev->mbox);
 	ndev->mbox = NULL;
-	aie2_psp_stop(ndev->psp_hdl);
+	aie_psp_stop(ndev->aie.psp_hdl);
 	aie2_smu_fini(ndev);
 	aie2_error_async_events_free(ndev);
 	pci_disable_device(pdev);
@@ -374,20 +352,20 @@ static int aie2_hw_start(struct amdxdna_dev *xdna)
 		goto disable_dev;
 	}
 
-	ndev->mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox);
-	if (!ndev->mgmt_chann) {
+	ndev->aie.mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox);
+	if (!ndev->aie.mgmt_chann) {
 		XDNA_ERR(xdna, "failed to alloc channel");
 		ret = -ENODEV;
 		goto disable_dev;
 	}
 
-	ret = aie2_smu_init(ndev);
+	ret = aie_smu_init(ndev->aie.smu_hdl);
 	if (ret) {
 		XDNA_ERR(xdna, "failed to init smu, ret %d", ret);
 		goto free_channel;
 	}
 
-	ret = aie2_psp_start(ndev->psp_hdl);
+	ret = aie_psp_start(ndev->aie.psp_hdl);
 	if (ret) {
 		XDNA_ERR(xdna, "failed to start psp, ret %d", ret);
 		goto fini_smu;
@@ -399,17 +377,17 @@ static int aie2_hw_start(struct amdxdna_dev *xdna)
 		goto stop_psp;
 	}
 
-	mgmt_mb_irq = pci_irq_vector(pdev, ndev->mgmt_chan_idx);
+	mgmt_mb_irq = pci_irq_vector(pdev, ndev->aie.mgmt_chan_idx);
 	if (mgmt_mb_irq < 0) {
 		ret = mgmt_mb_irq;
 		XDNA_ERR(xdna, "failed to alloc irq vector, ret %d", ret);
 		goto stop_psp;
 	}
 
-	xdna_mailbox_intr_reg = ndev->mgmt_i2x.mb_head_ptr_reg + 4;
-	ret = xdna_mailbox_start_channel(ndev->mgmt_chann,
-					 &ndev->mgmt_x2i,
-					 &ndev->mgmt_i2x,
+	xdna_mailbox_intr_reg = ndev->aie.mgmt_i2x.mb_head_ptr_reg + 4;
+	ret = xdna_mailbox_start_channel(ndev->aie.mgmt_chann,
+					 &ndev->aie.mgmt_x2i,
+					 &ndev->aie.mgmt_i2x,
 					 xdna_mailbox_intr_reg,
 					 mgmt_mb_irq);
 	if (ret) {
@@ -448,14 +426,14 @@ static int aie2_hw_start(struct amdxdna_dev *xdna)
 
 stop_fw:
 	aie2_suspend_fw(ndev);
-	xdna_mailbox_stop_channel(ndev->mgmt_chann);
+	xdna_mailbox_stop_channel(ndev->aie.mgmt_chann);
 stop_psp:
-	aie2_psp_stop(ndev->psp_hdl);
+	aie_psp_stop(ndev->aie.psp_hdl);
 fini_smu:
 	aie2_smu_fini(ndev);
 free_channel:
-	xdna_mailbox_free_channel(ndev->mgmt_chann);
-	ndev->mgmt_chann = NULL;
+	xdna_mailbox_free_channel(ndev->aie.mgmt_chann);
+	ndev->aie.mgmt_chann = NULL;
 disable_dev:
 	pci_disable_device(pdev);
 
@@ -500,7 +478,8 @@ static int aie2_init(struct amdxdna_dev *xdna)
 	void __iomem *tbl[PCI_NUM_RESOURCES] = {0};
 	struct init_config xrs_cfg = { 0 };
 	struct amdxdna_dev_hdl *ndev;
-	struct psp_config psp_conf;
+	struct psp_config psp_conf = { 0 };
+	struct smu_config smu_conf;
 	const struct firmware *fw;
 	unsigned long bars = 0;
 	char *fw_full_path;
@@ -516,7 +495,7 @@ static int aie2_init(struct amdxdna_dev *xdna)
 		return -ENOMEM;
 
 	ndev->priv = xdna->dev_info->dev_priv;
-	ndev->xdna = xdna;
+	ndev->aie.xdna = xdna;
 
 	for (i = 0; i < ARRAY_SIZE(npu_fw); i++) {
 		fw_full_path = kasprintf(GFP_KERNEL, "%s%s", ndev->priv->fw_path, npu_fw[i]);
@@ -545,9 +524,10 @@ static int aie2_init(struct amdxdna_dev *xdna)
 
 	for (i = 0; i < PSP_MAX_REGS; i++)
 		set_bit(PSP_REG_BAR(ndev, i), &bars);
+	for (i = 0; i < SMU_MAX_REGS; i++)
+		set_bit(SMU_REG_BAR(ndev, i), &bars);
 
 	set_bit(xdna->dev_info->sram_bar, &bars);
-	set_bit(xdna->dev_info->smu_bar, &bars);
 	set_bit(xdna->dev_info->mbox_bar, &bars);
 
 	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
@@ -562,7 +542,6 @@ static int aie2_init(struct amdxdna_dev *xdna)
 	}
 
 	ndev->sram_base = tbl[xdna->dev_info->sram_bar];
-	ndev->smu_base = tbl[xdna->dev_info->smu_bar];
 	ndev->mbox_base = tbl[xdna->dev_info->mbox_bar];
 
 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
@@ -586,14 +565,25 @@ static int aie2_init(struct amdxdna_dev *xdna)
 
 	psp_conf.fw_size = fw->size;
 	psp_conf.fw_buf = fw->data;
+	psp_conf.arg2_mask = GENMASK(23, 0);
+	psp_conf.notify_val = 1;
 	for (i = 0; i < PSP_MAX_REGS; i++)
 		psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i);
-	ndev->psp_hdl = aie2m_psp_create(&xdna->ddev, &psp_conf);
-	if (!ndev->psp_hdl) {
+	ndev->aie.psp_hdl = aiem_psp_create(&xdna->ddev, &psp_conf);
+	if (!ndev->aie.psp_hdl) {
 		XDNA_ERR(xdna, "failed to create psp");
 		ret = -ENOMEM;
 		goto release_fw;
 	}
+
+	for (i = 0; i < SMU_MAX_REGS; i++)
+		smu_conf.smu_regs[i] = tbl[SMU_REG_BAR(ndev, i)] + SMU_REG_OFF(ndev, i);
+	ndev->aie.smu_hdl = aiem_smu_create(&xdna->ddev, &smu_conf);
+	if (!ndev->aie.smu_hdl) {
+		XDNA_ERR(xdna, "failed to create smu");
+		ret = -ENOMEM;
+		goto release_fw;
+	}
 	xdna->dev_handle = ndev;
 
 	ret = aie2_hw_start(xdna);
@@ -605,7 +595,7 @@ static int aie2_init(struct amdxdna_dev *xdna)
 	xrs_cfg.clk_list.num_levels = ndev->max_dpm_level + 1;
 	for (i = 0; i < xrs_cfg.clk_list.num_levels; i++)
 		xrs_cfg.clk_list.cu_clk_list[i] = ndev->priv->dpm_clk_tbl[i].hclk;
-	xrs_cfg.sys_eff_factor = 1;
+	xrs_cfg.sys_eff_factor = 2;
 	xrs_cfg.ddev = &xdna->ddev;
 	xrs_cfg.actions = &aie2_xrs_actions;
 	xrs_cfg.total_col = ndev->total_col;
@@ -619,6 +609,7 @@ static int aie2_init(struct amdxdna_dev *xdna)
 
 	release_firmware(fw);
 	aie2_msg_init(ndev);
+	amdxdna_vbnv_init(xdna);
 	amdxdna_pm_init(xdna);
 	return 0;
 
@@ -639,23 +630,19 @@ static void aie2_fini(struct amdxdna_dev *xdna)
 static int aie2_get_aie_status(struct amdxdna_client *client,
 			       struct amdxdna_drm_get_info *args)
 {
-	struct amdxdna_drm_query_aie_status status;
+	struct amdxdna_drm_query_aie_status status = {};
 	struct amdxdna_dev *xdna = client->xdna;
 	struct amdxdna_dev_hdl *ndev;
+	u32 buf_sz;
 	int ret;
 
 	ndev = xdna->dev_handle;
-	if (copy_from_user(&status, u64_to_user_ptr(args->buffer), sizeof(status))) {
+	buf_sz = min(args->buffer_size, sizeof(status));
+	if (copy_from_user(&status, u64_to_user_ptr(args->buffer), buf_sz)) {
 		XDNA_ERR(xdna, "Failed to copy AIE request into kernel");
 		return -EFAULT;
 	}
 
-	if (ndev->metadata.cols * ndev->metadata.size < status.buffer_size) {
-		XDNA_ERR(xdna, "Invalid buffer size. Given Size: %u. Need Size: %u.",
-			 status.buffer_size, ndev->metadata.cols * ndev->metadata.size);
-		return -EINVAL;
-	}
-
 	ret = aie2_query_status(ndev, u64_to_user_ptr(status.buffer),
 				status.buffer_size, &status.cols_filled);
 	if (ret) {
@@ -663,7 +650,7 @@ static int aie2_get_aie_status(struct amdxdna_client *client,
 		return ret;
 	}
 
-	if (copy_to_user(u64_to_user_ptr(args->buffer), &status, sizeof(status))) {
+	if (copy_to_user(u64_to_user_ptr(args->buffer), &status, buf_sz)) {
 		XDNA_ERR(xdna, "Failed to copy AIE request info to user space");
 		return -EFAULT;
 	}
@@ -671,63 +658,20 @@ static int aie2_get_aie_status(struct amdxdna_client *client,
 	return 0;
 }
 
-static int aie2_get_aie_metadata(struct amdxdna_client *client,
-				 struct amdxdna_drm_get_info *args)
-{
-	struct amdxdna_drm_query_aie_metadata *meta;
-	struct amdxdna_dev *xdna = client->xdna;
-	struct amdxdna_dev_hdl *ndev;
-	int ret = 0;
-
-	ndev = xdna->dev_handle;
-	meta = kzalloc_obj(*meta);
-	if (!meta)
-		return -ENOMEM;
-
-	meta->col_size = ndev->metadata.size;
-	meta->cols = ndev->metadata.cols;
-	meta->rows = ndev->metadata.rows;
-
-	meta->version.major = ndev->metadata.version.major;
-	meta->version.minor = ndev->metadata.version.minor;
-
-	meta->core.row_count = ndev->metadata.core.row_count;
-	meta->core.row_start = ndev->metadata.core.row_start;
-	meta->core.dma_channel_count = ndev->metadata.core.dma_channel_count;
-	meta->core.lock_count = ndev->metadata.core.lock_count;
-	meta->core.event_reg_count = ndev->metadata.core.event_reg_count;
-
-	meta->mem.row_count = ndev->metadata.mem.row_count;
-	meta->mem.row_start = ndev->metadata.mem.row_start;
-	meta->mem.dma_channel_count = ndev->metadata.mem.dma_channel_count;
-	meta->mem.lock_count = ndev->metadata.mem.lock_count;
-	meta->mem.event_reg_count = ndev->metadata.mem.event_reg_count;
-
-	meta->shim.row_count = ndev->metadata.shim.row_count;
-	meta->shim.row_start = ndev->metadata.shim.row_start;
-	meta->shim.dma_channel_count = ndev->metadata.shim.dma_channel_count;
-	meta->shim.lock_count = ndev->metadata.shim.lock_count;
-	meta->shim.event_reg_count = ndev->metadata.shim.event_reg_count;
-
-	if (copy_to_user(u64_to_user_ptr(args->buffer), meta, sizeof(*meta)))
-		ret = -EFAULT;
-
-	kfree(meta);
-	return ret;
-}
-
 static int aie2_get_aie_version(struct amdxdna_client *client,
 				struct amdxdna_drm_get_info *args)
 {
 	struct amdxdna_drm_query_aie_version version;
 	struct amdxdna_dev *xdna = client->xdna;
 	struct amdxdna_dev_hdl *ndev;
+	u32 buf_sz;
 
 	ndev = xdna->dev_handle;
 	version.major = ndev->version.major;
 	version.minor = ndev->version.minor;
 
-	if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
+	buf_sz = min(args->buffer_size, sizeof(version));
+	if (copy_to_user(u64_to_user_ptr(args->buffer), &version, buf_sz))
 		return -EFAULT;
 
 	return 0;
@@ -738,13 +682,15 @@ static int aie2_get_firmware_version(struct amdxdna_client *client,
 {
 	struct amdxdna_drm_query_firmware_version version;
 	struct amdxdna_dev *xdna = client->xdna;
+	u32 buf_sz;
 
 	version.major = xdna->fw_ver.major;
 	version.minor = xdna->fw_ver.minor;
 	version.patch = xdna->fw_ver.sub;
 	version.build = xdna->fw_ver.build;
 
-	if (copy_to_user(u64_to_user_ptr(args->buffer), &version, sizeof(version)))
+	buf_sz = min(args->buffer_size, sizeof(version));
+	if (copy_to_user(u64_to_user_ptr(args->buffer), &version, buf_sz))
 		return -EFAULT;
 
 	return 0;
@@ -756,11 +702,13 @@ static int aie2_get_power_mode(struct amdxdna_client *client,
 	struct amdxdna_drm_get_power_mode mode = {};
 	struct amdxdna_dev *xdna = client->xdna;
 	struct amdxdna_dev_hdl *ndev;
+	u32 buf_sz;
 
 	ndev = xdna->dev_handle;
 	mode.power_mode = ndev->pw_mode;
 
-	if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, sizeof(mode)))
+	buf_sz = min(args->buffer_size, sizeof(mode));
+	if (copy_to_user(u64_to_user_ptr(args->buffer), &mode, buf_sz))
 		return -EFAULT;
 
 	return 0;
@@ -773,19 +721,22 @@ static int aie2_get_clock_metadata(struct amdxdna_client *client,
 	struct amdxdna_dev *xdna = client->xdna;
 	struct amdxdna_dev_hdl *ndev;
 	int ret = 0;
+	u32 buf_sz;
 
 	ndev = xdna->dev_handle;
 	clock = kzalloc_obj(*clock);
 	if (!clock)
 		return -ENOMEM;
 
+	aie2_update_counters(ndev);
 	snprintf(clock->mp_npu_clock.name, sizeof(clock->mp_npu_clock.name),
 		 "MP-NPU Clock");
 	clock->mp_npu_clock.freq_mhz = ndev->npuclk_freq;
 	snprintf(clock->h_clock.name, sizeof(clock->h_clock.name), "H Clock");
 	clock->h_clock.freq_mhz = ndev->hclk_freq;
 
-	if (copy_to_user(u64_to_user_ptr(args->buffer), clock, sizeof(*clock)))
+	buf_sz = min(args->buffer_size, sizeof(*clock));
+	if (copy_to_user(u64_to_user_ptr(args->buffer), clock, buf_sz))
 		ret = -EFAULT;
 
 	kfree(clock);
@@ -811,12 +762,14 @@ static int aie2_get_sensors(struct amdxdna_client *client,
 	scnprintf(sensor.label, sizeof(sensor.label), "Total Power");
 	scnprintf(sensor.units, sizeof(sensor.units), "mW");
 
+	if (args->buffer_size < sizeof(sensor))
+		goto out;
+
 	if (copy_to_user(u64_to_user_ptr(args->buffer), &sensor, sizeof(sensor)))
 		return -EFAULT;
 
+	args->buffer_size -= sizeof(sensor);
 	sensors_count++;
-	if (args->buffer_size <= sensors_count * sizeof(sensor))
-		goto out;
 
 	for (i = 0; i < min_t(u32, ndev->total_col, 8); i++) {
 		memset(&sensor, 0, sizeof(sensor));
@@ -826,13 +779,15 @@ static int aie2_get_sensors(struct amdxdna_client *client,
 		scnprintf(sensor.label, sizeof(sensor.label), "Column %d Utilization", i);
 		scnprintf(sensor.units, sizeof(sensor.units), "%%");
 
+		if (args->buffer_size < sizeof(sensor))
+			goto out;
+
 		if (copy_to_user(u64_to_user_ptr(args->buffer) + sensors_count * sizeof(sensor),
 				 &sensor, sizeof(sensor)))
 			return -EFAULT;
 
+		args->buffer_size -= sizeof(sensor);
 		sensors_count++;
-		if (args->buffer_size <= sensors_count * sizeof(sensor))
-			goto out;
 	}
 
 out:
@@ -928,18 +883,21 @@ static int aie2_query_resource_info(struct amdxdna_client *client,
 	const struct amdxdna_dev_priv *priv;
 	struct amdxdna_dev_hdl *ndev;
 	struct amdxdna_dev *xdna;
+	u32 buf_sz;
 
 	xdna = client->xdna;
 	ndev = xdna->dev_handle;
 	priv = ndev->priv;
 
+	aie2_update_counters(ndev);
 	res_info.npu_clk_max = priv->dpm_clk_tbl[ndev->max_dpm_level].hclk;
 	res_info.npu_tops_max = ndev->max_tops;
 	res_info.npu_task_max = priv->hwctx_limit;
 	res_info.npu_tops_curr = ndev->curr_tops;
 	res_info.npu_task_curr = ndev->hwctx_num;
 
-	if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, sizeof(res_info)))
+	buf_sz = min(args->buffer_size, sizeof(res_info));
+	if (copy_to_user(u64_to_user_ptr(args->buffer), &res_info, buf_sz))
 		return -EFAULT;
 
 	return 0;
@@ -975,12 +933,7 @@ static int aie2_get_telemetry(struct amdxdna_client *client,
 		XDNA_ERR(xdna, "Invalid buffer size");
 		return -EINVAL;
 	}
-
 	telemetry_data_sz = args->buffer_size - header_sz;
-	if (telemetry_data_sz > SZ_4M) {
-		XDNA_ERR(xdna, "Buffer size is too big, %d", telemetry_data_sz);
-		return -EINVAL;
-	}
 
 	header = kzalloc(header_sz, GFP_KERNEL);
 	if (!header)
@@ -1021,6 +974,7 @@ static int aie2_get_preempt_state(struct amdxdna_client *client,
 	struct amdxdna_drm_attribute_state state = {};
 	struct amdxdna_dev *xdna = client->xdna;
 	struct amdxdna_dev_hdl *ndev;
+	u32 buf_sz;
 
 	ndev = xdna->dev_handle;
 	if (args->param == DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE)
@@ -1028,7 +982,8 @@ static int aie2_get_preempt_state(struct amdxdna_client *client,
 	else if (args->param == DRM_AMDXDNA_GET_FRAME_BOUNDARY_PREEMPT_STATE)
 		state.state = ndev->frame_boundary_preempt;
 
-	if (copy_to_user(u64_to_user_ptr(args->buffer), &state, sizeof(state)))
+	buf_sz = min(args->buffer_size, sizeof(state));
+	if (copy_to_user(u64_to_user_ptr(args->buffer), &state, buf_sz))
 		return -EFAULT;
 
 	return 0;
@@ -1037,6 +992,7 @@ static int aie2_get_preempt_state(struct amdxdna_client *client,
 static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args)
 {
 	struct amdxdna_dev *xdna = client->xdna;
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
 	int ret, idx;
 
 	if (!drm_dev_enter(&xdna->ddev, &idx))
@@ -1051,7 +1007,7 @@ static int aie2_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_i
 		ret = aie2_get_aie_status(client, args);
 		break;
 	case DRM_AMDXDNA_QUERY_AIE_METADATA:
-		ret = aie2_get_aie_metadata(client, args);
+		ret = amdxdna_get_metadata(&ndev->aie, client, args);
 		break;
 	case DRM_AMDXDNA_QUERY_AIE_VERSION:
 		ret = aie2_get_aie_version(client, args);
@@ -1255,6 +1211,21 @@ static int aie2_set_state(struct amdxdna_client *client,
 	return ret;
 }
 
+static int aie2_get_dev_rev(struct amdxdna_dev *xdna, u32 *rev)
+{
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+	enum aie2_dev_revision aie2_rev;
+	int ret;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+	ret = aie2_get_dev_revision(ndev, &aie2_rev);
+
+	if (!ret)
+		*rev = (u32)aie2_rev;
+
+	return ret;
+}
+
 const struct amdxdna_dev_ops aie2_ops = {
 	.init = aie2_init,
 	.fini = aie2_fini,
@@ -1269,4 +1240,6 @@ const struct amdxdna_dev_ops aie2_ops = {
 	.cmd_submit = aie2_cmd_submit,
 	.hmm_invalidate = aie2_hmm_invalidate,
 	.get_array = aie2_get_array,
+	.get_dev_revision = aie2_get_dev_rev,
+	.hwctx_heap_expand = aie2_hwctx_heap_expand,
 };
diff --git a/drivers/accel/amdxdna/aie2_pci.h b/drivers/accel/amdxdna/aie2_pci.h
index efcf4be035f04..77648cc548b66 100644
--- a/drivers/accel/amdxdna/aie2_pci.h
+++ b/drivers/accel/amdxdna/aie2_pci.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (C) 2023-2024, Advanced Micro Devices, Inc.
+ * Copyright (C) 2023-2026, Advanced Micro Devices, Inc.
  */
 
 #ifndef _AIE2_PCI_H_
@@ -10,30 +10,22 @@
 #include <linux/limits.h>
 #include <linux/semaphore.h>
 
+#include "aie.h"
 #include "aie2_msg_priv.h"
 #include "amdxdna_mailbox.h"
 
-#define AIE2_INTERVAL	20000	/* us */
-#define AIE2_TIMEOUT	1000000	/* us */
-
 /* Firmware determines device memory base address and size */
-#define AIE2_DEVM_BASE	0x4000000
-#define AIE2_DEVM_SIZE	SZ_64M
+#define AIE2_DEVM_BASE		0x4000000
+#define AIE2_DEVM_SIZE		SZ_64M
+#define AIE2_DEVM_MAX_SIZE	SZ_512M
 
-#define NDEV2PDEV(ndev) (to_pci_dev((ndev)->xdna->ddev.dev))
+#define NDEV2PDEV(ndev) (to_pci_dev((ndev)->aie.xdna->ddev.dev))
 
 #define AIE2_SRAM_OFF(ndev, addr) ((addr) - (ndev)->priv->sram_dev_addr)
 #define AIE2_MBOX_OFF(ndev, addr) ((addr) - (ndev)->priv->mbox_dev_addr)
 
-#define PSP_REG_BAR(ndev, idx) ((ndev)->priv->psp_regs_off[(idx)].bar_idx)
-#define PSP_REG_OFF(ndev, idx) ((ndev)->priv->psp_regs_off[(idx)].offset)
 #define SRAM_REG_OFF(ndev, idx) ((ndev)->priv->sram_offs[(idx)].offset)
 
-#define SMU_REG(ndev, idx) \
-({ \
-	typeof(ndev) _ndev = ndev; \
-	((_ndev)->smu_base + (_ndev)->priv->smu_regs_off[(idx)].offset); \
-})
 #define SRAM_GET_ADDR(ndev, idx) \
 ({ \
 	typeof(ndev) _ndev = ndev; \
@@ -45,7 +37,7 @@
 ({ \
 	typeof(ndev) _ndev = (ndev); \
 	((_ndev)->priv->mbox_size) ? (_ndev)->priv->mbox_size : \
-	pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->xdna->dev_info->mbox_bar); \
+	pci_resource_len(NDEV2PDEV(_ndev), (_ndev)->aie.xdna->dev_info->mbox_bar); \
 })
 
 #if IS_ENABLED(CONFIG_AMD_PMF)
@@ -75,68 +67,17 @@
 })
 #endif
 
-enum aie2_smu_reg_idx {
-	SMU_CMD_REG = 0,
-	SMU_ARG_REG,
-	SMU_INTR_REG,
-	SMU_RESP_REG,
-	SMU_OUT_REG,
-	SMU_MAX_REGS /* Keep this at the end */
-};
-
 enum aie2_sram_reg_idx {
 	MBOX_CHANN_OFF = 0,
 	FW_ALIVE_OFF,
 	SRAM_MAX_INDEX /* Keep this at the end */
 };
 
-enum psp_reg_idx {
-	PSP_CMD_REG = 0,
-	PSP_ARG0_REG,
-	PSP_ARG1_REG,
-	PSP_ARG2_REG,
-	PSP_NUM_IN_REGS, /* number of input registers */
-	PSP_INTR_REG = PSP_NUM_IN_REGS,
-	PSP_STATUS_REG,
-	PSP_RESP_REG,
-	PSP_PWAITMODE_REG,
-	PSP_MAX_REGS /* Keep this at the end */
-};
-
 struct amdxdna_client;
 struct amdxdna_fw_ver;
 struct amdxdna_hwctx;
 struct amdxdna_sched_job;
 
-struct psp_config {
-	const void	*fw_buf;
-	u32		fw_size;
-	void __iomem	*psp_regs[PSP_MAX_REGS];
-};
-
-struct aie_version {
-	u16 major;
-	u16 minor;
-};
-
-struct aie_tile_metadata {
-	u16 row_count;
-	u16 row_start;
-	u16 dma_channel_count;
-	u16 lock_count;
-	u16 event_reg_count;
-};
-
-struct aie_metadata {
-	u32 size;
-	u16 cols;
-	u16 rows;
-	struct aie_version version;
-	struct aie_tile_metadata core;
-	struct aie_tile_metadata mem;
-	struct aie_tile_metadata shim;
-};
-
 enum rt_config_category {
 	AIE2_RT_CFG_INIT,
 	AIE2_RT_CFG_CLK_GATING,
@@ -202,24 +143,19 @@ struct aie2_exec_msg_ops {
 	u32 (*get_chain_msg_op)(u32 cmd_op);
 };
 
+enum aie2_tdr_status {
+	AIE2_TDR_WAIT,
+	AIE2_TDR_SIGNALED,
+};
+
 struct amdxdna_dev_hdl {
-	struct amdxdna_dev		*xdna;
+	struct aie_device		aie;
 	const struct amdxdna_dev_priv	*priv;
 	void			__iomem *sram_base;
-	void			__iomem *smu_base;
 	void			__iomem *mbox_base;
-	struct psp_device		*psp_hdl;
-
-	struct xdna_mailbox_chann_res	mgmt_x2i;
-	struct xdna_mailbox_chann_res	mgmt_i2x;
-	u32				mgmt_chan_idx;
-	u32				mgmt_prot_major;
-	u32				mgmt_prot_minor;
 
 	u32				total_col;
-	struct aie_version		version;
-	struct aie_metadata		metadata;
-	unsigned long			feature_mask;
+	struct amdxdna_drm_query_aie_version version;
 	struct aie2_exec_msg_ops	*exec_msg_ops;
 
 	/* power management and clock*/
@@ -237,63 +173,58 @@ struct amdxdna_dev_hdl {
 
 	/* Mailbox and the management channel */
 	struct mailbox			*mbox;
-	struct mailbox_channel		*mgmt_chann;
 	struct async_events		*async_events;
 
 	enum aie2_dev_status		dev_status;
 	u32				hwctx_num;
 
 	struct amdxdna_async_error	last_async_err;
-};
-
-#define DEFINE_BAR_OFFSET(reg_name, bar, reg_addr) \
-	[reg_name] = {bar##_BAR_INDEX, (reg_addr) - bar##_BAR_BASE}
-
-struct aie2_bar_off_pair {
-	int	bar_idx;
-	u32	offset;
+	enum aie2_tdr_status		tdr_status;
 };
 
 struct aie2_hw_ops {
 	int (*set_dpm)(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
+	int (*update_counters)(struct amdxdna_dev_hdl *ndev);
 };
 
+#define aie2_update_counters(ndev)				\
+({								\
+	typeof(ndev) _ndev = ndev;				\
+	if (_ndev->priv->hw_ops->update_counters)		\
+		_ndev->priv->hw_ops->update_counters(_ndev);	\
+})
+
 enum aie2_fw_feature {
 	AIE2_NPU_COMMAND,
 	AIE2_PREEMPT,
 	AIE2_TEMPORAL_ONLY,
 	AIE2_APP_HEALTH,
+	AIE2_ADD_HOST_BUFFER,
+	AIE2_UPDATE_PROPERTY,
+	AIE2_GET_DEV_REVISION,
 	AIE2_FEATURE_MAX
 };
 
-struct aie2_fw_feature_tbl {
-	u64 features;
-	u32 major;
-	u32 max_minor;
-	u32 min_minor;
-};
-
 #define AIE2_ALL_FEATURES	GENMASK_ULL(AIE2_FEATURE_MAX - 1, AIE2_NPU_COMMAND)
-#define AIE2_FEATURE_ON(ndev, feature)	test_bit(feature, &(ndev)->feature_mask)
 
 struct amdxdna_dev_priv {
 	const char			*fw_path;
 	const struct rt_config		*rt_config;
 	const struct dpm_clk_freq	*dpm_clk_tbl;
-	const struct aie2_fw_feature_tbl *fw_feature_tbl;
 
 #define COL_ALIGN_NONE   0
 #define COL_ALIGN_NATURE 1
 	u32				col_align;
+	u32				col_opc;
 	u32				mbox_dev_addr;
 	/* If mbox_size is 0, use BAR size. See MBOX_SIZE macro */
 	u32				mbox_size;
 	u32				hwctx_limit;
 	u32				sram_dev_addr;
-	struct aie2_bar_off_pair	sram_offs[SRAM_MAX_INDEX];
-	struct aie2_bar_off_pair	psp_regs_off[PSP_MAX_REGS];
-	struct aie2_bar_off_pair	smu_regs_off[SMU_MAX_REGS];
-	struct aie2_hw_ops		hw_ops;
+	struct aie_bar_off_pair		sram_offs[SRAM_MAX_INDEX];
+	struct aie_bar_off_pair		psp_regs_off[PSP_MAX_REGS];
+	struct aie_bar_off_pair		smu_regs_off[SMU_MAX_REGS];
+	const struct aie2_hw_ops	*hw_ops;
 };
 
 extern const struct amdxdna_dev_ops aie2_ops;
@@ -306,25 +237,15 @@ extern const struct dpm_clk_freq npu1_dpm_clk_table[];
 extern const struct dpm_clk_freq npu4_dpm_clk_table[];
 extern const struct rt_config npu1_default_rt_cfg[];
 extern const struct rt_config npu4_default_rt_cfg[];
-extern const struct aie2_fw_feature_tbl npu4_fw_feature_table[];
-
-/* aie2_smu.c */
-int aie2_smu_init(struct amdxdna_dev_hdl *ndev);
-void aie2_smu_fini(struct amdxdna_dev_hdl *ndev);
-int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
-int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
+extern const struct amdxdna_fw_feature_tbl npu4_fw_feature_table[];
+extern const struct amdxdna_rev_vbnv npu4_rev_vbnv_tbl[];
+extern const struct aie2_hw_ops npu4_hw_ops;
 
 /* aie2_pm.c */
 int aie2_pm_init(struct amdxdna_dev_hdl *ndev);
 int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target);
 int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level);
 
-/* aie2_psp.c */
-struct psp_device *aie2m_psp_create(struct drm_device *ddev, struct psp_config *conf);
-int aie2_psp_start(struct psp_device *psp);
-void aie2_psp_stop(struct psp_device *psp);
-int aie2_psp_waitmode_poll(struct psp_device *psp);
-
 /* aie2_error.c */
 int aie2_error_async_events_alloc(struct amdxdna_dev_hdl *ndev);
 void aie2_error_async_events_free(struct amdxdna_dev_hdl *ndev);
@@ -340,15 +261,19 @@ int aie2_resume_fw(struct amdxdna_dev_hdl *ndev);
 int aie2_set_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 value);
 int aie2_get_runtime_cfg(struct amdxdna_dev_hdl *ndev, u32 type, u64 *value);
 int aie2_assign_mgmt_pasid(struct amdxdna_dev_hdl *ndev, u16 pasid);
-int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev, struct aie_version *version);
-int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev, struct aie_metadata *metadata);
+int aie2_query_aie_version(struct amdxdna_dev_hdl *ndev,
+			   struct amdxdna_drm_query_aie_version *version);
+int aie2_query_aie_metadata(struct amdxdna_dev_hdl *ndev,
+			    struct amdxdna_drm_query_aie_metadata *metadata);
 int aie2_query_firmware_version(struct amdxdna_dev_hdl *ndev,
 				struct amdxdna_fw_ver *fw_ver);
 int aie2_query_app_health(struct amdxdna_dev_hdl *ndev, u32 context_id,
 			  struct app_health_report *report);
+int aie2_get_dev_revision(struct amdxdna_dev_hdl *ndev, enum aie2_dev_revision *rev);
 int aie2_create_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx);
 int aie2_destroy_context(struct amdxdna_dev_hdl *ndev, struct amdxdna_hwctx *hwctx);
 int aie2_map_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size);
+int aie2_add_host_buf(struct amdxdna_dev_hdl *ndev, u32 context_id, u64 addr, u64 size);
 int aie2_query_status(struct amdxdna_dev_hdl *ndev, char __user *buf, u32 size, u32 *cols_filled);
 int aie2_query_telemetry(struct amdxdna_dev_hdl *ndev,
 			 char __user *buf, u32 size,
@@ -369,10 +294,7 @@ int aie2_sync_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
 		 int (*notify_cb)(void *, void __iomem *, size_t));
 int aie2_config_debug_bo(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job,
 			 int (*notify_cb)(void *, void __iomem *, size_t));
-void *aie2_alloc_msg_buffer(struct amdxdna_dev_hdl *ndev, u32 *size,
-			    dma_addr_t *dma_addr);
-void aie2_free_msg_buffer(struct amdxdna_dev_hdl *ndev, size_t size,
-			  void *cpu_addr, dma_addr_t dma_addr);
+int aie2_update_prop_time_quota(struct amdxdna_dev_hdl *ndev, u32 us);
 
 /* aie2_hwctx.c */
 int aie2_hwctx_init(struct amdxdna_hwctx *hwctx);
@@ -383,5 +305,6 @@ void aie2_hwctx_suspend(struct amdxdna_client *client);
 int aie2_hwctx_resume(struct amdxdna_client *client);
 int aie2_cmd_submit(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq);
 void aie2_hmm_invalidate(struct amdxdna_gem_obj *abo, unsigned long cur_seq);
+int aie2_hwctx_heap_expand(struct amdxdna_hwctx *hwctx, struct amdxdna_gem_obj *heap);
 
 #endif /* _AIE2_PCI_H_ */
diff --git a/drivers/accel/amdxdna/aie2_pm.c b/drivers/accel/amdxdna/aie2_pm.c
index 29bd4403a94d4..4fe6030d2c417 100644
--- a/drivers/accel/amdxdna/aie2_pm.c
+++ b/drivers/accel/amdxdna/aie2_pm.c
@@ -31,14 +31,14 @@ int aie2_pm_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
 {
 	int ret;
 
-	ret = amdxdna_pm_resume_get_locked(ndev->xdna);
+	ret = amdxdna_pm_resume_get_locked(ndev->aie.xdna);
 	if (ret)
 		return ret;
 
-	ret = ndev->priv->hw_ops.set_dpm(ndev, dpm_level);
+	ret = ndev->priv->hw_ops->set_dpm(ndev, dpm_level);
 	if (!ret)
 		ndev->dpm_level = dpm_level;
-	amdxdna_pm_suspend_put(ndev->xdna);
+	amdxdna_pm_suspend_put(ndev->aie.xdna);
 
 	return ret;
 }
@@ -49,7 +49,7 @@ int aie2_pm_init(struct amdxdna_dev_hdl *ndev)
 
 	if (ndev->dev_status != AIE2_DEV_UNINIT) {
 		/* Resume device */
-		ret = ndev->priv->hw_ops.set_dpm(ndev, ndev->dpm_level);
+		ret = ndev->priv->hw_ops->set_dpm(ndev, ndev->dpm_level);
 		if (ret)
 			return ret;
 
@@ -64,7 +64,7 @@ int aie2_pm_init(struct amdxdna_dev_hdl *ndev)
 		ndev->max_dpm_level++;
 	ndev->max_dpm_level--;
 
-	ret = ndev->priv->hw_ops.set_dpm(ndev, ndev->max_dpm_level);
+	ret = ndev->priv->hw_ops->set_dpm(ndev, ndev->max_dpm_level);
 	if (ret)
 		return ret;
 	ndev->dpm_level = ndev->max_dpm_level;
@@ -74,14 +74,14 @@ int aie2_pm_init(struct amdxdna_dev_hdl *ndev)
 		return ret;
 
 	ndev->pw_mode = POWER_MODE_DEFAULT;
-	ndev->dft_dpm_level = ndev->max_dpm_level;
+	ndev->dft_dpm_level = 0;
 
 	return 0;
 }
 
 int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type target)
 {
-	struct amdxdna_dev *xdna = ndev->xdna;
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
 	u32 clk_gating, dpm_level;
 	int ret;
 
@@ -108,6 +108,14 @@ int aie2_pm_set_mode(struct amdxdna_dev_hdl *ndev, enum amdxdna_power_mode_type
 		clk_gating = AIE2_CLK_GATING_ENABLE;
 		dpm_level = ndev->dft_dpm_level;
 		break;
+	case POWER_MODE_LOW:
+		clk_gating = AIE2_CLK_GATING_ENABLE;
+		dpm_level = 0;
+		break;
+	case POWER_MODE_MEDIUM:
+		clk_gating = AIE2_CLK_GATING_ENABLE;
+		dpm_level = ndev->max_dpm_level / 2;
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
diff --git a/drivers/accel/amdxdna/aie2_psp.c b/drivers/accel/amdxdna/aie2_psp.c
deleted file mode 100644
index 3a7130577e3e2..0000000000000
--- a/drivers/accel/amdxdna/aie2_psp.c
+++ /dev/null
@@ -1,161 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
- */
-
-#include <drm/drm_device.h>
-#include <drm/drm_gem_shmem_helper.h>
-#include <drm/drm_managed.h>
-#include <drm/drm_print.h>
-#include <drm/gpu_scheduler.h>
-#include <linux/bitfield.h>
-#include <linux/iopoll.h>
-
-#include "aie2_pci.h"
-#include "amdxdna_mailbox.h"
-#include "amdxdna_pci_drv.h"
-
-#define PSP_STATUS_READY	BIT(31)
-
-/* PSP commands */
-#define PSP_VALIDATE		1
-#define PSP_START		2
-#define PSP_RELEASE_TMR		3
-
-/* PSP special arguments */
-#define PSP_START_COPY_FW	1
-
-/* PSP response error code */
-#define PSP_ERROR_CANCEL	0xFFFF0002
-#define PSP_ERROR_BAD_STATE	0xFFFF0007
-
-#define PSP_FW_ALIGN		0x10000
-#define PSP_POLL_INTERVAL	20000	/* us */
-#define PSP_POLL_TIMEOUT	1000000	/* us */
-
-#define PSP_REG(p, reg) ((p)->psp_regs[reg])
-
-struct psp_device {
-	struct drm_device	*ddev;
-	struct psp_config	conf;
-	u32			fw_buf_sz;
-	u64			fw_paddr;
-	void			*fw_buffer;
-	void __iomem		*psp_regs[PSP_MAX_REGS];
-};
-
-static int psp_exec(struct psp_device *psp, u32 *reg_vals)
-{
-	u32 resp_code;
-	int ret, i;
-	u32 ready;
-
-	/* Write command and argument registers */
-	for (i = 0; i < PSP_NUM_IN_REGS; i++)
-		writel(reg_vals[i], PSP_REG(psp, i));
-
-	/* clear and set PSP INTR register to kick off */
-	writel(0, PSP_REG(psp, PSP_INTR_REG));
-	writel(1, PSP_REG(psp, PSP_INTR_REG));
-
-	/* PSP should be busy. Wait for ready, so we know task is done. */
-	ret = readx_poll_timeout(readl, PSP_REG(psp, PSP_STATUS_REG), ready,
-				 FIELD_GET(PSP_STATUS_READY, ready),
-				 PSP_POLL_INTERVAL, PSP_POLL_TIMEOUT);
-	if (ret) {
-		drm_err(psp->ddev, "PSP is not ready, ret 0x%x", ret);
-		return ret;
-	}
-
-	resp_code = readl(PSP_REG(psp, PSP_RESP_REG));
-	if (resp_code) {
-		drm_err(psp->ddev, "fw return error 0x%x", resp_code);
-		return -EIO;
-	}
-
-	return 0;
-}
-
-int aie2_psp_waitmode_poll(struct psp_device *psp)
-{
-	struct amdxdna_dev *xdna = to_xdna_dev(psp->ddev);
-	u32 mode_reg;
-	int ret;
-
-	ret = readx_poll_timeout(readl, PSP_REG(psp, PSP_PWAITMODE_REG), mode_reg,
-				 (mode_reg & 0x1) == 1,
-				 PSP_POLL_INTERVAL, PSP_POLL_TIMEOUT);
-	if (ret)
-		XDNA_ERR(xdna, "fw waitmode reg error, ret %d", ret);
-
-	return ret;
-}
-
-void aie2_psp_stop(struct psp_device *psp)
-{
-	u32 reg_vals[PSP_NUM_IN_REGS] = { PSP_RELEASE_TMR, };
-	int ret;
-
-	ret = psp_exec(psp, reg_vals);
-	if (ret)
-		drm_err(psp->ddev, "release tmr failed, ret %d", ret);
-}
-
-int aie2_psp_start(struct psp_device *psp)
-{
-	u32 reg_vals[PSP_NUM_IN_REGS];
-	int ret;
-
-	reg_vals[0] = PSP_VALIDATE;
-	reg_vals[1] = lower_32_bits(psp->fw_paddr);
-	reg_vals[2] = upper_32_bits(psp->fw_paddr);
-	reg_vals[3] = psp->fw_buf_sz;
-
-	ret = psp_exec(psp, reg_vals);
-	if (ret) {
-		drm_err(psp->ddev, "failed to validate fw, ret %d", ret);
-		return ret;
-	}
-
-	memset(reg_vals, 0, sizeof(reg_vals));
-	reg_vals[0] = PSP_START;
-	reg_vals[1] = PSP_START_COPY_FW;
-	ret = psp_exec(psp, reg_vals);
-	if (ret) {
-		drm_err(psp->ddev, "failed to start fw, ret %d", ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-struct psp_device *aie2m_psp_create(struct drm_device *ddev, struct psp_config *conf)
-{
-	struct psp_device *psp;
-	u64 offset;
-
-	psp = drmm_kzalloc(ddev, sizeof(*psp), GFP_KERNEL);
-	if (!psp)
-		return NULL;
-
-	psp->ddev = ddev;
-	memcpy(psp->psp_regs, conf->psp_regs, sizeof(psp->psp_regs));
-
-	psp->fw_buf_sz = ALIGN(conf->fw_size, PSP_FW_ALIGN);
-	psp->fw_buffer = drmm_kmalloc(ddev, psp->fw_buf_sz + PSP_FW_ALIGN, GFP_KERNEL);
-	if (!psp->fw_buffer) {
-		drm_err(ddev, "no memory for fw buffer");
-		return NULL;
-	}
-
-	/*
-	 * AMD Platform Security Processor(PSP) requires host physical
-	 * address to load NPU firmware.
-	 */
-	psp->fw_paddr = virt_to_phys(psp->fw_buffer);
-	offset = ALIGN(psp->fw_paddr, PSP_FW_ALIGN) - psp->fw_paddr;
-	psp->fw_paddr += offset;
-	memcpy(psp->fw_buffer + offset, conf->fw_buf, conf->fw_size);
-
-	return psp;
-}
diff --git a/drivers/accel/amdxdna/aie2_smu.c b/drivers/accel/amdxdna/aie2_smu.c
deleted file mode 100644
index d8c31924e501b..0000000000000
--- a/drivers/accel/amdxdna/aie2_smu.c
+++ /dev/null
@@ -1,156 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
- */
-
-#include <drm/drm_device.h>
-#include <drm/drm_gem_shmem_helper.h>
-#include <drm/drm_print.h>
-#include <drm/gpu_scheduler.h>
-#include <linux/iopoll.h>
-
-#include "aie2_pci.h"
-#include "amdxdna_pci_drv.h"
-
-#define SMU_RESULT_OK		1
-
-/* SMU commands */
-#define AIE2_SMU_POWER_ON		0x3
-#define AIE2_SMU_POWER_OFF		0x4
-#define AIE2_SMU_SET_MPNPUCLK_FREQ	0x5
-#define AIE2_SMU_SET_HCLK_FREQ		0x6
-#define AIE2_SMU_SET_SOFT_DPMLEVEL	0x7
-#define AIE2_SMU_SET_HARD_DPMLEVEL	0x8
-
-#define NPU4_DPM_TOPS(ndev, dpm_level) \
-({ \
-	typeof(ndev) _ndev = ndev; \
-	(4096 * (_ndev)->total_col * \
-	 (_ndev)->priv->dpm_clk_tbl[dpm_level].hclk / 1000000); \
-})
-
-static int aie2_smu_exec(struct amdxdna_dev_hdl *ndev, u32 reg_cmd,
-			 u32 reg_arg, u32 *out)
-{
-	u32 resp;
-	int ret;
-
-	writel(0, SMU_REG(ndev, SMU_RESP_REG));
-	writel(reg_arg, SMU_REG(ndev, SMU_ARG_REG));
-	writel(reg_cmd, SMU_REG(ndev, SMU_CMD_REG));
-
-	/* Clear and set SMU_INTR_REG to kick off */
-	writel(0, SMU_REG(ndev, SMU_INTR_REG));
-	writel(1, SMU_REG(ndev, SMU_INTR_REG));
-
-	ret = readx_poll_timeout(readl, SMU_REG(ndev, SMU_RESP_REG), resp,
-				 resp, AIE2_INTERVAL, AIE2_TIMEOUT);
-	if (ret) {
-		XDNA_ERR(ndev->xdna, "smu cmd %d timed out", reg_cmd);
-		return ret;
-	}
-
-	if (out)
-		*out = readl(SMU_REG(ndev, SMU_OUT_REG));
-
-	if (resp != SMU_RESULT_OK) {
-		XDNA_ERR(ndev->xdna, "smu cmd %d failed, 0x%x", reg_cmd, resp);
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
-{
-	u32 freq;
-	int ret;
-
-	ret = aie2_smu_exec(ndev, AIE2_SMU_SET_MPNPUCLK_FREQ,
-			    ndev->priv->dpm_clk_tbl[dpm_level].npuclk, &freq);
-	if (ret) {
-		XDNA_ERR(ndev->xdna, "Set npu clock to %d failed, ret %d\n",
-			 ndev->priv->dpm_clk_tbl[dpm_level].npuclk, ret);
-		return ret;
-	}
-	ndev->npuclk_freq = freq;
-
-	ret = aie2_smu_exec(ndev, AIE2_SMU_SET_HCLK_FREQ,
-			    ndev->priv->dpm_clk_tbl[dpm_level].hclk, &freq);
-	if (ret) {
-		XDNA_ERR(ndev->xdna, "Set h clock to %d failed, ret %d\n",
-			 ndev->priv->dpm_clk_tbl[dpm_level].hclk, ret);
-		return ret;
-	}
-
-	ndev->hclk_freq = freq;
-	ndev->max_tops = 2 * ndev->total_col;
-	ndev->curr_tops = ndev->max_tops * freq / 1028;
-
-	XDNA_DBG(ndev->xdna, "MP-NPU clock %d, H clock %d\n",
-		 ndev->npuclk_freq, ndev->hclk_freq);
-
-	return 0;
-}
-
-int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
-{
-	int ret;
-
-	ret = aie2_smu_exec(ndev, AIE2_SMU_SET_HARD_DPMLEVEL, dpm_level, NULL);
-	if (ret) {
-		XDNA_ERR(ndev->xdna, "Set hard dpm level %d failed, ret %d ",
-			 dpm_level, ret);
-		return ret;
-	}
-
-	ret = aie2_smu_exec(ndev, AIE2_SMU_SET_SOFT_DPMLEVEL, dpm_level, NULL);
-	if (ret) {
-		XDNA_ERR(ndev->xdna, "Set soft dpm level %d failed, ret %d",
-			 dpm_level, ret);
-		return ret;
-	}
-
-	ndev->npuclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].npuclk;
-	ndev->hclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].hclk;
-	ndev->max_tops = NPU4_DPM_TOPS(ndev, ndev->max_dpm_level);
-	ndev->curr_tops = NPU4_DPM_TOPS(ndev, dpm_level);
-
-	XDNA_DBG(ndev->xdna, "MP-NPU clock %d, H clock %d\n",
-		 ndev->npuclk_freq, ndev->hclk_freq);
-
-	return 0;
-}
-
-int aie2_smu_init(struct amdxdna_dev_hdl *ndev)
-{
-	int ret;
-
-	/*
-	 * Failing to set power off indicates an unrecoverable hardware or
-	 * firmware error.
-	 */
-	ret = aie2_smu_exec(ndev, AIE2_SMU_POWER_OFF, 0, NULL);
-	if (ret) {
-		XDNA_ERR(ndev->xdna, "Access power failed, ret %d", ret);
-		return ret;
-	}
-
-	ret = aie2_smu_exec(ndev, AIE2_SMU_POWER_ON, 0, NULL);
-	if (ret) {
-		XDNA_ERR(ndev->xdna, "Power on failed, ret %d", ret);
-		return ret;
-	}
-
-	return 0;
-}
-
-void aie2_smu_fini(struct amdxdna_dev_hdl *ndev)
-{
-	int ret;
-
-	ndev->priv->hw_ops.set_dpm(ndev, 0);
-	ret = aie2_smu_exec(ndev, AIE2_SMU_POWER_OFF, 0, NULL);
-	if (ret)
-		XDNA_ERR(ndev->xdna, "Power off failed, ret %d", ret);
-}
diff --git a/drivers/accel/amdxdna/aie2_solver.c b/drivers/accel/amdxdna/aie2_solver.c
index 3611e3268d791..6f3ee77d5264f 100644
--- a/drivers/accel/amdxdna/aie2_solver.c
+++ b/drivers/accel/amdxdna/aie2_solver.c
@@ -52,7 +52,7 @@ static u32 calculate_gops(struct aie_qos *rqos)
 	u32 service_rate = 0;
 
 	if (rqos->latency)
-		service_rate = (1000 / rqos->latency);
+		service_rate = max_t(u32, 1000 / rqos->latency, 1);
 
 	if (rqos->fps > service_rate)
 		return rqos->fps * rqos->gops;
@@ -348,6 +348,7 @@ int xrs_release_resource(void *hdl, u64 rid)
 {
 	struct solver_state *xrs = hdl;
 	struct solver_node *node;
+	u32 level = 0;
 
 	node = rg_search_node(&xrs->rgp, rid);
 	if (!node) {
@@ -358,6 +359,13 @@ int xrs_release_resource(void *hdl, u64 rid)
 	xrs->cfg.actions->unload(node->cb_arg);
 	remove_solver_node(&xrs->rgp, node);
 
+	/* set the dpm level which fits all the sessions */
+	list_for_each_entry(node, &xrs->rgp.node_list, list) {
+		if (node->dpm_level > level)
+			level = node->dpm_level;
+	}
+	xrs->cfg.actions->set_dft_dpm_level(xrs->cfg.ddev, level);
+
 	return 0;
 }
 
diff --git a/drivers/accel/amdxdna/aie4_ctx.c b/drivers/accel/amdxdna/aie4_ctx.c
new file mode 100644
index 0000000000000..8408b0d2696f6
--- /dev/null
+++ b/drivers/accel/amdxdna/aie4_ctx.c
@@ -0,0 +1,333 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <drm/amdxdna_accel.h>
+#include <drm/drm_device.h>
+#include <drm/drm_gem.h>
+#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_print.h>
+#include <drm/gpu_scheduler.h>
+#include <linux/types.h>
+
+#include "aie.h"
+#include "aie4_host_queue.h"
+#include "aie4_msg_priv.h"
+#include "aie4_pci.h"
+#include "amdxdna_ctx.h"
+#include "amdxdna_gem.h"
+#include "amdxdna_mailbox.h"
+#include "amdxdna_mailbox_helper.h"
+#include "amdxdna_pci_drv.h"
+
+static irqreturn_t cert_comp_isr(int irq, void *p)
+{
+	struct cert_comp *cert_comp = p;
+
+	wake_up_all(&cert_comp->waitq);
+	return IRQ_HANDLED;
+}
+
+static struct cert_comp *aie4_lookup_cert_comp(struct amdxdna_dev_hdl *ndev, u32 msix_idx)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
+	struct cert_comp *cert_comp;
+	int ret;
+
+	guard(mutex)(&ndev->cert_comp_lock);
+
+	cert_comp = xa_load(&ndev->cert_comp_xa, msix_idx);
+	if (cert_comp) {
+		kref_get(&cert_comp->kref);
+		return cert_comp;
+	}
+
+	cert_comp = kzalloc_obj(*cert_comp);
+	if (!cert_comp)
+		return NULL;
+
+	cert_comp->ndev = ndev;
+	cert_comp->msix_idx = msix_idx;
+	init_waitqueue_head(&cert_comp->waitq);
+	kref_init(&cert_comp->kref);
+
+	ret = pci_irq_vector(pdev, cert_comp->msix_idx);
+	if (ret < 0) {
+		XDNA_ERR(xdna, "MSI-X idx %u is invalid, ret:%d", msix_idx, ret);
+		goto free_cert_comp;
+	}
+	cert_comp->irq = ret;
+
+	ret = request_irq(cert_comp->irq, cert_comp_isr, 0, "xdna_hsa", cert_comp);
+	if (ret) {
+		XDNA_ERR(xdna, "request irq %d failed %d", cert_comp->irq, ret);
+		goto free_cert_comp;
+	}
+
+	ret = xa_err(xa_store(&ndev->cert_comp_xa, msix_idx, cert_comp, GFP_KERNEL));
+	if (ret) {
+		XDNA_ERR(xdna, "store cert_comp for msix idx %d failed %d", msix_idx, ret);
+		goto free_irq;
+	}
+
+	return cert_comp;
+
+free_irq:
+	free_irq(cert_comp->irq, cert_comp);
+free_cert_comp:
+	kfree(cert_comp);
+	return NULL;
+}
+
+static void cert_comp_release(struct kref *kref)
+{
+	struct cert_comp *cert_comp = container_of(kref, struct cert_comp, kref);
+	struct amdxdna_dev_hdl *ndev = cert_comp->ndev;
+
+	drm_WARN_ON(&ndev->aie.xdna->ddev, !mutex_is_locked(&ndev->cert_comp_lock));
+
+	xa_erase(&ndev->cert_comp_xa, cert_comp->msix_idx);
+	free_irq(cert_comp->irq, cert_comp);
+	kfree(cert_comp);
+}
+
+static void aie4_put_cert_comp(struct cert_comp *cert_comp)
+{
+	struct amdxdna_dev_hdl *ndev;
+
+	ndev = cert_comp->ndev;
+	guard(mutex)(&ndev->cert_comp_lock);
+	kref_put(&cert_comp->kref, cert_comp_release);
+}
+
+static int aie4_msg_destroy_context(struct amdxdna_dev_hdl *ndev, u32 hw_context_id)
+{
+	DECLARE_AIE_MSG(aie4_msg_destroy_hw_context, AIE4_MSG_OP_DESTROY_HW_CONTEXT);
+
+	req.hw_context_id = hw_context_id;
+	return aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+}
+
+static int aie4_hwctx_create(struct amdxdna_hwctx *hwctx)
+{
+	DECLARE_AIE_MSG(aie4_msg_create_hw_context, AIE4_MSG_OP_CREATE_HW_CONTEXT);
+	struct amdxdna_client *client = hwctx->client;
+	struct amdxdna_hwctx_priv *priv = hwctx->priv;
+	struct amdxdna_dev *xdna = hwctx->client->xdna;
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+	int ret;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+
+	if (!ndev->partition_id || !hwctx->num_tiles) {
+		XDNA_ERR(xdna, "invalid request partition_id %d, num_tiles %d",
+			 ndev->partition_id, hwctx->num_tiles);
+		return -EINVAL;
+	}
+
+	req.partition_id = ndev->partition_id;
+	req.request_num_tiles = hwctx->num_tiles;
+	req.pasid = FIELD_PREP(AIE4_MSG_PASID, client->pasid) |
+		FIELD_PREP(AIE4_MSG_PASID_VLD, 1);
+	req.priority_band = hwctx->qos.priority;
+
+	req.hsa_addr_high = upper_32_bits(amdxdna_gem_dev_addr(priv->umq_bo));
+	req.hsa_addr_low = lower_32_bits(amdxdna_gem_dev_addr(priv->umq_bo));
+
+	XDNA_DBG(xdna, "pasid 0x%x, num_tiles %d, hsa[0x%x 0x%x]",
+		 req.pasid, req.request_num_tiles, req.hsa_addr_high, req.hsa_addr_low);
+
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret) {
+		XDNA_ERR(xdna, "create ctx failed: %d", ret);
+		return ret;
+	}
+
+	XDNA_DBG(xdna, "resp msix: %d, ctx id: %d, doorbell: %d",
+		 resp.job_complete_msix_idx,
+		 resp.hw_context_id,
+		 resp.doorbell_offset);
+
+	/* setup interrupt completion per msix index */
+	priv->cert_comp = aie4_lookup_cert_comp(ndev, resp.job_complete_msix_idx);
+	if (!priv->cert_comp) {
+		aie4_msg_destroy_context(ndev, resp.hw_context_id);
+		return -EINVAL;
+	}
+
+	priv->hw_ctx_id = resp.hw_context_id;
+	hwctx->doorbell_offset = resp.doorbell_offset;
+
+	return 0;
+}
+
+static void aie4_hwctx_destroy(struct amdxdna_hwctx *hwctx)
+{
+	struct amdxdna_client *client = hwctx->client;
+	struct amdxdna_hwctx_priv *priv = hwctx->priv;
+	struct amdxdna_dev *xdna = client->xdna;
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+
+	aie4_msg_destroy_context(ndev, priv->hw_ctx_id);
+	aie4_put_cert_comp(priv->cert_comp);
+}
+
+static void aie4_hwctx_umq_fini(struct amdxdna_hwctx *hwctx)
+{
+	if (hwctx->priv && hwctx->priv->umq_bo)
+		amdxdna_gem_put_obj(hwctx->priv->umq_bo);
+}
+
+static int aie4_hwctx_umq_init(struct amdxdna_hwctx *hwctx)
+{
+	struct amdxdna_hwctx_priv *priv = hwctx->priv;
+	struct amdxdna_dev *xdna = hwctx->client->xdna;
+	struct amdxdna_gem_obj *umq_bo;
+	struct host_queue_header *qhdr;
+	int ret;
+
+	umq_bo = amdxdna_gem_get_obj(hwctx->client, hwctx->umq_bo_hdl, AMDXDNA_BO_SHARE);
+	if (!umq_bo) {
+		XDNA_ERR(xdna, "cannot find umq_bo handle %d", hwctx->umq_bo_hdl);
+		return -ENOENT;
+	}
+	if (umq_bo->mem.size < sizeof(*qhdr)) {
+		XDNA_ERR(xdna, "umq_bo size is too small");
+		ret = -EINVAL;
+		goto put_umq_bo;
+	}
+
+	/* get kva address for host queue read index and write index */
+	qhdr = amdxdna_gem_vmap(umq_bo);
+	if (!qhdr) {
+		ret = -ENOMEM;
+		goto put_umq_bo;
+	}
+
+	priv->umq_bo = umq_bo;
+	priv->umq_read_index = &qhdr->read_index;
+	priv->umq_write_index = &qhdr->write_index;
+
+	return 0;
+
+put_umq_bo:
+	amdxdna_gem_put_obj(umq_bo);
+	return ret;
+}
+
+int aie4_hwctx_init(struct amdxdna_hwctx *hwctx)
+{
+	struct amdxdna_client *client = hwctx->client;
+	struct amdxdna_dev *xdna = client->xdna;
+	struct amdxdna_hwctx_priv *priv;
+	int ret;
+
+	priv = kzalloc_obj(*priv);
+	if (!priv)
+		return -ENOMEM;
+	hwctx->priv = priv;
+
+	ret = aie4_hwctx_umq_init(hwctx);
+	if (ret)
+		goto free_priv;
+
+	ret = aie4_hwctx_create(hwctx);
+	if (ret)
+		goto umq_fini;
+
+	XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name);
+	return 0;
+
+umq_fini:
+	aie4_hwctx_umq_fini(hwctx);
+free_priv:
+	kfree(priv);
+	hwctx->priv = NULL;
+	return ret;
+}
+
+void aie4_hwctx_fini(struct amdxdna_hwctx *hwctx)
+{
+	aie4_hwctx_destroy(hwctx);
+	aie4_hwctx_umq_fini(hwctx);
+	kfree(hwctx->priv);
+}
+
+static inline bool valid_queue_index(u64 read, u64 write, u32 capacity)
+{
+	return (write >= read) && ((write - read) <= capacity);
+}
+
+static u64 get_read_index(struct amdxdna_hwctx *hwctx)
+{
+	u64 wi = READ_ONCE(*hwctx->priv->umq_write_index);
+	u64 ri = READ_ONCE(*hwctx->priv->umq_read_index);
+	struct amdxdna_dev *xdna = hwctx->client->xdna;
+
+	/*
+	 * CERT cannot update read index as uint64 atomically. Driver may read
+	 * half-updated read index when it has bits in high 32bit. In case read
+	 * index is not valid, wait for some time and retry once. It should
+	 * allow CERT to complete the read index update.
+	 */
+	if (!valid_queue_index(ri, wi, CTX_MAX_CMDS)) {
+		XDNA_WARN(xdna, "Invalid index, ri %llu, wi %llu", ri, wi);
+		usleep_range(100, 200);
+		ri = READ_ONCE(*hwctx->priv->umq_read_index);
+		if (!valid_queue_index(ri, wi, CTX_MAX_CMDS)) {
+			XDNA_ERR(xdna, "Invalid index after retry, ri %llu, wi %llu", ri, wi);
+			ri = 0;
+		}
+	}
+
+	return ri;
+}
+
+static inline bool check_cmd_done(struct amdxdna_hwctx *hwctx, u64 seq)
+{
+	u64 read_idx = get_read_index(hwctx);
+
+	return read_idx > seq;
+}
+
+int aie4_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq, u32 timeout)
+{
+	unsigned long wait_jifs = MAX_SCHEDULE_TIMEOUT;
+	struct amdxdna_hwctx_priv *priv = hwctx->priv;
+	struct cert_comp *cert_comp = priv->cert_comp;
+	long ret;
+
+	if (timeout)
+		wait_jifs = msecs_to_jiffies(timeout);
+
+	ret = wait_event_interruptible_timeout(cert_comp->waitq,
+					       (check_cmd_done(hwctx, seq)),
+					       wait_jifs);
+
+	if (!ret)
+		ret = -ETIME;
+
+	return ret <= 0 ? ret : 0;
+}
+
+int aie4_hwctx_valid_doorbell(struct amdxdna_client *client, u32 vm_pgoff)
+{
+	struct amdxdna_hwctx *hwctx;
+	unsigned long hwctx_id;
+	int idx;
+
+	idx = srcu_read_lock(&client->hwctx_srcu);
+	amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
+		if (vm_pgoff == (hwctx->doorbell_offset >> PAGE_SHIFT)) {
+			srcu_read_unlock(&client->hwctx_srcu, idx);
+			return 1;
+		}
+	}
+	srcu_read_unlock(&client->hwctx_srcu, idx);
+
+	return 0;
+}
diff --git a/drivers/accel/amdxdna/aie4_host_queue.h b/drivers/accel/amdxdna/aie4_host_queue.h
new file mode 100644
index 0000000000000..1b33eda3f7278
--- /dev/null
+++ b/drivers/accel/amdxdna/aie4_host_queue.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _AIE4_HOST_QUEUE_H_
+#define _AIE4_HOST_QUEUE_H_
+
+#include <linux/types.h>
+
+#define CTX_MAX_CMDS                    32
+
+struct host_queue_header {
+	__u64 read_index;
+	struct {
+		__u16 major;
+		__u16 minor;
+	} version;
+	__u32 capacity; /* Queue capacity, must be power of two. */
+	__u64 write_index;
+	__u64 data_address; /* The xdna dev addr for payload. */
+};
+
+#endif /* _AIE4_HOST_QUEUE_H_ */
diff --git a/drivers/accel/amdxdna/aie4_message.c b/drivers/accel/amdxdna/aie4_message.c
new file mode 100644
index 0000000000000..88037edbb02aa
--- /dev/null
+++ b/drivers/accel/amdxdna/aie4_message.c
@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <drm/amdxdna_accel.h>
+#include <drm/drm_print.h>
+#include <linux/mutex.h>
+
+#include "aie.h"
+#include "aie4_msg_priv.h"
+#include "aie4_pci.h"
+#include "amdxdna_mailbox.h"
+#include "amdxdna_mailbox_helper.h"
+#include "amdxdna_pci_drv.h"
+
+int aie4_suspend_fw(struct amdxdna_dev_hdl *ndev)
+{
+	DECLARE_AIE_MSG(aie4_msg_suspend, AIE4_MSG_OP_SUSPEND);
+	int ret;
+
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret)
+		XDNA_ERR(ndev->aie.xdna, "Failed to suspend fw, ret %d", ret);
+
+	return ret;
+}
+
+int aie4_query_aie_metadata(struct amdxdna_dev_hdl *ndev,
+			    struct amdxdna_drm_query_aie_metadata *metadata)
+{
+	DECLARE_AIE_MSG(aie4_msg_aie4_tile_info, AIE4_MSG_OP_AIE_TILE_INFO);
+	int ret;
+
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret)
+		return ret;
+
+	metadata->col_size = resp.info.size;
+	metadata->cols = resp.info.cols;
+	metadata->rows = resp.info.rows;
+
+	metadata->version.major = resp.info.major;
+	metadata->version.minor = resp.info.minor;
+
+	metadata->core.row_count = resp.info.core_rows;
+	metadata->core.row_start = resp.info.core_row_start;
+	metadata->core.dma_channel_count = resp.info.core_dma_channels;
+	metadata->core.lock_count = resp.info.core_locks;
+	metadata->core.event_reg_count = resp.info.core_events;
+
+	metadata->mem.row_count = resp.info.mem_rows;
+	metadata->mem.row_start = resp.info.mem_row_start;
+	metadata->mem.dma_channel_count = resp.info.mem_dma_channels;
+	metadata->mem.lock_count = resp.info.mem_locks;
+	metadata->mem.event_reg_count = resp.info.mem_events;
+
+	metadata->shim.row_count = resp.info.shim_rows;
+	metadata->shim.row_start = resp.info.shim_row_start;
+	metadata->shim.dma_channel_count = resp.info.shim_dma_channels;
+	metadata->shim.lock_count = resp.info.shim_locks;
+	metadata->shim.event_reg_count = resp.info.shim_events;
+
+	return 0;
+}
+
+int aie4_attach_work_buffer(struct amdxdna_dev_hdl *ndev)
+{
+	DECLARE_AIE_MSG(aie4_msg_attach_work_buffer, AIE4_MSG_OP_ATTACH_WORK_BUFFER);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	int ret;
+
+	req.buff_addr = ndev->work_buf_addr;
+	req.buff_size = AIE4_WORK_BUFFER_MIN_SIZE;
+
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret)
+		XDNA_ERR(xdna, "Failed to attach work buffer, ret %d", ret);
+	else
+		XDNA_DBG(xdna, "Attached work buffer");
+
+	return ret;
+}
diff --git a/drivers/accel/amdxdna/aie4_msg_priv.h b/drivers/accel/amdxdna/aie4_msg_priv.h
new file mode 100644
index 0000000000000..af0866045b912
--- /dev/null
+++ b/drivers/accel/amdxdna/aie4_msg_priv.h
@@ -0,0 +1,147 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _AIE4_MSG_PRIV_H_
+#define _AIE4_MSG_PRIV_H_
+
+#include <linux/sizes.h>
+#include <linux/types.h>
+
+enum aie4_msg_opcode {
+	AIE4_MSG_OP_SUSPEND                          = 0x10003,
+	AIE4_MSG_OP_ATTACH_WORK_BUFFER               = 0x1000D,
+
+	AIE4_MSG_OP_CREATE_VFS                       = 0x20001,
+	AIE4_MSG_OP_DESTROY_VFS                      = 0x20002,
+
+	AIE4_MSG_OP_CREATE_PARTITION                 = 0x30001,
+	AIE4_MSG_OP_DESTROY_PARTITION                = 0x30002,
+	AIE4_MSG_OP_CREATE_HW_CONTEXT                = 0x30003,
+	AIE4_MSG_OP_DESTROY_HW_CONTEXT               = 0x30004,
+	AIE4_MSG_OP_AIE_TILE_INFO                    = 0x30006,
+};
+
+enum aie4_msg_status {
+	AIE4_MSG_STATUS_SUCCESS = 0x0,
+	AIE4_MSG_STATUS_ERROR = 0x1,
+	AIE4_MSG_STATUS_NOTSUPP = 0x2,
+	MAX_AIE4_MSG_STATUS_CODE = 0x4,
+};
+
+struct aie4_msg_suspend_req {
+	__u32 rsvd;
+} __packed;
+
+struct aie4_msg_suspend_resp {
+	enum aie4_msg_status status;
+} __packed;
+
+struct aie4_msg_create_vfs_req {
+	__u32 vf_cnt;
+} __packed;
+
+struct aie4_msg_create_vfs_resp {
+	enum aie4_msg_status status;
+} __packed;
+
+struct aie4_msg_destroy_vfs_req {
+	__u32 rsvd;
+} __packed;
+
+struct aie4_msg_destroy_vfs_resp {
+	enum aie4_msg_status status;
+} __packed;
+
+struct aie4_msg_create_partition_req {
+	__u32 partition_col_start;
+	__u32 partition_col_count;
+} __packed;
+
+struct aie4_msg_create_partition_resp {
+	enum aie4_msg_status status;
+	__u32 partition_id;
+} __packed;
+
+struct aie4_msg_destroy_partition_req {
+	__u32 partition_id;
+} __packed;
+
+struct aie4_msg_destroy_partition_resp {
+	enum aie4_msg_status status;
+} __packed;
+
+struct aie4_msg_create_hw_context_req {
+	__u32 partition_id;
+	__u32 request_num_tiles;
+	__u32 hsa_addr_high;
+	__u32 hsa_addr_low;
+#define AIE4_MSG_PASID GENMASK(19, 0)
+#define AIE4_MSG_PASID_VLD GENMASK(31, 31)
+	__u32 pasid;
+	__u32 priority_band;
+} __packed;
+
+struct aie4_msg_create_hw_context_resp {
+	enum aie4_msg_status status;
+	__u32 hw_context_id;
+	__u32 doorbell_offset;
+	__u32 job_complete_msix_idx;
+} __packed;
+
+struct aie4_msg_destroy_hw_context_req {
+	__u32 hw_context_id;
+	__u32 resvd1;
+} __packed;
+
+struct aie4_msg_destroy_hw_context_resp {
+	enum aie4_msg_status status;
+} __packed;
+
+struct aie4_tile_info {
+	__u32 size;
+	__u16 major;
+	__u16 minor;
+	__u16 cols;
+	__u16 rows;
+	__u16 core_rows;
+	__u16 mem_rows;
+	__u16 shim_rows;
+	__u16 core_row_start;
+	__u16 mem_row_start;
+	__u16 shim_row_start;
+	__u16 core_dma_channels;
+	__u16 mem_dma_channels;
+	__u16 shim_dma_channels;
+	__u16 core_locks;
+	__u16 mem_locks;
+	__u16 shim_locks;
+	__u16 core_events;
+	__u16 mem_events;
+	__u16 shim_events;
+	__u16 resvd;
+} __packed;
+
+struct aie4_msg_aie4_tile_info_req {
+	__u32 resvd;
+} __packed;
+
+struct aie4_msg_aie4_tile_info_resp {
+	enum aie4_msg_status status;
+	struct aie4_tile_info info;
+} __packed;
+
+#define AIE4_WORK_BUFFER_MIN_SIZE      SZ_4M
+
+struct aie4_msg_attach_work_buffer_req {
+	__u64 buff_addr;
+	__u32 reserved;
+	__u32 buff_size;
+} __packed;
+
+struct aie4_msg_attach_work_buffer_resp {
+	enum aie4_msg_status status;
+} __packed;
+
+#endif /* _AIE4_MSG_PRIV_H_ */
diff --git a/drivers/accel/amdxdna/aie4_pci.c b/drivers/accel/amdxdna/aie4_pci.c
new file mode 100644
index 0000000000000..a58a83af42a4a
--- /dev/null
+++ b/drivers/accel/amdxdna/aie4_pci.c
@@ -0,0 +1,667 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <drm/amdxdna_accel.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_print.h>
+#include <linux/firmware.h>
+#include <linux/sizes.h>
+
+#include "aie.h"
+#include "aie4_msg_priv.h"
+#include "aie4_pci.h"
+#include "amdxdna_mailbox.h"
+#include "amdxdna_mailbox_helper.h"
+#include "amdxdna_pci_drv.h"
+
+#define NO_IOHUB		0
+#define PSP_NOTIFY_INTR		0xD007BE11
+#define AIE4_TOTAL_COLUMN	3
+
+/*
+ * The management mailbox channel is allocated by firmware.
+ * The related register and ring buffer information is on SRAM BAR.
+ * This struct is the register layout.
+ */
+struct mailbox_info {
+	__u32 valid;
+	__u32 protocol_major;
+	__u32 protocol_minor;
+	__u32 x2i_tail_offset;
+	__u32 x2i_head_offset;
+	__u32 x2i_buffer_addr;
+	__u32 x2i_buffer_size;
+	__u32 i2x_tail_offset;
+	__u32 i2x_head_offset;
+	__u32 i2x_buffer_addr;
+	__u32 i2x_buffer_size;
+	__u32 i2x_msi_idx;
+	__u32 reserved[4];
+};
+
+static int aie4_fw_is_alive(struct amdxdna_dev *xdna)
+{
+	const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv;
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+	u32 __iomem *src;
+	u32 fw_is_valid;
+	int ret;
+
+	src = ndev->rbuf_base + npriv->mbox_info_off;
+
+	ret = readx_poll_timeout(readl, src + offsetof(struct mailbox_info, valid),
+				 fw_is_valid, (fw_is_valid == 0x1),
+				 AIE_INTERVAL, AIE_TIMEOUT);
+	if (ret)
+		XDNA_ERR(xdna, "fw_is_valid=%d after %d ms",
+			 fw_is_valid, DIV_ROUND_CLOSEST(AIE_TIMEOUT, 1000000));
+
+	return ret;
+}
+
+static void aie4_read_mbox_info(struct amdxdna_dev *xdna,
+				struct mailbox_info *mbox_info)
+{
+	const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv;
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+	u32 *dst = (u32 *)mbox_info;
+	u32 __iomem *src;
+	int i;
+
+	src = ndev->rbuf_base + npriv->mbox_info_off;
+
+	for (i = 0; i < sizeof(*mbox_info) / sizeof(u32); i++)
+		dst[i] = readl(&src[i]);
+}
+
+static int aie4_mailbox_info(struct amdxdna_dev *xdna,
+			     struct mailbox_info *mbox_info)
+{
+	int ret;
+
+	ret = aie4_fw_is_alive(xdna);
+	if (ret)
+		return ret;
+
+	aie4_read_mbox_info(xdna, mbox_info);
+
+	ret = aie_check_protocol(&xdna->dev_handle->aie,
+				 mbox_info->protocol_major,
+				 mbox_info->protocol_minor);
+	if (ret)
+		XDNA_ERR(xdna, "mailbox major.minor %d.%d is not supported",
+			 mbox_info->protocol_major, mbox_info->protocol_minor);
+
+	return ret;
+}
+
+static void aie4_mailbox_fini(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+
+	aie_destroy_chann(&ndev->aie, &ndev->aie.mgmt_chann);
+	drmm_kfree(&xdna->ddev, ndev->mbox);
+	ndev->mbox = NULL;
+}
+
+static int aie4_irq_init(struct amdxdna_dev *xdna)
+{
+	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
+	int ret, nvec;
+
+	nvec = pci_msix_vec_count(pdev);
+	XDNA_DBG(xdna, "irq vectors:%d", nvec);
+	if (nvec <= 0) {
+		XDNA_ERR(xdna, "does not get number of interrupt vector");
+		return -EINVAL;
+	}
+
+	ret = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
+	if (ret < 0) {
+		XDNA_ERR(xdna, "failed to alloc irq vector, ret: %d", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int aie4_mailbox_start(struct amdxdna_dev *xdna,
+			      struct mailbox_info *mbi)
+{
+	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+	const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv;
+	struct xdna_mailbox_chann_res *i2x;
+	struct xdna_mailbox_chann_res *x2i;
+	int mgmt_mb_irq;
+	int ret;
+
+	struct xdna_mailbox_res mbox_res = {
+		.ringbuf_base = ndev->rbuf_base,
+		.ringbuf_size = pci_resource_len(pdev, npriv->mbox_rbuf_bar),
+		.mbox_base = ndev->mbox_base,
+		.mbox_size = pci_resource_len(pdev, npriv->mbox_bar),
+		.name = "xdna_aie4_mailbox",
+	};
+
+	i2x = &ndev->aie.mgmt_i2x;
+	x2i = &ndev->aie.mgmt_x2i;
+
+	x2i->mb_head_ptr_reg = mbi->x2i_head_offset;
+	x2i->mb_tail_ptr_reg = mbi->x2i_tail_offset;
+	x2i->rb_start_addr = mbi->x2i_buffer_addr;
+	x2i->rb_size = mbi->x2i_buffer_size;
+
+	i2x->rb_start_addr = mbi->i2x_buffer_addr;
+	i2x->rb_size = mbi->i2x_buffer_size;
+	i2x->mb_head_ptr_reg = mbi->i2x_head_offset;
+	i2x->mb_tail_ptr_reg = mbi->i2x_tail_offset;
+
+	ndev->aie.mgmt_chan_idx = mbi->i2x_msi_idx;
+	aie_dump_mgmt_chann_debug(&ndev->aie);
+
+	ndev->mbox = xdnam_mailbox_create(&xdna->ddev, &mbox_res);
+	if (!ndev->mbox) {
+		XDNA_ERR(xdna, "failed to create mailbox device");
+		return -ENODEV;
+	}
+
+	ndev->aie.mgmt_chann = xdna_mailbox_alloc_channel(ndev->mbox);
+	if (!ndev->aie.mgmt_chann) {
+		XDNA_ERR(xdna, "failed to alloc mailbox channel");
+		return -ENODEV;
+	}
+
+	mgmt_mb_irq = pci_irq_vector(pdev, ndev->aie.mgmt_chan_idx);
+	if (mgmt_mb_irq < 0) {
+		XDNA_ERR(xdna, "failed to alloc irq vector, return %d", mgmt_mb_irq);
+		ret = mgmt_mb_irq;
+		goto free_channel;
+	}
+
+	ret = xdna_mailbox_start_channel(ndev->aie.mgmt_chann,
+					 &ndev->aie.mgmt_x2i,
+					 &ndev->aie.mgmt_i2x,
+					 NO_IOHUB,
+					 mgmt_mb_irq);
+	if (ret) {
+		XDNA_ERR(xdna, "failed to start management mailbox channel");
+		ret = -EINVAL;
+		goto free_channel;
+	}
+
+	XDNA_DBG(xdna, "Mailbox management channel created");
+	return 0;
+
+free_channel:
+	xdna_mailbox_free_channel(ndev->aie.mgmt_chann);
+	ndev->aie.mgmt_chann = NULL;
+	return ret;
+}
+
+static int aie4_mailbox_init(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	struct mailbox_info mbox_info;
+	int ret;
+
+	ret = aie4_mailbox_info(xdna, &mbox_info);
+	if (ret)
+		return ret;
+
+	return aie4_mailbox_start(xdna, &mbox_info);
+}
+
+static void aie4_fw_stop(struct amdxdna_dev_hdl *ndev)
+{
+	aie_psp_stop(ndev->aie.psp_hdl);
+	aie_smu_fini(ndev->aie.smu_hdl);
+}
+
+static int aie4_fw_start(struct amdxdna_dev_hdl *ndev)
+{
+	int ret;
+
+	ret = aie_smu_init(ndev->aie.smu_hdl);
+	if (ret) {
+		XDNA_ERR(ndev->aie.xdna, "failed to init smu, ret %d", ret);
+		return ret;
+	}
+
+	ret = aie_psp_start(ndev->aie.psp_hdl);
+	if (ret) {
+		XDNA_ERR(ndev->aie.xdna, "failed to start psp, ret %d", ret);
+		aie_smu_fini(ndev->aie.smu_hdl);
+	}
+
+	return ret;
+}
+
+static int aie4_partition_init(struct amdxdna_dev_hdl *ndev)
+{
+	DECLARE_AIE_MSG(aie4_msg_create_partition, AIE4_MSG_OP_CREATE_PARTITION);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	int ret;
+
+	req.partition_col_start = 0;
+	req.partition_col_count = AIE4_TOTAL_COLUMN;
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret) {
+		XDNA_ERR(xdna, "partition init failed: %d", ret);
+		return ret;
+	}
+
+	ndev->partition_id = resp.partition_id;
+	return 0;
+}
+
+static void aie4_partition_fini(struct amdxdna_dev_hdl *ndev)
+{
+	DECLARE_AIE_MSG(aie4_msg_destroy_partition, AIE4_MSG_OP_DESTROY_PARTITION);
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	int ret;
+
+	req.partition_id = ndev->partition_id;
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret)
+		XDNA_ERR(xdna, "partition fini failed: %d", ret);
+}
+
+static int aie4_query(struct amdxdna_dev_hdl *ndev)
+{
+	return aie4_query_aie_metadata(ndev, &ndev->aie.metadata);
+}
+
+static int aie4_pf_hw_start(struct amdxdna_dev_hdl *ndev)
+{
+	int ret;
+
+	ret = aie4_fw_start(ndev);
+	if (ret)
+		return ret;
+
+	ret = aie4_mailbox_init(ndev);
+	if (ret)
+		goto stop_fw;
+
+	ret = aie4_attach_work_buffer(ndev);
+	if (ret)
+		goto mbox_fini;
+
+	return 0;
+
+mbox_fini:
+	aie4_mailbox_fini(ndev);
+stop_fw:
+	aie4_fw_stop(ndev);
+
+	return ret;
+}
+
+static void aie4_pf_hw_stop(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+
+	aie4_suspend_fw(ndev);
+	aie4_mailbox_fini(ndev);
+	aie4_fw_stop(ndev);
+}
+
+static int aie4_vf_hw_start(struct amdxdna_dev_hdl *ndev)
+{
+	int ret;
+
+	ret = aie4_mailbox_init(ndev);
+	if (ret)
+		return ret;
+
+	ret = aie4_query(ndev);
+	if (ret)
+		goto mailbox_fini;
+
+	ret = aie4_partition_init(ndev);
+	if (ret)
+		goto mailbox_fini;
+
+	return 0;
+
+mailbox_fini:
+	aie4_mailbox_fini(ndev);
+	return ret;
+}
+
+static void aie4_vf_hw_stop(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+
+	aie4_partition_fini(ndev);
+	aie4_mailbox_fini(ndev);
+}
+
+static int aie4_request_firmware(struct amdxdna_dev_hdl *ndev,
+				 const struct firmware **npufw,
+				 const struct firmware **certfw)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
+	char fw_name[128];
+	int ret;
+
+	ret = snprintf(fw_name, sizeof(fw_name), "amdnpu/%04x_%02x/%s",
+		       pdev->device, pdev->revision, ndev->priv->npufw_path);
+	if (ret >= sizeof(fw_name)) {
+		XDNA_ERR(xdna, "npu firmware path is truncated");
+		return -EINVAL;
+	}
+
+	ret = request_firmware(npufw, fw_name, &pdev->dev);
+	if (ret) {
+		XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", fw_name, ret);
+		return ret;
+	}
+
+	ret = snprintf(fw_name, sizeof(fw_name), "amdnpu/%04x_%02x/%s",
+		       pdev->device, pdev->revision, ndev->priv->certfw_path);
+	if (ret >= sizeof(fw_name)) {
+		XDNA_ERR(xdna, "cert firmware path is truncated");
+		ret = -EINVAL;
+		goto release_npufw;
+	}
+
+	ret = request_firmware(certfw, fw_name, &pdev->dev);
+	if (ret) {
+		XDNA_ERR(xdna, "failed to request_firmware %s, ret %d", fw_name, ret);
+		goto release_npufw;
+	}
+
+	return 0;
+
+release_npufw:
+	release_firmware(*npufw);
+
+	return ret;
+}
+
+static void aie4_release_firmware(struct amdxdna_dev_hdl *ndev,
+				  const struct firmware *npufw,
+				  const struct firmware *certfw)
+{
+	release_firmware(certfw);
+	release_firmware(npufw);
+}
+
+static int aie4_prepare_firmware(struct amdxdna_dev_hdl *ndev,
+				 const struct firmware *npufw,
+				 const struct firmware *certfw,
+				 void __iomem *tbl[PCI_NUM_RESOURCES])
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	struct psp_config psp_conf;
+	struct smu_config smu_conf;
+	int i;
+
+	psp_conf.fw_size = npufw->size;
+	psp_conf.fw_buf = npufw->data;
+	psp_conf.certfw_size = certfw->size;
+	psp_conf.certfw_buf = certfw->data;
+	psp_conf.arg2_mask = ~0;
+	psp_conf.notify_val = PSP_NOTIFY_INTR;
+	for (i = 0; i < PSP_MAX_REGS; i++)
+		psp_conf.psp_regs[i] = tbl[PSP_REG_BAR(ndev, i)] + PSP_REG_OFF(ndev, i);
+	ndev->aie.psp_hdl = aiem_psp_create(&xdna->ddev, &psp_conf);
+	if (!ndev->aie.psp_hdl) {
+		XDNA_ERR(xdna, "failed to create psp");
+		return -ENOMEM;
+	}
+
+	for (i = 0; i < SMU_MAX_REGS; i++)
+		smu_conf.smu_regs[i] = tbl[SMU_REG_BAR(ndev, i)] + SMU_REG_OFF(ndev, i);
+	ndev->aie.smu_hdl = aiem_smu_create(&xdna->ddev, &smu_conf);
+	if (!ndev->aie.smu_hdl) {
+		XDNA_ERR(xdna, "failed to create smu");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static int aie4_load_fw(struct amdxdna_dev_hdl *ndev,
+			void __iomem *tbl[PCI_NUM_RESOURCES])
+{
+	const struct firmware *npufw, *certfw;
+	int ret;
+
+	if (!ndev->priv->npufw_path && !ndev->priv->certfw_path)
+		return 0;
+
+	ret = aie4_request_firmware(ndev, &npufw, &certfw);
+	if (ret)
+		return ret;
+
+	ret = aie4_prepare_firmware(ndev, npufw, certfw, tbl);
+	aie4_release_firmware(ndev, npufw, certfw);
+
+	return ret;
+}
+
+static int aie4m_pcidev_init(struct amdxdna_dev *xdna)
+{
+	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
+	struct amdxdna_dev_hdl *ndev;
+	void __iomem *tbl[PCI_NUM_RESOURCES] = {0};
+	unsigned long bars = 0;
+	int ret, i;
+
+	ndev = drmm_kzalloc(&xdna->ddev, sizeof(*ndev), GFP_KERNEL);
+	if (!ndev)
+		return -ENOMEM;
+
+	ndev->priv = xdna->dev_info->dev_priv;
+	ndev->aie.xdna = xdna;
+	xdna->dev_handle = ndev;
+
+	xa_init_flags(&ndev->cert_comp_xa, XA_FLAGS_ALLOC);
+	mutex_init(&ndev->cert_comp_lock);
+
+	/* Enable managed PCI device */
+	ret = pcim_enable_device(pdev);
+	if (ret) {
+		XDNA_ERR(xdna, "pcim enable device failed, ret %d", ret);
+		return ret;
+	}
+
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		XDNA_ERR(xdna, "failed to set DMA mask to 64:%d", ret);
+		return ret;
+	}
+
+	for (i = 0; i < PSP_MAX_REGS; i++)
+		set_bit(PSP_REG_BAR(ndev, i), &bars);
+	for (i = 0; i < SMU_MAX_REGS; i++)
+		set_bit(SMU_REG_BAR(ndev, i), &bars);
+	set_bit(xdna->dev_info->mbox_bar, &bars);
+	set_bit(xdna->dev_info->sram_bar, &bars);
+
+	for (i = 0; i < PCI_NUM_RESOURCES; i++) {
+		if (!test_bit(i, &bars))
+			continue;
+		tbl[i] = pcim_iomap(pdev, i, 0);
+		if (!tbl[i]) {
+			XDNA_ERR(xdna, "map bar %d failed", i);
+			return -ENOMEM;
+		}
+	}
+
+	ndev->mbox_base = tbl[xdna->dev_info->mbox_bar];
+	ndev->rbuf_base = tbl[xdna->dev_info->sram_bar];
+
+	pci_set_master(pdev);
+
+	ret = aie4_load_fw(ndev, tbl);
+	if (ret)
+		return ret;
+
+	ret = aie4_irq_init(xdna);
+	if (ret)
+		return ret;
+
+	amdxdna_vbnv_init(xdna);
+	XDNA_DBG(xdna, "init finished");
+
+	return 0;
+}
+
+static int aie4_doorbell_mmap(struct amdxdna_client *client, struct vm_area_struct *vma)
+{
+	struct amdxdna_dev *xdna = client->xdna;
+	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
+	const struct amdxdna_dev_priv *npriv = xdna->dev_info->dev_priv;
+	phys_addr_t res_start;
+	unsigned long pfn;
+	int ret;
+
+	if (!aie4_hwctx_valid_doorbell(client, vma->vm_pgoff)) {
+		XDNA_ERR(xdna, "Invalid doorbell page offset 0x%lx", vma->vm_pgoff);
+		return -EINVAL;
+	}
+
+	if (vma_pages(vma) != 1) {
+		XDNA_ERR(xdna, "can only map one page, got %ld", vma_pages(vma));
+		return -EINVAL;
+	}
+
+	res_start = pci_resource_start(pdev, xdna->dev_info->doorbell_bar) + npriv->doorbell_off;
+	pfn = PHYS_PFN(res_start) + vma->vm_pgoff;
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vm_flags_set(vma, VM_IO | VM_DONTEXPAND | VM_DONTDUMP);
+	ret = io_remap_pfn_range(vma, vma->vm_start,
+				 pfn,
+				 PAGE_SIZE,
+				 vma->vm_page_prot);
+
+	XDNA_DBG(xdna, "doorbell ret %d", ret);
+	return ret;
+}
+
+static int aie4_get_info(struct amdxdna_client *client, struct amdxdna_drm_get_info *args)
+{
+	struct amdxdna_dev *xdna = client->xdna;
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+	int ret;
+
+	switch (args->param) {
+	case DRM_AMDXDNA_QUERY_AIE_METADATA:
+		ret = amdxdna_get_metadata(&ndev->aie, client, args);
+		break;
+	default:
+		XDNA_ERR(xdna, "Not supported request parameter %u", args->param);
+		ret = -EOPNOTSUPP;
+	}
+
+	XDNA_DBG(xdna, "Got param %d", args->param);
+
+	return ret;
+}
+
+static int aie4_alloc_work_buffer(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	u32 buf_size = AIE4_WORK_BUFFER_MIN_SIZE;
+
+	ndev->work_buf = amdxdna_alloc_msg_buffer(xdna, &buf_size,
+						  &ndev->work_buf_addr);
+	if (IS_ERR(ndev->work_buf)) {
+		int ret = PTR_ERR(ndev->work_buf);
+
+		XDNA_ERR(xdna, "Failed to alloc work buffer, size 0x%x",
+			 AIE4_WORK_BUFFER_MIN_SIZE);
+		ndev->work_buf = NULL;
+		return ret;
+	}
+
+	ndev->work_buf_size = buf_size;
+	XDNA_DBG(xdna, "Work buffer allocated: size 0x%x", buf_size);
+
+	return 0;
+}
+
+static void aie4_free_work_buffer(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+
+	if (!ndev->work_buf)
+		return;
+
+	amdxdna_free_msg_buffer(xdna, ndev->work_buf_size, ndev->work_buf,
+				ndev->work_buf_addr);
+	ndev->work_buf = NULL;
+}
+
+static int aie4_pf_init(struct amdxdna_dev *xdna)
+{
+	int ret;
+
+	ret = aie4m_pcidev_init(xdna);
+	if (ret)
+		return ret;
+
+	ret = aie4_alloc_work_buffer(xdna->dev_handle);
+	if (ret)
+		return ret;
+
+	ret = aie4_pf_hw_start(xdna->dev_handle);
+	if (ret)
+		goto free_work_buf;
+
+	return 0;
+
+free_work_buf:
+	aie4_free_work_buffer(xdna->dev_handle);
+	return ret;
+}
+
+static int aie4_vf_init(struct amdxdna_dev *xdna)
+{
+	int ret;
+
+	ret = aie4m_pcidev_init(xdna);
+	if (ret)
+		return ret;
+
+	return aie4_vf_hw_start(xdna->dev_handle);
+}
+
+static void aie4_pf_fini(struct amdxdna_dev *xdna)
+{
+	aie4_sriov_stop(xdna->dev_handle);
+	aie4_pf_hw_stop(xdna->dev_handle);
+	aie4_free_work_buffer(xdna->dev_handle);
+}
+
+static void aie4_vf_fini(struct amdxdna_dev *xdna)
+{
+	aie4_vf_hw_stop(xdna->dev_handle);
+}
+
+const struct amdxdna_dev_ops aie4_pf_ops = {
+	.init			= aie4_pf_init,
+	.fini			= aie4_pf_fini,
+	.sriov_configure        = aie4_sriov_configure,
+};
+
+const struct amdxdna_dev_ops aie4_vf_ops = {
+	.init			= aie4_vf_init,
+	.fini			= aie4_vf_fini,
+	.hwctx_init		= aie4_hwctx_init,
+	.hwctx_fini		= aie4_hwctx_fini,
+	.mmap			= aie4_doorbell_mmap,
+	.cmd_wait		= aie4_cmd_wait,
+	.get_aie_info		= aie4_get_info,
+};
diff --git a/drivers/accel/amdxdna/aie4_pci.h b/drivers/accel/amdxdna/aie4_pci.h
new file mode 100644
index 0000000000000..3fd5eace3ed71
--- /dev/null
+++ b/drivers/accel/amdxdna/aie4_pci.h
@@ -0,0 +1,89 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#ifndef _AIE4_PCI_H_
+#define _AIE4_PCI_H_
+
+#include <linux/device.h>
+#include <linux/iopoll.h>
+#include <linux/pci.h>
+
+#include "aie.h"
+#include "amdxdna_mailbox.h"
+
+struct cert_comp {
+	struct amdxdna_dev_hdl          *ndev;
+	u32                             msix_idx;
+	int                             irq;
+	struct kref                     kref;
+	wait_queue_head_t               waitq;
+};
+
+struct amdxdna_hwctx_priv {
+	struct amdxdna_gem_obj          *umq_bo;
+	u64                             *umq_read_index;
+	u64                             *umq_write_index;
+
+	struct cert_comp                *cert_comp;
+	u32                             hw_ctx_id;
+};
+
+struct amdxdna_dev_priv {
+	const char              *npufw_path;
+	const char              *certfw_path;
+	u32			mbox_bar;
+	u32			mbox_rbuf_bar;
+	u64			mbox_info_off;
+	u32			doorbell_off;
+
+	struct aie_bar_off_pair	psp_regs_off[PSP_MAX_REGS];
+	struct aie_bar_off_pair	smu_regs_off[SMU_MAX_REGS];
+};
+
+struct amdxdna_dev_hdl {
+	struct aie_device		aie;
+	const struct amdxdna_dev_priv	*priv;
+	void			__iomem *mbox_base;
+	void			__iomem *rbuf_base;
+
+	struct mailbox			*mbox;
+	u32				partition_id;
+
+	struct xarray                   cert_comp_xa; /* device level indexed by msix id */
+	struct mutex                    cert_comp_lock; /* protects cert_comp operations*/
+
+	void				*work_buf;
+	dma_addr_t			work_buf_addr;
+	u32				work_buf_size;
+};
+
+/* aie4_message.c */
+int aie4_query_aie_metadata(struct amdxdna_dev_hdl *ndev,
+			    struct amdxdna_drm_query_aie_metadata *metadata);
+int aie4_suspend_fw(struct amdxdna_dev_hdl *ndev);
+int aie4_attach_work_buffer(struct amdxdna_dev_hdl *ndev);
+
+/* aie4_ctx.c */
+int aie4_hwctx_init(struct amdxdna_hwctx *hwctx);
+void aie4_hwctx_fini(struct amdxdna_hwctx *hwctx);
+int aie4_cmd_wait(struct amdxdna_hwctx *hwctx, u64 seq, u32 timeout);
+int aie4_hwctx_valid_doorbell(struct amdxdna_client *client, u32 vm_pgoff);
+
+/* aie4_sriov.c */
+#if IS_ENABLED(CONFIG_PCI_IOV)
+int aie4_sriov_configure(struct amdxdna_dev *xdna, int num_vfs);
+int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev);
+#else
+#define aie4_sriov_configure NULL
+static inline int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev)
+{
+	return 0;
+}
+#endif
+
+extern const struct amdxdna_dev_ops aie4_pf_ops;
+extern const struct amdxdna_dev_ops aie4_vf_ops;
+
+#endif /* _AIE4_PCI_H_ */
diff --git a/drivers/accel/amdxdna/aie4_sriov.c b/drivers/accel/amdxdna/aie4_sriov.c
new file mode 100644
index 0000000000000..e1ce633768a5b
--- /dev/null
+++ b/drivers/accel/amdxdna/aie4_sriov.c
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <drm/amdxdna_accel.h>
+#include <drm/drm_print.h>
+#include <linux/pci.h>
+
+#include "aie.h"
+#include "aie4_msg_priv.h"
+#include "aie4_pci.h"
+#include "amdxdna_mailbox.h"
+#include "amdxdna_mailbox_helper.h"
+#include "amdxdna_pci_drv.h"
+
+static int aie4_destroy_vfs(struct amdxdna_dev_hdl *ndev)
+{
+	DECLARE_AIE_MSG(aie4_msg_destroy_vfs, AIE4_MSG_OP_DESTROY_VFS);
+	int ret;
+
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret)
+		XDNA_ERR(ndev->aie.xdna, "destroy vfs op failed: %d", ret);
+
+	return ret;
+}
+
+static int aie4_create_vfs(struct amdxdna_dev_hdl *ndev, int num_vfs)
+{
+	DECLARE_AIE_MSG(aie4_msg_create_vfs, AIE4_MSG_OP_CREATE_VFS);
+	int ret;
+
+	req.vf_cnt = num_vfs;
+	ret = aie_send_mgmt_msg_wait(&ndev->aie, &msg);
+	if (ret)
+		XDNA_ERR(ndev->aie.xdna, "create vfs op failed: %d", ret);
+
+	return ret;
+}
+
+int aie4_sriov_stop(struct amdxdna_dev_hdl *ndev)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
+	int ret;
+
+	if (!pci_num_vf(pdev))
+		return 0;
+
+	ret = pci_vfs_assigned(pdev);
+	if (ret) {
+		XDNA_ERR(xdna, "VFs are still assigned to VMs");
+		return -EPERM;
+	}
+
+	pci_disable_sriov(pdev);
+	return aie4_destroy_vfs(ndev);
+}
+
+static int aie4_sriov_start(struct amdxdna_dev_hdl *ndev, int num_vfs)
+{
+	struct amdxdna_dev *xdna = ndev->aie.xdna;
+	struct pci_dev *pdev = to_pci_dev(xdna->ddev.dev);
+	int ret;
+
+	ret = aie4_create_vfs(ndev, num_vfs);
+	if (ret)
+		return ret;
+
+	ret = pci_enable_sriov(pdev, num_vfs);
+	if (ret) {
+		XDNA_ERR(xdna, "configure VFs failed, ret: %d", ret);
+		aie4_destroy_vfs(ndev);
+		return ret;
+	}
+
+	return num_vfs;
+}
+
+int aie4_sriov_configure(struct amdxdna_dev *xdna, int num_vfs)
+{
+	struct amdxdna_dev_hdl *ndev = xdna->dev_handle;
+
+	drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
+
+	return (num_vfs) ? aie4_sriov_start(ndev, num_vfs) : aie4_sriov_stop(ndev);
+}
diff --git a/drivers/accel/amdxdna/aie_psp.c b/drivers/accel/amdxdna/aie_psp.c
new file mode 100644
index 0000000000000..458dca7cc5a05
--- /dev/null
+++ b/drivers/accel/amdxdna/aie_psp.c
@@ -0,0 +1,235 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <drm/drm_device.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_print.h>
+#include <linux/bitfield.h>
+#include <linux/iopoll.h>
+#include <linux/slab.h>
+
+#include "aie.h"
+
+#define PSP_STATUS_READY	BIT(31)
+
+/* PSP commands */
+#define PSP_VALIDATE		1
+#define PSP_START		2
+#define PSP_RELEASE_TMR		3
+#define PSP_VALIDATE_CERT       4
+
+/* PSP special arguments */
+#define PSP_START_COPY_FW	1
+
+/* PSP response error code */
+#define PSP_ERROR_CANCEL	0xFFFF0002
+#define PSP_ERROR_BAD_STATE	0xFFFF0007
+
+#define PSP_FW_ALIGN		0x10000
+#define PSP_CFW_ALIGN           0x8000
+#define PSP_POLL_INTERVAL	20000	/* us */
+#define PSP_POLL_TIMEOUT	1000000	/* us */
+
+#define PSP_REG(p, reg) ((p)->conf.psp_regs[reg])
+#define PSP_SET_CMD(psp, reg_vals, cmd, arg0, arg1, arg2)		\
+({									\
+	u32 *_regs = reg_vals;						\
+	u32 _cmd = cmd;							\
+	_regs[0] = _cmd;						\
+	_regs[1] = arg0;						\
+	_regs[2] = arg1;						\
+	_regs[3] = ((arg2) | ((_cmd) << 24)) & (psp)->conf.arg2_mask;	\
+})
+
+struct psp_device {
+	struct drm_device	*ddev;
+	struct psp_config	conf;
+	u32			fw_buf_sz;
+	u64			fw_paddr;
+	void			*fw_buffer;
+	u32                     certfw_buf_sz;
+	u64                     certfw_paddr;
+	void                    *certfw_buffer;
+};
+
+static int psp_exec(struct psp_device *psp, u32 *reg_vals)
+{
+	u32 resp_code;
+	int ret, i;
+	u32 ready;
+
+	/* Check for PSP ready before any write */
+	ret = readx_poll_timeout(readl, PSP_REG(psp, PSP_STATUS_REG), ready,
+				 FIELD_GET(PSP_STATUS_READY, ready),
+				 PSP_POLL_INTERVAL, PSP_POLL_TIMEOUT);
+	if (ret) {
+		drm_err(psp->ddev, "PSP is not ready, ret 0x%x", ret);
+		return ret;
+	}
+
+	/* Write command and argument registers */
+	for (i = 0; i < PSP_NUM_IN_REGS; i++)
+		writel(reg_vals[i], PSP_REG(psp, i));
+
+	/* clear and set PSP INTR register to kick off */
+	writel(0, PSP_REG(psp, PSP_INTR_REG));
+	writel(psp->conf.notify_val, PSP_REG(psp, PSP_INTR_REG));
+
+	/* PSP should be busy. Wait for ready, so we know task is done. */
+	ret = readx_poll_timeout(readl, PSP_REG(psp, PSP_STATUS_REG), ready,
+				 FIELD_GET(PSP_STATUS_READY, ready),
+				 PSP_POLL_INTERVAL, PSP_POLL_TIMEOUT);
+	if (ret) {
+		drm_err(psp->ddev, "PSP is not ready, ret 0x%x", ret);
+		return ret;
+	}
+
+	resp_code = readl(PSP_REG(psp, PSP_RESP_REG));
+	if (resp_code) {
+		drm_err(psp->ddev, "fw return error 0x%x", resp_code);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+int aie_psp_waitmode_poll(struct psp_device *psp)
+{
+	struct amdxdna_dev *xdna = to_xdna_dev(psp->ddev);
+	u32 mode_reg;
+	int ret;
+
+	ret = readx_poll_timeout(readl, PSP_REG(psp, PSP_PWAITMODE_REG), mode_reg,
+				 (mode_reg & 0x1) == 1,
+				 PSP_POLL_INTERVAL, PSP_POLL_TIMEOUT);
+	if (ret)
+		XDNA_ERR(xdna, "fw waitmode reg error, ret %d", ret);
+
+	return ret;
+}
+
+void aie_psp_stop(struct psp_device *psp)
+{
+	u32 reg_vals[PSP_NUM_IN_REGS];
+	int ret;
+
+	PSP_SET_CMD(psp, reg_vals, PSP_RELEASE_TMR, 0, 0, 0);
+
+	ret = psp_exec(psp, reg_vals);
+	if (ret)
+		drm_err(psp->ddev, "release tmr failed, ret %d", ret);
+}
+
+static int psp_validate_fw(struct psp_device *psp, u8 cmd, u64 paddr, u32 buf_sz)
+{
+	u32 reg_vals[PSP_NUM_IN_REGS];
+	int ret;
+
+	PSP_SET_CMD(psp, reg_vals, cmd, lower_32_bits(paddr),
+		    upper_32_bits(paddr), buf_sz);
+
+	ret = psp_exec(psp, reg_vals);
+	if (ret)
+		drm_err(psp->ddev, "failed to validate fw, ret %d", ret);
+
+	return ret;
+}
+
+static int psp_start(struct psp_device *psp)
+{
+	u32 reg_vals[PSP_NUM_IN_REGS];
+	int ret;
+
+	PSP_SET_CMD(psp, reg_vals, PSP_START, PSP_START_COPY_FW, 0, 0);
+
+	ret = psp_exec(psp, reg_vals);
+	if (ret)
+		drm_err(psp->ddev, "failed to start fw, ret %d", ret);
+
+	return ret;
+}
+
+int aie_psp_start(struct psp_device *psp)
+{
+	int ret;
+
+	ret = psp_validate_fw(psp, PSP_VALIDATE,
+			      psp->fw_paddr, psp->fw_buf_sz);
+	if (ret)
+		return ret;
+
+	if (!psp->certfw_buf_sz)
+		goto psp_start;
+
+	ret = psp_validate_fw(psp, PSP_VALIDATE_CERT,
+			      psp->certfw_paddr, psp->certfw_buf_sz);
+	if (ret)
+		return ret;
+psp_start:
+	return psp_start(psp);
+}
+
+/*
+ * PSP requires host physical address to load firmware.
+ * Allocate a buffer, obtain its physical address, align, and copy data in.
+ */
+static void *psp_alloc_fw_buf(struct psp_device *psp, const void *fw_data,
+			      u32 fw_size, u32 align, u32 *buf_sz,
+			      u64 *paddr)
+{
+	u32 alloc_sz;
+	void *buffer;
+	u64 offset;
+
+	*buf_sz = ALIGN(fw_size, align);
+	alloc_sz = *buf_sz + align;
+
+	buffer = drmm_kmalloc(psp->ddev, alloc_sz, GFP_KERNEL);
+	if (!buffer)
+		return NULL;
+
+	*paddr = virt_to_phys(buffer);
+	offset = ALIGN(*paddr, align) - *paddr;
+	*paddr += offset;
+	memcpy(buffer + offset, fw_data, fw_size);
+
+	return buffer;
+}
+
+struct psp_device *aiem_psp_create(struct drm_device *ddev, struct psp_config *conf)
+{
+	struct psp_device *psp;
+
+	psp = drmm_kzalloc(ddev, sizeof(*psp), GFP_KERNEL);
+	if (!psp)
+		return NULL;
+
+	psp->ddev = ddev;
+	psp->fw_buffer = psp_alloc_fw_buf(psp, conf->fw_buf, conf->fw_size,
+					  PSP_FW_ALIGN, &psp->fw_buf_sz,
+					  &psp->fw_paddr);
+	if (!psp->fw_buffer)
+		return NULL;
+
+	if (!conf->certfw_size) {
+		drm_dbg(ddev, "no cert fw");
+		goto done;
+	}
+
+	/* CERT firmware */
+	psp->certfw_buffer = psp_alloc_fw_buf(psp, conf->certfw_buf,
+					      conf->certfw_size, PSP_CFW_ALIGN,
+					      &psp->certfw_buf_sz,
+					      &psp->certfw_paddr);
+	if (!psp->certfw_buffer) {
+		drm_err(ddev, "no memory for cert fw buffer");
+		return NULL;
+	}
+
+done:
+	memcpy(&psp->conf, conf, sizeof(psp->conf));
+
+	return psp;
+}
diff --git a/drivers/accel/amdxdna/aie_smu.c b/drivers/accel/amdxdna/aie_smu.c
new file mode 100644
index 0000000000000..62aea550aabcf
--- /dev/null
+++ b/drivers/accel/amdxdna/aie_smu.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include "drm/amdxdna_accel.h"
+#include <drm/drm_device.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_print.h>
+#include <drm/gpu_scheduler.h>
+#include <linux/iopoll.h>
+
+#include "aie.h"
+
+#define SMU_RESULT_OK   1
+
+/* SMU commands */
+#define AIE_SMU_POWER_ON                0x3
+#define AIE_SMU_POWER_OFF               0x4
+#define AIE_SMU_SET_MPNPUCLK_FREQ       0x5
+#define AIE_SMU_SET_HCLK_FREQ           0x6
+#define AIE_SMU_SET_SOFT_DPMLEVEL       0x7
+#define AIE_SMU_SET_HARD_DPMLEVEL       0x8
+
+#define SMU_REG(s, reg) ((s)->smu_regs[reg])
+
+struct smu_device {
+	struct drm_device	*ddev;
+	struct smu_config	conf;
+	void __iomem		*smu_regs[SMU_MAX_REGS];
+};
+
+static int aie_smu_exec(struct smu_device *smu, u32 reg_cmd, u32 reg_arg, u32 *out)
+{
+	u32 resp;
+	int ret;
+
+	writel(0, SMU_REG(smu, SMU_RESP_REG));
+	writel(reg_arg, SMU_REG(smu, SMU_ARG_REG));
+	writel(reg_cmd, SMU_REG(smu, SMU_CMD_REG));
+
+	/* Clear and set SMU_INTR_REG to kick off */
+	writel(0, SMU_REG(smu, SMU_INTR_REG));
+	writel(1, SMU_REG(smu, SMU_INTR_REG));
+
+	ret = readx_poll_timeout(readl, SMU_REG(smu, SMU_RESP_REG), resp,
+				 resp, AIE_INTERVAL, AIE_TIMEOUT);
+	if (ret) {
+		drm_err(smu->ddev, "smu cmd %d timed out", reg_cmd);
+		return ret;
+	}
+
+	if (out)
+		*out = readl(SMU_REG(smu, SMU_OUT_REG));
+
+	if (resp != SMU_RESULT_OK) {
+		drm_err(smu->ddev, "smu cmd %d failed, 0x%x", reg_cmd, resp);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int aie_smu_init(struct smu_device *smu)
+{
+	int ret;
+
+	/*
+	 * Failing to set power off indicates an unrecoverable hardware or
+	 * firmware error.
+	 */
+	ret = aie_smu_exec(smu, AIE_SMU_POWER_OFF, 0, NULL);
+	if (ret) {
+		drm_err(smu->ddev, "Access power failed, ret %d", ret);
+		return ret;
+	}
+
+	ret = aie_smu_exec(smu, AIE_SMU_POWER_ON, 0, NULL);
+	if (ret) {
+		drm_err(smu->ddev, "Power on failed, ret %d", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+void aie_smu_fini(struct smu_device *smu)
+{
+	int ret;
+
+	ret = aie_smu_exec(smu, AIE_SMU_POWER_OFF, 0, NULL);
+	if (ret)
+		drm_err(smu->ddev, "Power off failed, ret %d", ret);
+}
+
+int aie_smu_set_clocks(struct smu_device *smu, u32 *npuclk, u32 *hclk)
+{
+	int ret;
+
+	if (npuclk) {
+		ret = aie_smu_exec(smu, AIE_SMU_SET_MPNPUCLK_FREQ, *npuclk, npuclk);
+		if (ret) {
+			drm_err(smu->ddev, "Set mpnpu clock to %d failed, ret %d", *npuclk, ret);
+			return ret;
+		}
+	}
+
+	if (hclk) {
+		ret = aie_smu_exec(smu, AIE_SMU_SET_HCLK_FREQ, *hclk, hclk);
+		if (ret) {
+			drm_err(smu->ddev, "Set hclock to %d failed, ret %d",
+				*hclk, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+int aie_smu_set_dpm(struct smu_device *smu, u32 dpm_level)
+{
+	int ret;
+
+	ret = aie_smu_exec(smu, AIE_SMU_SET_HARD_DPMLEVEL, dpm_level, NULL);
+	if (ret) {
+		drm_err(smu->ddev, "Set hard dpm level %d failed, ret %d",
+			dpm_level, ret);
+		return ret;
+	}
+
+	ret = aie_smu_exec(smu, AIE_SMU_SET_SOFT_DPMLEVEL, dpm_level, NULL);
+	if (ret) {
+		drm_err(smu->ddev, "Set soft dpm level %d failed, ret %d",
+			dpm_level, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+struct smu_device *aiem_smu_create(struct drm_device *ddev, struct smu_config *conf)
+{
+	struct smu_device *smu;
+
+	smu = drmm_kzalloc(ddev, sizeof(*smu), GFP_KERNEL);
+	if (!smu)
+		return NULL;
+
+	smu->ddev = ddev;
+	memcpy(smu->smu_regs, conf->smu_regs, sizeof(smu->smu_regs));
+
+	return smu;
+}
diff --git a/drivers/accel/amdxdna/amdxdna_cbuf.c b/drivers/accel/amdxdna/amdxdna_cbuf.c
new file mode 100644
index 0000000000000..a504560aae986
--- /dev/null
+++ b/drivers/accel/amdxdna/amdxdna_cbuf.c
@@ -0,0 +1,280 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <drm/drm_mm.h>
+#include <drm/drm_prime.h>
+
+#include "amdxdna_cbuf.h"
+#include "amdxdna_pci_drv.h"
+
+/*
+ * Carveout memory is a chunk of memory which is physically contiguous and
+ * is reserved during early boot time. There is only one chunk of such memory
+ * per device. Once available, all BOs accessible from device should be
+ * allocated from this memory. This is a platform debug/bringup feature.
+ */
+struct amdxdna_carveout {
+	u64		addr;
+	u64		size;
+	struct drm_mm	mm;
+	struct mutex	lock; /* protect mm */
+};
+
+bool amdxdna_use_carveout(struct amdxdna_dev *xdna)
+{
+	return !!xdna->carveout;
+}
+
+void amdxdna_get_carveout_conf(struct amdxdna_dev *xdna, u64 *addr, u64 *size)
+{
+	if (amdxdna_use_carveout(xdna)) {
+		*addr = xdna->carveout->addr;
+		*size = xdna->carveout->size;
+	} else {
+		*addr = 0;
+		*size = 0;
+	}
+}
+
+int amdxdna_carveout_init(struct amdxdna_dev *xdna, u64 carveout_addr, u64 carveout_size)
+{
+	struct amdxdna_carveout *carveout;
+
+	/* Only allow carveout memory to be set up once. */
+	if (amdxdna_use_carveout(xdna)) {
+		XDNA_ERR(xdna, "Carveout memory has already been set up.");
+		return -EBUSY;
+	}
+
+	carveout = kzalloc_obj(*carveout);
+	if (!carveout)
+		return -ENOMEM;
+
+	carveout->addr = carveout_addr;
+	carveout->size = carveout_size;
+	mutex_init(&carveout->lock);
+	drm_mm_init(&carveout->mm, carveout->addr, carveout->size);
+
+	xdna->carveout = carveout;
+	XDNA_INFO(xdna, "Use carveout mem: 0x%llx@0x%llx\n", carveout->size, carveout->addr);
+	return 0;
+}
+
+void amdxdna_carveout_fini(struct amdxdna_dev *xdna)
+{
+	struct amdxdna_carveout *carveout = xdna->carveout;
+
+	if (!amdxdna_use_carveout(xdna))
+		return;
+
+	XDNA_INFO(xdna, "Cleanup carveout mem: 0x%llx@0x%llx\n", carveout->size, carveout->addr);
+	drm_mm_takedown(&carveout->mm);
+	mutex_destroy(&carveout->lock);
+	kfree(carveout);
+	xdna->carveout = NULL;
+}
+
+struct amdxdna_cbuf_priv {
+	struct amdxdna_dev *xdna;
+	struct drm_mm_node node;
+};
+
+static struct sg_table *amdxdna_cbuf_map(struct dma_buf_attachment *attach,
+					 enum dma_data_direction direction)
+{
+	struct amdxdna_cbuf_priv *cbuf = attach->dmabuf->priv;
+	struct device *dev = attach->dev;
+	struct scatterlist *sgl, *sg;
+	int ret, n_entries, i;
+	struct sg_table *sgt;
+	dma_addr_t dma_addr;
+	size_t dma_size;
+	size_t max_seg;
+
+	sgt = kzalloc_obj(*sgt);
+	if (!sgt)
+		return ERR_PTR(-ENOMEM);
+
+	max_seg = min_t(size_t, UINT_MAX, dma_max_mapping_size(dev));
+	n_entries = (cbuf->node.size + max_seg - 1) / max_seg;
+	sgl = kzalloc_objs(*sg, n_entries);
+	if (!sgl) {
+		ret = -ENOMEM;
+		goto free_sgt;
+	}
+	sg_init_table(sgl, n_entries);
+	sgt->orig_nents = n_entries;
+	sgt->nents = n_entries;
+	sgt->sgl = sgl;
+
+	dma_size = cbuf->node.size;
+	dma_addr = dma_map_resource(dev, cbuf->node.start, dma_size,
+				    direction, DMA_ATTR_SKIP_CPU_SYNC);
+	ret = dma_mapping_error(dev, dma_addr);
+	if (ret) {
+		pr_err("Failed to dma_map_resource carveout dma buf, ret %d\n", ret);
+		goto free_sgl;
+	}
+
+	for_each_sgtable_dma_sg(sgt, sg, i) {
+		size_t len = min_t(size_t, max_seg, dma_size);
+
+		sg_dma_address(sg) = dma_addr;
+		sg_dma_len(sg) = len;
+		dma_addr += len;
+		dma_size -= len;
+	}
+
+	return sgt;
+
+free_sgl:
+	kfree(sgl);
+free_sgt:
+	kfree(sgt);
+	return ERR_PTR(ret);
+}
+
+static void amdxdna_cbuf_unmap(struct dma_buf_attachment *attach,
+			       struct sg_table *sgt,
+			       enum dma_data_direction direction)
+{
+	dma_unmap_resource(attach->dev, sg_dma_address(sgt->sgl),
+			   drm_prime_get_contiguous_size(sgt), direction,
+			   DMA_ATTR_SKIP_CPU_SYNC);
+	sg_free_table(sgt);
+	kfree(sgt);
+}
+
+static void amdxdna_cbuf_release(struct dma_buf *dbuf)
+{
+	struct amdxdna_cbuf_priv *cbuf = dbuf->priv;
+	struct amdxdna_carveout *carveout;
+
+	carveout = cbuf->xdna->carveout;
+	mutex_lock(&carveout->lock);
+	drm_mm_remove_node(&cbuf->node);
+	mutex_unlock(&carveout->lock);
+
+	kfree(cbuf);
+}
+
+static vm_fault_t amdxdna_cbuf_vm_fault(struct vm_fault *vmf)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct amdxdna_cbuf_priv *cbuf;
+	unsigned long pfn;
+	pgoff_t pgoff;
+
+	cbuf = vma->vm_private_data;
+	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
+	pfn = (cbuf->node.start >> PAGE_SHIFT) + pgoff;
+
+	return vmf_insert_pfn(vma, vmf->address, pfn);
+}
+
+static const struct vm_operations_struct amdxdna_cbuf_vm_ops = {
+	.fault = amdxdna_cbuf_vm_fault,
+};
+
+static int amdxdna_cbuf_mmap(struct dma_buf *dbuf, struct vm_area_struct *vma)
+{
+	struct amdxdna_cbuf_priv *cbuf = dbuf->priv;
+
+	vma->vm_ops = &amdxdna_cbuf_vm_ops;
+	vma->vm_private_data = cbuf;
+	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
+
+	return 0;
+}
+
+static int amdxdna_cbuf_vmap(struct dma_buf *dbuf, struct iosys_map *map)
+{
+	struct amdxdna_cbuf_priv *cbuf = dbuf->priv;
+	void *kva;
+
+	kva = memremap(cbuf->node.start, cbuf->node.size, MEMREMAP_WB);
+	if (!kva) {
+		pr_err("Failed to vmap carveout dma buf\n");
+		return -ENOMEM;
+	}
+
+	iosys_map_set_vaddr(map, kva);
+	return 0;
+}
+
+static void amdxdna_cbuf_vunmap(struct dma_buf *dbuf, struct iosys_map *map)
+{
+	memunmap(map->vaddr);
+}
+
+static const struct dma_buf_ops amdxdna_cbuf_dmabuf_ops = {
+	.map_dma_buf = amdxdna_cbuf_map,
+	.unmap_dma_buf = amdxdna_cbuf_unmap,
+	.release = amdxdna_cbuf_release,
+	.mmap = amdxdna_cbuf_mmap,
+	.vmap = amdxdna_cbuf_vmap,
+	.vunmap = amdxdna_cbuf_vunmap,
+};
+
+static int amdxdna_cbuf_clear(struct dma_buf *dbuf)
+{
+	struct iosys_map vmap = IOSYS_MAP_INIT_VADDR(NULL);
+
+	dma_buf_vmap(dbuf, &vmap);
+	if (!vmap.vaddr)
+		return -EFAULT;
+
+	memset(vmap.vaddr, 0, dbuf->size);
+	dma_buf_vunmap(dbuf, &vmap);
+
+	return 0;
+}
+
+struct dma_buf *amdxdna_get_cbuf(struct drm_device *dev, size_t size, u64 alignment)
+{
+	struct amdxdna_dev *xdna = to_xdna_dev(dev);
+	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
+	struct amdxdna_carveout *carveout;
+	struct amdxdna_cbuf_priv *cbuf;
+	struct dma_buf *dbuf;
+	int ret;
+
+	cbuf = kzalloc_obj(*cbuf);
+	if (!cbuf)
+		return ERR_PTR(-ENOMEM);
+	cbuf->xdna = xdna;
+
+	carveout = xdna->carveout;
+	mutex_lock(&carveout->lock);
+	ret = drm_mm_insert_node_generic(&carveout->mm, &cbuf->node, size,
+					 alignment, 0, DRM_MM_INSERT_BEST);
+	mutex_unlock(&carveout->lock);
+	if (ret)
+		goto free_cbuf;
+
+	exp_info.size = size;
+	exp_info.ops = &amdxdna_cbuf_dmabuf_ops;
+	exp_info.priv = cbuf;
+	exp_info.flags = O_RDWR;
+	dbuf = dma_buf_export(&exp_info);
+	if (IS_ERR(dbuf)) {
+		ret = PTR_ERR(dbuf);
+		goto remove_node;
+	}
+
+	ret = amdxdna_cbuf_clear(dbuf);
+	if (ret) {
+		dma_buf_put(dbuf);
+		goto out;
+	}
+	return dbuf;
+
+remove_node:
+	drm_mm_remove_node(&cbuf->node);
+free_cbuf:
+	kfree(cbuf);
+out:
+	return ERR_PTR(ret);
+}
diff --git a/drivers/accel/amdxdna/amdxdna_cbuf.h b/drivers/accel/amdxdna/amdxdna_cbuf.h
new file mode 100644
index 0000000000000..8e89336ffd506
--- /dev/null
+++ b/drivers/accel/amdxdna/amdxdna_cbuf.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+#ifndef _AMDXDNA_CBUF_H_
+#define _AMDXDNA_CBUF_H_
+
+#include "amdxdna_pci_drv.h"
+#include <drm/drm_device.h>
+#include <linux/dma-buf.h>
+
+bool amdxdna_use_carveout(struct amdxdna_dev *xdna);
+int amdxdna_carveout_init(struct amdxdna_dev *xdna, u64 carveout_addr, u64 carveout_size);
+void amdxdna_carveout_fini(struct amdxdna_dev *xdna);
+void amdxdna_get_carveout_conf(struct amdxdna_dev *xdna, u64 *addr, u64 *size);
+struct dma_buf *amdxdna_get_cbuf(struct drm_device *dev, size_t size, u64 alignment);
+
+#endif
diff --git a/drivers/accel/amdxdna/amdxdna_ctx.c b/drivers/accel/amdxdna/amdxdna_ctx.c
index ff6c3e8e5a15b..855da8c79a1cd 100644
--- a/drivers/accel/amdxdna/amdxdna_ctx.c
+++ b/drivers/accel/amdxdna/amdxdna_ctx.c
@@ -61,16 +61,35 @@ static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx)
 	return &fence->base;
 }
 
+static void amdxdna_hwctx_release_expanded_heap(struct amdxdna_hwctx *hwctx)
+{
+	struct amdxdna_client *client = hwctx->client;
+	struct amdxdna_gem_obj *heap;
+	unsigned long heap_id;
+
+	mutex_lock(&client->mm_lock);
+	if (hwctx->last_attached_heap) {
+		xa_for_each_range(&client->dev_heap_xa, heap_id, heap, 1,
+				  hwctx->last_attached_heap) {
+			amdxdna_gem_unpin(heap);
+			drm_gem_object_put(to_gobj(heap));
+		}
+	}
+	mutex_unlock(&client->mm_lock);
+}
+
 static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx,
 				      struct srcu_struct *ss)
 {
-	struct amdxdna_dev *xdna = hwctx->client->xdna;
+	struct amdxdna_client *client = hwctx->client;
+	struct amdxdna_dev *xdna = client->xdna;
 
 	synchronize_srcu(ss);
 
 	/* At this point, user is not able to submit new commands */
 	xdna->dev_info->ops->hwctx_fini(hwctx);
 
+	amdxdna_hwctx_release_expanded_heap(hwctx);
 	kfree(hwctx->name);
 	kfree(hwctx);
 }
@@ -207,6 +226,9 @@ int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct dr
 	if (args->ext || args->ext_flags)
 		return -EINVAL;
 
+	if (!xdna->dev_info->ops->hwctx_init)
+		return -EOPNOTSUPP;
+
 	hwctx = kzalloc_obj(*hwctx);
 	if (!hwctx)
 		return -ENOMEM;
@@ -220,6 +242,8 @@ int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct dr
 	hwctx->client = client;
 	hwctx->fw_ctx_id = -1;
 	hwctx->num_tiles = args->num_tiles;
+	hwctx->umq_bo_hdl = args->umq_bo;
+	hwctx->doorbell_offset = AMDXDNA_INVALID_DOORBELL_OFFSET;
 	hwctx->mem_size = args->mem_size;
 	hwctx->max_opc = args->max_opc;
 
@@ -233,7 +257,7 @@ int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct dr
 	ret = xdna->dev_info->ops->hwctx_init(hwctx);
 	if (ret) {
 		XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret);
-		goto dev_exit;
+		goto release_expanded_heap;
 	}
 
 	hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->fw_ctx_id);
@@ -252,6 +276,7 @@ int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct dr
 
 	args->handle = hwctx->id;
 	args->syncobj_handle = hwctx->syncobj_hdl;
+	args->umq_doorbell = hwctx->doorbell_offset;
 
 	atomic64_set(&hwctx->job_submit_cnt, 0);
 	atomic64_set(&hwctx->job_free_cnt, 0);
@@ -263,7 +288,8 @@ int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct dr
 	kfree(hwctx->name);
 fini_hwctx:
 	xdna->dev_info->ops->hwctx_fini(hwctx);
-dev_exit:
+release_expanded_heap:
+	amdxdna_hwctx_release_expanded_heap(hwctx);
 	drm_dev_exit(idx);
 free_hwctx:
 	kfree(hwctx);
@@ -401,6 +427,62 @@ int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl)
 	return ret;
 }
 
+static int amdxdna_hwctx_expand_heap(struct amdxdna_hwctx *hwctx)
+{
+	struct amdxdna_client *client = hwctx->client;
+	struct amdxdna_dev *xdna = client->xdna;
+	struct amdxdna_gem_obj *heap;
+	unsigned long heap_id, nid;
+	int ret = 0;
+
+	nid = hwctx->last_attached_heap + 1;
+	if (nid == client->dev_heap_nid)
+		goto out;
+
+	xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
+			  nid, client->dev_heap_nid) {
+		drm_gem_object_get(to_gobj(heap));
+		ret = amdxdna_gem_pin(heap);
+		if (ret) {
+			drm_gem_object_put(to_gobj(heap));
+			break;
+		}
+
+		mutex_unlock(&client->mm_lock);
+		ret = xdna->dev_info->ops->hwctx_heap_expand(hwctx, heap);
+		mutex_lock(&client->mm_lock);
+		if (ret) {
+			amdxdna_gem_unpin(heap);
+			drm_gem_object_put(to_gobj(heap));
+			break;
+		}
+
+		hwctx->last_attached_heap = heap_id;
+	}
+
+out:
+	return ret;
+}
+
+int amdxdna_update_heap(struct amdxdna_client *client, struct amdxdna_hwctx *hwctx)
+{
+	unsigned long hwctx_id;
+	int ret;
+
+	guard(mutex)(&client->mm_lock);
+
+	if (hwctx)
+		return amdxdna_hwctx_expand_heap(hwctx);
+
+	amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
+		ret = amdxdna_hwctx_expand_heap(hwctx);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static void
 amdxdna_arg_bos_put(struct amdxdna_sched_job *job)
 {
@@ -514,7 +596,6 @@ int amdxdna_cmd_submit(struct amdxdna_client *client,
 		goto unlock_srcu;
 	}
 
-
 	job->hwctx = hwctx;
 	job->mm = current->mm;
 
@@ -612,6 +693,8 @@ int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_
 	if (args->ext || args->ext_flags)
 		return -EINVAL;
 
+	trace_amdxdna_debug_point(current->comm, args->type, "job received");
+
 	switch (args->type) {
 	case AMDXDNA_CMD_SUBMIT_EXEC_BUF:
 		return amdxdna_drm_submit_execbuf(client, args);
@@ -620,3 +703,37 @@ int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_
 	XDNA_ERR(client->xdna, "Invalid command type %d", args->type);
 	return -EINVAL;
 }
+
+int amdxdna_drm_wait_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
+{
+	struct amdxdna_client *client = filp->driver_priv;
+	struct amdxdna_dev *xdna = to_xdna_dev(dev);
+	struct amdxdna_drm_wait_cmd *args = data;
+	struct amdxdna_hwctx *hwctx;
+	int ret, idx;
+
+	XDNA_DBG(xdna, "PID %d ctx %d timeout set %d ms for cmd %llu",
+		 client->pid, args->hwctx, args->timeout, args->seq);
+
+	if (!xdna->dev_info->ops->cmd_wait)
+		return -EOPNOTSUPP;
+
+	idx = srcu_read_lock(&client->hwctx_srcu);
+	hwctx = xa_load(&client->hwctx_xa, args->hwctx);
+	if (!hwctx) {
+		XDNA_DBG(xdna, "PID %d failed to get ctx %d", client->pid, args->hwctx);
+		ret = -EINVAL;
+		goto unlock_ctx_srcu;
+	}
+
+	ret = xdna->dev_info->ops->cmd_wait(hwctx, args->seq, args->timeout);
+
+	XDNA_DBG(xdna, "PID %d ctx %d cmd %lld wait finished, ret %d",
+		 client->pid, args->hwctx, args->seq, ret);
+
+	trace_amdxdna_debug_point(current->comm, args->seq, "job returned to user");
+
+unlock_ctx_srcu:
+	srcu_read_unlock(&client->hwctx_srcu, idx);
+	return ret;
+}
diff --git a/drivers/accel/amdxdna/amdxdna_ctx.h b/drivers/accel/amdxdna/amdxdna_ctx.h
index a8557d7e89238..aaae164304661 100644
--- a/drivers/accel/amdxdna/amdxdna_ctx.h
+++ b/drivers/accel/amdxdna/amdxdna_ctx.h
@@ -14,6 +14,7 @@ struct amdxdna_hwctx_priv;
 
 enum ert_cmd_opcode {
 	ERT_START_CU = 0,
+	ERT_START_DPU = 18,
 	ERT_CMD_CHAIN = 19,
 	ERT_START_NPU = 20,
 	ERT_START_NPU_PREEMPT = 21,
@@ -105,7 +106,10 @@ struct amdxdna_hwctx {
 	u32				*col_list;
 	u32				start_col;
 	u32				num_col;
+	u32				umq_bo_hdl;
+	u32				doorbell_offset;
 	u32				num_unused_col;
+	u32				last_attached_heap;
 
 	struct amdxdna_qos_info		     qos;
 	struct amdxdna_hwctx_param_config_cu *cus;
@@ -119,6 +123,7 @@ struct amdxdna_hwctx {
 	container_of(j, struct amdxdna_sched_job, base)
 
 enum amdxdna_job_opcode {
+	DEFAULT_IO,
 	SYNC_DEBUG_BO,
 	ATTACH_DEBUG_BO,
 	DETACH_DEBUG_BO,
@@ -201,18 +206,17 @@ void amdxdna_hwctx_remove_all(struct amdxdna_client *client);
 int amdxdna_hwctx_walk(struct amdxdna_client *client, void *arg,
 		       int (*walk)(struct amdxdna_hwctx *hwctx, void *arg));
 int amdxdna_hwctx_sync_debug_bo(struct amdxdna_client *client, u32 debug_bo_hdl);
+int amdxdna_update_heap(struct amdxdna_client *client, struct amdxdna_hwctx *hwctx);
 
 int amdxdna_cmd_submit(struct amdxdna_client *client,
 		       struct amdxdna_drv_cmd *drv_cmd, u32 cmd_bo_hdls,
 		       u32 *arg_bo_hdls, u32 arg_bo_cnt,
 		       u32 hwctx_hdl, u64 *seq);
 
-int amdxdna_cmd_wait(struct amdxdna_client *client, u32 hwctx_hdl,
-		     u64 seq, u32 timeout);
-
 int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
+int amdxdna_drm_wait_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 
 #endif /* _AMDXDNA_CTX_H_ */
diff --git a/drivers/accel/amdxdna/amdxdna_debugfs.c b/drivers/accel/amdxdna/amdxdna_debugfs.c
new file mode 100644
index 0000000000000..a6ec17c636295
--- /dev/null
+++ b/drivers/accel/amdxdna/amdxdna_debugfs.c
@@ -0,0 +1,129 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include "amdxdna_cbuf.h"
+#include "amdxdna_debugfs.h"
+
+#include <drm/drm_file.h>
+#include <linux/debugfs.h>
+#include <linux/pm_runtime.h>
+#include <linux/seq_file.h>
+#include <linux/string.h>
+#include <linux/uaccess.h>
+
+#define _DBGFS_FOPS(_open, _release, _write) \
+{ \
+	.owner = THIS_MODULE, \
+	.open = _open, \
+	.read = seq_read, \
+	.llseek = seq_lseek, \
+	.release = _release, \
+	.write = _write, \
+}
+
+#define AMDXDNA_DBGFS_FOPS(_name, _show, _write) \
+	static int amdxdna_dbgfs_##_name##_open(struct inode *inode, struct file *file) \
+	{ \
+		return single_open(file, _show, inode->i_private); \
+	} \
+	static int amdxdna_dbgfs_##_name##_release(struct inode *inode, struct file *file) \
+	{ \
+		return single_release(inode, file); \
+	} \
+	static const struct file_operations amdxdna_fops_##_name = \
+		_DBGFS_FOPS(amdxdna_dbgfs_##_name##_open, amdxdna_dbgfs_##_name##_release, _write)
+
+#define AMDXDNA_DBGFS_FILE(_name, _mode) { #_name, &amdxdna_fops_##_name, _mode }
+
+#define file_to_xdna(file) (((struct seq_file *)(file)->private_data)->private)
+
+static ssize_t amdxdna_carveout_write(struct file *file, const char __user *buf,
+				      size_t count, loff_t *ppos)
+{
+	struct amdxdna_dev *xdna = file_to_xdna(file);
+	char kbuf[128];
+	u64 size, addr;
+	char *sep;
+	int ret;
+
+	if (count == 0 || count >= sizeof(kbuf))
+		return -EINVAL;
+
+	if (copy_from_user(kbuf, buf, count))
+		return -EFAULT;
+	kbuf[count] = '\0';
+	strim(kbuf);
+	XDNA_DBG(xdna, "Trying to set carveout to %s", kbuf);
+
+	sep = strchr(kbuf, '@');
+	if (!sep)
+		return -EINVAL;
+	*sep = '\0';
+	sep++;
+
+	ret = kstrtou64(kbuf, 0, &size);
+	if (ret)
+		return ret;
+
+	ret = kstrtou64(sep, 0, &addr);
+	if (ret)
+		return ret;
+
+	/* Sanity check the addr and size. */
+	if (!size)
+		return -EINVAL;
+	if (!IS_ALIGNED(addr, PAGE_SIZE) || !IS_ALIGNED(size, PAGE_SIZE))
+		return -EINVAL;
+
+	guard(mutex)(&xdna->dev_lock);
+
+	ret = amdxdna_carveout_init(xdna, addr, size);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static int amdxdna_carveout_show(struct seq_file *m, void *unused)
+{
+	struct amdxdna_dev *xdna = m->private;
+	u64 addr, size;
+
+	guard(mutex)(&xdna->dev_lock);
+	amdxdna_get_carveout_conf(xdna, &addr, &size);
+	seq_printf(m, "0x%llx@0x%llx\n", size, addr);
+	return 0;
+}
+
+/*
+ * Input/output format: <carveout_size>@<carveout_address>
+ */
+AMDXDNA_DBGFS_FOPS(carveout, amdxdna_carveout_show, amdxdna_carveout_write);
+
+static const struct {
+	const char *name;
+	const struct file_operations *fops;
+	umode_t mode;
+} amdxdna_dbgfs_files[] = {
+	AMDXDNA_DBGFS_FILE(carveout, 0600),
+};
+
+void amdxdna_debugfs_init(struct amdxdna_dev *xdna)
+{
+	struct drm_minor *minor = xdna->ddev.accel;
+	int i;
+
+	/*
+	 * It should be okay that debugfs fails to init.
+	 * We rely on DRM framework to finish debugfs.
+	 */
+	for (i = 0; i < ARRAY_SIZE(amdxdna_dbgfs_files); i++) {
+		debugfs_create_file(amdxdna_dbgfs_files[i].name,
+				    amdxdna_dbgfs_files[i].mode,
+				    minor->debugfs_root,
+				    xdna,
+				    amdxdna_dbgfs_files[i].fops);
+	}
+}
diff --git a/drivers/accel/amdxdna/amdxdna_debugfs.h b/drivers/accel/amdxdna/amdxdna_debugfs.h
new file mode 100644
index 0000000000000..2abb45de3f7e8
--- /dev/null
+++ b/drivers/accel/amdxdna/amdxdna_debugfs.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+#ifndef _AMDXDNA_DEBUGFS_H_
+#define _AMDXDNA_DEBUGFS_H_
+
+#include "amdxdna_pci_drv.h"
+
+#if defined(CONFIG_DEBUG_FS)
+void amdxdna_debugfs_init(struct amdxdna_dev *xdna);
+#else
+static inline void amdxdna_debugfs_init(struct amdxdna_dev *xdna)
+{
+}
+#endif /* CONFIG_DEBUG_FS */
+
+#endif
diff --git a/drivers/accel/amdxdna/amdxdna_gem.c b/drivers/accel/amdxdna/amdxdna_gem.c
index 238ee244d4a69..00efa8abfeeac 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.c
+++ b/drivers/accel/amdxdna/amdxdna_gem.c
@@ -16,6 +16,7 @@
 #include <linux/pagemap.h>
 #include <linux/vmalloc.h>
 
+#include "amdxdna_cbuf.h"
 #include "amdxdna_ctx.h"
 #include "amdxdna_gem.h"
 #include "amdxdna_pci_drv.h"
@@ -23,6 +24,77 @@
 
 MODULE_IMPORT_NS("DMA_BUF");
 
+/*
+ * The dev BO could be across multiple heap BO chunks. The heap chunks should
+ * be mapped to userspace and the userspace virtual address should be
+ * contiguous.
+ */
+static int
+amdxdna_init_dev_bo(struct amdxdna_gem_obj *dev_bo)
+{
+	struct amdxdna_client *client = dev_bo->client;
+	struct amdxdna_dev *xdna = client->xdna;
+	struct amdxdna_gem_obj *heap;
+	u64 heap_addr, exp_heap_uva;
+	u32 heap_id;
+
+	if (xa_empty(&client->dev_heap_xa)) {
+		XDNA_DBG(xdna, "Empty heap xa");
+		return -EAGAIN;
+	}
+
+	for (heap_id = 0; heap_id < client->dev_heap_nid; heap_id++) {
+		heap = xa_load(&client->dev_heap_xa, heap_id);
+		if (!heap) {
+			XDNA_ERR(xdna, "Failed to load heap %d", heap_id);
+			return -EINVAL;
+		}
+		heap_addr = amdxdna_gem_dev_addr(heap);
+		if (heap_addr > dev_bo->mm_node.start)
+			break;
+	}
+
+	heap_id--;
+	heap = xa_load(&client->dev_heap_xa, heap_id);
+	exp_heap_uva = amdxdna_gem_uva(heap);
+	heap_addr = amdxdna_gem_dev_addr(heap);
+	dev_bo->heap_start_id = heap_id;
+	dev_bo->mem.uva = dev_bo->mm_node.start - heap_addr + exp_heap_uva;
+
+	for (; heap_id < client->dev_heap_nid; heap_id++) {
+		heap = xa_load(&client->dev_heap_xa, heap_id);
+		if (!heap) {
+			XDNA_ERR(xdna, "Failed to load heap %d", heap_id);
+			return -EINVAL;
+		}
+		heap_addr = amdxdna_gem_uva(heap);
+		if (heap_addr == AMDXDNA_INVALID_ADDR) {
+			XDNA_ERR(xdna, "Heap %d is not mapped", heap_id);
+			return -EAGAIN;
+		}
+
+		if (heap_addr != exp_heap_uva) {
+			XDNA_ERR(xdna, "Heap %d uva is not contiguous", heap_id);
+			return -EINVAL;
+		}
+
+		if (heap->dev_addr + heap->mem.size >=
+		    dev_bo->mm_node.start + dev_bo->mem.size)
+			break;
+
+		exp_heap_uva += heap->mem.size;
+	}
+
+	if (heap_id == client->dev_heap_nid) {
+		XDNA_DBG(xdna, "Can not find heap end");
+		return -EAGAIN;
+	}
+
+	dev_bo->heap_end_id = heap_id;
+
+	return 0;
+}
+
 static int
 amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo)
 {
@@ -30,32 +102,22 @@ amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo)
 	struct amdxdna_dev *xdna = client->xdna;
 	struct amdxdna_mem *mem = &abo->mem;
 	struct amdxdna_gem_obj *heap;
+	unsigned long heap_id;
 	u32 align;
 	int ret;
 
 	mutex_lock(&client->mm_lock);
 
-	heap = client->dev_heap;
-	if (!heap) {
-		ret = -EINVAL;
-		goto unlock_out;
-	}
-
-	if (amdxdna_gem_uva(heap) == AMDXDNA_INVALID_ADDR) {
-		XDNA_ERR(xdna, "Invalid dev heap userptr");
-		ret = -EINVAL;
-		goto unlock_out;
-	}
-
-	if (mem->size == 0 || mem->size > heap->mem.size) {
-		XDNA_ERR(xdna, "Invalid dev bo size 0x%lx, limit 0x%lx",
-			 mem->size, heap->mem.size);
+	if (!mem->size || mem->size > xdna->dev_info->dev_heap_max_size) {
+		XDNA_ERR(xdna, "Invalid dev bo size 0x%lx, max heap 0x%lx",
+			 mem->size, xdna->dev_info->dev_heap_max_size);
 		ret = -EINVAL;
 		goto unlock_out;
 	}
 
 	align = 1 << max(PAGE_SHIFT, xdna->dev_info->dev_mem_buf_shift);
-	ret = drm_mm_insert_node_generic(&heap->mm, &abo->mm_node,
+	ret = drm_mm_insert_node_generic(&client->dev_heap_mm,
+					 &abo->mm_node,
 					 mem->size, align,
 					 0, DRM_MM_INSERT_BEST);
 	if (ret) {
@@ -63,9 +125,16 @@ amdxdna_gem_heap_alloc(struct amdxdna_gem_obj *abo)
 		goto unlock_out;
 	}
 
-	client->heap_usage += mem->size;
+	ret = amdxdna_init_dev_bo(abo);
+	if (ret) {
+		drm_mm_remove_node(&abo->mm_node);
+		goto unlock_out;
+	}
 
-	drm_gem_object_get(to_gobj(heap));
+	client->heap_usage += mem->size;
+	xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
+			  abo->heap_start_id, abo->heap_end_id)
+		drm_gem_object_get(to_gobj(heap));
 
 unlock_out:
 	mutex_unlock(&client->mm_lock);
@@ -78,13 +147,16 @@ amdxdna_gem_heap_free(struct amdxdna_gem_obj *abo)
 {
 	struct amdxdna_client *client = abo->client;
 	struct amdxdna_gem_obj *heap;
+	unsigned long heap_id;
 
 	mutex_lock(&client->mm_lock);
 
 	drm_mm_remove_node(&abo->mm_node);
 	client->heap_usage -= abo->mem.size;
-	heap = client->dev_heap;
-	drm_gem_object_put(to_gobj(heap));
+
+	xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
+			  abo->heap_start_id, abo->heap_end_id)
+		drm_gem_object_put(to_gobj(heap));
 
 	mutex_unlock(&client->mm_lock);
 }
@@ -160,31 +232,13 @@ static void amdxdna_gem_vunmap(struct amdxdna_gem_obj *abo)
 	}
 }
 
-/*
- * Obtain the user virtual address for accessing the BO.
- * It can be used for device to access the BO when PASID is enabled.
- */
-u64 amdxdna_gem_uva(struct amdxdna_gem_obj *abo)
-{
-	if (abo->type == AMDXDNA_BO_DEV) {
-		struct amdxdna_gem_obj *heap = abo->client->dev_heap;
-		u64 off = amdxdna_dev_bo_offset(abo);
-
-		if (amdxdna_gem_uva(heap) != AMDXDNA_INVALID_ADDR)
-			return amdxdna_gem_uva(heap) + off;
-		return AMDXDNA_INVALID_ADDR;
-	}
-
-	return abo->mem.uva;
-}
-
 /*
  * Obtain the address for device to access the BO.
  */
 u64 amdxdna_gem_dev_addr(struct amdxdna_gem_obj *abo)
 {
 	if (abo->type == AMDXDNA_BO_DEV_HEAP)
-		return abo->client->xdna->dev_info->dev_mem_base;
+		return abo->dev_addr;
 	if (abo->type == AMDXDNA_BO_DEV)
 		return abo->mm_node.start;
 	return amdxdna_obj_dma_addr(abo);
@@ -211,7 +265,8 @@ static bool amdxdna_hmm_invalidate(struct mmu_interval_notifier *mni,
 	mmu_interval_set_seq(&mapp->notifier, cur_seq);
 	up_write(&xdna->notifier_lock);
 
-	xdna->dev_info->ops->hmm_invalidate(abo, cur_seq);
+	if (xdna->dev_info->ops->hmm_invalidate)
+		xdna->dev_info->ops->hmm_invalidate(abo, cur_seq);
 
 	if (range->event == MMU_NOTIFY_UNMAP) {
 		down_write(&xdna->notifier_lock);
@@ -294,7 +349,7 @@ static int amdxdna_hmm_register(struct amdxdna_gem_obj *abo,
 	u32 nr_pages;
 	int ret;
 
-	if (!xdna->dev_info->ops->hmm_invalidate)
+	if (!amdxdna_pasid_on(abo->client))
 		return 0;
 
 	mapp = kzalloc_obj(*mapp);
@@ -490,6 +545,9 @@ static struct dma_buf *amdxdna_gem_prime_export(struct drm_gem_object *gobj, int
 	struct amdxdna_gem_obj *abo = to_xdna_obj(gobj);
 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 
+	if (abo->private_buffer)
+		return ERR_PTR(-EOPNOTSUPP);
+
 	if (abo->dma_buf) {
 		get_dma_buf(abo->dma_buf);
 		return abo->dma_buf;
@@ -516,10 +574,6 @@ static void amdxdna_imported_obj_free(struct amdxdna_gem_obj *abo)
 static inline bool
 amdxdna_gem_skip_bo_usage(struct amdxdna_gem_obj *abo)
 {
-	/* Do not count imported BOs since the buffer is not allocated by us. */
-	if (is_import_bo(abo))
-		return true;
-
 	/* Already counted as part of HEAP BO */
 	if (abo->type == AMDXDNA_BO_DEV)
 		return true;
@@ -568,12 +622,7 @@ static void amdxdna_gem_obj_free(struct drm_gem_object *gobj)
 	if (abo->pinned)
 		amdxdna_gem_unpin(abo);
 
-	if (abo->type == AMDXDNA_BO_DEV_HEAP)
-		drm_mm_takedown(&abo->mm);
-
-	if (amdxdna_iova_on(xdna))
-		amdxdna_iommu_unmap_bo(xdna, abo);
-
+	amdxdna_dma_unmap_bo(xdna, abo);
 	amdxdna_gem_vunmap(abo);
 	mutex_destroy(&abo->lock);
 
@@ -591,18 +640,20 @@ static int amdxdna_gem_obj_open(struct drm_gem_object *gobj, struct drm_file *fi
 
 	guard(mutex)(&abo->lock);
 	abo->open_ref++;
+	if (abo->open_ref > 1)
+		return 0;
 
-	if (abo->open_ref == 1) {
-		/* Attached to the client when first opened by it. */
-		abo->client = filp->driver_priv;
-		amdxdna_gem_add_bo_usage(abo);
-	}
-	if (amdxdna_iova_on(xdna)) {
-		ret = amdxdna_iommu_map_bo(xdna, abo);
+	/* Attached to the client when first opened by it. */
+	abo->client = filp->driver_priv;
+
+	/* No need to set up dma addr mapping in PASID mode. */
+	if (!amdxdna_pasid_on(abo->client)) {
+		ret = amdxdna_dma_map_bo(xdna, abo);
 		if (ret)
 			return ret;
 	}
 
+	amdxdna_gem_add_bo_usage(abo);
 	return 0;
 }
 
@@ -620,14 +671,59 @@ static void amdxdna_gem_obj_close(struct drm_gem_object *gobj, struct drm_file *
 	}
 }
 
+static int amdxdna_gem_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map)
+{
+	struct amdxdna_gem_obj *abo = to_xdna_obj(obj);
+	int ret;
+
+	iosys_map_clear(map);
+
+	dma_resv_assert_held(obj->resv);
+
+	if (is_import_bo(abo))
+		ret = dma_buf_vmap(abo->dma_buf, map);
+	else
+		ret = drm_gem_shmem_object_vmap(obj, map);
+	if (ret)
+		return ret;
+	if (!map->vaddr)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void amdxdna_gem_obj_vunmap(struct drm_gem_object *obj, struct iosys_map *map)
+{
+	struct amdxdna_gem_obj *abo = to_xdna_obj(obj);
+
+	dma_resv_assert_held(obj->resv);
+
+	if (is_import_bo(abo))
+		dma_buf_vunmap(abo->dma_buf, map);
+	else
+		drm_gem_shmem_object_vunmap(obj, map);
+}
+
 static int amdxdna_gem_dev_obj_vmap(struct drm_gem_object *obj, struct iosys_map *map)
 {
 	struct amdxdna_gem_obj *abo = to_xdna_obj(obj);
-	void *base = amdxdna_gem_vmap(abo->client->dev_heap);
-	u64 offset = amdxdna_dev_bo_offset(abo);
+	struct amdxdna_gem_obj *heap;
+	void *base;
+	u64 offset;
+
+	/* vmap dev bo which is across more than 1 heap is not allowed */
+	if (abo->heap_start_id != abo->heap_end_id)
+		return -ENOMEM;
 
+	heap = xa_load(&abo->client->dev_heap_xa, abo->heap_start_id);
+	if (!heap)
+		return -ENOMEM;
+
+	base = amdxdna_gem_vmap(heap);
 	if (!base)
 		return -ENOMEM;
+
+	offset = amdxdna_gem_dev_addr(abo) - amdxdna_gem_dev_addr(heap);
 	iosys_map_set_vaddr(map, base + offset);
 	return 0;
 }
@@ -645,8 +741,8 @@ static const struct drm_gem_object_funcs amdxdna_gem_shmem_funcs = {
 	.pin = drm_gem_shmem_object_pin,
 	.unpin = drm_gem_shmem_object_unpin,
 	.get_sg_table = drm_gem_shmem_object_get_sg_table,
-	.vmap = drm_gem_shmem_object_vmap,
-	.vunmap = drm_gem_shmem_object_vunmap,
+	.vmap = amdxdna_gem_obj_vmap,
+	.vunmap = amdxdna_gem_obj_vunmap,
 	.mmap = amdxdna_gem_obj_mmap,
 	.vm_ops = &drm_gem_shmem_vm_ops,
 	.export = amdxdna_gem_prime_export,
@@ -685,6 +781,7 @@ amdxdna_gem_create_ubuf_object(struct drm_device *dev, struct amdxdna_drm_create
 {
 	struct amdxdna_dev *xdna = to_xdna_dev(dev);
 	struct amdxdna_drm_va_tbl va_tbl;
+	struct amdxdna_gem_obj *abo;
 	struct drm_gem_object *gobj;
 	struct dma_buf *dma_buf;
 
@@ -711,7 +808,40 @@ amdxdna_gem_create_ubuf_object(struct drm_device *dev, struct amdxdna_drm_create
 
 	dma_buf_put(dma_buf);
 
-	return to_xdna_obj(gobj);
+	abo = to_xdna_obj(gobj);
+	abo->private_buffer = true;
+
+	return abo;
+}
+
+static struct amdxdna_gem_obj *
+amdxdna_gem_create_cbuf_object(struct drm_device *dev, struct amdxdna_drm_create_bo *args)
+{
+	struct amdxdna_dev *xdna = to_xdna_dev(dev);
+	size_t size = PAGE_ALIGN(args->size);
+	struct drm_gem_object *gobj;
+	struct amdxdna_gem_obj *ret;
+	struct dma_buf *dma_buf;
+	u64 align;
+
+	if (!size) {
+		XDNA_ERR(xdna, "Invalid BO size 0x%llx", args->size);
+		return ERR_PTR(-EINVAL);
+	}
+
+	align = (args->type == AMDXDNA_BO_DEV_HEAP) ?  xdna->dev_info->dev_mem_size : 0;
+	dma_buf = amdxdna_get_cbuf(dev, size, align);
+	if (IS_ERR(dma_buf))
+		return ERR_CAST(dma_buf);
+
+	gobj = amdxdna_gem_prime_import(dev, dma_buf);
+	if (IS_ERR(gobj))
+		ret = ERR_CAST(gobj);
+	else
+		ret = to_xdna_obj(gobj);
+
+	dma_buf_put(dma_buf);
+	return ret;
 }
 
 struct drm_gem_object *
@@ -769,6 +899,8 @@ amdxdna_drm_create_share_bo(struct drm_device *dev,
 
 	if (args->vaddr)
 		abo = amdxdna_gem_create_ubuf_object(dev, args);
+	else if (amdxdna_use_carveout(to_xdna_dev(dev)))
+		abo = amdxdna_gem_create_cbuf_object(dev, args);
 	else
 		abo = amdxdna_gem_create_shmem_object(dev, args);
 	if (IS_ERR(abo))
@@ -810,15 +942,32 @@ amdxdna_drm_create_dev_heap_bo(struct drm_device *dev,
 	/* Set up heap for this client. */
 	mutex_lock(&client->mm_lock);
 
-	if (client->dev_heap) {
-		XDNA_DBG(client->xdna, "dev heap is already created");
-		ret = -EBUSY;
+	if (!xdna->dev_info->ops->hwctx_heap_expand &&
+	    client->dev_heap_nid > 0) {
+		XDNA_ERR(xdna, "Heap expansion is not supported");
+		ret = -EOPNOTSUPP;
 		goto mm_unlock;
 	}
-	client->dev_heap = abo;
-	drm_gem_object_get(to_gobj(abo));
 
-	drm_mm_init(&abo->mm, xdna->dev_info->dev_mem_base, abo->mem.size);
+	if (client->total_heap_size + abo->mem.size >
+	    xdna->dev_info->dev_heap_max_size) {
+		XDNA_ERR(xdna, "Heap size 0x%lx + 0x%lx exceeds max 0x%lx",
+			 client->total_heap_size, abo->mem.size,
+			 xdna->dev_info->dev_heap_max_size);
+		ret = -ENOSPC;
+		goto mm_unlock;
+	}
+
+	ret = xa_insert(&client->dev_heap_xa, client->dev_heap_nid, abo, GFP_KERNEL);
+	if (ret) {
+		XDNA_ERR(xdna, "Add heap failed %d", ret);
+		goto mm_unlock;
+	}
+
+	abo->dev_addr = xdna->dev_info->dev_mem_base + client->total_heap_size;
+	client->total_heap_size += abo->mem.size;
+	client->dev_heap_nid++;
+	drm_gem_object_get(to_gobj(abo));
 
 	mutex_unlock(&client->mm_lock);
 
@@ -861,10 +1010,10 @@ amdxdna_drm_create_dev_bo(struct drm_device *dev,
 
 	ret = amdxdna_gem_heap_alloc(abo);
 	if (ret) {
-		XDNA_ERR(xdna, "Failed to alloc dev bo memory, ret %d", ret);
 		amdxdna_gem_destroy_obj(abo);
 		return ERR_PTR(ret);
 	}
+
 	drm_gem_private_object_init(dev, gobj, aligned_sz);
 
 	return abo;
@@ -872,6 +1021,7 @@ amdxdna_drm_create_dev_bo(struct drm_device *dev,
 
 int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 {
+	struct amdxdna_client *client = filp->driver_priv;
 	struct amdxdna_dev *xdna = to_xdna_dev(dev);
 	struct amdxdna_drm_create_bo *args = data;
 	struct amdxdna_gem_obj *abo;
@@ -884,7 +1034,6 @@ int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_f
 		 args->type, args->vaddr, args->size, args->flags);
 	switch (args->type) {
 	case AMDXDNA_BO_CMD:
-		fallthrough;
 	case AMDXDNA_BO_SHARE:
 		abo = amdxdna_drm_create_share_bo(dev, args, filp);
 		break;
@@ -893,6 +1042,13 @@ int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_f
 		break;
 	case AMDXDNA_BO_DEV:
 		abo = amdxdna_drm_create_dev_bo(dev, args, filp);
+		if (!IS_ERR(abo)) {
+			mutex_lock(&xdna->dev_lock);
+			ret = amdxdna_update_heap(client, NULL);
+			mutex_unlock(&xdna->dev_lock);
+			if (ret)
+				goto put_obj;
+		}
 		break;
 	default:
 		return -EINVAL;
@@ -916,14 +1072,11 @@ int amdxdna_drm_create_bo_ioctl(struct drm_device *dev, void *data, struct drm_f
 	return ret;
 }
 
-int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
+static int amdxdna_bo_pin(struct amdxdna_gem_obj *abo)
 {
 	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
 	int ret;
 
-	if (abo->type == AMDXDNA_BO_DEV)
-		abo = abo->client->dev_heap;
-
 	if (is_import_bo(abo))
 		return 0;
 
@@ -933,6 +1086,45 @@ int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
 	return ret;
 }
 
+static void amdxdna_bo_unpin(struct amdxdna_gem_obj *abo)
+{
+	struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev);
+
+	if (is_import_bo(abo))
+		return;
+
+	drm_gem_shmem_unpin(&abo->base);
+
+	XDNA_DBG(xdna, "BO type %d", abo->type);
+}
+
+int amdxdna_gem_pin_nolock(struct amdxdna_gem_obj *abo)
+{
+	struct amdxdna_client *client = abo->client;
+	struct amdxdna_gem_obj *heap;
+	unsigned long heap_id, last = ULONG_MAX;
+	int ret = 0;
+
+	if (abo->type != AMDXDNA_BO_DEV)
+		return amdxdna_bo_pin(abo);
+
+	xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
+			  abo->heap_start_id, abo->heap_end_id) {
+		ret = amdxdna_bo_pin(heap);
+		if (ret)
+			break;
+		last = heap_id;
+	}
+
+	if (ret && last <= abo->heap_end_id) {
+		xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
+				  abo->heap_start_id, last)
+			amdxdna_bo_unpin(heap);
+	}
+
+	return ret;
+}
+
 int amdxdna_gem_pin(struct amdxdna_gem_obj *abo)
 {
 	int ret;
@@ -946,14 +1138,18 @@ int amdxdna_gem_pin(struct amdxdna_gem_obj *abo)
 
 void amdxdna_gem_unpin(struct amdxdna_gem_obj *abo)
 {
-	if (abo->type == AMDXDNA_BO_DEV)
-		abo = abo->client->dev_heap;
+	mutex_lock(&abo->lock);
+	if (abo->type == AMDXDNA_BO_DEV) {
+		struct amdxdna_gem_obj *heap;
+		unsigned long heap_id;
 
-	if (is_import_bo(abo))
-		return;
+		xa_for_each_range(&abo->client->dev_heap_xa, heap_id, heap,
+				  abo->heap_start_id, abo->heap_end_id)
+			amdxdna_bo_unpin(heap);
+	} else {
+		amdxdna_bo_unpin(abo);
+	}
 
-	mutex_lock(&abo->lock);
-	drm_gem_shmem_unpin(&abo->base);
 	mutex_unlock(&abo->lock);
 }
 
@@ -1010,6 +1206,29 @@ int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm
 	return ret;
 }
 
+static int amdxdna_flush_bo(struct amdxdna_gem_obj *abo, u64 offset, u64 size)
+{
+	u64 end;
+
+	if (offset >= abo->mem.size)
+		return -EINVAL;
+
+	if (check_add_overflow(offset, size, &end))
+		return -EINVAL;
+
+	size = min(abo->mem.size, end) - offset;
+	if (is_import_bo(abo))
+		drm_clflush_sg(abo->base.sgt);
+	else if (amdxdna_gem_vmap(abo))
+		drm_clflush_virt_range(amdxdna_gem_vmap(abo) + offset, size);
+	else if (abo->base.pages)
+		drm_clflush_pages(abo->base.pages, abo->mem.size >> PAGE_SHIFT);
+	else
+		return -EINVAL;
+
+	return 0;
+}
+
 /*
  * The sync bo ioctl is to make sure the CPU cache is in sync with memory.
  * This is required because NPU is not cache coherent device. CPU cache
@@ -1020,11 +1239,12 @@ int amdxdna_drm_get_bo_info_ioctl(struct drm_device *dev, void *data, struct drm
 int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
 			      void *data, struct drm_file *filp)
 {
+	struct amdxdna_client *client = filp->driver_priv;
 	struct amdxdna_dev *xdna = to_xdna_dev(dev);
 	struct amdxdna_drm_sync_bo *args = data;
 	struct amdxdna_gem_obj *abo;
 	struct drm_gem_object *gobj;
-	int ret;
+	int ret = 0;
 
 	gobj = drm_gem_object_lookup(filp, args->handle);
 	if (!gobj) {
@@ -1033,22 +1253,45 @@ int amdxdna_drm_sync_bo_ioctl(struct drm_device *dev,
 	}
 	abo = to_xdna_obj(gobj);
 
-	ret = amdxdna_gem_pin(abo);
-	if (ret) {
-		XDNA_ERR(xdna, "Pin BO %d failed, ret %d", args->handle, ret);
-		goto put_obj;
-	}
+	if (abo->type == AMDXDNA_BO_DEV) {
+		struct amdxdna_gem_obj *heap;
+		unsigned long heap_id;
+		u64 bo_start = amdxdna_gem_dev_addr(abo);
+		u64 flush_start = bo_start + args->offset;
+		u64 flush_end = flush_start + args->size;
+
+		xa_for_each_range(&client->dev_heap_xa, heap_id, heap,
+				  abo->heap_start_id, abo->heap_end_id) {
+			u64 heap_start = amdxdna_gem_dev_addr(heap);
+			u64 heap_end = heap_start + heap->mem.size;
+			u64 start = max(flush_start, heap_start);
+			u64 end = min(flush_end, heap_end);
+
+			if (start >= end)
+				continue;
+
+			ret = amdxdna_flush_bo(heap, start - heap_start, end - start);
+			if (ret) {
+				XDNA_ERR(xdna, "Failed to flush heap %ld ret %d",
+					 heap_id, ret);
+				goto put_obj;
+			}
+		}
+	} else {
+		ret = amdxdna_gem_pin(abo);
+		if (ret) {
+			XDNA_ERR(xdna, "Pin BO %d failed, ret %d", args->handle, ret);
+			goto put_obj;
+		}
 
-	if (is_import_bo(abo))
-		drm_clflush_sg(abo->base.sgt);
-	else if (amdxdna_gem_vmap(abo))
-		drm_clflush_virt_range(amdxdna_gem_vmap(abo) + args->offset, args->size);
-	else if (abo->base.pages)
-		drm_clflush_pages(abo->base.pages, gobj->size >> PAGE_SHIFT);
-	else
-		drm_WARN(&xdna->ddev, 1, "Can not get flush memory");
+		ret = amdxdna_flush_bo(abo, args->offset, args->size);
+		amdxdna_gem_unpin(abo);
 
-	amdxdna_gem_unpin(abo);
+		if (ret) {
+			drm_WARN(&xdna->ddev, 1, "Can not get flush memory");
+			goto put_obj;
+		}
+	}
 
 	XDNA_DBG(xdna, "Sync bo %d offset 0x%llx, size 0x%llx\n",
 		 args->handle, args->offset, args->size);
diff --git a/drivers/accel/amdxdna/amdxdna_gem.h b/drivers/accel/amdxdna/amdxdna_gem.h
index 4fc48a1189d27..a3e44c7a23952 100644
--- a/drivers/accel/amdxdna/amdxdna_gem.h
+++ b/drivers/accel/amdxdna/amdxdna_gem.h
@@ -46,14 +46,18 @@ struct amdxdna_gem_obj {
 	int				open_ref;
 
 	/* Below members are initialized when needed */
-	struct drm_mm			mm; /* For AMDXDNA_BO_DEV_HEAP */
 	struct drm_mm_node		mm_node; /* For AMDXDNA_BO_DEV */
+	u32				heap_start_id;
+	u32				heap_end_id;
+	u64				dev_addr; /* For heap bo */
 	u32				assigned_hwctx;
 	struct dma_buf			*dma_buf;
 	struct dma_buf_attachment	*attach;
 
 	/* True, if BO is managed by XRT, not application */
 	bool				internal;
+	/* True, if BO is not exportable */
+	bool				private_buffer;
 };
 
 #define to_gobj(obj)    (&(obj)->base.base)
@@ -71,13 +75,21 @@ static inline void amdxdna_gem_put_obj(struct amdxdna_gem_obj *abo)
 	drm_gem_object_put(to_gobj(abo));
 }
 
+/*
+ * Obtain the user virtual address for accessing the BO.
+ * It can be used for device to access the BO when PASID is enabled.
+ */
+static inline u64 amdxdna_gem_uva(struct amdxdna_gem_obj *abo)
+{
+	return abo->mem.uva;
+}
+
 void *amdxdna_gem_vmap(struct amdxdna_gem_obj *abo);
-u64 amdxdna_gem_uva(struct amdxdna_gem_obj *abo);
 u64 amdxdna_gem_dev_addr(struct amdxdna_gem_obj *abo);
 
 static inline u64 amdxdna_dev_bo_offset(struct amdxdna_gem_obj *abo)
 {
-	return amdxdna_gem_dev_addr(abo) - amdxdna_gem_dev_addr(abo->client->dev_heap);
+	return amdxdna_gem_dev_addr(abo) - abo->client->xdna->dev_info->dev_mem_base;
 }
 
 static inline u64 amdxdna_obj_dma_addr(struct amdxdna_gem_obj *abo)
diff --git a/drivers/accel/amdxdna/amdxdna_iommu.c b/drivers/accel/amdxdna/amdxdna_iommu.c
index 4626434d4180c..eff00131d0f80 100644
--- a/drivers/accel/amdxdna/amdxdna_iommu.c
+++ b/drivers/accel/amdxdna/amdxdna_iommu.c
@@ -35,14 +35,15 @@ static struct iova *amdxdna_iommu_alloc_iova(struct amdxdna_dev *xdna,
 	return iova;
 }
 
-int amdxdna_iommu_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
+int amdxdna_dma_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
 {
+	unsigned long contig_sz;
 	struct sg_table *sgt;
 	dma_addr_t dma_addr;
 	struct iova *iova;
-	size_t size;
+	ssize_t size;
 
-	if (abo->type != AMDXDNA_BO_DEV_HEAP && abo->type != AMDXDNA_BO_SHMEM)
+	if (abo->type != AMDXDNA_BO_DEV_HEAP && abo->type != AMDXDNA_BO_SHARE)
 		return 0;
 
 	sgt = drm_gem_shmem_get_pages_sgt(&abo->base);
@@ -51,40 +52,63 @@ int amdxdna_iommu_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
 		return PTR_ERR(sgt);
 	}
 
-	if (!sgt->orig_nents || !sg_page(sgt->sgl)) {
-		XDNA_ERR(xdna, "sgl is zero length or not page backed");
+	if (!sgt->orig_nents) {
+		XDNA_ERR(xdna, "sgl is zero length");
 		return -EOPNOTSUPP;
 	}
 
-	iova = amdxdna_iommu_alloc_iova(xdna, abo->mem.size, &dma_addr,
-					(abo->type == AMDXDNA_BO_DEV_HEAP));
-	if (IS_ERR(iova)) {
-		XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova));
-		return PTR_ERR(iova);
-	}
-
-	size = iommu_map_sgtable(xdna->domain, dma_addr, sgt,
-				 IOMMU_READ | IOMMU_WRITE);
-	if (size < abo->mem.size) {
-		__free_iova(&xdna->iovad, iova);
-		return -ENXIO;
+	if (amdxdna_iova_on(xdna)) {
+		if (!sg_page(sgt->sgl)) {
+			XDNA_ERR(xdna, "sgl is not page backed");
+			return -EOPNOTSUPP;
+		}
+
+		iova = amdxdna_iommu_alloc_iova(xdna, abo->mem.size, &dma_addr,
+						(abo->type == AMDXDNA_BO_DEV_HEAP));
+		if (IS_ERR(iova)) {
+			XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova));
+			return PTR_ERR(iova);
+		}
+
+		size = iommu_map_sgtable(xdna->domain, dma_addr, sgt,
+					 IOMMU_READ | IOMMU_WRITE);
+		if (size < 0) {
+			XDNA_ERR(xdna, "iommu_map_sgtable failed: %zd", size);
+			__free_iova(&xdna->iovad, iova);
+			return size;
+		}
+		if (size < abo->mem.size) {
+			iommu_unmap(xdna->domain, dma_addr, size);
+			__free_iova(&xdna->iovad, iova);
+			return -ENXIO;
+		}
+		abo->mem.dma_addr = dma_addr;
+	} else {
+		/* Device doesn't support scatter/gather list, fail non-contiguous mapping. */
+		contig_sz = drm_prime_get_contiguous_size(sgt);
+		if (contig_sz < abo->mem.size) {
+			XDNA_ERR(xdna,
+				 "noncontiguous dma addr, contig size:%ld, expected size:%ld",
+				 contig_sz, abo->mem.size);
+			return -EINVAL;
+		}
+		abo->mem.dma_addr = sg_dma_address(sgt->sgl);
 	}
-
-	abo->mem.dma_addr = dma_addr;
-
 	return 0;
 }
 
-void amdxdna_iommu_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
+void amdxdna_dma_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
 {
 	size_t size;
 
 	if (abo->mem.dma_addr == AMDXDNA_INVALID_ADDR)
 		return;
 
-	size = iova_align(&xdna->iovad, abo->mem.size);
-	iommu_unmap(xdna->domain, abo->mem.dma_addr, size);
-	free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, abo->mem.dma_addr));
+	if (amdxdna_iova_on(xdna)) {
+		size = iova_align(&xdna->iovad, abo->mem.size);
+		iommu_unmap(xdna->domain, abo->mem.dma_addr, size);
+		free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, abo->mem.dma_addr));
+	}
 	abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
 }
 
@@ -110,10 +134,12 @@ void *amdxdna_iommu_alloc(struct amdxdna_dev *xdna, size_t size, dma_addr_t *dma
 			iova_align(&xdna->iovad, size),
 			IOMMU_READ | IOMMU_WRITE, GFP_KERNEL);
 	if (ret)
-		goto free_iova;
+		goto free_cpu_addr;
 
 	return cpu_addr;
 
+free_cpu_addr:
+	free_pages((unsigned long)cpu_addr, get_order(size));
 free_iova:
 	__free_iova(&xdna->iovad, iova);
 	return ERR_PTR(ret);
diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.c b/drivers/accel/amdxdna/amdxdna_mailbox.c
index e681a090752df..cc8865f4e79c9 100644
--- a/drivers/accel/amdxdna/amdxdna_mailbox.c
+++ b/drivers/accel/amdxdna/amdxdna_mailbox.c
@@ -112,6 +112,18 @@ static u32 mailbox_reg_read(struct mailbox_channel *mb_chann, u32 mbox_reg)
 	return readl(ringbuf_addr);
 }
 
+static inline void mailbox_irq_acknowledge(struct mailbox_channel *mb_chann)
+{
+	if (mb_chann->iohub_int_addr)
+		mailbox_reg_write(mb_chann, mb_chann->iohub_int_addr, 0);
+}
+
+static inline u32 mailbox_irq_status(struct mailbox_channel *mb_chann)
+{
+	return (mb_chann->iohub_int_addr) ?
+		mailbox_reg_read(mb_chann, mb_chann->iohub_int_addr) : 0;
+}
+
 static inline void
 mailbox_set_headptr(struct mailbox_channel *mb_chann, u32 headptr_val)
 {
@@ -199,7 +211,6 @@ mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg)
 	start_addr = mb_chann->res[CHAN_RES_X2I].rb_start_addr;
 	tmp_tail = tail + mb_msg->pkg_size;
 
-
 check_again:
 	if (tail >= head && tmp_tail > ringbuf_size) {
 		write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail;
@@ -350,6 +361,7 @@ static void mailbox_rx_worker(struct work_struct *rx_work)
 	int ret;
 
 	mb_chann = container_of(rx_work, struct mailbox_channel, rx_work);
+	trace_mbox_rx_worker(MAILBOX_NAME, mb_chann->msix_irq);
 
 	if (READ_ONCE(mb_chann->bad_state)) {
 		MB_ERR(mb_chann, "Channel in bad state, work aborted");
@@ -357,7 +369,7 @@ static void mailbox_rx_worker(struct work_struct *rx_work)
 	}
 
 again:
-	mailbox_reg_write(mb_chann, mb_chann->iohub_int_addr, 0);
+	mailbox_irq_acknowledge(mb_chann);
 
 	while (1) {
 		/*
@@ -382,7 +394,7 @@ static void mailbox_rx_worker(struct work_struct *rx_work)
 	 * the interrupt register to make sure there is not any new response
 	 * before exiting.
 	 */
-	if (mailbox_reg_read(mb_chann, mb_chann->iohub_int_addr))
+	if (mailbox_irq_status(mb_chann))
 		goto again;
 }
 
@@ -485,6 +497,9 @@ struct mailbox_channel *xdna_mailbox_alloc_channel(struct mailbox *mb)
 
 void xdna_mailbox_free_channel(struct mailbox_channel *mb_chann)
 {
+	if (!mb_chann)
+		return;
+
 	destroy_workqueue(mb_chann->work_q);
 	kfree(mb_chann);
 }
@@ -520,7 +535,7 @@ xdna_mailbox_start_channel(struct mailbox_channel *mb_chann,
 	}
 
 	mb_chann->bad_state = false;
-	mailbox_reg_write(mb_chann, mb_chann->iohub_int_addr, 0);
+	mailbox_irq_acknowledge(mb_chann);
 
 	MB_DBG(mb_chann, "Mailbox channel started (irq: %d)", mb_chann->msix_irq);
 	return 0;
@@ -531,6 +546,9 @@ void xdna_mailbox_stop_channel(struct mailbox_channel *mb_chann)
 	struct mailbox_msg *mb_msg;
 	unsigned long msg_id;
 
+	if (!mb_chann)
+		return;
+
 	/* Disable an irq and wait. This might sleep. */
 	free_irq(mb_chann->msix_irq, mb_chann);
 
@@ -538,7 +556,9 @@ void xdna_mailbox_stop_channel(struct mailbox_channel *mb_chann)
 	drain_workqueue(mb_chann->work_q);
 
 	/* We can clean up and release resources */
-	xa_for_each(&mb_chann->chan_xa, msg_id, mb_msg)
+	xa_for_each_start(&mb_chann->chan_xa, msg_id, mb_msg, mb_chann->next_msgid)
+		mailbox_release_msg(mb_chann, mb_msg);
+	xa_for_each_range(&mb_chann->chan_xa, msg_id, mb_msg, 0, mb_chann->next_msgid - 1)
 		mailbox_release_msg(mb_chann, mb_msg);
 	xa_destroy(&mb_chann->chan_xa);
 
diff --git a/drivers/accel/amdxdna/amdxdna_mailbox.h b/drivers/accel/amdxdna/amdxdna_mailbox.h
index 8b1e00945da48..2908404303aee 100644
--- a/drivers/accel/amdxdna/amdxdna_mailbox.h
+++ b/drivers/accel/amdxdna/amdxdna_mailbox.h
@@ -1,10 +1,10 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
+ * Copyright (C) 2022-2026, Advanced Micro Devices, Inc.
  */
 
-#ifndef _AIE2_MAILBOX_H_
-#define _AIE2_MAILBOX_H_
+#ifndef _AIE_MAILBOX_H_
+#define _AIE_MAILBOX_H_
 
 struct mailbox;
 struct mailbox_channel;
@@ -124,4 +124,4 @@ void xdna_mailbox_stop_channel(struct mailbox_channel *mailbox_chann);
 int xdna_mailbox_send_msg(struct mailbox_channel *mailbox_chann,
 			  const struct xdna_mailbox_msg *msg, u64 tx_timeout);
 
-#endif /* _AIE2_MAILBOX_ */
+#endif /* _AIE_MAILBOX_ */
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.c b/drivers/accel/amdxdna/amdxdna_pci_drv.c
index b50a7d1f8a118..c677293c1ae7f 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.c
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.c
@@ -14,7 +14,9 @@
 #include <linux/iommu.h>
 #include <linux/pci.h>
 
+#include "amdxdna_cbuf.h"
 #include "amdxdna_ctx.h"
+#include "amdxdna_debugfs.h"
 #include "amdxdna_gem.h"
 #include "amdxdna_pci_drv.h"
 #include "amdxdna_pm.h"
@@ -37,9 +39,11 @@ MODULE_FIRMWARE("amdnpu/17f0_11/npu_7.sbin");
  * 0.6: Support preemption
  * 0.7: Support getting power and utilization data
  * 0.8: Support BO usage query
+ * 0.9: Add new device type AMDXDNA_DEV_TYPE_PF
+ * 0.10: Support AIE4 UMQ
  */
 #define AMDXDNA_DRIVER_MAJOR		0
-#define AMDXDNA_DRIVER_MINOR		8
+#define AMDXDNA_DRIVER_MINOR		10
 
 /*
  * Bind the driver base on (vendor_id, device_id) pair and later use the
@@ -49,6 +53,10 @@ MODULE_FIRMWARE("amdnpu/17f0_11/npu_7.sbin");
 static const struct pci_device_id pci_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1502) },
 	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f0) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x17f3) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1B0B) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, 0x1B0C) },
 	{0}
 };
 
@@ -59,14 +67,47 @@ static const struct amdxdna_device_id amdxdna_ids[] = {
 	{ 0x17f0, 0x10, &dev_npu4_info },
 	{ 0x17f0, 0x11, &dev_npu5_info },
 	{ 0x17f0, 0x20, &dev_npu6_info },
+	{ 0x17f2, 0x10, &dev_npu3_pf_info },
+	{ 0x17f3, 0x10, &dev_npu3_vf_info },
+	{ 0x1B0B, 0x10, &dev_npu3_pf_info },
+	{ 0x1B0C, 0x10, &dev_npu3_vf_info },
 	{0}
 };
 
+static int amdxdna_sva_init(struct amdxdna_client *client)
+{
+	struct amdxdna_dev *xdna = client->xdna;
+
+	client->sva = iommu_sva_bind_device(xdna->ddev.dev, client->mm);
+	if (IS_ERR(client->sva)) {
+		XDNA_ERR(xdna, "SVA bind device failed, ret %ld", PTR_ERR(client->sva));
+		return PTR_ERR(client->sva);
+	}
+
+	client->pasid = iommu_sva_get_pasid(client->sva);
+	if (client->pasid == IOMMU_PASID_INVALID) {
+		iommu_sva_unbind_device(client->sva);
+		XDNA_ERR(xdna, "SVA get pasid failed");
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
+static void amdxdna_sva_fini(struct amdxdna_client *client)
+{
+	if (IS_ERR_OR_NULL(client->sva))
+		return;
+
+	iommu_sva_unbind_device(client->sva);
+	client->sva = NULL;
+	client->pasid = IOMMU_PASID_INVALID;
+}
+
 static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
 {
 	struct amdxdna_dev *xdna = to_xdna_dev(ddev);
 	struct amdxdna_client *client;
-	int ret;
 
 	client = kzalloc_obj(*client);
 	if (!client)
@@ -75,25 +116,19 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
 	client->pid = pid_nr(rcu_access_pointer(filp->pid));
 	client->xdna = xdna;
 	client->pasid = IOMMU_PASID_INVALID;
+	client->mm = current->mm;
 
 	if (!amdxdna_iova_on(xdna)) {
-		client->sva = iommu_sva_bind_device(xdna->ddev.dev, current->mm);
-		if (IS_ERR(client->sva)) {
-			ret = PTR_ERR(client->sva);
-			XDNA_ERR(xdna, "SVA bind device failed, ret %d", ret);
-			goto failed;
-		}
-		client->pasid = iommu_sva_get_pasid(client->sva);
-		if (client->pasid == IOMMU_PASID_INVALID) {
-			XDNA_ERR(xdna, "SVA get pasid failed");
-			ret = -ENODEV;
-			goto unbind_sva;
-		}
+		/* No need to fail open since user may use pa + carveout later. */
+		if (amdxdna_sva_init(client))
+			XDNA_WARN(xdna, "PASID not available for pid %d", client->pid);
 	}
-	client->mm = current->mm;
 	mmgrab(client->mm);
 	init_srcu_struct(&client->hwctx_srcu);
 	xa_init_flags(&client->hwctx_xa, XA_FLAGS_ALLOC);
+	xa_init_flags(&client->dev_heap_xa, XA_FLAGS_ALLOC);
+	drm_mm_init(&client->dev_heap_mm, xdna->dev_info->dev_mem_base,
+		    xdna->dev_info->dev_heap_max_size);
 	mutex_init(&client->mm_lock);
 
 	mutex_lock(&xdna->dev_lock);
@@ -105,32 +140,26 @@ static int amdxdna_drm_open(struct drm_device *ddev, struct drm_file *filp)
 
 	XDNA_DBG(xdna, "pid %d opened", client->pid);
 	return 0;
-
-unbind_sva:
-	if (!IS_ERR_OR_NULL(client->sva))
-		iommu_sva_unbind_device(client->sva);
-failed:
-	kfree(client);
-
-	return ret;
 }
 
 static void amdxdna_client_cleanup(struct amdxdna_client *client)
 {
+	struct amdxdna_gem_obj *heap;
+	unsigned long heap_id;
+
 	list_del(&client->node);
 	amdxdna_hwctx_remove_all(client);
 	xa_destroy(&client->hwctx_xa);
 	cleanup_srcu_struct(&client->hwctx_srcu);
 
-	if (client->dev_heap)
-		drm_gem_object_put(to_gobj(client->dev_heap));
+	xa_for_each(&client->dev_heap_xa, heap_id, heap)
+		drm_gem_object_put(to_gobj(heap));
+	xa_destroy(&client->dev_heap_xa);
+	drm_mm_takedown(&client->dev_heap_mm);
 
 	mutex_destroy(&client->mm_lock);
-
-	if (!IS_ERR_OR_NULL(client->sva))
-		iommu_sva_unbind_device(client->sva);
 	mmdrop(client->mm);
-
+	amdxdna_sva_fini(client);
 	kfree(client);
 }
 
@@ -204,6 +233,21 @@ static int amdxdna_drm_set_state_ioctl(struct drm_device *dev, void *data, struc
 	return ret;
 }
 
+static int amdxdna_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct drm_file *drm_filp = filp->private_data;
+	struct amdxdna_client *client = drm_filp->driver_priv;
+	struct amdxdna_dev *xdna = client->xdna;
+
+	if (likely(vma->vm_pgoff >= DRM_FILE_PAGE_OFFSET_START))
+		return drm_gem_mmap(filp, vma);
+
+	if (!xdna->dev_info->ops->mmap)
+		return -EOPNOTSUPP;
+
+	return xdna->dev_info->ops->mmap(client, vma);
+}
+
 static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = {
 	/* Context */
 	DRM_IOCTL_DEF_DRV(AMDXDNA_CREATE_HWCTX, amdxdna_drm_create_hwctx_ioctl, 0),
@@ -215,12 +259,44 @@ static const struct drm_ioctl_desc amdxdna_drm_ioctls[] = {
 	DRM_IOCTL_DEF_DRV(AMDXDNA_SYNC_BO, amdxdna_drm_sync_bo_ioctl, 0),
 	/* Execution */
 	DRM_IOCTL_DEF_DRV(AMDXDNA_EXEC_CMD, amdxdna_drm_submit_cmd_ioctl, 0),
+	DRM_IOCTL_DEF_DRV(AMDXDNA_WAIT_CMD, amdxdna_drm_wait_cmd_ioctl, 0),
 	/* AIE hardware */
 	DRM_IOCTL_DEF_DRV(AMDXDNA_GET_INFO, amdxdna_drm_get_info_ioctl, 0),
 	DRM_IOCTL_DEF_DRV(AMDXDNA_GET_ARRAY, amdxdna_drm_get_array_ioctl, 0),
 	DRM_IOCTL_DEF_DRV(AMDXDNA_SET_STATE, amdxdna_drm_set_state_ioctl, DRM_ROOT_ONLY),
 };
 
+static void amdxdna_show_fdinfo(struct drm_printer *p, struct drm_file *filp)
+{
+	struct amdxdna_client *client = filp->driver_priv;
+	size_t heap_usage, external_usage, internal_usage;
+	char *drv_name = filp->minor->dev->driver->name;
+
+	mutex_lock(&client->mm_lock);
+
+	heap_usage = client->heap_usage;
+	internal_usage = client->total_int_bo_usage;
+	external_usage = client->total_bo_usage - internal_usage;
+
+	mutex_unlock(&client->mm_lock);
+
+	/*
+	 * Note for driver specific BO memory usage stat.
+	 * Total memory in use = amdxdna-internal-alloc + amdxdna-external-alloc, which
+	 * includes both imported and created BOs. To avoid double counts, it includes
+	 * HEAP BO, but not DEV BO. DEV BO is counted by amdxdna-heap-alloc.
+	 */
+	drm_fdinfo_print_size(p, drv_name, "heap", "alloc", heap_usage);
+	drm_fdinfo_print_size(p, drv_name, "internal", "alloc", internal_usage);
+	drm_fdinfo_print_size(p, drv_name, "external", "alloc", external_usage);
+	/*
+	 * Note for DRM standard BO memory stat.
+	 * drm-total-memory counts both DEV BO and HEAP BO. The DEV BO size is double counted.
+	 * drm-shared-memory counts BO shared with other processes/devices.
+	 */
+	drm_show_memory_stats(p, filp);
+}
+
 static const struct file_operations amdxdna_fops = {
 	.owner		= THIS_MODULE,
 	.open		= accel_open,
@@ -230,7 +306,8 @@ static const struct file_operations amdxdna_fops = {
 	.poll		= drm_poll,
 	.read		= drm_read,
 	.llseek		= noop_llseek,
-	.mmap		= drm_gem_mmap,
+	.mmap		= amdxdna_drm_gem_mmap,
+	.show_fdinfo	= drm_show_fdinfo,
 	.fop_flags	= FOP_UNSIGNED_OFFSET,
 };
 
@@ -246,7 +323,7 @@ const struct drm_driver amdxdna_drm_drv = {
 	.postclose = amdxdna_drm_close,
 	.ioctls = amdxdna_drm_ioctls,
 	.num_ioctls = ARRAY_SIZE(amdxdna_drm_ioctls),
-
+	.show_fdinfo = amdxdna_show_fdinfo,
 	.gem_create_object = amdxdna_gem_create_shmem_object_cb,
 	.gem_prime_import = amdxdna_gem_prime_import,
 };
@@ -264,25 +341,38 @@ amdxdna_get_dev_info(struct pci_dev *pdev)
 	return NULL;
 }
 
+static void amdxdna_xdna_drm_release(struct drm_device *drm, void *res)
+{
+	struct amdxdna_dev *xdna = res;
+
+	amdxdna_carveout_fini(xdna);
+}
+
 static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 {
 	struct device *dev = &pdev->dev;
 	struct amdxdna_dev *xdna;
+	struct drm_device *ddev;
 	int ret;
 
 	xdna = devm_drm_dev_alloc(dev, &amdxdna_drm_drv, typeof(*xdna), ddev);
 	if (IS_ERR(xdna))
 		return PTR_ERR(xdna);
+	ddev = &xdna->ddev;
 
 	xdna->dev_info = amdxdna_get_dev_info(pdev);
 	if (!xdna->dev_info)
 		return -ENODEV;
 
-	drmm_mutex_init(&xdna->ddev, &xdna->dev_lock);
+	drmm_mutex_init(ddev, &xdna->dev_lock);
 	init_rwsem(&xdna->notifier_lock);
 	INIT_LIST_HEAD(&xdna->client_list);
 	pci_set_drvdata(pdev, xdna);
 
+	ret = drmm_add_action(ddev, amdxdna_xdna_drm_release, xdna);
+	if (ret)
+		return ret;
+
 	if (IS_ENABLED(CONFIG_LOCKDEP)) {
 		fs_reclaim_acquire(GFP_KERNEL);
 		might_lock(&xdna->notifier_lock);
@@ -313,12 +403,13 @@ static int amdxdna_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		goto failed_dev_fini;
 	}
 
-	ret = drm_dev_register(&xdna->ddev, 0);
+	ret = drm_dev_register(ddev, 0);
 	if (ret) {
 		XDNA_ERR(xdna, "DRM register failed, ret %d", ret);
 		goto failed_sysfs_fini;
 	}
 
+	amdxdna_debugfs_init(xdna);
 	return 0;
 
 failed_sysfs_fini:
@@ -365,12 +456,24 @@ static const struct dev_pm_ops amdxdna_pm_ops = {
 	RUNTIME_PM_OPS(amdxdna_pm_suspend, amdxdna_pm_resume, NULL)
 };
 
+static int amdxdna_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+	struct amdxdna_dev *xdna = pci_get_drvdata(pdev);
+
+	guard(mutex)(&xdna->dev_lock);
+	if (xdna->dev_info->ops->sriov_configure)
+		return xdna->dev_info->ops->sriov_configure(xdna, num_vfs);
+
+	return -ENOENT;
+}
+
 static struct pci_driver amdxdna_pci_driver = {
 	.name = KBUILD_MODNAME,
 	.id_table = pci_ids,
 	.probe = amdxdna_probe,
 	.remove = amdxdna_remove,
 	.driver.pm = &amdxdna_pm_ops,
+	.sriov_configure = amdxdna_sriov_configure,
 };
 
 module_pci_driver(amdxdna_pci_driver);
diff --git a/drivers/accel/amdxdna/amdxdna_pci_drv.h b/drivers/accel/amdxdna/amdxdna_pci_drv.h
index 0661749917d68..34271c14d359f 100644
--- a/drivers/accel/amdxdna/amdxdna_pci_drv.h
+++ b/drivers/accel/amdxdna/amdxdna_pci_drv.h
@@ -7,6 +7,7 @@
 #define _AMDXDNA_PCI_DRV_H_
 
 #include <drm/amdxdna_accel.h>
+#include <drm/drm_mm.h>
 #include <drm/drm_print.h>
 #include <linux/iommu.h>
 #include <linux/iova.h>
@@ -55,15 +56,27 @@ struct amdxdna_dev_ops {
 	void (*fini)(struct amdxdna_dev *xdna);
 	int (*resume)(struct amdxdna_dev *xdna);
 	int (*suspend)(struct amdxdna_dev *xdna);
+	int (*sriov_configure)(struct amdxdna_dev *xdna, int num_vfs);
+	int (*mmap)(struct amdxdna_client *client, struct vm_area_struct *vma);
 	int (*hwctx_init)(struct amdxdna_hwctx *hwctx);
 	void (*hwctx_fini)(struct amdxdna_hwctx *hwctx);
 	int (*hwctx_config)(struct amdxdna_hwctx *hwctx, u32 type, u64 value, void *buf, u32 size);
 	int (*hwctx_sync_debug_bo)(struct amdxdna_hwctx *hwctx, u32 debug_bo_hdl);
+	int (*hwctx_heap_expand)(struct amdxdna_hwctx *hwctx, struct amdxdna_gem_obj *heap);
 	void (*hmm_invalidate)(struct amdxdna_gem_obj *abo, unsigned long cur_seq);
 	int (*cmd_submit)(struct amdxdna_hwctx *hwctx, struct amdxdna_sched_job *job, u64 *seq);
+	int (*cmd_wait)(struct amdxdna_hwctx *hwctx, u64 seq, u32 timeout);
 	int (*get_aie_info)(struct amdxdna_client *client, struct amdxdna_drm_get_info *args);
 	int (*set_aie_state)(struct amdxdna_client *client, struct amdxdna_drm_set_state *args);
 	int (*get_array)(struct amdxdna_client *client, struct amdxdna_drm_get_array *args);
+	int (*get_dev_revision)(struct amdxdna_dev *xdna, u32 *rev);
+};
+
+struct amdxdna_fw_feature_tbl {
+	u64 features;
+	u32 major;
+	u32 max_minor;
+	u32 min_minor;
 };
 
 /*
@@ -76,13 +89,17 @@ struct amdxdna_dev_info {
 	int				sram_bar;
 	int				psp_bar;
 	int				smu_bar;
+	int				doorbell_bar;
 	int				device_type;
 	int				first_col;
 	u32				dev_mem_buf_shift;
 	u64				dev_mem_base;
 	size_t				dev_mem_size;
-	char				*vbnv;
+	const char			*default_vbnv;
+	const struct amdxdna_rev_vbnv	*rev_vbnv_tbl;
+	size_t				dev_heap_max_size;
 	const struct amdxdna_dev_priv	*dev_priv;
+	const struct amdxdna_fw_feature_tbl *fw_feature_tbl;
 	const struct amdxdna_dev_ops	*ops;
 };
 
@@ -93,6 +110,8 @@ struct amdxdna_fw_ver {
 	u32 build;
 };
 
+struct amdxdna_carveout;
+
 struct amdxdna_dev {
 	struct drm_device		ddev;
 	struct amdxdna_dev_hdl		*dev_handle;
@@ -108,6 +127,10 @@ struct amdxdna_dev {
 	struct iommu_group		*group;
 	struct iommu_domain		*domain;
 	struct iova_domain		iovad;
+	/* Accurate board name queried from firmware, or default_vbnv as fallback */
+	const char			*vbnv;
+
+	struct amdxdna_carveout		*carveout;
 };
 
 /*
@@ -133,7 +156,10 @@ struct amdxdna_client {
 	struct drm_file			*filp;
 
 	struct mutex			mm_lock; /* protect memory related */
-	struct amdxdna_gem_obj		*dev_heap;
+	struct xarray			dev_heap_xa;
+	struct drm_mm			dev_heap_mm;
+	u32				dev_heap_nid;
+	size_t				total_heap_size;
 
 	struct iommu_sva		*sva;
 	int				pasid;
@@ -149,6 +175,8 @@ struct amdxdna_client {
 
 /* Add device info below */
 extern const struct amdxdna_dev_info dev_npu1_info;
+extern const struct amdxdna_dev_info dev_npu3_pf_info;
+extern const struct amdxdna_dev_info dev_npu3_vf_info;
 extern const struct amdxdna_dev_info dev_npu4_info;
 extern const struct amdxdna_dev_info dev_npu5_info;
 extern const struct amdxdna_dev_info dev_npu6_info;
@@ -158,11 +186,11 @@ void amdxdna_sysfs_fini(struct amdxdna_dev *xdna);
 
 int amdxdna_iommu_init(struct amdxdna_dev *xdna);
 void amdxdna_iommu_fini(struct amdxdna_dev *xdna);
-int amdxdna_iommu_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo);
-void amdxdna_iommu_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo);
 void *amdxdna_iommu_alloc(struct amdxdna_dev *xdna, size_t size, dma_addr_t *dma_addr);
 void amdxdna_iommu_free(struct amdxdna_dev *xdna, size_t size,
 			void *cpu_addr, dma_addr_t dma_addr);
+int amdxdna_dma_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo);
+void amdxdna_dma_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo);
 
 static inline bool amdxdna_iova_on(struct amdxdna_dev *xdna)
 {
diff --git a/drivers/accel/amdxdna/amdxdna_sysfs.c b/drivers/accel/amdxdna/amdxdna_sysfs.c
index f27e4ee960a0c..d9e359ee8182c 100644
--- a/drivers/accel/amdxdna/amdxdna_sysfs.c
+++ b/drivers/accel/amdxdna/amdxdna_sysfs.c
@@ -17,7 +17,10 @@ static ssize_t vbnv_show(struct device *dev, struct device_attribute *attr, char
 {
 	struct amdxdna_dev *xdna = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%s\n", xdna->dev_info->vbnv);
+	if (!xdna->vbnv)
+		return sprintf(buf, "\n");
+
+	return sprintf(buf, "%s\n", xdna->vbnv);
 }
 static DEVICE_ATTR_RO(vbnv);
 
diff --git a/drivers/accel/amdxdna/amdxdna_ubuf.c b/drivers/accel/amdxdna/amdxdna_ubuf.c
index fb999aa253185..bb60fb80467ef 100644
--- a/drivers/accel/amdxdna/amdxdna_ubuf.c
+++ b/drivers/accel/amdxdna/amdxdna_ubuf.c
@@ -69,62 +69,32 @@ static void amdxdna_ubuf_release(struct dma_buf *dbuf)
 	kfree(ubuf);
 }
 
-static vm_fault_t amdxdna_ubuf_vm_fault(struct vm_fault *vmf)
-{
-	struct vm_area_struct *vma = vmf->vma;
-	struct amdxdna_ubuf_priv *ubuf;
-	unsigned long pfn;
-	pgoff_t pgoff;
-
-	ubuf = vma->vm_private_data;
-	pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT;
-
-	pfn = page_to_pfn(ubuf->pages[pgoff]);
-	return vmf_insert_pfn(vma, vmf->address, pfn);
-}
-
-static const struct vm_operations_struct amdxdna_ubuf_vm_ops = {
-	.fault = amdxdna_ubuf_vm_fault,
+static const struct dma_buf_ops amdxdna_ubuf_dmabuf_ops = {
+	.map_dma_buf = amdxdna_ubuf_map,
+	.unmap_dma_buf = amdxdna_ubuf_unmap,
+	.release = amdxdna_ubuf_release,
 };
 
-static int amdxdna_ubuf_mmap(struct dma_buf *dbuf, struct vm_area_struct *vma)
-{
-	struct amdxdna_ubuf_priv *ubuf = dbuf->priv;
-
-	vma->vm_ops = &amdxdna_ubuf_vm_ops;
-	vma->vm_private_data = ubuf;
-	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
-
-	return 0;
-}
-
-static int amdxdna_ubuf_vmap(struct dma_buf *dbuf, struct iosys_map *map)
+static int readonly_va_entry(struct amdxdna_drm_va_entry *va_ent)
 {
-	struct amdxdna_ubuf_priv *ubuf = dbuf->priv;
-	void *kva;
+	struct mm_struct *mm = current->mm;
+	struct vm_area_struct *vma;
+	int ret;
 
-	kva = vmap(ubuf->pages, ubuf->nr_pages, VM_MAP, PAGE_KERNEL);
-	if (!kva)
-		return -EINVAL;
+	mmap_read_lock(mm);
 
-	iosys_map_set_vaddr(map, kva);
-	return 0;
-}
+	vma = find_vma(mm, va_ent->vaddr);
+	if (!vma ||
+	    vma->vm_start > va_ent->vaddr ||
+	    vma->vm_end - va_ent->vaddr < va_ent->len)
+		ret = -ENOENT;
+	else
+		ret = vma->vm_flags & VM_WRITE ? 0 : 1;
 
-static void amdxdna_ubuf_vunmap(struct dma_buf *dbuf, struct iosys_map *map)
-{
-	vunmap(map->vaddr);
+	mmap_read_unlock(mm);
+	return ret;
 }
 
-static const struct dma_buf_ops amdxdna_ubuf_dmabuf_ops = {
-	.map_dma_buf = amdxdna_ubuf_map,
-	.unmap_dma_buf = amdxdna_ubuf_unmap,
-	.release = amdxdna_ubuf_release,
-	.mmap = amdxdna_ubuf_mmap,
-	.vmap = amdxdna_ubuf_vmap,
-	.vunmap = amdxdna_ubuf_vunmap,
-};
-
 struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev,
 				 u32 num_entries, void __user *va_entries)
 {
@@ -134,6 +104,7 @@ struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev,
 	struct amdxdna_ubuf_priv *ubuf;
 	u32 npages, start = 0;
 	struct dma_buf *dbuf;
+	bool readonly = true;
 	int i, ret;
 	DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
 
@@ -172,6 +143,10 @@ struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev,
 			ret = -EINVAL;
 			goto free_ent;
 		}
+
+		/* Pin pages as writable as long as not all entries are read-only. */
+		if (readonly && readonly_va_entry(&va_ent[i]) != 1)
+			readonly = false;
 	}
 
 	ubuf->nr_pages = exp_info.size >> PAGE_SHIFT;
@@ -194,20 +169,24 @@ struct dma_buf *amdxdna_get_ubuf(struct drm_device *dev,
 		npages = va_ent[i].len >> PAGE_SHIFT;
 
 		ret = pin_user_pages_fast(va_ent[i].vaddr, npages,
-					  FOLL_WRITE | FOLL_LONGTERM,
+					  (readonly ? 0 : FOLL_WRITE) | FOLL_LONGTERM,
 					  &ubuf->pages[start]);
-		if (ret < 0 || ret != npages) {
-			ret = -ENOMEM;
+		if (ret >= 0) {
+			start += ret;
+			if (ret != npages) {
+				XDNA_ERR(xdna, "Partially pinned pages %d/%u", ret, npages);
+				ret = -ENOMEM;
+				goto destroy_pages;
+			}
+		} else {
 			XDNA_ERR(xdna, "Failed to pin pages ret %d", ret);
 			goto destroy_pages;
 		}
-
-		start += ret;
 	}
 
 	exp_info.ops = &amdxdna_ubuf_dmabuf_ops;
 	exp_info.priv = ubuf;
-	exp_info.flags = O_RDWR | O_CLOEXEC;
+	exp_info.flags = (readonly ? O_RDONLY : O_RDWR) | O_CLOEXEC;
 
 	dbuf = dma_buf_export(&exp_info);
 	if (IS_ERR(dbuf)) {
diff --git a/drivers/accel/amdxdna/npu1_regs.c b/drivers/accel/amdxdna/npu1_regs.c
index 1320e924e5482..ca779674017ad 100644
--- a/drivers/accel/amdxdna/npu1_regs.c
+++ b/drivers/accel/amdxdna/npu1_regs.c
@@ -65,18 +65,39 @@ const struct dpm_clk_freq npu1_dpm_clk_table[] = {
 	{ 0 }
 };
 
-static const struct aie2_fw_feature_tbl npu1_fw_feature_table[] = {
+static const struct amdxdna_fw_feature_tbl npu1_fw_feature_table[] = {
 	{ .major = 5, .min_minor = 7 },
 	{ .features = BIT_U64(AIE2_NPU_COMMAND), .major = 5, .min_minor = 8 },
 	{ 0 }
 };
 
+static int npu1_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
+{
+	u32 npuclk, hclk;
+	int ret;
+
+	npuclk = ndev->priv->dpm_clk_tbl[dpm_level].npuclk;
+	hclk = ndev->priv->dpm_clk_tbl[dpm_level].hclk;
+	ret = aie_smu_set_clocks(ndev->aie.smu_hdl, &npuclk, &hclk);
+	if (ret)
+		return ret;
+
+	ndev->npuclk_freq = npuclk;
+	ndev->hclk_freq = hclk;
+	ndev->max_tops = 2 * ndev->total_col;
+	ndev->curr_tops = ndev->max_tops * hclk / 1028;
+
+	XDNA_DBG(ndev->aie.xdna, "MP-NPU clock %d, H clock %d\n",
+		 ndev->npuclk_freq, ndev->hclk_freq);
+	return 0;
+}
+
 static const struct amdxdna_dev_priv npu1_dev_priv = {
 	.fw_path        = "amdnpu/1502_00/",
 	.rt_config	= npu1_default_rt_cfg,
 	.dpm_clk_tbl	= npu1_dpm_clk_table,
-	.fw_feature_tbl = npu1_fw_feature_table,
 	.col_align	= COL_ALIGN_NONE,
+	.col_opc	= 2048,
 	.mbox_dev_addr  = NPU1_MBOX_BAR_BASE,
 	.mbox_size      = 0, /* Use BAR size */
 	.sram_dev_addr  = NPU1_SRAM_BAR_BASE,
@@ -102,7 +123,7 @@ static const struct amdxdna_dev_priv npu1_dev_priv = {
 		DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU1_SMU, MPNPU_PUB_SCRATCH6),
 		DEFINE_BAR_OFFSET(SMU_OUT_REG,  NPU1_SMU, MPNPU_PUB_SCRATCH7),
 	},
-	.hw_ops		= {
+	.hw_ops		= &(const struct aie2_hw_ops) {
 		.set_dpm = npu1_set_dpm,
 	},
 };
@@ -117,8 +138,10 @@ const struct amdxdna_dev_info dev_npu1_info = {
 	.dev_mem_buf_shift = 15, /* 32 KiB aligned */
 	.dev_mem_base      = AIE2_DEVM_BASE,
 	.dev_mem_size      = AIE2_DEVM_SIZE,
-	.vbnv              = "RyzenAI-npu1",
+	.default_vbnv      = "RyzenAI-npu1",
+	.dev_heap_max_size = AIE2_DEVM_SIZE,
 	.device_type       = AMDXDNA_DEV_TYPE_KMQ,
 	.dev_priv          = &npu1_dev_priv,
+	.fw_feature_tbl    = npu1_fw_feature_table,
 	.ops               = &aie2_ops,
 };
diff --git a/drivers/accel/amdxdna/npu3_regs.c b/drivers/accel/amdxdna/npu3_regs.c
new file mode 100644
index 0000000000000..d76b2e99c3089
--- /dev/null
+++ b/drivers/accel/amdxdna/npu3_regs.c
@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026, Advanced Micro Devices, Inc.
+ */
+
+#include <drm/amdxdna_accel.h>
+#include <drm/drm_device.h>
+
+#include "aie4_pci.h"
+#include "amdxdna_pci_drv.h"
+
+#define NPU3_MBOX_BAR		0
+
+#define NPU3_MBOX_BUFFER_BAR	2
+#define NPU3_MBOX_INFO_OFF	0x0
+
+#define NPU3_DOORBELL_BAR       2
+#define NPU3_DOORBELL_OFF       0x0
+
+/* PCIe BAR Index for NPU3 */
+#define NPU3_REG_BAR_INDEX	0
+#define NPU3_PSP_BAR_INDEX      4
+#define NPU3_SMU_BAR_INDEX      5
+
+#define MMNPU_APERTURE3_BASE    0x3810000
+#define MMNPU_APERTURE4_BASE    0x3B10000
+
+#define NPU3_PSP_BAR_BASE       MMNPU_APERTURE3_BASE
+#define NPU3_SMU_BAR_BASE       MMNPU_APERTURE4_BASE
+
+#define MPASP_C2PMSG_123_ALT_1  0x3810AEC
+#define MPASP_C2PMSG_156_ALT_1  0x3810B70
+#define MPASP_C2PMSG_157_ALT_1  0x3810B74
+#define MPASP_C2PMSG_73_ALT_1   0x3810A24
+
+#define MP1_C2PMSG_59_ALT_1     0x3B109EC
+#define MP1_C2PMSG_61_ALT_1     0x3B109F4
+#define MP1_C2PMSG_60_ALT_1     0x3B109F0
+
+static const struct amdxdna_fw_feature_tbl npu3_fw_feature_table[] = {
+	{ .major = 5, .min_minor = 10 },
+	{ 0 }
+};
+
+static const struct amdxdna_dev_priv npu3_dev_priv = {
+	.npufw_path             = "npu.dev.sbin",
+	.certfw_path            = "cert.dev.sbin",
+	.mbox_bar		= NPU3_MBOX_BAR,
+	.mbox_rbuf_bar		= NPU3_MBOX_BUFFER_BAR,
+	.mbox_info_off		= NPU3_MBOX_INFO_OFF,
+	.doorbell_off		= NPU3_DOORBELL_OFF,
+	.psp_regs_off   = {
+		DEFINE_BAR_OFFSET(PSP_CMD_REG,    NPU3_PSP, MPASP_C2PMSG_123_ALT_1),
+		DEFINE_BAR_OFFSET(PSP_ARG0_REG,   NPU3_PSP, MPASP_C2PMSG_156_ALT_1),
+		DEFINE_BAR_OFFSET(PSP_ARG1_REG,   NPU3_PSP, MPASP_C2PMSG_157_ALT_1),
+		DEFINE_BAR_OFFSET(PSP_ARG2_REG,   NPU3_PSP, MPASP_C2PMSG_123_ALT_1),
+		DEFINE_BAR_OFFSET(PSP_INTR_REG,   NPU3_PSP, MPASP_C2PMSG_73_ALT_1),
+		DEFINE_BAR_OFFSET(PSP_STATUS_REG, NPU3_PSP, MPASP_C2PMSG_123_ALT_1),
+		DEFINE_BAR_OFFSET(PSP_RESP_REG,   NPU3_PSP, MPASP_C2PMSG_156_ALT_1),
+		/* npu3 doesn't use 8th pwaitmode register */
+	},
+	.smu_regs_off   = {
+		DEFINE_BAR_OFFSET(SMU_CMD_REG,  NPU3_SMU, MP1_C2PMSG_59_ALT_1),
+		DEFINE_BAR_OFFSET(SMU_ARG_REG,  NPU3_SMU, MP1_C2PMSG_61_ALT_1),
+		DEFINE_BAR_OFFSET(SMU_INTR_REG, NPU3_SMU, MMNPU_APERTURE4_BASE),
+		DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU3_SMU, MP1_C2PMSG_60_ALT_1),
+		DEFINE_BAR_OFFSET(SMU_OUT_REG,  NPU3_SMU, MP1_C2PMSG_61_ALT_1),
+	},
+};
+
+static const struct amdxdna_dev_priv npu3_dev_vf_priv = {
+	/* vf device does not load firmware */
+	.mbox_bar		= NPU3_MBOX_BAR,
+	.mbox_rbuf_bar		= NPU3_MBOX_BUFFER_BAR,
+	.mbox_info_off		= NPU3_MBOX_INFO_OFF,
+	/* vf device does not have smu and psp */
+};
+
+const struct amdxdna_dev_info dev_npu3_pf_info = {
+	.mbox_bar		= NPU3_MBOX_BAR,
+	.sram_bar		= NPU3_MBOX_BUFFER_BAR,
+	.psp_bar                = NPU3_PSP_BAR_INDEX,
+	.smu_bar		= NPU3_SMU_BAR_INDEX,
+	.default_vbnv		= "RyzenAI-npu3-pf",
+	.device_type		= AMDXDNA_DEV_TYPE_PF,
+	.dev_priv		= &npu3_dev_priv,
+	.fw_feature_tbl		= npu3_fw_feature_table,
+	.ops			= &aie4_pf_ops,
+};
+
+const struct amdxdna_dev_info dev_npu3_vf_info = {
+	.mbox_bar		= NPU3_MBOX_BAR,
+	.sram_bar		= NPU3_MBOX_BUFFER_BAR,
+	.doorbell_bar		= NPU3_DOORBELL_BAR,
+	.default_vbnv		= "RyzenAI-npu3-vf",
+	.device_type		= AMDXDNA_DEV_TYPE_UMQ,
+	.dev_priv		= &npu3_dev_vf_priv,
+	.fw_feature_tbl		= npu3_fw_feature_table,
+	.ops			= &aie4_vf_ops,
+};
diff --git a/drivers/accel/amdxdna/npu4_regs.c b/drivers/accel/amdxdna/npu4_regs.c
index 619bff042e52c..15a1613846252 100644
--- a/drivers/accel/amdxdna/npu4_regs.c
+++ b/drivers/accel/amdxdna/npu4_regs.c
@@ -6,6 +6,7 @@
 #include <drm/amdxdna_accel.h>
 #include <drm/drm_device.h>
 #include <drm/gpu_scheduler.h>
+#include <linux/amd-pmf-io.h>
 #include <linux/bits.h>
 #include <linux/sizes.h>
 
@@ -63,6 +64,8 @@
 #define NPU4_SMU_BAR_BASE	MMNPU_APERTURE4_BASE
 #define NPU4_SRAM_BAR_BASE	MMNPU_APERTURE1_BASE
 
+#define NPU4_DPM_TOPS(ndev, hclk) (4096 * (ndev)->total_col * (hclk) / 1000000)
+
 const struct rt_config npu4_default_rt_cfg[] = {
 	{ 5, 1, AIE2_RT_CFG_INIT }, /* PDI APP LOAD MODE */
 	{ 10, 1, AIE2_RT_CFG_INIT }, /* DEBUG BUF */
@@ -88,22 +91,77 @@ const struct dpm_clk_freq npu4_dpm_clk_table[] = {
 	{ 0 }
 };
 
-const struct aie2_fw_feature_tbl npu4_fw_feature_table[] = {
+const struct amdxdna_fw_feature_tbl npu4_fw_feature_table[] = {
 	{ .major = 6, .min_minor = 12 },
-	{ .features = BIT_U64(AIE2_NPU_COMMAND), .major = 6, .min_minor = 15 },
 	{ .features = BIT_U64(AIE2_PREEMPT), .major = 6, .min_minor = 12 },
 	{ .features = BIT_U64(AIE2_TEMPORAL_ONLY), .major = 6, .min_minor = 12 },
+	{ .features = BIT_U64(AIE2_NPU_COMMAND), .major = 6, .min_minor = 15 },
+	{ .features = BIT_U64(AIE2_UPDATE_PROPERTY), .major = 6, .min_minor = 15 },
 	{ .features = BIT_U64(AIE2_APP_HEALTH), .major = 6, .min_minor = 18 },
+	{ .features = BIT_U64(AIE2_ADD_HOST_BUFFER), .major = 6, .min_minor = 18 },
+	{ .features = BIT_U64(AIE2_GET_DEV_REVISION), .major = 6, .min_minor = 24 },
 	{ .features = AIE2_ALL_FEATURES, .major = 7 },
 	{ 0 }
 };
 
+static int npu4_set_dpm(struct amdxdna_dev_hdl *ndev, u32 dpm_level)
+{
+	int ret;
+
+	ret = aie_smu_set_dpm(ndev->aie.smu_hdl, dpm_level);
+	if (ret)
+		return ret;
+
+	ndev->npuclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].npuclk;
+	ndev->hclk_freq = ndev->priv->dpm_clk_tbl[dpm_level].hclk;
+	ndev->max_tops = NPU4_DPM_TOPS(ndev, ndev->priv->dpm_clk_tbl[ndev->max_dpm_level].hclk);
+	ndev->curr_tops = NPU4_DPM_TOPS(ndev, ndev->hclk_freq);
+
+	XDNA_DBG(ndev->aie.xdna, "MP-NPU clock %d, H clock %d\n",
+		 ndev->npuclk_freq, ndev->hclk_freq);
+
+	return 0;
+}
+
+static int npu4_update_counters(struct amdxdna_dev_hdl *ndev)
+{
+	struct amd_pmf_npu_metrics npu_metrics;
+	int ret;
+
+	ret = AIE2_GET_PMF_NPU_METRICS(&npu_metrics);
+	if (ret)
+		return ret;
+
+	ndev->npuclk_freq = npu_metrics.mpnpuclk_freq;
+	ndev->hclk_freq = npu_metrics.npuclk_freq;
+	ndev->curr_tops = NPU4_DPM_TOPS(ndev, ndev->hclk_freq);
+
+	return 0;
+}
+
+const struct aie2_hw_ops npu4_hw_ops = {
+	.set_dpm = npu4_set_dpm,
+	.update_counters = npu4_update_counters,
+};
+
+const struct amdxdna_rev_vbnv npu4_rev_vbnv_tbl[] = {
+	{ AIE2_DEV_REVISION_STXA, "NPU Strix" },
+	{ AIE2_DEV_REVISION_STXB, "NPU Strix" },
+	{ AIE2_DEV_REVISION_KRK1, "NPU Krackan 1" },
+	{ AIE2_DEV_REVISION_KRK2, "NPU Krackan 2" },
+	{ AIE2_DEV_REVISION_HALO, "NPU Strix Halo" },
+	{ AIE2_DEV_REVISION_GPT1, "NPU Gorgon Point 1" },
+	{ AIE2_DEV_REVISION_GPT2, "NPU Gorgon Point 2" },
+	{ AIE2_DEV_REVISION_GPT3, "NPU Gorgon Point 3" },
+	{ 0 }
+};
+
 static const struct amdxdna_dev_priv npu4_dev_priv = {
 	.fw_path        = "amdnpu/17f0_10/",
 	.rt_config	= npu4_default_rt_cfg,
 	.dpm_clk_tbl	= npu4_dpm_clk_table,
-	.fw_feature_tbl = npu4_fw_feature_table,
 	.col_align	= COL_ALIGN_NATURE,
+	.col_opc	= 4096,
 	.mbox_dev_addr  = NPU4_MBOX_BAR_BASE,
 	.mbox_size      = 0, /* Use BAR size */
 	.sram_dev_addr  = NPU4_SRAM_BAR_BASE,
@@ -129,9 +187,7 @@ static const struct amdxdna_dev_priv npu4_dev_priv = {
 		DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU4_SMU, MP1_C2PMSG_61),
 		DEFINE_BAR_OFFSET(SMU_OUT_REG,  NPU4_SMU, MP1_C2PMSG_60),
 	},
-	.hw_ops		= {
-		.set_dpm = npu4_set_dpm,
-	},
+	.hw_ops		= &npu4_hw_ops
 };
 
 const struct amdxdna_dev_info dev_npu4_info = {
@@ -144,8 +200,11 @@ const struct amdxdna_dev_info dev_npu4_info = {
 	.dev_mem_buf_shift = 15, /* 32 KiB aligned */
 	.dev_mem_base      = AIE2_DEVM_BASE,
 	.dev_mem_size      = AIE2_DEVM_SIZE,
-	.vbnv              = "RyzenAI-npu4",
+	.default_vbnv      = "RyzenAI-npu4",
+	.dev_heap_max_size = AIE2_DEVM_MAX_SIZE,
 	.device_type       = AMDXDNA_DEV_TYPE_KMQ,
+	.rev_vbnv_tbl      = npu4_rev_vbnv_tbl,
 	.dev_priv          = &npu4_dev_priv,
+	.fw_feature_tbl    = npu4_fw_feature_table,
 	.ops               = &aie2_ops, /* NPU4 can share NPU1's callback */
 };
diff --git a/drivers/accel/amdxdna/npu5_regs.c b/drivers/accel/amdxdna/npu5_regs.c
index c0ac5daf32ee5..306b359d0cd39 100644
--- a/drivers/accel/amdxdna/npu5_regs.c
+++ b/drivers/accel/amdxdna/npu5_regs.c
@@ -66,8 +66,8 @@ static const struct amdxdna_dev_priv npu5_dev_priv = {
 	.fw_path        = "amdnpu/17f0_11/",
 	.rt_config	= npu4_default_rt_cfg,
 	.dpm_clk_tbl	= npu4_dpm_clk_table,
-	.fw_feature_tbl = npu4_fw_feature_table,
 	.col_align	= COL_ALIGN_NATURE,
+	.col_opc	= 4096,
 	.mbox_dev_addr  = NPU5_MBOX_BAR_BASE,
 	.mbox_size      = 0, /* Use BAR size */
 	.sram_dev_addr  = NPU5_SRAM_BAR_BASE,
@@ -93,9 +93,7 @@ static const struct amdxdna_dev_priv npu5_dev_priv = {
 		DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU5_SMU, MP1_C2PMSG_61),
 		DEFINE_BAR_OFFSET(SMU_OUT_REG,  NPU5_SMU, MP1_C2PMSG_60),
 	},
-	.hw_ops		= {
-		.set_dpm = npu4_set_dpm,
-	},
+	.hw_ops		= &npu4_hw_ops
 };
 
 const struct amdxdna_dev_info dev_npu5_info = {
@@ -108,8 +106,11 @@ const struct amdxdna_dev_info dev_npu5_info = {
 	.dev_mem_buf_shift = 15, /* 32 KiB aligned */
 	.dev_mem_base      = AIE2_DEVM_BASE,
 	.dev_mem_size      = AIE2_DEVM_SIZE,
-	.vbnv              = "RyzenAI-npu5",
+	.default_vbnv      = "RyzenAI-npu5",
+	.dev_heap_max_size = AIE2_DEVM_MAX_SIZE,
 	.device_type       = AMDXDNA_DEV_TYPE_KMQ,
+	.rev_vbnv_tbl      = npu4_rev_vbnv_tbl,
 	.dev_priv          = &npu5_dev_priv,
+	.fw_feature_tbl    = npu4_fw_feature_table,
 	.ops               = &aie2_ops,
 };
diff --git a/drivers/accel/amdxdna/npu6_regs.c b/drivers/accel/amdxdna/npu6_regs.c
index ce591ed0d4832..e68637d2a228e 100644
--- a/drivers/accel/amdxdna/npu6_regs.c
+++ b/drivers/accel/amdxdna/npu6_regs.c
@@ -66,8 +66,8 @@ static const struct amdxdna_dev_priv npu6_dev_priv = {
 	.fw_path        = "amdnpu/17f0_10/",
 	.rt_config	= npu4_default_rt_cfg,
 	.dpm_clk_tbl	= npu4_dpm_clk_table,
-	.fw_feature_tbl = npu4_fw_feature_table,
 	.col_align	= COL_ALIGN_NATURE,
+	.col_opc	= 4096,
 	.mbox_dev_addr  = NPU6_MBOX_BAR_BASE,
 	.mbox_size      = 0, /* Use BAR size */
 	.sram_dev_addr  = NPU6_SRAM_BAR_BASE,
@@ -93,9 +93,7 @@ static const struct amdxdna_dev_priv npu6_dev_priv = {
 		DEFINE_BAR_OFFSET(SMU_RESP_REG, NPU6_SMU, MP1_C2PMSG_61),
 		DEFINE_BAR_OFFSET(SMU_OUT_REG,  NPU6_SMU, MP1_C2PMSG_60),
 	},
-	.hw_ops         = {
-		.set_dpm = npu4_set_dpm,
-	},
+	.hw_ops         = &npu4_hw_ops
 
 };
 
@@ -109,8 +107,11 @@ const struct amdxdna_dev_info dev_npu6_info = {
 	.dev_mem_buf_shift = 15, /* 32 KiB aligned */
 	.dev_mem_base      = AIE2_DEVM_BASE,
 	.dev_mem_size      = AIE2_DEVM_SIZE,
-	.vbnv              = "RyzenAI-npu6",
+	.default_vbnv      = "RyzenAI-npu6",
+	.dev_heap_max_size = AIE2_DEVM_MAX_SIZE,
 	.device_type       = AMDXDNA_DEV_TYPE_KMQ,
+	.rev_vbnv_tbl      = npu4_rev_vbnv_tbl,
 	.dev_priv          = &npu6_dev_priv,
+	.fw_feature_tbl    = npu4_fw_feature_table,
 	.ops               = &aie2_ops,
 };
diff --git a/drivers/accel/ethosu/ethosu_job.c b/drivers/accel/ethosu/ethosu_job.c
index ec85f41567440..b76924645aaaf 100644
--- a/drivers/accel/ethosu/ethosu_job.c
+++ b/drivers/accel/ethosu/ethosu_job.c
@@ -296,7 +296,6 @@ int ethosu_job_init(struct ethosu_device *edev)
 	struct device *dev = edev->base.dev;
 	struct drm_sched_init_args args = {
 		.ops = &ethosu_sched_ops,
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit = 1,
 		.timeout = msecs_to_jiffies(JOB_TIMEOUT_MS),
 		.name = dev_name(dev),
@@ -417,9 +416,21 @@ static int ethosu_ioctl_submit_job(struct drm_device *dev, struct drm_file *file
 		struct drm_gem_object *gem;
 
 		/* Can only omit a BO handle if the region is not used or used for SRAM */
-		if (!job->region_bo_handles[i] &&
-		    (!cmd_info->region_size[i] || (i == ETHOSU_SRAM_REGION && job->sram_size)))
-			continue;
+		if (!job->region_bo_handles[i]) {
+			if (!cmd_info->region_size[i])
+				continue;
+			if (i == ETHOSU_SRAM_REGION) {
+				if (cmd_info->region_size[i] <= edev->npu_info.sram_size)
+					continue;
+
+				dev_err(dev->dev,
+					"cmd stream region %d size greater than SRAM size (%llu > %u)\n",
+					i, cmd_info->region_size[i],
+					edev->npu_info.sram_size);
+				ret = -EINVAL;
+				goto out_cleanup_job;
+			}
+		}
 
 		if (job->region_bo_handles[i] && !cmd_info->region_size[i]) {
 			dev_err(dev->dev,
diff --git a/drivers/accel/ivpu/ivpu_drv.c b/drivers/accel/ivpu/ivpu_drv.c
index 3b7b008bccfec..35e506074d5f1 100644
--- a/drivers/accel/ivpu/ivpu_drv.c
+++ b/drivers/accel/ivpu/ivpu_drv.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2025 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #include <linux/firmware.h>
@@ -234,7 +234,7 @@ static int ivpu_get_param_ioctl(struct drm_device *dev, void *data, struct drm_f
 		args->value = vdev->platform;
 		break;
 	case DRM_IVPU_PARAM_CORE_CLOCK_RATE:
-		args->value = ivpu_hw_dpu_max_freq_get(vdev);
+		args->value = ivpu_hw_btrs_pll_ratio_to_hz(vdev, vdev->hw->pll.max_ratio);
 		break;
 	case DRM_IVPU_PARAM_NUM_CONTEXTS:
 		args->value = file_priv->user_limits->max_ctx_count;
@@ -487,6 +487,10 @@ int ivpu_boot(struct ivpu_device *vdev)
 		ret = ivpu_hw_sched_init(vdev);
 		if (ret)
 			goto err_disable_ipc;
+
+		ret = ivpu_hw_btrs_cfg_freq_init(vdev);
+		if (ret)
+			goto err_disable_ipc;
 	}
 
 	return 0;
diff --git a/drivers/accel/ivpu/ivpu_hw.h b/drivers/accel/ivpu/ivpu_hw.h
index b6d0f0d0dcccd..487a918e2fa9a 100644
--- a/drivers/accel/ivpu/ivpu_hw.h
+++ b/drivers/accel/ivpu/ivpu_hw.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (C) 2020-2025 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #ifndef __IVPU_HW_H__
@@ -28,6 +28,7 @@ struct ivpu_hw_info {
 		struct ivpu_addr_range dma;
 	} ranges;
 	struct {
+		/* Hardware min and max pll ratio */
 		u8 min_ratio;
 		u8 max_ratio;
 		/*
@@ -35,6 +36,9 @@ struct ivpu_hw_info {
 		 * performance to power ratio at this frequency.
 		 */
 		u8 pn_ratio;
+		/* Pll ratios configured via sysfs interface */
+		u8 cfg_min_ratio;
+		u8 cfg_max_ratio;
 		u32 profiling_freq;
 	} pll;
 	struct {
@@ -80,16 +84,6 @@ static inline u64 ivpu_hw_range_size(const struct ivpu_addr_range *range)
 	return range->end - range->start;
 }
 
-static inline u32 ivpu_hw_dpu_max_freq_get(struct ivpu_device *vdev)
-{
-	return ivpu_hw_btrs_dpu_max_freq_get(vdev);
-}
-
-static inline u32 ivpu_hw_dpu_freq_get(struct ivpu_device *vdev)
-{
-	return ivpu_hw_btrs_dpu_freq_get(vdev);
-}
-
 static inline void ivpu_hw_irq_clear(struct ivpu_device *vdev)
 {
 	ivpu_hw_ip_irq_clear(vdev);
diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.c b/drivers/accel/ivpu/ivpu_hw_btrs.c
index 06e65c5926185..dac935164e11b 100644
--- a/drivers/accel/ivpu/ivpu_hw_btrs.c
+++ b/drivers/accel/ivpu/ivpu_hw_btrs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2025 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #include <linux/units.h>
@@ -11,6 +11,7 @@
 #include "ivpu_hw_btrs_lnl_reg.h"
 #include "ivpu_hw_btrs_mtl_reg.h"
 #include "ivpu_hw_reg_io.h"
+#include "ivpu_jsm_msg.h"
 #include "ivpu_pm.h"
 
 #define BTRS_MTL_IRQ_MASK ((REG_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR)) | \
@@ -33,8 +34,7 @@
 
 #define PLL_CDYN_DEFAULT               0x80
 #define PLL_EPP_DEFAULT                0x80
-#define PLL_REF_CLK_FREQ               50000000ull
-#define PLL_RATIO_TO_FREQ(x)           ((x) * PLL_REF_CLK_FREQ)
+#define PLL_REF_CLK_FREQ_MHZ           50
 
 #define PLL_TIMEOUT_US		       (1500 * USEC_PER_MSEC)
 #define IDLE_TIMEOUT_US		       (5 * USEC_PER_MSEC)
@@ -59,8 +59,6 @@
 #define DCT_ENABLE                     0x1
 #define DCT_DISABLE                    0x0
 
-static u32 pll_ratio_to_dpu_freq(struct ivpu_device *vdev, u32 ratio);
-
 int ivpu_hw_btrs_irqs_clear_with_0_mtl(struct ivpu_device *vdev)
 {
 	REGB_WR32(VPU_HW_BTRS_MTL_INTERRUPT_STAT, BTRS_MTL_ALL_IRQ_MASK);
@@ -111,6 +109,8 @@ void ivpu_hw_btrs_freq_ratios_init(struct ivpu_device *vdev)
 	hw->pll.min_ratio = clamp_t(u8, ivpu_pll_min_ratio, hw->pll.min_ratio, hw->pll.max_ratio);
 	hw->pll.max_ratio = clamp_t(u8, ivpu_pll_max_ratio, hw->pll.min_ratio, hw->pll.max_ratio);
 	hw->pll.pn_ratio = clamp_t(u8, hw->pll.pn_ratio, hw->pll.min_ratio, hw->pll.max_ratio);
+	hw->pll.cfg_max_ratio = hw->pll.max_ratio;
+	hw->pll.cfg_min_ratio = hw->pll.min_ratio;
 }
 
 static bool tile_disable_check(u32 config)
@@ -341,8 +341,8 @@ int ivpu_hw_btrs_wp_drive(struct ivpu_device *vdev, bool enable)
 
 	prepare_wp_request(vdev, &wp, enable);
 
-	ivpu_dbg(vdev, PM, "PLL workpoint request: %lu MHz, config: 0x%x, epp: 0x%x, cdyn: 0x%x\n",
-		 pll_ratio_to_dpu_freq(vdev, wp.target) / HZ_PER_MHZ, wp.cfg, wp.epp, wp.cdyn);
+	ivpu_dbg(vdev, PM, "PLL workpoint request: %u MHz, config: 0x%x, epp: 0x%x, cdyn: 0x%x\n",
+		 ivpu_hw_btrs_pll_ratio_to_mhz(vdev, wp.target), wp.cfg, wp.epp, wp.cdyn);
 
 	ret = wp_request_send(vdev, &wp);
 	if (ret) {
@@ -598,35 +598,101 @@ static u32 pll_config_get_lnl(struct ivpu_device *vdev)
 	return REGB_RD32(VPU_HW_BTRS_LNL_PLL_FREQ);
 }
 
-static u32 pll_ratio_to_dpu_freq_mtl(u16 ratio)
+static u32 pll_ratio_to_mhz_mtl(u8 pll_ratio)
 {
-	return (PLL_RATIO_TO_FREQ(ratio) * 2) / 3;
+	return (pll_ratio * PLL_REF_CLK_FREQ_MHZ * 2) / 3;
 }
 
-static u32 pll_ratio_to_dpu_freq_lnl(u16 ratio)
+static u32 pll_ratio_to_mhz_lnl(u8 pll_ratio)
 {
-	return PLL_RATIO_TO_FREQ(ratio) / 2;
+	return (pll_ratio * PLL_REF_CLK_FREQ_MHZ) / 2;
 }
 
-static u32 pll_ratio_to_dpu_freq(struct ivpu_device *vdev, u32 ratio)
+u32 ivpu_hw_btrs_pll_ratio_to_mhz(struct ivpu_device *vdev, u8 pll_ratio)
 {
 	if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
-		return pll_ratio_to_dpu_freq_mtl(ratio);
+		return pll_ratio_to_mhz_mtl(pll_ratio);
 	else
-		return pll_ratio_to_dpu_freq_lnl(ratio);
+		return pll_ratio_to_mhz_lnl(pll_ratio);
 }
 
-u32 ivpu_hw_btrs_dpu_max_freq_get(struct ivpu_device *vdev)
+u32 ivpu_hw_btrs_pll_ratio_to_hz(struct ivpu_device *vdev, u8 pll_ratio)
 {
-	return pll_ratio_to_dpu_freq(vdev, vdev->hw->pll.max_ratio);
+	return ivpu_hw_btrs_pll_ratio_to_mhz(vdev, pll_ratio) * HZ_PER_MHZ;
 }
 
-u32 ivpu_hw_btrs_dpu_freq_get(struct ivpu_device *vdev)
+u32 ivpu_hw_btrs_current_freq_get(struct ivpu_device *vdev)
 {
 	if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
-		return pll_ratio_to_dpu_freq_mtl(pll_config_get_mtl(vdev));
+		return pll_ratio_to_mhz_mtl(pll_config_get_mtl(vdev));
 	else
-		return pll_ratio_to_dpu_freq_lnl(pll_config_get_lnl(vdev));
+		return pll_ratio_to_mhz_lnl(pll_config_get_lnl(vdev));
+}
+
+static int ivpu_hw_btrs_cfg_freq_set(struct ivpu_device *vdev, u8 cfg_min_ratio, u8 cfg_max_ratio)
+{
+	u8 min_ratio = clamp_t(u8, cfg_min_ratio, vdev->hw->pll.min_ratio, cfg_max_ratio);
+	u8 pn_ratio = clamp_t(u8, vdev->hw->pll.pn_ratio, min_ratio, cfg_max_ratio);
+	int ret;
+
+	ivpu_dbg(vdev, PM, "Set frequency range to min: %u, pn: %u, max: %u MHz\n",
+		 ivpu_hw_btrs_pll_ratio_to_mhz(vdev, min_ratio),
+		 ivpu_hw_btrs_pll_ratio_to_mhz(vdev, pn_ratio),
+		 ivpu_hw_btrs_pll_ratio_to_mhz(vdev, cfg_max_ratio));
+
+	ret = ivpu_rpm_get(vdev);
+	if (ret < 0)
+		return ret;
+
+	ret = ivpu_jsm_msg_freq_config(vdev, min_ratio, pn_ratio, cfg_max_ratio);
+	ivpu_rpm_put(vdev);
+
+	if (ret) {
+		ivpu_warn(vdev,
+			  "Failed to set frequency to min: %u, pn: %u, max: %u MHz, ret %d\n",
+			  ivpu_hw_btrs_pll_ratio_to_mhz(vdev, min_ratio),
+			  ivpu_hw_btrs_pll_ratio_to_mhz(vdev, pn_ratio),
+			  ivpu_hw_btrs_pll_ratio_to_mhz(vdev, cfg_max_ratio),
+			  ret);
+		return ret;
+	}
+
+	vdev->hw->pll.cfg_min_ratio = cfg_min_ratio;
+	vdev->hw->pll.cfg_max_ratio = cfg_max_ratio;
+
+	return 0;
+}
+
+static u8 dpu_mhz_to_pll_ratio_lnl(u32 freq_mhz)
+{
+	return clamp_t(u32, freq_mhz / (PLL_REF_CLK_FREQ_MHZ / 2), 0, U8_MAX);
+}
+
+int ivpu_hw_btrs_cfg_max_freq_set(struct ivpu_device *vdev, u32 max_freq_mhz)
+{
+	u8 ratio = dpu_mhz_to_pll_ratio_lnl(max_freq_mhz);
+	u8 cfg_max_ratio = clamp_t(u8, ratio, vdev->hw->pll.min_ratio, vdev->hw->pll.max_ratio);
+
+	return ivpu_hw_btrs_cfg_freq_set(vdev, vdev->hw->pll.cfg_min_ratio, cfg_max_ratio);
+}
+
+int ivpu_hw_btrs_cfg_min_freq_set(struct ivpu_device *vdev, u32 min_freq_mhz)
+{
+	u8 ratio = dpu_mhz_to_pll_ratio_lnl(min_freq_mhz);
+	u8 cfg_min_ratio = clamp_t(u8, ratio, vdev->hw->pll.min_ratio, vdev->hw->pll.max_ratio);
+
+	return ivpu_hw_btrs_cfg_freq_set(vdev, cfg_min_ratio, vdev->hw->pll.cfg_max_ratio);
+}
+
+int ivpu_hw_btrs_cfg_freq_init(struct ivpu_device *vdev)
+{
+	if (vdev->hw->pll.min_ratio == vdev->hw->pll.cfg_min_ratio &&
+	    vdev->hw->pll.max_ratio == vdev->hw->pll.cfg_max_ratio)
+		return 0;
+
+	return ivpu_hw_btrs_cfg_freq_set(vdev,
+					 vdev->hw->pll.cfg_min_ratio,
+					 vdev->hw->pll.cfg_max_ratio);
 }
 
 /* Handler for IRQs from Buttress core (irqB) */
@@ -641,8 +707,8 @@ bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq)
 	if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, FREQ_CHANGE, status)) {
 		u32 pll = pll_config_get_mtl(vdev);
 
-		ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq, wp %08x, %lu MHz",
-			 pll, pll_ratio_to_dpu_freq_mtl(pll) / HZ_PER_MHZ);
+		ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq, wp %08x, %u MHz",
+			 pll, pll_ratio_to_mhz_mtl(pll));
 	}
 
 	if (REG_TEST_FLD(VPU_HW_BTRS_MTL_INTERRUPT_STAT, ATS_ERR, status)) {
@@ -695,8 +761,8 @@ bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq)
 	if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, FREQ_CHANGE, status)) {
 		u32 pll = pll_config_get_lnl(vdev);
 
-		ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq, wp %08x, %lu MHz",
-			 pll, pll_ratio_to_dpu_freq_lnl(pll) / HZ_PER_MHZ);
+		ivpu_dbg(vdev, IRQ, "FREQ_CHANGE irq, wp %08x, %u MHz",
+			 pll, pll_ratio_to_mhz_lnl(pll));
 	}
 
 	if (REG_TEST_FLD(VPU_HW_BTRS_LNL_INTERRUPT_STAT, ATS_ERR, status)) {
diff --git a/drivers/accel/ivpu/ivpu_hw_btrs.h b/drivers/accel/ivpu/ivpu_hw_btrs.h
index c4c10e22f30f3..d6343d7345706 100644
--- a/drivers/accel/ivpu/ivpu_hw_btrs.h
+++ b/drivers/accel/ivpu/ivpu_hw_btrs.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (C) 2020-2025 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #ifndef __IVPU_HW_BTRS_H__
@@ -31,8 +31,12 @@ int ivpu_hw_btrs_ip_reset(struct ivpu_device *vdev);
 void ivpu_hw_btrs_profiling_freq_reg_set_lnl(struct ivpu_device *vdev);
 void ivpu_hw_btrs_ats_print_lnl(struct ivpu_device *vdev);
 void ivpu_hw_btrs_clock_relinquish_disable_lnl(struct ivpu_device *vdev);
-u32 ivpu_hw_btrs_dpu_max_freq_get(struct ivpu_device *vdev);
-u32 ivpu_hw_btrs_dpu_freq_get(struct ivpu_device *vdev);
+u32 ivpu_hw_btrs_pll_ratio_to_mhz(struct ivpu_device *vdev, u8 pll_ratio);
+u32 ivpu_hw_btrs_pll_ratio_to_hz(struct ivpu_device *vdev, u8 pll_ratio);
+u32 ivpu_hw_btrs_current_freq_get(struct ivpu_device *vdev);
+int ivpu_hw_btrs_cfg_min_freq_set(struct ivpu_device *vdev, u32 freq_mhz);
+int ivpu_hw_btrs_cfg_max_freq_set(struct ivpu_device *vdev, u32 freq_mhz);
+int ivpu_hw_btrs_cfg_freq_init(struct ivpu_device *vdev);
 bool ivpu_hw_btrs_irq_handler_mtl(struct ivpu_device *vdev, int irq);
 bool ivpu_hw_btrs_irq_handler_lnl(struct ivpu_device *vdev, int irq);
 int ivpu_hw_btrs_dct_get_request(struct ivpu_device *vdev, bool *enable);
diff --git a/drivers/accel/ivpu/ivpu_hw_ip.c b/drivers/accel/ivpu/ivpu_hw_ip.c
index 37f95a0551eda..81f0b1f8f5a66 100644
--- a/drivers/accel/ivpu/ivpu_hw_ip.c
+++ b/drivers/accel/ivpu/ivpu_hw_ip.c
@@ -308,26 +308,26 @@ static void pwr_island_trickle_drive_40xx(struct ivpu_device *vdev, bool enable)
 
 static void pwr_island_drive_37xx(struct ivpu_device *vdev, bool enable)
 {
-	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0);
+	u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0);
 
 	if (enable)
-		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
+		val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
 	else
-		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
+		val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
 
-	REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
+	REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
 }
 
 static void pwr_island_drive_40xx(struct ivpu_device *vdev, bool enable)
 {
-	u32 val = REGV_RD32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0);
+	u32 val = REGV_RD32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0);
 
 	if (enable)
-		val = REG_SET_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
+		val = REG_SET_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
 	else
-		val = REG_CLR_FLD(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, MSS_CPU, val);
+		val = REG_CLR_FLD(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, CSS_CPU, val);
 
-	REGV_WR32(VPU_37XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
+	REGV_WR32(VPU_40XX_HOST_SS_AON_PWR_ISLAND_EN0, val);
 }
 
 static void pwr_island_enable(struct ivpu_device *vdev)
diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.c b/drivers/accel/ivpu/ivpu_jsm_msg.c
index 07b1d6f615a91..17b42a76aef92 100644
--- a/drivers/accel/ivpu/ivpu_jsm_msg.c
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2020-2024 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #include "ivpu_drv.h"
@@ -86,6 +86,9 @@ const char *ivpu_jsm_msg_type_to_str(enum vpu_ipc_msg_type type)
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_ENABLE_DONE);
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_DISABLE);
 	IVPU_CASE_TO_STR(VPU_JSM_MSG_DCT_DISABLE_DONE);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_FREQ_CONFIG);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_FREQ_CONFIG_RSP);
+	IVPU_CASE_TO_STR(VPU_JSM_MSG_RESERVED_111E);
 	}
 	#undef IVPU_CASE_TO_STR
 
@@ -571,3 +574,16 @@ int ivpu_jsm_state_dump_no_reply(struct ivpu_device *vdev)
 	return ivpu_ipc_send_and_wait(vdev, &req, VPU_IPC_CHAN_ASYNC_CMD,
 				      vdev->timeout.state_dump_msg);
 }
+
+int ivpu_jsm_msg_freq_config(struct ivpu_device *vdev, u16 min_ratio, u16 pn_ratio, u16 max_ratio)
+{
+	struct vpu_jsm_msg req = { .type = VPU_JSM_MSG_FREQ_CONFIG};
+	struct vpu_jsm_msg resp;
+
+	req.payload.freq_config.min_freq_pll_ratio = min_ratio;
+	req.payload.freq_config.pn_freq_pll_ratio = pn_ratio;
+	req.payload.freq_config.max_freq_pll_ratio = max_ratio;
+
+	return ivpu_ipc_send_receive_internal(vdev, &req, VPU_JSM_MSG_FREQ_CONFIG_RSP, &resp,
+					      VPU_IPC_CHAN_ASYNC_CMD, vdev->timeout.jsm);
+}
diff --git a/drivers/accel/ivpu/ivpu_jsm_msg.h b/drivers/accel/ivpu/ivpu_jsm_msg.h
index a74f5a0b0d935..28ffe07422b07 100644
--- a/drivers/accel/ivpu/ivpu_jsm_msg.h
+++ b/drivers/accel/ivpu/ivpu_jsm_msg.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (C) 2020-2024 Intel Corporation
+ * Copyright (C) 2020-2026 Intel Corporation
  */
 
 #ifndef __IVPU_JSM_MSG_H__
@@ -45,5 +45,6 @@ int ivpu_jsm_dct_enable(struct ivpu_device *vdev, u32 active_us, u32 inactive_us
 int ivpu_jsm_dct_disable(struct ivpu_device *vdev);
 int ivpu_jsm_state_dump(struct ivpu_device *vdev);
 int ivpu_jsm_state_dump_no_reply(struct ivpu_device *vdev);
+int ivpu_jsm_msg_freq_config(struct ivpu_device *vdev, u16 min_ratio, u16 pn_ratio, u16 max_ratio);
 
 #endif
diff --git a/drivers/accel/ivpu/ivpu_sysfs.c b/drivers/accel/ivpu/ivpu_sysfs.c
index d250a10caca9b..95d201b3b8aa7 100644
--- a/drivers/accel/ivpu/ivpu_sysfs.c
+++ b/drivers/accel/ivpu/ivpu_sysfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #include <linux/device.h>
@@ -82,8 +82,7 @@ static DEVICE_ATTR_RO(npu_memory_utilization);
  * - "OS"		- Operating System Scheduler mode
  *
  */
-static ssize_t
-sched_mode_show(struct device *dev, struct device_attribute *attr, char *buf)
+static ssize_t sched_mode_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
 	struct ivpu_device *vdev = to_ivpu_device(drm);
@@ -94,47 +93,162 @@ sched_mode_show(struct device *dev, struct device_attribute *attr, char *buf)
 static DEVICE_ATTR_RO(sched_mode);
 
 /**
- * DOC: npu_max_frequency
+ * DOC: NPU frequency control and information
+ *
+ * Hardware frequency capabilities:
+ * freq/hw_min_freq - Minimum frequency supported by the NPU hardware.
+ * freq/hw_max_freq - Maximum frequency supported by the NPU hardware.
+ * freq/hw_efficient_freq - Most efficient operating frequency for the NPU.
+ *
+ * Configurable frequency limits (50XX devices and newer):
+ * freq/set_min_freq - Configured minimum operating frequency.
+ * freq/set_max_freq - Configured maximum operating frequency.
+ *
+ * Clamping behavior: Values written to set_min_freq and set_max_freq are
+ * clamped to hardware limits. If set_min_freq exceeds set_max_freq, the driver
+ * clamps set_min_freq to set_max_freq when selecting the operating frequency.
+ *
+ * Current operating frequency:
+ * freq/current_freq - Current frequency in MHz. Valid only when the device is
+ * active; returns 0 when idle. May be lower than the requested range due to
+ * power or thermal constraints.
  *
- * The npu_max_frequency shows maximum frequency in MHz of the NPU's data
- * processing unit
+ * Legacy attributes (backward compatibility):
+ * npu_max_frequency_mhz - Alias for freq/hw_max_freq.
+ * npu_current_frequency_mhz - Alias for freq/current_freq.
  */
+
+static ssize_t hw_min_freq_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct ivpu_device *vdev = to_ivpu_device(drm);
+	u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.min_ratio);
+
+	return sysfs_emit(buf, "%u\n", freq_mhz);
+}
+
+static DEVICE_ATTR_RO(hw_min_freq);
+
+static ssize_t hw_efficient_freq_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct ivpu_device *vdev = to_ivpu_device(drm);
+	u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.pn_ratio);
+
+	return sysfs_emit(buf, "%u\n", freq_mhz);
+}
+
+static DEVICE_ATTR_RO(hw_efficient_freq);
+
+static ssize_t hw_max_freq_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct ivpu_device *vdev = to_ivpu_device(drm);
+	u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.max_ratio);
+
+	return sysfs_emit(buf, "%u\n", freq_mhz);
+}
+
+static DEVICE_ATTR_RO(hw_max_freq);
+
+static ssize_t set_min_freq_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct ivpu_device *vdev = to_ivpu_device(drm);
+	u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.cfg_min_ratio);
+
+	return sysfs_emit(buf, "%u\n", freq_mhz);
+}
+
 static ssize_t
-npu_max_frequency_mhz_show(struct device *dev, struct device_attribute *attr, char *buf)
+set_min_freq_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
 	struct ivpu_device *vdev = to_ivpu_device(drm);
-	u32 freq = ivpu_hw_dpu_max_freq_get(vdev);
+	u32 freq_mhz;
+	int ret;
+
+	ret = kstrtou32(buf, 10, &freq_mhz);
+	if (ret)
+		return ret;
 
-	return sysfs_emit(buf, "%lu\n", freq / HZ_PER_MHZ);
+	ret = ivpu_hw_btrs_cfg_min_freq_set(vdev, freq_mhz);
+	if (ret)
+		return ret;
+
+	return count;
 }
 
-static DEVICE_ATTR_RO(npu_max_frequency_mhz);
+static DEVICE_ATTR_RW(set_min_freq);
+
+static ssize_t set_max_freq_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct ivpu_device *vdev = to_ivpu_device(drm);
+	u32 freq_mhz = ivpu_hw_btrs_pll_ratio_to_mhz(vdev, vdev->hw->pll.cfg_max_ratio);
+
+	return sysfs_emit(buf, "%u\n", freq_mhz);
+}
 
-/**
- * DOC: npu_current_frequency_mhz
- *
- * The npu_current_frequency_mhz shows current frequency in MHz of the NPU's
- * data processing unit
- */
 static ssize_t
-npu_current_frequency_mhz_show(struct device *dev, struct device_attribute *attr, char *buf)
+set_max_freq_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct ivpu_device *vdev = to_ivpu_device(drm);
+	u32 freq_mhz;
+	int ret;
+
+	ret = kstrtou32(buf, 10, &freq_mhz);
+	if (ret)
+		return ret;
+
+	/* Convert MHz to Hz and set max frequency */
+	ret = ivpu_hw_btrs_cfg_max_freq_set(vdev, freq_mhz);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static DEVICE_ATTR_RW(set_max_freq);
+
+static ssize_t current_freq_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct drm_device *drm = dev_get_drvdata(dev);
 	struct ivpu_device *vdev = to_ivpu_device(drm);
-	u32 freq = 0;
+	u32 freq_mhz = 0;
 
 	/* Read frequency only if device is active, otherwise frequency is 0 */
 	if (pm_runtime_get_if_active(vdev->drm.dev) > 0) {
-		freq = ivpu_hw_dpu_freq_get(vdev);
+		freq_mhz = ivpu_hw_btrs_current_freq_get(vdev);
 
 		pm_runtime_put_autosuspend(vdev->drm.dev);
 	}
 
-	return sysfs_emit(buf, "%lu\n", freq / HZ_PER_MHZ);
+	return sysfs_emit(buf, "%u\n", freq_mhz);
 }
 
-static DEVICE_ATTR_RO(npu_current_frequency_mhz);
+static DEVICE_ATTR_RO(current_freq);
+
+/* Alias to current_freq for legacy compat */
+static struct device_attribute dev_attr_npu_max_frequency_mhz =
+	__ATTR(npu_max_frequency_mhz, 0444, hw_max_freq_show, NULL);
+
+static struct device_attribute dev_attr_npu_current_frequency_mhz =
+	__ATTR(npu_current_frequency_mhz, 0444, current_freq_show, NULL);
+
+static struct attribute *ivpu_freq_attrs[] = {
+	&dev_attr_hw_min_freq.attr,
+	&dev_attr_hw_efficient_freq.attr,
+	&dev_attr_hw_max_freq.attr,
+	&dev_attr_current_freq.attr,
+	NULL,
+};
+
+static struct attribute_group ivpu_freq_attr_group = {
+	.name = "freq",
+	.attrs = ivpu_freq_attrs,
+};
 
 static struct attribute *ivpu_dev_attrs[] = {
 	&dev_attr_npu_busy_time_us.attr,
@@ -154,6 +268,32 @@ void ivpu_sysfs_init(struct ivpu_device *vdev)
 	int ret;
 
 	ret = devm_device_add_group(vdev->drm.dev, &ivpu_dev_attr_group);
-	if (ret)
-		ivpu_warn(vdev, "Failed to add group to device, ret %d", ret);
+	if (ret) {
+		ivpu_warn(vdev, "Failed to add sysfs group to device, ret %d", ret);
+		return;
+	}
+
+	ret = devm_device_add_group(vdev->drm.dev, &ivpu_freq_attr_group);
+	if (ret) {
+		ivpu_warn(vdev, "Failed to add sysfs freq group, ret %d", ret);
+		return;
+	}
+
+	if (ivpu_hw_ip_gen(vdev) >= IVPU_HW_IP_50XX) {
+		ret = sysfs_add_file_to_group(&vdev->drm.dev->kobj,
+					      &dev_attr_set_min_freq.attr,
+					      "freq");
+		if (ret) {
+			ivpu_warn(vdev, "Failed to add sysfs set_min_freq to device, ret %d", ret);
+			return;
+		}
+
+		ret = sysfs_add_file_to_group(&vdev->drm.dev->kobj,
+					      &dev_attr_set_max_freq.attr,
+					      "freq");
+		if (ret) {
+			ivpu_warn(vdev, "Failed to add sysfs set_max_freq to device, ret %d", ret);
+			return;
+		}
+	}
 }
diff --git a/drivers/accel/ivpu/vpu_jsm_api.h b/drivers/accel/ivpu/vpu_jsm_api.h
index bca6a44dc041e..6d58d04cc0c41 100644
--- a/drivers/accel/ivpu/vpu_jsm_api.h
+++ b/drivers/accel/ivpu/vpu_jsm_api.h
@@ -23,7 +23,7 @@
 /*
  * Minor version changes when API backward compatibility is preserved.
  */
-#define VPU_JSM_API_VER_MINOR 33
+#define VPU_JSM_API_VER_MINOR 34
 
 /*
  * API header changed (field names, documentation, formatting) but API itself has not been changed
@@ -604,6 +604,16 @@ enum vpu_ipc_msg_type {
 	 * This command has no payload
 	 */
 	VPU_JSM_MSG_DCT_DISABLE = 0x111d,
+	/**
+	 * Reserved command ID to ensure that the following command requests /
+	 * responses have the same lower byte value.
+	 */
+	VPU_JSM_MSG_RESERVED_111E = 0x111e,
+	/**
+	 * Control command: Configure VPU frequency scaling parameters.
+	 * @see vpu_ipc_msg_payload_freq_config
+	 */
+	VPU_JSM_MSG_FREQ_CONFIG = 0x111f,
 	/**
 	 * Dump VPU state. To be used for debug purposes only.
 	 * This command has no payload.
@@ -766,6 +776,11 @@ enum vpu_ipc_msg_type {
 	 * This command has no payload
 	 */
 	VPU_JSM_MSG_DCT_DISABLE_DONE = 0x221e,
+	/**
+	 * Response to control command: Configure VPU frequency scaling parameters.
+	 * @see vpu_ipc_msg_payload_freq_config
+	 */
+	VPU_JSM_MSG_FREQ_CONFIG_RSP = 0x221f,
 	/**
 	 * Response to state dump control command.
 	 * This command has no payload.
@@ -1650,6 +1665,22 @@ struct vpu_ipc_msg_payload_pwr_dct_control {
 	u32 dct_inactive_us;
 };
 
+/**
+ * Payload for @ref VPU_JSM_MSG_FREQ_CONFIG message.
+ *
+ * This payload allows the host to configure the VPU frequency scaling parameters.
+ */
+struct vpu_ipc_msg_payload_freq_config {
+	/** Minimum frequency PLL ratio */
+	u32 min_freq_pll_ratio;
+	/** Efficiency frequency PLL ratio */
+	u32 pn_freq_pll_ratio;
+	/** Maximum frequency PLL ratio */
+	u32 max_freq_pll_ratio;
+	/** Reserved for 64-bit alignment */
+	u32 reserved_0;
+};
+
 /*
  * Payloads union, used to define complete message format.
  */
@@ -1692,6 +1723,7 @@ union vpu_ipc_msg_payload {
 	struct vpu_ipc_msg_payload_hws_resume_engine hws_resume_engine;
 	struct vpu_ipc_msg_payload_pwr_d0i3_enter pwr_d0i3_enter;
 	struct vpu_ipc_msg_payload_pwr_dct_control pwr_dct_control;
+	struct vpu_ipc_msg_payload_freq_config freq_config;
 };
 
 /**
diff --git a/drivers/accel/qaic/mhi_controller.c b/drivers/accel/qaic/mhi_controller.c
index 4d787f77ce419..40e6d262ef217 100644
--- a/drivers/accel/qaic/mhi_controller.c
+++ b/drivers/accel/qaic/mhi_controller.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
-/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <linux/delay.h>
 #include <linux/err.h>
diff --git a/drivers/accel/qaic/mhi_controller.h b/drivers/accel/qaic/mhi_controller.h
index 8939f6ae185ed..c1940c8392467 100644
--- a/drivers/accel/qaic/mhi_controller.h
+++ b/drivers/accel/qaic/mhi_controller.h
@@ -1,8 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0-only
- *
- * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved.
- * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
- */
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+/* Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #ifndef MHICONTROLLERQAIC_H_
 #define MHICONTROLLERQAIC_H_
diff --git a/drivers/accel/qaic/qaic.h b/drivers/accel/qaic/qaic.h
index fa7a8155658c4..02ca99000e2ae 100644
--- a/drivers/accel/qaic/qaic.h
+++ b/drivers/accel/qaic/qaic.h
@@ -1,8 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0-only
- *
- * Copyright (c) 2019-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved.
- */
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+/* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #ifndef _QAIC_H_
 #define _QAIC_H_
@@ -152,8 +151,6 @@ struct qaic_device {
 	struct list_head	cntl_xfer_list;
 	/* Synchronizes MHI control device transactions and its xfer list */
 	struct mutex		cntl_mutex;
-	/* Array of DBC struct of this device */
-	struct dma_bridge_chan	*dbc;
 	/* Work queue for tasks related to MHI control device */
 	struct workqueue_struct	*cntl_wq;
 	/* Synchronizes all the users of device during cleanup */
@@ -206,6 +203,8 @@ struct qaic_device {
 	void			*ssr_mhi_buf;
 	/* DBC which is under SSR. Sentinel U32_MAX would mean that no SSR in progress */
 	u32			ssr_dbc;
+	/* Array of DBC struct of this device */
+	struct dma_bridge_chan	dbc[] __counted_by(num_dbc);
 };
 
 struct qaic_drm_device {
diff --git a/drivers/accel/qaic/qaic_control.c b/drivers/accel/qaic/qaic_control.c
index 43f84d4389602..bb94d3556904a 100644
--- a/drivers/accel/qaic/qaic_control.c
+++ b/drivers/accel/qaic/qaic_control.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
-/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <asm/byteorder.h>
 #include <linux/completion.h>
diff --git a/drivers/accel/qaic/qaic_data.c b/drivers/accel/qaic/qaic_data.c
index 1e4c579d27256..ef0351b6dc9b5 100644
--- a/drivers/accel/qaic/qaic_data.c
+++ b/drivers/accel/qaic/qaic_data.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
-/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <linux/bitfield.h>
 #include <linux/bits.h>
diff --git a/drivers/accel/qaic/qaic_debugfs.c b/drivers/accel/qaic/qaic_debugfs.c
index 8dc4fe5bb560e..95c78e12dd61b 100644
--- a/drivers/accel/qaic/qaic_debugfs.c
+++ b/drivers/accel/qaic/qaic_debugfs.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 /* Copyright (c) 2020, The Linux Foundation. All rights reserved. */
-/* Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <linux/debugfs.h>
 #include <linux/device.h>
@@ -27,6 +27,8 @@
 struct bootlog_msg {
 	/* Buffer for bootlog messages */
 	char str[BOOTLOG_MSG_SIZE];
+	/* Length of bootlog message */
+	size_t len;
 	/* Root struct of device, used to access device resources */
 	struct qaic_device *qdev;
 	/* Work struct to schedule work coming on QAIC_LOGGING channel */
@@ -46,18 +48,15 @@ static int bootlog_show(struct seq_file *s, void *unused)
 {
 	struct bootlog_page *page;
 	struct qaic_device *qdev;
-	void *page_end;
+	size_t len;
 	void *log;
 
 	qdev = s->private;
 	mutex_lock(&qdev->bootlog_mutex);
 	list_for_each_entry(page, &qdev->bootlog, node) {
 		log = page + 1;
-		page_end = (void *)page + page->offset;
-		while (log < page_end) {
-			seq_printf(s, "%s", (char *)log);
-			log += strlen(log) + 1;
-		}
+		len = page->offset - sizeof(*page);
+		seq_write(s, log, len);
 	}
 	mutex_unlock(&qdev->bootlog_mutex);
 
@@ -182,15 +181,14 @@ static void bootlog_commit(struct qaic_device *qdev, unsigned int size)
 static void bootlog_log(struct work_struct *work)
 {
 	struct bootlog_msg *msg = container_of(work, struct bootlog_msg, work);
-	unsigned int len = strlen(msg->str) + 1;
 	struct qaic_device *qdev = msg->qdev;
 	void *log;
 
 	mutex_lock(&qdev->bootlog_mutex);
-	log = bootlog_get_space(qdev, len);
+	log = bootlog_get_space(qdev, msg->len);
 	if (log) {
-		memcpy(log, msg, len);
-		bootlog_commit(qdev, len);
+		memcpy(log, msg, msg->len);
+		bootlog_commit(qdev, msg->len);
 	}
 	mutex_unlock(&qdev->bootlog_mutex);
 
@@ -265,14 +263,18 @@ static void qaic_bootlog_mhi_dl_xfer_cb(struct mhi_device *mhi_dev, struct mhi_r
 {
 	struct qaic_device *qdev = dev_get_drvdata(&mhi_dev->dev);
 	struct bootlog_msg *msg = mhi_result->buf_addr;
+	int status = mhi_result->transaction_status;
 
-	if (mhi_result->transaction_status) {
+	if (status && status != -EOVERFLOW) {
 		devm_kfree(&qdev->pdev->dev, msg);
 		return;
 	}
 
-	/* Force a null at the end of the transferred string */
-	msg->str[mhi_result->bytes_xferd - 1] = 0;
+	msg->len = mhi_result->bytes_xferd;
+
+	/* Exclude trailing null to normalize AIC100/AIC200 line endings */
+	if (msg->len && msg->str[msg->len - 1] == '\0')
+		msg->len--;
 
 	queue_work(qdev->bootlog_wq, &msg->work);
 }
diff --git a/drivers/accel/qaic/qaic_debugfs.h b/drivers/accel/qaic/qaic_debugfs.h
index 05e74f84cf9f0..59a002bab07c2 100644
--- a/drivers/accel/qaic/qaic_debugfs.h
+++ b/drivers/accel/qaic/qaic_debugfs.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 
 /* Copyright (c) 2020, The Linux Foundation. All rights reserved. */
-/* Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #ifndef __QAIC_DEBUGFS_H__
 #define __QAIC_DEBUGFS_H__
diff --git a/drivers/accel/qaic/qaic_drv.c b/drivers/accel/qaic/qaic_drv.c
index 63fb8c7b4abcb..0acf6cdd44f81 100644
--- a/drivers/accel/qaic/qaic_drv.c
+++ b/drivers/accel/qaic/qaic_drv.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
-/* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
@@ -43,6 +43,7 @@ MODULE_IMPORT_NS("DMA_BUF");
 #define QAIC_DESC			"Qualcomm Cloud AI Accelerators"
 #define CNTL_MAJOR			5
 #define CNTL_MINOR			0
+#define DBC_NUM				16
 
 struct qaic_device_config {
 	/* Indicates the AIC family the device belongs to */
@@ -405,15 +406,12 @@ static struct qaic_device *create_qdev(struct pci_dev *pdev,
 	struct drm_device *drm;
 	int i, ret;
 
-	qdev = devm_kzalloc(dev, sizeof(*qdev), GFP_KERNEL);
+	qdev = devm_kzalloc(dev, struct_size(qdev, dbc, DBC_NUM), GFP_KERNEL);
 	if (!qdev)
 		return NULL;
 
+	qdev->num_dbc = DBC_NUM;
 	qdev->dev_state = QAIC_OFFLINE;
-	qdev->num_dbc = 16;
-	qdev->dbc = devm_kcalloc(dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL);
-	if (!qdev->dbc)
-		return NULL;
 
 	qddev = devm_drm_dev_alloc(&pdev->dev, &qaic_accel_driver, struct qaic_drm_device, drm);
 	if (IS_ERR(qddev))
diff --git a/drivers/accel/qaic/qaic_ras.c b/drivers/accel/qaic/qaic_ras.c
index 6791af366cba7..e0205c2cda3df 100644
--- a/drivers/accel/qaic/qaic_ras.c
+++ b/drivers/accel/qaic/qaic_ras.c
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 /* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. */
-/* Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved. */
 /* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <asm/byteorder.h>
diff --git a/drivers/accel/qaic/qaic_ras.h b/drivers/accel/qaic/qaic_ras.h
index d44a4eeeb0601..7b3fe9585ed9a 100644
--- a/drivers/accel/qaic/qaic_ras.h
+++ b/drivers/accel/qaic/qaic_ras.h
@@ -1,4 +1,5 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
+
 /* Copyright (c) 2020, The Linux Foundation. All rights reserved. */
 
 #ifndef __QAIC_RAS_H__
diff --git a/drivers/accel/qaic/qaic_ssr.c b/drivers/accel/qaic/qaic_ssr.c
index a5bb6078824b7..a989286549599 100644
--- a/drivers/accel/qaic/qaic_ssr.c
+++ b/drivers/accel/qaic/qaic_ssr.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
 /* Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. */
-/* Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <asm/byteorder.h>
 #include <drm/drm_file.h>
diff --git a/drivers/accel/qaic/qaic_ssr.h b/drivers/accel/qaic/qaic_ssr.h
index 97ccff305750f..af074edbf9675 100644
--- a/drivers/accel/qaic/qaic_ssr.h
+++ b/drivers/accel/qaic/qaic_ssr.h
@@ -1,8 +1,7 @@
-/* SPDX-License-Identifier: GPL-2.0-only
- *
- * Copyright (c) 2020, The Linux Foundation. All rights reserved.
- * Copyright (c) 2021, 2024 Qualcomm Innovation Center, Inc. All rights reserved.
- */
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+/* Copyright (c) 2020, The Linux Foundation. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #ifndef __QAIC_SSR_H__
 #define __QAIC_SSR_H__
diff --git a/drivers/accel/qaic/qaic_timesync.c b/drivers/accel/qaic/qaic_timesync.c
index 939462b9958d5..9faf71f47bdc2 100644
--- a/drivers/accel/qaic/qaic_timesync.c
+++ b/drivers/accel/qaic/qaic_timesync.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
-/* Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved. */
+
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <linux/io.h>
 #include <linux/kernel.h>
diff --git a/drivers/accel/qaic/qaic_timesync.h b/drivers/accel/qaic/qaic_timesync.h
index 77b9c2b55057c..6aeda1d62a353 100644
--- a/drivers/accel/qaic/qaic_timesync.h
+++ b/drivers/accel/qaic/qaic_timesync.h
@@ -1,7 +1,6 @@
-/* SPDX-License-Identifier: GPL-2.0-only
- *
- * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
- */
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #ifndef __QAIC_TIMESYNC_H__
 #define __QAIC_TIMESYNC_H__
diff --git a/drivers/accel/qaic/sahara.c b/drivers/accel/qaic/sahara.c
index fd3c3b2d1fd3b..9fea294e1d7bf 100644
--- a/drivers/accel/qaic/sahara.c
+++ b/drivers/accel/qaic/sahara.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
-/* Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #include <linux/devcoredump.h>
 #include <linux/firmware.h>
diff --git a/drivers/accel/qaic/sahara.h b/drivers/accel/qaic/sahara.h
index 640208acc0d13..08037281c80e2 100644
--- a/drivers/accel/qaic/sahara.h
+++ b/drivers/accel/qaic/sahara.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 
-/* Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. */
+/* Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */
 
 #ifndef __SAHARA_H__
 #define __SAHARA_H__
diff --git a/drivers/accel/rocket/rocket_job.c b/drivers/accel/rocket/rocket_job.c
index ac51bff39833f..2f1861f960ccb 100644
--- a/drivers/accel/rocket/rocket_job.c
+++ b/drivers/accel/rocket/rocket_job.c
@@ -437,7 +437,6 @@ int rocket_job_init(struct rocket_core *core)
 {
 	struct drm_sched_init_args args = {
 		.ops = &rocket_sched_ops,
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit = 1,
 		.timeout = msecs_to_jiffies(JOB_TIMEOUT_MS),
 		.name = dev_name(core->dev),
diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c
index d4ae21a710079..b82dd67d98c9a 100644
--- a/drivers/acpi/battery.c
+++ b/drivers/acpi/battery.c
@@ -94,6 +94,7 @@ struct acpi_battery {
 	struct power_supply *bat;
 	struct power_supply_desc bat_desc;
 	struct acpi_device *device;
+	struct device *phys_dev;
 	struct notifier_block pm_nb;
 	struct list_head list;
 	unsigned long update_time;
@@ -1033,7 +1034,7 @@ static int acpi_battery_update(struct acpi_battery *battery, bool resume)
 	if ((battery->state & ACPI_BATTERY_STATE_CRITICAL) ||
 	    (test_bit(ACPI_BATTERY_ALARM_PRESENT, &battery->flags) &&
 	     (battery->capacity_now <= battery->alarm)))
-		acpi_pm_wakeup_event(&battery->device->dev);
+		acpi_pm_wakeup_event(battery->phys_dev);
 
 	return result;
 }
@@ -1231,6 +1232,7 @@ static int acpi_battery_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, battery);
 
+	battery->phys_dev = &pdev->dev;
 	battery->device = device;
 
 	result = devm_mutex_init(&pdev->dev, &battery->update_lock);
diff --git a/drivers/acpi/bus.c b/drivers/acpi/bus.c
index 2ec095e2009e4..05118a471d4bc 100644
--- a/drivers/acpi/bus.c
+++ b/drivers/acpi/bus.c
@@ -831,9 +831,9 @@ const struct acpi_device *acpi_companion_match(const struct device *dev)
  * identifiers and a _DSD object with the "compatible" property, use that
  * property to match against the given list of identifiers.
  */
-static bool acpi_of_match_device(const struct acpi_device *adev,
-				 const struct of_device_id *of_match_table,
-				 const struct of_device_id **of_id)
+bool acpi_of_match_device(const struct acpi_device *adev,
+			  const struct of_device_id *of_match_table,
+			  const struct of_device_id **of_id)
 {
 	const union acpi_object *of_compatible, *obj;
 	int i, nval;
@@ -1181,6 +1181,21 @@ int acpi_bus_for_each_dev(int (*fn)(struct device *, void *), void *data)
 }
 EXPORT_SYMBOL_GPL(acpi_bus_for_each_dev);
 
+/**
+ * acpi_bus_find_device_by_name() - Locate an ACPI device by its name
+ * @name: Name of the device to match
+ *
+ * The caller is responsible for calling put_device() on the returned object.
+ *
+ * Returns:
+ * New reference to the matched device or NULL if the device can't be found.
+ */
+struct device *acpi_bus_find_device_by_name(const char *name)
+{
+	return bus_find_device_by_name(&acpi_bus_type, NULL, name);
+}
+EXPORT_SYMBOL_GPL(acpi_bus_find_device_by_name);
+
 struct acpi_dev_walk_context {
 	int (*fn)(struct acpi_device *, void *);
 	void *data;
diff --git a/drivers/acpi/pmic/intel_pmic.c b/drivers/acpi/pmic/intel_pmic.c
index 134e9ca8eaa23..9bd46defc5a27 100644
--- a/drivers/acpi/pmic/intel_pmic.c
+++ b/drivers/acpi/pmic/intel_pmic.c
@@ -67,14 +67,12 @@ static acpi_status intel_pmic_power_handler(u32 function,
 	if (result == -ENOENT)
 		return AE_BAD_PARAMETER;
 
-	mutex_lock(&opregion->lock);
+	guard(mutex)(&opregion->lock);
 
 	result = function == ACPI_READ ?
 		d->get_power(regmap, reg, bit, value64) :
 		d->update_power(regmap, reg, bit, *value64 == 1);
 
-	mutex_unlock(&opregion->lock);
-
 	return result ? AE_ERROR : AE_OK;
 }
 
@@ -182,19 +180,16 @@ static acpi_status intel_pmic_thermal_handler(u32 function,
 	if (result == -ENOENT)
 		return AE_BAD_PARAMETER;
 
-	mutex_lock(&opregion->lock);
-
-	if (pmic_thermal_is_temp(address))
-		result = pmic_thermal_temp(opregion, reg, function, value64);
-	else if (pmic_thermal_is_aux(address))
-		result = pmic_thermal_aux(opregion, reg, function, value64);
-	else if (pmic_thermal_is_pen(address))
-		result = pmic_thermal_pen(opregion, reg, bit,
-						function, value64);
-	else
-		result = -EINVAL;
-
-	mutex_unlock(&opregion->lock);
+	scoped_guard(mutex, &opregion->lock) {
+		if (pmic_thermal_is_temp(address))
+			result = pmic_thermal_temp(opregion, reg, function, value64);
+		else if (pmic_thermal_is_aux(address))
+			result = pmic_thermal_aux(opregion, reg, function, value64);
+		else if (pmic_thermal_is_pen(address))
+			result = pmic_thermal_pen(opregion, reg, bit, function, value64);
+		else
+			result = -EINVAL;
+	}
 
 	if (result < 0) {
 		if (result == -EINVAL)
@@ -354,13 +349,15 @@ int intel_soc_pmic_exec_mipi_pmic_seq_element(u16 i2c_address, u32 reg_address,
 
 	d = intel_pmic_opregion->data;
 
-	mutex_lock(&intel_pmic_opregion->lock);
+	guard(mutex)(&intel_pmic_opregion->lock);
 
 	if (d->exec_mipi_pmic_seq_element) {
-		ret = d->exec_mipi_pmic_seq_element(intel_pmic_opregion->regmap,
+		return d->exec_mipi_pmic_seq_element(intel_pmic_opregion->regmap,
 						    i2c_address, reg_address,
 						    value, mask);
-	} else if (d->pmic_i2c_address) {
+	}
+
+	if (d->pmic_i2c_address) {
 		if (i2c_address == d->pmic_i2c_address) {
 			ret = regmap_update_bits(intel_pmic_opregion->regmap,
 						 reg_address, mask, value);
@@ -376,8 +373,6 @@ int intel_soc_pmic_exec_mipi_pmic_seq_element(u16 i2c_address, u32 reg_address,
 		ret = -EOPNOTSUPP;
 	}
 
-	mutex_unlock(&intel_pmic_opregion->lock);
-
 	return ret;
 }
 EXPORT_SYMBOL_GPL(intel_soc_pmic_exec_mipi_pmic_seq_element);
diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c
index 530547cda8b28..a0c36bd968d30 100644
--- a/drivers/acpi/scan.c
+++ b/drivers/acpi/scan.c
@@ -122,7 +122,7 @@ bool acpi_scan_is_offline(struct acpi_device *adev, bool uevent)
 	mutex_lock_nested(&adev->physical_node_lock, SINGLE_DEPTH_NESTING);
 
 	list_for_each_entry(pn, &adev->physical_node_list, node)
-		if (device_supports_offline(pn->dev) && !pn->dev->offline) {
+		if (device_supports_offline(pn->dev) && !dev_offline(pn->dev)) {
 			if (uevent)
 				kobject_uevent_env(&pn->dev->kobj, KOBJ_CHANGE, envp);
 
diff --git a/drivers/ata/ahci.c b/drivers/ata/ahci.c
index 1d73a53370cf3..b4e5d347151a9 100644
--- a/drivers/ata/ahci.c
+++ b/drivers/ata/ahci.c
@@ -1888,6 +1888,24 @@ static ssize_t remapped_nvme_show(struct device *dev,
 
 static DEVICE_ATTR_RO(remapped_nvme);
 
+static int ahci_validate_bar_size(struct pci_dev *pdev, int bar,
+				  struct ahci_host_priv *hpriv)
+{
+	u32 cap = readl(hpriv->mmio + HOST_CAP);
+	unsigned int max_ports = ahci_nr_ports(cap);
+	u32 last_port_end = 0x100 + (max_ports * 0x80);
+	resource_size_t bar_size = pci_resource_len(pdev, bar);
+
+	if (last_port_end > bar_size) {
+		dev_warn(&pdev->dev,
+			 "BAR%d too small for %u ports (last port ends at %#x, BAR %pa)\n",
+			 bar, max_ports, last_port_end, &bar_size);
+		return -ENODEV;
+	}
+
+	return 0;
+}
+
 static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 {
 	unsigned int board_id = ent->driver_data;
@@ -1988,6 +2006,10 @@ static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (!hpriv->mmio)
 		return -ENOMEM;
 
+	rc = ahci_validate_bar_size(pdev, ahci_pci_bar, hpriv);
+	if (rc)
+		return rc;
+
 	/* detect remapped nvme devices */
 	ahci_remap_check(pdev, ahci_pci_bar, hpriv);
 
diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
index c79abdfcd7a9b..0c5e5b29bce4c 100644
--- a/drivers/ata/libahci.c
+++ b/drivers/ata/libahci.c
@@ -1527,6 +1527,7 @@ int ahci_do_softreset(struct ata_link *link, unsigned int *class,
 	ata_link_err(link, "softreset failed (%s)\n", reason);
 	return rc;
 }
+EXPORT_SYMBOL_GPL(ahci_do_softreset);
 
 int ahci_check_ready(struct ata_link *link)
 {
@@ -1544,7 +1545,6 @@ static int ahci_softreset(struct ata_link *link, unsigned int *class,
 
 	return ahci_do_softreset(link, class, pmp, deadline, ahci_check_ready);
 }
-EXPORT_SYMBOL_GPL(ahci_do_softreset);
 
 static int ahci_bad_pmp_check_ready(struct ata_link *link)
 {
@@ -2636,7 +2636,7 @@ void ahci_print_info(struct ata_host *host, const char *scc_s)
 		,
 
 		hweight32(impl),
-		(cap & 0x1f) + 1,
+		ahci_nr_ports(cap),
 		impl);
 
 	dev_info(host->dev,
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
index e76d15411e2ae..3d0027ec33c2a 100644
--- a/drivers/ata/libata-core.c
+++ b/drivers/ata/libata-core.c
@@ -5584,6 +5584,7 @@ void ata_link_init(struct ata_port *ap, struct ata_link *link, int pmp)
 	link->pmp = pmp;
 	link->active_tag = ATA_TAG_POISON;
 	link->hw_sata_spd_limit = UINT_MAX;
+	INIT_WORK(&link->deferred_qc_work, ata_scsi_deferred_qc_work);
 
 	/* can't use iterator, ap isn't initialized yet */
 	for (i = 0; i < ATA_MAX_DEVICES; i++) {
@@ -5666,7 +5667,6 @@ struct ata_port *ata_port_alloc(struct ata_host *host)
 	mutex_init(&ap->scsi_scan_mutex);
 	INIT_DELAYED_WORK(&ap->hotplug_task, ata_scsi_hotplug);
 	INIT_DELAYED_WORK(&ap->scsi_rescan_task, ata_scsi_dev_rescan);
-	INIT_WORK(&ap->deferred_qc_work, ata_scsi_deferred_qc_work);
 	INIT_LIST_HEAD(&ap->eh_done_q);
 	init_waitqueue_head(&ap->eh_wait_q);
 	init_completion(&ap->park_req_pending);
@@ -6291,12 +6291,15 @@ static void ata_port_detach(struct ata_port *ap)
 
 	/* It better be dead now and not have any remaining deferred qc. */
 	WARN_ON(!(ap->pflags & ATA_PFLAG_UNLOADED));
-	WARN_ON(ap->deferred_qc);
 
-	cancel_work_sync(&ap->deferred_qc_work);
 	cancel_delayed_work_sync(&ap->hotplug_task);
 	cancel_delayed_work_sync(&ap->scsi_rescan_task);
 
+	ata_for_each_link(link, ap, PMP_FIRST) {
+		WARN_ON(link->deferred_qc);
+		cancel_work_sync(&link->deferred_qc_work);
+	}
+
 	/* Delete port multiplier link transport devices */
 	if (ap->pmp_link) {
 		int i;
diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index 9a4b67b90b17b..715bc525b38ff 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -651,11 +651,11 @@ int ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
 			if (qc->scsicmd != scmd)
 				continue;
 			if ((qc->flags & ATA_QCFLAG_ACTIVE) ||
-			    qc == ap->deferred_qc)
+			    qc == qc->dev->link->deferred_qc)
 				break;
 		}
 
-		if (i < ATA_MAX_QUEUE && qc == ap->deferred_qc) {
+		if (i < ATA_MAX_QUEUE && qc == qc->dev->link->deferred_qc) {
 			/*
 			 * This is a deferred command that timed out while
 			 * waiting for the command queue to drain. Since the qc
@@ -666,8 +666,8 @@ int ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap,
 			 * deferred qc work from issuing this qc.
 			 */
 			WARN_ON_ONCE(qc->flags & ATA_QCFLAG_ACTIVE);
-			ap->deferred_qc = NULL;
-			cancel_work(&ap->deferred_qc_work);
+			qc->dev->link->deferred_qc = NULL;
+			cancel_work(&qc->dev->link->deferred_qc_work);
 			set_host_byte(scmd, DID_TIME_OUT);
 			scsi_eh_finish_cmd(scmd, &ap->eh_done_q);
 		} else if (i < ATA_MAX_QUEUE) {
@@ -819,7 +819,7 @@ void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap)
 		ap->pflags &= ~ATA_PFLAG_LOADING;
 	else if ((ap->pflags & ATA_PFLAG_SCSI_HOTPLUG) &&
 		!(ap->flags & ATA_FLAG_SAS_HOST))
-		schedule_delayed_work(&ap->hotplug_task, 0);
+		queue_delayed_work(system_dfl_long_wq, &ap->hotplug_task, 0);
 
 	if (ap->pflags & ATA_PFLAG_RECOVERED)
 		ata_port_info(ap, "EH complete\n");
diff --git a/drivers/ata/libata-pmp.c b/drivers/ata/libata-pmp.c
index e3adc008fed1e..e8540931b4a11 100644
--- a/drivers/ata/libata-pmp.c
+++ b/drivers/ata/libata-pmp.c
@@ -110,13 +110,24 @@ int sata_pmp_qc_defer_cmd_switch(struct ata_queued_cmd *qc)
 {
 	struct ata_link *link = qc->dev->link;
 	struct ata_port *ap = link->ap;
+	int ret;
 
 	if (ap->excl_link == NULL || ap->excl_link == link) {
 		if (ap->nr_active_links == 0 || ata_link_active(link)) {
 			qc->flags |= ATA_QCFLAG_CLEAR_EXCL;
-			return ata_std_qc_defer(qc);
+			ret = ata_std_qc_defer(qc);
+			if (ret == ATA_DEFER_LINK)
+				return ATA_DEFER_LINK_EXCL;
+			return ret;
 		}
 
+		/*
+		 * Note: ap->excl_link contains the link that is next in line,
+		 * i.e. implicit round robin. If there is only one link
+		 * dispatching, ap->excl_link will be left unclaimed, allowing
+		 * other links to set ap->excl_link, ensuring that the currently
+		 * active link cannot queue any more.
+		 */
 		ap->excl_link = link;
 	}
 
@@ -571,8 +582,11 @@ static void sata_pmp_detach(struct ata_device *dev)
 	if (ap->ops->pmp_detach)
 		ap->ops->pmp_detach(ap);
 
-	ata_for_each_link(tlink, ap, EDGE)
+	ata_for_each_link(tlink, ap, EDGE) {
+		WARN_ON(tlink->deferred_qc);
+		cancel_work_sync(&tlink->deferred_qc_work);
 		ata_eh_detach_dev(tlink->device);
+	}
 
 	spin_lock_irqsave(ap->lock, flags);
 	ap->nr_pmp_links = 0;
diff --git a/drivers/ata/libata-scsi.c b/drivers/ata/libata-scsi.c
index f44612e269a4d..1f4a4329fc11f 100644
--- a/drivers/ata/libata-scsi.c
+++ b/drivers/ata/libata-scsi.c
@@ -1664,8 +1664,9 @@ static void ata_scsi_qc_done(struct ata_queued_cmd *qc, bool set_result,
 
 void ata_scsi_deferred_qc_work(struct work_struct *work)
 {
-	struct ata_port *ap =
-		container_of(work, struct ata_port, deferred_qc_work);
+	struct ata_link *link =
+		container_of(work, struct ata_link, deferred_qc_work);
+	struct ata_port *ap = link->ap;
 	struct ata_queued_cmd *qc;
 	unsigned long flags;
 
@@ -1676,10 +1677,10 @@ void ata_scsi_deferred_qc_work(struct work_struct *work)
 	 * such case, we should not need any more deferring the qc, so warn if
 	 * qc_defer() says otherwise.
 	 */
-	qc = ap->deferred_qc;
+	qc = link->deferred_qc;
 	if (qc && !ata_port_eh_scheduled(ap)) {
 		WARN_ON_ONCE(ap->ops->qc_defer(qc));
-		ap->deferred_qc = NULL;
+		link->deferred_qc = NULL;
 		ata_qc_issue(qc);
 	}
 
@@ -1688,7 +1689,7 @@ void ata_scsi_deferred_qc_work(struct work_struct *work)
 
 void ata_scsi_requeue_deferred_qc(struct ata_port *ap)
 {
-	struct ata_queued_cmd *qc = ap->deferred_qc;
+	struct ata_link *link;
 
 	lockdep_assert_held(ap->lock);
 
@@ -1697,16 +1698,21 @@ void ata_scsi_requeue_deferred_qc(struct ata_port *ap)
 	 * do not try to be smart about what to do with this deferred command
 	 * and simply requeue it by completing it with DID_REQUEUE.
 	 */
-	if (qc) {
-		ap->deferred_qc = NULL;
-		cancel_work(&ap->deferred_qc_work);
-		ata_scsi_qc_done(qc, true, DID_REQUEUE << 16);
+	ata_for_each_link(link, ap, PMP_FIRST) {
+		struct ata_queued_cmd *qc = link->deferred_qc;
+
+		if (qc) {
+			link->deferred_qc = NULL;
+			cancel_work(&link->deferred_qc_work);
+			ata_scsi_qc_done(qc, true, DID_REQUEUE << 16);
+		}
 	}
 }
 
-static void ata_scsi_schedule_deferred_qc(struct ata_port *ap)
+static void ata_scsi_schedule_deferred_qc(struct ata_link *link)
 {
-	struct ata_queued_cmd *qc = ap->deferred_qc;
+	struct ata_queued_cmd *qc = link->deferred_qc;
+	struct ata_port *ap = link->ap;
 
 	lockdep_assert_held(ap->lock);
 
@@ -1723,12 +1729,12 @@ static void ata_scsi_schedule_deferred_qc(struct ata_port *ap)
 		return;
 	}
 	if (!ap->ops->qc_defer(qc))
-		queue_work(system_highpri_wq, &ap->deferred_qc_work);
+		queue_work(system_highpri_wq, &link->deferred_qc_work);
 }
 
 static void ata_scsi_qc_complete(struct ata_queued_cmd *qc)
 {
-	struct ata_port *ap = qc->ap;
+	struct ata_link *link = qc->dev->link;
 	struct scsi_cmnd *cmd = qc->scsicmd;
 	u8 *cdb = cmd->cmnd;
 	bool have_sense = qc->flags & ATA_QCFLAG_SENSE_VALID;
@@ -1759,22 +1765,23 @@ static void ata_scsi_qc_complete(struct ata_queued_cmd *qc)
 
 	ata_scsi_qc_done(qc, false, 0);
 
-	ata_scsi_schedule_deferred_qc(ap);
+	ata_scsi_schedule_deferred_qc(link);
 }
 
 static int ata_scsi_qc_issue(struct ata_port *ap, struct ata_queued_cmd *qc)
 {
+	struct ata_link *link = qc->dev->link;
 	int ret;
 
 	if (!ap->ops->qc_defer)
-		goto issue;
+		goto issue_qc;
 
 	/*
 	 * If we already have a deferred qc, then rely on the SCSI layer to
 	 * requeue and defer all incoming commands until the deferred qc is
 	 * processed, once all on-going commands complete.
 	 */
-	if (ap->deferred_qc) {
+	if (link->deferred_qc) {
 		ata_qc_free(qc);
 		return SCSI_MLQUEUE_DEVICE_BUSY;
 	}
@@ -1786,38 +1793,46 @@ static int ata_scsi_qc_issue(struct ata_port *ap, struct ata_queued_cmd *qc)
 		break;
 	case ATA_DEFER_LINK:
 		ret = SCSI_MLQUEUE_DEVICE_BUSY;
-		break;
+		goto defer_qc;
+	case ATA_DEFER_LINK_EXCL:
+		/*
+		 * Drivers making use of ap->excl_link cannot store the QC in
+		 * link->deferred_qc, because the ap->excl_link handling is
+		 * incompatible with the link->deferred_qc workqueue handling.
+		 */
+		ret = SCSI_MLQUEUE_DEVICE_BUSY;
+		goto free_qc;
 	case ATA_DEFER_PORT:
 		ret = SCSI_MLQUEUE_HOST_BUSY;
-		break;
+		goto free_qc;
 	default:
 		WARN_ON_ONCE(1);
 		ret = SCSI_MLQUEUE_HOST_BUSY;
-		break;
+		goto free_qc;
 	}
 
-	if (ret) {
-		/*
-		 * We must defer this qc: if this is not an NCQ command, keep
-		 * this qc as a deferred one and report to the SCSI layer that
-		 * we issued it so that it is not requeued. The deferred qc will
-		 * be issued with the port deferred_qc_work once all on-going
-		 * commands complete.
-		 */
-		if (!ata_is_ncq(qc->tf.protocol)) {
-			ap->deferred_qc = qc;
-			return 0;
-		}
+issue_qc:
+	ata_qc_issue(qc);
+	return 0;
 
-		/* Force a requeue of the command to defer its execution. */
-		ata_qc_free(qc);
-		return ret;
+defer_qc:
+	/*
+	 * We must defer this qc: if this is not an NCQ command, keep
+	 * this qc as a deferred one and report to the SCSI layer that
+	 * we issued it so that it is not requeued. The deferred qc will
+	 * be issued with the port deferred_qc_work once all on-going
+	 * commands complete.
+	 */
+	if (!ata_is_ncq(qc->tf.protocol)) {
+		link->deferred_qc = qc;
+		return 0;
 	}
 
-issue:
-	ata_qc_issue(qc);
+free_qc:
+	/* Force a requeue of the command to defer its execution. */
+	ata_qc_free(qc);
 
-	return 0;
+	return ret;
 }
 
 /**
@@ -4753,7 +4768,7 @@ void ata_scsi_scan_host(struct ata_port *ap, int sync)
 			     "WARNING: synchronous SCSI scan failed without making any progress, switching to async\n");
 	}
 
-	queue_delayed_work(system_long_wq, &ap->hotplug_task,
+	queue_delayed_work(system_dfl_long_wq, &ap->hotplug_task,
 			   round_jiffies_relative(HZ));
 }
 
diff --git a/drivers/ata/pata_arasan_cf.c b/drivers/ata/pata_arasan_cf.c
index b1e281b64f57c..a77fefd320c26 100644
--- a/drivers/ata/pata_arasan_cf.c
+++ b/drivers/ata/pata_arasan_cf.c
@@ -803,16 +803,6 @@ static int arasan_cf_probe(struct platform_device *pdev)
 	irq_handler_t irq_handler = NULL;
 	int ret;
 
-	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -EINVAL;
-
-	if (!devm_request_mem_region(&pdev->dev, res->start, resource_size(res),
-				DRIVER_NAME)) {
-		dev_warn(&pdev->dev, "Failed to get memory region resource\n");
-		return -ENOENT;
-	}
-
 	acdev = devm_kzalloc(&pdev->dev, sizeof(*acdev), GFP_KERNEL);
 	if (!acdev)
 		return -ENOMEM;
@@ -827,22 +817,20 @@ static int arasan_cf_probe(struct platform_device *pdev)
 	 * support only PIO
 	 */
 	ret = platform_get_irq(pdev, 0);
+	if (ret == -EPROBE_DEFER)
+		return ret;
 	if (ret > 0) {
 		acdev->irq = ret;
 		irq_handler = arasan_cf_interrupt;
-	} else	if (ret == -EPROBE_DEFER) {
-		return ret;
 	} else	{
 		quirk |= CF_BROKEN_MWDMA | CF_BROKEN_UDMA;
 	}
 
+	acdev->vbase = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
+	if (IS_ERR(acdev->vbase))
+		return PTR_ERR(acdev->vbase);
+
 	acdev->pbase = res->start;
-	acdev->vbase = devm_ioremap(&pdev->dev, res->start,
-			resource_size(res));
-	if (!acdev->vbase) {
-		dev_warn(&pdev->dev, "ioremap fail\n");
-		return -ENOMEM;
-	}
 
 	acdev->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(acdev->clk)) {
diff --git a/drivers/ata/sata_sil24.c b/drivers/ata/sata_sil24.c
index d642ece9f07a1..57f1081b86db3 100644
--- a/drivers/ata/sata_sil24.c
+++ b/drivers/ata/sata_sil24.c
@@ -789,6 +789,7 @@ static int sil24_qc_defer(struct ata_queued_cmd *qc)
 	struct ata_link *link = qc->dev->link;
 	struct ata_port *ap = link->ap;
 	u8 prot = qc->tf.protocol;
+	int ret;
 
 	/*
 	 * There is a bug in the chip:
@@ -826,7 +827,10 @@ static int sil24_qc_defer(struct ata_queued_cmd *qc)
 		qc->flags |= ATA_QCFLAG_CLEAR_EXCL;
 	}
 
-	return ata_std_qc_defer(qc);
+	ret = ata_std_qc_defer(qc);
+	if (ret == ATA_DEFER_LINK)
+		return ATA_DEFER_LINK_EXCL;
+	return ret;
 }
 
 static enum ata_completion_errors sil24_qc_prep(struct ata_queued_cmd *qc)
diff --git a/drivers/atm/solos-pci.c b/drivers/atm/solos-pci.c
index 24c764664c247..bcb1353877e4d 100644
--- a/drivers/atm/solos-pci.c
+++ b/drivers/atm/solos-pci.c
@@ -1464,8 +1464,8 @@ static void fpga_remove(struct pci_dev *dev)
 }
 
 static const struct pci_device_id fpga_pci_tbl[] = {
-	{ 0x10ee, 0x0300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ 0, }
+	{ PCI_DEVICE(0x10ee, 0x0300) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci,fpga_pci_tbl);
diff --git a/drivers/auxdisplay/Kconfig b/drivers/auxdisplay/Kconfig
index bedc6133f970a..1ea7c039160ce 100644
--- a/drivers/auxdisplay/Kconfig
+++ b/drivers/auxdisplay/Kconfig
@@ -327,7 +327,7 @@ config PANEL_CHANGE_MESSAGE
 	  say 'N' and keep the default message with the version.
 
 config PANEL_BOOT_MESSAGE
-	depends on PANEL_CHANGE_MESSAGE="y"
+	depends on PANEL_CHANGE_MESSAGE
 	string "New initialization message"
 	default ""
 	help
diff --git a/drivers/auxdisplay/line-display.c b/drivers/auxdisplay/line-display.c
index fb6d9294140d9..915eb5cd96b23 100644
--- a/drivers/auxdisplay/line-display.c
+++ b/drivers/auxdisplay/line-display.c
@@ -173,7 +173,7 @@ static int linedisp_display(struct linedisp *linedisp, const char *msg,
 		count = strlen(msg);
 
 	/* if the string ends with a newline, trim it */
-	if (msg[count - 1] == '\n')
+	if (count && msg[count - 1] == '\n')
 		count--;
 
 	if (!count) {
diff --git a/drivers/auxdisplay/max6959.c b/drivers/auxdisplay/max6959.c
index 6bbc8d48fb1b2..3bdef099a2259 100644
--- a/drivers/auxdisplay/max6959.c
+++ b/drivers/auxdisplay/max6959.c
@@ -86,10 +86,7 @@ static const struct linedisp_ops max6959_linedisp_ops = {
 
 static int max6959_enable(struct max6959_priv *priv, bool enable)
 {
-	u8 mask = REG_CONFIGURATION_S_BIT;
-	u8 value = enable ? mask : 0;
-
-	return regmap_update_bits(priv->regmap, REG_CONFIGURATION, mask, value);
+	return regmap_assign_bits(priv->regmap, REG_CONFIGURATION, REG_CONFIGURATION_S_BIT, enable);
 }
 
 static void max6959_power_off(void *priv)
diff --git a/drivers/base/base.h b/drivers/base/base.h
index 30b416588617b..483b99b4fa3d7 100644
--- a/drivers/base/base.h
+++ b/drivers/base/base.h
@@ -86,18 +86,6 @@ struct driver_private {
 };
 #define to_driver(obj) container_of(obj, struct driver_private, kobj)
 
-#ifdef CONFIG_RUST
-/**
- * struct driver_type - Representation of a Rust driver type.
- */
-struct driver_type {
-	/**
-	 * @id: Representation of core::any::TypeId.
-	 */
-	u8 id[16];
-} __packed;
-#endif
-
 /**
  * struct device_private - structure to hold the private to the driver core
  *			   portions of the device structure.
@@ -115,7 +103,6 @@ struct driver_type {
  *			   dev_err_probe() for later retrieval via debugfs
  * @device: pointer back to the struct device that this structure is
  *	    associated with.
- * @driver_type: The type of the bound Rust driver.
  * @dead: This device is currently either in the process of or has been
  *	  removed from the system. Any asynchronous events scheduled for this
  *	  device should exit without taking any action.
@@ -132,9 +119,6 @@ struct device_private {
 	const struct device_driver *async_driver;
 	char *deferred_probe_reason;
 	struct device *device;
-#ifdef CONFIG_RUST
-	struct driver_type driver_type;
-#endif
 	u8 dead:1;
 };
 #define to_device_private_parent(obj)	\
@@ -188,6 +172,20 @@ static inline int driver_match_device(const struct device_driver *drv,
 	return drv->bus->match ? drv->bus->match(dev, drv) : 1;
 }
 
+static inline bool dev_has_sync_state(struct device *dev)
+{
+	struct device_driver *drv;
+
+	if (!dev)
+		return false;
+	drv = READ_ONCE(dev->driver);
+	if (drv && drv->sync_state)
+		return true;
+	if (dev->bus && dev->bus->sync_state)
+		return true;
+	return false;
+}
+
 static inline void dev_sync_state(struct device *dev)
 {
 	if (dev->bus->sync_state)
diff --git a/drivers/base/bus.c b/drivers/base/bus.c
index 8b6722ff8590d..d17bd91490ee4 100644
--- a/drivers/base/bus.c
+++ b/drivers/base/bus.c
@@ -544,10 +544,10 @@ static const struct attribute_group driver_override_dev_group = {
  */
 int bus_add_device(struct device *dev)
 {
-	struct subsys_private *sp = bus_to_subsys(dev->bus);
+	struct subsys_private *sp;
 	int error;
 
-	if (!sp) {
+	if (!dev->bus) {
 		/*
 		 * This is a normal operation for many devices that do not
 		 * have a bus assigned to them, just say that all went
@@ -556,6 +556,13 @@ int bus_add_device(struct device *dev)
 		return 0;
 	}
 
+	sp = bus_to_subsys(dev->bus);
+	if (!sp) {
+		pr_err("%s: cannot add device '%s' to unregistered bus '%s'\n",
+		       __func__, dev_name(dev), dev->bus->name);
+		return -EINVAL;
+	}
+
 	/*
 	 * Reference in sp is now incremented and will be dropped when
 	 * the device is removed from the bus
diff --git a/drivers/base/cacheinfo.c b/drivers/base/cacheinfo.c
index 391ac5e3d2f56..70701d3bc81c6 100644
--- a/drivers/base/cacheinfo.c
+++ b/drivers/base/cacheinfo.c
@@ -17,6 +17,7 @@
 #include <linux/init.h>
 #include <linux/of.h>
 #include <linux/sched.h>
+#include <linux/sched/topology.h>
 #include <linux/slab.h>
 #include <linux/smp.h>
 #include <linux/sysfs.h>
@@ -68,6 +69,24 @@ bool last_level_cache_is_valid(unsigned int cpu)
 
 }
 
+/*
+ * Get the cacheinfo of the LLC associated with @cpu.
+ * Derived from update_per_cpu_data_slice_size_cpu().
+ */
+struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu)
+{
+	struct cacheinfo *llc;
+
+	if (!last_level_cache_is_valid(cpu))
+		return NULL;
+
+	llc = per_cpu_cacheinfo_idx(cpu, cache_leaves(cpu) - 1);
+	if (llc->type != CACHE_TYPE_DATA && llc->type != CACHE_TYPE_UNIFIED)
+		return NULL;
+
+	return llc;
+}
+
 bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y)
 {
 	struct cacheinfo *llc_x, *llc_y;
@@ -1018,6 +1037,7 @@ static int cacheinfo_cpu_online(unsigned int cpu)
 		goto err;
 	if (cpu_map_shared_cache(true, cpu, &cpu_map))
 		update_per_cpu_data_slice_size(true, cpu, cpu_map);
+	sched_update_llc_bytes(cpu);
 	return 0;
 err:
 	free_cache_attributes(cpu);
@@ -1036,6 +1056,9 @@ static int cacheinfo_cpu_pre_down(unsigned int cpu)
 	free_cache_attributes(cpu);
 	if (nr_shared > 1)
 		update_per_cpu_data_slice_size(false, cpu, cpu_map);
+
+	sched_update_llc_bytes(cpu);
+
 	return 0;
 }
 
diff --git a/drivers/base/core.c b/drivers/base/core.c
index bd2ddf2aab505..d49420e066de6 100644
--- a/drivers/base/core.c
+++ b/drivers/base/core.c
@@ -1011,7 +1011,7 @@ static void device_links_missing_supplier(struct device *dev)
 
 static bool dev_is_best_effort(struct device *dev)
 {
-	return (fw_devlink_best_effort && dev->can_match) ||
+	return (fw_devlink_best_effort && dev_can_match(dev)) ||
 		(dev->fwnode && fwnode_test_flag(dev->fwnode, FWNODE_FLAG_BEST_EFFORT));
 }
 
@@ -1079,7 +1079,7 @@ int device_links_check_suppliers(struct device *dev)
 
 			if (dev_is_best_effort(dev) &&
 			    device_link_test(link, DL_FLAG_INFERRED) &&
-			    !link->supplier->can_match) {
+			    !dev_can_match(link->supplier)) {
 				ret = -EAGAIN;
 				continue;
 			}
@@ -1123,7 +1123,7 @@ static void __device_links_queue_sync_state(struct device *dev,
 
 	if (!dev_has_sync_state(dev))
 		return;
-	if (dev->state_synced)
+	if (dev_state_synced(dev))
 		return;
 
 	list_for_each_entry(link, &dev->links.consumers, s_node) {
@@ -1138,7 +1138,7 @@ static void __device_links_queue_sync_state(struct device *dev,
 	 * than once. This can happen if new consumers get added to the device
 	 * and probed before the list is flushed.
 	 */
-	dev->state_synced = true;
+	dev_set_state_synced(dev);
 
 	if (WARN_ON(!list_empty(&dev->links.defer_sync)))
 		return;
@@ -1370,7 +1370,7 @@ void device_links_driver_bound(struct device *dev)
 		} else if (dev_is_best_effort(dev) &&
 			   device_link_test(link, DL_FLAG_INFERRED) &&
 			   link->status != DL_STATE_CONSUMER_PROBE &&
-			   !link->supplier->can_match) {
+			   !dev_can_match(link->supplier)) {
 			/*
 			 * When dev_is_best_effort() is true, we ignore device
 			 * links to suppliers that don't have a driver.  If the
@@ -1758,7 +1758,7 @@ static int fw_devlink_no_driver(struct device *dev, void *data)
 {
 	struct device_link *link = to_devlink(dev);
 
-	if (!link->supplier->can_match)
+	if (!dev_can_match(link->supplier))
 		fw_devlink_relax_link(link);
 
 	return 0;
@@ -1779,7 +1779,7 @@ static int fw_devlink_dev_sync_state(struct device *dev, void *data)
 	struct device *sup = link->supplier;
 
 	if (!device_link_test(link, DL_FLAG_MANAGED) ||
-	    link->status == DL_STATE_ACTIVE || sup->state_synced ||
+	    link->status == DL_STATE_ACTIVE || dev_state_synced(sup) ||
 	    !dev_has_sync_state(sup))
 		return 0;
 
@@ -1793,7 +1793,7 @@ static int fw_devlink_dev_sync_state(struct device *dev, void *data)
 		return 0;
 
 	dev_warn(sup, "Timed out. Forcing sync_state()\n");
-	sup->state_synced = true;
+	dev_set_state_synced(sup);
 	get_device(sup);
 	list_add_tail(&sup->links.defer_sync, data);
 
@@ -2787,7 +2787,7 @@ static ssize_t online_show(struct device *dev, struct device_attribute *attr,
 	bool val;
 
 	device_lock(dev);
-	val = !dev->offline;
+	val = !dev_offline(dev);
 	device_unlock(dev);
 	return sysfs_emit(buf, "%u\n", val);
 }
@@ -2913,7 +2913,7 @@ static int device_add_attrs(struct device *dev)
 	if (error)
 		goto err_remove_type_groups;
 
-	if (device_supports_offline(dev) && !dev->offline_disabled) {
+	if (device_supports_offline(dev) && !dev_offline_disabled(dev)) {
 		error = device_create_file(dev, &dev_attr_online);
 		if (error)
 			goto err_remove_dev_groups;
@@ -3170,11 +3170,7 @@ void device_initialize(struct device *dev)
 	INIT_LIST_HEAD(&dev->links.suppliers);
 	INIT_LIST_HEAD(&dev->links.defer_sync);
 	dev->links.status = DL_DEV_NO_DRIVER;
-#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
-    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
-    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
-	dev->dma_coherent = dma_default_coherent;
-#endif
+	dev_assign_dma_coherent(dev, dma_default_coherent);
 	swiotlb_dev_init(dev);
 }
 EXPORT_SYMBOL_GPL(device_initialize);
@@ -3710,7 +3706,7 @@ int device_add(struct device *dev)
 	 * match with any driver, don't block its consumers from probing in
 	 * case the consumer device is able to operate without this supplier.
 	 */
-	if (dev->fwnode && fw_devlink_drv_reg_done && !dev->can_match)
+	if (dev->fwnode && fw_devlink_drv_reg_done && !dev_can_match(dev))
 		fw_devlink_unblock_consumers(dev);
 
 	if (parent)
@@ -4180,7 +4176,7 @@ static int device_check_offline(struct device *dev, void *not_used)
 	if (ret)
 		return ret;
 
-	return device_supports_offline(dev) && !dev->offline ? -EBUSY : 0;
+	return device_supports_offline(dev) && !dev_offline(dev) ? -EBUSY : 0;
 }
 
 /**
@@ -4198,7 +4194,7 @@ int device_offline(struct device *dev)
 {
 	int ret;
 
-	if (dev->offline_disabled)
+	if (dev_offline_disabled(dev))
 		return -EPERM;
 
 	ret = device_for_each_child(dev, NULL, device_check_offline);
@@ -4207,13 +4203,13 @@ int device_offline(struct device *dev)
 
 	device_lock(dev);
 	if (device_supports_offline(dev)) {
-		if (dev->offline) {
+		if (dev_offline(dev)) {
 			ret = 1;
 		} else {
 			ret = dev->bus->offline(dev);
 			if (!ret) {
 				kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
-				dev->offline = true;
+				dev_set_offline(dev);
 			}
 		}
 	}
@@ -4238,11 +4234,11 @@ int device_online(struct device *dev)
 
 	device_lock(dev);
 	if (device_supports_offline(dev)) {
-		if (dev->offline) {
+		if (dev_offline(dev)) {
 			ret = dev->bus->online(dev);
 			if (!ret) {
 				kobject_uevent(&dev->kobj, KOBJ_ONLINE);
-				dev->offline = false;
+				dev_clear_offline(dev);
 			}
 		} else {
 			ret = 1;
@@ -4716,7 +4712,7 @@ static int device_attrs_change_owner(struct device *dev, kuid_t kuid,
 	if (error)
 		return error;
 
-	if (device_supports_offline(dev) && !dev->offline_disabled) {
+	if (device_supports_offline(dev) && !dev_offline_disabled(dev)) {
 		/* Change online device attributes of @dev to @kuid/@kgid. */
 		error = sysfs_file_change_owner(kobj, dev_attr_online.attr.name,
 						kuid, kgid);
@@ -5283,7 +5279,7 @@ void device_set_of_node_from_dev(struct device *dev, const struct device *dev2)
 {
 	of_node_put(dev->of_node);
 	dev->of_node = of_node_get(dev2->of_node);
-	dev->of_node_reused = true;
+	dev_set_of_node_reused(dev);
 }
 EXPORT_SYMBOL_GPL(device_set_of_node_from_dev);
 
diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c
index 875abdc9942e1..19d288a3c80cd 100644
--- a/drivers/base/cpu.c
+++ b/drivers/base/cpu.c
@@ -422,8 +422,8 @@ int register_cpu(struct cpu *cpu, int num)
 	cpu->dev.id = num;
 	cpu->dev.bus = &cpu_subsys;
 	cpu->dev.release = cpu_device_release;
-	cpu->dev.offline_disabled = !cpu->hotpluggable;
-	cpu->dev.offline = !cpu_online(num);
+	dev_assign_offline_disabled(&cpu->dev, !cpu->hotpluggable);
+	dev_assign_offline(&cpu->dev, !cpu_online(num));
 	cpu->dev.of_node = of_get_cpu_node(num, NULL);
 	cpu->dev.groups = common_cpu_attr_groups;
 	if (cpu->hotpluggable)
diff --git a/drivers/base/dd.c b/drivers/base/dd.c
index 1dc1e3528043c..5799a60fd0585 100644
--- a/drivers/base/dd.c
+++ b/drivers/base/dd.c
@@ -132,7 +132,7 @@ static DECLARE_WORK(deferred_probe_work, deferred_probe_work_func);
 
 void driver_deferred_probe_add(struct device *dev)
 {
-	if (!dev->can_match)
+	if (!dev_can_match(dev))
 		return;
 
 	mutex_lock(&deferred_probe_mutex);
@@ -569,12 +569,10 @@ static ssize_t state_synced_store(struct device *dev,
 		return -EINVAL;
 
 	device_lock(dev);
-	if (!dev->state_synced) {
-		dev->state_synced = true;
+	if (!dev_test_and_set_state_synced(dev))
 		dev_sync_state(dev);
-	} else {
+	else
 		ret = -EINVAL;
-	}
 	device_unlock(dev);
 
 	return ret ? ret : count;
@@ -586,7 +584,7 @@ static ssize_t state_synced_show(struct device *dev,
 	bool val;
 
 	device_lock(dev);
-	val = dev->state_synced;
+	val = dev_state_synced(dev);
 	device_unlock(dev);
 
 	return sysfs_emit(buf, "%u\n", val);
@@ -849,14 +847,14 @@ static int __driver_probe_device(const struct device_driver *drv, struct device
 		return dev_err_probe(dev, -EPROBE_DEFER, "Device not ready to probe\n");
 
 	/*
-	 * Set can_match = true after calling dev_ready_to_probe(), so
+	 * Call dev_set_can_match() after calling dev_ready_to_probe(), so
 	 * driver_deferred_probe_add() won't actually add the device to the
 	 * deferred probe list when dev_ready_to_probe() returns false.
 	 *
 	 * When dev_ready_to_probe() returns false, it means that device_add()
 	 * will do another probe() attempt for us.
 	 */
-	dev->can_match = true;
+	dev_set_can_match(dev);
 	dev_dbg(dev, "bus: '%s': %s: matched device with driver %s\n",
 		drv->bus->name, __func__, drv->name);
 
@@ -1002,7 +1000,7 @@ static int __device_attach_driver(struct device_driver *drv, void *_data)
 		return 0;
 	} else if (ret == -EPROBE_DEFER) {
 		dev_dbg(dev, "Device match requests probe deferral\n");
-		dev->can_match = true;
+		dev_set_can_match(dev);
 		driver_deferred_probe_add(dev);
 		/*
 		 * Device can't match with a driver right now, so don't attempt
@@ -1254,7 +1252,7 @@ static int __driver_attach(struct device *dev, void *data)
 		return 0;
 	} else if (ret == -EPROBE_DEFER) {
 		dev_dbg(dev, "Device match requests probe deferral\n");
-		dev->can_match = true;
+		dev_set_can_match(dev);
 		driver_deferred_probe_add(dev);
 		/*
 		 * Driver could not match with device, but may match with
diff --git a/drivers/base/faux.c b/drivers/base/faux.c
index fb3e42f213624..a8329f88222e3 100644
--- a/drivers/base/faux.c
+++ b/drivers/base/faux.c
@@ -133,6 +133,9 @@ struct faux_device *faux_device_create_with_groups(const char *name,
 	struct device *dev;
 	int ret;
 
+	if (!faux_bus_root)
+		return NULL;
+
 	faux_obj = kzalloc_obj(*faux_obj);
 	if (!faux_obj)
 		return NULL;
@@ -232,34 +235,29 @@ EXPORT_SYMBOL_GPL(faux_device_destroy);
 
 int __init faux_bus_init(void)
 {
+	struct device *root;
 	int ret;
 
-	faux_bus_root = kzalloc_obj(*faux_bus_root);
-	if (!faux_bus_root)
-		return -ENOMEM;
-
-	dev_set_name(faux_bus_root, "faux");
-
-	ret = device_register(faux_bus_root);
-	if (ret) {
-		put_device(faux_bus_root);
-		return ret;
-	}
+	root = root_device_register("faux");
+	if (IS_ERR(root))
+		return PTR_ERR(root);
 
 	ret = bus_register(&faux_bus_type);
 	if (ret)
-		goto error_bus;
+		goto err_deregister_root;
 
 	ret = driver_register(&faux_driver);
 	if (ret)
-		goto error_driver;
+		goto err_deregister_bus;
 
-	return ret;
+	faux_bus_root = root;
+
+	return 0;
 
-error_driver:
+err_deregister_bus:
 	bus_unregister(&faux_bus_type);
+err_deregister_root:
+	root_device_unregister(root);
 
-error_bus:
-	device_unregister(faux_bus_root);
 	return ret;
 }
diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c
index a11b30dda23be..78c5f05a2ec10 100644
--- a/drivers/base/firmware_loader/main.c
+++ b/drivers/base/firmware_loader/main.c
@@ -1132,6 +1132,7 @@ EXPORT_SYMBOL(release_firmware);
 /* Async support */
 struct firmware_work {
 	struct work_struct work;
+	struct list_head list;
 	struct module *module;
 	const char *name;
 	struct device *device;
@@ -1140,6 +1141,17 @@ struct firmware_work {
 	u32 opt_flags;
 };
 
+static LIST_HEAD(firmware_work_list);
+static DEFINE_SPINLOCK(firmware_work_lock);
+
+static void firmware_work_free(struct firmware_work *fw_work)
+{
+	put_device(fw_work->device); /* taken in request_firmware_nowait() */
+	module_put(fw_work->module);
+	kfree_const(fw_work->name);
+	kfree(fw_work);
+}
+
 static void request_firmware_work_func(struct work_struct *work)
 {
 	struct firmware_work *fw_work;
@@ -1150,11 +1162,15 @@ static void request_firmware_work_func(struct work_struct *work)
 	_request_firmware(&fw, fw_work->name, fw_work->device, NULL, 0, 0,
 			  fw_work->opt_flags);
 	fw_work->cont(fw, fw_work->context);
-	put_device(fw_work->device); /* taken in request_firmware_nowait() */
 
-	module_put(fw_work->module);
-	kfree_const(fw_work->name);
-	kfree(fw_work);
+	spin_lock_irq(&firmware_work_lock);
+	if (!list_empty(&fw_work->list)) {
+		list_del_init(&fw_work->list);
+		spin_unlock_irq(&firmware_work_lock);
+		firmware_work_free(fw_work);
+		return;
+	}
+	spin_unlock_irq(&firmware_work_lock);
 }
 
 
@@ -1164,6 +1180,7 @@ static int _request_firmware_nowait(
 	void (*cont)(const struct firmware *fw, void *context), bool nowarn)
 {
 	struct firmware_work *fw_work;
+	unsigned long flags;
 
 	fw_work = kzalloc_obj(struct firmware_work, gfp);
 	if (!fw_work)
@@ -1196,7 +1213,12 @@ static int _request_firmware_nowait(
 
 	get_device(fw_work->device);
 	INIT_WORK(&fw_work->work, request_firmware_work_func);
+
+	spin_lock_irqsave(&firmware_work_lock, flags);
+	list_add_tail(&fw_work->list, &firmware_work_list);
 	schedule_work(&fw_work->work);
+	spin_unlock_irqrestore(&firmware_work_lock, flags);
+
 	return 0;
 }
 
@@ -1259,6 +1281,44 @@ int firmware_request_nowait_nowarn(
 }
 EXPORT_SYMBOL_GPL(firmware_request_nowait_nowarn);
 
+/**
+ * request_firmware_nowait_cancel() - cancel an async firmware request
+ * @device: device for which the firmware is being loaded
+ * @context: context passed to request_firmware_nowait()
+ * @cont: callback passed to request_firmware_nowait()
+ *
+ * Cancel a pending request_firmware_nowait() request for @device, @context
+ * and @cont. If the associated work has already started, this function waits
+ * until the callback has returned. If the callback has already completed, this
+ * function does nothing.
+ *
+ * This function may sleep.
+ */
+void request_firmware_nowait_cancel(struct device *device, void *context,
+				    void (*cont)(const struct firmware *fw,
+						 void *context))
+{
+	struct firmware_work *fw_work = NULL;
+	struct firmware_work *tmp;
+
+	spin_lock_irq(&firmware_work_lock);
+	list_for_each_entry_reverse(tmp, &firmware_work_list, list) {
+		if (tmp->device == device && tmp->context == context &&
+		    tmp->cont == cont) {
+			fw_work = tmp;
+			list_del_init(&fw_work->list);
+			break;
+		}
+	}
+	spin_unlock_irq(&firmware_work_lock);
+
+	if (!fw_work)
+		return;
+	cancel_work_sync(&fw_work->work);
+	firmware_work_free(fw_work);
+}
+EXPORT_SYMBOL_GPL(request_firmware_nowait_cancel);
+
 #ifdef CONFIG_FW_CACHE
 static ASYNC_DOMAIN_EXCLUSIVE(fw_cache_domain);
 
diff --git a/drivers/base/firmware_loader/sysfs_upload.c b/drivers/base/firmware_loader/sysfs_upload.c
index f59a7856934ce..efc33294212fd 100644
--- a/drivers/base/firmware_loader/sysfs_upload.c
+++ b/drivers/base/firmware_loader/sysfs_upload.c
@@ -343,7 +343,6 @@ firmware_upload_register(struct module *module, struct device *parent,
 		goto free_fw_upload_priv;
 	}
 	fw_upload->priv = fw_sysfs;
-	fw_sysfs->fw_upload_priv = fw_upload_priv;
 	fw_dev = &fw_sysfs->dev;
 
 	ret = alloc_lookup_fw_priv(name, &fw_cache, &fw_priv,  NULL, 0, 0,
@@ -351,10 +350,12 @@ firmware_upload_register(struct module *module, struct device *parent,
 	if (ret != 0) {
 		if (ret > 0)
 			ret = -EINVAL;
-		goto free_fw_sysfs;
+		put_device(fw_dev);
+		goto free_fw_upload_priv;
 	}
 	fw_priv->is_paged_buf = true;
 	fw_sysfs->fw_priv = fw_priv;
+	fw_sysfs->fw_upload_priv = fw_upload_priv;
 
 	ret = device_add(fw_dev);
 	if (ret) {
@@ -365,9 +366,6 @@ firmware_upload_register(struct module *module, struct device *parent,
 
 	return fw_upload;
 
-free_fw_sysfs:
-	kfree(fw_sysfs);
-
 free_fw_upload_priv:
 	kfree(fw_upload_priv);
 
diff --git a/drivers/base/isa.c b/drivers/base/isa.c
index fd076cc63cb65..5887e4211f80b 100644
--- a/drivers/base/isa.c
+++ b/drivers/base/isa.c
@@ -11,9 +11,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/isa.h>
 
-static struct device isa_bus = {
-	.init_name	= "isa"
-};
+static struct device *isa_bus;
 
 struct isa_dev {
 	struct device dev;
@@ -131,7 +129,7 @@ int isa_register_driver(struct isa_driver *isa_driver, unsigned int ndev)
 			break;
 		}
 
-		isa_dev->dev.parent	= &isa_bus;
+		isa_dev->dev.parent	= isa_bus;
 		isa_dev->dev.bus	= &isa_bus_type;
 
 		dev_set_name(&isa_dev->dev, "%s.%u",
@@ -169,9 +167,11 @@ static int __init isa_bus_init(void)
 
 	error = bus_register(&isa_bus_type);
 	if (!error) {
-		error = device_register(&isa_bus);
-		if (error)
+		isa_bus = root_device_register("isa");
+		if (IS_ERR(isa_bus)) {
+			error = PTR_ERR(isa_bus);
 			bus_unregister(&isa_bus_type);
+		}
 	}
 	return error;
 }
diff --git a/drivers/base/memory.c b/drivers/base/memory.c
index f806a683b7672..5b5d41089e816 100644
--- a/drivers/base/memory.c
+++ b/drivers/base/memory.c
@@ -649,7 +649,7 @@ int __weak arch_get_memory_phys_device(unsigned long start_pfn)
  *
  * Called under device_hotplug_lock.
  */
-struct memory_block *find_memory_block_by_id(unsigned long block_id)
+struct memory_block *memory_block_get(unsigned long block_id)
 {
 	struct memory_block *mem;
 
@@ -659,16 +659,6 @@ struct memory_block *find_memory_block_by_id(unsigned long block_id)
 	return mem;
 }
 
-/*
- * Called under device_hotplug_lock.
- */
-struct memory_block *find_memory_block(unsigned long section_nr)
-{
-	unsigned long block_id = memory_block_id(section_nr);
-
-	return find_memory_block_by_id(block_id);
-}
-
 static struct attribute *memory_memblk_attrs[] = {
 	&dev_attr_phys_index.attr,
 	&dev_attr_state.attr,
@@ -697,11 +687,11 @@ static int __add_memory_block(struct memory_block *memory)
 	memory->dev.id = memory->start_section_nr / sections_per_block;
 	memory->dev.release = memory_block_release;
 	memory->dev.groups = memory_memblk_attr_groups;
-	memory->dev.offline = memory->state == MEM_OFFLINE;
+	dev_assign_offline(&memory->dev, memory->state == MEM_OFFLINE);
 
 	ret = device_register(&memory->dev);
 	if (ret) {
-		put_device(&memory->dev);
+		memory_block_put(memory);
 		return ret;
 	}
 	ret = xa_err(xa_store(&memory_blocks, memory->dev.id, memory,
@@ -795,9 +785,9 @@ static int add_memory_block(unsigned long block_id, int nid, unsigned long state
 	struct memory_block *mem;
 	int ret = 0;
 
-	mem = find_memory_block_by_id(block_id);
+	mem = memory_block_get(block_id);
 	if (mem) {
-		put_device(&mem->dev);
+		memory_block_put(mem);
 		return -EEXIST;
 	}
 	mem = kzalloc_obj(*mem);
@@ -845,8 +835,8 @@ static void remove_memory_block(struct memory_block *memory)
 		memory->group = NULL;
 	}
 
-	/* drop the ref. we got via find_memory_block() */
-	put_device(&memory->dev);
+	/* drop the ref. we got via memory_block_get() */
+	memory_block_put(memory);
 	device_unregister(&memory->dev);
 }
 
@@ -880,7 +870,7 @@ int create_memory_block_devices(unsigned long start, unsigned long size,
 		end_block_id = block_id;
 		for (block_id = start_block_id; block_id != end_block_id;
 		     block_id++) {
-			mem = find_memory_block_by_id(block_id);
+			mem = memory_block_get(block_id);
 			if (WARN_ON_ONCE(!mem))
 				continue;
 			remove_memory_block(mem);
@@ -908,7 +898,7 @@ void remove_memory_block_devices(unsigned long start, unsigned long size)
 		return;
 
 	for (block_id = start_block_id; block_id != end_block_id; block_id++) {
-		mem = find_memory_block_by_id(block_id);
+		mem = memory_block_get(block_id);
 		if (WARN_ON_ONCE(!mem))
 			continue;
 		num_poisoned_pages_sub(-1UL, memblk_nr_poison(mem));
@@ -1015,12 +1005,12 @@ int walk_memory_blocks(unsigned long start, unsigned long size,
 		return 0;
 
 	for (block_id = start_block_id; block_id <= end_block_id; block_id++) {
-		mem = find_memory_block_by_id(block_id);
+		mem = memory_block_get(block_id);
 		if (!mem)
 			continue;
 
 		ret = func(mem, arg);
-		put_device(&mem->dev);
+		memory_block_put(mem);
 		if (ret)
 			break;
 	}
@@ -1228,19 +1218,23 @@ int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func,
 void memblk_nr_poison_inc(unsigned long pfn)
 {
 	const unsigned long block_id = pfn_to_block_id(pfn);
-	struct memory_block *mem = find_memory_block_by_id(block_id);
+	struct memory_block *mem = memory_block_get(block_id);
 
-	if (mem)
+	if (mem) {
 		atomic_long_inc(&mem->nr_hwpoison);
+		memory_block_put(mem);
+	}
 }
 
 void memblk_nr_poison_sub(unsigned long pfn, long i)
 {
 	const unsigned long block_id = pfn_to_block_id(pfn);
-	struct memory_block *mem = find_memory_block_by_id(block_id);
+	struct memory_block *mem = memory_block_get(block_id);
 
-	if (mem)
+	if (mem) {
 		atomic_long_sub(i, &mem->nr_hwpoison);
+		memory_block_put(mem);
+	}
 }
 
 static unsigned long memblk_nr_poison(struct memory_block *mem)
diff --git a/drivers/base/node.c b/drivers/base/node.c
index 126f66aa2c3e8..3da91929ad4e3 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -523,6 +523,7 @@ static ssize_t node_read_meminfo(struct device *dev,
 #ifdef CONFIG_UNACCEPTED_MEMORY
 			     "Node %d Unaccepted:     %8lu kB\n"
 #endif
+			     "Node %d Balloon:        %8lu kB\n"
 			     "Node %d GPUActive:      %8lu kB\n"
 			     "Node %d GPUReclaim:     %8lu kB\n"
 			     ,
@@ -559,6 +560,7 @@ static ssize_t node_read_meminfo(struct device *dev,
 			     nid, K(sum_zone_node_page_state(nid, NR_UNACCEPTED))
 #endif
 			     ,
+			     nid, K(node_page_state(pgdat, NR_BALLOON_PAGES)),
 			     nid, K(node_page_state(pgdat, NR_GPU_ACTIVE)),
 			     nid, K(node_page_state(pgdat, NR_GPU_RECLAIM))
 			    );
@@ -847,13 +849,13 @@ static void register_memory_blocks_under_nodes(void)
 		for (block_id = start_block_id; block_id <= end_block_id; block_id++) {
 			struct memory_block *mem;
 
-			mem = find_memory_block_by_id(block_id);
+			mem = memory_block_get(block_id);
 			if (!mem)
 				continue;
 
 			memory_block_add_nid_early(mem, nid);
 			do_register_memory_block_under_node(nid, mem);
-			put_device(&mem->dev);
+			memory_block_put(mem);
 		}
 
 	}
diff --git a/drivers/base/pinctrl.c b/drivers/base/pinctrl.c
index 6e250272c8439..0bbc83231234e 100644
--- a/drivers/base/pinctrl.c
+++ b/drivers/base/pinctrl.c
@@ -24,7 +24,7 @@ int pinctrl_bind_pins(struct device *dev)
 {
 	int ret;
 
-	if (dev->of_node_reused)
+	if (dev_of_node_reused(dev))
 		return 0;
 
 	dev->pins = devm_kzalloc(dev, sizeof(*(dev->pins)), GFP_KERNEL);
diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index 70db08f3ac6fa..69eed058eb208 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -61,10 +61,6 @@ static const struct msi_domain_template platform_msi_template = {
  * parent. The parent domain sets up the new domain. The domain has
  * a fixed size of @nvec. The domain is managed by devres and will
  * be removed when the device is removed.
- *
- * Note: For migration purposes this falls back to the original platform_msi code
- *	 up to the point where all platforms have been converted to the MSI
- *	 parent model.
  */
 int platform_device_msi_init_and_alloc_irqs(struct device *dev, unsigned int nvec,
 					    irq_write_msi_msg_t write_msi_msg)
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index 75b4698d0e582..4824d9a2edbe9 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -606,6 +606,12 @@ static void platform_device_release(struct device *dev)
 	kfree(pa);
 }
 
+static void platform_device_release_full(struct device *dev)
+{
+	device_remove_software_node(dev);
+	platform_device_release(dev);
+}
+
 /**
  * platform_device_alloc - create a platform device
  * @name: base name of the device we're adding
@@ -848,7 +854,13 @@ struct platform_device *platform_device_register_full(const struct platform_devi
 	int ret;
 	struct platform_device *pdev;
 
-	if (pdevinfo->swnode && pdevinfo->properties)
+	/*
+	 * Only one software node per device is allowed. Make sure we don't
+	 * accept or create two.
+	 */
+	if ((pdevinfo->swnode && pdevinfo->properties) ||
+	    (pdevinfo->swnode && is_software_node(pdevinfo->fwnode)) ||
+	    (pdevinfo->properties && is_software_node(pdevinfo->fwnode)))
 		return ERR_PTR(-EINVAL);
 
 	pdev = platform_device_alloc(pdevinfo->name, pdevinfo->id);
@@ -858,7 +870,7 @@ struct platform_device *platform_device_register_full(const struct platform_devi
 	pdev->dev.parent = pdevinfo->parent;
 	pdev->dev.fwnode = pdevinfo->fwnode;
 	pdev->dev.of_node = of_node_get(to_of_node(pdev->dev.fwnode));
-	pdev->dev.of_node_reused = pdevinfo->of_node_reused;
+	dev_assign_of_node_reused(&pdev->dev, pdevinfo->of_node_reused);
 
 	if (pdevinfo->dma_mask) {
 		pdev->platform_dma_mask = pdevinfo->dma_mask;
@@ -878,6 +890,8 @@ struct platform_device *platform_device_register_full(const struct platform_devi
 		ret = device_add_software_node(&pdev->dev, pdevinfo->swnode);
 		if (ret)
 			goto err;
+
+		pdev->dev.release = platform_device_release_full;
 	} else if (pdevinfo->properties) {
 		ret = device_create_managed_software_node(&pdev->dev,
 							  pdevinfo->properties, NULL);
diff --git a/drivers/base/power/power.h b/drivers/base/power/power.h
index 922ed457db191..8823aceeac8b5 100644
--- a/drivers/base/power/power.h
+++ b/drivers/base/power/power.h
@@ -168,3 +168,10 @@ static inline void device_pm_init(struct device *dev)
 	device_pm_sleep_init(dev);
 	pm_runtime_init(dev);
 }
+
+#ifdef CONFIG_BPF_SYSCALL
+struct bpf_ws_lock { };
+struct bpf_ws_lock *bpf_wakeup_sources_read_lock(void);
+void bpf_wakeup_sources_read_unlock(struct bpf_ws_lock *lock);
+void *bpf_wakeup_sources_get_head(void);
+#endif
diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c
index b8e48a023bf0c..80b497de2deb0 100644
--- a/drivers/base/power/wakeup.c
+++ b/drivers/base/power/wakeup.c
@@ -1168,11 +1168,78 @@ static const struct file_operations wakeup_sources_stats_fops = {
 	.release = seq_release_private,
 };
 
-static int __init wakeup_sources_debugfs_init(void)
+#ifdef CONFIG_BPF_SYSCALL
+#include <linux/btf.h>
+
+__bpf_kfunc_start_defs();
+
+/**
+ * bpf_wakeup_sources_read_lock - Acquire the SRCU lock for wakeup sources
+ *
+ * The underlying SRCU lock returns an integer index. However, the BPF verifier
+ * requires a pointer (PTR_TO_BTF_ID) to strictly track the state of acquired
+ * resources using KF_ACQUIRE and KF_RELEASE semantics. We use an opaque
+ * structure pointer (struct bpf_ws_lock *) to satisfy the verifier while
+ * safely encoding the integer index within the pointer address itself.
+ *
+ * Return: An opaque pointer encoding the SRCU lock index + 1 (to avoid NULL).
+ */
+__bpf_kfunc struct bpf_ws_lock *bpf_wakeup_sources_read_lock(void)
+{
+	return (struct bpf_ws_lock *)(long)(wakeup_sources_read_lock() + 1);
+}
+
+/**
+ * bpf_wakeup_sources_read_unlock - Release the SRCU lock for wakeup sources
+ * @lock: The opaque pointer returned by bpf_wakeup_sources_read_lock()
+ *
+ * The BPF verifier guarantees that @lock is a valid, unreleased pointer from
+ * the acquire function. We decode the pointer back into the integer SRCU index
+ * by subtracting 1 and release the lock.
+ */
+__bpf_kfunc void bpf_wakeup_sources_read_unlock(struct bpf_ws_lock *lock)
+{
+	wakeup_sources_read_unlock((int)(long)lock - 1);
+}
+
+/**
+ * bpf_wakeup_sources_get_head - Get the head of the wakeup sources list
+ *
+ * Return: The head of the wakeup sources list.
+ */
+__bpf_kfunc void *bpf_wakeup_sources_get_head(void)
+{
+	return &wakeup_sources;
+}
+
+__bpf_kfunc_end_defs();
+
+BTF_KFUNCS_START(wakeup_source_kfunc_ids)
+BTF_ID_FLAGS(func, bpf_wakeup_sources_read_lock, KF_ACQUIRE)
+BTF_ID_FLAGS(func, bpf_wakeup_sources_read_unlock, KF_RELEASE)
+BTF_ID_FLAGS(func, bpf_wakeup_sources_get_head)
+BTF_KFUNCS_END(wakeup_source_kfunc_ids)
+
+static const struct btf_kfunc_id_set wakeup_source_kfunc_set = {
+	.set   = &wakeup_source_kfunc_ids,
+};
+
+static void __init wakeup_sources_bpf_init(void)
+{
+	if (register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &wakeup_source_kfunc_set))
+		pm_pr_dbg("Wakeup: failed to register BTF kfuncs\n");
+}
+#else
+static inline void wakeup_sources_bpf_init(void) {}
+#endif /* CONFIG_BPF_SYSCALL */
+
+static int __init wakeup_sources_init(void)
 {
 	debugfs_create_file("wakeup_sources", 0444, NULL, NULL,
 			    &wakeup_sources_stats_fops);
+	wakeup_sources_bpf_init();
+
 	return 0;
 }
 
-postcore_initcall(wakeup_sources_debugfs_init);
+postcore_initcall(wakeup_sources_init);
diff --git a/drivers/base/regmap/regmap-i2c.c b/drivers/base/regmap/regmap-i2c.c
index 31e30dfced192..51a04961faf7b 100644
--- a/drivers/base/regmap/regmap-i2c.c
+++ b/drivers/base/regmap/regmap-i2c.c
@@ -337,7 +337,7 @@ static int regmap_smbus_word_write_reg16(void *context, const void *data,
 	val = ((u8 *)data)[2];
 
 	return i2c_smbus_write_word_data(i2c, addr_hi,
-					 cpu_to_le16(((u16)val << 8) | addr_lo));
+					 ((u16)val << 8) | addr_lo);
 }
 
 static const struct regmap_bus regmap_smbus_byte_word_reg16 = {
diff --git a/drivers/base/swnode.c b/drivers/base/swnode.c
index a19f8f722bc8e..869228a65cb36 100644
--- a/drivers/base/swnode.c
+++ b/drivers/base/swnode.c
@@ -374,20 +374,28 @@ EXPORT_SYMBOL_GPL(property_entries_free);
 /* -------------------------------------------------------------------------- */
 /* fwnode operations */
 
-static struct fwnode_handle *software_node_get(struct fwnode_handle *fwnode)
+static struct swnode *swnode_get(struct swnode *swnode)
 {
-	struct swnode *swnode = to_swnode(fwnode);
-
 	kobject_get(&swnode->kobj);
 
+	return swnode;
+}
+
+static void swnode_put(struct swnode *swnode)
+{
+	kobject_put(&swnode->kobj);
+}
+
+static struct fwnode_handle *software_node_get(struct fwnode_handle *fwnode)
+{
+	struct swnode *swnode = swnode_get(to_swnode(fwnode));
+
 	return &swnode->fwnode;
 }
 
 static void software_node_put(struct fwnode_handle *fwnode)
 {
-	struct swnode *swnode = to_swnode(fwnode);
-
-	kobject_put(&swnode->kobj);
+	swnode_put(to_swnode(fwnode));
 }
 
 static bool software_node_property_present(const struct fwnode_handle *fwnode,
@@ -493,7 +501,7 @@ software_node_get_named_child_node(const struct fwnode_handle *fwnode,
 
 	list_for_each_entry(child, &swnode->children, entry) {
 		if (!strcmp(childname, kobject_name(&child->kobj))) {
-			kobject_get(&child->kobj);
+			swnode_get(child);
 			return &child->fwnode;
 		}
 	}
@@ -737,7 +745,7 @@ software_node_find_by_name(const struct software_node *parent, const char *name)
 		swnode = kobj_to_swnode(k);
 		if (parent == swnode->node->parent && swnode->node->name &&
 		    !strcmp(name, swnode->node->name)) {
-			kobject_get(&swnode->kobj);
+			swnode_get(swnode);
 			break;
 		}
 		swnode = NULL;
@@ -835,13 +843,13 @@ swnode_register(const struct software_node *node, struct swnode *parent,
 					   parent ? &parent->kobj : NULL,
 					   "node%d", swnode->id);
 	if (ret) {
-		kobject_put(&swnode->kobj);
+		swnode_put(swnode);
 		return ERR_PTR(ret);
 	}
 
 	/*
 	 * Assign the flag only in the successful case, so
-	 * the above kobject_put() won't mess up with properties.
+	 * the above swnode_put() won't mess up with properties.
 	 */
 	swnode->allocated = allocated;
 
@@ -978,7 +986,7 @@ void fwnode_remove_software_node(struct fwnode_handle *fwnode)
 	if (!swnode)
 		return;
 
-	kobject_put(&swnode->kobj);
+	swnode_put(swnode);
 }
 EXPORT_SYMBOL_GPL(fwnode_remove_software_node);
 
@@ -1002,7 +1010,7 @@ int device_add_software_node(struct device *dev, const struct software_node *nod
 
 	swnode = software_node_to_swnode(node);
 	if (swnode) {
-		kobject_get(&swnode->kobj);
+		swnode_get(swnode);
 	} else {
 		ret = software_node_register(node);
 		if (ret)
@@ -1044,7 +1052,7 @@ void device_remove_software_node(struct device *dev)
 		software_node_notify_remove(dev);
 
 	set_secondary_fwnode(dev, NULL);
-	kobject_put(&swnode->kobj);
+	swnode_put(swnode);
 }
 EXPORT_SYMBOL_GPL(device_remove_software_node);
 
@@ -1097,7 +1105,7 @@ void software_node_notify(struct device *dev)
 	if (!swnode)
 		return;
 
-	kobject_get(&swnode->kobj);
+	swnode_get(swnode);
 	ret = sysfs_create_link(&dev->kobj, &swnode->kobj, "software_node");
 	if (ret)
 		return;
@@ -1119,11 +1127,11 @@ void software_node_notify_remove(struct device *dev)
 
 	sysfs_remove_link(&swnode->kobj, dev_name(dev));
 	sysfs_remove_link(&dev->kobj, "software_node");
-	kobject_put(&swnode->kobj);
+	swnode_put(swnode);
 
 	if (swnode->managed) {
 		set_secondary_fwnode(dev, NULL);
-		kobject_put(&swnode->kobj);
+		swnode_put(swnode);
 	}
 }
 
diff --git a/drivers/block/drbd/Makefile b/drivers/block/drbd/Makefile
index 187eaf81f0f89..5faaa8a8e7f02 100644
--- a/drivers/block/drbd/Makefile
+++ b/drivers/block/drbd/Makefile
@@ -3,6 +3,7 @@ drbd-y := drbd_buildtag.o drbd_bitmap.o drbd_proc.o
 drbd-y += drbd_worker.o drbd_receiver.o drbd_req.o drbd_actlog.o
 drbd-y += drbd_main.o drbd_strings.o drbd_nl.o
 drbd-y += drbd_interval.o drbd_state.o
+drbd-y += drbd_nl_gen.o
 drbd-$(CONFIG_DEBUG_FS) += drbd_debugfs.o
 
 obj-$(CONFIG_BLK_DEV_DRBD)     += drbd.o
diff --git a/drivers/block/drbd/drbd_buildtag.c b/drivers/block/drbd/drbd_buildtag.c
index cb1aa66d7d5d9..cd0389488f63e 100644
--- a/drivers/block/drbd/drbd_buildtag.c
+++ b/drivers/block/drbd/drbd_buildtag.c
@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: GPL-2.0-only
-#include <linux/drbd_config.h>
+#include "drbd_config.h"
 #include <linux/module.h>
 
 const char *drbd_buildtag(void)
diff --git a/drivers/block/drbd/drbd_config.h b/drivers/block/drbd/drbd_config.h
new file mode 100644
index 0000000000000..d215365c6bb1b
--- /dev/null
+++ b/drivers/block/drbd/drbd_config.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * drbd_config.h
+ * DRBD's compile time configuration.
+ */
+
+#ifndef DRBD_CONFIG_H
+#define DRBD_CONFIG_H
+
+extern const char *drbd_buildtag(void);
+
+#define REL_VERSION "8.4.11"
+#define PRO_VERSION_MIN 86
+#define PRO_VERSION_MAX 101
+
+#endif
diff --git a/drivers/block/drbd/drbd_debugfs.c b/drivers/block/drbd/drbd_debugfs.c
index 12460b584bcb9..371abcd7e8800 100644
--- a/drivers/block/drbd/drbd_debugfs.c
+++ b/drivers/block/drbd/drbd_debugfs.c
@@ -844,7 +844,7 @@ static int drbd_version_show(struct seq_file *m, void *ignored)
 {
 	seq_printf(m, "# %s\n", drbd_buildtag());
 	seq_printf(m, "VERSION=%s\n", REL_VERSION);
-	seq_printf(m, "API_VERSION=%u\n", GENL_MAGIC_VERSION);
+	seq_printf(m, "API_VERSION=%u\n", DRBD_FAMILY_VERSION);
 	seq_printf(m, "PRO_VERSION_MIN=%u\n", PRO_VERSION_MIN);
 	seq_printf(m, "PRO_VERSION_MAX=%u\n", PRO_VERSION_MAX);
 	return 0;
diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h
index f6d6276974eef..48b45c3142f76 100644
--- a/drivers/block/drbd/drbd_int.h
+++ b/drivers/block/drbd/drbd_int.h
@@ -32,14 +32,16 @@
 #include <net/tcp.h>
 #include <linux/lru_cache.h>
 #include <linux/prefetch.h>
-#include <linux/drbd_genl_api.h>
 #include <linux/drbd.h>
-#include <linux/drbd_config.h>
+#include "drbd_config.h"
+#include "drbd_nl_gen.h"
 #include "drbd_strings.h"
 #include "drbd_state.h"
 #include "drbd_protocol.h"
 #include "drbd_polymorph_printk.h"
 
+extern struct genl_family drbd_nl_family;
+
 /* shared module parameters, defined in drbd_main.c */
 #ifdef CONFIG_DRBD_FAULT_INJECTION
 extern int drbd_enable_faults;
diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c
index b1a721dd04969..a2a841c892011 100644
--- a/drivers/block/drbd/drbd_main.c
+++ b/drivers/block/drbd/drbd_main.c
@@ -2324,7 +2324,7 @@ static void drbd_cleanup(void)
 	if (retry.wq)
 		destroy_workqueue(retry.wq);
 
-	drbd_genl_unregister();
+	genl_unregister_family(&drbd_nl_family);
 
 	idr_for_each_entry(&drbd_devices, device, i)
 		drbd_delete_device(device);
@@ -2846,7 +2846,7 @@ static int __init drbd_init(void)
 	mutex_init(&resources_mutex);
 	INIT_LIST_HEAD(&drbd_resources);
 
-	err = drbd_genl_register();
+	err = genl_register_family(&drbd_nl_family);
 	if (err) {
 		pr_err("unable to register generic netlink family\n");
 		goto fail;
@@ -2876,7 +2876,7 @@ static int __init drbd_init(void)
 
 	pr_info("initialized. "
 	       "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n",
-	       GENL_MAGIC_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
+	       DRBD_FAMILY_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX);
 	pr_info("%s\n", drbd_buildtag());
 	pr_info("registered as block device major %d\n", DRBD_MAJOR);
 	return 0; /* Success! */
diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c
index c2ac555473e7d..f9ffcd67607b5 100644
--- a/drivers/block/drbd/drbd_nl.c
+++ b/drivers/block/drbd/drbd_nl.c
@@ -31,59 +31,13 @@
 
 #include <net/genetlink.h>
 
-/* .doit */
-// int drbd_adm_create_resource(struct sk_buff *skb, struct genl_info *info);
-// int drbd_adm_delete_resource(struct sk_buff *skb, struct genl_info *info);
-
-int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_del_minor(struct sk_buff *skb, struct genl_info *info);
-
-int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_del_resource(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_down(struct sk_buff *skb, struct genl_info *info);
-
-int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_disconnect(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_pause_sync(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_suspend_io(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_outdate(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_get_status(struct sk_buff *skb, struct genl_info *info);
-int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info);
-/* .dumpit */
-int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_devices_done(struct netlink_callback *cb);
-int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_connections_done(struct netlink_callback *cb);
-int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb);
-int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb);
-int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb);
-
-#include <linux/drbd_genl_api.h>
-
-static int drbd_pre_doit(const struct genl_split_ops *ops,
-			 struct sk_buff *skb, struct genl_info *info);
-static void drbd_post_doit(const struct genl_split_ops *ops,
-			   struct sk_buff *skb, struct genl_info *info);
-
-#define GENL_MAGIC_FAMILY_PRE_DOIT	drbd_pre_doit
-#define GENL_MAGIC_FAMILY_POST_DOIT	drbd_post_doit
-
-#include <linux/genl_magic_func.h>
+#include "drbd_nl_gen.h"
+
+static int drbd_genl_multicast_events(struct sk_buff *skb, gfp_t flags)
+{
+	return genlmsg_multicast(&drbd_nl_family, skb, 0,
+				 DRBD_NLGRP_EVENTS, flags);
+}
 
 static atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */
 static atomic_t notify_genl_seq = ATOMIC_INIT(2); /* two. */
@@ -114,7 +68,7 @@ static int drbd_msg_put_info(struct sk_buff *skb, const char *info)
 	if (!nla)
 		return err;
 
-	err = nla_put_string(skb, T_info_text, info);
+	err = nla_put_string(skb, DRBD_A_DRBD_CFG_REPLY_INFO_TEXT, info);
 	if (err) {
 		nla_nest_cancel(skb, nla);
 		return err;
@@ -135,7 +89,7 @@ static int drbd_msg_sprintf_info(struct sk_buff *skb, const char *fmt, ...)
 	if (!nla)
 		return err;
 
-	txt = nla_reserve(skb, T_info_text, 256);
+	txt = nla_reserve(skb, DRBD_A_DRBD_CFG_REPLY_INFO_TEXT, 256);
 	if (!txt) {
 		nla_nest_cancel(skb, nla);
 		return err;
@@ -187,6 +141,15 @@ static const unsigned int drbd_genl_cmd_flags[] = {
 	[DRBD_ADM_DOWN]           = DRBD_ADM_NEED_RESOURCE,
 };
 
+/* Detect attempts to change invariant attributes in a _change_ handler. */
+#define has_invariant(ntb, attr)						\
+({									\
+	bool __found = !!(ntb)[attr];					\
+	if (__found)							\
+		pr_info("must not change invariant attr: %s\n", #attr);	\
+	__found;							\
+})
+
 /*
  * At this point, we still rely on the global genl_lock().
  * If we want to avoid that, and allow "genl_family.parallel_ops", we may need
@@ -210,7 +173,7 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx,
 	}
 
 	adm_ctx->reply_dh = genlmsg_put_reply(adm_ctx->reply_skb,
-					info, &drbd_genl_family, 0, cmd);
+					info, &drbd_nl_family, 0, cmd);
 	/* put of a few bytes into a fresh skb of >= 4k will always succeed.
 	 * but anyways */
 	if (!adm_ctx->reply_dh) {
@@ -223,9 +186,11 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx,
 
 	adm_ctx->volume = VOLUME_UNSPECIFIED;
 	if (info->attrs[DRBD_NLA_CFG_CONTEXT]) {
+		struct nlattr **ntb;
 		struct nlattr *nla;
-		/* parse and validate only */
-		err = drbd_cfg_context_from_attrs(NULL, info);
+
+		/* parse and validate, get nested attribute table */
+		err = drbd_cfg_context_ntb_from_attrs(&ntb, info);
 		if (err)
 			goto fail;
 
@@ -234,18 +199,21 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx,
 		err = nla_put_nohdr(adm_ctx->reply_skb,
 				info->attrs[DRBD_NLA_CFG_CONTEXT]->nla_len,
 				info->attrs[DRBD_NLA_CFG_CONTEXT]);
-		if (err)
+		if (err) {
+			kfree(ntb);
 			goto fail;
+		}
 
 		/* and assign stuff to the adm_ctx */
-		nla = nested_attr_tb[T_ctx_volume];
+		nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME];
 		if (nla)
 			adm_ctx->volume = nla_get_u32(nla);
-		nla = nested_attr_tb[T_ctx_resource_name];
+		nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME];
 		if (nla)
 			adm_ctx->resource_name = nla_data(nla);
-		adm_ctx->my_addr = nested_attr_tb[T_ctx_my_addr];
-		adm_ctx->peer_addr = nested_attr_tb[T_ctx_peer_addr];
+		adm_ctx->my_addr = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR];
+		adm_ctx->peer_addr = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR];
+		kfree(ntb);
 		if ((adm_ctx->my_addr &&
 		     nla_len(adm_ctx->my_addr) > sizeof(adm_ctx->connection->my_addr)) ||
 		    (adm_ctx->peer_addr &&
@@ -259,7 +227,7 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx,
 	adm_ctx->device = minor_to_device(d_in->minor);
 
 	/* We are protected by the global genl_lock().
-	 * But we may explicitly drop it/retake it in drbd_adm_set_role(),
+	 * But we may explicitly drop it/retake it in drbd_nl_set_role(),
 	 * so make sure this object stays around. */
 	if (adm_ctx->device)
 		kref_get(&adm_ctx->device->kref);
@@ -334,8 +302,8 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx,
 	return err;
 }
 
-static int drbd_pre_doit(const struct genl_split_ops *ops,
-			 struct sk_buff *skb, struct genl_info *info)
+int drbd_pre_doit(const struct genl_split_ops *ops,
+		  struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx;
 	u8 cmd = info->genlhdr->cmd;
@@ -362,8 +330,8 @@ static int drbd_pre_doit(const struct genl_split_ops *ops,
 	return 0;
 }
 
-static void drbd_post_doit(const struct genl_split_ops *ops,
-			   struct sk_buff *skb, struct genl_info *info)
+void drbd_post_doit(const struct genl_split_ops *ops,
+		    struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 
@@ -828,7 +796,7 @@ static const char *from_attrs_err_to_txt(int err)
 		"invalid attribute value";
 }
 
-int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info)
+static int drbd_nl_set_role(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct set_role_parms parms;
@@ -868,6 +836,16 @@ int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
+int drbd_nl_primary_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	return drbd_nl_set_role(skb, info);
+}
+
+int drbd_nl_secondary_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	return drbd_nl_set_role(skb, info);
+}
+
 /* Initializes the md.*_offset members, so we are able to find
  * the on disk meta data.
  *
@@ -962,7 +940,7 @@ char *ppsize(char *buf, unsigned long long size)
  *  peer may not initiate a resize.
  */
 /* Note these are not to be confused with
- * drbd_adm_suspend_io/drbd_adm_resume_io,
+ * drbd_nl_suspend_io_doit/drbd_nl_resume_io_doit,
  * which are (sub) state changes triggered by admin (drbdsetup),
  * and can be long lived.
  * This changes an device->flag, is triggered by drbd internals,
@@ -1574,13 +1552,14 @@ static int disk_opts_check_al_size(struct drbd_device *device, struct disk_conf
 	return err;
 }
 
-int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_chg_disk_opts_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	enum drbd_ret_code retcode;
 	struct drbd_device *device;
 	struct disk_conf *new_disk_conf, *old_disk_conf;
 	struct fifo_buffer *old_plan = NULL, *new_plan = NULL;
+	struct nlattr **ntb;
 	int err;
 	unsigned int fifo_size;
 
@@ -1612,13 +1591,29 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
 	if (should_set_defaults(info))
 		set_disk_conf_defaults(new_disk_conf);
 
-	err = disk_conf_from_attrs_for_change(new_disk_conf, info);
+	err = disk_conf_from_attrs(new_disk_conf, info);
 	if (err && err != -ENOMSG) {
 		retcode = ERR_MANDATORY_TAG;
 		drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err));
 		goto fail_unlock;
 	}
 
+	err = disk_conf_ntb_from_attrs(&ntb, info);
+	if (!err) {
+		if (has_invariant(ntb, DRBD_A_DISK_CONF_BACKING_DEV) ||
+		    has_invariant(ntb, DRBD_A_DISK_CONF_META_DEV) ||
+		    has_invariant(ntb, DRBD_A_DISK_CONF_META_DEV_IDX) ||
+		    has_invariant(ntb, DRBD_A_DISK_CONF_DISK_SIZE) ||
+		    has_invariant(ntb, DRBD_A_DISK_CONF_MAX_BIO_BVECS)) {
+			retcode = ERR_MANDATORY_TAG;
+			drbd_msg_put_info(adm_ctx->reply_skb,
+				"cannot change invariant setting");
+			kfree(ntb);
+			goto fail_unlock;
+		}
+		kfree(ntb);
+	}
+
 	if (!expect(device, new_disk_conf->resync_rate >= 1))
 		new_disk_conf->resync_rate = 1;
 
@@ -1796,7 +1791,7 @@ void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev *
 	kfree(ldev);
 }
 
-int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_attach_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct drbd_device *device;
@@ -2236,7 +2231,7 @@ static int adm_detach(struct drbd_device *device, int force)
  * Then we transition to D_DISKLESS, and wait for put_ldev() to return all
  * internal references as well.
  * Only then we have finally detached. */
-int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_detach_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	enum drbd_ret_code retcode;
@@ -2434,12 +2429,13 @@ static void free_crypto(struct crypto *crypto)
 	crypto_free_shash(crypto->verify_tfm);
 }
 
-int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_chg_net_opts_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	enum drbd_ret_code retcode;
 	struct drbd_connection *connection;
 	struct net_conf *old_net_conf, *new_net_conf = NULL;
+	struct nlattr **ntb;
 	int err;
 	int ovr; /* online verify running */
 	int rsr; /* re-sync running */
@@ -2476,13 +2472,26 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
 	if (should_set_defaults(info))
 		set_net_conf_defaults(new_net_conf);
 
-	err = net_conf_from_attrs_for_change(new_net_conf, info);
+	err = net_conf_from_attrs(new_net_conf, info);
 	if (err && err != -ENOMSG) {
 		retcode = ERR_MANDATORY_TAG;
 		drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err));
 		goto fail;
 	}
 
+	err = net_conf_ntb_from_attrs(&ntb, info);
+	if (!err) {
+		if (has_invariant(ntb, DRBD_A_NET_CONF_DISCARD_MY_DATA) ||
+		    has_invariant(ntb, DRBD_A_NET_CONF_TENTATIVE)) {
+			retcode = ERR_MANDATORY_TAG;
+			drbd_msg_put_info(adm_ctx->reply_skb,
+				"cannot change invariant setting");
+			kfree(ntb);
+			goto fail;
+		}
+		kfree(ntb);
+	}
+
 	retcode = check_net_options(connection, new_net_conf);
 	if (retcode != NO_ERROR)
 		goto fail;
@@ -2575,7 +2584,7 @@ static void peer_device_to_info(struct peer_device_info *info,
 	info->peer_resync_susp_dependency = device->state.aftr_isp;
 }
 
-int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_connect_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct connection_info connection_info;
 	enum drbd_notification_type flags;
@@ -2790,7 +2799,7 @@ static enum drbd_state_rv conn_try_disconnect(struct drbd_connection *connection
 	return rv;
 }
 
-int drbd_adm_disconnect(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_disconnect_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct disconnect_parms parms;
@@ -2845,7 +2854,7 @@ void resync_after_online_grow(struct drbd_device *device)
 		_drbd_request_state(device, NS(conn, C_WF_SYNC_UUID), CS_VERBOSE + CS_SERIALIZE);
 }
 
-int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_resize_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct disk_conf *old_disk_conf, *new_disk_conf = NULL;
@@ -2981,7 +2990,7 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
 	goto fail;
 }
 
-int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_resource_opts_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	enum drbd_ret_code retcode;
@@ -3019,7 +3028,7 @@ int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
-int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_invalidate_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct drbd_device *device;
@@ -3097,7 +3106,7 @@ static int drbd_bmio_set_susp_al(struct drbd_device *device,
 	return rv;
 }
 
-int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_inval_peer_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	int retcode; /* drbd_ret_code, drbd_state_rv */
@@ -3148,7 +3157,7 @@ int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
-int drbd_adm_pause_sync(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_pause_sync_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	enum drbd_ret_code retcode;
@@ -3168,7 +3177,7 @@ int drbd_adm_pause_sync(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
-int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_resume_sync_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	union drbd_dev_state s;
@@ -3196,12 +3205,12 @@ int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
-int drbd_adm_suspend_io(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_suspend_io_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	return drbd_adm_simple_request_state(skb, info, NS(susp, 1));
 }
 
-int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_resume_io_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct drbd_device *device;
@@ -3257,7 +3266,7 @@ int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
-int drbd_adm_outdate(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_outdate_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	return drbd_adm_simple_request_state(skb, info, NS(disk, D_OUTDATED));
 }
@@ -3272,16 +3281,20 @@ static int nla_put_drbd_cfg_context(struct sk_buff *skb,
 	if (!nla)
 		goto nla_put_failure;
 	if (device &&
-	    nla_put_u32(skb, T_ctx_volume, device->vnr))
+	    nla_put_u32(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME, device->vnr))
 		goto nla_put_failure;
-	if (nla_put_string(skb, T_ctx_resource_name, resource->name))
+	if (nla_put_string(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME, resource->name))
 		goto nla_put_failure;
 	if (connection) {
 		if (connection->my_addr_len &&
-		    nla_put(skb, T_ctx_my_addr, connection->my_addr_len, &connection->my_addr))
+		    nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR,
+			    connection->my_addr_len,
+			    &connection->my_addr))
 			goto nla_put_failure;
 		if (connection->peer_addr_len &&
-		    nla_put(skb, T_ctx_peer_addr, connection->peer_addr_len, &connection->peer_addr))
+		    nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR,
+			    connection->peer_addr_len,
+			    &connection->peer_addr))
 			goto nla_put_failure;
 	}
 	nla_nest_end(skb, nla);
@@ -3300,7 +3313,7 @@ static int nla_put_drbd_cfg_context(struct sk_buff *skb,
  */
 static struct nlattr *find_cfg_context_attr(const struct nlmsghdr *nlh, int attr)
 {
-	const unsigned hdrlen = GENL_HDRLEN + GENL_MAGIC_FAMILY_HDRSZ;
+	const unsigned int hdrlen = GENL_HDRLEN + sizeof(struct drbd_genlmsghdr);
 	struct nlattr *nla;
 
 	nla = nla_find(nlmsg_attrdata(nlh, hdrlen), nlmsg_attrlen(nlh, hdrlen),
@@ -3312,7 +3325,7 @@ static struct nlattr *find_cfg_context_attr(const struct nlmsghdr *nlh, int attr
 
 static void resource_to_info(struct resource_info *, struct drbd_resource *);
 
-int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb)
+int drbd_nl_get_resources_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct drbd_genlmsghdr *dh;
 	struct drbd_resource *resource;
@@ -3340,7 +3353,7 @@ int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb)
 
 put_result:
 	dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
-			cb->nlh->nlmsg_seq, &drbd_genl_family,
+			cb->nlh->nlmsg_seq, &drbd_nl_family,
 			NLM_F_MULTI, DRBD_ADM_GET_RESOURCES);
 	err = -ENOMEM;
 	if (!dh)
@@ -3350,15 +3363,15 @@ int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb)
 	err = nla_put_drbd_cfg_context(skb, resource, NULL, NULL);
 	if (err)
 		goto out;
-	err = res_opts_to_skb(skb, &resource->res_opts, !capable(CAP_SYS_ADMIN));
+	err = res_opts_to_skb(skb, &resource->res_opts);
 	if (err)
 		goto out;
 	resource_to_info(&resource_info, resource);
-	err = resource_info_to_skb(skb, &resource_info, !capable(CAP_SYS_ADMIN));
+	err = resource_info_to_skb(skb, &resource_info);
 	if (err)
 		goto out;
 	resource_statistics.res_stat_write_ordering = resource->write_ordering;
-	err = resource_statistics_to_skb(skb, &resource_statistics, !capable(CAP_SYS_ADMIN));
+	err = resource_statistics_to_skb(skb, &resource_statistics);
 	if (err)
 		goto out;
 	cb->args[0] = (long)resource;
@@ -3423,7 +3436,7 @@ int drbd_adm_dump_devices_done(struct netlink_callback *cb) {
 
 static void device_to_info(struct device_info *, struct drbd_device *);
 
-int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
+int drbd_nl_get_devices_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct nlattr *resource_filter;
 	struct drbd_resource *resource;
@@ -3436,7 +3449,8 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
 
 	resource = (struct drbd_resource *)cb->args[0];
 	if (!cb->args[0] && !cb->args[1]) {
-		resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name);
+		resource_filter = find_cfg_context_attr(cb->nlh,
+				DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME);
 		if (resource_filter) {
 			retcode = ERR_RES_NOT_KNOWN;
 			resource = drbd_find_resource(nla_data(resource_filter));
@@ -3465,7 +3479,7 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
 
 put_result:
 	dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
-			cb->nlh->nlmsg_seq, &drbd_genl_family,
+			cb->nlh->nlmsg_seq, &drbd_nl_family,
 			NLM_F_MULTI, DRBD_ADM_GET_DEVICES);
 	err = -ENOMEM;
 	if (!dh)
@@ -3481,18 +3495,18 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb)
 			struct disk_conf *disk_conf =
 				rcu_dereference(device->ldev->disk_conf);
 
-			err = disk_conf_to_skb(skb, disk_conf, !capable(CAP_SYS_ADMIN));
+			err = disk_conf_to_skb(skb, disk_conf);
 			put_ldev(device);
 			if (err)
 				goto out;
 		}
 		device_to_info(&device_info, device);
-		err = device_info_to_skb(skb, &device_info, !capable(CAP_SYS_ADMIN));
+		err = device_info_to_skb(skb, &device_info);
 		if (err)
 			goto out;
 
 		device_to_statistics(&device_statistics, device);
-		err = device_statistics_to_skb(skb, &device_statistics, !capable(CAP_SYS_ADMIN));
+		err = device_statistics_to_skb(skb, &device_statistics);
 		if (err)
 			goto out;
 		cb->args[1] = minor + 1;
@@ -3514,7 +3528,7 @@ int drbd_adm_dump_connections_done(struct netlink_callback *cb)
 
 enum { SINGLE_RESOURCE, ITERATE_RESOURCES };
 
-int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
+int drbd_nl_get_connections_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct nlattr *resource_filter;
 	struct drbd_resource *resource = NULL, *next_resource;
@@ -3527,7 +3541,8 @@ int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
 	rcu_read_lock();
 	resource = (struct drbd_resource *)cb->args[0];
 	if (!cb->args[0]) {
-		resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name);
+		resource_filter = find_cfg_context_attr(cb->nlh,
+				DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME);
 		if (resource_filter) {
 			retcode = ERR_RES_NOT_KNOWN;
 			resource = drbd_find_resource(nla_data(resource_filter));
@@ -3591,7 +3606,7 @@ int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
 
 put_result:
 	dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
-			cb->nlh->nlmsg_seq, &drbd_genl_family,
+			cb->nlh->nlmsg_seq, &drbd_nl_family,
 			NLM_F_MULTI, DRBD_ADM_GET_CONNECTIONS);
 	err = -ENOMEM;
 	if (!dh)
@@ -3606,16 +3621,16 @@ int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb)
 			goto out;
 		net_conf = rcu_dereference(connection->net_conf);
 		if (net_conf) {
-			err = net_conf_to_skb(skb, net_conf, !capable(CAP_SYS_ADMIN));
+			err = net_conf_to_skb(skb, net_conf);
 			if (err)
 				goto out;
 		}
 		connection_to_info(&connection_info, connection);
-		err = connection_info_to_skb(skb, &connection_info, !capable(CAP_SYS_ADMIN));
+		err = connection_info_to_skb(skb, &connection_info);
 		if (err)
 			goto out;
 		connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags);
-		err = connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN));
+		err = connection_statistics_to_skb(skb, &connection_statistics);
 		if (err)
 			goto out;
 		cb->args[2] = (long)connection;
@@ -3676,7 +3691,7 @@ int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb)
 	return put_resource_in_arg0(cb, 9);
 }
 
-int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
+int drbd_nl_get_peer_devices_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct nlattr *resource_filter;
 	struct drbd_resource *resource;
@@ -3688,7 +3703,8 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 
 	resource = (struct drbd_resource *)cb->args[0];
 	if (!cb->args[0] && !cb->args[1]) {
-		resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name);
+		resource_filter = find_cfg_context_attr(cb->nlh,
+				DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME);
 		if (resource_filter) {
 			retcode = ERR_RES_NOT_KNOWN;
 			resource = drbd_find_resource(nla_data(resource_filter));
@@ -3735,7 +3751,7 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 
 put_result:
 	dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
-			cb->nlh->nlmsg_seq, &drbd_genl_family,
+			cb->nlh->nlmsg_seq, &drbd_nl_family,
 			NLM_F_MULTI, DRBD_ADM_GET_PEER_DEVICES);
 	err = -ENOMEM;
 	if (!dh)
@@ -3751,11 +3767,11 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb)
 		if (err)
 			goto out;
 		peer_device_to_info(&peer_device_info, peer_device);
-		err = peer_device_info_to_skb(skb, &peer_device_info, !capable(CAP_SYS_ADMIN));
+		err = peer_device_info_to_skb(skb, &peer_device_info);
 		if (err)
 			goto out;
 		peer_device_to_statistics(&peer_device_statistics, peer_device);
-		err = peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN));
+		err = peer_device_statistics_to_skb(skb, &peer_device_statistics);
 		if (err)
 			goto out;
 		cb->args[1] = minor;
@@ -3795,11 +3811,11 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device,
 	/* If sib != NULL, this is drbd_bcast_event, which anyone can listen
 	 * to.  So we better exclude_sensitive information.
 	 *
-	 * If sib == NULL, this is drbd_adm_get_status, executed synchronously
+	 * If sib == NULL, this is drbd_nl_get_status_doit, executed synchronously
 	 * in the context of the requesting user process. Exclude sensitive
 	 * information, unless current has superuser.
 	 *
-	 * NOTE: for drbd_adm_get_status_all(), this is a netlink dump, and
+	 * NOTE: for drbd_nl_get_status_dumpit(), this is a netlink dump, and
 	 * relies on the current implementation of netlink_dump(), which
 	 * executes the dump callback successively from netlink_recvmsg(),
 	 * always in the context of the receiving process */
@@ -3812,7 +3828,7 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device,
 	if (nla_put_drbd_cfg_context(skb, resource, the_only_connection(resource), device))
 		goto nla_put_failure;
 
-	if (res_opts_to_skb(skb, &device->resource->res_opts, exclude_sensitive))
+	if (res_opts_to_skb(skb, &device->resource->res_opts))
 		goto nla_put_failure;
 
 	rcu_read_lock();
@@ -3820,14 +3836,24 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device,
 		struct disk_conf *disk_conf;
 
 		disk_conf = rcu_dereference(device->ldev->disk_conf);
-		err = disk_conf_to_skb(skb, disk_conf, exclude_sensitive);
+		err = disk_conf_to_skb(skb, disk_conf);
 	}
 	if (!err) {
 		struct net_conf *nc;
 
 		nc = rcu_dereference(first_peer_device(device)->connection->net_conf);
-		if (nc)
-			err = net_conf_to_skb(skb, nc, exclude_sensitive);
+		if (nc) {
+			if (exclude_sensitive) {
+				struct net_conf nc_clean = *nc;
+
+				memset(nc_clean.shared_secret, 0,
+				       sizeof(nc_clean.shared_secret));
+				nc_clean.shared_secret_len = 0;
+				err = net_conf_to_skb(skb, &nc_clean);
+			} else {
+				err = net_conf_to_skb(skb, nc);
+			}
+		}
 	}
 	rcu_read_unlock();
 	if (err)
@@ -3836,42 +3862,57 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device,
 	nla = nla_nest_start_noflag(skb, DRBD_NLA_STATE_INFO);
 	if (!nla)
 		goto nla_put_failure;
-	if (nla_put_u32(skb, T_sib_reason, sib ? sib->sib_reason : SIB_GET_STATUS_REPLY) ||
-	    nla_put_u32(skb, T_current_state, device->state.i) ||
-	    nla_put_u64_0pad(skb, T_ed_uuid, device->ed_uuid) ||
-	    nla_put_u64_0pad(skb, T_capacity, get_capacity(device->vdisk)) ||
-	    nla_put_u64_0pad(skb, T_send_cnt, device->send_cnt) ||
-	    nla_put_u64_0pad(skb, T_recv_cnt, device->recv_cnt) ||
-	    nla_put_u64_0pad(skb, T_read_cnt, device->read_cnt) ||
-	    nla_put_u64_0pad(skb, T_writ_cnt, device->writ_cnt) ||
-	    nla_put_u64_0pad(skb, T_al_writ_cnt, device->al_writ_cnt) ||
-	    nla_put_u64_0pad(skb, T_bm_writ_cnt, device->bm_writ_cnt) ||
-	    nla_put_u32(skb, T_ap_bio_cnt, atomic_read(&device->ap_bio_cnt)) ||
-	    nla_put_u32(skb, T_ap_pending_cnt, atomic_read(&device->ap_pending_cnt)) ||
-	    nla_put_u32(skb, T_rs_pending_cnt, atomic_read(&device->rs_pending_cnt)))
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_SIB_REASON,
+		       sib ? sib->sib_reason : SIB_GET_STATUS_REPLY) ||
+	    nla_put_u32(skb, DRBD_A_STATE_INFO_CURRENT_STATE,
+		       device->state.i) ||
+	    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_ED_UUID,
+			     device->ed_uuid, 0) ||
+	    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_CAPACITY,
+			     get_capacity(device->vdisk), 0) ||
+	    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_SEND_CNT,
+			     device->send_cnt, 0) ||
+	    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_RECV_CNT,
+			     device->recv_cnt, 0) ||
+	    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_READ_CNT,
+			     device->read_cnt, 0) ||
+	    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_WRIT_CNT,
+			     device->writ_cnt, 0) ||
+	    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_AL_WRIT_CNT,
+			     device->al_writ_cnt, 0) ||
+	    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BM_WRIT_CNT,
+			     device->bm_writ_cnt, 0) ||
+	    nla_put_u32(skb, DRBD_A_STATE_INFO_AP_BIO_CNT,
+		       atomic_read(&device->ap_bio_cnt)) ||
+	    nla_put_u32(skb, DRBD_A_STATE_INFO_AP_PENDING_CNT,
+		       atomic_read(&device->ap_pending_cnt)) ||
+	    nla_put_u32(skb, DRBD_A_STATE_INFO_RS_PENDING_CNT,
+		       atomic_read(&device->rs_pending_cnt)))
 		goto nla_put_failure;
 
 	if (got_ldev) {
 		int err;
 
 		spin_lock_irq(&device->ldev->md.uuid_lock);
-		err = nla_put(skb, T_uuids, sizeof(si->uuids), device->ldev->md.uuid);
+		err = nla_put(skb, DRBD_A_STATE_INFO_UUIDS,
+			     sizeof(si->uuids),
+			     device->ldev->md.uuid);
 		spin_unlock_irq(&device->ldev->md.uuid_lock);
 
 		if (err)
 			goto nla_put_failure;
 
-		if (nla_put_u32(skb, T_disk_flags, device->ldev->md.flags) ||
-		    nla_put_u64_0pad(skb, T_bits_total, drbd_bm_bits(device)) ||
-		    nla_put_u64_0pad(skb, T_bits_oos,
-				     drbd_bm_total_weight(device)))
+		if (nla_put_u32(skb, DRBD_A_STATE_INFO_DISK_FLAGS, device->ldev->md.flags) ||
+		    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_TOTAL, drbd_bm_bits(device), 0) ||
+		    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_OOS,
+				      drbd_bm_total_weight(device), 0))
 			goto nla_put_failure;
 		if (C_SYNC_SOURCE <= device->state.conn &&
 		    C_PAUSED_SYNC_T >= device->state.conn) {
-			if (nla_put_u64_0pad(skb, T_bits_rs_total,
-					     device->rs_total) ||
-			    nla_put_u64_0pad(skb, T_bits_rs_failed,
-					     device->rs_failed))
+			if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_RS_TOTAL,
+					      device->rs_total, 0) ||
+			    nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_RS_FAILED,
+					      device->rs_failed, 0))
 				goto nla_put_failure;
 		}
 	}
@@ -3882,17 +3923,17 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device,
 		case SIB_GET_STATUS_REPLY:
 			break;
 		case SIB_STATE_CHANGE:
-			if (nla_put_u32(skb, T_prev_state, sib->os.i) ||
-			    nla_put_u32(skb, T_new_state, sib->ns.i))
+			if (nla_put_u32(skb, DRBD_A_STATE_INFO_PREV_STATE, sib->os.i) ||
+			    nla_put_u32(skb, DRBD_A_STATE_INFO_NEW_STATE, sib->ns.i))
 				goto nla_put_failure;
 			break;
 		case SIB_HELPER_POST:
-			if (nla_put_u32(skb, T_helper_exit_code,
+			if (nla_put_u32(skb, DRBD_A_STATE_INFO_HELPER_EXIT_CODE,
 					sib->helper_exit_code))
 				goto nla_put_failure;
 			fallthrough;
 		case SIB_HELPER_PRE:
-			if (nla_put_string(skb, T_helper, sib->helper_name))
+			if (nla_put_string(skb, DRBD_A_STATE_INFO_HELPER, sib->helper_name))
 				goto nla_put_failure;
 			break;
 		}
@@ -3907,7 +3948,7 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device,
 	return err;
 }
 
-int drbd_adm_get_status(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_get_status_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	enum drbd_ret_code retcode;
@@ -3997,7 +4038,7 @@ static int get_one_status(struct sk_buff *skb, struct netlink_callback *cb)
 		}
 
 		dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
-				cb->nlh->nlmsg_seq, &drbd_genl_family,
+				cb->nlh->nlmsg_seq, &drbd_nl_family,
 				NLM_F_MULTI, DRBD_ADM_GET_STATUS);
 		if (!dh)
 			goto out;
@@ -4017,7 +4058,7 @@ static int get_one_status(struct sk_buff *skb, struct netlink_callback *cb)
 				struct net_conf *nc;
 
 				nc = rcu_dereference(connection->net_conf);
-				if (nc && net_conf_to_skb(skb, nc, 1) != 0)
+				if (nc && net_conf_to_skb(skb, nc) != 0)
 					goto cancel;
 			}
 			goto done;
@@ -4059,9 +4100,9 @@ static int get_one_status(struct sk_buff *skb, struct netlink_callback *cb)
  *
  * Once things are setup properly, we call into get_one_status().
  */
-int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb)
+int drbd_nl_get_status_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 {
-	const unsigned hdrlen = GENL_HDRLEN + GENL_MAGIC_FAMILY_HDRSZ;
+	const unsigned int hdrlen = GENL_HDRLEN + sizeof(struct drbd_genlmsghdr);
 	struct nlattr *nla;
 	const char *resource_name;
 	struct drbd_resource *resource;
@@ -4084,7 +4125,7 @@ int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb)
 	/* No explicit context given.  Dump all. */
 	if (!nla)
 		goto dump;
-	nla = nla_find_nested(nla, T_ctx_resource_name);
+	nla = nla_find_nested(nla, DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME);
 	/* context given, but no name present? */
 	if (!nla)
 		return -EINVAL;
@@ -4107,7 +4148,7 @@ int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb)
 	return get_one_status(skb, cb);
 }
 
-int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_get_timeout_type_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	enum drbd_ret_code retcode;
@@ -4125,7 +4166,7 @@ int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info)
 		test_bit(USE_DEGR_WFC_T, &adm_ctx->device->flags) ? UT_DEGRADED :
 		UT_DEFAULT;
 
-	err = timeout_parms_to_priv_skb(adm_ctx->reply_skb, &tp);
+	err = timeout_parms_to_skb(adm_ctx->reply_skb, &tp);
 	if (err) {
 		nlmsg_free(adm_ctx->reply_skb);
 		adm_ctx->reply_skb = NULL;
@@ -4136,7 +4177,7 @@ int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
-int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_start_ov_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct drbd_device *device;
@@ -4182,7 +4223,7 @@ int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info)
 }
 
 
-int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_new_c_uuid_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct drbd_device *device;
@@ -4285,7 +4326,7 @@ static void resource_to_info(struct resource_info *info,
 	info->res_susp_fen = resource->susp_fen;
 }
 
-int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_new_resource_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_connection *connection;
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
@@ -4348,7 +4389,7 @@ static void device_to_info(struct device_info *info,
 }
 
 
-int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_new_minor_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct drbd_genlmsghdr *dh = genl_info_userhdr(info);
@@ -4455,7 +4496,7 @@ static enum drbd_ret_code adm_del_minor(struct drbd_device *device)
 		return ERR_MINOR_CONFIGURED;
 }
 
-int drbd_adm_del_minor(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_del_minor_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	enum drbd_ret_code retcode;
@@ -4504,7 +4545,7 @@ static int adm_del_resource(struct drbd_resource *resource)
 	return NO_ERROR;
 }
 
-int drbd_adm_down(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_down_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct drbd_resource *resource;
@@ -4567,7 +4608,7 @@ int drbd_adm_down(struct sk_buff *skb, struct genl_info *info)
 	return 0;
 }
 
-int drbd_adm_del_resource(struct sk_buff *skb, struct genl_info *info)
+int drbd_nl_del_resource_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct drbd_config_context *adm_ctx = info->user_ptr[0];
 	struct drbd_resource *resource;
@@ -4601,7 +4642,7 @@ void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib)
 		goto failed;
 
 	err = -EMSGSIZE;
-	d_out = genlmsg_put(msg, 0, seq, &drbd_genl_family, 0, DRBD_EVENT);
+	d_out = genlmsg_put(msg, 0, seq, &drbd_nl_family, 0, DRBD_ADM_EVENT);
 	if (!d_out) /* cannot happen, but anyways. */
 		goto nla_put_failure;
 	d_out->minor = device_to_minor(device);
@@ -4632,7 +4673,7 @@ static int nla_put_notification_header(struct sk_buff *msg,
 		.nh_type = type,
 	};
 
-	return drbd_notification_header_to_skb(msg, &nh, true);
+	return drbd_notification_header_to_skb(msg, &nh);
 }
 
 int notify_resource_state(struct sk_buff *skb,
@@ -4656,7 +4697,7 @@ int notify_resource_state(struct sk_buff *skb,
 	}
 
 	err = -EMSGSIZE;
-	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_RESOURCE_STATE);
+	dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_RESOURCE_STATE);
 	if (!dh)
 		goto nla_put_failure;
 	dh->minor = -1U;
@@ -4664,10 +4705,10 @@ int notify_resource_state(struct sk_buff *skb,
 	if (nla_put_drbd_cfg_context(skb, resource, NULL, NULL) ||
 	    nla_put_notification_header(skb, type) ||
 	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
-	     resource_info_to_skb(skb, resource_info, true)))
+	     resource_info_to_skb(skb, resource_info)))
 		goto nla_put_failure;
 	resource_statistics.res_stat_write_ordering = resource->write_ordering;
-	err = resource_statistics_to_skb(skb, &resource_statistics, !capable(CAP_SYS_ADMIN));
+	err = resource_statistics_to_skb(skb, &resource_statistics);
 	if (err)
 		goto nla_put_failure;
 	genlmsg_end(skb, dh);
@@ -4708,7 +4749,7 @@ int notify_device_state(struct sk_buff *skb,
 	}
 
 	err = -EMSGSIZE;
-	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_DEVICE_STATE);
+	dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_DEVICE_STATE);
 	if (!dh)
 		goto nla_put_failure;
 	dh->minor = device->minor;
@@ -4716,10 +4757,10 @@ int notify_device_state(struct sk_buff *skb,
 	if (nla_put_drbd_cfg_context(skb, device->resource, NULL, device) ||
 	    nla_put_notification_header(skb, type) ||
 	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
-	     device_info_to_skb(skb, device_info, true)))
+	     device_info_to_skb(skb, device_info)))
 		goto nla_put_failure;
 	device_to_statistics(&device_statistics, device);
-	device_statistics_to_skb(skb, &device_statistics, !capable(CAP_SYS_ADMIN));
+	device_statistics_to_skb(skb, &device_statistics);
 	genlmsg_end(skb, dh);
 	if (multicast) {
 		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
@@ -4758,7 +4799,7 @@ int notify_connection_state(struct sk_buff *skb,
 	}
 
 	err = -EMSGSIZE;
-	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_CONNECTION_STATE);
+	dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_CONNECTION_STATE);
 	if (!dh)
 		goto nla_put_failure;
 	dh->minor = -1U;
@@ -4766,10 +4807,10 @@ int notify_connection_state(struct sk_buff *skb,
 	if (nla_put_drbd_cfg_context(skb, connection->resource, connection, NULL) ||
 	    nla_put_notification_header(skb, type) ||
 	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
-	     connection_info_to_skb(skb, connection_info, true)))
+	     connection_info_to_skb(skb, connection_info)))
 		goto nla_put_failure;
 	connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags);
-	connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN));
+	connection_statistics_to_skb(skb, &connection_statistics);
 	genlmsg_end(skb, dh);
 	if (multicast) {
 		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
@@ -4809,7 +4850,7 @@ int notify_peer_device_state(struct sk_buff *skb,
 	}
 
 	err = -EMSGSIZE;
-	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_PEER_DEVICE_STATE);
+	dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_PEER_DEVICE_STATE);
 	if (!dh)
 		goto nla_put_failure;
 	dh->minor = -1U;
@@ -4817,10 +4858,10 @@ int notify_peer_device_state(struct sk_buff *skb,
 	if (nla_put_drbd_cfg_context(skb, resource, peer_device->connection, peer_device->device) ||
 	    nla_put_notification_header(skb, type) ||
 	    ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY &&
-	     peer_device_info_to_skb(skb, peer_device_info, true)))
+	     peer_device_info_to_skb(skb, peer_device_info)))
 		goto nla_put_failure;
 	peer_device_to_statistics(&peer_device_statistics, peer_device);
-	peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN));
+	peer_device_statistics_to_skb(skb, &peer_device_statistics);
 	genlmsg_end(skb, dh);
 	if (multicast) {
 		err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
@@ -4859,7 +4900,7 @@ void notify_helper(enum drbd_notification_type type,
 		goto fail;
 
 	err = -EMSGSIZE;
-	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_HELPER);
+	dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_HELPER);
 	if (!dh)
 		goto fail;
 	dh->minor = device ? device->minor : -1;
@@ -4867,7 +4908,7 @@ void notify_helper(enum drbd_notification_type type,
 	mutex_lock(&notification_mutex);
 	if (nla_put_drbd_cfg_context(skb, resource, connection, device) ||
 	    nla_put_notification_header(skb, type) ||
-	    drbd_helper_info_to_skb(skb, &helper_info, true))
+	    drbd_helper_info_to_skb(skb, &helper_info))
 		goto unlock_fail;
 	genlmsg_end(skb, dh);
 	err = drbd_genl_multicast_events(skb, GFP_NOWAIT);
@@ -4892,7 +4933,7 @@ static int notify_initial_state_done(struct sk_buff *skb, unsigned int seq)
 	int err;
 
 	err = -EMSGSIZE;
-	dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_INITIAL_STATE_DONE);
+	dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_INITIAL_STATE_DONE);
 	if (!dh)
 		goto nla_put_failure;
 	dh->minor = -1U;
@@ -4987,7 +5028,7 @@ static int get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 		return skb->len;
 }
 
-int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
+int drbd_nl_get_initial_state_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
 {
 	struct drbd_resource *resource;
 	LIST_HEAD(head);
@@ -5035,3 +5076,20 @@ int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb)
 	cb->args[2] = cb->nlh->nlmsg_seq;
 	return get_initial_state(skb, cb);
 }
+
+static const struct genl_multicast_group drbd_nl_mcgrps[] = {
+	[DRBD_NLGRP_EVENTS] = { .name = "events", },
+};
+
+struct genl_family drbd_nl_family __ro_after_init = {
+	.name		= "drbd",
+	.version	= DRBD_FAMILY_VERSION,
+	.hdrsize	= NLA_ALIGN(sizeof(struct drbd_genlmsghdr)),
+	.split_ops	= drbd_nl_ops,
+	.n_split_ops	= ARRAY_SIZE(drbd_nl_ops),
+	.mcgrps		= drbd_nl_mcgrps,
+	.n_mcgrps	= ARRAY_SIZE(drbd_nl_mcgrps),
+	.resv_start_op	= 42,
+	.module		= THIS_MODULE,
+	.netnsok	= true,
+};
diff --git a/drivers/block/drbd/drbd_nl_gen.c b/drivers/block/drbd/drbd_nl_gen.c
new file mode 100644
index 0000000000000..fb44b948cec87
--- /dev/null
+++ b/drivers/block/drbd/drbd_nl_gen.c
@@ -0,0 +1,2606 @@
+// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include "drbd_nl_gen.h"
+
+#include <uapi/linux/drbd_genl.h>
+#include <linux/drbd.h>
+#include <linux/drbd_limits.h>
+
+/* Common nested types */
+const struct nla_policy drbd_connection_info_nl_policy[DRBD_A_CONNECTION_INFO_CONN_ROLE + 1] = {
+	[DRBD_A_CONNECTION_INFO_CONN_CONNECTION_STATE] = { .type = NLA_U32, },
+	[DRBD_A_CONNECTION_INFO_CONN_ROLE] = { .type = NLA_U32, },
+};
+
+const struct nla_policy drbd_connection_statistics_nl_policy[DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED + 1] = {
+	[DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED] = { .type = NLA_U8, },
+};
+
+const struct nla_policy drbd_detach_parms_nl_policy[DRBD_A_DETACH_PARMS_FORCE_DETACH + 1] = {
+	[DRBD_A_DETACH_PARMS_FORCE_DETACH] = { .type = NLA_U8, },
+};
+
+const struct nla_policy drbd_device_info_nl_policy[DRBD_A_DEVICE_INFO_DEV_DISK_STATE + 1] = {
+	[DRBD_A_DEVICE_INFO_DEV_DISK_STATE] = { .type = NLA_U32, },
+};
+
+const struct nla_policy drbd_device_statistics_nl_policy[DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS + 1] = {
+	[DRBD_A_DEVICE_STATISTICS_DEV_SIZE] = { .type = NLA_U64, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_READ] = { .type = NLA_U64, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_WRITE] = { .type = NLA_U64, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_AL_WRITES] = { .type = NLA_U64, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_BM_WRITES] = { .type = NLA_U64, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_UPPER_PENDING] = { .type = NLA_U32, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_LOWER_PENDING] = { .type = NLA_U32, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_UPPER_BLOCKED] = { .type = NLA_U8, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_LOWER_BLOCKED] = { .type = NLA_U8, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_AL_SUSPENDED] = { .type = NLA_U8, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_EXPOSED_DATA_UUID] = { .type = NLA_U64, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_CURRENT_UUID] = { .type = NLA_U64, },
+	[DRBD_A_DEVICE_STATISTICS_DEV_DISK_FLAGS] = { .type = NLA_U32, },
+	[DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS] = NLA_POLICY_MAX_LEN(DRBD_NL_HISTORY_UUIDS_SIZE),
+};
+
+const struct nla_policy drbd_disconnect_parms_nl_policy[DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT + 1] = {
+	[DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT] = { .type = NLA_U8, },
+};
+
+const struct nla_policy drbd_disk_conf_nl_policy[DRBD_A_DISK_CONF_DISABLE_WRITE_SAME + 1] = {
+	[DRBD_A_DISK_CONF_BACKING_DEV] = { .type = NLA_NUL_STRING, .len = 128, },
+	[DRBD_A_DISK_CONF_META_DEV] = { .type = NLA_NUL_STRING, .len = 128, },
+	[DRBD_A_DISK_CONF_META_DEV_IDX] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_DISK_SIZE] = { .type = NLA_U64, },
+	[DRBD_A_DISK_CONF_MAX_BIO_BVECS] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_ON_IO_ERROR] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_FENCING] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_RESYNC_RATE] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_RESYNC_AFTER] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_AL_EXTENTS] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_C_PLAN_AHEAD] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_C_DELAY_TARGET] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_C_FILL_TARGET] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_C_MAX_RATE] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_C_MIN_RATE] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_DISK_BARRIER] = { .type = NLA_U8, },
+	[DRBD_A_DISK_CONF_DISK_FLUSHES] = { .type = NLA_U8, },
+	[DRBD_A_DISK_CONF_DISK_DRAIN] = { .type = NLA_U8, },
+	[DRBD_A_DISK_CONF_MD_FLUSHES] = { .type = NLA_U8, },
+	[DRBD_A_DISK_CONF_DISK_TIMEOUT] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_READ_BALANCING] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_AL_UPDATES] = { .type = NLA_U8, },
+	[DRBD_A_DISK_CONF_DISCARD_ZEROES_IF_ALIGNED] = { .type = NLA_U8, },
+	[DRBD_A_DISK_CONF_RS_DISCARD_GRANULARITY] = { .type = NLA_U32, },
+	[DRBD_A_DISK_CONF_DISABLE_WRITE_SAME] = { .type = NLA_U8, },
+};
+
+const struct nla_policy drbd_drbd_cfg_context_nl_policy[DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR + 1] = {
+	[DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME] = { .type = NLA_U32, },
+	[DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME] = { .type = NLA_NUL_STRING, .len = 128, },
+	[DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR] = NLA_POLICY_MAX_LEN(128),
+	[DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR] = NLA_POLICY_MAX_LEN(128),
+};
+
+const struct nla_policy drbd_net_conf_nl_policy[DRBD_A_NET_CONF_SOCK_CHECK_TIMEO + 1] = {
+	[DRBD_A_NET_CONF_SHARED_SECRET] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, },
+	[DRBD_A_NET_CONF_CRAM_HMAC_ALG] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, },
+	[DRBD_A_NET_CONF_INTEGRITY_ALG] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, },
+	[DRBD_A_NET_CONF_VERIFY_ALG] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, },
+	[DRBD_A_NET_CONF_CSUMS_ALG] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, },
+	[DRBD_A_NET_CONF_WIRE_PROTOCOL] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_CONNECT_INT] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_TIMEOUT] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_PING_INT] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_PING_TIMEO] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_SNDBUF_SIZE] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_RCVBUF_SIZE] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_KO_COUNT] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_MAX_BUFFERS] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_MAX_EPOCH_SIZE] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_UNPLUG_WATERMARK] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_AFTER_SB_0P] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_AFTER_SB_1P] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_AFTER_SB_2P] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_RR_CONFLICT] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_ON_CONGESTION] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_CONG_FILL] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_CONG_EXTENTS] = { .type = NLA_U32, },
+	[DRBD_A_NET_CONF_TWO_PRIMARIES] = { .type = NLA_U8, },
+	[DRBD_A_NET_CONF_DISCARD_MY_DATA] = { .type = NLA_U8, },
+	[DRBD_A_NET_CONF_TCP_CORK] = { .type = NLA_U8, },
+	[DRBD_A_NET_CONF_ALWAYS_ASBP] = { .type = NLA_U8, },
+	[DRBD_A_NET_CONF_TENTATIVE] = { .type = NLA_U8, },
+	[DRBD_A_NET_CONF_USE_RLE] = { .type = NLA_U8, },
+	[DRBD_A_NET_CONF_CSUMS_AFTER_CRASH_ONLY] = { .type = NLA_U8, },
+	[DRBD_A_NET_CONF_SOCK_CHECK_TIMEO] = { .type = NLA_U32, },
+};
+
+const struct nla_policy drbd_new_c_uuid_parms_nl_policy[DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM + 1] = {
+	[DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM] = { .type = NLA_U8, },
+};
+
+const struct nla_policy drbd_peer_device_info_nl_policy[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY + 1] = {
+	[DRBD_A_PEER_DEVICE_INFO_PEER_REPL_STATE] = { .type = NLA_U32, },
+	[DRBD_A_PEER_DEVICE_INFO_PEER_DISK_STATE] = { .type = NLA_U32, },
+	[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_USER] = { .type = NLA_U32, },
+	[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_PEER] = { .type = NLA_U32, },
+	[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY] = { .type = NLA_U32, },
+};
+
+const struct nla_policy drbd_peer_device_statistics_nl_policy[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS + 1] = {
+	[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RECEIVED] = { .type = NLA_U64, },
+	[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_SENT] = { .type = NLA_U64, },
+	[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_PENDING] = { .type = NLA_U32, },
+	[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_UNACKED] = { .type = NLA_U32, },
+	[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_OUT_OF_SYNC] = { .type = NLA_U64, },
+	[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RESYNC_FAILED] = { .type = NLA_U64, },
+	[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_BITMAP_UUID] = { .type = NLA_U64, },
+	[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS] = { .type = NLA_U32, },
+};
+
+const struct nla_policy drbd_res_opts_nl_policy[DRBD_A_RES_OPTS_ON_NO_DATA + 1] = {
+	[DRBD_A_RES_OPTS_CPU_MASK] = { .type = NLA_NUL_STRING, .len = DRBD_CPU_MASK_SIZE, },
+	[DRBD_A_RES_OPTS_ON_NO_DATA] = { .type = NLA_U32, },
+};
+
+const struct nla_policy drbd_resize_parms_nl_policy[DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE + 1] = {
+	[DRBD_A_RESIZE_PARMS_RESIZE_SIZE] = { .type = NLA_U64, },
+	[DRBD_A_RESIZE_PARMS_RESIZE_FORCE] = { .type = NLA_U8, },
+	[DRBD_A_RESIZE_PARMS_NO_RESYNC] = { .type = NLA_U8, },
+	[DRBD_A_RESIZE_PARMS_AL_STRIPES] = { .type = NLA_U32, },
+	[DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE] = { .type = NLA_U32, },
+};
+
+const struct nla_policy drbd_resource_info_nl_policy[DRBD_A_RESOURCE_INFO_RES_SUSP_FEN + 1] = {
+	[DRBD_A_RESOURCE_INFO_RES_ROLE] = { .type = NLA_U32, },
+	[DRBD_A_RESOURCE_INFO_RES_SUSP] = { .type = NLA_U8, },
+	[DRBD_A_RESOURCE_INFO_RES_SUSP_NOD] = { .type = NLA_U8, },
+	[DRBD_A_RESOURCE_INFO_RES_SUSP_FEN] = { .type = NLA_U8, },
+};
+
+const struct nla_policy drbd_resource_statistics_nl_policy[DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING + 1] = {
+	[DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING] = { .type = NLA_U32, },
+};
+
+const struct nla_policy drbd_set_role_parms_nl_policy[DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE + 1] = {
+	[DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE] = { .type = NLA_U8, },
+};
+
+const struct nla_policy drbd_start_ov_parms_nl_policy[DRBD_A_START_OV_PARMS_OV_STOP_SECTOR + 1] = {
+	[DRBD_A_START_OV_PARMS_OV_START_SECTOR] = { .type = NLA_U64, },
+	[DRBD_A_START_OV_PARMS_OV_STOP_SECTOR] = { .type = NLA_U64, },
+};
+
+/* DRBD_ADM_GET_STATUS - do */
+static const struct nla_policy drbd_get_status_do_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_GET_STATUS - dump */
+static const struct nla_policy drbd_get_status_dump_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_NEW_MINOR - do */
+static const struct nla_policy drbd_new_minor_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_DEL_MINOR - do */
+static const struct nla_policy drbd_del_minor_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_NEW_RESOURCE - do */
+static const struct nla_policy drbd_new_resource_nl_policy[DRBD_NLA_RESOURCE_OPTS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_RESOURCE_OPTS] = NLA_POLICY_NESTED(drbd_res_opts_nl_policy),
+};
+
+/* DRBD_ADM_DEL_RESOURCE - do */
+static const struct nla_policy drbd_del_resource_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_RESOURCE_OPTS - do */
+static const struct nla_policy drbd_resource_opts_nl_policy[DRBD_NLA_RESOURCE_OPTS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_RESOURCE_OPTS] = NLA_POLICY_NESTED(drbd_res_opts_nl_policy),
+};
+
+/* DRBD_ADM_CONNECT - do */
+static const struct nla_policy drbd_connect_nl_policy[DRBD_NLA_NET_CONF + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_NET_CONF] = NLA_POLICY_NESTED(drbd_net_conf_nl_policy),
+};
+
+/* DRBD_ADM_DISCONNECT - do */
+static const struct nla_policy drbd_disconnect_nl_policy[DRBD_NLA_DISCONNECT_PARMS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_DISCONNECT_PARMS] = NLA_POLICY_NESTED(drbd_disconnect_parms_nl_policy),
+};
+
+/* DRBD_ADM_ATTACH - do */
+static const struct nla_policy drbd_attach_nl_policy[DRBD_NLA_DISK_CONF + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_DISK_CONF] = NLA_POLICY_NESTED(drbd_disk_conf_nl_policy),
+};
+
+/* DRBD_ADM_RESIZE - do */
+static const struct nla_policy drbd_resize_nl_policy[DRBD_NLA_RESIZE_PARMS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_RESIZE_PARMS] = NLA_POLICY_NESTED(drbd_resize_parms_nl_policy),
+};
+
+/* DRBD_ADM_PRIMARY - do */
+static const struct nla_policy drbd_primary_nl_policy[DRBD_NLA_SET_ROLE_PARMS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_SET_ROLE_PARMS] = NLA_POLICY_NESTED(drbd_set_role_parms_nl_policy),
+};
+
+/* DRBD_ADM_SECONDARY - do */
+static const struct nla_policy drbd_secondary_nl_policy[DRBD_NLA_SET_ROLE_PARMS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_SET_ROLE_PARMS] = NLA_POLICY_NESTED(drbd_set_role_parms_nl_policy),
+};
+
+/* DRBD_ADM_NEW_C_UUID - do */
+static const struct nla_policy drbd_new_c_uuid_nl_policy[DRBD_NLA_NEW_C_UUID_PARMS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_NEW_C_UUID_PARMS] = NLA_POLICY_NESTED(drbd_new_c_uuid_parms_nl_policy),
+};
+
+/* DRBD_ADM_START_OV - do */
+static const struct nla_policy drbd_start_ov_nl_policy[DRBD_NLA_START_OV_PARMS + 1] = {
+	[DRBD_NLA_START_OV_PARMS] = NLA_POLICY_NESTED(drbd_start_ov_parms_nl_policy),
+};
+
+/* DRBD_ADM_DETACH - do */
+static const struct nla_policy drbd_detach_nl_policy[DRBD_NLA_DETACH_PARMS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_DETACH_PARMS] = NLA_POLICY_NESTED(drbd_detach_parms_nl_policy),
+};
+
+/* DRBD_ADM_INVALIDATE - do */
+static const struct nla_policy drbd_invalidate_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_INVAL_PEER - do */
+static const struct nla_policy drbd_inval_peer_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_PAUSE_SYNC - do */
+static const struct nla_policy drbd_pause_sync_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_RESUME_SYNC - do */
+static const struct nla_policy drbd_resume_sync_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_SUSPEND_IO - do */
+static const struct nla_policy drbd_suspend_io_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_RESUME_IO - do */
+static const struct nla_policy drbd_resume_io_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_OUTDATE - do */
+static const struct nla_policy drbd_outdate_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_GET_TIMEOUT_TYPE - do */
+static const struct nla_policy drbd_get_timeout_type_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_DOWN - do */
+static const struct nla_policy drbd_down_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* DRBD_ADM_CHG_DISK_OPTS - do */
+static const struct nla_policy drbd_chg_disk_opts_nl_policy[DRBD_NLA_DISK_CONF + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_DISK_CONF] = NLA_POLICY_NESTED(drbd_disk_conf_nl_policy),
+};
+
+/* DRBD_ADM_CHG_NET_OPTS - do */
+static const struct nla_policy drbd_chg_net_opts_nl_policy[DRBD_NLA_NET_CONF + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_NET_CONF] = NLA_POLICY_NESTED(drbd_net_conf_nl_policy),
+};
+
+/* DRBD_ADM_GET_RESOURCES - dump */
+static const struct nla_policy drbd_get_resources_nl_policy[DRBD_NLA_RESOURCE_STATISTICS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_RESOURCE_INFO] = NLA_POLICY_NESTED(drbd_resource_info_nl_policy),
+	[DRBD_NLA_RESOURCE_STATISTICS] = NLA_POLICY_NESTED(drbd_resource_statistics_nl_policy),
+};
+
+/* DRBD_ADM_GET_DEVICES - dump */
+static const struct nla_policy drbd_get_devices_nl_policy[DRBD_NLA_DEVICE_STATISTICS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_DEVICE_INFO] = NLA_POLICY_NESTED(drbd_device_info_nl_policy),
+	[DRBD_NLA_DEVICE_STATISTICS] = NLA_POLICY_NESTED(drbd_device_statistics_nl_policy),
+};
+
+/* DRBD_ADM_GET_CONNECTIONS - dump */
+static const struct nla_policy drbd_get_connections_nl_policy[DRBD_NLA_CONNECTION_STATISTICS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_CONNECTION_INFO] = NLA_POLICY_NESTED(drbd_connection_info_nl_policy),
+	[DRBD_NLA_CONNECTION_STATISTICS] = NLA_POLICY_NESTED(drbd_connection_statistics_nl_policy),
+};
+
+/* DRBD_ADM_GET_PEER_DEVICES - dump */
+static const struct nla_policy drbd_get_peer_devices_nl_policy[DRBD_NLA_PEER_DEVICE_STATISTICS + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+	[DRBD_NLA_PEER_DEVICE_INFO] = NLA_POLICY_NESTED(drbd_peer_device_info_nl_policy),
+	[DRBD_NLA_PEER_DEVICE_STATISTICS] = NLA_POLICY_NESTED(drbd_peer_device_statistics_nl_policy),
+};
+
+/* DRBD_ADM_GET_INITIAL_STATE - dump */
+static const struct nla_policy drbd_get_initial_state_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = {
+	[DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy),
+};
+
+/* Ops table for drbd */
+const struct genl_split_ops drbd_nl_ops[32] = {
+	{
+		.cmd		= DRBD_ADM_GET_STATUS,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_get_status_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_get_status_do_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_GET_STATUS,
+		.dumpit		= drbd_nl_get_status_dumpit,
+		.policy		= drbd_get_status_dump_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= DRBD_ADM_NEW_MINOR,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_new_minor_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_new_minor_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_DEL_MINOR,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_del_minor_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_del_minor_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_NEW_RESOURCE,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_new_resource_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_new_resource_nl_policy,
+		.maxattr	= DRBD_NLA_RESOURCE_OPTS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_DEL_RESOURCE,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_del_resource_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_del_resource_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_RESOURCE_OPTS,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_resource_opts_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_resource_opts_nl_policy,
+		.maxattr	= DRBD_NLA_RESOURCE_OPTS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_CONNECT,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_connect_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_connect_nl_policy,
+		.maxattr	= DRBD_NLA_NET_CONF,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_DISCONNECT,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_disconnect_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_disconnect_nl_policy,
+		.maxattr	= DRBD_NLA_DISCONNECT_PARMS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_ATTACH,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_attach_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_attach_nl_policy,
+		.maxattr	= DRBD_NLA_DISK_CONF,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_RESIZE,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_resize_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_resize_nl_policy,
+		.maxattr	= DRBD_NLA_RESIZE_PARMS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_PRIMARY,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_primary_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_primary_nl_policy,
+		.maxattr	= DRBD_NLA_SET_ROLE_PARMS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_SECONDARY,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_secondary_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_secondary_nl_policy,
+		.maxattr	= DRBD_NLA_SET_ROLE_PARMS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_NEW_C_UUID,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_new_c_uuid_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_new_c_uuid_nl_policy,
+		.maxattr	= DRBD_NLA_NEW_C_UUID_PARMS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_START_OV,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_start_ov_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_start_ov_nl_policy,
+		.maxattr	= DRBD_NLA_START_OV_PARMS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_DETACH,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_detach_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_detach_nl_policy,
+		.maxattr	= DRBD_NLA_DETACH_PARMS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_INVALIDATE,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_invalidate_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_invalidate_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_INVAL_PEER,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_inval_peer_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_inval_peer_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_PAUSE_SYNC,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_pause_sync_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_pause_sync_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_RESUME_SYNC,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_resume_sync_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_resume_sync_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_SUSPEND_IO,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_suspend_io_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_suspend_io_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_RESUME_IO,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_resume_io_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_resume_io_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_OUTDATE,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_outdate_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_outdate_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_GET_TIMEOUT_TYPE,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_get_timeout_type_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_get_timeout_type_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_DOWN,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_down_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_down_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_CHG_DISK_OPTS,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_chg_disk_opts_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_chg_disk_opts_nl_policy,
+		.maxattr	= DRBD_NLA_DISK_CONF,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_CHG_NET_OPTS,
+		.pre_doit	= drbd_pre_doit,
+		.doit		= drbd_nl_chg_net_opts_doit,
+		.post_doit	= drbd_post_doit,
+		.policy		= drbd_chg_net_opts_nl_policy,
+		.maxattr	= DRBD_NLA_NET_CONF,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= DRBD_ADM_GET_RESOURCES,
+		.dumpit		= drbd_nl_get_resources_dumpit,
+		.policy		= drbd_get_resources_nl_policy,
+		.maxattr	= DRBD_NLA_RESOURCE_STATISTICS,
+		.flags		= GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= DRBD_ADM_GET_DEVICES,
+		.dumpit		= drbd_nl_get_devices_dumpit,
+		.done		= drbd_adm_dump_devices_done,
+		.policy		= drbd_get_devices_nl_policy,
+		.maxattr	= DRBD_NLA_DEVICE_STATISTICS,
+		.flags		= GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= DRBD_ADM_GET_CONNECTIONS,
+		.dumpit		= drbd_nl_get_connections_dumpit,
+		.done		= drbd_adm_dump_connections_done,
+		.policy		= drbd_get_connections_nl_policy,
+		.maxattr	= DRBD_NLA_CONNECTION_STATISTICS,
+		.flags		= GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= DRBD_ADM_GET_PEER_DEVICES,
+		.dumpit		= drbd_nl_get_peer_devices_dumpit,
+		.done		= drbd_adm_dump_peer_devices_done,
+		.policy		= drbd_get_peer_devices_nl_policy,
+		.maxattr	= DRBD_NLA_PEER_DEVICE_STATISTICS,
+		.flags		= GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= DRBD_ADM_GET_INITIAL_STATE,
+		.dumpit		= drbd_nl_get_initial_state_dumpit,
+		.policy		= drbd_get_initial_state_nl_policy,
+		.maxattr	= DRBD_NLA_CFG_CONTEXT,
+		.flags		= GENL_CMD_CAP_DUMP,
+	},
+};
+
+static const struct genl_multicast_group drbd_nl_mcgrps[] = {
+	[DRBD_NLGRP_EVENTS] = { "events", },
+};
+
+static int __drbd_cfg_context_from_attrs(struct drbd_cfg_context *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR;
+	struct nlattr *tla = info->attrs[DRBD_NLA_CFG_CONTEXT];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_drbd_cfg_context_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME];
+	if (nla && s)
+		s->ctx_volume = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME];
+	if (nla && s)
+		s->ctx_resource_name_len = nla_strscpy(s->ctx_resource_name, nla, 128);
+
+	nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR];
+	if (nla && s)
+		s->ctx_my_addr_len = nla_memcpy(s->ctx_my_addr, nla, 128);
+
+	nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR];
+	if (nla && s)
+		s->ctx_peer_addr_len = nla_memcpy(s->ctx_peer_addr, nla, 128);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int drbd_cfg_context_from_attrs(struct drbd_cfg_context *s,
+				struct genl_info *info)
+{
+	return __drbd_cfg_context_from_attrs(s, NULL, info);
+}
+
+int drbd_cfg_context_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __drbd_cfg_context_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __disk_conf_from_attrs(struct disk_conf *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_DISK_CONF_DISABLE_WRITE_SAME;
+	struct nlattr *tla = info->attrs[DRBD_NLA_DISK_CONF];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_DISK_CONF_DISABLE_WRITE_SAME + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_disk_conf_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_DISK_CONF_BACKING_DEV];
+	if (nla) {
+		if (s)
+			s->backing_dev_len = nla_strscpy(s->backing_dev, nla, 128);
+	} else {
+		pr_info("<< missing required attr: backing_dev\n");
+		err = -ENOMSG;
+	}
+
+	nla = ntb[DRBD_A_DISK_CONF_META_DEV];
+	if (nla) {
+		if (s)
+			s->meta_dev_len = nla_strscpy(s->meta_dev, nla, 128);
+	} else {
+		pr_info("<< missing required attr: meta_dev\n");
+		err = -ENOMSG;
+	}
+
+	nla = ntb[DRBD_A_DISK_CONF_META_DEV_IDX];
+	if (nla) {
+		if (s)
+			s->meta_dev_idx = nla_get_s32(nla);
+	} else {
+		pr_info("<< missing required attr: meta_dev_idx\n");
+		err = -ENOMSG;
+	}
+
+	nla = ntb[DRBD_A_DISK_CONF_DISK_SIZE];
+	if (nla && s)
+		s->disk_size = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_MAX_BIO_BVECS];
+	if (nla && s)
+		s->max_bio_bvecs = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_ON_IO_ERROR];
+	if (nla && s)
+		s->on_io_error = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_FENCING];
+	if (nla && s)
+		s->fencing = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_RESYNC_RATE];
+	if (nla && s)
+		s->resync_rate = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_RESYNC_AFTER];
+	if (nla && s)
+		s->resync_after = nla_get_s32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_AL_EXTENTS];
+	if (nla && s)
+		s->al_extents = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_C_PLAN_AHEAD];
+	if (nla && s)
+		s->c_plan_ahead = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_C_DELAY_TARGET];
+	if (nla && s)
+		s->c_delay_target = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_C_FILL_TARGET];
+	if (nla && s)
+		s->c_fill_target = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_C_MAX_RATE];
+	if (nla && s)
+		s->c_max_rate = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_C_MIN_RATE];
+	if (nla && s)
+		s->c_min_rate = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_DISK_BARRIER];
+	if (nla && s)
+		s->disk_barrier = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_DISK_FLUSHES];
+	if (nla && s)
+		s->disk_flushes = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_DISK_DRAIN];
+	if (nla && s)
+		s->disk_drain = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_MD_FLUSHES];
+	if (nla && s)
+		s->md_flushes = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_DISK_TIMEOUT];
+	if (nla && s)
+		s->disk_timeout = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_READ_BALANCING];
+	if (nla && s)
+		s->read_balancing = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_AL_UPDATES];
+	if (nla && s)
+		s->al_updates = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_DISCARD_ZEROES_IF_ALIGNED];
+	if (nla && s)
+		s->discard_zeroes_if_aligned = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_RS_DISCARD_GRANULARITY];
+	if (nla && s)
+		s->rs_discard_granularity = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DISK_CONF_DISABLE_WRITE_SAME];
+	if (nla && s)
+		s->disable_write_same = nla_get_u8(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int disk_conf_from_attrs(struct disk_conf *s,
+				struct genl_info *info)
+{
+	return __disk_conf_from_attrs(s, NULL, info);
+}
+
+int disk_conf_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __disk_conf_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __res_opts_from_attrs(struct res_opts *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_RES_OPTS_ON_NO_DATA;
+	struct nlattr *tla = info->attrs[DRBD_NLA_RESOURCE_OPTS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_RES_OPTS_ON_NO_DATA + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_res_opts_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_RES_OPTS_CPU_MASK];
+	if (nla && s)
+		s->cpu_mask_len = nla_strscpy(s->cpu_mask, nla, DRBD_CPU_MASK_SIZE);
+
+	nla = ntb[DRBD_A_RES_OPTS_ON_NO_DATA];
+	if (nla && s)
+		s->on_no_data = nla_get_u32(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int res_opts_from_attrs(struct res_opts *s,
+				struct genl_info *info)
+{
+	return __res_opts_from_attrs(s, NULL, info);
+}
+
+int res_opts_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __res_opts_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __net_conf_from_attrs(struct net_conf *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_NET_CONF_SOCK_CHECK_TIMEO;
+	struct nlattr *tla = info->attrs[DRBD_NLA_NET_CONF];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_NET_CONF_SOCK_CHECK_TIMEO + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_net_conf_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_NET_CONF_SHARED_SECRET];
+	if (nla && s)
+		s->shared_secret_len = nla_strscpy(s->shared_secret, nla, SHARED_SECRET_MAX);
+
+	nla = ntb[DRBD_A_NET_CONF_CRAM_HMAC_ALG];
+	if (nla && s)
+		s->cram_hmac_alg_len = nla_strscpy(s->cram_hmac_alg, nla, SHARED_SECRET_MAX);
+
+	nla = ntb[DRBD_A_NET_CONF_INTEGRITY_ALG];
+	if (nla && s)
+		s->integrity_alg_len = nla_strscpy(s->integrity_alg, nla, SHARED_SECRET_MAX);
+
+	nla = ntb[DRBD_A_NET_CONF_VERIFY_ALG];
+	if (nla && s)
+		s->verify_alg_len = nla_strscpy(s->verify_alg, nla, SHARED_SECRET_MAX);
+
+	nla = ntb[DRBD_A_NET_CONF_CSUMS_ALG];
+	if (nla && s)
+		s->csums_alg_len = nla_strscpy(s->csums_alg, nla, SHARED_SECRET_MAX);
+
+	nla = ntb[DRBD_A_NET_CONF_WIRE_PROTOCOL];
+	if (nla && s)
+		s->wire_protocol = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_CONNECT_INT];
+	if (nla && s)
+		s->connect_int = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_TIMEOUT];
+	if (nla && s)
+		s->timeout = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_PING_INT];
+	if (nla && s)
+		s->ping_int = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_PING_TIMEO];
+	if (nla && s)
+		s->ping_timeo = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_SNDBUF_SIZE];
+	if (nla && s)
+		s->sndbuf_size = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_RCVBUF_SIZE];
+	if (nla && s)
+		s->rcvbuf_size = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_KO_COUNT];
+	if (nla && s)
+		s->ko_count = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_MAX_BUFFERS];
+	if (nla && s)
+		s->max_buffers = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_MAX_EPOCH_SIZE];
+	if (nla && s)
+		s->max_epoch_size = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_UNPLUG_WATERMARK];
+	if (nla && s)
+		s->unplug_watermark = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_AFTER_SB_0P];
+	if (nla && s)
+		s->after_sb_0p = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_AFTER_SB_1P];
+	if (nla && s)
+		s->after_sb_1p = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_AFTER_SB_2P];
+	if (nla && s)
+		s->after_sb_2p = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_RR_CONFLICT];
+	if (nla && s)
+		s->rr_conflict = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_ON_CONGESTION];
+	if (nla && s)
+		s->on_congestion = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_CONG_FILL];
+	if (nla && s)
+		s->cong_fill = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_CONG_EXTENTS];
+	if (nla && s)
+		s->cong_extents = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_TWO_PRIMARIES];
+	if (nla && s)
+		s->two_primaries = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_DISCARD_MY_DATA];
+	if (nla && s)
+		s->discard_my_data = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_TCP_CORK];
+	if (nla && s)
+		s->tcp_cork = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_ALWAYS_ASBP];
+	if (nla && s)
+		s->always_asbp = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_TENTATIVE];
+	if (nla && s)
+		s->tentative = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_USE_RLE];
+	if (nla && s)
+		s->use_rle = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_CSUMS_AFTER_CRASH_ONLY];
+	if (nla && s)
+		s->csums_after_crash_only = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_NET_CONF_SOCK_CHECK_TIMEO];
+	if (nla && s)
+		s->sock_check_timeo = nla_get_u32(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int net_conf_from_attrs(struct net_conf *s,
+				struct genl_info *info)
+{
+	return __net_conf_from_attrs(s, NULL, info);
+}
+
+int net_conf_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __net_conf_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __set_role_parms_from_attrs(struct set_role_parms *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE;
+	struct nlattr *tla = info->attrs[DRBD_NLA_SET_ROLE_PARMS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_set_role_parms_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE];
+	if (nla && s)
+		s->assume_uptodate = nla_get_u8(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int set_role_parms_from_attrs(struct set_role_parms *s,
+				struct genl_info *info)
+{
+	return __set_role_parms_from_attrs(s, NULL, info);
+}
+
+int set_role_parms_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __set_role_parms_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __resize_parms_from_attrs(struct resize_parms *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE;
+	struct nlattr *tla = info->attrs[DRBD_NLA_RESIZE_PARMS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_resize_parms_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_RESIZE_PARMS_RESIZE_SIZE];
+	if (nla && s)
+		s->resize_size = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_RESIZE_PARMS_RESIZE_FORCE];
+	if (nla && s)
+		s->resize_force = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_RESIZE_PARMS_NO_RESYNC];
+	if (nla && s)
+		s->no_resync = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_RESIZE_PARMS_AL_STRIPES];
+	if (nla && s)
+		s->al_stripes = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE];
+	if (nla && s)
+		s->al_stripe_size = nla_get_u32(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int resize_parms_from_attrs(struct resize_parms *s,
+				struct genl_info *info)
+{
+	return __resize_parms_from_attrs(s, NULL, info);
+}
+
+int resize_parms_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __resize_parms_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __start_ov_parms_from_attrs(struct start_ov_parms *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_START_OV_PARMS_OV_STOP_SECTOR;
+	struct nlattr *tla = info->attrs[DRBD_NLA_START_OV_PARMS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_START_OV_PARMS_OV_STOP_SECTOR + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_start_ov_parms_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_START_OV_PARMS_OV_START_SECTOR];
+	if (nla && s)
+		s->ov_start_sector = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_START_OV_PARMS_OV_STOP_SECTOR];
+	if (nla && s)
+		s->ov_stop_sector = nla_get_u64(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int start_ov_parms_from_attrs(struct start_ov_parms *s,
+				struct genl_info *info)
+{
+	return __start_ov_parms_from_attrs(s, NULL, info);
+}
+
+int start_ov_parms_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __start_ov_parms_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __new_c_uuid_parms_from_attrs(struct new_c_uuid_parms *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM;
+	struct nlattr *tla = info->attrs[DRBD_NLA_NEW_C_UUID_PARMS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_new_c_uuid_parms_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM];
+	if (nla && s)
+		s->clear_bm = nla_get_u8(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int new_c_uuid_parms_from_attrs(struct new_c_uuid_parms *s,
+				struct genl_info *info)
+{
+	return __new_c_uuid_parms_from_attrs(s, NULL, info);
+}
+
+int new_c_uuid_parms_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __new_c_uuid_parms_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __disconnect_parms_from_attrs(struct disconnect_parms *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT;
+	struct nlattr *tla = info->attrs[DRBD_NLA_DISCONNECT_PARMS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_disconnect_parms_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT];
+	if (nla && s)
+		s->force_disconnect = nla_get_u8(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int disconnect_parms_from_attrs(struct disconnect_parms *s,
+				struct genl_info *info)
+{
+	return __disconnect_parms_from_attrs(s, NULL, info);
+}
+
+int disconnect_parms_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __disconnect_parms_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __detach_parms_from_attrs(struct detach_parms *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_DETACH_PARMS_FORCE_DETACH;
+	struct nlattr *tla = info->attrs[DRBD_NLA_DETACH_PARMS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_DETACH_PARMS_FORCE_DETACH + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_detach_parms_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_DETACH_PARMS_FORCE_DETACH];
+	if (nla && s)
+		s->force_detach = nla_get_u8(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int detach_parms_from_attrs(struct detach_parms *s,
+				struct genl_info *info)
+{
+	return __detach_parms_from_attrs(s, NULL, info);
+}
+
+int detach_parms_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __detach_parms_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __resource_info_from_attrs(struct resource_info *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_RESOURCE_INFO_RES_SUSP_FEN;
+	struct nlattr *tla = info->attrs[DRBD_NLA_RESOURCE_INFO];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_RESOURCE_INFO_RES_SUSP_FEN + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_resource_info_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_RESOURCE_INFO_RES_ROLE];
+	if (nla && s)
+		s->res_role = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_RESOURCE_INFO_RES_SUSP];
+	if (nla && s)
+		s->res_susp = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_RESOURCE_INFO_RES_SUSP_NOD];
+	if (nla && s)
+		s->res_susp_nod = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_RESOURCE_INFO_RES_SUSP_FEN];
+	if (nla && s)
+		s->res_susp_fen = nla_get_u8(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int resource_info_from_attrs(struct resource_info *s,
+				struct genl_info *info)
+{
+	return __resource_info_from_attrs(s, NULL, info);
+}
+
+int resource_info_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __resource_info_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __device_info_from_attrs(struct device_info *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_DEVICE_INFO_DEV_DISK_STATE;
+	struct nlattr *tla = info->attrs[DRBD_NLA_DEVICE_INFO];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_DEVICE_INFO_DEV_DISK_STATE + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_device_info_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_DEVICE_INFO_DEV_DISK_STATE];
+	if (nla && s)
+		s->dev_disk_state = nla_get_u32(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int device_info_from_attrs(struct device_info *s,
+				struct genl_info *info)
+{
+	return __device_info_from_attrs(s, NULL, info);
+}
+
+int device_info_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __device_info_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __connection_info_from_attrs(struct connection_info *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_CONNECTION_INFO_CONN_ROLE;
+	struct nlattr *tla = info->attrs[DRBD_NLA_CONNECTION_INFO];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_CONNECTION_INFO_CONN_ROLE + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_connection_info_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_CONNECTION_INFO_CONN_CONNECTION_STATE];
+	if (nla && s)
+		s->conn_connection_state = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_CONNECTION_INFO_CONN_ROLE];
+	if (nla && s)
+		s->conn_role = nla_get_u32(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int connection_info_from_attrs(struct connection_info *s,
+				struct genl_info *info)
+{
+	return __connection_info_from_attrs(s, NULL, info);
+}
+
+int connection_info_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __connection_info_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __peer_device_info_from_attrs(struct peer_device_info *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY;
+	struct nlattr *tla = info->attrs[DRBD_NLA_PEER_DEVICE_INFO];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_peer_device_info_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_REPL_STATE];
+	if (nla && s)
+		s->peer_repl_state = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_DISK_STATE];
+	if (nla && s)
+		s->peer_disk_state = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_USER];
+	if (nla && s)
+		s->peer_resync_susp_user = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_PEER];
+	if (nla && s)
+		s->peer_resync_susp_peer = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY];
+	if (nla && s)
+		s->peer_resync_susp_dependency = nla_get_u32(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int peer_device_info_from_attrs(struct peer_device_info *s,
+				struct genl_info *info)
+{
+	return __peer_device_info_from_attrs(s, NULL, info);
+}
+
+int peer_device_info_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __peer_device_info_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __resource_statistics_from_attrs(struct resource_statistics *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING;
+	struct nlattr *tla = info->attrs[DRBD_NLA_RESOURCE_STATISTICS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_resource_statistics_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING];
+	if (nla && s)
+		s->res_stat_write_ordering = nla_get_u32(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int resource_statistics_from_attrs(struct resource_statistics *s,
+				struct genl_info *info)
+{
+	return __resource_statistics_from_attrs(s, NULL, info);
+}
+
+int resource_statistics_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __resource_statistics_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __device_statistics_from_attrs(struct device_statistics *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS;
+	struct nlattr *tla = info->attrs[DRBD_NLA_DEVICE_STATISTICS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_device_statistics_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_SIZE];
+	if (nla && s)
+		s->dev_size = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_READ];
+	if (nla && s)
+		s->dev_read = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_WRITE];
+	if (nla && s)
+		s->dev_write = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_AL_WRITES];
+	if (nla && s)
+		s->dev_al_writes = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_BM_WRITES];
+	if (nla && s)
+		s->dev_bm_writes = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_UPPER_PENDING];
+	if (nla && s)
+		s->dev_upper_pending = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_LOWER_PENDING];
+	if (nla && s)
+		s->dev_lower_pending = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_UPPER_BLOCKED];
+	if (nla && s)
+		s->dev_upper_blocked = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_LOWER_BLOCKED];
+	if (nla && s)
+		s->dev_lower_blocked = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_AL_SUSPENDED];
+	if (nla && s)
+		s->dev_al_suspended = nla_get_u8(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_EXPOSED_DATA_UUID];
+	if (nla && s)
+		s->dev_exposed_data_uuid = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_CURRENT_UUID];
+	if (nla && s)
+		s->dev_current_uuid = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_DISK_FLAGS];
+	if (nla && s)
+		s->dev_disk_flags = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS];
+	if (nla && s)
+		s->history_uuids_len = nla_memcpy(s->history_uuids, nla, DRBD_NL_HISTORY_UUIDS_SIZE);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int device_statistics_from_attrs(struct device_statistics *s,
+				struct genl_info *info)
+{
+	return __device_statistics_from_attrs(s, NULL, info);
+}
+
+int device_statistics_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __device_statistics_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __connection_statistics_from_attrs(struct connection_statistics *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED;
+	struct nlattr *tla = info->attrs[DRBD_NLA_CONNECTION_STATISTICS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_connection_statistics_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED];
+	if (nla && s)
+		s->conn_congested = nla_get_u8(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int connection_statistics_from_attrs(struct connection_statistics *s,
+				struct genl_info *info)
+{
+	return __connection_statistics_from_attrs(s, NULL, info);
+}
+
+int connection_statistics_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __connection_statistics_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+static int __peer_device_statistics_from_attrs(struct peer_device_statistics *s,
+		struct nlattr ***ret_nested_attribute_table,
+		struct genl_info *info)
+{
+	const int maxtype = DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS;
+	struct nlattr *tla = info->attrs[DRBD_NLA_PEER_DEVICE_STATISTICS];
+	struct nlattr **ntb;
+	struct nlattr *nla;
+	int err = 0;
+
+	if (ret_nested_attribute_table)
+		*ret_nested_attribute_table = NULL;
+	if (!tla)
+		return -ENOMSG;
+	ntb = kcalloc(DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS + 1, sizeof(*ntb), GFP_KERNEL);
+	if (!ntb)
+		return -ENOMEM;
+	err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_peer_device_statistics_nl_policy, NULL);
+	if (err)
+		goto out;
+
+	nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RECEIVED];
+	if (nla && s)
+		s->peer_dev_received = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_SENT];
+	if (nla && s)
+		s->peer_dev_sent = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_PENDING];
+	if (nla && s)
+		s->peer_dev_pending = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_UNACKED];
+	if (nla && s)
+		s->peer_dev_unacked = nla_get_u32(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_OUT_OF_SYNC];
+	if (nla && s)
+		s->peer_dev_out_of_sync = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RESYNC_FAILED];
+	if (nla && s)
+		s->peer_dev_resync_failed = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_BITMAP_UUID];
+	if (nla && s)
+		s->peer_dev_bitmap_uuid = nla_get_u64(nla);
+
+	nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS];
+	if (nla && s)
+		s->peer_dev_flags = nla_get_u32(nla);
+
+out:
+	if (ret_nested_attribute_table && (!err || err == -ENOMSG))
+		*ret_nested_attribute_table = ntb;
+	else
+		kfree(ntb);
+	return err;
+}
+
+int peer_device_statistics_from_attrs(struct peer_device_statistics *s,
+				struct genl_info *info)
+{
+	return __peer_device_statistics_from_attrs(s, NULL, info);
+}
+
+int peer_device_statistics_ntb_from_attrs(
+			struct nlattr ***ret_nested_attribute_table,
+			struct genl_info *info)
+{
+	return __peer_device_statistics_from_attrs(NULL, ret_nested_attribute_table, info);
+}
+
+int drbd_cfg_reply_to_skb(struct sk_buff *skb, struct drbd_cfg_reply *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_CFG_REPLY);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put(skb, DRBD_A_DRBD_CFG_REPLY_INFO_TEXT, min_t(int, 0,
+			s->info_text_len + (s->info_text_len < 0)), s->info_text))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int drbd_cfg_context_to_skb(struct sk_buff *skb, struct drbd_cfg_context *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_CFG_CONTEXT);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME, s->ctx_volume))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME, min_t(int, 128,
+			s->ctx_resource_name_len + (s->ctx_resource_name_len < 128)), s->ctx_resource_name))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR, min_t(int, 128,
+			s->ctx_my_addr_len), s->ctx_my_addr))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR, min_t(int, 128,
+			s->ctx_peer_addr_len), s->ctx_peer_addr))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int disk_conf_to_skb(struct sk_buff *skb, struct disk_conf *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DISK_CONF);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put(skb, DRBD_A_DISK_CONF_BACKING_DEV, min_t(int, 128,
+			s->backing_dev_len + (s->backing_dev_len < 128)), s->backing_dev))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_DISK_CONF_META_DEV, min_t(int, 128,
+			s->meta_dev_len + (s->meta_dev_len < 128)), s->meta_dev))
+		goto nla_put_failure;
+	if (nla_put_s32(skb, DRBD_A_DISK_CONF_META_DEV_IDX, s->meta_dev_idx))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_DISK_CONF_DISK_SIZE, s->disk_size, 0))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_MAX_BIO_BVECS, s->max_bio_bvecs))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_ON_IO_ERROR, s->on_io_error))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_FENCING, s->fencing))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_RESYNC_RATE, s->resync_rate))
+		goto nla_put_failure;
+	if (nla_put_s32(skb, DRBD_A_DISK_CONF_RESYNC_AFTER, s->resync_after))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_AL_EXTENTS, s->al_extents))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_PLAN_AHEAD, s->c_plan_ahead))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_DELAY_TARGET, s->c_delay_target))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_FILL_TARGET, s->c_fill_target))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_MAX_RATE, s->c_max_rate))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_MIN_RATE, s->c_min_rate))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISK_BARRIER, s->disk_barrier))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISK_FLUSHES, s->disk_flushes))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISK_DRAIN, s->disk_drain))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DISK_CONF_MD_FLUSHES, s->md_flushes))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_DISK_TIMEOUT, s->disk_timeout))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_READ_BALANCING, s->read_balancing))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DISK_CONF_AL_UPDATES, s->al_updates))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISCARD_ZEROES_IF_ALIGNED, s->discard_zeroes_if_aligned))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DISK_CONF_RS_DISCARD_GRANULARITY, s->rs_discard_granularity))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISABLE_WRITE_SAME, s->disable_write_same))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int res_opts_to_skb(struct sk_buff *skb, struct res_opts *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_RESOURCE_OPTS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put(skb, DRBD_A_RES_OPTS_CPU_MASK, min_t(int, DRBD_CPU_MASK_SIZE,
+			s->cpu_mask_len + (s->cpu_mask_len < DRBD_CPU_MASK_SIZE)), s->cpu_mask))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_RES_OPTS_ON_NO_DATA, s->on_no_data))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int net_conf_to_skb(struct sk_buff *skb, struct net_conf *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_NET_CONF);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put(skb, DRBD_A_NET_CONF_SHARED_SECRET, min_t(int, SHARED_SECRET_MAX,
+			s->shared_secret_len + (s->shared_secret_len < SHARED_SECRET_MAX)), s->shared_secret))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_NET_CONF_CRAM_HMAC_ALG, min_t(int, SHARED_SECRET_MAX,
+			s->cram_hmac_alg_len + (s->cram_hmac_alg_len < SHARED_SECRET_MAX)), s->cram_hmac_alg))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_NET_CONF_INTEGRITY_ALG, min_t(int, SHARED_SECRET_MAX,
+			s->integrity_alg_len + (s->integrity_alg_len < SHARED_SECRET_MAX)), s->integrity_alg))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_NET_CONF_VERIFY_ALG, min_t(int, SHARED_SECRET_MAX,
+			s->verify_alg_len + (s->verify_alg_len < SHARED_SECRET_MAX)), s->verify_alg))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_NET_CONF_CSUMS_ALG, min_t(int, SHARED_SECRET_MAX,
+			s->csums_alg_len + (s->csums_alg_len < SHARED_SECRET_MAX)), s->csums_alg))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_WIRE_PROTOCOL, s->wire_protocol))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_CONNECT_INT, s->connect_int))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_TIMEOUT, s->timeout))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_PING_INT, s->ping_int))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_PING_TIMEO, s->ping_timeo))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_SNDBUF_SIZE, s->sndbuf_size))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_RCVBUF_SIZE, s->rcvbuf_size))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_KO_COUNT, s->ko_count))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_MAX_BUFFERS, s->max_buffers))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_MAX_EPOCH_SIZE, s->max_epoch_size))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_UNPLUG_WATERMARK, s->unplug_watermark))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_AFTER_SB_0P, s->after_sb_0p))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_AFTER_SB_1P, s->after_sb_1p))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_AFTER_SB_2P, s->after_sb_2p))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_RR_CONFLICT, s->rr_conflict))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_ON_CONGESTION, s->on_congestion))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_CONG_FILL, s->cong_fill))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_CONG_EXTENTS, s->cong_extents))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_NET_CONF_TWO_PRIMARIES, s->two_primaries))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_NET_CONF_DISCARD_MY_DATA, s->discard_my_data))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_NET_CONF_TCP_CORK, s->tcp_cork))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_NET_CONF_ALWAYS_ASBP, s->always_asbp))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_NET_CONF_TENTATIVE, s->tentative))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_NET_CONF_USE_RLE, s->use_rle))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_NET_CONF_CSUMS_AFTER_CRASH_ONLY, s->csums_after_crash_only))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_NET_CONF_SOCK_CHECK_TIMEO, s->sock_check_timeo))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int set_role_parms_to_skb(struct sk_buff *skb, struct set_role_parms *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_SET_ROLE_PARMS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u8(skb, DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE, s->assume_uptodate))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int resize_parms_to_skb(struct sk_buff *skb, struct resize_parms *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_RESIZE_PARMS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u64_64bit(skb, DRBD_A_RESIZE_PARMS_RESIZE_SIZE, s->resize_size, 0))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_RESIZE_PARMS_RESIZE_FORCE, s->resize_force))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_RESIZE_PARMS_NO_RESYNC, s->no_resync))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_RESIZE_PARMS_AL_STRIPES, s->al_stripes))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE, s->al_stripe_size))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int state_info_to_skb(struct sk_buff *skb, struct state_info *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_STATE_INFO);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_SIB_REASON, s->sib_reason))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_CURRENT_STATE, s->current_state))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_CAPACITY, s->capacity, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_ED_UUID, s->ed_uuid, 0))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_PREV_STATE, s->prev_state))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_NEW_STATE, s->new_state))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_STATE_INFO_UUIDS, min_t(int, DRBD_NL_UUIDS_SIZE,
+			s->uuids_len), s->uuids))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_DISK_FLAGS, s->disk_flags))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_TOTAL, s->bits_total, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_OOS, s->bits_oos, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_RS_TOTAL, s->bits_rs_total, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_RS_FAILED, s->bits_rs_failed, 0))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_STATE_INFO_HELPER, min_t(int, 32,
+			s->helper_len + (s->helper_len < 32)), s->helper))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_HELPER_EXIT_CODE, s->helper_exit_code))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_SEND_CNT, s->send_cnt, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_RECV_CNT, s->recv_cnt, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_READ_CNT, s->read_cnt, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_WRIT_CNT, s->writ_cnt, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_AL_WRIT_CNT, s->al_writ_cnt, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BM_WRIT_CNT, s->bm_writ_cnt, 0))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_AP_BIO_CNT, s->ap_bio_cnt))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_AP_PENDING_CNT, s->ap_pending_cnt))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_STATE_INFO_RS_PENDING_CNT, s->rs_pending_cnt))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int start_ov_parms_to_skb(struct sk_buff *skb, struct start_ov_parms *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_START_OV_PARMS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u64_64bit(skb, DRBD_A_START_OV_PARMS_OV_START_SECTOR, s->ov_start_sector, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_START_OV_PARMS_OV_STOP_SECTOR, s->ov_stop_sector, 0))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int new_c_uuid_parms_to_skb(struct sk_buff *skb, struct new_c_uuid_parms *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_NEW_C_UUID_PARMS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u8(skb, DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM, s->clear_bm))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int timeout_parms_to_skb(struct sk_buff *skb, struct timeout_parms *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_TIMEOUT_PARMS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, DRBD_A_TIMEOUT_PARMS_TIMEOUT_TYPE, s->timeout_type))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int disconnect_parms_to_skb(struct sk_buff *skb, struct disconnect_parms *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DISCONNECT_PARMS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u8(skb, DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT, s->force_disconnect))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int detach_parms_to_skb(struct sk_buff *skb, struct detach_parms *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DETACH_PARMS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u8(skb, DRBD_A_DETACH_PARMS_FORCE_DETACH, s->force_detach))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int resource_info_to_skb(struct sk_buff *skb, struct resource_info *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_RESOURCE_INFO);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, DRBD_A_RESOURCE_INFO_RES_ROLE, s->res_role))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_RESOURCE_INFO_RES_SUSP, s->res_susp))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_RESOURCE_INFO_RES_SUSP_NOD, s->res_susp_nod))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_RESOURCE_INFO_RES_SUSP_FEN, s->res_susp_fen))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int device_info_to_skb(struct sk_buff *skb, struct device_info *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DEVICE_INFO);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, DRBD_A_DEVICE_INFO_DEV_DISK_STATE, s->dev_disk_state))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int connection_info_to_skb(struct sk_buff *skb, struct connection_info *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_CONNECTION_INFO);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, DRBD_A_CONNECTION_INFO_CONN_CONNECTION_STATE, s->conn_connection_state))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_CONNECTION_INFO_CONN_ROLE, s->conn_role))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int peer_device_info_to_skb(struct sk_buff *skb, struct peer_device_info *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_PEER_DEVICE_INFO);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_REPL_STATE, s->peer_repl_state))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_DISK_STATE, s->peer_disk_state))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_USER, s->peer_resync_susp_user))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_PEER, s->peer_resync_susp_peer))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY, s->peer_resync_susp_dependency))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int resource_statistics_to_skb(struct sk_buff *skb, struct resource_statistics *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_RESOURCE_STATISTICS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING, s->res_stat_write_ordering))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int device_statistics_to_skb(struct sk_buff *skb, struct device_statistics *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DEVICE_STATISTICS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_SIZE, s->dev_size, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_READ, s->dev_read, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_WRITE, s->dev_write, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_AL_WRITES, s->dev_al_writes, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_BM_WRITES, s->dev_bm_writes, 0))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DEVICE_STATISTICS_DEV_UPPER_PENDING, s->dev_upper_pending))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DEVICE_STATISTICS_DEV_LOWER_PENDING, s->dev_lower_pending))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DEVICE_STATISTICS_DEV_UPPER_BLOCKED, s->dev_upper_blocked))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DEVICE_STATISTICS_DEV_LOWER_BLOCKED, s->dev_lower_blocked))
+		goto nla_put_failure;
+	if (nla_put_u8(skb, DRBD_A_DEVICE_STATISTICS_DEV_AL_SUSPENDED, s->dev_al_suspended))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_EXPOSED_DATA_UUID, s->dev_exposed_data_uuid, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_CURRENT_UUID, s->dev_current_uuid, 0))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DEVICE_STATISTICS_DEV_DISK_FLAGS, s->dev_disk_flags))
+		goto nla_put_failure;
+	if (nla_put(skb, DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS, min_t(int, DRBD_NL_HISTORY_UUIDS_SIZE,
+			s->history_uuids_len), s->history_uuids))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int connection_statistics_to_skb(struct sk_buff *skb, struct connection_statistics *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_CONNECTION_STATISTICS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u8(skb, DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED, s->conn_congested))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int peer_device_statistics_to_skb(struct sk_buff *skb, struct peer_device_statistics *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_PEER_DEVICE_STATISTICS);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RECEIVED, s->peer_dev_received, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_SENT, s->peer_dev_sent, 0))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_PENDING, s->peer_dev_pending))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_UNACKED, s->peer_dev_unacked))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_OUT_OF_SYNC, s->peer_dev_out_of_sync, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RESYNC_FAILED, s->peer_dev_resync_failed, 0))
+		goto nla_put_failure;
+	if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_BITMAP_UUID, s->peer_dev_bitmap_uuid, 0))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS, s->peer_dev_flags))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int drbd_notification_header_to_skb(struct sk_buff *skb, struct drbd_notification_header *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_NOTIFICATION_HEADER);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, DRBD_A_DRBD_NOTIFICATION_HEADER_NH_TYPE, s->nh_type))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+int drbd_helper_info_to_skb(struct sk_buff *skb, struct drbd_helper_info *s)
+{
+	struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_HELPER);
+
+	if (!tla)
+		goto nla_put_failure;
+
+	if (nla_put(skb, DRBD_A_DRBD_HELPER_INFO_HELPER_NAME, min_t(int, 32,
+			s->helper_name_len + (s->helper_name_len < 32)), s->helper_name))
+		goto nla_put_failure;
+	if (nla_put_u32(skb, DRBD_A_DRBD_HELPER_INFO_HELPER_STATUS, s->helper_status))
+		goto nla_put_failure;
+
+	nla_nest_end(skb, tla);
+	return 0;
+
+nla_put_failure:
+	if (tla)
+		nla_nest_cancel(skb, tla);
+	return -EMSGSIZE;
+}
+
+void set_disk_conf_defaults(struct disk_conf *x)
+{
+	x->on_io_error = DRBD_ON_IO_ERROR_DEF;
+	x->fencing = DRBD_FENCING_DEF;
+	x->resync_rate = DRBD_RESYNC_RATE_DEF;
+	x->resync_after = DRBD_MINOR_NUMBER_DEF;
+	x->al_extents = DRBD_AL_EXTENTS_DEF;
+	x->c_plan_ahead = DRBD_C_PLAN_AHEAD_DEF;
+	x->c_delay_target = DRBD_C_DELAY_TARGET_DEF;
+	x->c_fill_target = DRBD_C_FILL_TARGET_DEF;
+	x->c_max_rate = DRBD_C_MAX_RATE_DEF;
+	x->c_min_rate = DRBD_C_MIN_RATE_DEF;
+	x->disk_barrier = DRBD_DISK_BARRIER_DEF;
+	x->disk_flushes = DRBD_DISK_FLUSHES_DEF;
+	x->disk_drain = DRBD_DISK_DRAIN_DEF;
+	x->md_flushes = DRBD_MD_FLUSHES_DEF;
+	x->disk_timeout = DRBD_DISK_TIMEOUT_DEF;
+	x->read_balancing = DRBD_READ_BALANCING_DEF;
+	x->al_updates = DRBD_AL_UPDATES_DEF;
+	x->discard_zeroes_if_aligned = DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF;
+	x->rs_discard_granularity = DRBD_RS_DISCARD_GRANULARITY_DEF;
+	x->disable_write_same = DRBD_DISABLE_WRITE_SAME_DEF;
+}
+
+void set_res_opts_defaults(struct res_opts *x)
+{
+	memset(x->cpu_mask, 0, sizeof(x->cpu_mask));
+	x->cpu_mask_len = 0;
+	x->on_no_data = DRBD_ON_NO_DATA_DEF;
+}
+
+void set_net_conf_defaults(struct net_conf *x)
+{
+	memset(x->shared_secret, 0, sizeof(x->shared_secret));
+	x->shared_secret_len = 0;
+	memset(x->cram_hmac_alg, 0, sizeof(x->cram_hmac_alg));
+	x->cram_hmac_alg_len = 0;
+	memset(x->integrity_alg, 0, sizeof(x->integrity_alg));
+	x->integrity_alg_len = 0;
+	memset(x->verify_alg, 0, sizeof(x->verify_alg));
+	x->verify_alg_len = 0;
+	memset(x->csums_alg, 0, sizeof(x->csums_alg));
+	x->csums_alg_len = 0;
+	x->wire_protocol = DRBD_PROTOCOL_DEF;
+	x->connect_int = DRBD_CONNECT_INT_DEF;
+	x->timeout = DRBD_TIMEOUT_DEF;
+	x->ping_int = DRBD_PING_INT_DEF;
+	x->ping_timeo = DRBD_PING_TIMEO_DEF;
+	x->sndbuf_size = DRBD_SNDBUF_SIZE_DEF;
+	x->rcvbuf_size = DRBD_RCVBUF_SIZE_DEF;
+	x->ko_count = DRBD_KO_COUNT_DEF;
+	x->max_buffers = DRBD_MAX_BUFFERS_DEF;
+	x->max_epoch_size = DRBD_MAX_EPOCH_SIZE_DEF;
+	x->unplug_watermark = DRBD_UNPLUG_WATERMARK_DEF;
+	x->after_sb_0p = DRBD_AFTER_SB_0P_DEF;
+	x->after_sb_1p = DRBD_AFTER_SB_1P_DEF;
+	x->after_sb_2p = DRBD_AFTER_SB_2P_DEF;
+	x->rr_conflict = DRBD_RR_CONFLICT_DEF;
+	x->on_congestion = DRBD_ON_CONGESTION_DEF;
+	x->cong_fill = DRBD_CONG_FILL_DEF;
+	x->cong_extents = DRBD_CONG_EXTENTS_DEF;
+	x->two_primaries = DRBD_ALLOW_TWO_PRIMARIES_DEF;
+	x->tcp_cork = DRBD_TCP_CORK_DEF;
+	x->always_asbp = DRBD_ALWAYS_ASBP_DEF;
+	x->use_rle = DRBD_USE_RLE_DEF;
+	x->csums_after_crash_only = DRBD_CSUMS_AFTER_CRASH_ONLY_DEF;
+	x->sock_check_timeo = DRBD_SOCKET_CHECK_TIMEO_DEF;
+}
+
+void set_resize_parms_defaults(struct resize_parms *x)
+{
+	x->al_stripes = DRBD_AL_STRIPES_DEF;
+	x->al_stripe_size = DRBD_AL_STRIPE_SIZE_DEF;
+}
diff --git a/drivers/block/drbd/drbd_nl_gen.h b/drivers/block/drbd/drbd_nl_gen.h
new file mode 100644
index 0000000000000..f2140dd1ac4e1
--- /dev/null
+++ b/drivers/block/drbd/drbd_nl_gen.h
@@ -0,0 +1,395 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+
+#ifndef _LINUX_DRBD_GEN_H
+#define _LINUX_DRBD_GEN_H
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include <uapi/linux/drbd_genl.h>
+#include <linux/drbd.h>
+#include <linux/drbd_limits.h>
+
+/* Common nested types */
+extern const struct nla_policy drbd_connection_info_nl_policy[DRBD_A_CONNECTION_INFO_CONN_ROLE + 1];
+extern const struct nla_policy drbd_connection_statistics_nl_policy[DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED + 1];
+extern const struct nla_policy drbd_detach_parms_nl_policy[DRBD_A_DETACH_PARMS_FORCE_DETACH + 1];
+extern const struct nla_policy drbd_device_info_nl_policy[DRBD_A_DEVICE_INFO_DEV_DISK_STATE + 1];
+extern const struct nla_policy drbd_device_statistics_nl_policy[DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS + 1];
+extern const struct nla_policy drbd_disconnect_parms_nl_policy[DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT + 1];
+extern const struct nla_policy drbd_disk_conf_nl_policy[DRBD_A_DISK_CONF_DISABLE_WRITE_SAME + 1];
+extern const struct nla_policy drbd_drbd_cfg_context_nl_policy[DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR + 1];
+extern const struct nla_policy drbd_net_conf_nl_policy[DRBD_A_NET_CONF_SOCK_CHECK_TIMEO + 1];
+extern const struct nla_policy drbd_new_c_uuid_parms_nl_policy[DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM + 1];
+extern const struct nla_policy drbd_peer_device_info_nl_policy[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY + 1];
+extern const struct nla_policy drbd_peer_device_statistics_nl_policy[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS + 1];
+extern const struct nla_policy drbd_res_opts_nl_policy[DRBD_A_RES_OPTS_ON_NO_DATA + 1];
+extern const struct nla_policy drbd_resize_parms_nl_policy[DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE + 1];
+extern const struct nla_policy drbd_resource_info_nl_policy[DRBD_A_RESOURCE_INFO_RES_SUSP_FEN + 1];
+extern const struct nla_policy drbd_resource_statistics_nl_policy[DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING + 1];
+extern const struct nla_policy drbd_set_role_parms_nl_policy[DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE + 1];
+extern const struct nla_policy drbd_start_ov_parms_nl_policy[DRBD_A_START_OV_PARMS_OV_STOP_SECTOR + 1];
+
+/* Ops table for drbd */
+extern const struct genl_split_ops drbd_nl_ops[32];
+
+int drbd_pre_doit(const struct genl_split_ops *ops, struct sk_buff *skb,
+		  struct genl_info *info);
+void
+drbd_post_doit(const struct genl_split_ops *ops, struct sk_buff *skb,
+	       struct genl_info *info);
+int drbd_adm_dump_devices_done(struct netlink_callback *cb);
+int drbd_adm_dump_connections_done(struct netlink_callback *cb);
+int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb);
+
+int drbd_nl_get_status_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_get_status_dumpit(struct sk_buff *skb, struct netlink_callback *cb);
+int drbd_nl_new_minor_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_del_minor_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_new_resource_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_del_resource_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_resource_opts_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_connect_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_disconnect_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_attach_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_resize_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_primary_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_secondary_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_new_c_uuid_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_start_ov_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_detach_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_invalidate_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_inval_peer_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_pause_sync_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_resume_sync_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_suspend_io_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_resume_io_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_outdate_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_get_timeout_type_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_down_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_chg_disk_opts_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_chg_net_opts_doit(struct sk_buff *skb, struct genl_info *info);
+int drbd_nl_get_resources_dumpit(struct sk_buff *skb,
+				 struct netlink_callback *cb);
+int drbd_nl_get_devices_dumpit(struct sk_buff *skb,
+			       struct netlink_callback *cb);
+int drbd_nl_get_connections_dumpit(struct sk_buff *skb,
+				   struct netlink_callback *cb);
+int drbd_nl_get_peer_devices_dumpit(struct sk_buff *skb,
+				    struct netlink_callback *cb);
+int drbd_nl_get_initial_state_dumpit(struct sk_buff *skb,
+				     struct netlink_callback *cb);
+
+enum {
+	DRBD_NLGRP_EVENTS,
+};
+
+struct drbd_cfg_reply {
+	char info_text[0];
+	__u32 info_text_len;
+};
+
+struct drbd_cfg_context {
+	__u32 ctx_volume;
+	char ctx_resource_name[128];
+	__u32 ctx_resource_name_len;
+	char ctx_my_addr[128];
+	__u32 ctx_my_addr_len;
+	char ctx_peer_addr[128];
+	__u32 ctx_peer_addr_len;
+};
+
+struct disk_conf {
+	char backing_dev[128];
+	__u32 backing_dev_len;
+	char meta_dev[128];
+	__u32 meta_dev_len;
+	__s32 meta_dev_idx;
+	__u64 disk_size;
+	__u32 max_bio_bvecs;
+	__u32 on_io_error;
+	__u32 fencing;
+	__u32 resync_rate;
+	__s32 resync_after;
+	__u32 al_extents;
+	__u32 c_plan_ahead;
+	__u32 c_delay_target;
+	__u32 c_fill_target;
+	__u32 c_max_rate;
+	__u32 c_min_rate;
+	unsigned char disk_barrier;
+	unsigned char disk_flushes;
+	unsigned char disk_drain;
+	unsigned char md_flushes;
+	__u32 disk_timeout;
+	__u32 read_balancing;
+	unsigned char al_updates;
+	unsigned char discard_zeroes_if_aligned;
+	__u32 rs_discard_granularity;
+	unsigned char disable_write_same;
+};
+
+struct res_opts {
+	char cpu_mask[DRBD_CPU_MASK_SIZE];
+	__u32 cpu_mask_len;
+	__u32 on_no_data;
+};
+
+struct net_conf {
+	char shared_secret[SHARED_SECRET_MAX];
+	__u32 shared_secret_len;
+	char cram_hmac_alg[SHARED_SECRET_MAX];
+	__u32 cram_hmac_alg_len;
+	char integrity_alg[SHARED_SECRET_MAX];
+	__u32 integrity_alg_len;
+	char verify_alg[SHARED_SECRET_MAX];
+	__u32 verify_alg_len;
+	char csums_alg[SHARED_SECRET_MAX];
+	__u32 csums_alg_len;
+	__u32 wire_protocol;
+	__u32 connect_int;
+	__u32 timeout;
+	__u32 ping_int;
+	__u32 ping_timeo;
+	__u32 sndbuf_size;
+	__u32 rcvbuf_size;
+	__u32 ko_count;
+	__u32 max_buffers;
+	__u32 max_epoch_size;
+	__u32 unplug_watermark;
+	__u32 after_sb_0p;
+	__u32 after_sb_1p;
+	__u32 after_sb_2p;
+	__u32 rr_conflict;
+	__u32 on_congestion;
+	__u32 cong_fill;
+	__u32 cong_extents;
+	unsigned char two_primaries;
+	unsigned char discard_my_data;
+	unsigned char tcp_cork;
+	unsigned char always_asbp;
+	unsigned char tentative;
+	unsigned char use_rle;
+	unsigned char csums_after_crash_only;
+	__u32 sock_check_timeo;
+};
+
+struct set_role_parms {
+	unsigned char assume_uptodate;
+};
+
+struct resize_parms {
+	__u64 resize_size;
+	unsigned char resize_force;
+	unsigned char no_resync;
+	__u32 al_stripes;
+	__u32 al_stripe_size;
+};
+
+struct state_info {
+	__u32 sib_reason;
+	__u32 current_state;
+	__u64 capacity;
+	__u64 ed_uuid;
+	__u32 prev_state;
+	__u32 new_state;
+	char uuids[DRBD_NL_UUIDS_SIZE];
+	__u32 uuids_len;
+	__u32 disk_flags;
+	__u64 bits_total;
+	__u64 bits_oos;
+	__u64 bits_rs_total;
+	__u64 bits_rs_failed;
+	char helper[32];
+	__u32 helper_len;
+	__u32 helper_exit_code;
+	__u64 send_cnt;
+	__u64 recv_cnt;
+	__u64 read_cnt;
+	__u64 writ_cnt;
+	__u64 al_writ_cnt;
+	__u64 bm_writ_cnt;
+	__u32 ap_bio_cnt;
+	__u32 ap_pending_cnt;
+	__u32 rs_pending_cnt;
+};
+
+struct start_ov_parms {
+	__u64 ov_start_sector;
+	__u64 ov_stop_sector;
+};
+
+struct new_c_uuid_parms {
+	unsigned char clear_bm;
+};
+
+struct timeout_parms {
+	__u32 timeout_type;
+};
+
+struct disconnect_parms {
+	unsigned char force_disconnect;
+};
+
+struct detach_parms {
+	unsigned char force_detach;
+};
+
+struct resource_info {
+	__u32 res_role;
+	unsigned char res_susp;
+	unsigned char res_susp_nod;
+	unsigned char res_susp_fen;
+};
+
+struct device_info {
+	__u32 dev_disk_state;
+};
+
+struct connection_info {
+	__u32 conn_connection_state;
+	__u32 conn_role;
+};
+
+struct peer_device_info {
+	__u32 peer_repl_state;
+	__u32 peer_disk_state;
+	__u32 peer_resync_susp_user;
+	__u32 peer_resync_susp_peer;
+	__u32 peer_resync_susp_dependency;
+};
+
+struct resource_statistics {
+	__u32 res_stat_write_ordering;
+};
+
+struct device_statistics {
+	__u64 dev_size;
+	__u64 dev_read;
+	__u64 dev_write;
+	__u64 dev_al_writes;
+	__u64 dev_bm_writes;
+	__u32 dev_upper_pending;
+	__u32 dev_lower_pending;
+	unsigned char dev_upper_blocked;
+	unsigned char dev_lower_blocked;
+	unsigned char dev_al_suspended;
+	__u64 dev_exposed_data_uuid;
+	__u64 dev_current_uuid;
+	__u32 dev_disk_flags;
+	char history_uuids[DRBD_NL_HISTORY_UUIDS_SIZE];
+	__u32 history_uuids_len;
+};
+
+struct connection_statistics {
+	unsigned char conn_congested;
+};
+
+struct peer_device_statistics {
+	__u64 peer_dev_received;
+	__u64 peer_dev_sent;
+	__u32 peer_dev_pending;
+	__u32 peer_dev_unacked;
+	__u64 peer_dev_out_of_sync;
+	__u64 peer_dev_resync_failed;
+	__u64 peer_dev_bitmap_uuid;
+	__u32 peer_dev_flags;
+};
+
+struct drbd_notification_header {
+	__u32 nh_type;
+};
+
+struct drbd_helper_info {
+	char helper_name[32];
+	__u32 helper_name_len;
+	__u32 helper_status;
+};
+
+int drbd_cfg_reply_to_skb(struct sk_buff *skb, struct drbd_cfg_reply *s);
+
+int drbd_cfg_context_from_attrs(struct drbd_cfg_context *s, struct genl_info *info);
+int drbd_cfg_context_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int drbd_cfg_context_to_skb(struct sk_buff *skb, struct drbd_cfg_context *s);
+
+int disk_conf_from_attrs(struct disk_conf *s, struct genl_info *info);
+int disk_conf_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int disk_conf_to_skb(struct sk_buff *skb, struct disk_conf *s);
+void set_disk_conf_defaults(struct disk_conf *x);
+
+int res_opts_from_attrs(struct res_opts *s, struct genl_info *info);
+int res_opts_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int res_opts_to_skb(struct sk_buff *skb, struct res_opts *s);
+void set_res_opts_defaults(struct res_opts *x);
+
+int net_conf_from_attrs(struct net_conf *s, struct genl_info *info);
+int net_conf_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int net_conf_to_skb(struct sk_buff *skb, struct net_conf *s);
+void set_net_conf_defaults(struct net_conf *x);
+
+int set_role_parms_from_attrs(struct set_role_parms *s, struct genl_info *info);
+int set_role_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int set_role_parms_to_skb(struct sk_buff *skb, struct set_role_parms *s);
+
+int resize_parms_from_attrs(struct resize_parms *s, struct genl_info *info);
+int resize_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int resize_parms_to_skb(struct sk_buff *skb, struct resize_parms *s);
+void set_resize_parms_defaults(struct resize_parms *x);
+
+int state_info_to_skb(struct sk_buff *skb, struct state_info *s);
+
+int start_ov_parms_from_attrs(struct start_ov_parms *s, struct genl_info *info);
+int start_ov_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int start_ov_parms_to_skb(struct sk_buff *skb, struct start_ov_parms *s);
+
+int new_c_uuid_parms_from_attrs(struct new_c_uuid_parms *s, struct genl_info *info);
+int new_c_uuid_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int new_c_uuid_parms_to_skb(struct sk_buff *skb, struct new_c_uuid_parms *s);
+
+int timeout_parms_to_skb(struct sk_buff *skb, struct timeout_parms *s);
+
+int disconnect_parms_from_attrs(struct disconnect_parms *s, struct genl_info *info);
+int disconnect_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int disconnect_parms_to_skb(struct sk_buff *skb, struct disconnect_parms *s);
+
+int detach_parms_from_attrs(struct detach_parms *s, struct genl_info *info);
+int detach_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int detach_parms_to_skb(struct sk_buff *skb, struct detach_parms *s);
+
+int resource_info_from_attrs(struct resource_info *s, struct genl_info *info);
+int resource_info_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int resource_info_to_skb(struct sk_buff *skb, struct resource_info *s);
+
+int device_info_from_attrs(struct device_info *s, struct genl_info *info);
+int device_info_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int device_info_to_skb(struct sk_buff *skb, struct device_info *s);
+
+int connection_info_from_attrs(struct connection_info *s, struct genl_info *info);
+int connection_info_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int connection_info_to_skb(struct sk_buff *skb, struct connection_info *s);
+
+int peer_device_info_from_attrs(struct peer_device_info *s, struct genl_info *info);
+int peer_device_info_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int peer_device_info_to_skb(struct sk_buff *skb, struct peer_device_info *s);
+
+int resource_statistics_from_attrs(struct resource_statistics *s, struct genl_info *info);
+int resource_statistics_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int resource_statistics_to_skb(struct sk_buff *skb, struct resource_statistics *s);
+
+int device_statistics_from_attrs(struct device_statistics *s, struct genl_info *info);
+int device_statistics_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int device_statistics_to_skb(struct sk_buff *skb, struct device_statistics *s);
+
+int connection_statistics_from_attrs(struct connection_statistics *s, struct genl_info *info);
+int connection_statistics_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int connection_statistics_to_skb(struct sk_buff *skb, struct connection_statistics *s);
+
+int peer_device_statistics_from_attrs(struct peer_device_statistics *s, struct genl_info *info);
+int peer_device_statistics_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info);
+int peer_device_statistics_to_skb(struct sk_buff *skb, struct peer_device_statistics *s);
+
+int drbd_notification_header_to_skb(struct sk_buff *skb, struct drbd_notification_header *s);
+
+int drbd_helper_info_to_skb(struct sk_buff *skb, struct drbd_helper_info *s);
+
+#endif /* _LINUX_DRBD_GEN_H */
diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c
index 1d0feafceadcb..6d0c12c102602 100644
--- a/drivers/block/drbd/drbd_proc.c
+++ b/drivers/block/drbd/drbd_proc.c
@@ -228,7 +228,7 @@ int drbd_seq_show(struct seq_file *seq, void *v)
 	};
 
 	seq_printf(seq, "version: " REL_VERSION " (api:%d/proto:%d-%d)\n%s\n",
-		   GENL_MAGIC_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX, drbd_buildtag());
+		   DRBD_FAMILY_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX, drbd_buildtag());
 
 	/*
 	  cs .. connection state
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index 0000913f7efcc..f5c0915242951 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -1118,6 +1118,22 @@ static void __loop_clr_fd(struct loop_device *lo)
 	struct file *filp;
 	gfp_t gfp = lo->old_gfp_mask;
 
+	/*
+	 * Now that loop_queue_rq() sees lo->lo_state != Lo_bound,
+	 * wait for already started loop_queue_rq() to complete.
+	 */
+	synchronize_rcu();
+	/*
+	 * Now that no more works are scheduled by loop_queue_rq(),
+	 * wait for already scheduled works to complete.
+	 */
+	drain_workqueue(lo->workqueue);
+	/*
+	 * Now that no more AIO requests are scheduled by lo_rw_aio(),
+	 * wait for already started AIO to complete.
+	 */
+	blk_mq_unfreeze_queue(lo->lo_queue, blk_mq_freeze_queue(lo->lo_queue));
+
 	spin_lock_irq(&lo->lo_lock);
 	filp = lo->lo_backing_file;
 	lo->lo_backing_file = NULL;
diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c
index 4065336ebd1f1..94709466ad19f 100644
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -580,14 +580,7 @@ static const struct bus_type rbd_bus_type = {
 	.bus_groups	= rbd_bus_groups,
 };
 
-static void rbd_root_dev_release(struct device *dev)
-{
-}
-
-static struct device rbd_root_dev = {
-	.init_name =    "rbd",
-	.release =      rbd_root_dev_release,
-};
+static struct device *rbd_root_dev;
 
 static __printf(2, 3)
 void rbd_warn(struct rbd_device *rbd_dev, const char *fmt, ...)
@@ -4565,24 +4558,12 @@ static int rbd_register_watch(struct rbd_device *rbd_dev)
 	return ret;
 }
 
-static void cancel_tasks_sync(struct rbd_device *rbd_dev)
-{
-	dout("%s rbd_dev %p\n", __func__, rbd_dev);
-
-	cancel_work_sync(&rbd_dev->acquired_lock_work);
-	cancel_work_sync(&rbd_dev->released_lock_work);
-	cancel_delayed_work_sync(&rbd_dev->lock_dwork);
-	cancel_work_sync(&rbd_dev->unlock_work);
-}
-
 /*
  * header_rwsem must not be held to avoid a deadlock with
  * rbd_dev_refresh() when flushing notifies.
  */
 static void rbd_unregister_watch(struct rbd_device *rbd_dev)
 {
-	cancel_tasks_sync(rbd_dev);
-
 	mutex_lock(&rbd_dev->watch_mutex);
 	if (rbd_dev->watch_state == RBD_WATCH_STATE_REGISTERED)
 		__rbd_unregister_watch(rbd_dev);
@@ -5390,7 +5371,7 @@ static struct rbd_device *__rbd_dev_create(struct rbd_spec *spec)
 
 	rbd_dev->dev.bus = &rbd_bus_type;
 	rbd_dev->dev.type = &rbd_device_type;
-	rbd_dev->dev.parent = &rbd_root_dev;
+	rbd_dev->dev.parent = rbd_root_dev;
 	device_initialize(&rbd_dev->dev);
 
 	return rbd_dev;
@@ -6548,10 +6529,18 @@ static int rbd_add_parse_args(const char *buf,
 
 static void rbd_dev_image_unlock(struct rbd_device *rbd_dev)
 {
+	dout("%s rbd_dev %p\n", __func__, rbd_dev);
+
+	disable_delayed_work_sync(&rbd_dev->lock_dwork);
+	disable_work_sync(&rbd_dev->unlock_work);
+
 	down_write(&rbd_dev->lock_rwsem);
 	if (__rbd_is_lock_owner(rbd_dev))
 		__rbd_release_lock(rbd_dev);
 	up_write(&rbd_dev->lock_rwsem);
+
+	flush_work(&rbd_dev->acquired_lock_work);
+	flush_work(&rbd_dev->released_lock_work);
 }
 
 /*
@@ -7331,15 +7320,13 @@ static int __init rbd_sysfs_init(void)
 {
 	int ret;
 
-	ret = device_register(&rbd_root_dev);
-	if (ret < 0) {
-		put_device(&rbd_root_dev);
-		return ret;
-	}
+	rbd_root_dev = root_device_register("rbd");
+	if (IS_ERR(rbd_root_dev))
+		return PTR_ERR(rbd_root_dev);
 
 	ret = bus_register(&rbd_bus_type);
 	if (ret < 0)
-		device_unregister(&rbd_root_dev);
+		root_device_unregister(rbd_root_dev);
 
 	return ret;
 }
@@ -7347,7 +7334,7 @@ static int __init rbd_sysfs_init(void)
 static void __exit rbd_sysfs_cleanup(void)
 {
 	bus_unregister(&rbd_bus_type);
-	device_unregister(&rbd_root_dev);
+	root_device_unregister(rbd_root_dev);
 }
 
 static int __init rbd_slab_init(void)
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index aebc710f0d6ae..07111455eecff 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -33,6 +33,7 @@
 #include <linux/cpuhotplug.h>
 #include <linux/part_stat.h>
 #include <linux/kernel_read_file.h>
+#include <linux/rcupdate.h>
 
 #include "zram_drv.h"
 
@@ -504,6 +505,7 @@ struct zram_wb_ctl {
 	wait_queue_head_t done_wait;
 	spinlock_t done_lock;
 	atomic_t num_inflight;
+	struct rcu_head rcu;
 };
 
 struct zram_wb_req {
@@ -847,7 +849,7 @@ static void release_wb_ctl(struct zram_wb_ctl *wb_ctl)
 		release_wb_req(req);
 	}
 
-	kfree(wb_ctl);
+	kfree_rcu(wb_ctl, rcu);
 }
 
 static struct zram_wb_ctl *init_wb_ctl(struct zram *zram)
@@ -964,11 +966,13 @@ static void zram_writeback_endio(struct bio *bio)
 	struct zram_wb_ctl *wb_ctl = bio->bi_private;
 	unsigned long flags;
 
+	rcu_read_lock();
 	spin_lock_irqsave(&wb_ctl->done_lock, flags);
 	list_add(&req->entry, &wb_ctl->done_reqs);
 	spin_unlock_irqrestore(&wb_ctl->done_lock, flags);
 
 	wake_up(&wb_ctl->done_wait);
+	rcu_read_unlock();
 }
 
 static void zram_submit_wb_request(struct zram *zram,
diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig
index c5d45cf91f882..4e8c24d757e9a 100644
--- a/drivers/bluetooth/Kconfig
+++ b/drivers/bluetooth/Kconfig
@@ -337,46 +337,6 @@ config BT_HCIBFUSB
 	  Say Y here to compile support for HCI BFUSB devices into the
 	  kernel or say M to compile it as module (bfusb).
 
-config BT_HCIDTL1
-	tristate "HCI DTL1 (PC Card) driver"
-	depends on PCMCIA && HAS_IOPORT
-	help
-	  Bluetooth HCI DTL1 (PC Card) driver.
-	  This driver provides support for Bluetooth PCMCIA devices with
-	  Nokia DTL1 interface:
-	     Nokia Bluetooth Card
-	     Socket Bluetooth CF Card
-
-	  Say Y here to compile support for HCI DTL1 devices into the
-	  kernel or say M to compile it as module (dtl1_cs).
-
-config BT_HCIBT3C
-	tristate "HCI BT3C (PC Card) driver"
-	depends on PCMCIA && HAS_IOPORT
-	select FW_LOADER
-	help
-	  Bluetooth HCI BT3C (PC Card) driver.
-	  This driver provides support for Bluetooth PCMCIA devices with
-	  3Com BT3C interface:
-	     3Com Bluetooth Card (3CRWB6096)
-	     HP Bluetooth Card
-
-	  Say Y here to compile support for HCI BT3C devices into the
-	  kernel or say M to compile it as module (bt3c_cs).
-
-config BT_HCIBLUECARD
-	tristate "HCI BlueCard (PC Card) driver"
-	depends on PCMCIA && HAS_IOPORT
-	help
-	  Bluetooth HCI BlueCard (PC Card) driver.
-	  This driver provides support for Bluetooth PCMCIA devices with
-	  Anycom BlueCard interface:
-	     Anycom Bluetooth PC Card
-	     Anycom Bluetooth CF Card
-
-	  Say Y here to compile support for HCI BlueCard devices into the
-	  kernel or say M to compile it as module (bluecard_cs).
-
 config BT_HCIVHCI
 	tristate "HCI VHCI (Virtual HCI device) driver"
 	select WANT_DEV_COREDUMP
@@ -502,7 +462,7 @@ config BT_NXPUART
 
 config BT_INTEL_PCIE
 	tristate "Intel HCI PCIe driver"
-	depends on PCI
+	depends on PCI && ACPI
 	select BT_INTEL
 	select FW_LOADER
 	help
diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile
index 81856512ddd03..bafc26250b636 100644
--- a/drivers/bluetooth/Makefile
+++ b/drivers/bluetooth/Makefile
@@ -9,9 +9,6 @@ obj-$(CONFIG_BT_HCIBCM203X)	+= bcm203x.o
 obj-$(CONFIG_BT_HCIBCM4377)	+= hci_bcm4377.o
 obj-$(CONFIG_BT_HCIBPA10X)	+= bpa10x.o
 obj-$(CONFIG_BT_HCIBFUSB)	+= bfusb.o
-obj-$(CONFIG_BT_HCIDTL1)	+= dtl1_cs.o
-obj-$(CONFIG_BT_HCIBT3C)	+= bt3c_cs.o
-obj-$(CONFIG_BT_HCIBLUECARD)	+= bluecard_cs.o
 
 obj-$(CONFIG_BT_HCIBTUSB)	+= btusb.o
 obj-$(CONFIG_BT_HCIBTSDIO)	+= btsdio.o
diff --git a/drivers/bluetooth/bluecard_cs.c b/drivers/bluetooth/bluecard_cs.c
deleted file mode 100644
index 1e3a56e9b1399..0000000000000
--- a/drivers/bluetooth/bluecard_cs.c
+++ /dev/null
@@ -1,908 +0,0 @@
-/*
- *
- *  Bluetooth driver for the Anycom BlueCard (LSE039/LSE041)
- *
- *  Copyright (C) 2001-2002  Marcel Holtmann <marcel@holtmann.org>
- *
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation;
- *
- *  Software distributed under the License is distributed on an "AS
- *  IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
- *  implied. See the License for the specific language governing
- *  rights and limitations under the License.
- *
- *  The initial developer of the original code is David A. Hinds
- *  <dahinds@users.sourceforge.net>.  Portions created by David A. Hinds
- *  are Copyright (C) 1999 David A. Hinds.  All Rights Reserved.
- *
- */
-
-#include <linux/module.h>
-
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-#include <linux/sched.h>
-#include <linux/delay.h>
-#include <linux/timer.h>
-#include <linux/errno.h>
-#include <linux/ptrace.h>
-#include <linux/ioport.h>
-#include <linux/spinlock.h>
-#include <linux/moduleparam.h>
-#include <linux/wait.h>
-
-#include <linux/skbuff.h>
-#include <linux/io.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/ciscode.h>
-#include <pcmcia/ds.h>
-#include <pcmcia/cisreg.h>
-
-#include <net/bluetooth/bluetooth.h>
-#include <net/bluetooth/hci_core.h>
-
-
-
-/* ======================== Module parameters ======================== */
-
-
-MODULE_AUTHOR("Marcel Holtmann <marcel@holtmann.org>");
-MODULE_DESCRIPTION("Bluetooth driver for the Anycom BlueCard (LSE039/LSE041)");
-MODULE_LICENSE("GPL");
-
-
-
-/* ======================== Local structures ======================== */
-
-
-struct bluecard_info {
-	struct pcmcia_device *p_dev;
-
-	struct hci_dev *hdev;
-
-	spinlock_t lock;		/* For serializing operations */
-	struct timer_list timer;	/* For LED control */
-
-	struct sk_buff_head txq;
-	unsigned long tx_state;
-
-	unsigned long rx_state;
-	unsigned long rx_count;
-	struct sk_buff *rx_skb;
-
-	unsigned char ctrl_reg;
-	unsigned long hw_state;		/* Status of the hardware and LED control */
-};
-
-
-static int bluecard_config(struct pcmcia_device *link);
-static void bluecard_release(struct pcmcia_device *link);
-
-static void bluecard_detach(struct pcmcia_device *p_dev);
-
-
-/* Default baud rate: 57600, 115200, 230400 or 460800 */
-#define DEFAULT_BAUD_RATE  230400
-
-
-/* Hardware states */
-#define CARD_READY             1
-#define CARD_ACTIVITY	       2
-#define CARD_HAS_PCCARD_ID     4
-#define CARD_HAS_POWER_LED     5
-#define CARD_HAS_ACTIVITY_LED  6
-
-/* Transmit states  */
-#define XMIT_SENDING         1
-#define XMIT_WAKEUP          2
-#define XMIT_BUFFER_NUMBER   5	/* unset = buffer one, set = buffer two */
-#define XMIT_BUF_ONE_READY   6
-#define XMIT_BUF_TWO_READY   7
-#define XMIT_SENDING_READY   8
-
-/* Receiver states */
-#define RECV_WAIT_PACKET_TYPE   0
-#define RECV_WAIT_EVENT_HEADER  1
-#define RECV_WAIT_ACL_HEADER    2
-#define RECV_WAIT_SCO_HEADER    3
-#define RECV_WAIT_DATA          4
-
-/* Special packet types */
-#define PKT_BAUD_RATE_57600   0x80
-#define PKT_BAUD_RATE_115200  0x81
-#define PKT_BAUD_RATE_230400  0x82
-#define PKT_BAUD_RATE_460800  0x83
-
-
-/* These are the register offsets */
-#define REG_COMMAND     0x20
-#define REG_INTERRUPT   0x21
-#define REG_CONTROL     0x22
-#define REG_RX_CONTROL  0x24
-#define REG_CARD_RESET  0x30
-#define REG_LED_CTRL    0x30
-
-/* REG_COMMAND */
-#define REG_COMMAND_TX_BUF_ONE  0x01
-#define REG_COMMAND_TX_BUF_TWO  0x02
-#define REG_COMMAND_RX_BUF_ONE  0x04
-#define REG_COMMAND_RX_BUF_TWO  0x08
-#define REG_COMMAND_RX_WIN_ONE  0x00
-#define REG_COMMAND_RX_WIN_TWO  0x10
-
-/* REG_CONTROL */
-#define REG_CONTROL_BAUD_RATE_57600   0x00
-#define REG_CONTROL_BAUD_RATE_115200  0x01
-#define REG_CONTROL_BAUD_RATE_230400  0x02
-#define REG_CONTROL_BAUD_RATE_460800  0x03
-#define REG_CONTROL_RTS               0x04
-#define REG_CONTROL_BT_ON             0x08
-#define REG_CONTROL_BT_RESET          0x10
-#define REG_CONTROL_BT_RES_PU         0x20
-#define REG_CONTROL_INTERRUPT         0x40
-#define REG_CONTROL_CARD_RESET        0x80
-
-/* REG_RX_CONTROL */
-#define RTS_LEVEL_SHIFT_BITS  0x02
-
-
-
-/* ======================== LED handling routines ======================== */
-
-
-static void bluecard_activity_led_timeout(struct timer_list *t)
-{
-	struct bluecard_info *info = timer_container_of(info, t, timer);
-	unsigned int iobase = info->p_dev->resource[0]->start;
-
-	if (test_bit(CARD_ACTIVITY, &(info->hw_state))) {
-		/* leave LED in inactive state for HZ/10 for blink effect */
-		clear_bit(CARD_ACTIVITY, &(info->hw_state));
-		mod_timer(&(info->timer), jiffies + HZ / 10);
-	}
-
-	/* Disable activity LED, enable power LED */
-	outb(0x08 | 0x20, iobase + 0x30);
-}
-
-
-static void bluecard_enable_activity_led(struct bluecard_info *info)
-{
-	unsigned int iobase = info->p_dev->resource[0]->start;
-
-	/* don't disturb running blink timer */
-	if (timer_pending(&(info->timer)))
-		return;
-
-	set_bit(CARD_ACTIVITY, &(info->hw_state));
-
-	if (test_bit(CARD_HAS_ACTIVITY_LED, &(info->hw_state))) {
-		/* Enable activity LED, keep power LED enabled */
-		outb(0x18 | 0x60, iobase + 0x30);
-	} else {
-		/* Disable power LED */
-		outb(0x00, iobase + 0x30);
-	}
-
-	/* Stop the LED after HZ/10 */
-	mod_timer(&(info->timer), jiffies + HZ / 10);
-}
-
-
-
-/* ======================== Interrupt handling ======================== */
-
-
-static int bluecard_write(unsigned int iobase, unsigned int offset, __u8 *buf, int len)
-{
-	int i, actual;
-
-	actual = (len > 15) ? 15 : len;
-
-	outb_p(actual, iobase + offset);
-
-	for (i = 0; i < actual; i++)
-		outb_p(buf[i], iobase + offset + i + 1);
-
-	return actual;
-}
-
-
-static void bluecard_write_wakeup(struct bluecard_info *info)
-{
-	if (!info) {
-		BT_ERR("Unknown device");
-		return;
-	}
-
-	if (!test_bit(XMIT_SENDING_READY, &(info->tx_state)))
-		return;
-
-	if (test_and_set_bit(XMIT_SENDING, &(info->tx_state))) {
-		set_bit(XMIT_WAKEUP, &(info->tx_state));
-		return;
-	}
-
-	do {
-		unsigned int iobase = info->p_dev->resource[0]->start;
-		unsigned int offset;
-		unsigned char command;
-		unsigned long ready_bit;
-		register struct sk_buff *skb;
-		int len;
-
-		clear_bit(XMIT_WAKEUP, &(info->tx_state));
-
-		if (!pcmcia_dev_present(info->p_dev))
-			return;
-
-		if (test_bit(XMIT_BUFFER_NUMBER, &(info->tx_state))) {
-			if (!test_bit(XMIT_BUF_TWO_READY, &(info->tx_state)))
-				break;
-			offset = 0x10;
-			command = REG_COMMAND_TX_BUF_TWO;
-			ready_bit = XMIT_BUF_TWO_READY;
-		} else {
-			if (!test_bit(XMIT_BUF_ONE_READY, &(info->tx_state)))
-				break;
-			offset = 0x00;
-			command = REG_COMMAND_TX_BUF_ONE;
-			ready_bit = XMIT_BUF_ONE_READY;
-		}
-
-		skb = skb_dequeue(&(info->txq));
-		if (!skb)
-			break;
-
-		if (hci_skb_pkt_type(skb) & 0x80) {
-			/* Disable RTS */
-			info->ctrl_reg |= REG_CONTROL_RTS;
-			outb(info->ctrl_reg, iobase + REG_CONTROL);
-		}
-
-		/* Activate LED */
-		bluecard_enable_activity_led(info);
-
-		/* Send frame */
-		len = bluecard_write(iobase, offset, skb->data, skb->len);
-
-		/* Tell the FPGA to send the data */
-		outb_p(command, iobase + REG_COMMAND);
-
-		/* Mark the buffer as dirty */
-		clear_bit(ready_bit, &(info->tx_state));
-
-		if (hci_skb_pkt_type(skb) & 0x80) {
-			DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
-			DEFINE_WAIT(wait);
-
-			unsigned char baud_reg;
-
-			switch (hci_skb_pkt_type(skb)) {
-			case PKT_BAUD_RATE_460800:
-				baud_reg = REG_CONTROL_BAUD_RATE_460800;
-				break;
-			case PKT_BAUD_RATE_230400:
-				baud_reg = REG_CONTROL_BAUD_RATE_230400;
-				break;
-			case PKT_BAUD_RATE_115200:
-				baud_reg = REG_CONTROL_BAUD_RATE_115200;
-				break;
-			case PKT_BAUD_RATE_57600:
-			default:
-				baud_reg = REG_CONTROL_BAUD_RATE_57600;
-				break;
-			}
-
-			/* Wait until the command reaches the baseband */
-			mdelay(100);
-
-			/* Set baud on baseband */
-			info->ctrl_reg &= ~0x03;
-			info->ctrl_reg |= baud_reg;
-			outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-			/* Enable RTS */
-			info->ctrl_reg &= ~REG_CONTROL_RTS;
-			outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-			/* Wait before the next HCI packet can be send */
-			mdelay(1000);
-		}
-
-		if (len == skb->len) {
-			kfree_skb(skb);
-		} else {
-			skb_pull(skb, len);
-			skb_queue_head(&(info->txq), skb);
-		}
-
-		info->hdev->stat.byte_tx += len;
-
-		/* Change buffer */
-		change_bit(XMIT_BUFFER_NUMBER, &(info->tx_state));
-
-	} while (test_bit(XMIT_WAKEUP, &(info->tx_state)));
-
-	clear_bit(XMIT_SENDING, &(info->tx_state));
-}
-
-
-static int bluecard_read(unsigned int iobase, unsigned int offset, __u8 *buf, int size)
-{
-	int i, n, len;
-
-	outb(REG_COMMAND_RX_WIN_ONE, iobase + REG_COMMAND);
-
-	len = inb(iobase + offset);
-	n = 0;
-	i = 1;
-
-	while (n < len) {
-
-		if (i == 16) {
-			outb(REG_COMMAND_RX_WIN_TWO, iobase + REG_COMMAND);
-			i = 0;
-		}
-
-		buf[n] = inb(iobase + offset + i);
-
-		n++;
-		i++;
-
-	}
-
-	return len;
-}
-
-
-static void bluecard_receive(struct bluecard_info *info,
-			     unsigned int offset)
-{
-	unsigned int iobase;
-	unsigned char buf[31];
-	int i, len;
-
-	if (!info) {
-		BT_ERR("Unknown device");
-		return;
-	}
-
-	iobase = info->p_dev->resource[0]->start;
-
-	if (test_bit(XMIT_SENDING_READY, &(info->tx_state)))
-		bluecard_enable_activity_led(info);
-
-	len = bluecard_read(iobase, offset, buf, sizeof(buf));
-
-	for (i = 0; i < len; i++) {
-
-		/* Allocate packet */
-		if (!info->rx_skb) {
-			info->rx_state = RECV_WAIT_PACKET_TYPE;
-			info->rx_count = 0;
-			info->rx_skb = bt_skb_alloc(HCI_MAX_FRAME_SIZE, GFP_ATOMIC);
-			if (!info->rx_skb) {
-				BT_ERR("Can't allocate mem for new packet");
-				return;
-			}
-		}
-
-		if (info->rx_state == RECV_WAIT_PACKET_TYPE) {
-
-			hci_skb_pkt_type(info->rx_skb) = buf[i];
-
-			switch (hci_skb_pkt_type(info->rx_skb)) {
-
-			case 0x00:
-				/* init packet */
-				if (offset != 0x00) {
-					set_bit(XMIT_BUF_ONE_READY, &(info->tx_state));
-					set_bit(XMIT_BUF_TWO_READY, &(info->tx_state));
-					set_bit(XMIT_SENDING_READY, &(info->tx_state));
-					bluecard_write_wakeup(info);
-				}
-
-				kfree_skb(info->rx_skb);
-				info->rx_skb = NULL;
-				break;
-
-			case HCI_EVENT_PKT:
-				info->rx_state = RECV_WAIT_EVENT_HEADER;
-				info->rx_count = HCI_EVENT_HDR_SIZE;
-				break;
-
-			case HCI_ACLDATA_PKT:
-				info->rx_state = RECV_WAIT_ACL_HEADER;
-				info->rx_count = HCI_ACL_HDR_SIZE;
-				break;
-
-			case HCI_SCODATA_PKT:
-				info->rx_state = RECV_WAIT_SCO_HEADER;
-				info->rx_count = HCI_SCO_HDR_SIZE;
-				break;
-
-			default:
-				/* unknown packet */
-				BT_ERR("Unknown HCI packet with type 0x%02x received",
-				       hci_skb_pkt_type(info->rx_skb));
-				info->hdev->stat.err_rx++;
-
-				kfree_skb(info->rx_skb);
-				info->rx_skb = NULL;
-				break;
-
-			}
-
-		} else {
-
-			skb_put_u8(info->rx_skb, buf[i]);
-			info->rx_count--;
-
-			if (info->rx_count == 0) {
-
-				int dlen;
-				struct hci_event_hdr *eh;
-				struct hci_acl_hdr *ah;
-				struct hci_sco_hdr *sh;
-
-				switch (info->rx_state) {
-
-				case RECV_WAIT_EVENT_HEADER:
-					eh = hci_event_hdr(info->rx_skb);
-					info->rx_state = RECV_WAIT_DATA;
-					info->rx_count = eh->plen;
-					break;
-
-				case RECV_WAIT_ACL_HEADER:
-					ah = hci_acl_hdr(info->rx_skb);
-					dlen = __le16_to_cpu(ah->dlen);
-					info->rx_state = RECV_WAIT_DATA;
-					info->rx_count = dlen;
-					break;
-
-				case RECV_WAIT_SCO_HEADER:
-					sh = hci_sco_hdr(info->rx_skb);
-					info->rx_state = RECV_WAIT_DATA;
-					info->rx_count = sh->dlen;
-					break;
-
-				case RECV_WAIT_DATA:
-					hci_recv_frame(info->hdev, info->rx_skb);
-					info->rx_skb = NULL;
-					break;
-
-				}
-
-			}
-
-		}
-
-
-	}
-
-	info->hdev->stat.byte_rx += len;
-}
-
-
-static irqreturn_t bluecard_interrupt(int irq, void *dev_inst)
-{
-	struct bluecard_info *info = dev_inst;
-	unsigned int iobase;
-	unsigned char reg;
-
-	if (!info || !info->hdev)
-		/* our irq handler is shared */
-		return IRQ_NONE;
-
-	if (!test_bit(CARD_READY, &(info->hw_state)))
-		return IRQ_HANDLED;
-
-	iobase = info->p_dev->resource[0]->start;
-
-	spin_lock(&(info->lock));
-
-	/* Disable interrupt */
-	info->ctrl_reg &= ~REG_CONTROL_INTERRUPT;
-	outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-	reg = inb(iobase + REG_INTERRUPT);
-
-	if ((reg != 0x00) && (reg != 0xff)) {
-
-		if (reg & 0x04) {
-			bluecard_receive(info, 0x00);
-			outb(0x04, iobase + REG_INTERRUPT);
-			outb(REG_COMMAND_RX_BUF_ONE, iobase + REG_COMMAND);
-		}
-
-		if (reg & 0x08) {
-			bluecard_receive(info, 0x10);
-			outb(0x08, iobase + REG_INTERRUPT);
-			outb(REG_COMMAND_RX_BUF_TWO, iobase + REG_COMMAND);
-		}
-
-		if (reg & 0x01) {
-			set_bit(XMIT_BUF_ONE_READY, &(info->tx_state));
-			outb(0x01, iobase + REG_INTERRUPT);
-			bluecard_write_wakeup(info);
-		}
-
-		if (reg & 0x02) {
-			set_bit(XMIT_BUF_TWO_READY, &(info->tx_state));
-			outb(0x02, iobase + REG_INTERRUPT);
-			bluecard_write_wakeup(info);
-		}
-
-	}
-
-	/* Enable interrupt */
-	info->ctrl_reg |= REG_CONTROL_INTERRUPT;
-	outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-	spin_unlock(&(info->lock));
-
-	return IRQ_HANDLED;
-}
-
-
-
-/* ======================== Device specific HCI commands ======================== */
-
-
-static int bluecard_hci_set_baud_rate(struct hci_dev *hdev, int baud)
-{
-	struct bluecard_info *info = hci_get_drvdata(hdev);
-	struct sk_buff *skb;
-
-	/* Ericsson baud rate command */
-	unsigned char cmd[] = { HCI_COMMAND_PKT, 0x09, 0xfc, 0x01, 0x03 };
-
-	skb = bt_skb_alloc(HCI_MAX_FRAME_SIZE, GFP_KERNEL);
-	if (!skb) {
-		BT_ERR("Can't allocate mem for new packet");
-		return -1;
-	}
-
-	switch (baud) {
-	case 460800:
-		cmd[4] = 0x00;
-		hci_skb_pkt_type(skb) = PKT_BAUD_RATE_460800;
-		break;
-	case 230400:
-		cmd[4] = 0x01;
-		hci_skb_pkt_type(skb) = PKT_BAUD_RATE_230400;
-		break;
-	case 115200:
-		cmd[4] = 0x02;
-		hci_skb_pkt_type(skb) = PKT_BAUD_RATE_115200;
-		break;
-	case 57600:
-	default:
-		cmd[4] = 0x03;
-		hci_skb_pkt_type(skb) = PKT_BAUD_RATE_57600;
-		break;
-	}
-
-	skb_put_data(skb, cmd, sizeof(cmd));
-
-	skb_queue_tail(&(info->txq), skb);
-
-	bluecard_write_wakeup(info);
-
-	return 0;
-}
-
-
-
-/* ======================== HCI interface ======================== */
-
-
-static int bluecard_hci_flush(struct hci_dev *hdev)
-{
-	struct bluecard_info *info = hci_get_drvdata(hdev);
-
-	/* Drop TX queue */
-	skb_queue_purge(&(info->txq));
-
-	return 0;
-}
-
-
-static int bluecard_hci_open(struct hci_dev *hdev)
-{
-	struct bluecard_info *info = hci_get_drvdata(hdev);
-	unsigned int iobase = info->p_dev->resource[0]->start;
-
-	if (test_bit(CARD_HAS_PCCARD_ID, &(info->hw_state)))
-		bluecard_hci_set_baud_rate(hdev, DEFAULT_BAUD_RATE);
-
-	/* Enable power LED */
-	outb(0x08 | 0x20, iobase + 0x30);
-
-	return 0;
-}
-
-
-static int bluecard_hci_close(struct hci_dev *hdev)
-{
-	struct bluecard_info *info = hci_get_drvdata(hdev);
-	unsigned int iobase = info->p_dev->resource[0]->start;
-
-	bluecard_hci_flush(hdev);
-
-	/* Stop LED timer */
-	timer_delete_sync(&(info->timer));
-
-	/* Disable power LED */
-	outb(0x00, iobase + 0x30);
-
-	return 0;
-}
-
-
-static int bluecard_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct bluecard_info *info = hci_get_drvdata(hdev);
-
-	switch (hci_skb_pkt_type(skb)) {
-	case HCI_COMMAND_PKT:
-		hdev->stat.cmd_tx++;
-		break;
-	case HCI_ACLDATA_PKT:
-		hdev->stat.acl_tx++;
-		break;
-	case HCI_SCODATA_PKT:
-		hdev->stat.sco_tx++;
-		break;
-	}
-
-	/* Prepend skb with frame type */
-	memcpy(skb_push(skb, 1), &hci_skb_pkt_type(skb), 1);
-	skb_queue_tail(&(info->txq), skb);
-
-	bluecard_write_wakeup(info);
-
-	return 0;
-}
-
-
-
-/* ======================== Card services HCI interaction ======================== */
-
-
-static int bluecard_open(struct bluecard_info *info)
-{
-	unsigned int iobase = info->p_dev->resource[0]->start;
-	struct hci_dev *hdev;
-	unsigned char id;
-
-	spin_lock_init(&(info->lock));
-
-	timer_setup(&info->timer, bluecard_activity_led_timeout, 0);
-
-	skb_queue_head_init(&(info->txq));
-
-	info->rx_state = RECV_WAIT_PACKET_TYPE;
-	info->rx_count = 0;
-	info->rx_skb = NULL;
-
-	/* Initialize HCI device */
-	hdev = hci_alloc_dev();
-	if (!hdev) {
-		BT_ERR("Can't allocate HCI device");
-		return -ENOMEM;
-	}
-
-	info->hdev = hdev;
-
-	hdev->bus = HCI_PCCARD;
-	hci_set_drvdata(hdev, info);
-	SET_HCIDEV_DEV(hdev, &info->p_dev->dev);
-
-	hdev->open  = bluecard_hci_open;
-	hdev->close = bluecard_hci_close;
-	hdev->flush = bluecard_hci_flush;
-	hdev->send  = bluecard_hci_send_frame;
-
-	id = inb(iobase + 0x30);
-
-	if ((id & 0x0f) == 0x02)
-		set_bit(CARD_HAS_PCCARD_ID, &(info->hw_state));
-
-	if (id & 0x10)
-		set_bit(CARD_HAS_POWER_LED, &(info->hw_state));
-
-	if (id & 0x20)
-		set_bit(CARD_HAS_ACTIVITY_LED, &(info->hw_state));
-
-	/* Reset card */
-	info->ctrl_reg = REG_CONTROL_BT_RESET | REG_CONTROL_CARD_RESET;
-	outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-	/* Turn FPGA off */
-	outb(0x80, iobase + 0x30);
-
-	/* Wait some time */
-	msleep(10);
-
-	/* Turn FPGA on */
-	outb(0x00, iobase + 0x30);
-
-	/* Activate card */
-	info->ctrl_reg = REG_CONTROL_BT_ON | REG_CONTROL_BT_RES_PU;
-	outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-	/* Enable interrupt */
-	outb(0xff, iobase + REG_INTERRUPT);
-	info->ctrl_reg |= REG_CONTROL_INTERRUPT;
-	outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-	if ((id & 0x0f) == 0x03) {
-		/* Disable RTS */
-		info->ctrl_reg |= REG_CONTROL_RTS;
-		outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-		/* Set baud rate */
-		info->ctrl_reg |= 0x03;
-		outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-		/* Enable RTS */
-		info->ctrl_reg &= ~REG_CONTROL_RTS;
-		outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-		set_bit(XMIT_BUF_ONE_READY, &(info->tx_state));
-		set_bit(XMIT_BUF_TWO_READY, &(info->tx_state));
-		set_bit(XMIT_SENDING_READY, &(info->tx_state));
-	}
-
-	/* Start the RX buffers */
-	outb(REG_COMMAND_RX_BUF_ONE, iobase + REG_COMMAND);
-	outb(REG_COMMAND_RX_BUF_TWO, iobase + REG_COMMAND);
-
-	/* Signal that the hardware is ready */
-	set_bit(CARD_READY, &(info->hw_state));
-
-	/* Drop TX queue */
-	skb_queue_purge(&(info->txq));
-
-	/* Control the point at which RTS is enabled */
-	outb((0x0f << RTS_LEVEL_SHIFT_BITS) | 1, iobase + REG_RX_CONTROL);
-
-	/* Timeout before it is safe to send the first HCI packet */
-	msleep(1250);
-
-	/* Register HCI device */
-	if (hci_register_dev(hdev) < 0) {
-		BT_ERR("Can't register HCI device");
-		info->hdev = NULL;
-		hci_free_dev(hdev);
-		return -ENODEV;
-	}
-
-	return 0;
-}
-
-
-static int bluecard_close(struct bluecard_info *info)
-{
-	unsigned int iobase = info->p_dev->resource[0]->start;
-	struct hci_dev *hdev = info->hdev;
-
-	if (!hdev)
-		return -ENODEV;
-
-	bluecard_hci_close(hdev);
-
-	clear_bit(CARD_READY, &(info->hw_state));
-
-	/* Reset card */
-	info->ctrl_reg = REG_CONTROL_BT_RESET | REG_CONTROL_CARD_RESET;
-	outb(info->ctrl_reg, iobase + REG_CONTROL);
-
-	/* Turn FPGA off */
-	outb(0x80, iobase + 0x30);
-
-	hci_unregister_dev(hdev);
-	hci_free_dev(hdev);
-
-	return 0;
-}
-
-static int bluecard_probe(struct pcmcia_device *link)
-{
-	struct bluecard_info *info;
-
-	/* Create new info device */
-	info = devm_kzalloc(&link->dev, sizeof(*info), GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	info->p_dev = link;
-	link->priv = info;
-
-	link->config_flags |= CONF_ENABLE_IRQ;
-
-	return bluecard_config(link);
-}
-
-
-static void bluecard_detach(struct pcmcia_device *link)
-{
-	bluecard_release(link);
-}
-
-
-static int bluecard_config(struct pcmcia_device *link)
-{
-	struct bluecard_info *info = link->priv;
-	int i, n;
-
-	link->config_index = 0x20;
-
-	link->resource[0]->flags |= IO_DATA_PATH_WIDTH_8;
-	link->resource[0]->end = 64;
-	link->io_lines = 6;
-
-	for (n = 0; n < 0x400; n += 0x40) {
-		link->resource[0]->start = n ^ 0x300;
-		i = pcmcia_request_io(link);
-		if (i == 0)
-			break;
-	}
-
-	if (i != 0)
-		goto failed;
-
-	i = pcmcia_request_irq(link, bluecard_interrupt);
-	if (i != 0)
-		goto failed;
-
-	i = pcmcia_enable_device(link);
-	if (i != 0)
-		goto failed;
-
-	if (bluecard_open(info) != 0)
-		goto failed;
-
-	return 0;
-
-failed:
-	bluecard_release(link);
-	return -ENODEV;
-}
-
-
-static void bluecard_release(struct pcmcia_device *link)
-{
-	struct bluecard_info *info = link->priv;
-
-	bluecard_close(info);
-
-	timer_delete_sync(&(info->timer));
-
-	pcmcia_disable_device(link);
-}
-
-static const struct pcmcia_device_id bluecard_ids[] = {
-	PCMCIA_DEVICE_PROD_ID12("BlueCard", "LSE041", 0xbaf16fbf, 0x657cc15e),
-	PCMCIA_DEVICE_PROD_ID12("BTCFCARD", "LSE139", 0xe3987764, 0x2524b59c),
-	PCMCIA_DEVICE_PROD_ID12("WSS", "LSE039", 0x0a0736ec, 0x24e6dfab),
-	PCMCIA_DEVICE_NULL
-};
-MODULE_DEVICE_TABLE(pcmcia, bluecard_ids);
-
-static struct pcmcia_driver bluecard_driver = {
-	.owner		= THIS_MODULE,
-	.name		= "bluecard_cs",
-	.probe		= bluecard_probe,
-	.remove		= bluecard_detach,
-	.id_table	= bluecard_ids,
-};
-module_pcmcia_driver(bluecard_driver);
diff --git a/drivers/bluetooth/bt3c_cs.c b/drivers/bluetooth/bt3c_cs.c
deleted file mode 100644
index 54713833951a9..0000000000000
--- a/drivers/bluetooth/bt3c_cs.c
+++ /dev/null
@@ -1,749 +0,0 @@
-/*
- *
- *  Driver for the 3Com Bluetooth PCMCIA card
- *
- *  Copyright (C) 2001-2002  Marcel Holtmann <marcel@holtmann.org>
- *                           Jose Orlando Pereira <jop@di.uminho.pt>
- *
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation;
- *
- *  Software distributed under the License is distributed on an "AS
- *  IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
- *  implied. See the License for the specific language governing
- *  rights and limitations under the License.
- *
- *  The initial developer of the original code is David A. Hinds
- *  <dahinds@users.sourceforge.net>.  Portions created by David A. Hinds
- *  are Copyright (C) 1999 David A. Hinds.  All Rights Reserved.
- *
- */
-
-#include <linux/module.h>
-
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-#include <linux/delay.h>
-#include <linux/errno.h>
-#include <linux/ptrace.h>
-#include <linux/ioport.h>
-#include <linux/spinlock.h>
-#include <linux/moduleparam.h>
-
-#include <linux/skbuff.h>
-#include <linux/string.h>
-#include <linux/serial.h>
-#include <linux/serial_reg.h>
-#include <linux/bitops.h>
-#include <asm/io.h>
-
-#include <linux/device.h>
-#include <linux/firmware.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/ciscode.h>
-#include <pcmcia/ds.h>
-#include <pcmcia/cisreg.h>
-
-#include <net/bluetooth/bluetooth.h>
-#include <net/bluetooth/hci_core.h>
-
-
-
-/* ======================== Module parameters ======================== */
-
-
-MODULE_AUTHOR("Marcel Holtmann <marcel@holtmann.org>");
-MODULE_DESCRIPTION("Bluetooth driver for the 3Com Bluetooth PCMCIA card");
-MODULE_LICENSE("GPL");
-MODULE_FIRMWARE("BT3CPCC.bin");
-
-
-
-/* ======================== Local structures ======================== */
-
-
-struct bt3c_info {
-	struct pcmcia_device *p_dev;
-
-	struct hci_dev *hdev;
-
-	spinlock_t lock;		/* For serializing operations */
-
-	struct sk_buff_head txq;
-	unsigned long tx_state;
-
-	unsigned long rx_state;
-	unsigned long rx_count;
-	struct sk_buff *rx_skb;
-};
-
-
-static int bt3c_config(struct pcmcia_device *link);
-static void bt3c_release(struct pcmcia_device *link);
-
-static void bt3c_detach(struct pcmcia_device *p_dev);
-
-
-/* Transmit states  */
-#define XMIT_SENDING  1
-#define XMIT_WAKEUP   2
-#define XMIT_WAITING  8
-
-/* Receiver states */
-#define RECV_WAIT_PACKET_TYPE   0
-#define RECV_WAIT_EVENT_HEADER  1
-#define RECV_WAIT_ACL_HEADER    2
-#define RECV_WAIT_SCO_HEADER    3
-#define RECV_WAIT_DATA          4
-
-
-
-/* ======================== Special I/O functions ======================== */
-
-
-#define DATA_L   0
-#define DATA_H   1
-#define ADDR_L   2
-#define ADDR_H   3
-#define CONTROL  4
-
-
-static inline void bt3c_address(unsigned int iobase, unsigned short addr)
-{
-	outb(addr & 0xff, iobase + ADDR_L);
-	outb((addr >> 8) & 0xff, iobase + ADDR_H);
-}
-
-
-static inline void bt3c_put(unsigned int iobase, unsigned short value)
-{
-	outb(value & 0xff, iobase + DATA_L);
-	outb((value >> 8) & 0xff, iobase + DATA_H);
-}
-
-
-static inline void bt3c_io_write(unsigned int iobase, unsigned short addr, unsigned short value)
-{
-	bt3c_address(iobase, addr);
-	bt3c_put(iobase, value);
-}
-
-
-static inline unsigned short bt3c_get(unsigned int iobase)
-{
-	unsigned short value = inb(iobase + DATA_L);
-
-	value |= inb(iobase + DATA_H) << 8;
-
-	return value;
-}
-
-
-static inline unsigned short bt3c_read(unsigned int iobase, unsigned short addr)
-{
-	bt3c_address(iobase, addr);
-
-	return bt3c_get(iobase);
-}
-
-
-
-/* ======================== Interrupt handling ======================== */
-
-
-static int bt3c_write(unsigned int iobase, int fifo_size, __u8 *buf, int len)
-{
-	int actual = 0;
-
-	bt3c_address(iobase, 0x7080);
-
-	/* Fill FIFO with current frame */
-	while (actual < len) {
-		/* Transmit next byte */
-		bt3c_put(iobase, buf[actual]);
-		actual++;
-	}
-
-	bt3c_io_write(iobase, 0x7005, actual);
-
-	return actual;
-}
-
-
-static void bt3c_write_wakeup(struct bt3c_info *info)
-{
-	if (!info) {
-		BT_ERR("Unknown device");
-		return;
-	}
-
-	if (test_and_set_bit(XMIT_SENDING, &(info->tx_state)))
-		return;
-
-	do {
-		unsigned int iobase = info->p_dev->resource[0]->start;
-		register struct sk_buff *skb;
-		int len;
-
-		if (!pcmcia_dev_present(info->p_dev))
-			break;
-
-		skb = skb_dequeue(&(info->txq));
-		if (!skb) {
-			clear_bit(XMIT_SENDING, &(info->tx_state));
-			break;
-		}
-
-		/* Send frame */
-		len = bt3c_write(iobase, 256, skb->data, skb->len);
-
-		if (len != skb->len)
-			BT_ERR("Very strange");
-
-		kfree_skb(skb);
-
-		info->hdev->stat.byte_tx += len;
-
-	} while (0);
-}
-
-
-static void bt3c_receive(struct bt3c_info *info)
-{
-	unsigned int iobase;
-	int size = 0, avail;
-
-	if (!info) {
-		BT_ERR("Unknown device");
-		return;
-	}
-
-	iobase = info->p_dev->resource[0]->start;
-
-	avail = bt3c_read(iobase, 0x7006);
-
-	bt3c_address(iobase, 0x7480);
-	while (size < avail) {
-		size++;
-		info->hdev->stat.byte_rx++;
-
-		/* Allocate packet */
-		if (!info->rx_skb) {
-			info->rx_state = RECV_WAIT_PACKET_TYPE;
-			info->rx_count = 0;
-			info->rx_skb = bt_skb_alloc(HCI_MAX_FRAME_SIZE, GFP_ATOMIC);
-			if (!info->rx_skb) {
-				BT_ERR("Can't allocate mem for new packet");
-				return;
-			}
-		}
-
-
-		if (info->rx_state == RECV_WAIT_PACKET_TYPE) {
-
-			hci_skb_pkt_type(info->rx_skb) = inb(iobase + DATA_L);
-			inb(iobase + DATA_H);
-
-			switch (hci_skb_pkt_type(info->rx_skb)) {
-
-			case HCI_EVENT_PKT:
-				info->rx_state = RECV_WAIT_EVENT_HEADER;
-				info->rx_count = HCI_EVENT_HDR_SIZE;
-				break;
-
-			case HCI_ACLDATA_PKT:
-				info->rx_state = RECV_WAIT_ACL_HEADER;
-				info->rx_count = HCI_ACL_HDR_SIZE;
-				break;
-
-			case HCI_SCODATA_PKT:
-				info->rx_state = RECV_WAIT_SCO_HEADER;
-				info->rx_count = HCI_SCO_HDR_SIZE;
-				break;
-
-			default:
-				/* Unknown packet */
-				BT_ERR("Unknown HCI packet with type 0x%02x received",
-				       hci_skb_pkt_type(info->rx_skb));
-				info->hdev->stat.err_rx++;
-
-				kfree_skb(info->rx_skb);
-				info->rx_skb = NULL;
-				break;
-
-			}
-
-		} else {
-
-			__u8 x = inb(iobase + DATA_L);
-
-			skb_put_u8(info->rx_skb, x);
-			inb(iobase + DATA_H);
-			info->rx_count--;
-
-			if (info->rx_count == 0) {
-
-				int dlen;
-				struct hci_event_hdr *eh;
-				struct hci_acl_hdr *ah;
-				struct hci_sco_hdr *sh;
-
-				switch (info->rx_state) {
-
-				case RECV_WAIT_EVENT_HEADER:
-					eh = hci_event_hdr(info->rx_skb);
-					info->rx_state = RECV_WAIT_DATA;
-					info->rx_count = eh->plen;
-					break;
-
-				case RECV_WAIT_ACL_HEADER:
-					ah = hci_acl_hdr(info->rx_skb);
-					dlen = __le16_to_cpu(ah->dlen);
-					info->rx_state = RECV_WAIT_DATA;
-					info->rx_count = dlen;
-					break;
-
-				case RECV_WAIT_SCO_HEADER:
-					sh = hci_sco_hdr(info->rx_skb);
-					info->rx_state = RECV_WAIT_DATA;
-					info->rx_count = sh->dlen;
-					break;
-
-				case RECV_WAIT_DATA:
-					hci_recv_frame(info->hdev, info->rx_skb);
-					info->rx_skb = NULL;
-					break;
-
-				}
-
-			}
-
-		}
-
-	}
-
-	bt3c_io_write(iobase, 0x7006, 0x0000);
-}
-
-
-static irqreturn_t bt3c_interrupt(int irq, void *dev_inst)
-{
-	struct bt3c_info *info = dev_inst;
-	unsigned int iobase;
-	int iir;
-	irqreturn_t r = IRQ_NONE;
-
-	if (!info || !info->hdev)
-		/* our irq handler is shared */
-		return IRQ_NONE;
-
-	iobase = info->p_dev->resource[0]->start;
-
-	spin_lock(&(info->lock));
-
-	iir = inb(iobase + CONTROL);
-	if (iir & 0x80) {
-		int stat = bt3c_read(iobase, 0x7001);
-
-		if ((stat & 0xff) == 0x7f) {
-			BT_ERR("Very strange (stat=0x%04x)", stat);
-		} else if ((stat & 0xff) != 0xff) {
-			if (stat & 0x0020) {
-				int status = bt3c_read(iobase, 0x7002) & 0x10;
-				bt_dev_info(info->hdev, "Antenna %s",
-							status ? "out" : "in");
-			}
-			if (stat & 0x0001)
-				bt3c_receive(info);
-			if (stat & 0x0002) {
-				clear_bit(XMIT_SENDING, &(info->tx_state));
-				bt3c_write_wakeup(info);
-			}
-
-			bt3c_io_write(iobase, 0x7001, 0x0000);
-
-			outb(iir, iobase + CONTROL);
-		}
-		r = IRQ_HANDLED;
-	}
-
-	spin_unlock(&(info->lock));
-
-	return r;
-}
-
-
-
-/* ======================== HCI interface ======================== */
-
-
-static int bt3c_hci_flush(struct hci_dev *hdev)
-{
-	struct bt3c_info *info = hci_get_drvdata(hdev);
-
-	/* Drop TX queue */
-	skb_queue_purge(&(info->txq));
-
-	return 0;
-}
-
-
-static int bt3c_hci_open(struct hci_dev *hdev)
-{
-	return 0;
-}
-
-
-static int bt3c_hci_close(struct hci_dev *hdev)
-{
-	bt3c_hci_flush(hdev);
-
-	return 0;
-}
-
-
-static int bt3c_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct bt3c_info *info = hci_get_drvdata(hdev);
-	unsigned long flags;
-
-	switch (hci_skb_pkt_type(skb)) {
-	case HCI_COMMAND_PKT:
-		hdev->stat.cmd_tx++;
-		break;
-	case HCI_ACLDATA_PKT:
-		hdev->stat.acl_tx++;
-		break;
-	case HCI_SCODATA_PKT:
-		hdev->stat.sco_tx++;
-		break;
-	}
-
-	/* Prepend skb with frame type */
-	memcpy(skb_push(skb, 1), &hci_skb_pkt_type(skb), 1);
-	skb_queue_tail(&(info->txq), skb);
-
-	spin_lock_irqsave(&(info->lock), flags);
-
-	bt3c_write_wakeup(info);
-
-	spin_unlock_irqrestore(&(info->lock), flags);
-
-	return 0;
-}
-
-
-
-/* ======================== Card services HCI interaction ======================== */
-
-
-static int bt3c_load_firmware(struct bt3c_info *info,
-			      const unsigned char *firmware,
-			      int count)
-{
-	char *ptr = (char *) firmware;
-	char b[9];
-	unsigned int iobase, tmp, tn;
-	unsigned long size, addr, fcs;
-	int i, err = 0;
-
-	iobase = info->p_dev->resource[0]->start;
-
-	/* Reset */
-	bt3c_io_write(iobase, 0x8040, 0x0404);
-	bt3c_io_write(iobase, 0x8040, 0x0400);
-
-	udelay(1);
-
-	bt3c_io_write(iobase, 0x8040, 0x0404);
-
-	udelay(17);
-
-	/* Load */
-	while (count) {
-		if (ptr[0] != 'S') {
-			BT_ERR("Bad address in firmware");
-			err = -EFAULT;
-			goto error;
-		}
-
-		memset(b, 0, sizeof(b));
-		memcpy(b, ptr + 2, 2);
-		if (kstrtoul(b, 16, &size) < 0)
-			return -EINVAL;
-
-		memset(b, 0, sizeof(b));
-		memcpy(b, ptr + 4, 8);
-		if (kstrtoul(b, 16, &addr) < 0)
-			return -EINVAL;
-
-		memset(b, 0, sizeof(b));
-		memcpy(b, ptr + (size * 2) + 2, 2);
-		if (kstrtoul(b, 16, &fcs) < 0)
-			return -EINVAL;
-
-		memset(b, 0, sizeof(b));
-		for (tmp = 0, i = 0; i < size; i++) {
-			memcpy(b, ptr + (i * 2) + 2, 2);
-			if (kstrtouint(b, 16, &tn))
-				return -EINVAL;
-			tmp += tn;
-		}
-
-		if (((tmp + fcs) & 0xff) != 0xff) {
-			BT_ERR("Checksum error in firmware");
-			err = -EILSEQ;
-			goto error;
-		}
-
-		if (ptr[1] == '3') {
-			bt3c_address(iobase, addr);
-
-			memset(b, 0, sizeof(b));
-			for (i = 0; i < (size - 4) / 2; i++) {
-				memcpy(b, ptr + (i * 4) + 12, 4);
-				if (kstrtouint(b, 16, &tmp))
-					return -EINVAL;
-				bt3c_put(iobase, tmp);
-			}
-		}
-
-		ptr   += (size * 2) + 6;
-		count -= (size * 2) + 6;
-	}
-
-	udelay(17);
-
-	/* Boot */
-	bt3c_address(iobase, 0x3000);
-	outb(inb(iobase + CONTROL) | 0x40, iobase + CONTROL);
-
-error:
-	udelay(17);
-
-	/* Clear */
-	bt3c_io_write(iobase, 0x7006, 0x0000);
-	bt3c_io_write(iobase, 0x7005, 0x0000);
-	bt3c_io_write(iobase, 0x7001, 0x0000);
-
-	return err;
-}
-
-
-static int bt3c_open(struct bt3c_info *info)
-{
-	const struct firmware *firmware;
-	struct hci_dev *hdev;
-	int err;
-
-	spin_lock_init(&(info->lock));
-
-	skb_queue_head_init(&(info->txq));
-
-	info->rx_state = RECV_WAIT_PACKET_TYPE;
-	info->rx_count = 0;
-	info->rx_skb = NULL;
-
-	/* Initialize HCI device */
-	hdev = hci_alloc_dev();
-	if (!hdev) {
-		BT_ERR("Can't allocate HCI device");
-		return -ENOMEM;
-	}
-
-	info->hdev = hdev;
-
-	hdev->bus = HCI_PCCARD;
-	hci_set_drvdata(hdev, info);
-	SET_HCIDEV_DEV(hdev, &info->p_dev->dev);
-
-	hdev->open  = bt3c_hci_open;
-	hdev->close = bt3c_hci_close;
-	hdev->flush = bt3c_hci_flush;
-	hdev->send  = bt3c_hci_send_frame;
-
-	/* Load firmware */
-	err = request_firmware(&firmware, "BT3CPCC.bin", &info->p_dev->dev);
-	if (err < 0) {
-		BT_ERR("Firmware request failed");
-		goto error;
-	}
-
-	err = bt3c_load_firmware(info, firmware->data, firmware->size);
-
-	release_firmware(firmware);
-
-	if (err < 0) {
-		BT_ERR("Firmware loading failed");
-		goto error;
-	}
-
-	/* Timeout before it is safe to send the first HCI packet */
-	msleep(1000);
-
-	/* Register HCI device */
-	err = hci_register_dev(hdev);
-	if (err < 0) {
-		BT_ERR("Can't register HCI device");
-		goto error;
-	}
-
-	return 0;
-
-error:
-	info->hdev = NULL;
-	hci_free_dev(hdev);
-	return err;
-}
-
-
-static int bt3c_close(struct bt3c_info *info)
-{
-	struct hci_dev *hdev = info->hdev;
-
-	if (!hdev)
-		return -ENODEV;
-
-	bt3c_hci_close(hdev);
-
-	hci_unregister_dev(hdev);
-	hci_free_dev(hdev);
-
-	return 0;
-}
-
-static int bt3c_probe(struct pcmcia_device *link)
-{
-	struct bt3c_info *info;
-
-	/* Create new info device */
-	info = devm_kzalloc(&link->dev, sizeof(*info), GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	info->p_dev = link;
-	link->priv = info;
-
-	link->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_VPP |
-		CONF_AUTO_SET_IO;
-
-	return bt3c_config(link);
-}
-
-
-static void bt3c_detach(struct pcmcia_device *link)
-{
-	bt3c_release(link);
-}
-
-static int bt3c_check_config(struct pcmcia_device *p_dev, void *priv_data)
-{
-	int *try = priv_data;
-
-	if (!try)
-		p_dev->io_lines = 16;
-
-	if ((p_dev->resource[0]->end != 8) || (p_dev->resource[0]->start == 0))
-		return -EINVAL;
-
-	p_dev->resource[0]->end = 8;
-	p_dev->resource[0]->flags &= ~IO_DATA_PATH_WIDTH;
-	p_dev->resource[0]->flags |= IO_DATA_PATH_WIDTH_8;
-
-	return pcmcia_request_io(p_dev);
-}
-
-static int bt3c_check_config_notpicky(struct pcmcia_device *p_dev,
-				      void *priv_data)
-{
-	static unsigned int base[5] = { 0x3f8, 0x2f8, 0x3e8, 0x2e8, 0x0 };
-	int j;
-
-	if (p_dev->io_lines > 3)
-		return -ENODEV;
-
-	p_dev->resource[0]->flags &= ~IO_DATA_PATH_WIDTH;
-	p_dev->resource[0]->flags |= IO_DATA_PATH_WIDTH_8;
-	p_dev->resource[0]->end = 8;
-
-	for (j = 0; j < 5; j++) {
-		p_dev->resource[0]->start = base[j];
-		p_dev->io_lines = base[j] ? 16 : 3;
-		if (!pcmcia_request_io(p_dev))
-			return 0;
-	}
-	return -ENODEV;
-}
-
-static int bt3c_config(struct pcmcia_device *link)
-{
-	struct bt3c_info *info = link->priv;
-	int i;
-	unsigned long try;
-
-	/* First pass: look for a config entry that looks normal.
-	 * Two tries: without IO aliases, then with aliases
-	 */
-	for (try = 0; try < 2; try++)
-		if (!pcmcia_loop_config(link, bt3c_check_config, (void *) try))
-			goto found_port;
-
-	/* Second pass: try to find an entry that isn't picky about
-	 * its base address, then try to grab any standard serial port
-	 * address, and finally try to get any free port.
-	 */
-	if (!pcmcia_loop_config(link, bt3c_check_config_notpicky, NULL))
-		goto found_port;
-
-	BT_ERR("No usable port range found");
-	goto failed;
-
-found_port:
-	i = pcmcia_request_irq(link, &bt3c_interrupt);
-	if (i != 0)
-		goto failed;
-
-	i = pcmcia_enable_device(link);
-	if (i != 0)
-		goto failed;
-
-	if (bt3c_open(info) != 0)
-		goto failed;
-
-	return 0;
-
-failed:
-	bt3c_release(link);
-	return -ENODEV;
-}
-
-
-static void bt3c_release(struct pcmcia_device *link)
-{
-	struct bt3c_info *info = link->priv;
-
-	bt3c_close(info);
-
-	pcmcia_disable_device(link);
-}
-
-
-static const struct pcmcia_device_id bt3c_ids[] = {
-	PCMCIA_DEVICE_PROD_ID13("3COM", "Bluetooth PC Card", 0xefce0a31, 0xd4ce9b02),
-	PCMCIA_DEVICE_NULL
-};
-MODULE_DEVICE_TABLE(pcmcia, bt3c_ids);
-
-static struct pcmcia_driver bt3c_driver = {
-	.owner		= THIS_MODULE,
-	.name		= "bt3c_cs",
-	.probe		= bt3c_probe,
-	.remove		= bt3c_detach,
-	.id_table	= bt3c_ids,
-};
-module_pcmcia_driver(bt3c_driver);
diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c
index dcaaa4ca02b99..5e9cac090bd8f 100644
--- a/drivers/bluetooth/btintel.c
+++ b/drivers/bluetooth/btintel.c
@@ -67,9 +67,10 @@ static struct {
 	u32        fw_build_num;
 } coredump_info;
 
-static const guid_t btintel_guid_dsm =
+const guid_t btintel_guid_dsm =
 	GUID_INIT(0xaa10f4e0, 0x81ac, 0x4233,
 		  0xab, 0xf6, 0x3b, 0x2a, 0xc5, 0x0e, 0x28, 0xd9);
+EXPORT_SYMBOL_GPL(btintel_guid_dsm);
 
 int btintel_check_bdaddr(struct hci_dev *hdev)
 {
@@ -2624,7 +2625,7 @@ static void btintel_set_ppag(struct hci_dev *hdev, struct intel_version_tlv *ver
 	kfree_skb(skb);
 }
 
-static int btintel_acpi_reset_method(struct hci_dev *hdev)
+int btintel_acpi_reset_method(struct hci_dev *hdev)
 {
 	int ret = 0;
 	acpi_status status;
@@ -2632,14 +2633,14 @@ static int btintel_acpi_reset_method(struct hci_dev *hdev)
 	struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL };
 
 	status = acpi_evaluate_object(ACPI_HANDLE(GET_HCIDEV_DEV(hdev)), "_PRR", NULL, &buffer);
-	if (ACPI_FAILURE(status)) {
+	if (ACPI_FAILURE(status) || !buffer.pointer) {
 		bt_dev_err(hdev, "Failed to run _PRR method");
 		ret = -ENODEV;
 		return ret;
 	}
 	p = buffer.pointer;
 
-	if (p->package.count != 1 || p->type != ACPI_TYPE_PACKAGE) {
+	if (p->type != ACPI_TYPE_PACKAGE || p->package.count != 1) {
 		bt_dev_err(hdev, "Invalid arguments");
 		ret = -EINVAL;
 		goto exit_on_error;
@@ -2663,6 +2664,7 @@ static int btintel_acpi_reset_method(struct hci_dev *hdev)
 	kfree(buffer.pointer);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(btintel_acpi_reset_method);
 
 static void btintel_set_dsm_reset_method(struct hci_dev *hdev,
 					 struct intel_version_tlv *ver_tlv)
diff --git a/drivers/bluetooth/btintel.h b/drivers/bluetooth/btintel.h
index 0e9ca99aaaaea..70d812ad36a22 100644
--- a/drivers/bluetooth/btintel.h
+++ b/drivers/bluetooth/btintel.h
@@ -79,6 +79,8 @@ struct intel_tlv {
 #define BTINTEL_HWID_SCP2	0x20	/* Scorpius Peak2 - Nova Lake */
 #define BTINTEL_HWID_BZRIW	0x22	/* BlazarIW - Wildcat Lake */
 
+extern const guid_t btintel_guid_dsm;
+
 struct intel_version_tlv {
 	u32	cnvi_top;
 	u32	cnvr_top;
@@ -289,6 +291,7 @@ int btintel_bootloader_setup_tlv(struct hci_dev *hdev,
 int btintel_shutdown_combined(struct hci_dev *hdev);
 void btintel_hw_error(struct hci_dev *hdev, u8 code);
 void btintel_print_fseq_info(struct hci_dev *hdev);
+int btintel_acpi_reset_method(struct hci_dev *hdev);
 #else
 
 static inline int btintel_check_bdaddr(struct hci_dev *hdev)
@@ -422,4 +425,8 @@ static inline void btintel_hw_error(struct hci_dev *hdev, u8 code)
 static inline void btintel_print_fseq_info(struct hci_dev *hdev)
 {
 }
+static inline int btintel_acpi_reset_method(struct hci_dev *hdev)
+{
+	return -ENODEV;
+}
 #endif
diff --git a/drivers/bluetooth/btintel_pcie.c b/drivers/bluetooth/btintel_pcie.c
index a3643e67b33fd..eba563b660904 100644
--- a/drivers/bluetooth/btintel_pcie.c
+++ b/drivers/bluetooth/btintel_pcie.c
@@ -15,6 +15,7 @@
 #include <linux/wait.h>
 #include <linux/delay.h>
 #include <linux/interrupt.h>
+#include <linux/acpi.h>
 
 #include <linux/unaligned.h>
 #include <linux/devcoredump.h>
@@ -102,6 +103,22 @@ enum {
 	BTINTEL_PCIE_D3
 };
 
+enum {
+	BTINTEL_PCIE_DSM_SET_RESET_TIMING = 1,
+	BTINTEL_PCIE_DSM_GET_RESET_TIMING = 2,
+	BTINTEL_PCIE_DSM_BT_PLDR_CONFIG = 3,
+	BTINTEL_PCIE_DSM_GET_RESET_TYPE = 4,
+	BTINTEL_PCIE_DSM_DYNAMIC_PLDR = 5,
+	BTINTEL_PCIE_DSM_GET_RESET_METHOD = 6,
+	BTINTEL_PCIE_DSM_SET_PLDR_DELAY = 7,
+};
+
+enum btintel_dsm_internal_product_reset_mode {
+	BTINTEL_PCIE_DSM_PLDR_MODE_EN_PROD_RESET	= BIT(0),
+	BTINTEL_PCIE_DSM_PLDR_MODE_EN_WIFI_FLR		= BIT(1),
+	BTINTEL_PCIE_DSM_PLDR_MODE_EN_BT_OFF_ON		= BIT(2),
+};
+
 /* Structure for dbgc fragment buffer
  * @buf_addr_lsb: LSB of the buffer's physical address
  * @buf_addr_msb: MSB of the buffer's physical address
@@ -128,11 +145,6 @@ struct btintel_pcie_dbgc_ctxt {
 	struct btintel_pcie_dbgc_ctxt_buf bufs[BTINTEL_PCIE_DBGC_BUFFER_COUNT];
 };
 
-struct btintel_pcie_removal {
-	struct pci_dev *pdev;
-	struct work_struct work;
-};
-
 static LIST_HEAD(btintel_pcie_recovery_list);
 static DEFINE_SPINLOCK(btintel_pcie_recovery_lock);
 
@@ -582,12 +594,10 @@ static int btintel_pcie_get_mac_access(struct btintel_pcie_data *data)
 
 	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 
-	reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS;
-	reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ;
-	if ((reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS) == 0)
+	if (!(reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ)) {
 		reg |= BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ;
-
-	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
+		btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
+	}
 
 	do {
 		reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
@@ -607,16 +617,10 @@ static void btintel_pcie_release_mac_access(struct btintel_pcie_data *data)
 
 	reg = btintel_pcie_rd_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG);
 
-	if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ)
+	if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ) {
 		reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ;
-
-	if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS)
-		reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS;
-
-	if (reg & BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ)
-		reg &= ~BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ;
-
-	btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
+		btintel_pcie_wr_reg32(data, BTINTEL_PCIE_CSR_FUNC_CTRL_REG, reg);
+	}
 }
 
 static void *btintel_pcie_copy_tlv(void *dest, enum btintel_pcie_tlv_type type,
@@ -2273,21 +2277,132 @@ static void btintel_pcie_inc_recovery_count(struct pci_dev *pdev,
 }
 
 static int btintel_pcie_setup_hdev(struct btintel_pcie_data *data);
+static void btintel_pcie_reset(struct hci_dev *hdev);
 
-static void btintel_pcie_removal_work(struct work_struct *wk)
+static int btintel_pcie_acpi_reset_method(struct btintel_pcie_data *data)
 {
-	struct btintel_pcie_removal *removal =
-		container_of(wk, struct btintel_pcie_removal, work);
-	struct pci_dev *pdev = removal->pdev;
-	struct btintel_pcie_data *data;
+	union acpi_object *obj, argv4;
+	acpi_handle handle;
+	int ret;
+	struct pldr_mode {
+		__le16	cmd_type;
+		__le16	cmd_payload;
+	} __packed;
+
+	/* set 1 for _PRR mode
+	 * Product Reset (PLDR Abort flow)
+	 */
+	static const struct pldr_mode mode = {
+		.cmd_type = cpu_to_le16(1),
+		.cmd_payload = cpu_to_le16(BTINTEL_PCIE_DSM_PLDR_MODE_EN_PROD_RESET |
+			       BTINTEL_PCIE_DSM_PLDR_MODE_EN_WIFI_FLR),
+	};
+	struct hci_dev *hdev = data->hdev;
+
+	handle = ACPI_HANDLE(GET_HCIDEV_DEV(data->hdev));
+	if (!handle) {
+		bt_dev_err(data->hdev, "No support for bluetooth device in ACPI firmware");
+		return -EACCES;
+	}
+
+	if (!acpi_has_method(handle, "_PRR")) {
+		bt_dev_err(data->hdev, "No support for _PRR ACPI method, cold boot");
+		return -ENODEV;
+	}
+
+	argv4.buffer.type = ACPI_TYPE_BUFFER;
+	argv4.buffer.length = sizeof(mode);
+	argv4.buffer.pointer = (void *)&mode;
+
+	obj = acpi_evaluate_dsm(handle, &btintel_guid_dsm, 0,
+				BTINTEL_PCIE_DSM_DYNAMIC_PLDR, &argv4);
+	if (!obj) {
+		bt_dev_err(data->hdev, "Failed to call dsm to set reset method");
+		return -EIO;
+	}
+	ACPI_FREE(obj);
+
+	pci_dev_lock(data->pdev);
+	pci_save_state(data->pdev);
+	ret = btintel_acpi_reset_method(hdev);
+	if (ret)
+		bt_dev_err(data->hdev, "ACPI _PRR reset failed (%d), PLDR incomplete",
+			   ret);
+	pci_restore_state(data->pdev);
+	pci_dev_unlock(data->pdev);
+	return ret;
+}
+
+static void btintel_pcie_perform_pldr(struct btintel_pcie_data *data)
+{
+	struct pci_dev *pdev = data->pdev;
+	struct pci_dev *wifi = NULL;
+	struct pci_bus *bus;
+	int ret;
+	/* on integrated we have to look up by ID (same bus) */
+	static const struct pci_device_id wifi_device_ids[] = {
+	#define WIFI_DEV(_id) { PCI_DEVICE(PCI_VENDOR_ID_INTEL, _id) }
+		WIFI_DEV(0xA840), /* LNL */
+		WIFI_DEV(0xE440), /* PTL-P */
+		WIFI_DEV(0xE340), /* PTL-H */
+		WIFI_DEV(0xD340), /* NVL-H */
+		WIFI_DEV(0x6E70), /* NVL-S */
+		WIFI_DEV(0x4D40), /* WCL */
+		WIFI_DEV(0xD240), /* RZL-H */
+		WIFI_DEV(0x6C40), /* RZL-M */
+		{}
+	};
+	struct pci_dev *tmp = NULL;
+
+	bus = pdev->bus;
+	if (!bus)
+		return;
+
+	list_for_each_entry(tmp, &bus->devices, bus_list) {
+		if (pci_match_id(wifi_device_ids, tmp)) {
+			wifi = pci_dev_get(tmp);
+			break;
+		}
+	}
+
+	if (wifi)
+		device_release_driver(&wifi->dev);
+
+	/* Wi-Fi is fully unbound before the reset and fully reprobed after
+	 * the normal PCI probe path handles all state setup from scratch.
+	 * BT needs pci_save_state()/pci_restore_state() because the BT driver
+	 * is still partially attached when the _PRR runs (it hasn't been unbound yet).
+	 * The PCI device needs to remain minimally functional so that
+	 * device_reprobe(&pdev->dev) can work afterward
+	 */
+	ret = btintel_pcie_acpi_reset_method(data);
+
+	if (wifi) {
+		if (device_reprobe(&wifi->dev))
+			BT_ERR("WiFi reprobe failed for BDF:%s", pci_name(wifi));
+		pci_dev_put(wifi);
+	}
+
+	if (!ret) {
+		if (device_reprobe(&pdev->dev))
+			BT_ERR("BT reprobe failed for BDF:%s", pci_name(pdev));
+	}
+}
+
+static void btintel_pcie_reset_work(struct work_struct *wk)
+{
+	struct btintel_pcie_data *data =
+		container_of(wk, struct btintel_pcie_data, reset_work);
+	struct pci_dev *pdev = data->pdev;
 	int err;
 
 	pci_lock_rescan_remove();
 
 	if (!pdev->bus)
-		goto error;
+		goto out;
 
-	data = pci_get_drvdata(pdev);
+	if (!data)
+		goto out;
 
 	btintel_pcie_disable_interrupts(data);
 	btintel_pcie_synchronize_irqs(data);
@@ -2295,12 +2410,21 @@ static void btintel_pcie_removal_work(struct work_struct *wk)
 	flush_work(&data->rx_work);
 
 	bt_dev_dbg(data->hdev, "Release bluetooth interface");
+	if (data->reset_type == BTINTEL_PCIE_IOSF_PRR_PLDR) {
+		/* This function holds pci_lock_rescan_remove(), which acquires
+		 * pci_rescan_remove_lock. This mutex serializes against PCI device
+		 * addition/removal (hotplug), so no device can be added to or
+		 * removed from the bus list while this code runs.
+		 */
+		btintel_pcie_perform_pldr(data);
+		goto out;
+	}
 	btintel_pcie_release_hdev(data);
 
 	err = pci_reset_function(pdev);
 	if (err) {
 		BT_ERR("Failed resetting the pcie device (%d)", err);
-		goto error;
+		goto out;
 	}
 
 	btintel_pcie_enable_interrupts(data);
@@ -2310,7 +2434,7 @@ static void btintel_pcie_removal_work(struct work_struct *wk)
 	if (err) {
 		BT_ERR("Failed to enable bluetooth hardware after reset (%d)",
 		       err);
-		goto error;
+		goto out;
 	}
 
 	btintel_pcie_reset_ia(data);
@@ -2320,17 +2444,15 @@ static void btintel_pcie_removal_work(struct work_struct *wk)
 	err = btintel_pcie_setup_hdev(data);
 	if (err) {
 		BT_ERR("Failed registering hdev (%d)", err);
-		goto error;
+		goto out;
 	}
-error:
+out:
 	pci_dev_put(pdev);
 	pci_unlock_rescan_remove();
-	kfree(removal);
 }
 
 static void btintel_pcie_reset(struct hci_dev *hdev)
 {
-	struct btintel_pcie_removal *removal;
 	struct btintel_pcie_data *data;
 
 	data = hci_get_drvdata(hdev);
@@ -2341,14 +2463,8 @@ static void btintel_pcie_reset(struct hci_dev *hdev)
 	if (test_and_set_bit(BTINTEL_PCIE_RECOVERY_IN_PROGRESS, &data->flags))
 		return;
 
-	removal = kzalloc_obj(*removal, GFP_ATOMIC);
-	if (!removal)
-		return;
-
-	removal->pdev = data->pdev;
-	INIT_WORK(&removal->work, btintel_pcie_removal_work);
-	pci_dev_get(removal->pdev);
-	schedule_work(&removal->work);
+	pci_dev_get(data->pdev);
+	schedule_work(&data->reset_work);
 }
 
 static void btintel_pcie_hw_error(struct hci_dev *hdev, u8 code)
@@ -2358,15 +2474,19 @@ static void btintel_pcie_hw_error(struct hci_dev *hdev, u8 code)
 	struct pci_dev *pdev = dev_data->pdev;
 	time64_t retry_window;
 
-	if (code == 0x13) {
-		bt_dev_err(hdev, "Encountered top exception");
-		return;
-	}
+	btintel_pcie_dump_debug_registers(hdev);
 
 	data = btintel_pcie_get_recovery(pdev, &hdev->dev);
 	if (!data)
 		return;
 
+	if (code == 0x13)
+		dev_data->reset_type = BTINTEL_PCIE_IOSF_PRR_PLDR;
+	else
+		dev_data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR;
+
+	bt_dev_err(hdev, "Encountered exception err:0x%x triggering: %s", code,
+		   dev_data->reset_type == BTINTEL_PCIE_IOSF_PRR_PLDR ? "PLDR" : "FLR");
 	retry_window = ktime_get_boottime_seconds() - data->last_error;
 
 	if (retry_window < BTINTEL_PCIE_RESET_WINDOW_SECS &&
@@ -2519,10 +2639,14 @@ static int btintel_pcie_probe(struct pci_dev *pdev,
 
 	skb_queue_head_init(&data->rx_skb_q);
 	INIT_WORK(&data->rx_work, btintel_pcie_rx_work);
+	INIT_WORK(&data->reset_work, btintel_pcie_reset_work);
 
 	data->boot_stage_cache = 0x00;
 	data->img_resp_cache = 0x00;
-
+	/* FLR can be invoked by echoing to debugfs path, so explicitly
+	 * initialized
+	 */
+	data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR;
 	err = btintel_pcie_config_pcie(pdev, data);
 	if (err)
 		goto exit_error;
@@ -2571,6 +2695,14 @@ static void btintel_pcie_remove(struct pci_dev *pdev)
 
 	data = pci_get_drvdata(pdev);
 
+	/* Cancel pending reset work. Skip only when remove() is called from
+	 * within the reset work itself (PLDR device_reprobe path) to avoid
+	 * deadlock. current_work() returns the work_struct of the caller if
+	 * we are in a workqueue context.
+	 */
+	if (current_work() != &data->reset_work)
+		cancel_work_sync(&data->reset_work);
+
 	btintel_pcie_disable_interrupts(data);
 
 	btintel_pcie_synchronize_irqs(data);
@@ -2720,6 +2852,7 @@ static int btintel_pcie_resume(struct device *dev)
 	if (data->pm_sx_event == PM_EVENT_FREEZE ||
 	    data->pm_sx_event == PM_EVENT_HIBERNATE) {
 		set_bit(BTINTEL_PCIE_CORE_HALTED, &data->flags);
+		data->reset_type = BTINTEL_PCIE_IOSF_PRR_FLR;
 		btintel_pcie_reset(data->hdev);
 		return 0;
 	}
diff --git a/drivers/bluetooth/btintel_pcie.h b/drivers/bluetooth/btintel_pcie.h
index f922abd1e7d88..7fc8c46ed6899 100644
--- a/drivers/bluetooth/btintel_pcie.h
+++ b/drivers/bluetooth/btintel_pcie.h
@@ -34,9 +34,6 @@
 #define BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_STS	(BIT(20))
 
 #define BTINTEL_PCIE_CSR_FUNC_CTRL_MAC_ACCESS_REQ	(BIT(21))
-/* Stop MAC Access disconnection request */
-#define BTINTEL_PCIE_CSR_FUNC_CTRL_STOP_MAC_ACCESS_DIS	(BIT(22))
-#define BTINTEL_PCIE_CSR_FUNC_CTRL_XTAL_CLK_REQ		(BIT(23))
 
 #define BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_STS	(BIT(28))
 #define BTINTEL_PCIE_CSR_FUNC_CTRL_BUS_MASTER_DISCON	(BIT(29))
@@ -144,6 +141,11 @@ enum msix_mbox_int_causes {
 	BTINTEL_PCIE_CSR_MBOX_STATUS_MBOX4 = BIT(3), /* cause MBOX4 */
 };
 
+enum btintel_pcie_reset_type {
+	BTINTEL_PCIE_IOSF_PRR_FLR = 0,
+	BTINTEL_PCIE_IOSF_PRR_PLDR = 1,
+};
+
 #define BTINTEL_PCIE_MSIX_NON_AUTO_CLEAR_CAUSE	BIT(7)
 
 /* Minimum and Maximum number of MSI-X Vector
@@ -500,6 +502,7 @@ struct btintel_pcie_data {
 	struct workqueue_struct	*workqueue;
 	struct sk_buff_head	rx_skb_q;
 	struct work_struct	rx_work;
+	struct work_struct      reset_work;
 
 	struct dma_pool	*dma_pool;
 	dma_addr_t	dma_p_addr;
@@ -511,6 +514,7 @@ struct btintel_pcie_data {
 	struct txq	txq;
 	struct rxq	rxq;
 	u32	alive_intr_ctxt;
+	enum btintel_pcie_reset_type	reset_type;
 	struct btintel_pcie_dbgc	dbgc;
 	struct btintel_pcie_dump_header dmp_hdr;
 	u8	pm_sx_event;
diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c
index f70c1b0f89903..ea7a031000cd8 100644
--- a/drivers/bluetooth/btmtk.c
+++ b/drivers/bluetooth/btmtk.c
@@ -537,6 +537,7 @@ static void btmtk_usb_wmt_recv(struct urb *urb)
 		return;
 	} else if (urb->status == -ENOENT) {
 		/* Avoid suspend failed when usb_kill_urb */
+		kfree(urb->setup_packet);
 		return;
 	}
 
@@ -610,6 +611,7 @@ static int btmtk_usb_submit_wmt_recv_urb(struct hci_dev *hdev)
 		if (err != -EPERM && err != -ENODEV)
 			bt_dev_err(hdev, "urb %p submission failed (%d)",
 				   urb, -err);
+		kfree(dr);
 		usb_unanchor_urb(urb);
 	}
 
@@ -719,8 +721,8 @@ static int btmtk_usb_hci_wmt_sync(struct hci_dev *hdev,
 	case BTMTK_WMT_FUNC_CTRL:
 		if (!skb_pull_data(data->evt_skb,
 				   sizeof(wmt_evt_funcc->status))) {
-			err = -EINVAL;
-			goto err_free_skb;
+			status = BTMTK_WMT_ON_UNDONE;
+			break;
 		}
 
 		wmt_evt_funcc = (struct btmtk_hci_wmt_evt_funcc *)wmt_evt;
@@ -1545,6 +1547,29 @@ int btmtk_usb_shutdown(struct hci_dev *hdev)
 	return 0;
 }
 EXPORT_SYMBOL_GPL(btmtk_usb_shutdown);
+
+int btmtk_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	struct hci_event_hdr *hdr = (void *)skb->data;
+	struct hci_ev_cmd_complete *ec;
+
+	if (hdr->evt == HCI_EV_CMD_COMPLETE &&
+	    skb->len >= HCI_EVENT_HDR_SIZE + sizeof(*ec)) {
+		u16 opcode;
+
+		ec = (void *)(skb->data + HCI_EVENT_HDR_SIZE);
+		opcode = __le16_to_cpu(ec->opcode);
+
+		/* Filter vendor opcode */
+		if (opcode == 0xfc5d) {
+			kfree_skb(skb);
+			return 0;
+		}
+	}
+
+	return hci_recv_frame(hdev, skb);
+}
+EXPORT_SYMBOL_GPL(btmtk_recv_event);
 #endif
 
 MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
diff --git a/drivers/bluetooth/btmtk.h b/drivers/bluetooth/btmtk.h
index c564aedc0ce0f..c83c24897c954 100644
--- a/drivers/bluetooth/btmtk.h
+++ b/drivers/bluetooth/btmtk.h
@@ -220,6 +220,8 @@ int btmtk_usb_suspend(struct hci_dev *hdev);
 int btmtk_usb_setup(struct hci_dev *hdev);
 
 int btmtk_usb_shutdown(struct hci_dev *hdev);
+
+int btmtk_recv_event(struct hci_dev *hdev, struct sk_buff *skb);
 #else
 
 static inline int btmtk_set_bdaddr(struct hci_dev *hdev,
@@ -299,4 +301,9 @@ static inline int btmtk_usb_shutdown(struct hci_dev *hdev)
 {
 	return -EOPNOTSUPP;
 }
+
+static inline int btmtk_recv_event(struct hci_dev *hdev, struct sk_buff *skb)
+{
+	return hci_recv_frame(hdev, skb);
+}
 #endif
diff --git a/drivers/bluetooth/btrtl.c b/drivers/bluetooth/btrtl.c
index 62f9d4df3a4f8..49ecb18fea45f 100644
--- a/drivers/bluetooth/btrtl.c
+++ b/drivers/bluetooth/btrtl.c
@@ -1342,6 +1342,19 @@ void btrtl_set_quirks(struct hci_dev *hdev, struct btrtl_device_info *btrtl_dev)
 	if (!btrtl_dev->ic_info)
 		return;
 
+	switch (btrtl_dev->project_id) {
+	case CHIP_ID_8761B:
+		/* RTL8761B/BU reports HCI version 5.1 but does not support
+		 * the LE Extended Scan commands (Opcode 0x2042), causing
+		 * repeated -EBUSY failures when BlueZ attempts extended
+		 * scanning while a connection is active.
+		 */
+		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_EXT_SCAN);
+		break;
+	default:
+		break;
+	}
+
 	switch (btrtl_dev->ic_info->lmp_subver) {
 	case RTL_ROM_LMP_8703B:
 		/* 8723CS reports two pages for local ext features,
diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c
index 7f5fce93d9848..3afbad6677400 100644
--- a/drivers/bluetooth/btusb.c
+++ b/drivers/bluetooth/btusb.c
@@ -484,6 +484,7 @@ static const struct usb_device_id quirks_table[] = {
 	{ USB_DEVICE(0x8087, 0x0037), .driver_info = BTUSB_INTEL_COMBINED },
 	{ USB_DEVICE(0x8087, 0x0038), .driver_info = BTUSB_INTEL_COMBINED },
 	{ USB_DEVICE(0x8087, 0x0039), .driver_info = BTUSB_INTEL_COMBINED },
+	{ USB_DEVICE(0x8087, 0x0040), .driver_info = BTUSB_INTEL_COMBINED }, /* Lizard Peak 2 */
 	{ USB_DEVICE(0x8087, 0x07da), .driver_info = BTUSB_CSR },
 	{ USB_DEVICE(0x8087, 0x07dc), .driver_info = BTUSB_INTEL_COMBINED |
 						     BTUSB_INTEL_NO_WBS_SUPPORT |
@@ -601,6 +602,10 @@ static const struct usb_device_id quirks_table[] = {
 	/* Realtek 8922AE Bluetooth devices */
 	{ USB_DEVICE(0x0bda, 0x8922), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0xd922), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0bda, 0xd923), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3617), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3616), .driver_info = BTUSB_REALTEK |
@@ -723,6 +728,8 @@ static const struct usb_device_id quirks_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x04ca, 0x38e4), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0e8d, 0x223c), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3568), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3584), .driver_info = BTUSB_MEDIATEK |
@@ -773,6 +780,8 @@ static const struct usb_device_id quirks_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x0489, 0xe151), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x0e8d, 0x8c38), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3602), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3603), .driver_info = BTUSB_MEDIATEK |
@@ -781,6 +790,8 @@ static const struct usb_device_id quirks_table[] = {
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3608), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
+	{ USB_DEVICE(0x13d3, 0x3609), .driver_info = BTUSB_MEDIATEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3613), .driver_info = BTUSB_MEDIATEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x13d3, 0x3627), .driver_info = BTUSB_MEDIATEK |
@@ -814,6 +825,8 @@ static const struct usb_device_id quirks_table[] = {
 	{ USB_DEVICE(0x2ff8, 0xb011), .driver_info = BTUSB_REALTEK },
 
 	/* Additional Realtek 8761BUV Bluetooth devices */
+	{ USB_DEVICE(0x2c4e, 0x0115), .driver_info = BTUSB_REALTEK |
+						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x2357, 0x0604), .driver_info = BTUSB_REALTEK |
 						     BTUSB_WIDEBAND_SPEECH },
 	{ USB_DEVICE(0x0b05, 0x190e), .driver_info = BTUSB_REALTEK |
@@ -4242,6 +4255,7 @@ static int btusb_probe(struct usb_interface *intf,
 		hci_set_quirk(hdev, HCI_QUIRK_BROKEN_ENHANCED_SETUP_SYNC_CONN);
 		hci_set_quirk(hdev, HCI_QUIRK_NON_PERSISTENT_SETUP);
 		data->recv_acl = btmtk_usb_recv_acl;
+		data->recv_event = btmtk_recv_event;
 		data->suspend = btmtk_usb_suspend;
 		data->resume = btmtk_usb_resume;
 		data->disconnect = btusb_mtk_disconnect;
diff --git a/drivers/bluetooth/dtl1_cs.c b/drivers/bluetooth/dtl1_cs.c
deleted file mode 100644
index 2adfe4fade763..0000000000000
--- a/drivers/bluetooth/dtl1_cs.c
+++ /dev/null
@@ -1,614 +0,0 @@
-/*
- *
- *  A driver for Nokia Connectivity Card DTL-1 devices
- *
- *  Copyright (C) 2001-2002  Marcel Holtmann <marcel@holtmann.org>
- *
- *
- *  This program is free software; you can redistribute it and/or modify
- *  it under the terms of the GNU General Public License version 2 as
- *  published by the Free Software Foundation;
- *
- *  Software distributed under the License is distributed on an "AS
- *  IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or
- *  implied. See the License for the specific language governing
- *  rights and limitations under the License.
- *
- *  The initial developer of the original code is David A. Hinds
- *  <dahinds@users.sourceforge.net>.  Portions created by David A. Hinds
- *  are Copyright (C) 1999 David A. Hinds.  All Rights Reserved.
- *
- */
-
-#include <linux/module.h>
-
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-#include <linux/delay.h>
-#include <linux/errno.h>
-#include <linux/ptrace.h>
-#include <linux/ioport.h>
-#include <linux/spinlock.h>
-#include <linux/moduleparam.h>
-
-#include <linux/skbuff.h>
-#include <linux/string.h>
-#include <linux/serial.h>
-#include <linux/serial_reg.h>
-#include <linux/bitops.h>
-#include <asm/io.h>
-
-#include <pcmcia/cistpl.h>
-#include <pcmcia/ciscode.h>
-#include <pcmcia/ds.h>
-#include <pcmcia/cisreg.h>
-
-#include <net/bluetooth/bluetooth.h>
-#include <net/bluetooth/hci_core.h>
-
-
-
-/* ======================== Module parameters ======================== */
-
-
-MODULE_AUTHOR("Marcel Holtmann <marcel@holtmann.org>");
-MODULE_DESCRIPTION("Bluetooth driver for Nokia Connectivity Card DTL-1");
-MODULE_LICENSE("GPL");
-
-
-
-/* ======================== Local structures ======================== */
-
-
-struct dtl1_info {
-	struct pcmcia_device *p_dev;
-
-	struct hci_dev *hdev;
-
-	spinlock_t lock;		/* For serializing operations */
-
-	unsigned long flowmask;		/* HCI flow mask */
-	int ri_latch;
-
-	struct sk_buff_head txq;
-	unsigned long tx_state;
-
-	unsigned long rx_state;
-	unsigned long rx_count;
-	struct sk_buff *rx_skb;
-};
-
-
-static int dtl1_config(struct pcmcia_device *link);
-
-
-/* Transmit states  */
-#define XMIT_SENDING  1
-#define XMIT_WAKEUP   2
-#define XMIT_WAITING  8
-
-/* Receiver States */
-#define RECV_WAIT_NSH   0
-#define RECV_WAIT_DATA  1
-
-
-struct nsh {
-	u8 type;
-	u8 zero;
-	u16 len;
-} __packed;	/* Nokia Specific Header */
-
-#define NSHL  4				/* Nokia Specific Header Length */
-
-
-
-/* ======================== Interrupt handling ======================== */
-
-
-static int dtl1_write(unsigned int iobase, int fifo_size, __u8 *buf, int len)
-{
-	int actual = 0;
-
-	/* Tx FIFO should be empty */
-	if (!(inb(iobase + UART_LSR) & UART_LSR_THRE))
-		return 0;
-
-	/* Fill FIFO with current frame */
-	while ((fifo_size-- > 0) && (actual < len)) {
-		/* Transmit next byte */
-		outb(buf[actual], iobase + UART_TX);
-		actual++;
-	}
-
-	return actual;
-}
-
-
-static void dtl1_write_wakeup(struct dtl1_info *info)
-{
-	if (!info) {
-		BT_ERR("Unknown device");
-		return;
-	}
-
-	if (test_bit(XMIT_WAITING, &(info->tx_state))) {
-		set_bit(XMIT_WAKEUP, &(info->tx_state));
-		return;
-	}
-
-	if (test_and_set_bit(XMIT_SENDING, &(info->tx_state))) {
-		set_bit(XMIT_WAKEUP, &(info->tx_state));
-		return;
-	}
-
-	do {
-		unsigned int iobase = info->p_dev->resource[0]->start;
-		register struct sk_buff *skb;
-		int len;
-
-		clear_bit(XMIT_WAKEUP, &(info->tx_state));
-
-		if (!pcmcia_dev_present(info->p_dev))
-			return;
-
-		skb = skb_dequeue(&(info->txq));
-		if (!skb)
-			break;
-
-		/* Send frame */
-		len = dtl1_write(iobase, 32, skb->data, skb->len);
-
-		if (len == skb->len) {
-			set_bit(XMIT_WAITING, &(info->tx_state));
-			kfree_skb(skb);
-		} else {
-			skb_pull(skb, len);
-			skb_queue_head(&(info->txq), skb);
-		}
-
-		info->hdev->stat.byte_tx += len;
-
-	} while (test_bit(XMIT_WAKEUP, &(info->tx_state)));
-
-	clear_bit(XMIT_SENDING, &(info->tx_state));
-}
-
-
-static void dtl1_control(struct dtl1_info *info, struct sk_buff *skb)
-{
-	u8 flowmask = *(u8 *)skb->data;
-	int i;
-
-	printk(KERN_INFO "Bluetooth: Nokia control data =");
-	for (i = 0; i < skb->len; i++)
-		printk(" %02x", skb->data[i]);
-
-	printk("\n");
-
-	/* transition to active state */
-	if (((info->flowmask & 0x07) == 0) && ((flowmask & 0x07) != 0)) {
-		clear_bit(XMIT_WAITING, &(info->tx_state));
-		dtl1_write_wakeup(info);
-	}
-
-	info->flowmask = flowmask;
-
-	kfree_skb(skb);
-}
-
-
-static void dtl1_receive(struct dtl1_info *info)
-{
-	unsigned int iobase;
-	struct nsh *nsh;
-	int boguscount = 0;
-
-	if (!info) {
-		BT_ERR("Unknown device");
-		return;
-	}
-
-	iobase = info->p_dev->resource[0]->start;
-
-	do {
-		info->hdev->stat.byte_rx++;
-
-		/* Allocate packet */
-		if (info->rx_skb == NULL) {
-			info->rx_skb = bt_skb_alloc(HCI_MAX_FRAME_SIZE, GFP_ATOMIC);
-			if (!info->rx_skb) {
-				BT_ERR("Can't allocate mem for new packet");
-				info->rx_state = RECV_WAIT_NSH;
-				info->rx_count = NSHL;
-				return;
-			}
-		}
-
-		skb_put_u8(info->rx_skb, inb(iobase + UART_RX));
-		nsh = (struct nsh *)info->rx_skb->data;
-
-		info->rx_count--;
-
-		if (info->rx_count == 0) {
-
-			switch (info->rx_state) {
-			case RECV_WAIT_NSH:
-				info->rx_state = RECV_WAIT_DATA;
-				info->rx_count = nsh->len + (nsh->len & 0x0001);
-				break;
-			case RECV_WAIT_DATA:
-				hci_skb_pkt_type(info->rx_skb) = nsh->type;
-
-				/* remove PAD byte if it exists */
-				if (nsh->len & 0x0001) {
-					info->rx_skb->tail--;
-					info->rx_skb->len--;
-				}
-
-				/* remove NSH */
-				skb_pull(info->rx_skb, NSHL);
-
-				switch (hci_skb_pkt_type(info->rx_skb)) {
-				case 0x80:
-					/* control data for the Nokia Card */
-					dtl1_control(info, info->rx_skb);
-					break;
-				case 0x82:
-				case 0x83:
-				case 0x84:
-					/* send frame to the HCI layer */
-					hci_skb_pkt_type(info->rx_skb) &= 0x0f;
-					hci_recv_frame(info->hdev, info->rx_skb);
-					break;
-				default:
-					/* unknown packet */
-					BT_ERR("Unknown HCI packet with type 0x%02x received",
-					       hci_skb_pkt_type(info->rx_skb));
-					kfree_skb(info->rx_skb);
-					break;
-				}
-
-				info->rx_state = RECV_WAIT_NSH;
-				info->rx_count = NSHL;
-				info->rx_skb = NULL;
-				break;
-			}
-
-		}
-
-		/* Make sure we don't stay here too long */
-		if (boguscount++ > 32)
-			break;
-
-	} while (inb(iobase + UART_LSR) & UART_LSR_DR);
-}
-
-
-static irqreturn_t dtl1_interrupt(int irq, void *dev_inst)
-{
-	struct dtl1_info *info = dev_inst;
-	unsigned int iobase;
-	unsigned char msr;
-	int boguscount = 0;
-	int iir, lsr;
-	irqreturn_t r = IRQ_NONE;
-
-	if (!info || !info->hdev)
-		/* our irq handler is shared */
-		return IRQ_NONE;
-
-	iobase = info->p_dev->resource[0]->start;
-
-	spin_lock(&(info->lock));
-
-	iir = inb(iobase + UART_IIR) & UART_IIR_ID;
-	while (iir) {
-
-		r = IRQ_HANDLED;
-		/* Clear interrupt */
-		lsr = inb(iobase + UART_LSR);
-
-		switch (iir) {
-		case UART_IIR_RLSI:
-			BT_ERR("RLSI");
-			break;
-		case UART_IIR_RDI:
-			/* Receive interrupt */
-			dtl1_receive(info);
-			break;
-		case UART_IIR_THRI:
-			if (lsr & UART_LSR_THRE) {
-				/* Transmitter ready for data */
-				dtl1_write_wakeup(info);
-			}
-			break;
-		default:
-			BT_ERR("Unhandled IIR=%#x", iir);
-			break;
-		}
-
-		/* Make sure we don't stay here too long */
-		if (boguscount++ > 100)
-			break;
-
-		iir = inb(iobase + UART_IIR) & UART_IIR_ID;
-
-	}
-
-	msr = inb(iobase + UART_MSR);
-
-	if (info->ri_latch ^ (msr & UART_MSR_RI)) {
-		info->ri_latch = msr & UART_MSR_RI;
-		clear_bit(XMIT_WAITING, &(info->tx_state));
-		dtl1_write_wakeup(info);
-		r = IRQ_HANDLED;
-	}
-
-	spin_unlock(&(info->lock));
-
-	return r;
-}
-
-
-
-/* ======================== HCI interface ======================== */
-
-
-static int dtl1_hci_open(struct hci_dev *hdev)
-{
-	return 0;
-}
-
-
-static int dtl1_hci_flush(struct hci_dev *hdev)
-{
-	struct dtl1_info *info = hci_get_drvdata(hdev);
-
-	/* Drop TX queue */
-	skb_queue_purge(&(info->txq));
-
-	return 0;
-}
-
-
-static int dtl1_hci_close(struct hci_dev *hdev)
-{
-	dtl1_hci_flush(hdev);
-
-	return 0;
-}
-
-
-static int dtl1_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
-{
-	struct dtl1_info *info = hci_get_drvdata(hdev);
-	struct sk_buff *s;
-	struct nsh nsh;
-
-	switch (hci_skb_pkt_type(skb)) {
-	case HCI_COMMAND_PKT:
-		hdev->stat.cmd_tx++;
-		nsh.type = 0x81;
-		break;
-	case HCI_ACLDATA_PKT:
-		hdev->stat.acl_tx++;
-		nsh.type = 0x82;
-		break;
-	case HCI_SCODATA_PKT:
-		hdev->stat.sco_tx++;
-		nsh.type = 0x83;
-		break;
-	default:
-		return -EILSEQ;
-	}
-
-	nsh.zero = 0;
-	nsh.len = skb->len;
-
-	s = bt_skb_alloc(NSHL + skb->len + 1, GFP_ATOMIC);
-	if (!s)
-		return -ENOMEM;
-
-	skb_reserve(s, NSHL);
-	skb_copy_from_linear_data(skb, skb_put(s, skb->len), skb->len);
-	if (skb->len & 0x0001)
-		skb_put_u8(s, 0);	/* PAD */
-
-	/* Prepend skb with Nokia frame header and queue */
-	memcpy(skb_push(s, NSHL), &nsh, NSHL);
-	skb_queue_tail(&(info->txq), s);
-
-	dtl1_write_wakeup(info);
-
-	kfree_skb(skb);
-
-	return 0;
-}
-
-
-
-/* ======================== Card services HCI interaction ======================== */
-
-
-static int dtl1_open(struct dtl1_info *info)
-{
-	unsigned long flags;
-	unsigned int iobase = info->p_dev->resource[0]->start;
-	struct hci_dev *hdev;
-
-	spin_lock_init(&(info->lock));
-
-	skb_queue_head_init(&(info->txq));
-
-	info->rx_state = RECV_WAIT_NSH;
-	info->rx_count = NSHL;
-	info->rx_skb = NULL;
-
-	set_bit(XMIT_WAITING, &(info->tx_state));
-
-	/* Initialize HCI device */
-	hdev = hci_alloc_dev();
-	if (!hdev) {
-		BT_ERR("Can't allocate HCI device");
-		return -ENOMEM;
-	}
-
-	info->hdev = hdev;
-
-	hdev->bus = HCI_PCCARD;
-	hci_set_drvdata(hdev, info);
-	SET_HCIDEV_DEV(hdev, &info->p_dev->dev);
-
-	hdev->open  = dtl1_hci_open;
-	hdev->close = dtl1_hci_close;
-	hdev->flush = dtl1_hci_flush;
-	hdev->send  = dtl1_hci_send_frame;
-
-	spin_lock_irqsave(&(info->lock), flags);
-
-	/* Reset UART */
-	outb(0, iobase + UART_MCR);
-
-	/* Turn off interrupts */
-	outb(0, iobase + UART_IER);
-
-	/* Initialize UART */
-	outb(UART_LCR_WLEN8, iobase + UART_LCR);	/* Reset DLAB */
-	outb((UART_MCR_DTR | UART_MCR_RTS | UART_MCR_OUT2), iobase + UART_MCR);
-
-	info->ri_latch = inb(info->p_dev->resource[0]->start + UART_MSR)
-				& UART_MSR_RI;
-
-	/* Turn on interrupts */
-	outb(UART_IER_RLSI | UART_IER_RDI | UART_IER_THRI, iobase + UART_IER);
-
-	spin_unlock_irqrestore(&(info->lock), flags);
-
-	/* Timeout before it is safe to send the first HCI packet */
-	msleep(2000);
-
-	/* Register HCI device */
-	if (hci_register_dev(hdev) < 0) {
-		BT_ERR("Can't register HCI device");
-		info->hdev = NULL;
-		hci_free_dev(hdev);
-		return -ENODEV;
-	}
-
-	return 0;
-}
-
-
-static int dtl1_close(struct dtl1_info *info)
-{
-	unsigned long flags;
-	unsigned int iobase = info->p_dev->resource[0]->start;
-	struct hci_dev *hdev = info->hdev;
-
-	if (!hdev)
-		return -ENODEV;
-
-	dtl1_hci_close(hdev);
-
-	spin_lock_irqsave(&(info->lock), flags);
-
-	/* Reset UART */
-	outb(0, iobase + UART_MCR);
-
-	/* Turn off interrupts */
-	outb(0, iobase + UART_IER);
-
-	spin_unlock_irqrestore(&(info->lock), flags);
-
-	hci_unregister_dev(hdev);
-	hci_free_dev(hdev);
-
-	return 0;
-}
-
-static int dtl1_probe(struct pcmcia_device *link)
-{
-	struct dtl1_info *info;
-
-	/* Create new info device */
-	info = devm_kzalloc(&link->dev, sizeof(*info), GFP_KERNEL);
-	if (!info)
-		return -ENOMEM;
-
-	info->p_dev = link;
-	link->priv = info;
-
-	link->config_flags |= CONF_ENABLE_IRQ | CONF_AUTO_SET_IO;
-
-	return dtl1_config(link);
-}
-
-
-static void dtl1_detach(struct pcmcia_device *link)
-{
-	struct dtl1_info *info = link->priv;
-
-	dtl1_close(info);
-	pcmcia_disable_device(link);
-}
-
-static int dtl1_confcheck(struct pcmcia_device *p_dev, void *priv_data)
-{
-	if ((p_dev->resource[1]->end) || (p_dev->resource[1]->end < 8))
-		return -ENODEV;
-
-	p_dev->resource[0]->flags &= ~IO_DATA_PATH_WIDTH;
-	p_dev->resource[0]->flags |= IO_DATA_PATH_WIDTH_8;
-
-	return pcmcia_request_io(p_dev);
-}
-
-static int dtl1_config(struct pcmcia_device *link)
-{
-	struct dtl1_info *info = link->priv;
-	int ret;
-
-	/* Look for a generic full-sized window */
-	link->resource[0]->end = 8;
-	ret = pcmcia_loop_config(link, dtl1_confcheck, NULL);
-	if (ret)
-		goto failed;
-
-	ret = pcmcia_request_irq(link, dtl1_interrupt);
-	if (ret)
-		goto failed;
-
-	ret = pcmcia_enable_device(link);
-	if (ret)
-		goto failed;
-
-	ret = dtl1_open(info);
-	if (ret)
-		goto failed;
-
-	return 0;
-
-failed:
-	dtl1_detach(link);
-	return ret;
-}
-
-static const struct pcmcia_device_id dtl1_ids[] = {
-	PCMCIA_DEVICE_PROD_ID12("Nokia Mobile Phones", "DTL-1", 0xe1bfdd64, 0xe168480d),
-	PCMCIA_DEVICE_PROD_ID12("Nokia Mobile Phones", "DTL-4", 0xe1bfdd64, 0x9102bc82),
-	PCMCIA_DEVICE_PROD_ID12("Socket", "CF", 0xb38bcc2e, 0x44ebf863),
-	PCMCIA_DEVICE_PROD_ID12("Socket", "CF+ Personal Network Card", 0xb38bcc2e, 0xe732bae3),
-	PCMCIA_DEVICE_NULL
-};
-MODULE_DEVICE_TABLE(pcmcia, dtl1_ids);
-
-static struct pcmcia_driver dtl1_driver = {
-	.owner		= THIS_MODULE,
-	.name		= "dtl1_cs",
-	.probe		= dtl1_probe,
-	.remove		= dtl1_detach,
-	.id_table	= dtl1_ids,
-};
-module_pcmcia_driver(dtl1_driver);
diff --git a/drivers/bluetooth/hci_ldisc.c b/drivers/bluetooth/hci_ldisc.c
index 275ea865bc297..47f4902b40b47 100644
--- a/drivers/bluetooth/hci_ldisc.c
+++ b/drivers/bluetooth/hci_ldisc.c
@@ -194,7 +194,15 @@ void hci_uart_init_work(struct work_struct *work)
 	err = hci_register_dev(hu->hdev);
 	if (err < 0) {
 		BT_ERR("Can't register HCI device");
+
+		percpu_down_write(&hu->proto_lock);
 		clear_bit(HCI_UART_PROTO_READY, &hu->flags);
+		percpu_up_write(&hu->proto_lock);
+
+		/* Safely cancel work after clearing flags */
+		cancel_work_sync(&hu->write_work);
+
+		/* Close protocol before freeing hdev */
 		hu->proto->close(hu);
 		hdev = hu->hdev;
 		hu->hdev = NULL;
@@ -263,8 +271,12 @@ static int hci_uart_open(struct hci_dev *hdev)
 /* Close device */
 static int hci_uart_close(struct hci_dev *hdev)
 {
+	struct hci_uart *hu = hci_get_drvdata(hdev);
+
 	BT_DBG("hdev %p", hdev);
 
+	cancel_work_sync(&hu->write_work);
+
 	hci_uart_flush(hdev);
 	hdev->flush = NULL;
 	return 0;
@@ -531,6 +543,7 @@ static void hci_uart_tty_close(struct tty_struct *tty)
 {
 	struct hci_uart *hu = tty->disc_data;
 	struct hci_dev *hdev;
+	bool proto_ready;
 
 	BT_DBG("tty %p", tty);
 
@@ -540,24 +553,38 @@ static void hci_uart_tty_close(struct tty_struct *tty)
 	if (!hu)
 		return;
 
-	hdev = hu->hdev;
-	if (hdev)
-		hci_uart_close(hdev);
+	/* Wait for init_ready to finish to prevent registration races */
+	cancel_work_sync(&hu->init_ready);
 
-	if (test_bit(HCI_UART_PROTO_READY, &hu->flags)) {
+	proto_ready = test_bit(HCI_UART_PROTO_READY, &hu->flags);
+	if (proto_ready) {
 		percpu_down_write(&hu->proto_lock);
 		clear_bit(HCI_UART_PROTO_READY, &hu->flags);
 		percpu_up_write(&hu->proto_lock);
+	}
 
-		cancel_work_sync(&hu->init_ready);
-		cancel_work_sync(&hu->write_work);
+	/*
+	 * Unconditionally cancel write_work AFTER clearing PROTO_READY.
+	 * This ensures that concurrent protocol timers cannot requeue
+	 * write_work via hci_uart_tx_wakeup(), permanently preventing
+	 * double-free races and UAFs.
+	 */
+	cancel_work_sync(&hu->write_work);
+
+	hdev = hu->hdev;
+	if (hdev)
+		hci_uart_close(hdev); /* proto->flush is safely skipped */
 
+	if (proto_ready) {
 		if (hdev) {
 			if (test_bit(HCI_UART_REGISTERED, &hu->flags))
 				hci_unregister_dev(hdev);
-			hci_free_dev(hdev);
 		}
+		/* Close protocol before freeing hdev (intrinsically purges queues) */
 		hu->proto->close(hu);
+
+		if (hdev)
+			hci_free_dev(hdev);
 	}
 	clear_bit(HCI_UART_PROTO_SET, &hu->flags);
 
@@ -625,11 +652,12 @@ static void hci_uart_tty_receive(struct tty_struct *tty, const u8 *data,
 	 * tty caller
 	 */
 	hu->proto->recv(hu, data, count);
-	percpu_up_read(&hu->proto_lock);
 
 	if (hu->hdev)
 		hu->hdev->stat.byte_rx += count;
 
+	percpu_up_read(&hu->proto_lock);
+
 	tty_unthrottle(tty);
 }
 
@@ -695,6 +723,10 @@ static int hci_uart_register_dev(struct hci_uart *hu)
 		percpu_down_write(&hu->proto_lock);
 		clear_bit(HCI_UART_PROTO_INIT, &hu->flags);
 		percpu_up_write(&hu->proto_lock);
+		/* Cancel work after clearing flags */
+		cancel_work_sync(&hu->write_work);
+
+		/* Close protocol before freeing hdev */
 		hu->proto->close(hu);
 		hu->hdev = NULL;
 		hci_free_dev(hdev);
diff --git a/drivers/bluetooth/hci_qca.c b/drivers/bluetooth/hci_qca.c
index cd1834246b479..ed280399bf474 100644
--- a/drivers/bluetooth/hci_qca.c
+++ b/drivers/bluetooth/hci_qca.c
@@ -48,13 +48,12 @@
 #define HCI_MAX_IBS_SIZE	10
 
 #define IBS_WAKE_RETRANS_TIMEOUT_MS	100
-#define IBS_BTSOC_TX_IDLE_TIMEOUT_MS	200
+#define IBS_BTSOC_TX_IDLE_TIMEOUT	msecs_to_jiffies(200)
 #define IBS_HOST_TX_IDLE_TIMEOUT_MS	2000
-#define CMD_TRANS_TIMEOUT_MS		100
-#define MEMDUMP_TIMEOUT_MS		8000
-#define IBS_DISABLE_SSR_TIMEOUT_MS \
-	(MEMDUMP_TIMEOUT_MS + FW_DOWNLOAD_TIMEOUT_MS)
-#define FW_DOWNLOAD_TIMEOUT_MS		3000
+#define CMD_TRANS_TIMEOUT		msecs_to_jiffies(100)
+#define MEMDUMP_TIMEOUT			msecs_to_jiffies(8000)
+#define FW_DOWNLOAD_TIMEOUT		msecs_to_jiffies(3000)
+#define IBS_DISABLE_SSR_TIMEOUT		(MEMDUMP_TIMEOUT + FW_DOWNLOAD_TIMEOUT)
 
 /* susclk rate */
 #define SUSCLK_RATE_32KHZ	32768
@@ -1096,7 +1095,7 @@ static void qca_controller_memdump(struct work_struct *work)
 
 			queue_delayed_work(qca->workqueue,
 					   &qca->ctrl_memdump_timeout,
-					   msecs_to_jiffies(MEMDUMP_TIMEOUT_MS));
+					   MEMDUMP_TIMEOUT);
 			skb_pull(skb, sizeof(qca_memdump->ram_dump_size));
 			qca_memdump->current_seq_no = 0;
 			qca_memdump->received_dump = 0;
@@ -1369,7 +1368,7 @@ static int qca_set_baudrate(struct hci_dev *hdev, uint8_t baudrate)
 
 	if (hu->serdev)
 		serdev_device_wait_until_sent(hu->serdev,
-		      msecs_to_jiffies(CMD_TRANS_TIMEOUT_MS));
+		      CMD_TRANS_TIMEOUT);
 
 	/* Give the controller time to process the request */
 	switch (qca_soc_type(hu)) {
@@ -1401,8 +1400,8 @@ static inline void host_set_baudrate(struct hci_uart *hu, unsigned int speed)
 
 static int qca_send_power_pulse(struct hci_uart *hu, bool on)
 {
+	int timeout = CMD_TRANS_TIMEOUT;
 	int ret;
-	int timeout = msecs_to_jiffies(CMD_TRANS_TIMEOUT_MS);
 	u8 cmd = on ? QCA_WCN3990_POWERON_PULSE : QCA_WCN3990_POWEROFF_PULSE;
 
 	/* These power pulses are single byte command which are sent
@@ -1607,7 +1606,7 @@ static void qca_wait_for_dump_collection(struct hci_dev *hdev)
 	struct qca_data *qca = hu->priv;
 
 	wait_on_bit_timeout(&qca->flags, QCA_MEMDUMP_COLLECTION,
-			    TASK_UNINTERRUPTIBLE, MEMDUMP_TIMEOUT_MS);
+			    TASK_UNINTERRUPTIBLE, MEMDUMP_TIMEOUT);
 
 	clear_bit(QCA_MEMDUMP_COLLECTION, &qca->flags);
 }
@@ -2591,7 +2590,7 @@ static void qca_serdev_remove(struct serdev_device *serdev)
 static void qca_serdev_shutdown(struct serdev_device *serdev)
 {
 	int ret;
-	int timeout = msecs_to_jiffies(CMD_TRANS_TIMEOUT_MS);
+	int timeout = CMD_TRANS_TIMEOUT;
 	struct qca_serdev *qcadev = serdev_device_get_drvdata(serdev);
 	struct hci_uart *hu = &qcadev->serdev_hu;
 	struct hci_dev *hdev = hu->hdev;
@@ -2648,7 +2647,7 @@ static int __maybe_unused qca_suspend(struct device *dev)
 	bool tx_pending = false;
 	int ret = 0;
 	u8 cmd;
-	u32 wait_timeout = 0;
+	unsigned long wait_timeout = 0;
 
 	set_bit(QCA_SUSPENDING, &qca->flags);
 
@@ -2669,15 +2668,15 @@ static int __maybe_unused qca_suspend(struct device *dev)
 	if (test_bit(QCA_IBS_DISABLED, &qca->flags) ||
 	    test_bit(QCA_SSR_TRIGGERED, &qca->flags)) {
 		wait_timeout = test_bit(QCA_SSR_TRIGGERED, &qca->flags) ?
-					IBS_DISABLE_SSR_TIMEOUT_MS :
-					FW_DOWNLOAD_TIMEOUT_MS;
+					IBS_DISABLE_SSR_TIMEOUT :
+					FW_DOWNLOAD_TIMEOUT;
 
 		/* QCA_IBS_DISABLED flag is set to true, During FW download
 		 * and during memory dump collection. It is reset to false,
 		 * After FW download complete.
 		 */
 		wait_on_bit_timeout(&qca->flags, QCA_IBS_DISABLED,
-			    TASK_UNINTERRUPTIBLE, msecs_to_jiffies(wait_timeout));
+			    TASK_UNINTERRUPTIBLE, wait_timeout);
 
 		if (test_bit(QCA_IBS_DISABLED, &qca->flags)) {
 			bt_dev_err(hu->hdev, "SSR or FW download time out");
@@ -2729,7 +2728,7 @@ static int __maybe_unused qca_suspend(struct device *dev)
 
 	if (tx_pending) {
 		serdev_device_wait_until_sent(hu->serdev,
-					      msecs_to_jiffies(CMD_TRANS_TIMEOUT_MS));
+					      CMD_TRANS_TIMEOUT);
 		serial_clock_vote(HCI_IBS_TX_VOTE_CLOCK_OFF, hu);
 	}
 
@@ -2738,7 +2737,7 @@ static int __maybe_unused qca_suspend(struct device *dev)
 	 */
 	ret = wait_event_interruptible_timeout(qca->suspend_wait_q,
 			qca->rx_ibs_state == HCI_IBS_RX_ASLEEP,
-			msecs_to_jiffies(IBS_BTSOC_TX_IDLE_TIMEOUT_MS));
+			IBS_BTSOC_TX_IDLE_TIMEOUT);
 	if (ret == 0) {
 		ret = -ETIMEDOUT;
 		goto error;
diff --git a/drivers/bus/fsl-mc/dprc-driver.c b/drivers/bus/fsl-mc/dprc-driver.c
index db67442addad2..a85706826fa06 100644
--- a/drivers/bus/fsl-mc/dprc-driver.c
+++ b/drivers/bus/fsl-mc/dprc-driver.c
@@ -609,9 +609,8 @@ int dprc_setup(struct fsl_mc_device *mc_dev)
 {
 	struct device *parent_dev = mc_dev->dev.parent;
 	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_dev);
-	struct irq_domain *mc_msi_domain;
+	struct irq_domain *mc_msi_domain = NULL;
 	bool mc_io_created = false;
-	bool msi_domain_set = false;
 	bool uapi_created = false;
 	u16 major_ver, minor_ver;
 	size_t region_size;
@@ -652,14 +651,12 @@ int dprc_setup(struct fsl_mc_device *mc_dev)
 		uapi_created = true;
 	}
 
-	mc_msi_domain = fsl_mc_find_msi_domain(&mc_dev->dev);
-	if (!mc_msi_domain) {
+	mc_msi_domain = fsl_mc_get_msi_parent(&mc_dev->dev);
+	if (!mc_msi_domain)
 		dev_warn(&mc_dev->dev,
 			 "WARNING: MC bus without interrupt support\n");
-	} else {
+	else
 		dev_set_msi_domain(&mc_dev->dev, mc_msi_domain);
-		msi_domain_set = true;
-	}
 
 	error = dprc_open(mc_dev->mc_io, 0, mc_dev->obj_desc.id,
 			  &mc_dev->mc_handle);
@@ -699,8 +696,7 @@ int dprc_setup(struct fsl_mc_device *mc_dev)
 	(void)dprc_close(mc_dev->mc_io, 0, mc_dev->mc_handle);
 
 error_cleanup_msi_domain:
-	if (msi_domain_set)
-		dev_set_msi_domain(&mc_dev->dev, NULL);
+	dev_set_msi_domain(&mc_dev->dev, NULL);
 
 	if (mc_io_created) {
 		fsl_destroy_mc_io(mc_dev->mc_io);
diff --git a/drivers/bus/fsl-mc/fsl-mc-bus.c b/drivers/bus/fsl-mc/fsl-mc-bus.c
index 221146e4860be..64d75eed0d347 100644
--- a/drivers/bus/fsl-mc/fsl-mc-bus.c
+++ b/drivers/bus/fsl-mc/fsl-mc-bus.c
@@ -66,6 +66,13 @@ struct fsl_mc_addr_translation_range {
 #define GCR1_P1_STOP	BIT(31)
 #define GCR1_P2_STOP	BIT(30)
 
+#define FSL_MC_GSR		0x8
+#define FSL_MC_GSR_BOOT_DONE	BIT(0)
+#define FSL_MC_GSR_MCS_MASK	GENMASK(7, 0)
+#define FSL_MC_GSR_MCS_ERR_MASK	GENMASK(7, 1)
+#define FSL_MC_GSR_BC_MASK	GENMASK(15, 8)
+#define FSL_MC_GSR_BC_SHIFT	8
+
 #define FSL_MC_FAPR	0x28
 #define MC_FAPR_PL	BIT(18)
 #define MC_FAPR_BMT	BIT(17)
@@ -828,14 +835,12 @@ int fsl_mc_device_add(struct fsl_mc_obj_desc *obj_desc,
 	} else {
 		/*
 		 * A non-DPRC object has to be a child of a DPRC, use the
-		 * parent's ICID and interrupt domain.
+		 * parent's ICID.
 		 */
 		mc_dev->icid = parent_mc_dev->icid;
 		mc_dev->dma_mask = FSL_MC_DEFAULT_DMA_MASK;
 		mc_dev->dev.dma_mask = &mc_dev->dma_mask;
 		mc_dev->dev.coherent_dma_mask = mc_dev->dma_mask;
-		dev_set_msi_domain(&mc_dev->dev,
-				   dev_get_msi_domain(&parent_mc_dev->dev));
 	}
 
 	/*
@@ -992,6 +997,41 @@ static int get_mc_addr_translation_ranges(struct device *dev,
 	return 0;
 }
 
+static u32 fsl_mc_read_gsr(struct fsl_mc *mc)
+{
+	return readl(mc->fsl_mc_regs + FSL_MC_GSR);
+}
+
+static int fsl_mc_firmware_check(struct platform_device *pdev)
+{
+	struct fsl_mc *mc = platform_get_drvdata(pdev);
+	u32 gsr, boot_done, boot_code, mcs;
+
+	gsr = fsl_mc_read_gsr(mc);
+	boot_code = (gsr & FSL_MC_GSR_BC_MASK) >> FSL_MC_GSR_BC_SHIFT;
+	if (boot_code == 0xDD) {
+		dev_err(&pdev->dev,
+			"fsl-mc: DPL processing was not started, DPAA2 will not work!\n");
+		return -EOPNOTSUPP;
+	}
+
+	boot_done = gsr & FSL_MC_GSR_BOOT_DONE;
+	if (!boot_done) {
+		dev_dbg(&pdev->dev,
+			"fsl-mc: DPL processing in progress, defer probe\n");
+		return -EPROBE_DEFER;
+	}
+
+	mcs = gsr & FSL_MC_GSR_MCS_MASK;
+	if (mcs & FSL_MC_GSR_MCS_ERR_MASK) {
+		dev_err(&pdev->dev,
+			"fsl-mc: MC boot completed with error 0x%x\n", mcs);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 /*
  * fsl_mc_bus_probe - callback invoked when the root MC bus is being
  * added
@@ -1056,6 +1096,10 @@ static int fsl_mc_bus_probe(struct platform_device *pdev)
 		       mc->fsl_mc_regs + FSL_MC_GCR1);
 	}
 
+	error = fsl_mc_firmware_check(pdev);
+	if (error)
+		return error;
+
 	/*
 	 * Get physical address of MC portal for the root DPRC:
 	 */
diff --git a/drivers/bus/fsl-mc/fsl-mc-msi.c b/drivers/bus/fsl-mc/fsl-mc-msi.c
index 82cd69f7884c6..be38b43803dea 100644
--- a/drivers/bus/fsl-mc/fsl-mc-msi.c
+++ b/drivers/bus/fsl-mc/fsl-mc-msi.c
@@ -15,53 +15,16 @@
 
 #include "fsl-mc-private.h"
 
-#ifdef GENERIC_MSI_DOMAIN_OPS
-/*
- * Generate a unique ID identifying the interrupt (only used within the MSI
- * irqdomain.  Combine the icid with the interrupt index.
- */
-static irq_hw_number_t fsl_mc_domain_calc_hwirq(struct fsl_mc_device *dev,
-						struct msi_desc *desc)
-{
-	/*
-	 * Make the base hwirq value for ICID*10000 so it is readable
-	 * as a decimal value in /proc/interrupts.
-	 */
-	return (irq_hw_number_t)(desc->msi_index + (dev->icid * 10000));
-}
-
-static void fsl_mc_msi_set_desc(msi_alloc_info_t *arg,
-				struct msi_desc *desc)
-{
-	arg->desc = desc;
-	arg->hwirq = fsl_mc_domain_calc_hwirq(to_fsl_mc_device(desc->dev),
-					      desc);
-}
-#else
-#define fsl_mc_msi_set_desc NULL
-#endif
-
-static void fsl_mc_msi_update_dom_ops(struct msi_domain_info *info)
+static void fsl_mc_write_msi_msg(struct msi_desc *msi_desc, struct msi_msg *msg)
 {
-	struct msi_domain_ops *ops = info->ops;
-
-	if (!ops)
-		return;
-
-	/*
-	 * set_desc should not be set by the caller
-	 */
-	if (!ops->set_desc)
-		ops->set_desc = fsl_mc_msi_set_desc;
-}
-
-static void __fsl_mc_msi_write_msg(struct fsl_mc_device *mc_bus_dev,
-				   struct fsl_mc_device_irq *mc_dev_irq,
-				   struct msi_desc *msi_desc)
-{
-	int error;
+	struct fsl_mc_device *mc_bus_dev = to_fsl_mc_device(msi_desc->dev);
+	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
+	struct fsl_mc_device_irq *mc_dev_irq = &mc_bus->irq_resources[msi_desc->msi_index];
 	struct fsl_mc_device *owner_mc_dev = mc_dev_irq->mc_dev;
 	struct dprc_irq_cfg irq_cfg;
+	int error;
+
+	msi_desc->msg = *msg;
 
 	/*
 	 * msi_desc->msg.address is 0x0 when this function is invoked in
@@ -110,118 +73,27 @@ static void __fsl_mc_msi_write_msg(struct fsl_mc_device *mc_bus_dev,
 	}
 }
 
-/*
- * NOTE: This function is invoked with interrupts disabled
- */
-static void fsl_mc_msi_write_msg(struct irq_data *irq_data,
-				 struct msi_msg *msg)
-{
-	struct msi_desc *msi_desc = irq_data_get_msi_desc(irq_data);
-	struct fsl_mc_device *mc_bus_dev = to_fsl_mc_device(msi_desc->dev);
-	struct fsl_mc_bus *mc_bus = to_fsl_mc_bus(mc_bus_dev);
-	struct fsl_mc_device_irq *mc_dev_irq =
-		&mc_bus->irq_resources[msi_desc->msi_index];
-
-	msi_desc->msg = *msg;
-
-	/*
-	 * Program the MSI (paddr, value) pair in the device:
-	 */
-	__fsl_mc_msi_write_msg(mc_bus_dev, mc_dev_irq, msi_desc);
-}
-
-static void fsl_mc_msi_update_chip_ops(struct msi_domain_info *info)
-{
-	struct irq_chip *chip = info->chip;
-
-	if (!chip)
-		return;
-
-	/*
-	 * irq_write_msi_msg should not be set by the caller
-	 */
-	if (!chip->irq_write_msi_msg)
-		chip->irq_write_msi_msg = fsl_mc_msi_write_msg;
-}
-
-/**
- * fsl_mc_msi_create_irq_domain - Create a fsl-mc MSI interrupt domain
- * @fwnode:	Optional firmware node of the interrupt controller
- * @info:	MSI domain info
- * @parent:	Parent irq domain
- *
- * Updates the domain and chip ops and creates a fsl-mc MSI
- * interrupt domain.
- *
- * Returns:
- * A domain pointer or NULL in case of failure.
- */
-struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode,
-						struct msi_domain_info *info,
-						struct irq_domain *parent)
-{
-	struct irq_domain *domain;
-
-	if (WARN_ON((info->flags & MSI_FLAG_LEVEL_CAPABLE)))
-		info->flags &= ~MSI_FLAG_LEVEL_CAPABLE;
-	if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
-		fsl_mc_msi_update_dom_ops(info);
-	if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
-		fsl_mc_msi_update_chip_ops(info);
-	info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | MSI_FLAG_FREE_MSI_DESCS;
-
-	domain = msi_create_irq_domain(fwnode, info, parent);
-	if (domain)
-		irq_domain_update_bus_token(domain, DOMAIN_BUS_FSL_MC_MSI);
-
-	return domain;
-}
-
-struct irq_domain *fsl_mc_find_msi_domain(struct device *dev)
+struct irq_domain *fsl_mc_get_msi_parent(struct device *dev)
 {
+	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
 	struct device *root_dprc_dev;
 	struct device *bus_dev;
-	struct irq_domain *msi_domain;
-	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
 
 	fsl_mc_get_root_dprc(dev, &root_dprc_dev);
 	bus_dev = root_dprc_dev->parent;
 
-	if (bus_dev->of_node) {
-		msi_domain = of_msi_map_get_device_domain(dev,
-						  mc_dev->icid,
-						  DOMAIN_BUS_FSL_MC_MSI);
-
-		/*
-		 * if the msi-map property is missing assume that all the
-		 * child containers inherit the domain from the parent
-		 */
-		if (!msi_domain)
-
-			msi_domain = of_msi_get_domain(bus_dev,
-						bus_dev->of_node,
-						DOMAIN_BUS_FSL_MC_MSI);
-	} else {
-		msi_domain = iort_get_device_domain(dev, mc_dev->icid,
-						    DOMAIN_BUS_FSL_MC_MSI);
-	}
-
-	return msi_domain;
+	return (bus_dev->of_node ?
+		of_msi_get_domain(bus_dev, bus_dev->of_node, DOMAIN_BUS_NEXUS) :
+		iort_get_device_domain(bus_dev, mc_dev->icid, DOMAIN_BUS_NEXUS));
 }
 
 int fsl_mc_msi_domain_alloc_irqs(struct device *dev,  unsigned int irq_count)
 {
 	int error = msi_setup_device_data(dev);
-
 	if (error)
 		return error;
 
-	/*
-	 * NOTE: Calling this function will trigger the invocation of the
-	 * its_fsl_mc_msi_prepare() callback
-	 */
-	error = msi_domain_alloc_irqs_range(dev, MSI_DEFAULT_DOMAIN, 0, irq_count - 1);
-
+	error = platform_device_msi_init_and_alloc_irqs(dev, irq_count, fsl_mc_write_msi_msg);
 	if (error)
 		dev_err(dev, "Failed to allocate IRQs\n");
 	return error;
@@ -231,3 +103,15 @@ void fsl_mc_msi_domain_free_irqs(struct device *dev)
 {
 	msi_domain_free_irqs_all(dev, MSI_DEFAULT_DOMAIN);
 }
+
+u32 fsl_mc_get_msi_id(struct device *dev)
+{
+	struct fsl_mc_device *mc_dev = to_fsl_mc_device(dev);
+	struct device *root_dprc_dev;
+
+	fsl_mc_get_root_dprc(dev, &root_dprc_dev);
+
+	return (root_dprc_dev->parent->of_node ?
+		of_msi_xlate(dev, NULL, mc_dev->icid) :
+		iort_msi_map_id(dev, mc_dev->icid));
+}
diff --git a/drivers/bus/fsl-mc/fsl-mc-private.h b/drivers/bus/fsl-mc/fsl-mc-private.h
index beed4c53533d8..197edcc8cde46 100644
--- a/drivers/bus/fsl-mc/fsl-mc-private.h
+++ b/drivers/bus/fsl-mc/fsl-mc-private.h
@@ -641,7 +641,7 @@ int fsl_mc_msi_domain_alloc_irqs(struct device *dev,
 
 void fsl_mc_msi_domain_free_irqs(struct device *dev);
 
-struct irq_domain *fsl_mc_find_msi_domain(struct device *dev);
+struct irq_domain *fsl_mc_get_msi_parent(struct device *dev);
 
 int __must_check fsl_create_mc_io(struct device *dev,
 				  phys_addr_t mc_portal_phys_addr,
diff --git a/drivers/bus/mhi/ep/main.c b/drivers/bus/mhi/ep/main.c
index 0277e1ab11988..9db2a2a2c9134 100644
--- a/drivers/bus/mhi/ep/main.c
+++ b/drivers/bus/mhi/ep/main.c
@@ -232,7 +232,9 @@ static int mhi_ep_process_cmd_ring(struct mhi_ep_ring *ring, struct mhi_ring_ele
 			ret = mhi_ep_create_device(mhi_cntrl, ch_id);
 			if (ret) {
 				dev_err(dev, "Error creating device for channel (%u)\n", ch_id);
+				mutex_lock(&mhi_cntrl->state_lock);
 				mhi_ep_handle_syserr(mhi_cntrl);
+				mutex_unlock(&mhi_cntrl->state_lock);
 				return ret;
 			}
 		}
@@ -1087,11 +1089,12 @@ static void mhi_ep_reset_worker(struct work_struct *work)
 
 	mhi_ep_power_down(mhi_cntrl);
 
-	mutex_lock(&mhi_cntrl->state_lock);
-
 	/* Reset MMIO to signal host that the MHI_RESET is completed in endpoint */
 	mhi_ep_mmio_reset(mhi_cntrl);
+
+	mutex_lock(&mhi_cntrl->state_lock);
 	cur_state = mhi_cntrl->mhi_state;
+	mutex_unlock(&mhi_cntrl->state_lock);
 
 	/*
 	 * Only proceed further if the reset is due to SYS_ERR. The host will
@@ -1100,8 +1103,6 @@ static void mhi_ep_reset_worker(struct work_struct *work)
 	 */
 	if (cur_state == MHI_STATE_SYS_ERR)
 		mhi_ep_power_up(mhi_cntrl);
-
-	mutex_unlock(&mhi_cntrl->state_lock);
 }
 
 /*
@@ -1148,7 +1149,9 @@ int mhi_ep_power_up(struct mhi_ep_cntrl *mhi_cntrl)
 	for (i = 0; i < mhi_cntrl->event_rings; i++)
 		mhi_ep_ring_init(&mhi_cntrl->mhi_event[i].ring, RING_TYPE_ER, i);
 
+	mutex_lock(&mhi_cntrl->state_lock);
 	mhi_cntrl->mhi_state = MHI_STATE_RESET;
+	mutex_unlock(&mhi_cntrl->state_lock);
 
 	/* Set AMSS EE before signaling ready state */
 	mhi_ep_mmio_set_env(mhi_cntrl, MHI_EE_AMSS);
diff --git a/drivers/bus/mhi/host/pci_generic.c b/drivers/bus/mhi/host/pci_generic.c
index 750da3dbb4c6e..5836ecb0ea320 100644
--- a/drivers/bus/mhi/host/pci_generic.c
+++ b/drivers/bus/mhi/host/pci_generic.c
@@ -914,6 +914,16 @@ static const struct mhi_pci_dev_info mhi_telit_fe912c04_info = {
 	.edl_trigger = true,
 };
 
+static const struct mhi_pci_dev_info mhi_telit_fe910c04_info = {
+	.name = "telit-fe910c04",
+	.config = &modem_telit_fn920c04_config,
+	.bar_num = MHI_PCI_DEFAULT_BAR_NUM,
+	.dma_data_width = 32,
+	.sideband_wake = false,
+	.mru_default = 32768,
+	.edl_trigger = true,
+};
+
 static const struct mhi_pci_dev_info mhi_netprisma_lcur57_info = {
 	.name = "netprisma-lcur57",
 	.edl = "qcom/prog_firehose_sdx24.mbn",
@@ -941,6 +951,9 @@ static const struct pci_device_id mhi_pci_id_table[] = {
 	/* Telit FN920C04 (sdx35) */
 	{PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x011a, 0x1c5d, 0x2020),
 		.driver_data = (kernel_ulong_t) &mhi_telit_fn920c04_info },
+	/* Telit FE910C04 (sdx35) */
+	{PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x011a, 0x1c5d, 0x202a),
+		.driver_data = (kernel_ulong_t) &mhi_telit_fe910c04_info },
 	/* Telit FE912C04 (sdx35) */
 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_QCOM, 0x011a, 0x1c5d, 0x2045),
 		.driver_data = (kernel_ulong_t) &mhi_telit_fe912c04_info },
@@ -1189,7 +1202,8 @@ static int mhi_pci_get_irqs(struct mhi_controller *mhi_cntrl,
 	 */
 	mhi_cntrl->nr_irqs = 1 + mhi_cntrl_config->num_events;
 
-	nr_vectors = pci_alloc_irq_vectors(pdev, 1, mhi_cntrl->nr_irqs, PCI_IRQ_MSIX | PCI_IRQ_MSI);
+	nr_vectors = pci_alloc_irq_vectors(pdev, 1, roundup_pow_of_two(mhi_cntrl->nr_irqs),
+					   PCI_IRQ_MSIX | PCI_IRQ_MSI);
 	if (nr_vectors < 0) {
 		dev_err(&pdev->dev, "Error allocating MSI vectors %d\n",
 			nr_vectors);
diff --git a/drivers/bus/tegra-aconnect.c b/drivers/bus/tegra-aconnect.c
index 90e3b0a108161..9b2b227332b22 100644
--- a/drivers/bus/tegra-aconnect.c
+++ b/drivers/bus/tegra-aconnect.c
@@ -32,16 +32,14 @@ static int tegra_aconnect_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	aconnect->ape_clk = devm_clk_get(&pdev->dev, "ape");
-	if (IS_ERR(aconnect->ape_clk)) {
-		dev_err(&pdev->dev, "Can't retrieve ape clock\n");
-		return PTR_ERR(aconnect->ape_clk);
-	}
+	if (IS_ERR(aconnect->ape_clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(aconnect->ape_clk),
+				     "can't retrieve ape clock\n");
 
 	aconnect->apb2ape_clk = devm_clk_get(&pdev->dev, "apb2ape");
-	if (IS_ERR(aconnect->apb2ape_clk)) {
-		dev_err(&pdev->dev, "Can't retrieve apb2ape clock\n");
-		return PTR_ERR(aconnect->apb2ape_clk);
-	}
+	if (IS_ERR(aconnect->apb2ape_clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(aconnect->apb2ape_clk),
+				     "can't retrieve apb2ape clock\n");
 
 	dev_set_drvdata(&pdev->dev, aconnect);
 	pm_runtime_enable(&pdev->dev);
diff --git a/drivers/char/agp/efficeon-agp.c b/drivers/char/agp/efficeon-agp.c
index 0d25bbdc7e6af..4d0b7d7c0aad4 100644
--- a/drivers/char/agp/efficeon-agp.c
+++ b/drivers/char/agp/efficeon-agp.c
@@ -27,6 +27,7 @@
 #include <linux/gfp.h>
 #include <linux/page-flags.h>
 #include <linux/mm.h>
+#include <asm/cpuid/api.h>
 #include "agp.h"
 #include "intel-agp.h"
 
diff --git a/drivers/char/hw_random/amd-rng.c b/drivers/char/hw_random/amd-rng.c
index 332d594bdf709..dff80daae5877 100644
--- a/drivers/char/hw_random/amd-rng.c
+++ b/drivers/char/hw_random/amd-rng.c
@@ -47,9 +47,9 @@
  * want to register another driver on the same PCI id.
  */
 static const struct pci_device_id pci_tbl[] = {
-	{ PCI_VDEVICE(AMD, 0x7443), 0, },
-	{ PCI_VDEVICE(AMD, 0x746b), 0, },
-	{ 0, },	/* terminate list */
+	{ PCI_VDEVICE(AMD, 0x7443) },
+	{ PCI_VDEVICE(AMD, 0x746b) },
+	{ }	/* terminate list */
 };
 MODULE_DEVICE_TABLE(pci, pci_tbl);
 
diff --git a/drivers/char/hw_random/cavium-rng.c b/drivers/char/hw_random/cavium-rng.c
index d9d7b6038c065..3e2c042009f66 100644
--- a/drivers/char/hw_random/cavium-rng.c
+++ b/drivers/char/hw_random/cavium-rng.c
@@ -73,8 +73,8 @@ static void cavium_rng_remove(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id cavium_rng_pf_id_table[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa018), 0, 0, 0}, /* Thunder RNM */
-	{0,},
+	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa018) }, /* Thunder RNM */
+	{ },
 };
 
 MODULE_DEVICE_TABLE(pci, cavium_rng_pf_id_table);
diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
index aba92d777f726..6931657ad2caa 100644
--- a/drivers/char/hw_random/core.c
+++ b/drivers/char/hw_random/core.c
@@ -25,12 +25,13 @@
 #include <linux/sched/signal.h>
 #include <linux/slab.h>
 #include <linux/string.h>
+#include <linux/sysfs.h>
 #include <linux/uaccess.h>
 #include <linux/workqueue.h>
 
 #define RNG_MODULE_NAME		"hw_random"
 
-#define RNG_BUFFER_SIZE (SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES)
+#define RNG_BUFFER_SIZE		MAX(32, SMP_CACHE_BYTES)
 
 static struct hwrng __rcu *current_rng;
 /* the current rng has been explicitly chosen by user via sysfs */
@@ -54,13 +55,9 @@ module_param(default_quality, ushort, 0644);
 MODULE_PARM_DESC(default_quality,
 		 "default maximum entropy content of hwrng per 1024 bits of input");
 
-static void drop_current_rng(void);
 static int hwrng_init(struct hwrng *rng);
 static int hwrng_fillfn(void *unused);
 
-static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size,
-			       int wait);
-
 static size_t rng_buffer_size(void)
 {
 	return RNG_BUFFER_SIZE;
@@ -214,8 +211,8 @@ static int rng_dev_open(struct inode *inode, struct file *filp)
 	return 0;
 }
 
-static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size,
-			int wait) {
+static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, bool wait)
+{
 	int present;
 
 	BUG_ON(!mutex_is_locked(&reading_mutex));
@@ -418,21 +415,17 @@ static ssize_t rng_available_show(struct device *dev,
 				  struct device_attribute *attr,
 				  char *buf)
 {
-	int err;
 	struct hwrng *rng;
+	int len = 0;
 
-	err = mutex_lock_interruptible(&rng_mutex);
-	if (err)
+	if (mutex_lock_interruptible(&rng_mutex))
 		return -ERESTARTSYS;
-	buf[0] = '\0';
-	list_for_each_entry(rng, &rng_list, list) {
-		strlcat(buf, rng->name, PAGE_SIZE);
-		strlcat(buf, " ", PAGE_SIZE);
-	}
-	strlcat(buf, "none\n", PAGE_SIZE);
+	list_for_each_entry(rng, &rng_list, list)
+		len += sysfs_emit_at(buf, len, "%s ", rng->name);
+	len += sysfs_emit_at(buf, len, "none\n");
 	mutex_unlock(&rng_mutex);
 
-	return strlen(buf);
+	return len;
 }
 
 static ssize_t rng_selected_show(struct device *dev,
@@ -538,8 +531,7 @@ static int hwrng_fillfn(void *unused)
 		}
 
 		mutex_lock(&reading_mutex);
-		rc = rng_get_data(rng, rng_fillbuf,
-				  rng_buffer_size(), 1);
+		rc = rng_get_data(rng, rng_fillbuf, rng_buffer_size(), true);
 		if (current_quality != rng->quality)
 			rng->quality = current_quality; /* obsolete */
 		quality = rng->quality;
diff --git a/drivers/char/hw_random/geode-rng.c b/drivers/char/hw_random/geode-rng.c
index 1b21d58e1768e..ae63eff643448 100644
--- a/drivers/char/hw_random/geode-rng.c
+++ b/drivers/char/hw_random/geode-rng.c
@@ -46,8 +46,8 @@
  * want to register another driver on the same PCI id.
  */
 static const struct pci_device_id pci_tbl[] = {
-	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), 0, },
-	{ 0, },	/* terminate list */
+	{ PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES) },
+	{ }	/* terminate list */
 };
 MODULE_DEVICE_TABLE(pci, pci_tbl);
 
diff --git a/drivers/char/hw_random/mtk-rng.c b/drivers/char/hw_random/mtk-rng.c
index 5808d09d12c45..1f9793010e6c7 100644
--- a/drivers/char/hw_random/mtk-rng.c
+++ b/drivers/char/hw_random/mtk-rng.c
@@ -3,9 +3,11 @@
  * Driver for Mediatek Hardware Random Number Generator
  *
  * Copyright (C) 2017 Sean Wang <sean.wang@mediatek.com>
+ * Copyright (C) 2026 Daniel Golle <daniel@makrotopia.org>
  */
 #define MTK_RNG_DEV KBUILD_MODNAME
 
+#include <linux/arm-smccc.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/err.h>
@@ -17,6 +19,7 @@
 #include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/soc/mediatek/mtk_sip_svc.h>
 
 /* Runtime PM autosuspend timeout: */
 #define RNG_AUTOSUSPEND_TIMEOUT		100
@@ -30,6 +33,11 @@
 
 #define RNG_DATA			0x08
 
+/* Driver feature flags */
+#define MTK_RNG_SMC			BIT(0)
+
+#define MTK_SIP_KERNEL_GET_RND		MTK_SIP_SMC_CMD(0x550)
+
 #define to_mtk_rng(p)	container_of(p, struct mtk_rng, rng)
 
 struct mtk_rng {
@@ -37,6 +45,7 @@ struct mtk_rng {
 	struct clk *clk;
 	struct hwrng rng;
 	struct device *dev;
+	unsigned long flags;
 };
 
 static int mtk_rng_init(struct hwrng *rng)
@@ -103,6 +112,56 @@ static int mtk_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait)
 	return retval || !wait ? retval : -EIO;
 }
 
+static int mtk_rng_read_smc(struct hwrng *rng, void *buf, size_t max,
+			    bool wait)
+{
+	struct arm_smccc_res res;
+	int retval = 0;
+
+	while (max >= sizeof(u32)) {
+		arm_smccc_smc(MTK_SIP_KERNEL_GET_RND, 0, 0, 0, 0, 0, 0, 0,
+			      &res);
+		if (res.a0)
+			break;
+
+		*(u32 *)buf = res.a1;
+		retval += sizeof(u32);
+		buf += sizeof(u32);
+		max -= sizeof(u32);
+	}
+
+	return retval || !wait ? retval : -EIO;
+}
+
+static bool mtk_rng_hw_accessible(struct mtk_rng *priv)
+{
+	u32 val;
+	int err;
+
+	err = clk_prepare_enable(priv->clk);
+	if (err)
+		return false;
+
+	val = readl(priv->base + RNG_CTRL);
+	val |= RNG_EN;
+	writel(val, priv->base + RNG_CTRL);
+
+	val = readl(priv->base + RNG_CTRL);
+
+	if (val & RNG_EN) {
+		/* HW is accessible, clean up: disable RNG and clock */
+		writel(val & ~RNG_EN, priv->base + RNG_CTRL);
+		clk_disable_unprepare(priv->clk);
+		return true;
+	}
+
+	/*
+	 * If TF-A blocks direct access, the register reads back as 0.
+	 * Leave the clock enabled as TF-A needs it.
+	 */
+	return false;
+}
+
 static int mtk_rng_probe(struct platform_device *pdev)
 {
 	int ret;
@@ -114,23 +173,42 @@ static int mtk_rng_probe(struct platform_device *pdev)
 
 	priv->dev = &pdev->dev;
 	priv->rng.name = pdev->name;
-#ifndef CONFIG_PM
-	priv->rng.init = mtk_rng_init;
-	priv->rng.cleanup = mtk_rng_cleanup;
-#endif
-	priv->rng.read = mtk_rng_read;
 	priv->rng.quality = 900;
-
-	priv->clk = devm_clk_get(&pdev->dev, "rng");
-	if (IS_ERR(priv->clk)) {
-		ret = PTR_ERR(priv->clk);
-		dev_err(&pdev->dev, "no clock for device: %d\n", ret);
-		return ret;
+	priv->flags = (unsigned long)device_get_match_data(&pdev->dev);
+
+	if (!(priv->flags & MTK_RNG_SMC)) {
+		priv->clk = devm_clk_get(&pdev->dev, "rng");
+		if (IS_ERR(priv->clk)) {
+			ret = PTR_ERR(priv->clk);
+			dev_err(&pdev->dev, "no clock for device: %d\n", ret);
+			return ret;
+		}
+
+		priv->base = devm_platform_ioremap_resource(pdev, 0);
+		if (IS_ERR(priv->base))
+			return PTR_ERR(priv->base);
+
+		if (IS_ENABLED(CONFIG_HAVE_ARM_SMCCC) &&
+		    of_device_is_compatible(pdev->dev.of_node,
+					    "mediatek,mt7986-rng") &&
+		    !mtk_rng_hw_accessible(priv)) {
+			priv->flags |= MTK_RNG_SMC;
+			dev_info(&pdev->dev,
+				 "HW RNG not MMIO accessible, using SMC\n");
+		}
 	}
 
-	priv->base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(priv->base))
-		return PTR_ERR(priv->base);
+	if (priv->flags & MTK_RNG_SMC) {
+		if (!IS_ENABLED(CONFIG_HAVE_ARM_SMCCC))
+			return -ENODEV;
+		priv->rng.read = mtk_rng_read_smc;
+	} else {
+#ifndef CONFIG_PM
+		priv->rng.init = mtk_rng_init;
+		priv->rng.cleanup = mtk_rng_cleanup;
+#endif
+		priv->rng.read = mtk_rng_read;
+	}
 
 	ret = devm_hwrng_register(&pdev->dev, &priv->rng);
 	if (ret) {
@@ -139,12 +217,15 @@ static int mtk_rng_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	dev_set_drvdata(&pdev->dev, priv);
-	pm_runtime_set_autosuspend_delay(&pdev->dev, RNG_AUTOSUSPEND_TIMEOUT);
-	pm_runtime_use_autosuspend(&pdev->dev);
-	ret = devm_pm_runtime_enable(&pdev->dev);
-	if (ret)
-		return ret;
+	if (!(priv->flags & MTK_RNG_SMC)) {
+		dev_set_drvdata(&pdev->dev, priv);
+		pm_runtime_set_autosuspend_delay(&pdev->dev,
+						 RNG_AUTOSUSPEND_TIMEOUT);
+		pm_runtime_use_autosuspend(&pdev->dev);
+		ret = devm_pm_runtime_enable(&pdev->dev);
+		if (ret)
+			return ret;
+	}
 
 	dev_info(&pdev->dev, "registered RNG driver\n");
 
@@ -181,8 +262,11 @@ static const struct dev_pm_ops mtk_rng_pm_ops = {
 #endif	/* CONFIG_PM */
 
 static const struct of_device_id mtk_rng_match[] = {
-	{ .compatible = "mediatek,mt7986-rng" },
 	{ .compatible = "mediatek,mt7623-rng" },
+	{ .compatible = "mediatek,mt7981-rng", .data = (void *)MTK_RNG_SMC },
+	{ .compatible = "mediatek,mt7986-rng" },
+	{ .compatible = "mediatek,mt7987-rng", .data = (void *)MTK_RNG_SMC },
+	{ .compatible = "mediatek,mt7988-rng", .data = (void *)MTK_RNG_SMC },
 	{},
 };
 MODULE_DEVICE_TABLE(of, mtk_rng_match);
@@ -200,4 +284,5 @@ module_platform_driver(mtk_rng_driver);
 
 MODULE_DESCRIPTION("Mediatek Random Number Generator Driver");
 MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>");
+MODULE_AUTHOR("Daniel Golle <daniel@makrotopia.org>");
 MODULE_LICENSE("GPL");
diff --git a/drivers/char/ipmi/ipmb_dev_int.c b/drivers/char/ipmi/ipmb_dev_int.c
index 2fe1d205ce4e8..680ff15c30ab9 100644
--- a/drivers/char/ipmi/ipmb_dev_int.c
+++ b/drivers/char/ipmi/ipmb_dev_int.c
@@ -353,8 +353,8 @@ static void ipmb_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ipmb_id[] = {
-	{ "ipmb-dev" },
-	{}
+	{ .name = "ipmb-dev" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ipmb_id);
 
diff --git a/drivers/char/ipmi/ipmi_ipmb.c b/drivers/char/ipmi/ipmi_ipmb.c
index 28818952a7a4b..1f1e5718f082e 100644
--- a/drivers/char/ipmi/ipmi_ipmb.c
+++ b/drivers/char/ipmi/ipmi_ipmb.c
@@ -566,8 +566,8 @@ MODULE_DEVICE_TABLE(of, of_ipmi_ipmb_match);
 #endif
 
 static const struct i2c_device_id ipmi_ipmb_id[] = {
-	{ DEVICE_NAME },
-	{}
+	{ .name = DEVICE_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ipmi_ipmb_id);
 
diff --git a/drivers/char/ipmi/ipmi_kcs_sm.c b/drivers/char/ipmi/ipmi_kcs_sm.c
index efda90dcf5b3d..c89055db39e52 100644
--- a/drivers/char/ipmi/ipmi_kcs_sm.c
+++ b/drivers/char/ipmi/ipmi_kcs_sm.c
@@ -102,7 +102,7 @@ enum kcs_states {
 /* Timeouts in microseconds. */
 #define IBF_RETRY_TIMEOUT (5*USEC_PER_SEC)
 #define OBF_RETRY_TIMEOUT (5*USEC_PER_SEC)
-#define MAX_ERROR_RETRIES 10
+#define MAX_ERROR_RETRIES 2
 #define ERROR0_OBF_WAIT_JIFFIES (2*HZ)
 
 struct si_sm_data {
diff --git a/drivers/char/ipmi/ipmi_msghandler.c b/drivers/char/ipmi/ipmi_msghandler.c
index 869ac87a4b6ae..7a4566046b68c 100644
--- a/drivers/char/ipmi/ipmi_msghandler.c
+++ b/drivers/char/ipmi/ipmi_msghandler.c
@@ -1610,14 +1610,12 @@ int ipmi_set_gets_events(struct ipmi_user *user, bool val)
 {
 	struct ipmi_smi      *intf = user->intf;
 	struct ipmi_recv_msg *msg, *msg2;
-	struct list_head     msgs;
+	LIST_HEAD(msgs);
 
 	user = acquire_ipmi_user(user);
 	if (!user)
 		return -ENODEV;
 
-	INIT_LIST_HEAD(&msgs);
-
 	mutex_lock(&intf->events_mutex);
 	if (user->gets_events == val)
 		goto out;
@@ -3785,10 +3783,9 @@ static void cleanup_smi_msgs(struct ipmi_smi *intf)
 	struct seq_table *ent;
 	struct ipmi_smi_msg *msg;
 	struct list_head *entry;
-	struct list_head tmplist;
+	LIST_HEAD(tmplist);
 
 	/* Clear out our transmit queues and hold the messages. */
-	INIT_LIST_HEAD(&tmplist);
 	list_splice_tail(&intf->hp_xmit_msgs, &tmplist);
 	list_splice_tail(&intf->xmit_msgs, &tmplist);
 
@@ -4442,7 +4439,7 @@ static int handle_read_event_rsp(struct ipmi_smi *intf,
 				 struct ipmi_smi_msg *msg)
 {
 	struct ipmi_recv_msg *recv_msg, *recv_msg2;
-	struct list_head     msgs;
+	LIST_HEAD(msgs);
 	struct ipmi_user     *user;
 	int rv = 0, deliver_count = 0;
 
@@ -4457,8 +4454,6 @@ static int handle_read_event_rsp(struct ipmi_smi *intf,
 		return 0;
 	}
 
-	INIT_LIST_HEAD(&msgs);
-
 	mutex_lock(&intf->events_mutex);
 
 	ipmi_inc_stat(intf, events);
@@ -5101,7 +5096,7 @@ static void check_msg_timeout(struct ipmi_smi *intf, struct seq_table *ent,
 static bool ipmi_timeout_handler(struct ipmi_smi *intf,
 				 unsigned long timeout_period)
 {
-	struct list_head     timeouts;
+	LIST_HEAD(timeouts);
 	struct ipmi_recv_msg *msg, *msg2;
 	unsigned long        flags;
 	int                  i;
@@ -5120,7 +5115,6 @@ static bool ipmi_timeout_handler(struct ipmi_smi *intf,
 	 * have timed out, putting them in the timeouts
 	 * list.
 	 */
-	INIT_LIST_HEAD(&timeouts);
 	mutex_lock(&intf->seq_lock);
 	if (intf->ipmb_maintenance_mode_timeout) {
 		if (intf->ipmb_maintenance_mode_timeout <= timeout_period)
diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c
index f419b46bf0020..add043b812ea0 100644
--- a/drivers/char/ipmi/ipmi_ssif.c
+++ b/drivers/char/ipmi/ipmi_ssif.c
@@ -2094,7 +2094,7 @@ static int dmi_ipmi_probe(struct platform_device *pdev)
 #endif
 
 static const struct i2c_device_id ssif_id[] = {
-	{ DEVICE_NAME },
+	{ .name = DEVICE_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ssif_id);
diff --git a/drivers/char/ipmi/ssif_bmc.c b/drivers/char/ipmi/ssif_bmc.c
index 1df0e9284ad96..6036897725f3b 100644
--- a/drivers/char/ipmi/ssif_bmc.c
+++ b/drivers/char/ipmi/ssif_bmc.c
@@ -874,7 +874,7 @@ static const struct of_device_id ssif_bmc_match[] = {
 MODULE_DEVICE_TABLE(of, ssif_bmc_match);
 
 static const struct i2c_device_id ssif_bmc_id[] = {
-	{ DEVICE_NAME },
+	{ .name = DEVICE_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ssif_bmc_id);
diff --git a/drivers/char/sonypi.c b/drivers/char/sonypi.c
index ccda997a9098a..959949f04f7d9 100644
--- a/drivers/char/sonypi.c
+++ b/drivers/char/sonypi.c
@@ -37,6 +37,7 @@
 #include <linux/kfifo.h>
 #include <linux/platform_device.h>
 #include <linux/gfp.h>
+#include <linux/string.h>
 #include <linux/string_choices.h>
 
 #include <linux/uaccess.h>
@@ -1120,8 +1121,8 @@ static int sonypi_acpi_probe(struct platform_device *pdev)
 	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 
 	sonypi_acpi_device = device;
-	strcpy(acpi_device_name(device), "Sony laptop hotkeys");
-	strcpy(acpi_device_class(device), "sony/hotkey");
+	strscpy(acpi_device_name(device), "Sony laptop hotkeys");
+	strscpy(acpi_device_class(device), "sony/hotkey");
 	return 0;
 }
 
diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c
index b11e6fa8b740e..52ee350da8671 100644
--- a/drivers/char/tpm/tpm2-cmd.c
+++ b/drivers/char/tpm/tpm2-cmd.c
@@ -71,9 +71,9 @@ static const struct {
 	{TPM2_CC_HIERARCHY_CHANGE_AUTH, 2000},
 	{TPM2_CC_GET_CAPABILITY, 750},
 	{TPM2_CC_NV_READ, 2000},
-	{TPM2_CC_CREATE_PRIMARY, 30000},
-	{TPM2_CC_CREATE, 30000},
-	{TPM2_CC_CREATE_LOADED, 30000},
+	{TPM2_CC_CREATE_PRIMARY, 300000},
+	{TPM2_CC_CREATE, 300000},
+	{TPM2_CC_CREATE_LOADED, 300000},
 };
 
 /**
diff --git a/drivers/char/tpm/tpm2-sessions.c b/drivers/char/tpm/tpm2-sessions.c
index c4da6fde748f4..795cd99dc6fed 100644
--- a/drivers/char/tpm/tpm2-sessions.c
+++ b/drivers/char/tpm/tpm2-sessions.c
@@ -285,11 +285,14 @@ int tpm_buf_append_name(struct tpm_chip *chip, struct tpm_buf *buf,
 	    mso == TPM2_MSO_NVRAM) {
 		if (!name) {
 			ret = tpm2_read_public(chip, handle, auth->name[slot]);
-			if (ret < 0)
-				goto err;
-
-			name_size_alg = ret;
+		} else {
+			ret = name_size(name);
 		}
+
+		if (ret < 0)
+			goto err;
+
+		name_size_alg = ret;
 	} else {
 		if (name) {
 			dev_err(&chip->dev, "handle 0x%08x does not use a name\n",
diff --git a/drivers/char/tpm/tpm_crb.c b/drivers/char/tpm/tpm_crb.c
index 7d1377e8e616a..ceb4100ba400f 100644
--- a/drivers/char/tpm/tpm_crb.c
+++ b/drivers/char/tpm/tpm_crb.c
@@ -786,8 +786,8 @@ static int crb_map_pluton(struct device *dev, struct crb_priv *priv,
 static int crb_acpi_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct acpi_device *device = ACPI_COMPANION(dev);
 	struct acpi_table_tpm2 *buf;
+	struct acpi_device *device;
 	struct crb_priv *priv;
 	struct tpm_chip *chip;
 	struct tpm2_crb_smc *crb_smc;
@@ -797,6 +797,10 @@ static int crb_acpi_probe(struct platform_device *pdev)
 	u32 sm;
 	int rc;
 
+	device = ACPI_COMPANION(dev);
+	if (!device)
+		return -ENODEV;
+
 	status = acpi_get_table(ACPI_SIG_TPM2, 1,
 				(struct acpi_table_header **) &buf);
 	if (ACPI_FAILURE(status) || buf->header.length < sizeof(*buf)) {
diff --git a/drivers/char/tpm/tpm_svsm.c b/drivers/char/tpm/tpm_svsm.c
index f5ba0f64850b9..b74d60f687d5b 100644
--- a/drivers/char/tpm/tpm_svsm.c
+++ b/drivers/char/tpm/tpm_svsm.c
@@ -49,7 +49,7 @@ static int tpm_svsm_send(struct tpm_chip *chip, u8 *buf, size_t bufsiz,
 	return svsm_vtpm_cmd_response_parse(priv->buffer, buf, bufsiz);
 }
 
-static struct tpm_class_ops tpm_chip_ops = {
+static const struct tpm_class_ops tpm_chip_ops = {
 	.flags = TPM_OPS_AUTO_STARTUP,
 	.send = tpm_svsm_send,
 };
diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c
index 21d79ad3b164e..153a57c792400 100644
--- a/drivers/char/tpm/tpm_tis_core.c
+++ b/drivers/char/tpm/tpm_tis_core.c
@@ -66,8 +66,8 @@ static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask,
 		bool check_cancel)
 {
 	struct tpm_tis_data *priv = dev_get_drvdata(&chip->dev);
+	DEFINE_WAIT_FUNC(wait, woken_wake_function);
 	unsigned long stop;
-	long rc;
 	u8 status;
 	bool canceled = false;
 	u8 sts_mask;
@@ -87,23 +87,30 @@ static int wait_for_tpm_stat(struct tpm_chip *chip, u8 mask,
 	/* process status changes with irq support */
 	if (sts_mask) {
 		ret = -ETIME;
+		add_wait_queue(queue, &wait);
 again:
+		if (wait_for_tpm_stat_cond(chip, sts_mask, check_cancel,
+					   &canceled)) {
+			ret = canceled ? -ECANCELED : 0;
+			goto out;
+		}
+
 		timeout = stop - jiffies;
 		if ((long)timeout <= 0)
-			return -ETIME;
-		rc = wait_event_interruptible_timeout(*queue,
-			wait_for_tpm_stat_cond(chip, sts_mask, check_cancel,
-					       &canceled),
-			timeout);
-		if (rc > 0) {
-			if (canceled)
-				return -ECANCELED;
-			ret = 0;
-		}
-		if (rc == -ERESTARTSYS && freezing(current)) {
-			clear_thread_flag(TIF_SIGPENDING);
-			goto again;
+			goto out;
+
+		if (signal_pending(current)) {
+			if (freezing(current)) {
+				clear_thread_flag(TIF_SIGPENDING);
+				goto again;
+			}
+			goto out;
 		}
+
+		wait_woken(&wait, TASK_INTERRUPTIBLE, timeout);
+		goto again;
+out:
+		remove_wait_queue(queue, &wait);
 	}
 
 	if (ret)
diff --git a/drivers/clk/Makefile b/drivers/clk/Makefile
index a3e2862ebd7eb..e83c60320bfeb 100644
--- a/drivers/clk/Makefile
+++ b/drivers/clk/Makefile
@@ -141,7 +141,7 @@ obj-$(CONFIG_COMMON_CLK_PXA)		+= pxa/
 obj-$(CONFIG_COMMON_CLK_QCOM)		+= qcom/
 obj-y					+= ralink/
 obj-y					+= renesas/
-obj-$(CONFIG_ARCH_ROCKCHIP)		+= rockchip/
+obj-$(CONFIG_COMMON_CLK_ROCKCHIP)	+= rockchip/
 obj-$(CONFIG_COMMON_CLK_SAMSUNG)	+= samsung/
 obj-$(CONFIG_CLK_SIFIVE)		+= sifive/
 obj-y					+= socfpga/
diff --git a/drivers/clk/at91/sam9x7.c b/drivers/clk/at91/sam9x7.c
index 89868a0aeaba9..8b52da194849f 100644
--- a/drivers/clk/at91/sam9x7.c
+++ b/drivers/clk/at91/sam9x7.c
@@ -387,7 +387,7 @@ static const struct {
 	{ .n = "dma0_clk",	.id = 20, },
 	{ .n = "uhphs_clk",	.id = 22, },
 	{ .n = "udphs_clk",	.id = 23, },
-	{ .n = "macb0_clk",	.id = 24, },
+	{ .n = "gmac_clk",	.id = 24, },
 	{ .n = "lcd_clk",	.id = 25, },
 	{ .n = "sdmmc1_clk",	.id = 26, },
 	{ .n = "ssc_clk",	.id = 28, },
@@ -420,7 +420,6 @@ static const struct {
 	{ .n = "lvdsc_clk",	.id = 56, },
 	{ .n = "pit64b1_clk",	.id = 58, },
 	{ .n = "puf_clk",	.id = 59, },
-	{ .n = "gmactsu_clk",	.id = 67, },
 };
 
 /*
@@ -569,6 +568,15 @@ static const struct {
 		.pp_chg_id = INT_MIN,
 	},
 
+	{
+		.n = "gmac_gclk",
+		.id = 24,
+		.pp = { "audiopll_divpmcck", "plla_div2pmcck", },
+		.pp_mux_table = { 6, 8, },
+		.pp_count = 2,
+		.pp_chg_id = INT_MIN,
+	},
+
 	{
 		.n = "lcd_gclk",
 		.id = 25,
@@ -702,15 +710,6 @@ static const struct {
 		.pp_count = 1,
 		.pp_chg_id = INT_MIN,
 	},
-
-	{
-		.n = "gmac_gclk",
-		.id = 67,
-		.pp = { "audiopll_divpmcck", "plla_div2pmcck", },
-		.pp_mux_table = { 6, 8, },
-		.pp_count = 2,
-		.pp_chg_id = INT_MIN,
-	},
 };
 
 static void __init sam9x7_pmc_setup(struct device_node *np)
diff --git a/drivers/clk/bcm/clk-iproc-asiu.c b/drivers/clk/bcm/clk-iproc-asiu.c
index 819ab1b55df3b..56a8132279814 100644
--- a/drivers/clk/bcm/clk-iproc-asiu.c
+++ b/drivers/clk/bcm/clk-iproc-asiu.c
@@ -27,8 +27,7 @@ struct iproc_asiu {
 	void __iomem *div_base;
 	void __iomem *gate_base;
 
-	struct clk_hw_onecell_data *clk_data;
-	struct iproc_asiu_clk *clks;
+	struct iproc_asiu_clk clks[];
 };
 
 #define to_asiu_clk(hw) container_of(hw, struct iproc_asiu_clk, hw)
@@ -184,22 +183,19 @@ void __init iproc_asiu_setup(struct device_node *node,
 {
 	int i, ret;
 	struct iproc_asiu *asiu;
+	struct clk_hw_onecell_data *clk_data;
 
 	if (WARN_ON(!gate || !div))
 		return;
 
-	asiu = kzalloc_obj(*asiu);
+	asiu = kzalloc_flex(*asiu, clks, num_clks);
 	if (WARN_ON(!asiu))
 		return;
 
-	asiu->clk_data = kzalloc_flex(*asiu->clk_data, hws, num_clks);
-	if (WARN_ON(!asiu->clk_data))
+	clk_data = kzalloc_flex(*clk_data, hws, num_clks);
+	if (WARN_ON(!clk_data))
 		goto err_clks;
-	asiu->clk_data->num = num_clks;
-
-	asiu->clks = kzalloc_objs(*asiu->clks, num_clks);
-	if (WARN_ON(!asiu->clks))
-		goto err_asiu_clks;
+	clk_data->num = num_clks;
 
 	asiu->div_base = of_iomap(node, 0);
 	if (WARN_ON(!asiu->div_base))
@@ -236,11 +232,11 @@ void __init iproc_asiu_setup(struct device_node *node,
 		ret = clk_hw_register(NULL, &asiu_clk->hw);
 		if (WARN_ON(ret))
 			goto err_clk_register;
-		asiu->clk_data->hws[i] = &asiu_clk->hw;
+		clk_data->hws[i] = &asiu_clk->hw;
 	}
 
 	ret = of_clk_add_hw_provider(node, of_clk_hw_onecell_get,
-				     asiu->clk_data);
+				     clk_data);
 	if (WARN_ON(ret))
 		goto err_clk_register;
 
@@ -248,17 +244,14 @@ void __init iproc_asiu_setup(struct device_node *node,
 
 err_clk_register:
 	while (--i >= 0)
-		clk_hw_unregister(asiu->clk_data->hws[i]);
+		clk_hw_unregister(clk_data->hws[i]);
 	iounmap(asiu->gate_base);
 
 err_iomap_gate:
 	iounmap(asiu->div_base);
 
 err_iomap_div:
-	kfree(asiu->clks);
-
-err_asiu_clks:
-	kfree(asiu->clk_data);
+	kfree(clk_data);
 
 err_clks:
 	kfree(asiu);
diff --git a/drivers/clk/clk-axi-clkgen.c b/drivers/clk/clk-axi-clkgen.c
index fa5ccef73e60d..26f76a6db8202 100644
--- a/drivers/clk/clk-axi-clkgen.c
+++ b/drivers/clk/clk-axi-clkgen.c
@@ -521,7 +521,7 @@ static int axi_clkgen_setup_limits(struct axi_clkgen *axi_clkgen,
 		axi_clkgen->limits.fvco_max = 1200000;
 		axi_clkgen->limits.fpfd_max = 450000;
 		break;
-	case ADI_AXI_FPGA_SPEED_2 ... ADI_AXI_FPGA_SPEED_2LV:
+	case ADI_AXI_FPGA_SPEED_2 ... ADI_AXI_FPGA_SPEED_2MP:
 		axi_clkgen->limits.fvco_max = 1440000;
 		axi_clkgen->limits.fpfd_max = 500000;
 		if (family == ADI_AXI_FPGA_FAMILY_KINTEX || family == ADI_AXI_FPGA_FAMILY_ARTIX) {
@@ -546,6 +546,9 @@ static int axi_clkgen_setup_limits(struct axi_clkgen *axi_clkgen,
 	if (tech == ADI_AXI_FPGA_TECH_ULTRASCALE_PLUS) {
 		axi_clkgen->limits.fvco_max = 1600000;
 		axi_clkgen->limits.fvco_min = 800000;
+	} else if (tech == ADI_AXI_FPGA_TECH_VERSAL) {
+		axi_clkgen->limits.fvco_max = 4320000;
+		axi_clkgen->limits.fvco_min = 2160000;
 	}
 
 	return 0;
diff --git a/drivers/clk/clk-bulk.c b/drivers/clk/clk-bulk.c
index d85dae4bdf895..acd9dff300722 100644
--- a/drivers/clk/clk-bulk.c
+++ b/drivers/clk/clk-bulk.c
@@ -12,7 +12,8 @@
 #include <linux/of.h>
 #include <linux/slab.h>
 
-static int __must_check of_clk_bulk_get(struct device_node *np, int num_clks,
+static int __must_check of_clk_bulk_get(struct device *dev,
+					struct device_node *np, int num_clks,
 					struct clk_bulk_data *clks)
 {
 	int ret;
@@ -28,8 +29,8 @@ static int __must_check of_clk_bulk_get(struct device_node *np, int num_clks,
 		clks[i].clk = of_clk_get(np, i);
 		if (IS_ERR(clks[i].clk)) {
 			ret = PTR_ERR(clks[i].clk);
-			pr_err("%pOF: Failed to get clk index: %d ret: %d\n",
-			       np, i, ret);
+			dev_err_probe(dev, ret, "%pOF: Failed to get clk index: %d (%s)\n",
+				      np, i, clks[i].id);
 			clks[i].clk = NULL;
 			goto err;
 		}
@@ -43,7 +44,8 @@ static int __must_check of_clk_bulk_get(struct device_node *np, int num_clks,
 	return ret;
 }
 
-static int __must_check of_clk_bulk_get_all(struct device_node *np,
+static int __must_check of_clk_bulk_get_all(struct device *dev,
+					    struct device_node *np,
 					    struct clk_bulk_data **clks)
 {
 	struct clk_bulk_data *clk_bulk;
@@ -58,7 +60,7 @@ static int __must_check of_clk_bulk_get_all(struct device_node *np,
 	if (!clk_bulk)
 		return -ENOMEM;
 
-	ret = of_clk_bulk_get(np, num_clks, clk_bulk);
+	ret = of_clk_bulk_get(dev, np, num_clks, clk_bulk);
 	if (ret) {
 		kfree(clk_bulk);
 		return ret;
@@ -144,7 +146,7 @@ int __must_check clk_bulk_get_all(struct device *dev,
 	if (!np)
 		return 0;
 
-	return of_clk_bulk_get_all(np, clks);
+	return of_clk_bulk_get_all(dev, np, clks);
 }
 EXPORT_SYMBOL(clk_bulk_get_all);
 
diff --git a/drivers/clk/clk-max77686.c b/drivers/clk/clk-max77686.c
index 3727d54724500..9149ce4f702d5 100644
--- a/drivers/clk/clk-max77686.c
+++ b/drivers/clk/clk-max77686.c
@@ -47,8 +47,8 @@ struct max77686_clk_init_data {
 
 struct max77686_clk_driver_data {
 	enum max77686_chip_name chip;
-	struct max77686_clk_init_data *max_clk_data;
 	size_t num_clks;
+	struct max77686_clk_init_data max_clk_data[] __counted_by(num_clks);
 };
 
 static const struct
@@ -168,19 +168,7 @@ static int max77686_clk_probe(struct platform_device *pdev)
 	struct regmap *regmap;
 	int i, ret, num_clks;
 
-	drv_data = devm_kzalloc(dev, sizeof(*drv_data), GFP_KERNEL);
-	if (!drv_data)
-		return -ENOMEM;
-
-	regmap = dev_get_regmap(parent, NULL);
-	if (!regmap) {
-		dev_err(dev, "Failed to get rtc regmap\n");
-		return -ENODEV;
-	}
-
-	drv_data->chip = id->driver_data;
-
-	switch (drv_data->chip) {
+	switch (id->driver_data) {
 	case CHIP_MAX77686:
 		num_clks = MAX77686_CLKS_NUM;
 		hw_clks = max77686_hw_clks_info;
@@ -201,13 +189,19 @@ static int max77686_clk_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	drv_data->num_clks = num_clks;
-	drv_data->max_clk_data = devm_kcalloc(dev, num_clks,
-					      sizeof(*drv_data->max_clk_data),
-					      GFP_KERNEL);
-	if (!drv_data->max_clk_data)
+	drv_data = devm_kzalloc(dev, struct_size(drv_data, max_clk_data, num_clks), GFP_KERNEL);
+	if (!drv_data)
 		return -ENOMEM;
 
+	drv_data->num_clks = num_clks;
+	drv_data->chip = id->driver_data;
+
+	regmap = dev_get_regmap(parent, NULL);
+	if (!regmap) {
+		dev_err(dev, "Failed to get rtc regmap\n");
+		return -ENODEV;
+	}
+
 	for (i = 0; i < num_clks; i++) {
 		struct max77686_clk_init_data *max_clk_data;
 		const char *clk_name;
diff --git a/drivers/clk/clk-scmi.c b/drivers/clk/clk-scmi.c
index 6b286ea6f1218..7c562559ad8bb 100644
--- a/drivers/clk/clk-scmi.c
+++ b/drivers/clk/clk-scmi.c
@@ -12,7 +12,6 @@
 #include <linux/of.h>
 #include <linux/module.h>
 #include <linux/scmi_protocol.h>
-#include <asm/div64.h>
 
 #define NOT_ATOMIC	false
 #define ATOMIC		true
@@ -57,35 +56,17 @@ static unsigned long scmi_clk_recalc_rate(struct clk_hw *hw,
 static int scmi_clk_determine_rate(struct clk_hw *hw,
 				   struct clk_rate_request *req)
 {
-	u64 fmin, fmax, ftmp;
+	int ret;
 	struct scmi_clk *clk = to_scmi_clk(hw);
 
 	/*
-	 * We can't figure out what rate it will be, so just return the
-	 * rate back to the caller. scmi_clk_recalc_rate() will be called
-	 * after the rate is set and we'll know what rate the clock is
+	 * If we could not get a better rate scmi_clk_recalc_rate() will be
+	 * called after the rate is set and we'll know what rate the clock is
 	 * running at then.
 	 */
-	if (clk->info->rate_discrete)
-		return 0;
-
-	fmin = clk->info->range.min_rate;
-	fmax = clk->info->range.max_rate;
-	if (req->rate <= fmin) {
-		req->rate = fmin;
-
-		return 0;
-	} else if (req->rate >= fmax) {
-		req->rate = fmax;
-
-		return 0;
-	}
-
-	ftmp = req->rate - fmin;
-	ftmp += clk->info->range.step_size - 1; /* to round up */
-	do_div(ftmp, clk->info->range.step_size);
-
-	req->rate = ftmp * clk->info->range.step_size + fmin;
+	ret = scmi_proto_clk_ops->determine_rate(clk->ph, clk->id, &req->rate);
+	if (ret)
+		return ret;
 
 	return 0;
 }
@@ -221,7 +202,6 @@ static int scmi_clk_ops_init(struct device *dev, struct scmi_clk *sclk,
 			     const struct clk_ops *scmi_ops)
 {
 	int ret;
-	unsigned long min_rate, max_rate;
 
 	struct clk_init_data init = {
 		.flags = CLK_GET_RATE_NOCACHE,
@@ -236,20 +216,8 @@ static int scmi_clk_ops_init(struct device *dev, struct scmi_clk *sclk,
 	if (ret)
 		return ret;
 
-	if (sclk->info->rate_discrete) {
-		int num_rates = sclk->info->list.num_rates;
-
-		if (num_rates <= 0)
-			return -EINVAL;
-
-		min_rate = sclk->info->list.rates[0];
-		max_rate = sclk->info->list.rates[num_rates - 1];
-	} else {
-		min_rate = sclk->info->range.min_rate;
-		max_rate = sclk->info->range.max_rate;
-	}
-
-	clk_hw_set_rate_range(&sclk->hw, min_rate, max_rate);
+	clk_hw_set_rate_range(&sclk->hw, sclk->info->min_rate,
+			      sclk->info->max_rate);
 	return ret;
 }
 
diff --git a/drivers/clk/clk-scpi.c b/drivers/clk/clk-scpi.c
index 7806569cd0d5c..24cee7c9fda6c 100644
--- a/drivers/clk/clk-scpi.c
+++ b/drivers/clk/clk-scpi.c
@@ -258,7 +258,7 @@ static void scpi_clocks_remove(struct platform_device *pdev)
 	}
 
 	for_each_available_child_of_node(np, child)
-		of_clk_del_provider(np);
+		of_clk_del_provider(child);
 }
 
 static int scpi_clocks_probe(struct platform_device *pdev)
diff --git a/drivers/clk/hisilicon/clk.c b/drivers/clk/hisilicon/clk.c
index fae65127cd4aa..08050ff1c8cf9 100644
--- a/drivers/clk/hisilicon/clk.c
+++ b/drivers/clk/hisilicon/clk.c
@@ -70,7 +70,7 @@ struct hisi_clock_data *hisi_clk_init(struct device_node *np,
 
 	clk_data = kzalloc_obj(*clk_data);
 	if (!clk_data)
-		goto err;
+		goto err_base;
 
 	clk_data->base = base;
 	clk_table = kzalloc_objs(*clk_table, nr_clks);
@@ -83,6 +83,8 @@ struct hisi_clock_data *hisi_clk_init(struct device_node *np,
 	return clk_data;
 err_data:
 	kfree(clk_data);
+err_base:
+	iounmap(base);
 err:
 	return NULL;
 }
diff --git a/drivers/clk/hisilicon/clkdivider-hi6220.c b/drivers/clk/hisilicon/clkdivider-hi6220.c
index 1787ecefe601b..20a337383a1e1 100644
--- a/drivers/clk/hisilicon/clkdivider-hi6220.c
+++ b/drivers/clk/hisilicon/clkdivider-hi6220.c
@@ -26,8 +26,8 @@
  * @shift:	shift to the divider bit field
  * @width:	width of the divider bit field
  * @mask:	mask for setting divider rate
- * @table:	the div table that the divider supports
  * @lock:	register lock
+ * @table:	the div table that the divider supports
  */
 struct hi6220_clk_divider {
 	struct clk_hw	hw;
@@ -35,8 +35,8 @@ struct hi6220_clk_divider {
 	u8		shift;
 	u8		width;
 	u32		mask;
-	const struct clk_div_table *table;
 	spinlock_t	*lock;
+	struct clk_div_table table[];
 };
 
 #define to_hi6220_clk_divider(_hw)	\
@@ -108,24 +108,19 @@ struct clk *hi6220_register_clkdiv(struct device *dev, const char *name,
 	u32 max_div, min_div;
 	int i;
 
-	/* allocate the divider */
-	div = kzalloc_obj(*div);
-	if (!div)
-		return ERR_PTR(-ENOMEM);
-
 	/* Init the divider table */
 	max_div = div_mask(width) + 1;
 	min_div = 1;
 
-	table = kzalloc_objs(*table, max_div + 1);
-	if (!table) {
-		kfree(div);
+	/* allocate the divider */
+	div = kzalloc_flex(*div, table, max_div + 1);
+	if (!div)
 		return ERR_PTR(-ENOMEM);
-	}
 
 	for (i = 0; i < max_div; i++) {
-		table[i].div = min_div + i;
-		table[i].val = table[i].div - 1;
+		table = &div->table[i];
+		table->div = min_div + i;
+		table->val = table->div - 1;
 	}
 
 	init.name = name;
@@ -141,14 +136,11 @@ struct clk *hi6220_register_clkdiv(struct device *dev, const char *name,
 	div->mask = mask_bit ? BIT(mask_bit) : 0;
 	div->lock = lock;
 	div->hw.init = &init;
-	div->table = table;
 
 	/* register the clock */
 	clk = clk_register(dev, &div->hw);
-	if (IS_ERR(clk)) {
-		kfree(table);
+	if (IS_ERR(clk))
 		kfree(div);
-	}
 
 	return clk;
 }
diff --git a/drivers/clk/mediatek/clk-mt7988-infracfg.c b/drivers/clk/mediatek/clk-mt7988-infracfg.c
index ef8267319d91b..13ffa9d88e24b 100644
--- a/drivers/clk/mediatek/clk-mt7988-infracfg.c
+++ b/drivers/clk/mediatek/clk-mt7988-infracfg.c
@@ -56,49 +56,45 @@ static const char *const infra_pcie_gfmux_tl_ck_o_p3_parents[] __initconst = {
 
 static const struct mtk_mux infra_muxes[] = {
 	/* MODULE_CLK_SEL_0 */
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_MUX_UART0_SEL, "infra_mux_uart0_sel",
-			     infra_mux_uart0_parents, 0x0018, 0x0010, 0x0014, 0, 1, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_MUX_UART1_SEL, "infra_mux_uart1_sel",
-			     infra_mux_uart1_parents, 0x0018, 0x0010, 0x0014, 1, 1, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_MUX_UART2_SEL, "infra_mux_uart2_sel",
-			     infra_mux_uart2_parents, 0x0018, 0x0010, 0x0014, 2, 1, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_MUX_SPI0_SEL, "infra_mux_spi0_sel", infra_mux_spi0_parents,
-			     0x0018, 0x0010, 0x0014, 4, 1, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_MUX_SPI1_SEL, "infra_mux_spi1_sel", infra_mux_spi1_parents,
-			     0x0018, 0x0010, 0x0014, 5, 1, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_MUX_SPI2_SEL, "infra_mux_spi2_sel", infra_mux_spi0_parents,
-			     0x0018, 0x0010, 0x0014, 6, 1, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_SEL, "infra_pwm_sel", infra_pwm_bck_parents, 0x0018,
-			     0x0010, 0x0014, 14, 2, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_CK1_SEL, "infra_pwm_ck1_sel", infra_pwm_bck_parents,
-			     0x0018, 0x0010, 0x0014, 16, 2, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_CK2_SEL, "infra_pwm_ck2_sel", infra_pwm_bck_parents,
-			     0x0018, 0x0010, 0x0014, 18, 2, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_CK3_SEL, "infra_pwm_ck3_sel", infra_pwm_bck_parents,
-			     0x0018, 0x0010, 0x0014, 20, 2, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_CK4_SEL, "infra_pwm_ck4_sel", infra_pwm_bck_parents,
-			     0x0018, 0x0010, 0x0014, 22, 2, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_CK5_SEL, "infra_pwm_ck5_sel", infra_pwm_bck_parents,
-			     0x0018, 0x0010, 0x0014, 24, 2, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_CK6_SEL, "infra_pwm_ck6_sel", infra_pwm_bck_parents,
-			     0x0018, 0x0010, 0x0014, 26, 2, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_CK7_SEL, "infra_pwm_ck7_sel", infra_pwm_bck_parents,
-			     0x0018, 0x0010, 0x0014, 28, 2, -1, -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PWM_CK8_SEL, "infra_pwm_ck8_sel", infra_pwm_bck_parents,
-			     0x0018, 0x0010, 0x0014, 30, 2, -1, -1, -1),
+	MUX_CLR_SET(CLK_INFRA_MUX_UART0_SEL, "infra_mux_uart0_sel",
+		    infra_mux_uart0_parents, 0x0018, 0x0010, 0x0014, 0, 1),
+	MUX_CLR_SET(CLK_INFRA_MUX_UART1_SEL, "infra_mux_uart1_sel",
+		    infra_mux_uart1_parents, 0x0018, 0x0010, 0x0014, 1, 1),
+	MUX_CLR_SET(CLK_INFRA_MUX_UART2_SEL, "infra_mux_uart2_sel",
+		    infra_mux_uart2_parents, 0x0018, 0x0010, 0x0014, 2, 1),
+	MUX_CLR_SET(CLK_INFRA_MUX_SPI0_SEL, "infra_mux_spi0_sel",
+		    infra_mux_spi0_parents, 0x0018, 0x0010, 0x0014, 4, 1),
+	MUX_CLR_SET(CLK_INFRA_MUX_SPI1_SEL, "infra_mux_spi1_sel",
+		    infra_mux_spi1_parents, 0x0018, 0x0010, 0x0014, 5, 1),
+	MUX_CLR_SET(CLK_INFRA_MUX_SPI2_SEL, "infra_mux_spi2_sel",
+		    infra_mux_spi0_parents, 0x0018, 0x0010, 0x0014, 6, 1),
+	MUX_CLR_SET(CLK_INFRA_PWM_SEL, "infra_pwm_sel",
+		    infra_pwm_bck_parents, 0x0018, 0x0010, 0x0014, 14, 2),
+	MUX_CLR_SET(CLK_INFRA_PWM_CK1_SEL, "infra_pwm_ck1_sel",
+		    infra_pwm_bck_parents, 0x0018, 0x0010, 0x0014, 16, 2),
+	MUX_CLR_SET(CLK_INFRA_PWM_CK2_SEL, "infra_pwm_ck2_sel",
+		    infra_pwm_bck_parents, 0x0018, 0x0010, 0x0014, 18, 2),
+	MUX_CLR_SET(CLK_INFRA_PWM_CK3_SEL, "infra_pwm_ck3_sel",
+		    infra_pwm_bck_parents, 0x0018, 0x0010, 0x0014, 20, 2),
+	MUX_CLR_SET(CLK_INFRA_PWM_CK4_SEL, "infra_pwm_ck4_sel",
+		    infra_pwm_bck_parents, 0x0018, 0x0010, 0x0014, 22, 2),
+	MUX_CLR_SET(CLK_INFRA_PWM_CK5_SEL, "infra_pwm_ck5_sel",
+		    infra_pwm_bck_parents, 0x0018, 0x0010, 0x0014, 24, 2),
+	MUX_CLR_SET(CLK_INFRA_PWM_CK6_SEL, "infra_pwm_ck6_sel",
+		    infra_pwm_bck_parents, 0x0018, 0x0010, 0x0014, 26, 2),
+	MUX_CLR_SET(CLK_INFRA_PWM_CK7_SEL, "infra_pwm_ck7_sel",
+		    infra_pwm_bck_parents, 0x0018, 0x0010, 0x0014, 28, 2),
+	MUX_CLR_SET(CLK_INFRA_PWM_CK8_SEL, "infra_pwm_ck8_sel",
+		    infra_pwm_bck_parents, 0x0018, 0x0010, 0x0014, 30, 2),
 	/* MODULE_CLK_SEL_1 */
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PCIE_GFMUX_TL_O_P0_SEL, "infra_pcie_gfmux_tl_o_p0_sel",
-			     infra_pcie_gfmux_tl_ck_o_p0_parents, 0x0028, 0x0020, 0x0024, 0, 2, -1,
-			     -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PCIE_GFMUX_TL_O_P1_SEL, "infra_pcie_gfmux_tl_o_p1_sel",
-			     infra_pcie_gfmux_tl_ck_o_p1_parents, 0x0028, 0x0020, 0x0024, 2, 2, -1,
-			     -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PCIE_GFMUX_TL_O_P2_SEL, "infra_pcie_gfmux_tl_o_p2_sel",
-			     infra_pcie_gfmux_tl_ck_o_p2_parents, 0x0028, 0x0020, 0x0024, 4, 2, -1,
-			     -1, -1),
-	MUX_GATE_CLR_SET_UPD(CLK_INFRA_PCIE_GFMUX_TL_O_P3_SEL, "infra_pcie_gfmux_tl_o_p3_sel",
-			     infra_pcie_gfmux_tl_ck_o_p3_parents, 0x0028, 0x0020, 0x0024, 6, 2, -1,
-			     -1, -1),
+	MUX_CLR_SET(CLK_INFRA_PCIE_GFMUX_TL_O_P0_SEL, "infra_pcie_gfmux_tl_o_p0_sel",
+		    infra_pcie_gfmux_tl_ck_o_p0_parents, 0x0028, 0x0020, 0x0024, 0, 2),
+	MUX_CLR_SET(CLK_INFRA_PCIE_GFMUX_TL_O_P1_SEL, "infra_pcie_gfmux_tl_o_p1_sel",
+		    infra_pcie_gfmux_tl_ck_o_p1_parents, 0x0028, 0x0020, 0x0024, 2, 2),
+	MUX_CLR_SET(CLK_INFRA_PCIE_GFMUX_TL_O_P2_SEL, "infra_pcie_gfmux_tl_o_p2_sel",
+		    infra_pcie_gfmux_tl_ck_o_p2_parents, 0x0028, 0x0020, 0x0024, 4, 2),
+	MUX_CLR_SET(CLK_INFRA_PCIE_GFMUX_TL_O_P3_SEL, "infra_pcie_gfmux_tl_o_p3_sel",
+		    infra_pcie_gfmux_tl_ck_o_p3_parents, 0x0028, 0x0020, 0x0024, 6, 2),
 };
 
 static const struct mtk_gate_regs infra0_cg_regs = {
diff --git a/drivers/clk/mediatek/clk-mt8192.c b/drivers/clk/mediatek/clk-mt8192.c
index 50b43807c60cf..12c8890d922fe 100644
--- a/drivers/clk/mediatek/clk-mt8192.c
+++ b/drivers/clk/mediatek/clk-mt8192.c
@@ -579,8 +579,8 @@ static const struct mtk_mux top_mtk_muxes[] = {
 			     dsp7_parents, 0x050, 0x054, 0x058, 0, 3, 7, 0x004, 16),
 	MUX_GATE_CLR_SET_UPD(CLK_TOP_MFG_REF_SEL, "mfg_ref_sel",
 			     mfg_ref_parents, 0x050, 0x054, 0x058, 16, 2, 23, 0x004, 18),
-	MUX_CLR_SET_UPD(CLK_TOP_MFG_PLL_SEL, "mfg_pll_sel",
-			mfg_pll_parents, 0x050, 0x054, 0x058, 18, 1, -1, -1),
+	MUX_CLR_SET(CLK_TOP_MFG_PLL_SEL, "mfg_pll_sel", mfg_pll_parents,
+		   0x050, 0x054, 0x058, 18, 1),
 	MUX_GATE_CLR_SET_UPD(CLK_TOP_CAMTG_SEL, "camtg_sel",
 			     camtg_parents, 0x050, 0x054, 0x058, 24, 3, 31, 0x004, 19),
 	/* CLK_CFG_5 */
diff --git a/drivers/clk/mediatek/clk-mux.h b/drivers/clk/mediatek/clk-mux.h
index 151e56dcf8842..1a9baf306b4a7 100644
--- a/drivers/clk/mediatek/clk-mux.h
+++ b/drivers/clk/mediatek/clk-mux.h
@@ -126,6 +126,11 @@ extern const struct clk_ops mtk_mux_gate_hwv_fenc_clr_set_upd_ops;
 			0, _upd_ofs, _upd, CLK_SET_RATE_PARENT,		\
 			mtk_mux_clr_set_upd_ops)
 
+#define MUX_CLR_SET(_id, _name, _parents, _mux_ofs,			\
+		    _mux_set_ofs, _mux_clr_ofs, _shift, _width)		\
+		    MUX_CLR_SET_UPD(_id, _name, _parents, _mux_ofs,	\
+		    _mux_set_ofs, _mux_clr_ofs, _shift, _width, 0, -1)
+
 #define MUX_GATE_HWV_FENC_CLR_SET_UPD_FLAGS(_id, _name, _parents,			\
 				_mux_ofs, _mux_set_ofs, _mux_clr_ofs,			\
 				_hwv_sta_ofs, _hwv_set_ofs, _hwv_clr_ofs,		\
diff --git a/drivers/clk/microchip/Makefile b/drivers/clk/microchip/Makefile
index 13250e04e46cf..8e60bc1a03ae7 100644
--- a/drivers/clk/microchip/Makefile
+++ b/drivers/clk/microchip/Makefile
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_COMMON_CLK_PIC32) += clk-core.o
+obj-$(CONFIG_COMMON_CLK_PIC32) += clk-pic32.o
 obj-$(CONFIG_PIC32MZDA) += clk-pic32mzda.o
 obj-$(CONFIG_MCHP_CLK_MPFS) += clk-mpfs.o
 obj-$(CONFIG_MCHP_CLK_MPFS) += clk-mpfs-ccc.o
diff --git a/drivers/clk/microchip/clk-core.c b/drivers/clk/microchip/clk-core.c
deleted file mode 100644
index 692152b5094e0..0000000000000
--- a/drivers/clk/microchip/clk-core.c
+++ /dev/null
@@ -1,1019 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Purna Chandra Mandal,<purna.mandal@microchip.com>
- * Copyright (C) 2015 Microchip Technology Inc.  All rights reserved.
- */
-#include <linux/clk-provider.h>
-#include <linux/delay.h>
-#include <linux/device.h>
-#include <linux/interrupt.h>
-#include <linux/io.h>
-#include <linux/iopoll.h>
-#include <linux/platform_data/pic32.h>
-
-#include "clk-core.h"
-
-/* OSCCON Reg fields */
-#define OSC_CUR_MASK		0x07
-#define OSC_CUR_SHIFT		12
-#define OSC_NEW_MASK		0x07
-#define OSC_NEW_SHIFT		8
-#define OSC_SWEN		BIT(0)
-
-/* SPLLCON Reg fields */
-#define PLL_RANGE_MASK		0x07
-#define PLL_RANGE_SHIFT		0
-#define PLL_ICLK_MASK		0x01
-#define PLL_ICLK_SHIFT		7
-#define PLL_IDIV_MASK		0x07
-#define PLL_IDIV_SHIFT		8
-#define PLL_ODIV_MASK		0x07
-#define PLL_ODIV_SHIFT		24
-#define PLL_MULT_MASK		0x7F
-#define PLL_MULT_SHIFT		16
-#define PLL_MULT_MAX		128
-#define PLL_ODIV_MIN		1
-#define PLL_ODIV_MAX		5
-
-/* Peripheral Bus Clock Reg Fields */
-#define PB_DIV_MASK		0x7f
-#define PB_DIV_SHIFT		0
-#define PB_DIV_READY		BIT(11)
-#define PB_DIV_ENABLE		BIT(15)
-#define PB_DIV_MAX		128
-#define PB_DIV_MIN		0
-
-/* Reference Oscillator Control Reg fields */
-#define REFO_SEL_MASK		0x0f
-#define REFO_SEL_SHIFT		0
-#define REFO_ACTIVE		BIT(8)
-#define REFO_DIVSW_EN		BIT(9)
-#define REFO_OE			BIT(12)
-#define REFO_ON			BIT(15)
-#define REFO_DIV_SHIFT		16
-#define REFO_DIV_MASK		0x7fff
-
-/* Reference Oscillator Trim Register Fields */
-#define REFO_TRIM_REG		0x10
-#define REFO_TRIM_MASK		0x1ff
-#define REFO_TRIM_SHIFT		23
-#define REFO_TRIM_MAX		511
-
-/* Mux Slew Control Register fields */
-#define SLEW_BUSY		BIT(0)
-#define SLEW_DOWNEN		BIT(1)
-#define SLEW_UPEN		BIT(2)
-#define SLEW_DIV		0x07
-#define SLEW_DIV_SHIFT		8
-#define SLEW_SYSDIV		0x0f
-#define SLEW_SYSDIV_SHIFT	20
-
-/* Clock Poll Timeout */
-#define LOCK_TIMEOUT_US         USEC_PER_MSEC
-
-/* SoC specific clock needed during SPLL clock rate switch */
-static struct clk_hw *pic32_sclk_hw;
-
-/* add instruction pipeline delay while CPU clock is in-transition. */
-#define cpu_nop5()			\
-do {					\
-	__asm__ __volatile__("nop");	\
-	__asm__ __volatile__("nop");	\
-	__asm__ __volatile__("nop");	\
-	__asm__ __volatile__("nop");	\
-	__asm__ __volatile__("nop");	\
-} while (0)
-
-/* Perpheral bus clocks */
-struct pic32_periph_clk {
-	struct clk_hw hw;
-	void __iomem *ctrl_reg;
-	struct pic32_clk_common *core;
-};
-
-#define clkhw_to_pbclk(_hw)	container_of(_hw, struct pic32_periph_clk, hw)
-
-static int pbclk_is_enabled(struct clk_hw *hw)
-{
-	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
-
-	return readl(pb->ctrl_reg) & PB_DIV_ENABLE;
-}
-
-static int pbclk_enable(struct clk_hw *hw)
-{
-	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
-
-	writel(PB_DIV_ENABLE, PIC32_SET(pb->ctrl_reg));
-	return 0;
-}
-
-static void pbclk_disable(struct clk_hw *hw)
-{
-	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
-
-	writel(PB_DIV_ENABLE, PIC32_CLR(pb->ctrl_reg));
-}
-
-static unsigned long calc_best_divided_rate(unsigned long rate,
-					    unsigned long parent_rate,
-					    u32 divider_max,
-					    u32 divider_min)
-{
-	unsigned long divided_rate, divided_rate_down, best_rate;
-	unsigned long div, div_up;
-
-	/* eq. clk_rate = parent_rate / divider.
-	 *
-	 * Find best divider to produce closest of target divided rate.
-	 */
-	div = parent_rate / rate;
-	div = clamp_val(div, divider_min, divider_max);
-	div_up = clamp_val(div + 1, divider_min, divider_max);
-
-	divided_rate = parent_rate / div;
-	divided_rate_down = parent_rate / div_up;
-	if (abs(rate - divided_rate_down) < abs(rate - divided_rate))
-		best_rate = divided_rate_down;
-	else
-		best_rate = divided_rate;
-
-	return best_rate;
-}
-
-static inline u32 pbclk_read_pbdiv(struct pic32_periph_clk *pb)
-{
-	return ((readl(pb->ctrl_reg) >> PB_DIV_SHIFT) & PB_DIV_MASK) + 1;
-}
-
-static unsigned long pbclk_recalc_rate(struct clk_hw *hw,
-				       unsigned long parent_rate)
-{
-	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
-
-	return parent_rate / pbclk_read_pbdiv(pb);
-}
-
-static int pbclk_determine_rate(struct clk_hw *hw,
-				struct clk_rate_request *req)
-{
-	req->rate = calc_best_divided_rate(req->rate, req->best_parent_rate,
-					   PB_DIV_MAX, PB_DIV_MIN);
-
-	return 0;
-}
-
-static int pbclk_set_rate(struct clk_hw *hw, unsigned long rate,
-			  unsigned long parent_rate)
-{
-	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
-	unsigned long flags;
-	u32 v, div;
-	int err;
-
-	/* check & wait for DIV_READY */
-	err = readl_poll_timeout(pb->ctrl_reg, v, v & PB_DIV_READY,
-				 1, LOCK_TIMEOUT_US);
-	if (err)
-		return err;
-
-	/* calculate clkdiv and best rate */
-	div = DIV_ROUND_CLOSEST(parent_rate, rate);
-
-	spin_lock_irqsave(&pb->core->reg_lock, flags);
-
-	/* apply new div */
-	v = readl(pb->ctrl_reg);
-	v &= ~PB_DIV_MASK;
-	v |= (div - 1);
-
-	pic32_syskey_unlock();
-
-	writel(v, pb->ctrl_reg);
-
-	spin_unlock_irqrestore(&pb->core->reg_lock, flags);
-
-	/* wait again for DIV_READY */
-	err = readl_poll_timeout(pb->ctrl_reg, v, v & PB_DIV_READY,
-				 1, LOCK_TIMEOUT_US);
-	if (err)
-		return err;
-
-	/* confirm that new div is applied correctly */
-	return (pbclk_read_pbdiv(pb) == div) ? 0 : -EBUSY;
-}
-
-const struct clk_ops pic32_pbclk_ops = {
-	.enable		= pbclk_enable,
-	.disable	= pbclk_disable,
-	.is_enabled	= pbclk_is_enabled,
-	.recalc_rate	= pbclk_recalc_rate,
-	.determine_rate = pbclk_determine_rate,
-	.set_rate	= pbclk_set_rate,
-};
-
-struct clk *pic32_periph_clk_register(const struct pic32_periph_clk_data *desc,
-				      struct pic32_clk_common *core)
-{
-	struct pic32_periph_clk *pbclk;
-	struct clk *clk;
-
-	pbclk = devm_kzalloc(core->dev, sizeof(*pbclk), GFP_KERNEL);
-	if (!pbclk)
-		return ERR_PTR(-ENOMEM);
-
-	pbclk->hw.init = &desc->init_data;
-	pbclk->core = core;
-	pbclk->ctrl_reg = desc->ctrl_reg + core->iobase;
-
-	clk = devm_clk_register(core->dev, &pbclk->hw);
-	if (IS_ERR(clk)) {
-		dev_err(core->dev, "%s: clk_register() failed\n", __func__);
-		devm_kfree(core->dev, pbclk);
-	}
-
-	return clk;
-}
-
-/* Reference oscillator operations */
-struct pic32_ref_osc {
-	struct clk_hw hw;
-	void __iomem *ctrl_reg;
-	const u32 *parent_map;
-	struct pic32_clk_common *core;
-};
-
-#define clkhw_to_refosc(_hw)	container_of(_hw, struct pic32_ref_osc, hw)
-
-static int roclk_is_enabled(struct clk_hw *hw)
-{
-	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
-
-	return readl(refo->ctrl_reg) & REFO_ON;
-}
-
-static int roclk_enable(struct clk_hw *hw)
-{
-	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
-
-	writel(REFO_ON | REFO_OE, PIC32_SET(refo->ctrl_reg));
-	return 0;
-}
-
-static void roclk_disable(struct clk_hw *hw)
-{
-	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
-
-	writel(REFO_ON | REFO_OE, PIC32_CLR(refo->ctrl_reg));
-}
-
-static int roclk_init(struct clk_hw *hw)
-{
-	/* initialize clock in disabled state */
-	roclk_disable(hw);
-
-	return 0;
-}
-
-static u8 roclk_get_parent(struct clk_hw *hw)
-{
-	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
-	u32 v, i;
-
-	v = (readl(refo->ctrl_reg) >> REFO_SEL_SHIFT) & REFO_SEL_MASK;
-
-	if (refo->parent_map) {
-		for (i = 0; i < clk_hw_get_num_parents(hw); i++)
-			if (refo->parent_map[i] == v)
-				return i;
-	}
-
-	return v;
-}
-
-static unsigned long roclk_calc_rate(unsigned long parent_rate,
-				     u32 rodiv, u32 rotrim)
-{
-	u64 rate64;
-
-	/* fout = fin / [2 * {div + (trim / 512)}]
-	 *	= fin * 512 / [1024 * div + 2 * trim]
-	 *	= fin * 256 / (512 * div + trim)
-	 *	= (fin << 8) / ((div << 9) + trim)
-	 */
-	if (rotrim) {
-		rodiv = (rodiv << 9) + rotrim;
-		rate64 = parent_rate;
-		rate64 <<= 8;
-		do_div(rate64, rodiv);
-	} else if (rodiv) {
-		rate64 = parent_rate / (rodiv << 1);
-	} else {
-		rate64 = parent_rate;
-	}
-	return rate64;
-}
-
-static void roclk_calc_div_trim(unsigned long rate,
-				unsigned long parent_rate,
-				u32 *rodiv_p, u32 *rotrim_p)
-{
-	u32 div, rotrim, rodiv;
-	u64 frac;
-
-	/* Find integer approximation of floating-point arithmetic.
-	 *      fout = fin / [2 * {rodiv + (rotrim / 512)}] ... (1)
-	 * i.e. fout = fin / 2 * DIV
-	 *      whereas DIV = rodiv + (rotrim / 512)
-	 *
-	 * Since kernel does not perform floating-point arithmetic so
-	 * (rotrim/512) will be zero. And DIV & rodiv will result same.
-	 *
-	 * ie. fout = (fin * 256) / [(512 * rodiv) + rotrim]  ... from (1)
-	 * ie. rotrim = ((fin * 256) / fout) - (512 * DIV)
-	 */
-	if (parent_rate <= rate) {
-		div = 0;
-		frac = 0;
-		rodiv = 0;
-		rotrim = 0;
-	} else {
-		div = parent_rate / (rate << 1);
-		frac = parent_rate;
-		frac <<= 8;
-		do_div(frac, rate);
-		frac -= (u64)(div << 9);
-
-		rodiv = (div > REFO_DIV_MASK) ? REFO_DIV_MASK : div;
-		rotrim = (frac >= REFO_TRIM_MAX) ? REFO_TRIM_MAX : frac;
-	}
-
-	if (rodiv_p)
-		*rodiv_p = rodiv;
-
-	if (rotrim_p)
-		*rotrim_p = rotrim;
-}
-
-static unsigned long roclk_recalc_rate(struct clk_hw *hw,
-				       unsigned long parent_rate)
-{
-	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
-	u32 v, rodiv, rotrim;
-
-	/* get rodiv */
-	v = readl(refo->ctrl_reg);
-	rodiv = (v >> REFO_DIV_SHIFT) & REFO_DIV_MASK;
-
-	/* get trim */
-	v = readl(refo->ctrl_reg + REFO_TRIM_REG);
-	rotrim = (v >> REFO_TRIM_SHIFT) & REFO_TRIM_MASK;
-
-	return roclk_calc_rate(parent_rate, rodiv, rotrim);
-}
-
-static int roclk_determine_rate(struct clk_hw *hw,
-				struct clk_rate_request *req)
-{
-	struct clk_hw *parent_clk, *best_parent_clk = NULL;
-	unsigned int i, delta, best_delta = -1;
-	unsigned long parent_rate, best_parent_rate = 0;
-	unsigned long best = 0, nearest_rate;
-
-	/* find a parent which can generate nearest clkrate >= rate */
-	for (i = 0; i < clk_hw_get_num_parents(hw); i++) {
-		u32 rotrim, rodiv;
-
-		/* get parent */
-		parent_clk = clk_hw_get_parent_by_index(hw, i);
-		if (!parent_clk)
-			continue;
-
-		/* skip if parent runs slower than target rate */
-		parent_rate = clk_hw_get_rate(parent_clk);
-		if (req->rate > parent_rate)
-			continue;
-
-		/* calculate dividers for new rate */
-		roclk_calc_div_trim(req->rate, req->best_parent_rate, &rodiv, &rotrim);
-
-		/* caclulate new rate (rounding) based on new rodiv & rotrim */
-		nearest_rate = roclk_calc_rate(req->best_parent_rate, rodiv, rotrim);
-
-		delta = abs(nearest_rate - req->rate);
-		if ((nearest_rate >= req->rate) && (delta < best_delta)) {
-			best_parent_clk = parent_clk;
-			best_parent_rate = parent_rate;
-			best = nearest_rate;
-			best_delta = delta;
-
-			if (delta == 0)
-				break;
-		}
-	}
-
-	/* if no match found, retain old rate */
-	if (!best_parent_clk) {
-		pr_err("%s:%s, no parent found for rate %lu.\n",
-		       __func__, clk_hw_get_name(hw), req->rate);
-		return clk_hw_get_rate(hw);
-	}
-
-	pr_debug("%s,rate %lu, best_parent(%s, %lu), best %lu, delta %d\n",
-		 clk_hw_get_name(hw), req->rate,
-		 clk_hw_get_name(best_parent_clk), best_parent_rate,
-		 best, best_delta);
-
-	if (req->best_parent_rate)
-		req->best_parent_rate = best_parent_rate;
-
-	if (req->best_parent_hw)
-		req->best_parent_hw = best_parent_clk;
-
-	return best;
-}
-
-static int roclk_set_parent(struct clk_hw *hw, u8 index)
-{
-	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
-	unsigned long flags;
-	u32 v;
-	int err;
-
-	if (refo->parent_map)
-		index = refo->parent_map[index];
-
-	/* wait until ACTIVE bit is zero or timeout */
-	err = readl_poll_timeout(refo->ctrl_reg, v, !(v & REFO_ACTIVE),
-				 1, LOCK_TIMEOUT_US);
-	if (err) {
-		pr_err("%s: poll failed, clk active\n", clk_hw_get_name(hw));
-		return err;
-	}
-
-	spin_lock_irqsave(&refo->core->reg_lock, flags);
-
-	pic32_syskey_unlock();
-
-	/* calculate & apply new */
-	v = readl(refo->ctrl_reg);
-	v &= ~(REFO_SEL_MASK << REFO_SEL_SHIFT);
-	v |= index << REFO_SEL_SHIFT;
-
-	writel(v, refo->ctrl_reg);
-
-	spin_unlock_irqrestore(&refo->core->reg_lock, flags);
-
-	return 0;
-}
-
-static int roclk_set_rate_and_parent(struct clk_hw *hw,
-				     unsigned long rate,
-				     unsigned long parent_rate,
-				     u8 index)
-{
-	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
-	unsigned long flags;
-	u32 trim, rodiv, v;
-	int err;
-
-	/* calculate new rodiv & rotrim for new rate */
-	roclk_calc_div_trim(rate, parent_rate, &rodiv, &trim);
-
-	pr_debug("parent_rate = %lu, rate = %lu, div = %d, trim = %d\n",
-		 parent_rate, rate, rodiv, trim);
-
-	/* wait till source change is active */
-	err = readl_poll_timeout(refo->ctrl_reg, v,
-				 !(v & (REFO_ACTIVE | REFO_DIVSW_EN)),
-				 1, LOCK_TIMEOUT_US);
-	if (err) {
-		pr_err("%s: poll timedout, clock is still active\n", __func__);
-		return err;
-	}
-
-	spin_lock_irqsave(&refo->core->reg_lock, flags);
-	v = readl(refo->ctrl_reg);
-
-	pic32_syskey_unlock();
-
-	/* apply parent, if required */
-	if (refo->parent_map)
-		index = refo->parent_map[index];
-
-	v &= ~(REFO_SEL_MASK << REFO_SEL_SHIFT);
-	v |= index << REFO_SEL_SHIFT;
-
-	/* apply RODIV */
-	v &= ~(REFO_DIV_MASK << REFO_DIV_SHIFT);
-	v |= rodiv << REFO_DIV_SHIFT;
-	writel(v, refo->ctrl_reg);
-
-	/* apply ROTRIM */
-	v = readl(refo->ctrl_reg + REFO_TRIM_REG);
-	v &= ~(REFO_TRIM_MASK << REFO_TRIM_SHIFT);
-	v |= trim << REFO_TRIM_SHIFT;
-	writel(v, refo->ctrl_reg + REFO_TRIM_REG);
-
-	/* enable & activate divider switching */
-	writel(REFO_ON | REFO_DIVSW_EN, PIC32_SET(refo->ctrl_reg));
-
-	/* wait till divswen is in-progress */
-	err = readl_poll_timeout_atomic(refo->ctrl_reg, v, !(v & REFO_DIVSW_EN),
-					1, LOCK_TIMEOUT_US);
-	/* leave the clk gated as it was */
-	writel(REFO_ON, PIC32_CLR(refo->ctrl_reg));
-
-	spin_unlock_irqrestore(&refo->core->reg_lock, flags);
-
-	return err;
-}
-
-static int roclk_set_rate(struct clk_hw *hw, unsigned long rate,
-			  unsigned long parent_rate)
-{
-	u8 index = roclk_get_parent(hw);
-
-	return roclk_set_rate_and_parent(hw, rate, parent_rate, index);
-}
-
-const struct clk_ops pic32_roclk_ops = {
-	.enable			= roclk_enable,
-	.disable		= roclk_disable,
-	.is_enabled		= roclk_is_enabled,
-	.get_parent		= roclk_get_parent,
-	.set_parent		= roclk_set_parent,
-	.determine_rate		= roclk_determine_rate,
-	.recalc_rate		= roclk_recalc_rate,
-	.set_rate_and_parent	= roclk_set_rate_and_parent,
-	.set_rate		= roclk_set_rate,
-	.init			= roclk_init,
-};
-
-struct clk *pic32_refo_clk_register(const struct pic32_ref_osc_data *data,
-				    struct pic32_clk_common *core)
-{
-	struct pic32_ref_osc *refo;
-	struct clk *clk;
-
-	refo = devm_kzalloc(core->dev, sizeof(*refo), GFP_KERNEL);
-	if (!refo)
-		return ERR_PTR(-ENOMEM);
-
-	refo->core = core;
-	refo->hw.init = &data->init_data;
-	refo->ctrl_reg = data->ctrl_reg + core->iobase;
-	refo->parent_map = data->parent_map;
-
-	clk = devm_clk_register(core->dev, &refo->hw);
-	if (IS_ERR(clk))
-		dev_err(core->dev, "%s: clk_register() failed\n", __func__);
-
-	return clk;
-}
-
-struct pic32_sys_pll {
-	struct clk_hw hw;
-	void __iomem *ctrl_reg;
-	void __iomem *status_reg;
-	u32 lock_mask;
-	u32 idiv; /* PLL iclk divider, treated fixed */
-	struct pic32_clk_common *core;
-};
-
-#define clkhw_to_spll(_hw)	container_of(_hw, struct pic32_sys_pll, hw)
-
-static inline u32 spll_odiv_to_divider(u32 odiv)
-{
-	odiv = clamp_val(odiv, PLL_ODIV_MIN, PLL_ODIV_MAX);
-
-	return 1 << odiv;
-}
-
-static unsigned long spll_calc_mult_div(struct pic32_sys_pll *pll,
-					unsigned long rate,
-					unsigned long parent_rate,
-					u32 *mult_p, u32 *odiv_p)
-{
-	u32 mul, div, best_mul = 1, best_div = 1;
-	unsigned long new_rate, best_rate = rate;
-	unsigned int best_delta = -1, delta, match_found = 0;
-	u64 rate64;
-
-	parent_rate /= pll->idiv;
-
-	for (mul = 1; mul <= PLL_MULT_MAX; mul++) {
-		for (div = PLL_ODIV_MIN; div <= PLL_ODIV_MAX; div++) {
-			rate64 = parent_rate;
-			rate64 *= mul;
-			do_div(rate64, 1 << div);
-			new_rate = rate64;
-			delta = abs(rate - new_rate);
-			if ((new_rate >= rate) && (delta < best_delta)) {
-				best_delta = delta;
-				best_rate = new_rate;
-				best_mul = mul;
-				best_div = div;
-				match_found = 1;
-			}
-		}
-	}
-
-	if (!match_found) {
-		pr_warn("spll: no match found\n");
-		return 0;
-	}
-
-	pr_debug("rate %lu, par_rate %lu/mult %u, div %u, best_rate %lu\n",
-		 rate, parent_rate, best_mul, best_div, best_rate);
-
-	if (mult_p)
-		*mult_p = best_mul - 1;
-
-	if (odiv_p)
-		*odiv_p = best_div;
-
-	return best_rate;
-}
-
-static unsigned long spll_clk_recalc_rate(struct clk_hw *hw,
-					  unsigned long parent_rate)
-{
-	struct pic32_sys_pll *pll = clkhw_to_spll(hw);
-	unsigned long pll_in_rate;
-	u32 mult, odiv, div, v;
-	u64 rate64;
-
-	v = readl(pll->ctrl_reg);
-	odiv = ((v >> PLL_ODIV_SHIFT) & PLL_ODIV_MASK);
-	mult = ((v >> PLL_MULT_SHIFT) & PLL_MULT_MASK) + 1;
-	div = spll_odiv_to_divider(odiv);
-
-	/* pll_in_rate = parent_rate / idiv
-	 * pll_out_rate = pll_in_rate * mult / div;
-	 */
-	pll_in_rate = parent_rate / pll->idiv;
-	rate64 = pll_in_rate;
-	rate64 *= mult;
-	do_div(rate64, div);
-
-	return rate64;
-}
-
-static int spll_clk_determine_rate(struct clk_hw *hw,
-				   struct clk_rate_request *req)
-{
-	struct pic32_sys_pll *pll = clkhw_to_spll(hw);
-
-	req->rate = spll_calc_mult_div(pll, req->rate, req->best_parent_rate,
-				       NULL, NULL);
-
-	return 0;
-}
-
-static int spll_clk_set_rate(struct clk_hw *hw, unsigned long rate,
-			     unsigned long parent_rate)
-{
-	struct pic32_sys_pll *pll = clkhw_to_spll(hw);
-	unsigned long ret, flags;
-	u32 mult, odiv, v;
-	int err;
-
-	ret = spll_calc_mult_div(pll, rate, parent_rate, &mult, &odiv);
-	if (!ret)
-		return -EINVAL;
-
-	/*
-	 * We can't change SPLL counters when it is in-active use
-	 * by SYSCLK. So check before applying new counters/rate.
-	 */
-
-	/* Is spll_clk active parent of sys_clk ? */
-	if (unlikely(clk_hw_get_parent(pic32_sclk_hw) == hw)) {
-		pr_err("%s: failed, clk in-use\n", __func__);
-		return -EBUSY;
-	}
-
-	spin_lock_irqsave(&pll->core->reg_lock, flags);
-
-	/* apply new multiplier & divisor */
-	v = readl(pll->ctrl_reg);
-	v &= ~(PLL_MULT_MASK << PLL_MULT_SHIFT);
-	v &= ~(PLL_ODIV_MASK << PLL_ODIV_SHIFT);
-	v |= (mult << PLL_MULT_SHIFT) | (odiv << PLL_ODIV_SHIFT);
-
-	/* sys unlock before write */
-	pic32_syskey_unlock();
-
-	writel(v, pll->ctrl_reg);
-	cpu_relax();
-
-	/* insert few nops (5-stage) to ensure CPU does not hang */
-	cpu_nop5();
-	cpu_nop5();
-
-	/* Wait until PLL is locked (maximum 100 usecs). */
-	err = readl_poll_timeout_atomic(pll->status_reg, v,
-					v & pll->lock_mask, 1, 100);
-	spin_unlock_irqrestore(&pll->core->reg_lock, flags);
-
-	return err;
-}
-
-/* SPLL clock operation */
-const struct clk_ops pic32_spll_ops = {
-	.recalc_rate	= spll_clk_recalc_rate,
-	.determine_rate = spll_clk_determine_rate,
-	.set_rate	= spll_clk_set_rate,
-};
-
-struct clk *pic32_spll_clk_register(const struct pic32_sys_pll_data *data,
-				    struct pic32_clk_common *core)
-{
-	struct pic32_sys_pll *spll;
-	struct clk *clk;
-
-	spll = devm_kzalloc(core->dev, sizeof(*spll), GFP_KERNEL);
-	if (!spll)
-		return ERR_PTR(-ENOMEM);
-
-	spll->core = core;
-	spll->hw.init = &data->init_data;
-	spll->ctrl_reg = data->ctrl_reg + core->iobase;
-	spll->status_reg = data->status_reg + core->iobase;
-	spll->lock_mask = data->lock_mask;
-
-	/* cache PLL idiv; PLL driver uses it as constant.*/
-	spll->idiv = (readl(spll->ctrl_reg) >> PLL_IDIV_SHIFT) & PLL_IDIV_MASK;
-	spll->idiv += 1;
-
-	clk = devm_clk_register(core->dev, &spll->hw);
-	if (IS_ERR(clk))
-		dev_err(core->dev, "sys_pll: clk_register() failed\n");
-
-	return clk;
-}
-
-/* System mux clock(aka SCLK) */
-
-struct pic32_sys_clk {
-	struct clk_hw hw;
-	void __iomem *mux_reg;
-	void __iomem *slew_reg;
-	u32 slew_div;
-	const u32 *parent_map;
-	struct pic32_clk_common *core;
-};
-
-#define clkhw_to_sys_clk(_hw)	container_of(_hw, struct pic32_sys_clk, hw)
-
-static unsigned long sclk_get_rate(struct clk_hw *hw, unsigned long parent_rate)
-{
-	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
-	u32 div;
-
-	div = (readl(sclk->slew_reg) >> SLEW_SYSDIV_SHIFT) & SLEW_SYSDIV;
-	div += 1; /* sys-div to divider */
-
-	return parent_rate / div;
-}
-
-static int sclk_set_rate(struct clk_hw *hw,
-			 unsigned long rate, unsigned long parent_rate)
-{
-	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
-	unsigned long flags;
-	u32 v, div;
-	int err;
-
-	div = parent_rate / rate;
-
-	spin_lock_irqsave(&sclk->core->reg_lock, flags);
-
-	/* apply new div */
-	v = readl(sclk->slew_reg);
-	v &= ~(SLEW_SYSDIV << SLEW_SYSDIV_SHIFT);
-	v |= (div - 1) << SLEW_SYSDIV_SHIFT;
-
-	pic32_syskey_unlock();
-
-	writel(v, sclk->slew_reg);
-
-	/* wait until BUSY is cleared */
-	err = readl_poll_timeout_atomic(sclk->slew_reg, v,
-					!(v & SLEW_BUSY), 1, LOCK_TIMEOUT_US);
-
-	spin_unlock_irqrestore(&sclk->core->reg_lock, flags);
-
-	return err;
-}
-
-static u8 sclk_get_parent(struct clk_hw *hw)
-{
-	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
-	u32 i, v;
-
-	v = (readl(sclk->mux_reg) >> OSC_CUR_SHIFT) & OSC_CUR_MASK;
-
-	if (sclk->parent_map) {
-		for (i = 0; i < clk_hw_get_num_parents(hw); i++)
-			if (sclk->parent_map[i] == v)
-				return i;
-	}
-
-	return v;
-}
-
-static int sclk_set_parent(struct clk_hw *hw, u8 index)
-{
-	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
-	unsigned long flags;
-	u32 nosc, cosc, v;
-	int err;
-
-	spin_lock_irqsave(&sclk->core->reg_lock, flags);
-
-	/* find new_osc */
-	nosc = sclk->parent_map ? sclk->parent_map[index] : index;
-
-	/* set new parent */
-	v = readl(sclk->mux_reg);
-	v &= ~(OSC_NEW_MASK << OSC_NEW_SHIFT);
-	v |= nosc << OSC_NEW_SHIFT;
-
-	pic32_syskey_unlock();
-
-	writel(v, sclk->mux_reg);
-
-	/* initate switch */
-	writel(OSC_SWEN, PIC32_SET(sclk->mux_reg));
-	cpu_relax();
-
-	/* add nop to flush pipeline (as cpu_clk is in-flux) */
-	cpu_nop5();
-
-	/* wait for SWEN bit to clear */
-	err = readl_poll_timeout_atomic(sclk->slew_reg, v,
-					!(v & OSC_SWEN), 1, LOCK_TIMEOUT_US);
-
-	spin_unlock_irqrestore(&sclk->core->reg_lock, flags);
-
-	/*
-	 * SCLK clock-switching logic might reject a clock switching request
-	 * if pre-requisites (like new clk_src not present or unstable) are
-	 * not met.
-	 * So confirm before claiming success.
-	 */
-	cosc = (readl(sclk->mux_reg) >> OSC_CUR_SHIFT) & OSC_CUR_MASK;
-	if (cosc != nosc) {
-		pr_err("%s: err, failed to set_parent() to %d, current %d\n",
-		       clk_hw_get_name(hw), nosc, cosc);
-		err = -EBUSY;
-	}
-
-	return err;
-}
-
-static int sclk_init(struct clk_hw *hw)
-{
-	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
-	unsigned long flags;
-	u32 v;
-
-	/* Maintain reference to this clk, required in spll_clk_set_rate() */
-	pic32_sclk_hw = hw;
-
-	/* apply slew divider on both up and down scaling */
-	if (sclk->slew_div) {
-		spin_lock_irqsave(&sclk->core->reg_lock, flags);
-		v = readl(sclk->slew_reg);
-		v &= ~(SLEW_DIV << SLEW_DIV_SHIFT);
-		v |= sclk->slew_div << SLEW_DIV_SHIFT;
-		v |= SLEW_DOWNEN | SLEW_UPEN;
-		writel(v, sclk->slew_reg);
-		spin_unlock_irqrestore(&sclk->core->reg_lock, flags);
-	}
-
-	return 0;
-}
-
-/* sclk with post-divider */
-const struct clk_ops pic32_sclk_ops = {
-	.get_parent	= sclk_get_parent,
-	.set_parent	= sclk_set_parent,
-	.set_rate	= sclk_set_rate,
-	.recalc_rate	= sclk_get_rate,
-	.init		= sclk_init,
-	.determine_rate = __clk_mux_determine_rate,
-};
-
-/* sclk with no slew and no post-divider */
-const struct clk_ops pic32_sclk_no_div_ops = {
-	.get_parent	= sclk_get_parent,
-	.set_parent	= sclk_set_parent,
-	.init		= sclk_init,
-	.determine_rate = __clk_mux_determine_rate,
-};
-
-struct clk *pic32_sys_clk_register(const struct pic32_sys_clk_data *data,
-				   struct pic32_clk_common *core)
-{
-	struct pic32_sys_clk *sclk;
-	struct clk *clk;
-
-	sclk = devm_kzalloc(core->dev, sizeof(*sclk), GFP_KERNEL);
-	if (!sclk)
-		return ERR_PTR(-ENOMEM);
-
-	sclk->core = core;
-	sclk->hw.init = &data->init_data;
-	sclk->mux_reg = data->mux_reg + core->iobase;
-	sclk->slew_reg = data->slew_reg + core->iobase;
-	sclk->slew_div = data->slew_div;
-	sclk->parent_map = data->parent_map;
-
-	clk = devm_clk_register(core->dev, &sclk->hw);
-	if (IS_ERR(clk))
-		dev_err(core->dev, "%s: clk register failed\n", __func__);
-
-	return clk;
-}
-
-/* secondary oscillator */
-struct pic32_sec_osc {
-	struct clk_hw hw;
-	void __iomem *enable_reg;
-	void __iomem *status_reg;
-	u32 enable_mask;
-	u32 status_mask;
-	unsigned long fixed_rate;
-	struct pic32_clk_common *core;
-};
-
-#define clkhw_to_sosc(_hw)	container_of(_hw, struct pic32_sec_osc, hw)
-static int sosc_clk_enable(struct clk_hw *hw)
-{
-	struct pic32_sec_osc *sosc = clkhw_to_sosc(hw);
-	u32 v;
-
-	/* enable SOSC */
-	pic32_syskey_unlock();
-	writel(sosc->enable_mask, PIC32_SET(sosc->enable_reg));
-
-	/* wait till warm-up period expires or ready-status is updated */
-	return readl_poll_timeout_atomic(sosc->status_reg, v,
-					 v & sosc->status_mask, 1, 100);
-}
-
-static void sosc_clk_disable(struct clk_hw *hw)
-{
-	struct pic32_sec_osc *sosc = clkhw_to_sosc(hw);
-
-	pic32_syskey_unlock();
-	writel(sosc->enable_mask, PIC32_CLR(sosc->enable_reg));
-}
-
-static int sosc_clk_is_enabled(struct clk_hw *hw)
-{
-	struct pic32_sec_osc *sosc = clkhw_to_sosc(hw);
-	u32 enabled, ready;
-
-	/* check enabled and ready status */
-	enabled = readl(sosc->enable_reg) & sosc->enable_mask;
-	ready = readl(sosc->status_reg) & sosc->status_mask;
-
-	return enabled && ready;
-}
-
-static unsigned long sosc_clk_calc_rate(struct clk_hw *hw,
-					unsigned long parent_rate)
-{
-	return clkhw_to_sosc(hw)->fixed_rate;
-}
-
-const struct clk_ops pic32_sosc_ops = {
-	.enable = sosc_clk_enable,
-	.disable = sosc_clk_disable,
-	.is_enabled = sosc_clk_is_enabled,
-	.recalc_rate = sosc_clk_calc_rate,
-};
-
-struct clk *pic32_sosc_clk_register(const struct pic32_sec_osc_data *data,
-				    struct pic32_clk_common *core)
-{
-	struct pic32_sec_osc *sosc;
-
-	sosc = devm_kzalloc(core->dev, sizeof(*sosc), GFP_KERNEL);
-	if (!sosc)
-		return ERR_PTR(-ENOMEM);
-
-	sosc->core = core;
-	sosc->hw.init = &data->init_data;
-	sosc->fixed_rate = data->fixed_rate;
-	sosc->enable_mask = data->enable_mask;
-	sosc->status_mask = data->status_mask;
-	sosc->enable_reg = data->enable_reg + core->iobase;
-	sosc->status_reg = data->status_reg + core->iobase;
-
-	return devm_clk_register(core->dev, &sosc->hw);
-}
diff --git a/drivers/clk/microchip/clk-core.h b/drivers/clk/microchip/clk-core.h
deleted file mode 100644
index acb81739a16f8..0000000000000
--- a/drivers/clk/microchip/clk-core.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Purna Chandra Mandal,<purna.mandal@microchip.com>
- * Copyright (C) 2015 Microchip Technology Inc.  All rights reserved.
- */
-#ifndef __MICROCHIP_CLK_PIC32_H_
-#define __MICROCHIP_CLK_PIC32_H_
-
-#include <linux/clk-provider.h>
-
-/* PIC32 clock data */
-struct pic32_clk_common {
-	struct device *dev;
-	void __iomem *iobase;
-	spinlock_t reg_lock; /* clock lock */
-};
-
-/* System PLL clock */
-struct pic32_sys_pll_data {
-	struct clk_init_data init_data;
-	const u32 ctrl_reg;
-	const u32 status_reg;
-	const u32 lock_mask;
-};
-
-/* System clock */
-struct pic32_sys_clk_data {
-	struct clk_init_data init_data;
-	const u32 mux_reg;
-	const u32 slew_reg;
-	const u32 *parent_map;
-	const u32 slew_div;
-};
-
-/* Reference Oscillator clock */
-struct pic32_ref_osc_data {
-	struct clk_init_data init_data;
-	const u32 ctrl_reg;
-	const u32 *parent_map;
-};
-
-/* Peripheral Bus clock */
-struct pic32_periph_clk_data {
-	struct clk_init_data init_data;
-	const u32 ctrl_reg;
-};
-
-/* External Secondary Oscillator clock  */
-struct pic32_sec_osc_data {
-	struct clk_init_data init_data;
-	const u32 enable_reg;
-	const u32 status_reg;
-	const u32 enable_mask;
-	const u32 status_mask;
-	const unsigned long fixed_rate;
-};
-
-extern const struct clk_ops pic32_pbclk_ops;
-extern const struct clk_ops pic32_sclk_ops;
-extern const struct clk_ops pic32_sclk_no_div_ops;
-extern const struct clk_ops pic32_spll_ops;
-extern const struct clk_ops pic32_roclk_ops;
-extern const struct clk_ops pic32_sosc_ops;
-
-struct clk *pic32_periph_clk_register(const struct pic32_periph_clk_data *data,
-				      struct pic32_clk_common *core);
-struct clk *pic32_refo_clk_register(const struct pic32_ref_osc_data *data,
-				    struct pic32_clk_common *core);
-struct clk *pic32_sys_clk_register(const struct pic32_sys_clk_data *data,
-				   struct pic32_clk_common *core);
-struct clk *pic32_spll_clk_register(const struct pic32_sys_pll_data *data,
-				    struct pic32_clk_common *core);
-struct clk *pic32_sosc_clk_register(const struct pic32_sec_osc_data *data,
-				    struct pic32_clk_common *core);
-
-#endif /* __MICROCHIP_CLK_PIC32_H_*/
diff --git a/drivers/clk/microchip/clk-pic32.c b/drivers/clk/microchip/clk-pic32.c
new file mode 100644
index 0000000000000..9d128fba2cdea
--- /dev/null
+++ b/drivers/clk/microchip/clk-pic32.c
@@ -0,0 +1,1019 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Purna Chandra Mandal,<purna.mandal@microchip.com>
+ * Copyright (C) 2015 Microchip Technology Inc.  All rights reserved.
+ */
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/platform_data/pic32.h>
+
+#include "clk-pic32.h"
+
+/* OSCCON Reg fields */
+#define OSC_CUR_MASK		0x07
+#define OSC_CUR_SHIFT		12
+#define OSC_NEW_MASK		0x07
+#define OSC_NEW_SHIFT		8
+#define OSC_SWEN		BIT(0)
+
+/* SPLLCON Reg fields */
+#define PLL_RANGE_MASK		0x07
+#define PLL_RANGE_SHIFT		0
+#define PLL_ICLK_MASK		0x01
+#define PLL_ICLK_SHIFT		7
+#define PLL_IDIV_MASK		0x07
+#define PLL_IDIV_SHIFT		8
+#define PLL_ODIV_MASK		0x07
+#define PLL_ODIV_SHIFT		24
+#define PLL_MULT_MASK		0x7F
+#define PLL_MULT_SHIFT		16
+#define PLL_MULT_MAX		128
+#define PLL_ODIV_MIN		1
+#define PLL_ODIV_MAX		5
+
+/* Peripheral Bus Clock Reg Fields */
+#define PB_DIV_MASK		0x7f
+#define PB_DIV_SHIFT		0
+#define PB_DIV_READY		BIT(11)
+#define PB_DIV_ENABLE		BIT(15)
+#define PB_DIV_MAX		128
+#define PB_DIV_MIN		0
+
+/* Reference Oscillator Control Reg fields */
+#define REFO_SEL_MASK		0x0f
+#define REFO_SEL_SHIFT		0
+#define REFO_ACTIVE		BIT(8)
+#define REFO_DIVSW_EN		BIT(9)
+#define REFO_OE			BIT(12)
+#define REFO_ON			BIT(15)
+#define REFO_DIV_SHIFT		16
+#define REFO_DIV_MASK		0x7fff
+
+/* Reference Oscillator Trim Register Fields */
+#define REFO_TRIM_REG		0x10
+#define REFO_TRIM_MASK		0x1ff
+#define REFO_TRIM_SHIFT		23
+#define REFO_TRIM_MAX		511
+
+/* Mux Slew Control Register fields */
+#define SLEW_BUSY		BIT(0)
+#define SLEW_DOWNEN		BIT(1)
+#define SLEW_UPEN		BIT(2)
+#define SLEW_DIV		0x07
+#define SLEW_DIV_SHIFT		8
+#define SLEW_SYSDIV		0x0f
+#define SLEW_SYSDIV_SHIFT	20
+
+/* Clock Poll Timeout */
+#define LOCK_TIMEOUT_US         USEC_PER_MSEC
+
+/* SoC specific clock needed during SPLL clock rate switch */
+static struct clk_hw *pic32_sclk_hw;
+
+/* add instruction pipeline delay while CPU clock is in-transition. */
+#define cpu_nop5()			\
+do {					\
+	__asm__ __volatile__("nop");	\
+	__asm__ __volatile__("nop");	\
+	__asm__ __volatile__("nop");	\
+	__asm__ __volatile__("nop");	\
+	__asm__ __volatile__("nop");	\
+} while (0)
+
+/* Perpheral bus clocks */
+struct pic32_periph_clk {
+	struct clk_hw hw;
+	void __iomem *ctrl_reg;
+	struct pic32_clk_common *core;
+};
+
+#define clkhw_to_pbclk(_hw)	container_of(_hw, struct pic32_periph_clk, hw)
+
+static int pbclk_is_enabled(struct clk_hw *hw)
+{
+	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
+
+	return readl(pb->ctrl_reg) & PB_DIV_ENABLE;
+}
+
+static int pbclk_enable(struct clk_hw *hw)
+{
+	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
+
+	writel(PB_DIV_ENABLE, PIC32_SET(pb->ctrl_reg));
+	return 0;
+}
+
+static void pbclk_disable(struct clk_hw *hw)
+{
+	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
+
+	writel(PB_DIV_ENABLE, PIC32_CLR(pb->ctrl_reg));
+}
+
+static unsigned long calc_best_divided_rate(unsigned long rate,
+					    unsigned long parent_rate,
+					    u32 divider_max,
+					    u32 divider_min)
+{
+	unsigned long divided_rate, divided_rate_down, best_rate;
+	unsigned long div, div_up;
+
+	/* eq. clk_rate = parent_rate / divider.
+	 *
+	 * Find best divider to produce closest of target divided rate.
+	 */
+	div = parent_rate / rate;
+	div = clamp_val(div, divider_min, divider_max);
+	div_up = clamp_val(div + 1, divider_min, divider_max);
+
+	divided_rate = parent_rate / div;
+	divided_rate_down = parent_rate / div_up;
+	if (abs(rate - divided_rate_down) < abs(rate - divided_rate))
+		best_rate = divided_rate_down;
+	else
+		best_rate = divided_rate;
+
+	return best_rate;
+}
+
+static inline u32 pbclk_read_pbdiv(struct pic32_periph_clk *pb)
+{
+	return ((readl(pb->ctrl_reg) >> PB_DIV_SHIFT) & PB_DIV_MASK) + 1;
+}
+
+static unsigned long pbclk_recalc_rate(struct clk_hw *hw,
+				       unsigned long parent_rate)
+{
+	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
+
+	return parent_rate / pbclk_read_pbdiv(pb);
+}
+
+static int pbclk_determine_rate(struct clk_hw *hw,
+				struct clk_rate_request *req)
+{
+	req->rate = calc_best_divided_rate(req->rate, req->best_parent_rate,
+					   PB_DIV_MAX, PB_DIV_MIN);
+
+	return 0;
+}
+
+static int pbclk_set_rate(struct clk_hw *hw, unsigned long rate,
+			  unsigned long parent_rate)
+{
+	struct pic32_periph_clk *pb = clkhw_to_pbclk(hw);
+	unsigned long flags;
+	u32 v, div;
+	int err;
+
+	/* check & wait for DIV_READY */
+	err = readl_poll_timeout(pb->ctrl_reg, v, v & PB_DIV_READY,
+				 1, LOCK_TIMEOUT_US);
+	if (err)
+		return err;
+
+	/* calculate clkdiv and best rate */
+	div = DIV_ROUND_CLOSEST(parent_rate, rate);
+
+	spin_lock_irqsave(&pb->core->reg_lock, flags);
+
+	/* apply new div */
+	v = readl(pb->ctrl_reg);
+	v &= ~PB_DIV_MASK;
+	v |= (div - 1);
+
+	pic32_syskey_unlock();
+
+	writel(v, pb->ctrl_reg);
+
+	spin_unlock_irqrestore(&pb->core->reg_lock, flags);
+
+	/* wait again for DIV_READY */
+	err = readl_poll_timeout(pb->ctrl_reg, v, v & PB_DIV_READY,
+				 1, LOCK_TIMEOUT_US);
+	if (err)
+		return err;
+
+	/* confirm that new div is applied correctly */
+	return (pbclk_read_pbdiv(pb) == div) ? 0 : -EBUSY;
+}
+
+const struct clk_ops pic32_pbclk_ops = {
+	.enable		= pbclk_enable,
+	.disable	= pbclk_disable,
+	.is_enabled	= pbclk_is_enabled,
+	.recalc_rate	= pbclk_recalc_rate,
+	.determine_rate = pbclk_determine_rate,
+	.set_rate	= pbclk_set_rate,
+};
+
+struct clk *pic32_periph_clk_register(const struct pic32_periph_clk_data *desc,
+				      struct pic32_clk_common *core)
+{
+	struct pic32_periph_clk *pbclk;
+	struct clk *clk;
+
+	pbclk = devm_kzalloc(core->dev, sizeof(*pbclk), GFP_KERNEL);
+	if (!pbclk)
+		return ERR_PTR(-ENOMEM);
+
+	pbclk->hw.init = &desc->init_data;
+	pbclk->core = core;
+	pbclk->ctrl_reg = desc->ctrl_reg + core->iobase;
+
+	clk = devm_clk_register(core->dev, &pbclk->hw);
+	if (IS_ERR(clk)) {
+		dev_err(core->dev, "%s: clk_register() failed\n", __func__);
+		devm_kfree(core->dev, pbclk);
+	}
+
+	return clk;
+}
+
+/* Reference oscillator operations */
+struct pic32_ref_osc {
+	struct clk_hw hw;
+	void __iomem *ctrl_reg;
+	const u32 *parent_map;
+	struct pic32_clk_common *core;
+};
+
+#define clkhw_to_refosc(_hw)	container_of(_hw, struct pic32_ref_osc, hw)
+
+static int roclk_is_enabled(struct clk_hw *hw)
+{
+	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
+
+	return readl(refo->ctrl_reg) & REFO_ON;
+}
+
+static int roclk_enable(struct clk_hw *hw)
+{
+	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
+
+	writel(REFO_ON | REFO_OE, PIC32_SET(refo->ctrl_reg));
+	return 0;
+}
+
+static void roclk_disable(struct clk_hw *hw)
+{
+	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
+
+	writel(REFO_ON | REFO_OE, PIC32_CLR(refo->ctrl_reg));
+}
+
+static int roclk_init(struct clk_hw *hw)
+{
+	/* initialize clock in disabled state */
+	roclk_disable(hw);
+
+	return 0;
+}
+
+static u8 roclk_get_parent(struct clk_hw *hw)
+{
+	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
+	u32 v, i;
+
+	v = (readl(refo->ctrl_reg) >> REFO_SEL_SHIFT) & REFO_SEL_MASK;
+
+	if (refo->parent_map) {
+		for (i = 0; i < clk_hw_get_num_parents(hw); i++)
+			if (refo->parent_map[i] == v)
+				return i;
+	}
+
+	return v;
+}
+
+static unsigned long roclk_calc_rate(unsigned long parent_rate,
+				     u32 rodiv, u32 rotrim)
+{
+	u64 rate64;
+
+	/* fout = fin / [2 * {div + (trim / 512)}]
+	 *	= fin * 512 / [1024 * div + 2 * trim]
+	 *	= fin * 256 / (512 * div + trim)
+	 *	= (fin << 8) / ((div << 9) + trim)
+	 */
+	if (rotrim) {
+		rodiv = (rodiv << 9) + rotrim;
+		rate64 = parent_rate;
+		rate64 <<= 8;
+		do_div(rate64, rodiv);
+	} else if (rodiv) {
+		rate64 = parent_rate / (rodiv << 1);
+	} else {
+		rate64 = parent_rate;
+	}
+	return rate64;
+}
+
+static void roclk_calc_div_trim(unsigned long rate,
+				unsigned long parent_rate,
+				u32 *rodiv_p, u32 *rotrim_p)
+{
+	u32 div, rotrim, rodiv;
+	u64 frac;
+
+	/* Find integer approximation of floating-point arithmetic.
+	 *      fout = fin / [2 * {rodiv + (rotrim / 512)}] ... (1)
+	 * i.e. fout = fin / 2 * DIV
+	 *      whereas DIV = rodiv + (rotrim / 512)
+	 *
+	 * Since kernel does not perform floating-point arithmetic so
+	 * (rotrim/512) will be zero. And DIV & rodiv will result same.
+	 *
+	 * ie. fout = (fin * 256) / [(512 * rodiv) + rotrim]  ... from (1)
+	 * ie. rotrim = ((fin * 256) / fout) - (512 * DIV)
+	 */
+	if (parent_rate <= rate) {
+		div = 0;
+		frac = 0;
+		rodiv = 0;
+		rotrim = 0;
+	} else {
+		div = parent_rate / (rate << 1);
+		frac = parent_rate;
+		frac <<= 8;
+		do_div(frac, rate);
+		frac -= (u64)(div << 9);
+
+		rodiv = (div > REFO_DIV_MASK) ? REFO_DIV_MASK : div;
+		rotrim = (frac >= REFO_TRIM_MAX) ? REFO_TRIM_MAX : frac;
+	}
+
+	if (rodiv_p)
+		*rodiv_p = rodiv;
+
+	if (rotrim_p)
+		*rotrim_p = rotrim;
+}
+
+static unsigned long roclk_recalc_rate(struct clk_hw *hw,
+				       unsigned long parent_rate)
+{
+	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
+	u32 v, rodiv, rotrim;
+
+	/* get rodiv */
+	v = readl(refo->ctrl_reg);
+	rodiv = (v >> REFO_DIV_SHIFT) & REFO_DIV_MASK;
+
+	/* get trim */
+	v = readl(refo->ctrl_reg + REFO_TRIM_REG);
+	rotrim = (v >> REFO_TRIM_SHIFT) & REFO_TRIM_MASK;
+
+	return roclk_calc_rate(parent_rate, rodiv, rotrim);
+}
+
+static int roclk_determine_rate(struct clk_hw *hw,
+				struct clk_rate_request *req)
+{
+	struct clk_hw *parent_clk, *best_parent_clk = NULL;
+	unsigned int i, delta, best_delta = -1;
+	unsigned long parent_rate, best_parent_rate = 0;
+	unsigned long best = 0, nearest_rate;
+
+	/* find a parent which can generate nearest clkrate >= rate */
+	for (i = 0; i < clk_hw_get_num_parents(hw); i++) {
+		u32 rotrim, rodiv;
+
+		/* get parent */
+		parent_clk = clk_hw_get_parent_by_index(hw, i);
+		if (!parent_clk)
+			continue;
+
+		/* skip if parent runs slower than target rate */
+		parent_rate = clk_hw_get_rate(parent_clk);
+		if (req->rate > parent_rate)
+			continue;
+
+		/* calculate dividers for new rate */
+		roclk_calc_div_trim(req->rate, req->best_parent_rate, &rodiv, &rotrim);
+
+		/* caclulate new rate (rounding) based on new rodiv & rotrim */
+		nearest_rate = roclk_calc_rate(req->best_parent_rate, rodiv, rotrim);
+
+		delta = abs(nearest_rate - req->rate);
+		if ((nearest_rate >= req->rate) && (delta < best_delta)) {
+			best_parent_clk = parent_clk;
+			best_parent_rate = parent_rate;
+			best = nearest_rate;
+			best_delta = delta;
+
+			if (delta == 0)
+				break;
+		}
+	}
+
+	/* if no match found, retain old rate */
+	if (!best_parent_clk) {
+		pr_err("%s:%s, no parent found for rate %lu.\n",
+		       __func__, clk_hw_get_name(hw), req->rate);
+		return clk_hw_get_rate(hw);
+	}
+
+	pr_debug("%s,rate %lu, best_parent(%s, %lu), best %lu, delta %d\n",
+		 clk_hw_get_name(hw), req->rate,
+		 clk_hw_get_name(best_parent_clk), best_parent_rate,
+		 best, best_delta);
+
+	if (req->best_parent_rate)
+		req->best_parent_rate = best_parent_rate;
+
+	if (req->best_parent_hw)
+		req->best_parent_hw = best_parent_clk;
+
+	return best;
+}
+
+static int roclk_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
+	unsigned long flags;
+	u32 v;
+	int err;
+
+	if (refo->parent_map)
+		index = refo->parent_map[index];
+
+	/* wait until ACTIVE bit is zero or timeout */
+	err = readl_poll_timeout(refo->ctrl_reg, v, !(v & REFO_ACTIVE),
+				 1, LOCK_TIMEOUT_US);
+	if (err) {
+		pr_err("%s: poll failed, clk active\n", clk_hw_get_name(hw));
+		return err;
+	}
+
+	spin_lock_irqsave(&refo->core->reg_lock, flags);
+
+	pic32_syskey_unlock();
+
+	/* calculate & apply new */
+	v = readl(refo->ctrl_reg);
+	v &= ~(REFO_SEL_MASK << REFO_SEL_SHIFT);
+	v |= index << REFO_SEL_SHIFT;
+
+	writel(v, refo->ctrl_reg);
+
+	spin_unlock_irqrestore(&refo->core->reg_lock, flags);
+
+	return 0;
+}
+
+static int roclk_set_rate_and_parent(struct clk_hw *hw,
+				     unsigned long rate,
+				     unsigned long parent_rate,
+				     u8 index)
+{
+	struct pic32_ref_osc *refo = clkhw_to_refosc(hw);
+	unsigned long flags;
+	u32 trim, rodiv, v;
+	int err;
+
+	/* calculate new rodiv & rotrim for new rate */
+	roclk_calc_div_trim(rate, parent_rate, &rodiv, &trim);
+
+	pr_debug("parent_rate = %lu, rate = %lu, div = %d, trim = %d\n",
+		 parent_rate, rate, rodiv, trim);
+
+	/* wait till source change is active */
+	err = readl_poll_timeout(refo->ctrl_reg, v,
+				 !(v & (REFO_ACTIVE | REFO_DIVSW_EN)),
+				 1, LOCK_TIMEOUT_US);
+	if (err) {
+		pr_err("%s: poll timedout, clock is still active\n", __func__);
+		return err;
+	}
+
+	spin_lock_irqsave(&refo->core->reg_lock, flags);
+	v = readl(refo->ctrl_reg);
+
+	pic32_syskey_unlock();
+
+	/* apply parent, if required */
+	if (refo->parent_map)
+		index = refo->parent_map[index];
+
+	v &= ~(REFO_SEL_MASK << REFO_SEL_SHIFT);
+	v |= index << REFO_SEL_SHIFT;
+
+	/* apply RODIV */
+	v &= ~(REFO_DIV_MASK << REFO_DIV_SHIFT);
+	v |= rodiv << REFO_DIV_SHIFT;
+	writel(v, refo->ctrl_reg);
+
+	/* apply ROTRIM */
+	v = readl(refo->ctrl_reg + REFO_TRIM_REG);
+	v &= ~(REFO_TRIM_MASK << REFO_TRIM_SHIFT);
+	v |= trim << REFO_TRIM_SHIFT;
+	writel(v, refo->ctrl_reg + REFO_TRIM_REG);
+
+	/* enable & activate divider switching */
+	writel(REFO_ON | REFO_DIVSW_EN, PIC32_SET(refo->ctrl_reg));
+
+	/* wait till divswen is in-progress */
+	err = readl_poll_timeout_atomic(refo->ctrl_reg, v, !(v & REFO_DIVSW_EN),
+					1, LOCK_TIMEOUT_US);
+	/* leave the clk gated as it was */
+	writel(REFO_ON, PIC32_CLR(refo->ctrl_reg));
+
+	spin_unlock_irqrestore(&refo->core->reg_lock, flags);
+
+	return err;
+}
+
+static int roclk_set_rate(struct clk_hw *hw, unsigned long rate,
+			  unsigned long parent_rate)
+{
+	u8 index = roclk_get_parent(hw);
+
+	return roclk_set_rate_and_parent(hw, rate, parent_rate, index);
+}
+
+const struct clk_ops pic32_roclk_ops = {
+	.enable			= roclk_enable,
+	.disable		= roclk_disable,
+	.is_enabled		= roclk_is_enabled,
+	.get_parent		= roclk_get_parent,
+	.set_parent		= roclk_set_parent,
+	.determine_rate		= roclk_determine_rate,
+	.recalc_rate		= roclk_recalc_rate,
+	.set_rate_and_parent	= roclk_set_rate_and_parent,
+	.set_rate		= roclk_set_rate,
+	.init			= roclk_init,
+};
+
+struct clk *pic32_refo_clk_register(const struct pic32_ref_osc_data *data,
+				    struct pic32_clk_common *core)
+{
+	struct pic32_ref_osc *refo;
+	struct clk *clk;
+
+	refo = devm_kzalloc(core->dev, sizeof(*refo), GFP_KERNEL);
+	if (!refo)
+		return ERR_PTR(-ENOMEM);
+
+	refo->core = core;
+	refo->hw.init = &data->init_data;
+	refo->ctrl_reg = data->ctrl_reg + core->iobase;
+	refo->parent_map = data->parent_map;
+
+	clk = devm_clk_register(core->dev, &refo->hw);
+	if (IS_ERR(clk))
+		dev_err(core->dev, "%s: clk_register() failed\n", __func__);
+
+	return clk;
+}
+
+struct pic32_sys_pll {
+	struct clk_hw hw;
+	void __iomem *ctrl_reg;
+	void __iomem *status_reg;
+	u32 lock_mask;
+	u32 idiv; /* PLL iclk divider, treated fixed */
+	struct pic32_clk_common *core;
+};
+
+#define clkhw_to_spll(_hw)	container_of(_hw, struct pic32_sys_pll, hw)
+
+static inline u32 spll_odiv_to_divider(u32 odiv)
+{
+	odiv = clamp_val(odiv, PLL_ODIV_MIN, PLL_ODIV_MAX);
+
+	return 1 << odiv;
+}
+
+static unsigned long spll_calc_mult_div(struct pic32_sys_pll *pll,
+					unsigned long rate,
+					unsigned long parent_rate,
+					u32 *mult_p, u32 *odiv_p)
+{
+	u32 mul, div, best_mul = 1, best_div = 1;
+	unsigned long new_rate, best_rate = rate;
+	unsigned int best_delta = -1, delta, match_found = 0;
+	u64 rate64;
+
+	parent_rate /= pll->idiv;
+
+	for (mul = 1; mul <= PLL_MULT_MAX; mul++) {
+		for (div = PLL_ODIV_MIN; div <= PLL_ODIV_MAX; div++) {
+			rate64 = parent_rate;
+			rate64 *= mul;
+			do_div(rate64, 1 << div);
+			new_rate = rate64;
+			delta = abs(rate - new_rate);
+			if ((new_rate >= rate) && (delta < best_delta)) {
+				best_delta = delta;
+				best_rate = new_rate;
+				best_mul = mul;
+				best_div = div;
+				match_found = 1;
+			}
+		}
+	}
+
+	if (!match_found) {
+		pr_warn("spll: no match found\n");
+		return 0;
+	}
+
+	pr_debug("rate %lu, par_rate %lu/mult %u, div %u, best_rate %lu\n",
+		 rate, parent_rate, best_mul, best_div, best_rate);
+
+	if (mult_p)
+		*mult_p = best_mul - 1;
+
+	if (odiv_p)
+		*odiv_p = best_div;
+
+	return best_rate;
+}
+
+static unsigned long spll_clk_recalc_rate(struct clk_hw *hw,
+					  unsigned long parent_rate)
+{
+	struct pic32_sys_pll *pll = clkhw_to_spll(hw);
+	unsigned long pll_in_rate;
+	u32 mult, odiv, div, v;
+	u64 rate64;
+
+	v = readl(pll->ctrl_reg);
+	odiv = ((v >> PLL_ODIV_SHIFT) & PLL_ODIV_MASK);
+	mult = ((v >> PLL_MULT_SHIFT) & PLL_MULT_MASK) + 1;
+	div = spll_odiv_to_divider(odiv);
+
+	/* pll_in_rate = parent_rate / idiv
+	 * pll_out_rate = pll_in_rate * mult / div;
+	 */
+	pll_in_rate = parent_rate / pll->idiv;
+	rate64 = pll_in_rate;
+	rate64 *= mult;
+	do_div(rate64, div);
+
+	return rate64;
+}
+
+static int spll_clk_determine_rate(struct clk_hw *hw,
+				   struct clk_rate_request *req)
+{
+	struct pic32_sys_pll *pll = clkhw_to_spll(hw);
+
+	req->rate = spll_calc_mult_div(pll, req->rate, req->best_parent_rate,
+				       NULL, NULL);
+
+	return 0;
+}
+
+static int spll_clk_set_rate(struct clk_hw *hw, unsigned long rate,
+			     unsigned long parent_rate)
+{
+	struct pic32_sys_pll *pll = clkhw_to_spll(hw);
+	unsigned long ret, flags;
+	u32 mult, odiv, v;
+	int err;
+
+	ret = spll_calc_mult_div(pll, rate, parent_rate, &mult, &odiv);
+	if (!ret)
+		return -EINVAL;
+
+	/*
+	 * We can't change SPLL counters when it is in-active use
+	 * by SYSCLK. So check before applying new counters/rate.
+	 */
+
+	/* Is spll_clk active parent of sys_clk ? */
+	if (unlikely(clk_hw_get_parent(pic32_sclk_hw) == hw)) {
+		pr_err("%s: failed, clk in-use\n", __func__);
+		return -EBUSY;
+	}
+
+	spin_lock_irqsave(&pll->core->reg_lock, flags);
+
+	/* apply new multiplier & divisor */
+	v = readl(pll->ctrl_reg);
+	v &= ~(PLL_MULT_MASK << PLL_MULT_SHIFT);
+	v &= ~(PLL_ODIV_MASK << PLL_ODIV_SHIFT);
+	v |= (mult << PLL_MULT_SHIFT) | (odiv << PLL_ODIV_SHIFT);
+
+	/* sys unlock before write */
+	pic32_syskey_unlock();
+
+	writel(v, pll->ctrl_reg);
+	cpu_relax();
+
+	/* insert few nops (5-stage) to ensure CPU does not hang */
+	cpu_nop5();
+	cpu_nop5();
+
+	/* Wait until PLL is locked (maximum 100 usecs). */
+	err = readl_poll_timeout_atomic(pll->status_reg, v,
+					v & pll->lock_mask, 1, 100);
+	spin_unlock_irqrestore(&pll->core->reg_lock, flags);
+
+	return err;
+}
+
+/* SPLL clock operation */
+const struct clk_ops pic32_spll_ops = {
+	.recalc_rate	= spll_clk_recalc_rate,
+	.determine_rate = spll_clk_determine_rate,
+	.set_rate	= spll_clk_set_rate,
+};
+
+struct clk *pic32_spll_clk_register(const struct pic32_sys_pll_data *data,
+				    struct pic32_clk_common *core)
+{
+	struct pic32_sys_pll *spll;
+	struct clk *clk;
+
+	spll = devm_kzalloc(core->dev, sizeof(*spll), GFP_KERNEL);
+	if (!spll)
+		return ERR_PTR(-ENOMEM);
+
+	spll->core = core;
+	spll->hw.init = &data->init_data;
+	spll->ctrl_reg = data->ctrl_reg + core->iobase;
+	spll->status_reg = data->status_reg + core->iobase;
+	spll->lock_mask = data->lock_mask;
+
+	/* cache PLL idiv; PLL driver uses it as constant.*/
+	spll->idiv = (readl(spll->ctrl_reg) >> PLL_IDIV_SHIFT) & PLL_IDIV_MASK;
+	spll->idiv += 1;
+
+	clk = devm_clk_register(core->dev, &spll->hw);
+	if (IS_ERR(clk))
+		dev_err(core->dev, "sys_pll: clk_register() failed\n");
+
+	return clk;
+}
+
+/* System mux clock(aka SCLK) */
+
+struct pic32_sys_clk {
+	struct clk_hw hw;
+	void __iomem *mux_reg;
+	void __iomem *slew_reg;
+	u32 slew_div;
+	const u32 *parent_map;
+	struct pic32_clk_common *core;
+};
+
+#define clkhw_to_sys_clk(_hw)	container_of(_hw, struct pic32_sys_clk, hw)
+
+static unsigned long sclk_get_rate(struct clk_hw *hw, unsigned long parent_rate)
+{
+	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
+	u32 div;
+
+	div = (readl(sclk->slew_reg) >> SLEW_SYSDIV_SHIFT) & SLEW_SYSDIV;
+	div += 1; /* sys-div to divider */
+
+	return parent_rate / div;
+}
+
+static int sclk_set_rate(struct clk_hw *hw,
+			 unsigned long rate, unsigned long parent_rate)
+{
+	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
+	unsigned long flags;
+	u32 v, div;
+	int err;
+
+	div = parent_rate / rate;
+
+	spin_lock_irqsave(&sclk->core->reg_lock, flags);
+
+	/* apply new div */
+	v = readl(sclk->slew_reg);
+	v &= ~(SLEW_SYSDIV << SLEW_SYSDIV_SHIFT);
+	v |= (div - 1) << SLEW_SYSDIV_SHIFT;
+
+	pic32_syskey_unlock();
+
+	writel(v, sclk->slew_reg);
+
+	/* wait until BUSY is cleared */
+	err = readl_poll_timeout_atomic(sclk->slew_reg, v,
+					!(v & SLEW_BUSY), 1, LOCK_TIMEOUT_US);
+
+	spin_unlock_irqrestore(&sclk->core->reg_lock, flags);
+
+	return err;
+}
+
+static u8 sclk_get_parent(struct clk_hw *hw)
+{
+	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
+	u32 i, v;
+
+	v = (readl(sclk->mux_reg) >> OSC_CUR_SHIFT) & OSC_CUR_MASK;
+
+	if (sclk->parent_map) {
+		for (i = 0; i < clk_hw_get_num_parents(hw); i++)
+			if (sclk->parent_map[i] == v)
+				return i;
+	}
+
+	return v;
+}
+
+static int sclk_set_parent(struct clk_hw *hw, u8 index)
+{
+	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
+	unsigned long flags;
+	u32 nosc, cosc, v;
+	int err;
+
+	spin_lock_irqsave(&sclk->core->reg_lock, flags);
+
+	/* find new_osc */
+	nosc = sclk->parent_map ? sclk->parent_map[index] : index;
+
+	/* set new parent */
+	v = readl(sclk->mux_reg);
+	v &= ~(OSC_NEW_MASK << OSC_NEW_SHIFT);
+	v |= nosc << OSC_NEW_SHIFT;
+
+	pic32_syskey_unlock();
+
+	writel(v, sclk->mux_reg);
+
+	/* initate switch */
+	writel(OSC_SWEN, PIC32_SET(sclk->mux_reg));
+	cpu_relax();
+
+	/* add nop to flush pipeline (as cpu_clk is in-flux) */
+	cpu_nop5();
+
+	/* wait for SWEN bit to clear */
+	err = readl_poll_timeout_atomic(sclk->slew_reg, v,
+					!(v & OSC_SWEN), 1, LOCK_TIMEOUT_US);
+
+	spin_unlock_irqrestore(&sclk->core->reg_lock, flags);
+
+	/*
+	 * SCLK clock-switching logic might reject a clock switching request
+	 * if pre-requisites (like new clk_src not present or unstable) are
+	 * not met.
+	 * So confirm before claiming success.
+	 */
+	cosc = (readl(sclk->mux_reg) >> OSC_CUR_SHIFT) & OSC_CUR_MASK;
+	if (cosc != nosc) {
+		pr_err("%s: err, failed to set_parent() to %d, current %d\n",
+		       clk_hw_get_name(hw), nosc, cosc);
+		err = -EBUSY;
+	}
+
+	return err;
+}
+
+static int sclk_init(struct clk_hw *hw)
+{
+	struct pic32_sys_clk *sclk = clkhw_to_sys_clk(hw);
+	unsigned long flags;
+	u32 v;
+
+	/* Maintain reference to this clk, required in spll_clk_set_rate() */
+	pic32_sclk_hw = hw;
+
+	/* apply slew divider on both up and down scaling */
+	if (sclk->slew_div) {
+		spin_lock_irqsave(&sclk->core->reg_lock, flags);
+		v = readl(sclk->slew_reg);
+		v &= ~(SLEW_DIV << SLEW_DIV_SHIFT);
+		v |= sclk->slew_div << SLEW_DIV_SHIFT;
+		v |= SLEW_DOWNEN | SLEW_UPEN;
+		writel(v, sclk->slew_reg);
+		spin_unlock_irqrestore(&sclk->core->reg_lock, flags);
+	}
+
+	return 0;
+}
+
+/* sclk with post-divider */
+const struct clk_ops pic32_sclk_ops = {
+	.get_parent	= sclk_get_parent,
+	.set_parent	= sclk_set_parent,
+	.set_rate	= sclk_set_rate,
+	.recalc_rate	= sclk_get_rate,
+	.init		= sclk_init,
+	.determine_rate = __clk_mux_determine_rate,
+};
+
+/* sclk with no slew and no post-divider */
+const struct clk_ops pic32_sclk_no_div_ops = {
+	.get_parent	= sclk_get_parent,
+	.set_parent	= sclk_set_parent,
+	.init		= sclk_init,
+	.determine_rate = __clk_mux_determine_rate,
+};
+
+struct clk *pic32_sys_clk_register(const struct pic32_sys_clk_data *data,
+				   struct pic32_clk_common *core)
+{
+	struct pic32_sys_clk *sclk;
+	struct clk *clk;
+
+	sclk = devm_kzalloc(core->dev, sizeof(*sclk), GFP_KERNEL);
+	if (!sclk)
+		return ERR_PTR(-ENOMEM);
+
+	sclk->core = core;
+	sclk->hw.init = &data->init_data;
+	sclk->mux_reg = data->mux_reg + core->iobase;
+	sclk->slew_reg = data->slew_reg + core->iobase;
+	sclk->slew_div = data->slew_div;
+	sclk->parent_map = data->parent_map;
+
+	clk = devm_clk_register(core->dev, &sclk->hw);
+	if (IS_ERR(clk))
+		dev_err(core->dev, "%s: clk register failed\n", __func__);
+
+	return clk;
+}
+
+/* secondary oscillator */
+struct pic32_sec_osc {
+	struct clk_hw hw;
+	void __iomem *enable_reg;
+	void __iomem *status_reg;
+	u32 enable_mask;
+	u32 status_mask;
+	unsigned long fixed_rate;
+	struct pic32_clk_common *core;
+};
+
+#define clkhw_to_sosc(_hw)	container_of(_hw, struct pic32_sec_osc, hw)
+static int sosc_clk_enable(struct clk_hw *hw)
+{
+	struct pic32_sec_osc *sosc = clkhw_to_sosc(hw);
+	u32 v;
+
+	/* enable SOSC */
+	pic32_syskey_unlock();
+	writel(sosc->enable_mask, PIC32_SET(sosc->enable_reg));
+
+	/* wait till warm-up period expires or ready-status is updated */
+	return readl_poll_timeout_atomic(sosc->status_reg, v,
+					 v & sosc->status_mask, 1, 100);
+}
+
+static void sosc_clk_disable(struct clk_hw *hw)
+{
+	struct pic32_sec_osc *sosc = clkhw_to_sosc(hw);
+
+	pic32_syskey_unlock();
+	writel(sosc->enable_mask, PIC32_CLR(sosc->enable_reg));
+}
+
+static int sosc_clk_is_enabled(struct clk_hw *hw)
+{
+	struct pic32_sec_osc *sosc = clkhw_to_sosc(hw);
+	u32 enabled, ready;
+
+	/* check enabled and ready status */
+	enabled = readl(sosc->enable_reg) & sosc->enable_mask;
+	ready = readl(sosc->status_reg) & sosc->status_mask;
+
+	return enabled && ready;
+}
+
+static unsigned long sosc_clk_calc_rate(struct clk_hw *hw,
+					unsigned long parent_rate)
+{
+	return clkhw_to_sosc(hw)->fixed_rate;
+}
+
+const struct clk_ops pic32_sosc_ops = {
+	.enable = sosc_clk_enable,
+	.disable = sosc_clk_disable,
+	.is_enabled = sosc_clk_is_enabled,
+	.recalc_rate = sosc_clk_calc_rate,
+};
+
+struct clk *pic32_sosc_clk_register(const struct pic32_sec_osc_data *data,
+				    struct pic32_clk_common *core)
+{
+	struct pic32_sec_osc *sosc;
+
+	sosc = devm_kzalloc(core->dev, sizeof(*sosc), GFP_KERNEL);
+	if (!sosc)
+		return ERR_PTR(-ENOMEM);
+
+	sosc->core = core;
+	sosc->hw.init = &data->init_data;
+	sosc->fixed_rate = data->fixed_rate;
+	sosc->enable_mask = data->enable_mask;
+	sosc->status_mask = data->status_mask;
+	sosc->enable_reg = data->enable_reg + core->iobase;
+	sosc->status_reg = data->status_reg + core->iobase;
+
+	return devm_clk_register(core->dev, &sosc->hw);
+}
diff --git a/drivers/clk/microchip/clk-pic32.h b/drivers/clk/microchip/clk-pic32.h
new file mode 100644
index 0000000000000..acb81739a16f8
--- /dev/null
+++ b/drivers/clk/microchip/clk-pic32.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Purna Chandra Mandal,<purna.mandal@microchip.com>
+ * Copyright (C) 2015 Microchip Technology Inc.  All rights reserved.
+ */
+#ifndef __MICROCHIP_CLK_PIC32_H_
+#define __MICROCHIP_CLK_PIC32_H_
+
+#include <linux/clk-provider.h>
+
+/* PIC32 clock data */
+struct pic32_clk_common {
+	struct device *dev;
+	void __iomem *iobase;
+	spinlock_t reg_lock; /* clock lock */
+};
+
+/* System PLL clock */
+struct pic32_sys_pll_data {
+	struct clk_init_data init_data;
+	const u32 ctrl_reg;
+	const u32 status_reg;
+	const u32 lock_mask;
+};
+
+/* System clock */
+struct pic32_sys_clk_data {
+	struct clk_init_data init_data;
+	const u32 mux_reg;
+	const u32 slew_reg;
+	const u32 *parent_map;
+	const u32 slew_div;
+};
+
+/* Reference Oscillator clock */
+struct pic32_ref_osc_data {
+	struct clk_init_data init_data;
+	const u32 ctrl_reg;
+	const u32 *parent_map;
+};
+
+/* Peripheral Bus clock */
+struct pic32_periph_clk_data {
+	struct clk_init_data init_data;
+	const u32 ctrl_reg;
+};
+
+/* External Secondary Oscillator clock  */
+struct pic32_sec_osc_data {
+	struct clk_init_data init_data;
+	const u32 enable_reg;
+	const u32 status_reg;
+	const u32 enable_mask;
+	const u32 status_mask;
+	const unsigned long fixed_rate;
+};
+
+extern const struct clk_ops pic32_pbclk_ops;
+extern const struct clk_ops pic32_sclk_ops;
+extern const struct clk_ops pic32_sclk_no_div_ops;
+extern const struct clk_ops pic32_spll_ops;
+extern const struct clk_ops pic32_roclk_ops;
+extern const struct clk_ops pic32_sosc_ops;
+
+struct clk *pic32_periph_clk_register(const struct pic32_periph_clk_data *data,
+				      struct pic32_clk_common *core);
+struct clk *pic32_refo_clk_register(const struct pic32_ref_osc_data *data,
+				    struct pic32_clk_common *core);
+struct clk *pic32_sys_clk_register(const struct pic32_sys_clk_data *data,
+				   struct pic32_clk_common *core);
+struct clk *pic32_spll_clk_register(const struct pic32_sys_pll_data *data,
+				    struct pic32_clk_common *core);
+struct clk *pic32_sosc_clk_register(const struct pic32_sec_osc_data *data,
+				    struct pic32_clk_common *core);
+
+#endif /* __MICROCHIP_CLK_PIC32_H_*/
diff --git a/drivers/clk/microchip/clk-pic32mzda.c b/drivers/clk/microchip/clk-pic32mzda.c
index 27599829ea40a..e11cbdd982a61 100644
--- a/drivers/clk/microchip/clk-pic32mzda.c
+++ b/drivers/clk/microchip/clk-pic32mzda.c
@@ -14,7 +14,7 @@
 #include <linux/platform_device.h>
 #include <asm/traps.h>
 
-#include "clk-core.h"
+#include "clk-pic32.h"
 
 /* FRC Postscaler */
 #define OSC_FRCDIV_MASK		0x07
diff --git a/drivers/clk/mmp/clk-pxa1908-apbc.c b/drivers/clk/mmp/clk-pxa1908-apbc.c
index 3fd7b5e644f3b..438ece4f047dc 100644
--- a/drivers/clk/mmp/clk-pxa1908-apbc.c
+++ b/drivers/clk/mmp/clk-pxa1908-apbc.c
@@ -7,6 +7,7 @@
 #include <dt-bindings/clock/marvell,pxa1908.h>
 
 #include "clk.h"
+#include "reset.h"
 
 #define APBC_UART0		0x0
 #define APBC_UART1		0x4
@@ -44,22 +45,25 @@ static const char * const uart_parent_names[] = {"pll1_117", "uart_pll"};
 static const char * const ssp_parent_names[] = {"pll1_d16", "pll1_d48", "pll1_d24", "pll1_d12"};
 
 static struct mmp_param_gate_clk apbc_gate_clks[] = {
-	{PXA1908_CLK_TWSI0, "twsi0_clk", "pll1_32", CLK_SET_RATE_PARENT, APBC_TWSI0, 0x7, 3, 0, 0, NULL},
-	{PXA1908_CLK_TWSI1, "twsi1_clk", "pll1_32", CLK_SET_RATE_PARENT, APBC_TWSI1, 0x7, 3, 0, 0, NULL},
-	{PXA1908_CLK_TWSI3, "twsi3_clk", "pll1_32", CLK_SET_RATE_PARENT, APBC_TWSI3, 0x7, 3, 0, 0, NULL},
-	{PXA1908_CLK_GPIO, "gpio_clk", "vctcxo", CLK_SET_RATE_PARENT, APBC_GPIO, 0x7, 3, 0, 0, NULL},
-	{PXA1908_CLK_KPC, "kpc_clk", "clk32", CLK_SET_RATE_PARENT, APBC_KPC, 0x7, 3, 0, MMP_CLK_GATE_NEED_DELAY, NULL},
-	{PXA1908_CLK_RTC, "rtc_clk", "clk32", CLK_SET_RATE_PARENT, APBC_RTC, 0x87, 0x83, 0, MMP_CLK_GATE_NEED_DELAY, NULL},
+	{PXA1908_CLK_TWSI0, "twsi0_clk", "pll1_32", CLK_SET_RATE_PARENT, APBC_TWSI0, 0x3, 3, 0, 0, NULL},
+	{PXA1908_CLK_TWSI1, "twsi1_clk", "pll1_32", CLK_SET_RATE_PARENT, APBC_TWSI1, 0x3, 3, 0, 0, NULL},
+	{PXA1908_CLK_TWSI3, "twsi3_clk", "pll1_32", CLK_SET_RATE_PARENT, APBC_TWSI3, 0x3, 3, 0, 0, NULL},
+	{PXA1908_CLK_GPIO, "gpio_clk", "vctcxo", CLK_SET_RATE_PARENT, APBC_GPIO, 0x3, 3, 0, 0, NULL},
+	{PXA1908_CLK_KPC, "kpc_clk", "clk32", CLK_SET_RATE_PARENT, APBC_KPC, 0x3, 3, 0, MMP_CLK_GATE_NEED_DELAY, NULL},
+	{PXA1908_CLK_RTC, "rtc_clk", "clk32", CLK_SET_RATE_PARENT, APBC_RTC, 0x83, 0x83, 0, MMP_CLK_GATE_NEED_DELAY, NULL},
+	{PXA1908_CLK_PWM1, "pwm1_clk", "pwm01_apb_share", CLK_SET_RATE_PARENT, APBC_PWM1, 0x2, 2, 0, 0, NULL},
+	{PXA1908_CLK_PWM3, "pwm3_clk", "pwm23_apb_share", CLK_SET_RATE_PARENT, APBC_PWM3, 0x2, 2, 0, 0, NULL},
+	{PXA1908_CLK_UART0, "uart0_clk", "uart0_mux", CLK_SET_RATE_PARENT, APBC_UART0, 0x3, 3, 0, 0, &uart0_lock},
+	{PXA1908_CLK_UART1, "uart1_clk", "uart1_mux", CLK_SET_RATE_PARENT, APBC_UART1, 0x3, 3, 0, 0, &uart1_lock},
+	{PXA1908_CLK_THERMAL, "thermal_clk", NULL, 0, APBC_THERMAL, 0x3, 3, 0, 0, NULL},
+	{PXA1908_CLK_IPC_RST, "ipc_clk", NULL, 0, APBC_IPC_RST, 0x3, 3, 0, 0, NULL},
+	{PXA1908_CLK_SSP0, "ssp0_clk", "ssp0_mux", 0, APBC_SSP0, 0x3, 3, 0, 0, NULL},
+	{PXA1908_CLK_SSP2, "ssp2_clk", "ssp2_mux", 0, APBC_SSP2, 0x3, 3, 0, 0, NULL},
+};
+
+static struct mmp_param_gate_clk apbc_gate_no_reset_clks[] = {
 	{PXA1908_CLK_PWM0, "pwm0_clk", "pwm01_apb_share", CLK_SET_RATE_PARENT, APBC_PWM0, 0x2, 2, 0, 0, &pwm0_lock},
-	{PXA1908_CLK_PWM1, "pwm1_clk", "pwm01_apb_share", CLK_SET_RATE_PARENT, APBC_PWM1, 0x6, 2, 0, 0, NULL},
 	{PXA1908_CLK_PWM2, "pwm2_clk", "pwm23_apb_share", CLK_SET_RATE_PARENT, APBC_PWM2, 0x2, 2, 0, 0, NULL},
-	{PXA1908_CLK_PWM3, "pwm3_clk", "pwm23_apb_share", CLK_SET_RATE_PARENT, APBC_PWM3, 0x6, 2, 0, 0, NULL},
-	{PXA1908_CLK_UART0, "uart0_clk", "uart0_mux", CLK_SET_RATE_PARENT, APBC_UART0, 0x7, 3, 0, 0, &uart0_lock},
-	{PXA1908_CLK_UART1, "uart1_clk", "uart1_mux", CLK_SET_RATE_PARENT, APBC_UART1, 0x7, 3, 0, 0, &uart1_lock},
-	{PXA1908_CLK_THERMAL, "thermal_clk", NULL, 0, APBC_THERMAL, 0x7, 3, 0, 0, NULL},
-	{PXA1908_CLK_IPC_RST, "ipc_clk", NULL, 0, APBC_IPC_RST, 0x7, 3, 0, 0, NULL},
-	{PXA1908_CLK_SSP0, "ssp0_clk", "ssp0_mux", 0, APBC_SSP0, 0x7, 3, 0, 0, NULL},
-	{PXA1908_CLK_SSP2, "ssp2_clk", "ssp2_mux", 0, APBC_SSP2, 0x7, 3, 0, 0, NULL},
 };
 
 static struct mmp_param_mux_clk apbc_mux_clks[] = {
@@ -89,6 +93,30 @@ static void pxa1908_apb_periph_clk_init(struct pxa1908_clk_unit *pxa_unit)
 			ARRAY_SIZE(apbc_mux_clks));
 	mmp_register_gate_clks(unit, apbc_gate_clks, pxa_unit->base,
 			ARRAY_SIZE(apbc_gate_clks));
+	mmp_register_gate_clks(unit, apbc_gate_no_reset_clks, pxa_unit->base,
+			ARRAY_SIZE(apbc_gate_no_reset_clks));
+}
+
+/* Taken from clk-of-pxa1928.c */
+static void pxa1908_clk_reset_init(struct device_node *np,
+				   struct pxa1908_clk_unit *pxa_unit)
+{
+	struct mmp_clk_reset_cell *cells;
+	int nr_cells = ARRAY_SIZE(apbc_gate_clks);
+
+	cells = kzalloc_objs(*cells, nr_cells);
+	if (!cells)
+		return;
+
+	for (int i = 0; i < nr_cells; i++) {
+		cells[i].clk_id = apbc_gate_clks[i].id;
+		cells[i].reg = pxa_unit->base + apbc_gate_clks[i].offset;
+		cells[i].bits = BIT(2);
+		cells[i].flags = 0;
+		cells[i].lock = apbc_gate_clks[i].lock;
+	};
+
+	mmp_clk_reset_register(np, cells, nr_cells);
 }
 
 static int pxa1908_apbc_probe(struct platform_device *pdev)
@@ -107,6 +135,8 @@ static int pxa1908_apbc_probe(struct platform_device *pdev)
 
 	pxa1908_apb_periph_clk_init(pxa_unit);
 
+	pxa1908_clk_reset_init(pdev->dev.of_node, pxa_unit);
+
 	return 0;
 }
 
diff --git a/drivers/clk/mmp/clk-pxa1908-apbcp.c b/drivers/clk/mmp/clk-pxa1908-apbcp.c
index f638d7e89b472..1aa476103553e 100644
--- a/drivers/clk/mmp/clk-pxa1908-apbcp.c
+++ b/drivers/clk/mmp/clk-pxa1908-apbcp.c
@@ -7,6 +7,7 @@
 #include <dt-bindings/clock/marvell,pxa1908.h>
 
 #include "clk.h"
+#include "reset.h"
 
 #define APBCP_UART2		0x1c
 #define APBCP_TWSI2		0x28
@@ -24,9 +25,9 @@ static DEFINE_SPINLOCK(uart2_lock);
 static const char * const uart_parent_names[] = {"pll1_117", "uart_pll"};
 
 static struct mmp_param_gate_clk apbcp_gate_clks[] = {
-	{PXA1908_CLK_UART2, "uart2_clk", "uart2_mux", CLK_SET_RATE_PARENT, APBCP_UART2, 0x7, 0x3, 0x0, 0, &uart2_lock},
-	{PXA1908_CLK_TWSI2, "twsi2_clk", "pll1_32", CLK_SET_RATE_PARENT, APBCP_TWSI2, 0x7, 0x3, 0x0, 0, NULL},
-	{PXA1908_CLK_AICER, "ripc_clk", NULL, 0, APBCP_AICER, 0x7, 0x2, 0x0, 0, NULL},
+	{PXA1908_CLK_UART2, "uart2_clk", "uart2_mux", CLK_SET_RATE_PARENT, APBCP_UART2, 0x3, 0x3, 0x0, 0, &uart2_lock},
+	{PXA1908_CLK_TWSI2, "twsi2_clk", "pll1_32", CLK_SET_RATE_PARENT, APBCP_TWSI2, 0x3, 0x3, 0x0, 0, NULL},
+	{PXA1908_CLK_AICER, "ripc_clk", NULL, 0, APBCP_AICER, 0x3, 0x2, 0x0, 0, NULL},
 };
 
 static struct mmp_param_mux_clk apbcp_mux_clks[] = {
@@ -43,6 +44,28 @@ static void pxa1908_apb_p_periph_clk_init(struct pxa1908_clk_unit *pxa_unit)
 			ARRAY_SIZE(apbcp_gate_clks));
 }
 
+/* Taken from clk-of-pxa1928.c */
+static void pxa1908_clk_reset_init(struct device_node *np,
+				   struct pxa1908_clk_unit *pxa_unit)
+{
+	struct mmp_clk_reset_cell *cells;
+	int nr_cells = ARRAY_SIZE(apbcp_gate_clks);
+
+	cells = kzalloc_objs(*cells, nr_cells);
+	if (!cells)
+		return;
+
+	for (int i = 0; i < nr_cells; i++) {
+		cells[i].clk_id = apbcp_gate_clks[i].id;
+		cells[i].reg = pxa_unit->base + apbcp_gate_clks[i].offset;
+		cells[i].bits = BIT(2);
+		cells[i].flags = 0;
+		cells[i].lock = apbcp_gate_clks[i].lock;
+	};
+
+	mmp_clk_reset_register(np, cells, nr_cells);
+}
+
 static int pxa1908_apbcp_probe(struct platform_device *pdev)
 {
 	struct pxa1908_clk_unit *pxa_unit;
@@ -59,6 +82,8 @@ static int pxa1908_apbcp_probe(struct platform_device *pdev)
 
 	pxa1908_apb_p_periph_clk_init(pxa_unit);
 
+	pxa1908_clk_reset_init(pdev->dev.of_node, pxa_unit);
+
 	return 0;
 }
 
diff --git a/drivers/clk/mvebu/common.c b/drivers/clk/mvebu/common.c
index 28f2e1b2a9323..0f9ae5f5cefd2 100644
--- a/drivers/clk/mvebu/common.c
+++ b/drivers/clk/mvebu/common.c
@@ -189,10 +189,10 @@ DEFINE_SPINLOCK(ctrl_gating_lock);
 
 struct clk_gating_ctrl {
 	spinlock_t *lock;
-	struct clk **gates;
 	int num_gates;
 	void __iomem *base;
 	u32 saved_reg;
+	struct clk *gates[] __counted_by(num_gates);
 };
 
 static struct clk_gating_ctrl *ctrl;
@@ -257,24 +257,21 @@ void __init mvebu_clk_gating_setup(struct device_node *np,
 		clk_put(clk);
 	}
 
-	ctrl = kzalloc_obj(*ctrl);
+	/* Count, allocate, and register clock gates */
+	for (n = 0; desc[n].name;)
+		n++;
+
+	ctrl = kzalloc_flex(*ctrl, gates, n);
 	if (WARN_ON(!ctrl))
 		goto ctrl_out;
 
+	ctrl->num_gates = n;
+
 	/* lock must already be initialized */
 	ctrl->lock = &ctrl_gating_lock;
 
 	ctrl->base = base;
 
-	/* Count, allocate, and register clock gates */
-	for (n = 0; desc[n].name;)
-		n++;
-
-	ctrl->num_gates = n;
-	ctrl->gates = kzalloc_objs(*ctrl->gates, ctrl->num_gates);
-	if (WARN_ON(!ctrl->gates))
-		goto gates_out;
-
 	for (n = 0; n < ctrl->num_gates; n++) {
 		const char *parent =
 			(desc[n].parent) ? desc[n].parent : default_parent;
@@ -289,8 +286,6 @@ void __init mvebu_clk_gating_setup(struct device_node *np,
 	register_syscore(&clk_gate_syscore);
 
 	return;
-gates_out:
-	kfree(ctrl);
 ctrl_out:
 	iounmap(base);
 }
diff --git a/drivers/clk/nxp/clk-lpc18xx-ccu.c b/drivers/clk/nxp/clk-lpc18xx-ccu.c
index dcb6d0c0b41aa..3793e701835ff 100644
--- a/drivers/clk/nxp/clk-lpc18xx-ccu.c
+++ b/drivers/clk/nxp/clk-lpc18xx-ccu.c
@@ -27,8 +27,8 @@
 #define CCU_BRANCH_HAVE_DIV2	BIT(1)
 
 struct lpc18xx_branch_clk_data {
-	const char **name;
 	int num;
+	const char *name[] __counted_by(num);
 };
 
 struct lpc18xx_clk_branch {
@@ -266,6 +266,7 @@ static void __init lpc18xx_ccu_init(struct device_node *np)
 {
 	struct lpc18xx_branch_clk_data *clk_data;
 	void __iomem *reg_base;
+	size_t size;
 	int i, ret;
 
 	reg_base = of_iomap(np, 0);
@@ -274,19 +275,14 @@ static void __init lpc18xx_ccu_init(struct device_node *np)
 		return;
 	}
 
-	clk_data = kzalloc_obj(*clk_data);
+	size = of_property_count_strings(np, "clock-names");
+	clk_data = kzalloc_flex(*clk_data, name, size);
 	if (!clk_data) {
 		iounmap(reg_base);
 		return;
 	}
 
-	clk_data->num = of_property_count_strings(np, "clock-names");
-	clk_data->name = kcalloc(clk_data->num, sizeof(char *), GFP_KERNEL);
-	if (!clk_data->name) {
-		iounmap(reg_base);
-		kfree(clk_data);
-		return;
-	}
+	clk_data->num = size;
 
 	for (i = 0; i < clk_data->num; i++) {
 		ret = of_property_read_string_index(np, "clock-names", i,
diff --git a/drivers/clk/qcom/Kconfig b/drivers/clk/qcom/Kconfig
index df21ef5ffd686..d9cff5b0281d8 100644
--- a/drivers/clk/qcom/Kconfig
+++ b/drivers/clk/qcom/Kconfig
@@ -296,6 +296,22 @@ config QCOM_CLK_RPMH
 	 Say Y if you want to support the clocks exposed by RPMh on
 	 platforms such as SDM845.
 
+config CLK_HAWI_GCC
+	tristate "Hawi Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	select QCOM_GDSC
+	help
+	  Support for the global clock controller on Hawi devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  I2C, USB, UFS, SD/eMMC, PCIe, etc.
+
+config CLK_HAWI_TCSRCC
+	tristate "Hawi TCSR Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	help
+	  Support for the TCSR clock controller on Hawi devices.
+	  Say Y if you want to use peripheral devices such as PCIe, USB, UFS.
+
 config APQ_GCC_8084
 	tristate "APQ8084 Global Clock Controller"
 	depends on ARM || COMPILE_TEST
@@ -434,6 +450,16 @@ config IPQ_GCC_9574
 	  i2c, USB, SD/eMMC, etc. Select this for the root clock
 	  of ipq9574.
 
+config IPQ_GCC_9650
+	tristate "IPQ9650 Global Clock Controller"
+	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
+	help
+	  Support for global clock controller on ipq9650 devices.
+	  Say Y if you want to use peripheral devices such as UART, SPI,
+	  i2c, USB, SD/eMMC, etc. Select this for the root clock
+	  of ipq9650.
+
 config IPQ_NSSCC_5424
 	tristate "IPQ5424 NSS Clock Controller"
 	depends on ARM64 || COMPILE_TEST
diff --git a/drivers/clk/qcom/Makefile b/drivers/clk/qcom/Makefile
index 89d07c35e4d93..e100cfd6a52de 100644
--- a/drivers/clk/qcom/Makefile
+++ b/drivers/clk/qcom/Makefile
@@ -29,6 +29,8 @@ obj-$(CONFIG_CLK_GLYMUR_GCC) += gcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_GPUCC) += gpucc-glymur.o gxclkctl-kaanapali.o
 obj-$(CONFIG_CLK_GLYMUR_TCSRCC) += tcsrcc-glymur.o
 obj-$(CONFIG_CLK_GLYMUR_VIDEOCC) += videocc-glymur.o
+obj-$(CONFIG_CLK_HAWI_GCC) += gcc-hawi.o
+obj-$(CONFIG_CLK_HAWI_TCSRCC) += tcsrcc-hawi.o
 obj-$(CONFIG_CLK_KAANAPALI_CAMCC) += cambistmclkcc-kaanapali.o camcc-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_DISPCC) += dispcc-kaanapali.o
 obj-$(CONFIG_CLK_KAANAPALI_GCC) += gcc-kaanapali.o
@@ -57,6 +59,7 @@ obj-$(CONFIG_IPQ_GCC_6018) += gcc-ipq6018.o
 obj-$(CONFIG_IPQ_GCC_806X) += gcc-ipq806x.o
 obj-$(CONFIG_IPQ_GCC_8074) += gcc-ipq8074.o
 obj-$(CONFIG_IPQ_GCC_9574) += gcc-ipq9574.o
+obj-$(CONFIG_IPQ_GCC_9650) += gcc-ipq9650.o
 obj-$(CONFIG_IPQ_NSSCC_5424) += nsscc-ipq5424.o
 obj-$(CONFIG_IPQ_NSSCC_9574)	+= nsscc-ipq9574.o
 obj-$(CONFIG_IPQ_LCC_806X) += lcc-ipq806x.o
@@ -189,7 +192,7 @@ obj-$(CONFIG_SM_GPUCC_8450) += gpucc-sm8450.o
 obj-$(CONFIG_SM_GPUCC_8550) += gpucc-sm8550.o
 obj-$(CONFIG_SM_GPUCC_8650) += gpucc-sm8650.o
 obj-$(CONFIG_SM_GPUCC_8750) += gpucc-sm8750.o gxclkctl-kaanapali.o
-obj-$(CONFIG_SM_GPUCC_MILOS) += gpucc-milos.o
+obj-$(CONFIG_SM_GPUCC_MILOS) += gpucc-milos.o gxclkctl-kaanapali.o
 obj-$(CONFIG_SM_LPASSCC_6115) += lpasscc-sm6115.o
 obj-$(CONFIG_SM_TCSRCC_8550) += tcsrcc-sm8550.o
 obj-$(CONFIG_SM_TCSRCC_8650) += tcsrcc-sm8650.o
diff --git a/drivers/clk/qcom/clk-alpha-pll.h b/drivers/clk/qcom/clk-alpha-pll.h
index 42d3344921457..3a2157bebc52c 100644
--- a/drivers/clk/qcom/clk-alpha-pll.h
+++ b/drivers/clk/qcom/clk-alpha-pll.h
@@ -31,6 +31,7 @@ enum {
 	CLK_ALPHA_PLL_TYPE_PONGO_EKO_T = CLK_ALPHA_PLL_TYPE_PONGO_ELU,
 	CLK_ALPHA_PLL_TYPE_TAYCAN_ELU,
 	CLK_ALPHA_PLL_TYPE_TAYCAN_EKO_T = CLK_ALPHA_PLL_TYPE_TAYCAN_ELU,
+	CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T = CLK_ALPHA_PLL_TYPE_TAYCAN_ELU,
 	CLK_ALPHA_PLL_TYPE_RIVIAN_EVO,
 	CLK_ALPHA_PLL_TYPE_RIVIAN_ELU,
 	CLK_ALPHA_PLL_TYPE_RIVIAN_EKO_T = CLK_ALPHA_PLL_TYPE_RIVIAN_ELU,
@@ -198,16 +199,19 @@ extern const struct clk_ops clk_alpha_pll_zonda_ops;
 extern const struct clk_ops clk_alpha_pll_lucid_evo_ops;
 #define clk_alpha_pll_taycan_elu_ops clk_alpha_pll_lucid_evo_ops
 #define clk_alpha_pll_taycan_eko_t_ops clk_alpha_pll_lucid_evo_ops
+#define clk_alpha_pll_taycan_eha_t_ops clk_alpha_pll_lucid_evo_ops
 extern const struct clk_ops clk_alpha_pll_reset_lucid_evo_ops;
 #define clk_alpha_pll_reset_lucid_ole_ops clk_alpha_pll_reset_lucid_evo_ops
 extern const struct clk_ops clk_alpha_pll_fixed_lucid_evo_ops;
 #define clk_alpha_pll_fixed_lucid_ole_ops clk_alpha_pll_fixed_lucid_evo_ops
 #define clk_alpha_pll_fixed_taycan_elu_ops clk_alpha_pll_fixed_lucid_evo_ops
 #define clk_alpha_pll_fixed_taycan_eko_t_ops clk_alpha_pll_fixed_lucid_evo_ops
+#define clk_alpha_pll_fixed_taycan_eha_t_ops clk_alpha_pll_fixed_lucid_evo_ops
 extern const struct clk_ops clk_alpha_pll_postdiv_lucid_evo_ops;
 #define clk_alpha_pll_postdiv_lucid_ole_ops clk_alpha_pll_postdiv_lucid_evo_ops
 #define clk_alpha_pll_postdiv_taycan_elu_ops clk_alpha_pll_postdiv_lucid_evo_ops
 #define clk_alpha_pll_postdiv_taycan_eko_t_ops clk_alpha_pll_postdiv_lucid_evo_ops
+#define clk_alpha_pll_postdiv_taycan_eha_t_ops clk_alpha_pll_postdiv_lucid_evo_ops
 
 extern const struct clk_ops clk_alpha_pll_pongo_elu_ops;
 #define clk_alpha_pll_pongo_eko_t_ops clk_alpha_pll_pongo_elu_ops
@@ -246,6 +250,8 @@ void clk_pongo_elu_pll_configure(struct clk_alpha_pll *pll, struct regmap *regma
 	clk_lucid_evo_pll_configure(pll, regmap, config)
 #define clk_taycan_eko_t_pll_configure(pll, regmap, config) \
 	clk_lucid_evo_pll_configure(pll, regmap, config)
+#define clk_taycan_eha_t_pll_configure(pll, regmap, config) \
+	clk_lucid_evo_pll_configure(pll, regmap, config)
 
 void clk_rivian_evo_pll_configure(struct clk_alpha_pll *pll, struct regmap *regmap,
 				  const struct alpha_pll_config *config);
diff --git a/drivers/clk/qcom/clk-rpmh.c b/drivers/clk/qcom/clk-rpmh.c
index 339a6bbcdc4c7..6367b2a5a4e08 100644
--- a/drivers/clk/qcom/clk-rpmh.c
+++ b/drivers/clk/qcom/clk-rpmh.c
@@ -66,8 +66,6 @@ struct clk_rpmh {
 struct clk_rpmh_desc {
 	struct clk_hw **clks;
 	size_t num_clks;
-	/* RPMh clock clkaN are optional for this platform */
-	bool clka_optional;
 };
 
 static DEFINE_MUTEX(rpmh_clk_lock);
@@ -409,7 +407,9 @@ DEFINE_CLK_RPMH_VRM(clk5, _a2_e0, "C5A_E0", 2);
 DEFINE_CLK_RPMH_VRM(clk6, _a2_e0, "C6A_E0", 2);
 DEFINE_CLK_RPMH_VRM(clk7, _a2_e0, "C7A_E0", 2);
 DEFINE_CLK_RPMH_VRM(clk8, _a2_e0, "C8A_E0", 2);
+DEFINE_CLK_RPMH_VRM(clk9, _a2_e0, "C9A_E0", 2);
 
+DEFINE_CLK_RPMH_VRM(clk7, _a4_e0, "C7A_E0", 4);
 DEFINE_CLK_RPMH_VRM(clk11, _a4_e0, "C11A_E0", 4);
 
 DEFINE_CLK_RPMH_BCM(ce, "CE0");
@@ -697,7 +697,6 @@ static struct clk_hw *sm8550_rpmh_clocks[] = {
 static const struct clk_rpmh_desc clk_rpmh_sm8550 = {
 	.clks = sm8550_rpmh_clocks,
 	.num_clks = ARRAY_SIZE(sm8550_rpmh_clocks),
-	.clka_optional = true,
 };
 
 static struct clk_hw *sm8650_rpmh_clocks[] = {
@@ -729,7 +728,6 @@ static struct clk_hw *sm8650_rpmh_clocks[] = {
 static const struct clk_rpmh_desc clk_rpmh_sm8650 = {
 	.clks = sm8650_rpmh_clocks,
 	.num_clks = ARRAY_SIZE(sm8650_rpmh_clocks),
-	.clka_optional = true,
 };
 
 static struct clk_hw *sc7280_rpmh_clocks[] = {
@@ -899,7 +897,6 @@ static struct clk_hw *sm8750_rpmh_clocks[] = {
 static const struct clk_rpmh_desc clk_rpmh_sm8750 = {
 	.clks = sm8750_rpmh_clocks,
 	.num_clks = ARRAY_SIZE(sm8750_rpmh_clocks),
-	.clka_optional = true,
 };
 
 static struct clk_hw *glymur_rpmh_clocks[] = {
@@ -984,6 +981,36 @@ static const struct clk_rpmh_desc clk_rpmh_nord = {
 	.num_clks = ARRAY_SIZE(nord_rpmh_clocks),
 };
 
+static struct clk_hw *hawi_rpmh_clocks[] = {
+	[RPMH_CXO_CLK]		= &clk_rpmh_bi_tcxo_div2.hw,
+	[RPMH_CXO_CLK_A]	= &clk_rpmh_bi_tcxo_div2_ao.hw,
+	[RPMH_DIV_CLK1]		= &clk_rpmh_clk11_a4_e0.hw,
+	[RPMH_LN_BB_CLK1]	= &clk_rpmh_clk6_a2_e0.hw,
+	[RPMH_LN_BB_CLK1_A]	= &clk_rpmh_clk6_a2_e0_ao.hw,
+	[RPMH_LN_BB_CLK2]	= &clk_rpmh_clk7_a4_e0.hw,
+	[RPMH_LN_BB_CLK2_A]	= &clk_rpmh_clk7_a4_e0_ao.hw,
+	[RPMH_LN_BB_CLK3]	= &clk_rpmh_clk8_a2_e0.hw,
+	[RPMH_LN_BB_CLK3_A]	= &clk_rpmh_clk8_a2_e0_ao.hw,
+	[RPMH_LN_BB_CLK4]	= &clk_rpmh_clk9_a2_e0.hw,
+	[RPMH_LN_BB_CLK4_A]	= &clk_rpmh_clk9_a2_e0_ao.hw,
+	[RPMH_RF_CLK1]		= &clk_rpmh_clk1_a1_e0.hw,
+	[RPMH_RF_CLK1_A]	= &clk_rpmh_clk1_a1_e0_ao.hw,
+	[RPMH_RF_CLK2]		= &clk_rpmh_clk2_a1_e0.hw,
+	[RPMH_RF_CLK2_A]	= &clk_rpmh_clk2_a1_e0_ao.hw,
+	[RPMH_RF_CLK3]		= &clk_rpmh_clk3_a2_e0.hw,
+	[RPMH_RF_CLK3_A]	= &clk_rpmh_clk3_a2_e0_ao.hw,
+	[RPMH_RF_CLK4]		= &clk_rpmh_clk4_a2_e0.hw,
+	[RPMH_RF_CLK4_A]	= &clk_rpmh_clk4_a2_e0_ao.hw,
+	[RPMH_RF_CLK5]		= &clk_rpmh_clk5_a2_e0.hw,
+	[RPMH_RF_CLK5_A]	= &clk_rpmh_clk5_a2_e0_ao.hw,
+	[RPMH_IPA_CLK]		= &clk_rpmh_ipa.hw,
+};
+
+static const struct clk_rpmh_desc clk_rpmh_hawi = {
+	.clks = hawi_rpmh_clocks,
+	.num_clks = ARRAY_SIZE(hawi_rpmh_clocks),
+};
+
 static struct clk_hw *of_clk_rpmh_hw_get(struct of_phandle_args *clkspec,
 					 void *data)
 {
@@ -1027,8 +1054,7 @@ static int clk_rpmh_probe(struct platform_device *pdev)
 		if (!res_addr) {
 			hw_clks[i] = NULL;
 
-			if (desc->clka_optional &&
-			    !strncmp(rpmh_clk->res_name, "clka", sizeof("clka") - 1))
+			if (rpmh_clk->res_addr == CLK_RPMH_VRM_EN_OFFSET)
 				continue;
 
 			dev_err(&pdev->dev, "missing RPMh resource address for %s\n",
@@ -1075,6 +1101,7 @@ static int clk_rpmh_probe(struct platform_device *pdev)
 static const struct of_device_id clk_rpmh_match_table[] = {
 	{ .compatible = "qcom,eliza-rpmh-clk", .data = &clk_rpmh_eliza},
 	{ .compatible = "qcom,glymur-rpmh-clk", .data = &clk_rpmh_glymur},
+	{ .compatible = "qcom,hawi-rpmh-clk", .data = &clk_rpmh_hawi},
 	{ .compatible = "qcom,kaanapali-rpmh-clk", .data = &clk_rpmh_kaanapali},
 	{ .compatible = "qcom,milos-rpmh-clk", .data = &clk_rpmh_milos},
 	{ .compatible = "qcom,nord-rpmh-clk", .data = &clk_rpmh_nord},
diff --git a/drivers/clk/qcom/dispcc-x1e80100.c b/drivers/clk/qcom/dispcc-x1e80100.c
index aa7fd43969f9c..1d7c569dc0825 100644
--- a/drivers/clk/qcom/dispcc-x1e80100.c
+++ b/drivers/clk/qcom/dispcc-x1e80100.c
@@ -580,7 +580,7 @@ static struct clk_rcg2 disp_cc_mdss_mdp_clk_src = {
 		.parent_data = disp_cc_parent_data_6,
 		.num_parents = ARRAY_SIZE(disp_cc_parent_data_6),
 		.flags = CLK_SET_RATE_PARENT,
-		.ops = &clk_rcg2_shared_ops,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
 	},
 };
 
@@ -1634,7 +1634,7 @@ static const struct regmap_config disp_cc_x1e80100_regmap_config = {
 	.reg_bits = 32,
 	.reg_stride = 4,
 	.val_bits = 32,
-	.max_register = 0x11008,
+	.max_register = 0xf004, /* 0x10000, 0x10004 and maybe others are for TZ */
 	.fast_io = true,
 };
 
diff --git a/drivers/clk/qcom/gcc-hawi.c b/drivers/clk/qcom/gcc-hawi.c
new file mode 100644
index 0000000000000..6dd07c772c29b
--- /dev/null
+++ b/drivers/clk/qcom/gcc-hawi.c
@@ -0,0 +1,3657 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,hawi-gcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "clk-regmap-phy-mux.h"
+#include "common.h"
+#include "gdsc.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO,
+	DT_BI_TCXO_AO,
+	DT_SLEEP_CLK,
+	DT_PCIE_0_PIPE_CLK,
+	DT_PCIE_1_PIPE_CLK,
+	DT_UFS_PHY_RX_SYMBOL_0_CLK,
+	DT_UFS_PHY_RX_SYMBOL_1_CLK,
+	DT_UFS_PHY_TX_SYMBOL_0_CLK,
+	DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+};
+
+enum {
+	P_BI_TCXO,
+	P_GCC_GPLL0_OUT_EVEN,
+	P_GCC_GPLL0_OUT_MAIN,
+	P_GCC_GPLL4_OUT_MAIN,
+	P_GCC_GPLL5_OUT_MAIN,
+	P_GCC_GPLL7_OUT_MAIN,
+	P_GCC_GPLL9_OUT_MAIN,
+	P_PCIE_0_PIPE_CLK,
+	P_PCIE_1_PIPE_CLK,
+	P_SLEEP_CLK,
+	P_UFS_PHY_RX_SYMBOL_0_CLK,
+	P_UFS_PHY_RX_SYMBOL_1_CLK,
+	P_UFS_PHY_TX_SYMBOL_0_CLK,
+	P_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+};
+
+static struct clk_alpha_pll gcc_gpll0 = {
+	.offset = 0x0,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.enable_reg = 0x52028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll0",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gcc_gpll0_out_even[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gcc_gpll0_out_even = {
+	.offset = 0x0,
+	.post_div_shift = 10,
+	.post_div_table = post_div_table_gcc_gpll0_out_even,
+	.num_post_div = ARRAY_SIZE(post_div_table_gcc_gpll0_out_even),
+	.width = 4,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gpll0_out_even",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_gpll0.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_taycan_eha_t_ops,
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll4 = {
+	.offset = 0x4000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.enable_reg = 0x52028,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll4",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eha_t_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll5 = {
+	.offset = 0x5000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.enable_reg = 0x52028,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll5",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eha_t_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll7 = {
+	.offset = 0x7000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.enable_reg = 0x52028,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll7",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eha_t_ops,
+		},
+	},
+};
+
+static struct clk_alpha_pll gcc_gpll9 = {
+	.offset = 0x9000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_TAYCAN_EHA_T],
+	.clkr = {
+		.enable_reg = 0x52028,
+		.enable_mask = BIT(9),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpll9",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_BI_TCXO,
+			},
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_taycan_eha_t_ops,
+		},
+	},
+};
+
+static const struct parent_map gcc_parent_map_0[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_0[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_1[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL7_OUT_MAIN, 2 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_1[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll7.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_2[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_SLEEP_CLK, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_2[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_3[] = {
+	{ P_BI_TCXO, 0 },
+};
+
+static const struct clk_parent_data gcc_parent_data_3[] = {
+	{ .index = DT_BI_TCXO },
+};
+
+static const struct parent_map gcc_parent_map_4[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_SLEEP_CLK, 5 },
+};
+
+static const struct clk_parent_data gcc_parent_data_4[] = {
+	{ .index = DT_BI_TCXO },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_5[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL5_OUT_MAIN, 3 },
+	{ P_GCC_GPLL4_OUT_MAIN, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_5[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll5.clkr.hw },
+	{ .hw = &gcc_gpll4.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_8[] = {
+	{ P_BI_TCXO, 0 },
+	{ P_GCC_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL9_OUT_MAIN, 2 },
+	{ P_GCC_GPLL4_OUT_MAIN, 5 },
+	{ P_GCC_GPLL0_OUT_EVEN, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_8[] = {
+	{ .index = DT_BI_TCXO },
+	{ .hw = &gcc_gpll0.clkr.hw },
+	{ .hw = &gcc_gpll9.clkr.hw },
+	{ .hw = &gcc_gpll4.clkr.hw },
+	{ .hw = &gcc_gpll0_out_even.clkr.hw },
+};
+
+static struct clk_regmap_phy_mux gcc_pcie_0_pipe_clk_src = {
+	.reg = 0x6b0a8,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_PCIE_0_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie_1_pipe_clk_src = {
+	.reg = 0x670a4,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_PCIE_1_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_0_clk_src = {
+	.reg = 0x77068,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_0_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_RX_SYMBOL_0_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_ufs_phy_rx_symbol_1_clk_src = {
+	.reg = 0x770ec,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_1_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_RX_SYMBOL_1_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_ufs_phy_tx_symbol_0_clk_src = {
+	.reg = 0x77058,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_tx_symbol_0_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_UFS_PHY_TX_SYMBOL_0_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_usb3_prim_phy_pipe_clk_src = {
+	.reg = 0x39074,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_USB3_PHY_WRAPPER_GCC_USB30_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_gp1_clk_src[] = {
+	F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_gp1_clk_src = {
+	.cmd_rcgr = 0x64004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp1_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp2_clk_src = {
+	.cmd_rcgr = 0x65004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp2_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_gp3_clk_src = {
+	.cmd_rcgr = 0x66004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_gp1_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_gp3_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_0_aux_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_0_aux_clk_src = {
+	.cmd_rcgr = 0x6b0ac,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_aux_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_0_phy_aux_clk_src = {
+	.cmd_rcgr = 0x6b0c4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_0_phy_rchng_clk_src[] = {
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_0_phy_rchng_clk_src = {
+	.cmd_rcgr = 0x6b08c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_phy_rchng_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_1_aux_clk_src = {
+	.cmd_rcgr = 0x670a8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_1_aux_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_1_phy_aux_clk_src = {
+	.cmd_rcgr = 0x670c0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_1_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie_1_phy_rchng_clk_src = {
+	.cmd_rcgr = 0x67088,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_phy_rchng_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_1_phy_rchng_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pdm2_clk_src[] = {
+	F(60000000, P_GCC_GPLL0_OUT_MAIN, 10, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pdm2_clk_src = {
+	.cmd_rcgr = 0x33010,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pdm2_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pdm2_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s0_clk_src = {
+	.cmd_rcgr = 0x17008,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s0_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s1_clk_src = {
+	.cmd_rcgr = 0x17024,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s1_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s2_clk_src = {
+	.cmd_rcgr = 0x17040,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s2_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s3_clk_src = {
+	.cmd_rcgr = 0x1705c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s3_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_i2c_s4_clk_src = {
+	.cmd_rcgr = 0x17078,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_i2c_s4_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_qspi_ref_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(120000000, P_GCC_GPLL0_OUT_EVEN, 2.5, 0, 0),
+	F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0),
+	F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_qspi_ref_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_qspi_ref_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_qspi_ref_clk_src = {
+	.cmd_rcgr = 0x188c0,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_qspi_ref_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_qspi_ref_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s0_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s0_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s0_clk_src = {
+	.cmd_rcgr = 0x18014,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s1_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s1_clk_src = {
+	.cmd_rcgr = 0x18150,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s1_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s3_clk_src = {
+	.cmd_rcgr = 0x182a0,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s3_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap1_s4_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(60000000, P_GCC_GPLL0_OUT_EVEN, 5, 0, 0),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(120000000, P_GCC_GPLL0_OUT_MAIN, 5, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s4_clk_src = {
+	.cmd_rcgr = 0x183dc,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s4_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s5_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s5_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s5_clk_src = {
+	.cmd_rcgr = 0x18518,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s5_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s6_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s6_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s6_clk_src = {
+	.cmd_rcgr = 0x18654,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s6_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap1_s7_clk_src_init = {
+	.name = "gcc_qupv3_wrap1_s7_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap1_s7_clk_src = {
+	.cmd_rcgr = 0x18790,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap1_s7_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s0_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s0_clk_src = {
+	.cmd_rcgr = 0x1e01c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s1_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s1_clk_src = {
+	.cmd_rcgr = 0x1e160,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s1_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s2_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s2_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s2_clk_src = {
+	.cmd_rcgr = 0x1e29c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s2_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s3_clk_src = {
+	.cmd_rcgr = 0x1e3d8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap2_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap2_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap2_s4_clk_src = {
+	.cmd_rcgr = 0x1e514,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap2_s4_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap3_qspi_ref_clk_src[] = {
+	F(7372800, P_GCC_GPLL0_OUT_EVEN, 1, 384, 15625),
+	F(14745600, P_GCC_GPLL0_OUT_EVEN, 1, 768, 15625),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	F(29491200, P_GCC_GPLL0_OUT_EVEN, 1, 1536, 15625),
+	F(32000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 75),
+	F(48000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 25),
+	F(51200000, P_GCC_GPLL0_OUT_EVEN, 1, 64, 375),
+	F(64000000, P_GCC_GPLL0_OUT_EVEN, 1, 16, 75),
+	F(66666667, P_GCC_GPLL0_OUT_MAIN, 9, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(80000000, P_GCC_GPLL0_OUT_EVEN, 1, 4, 15),
+	F(96000000, P_GCC_GPLL0_OUT_EVEN, 1, 8, 25),
+	F(100000000, P_GCC_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(102400000, P_GCC_GPLL0_OUT_EVEN, 1, 128, 375),
+	F(112000000, P_GCC_GPLL0_OUT_EVEN, 1, 28, 75),
+	F(117964800, P_GCC_GPLL0_OUT_EVEN, 1, 6144, 15625),
+	F(120000000, P_GCC_GPLL0_OUT_EVEN, 2.5, 0, 0),
+	F(150000000, P_GCC_GPLL0_OUT_EVEN, 2, 0, 0),
+	F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_qspi_ref_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_qspi_ref_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_qspi_ref_clk_src = {
+	.cmd_rcgr = 0xa8650,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap3_qspi_ref_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap3_qspi_ref_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s0_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s0_clk_src = {
+	.cmd_rcgr = 0xa8014,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s2_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s2_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s2_clk_src = {
+	.cmd_rcgr = 0xa8168,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s2_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s3_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s3_clk_src = {
+	.cmd_rcgr = 0xa82a4,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s4_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s4_clk_src = {
+	.cmd_rcgr = 0xa83e0,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s4_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap3_s5_clk_src_init = {
+	.name = "gcc_qupv3_wrap3_s5_clk_src",
+	.parent_data = gcc_parent_data_0,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap3_s5_clk_src = {
+	.cmd_rcgr = 0xa851c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap3_s5_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s0_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s0_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s0_clk_src = {
+	.cmd_rcgr = 0xa9014,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s0_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s1_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s1_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s1_clk_src = {
+	.cmd_rcgr = 0xa9150,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s1_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s2_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s2_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s2_clk_src = {
+	.cmd_rcgr = 0xa928c,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s4_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s2_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s3_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s3_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s3_clk_src = {
+	.cmd_rcgr = 0xa93c8,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s3_clk_src_init,
+};
+
+static struct clk_init_data gcc_qupv3_wrap4_s4_clk_src_init = {
+	.name = "gcc_qupv3_wrap4_s4_clk_src",
+	.parent_data = gcc_parent_data_1,
+	.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+	.flags = CLK_SET_RATE_PARENT,
+	.ops = &clk_rcg2_shared_no_init_park_ops,
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap4_s4_clk_src = {
+	.cmd_rcgr = 0xa9504,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_qupv3_wrap1_s0_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &gcc_qupv3_wrap4_s4_clk_src_init,
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc2_apps_clk_src[] = {
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(37500000, P_GCC_GPLL0_OUT_EVEN, 8, 0, 0),
+	F(50000000, P_GCC_GPLL0_OUT_EVEN, 6, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(202000000, P_GCC_GPLL9_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc2_apps_clk_src = {
+	.cmd_rcgr = 0x1401c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_8,
+	.freq_tbl = ftbl_gcc_sdcc2_apps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc2_apps_clk_src",
+		.parent_data = gcc_parent_data_8,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_8),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc4_apps_clk_src[] = {
+	F(400000, P_BI_TCXO, 12, 1, 4),
+	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc4_apps_clk_src = {
+	.cmd_rcgr = 0x1601c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_sdcc4_apps_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc4_apps_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_axi_clk_src[] = {
+	F(25000000, P_GCC_GPLL0_OUT_EVEN, 12, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(225000000, P_GCC_GPLL5_OUT_MAIN, 4, 0, 0),
+	F(450000000, P_GCC_GPLL5_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_axi_clk_src = {
+	.cmd_rcgr = 0x77034,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_ufs_phy_axi_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_axi_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_ice_core_clk_src[] = {
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(225000000, P_GCC_GPLL5_OUT_MAIN, 4, 0, 0),
+	F(450000000, P_GCC_GPLL5_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_ice_core_clk_src = {
+	.cmd_rcgr = 0x7708c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_ufs_phy_ice_core_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_ice_core_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_phy_aux_clk_src[] = {
+	F(9600000, P_BI_TCXO, 2, 0, 0),
+	F(19200000, P_BI_TCXO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_phy_aux_clk_src = {
+	.cmd_rcgr = 0x770c0,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_ufs_phy_phy_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_ufs_phy_unipro_5_core_clk_src[] = {
+	F(75000000, P_GCC_GPLL0_OUT_EVEN, 4, 0, 0),
+	F(100000000, P_GCC_GPLL0_OUT_EVEN, 3, 0, 0),
+	F(201500000, P_GCC_GPLL4_OUT_MAIN, 4, 0, 0),
+	F(403000000, P_GCC_GPLL4_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_ufs_phy_unipro_5_core_clk_src = {
+	.cmd_rcgr = 0x770a4,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_ufs_phy_unipro_5_core_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_ufs_phy_unipro_5_core_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb30_prim_master_clk_src[] = {
+	F(66666667, P_GCC_GPLL0_OUT_EVEN, 4.5, 0, 0),
+	F(133333333, P_GCC_GPLL0_OUT_MAIN, 4.5, 0, 0),
+	F(200000000, P_GCC_GPLL0_OUT_MAIN, 3, 0, 0),
+	F(240000000, P_GCC_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb30_prim_master_clk_src = {
+	.cmd_rcgr = 0x39034,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_usb30_prim_master_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_master_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb30_prim_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x3904c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_mock_utmi_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb3_prim_phy_aux_clk_src = {
+	.cmd_rcgr = 0x39078,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_pcie_0_aux_clk_src,
+	.hw_clk_ctrl = true,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb3_prim_phy_aux_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_rcg2_shared_no_init_park_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_pcie_0_pipe_div_clk_src = {
+	.reg = 0x6b0a4,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_0_pipe_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_pcie_0_pipe_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_pcie_1_pipe_div_clk_src = {
+	.reg = 0x670a0,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_1_pipe_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_pcie_1_pipe_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_qupv3_wrap1_s2_clk_src = {
+	.reg = 0x1828c,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap1_s2_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_qupv3_wrap1_qspi_ref_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_qupv3_wrap3_s1_clk_src = {
+	.reg = 0xa8154,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap3_s1_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_qupv3_wrap3_qspi_ref_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_usb30_prim_mock_utmi_postdiv_clk_src = {
+	.reg = 0x39064,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb30_prim_mock_utmi_postdiv_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_usb30_prim_mock_utmi_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch gcc_aggre_noc_pcie_axi_clk = {
+	.halt_reg = 0x10068,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x10068,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(24),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_aggre_noc_pcie_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_stardustnoc_usb3_prim_axi_clk = {
+	.halt_reg = 0x39094,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x39094,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x39094,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_aggre_stardustnoc_usb3_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_aggre_ufs_phy_axi_clk = {
+	.halt_reg = 0x770f0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x770f0,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x770f0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_aggre_ufs_phy_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_boot_rom_ahb_clk = {
+	.halt_reg = 0x38004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x38004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(18),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_boot_rom_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camera_hf_axi_clk = {
+	.halt_reg = 0x26014,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x26014,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x26014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camera_hf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_camera_sf_axi_clk = {
+	.halt_reg = 0x2601c,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x2601c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x2601c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_camera_sf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_pcie_anoc_ahb_clk = {
+	.halt_reg = 0x10050,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x10050,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cfg_noc_pcie_anoc_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cfg_noc_usb3_prim_axi_clk = {
+	.halt_reg = 0x39090,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x39090,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x39090,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cfg_noc_usb3_prim_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cnoc_pcie_sf_axi_clk = {
+	.halt_reg = 0x10058,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(6),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cnoc_pcie_sf_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_eva_axi0_clk = {
+	.halt_reg = 0x9f008,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x9f008,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x9f008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_eva_axi0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_eva_axi0c_clk = {
+	.halt_reg = 0x9f010,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x9f010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x9f010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_eva_axi0c_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp1_clk = {
+	.halt_reg = 0x64000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x64000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp2_clk = {
+	.halt_reg = 0x65000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x65000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gp3_clk = {
+	.halt_reg = 0x66000,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x66000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gp3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gp3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gemnoc_gfx_clk = {
+	.halt_reg = 0x71010,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x71010,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x71010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gemnoc_gfx_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gpll0_clk_src",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gpll0.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_gpll0_div_clk_src = {
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x52000,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_gpll0_div_clk_src",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_gpll0_out_even.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_gpu_smmu_vote_clk = {
+	.halt_reg = 0x7d000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_gpu_smmu_vote_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mmu_tcu_vote_clk = {
+	.halt_reg = 0x7d02c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x7d02c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_mmu_tcu_vote_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_aux_clk = {
+	.halt_reg = 0x6b044,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b044,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_cfg_ahb_clk = {
+	.halt_reg = 0x6b040,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b040,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(3),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_mstr_axi_clk = {
+	.halt_reg = 0x6b030,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x6b030,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_phy_aux_clk = {
+	.halt_reg = 0x6b054,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b054,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_phy_rchng_clk = {
+	.halt_reg = 0x6b084,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b084,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_phy_rchng_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_phy_rchng_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_pipe_clk = {
+	.halt_reg = 0x6b074,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x6b074,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_pipe_div2_clk = {
+	.halt_reg = 0x6b064,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x6b064,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(6),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_pipe_div2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_0_pipe_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_axi_clk = {
+	.halt_reg = 0x6b020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b020,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_0_slv_q2a_axi_clk = {
+	.halt_reg = 0x6b01c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6b01c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_0_slv_q2a_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_aux_clk = {
+	.halt_reg = 0x67040,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_1_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_cfg_ahb_clk = {
+	.halt_reg = 0x6703c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6703c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(9),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_cfg_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_mstr_axi_clk = {
+	.halt_reg = 0x6702c,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x6702c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(17),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_mstr_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_phy_aux_clk = {
+	.halt_reg = 0x67050,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_1_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_phy_rchng_clk = {
+	.halt_reg = 0x67080,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(26),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_phy_rchng_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_1_phy_rchng_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_pipe_clk = {
+	.halt_reg = 0x67070,
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(17),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_1_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_pipe_div2_clk = {
+	.halt_reg = 0x67060,
+	.halt_check = BRANCH_HALT_SKIP,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_pipe_div2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_1_pipe_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_slv_axi_clk = {
+	.halt_reg = 0x6701c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x6701c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_slv_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie_1_slv_q2a_axi_clk = {
+	.halt_reg = 0x67018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie_1_slv_q2a_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm2_clk = {
+	.halt_reg = 0x3300c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3300c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pdm2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_ahb_clk = {
+	.halt_reg = 0x33004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x33004,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x33004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pdm_xo4_clk = {
+	.halt_reg = 0x33008,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x33008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pdm_xo4_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_core_clk = {
+	.halt_reg = 0x23004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s0_clk = {
+	.halt_reg = 0x17004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s1_clk = {
+	.halt_reg = 0x17020,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(11),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s2_clk = {
+	.halt_reg = 0x1703c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(12),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s3_clk = {
+	.halt_reg = 0x17058,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(13),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s4_clk = {
+	.halt_reg = 0x17074,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_i2c_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_i2c_s_ahb_clk = {
+	.halt_reg = 0x23000,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23000,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_i2c_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_core_2x_clk = {
+	.halt_reg = 0x2315c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(18),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_core_clk = {
+	.halt_reg = 0x23148,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(19),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_qspi_ref_clk = {
+	.halt_reg = 0x188bc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(29),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_qspi_ref_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_qspi_ref_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s0_clk = {
+	.halt_reg = 0x18004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(22),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s1_clk = {
+	.halt_reg = 0x18140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(23),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s2_clk = {
+	.halt_reg = 0x1827c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(24),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s3_clk = {
+	.halt_reg = 0x18290,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(25),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s4_clk = {
+	.halt_reg = 0x183cc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(26),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s5_clk = {
+	.halt_reg = 0x18508,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(27),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s6_clk = {
+	.halt_reg = 0x18644,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(28),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s6_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s6_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap1_s7_clk = {
+	.halt_reg = 0x18780,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap1_s7_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap1_s7_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_core_2x_clk = {
+	.halt_reg = 0x232b4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(3),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_core_clk = {
+	.halt_reg = 0x232a0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s0_clk = {
+	.halt_reg = 0x1e004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s1_clk = {
+	.halt_reg = 0x1e148,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(5),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s2_clk = {
+	.halt_reg = 0x1e28c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(6),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s3_clk = {
+	.halt_reg = 0x1e3c8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(7),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap2_s4_clk = {
+	.halt_reg = 0x1e504,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap2_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap2_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_core_2x_clk = {
+	.halt_reg = 0x2340c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(11),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_core_clk = {
+	.halt_reg = 0x233f8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(10),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_qspi_ref_clk = {
+	.halt_reg = 0xa8648,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(25),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_qspi_ref_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_qspi_ref_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s0_clk = {
+	.halt_reg = 0xa8004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(12),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s1_clk = {
+	.halt_reg = 0xa8140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(13),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s2_clk = {
+	.halt_reg = 0xa8158,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s3_clk = {
+	.halt_reg = 0xa8294,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(15),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s4_clk = {
+	.halt_reg = 0xa83d0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(16),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap3_s5_clk = {
+	.halt_reg = 0xa850c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(17),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap3_s5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap3_s5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_core_2x_clk = {
+	.halt_reg = 0x23564,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(25),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_core_2x_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_core_clk = {
+	.halt_reg = 0x23550,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(24),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_core_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s0_clk = {
+	.halt_reg = 0xa9004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(26),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s1_clk = {
+	.halt_reg = 0xa9140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(27),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s2_clk = {
+	.halt_reg = 0xa927c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(28),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s3_clk = {
+	.halt_reg = 0xa93b8,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(29),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap4_s4_clk = {
+	.halt_reg = 0xa94f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(30),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap4_s4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap4_s4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_1_m_axi_clk = {
+	.halt_reg = 0x23140,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23140,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(20),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_1_m_axi_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_1_s_ahb_clk = {
+	.halt_reg = 0x23144,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23144,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52008,
+		.enable_mask = BIT(21),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_1_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_2_m_ahb_clk = {
+	.halt_reg = 0x23298,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23298,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_2_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_2_s_ahb_clk = {
+	.halt_reg = 0x2329c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x2329c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52010,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_2_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_3_m_ahb_clk = {
+	.halt_reg = 0x233f0,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x233f0,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(8),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_3_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_3_s_ahb_clk = {
+	.halt_reg = 0x233f4,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x233f4,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(9),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_3_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_4_m_ahb_clk = {
+	.halt_reg = 0x23548,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x23548,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(22),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_4_m_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_4_s_ahb_clk = {
+	.halt_reg = 0x2354c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x2354c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x52018,
+		.enable_mask = BIT(23),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_4_s_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_ahb_clk = {
+	.halt_reg = 0x14014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc2_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc2_apps_clk = {
+	.halt_reg = 0x14004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x14004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc2_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc2_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_ahb_clk = {
+	.halt_reg = 0x16014,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x16014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc4_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc4_apps_clk = {
+	.halt_reg = 0x16004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x16004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc4_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc4_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_ahb_clk = {
+	.halt_reg = 0x77028,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x77028,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x77028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_ahb_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_axi_clk = {
+	.halt_reg = 0x77018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x77018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x77018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_axi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_axi_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_ice_core_clk = {
+	.halt_reg = 0x7707c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x7707c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7707c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_ice_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_ice_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_phy_aux_clk = {
+	.halt_reg = 0x770bc,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x770bc,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x770bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_rx_symbol_0_clk = {
+	.halt_reg = 0x77030,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x77030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_rx_symbol_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_rx_symbol_1_clk = {
+	.halt_reg = 0x770d8,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x770d8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_rx_symbol_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_rx_symbol_1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_tx_symbol_0_clk = {
+	.halt_reg = 0x7702c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x7702c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_tx_symbol_0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_tx_symbol_0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_ufs_phy_unipro_5_core_clk = {
+	.halt_reg = 0x7706c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.hwcg_reg = 0x7706c,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x7706c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_ufs_phy_unipro_5_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_ufs_phy_unipro_5_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_master_clk = {
+	.halt_reg = 0x39018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_master_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_mock_utmi_clk = {
+	.halt_reg = 0x3902c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3902c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_mock_utmi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb30_prim_sleep_clk = {
+	.halt_reg = 0x39028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb30_prim_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_aux_clk = {
+	.halt_reg = 0x39068,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x39068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_com_aux_clk = {
+	.halt_reg = 0x3906c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3906c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_com_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb3_prim_phy_pipe_clk = {
+	.halt_reg = 0x39070,
+	.halt_check = BRANCH_HALT_DELAY,
+	.hwcg_reg = 0x39070,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x39070,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb3_prim_phy_pipe_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb3_prim_phy_pipe_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_axi0_clk = {
+	.halt_reg = 0x32018,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x32018,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x32018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_axi0_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_video_axi0c_clk = {
+	.halt_reg = 0x32020,
+	.halt_check = BRANCH_HALT_SKIP,
+	.hwcg_reg = 0x32020,
+	.hwcg_bit = 1,
+	.clkr = {
+		.enable_reg = 0x32020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_video_axi0c_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct gdsc gcc_pcie_0_gdsc = {
+	.gdscr = 0x6b004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.collapse_ctrl = 0x52154,
+	.collapse_mask = BIT(0),
+	.pd = {
+		.name = "gcc_pcie_0_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_pcie_0_phy_gdsc = {
+	.gdscr = 0x6c000,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.collapse_ctrl = 0x52154,
+	.collapse_mask = BIT(1),
+	.pd = {
+		.name = "gcc_pcie_0_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_pcie_1_gdsc = {
+	.gdscr = 0x67004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.collapse_ctrl = 0x5214c,
+	.collapse_mask = BIT(2),
+	.pd = {
+		.name = "gcc_pcie_1_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_pcie_1_phy_gdsc = {
+	.gdscr = 0x68000,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.collapse_ctrl = 0x5214c,
+	.collapse_mask = BIT(3),
+	.pd = {
+		.name = "gcc_pcie_1_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE | VOTABLE,
+};
+
+static struct gdsc gcc_ufs_mem_phy_gdsc = {
+	.gdscr = 0x9e000,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "gcc_ufs_mem_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_ufs_phy_gdsc = {
+	.gdscr = 0x77004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_ufs_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_usb30_prim_gdsc = {
+	.gdscr = 0x39004,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0xf,
+	.pd = {
+		.name = "gcc_usb30_prim_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct gdsc gcc_usb3_phy_gdsc = {
+	.gdscr = 0x50018,
+	.en_rest_wait_val = 0x2,
+	.en_few_wait_val = 0x2,
+	.clk_dis_wait_val = 0x2,
+	.pd = {
+		.name = "gcc_usb3_phy_gdsc",
+	},
+	.pwrsts = PWRSTS_OFF_ON,
+	.flags = POLL_CFG_GDSCR | RETAIN_FF_ENABLE,
+};
+
+static struct clk_regmap *gcc_hawi_clocks[] = {
+	[GCC_AGGRE_NOC_PCIE_AXI_CLK] = &gcc_aggre_noc_pcie_axi_clk.clkr,
+	[GCC_AGGRE_STARDUSTNOC_USB3_PRIM_AXI_CLK] = &gcc_aggre_stardustnoc_usb3_prim_axi_clk.clkr,
+	[GCC_AGGRE_UFS_PHY_AXI_CLK] = &gcc_aggre_ufs_phy_axi_clk.clkr,
+	[GCC_BOOT_ROM_AHB_CLK] = &gcc_boot_rom_ahb_clk.clkr,
+	[GCC_CAMERA_HF_AXI_CLK] = &gcc_camera_hf_axi_clk.clkr,
+	[GCC_CAMERA_SF_AXI_CLK] = &gcc_camera_sf_axi_clk.clkr,
+	[GCC_CFG_NOC_PCIE_ANOC_AHB_CLK] = &gcc_cfg_noc_pcie_anoc_ahb_clk.clkr,
+	[GCC_CFG_NOC_USB3_PRIM_AXI_CLK] = &gcc_cfg_noc_usb3_prim_axi_clk.clkr,
+	[GCC_CNOC_PCIE_SF_AXI_CLK] = &gcc_cnoc_pcie_sf_axi_clk.clkr,
+	[GCC_EVA_AXI0_CLK] = &gcc_eva_axi0_clk.clkr,
+	[GCC_EVA_AXI0C_CLK] = &gcc_eva_axi0c_clk.clkr,
+	[GCC_GP1_CLK] = &gcc_gp1_clk.clkr,
+	[GCC_GP1_CLK_SRC] = &gcc_gp1_clk_src.clkr,
+	[GCC_GP2_CLK] = &gcc_gp2_clk.clkr,
+	[GCC_GP2_CLK_SRC] = &gcc_gp2_clk_src.clkr,
+	[GCC_GP3_CLK] = &gcc_gp3_clk.clkr,
+	[GCC_GP3_CLK_SRC] = &gcc_gp3_clk_src.clkr,
+	[GCC_GPLL0] = &gcc_gpll0.clkr,
+	[GCC_GPLL0_OUT_EVEN] = &gcc_gpll0_out_even.clkr,
+	[GCC_GPLL4] = &gcc_gpll4.clkr,
+	[GCC_GPLL5] = &gcc_gpll5.clkr,
+	[GCC_GPLL7] = &gcc_gpll7.clkr,
+	[GCC_GPLL9] = &gcc_gpll9.clkr,
+	[GCC_GPU_GEMNOC_GFX_CLK] = &gcc_gpu_gemnoc_gfx_clk.clkr,
+	[GCC_GPU_GPLL0_CLK_SRC] = &gcc_gpu_gpll0_clk_src.clkr,
+	[GCC_GPU_GPLL0_DIV_CLK_SRC] = &gcc_gpu_gpll0_div_clk_src.clkr,
+	[GCC_GPU_SMMU_VOTE_CLK] = &gcc_gpu_smmu_vote_clk.clkr,
+	[GCC_MMU_TCU_VOTE_CLK] = &gcc_mmu_tcu_vote_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK] = &gcc_pcie_0_aux_clk.clkr,
+	[GCC_PCIE_0_AUX_CLK_SRC] = &gcc_pcie_0_aux_clk_src.clkr,
+	[GCC_PCIE_0_CFG_AHB_CLK] = &gcc_pcie_0_cfg_ahb_clk.clkr,
+	[GCC_PCIE_0_MSTR_AXI_CLK] = &gcc_pcie_0_mstr_axi_clk.clkr,
+	[GCC_PCIE_0_PHY_AUX_CLK] = &gcc_pcie_0_phy_aux_clk.clkr,
+	[GCC_PCIE_0_PHY_AUX_CLK_SRC] = &gcc_pcie_0_phy_aux_clk_src.clkr,
+	[GCC_PCIE_0_PHY_RCHNG_CLK] = &gcc_pcie_0_phy_rchng_clk.clkr,
+	[GCC_PCIE_0_PHY_RCHNG_CLK_SRC] = &gcc_pcie_0_phy_rchng_clk_src.clkr,
+	[GCC_PCIE_0_PIPE_CLK] = &gcc_pcie_0_pipe_clk.clkr,
+	[GCC_PCIE_0_PIPE_CLK_SRC] = &gcc_pcie_0_pipe_clk_src.clkr,
+	[GCC_PCIE_0_PIPE_DIV2_CLK] = &gcc_pcie_0_pipe_div2_clk.clkr,
+	[GCC_PCIE_0_PIPE_DIV_CLK_SRC] = &gcc_pcie_0_pipe_div_clk_src.clkr,
+	[GCC_PCIE_0_SLV_AXI_CLK] = &gcc_pcie_0_slv_axi_clk.clkr,
+	[GCC_PCIE_0_SLV_Q2A_AXI_CLK] = &gcc_pcie_0_slv_q2a_axi_clk.clkr,
+	[GCC_PCIE_1_AUX_CLK] = &gcc_pcie_1_aux_clk.clkr,
+	[GCC_PCIE_1_AUX_CLK_SRC] = &gcc_pcie_1_aux_clk_src.clkr,
+	[GCC_PCIE_1_CFG_AHB_CLK] = &gcc_pcie_1_cfg_ahb_clk.clkr,
+	[GCC_PCIE_1_MSTR_AXI_CLK] = &gcc_pcie_1_mstr_axi_clk.clkr,
+	[GCC_PCIE_1_PHY_AUX_CLK] = &gcc_pcie_1_phy_aux_clk.clkr,
+	[GCC_PCIE_1_PHY_AUX_CLK_SRC] = &gcc_pcie_1_phy_aux_clk_src.clkr,
+	[GCC_PCIE_1_PHY_RCHNG_CLK] = &gcc_pcie_1_phy_rchng_clk.clkr,
+	[GCC_PCIE_1_PHY_RCHNG_CLK_SRC] = &gcc_pcie_1_phy_rchng_clk_src.clkr,
+	[GCC_PCIE_1_PIPE_CLK] = &gcc_pcie_1_pipe_clk.clkr,
+	[GCC_PCIE_1_PIPE_CLK_SRC] = &gcc_pcie_1_pipe_clk_src.clkr,
+	[GCC_PCIE_1_PIPE_DIV2_CLK] = &gcc_pcie_1_pipe_div2_clk.clkr,
+	[GCC_PCIE_1_PIPE_DIV_CLK_SRC] = &gcc_pcie_1_pipe_div_clk_src.clkr,
+	[GCC_PCIE_1_SLV_AXI_CLK] = &gcc_pcie_1_slv_axi_clk.clkr,
+	[GCC_PCIE_1_SLV_Q2A_AXI_CLK] = &gcc_pcie_1_slv_q2a_axi_clk.clkr,
+	[GCC_PDM2_CLK] = &gcc_pdm2_clk.clkr,
+	[GCC_PDM2_CLK_SRC] = &gcc_pdm2_clk_src.clkr,
+	[GCC_PDM_AHB_CLK] = &gcc_pdm_ahb_clk.clkr,
+	[GCC_PDM_XO4_CLK] = &gcc_pdm_xo4_clk.clkr,
+	[GCC_QUPV3_I2C_CORE_CLK] = &gcc_qupv3_i2c_core_clk.clkr,
+	[GCC_QUPV3_I2C_S0_CLK] = &gcc_qupv3_i2c_s0_clk.clkr,
+	[GCC_QUPV3_I2C_S0_CLK_SRC] = &gcc_qupv3_i2c_s0_clk_src.clkr,
+	[GCC_QUPV3_I2C_S1_CLK] = &gcc_qupv3_i2c_s1_clk.clkr,
+	[GCC_QUPV3_I2C_S1_CLK_SRC] = &gcc_qupv3_i2c_s1_clk_src.clkr,
+	[GCC_QUPV3_I2C_S2_CLK] = &gcc_qupv3_i2c_s2_clk.clkr,
+	[GCC_QUPV3_I2C_S2_CLK_SRC] = &gcc_qupv3_i2c_s2_clk_src.clkr,
+	[GCC_QUPV3_I2C_S3_CLK] = &gcc_qupv3_i2c_s3_clk.clkr,
+	[GCC_QUPV3_I2C_S3_CLK_SRC] = &gcc_qupv3_i2c_s3_clk_src.clkr,
+	[GCC_QUPV3_I2C_S4_CLK] = &gcc_qupv3_i2c_s4_clk.clkr,
+	[GCC_QUPV3_I2C_S4_CLK_SRC] = &gcc_qupv3_i2c_s4_clk_src.clkr,
+	[GCC_QUPV3_I2C_S_AHB_CLK] = &gcc_qupv3_i2c_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP1_CORE_2X_CLK] = &gcc_qupv3_wrap1_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP1_CORE_CLK] = &gcc_qupv3_wrap1_core_clk.clkr,
+	[GCC_QUPV3_WRAP1_QSPI_REF_CLK] = &gcc_qupv3_wrap1_qspi_ref_clk.clkr,
+	[GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC] = &gcc_qupv3_wrap1_qspi_ref_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S0_CLK] = &gcc_qupv3_wrap1_s0_clk.clkr,
+	[GCC_QUPV3_WRAP1_S0_CLK_SRC] = &gcc_qupv3_wrap1_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S1_CLK] = &gcc_qupv3_wrap1_s1_clk.clkr,
+	[GCC_QUPV3_WRAP1_S1_CLK_SRC] = &gcc_qupv3_wrap1_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S2_CLK] = &gcc_qupv3_wrap1_s2_clk.clkr,
+	[GCC_QUPV3_WRAP1_S2_CLK_SRC] = &gcc_qupv3_wrap1_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S3_CLK] = &gcc_qupv3_wrap1_s3_clk.clkr,
+	[GCC_QUPV3_WRAP1_S3_CLK_SRC] = &gcc_qupv3_wrap1_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S4_CLK] = &gcc_qupv3_wrap1_s4_clk.clkr,
+	[GCC_QUPV3_WRAP1_S4_CLK_SRC] = &gcc_qupv3_wrap1_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S5_CLK] = &gcc_qupv3_wrap1_s5_clk.clkr,
+	[GCC_QUPV3_WRAP1_S5_CLK_SRC] = &gcc_qupv3_wrap1_s5_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S6_CLK] = &gcc_qupv3_wrap1_s6_clk.clkr,
+	[GCC_QUPV3_WRAP1_S6_CLK_SRC] = &gcc_qupv3_wrap1_s6_clk_src.clkr,
+	[GCC_QUPV3_WRAP1_S7_CLK] = &gcc_qupv3_wrap1_s7_clk.clkr,
+	[GCC_QUPV3_WRAP1_S7_CLK_SRC] = &gcc_qupv3_wrap1_s7_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_CORE_2X_CLK] = &gcc_qupv3_wrap2_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP2_CORE_CLK] = &gcc_qupv3_wrap2_core_clk.clkr,
+	[GCC_QUPV3_WRAP2_S0_CLK] = &gcc_qupv3_wrap2_s0_clk.clkr,
+	[GCC_QUPV3_WRAP2_S0_CLK_SRC] = &gcc_qupv3_wrap2_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S1_CLK] = &gcc_qupv3_wrap2_s1_clk.clkr,
+	[GCC_QUPV3_WRAP2_S1_CLK_SRC] = &gcc_qupv3_wrap2_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S2_CLK] = &gcc_qupv3_wrap2_s2_clk.clkr,
+	[GCC_QUPV3_WRAP2_S2_CLK_SRC] = &gcc_qupv3_wrap2_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S3_CLK] = &gcc_qupv3_wrap2_s3_clk.clkr,
+	[GCC_QUPV3_WRAP2_S3_CLK_SRC] = &gcc_qupv3_wrap2_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP2_S4_CLK] = &gcc_qupv3_wrap2_s4_clk.clkr,
+	[GCC_QUPV3_WRAP2_S4_CLK_SRC] = &gcc_qupv3_wrap2_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_CORE_2X_CLK] = &gcc_qupv3_wrap3_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP3_CORE_CLK] = &gcc_qupv3_wrap3_core_clk.clkr,
+	[GCC_QUPV3_WRAP3_QSPI_REF_CLK] = &gcc_qupv3_wrap3_qspi_ref_clk.clkr,
+	[GCC_QUPV3_WRAP3_QSPI_REF_CLK_SRC] = &gcc_qupv3_wrap3_qspi_ref_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S0_CLK] = &gcc_qupv3_wrap3_s0_clk.clkr,
+	[GCC_QUPV3_WRAP3_S0_CLK_SRC] = &gcc_qupv3_wrap3_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S1_CLK] = &gcc_qupv3_wrap3_s1_clk.clkr,
+	[GCC_QUPV3_WRAP3_S1_CLK_SRC] = &gcc_qupv3_wrap3_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S2_CLK] = &gcc_qupv3_wrap3_s2_clk.clkr,
+	[GCC_QUPV3_WRAP3_S2_CLK_SRC] = &gcc_qupv3_wrap3_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S3_CLK] = &gcc_qupv3_wrap3_s3_clk.clkr,
+	[GCC_QUPV3_WRAP3_S3_CLK_SRC] = &gcc_qupv3_wrap3_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S4_CLK] = &gcc_qupv3_wrap3_s4_clk.clkr,
+	[GCC_QUPV3_WRAP3_S4_CLK_SRC] = &gcc_qupv3_wrap3_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP3_S5_CLK] = &gcc_qupv3_wrap3_s5_clk.clkr,
+	[GCC_QUPV3_WRAP3_S5_CLK_SRC] = &gcc_qupv3_wrap3_s5_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_CORE_2X_CLK] = &gcc_qupv3_wrap4_core_2x_clk.clkr,
+	[GCC_QUPV3_WRAP4_CORE_CLK] = &gcc_qupv3_wrap4_core_clk.clkr,
+	[GCC_QUPV3_WRAP4_S0_CLK] = &gcc_qupv3_wrap4_s0_clk.clkr,
+	[GCC_QUPV3_WRAP4_S0_CLK_SRC] = &gcc_qupv3_wrap4_s0_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_S1_CLK] = &gcc_qupv3_wrap4_s1_clk.clkr,
+	[GCC_QUPV3_WRAP4_S1_CLK_SRC] = &gcc_qupv3_wrap4_s1_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_S2_CLK] = &gcc_qupv3_wrap4_s2_clk.clkr,
+	[GCC_QUPV3_WRAP4_S2_CLK_SRC] = &gcc_qupv3_wrap4_s2_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_S3_CLK] = &gcc_qupv3_wrap4_s3_clk.clkr,
+	[GCC_QUPV3_WRAP4_S3_CLK_SRC] = &gcc_qupv3_wrap4_s3_clk_src.clkr,
+	[GCC_QUPV3_WRAP4_S4_CLK] = &gcc_qupv3_wrap4_s4_clk.clkr,
+	[GCC_QUPV3_WRAP4_S4_CLK_SRC] = &gcc_qupv3_wrap4_s4_clk_src.clkr,
+	[GCC_QUPV3_WRAP_1_M_AXI_CLK] = &gcc_qupv3_wrap_1_m_axi_clk.clkr,
+	[GCC_QUPV3_WRAP_1_S_AHB_CLK] = &gcc_qupv3_wrap_1_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_2_M_AHB_CLK] = &gcc_qupv3_wrap_2_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_2_S_AHB_CLK] = &gcc_qupv3_wrap_2_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_3_M_AHB_CLK] = &gcc_qupv3_wrap_3_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_3_S_AHB_CLK] = &gcc_qupv3_wrap_3_s_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_4_M_AHB_CLK] = &gcc_qupv3_wrap_4_m_ahb_clk.clkr,
+	[GCC_QUPV3_WRAP_4_S_AHB_CLK] = &gcc_qupv3_wrap_4_s_ahb_clk.clkr,
+	[GCC_SDCC2_AHB_CLK] = &gcc_sdcc2_ahb_clk.clkr,
+	[GCC_SDCC2_APPS_CLK] = &gcc_sdcc2_apps_clk.clkr,
+	[GCC_SDCC2_APPS_CLK_SRC] = &gcc_sdcc2_apps_clk_src.clkr,
+	[GCC_SDCC4_AHB_CLK] = &gcc_sdcc4_ahb_clk.clkr,
+	[GCC_SDCC4_APPS_CLK] = &gcc_sdcc4_apps_clk.clkr,
+	[GCC_SDCC4_APPS_CLK_SRC] = &gcc_sdcc4_apps_clk_src.clkr,
+	[GCC_UFS_PHY_AHB_CLK] = &gcc_ufs_phy_ahb_clk.clkr,
+	[GCC_UFS_PHY_AXI_CLK] = &gcc_ufs_phy_axi_clk.clkr,
+	[GCC_UFS_PHY_AXI_CLK_SRC] = &gcc_ufs_phy_axi_clk_src.clkr,
+	[GCC_UFS_PHY_ICE_CORE_CLK] = &gcc_ufs_phy_ice_core_clk.clkr,
+	[GCC_UFS_PHY_ICE_CORE_CLK_SRC] = &gcc_ufs_phy_ice_core_clk_src.clkr,
+	[GCC_UFS_PHY_PHY_AUX_CLK] = &gcc_ufs_phy_phy_aux_clk.clkr,
+	[GCC_UFS_PHY_PHY_AUX_CLK_SRC] = &gcc_ufs_phy_phy_aux_clk_src.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_0_CLK] = &gcc_ufs_phy_rx_symbol_0_clk.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_rx_symbol_0_clk_src.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_1_CLK] = &gcc_ufs_phy_rx_symbol_1_clk.clkr,
+	[GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC] = &gcc_ufs_phy_rx_symbol_1_clk_src.clkr,
+	[GCC_UFS_PHY_TX_SYMBOL_0_CLK] = &gcc_ufs_phy_tx_symbol_0_clk.clkr,
+	[GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC] = &gcc_ufs_phy_tx_symbol_0_clk_src.clkr,
+	[GCC_UFS_PHY_UNIPRO_5_CORE_CLK] = &gcc_ufs_phy_unipro_5_core_clk.clkr,
+	[GCC_UFS_PHY_UNIPRO_5_CORE_CLK_SRC] = &gcc_ufs_phy_unipro_5_core_clk_src.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK] = &gcc_usb30_prim_master_clk.clkr,
+	[GCC_USB30_PRIM_MASTER_CLK_SRC] = &gcc_usb30_prim_master_clk_src.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK] = &gcc_usb30_prim_mock_utmi_clk.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC] = &gcc_usb30_prim_mock_utmi_clk_src.clkr,
+	[GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC] = &gcc_usb30_prim_mock_utmi_postdiv_clk_src.clkr,
+	[GCC_USB30_PRIM_SLEEP_CLK] = &gcc_usb30_prim_sleep_clk.clkr,
+	[GCC_USB3_PRIM_PHY_AUX_CLK] = &gcc_usb3_prim_phy_aux_clk.clkr,
+	[GCC_USB3_PRIM_PHY_AUX_CLK_SRC] = &gcc_usb3_prim_phy_aux_clk_src.clkr,
+	[GCC_USB3_PRIM_PHY_COM_AUX_CLK] = &gcc_usb3_prim_phy_com_aux_clk.clkr,
+	[GCC_USB3_PRIM_PHY_PIPE_CLK] = &gcc_usb3_prim_phy_pipe_clk.clkr,
+	[GCC_USB3_PRIM_PHY_PIPE_CLK_SRC] = &gcc_usb3_prim_phy_pipe_clk_src.clkr,
+	[GCC_VIDEO_AXI0_CLK] = &gcc_video_axi0_clk.clkr,
+	[GCC_VIDEO_AXI0C_CLK] = &gcc_video_axi0c_clk.clkr,
+};
+
+static struct gdsc *gcc_hawi_gdscs[] = {
+	[GCC_PCIE_0_GDSC] = &gcc_pcie_0_gdsc,
+	[GCC_PCIE_0_PHY_GDSC] = &gcc_pcie_0_phy_gdsc,
+	[GCC_PCIE_1_GDSC] = &gcc_pcie_1_gdsc,
+	[GCC_PCIE_1_PHY_GDSC] = &gcc_pcie_1_phy_gdsc,
+	[GCC_UFS_MEM_PHY_GDSC] = &gcc_ufs_mem_phy_gdsc,
+	[GCC_UFS_PHY_GDSC] = &gcc_ufs_phy_gdsc,
+	[GCC_USB30_PRIM_GDSC] = &gcc_usb30_prim_gdsc,
+	[GCC_USB3_PHY_GDSC] = &gcc_usb3_phy_gdsc,
+};
+
+static const struct qcom_reset_map gcc_hawi_resets[] = {
+	[GCC_CAMERA_BCR] = { 0x26000 },
+	[GCC_EVA_AXI0_CLK_ARES] = { 0x9f008, 2 },
+	[GCC_EVA_AXI0C_CLK_ARES] = { 0x9f010, 2 },
+	[GCC_EVA_BCR] = { 0x9f000 },
+	[GCC_GPU_BCR] = { 0x71000 },
+	[GCC_PCIE_0_BCR] = { 0x6b000 },
+	[GCC_PCIE_0_LINK_DOWN_BCR] = { 0x6c014 },
+	[GCC_PCIE_0_NOCSR_COM_PHY_BCR] = { 0x6c020 },
+	[GCC_PCIE_0_PHY_BCR] = { 0x6c01c },
+	[GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR] = { 0x6c028 },
+	[GCC_PCIE_1_BCR] = { 0x67000 },
+	[GCC_PCIE_1_LINK_DOWN_BCR] = { 0x8e014 },
+	[GCC_PCIE_1_NOCSR_COM_PHY_BCR] = { 0x8e020 },
+	[GCC_PCIE_1_PHY_BCR] = { 0x8e01c },
+	[GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR] = { 0x8e024 },
+	[GCC_PCIE_PHY_BCR] = { 0x6f000 },
+	[GCC_PCIE_PHY_CFG_AHB_BCR] = { 0x6f00c },
+	[GCC_PCIE_PHY_COM_BCR] = { 0x6f010 },
+	[GCC_PCIE_RSCC_BCR] = { 0x11000 },
+	[GCC_PDM_BCR] = { 0x33000 },
+	[GCC_QUPV3_WRAPPER_1_BCR] = { 0x18000 },
+	[GCC_QUPV3_WRAPPER_2_BCR] = { 0x1e000 },
+	[GCC_QUPV3_WRAPPER_3_BCR] = { 0xa8000 },
+	[GCC_QUPV3_WRAPPER_4_BCR] = { 0xa9000 },
+	[GCC_QUPV3_WRAPPER_I2C_BCR] = { 0x17000 },
+	[GCC_QUSB2PHY_PRIM_BCR] = { 0x12000 },
+	[GCC_QUSB2PHY_SEC_BCR] = { 0x12004 },
+	[GCC_SDCC2_BCR] = { 0x14000 },
+	[GCC_SDCC4_BCR] = { 0x16000 },
+	[GCC_TCSR_PCIE_BCR] = { 0x6f018 },
+	[GCC_UFS_PHY_BCR] = { 0x77000 },
+	[GCC_USB30_PRIM_BCR] = { 0x39000 },
+	[GCC_USB3_DP_PHY_PRIM_BCR] = { 0x50008 },
+	[GCC_USB3_DP_PHY_SEC_BCR] = { 0x50014 },
+	[GCC_USB3_PHY_PRIM_BCR] = { 0x50000 },
+	[GCC_USB3_PHY_SEC_BCR] = { 0x5000c },
+	[GCC_USB3PHY_PHY_PRIM_BCR] = { 0x50004 },
+	[GCC_USB3PHY_PHY_SEC_BCR] = { 0x50010 },
+	[GCC_VIDEO_AXI0_CLK_ARES] = { 0x32018, 2 },
+	[GCC_VIDEO_AXI0C_CLK_ARES] = { 0x32020, 2 },
+	[GCC_VIDEO_BCR] = { 0x32000 },
+	[GCC_VIDEO_XO_CLK_ARES] = { 0x32028, 2 },
+};
+
+static const u32 gcc_hawi_critical_cbcrs[] = {
+	0xa0004, /* GCC_CAM_BIST_MCLK_AHB_CLK */
+	0x26004, /* GCC_CAMERA_AHB_CLK */
+	0x26028, /* GCC_CAMERA_RSC_CORE_CLK */
+	0x26024, /* GCC_CAMERA_XO_CLK */
+	0x9f004, /* GCC_EVA_AHB_CLK */
+	0x9f018, /* GCC_EVA_XO_CLK */
+	0x71004, /* GCC_GPU_CFG_AHB_CLK */
+	0x7101c, /* GCC_GPU_RSC_CORE_CLK */
+	0x67084, /* GCC_PCIE_1_RSC_CORE_CLK */
+	0x43014, /* GCC_PCIE_LINK_XO_CLK */
+	0x6b088, /* GCC_PCIE_RSC_CORE_CLK */
+	0x52010, /* GCC_PCIE_RSCC_CFG_AHB_CLK */
+	0x52010, /* GCC_PCIE_RSCC_XO_CLK */
+	0x32004, /* GCC_VIDEO_AHB_CLK */
+	0x32028, /* GCC_VIDEO_XO_CLK */
+};
+
+static const struct clk_rcg_dfs_data gcc_hawi_dfs_clocks[] = {
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_qspi_ref_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s4_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s5_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s6_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap1_s7_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s2_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap2_s4_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_qspi_ref_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s2_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s4_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap3_s5_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s0_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s1_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s2_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s3_clk_src),
+	DEFINE_RCG_DFS(gcc_qupv3_wrap4_s4_clk_src),
+};
+
+static const struct regmap_config gcc_hawi_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1f41f4,
+	.fast_io = true,
+};
+
+static void clk_hawi_regs_configure(struct device *dev, struct regmap *regmap)
+{
+	/* FORCE_MEM_CORE_ON for ufs phy ice core clocks */
+	qcom_branch_set_force_mem_core(regmap, gcc_ufs_phy_ice_core_clk, true);
+}
+
+static const struct qcom_cc_driver_data gcc_hawi_driver_data = {
+	.clk_cbcrs = gcc_hawi_critical_cbcrs,
+	.num_clk_cbcrs = ARRAY_SIZE(gcc_hawi_critical_cbcrs),
+	.dfs_rcgs = gcc_hawi_dfs_clocks,
+	.num_dfs_rcgs = ARRAY_SIZE(gcc_hawi_dfs_clocks),
+	.clk_regs_configure = clk_hawi_regs_configure,
+};
+
+static const struct qcom_cc_desc gcc_hawi_desc = {
+	.config = &gcc_hawi_regmap_config,
+	.clks = gcc_hawi_clocks,
+	.num_clks = ARRAY_SIZE(gcc_hawi_clocks),
+	.resets = gcc_hawi_resets,
+	.num_resets = ARRAY_SIZE(gcc_hawi_resets),
+	.gdscs = gcc_hawi_gdscs,
+	.num_gdscs = ARRAY_SIZE(gcc_hawi_gdscs),
+	.use_rpm = true,
+	.driver_data = &gcc_hawi_driver_data,
+};
+
+static const struct of_device_id gcc_hawi_match_table[] = {
+	{ .compatible = "qcom,hawi-gcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_hawi_match_table);
+
+static int gcc_hawi_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gcc_hawi_desc);
+}
+
+static struct platform_driver gcc_hawi_driver = {
+	.probe = gcc_hawi_probe,
+	.driver = {
+		.name = "gcc-hawi",
+		.of_match_table = gcc_hawi_match_table,
+	},
+};
+
+static int __init gcc_hawi_init(void)
+{
+	return platform_driver_register(&gcc_hawi_driver);
+}
+subsys_initcall(gcc_hawi_init);
+
+static void __exit gcc_hawi_exit(void)
+{
+	platform_driver_unregister(&gcc_hawi_driver);
+}
+module_exit(gcc_hawi_exit);
+
+MODULE_DESCRIPTION("QTI GCC HAWI Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-ipq9650.c b/drivers/clk/qcom/gcc-ipq9650.c
new file mode 100644
index 0000000000000..c556c2bbfd96b
--- /dev/null
+++ b/drivers/clk/qcom/gcc-ipq9650.c
@@ -0,0 +1,3445 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,ipq9650-gcc.h>
+#include <dt-bindings/reset/qcom,ipq9650-gcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "clk-regmap-phy-mux.h"
+#include "reset.h"
+
+enum {
+	DT_XO,
+	DT_SLEEP_CLK,
+	DT_PCIE30_PHY0_PIPE_CLK,
+	DT_PCIE30_PHY1_PIPE_CLK,
+	DT_PCIE30_PHY2_PIPE_CLK,
+	DT_PCIE30_PHY3_PIPE_CLK,
+	DT_PCIE30_PHY4_PIPE_CLK,
+	DT_USB3_PHY0_CC_PIPE_CLK,
+	DT_NSS_CMN_CLK,
+};
+
+enum {
+	P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC,
+	P_GPLL0_OUT_MAIN,
+	P_GPLL0_OUT_ODD,
+	P_GPLL2_OUT_AUX,
+	P_GPLL2_OUT_MAIN,
+	P_GPLL4_OUT_MAIN,
+	P_GPLL4_OUT_ODD,
+	P_NSS_CMN_CLK,
+	P_SLEEP_CLK,
+	P_XO,
+};
+
+static const struct clk_parent_data gcc_parent_data_xo = { .index = DT_XO };
+
+static struct clk_alpha_pll gpll0_main = {
+	.offset = 0x20000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID],
+	.clkr = {
+		.enable_reg = 0xb000,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll0_main",
+			.parent_data = &gcc_parent_data_xo,
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_fixed_lucid_ops,
+		},
+	},
+};
+
+static struct clk_fixed_factor gpll0_div2 = {
+	.mult = 1,
+	.div = 2,
+	.hw.init = &(const struct clk_init_data) {
+		.name = "gpll0_div2",
+		.parent_hws = (const struct clk_hw *[]) {
+			&gpll0_main.clkr.hw
+		},
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_alpha_pll_postdiv gpll0 = {
+	.offset = 0x20000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID],
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll0",
+		.parent_hws = (const struct clk_hw *[]) {
+			       &gpll0_main.clkr.hw },
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_ro_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll2 = {
+	.offset = 0x21000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_ZONDA],
+	.clkr = {
+		.enable_reg = 0xb000,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll2",
+			.parent_data = &gcc_parent_data_xo,
+			.num_parents = 1,
+			.ops = &clk_alpha_pll_zonda_ops,
+		},
+	},
+};
+
+static const struct clk_div_table post_div_table_gpll2_out_main[] = {
+	{ 0x1, 2 },
+	{ }
+};
+
+static struct clk_alpha_pll_postdiv gpll2_out_main = {
+	.offset = 0x21000,
+	.post_div_shift = 8,
+	.post_div_table = post_div_table_gpll2_out_main,
+	.num_post_div = ARRAY_SIZE(post_div_table_gpll2_out_main),
+	.width = 2,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_ZONDA],
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gpll2_out_main",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gpll2.clkr.hw,
+		},
+		.num_parents = 1,
+		.ops = &clk_alpha_pll_postdiv_zonda_ops,
+	},
+};
+
+static struct clk_alpha_pll gpll4 = {
+	.offset = 0x22000,
+	.regs = clk_alpha_pll_regs[CLK_ALPHA_PLL_TYPE_LUCID],
+	.clkr = {
+		.enable_reg = 0xb000,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gpll4",
+			.parent_data = &gcc_parent_data_xo,
+			.num_parents = 1,
+			/*
+			 * There are no consumers for this GPLL in kernel yet,
+			 * (will be added soon), so the clock framework
+			 * disables this source. But some of the clocks
+			 * initialized by boot loaders uses this source. So we
+			 * need to keep this clock ON. Add the
+			 * CLK_IGNORE_UNUSED flag so the clock will not be
+			 * disabled. Once the consumer in kernel is added, we
+			 * can get rid of this flag.
+			 */
+			.flags = CLK_IS_CRITICAL,
+			.ops = &clk_alpha_pll_fixed_lucid_ops,
+		},
+	},
+};
+
+static const struct parent_map gcc_parent_map_xo[] = {
+	{ P_XO, 0 },
+};
+
+static const struct parent_map gcc_parent_map_0[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_0[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_1[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_MAIN, 2 },
+};
+
+static const struct clk_parent_data gcc_parent_data_1[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll4.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_2[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+};
+
+static const struct clk_parent_data gcc_parent_data_2[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_3[] = {
+	{ P_XO, 0 },
+};
+
+static const struct clk_parent_data gcc_parent_data_3[] = {
+	{ .index = DT_XO },
+};
+
+static const struct parent_map gcc_parent_map_4[] = {
+	{ P_XO, 0 },
+	{ P_GPLL4_OUT_MAIN, 1 },
+	{ P_GPLL0_OUT_ODD, 2 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_4[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll4.clkr.hw },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_5[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL2_OUT_AUX, 2 },
+};
+
+static const struct clk_parent_data gcc_parent_data_5[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll2.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_6[] = {
+	{ P_XO, 0 },
+	{ P_GPLL4_OUT_ODD, 1 },
+	{ P_GPLL0_OUT_MAIN, 3 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_6[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll4.clkr.hw },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_7[] = {
+	{ P_XO, 0 },
+	{ P_NSS_CMN_CLK, 1 },
+	{ P_GPLL0_OUT_ODD, 2 },
+	{ P_GPLL2_OUT_AUX, 3 },
+};
+
+static const struct clk_parent_data gcc_parent_data_7[] = {
+	{ .index = DT_XO },
+	{ .index = DT_NSS_CMN_CLK },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll2.clkr.hw },
+};
+
+static const struct parent_map gcc_parent_map_8[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL0_OUT_ODD, 2 },
+	{ P_SLEEP_CLK, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_8[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll0.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_9[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL2_OUT_MAIN, 2 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_9[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll2_out_main.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_10[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_MAIN, 2 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 4 },
+};
+
+static const struct clk_parent_data gcc_parent_data_10[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll4.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct parent_map gcc_parent_map_11[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_ODD, 2 },
+	{ P_SLEEP_CLK, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_11[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_12[] = {
+	{ P_SLEEP_CLK, 6 },
+};
+
+static const struct clk_parent_data gcc_parent_data_12[] = {
+	{ .index = DT_SLEEP_CLK },
+};
+
+static const struct parent_map gcc_parent_map_13[] = {
+	{ P_XO, 0 },
+	{ P_GPLL0_OUT_MAIN, 1 },
+	{ P_GPLL4_OUT_MAIN, 2 },
+	{ P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 3 },
+};
+
+static const struct clk_parent_data gcc_parent_data_13[] = {
+	{ .index = DT_XO },
+	{ .hw = &gpll0.clkr.hw },
+	{ .hw = &gpll4.clkr.hw },
+	{ .hw = &gpll0_div2.hw },
+};
+
+static const struct freq_tbl ftbl_gcc_adss_pwm_clk_src[] = {
+	F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_adss_pwm_clk_src = {
+	.cmd_rcgr = 0x1c004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_adss_pwm_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_adss_pwm_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_gemnoc_anoc_pcie_clk_src[] = {
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static const struct freq_tbl ftbl_gcc_nss_ts_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_xo_clk_src = {
+	.cmd_rcgr = 0x34004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_xo,
+	.freq_tbl = ftbl_gcc_nss_ts_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_xo_clk_src",
+		.parent_data = &gcc_parent_data_xo,
+		.num_parents = 1,
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_fixed_factor gcc_xo_div4_clk_src = {
+	.mult = 1,
+	.div = 4,
+	.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_xo_div4_clk_src",
+		.parent_hws = (const struct clk_hw *[]) {
+			&gcc_xo_clk_src.clkr.hw
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_nss_ts_clk_src = {
+	.cmd_rcgr = 0x17088,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_nss_ts_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_nss_ts_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_nssnoc_memnoc_bfdcd_clk_src[] = {
+	F(462000000, P_NSS_CMN_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_nssnoc_memnoc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x17004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_7,
+	.freq_tbl = ftbl_gcc_nssnoc_memnoc_bfdcd_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_nssnoc_memnoc_bfdcd_clk_src",
+		.parent_data = gcc_parent_data_7,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_7),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_system_noc_bfdcd_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	F(133333333, P_GPLL0_OUT_MAIN, 6, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	F(266666667, P_GPLL4_OUT_MAIN, 4.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_system_noc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x2e004,
+	.freq_tbl = ftbl_gcc_system_noc_bfdcd_clk_src,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_13,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_system_noc_bfdcd_clk_src",
+		.parent_data = gcc_parent_data_13,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_13),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcnoc_bfdcd_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcnoc_bfdcd_clk_src = {
+	.cmd_rcgr = 0x31004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_pcnoc_bfdcd_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcnoc_bfdcd_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		/*
+		 * There are no consumers for this source in kernel yet,
+		 * (will be added soon), so the clock framework
+		 * disables this source. But some of the clocks
+		 * initialized by boot loaders uses this source. So we
+		 * need to keep this clock ON. Add the
+		 * CLK_IGNORE_UNUSED flag so the clock will not be
+		 * disabled. Once the consumer in kernel is added, we
+		 * can get rid of this flag.
+		 */
+		.flags = CLK_IS_CRITICAL,
+		.ops = &clk_rcg2_ops,
+	},
+};
+static const struct freq_tbl ftbl_gcc_pcie0_axi_m_clk_src[] = {
+	F(200000000, P_GPLL4_OUT_MAIN, 6, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie0_axi_m_clk_src = {
+	.cmd_rcgr = 0x28018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie0_axi_m_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie0_axi_s_clk_src = {
+	.cmd_rcgr = 0x28020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie0_axi_s_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie0_rchng_clk_src = {
+	.cmd_rcgr = 0x28028,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_adss_pwm_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie0_rchng_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie1_axi_m_clk_src[] = {
+	F(266666667, P_GPLL4_OUT_MAIN, 4.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie1_axi_m_clk_src = {
+	.cmd_rcgr = 0x29018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie1_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie1_axi_m_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie1_axi_s_clk_src = {
+	.cmd_rcgr = 0x29020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie1_axi_s_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie1_rchng_clk_src = {
+	.cmd_rcgr = 0x29028,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_adss_pwm_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie1_rchng_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie2_axi_m_clk_src = {
+	.cmd_rcgr = 0x2a018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie1_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie2_axi_m_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie2_axi_s_clk_src = {
+	.cmd_rcgr = 0x2a020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie2_axi_s_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie2_rchng_clk_src = {
+	.cmd_rcgr = 0x2a028,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_adss_pwm_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie2_rchng_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie3_axi_m_clk_src = {
+	.cmd_rcgr = 0x2b018,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie1_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie3_axi_m_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie3_axi_s_clk_src = {
+	.cmd_rcgr = 0x2b020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie3_axi_s_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie3_rchng_clk_src = {
+	.cmd_rcgr = 0x2b028,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_adss_pwm_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie3_rchng_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie4_axi_m_clk_src = {
+	.cmd_rcgr = 0x25004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie4_axi_m_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie4_axi_s_clk_src = {
+	.cmd_rcgr = 0x2500c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_1,
+	.freq_tbl = ftbl_gcc_pcie0_axi_m_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie4_axi_s_clk_src",
+		.parent_data = gcc_parent_data_1,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_1),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_pcie4_rchng_clk_src = {
+	.cmd_rcgr = 0x25014,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_2,
+	.freq_tbl = ftbl_gcc_adss_pwm_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie4_rchng_clk_src",
+		.parent_data = gcc_parent_data_2,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_2),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_pcie_aux_clk_src[] = {
+	F(20000000, P_GPLL0_OUT_MAIN, 10, 1, 4),
+	{ }
+};
+
+static struct clk_rcg2 gcc_pcie_aux_clk_src = {
+	.cmd_rcgr = 0x28004,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_8,
+	.freq_tbl = ftbl_gcc_pcie_aux_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_pcie_aux_clk_src",
+		.parent_data = gcc_parent_data_8,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_8),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qdss_at_clk_src[] = {
+	F(240000000, P_GPLL4_OUT_MAIN, 5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qdss_at_clk_src = {
+	.cmd_rcgr = 0x2d004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_qdss_at_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qdss_at_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_fixed_factor gcc_eud_at_div_clk_src = {
+	.mult = 1,
+	.div = 6,
+	.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_eud_at_div_clk_src",
+		.parent_hws = (const struct clk_hw *[]) {
+			&gcc_qdss_at_clk_src.clkr.hw },
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qdss_tsctr_clk_src[] = {
+	F(600000000, P_GPLL4_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qdss_tsctr_clk_src = {
+	.cmd_rcgr = 0x2d01c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_4,
+	.freq_tbl = ftbl_gcc_qdss_tsctr_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qdss_tsctr_clk_src",
+		.parent_data = gcc_parent_data_4,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_4),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_fixed_factor gcc_qdss_dap_sync_clk_src = {
+	.mult = 1,
+	.div = 4,
+	.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qdss_dap_sync_clk_src",
+		.parent_hws = (const struct clk_hw *[]) {
+			&gcc_qdss_tsctr_clk_src.clkr.hw
+		},
+		.num_parents = 1,
+		.ops = &clk_fixed_factor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sleep_clk_src[] = {
+	F(32000, P_SLEEP_CLK, 1, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sleep_clk_src = {
+	.cmd_rcgr = 0x3400c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_12,
+	.freq_tbl = ftbl_gcc_sleep_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sleep_clk_src",
+		.parent_data = gcc_parent_data_12,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_12),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qpic_io_macro_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	F(320000000, P_GPLL0_OUT_MAIN, 2.5, 0, 0),
+	F(400000000, P_GPLL0_OUT_MAIN, 2, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qpic_clk_src = {
+	.cmd_rcgr = 0x32020,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_qpic_io_macro_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qpic_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qpic_io_macro_clk_src = {
+	.cmd_rcgr = 0x32004,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_5,
+	.freq_tbl = ftbl_gcc_qpic_io_macro_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qpic_io_macro_clk_src",
+		.parent_data = gcc_parent_data_5,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_5),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_2x_core_clk_src = {
+	.cmd_rcgr = 0x100c,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_gemnoc_anoc_pcie_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_2x_core_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap_se0_clk_src[] = {
+	F(960000, P_XO, 10, 2, 5),
+	F(3686636, P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 1, 2, 217),
+	F(4800000, P_XO, 5, 0, 0),
+	F(7373272, P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 1, 4, 217),
+	F(9600000, P_XO, 2.5, 0, 0),
+	F(14746544, P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 1, 8, 217),
+	F(16000000, P_GPLL0_OUT_MAIN, 10, 1, 5),
+	F(24000000, P_XO, 1, 0, 0),
+	F(25000000, P_GPLL0_OUT_MAIN, 16, 1, 2),
+	F(32000000, P_GPLL0_OUT_MAIN, 1, 1, 25),
+	F(40000000, P_GPLL0_OUT_MAIN, 1, 1, 20),
+	F(46400000, P_GPLL0_OUT_MAIN, 2, 29, 250),
+	F(48000000, P_GPLL0_OUT_MAIN, 1, 3, 50),
+	F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0),
+	F(51200000, P_GPLL0_OUT_MAIN, 1, 8, 125),
+	F(56000000, P_GPLL0_OUT_MAIN, 1, 7, 100),
+	F(58986175, P_GPLL0_OUT_MAIN, 1, 16, 217),
+	F(60000000, P_GPLL0_OUT_MAIN, 1, 3, 40),
+	F(64000000, P_GPLL0_OUT_MAIN, 12.5, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se0_clk_src = {
+	.cmd_rcgr = 0x2018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se0_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se1_clk_src = {
+	.cmd_rcgr = 0x3018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se1_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se2_clk_src = {
+	.cmd_rcgr = 0x3034,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se2_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se3_clk_src = {
+	.cmd_rcgr = 0x3050,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se3_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_qupv3_wrap_se4_clk_src[] = {
+	F(960000, P_XO, 10, 2, 5),
+	F(3686636, P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 1, 2, 217),
+	F(4800000, P_XO, 5, 0, 0),
+	F(7373272, P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 1, 4, 217),
+	F(9600000, P_XO, 2.5, 0, 0),
+	F(14746544, P_GCC_GPLL0_OUT_MAIN_DIV_CLK_SRC, 1, 8, 217),
+	F(16000000, P_GPLL0_OUT_MAIN, 10, 1, 5),
+	F(24000000, P_XO, 1, 0, 0),
+	F(25000000, P_GPLL0_OUT_MAIN, 16, 1, 2),
+	F(32000000, P_GPLL0_OUT_MAIN, 1, 1, 25),
+	F(40000000, P_GPLL0_OUT_MAIN, 1, 1, 20),
+	F(46400000, P_GPLL0_OUT_MAIN, 2, 29, 250),
+	F(48000000, P_GPLL0_OUT_MAIN, 1, 3, 50),
+	F(50000000, P_GPLL0_OUT_MAIN, 16, 0, 0),
+	F(51200000, P_GPLL0_OUT_MAIN, 1, 8, 125),
+	F(56000000, P_GPLL0_OUT_MAIN, 1, 7, 100),
+	F(58986175, P_GPLL0_OUT_MAIN, 1, 16, 217),
+	F(60000000, P_GPLL0_OUT_MAIN, 1, 3, 40),
+	F(64000000, P_GPLL0_OUT_MAIN, 12.5, 0, 0),
+	F(100000000, P_GPLL0_OUT_MAIN, 8, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se4_clk_src = {
+	.cmd_rcgr = 0x306c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se4_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se4_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se5_clk_src = {
+	.cmd_rcgr = 0x3090,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se4_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se5_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se6_clk_src = {
+	.cmd_rcgr = 0x4004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se6_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_qupv3_wrap_se7_clk_src = {
+	.cmd_rcgr = 0x4020,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_qupv3_wrap_se0_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_qupv3_wrap_se7_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_apps_clk_src[] = {
+	F(144000, P_XO, 16, 12, 125),
+	F(400000, P_XO, 12, 1, 5),
+	F(24000000, P_GPLL2_OUT_MAIN, 12, 1, 2),
+	F(48000000, P_GPLL2_OUT_MAIN, 12, 0, 0),
+	F(96000000, P_GPLL2_OUT_MAIN, 6, 0, 0),
+	F(177777778, P_GPLL0_OUT_MAIN, 4.5, 0, 0),
+	F(192000000, P_GPLL2_OUT_MAIN, 3, 0, 0),
+	F(200000000, P_GPLL0_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc1_apps_clk_src = {
+	.cmd_rcgr = 0x33004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_9,
+	.freq_tbl = ftbl_gcc_sdcc1_apps_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc1_apps_clk_src",
+		.parent_data = gcc_parent_data_9,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_9),
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_sdcc1_ice_core_clk_src[] = {
+	F(300000000, P_GPLL4_OUT_MAIN, 4, 0, 0),
+	{ }
+};
+
+static struct clk_rcg2 gcc_sdcc1_ice_core_clk_src = {
+	.cmd_rcgr = 0x33018,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_10,
+	.freq_tbl = ftbl_gcc_sdcc1_ice_core_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_sdcc1_ice_core_clk_src",
+		.parent_data = gcc_parent_data_10,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_10),
+		.ops = &clk_rcg2_floor_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_uniphy_sys_clk_src = {
+	.cmd_rcgr = 0x17090,
+	.mnd_width = 0,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_3,
+	.freq_tbl = ftbl_gcc_nss_ts_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_uniphy_sys_clk_src",
+		.parent_data = gcc_parent_data_3,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_3),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb0_aux_clk_src = {
+	.cmd_rcgr = 0x2c018,
+	.mnd_width = 16,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_11,
+	.freq_tbl = ftbl_gcc_nss_ts_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb0_aux_clk_src",
+		.parent_data = gcc_parent_data_11,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_11),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb0_master_clk_src = {
+	.cmd_rcgr = 0x2c004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_0,
+	.freq_tbl = ftbl_gcc_gemnoc_anoc_pcie_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb0_master_clk_src",
+		.parent_data = gcc_parent_data_0,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_0),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static const struct freq_tbl ftbl_gcc_usb0_mock_utmi_clk_src[] = {
+	F(24000000, P_XO, 1, 0, 0),
+	F(60000000, P_GPLL4_OUT_ODD, 10, 1, 2),
+	{ }
+};
+
+static struct clk_rcg2 gcc_usb0_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x2c02c,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_6,
+	.freq_tbl = ftbl_gcc_usb0_mock_utmi_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb0_mock_utmi_clk_src",
+		.parent_data = gcc_parent_data_6,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_6),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_rcg2 gcc_usb1_mock_utmi_clk_src = {
+	.cmd_rcgr = 0x3c004,
+	.mnd_width = 8,
+	.hid_width = 5,
+	.parent_map = gcc_parent_map_6,
+	.freq_tbl = ftbl_gcc_usb0_mock_utmi_clk_src,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb1_mock_utmi_clk_src",
+		.parent_data = gcc_parent_data_6,
+		.num_parents = ARRAY_SIZE(gcc_parent_data_6),
+		.ops = &clk_rcg2_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_nssnoc_memnoc_div_clk_src = {
+	.reg = 0x1700c,
+	.shift = 0,
+	.width = 4,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_nssnoc_memnoc_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_nssnoc_memnoc_bfdcd_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_usb0_mock_utmi_div_clk_src = {
+	.reg = 0x2c040,
+	.shift = 0,
+	.width = 2,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb0_mock_utmi_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_usb0_mock_utmi_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_regmap_div gcc_usb1_mock_utmi_div_clk_src = {
+	.reg = 0x3c018,
+	.shift = 0,
+	.width = 2,
+	.clkr.hw.init = &(const struct clk_init_data) {
+		.name = "gcc_usb1_mock_utmi_div_clk_src",
+		.parent_hws = (const struct clk_hw*[]) {
+			&gcc_usb1_mock_utmi_clk_src.clkr.hw,
+		},
+		.num_parents = 1,
+		.flags = CLK_SET_RATE_PARENT,
+		.ops = &clk_regmap_div_ro_ops,
+	},
+};
+
+static struct clk_branch gcc_adss_pwm_clk = {
+	.halt_reg = 0x1c00c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1c00c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_adss_pwm_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_adss_pwm_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie0_1lane_m_clk = {
+	.halt_reg = 0x2e07c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e07c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie0_1lane_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie0_1lane_s_clk = {
+	.halt_reg = 0x2e0cc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e0cc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie0_1lane_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie1_2lane_m_clk = {
+	.halt_reg = 0x2e084,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e084,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie1_2lane_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie1_2lane_s_clk = {
+	.halt_reg = 0x2e0d0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e0d0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie1_2lane_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie2_2lane_m_clk = {
+	.halt_reg = 0x2e080,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e080,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie2_2lane_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie2_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie2_2lane_s_clk = {
+	.halt_reg = 0x2e0d4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e0d4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie2_2lane_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie2_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie3_2lane_m_clk = {
+	.halt_reg = 0x2e0bc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e0bc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie3_2lane_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie3_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie3_2lane_s_clk = {
+	.halt_reg = 0x2e0d8,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e0d8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie3_2lane_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie3_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie4_1lane_m_clk = {
+	.halt_reg = 0x2e0c0,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e0c0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie4_1lane_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie4_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_anoc_pcie4_1lane_s_clk = {
+	.halt_reg = 0x2e0dc,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e0dc,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_anoc_pcie4_1lane_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie4_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cmn_12gpll_ahb_clk = {
+	.halt_reg = 0x3a004,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3a004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cmn_12gpll_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_cmn_12gpll_sys_clk = {
+	.halt_reg = 0x3a008,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x3a008,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_cmn_12gpll_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_uniphy_sys_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_mdio_ahb_clk = {
+	.halt_reg = 0x17040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_mdio_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nss_ts_clk = {
+	.halt_reg = 0x17018,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x17018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nss_ts_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_nss_ts_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nsscc_clk = {
+	.halt_reg = 0x17034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nsscc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nsscfg_clk = {
+	.halt_reg = 0x1702c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1702c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nsscfg_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_atb_clk = {
+	.halt_reg = 0x17014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x17014,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_atb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qdss_at_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_memnoc_1_clk = {
+	.halt_reg = 0x17084,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17084,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_memnoc_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_nssnoc_memnoc_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_memnoc_clk = {
+	.halt_reg = 0x17024,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_memnoc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_nssnoc_memnoc_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_nsscc_clk = {
+	.halt_reg = 0x17030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_nsscc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_pcnoc_1_clk = {
+	.halt_reg = 0x17080,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17080,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_pcnoc_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_qosgen_ref_clk = {
+	.halt_reg = 0x1701c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1701c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_qosgen_ref_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_xo_div4_clk_src.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_snoc_1_clk = {
+	.halt_reg = 0x1707c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1707c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_snoc_1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_system_noc_bfdcd_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_snoc_clk = {
+	.halt_reg = 0x17028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_snoc_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_system_noc_bfdcd_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_timeout_ref_clk = {
+	.halt_reg = 0x17020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_timeout_ref_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_xo_div4_clk_src.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_nssnoc_xo_dcd_clk = {
+	.halt_reg = 0x17074,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x17074,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_nssnoc_xo_dcd_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_xo_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_ahb_clk = {
+	.halt_reg = 0x28030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_aux_clk = {
+	.halt_reg = 0x28070,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28070,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_axi_m_clk = {
+	.halt_reg = 0x28038,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28038,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_axi_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_axi_s_bridge_clk = {
+	.halt_reg = 0x28048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_axi_s_bridge_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_axi_s_clk = {
+	.halt_reg = 0x28040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x28040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_axi_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie0_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie0_pipe_clk_src = {
+	.reg = 0x28064,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "pcie0_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_PCIE30_PHY0_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_pipe_clk = {
+	.halt_reg = 0x28068,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x28068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_pipe_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie0_pipe_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_ahb_clk = {
+	.halt_reg = 0x29030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_aux_clk = {
+	.halt_reg = 0x29074,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29074,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_axi_m_clk = {
+	.halt_reg = 0x29038,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29038,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_axi_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_axi_s_bridge_clk = {
+	.halt_reg = 0x29048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_axi_s_bridge_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_axi_s_clk = {
+	.halt_reg = 0x29040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x29040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_axi_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie1_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie1_pipe_clk_src = {
+	.reg = 0x29064,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "pcie1_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_PCIE30_PHY1_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_pipe_clk = {
+	.halt_reg = 0x29068,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x29068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_pipe_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie1_pipe_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie2_ahb_clk = {
+	.halt_reg = 0x2a030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2a030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie2_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie2_aux_clk = {
+	.halt_reg = 0x2a078,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2a078,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie2_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie2_axi_m_clk = {
+	.halt_reg = 0x2a038,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2a038,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie2_axi_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie2_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie2_axi_s_bridge_clk = {
+	.halt_reg = 0x2a048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2a048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie2_axi_s_bridge_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie2_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie2_axi_s_clk = {
+	.halt_reg = 0x2a040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2a040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie2_axi_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie2_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie2_pipe_clk_src = {
+	.reg = 0x2a064,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "pcie2_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_PCIE30_PHY2_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie2_pipe_clk = {
+	.halt_reg = 0x2a068,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x2a068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie2_pipe_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie2_pipe_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie3_ahb_clk = {
+	.halt_reg = 0x2b030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2b030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie3_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie3_aux_clk = {
+	.halt_reg = 0x2b07c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2b07c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie3_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie3_axi_m_clk = {
+	.halt_reg = 0x2b038,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2b038,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie3_axi_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie3_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie3_axi_s_bridge_clk = {
+	.halt_reg = 0x2b048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2b048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie3_axi_s_bridge_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie3_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie3_axi_s_clk = {
+	.halt_reg = 0x2b040,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2b040,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie3_axi_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie3_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie3_pipe_clk_src = {
+	.reg = 0x2b064,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "pcie3_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_PCIE30_PHY3_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie3_pipe_clk = {
+	.halt_reg = 0x2b068,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x2b068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie3_pipe_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie3_pipe_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie4_ahb_clk = {
+	.halt_reg = 0x2501c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2501c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie4_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie4_aux_clk = {
+	.halt_reg = 0x25020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x25020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie4_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie4_axi_m_clk = {
+	.halt_reg = 0x25028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x25028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie4_axi_m_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie4_axi_m_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie4_axi_s_bridge_clk = {
+	.halt_reg = 0x25038,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x25038,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie4_axi_s_bridge_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie4_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie4_axi_s_clk = {
+	.halt_reg = 0x25030,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x25030,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie4_axi_s_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcie4_axi_s_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_pcie4_pipe_clk_src = {
+	.reg = 0x25058,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "pcie4_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_PCIE30_PHY4_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie4_pipe_clk = {
+	.halt_reg = 0x2503c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x2503c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie4_pipe_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie4_pipe_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie0_rchng_clk = {
+	.halt_reg = 0x28028,
+	.clkr = {
+		.enable_reg = 0x28028,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie0_rchng_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie0_rchng_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie1_rchng_clk = {
+	.halt_reg = 0x29028,
+	.clkr = {
+		.enable_reg = 0x29028,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie1_rchng_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie1_rchng_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie2_rchng_clk = {
+	.halt_reg = 0x2a028,
+	.clkr = {
+		.enable_reg = 0x2a028,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie2_rchng_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie2_rchng_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie3_rchng_clk = {
+	.halt_reg = 0x2b028,
+	.clkr = {
+		.enable_reg = 0x2b028,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie3_rchng_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie3_rchng_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_pcie4_rchng_clk = {
+	.halt_reg = 0x25014,
+	.clkr = {
+		.enable_reg = 0x25014,
+		.enable_mask = BIT(1),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_pcie4_rchng_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_pcie4_rchng_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qdss_at_clk = {
+	.halt_reg = 0x2d034,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x2d034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qdss_at_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qdss_at_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qdss_dap_clk = {
+	.halt_reg = 0x2d058,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0xb004,
+		.enable_mask = BIT(2),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qdss_dap_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qdss_dap_sync_clk_src.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qpic_ahb_clk = {
+	.halt_reg = 0x32010,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x32010,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qpic_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qpic_clk = {
+	.halt_reg = 0x32028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x32028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qpic_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qpic_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qpic_io_macro_clk = {
+	.halt_reg = 0x3200c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3200c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qpic_io_macro_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qpic_io_macro_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qpic_sleep_clk = {
+	.halt_reg = 0x32018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x32018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qpic_sleep_clk",
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_ahb_mst_clk = {
+	.halt_reg = 0x1014,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0xb004,
+		.enable_mask = BIT(14),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_ahb_mst_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_ahb_slv_clk = {
+	.halt_reg = 0x102c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0xb004,
+		.enable_mask = BIT(4),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_ahb_slv_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se0_clk = {
+	.halt_reg = 0x202c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x202c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se0_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se0_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se1_clk = {
+	.halt_reg = 0x302c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x302c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se1_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se1_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se2_clk = {
+	.halt_reg = 0x3048,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se2_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se2_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se3_clk = {
+	.halt_reg = 0x3064,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3064,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se3_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se3_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se4_clk = {
+	.halt_reg = 0x3080,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3080,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se4_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se4_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se5_clk = {
+	.halt_reg = 0x30a4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x30a4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se5_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se5_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se6_clk = {
+	.halt_reg = 0x4018,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4018,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se6_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se6_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_qupv3_wrap_se7_clk = {
+	.halt_reg = 0x4034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x4034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_qupv3_wrap_se7_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_qupv3_wrap_se7_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ahb_clk = {
+	.halt_reg = 0x3303c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3303c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_apps_clk = {
+	.halt_reg = 0x3302c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3302c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_apps_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc1_apps_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_sdcc1_ice_core_clk = {
+	.halt_reg = 0x33034,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x33034,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_sdcc1_ice_core_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sdcc1_ice_core_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_snoc_usb_clk = {
+	.halt_reg = 0x2e0c4,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x2e0c4,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_snoc_usb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb0_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy0_ahb_clk = {
+	.halt_reg = 0x1704c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1704c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy0_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy0_sys_clk = {
+	.halt_reg = 0x17048,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x17048,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy0_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_uniphy_sys_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy1_ahb_clk = {
+	.halt_reg = 0x1705c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1705c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy1_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy1_sys_clk = {
+	.halt_reg = 0x17058,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x17058,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy1_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_uniphy_sys_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy2_ahb_clk = {
+	.halt_reg = 0x1706c,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x1706c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy2_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_uniphy2_sys_clk = {
+	.halt_reg = 0x17068,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x17068,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_uniphy2_sys_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_uniphy_sys_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_aux_clk = {
+	.halt_reg = 0x2c04c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x2c04c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_aux_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb0_aux_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_eud_at_clk = {
+	.halt_reg = 0x30004,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x30004,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_eud_at_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_eud_at_div_clk_src.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_master_clk = {
+	.halt_reg = 0x2c044,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x2c044,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_master_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb0_master_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_mock_utmi_clk = {
+	.halt_reg = 0x2c050,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x2c050,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_mock_utmi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb0_mock_utmi_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_phy_cfg_ahb_clk = {
+	.halt_reg = 0x2c05c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x2c05c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_phy_cfg_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap_phy_mux gcc_usb0_pipe_clk_src = {
+	.reg = 0x2c074,
+	.clkr = {
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_pipe_clk_src",
+			.parent_data = &(const struct clk_parent_data) {
+				.index = DT_USB3_PHY0_CC_PIPE_CLK,
+			},
+			.num_parents = 1,
+			.ops = &clk_regmap_phy_mux_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_pipe_clk = {
+	.halt_reg = 0x2c054,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x2c054,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_pipe_clk",
+			.parent_hws = (const struct clk_hw *[]) {
+				&gcc_usb0_pipe_clk_src.clkr.hw
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb0_sleep_clk = {
+	.halt_reg = 0x2c058,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x2c058,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb0_sleep_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sleep_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_master_clk = {
+	.halt_reg = 0x3c028,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3c028,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb1_master_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_mock_utmi_clk = {
+	.halt_reg = 0x3c024,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3c024,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb1_mock_utmi_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_usb1_mock_utmi_div_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_phy_cfg_ahb_clk = {
+	.halt_reg = 0x3c01c,
+	.halt_check = BRANCH_HALT_VOTED,
+	.clkr = {
+		.enable_reg = 0x3c01c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb1_phy_cfg_ahb_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_pcnoc_bfdcd_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.flags = CLK_SET_RATE_PARENT,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch gcc_usb1_sleep_clk = {
+	.halt_reg = 0x3c020,
+	.halt_check = BRANCH_HALT,
+	.clkr = {
+		.enable_reg = 0x3c020,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "gcc_usb1_sleep_clk",
+			.parent_hws = (const struct clk_hw*[]) {
+				&gcc_sleep_clk_src.clkr.hw,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap *gcc_ipq9650_clocks[] = {
+	[GCC_ADSS_PWM_CLK] = &gcc_adss_pwm_clk.clkr,
+	[GCC_ADSS_PWM_CLK_SRC] = &gcc_adss_pwm_clk_src.clkr,
+	[GCC_ANOC_PCIE0_1LANE_M_CLK] = &gcc_anoc_pcie0_1lane_m_clk.clkr,
+	[GCC_ANOC_PCIE0_1LANE_S_CLK] = &gcc_anoc_pcie0_1lane_s_clk.clkr,
+	[GCC_ANOC_PCIE1_2LANE_M_CLK] = &gcc_anoc_pcie1_2lane_m_clk.clkr,
+	[GCC_ANOC_PCIE1_2LANE_S_CLK] = &gcc_anoc_pcie1_2lane_s_clk.clkr,
+	[GCC_ANOC_PCIE2_2LANE_M_CLK] = &gcc_anoc_pcie2_2lane_m_clk.clkr,
+	[GCC_ANOC_PCIE2_2LANE_S_CLK] = &gcc_anoc_pcie2_2lane_s_clk.clkr,
+	[GCC_ANOC_PCIE3_2LANE_M_CLK] = &gcc_anoc_pcie3_2lane_m_clk.clkr,
+	[GCC_ANOC_PCIE3_2LANE_S_CLK] = &gcc_anoc_pcie3_2lane_s_clk.clkr,
+	[GCC_ANOC_PCIE4_1LANE_M_CLK] = &gcc_anoc_pcie4_1lane_m_clk.clkr,
+	[GCC_ANOC_PCIE4_1LANE_S_CLK] = &gcc_anoc_pcie4_1lane_s_clk.clkr,
+	[GCC_CMN_12GPLL_AHB_CLK] = &gcc_cmn_12gpll_ahb_clk.clkr,
+	[GCC_CMN_12GPLL_SYS_CLK] = &gcc_cmn_12gpll_sys_clk.clkr,
+	[GCC_MDIO_AHB_CLK] = &gcc_mdio_ahb_clk.clkr,
+	[GCC_NSS_TS_CLK] = &gcc_nss_ts_clk.clkr,
+	[GCC_NSS_TS_CLK_SRC] = &gcc_nss_ts_clk_src.clkr,
+	[GCC_NSSCC_CLK] = &gcc_nsscc_clk.clkr,
+	[GCC_NSSCFG_CLK] = &gcc_nsscfg_clk.clkr,
+	[GCC_NSSNOC_ATB_CLK] = &gcc_nssnoc_atb_clk.clkr,
+	[GCC_NSSNOC_MEMNOC_1_CLK] = &gcc_nssnoc_memnoc_1_clk.clkr,
+	[GCC_NSSNOC_MEMNOC_BFDCD_CLK_SRC] = &gcc_nssnoc_memnoc_bfdcd_clk_src.clkr,
+	[GCC_NSSNOC_MEMNOC_CLK] = &gcc_nssnoc_memnoc_clk.clkr,
+	[GCC_NSSNOC_MEMNOC_DIV_CLK_SRC] = &gcc_nssnoc_memnoc_div_clk_src.clkr,
+	[GCC_NSSNOC_NSSCC_CLK] = &gcc_nssnoc_nsscc_clk.clkr,
+	[GCC_NSSNOC_PCNOC_1_CLK] = &gcc_nssnoc_pcnoc_1_clk.clkr,
+	[GCC_NSSNOC_QOSGEN_REF_CLK] = &gcc_nssnoc_qosgen_ref_clk.clkr,
+	[GCC_NSSNOC_SNOC_1_CLK] = &gcc_nssnoc_snoc_1_clk.clkr,
+	[GCC_NSSNOC_SNOC_CLK] = &gcc_nssnoc_snoc_clk.clkr,
+	[GCC_NSSNOC_TIMEOUT_REF_CLK] = &gcc_nssnoc_timeout_ref_clk.clkr,
+	[GCC_NSSNOC_XO_DCD_CLK] = &gcc_nssnoc_xo_dcd_clk.clkr,
+	[GCC_PCIE0_AHB_CLK] = &gcc_pcie0_ahb_clk.clkr,
+	[GCC_PCIE0_AUX_CLK] = &gcc_pcie0_aux_clk.clkr,
+	[GCC_PCIE0_AXI_M_CLK] = &gcc_pcie0_axi_m_clk.clkr,
+	[GCC_PCIE0_AXI_M_CLK_SRC] = &gcc_pcie0_axi_m_clk_src.clkr,
+	[GCC_PCIE0_AXI_S_BRIDGE_CLK] = &gcc_pcie0_axi_s_bridge_clk.clkr,
+	[GCC_PCIE0_AXI_S_CLK] = &gcc_pcie0_axi_s_clk.clkr,
+	[GCC_PCIE0_AXI_S_CLK_SRC] = &gcc_pcie0_axi_s_clk_src.clkr,
+	[GCC_PCIE0_PIPE_CLK] = &gcc_pcie0_pipe_clk.clkr,
+	[GCC_PCIE0_PIPE_CLK_SRC] = &gcc_pcie0_pipe_clk_src.clkr,
+	[GCC_PCIE0_RCHNG_CLK_SRC] = &gcc_pcie0_rchng_clk_src.clkr,
+	[GCC_PCIE0_RCHNG_CLK] = &gcc_pcie0_rchng_clk.clkr,
+	[GCC_PCIE1_AHB_CLK] = &gcc_pcie1_ahb_clk.clkr,
+	[GCC_PCIE1_AUX_CLK] = &gcc_pcie1_aux_clk.clkr,
+	[GCC_PCIE1_AXI_M_CLK] = &gcc_pcie1_axi_m_clk.clkr,
+	[GCC_PCIE1_AXI_M_CLK_SRC] = &gcc_pcie1_axi_m_clk_src.clkr,
+	[GCC_PCIE1_AXI_S_BRIDGE_CLK] = &gcc_pcie1_axi_s_bridge_clk.clkr,
+	[GCC_PCIE1_AXI_S_CLK] = &gcc_pcie1_axi_s_clk.clkr,
+	[GCC_PCIE1_AXI_S_CLK_SRC] = &gcc_pcie1_axi_s_clk_src.clkr,
+	[GCC_PCIE1_PIPE_CLK] = &gcc_pcie1_pipe_clk.clkr,
+	[GCC_PCIE1_PIPE_CLK_SRC] = &gcc_pcie1_pipe_clk_src.clkr,
+	[GCC_PCIE1_RCHNG_CLK_SRC] = &gcc_pcie1_rchng_clk_src.clkr,
+	[GCC_PCIE1_RCHNG_CLK] = &gcc_pcie1_rchng_clk.clkr,
+	[GCC_PCIE2_AHB_CLK] = &gcc_pcie2_ahb_clk.clkr,
+	[GCC_PCIE2_AUX_CLK] = &gcc_pcie2_aux_clk.clkr,
+	[GCC_PCIE2_AXI_M_CLK] = &gcc_pcie2_axi_m_clk.clkr,
+	[GCC_PCIE2_AXI_M_CLK_SRC] = &gcc_pcie2_axi_m_clk_src.clkr,
+	[GCC_PCIE2_AXI_S_BRIDGE_CLK] = &gcc_pcie2_axi_s_bridge_clk.clkr,
+	[GCC_PCIE2_AXI_S_CLK] = &gcc_pcie2_axi_s_clk.clkr,
+	[GCC_PCIE2_AXI_S_CLK_SRC] = &gcc_pcie2_axi_s_clk_src.clkr,
+	[GCC_PCIE2_PIPE_CLK] = &gcc_pcie2_pipe_clk.clkr,
+	[GCC_PCIE2_PIPE_CLK_SRC] = &gcc_pcie2_pipe_clk_src.clkr,
+	[GCC_PCIE2_RCHNG_CLK_SRC] = &gcc_pcie2_rchng_clk_src.clkr,
+	[GCC_PCIE2_RCHNG_CLK] = &gcc_pcie2_rchng_clk.clkr,
+	[GCC_PCIE3_AHB_CLK] = &gcc_pcie3_ahb_clk.clkr,
+	[GCC_PCIE3_AUX_CLK] = &gcc_pcie3_aux_clk.clkr,
+	[GCC_PCIE3_AXI_M_CLK] = &gcc_pcie3_axi_m_clk.clkr,
+	[GCC_PCIE3_AXI_M_CLK_SRC] = &gcc_pcie3_axi_m_clk_src.clkr,
+	[GCC_PCIE3_AXI_S_BRIDGE_CLK] = &gcc_pcie3_axi_s_bridge_clk.clkr,
+	[GCC_PCIE3_AXI_S_CLK] = &gcc_pcie3_axi_s_clk.clkr,
+	[GCC_PCIE3_AXI_S_CLK_SRC] = &gcc_pcie3_axi_s_clk_src.clkr,
+	[GCC_PCIE3_PIPE_CLK] = &gcc_pcie3_pipe_clk.clkr,
+	[GCC_PCIE3_PIPE_CLK_SRC] = &gcc_pcie3_pipe_clk_src.clkr,
+	[GCC_PCIE3_RCHNG_CLK_SRC] = &gcc_pcie3_rchng_clk_src.clkr,
+	[GCC_PCIE3_RCHNG_CLK] = &gcc_pcie3_rchng_clk.clkr,
+	[GCC_PCIE4_AHB_CLK] = &gcc_pcie4_ahb_clk.clkr,
+	[GCC_PCIE4_AUX_CLK] = &gcc_pcie4_aux_clk.clkr,
+	[GCC_PCIE4_AXI_M_CLK] = &gcc_pcie4_axi_m_clk.clkr,
+	[GCC_PCIE4_AXI_M_CLK_SRC] = &gcc_pcie4_axi_m_clk_src.clkr,
+	[GCC_PCIE4_AXI_S_BRIDGE_CLK] = &gcc_pcie4_axi_s_bridge_clk.clkr,
+	[GCC_PCIE4_AXI_S_CLK] = &gcc_pcie4_axi_s_clk.clkr,
+	[GCC_PCIE4_AXI_S_CLK_SRC] = &gcc_pcie4_axi_s_clk_src.clkr,
+	[GCC_PCIE4_PIPE_CLK] = &gcc_pcie4_pipe_clk.clkr,
+	[GCC_PCIE4_PIPE_CLK_SRC] = &gcc_pcie4_pipe_clk_src.clkr,
+	[GCC_PCIE4_RCHNG_CLK_SRC] = &gcc_pcie4_rchng_clk_src.clkr,
+	[GCC_PCIE4_RCHNG_CLK] = &gcc_pcie4_rchng_clk.clkr,
+	[GCC_PCIE_AUX_CLK_SRC] = &gcc_pcie_aux_clk_src.clkr,
+	[GCC_PCNOC_BFDCD_CLK_SRC] = &gcc_pcnoc_bfdcd_clk_src.clkr,
+	[GCC_QDSS_AT_CLK] = &gcc_qdss_at_clk.clkr,
+	[GCC_QDSS_AT_CLK_SRC] = &gcc_qdss_at_clk_src.clkr,
+	[GCC_QDSS_DAP_CLK] = &gcc_qdss_dap_clk.clkr,
+	[GCC_QDSS_TSCTR_CLK_SRC] = &gcc_qdss_tsctr_clk_src.clkr,
+	[GCC_QPIC_AHB_CLK] = &gcc_qpic_ahb_clk.clkr,
+	[GCC_QPIC_CLK] = &gcc_qpic_clk.clkr,
+	[GCC_QPIC_CLK_SRC] = &gcc_qpic_clk_src.clkr,
+	[GCC_QPIC_IO_MACRO_CLK] = &gcc_qpic_io_macro_clk.clkr,
+	[GCC_QPIC_IO_MACRO_CLK_SRC] = &gcc_qpic_io_macro_clk_src.clkr,
+	[GCC_QPIC_SLEEP_CLK] = &gcc_qpic_sleep_clk.clkr,
+	[GCC_QUPV3_2X_CORE_CLK_SRC] = &gcc_qupv3_2x_core_clk_src.clkr,
+	[GCC_QUPV3_AHB_MST_CLK] = &gcc_qupv3_ahb_mst_clk.clkr,
+	[GCC_QUPV3_AHB_SLV_CLK] = &gcc_qupv3_ahb_slv_clk.clkr,
+	[GCC_QUPV3_WRAP_SE0_CLK] = &gcc_qupv3_wrap_se0_clk.clkr,
+	[GCC_QUPV3_WRAP_SE0_CLK_SRC] = &gcc_qupv3_wrap_se0_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE1_CLK] = &gcc_qupv3_wrap_se1_clk.clkr,
+	[GCC_QUPV3_WRAP_SE1_CLK_SRC] = &gcc_qupv3_wrap_se1_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE2_CLK] = &gcc_qupv3_wrap_se2_clk.clkr,
+	[GCC_QUPV3_WRAP_SE2_CLK_SRC] = &gcc_qupv3_wrap_se2_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE3_CLK] = &gcc_qupv3_wrap_se3_clk.clkr,
+	[GCC_QUPV3_WRAP_SE3_CLK_SRC] = &gcc_qupv3_wrap_se3_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE4_CLK] = &gcc_qupv3_wrap_se4_clk.clkr,
+	[GCC_QUPV3_WRAP_SE4_CLK_SRC] = &gcc_qupv3_wrap_se4_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE5_CLK] = &gcc_qupv3_wrap_se5_clk.clkr,
+	[GCC_QUPV3_WRAP_SE5_CLK_SRC] = &gcc_qupv3_wrap_se5_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE6_CLK] = &gcc_qupv3_wrap_se6_clk.clkr,
+	[GCC_QUPV3_WRAP_SE6_CLK_SRC] = &gcc_qupv3_wrap_se6_clk_src.clkr,
+	[GCC_QUPV3_WRAP_SE7_CLK] = &gcc_qupv3_wrap_se7_clk.clkr,
+	[GCC_QUPV3_WRAP_SE7_CLK_SRC] = &gcc_qupv3_wrap_se7_clk_src.clkr,
+	[GCC_SDCC1_AHB_CLK] = &gcc_sdcc1_ahb_clk.clkr,
+	[GCC_SDCC1_APPS_CLK] = &gcc_sdcc1_apps_clk.clkr,
+	[GCC_SDCC1_APPS_CLK_SRC] = &gcc_sdcc1_apps_clk_src.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK] = &gcc_sdcc1_ice_core_clk.clkr,
+	[GCC_SDCC1_ICE_CORE_CLK_SRC] = &gcc_sdcc1_ice_core_clk_src.clkr,
+	[GCC_SLEEP_CLK_SRC] = &gcc_sleep_clk_src.clkr,
+	[GCC_SNOC_USB_CLK] = &gcc_snoc_usb_clk.clkr,
+	[GCC_SYSTEM_NOC_BFDCD_CLK_SRC] = &gcc_system_noc_bfdcd_clk_src.clkr,
+	[GCC_UNIPHY0_AHB_CLK] = &gcc_uniphy0_ahb_clk.clkr,
+	[GCC_UNIPHY0_SYS_CLK] = &gcc_uniphy0_sys_clk.clkr,
+	[GCC_UNIPHY1_AHB_CLK] = &gcc_uniphy1_ahb_clk.clkr,
+	[GCC_UNIPHY1_SYS_CLK] = &gcc_uniphy1_sys_clk.clkr,
+	[GCC_UNIPHY2_AHB_CLK] = &gcc_uniphy2_ahb_clk.clkr,
+	[GCC_UNIPHY2_SYS_CLK] = &gcc_uniphy2_sys_clk.clkr,
+	[GCC_UNIPHY_SYS_CLK_SRC] = &gcc_uniphy_sys_clk_src.clkr,
+	[GCC_USB0_AUX_CLK] = &gcc_usb0_aux_clk.clkr,
+	[GCC_USB0_AUX_CLK_SRC] = &gcc_usb0_aux_clk_src.clkr,
+	[GCC_USB0_EUD_AT_CLK] = &gcc_usb0_eud_at_clk.clkr,
+	[GCC_USB0_MASTER_CLK] = &gcc_usb0_master_clk.clkr,
+	[GCC_USB0_MASTER_CLK_SRC] = &gcc_usb0_master_clk_src.clkr,
+	[GCC_USB0_MOCK_UTMI_CLK] = &gcc_usb0_mock_utmi_clk.clkr,
+	[GCC_USB0_MOCK_UTMI_CLK_SRC] = &gcc_usb0_mock_utmi_clk_src.clkr,
+	[GCC_USB0_MOCK_UTMI_DIV_CLK_SRC] = &gcc_usb0_mock_utmi_div_clk_src.clkr,
+	[GCC_USB0_PHY_CFG_AHB_CLK] = &gcc_usb0_phy_cfg_ahb_clk.clkr,
+	[GCC_USB0_PIPE_CLK] = &gcc_usb0_pipe_clk.clkr,
+	[GCC_USB0_PIPE_CLK_SRC] = &gcc_usb0_pipe_clk_src.clkr,
+	[GCC_USB0_SLEEP_CLK] = &gcc_usb0_sleep_clk.clkr,
+	[GCC_USB1_MASTER_CLK] = &gcc_usb1_master_clk.clkr,
+	[GCC_USB1_MOCK_UTMI_CLK] = &gcc_usb1_mock_utmi_clk.clkr,
+	[GCC_USB1_MOCK_UTMI_CLK_SRC] = &gcc_usb1_mock_utmi_clk_src.clkr,
+	[GCC_USB1_MOCK_UTMI_DIV_CLK_SRC] = &gcc_usb1_mock_utmi_div_clk_src.clkr,
+	[GCC_USB1_PHY_CFG_AHB_CLK] = &gcc_usb1_phy_cfg_ahb_clk.clkr,
+	[GCC_USB1_SLEEP_CLK] = &gcc_usb1_sleep_clk.clkr,
+	[GCC_XO_CLK_SRC] = &gcc_xo_clk_src.clkr,
+	[GPLL0_MAIN] = &gpll0_main.clkr,
+	[GPLL0] = &gpll0.clkr,
+	[GPLL2] = &gpll2.clkr,
+	[GPLL2_OUT_MAIN] = &gpll2_out_main.clkr,
+	[GPLL4] = &gpll4.clkr,
+};
+
+static const struct qcom_reset_map gcc_ipq9650_resets[] = {
+	[GCC_ADSS_BCR] = { 0x1c000 },
+	[GCC_ADSS_PWM_CLK_ARES] = { 0x1c00c, 2 },
+	[GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR] = { 0x38000 },
+	[GCC_APC0_VOLTAGE_DROOP_DETECTOR_GPLL0_CLK_ARES] = { 0x3800c, 2 },
+	[GCC_APSS_AHB_CLK_ARES] = { 0x24014, 2 },
+	[GCC_APSS_ATB_CLK_ARES] = { 0x24034, 2 },
+	[GCC_APSS_AXI_CLK_ARES] = { 0x24018, 2 },
+	[GCC_APSS_TS_CLK_ARES] = { 0x24030, 2 },
+	[GCC_BOOT_ROM_AHB_CLK_ARES] = { 0x1302c, 2 },
+	[GCC_BOOT_ROM_BCR] = { 0x13028 },
+	[GCC_CPUSS_TRIG_CLK_ARES] = { 0x2401c, 2 },
+	[GCC_GP1_CLK_ARES] = { 0x8018, 2 },
+	[GCC_GP2_CLK_ARES] = { 0x8030, 2 },
+	[GCC_GP3_CLK_ARES] = { 0x8048, 2 },
+	[GCC_MDIO_AHB_CLK_ARES] = { 0x17040, 2 },
+	[GCC_MDIO_BCR] = { 0x1703c },
+	[GCC_NSS_BCR] = { 0x17000 },
+	[GCC_NSS_TS_CLK_ARES] = { 0x17018, 2 },
+	[GCC_NSSCC_CLK_ARES] = { 0x17034, 2 },
+	[GCC_NSSCFG_CLK_ARES] = { 0x1702c, 2 },
+	[GCC_NSSNOC_ATB_CLK_ARES] = { 0x17014, 2 },
+	[GCC_NSSNOC_MEMNOC_1_CLK_ARES] = { 0x17084, 2 },
+	[GCC_NSSNOC_MEMNOC_CLK_ARES] = { 0x17024, 2 },
+	[GCC_NSSNOC_NSSCC_CLK_ARES] = { 0x17030, 2 },
+	[GCC_NSSNOC_PCNOC_1_CLK_ARES] = { 0x17080, 2 },
+	[GCC_NSSNOC_QOSGEN_REF_CLK_ARES] = { 0x1701c, 2 },
+	[GCC_NSSNOC_SNOC_1_CLK_ARES] = { 0x1707c, 2 },
+	[GCC_NSSNOC_SNOC_CLK_ARES] = { 0x17028, 2 },
+	[GCC_NSSNOC_TIMEOUT_REF_CLK_ARES] = { 0x17020, 2 },
+	[GCC_NSSNOC_XO_DCD_CLK_ARES] = { 0x17074, 2 },
+	[GCC_PCIE0_AHB_CLK_ARES] = { 0x28030, 2 },
+	[GCC_PCIE0_AUX_CLK_ARES] = { 0x28070, 2 },
+	[GCC_PCIE0_AXI_M_CLK_ARES] = { 0x28038, 2 },
+	[GCC_PCIE0_AXI_S_BRIDGE_CLK_ARES] = { 0x28048, 2 },
+	[GCC_PCIE0_AXI_S_CLK_ARES] = { 0x28040, 2 },
+	[GCC_PCIE0_BCR] = { 0x28000 },
+	[GCC_PCIE0_LINK_DOWN_BCR] = { 0x28054 },
+	[GCC_PCIE0_PHY_BCR] = { 0x28060 },
+	[GCC_PCIE0_PIPE_CLK_ARES] = { 0x28068, 2 },
+	[GCC_PCIE0PHY_PHY_BCR] = { 0x2805c },
+	[GCC_PCIE0_PIPE_RESET] = { 0x28058, 0 },
+	[GCC_PCIE0_CORE_STICKY_RESET] = { 0x28058, 1 },
+	[GCC_PCIE0_AXI_S_STICKY_RESET] = { 0x28058, 2 },
+	[GCC_PCIE0_AXI_S_RESET] = { 0x28058, 3 },
+	[GCC_PCIE0_AXI_M_STICKY_RESET] = { 0x28058, 4 },
+	[GCC_PCIE0_AXI_M_RESET] = { 0x28058, 5 },
+	[GCC_PCIE0_AUX_RESET] = { 0x28058, 6 },
+	[GCC_PCIE0_AHB_RESET] = { 0x28058, 7 },
+	[GCC_PCIE1_AHB_CLK_ARES] = { 0x29030, 2 },
+	[GCC_PCIE1_AUX_CLK_ARES] = { 0x29074, 2 },
+	[GCC_PCIE1_AXI_M_CLK_ARES] = { 0x29038, 2 },
+	[GCC_PCIE1_AXI_S_BRIDGE_CLK_ARES] = { 0x29048, 2 },
+	[GCC_PCIE1_AXI_S_CLK_ARES] = { 0x29040, 2 },
+	[GCC_PCIE1_BCR] = { 0x29000 },
+	[GCC_PCIE1_LINK_DOWN_BCR] = { 0x29054 },
+	[GCC_PCIE1_PHY_BCR] = { 0x29060 },
+	[GCC_PCIE1_PIPE_CLK_ARES] = { 0x29068, 2 },
+	[GCC_PCIE1PHY_PHY_BCR] = { 0x2905c },
+	[GCC_PCIE1_PIPE_RESET] = { 0x29058, 0 },
+	[GCC_PCIE1_CORE_STICKY_RESET] = { 0x29058, 1 },
+	[GCC_PCIE1_AXI_S_STICKY_RESET] = { 0x29058, 2 },
+	[GCC_PCIE1_AXI_S_RESET] = { 0x29058, 3 },
+	[GCC_PCIE1_AXI_M_STICKY_RESET] = { 0x29058, 4 },
+	[GCC_PCIE1_AXI_M_RESET] = { 0x29058, 5 },
+	[GCC_PCIE1_AUX_RESET] = { 0x29058, 6 },
+	[GCC_PCIE1_AHB_RESET] = { 0x29058, 7 },
+	[GCC_PCIE2_AHB_CLK_ARES] = { 0x2a030, 2 },
+	[GCC_PCIE2_AUX_CLK_ARES] = { 0x2a078, 2 },
+	[GCC_PCIE2_AXI_M_CLK_ARES] = { 0x2a038, 2 },
+	[GCC_PCIE2_AXI_S_BRIDGE_CLK_ARES] = { 0x2a048, 2 },
+	[GCC_PCIE2_AXI_S_CLK_ARES] = { 0x2a040, 2 },
+	[GCC_PCIE2_BCR] = { 0x2a000 },
+	[GCC_PCIE2_LINK_DOWN_BCR] = { 0x2a054 },
+	[GCC_PCIE2_PHY_BCR] = { 0x2a060 },
+	[GCC_PCIE2_PIPE_CLK_ARES] = { 0x2a068, 2 },
+	[GCC_PCIE2PHY_PHY_BCR] = { 0x2a05c },
+	[GCC_PCIE2_PIPE_RESET] = { 0x2a058, 0 },
+	[GCC_PCIE2_CORE_STICKY_RESET] = { 0x2a058, 1 },
+	[GCC_PCIE2_AXI_S_STICKY_RESET] = { 0x2a058, 2 },
+	[GCC_PCIE2_AXI_S_RESET] = { 0x2a058, 3 },
+	[GCC_PCIE2_AXI_M_STICKY_RESET] = { 0x2a058, 4 },
+	[GCC_PCIE2_AXI_M_RESET] = { 0x2a058, 5 },
+	[GCC_PCIE2_AUX_RESET] = { 0x2a058, 6 },
+	[GCC_PCIE2_AHB_RESET] = { 0x2a058, 7 },
+	[GCC_PCIE3_AHB_CLK_ARES] = { 0x2b030, 2 },
+	[GCC_PCIE3_AUX_CLK_ARES] = { 0x2b07c, 2 },
+	[GCC_PCIE3_AXI_M_CLK_ARES] = { 0x2b038, 2 },
+	[GCC_PCIE3_AXI_S_BRIDGE_CLK_ARES] = { 0x2b048, 2 },
+	[GCC_PCIE3_AXI_S_CLK_ARES] = { 0x2b040, 2 },
+	[GCC_PCIE3_BCR] = { 0x2b000 },
+	[GCC_PCIE3_LINK_DOWN_BCR] = { 0x2b054 },
+	[GCC_PCIE3_PHY_BCR] = { 0x2b060 },
+	[GCC_PCIE3_PIPE_CLK_ARES] = { 0x2b068, 2 },
+	[GCC_PCIE3PHY_PHY_BCR] = { 0x2b05c },
+	[GCC_PCIE3_PIPE_RESET] = { 0x2b058, 0 },
+	[GCC_PCIE3_CORE_STICKY_RESET] = { 0x2b058, 1 },
+	[GCC_PCIE3_AXI_S_STICKY_RESET] = { 0x2b058, 2 },
+	[GCC_PCIE3_AXI_S_RESET] = { 0x2b058, 3 },
+	[GCC_PCIE3_AXI_M_STICKY_RESET] = { 0x2b058, 4 },
+	[GCC_PCIE3_AXI_M_RESET] = { 0x2b058, 5 },
+	[GCC_PCIE3_AUX_RESET] = { 0x2b058, 6 },
+	[GCC_PCIE3_AHB_RESET] = { 0x2b058, 7 },
+	[GCC_PCIE4_AHB_CLK_ARES] = { 0x2501c, 2 },
+	[GCC_PCIE4_AUX_CLK_ARES] = { 0x25020, 2 },
+	[GCC_PCIE4_AXI_M_CLK_ARES] = { 0x25028, 2 },
+	[GCC_PCIE4_AXI_S_BRIDGE_CLK_ARES] = { 0x25038, 2 },
+	[GCC_PCIE4_AXI_S_CLK_ARES] = { 0x25030, 2 },
+	[GCC_PCIE4_BCR] = { 0x25000 },
+	[GCC_PCIE4_LINK_DOWN_BCR] = { 0x25044 },
+	[GCC_PCIE4_PHY_BCR] = { 0x2504c },
+	[GCC_PCIE4_PIPE_CLK_ARES] = { 0x2503c, 2 },
+	[GCC_PCIE4_PIPE_RESET] = { 0x25054, 0 },
+	[GCC_PCIE4_CORE_STICKY_RESET] = { 0x25054, 1 },
+	[GCC_PCIE4_AXI_S_STICKY_RESET] = { 0x25054, 2 },
+	[GCC_PCIE4_AXI_S_RESET] = { 0x25054, 3 },
+	[GCC_PCIE4_AXI_M_STICKY_RESET] = { 0x25054, 4 },
+	[GCC_PCIE4_AXI_M_RESET] = { 0x25054, 5 },
+	[GCC_PCIE4_AUX_RESET] = { 0x25054, 6 },
+	[GCC_PCIE4_AHB_RESET] = { 0x25054, 7 },
+	[GCC_PCIE4PHY_PHY_BCR] = { 0x25048 },
+	[GCC_QDSS_APB2JTAG_CLK_ARES] = { 0x2d05c, 2 },
+	[GCC_QDSS_AT_CLK_ARES] = { 0x2d034, 2 },
+	[GCC_QDSS_BCR] = { 0x2d000 },
+	[GCC_QDSS_CFG_AHB_CLK_ARES] = { 0x2d068, 2 },
+	[GCC_QDSS_DAP_AHB_CLK_ARES] = { 0x2d064, 2 },
+	[GCC_QDSS_DAP_CLK_ARES] = { 0x2d058, 2 },
+	[GCC_QDSS_ETR_USB_CLK_ARES] = { 0x2d060, 2 },
+	[GCC_QDSS_EUD_AT_CLK_ARES] = { 0x2d06c, 2 },
+	[GCC_QDSS_STM_CLK_ARES] = { 0x2d03c, 2 },
+	[GCC_QDSS_TRACECLKIN_CLK_ARES] = { 0x2d040, 2 },
+	[GCC_QDSS_TS_CLK_ARES] = { 0x2d078, 2 },
+	[GCC_QDSS_TSCTR_DIV16_CLK_ARES] = { 0x2d054, 2 },
+	[GCC_QDSS_TSCTR_DIV2_CLK_ARES] = { 0x2d044, 2 },
+	[GCC_QDSS_TSCTR_DIV3_CLK_ARES] = { 0x2d048, 2 },
+	[GCC_QDSS_TSCTR_DIV4_CLK_ARES] = { 0x2d04c, 2 },
+	[GCC_QDSS_TSCTR_DIV8_CLK_ARES] = { 0x2d050, 2 },
+	[GCC_QPIC_AHB_CLK_ARES] = { 0x32010, 2 },
+	[GCC_QPIC_CLK_ARES] = { 0x32028, 2 },
+	[GCC_QPIC_BCR] = { 0x32000 },
+	[GCC_QPIC_IO_MACRO_CLK_ARES] = { 0x3200c, 2 },
+	[GCC_QPIC_SLEEP_CLK_ARES] = { 0x32018, 2 },
+	[GCC_QUPV3_2X_CORE_CLK_ARES] = { 0x1020, 2 },
+	[GCC_QUPV3_AHB_MST_CLK_ARES] = { 0x1014, 2 },
+	[GCC_QUPV3_AHB_SLV_CLK_ARES] = { 0x102c, 2 },
+	[GCC_QUPV3_BCR] = { 0x1000 },
+	[GCC_QUPV3_CORE_CLK_ARES] = { 0x1018, 2 },
+	[GCC_QUPV3_WRAP_SE0_CLK_ARES] = { 0x202c, 2 },
+	[GCC_QUPV3_WRAP_SE0_BCR] = { 0x2000 },
+	[GCC_QUPV3_WRAP_SE1_CLK_ARES] = { 0x302c, 2 },
+	[GCC_QUPV3_WRAP_SE1_BCR] = { 0x3000 },
+	[GCC_QUPV3_WRAP_SE2_CLK_ARES] = { 0x3048, 2 },
+	[GCC_QUPV3_WRAP_SE2_BCR] = { 0x3030 },
+	[GCC_QUPV3_WRAP_SE3_CLK_ARES] = { 0x3064, 2 },
+	[GCC_QUPV3_WRAP_SE3_BCR] = { 0x304c },
+	[GCC_QUPV3_WRAP_SE4_CLK_ARES] = { 0x3080, 2 },
+	[GCC_QUPV3_WRAP_SE4_BCR] = { 0x3068 },
+	[GCC_QUPV3_WRAP_SE5_CLK_ARES] = { 0x30a4, 2 },
+	[GCC_QUPV3_WRAP_SE5_BCR] = { 0x308c },
+	[GCC_QUPV3_WRAP_SE6_CLK_ARES] = { 0x4018, 2 },
+	[GCC_QUPV3_WRAP_SE6_BCR] = { 0x4000 },
+	[GCC_QUPV3_WRAP_SE7_CLK_ARES] = { 0x4034, 2 },
+	[GCC_QUPV3_WRAP_SE7_BCR] = { 0x401c },
+	[GCC_QUSB2_0_PHY_BCR] = { 0x2c068 },
+	[GCC_QUSB2_1_PHY_BCR] = { 0x3c030 },
+	[GCC_SDCC1_APPS_CLK_ARES] = { 0x3302c, 2 },
+	[GCC_SDCC1_ICE_CORE_CLK_ARES] = { 0x33034, 2 },
+	[GCC_SDCC_BCR] = { 0x33000 },
+	[GCC_TLMM_AHB_CLK_ARES] = { 0x3e004, 2 },
+	[GCC_TLMM_CLK_ARES] = { 0x3e008, 2 },
+	[GCC_TLMM_BCR] = { 0x3e000 },
+	[GCC_UNIPHY0_AHB_CLK_ARES] = { 0x1704c, 2 },
+	[GCC_UNIPHY0_BCR] = { 0x17044 },
+	[GCC_UNIPHY0_PMA_BCR] = { 0x17098 },
+	[GCC_UNIPHY0_SYS_CLK_ARES] = { 0x17048, 2 },
+	[GCC_UNIPHY1_AHB_CLK_ARES] = { 0x1705c, 2 },
+	[GCC_UNIPHY1_BCR] = { 0x17054 },
+	[GCC_UNIPHY1_PMA_BCR] = { 0x1709c },
+	[GCC_UNIPHY1_SYS_CLK_ARES] = { 0x17058, 2 },
+	[GCC_UNIPHY2_AHB_CLK_ARES] = { 0x1706c, 2 },
+	[GCC_UNIPHY2_BCR] = { 0x17064 },
+	[GCC_UNIPHY2_PMA_BCR] = { 0x170a0 },
+	[GCC_UNIPHY2_SYS_CLK_ARES] = { 0x17068, 2 },
+	[GCC_UNIPHY0_XPCS_ARES] = { 0x17050, 2 },
+	[GCC_UNIPHY1_XLGPCS_ARES] = { 0x17060, 1 },
+	[GCC_UNIPHY1_XPCS_ARES] = { 0x17060, 2 },
+	[GCC_UNIPHY2_XLGPCS_ARES] = { 0x17070, 1 },
+	[GCC_UNIPHY2_XPCS_ARES] = { 0x17070, 2 },
+	[GCC_USB0_AUX_CLK_ARES] = { 0x2c04c, 2 },
+	[GCC_USB0_MASTER_CLK_ARES] = { 0x2c044, 2 },
+	[GCC_USB0_MOCK_UTMI_CLK_ARES] = { 0x2c050, 2 },
+	[GCC_USB0_PHY_BCR] = { 0x2c06c },
+	[GCC_USB0_PHY_CFG_AHB_CLK_ARES] = { 0x2c05c, 2 },
+	[GCC_USB0_PIPE_CLK_ARES] = { 0x2c054, 2 },
+	[GCC_USB0_SLEEP_CLK_ARES] = { 0x2c058, 2 },
+	[GCC_USB1_BCR] = { 0x3c000 },
+	[GCC_USB1_MASTER_CLK_ARES] = { 0x3c028, 2 },
+	[GCC_USB1_MOCK_UTMI_CLK_ARES] = { 0x3c024, 2 },
+	[GCC_USB1_PHY_CFG_AHB_CLK_ARES] = { 0x3c01c, 2 },
+	[GCC_USB1_SLEEP_CLK_ARES] = { 0x3c020, 2 },
+	[GCC_USB3PHY_0_PHY_BCR] = { 0x2c070 },
+	[GCC_USB_BCR] = { 0x2c000 },
+};
+
+static const struct of_device_id gcc_ipq9650_match_table[] = {
+	{ .compatible = "qcom,ipq9650-gcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, gcc_ipq9650_match_table);
+
+static const struct regmap_config gcc_ipq9650_regmap_config = {
+	.reg_bits       = 32,
+	.reg_stride     = 4,
+	.val_bits       = 32,
+	.max_register   = 0x3f024,
+	.fast_io        = true,
+};
+
+static struct clk_hw *gcc_ipq9650_hws[] = {
+	&gpll0_div2.hw,
+	&gcc_xo_div4_clk_src.hw,
+	&gcc_qdss_dap_sync_clk_src.hw,
+	&gcc_eud_at_div_clk_src.hw,
+};
+
+static const struct qcom_cc_desc gcc_ipq9650_desc = {
+	.config = &gcc_ipq9650_regmap_config,
+	.clks = gcc_ipq9650_clocks,
+	.num_clks = ARRAY_SIZE(gcc_ipq9650_clocks),
+	.resets = gcc_ipq9650_resets,
+	.num_resets = ARRAY_SIZE(gcc_ipq9650_resets),
+	.clk_hws = gcc_ipq9650_hws,
+	.num_clk_hws = ARRAY_SIZE(gcc_ipq9650_hws),
+};
+
+static int gcc_ipq9650_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &gcc_ipq9650_desc);
+}
+
+static struct platform_driver gcc_ipq9650_driver = {
+	.probe = gcc_ipq9650_probe,
+	.driver = {
+		.name   = "qcom,gcc-ipq9650",
+		.of_match_table = gcc_ipq9650_match_table,
+	},
+};
+
+static int __init gcc_ipq9650_init(void)
+{
+	return platform_driver_register(&gcc_ipq9650_driver);
+}
+core_initcall(gcc_ipq9650_init);
+
+static void __exit gcc_ipq9650_exit(void)
+{
+	platform_driver_unregister(&gcc_ipq9650_driver);
+}
+module_exit(gcc_ipq9650_exit);
+
+MODULE_DESCRIPTION("QTI GCC IPQ9650 Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/qcom/gcc-nord.c b/drivers/clk/qcom/gcc-nord.c
index 3098d8fac0fb2..8a6e429f26402 100644
--- a/drivers/clk/qcom/gcc-nord.c
+++ b/drivers/clk/qcom/gcc-nord.c
@@ -1850,7 +1850,7 @@ static const struct regmap_config gcc_nord_regmap_config = {
 	.fast_io = true,
 };
 
-static struct qcom_cc_driver_data gcc_nord_driver_data = {
+static const struct qcom_cc_driver_data gcc_nord_driver_data = {
 	.dfs_rcgs = gcc_nord_dfs_clocks,
 	.num_dfs_rcgs = ARRAY_SIZE(gcc_nord_dfs_clocks),
 };
diff --git a/drivers/clk/qcom/gpucc-sm8750.c b/drivers/clk/qcom/gpucc-sm8750.c
index 5d52c6d8b5e51..1466bd36403f0 100644
--- a/drivers/clk/qcom/gpucc-sm8750.c
+++ b/drivers/clk/qcom/gpucc-sm8750.c
@@ -421,7 +421,7 @@ static struct clk_alpha_pll *gpu_cc_alpha_plls[] = {
 	&gpu_cc_pll0,
 };
 
-static u32 gpu_cc_sm8750_critical_cbcrs[] = {
+static const u32 gpu_cc_sm8750_critical_cbcrs[] = {
 	0x9004, /* GPU_CC_RSCC_XO_AON_CLK */
 	0x9008, /* GPU_CC_CXO_AON_CLK */
 	0x9064, /* GPU_CC_GX_AHB_FF_CLK */
@@ -430,7 +430,7 @@ static u32 gpu_cc_sm8750_critical_cbcrs[] = {
 	0x93a8, /* GPU_CC_RSCC_HUB_AON_CLK */
 };
 
-static struct qcom_cc_driver_data gpu_cc_sm8750_driver_data = {
+static const struct qcom_cc_driver_data gpu_cc_sm8750_driver_data = {
 	.alpha_plls = gpu_cc_alpha_plls,
 	.num_alpha_plls = ARRAY_SIZE(gpu_cc_alpha_plls),
 	.clk_cbcrs = gpu_cc_sm8750_critical_cbcrs,
diff --git a/drivers/clk/qcom/gxclkctl-kaanapali.c b/drivers/clk/qcom/gxclkctl-kaanapali.c
index 40d856378a74a..7b0af0ba1e685 100644
--- a/drivers/clk/qcom/gxclkctl-kaanapali.c
+++ b/drivers/clk/qcom/gxclkctl-kaanapali.c
@@ -53,6 +53,7 @@ static const struct qcom_cc_desc gx_clkctl_kaanapali_desc = {
 static const struct of_device_id gx_clkctl_kaanapali_match_table[] = {
 	{ .compatible = "qcom,glymur-gxclkctl" },
 	{ .compatible = "qcom,kaanapali-gxclkctl" },
+	{ .compatible = "qcom,milos-gxclkctl" },
 	{ .compatible = "qcom,sm8750-gxclkctl" },
 	{ }
 };
diff --git a/drivers/clk/qcom/ipq-cmn-pll.c b/drivers/clk/qcom/ipq-cmn-pll.c
index 5763e4df59a1a..441e88101ea30 100644
--- a/drivers/clk/qcom/ipq-cmn-pll.c
+++ b/drivers/clk/qcom/ipq-cmn-pll.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2024-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 /*
@@ -20,6 +20,11 @@
  * and an output clock to NSS (network subsystem) at 300 MHZ. The other output
  * clocks from CMN PLL on IPQ5424 are the same as IPQ9574.
  *
+ * On the IPQ5332 SoC, the CMN PLL provides a single 50 MHZ clock output to
+ * the Ethernet PHY (or switch) via the UNIPHY (PCS). It also supplies a 200
+ * MHZ clock to the PPE. The remaining fixed-rate clocks to the GCC and PCS
+ * are the same as those in the IPQ9574 SoC.
+ *
  *               +---------+
  *               |   GCC   |
  *               +--+---+--+
@@ -51,6 +56,7 @@
 
 #include <dt-bindings/clock/qcom,ipq-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq5018-cmn-pll.h>
+#include <dt-bindings/clock/qcom,ipq5332-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq5424-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq6018-cmn-pll.h>
 #include <dt-bindings/clock/qcom,ipq8074-cmn-pll.h>
@@ -131,6 +137,16 @@ static const struct cmn_pll_fixed_output_clk ipq8074_output_clks[] = {
 	{ /* Sentinel */ }
 };
 
+static const struct cmn_pll_fixed_output_clk ipq5332_output_clks[] = {
+	CLK_PLL_OUTPUT(IPQ5332_XO_24MHZ_CLK, "xo-24mhz", 24000000UL),
+	CLK_PLL_OUTPUT(IPQ5332_SLEEP_32KHZ_CLK, "sleep-32khz", 32000UL),
+	CLK_PLL_OUTPUT(IPQ5332_PCS_31P25MHZ_CLK, "pcs-31p25mhz", 31250000UL),
+	CLK_PLL_OUTPUT(IPQ5332_NSS_300MHZ_CLK, "nss-300mhz", 300000000UL),
+	CLK_PLL_OUTPUT(IPQ5332_PPE_200MHZ_CLK, "ppe-200mhz", 200000000UL),
+	CLK_PLL_OUTPUT(IPQ5332_ETH_50MHZ_CLK, "eth-50mhz", 50000000UL),
+	{ /* Sentinel */ }
+};
+
 static const struct cmn_pll_fixed_output_clk ipq5424_output_clks[] = {
 	CLK_PLL_OUTPUT(IPQ5424_XO_24MHZ_CLK, "xo-24mhz", 24000000UL),
 	CLK_PLL_OUTPUT(IPQ5424_SLEEP_32KHZ_CLK, "sleep-32khz", 32000UL),
@@ -199,7 +215,7 @@ static unsigned long clk_cmn_pll_recalc_rate(struct clk_hw *hw,
 					     unsigned long parent_rate)
 {
 	struct clk_cmn_pll *cmn_pll = to_clk_cmn_pll(hw);
-	u32 val, factor;
+	u32 val, factor, ref_div;
 
 	/*
 	 * The value of CMN_PLL_DIVIDER_CTRL_FACTOR is automatically adjusted
@@ -207,8 +223,15 @@ static unsigned long clk_cmn_pll_recalc_rate(struct clk_hw *hw,
 	 */
 	regmap_read(cmn_pll->regmap, CMN_PLL_DIVIDER_CTRL, &val);
 	factor = FIELD_GET(CMN_PLL_DIVIDER_CTRL_FACTOR, val);
+	if (WARN_ON(factor == 0))
+		factor = 1;
+
+	regmap_read(cmn_pll->regmap, CMN_PLL_REFCLK_CONFIG, &val);
+	ref_div = FIELD_GET(CMN_PLL_REFCLK_DIV, val);
+	if (WARN_ON(ref_div == 0))
+		ref_div = 1;
 
-	return parent_rate * 2 * factor;
+	return div_u64((u64)parent_rate * 2 * factor, ref_div);
 }
 
 static int clk_cmn_pll_determine_rate(struct clk_hw *hw,
@@ -461,6 +484,7 @@ static const struct dev_pm_ops ipq_cmn_pll_pm_ops = {
 
 static const struct of_device_id ipq_cmn_pll_clk_ids[] = {
 	{ .compatible = "qcom,ipq5018-cmn-pll", .data = &ipq5018_output_clks },
+	{ .compatible = "qcom,ipq5332-cmn-pll", .data = &ipq5332_output_clks },
 	{ .compatible = "qcom,ipq5424-cmn-pll", .data = &ipq5424_output_clks },
 	{ .compatible = "qcom,ipq6018-cmn-pll", .data = &ipq6018_output_clks },
 	{ .compatible = "qcom,ipq8074-cmn-pll", .data = &ipq8074_output_clks },
diff --git a/drivers/clk/qcom/negcc-nord.c b/drivers/clk/qcom/negcc-nord.c
index 1aa24e2784e53..2cb66b0691a65 100644
--- a/drivers/clk/qcom/negcc-nord.c
+++ b/drivers/clk/qcom/negcc-nord.c
@@ -1945,7 +1945,7 @@ static void clk_nord_regs_configure(struct device *dev, struct regmap *regmap)
 	qcom_branch_set_force_mem_core(regmap, ne_gcc_ufs_phy_axi_clk, true);
 }
 
-static struct qcom_cc_driver_data ne_gcc_nord_driver_data = {
+static const struct qcom_cc_driver_data ne_gcc_nord_driver_data = {
 	.dfs_rcgs = ne_gcc_nord_dfs_clocks,
 	.num_dfs_rcgs = ARRAY_SIZE(ne_gcc_nord_dfs_clocks),
 	.clk_regs_configure = clk_nord_regs_configure,
diff --git a/drivers/clk/qcom/nwgcc-nord.c b/drivers/clk/qcom/nwgcc-nord.c
index 163ab63c872bc..961cae47ff7ca 100644
--- a/drivers/clk/qcom/nwgcc-nord.c
+++ b/drivers/clk/qcom/nwgcc-nord.c
@@ -626,7 +626,7 @@ static const struct qcom_reset_map nw_gcc_nord_resets[] = {
 	[NW_GCC_VIDEO_BCR] = { 0x1a000 },
 };
 
-static u32 nw_gcc_nord_critical_cbcrs[] = {
+static const u32 nw_gcc_nord_critical_cbcrs[] = {
 	0x16004, /* NW_GCC_CAMERA_AHB_CLK */
 	0x16030, /* NW_GCC_CAMERA_XO_CLK */
 	0x18004, /* NW_GCC_DISP_0_AHB_CLK */
@@ -641,7 +641,7 @@ static u32 nw_gcc_nord_critical_cbcrs[] = {
 	0x1a044, /* NW_GCC_VIDEO_XO_CLK */
 };
 
-static struct qcom_cc_driver_data nw_gcc_nord_driver_data = {
+static const struct qcom_cc_driver_data nw_gcc_nord_driver_data = {
 	.clk_cbcrs = nw_gcc_nord_critical_cbcrs,
 	.num_clk_cbcrs = ARRAY_SIZE(nw_gcc_nord_critical_cbcrs),
 };
diff --git a/drivers/clk/qcom/segcc-nord.c b/drivers/clk/qcom/segcc-nord.c
index 1aab0999de4dd..c82a56d971544 100644
--- a/drivers/clk/qcom/segcc-nord.c
+++ b/drivers/clk/qcom/segcc-nord.c
@@ -1568,7 +1568,7 @@ static const struct regmap_config se_gcc_nord_regmap_config = {
 	.fast_io = true,
 };
 
-static struct qcom_cc_driver_data se_gcc_nord_driver_data = {
+static const struct qcom_cc_driver_data se_gcc_nord_driver_data = {
 	.dfs_rcgs = se_gcc_nord_dfs_clocks,
 	.num_dfs_rcgs = ARRAY_SIZE(se_gcc_nord_dfs_clocks),
 };
diff --git a/drivers/clk/qcom/tcsrcc-hawi.c b/drivers/clk/qcom/tcsrcc-hawi.c
new file mode 100644
index 0000000000000..c942b0c8e09f3
--- /dev/null
+++ b/drivers/clk/qcom/tcsrcc-hawi.c
@@ -0,0 +1,158 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/clk-provider.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include <dt-bindings/clock/qcom,hawi-tcsrcc.h>
+
+#include "clk-alpha-pll.h"
+#include "clk-branch.h"
+#include "clk-pll.h"
+#include "clk-rcg.h"
+#include "clk-regmap.h"
+#include "clk-regmap-divider.h"
+#include "clk-regmap-mux.h"
+#include "common.h"
+#include "reset.h"
+
+enum {
+	DT_BI_TCXO_PAD,
+};
+
+static struct clk_branch tcsr_pcie_0_clkref_en = {
+	.halt_reg = 0x4c,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x4c,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_pcie_0_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_pcie_1_clkref_en = {
+	.halt_reg = 0x0,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x0,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_pcie_1_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_ufs_clkref_en = {
+	.halt_reg = 0x10,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x10,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_ufs_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_usb2_clkref_en = {
+	.halt_reg = 0x18,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x18,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_usb2_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_branch tcsr_usb3_clkref_en = {
+	.halt_reg = 0x8,
+	.halt_check = BRANCH_HALT_DELAY,
+	.clkr = {
+		.enable_reg = 0x8,
+		.enable_mask = BIT(0),
+		.hw.init = &(const struct clk_init_data) {
+			.name = "tcsr_usb3_clkref_en",
+			.parent_data = &(const struct clk_parent_data){
+				.index = DT_BI_TCXO_PAD,
+			},
+			.num_parents = 1,
+			.ops = &clk_branch2_ops,
+		},
+	},
+};
+
+static struct clk_regmap *tcsr_cc_hawi_clocks[] = {
+	[TCSR_PCIE_0_CLKREF_EN] = &tcsr_pcie_0_clkref_en.clkr,
+	[TCSR_PCIE_1_CLKREF_EN] = &tcsr_pcie_1_clkref_en.clkr,
+	[TCSR_UFS_CLKREF_EN] = &tcsr_ufs_clkref_en.clkr,
+	[TCSR_USB2_CLKREF_EN] = &tcsr_usb2_clkref_en.clkr,
+	[TCSR_USB3_CLKREF_EN] = &tcsr_usb3_clkref_en.clkr,
+};
+
+static const struct regmap_config tcsr_cc_hawi_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x4c,
+	.fast_io = true,
+};
+
+static const struct qcom_cc_desc tcsr_cc_hawi_desc = {
+	.config = &tcsr_cc_hawi_regmap_config,
+	.clks = tcsr_cc_hawi_clocks,
+	.num_clks = ARRAY_SIZE(tcsr_cc_hawi_clocks),
+};
+
+static const struct of_device_id tcsr_cc_hawi_match_table[] = {
+	{ .compatible = "qcom,hawi-tcsrcc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tcsr_cc_hawi_match_table);
+
+static int tcsr_cc_hawi_probe(struct platform_device *pdev)
+{
+	return qcom_cc_probe(pdev, &tcsr_cc_hawi_desc);
+}
+
+static struct platform_driver tcsr_cc_hawi_driver = {
+	.probe = tcsr_cc_hawi_probe,
+	.driver = {
+		.name = "tcsrcc-hawi",
+		.of_match_table = tcsr_cc_hawi_match_table,
+	},
+};
+
+module_platform_driver(tcsr_cc_hawi_driver);
+
+MODULE_DESCRIPTION("QTI TCSRCC HAWI Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/clk/renesas/clk-r8a73a4.c b/drivers/clk/renesas/clk-r8a73a4.c
index 7f47644396e6c..1a2bbc5e92999 100644
--- a/drivers/clk/renesas/clk-r8a73a4.c
+++ b/drivers/clk/renesas/clk-r8a73a4.c
@@ -42,6 +42,8 @@ static struct div4_clk div4_clks[] = {
 	{ "b",	CPG_FRQCRA,  8 },
 	{ "m1", CPG_FRQCRA,  4 },
 	{ "m2", CPG_FRQCRA,  0 },
+	{ "ztr", CPG_FRQCRB, 20 },
+	{ "zt", CPG_FRQCRB, 16 },
 	{ "zx", CPG_FRQCRB, 12 },
 	{ "zs", CPG_FRQCRB,  8 },
 	{ "hp", CPG_FRQCRB,  4 },
diff --git a/drivers/clk/renesas/clk-r8a7740.c b/drivers/clk/renesas/clk-r8a7740.c
index 635d59ead499e..31a79674583e8 100644
--- a/drivers/clk/renesas/clk-r8a7740.c
+++ b/drivers/clk/renesas/clk-r8a7740.c
@@ -37,6 +37,8 @@ static struct div4_clk div4_clks[] = {
 	{ "zg", CPG_FRQCRA, 16 },
 	{ "b", CPG_FRQCRA,  8 },
 	{ "m1", CPG_FRQCRA,  4 },
+	{ "ztr", CPG_FRQCRB,  20 },
+	{ "zt", CPG_FRQCRB,  16 },
 	{ "hp", CPG_FRQCRB,  4 },
 	{ "hpp", CPG_FRQCRC, 20 },
 	{ "usbp", CPG_FRQCRC, 16 },
diff --git a/drivers/clk/renesas/r9a08g046-cpg.c b/drivers/clk/renesas/r9a08g046-cpg.c
index 6759957980f22..fc9db5a2f0ac6 100644
--- a/drivers/clk/renesas/r9a08g046-cpg.c
+++ b/drivers/clk/renesas/r9a08g046-cpg.c
@@ -17,17 +17,72 @@
 /* RZ/G3L Specific registers. */
 #define G3L_CPG_PL2_DDIV		(0x204)
 #define G3L_CPG_PL3_DDIV		(0x208)
+#define G3L_CPG_CA55CORE_DDIV		(0x234)
+#define G3L_CPG_RSCI_DDIV		(0x238)
+#define G3L_CPG_RSPI_DDIV		(0x23c)
 #define G3L_CLKDIVSTATUS		(0x280)
+#define G3L_CPG_ETH_SSEL		(0x410)
+#define G3L_CPG_RSCI_SSEL		(0x414)
+#define G3L_CPG_RSPI_SSEL		(0x418)
+#define G3L_CPG_ETH_SDIV		(0x434)
 
 /* RZ/G3L Specific division configuration.  */
 #define G3L_DIVPL2A		DDIV_PACK(G3L_CPG_PL2_DDIV, 0, 2)
 #define G3L_DIVPL2B		DDIV_PACK(G3L_CPG_PL2_DDIV, 4, 2)
 #define G3L_DIVPL3A		DDIV_PACK(G3L_CPG_PL3_DDIV, 0, 2)
+#define G3L_DIV_CA55_CORE0	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 0, 3)
+#define G3L_DIV_CA55_CORE1	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 4, 3)
+#define G3L_DIV_CA55_CORE2	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 8, 3)
+#define G3L_DIV_CA55_CORE3	DDIV_PACK(G3L_CPG_CA55CORE_DDIV, 12, 3)
+#define G3L_DIV_RSCI0		DDIV_PACK(G3L_CPG_RSCI_DDIV, 0, 2)
+#define G3L_DIV_RSCI1		DDIV_PACK(G3L_CPG_RSCI_DDIV, 2, 2)
+#define G3L_DIV_RSCI2		DDIV_PACK(G3L_CPG_RSCI_DDIV, 4, 2)
+#define G3L_DIV_RSCI3		DDIV_PACK(G3L_CPG_RSCI_DDIV, 6, 2)
+#define G3L_DIV_RSPI0		DDIV_PACK(G3L_CPG_RSPI_DDIV, 0, 2)
+#define G3L_DIV_RSPI1		DDIV_PACK(G3L_CPG_RSPI_DDIV, 2, 2)
+#define G3L_DIV_RSPI2		DDIV_PACK(G3L_CPG_RSPI_DDIV, 4, 2)
+#define G3L_SDIV_ETH_A		DDIV_PACK(G3L_CPG_ETH_SDIV, 0, 2)
+#define G3L_SDIV_ETH_B		DDIV_PACK(G3L_CPG_ETH_SDIV, 4, 1)
+#define G3L_SDIV_ETH_C		DDIV_PACK(G3L_CPG_ETH_SDIV, 8, 2)
+#define G3L_SDIV_ETH_D		DDIV_PACK(G3L_CPG_ETH_SDIV, 12, 1)
 
 /* RZ/G3L Clock status configuration. */
 #define G3L_DIVPL2A_STS		DDIV_PACK(G3L_CLKDIVSTATUS, 4, 1)
 #define G3L_DIVPL2B_STS		DDIV_PACK(G3L_CLKDIVSTATUS, 5, 1)
 #define G3L_DIVPL3A_STS		DDIV_PACK(G3L_CLKDIVSTATUS, 8, 1)
+#define G3L_DIV_CA55_CORE0_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 12, 1)
+#define G3L_DIV_CA55_CORE1_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 13, 1)
+#define G3L_DIV_CA55_CORE2_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 14, 1)
+#define G3L_DIV_CA55_CORE3_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 15, 1)
+#define G3L_DIV_RSCI0_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 16, 1)
+#define G3L_DIV_RSCI1_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 17, 1)
+#define G3L_DIV_RSCI2_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 18, 1)
+#define G3L_DIV_RSCI3_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 19, 1)
+#define G3L_DIV_RSPI0_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 20, 1)
+#define G3L_DIV_RSPI1_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 21, 1)
+#define G3L_DIV_RSPI2_STS	DDIV_PACK(G3L_CLKDIVSTATUS, 22, 1)
+
+/* RZ/G3L Specific clocks select. */
+#define G3L_SEL_ETH0_TX		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 0, 1)
+#define G3L_SEL_ETH0_RX		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 1, 1)
+#define G3L_SEL_ETH0_RM		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 2, 1)
+#define G3L_SEL_ETH0_CLK_TX_I	SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 3, 1)
+#define G3L_SEL_ETH0_CLK_RX_I	SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 4, 1)
+#define G3L_SEL_ETH1_TX		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 8, 1)
+#define G3L_SEL_ETH1_RX		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 9, 1)
+#define G3L_SEL_ETH1_RM		SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 10, 1)
+#define G3L_SEL_ETH1_CLK_TX_I	SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 11, 1)
+#define G3L_SEL_ETH1_CLK_RX_I	SEL_PLL_PACK(G3L_CPG_ETH_SSEL, 12, 1)
+#define G3L_SEL_RSCI0		SEL_PLL_PACK(G3L_CPG_RSCI_SSEL, 0, 2)
+#define G3L_SEL_RSCI1		SEL_PLL_PACK(G3L_CPG_RSCI_SSEL, 2, 2)
+#define G3L_SEL_RSCI2		SEL_PLL_PACK(G3L_CPG_RSCI_SSEL, 4, 2)
+#define G3L_SEL_RSCI3		SEL_PLL_PACK(G3L_CPG_RSCI_SSEL, 6, 2)
+#define G3L_SEL_RSPI0		SEL_PLL_PACK(G3L_CPG_RSPI_SSEL, 0, 2)
+#define G3L_SEL_RSPI1		SEL_PLL_PACK(G3L_CPG_RSPI_SSEL, 2, 2)
+#define G3L_SEL_RSPI2		SEL_PLL_PACK(G3L_CPG_RSPI_SSEL, 4, 2)
+
+/* PLL 1/4/6/7 configuration registers macro. */
+#define G3L_PLL1467_CONF(clk1, clk2, setting)	((clk1) << 22 | (clk2) << 12 | (setting))
 
 enum clk_ids {
 	/* Core Clock Outputs exported to DT */
@@ -41,16 +96,72 @@ enum clk_ids {
 	CLK_ETH1_RXC_RX_CLK_IN,
 
 	/* Internal Core Clocks */
+	CLK_PLL1,
 	CLK_PLL2,
 	CLK_PLL2_DIV2,
+	CLK_PLL2_DIV2_4,
+	CLK_PLL2_DIV5,
+	CLK_PLL2_DIV6,
+	CLK_PLL2_DIV7,
 	CLK_PLL3,
 	CLK_PLL3_DIV2,
+	CLK_PLL6,
+	CLK_PLL6_DIV10,
+	CLK_SEL_ETH0_TX,
+	CLK_SEL_ETH0_RX,
+	CLK_SEL_ETH0_RM,
+	CLK_SEL_ETH1_TX,
+	CLK_SEL_ETH1_RX,
+	CLK_SEL_ETH1_RM,
+	CLK_SEL_RSCI0,
+	CLK_SEL_RSCI1,
+	CLK_SEL_RSCI2,
+	CLK_SEL_RSCI3,
+	CLK_SEL_RSPI0,
+	CLK_SEL_RSPI1,
+	CLK_SEL_RSPI2,
+	CLK_ETH0_TR,
+	CLK_ETH0_RM,
+	CLK_ETH1_TR,
+	CLK_ETH1_RM,
 
 	/* Module Clocks */
 	MOD_CLK_BASE,
 };
 
 /* Divider tables */
+static const struct clk_div_table dtable_1_8[] = {
+	{ 0, 1 },
+	{ 1, 2 },
+	{ 2, 4 },
+	{ 3, 8 },
+	{ 0, 0 },
+};
+
+static const struct clk_div_table dtable_1_32[] = {
+	{ 0, 1 },
+	{ 1, 2 },
+	{ 2, 4 },
+	{ 3, 8 },
+	{ 4, 16 },
+	{ 5, 32 },
+	{ 0, 0 },
+};
+
+static const struct clk_div_table dtable_2_20[] = {
+	{ 0, 2 },
+	{ 1, 20 },
+	{ 0, 0 },
+};
+
+static const struct clk_div_table dtable_2_16[] = {
+	{ 0, 2 },
+	{ 1, 4 },
+	{ 2, 8 },
+	{ 3, 16 },
+	{ 0, 0 },
+};
+
 static const struct clk_div_table dtable_4_128[] = {
 	{ 0, 4 },
 	{ 1, 8 },
@@ -59,6 +170,13 @@ static const struct clk_div_table dtable_4_128[] = {
 	{ 0, 0 },
 };
 
+static const struct clk_div_table dtable_4_200[] = {
+	{ 0, 4 },
+	{ 1, 20 },
+	{ 2, 200 },
+	{ 0, 0 },
+};
+
 static const struct clk_div_table dtable_8_256[] = {
 	{ 0, 8 },
 	{ 1, 16 },
@@ -67,6 +185,19 @@ static const struct clk_div_table dtable_8_256[] = {
 	{ 0, 0 },
 };
 
+/* Mux clock names tables. */
+static const char * const sel_eth0_tx[] = { ".div_eth0_tr", "eth0_txc_tx_clk" };
+static const char * const sel_eth0_rx[] = { ".div_eth0_tr", "eth0_rxc_rx_clk" };
+static const char * const sel_eth0_rm[] = { ".pll6_div10", "eth0_rxc_rx_clk" };
+static const char * const sel_eth1_tx[] = { ".div_eth1_tr", "eth1_txc_tx_clk" };
+static const char * const sel_eth1_rx[] = { ".div_eth1_tr", "eth1_rxc_rx_clk" };
+static const char * const sel_eth1_rm[] = { ".pll6_div10", "eth1_rxc_rx_clk" };
+static const char * const sel_rsci_rspi[] = { ".pll2_div5", ".pll2_div6", ".pll2_div7", ".pll2_div2_4" };
+static const char * const sel_eth0_clk_tx_i[] = { ".sel_eth0_tx", ".div_eth0_rm" };
+static const char * const sel_eth0_clk_rx_i[] = { ".sel_eth0_rx", ".div_eth0_rm" };
+static const char * const sel_eth1_clk_tx_i[] = { ".sel_eth1_tx", ".div_eth1_rm" };
+static const char * const sel_eth1_clk_rx_i[] = { ".sel_eth1_rx", ".div_eth1_rm" };
+
 static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	/* External Clock Inputs */
 	DEF_INPUT("extal", CLK_EXTAL),
@@ -76,18 +207,82 @@ static const struct cpg_core_clk r9a08g046_core_clks[] __initconst = {
 	DEF_INPUT("eth1_rxc_rx_clk", CLK_ETH1_RXC_RX_CLK_IN),
 
 	/* Internal Core Clocks */
+	DEF_G3L_PLL(".pll1", CLK_PLL1, CLK_EXTAL, G3L_PLL1467_CONF(0x4, 0x8, 0x100),
+		    1200000000UL),
 	DEF_FIXED(".pll2", CLK_PLL2, CLK_EXTAL, 200, 3),
 	DEF_FIXED(".pll3", CLK_PLL3, CLK_EXTAL, 200, 3),
+	DEF_G3L_PLL(".pll6", CLK_PLL6, CLK_EXTAL, G3L_PLL1467_CONF(0x54, 0x58, 0),
+		    500000000UL),
 	DEF_FIXED(".pll2_div2", CLK_PLL2_DIV2, CLK_PLL2, 1, 2),
+	DEF_FIXED(".pll2_div2_4", CLK_PLL2_DIV2_4, CLK_PLL2_DIV2, 1, 4),
+	DEF_FIXED(".pll2_div5", CLK_PLL2_DIV5, CLK_PLL2, 1, 5),
+	DEF_FIXED(".pll2_div6", CLK_PLL2_DIV6, CLK_PLL2, 1, 6),
+	DEF_FIXED(".pll2_div7", CLK_PLL2_DIV7, CLK_PLL2, 1, 7),
 	DEF_FIXED(".pll3_div2", CLK_PLL3_DIV2, CLK_PLL3, 1, 2),
+	DEF_FIXED(".pll6_div10", CLK_PLL6_DIV10, CLK_PLL6, 1, 10),
+	DEF_MUX(".sel_rsci0", CLK_SEL_RSCI0, G3L_SEL_RSCI0, sel_rsci_rspi),
+	DEF_MUX(".sel_rsci1", CLK_SEL_RSCI1, G3L_SEL_RSCI1, sel_rsci_rspi),
+	DEF_MUX(".sel_rsci2", CLK_SEL_RSCI2, G3L_SEL_RSCI2, sel_rsci_rspi),
+	DEF_MUX(".sel_rsci3", CLK_SEL_RSCI3, G3L_SEL_RSCI3, sel_rsci_rspi),
+	DEF_MUX(".sel_rspi0", CLK_SEL_RSPI0, G3L_SEL_RSPI0, sel_rsci_rspi),
+	DEF_MUX(".sel_rspi1", CLK_SEL_RSPI1, G3L_SEL_RSPI1, sel_rsci_rspi),
+	DEF_MUX(".sel_rspi2", CLK_SEL_RSPI2, G3L_SEL_RSPI2, sel_rsci_rspi),
+	DEF_MUX(".sel_eth0_tx", CLK_SEL_ETH0_TX, G3L_SEL_ETH0_TX, sel_eth0_tx),
+	DEF_MUX(".sel_eth0_rx", CLK_SEL_ETH0_RX, G3L_SEL_ETH0_RX, sel_eth0_rx),
+	DEF_MUX(".sel_eth0_rm", CLK_SEL_ETH0_RM, G3L_SEL_ETH0_RM, sel_eth0_rm),
+	DEF_MUX(".sel_eth1_tx", CLK_SEL_ETH1_TX, G3L_SEL_ETH1_TX, sel_eth1_tx),
+	DEF_MUX(".sel_eth1_rx", CLK_SEL_ETH1_RX, G3L_SEL_ETH1_RX, sel_eth1_rx),
+	DEF_MUX(".sel_eth1_rm", CLK_SEL_ETH1_RM, G3L_SEL_ETH1_RM, sel_eth1_rm),
+	DEF_DIV(".div_eth0_tr", CLK_ETH0_TR, CLK_PLL6, G3L_SDIV_ETH_A, dtable_4_200),
+	DEF_DIV(".div_eth1_tr", CLK_ETH1_TR, CLK_PLL6, G3L_SDIV_ETH_C, dtable_4_200),
+	DEF_DIV(".div_eth0_rm", CLK_ETH0_RM, CLK_SEL_ETH0_RM, G3L_SDIV_ETH_B, dtable_2_20),
+	DEF_DIV(".div_eth1_rm", CLK_ETH1_RM, CLK_SEL_ETH1_RM, G3L_SDIV_ETH_D, dtable_2_20),
 
 	/* Core output clk */
+	DEF_G3S_DIV("IC0", R9A08G046_CLK_IC0, CLK_PLL1, G3L_DIV_CA55_CORE0, G3L_DIV_CA55_CORE0_STS,
+		    dtable_1_32, 0, 0, 0, NULL),
+	DEF_G3S_DIV("IC1", R9A08G046_CLK_IC1, CLK_PLL1, G3L_DIV_CA55_CORE1, G3L_DIV_CA55_CORE1_STS,
+		    dtable_1_32, 0, 0, 0, NULL),
+	DEF_G3S_DIV("IC2", R9A08G046_CLK_IC2, CLK_PLL1, G3L_DIV_CA55_CORE2, G3L_DIV_CA55_CORE2_STS,
+		    dtable_1_32, 0, 0, 0, NULL),
+	DEF_G3S_DIV("IC3", R9A08G046_CLK_IC3, CLK_PLL1, G3L_DIV_CA55_CORE3, G3L_DIV_CA55_CORE3_STS,
+		    dtable_1_32, 0, 0, 0, NULL),
 	DEF_G3S_DIV("P0", R9A08G046_CLK_P0, CLK_PLL2_DIV2, G3L_DIVPL2B, G3L_DIVPL2B_STS,
 		    dtable_8_256, 0, 0, 0, NULL),
 	DEF_G3S_DIV("P1", R9A08G046_CLK_P1, CLK_PLL3_DIV2, G3L_DIVPL3A, G3L_DIVPL3A_STS,
 		    dtable_4_128, 0, 0, 0, NULL),
 	DEF_G3S_DIV("P3", R9A08G046_CLK_P3, CLK_PLL2_DIV2, G3L_DIVPL2A, G3L_DIVPL2A_STS,
 		    dtable_4_128, 0, 0, 0, NULL),
+	DEF_G3S_DIV("P13", R9A08G046_CLK_P13, CLK_SEL_RSCI0, G3L_DIV_RSCI0, G3L_DIV_RSCI0_STS,
+		    dtable_2_16, 0, 100000000UL, 0, NULL),
+	DEF_G3S_DIV("P14", R9A08G046_CLK_P14, CLK_SEL_RSCI1, G3L_DIV_RSCI1, G3L_DIV_RSCI1_STS,
+		    dtable_2_16, 0, 100000000UL, 0, NULL),
+	DEF_G3S_DIV("P15", R9A08G046_CLK_P15, CLK_SEL_RSCI2, G3L_DIV_RSCI2, G3L_DIV_RSCI2_STS,
+		    dtable_2_16, 0, 100000000UL, 0, NULL),
+	DEF_G3S_DIV("P16", R9A08G046_CLK_P16, CLK_SEL_RSCI3, G3L_DIV_RSCI3, G3L_DIV_RSCI3_STS,
+		    dtable_2_16, 0, 100000000UL, 0, NULL),
+	DEF_G3S_DIV("P17", R9A08G046_CLK_P17, CLK_SEL_RSPI0, G3L_DIV_RSPI0, G3L_DIV_RSPI0_STS,
+		    dtable_1_8, 0, 200000000UL, 0, NULL),
+	DEF_G3S_DIV("P18", R9A08G046_CLK_P18, CLK_SEL_RSPI1, G3L_DIV_RSPI1, G3L_DIV_RSPI1_STS,
+		    dtable_1_8, 0, 200000000UL, 0, NULL),
+	DEF_G3S_DIV("P19", R9A08G046_CLK_P19, CLK_SEL_RSPI2, G3L_DIV_RSPI2, G3L_DIV_RSPI2_STS,
+		    dtable_1_8, 0, 200000000UL, 0, NULL),
+	DEF_FIXED("HP", R9A08G046_CLK_HP, CLK_PLL6_DIV10, 1, 1),
+	DEF_MUX_FLAGS("ETHTX01", R9A08G046_CLK_ETHTX01, G3L_SEL_ETH0_CLK_TX_I, sel_eth0_clk_tx_i,
+		      CLK_SET_RATE_PARENT),
+	DEF_MUX_FLAGS("ETHRX01", R9A08G046_CLK_ETHRX01, G3L_SEL_ETH0_CLK_RX_I, sel_eth0_clk_rx_i,
+		      CLK_SET_RATE_PARENT),
+	DEF_MUX_FLAGS("ETHTX11", R9A08G046_CLK_ETHTX11, G3L_SEL_ETH1_CLK_TX_I, sel_eth1_clk_tx_i,
+		      CLK_SET_RATE_PARENT),
+	DEF_MUX_FLAGS("ETHRX11", R9A08G046_CLK_ETHRX11, G3L_SEL_ETH1_CLK_RX_I, sel_eth1_clk_rx_i,
+		      CLK_SET_RATE_PARENT),
+	DEF_FIXED("ETHRM0", R9A08G046_CLK_ETHRM0, CLK_SEL_ETH0_RM, 1, 1),
+	DEF_FIXED("ETHTX02", R9A08G046_CLK_ETHTX02, CLK_SEL_ETH0_TX, 1, 1),
+	DEF_FIXED("ETHRX02", R9A08G046_CLK_ETHRX02, CLK_SEL_ETH0_RX, 1, 1),
+	DEF_FIXED("ETHRM1", R9A08G046_CLK_ETHRM1, CLK_SEL_ETH1_RM, 1, 1),
+	DEF_FIXED("ETHTX12", R9A08G046_CLK_ETHTX12, CLK_SEL_ETH1_TX, 1, 1),
+	DEF_FIXED("ETHRX12", R9A08G046_CLK_ETHRX12, CLK_SEL_ETH1_RX, 1, 1),
+	DEF_FIXED("OSCCLK", R9A08G046_OSCCLK, CLK_EXTAL, 1, 1),
 };
 
 static const struct rzg2l_mod_clk r9a08g046_mod_clks[] = {
@@ -101,8 +296,116 @@ static const struct rzg2l_mod_clk r9a08g046_mod_clks[] = {
 					MSTOP(BUS_REG1, BIT(2))),
 	DEF_MOD("dmac_pclk",		R9A08G046_DMAC_PCLK, R9A08G046_CLK_P3, 0x52c, 1,
 					MSTOP(BUS_REG1, BIT(3))),
+	DEF_MOD("wdt0_pclk",		R9A08G046_WDT0_PCLK, R9A08G046_CLK_P0, 0x548, 0,
+					MSTOP(BUS_REG0, BIT(0))),
+	DEF_MOD("wdt0_clk",		R9A08G046_WDT0_CLK, R9A08G046_OSCCLK, 0x548, 1,
+					MSTOP(BUS_REG0, BIT(0))),
+	DEF_MOD("ssi0_pclk2",		R9A08G046_SSI0_PCLK2, R9A08G046_CLK_P0,	0x570, 0,
+					MSTOP(BUS_MCPU1, BIT(10))),
+	DEF_MOD("ssi0_pclk_sfr",	R9A08G046_SSI0_PCLK_SFR, R9A08G046_CLK_P0, 0x570, 1,
+					MSTOP(BUS_MCPU1, BIT(10))),
+	DEF_MOD("ssi1_pclk2",		R9A08G046_SSI1_PCLK2, R9A08G046_CLK_P0, 0x570, 2,
+					MSTOP(BUS_MCPU1, BIT(11))),
+	DEF_MOD("ssi1_pclk_sfr",	R9A08G046_SSI1_PCLK_SFR, R9A08G046_CLK_P0, 0x570, 3,
+					MSTOP(BUS_MCPU1, BIT(11))),
+	DEF_MOD("ssi2_pclk2",		R9A08G046_SSI2_PCLK2, R9A08G046_CLK_P0,	0x570, 4,
+					MSTOP(BUS_MCPU1, BIT(12))),
+	DEF_MOD("ssi2_pclk_sfr",	R9A08G046_SSI2_PCLK_SFR, R9A08G046_CLK_P0, 0x570, 5,
+					MSTOP(BUS_MCPU1, BIT(12))),
+	DEF_MOD("ssi3_pclk2",		R9A08G046_SSI3_PCLK2, R9A08G046_CLK_P0,	0x570, 6,
+					MSTOP(BUS_MCPU1, BIT(13))),
+	DEF_MOD("ssi3_pclk_sfr",	R9A08G046_SSI3_PCLK_SFR, R9A08G046_CLK_P0, 0x570, 7,
+					MSTOP(BUS_MCPU1, BIT(13))),
+	DEF_MOD("eth0_clk_axi",		R9A08G046_ETH0_CLK_AXI, R9A08G046_CLK_P1, 0x57c, 0,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_MOD("eth1_clk_axi",		R9A08G046_ETH1_CLK_AXI, R9A08G046_CLK_P1, 0x57c, 1,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_MOD("eth0_clk_chi",		R9A08G046_ETH0_CLK_CHI, R9A08G046_CLK_P1, 0x57c, 2,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_MOD("eth1_clk_chi",		R9A08G046_ETH1_CLK_CHI, R9A08G046_CLK_P1, 0x57c, 3,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_COUPLED("eth0_tx_i",	R9A08G046_ETH0_CLK_TX_I, R9A08G046_CLK_ETHTX01, 0x57c, 4,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_COUPLED("eth0_tx_180_i",	R9A08G046_ETH0_CLK_TX_180_I, R9A08G046_CLK_ETHTX02, 0x57c, 4,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_COUPLED("eth1_tx_i",	R9A08G046_ETH1_CLK_TX_I, R9A08G046_CLK_ETHTX11, 0x57c, 5,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_COUPLED("eth1_tx_180_i",	R9A08G046_ETH1_CLK_TX_180_I, R9A08G046_CLK_ETHTX12, 0x57c, 5,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_COUPLED("eth0_rx_i",	R9A08G046_ETH0_CLK_RX_I, R9A08G046_CLK_ETHRX01, 0x57c, 6,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_COUPLED("eth0_rx_180_i",	R9A08G046_ETH0_CLK_RX_180_I, R9A08G046_CLK_ETHRX02, 0x57c, 6,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_COUPLED("eth1_rx_i",	R9A08G046_ETH1_CLK_RX_I, R9A08G046_CLK_ETHRX11, 0x57c, 7,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_COUPLED("eth1_rx_180_i",	R9A08G046_ETH1_CLK_RX_180_I, R9A08G046_CLK_ETHRX12, 0x57c, 7,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_MOD("eth0_ptp_ref_i",	R9A08G046_ETH0_CLK_PTP_REF_I, R9A08G046_CLK_HP, 0x57c, 8,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_MOD("eth1_ptp_ref_i",	R9A08G046_ETH1_CLK_PTP_REF_I, R9A08G046_CLK_HP, 0x57c, 9,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_MOD("eth0_rmii_i",		R9A08G046_ETH0_CLK_RMII_I, R9A08G046_CLK_ETHRM0, 0x57c, 10,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_MOD("eth1_rmii_i",		R9A08G046_ETH1_CLK_RMII_I, R9A08G046_CLK_ETHRM1, 0x57c, 11,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_COUPLED("eth0_tx_i_rmii",	R9A08G046_ETH0_CLK_TX_I_RMII, R9A08G046_CLK_ETHTX01, 0x57c, 12,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_COUPLED("eth0_rx_i_rmii",	R9A08G046_ETH0_CLK_RX_I_RMII, R9A08G046_CLK_ETHRX01, 0x57c, 12,
+					MSTOP(BUS_PERI_COM, BIT(2))),
+	DEF_COUPLED("eth1_tx_i_rmii",	R9A08G046_ETH1_CLK_TX_I_RMII, R9A08G046_CLK_ETHTX11, 0x57c, 13,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_COUPLED("eth1_rx_i_rmii",	R9A08G046_ETH1_CLK_RX_I_RMII, R9A08G046_CLK_ETHRX11, 0x57c, 13,
+					MSTOP(BUS_PERI_COM, BIT(3))),
+	DEF_MOD("i2c0_pclk",		R9A08G046_I2C0_PCLK, R9A08G046_CLK_P0, 0x580, 0,
+					MSTOP(BUS_MCPU2, BIT(10))),
+	DEF_MOD("i2c1_pclk",		R9A08G046_I2C1_PCLK, R9A08G046_CLK_P0, 0x580, 1,
+					MSTOP(BUS_MCPU2, BIT(11))),
+	DEF_MOD("i2c2_pclk",		R9A08G046_I2C2_PCLK, R9A08G046_CLK_P0, 0x580, 2,
+					MSTOP(BUS_MCPU2, BIT(12))),
+	DEF_MOD("i2c3_pclk",		R9A08G046_I2C3_PCLK, R9A08G046_CLK_P0, 0x580, 3,
+					MSTOP(BUS_MCPU2, BIT(13))),
 	DEF_MOD("scif0_clk_pck",	R9A08G046_SCIF0_CLK_PCK, R9A08G046_CLK_P0, 0x584, 0,
 					MSTOP(BUS_MCPU2, BIT(1))),
+	DEF_MOD("scif1_clk_pck",	R9A08G046_SCIF1_CLK_PCK, R9A08G046_CLK_P0, 0x584, 1,
+					MSTOP(BUS_MCPU2, BIT(2))),
+	DEF_MOD("scif2_clk_pck",	R9A08G046_SCIF2_CLK_PCK, R9A08G046_CLK_P0, 0x584, 2,
+					MSTOP(BUS_MCPU2, BIT(3))),
+	DEF_MOD("scif3_clk_pck",	R9A08G046_SCIF3_CLK_PCK, R9A08G046_CLK_P0, 0x584, 3,
+					MSTOP(BUS_MCPU2, BIT(4))),
+	DEF_MOD("scif4_clk_pck",	R9A08G046_SCIF4_CLK_PCK, R9A08G046_CLK_P0, 0x584, 4,
+					MSTOP(BUS_MCPU2, BIT(5))),
+	DEF_MOD("scif5_clk_pck",	R9A08G046_SCIF5_CLK_PCK, R9A08G046_CLK_P0, 0x584, 5,
+					MSTOP(BUS_MCPU3, BIT(4))),
+	DEF_MOD("rspi0_pclk",		R9A08G046_RSPI0_PCLK, R9A08G046_CLK_P3, 0x590, 0,
+					MSTOP(BUS_MCPU1, BIT(14))),
+	DEF_MOD("rspi1_pclk",		R9A08G046_RSPI1_PCLK, R9A08G046_CLK_P3, 0x590, 1,
+					MSTOP(BUS_MCPU1, BIT(15))),
+	DEF_MOD("rspi2_pclk",		R9A08G046_RSPI2_PCLK, R9A08G046_CLK_P3, 0x590, 2,
+					MSTOP(BUS_MCPU2, BIT(0))),
+	DEF_MOD("rspi0_tclk",		R9A08G046_RSPI0_TCLK, R9A08G046_CLK_P17, 0x590, 8,
+					MSTOP(BUS_MCPU1, BIT(14))),
+	DEF_MOD("rspi1_tclk",		R9A08G046_RSPI1_TCLK, R9A08G046_CLK_P18, 0x590, 9,
+					MSTOP(BUS_MCPU1, BIT(15))),
+	DEF_MOD("rspi2_tclk",		R9A08G046_RSPI2_TCLK, R9A08G046_CLK_P19, 0x590, 10,
+					MSTOP(BUS_MCPU2, BIT(0))),
+	DEF_MOD("gpio_hclk",		R9A08G046_GPIO_HCLK, R9A08G046_OSCCLK, 0x598, 0,
+					MSTOP(BUS_PERI_CPU, BIT(6))),
+	DEF_MOD("rsci0_pclk",		R9A08G046_RSCI0_PCLK, R9A08G046_CLK_P0, 0x618, 0,
+					MSTOP(BUS_MCPU2, BIT(7))),
+	DEF_MOD("rsci1_pclk",		R9A08G046_RSCI1_PCLK, R9A08G046_CLK_P0, 0x618, 1,
+					MSTOP(BUS_MCPU2, BIT(8))),
+	DEF_MOD("rsci2_pclk",		R9A08G046_RSCI2_PCLK, R9A08G046_CLK_P0, 0x618, 2,
+					MSTOP(BUS_MCPU3, BIT(11))),
+	DEF_MOD("rsci3_pclk",		R9A08G046_RSCI3_PCLK, R9A08G046_CLK_P0, 0x618, 3,
+					MSTOP(BUS_MCPU3, BIT(12))),
+	DEF_MOD("rsci0_tclk",		R9A08G046_RSCI0_TCLK, R9A08G046_CLK_P13, 0x618, 8,
+					MSTOP(BUS_MCPU2, BIT(7))),
+	DEF_MOD("rsci1_tclk",		R9A08G046_RSCI1_TCLK, R9A08G046_CLK_P14, 0x618, 9,
+					MSTOP(BUS_MCPU2, BIT(8))),
+	DEF_MOD("rsci2_tclk",		R9A08G046_RSCI2_TCLK, R9A08G046_CLK_P15, 0x618, 10,
+					MSTOP(BUS_MCPU3, BIT(11))),
+	DEF_MOD("rsci3_tclk",		R9A08G046_RSCI3_TCLK, R9A08G046_CLK_P16, 0x618, 11,
+					MSTOP(BUS_MCPU3, BIT(12))),
 };
 
 static const struct rzg2l_reset r9a08g046_resets[] = {
@@ -111,11 +414,45 @@ static const struct rzg2l_reset r9a08g046_resets[] = {
 	DEF_RST(R9A08G046_IA55_RESETN, 0x818, 0),
 	DEF_RST(R9A08G046_DMAC_ARESETN, 0x82c, 0),
 	DEF_RST(R9A08G046_DMAC_RST_ASYNC, 0x82c, 1),
+	DEF_RST(R9A08G046_WDT0_PRESETN, 0x848, 0),
+	DEF_RST(R9A08G046_SSI0_RST_M2_REG, 0x870, 0),
+	DEF_RST(R9A08G046_SSI1_RST_M2_REG, 0x870, 1),
+	DEF_RST(R9A08G046_SSI2_RST_M2_REG, 0x870, 2),
+	DEF_RST(R9A08G046_SSI3_RST_M2_REG, 0x870, 3),
+	DEF_RST(R9A08G046_ETH0_ARESET_N, 0x87c, 0),
+	DEF_RST(R9A08G046_ETH1_ARESET_N, 0x87c, 1),
+	DEF_RST(R9A08G046_I2C0_MRST, 0x880, 0),
+	DEF_RST(R9A08G046_I2C1_MRST, 0x880, 1),
+	DEF_RST(R9A08G046_I2C2_MRST, 0x880, 2),
+	DEF_RST(R9A08G046_I2C3_MRST, 0x880, 3),
 	DEF_RST(R9A08G046_SCIF0_RST_SYSTEM_N, 0x884, 0),
+	DEF_RST(R9A08G046_SCIF1_RST_SYSTEM_N, 0x884, 1),
+	DEF_RST(R9A08G046_SCIF2_RST_SYSTEM_N, 0x884, 2),
+	DEF_RST(R9A08G046_SCIF3_RST_SYSTEM_N, 0x884, 3),
+	DEF_RST(R9A08G046_SCIF4_RST_SYSTEM_N, 0x884, 4),
+	DEF_RST(R9A08G046_SCIF5_RST_SYSTEM_N, 0x884, 5),
+	DEF_RST(R9A08G046_RSPI0_PRESETN, 0x890, 0),
+	DEF_RST(R9A08G046_RSPI1_PRESETN, 0x890, 1),
+	DEF_RST(R9A08G046_RSPI2_PRESETN, 0x890, 2),
+	DEF_RST(R9A08G046_RSPI0_TRESETN, 0x890, 8),
+	DEF_RST(R9A08G046_RSPI1_TRESETN, 0x890, 9),
+	DEF_RST(R9A08G046_RSPI2_TRESETN, 0x890, 10),
+	DEF_RST(R9A08G046_GPIO_RSTN, 0x898, 0),
+	DEF_RST(R9A08G046_GPIO_PORT_RESETN, 0x898, 1),
+	DEF_RST(R9A08G046_GPIO_SPARE_RESETN, 0x898, 2),
+	DEF_RST(R9A08G046_RSCI0_PRESETN, 0x918, 0),
+	DEF_RST(R9A08G046_RSCI1_PRESETN, 0x918, 1),
+	DEF_RST(R9A08G046_RSCI2_PRESETN, 0x918, 2),
+	DEF_RST(R9A08G046_RSCI3_PRESETN, 0x918, 3),
+	DEF_RST(R9A08G046_RSCI0_TRESETN, 0x918, 8),
+	DEF_RST(R9A08G046_RSCI1_TRESETN, 0x918, 9),
+	DEF_RST(R9A08G046_RSCI2_TRESETN, 0x918, 10),
+	DEF_RST(R9A08G046_RSCI3_TRESETN, 0x918, 11),
 };
 
 static const unsigned int r9a08g046_crit_mod_clks[] __initconst = {
 	MOD_CLK_BASE + R9A08G046_GIC600_GICCLK,
+	MOD_CLK_BASE + R9A08G046_IA55_PCLK,
 	MOD_CLK_BASE + R9A08G046_IA55_CLK,
 	MOD_CLK_BASE + R9A08G046_DMAC_ACLK,
 };
@@ -125,6 +462,23 @@ static const unsigned int r9a08g046_crit_resets[] = {
 	R9A08G046_DMAC_RST_ASYNC,
 };
 
+static const unsigned int r9a08g046_no_pm_mod_clks[] = {
+	MOD_CLK_BASE + R9A08G046_ETH0_CLK_TX_I,
+	MOD_CLK_BASE + R9A08G046_ETH0_CLK_TX_180_I,
+	MOD_CLK_BASE + R9A08G046_ETH0_CLK_RX_I,
+	MOD_CLK_BASE + R9A08G046_ETH0_CLK_RX_180_I,
+	MOD_CLK_BASE + R9A08G046_ETH0_CLK_RMII_I,
+	MOD_CLK_BASE + R9A08G046_ETH0_CLK_TX_I_RMII,
+	MOD_CLK_BASE + R9A08G046_ETH0_CLK_RX_I_RMII,
+	MOD_CLK_BASE + R9A08G046_ETH1_CLK_TX_I,
+	MOD_CLK_BASE + R9A08G046_ETH1_CLK_TX_180_I,
+	MOD_CLK_BASE + R9A08G046_ETH1_CLK_RX_I,
+	MOD_CLK_BASE + R9A08G046_ETH1_CLK_RX_180_I,
+	MOD_CLK_BASE + R9A08G046_ETH1_CLK_RMII_I,
+	MOD_CLK_BASE + R9A08G046_ETH1_CLK_TX_I_RMII,
+	MOD_CLK_BASE + R9A08G046_ETH1_CLK_RX_I_RMII,
+};
+
 const struct rzg2l_cpg_info r9a08g046_cpg_info = {
 	/* Core Clocks */
 	.core_clks = r9a08g046_core_clks,
@@ -141,6 +495,10 @@ const struct rzg2l_cpg_info r9a08g046_cpg_info = {
 	.num_mod_clks = ARRAY_SIZE(r9a08g046_mod_clks),
 	.num_hw_mod_clks = R9A08G046_BSC_X_BCK_BSC + 1,
 
+	/* No PM modules Clocks */
+	.no_pm_mod_clks = r9a08g046_no_pm_mod_clks,
+	.num_no_pm_mod_clks = ARRAY_SIZE(r9a08g046_no_pm_mod_clks),
+
 	/* Resets */
 	.resets = r9a08g046_resets,
 	.num_resets = R9A08G046_BSC_X_PRESET_BSC + 1, /* Last reset ID + 1 */
diff --git a/drivers/clk/renesas/r9a09g047-cpg.c b/drivers/clk/renesas/r9a09g047-cpg.c
index e59ac4a05a7f5..94158b6834e6e 100644
--- a/drivers/clk/renesas/r9a09g047-cpg.c
+++ b/drivers/clk/renesas/r9a09g047-cpg.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/clk/renesas.h>
 #include <linux/device.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -30,6 +31,8 @@ enum clk_ids {
 	CLK_PLLCA55,
 	CLK_PLLVDO,
 	CLK_PLLETH,
+	CLK_PLLDSI0,
+	CLK_PLLDSI1,
 
 	/* Internal Core Clocks */
 	CLK_PLLCM33_DIV3,
@@ -57,6 +60,8 @@ enum clk_ids {
 	CLK_PLLETH_DIV_125_FIX,
 	CLK_CSDIV_PLLETH_GBE0,
 	CLK_CSDIV_PLLETH_GBE1,
+	CLK_SMUX2_DSI0_CLK,
+	CLK_SMUX2_DSI1_CLK,
 	CLK_SMUX2_GBE0_TXCLK,
 	CLK_SMUX2_GBE0_RXCLK,
 	CLK_SMUX2_GBE1_TXCLK,
@@ -64,6 +69,12 @@ enum clk_ids {
 	CLK_PLLDTY_DIV16,
 	CLK_PLLVDO_CRU0,
 	CLK_PLLVDO_GPU,
+	CLK_PLLETH_DIV4_LPCLK,
+	CLK_PLLETH_LPCLK,
+	CLK_PLLDSI0_DIV7,
+	CLK_PLLDSI1_DIV7,
+	CLK_PLLDSI0_CSDIV,
+	CLK_PLLDSI1_CSDIV,
 
 	/* Module Clocks */
 	MOD_CLK_BASE,
@@ -91,6 +102,18 @@ static const struct clk_div_table dtable_2_16[] = {
 	{0, 0},
 };
 
+static const struct clk_div_table dtable_2_16_plldsi[] = {
+	{0, 2},
+	{1, 4},
+	{2, 6},
+	{3, 8},
+	{4, 10},
+	{5, 12},
+	{6, 14},
+	{7, 16},
+	{0, 0},
+};
+
 static const struct clk_div_table dtable_2_64[] = {
 	{0, 2},
 	{1, 4},
@@ -107,7 +130,23 @@ static const struct clk_div_table dtable_2_100[] = {
 	{0, 0},
 };
 
+static const struct clk_div_table dtable_16_128[] = {
+	{0, 16},
+	{1, 32},
+	{2, 64},
+	{3, 128},
+	{0, 0},
+};
+
+RZG3E_CPG_PLL_DSI0_LIMITS(rzg3e_cpg_pll_dsi0_limits);
+RZG3E_CPG_PLL_DSI1_LIMITS(rzg3e_cpg_pll_dsi1_limits);
+
+#define PLLDSI0		PLL_PACK_LIMITS(0xc0, 1, 0, &rzg3e_cpg_pll_dsi0_limits)
+#define PLLDSI1		PLL_PACK_LIMITS(0x160, 1, 1, &rzg3e_cpg_pll_dsi1_limits)
+
 /* Mux clock tables */
+static const char * const smux2_dsi0_clk[] = { ".plldsi0_div7", ".plldsi0_csdiv" };
+static const char * const smux2_dsi1_clk[] = { ".plldsi1_div7", ".plldsi1_csdiv" };
 static const char * const smux2_gbe0_rxclk[] = { ".plleth_gbe0", "et0_rxclk" };
 static const char * const smux2_gbe0_txclk[] = { ".plleth_gbe0", "et0_txclk" };
 static const char * const smux2_gbe1_rxclk[] = { ".plleth_gbe1", "et1_rxclk" };
@@ -128,6 +167,8 @@ static const struct cpg_core_clk r9a09g047_core_clks[] __initconst = {
 	DEF_PLL(".pllca55", CLK_PLLCA55, CLK_QEXTAL, PLLCA55),
 	DEF_FIXED(".plleth", CLK_PLLETH, CLK_QEXTAL, 125, 3),
 	DEF_FIXED(".pllvdo", CLK_PLLVDO, CLK_QEXTAL, 105, 2),
+	DEF_PLLDSI(".plldsi0", CLK_PLLDSI0, CLK_QEXTAL, PLLDSI0),
+	DEF_PLLDSI(".plldsi1", CLK_PLLDSI1, CLK_QEXTAL, PLLDSI1),
 
 	/* Internal Core Clocks */
 	DEF_FIXED(".pllcm33_div3", CLK_PLLCM33_DIV3, CLK_PLLCM33, 1, 3),
@@ -171,6 +212,21 @@ static const struct cpg_core_clk r9a09g047_core_clks[] __initconst = {
 	DEF_DDIV(".pllvdo_cru0", CLK_PLLVDO_CRU0, CLK_PLLVDO, CDDIV3_DIVCTL3, dtable_2_4),
 	DEF_DDIV(".pllvdo_gpu", CLK_PLLVDO_GPU, CLK_PLLVDO, CDDIV3_DIVCTL1, dtable_2_64),
 
+	DEF_FIXED(".plleth_div4_lpclk", CLK_PLLETH_DIV4_LPCLK, CLK_PLLETH, 1, 4),
+	DEF_CSDIV(".plleth_lpclk", CLK_PLLETH_LPCLK, CLK_PLLETH_DIV4_LPCLK,
+		  CSDIV0_DIVCTL2, dtable_16_128),
+
+	DEF_PLLDSI_DIV(".plldsi0_csdiv", CLK_PLLDSI0_CSDIV, CLK_PLLDSI0,
+		       CSDIV1_DIVCTL2, dtable_2_16_plldsi),
+	DEF_PLLDSI_DIV(".plldsi1_csdiv", CLK_PLLDSI1_CSDIV, CLK_PLLDSI1,
+		       CSDIV1_DIVCTL3, dtable_2_16_plldsi),
+	DEF_FIXED(".plldsi0_div7", CLK_PLLDSI0_DIV7, CLK_PLLDSI0, 1, 7),
+	DEF_FIXED(".plldsi1_div7", CLK_PLLDSI1_DIV7, CLK_PLLDSI1, 1, 7),
+	DEF_PLLDSI_SMUX(".smux2_dsi0_clk", CLK_SMUX2_DSI0_CLK,
+			SSEL3_SELCTL0, smux2_dsi0_clk),
+	DEF_PLLDSI_SMUX(".smux2_dsi1_clk", CLK_SMUX2_DSI1_CLK,
+			SSEL3_SELCTL1, smux2_dsi1_clk),
+
 	/* Core Clocks */
 	DEF_FIXED("sys_0_pclk", R9A09G047_SYS_0_PCLK, CLK_QEXTAL, 1, 1),
 	DEF_DDIV("ca55_0_coreclk0", R9A09G047_CA55_0_CORECLK0, CLK_PLLCA55,
@@ -452,6 +508,22 @@ static const struct rzv2h_mod_clk r9a09g047_mod_clks[] __initconst = {
 						BUS_MSTOP(9, BIT(4))),
 	DEF_MOD("cru_0_pclk",			CLK_PLLDTY_DIV16, 13, 4, 6, 20,
 						BUS_MSTOP(9, BIT(4))),
+	DEF_MOD("dsi_0_pclk",			CLK_PLLDTY_DIV16, 14, 8, 7, 8,
+						BUS_MSTOP(9, BIT(15) | BIT(14))),
+	DEF_MOD("dsi_0_aclk",			CLK_PLLDTY_ACPU_DIV2, 14, 9, 7, 9,
+						BUS_MSTOP(9, BIT(15) | BIT(14))),
+	DEF_MOD("dsi_0_vclk1",			CLK_SMUX2_DSI0_CLK, 14, 10, 7, 10,
+						BUS_MSTOP(9, BIT(15) | BIT(14))),
+	DEF_MOD("dsi_0_lpclk",			CLK_PLLETH_LPCLK, 14, 11, 7, 11,
+						BUS_MSTOP(9, BIT(15) | BIT(14))),
+	DEF_MOD("dsi_0_pllref_clk",		CLK_QEXTAL, 14, 12, 7, 12,
+						BUS_MSTOP(9, BIT(15) | BIT(14))),
+	DEF_MOD("lcdc_0_clk_a",			CLK_PLLDTY_ACPU_DIV2, 14, 13, 7, 13,
+						BUS_MSTOP(10, BIT(3) | BIT(2) | BIT(1))),
+	DEF_MOD("lcdc_0_clk_p",			CLK_PLLDTY_DIV16, 14, 14, 7, 14,
+						BUS_MSTOP(10, BIT(3) | BIT(2) | BIT(1))),
+	DEF_MOD("lcdc_0_clk_d",			CLK_SMUX2_DSI0_CLK, 14, 15, 7, 15,
+						BUS_MSTOP(10, BIT(3) | BIT(2) | BIT(1))),
 	DEF_MOD("ge3d_clk",			CLK_PLLVDO_GPU, 15, 0, 7, 16,
 						BUS_MSTOP(3, BIT(4))),
 	DEF_MOD("ge3d_axi_clk",			CLK_PLLDTY_ACPU_DIV2, 15, 1, 7, 17,
@@ -460,6 +532,14 @@ static const struct rzv2h_mod_clk r9a09g047_mod_clks[] __initconst = {
 						BUS_MSTOP(3, BIT(4))),
 	DEF_MOD("tsu_1_pclk",			CLK_QEXTAL, 16, 10, 8, 10,
 						BUS_MSTOP(2, BIT(15))),
+	DEF_MOD("dsi_0_vclk2",			CLK_SMUX2_DSI1_CLK, 25, 0, 10, 21,
+						BUS_MSTOP(9, BIT(15) | BIT(14))),
+	DEF_MOD("lcdc_1_clk_a",			CLK_PLLDTY_ACPU_DIV2, 26, 8, 10, 30,
+						BUS_MSTOP(13, BIT(5) | BIT(4) | BIT(3))),
+	DEF_MOD("lcdc_1_clk_p",			CLK_PLLDTY_DIV16, 26, 9, 10, 31,
+						BUS_MSTOP(13, BIT(5) | BIT(4) | BIT(3))),
+	DEF_MOD("lcdc_1_clk_d",			CLK_SMUX2_DSI1_CLK, 26, 10, 11, 0,
+						BUS_MSTOP(13, BIT(5) | BIT(4) | BIT(3))),
 };
 
 static const struct rzv2h_reset r9a09g047_resets[] __initconst = {
@@ -535,10 +615,14 @@ static const struct rzv2h_reset r9a09g047_resets[] __initconst = {
 	DEF_RST(12, 5, 5, 22),		/* CRU_0_PRESETN */
 	DEF_RST(12, 6, 5, 23),		/* CRU_0_ARESETN */
 	DEF_RST(12, 7, 5, 24),		/* CRU_0_S_RESETN */
+	DEF_RST(13, 7, 6, 8),		/* DSI_0_PRESETN */
+	DEF_RST(13, 8, 6, 9),		/* DSI_0_ARESETN */
+	DEF_RST(13, 12, 6, 13),		/* LCDC_0_RESET_N */
 	DEF_RST(13, 13, 6, 14),		/* GE3D_RESETN */
 	DEF_RST(13, 14, 6, 15),		/* GE3D_AXI_RESETN */
 	DEF_RST(13, 15, 6, 16),		/* GE3D_ACE_RESETN */
 	DEF_RST(15, 8, 7, 9),		/* TSU_1_PRESETN */
+	DEF_RST(17, 14, 8, 15),		/* LCDC_1_RESET_N */
 };
 
 const struct rzv2h_cpg_info r9a09g047_cpg_info __initconst = {
diff --git a/drivers/clk/renesas/r9a09g077-cpg.c b/drivers/clk/renesas/r9a09g077-cpg.c
index 93b15e06a19bc..f777601a23b93 100644
--- a/drivers/clk/renesas/r9a09g077-cpg.c
+++ b/drivers/clk/renesas/r9a09g077-cpg.c
@@ -257,6 +257,7 @@ static const struct mssr_mod_clk r9a09g077_mod_clks[] __initconst = {
 	DEF_MOD("spi0", 104, CLK_SPI0ASYNC),
 	DEF_MOD("spi1", 105, CLK_SPI1ASYNC),
 	DEF_MOD("spi2", 106, CLK_SPI2ASYNC),
+	DEF_MOD("mtu3", 200, R9A09G077_CLK_PCLKH),
 	DEF_MOD("adc0", 206, R9A09G077_CLK_PCLKH),
 	DEF_MOD("adc1", 207, R9A09G077_CLK_PCLKH),
 	DEF_MOD("adc2", 225, R9A09G077_CLK_PCLKM),
diff --git a/drivers/clk/renesas/renesas-cpg-mssr.c b/drivers/clk/renesas/renesas-cpg-mssr.c
index 26ea85cfaa027..5b84cbee030b8 100644
--- a/drivers/clk/renesas/renesas-cpg-mssr.c
+++ b/drivers/clk/renesas/renesas-cpg-mssr.c
@@ -370,6 +370,9 @@ struct clk *cpg_mssr_clk_src_twocell_get(struct of_phandle_args *clkspec,
 	struct clk *clk;
 	int range_check;
 
+	if (clkspec->args_count != 2)
+		return ERR_PTR(-EINVAL);
+
 	switch (clkspec->args[0]) {
 	case CPG_CORE:
 		type = "core";
diff --git a/drivers/clk/renesas/rzg2l-cpg.c b/drivers/clk/renesas/rzg2l-cpg.c
index abfd8634d2bef..426e93dc7a989 100644
--- a/drivers/clk/renesas/rzg2l-cpg.c
+++ b/drivers/clk/renesas/rzg2l-cpg.c
@@ -58,6 +58,13 @@
 #define RZG3S_DIV_NF		GENMASK(12, 1)
 #define RZG3S_SEL_PLL		BIT(0)
 
+#define RZG3L_PLL_STBY_OFFSET(x)	(GET_REG_SAMPLL_CLK1(x) - 0x4)
+#define RZG3L_PLL_STBY_RESETB		BIT(0)
+#define RZG3L_PLL_STBY_RESETB_WEN	BIT(16)
+#define RZG3L_PLL_MON_OFFSET(x)		(GET_REG_SAMPLL_CLK1(x) + 0x8)
+#define RZG3L_PLL_MON_RESETB		BIT(0)
+#define RZG3L_PLL_MON_LOCK		BIT(4)
+
 #define CLK_ON_R(reg)		(reg)
 #define CLK_MON_R(reg)		(0x180 + (reg))
 #define CLK_RST_R(reg)		(reg)
@@ -1107,9 +1114,6 @@ static unsigned long rzg3s_cpg_pll_clk_recalc_rate(struct clk_hw *hw,
 	u32 nir, nfr, mr, pr, val, setting;
 	u64 rate;
 
-	if (pll_clk->type != CLK_TYPE_G3S_PLL)
-		return parent_rate;
-
 	setting = GET_REG_SAMPLL_SETTING(pll_clk->conf);
 	if (setting) {
 		val = readl(priv->base + setting);
@@ -1178,6 +1182,61 @@ rzg2l_cpg_pll_clk_register(const struct cpg_core_clk *core,
 	return pll_clk->hw.clk;
 }
 
+static int rzg3l_cpg_pll_clk_is_enabled(struct clk_hw *hw)
+{
+	struct pll_clk *pll_clk = to_pll(hw);
+	struct rzg2l_cpg_priv *priv = pll_clk->priv;
+	u32 val = readl(priv->base + RZG3L_PLL_MON_OFFSET(pll_clk->conf));
+	u32 mon_val = RZG3L_PLL_MON_RESETB | RZG3L_PLL_MON_LOCK;
+
+	/* Ensure both RESETB and LOCK bits are set */
+	return (mon_val == (val & mon_val));
+}
+
+static int rzg3l_cpg_pll_clk_endisable(struct clk_hw *hw, bool enable)
+{
+	struct pll_clk *pll_clk = to_pll(hw);
+	struct rzg2l_cpg_priv *priv = pll_clk->priv;
+	u32 mon_mask = RZG3L_PLL_MON_RESETB | RZG3L_PLL_MON_LOCK;
+	u32 val = RZG3L_PLL_STBY_RESETB_WEN;
+	u32 stby_offset, mon_offset;
+	u32 mon_val = 0;
+	int ret;
+
+	stby_offset = RZG3L_PLL_STBY_OFFSET(pll_clk->conf);
+	mon_offset = RZG3L_PLL_MON_OFFSET(pll_clk->conf);
+
+	if (enable) {
+		val |= RZG3L_PLL_STBY_RESETB;
+		mon_val = mon_mask;
+	}
+
+	writel(val, priv->base + stby_offset);
+
+	/* ensure PLL is in normal/standby mode */
+	ret = readl_poll_timeout_atomic(priv->base + mon_offset, val,
+					mon_val == (val & mon_mask), 10, 100);
+	if (ret)
+		dev_err(priv->dev, "Failed to %s PLL 0x%x/%pC\n", enable ?
+			"enable" : "disable", stby_offset, hw->clk);
+
+	return ret;
+}
+
+static int rzg3l_cpg_pll_clk_enable(struct clk_hw *hw)
+{
+	if (rzg3l_cpg_pll_clk_is_enabled(hw))
+		return 0;
+
+	return rzg3l_cpg_pll_clk_endisable(hw, true);
+}
+
+static const struct clk_ops rzg3l_cpg_pll_ops = {
+	.is_enabled = rzg3l_cpg_pll_clk_is_enabled,
+	.enable = rzg3l_cpg_pll_clk_enable,
+	.recalc_rate = rzg3s_cpg_pll_clk_recalc_rate,
+};
+
 static struct clk
 *rzg2l_cpg_clk_src_twocell_get(struct of_phandle_args *clkspec,
 			       void *data)
@@ -1261,6 +1320,9 @@ rzg2l_cpg_register_core_clk(const struct cpg_core_clk *core,
 	case CLK_TYPE_SAM_PLL:
 		clk = rzg2l_cpg_pll_clk_register(core, priv, &rzg2l_cpg_pll_ops);
 		break;
+	case CLK_TYPE_G3L_PLL:
+		clk = rzg2l_cpg_pll_clk_register(core, priv, &rzg3l_cpg_pll_ops);
+		break;
 	case CLK_TYPE_G3S_PLL:
 		clk = rzg2l_cpg_pll_clk_register(core, priv, &rzg3s_cpg_pll_ops);
 		break;
diff --git a/drivers/clk/renesas/rzg2l-cpg.h b/drivers/clk/renesas/rzg2l-cpg.h
index 10baf9e71a6e0..33f54ba0e64ee 100644
--- a/drivers/clk/renesas/rzg2l-cpg.h
+++ b/drivers/clk/renesas/rzg2l-cpg.h
@@ -123,6 +123,7 @@ enum clk_types {
 	CLK_TYPE_IN,		/* External Clock Input */
 	CLK_TYPE_FF,		/* Fixed Factor Clock */
 	CLK_TYPE_SAM_PLL,
+	CLK_TYPE_G3L_PLL,
 	CLK_TYPE_G3S_PLL,
 
 	/* Clock with divider */
@@ -152,6 +153,9 @@ enum clk_types {
 	DEF_TYPE(_name, _id, _type, .parent = _parent)
 #define DEF_SAMPLL(_name, _id, _parent, _conf) \
 	DEF_TYPE(_name, _id, CLK_TYPE_SAM_PLL, .parent = _parent, .conf = _conf)
+#define DEF_G3L_PLL(_name, _id, _parent, _conf, _default_rate) \
+	DEF_TYPE(_name, _id, CLK_TYPE_G3L_PLL, .parent = _parent, .conf = _conf, \
+		 .default_rate = _default_rate)
 #define DEF_G3S_PLL(_name, _id, _parent, _conf, _default_rate) \
 	DEF_TYPE(_name, _id, CLK_TYPE_G3S_PLL, .parent = _parent, .conf = _conf, \
 		 .default_rate = _default_rate)
@@ -174,11 +178,14 @@ enum clk_types {
 		 .invalid_rate = _invalid_rate, \
 		 .max_rate = _max_rate, .flag = (_clk_flags), \
 		 .notifier = _notif)
-#define DEF_MUX(_name, _id, _conf, _parent_names) \
+#define DEF_MUX_FLAGS(_name, _id, _conf, _parent_names, _flag) \
 	DEF_TYPE(_name, _id, CLK_TYPE_MUX, .conf = _conf, \
 		 .parent_names = _parent_names, \
 		 .num_parents = ARRAY_SIZE(_parent_names), \
-		 .mux_flags = CLK_MUX_HIWORD_MASK)
+		 .mux_flags = CLK_MUX_HIWORD_MASK, \
+		 .flag = _flag)
+#define DEF_MUX(_name, _id, _conf, _parent_names) \
+	DEF_MUX_FLAGS(_name, _id, _conf, _parent_names, 0)
 #define DEF_MUX_RO(_name, _id, _conf, _parent_names) \
 	DEF_TYPE(_name, _id, CLK_TYPE_MUX, .conf = _conf, \
 		 .parent_names = _parent_names, \
diff --git a/drivers/clk/renesas/rzv2h-cpg.c b/drivers/clk/renesas/rzv2h-cpg.c
index f6c47fb89bca5..e271c04cee34f 100644
--- a/drivers/clk/renesas/rzv2h-cpg.c
+++ b/drivers/clk/renesas/rzv2h-cpg.c
@@ -76,6 +76,11 @@
 /* On RZ/G3E SoC we have two DSI PLLs */
 #define MAX_CPG_DSI_PLL		2
 
+#define CPG_PLLDSI_SMUX_LVDS_DUTY_NUM		4
+#define CPG_PLLDSI_SMUX_LVDS_DUTY_DEN		7
+#define CPG_PLLDSI_SMUX_DSI_RGB_DUTY_NUM	1
+#define CPG_PLLDSI_SMUX_DSI_RGB_DUTY_DEN	2
+
 /**
  * struct rzv2h_pll_dsi_info - PLL DSI information, holds the limits and parameters
  *
@@ -418,6 +423,20 @@ bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits,
 }
 EXPORT_SYMBOL_NS_GPL(rzv2h_get_pll_divs_pars, "RZV2H_CPG");
 
+/**
+ * struct rzv2h_plldsi_mux_clk - PLL DSI MUX clock
+ *
+ * @priv: CPG private data
+ * @mux: mux clk
+ */
+struct rzv2h_plldsi_mux_clk {
+	struct rzv2h_cpg_priv *priv;
+	struct clk_mux mux;
+};
+
+#define to_plldsi_clk_mux(_mux) \
+	container_of(_mux, struct rzv2h_plldsi_mux_clk, mux)
+
 static unsigned long rzv2h_cpg_plldsi_div_recalc_rate(struct clk_hw *hw,
 						      unsigned long parent_rate)
 {
@@ -649,6 +668,165 @@ static int rzv2h_cpg_plldsi_set_rate(struct clk_hw *hw, unsigned long rate,
 	return rzv2h_cpg_pll_set_rate(pll_clk, &dsi_info->pll_dsi_parameters.pll, true);
 }
 
+static u8 rzv2h_cpg_plldsi_smux_get_parent(struct clk_hw *hw)
+{
+	return clk_mux_ops.get_parent(hw);
+}
+
+static int rzv2h_cpg_plldsi_smux_set_parent(struct clk_hw *hw, u8 index)
+{
+	return clk_mux_ops.set_parent(hw, index);
+}
+
+static int rzv2h_cpg_plldsi_smux_lvds_determine_rate(struct rzv2h_cpg_priv *priv,
+						     struct pll_clk *pll_clk,
+						     struct clk_rate_request *req)
+{
+	struct rzv2h_pll_div_pars *dsi_params;
+	struct rzv2h_pll_dsi_info *dsi_info;
+	u8 lvds_table[] = { 7 };
+	u64 rate_millihz;
+
+	dsi_info = &priv->pll_dsi_info[pll_clk->pll.instance];
+	dsi_params = &dsi_info->pll_dsi_parameters;
+
+	rate_millihz = mul_u32_u32(req->rate, MILLI);
+	if (!rzv2h_get_pll_divs_pars(dsi_info->pll_dsi_limits, dsi_params,
+				     lvds_table, ARRAY_SIZE(lvds_table), rate_millihz)) {
+		dev_err(priv->dev, "failed to determine rate for req->rate: %lu\n",
+			req->rate);
+		return -EINVAL;
+	}
+
+	req->rate = DIV_ROUND_CLOSEST_ULL(dsi_params->div.freq_millihz, MILLI);
+	req->best_parent_rate = req->rate;
+	dsi_info->req_pll_dsi_rate = req->best_parent_rate * dsi_params->div.divider_value;
+
+	return 0;
+}
+
+static int rzv2h_cpg_plldsi_smux_determine_rate(struct clk_hw *hw,
+						struct clk_rate_request *req)
+{
+	struct clk_mux *mux = to_clk_mux(hw);
+	struct rzv2h_plldsi_mux_clk *dsi_mux = to_plldsi_clk_mux(mux);
+	struct pll_clk *pll_clk = to_pll(clk_hw_get_parent(hw));
+	struct rzv2h_cpg_priv *priv = dsi_mux->priv;
+
+	/*
+	 * For LVDS output (parent index 0), calculate PLL parameters with
+	 * fixed divider value of 7. For DSI/RGB output (parent index 1) skip
+	 * PLL calculation here as it's handled by determine_rate of the
+	 * divider (up one level).
+	 */
+	if (!clk_mux_ops.get_parent(hw))
+		return rzv2h_cpg_plldsi_smux_lvds_determine_rate(priv, pll_clk, req);
+
+	req->best_parent_rate = req->rate;
+	return 0;
+}
+
+static int rzv2h_cpg_plldsi_smux_get_duty_cycle(struct clk_hw *hw,
+						struct clk_duty *duty)
+{
+	u8 parent = clk_mux_ops.get_parent(hw);
+
+	/*
+	 * CDIV7_DSIx_CLK - LVDS path (div7) - duty 4/7.
+	 * CSDIV_DSIx - DSI/RGB path (csdiv) - duty 1/2.
+	 */
+	if (parent == 0) {
+		duty->num = CPG_PLLDSI_SMUX_LVDS_DUTY_NUM;
+		duty->den = CPG_PLLDSI_SMUX_LVDS_DUTY_DEN;
+	} else {
+		duty->num = CPG_PLLDSI_SMUX_DSI_RGB_DUTY_NUM;
+		duty->den = CPG_PLLDSI_SMUX_DSI_RGB_DUTY_DEN;
+	}
+
+	return 0;
+}
+
+static int rzv2h_cpg_plldsi_smux_set_duty_cycle(struct clk_hw *hw,
+						struct clk_duty *duty)
+{
+	struct clk_hw *parent_hw;
+	u8 parent_idx;
+
+	/*
+	 * Select parent based on requested duty cycle:
+	 * - If duty > 50% (num/den > 1/2), select LVDS path (parent 0)
+	 * - Otherwise, select DSI/RGB path (parent 1)
+	 */
+	if (duty->num * CPG_PLLDSI_SMUX_DSI_RGB_DUTY_DEN >
+	    duty->den * CPG_PLLDSI_SMUX_DSI_RGB_DUTY_NUM)
+		parent_idx = 0;
+	else
+		parent_idx = 1;
+
+	if (parent_idx >= clk_hw_get_num_parents(hw))
+		return -EINVAL;
+
+	parent_hw = clk_hw_get_parent_by_index(hw, parent_idx);
+	if (!parent_hw)
+		return -EINVAL;
+
+	return clk_hw_set_parent(hw, parent_hw);
+}
+
+static const struct clk_ops rzv2h_cpg_plldsi_smux_ops = {
+	.determine_rate = rzv2h_cpg_plldsi_smux_determine_rate,
+	.get_parent = rzv2h_cpg_plldsi_smux_get_parent,
+	.set_parent = rzv2h_cpg_plldsi_smux_set_parent,
+	.get_duty_cycle = rzv2h_cpg_plldsi_smux_get_duty_cycle,
+	.set_duty_cycle = rzv2h_cpg_plldsi_smux_set_duty_cycle,
+};
+
+static struct clk * __init
+rzv2h_cpg_plldsi_smux_clk_register(const struct cpg_core_clk *core,
+				   struct rzv2h_cpg_priv *priv)
+{
+	struct rzv2h_plldsi_mux_clk *clk_hw_data;
+	struct clk_init_data init;
+	struct clk_hw *clk_hw;
+	struct smuxed smux;
+	int ret;
+
+	smux = core->cfg.smux;
+
+	if (smux.shift + smux.width > 16) {
+		dev_err(priv->dev, "mux value exceeds LOWORD field\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	clk_hw_data = devm_kzalloc(priv->dev, sizeof(*clk_hw_data), GFP_KERNEL);
+	if (!clk_hw_data)
+		return ERR_PTR(-ENOMEM);
+
+	clk_hw_data->priv = priv;
+
+	init.name = core->name;
+	init.ops = &rzv2h_cpg_plldsi_smux_ops;
+	init.flags = core->flag;
+	init.parent_names = core->parent_names;
+	init.num_parents = core->num_parents;
+
+	clk_hw_data->mux.reg = priv->base + smux.offset;
+
+	clk_hw_data->mux.shift = smux.shift;
+	clk_hw_data->mux.mask = clk_div_mask(smux.width);
+	clk_hw_data->mux.flags = core->mux_flags;
+	clk_hw_data->mux.lock = &priv->rmw_lock;
+
+	clk_hw = &clk_hw_data->mux.hw;
+	clk_hw->init = &init;
+
+	ret = devm_clk_hw_register(priv->dev, clk_hw);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return clk_hw->clk;
+}
+
 static int rzv2h_cpg_pll_clk_is_enabled(struct clk_hw *hw)
 {
 	struct pll_clk *pll_clk = to_pll(hw);
@@ -1085,6 +1263,9 @@ rzv2h_cpg_register_core_clk(const struct cpg_core_clk *core,
 	case CLK_TYPE_PLLDSI_DIV:
 		clk = rzv2h_cpg_plldsi_div_clk_register(core, priv);
 		break;
+	case CLK_TYPE_PLLDSI_SMUX:
+		clk = rzv2h_cpg_plldsi_smux_clk_register(core, priv);
+		break;
 	default:
 		goto fail;
 	}
diff --git a/drivers/clk/renesas/rzv2h-cpg.h b/drivers/clk/renesas/rzv2h-cpg.h
index dc957bdaf5e97..dec0f7b621d66 100644
--- a/drivers/clk/renesas/rzv2h-cpg.h
+++ b/drivers/clk/renesas/rzv2h-cpg.h
@@ -121,6 +121,7 @@ struct fixed_mod_conf {
 
 #define CPG_SSEL0		(0x300)
 #define CPG_SSEL1		(0x304)
+#define CPG_SSEL3		(0x30C)
 #define CPG_CDDIV0		(0x400)
 #define CPG_CDDIV1		(0x404)
 #define CPG_CDDIV2		(0x408)
@@ -148,6 +149,7 @@ struct fixed_mod_conf {
 #define CSDIV0_DIVCTL2	DDIV_PACK(CPG_CSDIV0, 8, 2, CSDIV_NO_MON)
 #define CSDIV0_DIVCTL3	DDIV_PACK_NO_RMW(CPG_CSDIV0, 12, 2, CSDIV_NO_MON)
 #define CSDIV1_DIVCTL2	DDIV_PACK(CPG_CSDIV1, 8, 4, CSDIV_NO_MON)
+#define CSDIV1_DIVCTL3	DDIV_PACK(CPG_CSDIV1, 12, 4, CSDIV_NO_MON)
 
 #define SSEL0_SELCTL2	SMUX_PACK(CPG_SSEL0, 8, 1)
 #define SSEL0_SELCTL3	SMUX_PACK(CPG_SSEL0, 12, 1)
@@ -155,6 +157,8 @@ struct fixed_mod_conf {
 #define SSEL1_SELCTL1	SMUX_PACK(CPG_SSEL1, 4, 1)
 #define SSEL1_SELCTL2	SMUX_PACK(CPG_SSEL1, 8, 1)
 #define SSEL1_SELCTL3	SMUX_PACK(CPG_SSEL1, 12, 1)
+#define SSEL3_SELCTL0	SMUX_PACK(CPG_SSEL3, 0, 1)
+#define SSEL3_SELCTL1	SMUX_PACK(CPG_SSEL3, 4, 1)
 
 #define BUS_MSTOP_IDX_MASK	GENMASK(31, 16)
 #define BUS_MSTOP_BITS_MASK	GENMASK(15, 0)
@@ -203,6 +207,7 @@ enum clk_types {
 	CLK_TYPE_SMUX,		/* Static Mux */
 	CLK_TYPE_PLLDSI,	/* PLLDSI */
 	CLK_TYPE_PLLDSI_DIV,	/* PLLDSI divider */
+	CLK_TYPE_PLLDSI_SMUX,	/* PLLDSI Static Mux */
 };
 
 #define DEF_TYPE(_name, _id, _type...) \
@@ -241,6 +246,13 @@ enum clk_types {
 		 .dtable = _dtable, \
 		 .parent = _parent, \
 		 .flag = CLK_SET_RATE_PARENT)
+#define DEF_PLLDSI_SMUX(_name, _id, _smux_packed, _parent_names) \
+	DEF_TYPE(_name, _id, CLK_TYPE_PLLDSI_SMUX, \
+		 .cfg.smux = _smux_packed, \
+		 .parent_names = _parent_names, \
+		 .num_parents = ARRAY_SIZE(_parent_names), \
+		 .flag = CLK_SET_RATE_PARENT, \
+		 .mux_flags = CLK_MUX_HIWORD_MASK)
 
 /**
  * struct rzv2h_mod_clk - Module Clocks definitions
diff --git a/drivers/clk/rockchip/Kconfig b/drivers/clk/rockchip/Kconfig
index 7e14335020618..85133498f0130 100644
--- a/drivers/clk/rockchip/Kconfig
+++ b/drivers/clk/rockchip/Kconfig
@@ -3,7 +3,7 @@
 
 config COMMON_CLK_ROCKCHIP
 	bool "Rockchip clock controller common support"
-	depends on ARCH_ROCKCHIP
+	depends on ARCH_ROCKCHIP || COMPILE_TEST
 	default ARCH_ROCKCHIP
 	help
 	  Say y here to enable common clock controller for Rockchip platforms.
diff --git a/drivers/clk/rockchip/clk-rk3588.c b/drivers/clk/rockchip/clk-rk3588.c
index 1694223f4f842..2ba9976654cf1 100644
--- a/drivers/clk/rockchip/clk-rk3588.c
+++ b/drivers/clk/rockchip/clk-rk3588.c
@@ -5,11 +5,13 @@
  */
 
 #include <linux/clk-provider.h>
+#include <linux/mfd/syscon.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/platform_device.h>
 #include <linux/syscore_ops.h>
 #include <dt-bindings/clock/rockchip,rk3588-cru.h>
+#include <soc/rockchip/rk3588_grf.h>
 #include "clk.h"
 
 #define RK3588_GRF_SOC_STATUS0		0x600
@@ -892,6 +894,8 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = {
 			RK3588_CLKGATE_CON(8), 0, GFLAGS),
 	MUX(I2S2_2CH_MCLKOUT, "i2s2_2ch_mclkout", i2s2_2ch_mclkout_p, CLK_SET_RATE_PARENT,
 			RK3588_CLKSEL_CON(30), 2, 1, MFLAGS),
+	GATE_GRF(I2S2_2CH_MCLKOUT_TO_IO, "i2s2_2ch_mclkout_to_io", "i2s2_2ch_mclkout",
+			0, RK3588_SYSGRF_SOC_CON6, 2, GFLAGS, grf_type_sys),
 
 	COMPOSITE(CLK_I2S3_2CH_SRC, "clk_i2s3_2ch_src", gpll_aupll_p, 0,
 			RK3588_CLKSEL_CON(30), 8, 1, MFLAGS, 3, 5, DFLAGS,
@@ -907,6 +911,8 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = {
 			RK3588_CLKGATE_CON(8), 4, GFLAGS),
 	MUX(I2S3_2CH_MCLKOUT, "i2s3_2ch_mclkout", i2s3_2ch_mclkout_p, CLK_SET_RATE_PARENT,
 			RK3588_CLKSEL_CON(32), 2, 1, MFLAGS),
+	GATE_GRF(I2S3_2CH_MCLKOUT_TO_IO, "i2s3_2ch_mclkout_to_io", "i2s3_2ch_mclkout",
+			0, RK3588_SYSGRF_SOC_CON6, 7, GFLAGS, grf_type_sys),
 	GATE(PCLK_ACDCDIG, "pclk_acdcdig", "pclk_audio_root", 0,
 			RK3588_CLKGATE_CON(7), 11, GFLAGS),
 	GATE(HCLK_I2S0_8CH, "hclk_i2s0_8ch", "hclk_audio_root", 0,
@@ -935,6 +941,8 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = {
 			RK3588_CLKGATE_CON(7), 10, GFLAGS),
 	MUX(I2S0_8CH_MCLKOUT, "i2s0_8ch_mclkout", i2s0_8ch_mclkout_p, CLK_SET_RATE_PARENT,
 			RK3588_CLKSEL_CON(28), 2, 2, MFLAGS),
+	GATE_GRF(I2S0_8CH_MCLKOUT_TO_IO, "i2s0_8ch_mclkout_to_io", "i2s0_8ch_mclkout",
+			0, RK3588_SYSGRF_SOC_CON6, 0, GFLAGS, grf_type_sys),
 
 	GATE(HCLK_PDM1, "hclk_pdm1", "hclk_audio_root", 0,
 			RK3588_CLKGATE_CON(9), 6, GFLAGS),
@@ -2220,6 +2228,8 @@ static struct rockchip_clk_branch rk3588_early_clk_branches[] __initdata = {
 			RK3588_PMU_CLKGATE_CON(2), 13, GFLAGS),
 	MUX(I2S1_8CH_MCLKOUT, "i2s1_8ch_mclkout", i2s1_8ch_mclkout_p, CLK_SET_RATE_PARENT,
 			RK3588_PMU_CLKSEL_CON(9), 2, 2, MFLAGS),
+	GATE_GRF(I2S1_8CH_MCLKOUT_TO_IO, "i2s1_8ch_mclkout_to_io", "i2s1_8ch_mclkout",
+			0, RK3588_SYSGRF_SOC_CON6, 1, GFLAGS, grf_type_sys),
 	GATE(PCLK_PMU1, "pclk_pmu1", "pclk_pmu0_root", CLK_IS_CRITICAL,
 			RK3588_PMU_CLKGATE_CON(1), 0, GFLAGS),
 	GATE(CLK_DDR_FAIL_SAFE, "clk_ddr_fail_safe", "clk_pmu0", CLK_IGNORE_UNUSED,
@@ -2439,6 +2449,7 @@ static struct rockchip_clk_branch rk3588_clk_branches[] = {
 static void __init rk3588_clk_early_init(struct device_node *np)
 {
 	struct rockchip_clk_provider *ctx;
+	struct regmap *sys_grf;
 	unsigned long clk_nr_clks, max_clk_id1, max_clk_id2;
 	void __iomem *reg_base;
 
@@ -2479,6 +2490,11 @@ static void __init rk3588_clk_early_init(struct device_node *np)
 			&rk3588_cpub1clk_data, rk3588_cpub1clk_rates,
 			ARRAY_SIZE(rk3588_cpub1clk_rates));
 
+	/* Register SYS_GRF for I2S MCLK output to IO gate clocks */
+	sys_grf = syscon_regmap_lookup_by_compatible("rockchip,rk3588-sys-grf");
+	if (!IS_ERR(sys_grf))
+		rockchip_clk_add_grf(ctx, sys_grf, grf_type_sys);
+
 	rockchip_clk_register_branches(ctx, rk3588_early_clk_branches,
 				       ARRAY_SIZE(rk3588_early_clk_branches));
 
diff --git a/drivers/clk/rockchip/clk.c b/drivers/clk/rockchip/clk.c
index e8b3b0b9a4f8f..ee8c79b938d3c 100644
--- a/drivers/clk/rockchip/clk.c
+++ b/drivers/clk/rockchip/clk.c
@@ -429,6 +429,24 @@ void rockchip_clk_of_add_provider(struct device_node *np,
 }
 EXPORT_SYMBOL_GPL(rockchip_clk_of_add_provider);
 
+int rockchip_clk_add_grf(struct rockchip_clk_provider *ctx,
+			 struct regmap *grf,
+			 enum rockchip_grf_type type)
+{
+	struct rockchip_aux_grf *aux_grf;
+
+	aux_grf = kzalloc_obj(*aux_grf);
+	if (!aux_grf)
+		return -ENOMEM;
+
+	aux_grf->grf = grf;
+	aux_grf->type = type;
+	hash_add(ctx->aux_grf_table, &aux_grf->node, type);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rockchip_clk_add_grf);
+
 void rockchip_clk_register_plls(struct rockchip_clk_provider *ctx,
 				struct rockchip_pll_clock *list,
 				unsigned int nr_pll, int grf_lock_offset)
@@ -509,10 +527,9 @@ void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx,
 		clk = NULL;
 
 		/* for GRF-dependent branches, choose the right grf first */
-		if ((list->branch_type == branch_grf_mux ||
-		     list->branch_type == branch_grf_gate ||
-		     list->branch_type == branch_grf_mmc) &&
-		    list->grf_type != grf_type_sys) {
+		if (list->branch_type == branch_grf_mux ||
+		    list->branch_type == branch_grf_gate ||
+		    list->branch_type == branch_grf_mmc) {
 			hash_for_each_possible(ctx->aux_grf_table, agrf, node, list->grf_type) {
 				if (agrf->type == list->grf_type) {
 					grf = agrf->grf;
diff --git a/drivers/clk/rockchip/clk.h b/drivers/clk/rockchip/clk.h
index cf0f5f11c34b5..9e3503e2ffc23 100644
--- a/drivers/clk/rockchip/clk.h
+++ b/drivers/clk/rockchip/clk.h
@@ -1329,6 +1329,9 @@ struct rockchip_clk_provider *rockchip_clk_init_early(struct device_node *np,
 void rockchip_clk_finalize(struct rockchip_clk_provider *ctx);
 void rockchip_clk_of_add_provider(struct device_node *np,
 				struct rockchip_clk_provider *ctx);
+int rockchip_clk_add_grf(struct rockchip_clk_provider *ctx,
+			 struct regmap *grf,
+			 enum rockchip_grf_type type);
 unsigned long rockchip_clk_find_max_clk_id(struct rockchip_clk_branch *list,
 					   unsigned int nr_clk);
 void rockchip_clk_register_branches(struct rockchip_clk_provider *ctx,
diff --git a/drivers/clk/samsung/clk-exynos850.c b/drivers/clk/samsung/clk-exynos850.c
index eb9c80b60225e..b143a42293f5a 100644
--- a/drivers/clk/samsung/clk-exynos850.c
+++ b/drivers/clk/samsung/clk-exynos850.c
@@ -686,10 +686,11 @@ static const struct samsung_gate_clock apm_gate_clks[] __initconst = {
 	     CLK_CON_GAT_GOUT_APM_APBIF_RTC_PCLK, 21, 0, 0),
 	GATE(CLK_GOUT_TOP_RTC_PCLK, "gout_top_rtc_pclk", "dout_apm_bus",
 	     CLK_CON_GAT_GOUT_APM_APBIF_TOP_RTC_PCLK, 21, 0, 0),
+	/* TODO: Should be dealt with or enabled in PMIC ACPM driver */
 	GATE(CLK_GOUT_I3C_PCLK, "gout_i3c_pclk", "dout_apm_bus",
-	     CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_PCLK, 21, 0, 0),
+	     CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_PCLK, 21, CLK_IS_CRITICAL, 0),
 	GATE(CLK_GOUT_I3C_SCLK, "gout_i3c_sclk", "mout_apm_i3c",
-	     CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_SCLK, 21, 0, 0),
+	     CLK_CON_GAT_GOUT_APM_I3C_APM_PMIC_I_SCLK, 21, CLK_IS_CRITICAL, 0),
 	GATE(CLK_GOUT_SPEEDY_PCLK, "gout_speedy_pclk", "dout_apm_bus",
 	     CLK_CON_GAT_GOUT_APM_SPEEDY_APM_PCLK, 21, 0, 0),
 	/* TODO: Should be enabled in GPIO driver (or made CLK_IS_CRITICAL) */
diff --git a/drivers/clk/samsung/clk-gs101.c b/drivers/clk/samsung/clk-gs101.c
index d2bcd3a9daf89..b44bb31f38b30 100644
--- a/drivers/clk/samsung/clk-gs101.c
+++ b/drivers/clk/samsung/clk-gs101.c
@@ -3921,7 +3921,7 @@ static const unsigned long peric0_clk_regs[] __initconst = {
 	CLK_CON_DIV_DIV_CLK_PERIC0_USI4_USI,
 	CLK_CON_DIV_DIV_CLK_PERIC0_USI5_USI,
 	CLK_CON_DIV_DIV_CLK_PERIC0_USI6_USI,
-	CLK_CON_DIV_DIV_CLK_PERIC0_USI6_USI,
+	CLK_CON_DIV_DIV_CLK_PERIC0_USI7_USI,
 	CLK_CON_DIV_DIV_CLK_PERIC0_USI8_USI,
 	CLK_CON_BUF_CLKBUF_PERIC0_IP,
 	CLK_CON_GAT_CLK_BLK_PERIC0_UID_PERIC0_CMU_PERIC0_IPCLKPORT_PCLK,
diff --git a/drivers/clk/visconti/pll.c b/drivers/clk/visconti/pll.c
index 805b954812817..3ce1a906fb0c3 100644
--- a/drivers/clk/visconti/pll.c
+++ b/drivers/clk/visconti/pll.c
@@ -21,9 +21,9 @@ struct visconti_pll {
 	void __iomem	*pll_base;
 	spinlock_t	*lock;
 	unsigned long flags;
-	const struct visconti_pll_rate_table *rate_table;
 	size_t rate_count;
 	struct visconti_pll_provider *ctx;
+	struct visconti_pll_rate_table rate_table[] __counted_by(rate_count);
 };
 
 #define PLL_CONF_REG		0x0000
@@ -255,10 +255,6 @@ static struct clk_hw *visconti_register_pll(struct visconti_pll_provider *ctx,
 	size_t len;
 	int ret;
 
-	pll = kzalloc_obj(*pll);
-	if (!pll)
-		return ERR_PTR(-ENOMEM);
-
 	init.name = name;
 	init.flags = CLK_IGNORE_UNUSED;
 	init.parent_names = &parent_name;
@@ -266,11 +262,13 @@ static struct clk_hw *visconti_register_pll(struct visconti_pll_provider *ctx,
 
 	for (len = 0; rate_table[len].rate != 0; )
 		len++;
+
+	pll = kzalloc_flex(*pll, rate_table, len);
+	if (!pll)
+		return ERR_PTR(-ENOMEM);
+
 	pll->rate_count = len;
-	pll->rate_table = kmemdup_array(rate_table,
-					pll->rate_count, sizeof(*pll->rate_table),
-					GFP_KERNEL);
-	WARN(!pll->rate_table, "%s: could not allocate rate table for %s\n", __func__, name);
+	memcpy(pll->rate_table, rate_table, len * sizeof(*pll->rate_table));
 
 	init.ops = &visconti_pll_ops;
 	pll->hw.init = &init;
@@ -282,7 +280,6 @@ static struct clk_hw *visconti_register_pll(struct visconti_pll_provider *ctx,
 	ret = clk_hw_register(NULL, &pll->hw);
 	if (ret) {
 		pr_err("failed to register pll clock %s : %d\n", name, ret);
-		kfree(pll->rate_table);
 		kfree(pll);
 		pll_hw_clk = ERR_PTR(ret);
 	}
diff --git a/drivers/clocksource/mmio.c b/drivers/clocksource/mmio.c
index cd5fbf49ac29f..0fee8edb837a9 100644
--- a/drivers/clocksource/mmio.c
+++ b/drivers/clocksource/mmio.c
@@ -21,21 +21,25 @@ u64 clocksource_mmio_readl_up(struct clocksource *c)
 {
 	return (u64)readl_relaxed(to_mmio_clksrc(c)->reg);
 }
+EXPORT_SYMBOL_GPL(clocksource_mmio_readl_up);
 
 u64 clocksource_mmio_readl_down(struct clocksource *c)
 {
 	return ~(u64)readl_relaxed(to_mmio_clksrc(c)->reg) & c->mask;
 }
+EXPORT_SYMBOL_GPL(clocksource_mmio_readl_down);
 
 u64 clocksource_mmio_readw_up(struct clocksource *c)
 {
 	return (u64)readw_relaxed(to_mmio_clksrc(c)->reg);
 }
+EXPORT_SYMBOL_GPL(clocksource_mmio_readw_up);
 
 u64 clocksource_mmio_readw_down(struct clocksource *c)
 {
 	return ~(u64)readw_relaxed(to_mmio_clksrc(c)->reg) & c->mask;
 }
+EXPORT_SYMBOL_GPL(clocksource_mmio_readw_down);
 
 /**
  * clocksource_mmio_init - Initialize a simple mmio based clocksource
@@ -46,9 +50,9 @@ u64 clocksource_mmio_readw_down(struct clocksource *c)
  * @bits:	Number of valid bits
  * @read:	One of clocksource_mmio_read*() above
  */
-int __init clocksource_mmio_init(void __iomem *base, const char *name,
-	unsigned long hz, int rating, unsigned bits,
-	u64 (*read)(struct clocksource *))
+int clocksource_mmio_init(void __iomem *base, const char *name,
+			  unsigned long hz, int rating, unsigned bits,
+			  u64 (*read)(struct clocksource *))
 {
 	struct clocksource_mmio *cs;
 
@@ -68,3 +72,4 @@ int __init clocksource_mmio_init(void __iomem *base, const char *name,
 
 	return clocksource_register_hz(&cs->clksrc, hz);
 }
+EXPORT_SYMBOL_GPL(clocksource_mmio_init);
diff --git a/drivers/clocksource/timer-of.c b/drivers/clocksource/timer-of.c
index 420202bf76e42..ba63433211b0f 100644
--- a/drivers/clocksource/timer-of.c
+++ b/drivers/clocksource/timer-of.c
@@ -19,7 +19,7 @@
  *
  * Free the irq resource
  */
-static __init void timer_of_irq_exit(struct of_timer_irq *of_irq)
+static void timer_of_irq_exit(struct of_timer_irq *of_irq)
 {
 	struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
 
@@ -41,8 +41,8 @@ static __init void timer_of_irq_exit(struct of_timer_irq *of_irq)
  *
  * Returns 0 on success, < 0 otherwise
  */
-static __init int timer_of_irq_init(struct device_node *np,
-				    struct of_timer_irq *of_irq)
+static int timer_of_irq_init(struct device_node *np,
+			     struct of_timer_irq *of_irq)
 {
 	int ret;
 	struct timer_of *to = container_of(of_irq, struct timer_of, of_irq);
@@ -82,7 +82,7 @@ static __init int timer_of_irq_init(struct device_node *np,
  *
  * Disables and releases the refcount on the clk
  */
-static __init void timer_of_clk_exit(struct of_timer_clk *of_clk)
+static void timer_of_clk_exit(struct of_timer_clk *of_clk)
 {
 	of_clk->rate = 0;
 	clk_disable_unprepare(of_clk->clk);
@@ -98,8 +98,8 @@ static __init void timer_of_clk_exit(struct of_timer_clk *of_clk)
  *
  * Returns 0 on success, < 0 otherwise
  */
-static __init int timer_of_clk_init(struct device_node *np,
-				    struct of_timer_clk *of_clk)
+static int timer_of_clk_init(struct device_node *np,
+			     struct of_timer_clk *of_clk)
 {
 	int ret;
 
@@ -137,13 +137,13 @@ static __init int timer_of_clk_init(struct device_node *np,
 	goto out;
 }
 
-static __init void timer_of_base_exit(struct of_timer_base *of_base)
+static void timer_of_base_exit(struct of_timer_base *of_base)
 {
 	iounmap(of_base->base);
 }
 
-static __init int timer_of_base_init(struct device_node *np,
-				     struct of_timer_base *of_base)
+static  int timer_of_base_init(struct device_node *np,
+			       struct of_timer_base *of_base)
 {
 	of_base->base = of_base->name ?
 		of_io_request_and_map(np, of_base->index, of_base->name) :
@@ -156,7 +156,7 @@ static __init int timer_of_base_init(struct device_node *np,
 	return 0;
 }
 
-int __init timer_of_init(struct device_node *np, struct timer_of *to)
+int timer_of_init(struct device_node *np, struct timer_of *to)
 {
 	int ret = -EINVAL;
 	int flags = 0;
@@ -200,6 +200,7 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
 		timer_of_base_exit(&to->of_base);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(timer_of_init);
 
 /**
  * timer_of_cleanup - release timer_of resources
@@ -208,7 +209,7 @@ int __init timer_of_init(struct device_node *np, struct timer_of *to)
  * Release the resources that has been used in timer_of_init().
  * This function should be called in init error cases
  */
-void __init timer_of_cleanup(struct timer_of *to)
+void timer_of_cleanup(struct timer_of *to)
 {
 	if (to->flags & TIMER_OF_IRQ)
 		timer_of_irq_exit(&to->of_irq);
@@ -219,3 +220,4 @@ void __init timer_of_cleanup(struct timer_of *to)
 	if (to->flags & TIMER_OF_BASE)
 		timer_of_base_exit(&to->of_base);
 }
+EXPORT_SYMBOL_GPL(timer_of_cleanup);
diff --git a/drivers/clocksource/timer-of.h b/drivers/clocksource/timer-of.h
index 01a2c6b7db065..74a632b85b472 100644
--- a/drivers/clocksource/timer-of.h
+++ b/drivers/clocksource/timer-of.h
@@ -65,9 +65,8 @@ static inline unsigned long timer_of_period(struct timer_of *to)
 	return to->of_clk.period;
 }
 
-extern int __init timer_of_init(struct device_node *np,
-				struct timer_of *to);
+int timer_of_init(struct device_node *np, struct timer_of *to);
 
-extern void __init timer_of_cleanup(struct timer_of *to);
+void timer_of_cleanup(struct timer_of *to);
 
 #endif
diff --git a/drivers/clocksource/timer-rtl-otto.c b/drivers/clocksource/timer-rtl-otto.c
index 6113d2fdd4de1..dd236a7babee8 100644
--- a/drivers/clocksource/timer-rtl-otto.c
+++ b/drivers/clocksource/timer-rtl-otto.c
@@ -225,7 +225,7 @@ static int rttm_enable_clocksource(struct clocksource *cs)
 	return 0;
 }
 
-struct rttm_cs rttm_cs = {
+static struct rttm_cs rttm_cs = {
 	.to = {
 		.flags	= TIMER_OF_BASE | TIMER_OF_CLOCK,
 	},
diff --git a/drivers/clocksource/timer-sun5i.c b/drivers/clocksource/timer-sun5i.c
index f827d3f98f60e..6ab300d226215 100644
--- a/drivers/clocksource/timer-sun5i.c
+++ b/drivers/clocksource/timer-sun5i.c
@@ -18,21 +18,30 @@
 #include <linux/slab.h>
 #include <linux/platform_device.h>
 
-#define TIMER_IRQ_EN_REG		0x00
+#define TIMER_IRQ_EN_REG			0x00
 #define TIMER_IRQ_EN(val)			BIT(val)
-#define TIMER_IRQ_ST_REG		0x04
-#define TIMER_CTL_REG(val)		(0x20 * (val) + 0x10)
+#define TIMER_IRQ_ST_REG			0x04
+#define TIMER_CTL_REG(val, offset)		(0x20 * (val) + 0x10 + (offset))
 #define TIMER_CTL_ENABLE			BIT(0)
 #define TIMER_CTL_RELOAD			BIT(1)
 #define TIMER_CTL_CLK_PRES(val)			(((val) & 0x7) << 4)
 #define TIMER_CTL_ONESHOT			BIT(7)
-#define TIMER_INTVAL_LO_REG(val)	(0x20 * (val) + 0x14)
-#define TIMER_INTVAL_HI_REG(val)	(0x20 * (val) + 0x18)
-#define TIMER_CNTVAL_LO_REG(val)	(0x20 * (val) + 0x1c)
-#define TIMER_CNTVAL_HI_REG(val)	(0x20 * (val) + 0x20)
+#define TIMER_INTVAL_LO_REG(val, offset)	(0x20 * (val) + 0x14 + (offset))
+#define TIMER_INTVAL_HI_REG(val, offset)	(0x20 * (val) + 0x18 + (offset))
+#define TIMER_CNTVAL_LO_REG(val, offset)	(0x20 * (val) + 0x1c + (offset))
+#define TIMER_CNTVAL_HI_REG(val, offset)	(0x20 * (val) + 0x20 + (offset))
 
 #define TIMER_SYNC_TICKS	3
 
+/**
+ * struct sunxi_timer_quirks - Differences between SoC variants.
+ *
+ * @from_ctl_base_offset: offset applied from ctl register onwards
+ */
+struct sunxi_timer_quirks {
+	u32 from_ctl_base_offset;
+};
+
 struct sun5i_timer {
 	void __iomem		*base;
 	struct clk		*clk;
@@ -40,6 +49,7 @@ struct sun5i_timer {
 	u32			ticks_per_jiffy;
 	struct clocksource	clksrc;
 	struct clock_event_device	clkevt;
+	const struct sunxi_timer_quirks *quirks;
 };
 
 #define nb_to_sun5i_timer(x) \
@@ -57,28 +67,36 @@ struct sun5i_timer {
  */
 static void sun5i_clkevt_sync(struct sun5i_timer *ce)
 {
-	u32 old = readl(ce->base + TIMER_CNTVAL_LO_REG(1));
+	u32 offset = ce->quirks->from_ctl_base_offset;
+	u32 old = readl(ce->base + TIMER_CNTVAL_LO_REG(1, offset));
 
-	while ((old - readl(ce->base + TIMER_CNTVAL_LO_REG(1))) < TIMER_SYNC_TICKS)
+	while ((old - readl(ce->base + TIMER_CNTVAL_LO_REG(1, offset))) <
+	       TIMER_SYNC_TICKS)
 		cpu_relax();
 }
 
 static void sun5i_clkevt_time_stop(struct sun5i_timer *ce, u8 timer)
 {
-	u32 val = readl(ce->base + TIMER_CTL_REG(timer));
-	writel(val & ~TIMER_CTL_ENABLE, ce->base + TIMER_CTL_REG(timer));
+	u32 offset = ce->quirks->from_ctl_base_offset;
+	u32 val = readl(ce->base + TIMER_CTL_REG(timer, offset));
+
+	writel(val & ~TIMER_CTL_ENABLE,
+	       ce->base + TIMER_CTL_REG(timer, offset));
 
 	sun5i_clkevt_sync(ce);
 }
 
 static void sun5i_clkevt_time_setup(struct sun5i_timer *ce, u8 timer, u32 delay)
 {
-	writel(delay, ce->base + TIMER_INTVAL_LO_REG(timer));
+	u32 offset = ce->quirks->from_ctl_base_offset;
+
+	writel(delay, ce->base + TIMER_INTVAL_LO_REG(timer, offset));
 }
 
 static void sun5i_clkevt_time_start(struct sun5i_timer *ce, u8 timer, bool periodic)
 {
-	u32 val = readl(ce->base + TIMER_CTL_REG(timer));
+	u32 offset = ce->quirks->from_ctl_base_offset;
+	u32 val = readl(ce->base + TIMER_CTL_REG(timer, offset));
 
 	if (periodic)
 		val &= ~TIMER_CTL_ONESHOT;
@@ -86,7 +104,7 @@ static void sun5i_clkevt_time_start(struct sun5i_timer *ce, u8 timer, bool perio
 		val |= TIMER_CTL_ONESHOT;
 
 	writel(val | TIMER_CTL_ENABLE | TIMER_CTL_RELOAD,
-	       ce->base + TIMER_CTL_REG(timer));
+	       ce->base + TIMER_CTL_REG(timer, offset));
 }
 
 static int sun5i_clkevt_shutdown(struct clock_event_device *clkevt)
@@ -141,8 +159,9 @@ static irqreturn_t sun5i_timer_interrupt(int irq, void *dev_id)
 static u64 sun5i_clksrc_read(struct clocksource *clksrc)
 {
 	struct sun5i_timer *cs = clksrc_to_sun5i_timer(clksrc);
+	u32 offset = cs->quirks->from_ctl_base_offset;
 
-	return ~readl(cs->base + TIMER_CNTVAL_LO_REG(1));
+	return ~readl(cs->base + TIMER_CNTVAL_LO_REG(1, offset));
 }
 
 static int sun5i_rate_cb(struct notifier_block *nb,
@@ -173,12 +192,13 @@ static int sun5i_setup_clocksource(struct platform_device *pdev,
 				   unsigned long rate)
 {
 	struct sun5i_timer *cs = platform_get_drvdata(pdev);
+	u32 offset = cs->quirks->from_ctl_base_offset;
 	void __iomem *base = cs->base;
 	int ret;
 
-	writel(~0, base + TIMER_INTVAL_LO_REG(1));
+	writel(~0, base + TIMER_INTVAL_LO_REG(1, offset));
 	writel(TIMER_CTL_ENABLE | TIMER_CTL_RELOAD,
-	       base + TIMER_CTL_REG(1));
+	       base + TIMER_CTL_REG(1, offset));
 
 	cs->clksrc.name = pdev->dev.of_node->name;
 	cs->clksrc.rating = 340;
@@ -237,6 +257,7 @@ static int sun5i_setup_clockevent(struct platform_device *pdev,
 
 static int sun5i_timer_probe(struct platform_device *pdev)
 {
+	const struct sunxi_timer_quirks *quirks;
 	struct device *dev = &pdev->dev;
 	struct sun5i_timer *st;
 	struct reset_control *rstc;
@@ -273,11 +294,18 @@ static int sun5i_timer_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+	quirks = of_device_get_match_data(&pdev->dev);
+	if (!quirks) {
+		dev_err(&pdev->dev, "Failed to determine the quirks to use\n");
+		return -ENODEV;
+	}
+
 	st->base = timer_base;
 	st->ticks_per_jiffy = DIV_ROUND_UP(rate, HZ);
 	st->clk = clk;
 	st->clk_rate_cb.notifier_call = sun5i_rate_cb;
 	st->clk_rate_cb.next = NULL;
+	st->quirks = quirks;
 
 	ret = devm_clk_notifier_register(dev, clk, &st->clk_rate_cb);
 	if (ret) {
@@ -286,6 +314,9 @@ static int sun5i_timer_probe(struct platform_device *pdev)
 	}
 
 	rstc = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(rstc))
+		return dev_err_probe(dev, PTR_ERR(rstc),
+				     "failed to get reset\n");
 	if (rstc)
 		reset_control_deassert(rstc);
 
@@ -311,9 +342,27 @@ static void sun5i_timer_remove(struct platform_device *pdev)
 	clocksource_unregister(&st->clksrc);
 }
 
+static const struct sunxi_timer_quirks sun5i_sun7i_hstimer_quirks = {
+	.from_ctl_base_offset = 0x0,
+};
+
+static const struct sunxi_timer_quirks sun20i_d1_hstimer_quirks = {
+	.from_ctl_base_offset = 0x10,
+};
+
 static const struct of_device_id sun5i_timer_of_match[] = {
-	{ .compatible = "allwinner,sun5i-a13-hstimer" },
-	{ .compatible = "allwinner,sun7i-a20-hstimer" },
+	{
+		.compatible = "allwinner,sun5i-a13-hstimer",
+		.data = &sun5i_sun7i_hstimer_quirks,
+	},
+	{
+		.compatible = "allwinner,sun7i-a20-hstimer",
+		.data = &sun5i_sun7i_hstimer_quirks,
+	},
+	{
+		.compatible = "allwinner,sun20i-d1-hstimer",
+		.data = &sun20i_d1_hstimer_quirks,
+	},
 	{},
 };
 MODULE_DEVICE_TABLE(of, sun5i_timer_of_match);
diff --git a/drivers/counter/counter-core.c b/drivers/counter/counter-core.c
index 50bd30ba3d037..0b1dac61b7b56 100644
--- a/drivers/counter/counter-core.c
+++ b/drivers/counter/counter-core.c
@@ -124,7 +124,8 @@ struct counter_device *counter_alloc(size_t sizeof_priv)
 
 err_dev_set_name:
 
-	counter_chrdev_remove(counter);
+	put_device(dev);
+	return NULL;
 err_chrdev_add:
 
 	ida_free(&counter_ida, dev->id);
diff --git a/drivers/counter/intel-qep.c b/drivers/counter/intel-qep.c
index c49c178056f45..9c6536f75afd2 100644
--- a/drivers/counter/intel-qep.c
+++ b/drivers/counter/intel-qep.c
@@ -188,25 +188,21 @@ static int intel_qep_ceiling_write(struct counter_device *counter,
 				   struct counter_count *count, u64 max)
 {
 	struct intel_qep *qep = counter_priv(counter);
-	int ret = 0;
 
 	/* Intel QEP ceiling configuration only supports 32-bit values */
 	if (max != (u32)max)
 		return -ERANGE;
 
-	mutex_lock(&qep->lock);
-	if (qep->enabled) {
-		ret = -EBUSY;
-		goto out;
-	}
+	guard(mutex)(&qep->lock);
+
+	if (qep->enabled)
+		return -EBUSY;
 
 	pm_runtime_get_sync(qep->dev);
 	intel_qep_writel(qep, INTEL_QEPMAX, max);
 	pm_runtime_put(qep->dev);
 
-out:
-	mutex_unlock(&qep->lock);
-	return ret;
+	return 0;
 }
 
 static int intel_qep_enable_read(struct counter_device *counter,
@@ -226,10 +222,11 @@ static int intel_qep_enable_write(struct counter_device *counter,
 	u32 reg;
 	bool changed;
 
-	mutex_lock(&qep->lock);
+	guard(mutex)(&qep->lock);
+
 	changed = val ^ qep->enabled;
 	if (!changed)
-		goto out;
+		return 0;
 
 	pm_runtime_get_sync(qep->dev);
 	reg = intel_qep_readl(qep, INTEL_QEPCON);
@@ -246,8 +243,6 @@ static int intel_qep_enable_write(struct counter_device *counter,
 	pm_runtime_put(qep->dev);
 	qep->enabled = val;
 
-out:
-	mutex_unlock(&qep->lock);
 	return 0;
 }
 
@@ -279,7 +274,6 @@ static int intel_qep_spike_filter_ns_write(struct counter_device *counter,
 	struct intel_qep *qep = counter_priv(counter);
 	u32 reg;
 	bool enable;
-	int ret = 0;
 
 	/*
 	 * Spike filter length is (MAX_COUNT + 2) clock periods.
@@ -300,11 +294,10 @@ static int intel_qep_spike_filter_ns_write(struct counter_device *counter,
 	if (length > INTEL_QEPFLT_MAX_COUNT(length))
 		return -ERANGE;
 
-	mutex_lock(&qep->lock);
-	if (qep->enabled) {
-		ret = -EBUSY;
-		goto out;
-	}
+	guard(mutex)(&qep->lock);
+
+	if (qep->enabled)
+		return -EBUSY;
 
 	pm_runtime_get_sync(qep->dev);
 	reg = intel_qep_readl(qep, INTEL_QEPCON);
@@ -316,9 +309,7 @@ static int intel_qep_spike_filter_ns_write(struct counter_device *counter,
 	intel_qep_writel(qep, INTEL_QEPCON, reg);
 	pm_runtime_put(qep->dev);
 
-out:
-	mutex_unlock(&qep->lock);
-	return ret;
+	return 0;
 }
 
 static int intel_qep_preset_enable_read(struct counter_device *counter,
@@ -342,13 +333,11 @@ static int intel_qep_preset_enable_write(struct counter_device *counter,
 {
 	struct intel_qep *qep = counter_priv(counter);
 	u32 reg;
-	int ret = 0;
 
-	mutex_lock(&qep->lock);
-	if (qep->enabled) {
-		ret = -EBUSY;
-		goto out;
-	}
+	guard(mutex)(&qep->lock);
+
+	if (qep->enabled)
+		return -EBUSY;
 
 	pm_runtime_get_sync(qep->dev);
 	reg = intel_qep_readl(qep, INTEL_QEPCON);
@@ -360,10 +349,7 @@ static int intel_qep_preset_enable_write(struct counter_device *counter,
 	intel_qep_writel(qep, INTEL_QEPCON, reg);
 	pm_runtime_put(qep->dev);
 
-out:
-	mutex_unlock(&qep->lock);
-
-	return ret;
+	return 0;
 }
 
 static struct counter_comp intel_qep_count_ext[] = {
diff --git a/drivers/cpufreq/Kconfig.arm b/drivers/cpufreq/Kconfig.arm
index 47c9b031f1b3b..a441668f9e0cc 100644
--- a/drivers/cpufreq/Kconfig.arm
+++ b/drivers/cpufreq/Kconfig.arm
@@ -153,7 +153,7 @@ config ARM_QCOM_CPUFREQ_NVMEM
 	  If in doubt, say N.
 
 config ARM_QCOM_CPUFREQ_HW
-	tristate "QCOM CPUFreq HW driver"
+	tristate "Qualcomm CPUFreq HW driver"
 	depends on ARCH_QCOM || COMPILE_TEST
 	depends on COMMON_CLK
 	help
diff --git a/drivers/cpufreq/Kconfig.x86 b/drivers/cpufreq/Kconfig.x86
index 027e6ea2e0384..53025a063d2f4 100644
--- a/drivers/cpufreq/Kconfig.x86
+++ b/drivers/cpufreq/Kconfig.x86
@@ -70,18 +70,6 @@ config X86_AMD_PSTATE_DEFAULT_MODE
 	  For details, take a look at:
 	  <file:Documentation/admin-guide/pm/amd-pstate.rst>.
 
-config X86_AMD_PSTATE_DYNAMIC_EPP
-	bool "AMD Processor P-State dynamic EPP support"
-	depends on X86_AMD_PSTATE
-	default n
-	help
-	  Allow the kernel to dynamically change the energy performance
-	  value from events like ACPI platform profile and AC adapter plug
-	  events.
-
-	  This feature can also be changed at runtime, this configuration
-	  option only sets the kernel default value behavior.
-
 config X86_AMD_PSTATE_UT
 	tristate "selftest for AMD Processor P-State driver"
 	depends on X86 && ACPI_PROCESSOR
@@ -126,32 +114,6 @@ config X86_ACPI_CPUFREQ_CPB
 	  By enabling this option the acpi_cpufreq driver provides the old
 	  entry in addition to the new boost ones, for compatibility reasons.
 
-config ELAN_CPUFREQ
-	tristate "AMD Elan SC400 and SC410"
-	depends on MELAN
-	help
-	  This adds the CPUFreq driver for AMD Elan SC400 and SC410
-	  processors.
-
-	  You need to specify the processor maximum speed as boot
-	  parameter: elanfreq=maxspeed (in kHz) or as module
-	  parameter "max_freq".
-
-	  For details, take a look at <file:Documentation/cpu-freq/>.
-
-	  If in doubt, say N.
-
-config SC520_CPUFREQ
-	tristate "AMD Elan SC520"
-	depends on MELAN
-	help
-	  This adds the CPUFreq driver for AMD Elan SC520 processor.
-
-	  For details, take a look at <file:Documentation/cpu-freq/>.
-
-	  If in doubt, say N.
-
-
 config X86_POWERNOW_K6
 	tristate "AMD Mobile K6-2/K6-3 PowerNow!"
 	depends on X86_32
diff --git a/drivers/cpufreq/Makefile b/drivers/cpufreq/Makefile
index 385c9fcc65c62..6c7a39b7f8d26 100644
--- a/drivers/cpufreq/Makefile
+++ b/drivers/cpufreq/Makefile
@@ -40,8 +40,6 @@ obj-$(CONFIG_X86_POWERNOW_K6)		+= powernow-k6.o
 obj-$(CONFIG_X86_POWERNOW_K7)		+= powernow-k7.o
 obj-$(CONFIG_X86_LONGHAUL)		+= longhaul.o
 obj-$(CONFIG_X86_E_POWERSAVER)		+= e_powersaver.o
-obj-$(CONFIG_ELAN_CPUFREQ)		+= elanfreq.o
-obj-$(CONFIG_SC520_CPUFREQ)		+= sc520_freq.o
 obj-$(CONFIG_X86_LONGRUN)		+= longrun.o
 obj-$(CONFIG_X86_GX_SUSPMOD)		+= gx-suspmod.o
 obj-$(CONFIG_X86_SPEEDSTEP_ICH)		+= speedstep-ich.o
diff --git a/drivers/cpufreq/amd-pstate-ut.c b/drivers/cpufreq/amd-pstate-ut.c
index aa8a464fab47a..13a23dac477d1 100644
--- a/drivers/cpufreq/amd-pstate-ut.c
+++ b/drivers/cpufreq/amd-pstate-ut.c
@@ -274,20 +274,21 @@ static int amd_pstate_set_mode(enum amd_pstate_mode mode)
 
 static int amd_pstate_ut_epp(u32 index)
 {
-	struct cpufreq_policy *policy __free(put_cpufreq_policy) = NULL;
-	char *buf __free(cleanup_page) = NULL;
 	static const char * const epp_strings[] = {
-		"performance",
-		"balance_performance",
-		"balance_power",
 		"power",
+		"balance_power",
+		"balance_performance",
+		"performance",
 	};
-	struct amd_cpudata *cpudata;
+	char *buf __free(cleanup_page) = NULL;
+	struct cpufreq_policy *policy = NULL;
 	enum amd_pstate_mode orig_mode;
+	struct amd_cpudata *cpudata;
+	unsigned long orig_policy;
 	bool orig_dynamic_epp;
 	int ret, cpu = 0;
-	int i;
 	u16 epp;
+	int i;
 
 	policy = cpufreq_cpu_get(cpu);
 	if (!policy)
@@ -297,6 +298,10 @@ static int amd_pstate_ut_epp(u32 index)
 	orig_mode = amd_pstate_get_status();
 	orig_dynamic_epp = cpudata->dynamic_epp;
 
+	/* Drop reference before potential driver change. */
+	cpufreq_cpu_put(policy);
+	policy = NULL;
+
 	/* disable dynamic EPP before running test */
 	if (cpudata->dynamic_epp) {
 		pr_debug("Dynamic EPP is enabled, disabling it\n");
@@ -311,6 +316,17 @@ static int amd_pstate_ut_epp(u32 index)
 	if (ret)
 		goto out;
 
+	policy = cpufreq_cpu_get(cpu);
+	if (!policy) {
+		ret = -ENODEV;
+		goto out;
+	}
+
+	down_write(&policy->rwsem);
+	cpudata = policy->driver_data;
+	orig_policy = cpudata->policy;
+	cpudata->policy = CPUFREQ_POLICY_POWERSAVE;
+
 	for (epp = 0; epp <= U8_MAX; epp++) {
 		u8 val;
 
@@ -358,6 +374,12 @@ static int amd_pstate_ut_epp(u32 index)
 	ret = 0;
 
 out:
+	if (policy) {
+		cpudata->policy = orig_policy;
+		up_write(&policy->rwsem);
+		cpufreq_cpu_put(policy);
+	}
+
 	if (orig_dynamic_epp) {
 		int ret2;
 
diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c
index 453084c67327f..6f415860bd2a4 100644
--- a/drivers/cpufreq/amd-pstate.c
+++ b/drivers/cpufreq/amd-pstate.c
@@ -87,11 +87,7 @@ static struct cpufreq_driver amd_pstate_driver;
 static struct cpufreq_driver amd_pstate_epp_driver;
 static int cppc_state = AMD_PSTATE_UNDEFINED;
 static bool amd_pstate_prefcore = true;
-#ifdef CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP
-static bool dynamic_epp = CONFIG_X86_AMD_PSTATE_DYNAMIC_EPP;
-#else
 static bool dynamic_epp;
-#endif
 static struct quirk_entry *quirks;
 
 /*
@@ -246,12 +242,10 @@ static int msr_update_perf(struct cpufreq_policy *policy, u8 min_perf,
 
 	value = prev = READ_ONCE(cpudata->cppc_req_cached);
 
-	value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK |
-		   AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK);
-	value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf);
-	value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf);
-	value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf);
-	value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp);
+	FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf);
+	FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf);
+	FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf);
+	FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp);
 
 	if (trace_amd_pstate_epp_perf_enabled()) {
 		union perf_cached perf = READ_ONCE(cpudata->perf);
@@ -300,8 +294,7 @@ static int msr_set_epp(struct cpufreq_policy *policy, u8 epp)
 	int ret;
 
 	value = prev = READ_ONCE(cpudata->cppc_req_cached);
-	value &= ~AMD_CPPC_EPP_PERF_MASK;
-	value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp);
+	FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp);
 
 	if (trace_amd_pstate_epp_perf_enabled()) {
 		union perf_cached perf = cpudata->perf;
@@ -441,8 +434,7 @@ static int shmem_set_epp(struct cpufreq_policy *policy, u8 epp)
 	}
 
 	value = READ_ONCE(cpudata->cppc_req_cached);
-	value &= ~AMD_CPPC_EPP_PERF_MASK;
-	value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp);
+	FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp);
 	WRITE_ONCE(cpudata->cppc_req_cached, value);
 
 	return ret;
@@ -575,12 +567,10 @@ static int shmem_update_perf(struct cpufreq_policy *policy, u8 min_perf,
 
 	value = prev = READ_ONCE(cpudata->cppc_req_cached);
 
-	value &= ~(AMD_CPPC_MAX_PERF_MASK | AMD_CPPC_MIN_PERF_MASK |
-		   AMD_CPPC_DES_PERF_MASK | AMD_CPPC_EPP_PERF_MASK);
-	value |= FIELD_PREP(AMD_CPPC_MAX_PERF_MASK, max_perf);
-	value |= FIELD_PREP(AMD_CPPC_DES_PERF_MASK, des_perf);
-	value |= FIELD_PREP(AMD_CPPC_MIN_PERF_MASK, min_perf);
-	value |= FIELD_PREP(AMD_CPPC_EPP_PERF_MASK, epp);
+	FIELD_MODIFY(AMD_CPPC_MAX_PERF_MASK, &value, max_perf);
+	FIELD_MODIFY(AMD_CPPC_DES_PERF_MASK, &value, des_perf);
+	FIELD_MODIFY(AMD_CPPC_MIN_PERF_MASK, &value, min_perf);
+	FIELD_MODIFY(AMD_CPPC_EPP_PERF_MASK, &value, epp);
 
 	if (trace_amd_pstate_epp_perf_enabled()) {
 		union perf_cached perf = READ_ONCE(cpudata->perf);
@@ -1291,6 +1281,8 @@ static int amd_pstate_set_dynamic_epp(struct cpufreq_policy *policy)
 		return ret;
 
 	cpudata->profile_name = kasprintf(GFP_KERNEL, "amd-pstate-epp-cpu%d", cpudata->cpu);
+	if (!cpudata->profile_name)
+		return -ENOMEM;
 
 	cpudata->ppdev = platform_profile_register(get_cpu_device(policy->cpu),
 						   cpudata->profile_name,
@@ -1427,7 +1419,7 @@ ssize_t store_energy_performance_preference(struct cpufreq_policy *policy,
 		if (ret)
 			epp = epp_values[ret];
 		else
-			epp = amd_pstate_get_balanced_epp(policy);
+			epp = cpudata->epp_default_dc;
 	}
 
 	if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) {
@@ -1707,6 +1699,8 @@ static int amd_pstate_change_driver_mode(int mode)
 {
 	int ret;
 
+	lockdep_assert_held(&amd_pstate_driver_lock);
+
 	ret = amd_pstate_unregister_driver(0);
 	if (ret)
 		return ret;
@@ -1821,8 +1815,16 @@ static ssize_t dynamic_epp_store(struct device *a, struct device_attribute *b,
 	if (ret)
 		return ret;
 
-	if (dynamic_epp == enabled)
+	guard(mutex)(&amd_pstate_driver_lock);
+
+	if (cppc_state != AMD_PSTATE_ACTIVE) {
+		pr_debug("dynamic_epp can only be toggled in active mode\n");
 		return -EINVAL;
+	}
+
+	/* Nothing to do */
+	if (dynamic_epp == enabled)
+		return count;
 
 	/* reinitialize with desired dynamic EPP value */
 	dynamic_epp = enabled;
@@ -1942,7 +1944,7 @@ static int amd_pstate_epp_cpu_init(struct cpufreq_policy *policy)
 	if (dynamic_epp)
 		ret = amd_pstate_set_dynamic_epp(policy);
 	else
-		ret = amd_pstate_set_epp(policy, amd_pstate_get_balanced_epp(policy));
+		ret = amd_pstate_set_epp(policy, cpudata->epp_default_dc);
 	if (ret)
 		goto free_cpudata1;
 
@@ -1970,12 +1972,13 @@ static void amd_pstate_epp_cpu_exit(struct cpufreq_policy *policy)
 	if (cpudata) {
 		union perf_cached perf = READ_ONCE(cpudata->perf);
 
+		if (cpudata->dynamic_epp)
+			amd_pstate_clear_dynamic_epp(policy);
+
 		/* Reset CPPC_REQ MSR to the BIOS value */
 		amd_pstate_update_perf(policy, perf.bios_min_perf, 0U, 0U, 0U, false);
 		amd_pstate_set_floor_perf(policy, cpudata->bios_floor_perf);
 
-		if (cpudata->dynamic_epp)
-			amd_pstate_clear_dynamic_epp(policy);
 		kfree(cpudata);
 		policy->driver_data = NULL;
 	}
diff --git a/drivers/cpufreq/apple-soc-cpufreq.c b/drivers/cpufreq/apple-soc-cpufreq.c
index 9396034167e50..638e5bf72185a 100644
--- a/drivers/cpufreq/apple-soc-cpufreq.c
+++ b/drivers/cpufreq/apple-soc-cpufreq.c
@@ -187,10 +187,8 @@ static int apple_soc_cpufreq_set_target(struct cpufreq_policy *policy,
 
 	reg &= ~priv->info->ps1_mask;
 	reg |= pstate << priv->info->ps1_shift;
-	if (priv->info->has_ps2) {
-		reg &= ~APPLE_DVFS_CMD_PS2;
-		reg |= FIELD_PREP(APPLE_DVFS_CMD_PS2, pstate);
-	}
+	if (priv->info->has_ps2)
+		FIELD_MODIFY(APPLE_DVFS_CMD_PS2, &reg, pstate);
 	reg |= APPLE_DVFS_CMD_SET;
 
 	writeq_relaxed(reg, priv->reg_base + APPLE_DVFS_CMD);
diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c
index 7e7f9dfb7a24c..bf5175f7551c7 100644
--- a/drivers/cpufreq/cppc_cpufreq.c
+++ b/drivers/cpufreq/cppc_cpufreq.c
@@ -982,7 +982,34 @@ store_energy_performance_preference_val(struct cpufreq_policy *policy,
 	return count;
 }
 
-CPPC_CPUFREQ_ATTR_RW_U64(perf_limited, cppc_get_perf_limited,
+static int cppc_get_perf_limited_filtered(int cpu, u64 *perf_limited)
+{
+	struct cpufreq_policy *policy;
+	struct cppc_cpudata *cpu_data;
+	int ret;
+
+	ret = cppc_get_perf_limited(cpu, perf_limited);
+	if (ret)
+		return ret;
+
+	policy = cpufreq_cpu_get_raw(cpu);
+	if (!policy)
+		return -EINVAL;
+
+	cpu_data = policy->driver_data;
+
+	/*
+	 * Desired Excursion is ignored when autonomous selection is
+	 * enabled. Clear the bit to avoid exposing meaningless state
+	 * to userspace.
+	 */
+	if (cpu_data && cpu_data->perf_ctrls.auto_sel)
+		*perf_limited &= ~CPPC_PERF_LIMITED_DESIRED_EXCURSION;
+
+	return 0;
+}
+
+CPPC_CPUFREQ_ATTR_RW_U64(perf_limited, cppc_get_perf_limited_filtered,
 			 cppc_set_perf_limited)
 
 cpufreq_freq_attr_ro(freqdomain_cpus);
diff --git a/drivers/cpufreq/elanfreq.c b/drivers/cpufreq/elanfreq.c
deleted file mode 100644
index fc5a58088b35c..0000000000000
--- a/drivers/cpufreq/elanfreq.c
+++ /dev/null
@@ -1,226 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *	elanfreq:	cpufreq driver for the AMD ELAN family
- *
- *	(c) Copyright 2002 Robert Schwebel <r.schwebel@pengutronix.de>
- *
- *	Parts of this code are (c) Sven Geggus <sven@geggus.net>
- *
- *      All Rights Reserved.
- *
- *	2002-02-13: - initial revision for 2.4.18-pre9 by Robert Schwebel
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-
-#include <linux/delay.h>
-#include <linux/cpufreq.h>
-
-#include <asm/cpu_device_id.h>
-#include <linux/timex.h>
-#include <linux/io.h>
-
-#define REG_CSCIR 0x22		/* Chip Setup and Control Index Register    */
-#define REG_CSCDR 0x23		/* Chip Setup and Control Data  Register    */
-
-/* Module parameter */
-static int max_freq;
-
-struct s_elan_multiplier {
-	int clock;		/* frequency in kHz                         */
-	int val40h;		/* PMU Force Mode register                  */
-	int val80h;		/* CPU Clock Speed Register                 */
-};
-
-/*
- * It is important that the frequencies
- * are listed in ascending order here!
- */
-static struct s_elan_multiplier elan_multiplier[] = {
-	{1000,	0x02,	0x18},
-	{2000,	0x02,	0x10},
-	{4000,	0x02,	0x08},
-	{8000,	0x00,	0x00},
-	{16000,	0x00,	0x02},
-	{33000,	0x00,	0x04},
-	{66000,	0x01,	0x04},
-	{99000,	0x01,	0x05}
-};
-
-static struct cpufreq_frequency_table elanfreq_table[] = {
-	{0, 0,	1000},
-	{0, 1,	2000},
-	{0, 2,	4000},
-	{0, 3,	8000},
-	{0, 4,	16000},
-	{0, 5,	33000},
-	{0, 6,	66000},
-	{0, 7,	99000},
-	{0, 0,	CPUFREQ_TABLE_END},
-};
-
-
-/**
- *	elanfreq_get_cpu_frequency: determine current cpu speed
- *
- *	Finds out at which frequency the CPU of the Elan SOC runs
- *	at the moment. Frequencies from 1 to 33 MHz are generated
- *	the normal way, 66 and 99 MHz are called "Hyperspeed Mode"
- *	and have the rest of the chip running with 33 MHz.
- */
-
-static unsigned int elanfreq_get_cpu_frequency(unsigned int cpu)
-{
-	u8 clockspeed_reg;    /* Clock Speed Register */
-
-	local_irq_disable();
-	outb_p(0x80, REG_CSCIR);
-	clockspeed_reg = inb_p(REG_CSCDR);
-	local_irq_enable();
-
-	if ((clockspeed_reg & 0xE0) == 0xE0)
-		return 0;
-
-	/* Are we in CPU clock multiplied mode (66/99 MHz)? */
-	if ((clockspeed_reg & 0xE0) == 0xC0) {
-		if ((clockspeed_reg & 0x01) == 0)
-			return 66000;
-		else
-			return 99000;
-	}
-
-	/* 33 MHz is not 32 MHz... */
-	if ((clockspeed_reg & 0xE0) == 0xA0)
-		return 33000;
-
-	return (1<<((clockspeed_reg & 0xE0) >> 5)) * 1000;
-}
-
-
-static int elanfreq_target(struct cpufreq_policy *policy,
-			    unsigned int state)
-{
-	/*
-	 * Access to the Elan's internal registers is indexed via
-	 * 0x22: Chip Setup & Control Register Index Register (CSCI)
-	 * 0x23: Chip Setup & Control Register Data  Register (CSCD)
-	 *
-	 */
-
-	/*
-	 * 0x40 is the Power Management Unit's Force Mode Register.
-	 * Bit 6 enables Hyperspeed Mode (66/100 MHz core frequency)
-	 */
-
-	local_irq_disable();
-	outb_p(0x40, REG_CSCIR);		/* Disable hyperspeed mode */
-	outb_p(0x00, REG_CSCDR);
-	local_irq_enable();		/* wait till internal pipelines and */
-	udelay(1000);			/* buffers have cleaned up          */
-
-	local_irq_disable();
-
-	/* now, set the CPU clock speed register (0x80) */
-	outb_p(0x80, REG_CSCIR);
-	outb_p(elan_multiplier[state].val80h, REG_CSCDR);
-
-	/* now, the hyperspeed bit in PMU Force Mode Register (0x40) */
-	outb_p(0x40, REG_CSCIR);
-	outb_p(elan_multiplier[state].val40h, REG_CSCDR);
-	udelay(10000);
-	local_irq_enable();
-
-	return 0;
-}
-/*
- *	Module init and exit code
- */
-
-static int elanfreq_cpu_init(struct cpufreq_policy *policy)
-{
-	struct cpuinfo_x86 *c = &cpu_data(0);
-	struct cpufreq_frequency_table *pos;
-
-	/* capability check */
-	if ((c->x86_vendor != X86_VENDOR_AMD) ||
-	    (c->x86 != 4) || (c->x86_model != 10))
-		return -ENODEV;
-
-	/* max freq */
-	if (!max_freq)
-		max_freq = elanfreq_get_cpu_frequency(0);
-
-	/* table init */
-	cpufreq_for_each_entry(pos, elanfreq_table)
-		if (pos->frequency > max_freq)
-			pos->frequency = CPUFREQ_ENTRY_INVALID;
-
-	policy->freq_table = elanfreq_table;
-	return 0;
-}
-
-
-#ifndef MODULE
-/**
- * elanfreq_setup - elanfreq command line parameter parsing
- *
- * elanfreq command line parameter.  Use:
- *  elanfreq=66000
- * to set the maximum CPU frequency to 66 MHz. Note that in
- * case you do not give this boot parameter, the maximum
- * frequency will fall back to _current_ CPU frequency which
- * might be lower. If you build this as a module, use the
- * max_freq module parameter instead.
- */
-static int __init elanfreq_setup(char *str)
-{
-	max_freq = simple_strtoul(str, &str, 0);
-	pr_warn("You're using the deprecated elanfreq command line option. Use elanfreq.max_freq instead, please!\n");
-	return 1;
-}
-__setup("elanfreq=", elanfreq_setup);
-#endif
-
-
-static struct cpufreq_driver elanfreq_driver = {
-	.get		= elanfreq_get_cpu_frequency,
-	.flags		= CPUFREQ_NO_AUTO_DYNAMIC_SWITCHING,
-	.verify		= cpufreq_generic_frequency_table_verify,
-	.target_index	= elanfreq_target,
-	.init		= elanfreq_cpu_init,
-	.name		= "elanfreq",
-};
-
-static const struct x86_cpu_id elan_id[] = {
-	X86_MATCH_VENDOR_FAM_MODEL(AMD, 4, 10, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, elan_id);
-
-static int __init elanfreq_init(void)
-{
-	if (!x86_match_cpu(elan_id))
-		return -ENODEV;
-	return cpufreq_register_driver(&elanfreq_driver);
-}
-
-
-static void __exit elanfreq_exit(void)
-{
-	cpufreq_unregister_driver(&elanfreq_driver);
-}
-
-
-module_param(max_freq, int, 0444);
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Robert Schwebel <r.schwebel@pengutronix.de>, "
-		"Sven Geggus <sven@geggus.net>");
-MODULE_DESCRIPTION("cpufreq driver for AMD's Elan CPUs");
-
-module_init(elanfreq_init);
-module_exit(elanfreq_exit);
diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
index 1292da53e5fcb..1f093e3464301 100644
--- a/drivers/cpufreq/intel_pstate.c
+++ b/drivers/cpufreq/intel_pstate.c
@@ -2279,7 +2279,7 @@ static int hwp_get_cpu_scaling(int cpu)
 		 * Return the hybrid scaling factor for P-cores and use the
 		 * default core scaling for E-cores.
 		 */
-		if (hybrid_get_cpu_type(cpu) == INTEL_CPU_TYPE_CORE)
+		if (hybrid_get_cpu_type(cpu) != INTEL_CPU_TYPE_ATOM)
 			return hybrid_scaling_factor;
 
 		return core_get_scaling();
@@ -3734,6 +3734,7 @@ static const struct x86_cpu_id intel_hybrid_scaling_factor[] = {
 	X86_MATCH_VFM(INTEL_RAPTORLAKE, HYBRID_SCALING_FACTOR_ADL),
 	X86_MATCH_VFM(INTEL_RAPTORLAKE_P, HYBRID_SCALING_FACTOR_ADL),
 	X86_MATCH_VFM(INTEL_RAPTORLAKE_S, HYBRID_SCALING_FACTOR_ADL),
+	X86_MATCH_VFM(INTEL_BARTLETTLAKE, HYBRID_SCALING_FACTOR_ADL),
 	X86_MATCH_VFM(INTEL_METEORLAKE_L, HYBRID_SCALING_FACTOR_MTL),
 	X86_MATCH_VFM(INTEL_LUNARLAKE_M, HYBRID_SCALING_FACTOR_LNL),
 	{}
diff --git a/drivers/cpufreq/longrun.c b/drivers/cpufreq/longrun.c
index 1caaec7c280ba..f3aaca0496a4c 100644
--- a/drivers/cpufreq/longrun.c
+++ b/drivers/cpufreq/longrun.c
@@ -14,6 +14,7 @@
 #include <asm/msr.h>
 #include <asm/processor.h>
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 
 static struct cpufreq_driver	longrun_driver;
 
diff --git a/drivers/cpufreq/powernow-k7.c b/drivers/cpufreq/powernow-k7.c
index 6b7caf4ae20df..6a930d7e6a5cb 100644
--- a/drivers/cpufreq/powernow-k7.c
+++ b/drivers/cpufreq/powernow-k7.c
@@ -29,6 +29,7 @@
 #include <asm/timer.h>		/* Needed for recalibrate_cpu_khz() */
 #include <asm/msr.h>
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 
 #ifdef CONFIG_X86_POWERNOW_K7_ACPI
 #include <linux/acpi.h>
diff --git a/drivers/cpufreq/powernow-k8.c b/drivers/cpufreq/powernow-k8.c
index 4d77eef53fe0f..2b791f1ec51bb 100644
--- a/drivers/cpufreq/powernow-k8.c
+++ b/drivers/cpufreq/powernow-k8.c
@@ -39,6 +39,7 @@
 
 #include <asm/msr.h>
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 
 #include <linux/acpi.h>
 #include <linux/mutex.h>
diff --git a/drivers/cpufreq/qcom-cpufreq-hw.c b/drivers/cpufreq/qcom-cpufreq-hw.c
index ea9a20d27b8fd..ef19faedbfec3 100644
--- a/drivers/cpufreq/qcom-cpufreq-hw.c
+++ b/drivers/cpufreq/qcom-cpufreq-hw.c
@@ -578,7 +578,6 @@ static void qcom_cpufreq_hw_cpu_exit(struct cpufreq_policy *policy)
 	dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);
 	qcom_cpufreq_hw_lmh_exit(data);
 	kfree(policy->freq_table);
-	kfree(data);
 }
 
 static void qcom_cpufreq_ready(struct cpufreq_policy *policy)
diff --git a/drivers/cpufreq/sc520_freq.c b/drivers/cpufreq/sc520_freq.c
deleted file mode 100644
index b360f03a116f3..0000000000000
--- a/drivers/cpufreq/sc520_freq.c
+++ /dev/null
@@ -1,136 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *	sc520_freq.c: cpufreq driver for the AMD Elan sc520
- *
- *	Copyright (C) 2005 Sean Young <sean@mess.org>
- *
- *	Based on elanfreq.c
- *
- *	2005-03-30: - initial revision
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-
-#include <linux/delay.h>
-#include <linux/cpufreq.h>
-#include <linux/timex.h>
-#include <linux/io.h>
-
-#include <asm/cpu_device_id.h>
-
-#define MMCR_BASE	0xfffef000	/* The default base address */
-#define OFFS_CPUCTL	0x2   /* CPU Control Register */
-
-static __u8 __iomem *cpuctl;
-
-static struct cpufreq_frequency_table sc520_freq_table[] = {
-	{0, 0x01,	100000},
-	{0, 0x02,	133000},
-	{0, 0,	CPUFREQ_TABLE_END},
-};
-
-static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu)
-{
-	u8 clockspeed_reg = *cpuctl;
-
-	switch (clockspeed_reg & 0x03) {
-	default:
-		pr_err("error: cpuctl register has unexpected value %02x\n",
-		       clockspeed_reg);
-		fallthrough;
-	case 0x01:
-		return 100000;
-	case 0x02:
-		return 133000;
-	}
-}
-
-static int sc520_freq_target(struct cpufreq_policy *policy, unsigned int state)
-{
-
-	u8 clockspeed_reg;
-
-	local_irq_disable();
-
-	clockspeed_reg = *cpuctl & ~0x03;
-	*cpuctl = clockspeed_reg | sc520_freq_table[state].driver_data;
-
-	local_irq_enable();
-
-	return 0;
-}
-
-/*
- *	Module init and exit code
- */
-
-static int sc520_freq_cpu_init(struct cpufreq_policy *policy)
-{
-	struct cpuinfo_x86 *c = &cpu_data(0);
-
-	/* capability check */
-	if (c->x86_vendor != X86_VENDOR_AMD ||
-	    c->x86 != 4 || c->x86_model != 9)
-		return -ENODEV;
-
-	/* cpuinfo and default policy values */
-	policy->cpuinfo.transition_latency = 1000000; /* 1ms */
-	policy->freq_table = sc520_freq_table;
-
-	return 0;
-}
-
-
-static struct cpufreq_driver sc520_freq_driver = {
-	.get	= sc520_freq_get_cpu_frequency,
-	.verify	= cpufreq_generic_frequency_table_verify,
-	.target_index = sc520_freq_target,
-	.init	= sc520_freq_cpu_init,
-	.name	= "sc520_freq",
-};
-
-static const struct x86_cpu_id sc520_ids[] = {
-	X86_MATCH_VENDOR_FAM_MODEL(AMD, 4, 9, NULL),
-	{}
-};
-MODULE_DEVICE_TABLE(x86cpu, sc520_ids);
-
-static int __init sc520_freq_init(void)
-{
-	int err;
-
-	if (!x86_match_cpu(sc520_ids))
-		return -ENODEV;
-
-	cpuctl = ioremap((unsigned long)(MMCR_BASE + OFFS_CPUCTL), 1);
-	if (!cpuctl) {
-		pr_err("sc520_freq: error: failed to remap memory\n");
-		return -ENOMEM;
-	}
-
-	err = cpufreq_register_driver(&sc520_freq_driver);
-	if (err)
-		iounmap(cpuctl);
-
-	return err;
-}
-
-
-static void __exit sc520_freq_exit(void)
-{
-	cpufreq_unregister_driver(&sc520_freq_driver);
-	iounmap(cpuctl);
-}
-
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Sean Young <sean@mess.org>");
-MODULE_DESCRIPTION("cpufreq driver for AMD's Elan sc520 CPU");
-
-module_init(sc520_freq_init);
-module_exit(sc520_freq_exit);
-
diff --git a/drivers/cpufreq/speedstep-lib.c b/drivers/cpufreq/speedstep-lib.c
index f8b42e981635a..973716c1c29ce 100644
--- a/drivers/cpufreq/speedstep-lib.c
+++ b/drivers/cpufreq/speedstep-lib.c
@@ -15,6 +15,7 @@
 #include <linux/init.h>
 #include <linux/cpufreq.h>
 
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 #include <asm/tsc.h>
 #include "speedstep-lib.h"
diff --git a/drivers/crypto/allwinner/Kconfig b/drivers/crypto/allwinner/Kconfig
index 7270e5fbc5738..b8e75210a0e31 100644
--- a/drivers/crypto/allwinner/Kconfig
+++ b/drivers/crypto/allwinner/Kconfig
@@ -14,7 +14,6 @@ config CRYPTO_DEV_SUN4I_SS
 	select CRYPTO_SHA1
 	select CRYPTO_AES
 	select CRYPTO_LIB_DES
-	select CRYPTO_RNG
 	select CRYPTO_SKCIPHER
 	help
 	  Some Allwinner SoC have a crypto accelerator named
@@ -50,7 +49,6 @@ config CRYPTO_DEV_SUN8I_CE
 	select CRYPTO_CBC
 	select CRYPTO_AES
 	select CRYPTO_DES
-	select CRYPTO_RNG
 	depends on CRYPTO_DEV_ALLWINNER
 	depends on PM
 	help
diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c
index 58a76e2ba64e2..813c4bc6312a5 100644
--- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c
+++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c
@@ -247,12 +247,14 @@ static int sun4i_ss_debugfs_show(struct seq_file *seq, void *v)
 				   ss_algs[i].stat_req, ss_algs[i].stat_opti, ss_algs[i].stat_fb,
 				   ss_algs[i].stat_bytes);
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n",
 				   ss_algs[i].alg.rng.base.cra_driver_name,
 				   ss_algs[i].alg.rng.base.cra_name,
 				   ss_algs[i].stat_req, ss_algs[i].stat_bytes);
 			break;
+#endif
 		case CRYPTO_ALG_TYPE_AHASH:
 			seq_printf(seq, "%s %s reqs=%lu\n",
 				   ss_algs[i].alg.hash.halg.base.cra_driver_name,
@@ -471,6 +473,7 @@ static int sun4i_ss_probe(struct platform_device *pdev)
 				goto error_alg;
 			}
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			err = crypto_register_rng(&ss_algs[i].alg.rng);
 			if (err) {
@@ -478,6 +481,7 @@ static int sun4i_ss_probe(struct platform_device *pdev)
 					ss_algs[i].alg.rng.base.cra_name);
 			}
 			break;
+#endif
 		}
 	}
 
@@ -497,9 +501,11 @@ static int sun4i_ss_probe(struct platform_device *pdev)
 		case CRYPTO_ALG_TYPE_AHASH:
 			crypto_unregister_ahash(&ss_algs[i].alg.hash);
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			crypto_unregister_rng(&ss_algs[i].alg.rng);
 			break;
+#endif
 		}
 	}
 error_pm:
@@ -520,9 +526,11 @@ static void sun4i_ss_remove(struct platform_device *pdev)
 		case CRYPTO_ALG_TYPE_AHASH:
 			crypto_unregister_ahash(&ss_algs[i].alg.hash);
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN4I_SS_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			crypto_unregister_rng(&ss_algs[i].alg.rng);
 			break;
+#endif
 		}
 	}
 
diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
index c16bb6ce6ee37..f3b58ed6aed0d 100644
--- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
+++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
@@ -676,6 +676,7 @@ static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v)
 			seq_printf(seq, "\tFallback due to SG numbers: %lu\n",
 				   ce_algs[i].stat_fb_maxsg);
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH
 		case CRYPTO_ALG_TYPE_AHASH:
 			seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
 				   ce_algs[i].alg.hash.base.halg.base.cra_driver_name,
@@ -692,12 +693,15 @@ static int sun8i_ce_debugfs_show(struct seq_file *seq, void *v)
 			seq_printf(seq, "\tFallback due to SG numbers: %lu\n",
 				   ce_algs[i].stat_fb_maxsg);
 			break;
+#endif
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			seq_printf(seq, "%s %s reqs=%lu bytes=%lu\n",
 				   ce_algs[i].alg.rng.base.cra_driver_name,
 				   ce_algs[i].alg.rng.base.cra_name,
 				   ce_algs[i].stat_req, ce_algs[i].stat_bytes);
 			break;
+#endif
 		}
 	}
 #if defined(CONFIG_CRYPTO_DEV_SUN8I_CE_TRNG) && \
@@ -905,6 +909,7 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce)
 				return err;
 			}
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH
 		case CRYPTO_ALG_TYPE_AHASH:
 			id = ce_algs[i].ce_algo_id;
 			ce_method = ce->variant->alg_hash[id];
@@ -925,6 +930,8 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce)
 				return err;
 			}
 			break;
+#endif
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			if (ce->variant->prng == CE_ID_NOTSUPP) {
 				dev_info(ce->dev,
@@ -942,6 +949,7 @@ static int sun8i_ce_register_algs(struct sun8i_ce_dev *ce)
 				ce_algs[i].ce = NULL;
 			}
 			break;
+#endif
 		default:
 			ce_algs[i].ce = NULL;
 			dev_err(ce->dev, "ERROR: tried to register an unknown algo\n");
@@ -963,16 +971,20 @@ static void sun8i_ce_unregister_algs(struct sun8i_ce_dev *ce)
 				 ce_algs[i].alg.skcipher.base.base.cra_name);
 			crypto_engine_unregister_skcipher(&ce_algs[i].alg.skcipher);
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_HASH
 		case CRYPTO_ALG_TYPE_AHASH:
 			dev_info(ce->dev, "Unregister %d %s\n", i,
 				 ce_algs[i].alg.hash.base.halg.base.cra_name);
 			crypto_engine_unregister_ahash(&ce_algs[i].alg.hash);
 			break;
+#endif
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			dev_info(ce->dev, "Unregister %d %s\n", i,
 				 ce_algs[i].alg.rng.base.cra_name);
 			crypto_unregister_rng(&ce_algs[i].alg.rng);
 			break;
+#endif
 		}
 	}
 }
diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
index f45685707e0d3..59c9bc45ec0f2 100644
--- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
+++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c
@@ -501,12 +501,15 @@ static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v)
 			seq_printf(seq, "\tFallback due to SG numbers: %lu\n",
 				   ss_algs[i].stat_fb_sgnum);
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			seq_printf(seq, "%s %s reqs=%lu tsize=%lu\n",
 				   ss_algs[i].alg.rng.base.cra_driver_name,
 				   ss_algs[i].alg.rng.base.cra_name,
 				   ss_algs[i].stat_req, ss_algs[i].stat_bytes);
 			break;
+#endif
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH
 		case CRYPTO_ALG_TYPE_AHASH:
 			seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n",
 				   ss_algs[i].alg.hash.base.halg.base.cra_driver_name,
@@ -523,6 +526,7 @@ static int sun8i_ss_debugfs_show(struct seq_file *seq, void *v)
 			seq_printf(seq, "\tFallback due to SG numbers: %lu\n",
 				   ss_algs[i].stat_fb_sgnum);
 			break;
+#endif
 		}
 	}
 	return 0;
@@ -707,6 +711,7 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss)
 				return err;
 			}
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			err = crypto_register_rng(&ss_algs[i].alg.rng);
 			if (err) {
@@ -715,6 +720,8 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss)
 				ss_algs[i].ss = NULL;
 			}
 			break;
+#endif
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH
 		case CRYPTO_ALG_TYPE_AHASH:
 			id = ss_algs[i].ss_algo_id;
 			ss_method = ss->variant->alg_hash[id];
@@ -735,6 +742,7 @@ static int sun8i_ss_register_algs(struct sun8i_ss_dev *ss)
 				return err;
 			}
 			break;
+#endif
 		default:
 			ss_algs[i].ss = NULL;
 			dev_err(ss->dev, "ERROR: tried to register an unknown algo\n");
@@ -756,16 +764,20 @@ static void sun8i_ss_unregister_algs(struct sun8i_ss_dev *ss)
 				 ss_algs[i].alg.skcipher.base.base.cra_name);
 			crypto_engine_unregister_skcipher(&ss_algs[i].alg.skcipher);
 			break;
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_PRNG
 		case CRYPTO_ALG_TYPE_RNG:
 			dev_info(ss->dev, "Unregister %d %s\n", i,
 				 ss_algs[i].alg.rng.base.cra_name);
 			crypto_unregister_rng(&ss_algs[i].alg.rng);
 			break;
+#endif
+#ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_HASH
 		case CRYPTO_ALG_TYPE_AHASH:
 			dev_info(ss->dev, "Unregister %d %s\n", i,
 				 ss_algs[i].alg.hash.base.halg.base.cra_name);
 			crypto_engine_unregister_ahash(&ss_algs[i].alg.hash);
 			break;
+#endif
 		}
 	}
 }
diff --git a/drivers/crypto/amlogic/amlogic-gxl-core.c b/drivers/crypto/amlogic/amlogic-gxl-core.c
index 1c18a5b8470e0..6cb33949915fe 100644
--- a/drivers/crypto/amlogic/amlogic-gxl-core.c
+++ b/drivers/crypto/amlogic/amlogic-gxl-core.c
@@ -291,8 +291,8 @@ static int meson_crypto_probe(struct platform_device *pdev)
 	return 0;
 error_alg:
 	meson_unregister_algs(mc);
-error_flow:
 	meson_free_chanlist(mc, MAXFLOW - 1);
+error_flow:
 	clk_disable_unprepare(mc->busclk);
 	return err;
 }
diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c
index 9c380351d2f9f..9660f6426a849 100644
--- a/drivers/crypto/atmel-ecc.c
+++ b/drivers/crypto/atmel-ecc.c
@@ -56,7 +56,7 @@ static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq,
 		goto free_work_data;
 
 	/* might want less than we've got */
-	n_sz = min_t(size_t, ATMEL_ECC_NIST_P256_N_SIZE, req->dst_len);
+	n_sz = min(ATMEL_ECC_NIST_P256_N_SIZE, req->dst_len);
 
 	/* copy the shared secret */
 	copied = sg_copy_from_buffer(req->dst, sg_nents_for_len(req->dst, n_sz),
@@ -150,7 +150,7 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req)
 		return -EINVAL;
 
 	/* might want less than we've got */
-	nbytes = min_t(size_t, ATMEL_ECC_PUBKEY_SIZE, req->dst_len);
+	nbytes = min(ATMEL_ECC_PUBKEY_SIZE, req->dst_len);
 
 	/* public key was saved at private key generation */
 	copied = sg_copy_from_buffer(req->dst,
@@ -368,19 +368,16 @@ static void atmel_ecc_remove(struct i2c_client *client)
 	spin_unlock(&driver_data.i2c_list_lock);
 }
 
-#ifdef CONFIG_OF
 static const struct of_device_id atmel_ecc_dt_ids[] = {
-	{
-		.compatible = "atmel,atecc508a",
-	}, {
-		/* sentinel */
-	}
+	{ .compatible = "atmel,atecc508a", },
+	{ .compatible = "atmel,atecc608b", },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, atmel_ecc_dt_ids);
-#endif
 
 static const struct i2c_device_id atmel_ecc_id[] = {
 	{ "atecc508a" },
+	{ "atecc608b" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, atmel_ecc_id);
@@ -388,7 +385,7 @@ MODULE_DEVICE_TABLE(i2c, atmel_ecc_id);
 static struct i2c_driver atmel_ecc_driver = {
 	.driver = {
 		.name	= "atmel-ecc",
-		.of_match_table = of_match_ptr(atmel_ecc_dt_ids),
+		.of_match_table = atmel_ecc_dt_ids,
 	},
 	.probe		= atmel_ecc_probe,
 	.remove		= atmel_ecc_remove,
diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c
index dbb39ed0cea11..6e6ac4770416e 100644
--- a/drivers/crypto/atmel-sha204a.c
+++ b/drivers/crypto/atmel-sha204a.c
@@ -19,6 +19,12 @@
 #include <linux/workqueue.h>
 #include "atmel-i2c.h"
 
+/*
+ * According to review by Bill Cox [1], the ATSHA204 has very low entropy.
+ * [1] https://www.metzdowd.com/pipermail/cryptography/2014-December/023858.html
+ */
+static const unsigned short atsha204_quality = 1;
+
 static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data,
 				   void *areq, int status)
 {
@@ -48,8 +54,8 @@ static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data,
 
 	if (rng->priv) {
 		work_data = (struct atmel_i2c_work_data *)rng->priv;
-		max = min(sizeof(work_data->cmd.data), max);
-		memcpy(data, &work_data->cmd.data, max);
+		max = min(RANDOM_RSP_SIZE - CMD_OVERHEAD_SIZE, max);
+		memcpy(data, &work_data->cmd.data[RSP_DATA_IDX], max);
 		rng->priv = 0;
 	} else {
 		work_data = kmalloc_obj(*work_data, GFP_ATOMIC);
@@ -87,8 +93,8 @@ static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max,
 	if (ret)
 		return ret;
 
-	max = min(sizeof(cmd.data), max);
-	memcpy(data, cmd.data, max);
+	max = min(RANDOM_RSP_SIZE - CMD_OVERHEAD_SIZE, max);
+	memcpy(data, &cmd.data[RSP_DATA_IDX], max);
 
 	return max;
 }
@@ -158,6 +164,7 @@ static const struct attribute_group atmel_sha204a_groups = {
 static int atmel_sha204a_probe(struct i2c_client *client)
 {
 	struct atmel_i2c_client_priv *i2c_priv;
+	const unsigned short *quality;
 	int ret;
 
 	ret = atmel_i2c_probe(client);
@@ -171,11 +178,9 @@ static int atmel_sha204a_probe(struct i2c_client *client)
 	i2c_priv->hwrng.name = dev_name(&client->dev);
 	i2c_priv->hwrng.read = atmel_sha204a_rng_read;
 
-	/*
-	 * According to review by Bill Cox [1], this HWRNG has very low entropy.
-	 * [1] https://www.metzdowd.com/pipermail/cryptography/2014-December/023858.html
-	 */
-	i2c_priv->hwrng.quality = 1;
+	quality = i2c_get_match_data(client);
+	if (quality)
+		i2c_priv->hwrng.quality = *quality;
 
 	ret = devm_hwrng_register(&client->dev, &i2c_priv->hwrng);
 	if (ret)
@@ -202,17 +207,17 @@ static void atmel_sha204a_remove(struct i2c_client *client)
 	kfree((void *)i2c_priv->hwrng.priv);
 }
 
-static const struct of_device_id atmel_sha204a_dt_ids[] __maybe_unused = {
-	{ .compatible = "atmel,atsha204", },
+static const struct of_device_id atmel_sha204a_dt_ids[] = {
+	{ .compatible = "atmel,atsha204", .data = &atsha204_quality },
 	{ .compatible = "atmel,atsha204a", },
-	{ /* sentinel */ }
+	{ }
 };
 MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids);
 
 static const struct i2c_device_id atmel_sha204a_id[] = {
-	{ "atsha204" },
+	{ "atsha204", (kernel_ulong_t)&atsha204_quality },
 	{ "atsha204a" },
-	{ /* sentinel */ }
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, atmel_sha204a_id);
 
@@ -222,7 +227,7 @@ static struct i2c_driver atmel_sha204a_driver = {
 	.id_table		= atmel_sha204a_id,
 
 	.driver.name		= "atmel-sha204a",
-	.driver.of_match_table	= of_match_ptr(atmel_sha204a_dt_ids),
+	.driver.of_match_table	= atmel_sha204a_dt_ids,
 };
 
 static int __init atmel_sha204a_init(void)
diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c
index a4793b76300cf..2bf8d71c86f42 100644
--- a/drivers/crypto/axis/artpec6_crypto.c
+++ b/drivers/crypto/axis/artpec6_crypto.c
@@ -706,22 +706,19 @@ artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common,
 			       void *dst, unsigned int len, bool eop,
 			       bool use_short)
 {
-	if (use_short && len < 7) {
+	dma_addr_t dma_addr;
+	int ret;
+
+	if (use_short && len < 7)
 		return artpec6_crypto_setup_out_descr_short(common, dst, len,
 							    eop);
-	} else {
-		int ret;
-		dma_addr_t dma_addr;
 
-		ret = artpec6_crypto_dma_map_single(common, dst, len,
-						   DMA_TO_DEVICE,
-						   &dma_addr);
-		if (ret)
-			return ret;
+	ret = artpec6_crypto_dma_map_single(common, dst, len, DMA_TO_DEVICE,
+					    &dma_addr);
+	if (ret)
+		return ret;
 
-		return artpec6_crypto_setup_out_descr_phys(common, dma_addr,
-							   len, eop);
-	}
+	return artpec6_crypto_setup_out_descr_phys(common, dma_addr, len, eop);
 }
 
 /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a
diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c
index 32a6e6e15ee25..ddbd60cf3b3c1 100644
--- a/drivers/crypto/caam/caamalg.c
+++ b/drivers/crypto/caam/caamalg.c
@@ -603,7 +603,7 @@ static int aead_setkey(struct crypto_aead *aead,
 	dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
 	       keys.authkeylen + keys.enckeylen, keys.enckeylen,
 	       keys.authkeylen);
-	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @"__stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	/*
@@ -639,7 +639,7 @@ static int aead_setkey(struct crypto_aead *aead,
 	dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
 				   keys.enckeylen, ctx->dir);
 
-	print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
+	print_hex_dump_devel("ctx.key@"__stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 			     ctx->adata.keylen_pad + keys.enckeylen, 1);
 
@@ -680,7 +680,7 @@ static int gcm_setkey(struct crypto_aead *aead,
 	if (err)
 		return err;
 
-	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @"__stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	memcpy(ctx->key, key, keylen);
@@ -701,7 +701,7 @@ static int rfc4106_setkey(struct crypto_aead *aead,
 	if (err)
 		return err;
 
-	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @"__stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	memcpy(ctx->key, key, keylen);
@@ -727,7 +727,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
 	if (err)
 		return err;
 
-	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @"__stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	memcpy(ctx->key, key, keylen);
@@ -754,7 +754,7 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 	u32 *desc;
 	const bool is_rfc3686 = alg->caam.rfc3686;
 
-	print_hex_dump_debug("key in @"__stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @"__stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	/* Here keylen is actual key length */
diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c
index 65f6adb6c673f..aa779caacfe59 100644
--- a/drivers/crypto/caam/caamalg_qi.c
+++ b/drivers/crypto/caam/caamalg_qi.c
@@ -212,7 +212,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 	dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n",
 		keys.authkeylen + keys.enckeylen, keys.enckeylen,
 		keys.authkeylen);
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	/*
@@ -248,7 +248,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 				   ctx->adata.keylen_pad + keys.enckeylen,
 				   ctx->dir);
 
-	print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
+	print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 			     ctx->adata.keylen_pad + keys.enckeylen, 1);
 
@@ -371,7 +371,7 @@ static int gcm_setkey(struct crypto_aead *aead,
 	if (ret)
 		return ret;
 
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	memcpy(ctx->key, key, keylen);
@@ -475,7 +475,7 @@ static int rfc4106_setkey(struct crypto_aead *aead,
 	if (ret)
 		return ret;
 
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	memcpy(ctx->key, key, keylen);
@@ -581,7 +581,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
 	if (ret)
 		return ret;
 
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	memcpy(ctx->key, key, keylen);
@@ -631,7 +631,7 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 	const bool is_rfc3686 = alg->caam.rfc3686;
 	int ret = 0;
 
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	ctx->cdata.keylen = keylen;
diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c
index bf10c3dda745a..6b47bcc16a506 100644
--- a/drivers/crypto/caam/caamalg_qi2.c
+++ b/drivers/crypto/caam/caamalg_qi2.c
@@ -301,7 +301,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 	dev_dbg(dev, "keylen %d enckeylen %d authkeylen %d\n",
 		keys.authkeylen + keys.enckeylen, keys.enckeylen,
 		keys.authkeylen);
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	ctx->adata.keylen = keys.authkeylen;
@@ -315,7 +315,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key,
 	memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
 	dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad +
 				   keys.enckeylen, ctx->dir);
-	print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
+	print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
 			     ctx->adata.keylen_pad + keys.enckeylen, 1);
 
@@ -732,7 +732,7 @@ static int gcm_setkey(struct crypto_aead *aead,
 	ret = aes_check_keylen(keylen);
 	if (ret)
 		return ret;
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	memcpy(ctx->key, key, keylen);
@@ -828,7 +828,7 @@ static int rfc4106_setkey(struct crypto_aead *aead,
 	if (ret)
 		return ret;
 
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	memcpy(ctx->key, key, keylen);
@@ -927,7 +927,7 @@ static int rfc4543_setkey(struct crypto_aead *aead,
 	if (ret)
 		return ret;
 
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	memcpy(ctx->key, key, keylen);
@@ -955,7 +955,7 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key,
 	u32 *desc;
 	const bool is_rfc3686 = alg->caam.rfc3686;
 
-	print_hex_dump_debug("key in @" __stringify(__LINE__)": ",
+	print_hex_dump_devel("key in @" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	ctx->cdata.keylen = keylen;
diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c
index ddb2a35aec2d5..3cd71810380f6 100644
--- a/drivers/crypto/caam/caamhash.c
+++ b/drivers/crypto/caam/caamhash.c
@@ -505,7 +505,7 @@ static int axcbc_setkey(struct crypto_ahash *ahash, const u8 *key,
 				   DMA_TO_DEVICE);
 	ctx->adata.keylen = keylen;
 
-	print_hex_dump_debug("axcbc ctx.key@" __stringify(__LINE__)" : ",
+	print_hex_dump_devel("axcbc ctx.key@" __stringify(__LINE__)" : ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, keylen, 1);
 
 	return axcbc_set_sh_desc(ahash);
@@ -525,7 +525,7 @@ static int acmac_setkey(struct crypto_ahash *ahash, const u8 *key,
 	ctx->adata.key_virt = key;
 	ctx->adata.keylen = keylen;
 
-	print_hex_dump_debug("acmac ctx.key@" __stringify(__LINE__)" : ",
+	print_hex_dump_devel("acmac ctx.key@" __stringify(__LINE__)" : ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
 
 	return acmac_set_sh_desc(ahash);
diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c
index 88cc4fe2a585d..de2fcc387477d 100644
--- a/drivers/crypto/caam/key_gen.c
+++ b/drivers/crypto/caam/key_gen.c
@@ -58,7 +58,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
 
 	dev_dbg(jrdev, "split keylen %d split keylen padded %d\n",
 		adata->keylen, adata->keylen_pad);
-	print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ",
+	print_hex_dump_devel("ctx.key@" __stringify(__LINE__)": ",
 			     DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1);
 
 	if (local_max > max_keylen)
@@ -113,7 +113,7 @@ int gen_split_key(struct device *jrdev, u8 *key_out,
 		wait_for_completion(&result.completion);
 		ret = result.err;
 
-		print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ",
+		print_hex_dump_devel("ctx.key@"__stringify(__LINE__)": ",
 				     DUMP_PREFIX_ADDRESS, 16, 4, key_out,
 				     adata->keylen_pad, 1);
 	}
diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c
index 54de869e5374c..9358c1c041d4e 100644
--- a/drivers/crypto/cavium/cpt/cptpf_main.c
+++ b/drivers/crypto/cavium/cpt/cptpf_main.c
@@ -651,6 +651,7 @@ static const struct pci_device_id cpt_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CPT_81XX_PCI_PF_DEVICE_ID) },
 	{ 0, }  /* end of table */
 };
+MODULE_DEVICE_TABLE(pci, cpt_id_table);
 
 static struct pci_driver cpt_pci_driver = {
 	.name = DRV_NAME,
@@ -666,4 +667,3 @@ MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
 MODULE_DESCRIPTION("Cavium Thunder CPT Physical Function Driver");
 MODULE_LICENSE("GPL v2");
 MODULE_VERSION(DRV_VERSION);
-MODULE_DEVICE_TABLE(pci, cpt_id_table);
diff --git a/drivers/crypto/cavium/cpt/cptvf_main.c b/drivers/crypto/cavium/cpt/cptvf_main.c
index 2c9a2af388765..db76df14c4a00 100644
--- a/drivers/crypto/cavium/cpt/cptvf_main.c
+++ b/drivers/crypto/cavium/cpt/cptvf_main.c
@@ -835,9 +835,10 @@ static void cptvf_shutdown(struct pci_dev *pdev)
 
 /* Supported devices */
 static const struct pci_device_id cptvf_id_table[] = {
-	{PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID), 0},
-	{ 0, }  /* end of table */
+	{ PCI_VDEVICE(CAVIUM, CPT_81XX_PCI_VF_DEVICE_ID) },
+	{ }  /* end of table */
 };
+MODULE_DEVICE_TABLE(pci, cptvf_id_table);
 
 static struct pci_driver cptvf_pci_driver = {
 	.name = DRV_NAME,
@@ -853,4 +854,3 @@ MODULE_AUTHOR("George Cherian <george.cherian@cavium.com>");
 MODULE_DESCRIPTION("Cavium Thunder CPT Virtual Function Driver");
 MODULE_LICENSE("GPL v2");
 MODULE_VERSION(DRV_VERSION);
-MODULE_DEVICE_TABLE(pci, cptvf_id_table);
diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c
index 8664d97261feb..e474c84d8d381 100644
--- a/drivers/crypto/cavium/nitrox/nitrox_main.c
+++ b/drivers/crypto/cavium/nitrox/nitrox_main.c
@@ -38,9 +38,9 @@ static unsigned int num_devices;
  * nitrox_pci_tbl - PCI Device ID Table
  */
 static const struct pci_device_id nitrox_pci_tbl[] = {
-	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
+	{ PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID) },
 	/* required last entry */
-	{0, }
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
 
diff --git a/drivers/crypto/ccp/psp-dev.c b/drivers/crypto/ccp/psp-dev.c
index 5c7f7e02a7d8a..b14ce51065d5d 100644
--- a/drivers/crypto/ccp/psp-dev.c
+++ b/drivers/crypto/ccp/psp-dev.c
@@ -316,15 +316,15 @@ void psp_dev_destroy(struct sp_device *sp)
 	if (!psp)
 		return;
 
-	sev_dev_destroy(psp);
+	dbc_dev_destroy(psp);
 
-	tee_dev_destroy(psp);
+	platform_access_dev_destroy(psp);
 
 	sfs_dev_destroy(psp);
 
-	dbc_dev_destroy(psp);
+	tee_dev_destroy(psp);
 
-	platform_access_dev_destroy(psp);
+	sev_dev_destroy(psp);
 
 	sp_free_psp_irq(sp, psp);
 
diff --git a/drivers/crypto/ccp/sev-dev-tsm.c b/drivers/crypto/ccp/sev-dev-tsm.c
index b07ae529b5910..f303d8f55991b 100644
--- a/drivers/crypto/ccp/sev-dev-tsm.c
+++ b/drivers/crypto/ccp/sev-dev-tsm.c
@@ -248,12 +248,19 @@ static void dsm_remove(struct pci_tsm *tsm)
 static int dsm_create(struct tio_dsm *dsm)
 {
 	struct pci_dev *pdev = dsm->tsm.base_tsm.pdev;
-	u8 segment_id = pdev->bus ? pci_domain_nr(pdev->bus) : 0;
-	struct pci_dev *rootport = pcie_find_root_port(pdev);
-	u16 device_id = pci_dev_id(pdev);
+	struct pci_dev *rootport;
+	u8 segment_id;
+	u16 device_id;
 	u16 root_port_id;
 	u32 lnkcap = 0;
 
+	if (!pdev->bus)
+		return -ENODEV;
+
+	segment_id = pci_domain_nr(pdev->bus);
+	rootport = pcie_find_root_port(pdev);
+	device_id = pci_dev_id(pdev);
+
 	if (pci_read_config_dword(rootport, pci_pcie_cap(rootport) + PCI_EXP_LNKCAP,
 				  &lnkcap))
 		return -ENODEV;
diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c
index d1e9e0ac63b60..ca473ca198b81 100644
--- a/drivers/crypto/ccp/sev-dev.c
+++ b/drivers/crypto/ccp/sev-dev.c
@@ -1328,10 +1328,11 @@ static int snp_filter_reserved_mem_regions(struct resource *rs, void *arg)
 	size_t size;
 
 	/*
-	 * Ensure the list of HV_FIXED pages that will be passed to firmware
-	 * do not exceed the page-sized argument buffer.
+	 * Ensure the list of HV_FIXED pages passed to the firmware including
+	 * the one about to be written to do not exceed the page-sized argument
+	 * buffer.
 	 */
-	if ((range_list->num_elements * sizeof(struct sev_data_range) +
+	if (((range_list->num_elements + 1) * sizeof(struct sev_data_range) +
 	     sizeof(struct sev_data_range_list)) > PAGE_SIZE)
 		return -E2BIG;
 
@@ -1355,7 +1356,7 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
 {
 	struct sev_data_range_list *snp_range_list __free(kfree) = NULL;
 	struct psp_device *psp = psp_master;
-	struct sev_data_snp_init_ex data;
+	struct sev_data_snp_init_ex data = {};
 	struct sev_device *sev;
 	void *arg = &data;
 	int cmd, rc = 0;
@@ -1374,7 +1375,9 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
 		return -EOPNOTSUPP;
 	}
 
-	snp_prepare();
+	rc = snp_prepare();
+	if (rc)
+		return rc;
 
 	/*
 	 * Starting in SNP firmware v1.52, the SNP_INIT_EX command takes a list
@@ -1419,8 +1422,6 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
 		 */
 		snp_add_hv_fixed_pages(sev, snp_range_list);
 
-		memset(&data, 0, sizeof(data));
-
 		if (max_snp_asid) {
 			data.ciphertext_hiding_en = 1;
 			data.max_snp_asid = max_snp_asid;
@@ -1487,6 +1488,8 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
 				       &snp_panic_notifier);
 
 	if (data.tio_en) {
+		struct page *page;
+
 		/*
 		 * This executes with the sev_cmd_mutex held so down the stack
 		 * snp_reclaim_pages(locked=false) might be needed (which is extremely
@@ -1494,12 +1497,14 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid)
 		 * Instead of exporting __snp_alloc_firmware_pages(), allocate a page
 		 * for this one call here.
 		 */
-		void *tio_status = page_address(__snp_alloc_firmware_pages(
-			GFP_KERNEL_ACCOUNT | __GFP_ZERO, 0, true));
+		page = __snp_alloc_firmware_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO,
+						  0, true);
+		if (page) {
+			void *tio_status = page_address(page);
 
-		if (tio_status) {
 			sev_tsm_init_locked(sev, tio_status);
-			__snp_free_firmware_pages(virt_to_page(tio_status), 0, true);
+
+			__snp_free_firmware_pages(page, 0, true);
 		}
 	}
 
@@ -1716,29 +1721,11 @@ static int sev_get_platform_state(int *state, int *error)
 
 static int sev_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required)
 {
-	struct sev_platform_init_args init_args = {0};
 	int rc;
 
-	rc = _sev_platform_init_locked(&init_args);
-	if (rc) {
-		argp->error = SEV_RET_INVALID_PLATFORM_STATE;
-		return rc;
-	}
-
-	*shutdown_required = true;
-
-	return 0;
-}
-
-static int snp_move_to_init_state(struct sev_issue_cmd *argp, bool *shutdown_required)
-{
-	int error, rc;
-
-	rc = __sev_snp_init_locked(&error, 0);
-	if (rc) {
-		argp->error = SEV_RET_INVALID_PLATFORM_STATE;
+	rc = __sev_platform_init_locked(&argp->error);
+	if (rc)
 		return rc;
-	}
 
 	*shutdown_required = true;
 
@@ -2301,7 +2288,8 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable)
 	/* Userspace wants to query the certificate length. */
 	if (!input.pdh_cert_address ||
 	    !input.pdh_cert_len ||
-	    !input.cert_chain_address)
+	    !input.cert_chain_address ||
+	    !input.cert_chain_len)
 		goto cmd;
 
 	/* Allocate a physically contiguous buffer to store the PDH blob. */
@@ -2381,7 +2369,8 @@ static int sev_ioctl_do_pdh_export(struct sev_issue_cmd *argp, bool writable)
 	return ret;
 }
 
-static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
+static int __sev_do_snp_platform_status(struct sev_user_data_snp_status *status,
+					int *error)
 {
 	struct sev_device *sev = psp_master->sev_data;
 	struct sev_data_snp_addr buf;
@@ -2389,9 +2378,6 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
 	void *data;
 	int ret;
 
-	if (!argp->data)
-		return -EINVAL;
-
 	status_page = alloc_page(GFP_KERNEL_ACCOUNT);
 	if (!status_page)
 		return -ENOMEM;
@@ -2414,7 +2400,7 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
 	}
 
 	buf.address = __psp_pa(data);
-	ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error);
+	ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, error);
 
 	if (sev->snp_initialized) {
 		/*
@@ -2429,35 +2415,41 @@ static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
 	if (ret)
 		goto cleanup;
 
-	if (copy_to_user((void __user *)argp->data, data,
-			 sizeof(struct sev_user_data_snp_status)))
-		ret = -EFAULT;
+	memcpy(status, data, sizeof(*status));
 
 cleanup:
 	__free_pages(status_page, 0);
 	return ret;
 }
 
+static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp)
+{
+	struct sev_user_data_snp_status status;
+	int ret;
+
+	if (!argp->data)
+		return -EINVAL;
+
+	ret = __sev_do_snp_platform_status(&status, &argp->error);
+	if (ret < 0)
+		return ret;
+
+	if (copy_to_user((void __user *)argp->data, &status,
+			 sizeof(struct sev_user_data_snp_status)))
+		ret = -EFAULT;
+
+	return ret;
+}
+
 static int sev_ioctl_do_snp_commit(struct sev_issue_cmd *argp)
 {
-	struct sev_device *sev = psp_master->sev_data;
 	struct sev_data_snp_commit buf;
-	bool shutdown_required = false;
-	int ret, error;
-
-	if (!sev->snp_initialized) {
-		ret = snp_move_to_init_state(argp, &shutdown_required);
-		if (ret)
-			return ret;
-	}
+	int ret;
 
 	buf.len = sizeof(buf);
 
 	ret = __sev_do_cmd_locked(SEV_CMD_SNP_COMMIT, &buf, &argp->error);
 
-	if (shutdown_required)
-		__sev_snp_shutdown_locked(&error, false);
-
 	return ret;
 }
 
@@ -2465,8 +2457,6 @@ static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable
 {
 	struct sev_device *sev = psp_master->sev_data;
 	struct sev_user_data_snp_config config;
-	bool shutdown_required = false;
-	int ret, error;
 
 	if (!argp->data)
 		return -EINVAL;
@@ -2474,30 +2464,21 @@ static int sev_ioctl_do_snp_set_config(struct sev_issue_cmd *argp, bool writable
 	if (!writable)
 		return -EPERM;
 
+	if (!sev->snp_initialized)
+		return -ENODEV;
+
 	if (copy_from_user(&config, (void __user *)argp->data, sizeof(config)))
 		return -EFAULT;
 
-	if (!sev->snp_initialized) {
-		ret = snp_move_to_init_state(argp, &shutdown_required);
-		if (ret)
-			return ret;
-	}
-
-	ret = __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error);
-
-	if (shutdown_required)
-		__sev_snp_shutdown_locked(&error, false);
-
-	return ret;
+	return __sev_do_cmd_locked(SEV_CMD_SNP_CONFIG, &config, &argp->error);
 }
 
 static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable)
 {
 	struct sev_device *sev = psp_master->sev_data;
 	struct sev_user_data_snp_vlek_load input;
-	bool shutdown_required = false;
-	int ret, error;
 	void *blob;
+	int ret;
 
 	if (!argp->data)
 		return -EINVAL;
@@ -2505,6 +2486,9 @@ static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable)
 	if (!writable)
 		return -EPERM;
 
+	if (!sev->snp_initialized)
+		return -ENODEV;
+
 	if (copy_from_user(&input, u64_to_user_ptr(argp->data), sizeof(input)))
 		return -EFAULT;
 
@@ -2518,18 +2502,7 @@ static int sev_ioctl_do_snp_vlek_load(struct sev_issue_cmd *argp, bool writable)
 
 	input.vlek_wrapped_address = __psp_pa(blob);
 
-	if (!sev->snp_initialized) {
-		ret = snp_move_to_init_state(argp, &shutdown_required);
-		if (ret)
-			goto cleanup;
-	}
-
 	ret = __sev_do_cmd_locked(SEV_CMD_SNP_VLEK_LOAD, &input, &argp->error);
-
-	if (shutdown_required)
-		__sev_snp_shutdown_locked(&error, false);
-
-cleanup:
 	kfree(blob);
 
 	return ret;
@@ -2939,3 +2912,73 @@ void sev_pci_exit(void)
 
 	sev_firmware_shutdown(sev);
 }
+
+static int get_v1_svn(struct sev_device *sev)
+{
+	struct sev_snp_tcb_version_genoa_milan *tcb;
+	struct sev_user_data_snp_status status;
+	int ret, error = 0;
+
+	mutex_lock(&sev_cmd_mutex);
+	ret = __sev_do_snp_platform_status(&status, &error);
+	mutex_unlock(&sev_cmd_mutex);
+	if (ret < 0)
+		return ret;
+
+	tcb = (struct sev_snp_tcb_version_genoa_milan *)&status
+		      .current_tcb_version;
+	return tcb->snp;
+}
+
+static int get_v2_svn(struct sev_device *sev)
+{
+	struct sev_user_data_snp_status status;
+	struct sev_snp_tcb_version_turin *tcb;
+	int ret, error = 0;
+
+	mutex_lock(&sev_cmd_mutex);
+	ret = __sev_do_snp_platform_status(&status, &error);
+	mutex_unlock(&sev_cmd_mutex);
+	if (ret < 0)
+		return ret;
+
+	tcb = (struct sev_snp_tcb_version_turin *)&status
+		      .current_tcb_version;
+	return tcb->snp;
+}
+
+static bool sev_firmware_allows_es(struct sev_device *sev)
+{
+	/* Documented in AMD-SB-3023 */
+	if (boot_cpu_has(X86_FEATURE_ZEN4) || boot_cpu_has(X86_FEATURE_ZEN3))
+		return get_v1_svn(sev) < 0x1b;
+	else if (boot_cpu_has(X86_FEATURE_ZEN5))
+		return get_v2_svn(sev) < 0x4;
+	else
+		return true;
+}
+
+int sev_firmware_supported_vm_types(void)
+{
+	int supported_vm_types = 0;
+	struct sev_device *sev;
+
+	if (!psp_master || !psp_master->sev_data)
+		return supported_vm_types;
+	sev = psp_master->sev_data;
+
+	supported_vm_types |= BIT(KVM_X86_SEV_VM);
+	supported_vm_types |= BIT(KVM_X86_SEV_ES_VM);
+
+	if (!sev->snp_initialized)
+		return supported_vm_types;
+
+	supported_vm_types |= BIT(KVM_X86_SNP_VM);
+
+	if (!sev_firmware_allows_es(sev))
+		supported_vm_types &= ~BIT(KVM_X86_SEV_ES_VM);
+
+	return supported_vm_types;
+
+}
+EXPORT_SYMBOL_FOR_MODULES(sev_firmware_supported_vm_types, "kvm-amd");
diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c
index 6ac805d99ccb3..ede6ff9ad0c24 100644
--- a/drivers/crypto/ccp/sp-pci.c
+++ b/drivers/crypto/ccp/sp-pci.c
@@ -552,21 +552,21 @@ static const struct sp_dev_vdata dev_vdata[] = {
 
 };
 static const struct pci_device_id sp_pci_table[] = {
-	{ PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] },
-	{ PCI_VDEVICE(AMD, 0x1456), (kernel_ulong_t)&dev_vdata[1] },
-	{ PCI_VDEVICE(AMD, 0x1468), (kernel_ulong_t)&dev_vdata[2] },
-	{ PCI_VDEVICE(AMD, 0x1486), (kernel_ulong_t)&dev_vdata[3] },
-	{ PCI_VDEVICE(AMD, 0x15DF), (kernel_ulong_t)&dev_vdata[4] },
-	{ PCI_VDEVICE(AMD, 0x14CA), (kernel_ulong_t)&dev_vdata[5] },
-	{ PCI_VDEVICE(AMD, 0x15C7), (kernel_ulong_t)&dev_vdata[6] },
-	{ PCI_VDEVICE(AMD, 0x1649), (kernel_ulong_t)&dev_vdata[6] },
-	{ PCI_VDEVICE(AMD, 0x1134), (kernel_ulong_t)&dev_vdata[7] },
-	{ PCI_VDEVICE(AMD, 0x17E0), (kernel_ulong_t)&dev_vdata[7] },
-	{ PCI_VDEVICE(AMD, 0x156E), (kernel_ulong_t)&dev_vdata[8] },
-	{ PCI_VDEVICE(AMD, 0x17D8), (kernel_ulong_t)&dev_vdata[8] },
-	{ PCI_VDEVICE(AMD, 0x115A), (kernel_ulong_t)&dev_vdata[9] },
+	{ PCI_VDEVICE(AMD, 0x1537), .driver_data = (kernel_ulong_t)&dev_vdata[0] },
+	{ PCI_VDEVICE(AMD, 0x1456), .driver_data = (kernel_ulong_t)&dev_vdata[1] },
+	{ PCI_VDEVICE(AMD, 0x1468), .driver_data = (kernel_ulong_t)&dev_vdata[2] },
+	{ PCI_VDEVICE(AMD, 0x1486), .driver_data = (kernel_ulong_t)&dev_vdata[3] },
+	{ PCI_VDEVICE(AMD, 0x15DF), .driver_data = (kernel_ulong_t)&dev_vdata[4] },
+	{ PCI_VDEVICE(AMD, 0x14CA), .driver_data = (kernel_ulong_t)&dev_vdata[5] },
+	{ PCI_VDEVICE(AMD, 0x15C7), .driver_data = (kernel_ulong_t)&dev_vdata[6] },
+	{ PCI_VDEVICE(AMD, 0x1649), .driver_data = (kernel_ulong_t)&dev_vdata[6] },
+	{ PCI_VDEVICE(AMD, 0x1134), .driver_data = (kernel_ulong_t)&dev_vdata[7] },
+	{ PCI_VDEVICE(AMD, 0x17E0), .driver_data = (kernel_ulong_t)&dev_vdata[7] },
+	{ PCI_VDEVICE(AMD, 0x156E), .driver_data = (kernel_ulong_t)&dev_vdata[8] },
+	{ PCI_VDEVICE(AMD, 0x17D8), .driver_data = (kernel_ulong_t)&dev_vdata[8] },
+	{ PCI_VDEVICE(AMD, 0x115A), .driver_data = (kernel_ulong_t)&dev_vdata[9] },
 	/* Last entry must be zero */
-	{ 0, }
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, sp_pci_table);
 
diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c
index 81533681f7fb5..088c4603047f0 100644
--- a/drivers/crypto/ccree/cc_aead.c
+++ b/drivers/crypto/ccree/cc_aead.c
@@ -3,11 +3,12 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/rtnetlink.h>
+#include <linux/string.h>
 #include <crypto/algapi.h>
 #include <crypto/internal/aead.h>
 #include <crypto/authenc.h>
 #include <crypto/gcm.h>
-#include <linux/rtnetlink.h>
 #include <crypto/internal/des.h>
 #include "cc_driver.h"
 #include "cc_buffer_mgr.h"
@@ -2569,11 +2570,9 @@ static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
 
 	alg = &tmpl->template_aead;
 
-	if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s",
-		     tmpl->name) >= CRYPTO_MAX_ALG_NAME)
+	if (strscpy(alg->base.cra_name, tmpl->name) < 0)
 		return ERR_PTR(-EINVAL);
-	if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-		     tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME)
+	if (strscpy(alg->base.cra_driver_name, tmpl->driver_name) < 0)
 		return ERR_PTR(-EINVAL);
 
 	alg->base.cra_module = THIS_MODULE;
diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c
index e2cbfdf7a0e46..5339b3796c80e 100644
--- a/drivers/crypto/ccree/cc_cipher.c
+++ b/drivers/crypto/ccree/cc_cipher.c
@@ -3,6 +3,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/string.h>
 #include <crypto/algapi.h>
 #include <crypto/internal/skcipher.h>
 #include <crypto/internal/des.h>
@@ -1386,11 +1387,9 @@ static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl,
 
 	memcpy(alg, &tmpl->template_skcipher, sizeof(*alg));
 
-	if (snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s",
-		     tmpl->name) >= CRYPTO_MAX_ALG_NAME)
+	if (strscpy(alg->base.cra_name, tmpl->name) < 0)
 		return ERR_PTR(-EINVAL);
-	if (snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-		     tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME)
+	if (strscpy(alg->base.cra_driver_name, tmpl->driver_name) < 0)
 		return ERR_PTR(-EINVAL);
 
 	alg->base.cra_module = THIS_MODULE;
diff --git a/drivers/crypto/ccree/cc_hash.c b/drivers/crypto/ccree/cc_hash.c
index 73179bf725a71..5de721a2ca30a 100644
--- a/drivers/crypto/ccree/cc_hash.c
+++ b/drivers/crypto/ccree/cc_hash.c
@@ -3,6 +3,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/string.h>
 #include <crypto/algapi.h>
 #include <crypto/hash.h>
 #include <crypto/md5.h>
@@ -1835,16 +1836,12 @@ static struct cc_hash_alg *cc_alloc_hash_alg(struct cc_hash_template *template,
 	alg = &halg->halg.base;
 
 	if (keyed) {
-		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->mac_name);
-		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->mac_driver_name);
+		strscpy(alg->cra_name, template->mac_name);
+		strscpy(alg->cra_driver_name, template->mac_driver_name);
 	} else {
 		halg->setkey = NULL;
-		snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->name);
-		snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
-			 template->driver_name);
+		strscpy(alg->cra_name, template->name);
+		strscpy(alg->cra_driver_name, template->driver_name);
 	}
 	alg->cra_module = THIS_MODULE;
 	alg->cra_ctxsize = sizeof(struct cc_hash_ctx) + crypto_dma_padding();
diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c
index fb4936e7afa2d..52809e57361aa 100644
--- a/drivers/crypto/inside-secure/safexcel.c
+++ b/drivers/crypto/inside-secure/safexcel.c
@@ -1475,7 +1475,7 @@ static int safexcel_probe_generic(void *pdev,
 	peid = version & 255;
 
 	/* Detect EIP206 processing pipe */
-	version = readl(EIP197_PE(priv) + + EIP197_PE_VERSION(0));
+	version = readl(EIP197_PE(priv) + EIP197_PE_VERSION(0));
 	if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) {
 		dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid);
 		return -ENODEV;
@@ -1893,7 +1893,7 @@ static int safexcel_pci_probe(struct pci_dev *pdev,
 		ent->vendor, ent->device, ent->subvendor,
 		ent->subdevice, ent->driver_data);
 
-	priv = kzalloc_obj(*priv);
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
diff --git a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c
index fcc0cf4df637d..5b90cf0fb0e41 100644
--- a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c
+++ b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c
@@ -884,8 +884,9 @@ static struct buffer_desc *chainup_buffers(struct device *dev,
 		ptr = sg_virt(sg);
 		next_buf = dma_pool_alloc(buffer_pool, flags, &next_buf_phys);
 		if (!next_buf) {
-			buf = NULL;
-			break;
+			buf->next = NULL;
+			buf->phys_next = 0;
+			return NULL;
 		}
 		sg_dma_address(sg) = dma_map_single(dev, ptr, len, dir);
 		buf->next = next_buf;
@@ -983,7 +984,7 @@ static int ablk_perform(struct skcipher_request *req, int encrypt)
 	unsigned int nbytes = req->cryptlen;
 	enum dma_data_direction src_direction = DMA_BIDIRECTIONAL;
 	struct ablk_ctx *req_ctx = skcipher_request_ctx(req);
-	struct buffer_desc src_hook;
+	struct buffer_desc *buf, src_hook;
 	struct device *dev = &pdev->dev;
 	unsigned int offset;
 	gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
@@ -1025,22 +1026,24 @@ static int ablk_perform(struct skcipher_request *req, int encrypt)
 		/* This was never tested by Intel
 		 * for more than one dst buffer, I think. */
 		req_ctx->dst = NULL;
-		if (!chainup_buffers(dev, req->dst, nbytes, &dst_hook,
-				     flags, DMA_FROM_DEVICE))
-			goto free_buf_dest;
-		src_direction = DMA_TO_DEVICE;
+		buf = chainup_buffers(dev, req->dst, nbytes, &dst_hook,
+				      flags, DMA_FROM_DEVICE);
 		req_ctx->dst = dst_hook.next;
 		crypt->dst_buf = dst_hook.phys_next;
+		if (!buf)
+			goto free_buf_dest;
+		src_direction = DMA_TO_DEVICE;
 	} else {
 		req_ctx->dst = NULL;
 	}
 	req_ctx->src = NULL;
-	if (!chainup_buffers(dev, req->src, nbytes, &src_hook, flags,
-			     src_direction))
-		goto free_buf_src;
-
+	buf = chainup_buffers(dev, req->src, nbytes, &src_hook, flags,
+			      src_direction);
 	req_ctx->src = src_hook.next;
 	crypt->src_buf = src_hook.phys_next;
+	if (!buf)
+		goto free_buf_src;
+
 	crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK;
 	qmgr_put_entry(send_qid, crypt_virt2phys(crypt));
 	BUG_ON(qmgr_stat_overflow(send_qid));
diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h
index 7b8b295ac4590..fb0fd46a79b00 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h
+++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h
@@ -92,7 +92,6 @@ void adf_exit_aer(void);
 int adf_init_arb(struct adf_accel_dev *accel_dev);
 void adf_exit_arb(struct adf_accel_dev *accel_dev);
 void adf_update_ring_arb(struct adf_etr_ring_data *ring);
-int adf_disable_arb_thd(struct adf_accel_dev *accel_dev, u32 ae, u32 thr);
 
 int adf_dev_get(struct adf_accel_dev *accel_dev);
 void adf_dev_put(struct adf_accel_dev *accel_dev);
diff --git a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c
index a3b474bdef6c8..023c5f1e78b07 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_heartbeat_inject.c
@@ -64,10 +64,8 @@ int adf_heartbeat_inject_error(struct adf_accel_dev *accel_dev)
 	if (ret)
 		return ret;
 
-	/* Configure worker threads to stop processing any packet */
-	ret = adf_disable_arb_thd(accel_dev, rand_ae, rand_thr);
-	if (ret)
-		return ret;
+	/* Disable arbiter to stop processing any packet */
+	hw_device->exit_arb(accel_dev);
 
 	/* Change HB counters memory to simulate a hang */
 	adf_set_hb_counters_fail(accel_dev, rand_ae, rand_thr);
diff --git a/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c
index f93d9cca70cee..dd9a31c20bc9c 100644
--- a/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c
+++ b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c
@@ -99,28 +99,3 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev)
 		csr_ops->write_csr_ring_srv_arb_en(csr, i, 0);
 }
 EXPORT_SYMBOL_GPL(adf_exit_arb);
-
-int adf_disable_arb_thd(struct adf_accel_dev *accel_dev, u32 ae, u32 thr)
-{
-	void __iomem *csr = accel_dev->transport->banks[0].csr_addr;
-	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
-	const u32 *thd_2_arb_cfg;
-	struct arb_info info;
-	u32 ae_thr_map;
-
-	if (ADF_AE_STRAND0_THREAD == thr || ADF_AE_STRAND1_THREAD == thr)
-		thr = ADF_AE_ADMIN_THREAD;
-
-	hw_data->get_arb_info(&info);
-	thd_2_arb_cfg = hw_data->get_arb_mapping(accel_dev);
-	if (!thd_2_arb_cfg)
-		return -EFAULT;
-
-	/* Disable scheduling for this particular AE and thread */
-	ae_thr_map = *(thd_2_arb_cfg + ae);
-	ae_thr_map &= ~(GENMASK(3, 0) << (thr * BIT(2)));
-
-	WRITE_CSR_ARB_WT2SAM(csr, info.arb_offset, info.wt2sam_offset, ae,
-			     ae_thr_map);
-	return 0;
-}
diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c
index 8afa3a87e38df..687ed730174d7 100644
--- a/drivers/crypto/marvell/cesa/cesa.c
+++ b/drivers/crypto/marvell/cesa/cesa.c
@@ -416,7 +416,7 @@ static int mv_cesa_probe(struct platform_device *pdev)
 	const struct mbus_dram_target_info *dram;
 	struct device *dev = &pdev->dev;
 	struct mv_cesa_dev *cesa;
-	struct mv_cesa_engine *engines;
+	struct mv_cesa_engine *engine;
 	int irq, ret, i, cpu;
 	u32 sram_size;
 
@@ -431,7 +431,8 @@ static int mv_cesa_probe(struct platform_device *pdev)
 			return -ENOTSUPP;
 	}
 
-	cesa = devm_kzalloc(dev, sizeof(*cesa), GFP_KERNEL);
+	cesa = devm_kzalloc(dev, struct_size(cesa, engines, caps->nengines),
+			GFP_KERNEL);
 	if (!cesa)
 		return -ENOMEM;
 
@@ -445,10 +446,6 @@ static int mv_cesa_probe(struct platform_device *pdev)
 		sram_size = CESA_SA_MIN_SRAM_SIZE;
 
 	cesa->sram_size = sram_size;
-	cesa->engines = devm_kcalloc(dev, caps->nengines, sizeof(*engines),
-				     GFP_KERNEL);
-	if (!cesa->engines)
-		return -ENOMEM;
 
 	spin_lock_init(&cesa->lock);
 
@@ -465,7 +462,7 @@ static int mv_cesa_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, cesa);
 
 	for (i = 0; i < caps->nengines; i++) {
-		struct mv_cesa_engine *engine = &cesa->engines[i];
+		engine = &cesa->engines[i];
 		char res_name[16];
 
 		engine->id = i;
diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h
index 50ca1039fdaa7..18f9f28040a61 100644
--- a/drivers/crypto/marvell/cesa/cesa.h
+++ b/drivers/crypto/marvell/cesa/cesa.h
@@ -402,27 +402,6 @@ struct mv_cesa_dev_dma {
 	struct dma_pool *padding_pool;
 };
 
-/**
- * struct mv_cesa_dev - CESA device
- * @caps:	device capabilities
- * @regs:	device registers
- * @sram_size:	usable SRAM size
- * @lock:	device lock
- * @engines:	array of engines
- * @dma:	dma pools
- *
- * Structure storing CESA device information.
- */
-struct mv_cesa_dev {
-	const struct mv_cesa_caps *caps;
-	void __iomem *regs;
-	struct device *dev;
-	unsigned int sram_size;
-	spinlock_t lock;
-	struct mv_cesa_engine *engines;
-	struct mv_cesa_dev_dma *dma;
-};
-
 /**
  * struct mv_cesa_engine - CESA engine
  * @id:			engine id
@@ -471,6 +450,27 @@ struct mv_cesa_engine {
 	int irq;
 };
 
+/**
+ * struct mv_cesa_dev - CESA device
+ * @caps:	device capabilities
+ * @regs:	device registers
+ * @sram_size:	usable SRAM size
+ * @lock:	device lock
+ * @dma:	dma pools
+ * @engines:	array of engines
+ *
+ * Structure storing CESA device information.
+ */
+struct mv_cesa_dev {
+	const struct mv_cesa_caps *caps;
+	void __iomem *regs;
+	struct device *dev;
+	unsigned int sram_size;
+	spinlock_t lock;
+	struct mv_cesa_dev_dma *dma;
+	struct mv_cesa_engine engines[];
+};
+
 /**
  * struct mv_cesa_req_ops - CESA request operations
  * @process:	process a request chunk result (should return 0 if the
diff --git a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c
index 14a42559f81d6..e4c828606a737 100644
--- a/drivers/crypto/marvell/octeontx/otx_cptpf_main.c
+++ b/drivers/crypto/marvell/octeontx/otx_cptpf_main.c
@@ -283,6 +283,7 @@ static const struct pci_device_id otx_cpt_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OTX_CPT_PCI_PF_DEVICE_ID) },
 	{ 0, }  /* end of table */
 };
+MODULE_DEVICE_TABLE(pci, otx_cpt_id_table);
 
 static struct pci_driver otx_cpt_pci_driver = {
 	.name = DRV_NAME,
@@ -298,4 +299,3 @@ MODULE_AUTHOR("Marvell International Ltd.");
 MODULE_DESCRIPTION("Marvell OcteonTX CPT Physical Function Driver");
 MODULE_LICENSE("GPL v2");
 MODULE_VERSION(DRV_VERSION);
-MODULE_DEVICE_TABLE(pci, otx_cpt_id_table);
diff --git a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c
index 587609db6c69f..159c1d290a36b 100644
--- a/drivers/crypto/marvell/octeontx/otx_cptvf_main.c
+++ b/drivers/crypto/marvell/octeontx/otx_cptvf_main.c
@@ -957,9 +957,10 @@ static void otx_cptvf_remove(struct pci_dev *pdev)
 
 /* Supported devices */
 static const struct pci_device_id otx_cptvf_id_table[] = {
-	{PCI_VDEVICE(CAVIUM, OTX_CPT_PCI_VF_DEVICE_ID), 0},
-	{ 0, }  /* end of table */
+	{ PCI_VDEVICE(CAVIUM, OTX_CPT_PCI_VF_DEVICE_ID) },
+	{ }  /* end of table */
 };
+MODULE_DEVICE_TABLE(pci, otx_cptvf_id_table);
 
 static struct pci_driver otx_cptvf_pci_driver = {
 	.name = DRV_NAME,
@@ -974,4 +975,3 @@ MODULE_AUTHOR("Marvell International Ltd.");
 MODULE_DESCRIPTION("Marvell OcteonTX CPT Virtual Function Driver");
 MODULE_LICENSE("GPL v2");
 MODULE_VERSION(DRV_VERSION);
-MODULE_DEVICE_TABLE(pci, otx_cptvf_id_table);
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
index 346d1345f11c2..f6f47f4e5d830 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
@@ -867,6 +867,7 @@ static const struct pci_device_id otx2_cpt_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CN10K_CPT_PCI_PF_DEVICE_ID) },
 	{ 0, }  /* end of table */
 };
+MODULE_DEVICE_TABLE(pci, otx2_cpt_id_table);
 
 static struct pci_driver otx2_cpt_pci_driver = {
 	.name = OTX2_CPT_DRV_NAME,
@@ -883,4 +884,3 @@ MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT");
 MODULE_AUTHOR("Marvell");
 MODULE_DESCRIPTION(OTX2_CPT_DRV_STRING);
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(pci, otx2_cpt_id_table);
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c
index 858f851c9c8ad..09ab85e6061c4 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c
@@ -460,10 +460,11 @@ static void otx2_cptvf_remove(struct pci_dev *pdev)
 
 /* Supported devices */
 static const struct pci_device_id otx2_cptvf_id_table[] = {
-	{PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID), 0},
-	{PCI_VDEVICE(CAVIUM, CN10K_CPT_PCI_VF_DEVICE_ID), 0},
-	{ 0, }  /* end of table */
+	{ PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID) },
+	{ PCI_VDEVICE(CAVIUM, CN10K_CPT_PCI_VF_DEVICE_ID) },
+	{ }  /* end of table */
 };
+MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table);
 
 static struct pci_driver otx2_cptvf_pci_driver = {
 	.name = OTX2_CPTVF_DRV_NAME,
@@ -479,4 +480,3 @@ MODULE_IMPORT_NS("CRYPTO_DEV_OCTEONTX2_CPT");
 MODULE_AUTHOR("Marvell");
 MODULE_DESCRIPTION("Marvell RVU CPT Virtual Function Driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table);
diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c
index 3eadaf7a64fab..f31555c0d715b 100644
--- a/drivers/crypto/omap-aes.c
+++ b/drivers/crypto/omap-aes.c
@@ -1089,6 +1089,20 @@ static struct attribute *omap_aes_attrs[] = {
 };
 ATTRIBUTE_GROUPS(omap_aes);
 
+static void omap_aes_unregister_algs(const struct omap_aes_pdata *pdata)
+{
+	struct omap_aes_algs_info *alg_info;
+	int i;
+
+	for (i = pdata->algs_info_size - 1; i >= 0; i--) {
+		alg_info = &pdata->algs_info[i];
+
+		crypto_engine_unregister_skciphers(alg_info->algs_list,
+						   alg_info->registered);
+		alg_info->registered = 0;
+	}
+}
+
 static int omap_aes_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -1215,15 +1229,11 @@ static int omap_aes_probe(struct platform_device *pdev)
 
 	return 0;
 err_aead_algs:
-	for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
-		aalg = &dd->pdata->aead_algs_info->algs_list[i];
-		crypto_engine_unregister_aead(aalg);
-	}
+	crypto_engine_unregister_aeads(dd->pdata->aead_algs_info->algs_list,
+				       dd->pdata->aead_algs_info->registered);
+	dd->pdata->aead_algs_info->registered = 0;
 err_algs:
-	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
-		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
-			crypto_engine_unregister_skcipher(
-					&dd->pdata->algs_info[i].algs_list[j]);
+	omap_aes_unregister_algs(dd->pdata);
 
 err_engine:
 	if (dd->engine)
@@ -1244,25 +1254,16 @@ static int omap_aes_probe(struct platform_device *pdev)
 static void omap_aes_remove(struct platform_device *pdev)
 {
 	struct omap_aes_dev *dd = platform_get_drvdata(pdev);
-	struct aead_engine_alg *aalg;
-	int i, j;
 
 	spin_lock_bh(&list_lock);
 	list_del(&dd->list);
 	spin_unlock_bh(&list_lock);
 
-	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
-		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
-			crypto_engine_unregister_skcipher(
-					&dd->pdata->algs_info[i].algs_list[j]);
-			dd->pdata->algs_info[i].registered--;
-		}
+	omap_aes_unregister_algs(dd->pdata);
 
-	for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) {
-		aalg = &dd->pdata->aead_algs_info->algs_list[i];
-		crypto_engine_unregister_aead(aalg);
-		dd->pdata->aead_algs_info->registered--;
-	}
+	crypto_engine_unregister_aeads(dd->pdata->aead_algs_info->algs_list,
+				       dd->pdata->aead_algs_info->registered);
+	dd->pdata->aead_algs_info->registered = 0;
 
 	crypto_engine_exit(dd->engine);
 
diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c
index 149ebd77710bf..16d5c617d5ee8 100644
--- a/drivers/crypto/omap-des.c
+++ b/drivers/crypto/omap-des.c
@@ -938,6 +938,20 @@ static int omap_des_get_pdev(struct omap_des_dev *dd,
 	return 0;
 }
 
+static void omap_des_unregister_algs(const struct omap_des_pdata *pdata)
+{
+	struct omap_des_algs_info *alg_info;
+	int i;
+
+	for (i = pdata->algs_info_size - 1; i >= 0; i--) {
+		alg_info = &pdata->algs_info[i];
+
+		crypto_engine_unregister_skciphers(alg_info->algs_list,
+						   alg_info->registered);
+		alg_info->registered = 0;
+	}
+}
+
 static int omap_des_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -1043,11 +1057,7 @@ static int omap_des_probe(struct platform_device *pdev)
 	return 0;
 
 err_algs:
-	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
-		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
-			crypto_engine_unregister_skcipher(
-					&dd->pdata->algs_info[i].algs_list[j]);
-
+	omap_des_unregister_algs(dd->pdata);
 err_engine:
 	if (dd->engine)
 		crypto_engine_exit(dd->engine);
@@ -1067,16 +1077,12 @@ static int omap_des_probe(struct platform_device *pdev)
 static void omap_des_remove(struct platform_device *pdev)
 {
 	struct omap_des_dev *dd = platform_get_drvdata(pdev);
-	int i, j;
 
 	spin_lock_bh(&list_lock);
 	list_del(&dd->list);
 	spin_unlock_bh(&list_lock);
 
-	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
-		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
-			crypto_engine_unregister_skcipher(
-					&dd->pdata->algs_info[i].algs_list[j]);
+	omap_des_unregister_algs(dd->pdata);
 
 	cancel_work_sync(&dd->done_task);
 	omap_des_dma_cleanup(dd);
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index b8c416c5ee70e..be1ac640ee592 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -2042,6 +2042,20 @@ static struct attribute *omap_sham_attrs[] = {
 };
 ATTRIBUTE_GROUPS(omap_sham);
 
+static void omap_sham_unregister_algs(const struct omap_sham_pdata *pdata)
+{
+	struct omap_sham_algs_info *alg_info;
+	int i;
+
+	for (i = pdata->algs_info_size - 1; i >= 0; i--) {
+		alg_info = &pdata->algs_info[i];
+
+		crypto_engine_unregister_ahashes(alg_info->algs_list,
+						 alg_info->registered);
+		alg_info->registered = 0;
+	}
+}
+
 static int omap_sham_probe(struct platform_device *pdev)
 {
 	struct omap_sham_dev *dd;
@@ -2158,10 +2172,7 @@ static int omap_sham_probe(struct platform_device *pdev)
 	return 0;
 
 err_algs:
-	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
-		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
-			crypto_engine_unregister_ahash(
-					&dd->pdata->algs_info[i].algs_list[j]);
+	omap_sham_unregister_algs(dd->pdata);
 err_engine_start:
 	crypto_engine_exit(dd->engine);
 err_engine:
@@ -2182,19 +2193,13 @@ static int omap_sham_probe(struct platform_device *pdev)
 static void omap_sham_remove(struct platform_device *pdev)
 {
 	struct omap_sham_dev *dd;
-	int i, j;
 
 	dd = platform_get_drvdata(pdev);
 
 	spin_lock_bh(&sham.lock);
 	list_del(&dd->list);
 	spin_unlock_bh(&sham.lock);
-	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
-		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
-			crypto_engine_unregister_ahash(
-					&dd->pdata->algs_info[i].algs_list[j]);
-			dd->pdata->algs_info[i].registered--;
-		}
+	omap_sham_unregister_algs(dd->pdata);
 	cancel_work_sync(&dd->done_task);
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
diff --git a/drivers/crypto/starfive/jh7110-cryp.c b/drivers/crypto/starfive/jh7110-cryp.c
index 42114e9364f0d..e19cd79459687 100644
--- a/drivers/crypto/starfive/jh7110-cryp.c
+++ b/drivers/crypto/starfive/jh7110-cryp.c
@@ -36,19 +36,14 @@ static struct starfive_dev_list dev_list = {
 
 struct starfive_cryp_dev *starfive_cryp_find_dev(struct starfive_cryp_ctx *ctx)
 {
-	struct starfive_cryp_dev *cryp = NULL, *tmp;
+	struct starfive_cryp_dev *cryp;
 
 	spin_lock_bh(&dev_list.lock);
-	if (!ctx->cryp) {
-		list_for_each_entry(tmp, &dev_list.dev_list, list) {
-			cryp = tmp;
-			break;
-		}
-		ctx->cryp = cryp;
-	} else {
-		cryp = ctx->cryp;
-	}
-
+	if (!ctx->cryp)
+		ctx->cryp = list_first_entry_or_null(&dev_list.dev_list,
+						     struct starfive_cryp_dev,
+						     list);
+	cryp = ctx->cryp;
 	spin_unlock_bh(&dev_list.lock);
 
 	return cryp;
diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c
index bc61d0fe35140..584508963241c 100644
--- a/drivers/crypto/talitos.c
+++ b/drivers/crypto/talitos.c
@@ -12,7 +12,6 @@
  * All rights reserved.
  */
 
-#include <linux/workqueue.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
@@ -255,6 +254,46 @@ static int init_device(struct device *dev)
 	return 0;
 }
 
+static void dma_map_request(struct device *dev, struct talitos_request *request,
+			    struct talitos_desc *desc, bool is_sec1)
+{
+	struct talitos_edesc *edesc =
+		container_of(desc, struct talitos_edesc, desc);
+	dma_addr_t dma_desc, prev_dma_desc;
+	struct talitos_edesc *prev_edesc = NULL;
+
+	if (is_sec1) {
+		while (edesc) {
+			edesc->desc.hdr1 = edesc->desc.hdr;
+
+			dma_desc = dma_map_single(dev, &edesc->desc.hdr1,
+						  TALITOS_DESC_SIZE,
+						  DMA_BIDIRECTIONAL);
+
+			if (!prev_edesc) {
+				request->dma_desc = dma_desc;
+				goto next;
+			}
+
+			/* Chain in any previous descriptors. */
+
+			prev_edesc->desc.next_desc = cpu_to_be32(dma_desc);
+
+			dma_sync_single_for_device(dev, prev_dma_desc,
+						   TALITOS_DESC_SIZE,
+						   DMA_TO_DEVICE);
+
+next:
+			prev_edesc = edesc;
+			prev_dma_desc = dma_desc;
+			edesc = edesc->next_desc;
+		}
+	} else {
+		request->dma_desc = dma_map_single(dev, desc, TALITOS_DESC_SIZE,
+						   DMA_BIDIRECTIONAL);
+	}
+}
+
 /**
  * talitos_submit - submits a descriptor to the device for processing
  * @dev:	the SEC device to be used
@@ -291,16 +330,7 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 	request = &priv->chan[ch].fifo[head];
 
 	/* map descriptor and save caller data */
-	if (is_sec1) {
-		desc->hdr1 = desc->hdr;
-		request->dma_desc = dma_map_single(dev, &desc->hdr1,
-						   TALITOS_DESC_SIZE,
-						   DMA_BIDIRECTIONAL);
-	} else {
-		request->dma_desc = dma_map_single(dev, desc,
-						   TALITOS_DESC_SIZE,
-						   DMA_BIDIRECTIONAL);
-	}
+	dma_map_request(dev, request, desc, is_sec1);
 	request->callback = callback;
 	request->context = context;
 
@@ -322,19 +352,51 @@ static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
 	return -EINPROGRESS;
 }
 
-static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1)
+static __be32 get_request_hdr(struct device *dev,
+			      struct talitos_request *request, bool is_sec1)
 {
 	struct talitos_edesc *edesc;
+	dma_addr_t dma_desc;
 
-	if (!is_sec1)
-		return request->desc->hdr;
+	if (!is_sec1) {
+		dma_sync_single_for_cpu(dev, request->dma_desc,
+					TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
 
-	if (!request->desc->next_desc)
-		return request->desc->hdr1;
+		return request->desc->hdr;
+	}
 
 	edesc = container_of(request->desc, struct talitos_edesc, desc);
+	dma_desc = request->dma_desc;
+	while (edesc->next_desc) {
+		dma_desc = be32_to_cpu(edesc->desc.next_desc);
+		edesc = edesc->next_desc;
+	}
+
+	dma_sync_single_for_cpu(dev, dma_desc, TALITOS_DESC_SIZE,
+				DMA_BIDIRECTIONAL);
 
-	return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1;
+	return edesc->desc.hdr1;
+}
+
+static void dma_unmap_request(struct device *dev,
+			      struct talitos_request *request, bool is_sec1)
+{
+	struct talitos_edesc *edesc;
+
+	if (is_sec1) {
+		dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE,
+				 DMA_BIDIRECTIONAL);
+		edesc = container_of(request->desc, struct talitos_edesc, desc);
+		while (edesc->next_desc) {
+			dma_unmap_single(dev,
+					 be32_to_cpu(edesc->desc.next_desc),
+					 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
+			edesc = edesc->next_desc;
+		}
+	} else {
+		dma_unmap_single(dev, request->dma_desc, TALITOS_DESC_SIZE,
+				 DMA_BIDIRECTIONAL);
+	}
 }
 
 /*
@@ -358,7 +420,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 
 		/* descriptors with their done bits set don't get the error */
 		rmb();
-		hdr = get_request_hdr(request, is_sec1);
+		hdr = get_request_hdr(dev, request, is_sec1);
 
 		if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 			status = 0;
@@ -368,9 +430,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 			else
 				status = error;
 
-		dma_unmap_single(dev, request->dma_desc,
-				 TALITOS_DESC_SIZE,
-				 DMA_BIDIRECTIONAL);
+		dma_unmap_request(dev, request, is_sec1);
 
 		/* copy entries so we can call callback outside lock */
 		saved_req.desc = request->desc;
@@ -459,14 +519,35 @@ DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
 DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
 DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
 
+static __be32 search_desc_hdr_in_request(struct talitos_request *request,
+					 dma_addr_t cur_desc, bool is_sec1)
+{
+	struct talitos_edesc *edesc;
+
+	if (request->dma_desc == cur_desc) {
+		return request->desc->hdr;
+	} else if (is_sec1) {
+		edesc = container_of(request->desc, struct talitos_edesc, desc);
+		while (edesc->next_desc) {
+			if (edesc->desc.next_desc == cpu_to_be32(cur_desc))
+				return edesc->next_desc->desc.hdr1;
+			edesc = edesc->next_desc;
+		}
+	}
+	return 0;
+}
+
 /*
  * locate current (offending) descriptor
  */
 static __be32 current_desc_hdr(struct device *dev, int ch)
 {
 	struct talitos_private *priv = dev_get_drvdata(dev);
+	bool is_sec1 = has_ftr_sec1(priv);
+	struct talitos_request *request;
 	int tail, iter;
 	dma_addr_t cur_desc;
+	__be32 hdr = 0;
 
 	cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
 	cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
@@ -477,27 +558,21 @@ static __be32 current_desc_hdr(struct device *dev, int ch)
 	}
 
 	tail = priv->chan[ch].tail;
-
 	iter = tail;
-	while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
-	       priv->chan[ch].fifo[iter].desc->next_desc != cpu_to_be32(cur_desc)) {
-		iter = (iter + 1) & (priv->fifo_len - 1);
-		if (iter == tail) {
-			dev_err(dev, "couldn't locate current descriptor\n");
-			return 0;
-		}
-	}
+	do {
+		request = &priv->chan[ch].fifo[iter];
 
-	if (priv->chan[ch].fifo[iter].desc->next_desc == cpu_to_be32(cur_desc)) {
-		struct talitos_edesc *edesc;
+		hdr = search_desc_hdr_in_request(request, cur_desc, is_sec1);
+		if (hdr)
+			break;
 
-		edesc = container_of(priv->chan[ch].fifo[iter].desc,
-				     struct talitos_edesc, desc);
-		return ((struct talitos_desc *)
-			(edesc->buf + edesc->dma_len))->hdr;
-	}
+		iter = (iter + 1) & (priv->fifo_len - 1);
+	} while (iter != tail);
+
+	if (!hdr)
+		dev_err(dev, "couldn't locate current descriptor\n");
 
-	return priv->chan[ch].fifo[iter].desc->hdr;
+	return hdr;
 }
 
 /*
@@ -869,28 +944,20 @@ struct talitos_ahash_req_ctx {
 	u8 buf[2][HASH_MAX_BLOCK_SIZE];
 	int buf_idx;
 	unsigned int swinit;
-	unsigned int first_desc;
-	unsigned int last_desc;
+	unsigned int first_request;
 	unsigned int last_request;
 	unsigned int to_hash_later;
 	unsigned int nbuf;
 	struct scatterlist bufsl[2];
 	struct scatterlist *psrc;
-
-	struct scatterlist request_bufsl[2];
-	struct ahash_request *areq;
-	struct scatterlist *request_sl;
-	unsigned int remaining_ahash_request_bytes;
-	unsigned int current_ahash_request_bytes;
-	struct work_struct sec1_ahash_process_remaining;
 };
 
 struct talitos_export_state {
 	u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
 	u8 buf[HASH_MAX_BLOCK_SIZE];
 	unsigned int swinit;
-	unsigned int first_desc;
-	unsigned int last_desc;
+	unsigned int first_request;
+	unsigned int last_request;
 	unsigned int to_hash_later;
 	unsigned int nbuf;
 };
@@ -1396,10 +1463,6 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
 		dma_len = 0;
 	}
 	alloc_len += icv_stashing ? authsize : 0;
-
-	/* if its a ahash, add space for a second desc next to the first one */
-	if (is_sec1 && !dst)
-		alloc_len += sizeof(struct talitos_desc);
 	alloc_len += ivsize;
 
 	edesc = kmalloc(ALIGN(alloc_len, dma_get_cache_alignment()), flags);
@@ -1415,6 +1478,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
 	edesc->dst_nents = dst_nents;
 	edesc->iv_dma = iv_dma;
 	edesc->dma_len = dma_len;
+	edesc->next_desc = NULL;
 	if (dma_len)
 		edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
 						     edesc->dma_len,
@@ -1715,39 +1779,37 @@ static void common_nonsnoop_hash_unmap(struct device *dev,
 	struct talitos_private *priv = dev_get_drvdata(dev);
 	bool is_sec1 = has_ftr_sec1(priv);
 	struct talitos_desc *desc = &edesc->desc;
-	struct talitos_desc *desc2 = (struct talitos_desc *)
-				     (edesc->buf + edesc->dma_len);
 
 	unmap_single_talitos_ptr(dev, &desc->ptr[5], DMA_FROM_DEVICE);
-	if (desc->next_desc &&
-	    desc->ptr[5].ptr != desc2->ptr[5].ptr)
-		unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE);
-	if (req_ctx->last_desc)
+
+	if (edesc->last && req_ctx->last_request)
 		memcpy(areq->result, req_ctx->hw_context,
 		       crypto_ahash_digestsize(tfm));
 
-	if (req_ctx->psrc)
-		talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
+	if (edesc->src)
+		talitos_sg_unmap(dev, edesc, edesc->src, NULL, 0, 0);
 
 	/* When using hashctx-in, must unmap it. */
 	if (from_talitos_ptr_len(&desc->ptr[1], is_sec1))
 		unmap_single_talitos_ptr(dev, &desc->ptr[1],
 					 DMA_TO_DEVICE);
-	else if (desc->next_desc)
-		unmap_single_talitos_ptr(dev, &desc2->ptr[1],
-					 DMA_TO_DEVICE);
-
-	if (is_sec1 && req_ctx->nbuf)
-		unmap_single_talitos_ptr(dev, &desc->ptr[3],
-					 DMA_TO_DEVICE);
 
 	if (edesc->dma_len)
 		dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
 				 DMA_BIDIRECTIONAL);
+}
+
+static void free_edesc_list_from(struct ahash_request *areq, struct talitos_edesc *edesc)
+{
+	struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
+	struct talitos_edesc *next;
 
-	if (desc->next_desc)
-		dma_unmap_single(dev, be32_to_cpu(desc->next_desc),
-				 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
+	while (edesc) {
+		next = edesc->next_desc;
+		common_nonsnoop_hash_unmap(ctx->dev, edesc, areq);
+		kfree(edesc);
+		edesc = next;
+	}
 }
 
 static void ahash_done(struct device *dev,
@@ -1758,30 +1820,46 @@ static void ahash_done(struct device *dev,
 	struct talitos_edesc *edesc =
 		 container_of(desc, struct talitos_edesc, desc);
 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
+	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
+	bool is_sec1 = has_ftr_sec1(dev_get_drvdata(dev));
+	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
+	struct talitos_edesc *next;
 
-	if (!req_ctx->last_desc && req_ctx->to_hash_later) {
-		/* Position any partial block for next update/final/finup */
-		req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
-		req_ctx->nbuf = req_ctx->to_hash_later;
-	}
-	common_nonsnoop_hash_unmap(dev, edesc, areq);
-
-	kfree(edesc);
+	if (is_sec1) {
+		if (!req_ctx->last_request && req_ctx->to_hash_later) {
+			/* Position any partial block for next update/final/finup */
+			req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
+			req_ctx->nbuf = req_ctx->to_hash_later;
+		}
 
-	if (err) {
+		free_edesc_list_from(areq, edesc);
 		ahash_request_complete(areq, err);
-		return;
-	}
+	} else {
+		next = edesc->next_desc;
 
-	req_ctx->remaining_ahash_request_bytes -=
-		req_ctx->current_ahash_request_bytes;
+		common_nonsnoop_hash_unmap(dev, edesc, areq);
+		kfree(edesc);
 
-	if (!req_ctx->remaining_ahash_request_bytes) {
-		ahash_request_complete(areq, 0);
-		return;
-	}
+		if (err)
+			goto out;
 
-	schedule_work(&req_ctx->sec1_ahash_process_remaining);
+		if (next) {
+			err = talitos_submit(dev, ctx->ch, &next->desc,
+					     ahash_done, areq);
+			if (err != -EINPROGRESS)
+				goto out;
+			return;
+		}
+out:
+		if (!req_ctx->last_request && req_ctx->to_hash_later) {
+			/* Position any partial block for next update/final/finup */
+			req_ctx->buf_idx = (req_ctx->buf_idx + 1) & 1;
+			req_ctx->nbuf = req_ctx->to_hash_later;
+		}
+		if (err && next)
+			free_edesc_list_from(areq, next);
+		ahash_request_complete(areq, err);
+	}
 }
 
 /*
@@ -1805,18 +1883,15 @@ static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
 			       (char *)padded_hash, DMA_TO_DEVICE);
 }
 
-static int common_nonsnoop_hash(struct talitos_edesc *edesc,
-				struct ahash_request *areq, unsigned int length,
-				void (*callback) (struct device *dev,
-						  struct talitos_desc *desc,
-						  void *context, int error))
+static void common_nonsnoop_hash(struct talitos_edesc *edesc,
+				 struct ahash_request *areq,
+				 unsigned int length)
 {
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
 	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
 	struct device *dev = ctx->dev;
 	struct talitos_desc *desc = &edesc->desc;
-	int ret;
 	bool sync_needed = false;
 	struct talitos_private *priv = dev_get_drvdata(dev);
 	bool is_sec1 = has_ftr_sec1(priv);
@@ -1825,7 +1900,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
 	/* first DWORD empty */
 
 	/* hash context in */
-	if (!req_ctx->first_desc || req_ctx->swinit) {
+	if (!edesc->first || !req_ctx->first_request || req_ctx->swinit) {
 		map_single_talitos_ptr_nosync(dev, &desc->ptr[1],
 					      req_ctx->hw_context_size,
 					      req_ctx->hw_context,
@@ -1833,40 +1908,31 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
 		req_ctx->swinit = 0;
 	}
 	/* Indicate next op is not the first. */
-	req_ctx->first_desc = 0;
+	req_ctx->first_request = 0;
 
 	/* HMAC key */
 	if (ctx->keylen)
 		to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen,
 			       is_sec1);
 
-	if (is_sec1 && req_ctx->nbuf)
-		length -= req_ctx->nbuf;
-
 	sg_count = edesc->src_nents ?: 1;
 	if (is_sec1 && sg_count > 1)
-		sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length);
+		sg_copy_to_buffer(edesc->src, sg_count, edesc->buf, length);
 	else if (length)
-		sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
-				      DMA_TO_DEVICE);
+		sg_count = dma_map_sg(dev, edesc->src, sg_count, DMA_TO_DEVICE);
+
 	/*
 	 * data in
 	 */
-	if (is_sec1 && req_ctx->nbuf) {
-		map_single_talitos_ptr(dev, &desc->ptr[3], req_ctx->nbuf,
-				       req_ctx->buf[req_ctx->buf_idx],
-				       DMA_TO_DEVICE);
-	} else {
-		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
-					  &desc->ptr[3], sg_count, 0, 0);
-		if (sg_count > 1)
-			sync_needed = true;
-	}
+	sg_count = talitos_sg_map(dev, edesc->src, length, edesc, &desc->ptr[3],
+				  sg_count, 0, 0);
+	if (sg_count > 1)
+		sync_needed = true;
 
 	/* fifth DWORD empty */
 
 	/* hash/HMAC out -or- hash context out */
-	if (req_ctx->last_desc)
+	if (edesc->last && req_ctx->last_request)
 		map_single_talitos_ptr(dev, &desc->ptr[5],
 				       crypto_ahash_digestsize(tfm),
 				       req_ctx->hw_context, DMA_FROM_DEVICE);
@@ -1881,73 +1947,94 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc,
 	if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
 		talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
 
-	if (is_sec1 && req_ctx->nbuf && length) {
-		struct talitos_desc *desc2 = (struct talitos_desc *)
-					     (edesc->buf + edesc->dma_len);
-		dma_addr_t next_desc;
-
-		memset(desc2, 0, sizeof(*desc2));
-		desc2->hdr = desc->hdr;
-		desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
-		desc2->hdr1 = desc2->hdr;
-		desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
-		desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
-		desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
-
-		if (desc->ptr[1].ptr)
-			copy_talitos_ptr(&desc2->ptr[1], &desc->ptr[1],
-					 is_sec1);
-		else
-			map_single_talitos_ptr_nosync(dev, &desc2->ptr[1],
-						      req_ctx->hw_context_size,
-						      req_ctx->hw_context,
-						      DMA_TO_DEVICE);
-		copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
-		sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
-					  &desc2->ptr[3], sg_count, 0, 0);
-		if (sg_count > 1)
-			sync_needed = true;
-		copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
-		if (req_ctx->last_desc)
-			map_single_talitos_ptr_nosync(dev, &desc->ptr[5],
-						      req_ctx->hw_context_size,
-						      req_ctx->hw_context,
-						      DMA_FROM_DEVICE);
-
-		next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
-					   DMA_BIDIRECTIONAL);
-		desc->next_desc = cpu_to_be32(next_desc);
-	}
-
 	if (sync_needed)
 		dma_sync_single_for_device(dev, edesc->dma_link_tbl,
 					   edesc->dma_len, DMA_BIDIRECTIONAL);
-
-	ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
-	if (ret != -EINPROGRESS) {
-		common_nonsnoop_hash_unmap(dev, edesc, areq);
-		kfree(edesc);
-	}
-	return ret;
 }
 
 static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
+					       struct scatterlist *src,
 					       unsigned int nbytes)
 {
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
 	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
+
+	return talitos_edesc_alloc(ctx->dev, src, NULL, NULL, 0,
+				   nbytes, 0, 0, 0, areq->base.flags, false);
+}
+
+static struct talitos_edesc *
+ahash_process_req_prepare(struct ahash_request *areq, unsigned int nbytes,
+			  unsigned int blocksize, bool is_sec1)
+{
+	struct talitos_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
 	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
-	struct talitos_private *priv = dev_get_drvdata(ctx->dev);
-	bool is_sec1 = has_ftr_sec1(priv);
+	struct talitos_edesc *first = NULL, *prev_edesc = NULL, *edesc;
+	size_t desc_max = is_sec1 ? TALITOS1_MAX_DATA_LEN :
+				    TALITOS2_MAX_DATA_LEN;
+	struct scatterlist tmp[2];
+	size_t to_hash_this_desc;
+	struct scatterlist *src;
+	size_t offset = 0;
+
+	do {
+		src = scatterwalk_ffwd(tmp, req_ctx->psrc, offset);
+
+		to_hash_this_desc =
+			min(nbytes, ALIGN_DOWN(desc_max, blocksize));
+
+		/* Allocate extended descriptor */
+		edesc = ahash_edesc_alloc(areq, src, to_hash_this_desc);
+		if (IS_ERR(edesc)) {
+			if (first)
+				free_edesc_list_from(areq, first);
+			return edesc;
+		}
 
-	if (is_sec1)
-		nbytes -= req_ctx->nbuf;
+		edesc->src =
+			scatterwalk_ffwd(edesc->bufsl, req_ctx->psrc, offset);
+		edesc->desc.hdr = ctx->desc_hdr_template;
+		edesc->first = offset == 0;
+		edesc->last = nbytes - to_hash_this_desc == 0;
 
-	return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
-				   nbytes, 0, 0, 0, areq->base.flags, false);
+		/* On last one, request SEC to pad; otherwise continue */
+		if (req_ctx->last_request && edesc->last)
+			edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
+		else
+			edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
+
+		/* request SEC to INIT hash. */
+		if (req_ctx->first_request && edesc->first && !req_ctx->swinit)
+			edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
+
+		/*
+		 * When the tfm context has a keylen, it's an HMAC.
+		 * A first or last (ie. not middle) descriptor must request HMAC.
+		 */
+		if (ctx->keylen && ((req_ctx->first_request && edesc->first) ||
+				    (req_ctx->last_request && edesc->last)))
+			edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
+
+		/* clear the DN bit  */
+		if (is_sec1 && !edesc->last)
+			edesc->desc.hdr &= ~DESC_HDR_DONE_NOTIFY;
+
+		common_nonsnoop_hash(edesc, areq, to_hash_this_desc);
+
+		offset += to_hash_this_desc;
+		nbytes -= to_hash_this_desc;
+
+		if (!prev_edesc)
+			first = edesc;
+		else
+			prev_edesc->next_desc = edesc;
+		prev_edesc = edesc;
+	} while (nbytes);
+
+	return first;
 }
 
-static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes)
+static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
 {
 	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
 	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
@@ -1955,23 +2042,23 @@ static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes
 	struct talitos_edesc *edesc;
 	unsigned int blocksize =
 			crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
+	bool is_sec1 = has_ftr_sec1(dev_get_drvdata(ctx->dev));
 	unsigned int nbytes_to_hash;
 	unsigned int to_hash_later;
 	unsigned int nsg;
 	int nents;
 	struct device *dev = ctx->dev;
-	struct talitos_private *priv = dev_get_drvdata(dev);
-	bool is_sec1 = has_ftr_sec1(priv);
 	u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
+	int ret;
 
-	if (!req_ctx->last_desc && (nbytes + req_ctx->nbuf <= blocksize)) {
+	if (!req_ctx->last_request && (nbytes + req_ctx->nbuf <= blocksize)) {
 		/* Buffer up to one whole block */
-		nents = sg_nents_for_len(req_ctx->request_sl, nbytes);
+		nents = sg_nents_for_len(areq->src, nbytes);
 		if (nents < 0) {
 			dev_err(dev, "Invalid number of src SG.\n");
 			return nents;
 		}
-		sg_copy_to_buffer(req_ctx->request_sl, nents,
+		sg_copy_to_buffer(areq->src, nents,
 				  ctx_buf + req_ctx->nbuf, nbytes);
 		req_ctx->nbuf += nbytes;
 		return 0;
@@ -1981,7 +2068,7 @@ static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes
 	nbytes_to_hash = nbytes + req_ctx->nbuf;
 	to_hash_later = nbytes_to_hash & (blocksize - 1);
 
-	if (req_ctx->last_desc)
+	if (req_ctx->last_request)
 		to_hash_later = 0;
 	else if (to_hash_later)
 		/* There is a partial block. Hash the full block(s) now */
@@ -1993,123 +2080,39 @@ static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes
 	}
 
 	/* Chain in any previously buffered data */
-	if (!is_sec1 && req_ctx->nbuf) {
+	if (req_ctx->nbuf) {
 		nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
 		sg_init_table(req_ctx->bufsl, nsg);
 		sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
 		if (nsg > 1)
-			sg_chain(req_ctx->bufsl, 2, req_ctx->request_sl);
+			sg_chain(req_ctx->bufsl, 2, areq->src);
 		req_ctx->psrc = req_ctx->bufsl;
-	} else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
-		int offset;
-
-		if (nbytes_to_hash > blocksize)
-			offset = blocksize - req_ctx->nbuf;
-		else
-			offset = nbytes_to_hash - req_ctx->nbuf;
-		nents = sg_nents_for_len(req_ctx->request_sl, offset);
-		if (nents < 0) {
-			dev_err(dev, "Invalid number of src SG.\n");
-			return nents;
-		}
-		sg_copy_to_buffer(req_ctx->request_sl, nents,
-				  ctx_buf + req_ctx->nbuf, offset);
-		req_ctx->nbuf += offset;
-		req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, req_ctx->request_sl,
-						 offset);
 	} else
-		req_ctx->psrc = req_ctx->request_sl;
+		req_ctx->psrc = areq->src;
 
 	if (to_hash_later) {
-		nents = sg_nents_for_len(req_ctx->request_sl, nbytes);
+		nents = sg_nents_for_len(areq->src, nbytes);
 		if (nents < 0) {
 			dev_err(dev, "Invalid number of src SG.\n");
 			return nents;
 		}
-		sg_pcopy_to_buffer(req_ctx->request_sl, nents,
+		sg_pcopy_to_buffer(areq->src, nents,
 				   req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
 				      to_hash_later,
 				      nbytes - to_hash_later);
 	}
 	req_ctx->to_hash_later = to_hash_later;
 
-	/* Allocate extended descriptor */
-	edesc = ahash_edesc_alloc(req_ctx->areq, nbytes_to_hash);
+	edesc = ahash_process_req_prepare(areq, nbytes_to_hash, blocksize,
+					  is_sec1);
 	if (IS_ERR(edesc))
 		return PTR_ERR(edesc);
 
-	edesc->desc.hdr = ctx->desc_hdr_template;
-
-	/* On last one, request SEC to pad; otherwise continue */
-	if (req_ctx->last_desc)
-		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
-	else
-		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
-
-	/* request SEC to INIT hash. */
-	if (req_ctx->first_desc && !req_ctx->swinit)
-		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
-
-	/* When the tfm context has a keylen, it's an HMAC.
-	 * A first or last (ie. not middle) descriptor must request HMAC.
-	 */
-	if (ctx->keylen && (req_ctx->first_desc || req_ctx->last_desc))
-		edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
+	ret = talitos_submit(dev, ctx->ch, &edesc->desc, ahash_done, areq);
+	if (ret != -EINPROGRESS)
+		free_edesc_list_from(areq, edesc);
 
-	return common_nonsnoop_hash(edesc, req_ctx->areq, nbytes_to_hash, ahash_done);
-}
-
-static void sec1_ahash_process_remaining(struct work_struct *work)
-{
-	struct talitos_ahash_req_ctx *req_ctx =
-		container_of(work, struct talitos_ahash_req_ctx,
-			     sec1_ahash_process_remaining);
-	int err = 0;
-
-	req_ctx->request_sl = scatterwalk_ffwd(req_ctx->request_bufsl,
-					       req_ctx->request_sl, TALITOS1_MAX_DATA_LEN);
-
-	if (req_ctx->remaining_ahash_request_bytes > TALITOS1_MAX_DATA_LEN)
-		req_ctx->current_ahash_request_bytes = TALITOS1_MAX_DATA_LEN;
-	else {
-		req_ctx->current_ahash_request_bytes =
-			req_ctx->remaining_ahash_request_bytes;
-
-		if (req_ctx->last_request)
-			req_ctx->last_desc = 1;
-	}
-
-	err = ahash_process_req_one(req_ctx->areq,
-				    req_ctx->current_ahash_request_bytes);
-
-	if (err != -EINPROGRESS)
-		ahash_request_complete(req_ctx->areq, err);
-}
-
-static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
-{
-	struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
-	struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
-	struct device *dev = ctx->dev;
-	struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
-	struct talitos_private *priv = dev_get_drvdata(dev);
-	bool is_sec1 = has_ftr_sec1(priv);
-
-	req_ctx->areq = areq;
-	req_ctx->request_sl = areq->src;
-	req_ctx->remaining_ahash_request_bytes = nbytes;
-
-	if (is_sec1) {
-		if (nbytes > TALITOS1_MAX_DATA_LEN)
-			nbytes = TALITOS1_MAX_DATA_LEN;
-		else if (req_ctx->last_request)
-			req_ctx->last_desc = 1;
-	}
-
-	req_ctx->current_ahash_request_bytes = nbytes;
-
-	return ahash_process_req_one(req_ctx->areq,
-				     req_ctx->current_ahash_request_bytes);
+	return ret;
 }
 
 static int ahash_init(struct ahash_request *areq)
@@ -2124,15 +2127,13 @@ static int ahash_init(struct ahash_request *areq)
 	/* Initialize the context */
 	req_ctx->buf_idx = 0;
 	req_ctx->nbuf = 0;
-	req_ctx->first_desc = 1; /* first_desc indicates h/w must init its context */
+	req_ctx->first_request = 1;
 	req_ctx->swinit = 0; /* assume h/w init of context */
 	size =	(crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
 			? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
 			: TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
 	req_ctx->hw_context_size = size;
 	req_ctx->last_request = 0;
-	req_ctx->last_desc = 0;
-	INIT_WORK(&req_ctx->sec1_ahash_process_remaining, sec1_ahash_process_remaining);
 
 	dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
 			     DMA_TO_DEVICE);
@@ -2224,8 +2225,8 @@ static int ahash_export(struct ahash_request *areq, void *out)
 	       req_ctx->hw_context_size);
 	memcpy(export->buf, req_ctx->buf[req_ctx->buf_idx], req_ctx->nbuf);
 	export->swinit = req_ctx->swinit;
-	export->first_desc = req_ctx->first_desc;
-	export->last_desc = req_ctx->last_desc;
+	export->first_request = req_ctx->first_request;
+	export->last_request = req_ctx->last_request;
 	export->to_hash_later = req_ctx->to_hash_later;
 	export->nbuf = req_ctx->nbuf;
 
@@ -2250,8 +2251,8 @@ static int ahash_import(struct ahash_request *areq, const void *in)
 	memcpy(req_ctx->hw_context, export->hw_context, size);
 	memcpy(req_ctx->buf[0], export->buf, export->nbuf);
 	req_ctx->swinit = export->swinit;
-	req_ctx->first_desc = export->first_desc;
-	req_ctx->last_desc = export->last_desc;
+	req_ctx->first_request = export->first_request;
+	req_ctx->last_request = export->last_request;
 	req_ctx->to_hash_later = export->to_hash_later;
 	req_ctx->nbuf = export->nbuf;
 
@@ -3409,14 +3410,19 @@ static int talitos_probe(struct platform_device *ofdev)
 	struct device *dev = &ofdev->dev;
 	struct device_node *np = ofdev->dev.of_node;
 	struct talitos_private *priv;
+	unsigned int num_channels;
 	int i, err;
 	int stride;
-	struct resource *res;
 
-	priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
+	if (of_property_read_u32(np, "fsl,num-channels", &num_channels))
+		return -EINVAL;
+
+	priv = devm_kzalloc(dev, struct_size(priv, chan, num_channels), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
+	priv->num_channels = num_channels;
+
 	INIT_LIST_HEAD(&priv->alg_list);
 
 	dev_set_drvdata(dev, priv);
@@ -3425,18 +3431,14 @@ static int talitos_probe(struct platform_device *ofdev)
 
 	spin_lock_init(&priv->reg_lock);
 
-	res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
-	if (!res)
-		return -ENXIO;
-	priv->reg = devm_ioremap(dev, res->start, resource_size(res));
-	if (!priv->reg) {
+	priv->reg = devm_platform_ioremap_resource(ofdev, 0);
+	if (IS_ERR(priv->reg)) {
 		dev_err(dev, "failed to of_iomap\n");
-		err = -ENOMEM;
+		err = PTR_ERR(priv->reg);
 		goto err_out;
 	}
 
 	/* get SEC version capabilities from device tree */
-	of_property_read_u32(np, "fsl,num-channels", &priv->num_channels);
 	of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
 	of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
 	of_property_read_u32(np, "fsl,descriptor-types-mask",
@@ -3511,16 +3513,6 @@ static int talitos_probe(struct platform_device *ofdev)
 		}
 	}
 
-	priv->chan = devm_kcalloc(dev,
-				  priv->num_channels,
-				  sizeof(struct talitos_channel),
-				  GFP_KERNEL);
-	if (!priv->chan) {
-		dev_err(dev, "failed to allocate channel management space\n");
-		err = -ENOMEM;
-		goto err_out;
-	}
-
 	priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
 
 	for (i = 0; i < priv->num_channels; i++) {
diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h
index 1a93ee355929d..d4ff8d589f461 100644
--- a/drivers/crypto/talitos.h
+++ b/drivers/crypto/talitos.h
@@ -44,11 +44,17 @@ struct talitos_desc {
 
 /*
  * talitos_edesc - s/w-extended descriptor
+ * @bufsl: scatterlist buffer
+ * @src: pointer to input scatterlist
+ * @first: first descriptor of a chain
+ * @last: last descriptor of a chain
+ *
  * @src_nents: number of segments in input scatterlist
  * @dst_nents: number of segments in output scatterlist
  * @iv_dma: dma address of iv for checking continuity and link table
  * @dma_len: length of dma mapped link_tbl space
  * @dma_link_tbl: bus physical address of link_tbl/buf
+ * @next_desc: next descriptor
  * @desc: h/w descriptor
  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
  * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
@@ -58,11 +64,17 @@ struct talitos_desc {
  * of link_tbl data
  */
 struct talitos_edesc {
+	struct scatterlist bufsl[2];
+	struct scatterlist *src;
+	int first;
+	int last;
+
 	int src_nents;
 	int dst_nents;
 	dma_addr_t iv_dma;
 	int dma_len;
 	dma_addr_t dma_link_tbl;
+	struct talitos_edesc *next_desc;
 	struct talitos_desc desc;
 	union {
 		DECLARE_FLEX_ARRAY(struct talitos_ptr, link_tbl);
@@ -139,8 +151,6 @@ struct talitos_private {
 	 */
 	unsigned int fifo_len;
 
-	struct talitos_channel *chan;
-
 	/* next channel to be assigned next incoming descriptor */
 	atomic_t last_chan ____cacheline_aligned;
 
@@ -153,6 +163,9 @@ struct talitos_private {
 	/* hwrng device */
 	struct hwrng rng;
 	bool rng_registered;
+
+	struct talitos_channel chan[] __counted_by(num_channels);
+
 };
 
 /* .features flag */
diff --git a/drivers/crypto/xilinx/xilinx-trng.c b/drivers/crypto/xilinx/xilinx-trng.c
index 5276ac2d82bbe..43a4832f07e7d 100644
--- a/drivers/crypto/xilinx/xilinx-trng.c
+++ b/drivers/crypto/xilinx/xilinx-trng.c
@@ -19,7 +19,6 @@
 #include <linux/platform_device.h>
 #include <crypto/aes.h>
 #include <crypto/df_sp80090a.h>
-#include <crypto/internal/drbg.h>
 #include <crypto/internal/cipher.h>
 #include <crypto/internal/rng.h>
 
diff --git a/drivers/dax/bus.c b/drivers/dax/bus.c
index ccfe65004888d..492573b47f662 100644
--- a/drivers/dax/bus.c
+++ b/drivers/dax/bus.c
@@ -1485,6 +1485,7 @@ static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data)
 	}
 
 	dev = &dev_dax->dev;
+	dev->type = &dev_dax_type;
 	device_initialize(dev);
 	dev_set_name(dev, "dax%d.%d", dax_region->id, dev_dax->id);
 
@@ -1531,7 +1532,6 @@ static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data)
 	dev->devt = inode->i_rdev;
 	dev->bus = &dax_bus_type;
 	dev->parent = parent;
-	dev->type = &dev_dax_type;
 
 	rc = device_add(dev);
 	if (rc) {
@@ -1554,14 +1554,13 @@ static struct dev_dax *__devm_create_dev_dax(struct dev_dax_data *data)
 	return dev_dax;
 
 err_alloc_dax:
-	kfree(dev_dax->pgmap);
 err_pgmap:
 	free_dev_dax_ranges(dev_dax);
 err_range:
-	free_dev_dax_id(dev_dax);
+	put_device(dev);
+	return ERR_PTR(rc);
 err_id:
 	kfree(dev_dax);
-
 	return ERR_PTR(rc);
 }
 
diff --git a/drivers/dax/kmem.c b/drivers/dax/kmem.c
index 2cc8749bc8711..a18e2b968e4da 100644
--- a/drivers/dax/kmem.c
+++ b/drivers/dax/kmem.c
@@ -227,6 +227,12 @@ static void dev_dax_kmem_remove(struct dev_dax *dev_dax)
 		if (rc)
 			continue;
 
+		/* range was never added during probe */
+		if (!data->res[i]) {
+			success++;
+			continue;
+		}
+
 		rc = remove_memory(range.start, range_len(&range));
 		if (rc == 0) {
 			remove_resource(data->res[i]);
diff --git a/drivers/devfreq/devfreq-event.c b/drivers/devfreq/devfreq-event.c
index 179de3cf6d6cc..9e183cd8818c1 100644
--- a/drivers/devfreq/devfreq-event.c
+++ b/drivers/devfreq/devfreq-event.c
@@ -17,7 +17,13 @@
 #include <linux/list.h>
 #include <linux/of.h>
 
-static struct class *devfreq_event_class;
+static struct attribute *devfreq_event_attrs[];
+ATTRIBUTE_GROUPS(devfreq_event);
+
+static const struct class devfreq_event_class = {
+	.name		= "devfreq-event",
+	.dev_groups	= devfreq_event_groups
+};
 
 /* The list of all devfreq event list */
 static LIST_HEAD(devfreq_event_list);
@@ -321,7 +327,7 @@ struct devfreq_event_dev *devfreq_event_add_edev(struct device *dev,
 	edev->desc = desc;
 	edev->enable_count = 0;
 	edev->dev.parent = dev;
-	edev->dev.class = devfreq_event_class;
+	edev->dev.class = &devfreq_event_class;
 	edev->dev.release = devfreq_event_release_edev;
 
 	dev_set_name(&edev->dev, "event%d", atomic_inc_return(&event_no));
@@ -461,18 +467,15 @@ static struct attribute *devfreq_event_attrs[] = {
 	&dev_attr_enable_count.attr,
 	NULL,
 };
-ATTRIBUTE_GROUPS(devfreq_event);
 
 static int __init devfreq_event_init(void)
 {
-	devfreq_event_class = class_create("devfreq-event");
-	if (IS_ERR(devfreq_event_class)) {
-		pr_err("%s: couldn't create class\n", __FILE__);
-		return PTR_ERR(devfreq_event_class);
-	}
+	int err;
 
-	devfreq_event_class->dev_groups = devfreq_event_groups;
+	err = class_register(&devfreq_event_class);
+	if (err)
+		pr_err("%s: couldn't create class\n", __FILE__);
 
-	return 0;
+	return err;
 }
 subsys_initcall(devfreq_event_init);
diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c
index 82dd9a43dc621..f08fc6966eaee 100644
--- a/drivers/devfreq/devfreq.c
+++ b/drivers/devfreq/devfreq.c
@@ -1261,7 +1261,6 @@ void devfreq_resume(void)
 int devfreq_add_governor(struct devfreq_governor *governor)
 {
 	struct devfreq_governor *g;
-	struct devfreq *devfreq;
 	int err = 0;
 
 	if (!governor) {
@@ -1280,38 +1279,6 @@ int devfreq_add_governor(struct devfreq_governor *governor)
 
 	list_add(&governor->node, &devfreq_governor_list);
 
-	list_for_each_entry(devfreq, &devfreq_list, node) {
-		int ret = 0;
-		struct device *dev = devfreq->dev.parent;
-
-		if (!strncmp(devfreq->governor->name, governor->name,
-			     DEVFREQ_NAME_LEN)) {
-			/* The following should never occur */
-			if (devfreq->governor) {
-				dev_warn(dev,
-					 "%s: Governor %s already present\n",
-					 __func__, devfreq->governor->name);
-				ret = devfreq->governor->event_handler(devfreq,
-							DEVFREQ_GOV_STOP, NULL);
-				if (ret) {
-					dev_warn(dev,
-						 "%s: Governor %s stop = %d\n",
-						 __func__,
-						 devfreq->governor->name, ret);
-				}
-				/* Fall through */
-			}
-			devfreq->governor = governor;
-			ret = devfreq->governor->event_handler(devfreq,
-						DEVFREQ_GOV_START, NULL);
-			if (ret) {
-				dev_warn(dev, "%s: Governor %s start=%d\n",
-					 __func__, devfreq->governor->name,
-					 ret);
-			}
-		}
-	}
-
 err_out:
 	mutex_unlock(&devfreq_list_lock);
 
@@ -1410,7 +1377,7 @@ static ssize_t governor_show(struct device *dev,
 	struct devfreq *df = to_devfreq(dev);
 
 	if (!df->governor)
-		return -EINVAL;
+		return -ENOENT;
 
 	return sprintf(buf, "%s\n", df->governor->name);
 }
@@ -1423,9 +1390,6 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 	char str_governor[DEVFREQ_NAME_LEN + 1];
 	const struct devfreq_governor *governor, *prev_governor;
 
-	if (!df->governor)
-		return -EINVAL;
-
 	ret = sscanf(buf, "%" __stringify(DEVFREQ_NAME_LEN) "s", str_governor);
 	if (ret != 1)
 		return -EINVAL;
@@ -1436,6 +1400,9 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 		ret = PTR_ERR(governor);
 		goto out;
 	}
+	if (!df->governor)
+		goto start_new_governor;
+
 	if (df->governor == governor) {
 		ret = 0;
 		goto out;
@@ -1456,6 +1423,7 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 		goto out;
 	}
 
+start_new_governor:
 	/*
 	 * Start the new governor and create the specific sysfs files
 	 * which depend on the new governor.
@@ -1469,6 +1437,9 @@ static ssize_t governor_store(struct device *dev, struct device_attribute *attr,
 
 		/* Restore previous governor */
 		df->governor = prev_governor;
+		if (!df->governor)
+			goto out;
+
 		ret = df->governor->event_handler(df, DEVFREQ_GOV_START, NULL);
 		if (ret) {
 			dev_err(dev,
@@ -1501,16 +1472,13 @@ static ssize_t available_governors_show(struct device *d,
 	struct devfreq *df = to_devfreq(d);
 	ssize_t count = 0;
 
-	if (!df->governor)
-		return -EINVAL;
-
 	mutex_lock(&devfreq_list_lock);
 
 	/*
 	 * The devfreq with immutable governor (e.g., passive) shows
 	 * only own governor.
 	 */
-	if (IS_SUPPORTED_FLAG(df->governor->flags, IMMUTABLE)) {
+	if (df->governor && IS_SUPPORTED_FLAG(df->governor->flags, IMMUTABLE)) {
 		count = scnprintf(&buf[count], DEVFREQ_NAME_LEN,
 				  "%s ", df->governor->name);
 	/*
diff --git a/drivers/devfreq/event/rockchip-dfi.c b/drivers/devfreq/event/rockchip-dfi.c
index 5e6e7e900bda0..8db0bceeded42 100644
--- a/drivers/devfreq/event/rockchip-dfi.c
+++ b/drivers/devfreq/event/rockchip-dfi.c
@@ -185,8 +185,10 @@ static int rockchip_dfi_enable(struct rockchip_dfi *dfi)
 	}
 
 	ret = rockchip_dfi_ddrtype_to_ctrl(dfi, &ctrl);
-	if (ret)
+	if (ret) {
+		clk_disable_unprepare(dfi->clk);
 		goto out;
+	}
 
 	for (i = 0; i < dfi->max_channels; i++) {
 
diff --git a/drivers/dibs/Kconfig b/drivers/dibs/Kconfig
index 5dc347b9b2350..8f41e56221004 100644
--- a/drivers/dibs/Kconfig
+++ b/drivers/dibs/Kconfig
@@ -1,18 +1,18 @@
 # SPDX-License-Identifier: GPL-2.0
 config DIBS
-	tristate "DIBS support"
+	tristate "Direct Internal Buffer Sharing support"
 	default n
 	help
-	  Direct Internal Buffer Sharing (DIBS)
-	  A communication method that uses common physical (internal) memory
-	  for synchronous direct access into a remote buffer.
+	  Direct Internal Buffer Sharing (DIBS) is a communication method that
+	  uses common physical (internal) memory for synchronous direct access
+	  into a remote buffer.
 
 	  Select this option to provide the abstraction layer between
 	  dibs devices and dibs clients like the SMC protocol.
 	  The module name is dibs.
 
 config DIBS_LO
-	bool "intra-OS shortcut with dibs loopback"
+	bool "Intra-OS shortcut with dibs loopback"
 	depends on DIBS
 	default n
 	help
diff --git a/drivers/dio/dio.c b/drivers/dio/dio.c
index 419b3c13d4913..7722d9eae6d00 100644
--- a/drivers/dio/dio.c
+++ b/drivers/dio/dio.c
@@ -178,7 +178,7 @@ static int __init dio_init(void)
 	if (!MACH_IS_HP300)
 		return 0;
 
-        printk(KERN_INFO "Scanning for DIO devices...\n");
+	pr_info("Scanning for DIO devices...\n");
 
 	/* Initialize the DIO bus */
 	INIT_LIST_HEAD(&dio_bus.devices);
@@ -247,18 +247,19 @@ static int __init dio_init(void)
 			dev->id = prid;
 
 		dev->ipl = DIO_IPL(va);
-		strcpy(dev->name, dio_getname(dev->id));
-                printk(KERN_INFO "select code %3d: ipl %d: ID %02X", dev->scode, dev->ipl, prid);
+		strscpy(dev->name, dio_getname(dev->id));
 		if (DIO_NEEDSSECID(prid))
-                        printk(":%02X", secid);
-                printk(": %s\n", dev->name);
+			pr_info("select code %3d: ipl %u: ID %02X:%02X: %s\n",
+				dev->scode, dev->ipl, prid, secid, dev->name);
+		else
+			pr_info("select code %3d: ipl %u: ID %02X: %s\n",
+				dev->scode, dev->ipl, prid, dev->name);
 
 		if (scode >= DIOII_SCBASE)
 			iounmap(va);
 		error = device_register(&dev->dev);
 		if (error) {
-			pr_err("DIO: Error registering device %s\n",
-			       dev->name);
+			pr_err("DIO: Error registering device %s\n", dev->name);
 			put_device(&dev->dev);
 			continue;
 		}
diff --git a/drivers/dma-buf/.kunitconfig b/drivers/dma-buf/.kunitconfig
new file mode 100644
index 0000000000000..1ce5fb7e6cf9f
--- /dev/null
+++ b/drivers/dma-buf/.kunitconfig
@@ -0,0 +1,2 @@
+CONFIG_KUNIT=y
+CONFIG_DMABUF_KUNIT_TEST=y
diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig
index 8d4f2f89f24e3..7efc0f0d07126 100644
--- a/drivers/dma-buf/Kconfig
+++ b/drivers/dma-buf/Kconfig
@@ -49,10 +49,13 @@ config DMABUF_DEBUG
 	  exporters. Specifically it validates that importers do not peek at the
 	  underlying struct page when they import a buffer.
 
-config DMABUF_SELFTESTS
-	tristate "Selftests for the dma-buf interfaces"
-	default n
-	depends on DMA_SHARED_BUFFER
+config DMABUF_KUNIT_TEST
+	tristate "KUnit tests for DMA-BUF" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	select DMA_SHARED_BUFFER
+	default KUNIT_ALL_TESTS
+	help
+	   Enable kunit tests for DMA-BUF
 
 menuconfig DMABUF_HEAPS
 	bool "DMA-BUF Userland Memory Heaps"
diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
index 7a85565d906ba..b25d7550bacfd 100644
--- a/drivers/dma-buf/Makefile
+++ b/drivers/dma-buf/Makefile
@@ -7,11 +7,10 @@ obj-$(CONFIG_SYNC_FILE)		+= sync_file.o
 obj-$(CONFIG_SW_SYNC)		+= sw_sync.o sync_debug.o
 obj-$(CONFIG_UDMABUF)		+= udmabuf.o
 
-dmabuf_selftests-y := \
-	selftest.o \
+dmabuf_kunit-y := \
 	st-dma-fence.o \
 	st-dma-fence-chain.o \
 	st-dma-fence-unwrap.o \
 	st-dma-resv.o
 
-obj-$(CONFIG_DMABUF_SELFTESTS)	+= dmabuf_selftests.o
+obj-$(CONFIG_DMABUF_KUNIT_TEST) += dmabuf_kunit.o
diff --git a/drivers/dma-buf/dma-fence-array.c b/drivers/dma-buf/dma-fence-array.c
index 089f694695248..5e10e8df372f1 100644
--- a/drivers/dma-buf/dma-fence-array.c
+++ b/drivers/dma-buf/dma-fence-array.c
@@ -190,15 +190,13 @@ EXPORT_SYMBOL(dma_fence_array_alloc);
  * @fences:		[in]	array containing the fences
  * @context:		[in]	fence context to use
  * @seqno:		[in]	sequence number to use
- * @signal_on_any:	[in]	signal on any fence in the array
  *
  * Implementation of @dma_fence_array_create without allocation. Useful to init
  * a preallocated dma fence array in the path of reclaim or dma fence signaling.
  */
 void dma_fence_array_init(struct dma_fence_array *array,
 			  int num_fences, struct dma_fence **fences,
-			  u64 context, unsigned seqno,
-			  bool signal_on_any)
+			  u64 context, unsigned seqno)
 {
 	static struct lock_class_key dma_fence_array_lock_key;
 
@@ -222,7 +220,7 @@ void dma_fence_array_init(struct dma_fence_array *array,
 	 */
 	lockdep_set_class(&array->base.inline_lock, &dma_fence_array_lock_key);
 
-	atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);
+	atomic_set(&array->num_pending, num_fences);
 	array->fences = fences;
 
 	array->base.error = PENDING_ERROR;
@@ -249,7 +247,6 @@ EXPORT_SYMBOL(dma_fence_array_init);
  * @fences:		[in]	array containing the fences
  * @context:		[in]	fence context to use
  * @seqno:		[in]	sequence number to use
- * @signal_on_any:	[in]	signal on any fence in the array
  *
  * Allocate a dma_fence_array object and initialize the base fence with
  * dma_fence_init().
@@ -264,8 +261,7 @@ EXPORT_SYMBOL(dma_fence_array_init);
  */
 struct dma_fence_array *dma_fence_array_create(int num_fences,
 					       struct dma_fence **fences,
-					       u64 context, unsigned seqno,
-					       bool signal_on_any)
+					       u64 context, unsigned seqno)
 {
 	struct dma_fence_array *array;
 
@@ -273,8 +269,7 @@ struct dma_fence_array *dma_fence_array_create(int num_fences,
 	if (!array)
 		return NULL;
 
-	dma_fence_array_init(array, num_fences, fences,
-			     context, seqno, signal_on_any);
+	dma_fence_array_init(array, num_fences, fences, context, seqno);
 
 	return array;
 }
diff --git a/drivers/dma-buf/dma-fence-unwrap.c b/drivers/dma-buf/dma-fence-unwrap.c
index 07fe9bf45aea7..53bb40e70b277 100644
--- a/drivers/dma-buf/dma-fence-unwrap.c
+++ b/drivers/dma-buf/dma-fence-unwrap.c
@@ -180,8 +180,7 @@ struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences,
 
 	if (count > 1) {
 		result = dma_fence_array_create(count, array,
-						dma_fence_context_alloc(1),
-						1, false);
+						dma_fence_context_alloc(1), 1);
 		if (!result) {
 			for (i = 0; i < count; i++)
 				dma_fence_put(array[i]);
diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c
index a2aa82f4eedd4..b3bfa6943a8e1 100644
--- a/drivers/dma-buf/dma-fence.c
+++ b/drivers/dma-buf/dma-fence.c
@@ -363,6 +363,8 @@ void dma_fence_signal_timestamp_locked(struct dma_fence *fence,
 				      &fence->flags)))
 		return;
 
+	trace_dma_fence_signaled(fence);
+
 	/*
 	 * When neither a release nor a wait operation is specified set the ops
 	 * pointer to NULL to allow the fence structure to become independent
@@ -377,7 +379,6 @@ void dma_fence_signal_timestamp_locked(struct dma_fence *fence,
 
 	fence->timestamp = timestamp;
 	set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
-	trace_dma_fence_signaled(fence);
 
 	list_for_each_entry_safe(cur, tmp, &cb_list, node) {
 		INIT_LIST_HEAD(&cur->node);
diff --git a/drivers/dma-buf/dma-resv.c b/drivers/dma-buf/dma-resv.c
index ce9e6c04897ff..39a92d9f24136 100644
--- a/drivers/dma-buf/dma-resv.c
+++ b/drivers/dma-buf/dma-resv.c
@@ -648,8 +648,7 @@ int dma_resv_get_singleton(struct dma_resv *obj, enum dma_resv_usage usage,
 	}
 
 	array = dma_fence_array_create(count, fences,
-				       dma_fence_context_alloc(1),
-				       1, false);
+				       dma_fence_context_alloc(1), 1);
 	if (!array) {
 		while (count--)
 			dma_fence_put(fences[count]);
diff --git a/drivers/dma-buf/selftest.c b/drivers/dma-buf/selftest.c
deleted file mode 100644
index c60b6944b4bd1..0000000000000
--- a/drivers/dma-buf/selftest.c
+++ /dev/null
@@ -1,167 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-
-/*
- * Copyright © 2019 Intel Corporation
- */
-
-#include <linux/compiler.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/sched/signal.h>
-#include <linux/slab.h>
-
-#include "selftest.h"
-
-enum {
-#define selftest(n, func) __idx_##n,
-#include "selftests.h"
-#undef selftest
-};
-
-#define selftest(n, f) [__idx_##n] = { .name = #n, .func = f },
-static struct selftest {
-	bool enabled;
-	const char *name;
-	int (*func)(void);
-} selftests[] = {
-#include "selftests.h"
-};
-#undef selftest
-
-/* Embed the line number into the parameter name so that we can order tests */
-#define param(n) __PASTE(igt__, __PASTE(__PASTE(__LINE__, __), n))
-#define selftest_0(n, func, id) \
-module_param_named(id, selftests[__idx_##n].enabled, bool, 0400);
-#define selftest(n, func) selftest_0(n, func, param(n))
-#include "selftests.h"
-#undef selftest
-
-int __sanitycheck__(void)
-{
-	pr_debug("Hello World!\n");
-	return 0;
-}
-
-static char *__st_filter;
-
-static bool apply_subtest_filter(const char *caller, const char *name)
-{
-	char *filter, *sep, *tok;
-	bool result = true;
-
-	filter = kstrdup(__st_filter, GFP_KERNEL);
-	for (sep = filter; (tok = strsep(&sep, ","));) {
-		bool allow = true;
-		char *sl;
-
-		if (*tok == '!') {
-			allow = false;
-			tok++;
-		}
-
-		if (*tok == '\0')
-			continue;
-
-		sl = strchr(tok, '/');
-		if (sl) {
-			*sl++ = '\0';
-			if (strcmp(tok, caller)) {
-				if (allow)
-					result = false;
-				continue;
-			}
-			tok = sl;
-		}
-
-		if (strcmp(tok, name)) {
-			if (allow)
-				result = false;
-			continue;
-		}
-
-		result = allow;
-		break;
-	}
-	kfree(filter);
-
-	return result;
-}
-
-int
-__subtests(const char *caller, const struct subtest *st, int count, void *data)
-{
-	int err;
-
-	for (; count--; st++) {
-		cond_resched();
-		if (signal_pending(current))
-			return -EINTR;
-
-		if (!apply_subtest_filter(caller, st->name))
-			continue;
-
-		pr_info("dma-buf: Running %s/%s\n", caller, st->name);
-
-		err = st->func(data);
-		if (err && err != -EINTR) {
-			pr_err("dma-buf/%s: %s failed with error %d\n",
-			       caller, st->name, err);
-			return err;
-		}
-	}
-
-	return 0;
-}
-
-static void set_default_test_all(struct selftest *st, unsigned long count)
-{
-	unsigned long i;
-
-	for (i = 0; i < count; i++)
-		if (st[i].enabled)
-			return;
-
-	for (i = 0; i < count; i++)
-		st[i].enabled = true;
-}
-
-static int run_selftests(struct selftest *st, unsigned long count)
-{
-	int err = 0;
-
-	set_default_test_all(st, count);
-
-	/* Tests are listed in natural order in selftests.h */
-	for (; count--; st++) {
-		if (!st->enabled)
-			continue;
-
-		pr_info("dma-buf: Running %s\n", st->name);
-		err = st->func();
-		if (err)
-			break;
-	}
-
-	if (WARN(err > 0 || err == -ENOTTY,
-		 "%s returned %d, conflicting with selftest's magic values!\n",
-		 st->name, err))
-		err = -1;
-
-	return err;
-}
-
-static int __init st_init(void)
-{
-	return run_selftests(selftests, ARRAY_SIZE(selftests));
-}
-
-static void __exit st_exit(void)
-{
-}
-
-module_param_named(st_filter, __st_filter, charp, 0400);
-module_init(st_init);
-module_exit(st_exit);
-
-MODULE_DESCRIPTION("Self-test harness for dma-buf");
-MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/dma-buf/selftest.h b/drivers/dma-buf/selftest.h
deleted file mode 100644
index 45793aff61425..0000000000000
--- a/drivers/dma-buf/selftest.h
+++ /dev/null
@@ -1,30 +0,0 @@
-// SPDX-License-Identifier: MIT
-
-/*
- * Copyright © 2019 Intel Corporation
- */
-
-#ifndef __SELFTEST_H__
-#define __SELFTEST_H__
-
-#include <linux/compiler.h>
-
-#define selftest(name, func) int func(void);
-#include "selftests.h"
-#undef selftest
-
-struct subtest {
-	int (*func)(void *data);
-	const char *name;
-};
-
-int __subtests(const char *caller,
-	       const struct subtest *st,
-	       int count,
-	       void *data);
-#define subtests(T, data) \
-	__subtests(__func__, T, ARRAY_SIZE(T), data)
-
-#define SUBTEST(x) { x, #x }
-
-#endif /* __SELFTEST_H__ */
diff --git a/drivers/dma-buf/selftests.h b/drivers/dma-buf/selftests.h
deleted file mode 100644
index 851965867d9c7..0000000000000
--- a/drivers/dma-buf/selftests.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/* List each unit test as selftest(name, function)
- *
- * The name is used as both an enum and expanded as subtest__name to create
- * a module parameter. It must be unique and legal for a C identifier.
- *
- * The function should be of type int function(void). It may be conditionally
- * compiled using #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST).
- *
- * Tests are executed in order by igt/dmabuf_selftest
- */
-selftest(sanitycheck, __sanitycheck__) /* keep first (igt selfcheck) */
-selftest(dma_fence, dma_fence)
-selftest(dma_fence_chain, dma_fence_chain)
-selftest(dma_fence_unwrap, dma_fence_unwrap)
-selftest(dma_resv, dma_resv)
diff --git a/drivers/dma-buf/st-dma-fence-chain.c b/drivers/dma-buf/st-dma-fence-chain.c
index 821023dd34df6..a3023d3fedc9d 100644
--- a/drivers/dma-buf/st-dma-fence-chain.c
+++ b/drivers/dma-buf/st-dma-fence-chain.c
@@ -4,6 +4,7 @@
  * Copyright © 2019 Intel Corporation
  */
 
+#include <kunit/test.h>
 #include <linux/delay.h>
 #include <linux/dma-fence.h>
 #include <linux/dma-fence-chain.h>
@@ -15,8 +16,6 @@
 #include <linux/spinlock.h>
 #include <linux/random.h>
 
-#include "selftest.h"
-
 #define CHAIN_SZ (4 << 10)
 
 static struct kmem_cache *slab_fences;
@@ -74,27 +73,23 @@ static struct dma_fence *mock_chain(struct dma_fence *prev,
 	return &f->base;
 }
 
-static int sanitycheck(void *arg)
+static void test_sanitycheck(struct kunit *test)
 {
 	struct dma_fence *f, *chain;
-	int err = 0;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	chain = mock_chain(NULL, f, 1);
 	if (chain)
 		dma_fence_enable_sw_signaling(chain);
 	else
-		err = -ENOMEM;
+		KUNIT_FAIL(test, "Failed to create chain");
 
 	dma_fence_signal(f);
 	dma_fence_put(f);
 
 	dma_fence_put(chain);
-
-	return err;
 }
 
 struct fence_chains {
@@ -176,7 +171,7 @@ static void fence_chains_fini(struct fence_chains *fc)
 	kvfree(fc->chains);
 }
 
-static int find_seqno(void *arg)
+static void test_find_seqno(struct kunit *test)
 {
 	struct fence_chains fc;
 	struct dma_fence *fence;
@@ -184,14 +179,13 @@ static int find_seqno(void *arg)
 	int i;
 
 	err = fence_chains_init(&fc, 64, seqno_inc);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	fence = dma_fence_get(fc.tail);
 	err = dma_fence_chain_find_seqno(&fence, 0);
 	dma_fence_put(fence);
 	if (err) {
-		pr_err("Reported %d for find_seqno(0)!\n", err);
+		KUNIT_FAIL(test, "Reported %d for find_seqno(0)!", err);
 		goto err;
 	}
 
@@ -200,14 +194,13 @@ static int find_seqno(void *arg)
 		err = dma_fence_chain_find_seqno(&fence, i + 1);
 		dma_fence_put(fence);
 		if (err) {
-			pr_err("Reported %d for find_seqno(%d:%d)!\n",
-			       err, fc.chain_length + 1, i + 1);
+			KUNIT_FAIL(test, "Reported %d for find_seqno(%d:%d)!",
+				   err, fc.chain_length + 1, i + 1);
 			goto err;
 		}
 		if (fence != fc.chains[i]) {
-			pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
-			       fc.chain_length + 1, i + 1);
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Incorrect fence reported by find_seqno(%d:%d)",
+				   fc.chain_length + 1, i + 1);
 			goto err;
 		}
 
@@ -215,12 +208,11 @@ static int find_seqno(void *arg)
 		err = dma_fence_chain_find_seqno(&fence, i + 1);
 		dma_fence_put(fence);
 		if (err) {
-			pr_err("Error reported for finding self\n");
+			KUNIT_FAIL(test, "Error reported for finding self");
 			goto err;
 		}
 		if (fence != fc.chains[i]) {
-			pr_err("Incorrect fence reported by find self\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Incorrect fence reported by find self");
 			goto err;
 		}
 
@@ -228,9 +220,8 @@ static int find_seqno(void *arg)
 		err = dma_fence_chain_find_seqno(&fence, i + 2);
 		dma_fence_put(fence);
 		if (!err) {
-			pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n",
-			       i + 1, i + 2);
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Error not reported for future fence: find_seqno(%d:%d)!",
+				   i + 1, i + 2);
 			goto err;
 		}
 
@@ -238,31 +229,28 @@ static int find_seqno(void *arg)
 		err = dma_fence_chain_find_seqno(&fence, i);
 		dma_fence_put(fence);
 		if (err) {
-			pr_err("Error reported for previous fence!\n");
+			KUNIT_FAIL(test, "Error reported for previous fence!");
 			goto err;
 		}
 		if (i > 0 && fence != fc.chains[i - 1]) {
-			pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
-			       i + 1, i);
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Incorrect fence reported by find_seqno(%d:%d)",
+				   i + 1, i);
 			goto err;
 		}
 	}
 
 err:
 	fence_chains_fini(&fc);
-	return err;
 }
 
-static int find_signaled(void *arg)
+static void test_find_signaled(struct kunit *test)
 {
 	struct fence_chains fc;
 	struct dma_fence *fence;
 	int err;
 
 	err = fence_chains_init(&fc, 2, seqno_inc);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	dma_fence_signal(fc.fences[0]);
 
@@ -270,37 +258,33 @@ static int find_signaled(void *arg)
 	err = dma_fence_chain_find_seqno(&fence, 1);
 	dma_fence_put(fence);
 	if (err) {
-		pr_err("Reported %d for find_seqno()!\n", err);
+		KUNIT_FAIL(test, "Reported %d for find_seqno()!", err);
 		goto err;
 	}
 
 	if (fence && fence != fc.chains[0]) {
-		pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n",
-		       fence->seqno);
+		KUNIT_FAIL(test, "Incorrect chain-fence.seqno:%lld reported for completed seqno:1",
+			   fence->seqno);
 
 		dma_fence_get(fence);
 		err = dma_fence_chain_find_seqno(&fence, 1);
 		dma_fence_put(fence);
 		if (err)
-			pr_err("Reported %d for finding self!\n", err);
-
-		err = -EINVAL;
+			KUNIT_FAIL(test, "Reported %d for finding self!", err);
 	}
 
 err:
 	fence_chains_fini(&fc);
-	return err;
 }
 
-static int find_out_of_order(void *arg)
+static void test_find_out_of_order(struct kunit *test)
 {
 	struct fence_chains fc;
 	struct dma_fence *fence;
 	int err;
 
 	err = fence_chains_init(&fc, 3, seqno_inc);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	dma_fence_signal(fc.fences[1]);
 
@@ -308,7 +292,7 @@ static int find_out_of_order(void *arg)
 	err = dma_fence_chain_find_seqno(&fence, 2);
 	dma_fence_put(fence);
 	if (err) {
-		pr_err("Reported %d for find_seqno()!\n", err);
+		KUNIT_FAIL(test, "Reported %d for find_seqno()!", err);
 		goto err;
 	}
 
@@ -319,16 +303,12 @@ static int find_out_of_order(void *arg)
 	 * we should get as fence to wait upon (fence 2 being garbage
 	 * collected during the traversal of the chain).
 	 */
-	if (fence != fc.chains[0]) {
-		pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n",
-		       fence ? fence->seqno : 0);
-
-		err = -EINVAL;
-	}
+	if (fence != fc.chains[0])
+		KUNIT_FAIL(test, "Incorrect chain-fence.seqno:%lld reported for completed seqno:2",
+			   fence ? fence->seqno : 0);
 
 err:
 	fence_chains_fini(&fc);
-	return err;
 }
 
 static uint64_t seqno_inc2(unsigned int i)
@@ -336,7 +316,7 @@ static uint64_t seqno_inc2(unsigned int i)
 	return 2 * i + 2;
 }
 
-static int find_gap(void *arg)
+static void test_find_gap(struct kunit *test)
 {
 	struct fence_chains fc;
 	struct dma_fence *fence;
@@ -344,24 +324,22 @@ static int find_gap(void *arg)
 	int i;
 
 	err = fence_chains_init(&fc, 64, seqno_inc2);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	for (i = 0; i < fc.chain_length; i++) {
 		fence = dma_fence_get(fc.tail);
 		err = dma_fence_chain_find_seqno(&fence, 2 * i + 1);
 		dma_fence_put(fence);
 		if (err) {
-			pr_err("Reported %d for find_seqno(%d:%d)!\n",
-			       err, fc.chain_length + 1, 2 * i + 1);
+			KUNIT_FAIL(test, "Reported %d for find_seqno(%d:%d)!",
+				   err, fc.chain_length + 1, 2 * i + 1);
 			goto err;
 		}
 		if (fence != fc.chains[i]) {
-			pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n",
-			       fence->seqno,
-			       fc.chain_length + 1,
-			       2 * i + 1);
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)",
+				   fence->seqno,
+				   fc.chain_length + 1,
+				   2 * i + 1);
 			goto err;
 		}
 
@@ -369,19 +347,17 @@ static int find_gap(void *arg)
 		err = dma_fence_chain_find_seqno(&fence, 2 * i + 2);
 		dma_fence_put(fence);
 		if (err) {
-			pr_err("Error reported for finding self\n");
+			KUNIT_FAIL(test, "Error reported for finding self");
 			goto err;
 		}
 		if (fence != fc.chains[i]) {
-			pr_err("Incorrect fence reported by find self\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Incorrect fence reported by find self");
 			goto err;
 		}
 	}
 
 err:
 	fence_chains_fini(&fc);
-	return err;
 }
 
 struct find_race {
@@ -437,7 +413,7 @@ static int __find_race(void *arg)
 	return err;
 }
 
-static int find_race(void *arg)
+static void test_find_race(struct kunit *test)
 {
 	struct find_race data;
 	int ncpus = num_online_cpus();
@@ -447,12 +423,11 @@ static int find_race(void *arg)
 	int i;
 
 	err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	threads = kmalloc_objs(*threads, ncpus);
 	if (!threads) {
-		err = -ENOMEM;
+		KUNIT_FAIL(test, "Failed to allocate threads array");
 		goto err;
 	}
 
@@ -486,74 +461,67 @@ static int find_race(void *arg)
 			count++;
 	pr_info("Completed %lu cycles\n", count);
 
+	KUNIT_EXPECT_EQ(test, err, 0);
+
 err:
 	fence_chains_fini(&data.fc);
-	return err;
 }
 
-static int signal_forward(void *arg)
+static void test_signal_forward(struct kunit *test)
 {
 	struct fence_chains fc;
 	int err;
 	int i;
 
 	err = fence_chains_init(&fc, 64, seqno_inc);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	for (i = 0; i < fc.chain_length; i++) {
 		dma_fence_signal(fc.fences[i]);
 
 		if (!dma_fence_is_signaled(fc.chains[i])) {
-			pr_err("chain[%d] not signaled!\n", i);
-			err = -EINVAL;
+			KUNIT_FAIL(test, "chain[%d] not signaled!", i);
 			goto err;
 		}
 
 		if (i + 1 < fc.chain_length &&
 		    dma_fence_is_signaled(fc.chains[i + 1])) {
-			pr_err("chain[%d] is signaled!\n", i);
-			err = -EINVAL;
+			KUNIT_FAIL(test, "chain[%d] is signaled!", i);
 			goto err;
 		}
 	}
 
 err:
 	fence_chains_fini(&fc);
-	return err;
 }
 
-static int signal_backward(void *arg)
+static void test_signal_backward(struct kunit *test)
 {
 	struct fence_chains fc;
 	int err;
 	int i;
 
 	err = fence_chains_init(&fc, 64, seqno_inc);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	for (i = fc.chain_length; i--; ) {
 		dma_fence_signal(fc.fences[i]);
 
 		if (i > 0 && dma_fence_is_signaled(fc.chains[i])) {
-			pr_err("chain[%d] is signaled!\n", i);
-			err = -EINVAL;
+			KUNIT_FAIL(test, "chain[%d] is signaled!", i);
 			goto err;
 		}
 	}
 
 	for (i = 0; i < fc.chain_length; i++) {
 		if (!dma_fence_is_signaled(fc.chains[i])) {
-			pr_err("chain[%d] was not signaled!\n", i);
-			err = -EINVAL;
+			KUNIT_FAIL(test, "chain[%d] was not signaled!", i);
 			goto err;
 		}
 	}
 
 err:
 	fence_chains_fini(&fc);
-	return err;
 }
 
 static int __wait_fence_chains(void *arg)
@@ -566,7 +534,7 @@ static int __wait_fence_chains(void *arg)
 	return 0;
 }
 
-static int wait_forward(void *arg)
+static void test_wait_forward(struct kunit *test)
 {
 	struct fence_chains fc;
 	struct task_struct *tsk;
@@ -574,12 +542,11 @@ static int wait_forward(void *arg)
 	int i;
 
 	err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
 	if (IS_ERR(tsk)) {
-		err = PTR_ERR(tsk);
+		KUNIT_FAIL(test, "Failed to create kthread");
 		goto err;
 	}
 	get_task_struct(tsk);
@@ -589,13 +556,13 @@ static int wait_forward(void *arg)
 		dma_fence_signal(fc.fences[i]);
 
 	err = kthread_stop_put(tsk);
+	KUNIT_EXPECT_EQ(test, err, 0);
 
 err:
 	fence_chains_fini(&fc);
-	return err;
 }
 
-static int wait_backward(void *arg)
+static void test_wait_backward(struct kunit *test)
 {
 	struct fence_chains fc;
 	struct task_struct *tsk;
@@ -603,12 +570,11 @@ static int wait_backward(void *arg)
 	int i;
 
 	err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
 	if (IS_ERR(tsk)) {
-		err = PTR_ERR(tsk);
+		KUNIT_FAIL(test, "Failed to create kthread");
 		goto err;
 	}
 	get_task_struct(tsk);
@@ -618,10 +584,10 @@ static int wait_backward(void *arg)
 		dma_fence_signal(fc.fences[i]);
 
 	err = kthread_stop_put(tsk);
+	KUNIT_EXPECT_EQ(test, err, 0);
 
 err:
 	fence_chains_fini(&fc);
-	return err;
 }
 
 static void randomise_fences(struct fence_chains *fc)
@@ -640,7 +606,7 @@ static void randomise_fences(struct fence_chains *fc)
 	}
 }
 
-static int wait_random(void *arg)
+static void test_wait_random(struct kunit *test)
 {
 	struct fence_chains fc;
 	struct task_struct *tsk;
@@ -648,14 +614,13 @@ static int wait_random(void *arg)
 	int i;
 
 	err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
-	if (err)
-		return err;
+	KUNIT_ASSERT_EQ_MSG(test, err, 0, "Failed to init fence chains");
 
 	randomise_fences(&fc);
 
 	tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
 	if (IS_ERR(tsk)) {
-		err = PTR_ERR(tsk);
+		KUNIT_FAIL(test, "Failed to create kthread");
 		goto err;
 	}
 	get_task_struct(tsk);
@@ -665,29 +630,14 @@ static int wait_random(void *arg)
 		dma_fence_signal(fc.fences[i]);
 
 	err = kthread_stop_put(tsk);
+	KUNIT_EXPECT_EQ(test, err, 0);
 
 err:
 	fence_chains_fini(&fc);
-	return err;
 }
 
-int dma_fence_chain(void)
+static int dma_fence_chain_suite_init(struct kunit_suite *suite)
 {
-	static const struct subtest tests[] = {
-		SUBTEST(sanitycheck),
-		SUBTEST(find_seqno),
-		SUBTEST(find_signaled),
-		SUBTEST(find_out_of_order),
-		SUBTEST(find_gap),
-		SUBTEST(find_race),
-		SUBTEST(signal_forward),
-		SUBTEST(signal_backward),
-		SUBTEST(wait_forward),
-		SUBTEST(wait_backward),
-		SUBTEST(wait_random),
-	};
-	int ret;
-
 	pr_info("sizeof(dma_fence_chain)=%zu\n",
 		sizeof(struct dma_fence_chain));
 
@@ -696,9 +646,34 @@ int dma_fence_chain(void)
 				 SLAB_HWCACHE_ALIGN);
 	if (!slab_fences)
 		return -ENOMEM;
+	return 0;
+}
 
-	ret = subtests(tests, NULL);
-
+static void dma_fence_chain_suite_exit(struct kunit_suite *suite)
+{
 	kmem_cache_destroy(slab_fences);
-	return ret;
 }
+
+static struct kunit_case dma_fence_chain_cases[] = {
+	KUNIT_CASE(test_sanitycheck),
+	KUNIT_CASE(test_find_seqno),
+	KUNIT_CASE(test_find_signaled),
+	KUNIT_CASE(test_find_out_of_order),
+	KUNIT_CASE(test_find_gap),
+	KUNIT_CASE(test_find_race),
+	KUNIT_CASE(test_signal_forward),
+	KUNIT_CASE(test_signal_backward),
+	KUNIT_CASE(test_wait_forward),
+	KUNIT_CASE(test_wait_backward),
+	KUNIT_CASE(test_wait_random),
+	{}
+};
+
+static struct kunit_suite dma_fence_chain_test_suite = {
+	.name = "dma-buf-fence-chain",
+	.suite_init = dma_fence_chain_suite_init,
+	.suite_exit = dma_fence_chain_suite_exit,
+	.test_cases = dma_fence_chain_cases,
+};
+
+kunit_test_suite(dma_fence_chain_test_suite);
diff --git a/drivers/dma-buf/st-dma-fence-unwrap.c b/drivers/dma-buf/st-dma-fence-unwrap.c
index 9c74195f47fd3..4e7ee25372ba6 100644
--- a/drivers/dma-buf/st-dma-fence-unwrap.c
+++ b/drivers/dma-buf/st-dma-fence-unwrap.c
@@ -4,13 +4,12 @@
  * Copyright (C) 2022 Advanced Micro Devices, Inc.
  */
 
+#include <kunit/test.h>
 #include <linux/dma-fence.h>
 #include <linux/dma-fence-array.h>
 #include <linux/dma-fence-chain.h>
 #include <linux/dma-fence-unwrap.h>
 
-#include "selftest.h"
-
 #define CHAIN_SZ (4 << 10)
 
 struct mock_fence {
@@ -65,7 +64,7 @@ static struct dma_fence *mock_array(unsigned int num_fences, ...)
 
 	array = dma_fence_array_create(num_fences, fences,
 				       dma_fence_context_alloc(1),
-				       1, false);
+				       1);
 	if (!array)
 		goto error_free;
 	return &array->base;
@@ -97,52 +96,45 @@ static struct dma_fence *mock_chain(struct dma_fence *prev,
 	return &f->base;
 }
 
-static int sanitycheck(void *arg)
+static void test_sanitycheck(struct kunit *test)
 {
 	struct dma_fence *f, *chain, *array;
-	int err = 0;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	array = mock_array(1, f);
-	if (!array)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, array);
 
 	chain = mock_chain(NULL, array);
-	if (!chain)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, chain);
 
 	dma_fence_put(chain);
-	return err;
 }
 
-static int unwrap_array(void *arg)
+static void test_unwrap_array(struct kunit *test)
 {
 	struct dma_fence *fence, *f1, *f2, *array;
 	struct dma_fence_unwrap iter;
-	int err = 0;
 
 	f1 = mock_fence();
-	if (!f1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f1);
 
 	dma_fence_enable_sw_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		dma_fence_put(f1);
-		return -ENOMEM;
+		return;
 	}
 
 	dma_fence_enable_sw_signaling(f2);
 
 	array = mock_array(2, f1, f2);
-	if (!array)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, array);
 
 	dma_fence_unwrap_for_each(fence, &iter, array) {
 		if (fence == f1) {
@@ -150,43 +142,37 @@ static int unwrap_array(void *arg)
 		} else if (fence == f2) {
 			f2 = NULL;
 		} else {
-			pr_err("Unexpected fence!\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Unexpected fence!");
 		}
 	}
 
-	if (f1 || f2) {
-		pr_err("Not all fences seen!\n");
-		err = -EINVAL;
-	}
+	if (f1 || f2)
+		KUNIT_FAIL(test, "Not all fences seen!");
 
 	dma_fence_put(array);
-	return err;
 }
 
-static int unwrap_chain(void *arg)
+static void test_unwrap_chain(struct kunit *test)
 {
 	struct dma_fence *fence, *f1, *f2, *chain;
 	struct dma_fence_unwrap iter;
-	int err = 0;
 
 	f1 = mock_fence();
-	if (!f1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f1);
 
 	dma_fence_enable_sw_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		dma_fence_put(f1);
-		return -ENOMEM;
+		return;
 	}
 
 	dma_fence_enable_sw_signaling(f2);
 
 	chain = mock_chain(f1, f2);
-	if (!chain)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, chain);
 
 	dma_fence_unwrap_for_each(fence, &iter, chain) {
 		if (fence == f1) {
@@ -194,47 +180,40 @@ static int unwrap_chain(void *arg)
 		} else if (fence == f2) {
 			f2 = NULL;
 		} else {
-			pr_err("Unexpected fence!\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Unexpected fence!");
 		}
 	}
 
-	if (f1 || f2) {
-		pr_err("Not all fences seen!\n");
-		err = -EINVAL;
-	}
+	if (f1 || f2)
+		KUNIT_FAIL(test, "Not all fences seen!");
 
 	dma_fence_put(chain);
-	return err;
 }
 
-static int unwrap_chain_array(void *arg)
+static void test_unwrap_chain_array(struct kunit *test)
 {
 	struct dma_fence *fence, *f1, *f2, *array, *chain;
 	struct dma_fence_unwrap iter;
-	int err = 0;
 
 	f1 = mock_fence();
-	if (!f1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f1);
 
 	dma_fence_enable_sw_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		dma_fence_put(f1);
-		return -ENOMEM;
+		return;
 	}
 
 	dma_fence_enable_sw_signaling(f2);
 
 	array = mock_array(2, f1, f2);
-	if (!array)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, array);
 
 	chain = mock_chain(NULL, array);
-	if (!chain)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, chain);
 
 	dma_fence_unwrap_for_each(fence, &iter, chain) {
 		if (fence == f1) {
@@ -242,35 +221,29 @@ static int unwrap_chain_array(void *arg)
 		} else if (fence == f2) {
 			f2 = NULL;
 		} else {
-			pr_err("Unexpected fence!\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Unexpected fence!");
 		}
 	}
 
-	if (f1 || f2) {
-		pr_err("Not all fences seen!\n");
-		err = -EINVAL;
-	}
+	if (f1 || f2)
+		KUNIT_FAIL(test, "Not all fences seen!");
 
 	dma_fence_put(chain);
-	return err;
 }
 
-static int unwrap_merge(void *arg)
+static void test_unwrap_merge(struct kunit *test)
 {
 	struct dma_fence *fence, *f1, *f2, *f3;
 	struct dma_fence_unwrap iter;
-	int err = 0;
 
 	f1 = mock_fence();
-	if (!f1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f1);
 
 	dma_fence_enable_sw_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
-		err = -ENOMEM;
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		goto error_put_f1;
 	}
 
@@ -278,7 +251,7 @@ static int unwrap_merge(void *arg)
 
 	f3 = dma_fence_unwrap_merge(f1, f2);
 	if (!f3) {
-		err = -ENOMEM;
+		KUNIT_FAIL(test, "Failed to merge fences");
 		goto error_put_f2;
 	}
 
@@ -290,39 +263,33 @@ static int unwrap_merge(void *arg)
 			dma_fence_put(f2);
 			f2 = NULL;
 		} else {
-			pr_err("Unexpected fence!\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Unexpected fence!");
 		}
 	}
 
-	if (f1 || f2) {
-		pr_err("Not all fences seen!\n");
-		err = -EINVAL;
-	}
+	if (f1 || f2)
+		KUNIT_FAIL(test, "Not all fences seen!");
 
 	dma_fence_put(f3);
 error_put_f2:
 	dma_fence_put(f2);
 error_put_f1:
 	dma_fence_put(f1);
-	return err;
 }
 
-static int unwrap_merge_duplicate(void *arg)
+static void test_unwrap_merge_duplicate(struct kunit *test)
 {
 	struct dma_fence *fence, *f1, *f2;
 	struct dma_fence_unwrap iter;
-	int err = 0;
 
 	f1 = mock_fence();
-	if (!f1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f1);
 
 	dma_fence_enable_sw_signaling(f1);
 
 	f2 = dma_fence_unwrap_merge(f1, f1);
 	if (!f2) {
-		err = -ENOMEM;
+		KUNIT_FAIL(test, "Failed to merge fences");
 		goto error_put_f1;
 	}
 
@@ -331,41 +298,35 @@ static int unwrap_merge_duplicate(void *arg)
 			dma_fence_put(f1);
 			f1 = NULL;
 		} else {
-			pr_err("Unexpected fence!\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Unexpected fence!");
 		}
 	}
 
-	if (f1) {
-		pr_err("Not all fences seen!\n");
-		err = -EINVAL;
-	}
+	if (f1)
+		KUNIT_FAIL(test, "Not all fences seen!");
 
 	dma_fence_put(f2);
 error_put_f1:
 	dma_fence_put(f1);
-	return err;
 }
 
-static int unwrap_merge_seqno(void *arg)
+static void test_unwrap_merge_seqno(struct kunit *test)
 {
 	struct dma_fence *fence, *f1, *f2, *f3, *f4;
 	struct dma_fence_unwrap iter;
-	int err = 0;
 	u64 ctx[2];
 
 	ctx[0] = dma_fence_context_alloc(1);
 	ctx[1] = dma_fence_context_alloc(1);
 
 	f1 = __mock_fence(ctx[1], 1);
-	if (!f1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f1);
 
 	dma_fence_enable_sw_signaling(f1);
 
 	f2 = __mock_fence(ctx[1], 2);
 	if (!f2) {
-		err = -ENOMEM;
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		goto error_put_f1;
 	}
 
@@ -373,7 +334,7 @@ static int unwrap_merge_seqno(void *arg)
 
 	f3 = __mock_fence(ctx[0], 1);
 	if (!f3) {
-		err = -ENOMEM;
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		goto error_put_f2;
 	}
 
@@ -381,7 +342,7 @@ static int unwrap_merge_seqno(void *arg)
 
 	f4 = dma_fence_unwrap_merge(f1, f2, f3);
 	if (!f4) {
-		err = -ENOMEM;
+		KUNIT_FAIL(test, "Failed to merge fences");
 		goto error_put_f3;
 	}
 
@@ -393,15 +354,12 @@ static int unwrap_merge_seqno(void *arg)
 			dma_fence_put(f2);
 			f2 = NULL;
 		} else {
-			pr_err("Unexpected fence!\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Unexpected fence!");
 		}
 	}
 
-	if (f2 || f3) {
-		pr_err("Not all fences seen!\n");
-		err = -EINVAL;
-	}
+	if (f2 || f3)
+		KUNIT_FAIL(test, "Not all fences seen!");
 
 	dma_fence_put(f4);
 error_put_f3:
@@ -410,40 +368,41 @@ static int unwrap_merge_seqno(void *arg)
 	dma_fence_put(f2);
 error_put_f1:
 	dma_fence_put(f1);
-	return err;
 }
 
-static int unwrap_merge_order(void *arg)
+static void test_unwrap_merge_order(struct kunit *test)
 {
 	struct dma_fence *fence, *f1, *f2, *a1, *a2, *c1, *c2;
 	struct dma_fence_unwrap iter;
-	int err = 0;
 
 	f1 = mock_fence();
-	if (!f1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f1);
 
 	dma_fence_enable_sw_signaling(f1);
 
 	f2 = mock_fence();
 	if (!f2) {
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		dma_fence_put(f1);
-		return -ENOMEM;
+		return;
 	}
 
 	dma_fence_enable_sw_signaling(f2);
 
 	a1 = mock_array(2, f1, f2);
-	if (!a1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, a1);
 
 	c1 = mock_chain(NULL, dma_fence_get(f1));
-	if (!c1)
+	if (!c1) {
+		KUNIT_FAIL(test, "Failed to create chain");
 		goto error_put_a1;
+	}
 
 	c2 = mock_chain(c1, dma_fence_get(f2));
-	if (!c2)
+	if (!c2) {
+		KUNIT_FAIL(test, "Failed to create chain");
 		goto error_put_a1;
+	}
 
 	/*
 	 * The fences in the chain are the same as in a1 but in oposite order,
@@ -455,63 +414,64 @@ static int unwrap_merge_order(void *arg)
 		if (fence == f1) {
 			f1 = NULL;
 			if (!f2)
-				pr_err("Unexpected order!\n");
+				KUNIT_FAIL(test, "Unexpected order!");
 		} else if (fence == f2) {
 			f2 = NULL;
 			if (f1)
-				pr_err("Unexpected order!\n");
+				KUNIT_FAIL(test, "Unexpected order!");
 		} else {
-			pr_err("Unexpected fence!\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Unexpected fence!");
 		}
 	}
 
-	if (f1 || f2) {
-		pr_err("Not all fences seen!\n");
-		err = -EINVAL;
-	}
+	if (f1 || f2)
+		KUNIT_FAIL(test, "Not all fences seen!");
 
 	dma_fence_put(a2);
-	return err;
+	return;
 
 error_put_a1:
 	dma_fence_put(a1);
-	return -ENOMEM;
 }
 
-static int unwrap_merge_complex(void *arg)
+static void test_unwrap_merge_complex(struct kunit *test)
 {
 	struct dma_fence *fence, *f1, *f2, *f3, *f4, *f5;
 	struct dma_fence_unwrap iter;
-	int err = -ENOMEM;
 
 	f1 = mock_fence();
-	if (!f1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f1);
 
 	dma_fence_enable_sw_signaling(f1);
 
 	f2 = mock_fence();
-	if (!f2)
+	if (!f2) {
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		goto error_put_f1;
+	}
 
 	dma_fence_enable_sw_signaling(f2);
 
 	f3 = dma_fence_unwrap_merge(f1, f2);
-	if (!f3)
+	if (!f3) {
+		KUNIT_FAIL(test, "Failed to merge fences");
 		goto error_put_f2;
+	}
 
 	/* The resulting array has the fences in reverse */
 	f4 = mock_array(2, dma_fence_get(f2), dma_fence_get(f1));
-	if (!f4)
+	if (!f4) {
+		KUNIT_FAIL(test, "Failed to create array");
 		goto error_put_f3;
+	}
 
 	/* Signaled fences should be filtered, the two arrays merged. */
 	f5 = dma_fence_unwrap_merge(f3, f4, dma_fence_get_stub());
-	if (!f5)
+	if (!f5) {
+		KUNIT_FAIL(test, "Failed to merge fences");
 		goto error_put_f4;
+	}
 
-	err = 0;
 	dma_fence_unwrap_for_each(fence, &iter, f5) {
 		if (fence == f1) {
 			dma_fence_put(f1);
@@ -520,15 +480,12 @@ static int unwrap_merge_complex(void *arg)
 			dma_fence_put(f2);
 			f2 = NULL;
 		} else {
-			pr_err("Unexpected fence!\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Unexpected fence!");
 		}
 	}
 
-	if (f1 || f2) {
-		pr_err("Not all fences seen!\n");
-		err = -EINVAL;
-	}
+	if (f1 || f2)
+		KUNIT_FAIL(test, "Not all fences seen!");
 
 	dma_fence_put(f5);
 error_put_f4:
@@ -539,56 +496,64 @@ static int unwrap_merge_complex(void *arg)
 	dma_fence_put(f2);
 error_put_f1:
 	dma_fence_put(f1);
-	return err;
 }
 
-static int unwrap_merge_complex_seqno(void *arg)
+static void test_unwrap_merge_complex_seqno(struct kunit *test)
 {
 	struct dma_fence *fence, *f1, *f2, *f3, *f4, *f5, *f6, *f7;
 	struct dma_fence_unwrap iter;
-	int err = -ENOMEM;
 	u64 ctx[2];
 
 	ctx[0] = dma_fence_context_alloc(1);
 	ctx[1] = dma_fence_context_alloc(1);
 
 	f1 = __mock_fence(ctx[0], 2);
-	if (!f1)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f1);
 
 	dma_fence_enable_sw_signaling(f1);
 
 	f2 = __mock_fence(ctx[1], 1);
-	if (!f2)
+	if (!f2) {
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		goto error_put_f1;
+	}
 
 	dma_fence_enable_sw_signaling(f2);
 
 	f3 = __mock_fence(ctx[0], 1);
-	if (!f3)
+	if (!f3) {
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		goto error_put_f2;
+	}
 
 	dma_fence_enable_sw_signaling(f3);
 
 	f4 = __mock_fence(ctx[1], 2);
-	if (!f4)
+	if (!f4) {
+		KUNIT_FAIL(test, "Failed to create mock fence");
 		goto error_put_f3;
+	}
 
 	dma_fence_enable_sw_signaling(f4);
 
 	f5 = mock_array(2, dma_fence_get(f1), dma_fence_get(f2));
-	if (!f5)
+	if (!f5) {
+		KUNIT_FAIL(test, "Failed to create array");
 		goto error_put_f4;
+	}
 
 	f6 = mock_array(2, dma_fence_get(f3), dma_fence_get(f4));
-	if (!f6)
+	if (!f6) {
+		KUNIT_FAIL(test, "Failed to create array");
 		goto error_put_f5;
+	}
 
 	f7 = dma_fence_unwrap_merge(f5, f6);
-	if (!f7)
+	if (!f7) {
+		KUNIT_FAIL(test, "Failed to merge fences");
 		goto error_put_f6;
+	}
 
-	err = 0;
 	dma_fence_unwrap_for_each(fence, &iter, f7) {
 		if (fence == f1 && f4) {
 			dma_fence_put(f1);
@@ -597,15 +562,12 @@ static int unwrap_merge_complex_seqno(void *arg)
 			dma_fence_put(f4);
 			f4 = NULL;
 		} else {
-			pr_err("Unexpected fence!\n");
-			err = -EINVAL;
+			KUNIT_FAIL(test, "Unexpected fence!");
 		}
 	}
 
-	if (f1 || f4) {
-		pr_err("Not all fences seen!\n");
-		err = -EINVAL;
-	}
+	if (f1 || f4)
+		KUNIT_FAIL(test, "Not all fences seen!");
 
 	dma_fence_put(f7);
 error_put_f6:
@@ -620,23 +582,25 @@ static int unwrap_merge_complex_seqno(void *arg)
 	dma_fence_put(f2);
 error_put_f1:
 	dma_fence_put(f1);
-	return err;
 }
 
-int dma_fence_unwrap(void)
-{
-	static const struct subtest tests[] = {
-		SUBTEST(sanitycheck),
-		SUBTEST(unwrap_array),
-		SUBTEST(unwrap_chain),
-		SUBTEST(unwrap_chain_array),
-		SUBTEST(unwrap_merge),
-		SUBTEST(unwrap_merge_duplicate),
-		SUBTEST(unwrap_merge_seqno),
-		SUBTEST(unwrap_merge_order),
-		SUBTEST(unwrap_merge_complex),
-		SUBTEST(unwrap_merge_complex_seqno),
-	};
-
-	return subtests(tests, NULL);
-}
+static struct kunit_case dma_fence_unwrap_cases[] = {
+	KUNIT_CASE(test_sanitycheck),
+	KUNIT_CASE(test_unwrap_array),
+	KUNIT_CASE(test_unwrap_chain),
+	KUNIT_CASE(test_unwrap_chain_array),
+	KUNIT_CASE(test_unwrap_merge),
+	KUNIT_CASE(test_unwrap_merge_duplicate),
+	KUNIT_CASE(test_unwrap_merge_seqno),
+	KUNIT_CASE(test_unwrap_merge_order),
+	KUNIT_CASE(test_unwrap_merge_complex),
+	KUNIT_CASE(test_unwrap_merge_complex_seqno),
+	{}
+};
+
+static struct kunit_suite dma_fence_unwrap_test_suite = {
+	.name = "dma-buf-fence-unwrap",
+	.test_cases = dma_fence_unwrap_cases,
+};
+
+kunit_test_suite(dma_fence_unwrap_test_suite);
diff --git a/drivers/dma-buf/st-dma-fence.c b/drivers/dma-buf/st-dma-fence.c
index 0d9d524d79b6d..4992722296968 100644
--- a/drivers/dma-buf/st-dma-fence.c
+++ b/drivers/dma-buf/st-dma-fence.c
@@ -4,6 +4,7 @@
  * Copyright © 2019 Intel Corporation
  */
 
+#include <kunit/test.h>
 #include <linux/delay.h>
 #include <linux/dma-fence.h>
 #include <linux/kernel.h>
@@ -12,8 +13,6 @@
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 
-#include "selftest.h"
-
 static const char *mock_name(struct dma_fence *f)
 {
 	return "mock";
@@ -36,62 +35,55 @@ static struct dma_fence *mock_fence(void)
 	return f;
 }
 
-static int sanitycheck(void *arg)
+static void test_sanitycheck(struct kunit *test)
 {
 	struct dma_fence *f;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	dma_fence_signal(f);
 	dma_fence_put(f);
-
-	return 0;
 }
 
-static int test_signaling(void *arg)
+static void test_signaling(struct kunit *test)
 {
 	struct dma_fence *f;
-	int err = -EINVAL;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	if (dma_fence_is_signaled(f)) {
-		pr_err("Fence unexpectedly signaled on creation\n");
+		KUNIT_FAIL(test, "Fence unexpectedly signaled on creation");
 		goto err_free;
 	}
 
 	if (dma_fence_check_and_signal(f)) {
-		pr_err("Fence reported being already signaled\n");
+		KUNIT_FAIL(test, "Fence reported being already signaled");
 		goto err_free;
 	}
 
 	if (!dma_fence_is_signaled(f)) {
-		pr_err("Fence not reporting signaled\n");
+		KUNIT_FAIL(test, "Fence not reporting signaled");
 		goto err_free;
 	}
 
 	if (!dma_fence_test_signaled_flag(f)) {
-		pr_err("Fence reported not being already signaled\n");
+		KUNIT_FAIL(test, "Fence reported not being already signaled");
 		goto err_free;
 	}
 
 	if (rcu_dereference_protected(f->ops, true)) {
-		pr_err("Fence ops not cleared on signal\n");
+		KUNIT_FAIL(test, "Fence ops not cleared on signal");
 		goto err_free;
 	}
 
-	err = 0;
 err_free:
 	dma_fence_put(f);
-	return err;
 }
 
 struct simple_cb {
@@ -104,215 +96,187 @@ static void simple_callback(struct dma_fence *f, struct dma_fence_cb *cb)
 	smp_store_mb(container_of(cb, struct simple_cb, cb)->seen, true);
 }
 
-static int test_add_callback(void *arg)
+static void test_add_callback(struct kunit *test)
 {
 	struct simple_cb cb = {};
 	struct dma_fence *f;
-	int err = -EINVAL;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
-		pr_err("Failed to add callback, fence already signaled!\n");
+		KUNIT_FAIL(test, "Failed to add callback, fence already signaled!");
 		goto err_free;
 	}
 
 	dma_fence_signal(f);
 	if (!cb.seen) {
-		pr_err("Callback failed!\n");
+		KUNIT_FAIL(test, "Callback failed!");
 		goto err_free;
 	}
 
-	err = 0;
 err_free:
 	dma_fence_put(f);
-	return err;
 }
 
-static int test_late_add_callback(void *arg)
+static void test_late_add_callback(struct kunit *test)
 {
 	struct simple_cb cb = {};
 	struct dma_fence *f;
-	int err = -EINVAL;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	dma_fence_signal(f);
 
 	if (!dma_fence_add_callback(f, &cb.cb, simple_callback)) {
-		pr_err("Added callback, but fence was already signaled!\n");
+		KUNIT_FAIL(test, "Added callback, but fence was already signaled!");
 		goto err_free;
 	}
 
 	dma_fence_signal(f);
 	if (cb.seen) {
-		pr_err("Callback called after failed attachment !\n");
+		KUNIT_FAIL(test, "Callback called after failed attachment!");
 		goto err_free;
 	}
 
-	err = 0;
 err_free:
 	dma_fence_put(f);
-	return err;
 }
 
-static int test_rm_callback(void *arg)
+static void test_rm_callback(struct kunit *test)
 {
 	struct simple_cb cb = {};
 	struct dma_fence *f;
-	int err = -EINVAL;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
-		pr_err("Failed to add callback, fence already signaled!\n");
+		KUNIT_FAIL(test, "Failed to add callback, fence already signaled!");
 		goto err_free;
 	}
 
 	if (!dma_fence_remove_callback(f, &cb.cb)) {
-		pr_err("Failed to remove callback!\n");
+		KUNIT_FAIL(test, "Failed to remove callback!");
 		goto err_free;
 	}
 
 	dma_fence_signal(f);
 	if (cb.seen) {
-		pr_err("Callback still signaled after removal!\n");
+		KUNIT_FAIL(test, "Callback still signaled after removal!");
 		goto err_free;
 	}
 
-	err = 0;
 err_free:
 	dma_fence_put(f);
-	return err;
 }
 
-static int test_late_rm_callback(void *arg)
+static void test_late_rm_callback(struct kunit *test)
 {
 	struct simple_cb cb = {};
 	struct dma_fence *f;
-	int err = -EINVAL;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	if (dma_fence_add_callback(f, &cb.cb, simple_callback)) {
-		pr_err("Failed to add callback, fence already signaled!\n");
+		KUNIT_FAIL(test, "Failed to add callback, fence already signaled!");
 		goto err_free;
 	}
 
 	dma_fence_signal(f);
 	if (!cb.seen) {
-		pr_err("Callback failed!\n");
+		KUNIT_FAIL(test, "Callback failed!");
 		goto err_free;
 	}
 
 	if (dma_fence_remove_callback(f, &cb.cb)) {
-		pr_err("Callback removal succeed after being executed!\n");
+		KUNIT_FAIL(test, "Callback removal succeeded after being executed!");
 		goto err_free;
 	}
 
-	err = 0;
 err_free:
 	dma_fence_put(f);
-	return err;
 }
 
-static int test_status(void *arg)
+static void test_status(struct kunit *test)
 {
 	struct dma_fence *f;
-	int err = -EINVAL;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	if (dma_fence_get_status(f)) {
-		pr_err("Fence unexpectedly has signaled status on creation\n");
+		KUNIT_FAIL(test, "Fence unexpectedly has signaled status on creation");
 		goto err_free;
 	}
 
 	dma_fence_signal(f);
 	if (!dma_fence_get_status(f)) {
-		pr_err("Fence not reporting signaled status\n");
+		KUNIT_FAIL(test, "Fence not reporting signaled status");
 		goto err_free;
 	}
 
-	err = 0;
 err_free:
 	dma_fence_put(f);
-	return err;
 }
 
-static int test_error(void *arg)
+static void test_error(struct kunit *test)
 {
 	struct dma_fence *f;
-	int err = -EINVAL;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	dma_fence_set_error(f, -EIO);
 
 	if (dma_fence_get_status(f)) {
-		pr_err("Fence unexpectedly has error status before signal\n");
+		KUNIT_FAIL(test, "Fence unexpectedly has error status before signal");
 		goto err_free;
 	}
 
 	dma_fence_signal(f);
 	if (dma_fence_get_status(f) != -EIO) {
-		pr_err("Fence not reporting error status, got %d\n",
-		       dma_fence_get_status(f));
+		KUNIT_FAIL(test, "Fence not reporting error status, got %d",
+			   dma_fence_get_status(f));
 		goto err_free;
 	}
 
-	err = 0;
 err_free:
 	dma_fence_put(f);
-	return err;
 }
 
-static int test_wait(void *arg)
+static void test_wait(struct kunit *test)
 {
 	struct dma_fence *f;
-	int err = -EINVAL;
 
 	f = mock_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	if (dma_fence_wait_timeout(f, false, 0) != 0) {
-		pr_err("Wait reported complete before being signaled\n");
+		KUNIT_FAIL(test, "Wait reported complete before being signaled");
 		goto err_free;
 	}
 
 	dma_fence_signal(f);
 
 	if (dma_fence_wait_timeout(f, false, 0) != 1) {
-		pr_err("Wait reported incomplete after being signaled\n");
+		KUNIT_FAIL(test, "Wait reported incomplete after being signaled");
 		goto err_free;
 	}
 
-	err = 0;
 err_free:
 	dma_fence_signal(f);
 	dma_fence_put(f);
-	return err;
 }
 
 struct wait_timer {
@@ -327,21 +291,19 @@ static void wait_timer(struct timer_list *timer)
 	dma_fence_signal(wt->f);
 }
 
-static int test_wait_timeout(void *arg)
+static void test_wait_timeout(struct kunit *test)
 {
 	struct wait_timer wt;
-	int err = -EINVAL;
 
 	timer_setup_on_stack(&wt.timer, wait_timer, 0);
 
 	wt.f = mock_fence();
-	if (!wt.f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, wt.f);
 
 	dma_fence_enable_sw_signaling(wt.f);
 
 	if (dma_fence_wait_timeout(wt.f, false, 1) != 0) {
-		pr_err("Wait reported complete before being signaled\n");
+		KUNIT_FAIL(test, "Wait reported complete before being signaled");
 		goto err_free;
 	}
 
@@ -349,42 +311,38 @@ static int test_wait_timeout(void *arg)
 
 	if (dma_fence_wait_timeout(wt.f, false, HZ) == 0) {
 		if (timer_pending(&wt.timer)) {
-			pr_notice("Timer did not fire within one HZ!\n");
-			err = 0; /* not our fault! */
+			kunit_mark_skipped(
+				test, "Timer did not fire within on HZ!\n");
 		} else {
-			pr_err("Wait reported incomplete after timeout\n");
+			KUNIT_FAIL(test,
+				   "Wait reported incomplete after timeout");
 		}
 		goto err_free;
 	}
 
-	err = 0;
 err_free:
 	timer_delete_sync(&wt.timer);
 	timer_destroy_on_stack(&wt.timer);
 	dma_fence_signal(wt.f);
 	dma_fence_put(wt.f);
-	return err;
 }
 
-static int test_stub(void *arg)
+static void test_stub(struct kunit *test)
 {
 	struct dma_fence *f[64];
-	int err = -EINVAL;
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(f); i++) {
 		f[i] = dma_fence_get_stub();
 		if (!dma_fence_is_signaled(f[i])) {
-			pr_err("Obtained unsignaled stub fence!\n");
+			KUNIT_FAIL(test, "Obtained unsignaled stub fence!");
 			goto err;
 		}
 	}
 
-	err = 0;
 err:
 	while (i--)
 		dma_fence_put(f[i]);
-	return err;
 }
 
 /* Now off to the races! */
@@ -473,12 +431,19 @@ static int thread_signal_callback(void *arg)
 	return err;
 }
 
-static int race_signal_callback(void *arg)
+static void test_race_signal_callback(struct kunit *test)
 {
 	struct dma_fence __rcu *f[2] = {};
 	int ret = 0;
 	int pass;
 
+	/*
+	 * thread_signal_callback() spins under RCU and it cannot make forward
+	 * progress unless the threads are truly running concurrently.
+	 */
+	if (num_online_cpus() < 2)
+		kunit_skip(test, "requires at least 2 CPUs");
+
 	for (pass = 0; !ret && pass <= 1; pass++) {
 		struct race_thread t[2];
 		int i;
@@ -490,10 +455,10 @@ static int race_signal_callback(void *arg)
 			t[i].task = kthread_run(thread_signal_callback, &t[i],
 						"dma-fence:%d", i);
 			if (IS_ERR(t[i].task)) {
-				ret = PTR_ERR(t[i].task);
+				KUNIT_FAIL(test, "Failed to create kthread");
 				while (--i >= 0)
 					kthread_stop_put(t[i].task);
-				return ret;
+				return;
 			}
 			get_task_struct(t[i].task);
 		}
@@ -509,26 +474,35 @@ static int race_signal_callback(void *arg)
 		}
 	}
 
-	return ret;
+	KUNIT_EXPECT_EQ(test, ret, 0);
 }
 
-int dma_fence(void)
+static int dma_fence_suite_init(struct kunit_suite *suite)
 {
-	static const struct subtest tests[] = {
-		SUBTEST(sanitycheck),
-		SUBTEST(test_signaling),
-		SUBTEST(test_add_callback),
-		SUBTEST(test_late_add_callback),
-		SUBTEST(test_rm_callback),
-		SUBTEST(test_late_rm_callback),
-		SUBTEST(test_status),
-		SUBTEST(test_error),
-		SUBTEST(test_wait),
-		SUBTEST(test_wait_timeout),
-		SUBTEST(test_stub),
-		SUBTEST(race_signal_callback),
-	};
-
 	pr_info("sizeof(dma_fence)=%zu\n", sizeof(struct dma_fence));
-	return subtests(tests, NULL);
+	return 0;
 }
+
+static struct kunit_case dma_fence_cases[] = {
+	KUNIT_CASE(test_sanitycheck),
+	KUNIT_CASE(test_signaling),
+	KUNIT_CASE(test_add_callback),
+	KUNIT_CASE(test_late_add_callback),
+	KUNIT_CASE(test_rm_callback),
+	KUNIT_CASE(test_late_rm_callback),
+	KUNIT_CASE(test_status),
+	KUNIT_CASE(test_error),
+	KUNIT_CASE(test_wait),
+	KUNIT_CASE(test_wait_timeout),
+	KUNIT_CASE(test_stub),
+	KUNIT_CASE(test_race_signal_callback),
+	{}
+};
+
+static struct kunit_suite dma_fence_test_suite = {
+	.name = "dma-buf-fence",
+	.suite_init = dma_fence_suite_init,
+	.test_cases = dma_fence_cases,
+};
+
+kunit_test_suite(dma_fence_test_suite);
diff --git a/drivers/dma-buf/st-dma-resv.c b/drivers/dma-buf/st-dma-resv.c
index ad4dfb49dcd9f..95a4becdb8926 100644
--- a/drivers/dma-buf/st-dma-resv.c
+++ b/drivers/dma-buf/st-dma-resv.c
@@ -5,13 +5,17 @@
 * Copyright © 2021 Advanced Micro Devices, Inc.
 */
 
+#include <kunit/test.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
 #include <linux/dma-resv.h>
 
-#include "selftest.h"
+static DEFINE_SPINLOCK(fence_lock);
 
-static struct spinlock fence_lock;
+struct dma_resv_usage_param {
+	enum dma_resv_usage usage;
+	const char *desc;
+};
 
 static const char *fence_name(struct dma_fence *f)
 {
@@ -35,15 +39,14 @@ static struct dma_fence *alloc_fence(void)
 	return f;
 }
 
-static int sanitycheck(void *arg)
+static void test_sanitycheck(struct kunit *test)
 {
 	struct dma_resv resv;
 	struct dma_fence *f;
 	int r;
 
 	f = alloc_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
@@ -53,49 +56,46 @@ static int sanitycheck(void *arg)
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
 	if (r)
-		pr_err("Resv locking failed\n");
+		KUNIT_FAIL(test, "Resv locking failed\n");
 	else
 		dma_resv_unlock(&resv);
 	dma_resv_fini(&resv);
-	return r;
 }
 
-static int test_signaling(void *arg)
+static void test_signaling(struct kunit *test)
 {
-	enum dma_resv_usage usage = (unsigned long)arg;
+	const struct dma_resv_usage_param *param = test->param_value;
+	enum dma_resv_usage usage = param->usage;
 	struct dma_resv resv;
 	struct dma_fence *f;
 	int r;
 
 	f = alloc_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
 	if (r) {
-		pr_err("Resv locking failed\n");
+		KUNIT_FAIL(test, "Resv locking failed");
 		goto err_free;
 	}
 
 	r = dma_resv_reserve_fences(&resv, 1);
 	if (r) {
-		pr_err("Resv shared slot allocation failed\n");
+		KUNIT_FAIL(test, "Resv shared slot allocation failed");
 		goto err_unlock;
 	}
 
 	dma_resv_add_fence(&resv, f, usage);
 	if (dma_resv_test_signaled(&resv, usage)) {
-		pr_err("Resv unexpectedly signaled\n");
-		r = -EINVAL;
+		KUNIT_FAIL(test, "Resv unexpectedly signaled");
 		goto err_unlock;
 	}
 	dma_fence_signal(f);
 	if (!dma_resv_test_signaled(&resv, usage)) {
-		pr_err("Resv not reporting signaled\n");
-		r = -EINVAL;
+		KUNIT_FAIL(test, "Resv not reporting signaled");
 		goto err_unlock;
 	}
 err_unlock:
@@ -103,33 +103,32 @@ static int test_signaling(void *arg)
 err_free:
 	dma_resv_fini(&resv);
 	dma_fence_put(f);
-	return r;
 }
 
-static int test_for_each(void *arg)
+static void test_for_each(struct kunit *test)
 {
-	enum dma_resv_usage usage = (unsigned long)arg;
+	const struct dma_resv_usage_param *param = test->param_value;
+	enum dma_resv_usage usage = param->usage;
 	struct dma_resv_iter cursor;
 	struct dma_fence *f, *fence;
 	struct dma_resv resv;
 	int r;
 
 	f = alloc_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
 	if (r) {
-		pr_err("Resv locking failed\n");
+		KUNIT_FAIL(test, "Resv locking failed");
 		goto err_free;
 	}
 
 	r = dma_resv_reserve_fences(&resv, 1);
 	if (r) {
-		pr_err("Resv shared slot allocation failed\n");
+		KUNIT_FAIL(test, "Resv shared slot allocation failed");
 		goto err_unlock;
 	}
 
@@ -138,24 +137,23 @@ static int test_for_each(void *arg)
 	r = -ENOENT;
 	dma_resv_for_each_fence(&cursor, &resv, usage, fence) {
 		if (!r) {
-			pr_err("More than one fence found\n");
-			r = -EINVAL;
+			KUNIT_FAIL(test, "More than one fence found");
 			goto err_unlock;
 		}
 		if (f != fence) {
-			pr_err("Unexpected fence\n");
+			KUNIT_FAIL(test, "Unexpected fence");
 			r = -EINVAL;
 			goto err_unlock;
 		}
 		if (dma_resv_iter_usage(&cursor) != usage) {
-			pr_err("Unexpected fence usage\n");
+			KUNIT_FAIL(test, "Unexpected fence usage");
 			r = -EINVAL;
 			goto err_unlock;
 		}
 		r = 0;
 	}
 	if (r) {
-		pr_err("No fence found\n");
+		KUNIT_FAIL(test, "No fence found");
 		goto err_unlock;
 	}
 	dma_fence_signal(f);
@@ -164,33 +162,32 @@ static int test_for_each(void *arg)
 err_free:
 	dma_resv_fini(&resv);
 	dma_fence_put(f);
-	return r;
 }
 
-static int test_for_each_unlocked(void *arg)
+static void test_for_each_unlocked(struct kunit *test)
 {
-	enum dma_resv_usage usage = (unsigned long)arg;
+	const struct dma_resv_usage_param *param = test->param_value;
+	enum dma_resv_usage usage = param->usage;
 	struct dma_resv_iter cursor;
 	struct dma_fence *f, *fence;
 	struct dma_resv resv;
 	int r;
 
 	f = alloc_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
 	if (r) {
-		pr_err("Resv locking failed\n");
+		KUNIT_FAIL(test, "Resv locking failed");
 		goto err_free;
 	}
 
 	r = dma_resv_reserve_fences(&resv, 1);
 	if (r) {
-		pr_err("Resv shared slot allocation failed\n");
+		KUNIT_FAIL(test, "Resv shared slot allocation failed");
 		dma_resv_unlock(&resv);
 		goto err_free;
 	}
@@ -202,21 +199,20 @@ static int test_for_each_unlocked(void *arg)
 	dma_resv_iter_begin(&cursor, &resv, usage);
 	dma_resv_for_each_fence_unlocked(&cursor, fence) {
 		if (!r) {
-			pr_err("More than one fence found\n");
-			r = -EINVAL;
+			KUNIT_FAIL(test, "More than one fence found");
 			goto err_iter_end;
 		}
 		if (!dma_resv_iter_is_restarted(&cursor)) {
-			pr_err("No restart flag\n");
+			KUNIT_FAIL(test, "No restart flag");
 			goto err_iter_end;
 		}
 		if (f != fence) {
-			pr_err("Unexpected fence\n");
+			KUNIT_FAIL(test, "Unexpected fence");
 			r = -EINVAL;
 			goto err_iter_end;
 		}
 		if (dma_resv_iter_usage(&cursor) != usage) {
-			pr_err("Unexpected fence usage\n");
+			KUNIT_FAIL(test, "Unexpected fence usage");
 			r = -EINVAL;
 			goto err_iter_end;
 		}
@@ -230,40 +226,38 @@ static int test_for_each_unlocked(void *arg)
 			r = 0;
 		}
 	}
-	if (r)
-		pr_err("No fence found\n");
+	KUNIT_EXPECT_EQ(test, r, 0);
 err_iter_end:
 	dma_resv_iter_end(&cursor);
 	dma_fence_signal(f);
 err_free:
 	dma_resv_fini(&resv);
 	dma_fence_put(f);
-	return r;
 }
 
-static int test_get_fences(void *arg)
+static void test_get_fences(struct kunit *test)
 {
-	enum dma_resv_usage usage = (unsigned long)arg;
+	const struct dma_resv_usage_param *param = test->param_value;
+	enum dma_resv_usage usage = param->usage;
 	struct dma_fence *f, **fences = NULL;
 	struct dma_resv resv;
 	int r, i;
 
 	f = alloc_fence();
-	if (!f)
-		return -ENOMEM;
+	KUNIT_ASSERT_NOT_NULL(test, f);
 
 	dma_fence_enable_sw_signaling(f);
 
 	dma_resv_init(&resv);
 	r = dma_resv_lock(&resv, NULL);
 	if (r) {
-		pr_err("Resv locking failed\n");
+		KUNIT_FAIL(test, "Resv locking failed");
 		goto err_resv;
 	}
 
 	r = dma_resv_reserve_fences(&resv, 1);
 	if (r) {
-		pr_err("Resv shared slot allocation failed\n");
+		KUNIT_FAIL(test, "Resv shared slot allocation failed");
 		dma_resv_unlock(&resv);
 		goto err_resv;
 	}
@@ -273,12 +267,12 @@ static int test_get_fences(void *arg)
 
 	r = dma_resv_get_fences(&resv, usage, &i, &fences);
 	if (r) {
-		pr_err("get_fences failed\n");
+		KUNIT_FAIL(test, "get_fences failed");
 		goto err_free;
 	}
 
 	if (i != 1 || fences[0] != f) {
-		pr_err("get_fences returned unexpected fence\n");
+		KUNIT_FAIL(test, "get_fences returned unexpected fence");
 		goto err_free;
 	}
 
@@ -290,27 +284,32 @@ static int test_get_fences(void *arg)
 err_resv:
 	dma_resv_fini(&resv);
 	dma_fence_put(f);
-	return r;
 }
 
-int dma_resv(void)
-{
-	static const struct subtest tests[] = {
-		SUBTEST(sanitycheck),
-		SUBTEST(test_signaling),
-		SUBTEST(test_for_each),
-		SUBTEST(test_for_each_unlocked),
-		SUBTEST(test_get_fences),
-	};
-	enum dma_resv_usage usage;
-	int r;
+static const struct dma_resv_usage_param dma_resv_usage_params[] = {
+	{ DMA_RESV_USAGE_KERNEL, "kernel" },
+	{ DMA_RESV_USAGE_WRITE, "write" },
+	{ DMA_RESV_USAGE_READ, "read" },
+	{ DMA_RESV_USAGE_BOOKKEEP, "bookkeep" },
+};
 
-	spin_lock_init(&fence_lock);
-	for (usage = DMA_RESV_USAGE_KERNEL; usage <= DMA_RESV_USAGE_BOOKKEEP;
-	     ++usage) {
-		r = subtests(tests, (void *)(unsigned long)usage);
-		if (r)
-			return r;
-	}
-	return 0;
-}
+KUNIT_ARRAY_PARAM_DESC(dma_resv_usage, dma_resv_usage_params, desc);
+
+static struct kunit_case dma_resv_cases[] = {
+	KUNIT_CASE(test_sanitycheck),
+	KUNIT_CASE_PARAM(test_signaling, dma_resv_usage_gen_params),
+	KUNIT_CASE_PARAM(test_for_each, dma_resv_usage_gen_params),
+	KUNIT_CASE_PARAM(test_for_each_unlocked, dma_resv_usage_gen_params),
+	KUNIT_CASE_PARAM(test_get_fences, dma_resv_usage_gen_params),
+	{}
+};
+
+static struct kunit_suite dma_resv_test_suite = {
+	.name = "dma-buf-resv",
+	.test_cases = dma_resv_cases,
+};
+
+kunit_test_suite(dma_resv_test_suite);
+
+MODULE_DESCRIPTION("KUnit tests for DMA-BUF");
+MODULE_LICENSE("GPL");
diff --git a/drivers/dma-buf/udmabuf.c b/drivers/dma-buf/udmabuf.c
index 94b8ecb892bb1..bced421c0d658 100644
--- a/drivers/dma-buf/udmabuf.c
+++ b/drivers/dma-buf/udmabuf.c
@@ -26,10 +26,10 @@ MODULE_PARM_DESC(size_limit_mb, "Max size of a dmabuf, in megabytes. Default is
 
 struct udmabuf {
 	pgoff_t pagecount;
-	struct folio **folios;
+	struct page **pages;
 
 	/**
-	 * Unlike folios, pinned_folios is only used for unpin.
+	 * Unlike pages, pinned_folios is only used for unpin.
 	 * So, nr_pinned is not the same to pagecount, the pinned_folios
 	 * only set each folio which already pinned when udmabuf_create.
 	 * Note that, since a folio may be pinned multiple times, each folio
@@ -40,8 +40,8 @@ struct udmabuf {
 	struct folio **pinned_folios;
 
 	struct sg_table *sg;
+	enum dma_data_direction sg_dir;
 	struct miscdevice *device;
-	pgoff_t *offsets;
 };
 
 static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)
@@ -55,8 +55,7 @@ static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)
 	if (pgoff >= ubuf->pagecount)
 		return VM_FAULT_SIGBUS;
 
-	pfn = folio_pfn(ubuf->folios[pgoff]);
-	pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT;
+	pfn = page_to_pfn(ubuf->pages[pgoff]);
 
 	ret = vmf_insert_pfn(vma, vmf->address, pfn);
 	if (ret & VM_FAULT_ERROR)
@@ -73,8 +72,7 @@ static vm_fault_t udmabuf_vm_fault(struct vm_fault *vmf)
 		if (WARN_ON(pgoff >= ubuf->pagecount))
 			break;
 
-		pfn = folio_pfn(ubuf->folios[pgoff]);
-		pfn += ubuf->offsets[pgoff] >> PAGE_SHIFT;
+		pfn = page_to_pfn(ubuf->pages[pgoff]);
 
 		/**
 		 * If the below vmf_insert_pfn() fails, we do not return an
@@ -109,22 +107,11 @@ static int mmap_udmabuf(struct dma_buf *buf, struct vm_area_struct *vma)
 static int vmap_udmabuf(struct dma_buf *buf, struct iosys_map *map)
 {
 	struct udmabuf *ubuf = buf->priv;
-	struct page **pages;
 	void *vaddr;
-	pgoff_t pg;
 
 	dma_resv_assert_held(buf->resv);
 
-	pages = kvmalloc_objs(*pages, ubuf->pagecount);
-	if (!pages)
-		return -ENOMEM;
-
-	for (pg = 0; pg < ubuf->pagecount; pg++)
-		pages[pg] = folio_page(ubuf->folios[pg],
-				       ubuf->offsets[pg] >> PAGE_SHIFT);
-
-	vaddr = vm_map_ram(pages, ubuf->pagecount, -1);
-	kvfree(pages);
+	vaddr = vm_map_ram(ubuf->pages, ubuf->pagecount, -1);
 	if (!vaddr)
 		return -EINVAL;
 
@@ -146,23 +133,19 @@ static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf,
 {
 	struct udmabuf *ubuf = buf->priv;
 	struct sg_table *sg;
-	struct scatterlist *sgl;
-	unsigned int i = 0;
 	int ret;
 
 	sg = kzalloc_obj(*sg);
 	if (!sg)
 		return ERR_PTR(-ENOMEM);
 
-	ret = sg_alloc_table(sg, ubuf->pagecount, GFP_KERNEL);
+	ret = sg_alloc_table_from_pages(sg, ubuf->pages, ubuf->pagecount, 0,
+					ubuf->pagecount << PAGE_SHIFT,
+					GFP_KERNEL);
 	if (ret < 0)
 		goto err_alloc;
 
-	for_each_sg(sg->sgl, sgl, ubuf->pagecount, i)
-		sg_set_folio(sgl, ubuf->folios[i], PAGE_SIZE,
-			     ubuf->offsets[i]);
-
-	ret = dma_map_sgtable(dev, sg, direction, 0);
+	ret = dma_map_sgtable(dev, sg, direction, DMA_ATTR_SKIP_CPU_SYNC);
 	if (ret < 0)
 		goto err_map;
 	return sg;
@@ -177,7 +160,7 @@ static struct sg_table *get_sg_table(struct device *dev, struct dma_buf *buf,
 static void put_sg_table(struct device *dev, struct sg_table *sg,
 			 enum dma_data_direction direction)
 {
-	dma_unmap_sgtable(dev, sg, direction, 0);
+	dma_unmap_sgtable(dev, sg, direction, DMA_ATTR_SKIP_CPU_SYNC);
 	sg_free_table(sg);
 	kfree(sg);
 }
@@ -207,12 +190,8 @@ static void unpin_all_folios(struct udmabuf *ubuf)
 
 static __always_inline int init_udmabuf(struct udmabuf *ubuf, pgoff_t pgcnt)
 {
-	ubuf->folios = kvmalloc_objs(*ubuf->folios, pgcnt);
-	if (!ubuf->folios)
-		return -ENOMEM;
-
-	ubuf->offsets = kvzalloc_objs(*ubuf->offsets, pgcnt);
-	if (!ubuf->offsets)
+	ubuf->pages = kvmalloc_objs(*ubuf->pages, pgcnt);
+	if (!ubuf->pages)
 		return -ENOMEM;
 
 	ubuf->pinned_folios = kvmalloc_objs(*ubuf->pinned_folios, pgcnt);
@@ -225,8 +204,7 @@ static __always_inline int init_udmabuf(struct udmabuf *ubuf, pgoff_t pgcnt)
 static __always_inline void deinit_udmabuf(struct udmabuf *ubuf)
 {
 	unpin_all_folios(ubuf);
-	kvfree(ubuf->offsets);
-	kvfree(ubuf->folios);
+	kvfree(ubuf->pages);
 }
 
 static void release_udmabuf(struct dma_buf *buf)
@@ -235,7 +213,7 @@ static void release_udmabuf(struct dma_buf *buf)
 	struct device *dev = ubuf->device->this_device;
 
 	if (ubuf->sg)
-		put_sg_table(dev, ubuf->sg, DMA_BIDIRECTIONAL);
+		put_sg_table(dev, ubuf->sg, ubuf->sg_dir);
 
 	deinit_udmabuf(ubuf);
 	kfree(ubuf);
@@ -253,6 +231,8 @@ static int begin_cpu_udmabuf(struct dma_buf *buf,
 		if (IS_ERR(ubuf->sg)) {
 			ret = PTR_ERR(ubuf->sg);
 			ubuf->sg = NULL;
+		} else {
+			ubuf->sg_dir = direction;
 		}
 	} else {
 		dma_sync_sgtable_for_cpu(dev, ubuf->sg, direction);
@@ -344,8 +324,8 @@ static long udmabuf_pin_folios(struct udmabuf *ubuf, struct file *memfd,
 		ubuf->pinned_folios[nr_pinned++] = folios[cur_folio];
 
 		for (; subpgoff < fsize; subpgoff += PAGE_SIZE) {
-			ubuf->folios[upgcnt] = folios[cur_folio];
-			ubuf->offsets[upgcnt] = subpgoff;
+			ubuf->pages[upgcnt] = folio_page(folios[cur_folio],
+						subpgoff >> PAGE_SHIFT);
 			++upgcnt;
 
 			if (++cur_pgcnt >= pgcnt)
diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c
index ed037fa883f6f..0de03611252ed 100644
--- a/drivers/dma/bcm-sba-raid.c
+++ b/drivers/dma/bcm-sba-raid.c
@@ -40,6 +40,7 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/raid/pq.h>
+#include <linux/raid/pq_tables.h>
 
 #include "dmaengine.h"
 
diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c
index 32a0e95c6a205..28bf818f9aa63 100644
--- a/drivers/dma/hisi_dma.c
+++ b/drivers/dma/hisi_dma.c
@@ -1037,6 +1037,7 @@ static const struct pci_device_id hisi_dma_pci_tbl[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa122) },
 	{ 0, }
 };
+MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);
 
 static struct pci_driver hisi_dma_pci_driver = {
 	.name		= "hisi_dma",
@@ -1050,4 +1051,3 @@ MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
 MODULE_AUTHOR("Zhenfa Qiu <qiuzhenfa@hisilicon.com>");
 MODULE_DESCRIPTION("HiSilicon Kunpeng DMA controller driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(pci, hisi_dma_pci_tbl);
diff --git a/drivers/dma/mmp_pdma.c b/drivers/dma/mmp_pdma.c
index d12e729ee12c5..386e85cd4882a 100644
--- a/drivers/dma/mmp_pdma.c
+++ b/drivers/dma/mmp_pdma.c
@@ -51,7 +51,10 @@
 #define DCSR_CMPST	BIT(10)	/* The Descriptor Compare Status */
 #define DCSR_EORINTR	BIT(9)	/* The end of Receive */
 
-#define DRCMR(n)	((((n) < 64) ? 0x0100 : 0x1100) + (((n) & 0x3f) << 2))
+#define DRCMR_BASE		0x0100
+#define DRCMR_EXT_BASE_K3	0x1000
+#define DRCMR_EXT_BASE_DEFAULT	0x1100
+#define DRCMR_REQ_LIMIT		64
 #define DRCMR_MAPVLD	BIT(7)	/* Map Valid (read / write) */
 #define DRCMR_CHLNUM	0x1f	/* mask for Channel Number (read / write) */
 
@@ -154,6 +157,7 @@ struct mmp_pdma_phy {
  * @run_bits:   Control bits in DCSR register for channel start/stop
  * @dma_width:  DMA addressing width in bits (32 or 64). Determines the
  *              DMA mask capability of the controller hardware.
+ * @drcmr_ext_base: Base DRCMR address for extended requests
  */
 struct mmp_pdma_ops {
 	/* Hardware Register Operations */
@@ -174,6 +178,7 @@ struct mmp_pdma_ops {
 	/* Controller Configuration */
 	u32 run_bits;
 	u32 dma_width;
+	u32 drcmr_ext_base;
 };
 
 struct mmp_pdma_device {
@@ -195,6 +200,13 @@ struct mmp_pdma_device {
 #define to_mmp_pdma_dev(dmadev)					\
 	container_of(dmadev, struct mmp_pdma_device, device)
 
+static u32 mmp_pdma_get_drcmr(struct mmp_pdma_device *pdev, u32 drcmr)
+{
+	if (drcmr < DRCMR_REQ_LIMIT)
+		return DRCMR_BASE + (drcmr << 2);
+	return pdev->ops->drcmr_ext_base + ((drcmr - DRCMR_REQ_LIMIT) << 2);
+}
+
 /* For 32-bit PDMA */
 static void write_next_addr_32(struct mmp_pdma_phy *phy, dma_addr_t addr)
 {
@@ -301,7 +313,7 @@ static void enable_chan(struct mmp_pdma_phy *phy)
 
 	pdev = to_mmp_pdma_dev(phy->vchan->chan.device);
 
-	reg = DRCMR(phy->vchan->drcmr);
+	reg = mmp_pdma_get_drcmr(pdev, phy->vchan->drcmr);
 	writel(DRCMR_MAPVLD | phy->idx, phy->base + reg);
 
 	dalgn = readl(phy->base + DALGN);
@@ -437,7 +449,7 @@ static void mmp_pdma_free_phy(struct mmp_pdma_chan *pchan)
 		return;
 
 	/* clear the channel mapping in DRCMR */
-	reg = DRCMR(pchan->drcmr);
+	reg = mmp_pdma_get_drcmr(pdev, pchan->drcmr);
 	writel(0, pchan->phy->base + reg);
 
 	spin_lock_irqsave(&pdev->phy_lock, flags);
@@ -1179,6 +1191,7 @@ static const struct mmp_pdma_ops marvell_pdma_v1_ops = {
 	.get_desc_dst_addr = get_desc_dst_addr_32,
 	.run_bits = (DCSR_RUN),
 	.dma_width = 32,
+	.drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT,
 };
 
 static const struct mmp_pdma_ops spacemit_k1_pdma_ops = {
@@ -1192,6 +1205,21 @@ static const struct mmp_pdma_ops spacemit_k1_pdma_ops = {
 	.get_desc_dst_addr = get_desc_dst_addr_64,
 	.run_bits = (DCSR_RUN | DCSR_LPAEEN),
 	.dma_width = 64,
+	.drcmr_ext_base = DRCMR_EXT_BASE_DEFAULT,
+};
+
+static const struct mmp_pdma_ops spacemit_k3_pdma_ops = {
+	.write_next_addr = write_next_addr_64,
+	.read_src_addr = read_src_addr_64,
+	.read_dst_addr = read_dst_addr_64,
+	.set_desc_next_addr = set_desc_next_addr_64,
+	.set_desc_src_addr = set_desc_src_addr_64,
+	.set_desc_dst_addr = set_desc_dst_addr_64,
+	.get_desc_src_addr = get_desc_src_addr_64,
+	.get_desc_dst_addr = get_desc_dst_addr_64,
+	.run_bits = (DCSR_RUN | DCSR_LPAEEN | DCSR_EORIRQEN | DCSR_EORSTOPEN),
+	.dma_width = 64,
+	.drcmr_ext_base = DRCMR_EXT_BASE_K3,
 };
 
 static const struct of_device_id mmp_pdma_dt_ids[] = {
@@ -1201,6 +1229,9 @@ static const struct of_device_id mmp_pdma_dt_ids[] = {
 	}, {
 		.compatible = "spacemit,k1-pdma",
 		.data = &spacemit_k1_pdma_ops
+	}, {
+		.compatible = "spacemit,k3-pdma",
+		.data = &spacemit_k3_pdma_ops
 	}, {
 		/* sentinel */
 	}
diff --git a/drivers/dma/pch_dma.c b/drivers/dma/pch_dma.c
index e9fbfd5a3d519..bf805f1024f64 100644
--- a/drivers/dma/pch_dma.c
+++ b/drivers/dma/pch_dma.c
@@ -970,6 +970,7 @@ static const struct pci_device_id pch_dma_id_table[] = {
 	{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7831_DMA2_4CH), 4}, /* SPI */
 	{ 0, },
 };
+MODULE_DEVICE_TABLE(pci, pch_dma_id_table);
 
 static SIMPLE_DEV_PM_OPS(pch_dma_pm_ops, pch_dma_suspend, pch_dma_resume);
 
@@ -987,4 +988,3 @@ MODULE_DESCRIPTION("Intel EG20T PCH / LAPIS Semicon ML7213/ML7223/ML7831 IOH "
 		   "DMA controller driver");
 MODULE_AUTHOR("Yong Wang <yong.y.wang@intel.com>");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(pci, pch_dma_id_table);
diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c
index 19116295f8325..1bb26af0405f3 100644
--- a/drivers/dma/qcom/bam_dma.c
+++ b/drivers/dma/qcom/bam_dma.c
@@ -199,6 +199,35 @@ static const struct reg_offset_data bam_v1_7_reg_info[] = {
 	[BAM_P_FIFO_SIZES]	= { 0x13820, 0x00, 0x1000, 0x00 },
 };
 
+static const struct reg_offset_data bam_v2_0_reg_info[] = {
+	[BAM_CTRL]		= { 0x0000, 0x00, 0x00, 0x00 },
+	[BAM_REVISION]		= { 0x1000, 0x00, 0x00, 0x00 },
+	[BAM_NUM_PIPES]		= { 0x1008, 0x00, 0x00, 0x00 },
+	[BAM_DESC_CNT_TRSHLD]	= { 0x0008, 0x00, 0x00, 0x00 },
+	[BAM_IRQ_SRCS]		= { 0x3010, 0x00, 0x00, 0x00 },
+	[BAM_IRQ_SRCS_MSK]	= { 0x3014, 0x00, 0x00, 0x00 },
+	[BAM_IRQ_SRCS_UNMASKED]	= { 0x3018, 0x00, 0x00, 0x00 },
+	[BAM_IRQ_STTS]		= { 0x0014, 0x00, 0x00, 0x00 },
+	[BAM_IRQ_CLR]		= { 0x0018, 0x00, 0x00, 0x00 },
+	[BAM_IRQ_EN]		= { 0x001C, 0x00, 0x00, 0x00 },
+	[BAM_CNFG_BITS]		= { 0x007C, 0x00, 0x00, 0x00 },
+	[BAM_IRQ_SRCS_EE]	= { 0x3000, 0x00, 0x00, 0x1000 },
+	[BAM_IRQ_SRCS_MSK_EE]	= { 0x3004, 0x00, 0x00, 0x1000 },
+	[BAM_P_CTRL]		= { 0xC000, 0x1000, 0x00, 0x00 },
+	[BAM_P_RST]		= { 0xC004, 0x1000, 0x00, 0x00 },
+	[BAM_P_HALT]		= { 0xC008, 0x1000, 0x00, 0x00 },
+	[BAM_P_IRQ_STTS]	= { 0xC010, 0x1000, 0x00, 0x00 },
+	[BAM_P_IRQ_CLR]		= { 0xC014, 0x1000, 0x00, 0x00 },
+	[BAM_P_IRQ_EN]		= { 0xC018, 0x1000, 0x00, 0x00 },
+	[BAM_P_EVNT_DEST_ADDR]	= { 0xC82C, 0x00, 0x1000, 0x00 },
+	[BAM_P_EVNT_REG]	= { 0xC818, 0x00, 0x1000, 0x00 },
+	[BAM_P_SW_OFSTS]	= { 0xC800, 0x00, 0x1000, 0x00 },
+	[BAM_P_DATA_FIFO_ADDR]	= { 0xC824, 0x00, 0x1000, 0x00 },
+	[BAM_P_DESC_FIFO_ADDR]	= { 0xC81C, 0x00, 0x1000, 0x00 },
+	[BAM_P_EVNT_GEN_TRSHLD]	= { 0xC828, 0x00, 0x1000, 0x00 },
+	[BAM_P_FIFO_SIZES]	= { 0xC820, 0x00, 0x1000, 0x00 },
+};
+
 /* BAM CTRL */
 #define BAM_SW_RST			BIT(0)
 #define BAM_EN				BIT(1)
@@ -1208,6 +1237,7 @@ static const struct of_device_id bam_of_match[] = {
 	{ .compatible = "qcom,bam-v1.3.0", .data = &bam_v1_3_reg_info },
 	{ .compatible = "qcom,bam-v1.4.0", .data = &bam_v1_4_reg_info },
 	{ .compatible = "qcom,bam-v1.7.0", .data = &bam_v1_7_reg_info },
+	{ .compatible = "qcom,bam-v2.0.0", .data = &bam_v2_0_reg_info },
 	{}
 };
 
diff --git a/drivers/dma/ti/k3-udma-glue.c b/drivers/dma/ti/k3-udma-glue.c
index f87d244cc2d67..686dc140293eb 100644
--- a/drivers/dma/ti/k3-udma-glue.c
+++ b/drivers/dma/ti/k3-udma-glue.c
@@ -312,7 +312,7 @@ k3_udma_glue_request_tx_chn_common(struct device *dev,
 
 	if (xudma_is_pktdma(tx_chn->common.udmax)) {
 		/* prepare the channel device as coherent */
-		tx_chn->common.chan_dev.dma_coherent = true;
+		dev_set_dma_coherent(&tx_chn->common.chan_dev);
 		dma_coerce_mask_and_coherent(&tx_chn->common.chan_dev,
 					     DMA_BIT_MASK(48));
 	}
@@ -1003,7 +1003,7 @@ k3_udma_glue_request_rx_chn_priv(struct device *dev, const char *name,
 
 	if (xudma_is_pktdma(rx_chn->common.udmax)) {
 		/* prepare the channel device as coherent */
-		rx_chn->common.chan_dev.dma_coherent = true;
+		dev_set_dma_coherent(&rx_chn->common.chan_dev);
 		dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
 					     DMA_BIT_MASK(48));
 	}
@@ -1104,7 +1104,7 @@ k3_udma_glue_request_remote_rx_chn_common(struct k3_udma_glue_rx_channel *rx_chn
 
 	if (xudma_is_pktdma(rx_chn->common.udmax)) {
 		/* prepare the channel device as coherent */
-		rx_chn->common.chan_dev.dma_coherent = true;
+		dev_set_dma_coherent(&rx_chn->common.chan_dev);
 		dma_coerce_mask_and_coherent(&rx_chn->common.chan_dev,
 					     DMA_BIT_MASK(48));
 		rx_chn->single_fdq = false;
diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c
index c964ebfcf3b68..1cf158eb7bdb5 100644
--- a/drivers/dma/ti/k3-udma.c
+++ b/drivers/dma/ti/k3-udma.c
@@ -428,18 +428,18 @@ static void k3_configure_chan_coherency(struct dma_chan *chan, u32 asel)
 		/* No special handling for the channel */
 		chan->dev->chan_dma_dev = false;
 
-		chan_dev->dma_coherent = false;
+		dev_clear_dma_coherent(chan_dev);
 		chan_dev->dma_parms = NULL;
 	} else if (asel == 14 || asel == 15) {
 		chan->dev->chan_dma_dev = true;
 
-		chan_dev->dma_coherent = true;
+		dev_set_dma_coherent(chan_dev);
 		dma_coerce_mask_and_coherent(chan_dev, DMA_BIT_MASK(48));
 		chan_dev->dma_parms = chan_dev->parent->dma_parms;
 	} else {
 		dev_warn(chan->device->dev, "Invalid ASEL value: %u\n", asel);
 
-		chan_dev->dma_coherent = false;
+		dev_clear_dma_coherent(chan_dev);
 		chan_dev->dma_parms = NULL;
 	}
 }
diff --git a/drivers/dpll/dpll_core.c b/drivers/dpll/dpll_core.c
index cbb635db43210..20a54728549cc 100644
--- a/drivers/dpll/dpll_core.c
+++ b/drivers/dpll/dpll_core.c
@@ -879,7 +879,8 @@ dpll_pin_register(struct dpll_device *dpll, struct dpll_pin *pin,
 	    WARN_ON(!ops->direction_get) ||
 	    WARN_ON(ops->measured_freq_get &&
 		    (!dpll_device_ops(dpll)->freq_monitor_get ||
-		     !dpll_device_ops(dpll)->freq_monitor_set)))
+		     !dpll_device_ops(dpll)->freq_monitor_set)) ||
+	    WARN_ON(ops->supported_ffo && !ops->ffo_get))
 		return -EINVAL;
 
 	mutex_lock(&dpll_lock);
diff --git a/drivers/dpll/dpll_netlink.c b/drivers/dpll/dpll_netlink.c
index 0ff1658c2dc1b..53ca24d0f191b 100644
--- a/drivers/dpll/dpll_netlink.c
+++ b/drivers/dpll/dpll_netlink.c
@@ -324,6 +324,30 @@ dpll_msg_add_pin_on_dpll_state(struct sk_buff *msg, struct dpll_pin *pin,
 	return 0;
 }
 
+static int
+dpll_msg_add_pin_operstate(struct sk_buff *msg, struct dpll_pin *pin,
+			   struct dpll_pin_ref *ref,
+			   struct netlink_ext_ack *extack)
+{
+	const struct dpll_pin_ops *ops = dpll_pin_ops(ref);
+	struct dpll_device *dpll = ref->dpll;
+	enum dpll_pin_operstate operstate;
+	int ret;
+
+	if (!ops->operstate_on_dpll_get)
+		return 0;
+	ret = ops->operstate_on_dpll_get(pin,
+					  dpll_pin_on_dpll_priv(dpll, pin),
+					  dpll, dpll_priv(dpll),
+					  &operstate, extack);
+	if (ret)
+		return ret;
+	if (nla_put_u32(msg, DPLL_A_PIN_OPERSTATE, operstate))
+		return -EMSGSIZE;
+
+	return 0;
+}
+
 static int
 dpll_msg_add_pin_direction(struct sk_buff *msg, struct dpll_pin *pin,
 			   struct dpll_pin_ref *ref,
@@ -393,31 +417,28 @@ dpll_msg_add_phase_offset(struct sk_buff *msg, struct dpll_pin *pin,
 
 static int dpll_msg_add_ffo(struct sk_buff *msg, struct dpll_pin *pin,
 			    struct dpll_pin_ref *ref,
+			    enum dpll_ffo_type type,
 			    struct netlink_ext_ack *extack)
 {
 	const struct dpll_pin_ops *ops = dpll_pin_ops(ref);
-	struct dpll_device *dpll = ref->dpll;
-	s64 ffo;
+	struct dpll_ffo_param ffo = { .type = type };
 	int ret;
 
-	if (!ops->ffo_get)
+	if (!ops->ffo_get || !(ops->supported_ffo & BIT(type)))
 		return 0;
-	ret = ops->ffo_get(pin, dpll_pin_on_dpll_priv(dpll, pin),
-			   dpll, dpll_priv(dpll), &ffo, extack);
+	ret = ops->ffo_get(pin, dpll_pin_on_dpll_priv(ref->dpll, pin),
+			   ref->dpll, dpll_priv(ref->dpll), &ffo, extack);
 	if (ret) {
 		if (ret == -ENODATA)
 			return 0;
 		return ret;
 	}
-	/* Put the FFO value in PPM to preserve compatibility with older
-	 * programs.
-	 */
-	ret = nla_put_sint(msg, DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET,
-			   div_s64(ffo, 1000000));
-	if (ret)
+	if (nla_put_sint(msg, DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET,
+			 div_s64(ffo.ffo, 1000000)))
 		return -EMSGSIZE;
-	return nla_put_sint(msg, DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET_PPT,
-			    ffo);
+	return nla_put_sint(msg,
+			    DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET_PPT,
+			    ffo.ffo);
 }
 
 static int dpll_msg_add_measured_freq(struct sk_buff *msg, struct dpll_pin *pin,
@@ -650,6 +671,9 @@ dpll_msg_add_pin_dplls(struct sk_buff *msg, struct dpll_pin *pin,
 		if (ret)
 			goto nest_cancel;
 		ret = dpll_msg_add_pin_on_dpll_state(msg, pin, ref, extack);
+		if (ret)
+			goto nest_cancel;
+		ret = dpll_msg_add_pin_operstate(msg, pin, ref, extack);
 		if (ret)
 			goto nest_cancel;
 		ret = dpll_msg_add_pin_prio(msg, pin, ref, extack);
@@ -659,6 +683,10 @@ dpll_msg_add_pin_dplls(struct sk_buff *msg, struct dpll_pin *pin,
 		if (ret)
 			goto nest_cancel;
 		ret = dpll_msg_add_phase_offset(msg, pin, ref, extack);
+		if (ret)
+			goto nest_cancel;
+		ret = dpll_msg_add_ffo(msg, pin, ref,
+				       DPLL_FFO_PIN_DEVICE, extack);
 		if (ret)
 			goto nest_cancel;
 		nla_nest_end(msg, attr);
@@ -721,7 +749,8 @@ dpll_cmd_pin_get_one(struct sk_buff *msg, struct dpll_pin *pin,
 	ret = dpll_msg_add_pin_phase_adjust(msg, pin, ref, extack);
 	if (ret)
 		return ret;
-	ret = dpll_msg_add_ffo(msg, pin, ref, extack);
+	ret = dpll_msg_add_ffo(msg, pin, ref,
+			       DPLL_FFO_PORT_RXTX_RATE, extack);
 	if (ret)
 		return ret;
 	ret = dpll_msg_add_measured_freq(msg, pin, ref, extack);
diff --git a/drivers/dpll/dpll_nl.c b/drivers/dpll/dpll_nl.c
index 1e652340a5d73..b1d9182c7802f 100644
--- a/drivers/dpll/dpll_nl.c
+++ b/drivers/dpll/dpll_nl.c
@@ -12,12 +12,15 @@
 #include <uapi/linux/dpll.h>
 
 /* Common nested types */
-const struct nla_policy dpll_pin_parent_device_nl_policy[DPLL_A_PIN_PHASE_OFFSET + 1] = {
+const struct nla_policy dpll_pin_parent_device_nl_policy[DPLL_A_PIN_OPERSTATE + 1] = {
 	[DPLL_A_PIN_PARENT_ID] = { .type = NLA_U32, },
 	[DPLL_A_PIN_DIRECTION] = NLA_POLICY_RANGE(NLA_U32, 1, 2),
 	[DPLL_A_PIN_PRIO] = { .type = NLA_U32, },
 	[DPLL_A_PIN_STATE] = NLA_POLICY_RANGE(NLA_U32, 1, 3),
+	[DPLL_A_PIN_OPERSTATE] = NLA_POLICY_RANGE(NLA_U32, 1, 4),
 	[DPLL_A_PIN_PHASE_OFFSET] = { .type = NLA_S64, },
+	[DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET] = { .type = NLA_SINT, },
+	[DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET_PPT] = { .type = NLA_SINT, },
 };
 
 const struct nla_policy dpll_pin_parent_pin_nl_policy[DPLL_A_PIN_STATE + 1] = {
diff --git a/drivers/dpll/dpll_nl.h b/drivers/dpll/dpll_nl.h
index 7419679b69779..fa8280e3dd14c 100644
--- a/drivers/dpll/dpll_nl.h
+++ b/drivers/dpll/dpll_nl.h
@@ -13,7 +13,7 @@
 #include <uapi/linux/dpll.h>
 
 /* Common nested types */
-extern const struct nla_policy dpll_pin_parent_device_nl_policy[DPLL_A_PIN_PHASE_OFFSET + 1];
+extern const struct nla_policy dpll_pin_parent_device_nl_policy[DPLL_A_PIN_OPERSTATE + 1];
 extern const struct nla_policy dpll_pin_parent_pin_nl_policy[DPLL_A_PIN_STATE + 1];
 extern const struct nla_policy dpll_reference_sync_nl_policy[DPLL_A_PIN_STATE + 1];
 
diff --git a/drivers/dpll/zl3073x/chan.c b/drivers/dpll/zl3073x/chan.c
index 2f48ca2391494..2fe3c3da84bb5 100644
--- a/drivers/dpll/zl3073x/chan.c
+++ b/drivers/dpll/zl3073x/chan.c
@@ -18,6 +18,7 @@
 int zl3073x_chan_state_update(struct zl3073x_dev *zldev, u8 index)
 {
 	struct zl3073x_chan *chan = &zldev->chan[index];
+	u64 val;
 	int rc;
 
 	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_MON_STATUS(index),
@@ -25,8 +26,34 @@ int zl3073x_chan_state_update(struct zl3073x_dev *zldev, u8 index)
 	if (rc)
 		return rc;
 
-	return zl3073x_read_u8(zldev, ZL_REG_DPLL_REFSEL_STATUS(index),
-			       &chan->refsel_status);
+	rc = zl3073x_read_u8(zldev, ZL_REG_DPLL_REFSEL_STATUS(index),
+			     &chan->refsel_status);
+	if (rc)
+		return rc;
+
+	/* Read df_offset vs tracked reference */
+	rc = zl3073x_poll_zero_u8(zldev, ZL_REG_DPLL_DF_READ(index),
+				  ZL_DPLL_DF_READ_SEM);
+	if (rc)
+		return rc;
+
+	rc = zl3073x_write_u8(zldev, ZL_REG_DPLL_DF_READ(index),
+			      ZL_DPLL_DF_READ_SEM | ZL_DPLL_DF_READ_REF_OFST);
+	if (rc)
+		return rc;
+
+	rc = zl3073x_poll_zero_u8(zldev, ZL_REG_DPLL_DF_READ(index),
+				  ZL_DPLL_DF_READ_SEM);
+	if (rc)
+		return rc;
+
+	rc = zl3073x_read_u48(zldev, ZL_REG_DPLL_DF_OFFSET(index), &val);
+	if (rc)
+		return rc;
+
+	chan->df_offset = sign_extend64(val, 47);
+
+	return 0;
 }
 
 /**
diff --git a/drivers/dpll/zl3073x/chan.h b/drivers/dpll/zl3073x/chan.h
index 481da2133202b..4353809c69122 100644
--- a/drivers/dpll/zl3073x/chan.h
+++ b/drivers/dpll/zl3073x/chan.h
@@ -17,6 +17,7 @@ struct zl3073x_dev;
  * @ref_prio: reference priority registers (4 bits per ref, P/N packed)
  * @mon_status: monitor status register value
  * @refsel_status: reference selection status register value
+ * @df_offset: frequency offset vs tracked reference in 2^-48 steps
  */
 struct zl3073x_chan {
 	struct_group(cfg,
@@ -26,6 +27,7 @@ struct zl3073x_chan {
 	struct_group(stat,
 		u8	mon_status;
 		u8	refsel_status;
+		s64	df_offset;
 	);
 };
 
@@ -37,6 +39,18 @@ int zl3073x_chan_state_set(struct zl3073x_dev *zldev, u8 index,
 
 int zl3073x_chan_state_update(struct zl3073x_dev *zldev, u8 index);
 
+/**
+ * zl3073x_chan_df_offset_get - get cached df_offset vs tracked reference
+ * @chan: pointer to channel state
+ *
+ * Return: frequency offset in 2^-48 steps
+ */
+static inline s64
+zl3073x_chan_df_offset_get(const struct zl3073x_chan *chan)
+{
+	return chan->df_offset;
+}
+
 /**
  * zl3073x_chan_mode_get - get DPLL channel operating mode
  * @chan: pointer to channel state
diff --git a/drivers/dpll/zl3073x/core.c b/drivers/dpll/zl3073x/core.c
index 5f1e70f3e40a0..b3345060490db 100644
--- a/drivers/dpll/zl3073x/core.c
+++ b/drivers/dpll/zl3073x/core.c
@@ -704,44 +704,6 @@ zl3073x_ref_freq_meas_update(struct zl3073x_dev *zldev)
 	return 0;
 }
 
-/**
- * zl3073x_ref_ffo_update - update reference fractional frequency offsets
- * @zldev: pointer to zl3073x_dev structure
- *
- * The function asks device to latch the latest measured fractional
- * frequency offset values, reads and stores them into the ref state.
- *
- * Return: 0 on success, <0 on error
- */
-static int
-zl3073x_ref_ffo_update(struct zl3073x_dev *zldev)
-{
-	int i, rc;
-
-	rc = zl3073x_ref_freq_meas_latch(zldev,
-					 ZL_REF_FREQ_MEAS_CTRL_REF_FREQ_OFF);
-	if (rc)
-		return rc;
-
-	/* Read DPLL-to-REFx frequency offset measurements */
-	for (i = 0; i < ZL3073X_NUM_REFS; i++) {
-		s32 value;
-
-		/* Read value stored in units of 2^-32 signed */
-		rc = zl3073x_read_u32(zldev, ZL_REG_REF_FREQ(i), &value);
-		if (rc)
-			return rc;
-
-		/* Convert to ppt
-		 * ffo = (10^12 * value) / 2^32
-		 * ffo = ( 5^12 * value) / 2^20
-		 */
-		zldev->ref[i].ffo = mul_s64_u64_shr(value, 244140625, 20);
-	}
-
-	return 0;
-}
-
 static void
 zl3073x_dev_periodic_work(struct kthread_work *work)
 {
@@ -776,13 +738,6 @@ zl3073x_dev_periodic_work(struct kthread_work *work)
 		}
 	}
 
-	/* Update references' fractional frequency offsets */
-	rc = zl3073x_ref_ffo_update(zldev);
-	if (rc)
-		dev_warn(zldev->dev,
-			 "Failed to update fractional frequency offsets: %pe\n",
-			 ERR_PTR(rc));
-
 	list_for_each_entry(zldpll, &zldev->dplls, list)
 		zl3073x_dpll_changes_check(zldpll);
 
diff --git a/drivers/dpll/zl3073x/dpll.c b/drivers/dpll/zl3073x/dpll.c
index c95e93ef3ab04..605801964c3cd 100644
--- a/drivers/dpll/zl3073x/dpll.c
+++ b/drivers/dpll/zl3073x/dpll.c
@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0-only
 
+#include <linux/atomic.h>
 #include <linux/bits.h>
 #include <linux/bitfield.h>
 #include <linux/bug.h>
@@ -38,7 +39,7 @@
  * @prio: pin priority <0, 14>
  * @esync_control: embedded sync is controllable
  * @phase_gran: phase adjustment granularity
- * @pin_state: last saved pin state
+ * @operstate: last saved operational state
  * @phase_offset: last saved pin phase offset
  * @freq_offset: last saved fractional frequency offset
  * @measured_freq: last saved measured frequency
@@ -55,9 +56,9 @@ struct zl3073x_dpll_pin {
 	u8			prio;
 	bool			esync_control;
 	s32			phase_gran;
-	enum dpll_pin_state	pin_state;
+	enum dpll_pin_operstate	operstate;
 	s64			phase_offset;
-	s64			freq_offset;
+	atomic64_t		freq_offset;
 	u32			measured_freq;
 };
 
@@ -295,11 +296,15 @@ zl3073x_dpll_input_pin_ref_sync_set(const struct dpll_pin *dpll_pin,
 static int
 zl3073x_dpll_input_pin_ffo_get(const struct dpll_pin *dpll_pin, void *pin_priv,
 			       const struct dpll_device *dpll, void *dpll_priv,
-			       s64 *ffo, struct netlink_ext_ack *extack)
+			       struct dpll_ffo_param *ffo,
+			       struct netlink_ext_ack *extack)
 {
 	struct zl3073x_dpll_pin *pin = pin_priv;
 
-	*ffo = pin->freq_offset;
+	if (pin->operstate != DPLL_PIN_OPERSTATE_ACTIVE)
+		return -ENODATA;
+
+	ffo->ffo = atomic64_read(&pin->freq_offset);
 
 	return 0;
 }
@@ -500,46 +505,41 @@ zl3073x_dpll_input_pin_phase_adjust_set(const struct dpll_pin *dpll_pin,
 }
 
 /**
- * zl3073x_dpll_ref_state_get - get status for given input pin
+ * zl3073x_dpll_ref_operstate_get - get operational state for input pin
  * @pin: pointer to pin
- * @state: place to store status
+ * @operstate: place to store operational state
  *
- * Checks current status for the given input pin and stores the value
- * to @state.
+ * Returns the actual hardware state of the pin: whether it is actively
+ * used by the DPLL, has no signal, failed qualification, or is simply
+ * not in use.
  *
  * Return: 0 on success, <0 on error
  */
 static int
-zl3073x_dpll_ref_state_get(struct zl3073x_dpll_pin *pin,
-			   enum dpll_pin_state *state)
+zl3073x_dpll_ref_operstate_get(struct zl3073x_dpll_pin *pin,
+			       enum dpll_pin_operstate *operstate)
 {
 	struct zl3073x_dpll *zldpll = pin->dpll;
 	struct zl3073x_dev *zldev = zldpll->dev;
-	const struct zl3073x_chan *chan;
-	u8 ref;
-
-	chan = zl3073x_chan_state_get(zldev, zldpll->id);
-	ref = zl3073x_input_pin_ref_get(pin->id);
+	const struct zl3073x_ref *ref;
+	u8 ref_id;
 
-	/* Check if the pin reference is connected */
-	if (ref == zl3073x_dpll_connected_ref_get(zldpll)) {
-		*state = DPLL_PIN_STATE_CONNECTED;
-		return 0;
-	}
+	ref_id = zl3073x_input_pin_ref_get(pin->id);
 
-	/* If the DPLL is running in automatic mode and the reference is
-	 * selectable and its monitor does not report any error then report
-	 * pin as selectable.
-	 */
-	if (zl3073x_chan_mode_get(chan) == ZL_DPLL_MODE_REFSEL_MODE_AUTO &&
-	    zl3073x_dev_ref_is_status_ok(zldev, ref) &&
-	    zl3073x_chan_ref_is_selectable(chan, ref)) {
-		*state = DPLL_PIN_STATE_SELECTABLE;
+	/* Check if this pin is the currently locked reference */
+	if (ref_id == zl3073x_dpll_connected_ref_get(zldpll)) {
+		*operstate = DPLL_PIN_OPERSTATE_ACTIVE;
 		return 0;
 	}
 
-	/* Otherwise report the pin as disconnected */
-	*state = DPLL_PIN_STATE_DISCONNECTED;
+	/* Check reference monitor status */
+	ref = zl3073x_ref_state_get(zldev, ref_id);
+	if (ref->mon_status & ZL_REF_MON_STATUS_LOS)
+		*operstate = DPLL_PIN_OPERSTATE_NO_SIGNAL;
+	else if (!zl3073x_ref_is_status_ok(ref))
+		*operstate = DPLL_PIN_OPERSTATE_QUAL_FAILED;
+	else
+		*operstate = DPLL_PIN_OPERSTATE_STANDBY;
 
 	return 0;
 }
@@ -551,10 +551,48 @@ zl3073x_dpll_input_pin_state_on_dpll_get(const struct dpll_pin *dpll_pin,
 					 void *dpll_priv,
 					 enum dpll_pin_state *state,
 					 struct netlink_ext_ack *extack)
+{
+	struct zl3073x_dpll *zldpll = dpll_priv;
+	struct zl3073x_dpll_pin *pin = pin_priv;
+	const struct zl3073x_chan *chan;
+	u8 mode, ref;
+
+	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	ref = zl3073x_input_pin_ref_get(pin->id);
+	mode = zl3073x_chan_mode_get(chan);
+
+	switch (mode) {
+	case ZL_DPLL_MODE_REFSEL_MODE_REFLOCK:
+		if (ref == zl3073x_chan_ref_get(chan))
+			*state = DPLL_PIN_STATE_CONNECTED;
+		else
+			*state = DPLL_PIN_STATE_DISCONNECTED;
+		break;
+	case ZL_DPLL_MODE_REFSEL_MODE_AUTO:
+		if (zl3073x_chan_ref_is_selectable(chan, ref))
+			*state = DPLL_PIN_STATE_SELECTABLE;
+		else
+			*state = DPLL_PIN_STATE_DISCONNECTED;
+		break;
+	default:
+		*state = DPLL_PIN_STATE_DISCONNECTED;
+		break;
+	}
+
+	return 0;
+}
+
+static int
+zl3073x_dpll_input_pin_operstate_on_dpll_get(const struct dpll_pin *dpll_pin,
+					     void *pin_priv,
+					     const struct dpll_device *dpll,
+					     void *dpll_priv,
+					     enum dpll_pin_operstate *operstate,
+					     struct netlink_ext_ack *extack)
 {
 	struct zl3073x_dpll_pin *pin = pin_priv;
 
-	return zl3073x_dpll_ref_state_get(pin, state);
+	return zl3073x_dpll_ref_operstate_get(pin, operstate);
 }
 
 static int
@@ -1241,6 +1279,7 @@ zl3073x_dpll_freq_monitor_set(const struct dpll_device *dpll,
 }
 
 static const struct dpll_pin_ops zl3073x_dpll_input_pin_ops = {
+	.supported_ffo = BIT(DPLL_FFO_PIN_DEVICE),
 	.direction_get = zl3073x_dpll_pin_direction_get,
 	.esync_get = zl3073x_dpll_input_pin_esync_get,
 	.esync_set = zl3073x_dpll_input_pin_esync_set,
@@ -1248,6 +1287,7 @@ static const struct dpll_pin_ops zl3073x_dpll_input_pin_ops = {
 	.frequency_get = zl3073x_dpll_input_pin_frequency_get,
 	.frequency_set = zl3073x_dpll_input_pin_frequency_set,
 	.measured_freq_get = zl3073x_dpll_input_pin_measured_freq_get,
+	.operstate_on_dpll_get = zl3073x_dpll_input_pin_operstate_on_dpll_get,
 	.phase_offset_get = zl3073x_dpll_input_pin_phase_offset_get,
 	.phase_adjust_get = zl3073x_dpll_input_pin_phase_adjust_get,
 	.phase_adjust_set = zl3073x_dpll_input_pin_phase_adjust_set,
@@ -1394,8 +1434,8 @@ zl3073x_dpll_pin_register(struct zl3073x_dpll_pin *pin, u32 index)
 
 err_register:
 	dpll_pin_put(pin->dpll_pin, &pin->tracker);
-	pin->dpll_pin = NULL;
 err_pin_get:
+	pin->dpll_pin = NULL;
 	fwnode_handle_put(pin->fwnode);
 	pin->fwnode = NULL;
 	zl3073x_pin_props_put(props);
@@ -1563,8 +1603,10 @@ zl3073x_dpll_pins_register(struct zl3073x_dpll *zldpll)
 		}
 
 		rc = zl3073x_dpll_pin_register(pin, index);
-		if (rc)
+		if (rc) {
+			zl3073x_dpll_pin_free(pin);
 			goto error;
+		}
 
 		list_add(&pin->list, &zldpll->pins);
 	}
@@ -1663,7 +1705,7 @@ zl3073x_dpll_pin_phase_offset_check(struct zl3073x_dpll_pin *pin)
 	 * 2) For other pins use appropriate ref_phase register if the phase
 	 *    monitor feature is enabled.
 	 */
-	if (pin->pin_state == DPLL_PIN_STATE_CONNECTED)
+	if (pin->operstate == DPLL_PIN_OPERSTATE_ACTIVE)
 		reg = ZL_REG_DPLL_PHASE_ERR_DATA(zldpll->id);
 	else if (zldpll->phase_monitor)
 		reg = ZL_REG_REF_PHASE(ref_id);
@@ -1695,37 +1737,29 @@ zl3073x_dpll_pin_phase_offset_check(struct zl3073x_dpll_pin *pin)
 }
 
 /**
- * zl3073x_dpll_pin_ffo_check - check for pin fractional frequency offset change
+ * zl3073x_dpll_pin_ffo_check - check for FFO change on active pin
  * @pin: pin to check
  *
- * Check for the given pin's fractional frequency change.
- *
- * Return: true on fractional frequency offset change, false otherwise
+ * Return: true on change, false otherwise
  */
 static bool
 zl3073x_dpll_pin_ffo_check(struct zl3073x_dpll_pin *pin)
 {
 	struct zl3073x_dpll *zldpll = pin->dpll;
 	struct zl3073x_dev *zldev = zldpll->dev;
-	const struct zl3073x_ref *ref;
-	u8 ref_id;
+	const struct zl3073x_chan *chan;
 	s64 ffo;
 
-	/* Get reference monitor status */
-	ref_id = zl3073x_input_pin_ref_get(pin->id);
-	ref = zl3073x_ref_state_get(zldev, ref_id);
-
-	/* Do not report ffo changes if the reference monitor report errors */
-	if (!zl3073x_ref_is_status_ok(ref))
+	if (pin->operstate != DPLL_PIN_OPERSTATE_ACTIVE)
 		return false;
 
-	/* Compare with previous value */
-	ffo = zl3073x_ref_ffo_get(ref);
-	if (pin->freq_offset != ffo) {
-		dev_dbg(zldev->dev, "%s freq offset changed: %lld -> %lld\n",
-			pin->label, pin->freq_offset, ffo);
-		pin->freq_offset = ffo;
+	chan = zl3073x_chan_state_get(zldpll->dev, zldpll->id);
+	ffo = mul_s64_u64_shr(zl3073x_chan_df_offset_get(chan),
+			      244140625, 36);
 
+	if (atomic64_xchg(&pin->freq_offset, ffo) != ffo) {
+		dev_dbg(zldev->dev, "%s freq offset changed to: %lld\n",
+			pin->label, ffo);
 		return true;
 	}
 
@@ -1828,7 +1862,7 @@ zl3073x_dpll_changes_check(struct zl3073x_dpll *zldpll)
 	}
 
 	list_for_each_entry(pin, &zldpll->pins, list) {
-		enum dpll_pin_state state;
+		enum dpll_pin_operstate operstate;
 		bool pin_changed = false;
 
 		/* Output pins change checks are not necessary because output
@@ -1837,18 +1871,18 @@ zl3073x_dpll_changes_check(struct zl3073x_dpll *zldpll)
 		if (!zl3073x_dpll_is_input_pin(pin))
 			continue;
 
-		rc = zl3073x_dpll_ref_state_get(pin, &state);
+		rc = zl3073x_dpll_ref_operstate_get(pin, &operstate);
 		if (rc) {
 			dev_err(dev,
-				"Failed to get %s on DPLL%u state: %pe\n",
+				"Failed to get %s on DPLL%u oper state: %pe\n",
 				pin->label, zldpll->id, ERR_PTR(rc));
 			return;
 		}
 
-		if (state != pin->pin_state) {
-			dev_dbg(dev, "%s state changed: %u->%u\n", pin->label,
-				pin->pin_state, state);
-			pin->pin_state = state;
+		if (operstate != pin->operstate) {
+			dev_dbg(dev, "%s oper state changed: %u->%u\n",
+				pin->label, pin->operstate, operstate);
+			pin->operstate = operstate;
 			pin_changed = true;
 		}
 
diff --git a/drivers/dpll/zl3073x/i2c.c b/drivers/dpll/zl3073x/i2c.c
index 979df85826abc..4a23340e29d08 100644
--- a/drivers/dpll/zl3073x/i2c.c
+++ b/drivers/dpll/zl3073x/i2c.c
@@ -26,11 +26,11 @@ static int zl3073x_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id zl3073x_i2c_id[] = {
-	{ "zl30731" },
-	{ "zl30732" },
-	{ "zl30733" },
-	{ "zl30734" },
-	{ "zl30735" },
+	{ .name = "zl30731" },
+	{ .name = "zl30732" },
+	{ .name = "zl30733" },
+	{ .name = "zl30734" },
+	{ .name = "zl30735" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, zl3073x_i2c_id);
diff --git a/drivers/dpll/zl3073x/ref.h b/drivers/dpll/zl3073x/ref.h
index 55e80e4f08734..e140ca3ea17dc 100644
--- a/drivers/dpll/zl3073x/ref.h
+++ b/drivers/dpll/zl3073x/ref.h
@@ -22,7 +22,6 @@ struct zl3073x_dev;
  * @freq_ratio_n: FEC mode divisor
  * @sync_ctrl: reference sync control
  * @config: reference config
- * @ffo: current fractional frequency offset
  * @meas_freq: measured input frequency in Hz
  * @mon_status: reference monitor status
  */
@@ -40,7 +39,6 @@ struct zl3073x_ref {
 		u8	config;
 	);
 	struct_group(stat, /* Status */
-		s64	ffo;
 		u32	meas_freq;
 		u8	mon_status;
 	);
@@ -58,18 +56,6 @@ int zl3073x_ref_state_update(struct zl3073x_dev *zldev, u8 index);
 
 int zl3073x_ref_freq_factorize(u32 freq, u16 *base, u16 *mult);
 
-/**
- * zl3073x_ref_ffo_get - get current fractional frequency offset
- * @ref: pointer to ref state
- *
- * Return: the latest measured fractional frequency offset
- */
-static inline s64
-zl3073x_ref_ffo_get(const struct zl3073x_ref *ref)
-{
-	return ref->ffo;
-}
-
 /**
  * zl3073x_ref_meas_freq_get - get measured input frequency
  * @ref: pointer to ref state
diff --git a/drivers/dpll/zl3073x/regs.h b/drivers/dpll/zl3073x/regs.h
index d425dc67250fe..9578f00095282 100644
--- a/drivers/dpll/zl3073x/regs.h
+++ b/drivers/dpll/zl3073x/regs.h
@@ -98,7 +98,14 @@
 
 #define ZL_REG_REF_MON_STATUS(_idx)					\
 	ZL_REG_IDX(_idx, 2, 0x02, 1, ZL3073X_NUM_REFS, 1)
-#define ZL_REF_MON_STATUS_OK			0 /* all bits zeroed */
+#define ZL_REF_MON_STATUS_OK			0
+#define ZL_REF_MON_STATUS_LOS			BIT(0)
+#define ZL_REF_MON_STATUS_SCM			BIT(1)
+#define ZL_REF_MON_STATUS_CFM			BIT(2)
+#define ZL_REF_MON_STATUS_GST			BIT(3)
+#define ZL_REF_MON_STATUS_PFM			BIT(4)
+#define ZL_REF_MON_STATUS_ESYNC			BIT(6)
+#define ZL_REF_MON_STATUS_SPLIT_XO		BIT(7)
 
 #define ZL_REG_DPLL_MON_STATUS(_idx)					\
 	ZL_REG_IDX(_idx, 2, 0x10, 1, ZL3073X_MAX_CHANNELS, 1)
@@ -157,6 +164,11 @@
 #define ZL_DPLL_MODE_REFSEL_MODE_NCO		4
 #define ZL_DPLL_MODE_REFSEL_REF			GENMASK(7, 4)
 
+#define ZL_REG_DPLL_DF_READ(_idx)					\
+	ZL_REG_IDX(_idx, 5, 0x28, 1, ZL3073X_MAX_CHANNELS, 1)
+#define ZL_DPLL_DF_READ_SEM			BIT(4)
+#define ZL_DPLL_DF_READ_REF_OFST		BIT(3)
+
 #define ZL_REG_DPLL_MEAS_CTRL			ZL_REG(5, 0x50, 1)
 #define ZL_DPLL_MEAS_CTRL_EN			BIT(0)
 #define ZL_DPLL_MEAS_CTRL_AVG_FACTOR		GENMASK(7, 4)
@@ -169,6 +181,16 @@
 #define ZL_REG_DPLL_PHASE_ERR_DATA(_idx)				\
 	ZL_REG_IDX(_idx, 5, 0x55, 6, ZL3073X_MAX_CHANNELS, 6)
 
+/*******************************
+ * Register Pages 6-7, DPLL Data
+ *******************************/
+
+#define ZL_REG_DPLL_DF_OFFSET_03(_idx)					\
+	ZL_REG_IDX(_idx, 6, 0x00, 6, 4, 0x20)
+#define ZL_REG_DPLL_DF_OFFSET_4		ZL_REG(7, 0x00, 6)
+#define ZL_REG_DPLL_DF_OFFSET(_idx)					\
+	((_idx) < 4 ? ZL_REG_DPLL_DF_OFFSET_03(_idx) : ZL_REG_DPLL_DF_OFFSET_4)
+
 /***********************************
  * Register Page 9, Synth and Output
  ***********************************/
diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c
index 63df35444214b..de6c52dbd9d25 100644
--- a/drivers/edac/i10nm_base.c
+++ b/drivers/edac/i10nm_base.c
@@ -79,6 +79,7 @@ static struct res_config *res_cfg;
 static int retry_rd_err_log;
 static int decoding_via_mca;
 static bool mem_cfg_2lm;
+static bool no_adxl;
 
 static struct reg_rrl icx_reg_rrl_ddr = {
 	.set_num = 2,
@@ -1222,8 +1223,14 @@ static int __init i10nm_init(void)
 	}
 
 	rc = skx_adxl_get();
-	if (rc)
-		goto fail;
+	if (rc) {
+		/* Decoding errors via MCA banks for 2LM isn't supported yet */
+		if (rc != -ENODEV || mem_cfg_2lm)
+			goto fail;
+		i10nm_printk(KERN_INFO, "ADXL not found, falling back to MCA-based decoding.\n");
+		no_adxl = true;
+		decoding_via_mca = true;
+	}
 
 	opstate_init();
 	mce_register_decode_chain(&i10nm_mce_dec);
@@ -1257,7 +1264,8 @@ static void __exit i10nm_exit(void)
 
 	skx_teardown_debug();
 	mce_unregister_decode_chain(&i10nm_mce_dec);
-	skx_adxl_put();
+	if (!no_adxl)
+		skx_adxl_put();
 	skx_remove();
 }
 
diff --git a/drivers/edac/i5000_edac.c b/drivers/edac/i5000_edac.c
index 471b8540d18b0..c0faf55f78129 100644
--- a/drivers/edac/i5000_edac.c
+++ b/drivers/edac/i5000_edac.c
@@ -352,6 +352,9 @@ struct i5000_pvt {
 	/* Actual values for this controller */
 	int maxch;		/* Max channels */
 	int maxdimmperch;	/* Max DIMMs per channel */
+
+	/* Hardware error reporting status */
+	bool enabled_error_reporting;
 };
 
 /* I5000 MCH error information retrieved from Hardware */
@@ -1302,10 +1305,10 @@ static int i5000_init_csrows(struct mem_ctl_info *mci)
 }
 
 /*
- *	i5000_enable_error_reporting
- *			Turn on the memory reporting features of the hardware
+ *	i5000_set_error_reporting
+ *			Turn on/off the memory reporting features of the hardware
  */
-static void i5000_enable_error_reporting(struct mem_ctl_info *mci)
+static void i5000_set_error_reporting(struct mem_ctl_info *mci, bool enable)
 {
 	struct i5000_pvt *pvt;
 	u32 fbd_error_mask;
@@ -1316,8 +1319,11 @@ static void i5000_enable_error_reporting(struct mem_ctl_info *mci)
 	pci_read_config_dword(pvt->branchmap_werrors, EMASK_FBD,
 			&fbd_error_mask);
 
-	/* Enable with a '0' */
-	fbd_error_mask &= ~(ENABLE_EMASK_ALL);
+	/* Enable with 0, disable with 1 */
+	if (enable)
+		fbd_error_mask &= ~(ENABLE_EMASK_ALL);
+	else
+		fbd_error_mask |= ENABLE_EMASK_ALL;
 
 	pci_write_config_dword(pvt->branchmap_werrors, EMASK_FBD,
 			fbd_error_mask);
@@ -1435,17 +1441,19 @@ static int i5000_probe1(struct pci_dev *pdev, int dev_idx)
 	if (i5000_init_csrows(mci)) {
 		edac_dbg(0, "MC: Setting mci->edac_cap to EDAC_FLAG_NONE because i5000_init_csrows() returned nonzero value\n");
 		mci->edac_cap = EDAC_FLAG_NONE;	/* no csrows found */
+		pvt->enabled_error_reporting = false;
 	} else {
 		edac_dbg(1, "MC: Enable error reporting now\n");
-		i5000_enable_error_reporting(mci);
+		i5000_set_error_reporting(mci, true);
+		pvt->enabled_error_reporting = true;
 	}
 
 	/* add this new MC control structure to EDAC's list of MCs */
 	if (edac_mc_add_mc(mci)) {
 		edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
-		/* FIXME: perhaps some code should go here that disables error
-		 * reporting if we just enabled it
-		 */
+		/* Disable error reporting if we previously enabled it */
+		if (pvt->enabled_error_reporting)
+			i5000_set_error_reporting(mci, false);
 		goto fail1;
 	}
 
@@ -1503,6 +1511,7 @@ static int i5000_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 static void i5000_remove_one(struct pci_dev *pdev)
 {
 	struct mem_ctl_info *mci;
+	struct i5000_pvt *pvt;
 
 	edac_dbg(0, "\n");
 
@@ -1512,6 +1521,12 @@ static void i5000_remove_one(struct pci_dev *pdev)
 	if ((mci = edac_mc_del_mc(&pdev->dev)) == NULL)
 		return;
 
+	pvt = mci->pvt_info;
+
+	/* Disable error reporting on teardown */
+	if (pvt->enabled_error_reporting)
+		i5000_set_error_reporting(mci, false);
+
 	/* retrieve references to resources, and free those resources */
 	i5000_put_devices(mci);
 	edac_mc_free(mci);
diff --git a/drivers/edac/i5100_edac.c b/drivers/edac/i5100_edac.c
index d470afe650018..d30919ceb22ba 100644
--- a/drivers/edac/i5100_edac.c
+++ b/drivers/edac/i5100_edac.c
@@ -859,6 +859,21 @@ static void i5100_init_csrows(struct mem_ctl_info *mci)
 	}
 }
 
+static void i5100_set_error_reporting(struct pci_dev *pdev, bool enable)
+{
+	u32 dw;
+
+	pci_read_config_dword(pdev, I5100_EMASK_MEM, &dw);
+
+	/* Enable with 0, disable with 1 */
+	if (enable)
+		dw &= ~I5100_FERR_NF_MEM_ANY_MASK;
+	else
+		dw |= I5100_FERR_NF_MEM_ANY_MASK;
+
+	pci_write_config_dword(pdev, I5100_EMASK_MEM, dw);
+}
+
 /****************************************************************************
  *                       Error injection routines
  ****************************************************************************/
@@ -1004,11 +1019,6 @@ static int i5100_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	pci_read_config_dword(pdev, I5100_MS, &dw);
 	ranksperch = !!(dw & (1 << 8)) * 2 + 4;
 
-	/* enable error reporting... */
-	pci_read_config_dword(pdev, I5100_EMASK_MEM, &dw);
-	dw &= ~I5100_FERR_NF_MEM_ANY_MASK;
-	pci_write_config_dword(pdev, I5100_EMASK_MEM, dw);
-
 	/* device 21, func 0, Channel 0 Memory Map, Error Flag/Mask, etc... */
 	ch0mm = pci_get_device_func(PCI_VENDOR_ID_INTEL,
 				    PCI_DEVICE_ID_INTEL_5100_21, 0);
@@ -1125,6 +1135,9 @@ static int i5100_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	i5100_setup_debugfs(mci);
 
+	/* Enable error reporting on success */
+	i5100_set_error_reporting(pdev, true);
+
 	return ret;
 
 bail_scrub:
@@ -1169,6 +1182,9 @@ static void i5100_remove_one(struct pci_dev *pdev)
 
 	priv = mci->pvt_info;
 
+	/* Disable error reporting at teardown */
+	i5100_set_error_reporting(pdev, false);
+
 	edac_debugfs_remove_recursive(priv->debugfs);
 
 	priv->scrub_enable = 0;
diff --git a/drivers/edac/i5400_edac.c b/drivers/edac/i5400_edac.c
index fb49a1d1df112..ae4f9298952da 100644
--- a/drivers/edac/i5400_edac.c
+++ b/drivers/edac/i5400_edac.c
@@ -353,6 +353,9 @@ struct i5400_pvt {
 	/* Actual values for this controller */
 	int maxch;				/* Max channels */
 	int maxdimmperch;			/* Max DIMMs per channel */
+
+	/* Hardware error reporting status */
+	bool enabled_error_reporting;
 };
 
 /* I5400 MCH error information retrieved from Hardware */
@@ -1223,10 +1226,10 @@ static int i5400_init_dimms(struct mem_ctl_info *mci)
 }
 
 /*
- *	i5400_enable_error_reporting
- *			Turn on the memory reporting features of the hardware
+ *	i5400_set_error_reporting
+ *			Turn on/off the memory reporting features of the hardware
  */
-static void i5400_enable_error_reporting(struct mem_ctl_info *mci)
+static void i5400_set_error_reporting(struct mem_ctl_info *mci, bool enable)
 {
 	struct i5400_pvt *pvt;
 	u32 fbd_error_mask;
@@ -1237,8 +1240,11 @@ static void i5400_enable_error_reporting(struct mem_ctl_info *mci)
 	pci_read_config_dword(pvt->branchmap_werrors, EMASK_FBD,
 			&fbd_error_mask);
 
-	/* Enable with a '0' */
-	fbd_error_mask &= ~(ENABLE_EMASK_ALL);
+	/* Enable with 0, disable with 1 */
+	if (enable)
+		fbd_error_mask &= ~(ENABLE_EMASK_ALL);
+	else
+		fbd_error_mask |= ENABLE_EMASK_ALL;
 
 	pci_write_config_dword(pvt->branchmap_werrors, EMASK_FBD,
 			fbd_error_mask);
@@ -1319,17 +1325,19 @@ static int i5400_probe1(struct pci_dev *pdev, int dev_idx)
 	if (i5400_init_dimms(mci)) {
 		edac_dbg(0, "MC: Setting mci->edac_cap to EDAC_FLAG_NONE because i5400_init_dimms() returned nonzero value\n");
 		mci->edac_cap = EDAC_FLAG_NONE;	/* no dimms found */
+		pvt->enabled_error_reporting = false;
 	} else {
 		edac_dbg(1, "MC: Enable error reporting now\n");
-		i5400_enable_error_reporting(mci);
+		i5400_set_error_reporting(mci, true);
+		pvt->enabled_error_reporting = true;
 	}
 
 	/* add this new MC control structure to EDAC's list of MCs */
 	if (edac_mc_add_mc(mci)) {
 		edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
-		/* FIXME: perhaps some code should go here that disables error
-		 * reporting if we just enabled it
-		 */
+		/* Disable error reporting if we just enabled it */
+		if (pvt->enabled_error_reporting)
+			i5400_set_error_reporting(mci, false);
 		goto fail1;
 	}
 
@@ -1387,6 +1395,7 @@ static int i5400_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 static void i5400_remove_one(struct pci_dev *pdev)
 {
 	struct mem_ctl_info *mci;
+	struct i5400_pvt *pvt;
 
 	edac_dbg(0, "\n");
 
@@ -1397,6 +1406,12 @@ static void i5400_remove_one(struct pci_dev *pdev)
 	if (!mci)
 		return;
 
+	pvt = mci->pvt_info;
+
+	/* Disable error reporting on teardown */
+	if (pvt->enabled_error_reporting)
+		i5400_set_error_reporting(mci, false);
+
 	/* retrieve references to resources, and free those resources */
 	i5400_put_devices(mci);
 
diff --git a/drivers/edac/i7300_edac.c b/drivers/edac/i7300_edac.c
index 69068f8d0cadf..64bc2d805a629 100644
--- a/drivers/edac/i7300_edac.c
+++ b/drivers/edac/i7300_edac.c
@@ -111,6 +111,9 @@ struct i7300_pvt {
 
 	/* Temporary buffer for use when preparing error messages */
 	char *tmp_prt_buffer;
+
+	/* Hardware error reporting status */
+	bool enabled_error_reporting;
 };
 
 /* FIXME: Why do we need to have this static? */
@@ -550,11 +553,12 @@ static void i7300_clear_error(struct mem_ctl_info *mci)
 }
 
 /**
- * i7300_enable_error_reporting() - Enable the memory reporting logic at the
+ * i7300_set_error_reporting() - Enable or disable the memory reporting logic at the
  *				    hardware
  * @mci: struct mem_ctl_info pointer
+ * @enable: enables if 'true', disables if 'false'
  */
-static void i7300_enable_error_reporting(struct mem_ctl_info *mci)
+static void i7300_set_error_reporting(struct mem_ctl_info *mci, bool enable)
 {
 	struct i7300_pvt *pvt = mci->pvt_info;
 	u32 fbd_error_mask;
@@ -563,8 +567,11 @@ static void i7300_enable_error_reporting(struct mem_ctl_info *mci)
 	pci_read_config_dword(pvt->pci_dev_16_1_fsb_addr_map,
 			      EMASK_FBD, &fbd_error_mask);
 
-	/* Enable with a '0' */
-	fbd_error_mask &= ~(EMASK_FBD_ERR_MASK);
+	/* Enable with 0, disable with 1 */
+	if (enable)
+		fbd_error_mask &= ~(EMASK_FBD_ERR_MASK);
+	else
+		fbd_error_mask |= EMASK_FBD_ERR_MASK;
 
 	pci_write_config_dword(pvt->pci_dev_16_1_fsb_addr_map,
 			       EMASK_FBD, fbd_error_mask);
@@ -1087,17 +1094,19 @@ static int i7300_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 	if (i7300_get_mc_regs(mci)) {
 		edac_dbg(0, "MC: Setting mci->edac_cap to EDAC_FLAG_NONE because i7300_init_csrows() returned nonzero value\n");
 		mci->edac_cap = EDAC_FLAG_NONE;	/* no csrows found */
+		pvt->enabled_error_reporting = false;
 	} else {
 		edac_dbg(1, "MC: Enable error reporting now\n");
-		i7300_enable_error_reporting(mci);
+		i7300_set_error_reporting(mci, true);
+		pvt->enabled_error_reporting = true;
 	}
 
 	/* add this new MC control structure to EDAC's list of MCs */
 	if (edac_mc_add_mc(mci)) {
 		edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
-		/* FIXME: perhaps some code should go here that disables error
-		 * reporting if we just enabled it
-		 */
+		/* Disable error reporting if we just enabled it */
+		if (pvt->enabled_error_reporting)
+			i7300_set_error_reporting(mci, false);
 		goto fail1;
 	}
 
@@ -1134,6 +1143,7 @@ static int i7300_init_one(struct pci_dev *pdev, const struct pci_device_id *id)
 static void i7300_remove_one(struct pci_dev *pdev)
 {
 	struct mem_ctl_info *mci;
+	struct i7300_pvt *pvt;
 	char *tmp;
 
 	edac_dbg(0, "\n");
@@ -1145,7 +1155,12 @@ static void i7300_remove_one(struct pci_dev *pdev)
 	if (!mci)
 		return;
 
-	tmp = ((struct i7300_pvt *)mci->pvt_info)->tmp_prt_buffer;
+	pvt = (struct i7300_pvt *)mci->pvt_info;
+	tmp = pvt->tmp_prt_buffer;
+
+	/* Disable error reporting before unregistering device */
+	if (pvt->enabled_error_reporting)
+		i7300_set_error_reporting(mci, false);
 
 	/* retrieve references to resources, and free those resources */
 	i7300_put_devices(mci);
diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c
index fcb8ab44cba55..f3e53d63eb54f 100644
--- a/drivers/edac/igen6_edac.c
+++ b/drivers/edac/igen6_edac.c
@@ -122,6 +122,20 @@
 #define MEM_SLICE_HASH_MASK(v)		(GET_BITFIELD(v, 6, 19) << 6)
 #define MEM_SLICE_HASH_LSB_MASK_BIT(v)	GET_BITFIELD(v, 24, 26)
 
+struct igen6_imc {
+	int mc;
+	struct mem_ctl_info *mci;
+	struct pci_dev *pdev;
+	struct device dev;
+	void __iomem *window;
+	u64 size;
+	u64 ch_s_size;
+	int ch_l_map;
+	u64 dimm_s_size[NUM_CHANNELS];
+	u64 dimm_l_size[NUM_CHANNELS];
+	int dimm_l_map[NUM_CHANNELS];
+};
+
 static struct res_config {
 	bool machine_check;
 	/* The number of present memory controllers. */
@@ -134,12 +148,29 @@ static struct res_config {
 	u64 reg_touud_mask;
 	/* IBECC error log */
 	u64 reg_eccerrlog_addr_mask;
+	/* MEMSS_PMA_CR registers. */
+	u32 reg_mem_config_offset;
+	u32 reg_mem_config_ddr_type_mask;
+	/* Memory controller registers. */
+	u32 reg_mad_inter_size_mask[NUM_CHANNELS];
+	u64 reg_mad_inter_size_granularity;
+	u32 reg_mad_intra_rank_mask[NUM_DIMMS];
+	u32 reg_mad_intra_width_mask[NUM_DIMMS];
+	u32 reg_mad_intra_density_mask[NUM_DIMMS];
 	u32 imc_base;
 	u32 cmf_base;
 	u32 cmf_size;
 	u32 ms_hash_offset;
 	u32 ibecc_base;
 	u32 ibecc_error_log_offset;
+	/* Get memory type. */
+	enum mem_type (*get_mem_type)(struct igen6_imc *imc);
+	/* Get DRAM chip type. */
+	enum dev_type (*get_dev_type)(struct igen6_imc *imc, int chan, int dimm_l);
+	/* Set imc->ch_{s_size,l_map}. */
+	void (*set_chan_params)(struct igen6_imc *imc);
+	/* Set imc->dimm_{l_size,s_size,l_map}[chan]. */
+	void (*set_dimm_params)(struct igen6_imc *imc, int chan);
 	bool (*ibecc_available)(struct pci_dev *pdev);
 	/* Extract error address logged in IBECC */
 	u64 (*err_addr)(u64 ecclog);
@@ -149,22 +180,9 @@ static struct res_config {
 	u64 (*err_addr_to_imc_addr)(u64 eaddr, int mc);
 } *res_cfg;
 
-struct igen6_imc {
-	int mc;
-	struct mem_ctl_info *mci;
-	struct pci_dev *pdev;
-	struct device dev;
-	void __iomem *window;
-	u64 size;
-	u64 ch_s_size;
-	int ch_l_map;
-	u64 dimm_s_size[NUM_CHANNELS];
-	u64 dimm_l_size[NUM_CHANNELS];
-	int dimm_l_map[NUM_CHANNELS];
-};
-
 static struct igen6_pvt {
 	struct igen6_imc imc[NUM_IMC];
+	void __iomem *memss_pma_cr;
 	u64 ms_hash;
 	u64 ms_s_size;
 	int ms_l_map;
@@ -291,6 +309,7 @@ static struct work_struct ecclog_work;
 #define DID_PTL_H_SKU11	0xb028
 #define DID_PTL_H_SKU12	0xb029
 #define DID_PTL_H_SKU13	0xb02a
+#define DID_PTL_H_SKU14	0xb00a
 
 /* Compute die IDs for Wildcat Lake with IBECC */
 #define DID_WCL_SKU1	0xfd00
@@ -500,6 +519,119 @@ static u64 rpl_p_err_addr(u64 ecclog)
 	return field_get(res_cfg->reg_eccerrlog_addr_mask, ecclog);
 }
 
+static enum mem_type ptl_h_get_mem_type(struct igen6_imc *imc)
+{
+	u32 mtype, val;
+
+	val = readl(igen6_pvt->memss_pma_cr + res_cfg->reg_mem_config_offset);
+	mtype = field_get(res_cfg->reg_mem_config_ddr_type_mask, val);
+
+	edac_dbg(2, "mtype %u (reg 0x%x)\n", mtype, val);
+
+	switch (mtype) {
+	case 1:
+		return MEM_DDR5;
+	case 2:
+		return MEM_LPDDR5;
+	case 3:
+		return MEM_LPDDR4;
+	default:
+		return MEM_UNKNOWN;
+	}
+}
+
+static enum dev_type ptl_h_get_dev_type(struct igen6_imc *imc, int chan, int dimm)
+{
+	u32 width, val;
+
+	val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4);
+	width = field_get(res_cfg->reg_mad_intra_width_mask[dimm], val);
+
+	switch (width) {
+	case 1:
+		return DEV_X8;
+	default:
+		return DEV_X16;
+	}
+}
+
+static u64 ptl_h_get_chan_size(struct igen6_imc *imc, int chan)
+{
+	u32 val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
+
+	return field_get(res_cfg->reg_mad_inter_size_mask[chan], val) *
+	       res_cfg->reg_mad_inter_size_granularity;
+}
+
+static u64 ptl_h_get_dimm_size(struct igen6_imc *imc, int chan, int dimm)
+{
+	u32 val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4);
+	u32 ranks = 1 << field_get(res_cfg->reg_mad_intra_rank_mask[dimm], val);
+	/* DRAM device density in Gb */
+	u64 density = field_get(res_cfg->reg_mad_intra_density_mask[dimm], val) * 4;
+
+	enum mem_type mtype = ptl_h_get_mem_type(imc);
+	enum dev_type dtype = ptl_h_get_dev_type(imc, chan, dimm);
+	u64 sub_ch_width, dev_num;
+
+	switch (mtype) {
+	case MEM_DDR5:
+		sub_ch_width = 32;
+		break;
+	case MEM_LPDDR5:
+	case MEM_LPDDR4:
+		sub_ch_width = 16;
+		break;
+	default:
+		sub_ch_width = 0;
+	}
+
+	switch (dtype) {
+	case DEV_X8:
+		dev_num = sub_ch_width / 8;
+		break;
+	case DEV_X16:
+		dev_num = sub_ch_width / 16;
+		break;
+	default:
+		dev_num = 0;
+	}
+
+	edac_dbg(2, "ranks %d, density %lluGb, sub_ch_width %llu, dev_num %llu (reg 0x%x)\n", ranks, density, sub_ch_width, dev_num, val);
+
+	return ((dev_num * density / 8) * ranks) << 30;
+}
+
+static void ptl_h_set_chan_params(struct igen6_imc *imc)
+{
+	u64 ch0_size = ptl_h_get_chan_size(imc, 0);
+	u64 ch1_size = ptl_h_get_chan_size(imc, 1);
+
+	if (ch0_size <= ch1_size) {
+		imc->ch_s_size = ch0_size;
+		imc->ch_l_map = 1;
+	} else {
+		imc->ch_s_size = ch1_size;
+		imc->ch_l_map = 0;
+	}
+}
+
+static void ptl_h_set_dimm_params(struct igen6_imc *imc, int chan)
+{
+	u64 dimm0_size = ptl_h_get_dimm_size(imc, chan, 0);
+	u64 dimm1_size = ptl_h_get_dimm_size(imc, chan, 1);
+
+	if (dimm0_size <= dimm1_size) {
+		imc->dimm_s_size[chan] = dimm0_size;
+		imc->dimm_l_size[chan] = dimm1_size;
+		imc->dimm_l_map[chan]  = 1;
+	} else {
+		imc->dimm_s_size[chan] = dimm1_size;
+		imc->dimm_l_size[chan] = dimm0_size;
+		imc->dimm_l_map[chan]  = 0;
+	}
+}
+
 static struct res_config ehl_cfg = {
 	.num_imc		= 1,
 	.reg_mchbar_mask	= GENMASK_ULL(38, 16),
@@ -622,6 +754,36 @@ static struct res_config mtl_p_cfg = {
 	.err_addr_to_imc_addr	= adl_err_addr_to_imc_addr,
 };
 
+static struct res_config ptl_h_cfg = {
+	.machine_check			= true,
+	.num_imc			= 2,
+	.reg_mchbar_mask		= GENMASK_ULL(41, 17),
+	.reg_tom_mask			= GENMASK_ULL(41, 20),
+	.reg_touud_mask			= GENMASK_ULL(41, 20),
+	.reg_eccerrlog_addr_mask	= GENMASK_ULL(38, 5),
+	.reg_mem_config_offset		= 0x13d04,
+	.reg_mem_config_ddr_type_mask	= GENMASK(8, 6),
+	.reg_mad_inter_size_mask[0]	= GENMASK(15, 8),
+	.reg_mad_inter_size_mask[1]	= GENMASK(23, 16),
+	.reg_mad_inter_size_granularity	= BIT_ULL(29),
+	.reg_mad_intra_rank_mask[0]	= BIT(7),
+	.reg_mad_intra_rank_mask[1]	= BIT(15),
+	.reg_mad_intra_width_mask[0]	= BIT(6),
+	.reg_mad_intra_width_mask[1]	= BIT(14),
+	.reg_mad_intra_density_mask[0]	= GENMASK(3, 0),
+	.reg_mad_intra_density_mask[1]	= GENMASK(11, 8),
+	.imc_base			= 0xd800,
+	.ibecc_base			= 0xd400,
+	.ibecc_error_log_offset		= 0x170,
+	.get_mem_type			= ptl_h_get_mem_type,
+	.get_dev_type			= ptl_h_get_dev_type,
+	.set_chan_params		= ptl_h_set_chan_params,
+	.set_dimm_params		= ptl_h_set_dimm_params,
+	.ibecc_available		= mtl_p_ibecc_available,
+	.err_addr_to_sys_addr		= adl_err_addr_to_sys_addr,
+	.err_addr_to_imc_addr		= adl_err_addr_to_imc_addr,
+};
+
 static struct res_config wcl_cfg = {
 	.machine_check		= true,
 	.num_imc		= 1,
@@ -689,46 +851,35 @@ static struct pci_device_id igen6_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, DID_ARL_UH_SKU1), (kernel_ulong_t)&mtl_p_cfg },
 	{ PCI_VDEVICE(INTEL, DID_ARL_UH_SKU2), (kernel_ulong_t)&mtl_p_cfg },
 	{ PCI_VDEVICE(INTEL, DID_ARL_UH_SKU3), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), (kernel_ulong_t)&mtl_p_cfg },
-	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), (kernel_ulong_t)&mtl_p_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU1), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU2), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU3), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU4), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU5), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU6), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU7), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU8), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU9), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU10), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU11), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU12), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU13), (kernel_ulong_t)&ptl_h_cfg },
+	{ PCI_VDEVICE(INTEL, DID_PTL_H_SKU14), (kernel_ulong_t)&ptl_h_cfg },
 	{ PCI_VDEVICE(INTEL, DID_WCL_SKU1), (kernel_ulong_t)&wcl_cfg },
 	{ },
 };
 MODULE_DEVICE_TABLE(pci, igen6_pci_tbl);
 
-static enum dev_type get_width(int dimm_l, u32 mad_dimm)
+static enum mem_type get_mem_type(struct igen6_imc *imc)
 {
-	u32 w = dimm_l ? MAD_DIMM_CH_DLW(mad_dimm) :
-			 MAD_DIMM_CH_DSW(mad_dimm);
+	u32 val;
 
-	switch (w) {
-	case 0:
-		return DEV_X8;
-	case 1:
-		return DEV_X16;
-	case 2:
-		return DEV_X32;
-	default:
-		return DEV_UNKNOWN;
-	}
-}
+	if (res_cfg->get_mem_type)
+		return res_cfg->get_mem_type(imc);
 
-static enum mem_type get_memory_type(u32 mad_inter)
-{
-	u32 t = MAD_INTER_CHANNEL_DDR_TYPE(mad_inter);
+	val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
 
-	switch (t) {
+	switch (MAD_INTER_CHANNEL_DDR_TYPE(val)) {
 	case 0:
 		return MEM_DDR4;
 	case 1:
@@ -744,6 +895,73 @@ static enum mem_type get_memory_type(u32 mad_inter)
 	}
 }
 
+static bool large_dimm(struct igen6_imc *imc, int chan, int dimm)
+{
+	return dimm == imc->dimm_l_map[chan];
+}
+
+static enum dev_type get_dev_type(struct igen6_imc *imc, int chan, int dimm)
+{
+	u32 width, val;
+
+	if (res_cfg->get_dev_type)
+		return res_cfg->get_dev_type(imc, chan, dimm);
+
+	val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4);
+	width = large_dimm(imc, chan, dimm) ? MAD_DIMM_CH_DLW(val) :
+					  MAD_DIMM_CH_DSW(val);
+
+	switch (width) {
+	case 0:
+		return DEV_X8;
+	case 1:
+		return DEV_X16;
+	case 2:
+		return DEV_X32;
+	default:
+		return DEV_UNKNOWN;
+	}
+}
+
+static u64 get_dimm_size(struct igen6_imc *imc, int chan, int dimm)
+{
+	if (large_dimm(imc, chan, dimm))
+		return imc->dimm_l_size[chan];
+
+	return imc->dimm_s_size[chan];
+}
+
+static void set_chan_params(struct igen6_imc *imc)
+{
+	u32 val;
+
+	if (res_cfg->set_chan_params) {
+		res_cfg->set_chan_params(imc);
+		return;
+	}
+
+	val = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
+	imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(val);
+	imc->ch_l_map = MAD_INTER_CHANNEL_CH_L_MAP(val);
+}
+
+static void set_dimm_params(struct igen6_imc *imc, int chan)
+{
+	u32 val;
+
+	if (res_cfg->set_dimm_params) {
+		res_cfg->set_dimm_params(imc, chan);
+		return;
+	}
+
+	val = readl(imc->window + MAD_INTRA_CH0_OFFSET + chan * 4);
+	imc->dimm_l_map[chan]  = MAD_INTRA_CH_DIMM_L_MAP(val);
+
+	val = readl(imc->window + MAD_DIMM_CH0_OFFSET + chan * 4);
+	imc->dimm_l_size[chan] = MAD_DIMM_CH_DIMM_L_SIZE(val);
+	imc->dimm_s_size[chan] = MAD_DIMM_CH_DIMM_S_SIZE(val);
+}
+
 static int decode_chan_idx(u64 addr, u64 mask, int intlv_bit)
 {
 	u64 hash_addr = addr & mask, hash = 0;
@@ -1084,7 +1302,6 @@ static bool igen6_check_ecc(struct igen6_imc *imc)
 static int igen6_get_dimm_config(struct mem_ctl_info *mci)
 {
 	struct igen6_imc *imc = mci->pvt_info;
-	u32 mad_inter, mad_intra, mad_dimm;
 	int i, j, ndimms, mc = imc->mc;
 	struct dimm_info *dimm;
 	enum mem_type mtype;
@@ -1094,33 +1311,20 @@ static int igen6_get_dimm_config(struct mem_ctl_info *mci)
 
 	edac_dbg(2, "\n");
 
-	mad_inter = readl(imc->window + MAD_INTER_CHANNEL_OFFSET);
-	mtype = get_memory_type(mad_inter);
+	mtype = get_mem_type(imc);
 	ecc = igen6_check_ecc(imc);
-	imc->ch_s_size = MAD_INTER_CHANNEL_CH_S_SIZE(mad_inter);
-	imc->ch_l_map  = MAD_INTER_CHANNEL_CH_L_MAP(mad_inter);
+	set_chan_params(imc);
 
 	for (i = 0; i < NUM_CHANNELS; i++) {
-		mad_intra = readl(imc->window + MAD_INTRA_CH0_OFFSET + i * 4);
-		mad_dimm  = readl(imc->window + MAD_DIMM_CH0_OFFSET + i * 4);
-
-		imc->dimm_l_size[i] = MAD_DIMM_CH_DIMM_L_SIZE(mad_dimm);
-		imc->dimm_s_size[i] = MAD_DIMM_CH_DIMM_S_SIZE(mad_dimm);
-		imc->dimm_l_map[i]  = MAD_INTRA_CH_DIMM_L_MAP(mad_intra);
+		set_dimm_params(imc, i);
 		imc->size += imc->dimm_s_size[i];
 		imc->size += imc->dimm_l_size[i];
 		ndimms = 0;
 
 		for (j = 0; j < NUM_DIMMS; j++) {
 			dimm = edac_get_dimm(mci, i, j, 0);
-
-			if (j ^ imc->dimm_l_map[i]) {
-				dtype = get_width(0, mad_dimm);
-				dsize = imc->dimm_s_size[i];
-			} else {
-				dtype = get_width(1, mad_dimm);
-				dsize = imc->dimm_l_size[i];
-			}
+			dtype = get_dev_type(imc, i, j);
+			dsize = get_dimm_size(imc, i, j);
 
 			if (!dsize)
 				continue;
@@ -1223,6 +1427,39 @@ static void igen6_debug_setup(void) {}
 static void igen6_debug_teardown(void) {}
 #endif
 
+static struct igen6_pvt *igen6_pvt_setup(struct pci_dev *pdev)
+{
+	void __iomem *memss_pma_cr;
+	struct igen6_pvt *pvt;
+	u64 mchbar;
+	int rc;
+
+	pvt = kzalloc_obj(*igen6_pvt);
+	if (!pvt)
+		return NULL;
+
+	rc = get_mchbar(pdev, &mchbar);
+	if (rc) {
+		kfree(pvt);
+		return NULL;
+	}
+
+	memss_pma_cr = ioremap(mchbar, MCHBAR_SIZE * 2);
+	if (!memss_pma_cr) {
+		kfree(pvt);
+		return NULL;
+	}
+	pvt->memss_pma_cr = memss_pma_cr;
+
+	return pvt;
+}
+
+static void igen6_pvt_release(struct igen6_pvt *pvt)
+{
+	iounmap(pvt->memss_pma_cr);
+	kfree(pvt);
+}
+
 static int igen6_pci_setup(struct pci_dev *pdev, u64 *mchbar)
 {
 	union  {
@@ -1296,6 +1533,11 @@ static bool igen6_imc_absent(void __iomem *window)
 	return readl(window + MAD_INTER_CHANNEL_OFFSET) == ~0;
 }
 
+static void imc_release(struct device *dev)
+{
+	/* Nothing to do, the 'imc' owns the 'dev' and will also release it. */
+}
+
 static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev)
 {
 	struct edac_mc_layer layers[2];
@@ -1334,6 +1576,7 @@ static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev
 	mci->pvt_info = &igen6_pvt->imc[mc];
 
 	imc = mci->pvt_info;
+	imc->dev.release = imc_release;
 	device_initialize(&imc->dev);
 	/*
 	 * EDAC core uses mci->pdev(pointer of structure device) as
@@ -1549,12 +1792,12 @@ static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	edac_dbg(2, "\n");
 
-	igen6_pvt = kzalloc_obj(*igen6_pvt);
+	res_cfg = (struct res_config *)ent->driver_data;
+
+	igen6_pvt = igen6_pvt_setup(pdev);
 	if (!igen6_pvt)
 		return -ENOMEM;
 
-	res_cfg = (struct res_config *)ent->driver_data;
-
 	rc = igen6_pci_setup(pdev, &mchbar);
 	if (rc)
 		goto fail;
@@ -1603,7 +1846,7 @@ static int igen6_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 fail2:
 	igen6_unregister_mcis();
 fail:
-	kfree(igen6_pvt);
+	igen6_pvt_release(igen6_pvt);
 	return rc;
 }
 
@@ -1618,7 +1861,7 @@ static void igen6_remove(struct pci_dev *pdev)
 	flush_work(&ecclog_work);
 	gen_pool_destroy(ecclog_pool);
 	igen6_unregister_mcis();
-	kfree(igen6_pvt);
+	igen6_pvt_release(igen6_pvt);
 }
 
 static struct pci_driver igen6_driver = {
diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c
index 7b282dfd093f2..35eb7a2038ab3 100644
--- a/drivers/edac/sb_edac.c
+++ b/drivers/edac/sb_edac.c
@@ -2016,7 +2016,7 @@ static bool sb_decode_ddr4(struct mem_ctl_info *mci, int ch, u8 rank,
 static bool sb_decode_ddr3(struct mem_ctl_info *mci, int ch, u8 rank,
 			   u64 rank_addr, char *msg)
 {
-	pr_warn_once("DDR3 row/column decode not support yet!\n");
+	pr_warn_once("DDR3 row/column decode is not supported yet!\n");
 	msg[0] = '\0';
 	return false;
 }
diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c
index a9557c8344bc1..f15de0ea96c87 100644
--- a/drivers/edac/skx_common.c
+++ b/drivers/edac/skx_common.c
@@ -466,6 +466,9 @@ int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm,
 	rows = numrow(mtr);
 	cols = imc->hbm_mc ? 6 : numcol(mtr);
 
+	if (ranks < 0 || rows < 0 || cols < 0)
+		return 0;
+
 	if (imc->hbm_mc) {
 		banks = 32;
 		mtype = MEM_HBM2;
diff --git a/drivers/firewire/core-cdev.c b/drivers/firewire/core-cdev.c
index f791db4c8dfff..c166e7617d2ab 100644
--- a/drivers/firewire/core-cdev.c
+++ b/drivers/firewire/core-cdev.c
@@ -128,19 +128,38 @@ struct descriptor_resource {
 	u32 data[];
 };
 
-struct iso_resource {
+struct iso_resource_params {
+	int generation;
+	u64 channels;
+	s32 bandwidth;
+};
+
+struct iso_resource_auto {
 	struct client_resource resource;
 	struct client *client;
 	/* Schedule work and access todo only with client->lock held. */
 	struct delayed_work work;
-	enum {ISO_RES_ALLOC, ISO_RES_REALLOC, ISO_RES_DEALLOC,
-	      ISO_RES_ALLOC_ONCE, ISO_RES_DEALLOC_ONCE,} todo;
-	int generation;
-	u64 channels;
-	s32 bandwidth;
+	enum {
+		ISO_RES_AUTO_ALLOC,
+		ISO_RES_AUTO_REALLOC,
+		ISO_RES_AUTO_DEALLOC,
+	} todo;
+	struct iso_resource_params params;
 	struct iso_resource_event *e_alloc, *e_dealloc;
 };
 
+struct iso_resource_once {
+	struct client *client;
+	// Schedule work and access todo only with client->lock held.
+	struct delayed_work work;
+	enum {
+		ISO_RES_ONCE_ALLOC,
+		ISO_RES_ONCE_DEALLOC,
+	} todo;
+	struct iso_resource_params params;
+	struct iso_resource_event *event;
+};
+
 static struct address_handler_resource *to_address_handler_resource(struct client_resource *resource)
 {
 	return container_of(resource, struct address_handler_resource, resource);
@@ -156,16 +175,16 @@ static struct descriptor_resource *to_descriptor_resource(struct client_resource
 	return container_of(resource, struct descriptor_resource, resource);
 }
 
-static struct iso_resource *to_iso_resource(struct client_resource *resource)
+static struct iso_resource_auto *to_iso_resource_auto(struct client_resource *resource)
 {
-	return container_of(resource, struct iso_resource, resource);
+	return container_of(resource, struct iso_resource_auto, resource);
 }
 
-static void release_iso_resource(struct client *, struct client_resource *);
+static void release_iso_resource_auto(struct client *, struct client_resource *);
 
-static int is_iso_resource(const struct client_resource *resource)
+static int is_iso_resource_auto(const struct client_resource *resource)
 {
-	return resource->release == release_iso_resource;
+	return resource->release == release_iso_resource_auto;
 }
 
 static void release_transaction(struct client *client,
@@ -176,7 +195,7 @@ static int is_outbound_transaction_resource(const struct client_resource *resour
 	return resource->release == release_transaction;
 }
 
-static void schedule_iso_resource(struct iso_resource *r, unsigned long delay)
+static void schedule_iso_resource_auto(struct iso_resource_auto *r, unsigned long delay)
 {
 	client_get(r->client);
 	if (!queue_delayed_work(fw_workqueue, &r->work, delay))
@@ -424,8 +443,8 @@ static void queue_bus_reset_event(struct client *client)
 	guard(spinlock_irq)(&client->lock);
 
 	xa_for_each(&client->resource_xa, index, resource) {
-		if (is_iso_resource(resource))
-			schedule_iso_resource(to_iso_resource(resource), 0);
+		if (is_iso_resource_auto(resource))
+			schedule_iso_resource_auto(to_iso_resource_auto(resource), 0);
 	}
 }
 
@@ -507,31 +526,28 @@ static int ioctl_get_info(struct client *client, union ioctl_arg *arg)
 static int add_client_resource(struct client *client, struct client_resource *resource,
 			       gfp_t gfp_mask)
 {
-	int ret;
-
 	scoped_guard(spinlock_irqsave, &client->lock) {
 		u32 index;
+		int ret;
 
-		if (client->in_shutdown) {
-			ret = -ECANCELED;
+		if (client->in_shutdown)
+			return  -ECANCELED;
+
+		if (gfpflags_allow_blocking(gfp_mask)) {
+			ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b,
+				       GFP_NOWAIT);
 		} else {
-			if (gfpflags_allow_blocking(gfp_mask)) {
-				ret = xa_alloc(&client->resource_xa, &index, resource, xa_limit_32b,
-					       GFP_NOWAIT);
-			} else {
-				ret = xa_alloc_bh(&client->resource_xa, &index, resource,
-						  xa_limit_32b, GFP_NOWAIT);
-			}
-		}
-		if (ret >= 0) {
-			resource->handle = index;
-			client_get(client);
-			if (is_iso_resource(resource))
-				schedule_iso_resource(to_iso_resource(resource), 0);
+			ret = xa_alloc_bh(&client->resource_xa, &index, resource,
+					  xa_limit_32b, GFP_NOWAIT);
 		}
+		if (ret < 0)
+			return ret;
+
+		resource->handle = index;
+		client_get(client);
 	}
 
-	return ret < 0 ? ret : 0;
+	return 0;
 }
 
 static int release_client_resource(struct client *client, u32 handle,
@@ -1293,83 +1309,107 @@ static int ioctl_get_cycle_timer(struct client *client, union ioctl_arg *arg)
 	return 0;
 }
 
-static void iso_resource_work(struct work_struct *work)
+static int fill_iso_resource_params(struct iso_resource_params *params,
+				    struct fw_cdev_allocate_iso_resource *request)
+{
+	if ((request->channels == 0 && request->bandwidth == 0) ||
+	    request->bandwidth > BANDWIDTH_AVAILABLE_INITIAL)
+		return -EINVAL;
+
+	params->generation = -1;
+	params->channels = request->channels;
+	params->bandwidth = request->bandwidth;
+
+	return 0;
+}
+
+static void iso_resource_auto_work(struct work_struct *work)
 {
 	struct iso_resource_event *e;
-	struct iso_resource *r = from_work(r, work, work.work);
+	struct iso_resource_auto *r = from_work(r, work, work.work);
 	struct client *client = r->client;
 	unsigned long index = r->resource.handle;
-	int generation, channel, bandwidth, todo;
-	bool skip, free, success;
+	int current_generation, resource_generation, channel, bandwidth, todo;
+	u64 reset_jiffies;
+	bool free;
 
 	scoped_guard(spinlock_irq, &client->lock) {
-		generation = client->device->generation;
+		reset_jiffies = client->device->card->reset_jiffies;
+		current_generation = client->device->generation;
+		resource_generation = r->params.generation;
+		r->params.generation = current_generation;
 		todo = r->todo;
+	}
+
+	switch (todo) {
+	case ISO_RES_AUTO_ALLOC:
 		// Allow 1000ms grace period for other reallocations.
-		if (todo == ISO_RES_ALLOC &&
-		    time_is_after_jiffies64(client->device->card->reset_jiffies + secs_to_jiffies(1))) {
-			schedule_iso_resource(r, msecs_to_jiffies(333));
-			skip = true;
-		} else {
-			// We could be called twice within the same generation.
-			skip = todo == ISO_RES_REALLOC &&
-			       r->generation == generation;
+		if (time_is_after_jiffies64(reset_jiffies + secs_to_jiffies(1))) {
+			schedule_iso_resource_auto(r, msecs_to_jiffies(333));
+			goto out;
 		}
-		free = todo == ISO_RES_DEALLOC ||
-		       todo == ISO_RES_ALLOC_ONCE ||
-		       todo == ISO_RES_DEALLOC_ONCE;
-		r->generation = generation;
+		break;
+	case ISO_RES_AUTO_REALLOC:
+		// We could be called twice within the same generation.
+		if (resource_generation == current_generation)
+			goto out;
+		break;
+	case ISO_RES_AUTO_DEALLOC:
+	default:
+		break;
 	}
 
-	if (skip)
-		goto out;
+	bandwidth = r->params.bandwidth;
 
-	bandwidth = r->bandwidth;
+	fw_iso_resource_manage(client->device->card, current_generation, r->params.channels,
+			       &channel, &bandwidth, todo != ISO_RES_AUTO_DEALLOC);
 
-	fw_iso_resource_manage(client->device->card, generation,
-			r->channels, &channel, &bandwidth,
-			todo == ISO_RES_ALLOC ||
-			todo == ISO_RES_REALLOC ||
-			todo == ISO_RES_ALLOC_ONCE);
-	/*
-	 * Is this generation outdated already?  As long as this resource sticks
-	 * in the xarray, it will be scheduled again for a newer generation or at
-	 * shutdown.
-	 */
-	if (channel == -EAGAIN &&
-	    (todo == ISO_RES_ALLOC || todo == ISO_RES_REALLOC))
-		goto out;
+	if (todo == ISO_RES_AUTO_DEALLOC) {
+		free = true;
+		e = r->e_dealloc;
+		r->e_dealloc = NULL;
+	} else {
+		free = false;
 
-	success = channel >= 0 || bandwidth > 0;
+		// Is this generation outdated already?  As long as this resource sticks in the
+		// xarray, it will be scheduled again for a newer generation or at shutdown.
+		if (channel == -EAGAIN)
+			goto out;
 
-	scoped_guard(spinlock_irq, &client->lock) {
-		// Transit from allocation to reallocation, except if the client
-		// requested deallocation in the meantime.
-		if (r->todo == ISO_RES_ALLOC)
-			r->todo = ISO_RES_REALLOC;
-		// Allocation or reallocation failure?  Pull this resource out of the
-		// xarray and prepare for deletion, unless the client is shutting down.
-		if (r->todo == ISO_RES_REALLOC && !success &&
-		    !client->in_shutdown &&
-		    xa_erase(&client->resource_xa, index)) {
-			client_put(client);
-			free = true;
+		bool success = channel >= 0 || bandwidth > 0;
+
+		if (!success) {
+			// Allocation or reallocation failure?  Pull this resource out of the
+			// xarray and prepare for deletion, unless the client is shutting down.
+			scoped_guard(spinlock_irq,  &client->lock) {
+				if (!client->in_shutdown && xa_erase(&client->resource_xa, index)) {
+					client_put(client);
+					free = true;
+				}
+			}
 		}
-	}
 
-	if (todo == ISO_RES_ALLOC && channel >= 0)
-		r->channels = 1ULL << channel;
+		if (todo == ISO_RES_AUTO_REALLOC) {
+			if (success)
+				goto out;
 
-	if (todo == ISO_RES_REALLOC && success)
-		goto out;
+			// Notify the userspace client of the failure through a deallocation event.
+			e = r->e_dealloc;
+			r->e_dealloc = NULL;
+		} else {
+			// Transit from allocation to reallocation, except if the client requested
+			// deallocation in the meantime.
+			scoped_guard(spinlock_irq,  &client->lock)
+				r->todo = ISO_RES_AUTO_REALLOC;
 
-	if (todo == ISO_RES_ALLOC || todo == ISO_RES_ALLOC_ONCE) {
-		e = r->e_alloc;
-		r->e_alloc = NULL;
-	} else {
-		e = r->e_dealloc;
-		r->e_dealloc = NULL;
+			if (channel >= 0)
+				r->params.channels = 1ULL << channel;
+
+			e = r->e_alloc;
+			r->e_alloc = NULL;
+		}
 	}
+
 	e->iso_resource.handle    = r->resource.handle;
 	e->iso_resource.channel   = channel;
 	e->iso_resource.bandwidth = bandwidth;
@@ -1387,97 +1427,137 @@ static void iso_resource_work(struct work_struct *work)
 	client_put(client);
 }
 
-static void release_iso_resource(struct client *client,
-				 struct client_resource *resource)
+static void release_iso_resource_auto(struct client *client, struct client_resource *resource)
 {
-	struct iso_resource *r = to_iso_resource(resource);
+	struct iso_resource_auto *r = to_iso_resource_auto(resource);
 
 	guard(spinlock_irq)(&client->lock);
 
-	r->todo = ISO_RES_DEALLOC;
-	schedule_iso_resource(r, 0);
+	r->todo = ISO_RES_AUTO_DEALLOC;
+	schedule_iso_resource_auto(r, 0);
 }
 
-static int init_iso_resource(struct client *client,
-		struct fw_cdev_allocate_iso_resource *request, int todo)
+static int ioctl_allocate_iso_resource(struct client *client, union ioctl_arg *arg)
 {
-	struct iso_resource_event *e1, *e2;
-	struct iso_resource *r;
-	int ret;
+	struct fw_cdev_allocate_iso_resource *request = &arg->allocate_iso_resource;
+	struct iso_resource_event *e1 __free(kfree) = kmalloc_obj(*e1);
+	struct iso_resource_event *e2 __free(kfree) = kmalloc_obj(*e2);
+	struct iso_resource_auto *r  __free(kfree) = kmalloc_obj(*r);
+	int err;
 
-	if ((request->channels == 0 && request->bandwidth == 0) ||
-	    request->bandwidth > BANDWIDTH_AVAILABLE_INITIAL)
-		return -EINVAL;
+	if (!r || !e1 || !e2)
+		return -ENOMEM;
 
-	r = kmalloc_obj(*r);
-	e1 = kmalloc_obj(*e1);
-	e2 = kmalloc_obj(*e2);
-	if (r == NULL || e1 == NULL || e2 == NULL) {
-		ret = -ENOMEM;
-		goto fail;
-	}
+	err = fill_iso_resource_params(&r->params, request);
+	if (err < 0)
+		return  err;
 
-	INIT_DELAYED_WORK(&r->work, iso_resource_work);
+	INIT_DELAYED_WORK(&r->work, iso_resource_auto_work);
 	r->client	= client;
-	r->todo		= todo;
-	r->generation	= -1;
-	r->channels	= request->channels;
-	r->bandwidth	= request->bandwidth;
+	r->todo		= ISO_RES_AUTO_ALLOC;
 	r->e_alloc	= e1;
 	r->e_dealloc	= e2;
 
 	e1->iso_resource.closure = request->closure;
-	e1->iso_resource.type    = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED;
+	e1->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED;
 	e2->iso_resource.closure = request->closure;
-	e2->iso_resource.type    = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;
+	e2->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;
 
-	if (todo == ISO_RES_ALLOC) {
-		r->resource.release = release_iso_resource;
-		ret = add_client_resource(client, &r->resource, GFP_KERNEL);
-		if (ret < 0)
-			goto fail;
-	} else {
-		r->resource.release = NULL;
-		r->resource.handle = -1;
-		schedule_iso_resource(r, 0);
-	}
+	r->resource.release = release_iso_resource_auto;
+	err = add_client_resource(client, &r->resource, GFP_KERNEL);
+	if (err < 0)
+		return err;
 	request->handle = r->resource.handle;
 
+	retain_and_null_ptr(e1);
+	retain_and_null_ptr(e2);
+	schedule_iso_resource_auto(no_free_ptr(r), 0);
+
 	return 0;
- fail:
-	kfree(r);
-	kfree(e1);
-	kfree(e2);
+}
 
-	return ret;
+static int ioctl_deallocate_iso_resource(struct client *client,
+					 union ioctl_arg *arg)
+{
+	return release_client_resource(client,
+			arg->deallocate.handle, release_iso_resource_auto, NULL);
 }
 
-static int ioctl_allocate_iso_resource(struct client *client,
-				       union ioctl_arg *arg)
+#define UNAVAILABLE_HANDLE	-1
+
+static void iso_resource_once_work(struct work_struct *work)
 {
-	return init_iso_resource(client,
-			&arg->allocate_iso_resource, ISO_RES_ALLOC);
+	struct iso_resource_once *r = from_work(r, work, work.work);
+	struct client *client = r->client;
+	struct iso_resource_event *e = r->event;
+	int generation, channel, bandwidth;
+
+	scoped_guard(spinlock_irq, &client->lock)
+		generation = client->device->generation;
+
+	r->params.generation = generation;
+	bandwidth = r->params.bandwidth;
+
+	fw_iso_resource_manage(client->device->card, generation, r->params.channels, &channel,
+			       &bandwidth, r->todo == ISO_RES_ONCE_ALLOC);
+
+	e->iso_resource.handle = UNAVAILABLE_HANDLE;
+	e->iso_resource.channel = channel;
+	e->iso_resource.bandwidth = bandwidth;
+
+	queue_event(client, &e->event, &e->iso_resource, sizeof(e->iso_resource), NULL, 0);
+
+	cancel_delayed_work(&r->work);
+	kfree(r);
+
+	client_put(client);
 }
 
-static int ioctl_deallocate_iso_resource(struct client *client,
-					 union ioctl_arg *arg)
+static int init_iso_resource_once(struct client *client,
+				  struct fw_cdev_allocate_iso_resource *request, int todo)
 {
-	return release_client_resource(client,
-			arg->deallocate.handle, release_iso_resource, NULL);
+	struct iso_resource_event *e __free(kfree) = kmalloc_obj(*e);
+	struct iso_resource_once *r __free(kfree) = kmalloc_obj(*r);
+	int err;
+
+	if (!r || !e)
+		return -ENOMEM;
+
+	err = fill_iso_resource_params(&r->params, request);
+	if (err < 0)
+		return err;
+
+	INIT_DELAYED_WORK(&r->work, iso_resource_once_work);
+	r->client = client;
+	r->todo	= todo;
+
+	if (todo == ISO_RES_ONCE_ALLOC)
+		e->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_ALLOCATED;
+	else
+		e->iso_resource.type = FW_CDEV_EVENT_ISO_RESOURCE_DEALLOCATED;
+	e->iso_resource.closure = request->closure;
+	r->event = no_free_ptr(e);
+
+	// Keep the client until work item finishing.
+	client_get(r->client);
+
+	queue_delayed_work(fw_workqueue, &no_free_ptr(r)->work, 0);
+
+	request->handle = UNAVAILABLE_HANDLE;
+
+	return 0;
 }
 
 static int ioctl_allocate_iso_resource_once(struct client *client,
 					    union ioctl_arg *arg)
 {
-	return init_iso_resource(client,
-			&arg->allocate_iso_resource, ISO_RES_ALLOC_ONCE);
+	return init_iso_resource_once(client, &arg->allocate_iso_resource, ISO_RES_ONCE_ALLOC);
 }
 
 static int ioctl_deallocate_iso_resource_once(struct client *client,
 					      union ioctl_arg *arg)
 {
-	return init_iso_resource(client,
-			&arg->allocate_iso_resource, ISO_RES_DEALLOC_ONCE);
+	return init_iso_resource_once(client, &arg->allocate_iso_resource, ISO_RES_ONCE_DEALLOC);
 }
 
 /*
diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig
index bbd2155d84838..bcfda01c2ab81 100644
--- a/drivers/firmware/Kconfig
+++ b/drivers/firmware/Kconfig
@@ -114,7 +114,10 @@ config ISCSI_IBFT
 
 config RASPBERRYPI_FIRMWARE
 	tristate "Raspberry Pi Firmware Driver"
-	depends on BCM2835_MBOX
+	depends on ARCH_BCM2835 || COMPILE_TEST
+	depends on ARM || ARM64
+	depends on MAILBOX
+	default ARCH_BCM2835
 	help
 	  This option enables support for communicating with the firmware on the
 	  Raspberry Pi.
diff --git a/drivers/firmware/arm_ffa/bus.c b/drivers/firmware/arm_ffa/bus.c
index 9576862d89c40..4df5c3b3fe434 100644
--- a/drivers/firmware/arm_ffa/bus.c
+++ b/drivers/firmware/arm_ffa/bus.c
@@ -26,6 +26,8 @@ static int ffa_device_match(struct device *dev, const struct device_driver *drv)
 
 	id_table = to_ffa_driver(drv)->id_table;
 	ffa_dev = to_ffa_dev(dev);
+	if (!id_table)
+		return 0;
 
 	while (!uuid_is_null(&id_table->uuid)) {
 		/*
@@ -123,7 +125,7 @@ int ffa_driver_register(struct ffa_driver *driver, struct module *owner,
 {
 	int ret;
 
-	if (!driver->probe)
+	if (!driver->probe || !driver->id_table)
 		return -EINVAL;
 
 	driver->driver.bus = &ffa_bus_type;
@@ -190,7 +192,7 @@ bool ffa_device_is_valid(struct ffa_device *ffa_dev)
 
 struct ffa_device *
 ffa_device_register(const struct ffa_partition_info *part_info,
-		    const struct ffa_ops *ops)
+		    const struct ffa_ops *ops, struct device *parent)
 {
 	int id, ret;
 	struct device *dev;
@@ -210,6 +212,7 @@ ffa_device_register(const struct ffa_partition_info *part_info,
 	}
 
 	dev = &ffa_dev->dev;
+	dev->parent = parent;
 	dev->bus = &ffa_bus_type;
 	dev->release = ffa_release_device;
 	dev->dma_mask = &dev->coherent_dma_mask;
diff --git a/drivers/firmware/arm_ffa/common.h b/drivers/firmware/arm_ffa/common.h
index 9c6425a81d0d5..5cdf4bd222c63 100644
--- a/drivers/firmware/arm_ffa/common.h
+++ b/drivers/firmware/arm_ffa/common.h
@@ -18,9 +18,9 @@ bool ffa_device_is_valid(struct ffa_device *ffa_dev);
 void ffa_device_match_uuid(struct ffa_device *ffa_dev, const uuid_t *uuid);
 
 #ifdef CONFIG_ARM_FFA_SMCCC
-int __init ffa_transport_init(ffa_fn **invoke_ffa_fn);
+int ffa_transport_init(ffa_fn **invoke_ffa_fn);
 #else
-static inline int __init ffa_transport_init(ffa_fn **invoke_ffa_fn)
+static inline int ffa_transport_init(ffa_fn **invoke_ffa_fn)
 {
 	return -EOPNOTSUPP;
 }
diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
index eb27828482837..54984e1b97413 100644
--- a/drivers/firmware/arm_ffa/driver.c
+++ b/drivers/firmware/arm_ffa/driver.c
@@ -36,16 +36,20 @@
 #include <linux/mm.h>
 #include <linux/mutex.h>
 #include <linux/of_irq.h>
+#include <linux/platform_device.h>
 #include <linux/scatterlist.h>
 #include <linux/slab.h>
 #include <linux/smp.h>
 #include <linux/uuid.h>
 #include <linux/xarray.h>
 
+#include <asm/virt.h>
+
 #include "common.h"
 
 #define FFA_DRIVER_VERSION	FFA_VERSION_1_2
 #define FFA_MIN_VERSION		FFA_VERSION_1_0
+#define FFA_PLATFORM_NAME	"arm-ffa"
 
 #define SENDER_ID_MASK		GENMASK(31, 16)
 #define RECEIVER_ID_MASK	GENMASK(15, 0)
@@ -87,6 +91,7 @@ static inline int ffa_to_linux_errno(int errno)
 
 struct ffa_pcpu_irq {
 	struct ffa_drv_info *info;
+	struct work_struct notif_pcpu_work;
 };
 
 struct ffa_drv_info {
@@ -100,13 +105,13 @@ struct ffa_drv_info {
 	bool mem_ops_native;
 	bool msg_direct_req2_supp;
 	bool bitmap_created;
+	bool bus_notifier_registered;
 	bool notif_enabled;
 	unsigned int sched_recv_irq;
 	unsigned int notif_pend_irq;
 	unsigned int cpuhp_state;
 	struct ffa_pcpu_irq __percpu *irq_pcpu;
 	struct workqueue_struct *notif_pcpu_wq;
-	struct work_struct notif_pcpu_work;
 	struct work_struct sched_recv_irq_work;
 	struct xarray partition_info;
 	DECLARE_HASHTABLE(notifier_hash, ilog2(FFA_MAX_NOTIFICATIONS));
@@ -114,6 +119,7 @@ struct ffa_drv_info {
 };
 
 static struct ffa_drv_info *drv_info;
+static struct platform_device *ffa_pdev;
 
 /*
  * The driver must be able to support all the versions from the earliest
@@ -322,6 +328,10 @@ __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
 #define PART_INFO_ID_MASK	GENMASK(15, 0)
 #define PART_INFO_EXEC_CXT_MASK	GENMASK(31, 16)
 #define PART_INFO_PROPS_MASK	GENMASK(63, 32)
+#define FFA_PART_INFO_GET_REGS_FIRST_REG	3
+#define FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC	3
+#define FFA_PART_INFO_GET_REGS_NUM_REGS \
+	(sizeof(ffa_value_t) / sizeof_field(ffa_value_t, a0))
 #define PART_INFO_ID(x)		((u16)(FIELD_GET(PART_INFO_ID_MASK, (x))))
 #define PART_INFO_EXEC_CXT(x)	((u16)(FIELD_GET(PART_INFO_EXEC_CXT_MASK, (x))))
 #define PART_INFO_PROPERTIES(x)	((u32)(FIELD_GET(PART_INFO_PROPS_MASK, (x))))
@@ -329,15 +339,13 @@ static int
 __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
 			      struct ffa_partition_info *buffer, int num_parts)
 {
-	u16 buf_sz, start_idx, cur_idx, count = 0, prev_idx = 0, tag = 0;
+	u16 buf_sz, start_idx = 0, cur_idx, count = 0, tag = 0;
 	struct ffa_partition_info *buf = buffer;
 	ffa_value_t partition_info;
 
 	do {
 		__le64 *regs;
-		int idx;
-
-		start_idx = prev_idx ? prev_idx + 1 : 0;
+		int idx, nr_desc, buf_idx, regs_per_desc, max_desc;
 
 		invoke_ffa_fn((ffa_value_t){
 			      .a0 = FFA_PARTITION_INFO_GET_REGS,
@@ -353,15 +361,35 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
 			count = PARTITION_COUNT(partition_info.a2);
 		if (!buffer || !num_parts) /* count only */
 			return count;
+		if (count > num_parts)
+			return -EINVAL;
 
 		cur_idx = CURRENT_INDEX(partition_info.a2);
-		tag = UUID_INFO_TAG(partition_info.a2);
+		if (cur_idx < start_idx || cur_idx >= count)
+			return -EINVAL;
+
 		buf_sz = PARTITION_INFO_SZ(partition_info.a2);
-		if (buf_sz > sizeof(*buffer))
-			buf_sz = sizeof(*buffer);
+		if (buf_sz % sizeof(*regs))
+			return -EINVAL;
+
+		regs_per_desc = buf_sz / sizeof(*regs);
+		if (regs_per_desc < FFA_PART_INFO_GET_REGS_MIN_REGS_PER_DESC)
+			return -EINVAL;
+
+		nr_desc = cur_idx - start_idx + 1;
+		max_desc = (FFA_PART_INFO_GET_REGS_NUM_REGS -
+			    FFA_PART_INFO_GET_REGS_FIRST_REG) / regs_per_desc;
+		if (nr_desc > max_desc)
+			return -EINVAL;
+
+		buf_idx = buf - buffer;
+		if (buf_idx + nr_desc > num_parts)
+			return -EINVAL;
+
+		tag = UUID_INFO_TAG(partition_info.a2);
 
 		regs = (void *)&partition_info.a3;
-		for (idx = 0; idx < cur_idx - start_idx + 1; idx++, buf++) {
+		for (idx = 0; idx < nr_desc; idx++, buf++) {
 			union {
 				uuid_t uuid;
 				u64 regs[2];
@@ -377,9 +405,9 @@ __ffa_partition_info_get_regs(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3,
 			buf->exec_ctxt = PART_INFO_EXEC_CXT(val);
 			buf->properties = PART_INFO_PROPERTIES(val);
 			uuid_copy(&buf->uuid, &uuid_regs.uuid);
-			regs += 3;
+			regs += regs_per_desc;
 		}
-		prev_idx = cur_idx;
+		start_idx = cur_idx + 1;
 
 	} while (cur_idx < (count - 1));
 
@@ -1189,7 +1217,7 @@ static int
 ffa_sched_recv_cb_update(struct ffa_device *dev, ffa_sched_recv_cb callback,
 			 void *cb_data, bool is_registration)
 {
-	struct ffa_dev_part_info *partition = NULL, *tmp;
+	struct ffa_dev_part_info *partition = NULL;
 	struct list_head *phead;
 	bool cb_valid;
 
@@ -1202,11 +1230,11 @@ ffa_sched_recv_cb_update(struct ffa_device *dev, ffa_sched_recv_cb callback,
 		return -EINVAL;
 	}
 
-	list_for_each_entry_safe(partition, tmp, phead, node)
+	list_for_each_entry(partition, phead, node)
 		if (partition->dev == dev)
 			break;
 
-	if (!partition) {
+	if (&partition->node == phead) {
 		pr_err("%s: No such partition ID 0x%x\n", __func__, dev->vm_id);
 		return -EINVAL;
 	}
@@ -1445,20 +1473,25 @@ static int ffa_notify_send(struct ffa_device *dev, int notify_id,
 
 static void handle_notif_callbacks(u64 bitmap, enum notify_type type)
 {
+	ffa_notifier_cb cb;
+	void *cb_data;
 	int notify_id;
-	struct notifier_cb_info *cb_info = NULL;
 
 	for (notify_id = 0; notify_id <= FFA_MAX_NOTIFICATIONS && bitmap;
 	     notify_id++, bitmap >>= 1) {
 		if (!(bitmap & 1))
 			continue;
 
-		read_lock(&drv_info->notify_lock);
-		cb_info = notifier_hnode_get_by_type(notify_id, type);
-		read_unlock(&drv_info->notify_lock);
+		scoped_guard(read_lock, &drv_info->notify_lock) {
+			struct notifier_cb_info *cb_info;
 
-		if (cb_info && cb_info->cb)
-			cb_info->cb(notify_id, cb_info->cb_data);
+			cb_info = notifier_hnode_get_by_type(notify_id, type);
+			cb = cb_info ? cb_info->cb : NULL;
+			cb_data = cb_info ? cb_info->cb_data : NULL;
+		}
+
+		if (cb)
+			cb(notify_id, cb_data);
 	}
 }
 
@@ -1466,39 +1499,56 @@ static void handle_fwk_notif_callbacks(u32 bitmap)
 {
 	void *buf;
 	uuid_t uuid;
+	void *fwk_cb_data;
 	int notify_id = 0, target;
+	ffa_fwk_notifier_cb fwk_cb;
 	struct ffa_indirect_msg_hdr *msg;
-	struct notifier_cb_info *cb_info = NULL;
+	size_t min_offset = offsetof(struct ffa_indirect_msg_hdr, uuid);
 
 	/* Only one framework notification defined and supported for now */
 	if (!(bitmap & FRAMEWORK_NOTIFY_RX_BUFFER_FULL))
 		return;
 
-	mutex_lock(&drv_info->rx_lock);
+	scoped_guard(mutex, &drv_info->rx_lock) {
+		u32 offset, size;
 
-	msg = drv_info->rx_buffer;
-	buf = kmemdup((void *)msg + msg->offset, msg->size, GFP_KERNEL);
-	if (!buf) {
-		mutex_unlock(&drv_info->rx_lock);
-		return;
-	}
+		msg = drv_info->rx_buffer;
+		offset = msg->offset;
+		size = msg->size;
 
-	target = SENDER_ID(msg->send_recv_id);
-	if (msg->offset >= sizeof(*msg))
-		uuid_copy(&uuid, &msg->uuid);
-	else
-		uuid_copy(&uuid, &uuid_null);
+		if (!size || (offset != min_offset && offset < sizeof(*msg)) ||
+		    offset > drv_info->rxtx_bufsz ||
+		    size > drv_info->rxtx_bufsz - offset) {
+			pr_err("invalid framework notification message\n");
+			ffa_rx_release();
+			return;
+		}
 
-	mutex_unlock(&drv_info->rx_lock);
+		buf = kmemdup((void *)msg + offset, size, GFP_KERNEL);
+		if (!buf) {
+			ffa_rx_release();
+			return;
+		}
 
-	ffa_rx_release();
+		target = SENDER_ID(msg->send_recv_id);
+		if (offset >= sizeof(*msg))
+			uuid_copy(&uuid, &msg->uuid);
+		else
+			uuid_copy(&uuid, &uuid_null);
+		ffa_rx_release();
+	}
 
-	read_lock(&drv_info->notify_lock);
-	cb_info = notifier_hnode_get_by_vmid_uuid(notify_id, target, &uuid);
-	read_unlock(&drv_info->notify_lock);
+	scoped_guard(read_lock, &drv_info->notify_lock) {
+		struct notifier_cb_info *cb_info;
 
-	if (cb_info && cb_info->fwk_cb)
-		cb_info->fwk_cb(notify_id, cb_info->cb_data, buf);
+		cb_info = notifier_hnode_get_by_vmid_uuid(notify_id, target,
+							  &uuid);
+		fwk_cb = cb_info ? cb_info->fwk_cb : NULL;
+		fwk_cb_data = cb_info ? cb_info->cb_data : NULL;
+	}
+
+	if (fwk_cb)
+		fwk_cb(notify_id, fwk_cb_data, buf);
 	kfree(buf);
 }
 
@@ -1539,10 +1589,11 @@ ffa_self_notif_handle(u16 vcpu, bool is_per_vcpu, void *cb_data)
 
 static void notif_pcpu_irq_work_fn(struct work_struct *work)
 {
-	struct ffa_drv_info *info = container_of(work, struct ffa_drv_info,
+	struct ffa_pcpu_irq *pcpu = container_of(work, struct ffa_pcpu_irq,
 						 notif_pcpu_work);
+	struct ffa_drv_info *info = pcpu->info;
 
-	ffa_self_notif_handle(smp_processor_id(), true, info);
+	notif_get_and_handle(info);
 }
 
 static const struct ffa_info_ops ffa_drv_info_ops = {
@@ -1629,6 +1680,15 @@ static struct notifier_block ffa_bus_nb = {
 	.notifier_call = ffa_bus_notifier,
 };
 
+static void ffa_bus_notifier_unregister(void)
+{
+	if (!drv_info->bus_notifier_registered)
+		return;
+
+	bus_unregister_notifier(&ffa_bus_type, &ffa_bus_nb);
+	drv_info->bus_notifier_registered = false;
+}
+
 static int ffa_xa_add_partition_info(struct ffa_device *dev)
 {
 	struct ffa_dev_part_info *info;
@@ -1685,7 +1745,7 @@ static int ffa_setup_host_partition(int vm_id)
 	int ret;
 
 	buf.id = vm_id;
-	ffa_dev = ffa_device_register(&buf, &ffa_drv_ops);
+	ffa_dev = ffa_device_register(&buf, &ffa_drv_ops, &ffa_pdev->dev);
 	if (!ffa_dev) {
 		pr_err("%s: failed to register host partition ID 0x%x\n",
 		       __func__, vm_id);
@@ -1712,6 +1772,8 @@ static void ffa_partitions_cleanup(void)
 	struct list_head *phead;
 	unsigned long idx;
 
+	ffa_bus_notifier_unregister();
+
 	/* Clean up/free all registered devices */
 	ffa_devices_unregister();
 
@@ -1739,11 +1801,14 @@ static int ffa_setup_partitions(void)
 		ret = bus_register_notifier(&ffa_bus_type, &ffa_bus_nb);
 		if (ret)
 			pr_err("Failed to register FF-A bus notifiers\n");
+		else
+			drv_info->bus_notifier_registered = true;
 	}
 
 	count = ffa_partition_probe(&uuid_null, &pbuf);
 	if (count <= 0) {
 		pr_info("%s: No partitions found, error %d\n", __func__, count);
+		ffa_bus_notifier_unregister();
 		return -EINVAL;
 	}
 
@@ -1755,7 +1820,8 @@ static int ffa_setup_partitions(void)
 		 * provides UUID here for each partition as part of the
 		 * discovery API and the same is passed.
 		 */
-		ffa_dev = ffa_device_register(tpbuf, &ffa_drv_ops);
+		ffa_dev = ffa_device_register(tpbuf, &ffa_drv_ops,
+					      &ffa_pdev->dev);
 		if (!ffa_dev) {
 			pr_err("%s: failed to register partition ID 0x%x\n",
 			       __func__, tpbuf->id);
@@ -1811,7 +1877,7 @@ static irqreturn_t notif_pend_irq_handler(int irq, void *irq_data)
 	struct ffa_drv_info *info = pcpu->info;
 
 	queue_work_on(smp_processor_id(), info->notif_pcpu_wq,
-		      &info->notif_pcpu_work);
+		      &pcpu->notif_pcpu_work);
 
 	return IRQ_HANDLED;
 }
@@ -1928,8 +1994,11 @@ static int ffa_init_pcpu_irq(void)
 	if (!irq_pcpu)
 		return -ENOMEM;
 
-	for_each_present_cpu(cpu)
+	for_each_present_cpu(cpu) {
 		per_cpu_ptr(irq_pcpu, cpu)->info = drv_info;
+		INIT_WORK(&per_cpu_ptr(irq_pcpu, cpu)->notif_pcpu_work,
+			  notif_pcpu_irq_work_fn);
+	}
 
 	drv_info->irq_pcpu = irq_pcpu;
 
@@ -1958,7 +2027,6 @@ static int ffa_init_pcpu_irq(void)
 	}
 
 	INIT_WORK(&drv_info->sched_recv_irq_work, ffa_sched_recv_irq_work_fn);
-	INIT_WORK(&drv_info->notif_pcpu_work, notif_pcpu_irq_work_fn);
 	drv_info->notif_pcpu_wq = create_workqueue("ffa_pcpu_irq_notification");
 	if (!drv_info->notif_pcpu_wq)
 		return -EINVAL;
@@ -2029,12 +2097,16 @@ static void ffa_notifications_setup(void)
 	ffa_notifications_cleanup();
 }
 
-static int __init ffa_init(void)
+static int ffa_probe(struct platform_device *pdev)
 {
 	int ret;
 	u32 buf_sz;
 	size_t rxtx_bufsz = SZ_4K;
 
+	if (IS_BUILTIN(CONFIG_ARM_FFA_TRANSPORT) &&
+	    is_protected_kvm_enabled() && !is_pkvm_initialized())
+		return -EPROBE_DEFER;
+
 	ret = ffa_transport_init(&invoke_ffa_fn);
 	if (ret)
 		return ret;
@@ -2042,6 +2114,7 @@ static int __init ffa_init(void)
 	drv_info = kzalloc_obj(*drv_info);
 	if (!drv_info)
 		return -ENOMEM;
+	platform_set_drvdata(pdev, drv_info);
 
 	ret = ffa_version_check(&drv_info->version);
 	if (ret)
@@ -2063,11 +2136,12 @@ static int __init ffa_init(void)
 			rxtx_bufsz = SZ_4K;
 	}
 
+	rxtx_bufsz = PAGE_ALIGN(rxtx_bufsz);
 	drv_info->rxtx_bufsz = rxtx_bufsz;
 	drv_info->rx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL);
 	if (!drv_info->rx_buffer) {
 		ret = -ENOMEM;
-		goto free_pages;
+		goto free_drv_info;
 	}
 
 	drv_info->tx_buffer = alloc_pages_exact(rxtx_bufsz, GFP_KERNEL);
@@ -2078,7 +2152,7 @@ static int __init ffa_init(void)
 
 	ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer),
 			   virt_to_phys(drv_info->rx_buffer),
-			   PAGE_ALIGN(rxtx_bufsz) / FFA_PAGE_SIZE);
+			   rxtx_bufsz / FFA_PAGE_SIZE);
 	if (ret) {
 		pr_err("failed to register FFA RxTx buffers\n");
 		goto free_pages;
@@ -2103,19 +2177,56 @@ static int __init ffa_init(void)
 		free_pages_exact(drv_info->tx_buffer, rxtx_bufsz);
 	free_pages_exact(drv_info->rx_buffer, rxtx_bufsz);
 free_drv_info:
+	platform_set_drvdata(pdev, NULL);
 	kfree(drv_info);
+	drv_info = NULL;
 	return ret;
 }
-rootfs_initcall(ffa_init);
 
-static void __exit ffa_exit(void)
+static void ffa_remove(struct platform_device *pdev)
 {
+	struct ffa_drv_info *info = platform_get_drvdata(pdev);
+
 	ffa_notifications_cleanup();
 	ffa_partitions_cleanup();
 	ffa_rxtx_unmap();
-	free_pages_exact(drv_info->tx_buffer, drv_info->rxtx_bufsz);
-	free_pages_exact(drv_info->rx_buffer, drv_info->rxtx_bufsz);
-	kfree(drv_info);
+	free_pages_exact(info->tx_buffer, info->rxtx_bufsz);
+	free_pages_exact(info->rx_buffer, info->rxtx_bufsz);
+	kfree(info);
+	platform_set_drvdata(pdev, NULL);
+	drv_info = NULL;
+}
+
+static struct platform_driver ffa_driver = {
+	.probe = ffa_probe,
+	.remove = ffa_remove,
+	.driver = {
+		.name = FFA_PLATFORM_NAME,
+	},
+};
+
+static int __init ffa_init(void)
+{
+	int ret;
+
+	ffa_pdev = platform_device_register_simple(FFA_PLATFORM_NAME,
+						   PLATFORM_DEVID_NONE,
+						   NULL, 0);
+	if (IS_ERR(ffa_pdev))
+		return PTR_ERR(ffa_pdev);
+
+	ret = platform_driver_register(&ffa_driver);
+	if (ret)
+		platform_device_unregister(ffa_pdev);
+
+	return ret;
+}
+module_init(ffa_init);
+
+static void __exit ffa_exit(void)
+{
+	platform_device_unregister(ffa_pdev);
+	platform_driver_unregister(&ffa_driver);
 }
 module_exit(ffa_exit);
 
diff --git a/drivers/firmware/arm_ffa/smccc.c b/drivers/firmware/arm_ffa/smccc.c
index 4d85bfff0a4e6..e6125dd9f58f8 100644
--- a/drivers/firmware/arm_ffa/smccc.c
+++ b/drivers/firmware/arm_ffa/smccc.c
@@ -17,7 +17,7 @@ static void __arm_ffa_fn_hvc(ffa_value_t args, ffa_value_t *res)
 	arm_smccc_1_2_hvc(&args, res);
 }
 
-int __init ffa_transport_init(ffa_fn **invoke_ffa_fn)
+int ffa_transport_init(ffa_fn **invoke_ffa_fn)
 {
 	enum arm_smccc_conduit conduit;
 
diff --git a/drivers/firmware/arm_scmi/base.c b/drivers/firmware/arm_scmi/base.c
index cd1331c2fc403..a9cea09355ff4 100644
--- a/drivers/firmware/arm_scmi/base.c
+++ b/drivers/firmware/arm_scmi/base.c
@@ -69,7 +69,7 @@ static int scmi_base_attributes_get(const struct scmi_protocol_handle *ph)
 	int ret;
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_base_attributes *attr_info;
-	struct scmi_revision_info *rev = ph->get_priv(ph);
+	struct scmi_base_info *rev = ph->get_priv(ph);
 
 	ret = ph->xops->xfer_get_init(ph, PROTOCOL_ATTRIBUTES,
 				      0, sizeof(*attr_info), &t);
@@ -103,7 +103,7 @@ scmi_base_vendor_id_get(const struct scmi_protocol_handle *ph, bool sub_vendor)
 	int ret, size;
 	char *vendor_id;
 	struct scmi_xfer *t;
-	struct scmi_revision_info *rev = ph->get_priv(ph);
+	struct scmi_base_info *rev = ph->get_priv(ph);
 
 	if (sub_vendor) {
 		cmd = BASE_DISCOVER_SUB_VENDOR;
@@ -143,7 +143,7 @@ scmi_base_implementation_version_get(const struct scmi_protocol_handle *ph)
 	int ret;
 	__le32 *impl_ver;
 	struct scmi_xfer *t;
-	struct scmi_revision_info *rev = ph->get_priv(ph);
+	struct scmi_base_info *rev = ph->get_priv(ph);
 
 	ret = ph->xops->xfer_get_init(ph, BASE_DISCOVER_IMPLEMENT_VERSION,
 				      0, sizeof(*impl_ver), &t);
@@ -180,7 +180,7 @@ scmi_base_implementation_list_get(const struct scmi_protocol_handle *ph,
 	__le32 *num_skip, *num_ret;
 	u32 tot_num_ret = 0, loop_num_ret;
 	struct device *dev = ph->dev;
-	struct scmi_revision_info *rev = ph->get_priv(ph);
+	struct scmi_base_info *rev = ph->get_priv(ph);
 
 	ret = ph->xops->xfer_get_init(ph, BASE_DISCOVER_LIST_PROTOCOLS,
 				      sizeof(*num_skip), 0, &t);
@@ -325,6 +325,8 @@ static void *scmi_base_fill_custom_report(const struct scmi_protocol_handle *ph,
 					  void *report, u32 *src_id)
 {
 	int i;
+	u32 error_status;
+	size_t expected_sz;
 	const struct scmi_base_error_notify_payld *p = payld;
 	struct scmi_base_error_report *r = report;
 
@@ -338,10 +340,19 @@ static void *scmi_base_fill_custom_report(const struct scmi_protocol_handle *ph,
 	if (evt_id != SCMI_EVENT_BASE_ERROR_EVENT || sizeof(*p) < payld_sz)
 		return NULL;
 
+	expected_sz = offsetof(typeof(*p), msg_reports);
+	if (payld_sz < expected_sz)
+		return NULL;
+
 	r->timestamp = timestamp;
 	r->agent_id = le32_to_cpu(p->agent_id);
-	r->fatal = IS_FATAL_ERROR(le32_to_cpu(p->error_status));
-	r->cmd_count = ERROR_CMD_COUNT(le32_to_cpu(p->error_status));
+	error_status = le32_to_cpu(p->error_status);
+	r->fatal = IS_FATAL_ERROR(error_status);
+	r->cmd_count = ERROR_CMD_COUNT(error_status);
+	expected_sz += r->cmd_count * sizeof(p->msg_reports[0]);
+	if (payld_sz < expected_sz)
+		return NULL;
+
 	for (i = 0; i < r->cmd_count; i++)
 		r->reports[i] = le64_to_cpu(p->msg_reports[i]);
 	*src_id = 0;
@@ -377,7 +388,7 @@ static int scmi_base_protocol_init(const struct scmi_protocol_handle *ph)
 	u8 *prot_imp;
 	char name[SCMI_SHORT_NAME_MAX_SIZE];
 	struct device *dev = ph->dev;
-	struct scmi_revision_info *rev = scmi_revision_area_get(ph);
+	struct scmi_base_info *rev = scmi_revision_area_get(ph);
 
 	rev->major_ver = PROTOCOL_REV_MAJOR(ph->version);
 	rev->minor_ver = PROTOCOL_REV_MINOR(ph->version);
diff --git a/drivers/firmware/arm_scmi/clock.c b/drivers/firmware/arm_scmi/clock.c
index ab36871650a1f..42e666a628c73 100644
--- a/drivers/firmware/arm_scmi/clock.c
+++ b/drivers/firmware/arm_scmi/clock.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2018-2022 ARM Ltd.
  */
 
+#include <linux/math64.h>
 #include <linux/module.h>
 #include <linux/limits.h>
 #include <linux/sort.h>
@@ -156,13 +157,26 @@ struct scmi_clock_rate_notify_payld {
 	__le32 rate_high;
 };
 
+struct scmi_clock_desc {
+	u32 id;
+	unsigned int tot_rates;
+	struct scmi_clock_rates r;
+#define	RATE_MIN	0
+#define	RATE_MAX	1
+#define	RATE_STEP	2
+	struct scmi_clock_info info;
+};
+
+#define to_desc(p)	(container_of(p, struct scmi_clock_desc, info))
+
 struct clock_info {
 	int num_clocks;
 	int max_async_req;
 	bool notify_rate_changed_cmd;
 	bool notify_rate_change_requested_cmd;
 	atomic_t cur_async_req;
-	struct scmi_clock_info *clk;
+	struct scmi_clock_desc *clkds;
+#define CLOCK_INFO(c, i)	(&(((c)->clkds + (i))->info))
 	int (*clock_config_set)(const struct scmi_protocol_handle *ph,
 				u32 clk_id, enum clk_state state,
 				enum scmi_clock_oem_config oem_type,
@@ -184,7 +198,7 @@ scmi_clock_domain_lookup(struct clock_info *ci, u32 clk_id)
 	if (clk_id >= ci->num_clocks)
 		return ERR_PTR(-EINVAL);
 
-	return ci->clk + clk_id;
+	return CLOCK_INFO(ci, clk_id);
 }
 
 static int
@@ -225,8 +239,7 @@ scmi_clock_protocol_attributes_get(const struct scmi_protocol_handle *ph,
 
 struct scmi_clk_ipriv {
 	struct device *dev;
-	u32 clk_id;
-	struct scmi_clock_info *clk;
+	struct scmi_clock_desc *clkd;
 };
 
 static void iter_clk_possible_parents_prepare_message(void *message, unsigned int desc_index,
@@ -235,7 +248,7 @@ static void iter_clk_possible_parents_prepare_message(void *message, unsigned in
 	struct scmi_msg_clock_possible_parents *msg = message;
 	const struct scmi_clk_ipriv *p = priv;
 
-	msg->id = cpu_to_le32(p->clk_id);
+	msg->id = cpu_to_le32(p->clkd->id);
 	/* Set the number of OPPs to be skipped/already read */
 	msg->skip_parents = cpu_to_le32(desc_index);
 }
@@ -245,7 +258,6 @@ static int iter_clk_possible_parents_update_state(struct scmi_iterator_state *st
 {
 	const struct scmi_msg_resp_clock_possible_parents *r = response;
 	struct scmi_clk_ipriv *p = priv;
-	struct device *dev = ((struct scmi_clk_ipriv *)p)->dev;
 	u32 flags;
 
 	flags = le32_to_cpu(r->num_parent_flags);
@@ -257,14 +269,15 @@ static int iter_clk_possible_parents_update_state(struct scmi_iterator_state *st
 	 * assume it's returned+remaining on first call.
 	 */
 	if (!st->max_resources) {
-		p->clk->num_parents = st->num_returned + st->num_remaining;
-		p->clk->parents = devm_kcalloc(dev, p->clk->num_parents,
-					       sizeof(*p->clk->parents),
-					       GFP_KERNEL);
-		if (!p->clk->parents) {
-			p->clk->num_parents = 0;
+		int num_parents = st->num_returned + st->num_remaining;
+
+		p->clkd->info.parents = devm_kcalloc(p->dev, num_parents,
+						     sizeof(*p->clkd->info.parents),
+						     GFP_KERNEL);
+		if (!p->clkd->info.parents)
 			return -ENOMEM;
-		}
+
+		/* max_resources is used by the iterators to control bounds */
 		st->max_resources = st->num_returned + st->num_remaining;
 	}
 
@@ -279,29 +292,29 @@ static int iter_clk_possible_parents_process_response(const struct scmi_protocol
 	const struct scmi_msg_resp_clock_possible_parents *r = response;
 	struct scmi_clk_ipriv *p = priv;
 
-	u32 *parent = &p->clk->parents[st->desc_index + st->loop_idx];
+	p->clkd->info.parents[st->desc_index + st->loop_idx] =
+		le32_to_cpu(r->possible_parents[st->loop_idx]);
 
-	*parent = le32_to_cpu(r->possible_parents[st->loop_idx]);
+	/* Count only effectively discovered parents */
+	p->clkd->info.num_parents++;
 
 	return 0;
 }
 
-static int scmi_clock_possible_parents(const struct scmi_protocol_handle *ph, u32 clk_id,
-				       struct scmi_clock_info *clk)
+static int scmi_clock_possible_parents(const struct scmi_protocol_handle *ph,
+				       u32 clk_id, struct clock_info *cinfo)
 {
 	struct scmi_iterator_ops ops = {
 		.prepare_message = iter_clk_possible_parents_prepare_message,
 		.update_state = iter_clk_possible_parents_update_state,
 		.process_response = iter_clk_possible_parents_process_response,
 	};
-
+	struct scmi_clock_desc *clkd = &cinfo->clkds[clk_id];
 	struct scmi_clk_ipriv ppriv = {
-		.clk_id = clk_id,
-		.clk = clk,
+		.clkd = clkd,
 		.dev = ph->dev,
 	};
 	void *iter;
-	int ret;
 
 	iter = ph->hops->iter_response_init(ph, &ops, 0,
 					    CLOCK_POSSIBLE_PARENTS_GET,
@@ -310,9 +323,7 @@ static int scmi_clock_possible_parents(const struct scmi_protocol_handle *ph, u3
 	if (IS_ERR(iter))
 		return PTR_ERR(iter);
 
-	ret = ph->hops->iter_response_run(iter);
-
-	return ret;
+	return ph->hops->iter_response_run(iter);
 }
 
 static int
@@ -351,7 +362,7 @@ static int scmi_clock_attributes_get(const struct scmi_protocol_handle *ph,
 	u32 attributes;
 	struct scmi_xfer *t;
 	struct scmi_msg_resp_clock_attributes *attr;
-	struct scmi_clock_info *clk = cinfo->clk + clk_id;
+	struct scmi_clock_info *clk = CLOCK_INFO(cinfo, clk_id);
 
 	ret = ph->xops->xfer_get_init(ph, CLOCK_ATTRIBUTES,
 				      sizeof(clk_id), sizeof(*attr), &t);
@@ -393,7 +404,7 @@ static int scmi_clock_attributes_get(const struct scmi_protocol_handle *ph,
 			clk->rate_change_requested_notifications = true;
 		if (PROTOCOL_REV_MAJOR(ph->version) >= 0x3) {
 			if (SUPPORTS_PARENT_CLOCK(attributes))
-				scmi_clock_possible_parents(ph, clk_id, clk);
+				scmi_clock_possible_parents(ph, clk_id, cinfo);
 			if (SUPPORTS_GET_PERMISSIONS(attributes))
 				scmi_clock_get_permissions(ph, clk_id, clk);
 			if (SUPPORTS_EXTENDED_CONFIG(attributes))
@@ -423,7 +434,7 @@ static void iter_clk_describe_prepare_message(void *message,
 	struct scmi_msg_clock_describe_rates *msg = message;
 	const struct scmi_clk_ipriv *p = priv;
 
-	msg->id = cpu_to_le32(p->clk_id);
+	msg->id = cpu_to_le32(p->clkd->id);
 	/* Set the number of rates to be skipped/already read */
 	msg->rate_index = cpu_to_le32(desc_index);
 }
@@ -456,20 +467,33 @@ iter_clk_describe_update_state(struct scmi_iterator_state *st,
 	flags = le32_to_cpu(r->num_rates_flags);
 	st->num_remaining = NUM_REMAINING(flags);
 	st->num_returned = NUM_RETURNED(flags);
-	p->clk->rate_discrete = RATE_DISCRETE(flags);
+	p->clkd->r.rate_discrete = RATE_DISCRETE(flags);
 
 	/* Warn about out of spec replies ... */
-	if (!p->clk->rate_discrete &&
+	if (!p->clkd->r.rate_discrete &&
 	    (st->num_returned != 3 || st->num_remaining != 0)) {
 		dev_warn(p->dev,
 			 "Out-of-spec CLOCK_DESCRIBE_RATES reply for %s - returned:%d remaining:%d rx_len:%zd\n",
-			 p->clk->name, st->num_returned, st->num_remaining,
+			 p->clkd->info.name, st->num_returned, st->num_remaining,
 			 st->rx_len);
 
 		SCMI_QUIRK(clock_rates_triplet_out_of_spec,
 			   QUIRK_OUT_OF_SPEC_TRIPLET);
 	}
 
+	if (!st->max_resources) {
+		unsigned int tot_rates = st->num_returned + st->num_remaining;
+
+		p->clkd->r.rates = devm_kcalloc(p->dev, tot_rates,
+						sizeof(*p->clkd->r.rates), GFP_KERNEL);
+		if (!p->clkd->r.rates)
+			return -ENOMEM;
+
+		/* max_resources is used by the iterators to control bounds */
+		p->clkd->tot_rates = tot_rates;
+		st->max_resources = tot_rates;
+	}
+
 	return 0;
 }
 
@@ -478,38 +502,20 @@ iter_clk_describe_process_response(const struct scmi_protocol_handle *ph,
 				   const void *response,
 				   struct scmi_iterator_state *st, void *priv)
 {
-	int ret = 0;
 	struct scmi_clk_ipriv *p = priv;
 	const struct scmi_msg_resp_clock_describe_rates *r = response;
 
-	if (!p->clk->rate_discrete) {
-		switch (st->desc_index + st->loop_idx) {
-		case 0:
-			p->clk->range.min_rate = RATE_TO_U64(r->rate[0]);
-			break;
-		case 1:
-			p->clk->range.max_rate = RATE_TO_U64(r->rate[1]);
-			break;
-		case 2:
-			p->clk->range.step_size = RATE_TO_U64(r->rate[2]);
-			break;
-		default:
-			ret = -EINVAL;
-			break;
-		}
-	} else {
-		u64 *rate = &p->clk->list.rates[st->desc_index + st->loop_idx];
+	p->clkd->r.rates[p->clkd->r.num_rates] = RATE_TO_U64(r->rate[st->loop_idx]);
 
-		*rate = RATE_TO_U64(r->rate[st->loop_idx]);
-		p->clk->list.num_rates++;
-	}
+	/* Count only effectively discovered rates */
+	p->clkd->r.num_rates++;
 
-	return ret;
+	return 0;
 }
 
 static int
-scmi_clock_describe_rates_get(const struct scmi_protocol_handle *ph, u32 clk_id,
-			      struct scmi_clock_info *clk)
+scmi_clock_describe_rates_get_full(const struct scmi_protocol_handle *ph,
+				   struct scmi_clock_desc *clkd)
 {
 	int ret;
 	void *iter;
@@ -519,12 +525,16 @@ scmi_clock_describe_rates_get(const struct scmi_protocol_handle *ph, u32 clk_id,
 		.process_response = iter_clk_describe_process_response,
 	};
 	struct scmi_clk_ipriv cpriv = {
-		.clk_id = clk_id,
-		.clk = clk,
+		.clkd = clkd,
 		.dev = ph->dev,
 	};
 
-	iter = ph->hops->iter_response_init(ph, &ops, SCMI_MAX_NUM_RATES,
+	/*
+	 * Using tot_rates as max_resources parameter here so as to trigger
+	 * the dynamic allocation only when strictly needed: when trying a
+	 * full enumeration after a lazy one tot_rates will be non-zero.
+	 */
+	iter = ph->hops->iter_response_init(ph, &ops, clkd->tot_rates,
 					    CLOCK_DESCRIBE_RATES,
 					    sizeof(struct scmi_msg_clock_describe_rates),
 					    &cpriv);
@@ -535,18 +545,99 @@ scmi_clock_describe_rates_get(const struct scmi_protocol_handle *ph, u32 clk_id,
 	if (ret)
 		return ret;
 
-	if (!clk->rate_discrete) {
-		dev_dbg(ph->dev, "Min %llu Max %llu Step %llu Hz\n",
-			clk->range.min_rate, clk->range.max_rate,
-			clk->range.step_size);
-	} else if (clk->list.num_rates) {
-		sort(clk->list.rates, clk->list.num_rates,
-		     sizeof(clk->list.rates[0]), rate_cmp_func, NULL);
+	/* empty set ? */
+	if (!clkd->r.num_rates)
+		return 0;
+
+	if (clkd->r.rate_discrete && PROTOCOL_REV_MAJOR(ph->version) == 0x1)
+		sort(clkd->r.rates, clkd->r.num_rates,
+		     sizeof(clkd->r.rates[0]), rate_cmp_func, NULL);
+
+	return 0;
+}
+
+static int
+scmi_clock_describe_rates_get_lazy(const struct scmi_protocol_handle *ph,
+				   struct scmi_clock_desc *clkd)
+{
+	struct scmi_iterator_ops ops = {
+		.prepare_message = iter_clk_describe_prepare_message,
+		.update_state = iter_clk_describe_update_state,
+		.process_response = iter_clk_describe_process_response,
+	};
+	struct scmi_clk_ipriv cpriv = {
+		.clkd = clkd,
+		.dev = ph->dev,
+	};
+	unsigned int first, last;
+	void *iter;
+	int ret;
+
+	iter = ph->hops->iter_response_init(ph, &ops, 0, CLOCK_DESCRIBE_RATES,
+					    sizeof(struct scmi_msg_clock_describe_rates),
+					    &cpriv);
+	if (IS_ERR(iter))
+		return PTR_ERR(iter);
+
+	/* Try to grab a triplet, so that in case is NON-discrete we are done */
+	first = 0;
+	last = 2;
+	ret = ph->hops->iter_response_run_bound(iter, &first, &last);
+	if (ret)
+		goto out;
+
+	/*
+	 * If discrete and we don't already have it, grab the last value, which
+	 * should be the max
+	 */
+	if (clkd->r.rate_discrete && clkd->tot_rates > clkd->r.num_rates) {
+		first = clkd->tot_rates - 1;
+		last = clkd->tot_rates - 1;
+		ret = ph->hops->iter_response_run_bound(iter, &first, &last);
 	}
 
+out:
+	ph->hops->iter_response_bound_cleanup(iter);
+
 	return ret;
 }
 
+static int
+scmi_clock_describe_rates_get(const struct scmi_protocol_handle *ph,
+			      u32 clk_id, struct clock_info *cinfo)
+{
+	struct scmi_clock_desc *clkd = &cinfo->clkds[clk_id];
+	int ret;
+
+	/*
+	 * Since only after SCMI Clock v1.0 the returned rates are guaranteed to
+	 * be discovered in ascending order, lazy enumeration cannot be use for
+	 * SCMI Clock v1.0 protocol.
+	 */
+	if (PROTOCOL_REV_MAJOR(ph->version) > 0x1)
+		ret = scmi_clock_describe_rates_get_lazy(ph, clkd);
+	else
+		ret = scmi_clock_describe_rates_get_full(ph, clkd);
+
+	if (ret)
+		return ret;
+
+	clkd->info.min_rate = clkd->r.rates[RATE_MIN];
+	if (!clkd->r.rate_discrete) {
+		clkd->info.max_rate = clkd->r.rates[RATE_MAX];
+		dev_dbg(ph->dev, "Min %llu Max %llu Step %llu Hz\n",
+			clkd->r.rates[RATE_MIN], clkd->r.rates[RATE_MAX],
+			clkd->r.rates[RATE_STEP]);
+	} else {
+		clkd->info.max_rate = clkd->r.rates[clkd->r.num_rates - 1];
+		dev_dbg(ph->dev, "Clock:%s Num_Rates:%u -> Min %llu Max %llu\n",
+			clkd->info.name, clkd->tot_rates,
+			clkd->info.min_rate, clkd->info.max_rate);
+	}
+
+	return 0;
+}
+
 static int
 scmi_clock_rate_get(const struct scmi_protocol_handle *ph,
 		    u32 clk_id, u64 *value)
@@ -624,6 +715,75 @@ static int scmi_clock_rate_set(const struct scmi_protocol_handle *ph,
 	return ret;
 }
 
+static int scmi_clock_determine_rate(const struct scmi_protocol_handle *ph,
+				     u32 clk_id, unsigned long *rate)
+{
+	u64 fmin, fmax, ftmp;
+	struct scmi_clock_info *clk;
+	struct scmi_clock_desc *clkd;
+	struct clock_info *ci = ph->get_priv(ph);
+
+	if (!rate)
+		return -EINVAL;
+
+	clk = scmi_clock_domain_lookup(ci, clk_id);
+	if (IS_ERR(clk))
+		return PTR_ERR(clk);
+
+	clkd = to_desc(clk);
+
+	/*
+	 * If we can't figure out what rate it will be, so just return the
+	 * rate back to the caller.
+	 */
+	if (clkd->r.rate_discrete)
+		return 0;
+
+	fmin = clk->min_rate;
+	fmax = clk->max_rate;
+	if (*rate <= fmin) {
+		*rate = fmin;
+		return 0;
+	} else if (*rate >= fmax) {
+		*rate = fmax;
+		return 0;
+	}
+
+	ftmp = *rate - fmin;
+	ftmp += clkd->r.rates[RATE_STEP] - 1; /* to round up */
+	ftmp = div64_ul(ftmp, clkd->r.rates[RATE_STEP]);
+
+	*rate = ftmp * clkd->r.rates[RATE_STEP] + fmin;
+
+	return 0;
+}
+
+static const struct scmi_clock_rates *
+scmi_clock_all_rates_get(const struct scmi_protocol_handle *ph, u32 clk_id)
+{
+	struct clock_info *ci = ph->get_priv(ph);
+	struct scmi_clock_desc *clkd;
+	struct scmi_clock_info *clk;
+
+	clk = scmi_clock_domain_lookup(ci, clk_id);
+	if (IS_ERR(clk) || !clk->name[0])
+		return NULL;
+
+	clkd = to_desc(clk);
+	/* Needs full enumeration ? */
+	if (clkd->r.rate_discrete && clkd->tot_rates != clkd->r.num_rates) {
+		int ret;
+
+		/* rates[] is already allocated BUT we need to re-enumerate */
+		clkd->r.num_rates = 0;
+		ret = scmi_clock_describe_rates_get_full(ph, clkd);
+		if (ret)
+			return NULL;
+	}
+
+	return &clkd->r;
+}
+
 static int
 scmi_clock_config_set(const struct scmi_protocol_handle *ph, u32 clk_id,
 		      enum clk_state state,
@@ -936,6 +1096,8 @@ static const struct scmi_clk_proto_ops clk_proto_ops = {
 	.info_get = scmi_clock_info_get,
 	.rate_get = scmi_clock_rate_get,
 	.rate_set = scmi_clock_rate_set,
+	.determine_rate = scmi_clock_determine_rate,
+	.all_rates_get = scmi_clock_all_rates_get,
 	.enable = scmi_clock_enable,
 	.disable = scmi_clock_disable,
 	.state_get = scmi_clock_state_get,
@@ -1080,17 +1242,16 @@ static int scmi_clock_protocol_init(const struct scmi_protocol_handle *ph)
 	if (ret)
 		return ret;
 
-	cinfo->clk = devm_kcalloc(ph->dev, cinfo->num_clocks,
-				  sizeof(*cinfo->clk), GFP_KERNEL);
-	if (!cinfo->clk)
+	cinfo->clkds = devm_kcalloc(ph->dev, cinfo->num_clocks,
+				    sizeof(*cinfo->clkds), GFP_KERNEL);
+	if (!cinfo->clkds)
 		return -ENOMEM;
 
 	for (clkid = 0; clkid < cinfo->num_clocks; clkid++) {
-		struct scmi_clock_info *clk = cinfo->clk + clkid;
-
+		cinfo->clkds[clkid].id = clkid;
 		ret = scmi_clock_attributes_get(ph, clkid, cinfo);
 		if (!ret)
-			scmi_clock_describe_rates_get(ph, clkid, clk);
+			scmi_clock_describe_rates_get(ph, clkid, cinfo);
 	}
 
 	if (PROTOCOL_REV_MAJOR(ph->version) >= 0x3) {
diff --git a/drivers/firmware/arm_scmi/common.h b/drivers/firmware/arm_scmi/common.h
index 7c9617d080a02..b9723c105fc1b 100644
--- a/drivers/firmware/arm_scmi/common.h
+++ b/drivers/firmware/arm_scmi/common.h
@@ -17,6 +17,7 @@
 #include <linux/hashtable.h>
 #include <linux/list.h>
 #include <linux/module.h>
+#include <linux/property.h>
 #include <linux/refcount.h>
 #include <linux/scmi_protocol.h>
 #include <linux/spinlock.h>
@@ -138,7 +139,7 @@ static inline void unpack_scmi_header(u32 msg_hdr, struct scmi_msg_hdr *hdr)
 	xfer_;							\
 })
 
-struct scmi_revision_info *
+struct scmi_base_info *
 scmi_revision_area_get(const struct scmi_protocol_handle *ph);
 void scmi_setup_protocol_implemented(const struct scmi_protocol_handle *ph,
 				     u8 *prot_imp);
@@ -462,6 +463,28 @@ struct scmi_transport_core_operations {
 	const struct scmi_message_operations *msg;
 };
 
+/**
+ * struct scmi_transport_handle  - Transport instance handle
+ * @supplier_get: A helper to retrieve the device descriptor, identifying the
+ *		  transport driver serving this SCMI instance, which will be
+ *		  used as a supplier for the core SCMI driver: returning an
+ *		  error here causes the probe sequence to be interrupted and
+ *		  return that same error code, so that each transport can decide
+ *		  which policy to implement by choosing an appropriate error.
+ * @supplier_put: A helper to signal that the specified transport supplier is
+ *		  no more being used and it is made available again.
+ *
+ * Note that these helpers are needed and provided only by those transports
+ * whose initialization relies on some other subsystem and whose relations to
+ * the core SCMI driver is not tracked by firmware descriptions.
+ */
+struct scmi_transport_handle {
+	struct device __must_check *(*supplier_get)
+		(const struct scmi_transport_handle *th);
+	int (*supplier_put)(const struct scmi_transport_handle *th,
+			    struct device *dev);
+};
+
 /**
  * struct scmi_transport  - A structure representing a configured transport
  *
@@ -470,35 +493,163 @@ struct scmi_transport_core_operations {
  * @desc: Transport descriptor
  * @core_ops: A pointer to a pointer used by the core SCMI stack to make the
  *	      core transport operations accessible to the transports.
+ * @th: An optional pointer to the transport handle
  */
 struct scmi_transport {
 	struct device *supplier;
 	struct scmi_desc desc;
 	struct scmi_transport_core_operations **core_ops;
+	const struct scmi_transport_handle *th;
+};
+
+/**
+ * struct scmi_transport_supplier  - Transport descriptor
+ * @mtx: A mutex to protect @available
+ * @available: A reference to an initialized transport device, when available.
+ *	       This reference is implicitly used to track the status of the
+ *	       supplier and it can cycle through the following 3 states:
+ *	       1. NOT_READY - PTR_ERR(-EPROBE_DEFER): no supplier available;
+ *		  this is the transport initial state.
+ *	       2. AVAILABLE - <supplier_dev>: a transport supplier has been
+ *		  initialized and it is available, ready to use.
+ *	       3. BUSY _ PTR_ERR(-EBUSY): transport supplier is currently in use.
+ * @th: An embedded transport handle object that embeds the helpers
+ *	implementing the above mentioned logic
+ *
+ * Note that this transport driver enforces single instance probing.
+ */
+struct scmi_transport_supplier {
+	/* Protect @available */
+	struct mutex mtx;
+	struct device *available;
+	const struct scmi_transport_handle th;
 };
 
+#define to_sup(t)	container_of(t, struct scmi_transport_supplier, th)
+
+/**
+ * scmi_transport_supplier_put  - A helper to dispose of a supplier
+ * @th: A reference to the transport handle to use
+ * @supplier: A reference to the device supplier to manage, cannot be NULL
+ *	      or ERR_PTR.
+ *
+ * Note that putting a supplier will have different effect based on the
+ * current state of scmi_transport_supplier.available:
+ *  - NOT_READY/BUSY: @supplier will be set as the new available device: this
+ *		      can be used to made available a supplier OR stop using one.
+ *  - AVAILABLE: if the @supplier we are disposing of matches the currently
+ *		 available one, roll back to NOT_READY state.
+ *		 Any other attempt to override an available supplier with a
+ *		 new one is rejected, effectively enforcing one single supplier.
+ *
+ * Return: 0 on Success, errno otherwise.
+ */
+static inline int
+scmi_transport_supplier_put(const struct scmi_transport_handle *th,
+			    struct device *supplier)
+{
+	struct scmi_transport_supplier *sup = to_sup(th);
+
+	/* Nothing to do when the provided supplier was never real */
+	if (IS_ERR_OR_NULL(supplier))
+		return 0;
+
+	guard(mutex)(&sup->mtx);
+	switch (PTR_ERR_OR_ZERO(sup->available)) {
+	case -EPROBE_DEFER:
+	case -EBUSY:
+		sup->available = supplier;
+		break;
+	case 0:
+		/* Putting a supplier when in the AVAILABLE state causes a
+		 * transition back to the NOT_READY state, BUT only if the
+		 * supplier we are disposing of was exactly the device that was
+		 * previously made readily available.
+		 */
+		if (supplier != sup->available)
+			return -EINVAL;
+		sup->available = ERR_PTR(-EPROBE_DEFER);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/**
+ * scmi_transport_supplier_get  - A helper to get hold of a supplier
+ * @th: A reference to the transport handle to use
+ *
+ * Note that, trying to get a supplier device can return:
+ *  - a ready to use supplier device, (subsequently made unavailable)
+ *  - PTR_ERR(-EPROBE_DEFER): no supplier is available
+ *  - PTR_ERR(-BUSY): supplier was already taken by a previous get
+ *
+ *  This allows the probe to defer and wait when a possible device can
+ *  be reasonably expected to appear.
+ *
+ * Return: a usable supplier device on Success or PTR_ERR on Failure.
+ */
+static inline struct device *
+scmi_transport_supplier_get(const struct scmi_transport_handle *th)
+{
+	struct scmi_transport_supplier *sup = to_sup(th);
+	struct device *supplier;
+
+	guard(mutex)(&sup->mtx);
+	supplier = sup->available;
+	if (!IS_ERR(sup->available))
+		sup->available = ERR_PTR(-EBUSY);
+
+	return supplier;
+}
+
+#define DEFINE_SCMI_TRANSPORT_SUPPLIER(__supplier)		\
+struct scmi_transport_supplier __supplier = {			\
+	.mtx = __MUTEX_INITIALIZER(__supplier.mtx),		\
+	.available = INIT_ERR_PTR(-EPROBE_DEFER),		\
+	.th.supplier_get = scmi_transport_supplier_get,		\
+	.th.supplier_put = scmi_transport_supplier_put,		\
+}
+
 #define DEFINE_SCMI_TRANSPORT_DRIVER(__tag, __drv, __desc, __match, __core_ops)\
 static void __tag##_dev_free(void *data)				       \
 {									       \
 	struct platform_device *spdev = data;				       \
+	struct scmi_transport *strans;					       \
+									       \
+	strans = dev_get_platdata(&spdev->dev);				       \
+	if (strans && strans->th)					       \
+		strans->th->supplier_put(strans->th, strans->supplier);	       \
 									       \
 	platform_device_unregister(spdev);				       \
 }									       \
 									       \
 static int __tag##_probe(struct platform_device *pdev)			       \
 {									       \
-	struct device *dev = &pdev->dev;				       \
+	struct device *dev = &pdev->dev, *supplier;			       \
 	struct platform_device *spdev;					       \
 	struct scmi_transport strans;					       \
 	int ret;							       \
 									       \
+	supplier = dev;							       \
+	strans.th = device_get_match_data(dev);				       \
+	if (strans.th) {						       \
+		supplier = strans.th->supplier_get(strans.th);		       \
+		if (IS_ERR(supplier))					       \
+			return PTR_ERR(supplier);			       \
+	}								       \
+									       \
 	spdev = platform_device_alloc("arm-scmi", PLATFORM_DEVID_AUTO);	       \
-	if (!spdev)							       \
-		return -ENOMEM;						       \
+	if (!spdev) {							       \
+		ret = -ENOMEM;						       \
+		goto err_mem;						       \
+	}								       \
 									       \
 	device_set_of_node_from_dev(&spdev->dev, dev);			       \
 									       \
-	strans.supplier = dev;						       \
+	strans.supplier = supplier;					       \
 	memcpy(&strans.desc, &(__desc), sizeof(strans.desc));		       \
 	strans.core_ops = &(__core_ops);				       \
 									       \
@@ -515,6 +666,10 @@ static int __tag##_probe(struct platform_device *pdev)			       \
 									       \
 err:									       \
 	platform_device_put(spdev);					       \
+err_mem:								       \
+	if (strans.th)							       \
+		strans.th->supplier_put(strans.th, supplier);			       \
+									       \
 	return ret;							       \
 }									       \
 									       \
diff --git a/drivers/firmware/arm_scmi/driver.c b/drivers/firmware/arm_scmi/driver.c
index f167194f7cf67..3e0d975ec94c4 100644
--- a/drivers/firmware/arm_scmi/driver.c
+++ b/drivers/firmware/arm_scmi/driver.c
@@ -17,6 +17,7 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/bitmap.h>
+#include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/device.h>
 #include <linux/export.h>
@@ -133,7 +134,7 @@ struct scmi_protocol_instance {
  *	       usage.
  * @protocols_mtx: A mutex to protect protocols instances initialization.
  * @protocols_imp: List of protocols implemented, currently maximum of
- *		   scmi_revision_info.num_protocols elements allocated by the
+ *		   scmi_base_info.num_protocols elements allocated by the
  *		   base protocol
  * @active_protocols: IDR storing device_nodes for protocols actually defined
  *		      in the DT and confirmed as implemented by fw.
@@ -151,7 +152,7 @@ struct scmi_info {
 	int id;
 	struct device *dev;
 	const struct scmi_desc *desc;
-	struct scmi_revision_info version;
+	struct scmi_base_info version;
 	struct scmi_handle handle;
 	struct scmi_xfers_info tx_minfo;
 	struct scmi_xfers_info rx_minfo;
@@ -265,7 +266,7 @@ scmi_vendor_protocol_lookup(int protocol_id, char *vendor_id,
 }
 
 static const struct scmi_protocol *
-scmi_vendor_protocol_get(int protocol_id, struct scmi_revision_info *version)
+scmi_vendor_protocol_get(int protocol_id, struct scmi_base_info *version)
 {
 	const struct scmi_protocol *proto;
 
@@ -303,7 +304,7 @@ scmi_vendor_protocol_get(int protocol_id, struct scmi_revision_info *version)
 }
 
 static const struct scmi_protocol *
-scmi_protocol_get(int protocol_id, struct scmi_revision_info *version)
+scmi_protocol_get(int protocol_id, struct scmi_base_info *version)
 {
 	const struct scmi_protocol *proto = NULL;
 
@@ -1789,86 +1790,115 @@ static void *scmi_iterator_init(const struct scmi_protocol_handle *ph,
 				size_t tx_size, void *priv)
 {
 	int ret;
-	struct scmi_iterator *i;
 
-	i = devm_kzalloc(ph->dev, sizeof(*i), GFP_KERNEL);
+	struct scmi_iterator *i __free(kfree) = kzalloc(sizeof(*i), GFP_KERNEL);
 	if (!i)
 		return ERR_PTR(-ENOMEM);
 
+	if (!ops || !ph)
+		return ERR_PTR(-EINVAL);
+
 	i->ph = ph;
 	i->ops = ops;
 	i->priv = priv;
 
 	ret = ph->xops->xfer_get_init(ph, msg_id, tx_size, 0, &i->t);
-	if (ret) {
-		devm_kfree(ph->dev, i);
+	if (ret)
 		return ERR_PTR(ret);
-	}
 
 	i->state.max_resources = max_resources;
 	i->msg = i->t->tx.buf;
 	i->resp = i->t->rx.buf;
 
-	return i;
+	return no_free_ptr(i);
 }
 
-static int scmi_iterator_run(void *iter)
+static int __scmi_iterator_run(void *iter, unsigned int *start, unsigned int *end)
 {
-	int ret = -EINVAL;
+	int ret;
 	struct scmi_iterator_ops *iops;
 	const struct scmi_protocol_handle *ph;
 	struct scmi_iterator_state *st;
-	struct scmi_iterator *i = iter;
+	struct scmi_iterator *i;
+	unsigned int n;
 
-	if (!i || !i->ops || !i->ph)
-		return ret;
+	if (!iter)
+		return -EINVAL;
 
+	i = iter;
 	iops = i->ops;
 	ph = i->ph;
 	st = &i->state;
 
+	/* Reinitialize state for next run */
+	st->num_returned = 0;
+	st->num_remaining = 0;
+	st->desc_index = start ? *start : 0;
+
 	do {
 		iops->prepare_message(i->msg, st->desc_index, i->priv);
 		ret = ph->xops->do_xfer(ph, i->t);
 		if (ret)
-			break;
+			return ret;
 
 		st->rx_len = i->t->rx.len;
 		ret = iops->update_state(st, i->resp, i->priv);
 		if (ret)
-			break;
+			return ret;
 
 		if (st->num_returned > st->max_resources - st->desc_index) {
 			dev_err(ph->dev,
 				"No. of resources can't exceed %d\n",
 				st->max_resources);
-			ret = -EINVAL;
-			break;
+			return -EINVAL;
 		}
 
-		for (st->loop_idx = 0; st->loop_idx < st->num_returned;
-		     st->loop_idx++) {
+		if (end)
+			n = min(st->num_returned, *end - st->desc_index + 1);
+		else
+			n = st->num_returned;
+		for (st->loop_idx = 0; st->loop_idx < n; st->loop_idx++) {
 			ret = iops->process_response(ph, i->resp, st, i->priv);
 			if (ret)
-				goto out;
+				return ret;
 		}
 
-		st->desc_index += st->num_returned;
+		st->desc_index += n;
 		ph->xops->reset_rx_to_maxsz(ph, i->t);
 		/*
 		 * check for both returned and remaining to avoid infinite
 		 * loop due to buggy firmware
 		 */
-	} while (st->num_returned && st->num_remaining);
+	} while (st->num_returned && st->num_remaining &&
+		 (!end || (st->desc_index <= min(*end, st->max_resources - 1))));
 
-out:
-	/* Finalize and destroy iterator */
-	ph->xops->xfer_put(ph, i->t);
-	devm_kfree(ph->dev, i);
+	return 0;
+}
+
+static void scmi_iterator_bound_cleanup(void *iter)
+{
+	struct scmi_iterator *i = iter;
+
+	i->ph->xops->xfer_put(i->ph, i->t);
+	kfree(i);
+}
+
+static int scmi_iterator_run(void *iter)
+{
+	int ret;
+
+	ret = __scmi_iterator_run(iter, NULL, NULL);
+	scmi_iterator_bound_cleanup(iter);
 
 	return ret;
 }
 
+static int scmi_iterator_run_bound(void *iter, unsigned int *start,
+				   unsigned int *end)
+{
+	return __scmi_iterator_run(iter, start, end);
+}
+
 struct scmi_msg_get_fc_info {
 	__le32 domain;
 	__le32 message_id;
@@ -2047,6 +2077,8 @@ static const struct scmi_proto_helpers_ops helpers_ops = {
 	.get_max_msg_size = scmi_common_get_max_msg_size,
 	.iter_response_init = scmi_iterator_init,
 	.iter_response_run = scmi_iterator_run,
+	.iter_response_run_bound = scmi_iterator_run_bound,
+	.iter_response_bound_cleanup = scmi_iterator_bound_cleanup,
 	.protocol_msg_check = scmi_protocol_msg_check,
 	.fastchannel_init = scmi_common_fastchannel_init,
 	.fastchannel_db_ring = scmi_common_fastchannel_db_ring,
@@ -2063,7 +2095,7 @@ static const struct scmi_proto_helpers_ops helpers_ops = {
  * Return: A reference to the version memory area associated to the SCMI
  *	   instance underlying this protocol handle.
  */
-struct scmi_revision_info *
+struct scmi_base_info *
 scmi_revision_area_get(const struct scmi_protocol_handle *ph)
 {
 	const struct scmi_protocol_instance *pi = ph_to_pi(ph);
@@ -2376,7 +2408,7 @@ scmi_is_protocol_implemented(const struct scmi_handle *handle, u8 prot_id)
 {
 	int i;
 	struct scmi_info *info = handle_to_scmi_info(handle);
-	struct scmi_revision_info *rev = handle->version;
+	struct scmi_base_info *rev = handle->version;
 
 	if (!info->protocols_imp)
 		return false;
@@ -2552,13 +2584,11 @@ static bool scmi_is_transport_atomic(const struct scmi_handle *handle,
  */
 static struct scmi_handle *scmi_handle_get(struct device *dev)
 {
-	struct list_head *p;
 	struct scmi_info *info;
 	struct scmi_handle *handle = NULL;
 
 	mutex_lock(&scmi_list_mutex);
-	list_for_each(p, &scmi_list) {
-		info = list_entry(p, struct scmi_info, node);
+	list_for_each_entry(info, &scmi_list, node) {
 		if (dev->parent == info->dev) {
 			info->users++;
 			handle = &info->handle;
@@ -3171,7 +3201,7 @@ static const struct scmi_desc *scmi_transport_setup(struct device *dev)
 
 static void scmi_enable_matching_quirks(struct scmi_info *info)
 {
-	struct scmi_revision_info *rev = &info->version;
+	struct scmi_base_info *rev = &info->version;
 
 	dev_dbg(info->dev, "Looking for quirks matching: %s/%s/0x%08X\n",
 		rev->vendor_id, rev->sub_vendor_id, rev->impl_ver);
diff --git a/drivers/firmware/arm_scmi/powercap.c b/drivers/firmware/arm_scmi/powercap.c
index ab9733f4458b8..27e3c805e9272 100644
--- a/drivers/firmware/arm_scmi/powercap.c
+++ b/drivers/firmware/arm_scmi/powercap.c
@@ -453,10 +453,14 @@ static int scmi_powercap_cap_set(const struct scmi_protocol_handle *ph,
 		return -EINVAL;
 
 	/* Just log the last set request if acting on a disabled domain */
-	if (PROTOCOL_REV_MAJOR(ph->version) >= 0x2 &&
-	    !pi->states[domain_id].enabled) {
-		pi->states[domain_id].last_pcap = power_cap;
-		return 0;
+	if (PROTOCOL_REV_MAJOR(ph->version) >= 0x2) {
+		if (!scmi_powercap_dom_info_get(ph, domain_id))
+			return -EINVAL;
+
+		if (!pi->states[domain_id].enabled) {
+			pi->states[domain_id].last_pcap = power_cap;
+			return 0;
+		}
 	}
 
 	return __scmi_powercap_cap_set(ph, pi, domain_id,
@@ -637,6 +641,9 @@ static int scmi_powercap_cap_enable_set(const struct scmi_protocol_handle *ph,
 	if (PROTOCOL_REV_MAJOR(ph->version) < 0x2)
 		return -EINVAL;
 
+	if (!scmi_powercap_dom_info_get(ph, domain_id))
+		return -EINVAL;
+
 	if (enable == pi->states[domain_id].enabled)
 		return 0;
 
@@ -678,6 +685,9 @@ static int scmi_powercap_cap_enable_get(const struct scmi_protocol_handle *ph,
 	if (PROTOCOL_REV_MAJOR(ph->version) < 0x2)
 		return 0;
 
+	if (!scmi_powercap_dom_info_get(ph, domain_id))
+		return -EINVAL;
+
 	/*
 	 * Report always real platform state; platform could have ignored
 	 * a previous disable request. Default true on any error.
diff --git a/drivers/firmware/arm_scmi/protocols.h b/drivers/firmware/arm_scmi/protocols.h
index f51245aca2594..15ad5162e37a9 100644
--- a/drivers/firmware/arm_scmi/protocols.h
+++ b/drivers/firmware/arm_scmi/protocols.h
@@ -259,7 +259,15 @@ struct scmi_fc_info {
  *			multi-part responses using the custom operations
  *			provided in @ops.
  * @iter_response_run: A common helper to trigger the run of a previously
- *		       initialized iterator.
+ *		       initialized iterator. Note that unbound iterators are
+ *		       automatically cleaned up.
+ * @iter_response_run_bound: A common helper to trigger the run of a previously
+ *			     initialized iterator, but only within the
+ *			     specified, optional, @start and @end resource
+ *			     indexes. Note that these bound-iterators need
+ *			     explicit cleanup via @iter_response_bound_cleanup.
+ * @iter_response_bound_cleanup: A common helper to finally release the iterator
+ *				 for bound iterators.
  * @protocol_msg_check: A common helper to check is a specific protocol message
  *			is supported.
  * @fastchannel_init: A common helper used to initialize FC descriptors by
@@ -276,6 +284,9 @@ struct scmi_proto_helpers_ops {
 				    unsigned int max_resources, u8 msg_id,
 				    size_t tx_size, void *priv);
 	int (*iter_response_run)(void *iter);
+	int (*iter_response_run_bound)(void *iter,
+				       unsigned int *start, unsigned int *end);
+	void (*iter_response_bound_cleanup)(void *iter);
 	int (*protocol_msg_check)(const struct scmi_protocol_handle *ph,
 				  u32 message_id, u32 *attributes);
 	void (*fastchannel_init)(const struct scmi_protocol_handle *ph,
diff --git a/drivers/firmware/arm_scmi/quirks.c b/drivers/firmware/arm_scmi/quirks.c
index 03848283c2a07..9ed5ce4feded1 100644
--- a/drivers/firmware/arm_scmi/quirks.c
+++ b/drivers/firmware/arm_scmi/quirks.c
@@ -219,9 +219,9 @@ static unsigned int scmi_quirk_signature(const char *vend, const char *sub_vend)
 static int scmi_quirk_range_parse(struct scmi_quirk *quirk)
 {
 	const char *last, *first __free(kfree) = NULL;
+	int ret = 0;
 	size_t len;
 	char *sep;
-	int ret;
 
 	quirk->start_range = 0;
 	quirk->end_range = 0xFFFFFFFF;
@@ -238,16 +238,15 @@ static int scmi_quirk_range_parse(struct scmi_quirk *quirk)
 	if (sep)
 		*sep = '\0';
 
-	if (sep == first) /* -X */
-		ret = kstrtouint(first + 1, 0, &quirk->end_range);
-	else /* X OR X- OR X-y */
+	if (sep != first) /* X OR X- OR X-y */ {
 		ret = kstrtouint(first, 0, &quirk->start_range);
-	if (ret)
-		return ret;
+		if (ret)
+			return ret;
+	}
 
 	if (!sep)
 		quirk->end_range = quirk->start_range;
-	else if (sep != last) /* x-Y */
+	else if (sep != last) /* -X OR x-Y */
 		ret = kstrtouint(sep + 1, 0, &quirk->end_range);
 
 	if (quirk->start_range > quirk->end_range)
@@ -259,10 +258,8 @@ static int scmi_quirk_range_parse(struct scmi_quirk *quirk)
 void scmi_quirks_initialize(void)
 {
 	struct scmi_quirk *quirk;
-	int i;
 
-	for (i = 0, quirk = scmi_quirks_table[0]; quirk;
-	     i++, quirk = scmi_quirks_table[i]) {
+	for (unsigned int i = 0; (quirk = scmi_quirks_table[i]); i++) {
 		int ret;
 
 		ret = scmi_quirk_range_parse(quirk);
diff --git a/drivers/firmware/arm_scmi/sensors.c b/drivers/firmware/arm_scmi/sensors.c
index 882d55f987d22..b14bb1146356f 100644
--- a/drivers/firmware/arm_scmi/sensors.c
+++ b/drivers/firmware/arm_scmi/sensors.c
@@ -793,7 +793,7 @@ static int scmi_sensor_config_get(const struct scmi_protocol_handle *ph,
 	if (!ret) {
 		struct scmi_sensor_info *s = si->sensors + sensor_id;
 
-		*sensor_config = get_unaligned_le64(t->rx.buf);
+		*sensor_config = get_unaligned_le32(t->rx.buf);
 		s->sensor_config = *sensor_config;
 	}
 
@@ -1072,12 +1072,15 @@ scmi_sensor_fill_custom_report(const struct scmi_protocol_handle *ph,
 	case SCMI_EVENT_SENSOR_UPDATE:
 	{
 		int i;
+		size_t expected_sz;
 		struct scmi_sensor_info *s;
 		const struct scmi_sensor_update_notify_payld *p = payld;
 		struct scmi_sensor_update_report *r = report;
 		struct sensors_info *sinfo = ph->get_priv(ph);
 
-		/* payld_sz is variable for this event */
+		if (payld_sz < sizeof(*p))
+			break;
+
 		r->sensor_id = le32_to_cpu(p->sensor_id);
 		if (r->sensor_id >= sinfo->num_sensors)
 			break;
@@ -1091,6 +1094,11 @@ scmi_sensor_fill_custom_report(const struct scmi_protocol_handle *ph,
 		 * readings defined for this sensor or 1 for scalar sensors.
 		 */
 		r->readings_count = s->num_axis ?: 1;
+		expected_sz = sizeof(*p) + r->readings_count *
+			      sizeof(p->readings[0]);
+		if (payld_sz < expected_sz)
+			break;
+
 		for (i = 0; i < r->readings_count; i++)
 			scmi_parse_sensor_readings(&r->readings[i],
 						   &p->readings[i]);
diff --git a/drivers/firmware/arm_scmi/transports/optee.c b/drivers/firmware/arm_scmi/transports/optee.c
index 07ae18d5279d9..dbe32e141748b 100644
--- a/drivers/firmware/arm_scmi/transports/optee.c
+++ b/drivers/firmware/arm_scmi/transports/optee.c
@@ -154,6 +154,8 @@ static struct scmi_transport_core_operations *core;
 /* There can be only 1 SCMI service in OP-TEE we connect to */
 static struct scmi_optee_agent *scmi_optee_private;
 
+static DEFINE_SCMI_TRANSPORT_SUPPLIER(scmi_optee_supplier);
+
 /* Open a session toward SCMI OP-TEE service with REE_KERNEL identity */
 static int open_session(struct scmi_optee_agent *agent, u32 *tee_session)
 {
@@ -522,7 +524,7 @@ static struct scmi_desc scmi_optee_desc = {
 };
 
 static const struct of_device_id scmi_of_match[] = {
-	{ .compatible = "linaro,scmi-optee" },
+	{ .compatible = "linaro,scmi-optee", .data = &scmi_optee_supplier.th},
 	{ /* Sentinel */ },
 };
 
@@ -561,18 +563,20 @@ static int scmi_optee_service_probe(struct tee_client_device *scmi_pta)
 	if (ret)
 		goto err;
 
-	/* Ensure agent resources are all visible before scmi_optee_private is */
+	/* Ensure initialized scmi_optee_private is visible */
 	smp_mb();
 	scmi_optee_private = agent;
 
-	ret = platform_driver_register(&scmi_optee_driver);
-	if (ret) {
-		scmi_optee_private = NULL;
-		goto err;
-	}
+	ret = scmi_transport_supplier_put(&scmi_optee_supplier.th, agent->dev);
+	if (ret)
+		goto err_put;
 
 	return 0;
 
+err_put:
+	/* Ensure cleared reference is visible before resources are released */
+	smp_store_mb(scmi_optee_private, NULL);
+
 err:
 	tee_client_close_context(tee_ctx);
 
@@ -586,13 +590,12 @@ static void scmi_optee_service_remove(struct tee_client_device *scmi_pta)
 	if (!scmi_optee_private)
 		return;
 
-	platform_driver_unregister(&scmi_optee_driver);
-
 	if (!list_empty(&scmi_optee_private->channel_list))
 		return;
 
 	/* Ensure cleared reference is visible before resources are released */
 	smp_store_mb(scmi_optee_private, NULL);
+	scmi_transport_supplier_put(&scmi_optee_supplier.th, agent->dev);
 
 	tee_client_close_context(agent->tee_ctx);
 }
@@ -616,7 +619,30 @@ static struct tee_client_driver scmi_optee_service_driver = {
 	},
 };
 
-module_tee_client_driver(scmi_optee_service_driver);
+static int __init scmi_transport_optee_init(void)
+{
+	int ret;
+
+	ret = tee_client_driver_register(&scmi_optee_service_driver);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&scmi_optee_driver);
+	if (ret) {
+		tee_client_driver_unregister(&scmi_optee_service_driver);
+		return ret;
+	}
+
+	return ret;
+}
+module_init(scmi_transport_optee_init);
+
+static void __exit scmi_transport_optee_exit(void)
+{
+	platform_driver_unregister(&scmi_optee_driver);
+	tee_client_driver_unregister(&scmi_optee_service_driver);
+}
+module_exit(scmi_transport_optee_exit);
 
 MODULE_AUTHOR("Etienne Carriere <etienne.carriere@foss.st.com>");
 MODULE_DESCRIPTION("SCMI OPTEE Transport driver");
diff --git a/drivers/firmware/arm_scmi/transports/virtio.c b/drivers/firmware/arm_scmi/transports/virtio.c
index 326c4a93e44b9..3282d8271839d 100644
--- a/drivers/firmware/arm_scmi/transports/virtio.c
+++ b/drivers/firmware/arm_scmi/transports/virtio.c
@@ -4,7 +4,7 @@
  * (SCMI).
  *
  * Copyright (C) 2020-2022 OpenSynergy.
- * Copyright (C) 2021-2024 ARM Ltd.
+ * Copyright (C) 2021-2026 ARM Ltd.
  */
 
 /**
@@ -116,6 +116,8 @@ static struct scmi_transport_core_operations *core;
 /* Only one SCMI VirtIO device can possibly exist */
 static struct virtio_device *scmi_vdev;
 
+static DEFINE_SCMI_TRANSPORT_SUPPLIER(scmi_virtio_supplier);
+
 static void scmi_vio_channel_ready(struct scmi_vio_channel *vioch,
 				   struct scmi_chan_info *cinfo)
 {
@@ -394,6 +396,10 @@ static bool virtio_chan_available(struct device_node *of_node, int idx)
 		return false;
 	}
 
+	dev_dbg(&scmi_vdev->dev, "%s Channel %sAVAILABLE on SCMI Virtio device.\n",
+		idx == VIRTIO_SCMI_VQ_TX ? "TX" : "RX",
+		(vioch && !vioch->cinfo) ? "" : "NOT ");
+
 	return vioch && !vioch->cinfo;
 }
 
@@ -410,7 +416,7 @@ static int virtio_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
 	int i;
 
 	if (!scmi_vdev)
-		return -EPROBE_DEFER;
+		return -EINVAL;
 
 	vioch = &((struct scmi_vio_channel *)scmi_vdev->priv)[index];
 
@@ -460,6 +466,9 @@ static int virtio_chan_setup(struct scmi_chan_info *cinfo, struct device *dev,
 
 	scmi_vio_channel_ready(vioch, cinfo);
 
+	dev_dbg(&scmi_vdev->dev, "%s Channel SETUP on SCMI Virtio device.\n",
+		tx ? "TX" : "RX");
+
 	return 0;
 }
 
@@ -801,7 +810,7 @@ static struct scmi_desc scmi_virtio_desc = {
 };
 
 static const struct of_device_id scmi_of_match[] = {
-	{ .compatible = "arm,scmi-virtio" },
+	{ .compatible = "arm,scmi-virtio", .data = &scmi_virtio_supplier.th},
 	{ /* Sentinel */ },
 };
 
@@ -864,33 +873,33 @@ static int scmi_vio_probe(struct virtio_device *vdev)
 			sz = MSG_TOKEN_MAX;
 		}
 		channels[i].max_msg = sz;
+		dev_dbg(dev, "VQ%d initialized with max_msg: %d\n", i, sz);
 	}
 
 	vdev->priv = channels;
-
 	/* Ensure initialized scmi_vdev is visible */
 	smp_store_mb(scmi_vdev, vdev);
 
 	/* Set device ready */
 	virtio_device_ready(vdev);
 
-	ret = platform_driver_register(&scmi_virtio_driver);
+	ret = scmi_transport_supplier_put(&scmi_virtio_supplier.th, &vdev->dev);
 	if (ret) {
+		virtio_reset_device(vdev);
 		vdev->priv = NULL;
 		vdev->config->del_vqs(vdev);
 		/* Ensure NULLified scmi_vdev is visible */
 		smp_store_mb(scmi_vdev, NULL);
-
 		return ret;
 	}
 
+	dev_dbg(dev, "Probed and initialized SCMI Virtio device.\n");
+
 	return 0;
 }
 
 static void scmi_vio_remove(struct virtio_device *vdev)
 {
-	platform_driver_unregister(&scmi_virtio_driver);
-
 	/*
 	 * Once we get here, virtio_chan_free() will have already been called by
 	 * the SCMI core for any existing channel and, as a consequence, all the
@@ -900,8 +909,10 @@ static void scmi_vio_remove(struct virtio_device *vdev)
 	 */
 	virtio_reset_device(vdev);
 	vdev->config->del_vqs(vdev);
+
 	/* Ensure scmi_vdev is visible as NULL */
 	smp_store_mb(scmi_vdev, NULL);
+	scmi_transport_supplier_put(&scmi_virtio_supplier.th, &vdev->dev);
 }
 
 static int scmi_vio_validate(struct virtio_device *vdev)
@@ -936,7 +947,30 @@ static struct virtio_driver virtio_scmi_driver = {
 	.validate = scmi_vio_validate,
 };
 
-module_virtio_driver(virtio_scmi_driver);
+static int __init scmi_transport_virtio_init(void)
+{
+	int ret;
+
+	ret = register_virtio_driver(&virtio_scmi_driver);
+	if (ret)
+		return ret;
+
+	ret = platform_driver_register(&scmi_virtio_driver);
+	if (ret) {
+		unregister_virtio_driver(&virtio_scmi_driver);
+		return ret;
+	}
+
+	return ret;
+}
+module_init(scmi_transport_virtio_init);
+
+static void __exit scmi_transport_virtio_exit(void)
+{
+	platform_driver_unregister(&scmi_virtio_driver);
+	unregister_virtio_driver(&virtio_scmi_driver);
+}
+module_exit(scmi_transport_virtio_exit);
 
 MODULE_AUTHOR("Igor Skalkin <igor.skalkin@opensynergy.com>");
 MODULE_AUTHOR("Peter Hilber <peter.hilber@opensynergy.com>");
diff --git a/drivers/firmware/arm_scmi/vendors/imx/imx-sm-misc.c b/drivers/firmware/arm_scmi/vendors/imx/imx-sm-misc.c
index 0ada753367efe..637973fb45e65 100644
--- a/drivers/firmware/arm_scmi/vendors/imx/imx-sm-misc.c
+++ b/drivers/firmware/arm_scmi/vendors/imx/imx-sm-misc.c
@@ -27,6 +27,7 @@ enum scmi_imx_misc_protocol_cmd {
 	SCMI_IMX_MISC_CTRL_GET	= 0x4,
 	SCMI_IMX_MISC_DISCOVER_BUILD_INFO = 0x6,
 	SCMI_IMX_MISC_CTRL_NOTIFY = 0x8,
+	SCMI_IMX_MISC_RESET_REASON_GET = 0xA,
 	SCMI_IMX_MISC_CFG_INFO_GET = 0xC,
 	SCMI_IMX_MISC_SYSLOG_GET = 0xD,
 	SCMI_IMX_MISC_BOARD_INFO = 0xE,
@@ -89,6 +90,37 @@ struct scmi_imx_misc_cfg_info_out {
 	u8 cfgname[MISC_MAX_CFGNAME];
 };
 
+struct scmi_imx_misc_reset_reason_in {
+#define MISC_REASON_FLAG_SYSTEM	BIT(0)
+	__le32 flags;
+};
+
+struct scmi_imx_misc_reset_reason_out {
+	/* Boot reason flags */
+#define MISC_BOOT_FLAG_VLD		BIT(31)
+#define MISC_BOOT_FLAG_ORG_VLD		BIT(28)
+#define MISC_BOOT_FLAG_ORIGIN		GENMASK(27, 24)
+#define MISC_BOOT_FLAG_O_SHIFT		24
+#define MISC_BOOT_FLAG_ERR_VLD		BIT(23)
+#define MISC_BOOT_FLAG_ERR_ID		GENMASK(22, 8)
+#define MISC_BOOT_FLAG_E_SHIFT		8
+#define MISC_BOOT_FLAG_REASON		GENMASK(7, 0)
+	__le32 b_flags;
+	/* Shutdown reason flags */
+#define MISC_SHUTDOWN_FLAG_VLD		BIT(31)
+#define MISC_SHUTDOWN_FLAG_EXT_LEN	GENMASK(30, 29)
+#define MISC_SHUTDOWN_FLAG_ORG_VLD	BIT(28)
+#define MISC_SHUTDOWN_FLAG_ORIGIN	GENMASK(27, 24)
+#define MISC_SHUTDOWN_FLAG_O_SHIFT	24
+#define MISC_SHUTDOWN_FLAG_ERR_VLD	BIT(23)
+#define MISC_SHUTDOWN_FLAG_ERR_ID	GENMASK(22, 8)
+#define MISC_SHUTDOWN_FLAG_E_SHIFT	8
+#define MISC_SHUTDOWN_FLAG_REASON	GENMASK(7, 0)
+	__le32 s_flags;
+	/* Array of extended info words */
+	__le32 extinfo[MISC_EXT_INFO_LEN_MAX];
+};
+
 struct scmi_imx_misc_syslog_in {
 	__le32 flags;
 	__le32 index;
@@ -452,11 +484,65 @@ static int scmi_imx_misc_syslog_get(const struct scmi_protocol_handle *ph, u16 *
 	return ph->hops->iter_response_run(iter);
 }
 
+static int scmi_imx_misc_reset_reason(const struct scmi_protocol_handle *ph, bool system,
+				      struct scmi_imx_misc_reset_reason *boot_r,
+				      struct scmi_imx_misc_reset_reason *shut_r,
+				      u32 *extinfo)
+{
+	struct scmi_imx_misc_reset_reason_in *in;
+	struct scmi_imx_misc_reset_reason_out *out;
+	struct scmi_xfer *t;
+	int ret;
+
+	ret = ph->xops->xfer_get_init(ph, SCMI_IMX_MISC_RESET_REASON_GET, sizeof(*in),
+				      sizeof(*out), &t);
+	if (ret)
+		return ret;
+
+	in = t->tx.buf;
+	if (system)
+		in->flags = le32_encode_bits(1, MISC_REASON_FLAG_SYSTEM);
+	else
+		in->flags = cpu_to_le32(0);
+
+	ret = ph->xops->do_xfer(ph, t);
+	if (!ret) {
+		out = t->rx.buf;
+		if (boot_r) {
+			boot_r->valid = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_VLD);
+			boot_r->orig_valid = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_ORG_VLD);
+			boot_r->err_valid = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_ERR_VLD);
+			boot_r->reason = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_REASON);
+			boot_r->origin = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_ORIGIN);
+			boot_r->errid = le32_get_bits(out->b_flags, MISC_BOOT_FLAG_ERR_ID);
+		}
+
+		if (shut_r) {
+			shut_r->valid = le32_get_bits(out->s_flags, MISC_SHUTDOWN_FLAG_VLD);
+			shut_r->orig_valid = le32_get_bits(out->s_flags,
+							   MISC_SHUTDOWN_FLAG_ORG_VLD);
+			shut_r->err_valid = le32_get_bits(out->s_flags,
+							  MISC_SHUTDOWN_FLAG_ERR_VLD);
+			shut_r->reason = le32_get_bits(out->s_flags, MISC_SHUTDOWN_FLAG_REASON);
+			shut_r->origin = le32_get_bits(out->s_flags, MISC_SHUTDOWN_FLAG_ORIGIN);
+			shut_r->errid = le32_get_bits(out->s_flags, MISC_SHUTDOWN_FLAG_ERR_ID);
+		}
+
+		if (extinfo)
+			memcpy_from_le32(extinfo, out->extinfo, MISC_EXT_INFO_LEN_MAX);
+	}
+
+	ph->xops->xfer_put(ph, t);
+
+	return ret;
+}
+
 static const struct scmi_imx_misc_proto_ops scmi_imx_misc_proto_ops = {
 	.misc_ctrl_set = scmi_imx_misc_ctrl_set,
 	.misc_ctrl_get = scmi_imx_misc_ctrl_get,
 	.misc_ctrl_req_notify = scmi_imx_misc_ctrl_notify,
 	.misc_syslog = scmi_imx_misc_syslog_get,
+	.misc_reset_reason = scmi_imx_misc_reset_reason,
 };
 
 static int scmi_imx_misc_protocol_init(const struct scmi_protocol_handle *ph)
diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c
index d04be38f1750d..318d1cc9a066c 100644
--- a/drivers/firmware/efi/efi.c
+++ b/drivers/firmware/efi/efi.c
@@ -402,21 +402,11 @@ static void __init efi_debugfs_init(void)
 static inline void efi_debugfs_init(void) {}
 #endif
 
-/*
- * We register the efi subsystem with the firmware subsystem and the
- * efivars subsystem with the efi subsystem, if the system was booted with
- * EFI.
- */
-static int __init efisubsys_init(void)
+static int __init efipostcore_init(void)
 {
-	int error;
-
 	if (!efi_enabled(EFI_RUNTIME_SERVICES))
 		efi.runtime_supported_mask = 0;
 
-	if (!efi_enabled(EFI_BOOT))
-		return 0;
-
 	if (efi.runtime_supported_mask) {
 		/*
 		 * Since we process only one efi_runtime_service() at a time, an
@@ -428,9 +418,23 @@ static int __init efisubsys_init(void)
 			pr_err("Creating efi_rts_wq failed, EFI runtime services disabled.\n");
 			clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
 			efi.runtime_supported_mask = 0;
-			return 0;
 		}
 	}
+	return 0;
+}
+postcore_initcall(efipostcore_init);
+
+/*
+ * We register the efi subsystem with the firmware subsystem and the
+ * efivars subsystem with the efi subsystem, if the system was booted with
+ * EFI.
+ */
+static int __init efisubsys_init(void)
+{
+	int error;
+
+	if (!efi_enabled(EFI_BOOT))
+		return 0;
 
 	if (efi_rt_services_supported(EFI_RT_SUPPORTED_TIME_SERVICES))
 		platform_device_register_simple("rtc-efi", 0, NULL, 0);
diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile
index cfedb3025c263..251571d293c3f 100644
--- a/drivers/firmware/efi/libstub/Makefile
+++ b/drivers/firmware/efi/libstub/Makefile
@@ -95,7 +95,8 @@ CFLAGS_zboot-decompress-gzip.o	+= -I$(srctree)/lib/zlib_inflate
 zboot-obj-$(CONFIG_KERNEL_ZSTD)	:= zboot-decompress-zstd.o lib-xxhash.o
 CFLAGS_zboot-decompress-zstd.o	+= -I$(srctree)/lib/zstd
 
-zboot-obj-$(CONFIG_RISCV)	+= lib-clz_ctz.o lib-ashldi3.o
+zboot-riscv-obj-$(CONFIG_32BIT)	+= lib-ashldi3.o lib-lshrdi3.o
+zboot-obj-$(CONFIG_RISCV)	+= lib-clz_ctz.o $(zboot-riscv-obj-y)
 zboot-obj-$(CONFIG_LOONGARCH)	+= lib-clz_ctz.o lib-ashldi3.o
 lib-$(CONFIG_EFI_ZBOOT)		+= zboot.o $(zboot-obj-y)
 
diff --git a/drivers/firmware/efi/libstub/x86-5lvl.c b/drivers/firmware/efi/libstub/x86-5lvl.c
index c00d0ae7ed5d5..c3da05c0df8b7 100644
--- a/drivers/firmware/efi/libstub/x86-5lvl.c
+++ b/drivers/firmware/efi/libstub/x86-5lvl.c
@@ -2,6 +2,7 @@
 #include <linux/efi.h>
 
 #include <asm/boot.h>
+#include <asm/cpuid/api.h>
 #include <asm/desc.h>
 #include <asm/efi.h>
 
diff --git a/drivers/firmware/efi/sysfb_efi.c b/drivers/firmware/efi/sysfb_efi.c
index 4c3986ddcd54a..685283bb7327a 100644
--- a/drivers/firmware/efi/sysfb_efi.c
+++ b/drivers/firmware/efi/sysfb_efi.c
@@ -311,11 +311,14 @@ static const struct dmi_system_id efifb_dmi_swap_width_height[] __initconst = {
 		.callback = efifb_swap_width_height,
 	},
 	{
-		/* Lenovo IdeaPad Duet 3 10IGL5 with 1200x1920 portrait screen */
+		/*
+		 * Lenovo IdeaPad Duet 3 10IGL5 and 10IGL5-LTE with
+		 * 1200x1920 portrait screen
+		 */
 		.matches = {
 			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "LENOVO"),
-			DMI_EXACT_MATCH(DMI_PRODUCT_VERSION,
-					"IdeaPad Duet 3 10IGL5"),
+			/* Non exact match to also match the LTE version */
+			DMI_MATCH(DMI_PRODUCT_VERSION, "IdeaPad Duet 3 10IGL5"),
 		},
 		.callback = efifb_swap_width_height,
 	},
diff --git a/drivers/firmware/google/coreboot_table.c b/drivers/firmware/google/coreboot_table.c
index c769631ea15d7..83f7eedf0b3fb 100644
--- a/drivers/firmware/google/coreboot_table.c
+++ b/drivers/firmware/google/coreboot_table.c
@@ -112,16 +112,20 @@ void coreboot_driver_unregister(struct coreboot_driver *driver)
 }
 EXPORT_SYMBOL(coreboot_driver_unregister);
 
-static int coreboot_table_populate(struct device *dev, void *ptr)
+static int coreboot_table_populate(struct device *dev, void *ptr, resource_size_t len)
 {
 	int i, ret;
 	void *ptr_entry;
 	struct coreboot_device *device;
 	struct coreboot_table_entry *entry;
 	struct coreboot_table_header *header = ptr;
+	void *ptr_end;
 
+	ptr_end = ptr + len;
 	ptr_entry = ptr + header->header_bytes;
 	for (i = 0; i < header->table_entries; i++) {
+		if (ptr_entry + sizeof(*entry) > ptr_end)
+			return -EINVAL;
 		entry = ptr_entry;
 
 		if (entry->size < sizeof(*entry)) {
@@ -129,6 +133,9 @@ static int coreboot_table_populate(struct device *dev, void *ptr)
 			return -EINVAL;
 		}
 
+		if (ptr_entry + entry->size > ptr_end)
+			return -EINVAL;
+
 		device = kzalloc(sizeof(device->dev) + entry->size, GFP_KERNEL);
 		if (!device)
 			return -ENOMEM;
@@ -148,13 +155,13 @@ static int coreboot_table_populate(struct device *dev, void *ptr)
 			break;
 		}
 
+		ptr_entry += entry->size;
+
 		ret = device_register(&device->dev);
 		if (ret) {
+			dev_warn(dev, "failed to register coreboot device: %d\n", ret);
 			put_device(&device->dev);
-			return ret;
 		}
-
-		ptr_entry += entry->size;
 	}
 
 	return 0;
@@ -194,7 +201,7 @@ static int coreboot_table_probe(struct platform_device *pdev)
 	if (!ptr)
 		return -ENOMEM;
 
-	ret = coreboot_table_populate(dev, ptr);
+	ret = coreboot_table_populate(dev, ptr, len);
 
 	memunmap(ptr);
 
diff --git a/drivers/firmware/imx/Kconfig b/drivers/firmware/imx/Kconfig
index 127ad752acf86..1f89dd140113e 100644
--- a/drivers/firmware/imx/Kconfig
+++ b/drivers/firmware/imx/Kconfig
@@ -55,3 +55,16 @@ config IMX_SCMI_MISC_DRV
 	  core that could provide misc functions such as board control.
 
 	  This driver can also be built as a module.
+
+config IMX_SEC_ENCLAVE
+	tristate "i.MX Embedded Secure Enclave - EdgeLock Enclave Firmware driver."
+	depends on (IMX_MBOX && ARCH_MXC && ARM64) || COMPILE_TEST
+	select FW_LOADER
+	default m if ARCH_MXC
+
+	help
+	  Exposes APIs supported by the iMX Secure Enclave HW IP called:
+	  - EdgeLock Enclave Firmware (for i.MX8ULP, i.MX93),
+	    like base, HSM, V2X & SHE using the SAB protocol via the shared Messaging
+	    Unit. This driver exposes these interfaces via a set of file descriptors
+	    allowing to configure shared memory, send and receive messages.
diff --git a/drivers/firmware/imx/Makefile b/drivers/firmware/imx/Makefile
index 3bbaffa6e3478..4412b15846b14 100644
--- a/drivers/firmware/imx/Makefile
+++ b/drivers/firmware/imx/Makefile
@@ -4,3 +4,5 @@ obj-$(CONFIG_IMX_SCU)		+= imx-scu.o misc.o imx-scu-irq.o rm.o imx-scu-soc.o
 obj-${CONFIG_IMX_SCMI_CPU_DRV}	+= sm-cpu.o
 obj-${CONFIG_IMX_SCMI_MISC_DRV}	+= sm-misc.o
 obj-${CONFIG_IMX_SCMI_LMM_DRV}	+= sm-lmm.o
+sec_enclave-objs		= se_ctrl.o ele_common.o ele_base_msg.o
+obj-${CONFIG_IMX_SEC_ENCLAVE}	+= sec_enclave.o
diff --git a/drivers/firmware/imx/ele_base_msg.c b/drivers/firmware/imx/ele_base_msg.c
new file mode 100644
index 0000000000000..23f68076b85ce
--- /dev/null
+++ b/drivers/firmware/imx/ele_base_msg.c
@@ -0,0 +1,298 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2025 NXP
+ */
+
+#include <linux/types.h>
+
+#include <linux/cleanup.h>
+#include <linux/completion.h>
+#include <linux/dma-mapping.h>
+#include <linux/genalloc.h>
+
+#include "ele_base_msg.h"
+#include "ele_common.h"
+
+#define FW_DBG_DUMP_FIXED_STR		"ELE"
+
+int ele_uapi_allowed_base_cmd(struct se_if_priv *priv,
+			      struct se_msg_hdr *header)
+{
+	switch (header->command) {
+	case ELE_PING_REQ: return 0;
+	case ELE_DEBUG_DUMP_REQ: return 0;
+	case ELE_OEM_AUTH_CONTAINER_REQ: return 0;
+	case ELE_OEM_VERIFY_IMAGE_REQ: return 0;
+	case ELE_OEM_REL_CONTAINER_REQ: return 0;
+	case ELE_FW_LIFE_CYCLE_REQ: return 0;
+	case ELE_READ_FUSE_REQ: return 0;
+	case ELE_GET_FW_VERS_REQ: return 0;
+	case ELE_RETURN_LIFE_CYCLE_REQ: return 0;
+	case ELE_GET_EVENT_REQ: return 0;
+	case ELE_COMMIT_REQ: return 0;
+	case ELE_GEN_KEY_BLOB_REQ: return 0;
+	case ELE_GET_FW_STATUS_REQ: return 0;
+	case ELE_XIP_DECRYPT_REQ: return 0;
+	case ELE_WRITE_FUSE: return 0;
+	case ELE_GET_INFO_REQ: return 0;
+	case ELE_DEV_ATTEST_REQ: return 0;
+	case ELE_WRITE_SHADOW_FUSE_REQ: return 0;
+	case ELE_READ_SHADOW_FUSE_REQ: return 0;
+	default:
+		return -EACCES;
+	}
+}
+
+int ele_get_info(struct se_if_priv *priv, struct ele_dev_info *s_info)
+{
+	dma_addr_t get_info_addr = 0;
+	u32 *get_info_data = NULL;
+	int ret = 0;
+
+	if (!priv)
+		return -EINVAL;
+
+	memset(s_info, 0x0, sizeof(*s_info));
+
+	struct se_api_msg *tx_msg __free(kfree) =
+		kzalloc(ELE_GET_INFO_REQ_MSG_SZ, GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(ELE_GET_INFO_RSP_MSG_SZ, GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	if (priv->mem_pool)
+		get_info_data = gen_pool_dma_alloc(priv->mem_pool,
+						   ELE_GET_INFO_BUFF_SZ,
+						   &get_info_addr);
+	else
+		get_info_data = dma_alloc_coherent(priv->dev,
+						   ELE_GET_INFO_BUFF_SZ,
+						   &get_info_addr,
+						   GFP_KERNEL);
+	if (!get_info_data) {
+		dev_dbg(priv->dev,
+			"%s: Failed to allocate get_info_addr.", __func__);
+		return -ENOMEM;
+	}
+
+	ret = se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+				  ELE_GET_INFO_REQ, ELE_GET_INFO_REQ_MSG_SZ,
+				  true);
+	if (ret)
+		goto exit;
+
+	tx_msg->data[0] = upper_32_bits(get_info_addr);
+	tx_msg->data[1] = lower_32_bits(get_info_addr);
+	tx_msg->data[2] = sizeof(*s_info);
+	ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_GET_INFO_REQ_MSG_SZ,
+			       rx_msg, ELE_GET_INFO_RSP_MSG_SZ);
+	if (ret < 0)
+		goto exit;
+
+	ret = se_val_rsp_hdr_n_status(priv, rx_msg, ELE_GET_INFO_REQ,
+				      ELE_GET_INFO_RSP_MSG_SZ, true);
+
+	memcpy(s_info, get_info_data, sizeof(*s_info));
+exit:
+	if (priv->mem_pool)
+		gen_pool_free(priv->mem_pool, (unsigned long)get_info_data,
+			      ELE_GET_INFO_BUFF_SZ);
+	else
+		dma_free_coherent(priv->dev, ELE_GET_INFO_BUFF_SZ,
+				  get_info_data, get_info_addr);
+
+	return ret;
+}
+
+int ele_fetch_soc_info(struct se_if_priv *priv, void *data)
+{
+	return ele_get_info(priv, data);
+}
+
+int ele_ping(struct se_if_priv *priv)
+{
+	int ret = 0;
+
+	if (!priv)
+		return -EINVAL;
+
+	struct se_api_msg *tx_msg __free(kfree) = kzalloc(ELE_PING_REQ_SZ,
+							  GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree) = kzalloc(ELE_PING_RSP_SZ,
+							  GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	ret = se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+				  ELE_PING_REQ, ELE_PING_REQ_SZ, true);
+	if (ret) {
+		dev_err(priv->dev, "Error: se_fill_cmd_msg_hdr failed.");
+		return ret;
+	}
+
+	ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_PING_REQ_SZ,
+			       rx_msg, ELE_PING_RSP_SZ);
+	if (ret < 0)
+		return ret;
+
+	ret = se_val_rsp_hdr_n_status(priv, rx_msg, ELE_PING_REQ,
+				      ELE_PING_RSP_SZ, true);
+
+	return ret;
+}
+
+int ele_service_swap(struct se_if_priv *priv,
+		     phys_addr_t addr,
+		     u32 addr_size, u16 flag)
+{
+	int ret = 0;
+
+	if (!priv)
+		return -EINVAL;
+
+	struct se_api_msg *tx_msg __free(kfree)	=
+		kzalloc(ELE_SERVICE_SWAP_REQ_MSG_SZ, GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(ELE_SERVICE_SWAP_RSP_MSG_SZ, GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	ret = se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+				  ELE_SERVICE_SWAP_REQ,
+				  ELE_SERVICE_SWAP_REQ_MSG_SZ, true);
+	if (ret)
+		return ret;
+
+	tx_msg->data[0] = flag;
+	tx_msg->data[1] = addr_size;
+	tx_msg->data[2] = ELE_NONE_VAL;
+	tx_msg->data[3] = lower_32_bits(addr);
+	tx_msg->data[4] = se_get_msg_chksum((u32 *)&tx_msg[0],
+					    ELE_SERVICE_SWAP_REQ_MSG_SZ);
+	if (!tx_msg->data[4])
+		return -EINVAL;
+
+	ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_SERVICE_SWAP_REQ_MSG_SZ,
+			       rx_msg, ELE_SERVICE_SWAP_RSP_MSG_SZ);
+	if (ret < 0)
+		return ret;
+
+	ret = se_val_rsp_hdr_n_status(priv, rx_msg, ELE_SERVICE_SWAP_REQ,
+				      ELE_SERVICE_SWAP_RSP_MSG_SZ, true);
+	if (ret)
+		return ret;
+
+	if (flag == ELE_IMEM_EXPORT)
+		ret = rx_msg->data[1];
+	else
+		ret = 0;
+
+	return ret;
+}
+
+int ele_fw_authenticate(struct se_if_priv *priv, phys_addr_t contnr_addr,
+			phys_addr_t img_addr)
+{
+	int ret = 0;
+
+	if (!priv)
+		return -EINVAL;
+
+	struct se_api_msg *tx_msg __free(kfree)	=
+		kzalloc(ELE_FW_AUTH_REQ_SZ, GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(ELE_FW_AUTH_RSP_MSG_SZ, GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	ret = se_fill_cmd_msg_hdr(priv, (struct se_msg_hdr *)&tx_msg->header,
+				  ELE_FW_AUTH_REQ, ELE_FW_AUTH_REQ_SZ, true);
+	if (ret)
+		return ret;
+
+	tx_msg->data[0] = lower_32_bits(contnr_addr);
+	tx_msg->data[1] = upper_32_bits(contnr_addr);
+	tx_msg->data[2] = img_addr;
+
+	ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_FW_AUTH_REQ_SZ, rx_msg,
+			       ELE_FW_AUTH_RSP_MSG_SZ);
+	if (ret < 0)
+		return ret;
+
+	ret = se_val_rsp_hdr_n_status(priv, rx_msg, ELE_FW_AUTH_REQ,
+				      ELE_FW_AUTH_RSP_MSG_SZ, true);
+
+	return ret;
+}
+
+int ele_debug_dump(struct se_if_priv *priv)
+{
+	bool keep_logging;
+	int msg_ex_cnt;
+	int ret = 0;
+	int i;
+
+	if (!priv)
+		return -EINVAL;
+
+	struct se_api_msg *tx_msg __free(kfree) = kzalloc(ELE_DEBUG_DUMP_REQ_SZ,
+							  GFP_KERNEL);
+	if (!tx_msg)
+		return -ENOMEM;
+
+	struct se_api_msg *rx_msg __free(kfree)	= kzalloc(ELE_DEBUG_DUMP_RSP_SZ,
+							  GFP_KERNEL);
+	if (!rx_msg)
+		return -ENOMEM;
+
+	ret = se_fill_cmd_msg_hdr(priv, &tx_msg->header, ELE_DEBUG_DUMP_REQ,
+				  ELE_DEBUG_DUMP_REQ_SZ, true);
+	if (ret)
+		return ret;
+
+	msg_ex_cnt = 0;
+	do {
+		memset(rx_msg, 0x0, ELE_DEBUG_DUMP_RSP_SZ);
+
+		ret = ele_msg_send_rcv(priv->priv_dev_ctx, tx_msg, ELE_DEBUG_DUMP_REQ_SZ,
+				       rx_msg, ELE_DEBUG_DUMP_RSP_SZ);
+		if (ret < 0)
+			return ret;
+
+		ret = se_val_rsp_hdr_n_status(priv, rx_msg, ELE_DEBUG_DUMP_REQ,
+					      ELE_DEBUG_DUMP_RSP_SZ, true);
+		if (ret) {
+			dev_err(priv->dev, "Dump_Debug_Buffer Error: %x.", ret);
+			break;
+		}
+		keep_logging = (rx_msg->header.size >= (ELE_DEBUG_DUMP_RSP_SZ >> 2) &&
+				msg_ex_cnt < ELE_MAX_DBG_DMP_PKT);
+
+		rx_msg->header.size -= 2;
+
+		if (rx_msg->header.size > 4)
+			rx_msg->header.size--;
+
+		for (i = 0; i < rx_msg->header.size; i += 2)
+			dev_info(priv->dev, "%s%02x_%02x: 0x%08x 0x%08x",
+				 FW_DBG_DUMP_FIXED_STR,	msg_ex_cnt, i,
+				 rx_msg->data[i + 1], rx_msg->data[i + 2]);
+
+		msg_ex_cnt++;
+	} while (keep_logging);
+
+	return ret;
+}
diff --git a/drivers/firmware/imx/ele_base_msg.h b/drivers/firmware/imx/ele_base_msg.h
new file mode 100644
index 0000000000000..75e65e2791934
--- /dev/null
+++ b/drivers/firmware/imx/ele_base_msg.h
@@ -0,0 +1,117 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2025 NXP
+ *
+ * Header file for the EdgeLock Enclave Base API(s).
+ */
+
+#ifndef ELE_BASE_MSG_H
+#define ELE_BASE_MSG_H
+
+#include <linux/device.h>
+#include <linux/types.h>
+
+#include "se_ctrl.h"
+
+#define ELE_NONE_VAL			0x0
+
+#define ELE_OEM_AUTH_CONTAINER_REQ	0x87
+#define ELE_OEM_VERIFY_IMAGE_REQ	0x88
+#define ELE_OEM_REL_CONTAINER_REQ	0x89
+#define ELE_FW_LIFE_CYCLE_REQ		0x95
+#define ELE_READ_FUSE_REQ		0x97
+#define ELE_GET_FW_VERS_REQ		0x9d
+#define ELE_RETURN_LIFE_CYCLE_REQ	0xa0
+#define ELE_GET_EVENT_REQ		0xa2
+#define ELE_COMMIT_REQ			0xa8
+#define ELE_GEN_KEY_BLOB_REQ		0xaf
+#define ELE_GET_FW_STATUS_REQ		0xc5
+#define ELE_XIP_DECRYPT_REQ		0xc6
+#define ELE_WRITE_FUSE                  0xd6
+#define ELE_DEV_ATTEST_REQ              0xdb
+#define ELE_WRITE_SHADOW_FUSE_REQ       0xf2
+#define ELE_READ_SHADOW_FUSE_REQ        0xf3
+
+#define ELE_GET_INFO_REQ		0xda
+#define ELE_GET_INFO_REQ_MSG_SZ		0x10
+#define ELE_GET_INFO_RSP_MSG_SZ		0x08
+
+#define MAX_UID_SIZE                     (16)
+#define DEV_GETINFO_ROM_PATCH_SHA_SZ     (32)
+#define DEV_GETINFO_FW_SHA_SZ            (32)
+#define DEV_GETINFO_OEM_SRKH_SZ          (64)
+#define DEV_GETINFO_MIN_VER_MASK	0xff
+#define DEV_GETINFO_MAJ_VER_MASK	0xff00
+#define ELE_DEV_INFO_EXTRA_SZ		0x60
+
+struct dev_info {
+	u8  cmd;
+	u8  ver;
+	u16 length;
+	u16 soc_id;
+	u16 soc_rev;
+	u16 lmda_val;
+	u8  ssm_state;
+	u8  dev_atts_api_ver;
+	u8  uid[MAX_UID_SIZE];
+	u8  sha_rom_patch[DEV_GETINFO_ROM_PATCH_SHA_SZ];
+	u8  sha_fw[DEV_GETINFO_FW_SHA_SZ];
+};
+
+struct dev_addn_info {
+	u8  oem_srkh[DEV_GETINFO_OEM_SRKH_SZ];
+	u8  trng_state;
+	u8  csal_state;
+	u8  imem_state;
+	u8  reserved2;
+};
+
+struct ele_dev_info {
+	struct dev_info d_info;
+	struct dev_addn_info d_addn_info;
+};
+
+#define ELE_GET_INFO_BUFF_SZ		(sizeof(struct ele_dev_info) \
+						+ ELE_DEV_INFO_EXTRA_SZ)
+
+#define GET_SERIAL_NUM_FROM_UID(x, uid_word_sz) ({\
+	const u32 *__x = (const u32 *)(x); \
+	size_t __sz = (uid_word_sz); \
+	((u64)__x[__sz - 1] << 32) | __x[0]; \
+	})
+
+#define ELE_MAX_DBG_DMP_PKT		50
+#define ELE_DEBUG_DUMP_REQ		0x21
+#define ELE_DEBUG_DUMP_REQ_SZ		0x4
+#define ELE_DEBUG_DUMP_RSP_SZ		0x5c
+
+#define ELE_PING_REQ			0x01
+#define ELE_PING_REQ_SZ			0x04
+#define ELE_PING_RSP_SZ			0x08
+
+#define ELE_SERVICE_SWAP_REQ		0xdf
+#define ELE_SERVICE_SWAP_REQ_MSG_SZ	0x18
+#define ELE_SERVICE_SWAP_RSP_MSG_SZ	0x0c
+#define ELE_IMEM_SIZE			0x10000
+#define ELE_IMEM_STATE_OK		0xca
+#define ELE_IMEM_STATE_BAD		0xfe
+#define ELE_IMEM_STATE_WORD		0x27
+#define ELE_IMEM_STATE_MASK		0x00ff0000
+#define ELE_IMEM_EXPORT			0x1
+#define ELE_IMEM_IMPORT			0x2
+
+#define ELE_FW_AUTH_REQ			0x02
+#define ELE_FW_AUTH_REQ_SZ		0x10
+#define ELE_FW_AUTH_RSP_MSG_SZ		0x08
+
+int ele_get_info(struct se_if_priv *priv, struct ele_dev_info *s_info);
+int ele_fetch_soc_info(struct se_if_priv *priv, void *data);
+int ele_ping(struct se_if_priv *priv);
+int ele_service_swap(struct se_if_priv *priv, phys_addr_t addr,
+		     u32 addr_size, u16 flag);
+int ele_fw_authenticate(struct se_if_priv *priv, phys_addr_t contnr_addr,
+			phys_addr_t img_addr);
+int ele_debug_dump(struct se_if_priv *priv);
+int ele_uapi_allowed_base_cmd(struct se_if_priv *priv,
+			      struct se_msg_hdr *header);
+#endif
diff --git a/drivers/firmware/imx/ele_common.c b/drivers/firmware/imx/ele_common.c
new file mode 100644
index 0000000000000..e5117a1f19959
--- /dev/null
+++ b/drivers/firmware/imx/ele_common.c
@@ -0,0 +1,373 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2025 NXP
+ */
+
+#include "ele_base_msg.h"
+#include "ele_common.h"
+
+int se_chk_tx_msg_hdr(struct se_if_priv *priv, struct se_msg_hdr *header)
+{
+	if (!header->size || header->size > MAX_WORD_SIZE)
+		return -EINVAL;
+
+	if (header->tag != priv->if_defs->cmd_tag &&
+	    header->tag != priv->if_defs->rsp_tag)
+		return -EINVAL;
+
+	if (header->ver == priv->if_defs->base_api_ver)
+		return ele_uapi_allowed_base_cmd(priv, header);
+	else if (header->ver == priv->if_defs->fw_api_ver)
+		return 0;
+
+	return -EINVAL;
+}
+
+/*
+ * se_get_msg_chksum() - to calculate checksum word by word.
+ *
+ * @msg : reference to the input msg-data.
+ * @msg_len : reference to the input msg-data length in bytes.
+ *            Includes extra 4 bytes (or 1 words) chksum.
+ *
+ * This function returns the checksum calculated by ORing word by word.
+ *
+ * Return:
+ *  0: if the input length is not 4 byte aligned, or num of words < 5.
+ *  chksum: calculated word by word.
+ */
+u32 se_get_msg_chksum(u32 *msg, u32 msg_len)
+{
+	u32 nb_words = msg_len / (u32)sizeof(u32);
+	u32 chksum = 0;
+	u32 i;
+
+	if (nb_words < 5)
+		return chksum;
+
+	if (msg_len % SE_MSG_WORD_SZ) {
+		pr_err("Msg-len is not 4-byte aligned.");
+		return chksum;
+	}
+
+	/* nb_words include one checksum word, so skip it. */
+	nb_words--;
+
+	for (i = 0; i < nb_words; i++)
+		chksum ^= *(msg + i);
+
+	return chksum;
+}
+
+void set_se_rcv_msg_timeout(struct se_if_priv *priv, u32 timeout_ms)
+{
+	priv->se_rcv_msg_timeout_ms = timeout_ms;
+}
+
+int ele_msg_rcv(struct se_if_device_ctx *dev_ctx, struct se_clbk_handle *se_clbk_hdl)
+{
+	unsigned long timeout_ms;
+	int ret;
+
+	do {
+		timeout_ms = MAX_SCHEDULE_TIMEOUT;
+		if (dev_ctx->priv->cmd_receiver_clbk_hdl.dev_ctx != dev_ctx)
+			timeout_ms = msecs_to_jiffies(dev_ctx->priv->se_rcv_msg_timeout_ms);
+
+		ret = wait_for_completion_interruptible_timeout(&se_clbk_hdl->done, timeout_ms);
+		if (ret == -ERESTARTSYS) {
+			if (dev_ctx->priv->waiting_rsp_clbk_hdl.dev_ctx) {
+				dev_ctx->priv->waiting_rsp_clbk_hdl.signal_rcvd = true;
+				continue;
+			}
+			ret = -EINTR;
+			break;
+		}
+		if (ret == 0) {
+			ret = -ETIMEDOUT;
+			dev_err(dev_ctx->priv->dev,
+				"Fatal Error: SE interface: %s0, hangs indefinitely.\n",
+				get_se_if_name(dev_ctx->priv->if_defs->se_if_type));
+			break;
+		}
+		ret = se_clbk_hdl->rx_msg_sz;
+		break;
+	} while (ret < 0);
+
+	return ret;
+}
+
+int ele_msg_send(struct se_if_device_ctx *dev_ctx,
+		 void *tx_msg,
+		 int tx_msg_sz)
+{
+	struct se_msg_hdr *header = tx_msg;
+	int err;
+
+	/*
+	 * Check that the size passed as argument matches the size
+	 * carried in the message.
+	 */
+	if (header->size << 2 != tx_msg_sz) {
+		dev_err(dev_ctx->priv->dev,
+			"%s: User buf hdr: 0x%x, sz mismatched with input-sz (%d != %d).",
+			dev_ctx->devname, *(u32 *)header, header->size << 2, tx_msg_sz);
+		return -EINVAL;
+	}
+
+	err = mbox_send_message(dev_ctx->priv->tx_chan, tx_msg);
+	if (err < 0) {
+		dev_err(dev_ctx->priv->dev,
+			"%s: Error: mbox_send_message failure.", dev_ctx->devname);
+		return err;
+	}
+
+	return tx_msg_sz;
+}
+
+/* API used for send/receive blocking call. */
+int ele_msg_send_rcv(struct se_if_device_ctx *dev_ctx, void *tx_msg,
+		     int tx_msg_sz, void *rx_msg, int exp_rx_msg_sz)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	int err;
+
+	guard(mutex)(&priv->se_if_cmd_lock);
+
+	priv->waiting_rsp_clbk_hdl.dev_ctx = dev_ctx;
+	priv->waiting_rsp_clbk_hdl.rx_msg_sz = exp_rx_msg_sz;
+	priv->waiting_rsp_clbk_hdl.rx_msg = rx_msg;
+
+	err = ele_msg_send(dev_ctx, tx_msg, tx_msg_sz);
+	if (err < 0)
+		return err;
+
+	err = ele_msg_rcv(dev_ctx, &priv->waiting_rsp_clbk_hdl);
+
+	if (priv->waiting_rsp_clbk_hdl.signal_rcvd) {
+		err = -EINTR;
+		priv->waiting_rsp_clbk_hdl.signal_rcvd = false;
+		dev_err(priv->dev, "%s: Err[0x%x]:Interrupted by signal.",
+			dev_ctx->devname, err);
+	}
+	priv->waiting_rsp_clbk_hdl.dev_ctx = NULL;
+
+	return err;
+}
+
+static bool check_hdr_exception_for_sz(struct se_if_priv *priv,
+				       struct se_msg_hdr *header)
+{
+	/*
+	 * List of API(s) header that can be accepte variable length
+	 * response buffer.
+	 */
+	if (header->command == ELE_DEBUG_DUMP_REQ &&
+	    header->ver == priv->if_defs->base_api_ver &&
+	    header->size >= 0 && header->size <= ELE_DEBUG_DUMP_RSP_SZ)
+		return true;
+
+	return false;
+}
+
+/*
+ * Callback called by mailbox FW, when data is received.
+ */
+void se_if_rx_callback(struct mbox_client *mbox_cl, void *msg)
+{
+	struct se_clbk_handle *se_clbk_hdl;
+	struct device *dev = mbox_cl->dev;
+	struct se_msg_hdr *header;
+	struct se_if_priv *priv;
+	u32 rx_msg_sz;
+
+	priv = dev_get_drvdata(dev);
+
+	/* The function can be called with NULL msg */
+	if (!msg) {
+		dev_err(dev, "Message is invalid\n");
+		return;
+	}
+
+	header = msg;
+	rx_msg_sz = header->size << 2;
+
+	/* Incoming command: wake up the receiver if any. */
+	if (header->tag == priv->if_defs->cmd_tag) {
+		se_clbk_hdl = &priv->cmd_receiver_clbk_hdl;
+		dev_dbg(dev, "Selecting cmd receiver:%s for mesg header:0x%x.",
+			se_clbk_hdl->dev_ctx->devname,  *(u32 *)header);
+
+		/*
+		 * Pre-allocated buffer of MAX_NVM_MSG_LEN
+		 * as the NVM command are initiated by FW.
+		 * Size is revealed as part of this call function.
+		 */
+		if (rx_msg_sz > MAX_NVM_MSG_LEN) {
+			dev_err(dev,
+				"%s: CMD-RCVER NVM: hdr(0x%x) with different sz(%d != %d).\n",
+				se_clbk_hdl->dev_ctx->devname, *(u32 *)header,
+				rx_msg_sz, se_clbk_hdl->rx_msg_sz);
+
+			se_clbk_hdl->rx_msg_sz = MAX_NVM_MSG_LEN;
+		}
+		se_clbk_hdl->rx_msg_sz = rx_msg_sz;
+
+	} else if (header->tag == priv->if_defs->rsp_tag) {
+		se_clbk_hdl = &priv->waiting_rsp_clbk_hdl;
+		dev_dbg(dev, "Selecting resp waiter:%s for mesg header:0x%x.",
+			se_clbk_hdl->dev_ctx->devname, *(u32 *)header);
+
+		if (rx_msg_sz != se_clbk_hdl->rx_msg_sz &&
+		    check_hdr_exception_for_sz(priv, header)) {
+			dev_err(dev,
+				"%s: Rsp to CMD: hdr(0x%x) with different sz(%d != %d).\n",
+				se_clbk_hdl->dev_ctx->devname, *(u32 *)header,
+				rx_msg_sz, se_clbk_hdl->rx_msg_sz);
+
+			se_clbk_hdl->rx_msg_sz = min(rx_msg_sz, se_clbk_hdl->rx_msg_sz);
+		}
+	} else {
+		dev_err(dev, "Failed to select a device for message: %.8x\n",
+			*((u32 *)header));
+		return;
+	}
+
+	memcpy(se_clbk_hdl->rx_msg, msg, se_clbk_hdl->rx_msg_sz);
+
+	/* Allow user to read */
+	complete(&se_clbk_hdl->done);
+}
+
+int se_val_rsp_hdr_n_status(struct se_if_priv *priv, struct se_api_msg *msg,
+			    u8 msg_id, u8 sz, bool is_base_api)
+{
+	struct se_msg_hdr *header = &msg->header;
+	u32 status;
+
+	if (header->tag != priv->if_defs->rsp_tag) {
+		dev_err(priv->dev, "MSG[0x%x] Hdr: Resp tag mismatch. (0x%x != 0x%x)",
+			msg_id, header->tag, priv->if_defs->rsp_tag);
+		return -EINVAL;
+	}
+
+	if (header->command != msg_id) {
+		dev_err(priv->dev, "MSG Header: Cmd id mismatch. (0x%x != 0x%x)",
+			header->command, msg_id);
+		return -EINVAL;
+	}
+
+	if ((sz % 4) || (header->size != (sz >> 2) &&
+			 !check_hdr_exception_for_sz(priv, header))) {
+		dev_err(priv->dev, "MSG[0x%x] Hdr: Cmd size mismatch. (0x%x != 0x%x)",
+			msg_id, header->size, (sz >> 2));
+		return -EINVAL;
+	}
+
+	if (is_base_api && header->ver != priv->if_defs->base_api_ver) {
+		dev_err(priv->dev,
+			"MSG[0x%x] Hdr: Base API Vers mismatch. (0x%x != 0x%x)",
+			msg_id, header->ver, priv->if_defs->base_api_ver);
+		return -EINVAL;
+	} else if (!is_base_api && header->ver != priv->if_defs->fw_api_ver) {
+		dev_err(priv->dev,
+			"MSG[0x%x] Hdr: FW API Vers mismatch. (0x%x != 0x%x)",
+			msg_id, header->ver, priv->if_defs->fw_api_ver);
+		return -EINVAL;
+	}
+
+	status = RES_STATUS(msg->data[0]);
+	if (status != priv->if_defs->success_tag) {
+		dev_err(priv->dev, "Command Id[%x], Response Failure = 0x%x",
+			header->command, status);
+		return -EPERM;
+	}
+
+	return 0;
+}
+
+int se_save_imem_state(struct se_if_priv *priv, struct se_imem_buf *imem)
+{
+	struct ele_dev_info s_info = {0};
+	int ret;
+
+	ret = ele_get_info(priv, &s_info);
+	if (ret) {
+		dev_err(priv->dev, "Failed to get info from ELE.\n");
+		return ret;
+	}
+
+	/* Check for the imem-state before continue to save imem state. */
+	if (s_info.d_addn_info.imem_state == ELE_IMEM_STATE_BAD)
+		return -EIO;
+
+	/*
+	 * EXPORT command will save encrypted IMEM to given address,
+	 * so later in resume, IMEM can be restored from the given
+	 * address.
+	 *
+	 * Size must be at least 64 kB.
+	 */
+	ret = ele_service_swap(priv, imem->phyaddr, ELE_IMEM_SIZE, ELE_IMEM_EXPORT);
+	if (ret < 0) {
+		dev_err(priv->dev, "Failed to export IMEM.");
+		imem->size = 0;
+	} else {
+		dev_dbg(priv->dev,
+			"Exported %d bytes of encrypted IMEM.",
+			ret);
+		imem->size = ret;
+	}
+
+	return ret > 0 ? 0 : ret;
+}
+
+int se_restore_imem_state(struct se_if_priv *priv, struct se_imem_buf *imem)
+{
+	struct ele_dev_info s_info;
+	int ret;
+
+	/* get info from ELE */
+	ret = ele_get_info(priv, &s_info);
+	if (ret) {
+		dev_err(priv->dev, "Failed to get info from ELE.");
+		return ret;
+	}
+	imem->state = s_info.d_addn_info.imem_state;
+
+	/* Check for the imem-state and imem-size before continue to
+	 * restore imem state.
+	 */
+	if (s_info.d_addn_info.imem_state != ELE_IMEM_STATE_BAD || !imem->size)
+		return -EIO;
+
+	/*
+	 * IMPORT command will restore IMEM from the given
+	 * address, here size is the actual size returned by ELE
+	 * during the export operation
+	 */
+	ret = ele_service_swap(priv, imem->phyaddr, imem->size, ELE_IMEM_IMPORT);
+	if (ret) {
+		dev_err(priv->dev, "Failed to import IMEM");
+		return ret;
+	}
+
+	/*
+	 * After importing IMEM, check if IMEM state is equal to 0xCA
+	 * to ensure IMEM is fully loaded and
+	 * ELE functionality can be used.
+	 */
+	ret = ele_get_info(priv, &s_info);
+	if (ret) {
+		dev_err(priv->dev, "Failed to get info from ELE.");
+		return ret;
+	}
+	imem->state = s_info.d_addn_info.imem_state;
+
+	if (s_info.d_addn_info.imem_state == ELE_IMEM_STATE_OK)
+		dev_dbg(priv->dev, "Successfully restored IMEM.");
+	else
+		dev_err(priv->dev, "Failed to restore IMEM.");
+
+	return ret;
+}
diff --git a/drivers/firmware/imx/ele_common.h b/drivers/firmware/imx/ele_common.h
new file mode 100644
index 0000000000000..bdc13d1b715d5
--- /dev/null
+++ b/drivers/firmware/imx/ele_common.h
@@ -0,0 +1,51 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2025 NXP
+ */
+
+#ifndef __ELE_COMMON_H__
+#define __ELE_COMMON_H__
+
+#include "se_ctrl.h"
+
+#define ELE_SUCCESS_IND			0xD6
+
+#define IMX_ELE_FW_DIR                 "imx/ele/"
+
+#define MAX_WORD_SIZE			0x20
+#define SE_RCV_MSG_DEFAULT_TIMEOUT	5000
+#define SE_RCV_MSG_LONG_TIMEOUT		5000000
+
+void set_se_rcv_msg_timeout(struct se_if_priv *priv, u32 val);
+u32 se_get_msg_chksum(u32 *msg, u32 msg_len);
+
+int ele_msg_rcv(struct se_if_device_ctx *dev_ctx, struct se_clbk_handle *se_clbk_hdl);
+
+int ele_msg_send(struct se_if_device_ctx *dev_ctx, void *tx_msg, int tx_msg_sz);
+
+int ele_msg_send_rcv(struct se_if_device_ctx *dev_ctx, void *tx_msg,
+		     int tx_msg_sz, void *rx_msg, int exp_rx_msg_sz);
+
+void se_if_rx_callback(struct mbox_client *mbox_cl, void *msg);
+
+int se_val_rsp_hdr_n_status(struct se_if_priv *priv, struct se_api_msg *msg,
+			    u8 msg_id, u8 sz, bool is_base_api);
+
+/* Fill a command message header with a given command ID and length in bytes. */
+static inline int se_fill_cmd_msg_hdr(struct se_if_priv *priv, struct se_msg_hdr *hdr,
+				      u8 cmd, u32 len, bool is_base_api)
+{
+	hdr->tag = priv->if_defs->cmd_tag;
+	hdr->ver = (is_base_api) ? priv->if_defs->base_api_ver : priv->if_defs->fw_api_ver;
+	hdr->command = cmd;
+	hdr->size = len >> 2;
+
+	return 0;
+}
+
+int se_save_imem_state(struct se_if_priv *priv, struct se_imem_buf *imem);
+
+int se_restore_imem_state(struct se_if_priv *priv, struct se_imem_buf *imem);
+
+int se_chk_tx_msg_hdr(struct se_if_priv *priv, struct se_msg_hdr *header);
+#endif /*__ELE_COMMON_H__ */
diff --git a/drivers/firmware/imx/se_ctrl.c b/drivers/firmware/imx/se_ctrl.c
new file mode 100644
index 0000000000000..9327d47e4312e
--- /dev/null
+++ b/drivers/firmware/imx/se_ctrl.c
@@ -0,0 +1,1193 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright 2026 NXP
+ */
+
+#include <linux/bitfield.h>
+#include <linux/cleanup.h>
+#include <linux/completion.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/dma-direct.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/export.h>
+#include <linux/firmware.h>
+#include <linux/firmware/imx/se_api.h>
+#include <linux/genalloc.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/miscdevice.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/sys_soc.h>
+#include <uapi/linux/se_ioctl.h>
+
+#include "ele_base_msg.h"
+#include "ele_common.h"
+#include "se_ctrl.h"
+
+#define MAX_SOC_INFO_DATA_SZ		256
+#define MBOX_TX_NAME			"tx"
+#define MBOX_RX_NAME			"rx"
+
+struct se_fw_img_name {
+	const u8 *prim_fw_nm_in_rfs;
+	const u8 *seco_fw_nm_in_rfs;
+};
+
+struct se_fw_load_info {
+	const struct se_fw_img_name *se_fw_img_nm;
+	bool is_fw_tobe_loaded;
+	bool imem_mgmt;
+	struct se_imem_buf imem;
+};
+
+struct se_var_info {
+	u16 soc_rev;
+	struct se_fw_load_info load_fw;
+};
+
+/* contains fixed information */
+struct se_soc_info {
+	const u16 soc_id;
+	const bool soc_register;
+	const struct se_fw_img_name se_fw_img_nm;
+};
+
+struct se_if_node {
+	struct se_soc_info *se_info;
+	u8 *pool_name;
+	bool reserved_dma_ranges;
+	struct se_if_defines if_defs;
+};
+
+/* common for all the SoC. */
+static struct se_var_info var_se_info;
+
+static struct se_soc_info se_imx8ulp_info = {
+	.soc_id = SOC_ID_OF_IMX8ULP,
+	.soc_register = true,
+	.se_fw_img_nm = {
+		.prim_fw_nm_in_rfs = IMX_ELE_FW_DIR
+			"mx8ulpa2-ahab-container.img",
+		.seco_fw_nm_in_rfs = IMX_ELE_FW_DIR
+			"mx8ulpa2ext-ahab-container.img",
+	},
+};
+
+static struct se_if_node imx8ulp_se_ele_hsm = {
+	.se_info = &se_imx8ulp_info,
+	.pool_name = "sram",
+	.reserved_dma_ranges = true,
+	.if_defs = {
+		.se_if_type = SE_TYPE_ID_HSM,
+		.cmd_tag = 0x17,
+		.rsp_tag = 0xe1,
+		.success_tag = ELE_SUCCESS_IND,
+		.base_api_ver = MESSAGING_VERSION_6,
+		.fw_api_ver = MESSAGING_VERSION_7,
+	},
+};
+
+static struct se_soc_info se_imx93_info = {
+	.soc_id = SOC_ID_OF_IMX93,
+};
+
+static struct se_if_node imx93_se_ele_hsm = {
+	.se_info = &se_imx93_info,
+	.reserved_dma_ranges = true,
+	.if_defs = {
+		.se_if_type = SE_TYPE_ID_HSM,
+		.cmd_tag = 0x17,
+		.rsp_tag = 0xe1,
+		.success_tag = ELE_SUCCESS_IND,
+		.base_api_ver = MESSAGING_VERSION_6,
+		.fw_api_ver = MESSAGING_VERSION_7,
+	},
+};
+
+static const struct of_device_id se_match[] = {
+	{ .compatible = "fsl,imx8ulp-se-ele-hsm", .data = &imx8ulp_se_ele_hsm},
+	{ .compatible = "fsl,imx93-se-ele-hsm", .data = &imx93_se_ele_hsm},
+	{},
+};
+
+char *get_se_if_name(u8 se_if_id)
+{
+	switch (se_if_id) {
+	case SE_TYPE_ID_DBG: return SE_TYPE_STR_DBG;
+	case SE_TYPE_ID_HSM: return SE_TYPE_STR_HSM;
+	}
+
+	return NULL;
+}
+
+static u32 get_se_soc_id(struct se_if_priv *priv)
+{
+	const struct se_soc_info *se_info = device_get_match_data(priv->dev);
+
+	return se_info->soc_id;
+}
+
+static struct se_fw_load_info *get_load_fw_instance(struct se_if_priv *priv)
+{
+	return &var_se_info.load_fw;
+}
+
+static int get_se_soc_info(struct se_if_priv *priv, const struct se_soc_info *se_info)
+{
+	struct se_fw_load_info *load_fw = get_load_fw_instance(priv);
+	struct soc_device_attribute *attr;
+	u8 data[MAX_SOC_INFO_DATA_SZ];
+	struct ele_dev_info *s_info;
+	struct soc_device *sdev;
+	int err = 0;
+
+	/*
+	 * This function should be called once.
+	 * Check if the se_soc_rev is zero to continue.
+	 */
+	if (var_se_info.soc_rev)
+		return err;
+
+	err = ele_fetch_soc_info(priv, &data);
+	if (err < 0)
+		return dev_err_probe(priv->dev, err, "Failed to fetch SoC Info.");
+	s_info = (void *)data;
+	var_se_info.soc_rev = s_info->d_info.soc_rev;
+	load_fw->imem.state = s_info->d_addn_info.imem_state;
+
+	if (!se_info->soc_register)
+		return 0;
+
+	attr = devm_kzalloc(priv->dev, sizeof(*attr), GFP_KERNEL);
+	if (!attr)
+		return -ENOMEM;
+
+	if (FIELD_GET(DEV_GETINFO_MIN_VER_MASK, var_se_info.soc_rev))
+		attr->revision = devm_kasprintf(priv->dev, GFP_KERNEL, "%x.%x",
+						FIELD_GET(DEV_GETINFO_MIN_VER_MASK,
+							  var_se_info.soc_rev),
+						FIELD_GET(DEV_GETINFO_MAJ_VER_MASK,
+							  var_se_info.soc_rev));
+	else
+		attr->revision = devm_kasprintf(priv->dev, GFP_KERNEL, "%x",
+						FIELD_GET(DEV_GETINFO_MAJ_VER_MASK,
+							  var_se_info.soc_rev));
+
+	switch (se_info->soc_id) {
+	case SOC_ID_OF_IMX8ULP:
+		attr->soc_id = "i.MX8ULP";
+		break;
+	case SOC_ID_OF_IMX93:
+		attr->soc_id = "i.MX93";
+		break;
+	}
+
+	err = of_property_read_string(of_root, "model", &attr->machine);
+	if (err)
+		return -EINVAL;
+
+	attr->family = "Freescale i.MX";
+
+	attr->serial_number = devm_kasprintf(priv->dev,
+					     GFP_KERNEL, "%016llX",
+					     GET_SERIAL_NUM_FROM_UID(s_info->d_info.uid,
+								     MAX_UID_SIZE >> 2));
+
+	sdev = soc_device_register(attr);
+	if (IS_ERR(sdev))
+		return PTR_ERR(sdev);
+
+	return 0;
+}
+
+static int load_firmware(struct se_if_priv *priv, const u8 *se_img_file_to_load)
+{
+	const struct firmware *fw = NULL;
+	dma_addr_t se_fw_dma_addr;
+	phys_addr_t se_fw_phyaddr;
+	u8 *se_fw_buf;
+	int ret;
+
+	if (!se_img_file_to_load) {
+		dev_err(priv->dev, "FW image is not provided.");
+		return -EINVAL;
+	}
+	ret = request_firmware(&fw, se_img_file_to_load, priv->dev);
+	if (ret)
+		return ret;
+
+	dev_info(priv->dev, "loading firmware %s.", se_img_file_to_load);
+
+	/* allocate buffer to store the SE FW */
+	se_fw_buf = dma_alloc_coherent(priv->dev, fw->size, &se_fw_dma_addr, GFP_KERNEL);
+	if (!se_fw_buf) {
+		ret = -ENOMEM;
+		goto exit;
+	}
+
+	memcpy(se_fw_buf, fw->data, fw->size);
+	se_fw_phyaddr = dma_to_phys(priv->dev, se_fw_dma_addr);
+	ret = ele_fw_authenticate(priv, se_fw_phyaddr, se_fw_phyaddr);
+	if (ret < 0) {
+		dev_err(priv->dev,
+			"Error %pe: Authenticate & load SE firmware %s.",
+			ERR_PTR(ret), se_img_file_to_load);
+		ret = -EPERM;
+	}
+	dma_free_coherent(priv->dev, fw->size, se_fw_buf, se_fw_dma_addr);
+exit:
+	release_firmware(fw);
+
+	return ret;
+}
+
+static int se_load_firmware(struct se_if_priv *priv)
+{
+	struct se_fw_load_info *load_fw = get_load_fw_instance(priv);
+	int ret = 0;
+
+	if (!load_fw->is_fw_tobe_loaded)
+		return 0;
+
+	if (load_fw->imem.state == ELE_IMEM_STATE_BAD) {
+		ret = load_firmware(priv, load_fw->se_fw_img_nm->prim_fw_nm_in_rfs);
+		if (ret) {
+			dev_err(priv->dev, "Failed to load boot firmware.");
+			return -EPERM;
+		}
+	}
+
+	ret = load_firmware(priv, load_fw->se_fw_img_nm->seco_fw_nm_in_rfs);
+	if (ret) {
+		dev_err(priv->dev, "Failed to load runtime firmware.");
+		return -EPERM;
+	}
+
+	load_fw->is_fw_tobe_loaded = false;
+
+	return ret;
+}
+
+static int init_se_shared_mem(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	INIT_LIST_HEAD(&se_shared_mem_mgmt->pending_out);
+	INIT_LIST_HEAD(&se_shared_mem_mgmt->pending_in);
+
+	/*
+	 * Allocate some memory for data exchanges with S40x.
+	 * This will be used for data not requiring secure memory.
+	 */
+	se_shared_mem_mgmt->non_secure_mem.ptr =
+			dma_alloc_coherent(priv->dev, MAX_DATA_SIZE_PER_USER,
+					   &se_shared_mem_mgmt->non_secure_mem.dma_addr,
+					   GFP_KERNEL);
+	if (!se_shared_mem_mgmt->non_secure_mem.ptr)
+		return -ENOMEM;
+
+	se_shared_mem_mgmt->non_secure_mem.size = MAX_DATA_SIZE_PER_USER;
+	se_shared_mem_mgmt->non_secure_mem.pos = 0;
+
+	return 0;
+}
+
+static void cleanup_se_shared_mem(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	/* Free non-secure shared buffer. */
+	dma_free_coherent(priv->dev, MAX_DATA_SIZE_PER_USER,
+			  se_shared_mem_mgmt->non_secure_mem.ptr,
+			  se_shared_mem_mgmt->non_secure_mem.dma_addr);
+
+	se_shared_mem_mgmt->non_secure_mem.ptr = NULL;
+	se_shared_mem_mgmt->non_secure_mem.dma_addr = 0;
+	se_shared_mem_mgmt->non_secure_mem.size = 0;
+	se_shared_mem_mgmt->non_secure_mem.pos = 0;
+}
+
+/* Need to copy the output data to user-device context.
+ */
+static int se_dev_ctx_cpy_out_data(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_buf_desc *b_desc, *temp;
+	bool do_cpy = true;
+
+	list_for_each_entry_safe(b_desc, temp, &se_shared_mem_mgmt->pending_out, link) {
+		if (b_desc->usr_buf_ptr && b_desc->shared_buf_ptr && do_cpy) {
+			dev_dbg(priv->dev, "Copying output data to user.");
+			if (do_cpy && copy_to_user(b_desc->usr_buf_ptr,
+						   b_desc->shared_buf_ptr,
+						   b_desc->size)) {
+				dev_err(priv->dev, "Failure copying output data to user.");
+				do_cpy = false;
+			}
+		}
+
+		if (b_desc->shared_buf_ptr)
+			memset(b_desc->shared_buf_ptr, 0, b_desc->size);
+
+		list_del(&b_desc->link);
+		kfree(b_desc);
+	}
+
+	return do_cpy ? 0 : -EFAULT;
+}
+
+/*
+ * Clean the used Shared Memory space,
+ * whether its Input Data copied from user buffers, or
+ * Data received from FW.
+ */
+static void se_dev_ctx_shared_mem_cleanup(struct se_if_device_ctx *dev_ctx)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct list_head *pending_lists[] = {&se_shared_mem_mgmt->pending_in,
+						&se_shared_mem_mgmt->pending_out};
+	struct se_buf_desc *b_desc, *temp;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(pending_lists); i++) {
+		list_for_each_entry_safe(b_desc, temp, pending_lists[i], link) {
+			if (b_desc->shared_buf_ptr)
+				memset(b_desc->shared_buf_ptr, 0, b_desc->size);
+
+			list_del(&b_desc->link);
+			kfree(b_desc);
+		}
+	}
+	se_shared_mem_mgmt->non_secure_mem.pos = 0;
+}
+
+static int add_b_desc_to_pending_list(void *shared_ptr_with_pos,
+				      struct se_ioctl_setup_iobuf *io,
+				      struct se_if_device_ctx *dev_ctx)
+{
+	struct se_shared_mem_mgmt_info *se_shared_mem_mgmt = &dev_ctx->se_shared_mem_mgmt;
+	struct se_buf_desc *b_desc;
+
+	b_desc = kzalloc_obj(*b_desc, GFP_KERNEL);
+	if (!b_desc)
+		return -ENOMEM;
+
+	b_desc->shared_buf_ptr = shared_ptr_with_pos;
+	b_desc->usr_buf_ptr = io->user_buf;
+	b_desc->size = io->length;
+
+	if (io->flags & SE_IO_BUF_FLAGS_IS_INPUT) {
+		/*
+		 * buffer is input:
+		 * add an entry in the "pending input buffers" list so
+		 * that copied data can be cleaned from shared memory
+		 * later.
+		 */
+		list_add_tail(&b_desc->link, &se_shared_mem_mgmt->pending_in);
+	} else {
+		/*
+		 * buffer is output:
+		 * add an entry in the "pending out buffers" list so data
+		 * can be copied to user space when receiving Secure-Enclave
+		 * response.
+		 */
+		list_add_tail(&b_desc->link, &se_shared_mem_mgmt->pending_out);
+	}
+
+	return 0;
+}
+
+/* interface for managed res to unregister a character device */
+static void if_misc_deregister(void *miscdevice)
+{
+	misc_deregister(miscdevice);
+}
+
+static int init_misc_device_context(struct se_if_priv *priv, int ch_id,
+				    struct se_if_device_ctx **new_dev_ctx,
+				    const struct file_operations *se_if_fops)
+{
+	struct se_if_device_ctx *dev_ctx;
+	int ret = 0;
+
+	dev_ctx = devm_kzalloc(priv->dev, sizeof(*dev_ctx), GFP_KERNEL);
+
+	if (!dev_ctx)
+		return -ENOMEM;
+
+	dev_ctx->devname = devm_kasprintf(priv->dev, GFP_KERNEL, "%s0_ch%d",
+					  get_se_if_name(priv->if_defs->se_if_type),
+					  ch_id);
+	if (!dev_ctx->devname)
+		return -ENOMEM;
+
+	mutex_init(&dev_ctx->fops_lock);
+
+	dev_ctx->priv = priv;
+	*new_dev_ctx = dev_ctx;
+
+	dev_ctx->miscdev = devm_kzalloc(priv->dev, sizeof(*dev_ctx->miscdev), GFP_KERNEL);
+	if (!dev_ctx->miscdev) {
+		*new_dev_ctx = NULL;
+		return -ENOMEM;
+	}
+
+	dev_ctx->miscdev->name = dev_ctx->devname;
+	dev_ctx->miscdev->minor = MISC_DYNAMIC_MINOR;
+	dev_ctx->miscdev->fops = se_if_fops;
+	dev_ctx->miscdev->parent = priv->dev;
+	ret = misc_register(dev_ctx->miscdev);
+	if (ret)
+		return dev_err_probe(priv->dev, ret, "Failed to register misc device.");
+
+	ret = devm_add_action_or_reset(priv->dev, if_misc_deregister, dev_ctx->miscdev);
+	if (ret)
+		return dev_err_probe(priv->dev, ret,
+				     "Failed to add action to the misc-dev.");
+	return ret;
+}
+
+static int init_device_context(struct se_if_priv *priv, int ch_id,
+			       struct se_if_device_ctx **new_dev_ctx)
+{
+	struct se_if_device_ctx *dev_ctx;
+	int ret = 0;
+
+	dev_ctx = kzalloc_obj(*dev_ctx, GFP_KERNEL);
+
+	if (!dev_ctx)
+		return -ENOMEM;
+
+	dev_ctx->devname = kasprintf(GFP_KERNEL, "%s0_ch%d",
+				     get_se_if_name(priv->if_defs->se_if_type),
+				     ch_id);
+	if (!dev_ctx->devname) {
+		kfree(dev_ctx);
+		return -ENOMEM;
+	}
+
+	mutex_init(&dev_ctx->fops_lock);
+	dev_ctx->priv = priv;
+	*new_dev_ctx = dev_ctx;
+
+	ret = init_se_shared_mem(dev_ctx);
+	if (ret < 0) {
+		kfree(dev_ctx->devname);
+		kfree(dev_ctx);
+		*new_dev_ctx = NULL;
+	}
+
+	list_add_tail(&dev_ctx->link, &priv->dev_ctx_list);
+	priv->active_devctx_count++;
+
+	return ret;
+}
+
+static int se_ioctl_cmd_snd_rcv_cleanup(struct se_if_device_ctx *dev_ctx, void __user *uarg,
+					struct se_ioctl_cmd_snd_rcv_rsp_info *cmd_snd_rcv_rsp_info)
+{
+	/* shared memory is allocated before this IOCTL */
+	se_dev_ctx_shared_mem_cleanup(dev_ctx);
+
+	if (copy_to_user(uarg, cmd_snd_rcv_rsp_info, sizeof(*cmd_snd_rcv_rsp_info))) {
+		dev_err(dev_ctx->priv->dev, "%s: Failed to copy cmd_snd_rcv_rsp_info from user.",
+			dev_ctx->devname);
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+static int se_ioctl_cmd_snd_rcv_rsp_handler(struct se_if_device_ctx *dev_ctx,
+					    void __user *uarg)
+{
+	struct se_ioctl_cmd_snd_rcv_rsp_info cmd_snd_rcv_rsp_info = {0};
+	struct se_if_priv *priv = dev_ctx->priv;
+	int err = 0;
+
+	if (copy_from_user(&cmd_snd_rcv_rsp_info, uarg,
+			   sizeof(cmd_snd_rcv_rsp_info))) {
+		dev_err(priv->dev,
+			"%s: Failed to copy cmd_snd_rcv_rsp_info from user.",
+			dev_ctx->devname);
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return -EFAULT;
+	}
+
+	if (cmd_snd_rcv_rsp_info.tx_buf_sz < SE_MU_HDR_SZ) {
+		dev_err(priv->dev, "%s: User buffer too small(%d < %d)",
+			dev_ctx->devname, cmd_snd_rcv_rsp_info.tx_buf_sz, SE_MU_HDR_SZ);
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return -ENOSPC;
+	}
+
+	err = se_chk_tx_msg_hdr(priv, (struct se_msg_hdr *)cmd_snd_rcv_rsp_info.tx_buf);
+	if (err) {
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return err;
+	}
+
+	struct se_api_msg *rx_msg __free(kfree) =
+		kzalloc(cmd_snd_rcv_rsp_info.rx_buf_sz, GFP_KERNEL);
+	if (!rx_msg) {
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return -ENOMEM;
+	}
+
+	struct se_api_msg *tx_msg __free(kfree) =
+		memdup_user(cmd_snd_rcv_rsp_info.tx_buf,
+			    cmd_snd_rcv_rsp_info.tx_buf_sz);
+	if (IS_ERR(tx_msg)) {
+		err = PTR_ERR(tx_msg);
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return err;
+	}
+
+	if (tx_msg->header.tag != priv->if_defs->cmd_tag) {
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return -EINVAL;
+	}
+
+	if (tx_msg->header.ver == priv->if_defs->fw_api_ver &&
+	    get_load_fw_instance(priv)->is_fw_tobe_loaded) {
+		err = se_load_firmware(priv);
+		if (err) {
+			dev_err(priv->dev, "Could not send msg as FW is not loaded.");
+			se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+			return -EPERM;
+		}
+	}
+	set_se_rcv_msg_timeout(priv, SE_RCV_MSG_LONG_TIMEOUT);
+
+	err = ele_msg_send_rcv(dev_ctx, tx_msg, cmd_snd_rcv_rsp_info.tx_buf_sz,
+			       rx_msg, cmd_snd_rcv_rsp_info.rx_buf_sz);
+	if (err < 0) {
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return err;
+	}
+
+	dev_dbg(priv->dev, "%s: %s %s.", dev_ctx->devname, __func__,
+		"message received, start transmit to user");
+
+	/* We may need to copy the output data to user before
+	 * delivering the completion message.
+	 */
+	err = se_dev_ctx_cpy_out_data(dev_ctx);
+	if (err < 0) {
+		se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+		return err;
+	}
+
+	/* Copy data from the buffer */
+	print_hex_dump_debug("to user ", DUMP_PREFIX_OFFSET, 4, 4, rx_msg,
+			     cmd_snd_rcv_rsp_info.rx_buf_sz, false);
+
+	if (copy_to_user(cmd_snd_rcv_rsp_info.rx_buf, rx_msg,
+			 cmd_snd_rcv_rsp_info.rx_buf_sz)) {
+		dev_err(priv->dev, "%s: Failed to copy to user.", dev_ctx->devname);
+		err = -EFAULT;
+	}
+
+	err = se_ioctl_cmd_snd_rcv_cleanup(dev_ctx, uarg, &cmd_snd_rcv_rsp_info);
+
+	return err;
+}
+
+static int se_ioctl_get_mu_info(struct se_if_device_ctx *dev_ctx,
+				void __user *uarg)
+{
+	struct se_if_priv *priv = dev_ctx->priv;
+	struct se_ioctl_get_if_info if_info;
+	struct se_if_node *if_node;
+	int err = 0;
+
+	if_node = container_of(priv->if_defs, typeof(*if_node), if_defs);
+
+	if_info.se_if_id = 0;
+	if_info.interrupt_idx = 0;
+	if_info.tz = 0;
+	if_info.did = 0;
+	if_info.cmd_tag = priv->if_defs->cmd_tag;
+	if_info.rsp_tag = priv->if_defs->rsp_tag;
+	if_info.success_tag = priv->if_defs->success_tag;
+	if_info.base_api_ver = priv->if_defs->base_api_ver;
+	if_info.fw_api_ver = priv->if_defs->fw_api_ver;
+
+	dev_dbg(priv->dev, "%s: info [se_if_id: %d, irq_idx: %d, tz: 0x%x, did: 0x%x].",
+		dev_ctx->devname, if_info.se_if_id, if_info.interrupt_idx, if_info.tz,
+		if_info.did);
+
+	if (copy_to_user(uarg, &if_info, sizeof(if_info))) {
+		dev_err(priv->dev, "%s: Failed to copy mu info to user.",
+			dev_ctx->devname);
+		err = -EFAULT;
+	}
+
+	return err;
+}
+
+/*
+ * Copy a buffer of data to/from the user and return the address to use in
+ * messages
+ */
+static int se_ioctl_setup_iobuf_handler(struct se_if_device_ctx *dev_ctx,
+					void __user *uarg)
+{
+	struct se_shared_mem *shared_mem = NULL;
+	struct se_ioctl_setup_iobuf io = {0};
+	size_t aligned_len;
+	int err = 0;
+	u32 pos;
+
+	if (copy_from_user(&io, uarg, sizeof(io))) {
+		dev_err(dev_ctx->priv->dev, "%s: Failed copy iobuf config from user.",
+			dev_ctx->devname);
+		return -EFAULT;
+	}
+
+	dev_dbg(dev_ctx->priv->dev, "%s: io [buf: %p(%d) flag: %x].", dev_ctx->devname,
+		io.user_buf, io.length, io.flags);
+
+	if (io.length == 0 || !io.user_buf) {
+		/*
+		 * Accept NULL pointers since some buffers are optional
+		 * in FW commands. In this case we should return 0 as
+		 * pointer to be embedded into the message.
+		 * Skip all data copy part of code below.
+		 */
+		io.ele_addr = 0;
+		goto copy;
+	}
+
+	if (io.length > SIZE_MAX - 7) {
+		dev_err(dev_ctx->priv->dev, "%s: Invalid buffer length.",
+			dev_ctx->devname);
+		return -EINVAL;
+	}
+	aligned_len = round_up((size_t)io.length, 8);
+
+	/* No specific requirement for this buffer. */
+	shared_mem = &dev_ctx->se_shared_mem_mgmt.non_secure_mem;
+
+	/* Check there is enough space in the shared memory. */
+	dev_dbg(dev_ctx->priv->dev, "%s: req_size = %zd, max_size= %d, curr_pos = %d",
+		dev_ctx->devname, aligned_len, shared_mem->size,
+		shared_mem->pos);
+
+	if (shared_mem->size < shared_mem->pos ||
+	    aligned_len > (shared_mem->size - shared_mem->pos)) {
+		dev_err(dev_ctx->priv->dev, "%s: Not enough space in shared memory.",
+			dev_ctx->devname);
+		return -ENOMEM;
+	}
+
+	/* Allocate space in shared memory. 8 bytes aligned. */
+	pos = shared_mem->pos;
+	shared_mem->pos += aligned_len;
+	io.ele_addr = (u64)shared_mem->dma_addr + pos;
+
+	memset(shared_mem->ptr + pos, 0, io.length);
+	if ((io.flags & SE_IO_BUF_FLAGS_IS_INPUT) ||
+	    (io.flags & SE_IO_BUF_FLAGS_IS_IN_OUT)) {
+		/*
+		 * buffer is input:
+		 * copy data from user space to this allocated buffer.
+		 */
+		if (copy_from_user(shared_mem->ptr + pos, io.user_buf, io.length)) {
+			dev_err(dev_ctx->priv->dev,
+				"%s: Failed copy data to shared memory.",
+				dev_ctx->devname);
+			return -EFAULT;
+		}
+	}
+
+	err = add_b_desc_to_pending_list(shared_mem->ptr + pos, &io, dev_ctx);
+	if (err < 0)
+		dev_err(dev_ctx->priv->dev, "%s: Failed to allocate/link b_desc.",
+			dev_ctx->devname);
+
+copy:
+	/* Provide the EdgeLock Enclave address to user space only if success.*/
+	if (copy_to_user(uarg, &io, sizeof(io))) {
+		dev_err(dev_ctx->priv->dev, "%s: Failed to copy iobuff setup to user.",
+			dev_ctx->devname);
+		err = -EFAULT;
+	}
+
+	return err;
+}
+
+/* IOCTL to provide SoC information */
+static int se_ioctl_get_se_soc_info_handler(struct se_if_device_ctx *dev_ctx,
+					    void __user *uarg)
+{
+	struct se_ioctl_get_soc_info soc_info;
+	int err = -EINVAL;
+
+	soc_info.soc_id = get_se_soc_id(dev_ctx->priv);
+	soc_info.soc_rev = var_se_info.soc_rev;
+
+	err = copy_to_user(uarg, (u8 *)(&soc_info), sizeof(soc_info));
+	if (err) {
+		dev_err(dev_ctx->priv->dev, "%s: Failed to copy soc info to user.",
+			dev_ctx->devname);
+		err = -EFAULT;
+	}
+
+	return err;
+}
+
+/*
+ * File operations for user-space
+ */
+
+/* Write a message to the MU. */
+static ssize_t se_if_fops_write(struct file *fp, const char __user *buf,
+				size_t size, loff_t *ppos)
+{
+	struct se_if_device_ctx *dev_ctx = fp->private_data;
+	struct se_if_priv *priv = dev_ctx->priv;
+	int err;
+
+	dev_dbg(priv->dev, "%s: write from buf (%p)%zu, ppos=%lld.", dev_ctx->devname,
+		buf, size, ((ppos) ? *ppos : 0));
+
+	scoped_cond_guard(mutex_intr, return -EBUSY, &dev_ctx->fops_lock) {
+		if (dev_ctx != priv->cmd_receiver_clbk_hdl.dev_ctx)
+			return -EINVAL;
+
+		err = se_chk_tx_msg_hdr(priv, (struct se_msg_hdr *)buf);
+		if (err)
+			return err;
+
+		if (size < SE_MU_HDR_SZ) {
+			dev_err(priv->dev, "%s: User buffer too small(%zu < %d).",
+				dev_ctx->devname, size, SE_MU_HDR_SZ);
+			return -ENOSPC;
+		}
+
+		struct se_api_msg *tx_msg __free(kfree) = memdup_user(buf, size);
+		if (IS_ERR(tx_msg))
+			return PTR_ERR(tx_msg);
+
+		print_hex_dump_debug("from user ", DUMP_PREFIX_OFFSET, 4, 4,
+				     tx_msg, size, false);
+
+		err = ele_msg_send(dev_ctx, tx_msg, size);
+
+		return err;
+	}
+}
+
+/*
+ * Read a message from the MU.
+ * Blocking until a message is available.
+ */
+static ssize_t se_if_fops_read(struct file *fp, char __user *buf, size_t size,
+			       loff_t *ppos)
+{
+	struct se_if_device_ctx *dev_ctx = fp->private_data;
+	struct se_if_priv *priv = dev_ctx->priv;
+	size_t copy_len;
+	int err;
+
+	dev_dbg(priv->dev, "%s: read to buf %p(%zu), ppos=%lld.", dev_ctx->devname,
+		buf, size, ((ppos) ? *ppos : 0));
+
+	scoped_cond_guard(mutex_intr, return -EBUSY, &dev_ctx->fops_lock) {
+		if (dev_ctx != priv->cmd_receiver_clbk_hdl.dev_ctx) {
+			err = -EINVAL;
+			goto exit;
+		}
+
+		err = ele_msg_rcv(dev_ctx, &priv->cmd_receiver_clbk_hdl);
+		if (err < 0) {
+			dev_err(priv->dev,
+				"%s: Er[0x%x]: Signal Interrupted. Current act-dev-ctx count: %d.",
+				dev_ctx->devname, err, dev_ctx->priv->active_devctx_count);
+			goto exit;
+		}
+
+		/* We may need to copy the output data to user before
+		 * delivering the completion message.
+		 */
+		err = se_dev_ctx_cpy_out_data(dev_ctx);
+		if (err < 0)
+			goto exit;
+
+		/* Copy data from the buffer */
+		print_hex_dump_debug("to user ", DUMP_PREFIX_OFFSET, 4, 4,
+				     priv->cmd_receiver_clbk_hdl.rx_msg,
+				     priv->cmd_receiver_clbk_hdl.rx_msg_sz,
+				     false);
+
+		copy_len = min(size, priv->cmd_receiver_clbk_hdl.rx_msg_sz);
+
+		if (copy_to_user(buf, priv->cmd_receiver_clbk_hdl.rx_msg, copy_len))
+			err = -EFAULT;
+		else
+			err = copy_len;
+
+exit:
+		priv->cmd_receiver_clbk_hdl.rx_msg_sz = 0;
+
+		se_dev_ctx_shared_mem_cleanup(dev_ctx);
+
+		return err;
+	}
+}
+
+/* Open a character device. */
+static int se_if_fops_open(struct inode *nd, struct file *fp)
+{
+	struct miscdevice *miscdev = fp->private_data;
+	struct se_if_device_ctx *misc_dev_ctx;
+	struct se_if_device_ctx *dev_ctx;
+	struct se_if_priv *priv;
+	int err = 0;
+
+	priv = dev_get_drvdata(miscdev->parent);
+	misc_dev_ctx = priv->priv_dev_ctx;
+
+	scoped_cond_guard(mutex_intr, return -EBUSY, &misc_dev_ctx->fops_lock) {
+		priv->dev_ctx_mono_count++;
+		err = init_device_context(priv,
+					  priv->dev_ctx_mono_count ?
+					  priv->dev_ctx_mono_count
+					  : priv->dev_ctx_mono_count++,
+					  &dev_ctx);
+		if (err)
+			dev_err(priv->dev, "Failed[0x%x] to create dev-ctx.", err);
+		else
+			fp->private_data = dev_ctx;
+
+		return err;
+	}
+}
+
+/* Close a character device. */
+static int se_if_fops_close(struct inode *nd, struct file *fp)
+{
+	struct se_if_device_ctx *dev_ctx = fp->private_data;
+	struct se_if_priv *priv = dev_ctx->priv;
+
+	scoped_cond_guard(mutex_intr, return -EBUSY, &dev_ctx->fops_lock) {
+		/* check if this device was registered as command receiver. */
+		if (priv->cmd_receiver_clbk_hdl.dev_ctx == dev_ctx) {
+			priv->cmd_receiver_clbk_hdl.dev_ctx = NULL;
+			kfree(priv->cmd_receiver_clbk_hdl.rx_msg);
+			priv->cmd_receiver_clbk_hdl.rx_msg = NULL;
+		}
+
+		se_dev_ctx_shared_mem_cleanup(dev_ctx);
+		cleanup_se_shared_mem(dev_ctx);
+
+		priv->active_devctx_count--;
+		list_del(&dev_ctx->link);
+
+		kfree(dev_ctx->devname);
+		kfree(dev_ctx);
+	}
+
+	return 0;
+}
+
+/* IOCTL entry point of a character device */
+static long se_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
+{
+	struct se_if_device_ctx *dev_ctx = fp->private_data;
+	struct se_if_priv *priv = dev_ctx->priv;
+	void __user *uarg = (void __user *)arg;
+	long err;
+
+	/* Prevent race during change of device context */
+	scoped_cond_guard(mutex_intr, return -EBUSY, &dev_ctx->fops_lock) {
+		switch (cmd) {
+		case SE_IOCTL_ENABLE_CMD_RCV:
+			if (!priv->cmd_receiver_clbk_hdl.dev_ctx) {
+				if (!priv->cmd_receiver_clbk_hdl.rx_msg) {
+					priv->cmd_receiver_clbk_hdl.rx_msg =
+						kzalloc(MAX_NVM_MSG_LEN,
+							GFP_KERNEL);
+					if (!priv->cmd_receiver_clbk_hdl.rx_msg) {
+						err = -ENOMEM;
+						break;
+					}
+				}
+				priv->cmd_receiver_clbk_hdl.rx_msg_sz = MAX_NVM_MSG_LEN;
+				priv->cmd_receiver_clbk_hdl.dev_ctx = dev_ctx;
+				err = 0;
+			} else {
+				err = -EBUSY;
+			}
+			break;
+		case SE_IOCTL_GET_MU_INFO:
+			err = se_ioctl_get_mu_info(dev_ctx, uarg);
+			break;
+		case SE_IOCTL_SETUP_IOBUF:
+			err = se_ioctl_setup_iobuf_handler(dev_ctx, uarg);
+			break;
+		case SE_IOCTL_GET_SOC_INFO:
+			err = se_ioctl_get_se_soc_info_handler(dev_ctx, uarg);
+			break;
+		case SE_IOCTL_CMD_SEND_RCV_RSP:
+			err = se_ioctl_cmd_snd_rcv_rsp_handler(dev_ctx, uarg);
+			break;
+		default:
+			err = -EINVAL;
+			dev_dbg(priv->dev, "%s: IOCTL %.8x not supported.",
+				dev_ctx->devname, cmd);
+		}
+	}
+
+	return err;
+}
+
+/* Char driver setup */
+static const struct file_operations se_if_fops = {
+	.open		= se_if_fops_open,
+	.owner		= THIS_MODULE,
+	.release	= se_if_fops_close,
+	.unlocked_ioctl = se_ioctl,
+	.read		= se_if_fops_read,
+	.write		= se_if_fops_write,
+};
+
+/* interface for managed res to free a mailbox channel */
+static void if_mbox_free_channel(void *mbox_chan)
+{
+	mbox_free_channel(mbox_chan);
+}
+
+static int se_if_request_channel(struct device *dev, struct mbox_chan **chan,
+				 struct mbox_client *cl, const char *name)
+{
+	struct mbox_chan *t_chan;
+	int ret = 0;
+
+	t_chan = mbox_request_channel_byname(cl, name);
+	if (IS_ERR(t_chan))
+		return dev_err_probe(dev, PTR_ERR(t_chan),
+				     "Failed to request %s channel.", name);
+
+	ret = devm_add_action_or_reset(dev, if_mbox_free_channel, t_chan);
+	if (ret)
+		return dev_err_probe(dev, -EPERM,
+				     "Failed to add-action for removal of mbox: %s.",
+				     name);
+	*chan = t_chan;
+
+	return ret;
+}
+
+static void se_if_probe_cleanup(void *plat_dev)
+{
+	struct se_if_device_ctx *dev_ctx, *t_dev_ctx;
+	struct platform_device *pdev = plat_dev;
+	struct se_fw_load_info *load_fw;
+	struct device *dev = &pdev->dev;
+	struct se_if_priv *priv;
+
+	priv = dev_get_drvdata(dev);
+	load_fw = get_load_fw_instance(priv);
+
+	/*
+	 * In se_if_request_channel(), passed the clean-up functional
+	 * pointer reference as action to devm_add_action_or_reset().
+	 * No need to free the mbox channels here.
+	 */
+
+	/*
+	 * free the buffer in se remove, previously allocated
+	 * in se probe to store encrypted IMEM
+	 */
+	if (load_fw && load_fw->imem.buf) {
+		dmam_free_coherent(dev, ELE_IMEM_SIZE, load_fw->imem.buf,
+				   load_fw->imem.phyaddr);
+		load_fw->imem.buf = NULL;
+	}
+
+	if (priv->dev_ctx_mono_count) {
+		list_for_each_entry_safe(dev_ctx, t_dev_ctx, &priv->dev_ctx_list, link) {
+			list_del(&dev_ctx->link);
+			priv->active_devctx_count--;
+		}
+	}
+
+	/*
+	 * No need to check, if reserved memory is allocated
+	 * before calling for its release. Or clearing the
+	 * un-set bit.
+	 */
+	of_reserved_mem_device_release(dev);
+}
+
+static int se_if_probe(struct platform_device *pdev)
+{
+	const struct se_soc_info *se_info;
+	const struct se_if_node *if_node;
+	struct se_fw_load_info *load_fw;
+	struct device *dev = &pdev->dev;
+	struct se_if_priv *priv;
+	dma_addr_t imem_dma_addr;
+	int ret;
+
+	if_node = device_get_match_data(dev);
+	if (!if_node)
+		return -EINVAL;
+
+	se_info = if_node->se_info;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	priv->if_defs = &if_node->if_defs;
+	dev_set_drvdata(dev, priv);
+
+	ret = devm_add_action_or_reset(dev, se_if_probe_cleanup, pdev);
+	if (ret)
+		return ret;
+
+	/* Mailbox client configuration */
+	priv->se_mb_cl.dev		= dev;
+	priv->se_mb_cl.tx_block		= false;
+	priv->se_mb_cl.knows_txdone	= true;
+	priv->se_mb_cl.rx_callback	= se_if_rx_callback;
+	set_se_rcv_msg_timeout(priv, SE_RCV_MSG_DEFAULT_TIMEOUT);
+
+	ret = se_if_request_channel(dev, &priv->tx_chan, &priv->se_mb_cl, MBOX_TX_NAME);
+	if (ret)
+		return ret;
+
+	ret = se_if_request_channel(dev, &priv->rx_chan, &priv->se_mb_cl, MBOX_RX_NAME);
+	if (ret)
+		return ret;
+
+	mutex_init(&priv->se_if_cmd_lock);
+
+	init_completion(&priv->waiting_rsp_clbk_hdl.done);
+	init_completion(&priv->cmd_receiver_clbk_hdl.done);
+
+	if (if_node->pool_name) {
+		priv->mem_pool = of_gen_pool_get(dev->of_node, if_node->pool_name, 0);
+		if (!priv->mem_pool)
+			return dev_err_probe(dev, -ENOMEM,
+					     "Unable to get sram pool = %s.",
+					     if_node->pool_name);
+	}
+	INIT_LIST_HEAD(&priv->dev_ctx_list);
+
+	if (if_node->reserved_dma_ranges) {
+		ret = of_reserved_mem_device_init(dev);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					    "Failed to init reserved memory region.");
+	}
+
+	ret = init_misc_device_context(priv, 0, &priv->priv_dev_ctx, &se_if_fops);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed[0x%x] to create device contexts.",
+				     ret);
+
+	if (if_node->if_defs.se_if_type == SE_TYPE_ID_HSM) {
+		ret = get_se_soc_info(priv, se_info);
+		if (ret)
+			return dev_err_probe(dev, ret, "Failed to fetch SoC Info.");
+	}
+
+	/* By default, there is no pending FW to be loaded.*/
+	if (se_info->se_fw_img_nm.seco_fw_nm_in_rfs) {
+		load_fw = get_load_fw_instance(priv);
+		load_fw->se_fw_img_nm = &se_info->se_fw_img_nm;
+		load_fw->is_fw_tobe_loaded = true;
+
+		if (load_fw->se_fw_img_nm->prim_fw_nm_in_rfs) {
+			/* allocate buffer where SE store encrypted IMEM */
+			imem_dma_addr = phys_to_dma(priv->dev, load_fw->imem.phyaddr);
+			load_fw->imem.buf = dmam_alloc_coherent(priv->dev, ELE_IMEM_SIZE,
+								&imem_dma_addr, GFP_KERNEL);
+			if (!load_fw->imem.buf)
+				return dev_err_probe(dev, -ENOMEM,
+						     "dmam-alloc-failed: To store encr-IMEM.");
+			load_fw->imem_mgmt = true;
+		}
+	}
+	dev_info(dev, "i.MX secure-enclave: %s0 interface to firmware, configured.",
+		 get_se_if_name(priv->if_defs->se_if_type));
+
+	return ret;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int se_suspend(struct device *dev)
+{
+	struct se_if_priv *priv = dev_get_drvdata(dev);
+	struct se_fw_load_info *load_fw;
+	int ret = 0;
+
+	set_se_rcv_msg_timeout(priv, SE_RCV_MSG_DEFAULT_TIMEOUT);
+	load_fw = get_load_fw_instance(priv);
+
+	if (load_fw->imem_mgmt) {
+		ret = se_save_imem_state(priv, &load_fw->imem);
+		if (ret)
+			dev_err(dev, "Failure saving IMEM state[0x%x]", ret);
+	}
+
+	return 0;
+}
+
+static int se_resume(struct device *dev)
+{
+	struct se_if_priv *priv = dev_get_drvdata(dev);
+	struct se_fw_load_info *load_fw;
+	int ret = 0;
+
+	load_fw = get_load_fw_instance(priv);
+
+	if (load_fw->imem_mgmt) {
+		se_restore_imem_state(priv, &load_fw->imem);
+		if (ret)
+			dev_err(dev, "Failure restoring IMEM state[0x%x]", ret);
+	}
+
+	return 0;
+}
+
+static const struct dev_pm_ops se_pm = {
+	SET_SYSTEM_SLEEP_PM_OPS(se_suspend, se_resume)
+};
+
+#define SE_PM_OPS	(&se_pm)
+#else
+#define SE_PM_OPS	NULL
+#endif
+
+static struct platform_driver se_driver = {
+	.driver = {
+		.name = "fsl-se",
+		.of_match_table = se_match,
+		.pm = SE_PM_OPS,
+	},
+	.probe = se_if_probe,
+};
+MODULE_DEVICE_TABLE(of, se_match);
+
+module_platform_driver(se_driver);
+MODULE_AUTHOR("Pankaj Gupta <pankaj.gupta@nxp.com>");
+MODULE_DESCRIPTION("iMX Secure Enclave Driver.");
+MODULE_LICENSE("GPL");
diff --git a/drivers/firmware/imx/se_ctrl.h b/drivers/firmware/imx/se_ctrl.h
new file mode 100644
index 0000000000000..7f9b0d199cfcf
--- /dev/null
+++ b/drivers/firmware/imx/se_ctrl.h
@@ -0,0 +1,129 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2026 NXP
+ */
+
+#ifndef SE_MU_H
+#define SE_MU_H
+
+#include <linux/bitfield.h>
+#include <linux/miscdevice.h>
+#include <linux/semaphore.h>
+#include <linux/mailbox_client.h>
+
+#define MAX_FW_LOAD_RETRIES		50
+#define SE_MSG_WORD_SZ			0x4
+
+#define RES_STATUS(x)			FIELD_GET(0x000000ff, x)
+#define MAX_DATA_SIZE_PER_USER		(65 * 1024)
+#define MAX_NVM_MSG_LEN			(256)
+#define MESSAGING_VERSION_6		0x6
+#define MESSAGING_VERSION_7		0x7
+
+struct se_clbk_handle {
+	struct se_if_device_ctx *dev_ctx;
+	struct completion done;
+	bool signal_rcvd;
+	u32 rx_msg_sz;
+	/*
+	 * Assignment of the rx_msg buffer to held till the
+	 * received content as part callback function, is copied.
+	 */
+	struct se_api_msg *rx_msg;
+};
+
+struct se_imem_buf {
+	u8 *buf;
+	phys_addr_t phyaddr;
+	u32 size;
+	u32 state;
+};
+
+struct se_buf_desc {
+	u8 *shared_buf_ptr;
+	void __user *usr_buf_ptr;
+	u32 size;
+	struct list_head link;
+};
+
+struct se_shared_mem {
+	dma_addr_t dma_addr;
+	u32 size;
+	u32 pos;
+	u8 *ptr;
+};
+
+struct se_shared_mem_mgmt_info {
+	struct list_head pending_in;
+	struct list_head pending_out;
+
+	struct se_shared_mem non_secure_mem;
+};
+
+/* Private struct for each char device instance. */
+struct se_if_device_ctx {
+	struct se_if_priv *priv;
+	struct miscdevice *miscdev;
+	const char *devname;
+
+	/* process one file operation at a time. */
+	struct mutex fops_lock;
+
+	struct se_shared_mem_mgmt_info se_shared_mem_mgmt;
+	struct list_head link;
+};
+
+/* Header of the messages exchange with the EdgeLock Enclave */
+struct se_msg_hdr {
+	u8 ver;
+	u8 size;
+	u8 command;
+	u8 tag;
+}  __packed;
+
+#define SE_MU_HDR_SZ	4
+
+struct se_api_msg {
+	struct se_msg_hdr header;
+	u32 data[];
+};
+
+struct se_if_defines {
+	const u8 se_if_type;
+	u8 cmd_tag;
+	u8 rsp_tag;
+	u8 success_tag;
+	u8 base_api_ver;
+	u8 fw_api_ver;
+};
+
+struct se_if_priv {
+	struct device *dev;
+
+	struct se_clbk_handle cmd_receiver_clbk_hdl;
+	/*
+	 * Update to the waiting_rsp_dev, to be protected
+	 * under se_if_cmd_lock.
+	 */
+	struct se_clbk_handle waiting_rsp_clbk_hdl;
+	/*
+	 * prevent new command to be sent on the se interface while previous
+	 * command is still processing. (response is awaited)
+	 */
+	struct mutex se_if_cmd_lock;
+
+	struct mbox_client se_mb_cl;
+	struct mbox_chan *tx_chan, *rx_chan;
+
+	struct gen_pool *mem_pool;
+	const struct se_if_defines *if_defs;
+
+	struct se_if_device_ctx *priv_dev_ctx;
+	struct list_head dev_ctx_list;
+	u32 active_devctx_count;
+	u32 dev_ctx_mono_count;
+	u32 se_rcv_msg_timeout_ms;
+};
+
+char *get_se_if_name(u8 se_if_id);
+#endif
diff --git a/drivers/firmware/imx/sm-misc.c b/drivers/firmware/imx/sm-misc.c
index 0a8ada329c9de..ac9af824c2d40 100644
--- a/drivers/firmware/imx/sm-misc.c
+++ b/drivers/firmware/imx/sm-misc.c
@@ -16,7 +16,30 @@
 
 static const struct scmi_imx_misc_proto_ops *imx_misc_ctrl_ops;
 static struct scmi_protocol_handle *ph;
-struct notifier_block scmi_imx_misc_ctrl_nb;
+static struct notifier_block scmi_imx_misc_ctrl_nb;
+
+static const char * const rst_imx95[] = {
+	"cm33_lockup", "cm33_swreq", "cm7_lockup", "cm7_swreq", "fccu",
+	"jtag_sw", "ele", "tempsense", "wdog1", "wdog2", "wdog3", "wdog4",
+	"wdog5", "jtag", "cm33_exc", "bbm", "sw", "sm_err", "fusa_sreco",
+	"pmic", "unused", "unused", "unused", "unused", "unused", "unused",
+	"unused", "unused", "unused", "unused", "unused", "por",
+};
+
+static const char * const rst_imx94[] = {
+	"cm33_lockup", "cm33_swreq", "cm70_lockup", "cm70_swreq", "fccu",
+	"jtag_sw", "ele", "tempsense", "wdog1", "wdog2", "wdog3", "wdog4",
+	"wdog5", "jtag", "wdog6", "wdog7", "wdog8", "wo_netc", "cm33s_lockup",
+	"cm33s_swreq", "cm71_lockup", "cm71_swreq", "cm33_exc", "bbm", "sw",
+	"sm_err", "fusa_sreco", "pmic", "unused", "unused", "unused", "por",
+};
+
+static const struct of_device_id allowlist[] = {
+	{ .compatible = "fsl,imx952", .data = rst_imx95 },
+	{ .compatible = "fsl,imx95", .data = rst_imx95 },
+	{ .compatible = "fsl,imx94", .data = rst_imx94 },
+	{ /* Sentinel */ }
+};
 
 int scmi_imx_misc_ctrl_set(u32 id, u32 val)
 {
@@ -75,6 +98,54 @@ static void scmi_imx_misc_put(void *p)
 	debugfs_remove((struct dentry *)p);
 }
 
+static int scmi_imx_misc_get_reason(struct scmi_device *sdev)
+{
+	struct scmi_imx_misc_reset_reason boot, shutdown;
+	const char **rst;
+	bool system = true;
+	int ret;
+
+	if (!of_machine_device_match(allowlist))
+		return 0;
+
+	rst = (const char **)of_machine_get_match_data(allowlist);
+
+	ret = imx_misc_ctrl_ops->misc_reset_reason(ph, system, &boot, &shutdown, NULL);
+	if (!ret) {
+		if (boot.valid)
+			dev_info(&sdev->dev, "%s Boot reason: %s, origin: %d, errid: %d\n",
+				 system ? "SYS" : "LM", rst[boot.reason],
+				 boot.orig_valid ? boot.origin : -1,
+				 boot.err_valid ? boot.errid : -1);
+		if (shutdown.valid)
+			dev_info(&sdev->dev, "%s shutdown reason: %s, origin: %d, errid: %d\n",
+				 system ? "SYS" : "LM", rst[shutdown.reason],
+				 shutdown.orig_valid ? shutdown.origin : -1,
+				 shutdown.err_valid ? shutdown.errid : -1);
+	} else {
+		dev_err(&sdev->dev, "Failed to get system reset reason: %d\n", ret);
+	}
+
+	system = false;
+	ret = imx_misc_ctrl_ops->misc_reset_reason(ph, system, &boot, &shutdown, NULL);
+	if (!ret) {
+		if (boot.valid)
+			dev_info(&sdev->dev, "%s Boot reason: %s, origin: %d, errid: %d\n",
+				 system ? "SYS" : "LM", rst[boot.reason],
+				 boot.orig_valid ? boot.origin : -1,
+				 boot.err_valid ? boot.errid : -1);
+		if (shutdown.valid)
+			dev_info(&sdev->dev, "%s shutdown reason: %s, origin: %d, errid: %d\n",
+				 system ? "SYS" : "LM", rst[shutdown.reason],
+				 shutdown.orig_valid ? shutdown.origin : -1,
+				 shutdown.err_valid ? shutdown.errid : -1);
+	} else {
+		dev_err(&sdev->dev, "Failed to get lm reset reason: %d\n", ret);
+	}
+
+	return 0;
+}
+
 static int scmi_imx_misc_ctrl_probe(struct scmi_device *sdev)
 {
 	const struct scmi_handle *handle = sdev->handle;
@@ -133,6 +204,8 @@ static int scmi_imx_misc_ctrl_probe(struct scmi_device *sdev)
 	scmi_imx_dentry = debugfs_create_dir("scmi_imx", NULL);
 	debugfs_create_file("syslog", 0444, scmi_imx_dentry, &sdev->dev, &syslog_fops);
 
+	scmi_imx_misc_get_reason(sdev);
+
 	return devm_add_action_or_reset(&sdev->dev, scmi_imx_misc_put, scmi_imx_dentry);
 }
 
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index 38ca190d4a22d..e73bae6cb23a3 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -539,12 +539,22 @@ static int psci_system_suspend(unsigned long unused)
 
 static int psci_system_suspend_enter(suspend_state_t state)
 {
+	pm_set_resume_via_firmware();
+
 	return cpu_suspend(0, psci_system_suspend);
 }
 
+static int psci_system_suspend_begin(suspend_state_t state)
+{
+	pm_set_suspend_via_firmware();
+
+	return 0;
+}
+
 static const struct platform_suspend_ops psci_suspend_ops = {
 	.valid          = suspend_valid_only_mem,
 	.enter          = psci_system_suspend_enter,
+	.begin          = psci_system_suspend_begin,
 };
 
 static void __init psci_init_system_reset2(void)
diff --git a/drivers/firmware/qcom/qcom_scm.c b/drivers/firmware/qcom/qcom_scm.c
index 9b06a69d3a6d7..6b601a4b89dbf 100644
--- a/drivers/firmware/qcom/qcom_scm.c
+++ b/drivers/firmware/qcom/qcom_scm.c
@@ -2306,6 +2306,7 @@ static const struct of_device_id qcom_scm_qseecom_allowlist[] __maybe_unused = {
 	{ .compatible = "lenovo,yoga-slim7x" },
 	{ .compatible = "medion,sprchrgd14s1" },
 	{ .compatible = "microsoft,arcata", },
+	{ .compatible = "microsoft,surface-pro-12in", },
 	{ .compatible = "microsoft,blackrock" },
 	{ .compatible = "microsoft,denali", },
 	{ .compatible = "microsoft,romulus13", },
diff --git a/drivers/firmware/samsung/exynos-acpm-dvfs.c b/drivers/firmware/samsung/exynos-acpm-dvfs.c
index 06bdf62dea1f3..fdea7aa24ca02 100644
--- a/drivers/firmware/samsung/exynos-acpm-dvfs.c
+++ b/drivers/firmware/samsung/exynos-acpm-dvfs.c
@@ -31,6 +31,9 @@ static void acpm_dvfs_set_xfer(struct acpm_xfer *xfer, u32 *cmd, size_t cmdlen,
 	if (response) {
 		xfer->rxcnt = cmdlen;
 		xfer->rxd = cmd;
+	} else {
+		xfer->rxcnt = 0;
+		xfer->rxd = NULL;
 	}
 }
 
diff --git a/drivers/firmware/samsung/exynos-acpm.c b/drivers/firmware/samsung/exynos-acpm.c
index 16c46ed608371..a2cac913b2bd5 100644
--- a/drivers/firmware/samsung/exynos-acpm.c
+++ b/drivers/firmware/samsung/exynos-acpm.c
@@ -5,13 +5,15 @@
  * Copyright 2024 Linaro Ltd.
  */
 
+#include <asm/barrier.h>
 #include <linux/bitfield.h>
 #include <linux/bitmap.h>
-#include <linux/bits.h>
+#include <linux/bitops.h>
 #include <linux/cleanup.h>
 #include <linux/container_of.h>
 #include <linux/delay.h>
 #include <linux/device.h>
+#include <linux/find.h>
 #include <linux/firmware/samsung/exynos-acpm-protocol.h>
 #include <linux/io.h>
 #include <linux/iopoll.h>
@@ -104,12 +106,15 @@ struct acpm_queue {
  *
  * @cmd:	pointer to where the data shall be saved.
  * @n_cmd:	number of 32-bit commands.
- * @response:	true if the client expects the RX data.
+ * @rxcnt:	expected length of the response in 32-bit words.
+ * @completed:	flag indicating if the firmware response has been fully
+ *		processed.
  */
 struct acpm_rx_data {
 	u32 *cmd;
 	size_t n_cmd;
-	bool response;
+	size_t rxcnt;
+	bool completed;
 };
 
 #define ACPM_SEQNUM_MAX    64
@@ -199,31 +204,33 @@ static void acpm_get_saved_rx(struct acpm_chan *achan,
 	const struct acpm_rx_data *rx_data = &achan->rx_data[tx_seqnum - 1];
 	u32 rx_seqnum;
 
-	if (!rx_data->response)
+	if (!rx_data->rxcnt)
 		return;
 
 	rx_seqnum = FIELD_GET(ACPM_PROTOCOL_SEQNUM, rx_data->cmd[0]);
 
-	if (rx_seqnum == tx_seqnum) {
+	if (rx_seqnum == tx_seqnum)
 		memcpy(xfer->rxd, rx_data->cmd, xfer->rxcnt * sizeof(*xfer->rxd));
-		clear_bit(rx_seqnum - 1, achan->bitmap_seqnum);
-	}
 }
 
 /**
  * acpm_get_rx() - get response from RX queue.
  * @achan:	ACPM channel info.
  * @xfer:	reference to the transfer to get response for.
+ * @native_match: pointer to a boolean set to true if the thread natively
+ *                processed its own sequence number during this call.
  *
  * Return: 0 on success, -errno otherwise.
  */
-static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
+static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer,
+		       bool *native_match)
 {
 	u32 rx_front, rx_seqnum, tx_seqnum, seqnum;
 	const void __iomem *base, *addr;
 	struct acpm_rx_data *rx_data;
 	u32 i, val, mlen;
-	bool rx_set = false;
+
+	*native_match = false;
 
 	guard(mutex)(&achan->rx_lock);
 
@@ -232,10 +239,8 @@ static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
 
 	tx_seqnum = FIELD_GET(ACPM_PROTOCOL_SEQNUM, xfer->txd[0]);
 
-	if (i == rx_front) {
-		acpm_get_saved_rx(achan, xfer, tx_seqnum);
+	if (i == rx_front)
 		return 0;
-	}
 
 	base = achan->rx.base;
 	mlen = achan->mlen;
@@ -256,11 +261,16 @@ static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
 		seqnum = rx_seqnum - 1;
 		rx_data = &achan->rx_data[seqnum];
 
-		if (rx_data->response) {
+		if (rx_data->rxcnt) {
 			if (rx_seqnum == tx_seqnum) {
 				__ioread32_copy(xfer->rxd, addr, xfer->rxcnt);
-				rx_set = true;
-				clear_bit(seqnum, achan->bitmap_seqnum);
+				/*
+				 * Signal completion to the polling thread.
+				 * Pairs with smp_load_acquire() in polling
+				 * loop.
+				 */
+				smp_store_release(&rx_data->completed, true);
+				*native_match = true;
 			} else {
 				/*
 				 * The RX data corresponds to another request.
@@ -268,25 +278,34 @@ static int acpm_get_rx(struct acpm_chan *achan, const struct acpm_xfer *xfer)
 				 * clear yet the bitmap. It will be cleared
 				 * after the response is copied to the request.
 				 */
-				__ioread32_copy(rx_data->cmd, addr, xfer->rxcnt);
+				__ioread32_copy(rx_data->cmd, addr,
+						rx_data->rxcnt);
+				/*
+				 * Signal completion to the polling thread.
+				 * Pairs with smp_load_acquire() in polling
+				 * loop.
+				 */
+				smp_store_release(&rx_data->completed, true);
 			}
 		} else {
-			clear_bit(seqnum, achan->bitmap_seqnum);
+			/*
+			 * Signal completion to the polling thread.
+			 * Pairs with smp_load_acquire() in polling loop.
+			 */
+			smp_store_release(&rx_data->completed, true);
+			if (rx_seqnum == tx_seqnum)
+				*native_match = true;
 		}
 
 		i = (i + 1) % achan->qlen;
 	} while (i != rx_front);
 
+	/* Ensure all payload reads complete before advancing the rear pointer */
+	mb();
+
 	/* We saved all responses, mark RX empty. */
 	writel(rx_front, achan->rx.rear);
 
-	/*
-	 * If the response was not in this iteration of the queue, check if the
-	 * RX data was previously saved.
-	 */
-	if (!rx_set)
-		acpm_get_saved_rx(achan, xfer, tx_seqnum);
-
 	return 0;
 }
 
@@ -301,6 +320,7 @@ static int acpm_dequeue_by_polling(struct acpm_chan *achan,
 				   const struct acpm_xfer *xfer)
 {
 	struct device *dev = achan->acpm->dev;
+	bool native_match;
 	ktime_t timeout;
 	u32 seqnum;
 	int ret;
@@ -309,12 +329,25 @@ static int acpm_dequeue_by_polling(struct acpm_chan *achan,
 
 	timeout = ktime_add_us(ktime_get(), ACPM_POLL_TIMEOUT_US);
 	do {
-		ret = acpm_get_rx(achan, xfer);
+		ret = acpm_get_rx(achan, xfer, &native_match);
 		if (ret)
 			return ret;
 
-		if (!test_bit(seqnum - 1, achan->bitmap_seqnum))
+		/*
+		 * Safely check if our specific transaction has been processed.
+		 * smp_load_acquire prevents the CPU from speculatively
+		 * executing subsequent instructions before the transaction is
+		 * synchronized.
+		 */
+		if (smp_load_acquire(&achan->rx_data[seqnum - 1].completed)) {
+			/* Retrieve payload if another thread cached it for us */
+			if (!native_match)
+				acpm_get_saved_rx(achan, xfer, seqnum);
+
+			/* Relinquish ownership of the sequence slot */
+			clear_bit_unlock(seqnum - 1, achan->bitmap_seqnum);
 			return 0;
+		}
 
 		/* Determined experimentally. */
 		udelay(20);
@@ -362,29 +395,48 @@ static int acpm_wait_for_queue_slots(struct acpm_chan *achan, u32 next_tx_front)
  * TX queue.
  * @achan:	ACPM channel info.
  * @xfer:	reference to the transfer being prepared.
+ *
+ * Return: 0 on success, -errno otherwise.
  */
-static void acpm_prepare_xfer(struct acpm_chan *achan,
-			      const struct acpm_xfer *xfer)
+static int acpm_prepare_xfer(struct acpm_chan *achan,
+			     const struct acpm_xfer *xfer)
 {
 	struct acpm_rx_data *rx_data;
 	u32 *txd = (u32 *)xfer->txd;
+	unsigned long size = ACPM_SEQNUM_MAX - 1;
+	unsigned long bit = achan->seqnum;
+
+	bit = find_next_zero_bit(achan->bitmap_seqnum, size, bit);
+	if (bit >= size) {
+		bit = find_first_zero_bit(achan->bitmap_seqnum, size);
+		if (bit >= size) {
+			dev_err_ratelimited(achan->acpm->dev,
+					    "ACPM sequence number pool exhausted\n");
+			return -EBUSY;
+		}
+	}
 
-	/* Prevent chan->seqnum from being re-used */
-	do {
-		if (++achan->seqnum == ACPM_SEQNUM_MAX)
-			achan->seqnum = 1;
-	} while (test_bit(achan->seqnum - 1, achan->bitmap_seqnum));
+	/*
+	 * Execute the atomic set to formally claim the bit and establish
+	 * LKMM Acquire semantics against the RX thread's clear_bit_unlock().
+	 * A loop is unnecessary because allocations are strictly serialized
+	 * by tx_lock.
+	 */
+	if (WARN_ON_ONCE(test_and_set_bit_lock(bit, achan->bitmap_seqnum)))
+		return -EIO;
 
+	/* Flag the index based on seqnum. (seqnum: 1~63, bitmap: 0~62) */
+	achan->seqnum = bit + 1;
 	txd[0] |= FIELD_PREP(ACPM_PROTOCOL_SEQNUM, achan->seqnum);
 
 	/* Clear data for upcoming responses */
-	rx_data = &achan->rx_data[achan->seqnum - 1];
+	rx_data = &achan->rx_data[bit];
+	rx_data->completed = false;
 	memset(rx_data->cmd, 0, sizeof(*rx_data->cmd) * rx_data->n_cmd);
-	if (xfer->rxd)
-		rx_data->response = true;
+	/* zero means no response expected */
+	rx_data->rxcnt = xfer->rxcnt;
 
-	/* Flag the index based on seqnum. (seqnum: 1~63, bitmap: 0~62) */
-	set_bit(achan->seqnum - 1, achan->bitmap_seqnum);
+	return 0;
 }
 
 /**
@@ -444,7 +496,9 @@ int acpm_do_xfer(struct acpm_handle *handle, const struct acpm_xfer *xfer)
 		if (ret)
 			return ret;
 
-		acpm_prepare_xfer(achan, xfer);
+		ret = acpm_prepare_xfer(achan, xfer);
+		if (ret)
+			return ret;
 
 		/* Write TX command. */
 		__iowrite32_copy(achan->tx.base + achan->mlen * tx_front,
@@ -526,10 +580,11 @@ static int acpm_achan_alloc_cmds(struct acpm_chan *achan)
 
 /**
  * acpm_free_mbox_chans() - free mailbox channels.
- * @acpm:	pointer to driver data.
+ * @data:	pointer to driver data.
  */
-static void acpm_free_mbox_chans(struct acpm_info *acpm)
+static void acpm_free_mbox_chans(void *data)
 {
+	struct acpm_info *acpm = data;
 	int i;
 
 	for (i = 0; i < acpm->num_chans; i++)
@@ -557,6 +612,10 @@ static int acpm_channels_init(struct acpm_info *acpm)
 	if (!acpm->chans)
 		return -ENOMEM;
 
+	ret = devm_add_action_or_reset(dev, acpm_free_mbox_chans, acpm);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to add mbox free action.\n");
+
 	chans_shmem = acpm->sram_base + readl(&shmem->chans);
 
 	for (i = 0; i < acpm->num_chans; i++) {
@@ -578,10 +637,8 @@ static int acpm_channels_init(struct acpm_info *acpm)
 		cl->dev = dev;
 
 		achan->chan = mbox_request_channel(cl, 0);
-		if (IS_ERR(achan->chan)) {
-			acpm_free_mbox_chans(acpm);
+		if (IS_ERR(achan->chan))
 			return PTR_ERR(achan->chan);
-		}
 	}
 
 	return 0;
diff --git a/drivers/firmware/smccc/soc_id.c b/drivers/firmware/smccc/soc_id.c
index 2f7475e66b3ca..a909d5e6dee50 100644
--- a/drivers/firmware/smccc/soc_id.c
+++ b/drivers/firmware/smccc/soc_id.c
@@ -60,7 +60,7 @@ static char __init *smccc_soc_name_init(void)
 	 * to the ARM_SMCCC_ARCH_SOC_ID function.  Fetch it if
 	 * available.
 	 */
-	args.a0 = ARM_SMCCC_ARCH_SOC_ID;
+	args.a0 = ARM_SMCCC_ARCH_SOC_ID64;
 	args.a1 = 2;    /* SOC_ID name */
 	arm_smccc_1_2_invoke(&args, &res);
 
diff --git a/drivers/firmware/xilinx/zynqmp.c b/drivers/firmware/xilinx/zynqmp.c
index fbe8510f49271..af838b2dc3279 100644
--- a/drivers/firmware/xilinx/zynqmp.c
+++ b/drivers/firmware/xilinx/zynqmp.c
@@ -1450,6 +1450,34 @@ int zynqmp_pm_get_node_status(const u32 node, u32 *const status,
 }
 EXPORT_SYMBOL_GPL(zynqmp_pm_get_node_status);
 
+/**
+ * zynqmp_pm_get_rpu_node_status - PM call to request a RPU node's current power state
+ * @node:		ID of the RPU component or sub-system in question
+ * @status:		Current operating state of the requested RPU node.
+ * @requirements:	Current requirements asserted on the RPU node.
+ * @usage:		Usage information, used for RPU slave nodes only:
+ *			PM_USAGE_NO_MASTER	- No master is currently using
+ *						  the node
+ *			PM_USAGE_CURRENT_MASTER	- Only requesting master is
+ *						  currently using the node
+ *			PM_USAGE_OTHER_MASTER	- Only other masters are
+ *						  currently using the node
+ *			PM_USAGE_BOTH_MASTERS	- Both the current and at least
+ *						  one other master is currently
+ *						  using the node
+ *
+ * Return:		Returns status, either success or error+reason
+ */
+int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status,
+				  u32 *const requirements, u32 *const usage)
+{
+	if (zynqmp_pm_feature(PM_GET_NODE_STATUS) < PM_API_VERSION_2)
+		return -EOPNOTSUPP;
+
+	return zynqmp_pm_get_node_status(node, status, requirements, usage);
+}
+EXPORT_SYMBOL_GPL(zynqmp_pm_get_rpu_node_status);
+
 /**
  * zynqmp_pm_force_pwrdwn - PM call to request for another PU or subsystem to
  *             be powered down forcefully
diff --git a/drivers/fpga/Kconfig b/drivers/fpga/Kconfig
index 37b35f58f0dfb..748fc210c1357 100644
--- a/drivers/fpga/Kconfig
+++ b/drivers/fpga/Kconfig
@@ -288,6 +288,13 @@ config FPGA_MGR_LATTICE_SYSCONFIG_SPI
 	  FPGA manager driver support for Lattice FPGAs programming over slave
 	  SPI sysCONFIG interface.
 
+config FPGA_MGR_EFINIX_SPI
+	tristate "Efinix FPGA configuration over SPI"
+	depends on SPI
+	help
+	  FPGA manager driver support for Efinix FPGAs configuration over SPI
+	  (passive mode only).
+
 source "drivers/fpga/tests/Kconfig"
 
 endif # FPGA
diff --git a/drivers/fpga/Makefile b/drivers/fpga/Makefile
index aeb89bb13517e..6f5798b27e0de 100644
--- a/drivers/fpga/Makefile
+++ b/drivers/fpga/Makefile
@@ -26,6 +26,7 @@ obj-$(CONFIG_FPGA_MGR_LATTICE_SYSCONFIG)	+= lattice-sysconfig.o
 obj-$(CONFIG_FPGA_MGR_LATTICE_SYSCONFIG_SPI)	+= lattice-sysconfig-spi.o
 obj-$(CONFIG_ALTERA_PR_IP_CORE)		+= altera-pr-ip-core.o
 obj-$(CONFIG_ALTERA_PR_IP_CORE_PLAT)	+= altera-pr-ip-core-plat.o
+obj-$(CONFIG_FPGA_MGR_EFINIX_SPI)	+= efinix-spi.o
 
 # FPGA Secure Update Drivers
 obj-$(CONFIG_FPGA_M10_BMC_SEC_UPDATE)	+= intel-m10-bmc-sec-update.o
diff --git a/drivers/fpga/efinix-spi.c b/drivers/fpga/efinix-spi.c
new file mode 100644
index 0000000000000..ed9a41232a32a
--- /dev/null
+++ b/drivers/fpga/efinix-spi.c
@@ -0,0 +1,260 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * FPGA Manager Driver for Efinix
+ *
+ * Copyright (C) 2025 iris-GmbH infrared & intelligent sensors
+ *
+ * Ian Dannapel <iansdannapel@gmail.com>
+ *
+ * Load Efinix FPGA firmware over SPI using the serial configuration interface.
+ *
+ * Note: Only passive mode (host initiates transfer) is currently supported.
+ */
+
+#include <linux/delay.h>
+#include <linux/fpga/fpga-mgr.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/spi/spi.h>
+
+/*
+ * 13 dummy bytes generate 104 SPI clock cycles (8 bits each).
+ * Used to meet the requirement for >100 clock cycles idle sequence.
+ */
+#define EFINIX_SPI_IDLE_CYCLES_BYTES 13
+
+/*
+ * tDMIN: Minimum time between deassertion of CRESET_N to first
+ * valid configuration data. (32 µs)
+ */
+#define EFINIX_TDMIN_US_MIN    35
+#define EFINIX_TDMIN_US_MAX    40
+
+/*
+ * tCRESET_N: Minimum CRESET_N low pulse width required to
+ * trigger re-configuration. (320 ns)
+ */
+#define EFINIX_TCRESETN_DELAY_MIN_US  1
+#define EFINIX_TCRESETN_DELAY_MAX_US  2
+
+/*
+ * tUSER: Minimum configuration duration after CDONE goes high
+ * before entering user mode. (25 µs)
+ */
+#define EFINIX_TUSER_US_MIN    30
+#define EFINIX_TUSER_US_MAX    35
+
+struct efinix_spi_conf {
+	struct spi_device *spi;
+	struct gpio_desc *cdone;
+	struct gpio_desc *reset;
+};
+
+static void efinix_spi_reset(struct efinix_spi_conf *conf)
+{
+	gpiod_set_value(conf->reset, 1);
+	usleep_range(EFINIX_TCRESETN_DELAY_MIN_US, EFINIX_TCRESETN_DELAY_MAX_US);
+	gpiod_set_value(conf->reset, 0);
+	usleep_range(EFINIX_TDMIN_US_MIN, EFINIX_TDMIN_US_MAX);
+}
+
+static enum fpga_mgr_states efinix_spi_state(struct fpga_manager *mgr)
+{
+	struct efinix_spi_conf *conf = mgr->priv;
+
+	if (conf->cdone && gpiod_get_value(conf->cdone) == 1)
+		return FPGA_MGR_STATE_OPERATING;
+
+	return FPGA_MGR_STATE_UNKNOWN;
+}
+
+static int efinix_spi_write_init(struct fpga_manager *mgr,
+				 struct fpga_image_info *info,
+				 const char *buf, size_t count)
+{
+	struct efinix_spi_conf *conf = mgr->priv;
+	struct spi_transfer assert_cs = {
+		/* Keep CS asserted across configuration. */
+		.cs_change = 1,
+	};
+	struct spi_message message;
+	int ret;
+
+	if (info->flags & FPGA_MGR_PARTIAL_RECONFIG) {
+		dev_err(&mgr->dev, "Partial reconfiguration not supported\n");
+		return -EOPNOTSUPP;
+	}
+
+	/*
+	 * Efinix passive SPI configuration requires chip select to stay
+	 * asserted from reset until the bitstream is fully clocked in.
+	 * Lock the SPI bus so no other device can toggle CS between the
+	 * reset pulse and the write/complete transfers.
+	 */
+	spi_bus_lock(conf->spi->controller);
+	spi_message_init_with_transfers(&message, &assert_cs, 1);
+	ret = spi_sync_locked(conf->spi, &message);
+	if (ret) {
+		spi_bus_unlock(conf->spi->controller);
+		return ret;
+	}
+
+	/* Reset with CS asserted */
+	efinix_spi_reset(conf);
+
+	return 0;
+}
+
+static int efinix_spi_write(struct fpga_manager *mgr, const char *buf,
+			    size_t count)
+{
+	struct spi_transfer write_xfer = {
+		.tx_buf = buf,
+		.len = count,
+		.cs_change = 1,
+	};
+	struct efinix_spi_conf *conf = mgr->priv;
+	struct spi_message message;
+	int ret;
+
+	spi_message_init_with_transfers(&message, &write_xfer, 1);
+	ret = spi_sync_locked(conf->spi, &message);
+	if (ret) {
+		dev_err(&mgr->dev, "SPI error in firmware write: %d\n", ret);
+		spi_bus_unlock(conf->spi->controller);
+	}
+
+	return ret;
+}
+
+static int efinix_spi_write_complete(struct fpga_manager *mgr,
+				     struct fpga_image_info *info)
+{
+	unsigned long timeout =
+		jiffies + usecs_to_jiffies(info->config_complete_timeout_us);
+	struct spi_transfer clk_cycles = {
+		.len = EFINIX_SPI_IDLE_CYCLES_BYTES,
+		/* Release CS after the trailing idle clocks are sent. */
+		.cs_change = 0,
+	};
+	struct efinix_spi_conf *conf = mgr->priv;
+	struct spi_message message;
+	int done, ret;
+	bool expired = false;
+	u8 *dummy_buf;
+
+	dummy_buf = kzalloc(EFINIX_SPI_IDLE_CYCLES_BYTES, GFP_KERNEL);
+	if (!dummy_buf) {
+		ret = -ENOMEM;
+		goto unlock_spi;
+	}
+
+	/*
+	 * Keep the bus locked while sending the trailing idle clocks, then
+	 * let this final transfer deassert CS to terminate configuration.
+	 */
+	clk_cycles.tx_buf = dummy_buf;
+	spi_message_init_with_transfers(&message, &clk_cycles, 1);
+	ret = spi_sync_locked(conf->spi, &message);
+	if (ret) {
+		dev_err(&mgr->dev, "SPI error in write complete: %d\n", ret);
+		goto free_buf;
+	}
+
+	if (conf->cdone) {
+		while (!expired) {
+			done = gpiod_get_value(conf->cdone);
+			if (done < 0) {
+				ret = done;
+				goto free_buf;
+			}
+			if (done)
+				break;
+
+			usleep_range(10, 20);
+			expired = time_after(jiffies, timeout);
+		}
+
+		if (expired) {
+			dev_err(&mgr->dev, "Timeout waiting for CDONE\n");
+			ret = -ETIMEDOUT;
+			goto free_buf;
+		}
+	}
+
+	usleep_range(EFINIX_TUSER_US_MIN, EFINIX_TUSER_US_MAX);
+
+free_buf:
+	kfree(dummy_buf);
+unlock_spi:
+	spi_bus_unlock(conf->spi->controller);
+
+	return ret;
+}
+
+static const struct fpga_manager_ops efinix_spi_ops = {
+	.state = efinix_spi_state,
+	.write_init = efinix_spi_write_init,
+	.write = efinix_spi_write,
+	.write_complete = efinix_spi_write_complete,
+};
+
+static int efinix_spi_probe(struct spi_device *spi)
+{
+	struct efinix_spi_conf *conf;
+	struct fpga_manager *mgr;
+
+	if (!(spi->mode & SPI_CPHA) || !(spi->mode & SPI_CPOL))
+		return dev_err_probe(&spi->dev, -EINVAL,
+				     "Unsupported SPI mode, set CPHA and CPOL\n");
+
+	conf = devm_kzalloc(&spi->dev, sizeof(*conf), GFP_KERNEL);
+	if (!conf)
+		return -ENOMEM;
+
+	conf->reset = devm_gpiod_get(&spi->dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(conf->reset))
+		return dev_err_probe(&spi->dev, PTR_ERR(conf->reset),
+				     "Failed to get RESET gpio\n");
+
+	conf->cdone = devm_gpiod_get_optional(&spi->dev, "cdone", GPIOD_IN);
+	if (IS_ERR(conf->cdone))
+		return dev_err_probe(&spi->dev, PTR_ERR(conf->cdone),
+				     "Failed to get CDONE gpio\n");
+
+	conf->spi = spi;
+
+	mgr = devm_fpga_mgr_register(&spi->dev,
+				     "Efinix FPGA Manager",
+				     &efinix_spi_ops, conf);
+
+	return PTR_ERR_OR_ZERO(mgr);
+}
+
+static const struct of_device_id efinix_spi_of_match[] = {
+	{ .compatible = "efinix,trion-config", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, efinix_spi_of_match);
+
+static const struct spi_device_id efinix_ids[] = {
+	{ "trion-config", 0 },
+	{},
+};
+MODULE_DEVICE_TABLE(spi, efinix_ids);
+
+static struct spi_driver efinix_spi_driver = {
+	.driver = {
+		.name = "efinix-spi",
+		.of_match_table = efinix_spi_of_match,
+	},
+	.probe = efinix_spi_probe,
+	.id_table = efinix_ids,
+};
+
+module_spi_driver(efinix_spi_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Ian Dannapel <iansdannapel@gmail.com>");
+MODULE_DESCRIPTION("Efinix FPGA SPI Programming Driver");
diff --git a/drivers/fpga/lattice-sysconfig-spi.c b/drivers/fpga/lattice-sysconfig-spi.c
index 44691cfcf50a5..5d195602b2615 100644
--- a/drivers/fpga/lattice-sysconfig-spi.c
+++ b/drivers/fpga/lattice-sysconfig-spi.c
@@ -3,8 +3,14 @@
  * Lattice FPGA programming over slave SPI sysCONFIG interface.
  */
 
-#include <linux/of.h>
+#include <linux/dev_printk.h>
+#include <linux/device/devres.h>
+#include <linux/errno.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/slab.h>
 #include <linux/spi/spi.h>
+#include <linux/types.h>
 
 #include "lattice-sysconfig.h"
 
@@ -85,7 +91,6 @@ static int sysconfig_spi_bitstream_burst_complete(struct sysconfig_priv *priv)
 
 static int sysconfig_spi_probe(struct spi_device *spi)
 {
-	const struct spi_device_id *dev_id;
 	struct device *dev = &spi->dev;
 	struct sysconfig_priv *priv;
 	const u32 *spi_max_speed;
@@ -94,15 +99,7 @@ static int sysconfig_spi_probe(struct spi_device *spi)
 	if (!priv)
 		return -ENOMEM;
 
-	spi_max_speed = device_get_match_data(dev);
-	if (!spi_max_speed) {
-		dev_id = spi_get_device_id(spi);
-		if (!dev_id)
-			return -ENODEV;
-
-		spi_max_speed = (const u32 *)dev_id->driver_data;
-	}
-
+	spi_max_speed = spi_get_device_match_data(spi);
 	if (!spi_max_speed)
 		return -EINVAL;
 
@@ -125,26 +122,26 @@ static const struct spi_device_id sysconfig_spi_ids[] = {
 	{
 		.name = "sysconfig-ecp5",
 		.driver_data = (kernel_ulong_t)&ecp5_spi_max_speed_hz,
-	}, {},
+	},
+	{}
 };
 MODULE_DEVICE_TABLE(spi, sysconfig_spi_ids);
 
-#if IS_ENABLED(CONFIG_OF)
 static const struct of_device_id sysconfig_of_ids[] = {
 	{
 		.compatible = "lattice,sysconfig-ecp5",
 		.data = &ecp5_spi_max_speed_hz,
-	}, {},
+	},
+	{}
 };
 MODULE_DEVICE_TABLE(of, sysconfig_of_ids);
-#endif /* IS_ENABLED(CONFIG_OF) */
 
 static struct spi_driver lattice_sysconfig_driver = {
 	.probe = sysconfig_spi_probe,
 	.id_table = sysconfig_spi_ids,
 	.driver = {
 		.name = "lattice_sysconfig_spi_fpga_mgr",
-		.of_match_table = of_match_ptr(sysconfig_of_ids),
+		.of_match_table = sysconfig_of_ids,
 	},
 };
 module_spi_driver(lattice_sysconfig_driver);
diff --git a/drivers/fpga/of-fpga-region.c b/drivers/fpga/of-fpga-region.c
index caa091224dc54..9107a5b461d35 100644
--- a/drivers/fpga/of-fpga-region.c
+++ b/drivers/fpga/of-fpga-region.c
@@ -168,11 +168,10 @@ static int child_regions_with_firmware(struct device_node *overlay)
 						     fpga_region_of_match);
 	}
 
-	of_node_put(child_region);
-
 	if (ret)
 		pr_err("firmware-name not allowed in child FPGA region: %pOF",
 		       child_region);
+	of_node_put(child_region);
 
 	return ret;
 }
diff --git a/drivers/fpga/ts73xx-fpga.c b/drivers/fpga/ts73xx-fpga.c
index 4e1d2a4d3df44..3460e4809f86f 100644
--- a/drivers/fpga/ts73xx-fpga.c
+++ b/drivers/fpga/ts73xx-fpga.c
@@ -11,6 +11,7 @@
 #include <linux/delay.h>
 #include <linux/io.h>
 #include <linux/module.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 #include <linux/string.h>
 #include <linux/iopoll.h>
@@ -119,9 +120,17 @@ static int ts73xx_fpga_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(mgr);
 }
 
+static const struct of_device_id ts73xx_fpga_of_match[] = {
+	{ .compatible = "technologic,ts7300-fpga" },
+	{},
+};
+
+MODULE_DEVICE_TABLE(of, ts73xx_fpga_of_match);
+
 static struct platform_driver ts73xx_fpga_driver = {
 	.driver	= {
 		.name	= "ts73xx-fpga-mgr",
+		.of_match_table = ts73xx_fpga_of_match,
 	},
 	.probe	= ts73xx_fpga_probe,
 };
diff --git a/drivers/fwctl/pds/main.c b/drivers/fwctl/pds/main.c
index 08872ee8422f8..68fe254dd10ad 100644
--- a/drivers/fwctl/pds/main.c
+++ b/drivers/fwctl/pds/main.c
@@ -362,6 +362,9 @@ static void *pdsfc_fw_rpc(struct fwctl_uctx *uctx, enum fwctl_rpc_scope scope,
 	void *out = NULL;
 	int err;
 
+	if (in_len < sizeof(*rpc))
+		return ERR_PTR(-EINVAL);
+
 	err = pdsfc_validate_rpc(pdsfc, rpc, scope);
 	if (err)
 		return ERR_PTR(err);
diff --git a/drivers/gpio/Kconfig b/drivers/gpio/Kconfig
index 020e51e30317a..9ea5c2523fd42 100644
--- a/drivers/gpio/Kconfig
+++ b/drivers/gpio/Kconfig
@@ -156,6 +156,7 @@ config GPIO_74XX_MMIO
 
 config GPIO_ALTERA
 	tristate "Altera GPIO"
+	select GPIO_GENERIC
 	select GPIOLIB_IRQCHIP
 	help
 	  Say Y or M here to build support for the Altera PIO device.
@@ -301,7 +302,7 @@ config GPIO_EM
 
 config GPIO_EN7523
 	tristate "Airoha GPIO support"
-	depends on ARCH_AIROHA
+	depends on ARCH_AIROHA || COMPILE_TEST
 	default ARCH_AIROHA
 	select GPIO_GENERIC
 	select GPIOLIB_IRQCHIP
@@ -698,7 +699,7 @@ config GPIO_SPACEMIT_K1
 
 config GPIO_SPEAR_SPICS
 	bool "ST SPEAr13xx SPI Chip Select as GPIO support"
-	depends on PLAT_SPEAR
+	depends on PLAT_SPEAR || COMPILE_TEST
 	select GENERIC_IRQ_CHIP
 	help
 	  Say yes here to support ST SPEAr SPI Chip Select as GPIO device.
@@ -805,8 +806,18 @@ config GPIO_VISCONTI
 	help
 	  Say yes here to support GPIO on Tohisba Visconti.
 
+config GPIO_WAVESHARE_DSI_TOUCH
+	tristate "Waveshare GPIO controller for DSI panels"
+	depends on BACKLIGHT_CLASS_DEVICE
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Enable support for the GPIO and PWM controller found on Waveshare DSI
+	  TOUCH panel kits. It provides GPIOs (used for regulator control and
+          resets) and backlight support.
+
 config GPIO_WCD934X
-	tristate "Qualcomm Technologies Inc WCD9340/WCD9341 GPIO controller driver"
+	tristate "Qualcomm WCD9340/WCD9341 GPIO controller driver"
 	depends on MFD_WCD934X
 	help
          This driver is to support GPIO block found on the Qualcomm Technologies
@@ -814,7 +825,7 @@ config GPIO_WCD934X
 
 config GPIO_XGENE
 	bool "APM X-Gene GPIO controller support"
-	depends on ARM64
+	depends on ARM64 || COMPILE_TEST
 	help
 	  This driver is to support the GPIO block within the APM X-Gene SoC
 	  platform's generic flash controller. The GPIO pins are muxed with
@@ -858,7 +869,7 @@ config GPIO_XTENSA
 
 config GPIO_ZEVIO
 	bool "LSI ZEVIO SoC memory mapped GPIOs"
-	depends on ARM
+	depends on ARM || COMPILE_TEST
 	help
 	  Say yes here to support the GPIO controller in LSI ZEVIO SoCs.
 
@@ -2060,6 +2071,7 @@ config GPIO_VIRTIO
 
 config GPIO_SIM
 	tristate "GPIO Simulator Module"
+	depends on SYSFS
 	select IRQ_SIM
 	select CONFIGFS_FS
 	help
diff --git a/drivers/gpio/Makefile b/drivers/gpio/Makefile
index b267598b517de..2ea47d9d3dca9 100644
--- a/drivers/gpio/Makefile
+++ b/drivers/gpio/Makefile
@@ -205,6 +205,7 @@ obj-$(CONFIG_GPIO_VIRTUSER)		+= gpio-virtuser.o
 obj-$(CONFIG_GPIO_VIRTIO)		+= gpio-virtio.o
 obj-$(CONFIG_GPIO_VISCONTI)		+= gpio-visconti.o
 obj-$(CONFIG_GPIO_VX855)		+= gpio-vx855.o
+obj-$(CONFIG_GPIO_WAVESHARE_DSI_TOUCH)	+= gpio-waveshare-dsi.o
 obj-$(CONFIG_GPIO_WCD934X)		+= gpio-wcd934x.o
 obj-$(CONFIG_GPIO_WHISKEY_COVE)		+= gpio-wcove.o
 obj-$(CONFIG_GPIO_WINBOND)		+= gpio-winbond.o
diff --git a/drivers/gpio/gpio-74x164.c b/drivers/gpio/gpio-74x164.c
index c226524efebab..5ca61cf5206aa 100644
--- a/drivers/gpio/gpio-74x164.c
+++ b/drivers/gpio/gpio-74x164.c
@@ -112,7 +112,7 @@ static int gen_74x164_probe(struct spi_device *spi)
 {
 	struct device *dev = &spi->dev;
 	struct gen_74x164_chip *chip;
-	u32 nregs;
+	u32 nregs, init_state;
 	int ret;
 
 	/*
@@ -134,6 +134,21 @@ static int gen_74x164_probe(struct spi_device *spi)
 
 	chip->registers = nregs;
 
+	/*
+	 * Optionally seed the chain with a board-specified pattern so the
+	 * outputs come up in a known state on the first SPI write. The
+	 * property follows the nxp,pcf8575 convention where bit N maps to
+	 * GPIO line N. On this output-only device, bit=0 drives the line
+	 * low and bit=1 drives it high. The bitmask covers up to 32 lines;
+	 * any further outputs come up zeroed by devm_kzalloc().
+	 */
+	if (!device_property_read_u32(dev, "lines-initial-states", &init_state)) {
+		unsigned int i;
+
+		for (i = 0; i < min(nregs, 4U); i++)
+			chip->buffer[nregs - 1 - i] = (init_state >> (i * 8)) & 0xff;
+	}
+
 	chip->gpiod_oe = devm_gpiod_get_optional(dev, "enable", GPIOD_OUT_LOW);
 	if (IS_ERR(chip->gpiod_oe))
 		return PTR_ERR(chip->gpiod_oe);
diff --git a/drivers/gpio/gpio-adnp.c b/drivers/gpio/gpio-adnp.c
index e5ac2d2110137..27a80d1143a19 100644
--- a/drivers/gpio/gpio-adnp.c
+++ b/drivers/gpio/gpio-adnp.c
@@ -499,8 +499,8 @@ static int adnp_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adnp_i2c_id[] = {
-	{ "gpio-adnp" },
-	{ },
+	{ .name = "gpio-adnp" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adnp_i2c_id);
 
diff --git a/drivers/gpio/gpio-altera.c b/drivers/gpio/gpio-altera.c
index 9508d764cce40..fe144360a88d8 100644
--- a/drivers/gpio/gpio-altera.c
+++ b/drivers/gpio/gpio-altera.c
@@ -17,6 +17,7 @@
 #include <linux/types.h>
 
 #include <linux/gpio/driver.h>
+#include <linux/gpio/generic.h>
 
 #define ALTERA_GPIO_MAX_NGPIO		32
 #define ALTERA_GPIO_DATA		0x0
@@ -26,7 +27,7 @@
 
 /**
 * struct altera_gpio_chip
-* @gc			: GPIO chip structure.
+* @chip			: Generic GPIO chip structure.
 * @regs			: memory mapped IO address for the controller registers.
 * @gpio_lock		: synchronization lock so that new irq/set/get requests
 *			  will be blocked until the current one completes.
@@ -34,7 +35,7 @@
 *			  (rising, falling, both, high)
 */
 struct altera_gpio_chip {
-	struct gpio_chip gc;
+	struct gpio_generic_chip chip;
 	void __iomem *regs;
 	raw_spinlock_t gpio_lock;
 	int interrupt_trigger;
@@ -106,72 +107,6 @@ static unsigned int altera_gpio_irq_startup(struct irq_data *d)
 	return 0;
 }
 
-static int altera_gpio_get(struct gpio_chip *gc, unsigned offset)
-{
-	struct altera_gpio_chip *altera_gc = gpiochip_get_data(gc);
-
-	return !!(readl(altera_gc->regs + ALTERA_GPIO_DATA) & BIT(offset));
-}
-
-static int altera_gpio_set(struct gpio_chip *gc, unsigned int offset, int value)
-{
-	struct altera_gpio_chip *altera_gc = gpiochip_get_data(gc);
-	unsigned long flags;
-	unsigned int data_reg;
-
-	raw_spin_lock_irqsave(&altera_gc->gpio_lock, flags);
-	data_reg = readl(altera_gc->regs + ALTERA_GPIO_DATA);
-	if (value)
-		data_reg |= BIT(offset);
-	else
-		data_reg &= ~BIT(offset);
-	writel(data_reg, altera_gc->regs + ALTERA_GPIO_DATA);
-	raw_spin_unlock_irqrestore(&altera_gc->gpio_lock, flags);
-
-	return 0;
-}
-
-static int altera_gpio_direction_input(struct gpio_chip *gc, unsigned offset)
-{
-	struct altera_gpio_chip *altera_gc = gpiochip_get_data(gc);
-	unsigned long flags;
-	unsigned int gpio_ddr;
-
-	raw_spin_lock_irqsave(&altera_gc->gpio_lock, flags);
-	/* Set pin as input, assumes software controlled IP */
-	gpio_ddr = readl(altera_gc->regs + ALTERA_GPIO_DIR);
-	gpio_ddr &= ~BIT(offset);
-	writel(gpio_ddr, altera_gc->regs + ALTERA_GPIO_DIR);
-	raw_spin_unlock_irqrestore(&altera_gc->gpio_lock, flags);
-
-	return 0;
-}
-
-static int altera_gpio_direction_output(struct gpio_chip *gc,
-		unsigned offset, int value)
-{
-	struct altera_gpio_chip *altera_gc = gpiochip_get_data(gc);
-	unsigned long flags;
-	unsigned int data_reg, gpio_ddr;
-
-	raw_spin_lock_irqsave(&altera_gc->gpio_lock, flags);
-	/* Sets the GPIO value */
-	data_reg = readl(altera_gc->regs + ALTERA_GPIO_DATA);
-	if (value)
-		data_reg |= BIT(offset);
-	else
-		data_reg &= ~BIT(offset);
-	writel(data_reg, altera_gc->regs + ALTERA_GPIO_DATA);
-
-	/* Set pin as output, assumes software controlled IP */
-	gpio_ddr = readl(altera_gc->regs + ALTERA_GPIO_DIR);
-	gpio_ddr |= BIT(offset);
-	writel(gpio_ddr, altera_gc->regs + ALTERA_GPIO_DIR);
-	raw_spin_unlock_irqrestore(&altera_gc->gpio_lock, flags);
-
-	return 0;
-}
-
 static void altera_gpio_irq_edge_handler(struct irq_desc *desc)
 {
 	struct gpio_chip *gc = irq_desc_get_handler_data(desc);
@@ -231,9 +166,12 @@ static const struct irq_chip altera_gpio_irq_chip = {
 
 static int altera_gpio_probe(struct platform_device *pdev)
 {
+	struct gpio_generic_chip_config config;
 	struct device *dev = &pdev->dev;
 	int reg, ret;
 	struct altera_gpio_chip *altera_gc;
+	struct gpio_generic_chip *chip;
+	struct gpio_chip *gc;
 	struct gpio_irq_chip *girq;
 	int mapped_irq;
 
@@ -243,35 +181,45 @@ static int altera_gpio_probe(struct platform_device *pdev)
 
 	raw_spin_lock_init(&altera_gc->gpio_lock);
 
+	altera_gc->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(altera_gc->regs))
+		return dev_err_probe(dev, PTR_ERR(altera_gc->regs),
+				     "failed to ioremap memory resource\n");
+
+	chip = &altera_gc->chip;
+
+	config = (struct gpio_generic_chip_config) {
+		.dev = dev,
+		.sz = 4,
+		.dat = altera_gc->regs + ALTERA_GPIO_DATA,
+		.set = altera_gc->regs + ALTERA_GPIO_DATA,
+		.dirout = altera_gc->regs + ALTERA_GPIO_DIR,
+	};
+
+	ret = gpio_generic_chip_init(chip, &config);
+	if (ret)
+		return dev_err_probe(dev, ret, "unable to init generic GPIO\n");
+
+	gc = &chip->gc;
+
 	if (device_property_read_u32(dev, "altr,ngpio", &reg))
 		/* By default assume maximum ngpio */
-		altera_gc->gc.ngpio = ALTERA_GPIO_MAX_NGPIO;
+		gc->ngpio = ALTERA_GPIO_MAX_NGPIO;
 	else
-		altera_gc->gc.ngpio = reg;
+		gc->ngpio = reg;
 
-	if (altera_gc->gc.ngpio > ALTERA_GPIO_MAX_NGPIO) {
+	if (gc->ngpio > ALTERA_GPIO_MAX_NGPIO) {
 		dev_warn(&pdev->dev,
 			"ngpio is greater than %d, defaulting to %d\n",
 			ALTERA_GPIO_MAX_NGPIO, ALTERA_GPIO_MAX_NGPIO);
-		altera_gc->gc.ngpio = ALTERA_GPIO_MAX_NGPIO;
+		gc->ngpio = ALTERA_GPIO_MAX_NGPIO;
 	}
 
-	altera_gc->gc.direction_input	= altera_gpio_direction_input;
-	altera_gc->gc.direction_output	= altera_gpio_direction_output;
-	altera_gc->gc.get		= altera_gpio_get;
-	altera_gc->gc.set		= altera_gpio_set;
-	altera_gc->gc.owner		= THIS_MODULE;
-	altera_gc->gc.parent		= &pdev->dev;
-	altera_gc->gc.base		= -1;
-
-	altera_gc->gc.label = devm_kasprintf(dev, GFP_KERNEL, "%pfw", dev_fwnode(dev));
-	if (!altera_gc->gc.label)
+	gc->base = -1;
+	gc->label = devm_kasprintf(dev, GFP_KERNEL, "%pfw", dev_fwnode(dev));
+	if (!gc->label)
 		return -ENOMEM;
 
-	altera_gc->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(altera_gc->regs))
-		return dev_err_probe(dev, PTR_ERR(altera_gc->regs), "failed to ioremap memory resource\n");
-
 	mapped_irq = platform_get_irq_optional(pdev, 0);
 	if (mapped_irq < 0)
 		goto skip_irq;
@@ -283,7 +231,7 @@ static int altera_gpio_probe(struct platform_device *pdev)
 	}
 	altera_gc->interrupt_trigger = reg;
 
-	girq = &altera_gc->gc.irq;
+	girq = &gc->irq;
 	gpio_irq_chip_set_chip(girq, &altera_gpio_irq_chip);
 
 	if (altera_gc->interrupt_trigger == IRQ_TYPE_LEVEL_HIGH)
@@ -300,7 +248,7 @@ static int altera_gpio_probe(struct platform_device *pdev)
 	girq->parents[0] = mapped_irq;
 
 skip_irq:
-	ret = devm_gpiochip_add_data(dev, &altera_gc->gc, altera_gc);
+	ret = devm_gpiochip_add_data(dev, gc, altera_gc);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed adding memory mapped gpiochip\n");
 		return ret;
diff --git a/drivers/gpio/gpio-amd8111.c b/drivers/gpio/gpio-amd8111.c
index 15fd5e210d747..8078b5d7b80c9 100644
--- a/drivers/gpio/gpio-amd8111.c
+++ b/drivers/gpio/gpio-amd8111.c
@@ -59,8 +59,8 @@
  * want to register another driver on the same PCI id.
  */
 static const struct pci_device_id pci_tbl[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8111_SMBUS), 0 },
-	{ 0, },	/* terminate list */
+	{ PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_8111_SMBUS) },
+	{ },	/* terminate list */
 };
 MODULE_DEVICE_TABLE(pci, pci_tbl);
 
diff --git a/drivers/gpio/gpio-ath79.c b/drivers/gpio/gpio-ath79.c
index 2ad9f6ac66362..85bd994d15d48 100644
--- a/drivers/gpio/gpio-ath79.c
+++ b/drivers/gpio/gpio-ath79.c
@@ -11,6 +11,7 @@
 #include <linux/device.h>
 #include <linux/gpio/driver.h>
 #include <linux/gpio/generic.h>
+#include <linux/gpio/machine.h> /* For WLAN GPIOs */
 #include <linux/interrupt.h>
 #include <linux/irq.h>
 #include <linux/mod_devicetable.h>
@@ -214,6 +215,56 @@ static const struct of_device_id ath79_gpio_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, ath79_gpio_of_match);
 
+#if IS_ENABLED(CONFIG_ATH9K_AHB)
+/*
+ * This registers all of the ath79k GPIOs as descriptors to be picked
+ * directly from the ATH79K wifi driver if the two are jitted together
+ * in the same SoC.
+ */
+#define ATH79K_WIFI_DESCS 32
+static int ath79_gpio_register_wifi_descriptors(struct device *dev,
+						const char *label)
+{
+	struct gpiod_lookup_table *lookup;
+	int i;
+
+	/* Create a gpiod lookup using gpiochip-local offsets + 1 for NULL */
+	lookup = devm_kzalloc(dev,
+			      struct_size(lookup, table, ATH79K_WIFI_DESCS + 1),
+			      GFP_KERNEL);
+	if (!lookup)
+		return -ENOMEM;
+
+	/*
+	 * Ugly system-wide lookup for the NULL device: we know this
+	 * is already NULL but explicitly assign it here for people to
+	 * know what is going on. (Yes this is an ugly legacy hack, live
+	 * with it.)
+	 */
+	lookup->dev_id = NULL;
+
+	for (i = 0; i < ATH79K_WIFI_DESCS; i++) {
+		lookup->table[i] =
+			/*
+			 * Set the HW offset on the chip and the lookup
+			 * index to the same value, so looking up index 0
+			 * will get HW offset 0, index 1 HW offset 1 etc.
+			 */
+			GPIO_LOOKUP_IDX(label, i, "ath9k", i, GPIO_ACTIVE_HIGH);
+	}
+
+	gpiod_add_lookup_table(lookup);
+
+	return 0;
+}
+#else
+static int ath79_gpio_register_wifi_descriptors(struct device *dev,
+						const char *label)
+{
+	return 0;
+}
+#endif
+
 static int ath79_gpio_probe(struct platform_device *pdev)
 {
 	struct gpio_generic_chip_config config;
@@ -276,7 +327,11 @@ static int ath79_gpio_probe(struct platform_device *pdev)
 		girq->handler = handle_simple_irq;
 	}
 
-	return devm_gpiochip_add_data(dev, &ctrl->chip.gc, ctrl);
+	err = devm_gpiochip_add_data(dev, &ctrl->chip.gc, ctrl);
+	if (err)
+		return err;
+
+	return ath79_gpio_register_wifi_descriptors(dev, ctrl->chip.gc.label);
 }
 
 static struct platform_driver ath79_gpio_driver = {
diff --git a/drivers/gpio/gpio-ds4520.c b/drivers/gpio/gpio-ds4520.c
index f52ecae382a45..5add662a7ef51 100644
--- a/drivers/gpio/gpio-ds4520.c
+++ b/drivers/gpio/gpio-ds4520.c
@@ -54,7 +54,7 @@ static const struct of_device_id ds4520_gpio_of_match_table[] = {
 MODULE_DEVICE_TABLE(of, ds4520_gpio_of_match_table);
 
 static const struct i2c_device_id ds4520_gpio_id_table[] = {
-	{ "ds4520-gpio" },
+	{ .name = "ds4520-gpio" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ds4520_gpio_id_table);
diff --git a/drivers/gpio/gpio-ep93xx.c b/drivers/gpio/gpio-ep93xx.c
index 1f56e44ffc9a3..8784e433e1ffb 100644
--- a/drivers/gpio/gpio-ep93xx.c
+++ b/drivers/gpio/gpio-ep93xx.c
@@ -323,8 +323,7 @@ static int ep93xx_setup_irqs(struct platform_device *pdev,
 	}
 
 	girq->default_type = IRQ_TYPE_NONE;
-	/* TODO: replace with handle_bad_irq() once we are fully hierarchical */
-	girq->handler = handle_simple_irq;
+	girq->handler = handle_bad_irq;
 
 	return 0;
 }
diff --git a/drivers/gpio/gpio-fxl6408.c b/drivers/gpio/gpio-fxl6408.c
index afc1b8461dabb..45b02d36e66ff 100644
--- a/drivers/gpio/gpio-fxl6408.c
+++ b/drivers/gpio/gpio-fxl6408.c
@@ -150,7 +150,7 @@ static const __maybe_unused struct of_device_id fxl6408_dt_ids[] = {
 MODULE_DEVICE_TABLE(of, fxl6408_dt_ids);
 
 static const struct i2c_device_id fxl6408_id[] = {
-	{ "fxl6408" },
+	{ .name = "fxl6408" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, fxl6408_id);
diff --git a/drivers/gpio/gpio-gw-pld.c b/drivers/gpio/gpio-gw-pld.c
index 2e5d97b7363fd..bf1f91c3c4a8b 100644
--- a/drivers/gpio/gpio-gw-pld.c
+++ b/drivers/gpio/gpio-gw-pld.c
@@ -109,7 +109,7 @@ static int gw_pld_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id gw_pld_id[] = {
-	{ "gw-pld", },
+	{ .name = "gw-pld" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, gw_pld_id);
diff --git a/drivers/gpio/gpio-ixp4xx.c b/drivers/gpio/gpio-ixp4xx.c
index f34d87869c8b0..669b139cd499b 100644
--- a/drivers/gpio/gpio-ixp4xx.c
+++ b/drivers/gpio/gpio-ixp4xx.c
@@ -311,12 +311,7 @@ static int ixp4xx_gpio_probe(struct platform_device *pdev)
 	}
 	g->chip.gc.ngpio = 16;
 	g->chip.gc.label = "IXP4XX_GPIO_CHIP";
-	/*
-	 * TODO: when we have migrated to device tree and all GPIOs
-	 * are fetched using phandles, set this to -1 to get rid of
-	 * the fixed gpiochip base.
-	 */
-	g->chip.gc.base = 0;
+	g->chip.gc.base = -1;
 	g->chip.gc.parent = &pdev->dev;
 	g->chip.gc.owner = THIS_MODULE;
 
diff --git a/drivers/gpio/gpio-max7300.c b/drivers/gpio/gpio-max7300.c
index 621d609ece904..62f2434c0d79b 100644
--- a/drivers/gpio/gpio-max7300.c
+++ b/drivers/gpio/gpio-max7300.c
@@ -53,7 +53,7 @@ static void max7300_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max7300_id[] = {
-	{ "max7300" },
+	{ .name = "max7300" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max7300_id);
diff --git a/drivers/gpio/gpio-max732x.c b/drivers/gpio/gpio-max732x.c
index 281ba1740a6a0..24c67c9129549 100644
--- a/drivers/gpio/gpio-max732x.c
+++ b/drivers/gpio/gpio-max732x.c
@@ -103,16 +103,16 @@ static uint64_t max732x_features[] = {
 };
 
 static const struct i2c_device_id max732x_id[] = {
-	{ "max7319", MAX7319 },
-	{ "max7320", MAX7320 },
-	{ "max7321", MAX7321 },
-	{ "max7322", MAX7322 },
-	{ "max7323", MAX7323 },
-	{ "max7324", MAX7324 },
-	{ "max7325", MAX7325 },
-	{ "max7326", MAX7326 },
-	{ "max7327", MAX7327 },
-	{ },
+	{ .name = "max7319", .driver_data = MAX7319 },
+	{ .name = "max7320", .driver_data = MAX7320 },
+	{ .name = "max7321", .driver_data = MAX7321 },
+	{ .name = "max7322", .driver_data = MAX7322 },
+	{ .name = "max7323", .driver_data = MAX7323 },
+	{ .name = "max7324", .driver_data = MAX7324 },
+	{ .name = "max7325", .driver_data = MAX7325 },
+	{ .name = "max7326", .driver_data = MAX7326 },
+	{ .name = "max7327", .driver_data = MAX7327 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max732x_id);
 
diff --git a/drivers/gpio/gpio-pca953x.c b/drivers/gpio/gpio-pca953x.c
index 52e96cc5f67bb..2ee35e855e4da 100644
--- a/drivers/gpio/gpio-pca953x.c
+++ b/drivers/gpio/gpio-pca953x.c
@@ -86,49 +86,49 @@
 #define PCA_CHIP_TYPE(x)	((x) & PCA_TYPE_MASK)
 
 static const struct i2c_device_id pca953x_id[] = {
-	{ "pca6408", 8  | PCA953X_TYPE | PCA_INT, },
-	{ "pca6416", 16 | PCA953X_TYPE | PCA_INT, },
-	{ "pca9505", 40 | PCA953X_TYPE | PCA_INT, },
-	{ "pca9506", 40 | PCA953X_TYPE | PCA_INT, },
-	{ "pca9534", 8  | PCA953X_TYPE | PCA_INT, },
-	{ "pca9535", 16 | PCA953X_TYPE | PCA_INT, },
-	{ "pca9536", 4  | PCA953X_TYPE, },
-	{ "pca9537", 4  | PCA953X_TYPE | PCA_INT, },
-	{ "pca9538", 8  | PCA953X_TYPE | PCA_INT, },
-	{ "pca9539", 16 | PCA953X_TYPE | PCA_INT, },
-	{ "pca9554", 8  | PCA953X_TYPE | PCA_INT, },
-	{ "pca9555", 16 | PCA953X_TYPE | PCA_INT, },
-	{ "pca9556", 8  | PCA953X_TYPE, },
-	{ "pca9557", 8  | PCA953X_TYPE, },
-	{ "pca9574", 8  | PCA957X_TYPE | PCA_INT, },
-	{ "pca9575", 16 | PCA957X_TYPE | PCA_INT, },
-	{ "pca9698", 40 | PCA953X_TYPE, },
-
-	{ "pcal6408", 8 | PCA953X_TYPE | PCA_LATCH_INT, },
-	{ "pcal6416", 16 | PCA953X_TYPE | PCA_LATCH_INT, },
-	{ "pcal6524", 24 | PCA953X_TYPE | PCA_LATCH_INT, },
-	{ "pcal6534", 34 | PCAL653X_TYPE | PCA_LATCH_INT, },
-	{ "pcal9535", 16 | PCA953X_TYPE | PCA_LATCH_INT, },
-	{ "pcal9554b", 8  | PCA953X_TYPE | PCA_LATCH_INT, },
-	{ "pcal9555a", 16 | PCA953X_TYPE | PCA_LATCH_INT, },
-
-	{ "max7310", 8  | PCA953X_TYPE, },
-	{ "max7312", 16 | PCA953X_TYPE | PCA_INT, },
-	{ "max7313", 16 | PCA953X_TYPE | PCA_INT, },
-	{ "max7315", 8  | PCA953X_TYPE | PCA_INT, },
-	{ "max7318", 16 | PCA953X_TYPE | PCA_INT, },
-	{ "pca6107", 8  | PCA953X_TYPE | PCA_INT, },
-	{ "tca6408", 8  | PCA953X_TYPE | PCA_INT, },
-	{ "tca6416", 16 | PCA953X_TYPE | PCA_INT, },
-	{ "tca6418", 18 | TCA6418_TYPE | PCA_INT, },
-	{ "tca6424", 24 | PCA953X_TYPE | PCA_INT, },
-	{ "tca9538", 8  | PCA953X_TYPE | PCA_INT, },
-	{ "tca9539", 16 | PCA953X_TYPE | PCA_INT, },
-	{ "tca9554", 8  | PCA953X_TYPE | PCA_INT, },
-	{ "xra1202", 8  | PCA953X_TYPE },
-
-	{ "tcal6408", 8  | PCA953X_TYPE | PCA_LATCH_INT, },
-	{ "tcal6416", 16 | PCA953X_TYPE | PCA_LATCH_INT, },
+	{ .name = "pca6408", .driver_data = 8  | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca6416", .driver_data = 16 | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9505", .driver_data = 40 | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9506", .driver_data = 40 | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9534", .driver_data = 8  | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9535", .driver_data = 16 | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9536", .driver_data = 4  | PCA953X_TYPE },
+	{ .name = "pca9537", .driver_data = 4  | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9538", .driver_data = 8  | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9539", .driver_data = 16 | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9554", .driver_data = 8  | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9555", .driver_data = 16 | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca9556", .driver_data = 8  | PCA953X_TYPE },
+	{ .name = "pca9557", .driver_data = 8  | PCA953X_TYPE },
+	{ .name = "pca9574", .driver_data = 8  | PCA957X_TYPE | PCA_INT },
+	{ .name = "pca9575", .driver_data = 16 | PCA957X_TYPE | PCA_INT },
+	{ .name = "pca9698", .driver_data = 40 | PCA953X_TYPE },
+
+	{ .name = "pcal6408",  .driver_data = 8  | PCA953X_TYPE | PCA_LATCH_INT },
+	{ .name = "pcal6416",  .driver_data = 16 | PCA953X_TYPE | PCA_LATCH_INT },
+	{ .name = "pcal6524",  .driver_data = 24 | PCA953X_TYPE | PCA_LATCH_INT },
+	{ .name = "pcal6534",  .driver_data = 34 | PCAL653X_TYPE | PCA_LATCH_INT },
+	{ .name = "pcal9535",  .driver_data = 16 | PCA953X_TYPE | PCA_LATCH_INT },
+	{ .name = "pcal9554b", .driver_data = 8  | PCA953X_TYPE | PCA_LATCH_INT },
+	{ .name = "pcal9555a", .driver_data = 16 | PCA953X_TYPE | PCA_LATCH_INT },
+
+	{ .name = "max7310", .driver_data = 8  | PCA953X_TYPE },
+	{ .name = "max7312", .driver_data = 16 | PCA953X_TYPE | PCA_INT },
+	{ .name = "max7313", .driver_data = 16 | PCA953X_TYPE | PCA_INT },
+	{ .name = "max7315", .driver_data = 8  | PCA953X_TYPE | PCA_INT },
+	{ .name = "max7318", .driver_data = 16 | PCA953X_TYPE | PCA_INT },
+	{ .name = "pca6107", .driver_data = 8  | PCA953X_TYPE | PCA_INT },
+	{ .name = "tca6408", .driver_data = 8  | PCA953X_TYPE | PCA_INT },
+	{ .name = "tca6416", .driver_data = 16 | PCA953X_TYPE | PCA_INT },
+	{ .name = "tca6418", .driver_data = 18 | TCA6418_TYPE | PCA_INT },
+	{ .name = "tca6424", .driver_data = 24 | PCA953X_TYPE | PCA_INT },
+	{ .name = "tca9538", .driver_data = 8  | PCA953X_TYPE | PCA_INT },
+	{ .name = "tca9539", .driver_data = 16 | PCA953X_TYPE | PCA_INT },
+	{ .name = "tca9554", .driver_data = 8  | PCA953X_TYPE | PCA_INT },
+	{ .name = "xra1202", .driver_data = 8  | PCA953X_TYPE },
+
+	{ .name = "tcal6408", .driver_data = 8  | PCA953X_TYPE | PCA_LATCH_INT },
+	{ .name = "tcal6416", .driver_data = 16 | PCA953X_TYPE | PCA_LATCH_INT },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pca953x_id);
@@ -877,11 +877,9 @@ static void pca953x_irq_bus_sync_unlock(struct irq_data *d)
 	bitmap_or(irq_mask, chip->irq_trig_fall, chip->irq_trig_raise, gc->ngpio);
 	bitmap_or(irq_mask, irq_mask, chip->irq_trig_level_high, gc->ngpio);
 	bitmap_or(irq_mask, irq_mask, chip->irq_trig_level_low, gc->ngpio);
-	bitmap_complement(reg_direction, reg_direction, gc->ngpio);
-	bitmap_and(irq_mask, irq_mask, reg_direction, gc->ngpio);
 
 	/* Look for any newly setup interrupt */
-	for_each_set_bit(level, irq_mask, gc->ngpio)
+	for_each_andnot_bit(level, irq_mask, reg_direction, gc->ngpio)
 		pca953x_gpio_direction_input(&chip->gpio_chip, level);
 
 	mutex_unlock(&chip->irq_lock);
@@ -1005,8 +1003,7 @@ static bool pca953x_irq_pending(struct pca953x_chip *chip, unsigned long *pendin
 	bitmap_and(cur_stat, cur_stat, chip->irq_mask, gc->ngpio);
 	bitmap_or(pending, pending, cur_stat, gc->ngpio);
 
-	bitmap_complement(cur_stat, new_stat, gc->ngpio);
-	bitmap_and(cur_stat, cur_stat, reg_direction, gc->ngpio);
+	bitmap_andnot(cur_stat, reg_direction, new_stat, gc->ngpio);
 	bitmap_and(old_stat, cur_stat, chip->irq_trig_level_low, gc->ngpio);
 	bitmap_and(old_stat, old_stat, chip->irq_mask, gc->ngpio);
 	bitmap_or(pending, pending, old_stat, gc->ngpio);
@@ -1411,7 +1408,7 @@ static int pca953x_resume(struct device *dev)
 		ret = regulator_enable(chip->regulator);
 		if (ret) {
 			dev_err(dev, "Failed to enable regulator: %d\n", ret);
-			return 0;
+			return ret;
 		}
 	}
 
diff --git a/drivers/gpio/gpio-pca9570.c b/drivers/gpio/gpio-pca9570.c
index 4a368803fb038..7a47a9aa0414e 100644
--- a/drivers/gpio/gpio-pca9570.c
+++ b/drivers/gpio/gpio-pca9570.c
@@ -163,9 +163,9 @@ static const struct pca9570_chip_data slg7xl45106_gpio = {
 };
 
 static const struct i2c_device_id pca9570_id_table[] = {
-	{ "pca9570", (kernel_ulong_t)&pca9570_gpio},
-	{ "pca9571", (kernel_ulong_t)&pca9571_gpio },
-	{ "slg7xl45106", (kernel_ulong_t)&slg7xl45106_gpio },
+	{ .name = "pca9570", .driver_data = (kernel_ulong_t)&pca9570_gpio },
+	{ .name = "pca9571", .driver_data = (kernel_ulong_t)&pca9571_gpio },
+	{ .name = "slg7xl45106", .driver_data = (kernel_ulong_t)&slg7xl45106_gpio },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, pca9570_id_table);
diff --git a/drivers/gpio/gpio-pcf857x.c b/drivers/gpio/gpio-pcf857x.c
index 3b9de8c3d924c..c942b959571b2 100644
--- a/drivers/gpio/gpio-pcf857x.c
+++ b/drivers/gpio/gpio-pcf857x.c
@@ -20,19 +20,19 @@
 #include <linux/spinlock.h>
 
 static const struct i2c_device_id pcf857x_id[] = {
-	{ "pcf8574", 8 },
-	{ "pcf8574a", 8 },
-	{ "pca8574", 8 },
-	{ "pca9670", 8 },
-	{ "pca9672", 8 },
-	{ "pca9674", 8 },
-	{ "pcf8575", 16 },
-	{ "pca8575", 16 },
-	{ "pca9671", 16 },
-	{ "pca9673", 16 },
-	{ "pca9675", 16 },
-	{ "max7328", 8 },
-	{ "max7329", 8 },
+	{ .name = "pcf8574", .driver_data = 8 },
+	{ .name = "pcf8574a", .driver_data = 8 },
+	{ .name = "pca8574", .driver_data = 8 },
+	{ .name = "pca9670", .driver_data = 8 },
+	{ .name = "pca9672", .driver_data = 8 },
+	{ .name = "pca9674", .driver_data = 8 },
+	{ .name = "pcf8575", .driver_data = 16 },
+	{ .name = "pca8575", .driver_data = 16 },
+	{ .name = "pca9671", .driver_data = 16 },
+	{ .name = "pca9673", .driver_data = 16 },
+	{ .name = "pca9675", .driver_data = 16 },
+	{ .name = "max7328", .driver_data = 8 },
+	{ .name = "max7329", .driver_data = 8 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pcf857x_id);
diff --git a/drivers/gpio/gpio-regmap.c b/drivers/gpio/gpio-regmap.c
index 9ae4a41a24277..51b4d69b87403 100644
--- a/drivers/gpio/gpio-regmap.c
+++ b/drivers/gpio/gpio-regmap.c
@@ -31,6 +31,7 @@ struct gpio_regmap {
 	unsigned int reg_clr_base;
 	unsigned int reg_dir_in_base;
 	unsigned int reg_dir_out_base;
+	unsigned long *fixed_direction_mask;
 	unsigned long *fixed_direction_output;
 
 #ifdef CONFIG_REGMAP_IRQ
@@ -138,6 +139,20 @@ static int gpio_regmap_set_with_clear(struct gpio_chip *chip,
 	return regmap_write(gpio->regmap, reg, mask);
 }
 
+static bool gpio_regmap_fixed_direction(struct gpio_regmap *gpio,
+					unsigned int offset)
+{
+	if (!gpio->fixed_direction_output)
+		return false;
+
+	/* In this case only some GPIOs are fixed as input/output */
+	if (gpio->fixed_direction_mask &&
+	    !test_bit(offset, gpio->fixed_direction_mask))
+		return false;
+
+	return true;
+}
+
 static int gpio_regmap_get_direction(struct gpio_chip *chip,
 				     unsigned int offset)
 {
@@ -145,7 +160,7 @@ static int gpio_regmap_get_direction(struct gpio_chip *chip,
 	unsigned int base, val, reg, mask;
 	int invert, ret;
 
-	if (gpio->fixed_direction_output) {
+	if (gpio_regmap_fixed_direction(gpio, offset)) {
 		if (test_bit(offset, gpio->fixed_direction_output))
 			return GPIO_LINE_DIRECTION_OUT;
 		else
@@ -181,6 +196,22 @@ static int gpio_regmap_get_direction(struct gpio_chip *chip,
 		return GPIO_LINE_DIRECTION_IN;
 }
 
+static int gpio_regmap_try_direction_fixed(struct gpio_regmap *gpio,
+					   unsigned int offset, bool output)
+{
+	if (test_bit(offset, gpio->fixed_direction_output)) {
+		if (output)
+			return 0;
+		else
+			return -EINVAL;
+	} else {
+		if (output)
+			return -EINVAL;
+		else
+			return 0;
+	}
+}
+
 static int gpio_regmap_set_direction(struct gpio_chip *chip,
 				     unsigned int offset, bool output)
 {
@@ -188,6 +219,13 @@ static int gpio_regmap_set_direction(struct gpio_chip *chip,
 	unsigned int base, val, reg, mask;
 	int invert, ret;
 
+	/*
+	 * If the direction is fixed, only accept the fixed
+	 * direction in this call.
+	 */
+	if (gpio_regmap_fixed_direction(gpio, offset))
+		return gpio_regmap_try_direction_fixed(gpio, offset, output);
+
 	if (gpio->reg_dir_out_base) {
 		base = gpio_regmap_addr(gpio->reg_dir_out_base);
 		invert = 0;
@@ -219,6 +257,20 @@ static int gpio_regmap_direction_input(struct gpio_chip *chip,
 static int gpio_regmap_direction_output(struct gpio_chip *chip,
 					unsigned int offset, int value)
 {
+	struct gpio_regmap *gpio = gpiochip_get_data(chip);
+	int ret;
+
+	/*
+	 * First check if this is gonna work on a fixed direction line,
+	 * if it doesn't (i.e. this is a fixed input line), then do not
+	 * attempt to set the output value either and just bail out.
+	 */
+	if (gpio_regmap_fixed_direction(gpio, offset)) {
+		ret = gpio_regmap_try_direction_fixed(gpio, offset, true);
+		if (ret)
+			return ret;
+	}
+
 	gpio_regmap_set(chip, offset, value);
 
 	return gpio_regmap_set_direction(chip, offset, true);
@@ -302,12 +354,23 @@ struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config
 			goto err_free_gpio;
 	}
 
+	if (config->fixed_direction_mask) {
+		gpio->fixed_direction_mask = bitmap_alloc(chip->ngpio,
+							    GFP_KERNEL);
+		if (!gpio->fixed_direction_mask) {
+			ret = -ENOMEM;
+			goto err_free_gpio;
+		}
+		bitmap_copy(gpio->fixed_direction_mask,
+			    config->fixed_direction_mask, chip->ngpio);
+	}
+
 	if (config->fixed_direction_output) {
 		gpio->fixed_direction_output = bitmap_alloc(chip->ngpio,
 							    GFP_KERNEL);
 		if (!gpio->fixed_direction_output) {
 			ret = -ENOMEM;
-			goto err_free_gpio;
+			goto err_free_bitmap_dirmask;
 		}
 		bitmap_copy(gpio->fixed_direction_output,
 			    config->fixed_direction_output, chip->ngpio);
@@ -329,7 +392,7 @@ struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config
 
 	ret = gpiochip_add_data(chip, gpio);
 	if (ret < 0)
-		goto err_free_bitmap;
+		goto err_free_bitmap_output;
 
 #ifdef CONFIG_REGMAP_IRQ
 	if (config->regmap_irq_chip) {
@@ -355,8 +418,10 @@ struct gpio_regmap *gpio_regmap_register(const struct gpio_regmap_config *config
 
 err_remove_gpiochip:
 	gpiochip_remove(chip);
-err_free_bitmap:
+err_free_bitmap_output:
 	bitmap_free(gpio->fixed_direction_output);
+err_free_bitmap_dirmask:
+	bitmap_free(gpio->fixed_direction_mask);
 err_free_gpio:
 	kfree(gpio);
 	return ERR_PTR(ret);
@@ -376,6 +441,7 @@ void gpio_regmap_unregister(struct gpio_regmap *gpio)
 
 	gpiochip_remove(&gpio->gpio_chip);
 	bitmap_free(gpio->fixed_direction_output);
+	bitmap_free(gpio->fixed_direction_mask);
 	kfree(gpio);
 }
 EXPORT_SYMBOL_GPL(gpio_regmap_unregister);
diff --git a/drivers/gpio/gpio-sim.c b/drivers/gpio/gpio-sim.c
index e19701c2ed673..b1f31861f40d3 100644
--- a/drivers/gpio/gpio-sim.c
+++ b/drivers/gpio/gpio-sim.c
@@ -695,9 +695,9 @@ static ssize_t gpio_sim_device_config_dev_name_show(struct config_item *item,
 
 	pdev = dev->pdev;
 	if (pdev)
-		return sprintf(page, "%s\n", dev_name(&pdev->dev));
+		return sysfs_emit(page, "%s\n", dev_name(&pdev->dev));
 
-	return sprintf(page, "gpio-sim.%d\n", dev->id);
+	return sysfs_emit(page, "gpio-sim.%d\n", dev->id);
 }
 
 CONFIGFS_ATTR_RO(gpio_sim_device_config_, dev_name);
@@ -711,7 +711,7 @@ gpio_sim_device_config_live_show(struct config_item *item, char *page)
 	scoped_guard(mutex, &dev->lock)
 		live = gpio_sim_device_is_live(dev);
 
-	return sprintf(page, "%c\n", live ? '1' : '0');
+	return sysfs_emit(page, "%c\n", live ? '1' : '0');
 }
 
 static unsigned int gpio_sim_get_line_names_size(struct gpio_sim_bank *bank)
@@ -1056,7 +1056,7 @@ static int gpio_sim_emit_chip_name(struct device *dev, void *data)
 		return 0;
 
 	if (device_match_fwnode(dev, ctx->swnode))
-		return sprintf(ctx->page, "%s\n", dev_name(dev));
+		return sysfs_emit(ctx->page, "%s\n", dev_name(dev));
 
 	return 0;
 }
@@ -1074,7 +1074,7 @@ static ssize_t gpio_sim_bank_config_chip_name_show(struct config_item *item,
 		return device_for_each_child(&dev->pdev->dev, &ctx,
 					     gpio_sim_emit_chip_name);
 
-	return sprintf(page, "none\n");
+	return sysfs_emit(page, "none\n");
 }
 
 CONFIGFS_ATTR_RO(gpio_sim_bank_config_, chip_name);
@@ -1087,7 +1087,7 @@ gpio_sim_bank_config_label_show(struct config_item *item, char *page)
 
 	guard(mutex)(&dev->lock);
 
-	return sprintf(page, "%s\n", bank->label ?: "");
+	return sysfs_emit(page, "%s\n", bank->label ?: "");
 }
 
 static ssize_t gpio_sim_bank_config_label_store(struct config_item *item,
@@ -1122,7 +1122,7 @@ gpio_sim_bank_config_num_lines_show(struct config_item *item, char *page)
 
 	guard(mutex)(&dev->lock);
 
-	return sprintf(page, "%u\n", bank->num_lines);
+	return sysfs_emit(page, "%u\n", bank->num_lines);
 }
 
 static ssize_t
@@ -1168,7 +1168,7 @@ gpio_sim_line_config_name_show(struct config_item *item, char *page)
 
 	guard(mutex)(&dev->lock);
 
-	return sprintf(page, "%s\n", line->name ?: "");
+	return sysfs_emit(page, "%s\n", line->name ?: "");
 }
 
 static ssize_t gpio_sim_line_config_name_store(struct config_item *item,
@@ -1203,7 +1203,7 @@ gpio_sim_line_config_valid_show(struct config_item *item, char *page)
 
 	guard(mutex)(&dev->lock);
 
-	return sprintf(page, "%c\n", line->valid ? '1' : '0');
+	return sysfs_emit(page, "%c\n", line->valid ? '1' : '0');
 }
 
 static ssize_t gpio_sim_line_config_valid_store(struct config_item *item,
@@ -1241,7 +1241,7 @@ static ssize_t gpio_sim_hog_config_name_show(struct config_item *item,
 
 	guard(mutex)(&dev->lock);
 
-	return sprintf(page, "%s\n", hog->name ?: "");
+	return sysfs_emit(page, "%s\n", hog->name ?: "");
 }
 
 static ssize_t gpio_sim_hog_config_name_store(struct config_item *item,
@@ -1295,7 +1295,7 @@ static ssize_t gpio_sim_hog_config_direction_show(struct config_item *item,
 		return -EINVAL;
 	}
 
-	return sprintf(page, "%s\n", repr);
+	return sysfs_emit(page, "%s\n", repr);
 }
 
 static ssize_t
@@ -1335,7 +1335,7 @@ static ssize_t gpio_sim_hog_config_active_low_show(struct config_item *item,
 
 	guard(mutex)(&dev->lock);
 
-	return sprintf(page, "%c\n", hog->active_low ? '1' : '0');
+	return sysfs_emit(page, "%c\n", hog->active_low ? '1' : '0');
 }
 
 static ssize_t
diff --git a/drivers/gpio/gpio-tegra186.c b/drivers/gpio/gpio-tegra186.c
index aa7c3e44234f5..f56617c298c0f 100644
--- a/drivers/gpio/gpio-tegra186.c
+++ b/drivers/gpio/gpio-tegra186.c
@@ -19,6 +19,7 @@
 #include <dt-bindings/gpio/tegra186-gpio.h>
 #include <dt-bindings/gpio/tegra194-gpio.h>
 #include <dt-bindings/gpio/tegra234-gpio.h>
+#include <dt-bindings/gpio/nvidia,tegra238-gpio.h>
 #include <dt-bindings/gpio/tegra241-gpio.h>
 #include <dt-bindings/gpio/tegra256-gpio.h>
 #include <dt-bindings/gpio/nvidia,tegra264-gpio.h>
@@ -1239,6 +1240,67 @@ static const struct tegra_gpio_soc tegra234_aon_soc = {
 	.has_vm_support = false,
 };
 
+#define TEGRA238_MAIN_GPIO_PORT(_name, _bank, _port, _pins) \
+	TEGRA_GPIO_PORT(TEGRA238_MAIN, _name, _bank, _port, _pins)
+
+static const struct tegra_gpio_port tegra238_main_ports[] = {
+	TEGRA238_MAIN_GPIO_PORT(A, 0, 0, 8),
+	TEGRA238_MAIN_GPIO_PORT(B, 0, 1, 5),
+	TEGRA238_MAIN_GPIO_PORT(C, 0, 2, 8),
+	TEGRA238_MAIN_GPIO_PORT(D, 0, 3, 8),
+	TEGRA238_MAIN_GPIO_PORT(E, 0, 4, 4),
+	TEGRA238_MAIN_GPIO_PORT(F, 0, 5, 8),
+	TEGRA238_MAIN_GPIO_PORT(G, 0, 6, 8),
+	TEGRA238_MAIN_GPIO_PORT(H, 0, 7, 6),
+	TEGRA238_MAIN_GPIO_PORT(J, 1, 0, 8),
+	TEGRA238_MAIN_GPIO_PORT(K, 1, 1, 4),
+	TEGRA238_MAIN_GPIO_PORT(L, 1, 2, 8),
+	TEGRA238_MAIN_GPIO_PORT(M, 1, 3, 8),
+	TEGRA238_MAIN_GPIO_PORT(N, 1, 4, 3),
+	TEGRA238_MAIN_GPIO_PORT(P, 1, 5, 8),
+	TEGRA238_MAIN_GPIO_PORT(Q, 1, 6, 3),
+	TEGRA238_MAIN_GPIO_PORT(R, 2, 0, 8),
+	TEGRA238_MAIN_GPIO_PORT(S, 2, 1, 8),
+	TEGRA238_MAIN_GPIO_PORT(T, 2, 2, 8),
+	TEGRA238_MAIN_GPIO_PORT(U, 2, 3, 6),
+	TEGRA238_MAIN_GPIO_PORT(V, 2, 4, 2),
+	TEGRA238_MAIN_GPIO_PORT(W, 3, 0, 8),
+	TEGRA238_MAIN_GPIO_PORT(X, 3, 1, 2)
+};
+
+static const struct tegra_gpio_soc tegra238_main_soc = {
+	.num_ports = ARRAY_SIZE(tegra238_main_ports),
+	.ports = tegra238_main_ports,
+	.name = "tegra238-gpio",
+	.instance = 0,
+	.num_irqs_per_bank = 8,
+	.has_vm_support = true,
+};
+
+#define TEGRA238_AON_GPIO_PORT(_name, _bank, _port, _pins) \
+	TEGRA_GPIO_PORT(TEGRA238_AON, _name, _bank, _port, _pins)
+
+static const struct tegra_gpio_port tegra238_aon_ports[] = {
+	TEGRA238_AON_GPIO_PORT(AA, 0, 0, 8),
+	TEGRA238_AON_GPIO_PORT(BB, 0, 1, 1),
+	TEGRA238_AON_GPIO_PORT(CC, 0, 2, 8),
+	TEGRA238_AON_GPIO_PORT(DD, 0, 3, 8),
+	TEGRA238_AON_GPIO_PORT(EE, 0, 4, 6),
+	TEGRA238_AON_GPIO_PORT(FF, 0, 5, 8),
+	TEGRA238_AON_GPIO_PORT(GG, 0, 6, 8),
+	TEGRA238_AON_GPIO_PORT(HH, 0, 7, 4)
+};
+
+static const struct tegra_gpio_soc tegra238_aon_soc = {
+	.num_ports = ARRAY_SIZE(tegra238_aon_ports),
+	.ports = tegra238_aon_ports,
+	.name = "tegra238-gpio-aon",
+	.instance = 1,
+	.num_irqs_per_bank = 8,
+	.has_gte = true,
+	.has_vm_support = false,
+};
+
 #define TEGRA241_MAIN_GPIO_PORT(_name, _bank, _port, _pins) \
 	TEGRA_GPIO_PORT(TEGRA241_MAIN, _name, _bank, _port, _pins)
 
@@ -1447,6 +1509,12 @@ static const struct of_device_id tegra186_gpio_of_match[] = {
 	}, {
 		.compatible = "nvidia,tegra234-gpio-aon",
 		.data = &tegra234_aon_soc
+	}, {
+		.compatible = "nvidia,tegra238-gpio",
+		.data = &tegra238_main_soc
+	}, {
+		.compatible = "nvidia,tegra238-gpio-aon",
+		.data = &tegra238_aon_soc
 	}, {
 		.compatible = "nvidia,tegra256-gpio",
 		.data = &tegra256_main_soc
diff --git a/drivers/gpio/gpio-timberdale.c b/drivers/gpio/gpio-timberdale.c
index f488939dd00a8..78fe133f5d323 100644
--- a/drivers/gpio/gpio-timberdale.c
+++ b/drivers/gpio/gpio-timberdale.c
@@ -14,7 +14,6 @@
 #include <linux/platform_device.h>
 #include <linux/irq.h>
 #include <linux/io.h>
-#include <linux/timb_gpio.h>
 #include <linux/interrupt.h>
 #include <linux/slab.h>
 
@@ -225,19 +224,21 @@ static int timbgpio_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct gpio_chip *gc;
 	struct timbgpio *tgpio;
-	struct timbgpio_platform_data *pdata = dev_get_platdata(&pdev->dev);
 	int irq = platform_get_irq(pdev, 0);
 
-	if (!pdata || pdata->nr_pins > 32) {
-		dev_err(dev, "Invalid platform data\n");
-		return -EINVAL;
-	}
-
 	tgpio = devm_kzalloc(dev, sizeof(*tgpio), GFP_KERNEL);
 	if (!tgpio)
 		return -EINVAL;
 
-	tgpio->irq_base = pdata->irq_base;
+	gc = &tgpio->gpio;
+
+	err = device_property_read_u32(dev, "irq-base", &tgpio->irq_base);
+	if (err)
+		return err;
+
+	err = device_property_read_u32(dev, "gpio-base", &gc->base);
+	if (err)
+		return err;
 
 	spin_lock_init(&tgpio->lock);
 
@@ -245,8 +246,6 @@ static int timbgpio_probe(struct platform_device *pdev)
 	if (IS_ERR(tgpio->membase))
 		return PTR_ERR(tgpio->membase);
 
-	gc = &tgpio->gpio;
-
 	gc->label = dev_name(&pdev->dev);
 	gc->owner = THIS_MODULE;
 	gc->parent = &pdev->dev;
@@ -256,21 +255,22 @@ static int timbgpio_probe(struct platform_device *pdev)
 	gc->set = timbgpio_gpio_set;
 	gc->to_irq = (irq >= 0 && tgpio->irq_base > 0) ? timbgpio_to_irq : NULL;
 	gc->dbg_show = NULL;
-	gc->base = pdata->gpio_base;
-	gc->ngpio = pdata->nr_pins;
 	gc->can_sleep = false;
 
 	err = devm_gpiochip_add_data(&pdev->dev, gc, tgpio);
 	if (err)
 		return err;
 
+	if (gc->ngpio > 32)
+		return dev_err_probe(dev, -EINVAL, "Invalid number of pins\n");
+
 	/* make sure to disable interrupts */
 	iowrite32(0x0, tgpio->membase + TGPIO_IER);
 
 	if (irq < 0 || tgpio->irq_base <= 0)
 		return 0;
 
-	for (i = 0; i < pdata->nr_pins; i++) {
+	for (i = 0; i < gc->ngpio; i++) {
 		irq_set_chip_and_handler(tgpio->irq_base + i,
 			&timbgpio_irqchip, handle_simple_irq);
 		irq_set_chip_data(tgpio->irq_base + i, tgpio);
diff --git a/drivers/gpio/gpio-tpic2810.c b/drivers/gpio/gpio-tpic2810.c
index 866ff2d436d53..c38538653e991 100644
--- a/drivers/gpio/gpio-tpic2810.c
+++ b/drivers/gpio/gpio-tpic2810.c
@@ -112,7 +112,7 @@ static int tpic2810_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tpic2810_id_table[] = {
-	{ "tpic2810", },
+	{ .name = "tpic2810" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, tpic2810_id_table);
diff --git a/drivers/gpio/gpio-ts4900.c b/drivers/gpio/gpio-ts4900.c
index d9ee8fc77ccdf..b46b48e56c565 100644
--- a/drivers/gpio/gpio-ts4900.c
+++ b/drivers/gpio/gpio-ts4900.c
@@ -175,7 +175,7 @@ static int ts4900_gpio_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ts4900_gpio_id_table[] = {
-	{ "ts4900-gpio", },
+	{ .name = "ts4900-gpio" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, ts4900_gpio_id_table);
diff --git a/drivers/gpio/gpio-usbio.c b/drivers/gpio/gpio-usbio.c
index 34d42c743d5bc..489c8ac6299e1 100644
--- a/drivers/gpio/gpio-usbio.c
+++ b/drivers/gpio/gpio-usbio.c
@@ -31,6 +31,7 @@ static const struct acpi_device_id usbio_gpio_acpi_hids[] = {
 	{ "INTC10B5" }, /* LNL */
 	{ "INTC10D1" }, /* MTL-CVF */
 	{ "INTC10E2" }, /* PTL */
+	{ "INTC1116" }, /* NVL */
 	{ }
 };
 
diff --git a/drivers/gpio/gpio-waveshare-dsi.c b/drivers/gpio/gpio-waveshare-dsi.c
new file mode 100644
index 0000000000000..38f52351bb584
--- /dev/null
+++ b/drivers/gpio/gpio-waveshare-dsi.c
@@ -0,0 +1,208 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2024 Waveshare International Limited
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/backlight.h>
+#include <linux/err.h>
+#include <linux/fb.h>
+#include <linux/gpio/driver.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+
+/* I2C registers of the microcontroller. */
+#define REG_TP		0x94
+#define REG_LCD		0x95
+#define REG_PWM		0x96
+#define REG_SIZE	0x97
+#define REG_ID		0x98
+#define REG_VERSION	0x99
+
+enum {
+	GPIO_AVDD = 0,
+	GPIO_PANEL_RESET = 1,
+	GPIO_BL_ENABLE = 2,
+	GPIO_IOVCC = 4,
+	GPIO_VCC = 8,
+	GPIO_TS_RESET = 9,
+};
+
+#define NUM_GPIO 16
+
+struct waveshare_gpio {
+	struct mutex dir_lock;
+	struct mutex pwr_lock;
+	struct regmap *regmap;
+	u16 poweron_state;
+
+	struct gpio_chip gc;
+};
+
+static const struct regmap_config waveshare_gpio_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = REG_VERSION,
+};
+
+static int waveshare_gpio_get(struct waveshare_gpio *state, unsigned int offset)
+{
+	u16 pwr_state;
+
+	guard(mutex)(&state->pwr_lock);
+	pwr_state = state->poweron_state & BIT(offset);
+
+	return !!pwr_state;
+}
+
+static int waveshare_gpio_set(struct waveshare_gpio *state, unsigned int offset, int value)
+{
+	u16 last_val;
+	int err;
+
+	guard(mutex)(&state->pwr_lock);
+
+	last_val = state->poweron_state;
+	if (value)
+		last_val |= BIT(offset);
+	else
+		last_val &= ~BIT(offset);
+
+	state->poweron_state = last_val;
+
+	err = regmap_write(state->regmap, REG_TP, last_val >> 8);
+	if (!err)
+		err = regmap_write(state->regmap, REG_LCD, last_val & 0xff);
+
+	return err;
+}
+
+static int waveshare_gpio_gpio_get_direction(struct gpio_chip *gc, unsigned int offset)
+{
+	return GPIO_LINE_DIRECTION_OUT;
+}
+
+static int waveshare_gpio_gpio_get(struct gpio_chip *gc, unsigned int offset)
+{
+	struct waveshare_gpio *state = gpiochip_get_data(gc);
+
+	return waveshare_gpio_get(state, offset);
+}
+
+static int waveshare_gpio_gpio_set(struct gpio_chip *gc, unsigned int offset, int value)
+{
+	struct waveshare_gpio *state = gpiochip_get_data(gc);
+
+	return waveshare_gpio_set(state, offset, value);
+}
+
+static int waveshare_gpio_update_status(struct backlight_device *bl)
+{
+	struct waveshare_gpio *state = bl_get_data(bl);
+	int brightness = backlight_get_brightness(bl);
+
+	waveshare_gpio_set(state, GPIO_BL_ENABLE, brightness);
+
+	return regmap_write(state->regmap, REG_PWM, brightness);
+}
+
+static const struct backlight_ops waveshare_gpio_bl = {
+	.update_status = waveshare_gpio_update_status,
+};
+
+static int waveshare_gpio_probe(struct i2c_client *i2c)
+{
+	struct backlight_properties props = {};
+	struct waveshare_gpio *state;
+	struct device *dev = &i2c->dev;
+	struct backlight_device *bl;
+	struct regmap *regmap;
+	unsigned int data;
+	int ret;
+
+	state = devm_kzalloc(dev, sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return -ENOMEM;
+
+	ret = devm_mutex_init(dev, &state->dir_lock);
+	if (ret)
+		return ret;
+
+	ret = devm_mutex_init(dev, &state->pwr_lock);
+	if (ret)
+		return ret;
+
+	regmap = devm_regmap_init_i2c(i2c, &waveshare_gpio_regmap_config);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Failed to allocate register map\n");
+
+	state->regmap = regmap;
+	i2c_set_clientdata(i2c, state);
+
+	ret = regmap_read(regmap, REG_ID, &data);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to read register\n");
+
+	dev_dbg(dev, "waveshare panel hw id = 0x%x\n", data);
+
+	ret = regmap_read(regmap, REG_SIZE, &data);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to read register\n");
+
+	dev_dbg(dev, "waveshare panel size = %d\n", data);
+
+	ret = regmap_read(regmap, REG_VERSION, &data);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to read register\n");
+
+	dev_dbg(dev, "waveshare panel mcu version = 0x%x\n", data);
+
+	ret = waveshare_gpio_set(state, GPIO_TS_RESET, 1);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to program GPIOs\n");
+
+	msleep(20);
+
+	state->gc.parent = dev;
+	state->gc.label = i2c->name;
+	state->gc.owner = THIS_MODULE;
+	state->gc.base = -1;
+	state->gc.ngpio = NUM_GPIO;
+
+	/* it is output only */
+	state->gc.get = waveshare_gpio_gpio_get;
+	state->gc.set = waveshare_gpio_gpio_set;
+	state->gc.get_direction = waveshare_gpio_gpio_get_direction;
+	state->gc.can_sleep = true;
+
+	ret = devm_gpiochip_add_data(dev, &state->gc, state);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to create gpiochip\n");
+
+	props.type = BACKLIGHT_RAW;
+	props.max_brightness = 255;
+	props.brightness = 255;
+	bl = devm_backlight_device_register(dev, dev_name(dev), dev, state,
+					    &waveshare_gpio_bl, &props);
+	return PTR_ERR_OR_ZERO(bl);
+}
+
+static const struct of_device_id waveshare_gpio_dt_ids[] = {
+	{ .compatible = "waveshare,dsi-touch-gpio" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, waveshare_gpio_dt_ids);
+
+static struct i2c_driver waveshare_gpio_regulator_driver = {
+	.driver = {
+		.name = "waveshare-regulator",
+		.of_match_table = of_match_ptr(waveshare_gpio_dt_ids),
+	},
+	.probe = waveshare_gpio_probe,
+};
+
+module_i2c_driver(waveshare_gpio_regulator_driver);
+
+MODULE_DESCRIPTION("GPIO controller driver for Waveshare DSI touch panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpio/gpio-xilinx.c b/drivers/gpio/gpio-xilinx.c
index be4b4d7305477..532205175827a 100644
--- a/drivers/gpio/gpio-xilinx.c
+++ b/drivers/gpio/gpio-xilinx.c
@@ -495,13 +495,11 @@ static void xgpio_irqhandler(struct irq_desc *desc)
 
 	xgpio_read_ch_all(chip, XGPIO_DATA_OFFSET, hw);
 
-	bitmap_complement(rising, chip->last_irq_read, 64);
-	bitmap_and(rising, rising, hw, 64);
+	bitmap_andnot(rising, hw, chip->last_irq_read, 64);
 	bitmap_and(rising, rising, chip->enable, 64);
 	bitmap_and(rising, rising, chip->rising_edge, 64);
 
-	bitmap_complement(falling, hw, 64);
-	bitmap_and(falling, falling, chip->last_irq_read, 64);
+	bitmap_andnot(falling, chip->last_irq_read, hw, 64);
 	bitmap_and(falling, falling, chip->enable, 64);
 	bitmap_and(falling, falling, chip->falling_edge, 64);
 
diff --git a/drivers/gpio/gpio-zevio.c b/drivers/gpio/gpio-zevio.c
index 29375bea2289b..af0158522ac5d 100644
--- a/drivers/gpio/gpio-zevio.c
+++ b/drivers/gpio/gpio-zevio.c
@@ -64,14 +64,14 @@ static inline u32 zevio_gpio_port_get(struct zevio_gpio *c, unsigned pin,
 					unsigned port_offset)
 {
 	unsigned section_offset = ((pin >> 3) & 3)*ZEVIO_GPIO_SECTION_SIZE;
-	return readl(IOMEM(c->regs + section_offset + port_offset));
+	return readl(c->regs + section_offset + port_offset);
 }
 
 static inline void zevio_gpio_port_set(struct zevio_gpio *c, unsigned pin,
 					unsigned port_offset, u32 val)
 {
 	unsigned section_offset = ((pin >> 3) & 3)*ZEVIO_GPIO_SECTION_SIZE;
-	writel(val, IOMEM(c->regs + section_offset + port_offset));
+	writel(val, c->regs + section_offset + port_offset);
 }
 
 /* Functions for struct gpio_chip */
diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c
index 571e366624d2a..8118ae3412c20 100644
--- a/drivers/gpio/gpio-zynq.c
+++ b/drivers/gpio/gpio-zynq.c
@@ -25,6 +25,7 @@
 #define VERSAL_GPIO_MAX_BANK	4
 #define PMC_GPIO_MAX_BANK	5
 #define VERSAL_UNUSED_BANKS	2
+#define EIO_GPIO_MAX_BANK	2
 
 #define ZYNQ_GPIO_BANK0_NGPIO	32
 #define ZYNQ_GPIO_BANK1_NGPIO	22
@@ -818,6 +819,16 @@ static const struct dev_pm_ops zynq_gpio_dev_pm_ops = {
 	RUNTIME_PM_OPS(zynq_gpio_runtime_suspend, zynq_gpio_runtime_resume, NULL)
 };
 
+static const struct zynq_platform_data eio_gpio_def = {
+	.label = "eio_gpio",
+	.ngpio = 52,
+	.max_bank = EIO_GPIO_MAX_BANK,
+	.bank_min[0] = 0,
+	.bank_max[0] = 25, /* 0 to 25 are connected to MIOs (26 pins) */
+	.bank_min[1] = 26,
+	.bank_max[1] = 51, /* Bank 1 are connected to MIOs (26 pins) */
+};
+
 static const struct zynq_platform_data versal_gpio_def = {
 	.label = "versal_gpio",
 	.quirks = GPIO_QUIRK_VERSAL,
@@ -882,6 +893,7 @@ static const struct of_device_id zynq_gpio_of_match[] = {
 	{ .compatible = "xlnx,zynqmp-gpio-1.0", .data = &zynqmp_gpio_def },
 	{ .compatible = "xlnx,versal-gpio-1.0", .data = &versal_gpio_def },
 	{ .compatible = "xlnx,pmc-gpio-1.0", .data = &pmc_gpio_def },
+	{ .compatible = "xlnx,eio-gpio-1.0", .data = &eio_gpio_def },
 	{ /* end of table */ }
 };
 MODULE_DEVICE_TABLE(of, zynq_gpio_of_match);
diff --git a/drivers/gpio/gpiolib-acpi-core.c b/drivers/gpio/gpiolib-acpi-core.c
index 09f860200a059..eb8a40cfb7a98 100644
--- a/drivers/gpio/gpiolib-acpi-core.c
+++ b/drivers/gpio/gpiolib-acpi-core.c
@@ -233,12 +233,23 @@ static void acpi_gpiochip_request_irq(struct acpi_gpio_chip *acpi_gpio,
 
 	event->irq_requested = true;
 
-	/* Make sure we trigger the initial state of edge-triggered IRQs */
+	/*
+	 * Make sure we trigger the initial state of ActiveBoth IRQs.
+	 *
+	 * According to the Microsoft GPIO documentation, triggering GPIO
+	 * interrupts marked as ActiveBoth during initialization is correct
+	 * as long as the associated GPIO line is already "asserted"
+	 * (logic level low). We should not trigger edge-based GPIO
+	 * interrupts not marked as ActiveBoth.
+	 *
+	 * See: https://learn.microsoft.com/en-us/windows-hardware/drivers/bringup/general-purpose-i-o--gpio-
+	 * Section: "GPIO controllers and ActiveBoth interrupts"
+	 */
 	if (acpi_gpio_need_run_edge_events_on_boot() &&
-	    (event->irqflags & (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING))) {
+	    ((event->irqflags & (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING)) ==
+	     (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING))) {
 		value = gpiod_get_raw_value_cansleep(event->desc);
-		if (((event->irqflags & IRQF_TRIGGER_RISING) && value == 1) ||
-		    ((event->irqflags & IRQF_TRIGGER_FALLING) && value == 0))
+		if (value == 0)
 			event->handler(event->irq, event);
 	}
 }
diff --git a/drivers/gpio/gpiolib-devres.c b/drivers/gpio/gpiolib-devres.c
index 72422c5db3641..2ec825ffab7da 100644
--- a/drivers/gpio/gpiolib-devres.c
+++ b/drivers/gpio/gpiolib-devres.c
@@ -353,6 +353,13 @@ int devm_gpiochip_add_data_with_key(struct device *dev, struct gpio_chip *gc, vo
 {
 	int ret;
 
+	/*
+	 * We are passing the devres device here so if the user did not pass
+	 * another parent, it's this one.
+	 */
+	if (!gc->parent)
+		gc->parent = dev;
+
 	ret = gpiochip_add_data_with_key(gc, data, lock_key, request_key);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/gpio/gpiolib-of.c b/drivers/gpio/gpiolib-of.c
index 2c923d17541f2..90f6295ab3384 100644
--- a/drivers/gpio/gpiolib-of.c
+++ b/drivers/gpio/gpiolib-of.c
@@ -240,6 +240,14 @@ static void of_gpio_try_fixup_polarity(const struct device_node *np,
 		 * treats it as "active low".
 		 */
 		{ "ti,tsc2005",		"reset-gpios",	false },
+#endif
+#if IS_ENABLED(CONFIG_SND_SOC_WSA881X)
+		/*
+		 * WSA881 powerdown is always active low, but some device trees
+		 * missed this when first contributed. It also has a very strange
+		 * compatible.
+		 */
+		{ "sdw10217201000",	"powerdown-gpios", false },
 #endif
 	};
 	unsigned int i;
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index 1e6dce430dca0..69743d6deeaf8 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -491,6 +491,28 @@ int gpiod_get_direction(struct gpio_desc *desc)
 }
 EXPORT_SYMBOL_GPL(gpiod_get_direction);
 
+/**
+ * gpiod_is_single_ended - check if the GPIO is configured as single-ended
+ * @desc: the GPIO descriptor to check
+ *
+ * Returns true if the GPIO is configured as either Open Drain or Open Source.
+ * In these modes, the direction of the line cannot always be reliably
+ * determined by reading hardware registers, as the "off" state (High-Z)
+ * is physically indistinguishable from an input state.
+ */
+bool gpiod_is_single_ended(struct gpio_desc *desc)
+{
+	if (!desc)
+		return false;
+
+	if (test_bit(GPIOD_FLAG_OPEN_DRAIN, &desc->flags) ||
+		test_bit(GPIOD_FLAG_OPEN_SOURCE, &desc->flags))
+		return true;
+
+	return false;
+}
+EXPORT_SYMBOL_GPL(gpiod_is_single_ended);
+
 /*
  * Add a new chip to the global chips list, keeping the list of chips sorted
  * by range(means [base, base + ngpio - 1]) order.
diff --git a/drivers/gpu/buddy.c b/drivers/gpu/buddy.c
index 52686672e99f5..eb1457376307d 100644
--- a/drivers/gpu/buddy.c
+++ b/drivers/gpu/buddy.c
@@ -437,6 +437,9 @@ int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size)
 		root_count++;
 	} while (size);
 
+#ifdef CONFIG_LOCKDEP
+	mm->lock_dep_map = NULL;
+#endif
 	return 0;
 
 out_free_roots:
@@ -538,6 +541,7 @@ void gpu_buddy_reset_clear(struct gpu_buddy *mm, bool is_clear)
 	unsigned int order;
 	int i;
 
+	gpu_buddy_driver_lock_held(mm);
 	size = mm->size;
 	for (i = 0; i < mm->n_roots; ++i) {
 		order = ilog2(size) - ilog2(mm->chunk_size);
@@ -580,6 +584,7 @@ EXPORT_SYMBOL(gpu_buddy_reset_clear);
 void gpu_buddy_free_block(struct gpu_buddy *mm,
 			  struct gpu_buddy_block *block)
 {
+	gpu_buddy_driver_lock_held(mm);
 	BUG_ON(!gpu_buddy_block_is_allocated(block));
 	mm->avail += gpu_buddy_block_size(mm, block);
 	if (gpu_buddy_block_is_clear(block))
@@ -633,6 +638,7 @@ void gpu_buddy_free_list(struct gpu_buddy *mm,
 {
 	bool mark_clear = flags & GPU_BUDDY_CLEARED;
 
+	gpu_buddy_driver_lock_held(mm);
 	__gpu_buddy_free_list(mm, objects, mark_clear, !mark_clear);
 }
 EXPORT_SYMBOL(gpu_buddy_free_list);
@@ -1172,6 +1178,8 @@ int gpu_buddy_block_trim(struct gpu_buddy *mm,
 	u64 new_start;
 	int err;
 
+	gpu_buddy_driver_lock_held(mm);
+
 	if (!list_is_singular(blocks))
 		return -EINVAL;
 
@@ -1287,6 +1295,8 @@ int gpu_buddy_alloc_blocks(struct gpu_buddy *mm,
 	unsigned long pages;
 	int err;
 
+	gpu_buddy_driver_lock_held(mm);
+
 	if (size < mm->chunk_size)
 		return -EINVAL;
 
@@ -1475,6 +1485,7 @@ void gpu_buddy_print(struct gpu_buddy *mm)
 {
 	int order;
 
+	gpu_buddy_driver_lock_held(mm);
 	pr_info("chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB, clear_free: %lluMiB\n",
 		mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
 
diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 8f5a8d3012e41..323422861e8f6 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -218,6 +218,13 @@ config DRM_GPUVM
 	  GPU-VM representation providing helpers to manage a GPUs virtual
 	  address space
 
+config RUST_DRM_GPUVM
+	bool
+	depends on DRM
+	select DRM_GPUVM
+	help
+	  Choose this if you need GPUVM functions in Rust
+
 config DRM_GPUSVM
 	tristate
 	depends on DRM
diff --git a/drivers/gpu/drm/adp/adp_drv.c b/drivers/gpu/drm/adp/adp_drv.c
index 4554cf75565e2..ce9dd6222634d 100644
--- a/drivers/gpu/drm/adp/adp_drv.c
+++ b/drivers/gpu/drm/adp/adp_drv.c
@@ -129,7 +129,7 @@ struct adp_drv_private {
 #define crtc_to_adp(x) container_of(x, struct adp_drv_private, crtc)
 
 static int adp_plane_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state;
 	struct drm_crtc_state *crtc_state;
@@ -151,7 +151,7 @@ static int adp_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void adp_plane_atomic_update(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct adp_drv_private *adp;
 	struct drm_rect src_rect;
@@ -193,7 +193,7 @@ static void adp_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void adp_plane_atomic_disable(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct adp_drv_private *adp = to_adp(plane->dev);
 
@@ -279,7 +279,7 @@ static void adp_crtc_disable_vblank(struct drm_crtc *crtc)
 }
 
 static void adp_crtc_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct adp_drv_private *adp = crtc_to_adp(crtc);
 
@@ -292,7 +292,7 @@ static void adp_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void adp_crtc_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct adp_drv_private *adp = crtc_to_adp(crtc);
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state, crtc);
@@ -308,7 +308,7 @@ static void adp_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void adp_crtc_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	u32 frame_num = 1;
 	unsigned long flags;
@@ -444,8 +444,6 @@ static int adp_setup_mode_config(struct adp_drv_private *adp)
 	if (IS_ERR(adp->connector))
 		return PTR_ERR(adp->connector);
 
-	drm_connector_attach_encoder(adp->connector, adp->encoder);
-
 	ret = drm_vblank_init(drm, drm->mode_config.num_crtc);
 	if (ret < 0) {
 		drm_err(drm, "failed to initialize vblank");
diff --git a/drivers/gpu/drm/amd/amdgpu/Kconfig b/drivers/gpu/drm/amd/amdgpu/Kconfig
index 7fb0b93bc1caa..12e4a41bf1f04 100644
--- a/drivers/gpu/drm/amd/amdgpu/Kconfig
+++ b/drivers/gpu/drm/amd/amdgpu/Kconfig
@@ -3,7 +3,6 @@
 config DRM_AMDGPU
 	tristate "AMD GPU"
 	depends on DRM && PCI
-	depends on !UML
 	select FW_LOADER
 	select DRM_CLIENT
 	select DRM_CLIENT_SELECTION
diff --git a/drivers/gpu/drm/amd/amdgpu/Makefile b/drivers/gpu/drm/amd/amdgpu/Makefile
index db66c63721999..ee3574797bc28 100644
--- a/drivers/gpu/drm/amd/amdgpu/Makefile
+++ b/drivers/gpu/drm/amd/amdgpu/Makefile
@@ -90,7 +90,7 @@ amdgpu-y += \
 	nbio_v7_2.o hdp_v4_0.o hdp_v5_0.o aldebaran_reg_init.o aldebaran.o soc21.o soc24.o \
 	sienna_cichlid.o smu_v13_0_10.o nbio_v4_3.o hdp_v6_0.o nbio_v7_7.o hdp_v5_2.o lsdma_v6_0.o \
 	nbio_v7_9.o aqua_vanjaram.o nbio_v7_11.o lsdma_v7_0.o hdp_v7_0.o nbif_v6_3_1.o \
-	cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o
+	cyan_skillfish_reg_init.o soc_v1_0.o lsdma_v7_1.o nbio_v6_3_2.o
 
 # add DF block
 amdgpu-y += \
@@ -234,7 +234,8 @@ amdgpu-y += \
 # add VPE block
 amdgpu-y += \
 	amdgpu_vpe.o \
-	vpe_v6_1.o
+	vpe_v6_1.o \
+	vpe_v2_0.o
 
 # add UMSCH block
 amdgpu-y += \
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 8bc591deb546d..5d7bfa59424a4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -267,6 +267,7 @@ extern int amdgpu_rebar;
 
 extern int amdgpu_wbrf;
 extern int amdgpu_user_queue;
+extern int amdgpu_ptl;
 
 extern uint amdgpu_hdmi_hpd_debounce_delay_ms;
 
@@ -464,7 +465,7 @@ int amdgpu_file_to_fpriv(struct file *filp, struct amdgpu_fpriv **fpriv);
 #define AMDGPU_MAX_WB 1024	/* Reserve at most 1024 WB slots for amdgpu-owned rings. */
 
 /**
- * amdgpu_wb - This struct is used for small GPU memory allocation.
+ * struct amdgpu_wb - This struct is used for small GPU memory allocation.
  *
  * This struct is used to allocate a small amount of GPU memory that can be
  * used to shadow certain states into the memory. This is especially useful for
@@ -538,44 +539,6 @@ struct amdgpu_allowed_register_entry {
 	bool grbm_indexed;
 };
 
-/**
- * enum amd_reset_method - Methods for resetting AMD GPU devices
- *
- * @AMD_RESET_METHOD_NONE: The device will not be reset.
- * @AMD_RESET_LEGACY: Method reserved for SI, CIK and VI ASICs.
- * @AMD_RESET_MODE0: Reset the entire ASIC. Not currently available for the
- *                   any device.
- * @AMD_RESET_MODE1: Resets all IP blocks on the ASIC (SDMA, GFX, VCN, etc.)
- *                   individually. Suitable only for some discrete GPU, not
- *                   available for all ASICs.
- * @AMD_RESET_MODE2: Resets a lesser level of IPs compared to MODE1. Which IPs
- *                   are reset depends on the ASIC. Notably doesn't reset IPs
- *                   shared with the CPU on APUs or the memory controllers (so
- *                   VRAM is not lost). Not available on all ASICs.
- * @AMD_RESET_LINK: Triggers SW-UP link reset on other GPUs
- * @AMD_RESET_BACO: BACO (Bus Alive, Chip Off) method powers off and on the card
- *                  but without powering off the PCI bus. Suitable only for
- *                  discrete GPUs.
- * @AMD_RESET_PCI: Does a full bus reset using core Linux subsystem PCI reset
- *                 and does a secondary bus reset or FLR, depending on what the
- *                 underlying hardware supports.
- *
- * Methods available for AMD GPU driver for resetting the device. Not all
- * methods are suitable for every device. User can override the method using
- * module parameter `reset_method`.
- */
-enum amd_reset_method {
-	AMD_RESET_METHOD_NONE = -1,
-	AMD_RESET_METHOD_LEGACY = 0,
-	AMD_RESET_METHOD_MODE0,
-	AMD_RESET_METHOD_MODE1,
-	AMD_RESET_METHOD_MODE2,
-	AMD_RESET_METHOD_LINK,
-	AMD_RESET_METHOD_BACO,
-	AMD_RESET_METHOD_PCI,
-	AMD_RESET_METHOD_ON_INIT,
-};
-
 struct amdgpu_video_codec_info {
 	u32 codec_type;
 	u32 max_width;
@@ -1111,6 +1074,7 @@ struct amdgpu_device {
 	long				psp_timeout;
 
 	uint64_t			unique_id;
+	uint8_t				unitid;
 	uint64_t	df_perfmon_config_assign_mask[AMDGPU_MAX_DF_PERFMONS];
 
 	/* enable runtime pm on the device */
@@ -1190,7 +1154,6 @@ struct amdgpu_device {
 	bool                            apu_prefer_gtt;
 
 	bool                            userq_halt_for_enforce_isolation;
-	struct work_struct              userq_reset_work;
 	struct amdgpu_uid *uid_info;
 
 	struct amdgpu_uma_carveout_info uma_info;
@@ -1373,6 +1336,8 @@ int emu_soc_asic_init(struct amdgpu_device *adev);
 #define RBIOS16(i) (RBIOS8(i) | (RBIOS8((i)+1) << 8))
 #define RBIOS32(i) ((RBIOS16(i)) | (RBIOS16((i)+2) << 16))
 
+#include "amdgpu_reset.h"
+
 /*
  * ASICs macro.
  */
@@ -1462,6 +1427,8 @@ ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring);
 ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset);
 void amdgpu_sdma_set_vm_pte_scheds(struct amdgpu_device *adev,
 				   const struct amdgpu_vm_pte_funcs *vm_pte_funcs);
+void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev,
+					 const struct amdgpu_buffer_funcs *buffer_funcs);
 
 /* atpx handler */
 #if defined(CONFIG_VGA_SWITCHEROO)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
index 62807b65f2af8..4c732e0f776e1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.c
@@ -228,7 +228,6 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block)
 	u64 acp_base;
 	u32 val = 0;
 	u32 count = 0;
-	struct i2s_platform_data *i2s_pdata = NULL;
 
 	struct amdgpu_device *adev = ip_block->adev;
 
@@ -272,18 +271,18 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block)
 			goto failure;
 		}
 
-		i2s_pdata = kzalloc_objs(struct i2s_platform_data, 1);
-		if (!i2s_pdata) {
+		adev->acp.i2s_pdata = kzalloc_objs(struct i2s_platform_data, 1);
+		if (!adev->acp.i2s_pdata) {
 			r = -ENOMEM;
 			goto failure;
 		}
 
-		i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
-				      DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
-		i2s_pdata[0].cap = DWC_I2S_PLAY | DWC_I2S_RECORD;
-		i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000;
-		i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET;
-		i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET;
+		adev->acp.i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
+						DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
+		adev->acp.i2s_pdata[0].cap = DWC_I2S_PLAY | DWC_I2S_RECORD;
+		adev->acp.i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000;
+		adev->acp.i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET;
+		adev->acp.i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET;
 
 		adev->acp.acp_res[0].name = "acp2x_dma";
 		adev->acp.acp_res[0].flags = IORESOURCE_MEM;
@@ -311,7 +310,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block)
 		adev->acp.acp_cell[1].id = 1;
 		adev->acp.acp_cell[1].num_resources = 1;
 		adev->acp.acp_cell[1].resources = &adev->acp.acp_res[1];
-		adev->acp.acp_cell[1].platform_data = &i2s_pdata[0];
+		adev->acp.acp_cell[1].platform_data = &adev->acp.i2s_pdata[0];
 		adev->acp.acp_cell[1].pdata_size = sizeof(struct i2s_platform_data);
 		r = mfd_add_devices(adev->acp.parent, 0, adev->acp.acp_cell, 2, NULL, 0, NULL);
 		if (r)
@@ -336,53 +335,53 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block)
 			goto failure;
 		}
 
-		i2s_pdata = kzalloc_objs(struct i2s_platform_data, 3);
-		if (!i2s_pdata) {
+		adev->acp.i2s_pdata = kzalloc_objs(struct i2s_platform_data, 3);
+		if (!adev->acp.i2s_pdata) {
 			r = -ENOMEM;
 			goto failure;
 		}
 
 		switch (adev->asic_type) {
 		case CHIP_STONEY:
-			i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
-				DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
+			adev->acp.i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
+							DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
 			break;
 		default:
-			i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET;
+			adev->acp.i2s_pdata[0].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET;
 		}
-		i2s_pdata[0].cap = DWC_I2S_PLAY;
-		i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000;
-		i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_PLAY_REG_OFFSET;
-		i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_PLAY_REG_OFFSET;
+		adev->acp.i2s_pdata[0].cap = DWC_I2S_PLAY;
+		adev->acp.i2s_pdata[0].snd_rates = SNDRV_PCM_RATE_8000_96000;
+		adev->acp.i2s_pdata[0].i2s_reg_comp1 = ACP_I2S_COMP1_PLAY_REG_OFFSET;
+		adev->acp.i2s_pdata[0].i2s_reg_comp2 = ACP_I2S_COMP2_PLAY_REG_OFFSET;
 		switch (adev->asic_type) {
 		case CHIP_STONEY:
-			i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
-				DW_I2S_QUIRK_COMP_PARAM1 |
-				DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
+			adev->acp.i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
+							DW_I2S_QUIRK_COMP_PARAM1 |
+							DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
 			break;
 		default:
-			i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
-				DW_I2S_QUIRK_COMP_PARAM1;
+			adev->acp.i2s_pdata[1].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET |
+							DW_I2S_QUIRK_COMP_PARAM1;
 		}
 
-		i2s_pdata[1].cap = DWC_I2S_RECORD;
-		i2s_pdata[1].snd_rates = SNDRV_PCM_RATE_8000_96000;
-		i2s_pdata[1].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET;
-		i2s_pdata[1].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET;
+		adev->acp.i2s_pdata[1].cap = DWC_I2S_RECORD;
+		adev->acp.i2s_pdata[1].snd_rates = SNDRV_PCM_RATE_8000_96000;
+		adev->acp.i2s_pdata[1].i2s_reg_comp1 = ACP_I2S_COMP1_CAP_REG_OFFSET;
+		adev->acp.i2s_pdata[1].i2s_reg_comp2 = ACP_I2S_COMP2_CAP_REG_OFFSET;
 
-		i2s_pdata[2].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET;
+		adev->acp.i2s_pdata[2].quirks = DW_I2S_QUIRK_COMP_REG_OFFSET;
 		switch (adev->asic_type) {
 		case CHIP_STONEY:
-			i2s_pdata[2].quirks |= DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
+			adev->acp.i2s_pdata[2].quirks |= DW_I2S_QUIRK_16BIT_IDX_OVERRIDE;
 			break;
 		default:
 			break;
 		}
 
-		i2s_pdata[2].cap = DWC_I2S_PLAY | DWC_I2S_RECORD;
-		i2s_pdata[2].snd_rates = SNDRV_PCM_RATE_8000_96000;
-		i2s_pdata[2].i2s_reg_comp1 = ACP_BT_COMP1_REG_OFFSET;
-		i2s_pdata[2].i2s_reg_comp2 = ACP_BT_COMP2_REG_OFFSET;
+		adev->acp.i2s_pdata[2].cap = DWC_I2S_PLAY | DWC_I2S_RECORD;
+		adev->acp.i2s_pdata[2].snd_rates = SNDRV_PCM_RATE_8000_96000;
+		adev->acp.i2s_pdata[2].i2s_reg_comp1 = ACP_BT_COMP1_REG_OFFSET;
+		adev->acp.i2s_pdata[2].i2s_reg_comp2 = ACP_BT_COMP2_REG_OFFSET;
 
 		adev->acp.acp_res[0].name = "acp2x_dma";
 		adev->acp.acp_res[0].flags = IORESOURCE_MEM;
@@ -420,21 +419,21 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block)
 		adev->acp.acp_cell[1].id = 1;
 		adev->acp.acp_cell[1].num_resources = 1;
 		adev->acp.acp_cell[1].resources = &adev->acp.acp_res[1];
-		adev->acp.acp_cell[1].platform_data = &i2s_pdata[0];
+		adev->acp.acp_cell[1].platform_data = &adev->acp.i2s_pdata[0];
 		adev->acp.acp_cell[1].pdata_size = sizeof(struct i2s_platform_data);
 
 		adev->acp.acp_cell[2].name = "designware-i2s";
 		adev->acp.acp_cell[2].id = 2;
 		adev->acp.acp_cell[2].num_resources = 1;
 		adev->acp.acp_cell[2].resources = &adev->acp.acp_res[2];
-		adev->acp.acp_cell[2].platform_data = &i2s_pdata[1];
+		adev->acp.acp_cell[2].platform_data = &adev->acp.i2s_pdata[1];
 		adev->acp.acp_cell[2].pdata_size = sizeof(struct i2s_platform_data);
 
 		adev->acp.acp_cell[3].name = "designware-i2s";
 		adev->acp.acp_cell[3].id = 3;
 		adev->acp.acp_cell[3].num_resources = 1;
 		adev->acp.acp_cell[3].resources = &adev->acp.acp_res[3];
-		adev->acp.acp_cell[3].platform_data = &i2s_pdata[2];
+		adev->acp.acp_cell[3].platform_data = &adev->acp.i2s_pdata[2];
 		adev->acp.acp_cell[3].pdata_size = sizeof(struct i2s_platform_data);
 
 		r = mfd_add_devices(adev->acp.parent, 0, adev->acp.acp_cell, ACP_DEVS, NULL, 0, NULL);
@@ -491,7 +490,7 @@ static int acp_hw_init(struct amdgpu_ip_block *ip_block)
 	return 0;
 
 failure:
-	kfree(i2s_pdata);
+	kfree(adev->acp.i2s_pdata);
 	kfree(adev->acp.acp_res);
 	kfree(adev->acp.acp_cell);
 	kfree(adev->acp.acp_genpd);
@@ -556,6 +555,7 @@ static int acp_hw_fini(struct amdgpu_ip_block *ip_block)
 			      acp_genpd_remove_device);
 
 	mfd_remove_devices(adev->acp.parent);
+	kfree(adev->acp.i2s_pdata);
 	kfree(adev->acp.acp_res);
 	kfree(adev->acp.acp_genpd);
 	kfree(adev->acp.acp_cell);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h
index a288ce25c1765..13b48c5823144 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_acp.h
@@ -35,6 +35,7 @@ struct amdgpu_acp {
 	struct mfd_cell *acp_cell;
 	struct resource *acp_res;
 	struct acp_pm_domain *acp_genpd;
+	struct i2s_platform_data *i2s_pdata;
 };
 
 extern const struct amdgpu_ip_block_version acp_ip_block;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index d9e283f3b57d5..9783a3cefb04b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -36,6 +36,9 @@
 #include "amdgpu_ras.h"
 #include "amdgpu_umc.h"
 #include "amdgpu_reset.h"
+#if IS_ENABLED(CONFIG_HSA_AMD)
+#include "kfd_priv.h"
+#endif
 
 /* Total memory size in system memory and all GPU VRAM. Used to
  * estimate worst case amount of memory to reserve for page tables
@@ -320,6 +323,28 @@ void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev)
 		(void)amdgpu_reset_domain_schedule(adev->reset_domain, &adev->kfd.reset_work);
 }
 
+void amdgpu_amdkfd_clear_kfd_mapping(struct amdgpu_device *adev)
+{
+#if IS_ENABLED(CONFIG_HSA_AMD)
+	struct kfd_dev *kfd = adev->kfd.dev;
+	unsigned int i;
+
+	if (!kfd)
+		return;
+
+	for (i = 0; i < kfd->num_nodes; i++) {
+		struct kfd_node *node = kfd->nodes[i];
+
+		kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_DOORBELL |
+					    KFD_MMAP_GPU_ID(node->id),
+					    kfd_doorbell_process_slice(kfd));
+		kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_MMIO |
+					    KFD_MMAP_GPU_ID(node->id),
+					    PAGE_SIZE);
+	}
+#endif
+}
+
 int amdgpu_amdkfd_alloc_kernel_mem(struct amdgpu_device *adev, size_t size,
 				u32 domain, void **mem_obj, uint64_t *gpu_addr,
 				void **cpu_ptr, bool cp_mqd_gfx9)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
index cdbab7f8cee8c..5333e052d56db 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h
@@ -38,6 +38,8 @@
 #include "amdgpu_vm.h"
 #include "amdgpu_xcp.h"
 #include "kfd_topology.h"
+#include "amdgpu_ptl.h"
+
 extern uint64_t amdgpu_amdkfd_total_mem_size;
 
 enum TLB_FLUSH_TYPE {
@@ -358,6 +360,7 @@ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev,
 		uint64_t size, u32 alloc_flag, int8_t xcp_id);
 void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev,
 		uint64_t size, u32 alloc_flag, int8_t xcp_id);
+void amdgpu_amdkfd_clear_kfd_mapping(struct amdgpu_device *adev);
 
 u64 amdgpu_amdkfd_xcp_memory_size(struct amdgpu_device *adev, int xcp_id);
 
@@ -434,8 +437,10 @@ int kgd2kfd_check_and_lock_kfd(struct kfd_dev *kfd);
 void kgd2kfd_unlock_kfd(struct kfd_dev *kfd);
 int kgd2kfd_start_sched(struct kfd_dev *kfd, uint32_t node_id);
 int kgd2kfd_start_sched_all_nodes(struct kfd_dev *kfd);
+int amdgpu_amdkfd_start_sched_all(struct amdgpu_device *adev);
 int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id);
 int kgd2kfd_stop_sched_all_nodes(struct kfd_dev *kfd);
+int amdgpu_amdkfd_stop_sched_all(struct amdgpu_device *adev);
 bool kgd2kfd_compute_active(struct kfd_dev *kfd, uint32_t node_id);
 bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entry *entry,
 			       bool retry_fault);
@@ -532,6 +537,11 @@ static inline int kgd2kfd_start_sched_all_nodes(struct kfd_dev *kfd)
 	return 0;
 }
 
+static inline int amdgpu_amdkfd_start_sched_all(struct amdgpu_device *adev)
+{
+	return 0;
+}
+
 static inline int kgd2kfd_stop_sched(struct kfd_dev *kfd, uint32_t node_id)
 {
 	return 0;
@@ -542,6 +552,11 @@ static inline int kgd2kfd_stop_sched_all_nodes(struct kfd_dev *kfd)
 	return 0;
 }
 
+static inline int amdgpu_amdkfd_stop_sched_all(struct amdgpu_device *adev)
+{
+	return 0;
+}
+
 static inline bool kgd2kfd_compute_active(struct kfd_dev *kfd, uint32_t node_id)
 {
 	return false;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
index f35947be37632..6ed399163547a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gc_9_4_3.c
@@ -520,6 +520,16 @@ static uint32_t kgd_gfx_v9_4_3_hqd_sdma_get_doorbell(struct amdgpu_device *adev,
 	return is_active ? doorbell_off >> 2 : 0;
 }
 
+static uint32_t kgd_v9_4_3_ptl_ctrl(struct amdgpu_device *adev,
+				uint32_t cmd,
+				uint32_t *ptl_state,
+				enum amdgpu_ptl_fmt *fmt1,
+				enum amdgpu_ptl_fmt *fmt2)
+{
+	return amdgpu_ptl_perf_monitor_ctrl(adev, cmd,
+			ptl_state, fmt1, fmt2);
+}
+
 const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = {
 	.program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings,
 	.set_pasid_vmid_mapping = kgd_gfx_v9_4_3_set_pasid_vmid_mapping,
@@ -555,5 +565,6 @@ const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = {
 	.clear_address_watch = kgd_gfx_v9_4_3_clear_address_watch,
 	.hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr,
 	.hqd_reset = kgd_gfx_v9_hqd_reset,
-	.hqd_sdma_get_doorbell = kgd_gfx_v9_4_3_hqd_sdma_get_doorbell
+	.hqd_sdma_get_doorbell = kgd_gfx_v9_4_3_hqd_sdma_get_doorbell,
+	.ptl_ctrl = kgd_v9_4_3_ptl_ctrl
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index 3698dd0330ff8..acd22bff1882d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -36,6 +36,21 @@
 #include "atombios_encoders.h"
 #include "bif/bif_4_1_d.h"
 
+/* VBIOS-reported table size is unchecked against the image; cap iterations and
+ * adev->i2c_bus[] indexing to AMDGPU_MAX_I2C_BUS.
+ */
+static int amdgpu_atombios_gpio_i2c_num_entries(uint16_t size)
+{
+	u32 bytes;
+
+	if (size < sizeof(ATOM_COMMON_TABLE_HEADER))
+		return 0;
+
+	bytes = size - sizeof(ATOM_COMMON_TABLE_HEADER);
+	return (int)min_t(u32, bytes / sizeof(ATOM_GPIO_I2C_ASSIGMENT),
+			  AMDGPU_MAX_I2C_BUS);
+}
+
 static struct amdgpu_i2c_bus_rec amdgpu_atombios_get_bus_rec_for_i2c_gpio(ATOM_GPIO_I2C_ASSIGMENT *gpio)
 {
 	struct amdgpu_i2c_bus_rec i2c;
@@ -96,8 +111,7 @@ struct amdgpu_i2c_bus_rec amdgpu_atombios_lookup_i2c_gpio(struct amdgpu_device *
 	if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) {
 		i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset);
 
-		num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) /
-			sizeof(ATOM_GPIO_I2C_ASSIGMENT);
+		num_indices = amdgpu_atombios_gpio_i2c_num_entries(size);
 
 		gpio = &i2c_info->asGPIO_Info[0];
 		for (i = 0; i < num_indices; i++) {
@@ -127,8 +141,7 @@ void amdgpu_atombios_i2c_init(struct amdgpu_device *adev)
 	if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) {
 		i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset);
 
-		num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) /
-			sizeof(ATOM_GPIO_I2C_ASSIGMENT);
+		num_indices = amdgpu_atombios_gpio_i2c_num_entries(size);
 
 		gpio = &i2c_info->asGPIO_Info[0];
 		for (i = 0; i < num_indices; i++) {
@@ -158,8 +171,7 @@ void amdgpu_atombios_oem_i2c_init(struct amdgpu_device *adev, u8 i2c_id)
 	if (amdgpu_atom_parse_data_header(ctx, index, &size, NULL, NULL, &data_offset)) {
 		i2c_info = (struct _ATOM_GPIO_I2C_INFO *)(ctx->bios + data_offset);
 
-		num_indices = (size - sizeof(ATOM_COMMON_TABLE_HEADER)) /
-			sizeof(ATOM_GPIO_I2C_ASSIGMENT);
+		num_indices = amdgpu_atombios_gpio_i2c_num_entries(size);
 
 		gpio = &i2c_info->asGPIO_Info[0];
 		for (i = 0; i < num_indices; i++) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index 35d04e69aec09..aa039e148a5ee 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -33,6 +33,7 @@
 #include <linux/pci.h>
 #include <linux/slab.h>
 #include <linux/acpi.h>
+#include <linux/vgaarb.h>
 /*
  * BIOS.
  */
@@ -467,7 +468,8 @@ static bool amdgpu_prefer_rom_resource(struct amdgpu_device *adev)
 {
 	struct resource *res = &adev->pdev->resource[PCI_ROM_RESOURCE];
 
-	return (res->flags & IORESOURCE_ROM_SHADOW);
+	return (res->flags & IORESOURCE_ROM_SHADOW) ||
+	       adev->pdev == vga_default_device();
 }
 
 static bool amdgpu_get_bios_dgpu(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
index 4e6e390854e6e..d5e59c24d907d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cper.c
@@ -153,7 +153,7 @@ int amdgpu_cper_entry_fill_fatal_section(struct amdgpu_device *adev,
 		   FATAL_SEC_OFFSET(hdr->sec_cnt, idx));
 
 	amdgpu_cper_entry_fill_section_desc(adev, section_desc, false, false,
-					    CPER_SEV_FATAL, CRASHDUMP, FATAL_SEC_LEN,
+					    CPER_SEV_FATAL_UNCORRECTED, CRASHDUMP, FATAL_SEC_LEN,
 					    FATAL_SEC_OFFSET(hdr->sec_cnt, idx));
 
 	section->body.reg_ctx_type = CPER_CTX_TYPE_CRASH;
@@ -215,7 +215,7 @@ int amdgpu_cper_entry_fill_bad_page_threshold_section(struct amdgpu_device *adev
 		   NONSTD_SEC_OFFSET(hdr->sec_cnt, idx));
 
 	amdgpu_cper_entry_fill_section_desc(adev, section_desc, true, false,
-					    CPER_SEV_FATAL, RUNTIME, NONSTD_SEC_LEN,
+					    CPER_SEV_FATAL_UNCORRECTED, RUNTIME, NONSTD_SEC_LEN,
 					    NONSTD_SEC_OFFSET(hdr->sec_cnt, idx));
 
 	section->hdr.valid_bits.err_info_cnt = 1;
@@ -312,7 +312,7 @@ int amdgpu_cper_generate_ue_record(struct amdgpu_device *adev,
 	reg_data.synd_lo   = lower_32_bits(bank->regs[ACA_REG_IDX_SYND]);
 	reg_data.synd_hi   = upper_32_bits(bank->regs[ACA_REG_IDX_SYND]);
 
-	amdgpu_cper_entry_fill_hdr(adev, fatal, AMDGPU_CPER_TYPE_FATAL, CPER_SEV_FATAL);
+	amdgpu_cper_entry_fill_hdr(adev, fatal, AMDGPU_CPER_TYPE_FATAL, CPER_SEV_FATAL_UNCORRECTED);
 	ret = amdgpu_cper_entry_fill_fatal_section(adev, fatal, 0, reg_data);
 	if (ret)
 		return ret;
@@ -337,7 +337,7 @@ int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev)
 
 	amdgpu_cper_entry_fill_hdr(adev, bp_threshold,
 				   AMDGPU_CPER_TYPE_BP_THRESHOLD,
-				   CPER_SEV_FATAL);
+				   CPER_SEV_FATAL_UNCORRECTED);
 	ret = amdgpu_cper_entry_fill_bad_page_threshold_section(adev, bp_threshold, 0);
 	if (ret)
 		return ret;
@@ -353,14 +353,14 @@ static enum cper_error_severity amdgpu_aca_err_type_to_cper_sev(struct amdgpu_de
 {
 	switch (aca_err_type) {
 	case ACA_ERROR_TYPE_UE:
-		return CPER_SEV_FATAL;
+		return CPER_SEV_FATAL_UNCORRECTED;
 	case ACA_ERROR_TYPE_CE:
 		return CPER_SEV_NON_FATAL_CORRECTED;
 	case ACA_ERROR_TYPE_DEFERRED:
 		return CPER_SEV_NON_FATAL_UNCORRECTED;
 	default:
 		dev_err(adev->dev, "Unknown ACA error type!\n");
-		return CPER_SEV_FATAL;
+		return CPER_SEV_FATAL_UNCORRECTED;
 	}
 }
 
@@ -484,7 +484,7 @@ static u32 amdgpu_cper_ring_get_ent_sz(struct amdgpu_ring *ring, u64 pos)
 
 void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count)
 {
-	u64 pos, wptr_old, rptr;
+	u64 pos, wptr_old, rptr, next_rptr;
 	int rec_cnt_dw = count >> 2;
 	u32 chunk, ent_sz;
 	u8 *s = (u8 *)src;
@@ -525,9 +525,19 @@ void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count)
 
 		do {
 			ent_sz = amdgpu_cper_ring_get_ent_sz(ring, pos);
-
-			rptr += (ent_sz >> 2);
-			rptr &= ring->ptr_mask;
+			next_rptr = rptr;
+			if (ent_sz >= sizeof(u32))
+				next_rptr = (rptr + (ent_sz >> 2)) & ring->ptr_mask;
+
+			if (next_rptr == rptr) {
+				/* Corrupt entry size, reset the ring to avoid an infinite loop. */
+				rptr = ring->wptr;
+				*ring->rptr_cpu_addr = rptr;
+				ring->count_dw = (ring->ring_size - 4) >> 2;
+				goto out_unlock;
+			}
+
+			rptr = next_rptr;
 			*ring->rptr_cpu_addr = rptr;
 
 			pos = rptr;
@@ -536,6 +546,8 @@ void amdgpu_cper_ring_write(struct amdgpu_ring *ring, void *src, int count)
 
 	if (ring->count_dw >= rec_cnt_dw)
 		ring->count_dw -= rec_cnt_dw;
+
+out_unlock:
 	mutex_unlock(&ring->adev->cper.ring_lock);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index b24d5d21be5f2..32af8cce3df8d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -1100,7 +1100,8 @@ static int amdgpu_cs_vm_handling(struct amdgpu_cs_parser *p)
 	if (p->gang_size > 1 && !adev->vm_manager.concurrent_flush) {
 		for (i = 0; i < p->gang_size; ++i) {
 			struct drm_sched_entity *entity = p->entities[i];
-			struct drm_gpu_scheduler *sched = entity->rq->sched;
+			struct drm_gpu_scheduler *sched =
+				container_of(entity->rq, typeof(*sched), rq);
 			struct amdgpu_ring *ring = to_amdgpu_ring(sched);
 
 			if (amdgpu_vmid_uses_reserved(vm, ring->vm_hub))
@@ -1231,7 +1232,8 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p)
 			return r;
 	}
 
-	sched = p->gang_leader->base.entity->rq->sched;
+	sched = container_of(p->gang_leader->base.entity->rq, typeof(*sched),
+			     rq);
 	while ((fence = amdgpu_sync_get_fence(&p->sync))) {
 		struct drm_sched_fence *s_fence = to_drm_sched_fence(fence);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
index 092fd33090998..389bad7242731 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c
@@ -2049,7 +2049,7 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
 		/* swap out the old fences */
 		amdgpu_ib_preempt_fences_swap(ring, fences);
 
-		amdgpu_fence_driver_force_completion(ring);
+		amdgpu_fence_driver_force_completion(ring, NULL);
 
 		/* resubmit unfinished jobs */
 		amdgpu_ib_preempt_job_recovery(&ring->sched);
@@ -2126,6 +2126,9 @@ int amdgpu_debugfs_init(struct amdgpu_device *adev)
 	debugfs_create_x32("amdgpu_smu_debug", 0600, root,
 			   &adev->pm.smu_debug_mask);
 
+	debugfs_create_x64("unique_id", 0444, root, &adev->unique_id);
+	debugfs_create_x8("unitid", 0444, root, &adev->unitid);
+
 	ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
 				  &fops_ib_preempt);
 	if (IS_ERR(ent)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 66ca043658ff8..21a3fb574d53e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -2221,7 +2221,6 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
 {
 	struct drm_sched_init_args args = {
 		.ops = &amdgpu_sched_ops,
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.timeout_wq = adev->reset_domain->wq,
 		.dev = adev->dev,
 	};
@@ -2460,7 +2459,7 @@ static int amdgpu_device_ip_init(struct amdgpu_device *adev)
 	if (r)
 		goto init_failed;
 
-	amdgpu_ttm_set_buffer_funcs_status(adev, true);
+	amdgpu_ttm_enable_buffer_funcs(adev);
 
 	/* Don't init kfd if whole hive need to be reset during init */
 	if (adev->init_lvl->level != AMDGPU_INIT_LEVEL_MINIMAL_XGMI) {
@@ -3148,7 +3147,7 @@ static int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
 		amdgpu_virt_request_full_gpu(adev, false);
 	}
 
-	amdgpu_ttm_set_buffer_funcs_status(adev, false);
+	amdgpu_ttm_disable_buffer_funcs(adev);
 
 	r = amdgpu_device_ip_suspend_phase1(adev);
 	if (r)
@@ -3363,7 +3362,7 @@ static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
 
 	r = amdgpu_device_ip_resume_phase2(adev);
 
-	amdgpu_ttm_set_buffer_funcs_status(adev, true);
+	amdgpu_ttm_enable_buffer_funcs(adev);
 
 	if (r)
 		return r;
@@ -3648,6 +3647,7 @@ static int amdgpu_device_sys_interface_init(struct amdgpu_device *adev)
 	amdgpu_reg_state_sysfs_init(adev);
 	amdgpu_xcp_sysfs_init(adev);
 	amdgpu_uma_sysfs_init(adev);
+	amdgpu_ptl_sysfs_init(adev);
 
 	return r;
 }
@@ -3664,6 +3664,7 @@ static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev)
 	amdgpu_reg_state_sysfs_fini(adev);
 	amdgpu_xcp_sysfs_fini(adev);
 	amdgpu_uma_sysfs_fini(adev);
+	amdgpu_ptl_sysfs_fini(adev);
 }
 
 /**
@@ -3701,7 +3702,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	adev->num_rings = 0;
 	RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
 	adev->mman.buffer_funcs = NULL;
-	adev->mman.buffer_funcs_ring = NULL;
+	adev->mman.num_buffer_funcs_scheds = 0;
 	adev->vm_manager.vm_pte_funcs = NULL;
 	adev->vm_manager.vm_pte_num_scheds = 0;
 	adev->gmc.gmc_funcs = NULL;
@@ -3732,6 +3733,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	mutex_init(&adev->virt.vf_errors.lock);
 	hash_init(adev->mn_hash);
 	mutex_init(&adev->psp.mutex);
+	mutex_init(&adev->psp.ptl.mutex);
 	mutex_init(&adev->notifier_lock);
 	mutex_init(&adev->pm.stable_pstate_ctx_lock);
 	mutex_init(&adev->benchmark_mutex);
@@ -3787,7 +3789,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	}
 
 	INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
-	INIT_WORK(&adev->userq_reset_work, amdgpu_userq_reset_work);
 
 	amdgpu_coredump_init(adev);
 
@@ -3858,6 +3859,9 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	 * completed before the need for a different level is detected.
 	 */
 	amdgpu_set_init_level(adev, AMDGPU_INIT_LEVEL_DEFAULT);
+
+	amdgpu_device_check_iommu_direct_map(adev);
+
 	/* early init functions */
 	r = amdgpu_device_ip_early_init(adev);
 	if (r)
@@ -4115,8 +4119,6 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	if (px)
 		vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
 
-	amdgpu_device_check_iommu_direct_map(adev);
-
 	adev->pm_nb.notifier_call = amdgpu_device_pm_notifier;
 	r = register_pm_notifier(&adev->pm_nb);
 	if (r)
@@ -4214,7 +4216,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev)
 	/* disable ras feature must before hw fini */
 	amdgpu_ras_pre_fini(adev);
 
-	amdgpu_ttm_set_buffer_funcs_status(adev, false);
+	amdgpu_ttm_disable_buffer_funcs(adev);
 
 	/*
 	 * device went through surprise hotplug; we need to destroy topology
@@ -4481,7 +4483,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
 	if (r)
 		goto unwind_userq;
 
-	amdgpu_ttm_set_buffer_funcs_status(adev, false);
+	amdgpu_ttm_disable_buffer_funcs(adev);
 
 	amdgpu_fence_driver_hw_fini(adev);
 
@@ -4495,7 +4497,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool notify_clients)
 	return 0;
 
 unwind_evict:
-	amdgpu_ttm_set_buffer_funcs_status(adev, true);
+	amdgpu_ttm_enable_buffer_funcs(adev);
 	amdgpu_fence_driver_hw_init(adev);
 
 unwind_userq:
@@ -5090,6 +5092,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 {
 	int i, r = 0;
 	struct amdgpu_job *job = NULL;
+	struct dma_fence *fence = NULL;
 	struct amdgpu_device *tmp_adev = reset_context->reset_req_dev;
 	bool need_full_reset =
 		test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
@@ -5102,6 +5105,9 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 
 	amdgpu_fence_driver_isr_toggle(adev, true);
 
+	if (job)
+		fence = &job->hw_fence->base;
+
 	/* block all schedulers and reset given job's ring */
 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
 		struct amdgpu_ring *ring = adev->rings[i];
@@ -5110,7 +5116,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
 			continue;
 
 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
-		amdgpu_fence_driver_force_completion(ring);
+		amdgpu_fence_driver_force_completion(ring, fence);
 	}
 
 	amdgpu_fence_driver_isr_toggle(adev, false);
@@ -5229,7 +5235,7 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
 				if (r)
 					goto out;
 
-				amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
+				amdgpu_ttm_enable_buffer_funcs(tmp_adev);
 
 				r = amdgpu_device_ip_resume_phase3(tmp_adev);
 				if (r)
@@ -5478,7 +5484,7 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
 	if (!amdgpu_sriov_vf(adev))
 		cancel_work(&adev->reset_work);
 #endif
-	cancel_work(&adev->userq_reset_work);
+	amdgpu_userq_mgr_cancel_reset_work(adev);
 
 	if (adev->kfd.dev)
 		cancel_work(&adev->kfd.reset_work);
@@ -5836,6 +5842,12 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
 	/* We need to lock reset domain only once both for XGMI and single device */
 	amdgpu_device_recovery_get_reset_lock(adev, &device_list);
 
+	/* unmap all the mappings of doorbell and framebuffer to prevent user space from
+	 * accessing them
+	 */
+	unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
+	amdgpu_amdkfd_clear_kfd_mapping(adev);
+
 	amdgpu_device_halt_activities(adev, job, reset_context, &device_list,
 				      hive, need_emergency_restart);
 	if (need_emergency_restart)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
index 8d99bfaa498f6..21f6761624c2c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c
@@ -64,6 +64,7 @@
 #include "nbio_v7_2.h"
 #include "nbio_v7_7.h"
 #include "nbif_v6_3_1.h"
+#include "nbio_v6_3_2.h"
 #include "hdp_v5_0.h"
 #include "hdp_v5_2.h"
 #include "hdp_v6_0.h"
@@ -304,7 +305,7 @@ static int amdgpu_discovery_get_tmr_info(struct amdgpu_device *adev,
 				adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].offset;
 			adev->discovery.size =
 				adev->virt.crit_regn_tbl[AMD_SRIOV_MSG_IPD_TABLE_ID].size_kb << 10;
-			if (!adev->discovery.offset || !adev->discovery.size)
+			if (!adev->discovery.size)
 				return -EINVAL;
 		} else {
 			goto out;
@@ -2737,6 +2738,9 @@ static int amdgpu_discovery_set_vpe_ip_blocks(struct amdgpu_device *adev)
 	case IP_VERSION(6, 1, 3):
 		amdgpu_device_ip_block_add(adev, &vpe_v6_1_ip_block);
 		break;
+	case IP_VERSION(2, 0, 0):
+		amdgpu_device_ip_block_add(adev, &vpe_v2_0_ip_block);
+		break;
 	default:
 		break;
 	}
@@ -3193,6 +3197,9 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 		adev->nbio.funcs = &nbif_v6_3_1_funcs;
 		adev->nbio.hdp_flush_reg = &nbif_v6_3_1_hdp_flush_reg;
 		break;
+	case IP_VERSION(6, 3, 2):
+		adev->nbio.funcs = &nbio_v6_3_2_funcs;
+		break;
 	default:
 		break;
 	}
@@ -3416,3 +3423,28 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev)
 	return 0;
 }
 
+int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev,
+						uint16_t *major, uint16_t *minor)
+{
+	uint8_t *discovery_bin = adev->discovery.bin;
+	struct table_info *info;
+	union gc_info *gc_info;
+	u16 offset;
+
+	if (!discovery_bin)
+		return -EINVAL;
+	if (amdgpu_discovery_get_table_info(adev, &info, GC))
+		return -EINVAL;
+
+	offset = le16_to_cpu(info->offset);
+	if (!offset)
+		return -EINVAL;
+
+	gc_info = (union gc_info *)(discovery_bin + offset);
+
+	if (major)
+		*major = le16_to_cpu(gc_info->v1.header.version_major);
+	if (minor)
+		*minor = le16_to_cpu(gc_info->v1.header.version_minor);
+	return 0;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
index 0ff1a7923eedf..e0010f6a3eda5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.h
@@ -48,6 +48,8 @@ int amdgpu_discovery_get_nps_info(struct amdgpu_device *adev,
 				  uint32_t *nps_type,
 				  struct amdgpu_gmc_memrange *ranges,
 				  int *range_cnt, bool refresh);
+int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev,
+						uint16_t *major, uint16_t *minor);
 
 void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 60debd543e44e..1781c0c3d010f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -246,6 +246,7 @@ int amdgpu_umsch_mm_fwlog;
 int amdgpu_rebar = -1; /* auto */
 int amdgpu_user_queue = -1;
 uint amdgpu_hdmi_hpd_debounce_delay_ms;
+int amdgpu_ptl = -1; /* auto */
 
 DECLARE_DYNDBG_CLASSMAP(drm_debug_classes, DD_CLASS_TYPE_DISJOINT_BITS, 0,
 			"DRM_UT_CORE",
@@ -1112,6 +1113,18 @@ module_param_named(user_queue, amdgpu_user_queue, int, 0444);
 MODULE_PARM_DESC(hdmi_hpd_debounce_delay_ms, "HDMI HPD disconnect debounce delay in milliseconds (0 to disable (by default), 1500 is common)");
 module_param_named(hdmi_hpd_debounce_delay_ms, amdgpu_hdmi_hpd_debounce_delay_ms, uint, 0644);
 
+/**
+ * DOC: ptl (int)
+ * Enable PTL feature at boot time. Possible values:
+ *
+ * - -1 = auto (ASIC specific default)
+ * -  0 = disable PTL (default)
+ * -  1 = enable PTL
+ * -  2 = permanently disable PTL (cannot be re-enabled at runtime)
+ */
+MODULE_PARM_DESC(ptl, "Enable PTL (-1 = auto, 0 = disable (default), 1 = enable, 2 = permanently disable)");
+module_param_named(ptl, amdgpu_ptl, int, 0444);
+
 /* These devices are not supported by amdgpu.
  * They are supported by the mach64, r128, radeon drivers
  */
@@ -2797,12 +2810,11 @@ static int amdgpu_runtime_idle_check_userq(struct device *dev)
 	return xa_empty(&adev->userq_doorbell_xa) ? 0 : -EBUSY;
 }
 
-static int amdgpu_pmops_runtime_suspend(struct device *dev)
+static int amdgpu_pmops_runtime_checks(struct device *dev)
 {
-	struct pci_dev *pdev = to_pci_dev(dev);
-	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	struct drm_device *drm_dev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = drm_to_adev(drm_dev);
-	int ret, i;
+	int ret;
 
 	if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) {
 		pm_runtime_forbid(dev);
@@ -2812,7 +2824,27 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev)
 	ret = amdgpu_runtime_idle_check_display(dev);
 	if (ret)
 		return ret;
-	ret = amdgpu_runtime_idle_check_userq(dev);
+
+	return amdgpu_runtime_idle_check_userq(dev);
+}
+
+static int amdgpu_pmops_runtime_idle(struct device *dev)
+{
+	int ret;
+
+	ret = amdgpu_pmops_runtime_checks(dev);
+	pm_runtime_autosuspend(dev);
+	return ret;
+}
+
+static int amdgpu_pmops_runtime_suspend(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct drm_device *drm_dev = pci_get_drvdata(pdev);
+	struct amdgpu_device *adev = drm_to_adev(drm_dev);
+	int ret, i;
+
+	ret = amdgpu_pmops_runtime_checks(dev);
 	if (ret)
 		return ret;
 
@@ -2924,27 +2956,6 @@ static int amdgpu_pmops_runtime_resume(struct device *dev)
 	return 0;
 }
 
-static int amdgpu_pmops_runtime_idle(struct device *dev)
-{
-	struct drm_device *drm_dev = dev_get_drvdata(dev);
-	struct amdgpu_device *adev = drm_to_adev(drm_dev);
-	int ret;
-
-	if (adev->pm.rpm_mode == AMDGPU_RUNPM_NONE) {
-		pm_runtime_forbid(dev);
-		return -EBUSY;
-	}
-
-	ret = amdgpu_runtime_idle_check_display(dev);
-	if (ret)
-		goto done;
-
-	ret = amdgpu_runtime_idle_check_userq(dev);
-done:
-	pm_runtime_autosuspend(dev);
-	return ret;
-}
-
 static int amdgpu_drm_release(struct inode *inode, struct file *filp)
 {
 	struct drm_file *file_priv = filp->private_data;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index 8048a4c04b47e..ea69b1bac7c6e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -547,7 +547,7 @@ void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev)
 			r = -ENODEV;
 		/* no need to trigger GPU reset as we are unloading */
 		if (r)
-			amdgpu_fence_driver_force_completion(ring);
+			amdgpu_fence_driver_force_completion(ring, NULL);
 
 		if (!drm_dev_is_unplugged(adev_to_drm(adev)) &&
 		    ring->fence_drv.irq_src &&
@@ -662,16 +662,34 @@ void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error)
  * amdgpu_fence_driver_force_completion - force signal latest fence of ring
  *
  * @ring: fence of the ring to signal
+ * @timedout_fence: fence of the timedout job
  *
  */
-void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
+void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring,
+					  struct dma_fence *timedout_fence)
 {
-	amdgpu_fence_driver_set_error(ring, -ECANCELED);
+	struct amdgpu_fence_driver *drv = &ring->fence_drv;
+	unsigned long flags;
+
+	spin_lock_irqsave(&drv->lock, flags);
+	for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) {
+		struct dma_fence *fence;
+
+		fence = rcu_dereference_protected(drv->fences[i],
+						  lockdep_is_held(&drv->lock));
+		if (fence && !dma_fence_is_signaled_locked(fence)) {
+			if (fence == timedout_fence)
+				dma_fence_set_error(fence, -ETIME);
+			else
+				dma_fence_set_error(fence, -ECANCELED);
+		}
+	}
+	spin_unlock_irqrestore(&drv->lock, flags);
+
 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
 	amdgpu_fence_process(ring);
 }
 
-
 /*
  * Kernel queue reset handling
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 5376035d32fe8..27be5083f2af6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -27,10 +27,12 @@
  */
 #include <linux/ktime.h>
 #include <linux/module.h>
+#include <linux/overflow.h>
 #include <linux/pagemap.h>
 #include <linux/pci.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-fence-unwrap.h>
+#include <linux/uaccess.h>
 
 #include <drm/amdgpu_drm.h>
 #include <drm/drm_drv.h>
@@ -508,6 +510,9 @@ int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data,
 	if (offset_in_page(args->addr | args->size))
 		return -EINVAL;
 
+	if (!access_ok((void __user *)(uintptr_t)args->addr, args->size))
+		return -EFAULT;
+
 	/* reject unknown flag values */
 	if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY |
 	    AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE |
@@ -821,7 +826,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 	struct drm_syncobj *timeline_syncobj = NULL;
 	struct dma_fence_chain *timeline_chain = NULL;
 	struct drm_exec exec;
-	uint64_t vm_size;
+	uint64_t vm_size, tmp;
 	int r = 0;
 
 	/* Validate virtual address range against reserved regions. */
@@ -845,7 +850,7 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 
 	vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
 	vm_size -= AMDGPU_VA_RESERVED_TOP;
-	if (args->va_address + args->map_size > vm_size) {
+	if (check_add_overflow(args->va_address, args->map_size, &tmp) || tmp > vm_size) {
 		dev_dbg(dev->dev,
 			"va_address 0x%llx is in top reserved area 0x%llx\n",
 			args->va_address + args->map_size, vm_size);
@@ -1217,13 +1222,14 @@ int amdgpu_gem_list_handles_ioctl(struct drm_device *dev, void *data,
 	return ret;
 }
 
-static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
-				  int width,
-				  int cpp,
-				  bool tiled)
+static unsigned int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
+					   unsigned int width,
+					   unsigned int cpp,
+					   bool tiled)
 {
-	int aligned = width;
-	int pitch_mask = 0;
+	unsigned int aligned = width;
+	unsigned int pitch_mask = 0;
+	unsigned int pitch;
 
 	switch (cpp) {
 	case 1:
@@ -1238,9 +1244,12 @@ static int amdgpu_gem_align_pitch(struct amdgpu_device *adev,
 		break;
 	}
 
-	aligned += pitch_mask;
+	if (check_add_overflow(aligned, pitch_mask, &aligned))
+		return 0;
 	aligned &= ~pitch_mask;
-	return aligned * cpp;
+	if (check_mul_overflow(aligned, cpp, &pitch))
+		return 0;
+	return pitch;
 }
 
 int amdgpu_mode_dumb_create(struct drm_file *file_priv,
@@ -1267,8 +1276,12 @@ int amdgpu_mode_dumb_create(struct drm_file *file_priv,
 
 	args->pitch = amdgpu_gem_align_pitch(adev, args->width,
 					     DIV_ROUND_UP(args->bpp, 8), 0);
+	if (!args->pitch)
+		return -EINVAL;
 	args->size = (u64)args->pitch * args->height;
 	args->size = ALIGN(args->size, PAGE_SIZE);
+	if (!args->size)
+		return -EINVAL;
 	domain = amdgpu_bo_get_preferred_domain(adev,
 				amdgpu_display_supported_domains(adev, flags));
 	r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index b8ca876694ff8..515cc4a2aeb4d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -2686,3 +2686,54 @@ void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev)
 #endif
 }
 
+int amdgpu_gfx_ring_preempt_ib(struct amdgpu_ring *ring)
+{
+	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
+	struct amdgpu_ring *kiq_ring = &kiq->ring;
+	unsigned long flags;
+	int i;
+
+	if (adev->enable_mes)
+		return -EINVAL;
+
+	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
+		return -EINVAL;
+
+	spin_lock_irqsave(&kiq->ring_lock, flags);
+
+	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
+		spin_unlock_irqrestore(&kiq->ring_lock, flags);
+		return -ENOMEM;
+	}
+
+	/* assert preemption condition */
+	amdgpu_ring_set_preempt_cond_exec(ring, false);
+
+	/* assert IB preemption, emit the trailing fence */
+	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
+					ring->trail_fence_gpu_addr,
+					++ring->trail_seq);
+	amdgpu_ring_commit(kiq_ring);
+
+	spin_unlock_irqrestore(&kiq->ring_lock, flags);
+
+	/* poll the trailing fence */
+	for (i = 0; i < adev->usec_timeout; i++) {
+		if (ring->trail_seq ==
+			le32_to_cpu(*(ring->trail_fence_cpu_addr)))
+			break;
+		udelay(1);
+	}
+
+	/* deassert preemption condition */
+    amdgpu_ring_set_preempt_cond_exec(ring, true);
+
+	if (i >= adev->usec_timeout) {
+		DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index a0cf0a3b41daf..77050f9884f20 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -664,6 +664,8 @@ void amdgpu_gfx_csb_preamble_end(u32 *buffer, u32 count);
 void amdgpu_debugfs_gfx_sched_mask_init(struct amdgpu_device *adev);
 void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev);
 
+int amdgpu_gfx_ring_preempt_ib(struct amdgpu_ring *ring);
+
 static inline const char *amdgpu_gfx_compute_mode_desc(int mode)
 {
 	switch (mode) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 3d9497d121ca2..276f9f49d6d15 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -280,6 +280,15 @@ void amdgpu_gmc_sysvm_location(struct amdgpu_device *adev, struct amdgpu_gmc *mc
 			mc->gart_size >> 20, mc->gart_start, mc->gart_end);
 }
 
+void amdgpu_gmc_set_gart_size(struct amdgpu_device *adev, u64 default_size)
+{
+	if (amdgpu_gart_size == -1)
+		adev->gmc.gart_size =
+			default_size + adev->pm.smu_prv_buffer_size;
+	else
+		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
+}
+
 /**
  * amdgpu_gmc_gart_location - try to find GART location
  *
@@ -711,12 +720,14 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev)
 void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 			      uint32_t vmhub, uint32_t flush_type)
 {
-	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+	struct amdgpu_ring *ring;
 	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
 	struct dma_fence *fence;
 	struct amdgpu_job *job;
 	int r;
 
+	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
+
 	if (!hub->sdma_invalidation_workaround || vmid ||
 	    !adev->mman.buffer_funcs_enabled || !adev->ib_pool_ready ||
 	    !ring->sched.ready) {
@@ -1005,6 +1016,9 @@ void amdgpu_gmc_noretry_set(struct amdgpu_device *adev)
 				gc_ver == IP_VERSION(9, 5, 0) ||
 				gc_ver >= IP_VERSION(10, 3, 0));
 
+	/* For GFX12.1 B0, set xnack (retry) on as default */
+	if (gc_ver == IP_VERSION(12, 1, 0) && (adev->rev_id & 0xf) == 0x1)
+		noretry_default = false;
 	if (!amdgpu_sriov_xnack_support(adev))
 		gmc->noretry = 1;
 	else
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
index 6ab4c1e297fce..676e3aaa1f27d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h
@@ -365,6 +365,8 @@ struct amdgpu_gmc {
 	bool flush_tlb_needs_extra_type_0;
 	bool flush_tlb_needs_extra_type_2;
 	bool flush_pasid_uses_kiq;
+
+	bool override_pte;
 };
 
 #define amdgpu_gmc_emit_flush_gpu_tlb(r, vmid, addr) (r)->adev->gmc.gmc_funcs->emit_flush_gpu_tlb((r), (vmid), (addr))
@@ -484,4 +486,6 @@ void amdgpu_gmc_init_sw_mem_ranges(struct amdgpu_device *adev,
 				   struct amdgpu_mem_partition_info *mem_ranges);
 int amdgpu_gmc_get_vram_info(struct amdgpu_device *adev,
 		int *vram_width, int *vram_type, int *vram_vendor);
+
+void amdgpu_gmc_set_gart_size(struct amdgpu_device *adev, u64 default_size);
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
index 620fddde4c4dd..a5d26b943f6d4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c
@@ -199,11 +199,18 @@ int amdgpu_gtt_mgr_alloc_entries(struct amdgpu_gtt_mgr *mgr,
 				 enum drm_mm_insert_mode mode)
 {
 	struct amdgpu_device *adev = container_of(mgr, typeof(*adev), mman.gtt_mgr);
+	u32 alignment = 0;
 	int r;
 
+	/* Align to TLB L2 cache entry size to work around "V bit HW bug" */
+	if (adev->asic_type == CHIP_TAHITI) {
+		alignment = 32 * 1024 / AMDGPU_GPU_PAGE_SIZE;
+		num_pages = ALIGN(num_pages, alignment);
+	}
+
 	spin_lock(&mgr->lock);
 	r = drm_mm_insert_node_in_range(&mgr->mm, mm_node, num_pages,
-					0, GART_ENTRY_WITHOUT_BO_COLOR, 0,
+					alignment, GART_ENTRY_WITHOUT_BO_COLOR, 0,
 					adev->gmc.gart_size >> PAGE_SHIFT,
 					mode);
 	spin_unlock(&mgr->lock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
index f58b6be7fccc0..444437c300886 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ih.h
@@ -91,6 +91,12 @@ struct amdgpu_ih_funcs {
 	uint64_t (*decode_iv_ts)(struct amdgpu_ih_ring *ih, u32 rptr,
 				 signed int offset);
 	void (*set_rptr)(struct amdgpu_device *adev, struct amdgpu_ih_ring *ih);
+	/* Decode IH cookie node_id into a human-readable die name string.
+	 * Returns buf, or NULL if this IH version does not support node_id decoding.
+	 */
+	const char *(*node_id_to_die_name)(struct amdgpu_device *adev,
+					   unsigned int node_id,
+					   char *buf, size_t size);
 };
 
 #define amdgpu_ih_get_wptr(adev, ih) (adev)->irq.ih_funcs->get_wptr((adev), (ih))
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
index 9c56be725ff3d..eb8537558f5fb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_imu.h
@@ -39,7 +39,6 @@ struct amdgpu_imu_funcs {
     int (*switch_compute_partition)(struct amdgpu_device *adev,
 				    int num_xccs_per_xcp,
 				    int compute_partition_mode);
-    void (*init_mcm_addr_lut)(struct amdgpu_device *adev);
 };
 
 struct imu_rlc_ram_golden {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 07771721af9d2..9ecc6387c1eb4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -388,7 +388,9 @@ static struct dma_fence *
 amdgpu_job_prepare_job(struct drm_sched_job *sched_job,
 		      struct drm_sched_entity *s_entity)
 {
-	struct amdgpu_ring *ring = to_amdgpu_ring(s_entity->rq->sched);
+	struct drm_gpu_scheduler *sched =
+		container_of(s_entity->rq, typeof(*sched), rq);
+	struct amdgpu_ring *ring = to_amdgpu_ring(sched);
 	struct amdgpu_job *job = to_amdgpu_job(sched_job);
 	struct dma_fence *fence;
 	int r;
@@ -481,25 +483,22 @@ drm_sched_entity_queue_pop(struct drm_sched_entity *entity)
 
 void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched)
 {
+	struct drm_sched_rq *rq = &sched->rq;
+	struct drm_sched_entity *s_entity;
 	struct drm_sched_job *s_job;
-	struct drm_sched_entity *s_entity = NULL;
-	int i;
 
 	/* Signal all jobs not yet scheduled */
-	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) {
-		struct drm_sched_rq *rq = sched->sched_rq[i];
-		spin_lock(&rq->lock);
-		list_for_each_entry(s_entity, &rq->entities, list) {
-			while ((s_job = drm_sched_entity_queue_pop(s_entity))) {
-				struct drm_sched_fence *s_fence = s_job->s_fence;
-
-				dma_fence_signal(&s_fence->scheduled);
-				dma_fence_set_error(&s_fence->finished, -EHWPOISON);
-				dma_fence_signal(&s_fence->finished);
-			}
+	spin_lock(&rq->lock);
+	list_for_each_entry(s_entity, &rq->entities, list) {
+		while ((s_job = drm_sched_entity_queue_pop(s_entity))) {
+			struct drm_sched_fence *s_fence = s_job->s_fence;
+
+			dma_fence_signal(&s_fence->scheduled);
+			dma_fence_set_error(&s_fence->finished, -EHWPOISON);
+			dma_fence_signal(&s_fence->finished);
 		}
-		spin_unlock(&rq->lock);
 	}
+	spin_unlock(&rq->lock);
 
 	/* Signal all jobs already scheduled to HW */
 	list_for_each_entry(s_job, &sched->pending_list, list) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
index 56a88e14a0448..e70a1117b8129 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.h
@@ -107,7 +107,10 @@ struct amdgpu_job {
 
 static inline struct amdgpu_ring *amdgpu_job_ring(struct amdgpu_job *job)
 {
-	return to_amdgpu_ring(job->base.entity->rq->sched);
+	struct drm_gpu_scheduler *sched =
+		container_of(job->base.entity->rq, typeof(*sched), rq);
+
+	return to_amdgpu_ring(sched);
 }
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, struct amdgpu_vm *vm,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
index 0d4c77c1b4b5f..c9467b26e42ce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c
@@ -103,7 +103,7 @@ static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe,
 
 int amdgpu_mes_init(struct amdgpu_device *adev)
 {
-	int i, r, num_pipes;
+	int i, r, num_pipes, num_queues = 0;
 	u32 total_vmid_mask, reserved_vmid_mask;
 	int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1;
 	u32 gfx_hqd_mask = amdgpu_mes_get_hqd_mask(adev->gfx.me.num_pipe_per_me,
@@ -159,7 +159,8 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 		adev->mes.compute_hqd_mask[i] = compute_hqd_mask;
 	}
 
-	num_pipes = adev->sdma.num_instances;
+	num_pipes = adev->sdma.num_inst_per_xcc ?
+		adev->sdma.num_inst_per_xcc : adev->sdma.num_instances;
 	if (num_pipes > AMDGPU_MES_MAX_SDMA_PIPES)
 		dev_warn(adev->dev, "more SDMA pipes than supported by MES! (%d vs %d)\n",
 			 num_pipes, AMDGPU_MES_MAX_SDMA_PIPES);
@@ -216,8 +217,27 @@ int amdgpu_mes_init(struct amdgpu_device *adev)
 	if (r)
 		goto error_doorbell;
 
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0)) {
+		/* When queue/pipe reset is done in MES instead of in the
+		 * driver, MES passes hung queues information to the driver in
+		 * hung_queue_hqd_info. Calculate required space to store this
+		 * information.
+		 */
+		for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++)
+			num_queues += hweight32(adev->mes.gfx_hqd_mask[i]);
+
+		for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++)
+			num_queues += hweight32(adev->mes.compute_hqd_mask[i]);
+
+		for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++)
+			num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc;
+
+		adev->mes.hung_queue_hqd_info_offset = num_queues;
+		adev->mes.hung_queue_db_array_size = num_queues * 2;
+	}
+
 	if (adev->mes.hung_queue_db_array_size) {
-		for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
+		for (i = 0; i < AMDGPU_MAX_MES_PIPES; i++) {
 			r = amdgpu_bo_create_kernel(adev,
 						    adev->mes.hung_queue_db_array_size * sizeof(u32),
 						    PAGE_SIZE,
@@ -264,10 +284,10 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
 			      &adev->mes.event_log_cpu_addr);
 
 	for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) {
-		amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i],
-				      &adev->mes.hung_queue_db_array_gpu_addr[i],
-				      &adev->mes.hung_queue_db_array_cpu_addr[i]);
-
+		if (adev->mes.hung_queue_db_array_gpu_obj[i])
+			 amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i],
+					 &adev->mes.hung_queue_db_array_gpu_addr[i],
+					 &adev->mes.hung_queue_db_array_cpu_addr[i]);
 		if (adev->mes.sch_ctx_ptr[i])
 			amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]);
 		if (adev->mes.query_status_fence_ptr[i])
@@ -281,7 +301,7 @@ void amdgpu_mes_fini(struct amdgpu_device *adev)
 	mutex_destroy(&adev->mes.mutex_hidden);
 }
 
-int amdgpu_mes_suspend(struct amdgpu_device *adev)
+int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id)
 {
 	struct mes_suspend_gang_input input;
 	int r;
@@ -291,6 +311,10 @@ int amdgpu_mes_suspend(struct amdgpu_device *adev)
 
 	memset(&input, 0x0, sizeof(struct mes_suspend_gang_input));
 	input.suspend_all_gangs = 1;
+	input.xcc_id = xcc_id;
+	if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) &&
+		((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x71))
+		input.suspend_all_sdma_gangs = 1;
 
 	/*
 	 * Avoid taking any other locks under MES lock to avoid circular
@@ -305,7 +329,7 @@ int amdgpu_mes_suspend(struct amdgpu_device *adev)
 	return r;
 }
 
-int amdgpu_mes_resume(struct amdgpu_device *adev)
+int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id)
 {
 	struct mes_resume_gang_input input;
 	int r;
@@ -315,6 +339,7 @@ int amdgpu_mes_resume(struct amdgpu_device *adev)
 
 	memset(&input, 0x0, sizeof(struct mes_resume_gang_input));
 	input.resume_all_gangs = 1;
+	input.xcc_id = xcc_id;
 
 	/*
 	 * Avoid taking any other locks under MES lock to avoid circular
@@ -428,7 +453,7 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 {
 	struct mes_detect_and_reset_queue_input input;
 	u32 *db_array = adev->mes.hung_queue_db_array_cpu_addr[xcc_id];
-	int r, i;
+	int hqd_info_offset = adev->mes.hung_queue_hqd_info_offset, r, i;
 
 	if (!hung_db_num || !hung_db_array)
 		return -EINVAL;
@@ -443,26 +468,34 @@ int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev,
 		adev->mes.hung_queue_db_array_size * sizeof(u32));
 	input.queue_type = queue_type;
 	input.detect_only = detect_only;
+	input.xcc_id = xcc_id;
 
 	r = adev->mes.funcs->detect_and_reset_hung_queues(&adev->mes,
 							  &input);
-	if (r) {
-		dev_err(adev->dev, "failed to detect and reset\n");
-	} else {
-		*hung_db_num = 0;
-		for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {
-			if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
-				hung_db_array[i] = db_array[i];
-				*hung_db_num += 1;
-			}
+
+	if (r && detect_only) {
+		dev_err(adev->dev, "Failed to detect hung queues\n");
+		return r;
+	}
+
+	*hung_db_num = 0;
+	/* MES passes hung queues' doorbell to driver */
+	for (i = 0; i < adev->mes.hung_queue_hqd_info_offset; i++) {
+		/* Finding hung queues where db_array[i] is a valid doorbell */
+		if (db_array[i] != AMDGPU_MES_INVALID_DB_OFFSET) {
+			hung_db_array[i] = db_array[i];
+			*hung_db_num += 1;
 		}
+	}
 
-		/*
-		 * TODO: return HQD info for MES scheduled user compute queue reset cases
-		 * stored in hung_db_array hqd info offset to full array size
-		 */
+	if (r && !(*hung_db_num)) {
+		dev_err(adev->dev, "Failed to detect and reset hung queues\n");
+		return r;
 	}
 
+	for (i = hqd_info_offset; i < hqd_info_offset + *hung_db_num; i++)
+		hung_db_array[i] = db_array[i];
+
 	return r;
 }
 
@@ -758,6 +791,12 @@ bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev)
 		amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 0, 0));
 }
 
+bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev)
+{
+	return (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) &&
+		(adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x73);
+}
+
 /* Fix me -- node_id is used to identify the correct MES instances in the future */
 static int amdgpu_mes_set_enforce_isolation(struct amdgpu_device *adev,
 					    uint32_t node_id, bool enable)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
index f80e3aca9c78e..93990d4990f2e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mes.h
@@ -170,6 +170,19 @@ struct amdgpu_mes {
 	uint64_t            shared_cmd_buf_gpu_addr[AMDGPU_MAX_MES_INST_PIPES];
 };
 
+struct amdgpu_mes_hung_queue_hqd_info {
+	union {
+		struct {
+			u32 queue_type: 3; // queue type
+			u32 pipe_index: 4; // pipe index
+			u32 queue_index: 8; // queue index
+			u32 reserved: 17;
+		};
+
+		u32 bit0_31;
+	};
+};
+
 struct amdgpu_mes_gang {
 	int 				gang_id;
 	int 				priority;
@@ -285,6 +298,7 @@ struct mes_unmap_legacy_queue_input {
 struct mes_suspend_gang_input {
 	uint32_t        xcc_id;
 	bool		suspend_all_gangs;
+	bool		suspend_all_sdma_gangs;
 	uint64_t	gang_context_addr;
 	uint64_t	suspend_fence_addr;
 	uint32_t	suspend_fence_value;
@@ -312,8 +326,9 @@ struct mes_reset_queue_input {
 };
 
 struct mes_detect_and_reset_queue_input {
-	uint32_t                           queue_type;
-	bool                               detect_only;
+	u32	queue_type;
+	bool		detect_only;
+	u32	xcc_id;
 };
 
 struct mes_inv_tlbs_pasid_input {
@@ -429,8 +444,8 @@ int amdgpu_mes_init_microcode(struct amdgpu_device *adev, int pipe);
 int amdgpu_mes_init(struct amdgpu_device *adev);
 void amdgpu_mes_fini(struct amdgpu_device *adev);
 
-int amdgpu_mes_suspend(struct amdgpu_device *adev);
-int amdgpu_mes_resume(struct amdgpu_device *adev);
+int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id);
+int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id);
 
 int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev,
 				struct amdgpu_ring *ring, uint32_t xcc_id);
@@ -534,6 +549,7 @@ static inline void amdgpu_mes_unlock(struct amdgpu_mes *mes)
 }
 
 bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev);
+bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev);
 
 int amdgpu_mes_update_enforce_isolation(struct amdgpu_device *adev);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
index 51ab1a3326157..8069fc41cc7fd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mode.h
@@ -38,7 +38,6 @@
 #include <drm/drm_probe_helper.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
-#include <linux/hrtimer.h>
 #include "amdgpu_irq.h"
 
 #include <drm/display/drm_dp_mst_helper.h>
@@ -505,9 +504,6 @@ struct amdgpu_crtc {
 	u32 line_time;
 	u32 lb_vblank_lead_lines;
 	struct drm_display_mode hw_mode;
-	/* for virtual dce */
-	struct hrtimer vblank_timer;
-	enum amdgpu_interrupt_state vsync_timer_enabled;
 
 	int otg_inst;
 	struct drm_pending_vblank_event *event;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
index a974265837f0d..e4c8e98725f01 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c
@@ -84,3 +84,13 @@ int amdgpu_nbio_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *
 	amdgpu_ras_block_late_fini(adev, ras_block);
 	return r;
 }
+
+
+void amdgpu_nbio_program_aspm(struct amdgpu_device *adev)
+{
+	if (!amdgpu_device_should_use_aspm(adev))
+		return;
+
+	if (adev->nbio.funcs->program_aspm)
+		adev->nbio.funcs->program_aspm(adev);
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
index b528de6a01f6d..a61f3a6e8ec77 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.h
@@ -121,4 +121,6 @@ u64 amdgpu_nbio_get_pcie_replay_count(struct amdgpu_device *adev);
 
 bool amdgpu_nbio_is_replay_cnt_supported(struct amdgpu_device *adev);
 
+void amdgpu_nbio_program_aspm(struct amdgpu_device *adev);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index b6aabac39b46d..f98bfba59a2ce 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -717,13 +717,17 @@ int amdgpu_bo_create(struct amdgpu_device *adev,
 	    bo->tbo.resource->mem_type == TTM_PL_VRAM) {
 		struct dma_fence *fence;
 
-		r = amdgpu_ttm_clear_buffer(bo, bo->tbo.base.resv, &fence);
+		r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
+					    bo, bo->tbo.base.resv, &fence,
+					    true, AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
 		if (unlikely(r))
 			goto fail_unreserve;
 
-		dma_resv_add_fence(bo->tbo.base.resv, fence,
-				   DMA_RESV_USAGE_KERNEL);
-		dma_fence_put(fence);
+		if (fence) {
+			dma_resv_add_fence(bo->tbo.base.resv, fence,
+					   DMA_RESV_USAGE_KERNEL);
+			dma_fence_put(fence);
+		}
 	}
 	if (!bp->resv)
 		amdgpu_bo_unreserve(bo);
@@ -1325,9 +1329,9 @@ void amdgpu_bo_release_notify(struct ttm_buffer_object *bo)
 	if (r)
 		goto out;
 
-	r = amdgpu_fill_buffer(amdgpu_ttm_next_clear_entity(adev),
-			       abo, 0, &bo->base._resv,
-			       &fence, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
+	r = amdgpu_ttm_clear_buffer(amdgpu_ttm_next_clear_entity(adev),
+				    abo, &bo->base._resv, &fence,
+				    false, AMDGPU_KERNEL_JOB_ID_CLEAR_ON_RELEASE);
 	if (WARN_ON(r))
 		goto out;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
index 912c9afaf9e11..4d68732d62235 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
@@ -96,7 +96,8 @@ struct amdgpu_bo_va {
 	 * if non-zero, cannot unmap from GPU because user queues may still access it
 	 */
 	unsigned int			queue_refcount;
-	atomic_t			userq_va_mapped;
+	/* Indicates if this buffer is mapped for any user queue. Once set, never reset. */
+	bool				userq_va_mapped;
 };
 
 struct amdgpu_bo {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index f0e4d020f4c7a..665a2632379de 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -52,6 +52,17 @@ static int psp_load_smu_fw(struct psp_context *psp);
 static int psp_rap_terminate(struct psp_context *psp);
 static int psp_securedisplay_terminate(struct psp_context *psp);
 
+static const char * const amdgpu_ptl_fmt_str[] = {
+	[AMDGPU_PTL_FMT_I8]      = "I8",
+	[AMDGPU_PTL_FMT_F16]     = "F16",
+	[AMDGPU_PTL_FMT_BF16]    = "BF16",
+	[AMDGPU_PTL_FMT_F32]     = "F32",
+	[AMDGPU_PTL_FMT_F64]     = "F64",
+	[AMDGPU_PTL_FMT_F8]      = "F8",
+	[AMDGPU_PTL_FMT_VECTOR]  = "VECTOR",
+	[AMDGPU_PTL_FMT_INVALID] = "INVALID",
+};
+
 static int psp_ring_init(struct psp_context *psp,
 			 enum psp_ring_type ring_type)
 {
@@ -518,7 +529,7 @@ static int psp_sw_init(struct amdgpu_ip_block *ip_block)
 	}
 
 	ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
-				      (amdgpu_sriov_vf(adev) || adev->debug_use_vram_fw_buf) ?
+				      (amdgpu_sriov_vf(adev) || adev->debug_use_vram_fw_buf || adev->gmc.xgmi.connected_to_cpu) ?
 				      AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
 				      &psp->fw_pri_bo,
 				      &psp->fw_pri_mc_addr,
@@ -682,6 +693,8 @@ static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 		return "SPATIAL_PARTITION";
 	case GFX_CMD_ID_FB_NPS_MODE:
 		return "NPS_MODE_CHANGE";
+	case GFX_CMD_ID_PERF_HW:
+		return "PERF MONITORING HW";
 	default:
 		return "UNKNOWN CMD";
 	}
@@ -844,7 +857,11 @@ static int psp_load_toc(struct psp_context *psp,
 	struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 
 	/* Copy toc to psp firmware private buffer */
-	psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes);
+	ret = psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes);
+	if (ret) {
+		release_psp_cmd_buf(psp);
+		return ret;
+	}
 
 	psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes);
 
@@ -1160,8 +1177,11 @@ static int psp_rl_load(struct amdgpu_device *adev)
 
 	cmd = acquire_psp_cmd_buf(psp);
 
-	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
-	memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes);
+	ret = psp_copy_fw(psp, psp->rl.start_addr, psp->rl.size_bytes);
+	if (ret) {
+		release_psp_cmd_buf(psp);
+		return ret;
+	}
 
 	cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
 	cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr);
@@ -1201,6 +1221,369 @@ int psp_memory_partition(struct psp_context *psp, int mode)
 	return ret;
 }
 
+static int psp_ptl_fmt_verify(struct psp_context *psp, enum amdgpu_ptl_fmt fmt,
+						 uint32_t *ptl_fmt)
+{
+	struct amdgpu_device *adev = psp->adev;
+
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 4))
+		return -EINVAL;
+
+	switch (fmt) {
+	case AMDGPU_PTL_FMT_I8:
+		*ptl_fmt = GFX_FTYPE_I8;
+		break;
+	case AMDGPU_PTL_FMT_F16:
+		*ptl_fmt = GFX_FTYPE_F16;
+		break;
+	case AMDGPU_PTL_FMT_BF16:
+		*ptl_fmt = GFX_FTYPE_BF16;
+		break;
+	case AMDGPU_PTL_FMT_F32:
+		*ptl_fmt = GFX_FTYPE_F32;
+		break;
+	case AMDGPU_PTL_FMT_F64:
+		*ptl_fmt = GFX_FTYPE_F64;
+		break;
+	case AMDGPU_PTL_FMT_F8:
+		*ptl_fmt = GFX_FTYPE_F8;
+		break;
+	case AMDGPU_PTL_FMT_VECTOR:
+		*ptl_fmt = GFX_FTYPE_VECTOR;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int psp_ptl_invoke(struct psp_context *psp, u32 req_code,
+		uint32_t *ptl_state, uint32_t *fmt1, uint32_t *fmt2)
+{
+	struct psp_gfx_cmd_resp *cmd;
+	struct amdgpu_ptl *ptl = &psp->ptl;
+	int ret;
+
+	cmd = acquire_psp_cmd_buf(psp);
+
+	cmd->cmd_id                     = GFX_CMD_ID_PERF_HW;
+	cmd->cmd.cmd_req_perf_hw.req    = req_code;
+	cmd->cmd.cmd_req_perf_hw.ptl_state    = *ptl_state;
+	cmd->cmd.cmd_req_perf_hw.pref_format1 = *fmt1;
+	cmd->cmd.cmd_req_perf_hw.pref_format2 = *fmt2;
+
+	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
+	if (ret)
+		goto out;
+
+	/*
+	 * Check response status explicitly to avoid
+	 * updating cached PTL state with invalid data.
+	 */
+	if (cmd->resp.status) {
+		dev_err(psp->adev->dev,
+				"PTL command 0x%x failed, PSP response status: 0x%X fw resp=0x%X\n",
+				req_code, cmd->resp.status,
+				cmd->resp.uresp.perf_hw_info.resp);
+		ret = -EIO;
+		goto out;
+	}
+
+	/* Parse response */
+	switch (req_code) {
+	case PSP_PTL_PERF_MON_QUERY:
+		*ptl_state = cmd->resp.uresp.perf_hw_info.ptl_state;
+		*fmt1      = cmd->resp.uresp.perf_hw_info.pref_format1;
+		*fmt2      = cmd->resp.uresp.perf_hw_info.pref_format2;
+		dev_dbg(psp->adev->dev, "PTL query: state=%d, fmt1=%d, fmt2=%d\n",
+				*ptl_state, *fmt1, *fmt2);
+		break;
+	case PSP_PTL_PERF_MON_SET:
+		/* Update cached state only on success */
+		ptl->enabled = *ptl_state;
+		ptl->fmt1    = *fmt1;
+		ptl->fmt2    = *fmt2;
+		dev_dbg(psp->adev->dev, "PTL set: state=%d, fmt1=%d, fmt2=%d\n",
+				*ptl_state, *fmt1, *fmt2);
+		break;
+	}
+
+out:
+	release_psp_cmd_buf(psp);
+	return ret;
+}
+
+int amdgpu_ptl_perf_monitor_ctrl(struct amdgpu_device *adev, u32 req_code,
+				uint32_t *ptl_state,
+				enum amdgpu_ptl_fmt *fmt1,
+				enum amdgpu_ptl_fmt *fmt2)
+{
+	uint32_t ptl_fmt1, ptl_fmt2;
+	struct psp_context *psp;
+	struct amdgpu_ptl *ptl;
+	int ret;
+
+	if (!adev || !ptl_state || !fmt1 || !fmt2)
+		return -EINVAL;
+
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	psp = &adev->psp;
+	ptl = &psp->ptl;
+
+	if (ptl->permanently_disabled && *ptl_state == 1)
+		return 0;
+
+	if (amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 4) ||
+			psp->sos.fw_version < 0x0036081a)
+		return -EOPNOTSUPP;
+
+	/* Verify formats */
+	if (psp_ptl_fmt_verify(psp, *fmt1, &ptl_fmt1) ||
+			psp_ptl_fmt_verify(psp, *fmt2, &ptl_fmt2))
+		return -EINVAL;
+
+	/*
+	 * Add check to skip if state and formats are identical to current ones
+	 */
+	if (req_code == PSP_PTL_PERF_MON_SET &&
+			ptl->enabled == *ptl_state &&
+			ptl->fmt1 == ptl_fmt1 &&
+			ptl->fmt2 == ptl_fmt2)
+		return 0;
+
+	/* If enabling PTL, check disable bitmap */
+	if (req_code == PSP_PTL_PERF_MON_SET && *ptl_state == 1) {
+		if (!bitmap_empty(ptl->disable_bitmap,
+					AMDGPU_PTL_DISABLE_MAX)) {
+			dev_dbg(adev->dev,
+					"PTL enable blocked: SYSFS=%d, PROFILER=%d (ref=%d)\n",
+					test_bit(AMDGPU_PTL_DISABLE_SYSFS,
+						ptl->disable_bitmap),
+					test_bit(AMDGPU_PTL_DISABLE_PROFILER,
+						ptl->disable_bitmap),
+					atomic_read(&ptl->disable_ref));
+			return 0;
+		}
+	}
+
+	if (req_code == PSP_PTL_PERF_MON_SET) {
+		amdgpu_amdkfd_stop_sched_all(adev);
+		/* Wait for GFX engine idle before PTL state transition */
+		ret = amdgpu_device_ip_wait_for_idle(adev,
+				AMD_IP_BLOCK_TYPE_GFX);
+		if (ret) {
+			amdgpu_amdkfd_start_sched_all(adev);
+			dev_err(adev->dev, "GFX not idle before PTL operation (%d)\n", ret);
+			return ret;
+		}
+		ret = psp_ptl_invoke(psp, req_code, ptl_state, &ptl_fmt1, &ptl_fmt2);
+		amdgpu_amdkfd_start_sched_all(adev);
+	} else {
+		ret = psp_ptl_invoke(psp, req_code, ptl_state, &ptl_fmt1, &ptl_fmt2);
+	}
+
+	return ret;
+}
+
+static enum amdgpu_ptl_fmt str_to_ptl_fmt(const char *str)
+{
+	int i;
+
+	for (i = 0; i < AMDGPU_PTL_FMT_INVALID; ++i) {
+		if (!strcmp(str, amdgpu_ptl_fmt_str[i]))
+			return (enum amdgpu_ptl_fmt)i;
+	}
+
+	return AMDGPU_PTL_FMT_INVALID;
+}
+
+static ssize_t ptl_supported_formats_show(struct device *dev,
+		struct device_attribute *attr, char *buf)
+{
+	ssize_t len = 0;
+
+	for (int i = 0; i < AMDGPU_PTL_FMT_INVALID; ++i) {
+		const char *fmt = amdgpu_ptl_fmt_str[i];
+
+		len += sysfs_emit_at(buf, len, "%s%s",
+				fmt ? fmt : "UNKNOWN",
+				(i < AMDGPU_PTL_FMT_INVALID - 1) ? "," : "\n");
+	}
+
+	return len;
+}
+
+static ssize_t ptl_enable_store(struct device *dev,
+		struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+	uint32_t ptl_state, fmt1, fmt2;
+	int ret;
+	bool enable;
+	bool bit_changed = false;
+
+	mutex_lock(&ptl->mutex);
+	if (sysfs_streq(buf, "enabled") || sysfs_streq(buf, "1")) {
+		enable = true;
+	} else if (sysfs_streq(buf, "disabled") || sysfs_streq(buf, "0")) {
+		enable = false;
+	} else {
+		mutex_unlock(&ptl->mutex);
+		return -EINVAL;
+	}
+
+	/* Block enable when permanently disabled */
+	if (ptl->permanently_disabled) {
+		mutex_unlock(&ptl->mutex);
+		return -EPERM;
+	}
+
+	fmt1 = ptl->fmt1;
+	fmt2 = ptl->fmt2;
+	ptl_state = enable ? 1 : 0;
+
+	if (enable)
+		bit_changed = test_and_clear_bit(AMDGPU_PTL_DISABLE_SYSFS,
+				ptl->disable_bitmap);
+
+	ret = amdgpu_ptl_perf_monitor_ctrl(adev, PSP_PTL_PERF_MON_SET, &ptl_state, &fmt1, &fmt2);
+	if (ret) {
+		dev_err(adev->dev, "Failed to set PTL err = %d\n", ret);
+		if (enable && bit_changed)
+			set_bit(AMDGPU_PTL_DISABLE_SYSFS, ptl->disable_bitmap);
+		mutex_unlock(&ptl->mutex);
+		return ret;
+	}
+
+	if (!enable)
+		set_bit(AMDGPU_PTL_DISABLE_SYSFS, ptl->disable_bitmap);
+
+	mutex_unlock(&ptl->mutex);
+
+	return count;
+}
+
+static ssize_t ptl_enable_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+
+	if (ptl->permanently_disabled)
+		return sysfs_emit(buf, "permanently disabled\n");
+
+	return sysfs_emit(buf, "%s\n", ptl->enabled ? "enabled" : "disabled");
+}
+
+static ssize_t ptl_format_store(struct device *dev,
+		struct device_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	char fmt1_str[8], fmt2_str[8];
+	enum amdgpu_ptl_fmt fmt1_enum, fmt2_enum;
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+	uint32_t ptl_state, fmt1, fmt2;
+	int ret;
+
+	/* Only allow format update when PTL is enabled */
+	if (!ptl->enabled)
+		return -EPERM;
+
+	mutex_lock(&ptl->mutex);
+	/* Parse input, expecting "FMT1,FMT2" */
+	if (sscanf(buf, "%7[^,],%7s", fmt1_str, fmt2_str) != 2) {
+		mutex_unlock(&ptl->mutex);
+		return -EINVAL;
+	}
+
+	fmt1_enum = str_to_ptl_fmt(fmt1_str);
+	fmt2_enum = str_to_ptl_fmt(fmt2_str);
+
+	if (fmt1_enum >= AMDGPU_PTL_FMT_INVALID ||
+			fmt2_enum >= AMDGPU_PTL_FMT_INVALID ||
+			fmt1_enum == fmt2_enum) {
+		mutex_unlock(&ptl->mutex);
+		return -EINVAL;
+	}
+
+	ptl_state = ptl->enabled;
+	fmt1 = fmt1_enum;
+	fmt2 = fmt2_enum;
+	ret = amdgpu_ptl_perf_monitor_ctrl(adev, PSP_PTL_PERF_MON_SET, &ptl_state, &fmt1, &fmt2);
+	if (ret) {
+		dev_err(adev->dev, "Failed to update PTL err = %d\n", ret);
+		mutex_unlock(&ptl->mutex);
+		return ret;
+	}
+	mutex_unlock(&ptl->mutex);
+
+	return count;
+}
+
+static ssize_t ptl_format_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+	struct psp_context *psp = &adev->psp;
+
+	return sysfs_emit(buf, "%s,%s\n",
+			amdgpu_ptl_fmt_str[psp->ptl.fmt1],
+			amdgpu_ptl_fmt_str[psp->ptl.fmt2]);
+}
+
+static umode_t amdgpu_ptl_is_visible(struct kobject *kobj, struct attribute *attr, int idx)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct drm_device *ddev = dev_get_drvdata(dev);
+	struct amdgpu_device *adev = drm_to_adev(ddev);
+
+	/* Only show PTL sysfs files if PTL hardware is supported */
+	if (!adev->psp.ptl.hw_supported)
+		return 0;
+
+	return attr->mode;
+}
+
+int amdgpu_ptl_sysfs_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+	int ret;
+
+	if (!ptl->hw_supported)
+		return 0;
+
+	if (ptl->ptl_sysfs_created)
+		return 0;
+
+	ret = sysfs_create_group(&adev->dev->kobj, &amdgpu_ptl_attr_group);
+	if (!ret)
+		ptl->ptl_sysfs_created = true;
+
+	return ret;
+}
+
+void amdgpu_ptl_sysfs_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+
+	if (!ptl->hw_supported)
+		return;
+
+	if (!ptl->ptl_sysfs_created)
+		return;
+
+	sysfs_remove_group(&adev->dev->kobj, &amdgpu_ptl_attr_group);
+	ptl->ptl_sysfs_created = false;
+}
+
 int psp_spatial_partition(struct psp_context *psp, int mode)
 {
 	struct psp_gfx_cmd_resp *cmd;
@@ -1384,8 +1767,12 @@ int psp_ta_load(struct psp_context *psp, struct ta_context *context)
 
 	cmd = acquire_psp_cmd_buf(psp);
 
-	psp_copy_fw(psp, context->bin_desc.start_addr,
-		    context->bin_desc.size_bytes);
+	ret = psp_copy_fw(psp, context->bin_desc.start_addr,
+			  context->bin_desc.size_bytes);
+	if (ret) {
+		release_psp_cmd_buf(psp);
+		return ret;
+	}
 
 	if (amdgpu_virt_xgmi_migrate_enabled(psp->adev) &&
 		context->mem_context.shared_bo)
@@ -3527,7 +3914,12 @@ int psp_init_toc_microcode(struct psp_context *psp, const char *chip_name)
 	const struct psp_firmware_header_v1_0 *toc_hdr;
 	int err = 0;
 
-	err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED,
+	if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(15, 0, 8) &&
+	    adev->rev_id == 0)
+		err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED,
+				   "amdgpu/%s_toc_1.bin", chip_name);
+	else
+		err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, AMDGPU_UCODE_REQUIRED,
 				   "amdgpu/%s_toc.bin", chip_name);
 	if (err)
 		goto out;
@@ -4181,17 +4573,24 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
 	return count;
 }
 
-void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
+int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
 {
 	int idx;
 
 	if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
-		return;
+		return -ENODEV;
+
+	if (!bin_size || bin_size > PSP_1_MEG) {
+		dev_err(psp->adev->dev, "PSP firmware is invalid\n");
+		drm_dev_exit(idx);
+		return -EINVAL;
+	}
 
 	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 	memcpy(psp->fw_pri_buf, start_addr, bin_size);
 
 	drm_dev_exit(idx);
+	return 0;
 }
 
 /**
@@ -4202,6 +4601,31 @@ void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size
 static DEVICE_ATTR(usbc_pd_fw, 0644,
 		   psp_usbc_pd_fw_sysfs_read,
 		   psp_usbc_pd_fw_sysfs_write);
+/**
+ * DOC: PTL sysfs attributes
+ * These sysfs files under /sys/class/drm/cardX/device/ptl allow users to enable or disable
+ * the Peak Tops Limiter (PTL), configure preferred PTL data formats, and query supported
+ * formats for each GPU.
+ */
+static DEVICE_ATTR(ptl_enable, 0644,
+			ptl_enable_show, ptl_enable_store);
+static DEVICE_ATTR(ptl_format, 0644,
+			ptl_format_show, ptl_format_store);
+static DEVICE_ATTR(ptl_supported_formats, 0444,
+			ptl_supported_formats_show, NULL);
+
+static struct attribute *ptl_attrs[] = {
+	&dev_attr_ptl_enable.attr,
+	&dev_attr_ptl_format.attr,
+	&dev_attr_ptl_supported_formats.attr,
+	NULL,
+};
+
+const struct attribute_group amdgpu_ptl_attr_group = {
+	.name = "ptl",
+	.attrs = ptl_attrs,
+	.is_visible = amdgpu_ptl_is_visible,
+};
 
 int is_psp_fw_valid(struct psp_bin_desc bin)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
index 7e94ec11c57e4..2d838b1b2b116 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.h
@@ -31,6 +31,8 @@
 #include "ta_ras_if.h"
 #include "ta_rap_if.h"
 #include "ta_secureDisplay_if.h"
+#include <linux/bitops.h>
+#include "amdgpu_ptl.h"
 
 #define PSP_FENCE_BUFFER_SIZE	0x1000
 #define PSP_CMD_BUFFER_SIZE	0x1000
@@ -358,6 +360,29 @@ struct spirom_bo {
 };
 #endif
 
+enum psp_ptl_cmd {
+	PSP_PTL_PERF_MON_QUERY = 0xA0000000,
+	PSP_PTL_PERF_MON_SET = 0xA0000001,
+};
+
+enum psp_ptl_format_type {
+	GFX_FTYPE_I8          = 0x00000000,
+	GFX_FTYPE_F16         = 0x00000001,
+	GFX_FTYPE_BF16        = 0x00000002,
+	GFX_FTYPE_F32         = 0x00000003,
+	GFX_FTYPE_F64         = 0x00000004,
+	GFX_FTYPE_F8          = 0x00000005,
+	GFX_FTYPE_VECTOR      = 0x00000006,
+	GFX_FTYPE_INVALID     = 0xFFFFFFFF,
+};
+
+struct psp_ptl_perf_req {
+	enum psp_ptl_cmd req;
+	uint32_t ptl_state;
+	uint32_t pref_format1;
+	uint32_t pref_format2;
+};
+
 struct psp_context {
 	struct amdgpu_device		*adev;
 	struct psp_ring			km_ring;
@@ -448,6 +473,7 @@ struct psp_context {
 #if defined(CONFIG_DEBUG_FS)
 	struct spirom_bo *spirom_dump_trip;
 #endif
+	struct amdgpu_ptl		ptl;
 };
 
 struct amdgpu_psp_funcs {
@@ -612,7 +638,7 @@ int psp_get_fw_attestation_records_addr(struct psp_context *psp,
 int psp_update_fw_reservation(struct psp_context *psp);
 int psp_load_fw_list(struct psp_context *psp,
 		     struct amdgpu_firmware_info **ucode_list, int ucode_count);
-void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size);
+int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size);
 
 int psp_spatial_partition(struct psp_context *psp, int mode);
 int psp_memory_partition(struct psp_context *psp, int mode);
@@ -631,5 +657,4 @@ void amdgpu_psp_debugfs_init(struct amdgpu_device *adev);
 int amdgpu_psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 			   enum psp_gfx_fw_type *type);
 
-
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
index 6c644cfe6695d..3e1f8b15f75de 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c
@@ -3093,6 +3093,25 @@ static int amdgpu_ras_mca2pa(struct amdgpu_device *adev,
 		return  -EINVAL;
 }
 
+static bool __check_record_in_range(struct amdgpu_device *adev,
+			struct eeprom_table_record *bps, int count)
+{
+	int i;
+
+	for (i = 0; i < count; i++) {
+		if (bps[i].retired_page >=
+			(adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT)) {
+			dev_warn(adev->dev,
+				"Recorded address out of range: 0x%llx, 0x%llx, 0x%x, 0x%x\n",
+				bps[i].address, bps[i].retired_page,
+				bps[i].mem_channel, bps[i].mcumc_id);
+			return false;
+		}
+	}
+
+	return true;
+}
+
 static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev,
 					struct eeprom_table_record *bps, int count)
 {
@@ -3100,6 +3119,9 @@ static int __amdgpu_ras_restore_bad_pages(struct amdgpu_device *adev,
 	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
 	struct ras_err_handler_data *data = con->eh_data;
 
+	if (!__check_record_in_range(adev, bps, count))
+		return 0;
+
 	for (j = 0; j < count; j++) {
 		if (!data->space_left &&
 		    amdgpu_ras_realloc_eh_data_space(adev, data, 256)) {
@@ -5641,6 +5663,11 @@ int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn)
 	uint64_t start = pfn << AMDGPU_GPU_PAGE_SHIFT;
 	int ret = 0;
 
+	if (pfn >= (adev->gmc.real_vram_size >> AMDGPU_GPU_PAGE_SHIFT)) {
+		dev_warn(adev->dev, "Ignoring out-of-range bad page 0x%llx", start);
+		return 0;
+	}
+
 	if (amdgpu_ras_check_critical_address(adev, start))
 		return 0;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
index 0c57fe2598945..c61389a079822 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ras_eeprom.c
@@ -1051,6 +1051,7 @@ int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control,
 	uint64_t ts, end_idx;
 	int i, ret;
 	u64 mca, ipid;
+	u32 cu, mem_channel, mcumc_id;
 
 	if (!amdgpu_ras_smu_eeprom_supported(adev))
 		return 0;
@@ -1079,9 +1080,10 @@ int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control,
 		record[i - rec_idx].err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE;
 
 		adev->umc.ras->mca_ipid_parse(adev, ipid,
-			(uint32_t *)&(record[i - rec_idx].cu),
-			(uint32_t *)&(record[i - rec_idx].mem_channel),
-			(uint32_t *)&(record[i - rec_idx].mcumc_id), NULL);
+			&cu, &mem_channel, &mcumc_id, NULL);
+		record[i - rec_idx].cu = (u8)cu;
+		record[i - rec_idx].mem_channel = (u8)mem_channel;
+		record[i - rec_idx].mcumc_id = (u8)mcumc_id;
 	}
 
 	return 0;
@@ -1632,6 +1634,14 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control)
 	}
 
 	control->ras_fri = RAS_OFFSET_TO_INDEX(control, hdr->first_rec_offset);
+	if (hdr->first_rec_offset < control->ras_record_offset ||
+	    control->ras_fri >= control->ras_max_record_count) {
+		dev_err(adev->dev,
+			"RAS header invalid, ras_fri: %u, first_rec_offset:0x%x",
+			control->ras_fri, hdr->first_rec_offset);
+		return -EINVAL;
+	}
+
 	control->ras_num_mca_recs = 0;
 	control->ras_num_pa_recs = 0;
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c
index 540040c76058b..daefbeeee4d2c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.c
@@ -956,3 +956,21 @@ uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst,
 	}
 	return ret;
 }
+
+
+uint32_t amdgpu_read_indexed_register(struct amdgpu_device *adev,
+			       u32 se_num, u32 sh_num, u32 reg_offset)
+{
+	uint32_t val;
+
+	mutex_lock(&adev->grbm_idx_mutex);
+	if (se_num != 0xffffffff || sh_num != 0xffffffff)
+		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
+
+	val = RREG32(reg_offset);
+
+	if (se_num != 0xffffffff || sh_num != 0xffffffff)
+		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
+	mutex_unlock(&adev->grbm_idx_mutex);
+	return val;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
index 4d88e5cd19fc9..a1011af6b52b4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reg_access.h
@@ -160,4 +160,7 @@ uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev, uint32_t inst,
 				    uint32_t reg_addr, char reg_name[],
 				    uint32_t expected_value, uint32_t mask);
 
+uint32_t amdgpu_read_indexed_register(struct amdgpu_device *adev,
+					u32 se_num, u32 sh_num, u32 reg_offset);
+
 #endif /* __AMDGPU_REG_ACCESS_H__ */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
index 1b982b803e6f3..428c3cbc4a401 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.c
@@ -90,6 +90,7 @@ static int amdgpu_reset_xgmi_reset_on_init_restore_hwctxt(
 			kgd2kfd_init_zone_device(tmp_adev);
 			amdgpu_amdkfd_device_init(tmp_adev);
 			amdgpu_amdkfd_drm_client_create(tmp_adev);
+			amdgpu_ptl_sysfs_init(tmp_adev);
 		}
 	}
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
index 07b4d37f1db69..c9f23a8e8db80 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_reset.h
@@ -46,6 +46,47 @@ enum AMDGPU_RESET_SRCS {
 	AMDGPU_RESET_SRC_USERQ,
 };
 
+/**
+ * enum amd_reset_method - Methods for resetting AMD GPU devices
+ *
+ * @AMD_RESET_METHOD_NONE: The device will not be reset.
+ * @AMD_RESET_METHOD_LEGACY: Method reserved for SI, CIK and VI ASICs.
+ * @AMD_RESET_METHOD_MODE0: Reset the entire ASIC. Not currently available for
+ *                          the any device.
+ * @AMD_RESET_METHOD_MODE1: Resets all IP blocks on the ASIC (SDMA, GFX, VCN,
+ *                   etc.) individually. Suitable only for some discrete GPU,
+ *                   not available for all ASICs.
+ * @AMD_RESET_METHOD_MODE2: Resets a lesser level of IPs compared to MODE1.
+ *                   Which IPs are reset depends on the ASIC. Notably doesn't
+ *                   reset IPs shared with the CPU on APUs or the memory
+ *                   controllers (so VRAM is not lost). Not available on all
+ *                   ASICs.
+ * @AMD_RESET_METHOD_LINK: Triggers SW-UP link reset on other GPUs
+ * @AMD_RESET_METHOD_BACO: BACO (Bus Alive, Chip Off) method powers off and on
+ *                   the card but without powering off the PCI bus. Suitable
+ *                   only for discrete GPUs.
+ * @AMD_RESET_METHOD_PCI: Does a full bus reset using core Linux subsystem
+ *                   PCI reset and does a secondary bus reset or FLR,
+ *                   depending on what the underlying hardware supports.
+ * @AMD_RESET_METHOD_ON_INIT: Does a device reset during the driver init
+ *                   sequence.
+ *
+ * Methods available for AMD GPU driver for resetting the device. Not all
+ * methods are suitable for every device. User can override the method using
+ * module parameter `reset_method`.
+ */
+enum amd_reset_method {
+	AMD_RESET_METHOD_NONE = -1,
+	AMD_RESET_METHOD_LEGACY = 0,
+	AMD_RESET_METHOD_MODE0,
+	AMD_RESET_METHOD_MODE1,
+	AMD_RESET_METHOD_MODE2,
+	AMD_RESET_METHOD_LINK,
+	AMD_RESET_METHOD_BACO,
+	AMD_RESET_METHOD_PCI,
+	AMD_RESET_METHOD_ON_INIT,
+};
+
 struct amdgpu_reset_context {
 	enum amd_reset_method method;
 	struct amdgpu_device *reset_req_dev;
@@ -56,6 +97,20 @@ struct amdgpu_reset_context {
 	enum AMDGPU_RESET_SRCS src;
 };
 
+struct amdgpu_reset_control {
+	void *handle;
+	struct work_struct reset_work;
+	struct mutex reset_lock;
+	struct amdgpu_reset_handler *(
+		*reset_handlers)[AMDGPU_RESET_MAX_HANDLERS];
+	atomic_t in_reset;
+	enum amd_reset_method active_reset;
+	struct amdgpu_reset_handler *(*get_reset_handler)(
+		struct amdgpu_reset_control *reset_ctl,
+		struct amdgpu_reset_context *context);
+	void (*async_reset)(struct work_struct *work);
+};
+
 struct amdgpu_reset_handler {
 	enum amd_reset_method reset_method;
 	int (*prepare_env)(struct amdgpu_reset_control *reset_ctl,
@@ -72,20 +127,6 @@ struct amdgpu_reset_handler {
 	int (*do_reset)(struct amdgpu_device *adev);
 };
 
-struct amdgpu_reset_control {
-	void *handle;
-	struct work_struct reset_work;
-	struct mutex reset_lock;
-	struct amdgpu_reset_handler *(
-		*reset_handlers)[AMDGPU_RESET_MAX_HANDLERS];
-	atomic_t in_reset;
-	enum amd_reset_method active_reset;
-	struct amdgpu_reset_handler *(*get_reset_handler)(
-		struct amdgpu_reset_control *reset_ctl,
-		struct amdgpu_reset_context *context);
-	void (*async_reset)(struct work_struct *work);
-};
-
 
 enum amdgpu_reset_domain_type {
 	SINGLE_DEVICE,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
index 715c9e43e13aa..8f28b3bd70106 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h
@@ -159,7 +159,8 @@ struct amdgpu_fence {
 extern const struct drm_sched_backend_ops amdgpu_sched_ops;
 
 void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error);
-void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring);
+void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring,
+					  struct dma_fence *timedout_fence);
 void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring,
 					     struct amdgpu_fence *guilty_fence);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
index 321310ba2c08e..fcd81242059e6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sdma.c
@@ -600,10 +600,10 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id,
 		 * to be submitted to the queues after the reset is complete.
 		 */
 		if (!ret) {
-			amdgpu_fence_driver_force_completion(gfx_ring);
+			amdgpu_fence_driver_force_completion(gfx_ring, NULL);
 			drm_sched_wqueue_start(&gfx_ring->sched);
 			if (adev->sdma.has_page_queue) {
-				amdgpu_fence_driver_force_completion(page_ring);
+				amdgpu_fence_driver_force_completion(page_ring, NULL);
 				drm_sched_wqueue_start(&page_ring->sched);
 			}
 		}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
index a0b479d5fff19..f4be192235889 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_seq64.c
@@ -175,11 +175,14 @@ int amdgpu_seq64_alloc(struct amdgpu_device *adev, u64 *va,
 {
 	unsigned long bit_pos;
 
-	bit_pos = find_first_zero_bit(adev->seq64.used, adev->seq64.num_sem);
-	if (bit_pos >= adev->seq64.num_sem)
-		return -ENOSPC;
+	for (;;) {
+		bit_pos = find_first_zero_bit(adev->seq64.used, adev->seq64.num_sem);
+		if (bit_pos >= adev->seq64.num_sem)
+			return -ENOSPC;
 
-	__set_bit(bit_pos, adev->seq64.used);
+		if (!test_and_set_bit(bit_pos, adev->seq64.used))
+			break;
+	}
 
 	*va = bit_pos * sizeof(u64) + amdgpu_seq64_get_va_base(adev);
 
@@ -205,7 +208,7 @@ void amdgpu_seq64_free(struct amdgpu_device *adev, u64 va)
 
 	bit_pos = (va - amdgpu_seq64_get_va_base(adev)) / sizeof(u64);
 	if (bit_pos < adev->seq64.num_sem)
-		__clear_bit(bit_pos, adev->seq64.used);
+		clear_bit(bit_pos, adev->seq64.used);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index d13e64a69e255..85724ec6aaf84 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -145,6 +145,7 @@ TRACE_EVENT(amdgpu_cs,
 		     struct amdgpu_ib *ib),
 	    TP_ARGS(p, job, ib),
 	    TP_STRUCT__entry(
+			     __field(struct drm_gpu_scheduler *, sched)
 			     __field(struct amdgpu_bo_list *, bo_list)
 			     __field(u32, ring)
 			     __field(u32, dw)
@@ -152,11 +153,14 @@ TRACE_EVENT(amdgpu_cs,
 			     ),
 
 	    TP_fast_assign(
+			   __entry->sched = container_of(job->base.entity->rq,
+							 typeof(*__entry->sched),
+							 rq);
 			   __entry->bo_list = p->bo_list;
-			   __entry->ring = to_amdgpu_ring(job->base.entity->rq->sched)->idx;
+			   __entry->ring = to_amdgpu_ring(__entry->sched)->idx;
 			   __entry->dw = ib->length_dw;
 			   __entry->fences = amdgpu_fence_count_emitted(
-				to_amdgpu_ring(job->base.entity->rq->sched));
+				to_amdgpu_ring(__entry->sched));
 			   ),
 	    TP_printk("bo_list=%p, ring=%u, dw=%u, fences=%u",
 		      __entry->bo_list, __entry->ring, __entry->dw,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 3d2e00efc7415..2740de94e93c3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -171,7 +171,7 @@ amdgpu_ttm_job_submit(struct amdgpu_device *adev, struct amdgpu_ttm_buffer_entit
 {
 	struct amdgpu_ring *ring;
 
-	ring = adev->mman.buffer_funcs_ring;
+	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
 	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
 	WARN_ON(job->ibs[0].length_dw > num_dw);
 
@@ -420,8 +420,8 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
 	if (old_mem->mem_type == TTM_PL_VRAM &&
 	    (abo->flags & AMDGPU_GEM_CREATE_VRAM_WIPE_ON_RELEASE)) {
 		struct dma_fence *wipe_fence = NULL;
-		r = amdgpu_fill_buffer(entity, abo, 0, NULL, &wipe_fence,
-				       AMDGPU_KERNEL_JOB_ID_MOVE_BLIT);
+		r = amdgpu_ttm_clear_buffer(entity, abo, NULL, &wipe_fence,
+					    false, AMDGPU_KERNEL_JOB_ID_MOVE_BLIT);
 		if (r) {
 			goto error;
 		} else if (wipe_fence) {
@@ -1563,7 +1563,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 	if (!adev->mman.sdma_access_ptr)
 		return -EACCES;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!adev->mman.buffer_funcs_enabled || !drm_dev_enter(adev_to_drm(adev), &idx))
 		return -ENODEV;
 
 	if (write)
@@ -1715,10 +1715,7 @@ static void amdgpu_ttm_init_fw_resv_region(struct amdgpu_device *adev)
 		reserve_size = max(reserve_size, (uint32_t)280 << 20);
 	else if (!adev->bios &&
 		 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) {
-		if (hweight32(adev->aid_mask) == 1)
-			reserve_size = max(reserve_size, (uint32_t)128 << 20);
-		else
-			reserve_size = max(reserve_size, (uint32_t)144 << 20);
+		reserve_size = max(reserve_size, (uint32_t)150 << 20);
 	} else if (!reserve_size)
 		reserve_size = DISCOVERY_TMR_OFFSET;
 
@@ -2101,7 +2098,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev)
 	}
 
 	/* Change the size here instead of the init above so only lpfn is affected */
-	amdgpu_ttm_set_buffer_funcs_status(adev, false);
+	amdgpu_ttm_disable_buffer_funcs(adev);
 #ifdef CONFIG_64BIT
 #ifdef CONFIG_X86
 	if (adev->gmc.xgmi.connected_to_cpu)
@@ -2281,115 +2278,92 @@ void amdgpu_ttm_fini(struct amdgpu_device *adev)
 }
 
 /**
- * amdgpu_ttm_set_buffer_funcs_status - enable/disable use of buffer functions
+ * amdgpu_ttm_enable_buffer_funcs - enable use of buffer functions
  *
  * @adev: amdgpu_device pointer
- * @enable: true when we can use buffer functions.
  *
- * Enable/disable use of buffer functions during suspend/resume. This should
+ * Enable use of buffer functions during suspend/resume. This should
  * only be called at bootup or when userspace isn't running.
  */
-void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
+void amdgpu_ttm_enable_buffer_funcs(struct amdgpu_device *adev)
 {
 	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
 	u32 num_clear_entities, num_move_entities;
-	uint64_t size;
 	int r, i, j;
 
 	if (!adev->mman.initialized || amdgpu_in_reset(adev) ||
-	    adev->mman.buffer_funcs_enabled == enable || adev->gmc.is_app_apu)
+	    adev->mman.buffer_funcs_enabled || adev->gmc.is_app_apu)
 		return;
 
-	if (enable) {
-		struct amdgpu_ring *ring;
-		struct drm_gpu_scheduler *sched;
+	if (!adev->mman.num_buffer_funcs_scheds) {
+		dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use");
+		return;
+	}
 
-		if (!adev->mman.buffer_funcs_ring || !adev->mman.buffer_funcs_ring->sched.ready) {
-			dev_warn(adev->dev, "Not enabling DMA transfers for in kernel use");
-			return;
-		}
+	/* default_entity doesn't need multiple schedulers so pass only 1. */
+	r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,
+						&adev->mman.default_entity,
+						DRM_SCHED_PRIORITY_KERNEL,
+						adev->mman.buffer_funcs_scheds, 1, 0);
+	if (r < 0) {
+		dev_err(adev->dev,
+			"Failed setting up TTM entity (%d)\n", r);
+		return;
+	}
 
-		num_clear_entities = 1;
-		num_move_entities = 1;
-		ring = adev->mman.buffer_funcs_ring;
-		sched = &ring->sched;
-		r = amdgpu_ttm_buffer_entity_init(&adev->mman.gtt_mgr,
-						  &adev->mman.default_entity,
-						  DRM_SCHED_PRIORITY_KERNEL,
-						  &sched, 1, 0);
-		if (r < 0) {
-			dev_err(adev->dev,
-				"Failed setting up TTM entity (%d)\n", r);
-			return;
-		}
+	num_clear_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES);
+	num_move_entities = MIN(adev->mman.num_buffer_funcs_scheds, TTM_NUM_MOVE_FENCES);
 
-		adev->mman.clear_entities = kcalloc(num_clear_entities,
-						    sizeof(struct amdgpu_ttm_buffer_entity),
-						    GFP_KERNEL);
-		atomic_set(&adev->mman.next_clear_entity, 0);
-		if (!adev->mman.clear_entities)
-			goto error_free_default_entity;
+	adev->mman.clear_entities = kcalloc(num_clear_entities,
+						sizeof(struct amdgpu_ttm_buffer_entity),
+						GFP_KERNEL);
+	atomic_set(&adev->mman.next_clear_entity, 0);
+	if (!adev->mman.clear_entities)
+		goto error_free_default_entity;
 
-		adev->mman.num_clear_entities = num_clear_entities;
-
-		for (i = 0; i < num_clear_entities; i++) {
-			r = amdgpu_ttm_buffer_entity_init(
-				&adev->mman.gtt_mgr, &adev->mman.clear_entities[i],
-				DRM_SCHED_PRIORITY_NORMAL, &sched, 1, 1);
-
-			if (r < 0) {
-				for (j = 0; j < i; j++)
-					amdgpu_ttm_buffer_entity_fini(
-						&adev->mman.gtt_mgr, &adev->mman.clear_entities[j]);
-				kfree(adev->mman.clear_entities);
-				adev->mman.num_clear_entities = 0;
-				adev->mman.clear_entities = NULL;
-				goto error_free_default_entity;
-			}
+	adev->mman.num_clear_entities = num_clear_entities;
+
+	for (i = 0; i < num_clear_entities; i++) {
+		r = amdgpu_ttm_buffer_entity_init(
+			&adev->mman.gtt_mgr,
+			&adev->mman.clear_entities[i],
+			DRM_SCHED_PRIORITY_KERNEL,
+			adev->mman.buffer_funcs_scheds,
+			adev->mman.num_buffer_funcs_scheds, 1);
+
+		if (r < 0) {
+			for (j = 0; j < i; j++)
+				amdgpu_ttm_buffer_entity_fini(
+					&adev->mman.gtt_mgr, &adev->mman.clear_entities[j]);
+			adev->mman.num_clear_entities = 0;
+			kfree(adev->mman.clear_entities);
+			goto error_free_default_entity;
 		}
+	}
 
-		adev->mman.num_move_entities = num_move_entities;
-		atomic_set(&adev->mman.next_move_entity, 0);
-		for (i = 0; i < num_move_entities; i++) {
-			r = amdgpu_ttm_buffer_entity_init(
-				&adev->mman.gtt_mgr,
-				&adev->mman.move_entities[i],
-				DRM_SCHED_PRIORITY_NORMAL, &sched, 1, 2);
-
-			if (r < 0) {
-				for (j = 0; j < i; j++)
-					amdgpu_ttm_buffer_entity_fini(
-						&adev->mman.gtt_mgr, &adev->mman.move_entities[j]);
-				adev->mman.num_move_entities = 0;
-				goto error_free_clear_entities;
-			}
+	adev->mman.num_move_entities = num_move_entities;
+	atomic_set(&adev->mman.next_move_entity, 0);
+	for (i = 0; i < num_move_entities; i++) {
+		r = amdgpu_ttm_buffer_entity_init(
+			&adev->mman.gtt_mgr,
+			&adev->mman.move_entities[i],
+			DRM_SCHED_PRIORITY_KERNEL,
+			adev->mman.buffer_funcs_scheds,
+			adev->mman.num_buffer_funcs_scheds, 2);
+
+		if (r < 0) {
+			for (j = 0; j < i; j++)
+				amdgpu_ttm_buffer_entity_fini(
+					&adev->mman.gtt_mgr,
+					&adev->mman.move_entities[j]);
+			adev->mman.num_move_entities = 0;
+			goto error_free_clear_entities;
 		}
-	} else {
-		amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
-					      &adev->mman.default_entity);
-		for (i = 0; i < adev->mman.num_clear_entities; i++)
-			amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
-						      &adev->mman.clear_entities[i]);
-		for (i = 0; i < adev->mman.num_move_entities; i++)
-			amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
-						      &adev->mman.move_entities[i]);
-		/* Drop all the old fences since re-creating the scheduler entities
-		 * will allocate new contexts.
-		 */
-		ttm_resource_manager_cleanup(man);
-		kfree(adev->mman.clear_entities);
-		adev->mman.clear_entities = NULL;
-		adev->mman.num_clear_entities = 0;
-		adev->mman.num_move_entities = 0;
 	}
 
 	/* this just adjusts TTM size idea, which sets lpfn to the correct value */
-	if (enable)
-		size = adev->gmc.real_vram_size;
-	else
-		size = adev->gmc.visible_vram_size;
-	man->size = size;
-	adev->mman.buffer_funcs_enabled = enable;
+	man->size = adev->gmc.real_vram_size;
+	adev->mman.buffer_funcs_enabled = true;
 
 	return;
 
@@ -2405,6 +2379,42 @@ void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev, bool enable)
 				      &adev->mman.default_entity);
 }
 
+/**
+ * amdgpu_ttm_disable_buffer_funcs - disable use of buffer functions
+ *
+ * @adev: amdgpu_device pointer
+ */
+void amdgpu_ttm_disable_buffer_funcs(struct amdgpu_device *adev)
+{
+	struct ttm_resource_manager *man =
+		ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
+	int i;
+
+	if (!adev->mman.buffer_funcs_enabled || amdgpu_in_reset(adev))
+		return;
+
+	amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
+				      &adev->mman.default_entity);
+	for (i = 0; i < adev->mman.num_move_entities; i++)
+		amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
+					      &adev->mman.move_entities[i]);
+	for (i = 0; i < adev->mman.num_clear_entities; i++)
+		amdgpu_ttm_buffer_entity_fini(&adev->mman.gtt_mgr,
+					      &adev->mman.clear_entities[i]);
+	/* Drop all the old fences since re-creating the scheduler entities
+	 * will allocate new contexts.
+	 */
+	ttm_resource_manager_cleanup(man);
+
+	kfree(adev->mman.clear_entities);
+	adev->mman.clear_entities = NULL;
+	adev->mman.num_clear_entities = 0;
+	adev->mman.num_move_entities = 0;
+
+	man->size = adev->gmc.visible_vram_size;
+	adev->mman.buffer_funcs_enabled = false;
+}
+
 static int amdgpu_ttm_prepare_job(struct amdgpu_device *adev,
 				  struct amdgpu_ttm_buffer_entity *entity,
 				  unsigned int num_dw,
@@ -2449,7 +2459,7 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,
 	unsigned int i;
 	int r;
 
-	ring = adev->mman.buffer_funcs_ring;
+	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
 
 	if (!ring->sched.ready) {
 		dev_err(adev->dev,
@@ -2524,77 +2534,23 @@ static int amdgpu_ttm_fill_mem(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_ttm_clear_buffer - clear memory buffers
- * @bo: amdgpu buffer object
- * @resv: reservation object
- * @fence: dma_fence associated with the operation
+ * amdgpu_ttm_clear_buffer - fill a buffer with 0
+ * @entity: entity to use
+ * @bo: the bo to fill
+ * @resv: fences contained in this reservation will be used as dependencies.
+ * @out_fence: the fence from the last clear will be stored here. It might be
+ *             NULL if no job was run.
+ * @consider_clear_status: true if region reported as cleared by amdgpu_res_cleared()
+ *                         are skipped.
+ * @k_job_id: trace id
  *
- * Clear the memory buffer resource.
- *
- * Returns:
- * 0 for success or a negative error code on failure.
  */
-int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
+int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity,
+			    struct amdgpu_bo *bo,
 			    struct dma_resv *resv,
-			    struct dma_fence **fence)
-{
-	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
-	struct amdgpu_ttm_buffer_entity *entity;
-	struct amdgpu_res_cursor cursor;
-	u64 addr;
-	int r = 0;
-
-	if (!adev->mman.buffer_funcs_enabled)
-		return -EINVAL;
-
-	if (!fence)
-		return -EINVAL;
-	entity = &adev->mman.clear_entities[0];
-	*fence = dma_fence_get_stub();
-
-	amdgpu_res_first(bo->tbo.resource, 0, amdgpu_bo_size(bo), &cursor);
-
-	mutex_lock(&entity->lock);
-	while (cursor.remaining) {
-		struct dma_fence *next = NULL;
-		u64 size;
-
-		if (amdgpu_res_cleared(&cursor)) {
-			amdgpu_res_next(&cursor, cursor.size);
-			continue;
-		}
-
-		/* Never clear more than 256MiB at once to avoid timeouts */
-		size = min(cursor.size, 256ULL << 20);
-
-		r = amdgpu_ttm_map_buffer(entity, &bo->tbo, bo->tbo.resource, &cursor,
-					  0, false, &size, &addr);
-		if (r)
-			goto err;
-
-		r = amdgpu_ttm_fill_mem(adev, entity, 0, addr, size, resv,
-					&next, true,
-					AMDGPU_KERNEL_JOB_ID_TTM_CLEAR_BUFFER);
-		if (r)
-			goto err;
-
-		dma_fence_put(*fence);
-		*fence = next;
-
-		amdgpu_res_next(&cursor, size);
-	}
-err:
-	mutex_unlock(&entity->lock);
-
-	return r;
-}
-
-int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
-		       struct amdgpu_bo *bo,
-		       uint32_t src_data,
-		       struct dma_resv *resv,
-		       struct dma_fence **f,
-		       u64 k_job_id)
+			    struct dma_fence **out_fence,
+			    bool consider_clear_status,
+			    u64 k_job_id)
 {
 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
 	struct dma_fence *fence = NULL;
@@ -2611,6 +2567,11 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
 		struct dma_fence *next;
 		uint64_t cur_size, to;
 
+		if (consider_clear_status && amdgpu_res_cleared(&dst)) {
+			amdgpu_res_next(&dst, dst.size);
+			continue;
+		}
+
 		/* Never fill more than 256MiB at once to avoid timeouts */
 		cur_size = min(dst.size, 256ULL << 20);
 
@@ -2620,7 +2581,7 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
 			goto error;
 
 		r = amdgpu_ttm_fill_mem(adev, entity,
-					src_data, to, cur_size, resv,
+					0, to, cur_size, resv,
 					&next, true, k_job_id);
 		if (r)
 			goto error;
@@ -2632,9 +2593,7 @@ int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
 	}
 error:
 	mutex_unlock(&entity->lock);
-	if (f)
-		*f = dma_fence_get(fence);
-	dma_fence_put(fence);
+	*out_fence = fence;
 	return r;
 }
 
@@ -2682,6 +2641,41 @@ int amdgpu_ttm_evict_resources(struct amdgpu_device *adev, int mem_type)
 	return ttm_resource_manager_evict_all(&adev->mman.bdev, man);
 }
 
+void amdgpu_sdma_set_buffer_funcs_scheds(struct amdgpu_device *adev,
+					 const struct amdgpu_buffer_funcs *buffer_funcs)
+{
+	struct drm_gpu_scheduler *sched;
+	struct amdgpu_vmhub *hub;
+	int i, n;
+
+	adev->mman.buffer_funcs = buffer_funcs;
+
+	for (i = 0, n = 0; i < adev->sdma.num_instances; i++) {
+		if (adev->sdma.has_page_queue)
+			sched = &adev->sdma.instance[i].page.sched;
+		else
+			sched = &adev->sdma.instance[i].ring.sched;
+
+		if (!sched->ready)
+			continue;
+
+		adev->mman.buffer_funcs_scheds[n++] = sched;
+	}
+
+	if (n == 0) {
+		adev->mman.num_buffer_funcs_scheds = 0;
+		drm_warn(&adev->ddev, "No working sdma ring available\n");
+		return;
+	}
+
+	/* Navi1x's workaround requires us to limit to a single SDMA sched
+	 * for ttm.
+	 */
+	hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
+	adev->mman.num_buffer_funcs_scheds = hub->sdma_invalidation_workaround ?
+		1 : n;
+}
+
 #if defined(CONFIG_DEBUG_FS)
 
 static int amdgpu_ttm_page_pool_show(struct seq_file *m, void *unused)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
index f2f23a42b3cc4..2d72fa2172745 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h
@@ -87,7 +87,8 @@ struct amdgpu_mman {
 
 	/* buffer handling */
 	const struct amdgpu_buffer_funcs	*buffer_funcs;
-	struct amdgpu_ring			*buffer_funcs_ring;
+	struct drm_gpu_scheduler		*buffer_funcs_scheds[AMDGPU_MAX_RINGS];
+	u32					num_buffer_funcs_scheds;
 	bool					buffer_funcs_enabled;
 
 	/* @default_entity: for workarounds, has no gart windows */
@@ -184,8 +185,8 @@ void amdgpu_ttm_unmark_vram_reserved(struct amdgpu_device *adev,
 
 int amdgpu_ttm_init(struct amdgpu_device *adev);
 void amdgpu_ttm_fini(struct amdgpu_device *adev);
-void amdgpu_ttm_set_buffer_funcs_status(struct amdgpu_device *adev,
-					bool enable);
+void amdgpu_ttm_enable_buffer_funcs(struct amdgpu_device *adev);
+void amdgpu_ttm_disable_buffer_funcs(struct amdgpu_device *adev);
 int amdgpu_copy_buffer(struct amdgpu_device *adev,
 		       struct amdgpu_ttm_buffer_entity *entity,
 		       uint64_t src_offset,
@@ -193,15 +194,12 @@ int amdgpu_copy_buffer(struct amdgpu_device *adev,
 		       struct dma_resv *resv,
 		       struct dma_fence **fence,
 		       bool vm_needs_flush, uint32_t copy_flags);
-int amdgpu_ttm_clear_buffer(struct amdgpu_bo *bo,
+int amdgpu_ttm_clear_buffer(struct amdgpu_ttm_buffer_entity *entity,
+			    struct amdgpu_bo *bo,
 			    struct dma_resv *resv,
-			    struct dma_fence **fence);
-int amdgpu_fill_buffer(struct amdgpu_ttm_buffer_entity *entity,
-		       struct amdgpu_bo *bo,
-		       uint32_t src_data,
-		       struct dma_resv *resv,
-		       struct dma_fence **f,
-		       u64 k_job_id);
+			    struct dma_fence **out_fence,
+			    bool consider_clear_status,
+			    u64 k_job_id);
 struct amdgpu_ttm_buffer_entity *amdgpu_ttm_next_clear_entity(struct amdgpu_device *adev);
 
 int amdgpu_ttm_alloc_gart(struct ttm_buffer_object *bo);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
index 0238c2798de4e..b8ed931f8a407 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_umc.c
@@ -130,6 +130,7 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 			if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops &&
 			    adev->umc.ras->ras_block.hw_ops->query_ras_error_address &&
 			    adev->umc.max_ras_err_cnt_per_query) {
+				kfree(err_data->err_addr);
 				err_data->err_addr =
 					kzalloc_objs(struct eeprom_table_record,
 						     adev->umc.max_ras_err_cnt_per_query);
@@ -160,6 +161,7 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev,
 			if (adev->umc.ras &&
 			    adev->umc.ras->ecc_info_query_ras_error_address &&
 			    adev->umc.max_ras_err_cnt_per_query) {
+				kfree(err_data->err_addr);
 				err_data->err_addr =
 					kzalloc_objs(struct eeprom_table_record,
 						     adev->umc.max_ras_err_cnt_per_query);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
index 70d74f04d2ddc..b634defb0be8b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.c
@@ -82,19 +82,11 @@ static bool amdgpu_userq_is_reset_type_supported(struct amdgpu_device *adev,
 	return false;
 }
 
-static void amdgpu_userq_gpu_reset(struct amdgpu_device *adev)
-{
-	if (amdgpu_device_should_recover_gpu(adev)) {
-		amdgpu_reset_domain_schedule(adev->reset_domain,
-					     &adev->userq_reset_work);
-		/* Wait for the reset job to complete */
-		flush_work(&adev->userq_reset_work);
-	}
-}
-
-static int
-amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
+static void amdgpu_userq_mgr_reset_work(struct work_struct *work)
 {
+	struct amdgpu_userq_mgr *uq_mgr =
+		container_of(work, struct amdgpu_userq_mgr,
+			     reset_work);
 	struct amdgpu_device *adev = uq_mgr->adev;
 	const int queue_types[] = {
 		AMDGPU_RING_TYPE_COMPUTE,
@@ -103,12 +95,11 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
 	};
 	const int num_queue_types = ARRAY_SIZE(queue_types);
 	bool gpu_reset = false;
-	int r = 0;
-	int i;
+	int i, r;
 
 	if (unlikely(adev->debug_disable_gpu_ring_reset)) {
 		dev_err(adev->dev, "userq reset disabled by debug mask\n");
-		return 0;
+		return;
 	}
 
 	/*
@@ -116,7 +107,7 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
 	 * skip all reset detection logic
 	 */
 	if (!amdgpu_gpu_recovery)
-		return 0;
+		return;
 
 	/*
 	 * Iterate through all queue types to detect and reset problematic queues
@@ -141,10 +132,19 @@ amdgpu_userq_detect_and_reset_queues(struct amdgpu_userq_mgr *uq_mgr)
 		}
 	}
 
-	if (gpu_reset)
-		amdgpu_userq_gpu_reset(adev);
+	if (gpu_reset) {
+		struct amdgpu_reset_context reset_context;
 
-	return r;
+		memset(&reset_context, 0, sizeof(reset_context));
+
+		reset_context.method = AMD_RESET_METHOD_NONE;
+		reset_context.reset_req_dev = adev;
+		reset_context.src = AMDGPU_RESET_SRC_USERQ;
+		set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
+		/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
+
+		amdgpu_device_gpu_recover(adev, NULL, &reset_context);
+	}
 }
 
 static void amdgpu_userq_hang_detect_work(struct work_struct *work)
@@ -153,7 +153,11 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work)
 		container_of(work, struct amdgpu_usermode_queue,
 			     hang_detect_work.work);
 
-	amdgpu_userq_detect_and_reset_queues(queue->userq_mgr);
+	/*
+	 * Don't schedule the work here! Scheduling or queue work from one reset
+	 * handler to another is illegal if you don't take extra precautions!
+	 */
+	amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work);
 }
 
 /*
@@ -182,8 +186,8 @@ void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue)
 		break;
 	}
 
-	schedule_delayed_work(&queue->hang_detect_work,
-		     msecs_to_jiffies(timeout_ms));
+	queue_delayed_work(adev->reset_domain->wq, &queue->hang_detect_work,
+			   msecs_to_jiffies(timeout_ms));
 }
 
 void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell)
@@ -223,7 +227,7 @@ static int amdgpu_userq_buffer_va_list_add(struct amdgpu_usermode_queue *queue,
 
 	INIT_LIST_HEAD(&va_cursor->list);
 	va_cursor->gpu_addr = addr;
-	atomic_set(&va_map->bo_va->userq_va_mapped, 1);
+	va_map->bo_va->userq_va_mapped = true;
 	list_add(&va_cursor->list, &queue->userq_va_list);
 
 	return 0;
@@ -270,7 +274,7 @@ static bool amdgpu_userq_buffer_va_mapped(struct amdgpu_vm *vm, u64 addr)
 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 
 	mapping = amdgpu_vm_bo_lookup_mapping(vm, addr);
-	if (!IS_ERR_OR_NULL(mapping) && atomic_read(&mapping->bo_va->userq_va_mapped))
+	if (!IS_ERR_OR_NULL(mapping) && mapping->bo_va->userq_va_mapped)
 		r = true;
 	else
 		r = false;
@@ -296,16 +300,8 @@ static bool amdgpu_userq_buffer_vas_mapped(struct amdgpu_usermode_queue *queue)
 	return false;
 }
 
-static void amdgpu_userq_buffer_va_list_del(struct amdgpu_bo_va_mapping *mapping,
-					    struct amdgpu_userq_va_cursor *va_cursor)
-{
-	atomic_set(&mapping->bo_va->userq_va_mapped, 0);
-	list_del(&va_cursor->list);
-	kfree(va_cursor);
-}
-
-static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev,
-						struct amdgpu_usermode_queue *queue)
+static void amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev,
+						 struct amdgpu_usermode_queue *queue)
 {
 	struct amdgpu_userq_va_cursor *va_cursor, *tmp;
 	struct amdgpu_bo_va_mapping *mapping;
@@ -315,15 +311,12 @@ static int amdgpu_userq_buffer_vas_list_cleanup(struct amdgpu_device *adev,
 
 	list_for_each_entry_safe(va_cursor, tmp, &queue->userq_va_list, list) {
 		mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, va_cursor->gpu_addr);
-		if (!mapping) {
-			return -EINVAL;
-		}
-		dev_dbg(adev->dev, "delete the userq:%p va:%llx\n",
-			queue, va_cursor->gpu_addr);
-		amdgpu_userq_buffer_va_list_del(mapping, va_cursor);
+		if (mapping)
+			dev_dbg(adev->dev, "delete the userq:%p va:%llx\n",
+				queue, va_cursor->gpu_addr);
+		list_del(&va_cursor->list);
+		kfree(va_cursor);
 	}
-
-	return 0;
 }
 
 static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue)
@@ -504,16 +497,20 @@ int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
 		goto free_obj;
 	}
 
+	r = amdgpu_bo_pin(userq_obj->obj, AMDGPU_GEM_DOMAIN_GTT);
+	if (r)
+		goto unresv;
+
 	r = amdgpu_ttm_alloc_gart(&(userq_obj->obj)->tbo);
 	if (r) {
 		drm_file_err(uq_mgr->file, "Failed to alloc GART for userqueue object (%d)", r);
-		goto unresv;
+		goto unpin_bo;
 	}
 
 	r = amdgpu_bo_kmap(userq_obj->obj, &userq_obj->cpu_ptr);
 	if (r) {
 		drm_file_err(uq_mgr->file, "Failed to map BO for userqueue (%d)", r);
-		goto unresv;
+		goto unpin_bo;
 	}
 
 	userq_obj->gpu_addr = amdgpu_bo_gpu_offset(userq_obj->obj);
@@ -521,11 +518,13 @@ int amdgpu_userq_create_object(struct amdgpu_userq_mgr *uq_mgr,
 	memset(userq_obj->cpu_ptr, 0, size);
 	return 0;
 
+unpin_bo:
+	amdgpu_bo_unpin(userq_obj->obj);
 unresv:
 	amdgpu_bo_unreserve(userq_obj->obj);
-
 free_obj:
 	amdgpu_bo_unref(&userq_obj->obj);
+
 	return r;
 }
 
@@ -533,6 +532,7 @@ void amdgpu_userq_destroy_object(struct amdgpu_userq_mgr *uq_mgr,
 				 struct amdgpu_userq_obj *userq_obj)
 {
 	amdgpu_bo_kunmap(userq_obj->obj);
+	amdgpu_bo_unpin(userq_obj->obj);
 	amdgpu_bo_unref(&userq_obj->obj);
 }
 
@@ -708,14 +708,14 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	const struct amdgpu_userq_funcs *uq_funcs;
 	struct amdgpu_usermode_queue *queue;
 	struct amdgpu_db_info db_info;
-	bool skip_map_queue;
-	u32 qid;
 	uint64_t index;
-	int r = 0;
-	int priority =
-		(args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK) >>
-		AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
+	int priority;
+	u32 qid;
+	int r;
 
+	priority =
+		(args->in.flags & AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_MASK)
+		>> AMDGPU_USERQ_CREATE_FLAGS_QUEUE_PRIORITY_SHIFT;
 	r = amdgpu_userq_priority_permit(filp, priority);
 	if (r)
 		return r;
@@ -728,40 +728,43 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 
 	uq_funcs = adev->userq_funcs[args->in.ip_type];
 	if (!uq_funcs) {
-		drm_file_err(uq_mgr->file, "Usermode queue is not supported for this IP (%u)\n",
-			     args->in.ip_type);
 		r = -EINVAL;
 		goto err_pm_runtime;
 	}
 
 	queue = kzalloc_obj(struct amdgpu_usermode_queue);
 	if (!queue) {
-		drm_file_err(uq_mgr->file, "Failed to allocate memory for queue\n");
 		r = -ENOMEM;
 		goto err_pm_runtime;
 	}
 
+	kref_init(&queue->refcount);
 	INIT_LIST_HEAD(&queue->userq_va_list);
 	queue->doorbell_handle = args->in.doorbell_handle;
 	queue->queue_type = args->in.ip_type;
 	queue->vm = &fpriv->vm;
 	queue->priority = priority;
-
-	db_info.queue_type = queue->queue_type;
-	db_info.doorbell_handle = queue->doorbell_handle;
-	db_info.db_obj = &queue->db_obj;
-	db_info.doorbell_offset = args->in.doorbell_offset;
-
 	queue->userq_mgr = uq_mgr;
+	INIT_DELAYED_WORK(&queue->hang_detect_work,
+			  amdgpu_userq_hang_detect_work);
 
-	/* Validate the userq virtual address.*/
-	r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
+	mutex_init(&queue->fence_drv_lock);
+	xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
+	r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
 	if (r)
 		goto free_queue;
 
-	if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va, args->in.queue_size) ||
-	    amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va, AMDGPU_GPU_PAGE_SIZE) ||
-	    amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va, AMDGPU_GPU_PAGE_SIZE)) {
+	/* Make sure the queue can actually run with those virtual addresses. */
+	r = amdgpu_bo_reserve(fpriv->vm.root.bo, false);
+	if (r)
+		goto free_fence_drv;
+
+	if (amdgpu_userq_input_va_validate(adev, queue, args->in.queue_va,
+					   args->in.queue_size) ||
+	    amdgpu_userq_input_va_validate(adev, queue, args->in.rptr_va,
+					   AMDGPU_GPU_PAGE_SIZE) ||
+	    amdgpu_userq_input_va_validate(adev, queue, args->in.wptr_va,
+					   AMDGPU_GPU_PAGE_SIZE)) {
 		r = -EINVAL;
 		amdgpu_bo_unreserve(fpriv->vm.root.bo);
 		goto clean_mapping;
@@ -769,6 +772,10 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	amdgpu_bo_unreserve(fpriv->vm.root.bo);
 
 	/* Convert relative doorbell offset into absolute doorbell index */
+	db_info.queue_type = queue->queue_type;
+	db_info.doorbell_handle = queue->doorbell_handle;
+	db_info.db_obj = &queue->db_obj;
+	db_info.doorbell_offset = args->in.doorbell_offset;
 	index = amdgpu_userq_get_doorbell_index(uq_mgr, &db_info, filp);
 	if (index == (uint64_t)-EINVAL) {
 		drm_file_err(uq_mgr->file, "Failed to get doorbell for queue\n");
@@ -777,82 +784,64 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args)
 	}
 
 	queue->doorbell_index = index;
-	mutex_init(&queue->fence_drv_lock);
-	xa_init_flags(&queue->fence_drv_xa, XA_FLAGS_ALLOC);
-	r = amdgpu_userq_fence_driver_alloc(adev, &queue->fence_drv);
-	if (r) {
-		drm_file_err(uq_mgr->file, "Failed to alloc fence driver\n");
-		goto clean_mapping;
-	}
-
 	r = uq_funcs->mqd_create(queue, &args->in);
 	if (r) {
 		drm_file_err(uq_mgr->file, "Failed to create Queue\n");
-		goto clean_fence_driver;
+		goto clean_mapping;
 	}
 
+	/* Update VM owner at userq submit-time for page-fault attribution. */
+	amdgpu_vm_set_task_info(&fpriv->vm);
+
+	r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue,
+				GFP_KERNEL));
+	if (r)
+		goto clean_mqd;
+
 	amdgpu_userq_ensure_ev_fence(&fpriv->userq_mgr, &fpriv->evf_mgr);
 
 	/* don't map the queue if scheduling is halted */
-	if (adev->userq_halt_for_enforce_isolation &&
-	    ((queue->queue_type == AMDGPU_HW_IP_GFX) ||
-	     (queue->queue_type == AMDGPU_HW_IP_COMPUTE)))
-		skip_map_queue = true;
-	else
-		skip_map_queue = false;
-	if (!skip_map_queue) {
+	if (!adev->userq_halt_for_enforce_isolation ||
+	    ((queue->queue_type != AMDGPU_HW_IP_GFX) &&
+	     (queue->queue_type != AMDGPU_HW_IP_COMPUTE))) {
 		r = amdgpu_userq_map_helper(queue);
 		if (r) {
 			drm_file_err(uq_mgr->file, "Failed to map Queue\n");
-			goto clean_mqd;
+			mutex_unlock(&uq_mgr->userq_mutex);
+			goto clean_doorbell;
 		}
 	}
 
-	/* drop this refcount during queue destroy */
-	kref_init(&queue->refcount);
-
-	/* Wait for mode-1 reset to complete */
-	down_read(&adev->reset_domain->sem);
+	atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
+	mutex_unlock(&uq_mgr->userq_mutex);
 
 	r = xa_alloc(&uq_mgr->userq_xa, &qid, queue,
-		     XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT), GFP_KERNEL);
-	if (r) {
-		if (!skip_map_queue)
-			amdgpu_userq_unmap_helper(queue);
-		r = -ENOMEM;
-		goto clean_reset_domain;
-	}
-
-	r = xa_err(xa_store_irq(&adev->userq_doorbell_xa, index, queue, GFP_KERNEL));
+		     XA_LIMIT(1, AMDGPU_MAX_USERQ_COUNT),
+		     GFP_KERNEL);
 	if (r) {
-		xa_erase(&uq_mgr->userq_xa, qid);
-		if (!skip_map_queue)
-			amdgpu_userq_unmap_helper(queue);
-		goto clean_reset_domain;
+		/*
+		 * This drops the last reference which should take care of
+		 * all cleanup.
+		 */
+		amdgpu_userq_put(queue);
+		return r;
 	}
-	up_read(&adev->reset_domain->sem);
 
 	amdgpu_debugfs_userq_init(filp, queue, qid);
-	INIT_DELAYED_WORK(&queue->hang_detect_work,
-			  amdgpu_userq_hang_detect_work);
-
 	args->out.queue_id = qid;
-	atomic_inc(&uq_mgr->userq_count[queue->queue_type]);
-	mutex_unlock(&uq_mgr->userq_mutex);
 	return 0;
 
-clean_reset_domain:
-	up_read(&adev->reset_domain->sem);
+clean_doorbell:
+	xa_erase_irq(&adev->userq_doorbell_xa, index);
 clean_mqd:
-	mutex_unlock(&uq_mgr->userq_mutex);
 	uq_funcs->mqd_destroy(queue);
-clean_fence_driver:
-	amdgpu_userq_fence_driver_free(queue);
 clean_mapping:
 	amdgpu_bo_reserve(fpriv->vm.root.bo, true);
 	amdgpu_userq_buffer_vas_list_cleanup(adev, queue);
 	amdgpu_bo_unreserve(fpriv->vm.root.bo);
 	mutex_destroy(&queue->fence_drv_lock);
+free_fence_drv:
+	amdgpu_userq_fence_driver_free(queue);
 free_queue:
 	kfree(queue);
 err_pm_runtime:
@@ -1039,12 +1028,12 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 	struct amdgpu_bo *bo;
 	int ret;
 
-	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->invalidated)) {
-		bo_va = list_first_entry(&vm->invalidated,
+	spin_lock(&vm->individual_lock);
+	while (!list_empty(&vm->always_valid.evicted)) {
+		bo_va = list_first_entry(&vm->always_valid.evicted,
 					 struct amdgpu_bo_va,
 					 base.vm_status);
-		spin_unlock(&vm->status_lock);
+		spin_unlock(&vm->individual_lock);
 
 		bo = bo_va->base.bo;
 		ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2);
@@ -1056,14 +1045,14 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec,
 		if (ret)
 			return ret;
 
-		/* This moves the bo_va to the done list */
+		/* This moves the bo_va to the idle list */
 		ret = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (ret)
 			return ret;
 
-		spin_lock(&vm->status_lock);
+		spin_lock(&vm->individual_lock);
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->individual_lock);
 
 	return 0;
 }
@@ -1095,7 +1084,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 		if (unlikely(ret))
 			goto unlock_all;
 
-		ret = amdgpu_vm_lock_done_list(vm, &exec, 1);
+		ret = amdgpu_vm_lock_individual(vm, &exec, 1);
 		drm_exec_retry_on_contention(&exec);
 		if (unlikely(ret))
 			goto unlock_all;
@@ -1138,7 +1127,7 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 
 	key = 0;
 	/* Validate User Ptr BOs */
-	list_for_each_entry(bo_va, &vm->done, base.vm_status) {
+	list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status) {
 		bo = bo_va->base.bo;
 		if (!bo)
 			continue;
@@ -1188,10 +1177,10 @@ amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr)
 
 	/*
 	 * We need to wait for all VM updates to finish before restarting the
-	 * queues. Using the done list like that is now ok since everything is
+	 * queues. Using the idle list like that is now ok since everything is
 	 * locked in place.
 	 */
-	list_for_each_entry(bo_va, &vm->done, base.vm_status)
+	list_for_each_entry(bo_va, &vm->always_valid.idle, base.vm_status)
 		dma_fence_wait(bo_va->last_pt_update, false);
 	dma_fence_wait(vm->last_update, false);
 
@@ -1252,28 +1241,13 @@ amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr)
 	if (ret) {
 		drm_file_err(uq_mgr->file,
 			     "Couldn't unmap all the queues, eviction failed ret=%d\n", ret);
-		amdgpu_userq_detect_and_reset_queues(uq_mgr);
+		amdgpu_reset_domain_schedule(uq_mgr->adev->reset_domain,
+					     &uq_mgr->reset_work);
+		flush_work(&uq_mgr->reset_work);
 	}
 	return ret;
 }
 
-void amdgpu_userq_reset_work(struct work_struct *work)
-{
-	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
-						  userq_reset_work);
-	struct amdgpu_reset_context reset_context;
-
-	memset(&reset_context, 0, sizeof(reset_context));
-
-	reset_context.method = AMD_RESET_METHOD_NONE;
-	reset_context.reset_req_dev = adev;
-	reset_context.src = AMDGPU_RESET_SRC_USERQ;
-	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
-	/*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/
-
-	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
-}
-
 static void
 amdgpu_userq_wait_for_signal(struct amdgpu_userq_mgr *uq_mgr)
 {
@@ -1307,9 +1281,24 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f
 	userq_mgr->file = file_priv;
 
 	INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker);
+	INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work);
 	return 0;
 }
 
+void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev)
+{
+	struct xarray *xa = &adev->userq_doorbell_xa;
+	struct amdgpu_usermode_queue *queue;
+	unsigned long flags, queue_id;
+
+	xa_lock_irqsave(xa, flags);
+	xa_for_each(xa, queue_id, queue) {
+		cancel_delayed_work(&queue->hang_detect_work);
+		cancel_work(&queue->userq_mgr->reset_work);
+	}
+	xa_unlock_irqrestore(xa, flags);
+}
+
 void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr)
 {
 	cancel_delayed_work_sync(&userq_mgr->resume_work);
@@ -1335,6 +1324,14 @@ void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr)
 	}
 
 	xa_destroy(&userq_mgr->userq_xa);
+
+	/*
+	 * Drain any in-flight reset_work. By this point all queues are freed
+	 * and userq_count is 0, so if reset_work starts now it exits early.
+	 * We still need to wait in case it was already executing gpu_recover.
+	 */
+	cancel_work_sync(&userq_mgr->reset_work);
+
 	mutex_destroy(&userq_mgr->userq_mutex);
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
index 85f460e7c31b2..49b33e2d6932f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq.h
@@ -84,7 +84,13 @@ struct amdgpu_usermode_queue {
 	u32			xcp_id;
 	int			priority;
 	struct dentry		*debugfs_queue;
-	struct delayed_work hang_detect_work;
+
+	/**
+	 * @hang_detect_work:
+	 *
+	 * Delayed work which runs when userq_fences time out.
+	 */
+	struct delayed_work	hang_detect_work;
 	struct kref		refcount;
 
 	struct list_head	userq_va_list;
@@ -116,6 +122,13 @@ struct amdgpu_userq_mgr {
 	struct amdgpu_device		*adev;
 	struct delayed_work		resume_work;
 	struct drm_file			*file;
+
+	/**
+	 * @reset_work:
+	 *
+	 * Reset work which is used when eviction fails.
+	 */
+	struct work_struct		reset_work;
 	atomic_t                        userq_count[AMDGPU_RING_TYPE_MAX];
 };
 
@@ -134,6 +147,7 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp
 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv,
 			  struct amdgpu_device *adev);
 
+void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev);
 void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr);
 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
index 53a8944bab05a..a41fb72dba940 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_userq_fence.c
@@ -370,51 +370,48 @@ static int amdgpu_userq_fence_read_wptr(struct amdgpu_device *adev,
 {
 	struct amdgpu_bo_va_mapping *mapping;
 	struct amdgpu_bo *bo;
+	struct drm_exec exec;
 	u64 addr, *ptr;
-	int r;
-
-	r = amdgpu_bo_reserve(queue->vm->root.bo, false);
-	if (r)
-		return r;
+	int ret;
 
 	addr = queue->userq_prop->wptr_gpu_addr;
 	addr &= AMDGPU_GMC_HOLE_MASK;
 
-	mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT);
-	if (!mapping) {
-		amdgpu_bo_unreserve(queue->vm->root.bo);
-		DRM_ERROR("Failed to lookup amdgpu_bo_va_mapping\n");
-		return -EINVAL;
-	}
+	drm_exec_init(&exec, DRM_EXEC_IGNORE_DUPLICATES, 2);
+	drm_exec_until_all_locked(&exec) {
+		ret = amdgpu_vm_lock_pd(queue->vm, &exec, 1);
+		drm_exec_retry_on_contention(&exec);
+		if (unlikely(ret))
+			goto lock_error;
 
-	bo = amdgpu_bo_ref(mapping->bo_va->base.bo);
-	amdgpu_bo_unreserve(queue->vm->root.bo);
-	r = amdgpu_bo_reserve(bo, true);
-	if (r) {
-		amdgpu_bo_unref(&bo);
-		DRM_ERROR("Failed to reserve userqueue wptr bo");
-		return r;
+		mapping = amdgpu_vm_bo_lookup_mapping(queue->vm, addr >> PAGE_SHIFT);
+		if (!mapping) {
+			ret = -EINVAL;
+			goto lock_error;
+		}
+
+		ret = drm_exec_lock_obj(&exec, &mapping->bo_va->base.bo->tbo.base);
+		drm_exec_retry_on_contention(&exec);
+		if (unlikely(ret))
+			goto lock_error;
 	}
 
-	r = amdgpu_bo_kmap(bo, (void **)&ptr);
-	if (r) {
+	bo = mapping->bo_va->base.bo;
+	ret = amdgpu_bo_kmap(bo, (void **)&ptr);
+	if (ret) {
 		DRM_ERROR("Failed mapping the userqueue wptr bo");
-		goto map_error;
+		goto lock_error;
 	}
 
 	*wptr = le64_to_cpu(*ptr);
 
 	amdgpu_bo_kunmap(bo);
-	amdgpu_bo_unreserve(bo);
-	amdgpu_bo_unref(&bo);
-
+	drm_exec_fini(&exec);
 	return 0;
 
-map_error:
-	amdgpu_bo_unreserve(bo);
-	amdgpu_bo_unref(&bo);
-
-	return r;
+lock_error:
+	drm_exec_fini(&exec);
+	return ret;
 }
 
 static void
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index 9d5cca7da1d9e..3a3bc0d370fa6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -512,7 +512,7 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 			}
 			memset_io(ptr, 0, size);
 			/* to restore uvd fence seq */
-			amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring);
+			amdgpu_fence_driver_force_completion(&adev->uvd.inst[i].ring, NULL);
 		}
 	}
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index debb82a2e031d..616967519869e 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -1486,18 +1486,27 @@ int vcn_set_powergating_state(struct amdgpu_ip_block *ip_block,
 }
 
 /**
- * amdgpu_vcn_reset_engine - Reset a specific VCN engine
- * @adev: Pointer to the AMDGPU device
- * @instance_id: VCN engine instance to reset
+ * amdgpu_vcn_ring_reset - Reset a VCN ring
+ * @ring: ring to reset
+ * @vmid: vmid of guilty job
+ * @timedout_fence: fence of timed out job
  *
+ * This helper is for VCN blocks without unified queues because
+ * resetting the engine resets all queues in that case.  With
+ * unified queues we have one queue per engine.
  * Returns: 0 on success, or a negative error code on failure.
  */
-static int amdgpu_vcn_reset_engine(struct amdgpu_device *adev,
-				   uint32_t instance_id)
+int amdgpu_vcn_ring_reset(struct amdgpu_ring *ring,
+			  unsigned int vmid,
+			  struct amdgpu_fence *timedout_fence)
 {
-	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[instance_id];
+	struct amdgpu_device *adev = ring->adev;
+	struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me];
 	int r, i;
 
+	if (adev->vcn.inst[ring->me].using_unified_queue)
+		return -EINVAL;
+
 	mutex_lock(&vinst->engine_reset_mutex);
 	/* Stop the scheduler's work queue for the dec and enc rings if they are running.
 	 * This ensures that no new tasks are submitted to the queues while
@@ -1519,9 +1528,13 @@ static int amdgpu_vcn_reset_engine(struct amdgpu_device *adev,
 		if (r)
 			goto unlock;
 	}
-	amdgpu_fence_driver_force_completion(&vinst->ring_dec);
+	amdgpu_fence_driver_force_completion(&vinst->ring_dec,
+					     (&vinst->ring_dec == ring) ?
+					     &timedout_fence->base : NULL);
 	for (i = 0; i < vinst->num_enc_rings; i++)
-		amdgpu_fence_driver_force_completion(&vinst->ring_enc[i]);
+		amdgpu_fence_driver_force_completion(&vinst->ring_enc[i],
+						     (&vinst->ring_enc[i] == ring) ?
+						     &timedout_fence->base : NULL);
 
 	/* Restart the scheduler's work queue for the dec and enc rings
 	 * if they were stopped by this function. This allows new tasks
@@ -1537,29 +1550,6 @@ static int amdgpu_vcn_reset_engine(struct amdgpu_device *adev,
 	return r;
 }
 
-/**
- * amdgpu_vcn_ring_reset - Reset a VCN ring
- * @ring: ring to reset
- * @vmid: vmid of guilty job
- * @timedout_fence: fence of timed out job
- *
- * This helper is for VCN blocks without unified queues because
- * resetting the engine resets all queues in that case.  With
- * unified queues we have one queue per engine.
- * Returns: 0 on success, or a negative error code on failure.
- */
-int amdgpu_vcn_ring_reset(struct amdgpu_ring *ring,
-			  unsigned int vmid,
-			  struct amdgpu_fence *timedout_fence)
-{
-	struct amdgpu_device *adev = ring->adev;
-
-	if (adev->vcn.inst[ring->me].using_unified_queue)
-		return -EINVAL;
-
-	return amdgpu_vcn_reset_engine(adev, ring->me);
-}
-
 int amdgpu_vcn_reg_dump_init(struct amdgpu_device *adev,
 			     const struct amdgpu_hwip_reg_entry *reg, u32 count)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
index e8d180a412d16..3fcbd722ee3b4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c
@@ -463,6 +463,9 @@ static void amdgpu_virt_add_bad_page(struct amdgpu_device *adev,
 static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 {
 	struct amd_sriov_msg_pf2vf_info_header *pf2vf_info = adev->virt.fw_reserve.p_pf2vf;
+	struct amdgim_pf2vf_info_v1 *pf2vf_v1;
+	struct amd_sriov_msg_pf2vf_info *pf2vf;
+
 	uint32_t checksum;
 	uint32_t checkval;
 
@@ -479,7 +482,8 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 
 	switch (pf2vf_info->version) {
 	case 1:
-		checksum = ((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->checksum;
+		pf2vf_v1 = (struct amdgim_pf2vf_info_v1 *)pf2vf_info;
+		checksum = pf2vf_v1->checksum;
 		checkval = amd_sriov_msg_checksum(
 			adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size,
 			adev->virt.fw_reserve.checksum_key, checksum);
@@ -490,12 +494,12 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 			return -EINVAL;
 		}
 
-		adev->virt.gim_feature =
-			((struct amdgim_pf2vf_info_v1 *)pf2vf_info)->feature_flags;
+		adev->virt.gim_feature = pf2vf_v1->feature_flags;
 		break;
 	case 2:
 		/* TODO: missing key, need to add it later */
-		checksum = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->checksum;
+		pf2vf = (struct amd_sriov_msg_pf2vf_info *)pf2vf_info;
+		checksum = pf2vf->checksum;
 		checkval = amd_sriov_msg_checksum(
 			adev->virt.fw_reserve.p_pf2vf, pf2vf_info->size,
 			0, checksum);
@@ -507,11 +511,9 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 		}
 
 		adev->virt.vf2pf_update_interval_ms =
-			((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->vf2pf_update_interval_ms;
-		adev->virt.gim_feature =
-			((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->feature_flags.all;
-		adev->virt.reg_access =
-			((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->reg_access_flags.all;
+			pf2vf->vf2pf_update_interval_ms;
+		adev->virt.gim_feature = pf2vf->feature_flags.all;
+		adev->virt.reg_access = pf2vf->reg_access_flags.all;
 
 		adev->virt.decode_max_dimension_pixels = 0;
 		adev->virt.decode_max_frame_pixels = 0;
@@ -519,26 +521,30 @@ static int amdgpu_virt_read_pf2vf_data(struct amdgpu_device *adev)
 		adev->virt.encode_max_frame_pixels = 0;
 		adev->virt.is_mm_bw_enabled = false;
 		for (i = 0; i < AMD_SRIOV_MSG_RESERVE_VCN_INST; i++) {
-			tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_dimension_pixels;
+			tmp = pf2vf->mm_bw_management[i].decode_max_dimension_pixels;
 			adev->virt.decode_max_dimension_pixels = max(tmp, adev->virt.decode_max_dimension_pixels);
 
-			tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].decode_max_frame_pixels;
+			tmp = pf2vf->mm_bw_management[i].decode_max_frame_pixels;
 			adev->virt.decode_max_frame_pixels = max(tmp, adev->virt.decode_max_frame_pixels);
 
-			tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_dimension_pixels;
+			tmp = pf2vf->mm_bw_management[i].encode_max_dimension_pixels;
 			adev->virt.encode_max_dimension_pixels = max(tmp, adev->virt.encode_max_dimension_pixels);
 
-			tmp = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->mm_bw_management[i].encode_max_frame_pixels;
+			tmp = pf2vf->mm_bw_management[i].encode_max_frame_pixels;
 			adev->virt.encode_max_frame_pixels = max(tmp, adev->virt.encode_max_frame_pixels);
 		}
 		if ((adev->virt.decode_max_dimension_pixels > 0) || (adev->virt.encode_max_dimension_pixels > 0))
 			adev->virt.is_mm_bw_enabled = true;
 
-		adev->unique_id =
-			((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->uuid;
-		adev->virt.ras_en_caps.all = ((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->ras_en_caps.all;
+		adev->unique_id = pf2vf->uuid;
+
+		adev->unitid = 0;
+		if (amdgpu_sriov_is_unitid_support(adev))
+			adev->unitid = pf2vf->unitid;
+
+		adev->virt.ras_en_caps.all = pf2vf->ras_en_caps.all;
 		adev->virt.ras_telemetry_en_caps.all =
-			((struct amd_sriov_msg_pf2vf_info *)pf2vf_info)->ras_telemetry_en_caps.all;
+			pf2vf->ras_telemetry_en_caps.all;
 		break;
 	default:
 		dev_err(adev->dev, "invalid pf2vf version: 0x%x\n", pf2vf_info->version);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
index 9da0c6e9b8695..d563deec0916b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_virt.h
@@ -163,6 +163,8 @@ enum AMDGIM_FEATURE_FLAG {
 	AMDGIM_FEATURE_RAS_CPER = (1 << 11),
 	AMDGIM_FEATURE_XGMI_TA_EXT_PEER_LINK = (1 << 12),
 	AMDGIM_FEATURE_XGMI_CONNECTED_TO_CPU = (1 << 13),
+	AMDGIM_FEATURE_PTL_SUPPORT = (1 << 14),
+	AMDGIM_FEATURE_UNITID_SUPPORT = (1 << 15),
 };
 
 enum AMDGIM_REG_ACCESS_FLAG {
@@ -441,6 +443,8 @@ static inline bool is_virtual_machine(void)
 	((adev)->virt.gim_feature & AMDGIM_FEATURE_VCN_RB_DECOUPLE)
 #define amdgpu_sriov_is_mes_info_enable(adev) \
 	((adev)->virt.gim_feature & AMDGIM_FEATURE_MES_INFO_ENABLE)
+#define amdgpu_sriov_is_unitid_support(adev) \
+	((adev)->virt.gim_feature & AMDGIM_FEATURE_UNITID_SUPPORT)
 
 #define amdgpu_virt_xgmi_migrate_enabled(adev) \
 	((adev)->virt.is_xgmi_node_migrate_enabled && (adev)->gmc.xgmi.node_segment_size != 0)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
index e54295b56282a..170adaf7e76ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vkms.c
@@ -5,6 +5,7 @@
 #include <drm/drm_simple_kms_helper.h>
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_vblank.h>
+#include <drm/drm_vblank_helper.h>
 
 #include "amdgpu.h"
 #ifdef CONFIG_DRM_AMDGPU_SI
@@ -42,81 +43,6 @@ static const u32 amdgpu_vkms_formats[] = {
 	DRM_FORMAT_XRGB8888,
 };
 
-static enum hrtimer_restart amdgpu_vkms_vblank_simulate(struct hrtimer *timer)
-{
-	struct amdgpu_crtc *amdgpu_crtc = container_of(timer, struct amdgpu_crtc, vblank_timer);
-	struct drm_crtc *crtc = &amdgpu_crtc->base;
-	struct amdgpu_vkms_output *output = drm_crtc_to_amdgpu_vkms_output(crtc);
-	u64 ret_overrun;
-	bool ret;
-
-	ret_overrun = hrtimer_forward_now(&amdgpu_crtc->vblank_timer,
-					  output->period_ns);
-	if (ret_overrun != 1)
-		drm_warn(amdgpu_crtc->base.dev,
-			 "%s: vblank timer overrun count: %llu\n",
-			 __func__, ret_overrun);
-
-	ret = drm_crtc_handle_vblank(crtc);
-	/* Don't queue timer again when vblank is disabled. */
-	if (!ret)
-		return HRTIMER_NORESTART;
-
-	return HRTIMER_RESTART;
-}
-
-static int amdgpu_vkms_enable_vblank(struct drm_crtc *crtc)
-{
-	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
-	struct amdgpu_vkms_output *out = drm_crtc_to_amdgpu_vkms_output(crtc);
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-
-	drm_calc_timestamping_constants(crtc, &crtc->mode);
-
-	out->period_ns = ktime_set(0, vblank->framedur_ns);
-	hrtimer_start(&amdgpu_crtc->vblank_timer, out->period_ns, HRTIMER_MODE_REL);
-
-	return 0;
-}
-
-static void amdgpu_vkms_disable_vblank(struct drm_crtc *crtc)
-{
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-
-	hrtimer_try_to_cancel(&amdgpu_crtc->vblank_timer);
-}
-
-static bool amdgpu_vkms_get_vblank_timestamp(struct drm_crtc *crtc,
-					     int *max_error,
-					     ktime_t *vblank_time,
-					     bool in_vblank_irq)
-{
-	struct amdgpu_vkms_output *output = drm_crtc_to_amdgpu_vkms_output(crtc);
-	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
-	struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-
-	if (!READ_ONCE(vblank->enabled)) {
-		*vblank_time = ktime_get();
-		return true;
-	}
-
-	*vblank_time = READ_ONCE(amdgpu_crtc->vblank_timer.node.expires);
-
-	if (WARN_ON(*vblank_time == vblank->time))
-		return true;
-
-	/*
-	 * To prevent races we roll the hrtimer forward before we do any
-	 * interrupt processing - this is how real hw works (the interrupt is
-	 * only generated after all the vblank registers are updated) and what
-	 * the vblank core expects. Therefore we need to always correct the
-	 * timestampe by one frame.
-	 */
-	*vblank_time -= output->period_ns;
-
-	return true;
-}
-
 static const struct drm_crtc_funcs amdgpu_vkms_crtc_funcs = {
 	.set_config             = drm_atomic_helper_set_config,
 	.destroy                = drm_crtc_cleanup,
@@ -124,45 +50,11 @@ static const struct drm_crtc_funcs amdgpu_vkms_crtc_funcs = {
 	.reset                  = drm_atomic_helper_crtc_reset,
 	.atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
 	.atomic_destroy_state   = drm_atomic_helper_crtc_destroy_state,
-	.enable_vblank		= amdgpu_vkms_enable_vblank,
-	.disable_vblank		= amdgpu_vkms_disable_vblank,
-	.get_vblank_timestamp	= amdgpu_vkms_get_vblank_timestamp,
+	DRM_CRTC_VBLANK_TIMER_FUNCS,
 };
 
-static void amdgpu_vkms_crtc_atomic_enable(struct drm_crtc *crtc,
-					   struct drm_atomic_state *state)
-{
-	drm_crtc_vblank_on(crtc);
-}
-
-static void amdgpu_vkms_crtc_atomic_disable(struct drm_crtc *crtc,
-					    struct drm_atomic_state *state)
-{
-	drm_crtc_vblank_off(crtc);
-}
-
-static void amdgpu_vkms_crtc_atomic_flush(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
-{
-	unsigned long flags;
-	if (crtc->state->event) {
-		spin_lock_irqsave(&crtc->dev->event_lock, flags);
-
-		if (drm_crtc_vblank_get(crtc) != 0)
-			drm_crtc_send_vblank_event(crtc, crtc->state->event);
-		else
-			drm_crtc_arm_vblank_event(crtc, crtc->state->event);
-
-		spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
-
-		crtc->state->event = NULL;
-	}
-}
-
 static const struct drm_crtc_helper_funcs amdgpu_vkms_crtc_helper_funcs = {
-	.atomic_flush	= amdgpu_vkms_crtc_atomic_flush,
-	.atomic_enable	= amdgpu_vkms_crtc_atomic_enable,
-	.atomic_disable	= amdgpu_vkms_crtc_atomic_disable,
+	DRM_CRTC_HELPER_VBLANK_FUNCS,
 };
 
 static int amdgpu_vkms_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
@@ -187,10 +79,6 @@ static int amdgpu_vkms_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
 	amdgpu_crtc->pll_id = ATOM_PPLL_INVALID;
 	amdgpu_crtc->encoder = NULL;
 	amdgpu_crtc->connector = NULL;
-	amdgpu_crtc->vsync_timer_enabled = AMDGPU_IRQ_STATE_DISABLE;
-
-	hrtimer_setup(&amdgpu_crtc->vblank_timer, &amdgpu_vkms_vblank_simulate, CLOCK_MONOTONIC,
-		      HRTIMER_MODE_REL);
 
 	return ret;
 }
@@ -262,13 +150,13 @@ static const struct drm_plane_funcs amdgpu_vkms_plane_funcs = {
 };
 
 static void amdgpu_vkms_plane_atomic_update(struct drm_plane *plane,
-					    struct drm_atomic_state *old_state)
+					    struct drm_atomic_commit *old_state)
 {
 	return;
 }
 
 static int amdgpu_vkms_plane_atomic_check(struct drm_plane *plane,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -538,11 +426,6 @@ static int amdgpu_vkms_sw_init(struct amdgpu_ip_block *ip_block)
 static int amdgpu_vkms_sw_fini(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
-	int i = 0;
-
-	for (i = 0; i < adev->mode_info.num_crtc; i++)
-		if (adev->mode_info.crtcs[i])
-			hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer);
 
 	drm_kms_helper_poll_fini(adev_to_drm(adev));
 	drm_mode_config_cleanup(adev_to_drm(adev));
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 9ba9de16a27a2..a99d01f9dc750 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -138,6 +138,47 @@ static void amdgpu_vm_assert_locked(struct amdgpu_vm *vm)
 	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 }
 
+/* Initialize the amdgpu_vm_bo_status object */
+static void amdgpu_vm_bo_status_init(struct amdgpu_vm_bo_status *lists)
+{
+	INIT_LIST_HEAD(&lists->evicted);
+	INIT_LIST_HEAD(&lists->moved);
+	INIT_LIST_HEAD(&lists->idle);
+}
+
+/*
+ * Make sure we have the lock to modify the vm_bo status and return the object
+ * with the status lists.
+ */
+static struct amdgpu_vm_bo_status *
+amdgpu_vm_bo_lock_lists(struct amdgpu_vm_bo_base *vm_bo)
+{
+	struct amdgpu_vm *vm = vm_bo->vm;
+	struct amdgpu_bo *bo = vm_bo->bo;
+
+	if (amdgpu_vm_is_bo_always_valid(vm, bo)) {
+		/* No extra locking needed, protected by the root PD resv lock */
+		amdgpu_vm_assert_locked(vm);
+
+		if (bo->tbo.type == ttm_bo_type_kernel)
+			return &vm->kernel;
+
+		return &vm->always_valid;
+	}
+
+	spin_lock(&vm_bo->vm->individual_lock);
+	return &vm->individual;
+}
+
+/* Eventually unlock the status list lock again */
+static void amdgpu_vm_bo_unlock_lists(struct amdgpu_vm_bo_base *vm_bo)
+{
+	if (amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
+		amdgpu_vm_assert_locked(vm_bo->vm);
+	else
+		spin_unlock(&vm_bo->vm->individual_lock);
+}
+
 /**
  * amdgpu_vm_is_bo_always_valid - check if the BO is VM always valid
  *
@@ -157,37 +198,44 @@ bool amdgpu_vm_is_bo_always_valid(struct amdgpu_vm *vm, struct amdgpu_bo *bo)
  *
  * @vm_bo: vm_bo which is evicted
  *
- * State for PDs/PTs and per VM BOs which are not at the location they should
- * be.
+ * State for vm_bo objects meaning the underlying BO was evicted and need to
+ * move in place again.
  */
 static void amdgpu_vm_bo_evicted(struct amdgpu_vm_bo_base *vm_bo)
 {
-	struct amdgpu_vm *vm = vm_bo->vm;
-	struct amdgpu_bo *bo = vm_bo->bo;
+	struct amdgpu_vm_bo_status *lists;
 
+	lists = amdgpu_vm_bo_lock_lists(vm_bo);
 	vm_bo->moved = true;
-	amdgpu_vm_assert_locked(vm);
-	spin_lock(&vm_bo->vm->status_lock);
-	if (bo->tbo.type == ttm_bo_type_kernel)
-		list_move(&vm_bo->vm_status, &vm->evicted);
-	else
-		list_move_tail(&vm_bo->vm_status, &vm->evicted);
-	spin_unlock(&vm_bo->vm->status_lock);
+	list_move(&vm_bo->vm_status, &lists->evicted);
+	amdgpu_vm_bo_unlock_lists(vm_bo);
 }
 /**
  * amdgpu_vm_bo_moved - vm_bo is moved
  *
  * @vm_bo: vm_bo which is moved
  *
- * State for per VM BOs which are moved, but that change is not yet reflected
- * in the page tables.
+ * State for vm_bo objects meaning the underlying BO was moved but the new
+ * location not yet reflected in the page tables.
  */
 static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
 {
-	amdgpu_vm_assert_locked(vm_bo->vm);
-	spin_lock(&vm_bo->vm->status_lock);
-	list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
-	spin_unlock(&vm_bo->vm->status_lock);
+	struct amdgpu_vm_bo_status *lists;
+	struct amdgpu_bo *bo = vm_bo->bo;
+
+	/*
+	 * The root PD doesn't have a parent PDE and goes directly into the
+	 * idle state.
+	 */
+	lists = amdgpu_vm_bo_lock_lists(vm_bo);
+	if (bo && bo->tbo.type == ttm_bo_type_kernel && !bo->parent) {
+		vm_bo->moved = false;
+		list_move(&vm_bo->vm_status, &lists->idle);
+	} else {
+		vm_bo->moved = true;
+		list_move(&vm_bo->vm_status, &lists->moved);
+	}
+	amdgpu_vm_bo_unlock_lists(vm_bo);
 }
 
 /**
@@ -195,120 +243,43 @@ static void amdgpu_vm_bo_moved(struct amdgpu_vm_bo_base *vm_bo)
  *
  * @vm_bo: vm_bo which is now idle
  *
- * State for PDs/PTs and per VM BOs which have gone through the state machine
- * and are now idle.
+ * State for vm_bo objects meaning we are done with the state machine and no
+ * further action is necessary.
  */
 static void amdgpu_vm_bo_idle(struct amdgpu_vm_bo_base *vm_bo)
 {
-	amdgpu_vm_assert_locked(vm_bo->vm);
-	spin_lock(&vm_bo->vm->status_lock);
-	list_move(&vm_bo->vm_status, &vm_bo->vm->idle);
-	spin_unlock(&vm_bo->vm->status_lock);
-	vm_bo->moved = false;
-}
+	struct amdgpu_vm_bo_status *lists;
 
-/**
- * amdgpu_vm_bo_invalidated - vm_bo is invalidated
- *
- * @vm_bo: vm_bo which is now invalidated
- *
- * State for normal BOs which are invalidated and that change not yet reflected
- * in the PTs.
- */
-static void amdgpu_vm_bo_invalidated(struct amdgpu_vm_bo_base *vm_bo)
-{
-	spin_lock(&vm_bo->vm->status_lock);
-	list_move(&vm_bo->vm_status, &vm_bo->vm->invalidated);
-	spin_unlock(&vm_bo->vm->status_lock);
-}
-
-/**
- * amdgpu_vm_bo_evicted_user - vm_bo is evicted
- *
- * @vm_bo: vm_bo which is evicted
- *
- * State for BOs used by user mode queues which are not at the location they
- * should be.
- */
-static void amdgpu_vm_bo_evicted_user(struct amdgpu_vm_bo_base *vm_bo)
-{
-	vm_bo->moved = true;
-	spin_lock(&vm_bo->vm->status_lock);
-	list_move(&vm_bo->vm_status, &vm_bo->vm->evicted_user);
-	spin_unlock(&vm_bo->vm->status_lock);
-}
-
-/**
- * amdgpu_vm_bo_relocated - vm_bo is reloacted
- *
- * @vm_bo: vm_bo which is relocated
- *
- * State for PDs/PTs which needs to update their parent PD.
- * For the root PD, just move to idle state.
- */
-static void amdgpu_vm_bo_relocated(struct amdgpu_vm_bo_base *vm_bo)
-{
-	amdgpu_vm_assert_locked(vm_bo->vm);
-	if (vm_bo->bo->parent) {
-		spin_lock(&vm_bo->vm->status_lock);
-		list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
-		spin_unlock(&vm_bo->vm->status_lock);
-	} else {
-		amdgpu_vm_bo_idle(vm_bo);
-	}
-}
-
-/**
- * amdgpu_vm_bo_done - vm_bo is done
- *
- * @vm_bo: vm_bo which is now done
- *
- * State for normal BOs which are invalidated and that change has been updated
- * in the PTs.
- */
-static void amdgpu_vm_bo_done(struct amdgpu_vm_bo_base *vm_bo)
-{
-	amdgpu_vm_assert_locked(vm_bo->vm);
-	spin_lock(&vm_bo->vm->status_lock);
-	list_move(&vm_bo->vm_status, &vm_bo->vm->done);
-	spin_unlock(&vm_bo->vm->status_lock);
+	lists = amdgpu_vm_bo_lock_lists(vm_bo);
+	if (!amdgpu_vm_is_bo_always_valid(vm_bo->vm, vm_bo->bo))
+		vm_bo->moved = false;
+	list_move(&vm_bo->vm_status, &lists->idle);
+	amdgpu_vm_bo_unlock_lists(vm_bo);
 }
 
 /**
  * amdgpu_vm_bo_reset_state_machine - reset the vm_bo state machine
  * @vm: the VM which state machine to reset
  *
- * Move all vm_bo object in the VM into a state where they will be updated
- * again during validation.
+ * Move all vm_bo object in the VM into a state where their location will be
+ * updated in the page tables again.
  */
 static void amdgpu_vm_bo_reset_state_machine(struct amdgpu_vm *vm)
 {
-	struct amdgpu_vm_bo_base *vm_bo, *tmp;
-
 	amdgpu_vm_assert_locked(vm);
+	list_splice_init(&vm->kernel.idle, &vm->kernel.moved);
+	list_splice_init(&vm->always_valid.idle, &vm->always_valid.moved);
 
-	spin_lock(&vm->status_lock);
-	list_splice_init(&vm->done, &vm->invalidated);
-	list_for_each_entry(vm_bo, &vm->invalidated, vm_status)
-		vm_bo->moved = true;
-
-	list_for_each_entry_safe(vm_bo, tmp, &vm->idle, vm_status) {
-		struct amdgpu_bo *bo = vm_bo->bo;
-
-		vm_bo->moved = true;
-		if (!bo || bo->tbo.type != ttm_bo_type_kernel)
-			list_move(&vm_bo->vm_status, &vm_bo->vm->moved);
-		else if (bo->parent)
-			list_move(&vm_bo->vm_status, &vm_bo->vm->relocated);
-	}
-	spin_unlock(&vm->status_lock);
+	spin_lock(&vm->individual_lock);
+	list_splice_init(&vm->individual.idle, &vm->individual.moved);
+	spin_unlock(&vm->individual_lock);
 }
 
 /**
  * amdgpu_vm_update_shared - helper to update shared memory stat
  * @base: base structure for tracking BO usage in a VM
  *
- * Takes the vm status_lock and updates the shared memory stat. If the basic
+ * Takes the vm stats_lock and updates the shared memory stat. If the basic
  * stat changed (e.g. buffer was moved) amdgpu_vm_update_stats need to be called
  * as well.
  */
@@ -321,7 +292,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
 	bool shared;
 
 	dma_resv_assert_held(bo->tbo.base.resv);
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
 	if (base->shared != shared) {
 		base->shared = shared;
@@ -333,7 +304,7 @@ static void amdgpu_vm_update_shared(struct amdgpu_vm_bo_base *base)
 			vm->stats[bo_memtype].drm.private += size;
 		}
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 }
 
 /**
@@ -358,11 +329,11 @@ void amdgpu_vm_bo_update_shared(struct amdgpu_bo *bo)
  *        be bo->tbo.resource
  * @sign: if we should add (+1) or subtract (-1) from the stat
  *
- * Caller need to have the vm status_lock held. Useful for when multiple update
+ * Caller need to have the vm stats_lock held. Useful for when multiple update
  * need to happen at the same time.
  */
 static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
-			    struct ttm_resource *res, int sign)
+					  struct ttm_resource *res, int sign)
 {
 	struct amdgpu_vm *vm = base->vm;
 	struct amdgpu_bo *bo = base->bo;
@@ -386,7 +357,8 @@ static void amdgpu_vm_update_stats_locked(struct amdgpu_vm_bo_base *base,
 		 */
 		if (bo->flags & AMDGPU_GEM_CREATE_DISCARDABLE)
 			vm->stats[res_memtype].drm.purgeable += size;
-		if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(res_memtype)))
+		if (!(bo->preferred_domains &
+		      amdgpu_mem_type_to_domain(res_memtype)))
 			vm->stats[bo_memtype].evicted += size;
 	}
 }
@@ -405,9 +377,9 @@ void amdgpu_vm_update_stats(struct amdgpu_vm_bo_base *base,
 {
 	struct amdgpu_vm *vm = base->vm;
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	amdgpu_vm_update_stats_locked(base, res, sign);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 }
 
 /**
@@ -428,37 +400,34 @@ void amdgpu_vm_bo_base_init(struct amdgpu_vm_bo_base *base,
 	base->next = NULL;
 	INIT_LIST_HEAD(&base->vm_status);
 
+	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 	if (!bo)
 		return;
+
 	base->next = bo->vm_bo;
 	bo->vm_bo = base;
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	base->shared = drm_gem_object_is_shared_for_memory_stats(&bo->tbo.base);
 	amdgpu_vm_update_stats_locked(base, bo->tbo.resource, +1);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 
-	if (!amdgpu_vm_is_bo_always_valid(vm, bo))
+	if (!amdgpu_vm_is_bo_always_valid(vm, bo)) {
+		amdgpu_vm_bo_idle(base);
 		return;
-
-	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
+	}
 
 	ttm_bo_set_bulk_move(&bo->tbo, &vm->lru_bulk_move);
-	if (bo->tbo.type == ttm_bo_type_kernel && bo->parent)
-		amdgpu_vm_bo_relocated(base);
-	else
-		amdgpu_vm_bo_idle(base);
 
+	/*
+	 * When a per VM isn't in the desired domain put it into the evicted
+	 * state to make sure that it gets validated on the next best occasion.
+	 */
 	if (bo->preferred_domains &
 	    amdgpu_mem_type_to_domain(bo->tbo.resource->mem_type))
-		return;
-
-	/*
-	 * we checked all the prerequisites, but it looks like this per vm bo
-	 * is currently evicted. add the bo to the evicted list to make sure it
-	 * is validated on next vm use to avoid fault.
-	 * */
-	amdgpu_vm_bo_evicted(base);
+		amdgpu_vm_bo_moved(base);
+	else
+		amdgpu_vm_bo_evicted(base);
 }
 
 /**
@@ -479,41 +448,41 @@ int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
 }
 
 /**
- * amdgpu_vm_lock_done_list - lock all BOs on the done list
+ * amdgpu_vm_lock_individual - lock all BOs on the individual idle list
  * @vm: vm providing the BOs
  * @exec: drm execution context
  * @num_fences: number of extra fences to reserve
  *
- * Lock the BOs on the done list in the DRM execution context.
+ * Lock the BOs on the individual idle list in the DRM execution context.
  */
-int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
-			     unsigned int num_fences)
+int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
+			      unsigned int num_fences)
 {
-	struct list_head *prev = &vm->done;
+	struct list_head *prev = &vm->individual.idle;
 	struct amdgpu_bo_va *bo_va;
 	struct amdgpu_bo *bo;
 	int ret;
 
 	/* We can only trust prev->next while holding the lock */
-	spin_lock(&vm->status_lock);
-	while (!list_is_head(prev->next, &vm->done)) {
+	spin_lock(&vm->individual_lock);
+	while (!list_is_head(prev->next, &vm->individual.idle)) {
 		bo_va = list_entry(prev->next, typeof(*bo_va), base.vm_status);
 
 		bo = bo_va->base.bo;
 		if (bo) {
 			amdgpu_bo_ref(bo);
-			spin_unlock(&vm->status_lock);
+			spin_unlock(&vm->individual_lock);
 
 			ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 1);
 			amdgpu_bo_unref(&bo);
 			if (unlikely(ret))
 				return ret;
 
-			spin_lock(&vm->status_lock);
+			spin_lock(&vm->individual_lock);
 		}
 		prev = prev->next;
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->individual_lock);
 
 	return 0;
 }
@@ -609,10 +578,10 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 		       void *param)
 {
 	uint64_t new_vm_generation = amdgpu_vm_generation(adev, vm);
-	struct amdgpu_vm_bo_base *bo_base;
-	struct amdgpu_bo *bo;
+	struct amdgpu_vm_bo_base *bo_base, *tmp;
 	int r;
 
+	dma_resv_assert_held(vm->root.bo->tbo.base.resv);
 	if (vm->generation != new_vm_generation) {
 		vm->generation = new_vm_generation;
 		amdgpu_vm_bo_reset_state_machine(vm);
@@ -622,49 +591,59 @@ int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 			return r;
 	}
 
-	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->evicted)) {
-		bo_base = list_first_entry(&vm->evicted,
-					   struct amdgpu_vm_bo_base,
-					   vm_status);
-		spin_unlock(&vm->status_lock);
+	list_for_each_entry_safe(bo_base, tmp, &vm->kernel.evicted, vm_status) {
+		r = validate(param, bo_base->bo);
+		if (r)
+			return r;
 
-		bo = bo_base->bo;
+		vm->update_funcs->map_table(to_amdgpu_bo_vm(bo_base->bo));
+		amdgpu_vm_bo_moved(bo_base);
+	}
 
-		r = validate(param, bo);
+	/*
+	 * As soon as all page tables are in place we can start updating them
+	 * again.
+	 */
+	amdgpu_vm_eviction_lock(vm);
+	vm->evicting = false;
+	amdgpu_vm_eviction_unlock(vm);
+
+	list_for_each_entry_safe(bo_base, tmp, &vm->always_valid.evicted,
+				 vm_status) {
+		r = validate(param, bo_base->bo);
 		if (r)
 			return r;
 
-		if (bo->tbo.type != ttm_bo_type_kernel) {
-			amdgpu_vm_bo_moved(bo_base);
-		} else {
-			vm->update_funcs->map_table(to_amdgpu_bo_vm(bo));
-			amdgpu_vm_bo_relocated(bo_base);
-		}
-		spin_lock(&vm->status_lock);
+		amdgpu_vm_bo_moved(bo_base);
 	}
-	while (ticket && !list_empty(&vm->evicted_user)) {
-		bo_base = list_first_entry(&vm->evicted_user,
-					   struct amdgpu_vm_bo_base,
-					   vm_status);
-		spin_unlock(&vm->status_lock);
 
-		bo = bo_base->bo;
-		dma_resv_assert_held(bo->tbo.base.resv);
+	if (!ticket)
+		return 0;
+
+	spin_lock(&vm->individual_lock);
+restart:
+	list_for_each_entry(bo_base, &vm->individual.evicted, vm_status) {
+		struct amdgpu_bo *bo = bo_base->bo;
+
+		if (dma_resv_locking_ctx(bo->tbo.base.resv) != ticket)
+			continue;
+
+		spin_unlock(&vm->individual_lock);
 
 		r = validate(param, bo);
 		if (r)
 			return r;
 
-		amdgpu_vm_bo_invalidated(bo_base);
+		amdgpu_vm_bo_moved(bo_base);
 
-		spin_lock(&vm->status_lock);
+		/* It's a bit inefficient to always jump back to the start, but
+		 * we would need to re-structure the KFD for properly fixing
+		 * that.
+		 */
+		spin_lock(&vm->individual_lock);
+		goto restart;
 	}
-	spin_unlock(&vm->status_lock);
-
-	amdgpu_vm_eviction_lock(vm);
-	vm->evicting = false;
-	amdgpu_vm_eviction_unlock(vm);
+	spin_unlock(&vm->individual_lock);
 
 	return 0;
 }
@@ -689,9 +668,7 @@ bool amdgpu_vm_ready(struct amdgpu_vm *vm)
 	ret = !vm->evicting;
 	amdgpu_vm_eviction_unlock(vm);
 
-	spin_lock(&vm->status_lock);
-	ret &= list_empty(&vm->evicted);
-	spin_unlock(&vm->status_lock);
+	ret &= list_empty(&vm->kernel.evicted);
 
 	spin_lock(&vm->immediate.lock);
 	ret &= !vm->immediate.stopped;
@@ -985,18 +962,13 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 			  struct amdgpu_vm *vm, bool immediate)
 {
 	struct amdgpu_vm_update_params params;
-	struct amdgpu_vm_bo_base *entry;
+	struct amdgpu_vm_bo_base *entry, *tmp;
 	bool flush_tlb_needed = false;
-	LIST_HEAD(relocated);
 	int r, idx;
 
 	amdgpu_vm_assert_locked(vm);
 
-	spin_lock(&vm->status_lock);
-	list_splice_init(&vm->relocated, &relocated);
-	spin_unlock(&vm->status_lock);
-
-	if (list_empty(&relocated))
+	if (list_empty(&vm->kernel.moved))
 		return 0;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx))
@@ -1012,7 +984,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (r)
 		goto error;
 
-	list_for_each_entry(entry, &relocated, vm_status) {
+	list_for_each_entry(entry, &vm->kernel.moved, vm_status) {
 		/* vm_flush_needed after updating moved PDEs */
 		flush_tlb_needed |= entry->moved;
 
@@ -1028,11 +1000,8 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (flush_tlb_needed)
 		atomic64_inc(&vm->tlb_seq);
 
-	while (!list_empty(&relocated)) {
-		entry = list_first_entry(&relocated, struct amdgpu_vm_bo_base,
-					 vm_status);
+	list_for_each_entry_safe(entry, tmp, &vm->kernel.moved, vm_status)
 		amdgpu_vm_bo_idle(entry);
-	}
 
 error:
 	drm_dev_exit(idx);
@@ -1163,7 +1132,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	params.pages_addr = pages_addr;
 	params.unlocked = unlocked;
 	params.needs_flush = flush_tlb;
-	params.allow_override = allow_override;
+	params.override_pte = allow_override && adev->gmc.override_pte;
 	INIT_LIST_HEAD(&params.tlb_flush_waitlist);
 
 	amdgpu_vm_eviction_lock(vm);
@@ -1260,9 +1229,9 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 void amdgpu_vm_get_memory(struct amdgpu_vm *vm,
 			  struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM])
 {
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->stats_lock);
 	memcpy(stats, vm->stats, sizeof(*stats) * __AMDGPU_PL_NUM);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->stats_lock);
 }
 
 /**
@@ -1406,7 +1375,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev, struct amdgpu_bo_va *bo_va,
 		else
 			amdgpu_vm_bo_idle(&bo_va->base);
 	} else {
-		amdgpu_vm_bo_done(&bo_va->base);
+		amdgpu_vm_bo_idle(&bo_va->base);
 	}
 
 	list_splice_init(&bo_va->invalids, &bo_va->valids);
@@ -1629,29 +1598,25 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 			   struct amdgpu_vm *vm,
 			   struct ww_acquire_ctx *ticket)
 {
-	struct amdgpu_bo_va *bo_va;
+	struct amdgpu_bo_va *bo_va, *tmp;
 	struct dma_resv *resv;
 	bool clear, unlock;
 	int r;
 
-	spin_lock(&vm->status_lock);
-	while (!list_empty(&vm->moved)) {
-		bo_va = list_first_entry(&vm->moved, struct amdgpu_bo_va,
-					 base.vm_status);
-		spin_unlock(&vm->status_lock);
-
+	list_for_each_entry_safe(bo_va, tmp, &vm->always_valid.moved,
+				 base.vm_status) {
 		/* Per VM BOs never need to bo cleared in the page tables */
 		r = amdgpu_vm_bo_update(adev, bo_va, false);
 		if (r)
 			return r;
-		spin_lock(&vm->status_lock);
 	}
 
-	while (!list_empty(&vm->invalidated)) {
-		bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
-					 base.vm_status);
+	spin_lock(&vm->individual_lock);
+	while (!list_empty(&vm->individual.moved)) {
+		bo_va = list_first_entry(&vm->individual.moved,
+					 typeof(*bo_va), base.vm_status);
 		resv = bo_va->base.bo->tbo.base.resv;
-		spin_unlock(&vm->status_lock);
+		spin_unlock(&vm->individual_lock);
 
 		/* Try to reserve the BO to avoid clearing its ptes */
 		if (!adev->debug_vm && dma_resv_trylock(resv)) {
@@ -1681,11 +1646,11 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 		    drm_gem_is_imported(&bo_va->base.bo->tbo.base) &&
 		    (!bo_va->base.bo->tbo.resource ||
 		     bo_va->base.bo->tbo.resource->mem_type == TTM_PL_SYSTEM))
-			amdgpu_vm_bo_evicted_user(&bo_va->base);
+			amdgpu_vm_bo_evicted(&bo_va->base);
 
-		spin_lock(&vm->status_lock);
+		spin_lock(&vm->individual_lock);
 	}
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->individual_lock);
 
 	return 0;
 }
@@ -2002,7 +1967,7 @@ int amdgpu_vm_bo_unmap(struct amdgpu_device *adev,
 	 * during user requests GEM unmap IOCTL except for forcing the unmap
 	 * from user space.
 	 */
-	if (unlikely(atomic_read(&bo_va->userq_va_mapped) > 0))
+	if (unlikely(bo_va->userq_va_mapped))
 		amdgpu_userq_gem_va_unmap_validate(adev, mapping, saddr);
 
 	list_del(&mapping->list);
@@ -2223,9 +2188,9 @@ void amdgpu_vm_bo_del(struct amdgpu_device *adev,
 		}
 	}
 
-	spin_lock(&vm->status_lock);
+	spin_lock(&vm->individual_lock);
 	list_del(&bo_va->base.vm_status);
-	spin_unlock(&vm->status_lock);
+	spin_unlock(&vm->individual_lock);
 
 	list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
 		list_del(&mapping->list);
@@ -2305,14 +2270,7 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_bo *bo, bool evicted)
 
 		if (bo_base->moved)
 			continue;
-		bo_base->moved = true;
-
-		if (bo->tbo.type == ttm_bo_type_kernel)
-			amdgpu_vm_bo_relocated(bo_base);
-		else if (amdgpu_vm_is_bo_always_valid(vm, bo))
-			amdgpu_vm_bo_moved(bo_base);
-		else
-			amdgpu_vm_bo_invalidated(bo_base);
+		amdgpu_vm_bo_moved(bo_base);
 	}
 }
 
@@ -2333,10 +2291,10 @@ void amdgpu_vm_bo_move(struct amdgpu_bo *bo, struct ttm_resource *new_mem,
 	for (bo_base = bo->vm_bo; bo_base; bo_base = bo_base->next) {
 		struct amdgpu_vm *vm = bo_base->vm;
 
-		spin_lock(&vm->status_lock);
+		spin_lock(&vm->stats_lock);
 		amdgpu_vm_update_stats_locked(bo_base, bo->tbo.resource, -1);
 		amdgpu_vm_update_stats_locked(bo_base, new_mem, +1);
-		spin_unlock(&vm->status_lock);
+		spin_unlock(&vm->stats_lock);
 	}
 
 	amdgpu_vm_bo_invalidate(bo, evicted);
@@ -2600,16 +2558,14 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	vm->va = RB_ROOT_CACHED;
 	for (i = 0; i < AMDGPU_MAX_VMHUBS; i++)
 		vm->reserved_vmid[i] = NULL;
-	INIT_LIST_HEAD(&vm->evicted);
-	INIT_LIST_HEAD(&vm->evicted_user);
-	INIT_LIST_HEAD(&vm->relocated);
-	INIT_LIST_HEAD(&vm->moved);
-	INIT_LIST_HEAD(&vm->idle);
-	INIT_LIST_HEAD(&vm->invalidated);
-	spin_lock_init(&vm->status_lock);
+
+	amdgpu_vm_bo_status_init(&vm->kernel);
+	amdgpu_vm_bo_status_init(&vm->always_valid);
+	spin_lock_init(&vm->individual_lock);
+	amdgpu_vm_bo_status_init(&vm->individual);
 	INIT_LIST_HEAD(&vm->freed);
-	INIT_LIST_HEAD(&vm->done);
 	INIT_KFIFO(vm->faults);
+	spin_lock_init(&vm->stats_lock);
 
 	r = amdgpu_vm_init_entities(adev, vm);
 	if (r)
@@ -3086,100 +3042,64 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
 }
 
 #if defined(CONFIG_DEBUG_FS)
-/**
- * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
- *
- * @vm: Requested VM for printing BO info
- * @m: debugfs file
- *
- * Print BO information in debugfs file for the VM
- */
-void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
-{
-	struct amdgpu_bo_va *bo_va, *tmp;
-	u64 total_idle = 0;
-	u64 total_evicted = 0;
-	u64 total_relocated = 0;
-	u64 total_moved = 0;
-	u64 total_invalidated = 0;
-	u64 total_done = 0;
-	unsigned int total_idle_objs = 0;
-	unsigned int total_evicted_objs = 0;
-	unsigned int total_relocated_objs = 0;
-	unsigned int total_moved_objs = 0;
-	unsigned int total_invalidated_objs = 0;
-	unsigned int total_done_objs = 0;
-	unsigned int id = 0;
 
-	amdgpu_vm_assert_locked(vm);
+/* print the debug info for a specific set of status lists */
+static void amdgpu_debugfs_vm_bo_status_info(struct seq_file *m,
+					     struct amdgpu_vm_bo_status *lists)
+{
+	struct amdgpu_vm_bo_base *base;
+	unsigned int id;
 
-	spin_lock(&vm->status_lock);
-	seq_puts(m, "\tIdle BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->idle, base.vm_status) {
-		if (!bo_va->base.bo)
-			continue;
-		total_idle += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
-	}
-	total_idle_objs = id;
 	id = 0;
-
 	seq_puts(m, "\tEvicted BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->evicted, base.vm_status) {
-		if (!bo_va->base.bo)
+	list_for_each_entry(base, &lists->evicted, vm_status) {
+		if (!base->bo)
 			continue;
-		total_evicted += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
-	}
-	total_evicted_objs = id;
-	id = 0;
 
-	seq_puts(m, "\tRelocated BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->relocated, base.vm_status) {
-		if (!bo_va->base.bo)
-			continue;
-		total_relocated += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
+		amdgpu_bo_print_info(id++, base->bo, m);
 	}
-	total_relocated_objs = id;
-	id = 0;
 
+	id = 0;
 	seq_puts(m, "\tMoved BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
-		if (!bo_va->base.bo)
+	list_for_each_entry(base, &lists->moved, vm_status) {
+		if (!base->bo)
 			continue;
-		total_moved += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
+
+		amdgpu_bo_print_info(id++, base->bo, m);
 	}
-	total_moved_objs = id;
-	id = 0;
 
-	seq_puts(m, "\tInvalidated BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->invalidated, base.vm_status) {
-		if (!bo_va->base.bo)
+	id = 0;
+	seq_puts(m, "\tIdle BOs:\n");
+	list_for_each_entry(base, &lists->moved, vm_status) {
+		if (!base->bo)
 			continue;
-		total_invalidated += amdgpu_bo_print_info(id++,	bo_va->base.bo, m);
+
+		amdgpu_bo_print_info(id++, base->bo, m);
 	}
-	total_invalidated_objs = id;
-	id = 0;
+}
 
-	seq_puts(m, "\tDone BOs:\n");
-	list_for_each_entry_safe(bo_va, tmp, &vm->done, base.vm_status) {
-		if (!bo_va->base.bo)
-			continue;
-		total_done += amdgpu_bo_print_info(id++, bo_va->base.bo, m);
-	}
-	spin_unlock(&vm->status_lock);
-	total_done_objs = id;
-
-	seq_printf(m, "\tTotal idle size:        %12lld\tobjs:\t%d\n", total_idle,
-		   total_idle_objs);
-	seq_printf(m, "\tTotal evicted size:     %12lld\tobjs:\t%d\n", total_evicted,
-		   total_evicted_objs);
-	seq_printf(m, "\tTotal relocated size:   %12lld\tobjs:\t%d\n", total_relocated,
-		   total_relocated_objs);
-	seq_printf(m, "\tTotal moved size:       %12lld\tobjs:\t%d\n", total_moved,
-		   total_moved_objs);
-	seq_printf(m, "\tTotal invalidated size: %12lld\tobjs:\t%d\n", total_invalidated,
-		   total_invalidated_objs);
-	seq_printf(m, "\tTotal done size:        %12lld\tobjs:\t%d\n", total_done,
-		   total_done_objs);
+/**
+ * amdgpu_debugfs_vm_bo_info  - print BO info for the VM
+ *
+ * @vm: Requested VM for printing BO info
+ * @m: debugfs file
+ *
+ * Print BO information in debugfs file for the VM
+ */
+void amdgpu_debugfs_vm_bo_info(struct amdgpu_vm *vm, struct seq_file *m)
+{
+	amdgpu_vm_assert_locked(vm);
+
+	seq_puts(m, "\tKernel PT/PDs:\n");
+	amdgpu_debugfs_vm_bo_status_info(m, &vm->kernel);
+
+	seq_puts(m, "\tPer VM BOs:\n");
+	amdgpu_debugfs_vm_bo_status_info(m, &vm->always_valid);
+
+	seq_puts(m, "\tIndividual BOs:\n");
+	spin_lock(&vm->individual_lock);
+	amdgpu_debugfs_vm_bo_status_info(m, &vm->individual);
+	spin_unlock(&vm->individual_lock);
 }
 #endif
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index d083d7aab75c6..3695299f1a039 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -205,17 +205,34 @@ struct amdgpu_vm_bo_base {
 	/* protected by bo being reserved */
 	struct amdgpu_vm_bo_base	*next;
 
-	/* protected by vm status_lock */
+	/* protected by vm reservation and invalidated_lock */
 	struct list_head		vm_status;
 
 	/* if the bo is counted as shared in mem stats
-	 * protected by vm status_lock */
+	 * protected by vm BO being reserved */
 	bool				shared;
 
 	/* protected by the BO being reserved */
 	bool				moved;
 };
 
+/*
+ * The following status lists contain amdgpu_vm_bo_base objects for
+ * either PD/PTs, per VM BOs or BOs with individual resv object.
+ *
+ * The state transits are: evicted -> moved -> idle
+ */
+struct amdgpu_vm_bo_status {
+	/* BOs evicted which need to move into place again */
+	struct list_head		evicted;
+
+	/* BOs which moved but new location hasn't been updated in the PDs/PTs */
+	struct list_head		moved;
+
+	/* BOs done with the state machine and need no further action */
+	struct list_head		idle;
+};
+
 /* provided by hw blocks that can write ptes, e.g., sdma */
 struct amdgpu_vm_pte_funcs {
 	/* number of dw to reserve per operation */
@@ -296,10 +313,10 @@ struct amdgpu_vm_update_params {
 	bool needs_flush;
 
 	/**
-	 * @allow_override: true for memory that is not uncached: allows MTYPE
-	 * to be overridden for NUMA local memory.
+	 * @override_pte: true for memory that is not uncached and gmc override function is
+	 * implemented to allow MTYPE to be overridden for NUMA local memory.
 	 */
-	bool allow_override;
+	bool override_pte;
 
 	/**
 	 * @tlb_flush_waitlist: temporary storage for BOs until tlb_flush
@@ -345,47 +362,29 @@ struct amdgpu_vm {
 	bool			evicting;
 	unsigned int		saved_flags;
 
-	/* Lock to protect vm_bo add/del/move on all lists of vm */
-	spinlock_t		status_lock;
-
-	/* Memory statistics for this vm, protected by status_lock */
+	/* Memory statistics for this vm, protected by stats_lock */
+	spinlock_t		stats_lock;
 	struct amdgpu_mem_stats stats[__AMDGPU_PL_NUM];
 
+	/* BO's belonging to PD/PT which are internal to the kernel. */
+	struct amdgpu_vm_bo_status	kernel;
+
 	/*
-	 * The following lists contain amdgpu_vm_bo_base objects for either
-	 * PDs, PTs or per VM BOs. The state transits are:
-	 *
-	 * evicted -> relocated (PDs, PTs) or moved (per VM BOs) -> idle
+	 * BOs allocated by userspace where the dma_resv is shared with the
+	 * root PD
 	 */
-
-	/* Per-VM and PT BOs who needs a validation */
-	struct list_head	evicted;
-
-	/* PT BOs which relocated and their parent need an update */
-	struct list_head	relocated;
-
-	/* per VM BOs moved, but not yet updated in the PT */
-	struct list_head	moved;
-
-	/* All BOs of this VM not currently in the state machine */
-	struct list_head	idle;
+	struct amdgpu_vm_bo_status	always_valid;
 
 	/*
 	 * The following lists contain amdgpu_vm_bo_base objects for BOs which
-	 * have their own dma_resv object and not depend on the root PD. Their
-	 * state transits are:
+	 * have their own dma_resv object and not depend on the root PD.
 	 *
-	 * evicted_user or invalidated -> done
+	 * Lists are protected by the individual_lock.
 	 */
+	spinlock_t			individual_lock;
 
-	/* BOs for user mode queues that need a validation */
-	struct list_head	evicted_user;
-
-	/* regular invalidated BOs, but not yet updated in the PT */
-	struct list_head	invalidated;
-
-	/* BOs which are invalidated, has been updated in the PTs */
-	struct list_head        done;
+	/* Userspace BOs with individual resv object */
+	struct amdgpu_vm_bo_status	individual;
 
 	/*
 	 * This list contains amdgpu_bo_va_mapping objects which have been freed
@@ -507,8 +506,8 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 int amdgpu_vm_lock_pd(struct amdgpu_vm *vm, struct drm_exec *exec,
 		      unsigned int num_fences);
-int amdgpu_vm_lock_done_list(struct amdgpu_vm *vm, struct drm_exec *exec,
-			     unsigned int num_fences);
+int amdgpu_vm_lock_individual(struct amdgpu_vm *vm, struct drm_exec *exec,
+			      unsigned int num_fences);
 bool amdgpu_vm_ready(struct amdgpu_vm *vm);
 uint64_t amdgpu_vm_generation(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 int amdgpu_vm_validate(struct amdgpu_device *adev, struct amdgpu_vm *vm,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
index f078db3fef79e..b31ff6f56f0d6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_cpu.c
@@ -88,12 +88,21 @@ static int amdgpu_vm_cpu_update(struct amdgpu_vm_update_params *p,
 
 	trace_amdgpu_vm_set_ptes(pe, addr, count, incr, flags, p->immediate);
 
+	if (!p->pages_addr && p->override_pte)
+		amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, addr, &flags);
+
 	for (i = 0; i < count; i++) {
+		u64 oflags = flags;
+
 		value = p->pages_addr ?
 			amdgpu_vm_map_gart(p->pages_addr, addr) :
 			addr;
+
+		if (p->pages_addr && p->override_pte)
+			amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, value, &oflags);
+
 		amdgpu_gmc_set_pte_pde(p->adev, (void *)(uintptr_t)pe,
-				       i, value, flags);
+				       i, value, oflags);
 		addr += incr;
 	}
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index a930f1522f962..e43a60d098082 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -544,9 +544,7 @@ static void amdgpu_vm_pt_free(struct amdgpu_vm_bo_base *entry)
 	entry->bo->vm_bo = NULL;
 	ttm_bo_set_bulk_move(&entry->bo->tbo, NULL);
 
-	spin_lock(&entry->vm->status_lock);
 	list_del(&entry->vm_status);
-	spin_unlock(&entry->vm->status_lock);
 	amdgpu_bo_unref(&entry->bo);
 }
 
@@ -590,7 +588,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
 	struct amdgpu_vm_pt_cursor seek;
 	struct amdgpu_vm_bo_base *entry;
 
-	spin_lock(&params->vm->status_lock);
 	for_each_amdgpu_vm_pt_dfs_safe(params->adev, params->vm, cursor, seek, entry) {
 		if (entry && entry->bo)
 			list_move(&entry->vm_status, &params->tlb_flush_waitlist);
@@ -598,7 +595,6 @@ static void amdgpu_vm_pt_add_list(struct amdgpu_vm_update_params *params,
 
 	/* enter start node now */
 	list_move(&cursor->entry->vm_status, &params->tlb_flush_waitlist);
-	spin_unlock(&params->vm->status_lock);
 }
 
 /**
@@ -710,15 +706,6 @@ static void amdgpu_vm_pte_update_flags(struct amdgpu_vm_update_params *params,
 	if (level == AMDGPU_VM_PTB)
 		amdgpu_vm_pte_update_noretry_flags(adev, &flags);
 
-	/* APUs mapping system memory may need different MTYPEs on different
-	 * NUMA nodes. Only do this for contiguous ranges that can be assumed
-	 * to be on the same NUMA node.
-	 */
-	if ((flags & AMDGPU_PTE_SYSTEM) && (adev->flags & AMD_IS_APU) &&
-	    adev->gmc.gmc_funcs->override_vm_pte_flags &&
-	    num_possible_nodes() > 1 && !params->pages_addr && params->allow_override)
-		amdgpu_gmc_override_vm_pte_flags(adev, params->vm, addr, &flags);
-
 	params->vm->update_funcs->update(params, pt, pe, addr, count, incr,
 					 flags);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
index 36805dcfa1598..fd09a2b5a147f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_sdma.c
@@ -106,13 +106,13 @@ static int amdgpu_vm_sdma_prepare(struct amdgpu_vm_update_params *p,
 static int amdgpu_vm_sdma_commit(struct amdgpu_vm_update_params *p,
 				 struct dma_fence **fence)
 {
+	struct drm_gpu_scheduler *sched =
+		container_of(p->vm->delayed.rq, typeof(*sched), rq);
+	struct amdgpu_ring *ring =
+		container_of(sched, struct amdgpu_ring, sched);
 	struct amdgpu_ib *ib = p->job->ibs;
-	struct amdgpu_ring *ring;
 	struct dma_fence *f;
 
-	ring = container_of(p->vm->delayed.rq->sched, struct amdgpu_ring,
-			    sched);
-
 	WARN_ON(ib->length_dw == 0);
 	amdgpu_ring_pad_ib(ring, ib);
 
@@ -257,6 +257,9 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
 		}
 
 		if (!p->pages_addr) {
+			if (p->override_pte)
+				amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, addr, &flags);
+
 			/* set page commands needed */
 			amdgpu_vm_sdma_set_ptes(p, bo, pe, addr, count,
 						incr, flags);
@@ -275,8 +278,14 @@ static int amdgpu_vm_sdma_update(struct amdgpu_vm_update_params *p,
 		p->num_dw_left -= nptes * 2;
 		pte = (uint64_t *)&(p->job->ibs->ptr[p->num_dw_left]);
 		for (i = 0; i < nptes; ++i, addr += incr) {
+			u64 oflags = flags;
+
 			pte[i] = amdgpu_vm_map_gart(p->pages_addr, addr);
-			pte[i] |= flags;
+
+			if (p->override_pte)
+				amdgpu_gmc_override_vm_pte_flags(p->adev, p->vm, pte[i], &oflags);
+
+			pte[i] |= oflags;
 		}
 
 		amdgpu_vm_sdma_copy_ptes(p, bo, pe, nptes);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
index fd881388d6125..6014204914774 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.c
@@ -29,6 +29,7 @@
 #include "amdgpu_smu.h"
 #include "soc15_common.h"
 #include "vpe_v6_1.h"
+#include "vpe_v2_0.h"
 
 #define AMDGPU_CSA_VPE_SIZE 	64
 /* VPE CSA resides in the 4th page of CSA */
@@ -310,6 +311,9 @@ static int vpe_early_init(struct amdgpu_ip_block *ip_block)
 		vpe_v6_1_set_funcs(vpe);
 		vpe->collaborate_mode = true;
 		break;
+	case IP_VERSION(2, 0, 0):
+		vpe_v2_0_set_funcs(vpe);
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -562,6 +566,11 @@ static void vpe_ring_emit_fence(struct amdgpu_ring *ring, uint64_t addr,
 		amdgpu_ring_write(ring, 0);
 	}
 
+	/* WA: Force sync after TRAP to avoid VPE1 fail to power off */
+	if (ring->adev->vpe.collaborate_mode) {
+		amdgpu_ring_write(ring, VPE_CMD_HEADER(VPE_CMD_OPCODE_COLLAB_SYNC, 0));
+		amdgpu_ring_write(ring, 0xabcd);
+	}
 }
 
 static void vpe_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
@@ -968,7 +977,7 @@ static const struct amdgpu_ring_funcs vpe_ring_funcs = {
 	.emit_frame_size =
 		5 + /* vpe_ring_init_cond_exec */
 		6 + /* vpe_ring_emit_pipeline_sync */
-		10 + 10 + 10 + /* vpe_ring_emit_fence */
+		12 + 12 + 12 + /* vpe_ring_emit_fence */
 		/* vpe_ring_emit_vm_flush */
 		SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 +
 		SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 6,
@@ -1009,6 +1018,19 @@ const struct amd_ip_funcs vpe_ip_funcs = {
 	.set_powergating_state = vpe_set_powergating_state,
 };
 
+const struct amd_ip_funcs vpe2_ip_funcs = {
+	.name = "vpe_v2_0",
+	.early_init = vpe_early_init,
+	.sw_init = vpe_sw_init,
+	.sw_fini = vpe_sw_fini,
+	.hw_init = vpe_hw_init,
+	.hw_fini = vpe_hw_fini,
+	.suspend = vpe_suspend,
+	.resume = vpe_resume,
+	.set_clockgating_state = vpe_set_clockgating_state,
+	.set_powergating_state = vpe_set_powergating_state,
+};
+
 const struct amdgpu_ip_block_version vpe_v6_1_ip_block = {
 	.type = AMD_IP_BLOCK_TYPE_VPE,
 	.major = 6,
@@ -1016,3 +1038,11 @@ const struct amdgpu_ip_block_version vpe_v6_1_ip_block = {
 	.rev = 0,
 	.funcs = &vpe_ip_funcs,
 };
+
+const struct amdgpu_ip_block_version vpe_v2_0_ip_block = {
+	.type = AMD_IP_BLOCK_TYPE_VPE,
+	.major = 2,
+	.minor = 0,
+	.rev = 0,
+	.funcs = &vpe2_ip_funcs,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.h
index 695da740a97ea..5c9a9f59a02b1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vpe.h
@@ -107,5 +107,6 @@ int amdgpu_vpe_sysfs_reset_mask_init(struct amdgpu_device *adev);
 		((vpe)->funcs->load_microcode ? (vpe)->funcs->load_microcode((vpe)) : 0)
 
 extern const struct amdgpu_ip_block_version vpe_v6_1_ip_block;
+extern const struct amdgpu_ip_block_version vpe_v2_0_ip_block;
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
index 42be8ee155dde..409e103ffe8c7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_xcp.c
@@ -466,15 +466,15 @@ int amdgpu_xcp_open_device(struct amdgpu_device *adev,
 void amdgpu_xcp_release_sched(struct amdgpu_device *adev,
 				  struct amdgpu_ctx_entity *entity)
 {
-	struct drm_gpu_scheduler *sched;
-	struct amdgpu_ring *ring;
+	struct drm_gpu_scheduler *sched =
+		container_of(entity->entity.rq, typeof(*sched), rq);
 
 	if (!adev->xcp_mgr)
 		return;
 
-	sched = entity->entity.rq->sched;
 	if (drm_sched_wqueue_ready(sched)) {
-		ring = to_amdgpu_ring(entity->entity.rq->sched);
+		struct amdgpu_ring *ring = to_amdgpu_ring(sched);
+
 		atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt);
 	}
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
index 847cfd1fd0046..d80f01c0e754d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h
@@ -162,7 +162,9 @@ union amd_sriov_msg_feature_flags {
 		uint32_t ras_cper		: 1;
 		uint32_t xgmi_ta_ext_peer_link	: 1;
 		uint32_t xgmi_connected_to_cpu  : 1;
-		uint32_t reserved		: 18;
+		uint32_t ptl_support		: 1;
+		uint32_t unitid_support		: 1;
+		uint32_t reserved		: 16;
 	} flags;
 	uint32_t all;
 };
@@ -256,7 +258,7 @@ struct amd_sriov_msg_pf2vf_info_header {
 	uint32_t reserved[2];
 };
 
-#define AMD_SRIOV_MSG_PF2VF_INFO_FILLED_SIZE (55)
+#define AMD_SRIOV_MSG_PF2VF_INFO_FILLED_SIZE (59)
 struct amd_sriov_msg_pf2vf_info {
 	/* header contains size and version */
 	struct amd_sriov_msg_pf2vf_info_header header;
@@ -293,7 +295,7 @@ struct amd_sriov_msg_pf2vf_info {
 	uint32_t vf2pf_update_interval_ms;
 	/* identification in ROCm SMI */
 	uint64_t uuid;
-	uint32_t fcn_idx;
+	uint32_t pad;
 	/* flags to indicate which register access method VF should use */
 	union amd_sriov_reg_access_flags reg_access_flags;
 	/* MM BW management */
@@ -314,6 +316,13 @@ struct amd_sriov_msg_pf2vf_info {
 	uint32_t more_bp;	//Reserved for future use.
 	union amd_sriov_ras_caps ras_en_caps;
 	union amd_sriov_ras_caps ras_telemetry_en_caps;
+	/* PTL status response for guest */
+	uint32_t ptl_enabled;        // PTL enable status: 0=disabled, 1=enabled
+	uint32_t ptl_pref_format1;   // Current preferred format 1
+	uint32_t ptl_pref_format2;   // Current preferred format 2
+	/* unit ID assigned by host; vf_idx [0..254] maps to unitid [1..255] (0 = pf) */
+	uint8_t unitid;
+	uint8_t padding[3];  //use the 3 bytes to align
 
 	/* reserved */
 	uint32_t reserved[256 - AMD_SRIOV_MSG_PF2VF_INFO_FILLED_SIZE];
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.c b/drivers/gpu/drm/amd/amdgpu/atom.c
index 6e37961f6be53..ca5d091549e1f 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.c
+++ b/drivers/gpu/drm/amd/amdgpu/atom.c
@@ -59,6 +59,9 @@
 
 #define ATOM_CMD_TIMEOUT_SEC	20
 
+/* Limit ATOM command table recursion (calltable) to avoid kernel stack overflow. */
+#define ATOM_EXECUTE_MAX_DEPTH	32
+
 typedef struct {
 	struct atom_context *ctx;
 	uint32_t *ps, *ws;
@@ -1229,6 +1232,13 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index,
 	if (!base)
 		return -EINVAL;
 
+	if (ctx->execute_depth >= ATOM_EXECUTE_MAX_DEPTH) {
+		DRM_ERROR("atombios command table nesting exceeded limit (%u)\n",
+			  ATOM_EXECUTE_MAX_DEPTH);
+		return -ELOOP;
+	}
+	ctx->execute_depth++;
+
 	len = CU16(base + ATOM_CT_SIZE_PTR);
 	ws = CU8(base + ATOM_CT_WS_PTR);
 	ps = CU8(base + ATOM_CT_PS_PTR) & ATOM_CT_PS_MASK;
@@ -1285,6 +1295,7 @@ static int amdgpu_atom_execute_table_locked(struct atom_context *ctx, int index,
 free:
 	if (ws)
 		kfree(ectx.ws);
+	ctx->execute_depth--;
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/atom.h b/drivers/gpu/drm/amd/amdgpu/atom.h
index 825ff28731f52..bb3d9eb7eb6b5 100644
--- a/drivers/gpu/drm/amd/amdgpu/atom.h
+++ b/drivers/gpu/drm/amd/amdgpu/atom.h
@@ -153,6 +153,9 @@ struct atom_context {
 	uint8_t vbios_ver_str[STRLEN_NORMAL];
 	uint8_t date[STRLEN_NORMAL];
 	uint8_t build_num[STRLEN_NORMAL];
+
+	/* Nesting depth for ATOM_OP_CALLTABLE */
+	unsigned int execute_depth;
 };
 
 extern int amdgpu_atom_debug;
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 22780c09177d8..120da838ac282 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -939,7 +939,6 @@ static int cik_sdma_early_init(struct amdgpu_ip_block *ip_block)
 
 	cik_sdma_set_ring_funcs(adev);
 	cik_sdma_set_irq_funcs(adev);
-	cik_sdma_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &cik_sdma_vm_pte_funcs);
 
 	return 0;
@@ -1000,8 +999,15 @@ static int cik_sdma_sw_fini(struct amdgpu_ip_block *ip_block)
 static int cik_sdma_hw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
+	int r;
+
+	r = cik_sdma_start(adev);
+	if (r)
+		return r;
+
+	cik_sdma_set_buffer_funcs(adev);
 
-	return cik_sdma_start(adev);
+	return 0;
 }
 
 static int cik_sdma_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -1340,8 +1346,7 @@ static const struct amdgpu_buffer_funcs cik_sdma_buffer_funcs = {
 
 static void cik_sdma_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	adev->mman.buffer_funcs = &cik_sdma_buffer_funcs;
-	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &cik_sdma_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version cik_sdma_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
index 8b60299b73ef7..58c69dcb527f7 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v10_0.c
@@ -7011,6 +7011,11 @@ static int gfx_v10_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
 	tmp = REG_SET_FIELD(tmp, CP_HQD_IB_CONTROL, MIN_IB_AVAIL_SIZE, 3);
 	mqd->cp_hqd_ib_control = tmp;
 
+	tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
+	mqd->cp_hqd_quantum = tmp;
+
 	/* set static priority for a compute queue/ring */
 	mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
 	mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
index d40ab1e954806..fabdbbd0abb7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v11_0.c
@@ -4417,6 +4417,11 @@ static int gfx_v11_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
 	mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
 	mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
 
+	tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
+	mqd->cp_hqd_quantum = tmp;
+
 	mqd->cp_hqd_active = prop->hqd_active;
 
 	/* set UQ fenceaddress */
@@ -5219,7 +5224,7 @@ static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block)
 	/**
 	 * GFX soft reset will impact MES, need resume MES when do GFX soft reset
 	 */
-	return amdgpu_mes_resume(adev);
+	return amdgpu_mes_resume(adev, 0);
 }
 
 static uint64_t gfx_v11_0_get_gpu_clock_counter(struct amdgpu_device *adev)
@@ -6227,56 +6232,6 @@ static void gfx_v11_0_ring_emit_gfx_shadow(struct amdgpu_ring *ring,
 	ring->set_q_mode_offs = offs;
 }
 
-static int gfx_v11_0_ring_preempt_ib(struct amdgpu_ring *ring)
-{
-	int i, r = 0;
-	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
-	struct amdgpu_ring *kiq_ring = &kiq->ring;
-	unsigned long flags;
-
-	if (adev->enable_mes)
-		return -EINVAL;
-
-	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
-		return -EINVAL;
-
-	spin_lock_irqsave(&kiq->ring_lock, flags);
-
-	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
-		spin_unlock_irqrestore(&kiq->ring_lock, flags);
-		return -ENOMEM;
-	}
-
-	/* assert preemption condition */
-	amdgpu_ring_set_preempt_cond_exec(ring, false);
-
-	/* assert IB preemption, emit the trailing fence */
-	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
-				   ring->trail_fence_gpu_addr,
-				   ++ring->trail_seq);
-	amdgpu_ring_commit(kiq_ring);
-
-	spin_unlock_irqrestore(&kiq->ring_lock, flags);
-
-	/* poll the trailing fence */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		if (ring->trail_seq ==
-		    le32_to_cpu(*(ring->trail_fence_cpu_addr)))
-			break;
-		udelay(1);
-	}
-
-	if (i >= adev->usec_timeout) {
-		r = -EINVAL;
-		DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
-	}
-
-	/* deassert preemption condition */
-	amdgpu_ring_set_preempt_cond_exec(ring, true);
-	return r;
-}
-
 static void gfx_v11_0_ring_emit_de_meta(struct amdgpu_ring *ring, bool resume)
 {
 	struct amdgpu_device *adev = ring->adev;
@@ -7308,7 +7263,7 @@ static const struct amdgpu_ring_funcs gfx_v11_0_ring_funcs_gfx = {
 	.emit_cntxcntl = gfx_v11_0_ring_emit_cntxcntl,
 	.emit_gfx_shadow = gfx_v11_0_ring_emit_gfx_shadow,
 	.init_cond_exec = gfx_v11_0_ring_emit_init_cond_exec,
-	.preempt_ib = gfx_v11_0_ring_preempt_ib,
+	.preempt_ib = amdgpu_gfx_ring_preempt_ib,
 	.emit_frame_cntl = gfx_v11_0_ring_emit_frame_cntl,
 	.emit_wreg = gfx_v11_0_ring_emit_wreg,
 	.emit_reg_wait = gfx_v11_0_ring_emit_reg_wait,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
index c35372e21261d..f47928dcd8480 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_0.c
@@ -3275,6 +3275,11 @@ static int gfx_v12_0_compute_mqd_init(struct amdgpu_device *adev, void *m,
 	mqd->cp_hqd_pipe_priority = prop->hqd_pipe_priority;
 	mqd->cp_hqd_queue_priority = prop->hqd_queue_priority;
 
+	tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
+	mqd->cp_hqd_quantum = tmp;
+
 	mqd->cp_hqd_active = prop->hqd_active;
 
 	/* set UQ fenceaddress */
@@ -4618,56 +4623,6 @@ static unsigned gfx_v12_0_ring_emit_init_cond_exec(struct amdgpu_ring *ring,
 	return ret;
 }
 
-static int gfx_v12_0_ring_preempt_ib(struct amdgpu_ring *ring)
-{
-	int i, r = 0;
-	struct amdgpu_device *adev = ring->adev;
-	struct amdgpu_kiq *kiq = &adev->gfx.kiq[0];
-	struct amdgpu_ring *kiq_ring = &kiq->ring;
-	unsigned long flags;
-
-	if (adev->enable_mes)
-		return -EINVAL;
-
-	if (!kiq->pmf || !kiq->pmf->kiq_unmap_queues)
-		return -EINVAL;
-
-	spin_lock_irqsave(&kiq->ring_lock, flags);
-
-	if (amdgpu_ring_alloc(kiq_ring, kiq->pmf->unmap_queues_size)) {
-		spin_unlock_irqrestore(&kiq->ring_lock, flags);
-		return -ENOMEM;
-	}
-
-	/* assert preemption condition */
-	amdgpu_ring_set_preempt_cond_exec(ring, false);
-
-	/* assert IB preemption, emit the trailing fence */
-	kiq->pmf->kiq_unmap_queues(kiq_ring, ring, PREEMPT_QUEUES_NO_UNMAP,
-				   ring->trail_fence_gpu_addr,
-				   ++ring->trail_seq);
-	amdgpu_ring_commit(kiq_ring);
-
-	spin_unlock_irqrestore(&kiq->ring_lock, flags);
-
-	/* poll the trailing fence */
-	for (i = 0; i < adev->usec_timeout; i++) {
-		if (ring->trail_seq ==
-		    le32_to_cpu(*(ring->trail_fence_cpu_addr)))
-			break;
-		udelay(1);
-	}
-
-	if (i >= adev->usec_timeout) {
-		r = -EINVAL;
-		DRM_ERROR("ring %d failed to preempt ib\n", ring->idx);
-	}
-
-	/* deassert preemption condition */
-	amdgpu_ring_set_preempt_cond_exec(ring, true);
-	return r;
-}
-
 static void gfx_v12_0_ring_emit_rreg(struct amdgpu_ring *ring, uint32_t reg,
 				     uint32_t reg_val_offs)
 {
@@ -5538,7 +5493,7 @@ static const struct amdgpu_ring_funcs gfx_v12_0_ring_funcs_gfx = {
 	.pad_ib = amdgpu_ring_generic_pad_ib,
 	.emit_cntxcntl = gfx_v12_0_ring_emit_cntxcntl,
 	.init_cond_exec = gfx_v12_0_ring_emit_init_cond_exec,
-	.preempt_ib = gfx_v12_0_ring_preempt_ib,
+	.preempt_ib = amdgpu_gfx_ring_preempt_ib,
 	.emit_wreg = gfx_v12_0_ring_emit_wreg,
 	.emit_reg_wait = gfx_v12_0_ring_emit_reg_wait,
 	.emit_reg_write_reg_wait = gfx_v12_0_ring_emit_reg_write_reg_wait,
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
index 68db1bc73bc7c..033f15e21ad33 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1.c
@@ -61,7 +61,7 @@
 #define regCP_HQD_IB_CONTROL_DEFAULT                                              0x00100000
 
 MODULE_FIRMWARE("amdgpu/gc_12_1_0_mec.bin");
-MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc.bin");
+MODULE_FIRMWARE("amdgpu/gc_12_1_0_rlc_1.bin");
 
 #define SH_MEM_ALIGNMENT_MODE_UNALIGNED_GFX12_1_0	0x00000001
 #define DEFAULT_SH_MEM_CONFIG \
@@ -243,9 +243,9 @@ static void gfx_v12_1_wait_reg_mem(struct amdgpu_ring *ring, int eng_sel,
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WAIT_REG_MEM, 5));
 	amdgpu_ring_write(ring,
 			  /* memory (1) or register (0) */
-			  (WAIT_REG_MEM_MEM_SPACE(mem_space) |
-			   WAIT_REG_MEM_OPERATION(opt) | /* wait */
-			   WAIT_REG_MEM_FUNCTION(3)));  /* equal */
+			  (PACKET3_WAIT_REG_MEM__MEM_SPACE(mem_space) |
+			   PACKET3_WAIT_REG_MEM__OPERATION(opt) | /* wait */
+			   PACKET3_WAIT_REG_MEM__FUNCTION(3)));  /* equal */
 
 	if (mem_space)
 		BUG_ON(addr0 & 0x3); /* Dword align */
@@ -335,7 +335,7 @@ static int gfx_v12_1_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 	}
 
 	ib.ptr[0] = PACKET3(PACKET3_WRITE_DATA, 3);
-	ib.ptr[1] = WRITE_DATA_DST_SEL(5) | WR_CONFIRM;
+	ib.ptr[1] = PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1);
 	ib.ptr[2] = lower_32_bits(gpu_addr);
 	ib.ptr[3] = upper_32_bits(gpu_addr);
 	ib.ptr[4] = 0xDEADBEEF;
@@ -409,7 +409,13 @@ static int gfx_v12_1_init_microcode(struct amdgpu_device *adev)
 	amdgpu_ucode_ip_version_decode(adev, GC_HWIP, ucode_prefix, sizeof(ucode_prefix));
 
 	if (!amdgpu_sriov_vf(adev)) {
-		err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
+		if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) &&
+		    adev->rev_id == 0)
+			err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
+					   AMDGPU_UCODE_REQUIRED,
+					   "amdgpu/%s_rlc_1.bin", ucode_prefix);
+		else
+			err = amdgpu_ucode_request(adev, &adev->gfx.rlc_fw,
 					   AMDGPU_UCODE_REQUIRED,
 					   "amdgpu/%s_rlc.bin", ucode_prefix);
 		if (err)
@@ -712,10 +718,19 @@ static void gfx_v12_1_select_me_pipe_q(struct amdgpu_device *adev,
 	soc_v1_0_grbm_select(adev, me, pipe, q, vm, GET_INST(GC, xcc_id));
 }
 
+#define regGFX_IMU_PARTITION_SWITCH		0x5f8c
+#define regGFX_IMU_PARTITION_SWITCH_BASE_IDX	1
+#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP__SHIFT	0x2
+#define GFX_IMU_PARTITION_SWITCH__TOTAL_XCCS_IN_XCP_MASK		0x0000003CL
+
 static int gfx_v12_1_get_xccs_per_xcp(struct amdgpu_device *adev)
 {
-	/* Fill this in when the interface is ready */
-	return 1;
+	u32 reg_data;
+
+	/* the register data is expected to be the same on all instances */
+	reg_data = RREG32_SOC15(GC, GET_INST(GC, 0), regGFX_IMU_PARTITION_SWITCH);
+
+	return REG_GET_FIELD(reg_data, GFX_IMU_PARTITION_SWITCH, TOTAL_XCCS_IN_XCP);
 }
 
 static int gfx_v12_1_ih_to_xcc_inst(struct amdgpu_device *adev, int ih_node)
@@ -1136,6 +1151,7 @@ static int gfx_v12_1_rlc_backdoor_autoload_enable(struct amdgpu_device *adev)
 
 static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 {
+	uint16_t major_ver, minor_ver;
 	int i, j, k, r, ring_id = 0;
 	unsigned num_compute_rings;
 	int xcc_id, num_xcc;
@@ -1146,6 +1162,15 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block)
 		adev->gfx.mec.num_mec = 1;
 		adev->gfx.mec.num_pipe_per_mec = 4;
 		adev->gfx.mec.num_queue_per_pipe = 8;
+
+		if (!amdgpu_discovery_get_gc_major_minor_version(
+			    adev, &major_ver, &minor_ver)) {
+			if (major_ver == 1 && minor_ver == 3) {
+				adev->gfx.config.max_cu_per_sh /= 2;
+				dev_dbg(adev->dev, "Halving max_cu_per_sh for GC Discovery table v1:3 %d\n",
+					adev->gfx.config.max_cu_per_sh);
+			}
+		}
 		break;
 	default:
 		adev->gfx.mec.num_mec = 2;
@@ -1450,6 +1475,19 @@ static void gfx_v12_1_get_tcc_info(struct amdgpu_device *adev)
 {
 }
 
+static void gfx_v12_1_xcc_xnack_set_chicken_bits(struct amdgpu_device *adev, int xcc_id)
+{
+	/* NOTE: COMPRESSION_ENABLE is used a chicken bit to enable/disable xcc xnack */
+	mutex_lock(&adev->srbm_mutex);
+	if (!adev->gmc.noretry) {
+		WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
+				      TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x1);
+	} else
+		WREG32_FIELD15_PREREG(GC, GET_INST(GC, xcc_id),
+				      TCP_PERFCOUNTER_FILTER, COMPRESSION_ENABLE, 0x0);
+	mutex_unlock(&adev->srbm_mutex);
+}
+
 static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev,
 					 int xcc_id)
 {
@@ -1477,6 +1515,8 @@ static void gfx_v12_1_xcc_constants_init(struct amdgpu_device *adev,
 	mutex_unlock(&adev->srbm_mutex);
 
 	gfx_v12_1_xcc_init_compute_vmid(adev, xcc_id);
+	gfx_v12_1_xcc_xnack_set_chicken_bits(adev, xcc_id);
+
 }
 
 static void gfx_v12_1_constants_init(struct amdgpu_device *adev)
@@ -2246,6 +2286,11 @@ static int gfx_v12_1_compute_mqd_init(struct amdgpu_device *adev, void *m,
 	mqd->cp_mqd_stride_size = prop->mqd_stride_size ? prop->mqd_stride_size :
 		AMDGPU_MQD_SIZE_ALIGN(adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size);
 
+	tmp = REG_SET_FIELD(0, CP_HQD_QUANTUM, QUANTUM_EN, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
+	mqd->cp_hqd_quantum = tmp;
+
 	mqd->cp_hqd_active = prop->hqd_active;
 
 	return 0;
@@ -3355,7 +3400,7 @@ static void gfx_v12_1_ring_emit_ib_compute(struct amdgpu_ring *ring,
 					   uint32_t flags)
 {
 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
-	u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vmid << 24);
+	u32 control = PACKET3_INDIRECT_BUFFER__VALID(1) | ib->length_dw | (vmid << 24);
 
 	/* Currently, there is a high possibility to get wave ID mismatch
 	 * between ME and GDS, leading to a hw deadlock, because ME generates
@@ -3391,15 +3436,15 @@ static void gfx_v12_1_ring_emit_fence(struct amdgpu_ring *ring, u64 addr,
 
 	/* RELEASE_MEM - flush caches, send int */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_RELEASE_MEM, 6));
-	amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_GCR_SEQ(1) |
-				 PACKET3_RELEASE_MEM_GCR_GLV_WB |
-				 PACKET3_RELEASE_MEM_GCR_GL2_WB |
-				 PACKET3_RELEASE_MEM_GCR_GL2_SCOPE(2) |
-				 PACKET3_RELEASE_MEM_TEMPORAL(3) |
-				 PACKET3_RELEASE_MEM_EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
-				 PACKET3_RELEASE_MEM_EVENT_INDEX(5)));
-	amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM_DATA_SEL(write64bit ? 2 : 1) |
-				 PACKET3_RELEASE_MEM_INT_SEL(int_sel ? 2 : 0)));
+	amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__GCR_SEQ(1) |
+				 PACKET3_RELEASE_MEM__GCR_GLV_WB |
+				 PACKET3_RELEASE_MEM__GCR_GL2_WB |
+				 PACKET3_RELEASE_MEM__GCR_GL2_SCOPE(2) |
+				 PACKET3_RELEASE_MEM__TEMPORAL(3) |
+				 PACKET3_RELEASE_MEM__EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) |
+				 PACKET3_RELEASE_MEM__EVENT_INDEX(5)));
+	amdgpu_ring_write(ring, (PACKET3_RELEASE_MEM__DATA_SEL(write64bit ? 2 : 1) |
+				 PACKET3_RELEASE_MEM__INT_SEL(int_sel ? 2 : 0)));
 
 	/*
 	 * the address should be Qword aligned if 64bit write, Dword
@@ -3460,7 +3505,7 @@ static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
 
 	/* write fence seq to the "addr" */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-	amdgpu_ring_write(ring, (WRITE_DATA_DST_SEL(5) | WR_CONFIRM));
+	amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(5) | PACKET3_WRITE_DATA__WR_CONFIRM(1)));
 	amdgpu_ring_write(ring, lower_32_bits(addr));
 	amdgpu_ring_write(ring, upper_32_bits(addr));
 	amdgpu_ring_write(ring, lower_32_bits(seq));
@@ -3468,7 +3513,7 @@ static void gfx_v12_1_ring_emit_fence_kiq(struct amdgpu_ring *ring, u64 addr,
 	if (flags & AMDGPU_FENCE_FLAG_INT) {
 		/* set register to trigger INT */
 		amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-		amdgpu_ring_write(ring, (WRITE_DATA_DST_SEL(0) | WR_CONFIRM));
+		amdgpu_ring_write(ring, (PACKET3_WRITE_DATA__DST_SEL(0) | PACKET3_WRITE_DATA__WR_CONFIRM(1)));
 		amdgpu_ring_write(ring, SOC15_REG_OFFSET(GC, GET_INST(GC, 0), regCPC_INT_STATUS));
 		amdgpu_ring_write(ring, 0);
 		amdgpu_ring_write(ring, 0x20000000); /* src_id is 178 */
@@ -3507,7 +3552,7 @@ static void gfx_v12_1_ring_emit_wreg(struct amdgpu_ring *ring,
 		cmd = (1 << 16); /* no inc addr */
 		break;
 	default:
-		cmd = WR_CONFIRM;
+		cmd = PACKET3_WRITE_DATA__WR_CONFIRM(1);
 		break;
 	}
 	amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
@@ -3810,12 +3855,12 @@ static int gfx_v12_1_rlc_poison_irq(struct amdgpu_device *adev,
 static void gfx_v12_1_emit_mem_sync(struct amdgpu_ring *ring)
 {
 	const unsigned int gcr_cntl =
-			PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(1) |
-			PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(1) |
-			PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(1) |
-			PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(1) |
-			PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(1) |
-			PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_SCOPE(2);
+			PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_INV(1) |
+			PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_WB(1) |
+			PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_INV(1) |
+			PACKET3_ACQUIRE_MEM__GCR_CNTL__GLK_INV(1) |
+			PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV(1) |
+			PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE(2);
 
 	/* ACQUIRE_MEM - make one or more surfaces valid for use by the subsequent operations */
 	amdgpu_ring_write(ring, PACKET3(PACKET3_ACQUIRE_MEM, 6));
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
index 21a07530c64d5..77b4f6ea532f5 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v12_1_pkt.h
@@ -58,11 +58,36 @@
 #define	PACKET3_DISPATCH_DIRECT				0x15
 #define	PACKET3_DISPATCH_INDIRECT			0x16
 #define	PACKET3_ATOMIC_MEM				0x1E
+#define 	PACKET3_ATOMIC_MEM__ATOMIC(x) ((((unsigned)(x)) & 0x7F) << 0)
+#define 	PACKET3_ATOMIC_MEM__COMMAND(x) ((((unsigned)(x)) & 0xF) << 8)
+#define 	PACKET3_ATOMIC_MEM__SCOPE(x) ((((unsigned)(x)) & 0x3) << 23)
+#define 	PACKET3_ATOMIC_MEM__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25)
+#define 	PACKET3_ATOMIC_MEM__ADDR_LO(x) (((unsigned)(x)))
+#define 	PACKET3_ATOMIC_MEM__ADDR_HI(x) (((unsigned)(x)))
+#define 	PACKET3_ATOMIC_MEM__SRC_DATA_LO(x) (((unsigned)(x)))
+#define 	PACKET3_ATOMIC_MEM__SRC_DATA_HI(x) (((unsigned)(x)))
+#define 	PACKET3_ATOMIC_MEM__CMP_DATA_LO(x) (((unsigned)(x)))
+#define 	PACKET3_ATOMIC_MEM__CMP_DATA_HI(x) (((unsigned)(x)))
+#define 	PACKET3_ATOMIC_MEM__LOOP_INTERVAL(x) ((((unsigned)(x)) & 0x1FFF) << 0)
+#define 	PACKET3_ATOMIC_MEM__COMMAND__SINGLE_PASS_ATOMIC 0
+#define 	PACKET3_ATOMIC_MEM__COMMAND__LOOP_UNTIL_COMPARE_SATISFIED 1
+#define 	PACKET3_ATOMIC_MEM__COMMAND__WAIT_FOR_WRITE_CONFIRMATION 2
+#define 	PACKET3_ATOMIC_MEM__COMMAND__SEND_AND_CONTINUE 3
+#define 	PACKET3_ATOMIC_MEM__SCOPE__CU 0
+#define 	PACKET3_ATOMIC_MEM__SCOPE__SE 1
+#define 	PACKET3_ATOMIC_MEM__SCOPE__DEVICE 2
+#define 	PACKET3_ATOMIC_MEM__SCOPE__SYSTEM 3
+#define 	PACKET3_ATOMIC_MEM__TEMPORAL__RT 0
+#define 	PACKET3_ATOMIC_MEM__TEMPORAL__NT 1
+#define 	PACKET3_ATOMIC_MEM__TEMPORAL__FW 2
+#define 	PACKET3_ATOMIC_MEM__TEMPORAL__UC 3
 #define	PACKET3_OCCLUSION_QUERY				0x1F
 #define	PACKET3_SET_PREDICATION				0x20
 #define	PACKET3_REG_RMW					0x21
 #define	PACKET3_COND_EXEC				0x22
 #define	PACKET3_PRED_EXEC				0x23
+#define 	PACKET3_PRED_EXEC__EXEC_COUNT(x) ((((unsigned)(x)) & 0x3FFF) << 0)
+#define 	PACKET3_PRED_EXEC__VIRTUALXCCID_SELECT(x) ((((unsigned)(x)) & 0xFF) << 24)
 #define	PACKET3_DRAW_INDIRECT				0x24
 #define	PACKET3_DRAW_INDEX_INDIRECT			0x25
 #define	PACKET3_INDEX_BASE				0x26
@@ -74,7 +99,11 @@
 #define	PACKET3_DRAW_INDEX_MULTI_AUTO			0x30
 #define	PACKET3_DRAW_INDEX_OFFSET_2			0x35
 #define	PACKET3_WRITE_DATA				0x37
-#define		WRITE_DATA_DST_SEL(x)                   (((x) & 0xf) << 8)
+#define 	PACKET3_WRITE_DATA__DST_SEL(x) ((((unsigned)(x)) & 0xF) << 8)
+#define 	PACKET3_WRITE_DATA__DST_SEL__MEM_MAPPED_REGISTER 0
+#define 	PACKET3_WRITE_DATA__DST_SEL__TC_L2 2
+#define 	PACKET3_WRITE_DATA__DST_SEL__MEMORY 5
+#define 	PACKET3_WRITE_DATA__DST_SEL__MEMORY_MAPPED_ADC_PERSISTENT_STATE 6
 		/* 0 - register
 		 * 1 - reserved
 		 * 2 - tc_l2
@@ -83,27 +112,52 @@
 		 * 5 - memory (same as tc_l2)
          * 6 - memory_mapped_adc_persistent_state
 		 */
-#define		WRITE_DATA_SCOPE(x)                     (((x) & 0x3) << 12)
-#define		WRITE_DATA_MODE(x)                      (((x) & 0x3) << 14)
+#define 	PACKET3_WRITE_DATA__SCOPE(x)   ((((unsigned)(x)) & 0x3) << 12)
+#define 	PACKET3_WRITE_DATA__SCOPE__CU 0
+#define 	PACKET3_WRITE_DATA__SCOPE__SE 1
+#define 	PACKET3_WRITE_DATA__SCOPE__DEVICE 2
+#define 	PACKET3_WRITE_DATA__SCOPE__SYSTEM 3
+#define 	PACKET3_WRITE_DATA__MODE(x)    ((((unsigned)(x)) & 0x3) << 14)
+#define 	PACKET3_WRITE_DATA__MODE__LOCAL_XCD 0
+#define 	PACKET3_WRITE_DATA__MODE__REMOTE_OR_LOCAL_AID 1
+#define 	PACKET3_WRITE_DATA__MODE__REMOTE_XCD 2
+#define 	PACKET3_WRITE_DATA__MODE__REMOTE_MID 3
         /* 0 - local xcd
          * 1 - remote/local aid
          * 2 - remote xcd
          * 3 - remote mid
          */
-#define		WRITE_DATA_ADDR_INCR                    (1 << 16)
-#define		WRITE_DATA_MID_DIE_ID(x)                (((x) & 0x3) << 18)
-#define		WR_CONFIRM                              (1 << 20)
-#define		WRITE_DATA_XCD_DIE_ID(x)                (((x) & 0xf) << 21)
-#define		WRITE_DATA_TEMPORAL(x)                  (((x) & 0x3) << 25)
-		/* 0 - rt
-		 * 1 - nt
-         * 2 - ht
-         * 3 - lu
-		 */
-#define		WRITE_DATA_COOP_DISABLE                 (1 << 27)
+#define 	PACKET3_WRITE_DATA__ADDR_INCR(x)  ((((unsigned)(x)) & 0x1) << 16)
+#define 	PACKET3_WRITE_DATA__ADDR_INCR__INCREMENT_ADDRESS 0
+#define 	PACKET3_WRITE_DATA__ADDR_INCR__DO_NOT_INCREMENT_ADDRESS 1
+#define 	PACKET3_WRITE_DATA__MID_DIE_ID(x)       ((((unsigned)(x)) & 0x3) << 18)
+#define 	PACKET3_WRITE_DATA__WR_CONFIRM(x) ((((unsigned)(x)) & 0x1) << 20)
+#define 	PACKET3_WRITE_DATA__WR_CONFIRM__DO_NOT_WAIT_FOR_WRITE_CONFIRMATION 0
+#define 	PACKET3_WRITE_DATA__WR_CONFIRM__WAIT_FOR_WRITE_CONFIRMATION 1
+#define 	PACKET3_WRITE_DATA__XCD_DIE_ID(x) ((((unsigned)(x)) & 0xF) << 21)
+#define 	PACKET3_WRITE_DATA__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25)
+#define 	PACKET3_WRITE_DATA__TEMPORAL__RT 0
+#define 	PACKET3_WRITE_DATA__TEMPORAL__NT 1
+#define 	PACKET3_WRITE_DATA__TEMPORAL__HT 2
+#define 	PACKET3_WRITE_DATA__TEMPORAL__LU 3
+#define 	PACKET3_WRITE_DATA__COOP_DISABLE(x)  ((((unsigned)(x)) & 0x1) << 27)
+#define 	PACKET3_WRITE_DATA__COOP_DISABLE__MASTER_AND_SLAVE_COOP 0
+#define 	PACKET3_WRITE_DATA__COOP_DISABLE__MASTER_ONLY 1
+#define 	PACKET3_WRITE_DATA__DST_MMREG_ADDR_LO(x) ((unsigned)(x))
+#define 	PACKET3_WRITE_DATA__DST_MEM_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
+#define 	PACKET3_WRITE_DATA__DST_MMREG_ADDR_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0)
+#define 	PACKET3_WRITE_DATA__DST_MEM_ADDR_HI(x) ((unsigned)(x))
+#define 	PACKET3_WRITE_DATA__DATA(x) ((unsigned)(x))
 #define	PACKET3_DRAW_INDEX_INDIRECT_MULTI		0x38
 #define	PACKET3_WAIT_REG_MEM				0x3C
-#define		WAIT_REG_MEM_FUNCTION(x)                (((x) & 0x7) << 0)
+#define		PACKET3_WAIT_REG_MEM__FUNCTION(x) ((((unsigned)(x)) & 0x7) << 0)
+#define		PACKET3_WAIT_REG_MEM__FUNCTION__ALWAYS_PASS 0
+#define		PACKET3_WAIT_REG_MEM__FUNCTION__LESS_THAN_REF_VALUE 1
+#define		PACKET3_WAIT_REG_MEM__FUNCTION__LESS_THAN_EQUAL_TO_THE_REF_VALUE 2
+#define		PACKET3_WAIT_REG_MEM__FUNCTION__EQUAL_TO_THE_REFERENCE_VALUE 3
+#define		PACKET3_WAIT_REG_MEM__FUNCTION__NOT_EQUAL_REFERENCE_VALUE 4
+#define		PACKET3_WAIT_REG_MEM__FUNCTION__GREATER_THAN_OR_EQUAL_REFERENCE_VALUE 5
+#define		PACKET3_WAIT_REG_MEM__FUNCTION__GREATER_THAN_REFERENCE_VALUE 6
 		/* 0 - always
 		 * 1 - <
 		 * 2 - <=
@@ -112,117 +166,222 @@
 		 * 5 - >=
 		 * 6 - >
 		 */
-#define		WAIT_REG_MEM_MEM_SPACE(x)               (((x) & 0x3) << 4)
+#define 	PACKET3_WAIT_REG_MEM__MEM_SPACE(x) ((((unsigned)(x)) & 0x3) << 4)
+#define 	PACKET3_WAIT_REG_MEM__MEM_SPACE__REGISTER_SPACE 0
+#define 	PACKET3_WAIT_REG_MEM__MEM_SPACE__MEMORY_SPACE 1
 		/* 0 - reg
 		 * 1 - mem
 		 */
-#define		WAIT_REG_MEM_OPERATION(x)               (((x) & 0x3) << 6)
+#define		PACKET3_WAIT_REG_MEM__OPERATION(x) ((((unsigned)(x)) & 0x3) << 6)
+#define 	PACKET3_WAIT_REG_MEM__OPERATION__WAIT_REG_MEM 0
+#define 	PACKET3_WAIT_REG_MEM__OPERATION__WR_WAIT_WR_REG 1
+#define 	PACKET3_WAIT_REG_MEM__OPERATION__WAIT_MEM_PREEMPTABLE 3
 		/* 0 - wait_reg_mem
 		 * 1 - wr_wait_wr_reg
+		 * 2 - reserved
+		 * 3 - wait_mem_preemptable
 		 */
-#define		WAIT_REG_MEM_MODE(x)                    (((x) & 0x3) << 10)
+#define		PACKET3_WAIT_REG_MEM__MODE(x)   ((((unsigned)(x)) & 0x3) << 10)
+#define		PACKET3_WAIT_REG_MEM__MODE__LOCAL_XCD   0
+#define		PACKET3_WAIT_REG_MEM__MODE__REMOTE_OR_LOCAL_AID   1
+#define		PACKET3_WAIT_REG_MEM__MODE__REMOTE_XCD   2
+#define		PACKET3_WAIT_REG_MEM__MODE__REMOTE_MID   3
         /* 0 - local xcd
          * 1 - remote/local aid
          * 2 - remote xcd
          * 3 - remote mid
          */
-#define		WAIT_REG_MEM_MID_DIE_ID(x)              (((x) & 0x3) << 12)
-#define		WAIT_REG_MEM_XCD_DIE_ID(x)              (((x) & 0xf) << 14)
-#define		WAIT_REG_MEM_MES_INTR_PIPE(x)           (((x) & 0x3) << 22)
-#define		WAIT_REG_MEM_MES_ACTION(x)              (((x) & 0x1) << 24)
-#define		WAIT_REG_MEM_TEMPORAL(x)                (((x) & 0x3) << 25)
+#define		PACKET3_WAIT_REG_MEM__MID_DIE_ID(x) ((((unsigned)(x)) & 0x3) << 12)
+#define		PACKET3_WAIT_REG_MEM__XCD_DIE_ID(x) ((((unsigned)(x)) & 0xf) << 14)
+#define		PACKET3_WAIT_REG_MEM__MES_INTR_PIPE(x) ((((unsigned)(x)) & 0x3) << 22)
+#define		PACKET3_WAIT_REG_MEM__MES_ACTION(x) ((((unsigned)(x)) & 0x1) << 24)
+#define		PACKET3_WAIT_REG_MEM__TEMPORAL(x)       ((((unsigned)(x)) & 0x3) << 25)
+#define 	PACKET3_WAIT_REG_MEM__TEMPORAL__RT 0
+#define 	PACKET3_WAIT_REG_MEM__TEMPORAL__NT 1
+#define 	PACKET3_WAIT_REG_MEM__TEMPORAL__HT 2
+#define 	PACKET3_WAIT_REG_MEM__TEMPORAL__LU 3
 		/* 0 - rt
 		 * 1 - nt
          * 2 - ht
          * 3 - lu
 		 */
+#define		PACKET3_WAIT_REG_MEM__MEM_POLL_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
+#define		PACKET3_WAIT_REG_MEM__REG_POLL_ADDR(x) ((unsigned)(x))
+#define		PACKET3_WAIT_REG_MEM__REG_WRITE_ADDR1(x) ((((unsigned)(x)) & 0x3FFFF) << 0)
+#define		PACKET3_WAIT_REG_MEM__MEM_POLL_ADDR_HI(x) ((unsigned)(x))
+#define		PACKET3_WAIT_REG_MEM__REG_POLL_ADDR_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0)
+#define		PACKET3_WAIT_REG_MEM__REG_WRITE_ADDR2(x) ((((unsigned)(x)) & 0x3FFFF) << 0)
+#define		PACKET3_WAIT_REG_MEM__REFERENCE(x) ((unsigned)(x))
+#define		PACKET3_WAIT_REG_MEM__MASK(x) ((unsigned)(x))
+#define		PACKET3_WAIT_REG_MEM__POLL_INTERVAL(x) ((((unsigned)(x)) & 0xFFFF) << 0)
+#define		PACKET3_WAIT_REG_MEM__OPTIMIZE_ACE_OFFLOAD_MODE(x) ((((unsigned)(x)) & 0x1) << 31)
 #define	PACKET3_INDIRECT_BUFFER				0x3F
-#define		INDIRECT_BUFFER_VALID                   (1 << 23)
-#define		INDIRECT_BUFFER_TEMPORAL(x)             (x) << 28)
-		/* 0 - rt
-		 * 1 - nt
-		 * 2 - ht
-         * 3 - lu
-		 */
+#define 	PACKET3_INDIRECT_BUFFER__IB_BASE_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
+#define 	PACKET3_INDIRECT_BUFFER__IB_BASE_HI(x) ((unsigned)(x))
+#define 	PACKET3_INDIRECT_BUFFER__IB_SIZE(x) ((((unsigned)(x)) & 0xFFFFF) << 0)
+#define 	PACKET3_INDIRECT_BUFFER__CHAIN(x) ((((unsigned)(x)) & 0x1) << 20)
+#define 	PACKET3_INDIRECT_BUFFER__OFFLOAD_POLLING(x) ((((unsigned)(x)) & 0x1) << 21)
+#define 	PACKET3_INDIRECT_BUFFER__VALID(x) ((((unsigned)(x)) & 0x1) << 23)
+#define 	PACKET3_INDIRECT_BUFFER__VMID(x) ((((unsigned)(x)) & 0xF) << 24)
+#define 	PACKET3_INDIRECT_BUFFER__TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 28)
+#define 	PACKET3_INDIRECT_BUFFER__INHERIT_VMID(x) ((((unsigned)(x)) & 0x1) << 30)
+#define 	PACKET3_INDIRECT_BUFFER__PRIV(x) ((((unsigned)(x)) & 0x1) << 31)
+#define 	PACKET3_INDIRECT_BUFFER__TEMPORAL__RT 0
+#define 	PACKET3_INDIRECT_BUFFER__TEMPORAL__NT 1
+#define 	PACKET3_INDIRECT_BUFFER__TEMPORAL__HT 2
+#define 	PACKET3_INDIRECT_BUFFER__TEMPORAL__LU 3
 #define	PACKET3_COND_INDIRECT_BUFFER			0x3F
 #define	PACKET3_COPY_DATA				0x40
-#define		COPY_DATA_SRC_SEL(x)                    (((x) & 0xf) << 0)
-#define		COPY_DATA_DST_SEL(x)                    (((x) & 0xf) << 8)
-#define		COPY_DATA_SRC_SCOPE(x)                  (((x) & 0x3) << 4)
-#define		COPY_DATA_DST_SCOPE(x)                  (((x) & 0x3) << 27)
-#define		COPY_DATA_MODE(x)                       (((x) & 0x3) << 6)
-        /* 0 - local xcd
-         * 1 - remote/local aid
-         * 2 - remote xcd
-         * 3 - remote mid
-         */
-#define		COPY_DATA_SRC_TEMPORAL(x)               (((x) & 0x3) << 13)
-#define		COPY_DATA_DST_TEMPORAL(x)               (((x) & 0x3) << 25)
-		/* 0 - rt
-		 * 1 - nt
-         * 2 - ht
-         * 3 - lu
-		 */
-#define		COPY_DATA_COUNT_SEL                     (1 << 16)
-#define		COPY_DATA_SRC_DST_REMOTE_MODE(x)        (((x)) & 0x1 << 16)
-        /* 0 - src remote
-         * 1 - dst remote
-         */
-#define		COPY_DATA_MID_DIE_ID(x)                 (((x) & 0x3) << 18)
-#define		COPY_DATA_XCD_DIE_ID(x)                 (((x) & 0xf) << 21)
-#define		COPY_DATA_PQ_EXE_STATUS                 (1 << 27)
+#define 	PACKET3_COPY_DATA__SRC_SEL(x) ((((unsigned)(x)) & 0xF) << 0)
+#define		PACKET3_COPY_DATA__SRC_SCOPE(x) ((((unsigned)(x)) & 0x3) << 4)
+#define 	PACKET3_COPY_DATA__MODE(x) ((((unsigned)(x)) & 0x3) << 6)
+#define 	PACKET3_COPY_DATA__DST_SEL(x) ((((unsigned)(x)) & 0xF) << 8)
+#define 	PACKET3_COPY_DATA__SRC_TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 13)
+#define 	PACKET3_COPY_DATA__COUNT_SEL(x) ((((unsigned)(x)) & 0x1) << 16)
+#define 	PACKET3_COPY_DATA__SRC_DST_REMOTE_MODE(x) ((((unsigned)(x)) & 0x1) << 17)
+#define 	PACKET3_COPY_DATA__MID_DIE_ID(x) ((((unsigned)(x)) & 0x3) << 18)
+#define 	PACKET3_COPY_DATA__WR_CONFIRM(x) ((((unsigned)(x)) & 0x1) << 20)
+#define 	PACKET3_COPY_DATA__XCD_DIE_ID(x) ((((unsigned)(x)) & 0xF) << 21)
+#define 	PACKET3_COPY_DATA__DST_TEMPORAL(x) ((((unsigned)(x)) & 0x3) << 25)
+#define		PACKET3_COPY_DATA__DST_SCOPE(x) ((((unsigned)(x)) & 0x3) << 27)
+#define 	PACKET3_COPY_DATA__PQ_EXE_STATUS(x) ((((unsigned)(x)) & 0x1) << 29)
+#define 	PACKET3_COPY_DATA__SRC_REG_OFFSET_LO(x) ((unsigned)(x))
+#define 	PACKET3_COPY_DATA__SRC_32B_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
+#define 	PACKET3_COPY_DATA__SRC_64B_ADDR_LO(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3)
+#define 	PACKET3_COPY_DATA__IMM_DATA(x) ((unsigned)(x))
+#define 	PACKET3_COPY_DATA__SRC_REG_OFFSET_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0)
+#define 	PACKET3_COPY_DATA__SRC_MEMTC_ADDR_HI(x) ((unsigned)(x))
+#define 	PACKET3_COPY_DATA__SRC_IMM_DATA(x) ((unsigned)(x))
+#define 	PACKET3_COPY_DATA__DST_REG_OFFSET_LO(x) ((unsigned)(x))
+#define 	PACKET3_COPY_DATA__DST_32B_ADDR_LO(x) ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
+#define 	PACKET3_COPY_DATA__DST_64B_ADDR_LO(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3)
+#define 	PACKET3_COPY_DATA__DST_REG_OFFSET_HI(x) ((((unsigned)(x)) & 0x3FFF) << 0)
+#define 	PACKET3_COPY_DATA__DST_ADDR_HI(x) ((unsigned)(x))
+#define 	PACKET3_COPY_DATA__SRC_SEL__MEM_MAPPED_REGISTER 0
+#define 	PACKET3_COPY_DATA__SRC_SEL__TC_L2_OBSOLETE 1
+#define 	PACKET3_COPY_DATA__SRC_SEL__TC_L2 2
+#define 	PACKET3_COPY_DATA__SRC_SEL__PERFCOUNTERS 4
+#define 	PACKET3_COPY_DATA__SRC_SEL__IMMEDIATE_DATA 5
+#define 	PACKET3_COPY_DATA__SRC_SEL__ATOMIC_RETURN_DATA 6
+#define 	PACKET3_COPY_DATA__SRC_SEL__GPU_CLOCK_COUNT 9
+#define 	PACKET3_COPY_DATA__SRC_SEL__SYSTEM_CLOCK_COUNT 10
+#define 	PACKET3_COPY_DATA__SRC_SCOPE__CU 0
+#define 	PACKET3_COPY_DATA__SRC_SCOPE__SE 1
+#define 	PACKET3_COPY_DATA__SRC_SCOPE__DEVICE 2
+#define 	PACKET3_COPY_DATA__SRC_SCOPE__SYSTEM 3
+#define 	PACKET3_COPY_DATA__MODE__LOCAL_XCD 0
+#define 	PACKET3_COPY_DATA__MODE__REMOTE_OR_LOCAL_AID 1
+#define 	PACKET3_COPY_DATA__MODE__REMOTE_XCD 2
+#define 	PACKET3_COPY_DATA__MODE__REMOTE_MID 3
+#define 	PACKET3_COPY_DATA__DST_SEL__MEM_MAPPED_REGISTER 0
+#define 	PACKET3_COPY_DATA__DST_SEL__TC_L2 2
+#define 	PACKET3_COPY_DATA__DST_SEL__PERFCOUNTERS 4
+#define 	PACKET3_COPY_DATA__DST_SEL__TC_L2_OBSOLETE 5
+#define 	PACKET3_COPY_DATA__DST_SEL__MEM_MAPPED_REG_DC 6
+#define 	PACKET3_COPY_DATA__SRC_TEMPORAL__RT 0
+#define 	PACKET3_COPY_DATA__SRC_TEMPORAL__NT 1
+#define 	PACKET3_COPY_DATA__SRC_TEMPORAL__HT 2
+#define 	PACKET3_COPY_DATA__SRC_TEMPORAL__LU 3
+#define 	PACKET3_COPY_DATA__COUNT_SEL__32_BITS_OF_DATA 0
+#define 	PACKET3_COPY_DATA__COUNT_SEL__64_BITS_OF_DATA 1
+#define 	PACKET3_COPY_DATA__SRC_DST_REMOTE_MODE__SRC_IS_REMOTE 0
+#define 	PACKET3_COPY_DATA__SRC_DST_REMOTE_MODE__DST_IS_REMOTE 1
+#define 	PACKET3_COPY_DATA__WR_CONFIRM__DO_NOT_WAIT_FOR_CONFIRMATION 0
+#define 	PACKET3_COPY_DATA__WR_CONFIRM__WAIT_FOR_CONFIRMATION 1
+#define 	PACKET3_COPY_DATA__DST_TEMPORAL__RT 0
+#define 	PACKET3_COPY_DATA__DST_TEMPORAL__NT 1
+#define 	PACKET3_COPY_DATA__DST_TEMPORAL__HT 2
+#define 	PACKET3_COPY_DATA__DST_TEMPORAL__LU 3
+#define 	PACKET3_COPY_DATA__DST_SCOPE__CU 0
+#define 	PACKET3_COPY_DATA__DST_SCOPE__SE 1
+#define 	PACKET3_COPY_DATA__DST_SCOPE__DEVICE 2
+#define 	PACKET3_COPY_DATA__DST_SCOPE__SYSTEM 3
+#define 	PACKET3_COPY_DATA__PQ_EXE_STATUS__DEFAULT 0
+#define 	PACKET3_COPY_DATA__PQ_EXE_STATUS__PHASE_UPDATE 1
 #define	PACKET3_PFP_SYNC_ME				0x42
 #define	PACKET3_COND_WRITE				0x45
 #define	PACKET3_EVENT_WRITE				0x46
-#define		EVENT_TYPE(x)                           ((x) << 0)
-#define		EVENT_INDEX(x)                          ((x) << 8)
-		/* 0 - any non-TS event
-		 * 1 - ZPASS_DONE, PIXEL_PIPE_STAT_*
-		 * 2 - SAMPLE_PIPELINESTAT
-		 * 3 - SAMPLE_STREAMOUTSTAT*
-		 * 4 - *S_PARTIAL_FLUSH
-		 */
+#define		PACKET3_EVENT_WRITE__EVENT_TYPE(x) ((((unsigned)(x)) & 0x3F) << 0)
+#define		PACKET3_EVENT_WRITE__EVENT_INDEX(x) ((((unsigned)(x)) & 0xF) << 8)
+#define		PACKET3_EVENT_WRITE__OFFLOAD_ENABLE(x) ((((unsigned)(x)) & 0x1) << 31)
+#define 	PACKET3_EVENT_WRITE__ADDRESS_LO(x) ((((unsigned)(x)) & 0x1FFFFFFF) << 3)
+#define 	PACKET3_EVENT_WRITE__ADDRESS_HI(x) ((unsigned)(x))
+#define 	PACKET3_EVENT_WRITE__EVENT_INDEX__OTHER 0
+#define 	PACKET3_EVENT_WRITE__EVENT_INDEX__SAMPLE_PIPELINESTAT 2
+#define 	PACKET3_EVENT_WRITE__EVENT_INDEX__CS_PARTIAL_FLUSH 4
+#define	PACKET3_EVENT_WRITE_EOP				0x47
+#define	PACKET3_EVENT_WRITE_EOS				0x48
 #define	PACKET3_RELEASE_MEM				0x49
-#define		PACKET3_RELEASE_MEM_EVENT_TYPE(x)	    ((x) << 0)
-#define		PACKET3_RELEASE_MEM_EVENT_INDEX(x)	    ((x) << 8)
-#define		PACKET3_RELEASE_MEM_GCR_GL2_SCOPE(x)	((x) << 12)
-#define		PACKET3_RELEASE_MEM_GCR_GLV_INV		    (1 << 14)
-#define		PACKET3_RELEASE_MEM_GCR_GL2_US		    (1 << 16)
-#define		PACKET3_RELEASE_MEM_GCR_GL2_RANGE(x)	((x) << 17)
-#define		PACKET3_RELEASE_MEM_GCR_GL2_DISCARD	    (1 << 19)
-#define		PACKET3_RELEASE_MEM_GCR_GL2_INV		    (1 << 20)
-#define		PACKET3_RELEASE_MEM_GCR_GL2_WB		    (1 << 21)
-#define		PACKET3_RELEASE_MEM_GCR_SEQ(x)		    ((x) << 22)
-#define		PACKET3_RELEASE_MEM_GCR_GLV_WB		    (1 << 24)
-#define		PACKET3_RELEASE_MEM_TEMPORAL(x)	        ((x) << 25)
+#define		PACKET3_RELEASE_MEM__EVENT_TYPE(x)  ((((unsigned)(x)) & 0x3F) << 0)
+#define		PACKET3_RELEASE_MEM__WAIT_SYNC(x)  ((((unsigned)(x)) & 0x1) << 7)
+#define		PACKET3_RELEASE_MEM__EVENT_INDEX(x)  ((((unsigned)(x)) & 0xF) << 8)
+#define		PACKET3_RELEASE_MEM__EVENT_INDEX__END_OF_PIPE   5
+#define		PACKET3_RELEASE_MEM__EVENT_INDEX__SHADER_DONE   6
+#define		PACKET3_RELEASE_MEM__GCR_CNTL(x)  ((((unsigned)(x)) & 0x1FFF) << 12)
+#define		PACKET3_RELEASE_MEM__GCR_GL2_SCOPE(x)	((x) << 12)
+#define		PACKET3_RELEASE_MEM__GCR_GLV_INV		    (1 << 14)
+#define		PACKET3_RELEASE_MEM__GCR_GL2_US		    (1 << 16)
+#define		PACKET3_RELEASE_MEM__GCR_GL2_RANGE(x)	((x) << 17)
+#define		PACKET3_RELEASE_MEM__GCR_GL2_DISCARD	    (1 << 19)
+#define		PACKET3_RELEASE_MEM__GCR_GL2_INV		    (1 << 20)
+#define		PACKET3_RELEASE_MEM__GCR_GL2_WB		    (1 << 21)
+#define		PACKET3_RELEASE_MEM__GCR_SEQ(x)		    ((x) << 22)
+#define		PACKET3_RELEASE_MEM__GCR_GLV_WB		    (1 << 24)
+#define		PACKET3_RELEASE_MEM__TEMPORAL(x)    ((((unsigned)(x)) & 0x3) << 25)
+#define		PACKET3_RELEASE_MEM__TEMPORAL__RT   0
+#define		PACKET3_RELEASE_MEM__TEMPORAL__NT   1
+#define		PACKET3_RELEASE_MEM__TEMPORAL__HT   2
+#define		PACKET3_RELEASE_MEM__TEMPORAL__LU   3
 		/* 0 - temporal__release_mem__rt
 		 * 1 - temporal__release_mem__nt
 		 * 2 - temporal__release_mem__ht
 		 * 3 - temporal__release_mem__lu
 		 */
-#define		PACKET3_RELEASE_MEM_PQ_EXE_STATUS		(1 << 28)
-#define		PACKET3_RELEASE_MEM_GCR_GLK_INV		    (1 << 30)
+#define		PACKET3_RELEASE_MEM__PQ_EXE_STATUS(x)    ((((unsigned)(x)) & 0x1) << 28)
+#define		PACKET3_RELEASE_MEM__PQ_EXE_STATUS__DEFAULT 0
+#define		PACKET3_RELEASE_MEM__PQ_EXE_STATUS__PHASE_UPDATE 1
+#define		PACKET3_RELEASE_MEM__GCR_GLK_INV		    (1 << 30)
 
-#define		PACKET3_RELEASE_MEM_DST_SEL(x)		((x) << 16)
-		/* 0 - memory controller
-		 * 1 - TC/L2
-         * 2 - register
-		 */
-#define		PACKET3_RELEASE_MEM_MES_INTR_PIPE(x)    ((x) << 20)
-#define		PACKET3_RELEASE_MEM_MES_ACTION_ID(x)    ((x) << 22)
-#define		PACKET3_RELEASE_MEM_INT_SEL(x)		((x) << 24)
-		/* 0 - none
-		 * 1 - interrupt only (DATA_SEL = 0)
-		 * 2 - interrupt when data write is confirmed
-		 */
-#define		PACKET3_RELEASE_MEM_ADD_DOOREBLL_OFFSET(x)		(1 << 28)
-#define		PACKET3_RELEASE_MEM_DATA_SEL(x)		((x) << 29)
+#define		PACKET3_RELEASE_MEM__DST_SEL(x) ((((unsigned)(x)) & 0x3) << 16)
+#define		PACKET3_RELEASE_MEM__DST_SEL__MEMORY_CONTROLLER 0
+#define		PACKET3_RELEASE_MEM__DST_SEL__TC_L2 1
+#define		PACKET3_RELEASE_MEM__DST_SEL__QUQUE_WRITE_POINTER_REGISTER 2
+#define		PACKET3_RELEASE_MEM__DST_SEL__QUQUE_WRITE_POINTER_POLL_MASK_BIT 3
+#define		PACKET3_RELEASE_MEM__MES_INTR_PIPE(x)    ((((unsigned)(x)) & 0x3) << 20)
+#define		PACKET3_RELEASE_MEM__MES_ACTION_ID(x)    ((((unsigned)(x)) & 0x3) << 22)
+#define		PACKET3_RELEASE_MEM__MES_ACTION_ID__NO_MES_NOTIFICATION  0
+#define		PACKET3_RELEASE_MEM__MES_ACTION_ID__INTERRUPT_AND_FENCE  1
+#define		PACKET3_RELEASE_MEM__MES_ACTION_ID__INTERRUPT_NO_FENCE_THEN_ADDRESS_PAYLOAD  2
+#define		PACKET3_RELEASE_MEM__MES_ACTION_ID__INTERRUPT_AND_ADDRESS_PAYLOAD  3
+#define		PACKET3_RELEASE_MEM__INT_SEL(x)  ((((unsigned)(x)) & 0x7) << 24)
+#define		PACKET3_RELEASE_MEM__INT_SEL__NONE   0
+#define		PACKET3_RELEASE_MEM__INT_SEL__SEND_INTERRUPT_ONLY   1
+#define		PACKET3_RELEASE_MEM__INT_SEL__SEND_INTERRUPT_AFTER_WRITE_CONFIRM   2
+#define		PACKET3_RELEASE_MEM__INT_SEL__SEND_DATA_AND_WRITE_CONFIRM   3
+#define		PACKET3_RELEASE_MEM__INT_SEL__UNCONDITIONALLY_SEND_INT_CTXID   4
+#define		PACKET3_RELEASE_MEM__INT_SEL__UNCONDITIONALLY_SEND_INT_CTXID_BASED_ON_32_BIT_COMPARE   5
+#define		PACKET3_RELEASE_MEM__INT_SEL__UNCONDITIONALLY_SEND_INT_CTXID_BASED_ON_64_BIT_COMPARE   6
+#define		PACKET3_RELEASE_MEM__ADD_DOOREBLL_OFFSET(x)  ((((unsigned)(x)) & 0x1) << 28)
+#define		PACKET3_RELEASE_MEM__DATA_SEL(x)     ((((unsigned)(x)) & 0x7) << 29)
+#define		PACKET3_RELEASE_MEM__DATA_SEL__NONE 0
+#define		PACKET3_RELEASE_MEM__DATA_SEL__SEND_32_BIT_LOW 1
+#define		PACKET3_RELEASE_MEM__DATA_SEL__SEND_64_BIT_DATA 2
+#define		PACKET3_RELEASE_MEM__DATA_SEL__SEND_GPU_CLOCK_COUNTER 3
+#define		PACKET3_RELEASE_MEM__DATA_SEL__SEND_SYSTEM_CLOCK_COUNTER 4
 		/* 0 - discard
 		 * 1 - send low 32bit data
 		 * 2 - send 64bit data
 		 * 3 - send 64bit GPU counter value
 		 * 4 - send 64bit sys counter value
 		 */
+#define		PACKET3_RELEASE_MEM__ADDRESS_LO_32B(x)     ((((unsigned)(x)) & 0x3FFFFFFF) << 2)
+#define		PACKET3_RELEASE_MEM__ADDRESS_LO_64B(x)     ((((unsigned)(x)) & 0x1FFFFFFF) << 3)
+#define		PACKET3_RELEASE_MEM__ADDRESS_HI(x)     ((unsigned)(x))
+#define		PACKET3_RELEASE_MEM__DATA_LO(x)     ((unsigned)(x))
+#define		PACKET3_RELEASE_MEM__CMP_DATA_LO(x)     ((unsigned)(x))
+#define		PACKET3_RELEASE_MEM__DATA_HI(x)     ((unsigned)(x))
+#define		PACKET3_RELEASE_MEM__CMP_DATA_HI(x)     ((unsigned)(x))
+#define		PACKET3_RELEASE_MEM__INT_CTXID(x)     ((unsigned)(x))
 
 #define	PACKET3_PREAMBLE_CNTL				0x4A
 #              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
@@ -279,58 +438,78 @@
 #              define PACKET3_DMA_DATA_CMD_DIS_WC   (1 << 30)
 #define	PACKET3_CONTEXT_REG_RMW				0x51
 #define	PACKET3_ACQUIRE_MEM				0x58
-/* 1.  HEADER
- * 2.  COHER_CNTL [30:0]
- * 2.1 ENGINE_SEL [31:31]
- * 2.  COHER_SIZE [31:0]
- * 3.  COHER_SIZE_HI [7:0]
- * 4.  COHER_BASE_LO [31:0]
- * 5.  COHER_BASE_HI [23:0]
- * 7.  POLL_INTERVAL [15:0]
- * 8.  GCR_CNTL [18:0]
- */
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GLI_INV(x) ((x) << 0)
+/* 1.  HEADER */
+#define		PACKET3_ACQUIRE_MEM__COHER_SIZE(x) ((unsigned)(x))
+/* 3.  COHER_SIZE [31:0] */
+#define		PACKET3_ACQUIRE_MEM__COHER_SIZE_HI(x) ((((unsigned)(x)) & 0xFF) << 0)
+/* 4.  COHER_SIZE_HI [7:0] */
+#define		PACKET3_ACQUIRE_MEM__COHER_BASE_LO(x) ((unsigned)(x))
+/* 5.  COHER_BASE_LO [31:0] */
+#define		PACKET3_ACQUIRE_MEM__COHER_BASE_HI(x) ((((unsigned)(x)) & 0xFFFFFF) << 0)
+/* 6.  COHER_BASE_HI [23:0] */
+#define		PACKET3_ACQUIRE_MEM__POLL_INTERVAL(x) ((((unsigned)(x)) & 0xFFFF) << 0)
+/* 7.  POLL_INTERVAL [15:0] */
+#define		PACKET3_ACQUIRE_MEM__GCR_CNTL(x) ((((unsigned)(x)) & 0x7FFFF) << 0)
+/* 8.  GCR_CNTL [18:0] */
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV(x) ((x) << 0)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__NOP 0
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__ALL 1
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__RANGE 2
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GLI_INV__FIRST_LAST 3
 		/*
 		 * 0:NOP
 		 * 1:ALL
 		 * 2:RANGE
 		 * 3:FIRST_LAST
 		 */
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL1_RANGE(x) ((x) << 2)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE(x) ((x) << 2)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE__ALL   0
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE__RANGE   2
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL1_RANGE__FIRST_LAST   3
 		/*
 		 * 0:ALL
 		 * 1:reserved
 		 * 2:RANGE
 		 * 3:FIRST_LAST
 		 */
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_SCOPE(x) ((x) << 4)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE(x) ((x) << 4)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE__DEVICE    0
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE__SYSTEM    1
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_SCOPE__FORCE_ALL    2
         /*
          * 0:Device scope
          * 1:System scope
          * 2:Force INV/WB all
          * 3:Reserved
          */
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_WB(x) ((x) << 6)
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GLK_INV(x) ((x) << 7)
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GLV_INV(x) ((x) << 8)
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_US(x) ((x) << 10)
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_RANGE(x) ((x) << 11)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_WB(x) ((x) << 6)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GLK_INV(x) ((x) << 7)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GLV_INV(x) ((x) << 8)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_US(x) ((x) << 10)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE(x) ((x) << 11)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__ALL   0
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__VOL   1
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__RANGE   2
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_RANGE__FIRST_LAST   3
 		/*
 		 * 0:ALL
 		 * 1:VOL
 		 * 2:RANGE
 		 * 3:FIRST_LAST
 		 */
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_DISCARD(x)  ((x) << 13)
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_INV(x) ((x) << 14)
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_GL2_WB(x) ((x) << 15)
-#define 	PACKET3_ACQUIRE_MEM_GCR_CNTL_SEQ(x) ((x) << 16)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_DISCARD(x)  ((x) << 13)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_INV(x) ((x) << 14)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GL2_WB(x) ((x) << 15)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ(x) ((x) << 16)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ__PARALLET    0
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ__FORWARD    1
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__SEQ__REVERSE    2
 		/*
 		 * 0: PARALLEL
 		 * 1: FORWARD
 		 * 2: REVERSE
 		 */
-#define 	PACKET3_ACQUIRE_MEM_GCR_RANGE_IS_PA  (1 << 18)
+#define 	PACKET3_ACQUIRE_MEM__GCR_CNTL__GCR_RANGE_IS_PA  (1 << 18)
 #define	PACKET3_GEN_PDEPTE				0x5B
 #define	PACKET3_PRIME_UTCL2				0x5D
 #define	PACKET3_LOAD_UCONFIG_REG			0x5E
@@ -348,11 +527,19 @@
 #define	PACKET3_SET_SH_REG				0x76
 #define		PACKET3_SET_SH_REG_START			0x00002c00
 #define		PACKET3_SET_SH_REG_END				0x00003000
+#define 	PACKET3_SET_SH_REG__REG_OFFSET(x) ((((unsigned)(x)) & 0xFFFF) << 0)
+#define 	PACKET3_SET_SH_REG__VMID_SHIFT(x) ((((unsigned)(x)) & 0x1F) << 23)
+#define 	PACKET3_SET_SH_REG__INDEX(x) ((((unsigned)(x)) & 0xF) << 28)
+#define 	PACKET3_SET_SH_REG__REG_DATA(x) (((unsigned)(x)))
+#define 	PACKET3_SET_SH_REG__INDEX__DEFAULT 0
+#define 	PACKET3_SET_SH_REG__INDEX__INSERT_VMID 1
 #define	PACKET3_SET_SH_REG_OFFSET			0x77
 #define	PACKET3_SET_QUEUE_REG				0x78
 #define	PACKET3_SET_UCONFIG_REG				0x79
 #define		PACKET3_SET_UCONFIG_REG_START			0x0000c000
 #define		PACKET3_SET_UCONFIG_REG_END			0x0000c400
+#define 	PACKET3_SET_UCONFIG_REG__REG_OFFSET(x) ((((unsigned)(x)) & 0xFFFF) << 0)
+#define 	PACKET3_SET_UCONFIG_REG__REG_DATA(x) (((unsigned)(x)))
 #define	PACKET3_SET_UCONFIG_REG_INDEX			0x7A
 #define	PACKET3_DISPATCH_DRAW_PREAMBLE			0x8C
 #define	PACKET3_DISPATCH_DRAW				0x8D
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index 2b691452775bc..65b8497ad5f02 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -2819,6 +2819,7 @@ static void gfx_v7_0_mqd_init(struct amdgpu_device *adev,
 {
 	u64 hqd_gpu_addr;
 	u64 wb_gpu_addr;
+	u32 tmp;
 
 	/* init the mqd struct */
 	memset(mqd, 0, sizeof(struct cik_mqd));
@@ -2923,7 +2924,11 @@ static void gfx_v7_0_mqd_init(struct amdgpu_device *adev,
 	mqd->cp_hqd_atomic1_preop_lo = RREG32(mmCP_HQD_ATOMIC1_PREOP_LO);
 	mqd->cp_hqd_atomic1_preop_hi = RREG32(mmCP_HQD_ATOMIC1_PREOP_HI);
 	mqd->cp_hqd_pq_rptr = RREG32(mmCP_HQD_PQ_RPTR);
-	mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
+	tmp = RREG32(mmCP_HQD_QUANTUM);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10);
+	mqd->cp_hqd_quantum = tmp;
 	mqd->cp_hqd_pipe_priority = RREG32(mmCP_HQD_PIPE_PRIORITY);
 	mqd->cp_hqd_queue_priority = RREG32(mmCP_HQD_QUEUE_PRIORITY);
 	mqd->cp_hqd_iq_rptr = RREG32(mmCP_HQD_IQ_RPTR);
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index a6b4c8f41dc11..130196859ff3f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -4532,7 +4532,11 @@ static int gfx_v8_0_mqd_init(struct amdgpu_ring *ring)
 
 	/* set static priority for a queue/ring */
 	gfx_v8_0_mqd_set_priority(ring, mqd);
-	mqd->cp_hqd_quantum = RREG32(mmCP_HQD_QUANTUM);
+	tmp = RREG32(mmCP_HQD_QUANTUM);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 10);
+	mqd->cp_hqd_quantum = tmp;
 
 	/* map_queues packet doesn't need activate the queue,
 	 * so only kiq need set this field.
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
index 86c7c2a429b71..bf270e605949f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c
@@ -3667,7 +3667,11 @@ static int gfx_v9_0_mqd_init(struct amdgpu_ring *ring)
 
 	/* set static priority for a queue/ring */
 	gfx_v9_0_mqd_set_priority(ring, mqd);
-	mqd->cp_hqd_quantum = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
+	tmp = RREG32_SOC15(GC, 0, mmCP_HQD_QUANTUM);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
+	mqd->cp_hqd_quantum = tmp;
 
 	/* map_queues packet doesn't need activate the queue,
 	 * so only kiq need set this field.
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
index ad4d442e7345e..9f76e1af8a553 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v9_4_3.c
@@ -1943,7 +1943,11 @@ static int gfx_v9_4_3_xcc_mqd_init(struct amdgpu_ring *ring, int xcc_id)
 
 	/* set static priority for a queue/ring */
 	gfx_v9_4_3_mqd_set_priority(ring, mqd);
-	mqd->cp_hqd_quantum = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_QUANTUM);
+	tmp = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_HQD_QUANTUM);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_EN, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_SCALE, 1);
+	tmp = REG_SET_FIELD(tmp, CP_HQD_QUANTUM, QUANTUM_DURATION, 1);
+	mqd->cp_hqd_quantum = tmp;
 
 	/* map_queues packet doesn't need activate the queue,
 	 * so only kiq need set this field.
@@ -2370,11 +2374,78 @@ static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block)
 	return r;
 }
 
+static int gfx_v9_4_3_perf_monitor_ptl_init(struct amdgpu_device *adev, bool enable)
+{
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+	uint32_t ptl_state = enable ? 1 : 0;
+	uint32_t fmt1, fmt2;
+	int r;
+
+	if (!adev->psp.funcs)
+		return -EOPNOTSUPP;
+
+	if (!ptl->hw_supported) {
+		fmt1 = GFX_FTYPE_VECTOR;
+		fmt2 = GFX_FTYPE_F8;
+	} else {
+		fmt1 = ptl->fmt1;
+		fmt2 = ptl->fmt2;
+	}
+
+	/* initialize PTL with default formats: GFX_FTYPE_VECTOR & GFX_FTYPE_F8 */
+	r = amdgpu_ptl_perf_monitor_ctrl(adev, PSP_PTL_PERF_MON_SET, &ptl_state,
+							&fmt1, &fmt2);
+	if (r)
+		return r;
+
+	ptl->hw_supported = true;
+
+	atomic_set(&ptl->disable_ref, 0);
+	if (!enable && !amdgpu_in_reset(adev) && !adev->in_suspend) {
+		dev_dbg(adev->dev,
+			"PTL disabled (amdgpu.ptl=%d)\
+			To enable, set amdgpu.ptl=1 via module param or kernel cmdline\n",
+			amdgpu_ptl);
+		set_bit(AMDGPU_PTL_DISABLE_SYSFS, ptl->disable_bitmap);
+	}
+
+	return 0;
+}
+
+static int gfx_v9_4_3_ptl_hw_init(struct amdgpu_device *adev)
+{
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+	bool enable;
+
+	switch (amdgpu_ptl) {
+	case 1:
+		enable = true;
+		break;
+	case 2:
+		/* Permanently disabled - cannot be re-enabled */
+		enable = false;
+		ptl->permanently_disabled = true;
+		break;
+	case -1:
+	case 0:
+	default:
+		enable = false;
+		break;
+	}
+
+	gfx_v9_4_3_perf_monitor_ptl_init(adev, enable ? 1 : 0);
+
+	return 0;
+}
+
 static int gfx_v9_4_3_hw_fini(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, num_xcc;
 
+	if (adev->psp.ptl.hw_supported && !amdgpu_in_reset(adev))
+		gfx_v9_4_3_perf_monitor_ptl_init(adev, false);
+
 	amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0);
 	amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0);
@@ -2401,12 +2472,21 @@ static bool gfx_v9_4_3_is_idle(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, num_xcc;
+	u32 gc_ip_version;
 
 	num_xcc = NUM_XCC(adev->gfx.xcc_mask);
+	gc_ip_version = amdgpu_ip_version(adev, GC_HWIP, 0);
+
 	for (i = 0; i < num_xcc; i++) {
-		if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), regGRBM_STATUS),
-					GRBM_STATUS, GUI_ACTIVE))
-			return false;
+		if (gc_ip_version == IP_VERSION(9, 4, 4)) {
+			if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), regGRBM_STATUS),
+					  GRBM_STATUS, SPI_BUSY))
+				return false;
+		} else {
+			if (REG_GET_FIELD(RREG32_SOC15(GC, GET_INST(GC, i), regGRBM_STATUS),
+					  GRBM_STATUS, GUI_ACTIVE))
+				return false;
+		}
 	}
 	return true;
 }
@@ -2549,6 +2629,8 @@ static int gfx_v9_4_3_late_init(struct amdgpu_ip_block *ip_block)
 	    adev->gfx.ras->enable_watchdog_timer)
 		adev->gfx.ras->enable_watchdog_timer(adev);
 
+	gfx_v9_4_3_ptl_hw_init(adev);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
index e1ace7d44ffdf..8523833a74fbf 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v10_0.c
@@ -707,20 +707,16 @@ static int gmc_v10_0_mc_init(struct amdgpu_device *adev)
 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
 
 	/* set the gart size */
-	if (amdgpu_gart_size == -1) {
-		switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
-		default:
-			adev->gmc.gart_size = 512ULL << 20;
-			break;
-		case IP_VERSION(10, 3, 1):   /* DCE SG support */
-		case IP_VERSION(10, 3, 3):   /* DCE SG support */
-		case IP_VERSION(10, 3, 6):   /* DCE SG support */
-		case IP_VERSION(10, 3, 7):   /* DCE SG support */
-			adev->gmc.gart_size = 1024ULL << 20;
-			break;
-		}
-	} else {
-		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(10, 3, 1):   /* DCE SG support */
+	case IP_VERSION(10, 3, 3):   /* DCE SG support */
+	case IP_VERSION(10, 3, 6):   /* DCE SG support */
+	case IP_VERSION(10, 3, 7):   /* DCE SG support */
+		amdgpu_gmc_set_gart_size(adev, SZ_1G);
+		break;
+	default:
+		amdgpu_gmc_set_gart_size(adev, SZ_512M);
+		break;
 	}
 
 	gmc_v10_0_vram_gtt_location(adev, &adev->gmc);
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
index 94d6631ce0bce..16388e3caea32 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v11_0.c
@@ -709,10 +709,7 @@ static int gmc_v11_0_mc_init(struct amdgpu_device *adev)
 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
 
 	/* set the gart size */
-	if (amdgpu_gart_size == -1)
-		adev->gmc.gart_size = 512ULL << 20;
-	else
-		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
+	amdgpu_gmc_set_gart_size(adev, SZ_512M);
 
 	gmc_v11_0_vram_gtt_location(adev, &adev->gmc);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
index e10ac9788d13a..586703ec0dfa0 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_0.c
@@ -729,8 +729,10 @@ static int gmc_v12_0_mc_init(struct amdgpu_device *adev)
 	int r;
 
 	if (adev->gmc.xgmi.connected_to_cpu)
-		adev->gmc.mc_vram_size =
-			adev->gmc.xgmi.node_segment_size * adev->gmc.xgmi.num_physical_nodes;
+		/* On A+A, manage driver allocation range to the local
+		 * node segment and prevent allocations on remote HBM.
+		 */
+		adev->gmc.mc_vram_size = adev->gmc.xgmi.node_segment_size;
 	else
 		adev->gmc.mc_vram_size =
 			adev->nbio.funcs->get_memsize(adev) * 1024ULL * 1024ULL;
@@ -763,10 +765,7 @@ static int gmc_v12_0_mc_init(struct amdgpu_device *adev)
 		adev->gmc.visible_vram_size = adev->gmc.real_vram_size;
 
 	/* set the gart size */
-	if (amdgpu_gart_size == -1) {
-		adev->gmc.gart_size = 512ULL << 20;
-	} else
-		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
+	amdgpu_gmc_set_gart_size(adev, SZ_512M);
 
 	gmc_v12_0_vram_gtt_location(adev, &adev->gmc);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
index 7ea7b9c30bca9..855cd29cbffaa 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v12_1.c
@@ -112,6 +112,8 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
 	const char *hub_name;
 	int ret, xcc_id = 0;
 	uint32_t status = 0;
+	const char *die_name;
+	char die_name_buf[32];
 	u64 addr;
 
 	node_id = entry->node_id;
@@ -201,6 +203,17 @@ static int gmc_v12_1_process_interrupt(struct amdgpu_device *adev,
 	dev_err(adev->dev, "  in page starting at address 0x%016llx from IH client %d (%s)\n",
 		addr, entry->client_id, soc_v1_0_ih_clientid_name[entry->client_id]);
 
+	if (adev->irq.ih_funcs &&
+	    adev->irq.ih_funcs->node_id_to_die_name) {
+		die_name = adev->irq.ih_funcs->node_id_to_die_name(adev, node_id,
+								   die_name_buf,
+								   sizeof(die_name_buf));
+		if (die_name)
+			dev_err(adev->dev,
+				"  cookie node_id %d fault from die %s\n",
+				node_id, die_name);
+	}
+
 	if (amdgpu_sriov_vf(adev))
 		return 0;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
index cc272a96fcef0..af6944d2d330d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v6_0.c
@@ -328,24 +328,18 @@ static int gmc_v6_0_mc_init(struct amdgpu_device *adev)
 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
 
 	/* set the gart size */
-	if (amdgpu_gart_size == -1) {
-		switch (adev->asic_type) {
-		case CHIP_HAINAN:    /* no MM engines */
-		default:
-			adev->gmc.gart_size = 256ULL << 20;
-			break;
-		case CHIP_VERDE:    /* UVD, VCE do not support GPUVM */
-		case CHIP_TAHITI:   /* UVD, VCE do not support GPUVM */
-		case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */
-		case CHIP_OLAND:    /* UVD, VCE do not support GPUVM */
-			adev->gmc.gart_size = 1024ULL << 20;
-			break;
-		}
-	} else {
-		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
+	switch (adev->asic_type) {
+	case CHIP_VERDE:    /* UVD, VCE do not support GPUVM */
+	case CHIP_TAHITI:   /* UVD, VCE do not support GPUVM */
+	case CHIP_PITCAIRN: /* UVD, VCE do not support GPUVM */
+	case CHIP_OLAND:    /* UVD, VCE do not support GPUVM */
+		amdgpu_gmc_set_gart_size(adev, SZ_1G);
+		break;
+	case CHIP_HAINAN:    /* no MM engines */
+	default:
+		amdgpu_gmc_set_gart_size(adev, SZ_256M);
+		break;
 	}
-
-	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
 	gmc_v6_0_vram_gtt_location(adev, &adev->gmc);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index bb16ba2ef6fd9..96023b4f5f923 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -394,27 +394,21 @@ static int gmc_v7_0_mc_init(struct amdgpu_device *adev)
 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
 
 	/* set the gart size */
-	if (amdgpu_gart_size == -1) {
-		switch (adev->asic_type) {
-		case CHIP_TOPAZ:     /* no MM engines */
-		default:
-			adev->gmc.gart_size = 256ULL << 20;
-			break;
+	switch (adev->asic_type) {
 #ifdef CONFIG_DRM_AMDGPU_CIK
-		case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */
-		case CHIP_HAWAII:  /* UVD, VCE do not support GPUVM */
-		case CHIP_KAVERI:  /* UVD, VCE do not support GPUVM */
-		case CHIP_KABINI:  /* UVD, VCE do not support GPUVM */
-		case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */
-			adev->gmc.gart_size = 1024ULL << 20;
-			break;
+	case CHIP_BONAIRE: /* UVD, VCE do not support GPUVM */
+	case CHIP_HAWAII:  /* UVD, VCE do not support GPUVM */
+	case CHIP_KAVERI:  /* UVD, VCE do not support GPUVM */
+	case CHIP_KABINI:  /* UVD, VCE do not support GPUVM */
+	case CHIP_MULLINS: /* UVD, VCE do not support GPUVM */
+		amdgpu_gmc_set_gart_size(adev, SZ_1G);
+		break;
 #endif
-		}
-	} else {
-		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
+	case CHIP_TOPAZ:     /* no MM engines */
+	default:
+		amdgpu_gmc_set_gart_size(adev, SZ_256M);
+		break;
 	}
-
-	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
 	gmc_v7_0_vram_gtt_location(adev, &adev->gmc);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index a59174f6bcc18..b76fc4441eb79 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -585,27 +585,21 @@ static int gmc_v8_0_mc_init(struct amdgpu_device *adev)
 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
 
 	/* set the gart size */
-	if (amdgpu_gart_size == -1) {
-		switch (adev->asic_type) {
-		case CHIP_POLARIS10: /* all engines support GPUVM */
-		case CHIP_POLARIS11: /* all engines support GPUVM */
-		case CHIP_POLARIS12: /* all engines support GPUVM */
-		case CHIP_VEGAM:     /* all engines support GPUVM */
-		default:
-			adev->gmc.gart_size = 256ULL << 20;
-			break;
-		case CHIP_TONGA:   /* UVD, VCE do not support GPUVM */
-		case CHIP_FIJI:    /* UVD, VCE do not support GPUVM */
-		case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */
-		case CHIP_STONEY:  /* UVD does not support GPUVM, DCE SG support */
-			adev->gmc.gart_size = 1024ULL << 20;
-			break;
-		}
-	} else {
-		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
+	switch (adev->asic_type) {
+	case CHIP_TONGA:   /* UVD, VCE do not support GPUVM */
+	case CHIP_FIJI:    /* UVD, VCE do not support GPUVM */
+	case CHIP_CARRIZO: /* UVD, VCE do not support GPUVM, DCE SG support */
+	case CHIP_STONEY:  /* UVD does not support GPUVM, DCE SG support */
+		amdgpu_gmc_set_gart_size(adev, SZ_1G);
+		break;
+	case CHIP_POLARIS10: /* all engines support GPUVM */
+	case CHIP_POLARIS11: /* all engines support GPUVM */
+	case CHIP_POLARIS12: /* all engines support GPUVM */
+	case CHIP_VEGAM:     /* all engines support GPUVM */
+	default:
+		amdgpu_gmc_set_gart_size(adev, SZ_256M);
+		break;
 	}
-
-	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
 	gmc_v8_0_vram_gtt_location(adev, &adev->gmc);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
index e7b78027002be..ced0f3941863f 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c
@@ -1204,21 +1204,6 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
 {
 	int local_node, nid;
 
-	/* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes. Local system
-	 * memory can use more efficient MTYPEs.
-	 */
-	if (!(adev->flags & AMD_IS_APU) ||
-	    amdgpu_ip_version(adev, GC_HWIP, 0) != IP_VERSION(9, 4, 3))
-		return;
-
-	/* Only direct-mapped memory allows us to determine the NUMA node from
-	 * the DMA address.
-	 */
-	if (!adev->ram_is_direct_mapped) {
-		dev_dbg_ratelimited(adev->dev, "RAM is not direct mapped\n");
-		return;
-	}
-
 	/* MTYPE_NC is the same default and can be overridden.
 	 * MTYPE_UC will be present if the memory is extended-coherent
 	 * and can also be overridden.
@@ -1231,11 +1216,7 @@ static void gmc_v9_0_override_vm_pte_flags(struct amdgpu_device *adev,
 		return;
 	}
 
-	/* FIXME: Only supported on native mode for now. For carve-out, the
-	 * NUMA affinity of the GPU/VM needs to come from the PCI info because
-	 * memory partitions are not associated with different NUMA nodes.
-	 */
-	if (adev->gmc.is_app_apu && vm->mem_id >= 0) {
+	if (vm->mem_id >= 0) {
 		local_node = adev->gmc.mem_partitions[vm->mem_id].numa.node;
 	} else {
 		dev_dbg_ratelimited(adev->dev, "Only native mode APU is supported.\n");
@@ -1344,6 +1325,20 @@ static const struct amdgpu_gmc_funcs gmc_v9_0_gmc_funcs = {
 static void gmc_v9_0_set_gmc_funcs(struct amdgpu_device *adev)
 {
 	adev->gmc.gmc_funcs = &gmc_v9_0_gmc_funcs;
+
+	/* Only GFX 9.4.3 APUs associate GPUs with NUMA nodes, local system
+	 * memory can use more efficient MTYPEs.
+	 *
+	 * APUs mapping system memory may need different MTYPEs on different
+	 * NUMA nodes.
+	 *
+	 * Only direct-mapped memory allows us to determine the NUMA node from
+	 * the DMA address.
+	 */
+	adev->gmc.override_pte = adev->gmc.is_app_apu &&
+				 num_possible_nodes() > 1 &&
+				 amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) &&
+				 adev->ram_is_direct_mapped;
 }
 
 static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev)
@@ -1736,31 +1731,25 @@ static int gmc_v9_0_mc_init(struct amdgpu_device *adev)
 	adev->gmc.visible_vram_size = adev->gmc.aper_size;
 
 	/* set the gart size */
-	if (amdgpu_gart_size == -1) {
-		switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
-		case IP_VERSION(9, 0, 1):  /* all engines support GPUVM */
-		case IP_VERSION(9, 2, 1):  /* all engines support GPUVM */
-		case IP_VERSION(9, 4, 0):
-		case IP_VERSION(9, 4, 1):
-		case IP_VERSION(9, 4, 2):
-		case IP_VERSION(9, 4, 3):
-		case IP_VERSION(9, 4, 4):
-		case IP_VERSION(9, 5, 0):
-		default:
-			adev->gmc.gart_size = 512ULL << 20;
-			break;
-		case IP_VERSION(9, 1, 0):   /* DCE SG support */
-		case IP_VERSION(9, 2, 2):   /* DCE SG support */
-		case IP_VERSION(9, 3, 0):
-			adev->gmc.gart_size = 1024ULL << 20;
-			break;
-		}
-	} else {
-		adev->gmc.gart_size = (u64)amdgpu_gart_size << 20;
+	switch (amdgpu_ip_version(adev, GC_HWIP, 0)) {
+	case IP_VERSION(9, 1, 0):   /* DCE SG support */
+	case IP_VERSION(9, 2, 2):   /* DCE SG support */
+	case IP_VERSION(9, 3, 0):
+		amdgpu_gmc_set_gart_size(adev, SZ_1G);
+		break;
+	case IP_VERSION(9, 0, 1):  /* all engines support GPUVM */
+	case IP_VERSION(9, 2, 1):  /* all engines support GPUVM */
+	case IP_VERSION(9, 4, 0):
+	case IP_VERSION(9, 4, 1):
+	case IP_VERSION(9, 4, 2):
+	case IP_VERSION(9, 4, 3):
+	case IP_VERSION(9, 4, 4):
+	case IP_VERSION(9, 5, 0):
+	default:
+		amdgpu_gmc_set_gart_size(adev, SZ_512M);
+		break;
 	}
 
-	adev->gmc.gart_size += adev->pm.smu_prv_buffer_size;
-
 	gmc_v9_0_vram_gtt_location(adev, &adev->gmc);
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
index 1fbe904f4223b..6de9e87e04e1a 100644
--- a/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/ih_v7_0.c
@@ -798,6 +798,43 @@ static void ih_v7_0_get_clockgating_state(struct amdgpu_ip_block *ip_block, u64
 	return;
 }
 
+/*
+ * ih_v7_0_node_id_to_die_name - Decode IH cookie node_id to a die name string
+ *
+ * Currently, only applies to IH v7_1. For other IH versions returns NULL.
+ *
+ * IH v7_1 node_id encoding:
+ * node_id[N:3] = MID index
+ * node_id[2:0] = sub-slot: 0=MID, 1=AID, 2-5=AID.XCD, 6-7=RSV
+ */
+static const char *ih_v7_0_node_id_to_die_name(struct amdgpu_device *adev,
+					       unsigned int node_id,
+					       char *buf, size_t size)
+{
+	int mid_id, sub_slot;
+
+	/* Node ID to die name decoding is only defined for IH v7_1 currenlty. */
+	if (amdgpu_ip_version(adev, OSSSYS_HWIP, 0) != IP_VERSION(7, 1, 0))
+		return NULL;
+
+	mid_id = node_id >> 3;
+	sub_slot = node_id & 0x7;
+
+	if (mid_id > 1)
+		return "UNKNOWN";
+
+	if (sub_slot == 0)
+		snprintf(buf, size, "MID%d", mid_id);
+	else if (sub_slot == 1)
+		snprintf(buf, size, "AID%d", mid_id);
+	else if (sub_slot <= 5)
+		snprintf(buf, size, "AID%d.XCD%d", mid_id, sub_slot - 2);
+	else
+		snprintf(buf, size, "RSV");
+
+	return buf;
+}
+
 static const struct amd_ip_funcs ih_v7_0_ip_funcs = {
 	.name = "ih_v7_0",
 	.early_init = ih_v7_0_early_init,
@@ -819,7 +856,8 @@ static const struct amdgpu_ih_funcs ih_v7_0_funcs = {
 	.get_wptr = ih_v7_0_get_wptr,
 	.decode_iv = amdgpu_ih_decode_iv_helper,
 	.decode_iv_ts = amdgpu_ih_decode_iv_ts_helper,
-	.set_rptr = ih_v7_0_set_rptr
+	.set_rptr = ih_v7_0_set_rptr,
+	.node_id_to_die_name = ih_v7_0_node_id_to_die_name,
 };
 
 static void ih_v7_0_set_interrupt_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
index 539785a51f69f..fe20cf2b94545 100644
--- a/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/imu_v12_1.c
@@ -140,27 +140,17 @@ static int imu_v12_1_switch_compute_partition(struct amdgpu_device *adev,
 	int ret;
 
 	if (adev->psp.funcs) {
-		/*TODO: revisit asp interface once it's avaialble */
-		ret = psp_spatial_partition(&adev->psp,
-					    NUM_XCC(adev->gfx.xcc_mask) /
-						    num_xccs_per_xcp);
+		ret = psp_spatial_partition(&adev->psp, compute_partition_mode);
 		if (ret)
 			return ret;
 	}
 
 	adev->gfx.num_xcc_per_xcp = num_xccs_per_xcp;
-
 	return 0;
 }
 
-static void imu_v12_1_init_mcm_addr_lut(struct amdgpu_device *adev)
-{
-	/* todo: fill in when interface is ready */
-}
-
 const struct amdgpu_imu_funcs gfx_v12_1_imu_funcs = {
 	.init_microcode = imu_v12_1_init_microcode,
 	.load_microcode = imu_v12_1_load_microcode,
 	.switch_compute_partition = imu_v12_1_switch_compute_partition,
-	.init_mcm_addr_lut = imu_v12_1_init_mcm_addr_lut,
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
index 13a6e24c624a2..5208312e7017c 100644
--- a/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/jpeg_v2_5.c
@@ -149,7 +149,7 @@ static int jpeg_v2_5_sw_init(struct amdgpu_ip_block *ip_block)
 		else
 			ring->vm_hub = AMDGPU_MMHUB0(0);
 		ring->doorbell_index = (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + 1 + 8 * i;
-		sprintf(ring->name, "jpeg_dec_%d", i);
+		snprintf(ring->name, sizeof(ring->name), "jpeg_dec_%d", i);
 		r = amdgpu_ring_init(adev, ring, 512, &adev->jpeg.inst[i].irq,
 				     0, AMDGPU_RING_PRIO_DEFAULT, NULL);
 		if (r)
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
index 5b4121ddc78c6..9bb8f73305377 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_userqueue.c
@@ -245,7 +245,7 @@ static int mes_userq_detect_and_reset(struct amdgpu_device *adev,
 
 	if (found_hung_queue) {
 		/* Resume scheduling after hang recovery */
-		r = amdgpu_mes_resume(adev);
+		r = amdgpu_mes_resume(adev, input.xcc_id);
 	}
 
 	return r;
diff --git a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
index cec8012781265..b169e577e5838 100644
--- a/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/mes_v12_1.c
@@ -44,8 +44,11 @@ static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block);
 static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id);
 static int mes_v12_1_kiq_hw_fini(struct amdgpu_device *adev, uint32_t xcc_id);
 static int mes_v12_1_self_test(struct amdgpu_device *adev, int xcc_id);
+static int mes_v12_1_setup_coop_mode(struct amdgpu_device *adev, int xcc_id);
 
 #define MES_EOP_SIZE   2048
+#define MES12_HUNG_DB_OFFSET_ARRAY_SIZE 8 /* [0:3] = db offset [4:7] hqd info */
+#define MES12_HUNG_HQD_INFO_OFFSET      4
 
 #define regCP_HQD_IB_CONTROL_MES_12_1_DEFAULT 0x100000
 #define XCC_MID_MASK 0x41000000
@@ -229,7 +232,7 @@ static int mes_v12_1_submit_pkt_and_poll_completion(struct amdgpu_mes *mes,
 			xcc_id, pipe, x_pkt->header.opcode);
 
 	r = amdgpu_fence_wait_polling(ring, seq, timeout);
-	if (r < 1 || !*status_ptr) {
+	if (r < 1 || !lower_32_bits(*status_ptr)) {
 		if (misc_op_str)
 			dev_err(adev->dev,
 				"MES(%d, %d) failed to respond to msg=%s (%s)\n",
@@ -468,13 +471,44 @@ static int mes_v12_1_unmap_legacy_queue(struct amdgpu_mes *mes,
 static int mes_v12_1_suspend_gang(struct amdgpu_mes *mes,
 				  struct mes_suspend_gang_input *input)
 {
-	return 0;
+	union MESAPI__SUSPEND mes_suspend_gang_pkt;
+
+	memset(&mes_suspend_gang_pkt, 0, sizeof(mes_suspend_gang_pkt));
+
+	mes_suspend_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_suspend_gang_pkt.header.opcode = MES_SCH_API_SUSPEND;
+	mes_suspend_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_suspend_gang_pkt.suspend_all_gangs = input->suspend_all_gangs;
+	mes_suspend_gang_pkt.suspend_all_sdma_gangs = input->suspend_all_sdma_gangs;
+	mes_suspend_gang_pkt.gang_context_addr = input->gang_context_addr;
+	mes_suspend_gang_pkt.suspend_fence_addr = input->suspend_fence_addr;
+	mes_suspend_gang_pkt.suspend_fence_value = input->suspend_fence_value;
+
+	/* Suspend gang is handled by master MES */
+	return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE,
+			&mes_suspend_gang_pkt, sizeof(mes_suspend_gang_pkt),
+			offsetof(union MESAPI__SUSPEND, api_status));
 }
 
 static int mes_v12_1_resume_gang(struct amdgpu_mes *mes,
 				 struct mes_resume_gang_input *input)
 {
-	return 0;
+	union MESAPI__RESUME mes_resume_gang_pkt;
+
+	memset(&mes_resume_gang_pkt, 0, sizeof(mes_resume_gang_pkt));
+
+	mes_resume_gang_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_resume_gang_pkt.header.opcode = MES_SCH_API_RESUME;
+	mes_resume_gang_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_resume_gang_pkt.resume_all_gangs = input->resume_all_gangs;
+	mes_resume_gang_pkt.gang_context_addr = input->gang_context_addr;
+
+	/* Resume gang is handled by master MES */
+	return mes_v12_1_submit_pkt_and_poll_completion(mes, input->xcc_id, AMDGPU_MES_SCHED_PIPE,
+			&mes_resume_gang_pkt, sizeof(mes_resume_gang_pkt),
+			offsetof(union MESAPI__RESUME, api_status));
 }
 
 static int mes_v12_1_query_sched_status(struct amdgpu_mes *mes,
@@ -621,11 +655,15 @@ static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes,
 	mes_set_hw_res_1_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
 	mes_set_hw_res_1_pkt.mes_kiq_unmap_timeout = 100;
 
-	if (mes->enable_coop_mode && pipe == AMDGPU_MES_SCHED_PIPE) {
+	/* From version 0x74 above, pipe1 support use shared command buffer
+	   to distribute some tasks on individual XCCs*/
+	if (mes->enable_coop_mode &&
+	    ((pipe == AMDGPU_MES_SCHED_PIPE) ||
+	    ((mes->kiq_version & AMDGPU_MES_VERSION_MASK) >= 0x74))) {
 		master_xcc_id = mes->master_xcc_ids[inst];
 		mes_set_hw_res_1_pkt.mes_coop_mode = 1;
 		mes_set_hw_res_1_pkt.coop_sch_shared_mc_addr =
-			mes->shared_cmd_buf_gpu_addr[master_xcc_id];
+			mes->shared_cmd_buf_gpu_addr[master_xcc_id + pipe];
 	}
 
 	return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
@@ -633,21 +671,10 @@ static int mes_v12_1_set_hw_resources_1(struct amdgpu_mes *mes,
 			offsetof(union MESAPI_SET_HW_RESOURCES_1, api_status));
 }
 
-static void mes_v12_1_set_gfx_hqd_mask(union MESAPI_SET_HW_RESOURCES *pkt)
-{
-	/*
-	 * GFX V12 has only one GFX pipe, but 8 queues in it.
-	 * GFX pipe 0 queue 0 is being used by Kernel queue.
-	 * Set GFX pipe 0 queue 1-7 for MES scheduling
-	 * mask = 1111 1110b
-	 */
-	pkt->gfx_hqd_mask[0] = 0xFE;
-}
-
 static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
 					int pipe, int xcc_id)
 {
-	int i;
+	int i, status;
 	struct amdgpu_device *adev = mes->adev;
 	union MESAPI_SET_HW_RESOURCES mes_set_hw_res_pkt;
 
@@ -667,7 +694,9 @@ static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
 			mes_set_hw_res_pkt.compute_hqd_mask[i] =
 				mes->compute_hqd_mask[i];
 
-		mes_v12_1_set_gfx_hqd_mask(&mes_set_hw_res_pkt);
+		for (i = 0; i < MAX_GFX_PIPES; i++)
+			mes_set_hw_res_pkt.gfx_hqd_mask[i] =
+				mes->gfx_hqd_mask[i];
 
 		for (i = 0; i < MAX_SDMA_PIPES; i++)
 			mes_set_hw_res_pkt.sdma_hqd_mask[i] =
@@ -715,9 +744,23 @@ static int mes_v12_1_set_hw_resources(struct amdgpu_mes *mes,
 	if (adev->enforce_isolation[0] == AMDGPU_ENFORCE_ISOLATION_ENABLE)
 		mes_set_hw_res_pkt.limit_single_process = 1;
 
-	return mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
+	status = mes_v12_1_submit_pkt_and_poll_completion(mes, xcc_id, pipe,
 			&mes_set_hw_res_pkt, sizeof(mes_set_hw_res_pkt),
 			offsetof(union MESAPI_SET_HW_RESOURCES, api_status));
+
+	/* get MES scheduler versions */
+	mutex_lock(&adev->srbm_mutex);
+	soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
+
+	if (pipe == AMDGPU_MES_SCHED_PIPE)
+		adev->mes.sched_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
+	else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
+		adev->mes.kiq_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
+
+	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
+	mutex_unlock(&adev->srbm_mutex);
+
+	return status;
 }
 
 static void mes_v12_1_init_aggregated_doorbell(struct amdgpu_mes *mes,
@@ -837,6 +880,33 @@ static int mes_v12_1_reset_legacy_queue(struct amdgpu_mes *mes,
 }
 #endif
 
+static int mes_v12_1_detect_and_reset_hung_queues(struct amdgpu_mes *mes,
+						  struct mes_detect_and_reset_queue_input *input)
+{
+	union MESAPI__RESET mes_reset_queue_pkt;
+
+	memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt));
+
+	mes_reset_queue_pkt.header.type = MES_API_TYPE_SCHEDULER;
+	mes_reset_queue_pkt.header.opcode = MES_SCH_API_RESET;
+	mes_reset_queue_pkt.header.dwsize = API_FRAME_SIZE_IN_DWORDS;
+
+	mes_reset_queue_pkt.queue_type =
+		convert_to_mes_queue_type(input->queue_type);
+	mes_reset_queue_pkt.doorbell_offset_addr =
+		mes->hung_queue_db_array_gpu_addr[0];
+
+	if (input->detect_only)
+		mes_reset_queue_pkt.hang_detect_only = 1;
+	else
+		mes_reset_queue_pkt.hang_detect_then_reset = 1;
+
+	return mes_v12_1_submit_pkt_and_poll_completion(mes,
+			input->xcc_id, AMDGPU_MES_SCHED_PIPE,
+			&mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt),
+			offsetof(union MESAPI__RESET, api_status));
+}
+
 static int mes_v12_inv_tlb_convert_hub_id(uint8_t id)
 {
 	/*
@@ -894,6 +964,7 @@ static const struct amdgpu_mes_funcs mes_v12_1_funcs = {
 	.resume_gang = mes_v12_1_resume_gang,
 	.misc_op = mes_v12_1_misc_op,
 	.reset_hw_queue = mes_v12_1_reset_hw_queue,
+	.detect_and_reset_hung_queues = mes_v12_1_detect_and_reset_hung_queues,
 	.invalidate_tlbs_pasid = mes_v12_1_inv_tlbs_pasid,
 };
 
@@ -1157,9 +1228,6 @@ static int mes_v12_1_allocate_shared_cmd_buf(struct amdgpu_device *adev,
 {
 	int r, inst = MES_PIPE_INST(xcc_id, pipe);
 
-	if (pipe == AMDGPU_MES_KIQ_PIPE)
-		return 0;
-
 	r = amdgpu_bo_create_kernel(adev, PAGE_SIZE, PAGE_SIZE,
 				    AMDGPU_GEM_DOMAIN_VRAM,
 				    &adev->mes.shared_cmd_buf_obj[inst],
@@ -1405,18 +1473,6 @@ static int mes_v12_1_queue_init(struct amdgpu_device *adev,
 		mes_v12_1_queue_init_register(ring, xcc_id);
 	}
 
-	/* get MES scheduler/KIQ versions */
-	mutex_lock(&adev->srbm_mutex);
-	soc_v1_0_grbm_select(adev, 3, pipe, 0, 0, GET_INST(GC, xcc_id));
-
-	if (pipe == AMDGPU_MES_SCHED_PIPE)
-		adev->mes.sched_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
-	else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq)
-		adev->mes.kiq_version = RREG32_SOC15(GC, GET_INST(GC, xcc_id), regCP_MES_GP3_LO);
-
-	soc_v1_0_grbm_select(adev, 0, 0, 0, 0, GET_INST(GC, xcc_id));
-	mutex_unlock(&adev->srbm_mutex);
-
 	return 0;
 }
 
@@ -1723,6 +1779,10 @@ static int mes_v12_1_kiq_hw_init(struct amdgpu_device *adev, uint32_t xcc_id)
 		goto failure;
 
 	if (adev->enable_uni_mes) {
+		r = mes_v12_1_setup_coop_mode(adev, xcc_id);
+		if (r)
+			goto failure;
+
 		r = mes_v12_1_set_hw_resources(&adev->mes,
 						 AMDGPU_MES_KIQ_PIPE, xcc_id);
 		if (r)
@@ -1845,9 +1905,6 @@ static int mes_v12_1_xcc_hw_init(struct amdgpu_ip_block *ip_block, int xcc_id)
 		goto failure;
 
 	if (adev->enable_uni_mes) {
-		r = mes_v12_1_setup_coop_mode(adev, xcc_id);
-		if (r)
-			goto failure;
 		mes_v12_1_set_hw_resources_1(&adev->mes,
 					       AMDGPU_MES_SCHED_PIPE, xcc_id);
 	}
@@ -1897,24 +1954,12 @@ static int mes_v12_1_hw_fini(struct amdgpu_ip_block *ip_block)
 
 static int mes_v12_1_suspend(struct amdgpu_ip_block *ip_block)
 {
-	int r;
-
-	r = amdgpu_mes_suspend(ip_block->adev);
-	if (r)
-		return r;
-
 	return mes_v12_1_hw_fini(ip_block);
 }
 
 static int mes_v12_1_resume(struct amdgpu_ip_block *ip_block)
 {
-	int r;
-
-	r = mes_v12_1_hw_init(ip_block);
-	if (r)
-		return r;
-
-	return amdgpu_mes_resume(ip_block->adev);
+	return mes_v12_1_hw_init(ip_block);
 }
 
 static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block)
@@ -1922,6 +1967,9 @@ static int mes_v12_1_early_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int pipe, r;
 
+	adev->mes.hung_queue_db_array_size = MES12_HUNG_DB_OFFSET_ARRAY_SIZE;
+	adev->mes.hung_queue_hqd_info_offset = MES12_HUNG_HQD_INFO_OFFSET;
+
 	for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) {
 		r = amdgpu_mes_init_microcode(adev, pipe);
 		if (r)
@@ -2016,7 +2064,7 @@ static int mes_v12_1_map_test_bo(struct amdgpu_device *adev,
 
 error:
 	amdgpu_sync_free(&sync);
-	return 0;
+	return r;
 }
 
 static int mes_v12_1_test_ring(struct amdgpu_device *adev, int xcc_id,
diff --git a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c
index b6f832c538603..28a99b52f59fd 100644
--- a/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/nbif_v6_3_1.c
@@ -144,10 +144,13 @@ static void nbif_v6_3_1_vcn_doorbell_range(struct amdgpu_device *adev,
 {
 	u32 doorbell_range;
 
-	if (instance)
+	if (instance) {
+		if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
+			return;
 		doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL);
-	else
+	} else {
 		doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL);
+	}
 
 	if (use_doorbell) {
 		doorbell_range = REG_SET_FIELD(doorbell_range,
@@ -177,10 +180,7 @@ static void nbif_v6_3_1_vcn_doorbell_range(struct amdgpu_device *adev,
 					       0);
 
 	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) {
-		if (instance)
-			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_nbif_4_10, doorbell_range);
-		else
-			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_nbif_4_10, doorbell_range);
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_nbif_4_10, doorbell_range);
 	} else {
 		if (instance)
 			WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
@@ -189,6 +189,51 @@ static void nbif_v6_3_1_vcn_doorbell_range(struct amdgpu_device *adev,
 	}
 }
 
+static void nbif_v6_3_1_vpe_doorbell_range(struct amdgpu_device *adev,
+							int instance, bool use_doorbell,
+							int doorbell_index,
+							int doorbell_size)
+{
+	if (instance)
+		return;
+
+	u32 doorbell_range = RREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL);
+
+	if (use_doorbell) {
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_ENABLE,
+					       0x1);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_AWID,
+					       0xf);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_RANGE_OFFSET,
+					       doorbell_index);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_RANGE_SIZE,
+					       doorbell_size);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE,
+					       0xf);
+	} else {
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL,
+					       S2A_DOORBELL_PORT5_RANGE_SIZE,
+					       0);
+	}
+
+	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4))
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_nbif_4_10, doorbell_range);
+	else
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
+
+}
+
 static void nbif_v6_3_1_gc_doorbell_init(struct amdgpu_device *adev)
 {
 	if (amdgpu_ip_version(adev, NBIO_HWIP, 0) == IP_VERSION(7, 11, 4)) {
@@ -517,6 +562,7 @@ const struct amdgpu_nbio_funcs nbif_v6_3_1_funcs = {
 	.get_memsize = nbif_v6_3_1_get_memsize,
 	.sdma_doorbell_range = nbif_v6_3_1_sdma_doorbell_range,
 	.vcn_doorbell_range = nbif_v6_3_1_vcn_doorbell_range,
+	.vpe_doorbell_range = nbif_v6_3_1_vpe_doorbell_range,
 	.gc_doorbell_init = nbif_v6_3_1_gc_doorbell_init,
 	.enable_doorbell_aperture = nbif_v6_3_1_enable_doorbell_aperture,
 	.enable_doorbell_selfring_aperture = nbif_v6_3_1_enable_doorbell_selfring_aperture,
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c
new file mode 100644
index 0000000000000..5e8f466f23ad3
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.c
@@ -0,0 +1,369 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#include "nbio/nbio_6_3_2_offset.h"
+#include "nbio/nbio_6_3_2_sh_mask.h"
+
+#include "amdgpu.h"
+#include "nbio_v6_3_2.h"
+
+static u32 nbio_v6_3_2_get_pcie_index_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2);
+}
+
+static u32 nbio_v6_3_2_get_pcie_data_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_DATA2);
+}
+
+static u32 nbio_v6_3_2_get_pcie_index_hi_offset(struct amdgpu_device *adev)
+{
+	return SOC15_REG_OFFSET(NBIO, 0, regBIF_BX0_PCIE_INDEX2_HI);
+}
+
+static u32 nbio_v6_3_2_get_rev_id(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	/* TODO: RCC_STRAP0_RCC_DEV0_EPF0_STRAP0 is not accessible from
+	 * guest side. It requires bootloader to update specific fields
+	 * in ip discovery table to identify soc revision id.
+	 * Return 0 when the function is called from guest side until
+	 * bootloader change is available.
+	 */
+	if (amdgpu_sriov_vf(adev))
+		return 0;
+
+	tmp = RREG32_SOC15(NBIO, 0, regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0);
+	tmp = REG_GET_FIELD(tmp, RCC_STRAP0_RCC_DEV0_EPF0_STRAP0,
+			    STRAP_ATI_REV_ID_DEV0_F0);
+
+	return tmp;
+}
+
+static void nbio_v6_3_2_mc_access_enable(struct amdgpu_device *adev,
+					 bool enable)
+{
+	if (enable)
+		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN,
+			BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK | BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK);
+	else
+		WREG32_SOC15(NBIO, 0, regBIF_BX0_BIF_FB_EN, 0);
+}
+
+static void nbio_v6_3_2_init_registers(struct amdgpu_device *adev)
+{
+	WREG32_SOC15(NBIO, 0, regXCD_DOORBELL_FENCE_1,
+		(0xff & ~(adev->gfx.xcc_mask)) <<
+		XCD_DOORBELL_FENCE_1__XCD_0_DOORBELL_DISABLE__SHIFT);
+}
+
+static u32 nbio_v6_3_2_get_memsize(struct amdgpu_device *adev)
+{
+	return RREG32_SOC15(NBIO, 0, regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE);
+}
+
+static void nbio_v6_3_2_enable_doorbell_aperture(struct amdgpu_device *adev,
+						 bool enable)
+{
+	/* Enable to allow doorbell pass thru on pre-silicon bare-metal */
+	WREG32_SOC15(NBIO, 0, regGDC0_DOORBELL_ACCESS_EN_PF, 0xfffff);
+	WREG32_FIELD15_PREREG(NBIO, 0, RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN,
+			BIF_DOORBELL_APER_EN, enable ? 1 : 0);
+}
+
+static void nbio_v6_3_2_enable_doorbell_selfring_aperture(struct amdgpu_device *adev,
+							  bool enable)
+{
+	u32 tmp = 0;
+
+	if (enable) {
+		tmp = REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_EN, 1) |
+		      REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_MODE, 1) |
+		      REG_SET_FIELD(tmp, BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL,
+				    DOORBELL_SELFRING_GPA_APER_SIZE, 0);
+
+		WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW,
+			     lower_32_bits(adev->doorbell.base));
+		WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH,
+			     upper_32_bits(adev->doorbell.base));
+	}
+
+	WREG32_SOC15(NBIO, 0, regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL, tmp);
+}
+
+static void nbio_v6_3_2_enable_doorbell_interrupt(struct amdgpu_device *adev,
+						  bool enable)
+{
+	WREG32_FIELD15_PREREG(NBIO, 0, BIF_BX0_BIF_DOORBELL_INT_CNTL,
+			      DOORBELL_INTERRUPT_DISABLE, enable ? 0 : 1);
+}
+
+static void nbio_v6_3_2_ih_control(struct amdgpu_device *adev)
+{
+	u32 interrupt_cntl;
+
+	/* setup interrupt control */
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL2, adev->dummy_page_addr >> 8);
+
+	interrupt_cntl = RREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL);
+	/*
+	 * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=0 - dummy read disabled with msi, enabled without msi
+	 * BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK=1 - dummy read controlled by IH_DUMMY_RD_EN
+	 */
+	interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL,
+				       IH_DUMMY_RD_OVERRIDE, 0);
+
+	/* BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK=1 if ring is in non-cacheable memory, e.g., vram */
+	interrupt_cntl = REG_SET_FIELD(interrupt_cntl, BIF_BX0_INTERRUPT_CNTL,
+				       IH_REQ_NONSNOOP_EN, 0);
+
+	WREG32_SOC15(NBIO, 0, regBIF_BX0_INTERRUPT_CNTL, interrupt_cntl);
+}
+
+static void nbio_v6_3_2_ih_doorbell_range(struct amdgpu_device *adev,
+					  bool use_doorbell, int doorbell_index)
+{
+	u32 ih_doorbell_range = 0;
+	u32 ih_doorbell_range1 = 0;
+
+	if (use_doorbell) {
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_ENABLE,
+						  0x1);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_AWID,
+						  0x0);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_RANGE_OFFSET,
+						  doorbell_index);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_RANGE_SIZE,
+						  8);
+		ih_doorbell_range = REG_SET_FIELD(ih_doorbell_range,
+						  GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL,
+						  S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE,
+						  0x0);
+		ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1,
+						   GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1,
+						   S2A_DOORBELL_PORT1_TARGET_PORT_TYPE,
+						   0x3);
+		ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1,
+						   GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1,
+						   S2A_DOORBELL_PORT1_TARGET_DIEID,
+						   0x0);
+		ih_doorbell_range1 = REG_SET_FIELD(ih_doorbell_range1,
+						   GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1,
+						   S2A_DOORBELL_PORT1_TARGET_PORT_ID,
+						   0x0);
+	}
+
+	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL, ih_doorbell_range);
+	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1, ih_doorbell_range1);
+}
+
+static void nbio_v6_3_2_gc_doorbell_init(struct amdgpu_device *adev)
+{
+	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL, 0x30000007);
+	WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1, 0x3);
+}
+
+static void nbio_v6_3_2_sdma_doorbell_range(struct amdgpu_device *adev,
+					    int instance, bool use_doorbell,
+					    int doorbell_index,
+					    int doorbell_size)
+{
+	if (instance == 0) {
+		u32 doorbell_range = 0;
+		u32 doorbell_range1 = 0;
+
+		if (use_doorbell) {
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
+						       S2A_DOORBELL_PORT6_ENABLE,
+						       0x1);
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
+						       S2A_DOORBELL_PORT6_AWID,
+						       0xe);
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
+						       S2A_DOORBELL_PORT6_RANGE_OFFSET,
+						       doorbell_index);
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
+						       S2A_DOORBELL_PORT6_RANGE_SIZE,
+						       doorbell_size);
+			doorbell_range = REG_SET_FIELD(doorbell_range,
+						       GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL,
+						       S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE,
+						       0xe);
+			doorbell_range1 = REG_SET_FIELD(doorbell_range1,
+						        GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1,
+						        S2A_DOORBELL_PORT6_TARGET_PORT_TYPE,
+						        0x3);
+			doorbell_range1 = REG_SET_FIELD(doorbell_range1,
+						        GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1,
+						        S2A_DOORBELL_PORT6_TARGET_DIEID,
+						        0x0);
+			doorbell_range1 = REG_SET_FIELD(doorbell_range1,
+						        GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1,
+						        S2A_DOORBELL_PORT6_TARGET_PORT_ID,
+						        0x0);
+		}
+
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL, doorbell_range);
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1, doorbell_range1);
+	}
+}
+
+static void nbio_v6_3_2_vcn_doorbell_range(struct amdgpu_device *adev,
+					   bool use_doorbell, int doorbell_index,
+					   int instance)
+{
+	u32 doorbell_range = 0;
+	u32 doorbell_range1 = 0;
+
+	if (use_doorbell) {
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+					       S2A_DOORBELL_PORT2_ENABLE,
+					       0x1);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+					       S2A_DOORBELL_PORT2_AWID,
+					       (instance % adev->vcn.num_inst_per_aid) ? 0x7 : 0x4);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+					       S2A_DOORBELL_PORT2_RANGE_OFFSET,
+					       doorbell_index);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+					       S2A_DOORBELL_PORT2_RANGE_SIZE,
+					       8);
+		doorbell_range = REG_SET_FIELD(doorbell_range,
+					       GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL,
+					       S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE,
+					       (instance % adev->vcn.num_inst_per_aid) ? 0x7 : 0x4);
+		doorbell_range1 = REG_SET_FIELD(doorbell_range1,
+						GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1,
+					        S2A_DOORBELL_PORT2_TARGET_PORT_TYPE,
+					        0x3);
+		doorbell_range1 = REG_SET_FIELD(doorbell_range1,
+						GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1,
+					        S2A_DOORBELL_PORT2_TARGET_DIEID,
+					        (instance / adev->vcn.num_inst_per_aid) ? 0x3 : 0x0);
+		doorbell_range1 = REG_SET_FIELD(doorbell_range1,
+						GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1,
+					        S2A_DOORBELL_PORT2_TARGET_PORT_ID,
+					        0x0);
+	}
+
+	switch (instance) {
+	case 0:
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL, doorbell_range);
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1, doorbell_range1);
+		break;
+	case 1:
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL, doorbell_range);
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1, doorbell_range1);
+		break;
+	case 2:
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL, doorbell_range);
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1, doorbell_range1);
+		break;
+	case 3:
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL, doorbell_range);
+		WREG32_SOC15(NBIO, 0, regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1, doorbell_range1);
+		break;
+	default:
+		dev_err(adev->dev,
+			"amdgpu: invalid vcn instance set when program doorbell range\n");
+		break;
+	}
+}
+
+static int nbio_v6_3_2_get_compute_partition_mode(struct amdgpu_device *adev)
+{
+	u32 tmp, px;
+
+	tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_COMPUTE_STATUS);
+	px = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_COMPUTE_STATUS,
+			   PARTITION_MODE);
+
+	return px;
+}
+
+static bool nbio_v6_3_2_is_nps_switch_requested(struct amdgpu_device *adev)
+{
+	u32 tmp;
+
+	tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_STATUS);
+	tmp = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_MEM_STATUS,
+			    CHANGE_STATUS);
+
+	/* 0x8 - NPS switch requested */
+	return (tmp == 0x8);
+}
+static u32 nbio_v6_3_2_get_memory_partition_mode(struct amdgpu_device *adev,
+						 u32 *supp_modes)
+{
+	u32 tmp;
+
+	tmp = RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_STATUS);
+	tmp = REG_GET_FIELD(tmp, BIF_BX_PF0_PARTITION_MEM_STATUS, NPS_MODE);
+
+	if (supp_modes) {
+		*supp_modes =
+			RREG32_SOC15(NBIO, 0, regBIF_BX_PF0_PARTITION_MEM_CAP);
+	}
+
+	return ffs(tmp);
+}
+
+const struct amdgpu_nbio_funcs nbio_v6_3_2_funcs = {
+	.get_pcie_index_offset = nbio_v6_3_2_get_pcie_index_offset,
+	.get_pcie_data_offset = nbio_v6_3_2_get_pcie_data_offset,
+	.get_pcie_index_hi_offset = nbio_v6_3_2_get_pcie_index_hi_offset,
+	.get_rev_id = nbio_v6_3_2_get_rev_id,
+	.mc_access_enable = nbio_v6_3_2_mc_access_enable,
+	.get_memsize = nbio_v6_3_2_get_memsize,
+	.enable_doorbell_aperture = nbio_v6_3_2_enable_doorbell_aperture,
+	.enable_doorbell_selfring_aperture = nbio_v6_3_2_enable_doorbell_selfring_aperture,
+	.enable_doorbell_interrupt = nbio_v6_3_2_enable_doorbell_interrupt,
+	.get_compute_partition_mode = nbio_v6_3_2_get_compute_partition_mode,
+	.get_memory_partition_mode = nbio_v6_3_2_get_memory_partition_mode,
+	.is_nps_switch_requested = nbio_v6_3_2_is_nps_switch_requested,
+	.ih_control = nbio_v6_3_2_ih_control,
+	.ih_doorbell_range = nbio_v6_3_2_ih_doorbell_range,
+	.gc_doorbell_init = nbio_v6_3_2_gc_doorbell_init,
+	.sdma_doorbell_range = nbio_v6_3_2_sdma_doorbell_range,
+	.vcn_doorbell_range = nbio_v6_3_2_vcn_doorbell_range,
+	.init_registers = nbio_v6_3_2_init_registers,
+};
diff --git a/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.h b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.h
new file mode 100644
index 0000000000000..bc7f747c88f47
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/nbio_v6_3_2.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __NBIO_V6_3_2_H__
+#define __NBIO_V6_3_2_H__
+
+#include "soc15_common.h"
+
+extern const struct amdgpu_nbio_funcs nbio_v6_3_2_funcs;
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdgpu/nv.c b/drivers/gpu/drm/amd/amdgpu/nv.c
index 7ce1a1b956065..72edf5326b05b 100644
--- a/drivers/gpu/drm/amd/amdgpu/nv.c
+++ b/drivers/gpu/drm/amd/amdgpu/nv.c
@@ -354,29 +354,12 @@ static struct soc15_allowed_register_entry nv_allowed_read_registers[] = {
 	{ SOC15_REG_ENTRY(GC, 0, mmGB_ADDR_CONFIG)},
 };
 
-static uint32_t nv_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
-					 u32 sh_num, u32 reg_offset)
-{
-	uint32_t val;
-
-	mutex_lock(&adev->grbm_idx_mutex);
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
-
-	val = RREG32(reg_offset);
-
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
-	mutex_unlock(&adev->grbm_idx_mutex);
-	return val;
-}
-
 static uint32_t nv_get_register_value(struct amdgpu_device *adev,
 				      bool indexed, u32 se_num,
 				      u32 sh_num, u32 reg_offset)
 {
 	if (indexed) {
-		return nv_read_indexed_register(adev, se_num, sh_num, reg_offset);
+		return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
 	} else {
 		if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG))
 			return adev->gfx.config.gb_addr_config;
@@ -511,16 +494,6 @@ static int nv_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk)
 	return 0;
 }
 
-static void nv_program_aspm(struct amdgpu_device *adev)
-{
-	if (!amdgpu_device_should_use_aspm(adev))
-		return;
-
-	if (adev->nbio.funcs->program_aspm)
-		adev->nbio.funcs->program_aspm(adev);
-
-}
-
 const struct amdgpu_ip_block_version nv_common_ip_block = {
 	.type = AMD_IP_BLOCK_TYPE_COMMON,
 	.major = 1,
@@ -984,7 +957,7 @@ static int nv_common_hw_init(struct amdgpu_ip_block *ip_block)
 		adev->nbio.funcs->apply_l1_link_width_reconfig_wa(adev);
 
 	/* enable aspm */
-	nv_program_aspm(adev);
+	amdgpu_nbio_program_aspm(adev);
 	/* setup nbio registers */
 	adev->nbio.funcs->init_registers(adev);
 	/* remap HDP registers to a hole in mmio space,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
index e8f768638fd59..ac34bac3c8391 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
+++ b/drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h
@@ -107,6 +107,7 @@ enum psp_gfx_cmd_id
     GFX_CMD_ID_CONFIG_SQ_PERFMON  = 0x00000046,   /* Config CGTT_SQ_CLK_CTRL */
     /* Dynamic memory partitioninig (NPS mode change)*/
     GFX_CMD_ID_FB_NPS_MODE        = 0x00000048,  /* Configure memory partitioning mode */
+    GFX_CMD_ID_PERF_HW            = 0x0000004C,   /* performance monitor */
     GFX_CMD_ID_FB_FW_RESERV_ADDR  = 0x00000050,  /* Query FW reservation addr */
     GFX_CMD_ID_FB_FW_RESERV_EXT_ADDR = 0x00000051,  /* Query FW reservation extended addr */
 };
@@ -373,6 +374,13 @@ struct psp_gfx_cmd_fb_memory_part {
 	uint32_t resvd;
 };
 
+struct psp_gfx_cmd_req_perf_hw {
+	uint32_t req;
+	uint32_t ptl_state;
+	uint32_t pref_format1;
+	uint32_t pref_format2;
+};
+
 /* All GFX ring buffer commands. */
 union psp_gfx_commands
 {
@@ -389,6 +397,7 @@ union psp_gfx_commands
     struct psp_gfx_cmd_sriov_spatial_part cmd_spatial_part;
     struct psp_gfx_cmd_config_sq_perfmon config_sq_perfmon;
     struct psp_gfx_cmd_fb_memory_part cmd_memory_part;
+    struct psp_gfx_cmd_req_perf_hw     cmd_req_perf_hw;
 };
 
 struct psp_gfx_uresp_reserved
@@ -415,12 +424,20 @@ struct psp_gfx_uresp_fw_reserve_info {
     uint32_t reserve_size;
 };
 
+struct psp_gfx_uresp_perf_hw {
+	uint32_t resp;
+	uint32_t ptl_state;
+	uint32_t pref_format1;
+	uint32_t pref_format2;
+};
+
 /* Union of command-specific responses for GPCOM ring. */
 union psp_gfx_uresp {
 	struct psp_gfx_uresp_reserved		reserved;
 	struct psp_gfx_uresp_bootcfg		boot_cfg;
 	struct psp_gfx_uresp_fwar_db_info	fwar_db_info;
 	struct psp_gfx_uresp_fw_reserve_info	fw_reserve_info;
+	struct psp_gfx_uresp_perf_hw		perf_hw_info;
 };
 
 /* Structure of GFX Response buffer.
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
index fb7aaf5ae05cf..479690c44f0d5 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
@@ -217,7 +217,9 @@ static int psp_v11_0_bootloader_load_component(struct psp_context  	*psp,
 		return ret;
 
 	/* Copy PSP System Driver binary to memory */
-	psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes);
+	ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the sys driver to bootloader */
 	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
@@ -263,7 +265,9 @@ static int psp_v11_0_bootloader_load_sos(struct psp_context *psp)
 		return ret;
 
 	/* Copy Secure OS binary to PSP memory */
-	psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes);
+	ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the PSP secure OS to bootloader */
 	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c
index c3cae29eeca1e..f823f042788d1 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v12_0.c
@@ -87,7 +87,9 @@ static int psp_v12_0_bootloader_load_sysdrv(struct psp_context *psp)
 		return ret;
 
 	/* Copy PSP System Driver binary to memory */
-	psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes);
+	ret = psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the sys driver to bootloader */
 	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
@@ -123,7 +125,9 @@ static int psp_v12_0_bootloader_load_sos(struct psp_context *psp)
 		return ret;
 
 	/* Copy Secure OS binary to PSP memory */
-	psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes);
+	ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the PSP secure OS to bootloader */
 	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
index a0c84f81c0c9d..00b4a34e6601f 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
@@ -271,10 +271,9 @@ static int psp_v13_0_bootloader_load_component(struct psp_context  	*psp,
 	if (ret)
 		return ret;
 
-	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
-
-	/* Copy PSP KDB binary to memory */
-	memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes);
+	ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the PSP KDB to bootloader */
 	WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36,
@@ -353,10 +352,9 @@ static int psp_v13_0_bootloader_load_sos(struct psp_context *psp)
 	if (ret)
 		return ret;
 
-	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
-
-	/* Copy Secure OS binary to PSP memory */
-	memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes);
+	ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the PSP secure OS to bootloader */
 	WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c
index 5f39a2edcc956..3d5e26b3fa00a 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0_4.c
@@ -105,10 +105,9 @@ static int psp_v13_0_4_bootloader_load_component(struct psp_context  	*psp,
 	if (ret)
 		return ret;
 
-	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
-
-	/* Copy PSP KDB binary to memory */
-	memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes);
+	ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the PSP KDB to bootloader */
 	WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36,
@@ -168,10 +167,9 @@ static int psp_v13_0_4_bootloader_load_sos(struct psp_context *psp)
 	if (ret)
 		return ret;
 
-	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
-
-	/* Copy Secure OS binary to PSP memory */
-	memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes);
+	ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the PSP secure OS to bootloader */
 	WREG32_SOC15(MP0, 0, regMP0_SMN_C2PMSG_36,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
index 38dfc5c19f2a7..040a61aefa866 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
@@ -140,10 +140,9 @@ static int psp_v14_0_bootloader_load_component(struct psp_context  	*psp,
 	if (ret)
 		return ret;
 
-	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
-
-	/* Copy PSP KDB binary to memory */
-	memcpy(psp->fw_pri_buf, bin_desc->start_addr, bin_desc->size_bytes);
+	ret = psp_copy_fw(psp, bin_desc->start_addr, bin_desc->size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the PSP KDB to bootloader */
 	WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_36,
@@ -214,10 +213,9 @@ static int psp_v14_0_bootloader_load_sos(struct psp_context *psp)
 	if (ret)
 		return ret;
 
-	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
-
-	/* Copy Secure OS binary to PSP memory */
-	memcpy(psp->fw_pri_buf, psp->sos.start_addr, psp->sos.size_bytes);
+	ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the PSP secure OS to bootloader */
 	WREG32_SOC15(MP0, 0, regMPASP_SMN_C2PMSG_36,
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
index b2d7cbd894c03..ec20cd5eb7552 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_8.c
@@ -32,6 +32,7 @@
 #include "mp/mp_15_0_8_sh_mask.h"
 
 MODULE_FIRMWARE("amdgpu/psp_15_0_8_toc.bin");
+MODULE_FIRMWARE("amdgpu/psp_15_0_8_toc_1.bin");
 
 static int psp_v15_0_8_init_microcode(struct psp_context *psp)
 {
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
index f5030efc6c804..24856c91c1351 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v3_1.c
@@ -96,7 +96,9 @@ static int psp_v3_1_bootloader_load_sysdrv(struct psp_context *psp)
 		return ret;
 
 	/* Copy PSP System Driver binary to memory */
-	psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes);
+	ret = psp_copy_fw(psp, psp->sys.start_addr, psp->sys.size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the sys driver to bootloader */
 	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
@@ -135,7 +137,9 @@ static int psp_v3_1_bootloader_load_sos(struct psp_context *psp)
 		return ret;
 
 	/* Copy Secure OS binary to PSP memory */
-	psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes);
+	ret = psp_copy_fw(psp, psp->sos.start_addr, psp->sos.size_bytes);
+	if (ret)
+		return ret;
 
 	/* Provide the PSP secure OS to bootloader */
 	WREG32_SOC15(MP0, 0, mmMP0_SMN_C2PMSG_36,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index 0090ace49024f..93ec52c1f3678 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -828,7 +828,6 @@ static int sdma_v2_4_early_init(struct amdgpu_ip_block *ip_block)
 		return r;
 
 	sdma_v2_4_set_ring_funcs(adev);
-	sdma_v2_4_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v2_4_vm_pte_funcs);
 	sdma_v2_4_set_irq_funcs(adev);
 
@@ -898,7 +897,9 @@ static int sdma_v2_4_hw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
-	return r;
+	sdma_v2_4_set_buffer_funcs(adev);
+
+	return 0;
 }
 
 static int sdma_v2_4_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -1235,8 +1236,7 @@ static const struct amdgpu_buffer_funcs sdma_v2_4_buffer_funcs = {
 
 static void sdma_v2_4_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	adev->mman.buffer_funcs = &sdma_v2_4_buffer_funcs;
-	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v2_4_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v2_4_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 2526d393162ac..3fde9be746900 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -1108,7 +1108,6 @@ static int sdma_v3_0_early_init(struct amdgpu_ip_block *ip_block)
 		return r;
 
 	sdma_v3_0_set_ring_funcs(adev);
-	sdma_v3_0_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v3_0_vm_pte_funcs);
 	sdma_v3_0_set_irq_funcs(adev);
 
@@ -1184,7 +1183,9 @@ static int sdma_v3_0_hw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
-	return r;
+	sdma_v3_0_set_buffer_funcs(adev);
+
+	return 0;
 }
 
 static int sdma_v3_0_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -1677,8 +1678,7 @@ static const struct amdgpu_buffer_funcs sdma_v3_0_buffer_funcs = {
 
 static void sdma_v3_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	adev->mman.buffer_funcs = &sdma_v3_0_buffer_funcs;
-	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v3_0_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v3_0_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
index e64f2f6df9a9e..cb64d17000df0 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c
@@ -1775,7 +1775,6 @@ static int sdma_v4_0_early_init(struct amdgpu_ip_block *ip_block)
 		adev->sdma.has_page_queue = true;
 
 	sdma_v4_0_set_ring_funcs(adev);
-	sdma_v4_0_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v4_0_vm_pte_funcs);
 	sdma_v4_0_set_irq_funcs(adev);
 	sdma_v4_0_set_ras_funcs(adev);
@@ -1961,6 +1960,7 @@ static int sdma_v4_0_sw_fini(struct amdgpu_ip_block *ip_block)
 static int sdma_v4_0_hw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
+	int r;
 
 	if (adev->flags & AMD_IS_APU)
 		amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_SDMA, false, 0);
@@ -1968,7 +1968,12 @@ static int sdma_v4_0_hw_init(struct amdgpu_ip_block *ip_block)
 	if (!amdgpu_sriov_vf(adev))
 		sdma_v4_0_init_golden_registers(adev);
 
-	return sdma_v4_0_start(adev);
+	r = sdma_v4_0_start(adev);
+	if (r)
+		return r;
+	sdma_v4_0_set_buffer_funcs(adev);
+
+	return 0;
 }
 
 static int sdma_v4_0_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -2626,13 +2631,9 @@ static const struct amdgpu_buffer_funcs sdma_v4_4_buffer_funcs = {
 static void sdma_v4_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
 	if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >= IP_VERSION(4, 4, 0))
-		adev->mman.buffer_funcs = &sdma_v4_4_buffer_funcs;
-	else
-		adev->mman.buffer_funcs = &sdma_v4_0_buffer_funcs;
-	if (adev->sdma.has_page_queue)
-		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
+		amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_4_buffer_funcs);
 	else
-		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+		amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_0_buffer_funcs);
 }
 
 static void sdma_v4_0_get_ras_error_count(uint32_t value,
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
index 78bdfed0a7fd3..88428b88e00fc 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v4_4_2.c
@@ -1368,7 +1368,6 @@ static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block)
 		adev->sdma.has_page_queue = true;
 
 	sdma_v4_4_2_set_ring_funcs(adev);
-	sdma_v4_4_2_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v4_4_2_vm_pte_funcs);
 	sdma_v4_4_2_set_irq_funcs(adev);
 	sdma_v4_4_2_set_ras_funcs(adev);
@@ -1568,8 +1567,11 @@ static int sdma_v4_4_2_hw_init(struct amdgpu_ip_block *ip_block)
 		sdma_v4_4_2_inst_init_golden_registers(adev, inst_mask);
 
 	r = sdma_v4_4_2_inst_start(adev, inst_mask, false);
+	if (r)
+		return r;
+	sdma_v4_4_2_set_buffer_funcs(adev);
 
-	return r;
+	return 0;
 }
 
 static int sdma_v4_4_2_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -2316,11 +2318,7 @@ static const struct amdgpu_buffer_funcs sdma_v4_4_2_buffer_funcs = {
 
 static void sdma_v4_4_2_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	adev->mman.buffer_funcs = &sdma_v4_4_2_buffer_funcs;
-	if (adev->sdma.has_page_queue)
-		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].page;
-	else
-		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v4_4_2_buffer_funcs);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
index 52f4e9e099cbf..fa02907217e08 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c
@@ -1373,7 +1373,6 @@ static int sdma_v5_0_early_init(struct amdgpu_ip_block *ip_block)
 		return r;
 
 	sdma_v5_0_set_ring_funcs(adev);
-	sdma_v5_0_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v5_0_vm_pte_funcs);
 	sdma_v5_0_set_irq_funcs(adev);
 	sdma_v5_0_set_mqd_funcs(adev);
@@ -1472,8 +1471,11 @@ static int sdma_v5_0_hw_init(struct amdgpu_ip_block *ip_block)
 	sdma_v5_0_init_golden_registers(adev);
 
 	r = sdma_v5_0_start(adev);
+	if (r)
+		return r;
+	sdma_v5_0_set_buffer_funcs(adev);
 
-	return r;
+	return 0;
 }
 
 static int sdma_v5_0_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -2052,10 +2054,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_0_buffer_funcs = {
 
 static void sdma_v5_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	if (adev->mman.buffer_funcs == NULL) {
-		adev->mman.buffer_funcs = &sdma_v5_0_buffer_funcs;
-		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
-	}
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_0_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v5_0_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
index b4fb90cc8f7d9..f6ecbc524c9b7 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c
@@ -1264,7 +1264,6 @@ static int sdma_v5_2_early_init(struct amdgpu_ip_block *ip_block)
 		return r;
 
 	sdma_v5_2_set_ring_funcs(adev);
-	sdma_v5_2_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v5_2_vm_pte_funcs);
 	sdma_v5_2_set_irq_funcs(adev);
 	sdma_v5_2_set_mqd_funcs(adev);
@@ -1385,8 +1384,14 @@ static int sdma_v5_2_sw_fini(struct amdgpu_ip_block *ip_block)
 static int sdma_v5_2_hw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
+	int r;
+
+	r = sdma_v5_2_start(adev);
+	if (r)
+		return r;
+	sdma_v5_2_set_buffer_funcs(adev);
 
-	return sdma_v5_2_start(adev);
+	return 0;
 }
 
 static int sdma_v5_2_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -2056,10 +2061,7 @@ static const struct amdgpu_buffer_funcs sdma_v5_2_buffer_funcs = {
 
 static void sdma_v5_2_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	if (adev->mman.buffer_funcs == NULL) {
-		adev->mman.buffer_funcs = &sdma_v5_2_buffer_funcs;
-		adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
-	}
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v5_2_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v5_2_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
index 8ca46e1e474ed..bf09ac841a684 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v6_0.c
@@ -1313,7 +1313,6 @@ static int sdma_v6_0_early_init(struct amdgpu_ip_block *ip_block)
 		return r;
 
 	sdma_v6_0_set_ring_funcs(adev);
-	sdma_v6_0_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v6_0_vm_pte_funcs);
 	sdma_v6_0_set_irq_funcs(adev);
 	sdma_v6_0_set_mqd_funcs(adev);
@@ -1477,6 +1476,7 @@ static int sdma_v6_0_hw_init(struct amdgpu_ip_block *ip_block)
 	r = sdma_v6_0_start(adev);
 	if (r)
 		return r;
+	sdma_v6_0_set_buffer_funcs(adev);
 
 	return sdma_v6_0_set_userq_trap_interrupts(adev, true);
 }
@@ -1886,8 +1886,7 @@ static const struct amdgpu_buffer_funcs sdma_v6_0_buffer_funcs = {
 
 static void sdma_v6_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	adev->mman.buffer_funcs = &sdma_v6_0_buffer_funcs;
-	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v6_0_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v6_0_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
index 37191e2918d45..f154b68dda701 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c
@@ -1299,7 +1299,6 @@ static int sdma_v7_0_early_init(struct amdgpu_ip_block *ip_block)
 	}
 
 	sdma_v7_0_set_ring_funcs(adev);
-	sdma_v7_0_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_0_vm_pte_funcs);
 	sdma_v7_0_set_irq_funcs(adev);
 	sdma_v7_0_set_mqd_funcs(adev);
@@ -1432,6 +1431,7 @@ static int sdma_v7_0_hw_init(struct amdgpu_ip_block *ip_block)
 	r = sdma_v7_0_start(adev);
 	if (r)
 		return r;
+	sdma_v7_0_set_buffer_funcs(adev);
 
 	return sdma_v7_0_set_userq_trap_interrupts(adev, true);
 }
@@ -1836,8 +1836,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_0_buffer_funcs = {
 
 static void sdma_v7_0_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	adev->mman.buffer_funcs = &sdma_v7_0_buffer_funcs;
-	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_0_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v7_0_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
index 061934a2e93a3..e1c0a4ff0e7ff 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_1.c
@@ -1287,7 +1287,6 @@ static int sdma_v7_1_early_init(struct amdgpu_ip_block *ip_block)
 	}
 
 	sdma_v7_1_set_ring_funcs(adev);
-	sdma_v7_1_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &sdma_v7_1_vm_pte_funcs);
 	sdma_v7_1_set_irq_funcs(adev);
 	sdma_v7_1_set_mqd_funcs(adev);
@@ -1386,10 +1385,16 @@ static int sdma_v7_1_hw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
 	uint32_t inst_mask;
+	int r;
 
 	inst_mask = GENMASK(adev->sdma.num_instances - 1, 0);
 
-	return sdma_v7_1_inst_start(adev, inst_mask);
+	r = sdma_v7_1_inst_start(adev, inst_mask);
+	if (r)
+		return r;
+	sdma_v7_1_set_buffer_funcs(adev);
+
+	return 0;
 }
 
 static int sdma_v7_1_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -1776,8 +1781,7 @@ static const struct amdgpu_buffer_funcs sdma_v7_1_buffer_funcs = {
 
 static void sdma_v7_1_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	adev->mman.buffer_funcs = &sdma_v7_1_buffer_funcs;
-	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &sdma_v7_1_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version sdma_v7_1_ip_block = {
diff --git a/drivers/gpu/drm/amd/amdgpu/si_dma.c b/drivers/gpu/drm/amd/amdgpu/si_dma.c
index 3e58feb2d5e4f..549708075eb48 100644
--- a/drivers/gpu/drm/amd/amdgpu/si_dma.c
+++ b/drivers/gpu/drm/amd/amdgpu/si_dma.c
@@ -487,7 +487,6 @@ static int si_dma_early_init(struct amdgpu_ip_block *ip_block)
 	adev->sdma.num_instances = SDMA_MAX_INSTANCE;
 
 	si_dma_set_ring_funcs(adev);
-	si_dma_set_buffer_funcs(adev);
 	amdgpu_sdma_set_vm_pte_scheds(adev, &si_dma_vm_pte_funcs);
 	si_dma_set_irq_funcs(adev);
 
@@ -543,8 +542,14 @@ static int si_dma_sw_fini(struct amdgpu_ip_block *ip_block)
 static int si_dma_hw_init(struct amdgpu_ip_block *ip_block)
 {
 	struct amdgpu_device *adev = ip_block->adev;
+	int r;
 
-	return si_dma_start(adev);
+	r = si_dma_start(adev);
+	if (r)
+		return r;
+	si_dma_set_buffer_funcs(adev);
+
+	return 0;
 }
 
 static int si_dma_hw_fini(struct amdgpu_ip_block *ip_block)
@@ -833,8 +838,7 @@ static const struct amdgpu_buffer_funcs si_dma_buffer_funcs = {
 
 static void si_dma_set_buffer_funcs(struct amdgpu_device *adev)
 {
-	adev->mman.buffer_funcs = &si_dma_buffer_funcs;
-	adev->mman.buffer_funcs_ring = &adev->sdma.instance[0].ring;
+	amdgpu_sdma_set_buffer_funcs_scheds(adev, &si_dma_buffer_funcs);
 }
 
 const struct amdgpu_ip_block_version si_dma_ip_block =
diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c
index b456e4541d9ac..87b398dd0769b 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc15.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc15.c
@@ -401,29 +401,12 @@ static struct soc15_allowed_register_entry soc15_allowed_read_registers[] = {
 	{ SOC15_REG_ENTRY(GC, 0, mmDB_DEBUG2)},
 };
 
-static uint32_t soc15_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
-					 u32 sh_num, u32 reg_offset)
-{
-	uint32_t val;
-
-	mutex_lock(&adev->grbm_idx_mutex);
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
-
-	val = RREG32(reg_offset);
-
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
-	mutex_unlock(&adev->grbm_idx_mutex);
-	return val;
-}
-
 static uint32_t soc15_get_register_value(struct amdgpu_device *adev,
 					 bool indexed, u32 se_num,
 					 u32 sh_num, u32 reg_offset)
 {
 	if (indexed) {
-		return soc15_read_indexed_register(adev, se_num, sh_num, reg_offset);
+		return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
 	} else {
 		if (reg_offset == SOC15_REG_OFFSET(GC, 0, mmGB_ADDR_CONFIG))
 			return adev->gfx.config.gb_addr_config;
@@ -695,15 +678,6 @@ static int soc15_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk
 	return 0;
 }
 
-static void soc15_program_aspm(struct amdgpu_device *adev)
-{
-	if (!amdgpu_device_should_use_aspm(adev))
-		return;
-
-	if (adev->nbio.funcs->program_aspm)
-		adev->nbio.funcs->program_aspm(adev);
-}
-
 const struct amdgpu_ip_block_version vega10_common_ip_block =
 {
 	.type = AMD_IP_BLOCK_TYPE_COMMON,
@@ -1284,7 +1258,7 @@ static int soc15_common_hw_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 
 	/* enable aspm */
-	soc15_program_aspm(adev);
+	amdgpu_nbio_program_aspm(adev);
 	/* setup nbio registers */
 	adev->nbio.funcs->init_registers(adev);
 	/* remap HDP registers to a hole in mmio space,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc21.c b/drivers/gpu/drm/amd/amdgpu/soc21.c
index fbd1d97f33add..93c002e511c7d 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc21.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc21.c
@@ -306,29 +306,12 @@ static struct soc15_allowed_register_entry soc21_allowed_read_registers[] = {
 	{ SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG)},
 };
 
-static uint32_t soc21_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
-					 u32 sh_num, u32 reg_offset)
-{
-	uint32_t val;
-
-	mutex_lock(&adev->grbm_idx_mutex);
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
-
-	val = RREG32(reg_offset);
-
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
-	mutex_unlock(&adev->grbm_idx_mutex);
-	return val;
-}
-
 static uint32_t soc21_get_register_value(struct amdgpu_device *adev,
 				      bool indexed, u32 se_num,
 				      u32 sh_num, u32 reg_offset)
 {
 	if (indexed) {
-		return soc21_read_indexed_register(adev, se_num, sh_num, reg_offset);
+		return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
 	} else {
 		if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG) && adev->gfx.config.gb_addr_config)
 			return adev->gfx.config.gb_addr_config;
@@ -470,15 +453,6 @@ static int soc21_set_vce_clocks(struct amdgpu_device *adev, u32 evclk, u32 ecclk
 	return 0;
 }
 
-static void soc21_program_aspm(struct amdgpu_device *adev)
-{
-	if (!amdgpu_device_should_use_aspm(adev))
-		return;
-
-	if (adev->nbio.funcs->program_aspm)
-		adev->nbio.funcs->program_aspm(adev);
-}
-
 const struct amdgpu_ip_block_version soc21_common_ip_block = {
 	.type = AMD_IP_BLOCK_TYPE_COMMON,
 	.major = 1,
@@ -925,7 +899,7 @@ static int soc21_common_hw_init(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 
 	/* enable aspm */
-	soc21_program_aspm(adev);
+	amdgpu_nbio_program_aspm(adev);
 	/* setup nbio registers */
 	adev->nbio.funcs->init_registers(adev);
 	/* remap HDP registers to a hole in mmio space,
diff --git a/drivers/gpu/drm/amd/amdgpu/soc24.c b/drivers/gpu/drm/amd/amdgpu/soc24.c
index d1adf19a51c45..265db9331d0bb 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc24.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc24.c
@@ -132,31 +132,12 @@ static struct soc15_allowed_register_entry soc24_allowed_read_registers[] = {
 	{ SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG)},
 };
 
-static uint32_t soc24_read_indexed_register(struct amdgpu_device *adev,
-					    u32 se_num,
-					    u32 sh_num,
-					    u32 reg_offset)
-{
-	uint32_t val;
-
-	mutex_lock(&adev->grbm_idx_mutex);
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
-
-	val = RREG32(reg_offset);
-
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
-	mutex_unlock(&adev->grbm_idx_mutex);
-	return val;
-}
-
 static uint32_t soc24_get_register_value(struct amdgpu_device *adev,
 					 bool indexed, u32 se_num,
 					 u32 sh_num, u32 reg_offset)
 {
 	if (indexed) {
-		return soc24_read_indexed_register(adev, se_num, sh_num, reg_offset);
+		return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
 	} else {
 		if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG) &&
 		    adev->gfx.config.gb_addr_config)
diff --git a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
index 709b1669b07bc..5f05c8e68297b 100644
--- a/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/soc_v1_0.c
@@ -184,31 +184,13 @@ static struct soc15_allowed_register_entry soc_v1_0_allowed_read_registers[] = {
 	{ SOC15_REG_ENTRY(GC, 0, regGB_ADDR_CONFIG_1) },
 };
 
-static uint32_t soc_v1_0_read_indexed_register(struct amdgpu_device *adev,
-					       u32 se_num,
-					       u32 sh_num,
-					       u32 reg_offset)
-{
-	uint32_t val;
-
-	mutex_lock(&adev->grbm_idx_mutex);
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff, 0);
-
-	val = RREG32(reg_offset);
-
-	if (se_num != 0xffffffff || sh_num != 0xffffffff)
-		amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, 0);
-	mutex_unlock(&adev->grbm_idx_mutex);
-	return val;
-}
 
 static uint32_t soc_v1_0_get_register_value(struct amdgpu_device *adev,
 					    bool indexed, u32 se_num,
 					    u32 sh_num, u32 reg_offset)
 {
 	if (indexed) {
-		return soc_v1_0_read_indexed_register(adev, se_num, sh_num, reg_offset);
+		return amdgpu_read_indexed_register(adev, se_num, sh_num, reg_offset);
 	} else {
 		if (reg_offset == SOC15_REG_OFFSET(GC, 0, regGB_ADDR_CONFIG_1) &&
 		    adev->gfx.config.gb_addr_config)
@@ -330,6 +312,8 @@ static int soc_v1_0_common_early_init(struct amdgpu_ip_block *ip_block)
 		return -EINVAL;
 	}
 
+	adev->nbio.funcs->init_registers(adev);
+
 	return 0;
 }
 
@@ -737,15 +721,8 @@ static int soc_v1_0_switch_partition_mode(struct amdgpu_xcp_mgr *xcp_mgr,
 
 	num_xcc_per_xcp = __soc_v1_0_get_xcc_per_xcp(xcp_mgr, mode);
 	if (adev->gfx.imu.funcs &&
-	    adev->gfx.imu.funcs->switch_compute_partition) {
-		ret = adev->gfx.imu.funcs->switch_compute_partition(xcp_mgr->adev, num_xcc_per_xcp, mode);
-		if (ret)
-			goto out;
-	}
-	if (adev->gfx.imu.funcs &&
-	    adev->gfx.imu.funcs->init_mcm_addr_lut &&
-	    amdgpu_emu_mode)
-		adev->gfx.imu.funcs->init_mcm_addr_lut(adev);
+	    adev->gfx.imu.funcs->switch_compute_partition)
+		adev->gfx.imu.funcs->switch_compute_partition(xcp_mgr->adev, num_xcc_per_xcp, mode);
 
 	/* Init info about new xcps */
 	*num_xcps = num_xcc / num_xcc_per_xcp;
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
index 5b7b46d242c6d..93253db5e2de4 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v1_0.c
@@ -42,9 +42,10 @@
 #include "oss/oss_1_0_d.h"
 #include "oss/oss_1_0_sh_mask.h"
 
+#define VCE_V1_0_ALIGNMENT	(32 * 1024)
 #define VCE_V1_0_FW_SIZE	(256 * 1024)
 #define VCE_V1_0_STACK_SIZE	(64 * 1024)
-#define VCE_V1_0_DATA_SIZE	(7808 * (AMDGPU_MAX_VCE_HANDLES + 1))
+#define VCE_V1_0_DATA_SIZE	(ALIGN(7808 * (AMDGPU_MAX_VCE_HANDLES + 1), VCE_V1_0_ALIGNMENT))
 #define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK	0x02
 
 static void vce_v1_0_set_ring_funcs(struct amdgpu_device *adev);
@@ -177,7 +178,7 @@ static void vce_v1_0_init_cg(struct amdgpu_device *adev)
 }
 
 /**
- * vce_v1_0_load_fw_signature - load firmware signature into VCPU BO
+ * vce_v1_0_load_fw() - load firmware signature into VCPU BO
  *
  * @adev: amdgpu_device pointer
  *
@@ -185,21 +186,26 @@ static void vce_v1_0_init_cg(struct amdgpu_device *adev)
  * This function finds the signature appropriate for the current
  * ASIC and writes that into the VCPU BO.
  */
-static int vce_v1_0_load_fw_signature(struct amdgpu_device *adev)
+static int vce_v1_0_load_fw(struct amdgpu_device *adev)
 {
 	const struct common_firmware_header *hdr;
 	struct vce_v1_0_fw_signature *sign;
-	unsigned int ucode_offset;
+	u32 ucode_offset;
+	u32 ucode_size;
 	uint32_t chip_id;
 	u32 *cpu_addr;
 	int i;
 
 	hdr = (const struct common_firmware_header *)adev->vce.fw->data;
 	ucode_offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
+	ucode_size = hdr->ucode_size_bytes - sizeof(struct vce_v1_0_fw_signature *);
 	cpu_addr = adev->vce.cpu_addr;
 
 	sign = (void *)adev->vce.fw->data + ucode_offset;
 
+	if (ucode_size > VCE_V1_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET)
+		return -EINVAL;
+
 	switch (adev->asic_type) {
 	case CHIP_TAHITI:
 		chip_id = 0x01000014;
@@ -226,12 +232,14 @@ static int vce_v1_0_load_fw_signature(struct amdgpu_device *adev)
 		return -EINVAL;
 	}
 
+	memset_io(&cpu_addr[0], 0, amdgpu_bo_size(adev->vce.vcpu_bo));
+
 	cpu_addr += (256 - 64) / 4;
 	memcpy_toio(&cpu_addr[0], &sign->val[i].nonce[0], 16);
 	cpu_addr[4] = cpu_to_le32(le32_to_cpu(sign->length) + 64);
 
 	memset_io(&cpu_addr[5], 0, 44);
-	memcpy_toio(&cpu_addr[16], &sign[1], hdr->ucode_size_bytes - sizeof(*sign));
+	memcpy_toio(&cpu_addr[16], &sign[1], ucode_size);
 
 	cpu_addr += (le32_to_cpu(sign->length) + 64) / 4;
 	memcpy_toio(&cpu_addr[0], &sign->val[i].sigval[0], 16);
@@ -312,18 +320,23 @@ static int vce_v1_0_mc_resume(struct amdgpu_device *adev)
 	WREG32(mmVCE_VCPU_SCRATCH7, AMDGPU_MAX_VCE_HANDLES);
 
 	offset =  adev->vce.gpu_addr + AMDGPU_VCE_FIRMWARE_OFFSET;
-	size = VCE_V1_0_FW_SIZE;
-	WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff);
+	size = VCE_V1_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET;
+	WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset);
 	WREG32(mmVCE_VCPU_CACHE_SIZE0, size);
 
 	offset += size;
 	size = VCE_V1_0_STACK_SIZE;
-	WREG32(mmVCE_VCPU_CACHE_OFFSET1, offset & 0x7fffffff);
+	WARN_ON(!IS_ALIGNED(offset, VCE_V1_0_ALIGNMENT));
+	WARN_ON(!IS_ALIGNED(size, VCE_V1_0_ALIGNMENT));
+	WREG32(mmVCE_VCPU_CACHE_OFFSET1, offset);
 	WREG32(mmVCE_VCPU_CACHE_SIZE1, size);
 
 	offset += size;
 	size = VCE_V1_0_DATA_SIZE;
-	WREG32(mmVCE_VCPU_CACHE_OFFSET2, offset & 0x7fffffff);
+	WARN_ON(!IS_ALIGNED(offset, VCE_V1_0_ALIGNMENT));
+	WARN_ON(!IS_ALIGNED(size, VCE_V1_0_ALIGNMENT));
+	WARN_ON((offset + size - adev->vce.gpu_addr) > amdgpu_bo_size(adev->vce.vcpu_bo));
+	WREG32(mmVCE_VCPU_CACHE_OFFSET2, offset);
 	WREG32(mmVCE_VCPU_CACHE_SIZE2, size);
 
 	WREG32_P(mmVCE_LMI_CTRL2, 0x0, ~0x100);
@@ -527,22 +540,31 @@ static int vce_v1_0_early_init(struct amdgpu_ip_block *ip_block)
  * To accomodate that, we put GART to the LOW address range
  * and reserve some GART pages where we map the VCPU BO,
  * so that it gets a 32-bit address.
+ *
+ * The BAR address is zero and we can't change it
+ * due to the firmware validation mechanism.
+ * It seems that it fails to initialize if the address is >= 128 MiB.
  */
 static int vce_v1_0_ensure_vcpu_bo_32bit_addr(struct amdgpu_device *adev)
 {
 	u64 bo_size = amdgpu_bo_size(adev->vce.vcpu_bo);
-	u64 max_vcpu_bo_addr = 0xffffffff - bo_size;
+	u64 max_vcpu_bo_addr = 0x07ffffff - bo_size;
 	u64 num_pages = ALIGN(bo_size, AMDGPU_GPU_PAGE_SIZE) / AMDGPU_GPU_PAGE_SIZE;
 	u64 pa = amdgpu_gmc_vram_pa(adev, adev->vce.vcpu_bo);
 	u64 flags = AMDGPU_PTE_READABLE | AMDGPU_PTE_WRITEABLE | AMDGPU_PTE_VALID;
 	u64 vce_gart_start_offs;
 	int r;
 
-	r = amdgpu_gtt_mgr_alloc_entries(&adev->mman.gtt_mgr,
-					 &adev->vce.gart_node, num_pages,
-					 DRM_MM_INSERT_LOW);
-	if (r)
-		return r;
+	if (adev->gmc.vram_start < adev->gmc.gart_start)
+		return amdgpu_bo_gpu_offset(adev->vce.vcpu_bo) <= max_vcpu_bo_addr ? 0 : -EINVAL;
+
+	if (!drm_mm_node_allocated(&adev->vce.gart_node)) {
+		r = amdgpu_gtt_mgr_alloc_entries(&adev->mman.gtt_mgr,
+						 &adev->vce.gart_node, num_pages,
+						 DRM_MM_INSERT_LOW);
+		if (r)
+			return r;
+	}
 
 	vce_gart_start_offs = amdgpu_gtt_node_to_byte_offset(&adev->vce.gart_node);
 
@@ -553,8 +575,6 @@ static int vce_v1_0_ensure_vcpu_bo_32bit_addr(struct amdgpu_device *adev)
 	amdgpu_gart_map_vram_range(adev, pa, adev->vce.gart_node.start,
 				   num_pages, flags, adev->gart.ptr);
 	adev->vce.gpu_addr = adev->gmc.gart_start + vce_gart_start_offs;
-	if (adev->vce.gpu_addr > max_vcpu_bo_addr)
-		return -EINVAL;
 
 	return 0;
 }
@@ -574,10 +594,7 @@ static int vce_v1_0_sw_init(struct amdgpu_ip_block *ip_block)
 	if (r)
 		return r;
 
-	r = amdgpu_vce_resume(adev);
-	if (r)
-		return r;
-	r = vce_v1_0_load_fw_signature(adev);
+	r = vce_v1_0_load_fw(adev);
 	if (r)
 		return r;
 	r = vce_v1_0_ensure_vcpu_bo_32bit_addr(adev);
@@ -696,10 +713,7 @@ static int vce_v1_0_resume(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int r;
 
-	r = amdgpu_vce_resume(adev);
-	if (r)
-		return r;
-	r = vce_v1_0_load_fw_signature(adev);
+	r = vce_v1_0_load_fw(adev);
 	if (r)
 		return r;
 	r = vce_v1_0_ensure_vcpu_bo_32bit_addr(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index db149eda62044..3a6fc8604108c 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -37,9 +37,14 @@
 #include "oss/oss_2_0_d.h"
 #include "oss/oss_2_0_sh_mask.h"
 
+
+/* Use 24K to be safe. The FW supposedly only requires 23744 bytes. */
+#define VCE_V2_0_DATA_ENTRY_SIZE (24 * 1024)
+
 #define VCE_V2_0_FW_SIZE	(256 * 1024)
 #define VCE_V2_0_STACK_SIZE	(64 * 1024)
-#define VCE_V2_0_DATA_SIZE	(23552 * AMDGPU_MAX_VCE_HANDLES)
+#define VCE_V2_0_DATA_SIZE	(VCE_V2_0_DATA_ENTRY_SIZE * (AMDGPU_MAX_VCE_HANDLES + 1))
+
 #define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK	0x02
 
 static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev);
@@ -183,7 +188,7 @@ static void vce_v2_0_mc_resume(struct amdgpu_device *adev)
 	WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR, (adev->vce.gpu_addr >> 8));
 
 	offset = AMDGPU_VCE_FIRMWARE_OFFSET;
-	size = VCE_V2_0_FW_SIZE;
+	size = VCE_V2_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET;
 	WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff);
 	WREG32(mmVCE_VCPU_CACHE_SIZE0, size);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 03d79e464f04f..c69f7d82060f0 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -574,7 +574,7 @@ static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx)
 	} else
 		WREG32(mmVCE_LMI_VCPU_CACHE_40BIT_BAR, (adev->vce.gpu_addr >> 8));
 	offset = AMDGPU_VCE_FIRMWARE_OFFSET;
-	size = VCE_V3_0_FW_SIZE;
+	size = VCE_V3_0_FW_SIZE - AMDGPU_VCE_FIRMWARE_OFFSET;
 	WREG32(mmVCE_VCPU_CACHE_OFFSET0, offset & 0x7fffffff);
 	WREG32(mmVCE_VCPU_CACHE_SIZE0, size);
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
index 10e8fc2821f37..7f001c32e9113 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
@@ -1686,7 +1686,7 @@ static int vcn_v4_0_3_reset_jpeg_post_helper(struct amdgpu_device *adev, int ins
 	for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) {
 		ring = &adev->jpeg.inst[inst].ring_dec[i];
 		/* Force completion of any remaining jobs */
-		amdgpu_fence_driver_force_completion(ring);
+		amdgpu_fence_driver_force_completion(ring, NULL);
 
 		if (ring->use_doorbell)
 			WREG32_SOC15_OFFSET(
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
index 54fbf8d73ca67..d3db0494341e3 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
@@ -1332,7 +1332,7 @@ static int vcn_v5_0_1_reset_jpeg_post_helper(struct amdgpu_device *adev, int ins
 	for (i = 0; i < adev->jpeg.num_jpeg_rings; ++i) {
 		ring = &adev->jpeg.inst[inst].ring_dec[i];
 		/* Force completion of any remaining jobs */
-		amdgpu_fence_driver_force_completion(ring);
+		amdgpu_fence_driver_force_completion(ring, NULL);
 
 		if (ring->use_doorbell)
 			WREG32_SOC15_OFFSET(
diff --git a/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.c
new file mode 100644
index 0000000000000..0d91e01fbf3a2
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.c
@@ -0,0 +1,350 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/firmware.h>
+#include "amdgpu.h"
+#include "amdgpu_ucode.h"
+#include "amdgpu_vpe.h"
+#include "vpe_v2_0.h"
+#include "soc15_common.h"
+#include "ivsrcid/vpe/irqsrcs_vpe_6_1.h"
+#include "vpe/vpe_2_0_0_offset.h"
+#include "vpe/vpe_2_0_0_sh_mask.h"
+
+MODULE_FIRMWARE("amdgpu/vpe_2_0_0.bin");
+
+#define VPE_THREAD1_UCODE_OFFSET	0x8000
+
+static uint32_t vpe_v2_0_get_reg_offset(struct amdgpu_vpe *vpe, uint32_t inst, uint32_t offset)
+{
+	uint32_t base;
+
+	base = vpe->ring.adev->reg_offset[VPE_HWIP][inst][0];
+
+	return base + offset;
+}
+
+static int vpe_v2_0_irq_init(struct amdgpu_vpe *vpe)
+{
+	struct amdgpu_device *adev = container_of(vpe, struct amdgpu_device, vpe);
+	int ret;
+
+	ret = amdgpu_irq_add_id(adev, SOC21_IH_CLIENTID_VPE,
+				VPE_6_1_SRCID__VPE_TRAP,
+				&adev->vpe.trap_irq);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int vpe_v2_0_load_microcode(struct amdgpu_vpe *vpe)
+{
+	struct amdgpu_device *adev = vpe->ring.adev;
+	const struct vpe_firmware_header_v1_0 *vpe_hdr;
+	const __le32 *data;
+	uint32_t ucode_offset[2], ucode_size[2], size_dw, ret;
+	uint32_t f32_offset, f32_cntl, reg_data;
+
+	ret = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL));
+	ret = REG_SET_FIELD(ret, VPEC_CNTL, UMSCH_INT_ENABLE, 0);
+	WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL), ret);
+
+	reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL2));
+	reg_data = REG_SET_FIELD(reg_data, VPEC_CNTL2, IB_FIFO_WATERMARK, 1);
+	WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL2), reg_data);
+
+	if (amdgpu_vpe_configure_dpm(vpe))
+		dev_warn(adev->dev, "VPE DPM not enabled.\n");
+
+	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
+
+		f32_offset = vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL);
+		f32_cntl = RREG32(f32_offset);
+		f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, HALT, 0);
+		f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_RESET, 0);
+
+		adev->vpe.cmdbuf_cpu_addr[0] = f32_offset;
+		adev->vpe.cmdbuf_cpu_addr[1] = f32_cntl;
+
+		return amdgpu_vpe_psp_update_sram(adev);
+	}
+
+	/* Halt and Check F32 cleaness */
+	f32_offset = vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL);
+	f32_cntl = RREG32(f32_offset);
+	f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, HALT, 1);
+	f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_RESET, 1);
+	f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_CHECKSUM_CLR, 1);
+	f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH0_CHECKSUM_CLR, 1);
+	WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL), f32_cntl);
+
+	f32_cntl = RREG32(f32_offset);
+	if (!REG_GET_FIELD(f32_cntl, VPEC_F32_CNTL, HALT)) {
+		dev_err(adev->dev, "VPEC is not halted");
+		return -EBUSY;
+	}
+
+	f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_CHECKSUM_CLR, 0);
+	f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH0_CHECKSUM_CLR, 0);
+	WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL), f32_cntl);
+
+	reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_UCODE_CHECKSUM));
+	if (reg_data) {
+		dev_err(adev->dev, "VPE FW checksum 0 not clean");
+		return -EBUSY;
+	}
+	reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_UCODE1_CHECKSUM));
+	if (reg_data) {
+		dev_err(adev->dev, "VPE FW checksum 1 not clean");
+		return -EBUSY;
+	}
+
+	reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_STATUS2));
+	if (REG_GET_FIELD(reg_data, VPEC_STATUS2, TH0F32_INSTR_PTR)) {
+		dev_err(adev->dev, "VPE FW initial status not clean");
+		return -EBUSY;
+	}
+
+	reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_STATUS6));
+	if (REG_GET_FIELD(reg_data, VPEC_STATUS6, TH1F32_INSTR_PTR)) {
+		dev_err(adev->dev, "VPE FW initial status not clean");
+		return -EBUSY;
+	}
+	/* end of F32 cleaness check */
+
+	vpe_hdr = (const struct vpe_firmware_header_v1_0 *)adev->vpe.fw->data;
+
+	/* Thread 0(command thread) ucode offset/size */
+	ucode_offset[0] = le32_to_cpu(vpe_hdr->header.ucode_array_offset_bytes);
+	ucode_size[0] = le32_to_cpu(vpe_hdr->ctx_ucode_size_bytes);
+	/* Thread 1(control thread) ucode offset/size */
+	ucode_offset[1] = le32_to_cpu(vpe_hdr->ctl_ucode_offset);
+	ucode_size[1] = le32_to_cpu(vpe_hdr->ctl_ucode_size_bytes);
+
+	reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_PG_CNTL));
+	reg_data = REG_SET_FIELD(reg_data, VPEC_PG_CNTL, PG_EN, 0);
+	WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_PG_CNTL), reg_data);
+
+	for (int j = 0; j < vpe->num_instances; j++) {
+		for (int i = 0; i < 2; i++) {
+			if (i > 0)
+				WREG32(vpe_get_reg_offset(vpe, j, regVPEC_UCODE_ADDR), VPE_THREAD1_UCODE_OFFSET);
+			else
+				WREG32(vpe_get_reg_offset(vpe, j, regVPEC_UCODE_ADDR), 0);
+
+			data = (const __le32 *)(adev->vpe.fw->data + ucode_offset[i]);
+			size_dw = ucode_size[i] / sizeof(__le32);
+
+			while (size_dw--) {
+				if (amdgpu_emu_mode && size_dw % 500 == 0)
+					msleep(1);
+				WREG32(vpe_get_reg_offset(vpe, j, regVPEC_UCODE_DATA), le32_to_cpup(data++));
+			}
+		}
+	}
+
+	reg_data = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_PG_CNTL));
+	reg_data = REG_SET_FIELD(reg_data, VPEC_PG_CNTL, PG_EN, 1);
+	WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_PG_CNTL), reg_data);
+
+	/* Unhalt F32 */
+	f32_cntl = RREG32(f32_offset);
+	f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, HALT, 0);
+	f32_cntl = REG_SET_FIELD(f32_cntl, VPEC_F32_CNTL, TH1_RESET, 0);
+	WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_F32_CNTL), f32_cntl);
+
+	return 0;
+}
+
+static int vpe_v2_0_ring_start(struct amdgpu_vpe *vpe)
+{
+	struct amdgpu_ring *ring = &vpe->ring;
+	struct amdgpu_device *adev = ring->adev;
+	uint32_t doorbell, doorbell_offset;
+	uint32_t rb_bufsz, rb_cntl;
+	uint32_t ib_cntl, i;
+	int ret;
+
+	for (i = 0; i < vpe->num_instances; i++) {
+		/* Set ring buffer size in dwords */
+		rb_bufsz = order_base_2(ring->ring_size / 4);
+		rb_cntl = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_CNTL));
+		rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RB_SIZE, rb_bufsz);
+		rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RB_PRIV, 1);
+		rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RB_VMID, 0);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_CNTL), rb_cntl);
+
+		/* Initialize the ring buffer's read and write pointers */
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_RPTR), 0);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_RPTR_HI), 0);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_WPTR), 0);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_WPTR_HI), 0);
+
+		/* set the wb address whether it's enabled or not */
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_RPTR_ADDR_LO),
+			lower_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFC);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_RPTR_ADDR_HI),
+			upper_32_bits(ring->rptr_gpu_addr) & 0xFFFFFFFF);
+
+		rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
+
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_BASE), ring->gpu_addr >> 8);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_BASE_HI), ring->gpu_addr >> 40);
+
+		ring->wptr = 0;
+
+		/* before programing wptr to a less value, need set minor_ptr_update first */
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_MINOR_PTR_UPDATE), 1);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_WPTR), lower_32_bits(ring->wptr) << 2);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_WPTR_HI), upper_32_bits(ring->wptr) << 2);
+		/* set minor_ptr_update to 0 after wptr programed */
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_MINOR_PTR_UPDATE), 0);
+
+		doorbell_offset = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_DOORBELL_OFFSET));
+		doorbell_offset = REG_SET_FIELD(doorbell_offset, VPEC_QUEUE0_DOORBELL_OFFSET, OFFSET, ring->doorbell_index + i*4);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_DOORBELL_OFFSET), doorbell_offset);
+
+		doorbell = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_DOORBELL));
+		doorbell = REG_SET_FIELD(doorbell, VPEC_QUEUE0_DOORBELL, ENABLE, ring->use_doorbell ? 1 : 0);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_DOORBELL), doorbell);
+
+		adev->nbio.funcs->vpe_doorbell_range(adev, i, ring->use_doorbell, ring->doorbell_index + i*4, 4);
+
+		rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RPTR_WRITEBACK_ENABLE, 1);
+		rb_cntl = REG_SET_FIELD(rb_cntl, VPEC_QUEUE0_RB_CNTL, RB_ENABLE, 1);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_RB_CNTL), rb_cntl);
+
+		ib_cntl = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_IB_CNTL));
+		ib_cntl = REG_SET_FIELD(ib_cntl, VPEC_QUEUE0_IB_CNTL, IB_ENABLE, 1);
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE0_IB_CNTL), ib_cntl);
+	}
+
+	ret = amdgpu_ring_test_helper(ring);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static int vpe_v2_0_ring_stop(struct amdgpu_vpe *vpe)
+{
+	struct amdgpu_device *adev = vpe->ring.adev;
+	uint32_t queue_reset, i;
+	int ret;
+
+	for (i = 0; i < vpe->num_instances; i++) {
+		queue_reset = RREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE_RESET_REQ));
+
+		queue_reset = REG_SET_FIELD(queue_reset, VPEC_QUEUE_RESET_REQ, QUEUE0_RESET, 1);
+
+		WREG32(vpe_get_reg_offset(vpe, i, regVPEC_QUEUE_RESET_REQ), queue_reset);
+		/* timeout length is adev->timeout_usec */
+		ret = SOC15_WAIT_ON_RREG(VPE, i, regVPEC_QUEUE_RESET_REQ, 0,
+					 VPEC_QUEUE_RESET_REQ__QUEUE0_RESET_MASK);
+
+		if (ret)
+			dev_err(adev->dev, "VPE queue reset failed\n");
+	}
+
+	vpe->ring.sched.ready = false;
+
+	return ret;
+}
+
+static int vpe_v2_0_set_trap_irq_state(struct amdgpu_device *adev,
+				       struct amdgpu_irq_src *source,
+				       unsigned int type,
+				       enum amdgpu_interrupt_state state)
+{
+	struct amdgpu_vpe *vpe = &adev->vpe;
+	uint32_t vpe_cntl;
+
+	vpe_cntl = RREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL));
+	vpe_cntl = REG_SET_FIELD(vpe_cntl, VPEC_CNTL, TRAP_ENABLE,
+				 state == AMDGPU_IRQ_STATE_ENABLE ? 1 : 0);
+
+	WREG32(vpe_get_reg_offset(vpe, 0, regVPEC_CNTL), vpe_cntl);
+
+	return 0;
+}
+
+static int vpe_v2_0_process_trap_irq(struct amdgpu_device *adev,
+				     struct amdgpu_irq_src *source,
+				     struct amdgpu_iv_entry *entry)
+{
+
+	DRM_DEBUG("IH: VPE trap\n");
+
+	switch (entry->client_id) {
+	case SOC21_IH_CLIENTID_VPE:
+		amdgpu_fence_process(&adev->vpe.ring);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int vpe_v2_0_set_regs(struct amdgpu_vpe *vpe)
+{
+	vpe->regs.queue0_rb_rptr_lo = regVPEC_QUEUE0_RB_RPTR;
+	vpe->regs.queue0_rb_rptr_hi = regVPEC_QUEUE0_RB_RPTR_HI;
+	vpe->regs.queue0_rb_wptr_lo = regVPEC_QUEUE0_RB_WPTR;
+	vpe->regs.queue0_rb_wptr_hi = regVPEC_QUEUE0_RB_WPTR_HI;
+	vpe->regs.queue0_preempt = regVPEC_QUEUE0_PREEMPT;
+	vpe->regs.dpm_enable = regVPEC_PUB_DUMMY2;
+
+	vpe->regs.dpm_pratio = regVPEC_QUEUE6_DUMMY4;
+	vpe->regs.dpm_request_interval = regVPEC_QUEUE5_DUMMY3;
+	vpe->regs.dpm_decision_threshold = regVPEC_QUEUE5_DUMMY4;
+	vpe->regs.dpm_busy_clamp_threshold = regVPEC_QUEUE7_DUMMY2;
+	vpe->regs.dpm_idle_clamp_threshold = regVPEC_QUEUE7_DUMMY3;
+	vpe->regs.dpm_request_lv = regVPEC_QUEUE7_DUMMY1;
+	vpe->regs.context_indicator = regVPEC_QUEUE6_DUMMY3;
+
+	return 0;
+}
+
+static struct vpe_funcs vpe_v2_0_funcs = {
+	.get_reg_offset = vpe_v2_0_get_reg_offset,
+	.set_regs = vpe_v2_0_set_regs,
+	.irq_init = vpe_v2_0_irq_init,
+	.init_microcode = amdgpu_vpe_init_microcode,
+	.load_microcode = vpe_v2_0_load_microcode,
+	.ring_init = amdgpu_vpe_ring_init,
+	.ring_start = vpe_v2_0_ring_start,
+	.ring_stop = vpe_v2_0_ring_stop,
+	.ring_fini = amdgpu_vpe_ring_fini,
+};
+
+static const struct amdgpu_irq_src_funcs vpe_v2_0_trap_irq_funcs = {
+	.set = vpe_v2_0_set_trap_irq_state,
+	.process = vpe_v2_0_process_trap_irq,
+};
+
+void vpe_v2_0_set_funcs(struct amdgpu_vpe *vpe)
+{
+	vpe->funcs = &vpe_v2_0_funcs;
+	vpe->trap_irq.funcs = &vpe_v2_0_trap_irq_funcs;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.h b/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.h
new file mode 100644
index 0000000000000..e9f2077bfdc2d
--- /dev/null
+++ b/drivers/gpu/drm/amd/amdgpu/vpe_v2_0.h
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#ifndef __VPE_V2_0_H__
+#define __VPE_V2_0_H__
+
+#include "amdgpu_vpe.h"
+
+void vpe_v2_0_set_funcs(struct amdgpu_vpe *vpe);
+
+#endif
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
index f95bf6d955346..81bcb16eb6ddc 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_chardev.c
@@ -21,6 +21,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include <linux/capability.h>
 #include <linux/device.h>
 #include <linux/err.h>
 #include <linux/fs.h>
@@ -44,6 +45,7 @@
 #include "kfd_smi_events.h"
 #include "amdgpu_dma_buf.h"
 #include "kfd_debug.h"
+#include "amdgpu_ptl.h"
 
 static long kfd_ioctl(struct file *, unsigned int, unsigned long);
 static int kfd_open(struct inode *, struct file *);
@@ -67,6 +69,21 @@ static const struct class kfd_class = {
 	.name = kfd_dev_name,
 };
 
+/*
+ * Cache the address space of the chardev on first open so that the reset
+ * path can drop all userspace mappings of doorbell and MMIO ranges via
+ * unmap_mapping_range().
+ */
+static struct address_space *kfd_dev_mapping;
+
+void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen)
+{
+	struct address_space *mapping = READ_ONCE(kfd_dev_mapping);
+
+	if (mapping)
+		unmap_mapping_range(mapping, holebegin, holelen, 1);
+}
+
 static inline struct kfd_process_device *kfd_lock_pdd_by_id(struct kfd_process *p, __u32 gpu_id)
 {
 	struct kfd_process_device *pdd;
@@ -133,6 +150,13 @@ static int kfd_open(struct inode *inode, struct file *filep)
 	if (iminor(inode) != 0)
 		return -ENODEV;
 
+	/*
+	 * /dev/kfd is a single chardev so all opens share one inode. Cache
+	 * its address_space on the first open for use by the reset path.
+	 */
+	if (!READ_ONCE(kfd_dev_mapping))
+		cmpxchg(&kfd_dev_mapping, NULL, inode->i_mapping);
+
 	is_32bit_user_mode = in_compat_syscall();
 
 	if (is_32bit_user_mode) {
@@ -147,11 +171,6 @@ static int kfd_open(struct inode *inode, struct file *filep)
 	if (IS_ERR(process))
 		return PTR_ERR(process);
 
-	if (kfd_process_init_cwsr_apu(process, filep)) {
-		kfd_unref_process(process);
-		return -EFAULT;
-	}
-
 	/* filep now owns the reference returned by kfd_create_process */
 	filep->private_data = process;
 
@@ -1777,6 +1796,108 @@ static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
 }
 #endif
 
+static int kfd_ptl_control(struct kfd_process_device *pdd, bool enable)
+{
+	struct amdgpu_device *adev = pdd->dev->adev;
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+	enum amdgpu_ptl_fmt pref_format1 = ptl->fmt1;
+	enum amdgpu_ptl_fmt pref_format2 = ptl->fmt2;
+	uint32_t ptl_state = enable ? 1 : 0;
+	int ret;
+
+	if (!ptl->hw_supported)
+		return -EOPNOTSUPP;
+
+	if (!pdd->dev->kfd2kgd || !pdd->dev->kfd2kgd->ptl_ctrl)
+		return -EOPNOTSUPP;
+
+	ret = pdd->dev->kfd2kgd->ptl_ctrl(adev, PSP_PTL_PERF_MON_SET,
+					  &ptl_state,
+					  &pref_format1,
+					  &pref_format2);
+
+	return ret;
+}
+
+int kfd_ptl_disable_request(struct kfd_process_device *pdd,
+		struct kfd_process *p)
+{
+	struct amdgpu_device *adev = pdd->dev->adev;
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+	int ret = 0;
+
+	mutex_lock(&ptl->mutex);
+
+	if (pdd->ptl_disable_req)
+		goto out;
+
+	if (atomic_inc_return(&ptl->disable_ref) == 1) {
+		ret = kfd_ptl_control(pdd, false);
+		if (ret) {
+			atomic_dec(&ptl->disable_ref);
+			dev_warn(pdd->dev->adev->dev,
+					"failed to disable PTL\n");
+			goto out;
+		}
+	}
+	set_bit(AMDGPU_PTL_DISABLE_PROFILER, ptl->disable_bitmap);
+	pdd->ptl_disable_req = true;
+
+out:
+	mutex_unlock(&ptl->mutex);
+	return ret;
+}
+
+int kfd_ptl_disable_release(struct kfd_process_device *pdd,
+		struct kfd_process *p)
+{
+	struct amdgpu_device *adev = pdd->dev->adev;
+	struct amdgpu_ptl *ptl = &adev->psp.ptl;
+	int ret = 0;
+
+	mutex_lock(&ptl->mutex);
+
+	if (!pdd->ptl_disable_req)
+		goto out;
+
+	if (atomic_dec_return(&ptl->disable_ref) == 0) {
+		clear_bit(AMDGPU_PTL_DISABLE_PROFILER, ptl->disable_bitmap);
+		ret = kfd_ptl_control(pdd, true);
+		if (ret) {
+			atomic_inc(&ptl->disable_ref);
+			set_bit(AMDGPU_PTL_DISABLE_PROFILER, ptl->disable_bitmap);
+			dev_warn(adev->dev, "Failed to enable PTL on release: %d\n", ret);
+			goto out;
+		}
+	}
+	pdd->ptl_disable_req = false;
+
+out:
+	mutex_unlock(&ptl->mutex);
+	return ret;
+}
+
+static int kfd_profiler_ptl_control(struct kfd_process *p,
+		struct kfd_ioctl_ptl_control *args)
+{
+	struct kfd_process_device *pdd;
+	int ret;
+
+	mutex_lock(&p->mutex);
+	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
+	mutex_unlock(&p->mutex);
+
+	if (!pdd || !pdd->dev || !pdd->dev->kfd)
+		return -EINVAL;
+
+	if (args->enable == 0)
+		ret = kfd_ptl_disable_request(pdd, p);
+	else
+		ret = kfd_ptl_disable_release(pdd, p);
+
+	return ret;
+}
+
 static int criu_checkpoint_process(struct kfd_process *p,
 			     uint8_t __user *user_priv_data,
 			     uint64_t *priv_offset)
@@ -2340,6 +2461,9 @@ static int criu_restore_memory_of_gpu(struct kfd_process_device *pdd,
 	const bool criu_resume = true;
 	u64 offset;
 
+	if (bo_priv->idr_handle > INT_MAX)
+		return -EINVAL;
+
 	if (bo_bucket->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL) {
 		if (bo_bucket->size !=
 				kfd_doorbell_process_slice(pdd->dev->kfd))
@@ -3218,6 +3342,107 @@ static int kfd_ioctl_create_process(struct file *filep, struct kfd_process *p, v
 	return 0;
 }
 
+static inline uint32_t profile_lock_device(struct kfd_process *p,
+					   uint32_t gpu_id, uint32_t op)
+{
+	struct kfd_process_device *pdd;
+	struct kfd_dev *kfd;
+	int status = -EINVAL;
+	struct amdgpu_ptl *ptl;
+
+	if (!p)
+		return -EINVAL;
+
+	mutex_lock(&p->mutex);
+	pdd = kfd_process_device_data_by_id(p, gpu_id);
+	mutex_unlock(&p->mutex);
+
+	if (!pdd || !pdd->dev || !pdd->dev->kfd)
+		return -EINVAL;
+
+	kfd = pdd->dev->kfd;
+	ptl = &pdd->dev->adev->psp.ptl;
+
+	mutex_lock(&kfd->profiler_lock);
+	if (op == 1) {
+		if (!kfd->profiler_process) {
+			kfd->profiler_process = p;
+			status = 0;
+			mutex_unlock(&kfd->profiler_lock);
+			if (ptl->hw_supported) {
+				status = kfd_ptl_disable_request(pdd, p);
+				if (status != 0)
+					dev_err(kfd_device,
+						"Failed to lock device %d for profiling, error %d\n",
+						gpu_id, status);
+			}
+			return status;
+		} else if (kfd->profiler_process == p) {
+			status = -EALREADY;
+		} else {
+			status = -EBUSY;
+		}
+	} else if (op == 0 && kfd->profiler_process == p) {
+		kfd->profiler_process = NULL;
+		status = 0;
+		mutex_unlock(&kfd->profiler_lock);
+
+		if (ptl->hw_supported) {
+			status = kfd_ptl_disable_release(pdd, p);
+			if (status)
+				dev_err(kfd_device,
+						"Failed to unlock device %d for profiling, error %d\n",
+						gpu_id, status);
+		}
+		return status;
+	}
+	mutex_unlock(&kfd->profiler_lock);
+
+	return status;
+}
+
+static inline int kfd_profiler_pmc(struct kfd_process *p,
+				   struct kfd_ioctl_pmc_settings *args)
+{
+	struct kfd_process_device *pdd;
+	struct device_queue_manager *dqm;
+	int status;
+
+	/* Check if we have the correct permissions. */
+	if (!perfmon_capable())
+		return -EPERM;
+
+	/* Lock/Unlock the device based on the parameter given in OP */
+	status = profile_lock_device(p, args->gpu_id, args->lock);
+	if (status != 0)
+		return status;
+
+	/* Enable/disable perfcount if requested */
+	mutex_lock(&p->mutex);
+	pdd = kfd_process_device_data_by_id(p, args->gpu_id);
+	dqm = pdd->dev->dqm;
+	mutex_unlock(&p->mutex);
+
+	dqm->ops.set_perfcount(dqm, args->perfcount_enable);
+	return status;
+}
+
+static int kfd_ioctl_profiler(struct file *filep, struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_profiler_args *args = data;
+
+	switch (args->op) {
+	case KFD_IOC_PROFILER_VERSION:
+		args->version = KFD_IOC_PROFILER_VERSION_NUM;
+		return 0;
+	case KFD_IOC_PROFILER_PMC:
+		return kfd_profiler_pmc(p, &args->pmc);
+	case KFD_IOC_PROFILER_PTL_CONTROL:
+		return kfd_profiler_ptl_control(p, &args->ptl);
+	}
+	return -EINVAL;
+}
+
 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
 	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
 			    .validate = NULL, .cmd_drv = 0, .name = #ioctl}
@@ -3344,6 +3569,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = {
 
 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_CREATE_PROCESS,
 			kfd_ioctl_create_process, 0),
+
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_PROFILER,
+			kfd_ioctl_profiler, 0),
 };
 
 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)
@@ -3537,9 +3765,8 @@ static int kfd_mmap(struct file *filep, struct vm_area_struct *vma)
 		return kfd_event_mmap(process, vma);
 
 	case KFD_MMAP_TYPE_RESERVED_MEM:
-		if (!dev)
-			return -ENODEV;
-		return kfd_reserved_mem_mmap(dev, process, vma);
+		pr_warn("KFD_MMAP_TYPE_RESERVED_MEM is no longer supported\n");
+		return -EINVAL;
 	case KFD_MMAP_TYPE_MMIO:
 		if (!dev)
 			return -ENODEV;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
index b7f8f7ff81983..c2c59781feee9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c
@@ -936,6 +936,9 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd,
 
 	svm_range_set_max_pages(kfd->adev);
 
+	kfd->profiler_process = NULL;
+	mutex_init(&kfd->profiler_lock);
+
 	kfd->init_complete = true;
 	dev_info(kfd_device, "added device %x:%x\n", kfd->adev->pdev->vendor,
 		 kfd->adev->pdev->device);
@@ -971,6 +974,7 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd)
 		ida_destroy(&kfd->doorbell_ida);
 		kfd_gtt_sa_fini(kfd);
 		amdgpu_amdkfd_free_kernel_mem(kfd->adev, &kfd->gtt_mem);
+		mutex_destroy(&kfd->profiler_lock);
 	}
 
 	kfree(kfd);
@@ -1647,6 +1651,22 @@ int kgd2kfd_stop_sched_all_nodes(struct kfd_dev *kfd)
 	return 0;
 }
 
+int amdgpu_amdkfd_stop_sched_all(struct amdgpu_device *adev)
+{
+	if (!adev->kfd.init_complete)
+		return 0;
+
+	return kgd2kfd_stop_sched_all_nodes(adev->kfd.dev);
+}
+
+int amdgpu_amdkfd_start_sched_all(struct amdgpu_device *adev)
+{
+	if (!adev->kfd.init_complete)
+		return 0;
+
+	return kgd2kfd_start_sched_all_nodes(adev->kfd.dev);
+}
+
 bool kgd2kfd_compute_active(struct kfd_dev *kfd, uint32_t node_id)
 {
 	struct kfd_node *node;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
index 9185ebe4c079a..5cba592ba941b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c
@@ -71,6 +71,12 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
 				struct queue *q, const uint32_t *restore_sdma_id);
 
 static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma);
+static int resume_all_queues_mes(struct device_queue_manager *dqm);
+static int suspend_all_queues_mes(struct device_queue_manager *dqm);
+static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm,
+						   u32 doorbell_offset);
+static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q,
+			       struct qcm_process_device *qpd);
 
 static inline
 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
@@ -273,13 +279,19 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 	return r;
 }
 
-static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
-			struct qcm_process_device *qpd)
+static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, struct queue *q,
+					    struct qcm_process_device *qpd,
+					    bool is_for_reset,
+					    bool flush_mes_queue)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
 	int r;
 	struct mes_remove_queue_input queue_input;
 
+	/* queue was already removed during reset */
+	if (q->properties.is_reset)
+		return 0;
+
 	if (!dqm->sched_running || dqm->sched_halt)
 		return 0;
 	if (!down_read_trylock(&adev->reset_domain->sem))
@@ -288,6 +300,7 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 	memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input));
 	queue_input.doorbell_offset = q->properties.doorbell_off;
 	queue_input.gang_context_addr = q->gang_ctx_gpu_addr;
+	queue_input.remove_queue_after_reset = flush_mes_queue;
 	queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1;
 
 	amdgpu_mes_lock(&adev->mes);
@@ -295,7 +308,13 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 	amdgpu_mes_unlock(&adev->mes);
 	up_read(&adev->reset_domain->sem);
 
+	if (is_for_reset)
+		return r;
+
 	if (r) {
+		if (!suspend_all_queues_mes(dqm))
+			return resume_all_queues_mes(dqm);
+
 		dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n",
 			q->properties.doorbell_off);
 		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
@@ -305,6 +324,35 @@ static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
 	return r;
 }
 
+static void set_perfcount(struct device_queue_manager *dqm, int enable)
+{
+	struct device_process_node *cur;
+	struct qcm_process_device *qpd;
+	struct queue *q;
+	struct mqd_update_info minfo = { 0 };
+
+	if (!dqm)
+		return;
+
+	minfo.update_flag = (enable == 1 ? UPDATE_FLAG_PERFCOUNT_ENABLE :
+						 UPDATE_FLAG_PERFCOUNT_DISABLE);
+	dqm_lock(dqm);
+	list_for_each_entry(cur, &dqm->queues, list) {
+		qpd = cur->qpd;
+		list_for_each_entry(q, &qpd->queues_list, list) {
+			pqm_update_mqd(qpd->pqm, q->properties.queue_id,
+						&minfo);
+		}
+	}
+	dqm_unlock(dqm);
+}
+
+static int remove_queue_mes(struct device_queue_manager *dqm, struct queue *q,
+			    struct qcm_process_device *qpd)
+{
+	return remove_queue_mes_on_reset_option(dqm, q, qpd, false, false);
+}
+
 static int remove_all_kfd_queues_mes(struct device_queue_manager *dqm)
 {
 	struct device_process_node *cur;
@@ -359,6 +407,92 @@ static int add_all_kfd_queues_mes(struct device_queue_manager *dqm)
 	return retval;
 }
 
+static int reset_queues_mes(struct device_queue_manager *dqm)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
+	int hqd_info_size = adev->mes.hung_queue_hqd_info_offset;
+	int num_hung = 0, r = 0, i, pipe, queue, queue_type;
+	u32 *hung_array = dqm->hung_db_array;
+	struct amdgpu_mes_hung_queue_hqd_info *hqd_info = dqm->hqd_info;
+	struct kfd_process_device *pdd;
+	struct queue *q;
+
+	if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) {
+		r = -ENOTRECOVERABLE;
+		goto fail;
+	}
+
+	/* reset should be used only in dqm locked queue reset */
+	if (WARN_ON(dqm->detect_hang_count > 0))
+		return 0;
+
+	if (!amdgpu_gpu_recovery) {
+		r = -ENOTRECOVERABLE;
+		goto fail;
+	}
+
+	if (!hung_array || !hqd_info) {
+		r = -ENOMEM;
+		goto fail;
+	}
+
+	memset(hqd_info, 0, hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info));
+
+	/*
+	 * AMDGPU_RING_TYPE_COMPUTE parameter does not matter if called
+	 * post suspend_all as reset & detect will return all hung queue types.
+	 *
+	 * Passed parameter is for targeting queues not scheduled by MES add_queue.
+	 */
+	r =  amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE,
+		false, &num_hung, hung_array, ffs(dqm->dev->xcc_mask) - 1);
+
+	if (!num_hung || r) {
+		r = -ENOTRECOVERABLE;
+		goto fail;
+	}
+
+	/* MES resets queue/pipe and cleans up internally */
+	for (i = 0; i < num_hung; i++) {
+		hqd_info[i].bit0_31 = hung_array[i + hqd_info_size];
+		pipe = hqd_info[i].pipe_index;
+		queue = hqd_info[i].queue_index;
+		queue_type = hqd_info[i].queue_type;
+
+		if (queue_type != MES_QUEUE_TYPE_COMPUTE &&
+		    queue_type != MES_QUEUE_TYPE_SDMA) {
+			pr_warn("Unsupported hung queue reset type: %d\n", queue_type);
+			hung_array[i] = AMDGPU_MES_INVALID_DB_OFFSET;
+			continue;
+		}
+
+		q = find_queue_by_doorbell_offset(dqm, hung_array[i]);
+		if (!q) {
+			r = -ENOTRECOVERABLE;
+			goto fail;
+		}
+
+		pdd = kfd_get_process_device_data(q->device, q->process);
+		if (!pdd) {
+			r = -ENODEV;
+			goto fail;
+		}
+
+		pr_warn("Hang detected doorbell %x pipe %d queue %d type %d\n",
+				hung_array[i], pipe, queue, queue_type);
+		/* Proceed remove_queue with reset=true */
+		remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, false);
+		set_queue_as_reset(dqm, q, &pdd->qpd);
+	}
+
+	dqm->detect_hang_count = num_hung;
+	kfd_signal_reset_event(dqm->dev);
+
+fail:
+	dqm->detect_hang_count = 0;
+	return r;
+}
+
 static int suspend_all_queues_mes(struct device_queue_manager *dqm)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
@@ -367,10 +501,13 @@ static int suspend_all_queues_mes(struct device_queue_manager *dqm)
 	if (!down_read_trylock(&adev->reset_domain->sem))
 		return -EIO;
 
-	r = amdgpu_mes_suspend(adev);
+	r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1);
 	up_read(&adev->reset_domain->sem);
 
 	if (r) {
+		if (!reset_queues_mes(dqm))
+			return 0;
+
 		dev_err(adev->dev, "failed to suspend gangs from MES\n");
 		dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n");
 		kfd_hws_hang(dqm);
@@ -387,7 +524,7 @@ static int resume_all_queues_mes(struct device_queue_manager *dqm)
 	if (!down_read_trylock(&adev->reset_domain->sem))
 		return -EIO;
 
-	r = amdgpu_mes_resume(adev);
+	r = amdgpu_mes_resume(adev, ffs(dqm->dev->xcc_mask) - 1);
 	up_read(&adev->reset_domain->sem);
 
 	if (r) {
@@ -475,6 +612,9 @@ static int allocate_doorbell(struct qcm_process_device *qpd,
 	} else {
 		/* For CP queues on SOC15 */
 		if (restore_id) {
+			if (*restore_id >= KFD_MAX_NUM_OF_QUEUES_PER_PROCESS)
+				return -EINVAL;
+
 			/* make sure that ID is free  */
 			if (__test_and_set_bit(*restore_id, qpd->doorbell_bitmap))
 				return -EINVAL;
@@ -1587,6 +1727,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
 		}
 
 		if (restore_sdma_id) {
+			if (*restore_sdma_id >= get_num_sdma_queues(dqm))
+				return -EINVAL;
+
 			/* Re-use existing sdma_id */
 			if (!test_bit(*restore_sdma_id, dqm->sdma_bitmap)) {
 				dev_err(dev, "SDMA queue already in use\n");
@@ -1613,6 +1756,9 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm,
 			return -ENOMEM;
 		}
 		if (restore_sdma_id) {
+			if (*restore_sdma_id >= get_num_xgmi_sdma_queues(dqm))
+				return -EINVAL;
+
 			/* Re-use existing sdma_id */
 			if (!test_bit(*restore_sdma_id, dqm->xgmi_sdma_bitmap)) {
 				dev_err(dev, "SDMA queue already in use\n");
@@ -1797,10 +1943,11 @@ static int halt_cpsch(struct device_queue_manager *dqm)
 static int unhalt_cpsch(struct device_queue_manager *dqm)
 {
 	int ret = 0;
+	struct amdgpu_device *adev = dqm->dev->adev;
 
 	dqm_lock(dqm);
 	if (!dqm->sched_running || !dqm->sched_halt) {
-		WARN_ONCE(!dqm->sched_halt, "Scheduling is not on halt.\n");
+		dev_dbg(adev->dev, "Scheduling is not on halt.\n");
 		dqm_unlock(dqm);
 		return 0;
 	}
@@ -1821,6 +1968,9 @@ static int start_cpsch(struct device_queue_manager *dqm)
 {
 	struct device *dev = dqm->dev->adev->dev;
 	int retval, num_hw_queue_slots;
+	struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev;
+	int hung_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev);
+	int hqd_info_size = adev->mes.hung_queue_hqd_info_offset;
 
 	dqm_lock(dqm);
 
@@ -1870,6 +2020,11 @@ static int start_cpsch(struct device_queue_manager *dqm)
 		goto fail_detect_hang_buffer;
 	}
 
+	dqm->hung_db_array = kzalloc(hung_array_size * sizeof(u32), GFP_KERNEL);
+	dqm->hqd_info = kzalloc(
+		hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info),
+		GFP_KERNEL);
+
 	dqm_unlock(dqm);
 
 	return 0;
@@ -1910,6 +2065,9 @@ static int stop_cpsch(struct device_queue_manager *dqm)
 		pm_uninit(&dqm->packet_mgr);
 	kfree(dqm->detect_hang_info);
 	dqm->detect_hang_info = NULL;
+	kfree(dqm->hung_db_array);
+	kfree(dqm->hqd_info);
+
 	dqm_unlock(dqm);
 
 	return ret;
@@ -2137,6 +2295,7 @@ static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q
 		q->properties.queue_id, pdd->process->lead_thread->pid);
 
 	pdd->has_reset_queue = true;
+	q->properties.is_reset = true;
 	if (q->properties.is_active) {
 		q->properties.is_active = false;
 		decrement_queue_count(dqm, qpd, q);
@@ -2203,6 +2362,23 @@ static struct queue *find_queue_by_address(struct device_queue_manager *dqm, uin
 	return NULL;
 }
 
+static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, u32 doorbell_offset)
+{
+	struct device_process_node *cur;
+	struct qcm_process_device *qpd;
+	struct queue *q;
+
+	list_for_each_entry(cur, &dqm->queues, list) {
+		qpd = cur->qpd;
+		list_for_each_entry(q, &qpd->queues_list, list) {
+			if (doorbell_offset == q->properties.doorbell_off)
+				return q;
+		}
+	}
+
+	return NULL;
+}
+
 static int reset_hung_queues(struct device_queue_manager *dqm)
 {
 	int r = 0, reset_count = 0, i;
@@ -2970,6 +3146,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 		dqm->ops.reset_queues = reset_queues_cpsch;
 		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
 		dqm->ops.checkpoint_mqd = checkpoint_mqd;
+		dqm->ops.set_perfcount = set_perfcount;
 		break;
 	case KFD_SCHED_POLICY_NO_HWS:
 		/* initialize dqm for no cp scheduling */
@@ -2990,6 +3167,7 @@ struct device_queue_manager *device_queue_manager_init(struct kfd_node *dev)
 		dqm->ops.get_wave_state = get_wave_state;
 		dqm->ops.get_queue_checkpoint_info = get_queue_checkpoint_info;
 		dqm->ops.checkpoint_mqd = checkpoint_mqd;
+		dqm->ops.set_perfcount = set_perfcount;
 		break;
 	default:
 		dev_err(dev->adev->dev, "Invalid scheduling policy %d\n", dqm->sched_policy);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index 3272328da11f9..e0b6a47e7722b 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -32,7 +32,6 @@
 #include "kfd_priv.h"
 #include "kfd_mqd_manager.h"
 
-
 #define VMID_NUM 16
 
 #define KFD_MES_PROCESS_QUANTUM		100000
@@ -200,6 +199,8 @@ struct device_queue_manager_ops {
 				  const struct queue *q,
 				  void *mqd,
 				  void *ctl_stack);
+	void	(*set_perfcount)(struct device_queue_manager *dqm,
+				  int enable);
 };
 
 struct device_queue_manager_asic_ops {
@@ -285,6 +286,9 @@ struct device_queue_manager {
 	struct dqm_detect_hang_info *detect_hang_info;
 	size_t detect_hang_info_size;
 	int detect_hang_count;
+	/* for per-queue reset with mes */
+	u32 *hung_db_array;
+	struct amdgpu_mes_hung_queue_hqd_info *hqd_info;
 };
 
 void device_queue_manager_init_cik(
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
index 9e70a5f8a50b0..5259893713782 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v12_1.c
@@ -54,6 +54,7 @@ static int update_qpd_v12_1(struct device_queue_manager *dqm,
 	struct kfd_process_device *pdd;
 	struct amdgpu_device *adev = dqm->dev->adev;
 	struct amdgpu_vmhub *hub = &adev->vmhub[AMDGPU_GFXHUB(0)];
+	bool xnack_enabled;
 
 	pdd = qpd_to_pdd(qpd);
 	qpd->vm_cntx_cntl = hub->vm_cntx_cntl;
@@ -71,16 +72,18 @@ static int update_qpd_v12_1(struct device_queue_manager *dqm,
 		qpd->sh_mem_ape1_base = 0;
 	}
 
-	if (KFD_SUPPORT_XNACK_PER_PROCESS(dqm->dev)) {
-		if (!pdd->process->xnack_enabled) {
-			qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
-			qpd->vm_cntx_cntl &=
+	xnack_enabled = KFD_SUPPORT_XNACK_PER_PROCESS(dqm->dev) ?
+			pdd->process->xnack_enabled :
+			!pdd->dev->kfd->noretry;
+
+	if (!xnack_enabled) {
+		qpd->sh_mem_config |= 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
+		qpd->vm_cntx_cntl &=
 			~(1 << GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT);
-		} else {
-			qpd->sh_mem_config &= ~(1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT);
-			qpd->vm_cntx_cntl |=
+	} else {
+		qpd->sh_mem_config &= ~(1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT);
+		qpd->vm_cntx_cntl |=
 			(1 << GCVM_CONTEXT0_CNTL__RETRY_PERMISSION_OR_INVALID_PAGE_FAULT__SHIFT);
-		}
 	}
 
 	qpd->sh_mem_bases = compute_sh_mem_bases_64bit(pdd);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_events.c b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
index 44150a71ffd51..a11c4ab3aafd9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_events.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_events.c
@@ -483,6 +483,11 @@ int kfd_criu_restore_event(struct file *devkfd,
 	}
 	*priv_data_offset += sizeof(*ev_priv);
 
+	if (ev_priv->event_id > INT_MAX) {
+		ret = -EINVAL;
+		goto exit;
+	}
+
 	if (ev_priv->user_handle) {
 		ret = kfd_kmap_event_page(p, ev_priv->user_handle);
 		if (ret)
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
index 964efa3259089..28dc6886c1ff1 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_migrate.c
@@ -129,13 +129,14 @@ svm_migrate_copy_memory_gart(struct amdgpu_device *adev, dma_addr_t *sys,
 			     struct dma_fence **mfence)
 {
 	const u64 GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE;
-	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
+	struct amdgpu_ring *ring;
 	struct amdgpu_ttm_buffer_entity *entity;
 	u64 gart_s, gart_d;
 	struct dma_fence *next;
 	u64 size;
 	int r;
 
+	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
 	entity = &adev->mman.move_entities[0];
 
 	mutex_lock(&entity->lock);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
index 77fb41e2486a4..8e8ec266ca46d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v10.c
@@ -123,10 +123,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 	 */
 	m->cp_hqd_hq_scheduler0 = 1 << 14;
 
-	if (q->format == KFD_QUEUE_FORMAT_AQL) {
+	if (q->format == KFD_QUEUE_FORMAT_AQL)
 		m->cp_hqd_aql_control =
 			1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
-	}
 
 	if (mm->dev->kfd->cwsr_enabled) {
 		m->cp_hqd_persistent_state |=
@@ -141,6 +140,12 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
 	}
 
+	mutex_lock(&mm->dev->kfd->profiler_lock);
+	if (mm->dev->kfd->profiler_process != NULL)
+		m->compute_perfcount_enable = 1;
+
+	mutex_unlock(&mm->dev->kfd->profiler_lock);
+
 	*mqd = m;
 	if (gart_addr)
 		*gart_addr = addr;
@@ -220,6 +225,13 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
 	if (mm->dev->kfd->cwsr_enabled)
 		m->cp_hqd_ctx_save_control = 0;
 
+	if (minfo) {
+		if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+			m->compute_perfcount_enable = 1;
+		else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+			m->compute_perfcount_enable = 0;
+	}
+
 	update_cu_mask(mm, mqd, minfo);
 	set_priority(m, q);
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
index a1e3cf2384dd3..7568e7ed5244d 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v11.c
@@ -163,10 +163,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 	if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev))
 		m->cp_hqd_hq_status0 |= 1 << 29;
 
-	if (q->format == KFD_QUEUE_FORMAT_AQL) {
+	if (q->format == KFD_QUEUE_FORMAT_AQL)
 		m->cp_hqd_aql_control =
 			1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
-	}
 
 	if (mm->dev->kfd->cwsr_enabled) {
 		m->cp_hqd_persistent_state |=
@@ -181,6 +180,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
 	}
 
+	mutex_lock(&mm->dev->kfd->profiler_lock);
+	if (mm->dev->kfd->profiler_process != NULL)
+		m->compute_perfcount_enable = 1;
+	mutex_unlock(&mm->dev->kfd->profiler_lock);
+
 	*mqd = m;
 	if (gart_addr)
 		*gart_addr = addr;
@@ -258,6 +262,12 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
 	}
 	if (mm->dev->kfd->cwsr_enabled)
 		m->cp_hqd_ctx_save_control = 0;
+	if (minfo) {
+		if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+			m->compute_perfcount_enable = 1;
+		else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+			m->compute_perfcount_enable = 0;
+	}
 
 	update_cu_mask(mm, mqd, minfo);
 	set_priority(m, q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
index b3e122d7876e0..8c815f129614c 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12.c
@@ -138,10 +138,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 	if (amdgpu_amdkfd_have_atomics_support(mm->dev->adev))
 		m->cp_hqd_hq_status0 |= 1 << 29;
 
-	if (q->format == KFD_QUEUE_FORMAT_AQL) {
+	if (q->format == KFD_QUEUE_FORMAT_AQL)
 		m->cp_hqd_aql_control =
 			1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
-	}
 
 	if (mm->dev->kfd->cwsr_enabled) {
 		m->cp_hqd_persistent_state |=
@@ -156,6 +155,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
 	}
 
+	mutex_lock(&mm->dev->kfd->profiler_lock);
+	if (mm->dev->kfd->profiler_process != NULL)
+		m->compute_perfcount_enable = 1;
+	mutex_unlock(&mm->dev->kfd->profiler_lock);
+
 	*mqd = m;
 	if (gart_addr)
 		*gart_addr = addr;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
index c90c0d99b1e3f..475589b924e90 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c
@@ -32,6 +32,10 @@
 #include "amdgpu_amdkfd.h"
 #include "kfd_device_queue_manager.h"
 
+static void update_mqd(struct mqd_manager *mm, void *mqd,
+		       struct queue_properties *q,
+		       struct mqd_update_info *minfo);
+
 static inline struct v12_1_compute_mqd *get_mqd(void *mqd)
 {
 	return (struct v12_1_compute_mqd *)mqd;
@@ -215,7 +219,7 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 	*mqd = m;
 	if (gart_addr)
 		*gart_addr = addr;
-	mm->update_mqd(mm, m, q, NULL);
+	update_mqd(mm, m, q, NULL);
 }
 
 static int load_mqd(struct mqd_manager *mm, void *mqd,
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
index e8f97de9d6e47..17bfb419b2020 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c
@@ -227,10 +227,9 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_aql_control =
 			1 << CP_HQD_AQL_CONTROL__CONTROL0__SHIFT;
 
-	if (q->tba_addr) {
+	if (q->tba_addr)
 		m->compute_pgm_rsrc2 |=
 			(1 << COMPUTE_PGM_RSRC2__TRAP_PRESENT__SHIFT);
-	}
 
 	if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address) {
 		m->cp_hqd_persistent_state |=
@@ -245,6 +244,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
 	}
 
+	mutex_lock(&mm->dev->kfd->profiler_lock);
+	if (mm->dev->kfd->profiler_process != NULL)
+		m->compute_perfcount_enable = 1;
+	mutex_unlock(&mm->dev->kfd->profiler_lock);
+
 	*mqd = m;
 	if (gart_addr)
 		*gart_addr = addr;
@@ -327,6 +331,13 @@ static void update_mqd(struct mqd_manager *mm, void *mqd,
 	if (mm->dev->kfd->cwsr_enabled && q->ctx_save_restore_area_address)
 		m->cp_hqd_ctx_save_control = 0;
 
+	if (minfo) {
+		if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+			m->compute_perfcount_enable = 1;
+		else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+			m->compute_perfcount_enable = 0;
+	}
+
 	if (KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 3) &&
 	    KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 4, 4) &&
 	    KFD_GC_VERSION(mm->dev) != IP_VERSION(9, 5, 0))
@@ -364,11 +375,15 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
 {
 	struct v9_mqd *m;
 	struct kfd_context_save_area_header header;
+	u32 cntl_stack_size;
+	u32 cntl_stack_offset;
 
 	/* Control stack is located one page after MQD. */
 	void *mqd_ctl_stack = (void *)((uintptr_t)mqd + AMDGPU_GPU_PAGE_SIZE);
 
 	m = get_mqd(mqd);
+	cntl_stack_size = min_t(u32, m->cp_hqd_cntl_stack_size,   q->ctl_stack_size);
+	cntl_stack_offset = min_t(u32, m->cp_hqd_cntl_stack_offset, cntl_stack_size);
 
 	*ctl_stack_used_size = m->cp_hqd_cntl_stack_size -
 		m->cp_hqd_cntl_stack_offset;
@@ -384,9 +399,10 @@ static int get_wave_state(struct mqd_manager *mm, void *mqd,
 	if (copy_to_user(ctl_stack, &header, sizeof(header.wave_state)))
 		return -EFAULT;
 
-	if (copy_to_user(ctl_stack + m->cp_hqd_cntl_stack_offset,
-				mqd_ctl_stack + m->cp_hqd_cntl_stack_offset,
-				*ctl_stack_used_size))
+	*ctl_stack_used_size = cntl_stack_size - cntl_stack_offset;
+
+	if (copy_to_user(ctl_stack + cntl_stack_offset, mqd_ctl_stack + cntl_stack_offset,
+					*ctl_stack_used_size))
 		return -EFAULT;
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
index 431a20323146b..c86779af323bb 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c
@@ -148,6 +148,11 @@ static void init_mqd(struct mqd_manager *mm, void **mqd,
 		m->cp_hqd_wg_state_offset = q->ctl_stack_size;
 	}
 
+	mutex_lock(&mm->dev->kfd->profiler_lock);
+	if (mm->dev->kfd->profiler_process != NULL)
+		m->compute_perfcount_enable = 1;
+	mutex_unlock(&mm->dev->kfd->profiler_lock);
+
 	*mqd = m;
 	if (gart_addr)
 		*gart_addr = addr;
@@ -230,6 +235,12 @@ static void __update_mqd(struct mqd_manager *mm, void *mqd,
 		m->cp_hqd_ctx_save_control =
 			atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
 			mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
+	if (minfo) {
+		if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_ENABLE)
+			m->compute_perfcount_enable = 1;
+		else if (minfo->update_flag == UPDATE_FLAG_PERFCOUNT_DISABLE)
+			m->compute_perfcount_enable = 0;
+	}
 
 	update_cu_mask(mm, mqd, minfo);
 	set_priority(m, q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index 7b5b122069198..acd0e41e744c9 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -215,8 +215,7 @@ enum cache_policy {
 	((KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 2)) ||	\
 	 (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 3)) ||	\
 	 (KFD_GC_VERSION(dev) == IP_VERSION(9, 4, 4)) ||	\
-	 (KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)) ||	\
-	 (KFD_GC_VERSION(dev) == IP_VERSION(12, 1, 0)))
+	 (KFD_GC_VERSION(dev) == IP_VERSION(9, 5, 0)))
 
 struct kfd_node;
 
@@ -384,6 +383,11 @@ struct kfd_dev {
 	int kfd_dev_lock;
 
 	atomic_t kfd_processes_count;
+
+	/* Lock for profiler process */
+	struct mutex profiler_lock;
+	/* Process currently holding the lock */
+	struct kfd_process *profiler_process;
 };
 
 enum kfd_mempool {
@@ -395,6 +399,7 @@ enum kfd_mempool {
 /* Character device interface */
 int kfd_chardev_init(void);
 void kfd_chardev_exit(void);
+void kfd_dev_unmap_mapping_range(loff_t const holebegin, loff_t const holelen);
 
 /**
  * enum kfd_unmap_queues_filter - Enum for queue filters.
@@ -523,6 +528,7 @@ struct queue_properties {
 	uint32_t pm4_target_xcc;
 	bool is_dbg_wa;
 	bool is_user_cu_masked;
+	bool is_reset;
 	/* Not relevant for user mode queues in cp scheduling */
 	unsigned int vmid;
 	/* Relevant only for sdma queues*/
@@ -556,6 +562,8 @@ enum mqd_update_flag {
 	UPDATE_FLAG_DBG_WA_ENABLE = 1,
 	UPDATE_FLAG_DBG_WA_DISABLE = 2,
 	UPDATE_FLAG_IS_GWS = 4, /* quirk for gfx9 IP */
+	UPDATE_FLAG_PERFCOUNT_ENABLE = 5,
+	UPDATE_FLAG_PERFCOUNT_DISABLE = 6,
 };
 
 struct mqd_update_info {
@@ -865,6 +873,8 @@ struct kfd_process_device {
 	bool has_reset_queue;
 
 	u32 pasid;
+	/* Indicates this process has requested PTL stay disabled */
+	bool ptl_disable_req;
 };
 
 #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
@@ -986,9 +996,6 @@ struct kfd_process {
 	struct kobject *kobj_queues;
 	struct attribute attr_pasid;
 
-	/* Keep track cwsr init */
-	bool has_cwsr;
-
 	/* Exception code enable mask and status */
 	uint64_t exception_enable_mask;
 	uint64_t exception_status;
@@ -1104,8 +1111,6 @@ struct kfd_process_device *kfd_create_process_device_data(struct kfd_node *dev,
 
 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);
 
-int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
-			  struct vm_area_struct *vma);
 void kfd_process_notifier_release_internal(struct kfd_process *p);
 
 /* KFD process API for creating and translating handles */
@@ -1222,9 +1227,6 @@ void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
 void kfd_process_set_trap_debug_flag(struct qcm_process_device *qpd,
 				     bool enabled);
 
-/* CWSR initialization */
-int kfd_process_init_cwsr_apu(struct kfd_process *process, struct file *filep);
-
 /* CRIU */
 /*
  * Need to increment KFD_CRIU_PRIV_VERSION each time a change is made to any of the CRIU private
@@ -1604,6 +1606,12 @@ static inline bool kfd_is_first_node(struct kfd_node *node)
 	return (node == node->kfd->nodes[0]);
 }
 
+/* PTL support */
+int kfd_ptl_disable_request(struct kfd_process_device *pdd,
+		struct kfd_process *p);
+int kfd_ptl_disable_release(struct kfd_process_device *pdd,
+		struct kfd_process *p);
+
 /* Debugfs */
 #if defined(CONFIG_DEBUG_FS)
 
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index d28ca581cad05..368283d53077e 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -1106,6 +1106,16 @@ static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
 		kfd_process_device_free_bos(p->pdds[i]);
 }
 
+static void kfd_process_profiler_release(struct kfd_process *p, struct kfd_process_device *pdd)
+{
+	mutex_lock(&pdd->dev->kfd->profiler_lock);
+	if (pdd->dev->kfd->profiler_process == p) {
+		pdd->qpd.dqm->ops.set_perfcount(pdd->qpd.dqm, 0);
+		pdd->dev->kfd->profiler_process = NULL;
+	}
+	mutex_unlock(&pdd->dev->kfd->profiler_lock);
+}
+
 static void kfd_process_destroy_pdds(struct kfd_process *p)
 {
 	int i;
@@ -1117,6 +1127,11 @@ static void kfd_process_destroy_pdds(struct kfd_process *p)
 
 		pr_debug("Releasing pdd (topology id %d, for pid %d)\n",
 			pdd->dev->id, p->lead_thread->pid);
+		kfd_process_profiler_release(p, pdd);
+
+		if (pdd->ptl_disable_req)
+			kfd_ptl_disable_release(pdd, p);
+
 		kfd_process_device_destroy_cwsr_dgpu(pdd);
 		kfd_process_device_destroy_ib_mem(pdd);
 
@@ -1409,50 +1424,6 @@ void kfd_cleanup_processes(void)
 	mmu_notifier_synchronize();
 }
 
-int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
-{
-	unsigned long  offset;
-	int i;
-
-	if (p->has_cwsr)
-		return 0;
-
-	for (i = 0; i < p->n_pdds; i++) {
-		struct kfd_node *dev = p->pdds[i]->dev;
-		struct qcm_process_device *qpd = &p->pdds[i]->qpd;
-
-		if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
-			continue;
-
-		offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
-		qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
-			KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
-			MAP_SHARED, offset);
-
-		if (IS_ERR_VALUE(qpd->tba_addr)) {
-			int err = qpd->tba_addr;
-
-			dev_err(dev->adev->dev,
-				"Failure to set tba address. error %d.\n", err);
-			qpd->tba_addr = 0;
-			qpd->cwsr_kaddr = NULL;
-			return err;
-		}
-
-		memcpy(qpd->cwsr_kaddr, dev->kfd->cwsr_isa, dev->kfd->cwsr_isa_size);
-
-		kfd_process_set_trap_debug_flag(qpd, p->debug_trap_enabled);
-
-		qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
-		pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
-			qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
-	}
-
-	p->has_cwsr = true;
-
-	return 0;
-}
-
 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
 {
 	struct kfd_node *dev = pdd->dev;
@@ -2228,38 +2199,6 @@ int kfd_resume_all_processes(void)
 	return ret;
 }
 
-int kfd_reserved_mem_mmap(struct kfd_node *dev, struct kfd_process *process,
-			  struct vm_area_struct *vma)
-{
-	struct kfd_process_device *pdd;
-	struct qcm_process_device *qpd;
-
-	if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
-		dev_err(dev->adev->dev, "Incorrect CWSR mapping size.\n");
-		return -EINVAL;
-	}
-
-	pdd = kfd_get_process_device_data(dev, process);
-	if (!pdd)
-		return -EINVAL;
-	qpd = &pdd->qpd;
-
-	qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
-					get_order(KFD_CWSR_TBA_TMA_SIZE));
-	if (!qpd->cwsr_kaddr) {
-		dev_err(dev->adev->dev,
-			"Error allocating per process CWSR buffer.\n");
-		return -ENOMEM;
-	}
-
-	vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND
-		| VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP);
-	/* Mapping pages to user process */
-	return remap_pfn_range(vma, vma->vm_start,
-			       PFN_DOWN(__pa(qpd->cwsr_kaddr)),
-			       KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
-}
-
 /* assumes caller holds process lock. */
 int kfd_process_drain_interrupts(struct kfd_process_device *pdd)
 {
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
index 29dee26261ab1..630f46091dc75 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_topology.c
@@ -514,7 +514,8 @@ static ssize_t node_show(struct kobject *kobj, struct attribute *attr,
 			dev->node_props.capability |=
 					HSA_CAP_AQL_QUEUE_DOUBLE_MAP;
 
-		if (KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0) &&
+		if ((KFD_GC_VERSION(dev->gpu) < IP_VERSION(10, 0, 0) ||
+			KFD_GC_VERSION(dev->gpu) == IP_VERSION(12, 1, 0)) &&
 			(dev->gpu->adev->sdma.supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE))
 				dev->node_props.capability2 |= HSA_CAP2_PER_SDMA_QUEUE_RESET_SUPPORTED;
 
@@ -2027,6 +2028,7 @@ static void kfd_topology_set_capabilities(struct kfd_topology_device *dev)
 		if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 1, 0)) {
 			dev->node_props.capability |=
 				HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED;
+			dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED;
 			dev->node_props.capability2 |=
 				HSA_CAP2_TRAP_DEBUG_LDS_OUT_OF_ADDR_RANGE_SUPPORTED;
 		}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
index 5fc5d56085066..ba7f98a87808c 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c
@@ -104,6 +104,7 @@
 #include "ivsrcid/dcn/irqsrcs_dcn_1_0.h"
 
 #include "modules/inc/mod_freesync.h"
+#include "modules/inc/mod_power.h"
 #include "modules/power/power_helpers.h"
 
 static_assert(AMDGPU_DMUB_NOTIFICATION_MAX == DMUB_NOTIFICATION_MAX, "AMDGPU_DMUB_NOTIFICATION_MAX mismatch");
@@ -232,11 +233,11 @@ static int amdgpu_dm_encoder_init(struct drm_device *dev,
 
 static int amdgpu_dm_connector_get_modes(struct drm_connector *connector);
 
-static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_state *state);
-static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state);
+static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state);
+static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state);
 
 static int amdgpu_dm_atomic_check(struct drm_device *dev,
-				  struct drm_atomic_state *state);
+				  struct drm_atomic_commit *state);
 
 static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector);
 static void handle_hpd_rx_irq(void *param);
@@ -1869,7 +1870,7 @@ static enum dmub_ips_disable_type dm_get_default_ips_mode(
 		ret =  DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF;
 		break;
 	case IP_VERSION(4, 2, 0):
-		ret =  DMUB_IPS_DISABLE_ALL;
+		ret =  DMUB_IPS_ENABLE;
 		break;
 	default:
 		/* ASICs older than DCN35 do not have IPSs */
@@ -1881,6 +1882,70 @@ static enum dmub_ips_disable_type dm_get_default_ips_mode(
 	return ret;
 }
 
+static int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm)
+{
+	struct mod_power_init_params init_data[MAX_NUM_EDP];
+
+	if (dm->num_of_edps == 0) {
+		drm_dbg_driver(
+			dm->ddev,
+			"amdgpu: No eDP detected, skip initializing power module\n");
+		return 0;
+	}
+
+	/* Initialize all the power module parameters */
+	for (int i = 0; i < dm->num_of_edps; i++) {
+		init_data[i].allow_psr_smu_optimizations =
+			!!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT);
+		init_data[i].allow_psr_multi_disp_optimizations =
+			!!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_MULTI_DISP_OPT);
+		/* See dm_late_init */
+		init_data[i].backlight_ramping_override = false;
+		init_data[i].backlight_ramping_start = 0xCCCC;
+		init_data[i].backlight_ramping_reduction = 0xCCCCCCCC;
+		init_data[i].def_varibright_level = 0;
+		init_data[i].abm_config_setting = 0;
+		init_data[i].num_backlight_levels = 101;
+		init_data[i].use_nits_based_brightness = false;
+		init_data[i].panel_max_millinits = 0;
+		init_data[i].panel_min_millinits = 0;
+		init_data[i].disable_fractional_pwm =
+			!(amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK);
+		init_data[i].use_custom_backlight_caps = false;
+		init_data[i].custom_backlight_caps_config_no = 0;
+		init_data[i].use_linear_backlight_curve = false;
+		init_data[i].def_varibright_enable = 0;
+		init_data[i].varibright_level = 0;
+		/*
+		 * Power module uses 16-bit backlight levels (0xFFFF max) rather
+		 * than 8-bit(0XFF max)
+		 */
+		init_data[i].min_backlight_pwm =
+			dm->backlight_caps[i].min_input_signal * 0x101;
+		init_data[i].max_backlight_pwm =
+			dm->backlight_caps[i].max_input_signal * 0x101;
+		init_data[i].min_abm_backlight =
+			dm->backlight_caps[i].min_input_signal * 0x101;
+
+		/* Min backlight level after ABM reduction,  Don't allow below 1%
+		 * 0xFFFF x 0.01 = 0x28F
+		 */
+		init_data[i].min_abm_backlight = (init_data[i].min_abm_backlight < 0x28F) ?
+			0x28F : init_data[i].min_abm_backlight;
+	}
+
+	dm->power_module = mod_power_create(dm->dc, init_data, dm->num_of_edps);
+	if (!dm->power_module) {
+		drm_err(dm->ddev, "amdgpu: Error allocating memory for power module\n");
+		return -ENOMEM;
+	}
+
+	mod_power_hw_init(dm->power_module);
+	drm_dbg_driver(dm->ddev, "amdgpu: Power module init done\n");
+
+	return 0;
+}
+
 static int amdgpu_dm_init(struct amdgpu_device *adev)
 {
 	struct dc_init_data init_data;
@@ -1898,6 +1963,8 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 	mutex_init(&adev->dm.dc_lock);
 	mutex_init(&adev->dm.audio_lock);
 
+	spin_lock_init(&adev->dm.dmub_lock);
+
 	if (amdgpu_dm_irq_init(adev)) {
 		drm_err(adev_to_drm(adev), "failed to initialize DM IRQ support.\n");
 		goto error;
@@ -2198,6 +2265,9 @@ static int amdgpu_dm_init(struct amdgpu_device *adev)
 		goto error;
 	}
 
+	if (amdgpu_dm_init_power_module(&adev->dm))
+		goto error;
+
 	/* create fake encoders for MST */
 	dm_dp_create_fake_mst_encoders(adev);
 
@@ -2260,9 +2330,16 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev)
 		adev->dm.idle_workqueue = NULL;
 	}
 
-	/* Disable ISM before dc_destroy() invalidates dm->dc */
+	/*
+	 * Disable ISM before dc_destroy() invalidates dm->dc.
+	 *
+	 * Quiesce workers first without dc_lock (they take dc_lock
+	 * themselves, so syncing under it would deadlock), then drive the
+	 * FSM back to FULL_POWER_RUNNING under dc_lock.
+	 */
+	amdgpu_dm_ism_disable(&adev->dm);
 	scoped_guard(mutex, &adev->dm.dc_lock)
-		amdgpu_dm_ism_disable(&adev->dm);
+		amdgpu_dm_ism_force_full_power(&adev->dm);
 
 	amdgpu_dm_destroy_drm_device(&adev->dm);
 
@@ -2334,6 +2411,10 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev)
 		adev->dm.freesync_module = NULL;
 	}
 
+	if (adev->dm.power_module) {
+		mod_power_destroy(adev->dm.power_module);
+		adev->dm.power_module = NULL;
+	}
 	mutex_destroy(&adev->dm.audio_lock);
 	mutex_destroy(&adev->dm.dc_lock);
 	mutex_destroy(&adev->dm.dpia_aux_lock);
@@ -3290,9 +3371,14 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block)
 	if (amdgpu_in_reset(adev)) {
 		enum dc_status res;
 
+		/* Quiesce ISM workers before taking dc_lock (workers take
+		 * dc_lock themselves; syncing under it would deadlock).
+		 */
+		amdgpu_dm_ism_disable(dm);
+
 		mutex_lock(&dm->dc_lock);
 
-		amdgpu_dm_ism_disable(dm);
+		amdgpu_dm_ism_force_full_power(dm);
 		dc_allow_idle_optimizations(adev->dm.dc, false);
 
 		dm->cached_dc_state = dc_state_create_copy(dm->dc->current_state);
@@ -3326,8 +3412,13 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block)
 
 	amdgpu_dm_irq_suspend(adev);
 
+	/*
+	 * Quiesce ISM workers before taking dc_lock (workers take dc_lock
+	 * themselves; syncing under it would deadlock).
+	 */
+	amdgpu_dm_ism_disable(dm);
 	scoped_guard(mutex, &dm->dc_lock)
-		amdgpu_dm_ism_disable(dm);
+		amdgpu_dm_ism_force_full_power(dm);
 
 	hpd_rx_irq_work_suspend(dm);
 
@@ -3342,7 +3433,7 @@ static int dm_suspend(struct amdgpu_ip_block *ip_block)
 }
 
 struct drm_connector *
-amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_state *state,
+amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_commit *state,
 					     struct drm_crtc *crtc)
 {
 	u32 i;
@@ -4906,7 +4997,7 @@ static int register_outbox_irq_handlers(struct amdgpu_device *adev)
  *
  * This should only be called during atomic check.
  */
-int dm_atomic_get_state(struct drm_atomic_state *state,
+int dm_atomic_get_state(struct drm_atomic_commit *state,
 			struct dm_atomic_state **dm_state)
 {
 	struct drm_device *dev = state->dev;
@@ -4927,7 +5018,7 @@ int dm_atomic_get_state(struct drm_atomic_state *state,
 }
 
 static struct dm_atomic_state *
-dm_atomic_get_new_state(struct drm_atomic_state *state)
+dm_atomic_get_new_state(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct amdgpu_device *adev = drm_to_adev(dev);
@@ -5054,8 +5145,8 @@ static int amdgpu_dm_mode_config_init(struct amdgpu_device *adev)
 #define AMDGPU_DM_MIN_SPREAD ((AMDGPU_DM_DEFAULT_MAX_BACKLIGHT - AMDGPU_DM_DEFAULT_MIN_BACKLIGHT) / 2)
 #define AUX_BL_DEFAULT_TRANSITION_TIME_MS 50
 
-static void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm,
-					    int bl_idx)
+void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm,
+				     int bl_idx)
 {
 	struct amdgpu_dm_backlight_caps *caps = &dm->backlight_caps[bl_idx];
 
@@ -5217,15 +5308,34 @@ static u32 convert_brightness_to_user(const struct amdgpu_dm_backlight_caps *cap
 				 max - min);
 }
 
+static struct dc_stream_state *dm_find_stream_with_link(
+	struct amdgpu_display_manager *dm,
+	struct dc_link *link)
+{
+	struct dc_state *cur_dc_state = dm->dc->current_state;
+	struct dc_stream_state *stream = NULL;
+	int i;
+
+	for (i = 0; i < cur_dc_state->stream_count; i++) {
+		stream = cur_dc_state->streams[i];
+		if (stream->link == link)
+			return stream;
+	}
+
+	return NULL;
+}
+
 static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
 					 int bl_idx,
 					 u32 user_brightness)
 {
 	struct amdgpu_dm_backlight_caps *caps;
 	struct dc_link *link;
-	u32 brightness;
-	bool rc, reallow_idle = false;
+	u32 brightness = 0;
+	bool rc = false, reallow_idle = false;
 	struct drm_connector *connector;
+	struct dc_stream_state *stream;
+	unsigned int min, max;
 
 	list_for_each_entry(connector, &dm->ddev->mode_config.connector_list, head) {
 		struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
@@ -5255,13 +5365,6 @@ static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
 	if (caps->brightness_mask)
 		brightness |= caps->brightness_mask;
 
-	/* Change brightness based on AUX property */
-	mutex_lock(&dm->dc_lock);
-	if (dm->dc->caps.ips_support && dm->dc->ctx->dmub_srv->idle_allowed) {
-		dc_allow_idle_optimizations(dm->dc, false);
-		reallow_idle = true;
-	}
-
 	if (trace_amdgpu_dm_brightness_enabled()) {
 		trace_amdgpu_dm_brightness(__builtin_return_address(0),
 					   user_brightness,
@@ -5270,22 +5373,45 @@ static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm,
 					   power_supply_is_system_supplied() > 0);
 	}
 
-	if (caps->aux_support) {
-		rc = dc_link_set_backlight_level_nits(link, true, brightness,
-						      AUX_BL_DEFAULT_TRANSITION_TIME_MS);
-		if (!rc)
-			DRM_DEBUG("DM: Failed to update backlight via AUX on eDP[%d]\n", bl_idx);
-	} else {
-		struct set_backlight_level_params backlight_level_params = { 0 };
+	stream = dm_find_stream_with_link(dm, link);
+	if (!stream)
+		return;
 
-		backlight_level_params.backlight_pwm_u16_16 = brightness;
-		backlight_level_params.transition_time_in_ms = 0;
+	mutex_lock(&dm->dc_lock);
+	if (dm->dc->caps.ips_support && dm->dc->ctx->dmub_srv->idle_allowed) {
+		dc_allow_idle_optimizations(dm->dc, false);
+		reallow_idle = true;
+	}
 
-		rc = dc_link_set_backlight_level(link, &backlight_level_params);
-		if (!rc)
-			DRM_DEBUG("DM: Failed to update backlight on eDP[%d]\n", bl_idx);
+	if (caps->aux_support) {
+		rc = mod_power_set_backlight_nits(dm->power_module, stream, brightness,
+			AUX_BL_DEFAULT_TRANSITION_TIME_MS, false, true);
+	} else {
+		/* power module uses millipercent */
+		get_brightness_range(caps, &min, &max);
+		brightness = DIV_ROUND_CLOSEST(brightness * 100, (max - min)) * 1000;
+		rc = mod_power_set_backlight_percent(dm->power_module, stream,
+						     brightness, 0, false);
 	}
 
+	/*
+	 * Some kms clients create a ramped backlight transition effect
+	 * by rapidly changing the backlight. Yet we must wait on dmcub
+	 * fw to exit psr/replay before programming backlight. To
+	 * prevent lag, keep disable psr/replay and let the next atomic
+	 * flip clear the event.
+	 *
+	 * ToDo: use ISM to handle rapidly backlight change
+	 *
+	 * Rapidly backlight change is similar to rapidly cursor events,
+	 * which is now handled by ISM. ISM can delay the event until system
+	 * is really idle, so we may use ISM to handle backlight change as well.
+	 */
+	amdgpu_dm_psr_set_event(dm, stream, true,
+		psr_event_hw_programming, true);
+	amdgpu_dm_replay_set_event(dm, stream, true,
+		replay_event_hw_programming, true);
+
 	if (dm->dc->caps.ips_support && reallow_idle)
 		dc_allow_idle_optimizations(dm->dc, true);
 
@@ -5503,6 +5629,8 @@ static void setup_backlight_device(struct amdgpu_display_manager *dm,
 
 static void amdgpu_set_panel_orientation(struct drm_connector *connector);
 
+
+
 /*
  * In this architecture, the association
  * connector -> encoder -> crtc
@@ -5744,7 +5872,7 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev)
 						psr_feature_enabled = false;
 
 				if (psr_feature_enabled) {
-					amdgpu_dm_set_psr_caps(link);
+					amdgpu_dm_set_psr_caps(link, aconnector);
 					drm_info(adev_to_drm(adev), "%s: PSR support %d, DC PSR ver %d, sink PSR ver %d DPCD caps 0x%x su_y_granularity %d\n",
 						 aconnector->base.name,
 						 link->psr_settings.psr_feature_enabled,
@@ -8370,7 +8498,7 @@ static int fill_hdr_info_packet(const struct drm_connector_state *state,
 
 static int
 amdgpu_dm_connector_atomic_check(struct drm_connector *conn,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *new_con_state =
 		drm_atomic_get_new_connector_state(state, conn);
@@ -8492,7 +8620,7 @@ static int dm_encoder_helper_atomic_check(struct drm_encoder *encoder,
 					  struct drm_crtc_state *crtc_state,
 					  struct drm_connector_state *conn_state)
 {
-	struct drm_atomic_state *state = crtc_state->state;
+	struct drm_atomic_commit *state = crtc_state->state;
 	struct drm_connector *connector = conn_state->connector;
 	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
 	struct dm_connector_state *dm_new_connector_state = to_dm_connector_state(conn_state);
@@ -8564,7 +8692,7 @@ const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs = {
 	.atomic_check = dm_encoder_helper_atomic_check
 };
 
-static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_state *state,
+static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_commit *state,
 					    struct dc_state *dc_state,
 					    struct dsc_mst_fairness_vars *vars)
 {
@@ -9808,7 +9936,8 @@ static void update_stream_irq_parameters(
 	spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags);
 }
 
-static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
+static void amdgpu_dm_handle_vrr_transition(struct amdgpu_display_manager *dm,
+					    struct dm_crtc_state *old_state,
 					    struct dm_crtc_state *new_state)
 {
 	bool old_vrr_active = amdgpu_dm_crtc_vrr_active(old_state);
@@ -9827,6 +9956,14 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
 		WARN_ON(drm_crtc_vblank_get(new_state->base.crtc) != 0);
 		drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR off->on: Get vblank ref\n",
 				 __func__, new_state->base.crtc->base.id);
+
+		scoped_guard(mutex, &dm->dc_lock) {
+			dc_exit_ips_for_hw_access(dm->dc);
+			amdgpu_dm_psr_set_event(dm, new_state->stream, true,
+				psr_event_vrr_transition, true);
+			amdgpu_dm_replay_set_event(dm, new_state->stream, true,
+				replay_event_vrr, true);
+		}
 	} else if (old_vrr_active && !new_vrr_active) {
 		/* Transition VRR active -> inactive:
 		 * Allow vblank irq disable again for fixed refresh rate.
@@ -9835,10 +9972,18 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state,
 		drm_crtc_vblank_put(new_state->base.crtc);
 		drm_dbg_driver(new_state->base.crtc->dev, "%s: crtc=%u VRR on->off: Drop vblank ref\n",
 				 __func__, new_state->base.crtc->base.id);
+
+		scoped_guard(mutex, &dm->dc_lock) {
+			dc_exit_ips_for_hw_access(dm->dc);
+			amdgpu_dm_psr_set_event(dm, new_state->stream, false,
+				psr_event_vrr_transition, false);
+			amdgpu_dm_replay_set_event(dm, new_state->stream, false,
+				replay_event_vrr, false);
+		}
 	}
 }
 
-static void amdgpu_dm_commit_cursors(struct drm_atomic_state *state)
+static void amdgpu_dm_commit_cursors(struct drm_atomic_commit *state)
 {
 	struct drm_plane *plane;
 	struct drm_plane_state *old_plane_state;
@@ -9932,7 +10077,8 @@ static void amdgpu_dm_update_cursor(struct drm_plane *plane,
 	}
 }
 
-static void amdgpu_dm_enable_self_refresh(struct amdgpu_crtc *acrtc_attach,
+static void amdgpu_dm_enable_self_refresh(struct amdgpu_display_manager *dm,
+					  struct amdgpu_crtc *acrtc_attach,
 					  const struct dm_crtc_state *acrtc_state,
 					  const u64 current_ts)
 {
@@ -9940,20 +10086,10 @@ static void amdgpu_dm_enable_self_refresh(struct amdgpu_crtc *acrtc_attach,
 	struct replay_settings *pr = &acrtc_state->stream->link->replay_settings;
 	struct amdgpu_dm_connector *aconn =
 		(struct amdgpu_dm_connector *)acrtc_state->stream->dm_stream_context;
-	bool vrr_active = amdgpu_dm_crtc_vrr_active(acrtc_state);
-
-	if (acrtc_state->update_type > UPDATE_TYPE_FAST) {
-		if (pr->config.replay_supported && !pr->replay_feature_enabled)
-			amdgpu_dm_link_setup_replay(acrtc_state->stream->link, aconn);
-		else if (psr->psr_version != DC_PSR_VERSION_UNSUPPORTED &&
-			     !psr->psr_feature_enabled)
-			if (!aconn->disallow_edp_enter_psr)
-				amdgpu_dm_link_setup_psr(acrtc_state->stream);
-	}
 
 	/* Decrement skip count when SR is enabled and we're doing fast updates. */
 	if (acrtc_state->update_type == UPDATE_TYPE_FAST &&
-	    (psr->psr_feature_enabled || pr->config.replay_supported)) {
+	    (psr->psr_feature_enabled || pr->replay_feature_enabled)) {
 		if (aconn->sr_skip_count > 0)
 			aconn->sr_skip_count--;
 
@@ -9968,24 +10104,20 @@ static void amdgpu_dm_enable_self_refresh(struct amdgpu_crtc *acrtc_attach,
 		 * of update events.
 		 * See `amdgpu_dm_crtc_vblank_control_worker()`.
 		 */
-		if (!vrr_active &&
-		    acrtc_attach->dm_irq_params.allow_sr_entry &&
-#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY
-		    !amdgpu_dm_crc_window_is_activated(acrtc_state->base.crtc) &&
-#endif
-		    (current_ts - psr->psr_dirty_rects_change_timestamp_ns) > 500000000) {
-			if (pr->replay_feature_enabled && !pr->replay_allow_active)
-				amdgpu_dm_replay_enable(acrtc_state->stream, true);
-			if (psr->psr_version == DC_PSR_VERSION_SU_1 &&
-			    !psr->psr_allow_active && !aconn->disallow_edp_enter_psr)
-				amdgpu_dm_psr_enable(acrtc_state->stream);
+		if (acrtc_attach->dm_irq_params.allow_sr_entry &&
+			(current_ts - psr->psr_dirty_rects_change_timestamp_ns) > 500000000) {
+			amdgpu_dm_psr_set_event(dm, acrtc_state->stream, false,
+				psr_event_hw_programming, false);
+
+			amdgpu_dm_replay_set_event(dm, acrtc_state->stream, false,
+				replay_event_hw_programming, false);
 		}
 	} else {
 		acrtc_attach->dm_irq_params.allow_sr_entry = false;
 	}
 }
 
-static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
+static void amdgpu_dm_commit_planes(struct drm_atomic_commit *state,
 				    struct drm_device *dev,
 				    struct amdgpu_display_manager *dm,
 				    struct drm_crtc *pcrtc,
@@ -10140,15 +10272,13 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 			 */
 			if (acrtc_state->stream->link->psr_settings.psr_version >= DC_PSR_VERSION_SU_1 &&
 			    acrtc_attach->dm_irq_params.allow_sr_entry &&
-#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY
-			    !amdgpu_dm_crc_window_is_activated(acrtc_state->base.crtc) &&
-#endif
 			    dirty_rects_changed) {
 				mutex_lock(&dm->dc_lock);
 				acrtc_state->stream->link->psr_settings.psr_dirty_rects_change_timestamp_ns =
 				timestamp_ns;
-				if (acrtc_state->stream->link->psr_settings.psr_allow_active)
-					amdgpu_dm_psr_disable(acrtc_state->stream, true);
+				dc_exit_ips_for_hw_access(dm->dc);
+				amdgpu_dm_psr_set_event(dm, acrtc_state->stream, true,
+					psr_event_hw_programming, true);
 				mutex_unlock(&dm->dc_lock);
 			}
 		}
@@ -10313,15 +10443,6 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 		if (acrtc_state->abm_level != dm_old_crtc_state->abm_level)
 			bundle->stream_update.abm_level = &acrtc_state->abm_level;
 
-		mutex_lock(&dm->dc_lock);
-		if ((acrtc_state->update_type > UPDATE_TYPE_FAST) || vrr_active) {
-			if (acrtc_state->stream->link->replay_settings.replay_allow_active)
-				amdgpu_dm_replay_disable(acrtc_state->stream);
-			if (acrtc_state->stream->link->psr_settings.psr_allow_active)
-				amdgpu_dm_psr_disable(acrtc_state->stream, true);
-		}
-		mutex_unlock(&dm->dc_lock);
-
 		/*
 		 * If FreeSync state on the stream has changed then we need to
 		 * re-adjust the min/max bounds now that DC doesn't handle this
@@ -10359,8 +10480,8 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 		if (dm_old_crtc_state->active_planes != acrtc_state->active_planes)
 			dm_update_pflip_irq_state(drm_to_adev(dev),
 						  acrtc_attach);
-
-		amdgpu_dm_enable_self_refresh(acrtc_attach, acrtc_state, timestamp_ns);
+		amdgpu_dm_enable_self_refresh(dm, acrtc_attach, acrtc_state,
+					      timestamp_ns);
 		mutex_unlock(&dm->dc_lock);
 	}
 
@@ -10379,7 +10500,7 @@ static void amdgpu_dm_commit_planes(struct drm_atomic_state *state,
 }
 
 static void amdgpu_dm_commit_audio(struct drm_device *dev,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
 	struct amdgpu_dm_connector *aconnector;
@@ -10479,7 +10600,118 @@ static void dm_clear_writeback(struct amdgpu_display_manager *dm,
 	dc_stream_remove_writeback(dm->dc, crtc_state->stream, 0);
 }
 
-static void amdgpu_dm_commit_streams(struct drm_atomic_state *state,
+/**
+ * amdgpu_dm_mod_power_update_streams - update mod_power stream state on modeset
+ * @state: the drm atomic state
+ * @dm: the display manager to update mod_power on
+ *
+ * Notify mod_power of stream changes on modeset events, and disable PSR/Replay
+ * in preparation for hardware programming. See also
+ * amdgpu_dm_mod_power_setup_streams() for post-modeset mod_power setup.
+ */
+static void amdgpu_dm_mod_power_update_streams(struct drm_atomic_commit *state,
+					       struct amdgpu_display_manager *dm)
+{
+	struct dm_crtc_state *dm_old_crtc_state, *dm_new_crtc_state;
+	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
+	struct amdgpu_dm_connector *aconnector;
+	struct drm_crtc *crtc;
+	int i = 0;
+
+	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
+		dm_old_crtc_state = to_dm_crtc_state(old_crtc_state);
+		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
+
+		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
+			continue;
+
+		/*
+		 * Update mod_power on modeset event in preparation for hw
+		 * programming. Always use the old stream, since it would have
+		 * been previously added to mod_power. If old stream is null (on
+		 * crtc enable, for example), mod_power will no-op, which is the
+		 * desried behavior.
+		 */
+		if (old_crtc_state->active) {
+			scoped_guard(mutex, &dm->dc_lock) {
+				dc_exit_ips_for_hw_access(dm->dc);
+				amdgpu_dm_psr_set_event(dm, dm_old_crtc_state->stream, true,
+					psr_event_hw_programming, true);
+				amdgpu_dm_replay_set_event(dm, dm_old_crtc_state->stream, true,
+					replay_event_hw_programming, true);
+				amdgpu_dm_replay_set_event(dm, dm_old_crtc_state->stream, false,
+					replay_event_general_ui, false);
+			}
+		}
+
+		if (new_crtc_state->active) {
+			aconnector = (struct amdgpu_dm_connector *)
+				dm_new_crtc_state->stream->dm_stream_context;
+			if (old_crtc_state->active) {
+				mod_power_replace_stream(dm->power_module,
+					dm_old_crtc_state->stream,
+					dm_new_crtc_state->stream,
+					&aconnector->psr_caps);
+			} else {
+				mod_power_add_stream(dm->power_module,
+					dm_new_crtc_state->stream,
+					&aconnector->psr_caps);
+			}
+		} else if (old_crtc_state->active) {
+			mod_power_remove_stream(dm->power_module,
+				dm_old_crtc_state->stream);
+		}
+	}
+}
+
+/**
+ * amdgpu_dm_mod_power_setup_streams - setup mod_power stream state post modeset
+ * @state: the drm atomic state
+ * @dm: the display manager to update mod_power on
+ *
+ * Notify mod_power of mode_change. This needs to be done after dc_stream
+ * updates have been committed, and VRR parameters have been updated.
+ */
+static void amdgpu_dm_mod_power_setup_streams(struct drm_atomic_commit *state,
+					      struct amdgpu_display_manager *dm)
+{
+	struct dm_crtc_state *dm_new_crtc_state;
+	struct drm_crtc_state *new_crtc_state;
+	struct amdgpu_crtc *acrtc;
+	struct drm_crtc *crtc;
+	int i = 0;
+
+	for_each_new_crtc_in_state(state, crtc, new_crtc_state, i) {
+		dm_new_crtc_state = to_dm_crtc_state(new_crtc_state);
+		acrtc = to_amdgpu_crtc(crtc);
+
+		if (!drm_atomic_crtc_needs_modeset(new_crtc_state))
+			continue;
+
+		if (new_crtc_state->active) {
+			amdgpu_dm_link_setup_replay(dm_new_crtc_state->stream,
+					&acrtc->dm_irq_params.vrr_params);
+			mod_power_notify_mode_change(dm->power_module,
+						dm_new_crtc_state->stream,
+						false);
+
+			/*
+			 * Block PSR / Replay on the new stream until display settles post-modeset.
+			 * These events will be cleared by amdgpu_dm_enable_self_refresh() once
+			 * allow_sr_entry becomes true.
+			 */
+			amdgpu_dm_psr_set_event(dm, dm_new_crtc_state->stream, true,
+				psr_event_hw_programming, true);
+
+			amdgpu_dm_replay_set_event(dm, dm_new_crtc_state->stream, true,
+				replay_event_hw_programming | replay_event_general_ui,
+				true);
+		}
+	}
+
+}
+
+static void amdgpu_dm_commit_streams(struct drm_atomic_commit *state,
 					struct dc_state *dc_state)
 {
 	struct drm_device *dev = state->dev;
@@ -10522,6 +10754,8 @@ static void amdgpu_dm_commit_streams(struct drm_atomic_state *state,
 		acrtc->wb_enabled = false;
 	}
 
+	amdgpu_dm_mod_power_update_streams(state, dm);
+
 	for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state,
 				      new_crtc_state, i) {
 		struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
@@ -10626,13 +10860,10 @@ static void amdgpu_dm_commit_streams(struct drm_atomic_state *state,
 		}
 	} /* for_each_crtc_in_state() */
 
-	/* if there mode set or reset, disable eDP PSR, Replay */
+	/* if there mode set or reset, flush vblank work queue */
 	if (mode_set_reset_required) {
 		if (dm->vblank_control_workqueue)
 			flush_workqueue(dm->vblank_control_workqueue);
-
-		amdgpu_dm_replay_disable_all(dm);
-		amdgpu_dm_psr_disable_all(dm);
 	}
 
 	dm_enable_per_frame_crtc_master_sync(dc_state);
@@ -10779,7 +11010,7 @@ static void dm_set_writeback(struct amdgpu_display_manager *dm,
 	drm_writeback_queue_job(wb_conn, new_con_state);
 }
 
-static void amdgpu_dm_update_hdcp(struct drm_atomic_state *state)
+static void amdgpu_dm_update_hdcp(struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *old_con_state, *new_con_state;
 	struct drm_device *dev = state->dev;
@@ -10894,7 +11125,7 @@ static void amdgpu_dm_update_hdcp(struct drm_atomic_state *state)
 	}
 }
 
-static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_state *state)
+static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
@@ -10935,7 +11166,7 @@ static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_state *state)
  * programming the hardware. Any failures here implies a hardware failure, since
  * atomic check should have filtered anything non-kosher.
  */
-static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
+static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct amdgpu_device *adev = drm_to_adev(dev);
@@ -11105,7 +11336,7 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 			manage_dm_interrupts(adev, acrtc, dm_new_crtc_state);
 		}
 		/* Handle vrr on->off / off->on transitions */
-		amdgpu_dm_handle_vrr_transition(dm_old_crtc_state, dm_new_crtc_state);
+		amdgpu_dm_handle_vrr_transition(dm, dm_old_crtc_state, dm_new_crtc_state);
 
 #ifdef CONFIG_DEBUG_FS
 		if (new_crtc_state->active &&
@@ -11143,6 +11374,8 @@ static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_state *state)
 #endif
 	}
 
+	amdgpu_dm_mod_power_setup_streams(state, dm);
+
 	for_each_new_crtc_in_state(state, crtc, new_crtc_state, j)
 		if (new_crtc_state->async_flip)
 			wait_for_vblank = false;
@@ -11240,7 +11473,7 @@ static int dm_force_atomic_commit(struct drm_connector *connector)
 {
 	int ret = 0;
 	struct drm_device *ddev = connector->dev;
-	struct drm_atomic_state *state = drm_atomic_state_alloc(ddev);
+	struct drm_atomic_commit *state = drm_atomic_commit_alloc(ddev);
 	struct amdgpu_crtc *disconnected_acrtc = to_amdgpu_crtc(connector->encoder->crtc);
 	struct drm_plane *plane = disconnected_acrtc->base.primary;
 	struct drm_connector_state *conn_state;
@@ -11255,7 +11488,7 @@ static int dm_force_atomic_commit(struct drm_connector *connector)
 	/* Construct an atomic state to restore previous display setting */
 
 	/*
-	 * Attach connectors to drm_atomic_state
+	 * Attach connectors to drm_atomic_commit
 	 */
 	conn_state = drm_atomic_get_connector_state(state, connector);
 
@@ -11265,7 +11498,7 @@ static int dm_force_atomic_commit(struct drm_connector *connector)
 		goto out;
 	}
 
-	/* Attach crtc to drm_atomic_state*/
+	/* Attach crtc to drm_atomic_commit*/
 	crtc_state = drm_atomic_get_crtc_state(state, &disconnected_acrtc->base);
 
 	/* Check for error in getting crtc state */
@@ -11277,7 +11510,7 @@ static int dm_force_atomic_commit(struct drm_connector *connector)
 	/* force a restore */
 	crtc_state->mode_changed = true;
 
-	/* Attach plane to drm_atomic_state */
+	/* Attach plane to drm_atomic_commit */
 	plane_state = drm_atomic_get_plane_state(state, plane);
 
 	/* Check for error in getting plane state */
@@ -11290,7 +11523,7 @@ static int dm_force_atomic_commit(struct drm_connector *connector)
 	ret = drm_atomic_commit(state);
 
 out:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 	if (ret)
 		drm_err(ddev, "Restoring old state failed with %i\n", ret);
 
@@ -11339,7 +11572,7 @@ void dm_restore_drm_connector_state(struct drm_device *dev,
  * Waits for completion of all non blocking commits.
  */
 static int do_aquire_global_lock(struct drm_device *dev,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_commit *commit;
@@ -11484,7 +11717,7 @@ static void set_freesync_fixed_config(struct dm_crtc_state *dm_new_crtc_state)
 }
 
 static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
-			 struct drm_atomic_state *state,
+			 struct drm_atomic_commit *state,
 			 struct drm_crtc *crtc,
 			 struct drm_crtc_state *old_crtc_state,
 			 struct drm_crtc_state *new_crtc_state,
@@ -11758,7 +11991,7 @@ static int dm_update_crtc_state(struct amdgpu_display_manager *dm,
 	return ret;
 }
 
-static bool should_reset_plane(struct drm_atomic_state *state,
+static bool should_reset_plane(struct drm_atomic_commit *state,
 			       struct drm_plane *plane,
 			       struct drm_plane_state *old_plane_state,
 			       struct drm_plane_state *new_plane_state)
@@ -12016,7 +12249,7 @@ static int dm_check_native_cursor_state(struct drm_crtc *new_plane_crtc,
 	return 0;
 }
 
-static bool dm_should_update_native_cursor(struct drm_atomic_state *state,
+static bool dm_should_update_native_cursor(struct drm_atomic_commit *state,
 					   struct drm_crtc *old_plane_crtc,
 					   struct drm_crtc *new_plane_crtc,
 					   bool enable)
@@ -12046,7 +12279,7 @@ static bool dm_should_update_native_cursor(struct drm_atomic_state *state,
 }
 
 static int dm_update_plane_state(struct dc *dc,
-				 struct drm_atomic_state *state,
+				 struct drm_atomic_commit *state,
 				 struct drm_plane *plane,
 				 struct drm_plane_state *old_plane_state,
 				 struct drm_plane_state *new_plane_state,
@@ -12254,7 +12487,7 @@ dm_get_plane_scale(struct drm_plane_state *plane_state,
  * indefinitely.
  */
 static inline struct __drm_planes_state *__get_next_zpos(
-	struct drm_atomic_state *state,
+	struct drm_atomic_commit *state,
 	struct __drm_planes_state *prev)
 {
 	unsigned int highest_zpos = 0, prev_zpos = 256;
@@ -12304,7 +12537,7 @@ static inline struct __drm_planes_state *__get_next_zpos(
 			     (old_plane_state) = __i->old_state,		\
 			     (new_plane_state) = __i->new_state, 1))
 
-static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm_crtc *crtc)
+static int add_affected_mst_dsc_crtcs(struct drm_atomic_commit *state, struct drm_crtc *crtc)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *conn_state, *old_conn_state;
@@ -12363,7 +12596,7 @@ static int add_affected_mst_dsc_crtcs(struct drm_atomic_state *state, struct drm
  *
  * Return: true if the pipeline modifies pixels, false otherwise.
  */
-static bool dm_plane_color_pipeline_active(struct drm_atomic_state *state,
+static bool dm_plane_color_pipeline_active(struct drm_atomic_commit *state,
 					   struct drm_plane *plane,
 					   bool use_old)
 {
@@ -12403,7 +12636,7 @@ static bool dm_plane_color_pipeline_active(struct drm_atomic_state *state,
  * failed.
  */
 static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
-				   struct drm_atomic_state *state,
+				   struct drm_atomic_commit *state,
 				   struct dm_crtc_state *dm_crtc_state,
 				   enum amdgpu_dm_cursor_mode *cursor_mode)
 {
@@ -12542,7 +12775,7 @@ static int dm_crtc_get_cursor_mode(struct amdgpu_device *adev,
 }
 
 static bool amdgpu_dm_crtc_mem_type_changed(struct drm_device *dev,
-					    struct drm_atomic_state *state,
+					    struct drm_atomic_commit *state,
 					    struct drm_crtc_state *crtc_state)
 {
 	struct drm_plane *plane;
@@ -12591,7 +12824,7 @@ static bool amdgpu_dm_crtc_mem_type_changed(struct drm_device *dev,
  * Return: -Error code if validation failed.
  */
 static int amdgpu_dm_atomic_check(struct drm_device *dev,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
 	struct dm_atomic_state *dm_state = NULL;
@@ -13701,11 +13934,17 @@ int amdgpu_dm_process_dmub_set_config_sync(
 
 bool dm_execute_dmub_cmd(const struct dc_context *ctx, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type)
 {
+	struct amdgpu_device *adev = ctx->driver_context;
+
+	guard(spinlock_irqsave)(&adev->dm.dmub_lock);
 	return dc_dmub_srv_cmd_run(ctx->dmub_srv, cmd, wait_type);
 }
 
 bool dm_execute_dmub_cmd_list(const struct dc_context *ctx, unsigned int count, union dmub_rb_cmd *cmd, enum dm_dmub_wait_type wait_type)
 {
+	struct amdgpu_device *adev = ctx->driver_context;
+
+	guard(spinlock_irqsave)(&adev->dm.dmub_lock);
 	return dc_dmub_srv_cmd_run_list(ctx->dmub_srv, count, cmd, wait_type);
 }
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
index 74a8fe1a1999f..74f700fbeb6f1 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h
@@ -463,6 +463,13 @@ struct amdgpu_display_manager {
 	 */
 	struct mutex dc_lock;
 
+	/**
+	 * @dmub_lock:
+	 *
+	 * Guards access to DMUB command submission.
+	 */
+	spinlock_t dmub_lock;
+
 	/**
 	 * @audio_lock:
 	 *
@@ -568,6 +575,7 @@ struct amdgpu_display_manager {
 	struct amdgpu_dm_backlight_caps backlight_caps[AMDGPU_DM_MAX_NUM_EDP];
 
 	struct mod_freesync *freesync_module;
+	struct mod_power *power_module;
 	struct hdcp_workqueue *hdcp_workqueue;
 
 	/**
@@ -584,7 +592,7 @@ struct amdgpu_display_manager {
 	 */
 	struct idle_workqueue *idle_workqueue;
 
-	struct drm_atomic_state *cached_state;
+	struct drm_atomic_commit *cached_state;
 	struct dc_state *cached_dc_state;
 
 	struct dm_compressor_info compressor;
@@ -835,6 +843,7 @@ struct amdgpu_dm_connector {
 	bool force_yuv420_output;
 	bool force_yuv422_output;
 	struct dsc_preferred_settings dsc_settings;
+	struct psr_caps psr_caps;
 	union dp_downstream_port_present mst_downstream_port_present;
 	/* Cached display modes */
 	struct drm_display_mode freesync_vid_base;
@@ -1123,11 +1132,11 @@ struct dc_stream_state *
 					const struct dm_connector_state *dm_state,
 					const struct dc_stream_state *old_stream);
 
-int dm_atomic_get_state(struct drm_atomic_state *state,
+int dm_atomic_get_state(struct drm_atomic_commit *state,
 			struct dm_atomic_state **dm_state);
 
 struct drm_connector *
-amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_state *state,
+amdgpu_dm_find_first_crtc_matching_connector(struct drm_atomic_commit *state,
 					     struct drm_crtc *crtc);
 
 int convert_dc_color_depth_into_bpc(enum dc_color_depth display_color_depth);
@@ -1149,4 +1158,5 @@ int amdgpu_dm_initialize_hdmi_connector(struct amdgpu_dm_connector *aconnector);
 
 void retrieve_dmi_info(struct amdgpu_display_manager *dm);
 
+void amdgpu_dm_update_backlight_caps(struct amdgpu_display_manager *dm, int bl_idx);
 #endif /* __AMDGPU_DM_H__ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
index cd1e58b8defcb..fa6883ae4dfb8 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c
@@ -1499,7 +1499,7 @@ __set_dm_plane_colorop_degamma(struct drm_plane_state *plane_state,
 {
 	struct drm_colorop *old_colorop;
 	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
-	struct drm_atomic_state *state = plane_state->state;
+	struct drm_atomic_commit *state = plane_state->state;
 	int i = 0;
 
 	old_colorop = colorop;
@@ -1526,7 +1526,7 @@ __set_dm_plane_colorop_3x4_matrix(struct drm_plane_state *plane_state,
 {
 	struct drm_colorop *old_colorop;
 	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
-	struct drm_atomic_state *state = plane_state->state;
+	struct drm_atomic_commit *state = plane_state->state;
 	const struct drm_device *dev = colorop->dev;
 	const struct drm_property_blob *blob;
 	struct drm_color_ctm_3x4 *ctm = NULL;
@@ -1567,7 +1567,7 @@ __set_dm_plane_colorop_multiplier(struct drm_plane_state *plane_state,
 {
 	struct drm_colorop *old_colorop;
 	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
-	struct drm_atomic_state *state = plane_state->state;
+	struct drm_atomic_commit *state = plane_state->state;
 	const struct drm_device *dev = colorop->dev;
 	int i = 0;
 
@@ -1596,7 +1596,7 @@ __set_dm_plane_colorop_shaper(struct drm_plane_state *plane_state,
 {
 	struct drm_colorop *old_colorop;
 	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
-	struct drm_atomic_state *state = plane_state->state;
+	struct drm_atomic_commit *state = plane_state->state;
 	enum dc_transfer_func_predefined default_tf = TRANSFER_FUNCTION_LINEAR;
 	struct dc_transfer_func *tf = &dc_plane_state->in_shaper_func;
 	const struct drm_color_lut32 *shaper_lut;
@@ -1704,7 +1704,7 @@ __set_dm_plane_colorop_3dlut(struct drm_plane_state *plane_state,
 	struct drm_colorop *old_colorop;
 	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
 	struct dc_transfer_func *tf = &dc_plane_state->in_shaper_func;
-	struct drm_atomic_state *state = plane_state->state;
+	struct drm_atomic_commit *state = plane_state->state;
 	const struct amdgpu_device *adev = drm_to_adev(colorop->dev);
 	bool has_3dlut = adev->dm.dc->caps.color.dpp.hw_3d_lut || adev->dm.dc->caps.color.mpc.preblend;
 	const struct drm_device *dev = colorop->dev;
@@ -1759,7 +1759,7 @@ __set_dm_plane_colorop_blend(struct drm_plane_state *plane_state,
 {
 	struct drm_colorop *old_colorop;
 	struct drm_colorop_state *colorop_state = NULL, *new_colorop_state;
-	struct drm_atomic_state *state = plane_state->state;
+	struct drm_atomic_commit *state = plane_state->state;
 	enum dc_transfer_func_predefined default_tf = TRANSFER_FUNCTION_LINEAR;
 	struct dc_transfer_func *tf = &dc_plane_state->blend_tf;
 	const struct drm_color_lut32 *blend_lut = NULL;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
index 62573173e2ac2..5adbb0f6a0c88 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c
@@ -503,7 +503,6 @@ int amdgpu_dm_crtc_configure_crc_source(struct drm_crtc *crtc,
 {
 	struct amdgpu_device *adev = drm_to_adev(crtc->dev);
 	struct dc_stream_state *stream_state = dm_crtc_state->stream;
-	struct amdgpu_dm_connector *aconnector = NULL;
 	bool enable = amdgpu_dm_is_valid_crc_source(source);
 	int ret = 0;
 	enum crc_poly_mode crc_poly_mode = CRC_POLY_MODE_16;
@@ -512,21 +511,17 @@ int amdgpu_dm_crtc_configure_crc_source(struct drm_crtc *crtc,
 	if (!stream_state)
 		return -EINVAL;
 
-	/* Get connector from stream */
-	aconnector = (struct amdgpu_dm_connector *)stream_state->dm_stream_context;
-
 	mutex_lock(&adev->dm.dc_lock);
 
-
+	/* Notify power module about CRC window active to disable PSR/Replay
+	 * Power module will check caps internally and skip if not supported
+	 */
 	if (enable) {
-		/* For PSR1, check that the panel has exited PSR */
-		if (stream_state->link->psr_settings.psr_version < DC_PSR_VERSION_SU_1)
-			amdgpu_dm_psr_wait_disable(stream_state);
+		amdgpu_dm_psr_set_event(&adev->dm, stream_state, true,
+			psr_event_crc_window_active, true);
 
-		/* Set flag to disallow enter replay when CRC source is enabled */
-		if (aconnector)
-			aconnector->disallow_edp_enter_replay = true;
-		amdgpu_dm_replay_disable(stream_state);
+		amdgpu_dm_replay_set_event(&adev->dm, stream_state, true,
+			replay_event_crc_window_active, true);
 	}
 
 	/* CRC polynomial selection only support for DCN3.6+ except DCN4.0.1 */
@@ -559,11 +554,15 @@ int amdgpu_dm_crtc_configure_crc_source(struct drm_crtc *crtc,
 	}
 
 	if (!enable) {
-		/* Clear flag to allow enter replay when CRC source is disabled */
-		if (aconnector)
-			aconnector->disallow_edp_enter_replay = false;
-	}
+		/* Notify power module about CRC window inactive to re-enable PSR/Replay
+		 * Power module will check caps internally and skip if not supported
+		 */
+		amdgpu_dm_psr_set_event(&adev->dm, stream_state, false,
+			psr_event_crc_window_active, false);
 
+		amdgpu_dm_replay_set_event(&adev->dm, stream_state, false,
+			replay_event_crc_window_active, false);
+	}
 unlock:
 	mutex_unlock(&adev->dm.dc_lock);
 
@@ -614,8 +613,13 @@ int amdgpu_dm_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name)
 		 */
 		ret = wait_for_completion_interruptible_timeout(
 			&commit->hw_done, 10 * HZ);
-		if (ret)
+		if (ret < 0)
 			goto cleanup;
+
+		if (ret == 0) {
+			ret = -ETIMEDOUT;
+			goto cleanup;
+		}
 	}
 
 	enable = amdgpu_dm_is_valid_crc_source(source);
@@ -760,10 +764,13 @@ void amdgpu_dm_crtc_handle_crc_irq(struct drm_crtc *crtc)
 	uint32_t crcs[3];
 	unsigned long flags;
 
-	if (crtc == NULL)
+	if (!crtc || !crtc->state || !crtc->dev)
 		return;
 
 	crtc_state = to_dm_crtc_state(crtc->state);
+	if (!crtc_state->stream)
+		return;
+
 	stream_state = crtc_state->stream;
 	acrtc = to_amdgpu_crtc(crtc);
 	drm_dev = crtc->dev;
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
index d69f5a75b685a..3dcedaa67ed8e 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c
@@ -34,6 +34,7 @@
 #include "amdgpu_dm_plane.h"
 #include "amdgpu_dm_trace.h"
 #include "amdgpu_dm_debugfs.h"
+#include "modules/inc/mod_power.h"
 
 #define HPD_DETECTION_PERIOD_uS 2000000
 #define HPD_DETECTION_TIME_uS 100000
@@ -100,68 +101,33 @@ bool amdgpu_dm_crtc_vrr_active(const struct dm_crtc_state *dm_state)
 }
 
 /**
- * amdgpu_dm_crtc_set_panel_sr_feature() - Manage panel self-refresh features.
- * @dm: amdgpu display manager instance.
- * @acrtc: CRTC whose panel self-refresh state is being updated.
- * @stream: DC stream associated with @acrtc.
- * @vblank_enabled: Whether the DRM vblank counter is currently enabled.
- * @allow_sr_entry: Whether entry into self-refresh mode is allowed.
+ * amdgpu_dm_crtc_set_static_screen_optimze() - Toggle static screen optimizations.
  *
- * The DRM vblank counter enable/disable action is used as the trigger to enable
- * or disable various panel self-refresh features:
+ * @dm: display manager
+ * @stream: DC stream state
+ * @sso_enable: desired static screen optimization state
+ * @allow_sr_entry: whether entry into self-refresh mode is allowed
  *
- * Panel Replay and PSR SU
- * - Enable when:
- *   - VRR is disabled
- *   - vblank counter is disabled
- *   - entry is allowed: usermode demonstrates an adequate number of fast
- *     commits
- *   - CRC capture window isn't active
- * - Keep enabled even when vblank counter gets enabled
- *
- * PSR1
- * - Enable condition same as above
- * - Disable when vblank counter is enabled
+ * This function uses the static-screen optimization state as the trigger to
+ * set/clear the Replay and PSR vsync-related events.
  */
-void amdgpu_dm_crtc_set_panel_sr_feature(
+void amdgpu_dm_crtc_set_static_screen_optimze(
 	struct amdgpu_display_manager *dm,
-	struct amdgpu_crtc *acrtc,
 	struct dc_stream_state *stream,
-	bool vblank_enabled, bool allow_sr_entry)
+	bool sso_enable, bool allow_sr_entry)
 {
 	struct dc_link *link = stream->link;
-	bool is_sr_active = (link->replay_settings.replay_allow_active ||
-				 link->psr_settings.psr_allow_active);
-	bool is_crc_window_active = false;
-	bool vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc);
-
-#ifdef CONFIG_DRM_AMD_SECURE_DISPLAY
-	is_crc_window_active =
-		amdgpu_dm_crc_window_is_activated(&acrtc->base);
-#endif
+	bool set_vsync_event = !sso_enable;
 
-	if (link->replay_settings.replay_feature_enabled && !vrr_active &&
-		allow_sr_entry && !is_sr_active && !is_crc_window_active) {
-		amdgpu_dm_replay_enable(stream, true);
-	} else if (vblank_enabled) {
-		if (link->psr_settings.psr_version < DC_PSR_VERSION_SU_1 && is_sr_active)
-			amdgpu_dm_psr_disable(stream, false);
-	} else if (link->psr_settings.psr_feature_enabled && !vrr_active &&
-		allow_sr_entry && !is_sr_active && !is_crc_window_active) {
-
-		struct amdgpu_dm_connector *aconn =
-			(struct amdgpu_dm_connector *) stream->dm_stream_context;
-
-		if (!aconn->disallow_edp_enter_psr) {
-			amdgpu_dm_psr_enable(stream);
-			if (dm->idle_workqueue &&
-			    (dm->dc->config.disable_ips == DMUB_IPS_ENABLE) &&
-			    dm->dc->idle_optimizations_allowed &&
-			    dm->idle_workqueue->enable &&
-			    !dm->idle_workqueue->running)
-				schedule_work(&dm->idle_workqueue->work);
-		}
-	}
+	if (!allow_sr_entry)
+		return;
+
+	amdgpu_dm_replay_set_event(dm, stream,
+		set_vsync_event, replay_event_vsync, set_vsync_event);
+
+	if (link->psr_settings.psr_version < DC_PSR_VERSION_SU_1)
+		amdgpu_dm_psr_set_event(dm, stream,
+			set_vsync_event, psr_event_vsync, set_vsync_event);
 }
 
 bool amdgpu_dm_is_headless(struct amdgpu_device *adev)
@@ -584,7 +550,7 @@ static void amdgpu_dm_crtc_helper_disable(struct drm_crtc *crtc)
 
 static int amdgpu_dm_crtc_count_crtc_active_planes(struct drm_crtc_state *new_crtc_state)
 {
-	struct drm_atomic_state *state = new_crtc_state->state;
+	struct drm_atomic_commit *state = new_crtc_state->state;
 	struct drm_plane *plane;
 	int num_active = 0;
 
@@ -637,7 +603,7 @@ static bool amdgpu_dm_crtc_helper_mode_fixup(struct drm_crtc *crtc,
 }
 
 static int amdgpu_dm_crtc_helper_atomic_check(struct drm_crtc *crtc,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 										crtc);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
index 3a8094013a5d0..e9fb52f0e66d9 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.h
@@ -27,11 +27,10 @@
 #ifndef __AMDGPU_DM_CRTC_H__
 #define __AMDGPU_DM_CRTC_H__
 
-void amdgpu_dm_crtc_set_panel_sr_feature(
+void amdgpu_dm_crtc_set_static_screen_optimze(
 	struct amdgpu_display_manager *dm,
-	struct amdgpu_crtc *acrtc,
 	struct dc_stream_state *stream,
-	bool vblank_enabled, bool allow_sr_entry);
+	bool sso_enable, bool allow_sr_entry);
 
 void amdgpu_dm_crtc_handle_vblank(struct amdgpu_crtc *acrtc);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
index 2409ac72b1667..4b09a740f2054 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c
@@ -33,6 +33,7 @@
 #include "amdgpu_dm.h"
 #include "amdgpu_dm_debugfs.h"
 #include "amdgpu_dm_replay.h"
+#include "amdgpu_dm_psr.h"
 #include "dm_helpers.h"
 #include "dmub/dmub_srv.h"
 #include "resource.h"
@@ -3162,10 +3163,25 @@ static int replay_get_state(void *data, u64 *val)
 {
 	struct amdgpu_dm_connector *connector = data;
 	struct dc_link *link = connector->dc_link;
+	struct amdgpu_device *adev = drm_to_adev(connector->base.dev);
+	struct dc *dc = adev->dm.dc;
 	uint64_t state = REPLAY_STATE_INVALID;
+	bool reallow_idle = false;
+
+	mutex_lock(&adev->dm.dc_lock);
+
+	if (dc->idle_optimizations_allowed) {
+		dc_allow_idle_optimizations(dc, false);
+		reallow_idle = true;
+	}
 
 	dc_link_get_replay_state(link, &state);
 
+	if (reallow_idle)
+		dc_allow_idle_optimizations(dc, true);
+
+	mutex_unlock(&adev->dm.dc_lock);
+
 	*val = state;
 
 	return 0;
@@ -3178,10 +3194,26 @@ static int replay_set_residency(void *data, u64 val)
 {
 	struct amdgpu_dm_connector *connector = data;
 	struct dc_link *link = connector->dc_link;
+	struct amdgpu_device *adev = drm_to_adev(connector->base.dev);
+	struct dc *dc = adev->dm.dc;
 	bool is_start = (val != 0);
 	u32 residency = 0;
+	bool reallow_idle = false;
+
+	mutex_lock(&adev->dm.dc_lock);
+
+	if (dc->idle_optimizations_allowed) {
+		dc_allow_idle_optimizations(dc, false);
+		reallow_idle = true;
+	}
 
 	link->dc->link_srv->edp_replay_residency(link, &residency, is_start, PR_RESIDENCY_MODE_PHY);
+
+	if (reallow_idle)
+		dc_allow_idle_optimizations(dc, true);
+
+	mutex_unlock(&adev->dm.dc_lock);
+
 	return 0;
 }
 
@@ -3192,9 +3224,25 @@ static int replay_get_residency(void *data, u64 *val)
 {
 	struct amdgpu_dm_connector *connector = data;
 	struct dc_link *link = connector->dc_link;
+	struct amdgpu_device *adev = drm_to_adev(connector->base.dev);
+	struct dc *dc = adev->dm.dc;
 	u32 residency = 0;
+	bool reallow_idle = false;
+
+	mutex_lock(&adev->dm.dc_lock);
+
+	if (dc->idle_optimizations_allowed) {
+		dc_allow_idle_optimizations(dc, false);
+		reallow_idle = true;
+	}
 
 	link->dc->link_srv->edp_replay_residency(link, &residency, false, PR_RESIDENCY_MODE_PHY);
+
+	if (reallow_idle)
+		dc_allow_idle_optimizations(dc, true);
+
+	mutex_unlock(&adev->dm.dc_lock);
+
 	*val = (u64)residency;
 
 	return 0;
@@ -3207,10 +3255,25 @@ static int psr_get(void *data, u64 *val)
 {
 	struct amdgpu_dm_connector *connector = data;
 	struct dc_link *link = connector->dc_link;
+	struct amdgpu_device *adev = drm_to_adev(connector->base.dev);
+	struct dc *dc = adev->dm.dc;
 	enum dc_psr_state state = PSR_STATE0;
+	bool reallow_idle = false;
+
+	mutex_lock(&adev->dm.dc_lock);
+
+	if (dc->idle_optimizations_allowed) {
+		dc_allow_idle_optimizations(dc, false);
+		reallow_idle = true;
+	}
 
 	dc_link_get_psr_state(link, &state);
 
+	if (reallow_idle)
+		dc_allow_idle_optimizations(dc, true);
+
+	mutex_unlock(&adev->dm.dc_lock);
+
 	*val = state;
 
 	return 0;
@@ -3223,10 +3286,25 @@ static int psr_read_residency(void *data, u64 *val)
 {
 	struct amdgpu_dm_connector *connector = data;
 	struct dc_link *link = connector->dc_link;
+	struct amdgpu_device *adev = drm_to_adev(connector->base.dev);
+	struct dc *dc = adev->dm.dc;
 	u32 residency = 0;
+	bool reallow_idle = false;
+
+	mutex_lock(&adev->dm.dc_lock);
+
+	if (dc->idle_optimizations_allowed) {
+		dc_allow_idle_optimizations(dc, false);
+		reallow_idle = true;
+	}
 
 	link->dc->link_srv->edp_get_psr_residency(link, &residency, PSR_RESIDENCY_MODE_PHY);
 
+	if (reallow_idle)
+		dc_allow_idle_optimizations(dc, true);
+
+	mutex_unlock(&adev->dm.dc_lock);
+
 	*val = (u64)residency;
 
 	return 0;
@@ -3300,11 +3378,26 @@ static int disallow_edp_enter_psr_get(void *data, u64 *val)
 static int disallow_edp_enter_psr_set(void *data, u64 val)
 {
 	struct amdgpu_dm_connector *aconnector = data;
+	struct dc_link *link = aconnector->dc_link;
 
-	aconnector->disallow_edp_enter_psr = val ? true : false;
+	aconnector->disallow_edp_enter_psr = (val != 0);
+
+	/* eDP PSR enable / disable is happened during mode change in power module.
+	 * Only psr_settings.psr_version is used to decide whether PSR is enabled or not.
+	 * So here we only update psr_version based on debugfs setting.
+	 * If disallow_edp_enter_psr is true, set psr_version to unsupported;
+	 * if disallow_edp_enter_psr is false, set psr_version based on sink capability.
+	 */
+	if (aconnector->disallow_edp_enter_psr)
+		link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
+	else if (aconnector->psr_caps.psr_version == 1)
+		link->psr_settings.psr_version = DC_PSR_VERSION_1;
+	else if (aconnector->psr_caps.psr_version == 2)
+		link->psr_settings.psr_version = DC_PSR_VERSION_SU_1;
 	return 0;
 }
 
+
 /* check if kernel disallow eDP enter replay state
  * cat /sys/kernel/debug/dri/0/eDP-X/disallow_edp_enter_replay
  * 0: allow edp enter replay; 1: disallow
@@ -3346,11 +3439,27 @@ static int disallow_edp_enter_replay_get(void *data, u64 *val)
 static int disallow_edp_enter_replay_set(void *data, u64 val)
 {
 	struct amdgpu_dm_connector *aconnector = data;
+	struct dc_link *link = aconnector->dc_link;
 
-	aconnector->disallow_edp_enter_replay = val ? true : false;
+	aconnector->disallow_edp_enter_replay = (val != 0);
+
+	/* eDP replay enable / disable is happened during mode change in power module.
+	 * Only replay_settings.config.replay_supported is used to decide whether
+	 * replay is enabled or not. So here we only update replay_supported based on
+	 * debugfs setting.
+	 * If disallow_edp_enter_replay is true, set replay_supported to false.
+	 * if disallow_edp_enter_replay is false, set replay_supported back based on
+	 * sink replay capability.
+	 */
+	if (aconnector->disallow_edp_enter_replay)
+		link->replay_settings.config.replay_supported = false;
+	else
+		link->replay_settings.config.replay_supported =
+			link->replay_settings.config.replay_cap_support;
 	return 0;
 }
 
+
 static int dmub_trace_mask_set(void *data, u64 val)
 {
 	struct amdgpu_device *adev = data;
@@ -3485,6 +3594,7 @@ DEFINE_DEBUGFS_ATTRIBUTE(disallow_edp_enter_replay_fops,
 
 DEFINE_DEBUGFS_ATTRIBUTE(ips_residency_cntl_fops, ips_residency_cntl_get,
 			   ips_residency_cntl_set, "%llu\n");
+
 DEFINE_SHOW_ATTRIBUTE(current_backlight);
 DEFINE_SHOW_ATTRIBUTE(target_backlight);
 DEFINE_SHOW_ATTRIBUTE(ips_status);
@@ -3855,28 +3965,35 @@ DEFINE_DEBUGFS_ATTRIBUTE(crc_win_y_end_fops, crc_win_y_end_get,
 static int crc_win_update_set(void *data, u64 val)
 {
 	struct drm_crtc *crtc = data;
-	struct amdgpu_crtc *acrtc;
+	struct amdgpu_crtc *acrtc = to_amdgpu_crtc(crtc);
 	struct amdgpu_device *adev = drm_to_adev(crtc->dev);
 
 	if (val) {
-		acrtc = to_amdgpu_crtc(crtc);
 		mutex_lock(&adev->dm.dc_lock);
-		/* PSR may write to OTG CRC window control register,
-		 * so close it before starting secure_display.
+		/* PSR Replay may write to OTG CRC window control register,
+		 * so inactive it before starting secure_display by sending disable event.
 		 */
-		amdgpu_dm_psr_disable(acrtc->dm_irq_params.stream, true);
+		amdgpu_dm_psr_set_event(&adev->dm, acrtc->dm_irq_params.stream, true,
+			psr_event_crc_window_active, true);
+		amdgpu_dm_replay_set_event(&adev->dm, acrtc->dm_irq_params.stream, true,
+			replay_event_crc_window_active, true);
 
 		spin_lock_irq(&adev_to_drm(adev)->event_lock);
-
 		acrtc->dm_irq_params.window_param[0].enable = true;
 		acrtc->dm_irq_params.window_param[0].update_win = true;
 		acrtc->dm_irq_params.window_param[0].skip_frame_cnt = 0;
 		acrtc->dm_irq_params.crc_window_activated = true;
-
 		spin_unlock_irq(&adev_to_drm(adev)->event_lock);
 		mutex_unlock(&adev->dm.dc_lock);
+	} else {
+		/* Clear disable events to allow PSR/Replay to active */
+		mutex_lock(&adev->dm.dc_lock);
+		amdgpu_dm_psr_set_event(&adev->dm, acrtc->dm_irq_params.stream, false,
+			psr_event_crc_window_active, false);
+		amdgpu_dm_replay_set_event(&adev->dm, acrtc->dm_irq_params.stream, false,
+			replay_event_crc_window_active, false);
+		mutex_unlock(&adev->dm.dc_lock);
 	}
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
index a64e95860e99b..7dcc587c45e91 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.c
@@ -291,24 +291,16 @@ static void dm_ism_commit_idle_optimization_state(struct amdgpu_dm_ism *ism,
 	 */
 	if (stream && stream->link) {
 		/*
-		 * If allow_panel_sso is true when disabling vblank, allow
-		 * deeper panel sleep states such as PSR1 and Replay static
-		 * screen optimization.
-		 */
-		if (!vblank_enabled && allow_panel_sso) {
-			amdgpu_dm_crtc_set_panel_sr_feature(
-				dm, acrtc, stream, false,
-				acrtc->dm_irq_params.allow_sr_entry);
-		} else if (vblank_enabled) {
-			/* Make sure to exit SSO on vblank enable */
-			amdgpu_dm_crtc_set_panel_sr_feature(
-				dm, acrtc, stream, true,
-				acrtc->dm_irq_params.allow_sr_entry);
-		}
-		/*
-		 * Else, vblank_enabled == false and allow_panel_sso == false;
-		 * do nothing here.
+		 * If the OS requires vblank events (or vblank is otherwise enabled),
+		 * do not allow static screen optimizations.
+		 *
+		 * Keep ism->allow_static_screen_optimizations unchanged so the
+		 * hysteresis-based decision can be reused once vblank is disabled.
 		 */
+		allow_panel_sso = allow_panel_sso && !vblank_enabled;
+		amdgpu_dm_crtc_set_static_screen_optimze(
+			dm, stream, allow_panel_sso,
+			acrtc->dm_irq_params.allow_sr_entry);
 	}
 
 	/*
@@ -459,8 +451,8 @@ void amdgpu_dm_ism_commit_event(struct amdgpu_dm_ism *ism,
 	struct amdgpu_display_manager *dm = &adev->dm;
 	struct dm_crtc_state *acrtc_state = to_dm_crtc_state(acrtc->base.state);
 
-	/* ISM transitions must be called with mutex acquired */
-	ASSERT(mutex_is_locked(&dm->dc_lock));
+	/* ISM transitions must be called with dc_lock held */
+	lockdep_assert_held(&dm->dc_lock);
 
 	/* ISM should not run after dc is destroyed */
 	ASSERT(dm->dc);
@@ -524,13 +516,20 @@ static void dm_ism_sso_delayed_work_func(struct work_struct *work)
 }
 
 /**
- * amdgpu_dm_ism_disable - Disable the ISM
+ * amdgpu_dm_ism_disable - Quiesce ISM workers
  *
  * @dm: The amdgpu display manager
  *
- * Disable the idle state manager by disabling any ISM work, canceling pending
- * work, and waiting for in-progress work to finish. After disabling, the system
- * is left in DM_ISM_STATE_FULL_POWER_RUNNING state.
+ * Cancels and disables any pending or in-flight ISM delayed work and waits
+ * for in-progress work to finish. After this returns, no ISM worker can run
+ * and subsequent mod_delayed_work() calls become no-ops via
+ * clear_pending_if_disabled().
+ *
+ * Must NOT be called with dc_lock held: the workers themselves take dc_lock,
+ * so a synchronous wait under dc_lock would deadlock.
+ *
+ * The caller is responsible for driving the FSM back to FULL_POWER_RUNNING
+ * (under dc_lock) by calling amdgpu_dm_ism_force_full_power().
  */
 void amdgpu_dm_ism_disable(struct amdgpu_display_manager *dm)
 {
@@ -538,21 +537,54 @@ void amdgpu_dm_ism_disable(struct amdgpu_display_manager *dm)
 	struct amdgpu_crtc *acrtc;
 	struct amdgpu_dm_ism *ism;
 
-	ASSERT(mutex_is_locked(&dm->dc_lock));
+	/*
+	 * Caller must NOT hold dc_lock: the ISM delayed work handlers
+	 * acquire dc_lock themselves, so waiting for them via
+	 * disable_delayed_work_sync() while holding dc_lock would
+	 * self-deadlock against an in-flight worker.
+	 */
+	lockdep_assert_not_held(&dm->dc_lock);
 
 	drm_for_each_crtc(crtc, dm->ddev) {
 		acrtc = to_amdgpu_crtc(crtc);
 		ism = &acrtc->ism;
 
-		/* Cancel and disable any pending work */
 		disable_delayed_work_sync(&ism->delayed_work);
 		disable_delayed_work_sync(&ism->sso_delayed_work);
+	}
+}
+
+/**
+ * amdgpu_dm_ism_force_full_power - Force every CRTC's ISM FSM to FULL_POWER
+ *
+ * @dm: The amdgpu display manager
+ *
+ * Sends DM_ISM_EVENT_EXIT_IDLE_REQUESTED to every CRTC's ISM, leaving each
+ * FSM in FULL_POWER_RUNNING. Intended to be paired with
+ * amdgpu_dm_ism_disable(): callers should first quiesce workers (without
+ * dc_lock), then take dc_lock and call this helper.
+ *
+ * Must be called with dc_lock held.
+ */
+void amdgpu_dm_ism_force_full_power(struct amdgpu_display_manager *dm)
+{
+	struct drm_crtc *crtc;
+	struct amdgpu_crtc *acrtc;
+
+	/*
+	 * Caller must hold dc_lock: commit_event() drives the FSM and
+	 * may touch dc state via dc_allow_idle_optimizations() etc.
+	 */
+	lockdep_assert_held(&dm->dc_lock);
+
+	drm_for_each_crtc(crtc, dm->ddev) {
+		acrtc = to_amdgpu_crtc(crtc);
 
 		/*
 		 * When disabled, leave in FULL_POWER_RUNNING state.
-		 * EXIT_IDLE will not queue any work
+		 * EXIT_IDLE will not queue any work.
 		 */
-		amdgpu_dm_ism_commit_event(ism,
+		amdgpu_dm_ism_commit_event(&acrtc->ism,
 					   DM_ISM_EVENT_EXIT_IDLE_REQUESTED);
 	}
 }
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h
index fde0ddc8d4e4b..964408cd9a839 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_ism.h
@@ -146,6 +146,7 @@ void amdgpu_dm_ism_fini(struct amdgpu_dm_ism *ism);
 void amdgpu_dm_ism_commit_event(struct amdgpu_dm_ism *ism,
 				enum amdgpu_dm_ism_event event);
 void amdgpu_dm_ism_disable(struct amdgpu_display_manager *dm);
+void amdgpu_dm_ism_force_full_power(struct amdgpu_display_manager *dm);
 void amdgpu_dm_ism_enable(struct amdgpu_display_manager *dm);
 
 #endif
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
index be038d9014bb4..51924ae87705a 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c
@@ -497,7 +497,7 @@ static int dm_dp_mst_get_modes(struct drm_connector *connector)
 
 static struct drm_encoder *
 dm_mst_atomic_best_encoder(struct drm_connector *connector,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *connector_state = drm_atomic_get_new_connector_state(state,
 											 connector);
@@ -579,7 +579,7 @@ dm_dp_mst_detect(struct drm_connector *connector,
 }
 
 static int dm_dp_mst_atomic_check(struct drm_connector *connector,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
 	struct drm_dp_mst_topology_mgr *mst_mgr = &aconnector->mst_root->mst_mgr;
@@ -986,7 +986,7 @@ static int bpp_x16_from_pbn(struct dsc_mst_fairness_params param, int pbn)
 	return dsc_config.bits_per_pixel;
 }
 
-static int increase_dsc_bpp(struct drm_atomic_state *state,
+static int increase_dsc_bpp(struct drm_atomic_commit *state,
 			    struct drm_dp_mst_topology_state *mst_state,
 			    struct dc_link *dc_link,
 			    struct dsc_mst_fairness_params *params,
@@ -1090,7 +1090,7 @@ static int increase_dsc_bpp(struct drm_atomic_state *state,
 	return 0;
 }
 
-static int try_disable_dsc(struct drm_atomic_state *state,
+static int try_disable_dsc(struct drm_atomic_commit *state,
 			   struct dc_link *dc_link,
 			   struct dsc_mst_fairness_params *params,
 			   struct dsc_mst_fairness_vars *vars,
@@ -1183,7 +1183,7 @@ static void log_dsc_params(int count, struct dsc_mst_fairness_vars *vars, int k)
 				 i, vars[i + k].dsc_enabled, vars[i + k].bpp_x16, vars[i + k].pbn);
 }
 
-static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
+static int compute_mst_dsc_configs_for_link(struct drm_atomic_commit *state,
 					    struct dc_state *dc_state,
 					    struct dc_link *dc_link,
 					    struct dsc_mst_fairness_vars *vars,
@@ -1348,7 +1348,7 @@ static int compute_mst_dsc_configs_for_link(struct drm_atomic_state *state,
 }
 
 static bool is_dsc_need_re_compute(
-	struct drm_atomic_state *state,
+	struct drm_atomic_commit *state,
 	struct dc_state *dc_state,
 	struct dc_link *dc_link)
 {
@@ -1479,7 +1479,7 @@ static bool is_dsc_need_re_compute(
 	return is_dsc_need_re_compute;
 }
 
-int compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
+int compute_mst_dsc_configs_for_state(struct drm_atomic_commit *state,
 				      struct dc_state *dc_state,
 				      struct dsc_mst_fairness_vars *vars)
 {
@@ -1549,7 +1549,7 @@ int compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
 	return ret;
 }
 
-static int pre_compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
+static int pre_compute_mst_dsc_configs_for_state(struct drm_atomic_commit *state,
 						 struct dc_state *dc_state,
 						 struct dsc_mst_fairness_vars *vars)
 {
@@ -1602,7 +1602,7 @@ static int pre_compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
 	return ret;
 }
 
-static int find_crtc_index_in_state_by_stream(struct drm_atomic_state *state,
+static int find_crtc_index_in_state_by_stream(struct drm_atomic_commit *state,
 					      struct dc_stream_state *stream)
 {
 	int i;
@@ -1633,7 +1633,7 @@ static bool is_link_to_dschub(struct dc_link *dc_link)
 	return true;
 }
 
-static bool is_dsc_precompute_needed(struct drm_atomic_state *state)
+static bool is_dsc_precompute_needed(struct drm_atomic_commit *state)
 {
 	int i;
 	struct drm_crtc *crtc;
@@ -1654,7 +1654,7 @@ static bool is_dsc_precompute_needed(struct drm_atomic_state *state)
 	return ret;
 }
 
-int pre_validate_dsc(struct drm_atomic_state *state,
+int pre_validate_dsc(struct drm_atomic_commit *state,
 		     struct dm_atomic_state **dm_state_ptr,
 		     struct dsc_mst_fairness_vars *vars)
 {
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
index 6f7ea684b555f..0e8eef5bdb743 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.h
@@ -80,13 +80,13 @@ struct dsc_mst_fairness_vars {
 	struct amdgpu_dm_connector *aconnector;
 };
 
-int compute_mst_dsc_configs_for_state(struct drm_atomic_state *state,
+int compute_mst_dsc_configs_for_state(struct drm_atomic_commit *state,
 				      struct dc_state *dc_state,
 				      struct dsc_mst_fairness_vars *vars);
 
 bool needs_dsc_aux_workaround(struct dc_link *link);
 
-int pre_validate_dsc(struct drm_atomic_state *state,
+int pre_validate_dsc(struct drm_atomic_commit *state,
 		     struct dm_atomic_state **dm_state_ptr,
 		     struct dsc_mst_fairness_vars *vars);
 
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
index 81e43534ec599..e957657b06c78 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c
@@ -1227,7 +1227,7 @@ int amdgpu_dm_plane_fill_dc_scaling_info(struct amdgpu_device *adev,
 }
 
 static int amdgpu_dm_plane_atomic_check(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -1274,7 +1274,7 @@ static int amdgpu_dm_plane_atomic_check(struct drm_plane *plane,
 }
 
 static int amdgpu_dm_plane_atomic_async_check(struct drm_plane *plane,
-					      struct drm_atomic_state *state, bool flip)
+					      struct drm_atomic_commit *state, bool flip)
 {
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_plane_state *new_plane_state;
@@ -1431,7 +1431,7 @@ void amdgpu_dm_plane_handle_cursor_update(struct drm_plane *plane,
 }
 
 static void amdgpu_dm_plane_atomic_async_update(struct drm_plane *plane,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
index 99d6d6c935610..dc5913a6456e2 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.c
@@ -58,171 +58,76 @@ static bool link_supports_psrsu(struct dc_link *link)
 	return false;
 }
 
-/*
- * amdgpu_dm_set_psr_caps() - set link psr capabilities
- * @link: link
- *
- */
-void amdgpu_dm_set_psr_caps(struct dc_link *link)
+static void amdgpu_dm_psr_fill_caps(struct dc_link *link, struct psr_caps *caps)
 {
-	if (!(link->connector_signal & SIGNAL_TYPE_EDP)) {
-		link->psr_settings.psr_feature_enabled = false;
-		return;
-	}
-
-	if (link->type == dc_connection_none) {
-		link->psr_settings.psr_feature_enabled = false;
-		return;
-	}
-
-	if (link->dpcd_caps.psr_info.psr_version == 0) {
-		link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
-		link->psr_settings.psr_feature_enabled = false;
-
-	} else {
-		unsigned int panel_inst = 0;
-
-		if (link_supports_psrsu(link))
-			link->psr_settings.psr_version = DC_PSR_VERSION_SU_1;
-		else
-			link->psr_settings.psr_version = DC_PSR_VERSION_1;
-
-		link->psr_settings.psr_feature_enabled = true;
-
-		/*disable allow psr/psrsu/replay on eDP1*/
-		if (dc_get_edp_link_panel_inst(link->ctx->dc, link, &panel_inst) && panel_inst == 1) {
-			link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
-			link->psr_settings.psr_feature_enabled = false;
-		}
-	}
+	struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
+	unsigned int power_opts = 0;
+
+	if (amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT)
+		power_opts |= psr_power_opt_smu_opt_static_screen;
+	power_opts |= psr_power_opt_z10_static_screen;
+
+	if (link->psr_settings.psr_version == DC_PSR_VERSION_1)
+		caps->psr_version = 1;
+	else if (link->psr_settings.psr_version == DC_PSR_VERSION_SU_1)
+		caps->psr_version = 2;
+
+	caps->psr_rfb_setup_time = (6 - dpcd_caps->psr_info.psr_dpcd_caps.bits.PSR_SETUP_TIME) * 55;
+	caps->psr_exit_link_training_required =
+		!dpcd_caps->psr_info.psr_dpcd_caps.bits.LINK_TRAINING_ON_EXIT_NOT_REQUIRED;
+	caps->edp_revision = dpcd_caps->edp_rev;
+	caps->support_ver = dpcd_caps->psr_info.psr_version;
+	caps->su_granularity_required =
+		dpcd_caps->psr_info.psr_dpcd_caps.bits.SU_GRANULARITY_REQUIRED;
+	caps->y_coordinate_required = dpcd_caps->psr_info.psr_dpcd_caps.bits.Y_COORDINATE_REQUIRED;
+	caps->su_y_granularity = dpcd_caps->psr_info.psr2_su_y_granularity_cap;
+	caps->alpm_cap = dpcd_caps->alpm_caps.bits.AUX_WAKE_ALPM_CAP;
+	caps->standby_support = dpcd_caps->alpm_caps.bits.PM_STATE_2A_SUPPORT;
+	caps->rate_control_caps = 0; /* TODO: read in rc caps from aux */
+	caps->psr_power_opt_flag = power_opts;
 }
 
 /*
- * amdgpu_dm_link_setup_psr() - configure psr link
- * @stream: stream state
- *
- * Return: true if success
+ * amdgpu_dm_set_psr_caps() - set link psr capabilities
+ * @link: link
+ * @aconnector: amdgpu_dm_connector
  */
-bool amdgpu_dm_link_setup_psr(struct dc_stream_state *stream)
+bool amdgpu_dm_set_psr_caps(struct dc_link *link, struct amdgpu_dm_connector *aconnector)
 {
-	struct dc_link *link = NULL;
-	struct psr_config psr_config = {0};
-	struct psr_context psr_context = {0};
-	struct dc *dc = NULL;
-	bool ret = false;
+	struct dc *dc;
+	unsigned int panel_inst = 0;
 
-	if (stream == NULL)
+	if (!link || !aconnector)
 		return false;
 
-	link = stream->link;
 	dc = link->ctx->dc;
 
-	if (link->psr_settings.psr_version != DC_PSR_VERSION_UNSUPPORTED) {
-		mod_power_calc_psr_configs(&psr_config, link, stream);
-
-		/* linux DM specific updating for psr config fields */
-		psr_config.allow_smu_optimizations =
-			(amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT) &&
-			mod_power_only_edp(dc->current_state, stream);
-		psr_config.allow_multi_disp_optimizations =
-			(amdgpu_dc_feature_mask & DC_PSR_ALLOW_MULTI_DISP_OPT);
-
-		if (link->psr_settings.psr_version == DC_PSR_VERSION_SU_1) {
-			if (!psr_su_set_dsc_slice_height(dc, link, stream, &psr_config))
-				return false;
-		}
-
-		ret = dc_link_setup_psr(link, stream, &psr_config, &psr_context);
-
-	}
-	DRM_DEBUG_DRIVER("PSR link: %d\n",	link->psr_settings.psr_feature_enabled);
-
-	return ret;
-}
-
-/*
- * amdgpu_dm_psr_enable() - enable psr f/w
- * @stream: stream state
- *
- */
-void amdgpu_dm_psr_enable(struct dc_stream_state *stream)
-{
-	struct dc_link *link = stream->link;
-	unsigned int vsync_rate_hz = 0;
-	struct dc_static_screen_params params = {0};
-	/* Calculate number of static frames before generating interrupt to
-	 * enter PSR.
-	 */
-	// Init fail safe of 2 frames static
-	unsigned int num_frames_static = 2;
-	unsigned int power_opt = 0;
-	bool psr_enable = true;
-
-	DRM_DEBUG_DRIVER("Enabling psr...\n");
-
-	vsync_rate_hz = div64_u64(div64_u64((
-			stream->timing.pix_clk_100hz * (uint64_t)100),
-			stream->timing.v_total),
-			stream->timing.h_total);
-
-	/* Round up
-	 * Calculate number of frames such that at least 30 ms of time has
-	 * passed.
-	 */
-	if (vsync_rate_hz != 0) {
-		unsigned int frame_time_microsec = 1000000 / vsync_rate_hz;
-
-		num_frames_static = (30000 / frame_time_microsec) + 1;
-	}
-
-	params.triggers.cursor_update = true;
-	params.triggers.overlay_update = true;
-	params.triggers.surface_update = true;
-	params.num_frames = num_frames_static;
+	/* Reset psr version first */
+	link->psr_settings.psr_version = DC_PSR_VERSION_UNSUPPORTED;
 
-	dc_stream_set_static_screen_params(link->ctx->dc,
-					   &stream, 1,
-					   &params);
+	if (!dc->caps.dmub_caps.psr)
+		return false;
 
-	/*
-	 * Only enable static-screen optimizations for PSR1. For PSR SU, this
-	 * causes vstartup interrupt issues, used by amdgpu_dm to send vblank
-	 * events.
-	 */
-	if (link->psr_settings.psr_version < DC_PSR_VERSION_SU_1)
-		power_opt |= psr_power_opt_z10_static_screen;
+	if (!(link->connector_signal & SIGNAL_TYPE_EDP))
+		return false;
 
-	dc_link_set_psr_allow_active(link, &psr_enable, false, false, &power_opt);
+	if (link->type == dc_connection_none)
+		return false;
 
-	if (link->ctx->dc->caps.ips_support)
-		dc_allow_idle_optimizations(link->ctx->dc, true);
-}
+	if (link->dpcd_caps.psr_info.psr_version == 0)
+		return false;
 
-/*
- * amdgpu_dm_psr_disable() - disable psr f/w
- * @stream:  stream state
- *
- * Return: true if success
- */
-bool amdgpu_dm_psr_disable(struct dc_stream_state *stream, bool wait)
-{
-	bool psr_enable = false;
+	/*disable allow psr/psrsu/replay on eDP1*/
+	if (dc_get_edp_link_panel_inst(link->ctx->dc, link, &panel_inst) && panel_inst == 1)
+		return false;
 
-	DRM_DEBUG_DRIVER("Disabling psr...\n");
+	if (link_supports_psrsu(link))
+		link->psr_settings.psr_version = DC_PSR_VERSION_SU_1;
+	else
+		link->psr_settings.psr_version = DC_PSR_VERSION_1;
 
-	return dc_link_set_psr_allow_active(stream->link, &psr_enable, wait, false, NULL);
-}
-
-/*
- * amdgpu_dm_psr_disable_all() - disable psr f/w for all streams
- * if psr is enabled on any stream
- *
- * Return: true if success
- */
-bool amdgpu_dm_psr_disable_all(struct amdgpu_display_manager *dm)
-{
-	DRM_DEBUG_DRIVER("Disabling psr if psr is enabled on any stream\n");
-	return dc_set_psr_allow_active(dm->dc, false);
+	amdgpu_dm_psr_fill_caps(link, &aconnector->psr_caps);
+	return true;
 }
 
 /*
@@ -250,36 +155,37 @@ bool amdgpu_dm_psr_is_active_allowed(struct amdgpu_display_manager *dm)
 			break;
 		}
 	}
-
 	return allow_active;
 }
 
-/**
- * amdgpu_dm_psr_wait_disable() - Wait for eDP panel to exit PSR
- * @stream: stream state attached to the eDP link
- *
- * Waits for a max of 500ms for the eDP panel to exit PSR.
+/*
+ * amdgpu_dm_psr_set_event() - set or clear PSR event for stream
+ * @dm: pointer to amdgpu_display_manager
+ * @stream: pointer to dc_stream_state
+ * @set_event: true to set event, false to clear event
+ * @event: PSR event type
+ * @wait_for_disable: whether to wait for PSR to be disabled
  *
- * Return: true if panel exited PSR, false otherwise.
+ * Return: true if successful, false otherwise
  */
-bool amdgpu_dm_psr_wait_disable(struct dc_stream_state *stream)
+bool amdgpu_dm_psr_set_event(struct amdgpu_display_manager *dm, struct dc_stream_state *stream,
+		bool set_event, enum psr_event event, bool wait_for_disable)
 {
-	enum dc_psr_state psr_state = PSR_STATE0;
-	struct dc_link *link = stream->link;
-	int retry_count;
+	unsigned int psr_events;
 
-	if (link == NULL)
+	/* Validate all required parameters */
+	if (!stream || !stream->link ||
+		!stream->link->psr_settings.psr_feature_enabled)
 		return false;
 
-	for (retry_count = 0; retry_count <= 1000; retry_count++) {
-		dc_link_get_psr_state(link, &psr_state);
-		if (psr_state == PSR_STATE0)
-			break;
-		udelay(500);
-	}
-
-	if (retry_count == 1000)
+	/* Get current psr events */
+	if (!mod_power_get_psr_event(dm->power_module, stream, &psr_events))
 		return false;
 
-	return true;
+	/* If all events already in desired state, return true. */
+	if ((psr_events & event) == (set_event ? event : 0))
+		return true;
+
+	return mod_power_set_psr_event(dm->power_module, stream,
+				       set_event, event, wait_for_disable);
 }
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h
index 4fb8626913cfb..16d535806ad63 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_psr.h
@@ -28,16 +28,15 @@
 #define AMDGPU_DM_AMDGPU_DM_PSR_H_
 
 #include "amdgpu.h"
+#include "dc.h"
+#include "modules/inc/mod_power.h"
 
 /* the number of pageflips before enabling psr */
 #define AMDGPU_DM_PSR_ENTRY_DELAY 5
 
-void amdgpu_dm_set_psr_caps(struct dc_link *link);
-void amdgpu_dm_psr_enable(struct dc_stream_state *stream);
-bool amdgpu_dm_link_setup_psr(struct dc_stream_state *stream);
-bool amdgpu_dm_psr_disable(struct dc_stream_state *stream, bool wait);
-bool amdgpu_dm_psr_disable_all(struct amdgpu_display_manager *dm);
+bool amdgpu_dm_set_psr_caps(struct dc_link *link, struct amdgpu_dm_connector *aconnector);
 bool amdgpu_dm_psr_is_active_allowed(struct amdgpu_display_manager *dm);
-bool amdgpu_dm_psr_wait_disable(struct dc_stream_state *stream);
-
+bool amdgpu_dm_psr_set_event(struct amdgpu_display_manager *dm,
+		struct dc_stream_state *stream, bool set_event,	enum psr_event event,
+		bool wait_for_disable);
 #endif /* AMDGPU_DM_AMDGPU_DM_PSR_H_ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c
index 8c150b0011059..297125d1db708 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.c
@@ -27,7 +27,6 @@
 #include "amdgpu_dm_replay.h"
 #include "dc_dmub_srv.h"
 #include "dc.h"
-#include "dm_helpers.h"
 #include "amdgpu_dm.h"
 #include "modules/power/power_helpers.h"
 #include "dmub/inc/dmub_cmd.h"
@@ -99,13 +98,29 @@ bool amdgpu_dm_set_replay_caps(struct dc_link *link, struct amdgpu_dm_connector
 		!dc->ctx->dmub_srv->dmub->feature_caps.replay_supported)
 		return false;
 
+	/* Mark Replay is supported in link and update related attributes
+	 * This flag presents DPCD caps & amd_vsdb caps satisfy replay requirement.
+	 */
+	pr_config.replay_cap_support = true;
+
 	// Mark Replay is supported in pr_config
 	pr_config.replay_supported = true;
 
+	pr_config.replay_enable_option = pr_enable_option_general_ui |
+					 pr_enable_option_static_screen	|
+					 pr_enable_option_static_screen_coasting;
+	pr_config.replay_power_opt_supported = replay_power_opt_smu_opt_static_screen |
+					       replay_power_opt_z10_static_screen;
+	pr_config.replay_smu_opt_supported = false;
+	pr_config.replay_support_fast_resync_in_ultra_sleep_mode =
+		aconnector->max_vfreq >= 2 * aconnector->min_vfreq;
+	pr_config.force_disable_desync_error_check = false;
+
 	debug_flags = (union replay_debug_flags *)&pr_config.debug_flags;
 	debug_flags->u32All = 0;
 	debug_flags->bitfields.visual_confirm =
 		link->ctx->dc->debug.visual_confirm == VISUAL_CONFIRM_REPLAY;
+	debug_flags->bitfields.skip_crtc_disabled = dc->debug.replay_skip_crtc_disabled;
 
 	init_replay_config(link, &pr_config);
 
@@ -113,104 +128,80 @@ bool amdgpu_dm_set_replay_caps(struct dc_link *link, struct amdgpu_dm_connector
 }
 
 /*
- * amdgpu_dm_link_setup_replay() - configure replay link
- * @link: link
- * @aconnector: aconnector
+ * amdgpu_dm_link_setup_replay() - config replay settings
+ * @stream: pointer to dc_stream_state structure
+ * @vrr_params: pointer to mod_vrr_params structure containing VRR parameters
  *
+ * config replay link settings including coasting vtotal calculations.
+ *
+ * Return: true if successful, false if any parameter is invalid or replay not supported
  */
-bool amdgpu_dm_link_setup_replay(struct dc_link *link, struct amdgpu_dm_connector *aconnector)
+bool amdgpu_dm_link_setup_replay(struct dc_stream_state *stream,
+		struct mod_vrr_params *vrr_params)
 {
-	struct replay_config *pr_config;
+	struct dc_link *link;
+	unsigned int static_coasting_vtotal;
+	unsigned int nom_coasting_vtotal;
 
-	if (link == NULL || aconnector == NULL)
+	if (!stream || !stream->link || !vrr_params)
 		return false;
 
-	pr_config = &link->replay_settings.config;
-
-	if (!pr_config->replay_supported)
+	link = stream->link;
+	if (!link->replay_settings.config.replay_supported)
 		return false;
 
-	pr_config->replay_power_opt_supported = 0x11;
-	pr_config->replay_smu_opt_supported = false;
-	pr_config->replay_enable_option |= pr_enable_option_static_screen;
-	pr_config->replay_support_fast_resync_in_ultra_sleep_mode = aconnector->max_vfreq >= 2 * aconnector->min_vfreq;
-	pr_config->replay_timing_sync_supported = false;
+	if (link->replay_settings.replay_feature_enabled)
+		return true;
 
-	if (!pr_config->replay_timing_sync_supported)
-		pr_config->replay_enable_option &= ~pr_enable_option_general_ui;
+	calculate_replay_link_off_frame_count(link, stream->timing.v_total,
+			stream->timing.h_total);
 
-	link->replay_settings.replay_feature_enabled = true;
+	nom_coasting_vtotal = stream->timing.v_total;
+	static_coasting_vtotal = mod_freesync_calc_v_total_from_refresh(stream,
+			vrr_params->min_refresh_in_uhz);
 
+	set_replay_coasting_vtotal(link, PR_COASTING_TYPE_NOM,
+			nom_coasting_vtotal);
+	set_replay_coasting_vtotal(link, PR_COASTING_TYPE_STATIC,
+			static_coasting_vtotal);
 	return true;
 }
 
 /*
- * amdgpu_dm_replay_enable() - enable replay f/w
- * @stream: stream state
+ * amdgpu_dm_replay_set_event() - set or clear replay event for a stream
+ * @dm: pointer to amdgpu_display_manager
+ * @stream: pointer to dc_stream_state
+ * @set_event: true to set event, false to clear event
+ * @event: replay event type to set or clear
+ * @wait_for_disable: whether to wait for replay to be disabled before returning
  *
- * Return: true if success
- */
-bool amdgpu_dm_replay_enable(struct dc_stream_state *stream, bool wait)
-{
-	bool replay_active = true;
-	struct dc_link *link = NULL;
-	struct amdgpu_dm_connector *aconnector = NULL;
-
-	if (stream == NULL)
-		return false;
-
-	/* Check if replay is disabled by connector flag */
-	aconnector = (struct amdgpu_dm_connector *)stream->dm_stream_context;
-	if (!aconnector || aconnector->disallow_edp_enter_replay) {
-		return false;
-	}
-
-	link = stream->link;
-
-	if (link) {
-		link->dc->link_srv->dp_setup_replay(link, stream);
-		link->dc->link_srv->edp_set_coasting_vtotal(link, stream->timing.v_total, 0);
-		DRM_DEBUG_DRIVER("Enabling replay...\n");
-		link->dc->link_srv->edp_set_replay_allow_active(link, &replay_active, wait, false, NULL);
-		return true;
-	}
-
-	return false;
-}
-
-/*
- * amdgpu_dm_replay_disable() - disable replay f/w
- * @stream:  stream state
+ * This function sets or clears a specific replay event for the given stream.
+ * It temporarily disables idle optimizations during the operation to ensure
+ * hardware access is available.
  *
- * Return: true if success
+ * Return: true if successful, false if any parameter is invalid or operation fails
  */
-bool amdgpu_dm_replay_disable(struct dc_stream_state *stream)
+bool amdgpu_dm_replay_set_event(struct amdgpu_display_manager *dm,
+		struct dc_stream_state *stream,
+		bool set_event,
+		enum replay_event event,
+		bool wait_for_disable)
 {
-	bool replay_active = false;
-	struct dc_link *link = NULL;
+	unsigned int replay_events;
 
-	if (stream == NULL)
+	/* Validate all required parameters */
+	if (!stream || !stream->link ||
+		!stream->link->replay_settings.replay_feature_enabled)
 		return false;
 
-	link = stream->link;
+	/* Get current replay events */
+	if (!mod_power_get_replay_event(dm->power_module, stream, &replay_events))
+		return false;
 
-	if (link) {
-		DRM_DEBUG_DRIVER("Disabling replay...\n");
-		link->dc->link_srv->edp_set_replay_allow_active(stream->link, &replay_active, true, false, NULL);
+	/* If all events already in desired state, return true. */
+	if ((replay_events & event) == (set_event ? event : 0))
 		return true;
-	}
-
-	return false;
-}
 
-/*
- * amdgpu_dm_replay_disable_all() - disable replay f/w
- * if replay is enabled on any stream
- *
- * Return: true if success
- */
-bool amdgpu_dm_replay_disable_all(struct amdgpu_display_manager *dm)
-{
-	DRM_DEBUG_DRIVER("Disabling replay if replay is enabled on any stream\n");
-	return dc_set_replay_allow_active(dm->dc, false);
+	return mod_power_set_replay_event(dm->power_module, stream,
+					 set_event, event, wait_for_disable);
 }
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.h
index 73b6c67ae5e76..021bf02555166 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_replay.h
@@ -28,22 +28,16 @@
 #define AMDGPU_DM_AMDGPU_DM_REPLAY_H_
 
 #include "amdgpu.h"
+#include "dc.h"
+#include "modules/inc/mod_power.h"
 
-enum replay_enable_option {
-	pr_enable_option_static_screen = 0x1,
-	pr_enable_option_mpo_video = 0x2,
-	pr_enable_option_full_screen_video = 0x4,
-	pr_enable_option_general_ui = 0x8,
-	pr_enable_option_static_screen_coasting = 0x10000,
-	pr_enable_option_mpo_video_coasting = 0x20000,
-	pr_enable_option_full_screen_video_coasting = 0x40000,
-};
-
-bool amdgpu_dm_link_supports_replay(struct dc_link *link, struct amdgpu_dm_connector *aconnector);
-bool amdgpu_dm_replay_enable(struct dc_stream_state *stream, bool enable);
-bool amdgpu_dm_set_replay_caps(struct dc_link *link, struct amdgpu_dm_connector *aconnector);
-bool amdgpu_dm_link_setup_replay(struct dc_link *link, struct amdgpu_dm_connector *aconnector);
-bool amdgpu_dm_replay_disable(struct dc_stream_state *stream);
-bool amdgpu_dm_replay_disable_all(struct amdgpu_display_manager *dm);
-
+bool amdgpu_dm_link_supports_replay(struct dc_link *link,
+		struct amdgpu_dm_connector *aconnector);
+bool amdgpu_dm_set_replay_caps(struct dc_link *link,
+		struct amdgpu_dm_connector *aconnector);
+bool amdgpu_dm_link_setup_replay(struct dc_stream_state *stream,
+		struct mod_vrr_params *vrr_params);
+bool amdgpu_dm_replay_set_event(struct amdgpu_display_manager *dm,
+		struct dc_stream_state *stream, bool set_event,
+		enum replay_event event, bool wait_for_disable);
 #endif /* AMDGPU_DM_AMDGPU_DM_REPLAY_H_ */
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
index 8550d5e8b7530..84dcb573d98f5 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_services.c
@@ -62,3 +62,30 @@ void dm_trace_smu_exit(bool success, uint32_t response, struct dc_context *ctx)
 }
 
 /**** power component interfaces ****/
+
+bool dm_query_extended_brightness_caps(struct dc_context *ctx,
+	enum dm_acpi_display_type display, struct dm_acpi_atif_backlight_caps *pCaps)
+{
+	struct amdgpu_device *adev;
+	struct amdgpu_display_manager *dm;
+	int bl_index = (display == AcpiDisplayType_LCD1) ? 0 : 1;
+
+	if (!ctx || !pCaps || !ctx->driver_context)
+		return false;
+
+	adev = (struct amdgpu_device *)ctx->driver_context;
+	dm = &adev->dm;
+
+	amdgpu_dm_update_backlight_caps(dm, bl_index);
+
+	pCaps->num_data_points = dm->backlight_caps[bl_index].data_points;
+	pCaps->max_input_signal = dm->backlight_caps[bl_index].max_input_signal;
+	pCaps->min_input_signal = dm->backlight_caps[bl_index].min_input_signal;
+	pCaps->ac_level_percentage = dm->backlight_caps[bl_index].ac_level;
+	pCaps->dc_level_percentage = dm->backlight_caps[bl_index].dc_level;
+
+	if (pCaps->num_data_points > 0)
+		memcpy(pCaps->data_points, dm->backlight_caps[bl_index].luminance_data,
+			sizeof(struct dm_bl_data_point) * pCaps->num_data_points);
+	return true;
+}
diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h
index e0fab8878d19c..f33a2c1e0da5f 100644
--- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h
+++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_trace.h
@@ -108,7 +108,7 @@ TRACE_EVENT(amdgpu_dm_connector_atomic_check,
 	    TP_STRUCT__entry(
 			     __field(uint32_t, conn_id)
 			     __field(const struct drm_connector_state *, conn_state)
-			     __field(const struct drm_atomic_state *, state)
+			     __field(const struct drm_atomic_commit *, state)
 			     __field(const struct drm_crtc_commit *, commit)
 			     __field(uint32_t, crtc_id)
 			     __field(uint32_t, best_encoder_id)
@@ -163,7 +163,7 @@ TRACE_EVENT(amdgpu_dm_crtc_atomic_check,
 	    TP_ARGS(state),
 
 	    TP_STRUCT__entry(
-			     __field(const struct drm_atomic_state *, state)
+			     __field(const struct drm_atomic_commit *, state)
 			     __field(const struct drm_crtc_state *, crtc_state)
 			     __field(const struct drm_crtc_commit *, commit)
 			     __field(uint32_t, crtc_id)
@@ -229,7 +229,7 @@ DECLARE_EVENT_CLASS(amdgpu_dm_plane_state_template,
 			     __field(uint32_t, plane_id)
 			     __field(enum drm_plane_type, plane_type)
 			     __field(const struct drm_plane_state *, plane_state)
-			     __field(const struct drm_atomic_state *, state)
+			     __field(const struct drm_atomic_commit *, state)
 			     __field(uint32_t, crtc_id)
 			     __field(uint32_t, fb_id)
 			     __field(uint32_t, fb_format)
@@ -315,11 +315,11 @@ DEFINE_EVENT(amdgpu_dm_plane_state_template, amdgpu_dm_atomic_update_cursor,
 	     TP_ARGS(state));
 
 TRACE_EVENT(amdgpu_dm_atomic_state_template,
-	    TP_PROTO(const struct drm_atomic_state *state),
+	    TP_PROTO(const struct drm_atomic_commit *state),
 	    TP_ARGS(state),
 
 	    TP_STRUCT__entry(
-			     __field(const struct drm_atomic_state *, state)
+			     __field(const struct drm_atomic_commit *, state)
 			     __field(bool, allow_modeset)
 			     __field(bool, legacy_cursor_update)
 			     __field(bool, async_update)
@@ -347,23 +347,23 @@ TRACE_EVENT(amdgpu_dm_atomic_state_template,
 );
 
 DEFINE_EVENT(amdgpu_dm_atomic_state_template, amdgpu_dm_atomic_commit_tail_begin,
-	     TP_PROTO(const struct drm_atomic_state *state),
+	     TP_PROTO(const struct drm_atomic_commit *state),
 	     TP_ARGS(state));
 
 DEFINE_EVENT(amdgpu_dm_atomic_state_template, amdgpu_dm_atomic_commit_tail_finish,
-	     TP_PROTO(const struct drm_atomic_state *state),
+	     TP_PROTO(const struct drm_atomic_commit *state),
 	     TP_ARGS(state));
 
 DEFINE_EVENT(amdgpu_dm_atomic_state_template, amdgpu_dm_atomic_check_begin,
-	     TP_PROTO(const struct drm_atomic_state *state),
+	     TP_PROTO(const struct drm_atomic_commit *state),
 	     TP_ARGS(state));
 
 TRACE_EVENT(amdgpu_dm_atomic_check_finish,
-	    TP_PROTO(const struct drm_atomic_state *state, int res),
+	    TP_PROTO(const struct drm_atomic_commit *state, int res),
 	    TP_ARGS(state, res),
 
 	    TP_STRUCT__entry(
-			     __field(const struct drm_atomic_state *, state)
+			     __field(const struct drm_atomic_commit *, state)
 			     __field(int, res)
 			     __field(bool, async_update)
 			     __field(bool, allow_modeset)
diff --git a/drivers/gpu/drm/amd/display/dc/basics/conversion.c b/drivers/gpu/drm/amd/display/dc/basics/conversion.c
index bd1f60ecaba4f..a1f2350388e2f 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/conversion.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/conversion.c
@@ -131,7 +131,7 @@ void convert_hw_matrix(struct fixed31_32 *matrix,
 		       uint16_t *reg,
 		       uint32_t buffer_size)
 {
-	for (int i = 0; i < buffer_size; ++i)
+	for (uint32_t i = 0; i < buffer_size; ++i)
 		matrix[i] = int_frac_to_fixed_point(reg[i], 2, 13);
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/basics/custom_float.c b/drivers/gpu/drm/amd/display/dc/basics/custom_float.c
index ae05ded9a7f35..d313584335d31 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/custom_float.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/custom_float.c
@@ -90,7 +90,7 @@ static bool build_custom_float(struct fixed31_32 value,
 	    dc_fixpt_lt(dc_fixpt_one, mantiss))
 		mantiss = dc_fixpt_zero;
 	else
-		mantiss = dc_fixpt_shl(mantiss, format->mantissa_bits);
+		mantiss = dc_fixpt_shl(mantiss, (unsigned char)format->mantissa_bits);
 
 	*mantissa = dc_fixpt_floor(mantiss);
 
diff --git a/drivers/gpu/drm/amd/display/dc/basics/dce_calcs.c b/drivers/gpu/drm/amd/display/dc/basics/dce_calcs.c
index f37a43f4172e1..1b8e843020d6c 100644
--- a/drivers/gpu/drm/amd/display/dc/basics/dce_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/basics/dce_calcs.c
@@ -302,7 +302,7 @@ static void calculate_bandwidth(
 	fbc_enabled = false;
 	lpt_enabled = false;
 	for (i = 4; i <= maximum_number_of_surfaces - 3; i++) {
-		if (i < data->number_of_displays + 4) {
+		if (i < (int32_t)(data->number_of_displays + 4)) {
 			if (i == 4 && data->d0_underlay_mode == bw_def_underlay_only) {
 				data->enable[i] = 0;
 				data->use_alpha[i] = 0;
@@ -2767,7 +2767,7 @@ static bool is_display_configuration_supported(
 	const struct bw_calcs_vbios *vbios,
 	const struct dce_bw_output *calcs_output)
 {
-	uint32_t int_max_clk;
+	int32_t int_max_clk;
 
 	int_max_clk = bw_fixed_to_int(vbios->high_voltage_max_dispclk);
 	int_max_clk *= 1000; /* MHz to kHz */
@@ -3077,7 +3077,7 @@ bool bw_calcs(struct dc_context *ctx,
 		}
 		calculate_bandwidth(dceip, vbios, data);
 
-		yclk_lvl = data->y_clk_level;
+		yclk_lvl = (uint8_t)data->y_clk_level;
 
 		calcs_output->nbp_state_change_enable =
 			data->nbp_state_change_enable;
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
index c307f42fe0b98..30567ef74e5c8 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser.c
@@ -805,8 +805,8 @@ static enum bp_result bios_parser_dac_load_detection(
 	uint32_t bios_0_scratch;
 	uint32_t device_id_mask = 0;
 
-	bp_params.device_id = get_support_mask_for_device_id(
-		DEVICE_TYPE_CRT, engine_id == ENGINE_ID_DACB ? 2 : 1);
+	bp_params.device_id = (uint16_t)get_support_mask_for_device_id(
+			DEVICE_TYPE_CRT, engine_id == ENGINE_ID_DACB ? 2 : 1);
 
 	if (bp_params.device_id == ATOM_DEVICE_CRT1_SUPPORT)
 		device_id_mask = ATOM_S0_CRT1_MASK;
@@ -1445,7 +1445,7 @@ static enum bp_result get_embedded_panel_info_v1_2(
 	info->ss_id = lvds->ucSS_Id;
 
 	{
-		uint8_t rr = le16_to_cpu(lvds->usSupportedRefreshRate);
+		uint16_t rr = le16_to_cpu(lvds->usSupportedRefreshRate);
 		/* Get minimum supported refresh rate*/
 		if (SUPPORTED_LCD_REFRESHRATE_30Hz & rr)
 			info->supported_rr.REFRESH_RATE_30HZ = 1;
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
index a1c08e1cc411d..ee6d52b87b029 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c
@@ -157,7 +157,7 @@ static uint8_t bios_parser_get_connectors_number(struct dc_bios *dcb)
 
 		break;
 	}
-	return count;
+	return (uint8_t)count;
 }
 
 static struct graphics_object_id bios_parser_get_connector_id(
@@ -401,7 +401,7 @@ static enum bp_result bios_parser_get_i2c_info(struct dc_bios *dcb,
 		return BP_RESULT_BADINPUT;
 
 	if (id.type == OBJECT_TYPE_GENERIC) {
-		dummy_record.i2c_id = id.id;
+		dummy_record.i2c_id = (uint8_t)id.id;
 
 		if (get_gpio_i2c_info(bp, &dummy_record, info) == BP_RESULT_OK)
 			return BP_RESULT_OK;
@@ -493,6 +493,10 @@ static enum bp_result get_gpio_i2c_info(
 			- sizeof(struct atom_common_table_header))
 				/ sizeof(struct atom_gpio_pin_assignment);
 
+	if (!bios_get_image(&bp->base, DATA_TABLES(gpio_pin_lut),
+			    le16_to_cpu(header->table_header.structuresize)))
+		return BP_RESULT_BADBIOSTABLE;
+
 	pin = (struct atom_gpio_pin_assignment *) header->gpio_pin;
 
 	for (table_index = 0; table_index < count; table_index++) {
@@ -681,6 +685,11 @@ static enum bp_result bios_parser_get_gpio_pin_info(
 	count = (le16_to_cpu(header->table_header.structuresize)
 			- sizeof(struct atom_common_table_header))
 				/ sizeof(struct atom_gpio_pin_assignment);
+
+	if (!bios_get_image(&bp->base, DATA_TABLES(gpio_pin_lut),
+			    le16_to_cpu(header->table_header.structuresize)))
+		return BP_RESULT_BADBIOSTABLE;
+
 	for (i = 0; i < count; ++i) {
 		if (header->gpio_pin[i].gpio_id != gpio_id)
 			continue;
@@ -1228,7 +1237,7 @@ static enum bp_result get_disp_caps_v4_1(
 	if (!disp_cntl_tbl)
 		return BP_RESULT_BADBIOSTABLE;
 
-	*dce_caps = disp_cntl_tbl->display_caps;
+	*dce_caps = (uint8_t)disp_cntl_tbl->display_caps;
 
 	return result;
 }
@@ -1252,7 +1261,7 @@ static enum bp_result get_disp_caps_v4_2(
 	if (!disp_cntl_tbl)
 		return BP_RESULT_BADBIOSTABLE;
 
-	*dce_caps = disp_cntl_tbl->display_caps;
+	*dce_caps = (uint8_t)disp_cntl_tbl->display_caps;
 
 	return result;
 }
@@ -1276,7 +1285,7 @@ static enum bp_result get_disp_caps_v4_3(
 	if (!disp_cntl_tbl)
 		return BP_RESULT_BADBIOSTABLE;
 
-	*dce_caps = disp_cntl_tbl->display_caps;
+	*dce_caps = (uint8_t)disp_cntl_tbl->display_caps;
 
 	return result;
 }
@@ -1300,7 +1309,7 @@ static enum bp_result get_disp_caps_v4_4(
 	if (!disp_cntl_tbl)
 		return BP_RESULT_BADBIOSTABLE;
 
-	*dce_caps = disp_cntl_tbl->display_caps;
+	*dce_caps = (uint8_t)disp_cntl_tbl->display_caps;
 
 	return result;
 }
@@ -1324,7 +1333,7 @@ static enum bp_result get_disp_caps_v4_5(
 	if (!disp_cntl_tbl)
 		return BP_RESULT_BADBIOSTABLE;
 
-	*dce_caps = disp_cntl_tbl->display_caps;
+	*dce_caps = (uint8_t)disp_cntl_tbl->display_caps;
 
 	return result;
 }
@@ -2585,7 +2594,7 @@ static enum bp_result get_integrated_info_v11(
 		info->ext_disp_conn_info.path[i].channel_mapping.raw =
 			info_v11->extdispconninfo.path[i].channelmapping;
 		info->ext_disp_conn_info.path[i].caps =
-				le16_to_cpu(info_v11->extdispconninfo.path[i].caps);
+				(unsigned short)le16_to_cpu(info_v11->extdispconninfo.path[i].caps);
 	}
 	info->ext_disp_conn_info.checksum =
 	info_v11->extdispconninfo.checksum;
@@ -2790,7 +2799,7 @@ static enum bp_result get_integrated_info_v2_1(
 		info->ext_disp_conn_info.path[i].channel_mapping.raw =
 			info_v2_1->extdispconninfo.path[i].channelmapping;
 		info->ext_disp_conn_info.path[i].caps =
-				le16_to_cpu(info_v2_1->extdispconninfo.path[i].caps);
+				(unsigned short)le16_to_cpu(info_v2_1->extdispconninfo.path[i].caps);
 	}
 
 	info->ext_disp_conn_info.checksum =
@@ -2954,7 +2963,7 @@ static enum bp_result get_integrated_info_v2_2(
 		info->ext_disp_conn_info.path[i].channel_mapping.raw =
 			info_v2_2->extdispconninfo.path[i].channelmapping;
 		info->ext_disp_conn_info.path[i].caps =
-				le16_to_cpu(info_v2_2->extdispconninfo.path[i].caps);
+				(unsigned short)le16_to_cpu(info_v2_2->extdispconninfo.path[i].caps);
 	}
 
 	info->ext_disp_conn_info.checksum =
@@ -2977,6 +2986,7 @@ static enum bp_result get_integrated_info_v2_2(
 	info->edp1_info.edp_panel_bpc =
 		info_v2_2->edp1_info.edp_panel_bpc;
 	info->edp1_info.edp_bootup_bl_level =
+		info_v2_2->edp1_info.edp_bootup_bl_level;
 
 	info->edp2_info.edp_backlight_pwm_hz =
 	le16_to_cpu(info_v2_2->edp2_info.edp_backlight_pwm_hz);
diff --git a/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c b/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c
index 8d2cf95ae7393..e00dc05c2d9d9 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/bios_parser_helper.c
@@ -37,10 +37,13 @@ uint8_t *bios_get_image(struct dc_bios *bp,
 	uint32_t offset,
 	uint32_t size)
 {
-	if (bp->bios && offset + size < bp->bios_size)
-		return bp->bios + offset;
-	else
+	if (!bp->bios)
 		return NULL;
+
+	if (offset > bp->bios_size || size > bp->bios_size - offset)
+		return NULL;
+
+	return bp->bios + offset;
 }
 
 #include "reg_helper.h"
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table.c b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
index f6e22dcecf29e..0df84394a3255 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table.c
@@ -1521,8 +1521,8 @@ static enum bp_result adjust_display_pll_v2(
 
 		if (pixel_clock_10KHz_in != 0) {
 			bp_params->adjusted_pixel_clock =
-					div_u64(pixel_clk * pixel_clk_10_khz_out,
-							pixel_clock_10KHz_in);
+					(uint32_t)div_u64(pixel_clk * pixel_clk_10_khz_out,
+							  pixel_clock_10KHz_in);
 		} else {
 			bp_params->adjusted_pixel_clock = 0;
 			BREAK_TO_DEBUGGER();
@@ -1571,8 +1571,8 @@ static enum bp_result adjust_display_pll_v3(
 
 		if (pixel_clk_10_kHz_in != 0) {
 			bp_params->adjusted_pixel_clock =
-					div_u64(pixel_clk * pixel_clk_10_khz_out,
-							pixel_clk_10_kHz_in);
+					(uint32_t)div_u64(pixel_clk * pixel_clk_10_khz_out,
+							  pixel_clk_10_kHz_in);
 		} else {
 			bp_params->adjusted_pixel_clock = 0;
 			BREAK_TO_DEBUGGER();
@@ -2662,8 +2662,8 @@ static enum bp_result set_dce_clock_v2_1(
 			!cmd->dc_clock_type_to_atom(bp_params->clock_type, &atom_clock_type))
 		return BP_RESULT_BADINPUT;
 
-	params.asParam.ucDCEClkSrc  = atom_pll_id;
-	params.asParam.ucDCEClkType = atom_clock_type;
+	params.asParam.ucDCEClkSrc = (uint8_t)atom_pll_id;
+	params.asParam.ucDCEClkType = (uint8_t)atom_clock_type;
 
 	if (bp_params->clock_type == DCECLOCK_TYPE_DPREFCLK) {
 		if (bp_params->flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK)
diff --git a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
index 17ef515c6c693..88625daf5378e 100644
--- a/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
+++ b/drivers/gpu/drm/amd/display/dc/bios/command_table2.c
@@ -929,8 +929,8 @@ static enum bp_result set_dce_clock_v2_1(
 					&atom_clock_type))
 		return BP_RESULT_BADINPUT;
 
-	params.param.dceclksrc  = atom_pll_id;
-	params.param.dceclktype = atom_clock_type;
+	params.param.dceclksrc = (uint8_t)atom_pll_id;
+	params.param.dceclktype = (uint8_t)atom_clock_type;
 
 	if (bp_params->clock_type == DCECLOCK_TYPE_DPREFCLK) {
 		if (bp_params->flags.USE_GENLOCK_AS_SOURCE_FOR_DPREFCLK)
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
index 6d41df52d7c9b..808e24f0e88fd 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c
@@ -88,7 +88,7 @@ static const struct state_dependent_clocks dce80_max_clks_by_state[] = {
 /* ClocksStatePerformance */
 { .display_clk_khz = 625000, .pixel_clk_khz = 400000 } };
 
-int dentist_get_divider_from_did(int did)
+unsigned int dentist_get_divider_from_did(unsigned int did)
 {
 	if (did < DENTIST_BASE_DID_1)
 		did = DENTIST_BASE_DID_1;
@@ -155,8 +155,8 @@ static int dce60_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
 int dce_get_dp_ref_freq_khz(struct clk_mgr *clk_mgr_base)
 {
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-	int dprefclk_wdivider;
-	int dprefclk_src_sel;
+	uint32_t dprefclk_wdivider;
+	uint32_t dprefclk_src_sel;
 	int dp_ref_clk_khz;
 	int target_div;
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.h
index f6622f58f62eb..9ea1b0a9923da 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.h
@@ -54,6 +54,6 @@ int dce_set_clock(
 
 void dce_clk_mgr_destroy(struct clk_mgr **clk_mgr);
 
-int dentist_get_divider_from_did(int did);
+unsigned int dentist_get_divider_from_did(unsigned int did);
 
 #endif /* _DCE_CLK_MGR_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c
index cd4c455166164..b35a44976477a 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce110/dce110_clk_mgr.c
@@ -63,11 +63,11 @@ static const struct state_dependent_clocks dce110_max_clks_by_state[] = {
 /*ClocksStatePerformance*/
 { .display_clk_khz = 643000, .pixel_clk_khz = 400000 } };
 
-static int determine_sclk_from_bounding_box(
+static uint32_t determine_sclk_from_bounding_box(
 		const struct dc *dc,
-		int required_sclk)
+		uint32_t required_sclk)
 {
-	int i;
+	uint32_t i;
 
 	/*
 	 * Some asics do not give us sclk levels, so we just report the actual
@@ -127,7 +127,7 @@ void dce110_fill_display_configs(
 	pp_display_cfg->avail_mclk_switch_time_us = dce110_get_min_vblank_time_us(context);
 	pp_display_cfg->disp_clk_khz = dc->clk_mgr->clks.dispclk_khz;
 	pp_display_cfg->avail_mclk_switch_time_in_disp_active_us = 0;
-	pp_display_cfg->crtc_index = dc->res_pool->res_cap->num_timing_generator;
+	pp_display_cfg->crtc_index = (uint8_t)dc->res_pool->res_cap->num_timing_generator;
 
 	for (j = 0; j < context->stream_count; j++) {
 		int k;
@@ -151,7 +151,7 @@ void dce110_fill_display_configs(
 
 		num_cfgs++;
 		cfg->signal = pipe_ctx->stream->signal;
-		cfg->pipe_idx = pipe_ctx->stream_res.tg->inst;
+		cfg->pipe_idx = (uint8_t)pipe_ctx->stream_res.tg->inst;
 		cfg->src_height = stream->src.height;
 		cfg->src_width = stream->src.width;
 		cfg->ddi_channel_mapping =
@@ -189,7 +189,7 @@ void dce110_fill_display_configs(
 		pp_display_cfg->line_time_in_us = 0;
 	}
 
-	pp_display_cfg->display_count = num_cfgs;
+	pp_display_cfg->display_count = (uint8_t)num_cfgs;
 }
 
 void dce11_pplib_apply_display_requirements(
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
index 369421e46c52d..06a51f47aed78 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn10/rv1_clk_mgr.c
@@ -88,7 +88,7 @@ static void ramp_up_dispclk_with_dpp(
 		struct dc_clocks *new_clocks,
 		bool safe_to_lower)
 {
-	int i;
+	uint32_t i;
 	int dispclk_to_dpp_threshold = rv1_determine_dppclk_threshold(clk_mgr, new_clocks);
 	bool request_dpp_div = new_clocks->dispclk_khz > new_clocks->dppclk_khz;
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
index bb4f3bd7532ef..cbd989b6a3df0 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn20/dcn20_clk_mgr.c
@@ -104,7 +104,7 @@ uint32_t dentist_get_did_from_divider(int divider)
 void dcn20_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
 		struct dc_state *context, bool safe_to_lower)
 {
-	int i;
+	uint32_t i;
 
 	clk_mgr->dccg->ref_dppclk = clk_mgr->base.clks.dppclk_khz;
 	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
@@ -255,9 +255,12 @@ void dcn2_update_clocks(struct clk_mgr *clk_mgr_base,
 			pp_smu->set_display_count(&pp_smu->pp_smu, display_count);
 	}
 
-	if (dc->debug.force_min_dcfclk_mhz > 0)
-		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ?
-				new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000);
+	if (dc->debug.force_min_dcfclk_mhz > 0) {
+		int force_min_dcfclk_khz = dc->debug.force_min_dcfclk_mhz * 1000;
+
+		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > force_min_dcfclk_khz) ?
+				new_clocks->dcfclk_khz : force_min_dcfclk_khz;
+	}
 
 	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz)) {
 		clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz;
@@ -427,8 +430,8 @@ void dcn2_read_clocks_from_hw_dentist(struct clk_mgr *clk_mgr_base)
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	uint32_t dispclk_wdivider;
 	uint32_t dppclk_wdivider;
-	int disp_divider;
-	int dpp_divider;
+	unsigned int disp_divider;
+	unsigned int dpp_divider;
 
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DPPCLK_WDIVIDER, &dppclk_wdivider);
@@ -531,8 +534,8 @@ void dcn20_clk_mgr_construct(
 		struct pp_smu_funcs *pp_smu,
 		struct dccg *dccg)
 {
-	int dprefclk_did;
-	int target_div;
+	unsigned int dprefclk_did;
+	unsigned int target_div;
 	uint32_t pll_req_reg;
 	struct fixed31_32 pll_req;
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
index 09e83097a623b..b5378344d2bc0 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c
@@ -106,7 +106,7 @@ static void rn_set_low_power_state(struct clk_mgr *clk_mgr_base)
 static void rn_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
 		struct dc_state *context, int ref_dpp_clk, bool safe_to_lower)
 {
-	int i;
+	uint32_t i;
 
 	clk_mgr->dccg->ref_dppclk = ref_dpp_clk;
 
@@ -462,8 +462,10 @@ static void build_watermark_ranges(struct clk_bw_params *bw_params, struct pp_sm
 		if (!bw_params->wm_table.entries[i].valid)
 			continue;
 
-		ranges->reader_wm_sets[num_valid_sets].wm_inst = bw_params->wm_table.entries[i].wm_inst;
-		ranges->reader_wm_sets[num_valid_sets].wm_type = bw_params->wm_table.entries[i].wm_type;
+		ranges->reader_wm_sets[num_valid_sets].wm_inst =
+			(uint8_t)bw_params->wm_table.entries[i].wm_inst;
+		ranges->reader_wm_sets[num_valid_sets].wm_type =
+			(uint8_t)bw_params->wm_table.entries[i].wm_type;
 		/* We will not select WM based on fclk, so leave it as unconstrained */
 		ranges->reader_wm_sets[num_valid_sets].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
 		ranges->reader_wm_sets[num_valid_sets].max_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
@@ -474,9 +476,11 @@ static void build_watermark_ranges(struct clk_bw_params *bw_params, struct pp_sm
 				ranges->reader_wm_sets[num_valid_sets].min_drain_clk_mhz = 0;
 			else {
 				/* add 1 to make it non-overlapping with next lvl */
-				ranges->reader_wm_sets[num_valid_sets].min_drain_clk_mhz = bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1;
+				ranges->reader_wm_sets[num_valid_sets].min_drain_clk_mhz =
+					(uint16_t)(bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1);
 			}
-			ranges->reader_wm_sets[num_valid_sets].max_drain_clk_mhz = bw_params->clk_table.entries[i].dcfclk_mhz;
+			ranges->reader_wm_sets[num_valid_sets].max_drain_clk_mhz =
+				(uint16_t)bw_params->clk_table.entries[i].dcfclk_mhz;
 
 		} else {
 			/* unconstrained for memory retraining */
@@ -640,6 +644,8 @@ static unsigned int find_dcfclk_for_voltage(struct dpm_clocks *clock_table, unsi
 static void rn_clk_mgr_helper_populate_bw_params(struct clk_bw_params *bw_params, struct dpm_clocks *clock_table, struct integrated_info *bios_info)
 {
 	int i, j = 0;
+	unsigned int entry_idx;
+	unsigned int wm_idx;
 
 	j = -1;
 
@@ -663,28 +669,28 @@ static void rn_clk_mgr_helper_populate_bw_params(struct clk_bw_params *bw_params
 
 	bw_params->clk_table.num_entries = j + 1;
 
-	for (i = 0; i < bw_params->clk_table.num_entries; i++, j--) {
-		bw_params->clk_table.entries[i].fclk_mhz = clock_table->FClocks[j].Freq;
-		bw_params->clk_table.entries[i].memclk_mhz = clock_table->MemClocks[j].Freq;
-		bw_params->clk_table.entries[i].voltage = clock_table->FClocks[j].Vol;
-		bw_params->clk_table.entries[i].dcfclk_mhz = find_dcfclk_for_voltage(clock_table, clock_table->FClocks[j].Vol);
-		bw_params->clk_table.entries[i].socclk_mhz = find_socclk_for_voltage(clock_table,
-									bw_params->clk_table.entries[i].voltage);
+	for (entry_idx = 0; entry_idx < bw_params->clk_table.num_entries; entry_idx++, j--) {
+		bw_params->clk_table.entries[entry_idx].fclk_mhz = clock_table->FClocks[j].Freq;
+		bw_params->clk_table.entries[entry_idx].memclk_mhz = clock_table->MemClocks[j].Freq;
+		bw_params->clk_table.entries[entry_idx].voltage = clock_table->FClocks[j].Vol;
+		bw_params->clk_table.entries[entry_idx].dcfclk_mhz = find_dcfclk_for_voltage(clock_table, clock_table->FClocks[j].Vol);
+		bw_params->clk_table.entries[entry_idx].socclk_mhz = find_socclk_for_voltage(clock_table,
+									bw_params->clk_table.entries[entry_idx].voltage);
 	}
 
 	bw_params->vram_type = bios_info->memory_type;
 	bw_params->num_channels = bios_info->ma_channel_number;
 
-	for (i = 0; i < WM_SET_COUNT; i++) {
-		bw_params->wm_table.entries[i].wm_inst = i;
+	for (wm_idx = 0; wm_idx < (unsigned int)WM_SET_COUNT; wm_idx++) {
+		bw_params->wm_table.entries[wm_idx].wm_inst = wm_idx;
 
-		if (i >= bw_params->clk_table.num_entries) {
-			bw_params->wm_table.entries[i].valid = false;
+		if (wm_idx >= bw_params->clk_table.num_entries) {
+			bw_params->wm_table.entries[wm_idx].valid = false;
 			continue;
 		}
 
-		bw_params->wm_table.entries[i].wm_type = WM_TYPE_PSTATE_CHG;
-		bw_params->wm_table.entries[i].valid = true;
+		bw_params->wm_table.entries[wm_idx].wm_type = WM_TYPE_PSTATE_CHG;
+		bw_params->wm_table.entries[wm_idx].valid = true;
 	}
 
 	if (bw_params->vram_type == LpDdr4MemType) {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
index dcec9d0f8c340..5641f9089b0b2 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr.c
@@ -78,9 +78,9 @@ static const struct clk_mgr_mask clk_mgr_mask = {
 
 
 /* Query SMU for all clock states for a particular clock */
-static void dcn3_init_single_clock(struct clk_mgr_internal *clk_mgr, uint32_t clk, unsigned int *entry_0, unsigned int *num_levels)
+static void dcn3_init_single_clock(struct clk_mgr_internal *clk_mgr, uint32_t clk, unsigned int *entry_0, uint8_t *num_levels)
 {
-	unsigned int i;
+	uint8_t i;
 	char *entry_i = (char *)entry_0;
 	uint32_t ret = dcn30_smu_get_dpm_freq_by_index(clk_mgr, clk, 0xFF);
 
@@ -109,7 +109,7 @@ static void dcn3_build_wm_range_table(struct clk_mgr_internal *clk_mgr)
 void dcn3_init_clocks(struct clk_mgr *clk_mgr_base)
 {
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-	unsigned int num_levels;
+	uint8_t num_levels;
 
 	memset(&(clk_mgr_base->clks), 0, sizeof(struct dc_clocks));
 	clk_mgr_base->clks.p_state_change_support = true;
@@ -228,13 +228,16 @@ static void dcn3_update_clocks(struct clk_mgr *clk_mgr_base,
 	if (enter_display_off == safe_to_lower)
 		dcn30_smu_set_num_of_displays(clk_mgr, display_count);
 
-	if (dc->debug.force_min_dcfclk_mhz > 0)
-		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ?
-				new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000);
+	if (dc->debug.force_min_dcfclk_mhz > 0) {
+		int force_min_dcfclk_khz = dc->debug.force_min_dcfclk_mhz * 1000;
+
+		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > force_min_dcfclk_khz) ?
+				new_clocks->dcfclk_khz : force_min_dcfclk_khz;
+	}
 
 	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz)) {
 		clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz;
-		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DCEFCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dcfclk_khz));
+		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DCEFCLK, (uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dcfclk_khz));
 	}
 
 	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_deep_sleep_khz, clk_mgr_base->clks.dcfclk_deep_sleep_khz)) {
@@ -251,8 +254,10 @@ static void dcn3_update_clocks(struct clk_mgr *clk_mgr_base,
 
 	// invalidate the current P-State forced min in certain dc_mode_softmax situations
 	if (dc->clk_mgr->dc_mode_softmax_enabled && safe_to_lower && !p_state_change_support) {
-		if ((new_clocks->dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000) !=
-				(clk_mgr_base->clks.dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000))
+		int softmax_memclk_khz = dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000;
+
+		if ((new_clocks->dramclk_khz <= softmax_memclk_khz) !=
+				(clk_mgr_base->clks.dramclk_khz <= softmax_memclk_khz))
 			update_pstate_unsupported_clk = true;
 	}
 
@@ -263,12 +268,13 @@ static void dcn3_update_clocks(struct clk_mgr *clk_mgr_base,
 		/* to disable P-State switching, set UCLK min = max */
 		if (!clk_mgr_base->clks.p_state_change_support) {
 			if (dc->clk_mgr->dc_mode_softmax_enabled &&
-				new_clocks->dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000)
+				new_clocks->dramclk_khz <= (int)(dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000))
 				dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-					dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
+					(uint16_t)dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
 			else
 				dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-					clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz);
+					(uint16_t)clk_mgr_base->bw_params->clk_table.entries[
+						clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz);
 		}
 	}
 
@@ -281,20 +287,20 @@ static void dcn3_update_clocks(struct clk_mgr *clk_mgr_base,
 	/* set UCLK to requested value if P-State switching is supported, or to re-enable P-State switching */
 	if (clk_mgr_base->clks.p_state_change_support &&
 			(update_uclk || !clk_mgr_base->clks.prev_p_state_change_support))
-		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
+		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
 
 	if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr_base->clks.dppclk_khz)) {
 		if (clk_mgr_base->clks.dppclk_khz > new_clocks->dppclk_khz)
 			dpp_clock_lowered = true;
 
 		clk_mgr_base->clks.dppclk_khz = new_clocks->dppclk_khz;
-		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_PIXCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dppclk_khz));
+		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_PIXCLK, (uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dppclk_khz));
 		update_dppclk = true;
 	}
 
 	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz)) {
 		clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz;
-		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DISPCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dispclk_khz));
+		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DISPCLK, (uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dispclk_khz));
 		update_dispclk = true;
 	}
 
@@ -323,7 +329,7 @@ static void dcn3_update_clocks(struct clk_mgr *clk_mgr_base,
 
 static void dcn3_notify_wm_ranges(struct clk_mgr *clk_mgr_base)
 {
-	unsigned int i;
+	uint8_t i;
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	WatermarksExternal_t *table = (WatermarksExternal_t *) clk_mgr->wm_range_table;
 
@@ -363,13 +369,14 @@ static void dcn3_set_hard_min_memclk(struct clk_mgr *clk_mgr_base, bool current_
 	if (current_mode) {
 		if (clk_mgr_base->clks.p_state_change_support)
 			dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-					khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
+					(uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
 		else
 			dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-					clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz);
+					(uint16_t)clk_mgr_base->bw_params->clk_table.entries[
+							clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz);
 	} else {
 		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-				clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz);
+				(uint16_t)clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz);
 	}
 }
 
@@ -382,7 +389,8 @@ static void dcn3_set_hard_max_memclk(struct clk_mgr *clk_mgr_base)
 		return;
 
 	dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK,
-			clk_mgr_base->bw_params->clk_table.entries[clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz);
+			(uint16_t)clk_mgr_base->bw_params->clk_table.entries[
+				clk_mgr_base->bw_params->clk_table.num_entries - 1].memclk_mhz);
 }
 
 static void dcn3_set_max_memclk(struct clk_mgr *clk_mgr_base, unsigned int memclk_mhz)
@@ -392,7 +400,7 @@ static void dcn3_set_max_memclk(struct clk_mgr *clk_mgr_base, unsigned int memcl
 	if (!clk_mgr->smu_present)
 		return;
 
-	dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, memclk_mhz);
+	dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)memclk_mhz);
 }
 static void dcn3_set_min_memclk(struct clk_mgr *clk_mgr_base, unsigned int memclk_mhz)
 {
@@ -400,14 +408,14 @@ static void dcn3_set_min_memclk(struct clk_mgr *clk_mgr_base, unsigned int memcl
 
 	if (!clk_mgr->smu_present)
 		return;
-	dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, memclk_mhz);
+	dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)memclk_mhz);
 }
 
 /* Get current memclk states, update bounding box */
 static void dcn3_get_memclk_states_from_smu(struct clk_mgr *clk_mgr_base)
 {
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
-	unsigned int num_levels;
+	uint8_t num_levels;
 
 	if (!clk_mgr->smu_present)
 		return;
@@ -480,7 +488,7 @@ static void dcn30_notify_link_rate_change(struct clk_mgr *clk_mgr_base, struct d
 
 	if (max_phyclk_req != clk_mgr_base->clks.phyclk_khz) {
 		clk_mgr_base->clks.phyclk_khz = max_phyclk_req;
-		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_PHYCLK, khz_to_mhz_ceil(clk_mgr_base->clks.phyclk_khz));
+		dcn30_smu_set_hard_min_by_freq(clk_mgr, PPCLK_PHYCLK, (uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.phyclk_khz));
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.c
index 827bc2431d5d8..e36233127b15c 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.c
@@ -123,12 +123,12 @@ bool dcn30_smu_test_message(struct clk_mgr_internal *clk_mgr, uint32_t input)
 	return false;
 }
 
-bool dcn30_smu_get_smu_version(struct clk_mgr_internal *clk_mgr, unsigned int *version)
+bool dcn30_smu_get_smu_version(struct clk_mgr_internal *clk_mgr, int *version)
 {
 	smu_print("SMU Get SMU version\n");
 
 	if (dcn30_smu_send_msg_with_param(clk_mgr,
-			DALSMC_MSG_GetSmuVersion, 0, version)) {
+			DALSMC_MSG_GetSmuVersion, 0, (uint32_t *)version)) {
 
 		smu_print("SMU version: %d\n", *version);
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.h
index ca9f5296be94a..67e93f9cac719 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn30/dcn30_clk_mgr_smu_msg.h
@@ -31,7 +31,7 @@
 struct clk_mgr_internal;
 
 bool         dcn30_smu_test_message(struct clk_mgr_internal *clk_mgr, uint32_t input);
-bool         dcn30_smu_get_smu_version(struct clk_mgr_internal *clk_mgr, unsigned int *version);
+bool         dcn30_smu_get_smu_version(struct clk_mgr_internal *clk_mgr, int *version);
 bool         dcn30_smu_check_driver_if_version(struct clk_mgr_internal *clk_mgr);
 bool         dcn30_smu_check_msg_header_version(struct clk_mgr_internal *clk_mgr);
 void         dcn30_smu_set_dram_addr_high(struct clk_mgr_internal *clk_mgr, uint32_t addr_high);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
index 57ba7bc4d16e6..e40bbc495fc43 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn301/vg_clk_mgr.c
@@ -385,7 +385,7 @@ static void vg_init_clocks(struct clk_mgr *clk_mgr)
 
 static void vg_build_watermark_ranges(struct clk_bw_params *bw_params, struct watermarks *table)
 {
-	int i, num_valid_sets;
+	uint8_t i, num_valid_sets;
 
 	num_valid_sets = 0;
 
@@ -394,8 +394,11 @@ static void vg_build_watermark_ranges(struct clk_bw_params *bw_params, struct wa
 		if (!bw_params->wm_table.entries[i].valid)
 			continue;
 
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting = bw_params->wm_table.entries[i].wm_inst;
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType = bw_params->wm_table.entries[i].wm_type;
+
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting =
+			(uint8_t)bw_params->wm_table.entries[i].wm_inst;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType =
+			(uint8_t)bw_params->wm_table.entries[i].wm_type;
 		/* We will not select WM based on fclk, so leave it as unconstrained */
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinClock = 0;
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxClock = 0xFFFF;
@@ -406,10 +409,10 @@ static void vg_build_watermark_ranges(struct clk_bw_params *bw_params, struct wa
 			else {
 				/* add 1 to make it non-overlapping with next lvl */
 				table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinMclk =
-						bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1;
+						(uint16_t)(bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1);
 			}
 			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxMclk =
-					bw_params->clk_table.entries[i].dcfclk_mhz;
+					(uint16_t)bw_params->clk_table.entries[i].dcfclk_mhz;
 
 		} else {
 			/* unconstrained for memory retraining */
@@ -530,7 +533,7 @@ static struct clk_bw_params vg_bw_params = {
 static uint32_t find_max_clk_value(const uint32_t clocks[], uint32_t num_clocks)
 {
 	uint32_t max = 0;
-	int i;
+	uint32_t i;
 
 	for (i = 0; i < num_clocks; ++i) {
 		if (clocks[i] > max)
@@ -562,6 +565,7 @@ static void vg_clk_mgr_helper_populate_bw_params(
 		const struct vg_dpm_clocks *clock_table)
 {
 	int i, j;
+	unsigned int entry_idx;
 	struct clk_bw_params *bw_params = clk_mgr->base.bw_params;
 	uint32_t max_dispclk = 0, max_dppclk = 0;
 
@@ -596,16 +600,17 @@ static void vg_clk_mgr_helper_populate_bw_params(
 
 	bw_params->clk_table.num_entries = j + 1;
 
-	for (i = 0; i < bw_params->clk_table.num_entries - 1; i++, j--) {
-		bw_params->clk_table.entries[i].fclk_mhz = clock_table->DfPstateTable[j].fclk;
-		bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[j].memclk;
-		bw_params->clk_table.entries[i].voltage = clock_table->DfPstateTable[j].voltage;
-		bw_params->clk_table.entries[i].dcfclk_mhz = find_dcfclk_for_voltage(clock_table, clock_table->DfPstateTable[j].voltage);
+	for (entry_idx = 0; entry_idx < bw_params->clk_table.num_entries - 1; entry_idx++, j--) {
+		bw_params->clk_table.entries[entry_idx].fclk_mhz = clock_table->DfPstateTable[j].fclk;
+		bw_params->clk_table.entries[entry_idx].memclk_mhz = clock_table->DfPstateTable[j].memclk;
+		bw_params->clk_table.entries[entry_idx].voltage = clock_table->DfPstateTable[j].voltage;
+		bw_params->clk_table.entries[entry_idx].dcfclk_mhz = find_dcfclk_for_voltage(clock_table, clock_table->DfPstateTable[j].voltage);
 
 		/* Now update clocks we do read */
-		bw_params->clk_table.entries[i].dispclk_mhz = max_dispclk;
-		bw_params->clk_table.entries[i].dppclk_mhz = max_dppclk;
+		bw_params->clk_table.entries[entry_idx].dispclk_mhz = max_dispclk;
+		bw_params->clk_table.entries[entry_idx].dppclk_mhz = max_dppclk;
 	}
+	i = (int)entry_idx;
 	bw_params->clk_table.entries[i].fclk_mhz = clock_table->DfPstateTable[j].fclk;
 	bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[j].memclk;
 	bw_params->clk_table.entries[i].voltage = clock_table->DfPstateTable[j].voltage;
@@ -616,16 +621,16 @@ static void vg_clk_mgr_helper_populate_bw_params(
 	bw_params->vram_type = bios_info->memory_type;
 	bw_params->num_channels = bios_info->ma_channel_number;
 
-	for (i = 0; i < WM_SET_COUNT; i++) {
-		bw_params->wm_table.entries[i].wm_inst = i;
+	for (entry_idx = 0; entry_idx < (unsigned int)WM_SET_COUNT; entry_idx++) {
+		bw_params->wm_table.entries[entry_idx].wm_inst = entry_idx;
 
-		if (i >= bw_params->clk_table.num_entries) {
-			bw_params->wm_table.entries[i].valid = false;
+		if (entry_idx >= bw_params->clk_table.num_entries) {
+			bw_params->wm_table.entries[entry_idx].valid = false;
 			continue;
 		}
 
-		bw_params->wm_table.entries[i].wm_type = WM_TYPE_PSTATE_CHG;
-		bw_params->wm_table.entries[i].valid = true;
+		bw_params->wm_table.entries[entry_idx].wm_type = WM_TYPE_PSTATE_CHG;
+		bw_params->wm_table.entries[entry_idx].valid = true;
 	}
 
 	if (bw_params->vram_type == LpDdr4MemType) {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
index 89fc482947eff..dc7f50095a13c 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn31/dcn31_clk_mgr.c
@@ -114,7 +114,7 @@ static int dcn31_get_active_display_cnt_wa(
 static void dcn31_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state *context, bool disable)
 {
 	struct dc *dc = clk_mgr_base->ctx->dc;
-	int i;
+	uint8_t i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; ++i) {
 		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
@@ -424,7 +424,7 @@ static struct dcn31_watermarks dummy_wms = { 0 };
 
 static void dcn31_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn31_watermarks *table)
 {
-	int i, num_valid_sets;
+	uint8_t i, num_valid_sets;
 
 	num_valid_sets = 0;
 
@@ -433,8 +433,10 @@ static void dcn31_build_watermark_ranges(struct clk_bw_params *bw_params, struct
 		if (!bw_params->wm_table.entries[i].valid)
 			continue;
 
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting = bw_params->wm_table.entries[i].wm_inst;
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType = bw_params->wm_table.entries[i].wm_type;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting =
+			(uint8_t)bw_params->wm_table.entries[i].wm_inst;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType =
+			(uint8_t)bw_params->wm_table.entries[i].wm_type;
 		/* We will not select WM based on fclk, so leave it as unconstrained */
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinClock = 0;
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxClock = 0xFFFF;
@@ -445,10 +447,10 @@ static void dcn31_build_watermark_ranges(struct clk_bw_params *bw_params, struct
 			else {
 				/* add 1 to make it non-overlapping with next lvl */
 				table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinMclk =
-						bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1;
+						(uint16_t)(bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1);
 			}
 			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxMclk =
-					bw_params->clk_table.entries[i].dcfclk_mhz;
+					(uint16_t)bw_params->clk_table.entries[i].dcfclk_mhz;
 
 		} else {
 			/* unconstrained for memory retraining */
@@ -523,7 +525,7 @@ static void dcn31_get_dpm_table_from_smu(struct clk_mgr_internal *clk_mgr,
 static uint32_t find_max_clk_value(const uint32_t clocks[], uint32_t num_clocks)
 {
 	uint32_t max = 0;
-	int i;
+	uint32_t i;
 
 	for (i = 0; i < num_clocks; ++i) {
 		if (clocks[i] > max)
@@ -539,8 +541,8 @@ static unsigned int find_clk_for_voltage(
 		unsigned int voltage)
 {
 	int i;
-	int max_voltage = 0;
-	int clock = 0;
+	unsigned int max_voltage = 0;
+	unsigned int clock = 0;
 
 	for (i = 0; i < NUM_SOC_VOLTAGE_LEVELS; i++) {
 		if (clock_table->SocVoltage[i] == voltage) {
@@ -561,6 +563,7 @@ static void dcn31_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *clk
 						    const DpmClocks_t *clock_table)
 {
 	int i, j;
+	unsigned int entry_idx;
 	struct clk_bw_params *bw_params = clk_mgr->base.bw_params;
 	uint32_t max_dispclk = 0, max_dppclk = 0;
 
@@ -595,24 +598,24 @@ static void dcn31_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *clk
 		ASSERT(0);
 	}
 
-	for (i = 0; i < bw_params->clk_table.num_entries; i++, j--) {
-		bw_params->clk_table.entries[i].fclk_mhz = clock_table->DfPstateTable[j].FClk;
-		bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[j].MemClk;
-		bw_params->clk_table.entries[i].voltage = clock_table->DfPstateTable[j].Voltage;
+	for (entry_idx = 0; entry_idx < bw_params->clk_table.num_entries; entry_idx++, j--) {
+		bw_params->clk_table.entries[entry_idx].fclk_mhz = clock_table->DfPstateTable[j].FClk;
+		bw_params->clk_table.entries[entry_idx].memclk_mhz = clock_table->DfPstateTable[j].MemClk;
+		bw_params->clk_table.entries[entry_idx].voltage = clock_table->DfPstateTable[j].Voltage;
 		switch (clock_table->DfPstateTable[j].WckRatio) {
 		case WCK_RATIO_1_2:
-			bw_params->clk_table.entries[i].wck_ratio = 2;
+			bw_params->clk_table.entries[entry_idx].wck_ratio = 2;
 			break;
 		case WCK_RATIO_1_4:
-			bw_params->clk_table.entries[i].wck_ratio = 4;
+			bw_params->clk_table.entries[entry_idx].wck_ratio = 4;
 			break;
 		default:
-			bw_params->clk_table.entries[i].wck_ratio = 1;
+			bw_params->clk_table.entries[entry_idx].wck_ratio = 1;
 		}
-		bw_params->clk_table.entries[i].dcfclk_mhz = find_clk_for_voltage(clock_table, clock_table->DcfClocks, clock_table->DfPstateTable[j].Voltage);
-		bw_params->clk_table.entries[i].socclk_mhz = find_clk_for_voltage(clock_table, clock_table->SocClocks, clock_table->DfPstateTable[j].Voltage);
-		bw_params->clk_table.entries[i].dispclk_mhz = max_dispclk;
-		bw_params->clk_table.entries[i].dppclk_mhz = max_dppclk;
+		bw_params->clk_table.entries[entry_idx].dcfclk_mhz = find_clk_for_voltage(clock_table, clock_table->DcfClocks, clock_table->DfPstateTable[j].Voltage);
+		bw_params->clk_table.entries[entry_idx].socclk_mhz = find_clk_for_voltage(clock_table, clock_table->SocClocks, clock_table->DfPstateTable[j].Voltage);
+		bw_params->clk_table.entries[entry_idx].dispclk_mhz = max_dispclk;
+		bw_params->clk_table.entries[entry_idx].dppclk_mhz = max_dppclk;
 	}
 
 	bw_params->vram_type = bios_info->memory_type;
@@ -620,16 +623,16 @@ static void dcn31_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *clk
 	bw_params->dram_channel_width_bytes = bios_info->memory_type == 0x22 ? 8 : 4;
 	//bw_params->dram_channel_width_bytes = dc->ctx->asic_id.vram_width;
 	bw_params->num_channels = bios_info->ma_channel_number ? bios_info->ma_channel_number : 4;
-	for (i = 0; i < WM_SET_COUNT; i++) {
-		bw_params->wm_table.entries[i].wm_inst = i;
+	for (entry_idx = 0; entry_idx < (unsigned int)WM_SET_COUNT; entry_idx++) {
+		bw_params->wm_table.entries[entry_idx].wm_inst = entry_idx;
 
-		if (i >= bw_params->clk_table.num_entries) {
-			bw_params->wm_table.entries[i].valid = false;
+		if (entry_idx >= bw_params->clk_table.num_entries) {
+			bw_params->wm_table.entries[entry_idx].valid = false;
 			continue;
 		}
 
-		bw_params->wm_table.entries[i].wm_type = WM_TYPE_PSTATE_CHG;
-		bw_params->wm_table.entries[i].valid = true;
+		bw_params->wm_table.entries[entry_idx].wm_type = WM_TYPE_PSTATE_CHG;
+		bw_params->wm_table.entries[entry_idx].valid = true;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
index b08a70a2f571d..5fe59adc862fe 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn314/dcn314_clk_mgr.c
@@ -149,7 +149,7 @@ static void dcn314_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state
 				  bool safe_to_lower, bool disable)
 {
 	struct dc *dc = clk_mgr_base->ctx->dc;
-	int i;
+	uint8_t i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; ++i) {
 		struct pipe_ctx *pipe = safe_to_lower
@@ -295,7 +295,7 @@ void dcn314_update_clocks(struct clk_mgr *clk_mgr_base,
 
 	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) &&
 		(new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) {
-		int requested_dispclk_khz = new_clocks->dispclk_khz;
+		uint32_t requested_dispclk_khz = new_clocks->dispclk_khz;
 
 		dcn314_disable_otg_wa(clk_mgr_base, context, safe_to_lower, true);
 
@@ -495,7 +495,7 @@ static struct dcn314_ss_info_table ss_info_table = {
 
 static void dcn314_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn314_watermarks *table)
 {
-	int i, num_valid_sets;
+	uint8_t i, num_valid_sets;
 
 	num_valid_sets = 0;
 
@@ -504,8 +504,10 @@ static void dcn314_build_watermark_ranges(struct clk_bw_params *bw_params, struc
 		if (!bw_params->wm_table.entries[i].valid)
 			continue;
 
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting = bw_params->wm_table.entries[i].wm_inst;
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType = bw_params->wm_table.entries[i].wm_type;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting =
+			(uint8_t)bw_params->wm_table.entries[i].wm_inst;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType =
+			(uint8_t)bw_params->wm_table.entries[i].wm_type;
 		/* We will not select WM based on fclk, so leave it as unconstrained */
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinClock = 0;
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxClock = 0xFFFF;
@@ -516,10 +518,10 @@ static void dcn314_build_watermark_ranges(struct clk_bw_params *bw_params, struc
 			else {
 				/* add 1 to make it non-overlapping with next lvl */
 				table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinMclk =
-						bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1;
+						(uint16_t)(bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1);
 			}
 			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxMclk =
-					bw_params->clk_table.entries[i].dcfclk_mhz;
+					(uint16_t)bw_params->clk_table.entries[i].dcfclk_mhz;
 
 		} else {
 			/* unconstrained for memory retraining */
@@ -614,7 +616,7 @@ static unsigned int convert_wck_ratio(uint8_t wck_ratio)
 static uint32_t find_max_clk_value(const uint32_t clocks[], uint32_t num_clocks)
 {
 	uint32_t max = 0;
-	int i;
+	uint32_t i;
 
 	for (i = 0; i < num_clocks; ++i) {
 		if (clocks[i] > max)
@@ -632,6 +634,7 @@ static void dcn314_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *cl
 	struct clk_limit_table_entry def_max = bw_params->clk_table.entries[bw_params->clk_table.num_entries - 1];
 	uint32_t max_pstate = 0,  max_fclk = 0,  min_pstate = 0, max_dispclk = 0, max_dppclk = 0;
 	int i;
+	unsigned int entry_idx;
 
 	/* Find highest valid fclk pstate */
 	for (i = 0; i < clock_table->NumDfPstatesEnabled; i++) {
@@ -721,26 +724,26 @@ static void dcn314_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *cl
 	 * Set any 0 clocks to max default setting. Not an issue for
 	 * power since we aren't doing switching in such case anyway
 	 */
-	for (i = 0; i < bw_params->clk_table.num_entries; i++) {
-		if (!bw_params->clk_table.entries[i].fclk_mhz) {
-			bw_params->clk_table.entries[i].fclk_mhz = def_max.fclk_mhz;
-			bw_params->clk_table.entries[i].memclk_mhz = def_max.memclk_mhz;
-			bw_params->clk_table.entries[i].voltage = def_max.voltage;
+	for (entry_idx = 0; entry_idx < bw_params->clk_table.num_entries; entry_idx++) {
+		if (!bw_params->clk_table.entries[entry_idx].fclk_mhz) {
+			bw_params->clk_table.entries[entry_idx].fclk_mhz = def_max.fclk_mhz;
+			bw_params->clk_table.entries[entry_idx].memclk_mhz = def_max.memclk_mhz;
+			bw_params->clk_table.entries[entry_idx].voltage = def_max.voltage;
 		}
-		if (!bw_params->clk_table.entries[i].dcfclk_mhz)
-			bw_params->clk_table.entries[i].dcfclk_mhz = def_max.dcfclk_mhz;
-		if (!bw_params->clk_table.entries[i].socclk_mhz)
-			bw_params->clk_table.entries[i].socclk_mhz = def_max.socclk_mhz;
-		if (!bw_params->clk_table.entries[i].dispclk_mhz)
-			bw_params->clk_table.entries[i].dispclk_mhz = def_max.dispclk_mhz;
-		if (!bw_params->clk_table.entries[i].dppclk_mhz)
-			bw_params->clk_table.entries[i].dppclk_mhz = def_max.dppclk_mhz;
-		if (!bw_params->clk_table.entries[i].phyclk_mhz)
-			bw_params->clk_table.entries[i].phyclk_mhz = def_max.phyclk_mhz;
-		if (!bw_params->clk_table.entries[i].phyclk_d18_mhz)
-			bw_params->clk_table.entries[i].phyclk_d18_mhz = def_max.phyclk_d18_mhz;
-		if (!bw_params->clk_table.entries[i].dtbclk_mhz)
-			bw_params->clk_table.entries[i].dtbclk_mhz = def_max.dtbclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dcfclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dcfclk_mhz = def_max.dcfclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].socclk_mhz)
+			bw_params->clk_table.entries[entry_idx].socclk_mhz = def_max.socclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dispclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dispclk_mhz = def_max.dispclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dppclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dppclk_mhz = def_max.dppclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].phyclk_mhz)
+			bw_params->clk_table.entries[entry_idx].phyclk_mhz = def_max.phyclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].phyclk_d18_mhz)
+			bw_params->clk_table.entries[entry_idx].phyclk_d18_mhz = def_max.phyclk_d18_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dtbclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dtbclk_mhz = def_max.dtbclk_mhz;
 	}
 	ASSERT(bw_params->clk_table.entries[i-1].dcfclk_mhz);
 	bw_params->vram_type = bios_info->memory_type;
@@ -748,16 +751,16 @@ static void dcn314_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *cl
 	bw_params->dram_channel_width_bytes = bios_info->memory_type == 0x22 ? 8 : 4;
 	bw_params->num_channels = bios_info->ma_channel_number ? bios_info->ma_channel_number : 4;
 
-	for (i = 0; i < WM_SET_COUNT; i++) {
-		bw_params->wm_table.entries[i].wm_inst = i;
+	for (entry_idx = 0; entry_idx < (unsigned int)WM_SET_COUNT; entry_idx++) {
+		bw_params->wm_table.entries[entry_idx].wm_inst = entry_idx;
 
-		if (i >= bw_params->clk_table.num_entries) {
-			bw_params->wm_table.entries[i].valid = false;
+		if (entry_idx >= bw_params->clk_table.num_entries) {
+			bw_params->wm_table.entries[entry_idx].valid = false;
 			continue;
 		}
 
-		bw_params->wm_table.entries[i].wm_type = WM_TYPE_PSTATE_CHG;
-		bw_params->wm_table.entries[i].valid = true;
+		bw_params->wm_table.entries[entry_idx].wm_type = WM_TYPE_PSTATE_CHG;
+		bw_params->wm_table.entries[entry_idx].valid = true;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
index 3a651c1a866d9..b6f26475ac164 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn315/dcn315_clk_mgr.c
@@ -100,7 +100,7 @@ static bool should_disable_otg(struct pipe_ctx *pipe)
 static void dcn315_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state *context, bool disable)
 {
 	struct dc *dc = clk_mgr_base->ctx->dc;
-	int i;
+	uint8_t i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; ++i) {
 		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
@@ -203,7 +203,7 @@ static void dcn315_update_clocks(struct clk_mgr *clk_mgr_base,
 
 	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) &&
 	    (new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) {
-		int requested_dispclk_khz = new_clocks->dispclk_khz;
+		uint32_t requested_dispclk_khz = new_clocks->dispclk_khz;
 
 		dcn315_disable_otg_wa(clk_mgr_base, context, true);
 
@@ -384,7 +384,7 @@ static struct dcn315_watermarks dummy_wms = { 0 };
 
 static void dcn315_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn315_watermarks *table)
 {
-	int i, num_valid_sets;
+	uint8_t i, num_valid_sets;
 
 	num_valid_sets = 0;
 
@@ -393,8 +393,11 @@ static void dcn315_build_watermark_ranges(struct clk_bw_params *bw_params, struc
 		if (!bw_params->wm_table.entries[i].valid)
 			continue;
 
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting = bw_params->wm_table.entries[i].wm_inst;
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType = bw_params->wm_table.entries[i].wm_type;
+
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting =
+			(uint8_t)bw_params->wm_table.entries[i].wm_inst;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType =
+			(uint8_t)bw_params->wm_table.entries[i].wm_type;
 		/* We will not select WM based on fclk, so leave it as unconstrained */
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinClock = 0;
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxClock = 0xFFFF;
@@ -405,10 +408,10 @@ static void dcn315_build_watermark_ranges(struct clk_bw_params *bw_params, struc
 			else {
 				/* add 1 to make it non-overlapping with next lvl */
 				table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinMclk =
-						bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1;
+						(uint16_t)(bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1);
 			}
 			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxMclk =
-					bw_params->clk_table.entries[i].dcfclk_mhz;
+					(uint16_t)bw_params->clk_table.entries[i].dcfclk_mhz;
 
 		} else {
 			/* unconstrained for memory retraining */
@@ -486,6 +489,7 @@ static void dcn315_clk_mgr_helper_populate_bw_params(
 		const DpmClocks_315_t *clock_table)
 {
 	int i;
+	unsigned int entry_idx;
 	struct clk_bw_params *bw_params = clk_mgr->base.bw_params;
 	uint32_t max_pstate = clock_table->NumDfPstatesEnabled - 1;
 	struct clk_limit_table_entry def_max = bw_params->clk_table.entries[bw_params->clk_table.num_entries - 1];
@@ -541,26 +545,26 @@ static void dcn315_clk_mgr_helper_populate_bw_params(
 	/* Set any 0 clocks to max default setting. Not an issue for
 	 * power since we aren't doing switching in such case anyway
 	 */
-	for (i = 0; i < bw_params->clk_table.num_entries; i++) {
-		if (!bw_params->clk_table.entries[i].fclk_mhz) {
-			bw_params->clk_table.entries[i].fclk_mhz = def_max.fclk_mhz;
-			bw_params->clk_table.entries[i].memclk_mhz = def_max.memclk_mhz;
-			bw_params->clk_table.entries[i].voltage = def_max.voltage;
+	for (entry_idx = 0; entry_idx < bw_params->clk_table.num_entries; entry_idx++) {
+		if (!bw_params->clk_table.entries[entry_idx].fclk_mhz) {
+			bw_params->clk_table.entries[entry_idx].fclk_mhz = def_max.fclk_mhz;
+			bw_params->clk_table.entries[entry_idx].memclk_mhz = def_max.memclk_mhz;
+			bw_params->clk_table.entries[entry_idx].voltage = def_max.voltage;
 		}
-		if (!bw_params->clk_table.entries[i].dcfclk_mhz)
-			bw_params->clk_table.entries[i].dcfclk_mhz = def_max.dcfclk_mhz;
-		if (!bw_params->clk_table.entries[i].socclk_mhz)
-			bw_params->clk_table.entries[i].socclk_mhz = def_max.socclk_mhz;
-		if (!bw_params->clk_table.entries[i].dispclk_mhz)
-			bw_params->clk_table.entries[i].dispclk_mhz = def_max.dispclk_mhz;
-		if (!bw_params->clk_table.entries[i].dppclk_mhz)
-			bw_params->clk_table.entries[i].dppclk_mhz = def_max.dppclk_mhz;
-		if (!bw_params->clk_table.entries[i].phyclk_mhz)
-			bw_params->clk_table.entries[i].phyclk_mhz = def_max.phyclk_mhz;
-		if (!bw_params->clk_table.entries[i].phyclk_d18_mhz)
-			bw_params->clk_table.entries[i].phyclk_d18_mhz = def_max.phyclk_d18_mhz;
-		if (!bw_params->clk_table.entries[i].dtbclk_mhz)
-			bw_params->clk_table.entries[i].dtbclk_mhz = def_max.dtbclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dcfclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dcfclk_mhz = def_max.dcfclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].socclk_mhz)
+			bw_params->clk_table.entries[entry_idx].socclk_mhz = def_max.socclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dispclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dispclk_mhz = def_max.dispclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dppclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dppclk_mhz = def_max.dppclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].phyclk_mhz)
+			bw_params->clk_table.entries[entry_idx].phyclk_mhz = def_max.phyclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].phyclk_d18_mhz)
+			bw_params->clk_table.entries[entry_idx].phyclk_d18_mhz = def_max.phyclk_d18_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dtbclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dtbclk_mhz = def_max.dtbclk_mhz;
 	}
 
 	/* Make sure all highest default clocks are included*/
@@ -572,16 +576,16 @@ static void dcn315_clk_mgr_helper_populate_bw_params(
 	bw_params->num_channels = bios_info->ma_channel_number;
 	bw_params->dram_channel_width_bytes = bios_info->memory_type == 0x22 ? 8 : 4;
 
-	for (i = 0; i < WM_SET_COUNT; i++) {
-		bw_params->wm_table.entries[i].wm_inst = i;
+	for (entry_idx = 0; entry_idx < (unsigned int)WM_SET_COUNT; entry_idx++) {
+		bw_params->wm_table.entries[entry_idx].wm_inst = entry_idx;
 
-		if (i >= bw_params->clk_table.num_entries) {
-			bw_params->wm_table.entries[i].valid = false;
+		if (entry_idx >= bw_params->clk_table.num_entries) {
+			bw_params->wm_table.entries[entry_idx].valid = false;
 			continue;
 		}
 
-		bw_params->wm_table.entries[i].wm_type = WM_TYPE_PSTATE_CHG;
-		bw_params->wm_table.entries[i].valid = true;
+		bw_params->wm_table.entries[entry_idx].wm_type = WM_TYPE_PSTATE_CHG;
+		bw_params->wm_table.entries[entry_idx].valid = true;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c
index e9d492d8c8d49..72a38f7a761c6 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn316/dcn316_clk_mgr.c
@@ -103,7 +103,7 @@ static void dcn316_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state
 		bool safe_to_lower, bool disable)
 {
 	struct dc *dc = clk_mgr_base->ctx->dc;
-	int i;
+	uint8_t i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; ++i) {
 		struct pipe_ctx *pipe = safe_to_lower
@@ -211,7 +211,7 @@ static void dcn316_update_clocks(struct clk_mgr *clk_mgr_base,
 
 	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) &&
 	    (new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) {
-		int requested_dispclk_khz = new_clocks->dispclk_khz;
+		uint32_t requested_dispclk_khz = new_clocks->dispclk_khz;
 
 		dcn316_disable_otg_wa(clk_mgr_base, context, safe_to_lower, true);
 
@@ -350,7 +350,7 @@ static struct dcn316_watermarks dummy_wms = { 0 };
 
 static void dcn316_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn316_watermarks *table)
 {
-	int i, num_valid_sets;
+	uint8_t i, num_valid_sets;
 
 	num_valid_sets = 0;
 
@@ -359,8 +359,11 @@ static void dcn316_build_watermark_ranges(struct clk_bw_params *bw_params, struc
 		if (!bw_params->wm_table.entries[i].valid)
 			continue;
 
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting = bw_params->wm_table.entries[i].wm_inst;
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType = bw_params->wm_table.entries[i].wm_type;
+
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting =
+			(uint8_t)bw_params->wm_table.entries[i].wm_inst;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType =
+			(uint8_t)bw_params->wm_table.entries[i].wm_type;
 		/* We will not select WM based on fclk, so leave it as unconstrained */
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinClock = 0;
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxClock = 0xFFFF;
@@ -371,10 +374,10 @@ static void dcn316_build_watermark_ranges(struct clk_bw_params *bw_params, struc
 			else {
 				/* add 1 to make it non-overlapping with next lvl */
 				table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinMclk =
-						bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1;
+						(uint16_t)(bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1);
 			}
 			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxMclk =
-					bw_params->clk_table.entries[i].dcfclk_mhz;
+					(uint16_t)bw_params->clk_table.entries[i].dcfclk_mhz;
 
 		} else {
 			/* unconstrained for memory retraining */
@@ -449,7 +452,7 @@ static void dcn316_get_dpm_table_from_smu(struct clk_mgr_internal *clk_mgr,
 static uint32_t find_max_clk_value(const uint32_t clocks[], uint32_t num_clocks)
 {
 	uint32_t max = 0;
-	int i;
+	uint32_t i;
 
 	for (i = 0; i < num_clocks; ++i) {
 		if (clocks[i] > max)
@@ -465,8 +468,8 @@ static unsigned int find_clk_for_voltage(
 		unsigned int voltage)
 {
 	int i;
-	int max_voltage = 0;
-	int clock = 0;
+	unsigned int max_voltage = 0;
+	unsigned int clock = 0;
 
 	for (i = 0; i < NUM_SOC_VOLTAGE_LEVELS; i++) {
 		if (clock_table->SocVoltage[i] == voltage) {
@@ -488,6 +491,7 @@ static void dcn316_clk_mgr_helper_populate_bw_params(
 		const DpmClocks_316_t *clock_table)
 {
 	int i, j;
+	unsigned int entry_idx;
 	struct clk_bw_params *bw_params = clk_mgr->base.bw_params;
 	uint32_t max_dispclk = 0, max_dppclk = 0;
 
@@ -522,46 +526,46 @@ static void dcn316_clk_mgr_helper_populate_bw_params(
 		ASSERT(0);
 	}
 
-	for (i = 0; i < bw_params->clk_table.num_entries; i++, j--) {
+	for (entry_idx = 0; entry_idx < bw_params->clk_table.num_entries; entry_idx++, j--) {
 		int temp;
 
-		bw_params->clk_table.entries[i].fclk_mhz = clock_table->DfPstateTable[j].FClk;
-		bw_params->clk_table.entries[i].memclk_mhz = clock_table->DfPstateTable[j].MemClk;
-		bw_params->clk_table.entries[i].voltage = clock_table->DfPstateTable[j].Voltage;
+		bw_params->clk_table.entries[entry_idx].fclk_mhz = clock_table->DfPstateTable[j].FClk;
+		bw_params->clk_table.entries[entry_idx].memclk_mhz = clock_table->DfPstateTable[j].MemClk;
+		bw_params->clk_table.entries[entry_idx].voltage = clock_table->DfPstateTable[j].Voltage;
 		switch (clock_table->DfPstateTable[j].WckRatio) {
 		case WCK_RATIO_1_2:
-			bw_params->clk_table.entries[i].wck_ratio = 2;
+			bw_params->clk_table.entries[entry_idx].wck_ratio = 2;
 			break;
 		case WCK_RATIO_1_4:
-			bw_params->clk_table.entries[i].wck_ratio = 4;
+			bw_params->clk_table.entries[entry_idx].wck_ratio = 4;
 			break;
 		default:
-			bw_params->clk_table.entries[i].wck_ratio = 1;
+			bw_params->clk_table.entries[entry_idx].wck_ratio = 1;
 		}
 		temp = find_clk_for_voltage(clock_table, clock_table->DcfClocks, clock_table->DfPstateTable[j].Voltage);
 		if (temp)
-			bw_params->clk_table.entries[i].dcfclk_mhz = temp;
+			bw_params->clk_table.entries[entry_idx].dcfclk_mhz = temp;
 		temp = find_clk_for_voltage(clock_table, clock_table->SocClocks, clock_table->DfPstateTable[j].Voltage);
 		if (temp)
-			bw_params->clk_table.entries[i].socclk_mhz = temp;
-		bw_params->clk_table.entries[i].dispclk_mhz = max_dispclk;
-		bw_params->clk_table.entries[i].dppclk_mhz = max_dppclk;
+			bw_params->clk_table.entries[entry_idx].socclk_mhz = temp;
+		bw_params->clk_table.entries[entry_idx].dispclk_mhz = max_dispclk;
+		bw_params->clk_table.entries[entry_idx].dppclk_mhz = max_dppclk;
 	}
 
 	bw_params->vram_type = bios_info->memory_type;
 	bw_params->num_channels = bios_info->ma_channel_number;
 	bw_params->dram_channel_width_bytes = bios_info->memory_type == 0x22 ? 8 : 4;
 
-	for (i = 0; i < WM_SET_COUNT; i++) {
-		bw_params->wm_table.entries[i].wm_inst = i;
+	for (entry_idx = 0; entry_idx < (unsigned int)WM_SET_COUNT; entry_idx++) {
+		bw_params->wm_table.entries[entry_idx].wm_inst = entry_idx;
 
-		if (i >= bw_params->clk_table.num_entries) {
-			bw_params->wm_table.entries[i].valid = false;
+		if (entry_idx >= bw_params->clk_table.num_entries) {
+			bw_params->wm_table.entries[entry_idx].valid = false;
 			continue;
 		}
 
-		bw_params->wm_table.entries[i].wm_type = WM_TYPE_PSTATE_CHG;
-		bw_params->wm_table.entries[i].valid = true;
+		bw_params->wm_table.entries[entry_idx].wm_type = WM_TYPE_PSTATE_CHG;
+		bw_params->wm_table.entries[entry_idx].valid = true;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
index fda6cade30a8d..90fdbf2798686 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn32/dcn32_clk_mgr.c
@@ -132,7 +132,7 @@ static const struct clk_mgr_mask clk_mgr_mask_dcn321 = {
 static void dcn32_init_single_clock(struct clk_mgr_internal *clk_mgr, PPCLK_e clk, unsigned int *entry_0,
 		unsigned int *num_levels)
 {
-	unsigned int i;
+	uint8_t i;
 	char *entry_i = (char *)entry_0;
 
 	uint32_t ret = dcn30_smu_get_dpm_freq_by_index(clk_mgr, clk, 0xFF);
@@ -236,9 +236,9 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base)
 	if (clk_mgr_base->ctx->dc->debug.min_disp_clk_khz) {
 		for (i = 0; i < num_levels; i++)
 			if (clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz
-					< khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz))
+					< (unsigned int)khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz))
 				clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz
-					= khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz);
+					= (unsigned int)khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz);
 	}
 	for (i = 0; i < num_levels; i++)
 		if (clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz > 1950)
@@ -247,9 +247,9 @@ void dcn32_init_clocks(struct clk_mgr *clk_mgr_base)
 	if (clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz) {
 		for (i = 0; i < num_levels; i++)
 			if (clk_mgr_base->bw_params->clk_table.entries[i].dppclk_mhz
-					< khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz))
+					< (unsigned int)khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz))
 				clk_mgr_base->bw_params->clk_table.entries[i].dppclk_mhz
-					= khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz);
+					= (unsigned int)khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz);
 	}
 
 	for (i = 0; i < num_levels; i++)
@@ -269,7 +269,7 @@ static void dcn32_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr,
 {
 	struct dccg *dccg = clk_mgr->dccg;
 	uint32_t tg_mask = 0;
-	int i;
+	uint32_t i;
 
 	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
@@ -314,7 +314,7 @@ static void dcn32_update_dppclk_dispclk_freq(struct clk_mgr_internal *clk_mgr, s
 void dcn32_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
 		struct dc_state *context, bool safe_to_lower)
 {
-	int i;
+	uint32_t i;
 
 	clk_mgr->dccg->ref_dppclk = clk_mgr->base.clks.dppclk_khz;
 	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
@@ -409,7 +409,7 @@ static void dcn32_update_clocks_update_dentist(
 			 * floored in Mhz to describe the intended clock.
 			 */
 			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DISPCLK,
-					khz_to_mhz_floor(temp_dispclk_khz));
+					(uint16_t)khz_to_mhz_floor(temp_dispclk_khz));
 
 		if (dc->debug.override_dispclk_programming) {
 			REG_GET(DENTIST_DISPCLK_CNTL,
@@ -456,7 +456,7 @@ static void dcn32_update_clocks_update_dentist(
 		 * floored in Mhz to describe the intended clock.
 		 */
 		dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DISPCLK,
-				khz_to_mhz_floor(clk_mgr->base.clks.dispclk_khz));
+				(uint16_t)khz_to_mhz_floor(clk_mgr->base.clks.dispclk_khz));
 
 	if (dc->debug.override_dispclk_programming) {
 		REG_GET(DENTIST_DISPCLK_CNTL,
@@ -475,7 +475,7 @@ static int dcn32_get_dispclk_from_dentist(struct clk_mgr *clk_mgr_base)
 {
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	uint32_t dispclk_wdivider;
-	int disp_divider;
+	unsigned int disp_divider;
 
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
 	disp_divider = dentist_get_divider_from_did(dispclk_wdivider);
@@ -673,14 +673,17 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 			}
 		}
 
-		if (dc->debug.force_min_dcfclk_mhz > 0)
-			new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ?
-					new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000);
+		if (dc->debug.force_min_dcfclk_mhz > 0) {
+			int force_min_dcfclk_khz = dc->debug.force_min_dcfclk_mhz * 1000;
+
+			new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > force_min_dcfclk_khz) ?
+					new_clocks->dcfclk_khz : force_min_dcfclk_khz;
+		}
 
 		if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz) &&
 				!dc->work_arounds.clock_update_disable_mask.dcfclk) {
 			clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz;
-			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DCFCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dcfclk_khz));
+			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DCFCLK, (uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dcfclk_khz));
 		}
 
 		if (should_set_clock(safe_to_lower, new_clocks->dcfclk_deep_sleep_khz, clk_mgr_base->clks.dcfclk_deep_sleep_khz) &&
@@ -715,13 +718,13 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 					 * frequency.
 					 */
 					if (dc->debug.disable_dc_mode_overwrite) {
-						dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, dc->clk_mgr->bw_params->max_memclk_mhz);
-						dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, dc->clk_mgr->bw_params->max_memclk_mhz);
+						dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)dc->clk_mgr->bw_params->max_memclk_mhz);
+						dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)dc->clk_mgr->bw_params->max_memclk_mhz);
 					} else
 						dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-								dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
+								(uint16_t)dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
 				} else {
-					dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, dc->clk_mgr->bw_params->max_memclk_mhz);
+					dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)dc->clk_mgr->bw_params->max_memclk_mhz);
 				}
 			}
 		}
@@ -755,9 +758,10 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 				!dc->work_arounds.clock_update_disable_mask.uclk) {
 			if (dc->clk_mgr->dc_mode_softmax_enabled && dc->debug.disable_dc_mode_overwrite)
 				dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK,
-						max((int)dc->clk_mgr->bw_params->dc_mode_softmax_memclk, khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz)));
+						(uint16_t)max((int)dc->clk_mgr->bw_params->dc_mode_softmax_memclk,
+								khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz)));
 
-			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
+			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
 		}
 
 		if (clk_mgr_base->clks.num_ways != new_clocks->num_ways &&
@@ -783,7 +787,7 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 			 * floored in Mhz to describe the intended clock.
 			 */
 			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DPPCLK,
-					khz_to_mhz_floor(clk_mgr_base->clks.dppclk_khz));
+					(uint16_t)khz_to_mhz_floor(clk_mgr_base->clks.dppclk_khz));
 
 		update_dppclk = true;
 	}
@@ -803,7 +807,7 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 			should_set_clock(safe_to_lower, new_clocks->ref_dtbclk_khz / 1000, clk_mgr_base->clks.ref_dtbclk_khz / 1000)) {
 		/* DCCG requires KHz precision for DTBCLK */
 		clk_mgr_base->clks.ref_dtbclk_khz =
-				dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DTBCLK, khz_to_mhz_ceil(new_clocks->ref_dtbclk_khz));
+				dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DTBCLK, (uint16_t)khz_to_mhz_ceil(new_clocks->ref_dtbclk_khz));
 
 		dcn32_update_clocks_update_dtb_dto(clk_mgr, context, clk_mgr_base->clks.ref_dtbclk_khz);
 	}
@@ -822,7 +826,7 @@ static void dcn32_update_clocks(struct clk_mgr *clk_mgr_base,
 				 * floored in Mhz to describe the intended clock.
 				 */
 				dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_DPPCLK,
-						khz_to_mhz_floor(clk_mgr_base->clks.dppclk_khz));
+						(uint16_t)khz_to_mhz_floor(clk_mgr_base->clks.dppclk_khz));
 		} else {
 			/* if clock is being raised, increase refclk before lowering DTO */
 			if (update_dppclk || update_dispclk)
@@ -968,7 +972,7 @@ static void dcn32_clock_read_ss_info(struct clk_mgr_internal *clk_mgr)
 }
 static void dcn32_notify_wm_ranges(struct clk_mgr *clk_mgr_base)
 {
-	unsigned int i;
+	uint8_t i;
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	WatermarksExternal_t *table = (WatermarksExternal_t *) clk_mgr->wm_range_table;
 
@@ -1002,13 +1006,13 @@ static void dcn32_set_hard_min_memclk(struct clk_mgr *clk_mgr_base, bool current
 	if (current_mode) {
 		if (clk_mgr_base->clks.p_state_change_support)
 			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-					khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
+					(uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dramclk_khz));
 		else
 			dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-					clk_mgr_base->bw_params->max_memclk_mhz);
+					(uint16_t)clk_mgr_base->bw_params->max_memclk_mhz);
 	} else {
 		dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK,
-				clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz);
+				(uint16_t)clk_mgr_base->bw_params->clk_table.entries[0].memclk_mhz);
 	}
 }
 
@@ -1020,7 +1024,7 @@ static void dcn32_set_hard_max_memclk(struct clk_mgr *clk_mgr_base)
 	if (!clk_mgr->smu_present)
 		return;
 
-	dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, clk_mgr_base->bw_params->max_memclk_mhz);
+	dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)clk_mgr_base->bw_params->max_memclk_mhz);
 }
 
 /* Get current memclk states, update bounding box */
@@ -1052,7 +1056,7 @@ static void dcn32_get_memclk_states_from_smu(struct clk_mgr *clk_mgr_base)
 
 	clk_mgr_base->bw_params->max_memclk_mhz =
 			clk_mgr_base->bw_params->clk_table.entries[num_entries_per_clk->num_memclk_levels - 1].memclk_mhz;
-	clk_mgr_base->bw_params->clk_table.num_entries = num_levels ? num_levels : 1;
+	clk_mgr_base->bw_params->clk_table.num_entries = (uint8_t)(num_levels ? num_levels : 1);
 
 	if (clk_mgr->dpm_present && !num_levels)
 		clk_mgr->dpm_present = false;
@@ -1109,7 +1113,7 @@ static void dcn32_set_max_memclk(struct clk_mgr *clk_mgr_base, unsigned int memc
 	if (!clk_mgr->smu_present)
 		return;
 
-	dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, memclk_mhz);
+	dcn30_smu_set_hard_max_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)memclk_mhz);
 }
 
 static void dcn32_set_min_memclk(struct clk_mgr *clk_mgr_base, unsigned int memclk_mhz)
@@ -1119,7 +1123,7 @@ static void dcn32_set_min_memclk(struct clk_mgr *clk_mgr_base, unsigned int memc
 	if (!clk_mgr->smu_present)
 		return;
 
-	dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, memclk_mhz);
+	dcn32_smu_set_hard_min_by_freq(clk_mgr, PPCLK_UCLK, (uint16_t)memclk_mhz);
 }
 
 static struct clk_mgr_funcs dcn32_funcs = {
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
index 2798088842f41..6c6848e375e17 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn35/dcn35_clk_mgr.c
@@ -190,7 +190,7 @@ void dcn35_disable_otg_wa(struct clk_mgr *clk_mgr_base, struct dc_state *context
 		bool safe_to_lower, bool disable)
 {
 	struct dc *dc = clk_mgr_base->ctx->dc;
-	int i;
+	uint8_t i;
 
 	if (dc->ctx->dce_environment == DCE_ENV_DIAG)
 		return;
@@ -257,7 +257,7 @@ static void dcn35_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr,
 {
 	struct dccg *dccg = clk_mgr->dccg;
 	uint32_t tg_mask = 0;
-	int i;
+	uint32_t i;
 
 	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
@@ -280,7 +280,7 @@ static void dcn35_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr,
 static void dcn35_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
 		struct dc_state *context, bool safe_to_lower)
 {
-	int i;
+	uint32_t i;
 	bool dppclk_active[MAX_PIPES] = {0};
 
 
@@ -332,7 +332,7 @@ static uint8_t get_lowest_dpia_index(const struct dc_link *link)
 			continue;
 
 		if (idx > dc_struct->links[i]->link_index)
-			idx = dc_struct->links[i]->link_index;
+			idx = (uint8_t)dc_struct->links[i]->link_index;
 	}
 
 	return idx;
@@ -450,9 +450,12 @@ void dcn35_update_clocks(struct clk_mgr *clk_mgr_base,
 			clk_mgr_base->clks.pwr_state = DCN_PWR_STATE_MISSION_MODE;
 		}
 	}
-	if (dc->debug.force_min_dcfclk_mhz > 0)
-		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ?
-				new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000);
+	if (dc->debug.force_min_dcfclk_mhz > 0) {
+		int force_min_dcfclk_khz = dc->debug.force_min_dcfclk_mhz * 1000;
+
+		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > force_min_dcfclk_khz) ?
+				new_clocks->dcfclk_khz : force_min_dcfclk_khz;
+	}
 
 	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz)) {
 		clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz;
@@ -478,7 +481,7 @@ void dcn35_update_clocks(struct clk_mgr *clk_mgr_base,
 
 	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) &&
 	    (new_clocks->dispclk_khz > 0 || (safe_to_lower && display_count == 0))) {
-		int requested_dispclk_khz = new_clocks->dispclk_khz;
+		uint32_t requested_dispclk_khz = new_clocks->dispclk_khz;
 
 		dcn35_disable_otg_wa(clk_mgr_base, context, safe_to_lower, true);
 
@@ -863,7 +866,7 @@ static void dcn35_read_ss_info_from_lut(struct clk_mgr_internal *clk_mgr)
 
 static void dcn35_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn35_watermarks *table)
 {
-	int i, num_valid_sets;
+	uint8_t i, num_valid_sets;
 
 	num_valid_sets = 0;
 
@@ -872,8 +875,10 @@ static void dcn35_build_watermark_ranges(struct clk_bw_params *bw_params, struct
 		if (!bw_params->wm_table.entries[i].valid)
 			continue;
 
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting = bw_params->wm_table.entries[i].wm_inst;
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType = bw_params->wm_table.entries[i].wm_type;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting =
+			(uint8_t)bw_params->wm_table.entries[i].wm_inst;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType =
+			(uint8_t)bw_params->wm_table.entries[i].wm_type;
 		/* We will not select WM based on fclk, so leave it as unconstrained */
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinClock = 0;
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxClock = 0xFFFF;
@@ -884,10 +889,10 @@ static void dcn35_build_watermark_ranges(struct clk_bw_params *bw_params, struct
 			else {
 				/* add 1 to make it non-overlapping with next lvl */
 				table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinMclk =
-						bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1;
+						(uint16_t)(bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1);
 			}
 			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxMclk =
-					bw_params->clk_table.entries[i].dcfclk_mhz;
+					(uint16_t)bw_params->clk_table.entries[i].dcfclk_mhz;
 
 		} else {
 			/* unconstrained for memory retraining */
@@ -978,7 +983,7 @@ static void dcn351_get_dpm_table_from_smu(struct clk_mgr_internal *clk_mgr,
 static uint32_t find_max_clk_value(const uint32_t clocks[], uint32_t num_clocks)
 {
 	uint32_t max = 0;
-	int i;
+	uint32_t i;
 
 	for (i = 0; i < num_clocks; ++i) {
 		if (clocks[i] > max)
@@ -1025,28 +1030,29 @@ static void dcn35_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *clk
 	uint32_t max_pstate = 0, max_dram_speed_mts = 0, min_dram_speed_mts = 0;
 	uint32_t num_memps, num_fclk, num_dcfclk;
 	int i;
+	unsigned int entry_idx;
 
 	/* Determine min/max p-state values. */
 	num_memps = (clock_table->NumMemPstatesEnabled > NUM_MEM_PSTATE_LEVELS) ? NUM_MEM_PSTATE_LEVELS :
 		clock_table->NumMemPstatesEnabled;
-	for (i = 0; i < num_memps; i++) {
-		uint32_t dram_speed_mts = calc_dram_speed_mts(&clock_table->MemPstateTable[i]);
+	for (entry_idx = 0; entry_idx < num_memps; entry_idx++) {
+		uint32_t dram_speed_mts = calc_dram_speed_mts(&clock_table->MemPstateTable[entry_idx]);
 
 		if (is_valid_clock_value(dram_speed_mts) && dram_speed_mts > max_dram_speed_mts) {
 			max_dram_speed_mts = dram_speed_mts;
-			max_pstate = i;
+			max_pstate = entry_idx;
 		}
 	}
 
 	min_dram_speed_mts = max_dram_speed_mts;
 	min_pstate = max_pstate;
 
-	for (i = 0; i < num_memps; i++) {
-		uint32_t dram_speed_mts = calc_dram_speed_mts(&clock_table->MemPstateTable[i]);
+	for (entry_idx = 0; entry_idx < num_memps; entry_idx++) {
+		uint32_t dram_speed_mts = calc_dram_speed_mts(&clock_table->MemPstateTable[entry_idx]);
 
 		if (is_valid_clock_value(dram_speed_mts) && dram_speed_mts < min_dram_speed_mts) {
 			min_dram_speed_mts = dram_speed_mts;
-			min_pstate = i;
+			min_pstate = entry_idx;
 		}
 	}
 
@@ -1076,31 +1082,32 @@ static void dcn35_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *clk
 
 	num_dcfclk = (clock_table->NumDcfClkLevelsEnabled > NUM_DCFCLK_DPM_LEVELS) ? NUM_DCFCLK_DPM_LEVELS :
 		clock_table->NumDcfClkLevelsEnabled;
-	for (i = 0; i < num_dcfclk; i++) {
+	for (entry_idx = 0; entry_idx < num_dcfclk; entry_idx++) {
 		int j;
 
 		/* First search defaults for the clocks we don't read using closest lower or equal default dcfclk */
 		for (j = bw_params->clk_table.num_entries - 1; j > 0; j--)
-			if (bw_params->clk_table.entries[j].dcfclk_mhz <= clock_table->DcfClocks[i])
+			if (bw_params->clk_table.entries[j].dcfclk_mhz <= clock_table->DcfClocks[entry_idx])
 				break;
 
-		bw_params->clk_table.entries[i].phyclk_mhz = bw_params->clk_table.entries[j].phyclk_mhz;
-		bw_params->clk_table.entries[i].phyclk_d18_mhz = bw_params->clk_table.entries[j].phyclk_d18_mhz;
-		bw_params->clk_table.entries[i].dtbclk_mhz = bw_params->clk_table.entries[j].dtbclk_mhz;
+		bw_params->clk_table.entries[entry_idx].phyclk_mhz = bw_params->clk_table.entries[j].phyclk_mhz;
+		bw_params->clk_table.entries[entry_idx].phyclk_d18_mhz = bw_params->clk_table.entries[j].phyclk_d18_mhz;
+		bw_params->clk_table.entries[entry_idx].dtbclk_mhz = bw_params->clk_table.entries[j].dtbclk_mhz;
 
 		/* Now update clocks we do read */
-		bw_params->clk_table.entries[i].memclk_mhz = clock_table->MemPstateTable[min_pstate].MemClk;
-		bw_params->clk_table.entries[i].voltage = clock_table->MemPstateTable[min_pstate].Voltage;
-		bw_params->clk_table.entries[i].dcfclk_mhz = clock_table->DcfClocks[i];
-		bw_params->clk_table.entries[i].socclk_mhz = clock_table->SocClocks[i];
-		bw_params->clk_table.entries[i].dispclk_mhz = max_dispclk;
-		bw_params->clk_table.entries[i].dppclk_mhz = max_dppclk;
-		bw_params->clk_table.entries[i].wck_ratio =
+		bw_params->clk_table.entries[entry_idx].memclk_mhz = clock_table->MemPstateTable[min_pstate].MemClk;
+		bw_params->clk_table.entries[entry_idx].voltage = clock_table->MemPstateTable[min_pstate].Voltage;
+		bw_params->clk_table.entries[entry_idx].dcfclk_mhz = clock_table->DcfClocks[entry_idx];
+		bw_params->clk_table.entries[entry_idx].socclk_mhz = clock_table->SocClocks[entry_idx];
+		bw_params->clk_table.entries[entry_idx].dispclk_mhz = max_dispclk;
+		bw_params->clk_table.entries[entry_idx].dppclk_mhz = max_dppclk;
+		bw_params->clk_table.entries[entry_idx].wck_ratio =
 			convert_wck_ratio(clock_table->MemPstateTable[min_pstate].WckRatio);
 
 		/* Dcfclk and Fclk are tied, but at a different ratio */
-		bw_params->clk_table.entries[i].fclk_mhz = min(max_fclk, 2 * clock_table->DcfClocks[i]);
+		bw_params->clk_table.entries[entry_idx].fclk_mhz = min(max_fclk, 2 * clock_table->DcfClocks[entry_idx]);
 	}
+	i = (int)entry_idx;
 
 	/* Make sure to include at least one entry at highest pstate */
 	if (max_pstate != min_pstate || i == 0) {
@@ -1146,44 +1153,44 @@ static void dcn35_clk_mgr_helper_populate_bw_params(struct clk_mgr_internal *clk
 	 * Set any 0 clocks to max default setting. Not an issue for
 	 * power since we aren't doing switching in such case anyway
 	 */
-	for (i = 0; i < bw_params->clk_table.num_entries; i++) {
-		if (!bw_params->clk_table.entries[i].fclk_mhz) {
-			bw_params->clk_table.entries[i].fclk_mhz = def_max.fclk_mhz;
-			bw_params->clk_table.entries[i].memclk_mhz = def_max.memclk_mhz;
-			bw_params->clk_table.entries[i].voltage = def_max.voltage;
+	for (entry_idx = 0; entry_idx < bw_params->clk_table.num_entries; entry_idx++) {
+		if (!bw_params->clk_table.entries[entry_idx].fclk_mhz) {
+			bw_params->clk_table.entries[entry_idx].fclk_mhz = def_max.fclk_mhz;
+			bw_params->clk_table.entries[entry_idx].memclk_mhz = def_max.memclk_mhz;
+			bw_params->clk_table.entries[entry_idx].voltage = def_max.voltage;
 		}
-		if (!bw_params->clk_table.entries[i].dcfclk_mhz)
-			bw_params->clk_table.entries[i].dcfclk_mhz = def_max.dcfclk_mhz;
-		if (!bw_params->clk_table.entries[i].socclk_mhz)
-			bw_params->clk_table.entries[i].socclk_mhz = def_max.socclk_mhz;
-		if (!bw_params->clk_table.entries[i].dispclk_mhz)
-			bw_params->clk_table.entries[i].dispclk_mhz = def_max.dispclk_mhz;
-		if (!bw_params->clk_table.entries[i].dppclk_mhz)
-			bw_params->clk_table.entries[i].dppclk_mhz = def_max.dppclk_mhz;
-		if (!bw_params->clk_table.entries[i].fclk_mhz)
-			bw_params->clk_table.entries[i].fclk_mhz = def_max.fclk_mhz;
-		if (!bw_params->clk_table.entries[i].phyclk_mhz)
-			bw_params->clk_table.entries[i].phyclk_mhz = def_max.phyclk_mhz;
-		if (!bw_params->clk_table.entries[i].phyclk_d18_mhz)
-			bw_params->clk_table.entries[i].phyclk_d18_mhz = def_max.phyclk_d18_mhz;
-		if (!bw_params->clk_table.entries[i].dtbclk_mhz)
-			bw_params->clk_table.entries[i].dtbclk_mhz = def_max.dtbclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dcfclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dcfclk_mhz = def_max.dcfclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].socclk_mhz)
+			bw_params->clk_table.entries[entry_idx].socclk_mhz = def_max.socclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dispclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dispclk_mhz = def_max.dispclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dppclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dppclk_mhz = def_max.dppclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].fclk_mhz)
+			bw_params->clk_table.entries[entry_idx].fclk_mhz = def_max.fclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].phyclk_mhz)
+			bw_params->clk_table.entries[entry_idx].phyclk_mhz = def_max.phyclk_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].phyclk_d18_mhz)
+			bw_params->clk_table.entries[entry_idx].phyclk_d18_mhz = def_max.phyclk_d18_mhz;
+		if (!bw_params->clk_table.entries[entry_idx].dtbclk_mhz)
+			bw_params->clk_table.entries[entry_idx].dtbclk_mhz = def_max.dtbclk_mhz;
 	}
 	ASSERT(bw_params->clk_table.entries[i-1].dcfclk_mhz);
 	bw_params->vram_type = bios_info->memory_type;
 	bw_params->dram_channel_width_bytes = bios_info->memory_type == 0x22 ? 8 : 4;
 	bw_params->num_channels = bios_info->ma_channel_number ? bios_info->ma_channel_number : 4;
 
-	for (i = 0; i < WM_SET_COUNT; i++) {
-		bw_params->wm_table.entries[i].wm_inst = i;
+	for (entry_idx = 0; entry_idx < (unsigned int)WM_SET_COUNT; entry_idx++) {
+		bw_params->wm_table.entries[entry_idx].wm_inst = entry_idx;
 
-		if (i >= bw_params->clk_table.num_entries) {
-			bw_params->wm_table.entries[i].valid = false;
+		if (entry_idx >= bw_params->clk_table.num_entries) {
+			bw_params->wm_table.entries[entry_idx].valid = false;
 			continue;
 		}
 
-		bw_params->wm_table.entries[i].wm_type = WM_TYPE_PSTATE_CHG;
-		bw_params->wm_table.entries[i].valid = true;
+		bw_params->wm_table.entries[entry_idx].wm_type = WM_TYPE_PSTATE_CHG;
+		bw_params->wm_table.entries[entry_idx].valid = true;
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c
index 82c1a55a22710..5bf6f2d50705c 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.c
@@ -174,7 +174,7 @@ static void dcn401_init_single_clock(struct clk_mgr_internal *clk_mgr, PPCLK_e c
 
 	/* if the initial message failed, num_levels will be 0 */
 	for (i = 0; i < *num_levels && i < ARRAY_SIZE(clk_mgr->base.bw_params->clk_table.entries); i++) {
-		*((unsigned int *)entry_i) = (dcn401_smu_get_dpm_freq_by_index(clk_mgr, clk, i) & 0xFFFF);
+		*((unsigned int *)entry_i) = (dcn401_smu_get_dpm_freq_by_index(clk_mgr, clk, (uint8_t)i) & 0xFFFF);
 		entry_i += sizeof(clk_mgr->base.bw_params->clk_table.entries[0]);
 	}
 }
@@ -182,15 +182,15 @@ static void dcn401_init_single_clock(struct clk_mgr_internal *clk_mgr, PPCLK_e c
 static void dcn401_build_wm_range_table(struct clk_mgr *clk_mgr)
 {
 	/* For min clocks use as reported by PM FW and report those as min */
-	uint16_t min_uclk_mhz = clk_mgr->bw_params->clk_table.entries[0].memclk_mhz;
-	uint16_t min_dcfclk_mhz	= clk_mgr->bw_params->clk_table.entries[0].dcfclk_mhz;
+	unsigned int min_uclk_mhz = clk_mgr->bw_params->clk_table.entries[0].memclk_mhz;
+	unsigned int min_dcfclk_mhz = clk_mgr->bw_params->clk_table.entries[0].dcfclk_mhz;
 
 	/* Set A - Normal - default values */
 	clk_mgr->bw_params->wm_table.nv_entries[WM_A].valid = true;
 	clk_mgr->bw_params->wm_table.nv_entries[WM_A].pmfw_breakdown.wm_type = WATERMARKS_CLOCK_RANGE;
-	clk_mgr->bw_params->wm_table.nv_entries[WM_A].pmfw_breakdown.min_dcfclk = min_dcfclk_mhz;
+	clk_mgr->bw_params->wm_table.nv_entries[WM_A].pmfw_breakdown.min_dcfclk = (uint16_t)min_dcfclk_mhz;
 	clk_mgr->bw_params->wm_table.nv_entries[WM_A].pmfw_breakdown.max_dcfclk = 0xFFFF;
-	clk_mgr->bw_params->wm_table.nv_entries[WM_A].pmfw_breakdown.min_uclk = min_uclk_mhz;
+	clk_mgr->bw_params->wm_table.nv_entries[WM_A].pmfw_breakdown.min_uclk = (uint16_t)min_uclk_mhz;
 	clk_mgr->bw_params->wm_table.nv_entries[WM_A].pmfw_breakdown.max_uclk = 0xFFFF;
 
 	/* Set B - Unused on dcn4 */
@@ -201,9 +201,9 @@ static void dcn401_build_wm_range_table(struct clk_mgr *clk_mgr)
 	if (clk_mgr->ctx->dc->bb_overrides.dummy_clock_change_latency_ns != 0x7FFFFFFF) {
 		clk_mgr->bw_params->wm_table.nv_entries[WM_1A].valid = true;
 		clk_mgr->bw_params->wm_table.nv_entries[WM_1A].pmfw_breakdown.wm_type = WATERMARKS_DUMMY_PSTATE;
-		clk_mgr->bw_params->wm_table.nv_entries[WM_1A].pmfw_breakdown.min_dcfclk = min_dcfclk_mhz;
+		clk_mgr->bw_params->wm_table.nv_entries[WM_1A].pmfw_breakdown.min_dcfclk = (uint16_t)min_dcfclk_mhz;
 		clk_mgr->bw_params->wm_table.nv_entries[WM_1A].pmfw_breakdown.max_dcfclk = 0xFFFF;
-		clk_mgr->bw_params->wm_table.nv_entries[WM_1A].pmfw_breakdown.min_uclk = min_uclk_mhz;
+		clk_mgr->bw_params->wm_table.nv_entries[WM_1A].pmfw_breakdown.min_uclk = (uint16_t)min_uclk_mhz;
 		clk_mgr->bw_params->wm_table.nv_entries[WM_1A].pmfw_breakdown.max_uclk = 0xFFFF;
 	} else {
 		clk_mgr->bw_params->wm_table.nv_entries[WM_1A].valid = false;
@@ -291,17 +291,17 @@ void dcn401_init_clocks(struct clk_mgr *clk_mgr_base)
 	if (clk_mgr_base->ctx->dc->debug.min_disp_clk_khz) {
 		for (i = 0; i < num_entries_per_clk->num_dispclk_levels; i++)
 			if (clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz
-					< khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz))
+					< (unsigned int)khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz))
 				clk_mgr_base->bw_params->clk_table.entries[i].dispclk_mhz
-					= khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz);
+					= (unsigned int)khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_disp_clk_khz);
 	}
 
 	if (clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz) {
 		for (i = 0; i < num_entries_per_clk->num_dppclk_levels; i++)
 			if (clk_mgr_base->bw_params->clk_table.entries[i].dppclk_mhz
-					< khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz))
+					< (unsigned int)khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz))
 				clk_mgr_base->bw_params->clk_table.entries[i].dppclk_mhz
-					= khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz);
+					= (unsigned int)khz_to_mhz_ceil(clk_mgr_base->ctx->dc->debug.min_dpp_clk_khz);
 	}
 
 	/* Get UCLK, update bounding box */
@@ -558,7 +558,7 @@ static void dcn401_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr
 static void dcn401_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
 		struct dc_state *context, bool safe_to_lower, int ref_dppclk_khz)
 {
-	int i;
+	uint32_t i;
 
 	clk_mgr->dccg->ref_dppclk = ref_dppclk_khz;
 	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
@@ -604,10 +604,10 @@ static int dcn401_set_hard_min_by_freq_optimized(struct clk_mgr_internal *clk_mg
 	 * clock returned is less than requested, then we will ceil the
 	 * requested value to mhz and call it again.
 	 */
-	int actual_clk_khz = dcn401_smu_set_hard_min_by_freq(clk_mgr, clk, khz_to_mhz_floor(requested_clk_khz));
+	int actual_clk_khz = dcn401_smu_set_hard_min_by_freq(clk_mgr, clk, (uint16_t)khz_to_mhz_floor(requested_clk_khz));
 
 	if (actual_clk_khz < requested_clk_khz)
-		actual_clk_khz = dcn401_smu_set_hard_min_by_freq(clk_mgr, clk, khz_to_mhz_ceil(requested_clk_khz));
+		actual_clk_khz = dcn401_smu_set_hard_min_by_freq(clk_mgr, clk, (uint16_t)khz_to_mhz_ceil(requested_clk_khz));
 
 	return actual_clk_khz;
 }
@@ -841,15 +841,18 @@ static unsigned int dcn401_build_update_bandwidth_clocks_sequence(
 	}
 
 	/* UPDATE DCFCLK */
-	if (dc->debug.force_min_dcfclk_mhz > 0)
-		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ?
-				new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000);
+	if (dc->debug.force_min_dcfclk_mhz > 0) {
+		int force_min_dcfclk_khz = dc->debug.force_min_dcfclk_mhz * 1000;
+
+		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > force_min_dcfclk_khz) ?
+				new_clocks->dcfclk_khz : force_min_dcfclk_khz;
+	}
 
 	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz)) {
 		clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz;
 		if (dcn401_is_ppclk_dpm_enabled(clk_mgr_internal, PPCLK_DCFCLK)) {
 			block_sequence[num_steps].params.update_hardmin_params.ppclk = PPCLK_DCFCLK;
-			block_sequence[num_steps].params.update_hardmin_params.freq_mhz = khz_to_mhz_ceil(clk_mgr_base->clks.dcfclk_khz);
+			block_sequence[num_steps].params.update_hardmin_params.freq_mhz = (uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dcfclk_khz);
 			block_sequence[num_steps].params.update_hardmin_params.response = NULL;
 			block_sequence[num_steps].func = CLK_MGR401_UPDATE_HARDMIN_PPCLK;
 			num_steps++;
@@ -860,7 +863,7 @@ static unsigned int dcn401_build_update_bandwidth_clocks_sequence(
 	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_deep_sleep_khz, clk_mgr_base->clks.dcfclk_deep_sleep_khz)) {
 		clk_mgr_base->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
 		if (dcn401_is_ppclk_dpm_enabled(clk_mgr_internal, PPCLK_DCFCLK)) {
-			block_sequence[num_steps].params.update_deep_sleep_dcfclk_params.freq_mhz = khz_to_mhz_ceil(clk_mgr_base->clks.dcfclk_deep_sleep_khz);
+			block_sequence[num_steps].params.update_deep_sleep_dcfclk_params.freq_mhz = (uint16_t)khz_to_mhz_ceil(clk_mgr_base->clks.dcfclk_deep_sleep_khz);
 			block_sequence[num_steps].func = CLK_MGR401_UPDATE_DEEP_SLEEP_DCFCLK;
 			num_steps++;
 		}
@@ -984,24 +987,24 @@ static unsigned int dcn401_build_update_bandwidth_clocks_sequence(
 	/* When idle DPM is enabled, need to send active and idle hardmins separately */
 	/* CLK_MGR401_UPDATE_ACTIVE_HARDMINS */
 	if ((update_active_uclk || update_active_fclk) && is_idle_dpm_enabled) {
-		block_sequence[num_steps].params.update_idle_hardmin_params.uclk_mhz = active_uclk_mhz;
-		block_sequence[num_steps].params.update_idle_hardmin_params.fclk_mhz = active_fclk_mhz;
+		block_sequence[num_steps].params.update_idle_hardmin_params.uclk_mhz = (uint16_t)active_uclk_mhz;
+		block_sequence[num_steps].params.update_idle_hardmin_params.fclk_mhz = (uint16_t)active_fclk_mhz;
 		block_sequence[num_steps].func = CLK_MGR401_UPDATE_ACTIVE_HARDMINS;
 		num_steps++;
 	}
 
 	/* CLK_MGR401_UPDATE_IDLE_HARDMINS */
 	if ((update_idle_uclk || update_idle_fclk) && is_idle_dpm_enabled) {
-		block_sequence[num_steps].params.update_idle_hardmin_params.uclk_mhz = idle_uclk_mhz;
-		block_sequence[num_steps].params.update_idle_hardmin_params.fclk_mhz = idle_fclk_mhz;
+		block_sequence[num_steps].params.update_idle_hardmin_params.uclk_mhz = (uint16_t)idle_uclk_mhz;
+		block_sequence[num_steps].params.update_idle_hardmin_params.fclk_mhz = (uint16_t)idle_fclk_mhz;
 		block_sequence[num_steps].func = CLK_MGR401_UPDATE_IDLE_HARDMINS;
 		num_steps++;
 	}
 
 	/* CLK_MGR401_UPDATE_SUBVP_HARDMINS */
 	if ((update_subvp_prefetch_dramclk || update_subvp_prefetch_fclk) && is_df_throttle_opt_enabled) {
-		block_sequence[num_steps].params.update_idle_hardmin_params.uclk_mhz = subvp_prefetch_dramclk_mhz;
-		block_sequence[num_steps].params.update_idle_hardmin_params.fclk_mhz = subvp_prefetch_fclk_mhz;
+		block_sequence[num_steps].params.update_idle_hardmin_params.uclk_mhz = (uint16_t)subvp_prefetch_dramclk_mhz;
+		block_sequence[num_steps].params.update_idle_hardmin_params.fclk_mhz = (uint16_t)subvp_prefetch_fclk_mhz;
 		block_sequence[num_steps].func = CLK_MGR401_UPDATE_SUBVP_HARDMINS;
 		num_steps++;
 	}
@@ -1010,7 +1013,7 @@ static unsigned int dcn401_build_update_bandwidth_clocks_sequence(
 	if (update_active_uclk || update_idle_uclk) {
 		if (!is_idle_dpm_enabled) {
 			block_sequence[num_steps].params.update_hardmin_params.ppclk = PPCLK_UCLK;
-			block_sequence[num_steps].params.update_hardmin_params.freq_mhz = active_uclk_mhz;
+			block_sequence[num_steps].params.update_hardmin_params.freq_mhz = (uint16_t)active_uclk_mhz;
 			block_sequence[num_steps].params.update_hardmin_params.response = NULL;
 			block_sequence[num_steps].func = CLK_MGR401_UPDATE_HARDMIN_PPCLK;
 			num_steps++;
@@ -1123,7 +1126,7 @@ static unsigned int dcn401_build_update_display_clocks_sequence(
 			dcn401_is_ppclk_dpm_enabled(clk_mgr_internal, PPCLK_DTBCLK)) {
 		/* DCCG requires KHz precision for DTBCLK */
 		block_sequence[num_steps].params.update_hardmin_params.ppclk = PPCLK_DTBCLK;
-		block_sequence[num_steps].params.update_hardmin_params.freq_mhz = khz_to_mhz_ceil(new_clocks->ref_dtbclk_khz);
+		block_sequence[num_steps].params.update_hardmin_params.freq_mhz = (uint16_t)khz_to_mhz_ceil(new_clocks->ref_dtbclk_khz);
 		block_sequence[num_steps].params.update_hardmin_params.response = &clk_mgr_base->clks.ref_dtbclk_khz;
 		block_sequence[num_steps].func = CLK_MGR401_UPDATE_HARDMIN_PPCLK;
 		num_steps++;
@@ -1318,7 +1321,7 @@ static void dcn401_notify_wm_ranges(struct clk_mgr *clk_mgr_base)
 	/* collect valid ranges, place in pmfw table */
 	for (i = 0; i < WM_SET_COUNT; i++)
 		if (clk_mgr->base.bw_params->wm_table.nv_entries[i].valid) {
-			table->Watermarks.WatermarkRow[i].WmSetting = i;
+			table->Watermarks.WatermarkRow[i].WmSetting = (uint8_t)i;
 			table->Watermarks.WatermarkRow[i].Flags = clk_mgr->base.bw_params->wm_table.nv_entries[i].pmfw_breakdown.wm_type;
 		}
 	dcn401_smu_set_dram_addr_high(clk_mgr, clk_mgr->wm_range_table_addr >> 32);
@@ -1487,7 +1490,7 @@ static int dcn401_get_dispclk_from_dentist(struct clk_mgr *clk_mgr_base)
 {
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	uint32_t dispclk_wdivider;
-	int disp_divider;
+	unsigned int disp_divider;
 
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
 	disp_divider = dentist_get_divider_from_did(dispclk_wdivider);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h
index 97a1ce1e8a9e8..370d2ddd6064c 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr.h
@@ -17,14 +17,14 @@ union dcn401_clk_mgr_block_sequence_params {
 		uint32_t ppclk;
 		uint16_t freq_mhz;
 		/* outputs */
-		uint32_t *response;
+		int *response;
 	} update_hardmin_params;
 	struct {
 		/* inputs */
 		uint32_t ppclk;
 		int freq_khz;
 		/* outputs */
-		uint32_t *response;
+		int *response;
 	} update_hardmin_optimized_params;
 	struct {
 		/* inputs */
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.c
index 3a263840893e3..82ccd9b407f47 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.c
@@ -143,12 +143,12 @@ static bool dcn401_smu_send_msg_with_param_delay(struct clk_mgr_internal *clk_mg
 	return false;
 }
 
-bool dcn401_smu_get_smu_version(struct clk_mgr_internal *clk_mgr, unsigned int *version)
+bool dcn401_smu_get_smu_version(struct clk_mgr_internal *clk_mgr, int *version)
 {
 	smu_print("SMU Get SMU version\n");
 
 	if (dcn401_smu_send_msg_with_param(clk_mgr,
-			DALSMC_MSG_GetSmuVersion, 0, version)) {
+			DALSMC_MSG_GetSmuVersion, 0, (uint32_t *)version)) {
 
 		smu_print("SMU version: %d\n", *version);
 
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.h b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.h
index 4f5ac603e8223..1addbed1cb99a 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.h
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn401/dcn401_clk_mgr_smu_msg.h
@@ -10,7 +10,7 @@
 
 struct clk_mgr_internal;
 
-bool dcn401_smu_get_smu_version(struct clk_mgr_internal *clk_mgr, unsigned int *version);
+bool dcn401_smu_get_smu_version(struct clk_mgr_internal *clk_mgr, int *version);
 bool dcn401_smu_check_driver_if_version(struct clk_mgr_internal *clk_mgr);
 bool dcn401_smu_check_msg_header_version(struct clk_mgr_internal *clk_mgr);
 void dcn401_smu_send_fclk_pstate_message(struct clk_mgr_internal *clk_mgr, bool support);
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
index 6a97ce69a5620..d01e5969a6708 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_clk_mgr.c
@@ -167,7 +167,7 @@ void dcn42_update_clocks_update_dtb_dto(struct clk_mgr_internal *clk_mgr,
 void dcn42_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr,
 		struct dc_state *context, bool safe_to_lower)
 {
-	int i;
+	uint32_t i;
 	bool dppclk_active[MAX_PIPES] = {0};
 
 
@@ -282,9 +282,12 @@ void dcn42_update_clocks(struct clk_mgr *clk_mgr_base,
 			clk_mgr_base->clks.pwr_state = DCN_PWR_STATE_MISSION_MODE;
 		}
 	}
-	if (dc->debug.force_min_dcfclk_mhz > 0)
-		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > (dc->debug.force_min_dcfclk_mhz * 1000)) ?
-				new_clocks->dcfclk_khz : (dc->debug.force_min_dcfclk_mhz * 1000);
+	if (dc->debug.force_min_dcfclk_mhz > 0) {
+		int force_min_dcfclk_khz = dc->debug.force_min_dcfclk_mhz * 1000;
+
+		new_clocks->dcfclk_khz = (new_clocks->dcfclk_khz > force_min_dcfclk_khz) ?
+				new_clocks->dcfclk_khz : force_min_dcfclk_khz;
+	}
 
 	if (should_set_clock(safe_to_lower, new_clocks->dcfclk_khz, clk_mgr_base->clks.dcfclk_khz)) {
 		clk_mgr_base->clks.dcfclk_khz = new_clocks->dcfclk_khz;
@@ -298,13 +301,16 @@ void dcn42_update_clocks(struct clk_mgr *clk_mgr_base,
 		clk_mgr_base->clks.dcfclk_deep_sleep_khz = new_clocks->dcfclk_deep_sleep_khz;
 
 		/* Clamp the requested clock to PMFW based on DCN limit. */
-		if (dc->debug.min_deep_sleep_dcfclk_khz > 0 && clk_mgr_base->clks.dcfclk_deep_sleep_khz < dc->debug.min_deep_sleep_dcfclk_khz)
-			clk_mgr_base->clks.dcfclk_deep_sleep_khz = dc->debug.min_deep_sleep_dcfclk_khz;
+		if (dc->debug.min_deep_sleep_dcfclk_khz > 0 &&
+			clk_mgr_base->clks.dcfclk_deep_sleep_khz < (int)dc->debug.min_deep_sleep_dcfclk_khz)
+			clk_mgr_base->clks.dcfclk_deep_sleep_khz = (int)dc->debug.min_deep_sleep_dcfclk_khz;
 
 		dcn42_smu_set_min_deep_sleep_dcfclk(clk_mgr, clk_mgr_base->clks.dcfclk_deep_sleep_khz);
 	}
 
 	// workaround: Limit dppclk to 100Mhz to avoid lower eDP panel switch to plus 4K monitor underflow.
+	if (new_clocks->dppclk_khz < 100000)
+		new_clocks->dppclk_khz = 100000;
 
 	if (should_set_clock(safe_to_lower, new_clocks->dppclk_khz, clk_mgr->base.clks.dppclk_khz)) {
 		if (clk_mgr->base.clks.dppclk_khz > new_clocks->dppclk_khz)
@@ -315,13 +321,13 @@ void dcn42_update_clocks(struct clk_mgr *clk_mgr_base,
 
 	if (should_set_clock(safe_to_lower, new_clocks->dispclk_khz, clk_mgr_base->clks.dispclk_khz) &&
 	    (new_clocks->dispclk_khz > 0 || (safe_to_lower && has_active_display == false))) {
-		int requested_dispclk_khz = new_clocks->dispclk_khz;
+		uint32_t requested_dispclk_khz = new_clocks->dispclk_khz;
 
 		dcn35_disable_otg_wa(clk_mgr_base, context, safe_to_lower, true);
 
 		/* Clamp the requested clock to PMFW based on their limit. */
-		if (dc->debug.min_disp_clk_khz > 0 && requested_dispclk_khz < dc->debug.min_disp_clk_khz)
-			requested_dispclk_khz = dc->debug.min_disp_clk_khz;
+		if (dc->debug.min_disp_clk_khz > 0 && requested_dispclk_khz < (uint32_t)dc->debug.min_disp_clk_khz)
+			requested_dispclk_khz = (uint32_t)dc->debug.min_disp_clk_khz;
 
 		dcn42_smu_set_dispclk(clk_mgr, requested_dispclk_khz);
 		clk_mgr_base->clks.dispclk_khz = new_clocks->dispclk_khz;
@@ -634,7 +640,7 @@ static void dcn42_read_ss_info_from_lut(struct clk_mgr_internal *clk_mgr)
 
 void dcn42_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn42_watermarks *table)
 {
-	int i, num_valid_sets;
+	uint8_t i, num_valid_sets;
 
 	num_valid_sets = 0;
 
@@ -643,8 +649,10 @@ void dcn42_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn42_
 		if (!bw_params->wm_table.entries[i].valid)
 			continue;
 
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting = bw_params->wm_table.entries[i].wm_inst;
-		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType = bw_params->wm_table.entries[i].wm_type;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmSetting =
+			(uint8_t)bw_params->wm_table.entries[i].wm_inst;
+		table->WatermarkRow[WM_DCFCLK][num_valid_sets].WmType =
+			(uint8_t)bw_params->wm_table.entries[i].wm_type;
 		/* We will not select WM based on fclk, so leave it as unconstrained */
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinClock = 0;
 		table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxClock = 0xFFFF;
@@ -655,10 +663,10 @@ void dcn42_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn42_
 			else {
 				/* add 1 to make it non-overlapping with next lvl */
 				table->WatermarkRow[WM_DCFCLK][num_valid_sets].MinMclk =
-						bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1;
+						(uint16_t)(bw_params->clk_table.entries[i - 1].dcfclk_mhz + 1);
 			}
 			table->WatermarkRow[WM_DCFCLK][num_valid_sets].MaxMclk =
-					bw_params->clk_table.entries[i].dcfclk_mhz;
+					(uint16_t)bw_params->clk_table.entries[i].dcfclk_mhz;
 
 		} else {
 			/* unconstrained for memory retraining */
@@ -690,7 +698,7 @@ void dcn42_build_watermark_ranges(struct clk_bw_params *bw_params, struct dcn42_
 
 void dcn42_notify_wm_ranges(struct clk_mgr *clk_mgr_base)
 {
-	int i = 0;
+	unsigned int i = 0;
 	struct dcn42_watermarks *table = NULL;
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	struct clk_mgr_dcn42 *clk_mgr_dcn42 = TO_CLK_MGR_DCN42(clk_mgr);
@@ -734,9 +742,12 @@ void dcn42_notify_wm_ranges(struct clk_mgr *clk_mgr_base)
 			clk_mgr_dcn42->smu_wm_set.mc_address.low_part);
 	dcn42_smu_transfer_wm_table_dram_2_smu(clk_mgr);
 
-	if (clk_mgr_dcn42->smu_wm_set.wm_set && clk_mgr_dcn42->smu_wm_set.mc_address.quad_part != 0)
+	if (clk_mgr_dcn42->smu_wm_set.wm_set && clk_mgr_dcn42->smu_wm_set.mc_address.quad_part != 0) {
 		dm_helpers_free_gpu_mem(clk_mgr->base.ctx, DC_MEM_ALLOC_TYPE_GART,
 				clk_mgr_dcn42->smu_wm_set.wm_set);
+		clk_mgr_dcn42->smu_wm_set.wm_set = NULL;
+		clk_mgr_dcn42->smu_wm_set.mc_address.quad_part = 0;
+	}
 
 }
 
@@ -870,7 +881,7 @@ int dcn42_get_dispclk_from_dentist(struct clk_mgr *clk_mgr_base)
 	(void)clk_mgr_base;
 	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
 	uint32_t dispclk_wdivider;
-	int disp_divider;
+	unsigned int disp_divider;
 
 	REG_GET(DENTIST_DISPCLK_CNTL, DENTIST_DISPCLK_WDIVIDER, &dispclk_wdivider);
 	disp_divider = dentist_get_divider_from_did(dispclk_wdivider);
@@ -1015,7 +1026,7 @@ static struct clk_mgr_funcs dcn42_funcs = {
 	.init_clocks = dcn42_init_clocks,
 	.enable_pme_wa = dcn42_enable_pme_wa,
 	.are_clock_states_equal = dcn42_are_clock_states_equal,
-	.notify_wm_ranges = dcn42_notify_wm_ranges,
+	.notify_wm_ranges = NULL,
 	.set_low_power_state = dcn42_set_low_power_state,
 	.exit_low_power_state = dcn42_exit_low_power_state,
 	.get_max_clock_khz = dcn42_get_max_clock_khz,
@@ -1069,10 +1080,11 @@ void dcn42_clk_mgr_construct(
 
 			dcn42_bw_params.vram_type = ctx->dc_bios->integrated_info->memory_type;
 			dcn42_bw_params.dram_channel_width_bytes = ctx->dc_bios->integrated_info->memory_type == 0x22 ? 8 : 4;
-			dcn42_bw_params.num_channels = ctx->dc_bios->integrated_info->ma_channel_number ? ctx->dc_bios->integrated_info->ma_channel_number : 1;
-			clk_mgr->base.base.dprefclk_khz = dcn42_smu_get_dprefclk(&clk_mgr->base);
-			clk_mgr->base.base.clks.ref_dtbclk_khz = dcn42_smu_get_dtbclk(&clk_mgr->base);
-
+			dcn42_bw_params.num_channels = ctx->dc_bios->integrated_info->ma_channel_number ? ctx->dc_bios->integrated_info->ma_channel_number : 2;
+			if (clk_mgr->base.smu_present) {
+				clk_mgr->base.base.clks.ref_dtbclk_khz = dcn42_smu_get_dtbclk(&clk_mgr->base);
+				clk_mgr->base.base.dprefclk_khz = dcn42_smu_get_dprefclk(&clk_mgr->base);
+			}
 			clk_mgr->base.base.bw_params = &dcn42_bw_params;
 
 			if (clk_mgr->base.smu_present)
@@ -1099,7 +1111,10 @@ void dcn42_clk_mgr_destroy(struct clk_mgr_internal *clk_mgr_int)
 {
 	struct clk_mgr_dcn42 *clk_mgr = TO_CLK_MGR_DCN42(clk_mgr_int);
 
-	if (clk_mgr->smu_wm_set.wm_set && clk_mgr->smu_wm_set.mc_address.quad_part != 0)
+	if (clk_mgr->smu_wm_set.wm_set && clk_mgr->smu_wm_set.mc_address.quad_part != 0) {
 		dm_helpers_free_gpu_mem(clk_mgr_int->base.ctx, DC_MEM_ALLOC_TYPE_GART,
 				clk_mgr->smu_wm_set.wm_set);
+		clk_mgr->smu_wm_set.wm_set = NULL;
+		clk_mgr->smu_wm_set.mc_address.quad_part = 0;
+	}
 }
diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c
index c791bb1edb47e..6d0012b7d6dc3 100644
--- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c
+++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dcn42/dcn42_smu.c
@@ -428,4 +428,3 @@ void dcn42_smu_set_dtbclk(struct clk_mgr_internal *clk_mgr, bool enable)
 			enable);
 	smu_print("%s: smu_set_dtbclk = %d\n", __func__, enable ? 1 : 0);
 }
-
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c
index 419f894c87b03..6d520d5b02e97 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc.c
@@ -32,6 +32,7 @@
 #include "core_types.h"
 #include "hw_sequencer.h"
 #include "dce/dce_hwseq.h"
+#include "hw/dccg.h"
 
 #include "resource.h"
 #include "dc_state.h"
@@ -61,6 +62,7 @@
 #include "link_enc_cfg.h"
 
 #include "link_service.h"
+#include "link/protocols/link_dp_capability.h"
 #include "dm_helpers.h"
 #include "mem_input.h"
 
@@ -172,7 +174,7 @@ static void destroy_links(struct dc *dc)
 
 static uint32_t get_num_of_internal_disp(struct dc_link **links, uint32_t num_links)
 {
-	int i;
+	uint32_t i;
 	uint32_t count = 0;
 
 	for (i = 0; i < num_links; i++) {
@@ -200,7 +202,7 @@ static bool create_links(
 		struct dc *dc,
 		uint32_t num_virtual_links)
 {
-	int i;
+	uint32_t i;
 	int connectors_num;
 	struct dc_bios *bios = dc->ctx->dc_bios;
 
@@ -235,7 +237,7 @@ static bool create_links(
 	 *   variants of the same card.
 	 */
 	for (i = 0; dc->link_count < connectors_num && i < MAX_LINKS; i++) {
-		struct graphics_object_id connector_id = bios->funcs->get_connector_id(bios, i);
+		struct graphics_object_id connector_id = bios->funcs->get_connector_id(bios, (uint8_t)i);
 		struct link_init_data link_init_params = {0};
 		struct dc_link *link;
 
@@ -246,7 +248,7 @@ static bool create_links(
 
 		link_init_params.ctx = dc->ctx;
 		/* next BIOS object table connector */
-		link_init_params.connector_index = i;
+		link_init_params.connector_index = (uint8_t)i;
 		link_init_params.link_index = dc->link_count;
 		link_init_params.dc = dc;
 		link = dc->link_srv->create_link(&link_init_params);
@@ -267,7 +269,7 @@ static bool create_links(
 		struct dc_link *link;
 
 		link_init_params.ctx = dc->ctx;
-		link_init_params.connector_index = i;
+		link_init_params.connector_index = (uint8_t)i;
 		link_init_params.link_index = dc->link_count;
 		link_init_params.dc = dc;
 		link_init_params.is_dpia_link = true;
@@ -341,7 +343,7 @@ static bool create_link_encoders(struct dc *dc)
 	bool res = true;
 	unsigned int num_usb4_dpia = dc->res_pool->res_cap->num_usb4_dpia;
 	unsigned int num_dig_link_enc = dc->res_pool->res_cap->num_dig_link_enc;
-	int i;
+	unsigned int i;
 
 	/* A platform without USB4 DPIA endpoints has a fixed mapping between DIG
 	 * link encoders and physical display endpoints and does not require
@@ -381,7 +383,7 @@ static void destroy_link_encoders(struct dc *dc)
 {
 	unsigned int num_usb4_dpia;
 	unsigned int num_dig_link_enc;
-	int i;
+	unsigned int i;
 
 	if (!dc->res_pool)
 		return;
@@ -659,12 +661,12 @@ bool dc_stream_configure_crc(struct dc *dc, struct dc_stream_state *stream,
 	/* By default, capture the full frame */
 	param.windowa_x_start = 0;
 	param.windowa_y_start = 0;
-	param.windowa_x_end = pipe->stream->timing.h_addressable;
-	param.windowa_y_end = pipe->stream->timing.v_addressable;
+	param.windowa_x_end = (uint16_t)pipe->stream->timing.h_addressable;
+	param.windowa_y_end = (uint16_t)pipe->stream->timing.v_addressable;
 	param.windowb_x_start = 0;
 	param.windowb_y_start = 0;
-	param.windowb_x_end = pipe->stream->timing.h_addressable;
-	param.windowb_y_end = pipe->stream->timing.v_addressable;
+	param.windowb_x_end = (uint16_t)pipe->stream->timing.h_addressable;
+	param.windowb_y_end = (uint16_t)pipe->stream->timing.v_addressable;
 	param.crc_poly_mode = crc_poly_mode;
 
 	if (crc_window) {
@@ -1136,7 +1138,7 @@ static void disable_all_writeback_pipes_for_stream(
 {
 	(void)dc;
 	(void)context;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < stream->num_wb_info; i++)
 		stream->writeback_info[i].wb_enabled = false;
@@ -1149,7 +1151,7 @@ static void apply_ctx_interdependent_lock(struct dc *dc,
 {
 	(void)dc;
 	(void)context;
-	int i;
+	unsigned int i;
 
 	/* Checks if interdependent update function pointer is NULL or not, takes care of DCE110 case */
 	if (dc->hwss.interdependent_update_lock)
@@ -1336,7 +1338,7 @@ bool dc_stream_adjust_vmin_vmax(struct dc *dc,
 
 static void disable_dangling_plane(struct dc *dc, struct dc_state *context)
 {
-	int i, j;
+	unsigned int i, j;
 	struct dc_state *dangling_context = dc_state_create_current_copy(dc);
 	struct dc_state *current_ctx;
 	struct pipe_ctx *pipe;
@@ -1799,7 +1801,8 @@ bool dc_validate_boot_timing(const struct dc *dc,
 	struct dc_crtc_timing hw_crtc_timing = {0};
 
 	struct dc_link *link = sink->link;
-	unsigned int i, enc_inst, tg_inst = 0;
+	unsigned int i, enc_inst;
+	unsigned int tg_inst = 0;
 
 	/* Support seamless boot on EDP displays only */
 	if (sink->sink_signal != SIGNAL_TYPE_EDP) {
@@ -1928,7 +1931,7 @@ bool dc_validate_boot_timing(const struct dc *dc,
 		struct dcn_dsc_state dsc_state = {0};
 
 		/* Find DSC associated with this timing generator */
-		if (tg_inst < dc->res_pool->res_cap->num_dsc) {
+		if (tg_inst < (unsigned int)dc->res_pool->res_cap->num_dsc) {
 			dsc = dc->res_pool->dscs[tg_inst];
 		}
 
@@ -2129,7 +2132,7 @@ void dc_trigger_sync(struct dc *dc, struct dc_state *context)
 
 static uint8_t get_stream_mask(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	unsigned int stream_mask = 0;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -2137,7 +2140,7 @@ static uint8_t get_stream_mask(struct dc *dc, struct dc_state *context)
 			stream_mask |= 1 << i;
 	}
 
-	return stream_mask;
+	return (uint8_t)stream_mask;
 }
 
 void dc_z10_restore(const struct dc *dc)
@@ -2203,7 +2206,7 @@ static enum dc_status dc_commit_state_no_check(struct dc *dc, struct dc_state *c
 	struct dc_bios *dcb = dc->ctx->dc_bios;
 	enum dc_status result = DC_ERROR_UNEXPECTED;
 	struct pipe_ctx *pipe;
-	int i, k, l;
+	unsigned int i, k, l;
 	struct dc_stream_state *dc_streams[MAX_STREAMS] = {0};
 	struct dc_state *old_state;
 	bool subvp_prev_use = false;
@@ -2443,7 +2446,7 @@ static bool commit_minimal_transition_state(struct dc *dc,
  */
 enum dc_status dc_commit_streams(struct dc *dc, struct dc_commit_streams_params *params)
 {
-	int i, j;
+	unsigned int i, j;
 	struct dc_state *context;
 	enum dc_status res = DC_OK;
 	struct dc_validation_set set[MAX_STREAMS] = {0};
@@ -2482,8 +2485,8 @@ enum dc_status dc_commit_streams(struct dc *dc, struct dc_commit_streams_params
 		set[i].stream = stream;
 
 		if (status) {
-			set[i].plane_count = status->plane_count;
-			for (j = 0; j < status->plane_count; j++)
+			set[i].plane_count = (uint8_t)status->plane_count;
+			for (j = 0; j < (unsigned int)status->plane_count; j++)
 				set[i].plane_states[j] = status->plane_states[j];
 		}
 	}
@@ -2541,7 +2544,7 @@ enum dc_status dc_commit_streams(struct dc *dc, struct dc_commit_streams_params
 	for (i = 0; i < params->stream_count; i++) {
 		for (j = 0; j < context->stream_count; j++) {
 			if (params->streams[i]->stream_id == context->streams[j]->stream_id)
-				params->streams[i]->out.otg_offset = context->stream_status[j].primary_otg_inst;
+				params->streams[i]->out.otg_offset = (uint8_t)context->stream_status[j].primary_otg_inst;
 
 			if (dc_is_embedded_signal(params->streams[i]->signal)) {
 				struct dc_stream_status *status = dc_state_get_stream_status(context, params->streams[i]);
@@ -2573,7 +2576,7 @@ bool dc_acquire_release_mpc_3dlut(
 		struct dc_3dlut **lut,
 		struct dc_transfer_func **shaper)
 {
-	int pipe_idx;
+	unsigned int pipe_idx;
 	bool ret = false;
 	bool found_pipe_idx = false;
 	const struct resource_pool *pool = dc->res_pool;
@@ -2645,7 +2648,7 @@ static void process_deferred_updates(struct dc *dc)
 
 void dc_post_update_surfaces_to_stream(struct dc *dc)
 {
-	int i;
+	unsigned int i;
 	struct dc_state *context = dc->current_state;
 
 	if ((!dc->optimized_required) || get_seamless_boot_stream_count(context) > 0)
@@ -2669,7 +2672,7 @@ void dc_post_update_surfaces_to_stream(struct dc *dc)
 		for (i = 0; i < dc->res_pool->pipe_count; i++)
 			if (context->res_ctx.pipe_ctx[i].stream == NULL ||
 					context->res_ctx.pipe_ctx[i].plane_state == NULL) {
-				context->res_ctx.pipe_ctx[i].pipe_idx = i;
+				context->res_ctx.pipe_ctx[i].pipe_idx = (uint8_t)i;
 				dc->hwss.disable_plane(dc, context, &context->res_ctx.pipe_ctx[i]);
 			}
 
@@ -2885,7 +2888,7 @@ static struct surface_update_descriptor get_scaling_info_update_type(
 			/* Making dst rect smaller requires a bandwidth change */
 			update_flags->bits.bandwidth_change = 1;
 
-		if (u->scaling_info->src_rect.width > check_config->max_optimizable_video_width &&
+		if (u->scaling_info->src_rect.width > (int)check_config->max_optimizable_video_width &&
 			(u->scaling_info->clip_rect.width > u->surface->clip_rect.width ||
 			 u->scaling_info->clip_rect.height > u->surface->clip_rect.height))
 			 /* Changing clip size of a large surface may result in MPC slice count change */
@@ -2946,6 +2949,11 @@ static struct surface_update_descriptor det_surface_update(
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
+	if (u->cursor_csc_color_matrix) {
+		update_flags->bits.cursor_csc_color_matrix_change = 1;
+		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+	}
+
 	if (u->coeff_reduction_factor) {
 		update_flags->bits.coeff_reduction_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
@@ -2956,27 +2964,16 @@ static struct surface_update_descriptor det_surface_update(
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
-	if (u->cm || (u->gamma && dce_use_lut(u->plane_info ? u->plane_info->format : u->surface->format))) {
+	if (u->blend_tf || (u->gamma && dce_use_lut(u->plane_info ? u->plane_info->format : u->surface->format))) {
 		update_flags->bits.gamma_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
-	if (u->cm && (u->cm->flags.bits.lut3d_enable || u->surface->cm.flags.bits.lut3d_enable)) {
+	if (u->lut3d_func || u->func_shaper) {
 		update_flags->bits.lut_3d = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
 
-	if (u->cm && u->cm->flags.bits.lut3d_dma_enable != u->surface->cm.flags.bits.lut3d_dma_enable &&
-			u->cm->flags.bits.lut3d_enable && u->surface->cm.flags.bits.lut3d_enable) {
-		/* Toggling 3DLUT loading between DMA and Host is illegal */
-		BREAK_TO_DEBUGGER();
-	}
-
-	if (u->cm && u->cm->flags.bits.lut3d_enable && !u->cm->flags.bits.lut3d_dma_enable) {
-		/* Host loading 3DLUT requires full update but only stream lock  */
-		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_STREAM);
-	}
-
 	if (u->hdr_mult.value)
 		if (u->hdr_mult.value != u->surface->hdr_mult.value) {
 			// TODO: Should be fast?
@@ -2995,6 +2992,20 @@ static struct surface_update_descriptor det_surface_update(
 		update_flags->bits.cm_hist_change = 1;
 		elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 	}
+	if (u->cm2_params) {
+		if (u->cm2_params->component_settings.shaper_3dlut_setting != u->surface->mcm_shaper_3dlut_setting
+				|| u->cm2_params->component_settings.lut1d_enable != u->surface->mcm_lut1d_enable
+				|| u->cm2_params->cm2_luts.lut3d_data.lut3d_src != u->surface->mcm_luts.lut3d_data.lut3d_src) {
+			update_flags->bits.mcm_transfer_function_enable_change = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+		}
+	}
+
+	if (update_flags->bits.lut_3d &&
+			u->surface->mcm_luts.lut3d_data.lut3d_src != DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
+		elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
+	}
+
 	if (check_config->enable_legacy_fast_update &&
 			(update_flags->bits.gamma_change ||
 			update_flags->bits.gamut_remap_change ||
@@ -3073,7 +3084,7 @@ static struct surface_update_descriptor check_update_surfaces_for_stream(
 			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL | LOCK_DESCRIPTOR_LINK);
 		}
 
-		if (stream_update->gamut_remap)
+		if (check_config->enable_legacy_fast_update && stream_update->gamut_remap)
 			su_flags->bits.gamut_remap = 1;
 
 		if (stream_update->wb_update)
@@ -3104,6 +3115,29 @@ static struct surface_update_descriptor check_update_surfaces_for_stream(
 			elevate_update_type(&overall_type, UPDATE_TYPE_FULL, LOCK_DESCRIPTOR_GLOBAL);
 
 		// Non-global cases
+
+		if (stream_update->gamut_remap) {
+			su_flags->bits.gamut_remap = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if ((stream_update->hdr_static_metadata && !stream_update->stream->use_dynamic_meta) ||
+				stream_update->vrr_infopacket ||
+				stream_update->vsc_infopacket ||
+				stream_update->vsp_infopacket ||
+				stream_update->hfvsif_infopacket ||
+				stream_update->adaptive_sync_infopacket ||
+				stream_update->vtem_infopacket ||
+				stream_update->avi_infopacket) {
+			su_flags->bits.info_frame = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->hdr_static_metadata && stream_update->stream->use_dynamic_meta) {
+			su_flags->bits.dmdata = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
 		if (stream_update->output_csc_transform) {
 			su_flags->bits.out_csc = 1;
 			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
@@ -3113,6 +3147,26 @@ static struct surface_update_descriptor check_update_surfaces_for_stream(
 			su_flags->bits.out_tf = 1;
 			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
 		}
+
+		if (stream_update->periodic_interrupt) {
+			su_flags->bits.periodic_interrupt = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->dither_option) {
+			su_flags->bits.dither = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->cursor_attributes) {
+			su_flags->bits.cursor_attr = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
+
+		if (stream_update->cursor_position) {
+			su_flags->bits.cursor_pos = 1;
+			elevate_update_type(&overall_type, UPDATE_TYPE_FAST, LOCK_DESCRIPTOR_STREAM);
+		}
 	}
 
 	for (int i = 0 ; i < surface_count; i++) {
@@ -3170,10 +3224,10 @@ static void copy_surface_update_to_plane(
 		surface->flip_immediate =
 			srf_update->flip_addr->flip_immediate;
 		surface->time.time_elapsed_in_us[surface->time.index] =
-			srf_update->flip_addr->flip_timestamp_in_us -
-				surface->time.prev_update_time_in_us;
+			(unsigned int)(srf_update->flip_addr->flip_timestamp_in_us -
+				surface->time.prev_update_time_in_us);
 		surface->time.prev_update_time_in_us =
-			srf_update->flip_addr->flip_timestamp_in_us;
+			(unsigned int)srf_update->flip_addr->flip_timestamp_in_us;
 		surface->time.index++;
 		if (surface->time.index >= DC_PLANE_UPDATE_TIMES_MAX)
 			surface->time.index = 0;
@@ -3250,12 +3304,24 @@ static void copy_surface_update_to_plane(
 			sizeof(struct dc_transfer_func_distributed_points));
 	}
 
-	/* Shaper, 3DLUT, 1DLUT */
-	if (srf_update->cm) {
-		memcpy(&surface->cm, srf_update->cm,
-				sizeof(surface->cm));
+	if (srf_update->cm2_params) {
+		surface->mcm_shaper_3dlut_setting = srf_update->cm2_params->component_settings.shaper_3dlut_setting;
+		surface->mcm_lut1d_enable = srf_update->cm2_params->component_settings.lut1d_enable;
+		surface->mcm_luts = srf_update->cm2_params->cm2_luts;
+	}
+
+	if (srf_update->func_shaper) {
+		memcpy(&surface->in_shaper_func, srf_update->func_shaper,
+		sizeof(surface->in_shaper_func));
+
+		if (surface->mcm_shaper_3dlut_setting >= DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER)
+			surface->mcm_luts.shaper = &surface->in_shaper_func;
 	}
 
+	if (srf_update->lut3d_func)
+		memcpy(&surface->lut3d_func, srf_update->lut3d_func,
+		sizeof(surface->lut3d_func));
+
 	if (srf_update->hdr_mult.value)
 		surface->hdr_mult =
 				srf_update->hdr_mult;
@@ -3264,6 +3330,17 @@ static void copy_surface_update_to_plane(
 		surface->sdr_white_level_nits =
 				srf_update->sdr_white_level_nits;
 
+	if (srf_update->blend_tf) {
+		memcpy(&surface->blend_tf, srf_update->blend_tf,
+		sizeof(surface->blend_tf));
+
+		if (surface->mcm_lut1d_enable)
+			surface->mcm_luts.lut1d_func = &surface->blend_tf;
+	}
+
+	if (srf_update->cm2_params || srf_update->blend_tf)
+		surface->lut_bank_a = !surface->lut_bank_a;
+
 	if (srf_update->input_csc_color_matrix)
 		surface->input_csc_color_matrix =
 			*srf_update->input_csc_color_matrix;
@@ -3387,7 +3464,7 @@ static void copy_stream_update_to_stream(struct dc *dc,
 		stream->test_pattern = *update->pending_test_pattern;
 	/* update current stream with writeback info */
 	if (update->wb_update) {
-		int i;
+		unsigned int i;
 
 		stream->num_wb_info = update->wb_update->num_wb_info;
 		ASSERT(stream->num_wb_info <= MAX_DWB_PIPES);
@@ -3582,7 +3659,8 @@ static bool update_planes_and_stream_state(struct dc *dc,
 		struct dc_state **new_context)
 {
 	struct dc_state *context;
-	int i, j;
+	int i;
+	unsigned int j;
 	enum surface_update_type update_type;
 	const struct dc_stream_status *stream_status;
 	struct dc_context *dc_ctx = dc->ctx;
@@ -3743,14 +3821,433 @@ static bool update_planes_and_stream_state(struct dc *dc,
 
 }
 
+static void program_cursor_attributes_sequence(
+		struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_state *context,
+		struct block_sequence_state *seq_state)
+{
+	int k;
+	struct pipe_ctx *pipe_to_program = NULL;
+	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+
+	for (k = 0; k < (int)dc->res_pool->pipe_count; k++) {
+		struct pipe_ctx *tmp_pipe = &context->res_ctx.pipe_ctx[k];
+
+		if (tmp_pipe->stream != stream)
+			continue;
+
+		if (!pipe_to_program) {
+			pipe_to_program = tmp_pipe;
+
+			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
+				hwss_add_begin_cursor_offload_update(seq_state, dc, tmp_pipe);
+			} else {
+				hwss_add_cursor_lock(seq_state, dc, pipe_to_program, true);
+				if (pipe_to_program->next_odm_pipe)
+					hwss_add_cursor_lock(seq_state, dc, pipe_to_program->next_odm_pipe, true);
+			}
+		}
+
+		hwss_add_set_cursor_attribute(seq_state, dc, tmp_pipe);
+		if (dc->ctx->dmub_srv)
+			hwss_add_send_update_cursor_info_to_dmu(seq_state, tmp_pipe, k);
+		if (dc->hwss.set_cursor_sdr_white_level)
+			hwss_add_set_cursor_sdr_white_level(seq_state, dc, tmp_pipe);
+		if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe)
+			hwss_add_update_cursor_offload_pipe(seq_state, dc, tmp_pipe);
+	}
+
+	if (pipe_to_program) {
+		if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update) {
+			hwss_add_commit_cursor_offload_update(seq_state, dc, pipe_to_program);
+		} else {
+			hwss_add_cursor_lock(seq_state, dc, pipe_to_program, false);
+			if (pipe_to_program->next_odm_pipe)
+				hwss_add_cursor_lock(seq_state, dc, pipe_to_program->next_odm_pipe, false);
+		}
+	}
+}
+
+static void program_cursor_position_sequence(
+		struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_state *context,
+		struct block_sequence_state *seq_state)
+{
+	int k;
+	struct pipe_ctx *pipe_to_program = NULL;
+	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+
+	for (k = 0; k < (int)dc->res_pool->pipe_count; k++) {
+		struct pipe_ctx *tmp_pipe = &context->res_ctx.pipe_ctx[k];
+
+		if (tmp_pipe->stream != stream ||
+				(!tmp_pipe->plane_res.mi  && !tmp_pipe->plane_res.hubp) ||
+				!tmp_pipe->plane_state ||
+				(!tmp_pipe->plane_res.xfm && !tmp_pipe->plane_res.dpp) ||
+				(!tmp_pipe->plane_res.ipp && !tmp_pipe->plane_res.dpp))
+			continue;
+
+		if (!pipe_to_program) {
+			pipe_to_program = tmp_pipe;
+
+			if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update)
+				hwss_add_begin_cursor_offload_update(seq_state, dc, tmp_pipe);
+			else
+				hwss_add_cursor_lock(seq_state, dc, pipe_to_program, true);
+		}
+
+		hwss_add_set_cursor_position(seq_state, dc, tmp_pipe);
+		if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe)
+			hwss_add_update_cursor_offload_pipe(seq_state, dc, tmp_pipe);
+
+		if (dc->ctx->dmub_srv)
+			hwss_add_send_update_cursor_info_to_dmu(seq_state, tmp_pipe, k);
+	}
+
+	if (pipe_to_program) {
+		if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update)
+			hwss_add_commit_cursor_offload_update(seq_state, dc, pipe_to_program);
+		else
+			hwss_add_cursor_lock(seq_state, dc, pipe_to_program, false);
+	}
+}
+
+static void add_update_info_frame_sequence(
+		struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	bool is_hdmi_tmds;
+	bool is_dp;
+
+	if (!pipe_ctx || !pipe_ctx->stream)
+		return;
+
+	if (pipe_ctx->stream_res.stream_enc == NULL)
+		return;
+
+	is_hdmi_tmds = dc_is_hdmi_tmds_signal(pipe_ctx->stream->signal);
+	is_dp = dc_is_dp_signal(pipe_ctx->stream->signal);
+
+	if (!is_hdmi_tmds && !is_dp)
+		return;
+
+	if (is_hdmi_tmds) {
+		hwss_add_stream_enc_update_hdmi_info_packets(seq_state, pipe_ctx);
+		return;
+	}
+
+	if (is_dp) {
+		if (dp_is_128b_132b_signal(pipe_ctx)) {
+			hwss_add_hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num(seq_state, pipe_ctx);
+			hwss_add_hpo_dp_stream_enc_update_dp_info_packets(seq_state, pipe_ctx);
+		} else {
+			hwss_add_stream_enc_update_dp_info_packets_sdp_line_num(seq_state, pipe_ctx);
+			hwss_add_stream_enc_update_dp_info_packets(seq_state, pipe_ctx);
+		}
+	}
+}
+
+static void add_link_update_dsc_config_sequence(
+		struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx,
+		struct dsc_config *dsc_cfg,
+		struct dsc_optc_config *dsc_optc_cfg)
+{
+	struct display_stream_compressor *dsc = pipe_ctx->stream_res.dsc;
+	struct dc_stream_state *stream = pipe_ctx->stream;
+	struct dc *dc = stream->ctx->dc;
+	struct dccg *dccg = dc->res_pool->dccg;
+	struct pipe_ctx *top_pipe = pipe_ctx;
+	struct pipe_ctx *odm_pipe = NULL;
+	int opp_cnt = 1;
+	bool should_use_dto_dscclk = false;
+	struct dsc_config dsc_pps_cfg;
+	uint8_t *dsc_packed_pps = stream->dsc_packed_pps;
+	int last_dsc_set_config_step = 0;
+
+	if (!stream->timing.flags.DSC || !dsc)
+		return;
+
+	while (top_pipe->prev_odm_pipe)
+		top_pipe = top_pipe->prev_odm_pipe;
+
+	for (odm_pipe = top_pipe->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe)
+		opp_cnt++;
+
+	memset(dsc_cfg, 0, sizeof(*dsc_cfg));
+	memset(dsc_optc_cfg, 0, sizeof(*dsc_optc_cfg));
+
+	dsc_cfg->pic_width = (stream->timing.h_addressable +
+		top_pipe->dsc_padding_params.dsc_hactive_padding +
+		stream->timing.h_border_left +
+		stream->timing.h_border_right) / opp_cnt;
+	dsc_cfg->pic_height = stream->timing.v_addressable +
+		stream->timing.v_border_top +
+		stream->timing.v_border_bottom;
+	dsc_cfg->pixel_encoding = stream->timing.pixel_encoding;
+	dsc_cfg->color_depth = stream->timing.display_color_depth;
+	dsc_cfg->is_odm = top_pipe->next_odm_pipe ? true : false;
+	dsc_cfg->dc_dsc_cfg = stream->timing.dsc_cfg;
+	ASSERT(dsc_cfg->dc_dsc_cfg.num_slices_h % opp_cnt == 0);
+	dsc_cfg->dc_dsc_cfg.num_slices_h /= opp_cnt;
+	dsc_cfg->dsc_padding = 0;
+
+	if (dccg && dccg->funcs->set_dto_dscclk &&
+			stream->timing.pix_clk_100hz > 480000)
+		should_use_dto_dscclk = true;
+
+	if (should_use_dto_dscclk)
+		hwss_add_dccg_set_dto_dscclk(seq_state, dccg, dsc->inst,
+			dsc_cfg->dc_dsc_cfg.num_slices_h);
+
+	last_dsc_set_config_step = *seq_state->num_steps;
+	hwss_add_dsc_set_config(seq_state, dsc, dsc_cfg, dsc_optc_cfg);
+	hwss_add_dsc_enable_with_opp(seq_state, top_pipe);
+
+	for (odm_pipe = top_pipe->next_odm_pipe; odm_pipe; odm_pipe = odm_pipe->next_odm_pipe) {
+		struct display_stream_compressor *odm_dsc = odm_pipe->stream_res.dsc;
+
+		if (should_use_dto_dscclk)
+			hwss_add_dccg_set_dto_dscclk(seq_state, dccg, odm_dsc->inst,
+				dsc_cfg->dc_dsc_cfg.num_slices_h);
+
+		last_dsc_set_config_step = *seq_state->num_steps;
+		hwss_add_dsc_set_config(seq_state, odm_dsc, dsc_cfg, dsc_optc_cfg);
+		hwss_add_dsc_enable_with_opp(seq_state, odm_pipe);
+	}
+
+	if (dc_is_dp_signal(stream->signal) && !dp_is_128b_132b_signal(pipe_ctx))
+		hwss_add_stream_enc_dp_set_dsc_config(seq_state,
+			pipe_ctx->stream_res.stream_enc,
+			&seq_state->steps[last_dsc_set_config_step].params.dsc_set_config_simple_params.dsc_optc_cfg);
+
+	hwss_add_tg_set_dsc_config(seq_state, top_pipe->stream_res.tg,
+		&seq_state->steps[last_dsc_set_config_step].params.dsc_set_config_simple_params.dsc_optc_cfg, true);
+
+	memset(&dsc_pps_cfg, 0, sizeof(dsc_pps_cfg));
+	dsc_pps_cfg.pic_width = stream->timing.h_addressable +
+		stream->timing.h_border_left + stream->timing.h_border_right;
+	dsc_pps_cfg.pic_height = stream->timing.v_addressable +
+		stream->timing.v_border_top + stream->timing.v_border_bottom;
+	dsc_pps_cfg.pixel_encoding = stream->timing.pixel_encoding;
+	dsc_pps_cfg.color_depth = stream->timing.display_color_depth;
+	dsc_pps_cfg.is_odm = top_pipe->next_odm_pipe ? true : false;
+	dsc_pps_cfg.dc_dsc_cfg = stream->timing.dsc_cfg;
+	dsc_pps_cfg.dsc_padding = top_pipe->dsc_padding_params.dsc_hactive_padding;
+
+	if (dsc->funcs->dsc_get_packed_pps) {
+		dsc->funcs->dsc_get_packed_pps(dsc, &dsc_pps_cfg, dsc_packed_pps);
+
+		if (dc_is_dp_signal(stream->signal)) {
+			if (dp_is_128b_132b_signal(pipe_ctx))
+				hwss_add_hpo_dp_stream_enc_dp_set_dsc_pps_info_packet(seq_state,
+					pipe_ctx->stream_res.hpo_dp_stream_enc,
+					true, dsc_packed_pps, false);
+			else
+				hwss_add_stream_enc_dp_set_dsc_pps_info_packet(seq_state,
+					pipe_ctx->stream_res.stream_enc,
+					true, dsc_packed_pps, false);
+		}
+	}
+}
+
+static void commit_planes_do_stream_update_sequence(struct dc *dc,
+		struct dc_stream_state *stream,
+		struct dc_stream_update *stream_update,
+		enum surface_update_type update_type,
+		struct dc_state *context,
+		struct block_sequence block_sequence[MAX_HWSS_BLOCK_SEQUENCE_SIZE],
+		unsigned int *num_steps)
+{
+	int j;
+	struct block_sequence_state seq_state = { .steps = block_sequence, .num_steps = num_steps };
+	struct dsc_config dsc_cfgs[MAX_PIPES];
+	struct dsc_optc_config dsc_optc_cfgs[MAX_PIPES];
+	unsigned int dsc_cfg_index = 0;
+	*num_steps = 0; // Initialize to 0
+
+	// Stream updates
+	for (j = 0; j < (int)dc->res_pool->pipe_count; j++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
+
+		if (resource_is_pipe_type(pipe_ctx, OTG_MASTER) && pipe_ctx->stream == stream) {
+
+			if (stream_update->periodic_interrupt && dc->hwss.setup_periodic_interrupt)
+				hwss_add_setup_periodic_interrupt(&seq_state, dc, pipe_ctx);
+
+			if ((stream_update->hdr_static_metadata && !stream->use_dynamic_meta) ||
+					stream_update->vrr_infopacket ||
+					stream_update->vsc_infopacket ||
+					stream_update->vsp_infopacket ||
+					stream_update->hfvsif_infopacket ||
+					stream_update->adaptive_sync_infopacket ||
+					stream_update->vtem_infopacket ||
+					stream_update->avi_infopacket) {
+				resource_build_info_frame(pipe_ctx);
+				add_update_info_frame_sequence(&seq_state, pipe_ctx);
+
+				if (dc_is_dp_signal(pipe_ctx->stream->signal))
+					hwss_add_dp_trace_source_sequence(&seq_state,
+							pipe_ctx->stream->link,
+							DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME);
+			}
+
+			if (stream_update->hdr_static_metadata &&
+					stream->use_dynamic_meta &&
+					dc->hwss.set_dmdata_attributes &&
+					pipe_ctx->stream->dmdata_address.quad_part != 0)
+				hwss_add_set_dmdata_attributes(&seq_state, pipe_ctx);
+
+			if (stream_update->gamut_remap)
+				hwss_add_dpp_program_gamut_remap(&seq_state, pipe_ctx);
+
+			if (stream_update->output_csc_transform)
+				hwss_add_program_output_csc(&seq_state, dc, pipe_ctx,
+					stream->output_color_space,
+					stream->csc_color_matrix.matrix,
+					pipe_ctx->stream_res.opp->inst);
+
+			if (stream_update->dither_option) {
+				struct pipe_ctx *odm_pipe = pipe_ctx->next_odm_pipe;
+				resource_build_bit_depth_reduction_params(pipe_ctx->stream,
+								&pipe_ctx->stream->bit_depth_params);
+				hwss_add_opp_program_fmt(&seq_state, pipe_ctx->stream_res.opp,
+						&stream->bit_depth_params,
+						&stream->clamping);
+				while (odm_pipe) {
+					hwss_add_opp_program_fmt(&seq_state, odm_pipe->stream_res.opp,
+							&stream->bit_depth_params,
+							&stream->clamping);
+					odm_pipe = odm_pipe->next_odm_pipe;
+				}
+			}
+
+			if (stream_update->cursor_attributes)
+				program_cursor_attributes_sequence(dc, stream, context, &seq_state);
+
+			if (stream_update->cursor_position)
+				program_cursor_position_sequence(dc, stream, context, &seq_state);
+
+			/* Full fe update*/
+			if (update_type == UPDATE_TYPE_FAST)
+				continue;
+
+			if (stream_update->dsc_config)
+				if (dsc_cfg_index < MAX_PIPES) {
+					add_link_update_dsc_config_sequence(&seq_state,
+						pipe_ctx,
+						&dsc_cfgs[dsc_cfg_index],
+						&dsc_optc_cfgs[dsc_cfg_index]);
+					dsc_cfg_index++;
+				}
+
+			if (stream_update->mst_bw_update) {
+				if (stream_update->mst_bw_update->is_increase)
+					hwss_add_link_increase_mst_payload(&seq_state,
+							pipe_ctx,
+							stream_update->mst_bw_update->mst_stream_bw);
+				else
+					hwss_add_link_reduce_mst_payload(&seq_state,
+							pipe_ctx,
+							stream_update->mst_bw_update->mst_stream_bw);
+			}
+
+			if (stream_update->pending_test_pattern) {
+				/*
+				 * test pattern params depends on ODM topology
+				 * changes that we could be applying to front
+				 * end. Since at the current stage front end
+				 * changes are not yet applied. We can only
+				 * apply test pattern in hw based on current
+				 * state and populate the final test pattern
+				 * params in new state. If current and new test
+				 * pattern params are different as result of
+				 * different ODM topology being used, it will be
+				 * detected and handle during front end
+				 * programming update.
+				 */
+				hwss_add_dp_set_test_pattern(&seq_state,
+					stream->link,
+					stream->test_pattern.type,
+					stream->test_pattern.color_space,
+					stream->test_pattern.p_link_settings,
+					stream->test_pattern.p_custom_pattern,
+					stream->test_pattern.cust_pattern_size);
+				resource_build_test_pattern_params(&context->res_ctx, pipe_ctx);
+			}
+
+			if (stream_update->dpms_off) {
+				// DPMS should not use partially updated pipe context
+				struct pipe_ctx *dpms_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[j];
+
+				if (*stream_update->dpms_off) {
+					hwss_add_link_set_dpms_off(&seq_state, dpms_pipe_ctx);
+					/* for dpms, keep acquired resources*/
+					if (dpms_pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only)
+						hwss_add_disable_audio_stream(&seq_state, dpms_pipe_ctx);
+
+					hwss_add_dc_set_optimized_required(&seq_state, dc, true);
+
+				} else {
+					if (get_seamless_boot_stream_count(context) == 0)
+						hwss_add_prepare_bandwidth(&seq_state, dc, dc->current_state);
+					hwss_add_link_set_dpms_on(&seq_state, dc->current_state, dpms_pipe_ctx);
+				}
+			} else if (pipe_ctx->stream->link->wa_flags.blank_stream_on_ocs_change && stream_update->output_color_space
+					&& !stream->dpms_off && dc_is_dp_signal(pipe_ctx->stream->signal)) {
+				/*
+				 * Workaround for firmware issue in some receivers where they don't pick up
+				 * correct output color space unless DP link is disabled/re-enabled
+				 */
+					hwss_add_link_set_dpms_on(&seq_state, dc->current_state, pipe_ctx);
+			}
+
+			if (stream_update->abm_level && pipe_ctx->stream_res.abm) {
+				bool should_program_abm = true;
+
+				// if otg funcs defined check if blanked before programming
+				if (pipe_ctx->stream_res.tg->funcs->is_blanked)
+					if (pipe_ctx->stream_res.tg->funcs->is_blanked(pipe_ctx->stream_res.tg))
+						should_program_abm = false;
+
+				if (should_program_abm) {
+					if (*stream_update->abm_level == ABM_LEVEL_IMMEDIATE_DISABLE) {
+						hwss_add_abm_set_immediate_disable(&seq_state, dc, pipe_ctx);
+					} else {
+						hwss_add_abm_set_level(&seq_state, pipe_ctx->stream_res.abm, stream->abm_level);
+					}
+				}
+			}
+		}
+	}
+}
+
 static void commit_planes_do_stream_update(struct dc *dc,
 		struct dc_stream_state *stream,
 		struct dc_stream_update *stream_update,
 		enum surface_update_type update_type,
 		struct dc_state *context)
 {
-	int j;
+	unsigned int j;
+
+	// Check if block sequence programming is enabled
+	if (dc->debug.enable_block_sequence_programming) {
+		unsigned int num_steps = 0;
+
+		// Build the block sequence using context's pre-allocated array
+		commit_planes_do_stream_update_sequence(dc, stream, stream_update,
+				update_type, context, context->block_sequence, &num_steps);
+
+		// Execute the block sequence
+		if (num_steps > 0)
+			hwss_execute_sequence(dc, context->block_sequence, num_steps);
+
+		return;
+	}
 
+	// Legacy path (existing implementation)
 	// Stream updates
 	for (j = 0; j < dc->res_pool->pipe_count; j++) {
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[j];
@@ -3849,27 +4346,34 @@ static void commit_planes_do_stream_update(struct dc *dc,
 				resource_build_test_pattern_params(&context->res_ctx, pipe_ctx);
 			}
 
+			// DPMS should not use partially updated pipe context
+			struct pipe_ctx *dpms_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[j];
+
 			if (stream_update->dpms_off) {
 				if (*stream_update->dpms_off) {
-					dc->link_srv->set_dpms_off(pipe_ctx);
+					dc->link_srv->set_dpms_off(dpms_pipe_ctx);
 					/* for dpms, keep acquired resources*/
-					if (pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only)
-						pipe_ctx->stream_res.audio->funcs->az_disable(pipe_ctx->stream_res.audio);
+					if (dpms_pipe_ctx->stream_res.audio && !dc->debug.az_endpoint_mute_only) {
+						struct audio *audio = dpms_pipe_ctx->stream_res.audio;
+
+						audio->funcs->az_disable(audio);
+					}
 
 					dc->optimized_required = true;
 
 				} else {
 					if (get_seamless_boot_stream_count(context) == 0)
 						dc->hwss.prepare_bandwidth(dc, dc->current_state);
-					dc->link_srv->set_dpms_on(dc->current_state, pipe_ctx);
+					dc->link_srv->set_dpms_on(dc->current_state, dpms_pipe_ctx);
 				}
-			} else if (pipe_ctx->stream->link->wa_flags.blank_stream_on_ocs_change && stream_update->output_color_space
-					&& !stream->dpms_off && dc_is_dp_signal(pipe_ctx->stream->signal)) {
+			} else if (dpms_pipe_ctx->stream->link->wa_flags.blank_stream_on_ocs_change &&
+					stream_update->output_color_space &&
+					!stream->dpms_off && dc_is_dp_signal(dpms_pipe_ctx->stream->signal)) {
 				/*
 				 * Workaround for firmware issue in some receivers where they don't pick up
 				 * correct output color space unless DP link is disabled/re-enabled
 				 */
-				dc->link_srv->set_dpms_on(dc->current_state, pipe_ctx);
+				dc->link_srv->set_dpms_on(dc->current_state, dpms_pipe_ctx);
 			}
 
 			if (stream_update->abm_level && pipe_ctx->stream_res.abm) {
@@ -3918,7 +4422,8 @@ void dc_dmub_update_dirty_rect(struct dc *dc,
 {
 	union dmub_rb_cmd cmd;
 	struct dmub_cmd_update_dirty_rect_data *update_dirty_rect;
-	unsigned int i, j;
+	int i;
+	unsigned int j;
 	unsigned int panel_inst = 0;
 
 	if (!dc_dmub_should_send_dirty_rect_cmd(dc, stream))
@@ -3949,7 +4454,7 @@ void dc_dmub_update_dirty_rect(struct dc *dc,
 		else
 			update_dirty_rect->cmd_version = DMUB_CMD_CURSOR_UPDATE_VERSION_1;
 
-		update_dirty_rect->dirty_rect_count = flip_addr->dirty_rect_count;
+		update_dirty_rect->dirty_rect_count = (uint8_t)flip_addr->dirty_rect_count;
 		memcpy(update_dirty_rect->src_dirty_rects, flip_addr->dirty_rects,
 				sizeof(flip_addr->dirty_rects));
 		for (j = 0; j < dc->res_pool->pipe_count; j++) {
@@ -3960,9 +4465,9 @@ void dc_dmub_update_dirty_rect(struct dc *dc,
 			if (pipe_ctx->plane_state != plane_state)
 				continue;
 
-			update_dirty_rect->panel_inst = panel_inst;
-			update_dirty_rect->pipe_idx = j;
-			update_dirty_rect->otg_inst = pipe_ctx->stream_res.tg->inst;
+			update_dirty_rect->panel_inst = (uint8_t)panel_inst;
+			update_dirty_rect->pipe_idx = (uint8_t)j;
+			update_dirty_rect->otg_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
 			dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_NO_WAIT);
 		}
 	}
@@ -3979,7 +4484,8 @@ static void build_dmub_update_dirty_rect(
 {
 	union dmub_rb_cmd cmd;
 	struct dmub_cmd_update_dirty_rect_data *update_dirty_rect;
-	unsigned int i, j;
+	int i;
+	unsigned int j;
 	unsigned int panel_inst = 0;
 
 	if (!dc_dmub_should_send_dirty_rect_cmd(dc, stream))
@@ -4010,7 +4516,7 @@ static void build_dmub_update_dirty_rect(
 		else
 			update_dirty_rect->cmd_version = DMUB_CMD_CURSOR_UPDATE_VERSION_1;
 
-		update_dirty_rect->dirty_rect_count = flip_addr->dirty_rect_count;
+		update_dirty_rect->dirty_rect_count = (uint8_t)flip_addr->dirty_rect_count;
 		memcpy(update_dirty_rect->src_dirty_rects, flip_addr->dirty_rects,
 				sizeof(flip_addr->dirty_rects));
 		for (j = 0; j < dc->res_pool->pipe_count; j++) {
@@ -4020,9 +4526,9 @@ static void build_dmub_update_dirty_rect(
 				continue;
 			if (pipe_ctx->plane_state != plane_state)
 				continue;
-			update_dirty_rect->panel_inst = panel_inst;
-			update_dirty_rect->pipe_idx = j;
-			update_dirty_rect->otg_inst = pipe_ctx->stream_res.tg->inst;
+			update_dirty_rect->panel_inst = (uint8_t)panel_inst;
+			update_dirty_rect->pipe_idx = (uint8_t)j;
+			update_dirty_rect->otg_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
 			dc_dmub_cmd[*dmub_cmd_count].dmub_cmd = cmd;
 			dc_dmub_cmd[*dmub_cmd_count].wait_type = DM_DMUB_WAIT_TYPE_NO_WAIT;
 			(*dmub_cmd_count)++;
@@ -4076,7 +4582,8 @@ static void commit_plane_for_stream_offload_fams2_flip(struct dc *dc,
 		struct dc_stream_state *stream,
 		struct dc_state *context)
 {
-	int i, j;
+	int i;
+	unsigned int j;
 
 	/* update dirty rect for PSR */
 	dc_dmub_update_dirty_rect(dc, surface_count, stream,
@@ -4117,7 +4624,8 @@ static void commit_planes_for_stream_fast(struct dc *dc,
 		enum surface_update_type update_type,
 		struct dc_state *context)
 {
-	int i, j;
+	int i;
+	unsigned int j;
 	struct pipe_ctx *top_pipe_to_program = NULL;
 	struct dc_stream_status *stream_status = NULL;
 	bool should_offload_fams2_flip = false;
@@ -4157,7 +4665,7 @@ static void commit_planes_for_stream_fast(struct dc *dc,
 	if (!top_pipe_to_program)
 		return;
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < (int)dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 
 		if (pipe->stream && pipe->plane_state) {
@@ -4236,7 +4744,8 @@ static void commit_planes_for_stream(struct dc *dc,
 		enum surface_update_type update_type,
 		struct dc_state *context)
 {
-	int i, j;
+	int i;
+	unsigned int j, pipe_idx;
 	struct pipe_ctx *top_pipe_to_program = NULL;
 	bool should_lock_all_pipes = (update_type != UPDATE_TYPE_FAST);
 	bool subvp_prev_use = false;
@@ -4258,8 +4767,8 @@ static void commit_planes_for_stream(struct dc *dc,
 	if (update_type != UPDATE_TYPE_FAST && dc->res_pool->funcs->prepare_mcache_programming)
 		dc->res_pool->funcs->prepare_mcache_programming(dc, context);
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+	for (pipe_idx = 0; pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[pipe_idx];
 
 		if (pipe->stream && pipe->plane_state) {
 			if (!dc->debug.using_dml2)
@@ -4290,8 +4799,8 @@ static void commit_planes_for_stream(struct dc *dc,
 				stream);
 	ASSERT(top_pipe_to_program != NULL);
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
+	for (pipe_idx = 0; pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+		struct pipe_ctx *old_pipe = &dc->current_state->res_ctx.pipe_ctx[pipe_idx];
 
 		// Check old context for SubVP
 		subvp_prev_use |= (dc_state_get_pipe_subvp_type(dc->current_state, old_pipe) == SUBVP_PHANTOM);
@@ -4299,8 +4808,8 @@ static void commit_planes_for_stream(struct dc *dc,
 			break;
 	}
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+	for (pipe_idx = 0; pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[pipe_idx];
 
 		if (dc_state_get_pipe_subvp_type(context, pipe) == SUBVP_PHANTOM) {
 			subvp_curr_use = true;
@@ -4325,7 +4834,7 @@ static void commit_planes_for_stream(struct dc *dc,
 				struct dmub_hw_lock_inst_flags inst_flags = { 0 };
 
 				hw_locks.bits.lock_dig = 1;
-				inst_flags.dig_inst = top_pipe_to_program->stream_res.tg->inst;
+				inst_flags.dig_inst = (uint8_t)top_pipe_to_program->stream_res.tg->inst;
 
 				dmub_hw_lock_mgr_cmd(dc->ctx->dmub_srv,
 							true,
@@ -4453,7 +4962,7 @@ static void commit_planes_for_stream(struct dc *dc,
 		if (!pipe_ctx->top_pipe &&
 			!pipe_ctx->prev_odm_pipe &&
 			should_update_pipe_for_stream(context, pipe_ctx, stream)) {
-			struct dc_stream_status *stream_status = NULL;
+			struct dc_stream_status *pipe_stream_status = NULL;
 
 			if (!pipe_ctx->plane_state)
 				continue;
@@ -4462,12 +4971,12 @@ static void commit_planes_for_stream(struct dc *dc,
 			if (update_type == UPDATE_TYPE_FAST)
 				continue;
 
-			stream_status =
+			pipe_stream_status =
 				stream_get_status(context, pipe_ctx->stream);
 
-			if (dc->hwss.apply_ctx_for_surface && stream_status)
+			if (dc->hwss.apply_ctx_for_surface && pipe_stream_status)
 				dc->hwss.apply_ctx_for_surface(
-					dc, pipe_ctx->stream, stream_status->plane_count, context);
+					dc, pipe_ctx->stream, pipe_stream_status->plane_count, context);
 		}
 	}
 
@@ -4502,16 +5011,16 @@ static void commit_planes_for_stream(struct dc *dc,
 		}
 
 		if (dc->debug.validate_dml_output) {
-			for (i = 0; i < dc->res_pool->pipe_count; i++) {
-				struct pipe_ctx *cur_pipe = &context->res_ctx.pipe_ctx[i];
+			for (pipe_idx = 0; pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+				struct pipe_ctx *cur_pipe = &context->res_ctx.pipe_ctx[pipe_idx];
 				if (cur_pipe->stream == NULL)
 					continue;
 
 				cur_pipe->plane_res.hubp->funcs->validate_dml_output(
 						cur_pipe->plane_res.hubp, dc->ctx,
-						&context->res_ctx.pipe_ctx[i].rq_regs,
-						&context->res_ctx.pipe_ctx[i].dlg_regs,
-						&context->res_ctx.pipe_ctx[i].ttu_regs);
+						&context->res_ctx.pipe_ctx[pipe_idx].rq_regs,
+						&context->res_ctx.pipe_ctx[pipe_idx].dlg_regs,
+						&context->res_ctx.pipe_ctx[pipe_idx].ttu_regs);
 			}
 		}
 	}
@@ -4550,9 +5059,11 @@ static void commit_planes_for_stream(struct dc *dc,
 				if (!should_update_pipe_for_plane(context, pipe_ctx, plane_state))
 					continue;
 
-				if (srf_updates[i].cm &&
-						srf_updates[i].cm->flags.bits.lut3d_enable &&
-						srf_updates[i].cm->flags.bits.lut3d_dma_enable &&
+				if (srf_updates[i].cm2_params &&
+						srf_updates[i].cm2_params->cm2_luts.lut3d_data.lut3d_src ==
+								DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM &&
+						srf_updates[i].cm2_params->component_settings.shaper_3dlut_setting ==
+								DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT &&
 						dc->hwss.trigger_3dlut_dma_load)
 					dc->hwss.trigger_3dlut_dma_load(dc, pipe_ctx);
 
@@ -4592,7 +5103,7 @@ static void commit_planes_for_stream(struct dc *dc,
 				struct dmub_hw_lock_inst_flags inst_flags = { 0 };
 
 				hw_locks.bits.lock_dig = 1;
-				inst_flags.dig_inst = top_pipe_to_program->stream_res.tg->inst;
+				inst_flags.dig_inst = (uint8_t)top_pipe_to_program->stream_res.tg->inst;
 
 				dmub_hw_lock_mgr_cmd(dc->ctx->dmub_srv,
 							false,
@@ -4831,7 +5342,7 @@ static void swap_and_release_current_context(struct dc *dc,
 		struct dc_state *new_context, struct dc_stream_state *stream)
 {
 
-	int i;
+	unsigned int i;
 	struct dc_state *old = dc->current_state;
 	struct pipe_ctx *pipe_ctx;
 
@@ -5091,7 +5602,7 @@ static bool commit_minimal_transition_state(struct dc *dc,
 
 	/* force full surface update */
 	for (i = 0; i < dc->current_state->stream_count; i++) {
-		for (j = 0; j < dc->current_state->stream_status[i].plane_count; j++) {
+		for (j = 0; j < (unsigned int)dc->current_state->stream_status[i].plane_count; j++) {
 			dc->current_state->stream_status[i].plane_states[j]->update_flags.raw = 0xFFFFFFFF;
 		}
 	}
@@ -5109,9 +5620,35 @@ void populate_fast_updates(struct dc_fast_update *fast_update,
 	if (stream_update) {
 		fast_update[0].out_transfer_func = stream_update->out_transfer_func;
 		fast_update[0].output_csc_transform = stream_update->output_csc_transform;
+		fast_update[0].cursor_attributes = stream_update->cursor_attributes;
+		fast_update[0].cursor_position = stream_update->cursor_position;
+		fast_update[0].periodic_interrupt = stream_update->periodic_interrupt;
+		fast_update[0].dither_option = stream_update->dither_option;
+		fast_update[0].gamut_remap = stream_update->gamut_remap;
+		fast_update[0].vrr_infopacket = stream_update->vrr_infopacket;
+		fast_update[0].vsc_infopacket = stream_update->vsc_infopacket;
+		fast_update[0].vsp_infopacket = stream_update->vsp_infopacket;
+		fast_update[0].hfvsif_infopacket = stream_update->hfvsif_infopacket;
+		fast_update[0].vtem_infopacket = stream_update->vtem_infopacket;
+		fast_update[0].adaptive_sync_infopacket = stream_update->adaptive_sync_infopacket;
+		fast_update[0].avi_infopacket = stream_update->avi_infopacket;
+		fast_update[0].hdr_static_metadata = stream_update->hdr_static_metadata;
 	} else {
 		fast_update[0].out_transfer_func = NULL;
 		fast_update[0].output_csc_transform = NULL;
+		fast_update[0].cursor_attributes = NULL;
+		fast_update[0].cursor_position = NULL;
+		fast_update[0].periodic_interrupt = NULL;
+		fast_update[0].dither_option = NULL;
+		fast_update[0].gamut_remap = NULL;
+		fast_update[0].vrr_infopacket = NULL;
+		fast_update[0].vsc_infopacket = NULL;
+		fast_update[0].vsp_infopacket = NULL;
+		fast_update[0].hfvsif_infopacket = NULL;
+		fast_update[0].vtem_infopacket = NULL;
+		fast_update[0].adaptive_sync_infopacket = NULL;
+		fast_update[0].avi_infopacket = NULL;
+		fast_update[0].hdr_static_metadata = NULL;
 	}
 
 	for (i = 0; i < surface_count; i++) {
@@ -5121,9 +5658,7 @@ void populate_fast_updates(struct dc_fast_update *fast_update,
 		fast_update[i].input_csc_color_matrix = srf_updates[i].input_csc_color_matrix;
 		fast_update[i].coeff_reduction_factor = srf_updates[i].coeff_reduction_factor;
 		fast_update[i].cursor_csc_color_matrix = srf_updates[i].cursor_csc_color_matrix;
-#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
 		fast_update[i].cm_hist_control = srf_updates[i].cm_hist_control;
-#endif
 	}
 }
 
@@ -5132,7 +5667,20 @@ static bool fast_updates_exist(const struct dc_fast_update *fast_update, int sur
 	int i;
 
 	if (fast_update[0].out_transfer_func ||
-		fast_update[0].output_csc_transform)
+		fast_update[0].output_csc_transform ||
+		fast_update[0].cursor_attributes ||
+		fast_update[0].cursor_position ||
+		fast_update[0].periodic_interrupt ||
+		fast_update[0].dither_option ||
+		fast_update[0].gamut_remap ||
+		fast_update[0].vrr_infopacket ||
+		fast_update[0].vsc_infopacket ||
+		fast_update[0].vsp_infopacket ||
+		fast_update[0].hfvsif_infopacket ||
+		fast_update[0].vtem_infopacket ||
+		fast_update[0].adaptive_sync_infopacket ||
+		fast_update[0].avi_infopacket ||
+		fast_update[0].hdr_static_metadata)
 		return true;
 
 	for (i = 0; i < surface_count; i++) {
@@ -5141,9 +5689,7 @@ static bool fast_updates_exist(const struct dc_fast_update *fast_update, int sur
 				fast_update[i].gamut_remap_matrix ||
 				fast_update[i].input_csc_color_matrix ||
 				fast_update[i].cursor_csc_color_matrix ||
-#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
 				fast_update[i].cm_hist_control ||
-#endif
 				fast_update[i].coeff_reduction_factor)
 			return true;
 	}
@@ -5156,7 +5702,20 @@ bool fast_nonaddr_updates_exist(struct dc_fast_update *fast_update, int surface_
 	int i;
 
 	if (fast_update[0].out_transfer_func ||
-		fast_update[0].output_csc_transform)
+		fast_update[0].output_csc_transform ||
+		fast_update[0].gamut_remap ||
+		fast_update[0].cursor_attributes ||
+		fast_update[0].cursor_position ||
+		fast_update[0].periodic_interrupt ||
+		fast_update[0].dither_option ||
+		fast_update[0].vrr_infopacket ||
+		fast_update[0].vsc_infopacket ||
+		fast_update[0].vsp_infopacket ||
+		fast_update[0].hfvsif_infopacket ||
+		fast_update[0].vtem_infopacket ||
+		fast_update[0].adaptive_sync_infopacket ||
+		fast_update[0].avi_infopacket ||
+		fast_update[0].hdr_static_metadata)
 		return true;
 
 	for (i = 0; i < surface_count; i++) {
@@ -5164,9 +5723,7 @@ bool fast_nonaddr_updates_exist(struct dc_fast_update *fast_update, int surface_
 				fast_update[i].gamma ||
 				fast_update[i].gamut_remap_matrix ||
 				fast_update[i].coeff_reduction_factor ||
-#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
 				fast_update[i].cm_hist_control ||
-#endif
 				fast_update[i].cursor_csc_color_matrix)
 			return true;
 	}
@@ -5209,12 +5766,6 @@ static bool full_update_required(
 		const struct dc_stream_update *stream_update,
 		const struct dc_stream_state *stream)
 {
-	const union dc_plane_cm_flags blend_only_flags = {
-		.bits = {
-			.blend_enable = 1,
-		}
-	};
-
 	if (full_update_required_weak(dc, srf_updates, surface_count, stream_update, stream))
 		return true;
 
@@ -5227,12 +5778,14 @@ static bool full_update_required(
 				(srf_updates[i].sdr_white_level_nits &&
 				srf_updates[i].sdr_white_level_nits != srf_updates->surface->sdr_white_level_nits) ||
 				srf_updates[i].in_transfer_func ||
+				srf_updates[i].func_shaper ||
+				srf_updates[i].lut3d_func ||
 				srf_updates[i].surface->force_full_update ||
 				(srf_updates[i].flip_addr &&
 				srf_updates[i].flip_addr->address.tmz_surface != srf_updates[i].surface->address.tmz_surface) ||
-				(srf_updates[i].cm &&
-				((srf_updates[i].cm->flags.all != blend_only_flags.all && srf_updates[i].cm->flags.all != 0) ||
-				(srf_updates[i].surface->cm.flags.all != blend_only_flags.all && srf_updates[i].surface->cm.flags.all != 0)))))
+				(srf_updates[i].cm2_params &&
+				 (srf_updates[i].cm2_params->component_settings.shaper_3dlut_setting != srf_updates[i].surface->mcm_shaper_3dlut_setting ||
+				  srf_updates[i].cm2_params->component_settings.lut1d_enable != srf_updates[i].surface->mcm_lut1d_enable))))
 			return true;
 	}
 
@@ -5240,23 +5793,12 @@ static bool full_update_required(
 			(((stream_update->src.height != 0 && stream_update->src.width != 0) ||
 			(stream_update->dst.height != 0 && stream_update->dst.width != 0) ||
 			stream_update->integer_scaling_update) ||
-			stream_update->hdr_static_metadata ||
 			stream_update->abm_level ||
-			stream_update->periodic_interrupt ||
-			stream_update->vrr_infopacket ||
-			stream_update->vsc_infopacket ||
-			stream_update->vsp_infopacket ||
-			stream_update->hfvsif_infopacket ||
-			stream_update->vtem_infopacket ||
-			stream_update->adaptive_sync_infopacket ||
-			stream_update->avi_infopacket ||
 			stream_update->dpms_off ||
 			stream_update->allow_freesync ||
 			stream_update->vrr_active_variable ||
 			stream_update->vrr_active_fixed ||
-			stream_update->gamut_remap ||
 			stream_update->output_color_space ||
-			stream_update->dither_option ||
 			stream_update->wb_update ||
 			stream_update->dsc_config ||
 			stream_update->mst_bw_update ||
@@ -5766,7 +6308,8 @@ bool dc_set_ips_disable(struct dc *dc, unsigned int disable_ips)
 
 void dc_allow_idle_optimizations_internal(struct dc *dc, bool allow, char const *caller_name)
 {
-	int idle_fclk_khz = 0, idle_dramclk_khz = 0, i = 0;
+	int idle_fclk_khz = 0, idle_dramclk_khz = 0;
+	unsigned int i = 0;
 	enum mall_stream_type subvp_pipe_type[MAX_PIPES] = {0};
 	struct pipe_ctx *pipe = NULL;
 	struct dc_state *context = dc->current_state;
@@ -5865,7 +6408,7 @@ static void blank_and_force_memclk(struct dc *dc, bool apply, unsigned int memcl
 	struct dc_state *context = dc->current_state;
 	struct hubp *hubp;
 	struct pipe_ctx *pipe;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		pipe = &context->res_ctx.pipe_ctx[i];
@@ -6071,7 +6614,11 @@ bool dc_process_dmub_aux_transfer_async(struct dc *dc,
 	uint8_t action;
 	union dmub_rb_cmd cmd = {0};
 
-	ASSERT(payload->length <= 16);
+	if (link_index >= dc->link_count || !dc->links[link_index])
+		return false;
+
+	if (payload->length > sizeof(cmd.dp_aux_access.aux_control.dpaux.data))
+		return false;
 
 	cmd.dp_aux_access.header.type = DMUB_CMD__DP_AUX_ACCESS;
 	cmd.dp_aux_access.header.payload_bytes = 0;
@@ -6081,12 +6628,12 @@ bool dc_process_dmub_aux_transfer_async(struct dc *dc,
 	else
 		cmd.dp_aux_access.aux_control.type = AUX_CHANNEL_LEGACY_DDC;
 
-	cmd.dp_aux_access.aux_control.instance = dc->links[link_index]->ddc_hw_inst;
+	cmd.dp_aux_access.aux_control.instance = (uint8_t)dc->links[link_index]->ddc_hw_inst;
 	cmd.dp_aux_access.aux_control.sw_crc_enabled = 0;
 	cmd.dp_aux_access.aux_control.timeout = 0;
 	cmd.dp_aux_access.aux_control.dpaux.address = payload->address;
 	cmd.dp_aux_access.aux_control.dpaux.is_i2c_over_aux = payload->i2c_over_aux;
-	cmd.dp_aux_access.aux_control.dpaux.length = payload->length;
+	cmd.dp_aux_access.aux_control.dpaux.length = (uint8_t)payload->length;
 
 	/* set aux action */
 	if (payload->i2c_over_aux) {
@@ -6152,7 +6699,7 @@ bool dc_smart_power_oled_enable(const struct dc_link *link, bool enable, uint16_
 	}
 
 	if (pipe_ctx)
-		otg_inst = pipe_ctx->stream_res.tg->inst;
+		otg_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
 
 	// before enable smart power OLED, we need to call set pipe for DMUB to set ABM config
 	if (enable) {
@@ -6169,11 +6716,11 @@ bool dc_smart_power_oled_enable(const struct dc_link *link, bool enable, uint16_
 		sizeof(struct dmub_rb_cmd_smart_power_oled_enable_data) - sizeof(struct dmub_cmd_header);
 	cmd.smart_power_oled_enable.header.ret_status = 1;
 	cmd.smart_power_oled_enable.data.enable = enable;
-	cmd.smart_power_oled_enable.data.panel_inst = panel_inst;
+	cmd.smart_power_oled_enable.data.panel_inst = (uint8_t)panel_inst;
 	cmd.smart_power_oled_enable.data.peak_nits = peak_nits;
 	cmd.smart_power_oled_enable.data.otg_inst = otg_inst;
-	cmd.smart_power_oled_enable.data.digfe_inst = link->link_enc->preferred_engine;
-	cmd.smart_power_oled_enable.data.digbe_inst = link->link_enc->transmitter;
+	cmd.smart_power_oled_enable.data.digfe_inst = (uint8_t)link->link_enc->preferred_engine;
+	cmd.smart_power_oled_enable.data.digbe_inst = (uint8_t)link->link_enc->transmitter;
 
 	cmd.smart_power_oled_enable.data.debugcontrol = debug_control;
 	cmd.smart_power_oled_enable.data.triggerline = triggerline;
@@ -6182,6 +6729,10 @@ bool dc_smart_power_oled_enable(const struct dc_link *link, bool enable, uint16_
 	// send cmd
 	status = dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 
+	// Update firmware_controlled_hdr_info_packet state on successful command execution
+	if (status && pipe_ctx)
+		pipe_ctx->stream->firmware_controlled_hdr_info_packet = enable;
+
 	return status;
 }
 
@@ -6204,7 +6755,7 @@ bool dc_smart_power_oled_get_max_cll(const struct dc_link *link, unsigned int *p
 	cmd.smart_power_oled_getmaxcll.header.payload_bytes = sizeof(cmd.smart_power_oled_getmaxcll.data);
 	cmd.smart_power_oled_getmaxcll.header.ret_status = 1;
 
-	cmd.smart_power_oled_getmaxcll.data.input.panel_inst = panel_inst;
+	cmd.smart_power_oled_getmaxcll.data.input.panel_inst = (uint8_t)panel_inst;
 
 	// send cmd and wait for reply
 	status = dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY);
@@ -6262,7 +6813,7 @@ bool dc_process_dmub_set_config_async(struct dc *dc,
 	cmd.set_config_access.header.type = DMUB_CMD__DPIA;
 	cmd.set_config_access.header.sub_type = DMUB_CMD__DPIA_SET_CONFIG_ACCESS;
 
-	cmd.set_config_access.set_config_control.instance = dc->links[link_index]->ddc_hw_inst;
+	cmd.set_config_access.set_config_control.instance = (uint8_t)dc->links[link_index]->ddc_hw_inst;
 	cmd.set_config_access.set_config_control.cmd_pkt.msg_type = payload->msg_type;
 	cmd.set_config_access.set_config_control.cmd_pkt.msg_data = payload->msg_data;
 
@@ -6306,7 +6857,7 @@ enum dc_status dc_process_dmub_set_mst_slots(const struct dc *dc,
 	cmd.set_mst_alloc_slots.header.type = DMUB_CMD__DPIA;
 	cmd.set_mst_alloc_slots.header.sub_type = DMUB_CMD__DPIA_MST_ALLOC_SLOTS;
 
-	cmd.set_mst_alloc_slots.mst_slots_control.instance = dc->links[link_index]->ddc_hw_inst;
+	cmd.set_mst_alloc_slots.mst_slots_control.instance = (uint8_t)dc->links[link_index]->ddc_hw_inst;
 	cmd.set_mst_alloc_slots.mst_slots_control.mst_alloc_slots = mst_alloc_slots;
 
 	if (!dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY))
@@ -6346,7 +6897,7 @@ void dc_process_dmub_dpia_set_tps_notification(const struct dc *dc, uint32_t lin
 
 	cmd.set_tps_notification.header.type = DMUB_CMD__DPIA;
 	cmd.set_tps_notification.header.sub_type = DMUB_CMD__DPIA_SET_TPS_NOTIFICATION;
-	cmd.set_tps_notification.tps_notification.instance = dc->links[link_index]->ddc_hw_inst;
+	cmd.set_tps_notification.tps_notification.instance = (uint8_t)dc->links[link_index]->ddc_hw_inst;
 	cmd.set_tps_notification.tps_notification.tps = tps;
 
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
@@ -6642,7 +7193,7 @@ bool dc_can_clear_cursor_limit(const struct dc *dc)
 	return false;
 }
 
-void dc_get_underflow_debug_data_for_otg(struct dc *dc, int primary_otg_inst,
+void dc_get_underflow_debug_data_for_otg(struct dc *dc, unsigned int primary_otg_inst,
 				struct dc_underflow_debug_data *out_data)
 {
 	struct timing_generator *tg = NULL;
@@ -6660,7 +7211,7 @@ void dc_get_underflow_debug_data_for_otg(struct dc *dc, int primary_otg_inst,
 		dc->hwss.get_underflow_debug_data(dc, tg, out_data);
 }
 
-void dc_get_power_feature_status(struct dc *dc, int primary_otg_inst,
+void dc_get_power_feature_status(struct dc *dc, unsigned int primary_otg_inst,
 				struct power_features *out_data)
 {
 	(void)primary_otg_inst;
@@ -6672,7 +7223,8 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 {
 	struct dc_state *context;
 	struct resource_context *res_ctx;
-	int i;
+	unsigned int i;
+	const unsigned int max_pipes = MAX_PIPES;
 
 	if (!dc || !dc->current_state || !state) {
 		if (state)
@@ -6696,7 +7248,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 	}
 
 	/* Capture HUBP programming state for each pipe */
-	for (i = 0; i < MAX_PIPES && i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
 
 		state->hubp[i].valid_stream = false;
@@ -6818,7 +7370,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 	/* Capture HUBBUB programming state */
 	if (dc->res_pool->hubbub) {
 		/* Individual DET buffer sizes - software state variables that program DET registers */
-		for (i = 0; i < 4 && i < dc->res_pool->pipe_count; i++) {
+		for (i = 0; i < 4u && i < dc->res_pool->pipe_count; i++) {
 			uint32_t det_size = res_ctx->pipe_ctx[i].det_buffer_size_kb;
 			switch (i) {
 			case 0:
@@ -6842,7 +7394,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 	}
 
 	/* Capture DPP programming state for each pipe */
-	for (i = 0; i < MAX_PIPES && i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
 
 		if (!pipe_ctx->stream)
@@ -6883,7 +7435,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 		state->dccg.dispclk_khz = dc->clk_mgr->clks.dispclk_khz;
 
 		/* Per-pipe clock configuration - only capture what's essential */
-		for (i = 0; i < MAX_PIPES && i < dc->res_pool->pipe_count; i++) {
+		for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
 			struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
 			if (pipe_ctx->stream) {
 				/* Essential clocks that directly affect underflow risk */
@@ -6910,7 +7462,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 		}
 
 		/* DSC clock state - only when actually using DSC */
-		for (i = 0; i < MAX_PIPES; i++) {
+		for (i = 0; i < max_pipes; i++) {
 			struct pipe_ctx *pipe_ctx = (i < dc->res_pool->pipe_count) ? &res_ctx->pipe_ctx[i] : NULL;
 			if (pipe_ctx && pipe_ctx->stream && pipe_ctx->stream->timing.dsc_cfg.num_slices_h > 0) {
 				state->dccg.dscclk_khz[i] = 400000; /* Typical DSC clock frequency */
@@ -6927,7 +7479,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 	}
 
 	/* Capture essential DSC configuration for underflow analysis */
-	for (i = 0; i < MAX_PIPES && i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
 
 		if (pipe_ctx->stream && pipe_ctx->stream->timing.dsc_cfg.num_slices_h > 0) {
@@ -6955,14 +7507,14 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 	}
 
 	/* Capture MPC programming state - comprehensive register field coverage */
-	for (i = 0; i < MAX_PIPES && i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
 
 		if (pipe_ctx->plane_state && pipe_ctx->stream) {
 			struct dc_plane_state *plane_state = pipe_ctx->plane_state;
 
 			/* MPCC blending tree and mode control - capture actual blend configuration */
-			state->mpc.mpcc_mode[i] = (plane_state->cm.blend_func.type != TF_TYPE_BYPASS) ? 1 : 0;
+			state->mpc.mpcc_mode[i] = (plane_state->blend_tf.type != TF_TYPE_BYPASS) ? 1 : 0;
 			state->mpc.mpcc_alpha_blend_mode[i] = plane_state->per_pixel_alpha ? 1 : 0;
 			state->mpc.mpcc_alpha_multiplied_mode[i] = plane_state->pre_multiplied_alpha ? 1 : 0;
 			state->mpc.mpcc_blnd_active_overlap_only[i] = 0; /* Default - no overlap restriction */
@@ -7023,7 +7575,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 	}
 
 	/* Capture OPP programming state for each pipe - comprehensive register field coverage */
-	for (i = 0; i < MAX_PIPES && i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
 
 		if (!pipe_ctx->stream)
@@ -7141,7 +7693,7 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
 	}
 
 	/* Capture OPTC programming state for each pipe - comprehensive register field coverage */
-	for (i = 0; i < MAX_PIPES && i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < max_pipes && i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &res_ctx->pipe_ctx[i];
 
 		if (!pipe_ctx->stream)
@@ -7237,7 +7789,8 @@ void dc_log_preos_dmcub_info(const struct dc *dc)
 bool dc_get_qos_info(struct dc *dc, struct dc_qos_info *info)
 {
 	const struct dc_clocks *clk = &dc->current_state->bw_ctx.bw.dcn.clk;
-	struct memory_qos qos;
+	struct dc_measured_memory_qos measured = {};
+	struct dc_requested_memory_qos requested = {};
 
 	memset(info, 0, sizeof(*info));
 
@@ -7246,20 +7799,38 @@ bool dc_get_qos_info(struct dc *dc, struct dc_qos_info *info)
 		return false;
 	}
 
-	// Call unified measurement function
-	dc->hwss.measure_memory_qos(dc, &qos);
+	dc->hwss.measure_memory_qos(dc, &measured);
+
+	info->actual_peak_bw_in_mbps    = measured.peak_bw_mbps;
+	info->actual_avg_bw_in_mbps     = measured.avg_bw_mbps;
+	info->actual_min_latency_in_ns  = measured.min_latency_ns;
+	info->actual_max_latency_in_ns  = measured.max_latency_ns;
+	info->actual_avg_latency_in_ns  = measured.avg_latency_ns;
+	info->dcn_bandwidth_ub_in_mbps  = (uint32_t)(clk->fclk_khz / 1000 * 64);
 
-	// Populate info from measured qos
-	info->actual_peak_bw_in_mbps = qos.peak_bw_mbps;
-	info->actual_avg_bw_in_mbps = qos.avg_bw_mbps;
-	info->actual_min_latency_in_ns = qos.min_latency_ns;
-	info->actual_max_latency_in_ns = qos.max_latency_ns;
-	info->actual_avg_latency_in_ns = qos.avg_latency_ns;
-	info->dcn_bandwidth_ub_in_mbps = (uint32_t)(clk->fclk_khz / 1000 * 64);
+	if (dc->clk_mgr && dc->clk_mgr->funcs->get_requested_memory_qos) {
+		dc->clk_mgr->funcs->get_requested_memory_qos(dc->clk_mgr, &requested);
+		info->qos_bandwidth_lb_in_mbps    = requested.bandwidth_lb_in_mbps;
+		info->calculated_avg_bw_in_mbps   = requested.calculated_avg_bw_in_mbps;
+		info->qos_max_latency_ub_in_ns    = requested.max_latency_ub_in_ns;
+		info->qos_avg_latency_ub_in_ns    = requested.avg_latency_ub_in_ns;
+		info->qos_max_bw_budget_in_mbps   = requested.max_bw_budget_in_mbps;
+	}
 
 	return true;
 }
 
+unsigned int dc_override_memory_bandwidth_request(
+		struct dc *dc,
+		unsigned int bw_mbps)
+{
+	if (!dc->clk_mgr || !dc->clk_mgr->funcs)
+		return 0;
+
+	return dc->clk_mgr->funcs->override_memory_bandwidth_request(
+			dc->clk_mgr, bw_mbps * 1000) / 1000;
+}
+
 enum update_v3_flow {
 	UPDATE_V3_FLOW_INVALID,
 	UPDATE_V3_FLOW_NO_NEW_CONTEXT_CONTEXT_FAST,
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
index 7333f5905330c..1916aa3ebaea1 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_hw_sequencer.c
@@ -39,6 +39,7 @@
 #include "abm.h"
 #include "dcn10/dcn10_hubbub.h"
 #include "dce/dmub_hw_lock_mgr.h"
+#include "link_service.h"
 
 #define MAX_NUM_MCACHE 8
 
@@ -345,39 +346,39 @@ void get_surface_visual_confirm_color(
 	switch (pipe_ctx->plane_res.scl_data.format) {
 	case PIXEL_FORMAT_ARGB8888:
 		/* set border color to red */
-		color->color_r_cr = color_value;
+		color->color_r_cr = (uint16_t)color_value;
 		if (pipe_ctx->plane_state->layer_index > 0) {
 			/* set border color to pink */
-			color->color_b_cb = color_value;
-			color->color_g_y = color_value * 0.5;
+			color->color_b_cb = (uint16_t)color_value;
+			color->color_g_y = (uint16_t)(color_value / 2);
 		}
 		break;
 
 	case PIXEL_FORMAT_ARGB2101010:
 		/* set border color to blue */
-		color->color_b_cb = color_value;
+		color->color_b_cb = (uint16_t)color_value;
 		if (pipe_ctx->plane_state->layer_index > 0) {
 			/* set border color to cyan */
-			color->color_g_y = color_value;
+			color->color_g_y = (uint16_t)color_value;
 		}
 		break;
 	case PIXEL_FORMAT_420BPP8:
 		/* set border color to green */
-		color->color_g_y = color_value;
+		color->color_g_y = (uint16_t)color_value;
 		break;
 	case PIXEL_FORMAT_420BPP10:
 		/* set border color to yellow */
-		color->color_g_y = color_value;
-		color->color_r_cr = color_value;
+		color->color_g_y = (uint16_t)color_value;
+		color->color_r_cr = (uint16_t)color_value;
 		break;
 	case PIXEL_FORMAT_FP16:
 		/* set border color to white */
-		color->color_r_cr = color_value;
-		color->color_b_cb = color_value;
-		color->color_g_y = color_value;
+		color->color_r_cr = (uint16_t)color_value;
+		color->color_b_cb = (uint16_t)color_value;
+		color->color_g_y = (uint16_t)color_value;
 		if (pipe_ctx->plane_state->layer_index > 0) {
 			/* set border color to orange */
-			color->color_g_y = 0.22 * color_value;
+			color->color_g_y = (uint16_t)((color_value * 22) / 100);
 			color->color_b_cb = 0;
 		}
 		break;
@@ -403,21 +404,21 @@ void get_hdr_visual_confirm_color(
 	case PIXEL_FORMAT_ARGB2101010:
 		if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_PQ) {
 			/* HDR10, ARGB2101010 - set border color to red */
-			color->color_r_cr = color_value;
+			color->color_r_cr = (uint16_t)color_value;
 		} else if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) {
 			/* FreeSync 2 ARGB2101010 - set border color to pink */
-			color->color_r_cr = color_value;
-			color->color_b_cb = color_value;
+			color->color_r_cr = (uint16_t)color_value;
+			color->color_b_cb = (uint16_t)color_value;
 		} else
 			is_sdr = true;
 		break;
 	case PIXEL_FORMAT_FP16:
 		if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_PQ) {
 			/* HDR10, FP16 - set border color to blue */
-			color->color_b_cb = color_value;
+			color->color_b_cb = (uint16_t)color_value;
 		} else if (top_pipe_ctx->stream->out_transfer_func.tf == TRANSFER_FUNCTION_GAMMA22) {
 			/* FreeSync 2 HDR - set border color to green */
-			color->color_g_y = color_value;
+			color->color_g_y = (uint16_t)color_value;
 		} else
 			is_sdr = true;
 		break;
@@ -428,9 +429,9 @@ void get_hdr_visual_confirm_color(
 
 	if (is_sdr) {
 		/* SDR - set border color to Gray */
-		color->color_r_cr = color_value/2;
-		color->color_b_cb = color_value/2;
-		color->color_g_y = color_value/2;
+		color->color_r_cr = (uint16_t)(color_value / 2);
+		color->color_b_cb = (uint16_t)(color_value / 2);
+		color->color_g_y = (uint16_t)(color_value / 2);
 	}
 }
 
@@ -454,7 +455,7 @@ void get_smartmux_visual_confirm_color(
 		*color = sm_ver_colors[dc->config.smart_mux_version];
 	} else {
 		/* dGPU driving the eDP - red */
-		color->color_r_cr = color_value;
+		color->color_r_cr = (uint16_t)color_value;
 		color->color_g_y = 0;
 		color->color_b_cb = 0;
 	}
@@ -476,19 +477,19 @@ void get_vabc_visual_confirm_color(
 	if (edp_link) {
 		switch (edp_link->backlight_control_type) {
 		case BACKLIGHT_CONTROL_PWM:
-			color->color_r_cr = color_value;
+			color->color_r_cr = (uint16_t)color_value;
 			color->color_g_y = 0;
 			color->color_b_cb = 0;
 			break;
 		case BACKLIGHT_CONTROL_AMD_AUX:
 			color->color_r_cr = 0;
-			color->color_g_y = color_value;
+			color->color_g_y = (uint16_t)color_value;
 			color->color_b_cb = 0;
 			break;
 		case BACKLIGHT_CONTROL_VESA_AUX:
 			color->color_r_cr = 0;
 			color->color_g_y = 0;
-			color->color_b_cb = color_value;
+			color->color_b_cb = (uint16_t)color_value;
 			break;
 		}
 	} else {
@@ -506,19 +507,19 @@ void get_subvp_visual_confirm_color(
 	if (pipe_ctx) {
 		switch (pipe_ctx->p_state_type) {
 		case P_STATE_SUB_VP:
-			color->color_r_cr = color_value;
+			color->color_r_cr = (uint16_t)color_value;
 			color->color_g_y  = 0;
 			color->color_b_cb = 0;
 			break;
 		case P_STATE_DRR_SUB_VP:
 			color->color_r_cr = 0;
-			color->color_g_y  = color_value;
+			color->color_g_y  = (uint16_t)color_value;
 			color->color_b_cb = 0;
 			break;
 		case P_STATE_V_BLANK_SUB_VP:
 			color->color_r_cr = 0;
 			color->color_g_y  = 0;
-			color->color_b_cb = color_value;
+			color->color_b_cb = (uint16_t)color_value;
 			break;
 		default:
 			break;
@@ -535,34 +536,34 @@ void get_mclk_switch_visual_confirm_color(
 	if (pipe_ctx) {
 		switch (pipe_ctx->p_state_type) {
 		case P_STATE_V_BLANK:
-			color->color_r_cr = color_value;
-			color->color_g_y = color_value;
+			color->color_r_cr = (uint16_t)color_value;
+			color->color_g_y = (uint16_t)color_value;
 			color->color_b_cb = 0;
 			break;
 		case P_STATE_FPO:
 			color->color_r_cr = 0;
-			color->color_g_y  = color_value;
-			color->color_b_cb = color_value;
+			color->color_g_y  = (uint16_t)color_value;
+			color->color_b_cb = (uint16_t)color_value;
 			break;
 		case P_STATE_V_ACTIVE:
-			color->color_r_cr = color_value;
+			color->color_r_cr = (uint16_t)color_value;
 			color->color_g_y  = 0;
-			color->color_b_cb = color_value;
+			color->color_b_cb = (uint16_t)color_value;
 			break;
 		case P_STATE_SUB_VP:
-			color->color_r_cr = color_value;
+			color->color_r_cr = (uint16_t)color_value;
 			color->color_g_y  = 0;
 			color->color_b_cb = 0;
 			break;
 		case P_STATE_DRR_SUB_VP:
 			color->color_r_cr = 0;
-			color->color_g_y  = color_value;
+			color->color_g_y  = (uint16_t)color_value;
 			color->color_b_cb = 0;
 			break;
 		case P_STATE_V_BLANK_SUB_VP:
 			color->color_r_cr = 0;
 			color->color_g_y  = 0;
-			color->color_b_cb = color_value;
+			color->color_b_cb = (uint16_t)color_value;
 			break;
 		default:
 			break;
@@ -577,13 +578,13 @@ void get_cursor_visual_confirm_color(
 	uint32_t color_value = MAX_TG_COLOR_VALUE;
 
 	if (pipe_ctx->stream && pipe_ctx->stream->cursor_position.enable) {
-		color->color_r_cr = color_value;
+		color->color_r_cr = (uint16_t)color_value;
 		color->color_g_y = 0;
 		color->color_b_cb = 0;
 	} else {
 		color->color_r_cr = 0;
 		color->color_g_y = 0;
-		color->color_b_cb = color_value;
+		color->color_b_cb = (uint16_t)color_value;
 	}
 }
 
@@ -665,7 +666,7 @@ void set_p_state_switch_method(
 		/* SubVP */
 		enable_subvp = false;
 
-		for (int i = 0; i < dc->res_pool->pipe_count; i++) {
+		for (unsigned int i = 0; i < dc->res_pool->pipe_count; i++) {
 			struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 
 			if (pipe->stream && dc_state_get_paired_subvp_stream(context, pipe->stream) &&
@@ -721,9 +722,9 @@ void get_fams2_visual_confirm_color(
 	/* driver only handles visual confirm when FAMS2 is disabled */
 	if (!dc_state_is_fams2_in_use(dc, context)) {
 		/* when FAMS2 is disabled, all pipes are grey */
-		color->color_g_y = color_value / 2;
-		color->color_b_cb = color_value / 2;
-		color->color_r_cr = color_value / 2;
+		color->color_g_y = (uint16_t)(color_value / 2);
+		color->color_b_cb = (uint16_t)(color_value / 2);
+		color->color_r_cr = (uint16_t)(color_value / 2);
 	}
 }
 
@@ -789,8 +790,241 @@ void hwss_build_fast_sequence(struct dc *dc,
 		(*num_steps)++;
 	}
 
+	if (dc->hwss.setup_periodic_interrupt && stream->update_flags.bits.periodic_interrupt) {
+		block_sequence[*num_steps].params.setup_periodic_interrupt_params.dc = dc;
+		block_sequence[*num_steps].params.setup_periodic_interrupt_params.pipe_ctx = pipe_ctx;
+		block_sequence[*num_steps].func = HWSS_SETUP_PERIODIC_INTERRUPT;
+		(*num_steps)++;
+	}
+
+	if (stream->update_flags.bits.info_frame) {
+		resource_build_info_frame(pipe_ctx);
+		block_sequence[*num_steps].params.update_info_frame_params.dc = dc;
+		block_sequence[*num_steps].params.update_info_frame_params.pipe_ctx = pipe_ctx;
+		block_sequence[*num_steps].func = HWSS_UPDATE_INFO_FRAME;
+		(*num_steps)++;
+
+		if (dc_is_dp_signal(pipe_ctx->stream->signal)) {
+			block_sequence[*num_steps].params.dp_trace_source_sequence_params.link = pipe_ctx->stream->link;
+			block_sequence[*num_steps].params.dp_trace_source_sequence_params.source = DPCD_SOURCE_SEQ_AFTER_UPDATE_INFO_FRAME;
+			block_sequence[*num_steps].func = DP_TRACE_SOURCE_SEQUENCE;
+			(*num_steps)++;
+		}
+	}
+
+	if (dc->hwss.set_dmdata_attributes && stream->update_flags.bits.dmdata &&
+			stream->use_dynamic_meta && pipe_ctx->stream->dmdata_address.quad_part != 0) {
+		struct dc_dmdata_attributes attr = { 0 };
+
+		attr.dmdata_mode = DMDATA_HW_MODE;
+		attr.dmdata_size = dc_is_hdmi_signal(pipe_ctx->stream->signal) ? 32 : 36;
+		attr.address.quad_part = pipe_ctx->stream->dmdata_address.quad_part;
+		attr.dmdata_dl_delta = 0;
+		attr.dmdata_qos_mode = 0;
+		attr.dmdata_qos_level = 0;
+		attr.dmdata_repeat = 1; /* always repeat */
+		attr.dmdata_updated = 1;
+		attr.dmdata_sw_data = NULL;
+
+		block_sequence[*num_steps].params.set_dmdata_attributes_params.hubp = pipe_ctx->plane_res.hubp;
+		block_sequence[*num_steps].params.set_dmdata_attributes_params.attr = attr;
+		block_sequence[*num_steps].func = HUBP_SET_DMDATA_ATTRIBUTES;
+		(*num_steps)++;
+	}
+
+	/* Track cursor lock state - separate locks for attribute and position updates */
+	bool enable_cursor_offload = false;
+
+	if ((dc->hwss.set_cursor_attribute && stream->update_flags.bits.cursor_attr) ||
+		(dc->hwss.set_cursor_position && stream->update_flags.bits.cursor_pos))
+		enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
+
+	/* Cursor attribute updates - separate lock/iterate/unlock */
+	if (dc->hwss.set_cursor_attribute && stream->update_flags.bits.cursor_attr) {
+		struct pipe_ctx *cursor_pipe_to_program = NULL;
+
+		for (i = 0; i < MAX_PIPES; i++) {
+			current_pipe = &context->res_ctx.pipe_ctx[i];
+
+			if (current_pipe->stream != stream)
+				continue;
+
+			if (!cursor_pipe_to_program) {
+				cursor_pipe_to_program = current_pipe;
+
+				if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
+					block_sequence[*num_steps].params.begin_cursor_offload_update_params.dc = dc;
+					block_sequence[*num_steps].params.begin_cursor_offload_update_params.pipe_ctx =
+						current_pipe;
+					block_sequence[*num_steps].func = HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE;
+					(*num_steps)++;
+				} else {
+					block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
+					block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx = current_pipe;
+					block_sequence[*num_steps].params.cursor_lock_params.lock = true;
+					block_sequence[*num_steps].func = HWSS_CURSOR_LOCK;
+					(*num_steps)++;
+
+					if (current_pipe->next_odm_pipe) {
+						block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
+						block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx =
+							current_pipe->next_odm_pipe;
+						block_sequence[*num_steps].params.cursor_lock_params.lock = true;
+						block_sequence[*num_steps].func = HWSS_CURSOR_LOCK;
+						(*num_steps)++;
+					}
+				}
+			}
+
+			block_sequence[*num_steps].params.hubp_set_cursor_attributes_params.hubp =
+				current_pipe->plane_res.hubp;
+			block_sequence[*num_steps].params.hubp_set_cursor_attributes_params.attributes =
+				&current_pipe->stream->cursor_attributes;
+			block_sequence[*num_steps].func = HUBP_SET_CURSOR_ATTRIBUTES;
+			(*num_steps)++;
+
+			block_sequence[*num_steps].params.dpp_set_cursor_attributes_params.dpp =
+				current_pipe->plane_res.dpp;
+			block_sequence[*num_steps].params.dpp_set_cursor_attributes_params.attributes =
+				&current_pipe->stream->cursor_attributes;
+			block_sequence[*num_steps].func = DPP_SET_CURSOR_ATTRIBUTES;
+			(*num_steps)++;
+
+			if (dc->ctx->dmub_srv) {
+				block_sequence[*num_steps].params.send_cursor_info_to_dmu_params.pipe_ctx =
+					current_pipe;
+				block_sequence[*num_steps].params.send_cursor_info_to_dmu_params.pipe_idx =
+					current_pipe->pipe_idx;
+				block_sequence[*num_steps].func = DC_SEND_CURSOR_INFO_TO_DMU;
+				(*num_steps)++;
+			}
+
+			block_sequence[*num_steps].params.set_cursor_sdr_white_level_params.dc = dc;
+			block_sequence[*num_steps].params.set_cursor_sdr_white_level_params.pipe_ctx =
+				current_pipe;
+			block_sequence[*num_steps].func = SET_CURSOR_SDR_WHITE_LEVEL;
+			(*num_steps)++;
+
+			if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe) {
+				block_sequence[*num_steps].params.update_cursor_offload_pipe_params.dc = dc;
+				block_sequence[*num_steps].params.update_cursor_offload_pipe_params.pipe_ctx =
+					current_pipe;
+				block_sequence[*num_steps].func = HWSS_UPDATE_CURSOR_OFFLOAD_PIPE;
+				(*num_steps)++;
+			}
+		}
+
+		/* Unlock cursor attributes after all pipes have been programmed */
+		if (cursor_pipe_to_program) {
+			if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update) {
+				block_sequence[*num_steps].params.commit_cursor_offload_update_params.dc = dc;
+				block_sequence[*num_steps].params.commit_cursor_offload_update_params.pipe_ctx =
+					cursor_pipe_to_program;
+				block_sequence[*num_steps].func = HWSS_COMMIT_CURSOR_OFFLOAD_UPDATE;
+				(*num_steps)++;
+			} else {
+				block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
+				block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx = cursor_pipe_to_program;
+				block_sequence[*num_steps].params.cursor_lock_params.lock = false;
+				block_sequence[*num_steps].func = HWSS_CURSOR_LOCK;
+				(*num_steps)++;
+
+				if (cursor_pipe_to_program->next_odm_pipe) {
+					block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
+					block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx =
+						cursor_pipe_to_program->next_odm_pipe;
+					block_sequence[*num_steps].params.cursor_lock_params.lock = false;
+					block_sequence[*num_steps].func = HWSS_CURSOR_LOCK;
+					(*num_steps)++;
+				}
+			}
+		}
+	}
+
+	/* Cursor position updates */
+	if (dc->hwss.set_cursor_position && stream->update_flags.bits.cursor_pos) {
+		struct pipe_ctx *cursor_pipe_to_program = NULL;
+
+		for (i = 0; i < MAX_PIPES; i++) {
+			current_pipe = &context->res_ctx.pipe_ctx[i];
+
+			if (current_pipe->stream != stream ||
+					(!current_pipe->plane_res.mi && !current_pipe->plane_res.hubp) ||
+					!current_pipe->plane_state ||
+					(!current_pipe->plane_res.xfm && !current_pipe->plane_res.dpp) ||
+					(!current_pipe->plane_res.ipp && !current_pipe->plane_res.dpp))
+				continue;
+
+			if (!cursor_pipe_to_program) {
+				cursor_pipe_to_program = current_pipe;
+
+				if (enable_cursor_offload && dc->hwss.begin_cursor_offload_update) {
+					block_sequence[*num_steps].params.begin_cursor_offload_update_params.dc = dc;
+					block_sequence[*num_steps].params.begin_cursor_offload_update_params.pipe_ctx =
+						current_pipe;
+					block_sequence[*num_steps].func = HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE;
+					(*num_steps)++;
+				} else {
+					block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
+					block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx = current_pipe;
+					block_sequence[*num_steps].params.cursor_lock_params.lock = true;
+					block_sequence[*num_steps].func = HWSS_CURSOR_LOCK;
+					(*num_steps)++;
+				}
+			}
+
+			block_sequence[*num_steps].params.set_cursor_position_params.dc = dc;
+			block_sequence[*num_steps].params.set_cursor_position_params.pipe_ctx = current_pipe;
+			block_sequence[*num_steps].func = SET_CURSOR_POSITION;
+			(*num_steps)++;
+
+			if (enable_cursor_offload && dc->hwss.update_cursor_offload_pipe) {
+				block_sequence[*num_steps].params.update_cursor_offload_pipe_params.dc = dc;
+				block_sequence[*num_steps].params.update_cursor_offload_pipe_params.pipe_ctx =
+					current_pipe;
+				block_sequence[*num_steps].func = HWSS_UPDATE_CURSOR_OFFLOAD_PIPE;
+				(*num_steps)++;
+			}
+
+			if (dc->ctx->dmub_srv) {
+				block_sequence[*num_steps].params.send_cursor_info_to_dmu_params.pipe_ctx =
+					current_pipe;
+				block_sequence[*num_steps].params.send_cursor_info_to_dmu_params.pipe_idx =
+					current_pipe->pipe_idx;
+				block_sequence[*num_steps].func = DC_SEND_CURSOR_INFO_TO_DMU;
+				(*num_steps)++;
+			}
+		}
+
+		/* Unlock cursor position after all pipes have been programmed */
+		if (cursor_pipe_to_program) {
+			if (enable_cursor_offload && dc->hwss.commit_cursor_offload_update) {
+				block_sequence[*num_steps].params.commit_cursor_offload_update_params.dc = dc;
+				block_sequence[*num_steps].params.commit_cursor_offload_update_params.pipe_ctx =
+					cursor_pipe_to_program;
+				block_sequence[*num_steps].func = HWSS_COMMIT_CURSOR_OFFLOAD_UPDATE;
+				(*num_steps)++;
+			} else {
+				block_sequence[*num_steps].params.cursor_lock_params.dc = dc;
+				block_sequence[*num_steps].params.cursor_lock_params.pipe_ctx = cursor_pipe_to_program;
+				block_sequence[*num_steps].params.cursor_lock_params.lock = false;
+				block_sequence[*num_steps].func = HWSS_CURSOR_LOCK;
+				(*num_steps)++;
+			}
+		}
+	}
+
 	current_pipe = pipe_ctx;
 	while (current_pipe) {
+		if (current_pipe->stream->update_flags.bits.dither) {
+			resource_build_bit_depth_reduction_params(current_pipe->stream, &current_pipe->stream->bit_depth_params);
+			block_sequence[*num_steps].params.opp_program_fmt_params.opp = current_pipe->stream_res.opp;
+			block_sequence[*num_steps].params.opp_program_fmt_params.fmt_bit_depth = &current_pipe->stream->bit_depth_params;
+			block_sequence[*num_steps].params.opp_program_fmt_params.clamping = &current_pipe->stream->clamping;
+			block_sequence[*num_steps].func = OPP_PROGRAM_FMT;
+			(*num_steps)++;
+		}
+
 		current_mpc_pipe = current_pipe;
 		while (current_mpc_pipe) {
 			if (current_mpc_pipe->plane_state) {
@@ -831,7 +1065,9 @@ void hwss_build_fast_sequence(struct dc *dc,
 					(*num_steps)++;
 				}
 
-				if (dc->hwss.program_gamut_remap && current_mpc_pipe->plane_state->update_flags.bits.gamut_remap_change) {
+				if (dc->hwss.program_gamut_remap &&
+						(current_mpc_pipe->plane_state->update_flags.bits.gamut_remap_change ||
+						 current_mpc_pipe->stream->update_flags.bits.gamut_remap)) {
 					block_sequence[*num_steps].params.program_gamut_remap_params.pipe_ctx = current_mpc_pipe;
 					block_sequence[*num_steps].func = DPP_PROGRAM_GAMUT_REMAP;
 					(*num_steps)++;
@@ -856,6 +1092,16 @@ void hwss_build_fast_sequence(struct dc *dc,
 					block_sequence[*num_steps].func = DPP_PROGRAM_CM_HIST;
 					(*num_steps)++;
 				}
+
+				if (current_mpc_pipe->plane_res.dpp &&
+						current_mpc_pipe->plane_res.dpp->funcs->set_cursor_matrix &&
+						current_mpc_pipe->plane_state->update_flags.bits.cursor_csc_color_matrix_change) {
+					block_sequence[*num_steps].params.dpp_set_cursor_matrix_params.dpp = current_mpc_pipe->plane_res.dpp;
+					block_sequence[*num_steps].params.dpp_set_cursor_matrix_params.color_space = current_mpc_pipe->plane_state->color_space;
+					block_sequence[*num_steps].params.dpp_set_cursor_matrix_params.cursor_csc_color_matrix = &current_mpc_pipe->plane_state->cursor_csc_color_matrix;
+					block_sequence[*num_steps].func = DPP_SET_CURSOR_MATRIX;
+					(*num_steps)++;
+				}
 			}
 			if (hws->funcs.set_output_transfer_func && current_mpc_pipe->stream->update_flags.bits.out_tf) {
 				block_sequence[*num_steps].params.set_output_transfer_func_params.dc = dc;
@@ -953,7 +1199,7 @@ void hwss_execute_sequence(struct dc *dc,
 		struct block_sequence block_sequence[MAX_HWSS_BLOCK_SEQUENCE_SIZE],
 		int num_steps)
 {
-	unsigned int i;
+	int i;
 	union block_sequence_params *params;
 	struct dce_hwseq *hws = dc->hwseq;
 
@@ -989,7 +1235,22 @@ void hwss_execute_sequence(struct dc *dc,
 					params->set_input_transfer_func_params.plane_state);
 			break;
 		case DPP_PROGRAM_GAMUT_REMAP:
-			dc->hwss.program_gamut_remap(params->program_gamut_remap_params.pipe_ctx);
+			hwss_program_gamut_remap(params);
+			break;
+		case HUBP_ENABLE_3DLUT_FL:
+			hwss_hubp_enable_3dlut_fl(params);
+			break;
+		case OTG_SETUP_VERTICAL_INTERRUPT:
+			hwss_tg_setup_vertical_interrupt0(params);
+			break;
+		case HWSS_SETUP_PERIODIC_INTERRUPT:
+			hwss_setup_periodic_interrupt(dc, params);
+			break;
+		case HWSS_UPDATE_INFO_FRAME:
+			hwss_update_info_frame(dc, params);
+			break;
+		case HUBP_SET_DMDATA_ATTRIBUTES:
+			hwss_set_dmdata_attributes(params);
 			break;
 		case DPP_SETUP_DPP:
 			hwss_setup_dpp(params);
@@ -1311,9 +1572,30 @@ void hwss_execute_sequence(struct dc *dc,
 		case ABORT_CURSOR_OFFLOAD_UPDATE:
 			hwss_abort_cursor_offload_update(params);
 			break;
+		case HWSS_CURSOR_LOCK:
+			hwss_cursor_lock(params);
+			break;
+		case HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE:
+			hwss_begin_cursor_offload_update(params);
+			break;
+		case HWSS_COMMIT_CURSOR_OFFLOAD_UPDATE:
+			hwss_commit_cursor_offload_update(params);
+			break;
+		case HWSS_UPDATE_CURSOR_OFFLOAD_PIPE:
+			hwss_update_cursor_offload_pipe(params);
+			break;
+		case DC_SEND_CURSOR_INFO_TO_DMU:
+			hwss_send_cursor_info_to_dmu(params);
+			break;
 		case SET_CURSOR_ATTRIBUTE:
 			hwss_set_cursor_attribute(params);
 			break;
+		case HUBP_SET_CURSOR_ATTRIBUTES:
+			hwss_hubp_set_cursor_attributes(params);
+			break;
+		case DPP_SET_CURSOR_ATTRIBUTES:
+			hwss_dpp_set_cursor_attributes(params);
+			break;
 		case SET_CURSOR_POSITION:
 			hwss_set_cursor_position(params);
 			break;
@@ -1329,6 +1611,57 @@ void hwss_execute_sequence(struct dc *dc,
 		case PHANTOM_HUBP_POST_ENABLE:
 			hwss_phantom_hubp_post_enable(params);
 			break;
+		case STREAM_ENC_UPDATE_HDMI_INFO_PACKETS:
+			hwss_stream_enc_update_hdmi_info_packets(params);
+			break;
+		case HPO_DP_STREAM_ENC_UPDATE_DP_INFO_PACKETS_SDP_LINE_NUM:
+			hwss_hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num(params);
+			break;
+		case HPO_DP_STREAM_ENC_UPDATE_DP_INFO_PACKETS:
+			hwss_hpo_dp_stream_enc_update_dp_info_packets(params);
+			break;
+		case STREAM_ENC_UPDATE_DP_INFO_PACKETS_SDP_LINE_NUM:
+			hwss_stream_enc_update_dp_info_packets_sdp_line_num(params);
+			break;
+		case STREAM_ENC_UPDATE_DP_INFO_PACKETS:
+			hwss_stream_enc_update_dp_info_packets(params);
+			break;
+		case DSC_SET_CONFIG_SIMPLE:
+			hwss_dsc_set_config_simple(params);
+			break;
+		case STREAM_ENC_DP_SET_DSC_CONFIG:
+			hwss_stream_enc_dp_set_dsc_config(params);
+			break;
+		case HPO_DP_STREAM_ENC_DP_SET_DSC_PPS_INFO_PACKET:
+			hwss_hpo_dp_stream_enc_dp_set_dsc_pps_info_packet(params);
+			break;
+		case STREAM_ENC_DP_SET_DSC_PPS_INFO_PACKET:
+			hwss_stream_enc_dp_set_dsc_pps_info_packet(params);
+			break;
+		case DP_TRACE_SOURCE_SEQUENCE:
+			hwss_dp_trace_source_sequence(params);
+			break;
+		case LINK_INCREASE_MST_PAYLOAD:
+			hwss_link_increase_mst_payload(params);
+			break;
+		case LINK_REDUCE_MST_PAYLOAD:
+			hwss_link_reduce_mst_payload(params);
+			break;
+		case DP_SET_TEST_PATTERN:
+			hwss_dp_set_test_pattern(params);
+			break;
+		case LINK_SET_DPMS_OFF:
+			hwss_link_set_dpms_off(params);
+			break;
+		case DISABLE_AUDIO_STREAM:
+			hwss_disable_audio_stream(dc, params);
+			break;
+		case PREPARE_BANDWIDTH:
+			hwss_prepare_bandwidth(dc, params);
+			break;
+		case LINK_SET_DPMS_ON:
+			hwss_link_set_dpms_on(params);
+			break;
 		default:
 			ASSERT(false);
 			break;
@@ -1732,6 +2065,21 @@ void hwss_add_tg_set_vtg_params(struct block_sequence_state *seq_state,
 	}
 }
 
+/*
+ * Helper function to add OTG setup vertical interrupt0 to block sequence
+ */
+void hwss_add_vertical_interrupt_setup(struct block_sequence_state *seq_state,
+		struct timing_generator *tg, uint32_t start_line, uint32_t end_line)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].params.tg_setup_vertical_interrupt0_params.tg = tg;
+		seq_state->steps[*seq_state->num_steps].params.tg_setup_vertical_interrupt0_params.start_line = start_line;
+		seq_state->steps[*seq_state->num_steps].params.tg_setup_vertical_interrupt0_params.end_line = end_line;
+		seq_state->steps[*seq_state->num_steps].func = OTG_SETUP_VERTICAL_INTERRUPT;
+		(*seq_state->num_steps)++;
+	}
+}
+
 /*
  * Helper function to add TG setup vertical interrupt2 to block sequence
  */
@@ -1947,6 +2295,16 @@ void hwss_add_tg_wait_double_buffer_pending(struct block_sequence_state *seq_sta
 	}
 }
 
+void hwss_add_hubp_enable_3dlut_fl(struct block_sequence_state *seq_state,
+		struct hubp *hubp)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].params.hubp_enable_3dlut_fl_params.hubp = hubp;
+		seq_state->steps[*seq_state->num_steps].func = HUBP_ENABLE_3DLUT_FL;
+		(*seq_state->num_steps)++;
+	}
+}
+
 void hwss_program_manual_trigger(union block_sequence_params *params)
 {
 	struct pipe_ctx *pipe_ctx = params->program_manual_trigger_params.pipe_ctx;
@@ -2091,7 +2449,7 @@ void get_surface_tile_visual_confirm_color(
 	switch (bottom_pipe_ctx->plane_state->tiling_info.gfx9.swizzle) {
 	case DC_SW_LINEAR:
 		/* LINEAR Surface - set border color to red */
-		color->color_r_cr = color_value;
+		color->color_r_cr = (uint16_t)color_value;
 		break;
 	default:
 		break;
@@ -2180,7 +2538,7 @@ void hwss_wait_for_outstanding_hw_updates(struct dc *dc, struct dc_state *dc_con
  * operations to complete. It should be invoked as a pre-amble prior
  * to full update programming before asserting any HW locks.
  */
-	int pipe_idx;
+	unsigned int pipe_idx;
 	int opp_inst;
 	int opp_count = dc->res_pool->res_cap->num_opp;
 	struct hubp *hubp;
@@ -2416,6 +2774,40 @@ void hwss_tg_set_vtg_params(union block_sequence_params *params)
 		tg->funcs->set_vtg_params(tg, timing, program_fp2);
 }
 
+void hwss_hubp_enable_3dlut_fl(union block_sequence_params *params)
+{
+	struct hubp *hubp = params->hubp_enable_3dlut_fl_params.hubp;
+
+	if (hubp->funcs->hubp_enable_3dlut_fl)
+		hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
+}
+
+void hwss_update_info_frame(struct dc *dc, union block_sequence_params *params)
+{
+	struct pipe_ctx *pipe_ctx = params->update_info_frame_params.pipe_ctx;
+
+	if (dc->hwss.update_info_frame)
+		dc->hwss.update_info_frame(pipe_ctx);
+}
+
+void hwss_setup_periodic_interrupt(struct dc *dc, union block_sequence_params *params)
+{
+	struct pipe_ctx *pipe_ctx = params->setup_periodic_interrupt_params.pipe_ctx;
+
+	if (dc->hwss.setup_periodic_interrupt)
+		dc->hwss.setup_periodic_interrupt(dc, pipe_ctx);
+}
+
+void hwss_tg_setup_vertical_interrupt0(union block_sequence_params *params)
+{
+	struct timing_generator *tg = params->tg_setup_vertical_interrupt0_params.tg;
+	uint32_t start_line = params->tg_setup_vertical_interrupt0_params.start_line;
+	uint32_t end_line = params->tg_setup_vertical_interrupt0_params.end_line;
+
+	if (tg->funcs->setup_vertical_interrupt0)
+		tg->funcs->setup_vertical_interrupt0(tg, start_line, end_line);
+}
+
 void hwss_tg_setup_vertical_interrupt2(union block_sequence_params *params)
 {
 	struct timing_generator *tg = params->tg_setup_vertical_interrupt2_params.tg;
@@ -2615,7 +3007,7 @@ void hwss_hubp_wait_flip_pending(union block_sequence_params *params)
 	struct hubp *hubp = params->hubp_wait_flip_pending_params.hubp;
 	unsigned int timeout_us = params->hubp_wait_flip_pending_params.timeout_us;
 	unsigned int polling_interval_us = params->hubp_wait_flip_pending_params.polling_interval_us;
-	int j = 0;
+	unsigned int j = 0;
 
 	for (j = 0; j < timeout_us / polling_interval_us
 		&& hubp->funcs->hubp_is_flip_pending(hubp); j++)
@@ -2627,7 +3019,7 @@ void hwss_tg_wait_double_buffer_pending(union block_sequence_params *params)
 	struct timing_generator *tg = params->tg_wait_double_buffer_pending_params.tg;
 	unsigned int timeout_us = params->tg_wait_double_buffer_pending_params.timeout_us;
 	unsigned int polling_interval_us = params->tg_wait_double_buffer_pending_params.polling_interval_us;
-	int j = 0;
+	unsigned int j = 0;
 
 	if (tg->funcs->get_optc_double_buffer_pending) {
 		for (j = 0; j < timeout_us / polling_interval_us
@@ -3113,6 +3505,51 @@ void hwss_abort_cursor_offload_update(union block_sequence_params *params)
 		dc->hwss.abort_cursor_offload_update(dc, pipe_ctx);
 }
 
+void hwss_cursor_lock(union block_sequence_params *params)
+{
+	struct dc *dc = params->cursor_lock_params.dc;
+	struct pipe_ctx *pipe_ctx = params->cursor_lock_params.pipe_ctx;
+	bool lock = params->cursor_lock_params.lock;
+
+	if (dc && dc->hwss.cursor_lock)
+		dc->hwss.cursor_lock(dc, pipe_ctx, lock);
+}
+
+void hwss_begin_cursor_offload_update(union block_sequence_params *params)
+{
+	struct dc *dc = params->begin_cursor_offload_update_params.dc;
+	struct pipe_ctx *pipe_ctx = params->begin_cursor_offload_update_params.pipe_ctx;
+
+	if (dc && dc->hwss.begin_cursor_offload_update)
+		dc->hwss.begin_cursor_offload_update(dc, pipe_ctx);
+}
+
+void hwss_commit_cursor_offload_update(union block_sequence_params *params)
+{
+	struct dc *dc = params->commit_cursor_offload_update_params.dc;
+	struct pipe_ctx *pipe_ctx = params->commit_cursor_offload_update_params.pipe_ctx;
+
+	if (dc && dc->hwss.commit_cursor_offload_update)
+		dc->hwss.commit_cursor_offload_update(dc, pipe_ctx);
+}
+
+void hwss_update_cursor_offload_pipe(union block_sequence_params *params)
+{
+	struct dc *dc = params->update_cursor_offload_pipe_params.dc;
+	struct pipe_ctx *pipe_ctx = params->update_cursor_offload_pipe_params.pipe_ctx;
+
+	if (dc && dc->hwss.update_cursor_offload_pipe)
+		dc->hwss.update_cursor_offload_pipe(dc, pipe_ctx);
+}
+
+void hwss_send_cursor_info_to_dmu(union block_sequence_params *params)
+{
+	struct pipe_ctx *pipe_ctx = params->send_cursor_info_to_dmu_params.pipe_ctx;
+	int pipe_idx = params->send_cursor_info_to_dmu_params.pipe_idx;
+
+	dc_send_update_cursor_info_to_dmu(pipe_ctx, (uint8_t)pipe_idx);
+}
+
 void hwss_set_cursor_attribute(union block_sequence_params *params)
 {
 	struct dc *dc = params->set_cursor_attribute_params.dc;
@@ -3122,6 +3559,24 @@ void hwss_set_cursor_attribute(union block_sequence_params *params)
 		dc->hwss.set_cursor_attribute(pipe_ctx);
 }
 
+void hwss_hubp_set_cursor_attributes(union block_sequence_params *params)
+{
+	struct hubp *hubp = params->hubp_set_cursor_attributes_params.hubp;
+	const struct dc_cursor_attributes *attributes = params->hubp_set_cursor_attributes_params.attributes;
+
+	if (hubp && hubp->funcs->set_cursor_attributes)
+		hubp->funcs->set_cursor_attributes(hubp, attributes);
+}
+
+void hwss_dpp_set_cursor_attributes(union block_sequence_params *params)
+{
+	struct dpp *dpp = params->dpp_set_cursor_attributes_params.dpp;
+	struct dc_cursor_attributes *attributes = params->dpp_set_cursor_attributes_params.attributes;
+
+	if (dpp && dpp->funcs->set_cursor_attributes)
+		dpp->funcs->set_cursor_attributes(dpp, attributes);
+}
+
 void hwss_set_cursor_position(union block_sequence_params *params)
 {
 	struct dc *dc = params->set_cursor_position_params.dc;
@@ -3140,6 +3595,14 @@ void hwss_set_cursor_sdr_white_level(union block_sequence_params *params)
 		dc->hwss.set_cursor_sdr_white_level(pipe_ctx);
 }
 
+void hwss_program_gamut_remap(union block_sequence_params *params)
+{
+	struct dc *dc = params->program_gamut_remap_params.pipe_ctx->stream->ctx->dc;
+
+	if (dc && dc->hwss.program_gamut_remap)
+		dc->hwss.program_gamut_remap(params->program_gamut_remap_params.pipe_ctx);
+}
+
 void hwss_program_output_csc(union block_sequence_params *params)
 {
 	struct dc *dc = params->program_output_csc_params.dc;
@@ -3169,6 +3632,198 @@ void hwss_phantom_hubp_post_enable(union block_sequence_params *params)
 		hubp->funcs->phantom_hubp_post_enable(hubp);
 }
 
+void hwss_stream_enc_update_hdmi_info_packets(union block_sequence_params *params)
+{
+	if (params->stream_enc_update_hdmi_info_packets_params.pipe_ctx->stream_res.stream_enc &&
+	    params->stream_enc_update_hdmi_info_packets_params.pipe_ctx->stream_res.stream_enc->funcs->update_hdmi_info_packets)
+		params->stream_enc_update_hdmi_info_packets_params.pipe_ctx->stream_res.stream_enc->funcs->update_hdmi_info_packets(
+			params->stream_enc_update_hdmi_info_packets_params.pipe_ctx->stream_res.stream_enc,
+			&params->stream_enc_update_hdmi_info_packets_params.pipe_ctx->stream_res.encoder_info_frame);
+}
+
+void hwss_hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num(union block_sequence_params *params)
+{
+	if (params->hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.hpo_dp_stream_enc &&
+	    params->hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->update_dp_info_packets_sdp_line_num)
+		params->hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->update_dp_info_packets_sdp_line_num(
+			params->hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.hpo_dp_stream_enc,
+			&params->hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.encoder_info_frame);
+}
+
+void hwss_hpo_dp_stream_enc_update_dp_info_packets(union block_sequence_params *params)
+{
+	if (params->hpo_dp_stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.hpo_dp_stream_enc &&
+	    params->hpo_dp_stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->update_dp_info_packets)
+		params->hpo_dp_stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->update_dp_info_packets(
+			params->hpo_dp_stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.hpo_dp_stream_enc,
+			&params->hpo_dp_stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.encoder_info_frame);
+}
+
+void hwss_stream_enc_update_dp_info_packets_sdp_line_num(union block_sequence_params *params)
+{
+	if (params->stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.stream_enc &&
+	    params->stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.stream_enc->funcs->update_dp_info_packets_sdp_line_num)
+		params->stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.stream_enc->funcs->update_dp_info_packets_sdp_line_num(
+			params->stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.stream_enc,
+			&params->stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx->stream_res.encoder_info_frame);
+}
+
+void hwss_stream_enc_update_dp_info_packets(union block_sequence_params *params)
+{
+	if (params->stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.stream_enc &&
+	    params->stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.stream_enc->funcs->update_dp_info_packets)
+		params->stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.stream_enc->funcs->update_dp_info_packets(
+			params->stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.stream_enc,
+			&params->stream_enc_update_dp_info_packets_params.pipe_ctx->stream_res.encoder_info_frame);
+}
+
+void hwss_stream_enc_dp_set_dsc_config(union block_sequence_params *params)
+{
+	if (params->stream_enc_dp_set_dsc_config_params.stream_enc &&
+	    params->stream_enc_dp_set_dsc_config_params.stream_enc->funcs->dp_set_dsc_config) {
+		enum optc_dsc_mode dsc_mode = OPTC_DSC_DISABLED;
+		uint32_t dsc_bytes_per_pixel = 0;
+		uint32_t dsc_slice_width = 0;
+		const struct dsc_optc_config *dsc_optc_cfg = params->stream_enc_dp_set_dsc_config_params.dsc_optc_cfg;
+
+		if (dsc_optc_cfg) {
+			dsc_mode = dsc_optc_cfg->is_pixel_format_444 ?
+				OPTC_DSC_ENABLED_444 : OPTC_DSC_ENABLED_NATIVE_SUBSAMPLED;
+			dsc_bytes_per_pixel = dsc_optc_cfg->bytes_per_pixel;
+			dsc_slice_width = dsc_optc_cfg->slice_width;
+		}
+
+		params->stream_enc_dp_set_dsc_config_params.stream_enc->funcs->dp_set_dsc_config(
+			params->stream_enc_dp_set_dsc_config_params.stream_enc,
+			dsc_mode, dsc_bytes_per_pixel, dsc_slice_width);
+	}
+}
+
+void hwss_hpo_dp_stream_enc_dp_set_dsc_pps_info_packet(union block_sequence_params *params)
+{
+	if (params->hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.hpo_dp_stream_enc &&
+	    params->hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.hpo_dp_stream_enc->funcs->dp_set_dsc_pps_info_packet)
+		params->hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.hpo_dp_stream_enc->funcs->dp_set_dsc_pps_info_packet(
+			params->hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.hpo_dp_stream_enc,
+			params->hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.immediate_update,
+			params->hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.dsc_packed_pps,
+			params->hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.pps_sdp_stream);
+}
+
+void hwss_stream_enc_dp_set_dsc_pps_info_packet(union block_sequence_params *params)
+{
+	if (params->stream_enc_dp_set_dsc_pps_info_packet_params.stream_enc &&
+	    params->stream_enc_dp_set_dsc_pps_info_packet_params.stream_enc->funcs->dp_set_dsc_pps_info_packet)
+		params->stream_enc_dp_set_dsc_pps_info_packet_params.stream_enc->funcs->dp_set_dsc_pps_info_packet(
+			params->stream_enc_dp_set_dsc_pps_info_packet_params.stream_enc,
+			params->stream_enc_dp_set_dsc_pps_info_packet_params.immediate_update,
+			params->stream_enc_dp_set_dsc_pps_info_packet_params.dsc_packed_pps,
+			params->stream_enc_dp_set_dsc_pps_info_packet_params.pps_sdp_stream);
+}
+
+void hwss_set_dmdata_attributes(union block_sequence_params *params)
+{
+	struct hubp *hubp = params->set_dmdata_attributes_params.hubp;
+
+	if (!hubp || !hubp->funcs->dmdata_set_attributes)
+		return;
+
+	hubp->funcs->dmdata_set_attributes(hubp,
+		&params->set_dmdata_attributes_params.attr);
+}
+
+void hwss_dp_trace_source_sequence(union block_sequence_params *params)
+{
+	struct dc_link *link = params->dp_trace_source_sequence_params.link;
+	enum dpcd_source_sequence source = params->dp_trace_source_sequence_params.source;
+
+	if (link && link->dc && link->dc->link_srv && link->dc->link_srv->dp_trace_source_sequence)
+		link->dc->link_srv->dp_trace_source_sequence(link, source);
+}
+
+void hwss_link_increase_mst_payload(union block_sequence_params *params)
+{
+	struct pipe_ctx *pipe_ctx = params->link_increase_mst_payload_params.pipe_ctx;
+	uint32_t mst_stream_bw = params->link_increase_mst_payload_params.mst_stream_bw;
+
+	if (pipe_ctx && pipe_ctx->stream && pipe_ctx->stream->link &&
+		pipe_ctx->stream->link->dc && pipe_ctx->stream->link->dc->link_srv &&
+		pipe_ctx->stream->link->dc->link_srv->increase_mst_payload)
+		pipe_ctx->stream->link->dc->link_srv->increase_mst_payload(pipe_ctx, mst_stream_bw);
+}
+
+void hwss_link_reduce_mst_payload(union block_sequence_params *params)
+{
+	struct pipe_ctx *pipe_ctx = params->link_reduce_mst_payload_params.pipe_ctx;
+	uint32_t mst_stream_bw = params->link_reduce_mst_payload_params.mst_stream_bw;
+
+	if (pipe_ctx && pipe_ctx->stream && pipe_ctx->stream->link &&
+		pipe_ctx->stream->link->dc && pipe_ctx->stream->link->dc->link_srv &&
+		pipe_ctx->stream->link->dc->link_srv->reduce_mst_payload)
+		pipe_ctx->stream->link->dc->link_srv->reduce_mst_payload(pipe_ctx, mst_stream_bw);
+}
+
+void hwss_dp_set_test_pattern(union block_sequence_params *params)
+{
+	struct dc_link *link = params->dp_set_test_pattern_params.link;
+	enum dp_test_pattern test_pattern = params->dp_set_test_pattern_params.test_pattern;
+	enum dp_test_pattern_color_space test_pattern_color_space =
+	params->dp_set_test_pattern_params.test_pattern_color_space;
+	const struct link_training_settings *p_link_settings =
+	params->dp_set_test_pattern_params.p_link_settings;
+	const unsigned char *p_custom_pattern = params->dp_set_test_pattern_params.p_custom_pattern;
+	unsigned int cust_pattern_size = params->dp_set_test_pattern_params.cust_pattern_size;
+
+	if (link && link->dc && link->dc->link_srv && link->dc->link_srv->dp_set_test_pattern)
+		link->dc->link_srv->dp_set_test_pattern(link, test_pattern, test_pattern_color_space,
+			p_link_settings, p_custom_pattern, cust_pattern_size);
+}
+
+void hwss_link_set_dpms_off(union block_sequence_params *params)
+{
+	struct pipe_ctx *pipe_ctx = params->link_set_dpms_off_params.pipe_ctx;
+
+	if (pipe_ctx && pipe_ctx->stream && pipe_ctx->stream->link &&
+	    pipe_ctx->stream->link->dc && pipe_ctx->stream->link->dc->link_srv &&
+	    pipe_ctx->stream->link->dc->link_srv->set_dpms_off)
+		pipe_ctx->stream->link->dc->link_srv->set_dpms_off(pipe_ctx);
+}
+
+void hwss_disable_audio_stream(struct dc *dc, union block_sequence_params *params)
+{
+	if (dc->hwss.disable_audio_stream)
+		dc->hwss.disable_audio_stream(
+			params->disable_audio_stream_params.pipe_ctx);
+}
+
+void hwss_prepare_bandwidth(struct dc *dc, union block_sequence_params *params)
+{
+	if (dc && dc->hwss.prepare_bandwidth)
+		dc->hwss.prepare_bandwidth(dc,
+			params->prepare_bandwidth_params.context);
+}
+
+void hwss_link_set_dpms_on(union block_sequence_params *params)
+{
+	struct dc_state *state = params->link_set_dpms_on_params.state;
+	struct pipe_ctx *pipe_ctx = params->link_set_dpms_on_params.pipe_ctx;
+
+	if (pipe_ctx && pipe_ctx->stream && pipe_ctx->stream->link &&
+	    pipe_ctx->stream->link->dc && pipe_ctx->stream->link->dc->link_srv &&
+	    pipe_ctx->stream->link->dc->link_srv->set_dpms_on)
+		pipe_ctx->stream->link->dc->link_srv->set_dpms_on(state, pipe_ctx);
+}
+
+void hwss_dsc_set_config_simple(union block_sequence_params *params)
+{
+	struct display_stream_compressor *dsc = params->dsc_set_config_simple_params.dsc;
+	struct dsc_config *dsc_cfg = &params->dsc_set_config_simple_params.dsc_cfg;
+	struct dsc_optc_config *dsc_optc_cfg = &params->dsc_set_config_simple_params.dsc_optc_cfg;
+
+	if (dsc && dsc->funcs && dsc->funcs->dsc_set_config)
+		dsc->funcs->dsc_set_config(dsc, dsc_cfg, dsc_optc_cfg);
+}
+
 void hwss_add_dccg_set_dto_dscclk(struct block_sequence_state *seq_state,
 		struct dccg *dccg, int inst, int num_slices_h)
 {
@@ -4009,6 +4664,30 @@ void hwss_add_set_cursor_attribute(struct block_sequence_state *seq_state,
 	}
 }
 
+void hwss_add_hubp_set_cursor_attributes(struct block_sequence_state *seq_state,
+		struct hubp *hubp,
+		const struct dc_cursor_attributes *attributes)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HUBP_SET_CURSOR_ATTRIBUTES;
+		seq_state->steps[*seq_state->num_steps].params.hubp_set_cursor_attributes_params.hubp = hubp;
+		seq_state->steps[*seq_state->num_steps].params.hubp_set_cursor_attributes_params.attributes = attributes;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_dpp_set_cursor_attributes(struct block_sequence_state *seq_state,
+		struct dpp *dpp,
+		struct dc_cursor_attributes *attributes)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = DPP_SET_CURSOR_ATTRIBUTES;
+		seq_state->steps[*seq_state->num_steps].params.dpp_set_cursor_attributes_params.dpp = dpp;
+		seq_state->steps[*seq_state->num_steps].params.dpp_set_cursor_attributes_params.attributes = attributes;
+		(*seq_state->num_steps)++;
+	}
+}
+
 void hwss_add_set_cursor_position(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx)
@@ -4109,6 +4788,313 @@ void hwss_add_tg_get_frame_count(struct block_sequence_state *seq_state,
 	}
 }
 
+void hwss_add_begin_cursor_offload_update(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE;
+		seq_state->steps[*seq_state->num_steps].params.begin_cursor_offload_update_params.dc = dc;
+		seq_state->steps[*seq_state->num_steps].params.begin_cursor_offload_update_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_cursor_lock(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		bool lock)
+{
+	/* cursor lock is per MPCC tree, so only need to lock one pipe per stream */
+	if (!pipe_ctx || pipe_ctx->top_pipe)
+		return;
+
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HWSS_CURSOR_LOCK;
+		seq_state->steps[*seq_state->num_steps].params.cursor_lock_params.dc = dc;
+		seq_state->steps[*seq_state->num_steps].params.cursor_lock_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.cursor_lock_params.lock = lock;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_send_update_cursor_info_to_dmu(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx,
+		int index)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = DC_SEND_CURSOR_INFO_TO_DMU;
+		seq_state->steps[*seq_state->num_steps].params.send_cursor_info_to_dmu_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.send_cursor_info_to_dmu_params.pipe_idx = index;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_update_cursor_offload_pipe(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HWSS_UPDATE_CURSOR_OFFLOAD_PIPE;
+		seq_state->steps[*seq_state->num_steps].params.update_cursor_offload_pipe_params.dc = dc;
+		seq_state->steps[*seq_state->num_steps].params.update_cursor_offload_pipe_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_commit_cursor_offload_update(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HWSS_COMMIT_CURSOR_OFFLOAD_UPDATE;
+		seq_state->steps[*seq_state->num_steps].params.commit_cursor_offload_update_params.dc = dc;
+		seq_state->steps[*seq_state->num_steps].params.commit_cursor_offload_update_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_stream_enc_update_hdmi_info_packets(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = STREAM_ENC_UPDATE_HDMI_INFO_PACKETS;
+		seq_state->steps[*seq_state->num_steps].params.stream_enc_update_hdmi_info_packets_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HPO_DP_STREAM_ENC_UPDATE_DP_INFO_PACKETS_SDP_LINE_NUM;
+		seq_state->steps[*seq_state->num_steps].params.hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_hpo_dp_stream_enc_update_dp_info_packets(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HPO_DP_STREAM_ENC_UPDATE_DP_INFO_PACKETS;
+		seq_state->steps[*seq_state->num_steps].params.hpo_dp_stream_enc_update_dp_info_packets_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_stream_enc_update_dp_info_packets_sdp_line_num(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = STREAM_ENC_UPDATE_DP_INFO_PACKETS_SDP_LINE_NUM;
+		seq_state->steps[*seq_state->num_steps].params.stream_enc_update_dp_info_packets_sdp_line_num_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_stream_enc_update_dp_info_packets(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = STREAM_ENC_UPDATE_DP_INFO_PACKETS;
+		seq_state->steps[*seq_state->num_steps].params.stream_enc_update_dp_info_packets_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_dsc_set_config(struct block_sequence_state *seq_state,
+		struct display_stream_compressor *dsc,
+		const struct dsc_config *dsc_cfg,
+		const struct dsc_optc_config *dsc_optc_cfg)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = DSC_SET_CONFIG_SIMPLE;
+		seq_state->steps[*seq_state->num_steps].params.dsc_set_config_simple_params.dsc = dsc;
+		seq_state->steps[*seq_state->num_steps].params.dsc_set_config_simple_params.dsc_cfg = *dsc_cfg;
+		seq_state->steps[*seq_state->num_steps].params.dsc_set_config_simple_params.dsc_optc_cfg = *dsc_optc_cfg;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_stream_enc_dp_set_dsc_config(struct block_sequence_state *seq_state,
+		struct stream_encoder *stream_enc,
+		const struct dsc_optc_config *dsc_optc_cfg)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = STREAM_ENC_DP_SET_DSC_CONFIG;
+		seq_state->steps[*seq_state->num_steps].params.stream_enc_dp_set_dsc_config_params.stream_enc = stream_enc;
+		seq_state->steps[*seq_state->num_steps].params.stream_enc_dp_set_dsc_config_params.dsc_optc_cfg = dsc_optc_cfg;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_hpo_dp_stream_enc_dp_set_dsc_pps_info_packet(struct block_sequence_state *seq_state,
+		struct hpo_dp_stream_encoder *hpo_dp_stream_enc,
+		bool immediate_update,
+		uint8_t *dsc_packed_pps,
+		bool pps_sdp_stream)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HPO_DP_STREAM_ENC_DP_SET_DSC_PPS_INFO_PACKET;
+		seq_state->steps[*seq_state->num_steps].params.hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.hpo_dp_stream_enc = hpo_dp_stream_enc;
+		seq_state->steps[*seq_state->num_steps].params.hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.immediate_update = immediate_update;
+		seq_state->steps[*seq_state->num_steps].params.hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.dsc_packed_pps = dsc_packed_pps;
+		seq_state->steps[*seq_state->num_steps].params.hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params.pps_sdp_stream = pps_sdp_stream;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_stream_enc_dp_set_dsc_pps_info_packet(struct block_sequence_state *seq_state,
+		struct stream_encoder *stream_enc,
+		bool immediate_update,
+		uint8_t *dsc_packed_pps,
+		bool pps_sdp_stream)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = STREAM_ENC_DP_SET_DSC_PPS_INFO_PACKET;
+		seq_state->steps[*seq_state->num_steps].params.stream_enc_dp_set_dsc_pps_info_packet_params.stream_enc = stream_enc;
+		seq_state->steps[*seq_state->num_steps].params.stream_enc_dp_set_dsc_pps_info_packet_params.immediate_update = immediate_update;
+		seq_state->steps[*seq_state->num_steps].params.stream_enc_dp_set_dsc_pps_info_packet_params.dsc_packed_pps = dsc_packed_pps;
+		seq_state->steps[*seq_state->num_steps].params.stream_enc_dp_set_dsc_pps_info_packet_params.pps_sdp_stream = pps_sdp_stream;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_setup_periodic_interrupt(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = HWSS_SETUP_PERIODIC_INTERRUPT;
+		seq_state->steps[*seq_state->num_steps].params.setup_periodic_interrupt_params.dc = dc;
+		seq_state->steps[*seq_state->num_steps].params.setup_periodic_interrupt_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_dp_trace_source_sequence(struct block_sequence_state *seq_state,
+		struct dc_link *link,
+		enum dpcd_source_sequence source)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = DP_TRACE_SOURCE_SEQUENCE;
+		seq_state->steps[*seq_state->num_steps].params.dp_trace_source_sequence_params.link = link;
+		seq_state->steps[*seq_state->num_steps].params.dp_trace_source_sequence_params.source = source;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_set_dmdata_attributes(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		struct dc_dmdata_attributes *attr =
+			&seq_state->steps[*seq_state->num_steps].params.set_dmdata_attributes_params.attr;
+
+		memset(attr, 0, sizeof(*attr));
+		attr->dmdata_mode = DMDATA_HW_MODE;
+		attr->dmdata_size = dc_is_hdmi_signal(pipe_ctx->stream->signal) ? 32 : 36;
+		attr->address.quad_part = pipe_ctx->stream->dmdata_address.quad_part;
+		attr->dmdata_repeat = 1; /* always repeat */
+		attr->dmdata_updated = 1;
+
+		seq_state->steps[*seq_state->num_steps].func = HUBP_SET_DMDATA_ATTRIBUTES;
+		seq_state->steps[*seq_state->num_steps].params.set_dmdata_attributes_params.hubp =
+			pipe_ctx->plane_res.hubp;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_link_increase_mst_payload(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx,
+		uint32_t mst_stream_bw)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = LINK_INCREASE_MST_PAYLOAD;
+		seq_state->steps[*seq_state->num_steps].params.link_increase_mst_payload_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.link_increase_mst_payload_params.mst_stream_bw = mst_stream_bw;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_link_reduce_mst_payload(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx,
+		uint32_t mst_stream_bw)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = LINK_REDUCE_MST_PAYLOAD;
+		seq_state->steps[*seq_state->num_steps].params.link_reduce_mst_payload_params.pipe_ctx = pipe_ctx;
+		seq_state->steps[*seq_state->num_steps].params.link_reduce_mst_payload_params.mst_stream_bw = mst_stream_bw;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_dp_set_test_pattern(struct block_sequence_state *seq_state,
+		struct dc_link *link,
+		enum dp_test_pattern test_pattern,
+		enum dp_test_pattern_color_space test_pattern_color_space,
+		const struct link_training_settings *p_link_settings,
+		const unsigned char *p_custom_pattern,
+		unsigned int cust_pattern_size)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = DP_SET_TEST_PATTERN;
+		seq_state->steps[*seq_state->num_steps].params.dp_set_test_pattern_params.link = link;
+		seq_state->steps[*seq_state->num_steps].params.dp_set_test_pattern_params.test_pattern = test_pattern;
+		seq_state->steps[*seq_state->num_steps].params.dp_set_test_pattern_params.test_pattern_color_space = test_pattern_color_space;
+		seq_state->steps[*seq_state->num_steps].params.dp_set_test_pattern_params.p_link_settings = p_link_settings;
+		seq_state->steps[*seq_state->num_steps].params.dp_set_test_pattern_params.p_custom_pattern = p_custom_pattern;
+		seq_state->steps[*seq_state->num_steps].params.dp_set_test_pattern_params.cust_pattern_size = cust_pattern_size;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_link_set_dpms_off(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = LINK_SET_DPMS_OFF;
+		seq_state->steps[*seq_state->num_steps].params.link_set_dpms_off_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_disable_audio_stream(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = DISABLE_AUDIO_STREAM;
+		seq_state->steps[*seq_state->num_steps].params.disable_audio_stream_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_prepare_bandwidth(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct dc_state *context)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = PREPARE_BANDWIDTH;
+		seq_state->steps[*seq_state->num_steps].params.prepare_bandwidth_params.dc = dc;
+		seq_state->steps[*seq_state->num_steps].params.prepare_bandwidth_params.context = context;
+		(*seq_state->num_steps)++;
+	}
+}
+
+void hwss_add_link_set_dpms_on(struct block_sequence_state *seq_state,
+		struct dc_state *state,
+		struct pipe_ctx *pipe_ctx)
+{
+	if (*seq_state->num_steps < MAX_HWSS_BLOCK_SEQUENCE_SIZE) {
+		seq_state->steps[*seq_state->num_steps].func = LINK_SET_DPMS_ON;
+		seq_state->steps[*seq_state->num_steps].params.link_set_dpms_on_params.state = state;
+		seq_state->steps[*seq_state->num_steps].params.link_set_dpms_on_params.pipe_ctx = pipe_ctx;
+		(*seq_state->num_steps)++;
+	}
+}
+
 
 void get_refresh_rate_confirm_color(struct pipe_ctx *pipe_ctx, struct tg_color *color)
 {
@@ -4124,8 +5110,8 @@ void get_refresh_rate_confirm_color(struct pipe_ctx *pipe_ctx, struct tg_color *
 		if (max_refresh_rate - min_refresh_rate)
 			scaling_factor = MAX_TG_COLOR_VALUE * (refresh_rate - min_refresh_rate) / (max_refresh_rate - min_refresh_rate);
 
-		pipe_ctx->visual_confirm_color.color_r_cr = color_value;
-		pipe_ctx->visual_confirm_color.color_g_y = scaling_factor;
-		pipe_ctx->visual_confirm_color.color_b_cb = color_value;
+		pipe_ctx->visual_confirm_color.color_r_cr = (uint16_t)color_value;
+		pipe_ctx->visual_confirm_color.color_g_y = (uint16_t)scaling_factor;
+		pipe_ctx->visual_confirm_color.color_b_cb = (uint16_t)color_value;
 	}
 }
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c
index deb23d20bca67..afdf9f8b16d75 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c
@@ -33,7 +33,7 @@ static bool is_dig_link_enc_stream(struct dc_stream_state *stream)
 {
 	bool is_dig_stream = false;
 	struct link_encoder *link_enc = NULL;
-	int i;
+	unsigned int i;
 
 	/* Loop over created link encoder objects. */
 	if (stream) {
@@ -161,7 +161,7 @@ static enum engine_id find_first_avail_link_enc(
 		enum engine_id eng_id_requested)
 {
 	enum engine_id eng_id = ENGINE_ID_UNKNOWN;
-	int i;
+	unsigned int i;
 
 	if (eng_id_requested != ENGINE_ID_UNKNOWN) {
 
@@ -248,7 +248,7 @@ static struct link_encoder *get_link_enc_used_by_link(
 /* Clear all link encoder assignments. */
 static void clear_enc_assignments(const struct dc *dc, struct dc_state *state)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < MAX_PIPES; i++) {
 		state->res_ctx.link_enc_cfg_ctx.link_enc_assignments[i].valid = false;
@@ -523,7 +523,7 @@ struct link_encoder *link_enc_cfg_get_next_avail_link_enc(struct dc *dc)
 {
 	struct link_encoder *link_enc = NULL;
 	enum engine_id encs_assigned[MAX_LINK_ENCODERS];
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < MAX_LINK_ENCODERS; i++)
 		encs_assigned[i] = ENGINE_ID_UNKNOWN;
@@ -624,7 +624,8 @@ bool link_enc_cfg_validate(struct dc *dc, struct dc_state *state)
 	bool valid_uniqueness = true;
 	bool valid_avail = true;
 	bool valid_streams = true;
-	int i, j;
+	int i;
+	unsigned int j;
 	uint8_t valid_count = 0;
 	uint8_t dig_stream_count = 0;
 	int eng_ids_per_ep_id[MAX_PIPES] = {0};
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
index f4e99ca7918fa..5ac5ad86bd010 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_link_exports.c
@@ -282,7 +282,7 @@ unsigned int dc_dp_trace_get_link_loss_count(struct dc_link *link)
 struct dc_sink *dc_link_add_remote_sink(
 		struct dc_link *link,
 		const uint8_t *edid,
-		int len,
+		unsigned int len,
 		struct dc_sink_init_data *init_data)
 {
 	return link->dc->link_srv->add_remote_sink(link, edid, len, init_data);
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
index 05991a10f8bf2..f57e9d85563e4 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c
@@ -286,34 +286,34 @@ struct resource_pool *dc_create_resource_pool(struct dc  *dc,
 #endif
 	case DCE_VERSION_8_0:
 		res_pool = dce80_create_resource_pool(
-				init_data->num_virtual_links, dc);
+				(uint8_t)init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_8_1:
 		res_pool = dce81_create_resource_pool(
-				init_data->num_virtual_links, dc);
+				(uint8_t)init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_8_3:
 		res_pool = dce83_create_resource_pool(
-				init_data->num_virtual_links, dc);
+				(uint8_t)init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_10_0:
 		res_pool = dce100_create_resource_pool(
-				init_data->num_virtual_links, dc);
+				(uint8_t)init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_11_0:
 		res_pool = dce110_create_resource_pool(
-				init_data->num_virtual_links, dc,
+				(uint8_t)init_data->num_virtual_links, dc,
 				init_data->asic_id);
 		break;
 	case DCE_VERSION_11_2:
 	case DCE_VERSION_11_22:
 		res_pool = dce112_create_resource_pool(
-				init_data->num_virtual_links, dc);
+				(uint8_t)init_data->num_virtual_links, dc);
 		break;
 	case DCE_VERSION_12_0:
 	case DCE_VERSION_12_1:
 		res_pool = dce120_create_resource_pool(
-				init_data->num_virtual_links, dc);
+				(uint8_t)init_data->num_virtual_links, dc);
 		break;
 
 #if defined(CONFIG_DRM_AMD_DC_FP)
@@ -446,7 +446,7 @@ bool resource_construct(
 {
 	struct dc_context *ctx = dc->ctx;
 	const struct resource_caps *caps = pool->res_cap;
-	int i;
+	unsigned int i;
 	unsigned int num_audio = caps->num_audio;
 	struct resource_straps straps = {0};
 
@@ -463,7 +463,7 @@ bool resource_construct(
 		 * PORT_CONNECTIVITY == 1 (as instructed by HW team).
 		 */
 		update_num_audio(&straps, &num_audio, &pool->audio_support);
-		for (i = 0; i < caps->num_audio; i++) {
+		for (i = 0; i < (unsigned int)caps->num_audio; i++) {
 			struct audio *aud = create_funcs->create_audio(ctx, i);
 
 			if (aud == NULL) {
@@ -481,14 +481,14 @@ bool resource_construct(
 
 	pool->stream_enc_count = 0;
 	if (create_funcs->create_stream_encoder) {
-		for (i = 0; i < caps->num_stream_encoder; i++) {
+		for (i = 0; i < (unsigned int)caps->num_stream_encoder; i++) {
 			pool->stream_enc[i] = create_funcs->create_stream_encoder(i, ctx);
 			if (pool->stream_enc[i] == NULL)
 				DC_ERR("DC: failed to create stream_encoder!\n");
 			pool->stream_enc_count++;
 		}
 
-		for (i = 0; i < caps->num_analog_stream_encoder; i++) {
+		for (i = 0; i < (unsigned int)caps->num_analog_stream_encoder; i++) {
 			pool->stream_enc[caps->num_stream_encoder + i] =
 				create_funcs->create_stream_encoder(ENGINE_ID_DACA + i, ctx);
 			if (pool->stream_enc[caps->num_stream_encoder + i] == NULL)
@@ -499,7 +499,7 @@ bool resource_construct(
 
 	pool->hpo_dp_stream_enc_count = 0;
 	if (create_funcs->create_hpo_dp_stream_encoder) {
-		for (i = 0; i < caps->num_hpo_dp_stream_encoder; i++) {
+		for (i = 0; i < (unsigned int)caps->num_hpo_dp_stream_encoder; i++) {
 			pool->hpo_dp_stream_enc[i] = create_funcs->create_hpo_dp_stream_encoder(i+ENGINE_ID_HPO_DP_0, ctx);
 			if (pool->hpo_dp_stream_enc[i] == NULL)
 				DC_ERR("DC: failed to create HPO DP stream encoder!\n");
@@ -510,15 +510,15 @@ bool resource_construct(
 
 	pool->hpo_dp_link_enc_count = 0;
 	if (create_funcs->create_hpo_dp_link_encoder) {
-		for (i = 0; i < caps->num_hpo_dp_link_encoder; i++) {
-			pool->hpo_dp_link_enc[i] = create_funcs->create_hpo_dp_link_encoder(i, ctx);
+		for (i = 0; i < (unsigned int)caps->num_hpo_dp_link_encoder; i++) {
+			pool->hpo_dp_link_enc[i] = create_funcs->create_hpo_dp_link_encoder((uint8_t)i, ctx);
 			if (pool->hpo_dp_link_enc[i] == NULL)
 				DC_ERR("DC: failed to create HPO DP link encoder!\n");
 			pool->hpo_dp_link_enc_count++;
 		}
 	}
 
-	for (i = 0; i < caps->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)caps->num_mpc_3dlut; i++) {
 		pool->mpc_lut[i] = dc_create_3dlut_func();
 		if (pool->mpc_lut[i] == NULL)
 			DC_ERR("DC: failed to create MPC 3dlut!\n");
@@ -551,11 +551,11 @@ static int find_matching_clock_source(
 		struct clock_source *clock_source)
 {
 
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->clk_src_count; i++) {
 		if (pool->clock_sources[i] == clock_source)
-			return i;
+			return (int)i;
 	}
 	return -1;
 }
@@ -610,7 +610,7 @@ bool resource_are_vblanks_synchronizable(
 {
 	uint32_t base60_refresh_rates[] = {10, 20, 5};
 	uint8_t i;
-	uint8_t rr_count = ARRAY_SIZE(base60_refresh_rates);
+	uint8_t rr_count = (uint8_t)ARRAY_SIZE(base60_refresh_rates);
 	uint64_t frame_time_diff;
 
 	if (stream1->ctx->dc->config.vblank_alignment_dto_params &&
@@ -982,7 +982,7 @@ static struct rect calculate_mpc_slice_in_timing_active(
 }
 
 static void calculate_adjust_recout_for_visual_confirm(struct pipe_ctx *pipe_ctx,
-	int *base_offset, int *dpp_offset)
+	unsigned int *base_offset, unsigned int *dpp_offset)
 {
 	struct dc *dc = pipe_ctx->stream->ctx->dc;
 	*base_offset = 0;
@@ -1004,7 +1004,7 @@ static void calculate_adjust_recout_for_visual_confirm(struct pipe_ctx *pipe_ctx
 static void reverse_adjust_recout_for_visual_confirm(struct rect *recout,
 		struct pipe_ctx *pipe_ctx)
 {
-	int dpp_offset, base_offset;
+	unsigned int dpp_offset, base_offset;
 
 	calculate_adjust_recout_for_visual_confirm(pipe_ctx, &base_offset,
 		&dpp_offset);
@@ -1015,7 +1015,7 @@ static void reverse_adjust_recout_for_visual_confirm(struct rect *recout,
 static void adjust_recout_for_visual_confirm(struct rect *recout,
 		struct pipe_ctx *pipe_ctx)
 {
-	int dpp_offset, base_offset;
+	unsigned int dpp_offset, base_offset;
 
 	calculate_adjust_recout_for_visual_confirm(pipe_ctx, &base_offset,
 		&dpp_offset);
@@ -1692,7 +1692,7 @@ bool resource_can_pipe_disable_cursor(struct pipe_ctx *pipe_ctx)
 	struct pipe_ctx *test_pipe, *split_pipe;
 	struct rect r1 = pipe_ctx->plane_res.scl_data.recout;
 	int r1_right, r1_bottom;
-	int cur_layer = pipe_ctx->plane_state->layer_index;
+	unsigned int cur_layer = pipe_ctx->plane_state->layer_index;
 
 	reverse_adjust_recout_for_visual_confirm(&r1, pipe_ctx);
 	r1_right = r1.x + r1.width;
@@ -1801,7 +1801,7 @@ struct pipe_ctx *resource_find_free_secondary_pipe_legacy(
 		int preferred_pipe_idx = (pool->pipe_count - 1) - primary_pipe->pipe_idx;
 		if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
 			secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-			secondary_pipe->pipe_idx = preferred_pipe_idx;
+			secondary_pipe->pipe_idx = (uint8_t)preferred_pipe_idx;
 		}
 	}
 
@@ -1813,7 +1813,7 @@ struct pipe_ctx *resource_find_free_secondary_pipe_legacy(
 		for (i = pool->pipe_count - 1; i >= 0; i--) {
 			if (res_ctx->pipe_ctx[i].stream == NULL) {
 				secondary_pipe = &res_ctx->pipe_ctx[i];
-				secondary_pipe->pipe_idx = i;
+				secondary_pipe->pipe_idx = (uint8_t)i;
 				break;
 			}
 		}
@@ -1876,7 +1876,7 @@ int recource_find_free_pipe_not_used_in_cur_res_ctx(
 {
 	int free_pipe_idx = FREE_PIPE_INDEX_NOT_FOUND;
 	const struct pipe_ctx *new_pipe, *cur_pipe;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->pipe_count; i++) {
 		cur_pipe = &cur_res_ctx->pipe_ctx[i];
@@ -1899,7 +1899,7 @@ int recource_find_free_pipe_used_as_otg_master_in_cur_res_ctx(
 {
 	int free_pipe_idx = FREE_PIPE_INDEX_NOT_FOUND;
 	const struct pipe_ctx *new_pipe, *cur_pipe;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->pipe_count; i++) {
 		cur_pipe = &cur_res_ctx->pipe_ctx[i];
@@ -1922,7 +1922,7 @@ int resource_find_free_pipe_used_as_cur_sec_dpp(
 {
 	int free_pipe_idx = FREE_PIPE_INDEX_NOT_FOUND;
 	const struct pipe_ctx *new_pipe, *cur_pipe;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->pipe_count; i++) {
 		cur_pipe = &cur_res_ctx->pipe_ctx[i];
@@ -1946,7 +1946,7 @@ int resource_find_free_pipe_used_as_cur_sec_dpp_in_mpcc_combine(
 {
 	int free_pipe_idx = FREE_PIPE_INDEX_NOT_FOUND;
 	const struct pipe_ctx *new_pipe, *cur_pipe;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->pipe_count; i++) {
 		cur_pipe = &cur_res_ctx->pipe_ctx[i];
@@ -1969,7 +1969,7 @@ int resource_find_any_free_pipe(struct resource_context *new_res_ctx,
 {
 	int free_pipe_idx = FREE_PIPE_INDEX_NOT_FOUND;
 	const struct pipe_ctx *new_pipe;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->pipe_count; i++) {
 		new_pipe = &new_res_ctx->pipe_ctx[i];
@@ -2564,7 +2564,7 @@ static bool update_pipe_params_after_odm_slice_count_change(
 		struct dc_state *context,
 		const struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	struct pipe_ctx *pipe;
 	bool result = true;
 
@@ -2587,7 +2587,7 @@ static bool update_pipe_params_after_mpc_slice_count_change(
 		struct dc_state *context,
 		const struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	struct pipe_ctx *pipe;
 	bool result = true;
 
@@ -2604,7 +2604,7 @@ static int acquire_first_split_pipe(
 		const struct resource_pool *pool,
 		struct dc_stream_state *stream)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->pipe_count; i++) {
 		struct pipe_ctx *split_pipe = &res_ctx->pipe_ctx[i];
@@ -2624,11 +2624,11 @@ static int acquire_first_split_pipe(
 			split_pipe->plane_res.ipp = pool->ipps[i];
 			split_pipe->plane_res.dpp = pool->dpps[i];
 			split_pipe->stream_res.opp = pool->opps[i];
-			split_pipe->plane_res.mpcc_inst = pool->dpps[i]->inst;
-			split_pipe->pipe_idx = i;
+			split_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[i]->inst;
+			split_pipe->pipe_idx = (uint8_t)i;
 
 			split_pipe->stream = stream;
-			return i;
+			return (int)i;
 		}
 	}
 	return FREE_PIPE_INDEX_NOT_FOUND;
@@ -2640,7 +2640,7 @@ static void update_stream_engine_usage(
 		struct stream_encoder *stream_enc,
 		bool acquired)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->stream_enc_count; i++) {
 		if (pool->stream_enc[i] == stream_enc)
@@ -2654,7 +2654,7 @@ static void update_hpo_dp_stream_engine_usage(
 		struct hpo_dp_stream_encoder *hpo_dp_stream_enc,
 		bool acquired)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->hpo_dp_stream_enc_count; i++) {
 		if (pool->hpo_dp_stream_enc[i] == hpo_dp_stream_enc)
@@ -2679,14 +2679,14 @@ static inline int find_acquired_hpo_dp_link_enc_for_link(
 static inline int find_free_hpo_dp_link_enc(const struct resource_context *res_ctx,
 		const struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < ARRAY_SIZE(res_ctx->hpo_dp_link_enc_ref_cnts); i++)
 		if (res_ctx->hpo_dp_link_enc_ref_cnts[i] == 0)
 			break;
 
 	return (i < ARRAY_SIZE(res_ctx->hpo_dp_link_enc_ref_cnts) &&
-			i < pool->hpo_dp_link_enc_count) ? i : -1;
+			i < pool->hpo_dp_link_enc_count) ? (int)i : -1;
 }
 
 static inline void acquire_hpo_dp_link_enc(
@@ -2773,9 +2773,10 @@ static inline int find_fixed_dio_link_enc(const struct dc_link *link)
 static inline int find_free_dio_link_enc(const struct resource_context *res_ctx,
 		const struct dc_link *link, const struct resource_pool *pool, struct dc_stream_state *stream)
 {
-	int i, j = -1;
+	unsigned int i;
+	int j = -1;
 	int stream_enc_inst = -1;
-	int enc_count = pool->dig_link_enc_count;
+	unsigned int enc_count = pool->dig_link_enc_count;
 
 	/* Find stream encoder instance for the stream */
 	if (stream) {
@@ -2931,7 +2932,7 @@ static void remove_dio_link_enc_from_ctx(struct resource_context *res_ctx,
 
 static int get_num_of_free_pipes(const struct resource_pool *pool, const struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	int count = 0;
 
 	for (i = 0; i < pool->pipe_count; i++)
@@ -3608,7 +3609,7 @@ void update_audio_usage(
 		struct audio *audio,
 		bool acquired)
 {
-	int i;
+	unsigned int i;
 	for (i = 0; i < pool->audio_count; i++) {
 		if (pool->audios[i] == audio)
 			res_ctx->is_audio_acquired[i] = acquired;
@@ -3621,7 +3622,7 @@ static struct hpo_dp_stream_encoder *find_first_free_match_hpo_dp_stream_enc_for
 		struct dc_stream_state *stream)
 {
 	(void)stream;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->hpo_dp_stream_enc_count; i++) {
 		if (!res_ctx->is_hpo_dp_stream_enc_acquired[i] &&
@@ -3804,7 +3805,7 @@ static int acquire_resource_from_hw_enabled_state(
 			pipe_ctx->stream_res.opp = pool->opps[id_src[i]];
 
 			if (pool->dpps[id_src[i]]) {
-				pipe_ctx->plane_res.mpcc_inst = pool->dpps[id_src[i]]->inst;
+				pipe_ctx->plane_res.mpcc_inst = (uint8_t)pool->dpps[id_src[i]]->inst;
 
 				if (pool->mpc->funcs->read_mpcc_state) {
 					struct mpcc_state s = {0};
@@ -3823,7 +3824,7 @@ static int acquire_resource_from_hw_enabled_state(
 						pipe_ctx->stream_res.opp->mpc_tree_params.opp_id = s.opp_id;
 				}
 			}
-			pipe_ctx->pipe_idx = id_src[i];
+			pipe_ctx->pipe_idx = (uint8_t)id_src[i];
 
 			if (id_src[i] >= pool->timing_generator_count) {
 				id_src[i] = pool->timing_generator_count - 1;
@@ -3939,7 +3940,7 @@ static bool acquire_otg_master_pipe_for_stream(
 	if (pipe_idx != FREE_PIPE_INDEX_NOT_FOUND) {
 		pipe_ctx = &new_ctx->res_ctx.pipe_ctx[pipe_idx];
 		memset(pipe_ctx, 0, sizeof(*pipe_ctx));
-		pipe_ctx->pipe_idx = pipe_idx;
+		pipe_ctx->pipe_idx = (uint8_t)pipe_idx;
 		pipe_ctx->stream_res.tg = pool->timing_generators[pipe_idx];
 		pipe_ctx->plane_res.mi = pool->mis[pipe_idx];
 		pipe_ctx->plane_res.hubp = pool->hubps[pipe_idx];
@@ -3948,9 +3949,9 @@ static bool acquire_otg_master_pipe_for_stream(
 		pipe_ctx->plane_res.dpp = pool->dpps[pipe_idx];
 		pipe_ctx->stream_res.opp = pool->opps[pipe_idx];
 		if (pool->dpps[pipe_idx])
-			pipe_ctx->plane_res.mpcc_inst = pool->dpps[pipe_idx]->inst;
+			pipe_ctx->plane_res.mpcc_inst = (uint8_t)pool->dpps[pipe_idx]->inst;
 
-		if (pipe_idx >= pool->timing_generator_count && pool->timing_generator_count != 0) {
+		if ((unsigned int)pipe_idx >= pool->timing_generator_count && pool->timing_generator_count != 0) {
 			int tg_inst = pool->timing_generator_count - 1;
 
 			pipe_ctx->stream_res.tg = pool->timing_generators[tg_inst];
@@ -4102,9 +4103,9 @@ bool dc_resource_is_dsc_encoding_supported(const struct dc *dc)
 static bool planes_changed_for_existing_stream(struct dc_state *context,
 					       struct dc_stream_state *stream,
 					       const struct dc_validation_set set[],
-					       int set_count)
+					       unsigned int set_count)
 {
-	int i, j;
+	unsigned int i, j;
 	struct dc_stream_status *stream_status = NULL;
 
 	for (i = 0; i < context->stream_count; i++) {
@@ -4140,10 +4141,10 @@ static bool add_all_planes_for_stream(
 		const struct dc *dc,
 		struct dc_stream_state *stream,
 		const struct dc_validation_set set[],
-		int set_count,
+		unsigned int set_count,
 		struct dc_state *state)
 {
-	int i, j;
+	unsigned int i, j;
 
 	for (i = 0; i < set_count; i++)
 		if (set[i].stream == stream)
@@ -4181,7 +4182,7 @@ static bool add_all_planes_for_stream(
  */
 enum dc_status dc_validate_with_context(struct dc *dc,
 					const struct dc_validation_set set[],
-					int set_count,
+					unsigned int set_count,
 					struct dc_state *context,
 					enum dc_validate_mode validate_mode)
 {
@@ -4194,7 +4195,8 @@ enum dc_status dc_validate_with_context(struct dc *dc,
 	int del_streams_count = 0;
 	int add_streams_count = 0;
 	bool found = false;
-	int i, j, k;
+	int i, j;
+	unsigned int k;
 
 	DC_LOGGER_INIT(dc->ctx->logger);
 
@@ -4421,7 +4423,8 @@ enum dc_status dc_validate_global_state(
 		enum dc_validate_mode validate_mode)
 {
 	enum dc_status result = DC_ERROR_UNEXPECTED;
-	int i, j;
+	int i;
+	unsigned int j;
 
 	if (!new_ctx)
 		return DC_ERROR_UNEXPECTED;
@@ -4497,7 +4500,7 @@ static void patch_gamut_packet_checksum(
 		for (i = 0; i <= gamut_packet->sb[1]; i++)
 			chk_sum += ptr[i];
 
-		gamut_packet->sb[2] = (uint8_t) (0x100 - chk_sum);
+		gamut_packet->sb[2] = (uint8_t)(0x100 - chk_sum);
 	}
 }
 
@@ -4562,7 +4565,7 @@ static void set_avi_info_frame(
 
 	/* Y0_Y1_Y2 : The pixel encoding */
 	/* H14b AVI InfoFrame has extension on Y-field from 2 bits to 3 bits */
-	hdmi_info.bits.Y0_Y1_Y2 = pixel_encoding;
+	hdmi_info.bits.Y0_Y1_Y2 = (uint8_t)pixel_encoding;
 
 	/* A0 = 1 Active Format Information valid */
 	hdmi_info.bits.A0 = ACTIVE_FORMAT_VALID;
@@ -4692,7 +4695,7 @@ static void set_avi_info_frame(
 		}
 	}
 	/* If VIC >= 128, the Source shall use AVI InfoFrame Version 3*/
-	hdmi_info.bits.VIC0_VIC7 = vic;
+	hdmi_info.bits.VIC0_VIC7 = (uint8_t)vic;
 	if (vic >= 128)
 		hdmi_info.bits.header.version = 3;
 	/* If (C1, C0)=(1, 1) and (EC2, EC1, EC0)=(1, 1, 1),
@@ -4710,7 +4713,7 @@ static void set_avi_info_frame(
 
 		hdmi_info.bits.FR0_FR3 = fr_ind & 0xF;
 		hdmi_info.bits.FR4 = (fr_ind >> 4) & 0x1;
-		hdmi_info.bits.RID0_RID5 = rid;
+		hdmi_info.bits.RID0_RID5 = (uint8_t)rid;
 	}
 
 	/* pixel repetition
@@ -4723,11 +4726,11 @@ static void set_avi_info_frame(
 	 * barBottom: Line Number of Start of Bottom Bar.
 	 * barLeft:   Pixel Number of End of Left Bar.
 	 * barRight:  Pixel Number of Start of Right Bar. */
-	hdmi_info.bits.bar_top = stream->timing.v_border_top;
-	hdmi_info.bits.bar_bottom = (stream->timing.v_total
+	hdmi_info.bits.bar_top = (uint16_t)stream->timing.v_border_top;
+	hdmi_info.bits.bar_bottom = (uint16_t)(stream->timing.v_total
 			- stream->timing.v_border_bottom + 1);
-	hdmi_info.bits.bar_left  = stream->timing.h_border_left;
-	hdmi_info.bits.bar_right = (stream->timing.h_total
+	hdmi_info.bits.bar_left  = (uint16_t)stream->timing.h_border_left;
+	hdmi_info.bits.bar_right = (uint16_t)(stream->timing.h_total
 			- stream->timing.h_border_right + 1);
 
     /* Additional Colorimetry Extension
@@ -4746,7 +4749,7 @@ static void set_avi_info_frame(
 		*check_sum += hdmi_info.packet_raw_data.sb[byte_index];
 
 	/* one byte complement */
-	*check_sum = (uint8_t) (0x100 - *check_sum);
+	*check_sum = (uint8_t)(0x100 - *check_sum);
 
 	/* Store in hw_path_mode */
 	info_packet->hb0 = hdmi_info.packet_raw_data.hb0;
@@ -4880,7 +4883,7 @@ struct clock_source *dc_resource_find_first_free_pll(
 		struct resource_context *res_ctx,
 		const struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->clk_src_count; ++i) {
 		if (res_ctx->clock_source_ref_count[i] == 0)
@@ -5369,7 +5372,7 @@ bool get_temp_dp_link_res(struct dc_link *link,
 void reset_syncd_pipes_from_disabled_pipes(struct dc *dc,
 		struct dc_state *context)
 {
-	int i, j;
+	uint8_t i, j;
 	struct pipe_ctx *pipe_ctx_old, *pipe_ctx, *pipe_ctx_syncd;
 
 	/* If pipe backend is reset, need to reset pipe syncd status */
@@ -5398,7 +5401,7 @@ void check_syncd_pipes_for_disabled_master_pipe(struct dc *dc,
 	struct dc_state *context,
 	uint8_t disabled_master_pipe_idx)
 {
-	int i;
+	unsigned int i;
 	struct pipe_ctx *pipe_ctx, *pipe_ctx_check;
 
 	pipe_ctx = &context->res_ctx.pipe_ctx[disabled_master_pipe_idx];
@@ -5432,7 +5435,7 @@ void reset_sync_context_for_pipe(const struct dc *dc,
 	struct dc_state *context,
 	uint8_t pipe_idx)
 {
-	int i;
+	uint8_t i;
 	struct pipe_ctx *pipe_ctx_reset;
 
 	/* reset the otg sync context for the pipe and its slave pipes if any */
@@ -5448,7 +5451,7 @@ void reset_sync_context_for_pipe(const struct dc *dc,
 uint8_t resource_transmitter_to_phy_idx(const struct dc *dc, enum transmitter transmitter)
 {
 	/* TODO - get transmitter to phy idx mapping from DMUB */
-	uint8_t phy_idx = transmitter - TRANSMITTER_UNIPHY_A;
+	uint8_t phy_idx = (uint8_t)(transmitter - TRANSMITTER_UNIPHY_A);
 
 	if (dc->ctx->dce_version == DCN_VERSION_3_1 &&
 			dc->ctx->asic_id.hw_internal_rev == YELLOW_CARP_B0) {
@@ -5515,8 +5518,8 @@ const struct link_hwss *get_link_hwss(const struct dc_link *link,
 bool is_h_timing_divisible_by_2(struct dc_stream_state *stream)
 {
 	bool divisible = false;
-	uint16_t h_blank_start = 0;
-	uint16_t h_blank_end = 0;
+	uint32_t h_blank_start = 0;
+	uint32_t h_blank_end = 0;
 
 	if (stream) {
 		h_blank_start = stream->timing.h_total - stream->timing.h_front_porch;
@@ -5570,13 +5573,13 @@ bool dc_resource_acquire_secondary_pipe_for_mpc_odm_legacy(
 	sec_pipe->next_odm_pipe = sec_next;
 	sec_pipe->prev_odm_pipe = sec_prev;
 
-	sec_pipe->pipe_idx = pipe_idx;
+	sec_pipe->pipe_idx = (uint8_t)pipe_idx;
 	sec_pipe->plane_res.mi = pool->mis[pipe_idx];
 	sec_pipe->plane_res.hubp = pool->hubps[pipe_idx];
 	sec_pipe->plane_res.ipp = pool->ipps[pipe_idx];
 	sec_pipe->plane_res.xfm = pool->transforms[pipe_idx];
 	sec_pipe->plane_res.dpp = pool->dpps[pipe_idx];
-	sec_pipe->plane_res.mpcc_inst = pool->dpps[pipe_idx]->inst;
+	sec_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[pipe_idx]->inst;
 	sec_pipe->stream_res.dsc = NULL;
 	if (odm) {
 		if (!sec_pipe->top_pipe)
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_state.c b/drivers/gpu/drm/amd/display/dc/core/dc_state.c
index 40f7aa7322581..7edeed52cc328 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_state.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_state.c
@@ -218,13 +218,13 @@ struct dc_state *dc_state_create(struct dc *dc, struct dc_state_create_params *p
 		}
 
 		if (dc->caps.dcmode_power_limits_present) {
-			bool status;
+			bool dc_power_status;
 
 			DC_FP_START();
-			status = dml2_create(dc, &dc->dml2_dc_power_options, &state->bw_ctx.dml2_dc_power_source);
+			dc_power_status = dml2_create(dc, &dc->dml2_dc_power_options, &state->bw_ctx.dml2_dc_power_source);
 			DC_FP_END();
 
-			if (!status) {
+			if (!dc_power_status) {
 				dc_state_release(state);
 				return NULL;
 			}
@@ -953,7 +953,7 @@ bool dc_state_remove_phantom_streams_and_planes(
 	const struct dc *dc,
 	struct dc_state *state)
 {
-	int i;
+	unsigned int i;
 	bool removed_phantom = false;
 	struct dc_stream_state *phantom_stream = NULL;
 
@@ -978,7 +978,7 @@ void dc_state_release_phantom_streams_and_planes(
 	unsigned int phantom_count;
 	struct dc_stream_state *phantom_streams[MAX_PHANTOM_PIPES];
 	struct dc_plane_state *phantom_planes[MAX_PHANTOM_PIPES];
-	int i;
+	unsigned int i;
 
 	phantom_count = state->phantom_stream_count;
 	memcpy(phantom_streams, state->phantom_streams, sizeof(struct dc_stream_state *) * MAX_PHANTOM_PIPES);
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
index e16de323f39c0..434654e903b11 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c
@@ -259,11 +259,17 @@ const struct dc_stream_status *dc_stream_get_status_const(
 	return dc_state_get_stream_status(dc->current_state, stream);
 }
 
+struct dc_link *dc_stream_get_link(
+	const struct dc_stream_state *stream)
+{
+	return stream->link;
+}
+
 void program_cursor_attributes(
 	struct dc *dc,
 	struct dc_stream_state *stream)
 {
-	int i;
+	uint8_t i;
 	struct resource_context *res_ctx;
 	struct pipe_ctx *pipe_to_program = NULL;
 	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
@@ -410,7 +416,7 @@ void program_cursor_position(
 	struct dc *dc,
 	struct dc_stream_state *stream)
 {
-	int i;
+	uint8_t i;
 	struct resource_context *res_ctx;
 	struct pipe_ctx *pipe_to_program = NULL;
 	bool enable_cursor_offload = dc_dmub_srv_is_cursor_offload_enabled(dc);
@@ -508,7 +514,7 @@ bool dc_stream_program_cursor_position(
 		/* apply/update visual confirm */
 		if (dc->debug.visual_confirm == VISUAL_CONFIRM_HW_CURSOR) {
 			/* update software state */
-			int i;
+			unsigned int i;
 
 			for (i = 0; i < dc->res_pool->pipe_count; i++) {
 				struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
@@ -527,7 +533,7 @@ bool dc_stream_program_cursor_position(
 
 		if (stream->drr_trigger_mode == DRR_TRIGGER_ON_FLIP_AND_CURSOR) {
 			/* apply manual trigger */
-			int i;
+			unsigned int i;
 
 			for (i = 0; i < dc->res_pool->pipe_count; i++) {
 				struct pipe_ctx *pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
@@ -553,7 +559,7 @@ bool dc_stream_add_writeback(struct dc *dc,
 		struct dc_writeback_info *wb_info)
 {
 	bool isDrc = false;
-	int i = 0;
+	unsigned int i = 0;
 	struct dwbc *dwb;
 
 	if (stream == NULL) {
@@ -596,7 +602,7 @@ bool dc_stream_add_writeback(struct dc *dc,
 
 	if (dc->hwss.enable_writeback) {
 		struct dc_stream_status *stream_status = dc_stream_get_status(stream);
-		struct dwbc *dwb = dc->res_pool->dwbc[wb_info->dwb_pipe_inst];
+		dwb = dc->res_pool->dwbc[wb_info->dwb_pipe_inst];
 		if (stream_status)
 			dwb->otg_inst = stream_status->primary_otg_inst;
 	}
@@ -608,7 +614,7 @@ bool dc_stream_add_writeback(struct dc *dc,
 
 	/* enable writeback */
 	if (dc->hwss.enable_writeback) {
-		struct dwbc *dwb = dc->res_pool->dwbc[wb_info->dwb_pipe_inst];
+		dwb = dc->res_pool->dwbc[wb_info->dwb_pipe_inst];
 
 		if (dwb->funcs->is_enabled(dwb)) {
 			/* writeback pipe already enabled, only need to update */
@@ -962,7 +968,7 @@ struct dc_rmcm_3dlut *dc_stream_get_3dlut_for_stream(
 	unsigned int num_rmcm = dc->caps.color.mpc.num_rmcm_3dluts;
 
 	// see if one is allocated for this stream
-	for (int i = 0; i < num_rmcm; i++) {
+	for (unsigned int i = 0; i < num_rmcm; i++) {
 		if (dc->res_pool->rmcm_3dlut[i].isInUse &&
 			dc->res_pool->rmcm_3dlut[i].stream == stream)
 			return &dc->res_pool->rmcm_3dlut[i];
@@ -973,7 +979,7 @@ struct dc_rmcm_3dlut *dc_stream_get_3dlut_for_stream(
 		return NULL;
 
 	//see if there is an unused 3dlut, allocate
-	for (int i = 0; i < num_rmcm; i++) {
+	for (unsigned int i = 0; i < num_rmcm; i++) {
 		if (!dc->res_pool->rmcm_3dlut[i].isInUse) {
 			dc->res_pool->rmcm_3dlut[i].isInUse = true;
 			dc->res_pool->rmcm_3dlut[i].stream = stream;
@@ -996,6 +1002,7 @@ void dc_stream_release_3dlut_for_stream(
 	if (rmcm_3dlut) {
 		rmcm_3dlut->isInUse = false;
 		rmcm_3dlut->stream  = NULL;
+		rmcm_3dlut->protection_bits = 0;
 	}
 }
 
@@ -1004,9 +1011,10 @@ void dc_stream_init_rmcm_3dlut(struct dc *dc)
 {
 	unsigned int num_rmcm = dc->caps.color.mpc.num_rmcm_3dluts;
 
-	for (int i = 0; i < num_rmcm; i++) {
+	for (unsigned int i = 0; i < num_rmcm; i++) {
 		dc->res_pool->rmcm_3dlut[i].isInUse = false;
 		dc->res_pool->rmcm_3dlut[i].stream = NULL;
+		dc->res_pool->rmcm_3dlut[i].protection_bits = 0;
 	}
 }
 
@@ -1032,14 +1040,18 @@ static int dc_stream_get_brightness_millinits_linear_interpolation (struct dc_st
 								     int refresh_hz)
 {
 	long long slope = 0;
+	long long y_intercept = 0;
+	long long brightness_millinits = 0;
+
 	if (stream->lumin_data.refresh_rate_hz[index2] != stream->lumin_data.refresh_rate_hz[index1]) {
 		slope = (stream->lumin_data.luminance_millinits[index2] - stream->lumin_data.luminance_millinits[index1]) /
 			    (stream->lumin_data.refresh_rate_hz[index2] - stream->lumin_data.refresh_rate_hz[index1]);
 	}
 
-	int y_intercept = stream->lumin_data.luminance_millinits[index2] - slope * stream->lumin_data.refresh_rate_hz[index2];
+	y_intercept = stream->lumin_data.luminance_millinits[index2] - slope * stream->lumin_data.refresh_rate_hz[index2];
+	brightness_millinits = y_intercept + (long long)refresh_hz * slope;
 
-	return (y_intercept + refresh_hz * slope);
+	return (int)brightness_millinits;
 }
 
 /*
@@ -1051,14 +1063,18 @@ static int dc_stream_get_refresh_hz_linear_interpolation (struct dc_stream_state
 							   int brightness_millinits)
 {
 	long long slope = 1;
+	long long y_intercept = 0;
+	long long refresh_hz = 0;
+
 	if (stream->lumin_data.refresh_rate_hz[index2] != stream->lumin_data.refresh_rate_hz[index1]) {
 		slope = (stream->lumin_data.luminance_millinits[index2] - stream->lumin_data.luminance_millinits[index1]) /
 				(stream->lumin_data.refresh_rate_hz[index2] - stream->lumin_data.refresh_rate_hz[index1]);
 	}
 
-	int y_intercept = stream->lumin_data.luminance_millinits[index2] - slope * stream->lumin_data.refresh_rate_hz[index2];
+	y_intercept = stream->lumin_data.luminance_millinits[index2] - slope * stream->lumin_data.refresh_rate_hz[index2];
+	refresh_hz = div64_s64((brightness_millinits - y_intercept), slope);
 
-	return ((int)div64_s64((brightness_millinits - y_intercept), slope));
+	return (int)refresh_hz;
 }
 
 /*
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
index a59b176d8e552..72845fc788f3f 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_surface.c
@@ -69,13 +69,13 @@ void dc_plane_destruct(struct dc_plane_state *plane_state)
 uint8_t  dc_plane_get_pipe_mask(struct dc_state *dc_state, const struct dc_plane_state *plane_state)
 {
 	uint8_t pipe_mask = 0;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < plane_state->ctx->dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &dc_state->res_ctx.pipe_ctx[i];
 
 		if (pipe_ctx->plane_state == plane_state && pipe_ctx->plane_res.hubp)
-			pipe_mask |= 1 << pipe_ctx->plane_res.hubp->inst;
+			pipe_mask |= (uint8_t)(1 << pipe_ctx->plane_res.hubp->inst);
 	}
 
 	return pipe_mask;
@@ -115,7 +115,7 @@ const struct dc_plane_status *dc_plane_get_status(
 {
 	const struct dc_plane_status *plane_status;
 	struct dc  *dc;
-	int i;
+	unsigned int i;
 
 	if (!plane_state ||
 		!plane_state->ctx ||
@@ -286,7 +286,7 @@ void dc_plane_force_dcc_and_tiling_disable(struct dc_plane_state *plane_state,
 					   bool clear_tiling)
 {
 	struct dc *dc;
-	int i;
+	unsigned int i;
 
 	if (!plane_state)
 		return;
diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c b/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c
index d1e68dc57a2ae..3743555133f6f 100644
--- a/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/core/dc_vm_helper.c
@@ -34,9 +34,9 @@ void vm_helper_mark_vmid_used(struct vm_helper *vm_helper, unsigned int pos, uin
 	vmids.vmid_usage[1] = 1 << pos;
 }
 
-int dc_setup_system_context(struct dc *dc, struct dc_phy_addr_space_config *pa_config)
+unsigned int dc_setup_system_context(struct dc *dc, struct dc_phy_addr_space_config *pa_config)
 {
-	int num_vmids = 0;
+	unsigned int num_vmids = 0;
 
 	/* Call HWSS to setup HUBBUB for address config */
 	if (dc->hwss.init_sys_ctx) {
diff --git a/drivers/gpu/drm/amd/display/dc/dc.h b/drivers/gpu/drm/amd/display/dc/dc.h
index 37714d4371fba..75f59ca927ffe 100644
--- a/drivers/gpu/drm/amd/display/dc/dc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc.h
@@ -63,7 +63,7 @@ struct dcn_dsc_reg_state;
 struct dcn_optc_reg_state;
 struct dcn_dccg_reg_state;
 
-#define DC_VER "3.2.378"
+#define DC_VER "3.2.383"
 
 /**
  * MAX_SURFACES - representative of the upper bound of surfaces that can be piped to a single CRTC
@@ -540,7 +540,7 @@ struct dc_config {
 	bool use_default_clock_table;
 	bool force_bios_enable_lttpr;
 	uint8_t force_bios_fixed_vs;
-	int sdpif_request_limit_words_per_umc;
+	unsigned int sdpif_request_limit_words_per_umc;
 	bool dc_mode_clk_limit_support;
 	bool EnableMinDispClkODM;
 	bool enable_auto_dpm_test_logs;
@@ -944,20 +944,20 @@ struct dc_virtual_addr_space_config {
 };
 
 struct dc_bounding_box_overrides {
-	int sr_exit_time_ns;
-	int sr_enter_plus_exit_time_ns;
-	int sr_exit_z8_time_ns;
-	int sr_enter_plus_exit_z8_time_ns;
-	int urgent_latency_ns;
-	int percent_of_ideal_drambw;
-	int dram_clock_change_latency_ns;
-	int dummy_clock_change_latency_ns;
-	int fclk_clock_change_latency_ns;
+	unsigned int sr_exit_time_ns;
+	unsigned int sr_enter_plus_exit_time_ns;
+	unsigned int sr_exit_z8_time_ns;
+	unsigned int sr_enter_plus_exit_z8_time_ns;
+	unsigned int urgent_latency_ns;
+	unsigned int percent_of_ideal_drambw;
+	unsigned int dram_clock_change_latency_ns;
+	unsigned int dummy_clock_change_latency_ns;
+	unsigned int fclk_clock_change_latency_ns;
 	/* This forces a hard min on the DCFCLK we use
 	 * for DML.  Unlike the debug option for forcing
 	 * DCFCLK, this override affects watermark calculations
 	 */
-	int min_dcfclk_mhz;
+	unsigned int min_dcfclk_mhz;
 };
 
 struct dc_qos_info {
@@ -971,6 +971,7 @@ struct dc_qos_info {
 	uint32_t actual_avg_latency_in_ns;
 	uint32_t qos_avg_latency_ub_in_ns;
 	uint32_t dcn_bandwidth_ub_in_mbps;
+	uint32_t qos_max_bw_budget_in_mbps;
 };
 
 struct dc_state;
@@ -989,7 +990,7 @@ struct link_service;
 struct dc_debug_options {
 	bool disable_dsc;
 	enum visual_confirm visual_confirm;
-	int visual_confirm_rect_height;
+	unsigned int visual_confirm_rect_height;
 
 	bool sanity_checks;
 	bool max_disp_clk;
@@ -1025,23 +1026,23 @@ struct dc_debug_options {
 	bool disable_io_clk_power_gate;
 	bool disable_mem_power_gate;
 	bool disable_dio_power_gate;
-	int dsc_min_slice_height_override;
-	int dsc_bpp_increment_div;
+	unsigned int dsc_min_slice_height_override;
+	unsigned int dsc_bpp_increment_div;
 	bool disable_pplib_wm_range;
 	enum wm_report_mode pplib_wm_report_mode;
 	unsigned int min_disp_clk_khz;
 	unsigned int min_dpp_clk_khz;
 	unsigned int min_dram_clk_khz;
-	int sr_exit_time_dpm0_ns;
-	int sr_enter_plus_exit_time_dpm0_ns;
-	int sr_exit_time_ns;
-	int sr_enter_plus_exit_time_ns;
-	int sr_exit_z8_time_ns;
-	int sr_enter_plus_exit_z8_time_ns;
-	int urgent_latency_ns;
+	unsigned int sr_exit_time_dpm0_ns;
+	unsigned int sr_enter_plus_exit_time_dpm0_ns;
+	unsigned int sr_exit_time_ns;
+	unsigned int sr_enter_plus_exit_time_ns;
+	unsigned int sr_exit_z8_time_ns;
+	unsigned int sr_enter_plus_exit_z8_time_ns;
+	unsigned int urgent_latency_ns;
 	uint32_t underflow_assert_delay_us;
-	int percent_of_ideal_drambw;
-	int dram_clock_change_latency_ns;
+	unsigned int percent_of_ideal_drambw;
+	unsigned int dram_clock_change_latency_ns;
 	bool optimized_watermark;
 	int always_scale;
 	bool disable_pplib_clock_request;
@@ -1066,8 +1067,8 @@ struct dc_debug_options {
 	uint8_t seamless_boot_odm_combine;
 	uint8_t force_odm_combine_4to1; //bit vector based on otg inst
 
-	int minimum_z8_residency_time;
-	int minimum_z10_residency_time;
+	unsigned int minimum_z8_residency_time;
+	unsigned int minimum_z10_residency_time;
 	bool disable_z9_mpc;
 	unsigned int force_fclk_khz;
 	bool enable_tri_buf;
@@ -1116,7 +1117,7 @@ struct dc_debug_options {
 	uint8_t fec_enable_delay_in100us;
 	bool enable_driver_sequence_debug;
 	enum det_size crb_alloc_policy;
-	int crb_alloc_policy_min_disp_count;
+	unsigned int crb_alloc_policy_min_disp_count;
 	bool disable_z10;
 	bool enable_z9_disable_interface;
 	bool psr_skip_crtc_disable;
@@ -1218,6 +1219,8 @@ struct dc_debug_options {
 	unsigned int force_vmin_threshold;
 	bool enable_otg_frame_sync_pwa;
 	unsigned int min_deep_sleep_dcfclk_khz;
+	unsigned int force_odm2to1_for_edp_pixclk_mhz;
+	bool enable_replay_esd_recovery;
 };
 
 
@@ -1290,7 +1293,7 @@ void dc_hardware_init(struct dc *dc);
 int dc_get_vmid_use_vector(struct dc *dc);
 void dc_setup_vm_context(struct dc *dc, struct dc_virtual_addr_space_config *va_config, int vmid);
 /* Returns the number of vmids supported */
-int dc_setup_system_context(struct dc *dc, struct dc_phy_addr_space_config *pa_config);
+unsigned int dc_setup_system_context(struct dc *dc, struct dc_phy_addr_space_config *pa_config);
 void dc_init_callbacks(struct dc *dc,
 		const struct dc_callback_init *init_params);
 void dc_deinit_callbacks(struct dc *dc);
@@ -1407,50 +1410,15 @@ struct lut_mem_mapping {
 struct dc_rmcm_3dlut {
 	bool isInUse;
 	const struct dc_stream_state *stream;
+	uint8_t protection_bits;
 };
 
 struct dc_3dlut {
 	struct kref refcount;
 	struct tetrahedral_params lut_3d;
+	struct fixed31_32 hdr_multiplier;
 	union dc_3dlut_state state;
 };
-
-/* 3DLUT DMA (Fast Load) params */
-struct dc_3dlut_dma {
-	struct dc_plane_address addr;
-	enum dc_cm_lut_swizzle swizzle;
-	enum dc_cm_lut_pixel_format format;
-	uint16_t bias; /* FP1.5.10 */
-	uint16_t scale; /* FP1.5.10 */
-	enum dc_cm_lut_size size;
-};
-
-/* color manager */
-union dc_plane_cm_flags {
-	unsigned int all;
-	struct {
-		unsigned int shaper_enable    : 1;
-		unsigned int lut3d_enable     : 1;
-		unsigned int blend_enable     : 1;
-		/* whether legacy (lut3d_func) or DMA is valid */
-		unsigned int lut3d_dma_enable : 1;
-		/* RMCM lut to be used instead of MCM */
-		unsigned int rmcm_enable	 : 1;
-		unsigned int reserved: 27;
-	} bits;
-};
-
-struct dc_plane_cm {
-	struct kref refcount;
-	struct dc_transfer_func shaper_func;
-	union {
-		struct dc_3dlut lut3d_func;
-		struct dc_3dlut_dma lut3d_dma;
-	};
-	struct dc_transfer_func blend_func;
-	union dc_plane_cm_flags flags;
-};
-
 /*
  * This structure is filled in by dc_surface_get_status and contains
  * the last requested address and the currently active address so the called
@@ -1485,6 +1453,7 @@ union surface_update_flags {
 		uint32_t pixel_format_change:1;
 		uint32_t plane_size_change:1;
 		uint32_t gamut_remap_change:1;
+		uint32_t cursor_csc_color_matrix_change:1;
 
 		/* Full updates */
 		uint32_t new_plane:1;
@@ -1528,18 +1497,14 @@ struct dc_plane_state {
 	struct fixed31_32 hdr_mult;
 	struct colorspace_transform gamut_remap_matrix;
 
+	// TODO: No longer used, remove
+	struct dc_hdr_static_metadata hdr_static_ctx;
+
 	enum dc_color_space color_space;
 
-	bool lut_bank_a;
-	struct dc_hdr_static_metadata hdr_static_ctx;
 	struct dc_3dlut lut3d_func;
 	struct dc_transfer_func in_shaper_func;
 	struct dc_transfer_func blend_tf;
-	enum dc_cm2_shaper_3dlut_setting mcm_shaper_3dlut_setting;
-	bool mcm_lut1d_enable;
-	struct dc_cm2_func_luts mcm_luts;
-	enum mpcc_movable_cm_location mcm_location;
-	struct dc_plane_cm cm;
 
 	struct dc_transfer_func *gamcor_tf;
 	enum surface_pixel_format format;
@@ -1554,7 +1519,7 @@ struct dc_plane_state {
 	bool visible;
 	bool flip_immediate;
 	bool horizontal_mirror;
-	int layer_index;
+	unsigned int layer_index;
 
 	union surface_update_flags update_flags;
 	bool flip_int_enabled;
@@ -1576,10 +1541,15 @@ struct dc_plane_state {
 
 	bool is_statically_allocated;
 	enum chroma_cositing cositing;
+	enum dc_cm2_shaper_3dlut_setting mcm_shaper_3dlut_setting;
+	bool mcm_lut1d_enable;
+	struct dc_cm2_func_luts mcm_luts;
+	bool lut_bank_a;
+	enum mpcc_movable_cm_location mcm_location;
 	struct dc_csc_transform cursor_csc_color_matrix;
 	bool adaptive_sharpness_en;
 	int adaptive_sharpness_policy;
-	int sharpness_level;
+	unsigned int sharpness_level;
 	enum linear_light_scaling linear_light_scaling;
 	unsigned int sdr_white_level_nits;
 	struct cm_hist_control cm_hist_control;
@@ -1602,7 +1572,7 @@ struct dc_plane_info {
 	bool global_alpha;
 	int  global_alpha_value;
 	bool input_csc_enabled;
-	int layer_index;
+	unsigned int layer_index;
 	enum chroma_cositing cositing;
 };
 
@@ -1891,9 +1861,21 @@ struct dc_fast_update {
 	struct dc_transfer_func *out_transfer_func;
 	struct dc_csc_transform *output_csc_transform;
 	const struct dc_csc_transform *cursor_csc_color_matrix;
-#if defined(CONFIG_DRM_AMD_DC_DCN4_2)
 	struct cm_hist_control *cm_hist_control;
-#endif
+	/* stream-level fast updates */
+	const struct colorspace_transform *gamut_remap;
+	const struct dc_cursor_attributes *cursor_attributes;
+	const struct dc_cursor_position *cursor_position;
+	const struct periodic_interrupt_config *periodic_interrupt;
+	const enum dc_dither_option *dither_option;
+	struct dc_info_packet *vrr_infopacket;
+	struct dc_info_packet *vsc_infopacket;
+	struct dc_info_packet *vsp_infopacket;
+	struct dc_info_packet *hfvsif_infopacket;
+	struct dc_info_packet *vtem_infopacket;
+	struct dc_info_packet *adaptive_sync_infopacket;
+	struct dc_info_packet *avi_infopacket;
+	struct dc_info_packet *hdr_static_metadata;
 };
 
 struct dc_surface_update {
@@ -1968,10 +1950,6 @@ struct dc_3dlut *dc_create_3dlut_func(void);
 void dc_3dlut_func_release(struct dc_3dlut *lut);
 void dc_3dlut_func_retain(struct dc_3dlut *lut);
 
-struct dc_plane_cm *dc_plane_cm_create(void);
-void dc_plane_cm_release(struct dc_plane_cm *cm);
-void dc_plane_cm_retain(struct dc_plane_cm *cm);
-
 void dc_post_update_surfaces_to_stream(
 		struct dc *dc);
 
@@ -2012,7 +1990,7 @@ enum dc_status dc_validate_plane(struct dc *dc, const struct dc_plane_state *pla
 
 enum dc_status dc_validate_with_context(struct dc *dc,
 					const struct dc_validation_set set[],
-					int set_count,
+					unsigned int set_count,
 					struct dc_state *context,
 					enum dc_validate_mode validate_mode);
 
@@ -2124,7 +2102,7 @@ struct dc_sink_init_data;
 struct dc_sink *dc_link_add_remote_sink(
 		struct dc_link *dc_link,
 		const uint8_t *edid,
-		int len,
+		unsigned int len,
 		struct dc_sink_init_data *init_data);
 
 /* Remove remote sink from a link with dc_connection_mst_branch connection type.
@@ -2921,9 +2899,9 @@ bool dc_can_clear_cursor_limit(const struct dc *dc);
  * including OTG underflow status, current read positions, frame count, and per-HUBP debug data.
  * The results are stored in the provided out_data structure for further analysis or logging.
  */
-void dc_get_underflow_debug_data_for_otg(struct dc *dc, int primary_otg_inst, struct dc_underflow_debug_data *out_data);
+void dc_get_underflow_debug_data_for_otg(struct dc *dc, unsigned int primary_otg_inst, struct dc_underflow_debug_data *out_data);
 
-void dc_get_power_feature_status(struct dc *dc, int primary_otg_inst, struct power_features *out_data);
+void dc_get_power_feature_status(struct dc *dc, unsigned int primary_otg_inst, struct power_features *out_data);
 
 /*
  * Software state variables used to program register fields across the display pipeline
@@ -3436,4 +3414,17 @@ bool dc_capture_register_software_state(struct dc *dc, struct dc_register_softwa
  */
 bool dc_get_qos_info(struct dc *dc, struct dc_qos_info *info);
 
+/**
+ * dc_override_memory_bandwidth_request - Override the DCN nominal memory
+ *     bandwidth request sent to PMFW, independent of the current display mode.
+ *     For debug use only.
+ * @dc: DC instance
+ * @bw_mbps: requested bandwidth in MB/s; 0 clears the override
+ *
+ * Return: capped bandwidth value actually applied (MB/s)
+ */
+unsigned int dc_override_memory_bandwidth_request(
+		struct dc *dc,
+		unsigned int bw_mbps);
+
 #endif /* DC_INTERFACE_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
index 79c8b4cab053b..0ee5c0c5545ca 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c
@@ -151,7 +151,7 @@ static bool dc_dmub_srv_reg_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_s
 	struct dc_context *dc_ctx;
 	struct dmub_srv *dmub;
 	enum dmub_status status = DMUB_STATUS_OK;
-	int i;
+	unsigned int i;
 
 	if (!dc_dmub_srv || !dc_dmub_srv->dmub)
 		return false;
@@ -193,7 +193,7 @@ static bool dc_dmub_srv_fb_cmd_list_queue_execute(struct dc_dmub_srv *dc_dmub_sr
 	struct dc_context *dc_ctx;
 	struct dmub_srv *dmub;
 	enum dmub_status status;
-	int i;
+	unsigned int i;
 
 	if (!dc_dmub_srv || !dc_dmub_srv->dmub)
 		return false;
@@ -336,7 +336,7 @@ bool dc_dmub_srv_optimized_init_done(struct dc_dmub_srv *dc_dmub_srv)
 		return false;
 	}
 
-	return boot_status.bits.optimized_init_done;
+	return (bool)boot_status.bits.optimized_init_done;
 }
 
 bool dc_dmub_srv_notify_stream_mask(struct dc_dmub_srv *dc_dmub_srv,
@@ -346,7 +346,7 @@ bool dc_dmub_srv_notify_stream_mask(struct dc_dmub_srv *dc_dmub_srv,
 		return false;
 
 	return dc_wake_and_execute_gpint(dc_dmub_srv->ctx, DMUB_GPINT__IDLE_OPT_NOTIFY_STREAM_MASK,
-					 stream_mask, NULL, DM_DMUB_WAIT_TYPE_WAIT);
+				 (uint16_t)stream_mask, NULL, DM_DMUB_WAIT_TYPE_WAIT);
 }
 
 bool dc_dmub_srv_is_restore_required(struct dc_dmub_srv *dc_dmub_srv)
@@ -368,7 +368,7 @@ bool dc_dmub_srv_is_restore_required(struct dc_dmub_srv *dc_dmub_srv)
 		return false;
 	}
 
-	return boot_status.bits.restore_required;
+	return (bool)boot_status.bits.restore_required;
 }
 
 bool dc_dmub_srv_get_dmub_outbox0_msg(const struct dc *dc, struct dmcub_trace_buf_entry *entry)
@@ -415,7 +415,7 @@ void dc_dmub_srv_set_drr_manual_trigger_cmd(struct dc *dc, uint32_t tg_inst)
 static uint8_t dc_dmub_srv_get_pipes_for_stream(struct dc *dc, struct dc_stream_state *stream)
 {
 	uint8_t pipes = 0;
-	int i = 0;
+	uint8_t i = 0;
 
 	for (i = 0; i < MAX_PIPES; i++) {
 		struct pipe_ctx *pipe = &dc->current_state->res_ctx.pipe_ctx[i];
@@ -430,25 +430,26 @@ static void dc_dmub_srv_populate_fams_pipe_info(struct dc *dc, struct dc_state *
 		struct pipe_ctx *head_pipe,
 		struct dmub_cmd_fw_assisted_mclk_switch_pipe_data *fams_pipe_data)
 {
-	int j;
+	unsigned int j;
 	int pipe_idx = 0;
 
-	fams_pipe_data->pipe_index[pipe_idx++] = head_pipe->plane_res.hubp->inst;
+	fams_pipe_data->pipe_index[pipe_idx++] = (uint8_t)head_pipe->plane_res.hubp->inst;
 	for (j = 0; j < dc->res_pool->pipe_count; j++) {
 		struct pipe_ctx *split_pipe = &context->res_ctx.pipe_ctx[j];
 
 		if (split_pipe->stream == head_pipe->stream && (split_pipe->top_pipe || split_pipe->prev_odm_pipe)) {
-			fams_pipe_data->pipe_index[pipe_idx++] = split_pipe->plane_res.hubp->inst;
+			fams_pipe_data->pipe_index[pipe_idx++] = (uint8_t)split_pipe->plane_res.hubp->inst;
 		}
 	}
-	fams_pipe_data->pipe_count = pipe_idx;
+	fams_pipe_data->pipe_count = (uint8_t)pipe_idx;
 }
 
 bool dc_dmub_srv_p_state_delegate(struct dc *dc, bool should_manage_pstate, struct dc_state *context)
 {
 	union dmub_rb_cmd cmd = { 0 };
 	struct dmub_cmd_fw_assisted_mclk_switch_config *config_data = &cmd.fw_assisted_mclk_switch.config_data;
-	int i = 0, k = 0;
+	unsigned int i = 0;
+	int k = 0;
 	int ramp_up_num_steps = 1; // TODO: Ramp is currently disabled. Reenable it.
 	uint8_t visual_confirm_enabled;
 	struct dc_stream_status *stream_status = NULL;
@@ -456,7 +457,7 @@ bool dc_dmub_srv_p_state_delegate(struct dc *dc, bool should_manage_pstate, stru
 	if (dc == NULL)
 		return false;
 
-	visual_confirm_enabled = dc->debug.visual_confirm == VISUAL_CONFIRM_FAMS;
+	visual_confirm_enabled = (uint8_t)(dc->debug.visual_confirm == VISUAL_CONFIRM_FAMS);
 
 	// Format command.
 	cmd.fw_assisted_mclk_switch.header.type = DMUB_CMD__FW_ASSISTED_MCLK_SWITCH;
@@ -477,7 +478,8 @@ bool dc_dmub_srv_p_state_delegate(struct dc *dc, bool should_manage_pstate, stru
 			 */
 			stream_status = dc_state_get_stream_status(context, pipe->stream);
 			if (stream_status && !stream_status->fpo_in_use) {
-				cmd.fw_assisted_mclk_switch.config_data.vactive_stretch_margin_us = dc->debug.fpo_vactive_margin_us;
+				cmd.fw_assisted_mclk_switch.config_data.vactive_stretch_margin_us =
+					(uint16_t)dc->debug.fpo_vactive_margin_us;
 				break;
 			}
 		}
@@ -486,17 +488,18 @@ bool dc_dmub_srv_p_state_delegate(struct dc *dc, bool should_manage_pstate, stru
 	for (i = 0, k = 0; context && i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 
-		if (!resource_is_pipe_type(pipe, OTG_MASTER))
+		if (!resource_is_pipe_type(pipe, OTG_MASTER) || !pipe->stream)
 			continue;
 
 		stream_status = dc_state_get_stream_status(context, pipe->stream);
 		if (stream_status && stream_status->fpo_in_use) {
-			struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
-			uint8_t min_refresh_in_hz = (pipe->stream->timing.min_refresh_in_uhz + 999999) / 1000000;
+			uint8_t min_refresh_in_hz;
+
+			min_refresh_in_hz = (uint8_t)((pipe->stream->timing.min_refresh_in_uhz + 999999) / 1000000);
 
 			config_data->pipe_data[k].pix_clk_100hz = pipe->stream->timing.pix_clk_100hz;
 			config_data->pipe_data[k].min_refresh_in_hz = min_refresh_in_hz;
-			config_data->pipe_data[k].max_ramp_step = ramp_up_num_steps;
+			config_data->pipe_data[k].max_ramp_step = (uint8_t)ramp_up_num_steps;
 			config_data->pipe_data[k].pipes = dc_dmub_srv_get_pipes_for_stream(dc, pipe->stream);
 			dc_dmub_srv_populate_fams_pipe_info(dc, context, pipe, &config_data->pipe_data[k]);
 			k++;
@@ -551,7 +554,7 @@ void dc_dmub_srv_get_visual_confirm_color_cmd(struct dc *dc, struct pipe_ctx *pi
 	cmd.visual_confirm_color.header.sub_type = 0;
 	cmd.visual_confirm_color.header.ret_status = 1;
 	cmd.visual_confirm_color.header.payload_bytes = sizeof(struct dmub_cmd_visual_confirm_color_data);
-	cmd.visual_confirm_color.visual_confirm_color_data.visual_confirm_color.panel_inst = panel_inst;
+	cmd.visual_confirm_color.visual_confirm_color_data.visual_confirm_color.panel_inst = (uint16_t)panel_inst;
 
 	// If command was processed, copy feature caps to dmub srv
 	if (dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY) &&
@@ -589,17 +592,17 @@ static void populate_subvp_cmd_drr_info(struct dc *dc,
 	struct dc_crtc_timing *main_timing = &subvp_pipe->stream->timing;
 	struct dc_crtc_timing *phantom_timing;
 	struct dc_crtc_timing *drr_timing = &vblank_pipe->stream->timing;
-	uint16_t drr_frame_us = 0;
-	uint16_t min_drr_supported_us = 0;
-	uint16_t max_drr_supported_us = 0;
-	uint16_t max_drr_vblank_us = 0;
-	uint16_t max_drr_mallregion_us = 0;
-	uint16_t mall_region_us = 0;
-	uint16_t prefetch_us = 0;
-	uint16_t subvp_active_us = 0;
-	uint16_t drr_active_us = 0;
-	uint16_t min_vtotal_supported = 0;
-	uint16_t max_vtotal_supported = 0;
+	uint64_t drr_frame_us = 0;
+	uint64_t min_drr_supported_us = 0;
+	uint64_t max_drr_supported_us = 0;
+	uint64_t max_drr_vblank_us = 0;
+	uint64_t max_drr_mallregion_us = 0;
+	uint64_t mall_region_us = 0;
+	uint64_t prefetch_us = 0;
+	uint64_t subvp_active_us = 0;
+	uint64_t drr_active_us = 0;
+	uint64_t min_vtotal_supported = 0;
+	uint64_t max_vtotal_supported = 0;
 
 	if (!phantom_stream)
 		return;
@@ -639,9 +642,10 @@ static void populate_subvp_cmd_drr_info(struct dc *dc,
 	 */
 	max_vtotal_supported = max_vtotal_supported - dc->caps.subvp_drr_max_vblank_margin_us;
 
-	pipe_data->pipe_config.vblank_data.drr_info.min_vtotal_supported = min_vtotal_supported;
-	pipe_data->pipe_config.vblank_data.drr_info.max_vtotal_supported = max_vtotal_supported;
-	pipe_data->pipe_config.vblank_data.drr_info.drr_vblank_start_margin = dc->caps.subvp_drr_vblank_start_margin_us;
+	pipe_data->pipe_config.vblank_data.drr_info.min_vtotal_supported = (uint16_t)min_vtotal_supported;
+	pipe_data->pipe_config.vblank_data.drr_info.max_vtotal_supported = (uint16_t)max_vtotal_supported;
+	pipe_data->pipe_config.vblank_data.drr_info.drr_vblank_start_margin =
+		(uint16_t)dc->caps.subvp_drr_vblank_start_margin_us;
 }
 
 /**
@@ -686,14 +690,15 @@ static void populate_subvp_cmd_vblank_pipe_info(struct dc *dc,
 
 	pipe_data->mode = VBLANK;
 	pipe_data->pipe_config.vblank_data.pix_clk_100hz = vblank_pipe->stream->timing.pix_clk_100hz;
-	pipe_data->pipe_config.vblank_data.vblank_start = vblank_pipe->stream->timing.v_total -
-							vblank_pipe->stream->timing.v_front_porch;
-	pipe_data->pipe_config.vblank_data.vtotal = vblank_pipe->stream->timing.v_total;
-	pipe_data->pipe_config.vblank_data.htotal = vblank_pipe->stream->timing.h_total;
+	pipe_data->pipe_config.vblank_data.vblank_start = (uint16_t)(vblank_pipe->stream->timing.v_total -
+							vblank_pipe->stream->timing.v_front_porch);
+	pipe_data->pipe_config.vblank_data.vtotal = (uint16_t)vblank_pipe->stream->timing.v_total;
+	pipe_data->pipe_config.vblank_data.htotal = (uint16_t)vblank_pipe->stream->timing.h_total;
 	pipe_data->pipe_config.vblank_data.vblank_pipe_index = vblank_pipe->pipe_idx;
-	pipe_data->pipe_config.vblank_data.vstartup_start = vblank_pipe->pipe_dlg_param.vstartup_start;
+	pipe_data->pipe_config.vblank_data.vstartup_start = (uint16_t)vblank_pipe->pipe_dlg_param.vstartup_start;
 	pipe_data->pipe_config.vblank_data.vblank_end =
-			vblank_pipe->stream->timing.v_total - vblank_pipe->stream->timing.v_front_porch - vblank_pipe->stream->timing.v_addressable;
+			(uint16_t)(vblank_pipe->stream->timing.v_total -
+			vblank_pipe->stream->timing.v_front_porch - vblank_pipe->stream->timing.v_addressable);
 
 	if (vblank_pipe->stream->ignore_msa_timing_param &&
 		(vblank_pipe->stream->allow_freesync || vblank_pipe->stream->vrr_active_variable || vblank_pipe->stream->vrr_active_fixed))
@@ -739,10 +744,10 @@ static void update_subvp_prefetch_end_to_mall_start(struct dc *dc,
 	phantom_timing0 = &phantom_stream0->timing;
 	phantom_timing1 = &phantom_stream1->timing;
 
-	subvp0_prefetch_us = div64_u64(((uint64_t)(phantom_timing0->v_total - phantom_timing0->v_front_porch) *
+	subvp0_prefetch_us = (uint32_t)div64_u64(((uint64_t)(phantom_timing0->v_total - phantom_timing0->v_front_porch) *
 			(uint64_t)phantom_timing0->h_total * 1000000),
 			(((uint64_t)phantom_timing0->pix_clk_100hz * 100) + dc->caps.subvp_prefetch_end_to_mall_start_us));
-	subvp1_prefetch_us = div64_u64(((uint64_t)(phantom_timing1->v_total - phantom_timing1->v_front_porch) *
+	subvp1_prefetch_us = (uint32_t)div64_u64(((uint64_t)(phantom_timing1->v_total - phantom_timing1->v_front_porch) *
 			(uint64_t)phantom_timing1->h_total * 1000000),
 			(((uint64_t)phantom_timing1->pix_clk_100hz * 100) + dc->caps.subvp_prefetch_end_to_mall_start_us));
 
@@ -751,8 +756,8 @@ static void update_subvp_prefetch_end_to_mall_start(struct dc *dc,
 	if (subvp0_prefetch_us > subvp1_prefetch_us) {
 		pipe_data = &cmd->fw_assisted_mclk_switch_v2.config_data.pipe_data[1];
 		prefetch_delta_us = subvp0_prefetch_us - subvp1_prefetch_us;
-		pipe_data->pipe_config.subvp_data.prefetch_to_mall_start_lines =
-				div64_u64(((uint64_t)(dc->caps.subvp_prefetch_end_to_mall_start_us + prefetch_delta_us) *
+pipe_data->pipe_config.subvp_data.prefetch_to_mall_start_lines =
+				(uint16_t)div64_u64(((uint64_t)(dc->caps.subvp_prefetch_end_to_mall_start_us + prefetch_delta_us) *
 					((uint64_t)phantom_timing1->pix_clk_100hz * 100) + ((uint64_t)phantom_timing1->h_total * 1000000 - 1)),
 					((uint64_t)phantom_timing1->h_total * 1000000));
 
@@ -760,7 +765,7 @@ static void update_subvp_prefetch_end_to_mall_start(struct dc *dc,
 		pipe_data = &cmd->fw_assisted_mclk_switch_v2.config_data.pipe_data[0];
 		prefetch_delta_us = subvp1_prefetch_us - subvp0_prefetch_us;
 		pipe_data->pipe_config.subvp_data.prefetch_to_mall_start_lines =
-				div64_u64(((uint64_t)(dc->caps.subvp_prefetch_end_to_mall_start_us + prefetch_delta_us) *
+				(uint16_t)div64_u64(((uint64_t)(dc->caps.subvp_prefetch_end_to_mall_start_us + prefetch_delta_us) *
 					((uint64_t)phantom_timing0->pix_clk_100hz * 100) + ((uint64_t)phantom_timing0->h_total * 1000000 - 1)),
 					((uint64_t)phantom_timing0->h_total * 1000000));
 	}
@@ -800,14 +805,14 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc,
 
 	pipe_data->mode = SUBVP;
 	pipe_data->pipe_config.subvp_data.pix_clk_100hz = subvp_pipe->stream->timing.pix_clk_100hz;
-	pipe_data->pipe_config.subvp_data.htotal = subvp_pipe->stream->timing.h_total;
-	pipe_data->pipe_config.subvp_data.vtotal = subvp_pipe->stream->timing.v_total;
+	pipe_data->pipe_config.subvp_data.htotal = (uint16_t)subvp_pipe->stream->timing.h_total;
+	pipe_data->pipe_config.subvp_data.vtotal = (uint16_t)subvp_pipe->stream->timing.v_total;
 	pipe_data->pipe_config.subvp_data.main_vblank_start =
-			main_timing->v_total - main_timing->v_front_porch;
+			(uint16_t)(main_timing->v_total - main_timing->v_front_porch);
 	pipe_data->pipe_config.subvp_data.main_vblank_end =
-			main_timing->v_total - main_timing->v_front_porch - main_timing->v_addressable;
-	pipe_data->pipe_config.subvp_data.mall_region_lines = phantom_timing->v_addressable;
-	pipe_data->pipe_config.subvp_data.main_pipe_index = subvp_pipe->stream_res.tg->inst;
+			(uint16_t)(main_timing->v_total - main_timing->v_front_porch - main_timing->v_addressable);
+	pipe_data->pipe_config.subvp_data.mall_region_lines = (uint16_t)phantom_timing->v_addressable;
+	pipe_data->pipe_config.subvp_data.main_pipe_index = (uint8_t)subvp_pipe->stream_res.tg->inst;
 	pipe_data->pipe_config.subvp_data.is_drr = subvp_pipe->stream->ignore_msa_timing_param &&
 		(subvp_pipe->stream->allow_freesync || subvp_pipe->stream->vrr_active_variable || subvp_pipe->stream->vrr_active_fixed);
 
@@ -822,25 +827,25 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc,
 	reduce_fraction(subvp_pipe->plane_state->src_rect.height, subvp_pipe->plane_state->dst_rect.height,
 			&out_num_plane, &out_den_plane);
 	reduce_fraction(out_num_stream * out_num_plane, out_den_stream * out_den_plane, &out_num, &out_den);
-	pipe_data->pipe_config.subvp_data.scale_factor_numerator = out_num;
-	pipe_data->pipe_config.subvp_data.scale_factor_denominator = out_den;
+	pipe_data->pipe_config.subvp_data.scale_factor_numerator = (uint8_t)out_num;
+	pipe_data->pipe_config.subvp_data.scale_factor_denominator = (uint8_t)out_den;
 
 	// Prefetch lines is equal to VACTIVE + BP + VSYNC
 	pipe_data->pipe_config.subvp_data.prefetch_lines =
-			phantom_timing->v_total - phantom_timing->v_front_porch;
+			(uint16_t)(phantom_timing->v_total - phantom_timing->v_front_porch);
 
 	// Round up
 	pipe_data->pipe_config.subvp_data.prefetch_to_mall_start_lines =
-			div64_u64(((uint64_t)dc->caps.subvp_prefetch_end_to_mall_start_us * ((uint64_t)phantom_timing->pix_clk_100hz * 100) +
+			(uint16_t)div64_u64(((uint64_t)dc->caps.subvp_prefetch_end_to_mall_start_us * ((uint64_t)phantom_timing->pix_clk_100hz * 100) +
 					((uint64_t)phantom_timing->h_total * 1000000 - 1)), ((uint64_t)phantom_timing->h_total * 1000000));
 	pipe_data->pipe_config.subvp_data.processing_delay_lines =
-			div64_u64(((uint64_t)(dc->caps.subvp_fw_processing_delay_us) * ((uint64_t)phantom_timing->pix_clk_100hz * 100) +
+			(uint16_t)div64_u64(((uint64_t)(dc->caps.subvp_fw_processing_delay_us) * ((uint64_t)phantom_timing->pix_clk_100hz * 100) +
 					((uint64_t)phantom_timing->h_total * 1000000 - 1)), ((uint64_t)phantom_timing->h_total * 1000000));
 
 	if (subvp_pipe->bottom_pipe) {
-		pipe_data->pipe_config.subvp_data.main_split_pipe_index = subvp_pipe->bottom_pipe->pipe_idx;
+		pipe_data->pipe_config.subvp_data.main_split_pipe_index = (uint8_t)subvp_pipe->bottom_pipe->pipe_idx;
 	} else if (subvp_pipe->next_odm_pipe) {
-		pipe_data->pipe_config.subvp_data.main_split_pipe_index = subvp_pipe->next_odm_pipe->pipe_idx;
+		pipe_data->pipe_config.subvp_data.main_split_pipe_index = (uint8_t)subvp_pipe->next_odm_pipe->pipe_idx;
 	} else {
 		pipe_data->pipe_config.subvp_data.main_split_pipe_index = 0xF;
 	}
@@ -851,11 +856,11 @@ static void populate_subvp_cmd_pipe_info(struct dc *dc,
 
 		if (resource_is_pipe_type(phantom_pipe, OTG_MASTER) &&
 				phantom_pipe->stream == dc_state_get_paired_subvp_stream(context, subvp_pipe->stream)) {
-			pipe_data->pipe_config.subvp_data.phantom_pipe_index = phantom_pipe->stream_res.tg->inst;
+			pipe_data->pipe_config.subvp_data.phantom_pipe_index = (uint8_t)phantom_pipe->stream_res.tg->inst;
 			if (phantom_pipe->bottom_pipe) {
-				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->bottom_pipe->plane_res.hubp->inst;
+				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = (uint8_t)phantom_pipe->bottom_pipe->plane_res.hubp->inst;
 			} else if (phantom_pipe->next_odm_pipe) {
-				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = phantom_pipe->next_odm_pipe->plane_res.hubp->inst;
+				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = (uint8_t)phantom_pipe->next_odm_pipe->plane_res.hubp->inst;
 			} else {
 				pipe_data->pipe_config.subvp_data.phantom_split_pipe_index = 0xF;
 			}
@@ -933,15 +938,15 @@ void dc_dmub_setup_subvp_dmub_command(struct dc *dc,
 		if (subvp_count == 2) {
 			update_subvp_prefetch_end_to_mall_start(dc, context, &cmd, subvp_pipes);
 		}
-		cmd.fw_assisted_mclk_switch_v2.config_data.pstate_allow_width_us = dc->caps.subvp_pstate_allow_width_us;
-		cmd.fw_assisted_mclk_switch_v2.config_data.vertical_int_margin_us = dc->caps.subvp_vertical_int_margin_us;
+		cmd.fw_assisted_mclk_switch_v2.config_data.pstate_allow_width_us = (uint8_t)dc->caps.subvp_pstate_allow_width_us;
+		cmd.fw_assisted_mclk_switch_v2.config_data.vertical_int_margin_us = (uint8_t)dc->caps.subvp_vertical_int_margin_us;
 
 		// Store the original watermark value for this SubVP config so we can lower it when the
 		// MCLK switch starts
 		wm_val_refclk = context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns *
 				(dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000) / 1000;
 
-		cmd.fw_assisted_mclk_switch_v2.config_data.watermark_a_cache = wm_val_refclk < 0xFFFF ? wm_val_refclk : 0xFFFF;
+		cmd.fw_assisted_mclk_switch_v2.config_data.watermark_a_cache = (uint16_t)(wm_val_refclk < 0xFFFF ? wm_val_refclk : 0xFFFF);
 	}
 
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
@@ -1060,10 +1065,10 @@ static void dc_build_cursor_update_payload0(
 	payload->cursor_rect.width  = hubp->cur_rect.w;
 	payload->cursor_rect.height = hubp->cur_rect.h;
 
-	payload->enable      = hubp->pos.cur_ctl.bits.cur_enable;
+	payload->enable      = (uint8_t)hubp->pos.cur_ctl.bits.cur_enable;
 	payload->pipe_idx    = p_idx;
-	payload->panel_inst  = panel_inst;
-	payload->otg_inst    = pipe_ctx->stream_res.tg->inst;
+	payload->panel_inst  = (uint8_t)panel_inst;
+	payload->otg_inst    = (uint8_t)pipe_ctx->stream_res.tg->inst;
 }
 
 static void dc_build_cursor_position_update_payload0(
@@ -1645,7 +1650,7 @@ bool dc_dmub_srv_should_detect(struct dc_dmub_srv *dc_dmub_srv)
 	if (dc_dmub_srv->dmub->shared_state &&
 	    dc_dmub_srv->dmub->meta_info.feature_bits.bits.shared_state_link_detection) {
 		ips_fw = &dc_dmub_srv->dmub->shared_state[DMUB_SHARED_SHARE_FEATURE__IPS_FW].data.ips_fw;
-		return ips_fw->signals.bits.detection_required;
+		return (bool)ips_fw->signals.bits.detection_required;
 	}
 
 	/* Detection may require reading scratch 0 - exit out of idle prior to the read. */
@@ -1807,7 +1812,7 @@ static void dc_dmub_srv_rb_based_fams2_update_config(struct dc *dc,
 		struct dc_state *context,
 		bool enable)
 {
-	uint8_t num_cmds = 1;
+	uint32_t num_cmds = 1;
 	uint32_t i;
 	union dmub_rb_cmd cmd[2 * MAX_STREAMS + 1];
 	struct dmub_rb_cmd_fams2 *global_cmd = &cmd[0].fams2_config;
@@ -1940,7 +1945,7 @@ void dc_dmub_srv_fams2_drr_update(struct dc *dc,
 
 	cmd.fams2_drr_update.header.type = DMUB_CMD__FW_ASSISTED_MCLK_SWITCH;
 	cmd.fams2_drr_update.header.sub_type = DMUB_CMD__FAMS2_DRR_UPDATE;
-	cmd.fams2_drr_update.dmub_optc_state_req.tg_inst = tg_inst;
+	cmd.fams2_drr_update.dmub_optc_state_req.tg_inst = (uint8_t)tg_inst;
 	cmd.fams2_drr_update.dmub_optc_state_req.v_total_max = vtotal_max;
 	cmd.fams2_drr_update.dmub_optc_state_req.v_total_min = vtotal_min;
 	cmd.fams2_drr_update.dmub_optc_state_req.v_total_mid = vtotal_mid;
@@ -1991,10 +1996,10 @@ void dc_dmub_srv_fams2_passthrough_flip(
 		cmds[num_cmds].fams2_flip.header.multi_cmd_pending = 1;
 
 		/* set topology info */
-		cmds[num_cmds].fams2_flip.flip_info.pipe_mask = dc_plane_get_pipe_mask(state, plane_state);
-		if (stream_status)
-			cmds[num_cmds].fams2_flip.flip_info.otg_inst = stream_status->primary_otg_inst;
-
+		cmds[num_cmds].fams2_flip.flip_info.pipe_mask = (uint8_t)dc_plane_get_pipe_mask(state, plane_state);
+		if (stream_status) {
+			cmds[num_cmds].fams2_flip.flip_info.otg_inst = (uint8_t)stream_status->primary_otg_inst;
+		}
 		cmds[num_cmds].fams2_flip.flip_info.config.bits.is_immediate = plane_state->flip_immediate;
 
 		/* build address info for command */
@@ -2249,6 +2254,8 @@ bool dmub_lsdma_send_tiled_to_tiled_copy_command(
 	lsdma_data->u.tiled_copy_data.data_format      = params.data_format;
 	lsdma_data->u.tiled_copy_data.max_com          = params.max_com;
 	lsdma_data->u.tiled_copy_data.max_uncom        = params.max_uncom;
+	lsdma_data->u.tiled_copy_data.cache_policy_src = params.src_cache_policy;
+	lsdma_data->u.tiled_copy_data.cache_policy_dst = params.dst_cache_policy;
 
 	result = dc_wake_and_execute_dmub_cmd(dc_ctx, &cmd, wait_type);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
index 6579cf9cce3c5..ebcaf49e5961e 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dmub_srv.h
@@ -240,11 +240,11 @@ struct lsdma_linear_sub_window_copy_params {
 	uint32_t src_slice_pitch;
 	uint32_t dst_slice_pitch;
 
-	uint32_t tmz              : 1;
+	uint32_t tmz              : 4;
 	uint32_t element_size     : 3;
 	uint32_t src_cache_policy : 3;
 	uint32_t dst_cache_policy : 3;
-	uint32_t padding          : 22;
+	uint32_t padding          : 19;
 };
 
 bool dmub_lsdma_send_linear_sub_window_copy_command(
@@ -286,12 +286,13 @@ struct lsdma_send_tiled_to_tiled_copy_command_params {
 	uint32_t swizzle_mode     : 5;
 	uint32_t element_size     : 3;
 	uint32_t dcc              : 1;
-	uint32_t tmz              : 1;
+	uint32_t tmz              : 4;
 	uint32_t read_compress    : 2;
 	uint32_t write_compress   : 2;
 	uint32_t max_com          : 2;
 	uint32_t max_uncom        : 1;
-	uint32_t padding          : 9;
+	uint32_t src_cache_policy : 3;
+	uint32_t dst_cache_policy : 3;
 };
 
 bool dmub_lsdma_send_tiled_to_tiled_copy_command(
diff --git a/drivers/gpu/drm/amd/display/dc/dc_dsc.h b/drivers/gpu/drm/amd/display/dc/dc_dsc.h
index 101bce6b8de64..7cb34e2b44a1b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_dsc.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_dsc.h
@@ -92,7 +92,7 @@ uint32_t dc_dsc_stream_bandwidth_in_kbps(const struct dc_crtc_timing *timing,
 
 uint32_t dc_dsc_stream_bandwidth_overhead_in_kbps(
 		const struct dc_crtc_timing *timing,
-		const int num_slices_h,
+		const uint32_t num_slices_h,
 		const bool is_dp);
 
 void dc_dsc_dump_decoder_caps(const struct display_stream_compressor *dsc,
diff --git a/drivers/gpu/drm/amd/display/dc/dc_fused_io.c b/drivers/gpu/drm/amd/display/dc/dc_fused_io.c
index fee69642fb93f..664cb4abf623b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_fused_io.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_fused_io.c
@@ -23,11 +23,11 @@ static bool op_i2c_convert(
 
 	req->type = type;
 	loc->is_aux = false;
-	loc->ddc_line = ddc_line;
+	loc->ddc_line = (uint8_t)ddc_line;
 	loc->over_aux = over_aux;
 	loc->address = op->address;
 	loc->offset = op->offset;
-	loc->length = op->size;
+	loc->length = (uint8_t)op->size;
 	memcpy(req->buffer, op->data, op->size);
 
 	return true;
@@ -84,7 +84,7 @@ static bool atomic_write_poll_read(
 			timeout_us += timeout_per_aux_transaction_us * (io->request.u.aux.length / 16);
 	}
 
-	if (!dm_helpers_execute_fused_io(link->ctx, link, commands, count, timeout_us))
+	if (!dm_helpers_execute_fused_io(link->ctx, link, commands, count, (uint32_t)timeout_us))
 		return false;
 
 	return commands[0].fused_io.request.status == FUSED_REQUEST_STATUS_SUCCESS;
diff --git a/drivers/gpu/drm/amd/display/dc/dc_helper.c b/drivers/gpu/drm/amd/display/dc/dc_helper.c
index 77299767096ff..9650cccf1b8d4 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dc_helper.c
@@ -122,7 +122,7 @@ static void set_reg_field_values(struct dc_reg_value_masks *field_value_mask,
 		field_value = va_arg(ap, uint32_t);
 
 		set_reg_field_value_masks(field_value_mask,
-				field_value, mask, shift);
+				field_value, mask, (uint8_t)shift);
 		i++;
 	}
 }
@@ -432,7 +432,7 @@ void generic_reg_wait(const struct dc_context *ctx,
 {
 	uint32_t field_value;
 	uint32_t reg_val;
-	int i;
+	unsigned int i;
 
 	if (ctx->dmub_srv &&
 	    ctx->dmub_srv->reg_helper_offload.gather_in_progress) {
@@ -447,7 +447,7 @@ void generic_reg_wait(const struct dc_context *ctx,
 	 * This value comes from experiments.
 	 *
 	 */
-	ASSERT(delay_between_poll_us * time_out_num_tries <= 3000000);
+	ASSERT(delay_between_poll_us * time_out_num_tries <= 3000000u);
 
 	for (i = 0; i <= time_out_num_tries; i++) {
 		if (i) {
@@ -459,7 +459,7 @@ void generic_reg_wait(const struct dc_context *ctx,
 
 		reg_val = dm_read_reg(ctx, addr);
 
-		field_value = get_reg_field_value_ex(reg_val, mask, shift);
+		field_value = get_reg_field_value_ex(reg_val, mask, (uint8_t)shift);
 
 		if (field_value == condition_value) {
 			if (i * delay_between_poll_us > 1000)
@@ -525,7 +525,7 @@ uint32_t generic_indirect_reg_get(const struct dc_context *ctx,
 		mask = va_arg(ap, uint32_t);
 		field_value = va_arg(ap, uint32_t *);
 
-		*field_value = get_reg_field_value_ex(value, mask, shift);
+		*field_value = get_reg_field_value_ex(value, mask, (uint8_t)shift);
 		i++;
 	}
 
@@ -554,7 +554,7 @@ uint32_t generic_indirect_reg_update_ex(const struct dc_context *ctx,
 		mask = va_arg(ap, uint32_t);
 		field_value = va_arg(ap, uint32_t);
 
-		reg_val = set_reg_field_value_ex(reg_val, field_value, mask, shift);
+		reg_val = set_reg_field_value_ex(reg_val, field_value, mask, (uint8_t)shift);
 		i++;
 	}
 
@@ -584,7 +584,7 @@ uint32_t generic_indirect_reg_update_ex_sync(const struct dc_context *ctx,
 		mask = va_arg(ap, uint32_t);
 		field_value = va_arg(ap, uint32_t);
 
-		reg_val = set_reg_field_value_ex(reg_val, field_value, mask, shift);
+		reg_val = set_reg_field_value_ex(reg_val, field_value, mask, (uint8_t)shift);
 		i++;
 	}
 
@@ -615,7 +615,7 @@ uint32_t generic_indirect_reg_get_sync(const struct dc_context *ctx,
 		mask = va_arg(ap, uint32_t);
 		field_value = va_arg(ap, uint32_t *);
 
-		*field_value = get_reg_field_value_ex(value, mask, shift);
+		*field_value = get_reg_field_value_ex(value, mask, (uint8_t)shift);
 		i++;
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
index c2ca08d26e371..fa64bf6e711ce 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_hw_types.h
@@ -346,7 +346,9 @@ enum swizzle_mode_addr3_values {
 	DC_ADDR3_SW_4KB_3D = 5,
 	DC_ADDR3_SW_64KB_3D = 6,
 	DC_ADDR3_SW_256KB_3D = 7,
-	DC_ADDR3_SW_MAX = 8,
+	DC_ADDR3_SW_64KB_2D_Z = 8,
+	DC_ADDR3_SW_256KB_2D_Z = 9,
+	DC_ADDR3_SW_MAX = 10,
 	DC_ADDR3_SW_UNKNOWN = DC_ADDR3_SW_MAX
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dc_stream.h b/drivers/gpu/drm/amd/display/dc/dc_stream.h
index 7c38fa6f8cb19..4154cd059562b 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_stream.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_stream.h
@@ -64,35 +64,6 @@ struct dc_stream_status {
 	bool fpo_in_use;
 };
 
-enum hubp_dmdata_mode {
-	DMDATA_SW_MODE,
-	DMDATA_HW_MODE
-};
-
-struct dc_dmdata_attributes {
-	/* Specifies whether dynamic meta data will be updated by software
-	 * or has to be fetched by hardware (DMA mode)
-	 */
-	enum hubp_dmdata_mode dmdata_mode;
-	/* Specifies if current dynamic meta data is to be used only for the current frame */
-	bool dmdata_repeat;
-	/* Specifies the size of Dynamic Metadata surface in byte.  Size of 0 means no Dynamic metadata is fetched */
-	uint32_t dmdata_size;
-	/* Specifies if a new dynamic meta data should be fetched for an upcoming frame */
-	bool dmdata_updated;
-	/* If hardware mode is used, the base address where DMDATA surface is located */
-	PHYSICAL_ADDRESS_LOC address;
-	/* Specifies whether QOS level will be provided by TTU or it will come from DMDATA_QOS_LEVEL */
-	bool dmdata_qos_mode;
-	/* If qos_mode = 1, this is the QOS value to be used: */
-	uint32_t dmdata_qos_level;
-	/* Specifies the value in unit of REFCLK cycles to be added to the
-	 * current time to produce the Amortized deadline for Dynamic Metadata chunk request
-	 */
-	uint32_t dmdata_dl_delta;
-	/* An unbounded array of uint32s, represents software dmdata to be loaded */
-	uint32_t *dmdata_sw_data;
-};
 
 struct dc_writeback_info {
 	bool wb_enabled;
@@ -146,6 +117,12 @@ union stream_update_flags {
 		uint32_t fams_changed : 1;
 		uint32_t scaler_sharpener : 1;
 		uint32_t sharpening_required : 1;
+		uint32_t cursor_attr : 1;
+		uint32_t cursor_pos : 1;
+		uint32_t periodic_interrupt : 1;
+		uint32_t info_frame : 1;
+		uint32_t dmdata : 1;
+		uint32_t dither : 1;
 	} bits;
 
 	uint32_t raw;
@@ -179,8 +156,8 @@ struct luminance_data {
 	int luminance_millinits[LUMINANCE_DATA_TABLE_SIZE];
 	int flicker_criteria_milli_nits_GAMING;
 	int flicker_criteria_milli_nits_STATIC;
-	int nominal_refresh_rate;
-	int dm_max_decrease_from_nominal;
+	unsigned int nominal_refresh_rate;
+	unsigned int dm_max_decrease_from_nominal;
 };
 
 enum dc_drr_trigger_mode {
@@ -324,6 +301,7 @@ struct dc_stream_state {
 	enum dc_drr_trigger_mode drr_trigger_mode;
 
 	struct dc_update_scratch_space *update_scratch;
+	bool firmware_controlled_hdr_info_packet;
 };
 
 #define ABM_LEVEL_IMMEDIATE_DISABLE 255
@@ -517,6 +495,9 @@ struct surface_update_descriptor dc_check_update_surfaces_for_stream(
 		int surface_count,
 		struct dc_stream_update *stream_update);
 
+struct dc_link *dc_stream_get_link(
+	const struct dc_stream_state *dc_stream);
+
 /**
  * Create a new default stream for the requested sink
  */
diff --git a/drivers/gpu/drm/amd/display/dc/dc_types.h b/drivers/gpu/drm/amd/display/dc/dc_types.h
index c08d5c005df6e..3ec49f4e277f2 100644
--- a/drivers/gpu/drm/amd/display/dc/dc_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dc_types.h
@@ -790,6 +790,36 @@ struct dc_clock_config {
 	uint32_t current_clock_khz;/*current clock in use*/
 };
 
+enum hubp_dmdata_mode {
+	DMDATA_SW_MODE,
+	DMDATA_HW_MODE
+};
+
+struct dc_dmdata_attributes {
+	/* Specifies whether dynamic meta data will be updated by software
+	 * or has to be fetched by hardware (DMA mode)
+	 */
+	enum hubp_dmdata_mode dmdata_mode;
+	/* Specifies if current dynamic meta data is to be used only for the current frame */
+	bool dmdata_repeat;
+	/* Specifies the size of Dynamic Metadata surface in byte.  Size of 0 means no Dynamic metadata is fetched */
+	uint32_t dmdata_size;
+	/* Specifies if a new dynamic meta data should be fetched for an upcoming frame */
+	bool dmdata_updated;
+	/* If hardware mode is used, the base address where DMDATA surface is located */
+	PHYSICAL_ADDRESS_LOC address;
+	/* Specifies whether QOS level will be provided by TTU or it will come from DMDATA_QOS_LEVEL */
+	bool dmdata_qos_mode;
+	/* If qos_mode = 1, this is the QOS value to be used: */
+	uint32_t dmdata_qos_level;
+	/* Specifies the value in unit of REFCLK cycles to be added to the
+	 * current time to produce the Amortized deadline for Dynamic Metadata chunk request
+	 */
+	uint32_t dmdata_dl_delta;
+	/* An unbounded array of uint32s, represents software dmdata to be loaded */
+	uint32_t *dmdata_sw_data;
+};
+
 struct hw_asic_id {
 	uint32_t chip_id;
 	uint32_t chip_family;
@@ -887,7 +917,7 @@ struct dsc_dec_dpcd_caps {
 	union dsc_slice_caps2 slice_caps2;
 	int32_t lb_bit_depth;
 	bool is_block_pred_supported;
-	int32_t edp_max_bits_per_pixel; /* Valid only in eDP */
+	uint32_t edp_max_bits_per_pixel; /* Valid only in eDP */
 	union dsc_color_formats color_formats;
 	union dsc_color_depth color_depth;
 	int32_t throughput_mode_0_mps; /* In MPs */
@@ -1493,28 +1523,4 @@ struct dc_validation_dpia_set {
 	uint32_t required_bw;
 };
 
-enum dc_cm_lut_swizzle {
-	CM_LUT_3D_SWIZZLE_LINEAR_RGB,
-	CM_LUT_3D_SWIZZLE_LINEAR_BGR,
-	CM_LUT_1D_PACKED_LINEAR
-};
-
-enum dc_cm_lut_pixel_format {
-	CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB,
-	CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB,
-	CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB,
-	CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB,
-	CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10,
-	CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10
-};
-
-enum dc_cm_lut_size {
-	CM_LUT_SIZE_NONE,
-	CM_LUT_SIZE_999,
-	CM_LUT_SIZE_171717,
-	CM_LUT_SIZE_333333,
-	CM_LUT_SIZE_454545,
-	CM_LUT_SIZE_656565,
-};
-
 #endif /* DC_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c
index a72bf413fad6b..21dcb812fbf71 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn31/dcn31_dccg.c
@@ -576,7 +576,7 @@ void dccg31_set_dtbclk_dto(
 
 		// phase / modulo = dtbclk / dtbclk ref
 		modulo = params->ref_dtbclk_khz * 1000;
-		phase = div_u64((((unsigned long long)modulo * req_dtbclk_khz) + params->ref_dtbclk_khz - 1),
+		phase = (uint32_t)div_u64((((unsigned long long)modulo * req_dtbclk_khz) + params->ref_dtbclk_khz - 1),
 				params->ref_dtbclk_khz);
 
 		REG_UPDATE(OTG_PIXEL_RATE_CNTL[params->otg_inst],
@@ -620,7 +620,7 @@ void dccg31_set_audio_dtbclk_dto(
 
 		// phase / modulo = dtbclk / dtbclk ref
 		modulo = params->ref_dtbclk_khz * 1000;
-		phase = div_u64((((unsigned long long)modulo * params->req_audio_dtbclk_khz) + params->ref_dtbclk_khz - 1),
+		phase = (uint32_t)div_u64((((unsigned long long)modulo * params->req_audio_dtbclk_khz) + params->ref_dtbclk_khz - 1),
 			params->ref_dtbclk_khz);
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c
index 97605a416031b..417ed9d0e6bbb 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn401/dcn401_dccg.c
@@ -610,11 +610,13 @@ void dccg401_set_dp_dto(
 		 * int = target_pix_rate / reference_clock
 		 * phase = target_pix_rate - int * reference_clock,
 		 * modulo = reference_clock */
-		dto_integer = div_u64(params->pixclk_hz, dto_modulo_hz);
+
+		/* dto_modulo_hz = refclk (~100 MHz), well within uint32_t range */
+		dto_integer = div_u64(params->pixclk_hz, (uint32_t)dto_modulo_hz);
 		dto_phase_hz = params->pixclk_hz - dto_integer * dto_modulo_hz;
 
-		if (dto_phase_hz <= 0 && dto_integer <= 0) {
-			/* negative pixel rate should never happen */
+		if (dto_phase_hz == 0 && dto_integer == 0) {
+			/* zero pixel rate should never happen */
 			BREAK_TO_DEBUGGER();
 			return;
 		}
@@ -656,25 +658,25 @@ void dccg401_set_dp_dto(
 
 		dccg401_set_dtbclk_p_src(dccg, params->clk_src, params->otg_inst);
 
-		REG_WRITE(DP_DTO_PHASE[params->otg_inst], dto_phase_hz);
-		REG_WRITE(DP_DTO_MODULO[params->otg_inst], dto_modulo_hz);
+		REG_WRITE(DP_DTO_PHASE[params->otg_inst], (uint32_t)dto_phase_hz);
+		REG_WRITE(DP_DTO_MODULO[params->otg_inst], (uint32_t)dto_modulo_hz);
 
 		switch (params->otg_inst) {
 		case 0:
 			REG_UPDATE(OTG_PIXEL_RATE_DIV,
-					DPDTO0_INT, dto_integer);
+					DPDTO0_INT, (uint32_t)dto_integer);
 			break;
 		case 1:
 			REG_UPDATE(OTG_PIXEL_RATE_DIV,
-					DPDTO1_INT, dto_integer);
+					DPDTO1_INT, (uint32_t)dto_integer);
 			break;
 		case 2:
 			REG_UPDATE(OTG_PIXEL_RATE_DIV,
-					DPDTO2_INT, dto_integer);
+					DPDTO2_INT, (uint32_t)dto_integer);
 			break;
 		case 3:
 			REG_UPDATE(OTG_PIXEL_RATE_DIV,
-					DPDTO3_INT, dto_integer);
+					DPDTO3_INT, (uint32_t)dto_integer);
 			break;
 		default:
 			BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
index b813310763e51..9612f4498ef6c 100644
--- a/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
+++ b/drivers/gpu/drm/amd/display/dc/dccg/dcn42/dcn42_dccg.c
@@ -6,6 +6,7 @@
 #include "core_types.h"
 #include "dcn35/dcn35_dccg.h"
 #include "dcn42_dccg.h"
+#include "dcn20/dcn20_dccg.h"
 
 #define TO_DCN_DCCG(dccg)\
 	container_of(dccg, struct dcn_dccg, base)
@@ -306,6 +307,7 @@ static const struct dccg_funcs dccg42_funcs = {
 	.dccg_root_gate_disable_control = dccg35_root_gate_disable_control,
 	.dccg_read_reg_state = dccg31_read_reg_state,
 	.dccg_enable_global_fgcg = dccg42_enable_global_fgcg,
+	.allow_clock_gating = dccg2_allow_clock_gating
 };
 
 struct dccg *dccg42_create(
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
index 673bb87d2c172..181944ce77aba 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.c
@@ -321,7 +321,7 @@ static int read_channel_reply(struct dce_aux *engine, uint32_t size,
 			uint32_t aux_sw_data_val;
 
 			REG_GET(AUX_SW_DATA, AUX_SW_DATA, &aux_sw_data_val);
-			buffer[i] = aux_sw_data_val;
+			buffer[i] = (uint8_t)aux_sw_data_val;
 			++i;
 		}
 
@@ -375,7 +375,7 @@ static enum aux_return_code_type get_channel_status(
 			(value & AUX_SW_STATUS__AUX_SW_RX_RECV_INVALID_L_MASK))
 			return AUX_RET_ERROR_INVALID_REPLY;
 
-		*returned_bytes = get_reg_field_value(value,
+		*returned_bytes = (uint8_t)get_reg_field_value(value,
 				AUX_SW_STATUS,
 				AUX_SW_REPLY_BYTE_COUNT);
 
@@ -559,6 +559,18 @@ static enum i2caux_transaction_action i2caux_action_from_payload(struct aux_payl
 int dce_aux_transfer_raw(struct ddc_service *ddc,
 		struct aux_payload *payload,
 		enum aux_return_code_type *operation_result)
+{
+	if (ddc->ctx->dc->debug.enable_dmub_aux_for_legacy_ddc ||
+	    !ddc->ddc_pin) {
+		return dce_aux_transfer_dmub_raw(ddc, payload, operation_result);
+	} else {
+		return dce_aux_transfer_raw_with_ddc_pin(ddc, payload, operation_result);
+	}
+}
+
+int dce_aux_transfer_raw_with_ddc_pin(struct ddc_service *ddc,
+		struct aux_payload *payload,
+		enum aux_return_code_type *operation_result)
 {
 	struct ddc *ddc_pin = ddc->ddc_pin;
 	struct dce_aux *aux_engine;
@@ -740,13 +752,7 @@ bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
 		if (payload->write)
 			dce_aux_log_payload("  write", payload->data, payload->length, 16);
 
-		/* Check whether aux to be processed via dmub or dcn directly */
-		if (ddc->ctx->dc->debug.enable_dmub_aux_for_legacy_ddc
-			|| ddc->ddc_pin == NULL) {
-			ret = dce_aux_transfer_dmub_raw(ddc, payload, &operation_result);
-		} else {
-			ret = dce_aux_transfer_raw(ddc, payload, &operation_result);
-		}
+		ret = dce_aux_transfer_raw(ddc, payload, &operation_result);
 
 		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION,
 					LOG_FLAG_I2cAux_DceAux,
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
index c850ed49281f3..8f47456940166 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_aux.h
@@ -304,6 +304,10 @@ int dce_aux_transfer_raw(struct ddc_service *ddc,
 		struct aux_payload *cmd,
 		enum aux_return_code_type *operation_result);
 
+int dce_aux_transfer_raw_with_ddc_pin(struct ddc_service *ddc,
+		struct aux_payload *cmd,
+		enum aux_return_code_type *operation_result);
+
 int dce_aux_transfer_dmub_raw(struct ddc_service *ddc,
 		struct aux_payload *payload,
 		enum aux_return_code_type *operation_result);
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
index 25c13822fedef..ac9eed58c4157 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_clock_source.c
@@ -162,11 +162,9 @@ static bool calculate_fb_and_fractional_fb_divider(
 	feedback_divider *= (uint64_t)
 			(calc_pll_cs->fract_fb_divider_precision_factor);
 
-	*feedback_divider_param =
-		div_u64_rem(
-			feedback_divider,
-			calc_pll_cs->fract_fb_divider_factor,
-			fract_feedback_divider_param);
+	*feedback_divider_param = (uint32_t)div_u64_rem(
+		feedback_divider, calc_pll_cs->fract_fb_divider_factor,
+		fract_feedback_divider_param);
 
 	if (*feedback_divider_param != 0)
 		return true;
@@ -240,7 +238,7 @@ static bool calc_fb_divider_checking_tolerance(
 		pll_settings->calculated_pix_clk_100hz =
 			actual_calculated_clock_100hz;
 		pll_settings->vco_freq =
-			div_u64((u64)actual_calculated_clock_100hz * post_divider, 10);
+			(uint32_t)div_u64((u64)actual_calculated_clock_100hz * post_divider, 10);
 		return true;
 	}
 	return false;
@@ -440,8 +438,7 @@ static bool pll_adjust_pix_clk(
 	bp_adjust_pixel_clock_params.
 		encoder_object_id = pix_clk_params->encoder_object_id;
 	bp_adjust_pixel_clock_params.signal_type = pix_clk_params->signal_type;
-	bp_adjust_pixel_clock_params.
-		ss_enable = pix_clk_params->flags.ENABLE_SS;
+	bp_adjust_pixel_clock_params.ss_enable = pix_clk_params->flags.ENABLE_SS != 0;
 	bp_result = clk_src->bios->funcs->adjust_pixel_clock(
 			clk_src->bios, &bp_adjust_pixel_clock_params);
 	if (bp_result == BP_RESULT_OK) {
@@ -958,7 +955,7 @@ static bool dce112_program_pix_clk(
 		dce112_program_pixel_clk_resync(clk_src,
 					pix_clk_params->signal_type,
 					pix_clk_params->color_depth,
-					pix_clk_params->flags.SUPPORT_YCBCR420);
+					pix_clk_params->flags.SUPPORT_YCBCR420 != 0);
 
 	return true;
 }
@@ -1059,7 +1056,7 @@ static bool dcn31_program_pix_clk(
 			dce112_program_pixel_clk_resync(clk_src,
 						pix_clk_params->signal_type,
 						pix_clk_params->color_depth,
-						pix_clk_params->flags.SUPPORT_YCBCR420);
+						pix_clk_params->flags.SUPPORT_YCBCR420 != 0);
 	}
 
 	return true;
@@ -1162,7 +1159,7 @@ static bool dcn401_program_pix_clk(
 			dce112_program_pixel_clk_resync(clk_src,
 						pix_clk_params->signal_type,
 						pix_clk_params->color_depth,
-						pix_clk_params->flags.SUPPORT_YCBCR420);
+						pix_clk_params->flags.SUPPORT_YCBCR420 != 0);
 	}
 
 	return true;
@@ -1211,9 +1208,8 @@ static bool get_pixel_clk_frequency_100hz(
 			 */
 			modulo_hz = REG_READ(MODULO[inst]);
 			if (modulo_hz)
-				*pixel_clk_khz = div_u64((uint64_t)clock_hz*
-					dp_dto_ref_khz*10,
-					modulo_hz);
+				*pixel_clk_khz = (unsigned int)div_u64((uint64_t)clock_hz *
+					dp_dto_ref_khz * 10, modulo_hz);
 			else
 				*pixel_clk_khz = 0;
 		} else {
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
index fe239a96121e2..05892ab4529f6 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_hw.c
@@ -120,7 +120,7 @@ static void process_channel_reply(
 		uint32_t i2c_data;
 
 		REG_GET(DC_I2C_DATA, DC_I2C_DATA, &i2c_data);
-		*buffer++ = i2c_data;
+		*buffer++ = (uint8_t)i2c_data;
 
 		--length;
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c
index 52e05b9185f1f..e64b3b6bff5c2 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_i2c_sw.c
@@ -75,7 +75,7 @@ static void release_engine_dce_sw(
 static bool wait_for_scl_high_sw(
 	struct dc_context *ctx,
 	struct ddc *ddc,
-	uint16_t clock_delay_div_4)
+	uint32_t clock_delay_div_4)
 {
 	(void)ctx;
 	uint32_t scl_retry = 0;
@@ -97,7 +97,7 @@ static bool wait_for_scl_high_sw(
 static bool write_byte_sw(
 	struct dc_context *ctx,
 	struct ddc *ddc_handle,
-	uint16_t clock_delay_div_4,
+	uint32_t clock_delay_div_4,
 	uint8_t byte)
 {
 	int32_t shift = 7;
@@ -154,7 +154,7 @@ static bool write_byte_sw(
 static bool read_byte_sw(
 	struct dc_context *ctx,
 	struct ddc *ddc_handle,
-	uint16_t clock_delay_div_4,
+	uint32_t clock_delay_div_4,
 	uint8_t *byte,
 	bool more)
 {
@@ -214,7 +214,7 @@ static bool read_byte_sw(
 static bool stop_sync_sw(
 	struct dc_context *ctx,
 	struct ddc *ddc_handle,
-	uint16_t clock_delay_div_4)
+	uint32_t clock_delay_div_4)
 {
 	uint32_t retry = 0;
 
@@ -251,7 +251,7 @@ static bool stop_sync_sw(
 static bool i2c_write_sw(
 	struct dc_context *ctx,
 	struct ddc *ddc_handle,
-	uint16_t clock_delay_div_4,
+	uint32_t clock_delay_div_4,
 	uint8_t address,
 	uint32_t length,
 	const uint8_t *data)
@@ -273,7 +273,7 @@ static bool i2c_write_sw(
 static bool i2c_read_sw(
 	struct dc_context *ctx,
 	struct ddc *ddc_handle,
-	uint16_t clock_delay_div_4,
+	uint32_t clock_delay_div_4,
 	uint8_t address,
 	uint32_t length,
 	uint8_t *data)
@@ -298,7 +298,7 @@ static bool i2c_read_sw(
 static bool start_sync_sw(
 	struct dc_context *ctx,
 	struct ddc *ddc_handle,
-	uint16_t clock_delay_div_4)
+	uint32_t clock_delay_div_4)
 {
 	uint32_t retry = 0;
 
@@ -399,7 +399,7 @@ static void dce_i2c_sw_engine_submit_channel_request(struct dce_i2c_sw *engine,
 						     struct i2c_request_transaction_data *req)
 {
 	struct ddc *ddc = engine->ddc;
-	uint16_t clock_delay_div_4 = engine->clock_delay >> 2;
+	uint32_t clock_delay_div_4 = engine->clock_delay >> 2;
 
 	/* send sync (start / repeated start) */
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
index ee55ec21d2706..72c0950808eca 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_ipp.c
@@ -171,7 +171,7 @@ static void dce_ipp_program_input_lut(
 	struct input_pixel_processor *ipp,
 	const struct dc_gamma *gamma)
 {
-	int i;
+	unsigned int i;
 	struct dce_ipp *ipp_dce = TO_DCE_IPP(ipp);
 
 	/* power on LUT memory */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
index e15fd1454d3b5..c6a20378121b3 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c
@@ -815,6 +815,7 @@ static bool dce110_link_encoder_validate_hdmi_output(
 {
 	enum dc_color_depth max_deep_color =
 			enc110->base.features.max_hdmi_deep_color;
+	int max_hdmi_pixel_clock_khz = (int)enc110->base.features.max_hdmi_pixel_clock;
 
 	if (max_deep_color < crtc_timing->display_color_depth)
 		return false;
@@ -825,7 +826,7 @@ static bool dce110_link_encoder_validate_hdmi_output(
 		return false;
 
 	if ((adjusted_pix_clk_khz == 0) ||
-		(adjusted_pix_clk_khz > enc110->base.features.max_hdmi_pixel_clock))
+		(adjusted_pix_clk_khz > max_hdmi_pixel_clock_khz))
 		return false;
 
 	/* DCE11 HW does not support 420 */
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c
index de31fb1b6819a..31a9181c6a2b4 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_panel_cntl.c
@@ -209,7 +209,7 @@ static void dce_driver_set_backlight(struct panel_cntl *panel_cntl,
 	if (pwm_period_bitcnt == 0)
 		bit_count = 16;
 	else
-		bit_count = pwm_period_bitcnt;
+		bit_count = (uint8_t)pwm_period_bitcnt;
 
 	/* e.g. maskedPwmPeriod = 0x24 when bitCount is 6 */
 	masked_pwm_period = masked_pwm_period & ((1 << bit_count) - 1);
@@ -224,7 +224,7 @@ static void dce_driver_set_backlight(struct panel_cntl *panel_cntl,
 	 * components shift by bitCount then mask 16 bits and add rounding bit
 	 * from MSB of fraction e.g. 0x86F7 = ((0x21BDC0 >> 6) & 0xFFF) + 0
 	 */
-	backlight_16bit = active_duty_cycle >> bit_count;
+	backlight_16bit = (uint32_t)(active_duty_cycle >> bit_count);
 	backlight_16bit &= 0xFFFF;
 	backlight_16bit += (active_duty_cycle >> (bit_count - 1)) & 0x1;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
index c1448ae473661..b3640682fa603 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dce_transform.c
@@ -802,7 +802,7 @@ static void program_bit_depth_reduction(
 
 	ASSERT(depth <= COLOR_DEPTH_121212); /* Invalid clamp bit depth */
 
-	spatial_dither_enable = bit_depth_params->flags.SPATIAL_DITHER_ENABLED;
+	spatial_dither_enable = bit_depth_params->flags.SPATIAL_DITHER_ENABLED != 0;
 	/* Default to 12 bit truncation without rounding */
 	trunc_round_depth = DCP_OUT_TRUNC_ROUND_DEPTH_12BIT;
 	trunc_mode = DCP_OUT_TRUNC_ROUND_MODE_TRUNCATE;
@@ -835,9 +835,9 @@ static void program_bit_depth_reduction(
 		   spatial_dither_enable,
 		   DCP_SPATIAL_DITHER_MODE_A_AA_A,
 		   DCP_SPATIAL_DITHER_DEPTH_30BPP,
-		   bit_depth_params->flags.FRAME_RANDOM,
-		   bit_depth_params->flags.RGB_RANDOM,
-		   bit_depth_params->flags.HIGHPASS_RANDOM);
+		   bit_depth_params->flags.FRAME_RANDOM != 0,
+		   bit_depth_params->flags.RGB_RANDOM != 0,
+		   bit_depth_params->flags.HIGHPASS_RANDOM != 0);
 }
 
 #if defined(CONFIG_DRM_AMD_DC_SI)
@@ -1173,13 +1173,13 @@ bool dce_transform_get_optimal_number_of_taps(
 {
 	struct dce_transform *xfm_dce = TO_DCE_TRANSFORM(xfm);
 	int pixel_width = scl_data->viewport.width;
-	int max_num_of_lines;
+	uint32_t max_num_of_lines;
 
 	if (xfm_dce->prescaler_on &&
 			(scl_data->viewport.width > scl_data->recout.width))
 		pixel_width = scl_data->recout.width;
 
-	max_num_of_lines = dce_transform_get_max_num_of_supported_lines(
+	max_num_of_lines = (uint32_t)dce_transform_get_max_num_of_supported_lines(
 		xfm_dce,
 		scl_data->lb_params.depth,
 		pixel_width);
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c
index 806b5709c9e7f..2f4d368bd3fd8 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_abm_lcd.c
@@ -58,7 +58,7 @@ static void dmub_abm_enable_fractional_pwm(struct dc_context *dc)
 	union dmub_rb_cmd cmd;
 	uint32_t fractional_pwm = (dc->dc->config.disable_fractional_pwm == false) ? 1 : 0;
 	uint32_t edp_id_count = dc->dc_edp_id_count;
-	int i;
+	unsigned int i;
 	uint8_t panel_mask = 0;
 
 	for (i = 0; i < edp_id_count; i++)
@@ -176,7 +176,7 @@ void dmub_abm_init_config(struct abm *abm,
 	cmd.abm_init_config.header.type = DMUB_CMD__ABM;
 	cmd.abm_init_config.header.sub_type = DMUB_CMD__ABM_INIT_CONFIG;
 	cmd.abm_init_config.abm_init_config_data.src.quad_part = dc->dmub_srv->dmub->scratch_mem_fb.gpu_addr;
-	cmd.abm_init_config.abm_init_config_data.bytes = bytes;
+	cmd.abm_init_config.abm_init_config_data.bytes = (uint16_t)bytes;
 	cmd.abm_init_config.abm_init_config_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
 	cmd.abm_init_config.abm_init_config_data.panel_mask = panel_mask;
 
@@ -237,7 +237,7 @@ bool dmub_abm_save_restore(
 	cmd.abm_save_restore.header.sub_type = DMUB_CMD__ABM_SAVE_RESTORE;
 
 	cmd.abm_save_restore.abm_init_config_data.src.quad_part = dc->dmub_srv->dmub->scratch_mem_fb.gpu_addr;
-	cmd.abm_save_restore.abm_init_config_data.bytes = bytes;
+	cmd.abm_save_restore.abm_init_config_data.bytes = (uint16_t)bytes;
 	cmd.abm_save_restore.abm_init_config_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
 	cmd.abm_save_restore.abm_init_config_data.panel_mask = panel_mask;
 
@@ -265,10 +265,10 @@ bool dmub_abm_set_pipe(struct abm *abm,
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.abm_set_pipe.header.type = DMUB_CMD__ABM;
 	cmd.abm_set_pipe.header.sub_type = DMUB_CMD__ABM_SET_PIPE;
-	cmd.abm_set_pipe.abm_set_pipe_data.otg_inst = otg_inst;
-	cmd.abm_set_pipe.abm_set_pipe_data.pwrseq_inst = pwrseq_inst;
-	cmd.abm_set_pipe.abm_set_pipe_data.set_pipe_option = option;
-	cmd.abm_set_pipe.abm_set_pipe_data.panel_inst = panel_inst;
+	cmd.abm_set_pipe.abm_set_pipe_data.otg_inst = (uint8_t)otg_inst;
+	cmd.abm_set_pipe.abm_set_pipe_data.pwrseq_inst = (uint8_t)pwrseq_inst;
+	cmd.abm_set_pipe.abm_set_pipe_data.set_pipe_option = (uint8_t)option;
+	cmd.abm_set_pipe.abm_set_pipe_data.panel_inst = (uint8_t)panel_inst;
 	cmd.abm_set_pipe.abm_set_pipe_data.ramping_boundary = ramping_boundary;
 	cmd.abm_set_pipe.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_pipe_data);
 
@@ -308,7 +308,7 @@ bool dmub_abm_set_event(struct abm *abm, unsigned int scaling_enable, unsigned i
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.abm_set_event.header.type = DMUB_CMD__ABM;
 	cmd.abm_set_event.header.sub_type = DMUB_CMD__ABM_SET_EVENT;
-	cmd.abm_set_event.abm_set_event_data.vb_scaling_enable = scaling_enable;
+	cmd.abm_set_event.abm_set_event_data.vb_scaling_enable = (uint8_t)scaling_enable;
 	cmd.abm_set_event.abm_set_event_data.vb_scaling_strength_mapping = scaling_strength_map;
 	cmd.abm_set_event.abm_set_event_data.panel_mask = (1<<panel_inst);
 	cmd.abm_set_event.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_event_data);
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
index f94fd007af233..45630c3effe1a 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_psr.c
@@ -341,26 +341,26 @@ static bool dmub_psr_copy_settings(struct dmub_psr *dmub,
 	copy_settings_data->mpcc_inst				= pipe_ctx->plane_res.mpcc_inst;
 
 	if (pipe_ctx->plane_res.dpp)
-		copy_settings_data->dpp_inst			= pipe_ctx->plane_res.dpp->inst;
+		copy_settings_data->dpp_inst			= (uint8_t)pipe_ctx->plane_res.dpp->inst;
 	else
 		copy_settings_data->dpp_inst			= 0;
 	if (pipe_ctx->stream_res.opp)
-		copy_settings_data->opp_inst			= pipe_ctx->stream_res.opp->inst;
+		copy_settings_data->opp_inst			= (uint8_t)pipe_ctx->stream_res.opp->inst;
 	else
 		copy_settings_data->opp_inst			= 0;
 	if (pipe_ctx->stream_res.tg)
-		copy_settings_data->otg_inst			= pipe_ctx->stream_res.tg->inst;
+		copy_settings_data->otg_inst			= (uint8_t)pipe_ctx->stream_res.tg->inst;
 	else
 		copy_settings_data->otg_inst			= 0;
 
 	// Misc
 	copy_settings_data->use_phy_fsm             = link->ctx->dc->debug.psr_power_use_phy_fsm;
-	copy_settings_data->psr_level				= psr_context->psr_level.u32all;
+	copy_settings_data->psr_level				= (uint16_t)psr_context->psr_level.u32all;
 	copy_settings_data->smu_optimizations_en		= psr_context->allow_smu_optimizations;
 	copy_settings_data->multi_disp_optimizations_en	= psr_context->allow_multi_disp_optimizations;
-	copy_settings_data->frame_delay				= psr_context->frame_delay;
+	copy_settings_data->frame_delay				= (uint8_t)psr_context->frame_delay;
 	copy_settings_data->frame_cap_ind			= psr_context->psrFrameCaptureIndicationReq;
-	copy_settings_data->init_sdp_deadline			= psr_context->sdpTransmitLineNumDeadline;
+	copy_settings_data->init_sdp_deadline			= (uint16_t)psr_context->sdpTransmitLineNumDeadline;
 	copy_settings_data->debug.u32All = 0;
 	copy_settings_data->debug.bitfields.visual_confirm	= dc->dc->debug.visual_confirm == VISUAL_CONFIRM_PSR;
 	copy_settings_data->debug.bitfields.use_hw_lock_mgr		= 1;
diff --git a/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c b/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c
index 0af1b8e0a49ec..9b50bf1d80fc8 100644
--- a/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c
+++ b/drivers/gpu/drm/amd/display/dc/dce/dmub_replay.c
@@ -151,22 +151,23 @@ static bool dmub_replay_copy_settings(struct dmub_replay *dmub,
 	copy_settings_data->digfe_inst				= replay_context->digfe_inst;
 
 	if (pipe_ctx->plane_res.dpp)
-		copy_settings_data->dpp_inst			= pipe_ctx->plane_res.dpp->inst;
+		copy_settings_data->dpp_inst			= (uint8_t)pipe_ctx->plane_res.dpp->inst;
 	else
 		copy_settings_data->dpp_inst			= 0;
+
 	if (pipe_ctx->stream_res.tg)
-		copy_settings_data->otg_inst			= pipe_ctx->stream_res.tg->inst;
+		copy_settings_data->otg_inst			= (uint8_t)pipe_ctx->stream_res.tg->inst;
 	else
 		copy_settings_data->otg_inst			= 0;
 
 	copy_settings_data->dpphy_inst				= link->link_enc->transmitter;
 
 	// Misc
-	copy_settings_data->line_time_in_ns			= replay_context->line_time_in_ns;
-	copy_settings_data->panel_inst				= panel_inst;
-	copy_settings_data->debug.u32All			= link->replay_settings.config.debug_flags;
+	copy_settings_data->line_time_in_ns			= (uint16_t)replay_context->line_time_in_ns;
+	copy_settings_data->panel_inst				= (uint16_t)panel_inst;
+	copy_settings_data->debug.u32All			= (uint16_t)link->replay_settings.config.debug_flags;
 	copy_settings_data->pixel_deviation_per_line		= link->dpcd_caps.pr_info.pixel_deviation_per_line;
-	copy_settings_data->max_deviation_line			= link->dpcd_caps.pr_info.max_deviation_line;
+	copy_settings_data->max_deviation_line			= (uint16_t)link->dpcd_caps.pr_info.max_deviation_line;
 	copy_settings_data->smu_optimizations_en		= link->replay_settings.replay_smu_opt_enable;
 	copy_settings_data->replay_timing_sync_supported = link->replay_settings.config.replay_timing_sync_supported;
 	copy_settings_data->replay_support_fast_resync_in_ultra_sleep_mode = link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode;
@@ -193,13 +194,13 @@ static bool dmub_replay_copy_settings(struct dmub_replay *dmub,
 
 	copy_settings_data->flags.bitfields.alpm_mode = (enum dmub_alpm_mode)link->replay_settings.config.alpm_mode;
 	if (link->replay_settings.config.alpm_mode == DC_ALPM_AUXLESS) {
-		copy_settings_data->auxless_alpm_data.lfps_setup_ns = dc->dc->debug.auxless_alpm_lfps_setup_ns;
-		copy_settings_data->auxless_alpm_data.lfps_period_ns = dc->dc->debug.auxless_alpm_lfps_period_ns;
-		copy_settings_data->auxless_alpm_data.lfps_silence_ns = dc->dc->debug.auxless_alpm_lfps_silence_ns;
+		copy_settings_data->auxless_alpm_data.lfps_setup_ns = (uint16_t)dc->dc->debug.auxless_alpm_lfps_setup_ns;
+		copy_settings_data->auxless_alpm_data.lfps_period_ns = (uint16_t)dc->dc->debug.auxless_alpm_lfps_period_ns;
+		copy_settings_data->auxless_alpm_data.lfps_silence_ns = (uint16_t)dc->dc->debug.auxless_alpm_lfps_silence_ns;
 		copy_settings_data->auxless_alpm_data.lfps_t1_t2_override_us =
-			dc->dc->debug.auxless_alpm_lfps_t1t2_us;
+			(uint16_t)dc->dc->debug.auxless_alpm_lfps_t1t2_us;
 		copy_settings_data->auxless_alpm_data.lfps_t1_t2_offset_us =
-			dc->dc->debug.auxless_alpm_lfps_t1t2_offset_us;
+			(uint16_t)dc->dc->debug.auxless_alpm_lfps_t1t2_offset_us;
 		copy_settings_data->auxless_alpm_data.lttpr_count = link->dc->link_srv->dp_get_lttpr_count(link);
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dce80/dce80_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce80/dce80_timing_generator.c
index 53c03364f5d48..20c18ac87998e 100644
--- a/drivers/gpu/drm/amd/display/dc/dce80/dce80_timing_generator.c
+++ b/drivers/gpu/drm/amd/display/dc/dce80/dce80_timing_generator.c
@@ -98,7 +98,7 @@ static void program_pix_dur(struct timing_generator *tg, uint32_t pix_clk_100hz)
 
 	set_reg_field_value(
 		value,
-		pix_dur,
+		(uint32_t)pix_dur,
 		DPG_PIPE_ARBITRATION_CONTROL1,
 		PIXEL_DURATION);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
index c702a30563f95..502a2d9b155f6 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_cm_common.c
@@ -75,8 +75,8 @@ void cm_helper_read_color_matrices(struct dc_context *ctx,
 				csc_c11, &regval0,
 				csc_c12, &regval1);
 
-		regval[2 * i] = regval0;
-		regval[(2 * i) + 1] = regval1;
+		regval[2 * i] = (uint16_t)regval0;
+		regval[(2 * i) + 1] = (uint16_t)regval1;
 
 		i++;
 	}
@@ -342,8 +342,8 @@ bool cm_helper_translate_curve_to_hw_format(struct dc_context *ctx,
 	struct pwl_result_data *rgb_minus_1;
 
 	int32_t region_start, region_end;
-	int32_t i;
-	uint32_t j, k, seg_distr[MAX_REGIONS_NUMBER], increment, start_index, hw_points;
+	int32_t k;
+	uint32_t i, j, seg_distr[MAX_REGIONS_NUMBER], increment, start_index, hw_points;
 
 	if (output_tf == NULL || lut_params == NULL || output_tf->type == TF_TYPE_BYPASS)
 		return false;
@@ -398,8 +398,8 @@ bool cm_helper_translate_curve_to_hw_format(struct dc_context *ctx,
 			if (j == hw_points - 1)
 				break;
 			if (i >= TRANSFER_FUNC_POINTS) {
-				DC_LOG_ERROR("Index out of bounds: i=%d, TRANSFER_FUNC_POINTS=%d\n",
-					     i, TRANSFER_FUNC_POINTS);
+				DC_LOG_ERROR("Index out of bounds: i=%u, TRANSFER_FUNC_POINTS=%u\n",
+					     i, (uint32_t)TRANSFER_FUNC_POINTS);
 				return false;
 			}
 			rgb_resulted[j].red = output_tf->tf_pts.red[i];
@@ -547,8 +547,8 @@ bool cm_helper_translate_curve_to_degamma_hw_format(
 	struct pwl_result_data *rgb_plus_1;
 
 	int32_t region_start, region_end;
-	int32_t i;
-	uint32_t j, k, seg_distr[MAX_REGIONS_NUMBER], increment, start_index, hw_points;
+	int32_t k;
+	uint32_t i, j, seg_distr[MAX_REGIONS_NUMBER], increment, start_index, hw_points;
 
 	if (output_tf == NULL || lut_params == NULL || output_tf->type == TF_TYPE_BYPASS)
 		return false;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
index baf663b661c84..feb6ca92802c3 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer_debug.c
@@ -59,7 +59,7 @@ unsigned int snprintf_count(char *pbuf, unsigned int bufsize, const char *fmt, .
 	va_end(args);
 
 	if (ret_vsnprintf > 0) {
-		if (ret_vsnprintf < bufsize)
+		if ((unsigned int)ret_vsnprintf < bufsize)
 			chars_printed = ret_vsnprintf;
 		else
 			chars_printed = bufsize - 1;
@@ -73,7 +73,7 @@ static unsigned int dcn10_get_hubbub_state(struct dc *dc, char *pBuf, unsigned i
 {
 	struct dc_context *dc_ctx = dc->ctx;
 	struct dcn_hubbub_wm wm;
-	int i;
+	unsigned int i;
 
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
@@ -111,7 +111,7 @@ static unsigned int dcn10_get_hubp_states(struct dc *dc, char *pBuf, unsigned in
 {
 	struct dc_context *dc_ctx = dc->ctx;
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
@@ -189,7 +189,7 @@ static unsigned int dcn10_get_hubp_states(struct dc *dc, char *pBuf, unsigned in
 static unsigned int dcn10_get_rq_states(struct dc *dc, char *pBuf, unsigned int bufSize)
 {
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
@@ -231,7 +231,7 @@ static unsigned int dcn10_get_rq_states(struct dc *dc, char *pBuf, unsigned int
 static unsigned int dcn10_get_dlg_states(struct dc *dc, char *pBuf, unsigned int bufSize)
 {
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
@@ -288,7 +288,7 @@ static unsigned int dcn10_get_dlg_states(struct dc *dc, char *pBuf, unsigned int
 static unsigned int dcn10_get_ttu_states(struct dc *dc, char *pBuf, unsigned int bufSize)
 {
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
@@ -328,7 +328,7 @@ static unsigned int dcn10_get_ttu_states(struct dc *dc, char *pBuf, unsigned int
 static unsigned int dcn10_get_cm_states(struct dc *dc, char *pBuf, unsigned int bufSize)
 {
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
@@ -383,7 +383,7 @@ static unsigned int dcn10_get_cm_states(struct dc *dc, char *pBuf, unsigned int
 static unsigned int dcn10_get_mpcc_states(struct dc *dc, char *pBuf, unsigned int bufSize)
 {
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
@@ -414,7 +414,7 @@ static unsigned int dcn10_get_mpcc_states(struct dc *dc, char *pBuf, unsigned in
 static unsigned int dcn10_get_otg_states(struct dc *dc, char *pBuf, unsigned int bufSize)
 {
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	unsigned int chars_printed = 0;
 	unsigned int remaining_buffer = bufSize;
@@ -491,7 +491,7 @@ static unsigned int dcn10_get_clock_states(struct dc *dc, char *pBuf, unsigned i
 static void dcn10_clear_otpc_underflow(struct dc *dc)
 {
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->timing_generator_count; i++) {
 		struct timing_generator *tg = pool->timing_generators[i];
@@ -508,7 +508,7 @@ static void dcn10_clear_otpc_underflow(struct dc *dc)
 static void dcn10_clear_hubp_underflow(struct dc *dc)
 {
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->pipe_count; i++) {
 		struct hubp *hubp = pool->hubps[i];
diff --git a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb_scl.c b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb_scl.c
index f73c5f42ea68e..7163426a0920a 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb_scl.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dwb_scl.c
@@ -696,7 +696,7 @@ static void wbscl_set_scaler_filter(
 	for (phase = 0; phase < (NUM_PHASES / 2 + 1); phase++) {
 		for (pair = 0; pair < tap_pairs; pair++) {
 			even_coef = filter[phase * taps + 2 * pair];
-			if ((pair * 2 + 1) < taps)
+			if ((uint32_t)(pair * 2 + 1) < taps)
 				odd_coef = filter[phase * taps + 2 * pair + 1];
 			else
 				odd_coef = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_link_encoder.c
index 36456c9971c84..fcaa3884d705e 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn21/dcn21_link_encoder.c
@@ -205,7 +205,7 @@ static bool update_cfg_data(
 static bool dcn21_link_encoder_acquire_phy(struct link_encoder *enc)
 {
 	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
-	int value;
+	uint32_t value;
 
 	if (enc->features.flags.bits.DP_IS_USB_C) {
 		REG_GET(RDPCSTX_PHY_CNTL6,
diff --git a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
index 9dbccf58dde5e..bfd5515c2f4fb 100644
--- a/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
+++ b/drivers/gpu/drm/amd/display/dc/dcn30/dcn30_cm_common.c
@@ -172,11 +172,13 @@ bool cm3_helper_translate_curve_to_hw_format(struct dc_context *ctx,
 	hw_points = fixpoint ? (hw_points - 1) : hw_points;
 
 	j = 0;
-	for (k = 0; k < (region_end - region_start); k++) {
+	uint32_t region_span = (uint32_t)(region_end - region_start);
+	for (k = 0; k < region_span; k++) {
 		increment = NUMBER_SW_SEGMENTS / (1 << seg_distr[k]);
 		start_index = (region_start + k + MAX_LOW_POINT) *
 				NUMBER_SW_SEGMENTS;
-		for (i = start_index; i < start_index + NUMBER_SW_SEGMENTS;
+		int32_t index_end = (int32_t)(start_index + NUMBER_SW_SEGMENTS);
+		for (i = start_index; i < index_end;
 				i += increment) {
 			if (j == hw_points)
 				break;
@@ -258,7 +260,7 @@ bool cm3_helper_translate_curve_to_hw_format(struct dc_context *ctx,
 			uint32_t green_clamp;
 			uint32_t blue_clamp;
 
-			if (i >= hw_points) {
+			if ((uint32_t)i >= hw_points) {
 				if (dc_fixpt_lt(rgb_plus_1->red, rgb->red))
 					rgb_plus_1->red = dc_fixpt_add(rgb->red,
 							rgb_minus_1->delta_red);
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
index 13e14aad3daac..59b68422334dd 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn10/dcn10_link_encoder.c
@@ -450,7 +450,7 @@ static uint8_t get_frontend_source(
 unsigned int dcn10_get_dig_frontend(struct link_encoder *enc)
 {
 	struct dcn10_link_encoder *enc10 = TO_DCN10_LINK_ENC(enc);
-	int32_t value;
+	uint32_t value;
 	enum engine_id result;
 
 	REG_GET(DIG_BE_CNTL, DIG_FE_SOURCE_SELECT, &value);
@@ -622,10 +622,11 @@ static bool dcn10_link_encoder_validate_hdmi_output(
 {
 	enum dc_color_depth max_deep_color =
 			enc10->base.features.max_hdmi_deep_color;
+	uint32_t pix_clk_100hz = (uint32_t)adjusted_pix_clk_100hz;
 
 	// check pixel clock against edid specified max TMDS clk
 	if (edid_caps->max_tmds_clk_mhz != 0 &&
-			adjusted_pix_clk_100hz > edid_caps->max_tmds_clk_mhz * 10000)
+			pix_clk_100hz > edid_caps->max_tmds_clk_mhz * 10000)
 		return false;
 
 	if (max_deep_color < crtc_timing->display_color_depth)
@@ -633,11 +634,11 @@ static bool dcn10_link_encoder_validate_hdmi_output(
 
 	if (crtc_timing->display_color_depth < COLOR_DEPTH_888)
 		return false;
-	if (adjusted_pix_clk_100hz < (TMDS_MIN_PIXEL_CLOCK * 10))
+	if (pix_clk_100hz < (TMDS_MIN_PIXEL_CLOCK * 10))
 		return false;
 
-	if ((adjusted_pix_clk_100hz == 0) ||
-		(adjusted_pix_clk_100hz > (enc10->base.features.max_hdmi_pixel_clock * 10)))
+	if ((pix_clk_100hz == 0) ||
+		(pix_clk_100hz > (enc10->base.features.max_hdmi_pixel_clock * 10)))
 		return false;
 
 	/* DCE11 HW does not support 420 */
@@ -647,7 +648,7 @@ static bool dcn10_link_encoder_validate_hdmi_output(
 
 	if ((!enc10->base.features.flags.bits.HDMI_6GB_EN ||
 			enc10->base.ctx->dc->debug.hdmi20_disable) &&
-			adjusted_pix_clk_100hz >= 3000000)
+			pix_clk_100hz >= 3000000)
 		return false;
 	if (enc10->base.ctx->dc->debug.hdmi20_disable &&
 		crtc_timing->pixel_encoding == PIXEL_ENCODING_YCBCR420)
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_stream_encoder.c
index e93be7b6d9b03..c14fb51c26116 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn30/dcn30_dio_stream_encoder.c
@@ -470,7 +470,11 @@ void enc3_stream_encoder_update_dp_info_packets(
 				&info_frame->spd,
 				true);
 	}
-	if (info_frame->hdrsmd.valid) {
+	/* While smart power oled is enabled DMUB is scanning the contents of each frame
+	 * and updating the HDR infopacket contents. Therefore we transition the infopacket
+	 * programming control to DMUB while Smart Power OLED is enabled.
+	 */
+	if (info_frame->hdrsmd.valid && !info_frame->firmware_controlled_hdr_info_packet) {
 		enc->vpg->funcs->update_generic_info_packet(
 				enc->vpg,
 				3,  /* packetIndex */
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_stream_encoder.c
index 2d33ed0c062dd..79d0c2896d295 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn401/dcn401_dio_stream_encoder.c
@@ -43,7 +43,7 @@
 
 #undef FN
 #define FN(reg_name, field_name) \
-	enc1->se_shift->field_name, enc1->se_mask->field_name
+	(uint8_t)enc1->se_shift->field_name, enc1->se_mask->field_name
 
 #define VBI_LINE_0 0
 #define HDMI_CLOCK_CHANNEL_RATE_MORE_340M 340000
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.h b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.h
index 4b5a9594f2790..9c607b24ec1c9 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_link_encoder.h
@@ -121,8 +121,6 @@
 	LE_SF(DIO_CLK_CNTL, SYMCLKC_G_HDCP_GATE_DIS, mask_sh),\
 	LE_SF(DIO_CLK_CNTL, SYMCLKD_G_HDCP_GATE_DIS, mask_sh),\
 	LE_SF(DIO_CLK_CNTL, SYMCLKE_G_HDCP_GATE_DIS, mask_sh),\
-	LE_SF(DIO_CLK_CNTL, SYMCLKF_G_HDCP_GATE_DIS, mask_sh),\
-	LE_SF(DIO_CLK_CNTL, SYMCLKG_G_HDCP_GATE_DIS, mask_sh)
 
 void dcn42_link_encoder_construct(
 	struct dcn20_link_encoder *enc20,
diff --git a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c
index 65afbfcaa96b0..55ddb9cf8a520 100644
--- a/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/dio/dcn42/dcn42_dio_stream_encoder.c
@@ -23,7 +23,7 @@
 
 #undef FN
 #define FN(reg_name, field_name) \
-	enc1->se_shift->field_name, enc1->se_mask->field_name
+	(uint8_t)enc1->se_shift->field_name, enc1->se_mask->field_name
 
 #define VBI_LINE_0 0
 #define HDMI_CLOCK_CHANNEL_RATE_MORE_340M 340000
@@ -401,7 +401,7 @@ void enc42_se_enable_audio_clock(
 {
 	struct dcn10_stream_encoder *enc1 = DCN10STRENC_FROM_STRENC(enc);
 
-	REG_UPDATE(DIG_FE_AUDIO_CNTL, APG_CLOCK_ENABLE, !!enable);
+	REG_UPDATE(DIG_FE_AUDIO_CNTL, APG_CLOCK_ENABLE, enable);
 }
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dm_helpers.h b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
index 107aec6a12657..63704d21a0b5c 100644
--- a/drivers/gpu/drm/amd/display/dc/dm_helpers.h
+++ b/drivers/gpu/drm/amd/display/dc/dm_helpers.h
@@ -30,6 +30,7 @@
 #ifndef __DM_HELPERS__
 #define __DM_HELPERS__
 
+#include "modules/inc/mod_info_packet_types.h"
 #include "dc_types.h"
 #include "dc.h"
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c
index 288d22a16cf25..5c00ef4239680 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_auto.c
@@ -45,25 +45,22 @@ void scaler_settings_calculation(struct dcn_bw_internal_vars *v)
 			if (v->source_scan[k] == dcn_bw_hor) {
 				v->h_ratio[k] = v->viewport_width[k] / v->scaler_rec_out_width[k];
 				v->v_ratio[k] = v->viewport_height[k] / v->scaler_recout_height[k];
-			}
-			else {
+			} else {
 				v->h_ratio[k] = v->viewport_height[k] / v->scaler_rec_out_width[k];
 				v->v_ratio[k] = v->viewport_width[k] / v->scaler_recout_height[k];
 			}
-		}
-		else {
+		} else {
 			if (v->source_scan[k] == dcn_bw_hor) {
-				v->h_ratio[k] =dcn_bw_max2(v->viewport_width[k] / v->scaler_rec_out_width[k], v->viewport_height[k] / v->scaler_recout_height[k]);
-			}
-			else {
-				v->h_ratio[k] =dcn_bw_max2(v->viewport_height[k] / v->scaler_rec_out_width[k], v->viewport_width[k] / v->scaler_recout_height[k]);
+				v->h_ratio[k] = dcn_bw_max2(v->viewport_width[k] / v->scaler_rec_out_width[k], v->viewport_height[k] / v->scaler_recout_height[k]);
+			} else {
+				v->h_ratio[k] = dcn_bw_max2(v->viewport_height[k] / v->scaler_rec_out_width[k], v->viewport_width[k] / v->scaler_recout_height[k]);
 			}
 			v->v_ratio[k] = v->h_ratio[k];
 		}
-		if (v->interlace_output[k] == 1.0) {
-			v->v_ratio[k] = 2.0 * v->v_ratio[k];
+		if (v->interlace_output[k] == 1.0f) {
+			v->v_ratio[k] = 2.0f * v->v_ratio[k];
 		}
-		if (v->underscan_output[k] == 1.0) {
+		if (v->underscan_output[k] == 1.0f) {
 			v->h_ratio[k] = v->h_ratio[k] * v->under_scan_factor;
 			v->v_ratio[k] = v->v_ratio[k] * v->under_scan_factor;
 		}
@@ -71,46 +68,38 @@ void scaler_settings_calculation(struct dcn_bw_internal_vars *v)
 	/*scaler taps calculation*/
 
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->h_ratio[k] > 1.0) {
-			v->acceptable_quality_hta_ps =dcn_bw_min2(v->max_hscl_taps, 2.0 *dcn_bw_ceil2(v->h_ratio[k], 1.0));
-		}
-		else if (v->h_ratio[k] < 1.0) {
-			v->acceptable_quality_hta_ps = 4.0;
-		}
-		else {
-			v->acceptable_quality_hta_ps = 1.0;
+		if (v->h_ratio[k] > 1.0f) {
+			v->acceptable_quality_hta_ps = dcn_bw_min2(v->max_hscl_taps, 2.0f * dcn_bw_ceil2(v->h_ratio[k], 1.0f));
+		} else if (v->h_ratio[k] < 1.0f) {
+			v->acceptable_quality_hta_ps = 4.0f;
+		} else {
+			v->acceptable_quality_hta_ps = 1.0f;
 		}
 		if (v->ta_pscalculation == dcn_bw_override) {
 			v->htaps[k] = v->override_hta_ps[k];
-		}
-		else {
+		} else {
 			v->htaps[k] = v->acceptable_quality_hta_ps;
 		}
-		if (v->v_ratio[k] > 1.0) {
-			v->acceptable_quality_vta_ps =dcn_bw_min2(v->max_vscl_taps, 2.0 *dcn_bw_ceil2(v->v_ratio[k], 1.0));
-		}
-		else if (v->v_ratio[k] < 1.0) {
-			v->acceptable_quality_vta_ps = 4.0;
-		}
-		else {
-			v->acceptable_quality_vta_ps = 1.0;
+		if (v->v_ratio[k] > 1.0f) {
+			v->acceptable_quality_vta_ps = dcn_bw_min2(v->max_vscl_taps, 2.0f * dcn_bw_ceil2(v->v_ratio[k], 1.0f));
+		} else if (v->v_ratio[k] < 1.0f) {
+			v->acceptable_quality_vta_ps = 4.0f;
+		} else {
+			v->acceptable_quality_vta_ps = 1.0f;
 		}
 		if (v->ta_pscalculation == dcn_bw_override) {
 			v->vtaps[k] = v->override_vta_ps[k];
-		}
-		else {
+		} else {
 			v->vtaps[k] = v->acceptable_quality_vta_ps;
 		}
 		if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64 || v->source_pixel_format[k] == dcn_bw_rgb_sub_32 || v->source_pixel_format[k] == dcn_bw_rgb_sub_16) {
-			v->vta_pschroma[k] = 0.0;
-			v->hta_pschroma[k] = 0.0;
-		}
-		else {
+			v->vta_pschroma[k] = 0.0f;
+			v->hta_pschroma[k] = 0.0f;
+		} else {
 			if (v->ta_pscalculation == dcn_bw_override) {
 				v->vta_pschroma[k] = v->override_vta_pschroma[k];
 				v->hta_pschroma[k] = v->override_hta_pschroma[k];
-			}
-			else {
+			} else {
 				v->vta_pschroma[k] = v->acceptable_quality_vta_ps;
 				v->hta_pschroma[k] = v->acceptable_quality_hta_ps;
 			}
@@ -129,7 +118,7 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 
 	v->scale_ratio_support = dcn_bw_yes;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->h_ratio[k] > v->max_hscl_ratio || v->v_ratio[k] > v->max_vscl_ratio || v->h_ratio[k] > v->htaps[k] || v->v_ratio[k] > v->vtaps[k] || (v->source_pixel_format[k] != dcn_bw_rgb_sub_64 && v->source_pixel_format[k] != dcn_bw_rgb_sub_32 && v->source_pixel_format[k] != dcn_bw_rgb_sub_16 && (v->h_ratio[k] / 2.0 > v->hta_pschroma[k] || v->v_ratio[k] / 2.0 > v->vta_pschroma[k]))) {
+		if (v->h_ratio[k] > v->max_hscl_ratio || v->v_ratio[k] > v->max_vscl_ratio || v->h_ratio[k] > v->htaps[k] || v->v_ratio[k] > v->vtaps[k] || (v->source_pixel_format[k] != dcn_bw_rgb_sub_64 && v->source_pixel_format[k] != dcn_bw_rgb_sub_32 && v->source_pixel_format[k] != dcn_bw_rgb_sub_16 && (v->h_ratio[k] / 2.0f > v->hta_pschroma[k] || v->v_ratio[k] / 2.0f > v->vta_pschroma[k]))) {
 			v->scale_ratio_support = dcn_bw_no;
 		}
 	}
@@ -146,60 +135,51 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->source_scan[k] == dcn_bw_hor) {
 			v->swath_width_ysingle_dpp[k] = v->viewport_width[k];
-		}
-		else {
+		} else {
 			v->swath_width_ysingle_dpp[k] = v->viewport_height[k];
 		}
 		if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
-			v->byte_per_pixel_in_dety[k] = 8.0;
-			v->byte_per_pixel_in_detc[k] = 0.0;
-		}
-		else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_32) {
-			v->byte_per_pixel_in_dety[k] = 4.0;
-			v->byte_per_pixel_in_detc[k] = 0.0;
-		}
-		else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_16) {
-			v->byte_per_pixel_in_dety[k] = 2.0;
-			v->byte_per_pixel_in_detc[k] = 0.0;
-		}
-		else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
-			v->byte_per_pixel_in_dety[k] = 1.0;
-			v->byte_per_pixel_in_detc[k] = 2.0;
-		}
-		else {
+			v->byte_per_pixel_in_dety[k] = 8.0f;
+			v->byte_per_pixel_in_detc[k] = 0.0f;
+		} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_32) {
+			v->byte_per_pixel_in_dety[k] = 4.0f;
+			v->byte_per_pixel_in_detc[k] = 0.0f;
+		} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_16) {
+			v->byte_per_pixel_in_dety[k] = 2.0f;
+			v->byte_per_pixel_in_detc[k] = 0.0f;
+		} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
+			v->byte_per_pixel_in_dety[k] = 1.0f;
+			v->byte_per_pixel_in_detc[k] = 2.0f;
+		} else {
 			v->byte_per_pixel_in_dety[k] = 4.0f / 3.0f;
 			v->byte_per_pixel_in_detc[k] = 8.0f / 3.0f;
 		}
 	}
-	v->total_read_bandwidth_consumed_gbyte_per_second = 0.0;
+	v->total_read_bandwidth_consumed_gbyte_per_second = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		v->read_bandwidth[k] = v->swath_width_ysingle_dpp[k] * (dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) * v->v_ratio[k] +dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / 2.0 * v->v_ratio[k] / 2) / (v->htotal[k] / v->pixel_clock[k]);
+		v->read_bandwidth[k] = v->swath_width_ysingle_dpp[k] * (dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) * v->v_ratio[k] + dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / 2.0f * v->v_ratio[k] / 2) / (v->htotal[k] / v->pixel_clock[k]);
 		if (v->dcc_enable[k] == dcn_bw_yes) {
 			v->read_bandwidth[k] = v->read_bandwidth[k] * (1 + 1 / 256);
 		}
 		if (v->pte_enable == dcn_bw_yes && v->source_scan[k] != dcn_bw_hor && (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d_x)) {
 			v->read_bandwidth[k] = v->read_bandwidth[k] * (1 + 1 / 64);
-		}
-		else if (v->pte_enable == dcn_bw_yes && v->source_scan[k] == dcn_bw_hor && (v->source_pixel_format[k] == dcn_bw_rgb_sub_64 || v->source_pixel_format[k] == dcn_bw_rgb_sub_32) && (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x)) {
+		} else if (v->pte_enable == dcn_bw_yes && v->source_scan[k] == dcn_bw_hor && (v->source_pixel_format[k] == dcn_bw_rgb_sub_64 || v->source_pixel_format[k] == dcn_bw_rgb_sub_32) && (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x)) {
 			v->read_bandwidth[k] = v->read_bandwidth[k] * (1 + 1 / 256);
-		}
-		else if (v->pte_enable == dcn_bw_yes) {
+		} else if (v->pte_enable == dcn_bw_yes) {
 			v->read_bandwidth[k] = v->read_bandwidth[k] * (1 + 1 / 512);
 		}
-		v->total_read_bandwidth_consumed_gbyte_per_second = v->total_read_bandwidth_consumed_gbyte_per_second + v->read_bandwidth[k] / 1000.0;
+		v->total_read_bandwidth_consumed_gbyte_per_second = v->total_read_bandwidth_consumed_gbyte_per_second + v->read_bandwidth[k] / 1000.0f;
 	}
-	v->total_write_bandwidth_consumed_gbyte_per_second = 0.0;
+	v->total_write_bandwidth_consumed_gbyte_per_second = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->output[k] == dcn_bw_writeback && v->output_format[k] == dcn_bw_444) {
-			v->write_bandwidth[k] = v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 4.0;
-		}
-		else if (v->output[k] == dcn_bw_writeback) {
-			v->write_bandwidth[k] = v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 1.5;
+			v->write_bandwidth[k] = v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 4.0f;
+		} else if (v->output[k] == dcn_bw_writeback) {
+			v->write_bandwidth[k] = v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 1.5f;
+		} else {
+			v->write_bandwidth[k] = 0.0f;
 		}
-		else {
-			v->write_bandwidth[k] = 0.0;
-		}
-		v->total_write_bandwidth_consumed_gbyte_per_second = v->total_write_bandwidth_consumed_gbyte_per_second + v->write_bandwidth[k] / 1000.0;
+		v->total_write_bandwidth_consumed_gbyte_per_second = v->total_write_bandwidth_consumed_gbyte_per_second + v->write_bandwidth[k] / 1000.0f;
 	}
 	v->total_bandwidth_consumed_gbyte_per_second = v->total_read_bandwidth_consumed_gbyte_per_second + v->total_write_bandwidth_consumed_gbyte_per_second;
 	v->dcc_enabled_in_any_plane = dcn_bw_no;
@@ -209,29 +189,28 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 		}
 	}
 	for (i = 0; i <= number_of_states_plus_one; i++) {
-		v->return_bw_todcn_per_state =dcn_bw_min2(v->return_bus_width * v->dcfclk_per_state[i], v->fabric_and_dram_bandwidth_per_state[i] * 1000.0 * v->percent_of_ideal_drambw_received_after_urg_latency / 100.0);
+		v->return_bw_todcn_per_state = dcn_bw_min2(v->return_bus_width * v->dcfclk_per_state[i], v->fabric_and_dram_bandwidth_per_state[i] * 1000.0f * v->percent_of_ideal_drambw_received_after_urg_latency / 100.0f);
 		v->return_bw_per_state[i] = v->return_bw_todcn_per_state;
-		if (v->dcc_enabled_in_any_plane == dcn_bw_yes && v->return_bw_todcn_per_state > v->dcfclk_per_state[i] * v->return_bus_width / 4.0) {
-			v->return_bw_per_state[i] =dcn_bw_min2(v->return_bw_per_state[i], v->return_bw_todcn_per_state * 4.0 * (1.0 - v->urgent_latency / ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0 / (v->return_bw_todcn_per_state - v->dcfclk_per_state[i] * v->return_bus_width / 4.0) + v->urgent_latency)));
+		if (v->dcc_enabled_in_any_plane == dcn_bw_yes && v->return_bw_todcn_per_state > v->dcfclk_per_state[i] * v->return_bus_width / 4.0f) {
+			v->return_bw_per_state[i] = dcn_bw_min2(v->return_bw_per_state[i], v->return_bw_todcn_per_state * 4.0f * (1.0f - v->urgent_latency / ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f / (v->return_bw_todcn_per_state - v->dcfclk_per_state[i] * v->return_bus_width / 4.0f) + v->urgent_latency)));
 		}
-		v->critical_point = 2.0 * v->return_bus_width * v->dcfclk_per_state[i] * v->urgent_latency / (v->return_bw_todcn_per_state * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0);
-		if (v->dcc_enabled_in_any_plane == dcn_bw_yes && v->critical_point > 1.0 && v->critical_point < 4.0) {
-			v->return_bw_per_state[i] =dcn_bw_min2(v->return_bw_per_state[i], dcn_bw_pow(4.0 * v->return_bw_todcn_per_state * (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0 * v->return_bus_width * v->dcfclk_per_state[i] * v->urgent_latency / (v->return_bw_todcn_per_state * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0), 2));
+		v->critical_point = 2.0f * v->return_bus_width * v->dcfclk_per_state[i] * v->urgent_latency / (v->return_bw_todcn_per_state * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f);
+		if (v->dcc_enabled_in_any_plane == dcn_bw_yes && v->critical_point > 1.0f && v->critical_point < 4.0f) {
+			v->return_bw_per_state[i] = dcn_bw_min2(v->return_bw_per_state[i], dcn_bw_pow(4.0f * v->return_bw_todcn_per_state * (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f * v->return_bus_width * v->dcfclk_per_state[i] * v->urgent_latency / (v->return_bw_todcn_per_state * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f), 2));
 		}
-		v->return_bw_todcn_per_state =dcn_bw_min2(v->return_bus_width * v->dcfclk_per_state[i], v->fabric_and_dram_bandwidth_per_state[i] * 1000.0);
-		if (v->dcc_enabled_in_any_plane == dcn_bw_yes && v->return_bw_todcn_per_state > v->dcfclk_per_state[i] * v->return_bus_width / 4.0) {
-			v->return_bw_per_state[i] =dcn_bw_min2(v->return_bw_per_state[i], v->return_bw_todcn_per_state * 4.0 * (1.0 - v->urgent_latency / ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0 / (v->return_bw_todcn_per_state - v->dcfclk_per_state[i] * v->return_bus_width / 4.0) + v->urgent_latency)));
+		v->return_bw_todcn_per_state = dcn_bw_min2(v->return_bus_width * v->dcfclk_per_state[i], v->fabric_and_dram_bandwidth_per_state[i] * 1000.0f);
+		if (v->dcc_enabled_in_any_plane == dcn_bw_yes && v->return_bw_todcn_per_state > v->dcfclk_per_state[i] * v->return_bus_width / 4.0f) {
+			v->return_bw_per_state[i] = dcn_bw_min2(v->return_bw_per_state[i], v->return_bw_todcn_per_state * 4.0f * (1.0f - v->urgent_latency / ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f / (v->return_bw_todcn_per_state - v->dcfclk_per_state[i] * v->return_bus_width / 4.0f) + v->urgent_latency)));
 		}
-		v->critical_point = 2.0 * v->return_bus_width * v->dcfclk_per_state[i] * v->urgent_latency / (v->return_bw_todcn_per_state * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0);
-		if (v->dcc_enabled_in_any_plane == dcn_bw_yes && v->critical_point > 1.0 && v->critical_point < 4.0) {
-			v->return_bw_per_state[i] =dcn_bw_min2(v->return_bw_per_state[i], dcn_bw_pow(4.0 * v->return_bw_todcn_per_state * (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0 * v->return_bus_width * v->dcfclk_per_state[i] * v->urgent_latency / (v->return_bw_todcn_per_state * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0), 2));
+		v->critical_point = 2.0f * v->return_bus_width * v->dcfclk_per_state[i] * v->urgent_latency / (v->return_bw_todcn_per_state * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f);
+		if (v->dcc_enabled_in_any_plane == dcn_bw_yes && v->critical_point > 1.0f && v->critical_point < 4.0f) {
+			v->return_bw_per_state[i] = dcn_bw_min2(v->return_bw_per_state[i], dcn_bw_pow(4.0f * v->return_bw_todcn_per_state * (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f * v->return_bus_width * v->dcfclk_per_state[i] * v->urgent_latency / (v->return_bw_todcn_per_state * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f), 2));
 		}
 	}
 	for (i = 0; i <= number_of_states_plus_one; i++) {
-		if ((v->total_read_bandwidth_consumed_gbyte_per_second * 1000.0 <= v->return_bw_per_state[i]) && (v->total_bandwidth_consumed_gbyte_per_second * 1000.0 <= v->fabric_and_dram_bandwidth_per_state[i] * 1000.0 * v->percent_of_ideal_drambw_received_after_urg_latency / 100.0)) {
+		if ((v->total_read_bandwidth_consumed_gbyte_per_second * 1000.0f <= v->return_bw_per_state[i]) && (v->total_bandwidth_consumed_gbyte_per_second * 1000.0f <= v->fabric_and_dram_bandwidth_per_state[i] * 1000.0f * v->percent_of_ideal_drambw_received_after_urg_latency / 100.0f)) {
 			v->bandwidth_support[i] = dcn_bw_yes;
-		}
-		else {
+		} else {
 			v->bandwidth_support[i] = dcn_bw_no;
 		}
 	}
@@ -239,21 +218,19 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 
 	v->writeback_latency_support = dcn_bw_yes;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->output[k] == dcn_bw_writeback && v->output_format[k] == dcn_bw_444 && v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 4.0 > (v->writeback_luma_buffer_size + v->writeback_chroma_buffer_size) * 1024.0 / v->write_back_latency) {
+		if (v->output[k] == dcn_bw_writeback && v->output_format[k] == dcn_bw_444 && v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 4.0f > (v->writeback_luma_buffer_size + v->writeback_chroma_buffer_size) * 1024.0f / v->write_back_latency) {
 			v->writeback_latency_support = dcn_bw_no;
-		}
-		else if (v->output[k] == dcn_bw_writeback && v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) >dcn_bw_min2(v->writeback_luma_buffer_size, 2.0 * v->writeback_chroma_buffer_size) * 1024.0 / v->write_back_latency) {
+		} else if (v->output[k] == dcn_bw_writeback && v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) > dcn_bw_min2(v->writeback_luma_buffer_size, 2.0f * v->writeback_chroma_buffer_size) * 1024.0f / v->write_back_latency) {
 			v->writeback_latency_support = dcn_bw_no;
 		}
 	}
 	/*re-ordering buffer support check*/
 
 	for (i = 0; i <= number_of_states_plus_one; i++) {
-		v->urgent_round_trip_and_out_of_order_latency_per_state[i] = (v->round_trip_ping_latency_cycles + 32.0) / v->dcfclk_per_state[i] + v->urgent_out_of_order_return_per_channel * v->number_of_channels / v->return_bw_per_state[i];
-		if ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0 / v->return_bw_per_state[i] > v->urgent_round_trip_and_out_of_order_latency_per_state[i]) {
+		v->urgent_round_trip_and_out_of_order_latency_per_state[i] = (v->round_trip_ping_latency_cycles + 32.0f) / v->dcfclk_per_state[i] + v->urgent_out_of_order_return_per_channel * v->number_of_channels / v->return_bw_per_state[i];
+		if ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f / v->return_bw_per_state[i] > v->urgent_round_trip_and_out_of_order_latency_per_state[i]) {
 			v->rob_support[i] = dcn_bw_yes;
-		}
-		else {
+		} else {
 			v->rob_support[i] = dcn_bw_no;
 		}
 	}
@@ -262,229 +239,201 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->output[k] == dcn_bw_dp && v->dsc_capability == dcn_bw_yes) {
 			if (v->output_format[k] == dcn_bw_420) {
-				v->required_output_bw = v->pixel_clock[k] / 2.0;
-			}
-			else {
+				v->required_output_bw = v->pixel_clock[k] / 2.0f;
+			} else {
 				v->required_output_bw = v->pixel_clock[k];
 			}
-		}
-		else if (v->output_format[k] == dcn_bw_420) {
-			v->required_output_bw = v->pixel_clock[k] * 3.0 / 2.0;
-		}
-		else {
-			v->required_output_bw = v->pixel_clock[k] * 3.0;
+		} else if (v->output_format[k] == dcn_bw_420) {
+			v->required_output_bw = v->pixel_clock[k] * 3.0f / 2.0f;
+		} else {
+			v->required_output_bw = v->pixel_clock[k] * 3.0f;
 		}
 		if (v->output[k] == dcn_bw_hdmi) {
 			v->required_phyclk[k] = v->required_output_bw;
 			switch (v->output_deep_color[k]) {
 			case dcn_bw_encoder_10bpc:
-				v->required_phyclk[k] =  v->required_phyclk[k] * 5.0 / 4;
+				v->required_phyclk[k] =  v->required_phyclk[k] * 5.0f / 4;
 			break;
 			case dcn_bw_encoder_12bpc:
-				v->required_phyclk[k] =  v->required_phyclk[k] * 3.0 / 2;
+				v->required_phyclk[k] =  v->required_phyclk[k] * 3.0f / 2;
 				break;
 			default:
 				break;
 			}
-			v->required_phyclk[k] = v->required_phyclk[k] / 3.0;
-		}
-		else if (v->output[k] == dcn_bw_dp) {
-			v->required_phyclk[k] = v->required_output_bw / 4.0;
-		}
-		else {
-			v->required_phyclk[k] = 0.0;
+			v->required_phyclk[k] = v->required_phyclk[k] / 3.0f;
+		} else if (v->output[k] == dcn_bw_dp) {
+			v->required_phyclk[k] = v->required_output_bw / 4.0f;
+		} else {
+			v->required_phyclk[k] = 0.0f;
 		}
 	}
 	for (i = 0; i <= number_of_states_plus_one; i++) {
 		v->dio_support[i] = dcn_bw_yes;
 		for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-			if (v->required_phyclk[k] > v->phyclk_per_state[i] || (v->output[k] == dcn_bw_hdmi && v->required_phyclk[k] > 600.0)) {
+			if (v->required_phyclk[k] > v->phyclk_per_state[i] || (v->output[k] == dcn_bw_hdmi && v->required_phyclk[k] > 600.0f)) {
 				v->dio_support[i] = dcn_bw_no;
 			}
 		}
 	}
 	/*total available writeback support check*/
 
-	v->total_number_of_active_writeback = 0.0;
+	v->total_number_of_active_writeback = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->output[k] == dcn_bw_writeback) {
-			v->total_number_of_active_writeback = v->total_number_of_active_writeback + 1.0;
+			v->total_number_of_active_writeback = v->total_number_of_active_writeback + 1.0f;
 		}
 	}
 	if (v->total_number_of_active_writeback <= v->max_num_writeback) {
 		v->total_available_writeback_support = dcn_bw_yes;
-	}
-	else {
+	} else {
 		v->total_available_writeback_support = dcn_bw_no;
 	}
 	/*maximum dispclk/dppclk support check*/
 
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->h_ratio[k] > 1.0) {
-			v->pscl_factor[k] =dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput * v->h_ratio[k] /dcn_bw_ceil2(v->htaps[k] / 6.0, 1.0));
-		}
-		else {
-			v->pscl_factor[k] =dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput);
+		if (v->h_ratio[k] > 1.0f) {
+			v->pscl_factor[k] = dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput * v->h_ratio[k] / dcn_bw_ceil2(v->htaps[k] / 6.0f, 1.0f));
+		} else {
+			v->pscl_factor[k] = dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput);
 		}
-		if (v->byte_per_pixel_in_detc[k] == 0.0) {
-			v->pscl_factor_chroma[k] = 0.0;
-			v->min_dppclk_using_single_dpp[k] = v->pixel_clock[k] *dcn_bw_max3(v->vtaps[k] / 6.0 *dcn_bw_min2(1.0, v->h_ratio[k]), v->h_ratio[k] * v->v_ratio[k] / v->pscl_factor[k], 1.0);
-		}
-		else {
-			if (v->h_ratio[k] / 2.0 > 1.0) {
-				v->pscl_factor_chroma[k] =dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput * v->h_ratio[k] / 2.0 /dcn_bw_ceil2(v->hta_pschroma[k] / 6.0, 1.0));
-			}
-			else {
-				v->pscl_factor_chroma[k] =dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput);
+		if (v->byte_per_pixel_in_detc[k] == 0.0f) {
+			v->pscl_factor_chroma[k] = 0.0f;
+			v->min_dppclk_using_single_dpp[k] = v->pixel_clock[k] * dcn_bw_max3(v->vtaps[k] / 6.0f * dcn_bw_min2(1.0f, v->h_ratio[k]), v->h_ratio[k] * v->v_ratio[k] / v->pscl_factor[k], 1.0f);
+		} else {
+			if (v->h_ratio[k] / 2.0f > 1.0f) {
+				v->pscl_factor_chroma[k] = dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput * v->h_ratio[k] / 2.0f / dcn_bw_ceil2(v->hta_pschroma[k] / 6.0f, 1.0f));
+			} else {
+				v->pscl_factor_chroma[k] = dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput);
 			}
-			v->min_dppclk_using_single_dpp[k] = v->pixel_clock[k] *dcn_bw_max5(v->vtaps[k] / 6.0 *dcn_bw_min2(1.0, v->h_ratio[k]), v->h_ratio[k] * v->v_ratio[k] / v->pscl_factor[k], v->vta_pschroma[k] / 6.0 *dcn_bw_min2(1.0, v->h_ratio[k] / 2.0), v->h_ratio[k] * v->v_ratio[k] / 4.0 / v->pscl_factor_chroma[k], 1.0);
+			v->min_dppclk_using_single_dpp[k] = v->pixel_clock[k] * dcn_bw_max5(v->vtaps[k] / 6.0f * dcn_bw_min2(1.0f, v->h_ratio[k]), v->h_ratio[k] * v->v_ratio[k] / v->pscl_factor[k], v->vta_pschroma[k] / 6.0f * dcn_bw_min2(1.0f, v->h_ratio[k] / 2.0f), v->h_ratio[k] * v->v_ratio[k] / 4.0f / v->pscl_factor_chroma[k], 1.0f);
 		}
 	}
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if ((v->source_pixel_format[k] == dcn_bw_rgb_sub_64 || v->source_pixel_format[k] == dcn_bw_rgb_sub_32 || v->source_pixel_format[k] == dcn_bw_rgb_sub_16)) {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-				v->read256_block_height_y[k] = 1.0;
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
-				v->read256_block_height_y[k] = 4.0;
-			}
-			else {
-				v->read256_block_height_y[k] = 8.0;
-			}
-			v->read256_block_width_y[k] = 256.0 /dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / v->read256_block_height_y[k];
-			v->read256_block_height_c[k] = 0.0;
-			v->read256_block_width_c[k] = 0.0;
-		}
-		else {
+				v->read256_block_height_y[k] = 1.0f;
+			} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
+				v->read256_block_height_y[k] = 4.0f;
+			} else {
+				v->read256_block_height_y[k] = 8.0f;
+			}
+			v->read256_block_width_y[k] = 256.0f / dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / v->read256_block_height_y[k];
+			v->read256_block_height_c[k] = 0.0f;
+			v->read256_block_width_c[k] = 0.0f;
+		} else {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-				v->read256_block_height_y[k] = 1.0;
-				v->read256_block_height_c[k] = 1.0;
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
-				v->read256_block_height_y[k] = 16.0;
-				v->read256_block_height_c[k] = 8.0;
-			}
-			else {
-				v->read256_block_height_y[k] = 8.0;
-				v->read256_block_height_c[k] = 8.0;
-			}
-			v->read256_block_width_y[k] = 256.0 /dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / v->read256_block_height_y[k];
-			v->read256_block_width_c[k] = 256.0 /dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / v->read256_block_height_c[k];
+				v->read256_block_height_y[k] = 1.0f;
+				v->read256_block_height_c[k] = 1.0f;
+			} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
+				v->read256_block_height_y[k] = 16.0f;
+				v->read256_block_height_c[k] = 8.0f;
+			} else {
+				v->read256_block_height_y[k] = 8.0f;
+				v->read256_block_height_c[k] = 8.0f;
+			}
+			v->read256_block_width_y[k] = 256.0f / dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / v->read256_block_height_y[k];
+			v->read256_block_width_c[k] = 256.0f / dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / v->read256_block_height_c[k];
 		}
 		if (v->source_scan[k] == dcn_bw_hor) {
 			v->max_swath_height_y[k] = v->read256_block_height_y[k];
 			v->max_swath_height_c[k] = v->read256_block_height_c[k];
-		}
-		else {
+		} else {
 			v->max_swath_height_y[k] = v->read256_block_width_y[k];
 			v->max_swath_height_c[k] = v->read256_block_width_c[k];
 		}
 		if ((v->source_pixel_format[k] == dcn_bw_rgb_sub_64 || v->source_pixel_format[k] == dcn_bw_rgb_sub_32 || v->source_pixel_format[k] == dcn_bw_rgb_sub_16)) {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear || (v->source_pixel_format[k] == dcn_bw_rgb_sub_64 && (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_var_s || v->source_surface_mode[k] == dcn_bw_sw_var_s_x) && v->source_scan[k] == dcn_bw_hor)) {
 				v->min_swath_height_y[k] = v->max_swath_height_y[k];
-			}
-			else {
-				v->min_swath_height_y[k] = v->max_swath_height_y[k] / 2.0;
+			} else {
+				v->min_swath_height_y[k] = v->max_swath_height_y[k] / 2.0f;
 			}
 			v->min_swath_height_c[k] = v->max_swath_height_c[k];
-		}
-		else {
+		} else {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
 				v->min_swath_height_y[k] = v->max_swath_height_y[k];
 				v->min_swath_height_c[k] = v->max_swath_height_c[k];
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8 && v->source_scan[k] == dcn_bw_hor) {
-				v->min_swath_height_y[k] = v->max_swath_height_y[k] / 2.0;
+			} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8 && v->source_scan[k] == dcn_bw_hor) {
+				v->min_swath_height_y[k] = v->max_swath_height_y[k] / 2.0f;
 				if (v->bug_forcing_luma_and_chroma_request_to_same_size_fixed == dcn_bw_yes) {
 					v->min_swath_height_c[k] = v->max_swath_height_c[k];
+				} else {
+					v->min_swath_height_c[k] = v->max_swath_height_c[k] / 2.0f;
 				}
-				else {
-					v->min_swath_height_c[k] = v->max_swath_height_c[k] / 2.0;
-				}
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10 && v->source_scan[k] == dcn_bw_hor) {
-				v->min_swath_height_c[k] = v->max_swath_height_c[k] / 2.0;
+			} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10 && v->source_scan[k] == dcn_bw_hor) {
+				v->min_swath_height_c[k] = v->max_swath_height_c[k] / 2.0f;
 				if (v->bug_forcing_luma_and_chroma_request_to_same_size_fixed == dcn_bw_yes) {
 					v->min_swath_height_y[k] = v->max_swath_height_y[k];
+				} else {
+					v->min_swath_height_y[k] = v->max_swath_height_y[k] / 2.0f;
 				}
-				else {
-					v->min_swath_height_y[k] = v->max_swath_height_y[k] / 2.0;
-				}
-			}
-			else {
+			} else {
 				v->min_swath_height_y[k] = v->max_swath_height_y[k];
 				v->min_swath_height_c[k] = v->max_swath_height_c[k];
 			}
 		}
 		if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-			v->maximum_swath_width = 8192.0;
-		}
-		else {
-			v->maximum_swath_width = 5120.0;
-		}
-		v->number_of_dpp_required_for_det_size =dcn_bw_ceil2(v->swath_width_ysingle_dpp[k] /dcn_bw_min2(v->maximum_swath_width, v->det_buffer_size_in_kbyte * 1024.0 / 2.0 / (v->byte_per_pixel_in_dety[k] * v->min_swath_height_y[k] + v->byte_per_pixel_in_detc[k] / 2.0 * v->min_swath_height_c[k])), 1.0);
-		if (v->byte_per_pixel_in_detc[k] == 0.0) {
-			v->number_of_dpp_required_for_lb_size =dcn_bw_ceil2((v->vtaps[k] +dcn_bw_max2(dcn_bw_ceil2(v->v_ratio[k], 1.0) - 2, 0.0)) * v->swath_width_ysingle_dpp[k] /dcn_bw_max2(v->h_ratio[k], 1.0) * v->lb_bit_per_pixel[k] / v->line_buffer_size, 1.0);
+			v->maximum_swath_width = 8192.0f;
+		} else {
+			v->maximum_swath_width = 5120.0f;
 		}
-		else {
-			v->number_of_dpp_required_for_lb_size =dcn_bw_max2(dcn_bw_ceil2((v->vtaps[k] +dcn_bw_max2(dcn_bw_ceil2(v->v_ratio[k], 1.0) - 2, 0.0)) * v->swath_width_ysingle_dpp[k] /dcn_bw_max2(v->h_ratio[k], 1.0) * v->lb_bit_per_pixel[k] / v->line_buffer_size, 1.0),dcn_bw_ceil2((v->vta_pschroma[k] +dcn_bw_max2(dcn_bw_ceil2(v->v_ratio[k] / 2.0, 1.0) - 2, 0.0)) * v->swath_width_ysingle_dpp[k] / 2.0 /dcn_bw_max2(v->h_ratio[k] / 2.0, 1.0) * v->lb_bit_per_pixel[k] / v->line_buffer_size, 1.0));
+		v->number_of_dpp_required_for_det_size = dcn_bw_ceil2(v->swath_width_ysingle_dpp[k] / dcn_bw_min2(v->maximum_swath_width, v->det_buffer_size_in_kbyte * 1024.0f / 2.0f / (v->byte_per_pixel_in_dety[k] * v->min_swath_height_y[k] + v->byte_per_pixel_in_detc[k] / 2.0f * v->min_swath_height_c[k])), 1.0f);
+		if (v->byte_per_pixel_in_detc[k] == 0.0f) {
+			v->number_of_dpp_required_for_lb_size = dcn_bw_ceil2((v->vtaps[k] + dcn_bw_max2(dcn_bw_ceil2(v->v_ratio[k], 1.0f) - 2, 0.0f)) * v->swath_width_ysingle_dpp[k] / dcn_bw_max2(v->h_ratio[k], 1.0f) * v->lb_bit_per_pixel[k] / v->line_buffer_size, 1.0f);
+		} else {
+			v->number_of_dpp_required_for_lb_size = dcn_bw_max2(dcn_bw_ceil2((v->vtaps[k] + dcn_bw_max2(dcn_bw_ceil2(v->v_ratio[k], 1.0f) - 2, 0.0f)) * v->swath_width_ysingle_dpp[k] / dcn_bw_max2(v->h_ratio[k], 1.0f) * v->lb_bit_per_pixel[k] / v->line_buffer_size, 1.0f), dcn_bw_ceil2((v->vta_pschroma[k] + dcn_bw_max2(dcn_bw_ceil2(v->v_ratio[k] / 2.0f, 1.0f) - 2, 0.0f)) * v->swath_width_ysingle_dpp[k] / 2.0f / dcn_bw_max2(v->h_ratio[k] / 2.0f, 1.0f) * v->lb_bit_per_pixel[k] / v->line_buffer_size, 1.0f));
 		}
-		v->number_of_dpp_required_for_det_and_lb_size[k] =dcn_bw_max2(v->number_of_dpp_required_for_det_size, v->number_of_dpp_required_for_lb_size);
+		v->number_of_dpp_required_for_det_and_lb_size[k] = dcn_bw_max2(v->number_of_dpp_required_for_det_size, v->number_of_dpp_required_for_lb_size);
 	}
 	for (i = 0; i <= number_of_states_plus_one; i++) {
 		for (j = 0; j <= 1; j++) {
-			v->total_number_of_active_dpp[i][j] = 0.0;
-			v->required_dispclk[i][j] = 0.0;
+			v->total_number_of_active_dpp[i][j] = 0.0f;
+			v->required_dispclk[i][j] = 0.0f;
 			v->dispclk_dppclk_support[i][j] = dcn_bw_yes;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				v->min_dispclk_using_single_dpp =dcn_bw_max2(v->pixel_clock[k], v->min_dppclk_using_single_dpp[k] * (j + 1)) * (1.0 + v->downspreading / 100.0);
+				v->min_dispclk_using_single_dpp = dcn_bw_max2(v->pixel_clock[k], v->min_dppclk_using_single_dpp[k] * (j + 1)) * (1.0f + v->downspreading / 100.0f);
 				if (v->odm_capability == dcn_bw_yes) {
-					v->min_dispclk_using_dual_dpp =dcn_bw_max2(v->pixel_clock[k] / 2.0, v->min_dppclk_using_single_dpp[k] / 2.0 * (j + 1)) * (1.0 + v->downspreading / 100.0);
-				}
-				else {
-					v->min_dispclk_using_dual_dpp =dcn_bw_max2(v->pixel_clock[k], v->min_dppclk_using_single_dpp[k] / 2.0 * (j + 1)) * (1.0 + v->downspreading / 100.0);
+					v->min_dispclk_using_dual_dpp = dcn_bw_max2(v->pixel_clock[k] / 2.0f, v->min_dppclk_using_single_dpp[k] / 2.0f * (j + 1)) * (1.0f + v->downspreading / 100.0f);
+				} else {
+					v->min_dispclk_using_dual_dpp = dcn_bw_max2(v->pixel_clock[k], v->min_dppclk_using_single_dpp[k] / 2.0f * (j + 1)) * (1.0f + v->downspreading / 100.0f);
 				}
 				if (i < number_of_states) {
-					v->min_dispclk_using_single_dpp = v->min_dispclk_using_single_dpp * (1.0 + v->dispclk_ramping_margin / 100.0);
-					v->min_dispclk_using_dual_dpp = v->min_dispclk_using_dual_dpp * (1.0 + v->dispclk_ramping_margin / 100.0);
-				}
-				if (v->min_dispclk_using_single_dpp <=dcn_bw_min2(v->max_dispclk[i], (j + 1) * v->max_dppclk[i]) && v->number_of_dpp_required_for_det_and_lb_size[k] <= 1.0) {
-					v->no_of_dpp[i][j][k] = 1.0;
-					v->required_dispclk[i][j] =dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_single_dpp);
-				}
-				else if (v->min_dispclk_using_dual_dpp <=dcn_bw_min2(v->max_dispclk[i], (j + 1) * v->max_dppclk[i])) {
-					v->no_of_dpp[i][j][k] = 2.0;
-					v->required_dispclk[i][j] =dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_dual_dpp);
-				}
-				else {
-					v->no_of_dpp[i][j][k] = 2.0;
-					v->required_dispclk[i][j] =dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_dual_dpp);
+					v->min_dispclk_using_single_dpp = v->min_dispclk_using_single_dpp * (1.0f + v->dispclk_ramping_margin / 100.0f);
+					v->min_dispclk_using_dual_dpp = v->min_dispclk_using_dual_dpp * (1.0f + v->dispclk_ramping_margin / 100.0f);
+				}
+				if (v->min_dispclk_using_single_dpp <= dcn_bw_min2(v->max_dispclk[i], (j + 1) * v->max_dppclk[i]) && v->number_of_dpp_required_for_det_and_lb_size[k] <= 1.0f) {
+					v->no_of_dpp[i][j][k] = 1.0f;
+					v->required_dispclk[i][j] = dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_single_dpp);
+				} else if (v->min_dispclk_using_dual_dpp <= dcn_bw_min2(v->max_dispclk[i], (j + 1) * v->max_dppclk[i])) {
+					v->no_of_dpp[i][j][k] = 2.0f;
+					v->required_dispclk[i][j] = dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_dual_dpp);
+				} else {
+					v->no_of_dpp[i][j][k] = 2.0f;
+					v->required_dispclk[i][j] = dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_dual_dpp);
 					v->dispclk_dppclk_support[i][j] = dcn_bw_no;
 				}
 				v->total_number_of_active_dpp[i][j] = v->total_number_of_active_dpp[i][j] + v->no_of_dpp[i][j][k];
 			}
 			if (v->total_number_of_active_dpp[i][j] > v->max_num_dpp) {
-				v->total_number_of_active_dpp[i][j] = 0.0;
-				v->required_dispclk[i][j] = 0.0;
+				v->total_number_of_active_dpp[i][j] = 0.0f;
+				v->required_dispclk[i][j] = 0.0f;
 				v->dispclk_dppclk_support[i][j] = dcn_bw_yes;
 				for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-					v->min_dispclk_using_single_dpp =dcn_bw_max2(v->pixel_clock[k], v->min_dppclk_using_single_dpp[k] * (j + 1)) * (1.0 + v->downspreading / 100.0);
-					v->min_dispclk_using_dual_dpp =dcn_bw_max2(v->pixel_clock[k], v->min_dppclk_using_single_dpp[k] / 2.0 * (j + 1)) * (1.0 + v->downspreading / 100.0);
+					v->min_dispclk_using_single_dpp = dcn_bw_max2(v->pixel_clock[k], v->min_dppclk_using_single_dpp[k] * (j + 1)) * (1.0f + v->downspreading / 100.0f);
+					v->min_dispclk_using_dual_dpp = dcn_bw_max2(v->pixel_clock[k], v->min_dppclk_using_single_dpp[k] / 2.0f * (j + 1)) * (1.0f + v->downspreading / 100.0f);
 					if (i < number_of_states) {
-						v->min_dispclk_using_single_dpp = v->min_dispclk_using_single_dpp * (1.0 + v->dispclk_ramping_margin / 100.0);
-						v->min_dispclk_using_dual_dpp = v->min_dispclk_using_dual_dpp * (1.0 + v->dispclk_ramping_margin / 100.0);
+						v->min_dispclk_using_single_dpp = v->min_dispclk_using_single_dpp * (1.0f + v->dispclk_ramping_margin / 100.0f);
+						v->min_dispclk_using_dual_dpp = v->min_dispclk_using_dual_dpp * (1.0f + v->dispclk_ramping_margin / 100.0f);
 					}
-					if (v->number_of_dpp_required_for_det_and_lb_size[k] <= 1.0) {
-						v->no_of_dpp[i][j][k] = 1.0;
-						v->required_dispclk[i][j] =dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_single_dpp);
-						if (v->min_dispclk_using_single_dpp >dcn_bw_min2(v->max_dispclk[i], (j + 1) * v->max_dppclk[i])) {
+					if (v->number_of_dpp_required_for_det_and_lb_size[k] <= 1.0f) {
+						v->no_of_dpp[i][j][k] = 1.0f;
+						v->required_dispclk[i][j] = dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_single_dpp);
+						if (v->min_dispclk_using_single_dpp > dcn_bw_min2(v->max_dispclk[i], (j + 1) * v->max_dppclk[i])) {
 							v->dispclk_dppclk_support[i][j] = dcn_bw_no;
 						}
-					}
-					else {
-						v->no_of_dpp[i][j][k] = 2.0;
-						v->required_dispclk[i][j] =dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_dual_dpp);
-						if (v->min_dispclk_using_dual_dpp >dcn_bw_min2(v->max_dispclk[i], (j + 1) * v->max_dppclk[i])) {
+					} else {
+						v->no_of_dpp[i][j][k] = 2.0f;
+						v->required_dispclk[i][j] = dcn_bw_max2(v->required_dispclk[i][j], v->min_dispclk_using_dual_dpp);
+						if (v->min_dispclk_using_dual_dpp > dcn_bw_min2(v->max_dispclk[i], (j + 1) * v->max_dppclk[i])) {
 							v->dispclk_dppclk_support[i][j] = dcn_bw_no;
 						}
 					}
@@ -497,7 +446,7 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 
 	v->viewport_size_support = dcn_bw_yes;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->number_of_dpp_required_for_det_and_lb_size[k] > 2.0) {
+		if (v->number_of_dpp_required_for_det_and_lb_size[k] > 2.0f) {
 			v->viewport_size_support = dcn_bw_no;
 		}
 	}
@@ -507,8 +456,7 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 		for (j = 0; j <= 1; j++) {
 			if (v->total_number_of_active_dpp[i][j] <= v->max_num_dpp) {
 				v->total_available_pipes_support[i][j] = dcn_bw_yes;
-			}
-			else {
+			} else {
 				v->total_available_pipes_support[i][j] = dcn_bw_no;
 			}
 		}
@@ -519,47 +467,43 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 		for (i = 0; i <= number_of_states_plus_one; i++) {
 			for (j = 0; j <= 1; j++) {
 				v->swath_width_yper_state[i][j][k] = v->swath_width_ysingle_dpp[k] / v->no_of_dpp[i][j][k];
-				v->swath_width_granularity_y = 256.0 /dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / v->max_swath_height_y[k];
-				v->rounded_up_max_swath_size_bytes_y = (dcn_bw_ceil2(v->swath_width_yper_state[i][j][k] - 1.0, v->swath_width_granularity_y) + v->swath_width_granularity_y) * v->byte_per_pixel_in_dety[k] * v->max_swath_height_y[k];
+				v->swath_width_granularity_y = 256.0f / dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / v->max_swath_height_y[k];
+				v->rounded_up_max_swath_size_bytes_y = (dcn_bw_ceil2(v->swath_width_yper_state[i][j][k] - 1.0f, v->swath_width_granularity_y) + v->swath_width_granularity_y) * v->byte_per_pixel_in_dety[k] * v->max_swath_height_y[k];
 				if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) {
-					v->rounded_up_max_swath_size_bytes_y =dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_y, 256.0) + 256;
+					v->rounded_up_max_swath_size_bytes_y = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_y, 256.0f) + 256;
 				}
-				if (v->max_swath_height_c[k] > 0.0) {
-					v->swath_width_granularity_c = 256.0 /dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / v->max_swath_height_c[k];
-					v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width_yper_state[i][j][k] / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pixel_in_detc[k] * v->max_swath_height_c[k];
+				if (v->max_swath_height_c[k] > 0.0f) {
+					v->swath_width_granularity_c = 256.0f / dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / v->max_swath_height_c[k];
+					v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width_yper_state[i][j][k] / 2.0f - 1.0f, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pixel_in_detc[k] * v->max_swath_height_c[k];
 					if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) {
-						v->rounded_up_max_swath_size_bytes_c = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256;
+						v->rounded_up_max_swath_size_bytes_c = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0f) + 256;
 					}
 				}
-				if (v->rounded_up_max_swath_size_bytes_y + v->rounded_up_max_swath_size_bytes_c <= v->det_buffer_size_in_kbyte * 1024.0 / 2.0) {
+				if (v->rounded_up_max_swath_size_bytes_y + v->rounded_up_max_swath_size_bytes_c <= v->det_buffer_size_in_kbyte * 1024.0f / 2.0f) {
 					v->swath_height_yper_state[i][j][k] = v->max_swath_height_y[k];
 					v->swath_height_cper_state[i][j][k] = v->max_swath_height_c[k];
-				}
-				else {
+				} else {
 					v->swath_height_yper_state[i][j][k] = v->min_swath_height_y[k];
 					v->swath_height_cper_state[i][j][k] = v->min_swath_height_c[k];
 				}
-				if (v->byte_per_pixel_in_detc[k] == 0.0) {
-					v->lines_in_det_luma = v->det_buffer_size_in_kbyte * 1024.0 / v->byte_per_pixel_in_dety[k] / v->swath_width_yper_state[i][j][k];
-					v->lines_in_det_chroma = 0.0;
-				}
-				else if (v->swath_height_yper_state[i][j][k] <= v->swath_height_cper_state[i][j][k]) {
-					v->lines_in_det_luma = v->det_buffer_size_in_kbyte * 1024.0 / 2.0 / v->byte_per_pixel_in_dety[k] / v->swath_width_yper_state[i][j][k];
-					v->lines_in_det_chroma = v->det_buffer_size_in_kbyte * 1024.0 / 2.0 / v->byte_per_pixel_in_detc[k] / (v->swath_width_yper_state[i][j][k] / 2.0);
-				}
-				else {
-					v->lines_in_det_luma = v->det_buffer_size_in_kbyte * 1024.0 * 2.0 / 3.0 / v->byte_per_pixel_in_dety[k] / v->swath_width_yper_state[i][j][k];
-					v->lines_in_det_chroma = v->det_buffer_size_in_kbyte * 1024.0 / 3.0 / v->byte_per_pixel_in_dety[k] / (v->swath_width_yper_state[i][j][k] / 2.0);
-				}
-				v->effective_lb_latency_hiding_source_lines_luma =dcn_bw_min2(v->max_line_buffer_lines,dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] /dcn_bw_max2(v->h_ratio[k], 1.0)), 1.0)) - (v->vtaps[k] - 1.0);
-				v->effective_detlb_lines_luma =dcn_bw_floor2(v->lines_in_det_luma +dcn_bw_min2(v->lines_in_det_luma * v->required_dispclk[i][j] * v->byte_per_pixel_in_dety[k] * v->pscl_factor[k] / v->return_bw_per_state[i], v->effective_lb_latency_hiding_source_lines_luma), v->swath_height_yper_state[i][j][k]);
-				if (v->byte_per_pixel_in_detc[k] == 0.0) {
-					v->urgent_latency_support_us_per_state[i][j][k] = v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]);
-				}
-				else {
-					v->effective_lb_latency_hiding_source_lines_chroma = dcn_bw_min2(v->max_line_buffer_lines, dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] / 2.0 / dcn_bw_max2(v->h_ratio[k] / 2.0, 1.0)), 1.0)) - (v->vta_pschroma[k] - 1.0);
+				if (v->byte_per_pixel_in_detc[k] == 0.0f) {
+					v->lines_in_det_luma = v->det_buffer_size_in_kbyte * 1024.0f / v->byte_per_pixel_in_dety[k] / v->swath_width_yper_state[i][j][k];
+					v->lines_in_det_chroma = 0.0f;
+				} else if (v->swath_height_yper_state[i][j][k] <= v->swath_height_cper_state[i][j][k]) {
+					v->lines_in_det_luma = v->det_buffer_size_in_kbyte * 1024.0f / 2.0f / v->byte_per_pixel_in_dety[k] / v->swath_width_yper_state[i][j][k];
+					v->lines_in_det_chroma = v->det_buffer_size_in_kbyte * 1024.0f / 2.0f / v->byte_per_pixel_in_detc[k] / (v->swath_width_yper_state[i][j][k] / 2.0f);
+				} else {
+					v->lines_in_det_luma = v->det_buffer_size_in_kbyte * 1024.0f * 2.0f / 3.0f / v->byte_per_pixel_in_dety[k] / v->swath_width_yper_state[i][j][k];
+					v->lines_in_det_chroma = v->det_buffer_size_in_kbyte * 1024.0f / 3.0f / v->byte_per_pixel_in_dety[k] / (v->swath_width_yper_state[i][j][k] / 2.0f);
+				}
+				v->effective_lb_latency_hiding_source_lines_luma = dcn_bw_min2(v->max_line_buffer_lines, dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] / dcn_bw_max2(v->h_ratio[k], 1.0f)), 1.0f)) - (v->vtaps[k] - 1.0f);
+				v->effective_detlb_lines_luma = dcn_bw_floor2(v->lines_in_det_luma + dcn_bw_min2(v->lines_in_det_luma * v->required_dispclk[i][j] * v->byte_per_pixel_in_dety[k] * v->pscl_factor[k] / v->return_bw_per_state[i], v->effective_lb_latency_hiding_source_lines_luma), v->swath_height_yper_state[i][j][k]);
+				if (v->byte_per_pixel_in_detc[k] == 0.0f) {
+					v->urgent_latency_support_us_per_state[i][j][k] = v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]);
+				} else {
+					v->effective_lb_latency_hiding_source_lines_chroma = dcn_bw_min2(v->max_line_buffer_lines, dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_yper_state[i][j][k] / 2.0f / dcn_bw_max2(v->h_ratio[k] / 2.0f, 1.0f)), 1.0f)) - (v->vta_pschroma[k] - 1.0f);
 					v->effective_detlb_lines_chroma = dcn_bw_floor2(v->lines_in_det_chroma + dcn_bw_min2(v->lines_in_det_chroma * v->required_dispclk[i][j] * v->byte_per_pixel_in_detc[k] * v->pscl_factor_chroma[k] / v->return_bw_per_state[i], v->effective_lb_latency_hiding_source_lines_chroma), v->swath_height_cper_state[i][j][k]);
-					v->urgent_latency_support_us_per_state[i][j][k] = dcn_bw_min2(v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]), v->effective_detlb_lines_chroma * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0) - v->effective_detlb_lines_chroma * v->swath_width_yper_state[i][j][k] / 2.0 * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]));
+					v->urgent_latency_support_us_per_state[i][j][k] = dcn_bw_min2(v->effective_detlb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_detlb_lines_luma * v->swath_width_yper_state[i][j][k] * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]), v->effective_detlb_lines_chroma * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0f) - v->effective_detlb_lines_chroma * v->swath_width_yper_state[i][j][k] / 2.0f * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / (v->return_bw_per_state[i] / v->no_of_dpp[i][j][k]));
 				}
 			}
 		}
@@ -568,7 +512,7 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 		for (j = 0; j <= 1; j++) {
 			v->urgent_latency_support[i][j] = dcn_bw_yes;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				if (v->urgent_latency_support_us_per_state[i][j][k] < v->urgent_latency / 1.0) {
+				if (v->urgent_latency_support_us_per_state[i][j][k] < v->urgent_latency / 1.0f) {
 					v->urgent_latency_support[i][j] = dcn_bw_no;
 				}
 			}
@@ -578,7 +522,7 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 
 	for (i = 0; i <= number_of_states_plus_one; i++) {
 		for (j = 0; j <= 1; j++) {
-			v->total_number_of_dcc_active_dpp[i][j] = 0.0;
+			v->total_number_of_dcc_active_dpp[i][j] = 0.0f;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 				if (v->dcc_enable[k] == dcn_bw_yes) {
 					v->total_number_of_dcc_active_dpp[i][j] = v->total_number_of_dcc_active_dpp[i][j] + v->no_of_dpp[i][j][k];
@@ -588,228 +532,198 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 	}
 	for (i = 0; i <= number_of_states_plus_one; i++) {
 		for (j = 0; j <= 1; j++) {
-			v->projected_dcfclk_deep_sleep = 8.0;
+			v->projected_dcfclk_deep_sleep = 8.0f;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				v->projected_dcfclk_deep_sleep =dcn_bw_max2(v->projected_dcfclk_deep_sleep, v->pixel_clock[k] / 16.0);
-				if (v->byte_per_pixel_in_detc[k] == 0.0) {
-					if (v->v_ratio[k] <= 1.0) {
-						v->projected_dcfclk_deep_sleep =dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1 *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / 64.0 * v->h_ratio[k] * v->pixel_clock[k] / v->no_of_dpp[i][j][k]);
-					}
-					else {
-						v->projected_dcfclk_deep_sleep =dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1 *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / 64.0 * v->pscl_factor[k] * v->required_dispclk[i][j] / (1 + j));
-					}
-				}
-				else {
-					if (v->v_ratio[k] <= 1.0) {
-						v->projected_dcfclk_deep_sleep =dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1 *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / 32.0 * v->h_ratio[k] * v->pixel_clock[k] / v->no_of_dpp[i][j][k]);
-					}
-					else {
-						v->projected_dcfclk_deep_sleep =dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1 *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / 32.0 * v->pscl_factor[k] * v->required_dispclk[i][j] / (1 + j));
+				v->projected_dcfclk_deep_sleep = dcn_bw_max2(v->projected_dcfclk_deep_sleep, v->pixel_clock[k] / 16.0f);
+				if (v->byte_per_pixel_in_detc[k] == 0.0f) {
+					if (v->v_ratio[k] <= 1.0f) {
+						v->projected_dcfclk_deep_sleep = dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1f * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / 64.0f * v->h_ratio[k] * v->pixel_clock[k] / v->no_of_dpp[i][j][k]);
+					} else {
+						v->projected_dcfclk_deep_sleep = dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1f * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / 64.0f * v->pscl_factor[k] * v->required_dispclk[i][j] / (1 + j));
 					}
-					if (v->v_ratio[k] / 2.0 <= 1.0) {
-						v->projected_dcfclk_deep_sleep =dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1 *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / 32.0 * v->h_ratio[k] / 2.0 * v->pixel_clock[k] / v->no_of_dpp[i][j][k]);
+				} else {
+					if (v->v_ratio[k] <= 1.0f) {
+						v->projected_dcfclk_deep_sleep = dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1f * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / 32.0f * v->h_ratio[k] * v->pixel_clock[k] / v->no_of_dpp[i][j][k]);
+					} else {
+						v->projected_dcfclk_deep_sleep = dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1f * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / 32.0f * v->pscl_factor[k] * v->required_dispclk[i][j] / (1 + j));
 					}
-					else {
-						v->projected_dcfclk_deep_sleep =dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1 *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / 32.0 * v->pscl_factor_chroma[k] * v->required_dispclk[i][j] / (1 + j));
+					if (v->v_ratio[k] / 2.0f <= 1.0f) {
+						v->projected_dcfclk_deep_sleep = dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1f * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / 32.0f * v->h_ratio[k] / 2.0f * v->pixel_clock[k] / v->no_of_dpp[i][j][k]);
+					} else {
+						v->projected_dcfclk_deep_sleep = dcn_bw_max2(v->projected_dcfclk_deep_sleep, 1.1f * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / 32.0f * v->pscl_factor_chroma[k] * v->required_dispclk[i][j] / (1 + j));
 					}
 				}
 			}
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 				if (v->dcc_enable[k] == dcn_bw_yes) {
-					v->meta_req_height_y = 8.0 * v->read256_block_height_y[k];
-					v->meta_req_width_y = 64.0 * 256.0 /dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / v->meta_req_height_y;
-					v->meta_surface_width_y =dcn_bw_ceil2(v->viewport_width[k] / v->no_of_dpp[i][j][k] - 1.0, v->meta_req_width_y) + v->meta_req_width_y;
-					v->meta_surface_height_y =dcn_bw_ceil2(v->viewport_height[k] - 1.0, v->meta_req_height_y) + v->meta_req_height_y;
+					v->meta_req_height_y = 8.0f * v->read256_block_height_y[k];
+					v->meta_req_width_y = 64.0f * 256.0f / dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / v->meta_req_height_y;
+					v->meta_surface_width_y = dcn_bw_ceil2(v->viewport_width[k] / v->no_of_dpp[i][j][k] - 1.0f, v->meta_req_width_y) + v->meta_req_width_y;
+					v->meta_surface_height_y = dcn_bw_ceil2(v->viewport_height[k] - 1.0f, v->meta_req_height_y) + v->meta_req_height_y;
 					if (v->pte_enable == dcn_bw_yes) {
-						v->meta_pte_bytes_per_frame_y = (dcn_bw_ceil2((v->meta_surface_width_y * v->meta_surface_height_y *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / 256.0 - 4096.0) / 8.0 / 4096.0, 1.0) + 1) * 64.0;
-					}
-					else {
-						v->meta_pte_bytes_per_frame_y = 0.0;
+						v->meta_pte_bytes_per_frame_y = (dcn_bw_ceil2((v->meta_surface_width_y * v->meta_surface_height_y * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / 256.0f - 4096.0f) / 8.0f / 4096.0f, 1.0f) + 1) * 64.0f;
+					} else {
+						v->meta_pte_bytes_per_frame_y = 0.0f;
 					}
 					if (v->source_scan[k] == dcn_bw_hor) {
-						v->meta_row_bytes_y = v->meta_surface_width_y * v->meta_req_height_y *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / 256.0;
-					}
-					else {
-						v->meta_row_bytes_y = v->meta_surface_height_y * v->meta_req_width_y *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / 256.0;
+						v->meta_row_bytes_y = v->meta_surface_width_y * v->meta_req_height_y * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / 256.0f;
+					} else {
+						v->meta_row_bytes_y = v->meta_surface_height_y * v->meta_req_width_y * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / 256.0f;
 					}
-				}
-				else {
-					v->meta_pte_bytes_per_frame_y = 0.0;
-					v->meta_row_bytes_y = 0.0;
+				} else {
+					v->meta_pte_bytes_per_frame_y = 0.0f;
+					v->meta_row_bytes_y = 0.0f;
 				}
 				if (v->pte_enable == dcn_bw_yes) {
 					if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-						v->macro_tile_block_size_bytes_y = 256.0;
-						v->macro_tile_block_height_y = 1.0;
-					}
-					else if (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d_x) {
-						v->macro_tile_block_size_bytes_y = 4096.0;
-						v->macro_tile_block_height_y = 4.0 * v->read256_block_height_y[k];
-					}
-					else if (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x) {
-						v->macro_tile_block_size_bytes_y = 64.0 * 1024;
-						v->macro_tile_block_height_y = 16.0 * v->read256_block_height_y[k];
-					}
-					else {
-						v->macro_tile_block_size_bytes_y = 256.0 * 1024;
-						v->macro_tile_block_height_y = 32.0 * v->read256_block_height_y[k];
+						v->macro_tile_block_size_bytes_y = 256.0f;
+						v->macro_tile_block_height_y = 1.0f;
+					} else if (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d_x) {
+						v->macro_tile_block_size_bytes_y = 4096.0f;
+						v->macro_tile_block_height_y = 4.0f * v->read256_block_height_y[k];
+					} else if (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x) {
+						v->macro_tile_block_size_bytes_y = 64.0f * 1024;
+						v->macro_tile_block_height_y = 16.0f * v->read256_block_height_y[k];
+					} else {
+						v->macro_tile_block_size_bytes_y = 256.0f * 1024;
+						v->macro_tile_block_height_y = 32.0f * v->read256_block_height_y[k];
 					}
-					if (v->macro_tile_block_size_bytes_y <= 65536.0) {
+					if (v->macro_tile_block_size_bytes_y <= 65536.0f) {
 						v->data_pte_req_height_y = v->macro_tile_block_height_y;
+					} else {
+						v->data_pte_req_height_y = 16.0f * v->read256_block_height_y[k];
 					}
-					else {
-						v->data_pte_req_height_y = 16.0 * v->read256_block_height_y[k];
-					}
-					v->data_pte_req_width_y = 4096.0 /dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) / v->data_pte_req_height_y * 8;
+					v->data_pte_req_width_y = 4096.0f / dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) / v->data_pte_req_height_y * 8;
 					if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-						v->dpte_bytes_per_row_y = 64.0 * (dcn_bw_ceil2((v->viewport_width[k] / v->no_of_dpp[i][j][k] *dcn_bw_min2(128.0, dcn_bw_pow(2.0,dcn_bw_floor2(dcn_bw_log(v->pte_buffer_size_in_requests * v->data_pte_req_width_y / (v->viewport_width[k] / v->no_of_dpp[i][j][k]), 2.0), 1.0))) - 1.0) / v->data_pte_req_width_y, 1.0) + 1);
-					}
-					else if (v->source_scan[k] == dcn_bw_hor) {
-						v->dpte_bytes_per_row_y = 64.0 * (dcn_bw_ceil2((v->viewport_width[k] / v->no_of_dpp[i][j][k] - 1.0) / v->data_pte_req_width_y, 1.0) + 1);
-					}
-					else {
-						v->dpte_bytes_per_row_y = 64.0 * (dcn_bw_ceil2((v->viewport_height[k] - 1.0) / v->data_pte_req_height_y, 1.0) + 1);
+						v->dpte_bytes_per_row_y = 64.0f * (dcn_bw_ceil2((v->viewport_width[k] / v->no_of_dpp[i][j][k] * dcn_bw_min2(128.0f, dcn_bw_pow(2.0f, dcn_bw_floor2(dcn_bw_log(v->pte_buffer_size_in_requests * v->data_pte_req_width_y / (v->viewport_width[k] / v->no_of_dpp[i][j][k]), 2.0f), 1.0f))) - 1.0f) / v->data_pte_req_width_y, 1.0f) + 1);
+					} else if (v->source_scan[k] == dcn_bw_hor) {
+						v->dpte_bytes_per_row_y = 64.0f * (dcn_bw_ceil2((v->viewport_width[k] / v->no_of_dpp[i][j][k] - 1.0f) / v->data_pte_req_width_y, 1.0f) + 1);
+					} else {
+						v->dpte_bytes_per_row_y = 64.0f * (dcn_bw_ceil2((v->viewport_height[k] - 1.0f) / v->data_pte_req_height_y, 1.0f) + 1);
 					}
-				}
-				else {
-					v->dpte_bytes_per_row_y = 0.0;
+				} else {
+					v->dpte_bytes_per_row_y = 0.0f;
 				}
 				if ((v->source_pixel_format[k] != dcn_bw_rgb_sub_64 && v->source_pixel_format[k] != dcn_bw_rgb_sub_32 && v->source_pixel_format[k] != dcn_bw_rgb_sub_16)) {
 					if (v->dcc_enable[k] == dcn_bw_yes) {
-						v->meta_req_height_c = 8.0 * v->read256_block_height_c[k];
-						v->meta_req_width_c = 64.0 * 256.0 /dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / v->meta_req_height_c;
-						v->meta_surface_width_c =dcn_bw_ceil2(v->viewport_width[k] / v->no_of_dpp[i][j][k] / 2.0 - 1.0, v->meta_req_width_c) + v->meta_req_width_c;
-						v->meta_surface_height_c =dcn_bw_ceil2(v->viewport_height[k] / 2.0 - 1.0, v->meta_req_height_c) + v->meta_req_height_c;
+						v->meta_req_height_c = 8.0f * v->read256_block_height_c[k];
+						v->meta_req_width_c = 64.0f * 256.0f / dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / v->meta_req_height_c;
+						v->meta_surface_width_c = dcn_bw_ceil2(v->viewport_width[k] / v->no_of_dpp[i][j][k] / 2.0f - 1.0f, v->meta_req_width_c) + v->meta_req_width_c;
+						v->meta_surface_height_c = dcn_bw_ceil2(v->viewport_height[k] / 2.0f - 1.0f, v->meta_req_height_c) + v->meta_req_height_c;
 						if (v->pte_enable == dcn_bw_yes) {
-							v->meta_pte_bytes_per_frame_c = (dcn_bw_ceil2((v->meta_surface_width_c * v->meta_surface_height_c *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / 256.0 - 4096.0) / 8.0 / 4096.0, 1.0) + 1) * 64.0;
-						}
-						else {
-							v->meta_pte_bytes_per_frame_c = 0.0;
+							v->meta_pte_bytes_per_frame_c = (dcn_bw_ceil2((v->meta_surface_width_c * v->meta_surface_height_c * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / 256.0f - 4096.0f) / 8.0f / 4096.0f, 1.0f) + 1) * 64.0f;
+						} else {
+							v->meta_pte_bytes_per_frame_c = 0.0f;
 						}
 						if (v->source_scan[k] == dcn_bw_hor) {
-							v->meta_row_bytes_c = v->meta_surface_width_c * v->meta_req_height_c *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / 256.0;
-						}
-						else {
-							v->meta_row_bytes_c = v->meta_surface_height_c * v->meta_req_width_c *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / 256.0;
+							v->meta_row_bytes_c = v->meta_surface_width_c * v->meta_req_height_c * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / 256.0f;
+						} else {
+							v->meta_row_bytes_c = v->meta_surface_height_c * v->meta_req_width_c * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / 256.0f;
 						}
-					}
-					else {
-						v->meta_pte_bytes_per_frame_c = 0.0;
-						v->meta_row_bytes_c = 0.0;
+					} else {
+						v->meta_pte_bytes_per_frame_c = 0.0f;
+						v->meta_row_bytes_c = 0.0f;
 					}
 					if (v->pte_enable == dcn_bw_yes) {
 						if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-							v->macro_tile_block_size_bytes_c = 256.0;
-							v->macro_tile_block_height_c = 1.0;
-						}
-						else if (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d_x) {
-							v->macro_tile_block_size_bytes_c = 4096.0;
-							v->macro_tile_block_height_c = 4.0 * v->read256_block_height_c[k];
+							v->macro_tile_block_size_bytes_c = 256.0f;
+							v->macro_tile_block_height_c = 1.0f;
+						} else if (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d_x) {
+							v->macro_tile_block_size_bytes_c = 4096.0f;
+							v->macro_tile_block_height_c = 4.0f * v->read256_block_height_c[k];
+						} else if (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x) {
+							v->macro_tile_block_size_bytes_c = 64.0f * 1024;
+							v->macro_tile_block_height_c = 16.0f * v->read256_block_height_c[k];
+						} else {
+							v->macro_tile_block_size_bytes_c = 256.0f * 1024;
+							v->macro_tile_block_height_c = 32.0f * v->read256_block_height_c[k];
 						}
-						else if (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x) {
-							v->macro_tile_block_size_bytes_c = 64.0 * 1024;
-							v->macro_tile_block_height_c = 16.0 * v->read256_block_height_c[k];
-						}
-						else {
-							v->macro_tile_block_size_bytes_c = 256.0 * 1024;
-							v->macro_tile_block_height_c = 32.0 * v->read256_block_height_c[k];
-						}
-						v->macro_tile_block_width_c = v->macro_tile_block_size_bytes_c /dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / v->macro_tile_block_height_c;
-						if (v->macro_tile_block_size_bytes_c <= 65536.0) {
+						v->macro_tile_block_width_c = v->macro_tile_block_size_bytes_c / dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / v->macro_tile_block_height_c;
+						if (v->macro_tile_block_size_bytes_c <= 65536.0f) {
 							v->data_pte_req_height_c = v->macro_tile_block_height_c;
+						} else {
+							v->data_pte_req_height_c = 16.0f * v->read256_block_height_c[k];
 						}
-						else {
-							v->data_pte_req_height_c = 16.0 * v->read256_block_height_c[k];
-						}
-						v->data_pte_req_width_c = 4096.0 /dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / v->data_pte_req_height_c * 8;
+						v->data_pte_req_width_c = 4096.0f / dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / v->data_pte_req_height_c * 8;
 						if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-							v->dpte_bytes_per_row_c = 64.0 * (dcn_bw_ceil2((v->viewport_width[k] / v->no_of_dpp[i][j][k] / 2.0 * dcn_bw_min2(128.0, dcn_bw_pow(2.0,dcn_bw_floor2(dcn_bw_log(v->pte_buffer_size_in_requests * v->data_pte_req_width_c / (v->viewport_width[k] / v->no_of_dpp[i][j][k] / 2.0), 2.0), 1.0))) - 1.0) / v->data_pte_req_width_c, 1.0) + 1);
-						}
-						else if (v->source_scan[k] == dcn_bw_hor) {
-							v->dpte_bytes_per_row_c = 64.0 * (dcn_bw_ceil2((v->viewport_width[k] / v->no_of_dpp[i][j][k] / 2.0 - 1.0) / v->data_pte_req_width_c, 1.0) + 1);
+							v->dpte_bytes_per_row_c = 64.0f * (dcn_bw_ceil2((v->viewport_width[k] / v->no_of_dpp[i][j][k] / 2.0f * dcn_bw_min2(128.0f, dcn_bw_pow(2.0f, dcn_bw_floor2(dcn_bw_log(v->pte_buffer_size_in_requests * v->data_pte_req_width_c / (v->viewport_width[k] / v->no_of_dpp[i][j][k] / 2.0f), 2.0f), 1.0f))) - 1.0f) / v->data_pte_req_width_c, 1.0f) + 1);
+						} else if (v->source_scan[k] == dcn_bw_hor) {
+							v->dpte_bytes_per_row_c = 64.0f * (dcn_bw_ceil2((v->viewport_width[k] / v->no_of_dpp[i][j][k] / 2.0f - 1.0f) / v->data_pte_req_width_c, 1.0f) + 1);
+						} else {
+							v->dpte_bytes_per_row_c = 64.0f * (dcn_bw_ceil2((v->viewport_height[k] / 2.0f - 1.0f) / v->data_pte_req_height_c, 1.0f) + 1);
 						}
-						else {
-							v->dpte_bytes_per_row_c = 64.0 * (dcn_bw_ceil2((v->viewport_height[k] / 2.0 - 1.0) / v->data_pte_req_height_c, 1.0) + 1);
-						}
-					}
-					else {
-						v->dpte_bytes_per_row_c = 0.0;
+					} else {
+						v->dpte_bytes_per_row_c = 0.0f;
 					}
-				}
-				else {
-					v->dpte_bytes_per_row_c = 0.0;
-					v->meta_pte_bytes_per_frame_c = 0.0;
-					v->meta_row_bytes_c = 0.0;
+				} else {
+					v->dpte_bytes_per_row_c = 0.0f;
+					v->meta_pte_bytes_per_frame_c = 0.0f;
+					v->meta_row_bytes_c = 0.0f;
 				}
 				v->dpte_bytes_per_row[k] = v->dpte_bytes_per_row_y + v->dpte_bytes_per_row_c;
 				v->meta_pte_bytes_per_frame[k] = v->meta_pte_bytes_per_frame_y + v->meta_pte_bytes_per_frame_c;
 				v->meta_row_bytes[k] = v->meta_row_bytes_y + v->meta_row_bytes_c;
-				v->v_init_y = (v->v_ratio[k] + v->vtaps[k] + 1.0 + v->interlace_output[k] * 0.5 * v->v_ratio[k]) / 2.0;
-				v->prefill_y[k] =dcn_bw_floor2(v->v_init_y, 1.0);
-				v->max_num_sw_y[k] =dcn_bw_ceil2((v->prefill_y[k] - 1.0) / v->swath_height_yper_state[i][j][k], 1.0) + 1;
-				if (v->prefill_y[k] > 1.0) {
-					v->max_partial_sw_y =dcn_bw_mod((v->prefill_y[k] - 2.0), v->swath_height_yper_state[i][j][k]);
-				}
-				else {
-					v->max_partial_sw_y =dcn_bw_mod((v->prefill_y[k] + v->swath_height_yper_state[i][j][k] - 2.0), v->swath_height_yper_state[i][j][k]);
-				}
-				v->max_partial_sw_y =dcn_bw_max2(1.0, v->max_partial_sw_y);
+				v->v_init_y = (v->v_ratio[k] + v->vtaps[k] + 1.0f + v->interlace_output[k] * 0.5f * v->v_ratio[k]) / 2.0f;
+				v->prefill_y[k] = dcn_bw_floor2(v->v_init_y, 1.0f);
+				v->max_num_sw_y[k] = dcn_bw_ceil2((v->prefill_y[k] - 1.0f) / v->swath_height_yper_state[i][j][k], 1.0f) + 1;
+				if (v->prefill_y[k] > 1.0f) {
+					v->max_partial_sw_y = dcn_bw_mod((v->prefill_y[k] - 2.0f), v->swath_height_yper_state[i][j][k]);
+				} else {
+					v->max_partial_sw_y = dcn_bw_mod((v->prefill_y[k] + v->swath_height_yper_state[i][j][k] - 2.0f), v->swath_height_yper_state[i][j][k]);
+				}
+				v->max_partial_sw_y = dcn_bw_max2(1.0f, v->max_partial_sw_y);
 				v->prefetch_lines_y[k] = v->max_num_sw_y[k] * v->swath_height_yper_state[i][j][k] + v->max_partial_sw_y;
 				if ((v->source_pixel_format[k] != dcn_bw_rgb_sub_64 && v->source_pixel_format[k] != dcn_bw_rgb_sub_32 && v->source_pixel_format[k] != dcn_bw_rgb_sub_16)) {
-					v->v_init_c = (v->v_ratio[k] / 2.0 + v->vtaps[k] + 1.0 + v->interlace_output[k] * 0.5 * v->v_ratio[k] / 2.0) / 2.0;
-					v->prefill_c[k] =dcn_bw_floor2(v->v_init_c, 1.0);
-					v->max_num_sw_c[k] =dcn_bw_ceil2((v->prefill_c[k] - 1.0) / v->swath_height_cper_state[i][j][k], 1.0) + 1;
-					if (v->prefill_c[k] > 1.0) {
-						v->max_partial_sw_c =dcn_bw_mod((v->prefill_c[k] - 2.0), v->swath_height_cper_state[i][j][k]);
-					}
-					else {
-						v->max_partial_sw_c =dcn_bw_mod((v->prefill_c[k] + v->swath_height_cper_state[i][j][k] - 2.0), v->swath_height_cper_state[i][j][k]);
+					v->v_init_c = (v->v_ratio[k] / 2.0f + v->vtaps[k] + 1.0f + v->interlace_output[k] * 0.5f * v->v_ratio[k] / 2.0f) / 2.0f;
+					v->prefill_c[k] = dcn_bw_floor2(v->v_init_c, 1.0f);
+					v->max_num_sw_c[k] = dcn_bw_ceil2((v->prefill_c[k] - 1.0f) / v->swath_height_cper_state[i][j][k], 1.0f) + 1;
+					if (v->prefill_c[k] > 1.0f) {
+						v->max_partial_sw_c = dcn_bw_mod((v->prefill_c[k] - 2.0f), v->swath_height_cper_state[i][j][k]);
+					} else {
+						v->max_partial_sw_c = dcn_bw_mod((v->prefill_c[k] + v->swath_height_cper_state[i][j][k] - 2.0f), v->swath_height_cper_state[i][j][k]);
 					}
-					v->max_partial_sw_c =dcn_bw_max2(1.0, v->max_partial_sw_c);
+					v->max_partial_sw_c = dcn_bw_max2(1.0f, v->max_partial_sw_c);
 					v->prefetch_lines_c[k] = v->max_num_sw_c[k] * v->swath_height_cper_state[i][j][k] + v->max_partial_sw_c;
+				} else {
+					v->prefetch_lines_c[k] = 0.0f;
 				}
-				else {
-					v->prefetch_lines_c[k] = 0.0;
-				}
-				v->dst_x_after_scaler = 90.0 * v->pixel_clock[k] / (v->required_dispclk[i][j] / (j + 1)) + 42.0 * v->pixel_clock[k] / v->required_dispclk[i][j];
-				if (v->no_of_dpp[i][j][k] > 1.0) {
-					v->dst_x_after_scaler = v->dst_x_after_scaler + v->scaler_rec_out_width[k] / 2.0;
+				v->dst_x_after_scaler = 90.0f * v->pixel_clock[k] / (v->required_dispclk[i][j] / (j + 1)) + 42.0f * v->pixel_clock[k] / v->required_dispclk[i][j];
+				if (v->no_of_dpp[i][j][k] > 1.0f) {
+					v->dst_x_after_scaler = v->dst_x_after_scaler + v->scaler_rec_out_width[k] / 2.0f;
 				}
 				if (v->output_format[k] == dcn_bw_420) {
-					v->dst_y_after_scaler = 1.0;
-				}
-				else {
-					v->dst_y_after_scaler = 0.0;
-				}
-				v->time_calc = 24.0 / v->projected_dcfclk_deep_sleep;
-				v->v_update_offset[k][j] = dcn_bw_ceil2(v->htotal[k] / 4.0, 1.0);
-				v->total_repeater_delay = v->max_inter_dcn_tile_repeaters * (2.0 / (v->required_dispclk[i][j] / (j + 1)) + 3.0 / v->required_dispclk[i][j]);
-				v->v_update_width[k][j] = (14.0 / v->projected_dcfclk_deep_sleep + 12.0 / (v->required_dispclk[i][j] / (j + 1)) + v->total_repeater_delay) * v->pixel_clock[k];
-				v->v_ready_offset[k][j] = dcn_bw_max2(150.0 / (v->required_dispclk[i][j] / (j + 1)), v->total_repeater_delay + 20.0 / v->projected_dcfclk_deep_sleep + 10.0 / (v->required_dispclk[i][j] / (j + 1))) * v->pixel_clock[k];
+					v->dst_y_after_scaler = 1.0f;
+				} else {
+					v->dst_y_after_scaler = 0.0f;
+				}
+				v->time_calc = 24.0f / v->projected_dcfclk_deep_sleep;
+				v->v_update_offset[k][j] = dcn_bw_ceil2(v->htotal[k] / 4.0f, 1.0f);
+				v->total_repeater_delay = v->max_inter_dcn_tile_repeaters * (2.0f / (v->required_dispclk[i][j] / (j + 1)) + 3.0f / v->required_dispclk[i][j]);
+				v->v_update_width[k][j] = (14.0f / v->projected_dcfclk_deep_sleep + 12.0f / (v->required_dispclk[i][j] / (j + 1)) + v->total_repeater_delay) * v->pixel_clock[k];
+				v->v_ready_offset[k][j] = dcn_bw_max2(150.0f / (v->required_dispclk[i][j] / (j + 1)), v->total_repeater_delay + 20.0f / v->projected_dcfclk_deep_sleep + 10.0f / (v->required_dispclk[i][j] / (j + 1))) * v->pixel_clock[k];
 				v->time_setup = (v->v_update_offset[k][j] + v->v_update_width[k][j] + v->v_ready_offset[k][j]) / v->pixel_clock[k];
-				v->extra_latency = v->urgent_round_trip_and_out_of_order_latency_per_state[i] + (v->total_number_of_active_dpp[i][j] * v->pixel_chunk_size_in_kbyte + v->total_number_of_dcc_active_dpp[i][j] * v->meta_chunk_size) * 1024.0 / v->return_bw_per_state[i];
+				v->extra_latency = v->urgent_round_trip_and_out_of_order_latency_per_state[i] + (v->total_number_of_active_dpp[i][j] * v->pixel_chunk_size_in_kbyte + v->total_number_of_dcc_active_dpp[i][j] * v->meta_chunk_size) * 1024.0f / v->return_bw_per_state[i];
 				if (v->pte_enable == dcn_bw_yes) {
-					v->extra_latency = v->extra_latency + v->total_number_of_active_dpp[i][j] * v->pte_chunk_size * 1024.0 / v->return_bw_per_state[i];
+					v->extra_latency = v->extra_latency + v->total_number_of_active_dpp[i][j] * v->pte_chunk_size * 1024.0f / v->return_bw_per_state[i];
 				}
 				if (v->can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one == dcn_bw_yes) {
-					v->maximum_vstartup = v->vtotal[k] - v->vactive[k] - 1.0;
-				}
-				else {
-					v->maximum_vstartup = v->v_sync_plus_back_porch[k] - 1.0;
+					v->maximum_vstartup = v->vtotal[k] - v->vactive[k] - 1.0f;
+				} else {
+					v->maximum_vstartup = v->v_sync_plus_back_porch[k] - 1.0f;
 				}
 
 				do {
 					v->line_times_for_prefetch[k] = v->maximum_vstartup - v->urgent_latency / (v->htotal[k] / v->pixel_clock[k]) - (v->time_calc + v->time_setup) / (v->htotal[k] / v->pixel_clock[k]) - (v->dst_y_after_scaler + v->dst_x_after_scaler / v->htotal[k]);
-					v->line_times_for_prefetch[k] =dcn_bw_floor2(4.0 * (v->line_times_for_prefetch[k] + 0.125), 1.0) / 4;
-					v->prefetch_bw[k] = (v->meta_pte_bytes_per_frame[k] + 2.0 * v->meta_row_bytes[k] + 2.0 * v->dpte_bytes_per_row[k] + v->prefetch_lines_y[k] * v->swath_width_yper_state[i][j][k] *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) + v->prefetch_lines_c[k] * v->swath_width_yper_state[i][j][k] / 2.0 *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0)) / (v->line_times_for_prefetch[k] * v->htotal[k] / v->pixel_clock[k]);
+					v->line_times_for_prefetch[k] = dcn_bw_floor2(4.0f * (v->line_times_for_prefetch[k] + 0.125f), 1.0f) / 4;
+					v->prefetch_bw[k] = (v->meta_pte_bytes_per_frame[k] + 2.0f * v->meta_row_bytes[k] + 2.0f * v->dpte_bytes_per_row[k] + v->prefetch_lines_y[k] * v->swath_width_yper_state[i][j][k] * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) + v->prefetch_lines_c[k] * v->swath_width_yper_state[i][j][k] / 2.0f * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f)) / (v->line_times_for_prefetch[k] * v->htotal[k] / v->pixel_clock[k]);
 
 					if (v->pte_enable == dcn_bw_yes && v->dcc_enable[k] == dcn_bw_yes) {
 						v->time_for_meta_pte_without_immediate_flip = dcn_bw_max3(
 								v->meta_pte_bytes_frame[k] / v->prefetch_bw[k],
 								v->extra_latency,
-								v->htotal[k] / v->pixel_clock[k] / 4.0);
+								v->htotal[k] / v->pixel_clock[k] / 4.0f);
 					} else {
-						v->time_for_meta_pte_without_immediate_flip = v->htotal[k] / v->pixel_clock[k] / 4.0;
+						v->time_for_meta_pte_without_immediate_flip = v->htotal[k] / v->pixel_clock[k] / 4.0f;
 					}
 
 					if (v->pte_enable == dcn_bw_yes || v->dcc_enable[k] == dcn_bw_yes) {
@@ -823,11 +737,11 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 								v->extra_latency - v->time_for_meta_pte_with_immediate_flip);
 					}
 
-					v->lines_for_meta_pte_without_immediate_flip[k] =dcn_bw_floor2(4.0 * (v->time_for_meta_pte_without_immediate_flip / (v->htotal[k] / v->pixel_clock[k]) + 0.125), 1.0) / 4;
-					v->lines_for_meta_and_dpte_row_without_immediate_flip[k] =dcn_bw_floor2(4.0 * (v->time_for_meta_and_dpte_row_without_immediate_flip / (v->htotal[k] / v->pixel_clock[k]) + 0.125), 1.0) / 4;
+					v->lines_for_meta_pte_without_immediate_flip[k] = dcn_bw_floor2(4.0f * (v->time_for_meta_pte_without_immediate_flip / (v->htotal[k] / v->pixel_clock[k]) + 0.125f), 1.0f) / 4;
+					v->lines_for_meta_and_dpte_row_without_immediate_flip[k] = dcn_bw_floor2(4.0f * (v->time_for_meta_and_dpte_row_without_immediate_flip / (v->htotal[k] / v->pixel_clock[k]) + 0.125f), 1.0f) / 4;
 					v->maximum_vstartup = v->maximum_vstartup - 1;
 
-					if (v->lines_for_meta_pte_without_immediate_flip[k] < 32.0 && v->lines_for_meta_and_dpte_row_without_immediate_flip[k] < 16.0)
+					if (v->lines_for_meta_pte_without_immediate_flip[k] < 32.0f && v->lines_for_meta_and_dpte_row_without_immediate_flip[k] < 16.0f)
 						break;
 
 				} while(1);
@@ -837,100 +751,91 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 				v->bw_available_for_immediate_flip = v->bw_available_for_immediate_flip -dcn_bw_max2(v->read_bandwidth[k], v->prefetch_bw[k]);
 			}
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				v->total_immediate_flip_bytes[k] = 0.0;
+				v->total_immediate_flip_bytes[k] = 0.0f;
 				if ((v->source_pixel_format[k] != dcn_bw_yuv420_sub_8 && v->source_pixel_format[k] != dcn_bw_yuv420_sub_10)) {
 					v->total_immediate_flip_bytes[k] = v->total_immediate_flip_bytes[k] + v->meta_pte_bytes_per_frame[k] + v->meta_row_bytes[k] + v->dpte_bytes_per_row[k];
 				}
 			}
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 				if (v->pte_enable == dcn_bw_yes && v->dcc_enable[k] == dcn_bw_yes) {
-					v->time_for_meta_pte_with_immediate_flip =dcn_bw_max5(v->meta_pte_bytes_per_frame[k] / v->prefetch_bw[k], v->meta_pte_bytes_per_frame[k] * v->total_immediate_flip_bytes[k] / (v->bw_available_for_immediate_flip * (v->meta_pte_bytes_per_frame[k] + v->meta_row_bytes[k] + v->dpte_bytes_per_row[k])), v->extra_latency, v->urgent_latency, v->htotal[k] / v->pixel_clock[k] / 4.0);
-				}
-				else {
-					v->time_for_meta_pte_with_immediate_flip = v->htotal[k] / v->pixel_clock[k] / 4.0;
+					v->time_for_meta_pte_with_immediate_flip = dcn_bw_max5(v->meta_pte_bytes_per_frame[k] / v->prefetch_bw[k], v->meta_pte_bytes_per_frame[k] * v->total_immediate_flip_bytes[k] / (v->bw_available_for_immediate_flip * (v->meta_pte_bytes_per_frame[k] + v->meta_row_bytes[k] + v->dpte_bytes_per_row[k])), v->extra_latency, v->urgent_latency, v->htotal[k] / v->pixel_clock[k] / 4.0f);
+				} else {
+					v->time_for_meta_pte_with_immediate_flip = v->htotal[k] / v->pixel_clock[k] / 4.0f;
 				}
 				if (v->pte_enable == dcn_bw_yes || v->dcc_enable[k] == dcn_bw_yes) {
-					v->time_for_meta_and_dpte_row_with_immediate_flip =dcn_bw_max5((v->meta_row_bytes[k] + v->dpte_bytes_per_row[k]) / v->prefetch_bw[k], (v->meta_row_bytes[k] + v->dpte_bytes_per_row[k]) * v->total_immediate_flip_bytes[k] / (v->bw_available_for_immediate_flip * (v->meta_pte_bytes_per_frame[k] + v->meta_row_bytes[k] + v->dpte_bytes_per_row[k])), v->htotal[k] / v->pixel_clock[k] - v->time_for_meta_pte_with_immediate_flip, v->extra_latency, 2.0 * v->urgent_latency);
+					v->time_for_meta_and_dpte_row_with_immediate_flip = dcn_bw_max5((v->meta_row_bytes[k] + v->dpte_bytes_per_row[k]) / v->prefetch_bw[k], (v->meta_row_bytes[k] + v->dpte_bytes_per_row[k]) * v->total_immediate_flip_bytes[k] / (v->bw_available_for_immediate_flip * (v->meta_pte_bytes_per_frame[k] + v->meta_row_bytes[k] + v->dpte_bytes_per_row[k])), v->htotal[k] / v->pixel_clock[k] - v->time_for_meta_pte_with_immediate_flip, v->extra_latency, 2.0f * v->urgent_latency);
+				} else {
+					v->time_for_meta_and_dpte_row_with_immediate_flip = dcn_bw_max2(v->htotal[k] / v->pixel_clock[k] - v->time_for_meta_pte_with_immediate_flip, v->extra_latency - v->time_for_meta_pte_with_immediate_flip);
 				}
-				else {
-					v->time_for_meta_and_dpte_row_with_immediate_flip =dcn_bw_max2(v->htotal[k] / v->pixel_clock[k] - v->time_for_meta_pte_with_immediate_flip, v->extra_latency - v->time_for_meta_pte_with_immediate_flip);
-				}
-				v->lines_for_meta_pte_with_immediate_flip[k] =dcn_bw_floor2(4.0 * (v->time_for_meta_pte_with_immediate_flip / (v->htotal[k] / v->pixel_clock[k]) + 0.125), 1.0) / 4;
-				v->lines_for_meta_and_dpte_row_with_immediate_flip[k] =dcn_bw_floor2(4.0 * (v->time_for_meta_and_dpte_row_with_immediate_flip / (v->htotal[k] / v->pixel_clock[k]) + 0.125), 1.0) / 4;
+				v->lines_for_meta_pte_with_immediate_flip[k] = dcn_bw_floor2(4.0f * (v->time_for_meta_pte_with_immediate_flip / (v->htotal[k] / v->pixel_clock[k]) + 0.125f), 1.0f) / 4;
+				v->lines_for_meta_and_dpte_row_with_immediate_flip[k] = dcn_bw_floor2(4.0f * (v->time_for_meta_and_dpte_row_with_immediate_flip / (v->htotal[k] / v->pixel_clock[k]) + 0.125f), 1.0f) / 4;
 				v->line_times_to_request_prefetch_pixel_data_with_immediate_flip = v->line_times_for_prefetch[k] - v->lines_for_meta_pte_with_immediate_flip[k] - v->lines_for_meta_and_dpte_row_with_immediate_flip[k];
 				v->line_times_to_request_prefetch_pixel_data_without_immediate_flip = v->line_times_for_prefetch[k] - v->lines_for_meta_pte_without_immediate_flip[k] - v->lines_for_meta_and_dpte_row_without_immediate_flip[k];
-				if (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip > 0.0) {
+				if (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip > 0.0f) {
 					v->v_ratio_pre_ywith_immediate_flip[i][j][k] = v->prefetch_lines_y[k] / v->line_times_to_request_prefetch_pixel_data_with_immediate_flip;
-					if ((v->swath_height_yper_state[i][j][k] > 4.0)) {
-						if (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip - (v->prefill_y[k] - 3.0) / 2.0 > 0.0) {
-							v->v_ratio_pre_ywith_immediate_flip[i][j][k] =dcn_bw_max2(v->v_ratio_pre_ywith_immediate_flip[i][j][k], (v->max_num_sw_y[k] * v->swath_height_yper_state[i][j][k]) / (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip - (v->prefill_y[k] - 3.0) / 2.0));
-						}
-						else {
-							v->v_ratio_pre_ywith_immediate_flip[i][j][k] = 999999.0;
+					if ((v->swath_height_yper_state[i][j][k] > 4.0f)) {
+						if (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip - (v->prefill_y[k] - 3.0f) / 2.0f > 0.0f) {
+							v->v_ratio_pre_ywith_immediate_flip[i][j][k] = dcn_bw_max2(v->v_ratio_pre_ywith_immediate_flip[i][j][k], (v->max_num_sw_y[k] * v->swath_height_yper_state[i][j][k]) / (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip - (v->prefill_y[k] - 3.0f) / 2.0f));
+						} else {
+							v->v_ratio_pre_ywith_immediate_flip[i][j][k] = 999999.0f;
 						}
 					}
 					v->v_ratio_pre_cwith_immediate_flip[i][j][k] = v->prefetch_lines_c[k] / v->line_times_to_request_prefetch_pixel_data_with_immediate_flip;
-					if ((v->swath_height_cper_state[i][j][k] > 4.0)) {
-						if (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip - (v->prefill_c[k] - 3.0) / 2.0 > 0.0) {
-							v->v_ratio_pre_cwith_immediate_flip[i][j][k] =dcn_bw_max2(v->v_ratio_pre_cwith_immediate_flip[i][j][k], (v->max_num_sw_c[k] * v->swath_height_cper_state[i][j][k]) / (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip - (v->prefill_c[k] - 3.0) / 2.0));
-						}
-						else {
-							v->v_ratio_pre_cwith_immediate_flip[i][j][k] = 999999.0;
+					if ((v->swath_height_cper_state[i][j][k] > 4.0f)) {
+						if (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip - (v->prefill_c[k] - 3.0f) / 2.0f > 0.0f) {
+							v->v_ratio_pre_cwith_immediate_flip[i][j][k] = dcn_bw_max2(v->v_ratio_pre_cwith_immediate_flip[i][j][k], (v->max_num_sw_c[k] * v->swath_height_cper_state[i][j][k]) / (v->line_times_to_request_prefetch_pixel_data_with_immediate_flip - (v->prefill_c[k] - 3.0f) / 2.0f));
+						} else {
+							v->v_ratio_pre_cwith_immediate_flip[i][j][k] = 999999.0f;
 						}
 					}
-					v->required_prefetch_pixel_data_bw_with_immediate_flip[i][j][k] = v->no_of_dpp[i][j][k] * (v->prefetch_lines_y[k] / v->line_times_to_request_prefetch_pixel_data_with_immediate_flip *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) + v->prefetch_lines_c[k] / v->line_times_to_request_prefetch_pixel_data_with_immediate_flip *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / 2.0) * v->swath_width_yper_state[i][j][k] / (v->htotal[k] / v->pixel_clock[k]);
-				}
-				else {
-					v->v_ratio_pre_ywith_immediate_flip[i][j][k] = 999999.0;
-					v->v_ratio_pre_cwith_immediate_flip[i][j][k] = 999999.0;
-					v->required_prefetch_pixel_data_bw_with_immediate_flip[i][j][k] = 999999.0;
+					v->required_prefetch_pixel_data_bw_with_immediate_flip[i][j][k] = v->no_of_dpp[i][j][k] * (v->prefetch_lines_y[k] / v->line_times_to_request_prefetch_pixel_data_with_immediate_flip * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) + v->prefetch_lines_c[k] / v->line_times_to_request_prefetch_pixel_data_with_immediate_flip * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / 2.0f) * v->swath_width_yper_state[i][j][k] / (v->htotal[k] / v->pixel_clock[k]);
+				} else {
+					v->v_ratio_pre_ywith_immediate_flip[i][j][k] = 999999.0f;
+					v->v_ratio_pre_cwith_immediate_flip[i][j][k] = 999999.0f;
+					v->required_prefetch_pixel_data_bw_with_immediate_flip[i][j][k] = 999999.0f;
 				}
-				if (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip > 0.0) {
+				if (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip > 0.0f) {
 					v->v_ratio_pre_ywithout_immediate_flip[i][j][k] = v->prefetch_lines_y[k] / v->line_times_to_request_prefetch_pixel_data_without_immediate_flip;
-					if ((v->swath_height_yper_state[i][j][k] > 4.0)) {
-						if (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip - (v->prefill_y[k] - 3.0) / 2.0 > 0.0) {
-							v->v_ratio_pre_ywithout_immediate_flip[i][j][k] =dcn_bw_max2(v->v_ratio_pre_ywithout_immediate_flip[i][j][k], (v->max_num_sw_y[k] * v->swath_height_yper_state[i][j][k]) / (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip - (v->prefill_y[k] - 3.0) / 2.0));
-						}
-						else {
-							v->v_ratio_pre_ywithout_immediate_flip[i][j][k] = 999999.0;
+					if ((v->swath_height_yper_state[i][j][k] > 4.0f)) {
+						if (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip - (v->prefill_y[k] - 3.0f) / 2.0f > 0.0f) {
+							v->v_ratio_pre_ywithout_immediate_flip[i][j][k] = dcn_bw_max2(v->v_ratio_pre_ywithout_immediate_flip[i][j][k], (v->max_num_sw_y[k] * v->swath_height_yper_state[i][j][k]) / (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip - (v->prefill_y[k] - 3.0f) / 2.0f));
+						} else {
+							v->v_ratio_pre_ywithout_immediate_flip[i][j][k] = 999999.0f;
 						}
 					}
 					v->v_ratio_pre_cwithout_immediate_flip[i][j][k] = v->prefetch_lines_c[k] / v->line_times_to_request_prefetch_pixel_data_without_immediate_flip;
-					if ((v->swath_height_cper_state[i][j][k] > 4.0)) {
-						if (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip - (v->prefill_c[k] - 3.0) / 2.0 > 0.0) {
-							v->v_ratio_pre_cwithout_immediate_flip[i][j][k] =dcn_bw_max2(v->v_ratio_pre_cwithout_immediate_flip[i][j][k], (v->max_num_sw_c[k] * v->swath_height_cper_state[i][j][k]) / (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip - (v->prefill_c[k] - 3.0) / 2.0));
-						}
-						else {
-							v->v_ratio_pre_cwithout_immediate_flip[i][j][k] = 999999.0;
+					if ((v->swath_height_cper_state[i][j][k] > 4.0f)) {
+						if (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip - (v->prefill_c[k] - 3.0f) / 2.0f > 0.0f) {
+							v->v_ratio_pre_cwithout_immediate_flip[i][j][k] = dcn_bw_max2(v->v_ratio_pre_cwithout_immediate_flip[i][j][k], (v->max_num_sw_c[k] * v->swath_height_cper_state[i][j][k]) / (v->line_times_to_request_prefetch_pixel_data_without_immediate_flip - (v->prefill_c[k] - 3.0f) / 2.0f));
+						} else {
+							v->v_ratio_pre_cwithout_immediate_flip[i][j][k] = 999999.0f;
 						}
 					}
-					v->required_prefetch_pixel_data_bw_without_immediate_flip[i][j][k] = v->no_of_dpp[i][j][k] * (v->prefetch_lines_y[k] / v->line_times_to_request_prefetch_pixel_data_without_immediate_flip *dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0) + v->prefetch_lines_c[k] / v->line_times_to_request_prefetch_pixel_data_without_immediate_flip *dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0) / 2.0) * v->swath_width_yper_state[i][j][k] / (v->htotal[k] / v->pixel_clock[k]);
-				}
-				else {
-					v->v_ratio_pre_ywithout_immediate_flip[i][j][k] = 999999.0;
-					v->v_ratio_pre_cwithout_immediate_flip[i][j][k] = 999999.0;
-					v->required_prefetch_pixel_data_bw_without_immediate_flip[i][j][k] = 999999.0;
+					v->required_prefetch_pixel_data_bw_without_immediate_flip[i][j][k] = v->no_of_dpp[i][j][k] * (v->prefetch_lines_y[k] / v->line_times_to_request_prefetch_pixel_data_without_immediate_flip * dcn_bw_ceil2(v->byte_per_pixel_in_dety[k], 1.0f) + v->prefetch_lines_c[k] / v->line_times_to_request_prefetch_pixel_data_without_immediate_flip * dcn_bw_ceil2(v->byte_per_pixel_in_detc[k], 2.0f) / 2.0f) * v->swath_width_yper_state[i][j][k] / (v->htotal[k] / v->pixel_clock[k]);
+				} else {
+					v->v_ratio_pre_ywithout_immediate_flip[i][j][k] = 999999.0f;
+					v->v_ratio_pre_cwithout_immediate_flip[i][j][k] = 999999.0f;
+					v->required_prefetch_pixel_data_bw_without_immediate_flip[i][j][k] = 999999.0f;
 				}
 			}
-			v->maximum_read_bandwidth_with_prefetch_with_immediate_flip = 0.0;
+			v->maximum_read_bandwidth_with_prefetch_with_immediate_flip = 0.0f;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 				if ((v->source_pixel_format[k] != dcn_bw_yuv420_sub_8 && v->source_pixel_format[k] != dcn_bw_yuv420_sub_10)) {
-					v->maximum_read_bandwidth_with_prefetch_with_immediate_flip = v->maximum_read_bandwidth_with_prefetch_with_immediate_flip +dcn_bw_max2(v->read_bandwidth[k], v->required_prefetch_pixel_data_bw_with_immediate_flip[i][j][k]) +dcn_bw_max2(v->meta_pte_bytes_per_frame[k] / (v->lines_for_meta_pte_with_immediate_flip[k] * v->htotal[k] / v->pixel_clock[k]), (v->meta_row_bytes[k] + v->dpte_bytes_per_row[k]) / (v->lines_for_meta_and_dpte_row_with_immediate_flip[k] * v->htotal[k] / v->pixel_clock[k]));
-				}
-				else {
-					v->maximum_read_bandwidth_with_prefetch_with_immediate_flip = v->maximum_read_bandwidth_with_prefetch_with_immediate_flip +dcn_bw_max2(v->read_bandwidth[k], v->required_prefetch_pixel_data_bw_without_immediate_flip[i][j][k]);
+					v->maximum_read_bandwidth_with_prefetch_with_immediate_flip = v->maximum_read_bandwidth_with_prefetch_with_immediate_flip + dcn_bw_max2(v->read_bandwidth[k], v->required_prefetch_pixel_data_bw_with_immediate_flip[i][j][k]) + dcn_bw_max2(v->meta_pte_bytes_per_frame[k] / (v->lines_for_meta_pte_with_immediate_flip[k] * v->htotal[k] / v->pixel_clock[k]), (v->meta_row_bytes[k] + v->dpte_bytes_per_row[k]) / (v->lines_for_meta_and_dpte_row_with_immediate_flip[k] * v->htotal[k] / v->pixel_clock[k]));
+				} else {
+					v->maximum_read_bandwidth_with_prefetch_with_immediate_flip = v->maximum_read_bandwidth_with_prefetch_with_immediate_flip + dcn_bw_max2(v->read_bandwidth[k], v->required_prefetch_pixel_data_bw_without_immediate_flip[i][j][k]);
 				}
 			}
-			v->maximum_read_bandwidth_with_prefetch_without_immediate_flip = 0.0;
+			v->maximum_read_bandwidth_with_prefetch_without_immediate_flip = 0.0f;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				v->maximum_read_bandwidth_with_prefetch_without_immediate_flip = v->maximum_read_bandwidth_with_prefetch_without_immediate_flip +dcn_bw_max2(v->read_bandwidth[k], v->required_prefetch_pixel_data_bw_without_immediate_flip[i][j][k]);
+				v->maximum_read_bandwidth_with_prefetch_without_immediate_flip = v->maximum_read_bandwidth_with_prefetch_without_immediate_flip + dcn_bw_max2(v->read_bandwidth[k], v->required_prefetch_pixel_data_bw_without_immediate_flip[i][j][k]);
 			}
 			v->prefetch_supported_with_immediate_flip[i][j] = dcn_bw_yes;
 			if (v->maximum_read_bandwidth_with_prefetch_with_immediate_flip > v->return_bw_per_state[i]) {
 				v->prefetch_supported_with_immediate_flip[i][j] = dcn_bw_no;
 			}
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				if (v->line_times_for_prefetch[k] < 2.0 || v->lines_for_meta_pte_with_immediate_flip[k] >= 8.0 || v->lines_for_meta_and_dpte_row_with_immediate_flip[k] >= 16.0) {
+				if (v->line_times_for_prefetch[k] < 2.0f || v->lines_for_meta_pte_with_immediate_flip[k] >= 8.0f || v->lines_for_meta_and_dpte_row_with_immediate_flip[k] >= 16.0f) {
 					v->prefetch_supported_with_immediate_flip[i][j] = dcn_bw_no;
 				}
 			}
@@ -939,7 +844,7 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 				v->prefetch_supported_without_immediate_flip[i][j] = dcn_bw_no;
 			}
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				if (v->line_times_for_prefetch[k] < 2.0 || v->lines_for_meta_pte_without_immediate_flip[k] >= 8.0 || v->lines_for_meta_and_dpte_row_without_immediate_flip[k] >= 16.0) {
+				if (v->line_times_for_prefetch[k] < 2.0f || v->lines_for_meta_pte_without_immediate_flip[k] >= 8.0f || v->lines_for_meta_and_dpte_row_without_immediate_flip[k] >= 16.0f) {
 					v->prefetch_supported_without_immediate_flip[i][j] = dcn_bw_no;
 				}
 			}
@@ -949,13 +854,13 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 		for (j = 0; j <= 1; j++) {
 			v->v_ratio_in_prefetch_supported_with_immediate_flip[i][j] = dcn_bw_yes;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				if ((((v->source_pixel_format[k] != dcn_bw_yuv420_sub_8 && v->source_pixel_format[k] != dcn_bw_yuv420_sub_10) && (v->v_ratio_pre_ywith_immediate_flip[i][j][k] > 4.0 || v->v_ratio_pre_cwith_immediate_flip[i][j][k] > 4.0)) || ((v->source_pixel_format[k] == dcn_bw_yuv420_sub_8 || v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) && (v->v_ratio_pre_ywithout_immediate_flip[i][j][k] > 4.0 || v->v_ratio_pre_cwithout_immediate_flip[i][j][k] > 4.0)))) {
+				if ((((v->source_pixel_format[k] != dcn_bw_yuv420_sub_8 && v->source_pixel_format[k] != dcn_bw_yuv420_sub_10) && (v->v_ratio_pre_ywith_immediate_flip[i][j][k] > 4.0f || v->v_ratio_pre_cwith_immediate_flip[i][j][k] > 4.0f)) || ((v->source_pixel_format[k] == dcn_bw_yuv420_sub_8 || v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) && (v->v_ratio_pre_ywithout_immediate_flip[i][j][k] > 4.0f || v->v_ratio_pre_cwithout_immediate_flip[i][j][k] > 4.0f)))) {
 					v->v_ratio_in_prefetch_supported_with_immediate_flip[i][j] = dcn_bw_no;
 				}
 			}
 			v->v_ratio_in_prefetch_supported_without_immediate_flip[i][j] = dcn_bw_yes;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				if ((v->v_ratio_pre_ywithout_immediate_flip[i][j][k] > 4.0 || v->v_ratio_pre_cwithout_immediate_flip[i][j][k] > 4.0)) {
+				if ((v->v_ratio_pre_ywithout_immediate_flip[i][j][k] > 4.0f || v->v_ratio_pre_cwithout_immediate_flip[i][j][k] > 4.0f)) {
 					v->v_ratio_in_prefetch_supported_without_immediate_flip[i][j] = dcn_bw_no;
 				}
 			}
@@ -968,18 +873,15 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 			if (v->scale_ratio_support == dcn_bw_yes && v->source_format_pixel_and_scan_support == dcn_bw_yes && v->viewport_size_support == dcn_bw_yes && v->bandwidth_support[i] == dcn_bw_yes && v->dio_support[i] == dcn_bw_yes && v->urgent_latency_support[i][j] == dcn_bw_yes && v->rob_support[i] == dcn_bw_yes && v->dispclk_dppclk_support[i][j] == dcn_bw_yes && v->total_available_pipes_support[i][j] == dcn_bw_yes && v->total_available_writeback_support == dcn_bw_yes && v->writeback_latency_support == dcn_bw_yes) {
 				if (v->prefetch_supported_with_immediate_flip[i][j] == dcn_bw_yes && v->v_ratio_in_prefetch_supported_with_immediate_flip[i][j] == dcn_bw_yes) {
 					v->mode_support_with_immediate_flip[i][j] = dcn_bw_yes;
-				}
-				else {
+				} else {
 					v->mode_support_with_immediate_flip[i][j] = dcn_bw_no;
 				}
 				if (v->prefetch_supported_without_immediate_flip[i][j] == dcn_bw_yes && v->v_ratio_in_prefetch_supported_without_immediate_flip[i][j] == dcn_bw_yes) {
 					v->mode_support_without_immediate_flip[i][j] = dcn_bw_yes;
-				}
-				else {
+				} else {
 					v->mode_support_without_immediate_flip[i][j] = dcn_bw_no;
 				}
-			}
-			else {
+			} else {
 				v->mode_support_with_immediate_flip[i][j] = dcn_bw_no;
 				v->mode_support_without_immediate_flip[i][j] = dcn_bw_no;
 			}
@@ -987,21 +889,20 @@ void mode_support_and_system_configuration(struct dcn_bw_internal_vars *v)
 	}
 	for (i = number_of_states_plus_one; i >= 0; i--) {
 		if ((i == number_of_states_plus_one || v->mode_support_with_immediate_flip[i][1] == dcn_bw_yes || v->mode_support_with_immediate_flip[i][0] == dcn_bw_yes) && i >= v->voltage_override_level) {
-			v->voltage_level_with_immediate_flip = i;
+			v->voltage_level_with_immediate_flip = (float)i;
 		}
 	}
 	for (i = number_of_states_plus_one; i >= 0; i--) {
 		if ((i == number_of_states_plus_one || v->mode_support_without_immediate_flip[i][1] == dcn_bw_yes || v->mode_support_without_immediate_flip[i][0] == dcn_bw_yes) && i >= v->voltage_override_level) {
-			v->voltage_level_without_immediate_flip = i;
+			v->voltage_level_without_immediate_flip = (float)i;
 		}
 	}
 	if (v->voltage_level_with_immediate_flip == number_of_states_plus_one) {
 		v->immediate_flip_supported = dcn_bw_no;
-		v->voltage_level = v->voltage_level_without_immediate_flip;
-	}
-	else {
+		v->voltage_level = (int)v->voltage_level_without_immediate_flip;
+	} else {
 		v->immediate_flip_supported = dcn_bw_yes;
-		v->voltage_level = v->voltage_level_with_immediate_flip;
+		v->voltage_level = (int)v->voltage_level_with_immediate_flip;
 	}
 	v->dcfclk = v->dcfclk_per_state[v->voltage_level];
 	v->fabric_and_dram_bandwidth = v->fabric_and_dram_bandwidth_per_state[v->voltage_level];
@@ -1021,16 +922,15 @@ void display_pipe_configuration(struct dcn_bw_internal_vars *v)
 	/*display pipe configuration*/
 
 	for (j = 0; j <= 1; j++) {
-		v->total_number_of_active_dpp_per_ratio[j] = 0.0;
+		v->total_number_of_active_dpp_per_ratio[j] = 0.0f;
 		for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 			v->total_number_of_active_dpp_per_ratio[j] = v->total_number_of_active_dpp_per_ratio[j] + v->dpp_per_plane_per_ratio[j][k];
 		}
 	}
-	if ((v->dispclk_dppclk_support_per_ratio[0] == dcn_bw_yes && v->dispclk_dppclk_support_per_ratio[1] == dcn_bw_no) || (v->dispclk_dppclk_support_per_ratio[0] == v->dispclk_dppclk_support_per_ratio[1] && (v->total_number_of_active_dpp_per_ratio[0] < v->total_number_of_active_dpp_per_ratio[1] || (((v->total_number_of_active_dpp_per_ratio[0] == v->total_number_of_active_dpp_per_ratio[1]) && v->required_dispclk_per_ratio[0] <= 0.5 * v->required_dispclk_per_ratio[1]))))) {
+	if ((v->dispclk_dppclk_support_per_ratio[0] == dcn_bw_yes && v->dispclk_dppclk_support_per_ratio[1] == dcn_bw_no) || (v->dispclk_dppclk_support_per_ratio[0] == v->dispclk_dppclk_support_per_ratio[1] && (v->total_number_of_active_dpp_per_ratio[0] < v->total_number_of_active_dpp_per_ratio[1] || (((v->total_number_of_active_dpp_per_ratio[0] == v->total_number_of_active_dpp_per_ratio[1]) && v->required_dispclk_per_ratio[0] <= 0.5f * v->required_dispclk_per_ratio[1]))))) {
 		v->dispclk_dppclk_ratio = 1;
 		v->final_error_message = v->error_message[0];
-	}
-	else {
+	} else {
 		v->dispclk_dppclk_ratio = 2;
 		v->final_error_message = v->error_message[1];
 	}
@@ -1039,137 +939,116 @@ void display_pipe_configuration(struct dcn_bw_internal_vars *v)
 	}
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
-			v->byte_per_pix_dety = 8.0;
-			v->byte_per_pix_detc = 0.0;
-		}
-		else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_32) {
-			v->byte_per_pix_dety = 4.0;
-			v->byte_per_pix_detc = 0.0;
-		}
-		else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_16) {
-			v->byte_per_pix_dety = 2.0;
-			v->byte_per_pix_detc = 0.0;
-		}
-		else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
-			v->byte_per_pix_dety = 1.0;
-			v->byte_per_pix_detc = 2.0;
-		}
-		else {
+			v->byte_per_pix_dety = 8.0f;
+			v->byte_per_pix_detc = 0.0f;
+		} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_32) {
+			v->byte_per_pix_dety = 4.0f;
+			v->byte_per_pix_detc = 0.0f;
+		} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_16) {
+			v->byte_per_pix_dety = 2.0f;
+			v->byte_per_pix_detc = 0.0f;
+		} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
+			v->byte_per_pix_dety = 1.0f;
+			v->byte_per_pix_detc = 2.0f;
+		} else {
 			v->byte_per_pix_dety = 4.0f / 3.0f;
 			v->byte_per_pix_detc = 8.0f / 3.0f;
 		}
 		if ((v->source_pixel_format[k] == dcn_bw_rgb_sub_64 || v->source_pixel_format[k] == dcn_bw_rgb_sub_32 || v->source_pixel_format[k] == dcn_bw_rgb_sub_16)) {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-				v->read256_bytes_block_height_y = 1.0;
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
-				v->read256_bytes_block_height_y = 4.0;
-			}
-			else {
-				v->read256_bytes_block_height_y = 8.0;
-			}
-			v->read256_bytes_block_width_y = 256.0 /dcn_bw_ceil2(v->byte_per_pix_dety, 1.0) / v->read256_bytes_block_height_y;
-			v->read256_bytes_block_height_c = 0.0;
-			v->read256_bytes_block_width_c = 0.0;
-		}
-		else {
+				v->read256_bytes_block_height_y = 1.0f;
+			} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
+				v->read256_bytes_block_height_y = 4.0f;
+			} else {
+				v->read256_bytes_block_height_y = 8.0f;
+			}
+			v->read256_bytes_block_width_y = 256.0f / dcn_bw_ceil2(v->byte_per_pix_dety, 1.0f) / v->read256_bytes_block_height_y;
+			v->read256_bytes_block_height_c = 0.0f;
+			v->read256_bytes_block_width_c = 0.0f;
+		} else {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-				v->read256_bytes_block_height_y = 1.0;
-				v->read256_bytes_block_height_c = 1.0;
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
-				v->read256_bytes_block_height_y = 16.0;
-				v->read256_bytes_block_height_c = 8.0;
-			}
-			else {
-				v->read256_bytes_block_height_y = 8.0;
-				v->read256_bytes_block_height_c = 8.0;
-			}
-			v->read256_bytes_block_width_y = 256.0 /dcn_bw_ceil2(v->byte_per_pix_dety, 1.0) / v->read256_bytes_block_height_y;
-			v->read256_bytes_block_width_c = 256.0 /dcn_bw_ceil2(v->byte_per_pix_detc, 2.0) / v->read256_bytes_block_height_c;
+				v->read256_bytes_block_height_y = 1.0f;
+				v->read256_bytes_block_height_c = 1.0f;
+			} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
+				v->read256_bytes_block_height_y = 16.0f;
+				v->read256_bytes_block_height_c = 8.0f;
+			} else {
+				v->read256_bytes_block_height_y = 8.0f;
+				v->read256_bytes_block_height_c = 8.0f;
+			}
+			v->read256_bytes_block_width_y = 256.0f / dcn_bw_ceil2(v->byte_per_pix_dety, 1.0f) / v->read256_bytes_block_height_y;
+			v->read256_bytes_block_width_c = 256.0f / dcn_bw_ceil2(v->byte_per_pix_detc, 2.0f) / v->read256_bytes_block_height_c;
 		}
 		if (v->source_scan[k] == dcn_bw_hor) {
 			v->maximum_swath_height_y = v->read256_bytes_block_height_y;
 			v->maximum_swath_height_c = v->read256_bytes_block_height_c;
-		}
-		else {
+		} else {
 			v->maximum_swath_height_y = v->read256_bytes_block_width_y;
 			v->maximum_swath_height_c = v->read256_bytes_block_width_c;
 		}
 		if ((v->source_pixel_format[k] == dcn_bw_rgb_sub_64 || v->source_pixel_format[k] == dcn_bw_rgb_sub_32 || v->source_pixel_format[k] == dcn_bw_rgb_sub_16)) {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear || (v->source_pixel_format[k] == dcn_bw_rgb_sub_64 && (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_var_s || v->source_surface_mode[k] == dcn_bw_sw_var_s_x) && v->source_scan[k] == dcn_bw_hor)) {
 				v->minimum_swath_height_y = v->maximum_swath_height_y;
-			}
-			else {
-				v->minimum_swath_height_y = v->maximum_swath_height_y / 2.0;
+			} else {
+				v->minimum_swath_height_y = v->maximum_swath_height_y / 2.0f;
 			}
 			v->minimum_swath_height_c = v->maximum_swath_height_c;
-		}
-		else {
+		} else {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
 				v->minimum_swath_height_y = v->maximum_swath_height_y;
 				v->minimum_swath_height_c = v->maximum_swath_height_c;
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8 && v->source_scan[k] == dcn_bw_hor) {
-				v->minimum_swath_height_y = v->maximum_swath_height_y / 2.0;
+			} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8 && v->source_scan[k] == dcn_bw_hor) {
+				v->minimum_swath_height_y = v->maximum_swath_height_y / 2.0f;
 				if (v->bug_forcing_luma_and_chroma_request_to_same_size_fixed == dcn_bw_yes) {
 					v->minimum_swath_height_c = v->maximum_swath_height_c;
+				} else {
+					v->minimum_swath_height_c = v->maximum_swath_height_c / 2.0f;
 				}
-				else {
-					v->minimum_swath_height_c = v->maximum_swath_height_c / 2.0;
-				}
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10 && v->source_scan[k] == dcn_bw_hor) {
-				v->minimum_swath_height_c = v->maximum_swath_height_c / 2.0;
+			} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10 && v->source_scan[k] == dcn_bw_hor) {
+				v->minimum_swath_height_c = v->maximum_swath_height_c / 2.0f;
 				if (v->bug_forcing_luma_and_chroma_request_to_same_size_fixed == dcn_bw_yes) {
 					v->minimum_swath_height_y = v->maximum_swath_height_y;
+				} else {
+					v->minimum_swath_height_y = v->maximum_swath_height_y / 2.0f;
 				}
-				else {
-					v->minimum_swath_height_y = v->maximum_swath_height_y / 2.0;
-				}
-			}
-			else {
+			} else {
 				v->minimum_swath_height_y = v->maximum_swath_height_y;
 				v->minimum_swath_height_c = v->maximum_swath_height_c;
 			}
 		}
 		if (v->source_scan[k] == dcn_bw_hor) {
 			v->swath_width = v->viewport_width[k] / v->dpp_per_plane[k];
-		}
-		else {
+		} else {
 			v->swath_width = v->viewport_height[k] / v->dpp_per_plane[k];
 		}
-		v->swath_width_granularity_y = 256.0 /dcn_bw_ceil2(v->byte_per_pix_dety, 1.0) / v->maximum_swath_height_y;
-		v->rounded_up_max_swath_size_bytes_y = (dcn_bw_ceil2(v->swath_width - 1.0, v->swath_width_granularity_y) + v->swath_width_granularity_y) * v->byte_per_pix_dety * v->maximum_swath_height_y;
+		v->swath_width_granularity_y = 256.0f / dcn_bw_ceil2(v->byte_per_pix_dety, 1.0f) / v->maximum_swath_height_y;
+		v->rounded_up_max_swath_size_bytes_y = (dcn_bw_ceil2(v->swath_width - 1.0f, v->swath_width_granularity_y) + v->swath_width_granularity_y) * v->byte_per_pix_dety * v->maximum_swath_height_y;
 		if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) {
-			v->rounded_up_max_swath_size_bytes_y =dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_y, 256.0) + 256;
+			v->rounded_up_max_swath_size_bytes_y = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_y, 256.0f) + 256;
 		}
-		if (v->maximum_swath_height_c > 0.0) {
-			v->swath_width_granularity_c = 256.0 /dcn_bw_ceil2(v->byte_per_pix_detc, 2.0) / v->maximum_swath_height_c;
-			v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width / 2.0 - 1.0, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pix_detc * v->maximum_swath_height_c;
+		if (v->maximum_swath_height_c > 0.0f) {
+			v->swath_width_granularity_c = 256.0f / dcn_bw_ceil2(v->byte_per_pix_detc, 2.0f) / v->maximum_swath_height_c;
+			v->rounded_up_max_swath_size_bytes_c = (dcn_bw_ceil2(v->swath_width / 2.0f - 1.0f, v->swath_width_granularity_c) + v->swath_width_granularity_c) * v->byte_per_pix_detc * v->maximum_swath_height_c;
 			if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_10) {
-				v->rounded_up_max_swath_size_bytes_c = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0) + 256;
+				v->rounded_up_max_swath_size_bytes_c = dcn_bw_ceil2(v->rounded_up_max_swath_size_bytes_c, 256.0f) + 256;
 			}
 		}
-		if (v->rounded_up_max_swath_size_bytes_y + v->rounded_up_max_swath_size_bytes_c <= v->det_buffer_size_in_kbyte * 1024.0 / 2.0) {
+		if (v->rounded_up_max_swath_size_bytes_y + v->rounded_up_max_swath_size_bytes_c <= v->det_buffer_size_in_kbyte * 1024.0f / 2.0f) {
 			v->swath_height_y[k] = v->maximum_swath_height_y;
 			v->swath_height_c[k] = v->maximum_swath_height_c;
-		}
-		else {
+		} else {
 			v->swath_height_y[k] = v->minimum_swath_height_y;
 			v->swath_height_c[k] = v->minimum_swath_height_c;
 		}
-		if (v->swath_height_c[k] == 0.0) {
-			v->det_buffer_size_y[k] = v->det_buffer_size_in_kbyte * 1024.0;
-			v->det_buffer_size_c[k] = 0.0;
-		}
-		else if (v->swath_height_y[k] <= v->swath_height_c[k]) {
-			v->det_buffer_size_y[k] = v->det_buffer_size_in_kbyte * 1024.0 / 2.0;
-			v->det_buffer_size_c[k] = v->det_buffer_size_in_kbyte * 1024.0 / 2.0;
-		}
-		else {
-			v->det_buffer_size_y[k] = v->det_buffer_size_in_kbyte * 1024.0 * 2.0 / 3.0;
-			v->det_buffer_size_c[k] = v->det_buffer_size_in_kbyte * 1024.0 / 3.0;
+		if (v->swath_height_c[k] == 0.0f) {
+			v->det_buffer_size_y[k] = v->det_buffer_size_in_kbyte * 1024.0f;
+			v->det_buffer_size_c[k] = 0.0f;
+		} else if (v->swath_height_y[k] <= v->swath_height_c[k]) {
+			v->det_buffer_size_y[k] = v->det_buffer_size_in_kbyte * 1024.0f / 2.0f;
+			v->det_buffer_size_c[k] = v->det_buffer_size_in_kbyte * 1024.0f / 2.0f;
+		} else {
+			v->det_buffer_size_y[k] = v->det_buffer_size_in_kbyte * 1024.0f * 2.0f / 3.0f;
+			v->det_buffer_size_c[k] = v->det_buffer_size_in_kbyte * 1024.0f / 3.0f;
 		}
 	}
 }
@@ -1178,52 +1057,46 @@ void dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performan
 	int k;
 	/*dispclk and dppclk calculation*/
 
-	v->dispclk_with_ramping = 0.0;
-	v->dispclk_without_ramping = 0.0;
+	v->dispclk_with_ramping = 0.0f;
+	v->dispclk_without_ramping = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->h_ratio[k] > 1.0) {
-			v->pscl_throughput[k] =dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput * v->h_ratio[k] /dcn_bw_ceil2(v->htaps[k] / 6.0, 1.0));
+		if (v->h_ratio[k] > 1.0f) {
+			v->pscl_throughput[k] = dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput * v->h_ratio[k] / dcn_bw_ceil2(v->htaps[k] / 6.0f, 1.0f));
+		} else {
+			v->pscl_throughput[k] = dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput);
 		}
-		else {
-			v->pscl_throughput[k] =dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput);
-		}
-		v->dppclk_using_single_dpp_luma = v->pixel_clock[k] *dcn_bw_max3(v->vtaps[k] / 6.0 *dcn_bw_min2(1.0, v->h_ratio[k]), v->h_ratio[k] * v->v_ratio[k] / v->pscl_throughput[k], 1.0);
+		v->dppclk_using_single_dpp_luma = v->pixel_clock[k] * dcn_bw_max3(v->vtaps[k] / 6.0f * dcn_bw_min2(1.0f, v->h_ratio[k]), v->h_ratio[k] * v->v_ratio[k] / v->pscl_throughput[k], 1.0f);
 		if ((v->source_pixel_format[k] != dcn_bw_yuv420_sub_8 && v->source_pixel_format[k] != dcn_bw_yuv420_sub_10)) {
-			v->pscl_throughput_chroma[k] = 0.0;
+			v->pscl_throughput_chroma[k] = 0.0f;
 			v->dppclk_using_single_dpp = v->dppclk_using_single_dpp_luma;
-		}
-		else {
-			if (v->h_ratio[k] > 1.0) {
-				v->pscl_throughput_chroma[k] =dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput * v->h_ratio[k] / 2.0 /dcn_bw_ceil2(v->hta_pschroma[k] / 6.0, 1.0));
+		} else {
+			if (v->h_ratio[k] > 1.0f) {
+				v->pscl_throughput_chroma[k] = dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput * v->h_ratio[k] / 2.0f / dcn_bw_ceil2(v->hta_pschroma[k] / 6.0f, 1.0f));
+			} else {
+				v->pscl_throughput_chroma[k] = dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput);
 			}
-			else {
-				v->pscl_throughput_chroma[k] =dcn_bw_min2(v->max_dchub_topscl_throughput, v->max_pscl_tolb_throughput);
-			}
-			v->dppclk_using_single_dpp_chroma = v->pixel_clock[k] *dcn_bw_max3(v->vta_pschroma[k] / 6.0 *dcn_bw_min2(1.0, v->h_ratio[k] / 2.0), v->h_ratio[k] * v->v_ratio[k] / 4.0 / v->pscl_throughput_chroma[k], 1.0);
-			v->dppclk_using_single_dpp =dcn_bw_max2(v->dppclk_using_single_dpp_luma, v->dppclk_using_single_dpp_chroma);
+			v->dppclk_using_single_dpp_chroma = v->pixel_clock[k] * dcn_bw_max3(v->vta_pschroma[k] / 6.0f * dcn_bw_min2(1.0f, v->h_ratio[k] / 2.0f), v->h_ratio[k] * v->v_ratio[k] / 4.0f / v->pscl_throughput_chroma[k], 1.0f);
+			v->dppclk_using_single_dpp = dcn_bw_max2(v->dppclk_using_single_dpp_luma, v->dppclk_using_single_dpp_chroma);
 		}
 		if (v->odm_capable == dcn_bw_yes) {
-			v->dispclk_with_ramping =dcn_bw_max2(v->dispclk_with_ramping,dcn_bw_max2(v->dppclk_using_single_dpp / v->dpp_per_plane[k] * v->dispclk_dppclk_ratio, v->pixel_clock[k] / v->dpp_per_plane[k]) * (1.0 + v->downspreading / 100.0) * (1.0 + v->dispclk_ramping_margin / 100.0));
-			v->dispclk_without_ramping =dcn_bw_max2(v->dispclk_without_ramping,dcn_bw_max2(v->dppclk_using_single_dpp / v->dpp_per_plane[k] * v->dispclk_dppclk_ratio, v->pixel_clock[k] / v->dpp_per_plane[k]) * (1.0 + v->downspreading / 100.0));
-		}
-		else {
-			v->dispclk_with_ramping =dcn_bw_max2(v->dispclk_with_ramping,dcn_bw_max2(v->dppclk_using_single_dpp / v->dpp_per_plane[k] * v->dispclk_dppclk_ratio, v->pixel_clock[k]) * (1.0 + v->downspreading / 100.0) * (1.0 + v->dispclk_ramping_margin / 100.0));
-			v->dispclk_without_ramping =dcn_bw_max2(v->dispclk_without_ramping,dcn_bw_max2(v->dppclk_using_single_dpp / v->dpp_per_plane[k] * v->dispclk_dppclk_ratio, v->pixel_clock[k]) * (1.0 + v->downspreading / 100.0));
+			v->dispclk_with_ramping = dcn_bw_max2(v->dispclk_with_ramping, dcn_bw_max2(v->dppclk_using_single_dpp / v->dpp_per_plane[k] * v->dispclk_dppclk_ratio, v->pixel_clock[k] / v->dpp_per_plane[k]) * (1.0f + v->downspreading / 100.0f) * (1.0f + v->dispclk_ramping_margin / 100.0f));
+			v->dispclk_without_ramping = dcn_bw_max2(v->dispclk_without_ramping, dcn_bw_max2(v->dppclk_using_single_dpp / v->dpp_per_plane[k] * v->dispclk_dppclk_ratio, v->pixel_clock[k] / v->dpp_per_plane[k]) * (1.0f + v->downspreading / 100.0f));
+		} else {
+			v->dispclk_with_ramping = dcn_bw_max2(v->dispclk_with_ramping, dcn_bw_max2(v->dppclk_using_single_dpp / v->dpp_per_plane[k] * v->dispclk_dppclk_ratio, v->pixel_clock[k]) * (1.0f + v->downspreading / 100.0f) * (1.0f + v->dispclk_ramping_margin / 100.0f));
+			v->dispclk_without_ramping = dcn_bw_max2(v->dispclk_without_ramping, dcn_bw_max2(v->dppclk_using_single_dpp / v->dpp_per_plane[k] * v->dispclk_dppclk_ratio, v->pixel_clock[k]) * (1.0f + v->downspreading / 100.0f));
 		}
 	}
 	if (v->dispclk_without_ramping > v->max_dispclk[number_of_states]) {
 		v->dispclk = v->dispclk_without_ramping;
-	}
-	else if (v->dispclk_with_ramping > v->max_dispclk[number_of_states]) {
+	} else if (v->dispclk_with_ramping > v->max_dispclk[number_of_states]) {
 		v->dispclk = v->max_dispclk[number_of_states];
-	}
-	else {
+	} else {
 		v->dispclk = v->dispclk_with_ramping;
 	}
 	v->dppclk = v->dispclk / v->dispclk_dppclk_ratio;
 	/*urgent watermark*/
 
-	v->return_bandwidth_to_dcn =dcn_bw_min2(v->return_bus_width * v->dcfclk, v->fabric_and_dram_bandwidth * 1000.0 * v->percent_of_ideal_drambw_received_after_urg_latency / 100.0);
+	v->return_bandwidth_to_dcn = dcn_bw_min2(v->return_bus_width * v->dcfclk, v->fabric_and_dram_bandwidth * 1000.0f * v->percent_of_ideal_drambw_received_after_urg_latency / 100.0f);
 	v->dcc_enabled_any_plane = dcn_bw_no;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->dcc_enable[k] == dcn_bw_yes) {
@@ -1231,129 +1104,119 @@ void dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performan
 		}
 	}
 	v->return_bw = v->return_bandwidth_to_dcn;
-	if (v->dcc_enabled_any_plane == dcn_bw_yes && v->return_bandwidth_to_dcn > v->dcfclk * v->return_bus_width / 4.0) {
-		v->return_bw =dcn_bw_min2(v->return_bw, v->return_bandwidth_to_dcn * 4.0 * (1.0 - v->urgent_latency / ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0 / (v->return_bandwidth_to_dcn - v->dcfclk * v->return_bus_width / 4.0) + v->urgent_latency)));
+	if (v->dcc_enabled_any_plane == dcn_bw_yes && v->return_bandwidth_to_dcn > v->dcfclk * v->return_bus_width / 4.0f) {
+		v->return_bw = dcn_bw_min2(v->return_bw, v->return_bandwidth_to_dcn * 4.0f * (1.0f - v->urgent_latency / ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f / (v->return_bandwidth_to_dcn - v->dcfclk * v->return_bus_width / 4.0f) + v->urgent_latency)));
 	}
-	v->critical_compression = 2.0 * v->return_bus_width * v->dcfclk * v->urgent_latency / (v->return_bandwidth_to_dcn * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0);
-	if (v->dcc_enabled_any_plane == dcn_bw_yes && v->critical_compression > 1.0 && v->critical_compression < 4.0) {
-		v->return_bw =dcn_bw_min2(v->return_bw, dcn_bw_pow(4.0 * v->return_bandwidth_to_dcn * (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0 * v->return_bus_width * v->dcfclk * v->urgent_latency / (v->return_bandwidth_to_dcn * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0), 2));
+	v->critical_compression = 2.0f * v->return_bus_width * v->dcfclk * v->urgent_latency / (v->return_bandwidth_to_dcn * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f);
+	if (v->dcc_enabled_any_plane == dcn_bw_yes && v->critical_compression > 1.0f && v->critical_compression < 4.0f) {
+		v->return_bw = dcn_bw_min2(v->return_bw, dcn_bw_pow(4.0f * v->return_bandwidth_to_dcn * (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f * v->return_bus_width * v->dcfclk * v->urgent_latency / (v->return_bandwidth_to_dcn * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f), 2));
 	}
-	v->return_bandwidth_to_dcn =dcn_bw_min2(v->return_bus_width * v->dcfclk, v->fabric_and_dram_bandwidth * 1000.0);
-	if (v->dcc_enabled_any_plane == dcn_bw_yes && v->return_bandwidth_to_dcn > v->dcfclk * v->return_bus_width / 4.0) {
-		v->return_bw =dcn_bw_min2(v->return_bw, v->return_bandwidth_to_dcn * 4.0 * (1.0 - v->urgent_latency / ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0 / (v->return_bandwidth_to_dcn - v->dcfclk * v->return_bus_width / 4.0) + v->urgent_latency)));
+	v->return_bandwidth_to_dcn = dcn_bw_min2(v->return_bus_width * v->dcfclk, v->fabric_and_dram_bandwidth * 1000.0f);
+	if (v->dcc_enabled_any_plane == dcn_bw_yes && v->return_bandwidth_to_dcn > v->dcfclk * v->return_bus_width / 4.0f) {
+		v->return_bw = dcn_bw_min2(v->return_bw, v->return_bandwidth_to_dcn * 4.0f * (1.0f - v->urgent_latency / ((v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f / (v->return_bandwidth_to_dcn - v->dcfclk * v->return_bus_width / 4.0f) + v->urgent_latency)));
 	}
-	v->critical_compression = 2.0 * v->return_bus_width * v->dcfclk * v->urgent_latency / (v->return_bandwidth_to_dcn * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0);
-	if (v->dcc_enabled_any_plane == dcn_bw_yes && v->critical_compression > 1.0 && v->critical_compression < 4.0) {
-		v->return_bw =dcn_bw_min2(v->return_bw, dcn_bw_pow(4.0 * v->return_bandwidth_to_dcn * (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0 * v->return_bus_width * v->dcfclk * v->urgent_latency / (v->return_bandwidth_to_dcn * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0), 2));
+	v->critical_compression = 2.0f * v->return_bus_width * v->dcfclk * v->urgent_latency / (v->return_bandwidth_to_dcn * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f);
+	if (v->dcc_enabled_any_plane == dcn_bw_yes && v->critical_compression > 1.0f && v->critical_compression < 4.0f) {
+		v->return_bw = dcn_bw_min2(v->return_bw, dcn_bw_pow(4.0f * v->return_bandwidth_to_dcn * (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f * v->return_bus_width * v->dcfclk * v->urgent_latency / (v->return_bandwidth_to_dcn * v->urgent_latency + (v->rob_buffer_size_in_kbyte - v->pixel_chunk_size_in_kbyte) * 1024.0f), 2));
 	}
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->source_scan[k] == dcn_bw_hor) {
 			v->swath_width_y[k] = v->viewport_width[k] / v->dpp_per_plane[k];
-		}
-		else {
+		} else {
 			v->swath_width_y[k] = v->viewport_height[k] / v->dpp_per_plane[k];
 		}
 	}
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
-			v->byte_per_pixel_dety[k] = 8.0;
-			v->byte_per_pixel_detc[k] = 0.0;
-		}
-		else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_32) {
-			v->byte_per_pixel_dety[k] = 4.0;
-			v->byte_per_pixel_detc[k] = 0.0;
-		}
-		else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_16) {
-			v->byte_per_pixel_dety[k] = 2.0;
-			v->byte_per_pixel_detc[k] = 0.0;
-		}
-		else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
-			v->byte_per_pixel_dety[k] = 1.0;
-			v->byte_per_pixel_detc[k] = 2.0;
-		}
-		else {
+			v->byte_per_pixel_dety[k] = 8.0f;
+			v->byte_per_pixel_detc[k] = 0.0f;
+		} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_32) {
+			v->byte_per_pixel_dety[k] = 4.0f;
+			v->byte_per_pixel_detc[k] = 0.0f;
+		} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_16) {
+			v->byte_per_pixel_dety[k] = 2.0f;
+			v->byte_per_pixel_detc[k] = 0.0f;
+		} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
+			v->byte_per_pixel_dety[k] = 1.0f;
+			v->byte_per_pixel_detc[k] = 2.0f;
+		} else {
 			v->byte_per_pixel_dety[k] = 4.0f / 3.0f;
 			v->byte_per_pixel_detc[k] = 8.0f / 3.0f;
 		}
 	}
-	v->total_data_read_bandwidth = 0.0;
+	v->total_data_read_bandwidth = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		v->read_bandwidth_plane_luma[k] = v->swath_width_y[k] * v->dpp_per_plane[k] *dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k];
-		v->read_bandwidth_plane_chroma[k] = v->swath_width_y[k] / 2.0 * v->dpp_per_plane[k] *dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k] / 2.0;
+		v->read_bandwidth_plane_luma[k] = v->swath_width_y[k] * v->dpp_per_plane[k] * dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k];
+		v->read_bandwidth_plane_chroma[k] = v->swath_width_y[k] / 2.0f * v->dpp_per_plane[k] * dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k] / 2.0f;
 		v->total_data_read_bandwidth = v->total_data_read_bandwidth + v->read_bandwidth_plane_luma[k] + v->read_bandwidth_plane_chroma[k];
 	}
-	v->total_active_dpp = 0.0;
-	v->total_dcc_active_dpp = 0.0;
+	v->total_active_dpp = 0.0f;
+	v->total_dcc_active_dpp = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		v->total_active_dpp = v->total_active_dpp + v->dpp_per_plane[k];
 		if (v->dcc_enable[k] == dcn_bw_yes) {
 			v->total_dcc_active_dpp = v->total_dcc_active_dpp + v->dpp_per_plane[k];
 		}
 	}
-	v->urgent_round_trip_and_out_of_order_latency = (v->round_trip_ping_latency_cycles + 32.0) / v->dcfclk + v->urgent_out_of_order_return_per_channel * v->number_of_channels / v->return_bw;
-	v->last_pixel_of_line_extra_watermark = 0.0;
+	v->urgent_round_trip_and_out_of_order_latency = (v->round_trip_ping_latency_cycles + 32.0f) / v->dcfclk + v->urgent_out_of_order_return_per_channel * v->number_of_channels / v->return_bw;
+	v->last_pixel_of_line_extra_watermark = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->v_ratio[k] <= 1.0) {
+		if (v->v_ratio[k] <= 1.0f) {
 			v->display_pipe_line_delivery_time_luma[k] = v->swath_width_y[k] * v->dpp_per_plane[k] / v->h_ratio[k] / v->pixel_clock[k];
-		}
-		else {
+		} else {
 			v->display_pipe_line_delivery_time_luma[k] = v->swath_width_y[k] / v->pscl_throughput[k] / v->dppclk;
 		}
-		v->data_fabric_line_delivery_time_luma = v->swath_width_y[k] * v->swath_height_y[k] *dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / (v->return_bw * v->read_bandwidth_plane_luma[k] / v->dpp_per_plane[k] / v->total_data_read_bandwidth);
-		v->last_pixel_of_line_extra_watermark =dcn_bw_max2(v->last_pixel_of_line_extra_watermark, v->data_fabric_line_delivery_time_luma - v->display_pipe_line_delivery_time_luma[k]);
-		if (v->byte_per_pixel_detc[k] == 0.0) {
-			v->display_pipe_line_delivery_time_chroma[k] = 0.0;
-		}
-		else {
-			if (v->v_ratio[k] / 2.0 <= 1.0) {
-				v->display_pipe_line_delivery_time_chroma[k] = v->swath_width_y[k] / 2.0 * v->dpp_per_plane[k] / (v->h_ratio[k] / 2.0) / v->pixel_clock[k];
+		v->data_fabric_line_delivery_time_luma = v->swath_width_y[k] * v->swath_height_y[k] * dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / (v->return_bw * v->read_bandwidth_plane_luma[k] / v->dpp_per_plane[k] / v->total_data_read_bandwidth);
+		v->last_pixel_of_line_extra_watermark = dcn_bw_max2(v->last_pixel_of_line_extra_watermark, v->data_fabric_line_delivery_time_luma - v->display_pipe_line_delivery_time_luma[k]);
+		if (v->byte_per_pixel_detc[k] == 0.0f) {
+			v->display_pipe_line_delivery_time_chroma[k] = 0.0f;
+		} else {
+			if (v->v_ratio[k] / 2.0f <= 1.0f) {
+				v->display_pipe_line_delivery_time_chroma[k] = v->swath_width_y[k] / 2.0f * v->dpp_per_plane[k] / (v->h_ratio[k] / 2.0f) / v->pixel_clock[k];
+			} else {
+				v->display_pipe_line_delivery_time_chroma[k] = v->swath_width_y[k] / 2.0f / v->pscl_throughput_chroma[k] / v->dppclk;
 			}
-			else {
-				v->display_pipe_line_delivery_time_chroma[k] = v->swath_width_y[k] / 2.0 / v->pscl_throughput_chroma[k] / v->dppclk;
-			}
-			v->data_fabric_line_delivery_time_chroma = v->swath_width_y[k] / 2.0 * v->swath_height_c[k] *dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / (v->return_bw * v->read_bandwidth_plane_chroma[k] / v->dpp_per_plane[k] / v->total_data_read_bandwidth);
-			v->last_pixel_of_line_extra_watermark =dcn_bw_max2(v->last_pixel_of_line_extra_watermark, v->data_fabric_line_delivery_time_chroma - v->display_pipe_line_delivery_time_chroma[k]);
+			v->data_fabric_line_delivery_time_chroma = v->swath_width_y[k] / 2.0f * v->swath_height_c[k] * dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / (v->return_bw * v->read_bandwidth_plane_chroma[k] / v->dpp_per_plane[k] / v->total_data_read_bandwidth);
+			v->last_pixel_of_line_extra_watermark = dcn_bw_max2(v->last_pixel_of_line_extra_watermark, v->data_fabric_line_delivery_time_chroma - v->display_pipe_line_delivery_time_chroma[k]);
 		}
 	}
-	v->urgent_extra_latency = v->urgent_round_trip_and_out_of_order_latency + (v->total_active_dpp * v->pixel_chunk_size_in_kbyte + v->total_dcc_active_dpp * v->meta_chunk_size) * 1024.0 / v->return_bw;
+	v->urgent_extra_latency = v->urgent_round_trip_and_out_of_order_latency + (v->total_active_dpp * v->pixel_chunk_size_in_kbyte + v->total_dcc_active_dpp * v->meta_chunk_size) * 1024.0f / v->return_bw;
 	if (v->pte_enable == dcn_bw_yes) {
-		v->urgent_extra_latency = v->urgent_extra_latency + v->total_active_dpp * v->pte_chunk_size * 1024.0 / v->return_bw;
+		v->urgent_extra_latency = v->urgent_extra_latency + v->total_active_dpp * v->pte_chunk_size * 1024.0f / v->return_bw;
 	}
 	v->urgent_watermark = v->urgent_latency + v->last_pixel_of_line_extra_watermark + v->urgent_extra_latency;
-	v->ptemeta_urgent_watermark = v->urgent_watermark + 2.0 * v->urgent_latency;
+	v->ptemeta_urgent_watermark = v->urgent_watermark + 2.0f * v->urgent_latency;
 	/*nb p-state/dram clock change watermark*/
 
 	v->dram_clock_change_watermark = v->dram_clock_change_latency + v->urgent_watermark;
-	v->total_active_writeback = 0.0;
+	v->total_active_writeback = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->output[k] == dcn_bw_writeback) {
-			v->total_active_writeback = v->total_active_writeback + 1.0;
+			v->total_active_writeback = v->total_active_writeback + 1.0f;
 		}
 	}
-	if (v->total_active_writeback <= 1.0) {
+	if (v->total_active_writeback <= 1.0f) {
 		v->writeback_dram_clock_change_watermark = v->dram_clock_change_latency + v->write_back_latency;
-	}
-	else {
-		v->writeback_dram_clock_change_watermark = v->dram_clock_change_latency + v->write_back_latency + v->writeback_chunk_size * 1024.0 / 32.0 / v->socclk;
+	} else {
+		v->writeback_dram_clock_change_watermark = v->dram_clock_change_latency + v->write_back_latency + v->writeback_chunk_size * 1024.0f / 32.0f / v->socclk;
 	}
 	/*stutter efficiency*/
 
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		v->lines_in_dety[k] = v->det_buffer_size_y[k] / v->byte_per_pixel_dety[k] / v->swath_width_y[k];
-		v->lines_in_dety_rounded_down_to_swath[k] =dcn_bw_floor2(v->lines_in_dety[k], v->swath_height_y[k]);
+		v->lines_in_dety_rounded_down_to_swath[k] = dcn_bw_floor2(v->lines_in_dety[k], v->swath_height_y[k]);
 		v->full_det_buffering_time_y[k] = v->lines_in_dety_rounded_down_to_swath[k] * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k];
-		if (v->byte_per_pixel_detc[k] > 0.0) {
-			v->lines_in_detc[k] = v->det_buffer_size_c[k] / v->byte_per_pixel_detc[k] / (v->swath_width_y[k] / 2.0);
-			v->lines_in_detc_rounded_down_to_swath[k] =dcn_bw_floor2(v->lines_in_detc[k], v->swath_height_c[k]);
-			v->full_det_buffering_time_c[k] = v->lines_in_detc_rounded_down_to_swath[k] * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0);
-		}
-		else {
-			v->lines_in_detc[k] = 0.0;
-			v->lines_in_detc_rounded_down_to_swath[k] = 0.0;
-			v->full_det_buffering_time_c[k] = 999999.0;
+		if (v->byte_per_pixel_detc[k] > 0.0f) {
+			v->lines_in_detc[k] = v->det_buffer_size_c[k] / v->byte_per_pixel_detc[k] / (v->swath_width_y[k] / 2.0f);
+			v->lines_in_detc_rounded_down_to_swath[k] = dcn_bw_floor2(v->lines_in_detc[k], v->swath_height_c[k]);
+			v->full_det_buffering_time_c[k] = v->lines_in_detc_rounded_down_to_swath[k] * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0f);
+		} else {
+			v->lines_in_detc[k] = 0.0f;
+			v->lines_in_detc_rounded_down_to_swath[k] = 0.0f;
+			v->full_det_buffering_time_c[k] = 999999.0f;
 		}
 	}
-	v->min_full_det_buffering_time = 999999.0;
+	v->min_full_det_buffering_time = 999999.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->full_det_buffering_time_y[k] < v->min_full_det_buffering_time) {
 			v->min_full_det_buffering_time = v->full_det_buffering_time_y[k];
@@ -1364,76 +1227,71 @@ void dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performan
 			v->frame_time_for_min_full_det_buffering_time = v->vtotal[k] * v->htotal[k] / v->pixel_clock[k];
 		}
 	}
-	v->average_read_bandwidth_gbyte_per_second = 0.0;
+	v->average_read_bandwidth_gbyte_per_second = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->dcc_enable[k] == dcn_bw_yes) {
-			v->average_read_bandwidth_gbyte_per_second = v->average_read_bandwidth_gbyte_per_second + v->read_bandwidth_plane_luma[k] / v->dcc_rate[k] / 1000.0 + v->read_bandwidth_plane_chroma[k] / v->dcc_rate[k] / 1000.0;
-		}
-		else {
-			v->average_read_bandwidth_gbyte_per_second = v->average_read_bandwidth_gbyte_per_second + v->read_bandwidth_plane_luma[k] / 1000.0 + v->read_bandwidth_plane_chroma[k] / 1000.0;
+			v->average_read_bandwidth_gbyte_per_second = v->average_read_bandwidth_gbyte_per_second + v->read_bandwidth_plane_luma[k] / v->dcc_rate[k] / 1000.0f + v->read_bandwidth_plane_chroma[k] / v->dcc_rate[k] / 1000.0f;
+		} else {
+			v->average_read_bandwidth_gbyte_per_second = v->average_read_bandwidth_gbyte_per_second + v->read_bandwidth_plane_luma[k] / 1000.0f + v->read_bandwidth_plane_chroma[k] / 1000.0f;
 		}
 		if (v->dcc_enable[k] == dcn_bw_yes) {
-			v->average_read_bandwidth_gbyte_per_second = v->average_read_bandwidth_gbyte_per_second + v->read_bandwidth_plane_luma[k] / 1000.0 / 256.0 + v->read_bandwidth_plane_chroma[k] / 1000.0 / 256.0;
+			v->average_read_bandwidth_gbyte_per_second = v->average_read_bandwidth_gbyte_per_second + v->read_bandwidth_plane_luma[k] / 1000.0f / 256.0f + v->read_bandwidth_plane_chroma[k] / 1000.0f / 256.0f;
 		}
 		if (v->pte_enable == dcn_bw_yes) {
-			v->average_read_bandwidth_gbyte_per_second = v->average_read_bandwidth_gbyte_per_second + v->read_bandwidth_plane_luma[k] / 1000.0 / 512.0 + v->read_bandwidth_plane_chroma[k] / 1000.0 / 512.0;
+			v->average_read_bandwidth_gbyte_per_second = v->average_read_bandwidth_gbyte_per_second + v->read_bandwidth_plane_luma[k] / 1000.0f / 512.0f + v->read_bandwidth_plane_chroma[k] / 1000.0f / 512.0f;
 		}
 	}
-	v->part_of_burst_that_fits_in_rob =dcn_bw_min2(v->min_full_det_buffering_time * v->total_data_read_bandwidth, v->rob_buffer_size_in_kbyte * 1024.0 * v->total_data_read_bandwidth / (v->average_read_bandwidth_gbyte_per_second * 1000.0));
-	v->stutter_burst_time = v->part_of_burst_that_fits_in_rob * (v->average_read_bandwidth_gbyte_per_second * 1000.0) / v->total_data_read_bandwidth / v->return_bw + (v->min_full_det_buffering_time * v->total_data_read_bandwidth - v->part_of_burst_that_fits_in_rob) / (v->dcfclk * 64.0);
-	if (v->total_active_writeback == 0.0) {
-		v->stutter_efficiency_not_including_vblank = (1.0 - (v->sr_exit_time + v->stutter_burst_time) / v->min_full_det_buffering_time) * 100.0;
+	v->part_of_burst_that_fits_in_rob = dcn_bw_min2(v->min_full_det_buffering_time * v->total_data_read_bandwidth, v->rob_buffer_size_in_kbyte * 1024.0f * v->total_data_read_bandwidth / (v->average_read_bandwidth_gbyte_per_second * 1000.0f));
+	v->stutter_burst_time = v->part_of_burst_that_fits_in_rob * (v->average_read_bandwidth_gbyte_per_second * 1000.0f) / v->total_data_read_bandwidth / v->return_bw + (v->min_full_det_buffering_time * v->total_data_read_bandwidth - v->part_of_burst_that_fits_in_rob) / (v->dcfclk * 64.0f);
+	if (v->total_active_writeback == 0.0f) {
+		v->stutter_efficiency_not_including_vblank = (1.0f - (v->sr_exit_time + v->stutter_burst_time) / v->min_full_det_buffering_time) * 100.0f;
+	} else {
+		v->stutter_efficiency_not_including_vblank = 0.0f;
 	}
-	else {
-		v->stutter_efficiency_not_including_vblank = 0.0;
-	}
-	v->smallest_vblank = 999999.0;
+	v->smallest_vblank = 999999.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->synchronized_vblank == dcn_bw_yes || v->number_of_active_planes == 1) {
 			v->v_blank_time = (v->vtotal[k] - v->vactive[k]) * v->htotal[k] / v->pixel_clock[k];
+		} else {
+			v->v_blank_time = 0.0f;
 		}
-		else {
-			v->v_blank_time = 0.0;
-		}
-		v->smallest_vblank =dcn_bw_min2(v->smallest_vblank, v->v_blank_time);
+		v->smallest_vblank = dcn_bw_min2(v->smallest_vblank, v->v_blank_time);
 	}
-	v->stutter_efficiency = (v->stutter_efficiency_not_including_vblank / 100.0 * (v->frame_time_for_min_full_det_buffering_time - v->smallest_vblank) + v->smallest_vblank) / v->frame_time_for_min_full_det_buffering_time * 100.0;
+	v->stutter_efficiency = (v->stutter_efficiency_not_including_vblank / 100.0f * (v->frame_time_for_min_full_det_buffering_time - v->smallest_vblank) + v->smallest_vblank) / v->frame_time_for_min_full_det_buffering_time * 100.0f;
 	/*dcfclk deep sleep*/
 
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->byte_per_pixel_detc[k] > 0.0) {
-			v->dcfclk_deep_sleep_per_plane[k] =dcn_bw_max2(1.1 * v->swath_width_y[k] *dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / 32.0 / v->display_pipe_line_delivery_time_luma[k], 1.1 * v->swath_width_y[k] / 2.0 *dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / 32.0 / v->display_pipe_line_delivery_time_chroma[k]);
-		}
-		else {
-			v->dcfclk_deep_sleep_per_plane[k] = 1.1 * v->swath_width_y[k] *dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / 64.0 / v->display_pipe_line_delivery_time_luma[k];
+		if (v->byte_per_pixel_detc[k] > 0.0f) {
+			v->dcfclk_deep_sleep_per_plane[k] = dcn_bw_max2(1.1f * v->swath_width_y[k] * dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / 32.0f / v->display_pipe_line_delivery_time_luma[k], 1.1f * v->swath_width_y[k] / 2.0f * dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / 32.0f / v->display_pipe_line_delivery_time_chroma[k]);
+		} else {
+			v->dcfclk_deep_sleep_per_plane[k] = 1.1f * v->swath_width_y[k] * dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / 64.0f / v->display_pipe_line_delivery_time_luma[k];
 		}
-		v->dcfclk_deep_sleep_per_plane[k] =dcn_bw_max2(v->dcfclk_deep_sleep_per_plane[k], v->pixel_clock[k] / 16.0);
+		v->dcfclk_deep_sleep_per_plane[k] = dcn_bw_max2(v->dcfclk_deep_sleep_per_plane[k], v->pixel_clock[k] / 16.0f);
 	}
-	v->dcf_clk_deep_sleep = 8.0;
+	v->dcf_clk_deep_sleep = 8.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		v->dcf_clk_deep_sleep =dcn_bw_max2(v->dcf_clk_deep_sleep, v->dcfclk_deep_sleep_per_plane[k]);
+		v->dcf_clk_deep_sleep = dcn_bw_max2(v->dcf_clk_deep_sleep, v->dcfclk_deep_sleep_per_plane[k]);
 	}
 	/*stutter watermark*/
 
-	v->stutter_exit_watermark = v->sr_exit_time + v->last_pixel_of_line_extra_watermark + v->urgent_extra_latency + 10.0 / v->dcf_clk_deep_sleep;
+	v->stutter_exit_watermark = v->sr_exit_time + v->last_pixel_of_line_extra_watermark + v->urgent_extra_latency + 10.0f / v->dcf_clk_deep_sleep;
 	v->stutter_enter_plus_exit_watermark = v->sr_enter_plus_exit_time + v->last_pixel_of_line_extra_watermark + v->urgent_extra_latency;
 	/*urgent latency supported*/
 
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		v->effective_det_plus_lb_lines_luma =dcn_bw_floor2(v->lines_in_dety[k] +dcn_bw_min2(v->lines_in_dety[k] * v->dppclk * v->byte_per_pixel_dety[k] * v->pscl_throughput[k] / (v->return_bw / v->dpp_per_plane[k]), v->effective_lb_latency_hiding_source_lines_luma), v->swath_height_y[k]);
+		v->effective_det_plus_lb_lines_luma = dcn_bw_floor2(v->lines_in_dety[k] + dcn_bw_min2(v->lines_in_dety[k] * v->dppclk * v->byte_per_pixel_dety[k] * v->pscl_throughput[k] / (v->return_bw / v->dpp_per_plane[k]), v->effective_lb_latency_hiding_source_lines_luma), v->swath_height_y[k]);
 		v->urgent_latency_support_us_luma = v->effective_det_plus_lb_lines_luma * (v->htotal[k] / v->pixel_clock[k]) / v->v_ratio[k] - v->effective_det_plus_lb_lines_luma * v->swath_width_y[k] * v->byte_per_pixel_dety[k] / (v->return_bw / v->dpp_per_plane[k]);
-		if (v->byte_per_pixel_detc[k] > 0.0) {
-			v->effective_det_plus_lb_lines_chroma =dcn_bw_floor2(v->lines_in_detc[k] +dcn_bw_min2(v->lines_in_detc[k] * v->dppclk * v->byte_per_pixel_detc[k] * v->pscl_throughput_chroma[k] / (v->return_bw / v->dpp_per_plane[k]), v->effective_lb_latency_hiding_source_lines_chroma), v->swath_height_c[k]);
-			v->urgent_latency_support_us_chroma = v->effective_det_plus_lb_lines_chroma * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0) - v->effective_det_plus_lb_lines_chroma * (v->swath_width_y[k] / 2.0) * v->byte_per_pixel_detc[k] / (v->return_bw / v->dpp_per_plane[k]);
-			v->urgent_latency_support_us[k] =dcn_bw_min2(v->urgent_latency_support_us_luma, v->urgent_latency_support_us_chroma);
-		}
-		else {
+		if (v->byte_per_pixel_detc[k] > 0.0f) {
+			v->effective_det_plus_lb_lines_chroma = dcn_bw_floor2(v->lines_in_detc[k] + dcn_bw_min2(v->lines_in_detc[k] * v->dppclk * v->byte_per_pixel_detc[k] * v->pscl_throughput_chroma[k] / (v->return_bw / v->dpp_per_plane[k]), v->effective_lb_latency_hiding_source_lines_chroma), v->swath_height_c[k]);
+			v->urgent_latency_support_us_chroma = v->effective_det_plus_lb_lines_chroma * (v->htotal[k] / v->pixel_clock[k]) / (v->v_ratio[k] / 2.0f) - v->effective_det_plus_lb_lines_chroma * (v->swath_width_y[k] / 2.0f) * v->byte_per_pixel_detc[k] / (v->return_bw / v->dpp_per_plane[k]);
+			v->urgent_latency_support_us[k] = dcn_bw_min2(v->urgent_latency_support_us_luma, v->urgent_latency_support_us_chroma);
+		} else {
 			v->urgent_latency_support_us[k] = v->urgent_latency_support_us_luma;
 		}
 	}
-	v->min_urgent_latency_support_us = 999999.0;
+	v->min_urgent_latency_support_us = 999999.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		v->min_urgent_latency_support_us =dcn_bw_min2(v->min_urgent_latency_support_us, v->urgent_latency_support_us[k]);
+		v->min_urgent_latency_support_us = dcn_bw_min2(v->min_urgent_latency_support_us, v->urgent_latency_support_us[k]);
 	}
 	/*non-urgent latency tolerance*/
 
@@ -1443,195 +1301,165 @@ void dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performan
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if ((v->source_pixel_format[k] == dcn_bw_rgb_sub_64 || v->source_pixel_format[k] == dcn_bw_rgb_sub_32 || v->source_pixel_format[k] == dcn_bw_rgb_sub_16)) {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-				v->block_height256_bytes_y = 1.0;
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
-				v->block_height256_bytes_y = 4.0;
-			}
-			else {
-				v->block_height256_bytes_y = 8.0;
-			}
-			v->block_height256_bytes_c = 0.0;
-		}
-		else {
+				v->block_height256_bytes_y = 1.0f;
+			} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
+				v->block_height256_bytes_y = 4.0f;
+			} else {
+				v->block_height256_bytes_y = 8.0f;
+			}
+			v->block_height256_bytes_c = 0.0f;
+		} else {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-				v->block_height256_bytes_y = 1.0;
-				v->block_height256_bytes_c = 1.0;
-			}
-			else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
-				v->block_height256_bytes_y = 16.0;
-				v->block_height256_bytes_c = 8.0;
-			}
-			else {
-				v->block_height256_bytes_y = 8.0;
-				v->block_height256_bytes_c = 8.0;
+				v->block_height256_bytes_y = 1.0f;
+				v->block_height256_bytes_c = 1.0f;
+			} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
+				v->block_height256_bytes_y = 16.0f;
+				v->block_height256_bytes_c = 8.0f;
+			} else {
+				v->block_height256_bytes_y = 8.0f;
+				v->block_height256_bytes_c = 8.0f;
 			}
 		}
 		if (v->dcc_enable[k] == dcn_bw_yes) {
-			v->meta_request_width_y = 64.0 * 256.0 /dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / (8.0 * v->block_height256_bytes_y);
-			v->meta_surf_width_y =dcn_bw_ceil2(v->swath_width_y[k] - 1.0, v->meta_request_width_y) + v->meta_request_width_y;
-			v->meta_surf_height_y =dcn_bw_ceil2(v->viewport_height[k] - 1.0, 8.0 * v->block_height256_bytes_y) + 8.0 * v->block_height256_bytes_y;
+			v->meta_request_width_y = 64.0f * 256.0f / dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / (8.0f * v->block_height256_bytes_y);
+			v->meta_surf_width_y = dcn_bw_ceil2(v->swath_width_y[k] - 1.0f, v->meta_request_width_y) + v->meta_request_width_y;
+			v->meta_surf_height_y = dcn_bw_ceil2(v->viewport_height[k] - 1.0f, 8.0f * v->block_height256_bytes_y) + 8.0f * v->block_height256_bytes_y;
 			if (v->pte_enable == dcn_bw_yes) {
-				v->meta_pte_bytes_frame_y = (dcn_bw_ceil2((v->meta_surf_width_y * v->meta_surf_height_y *dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / 256.0 - 4096.0) / 8.0 / 4096.0, 1.0) + 1) * 64.0;
-			}
-			else {
-				v->meta_pte_bytes_frame_y = 0.0;
+				v->meta_pte_bytes_frame_y = (dcn_bw_ceil2((v->meta_surf_width_y * v->meta_surf_height_y * dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / 256.0f - 4096.0f) / 8.0f / 4096.0f, 1.0f) + 1) * 64.0f;
+			} else {
+				v->meta_pte_bytes_frame_y = 0.0f;
 			}
 			if (v->source_scan[k] == dcn_bw_hor) {
-				v->meta_row_byte_y = v->meta_surf_width_y * 8.0 * v->block_height256_bytes_y *dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / 256.0;
-			}
-			else {
-				v->meta_row_byte_y = v->meta_surf_height_y * v->meta_request_width_y *dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / 256.0;
+				v->meta_row_byte_y = v->meta_surf_width_y * 8.0f * v->block_height256_bytes_y * dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / 256.0f;
+			} else {
+				v->meta_row_byte_y = v->meta_surf_height_y * v->meta_request_width_y * dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / 256.0f;
 			}
-		}
-		else {
-			v->meta_pte_bytes_frame_y = 0.0;
-			v->meta_row_byte_y = 0.0;
+		} else {
+			v->meta_pte_bytes_frame_y = 0.0f;
+			v->meta_row_byte_y = 0.0f;
 		}
 		if (v->pte_enable == dcn_bw_yes) {
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-				v->macro_tile_size_byte_y = 256.0;
-				v->macro_tile_height_y = 1.0;
-			}
-			else if (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d_x) {
-				v->macro_tile_size_byte_y = 4096.0;
-				v->macro_tile_height_y = 4.0 * v->block_height256_bytes_y;
-			}
-			else if (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x) {
-				v->macro_tile_size_byte_y = 64.0 * 1024;
-				v->macro_tile_height_y = 16.0 * v->block_height256_bytes_y;
-			}
-			else {
-				v->macro_tile_size_byte_y = 256.0 * 1024;
-				v->macro_tile_height_y = 32.0 * v->block_height256_bytes_y;
-			}
-			if (v->macro_tile_size_byte_y <= 65536.0) {
+				v->macro_tile_size_byte_y = 256.0f;
+				v->macro_tile_height_y = 1.0f;
+			} else if (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d_x) {
+				v->macro_tile_size_byte_y = 4096.0f;
+				v->macro_tile_height_y = 4.0f * v->block_height256_bytes_y;
+			} else if (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x) {
+				v->macro_tile_size_byte_y = 64.0f * 1024;
+				v->macro_tile_height_y = 16.0f * v->block_height256_bytes_y;
+			} else {
+				v->macro_tile_size_byte_y = 256.0f * 1024;
+				v->macro_tile_height_y = 32.0f * v->block_height256_bytes_y;
+			}
+			if (v->macro_tile_size_byte_y <= 65536.0f) {
 				v->pixel_pte_req_height_y = v->macro_tile_height_y;
+			} else {
+				v->pixel_pte_req_height_y = 16.0f * v->block_height256_bytes_y;
 			}
-			else {
-				v->pixel_pte_req_height_y = 16.0 * v->block_height256_bytes_y;
-			}
-			v->pixel_pte_req_width_y = 4096.0 /dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / v->pixel_pte_req_height_y * 8;
+			v->pixel_pte_req_width_y = 4096.0f / dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / v->pixel_pte_req_height_y * 8;
 			if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-				v->pixel_pte_bytes_per_row_y = 64.0 * (dcn_bw_ceil2((v->swath_width_y[k] *dcn_bw_min2(128.0, dcn_bw_pow(2.0,dcn_bw_floor2(dcn_bw_log(v->pte_buffer_size_in_requests * v->pixel_pte_req_width_y / v->swath_width_y[k], 2.0), 1.0))) - 1.0) / v->pixel_pte_req_width_y, 1.0) + 1);
-			}
-			else if (v->source_scan[k] == dcn_bw_hor) {
-				v->pixel_pte_bytes_per_row_y = 64.0 * (dcn_bw_ceil2((v->swath_width_y[k] - 1.0) / v->pixel_pte_req_width_y, 1.0) + 1);
+				v->pixel_pte_bytes_per_row_y = 64.0f * (dcn_bw_ceil2((v->swath_width_y[k] * dcn_bw_min2(128.0f, dcn_bw_pow(2.0f, dcn_bw_floor2(dcn_bw_log(v->pte_buffer_size_in_requests * v->pixel_pte_req_width_y / v->swath_width_y[k], 2.0f), 1.0f))) - 1.0f) / v->pixel_pte_req_width_y, 1.0f) + 1);
+			} else if (v->source_scan[k] == dcn_bw_hor) {
+				v->pixel_pte_bytes_per_row_y = 64.0f * (dcn_bw_ceil2((v->swath_width_y[k] - 1.0f) / v->pixel_pte_req_width_y, 1.0f) + 1);
+			} else {
+				v->pixel_pte_bytes_per_row_y = 64.0f * (dcn_bw_ceil2((v->viewport_height[k] - 1.0f) / v->pixel_pte_req_height_y, 1.0f) + 1);
 			}
-			else {
-				v->pixel_pte_bytes_per_row_y = 64.0 * (dcn_bw_ceil2((v->viewport_height[k] - 1.0) / v->pixel_pte_req_height_y, 1.0) + 1);
-			}
-		}
-		else {
-			v->pixel_pte_bytes_per_row_y = 0.0;
+		} else {
+			v->pixel_pte_bytes_per_row_y = 0.0f;
 		}
 		if ((v->source_pixel_format[k] != dcn_bw_rgb_sub_64 && v->source_pixel_format[k] != dcn_bw_rgb_sub_32 && v->source_pixel_format[k] != dcn_bw_rgb_sub_16)) {
 			if (v->dcc_enable[k] == dcn_bw_yes) {
-				v->meta_request_width_c = 64.0 * 256.0 /dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / (8.0 * v->block_height256_bytes_c);
-				v->meta_surf_width_c =dcn_bw_ceil2(v->swath_width_y[k] / 2.0 - 1.0, v->meta_request_width_c) + v->meta_request_width_c;
-				v->meta_surf_height_c =dcn_bw_ceil2(v->viewport_height[k] / 2.0 - 1.0, 8.0 * v->block_height256_bytes_c) + 8.0 * v->block_height256_bytes_c;
+				v->meta_request_width_c = 64.0f * 256.0f / dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / (8.0f * v->block_height256_bytes_c);
+				v->meta_surf_width_c = dcn_bw_ceil2(v->swath_width_y[k] / 2.0f - 1.0f, v->meta_request_width_c) + v->meta_request_width_c;
+				v->meta_surf_height_c = dcn_bw_ceil2(v->viewport_height[k] / 2.0f - 1.0f, 8.0f * v->block_height256_bytes_c) + 8.0f * v->block_height256_bytes_c;
 				if (v->pte_enable == dcn_bw_yes) {
-					v->meta_pte_bytes_frame_c = (dcn_bw_ceil2((v->meta_surf_width_c * v->meta_surf_height_c *dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / 256.0 - 4096.0) / 8.0 / 4096.0, 1.0) + 1) * 64.0;
-				}
-				else {
-					v->meta_pte_bytes_frame_c = 0.0;
+					v->meta_pte_bytes_frame_c = (dcn_bw_ceil2((v->meta_surf_width_c * v->meta_surf_height_c * dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / 256.0f - 4096.0f) / 8.0f / 4096.0f, 1.0f) + 1) * 64.0f;
+				} else {
+					v->meta_pte_bytes_frame_c = 0.0f;
 				}
 				if (v->source_scan[k] == dcn_bw_hor) {
-					v->meta_row_byte_c = v->meta_surf_width_c * 8.0 * v->block_height256_bytes_c *dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / 256.0;
-				}
-				else {
-					v->meta_row_byte_c = v->meta_surf_height_c * v->meta_request_width_c *dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / 256.0;
+					v->meta_row_byte_c = v->meta_surf_width_c * 8.0f * v->block_height256_bytes_c * dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / 256.0f;
+				} else {
+					v->meta_row_byte_c = v->meta_surf_height_c * v->meta_request_width_c * dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / 256.0f;
 				}
-			}
-			else {
-				v->meta_pte_bytes_frame_c = 0.0;
-				v->meta_row_byte_c = 0.0;
+			} else {
+				v->meta_pte_bytes_frame_c = 0.0f;
+				v->meta_row_byte_c = 0.0f;
 			}
 			if (v->pte_enable == dcn_bw_yes) {
 				if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-					v->macro_tile_size_bytes_c = 256.0;
-					v->macro_tile_height_c = 1.0;
-				}
-				else if (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d_x) {
-					v->macro_tile_size_bytes_c = 4096.0;
-					v->macro_tile_height_c = 4.0 * v->block_height256_bytes_c;
-				}
-				else if (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x) {
-					v->macro_tile_size_bytes_c = 64.0 * 1024;
-					v->macro_tile_height_c = 16.0 * v->block_height256_bytes_c;
-				}
-				else {
-					v->macro_tile_size_bytes_c = 256.0 * 1024;
-					v->macro_tile_height_c = 32.0 * v->block_height256_bytes_c;
-				}
-				if (v->macro_tile_size_bytes_c <= 65536.0) {
+					v->macro_tile_size_bytes_c = 256.0f;
+					v->macro_tile_height_c = 1.0f;
+				} else if (v->source_surface_mode[k] == dcn_bw_sw_4_kb_s || v->source_surface_mode[k] == dcn_bw_sw_4_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d || v->source_surface_mode[k] == dcn_bw_sw_4_kb_d_x) {
+					v->macro_tile_size_bytes_c = 4096.0f;
+					v->macro_tile_height_c = 4.0f * v->block_height256_bytes_c;
+				} else if (v->source_surface_mode[k] == dcn_bw_sw_64_kb_s || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_s_x || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_t || v->source_surface_mode[k] == dcn_bw_sw_64_kb_d_x) {
+					v->macro_tile_size_bytes_c = 64.0f * 1024;
+					v->macro_tile_height_c = 16.0f * v->block_height256_bytes_c;
+				} else {
+					v->macro_tile_size_bytes_c = 256.0f * 1024;
+					v->macro_tile_height_c = 32.0f * v->block_height256_bytes_c;
+				}
+				if (v->macro_tile_size_bytes_c <= 65536.0f) {
 					v->pixel_pte_req_height_c = v->macro_tile_height_c;
+				} else {
+					v->pixel_pte_req_height_c = 16.0f * v->block_height256_bytes_c;
 				}
-				else {
-					v->pixel_pte_req_height_c = 16.0 * v->block_height256_bytes_c;
-				}
-				v->pixel_pte_req_width_c = 4096.0 /dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / v->pixel_pte_req_height_c * 8;
+				v->pixel_pte_req_width_c = 4096.0f / dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / v->pixel_pte_req_height_c * 8;
 				if (v->source_surface_mode[k] == dcn_bw_sw_linear) {
-					v->pixel_pte_bytes_per_row_c = 64.0 * (dcn_bw_ceil2((v->swath_width_y[k] / 2.0 * dcn_bw_min2(128.0, dcn_bw_pow(2.0,dcn_bw_floor2(dcn_bw_log(v->pte_buffer_size_in_requests * v->pixel_pte_req_width_c / (v->swath_width_y[k] / 2.0), 2.0), 1.0))) - 1.0) / v->pixel_pte_req_width_c, 1.0) + 1);
+					v->pixel_pte_bytes_per_row_c = 64.0f * (dcn_bw_ceil2((v->swath_width_y[k] / 2.0f * dcn_bw_min2(128.0f, dcn_bw_pow(2.0f, dcn_bw_floor2(dcn_bw_log(v->pte_buffer_size_in_requests * v->pixel_pte_req_width_c / (v->swath_width_y[k] / 2.0f), 2.0f), 1.0f))) - 1.0f) / v->pixel_pte_req_width_c, 1.0f) + 1);
+				} else if (v->source_scan[k] == dcn_bw_hor) {
+					v->pixel_pte_bytes_per_row_c = 64.0f * (dcn_bw_ceil2((v->swath_width_y[k] / 2.0f - 1.0f) / v->pixel_pte_req_width_c, 1.0f) + 1);
+				} else {
+					v->pixel_pte_bytes_per_row_c = 64.0f * (dcn_bw_ceil2((v->viewport_height[k] / 2.0f - 1.0f) / v->pixel_pte_req_height_c, 1.0f) + 1);
 				}
-				else if (v->source_scan[k] == dcn_bw_hor) {
-					v->pixel_pte_bytes_per_row_c = 64.0 * (dcn_bw_ceil2((v->swath_width_y[k] / 2.0 - 1.0) / v->pixel_pte_req_width_c, 1.0) + 1);
-				}
-				else {
-					v->pixel_pte_bytes_per_row_c = 64.0 * (dcn_bw_ceil2((v->viewport_height[k] / 2.0 - 1.0) / v->pixel_pte_req_height_c, 1.0) + 1);
-				}
-			}
-			else {
-				v->pixel_pte_bytes_per_row_c = 0.0;
+			} else {
+				v->pixel_pte_bytes_per_row_c = 0.0f;
 			}
-		}
-		else {
-			v->pixel_pte_bytes_per_row_c = 0.0;
-			v->meta_pte_bytes_frame_c = 0.0;
-			v->meta_row_byte_c = 0.0;
+		} else {
+			v->pixel_pte_bytes_per_row_c = 0.0f;
+			v->meta_pte_bytes_frame_c = 0.0f;
+			v->meta_row_byte_c = 0.0f;
 		}
 		v->pixel_pte_bytes_per_row[k] = v->pixel_pte_bytes_per_row_y + v->pixel_pte_bytes_per_row_c;
 		v->meta_pte_bytes_frame[k] = v->meta_pte_bytes_frame_y + v->meta_pte_bytes_frame_c;
 		v->meta_row_byte[k] = v->meta_row_byte_y + v->meta_row_byte_c;
-		v->v_init_pre_fill_y[k] =dcn_bw_floor2((v->v_ratio[k] + v->vtaps[k] + 1.0 + v->interlace_output[k] * 0.5 * v->v_ratio[k]) / 2.0, 1.0);
-		v->max_num_swath_y[k] =dcn_bw_ceil2((v->v_init_pre_fill_y[k] - 1.0) / v->swath_height_y[k], 1.0) + 1;
-		if (v->v_init_pre_fill_y[k] > 1.0) {
-			v->max_partial_swath_y =dcn_bw_mod((v->v_init_pre_fill_y[k] - 2.0), v->swath_height_y[k]);
-		}
-		else {
-			v->max_partial_swath_y =dcn_bw_mod((v->v_init_pre_fill_y[k] + v->swath_height_y[k] - 2.0), v->swath_height_y[k]);
-		}
-		v->max_partial_swath_y =dcn_bw_max2(1.0, v->max_partial_swath_y);
+		v->v_init_pre_fill_y[k] = dcn_bw_floor2((v->v_ratio[k] + v->vtaps[k] + 1.0f + v->interlace_output[k] * 0.5f * v->v_ratio[k]) / 2.0f, 1.0f);
+		v->max_num_swath_y[k] = dcn_bw_ceil2((v->v_init_pre_fill_y[k] - 1.0f) / v->swath_height_y[k], 1.0f) + 1;
+		if (v->v_init_pre_fill_y[k] > 1.0f) {
+			v->max_partial_swath_y = dcn_bw_mod((v->v_init_pre_fill_y[k] - 2.0f), v->swath_height_y[k]);
+		} else {
+			v->max_partial_swath_y = dcn_bw_mod((v->v_init_pre_fill_y[k] + v->swath_height_y[k] - 2.0f), v->swath_height_y[k]);
+		}
+		v->max_partial_swath_y = dcn_bw_max2(1.0f, v->max_partial_swath_y);
 		v->prefetch_source_lines_y[k] = v->max_num_swath_y[k] * v->swath_height_y[k] + v->max_partial_swath_y;
 		if ((v->source_pixel_format[k] != dcn_bw_rgb_sub_64 && v->source_pixel_format[k] != dcn_bw_rgb_sub_32 && v->source_pixel_format[k] != dcn_bw_rgb_sub_16)) {
-			v->v_init_pre_fill_c[k] =dcn_bw_floor2((v->v_ratio[k] / 2.0 + v->vtaps[k] + 1.0 + v->interlace_output[k] * 0.5 * v->v_ratio[k] / 2.0) / 2.0, 1.0);
-			v->max_num_swath_c[k] =dcn_bw_ceil2((v->v_init_pre_fill_c[k] - 1.0) / v->swath_height_c[k], 1.0) + 1;
-			if (v->v_init_pre_fill_c[k] > 1.0) {
-				v->max_partial_swath_c =dcn_bw_mod((v->v_init_pre_fill_c[k] - 2.0), v->swath_height_c[k]);
-			}
-			else {
-				v->max_partial_swath_c =dcn_bw_mod((v->v_init_pre_fill_c[k] + v->swath_height_c[k] - 2.0), v->swath_height_c[k]);
-			}
-			v->max_partial_swath_c =dcn_bw_max2(1.0, v->max_partial_swath_c);
-		}
-		else {
-			v->max_num_swath_c[k] = 0.0;
-			v->max_partial_swath_c = 0.0;
+			v->v_init_pre_fill_c[k] = dcn_bw_floor2((v->v_ratio[k] / 2.0f + v->vtaps[k] + 1.0f + v->interlace_output[k] * 0.5f * v->v_ratio[k] / 2.0f) / 2.0f, 1.0f);
+			v->max_num_swath_c[k] = dcn_bw_ceil2((v->v_init_pre_fill_c[k] - 1.0f) / v->swath_height_c[k], 1.0f) + 1;
+			if (v->v_init_pre_fill_c[k] > 1.0f) {
+				v->max_partial_swath_c = dcn_bw_mod((v->v_init_pre_fill_c[k] - 2.0f), v->swath_height_c[k]);
+			} else {
+				v->max_partial_swath_c = dcn_bw_mod((v->v_init_pre_fill_c[k] + v->swath_height_c[k] - 2.0f), v->swath_height_c[k]);
+			}
+			v->max_partial_swath_c = dcn_bw_max2(1.0f, v->max_partial_swath_c);
+		} else {
+			v->max_num_swath_c[k] = 0.0f;
+			v->max_partial_swath_c = 0.0f;
 		}
 		v->prefetch_source_lines_c[k] = v->max_num_swath_c[k] * v->swath_height_c[k] + v->max_partial_swath_c;
 	}
-	v->t_calc = 24.0 / v->dcf_clk_deep_sleep;
+	v->t_calc = 24.0f / v->dcf_clk_deep_sleep;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one == dcn_bw_yes) {
-			v->max_vstartup_lines[k] = v->vtotal[k] - v->vactive[k] - 1.0;
-		}
-		else {
-			v->max_vstartup_lines[k] = v->v_sync_plus_back_porch[k] - 1.0;
+			v->max_vstartup_lines[k] = v->vtotal[k] - v->vactive[k] - 1.0f;
+		} else {
+			v->max_vstartup_lines[k] = v->v_sync_plus_back_porch[k] - 1.0f;
 		}
 	}
-	v->next_prefetch_mode = 0.0;
+	v->next_prefetch_mode = 0.0f;
 	do {
-		v->v_startup_lines = 13.0;
+		v->v_startup_lines = 13.0f;
 		do {
 			v->planes_with_room_to_increase_vstartup_prefetch_bw_less_than_active_bw = dcn_bw_yes;
 			v->planes_with_room_to_increase_vstartup_vratio_prefetch_more_than4 = dcn_bw_no;
@@ -1640,159 +1468,142 @@ void dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performan
 			v->destination_line_times_for_prefetch_less_than2 = dcn_bw_no;
 			v->prefetch_mode = v->next_prefetch_mode;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-				v->dstx_after_scaler = 90.0 * v->pixel_clock[k] / v->dppclk + 42.0 * v->pixel_clock[k] / v->dispclk;
-				if (v->dpp_per_plane[k] > 1.0) {
-					v->dstx_after_scaler = v->dstx_after_scaler + v->scaler_rec_out_width[k] / 2.0;
+				v->dstx_after_scaler = 90.0f * v->pixel_clock[k] / v->dppclk + 42.0f * v->pixel_clock[k] / v->dispclk;
+				if (v->dpp_per_plane[k] > 1.0f) {
+					v->dstx_after_scaler = v->dstx_after_scaler + v->scaler_rec_out_width[k] / 2.0f;
 				}
 				if (v->output_format[k] == dcn_bw_420) {
-					v->dsty_after_scaler = 1.0;
-				}
-				else {
-					v->dsty_after_scaler = 0.0;
-				}
-				v->v_update_offset_pix[k] = dcn_bw_ceil2(v->htotal[k] / 4.0, 1.0);
-				v->total_repeater_delay_time = v->max_inter_dcn_tile_repeaters * (2.0 / v->dppclk + 3.0 / v->dispclk);
-				v->v_update_width_pix[k] = (14.0 / v->dcf_clk_deep_sleep + 12.0 / v->dppclk + v->total_repeater_delay_time) * v->pixel_clock[k];
-				v->v_ready_offset_pix[k] = dcn_bw_max2(150.0 / v->dppclk, v->total_repeater_delay_time + 20.0 / v->dcf_clk_deep_sleep + 10.0 / v->dppclk) * v->pixel_clock[k];
+					v->dsty_after_scaler = 1.0f;
+				} else {
+					v->dsty_after_scaler = 0.0f;
+				}
+				v->v_update_offset_pix[k] = dcn_bw_ceil2(v->htotal[k] / 4.0f, 1.0f);
+				v->total_repeater_delay_time = v->max_inter_dcn_tile_repeaters * (2.0f / v->dppclk + 3.0f / v->dispclk);
+				v->v_update_width_pix[k] = (14.0f / v->dcf_clk_deep_sleep + 12.0f / v->dppclk + v->total_repeater_delay_time) * v->pixel_clock[k];
+				v->v_ready_offset_pix[k] = dcn_bw_max2(150.0f / v->dppclk, v->total_repeater_delay_time + 20.0f / v->dcf_clk_deep_sleep + 10.0f / v->dppclk) * v->pixel_clock[k];
 				v->t_setup = (v->v_update_offset_pix[k] + v->v_update_width_pix[k] + v->v_ready_offset_pix[k]) / v->pixel_clock[k];
-				v->v_startup[k] =dcn_bw_min2(v->v_startup_lines, v->max_vstartup_lines[k]);
-				if (v->prefetch_mode == 0.0) {
-					v->t_wait =dcn_bw_max3(v->dram_clock_change_latency + v->urgent_latency, v->sr_enter_plus_exit_time, v->urgent_latency);
-				}
-				else if (v->prefetch_mode == 1.0) {
-					v->t_wait =dcn_bw_max2(v->sr_enter_plus_exit_time, v->urgent_latency);
-				}
-				else {
+				v->v_startup[k] = dcn_bw_min2(v->v_startup_lines, v->max_vstartup_lines[k]);
+				if (v->prefetch_mode == 0.0f) {
+					v->t_wait = dcn_bw_max3(v->dram_clock_change_latency + v->urgent_latency, v->sr_enter_plus_exit_time, v->urgent_latency);
+				} else if (v->prefetch_mode == 1.0f) {
+					v->t_wait = dcn_bw_max2(v->sr_enter_plus_exit_time, v->urgent_latency);
+				} else {
 					v->t_wait = v->urgent_latency;
 				}
-				v->destination_lines_for_prefetch[k] =dcn_bw_floor2(4.0 * (v->v_startup[k] - v->t_wait / (v->htotal[k] / v->pixel_clock[k]) - (v->t_calc + v->t_setup) / (v->htotal[k] / v->pixel_clock[k]) - (v->dsty_after_scaler + v->dstx_after_scaler / v->htotal[k]) + 0.125), 1.0) / 4;
-				if (v->destination_lines_for_prefetch[k] > 0.0) {
-					v->prefetch_bandwidth[k] = (v->meta_pte_bytes_frame[k] + 2.0 * v->meta_row_byte[k] + 2.0 * v->pixel_pte_bytes_per_row[k] + v->prefetch_source_lines_y[k] * v->swath_width_y[k] *dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) + v->prefetch_source_lines_c[k] * v->swath_width_y[k] / 2.0 *dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0)) / (v->destination_lines_for_prefetch[k] * v->htotal[k] / v->pixel_clock[k]);
-				}
-				else {
-					v->prefetch_bandwidth[k] = 999999.0;
+				v->destination_lines_for_prefetch[k] = dcn_bw_floor2(4.0f * (v->v_startup[k] - v->t_wait / (v->htotal[k] / v->pixel_clock[k]) - (v->t_calc + v->t_setup) / (v->htotal[k] / v->pixel_clock[k]) - (v->dsty_after_scaler + v->dstx_after_scaler / v->htotal[k]) + 0.125f), 1.0f) / 4;
+				if (v->destination_lines_for_prefetch[k] > 0.0f) {
+					v->prefetch_bandwidth[k] = (v->meta_pte_bytes_frame[k] + 2.0f * v->meta_row_byte[k] + 2.0f * v->pixel_pte_bytes_per_row[k] + v->prefetch_source_lines_y[k] * v->swath_width_y[k] * dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) + v->prefetch_source_lines_c[k] * v->swath_width_y[k] / 2.0f * dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f)) / (v->destination_lines_for_prefetch[k] * v->htotal[k] / v->pixel_clock[k]);
+				} else {
+					v->prefetch_bandwidth[k] = 999999.0f;
 				}
 			}
 			v->bandwidth_available_for_immediate_flip = v->return_bw;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 				v->bandwidth_available_for_immediate_flip = v->bandwidth_available_for_immediate_flip -dcn_bw_max2(v->read_bandwidth_plane_luma[k] + v->read_bandwidth_plane_chroma[k], v->prefetch_bandwidth[k]);
 			}
-			v->tot_immediate_flip_bytes = 0.0;
+			v->tot_immediate_flip_bytes = 0.0f;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 				if (v->immediate_flip_supported == dcn_bw_yes && (v->source_pixel_format[k] != dcn_bw_yuv420_sub_8 && v->source_pixel_format[k] != dcn_bw_yuv420_sub_10)) {
 					v->tot_immediate_flip_bytes = v->tot_immediate_flip_bytes + v->meta_pte_bytes_frame[k] + v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k];
 				}
 			}
-			v->max_rd_bandwidth = 0.0;
+			v->max_rd_bandwidth = 0.0f;
 			for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 				if (v->pte_enable == dcn_bw_yes && v->dcc_enable[k] == dcn_bw_yes) {
 					if (v->immediate_flip_supported == dcn_bw_yes && (v->source_pixel_format[k] != dcn_bw_yuv420_sub_8 && v->source_pixel_format[k] != dcn_bw_yuv420_sub_10)) {
-						v->time_for_fetching_meta_pte =dcn_bw_max5(v->meta_pte_bytes_frame[k] / v->prefetch_bandwidth[k], v->meta_pte_bytes_frame[k] * v->tot_immediate_flip_bytes / (v->bandwidth_available_for_immediate_flip * (v->meta_pte_bytes_frame[k] + v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k])), v->urgent_extra_latency, v->urgent_latency, v->htotal[k] / v->pixel_clock[k] / 4.0);
-					}
-					else {
-						v->time_for_fetching_meta_pte =dcn_bw_max3(v->meta_pte_bytes_frame[k] / v->prefetch_bandwidth[k], v->urgent_extra_latency, v->htotal[k] / v->pixel_clock[k] / 4.0);
+						v->time_for_fetching_meta_pte = dcn_bw_max5(v->meta_pte_bytes_frame[k] / v->prefetch_bandwidth[k], v->meta_pte_bytes_frame[k] * v->tot_immediate_flip_bytes / (v->bandwidth_available_for_immediate_flip * (v->meta_pte_bytes_frame[k] + v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k])), v->urgent_extra_latency, v->urgent_latency, v->htotal[k] / v->pixel_clock[k] / 4.0f);
+					} else {
+						v->time_for_fetching_meta_pte = dcn_bw_max3(v->meta_pte_bytes_frame[k] / v->prefetch_bandwidth[k], v->urgent_extra_latency, v->htotal[k] / v->pixel_clock[k] / 4.0f);
 					}
+				} else {
+					v->time_for_fetching_meta_pte = v->htotal[k] / v->pixel_clock[k] / 4.0f;
 				}
-				else {
-					v->time_for_fetching_meta_pte = v->htotal[k] / v->pixel_clock[k] / 4.0;
-				}
-				v->destination_lines_to_request_vm_inv_blank[k] =dcn_bw_floor2(4.0 * (v->time_for_fetching_meta_pte / (v->htotal[k] / v->pixel_clock[k]) + 0.125), 1.0) / 4;
+				v->destination_lines_to_request_vm_inv_blank[k] = dcn_bw_floor2(4.0f * (v->time_for_fetching_meta_pte / (v->htotal[k] / v->pixel_clock[k]) + 0.125f), 1.0f) / 4;
 				if ((v->pte_enable == dcn_bw_yes || v->dcc_enable[k] == dcn_bw_yes)) {
 					if (v->immediate_flip_supported == dcn_bw_yes && (v->source_pixel_format[k] != dcn_bw_yuv420_sub_8 && v->source_pixel_format[k] != dcn_bw_yuv420_sub_10)) {
-						v->time_for_fetching_row_in_vblank =dcn_bw_max5((v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k]) / v->prefetch_bandwidth[k], (v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k]) * v->tot_immediate_flip_bytes / (v->bandwidth_available_for_immediate_flip * (v->meta_pte_bytes_frame[k] + v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k])), v->urgent_extra_latency, 2.0 * v->urgent_latency, v->htotal[k] / v->pixel_clock[k] - v->time_for_fetching_meta_pte);
-					}
-					else {
-						v->time_for_fetching_row_in_vblank =dcn_bw_max3((v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k]) / v->prefetch_bandwidth[k], v->urgent_extra_latency, v->htotal[k] / v->pixel_clock[k] - v->time_for_fetching_meta_pte);
+						v->time_for_fetching_row_in_vblank = dcn_bw_max5((v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k]) / v->prefetch_bandwidth[k], (v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k]) * v->tot_immediate_flip_bytes / (v->bandwidth_available_for_immediate_flip * (v->meta_pte_bytes_frame[k] + v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k])), v->urgent_extra_latency, 2.0f * v->urgent_latency, v->htotal[k] / v->pixel_clock[k] - v->time_for_fetching_meta_pte);
+					} else {
+						v->time_for_fetching_row_in_vblank = dcn_bw_max3((v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k]) / v->prefetch_bandwidth[k], v->urgent_extra_latency, v->htotal[k] / v->pixel_clock[k] - v->time_for_fetching_meta_pte);
 					}
+				} else {
+					v->time_for_fetching_row_in_vblank = dcn_bw_max2(v->urgent_extra_latency - v->time_for_fetching_meta_pte, v->htotal[k] / v->pixel_clock[k] - v->time_for_fetching_meta_pte);
 				}
-				else {
-					v->time_for_fetching_row_in_vblank =dcn_bw_max2(v->urgent_extra_latency - v->time_for_fetching_meta_pte, v->htotal[k] / v->pixel_clock[k] - v->time_for_fetching_meta_pte);
-				}
-				v->destination_lines_to_request_row_in_vblank[k] =dcn_bw_floor2(4.0 * (v->time_for_fetching_row_in_vblank / (v->htotal[k] / v->pixel_clock[k]) + 0.125), 1.0) / 4;
+				v->destination_lines_to_request_row_in_vblank[k] = dcn_bw_floor2(4.0f * (v->time_for_fetching_row_in_vblank / (v->htotal[k] / v->pixel_clock[k]) + 0.125f), 1.0f) / 4;
 				v->lines_to_request_prefetch_pixel_data = v->destination_lines_for_prefetch[k] - v->destination_lines_to_request_vm_inv_blank[k] - v->destination_lines_to_request_row_in_vblank[k];
-				if (v->lines_to_request_prefetch_pixel_data > 0.0) {
+				if (v->lines_to_request_prefetch_pixel_data > 0.0f) {
 					v->v_ratio_prefetch_y[k] = v->prefetch_source_lines_y[k] / v->lines_to_request_prefetch_pixel_data;
-					if ((v->swath_height_y[k] > 4.0)) {
-						if (v->lines_to_request_prefetch_pixel_data > (v->v_init_pre_fill_y[k] - 3.0) / 2.0) {
-							v->v_ratio_prefetch_y[k] =dcn_bw_max2(v->v_ratio_prefetch_y[k], v->max_num_swath_y[k] * v->swath_height_y[k] / (v->lines_to_request_prefetch_pixel_data - (v->v_init_pre_fill_y[k] - 3.0) / 2.0));
-						}
-						else {
-							v->v_ratio_prefetch_y[k] = 999999.0;
+					if ((v->swath_height_y[k] > 4.0f)) {
+						if (v->lines_to_request_prefetch_pixel_data > (v->v_init_pre_fill_y[k] - 3.0f) / 2.0f) {
+							v->v_ratio_prefetch_y[k] = dcn_bw_max2(v->v_ratio_prefetch_y[k], v->max_num_swath_y[k] * v->swath_height_y[k] / (v->lines_to_request_prefetch_pixel_data - (v->v_init_pre_fill_y[k] - 3.0f) / 2.0f));
+						} else {
+							v->v_ratio_prefetch_y[k] = 999999.0f;
 						}
 					}
+				} else {
+					v->v_ratio_prefetch_y[k] = 999999.0f;
 				}
-				else {
-					v->v_ratio_prefetch_y[k] = 999999.0;
-				}
-				v->v_ratio_prefetch_y[k] =dcn_bw_max2(v->v_ratio_prefetch_y[k], 1.0);
-				if (v->lines_to_request_prefetch_pixel_data > 0.0) {
+				v->v_ratio_prefetch_y[k] = dcn_bw_max2(v->v_ratio_prefetch_y[k], 1.0f);
+				if (v->lines_to_request_prefetch_pixel_data > 0.0f) {
 					v->v_ratio_prefetch_c[k] = v->prefetch_source_lines_c[k] / v->lines_to_request_prefetch_pixel_data;
-					if ((v->swath_height_c[k] > 4.0)) {
-						if (v->lines_to_request_prefetch_pixel_data > (v->v_init_pre_fill_c[k] - 3.0) / 2.0) {
-							v->v_ratio_prefetch_c[k] =dcn_bw_max2(v->v_ratio_prefetch_c[k], v->max_num_swath_c[k] * v->swath_height_c[k] / (v->lines_to_request_prefetch_pixel_data - (v->v_init_pre_fill_c[k] - 3.0) / 2.0));
-						}
-						else {
-							v->v_ratio_prefetch_c[k] = 999999.0;
+					if ((v->swath_height_c[k] > 4.0f)) {
+						if (v->lines_to_request_prefetch_pixel_data > (v->v_init_pre_fill_c[k] - 3.0f) / 2.0f) {
+							v->v_ratio_prefetch_c[k] = dcn_bw_max2(v->v_ratio_prefetch_c[k], v->max_num_swath_c[k] * v->swath_height_c[k] / (v->lines_to_request_prefetch_pixel_data - (v->v_init_pre_fill_c[k] - 3.0f) / 2.0f));
+						} else {
+							v->v_ratio_prefetch_c[k] = 999999.0f;
 						}
 					}
+				} else {
+					v->v_ratio_prefetch_c[k] = 999999.0f;
 				}
-				else {
-					v->v_ratio_prefetch_c[k] = 999999.0;
-				}
-				v->v_ratio_prefetch_c[k] =dcn_bw_max2(v->v_ratio_prefetch_c[k], 1.0);
-				if (v->lines_to_request_prefetch_pixel_data > 0.0) {
-					v->required_prefetch_pix_data_bw = v->dpp_per_plane[k] * (v->prefetch_source_lines_y[k] / v->lines_to_request_prefetch_pixel_data *dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) + v->prefetch_source_lines_c[k] / v->lines_to_request_prefetch_pixel_data *dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / 2.0) * v->swath_width_y[k] / (v->htotal[k] / v->pixel_clock[k]);
+				v->v_ratio_prefetch_c[k] = dcn_bw_max2(v->v_ratio_prefetch_c[k], 1.0f);
+				if (v->lines_to_request_prefetch_pixel_data > 0.0f) {
+					v->required_prefetch_pix_data_bw = v->dpp_per_plane[k] * (v->prefetch_source_lines_y[k] / v->lines_to_request_prefetch_pixel_data * dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) + v->prefetch_source_lines_c[k] / v->lines_to_request_prefetch_pixel_data * dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / 2.0f) * v->swath_width_y[k] / (v->htotal[k] / v->pixel_clock[k]);
+				} else {
+					v->required_prefetch_pix_data_bw = 999999.0f;
 				}
-				else {
-					v->required_prefetch_pix_data_bw = 999999.0;
-				}
-				v->max_rd_bandwidth = v->max_rd_bandwidth +dcn_bw_max2(v->read_bandwidth_plane_luma[k] + v->read_bandwidth_plane_chroma[k], v->required_prefetch_pix_data_bw);
+				v->max_rd_bandwidth = v->max_rd_bandwidth + dcn_bw_max2(v->read_bandwidth_plane_luma[k] + v->read_bandwidth_plane_chroma[k], v->required_prefetch_pix_data_bw);
 				if (v->immediate_flip_supported == dcn_bw_yes && (v->source_pixel_format[k] != dcn_bw_yuv420_sub_8 && v->source_pixel_format[k] != dcn_bw_yuv420_sub_10)) {
-					v->max_rd_bandwidth = v->max_rd_bandwidth +dcn_bw_max2(v->meta_pte_bytes_frame[k] / (v->destination_lines_to_request_vm_inv_blank[k] * v->htotal[k] / v->pixel_clock[k]), (v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k]) / (v->destination_lines_to_request_row_in_vblank[k] * v->htotal[k] / v->pixel_clock[k]));
+					v->max_rd_bandwidth = v->max_rd_bandwidth + dcn_bw_max2(v->meta_pte_bytes_frame[k] / (v->destination_lines_to_request_vm_inv_blank[k] * v->htotal[k] / v->pixel_clock[k]), (v->meta_row_byte[k] + v->pixel_pte_bytes_per_row[k]) / (v->destination_lines_to_request_row_in_vblank[k] * v->htotal[k] / v->pixel_clock[k]));
 				}
-				if (v->v_ratio_prefetch_y[k] > 4.0 || v->v_ratio_prefetch_c[k] > 4.0) {
+				if (v->v_ratio_prefetch_y[k] > 4.0f || v->v_ratio_prefetch_c[k] > 4.0f) {
 					v->v_ratio_prefetch_more_than4 = dcn_bw_yes;
 				}
-				if (v->destination_lines_for_prefetch[k] < 2.0) {
+				if (v->destination_lines_for_prefetch[k] < 2.0f) {
 					v->destination_line_times_for_prefetch_less_than2 = dcn_bw_yes;
 				}
 				if (v->max_vstartup_lines[k] > v->v_startup_lines) {
 					if (v->required_prefetch_pix_data_bw > (v->read_bandwidth_plane_luma[k] + v->read_bandwidth_plane_chroma[k])) {
 						v->planes_with_room_to_increase_vstartup_prefetch_bw_less_than_active_bw = dcn_bw_no;
 					}
-					if (v->v_ratio_prefetch_y[k] > 4.0 || v->v_ratio_prefetch_c[k] > 4.0) {
+					if (v->v_ratio_prefetch_y[k] > 4.0f || v->v_ratio_prefetch_c[k] > 4.0f) {
 						v->planes_with_room_to_increase_vstartup_vratio_prefetch_more_than4 = dcn_bw_yes;
 					}
-					if (v->destination_lines_for_prefetch[k] < 2.0) {
+					if (v->destination_lines_for_prefetch[k] < 2.0f) {
 						v->planes_with_room_to_increase_vstartup_destination_line_times_for_prefetch_less_than2 = dcn_bw_yes;
 					}
 				}
 			}
 			if (v->max_rd_bandwidth <= v->return_bw && v->v_ratio_prefetch_more_than4 == dcn_bw_no && v->destination_line_times_for_prefetch_less_than2 == dcn_bw_no) {
 				v->prefetch_mode_supported = dcn_bw_yes;
-			}
-			else {
+			} else {
 				v->prefetch_mode_supported = dcn_bw_no;
 			}
-			v->v_startup_lines = v->v_startup_lines + 1.0;
+			v->v_startup_lines = v->v_startup_lines + 1.0f;
 		} while (!(v->prefetch_mode_supported == dcn_bw_yes || (v->planes_with_room_to_increase_vstartup_prefetch_bw_less_than_active_bw == dcn_bw_yes && v->planes_with_room_to_increase_vstartup_vratio_prefetch_more_than4 == dcn_bw_no && v->planes_with_room_to_increase_vstartup_destination_line_times_for_prefetch_less_than2 == dcn_bw_no)));
-		v->next_prefetch_mode = v->next_prefetch_mode + 1.0;
-	} while (!(v->prefetch_mode_supported == dcn_bw_yes || v->prefetch_mode == 2.0));
+		v->next_prefetch_mode = v->next_prefetch_mode + 1.0f;
+	} while (!(v->prefetch_mode_supported == dcn_bw_yes || v->prefetch_mode == 2.0f));
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->v_ratio_prefetch_y[k] <= 1.0) {
+		if (v->v_ratio_prefetch_y[k] <= 1.0f) {
 			v->display_pipe_line_delivery_time_luma_prefetch[k] = v->swath_width_y[k] * v->dpp_per_plane[k] / v->h_ratio[k] / v->pixel_clock[k];
-		}
-		else {
+		} else {
 			v->display_pipe_line_delivery_time_luma_prefetch[k] = v->swath_width_y[k] / v->pscl_throughput[k] / v->dppclk;
 		}
-		if (v->byte_per_pixel_detc[k] == 0.0) {
-			v->display_pipe_line_delivery_time_chroma_prefetch[k] = 0.0;
-		}
-		else {
-			if (v->v_ratio_prefetch_c[k] <= 1.0) {
+		if (v->byte_per_pixel_detc[k] == 0.0f) {
+			v->display_pipe_line_delivery_time_chroma_prefetch[k] = 0.0f;
+		} else {
+			if (v->v_ratio_prefetch_c[k] <= 1.0f) {
 				v->display_pipe_line_delivery_time_chroma_prefetch[k] = v->swath_width_y[k] * v->dpp_per_plane[k] / v->h_ratio[k] / v->pixel_clock[k];
-			}
-			else {
+			} else {
 				v->display_pipe_line_delivery_time_chroma_prefetch[k] = v->swath_width_y[k] / v->pscl_throughput[k] / v->dppclk;
 			}
 		}
@@ -1800,17 +1611,15 @@ void dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performan
 	/*min ttuv_blank*/
 
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		if (v->prefetch_mode == 0.0) {
+		if (v->prefetch_mode == 0.0f) {
 			v->allow_dram_clock_change_during_vblank[k] = dcn_bw_yes;
 			v->allow_dram_self_refresh_during_vblank[k] = dcn_bw_yes;
-			v->min_ttuv_blank[k] = v->t_calc +dcn_bw_max3(v->dram_clock_change_watermark, v->stutter_enter_plus_exit_watermark, v->urgent_watermark);
-		}
-		else if (v->prefetch_mode == 1.0) {
+			v->min_ttuv_blank[k] = v->t_calc + dcn_bw_max3(v->dram_clock_change_watermark, v->stutter_enter_plus_exit_watermark, v->urgent_watermark);
+		} else if (v->prefetch_mode == 1.0f) {
 			v->allow_dram_clock_change_during_vblank[k] = dcn_bw_no;
 			v->allow_dram_self_refresh_during_vblank[k] = dcn_bw_yes;
-			v->min_ttuv_blank[k] = v->t_calc +dcn_bw_max2(v->stutter_enter_plus_exit_watermark, v->urgent_watermark);
-		}
-		else {
+			v->min_ttuv_blank[k] = v->t_calc + dcn_bw_max2(v->stutter_enter_plus_exit_watermark, v->urgent_watermark);
+		} else {
 			v->allow_dram_clock_change_during_vblank[k] = dcn_bw_no;
 			v->allow_dram_self_refresh_during_vblank[k] = dcn_bw_no;
 			v->min_ttuv_blank[k] = v->t_calc + v->urgent_watermark;
@@ -1818,115 +1627,102 @@ void dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performan
 	}
 	/*nb p-state/dram clock change support*/
 
-	v->active_dp_ps = 0.0;
+	v->active_dp_ps = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		v->active_dp_ps = v->active_dp_ps + v->dpp_per_plane[k];
 	}
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
-		v->lb_latency_hiding_source_lines_y =dcn_bw_min2(v->max_line_buffer_lines,dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_y[k] /dcn_bw_max2(v->h_ratio[k], 1.0)), 1.0)) - (v->vtaps[k] - 1.0);
-		v->lb_latency_hiding_source_lines_c =dcn_bw_min2(v->max_line_buffer_lines,dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_y[k] / 2.0 /dcn_bw_max2(v->h_ratio[k] / 2.0, 1.0)), 1.0)) - (v->vta_pschroma[k] - 1.0);
+		v->lb_latency_hiding_source_lines_y = dcn_bw_min2(v->max_line_buffer_lines, dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_y[k] / dcn_bw_max2(v->h_ratio[k], 1.0f)), 1.0f)) - (v->vtaps[k] - 1.0f);
+		v->lb_latency_hiding_source_lines_c = dcn_bw_min2(v->max_line_buffer_lines, dcn_bw_floor2(v->line_buffer_size / v->lb_bit_per_pixel[k] / (v->swath_width_y[k] / 2.0f / dcn_bw_max2(v->h_ratio[k] / 2.0f, 1.0f)), 1.0f)) - (v->vta_pschroma[k] - 1.0f);
 		v->effective_lb_latency_hiding_y = v->lb_latency_hiding_source_lines_y / v->v_ratio[k] * (v->htotal[k] / v->pixel_clock[k]);
-		v->effective_lb_latency_hiding_c = v->lb_latency_hiding_source_lines_c / (v->v_ratio[k] / 2.0) * (v->htotal[k] / v->pixel_clock[k]);
-		if (v->swath_width_y[k] > 2.0 * v->dpp_output_buffer_pixels) {
+		v->effective_lb_latency_hiding_c = v->lb_latency_hiding_source_lines_c / (v->v_ratio[k] / 2.0f) * (v->htotal[k] / v->pixel_clock[k]);
+		if (v->swath_width_y[k] > 2.0f * v->dpp_output_buffer_pixels) {
 			v->dpp_output_buffer_lines_y = v->dpp_output_buffer_pixels / v->swath_width_y[k];
-		}
-		else if (v->swath_width_y[k] > v->dpp_output_buffer_pixels) {
-			v->dpp_output_buffer_lines_y = 0.5;
-		}
-		else {
-			v->dpp_output_buffer_lines_y = 1.0;
-		}
-		if (v->swath_width_y[k] / 2.0 > 2.0 * v->dpp_output_buffer_pixels) {
-			v->dpp_output_buffer_lines_c = v->dpp_output_buffer_pixels / (v->swath_width_y[k] / 2.0);
-		}
-		else if (v->swath_width_y[k] / 2.0 > v->dpp_output_buffer_pixels) {
-			v->dpp_output_buffer_lines_c = 0.5;
-		}
-		else {
-			v->dpp_output_buffer_lines_c = 1.0;
+		} else if (v->swath_width_y[k] > v->dpp_output_buffer_pixels) {
+			v->dpp_output_buffer_lines_y = 0.5f;
+		} else {
+			v->dpp_output_buffer_lines_y = 1.0f;
+		}
+		if (v->swath_width_y[k] / 2.0f > 2.0f * v->dpp_output_buffer_pixels) {
+			v->dpp_output_buffer_lines_c = v->dpp_output_buffer_pixels / (v->swath_width_y[k] / 2.0f);
+		} else if (v->swath_width_y[k] / 2.0f > v->dpp_output_buffer_pixels) {
+			v->dpp_output_buffer_lines_c = 0.5f;
+		} else {
+			v->dpp_output_buffer_lines_c = 1.0f;
 		}
 		v->dppopp_buffering_y = (v->htotal[k] / v->pixel_clock[k]) * (v->dpp_output_buffer_lines_y + v->opp_output_buffer_lines);
 		v->max_det_buffering_time_y = v->full_det_buffering_time_y[k] + (v->lines_in_dety[k] - v->lines_in_dety_rounded_down_to_swath[k]) / v->swath_height_y[k] * (v->htotal[k] / v->pixel_clock[k]);
 		v->active_dram_clock_change_latency_margin_y = v->dppopp_buffering_y + v->effective_lb_latency_hiding_y + v->max_det_buffering_time_y - v->dram_clock_change_watermark;
-		if (v->active_dp_ps > 1.0) {
-			v->active_dram_clock_change_latency_margin_y = v->active_dram_clock_change_latency_margin_y - (1.0 - 1.0 / (v->active_dp_ps - 1.0)) * v->swath_height_y[k] * (v->htotal[k] / v->pixel_clock[k]);
+		if (v->active_dp_ps > 1.0f) {
+			v->active_dram_clock_change_latency_margin_y = v->active_dram_clock_change_latency_margin_y - (1.0f - 1.0f / (v->active_dp_ps - 1.0f)) * v->swath_height_y[k] * (v->htotal[k] / v->pixel_clock[k]);
 		}
-		if (v->byte_per_pixel_detc[k] > 0.0) {
+		if (v->byte_per_pixel_detc[k] > 0.0f) {
 			v->dppopp_buffering_c = (v->htotal[k] / v->pixel_clock[k]) * (v->dpp_output_buffer_lines_c + v->opp_output_buffer_lines);
 			v->max_det_buffering_time_c = v->full_det_buffering_time_c[k] + (v->lines_in_detc[k] - v->lines_in_detc_rounded_down_to_swath[k]) / v->swath_height_c[k] * (v->htotal[k] / v->pixel_clock[k]);
 			v->active_dram_clock_change_latency_margin_c = v->dppopp_buffering_c + v->effective_lb_latency_hiding_c + v->max_det_buffering_time_c - v->dram_clock_change_watermark;
-			if (v->active_dp_ps > 1.0) {
-				v->active_dram_clock_change_latency_margin_c = v->active_dram_clock_change_latency_margin_c - (1.0 - 1.0 / (v->active_dp_ps - 1.0)) * v->swath_height_c[k] * (v->htotal[k] / v->pixel_clock[k]);
+			if (v->active_dp_ps > 1.0f) {
+				v->active_dram_clock_change_latency_margin_c = v->active_dram_clock_change_latency_margin_c - (1.0f - 1.0f / (v->active_dp_ps - 1.0f)) * v->swath_height_c[k] * (v->htotal[k] / v->pixel_clock[k]);
 			}
-			v->active_dram_clock_change_latency_margin[k] =dcn_bw_min2(v->active_dram_clock_change_latency_margin_y, v->active_dram_clock_change_latency_margin_c);
-		}
-		else {
+			v->active_dram_clock_change_latency_margin[k] = dcn_bw_min2(v->active_dram_clock_change_latency_margin_y, v->active_dram_clock_change_latency_margin_c);
+		} else {
 			v->active_dram_clock_change_latency_margin[k] = v->active_dram_clock_change_latency_margin_y;
 		}
 		if (v->output_format[k] == dcn_bw_444) {
-			v->writeback_dram_clock_change_latency_margin = (v->writeback_luma_buffer_size + v->writeback_chroma_buffer_size) * 1024.0 / (v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 4.0) - v->writeback_dram_clock_change_watermark;
-		}
-		else {
-			v->writeback_dram_clock_change_latency_margin =dcn_bw_min2(v->writeback_luma_buffer_size, 2.0 * v->writeback_chroma_buffer_size) * 1024.0 / (v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k])) - v->writeback_dram_clock_change_watermark;
+			v->writeback_dram_clock_change_latency_margin = (v->writeback_luma_buffer_size + v->writeback_chroma_buffer_size) * 1024.0f / (v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 4.0f) - v->writeback_dram_clock_change_watermark;
+		} else {
+			v->writeback_dram_clock_change_latency_margin = dcn_bw_min2(v->writeback_luma_buffer_size, 2.0f * v->writeback_chroma_buffer_size) * 1024.0f / (v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k])) - v->writeback_dram_clock_change_watermark;
 		}
 		if (v->output[k] == dcn_bw_writeback) {
-			v->active_dram_clock_change_latency_margin[k] =dcn_bw_min2(v->active_dram_clock_change_latency_margin[k], v->writeback_dram_clock_change_latency_margin);
+			v->active_dram_clock_change_latency_margin[k] = dcn_bw_min2(v->active_dram_clock_change_latency_margin[k], v->writeback_dram_clock_change_latency_margin);
 		}
 	}
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->allow_dram_clock_change_during_vblank[k] == dcn_bw_yes) {
 			v->v_blank_dram_clock_change_latency_margin[k] = (v->vtotal[k] - v->scaler_recout_height[k]) * (v->htotal[k] / v->pixel_clock[k]) -dcn_bw_max2(v->dram_clock_change_watermark, v->writeback_dram_clock_change_watermark);
-		}
-		else {
-			v->v_blank_dram_clock_change_latency_margin[k] = 0.0;
+		} else {
+			v->v_blank_dram_clock_change_latency_margin[k] = 0.0f;
 		}
 	}
-	v->min_active_dram_clock_change_margin = 999999.0;
-	v->v_blank_of_min_active_dram_clock_change_margin = 999999.0;
-	v->second_min_active_dram_clock_change_margin = 999999.0;
+	v->min_active_dram_clock_change_margin = 999999.0f;
+	v->v_blank_of_min_active_dram_clock_change_margin = 999999.0f;
+	v->second_min_active_dram_clock_change_margin = 999999.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->active_dram_clock_change_latency_margin[k] < v->min_active_dram_clock_change_margin) {
 			v->second_min_active_dram_clock_change_margin = v->min_active_dram_clock_change_margin;
 			v->min_active_dram_clock_change_margin = v->active_dram_clock_change_latency_margin[k];
 			v->v_blank_of_min_active_dram_clock_change_margin = v->v_blank_dram_clock_change_latency_margin[k];
-		}
-		else if (v->active_dram_clock_change_latency_margin[k] < v->second_min_active_dram_clock_change_margin) {
+		} else if (v->active_dram_clock_change_latency_margin[k] < v->second_min_active_dram_clock_change_margin) {
 			v->second_min_active_dram_clock_change_margin = v->active_dram_clock_change_latency_margin[k];
 		}
 	}
-	v->min_vblank_dram_clock_change_margin = 999999.0;
+	v->min_vblank_dram_clock_change_margin = 999999.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->min_vblank_dram_clock_change_margin > v->v_blank_dram_clock_change_latency_margin[k]) {
 			v->min_vblank_dram_clock_change_margin = v->v_blank_dram_clock_change_latency_margin[k];
 		}
 	}
 	if (v->synchronized_vblank == dcn_bw_yes || v->number_of_active_planes == 1) {
-		v->dram_clock_change_margin =dcn_bw_max2(v->min_active_dram_clock_change_margin, v->min_vblank_dram_clock_change_margin);
-	}
-	else if (v->v_blank_of_min_active_dram_clock_change_margin > v->min_active_dram_clock_change_margin) {
-		v->dram_clock_change_margin =dcn_bw_min2(v->second_min_active_dram_clock_change_margin, v->v_blank_of_min_active_dram_clock_change_margin);
-	}
-	else {
+		v->dram_clock_change_margin = dcn_bw_max2(v->min_active_dram_clock_change_margin, v->min_vblank_dram_clock_change_margin);
+	} else if (v->v_blank_of_min_active_dram_clock_change_margin > v->min_active_dram_clock_change_margin) {
+		v->dram_clock_change_margin = dcn_bw_min2(v->second_min_active_dram_clock_change_margin, v->v_blank_of_min_active_dram_clock_change_margin);
+	} else {
 		v->dram_clock_change_margin = v->min_active_dram_clock_change_margin;
 	}
-	if (v->min_active_dram_clock_change_margin > 0.0) {
+	if (v->min_active_dram_clock_change_margin > 0.0f) {
 		v->dram_clock_change_support = dcn_bw_supported_in_v_active;
-	}
-	else if (v->dram_clock_change_margin > 0.0) {
+	} else if (v->dram_clock_change_margin > 0.0f) {
 		v->dram_clock_change_support = dcn_bw_supported_in_v_blank;
-	}
-	else {
+	} else {
 		v->dram_clock_change_support = dcn_bw_not_supported;
 	}
 	/*maximum bandwidth used*/
 
-	v->wr_bandwidth = 0.0;
+	v->wr_bandwidth = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->output[k] == dcn_bw_writeback && v->output_format[k] == dcn_bw_444) {
-			v->wr_bandwidth = v->wr_bandwidth + v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 4.0;
-		}
-		else if (v->output[k] == dcn_bw_writeback) {
-			v->wr_bandwidth = v->wr_bandwidth + v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 1.5;
+			v->wr_bandwidth = v->wr_bandwidth + v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 4.0f;
+		} else if (v->output[k] == dcn_bw_writeback) {
+			v->wr_bandwidth = v->wr_bandwidth + v->scaler_rec_out_width[k] / (v->htotal[k] / v->pixel_clock[k]) * 1.5f;
 		}
 	}
 	v->max_used_bw = v->max_rd_bandwidth + v->wr_bandwidth;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c
index cac72413a0977..16b990147d1dd 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calc_math.c
@@ -76,12 +76,12 @@ float dcn_bw_floor2(const float arg, const float significance)
 }
 float dcn_bw_floor(const float arg)
 {
-	return ((int) (arg));
+	return (float)((int)(arg));
 }
 
 float dcn_bw_ceil(const float arg)
 {
-	return (int) (arg + 0.99999);
+	return (float)((int)(arg + 0.99999f));
 }
 
 float dcn_bw_ceil2(const float arg, const float significance)
@@ -107,7 +107,7 @@ float dcn_bw_pow(float a, float exp)
 	/*ASSERT(exp == (int)exp);*/
 	if ((int)exp == 0)
 		return 1;
-	temp = dcn_bw_pow(a, (int)(exp / 2));
+	temp = dcn_bw_pow(a, (float)((int)(exp / 2)));
 	if (((int)exp % 2) == 0) {
 		return temp * temp;
 	} else {
diff --git a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
index 61553e24d53e4..dcca23d53261e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/calcs/dcn_calcs.c
@@ -72,21 +72,21 @@ const struct dcn_soc_bounding_box dcn10_soc_defaults = {
 		 * Use these value if fused value doesn't make sense as earlier
 		 * part don't have correct value fused */
 		/* default DCF CLK DPM on RV*/
-		.dcfclkv_max0p9 = 655,	/* MHz, = 3600/5.5 */
-		.dcfclkv_nom0p8 = 626,	/* MHz, = 3600/5.75 */
+		.dcfclkv_max0p9 = 655,	/* MHz, = 3600/5.5f */
+		.dcfclkv_nom0p8 = 626,	/* MHz, = 3600/5.75f */
 		.dcfclkv_mid0p72 = 600,	/* MHz, = 3600/6, bypass */
 		.dcfclkv_min0p65 = 300,	/* MHz, = 3600/12, bypass */
 
 		/* default DISP CLK voltage state on RV */
-		.max_dispclk_vmax0p9 = 1108,	/* MHz, = 3600/3.25 */
-		.max_dispclk_vnom0p8 = 1029,	/* MHz, = 3600/3.5 */
-		.max_dispclk_vmid0p72 = 960,	/* MHz, = 3600/3.75 */
-		.max_dispclk_vmin0p65 = 626,	/* MHz, = 3600/5.75 */
+		.max_dispclk_vmax0p9 = 1108,	/* MHz, = 3600/3.25f */
+		.max_dispclk_vnom0p8 = 1029,	/* MHz, = 3600/3.5f */
+		.max_dispclk_vmid0p72 = 960,	/* MHz, = 3600/3.75f */
+		.max_dispclk_vmin0p65 = 626,	/* MHz, = 3600/5.75f */
 
 		/* default DPP CLK voltage state on RV */
 		.max_dppclk_vmax0p9 = 720,	/* MHz, = 3600/5 */
-		.max_dppclk_vnom0p8 = 686,	/* MHz, = 3600/5.25 */
-		.max_dppclk_vmid0p72 = 626,	/* MHz, = 3600/5.75 */
+		.max_dppclk_vnom0p8 = 686,	/* MHz, = 3600/5.25f */
+		.max_dppclk_vmid0p72 = 626,	/* MHz, = 3600/5.75f */
 		.max_dppclk_vmin0p65 = 400,	/* MHz, = 3600/9 */
 
 		/* default PHY CLK voltage state on RV */
@@ -326,7 +326,7 @@ static void pipe_ctx_to_e2e_pipe_params (
 		 * allow us to disable dcc on the fly without re-calculating WM
 		 *
 		 * extra overhead for DCC is quite small.  for 1080p WM without
-		 * DCC is only 0.417us lower (urgent goes from 6.979us to 6.562us)
+		 * DCC is only 0.417fus lower (urgent goes from 6.979fus to 6.562fus)
 		 */
 		unsigned int bpe;
 
@@ -396,18 +396,18 @@ static void pipe_ctx_to_e2e_pipe_params (
 	}
 
 	input->scale_taps.htaps                = pipe->plane_res.scl_data.taps.h_taps;
-	input->scale_ratio_depth.hscl_ratio    = pipe->plane_res.scl_data.ratios.horz.value/4294967296.0;
-	input->scale_ratio_depth.vscl_ratio    = pipe->plane_res.scl_data.ratios.vert.value/4294967296.0;
-	input->scale_ratio_depth.vinit =  pipe->plane_res.scl_data.inits.v.value/4294967296.0;
-	if (input->scale_ratio_depth.vinit < 1.0)
+	input->scale_ratio_depth.hscl_ratio    = pipe->plane_res.scl_data.ratios.horz.value/4294967296.0f;
+	input->scale_ratio_depth.vscl_ratio    = pipe->plane_res.scl_data.ratios.vert.value/4294967296.0f;
+	input->scale_ratio_depth.vinit =  pipe->plane_res.scl_data.inits.v.value/4294967296.0f;
+	if (input->scale_ratio_depth.vinit < 1.0f)
 			input->scale_ratio_depth.vinit = 1;
 	input->scale_taps.vtaps = pipe->plane_res.scl_data.taps.v_taps;
 	input->scale_taps.vtaps_c = pipe->plane_res.scl_data.taps.v_taps_c;
 	input->scale_taps.htaps_c              = pipe->plane_res.scl_data.taps.h_taps_c;
-	input->scale_ratio_depth.hscl_ratio_c  = pipe->plane_res.scl_data.ratios.horz_c.value/4294967296.0;
-	input->scale_ratio_depth.vscl_ratio_c  = pipe->plane_res.scl_data.ratios.vert_c.value/4294967296.0;
-	input->scale_ratio_depth.vinit_c       = pipe->plane_res.scl_data.inits.v_c.value/4294967296.0;
-	if (input->scale_ratio_depth.vinit_c < 1.0)
+	input->scale_ratio_depth.hscl_ratio_c  = pipe->plane_res.scl_data.ratios.horz_c.value/4294967296.0f;
+	input->scale_ratio_depth.vscl_ratio_c  = pipe->plane_res.scl_data.ratios.vert_c.value/4294967296.0f;
+	input->scale_ratio_depth.vinit_c       = pipe->plane_res.scl_data.inits.v_c.value/4294967296.0f;
+	if (input->scale_ratio_depth.vinit_c < 1.0f)
 			input->scale_ratio_depth.vinit_c = 1;
 	switch (pipe->plane_res.scl_data.lb_params.depth) {
 	case LB_PIXEL_DEPTH_30BPP:
@@ -441,7 +441,7 @@ static void pipe_ctx_to_e2e_pipe_params (
 			- pipe->stream->timing.v_addressable
 			- pipe->stream->timing.v_border_bottom
 			- pipe->stream->timing.v_border_top;
-	input->dest.pixel_rate_mhz = pipe->stream->timing.pix_clk_100hz/10000.0;
+	input->dest.pixel_rate_mhz = pipe->stream->timing.pix_clk_100hz/10000.0f;
 	input->dest.vstartup_start = pipe->pipe_dlg_param.vstartup_start;
 	input->dest.vupdate_offset = pipe->pipe_dlg_param.vupdate_offset;
 	input->dest.vupdate_offset = pipe->pipe_dlg_param.vupdate_offset;
@@ -477,10 +477,10 @@ static void dcn_bw_calc_rq_dlg_ttu(
 	for (i = 0; i < number_of_planes; i++) {
 		total_active_bw += v->read_bandwidth[i];
 		total_prefetch_bw += v->prefetch_bandwidth[i];
-		total_flip_bytes += v->total_immediate_flip_bytes[i];
+		total_flip_bytes += (int)v->total_immediate_flip_bytes[i];
 	}
 	dlg_sys_param->total_flip_bw = v->return_bw - dcn_bw_max2(total_active_bw, total_prefetch_bw);
-	if (dlg_sys_param->total_flip_bw < 0.0)
+	if (dlg_sys_param->total_flip_bw < 0.0f)
 		dlg_sys_param->total_flip_bw = 0;
 
 	dlg_sys_param->t_mclk_wm_us = v->dram_clock_change_watermark;
@@ -494,7 +494,7 @@ static void dcn_bw_calc_rq_dlg_ttu(
 	input->clks_cfg.dcfclk_mhz = v->dcfclk;
 	input->clks_cfg.dispclk_mhz = v->dispclk;
 	input->clks_cfg.dppclk_mhz = v->dppclk;
-	input->clks_cfg.refclk_mhz = dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000.0;
+	input->clks_cfg.refclk_mhz = dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000.0f;
 	input->clks_cfg.socclk_mhz = v->socclk;
 	input->clks_cfg.voltage = v->voltage_level;
 //	dc->dml.logger = pool->base.logger;
@@ -533,13 +533,14 @@ static void split_stream_across_pipes(
 
 	*secondary_pipe = *primary_pipe;
 
-	secondary_pipe->pipe_idx = pipe_idx;
+	secondary_pipe->pipe_idx = (uint8_t)pipe_idx;
 	secondary_pipe->plane_res.mi = pool->mis[secondary_pipe->pipe_idx];
 	secondary_pipe->plane_res.hubp = pool->hubps[secondary_pipe->pipe_idx];
 	secondary_pipe->plane_res.ipp = pool->ipps[secondary_pipe->pipe_idx];
 	secondary_pipe->plane_res.xfm = pool->transforms[secondary_pipe->pipe_idx];
 	secondary_pipe->plane_res.dpp = pool->dpps[secondary_pipe->pipe_idx];
-	secondary_pipe->plane_res.mpcc_inst = pool->dpps[secondary_pipe->pipe_idx]->inst;
+	secondary_pipe->plane_res.mpcc_inst =
+			(uint8_t)pool->dpps[secondary_pipe->pipe_idx]->inst;
 	if (primary_pipe->bottom_pipe) {
 		ASSERT(primary_pipe->bottom_pipe != secondary_pipe);
 		secondary_pipe->bottom_pipe = primary_pipe->bottom_pipe;
@@ -643,7 +644,7 @@ static bool dcn_bw_apply_registry_override(struct dc *dc)
 	if ((int)(dc->dcn_soc->sr_exit_time * 1000) != dc->debug.sr_exit_time_ns
 			&& dc->debug.sr_exit_time_ns) {
 		updated = true;
-		dc->dcn_soc->sr_exit_time = dc->debug.sr_exit_time_ns / 1000.0;
+		dc->dcn_soc->sr_exit_time = dc->debug.sr_exit_time_ns / 1000.0f;
 	}
 
 	if ((int)(dc->dcn_soc->sr_enter_plus_exit_time * 1000)
@@ -651,13 +652,13 @@ static bool dcn_bw_apply_registry_override(struct dc *dc)
 			&& dc->debug.sr_enter_plus_exit_time_ns) {
 		updated = true;
 		dc->dcn_soc->sr_enter_plus_exit_time =
-				dc->debug.sr_enter_plus_exit_time_ns / 1000.0;
+				dc->debug.sr_enter_plus_exit_time_ns / 1000.0f;
 	}
 
 	if ((int)(dc->dcn_soc->urgent_latency * 1000) != dc->debug.urgent_latency_ns
 			&& dc->debug.urgent_latency_ns) {
 		updated = true;
-		dc->dcn_soc->urgent_latency = dc->debug.urgent_latency_ns / 1000.0;
+		dc->dcn_soc->urgent_latency = dc->debug.urgent_latency_ns / 1000.0f;
 	}
 
 	if ((int)(dc->dcn_soc->percent_of_ideal_drambw_received_after_urg_latency * 1000)
@@ -665,7 +666,7 @@ static bool dcn_bw_apply_registry_override(struct dc *dc)
 			&& dc->debug.percent_of_ideal_drambw) {
 		updated = true;
 		dc->dcn_soc->percent_of_ideal_drambw_received_after_urg_latency =
-				dc->debug.percent_of_ideal_drambw;
+				(float)dc->debug.percent_of_ideal_drambw;
 	}
 
 	if ((int)(dc->dcn_soc->dram_clock_change_latency * 1000)
@@ -673,7 +674,7 @@ static bool dcn_bw_apply_registry_override(struct dc *dc)
 			&& dc->debug.dram_clock_change_latency_ns) {
 		updated = true;
 		dc->dcn_soc->dram_clock_change_latency =
-				dc->debug.dram_clock_change_latency_ns / 1000.0;
+				dc->debug.dram_clock_change_latency_ns / 1000.0f;
 	}
 
 	return updated;
@@ -691,7 +692,7 @@ static void hack_disable_optional_pipe_split(struct dcn_bw_internal_vars *v)
 static void hack_force_pipe_split(struct dcn_bw_internal_vars *v,
 		unsigned int pixel_rate_100hz)
 {
-	float pixel_rate_mhz = pixel_rate_100hz / 10000.0;
+	float pixel_rate_mhz = pixel_rate_100hz / 10000.0f;
 
 	/*
 	 * force enabling pipe split by lower dpp clock for DPM0 to just
@@ -759,7 +760,8 @@ bool dcn_validate_bandwidth(
 
 	const struct resource_pool *pool = dc->res_pool;
 	struct dcn_bw_internal_vars *v = &context->dcn_bw_vars;
-	int i, input_idx, k;
+	unsigned int i;
+	int input_idx, k;
 	int vesa_sync_start, asic_blank_end, asic_blank_start;
 	bool bw_limit_pass;
 	float bw_limit;
@@ -808,12 +810,12 @@ bool dcn_validate_bandwidth(
 	v->phyclkv_max0p9 = dc->dcn_soc->phyclkv_max0p9;
 
 	v->downspreading = dc->dcn_soc->downspreading;
-	v->round_trip_ping_latency_cycles = dc->dcn_soc->round_trip_ping_latency_cycles;
-	v->urgent_out_of_order_return_per_channel = dc->dcn_soc->urgent_out_of_order_return_per_channel;
-	v->number_of_channels = dc->dcn_soc->number_of_channels;
-	v->vmm_page_size = dc->dcn_soc->vmm_page_size;
+	v->round_trip_ping_latency_cycles = (float)dc->dcn_soc->round_trip_ping_latency_cycles;
+	v->urgent_out_of_order_return_per_channel = (float)dc->dcn_soc->urgent_out_of_order_return_per_channel;
+	v->number_of_channels = (float)dc->dcn_soc->number_of_channels;
+	v->vmm_page_size = (float)dc->dcn_soc->vmm_page_size;
 	v->dram_clock_change_latency = dc->dcn_soc->dram_clock_change_latency;
-	v->return_bus_width = dc->dcn_soc->return_bus_width;
+	v->return_bus_width = (float)dc->dcn_soc->return_bus_width;
 
 	v->rob_buffer_size_in_kbyte = dc->dcn_ip->rob_buffer_size_in_kbyte;
 	v->det_buffer_size_in_kbyte = dc->dcn_ip->det_buffer_size_in_kbyte;
@@ -821,31 +823,31 @@ bool dcn_validate_bandwidth(
 	v->opp_output_buffer_lines = dc->dcn_ip->opp_output_buffer_lines;
 	v->pixel_chunk_size_in_kbyte = dc->dcn_ip->pixel_chunk_size_in_kbyte;
 	v->pte_enable = dc->dcn_ip->pte_enable;
-	v->pte_chunk_size = dc->dcn_ip->pte_chunk_size;
-	v->meta_chunk_size = dc->dcn_ip->meta_chunk_size;
-	v->writeback_chunk_size = dc->dcn_ip->writeback_chunk_size;
+	v->pte_chunk_size = (float)dc->dcn_ip->pte_chunk_size;
+	v->meta_chunk_size = (float)dc->dcn_ip->meta_chunk_size;
+	v->writeback_chunk_size = (float)dc->dcn_ip->writeback_chunk_size;
 	v->odm_capability = dc->dcn_ip->odm_capability;
 	v->dsc_capability = dc->dcn_ip->dsc_capability;
-	v->line_buffer_size = dc->dcn_ip->line_buffer_size;
+	v->line_buffer_size = (float)dc->dcn_ip->line_buffer_size;
 	v->is_line_buffer_bpp_fixed = dc->dcn_ip->is_line_buffer_bpp_fixed;
-	v->line_buffer_fixed_bpp = dc->dcn_ip->line_buffer_fixed_bpp;
-	v->max_line_buffer_lines = dc->dcn_ip->max_line_buffer_lines;
-	v->writeback_luma_buffer_size = dc->dcn_ip->writeback_luma_buffer_size;
-	v->writeback_chroma_buffer_size = dc->dcn_ip->writeback_chroma_buffer_size;
-	v->max_num_dpp = dc->dcn_ip->max_num_dpp;
-	v->max_num_writeback = dc->dcn_ip->max_num_writeback;
-	v->max_dchub_topscl_throughput = dc->dcn_ip->max_dchub_topscl_throughput;
-	v->max_pscl_tolb_throughput = dc->dcn_ip->max_pscl_tolb_throughput;
-	v->max_lb_tovscl_throughput = dc->dcn_ip->max_lb_tovscl_throughput;
-	v->max_vscl_tohscl_throughput = dc->dcn_ip->max_vscl_tohscl_throughput;
+	v->line_buffer_fixed_bpp = (float)dc->dcn_ip->line_buffer_fixed_bpp;
+	v->max_line_buffer_lines = (float)dc->dcn_ip->max_line_buffer_lines;
+	v->writeback_luma_buffer_size = (float)dc->dcn_ip->writeback_luma_buffer_size;
+	v->writeback_chroma_buffer_size = (float)dc->dcn_ip->writeback_chroma_buffer_size;
+	v->max_num_dpp = (float)dc->dcn_ip->max_num_dpp;
+	v->max_num_writeback = (float)dc->dcn_ip->max_num_writeback;
+	v->max_dchub_topscl_throughput = (float)dc->dcn_ip->max_dchub_topscl_throughput;
+	v->max_pscl_tolb_throughput = (float)dc->dcn_ip->max_pscl_tolb_throughput;
+	v->max_lb_tovscl_throughput = (float)dc->dcn_ip->max_lb_tovscl_throughput;
+	v->max_vscl_tohscl_throughput = (float)dc->dcn_ip->max_vscl_tohscl_throughput;
 	v->max_hscl_ratio = dc->dcn_ip->max_hscl_ratio;
 	v->max_vscl_ratio = dc->dcn_ip->max_vscl_ratio;
-	v->max_hscl_taps = dc->dcn_ip->max_hscl_taps;
-	v->max_vscl_taps = dc->dcn_ip->max_vscl_taps;
+	v->max_hscl_taps = (float)dc->dcn_ip->max_hscl_taps;
+	v->max_vscl_taps = (float)dc->dcn_ip->max_vscl_taps;
 	v->under_scan_factor = dc->dcn_ip->under_scan_factor;
-	v->pte_buffer_size_in_requests = dc->dcn_ip->pte_buffer_size_in_requests;
-	v->dispclk_ramping_margin = dc->dcn_ip->dispclk_ramping_margin;
-	v->max_inter_dcn_tile_repeaters = dc->dcn_ip->max_inter_dcn_tile_repeaters;
+	v->pte_buffer_size_in_requests = (float)dc->dcn_ip->pte_buffer_size_in_requests;
+	v->dispclk_ramping_margin = (float)dc->dcn_ip->dispclk_ramping_margin;
+	v->max_inter_dcn_tile_repeaters = (float)dc->dcn_ip->max_inter_dcn_tile_repeaters;
 	v->can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one =
 			dc->dcn_ip->can_vstartup_lines_exceed_vsync_plus_back_porch_lines_minus_one;
 	v->bug_forcing_luma_and_chroma_request_to_same_size_fixed =
@@ -903,14 +905,14 @@ bool dcn_validate_bandwidth(
 		v->underscan_output[input_idx] = false; /* taken care of in recout already*/
 		v->interlace_output[input_idx] = false;
 
-		v->htotal[input_idx] = pipe->stream->timing.h_total;
-		v->vtotal[input_idx] = pipe->stream->timing.v_total;
-		v->vactive[input_idx] = pipe->stream->timing.v_addressable +
-				pipe->stream->timing.v_border_top + pipe->stream->timing.v_border_bottom;
-		v->v_sync_plus_back_porch[input_idx] = pipe->stream->timing.v_total
+		v->htotal[input_idx] = (float)pipe->stream->timing.h_total;
+		v->vtotal[input_idx] = (float)pipe->stream->timing.v_total;
+		v->vactive[input_idx] = (float)(pipe->stream->timing.v_addressable +
+				pipe->stream->timing.v_border_top + pipe->stream->timing.v_border_bottom);
+		v->v_sync_plus_back_porch[input_idx] = (float)(pipe->stream->timing.v_total
 				- v->vactive[input_idx]
-				- pipe->stream->timing.v_front_porch;
-		v->pixel_clock[input_idx] = pipe->stream->timing.pix_clk_100hz/10000.0;
+				- pipe->stream->timing.v_front_porch);
+		v->pixel_clock[input_idx] = pipe->stream->timing.pix_clk_100hz/10000.0f;
 		if (pipe->stream->timing.timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
 			v->pixel_clock[input_idx] *= 2;
 		if (!pipe->plane_state) {
@@ -918,8 +920,8 @@ bool dcn_validate_bandwidth(
 			v->source_pixel_format[input_idx] = dcn_bw_rgb_sub_32;
 			v->source_surface_mode[input_idx] = dcn_bw_sw_4_kb_s;
 			v->lb_bit_per_pixel[input_idx] = 30;
-			v->viewport_width[input_idx] = pipe->stream->timing.h_addressable;
-			v->viewport_height[input_idx] = pipe->stream->timing.v_addressable;
+			v->viewport_width[input_idx] = (float)pipe->stream->timing.h_addressable;
+			v->viewport_height[input_idx] = (float)pipe->stream->timing.v_addressable;
 			/*
 			 * for cases where we have no plane, we want to validate up to 1080p
 			 * source size because here we are only interested in if the output
@@ -933,17 +935,17 @@ bool dcn_validate_bandwidth(
 				v->viewport_height[input_idx] = 1080;
 			v->scaler_rec_out_width[input_idx] = v->viewport_width[input_idx];
 			v->scaler_recout_height[input_idx] = v->viewport_height[input_idx];
-			v->override_hta_ps[input_idx] = 1;
-			v->override_vta_ps[input_idx] = 1;
-			v->override_hta_pschroma[input_idx] = 1;
-			v->override_vta_pschroma[input_idx] = 1;
+			v->override_hta_ps[input_idx] = 1.0f;
+			v->override_vta_ps[input_idx] = 1.0f;
+			v->override_hta_pschroma[input_idx] = 1.0f;
+			v->override_vta_pschroma[input_idx] = 1.0f;
 			v->source_scan[input_idx] = dcn_bw_hor;
 
 		} else {
-			v->viewport_height[input_idx] =  pipe->plane_res.scl_data.viewport.height;
-			v->viewport_width[input_idx] = pipe->plane_res.scl_data.viewport.width;
-			v->scaler_rec_out_width[input_idx] = pipe->plane_res.scl_data.recout.width;
-			v->scaler_recout_height[input_idx] = pipe->plane_res.scl_data.recout.height;
+			v->viewport_height[input_idx] =  (float)pipe->plane_res.scl_data.viewport.height;
+			v->viewport_width[input_idx] = (float)pipe->plane_res.scl_data.viewport.width;
+			v->scaler_rec_out_width[input_idx] = (float)pipe->plane_res.scl_data.recout.width;
+			v->scaler_recout_height[input_idx] = (float)pipe->plane_res.scl_data.recout.height;
 			if (pipe->bottom_pipe && pipe->bottom_pipe->plane_state == pipe->plane_state) {
 				if (pipe->plane_state->rotation % 2 == 0) {
 					int viewport_end = pipe->plane_res.scl_data.viewport.width
@@ -952,11 +954,11 @@ bool dcn_validate_bandwidth(
 							+ pipe->bottom_pipe->plane_res.scl_data.viewport.x;
 
 					if (viewport_end > viewport_b_end)
-						v->viewport_width[input_idx] = viewport_end
-							- pipe->bottom_pipe->plane_res.scl_data.viewport.x;
+						v->viewport_width[input_idx] = (float)(viewport_end
+							- pipe->bottom_pipe->plane_res.scl_data.viewport.x);
 					else
-						v->viewport_width[input_idx] = viewport_b_end
-									- pipe->plane_res.scl_data.viewport.x;
+						v->viewport_width[input_idx] = (float)(viewport_b_end
+									- pipe->plane_res.scl_data.viewport.x);
 				} else  {
 					int viewport_end = pipe->plane_res.scl_data.viewport.height
 						+ pipe->plane_res.scl_data.viewport.y;
@@ -964,14 +966,14 @@ bool dcn_validate_bandwidth(
 						+ pipe->bottom_pipe->plane_res.scl_data.viewport.y;
 
 					if (viewport_end > viewport_b_end)
-						v->viewport_height[input_idx] = viewport_end
-							- pipe->bottom_pipe->plane_res.scl_data.viewport.y;
+						v->viewport_height[input_idx] = (float)(viewport_end
+							- pipe->bottom_pipe->plane_res.scl_data.viewport.y);
 					else
-						v->viewport_height[input_idx] = viewport_b_end
-									- pipe->plane_res.scl_data.viewport.y;
+						v->viewport_height[input_idx] = (float)(viewport_b_end
+									- pipe->plane_res.scl_data.viewport.y);
 				}
-				v->scaler_rec_out_width[input_idx] = pipe->plane_res.scl_data.recout.width
-						+ pipe->bottom_pipe->plane_res.scl_data.recout.width;
+				v->scaler_rec_out_width[input_idx] = (float)(pipe->plane_res.scl_data.recout.width
+						+ pipe->bottom_pipe->plane_res.scl_data.recout.width);
 			}
 
 			if (pipe->plane_state->rotation % 2 == 0) {
@@ -997,7 +999,7 @@ bool dcn_validate_bandwidth(
 				 * allow us to disable dcc on the fly without re-calculating WM
 				 *
 				 * extra overhead for DCC is quite small.  for 1080p WM without
-				 * DCC is only 0.417us lower (urgent goes from 6.979us to 6.562us)
+				 * DCC is only 0.417fus lower (urgent goes from 6.979fus to 6.562fus)
 				 */
 				unsigned int bpe;
 
@@ -1009,11 +1011,11 @@ bool dcn_validate_bandwidth(
 					pipe->plane_state->format);
 			v->source_surface_mode[input_idx] = tl_sw_mode_to_bw_defs(
 					pipe->plane_state->tiling_info.gfx9.swizzle);
-			v->lb_bit_per_pixel[input_idx] = tl_lb_bpp_to_int(pipe->plane_res.scl_data.lb_params.depth);
-			v->override_hta_ps[input_idx] = pipe->plane_res.scl_data.taps.h_taps;
-			v->override_vta_ps[input_idx] = pipe->plane_res.scl_data.taps.v_taps;
-			v->override_hta_pschroma[input_idx] = pipe->plane_res.scl_data.taps.h_taps_c;
-			v->override_vta_pschroma[input_idx] = pipe->plane_res.scl_data.taps.v_taps_c;
+			v->lb_bit_per_pixel[input_idx] = (float)tl_lb_bpp_to_int(pipe->plane_res.scl_data.lb_params.depth);
+			v->override_hta_ps[input_idx] = (float)pipe->plane_res.scl_data.taps.h_taps;
+			v->override_vta_ps[input_idx] = (float)pipe->plane_res.scl_data.taps.v_taps;
+			v->override_hta_pschroma[input_idx] = (float)pipe->plane_res.scl_data.taps.h_taps_c;
+			v->override_vta_pschroma[input_idx] = (float)pipe->plane_res.scl_data.taps.v_taps_c;
 			/*
 			 * Spreadsheet doesn't handle taps_c is one properly,
 			 * need to force Chroma to always be scaled to pass
@@ -1091,29 +1093,29 @@ bool dcn_validate_bandwidth(
 	}
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		if (v->source_pixel_format[k] == dcn_bw_rgb_sub_64) {
-			v->byte_per_pixel_dety[k] = 8.0;
-			v->byte_per_pixel_detc[k] = 0.0;
+			v->byte_per_pixel_dety[k] = 8.0f;
+			v->byte_per_pixel_detc[k] = 0.0f;
 		} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_32) {
-			v->byte_per_pixel_dety[k] = 4.0;
-			v->byte_per_pixel_detc[k] = 0.0;
+			v->byte_per_pixel_dety[k] = 4.0f;
+			v->byte_per_pixel_detc[k] = 0.0f;
 		} else if (v->source_pixel_format[k] == dcn_bw_rgb_sub_16) {
-			v->byte_per_pixel_dety[k] = 2.0;
-			v->byte_per_pixel_detc[k] = 0.0;
+			v->byte_per_pixel_dety[k] = 2.0f;
+			v->byte_per_pixel_detc[k] = 0.0f;
 		} else if (v->source_pixel_format[k] == dcn_bw_yuv420_sub_8) {
-			v->byte_per_pixel_dety[k] = 1.0;
-			v->byte_per_pixel_detc[k] = 2.0;
+			v->byte_per_pixel_dety[k] = 1.0f;
+			v->byte_per_pixel_detc[k] = 2.0f;
 		} else {
 			v->byte_per_pixel_dety[k] = 4.0f / 3.0f;
 			v->byte_per_pixel_detc[k] = 8.0f / 3.0f;
 		}
 	}
 
-	v->total_data_read_bandwidth = 0.0;
+	v->total_data_read_bandwidth = 0.0f;
 	for (k = 0; k <= v->number_of_active_planes - 1; k++) {
 		v->read_bandwidth_plane_luma[k] = v->swath_width_y[k] * v->dpp_per_plane[k] *
-				dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k];
-		v->read_bandwidth_plane_chroma[k] = v->swath_width_y[k] / 2.0 * v->dpp_per_plane[k] *
-				dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k] / 2.0;
+				dcn_bw_ceil2(v->byte_per_pixel_dety[k], 1.0f) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k];
+		v->read_bandwidth_plane_chroma[k] = v->swath_width_y[k] / 2.0f * v->dpp_per_plane[k] *
+				dcn_bw_ceil2(v->byte_per_pixel_detc[k], 2.0f) / (v->htotal[k] / v->pixel_clock[k]) * v->v_ratio[k] / 2.0f;
 		v->total_data_read_bandwidth = v->total_data_read_bandwidth +
 				v->read_bandwidth_plane_luma[k] + v->read_bandwidth_plane_chroma[k];
 	}
@@ -1144,13 +1146,13 @@ bool dcn_validate_bandwidth(
 		dispclkdppclkdcfclk_deep_sleep_prefetch_parameters_watermarks_and_performance_calculation(v);
 
 		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns =
-			v->stutter_exit_watermark * 1000;
+			(uint32_t)(v->stutter_exit_watermark * 1000.0f);
 		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns =
-				v->stutter_enter_plus_exit_watermark * 1000;
+				(uint32_t)(v->stutter_enter_plus_exit_watermark * 1000.0f);
 		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns =
-				v->dram_clock_change_watermark * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = v->ptemeta_urgent_watermark * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = v->urgent_watermark * 1000;
+				(uint32_t)(v->dram_clock_change_watermark * 1000.0f);
+		context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = (uint32_t)(v->ptemeta_urgent_watermark * 1000.0f);
+		context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = (uint32_t)(v->urgent_watermark * 1000.0f);
 		context->bw_ctx.bw.dcn.watermarks.b = context->bw_ctx.bw.dcn.watermarks.a;
 		context->bw_ctx.bw.dcn.watermarks.c = context->bw_ctx.bw.dcn.watermarks.a;
 		context->bw_ctx.bw.dcn.watermarks.d = context->bw_ctx.bw.dcn.watermarks.a;
@@ -1168,14 +1170,14 @@ bool dcn_validate_bandwidth(
 			context->bw_ctx.bw.dcn.clk.dispclk_khz = (int)(dc->dcn_soc->max_dispclk_vmax0p9 * 1000);
 
 		if (context->bw_ctx.bw.dcn.clk.dispclk_khz <
-				dc->debug.min_disp_clk_khz) {
+				(int)dc->debug.min_disp_clk_khz) {
 			context->bw_ctx.bw.dcn.clk.dispclk_khz =
-					dc->debug.min_disp_clk_khz;
+					(int)dc->debug.min_disp_clk_khz;
 		}
 
 		context->bw_ctx.bw.dcn.clk.dppclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz /
 				v->dispclk_dppclk_ratio;
-		context->bw_ctx.bw.dcn.clk.phyclk_khz = v->phyclk_per_state[v->voltage_level];
+		context->bw_ctx.bw.dcn.clk.phyclk_khz = (int)v->phyclk_per_state[v->voltage_level];
 		switch (v->voltage_level) {
 		case 0:
 			context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz =
@@ -1207,10 +1209,10 @@ bool dcn_validate_bandwidth(
 			if (pipe->top_pipe && pipe->top_pipe->plane_state == pipe->plane_state)
 				continue;
 
-			pipe->pipe_dlg_param.vupdate_width = v->v_update_width_pix[input_idx];
-			pipe->pipe_dlg_param.vupdate_offset = v->v_update_offset_pix[input_idx];
-			pipe->pipe_dlg_param.vready_offset = v->v_ready_offset_pix[input_idx];
-			pipe->pipe_dlg_param.vstartup_start = v->v_startup[input_idx];
+			pipe->pipe_dlg_param.vupdate_width = (unsigned int)v->v_update_width_pix[input_idx];
+			pipe->pipe_dlg_param.vupdate_offset = (unsigned int)v->v_update_offset_pix[input_idx];
+			pipe->pipe_dlg_param.vready_offset = (unsigned int)v->v_ready_offset_pix[input_idx];
+			pipe->pipe_dlg_param.vstartup_start = (unsigned int)v->v_startup[input_idx];
 
 			pipe->pipe_dlg_param.htotal = pipe->stream->timing.h_total;
 			pipe->pipe_dlg_param.vtotal = pipe->stream->timing.v_total;
@@ -1248,10 +1250,10 @@ bool dcn_validate_bandwidth(
 					 TIMING_3D_FORMAT_SIDE_BY_SIDE))) {
 					if (hsplit_pipe && hsplit_pipe->plane_state == pipe->plane_state) {
 						/* update previously split pipe */
-						hsplit_pipe->pipe_dlg_param.vupdate_width = v->v_update_width_pix[input_idx];
-						hsplit_pipe->pipe_dlg_param.vupdate_offset = v->v_update_offset_pix[input_idx];
-						hsplit_pipe->pipe_dlg_param.vready_offset = v->v_ready_offset_pix[input_idx];
-						hsplit_pipe->pipe_dlg_param.vstartup_start = v->v_startup[input_idx];
+						hsplit_pipe->pipe_dlg_param.vupdate_width = (unsigned int)v->v_update_width_pix[input_idx];
+						hsplit_pipe->pipe_dlg_param.vupdate_offset = (unsigned int)v->v_update_offset_pix[input_idx];
+						hsplit_pipe->pipe_dlg_param.vready_offset = (unsigned int)v->v_ready_offset_pix[input_idx];
+						hsplit_pipe->pipe_dlg_param.vstartup_start = (unsigned int)v->v_startup[input_idx];
 
 						hsplit_pipe->pipe_dlg_param.htotal = pipe->stream->timing.h_total;
 						hsplit_pipe->pipe_dlg_param.vtotal = pipe->stream->timing.v_total;
@@ -1302,12 +1304,12 @@ bool dcn_validate_bandwidth(
 	 */
 
 	bw_limit = dc->dcn_soc->percent_disp_bw_limit * v->fabric_and_dram_bandwidth_vmax0p9;
-	bw_limit_pass = (v->total_data_read_bandwidth / 1000.0) < bw_limit;
+	bw_limit_pass = (v->total_data_read_bandwidth / 1000.0f) < bw_limit;
 
 	PERFORMANCE_TRACE_END();
 	BW_VAL_TRACE_FINISH();
 
-	if (bw_limit_pass && v->voltage_level <= get_highest_allowed_voltage_level(dc->config.is_vmin_only_asic))
+	if (bw_limit_pass && v->voltage_level <= (int)get_highest_allowed_voltage_level(dc->config.is_vmin_only_asic))
 		return true;
 	else
 		return false;
@@ -1327,19 +1329,19 @@ void dcn_bw_update_from_pplib_fclks(
 	vmax0p9_idx = fclks->num_levels > 0 ? fclks->num_levels - 1 : 0;
 
 	dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 =
-		32 * (fclks->data[vmin0p65_idx].clocks_in_khz / 1000.0) / 1000.0;
+		32 * (fclks->data[vmin0p65_idx].clocks_in_khz / 1000.0f) / 1000.0f;
 	dc->dcn_soc->fabric_and_dram_bandwidth_vmid0p72 =
 		dc->dcn_soc->number_of_channels *
-		(fclks->data[vmid0p72_idx].clocks_in_khz / 1000.0)
-		* ddr4_dram_factor_single_Channel / 1000.0;
+		(fclks->data[vmid0p72_idx].clocks_in_khz / 1000.0f)
+		* ddr4_dram_factor_single_Channel / 1000.0f;
 	dc->dcn_soc->fabric_and_dram_bandwidth_vnom0p8 =
 		dc->dcn_soc->number_of_channels *
-		(fclks->data[vnom0p8_idx].clocks_in_khz / 1000.0)
-		* ddr4_dram_factor_single_Channel / 1000.0;
+		(fclks->data[vnom0p8_idx].clocks_in_khz / 1000.0f)
+		* ddr4_dram_factor_single_Channel / 1000.0f;
 	dc->dcn_soc->fabric_and_dram_bandwidth_vmax0p9 =
 		dc->dcn_soc->number_of_channels *
-		(fclks->data[vmax0p9_idx].clocks_in_khz / 1000.0)
-		* ddr4_dram_factor_single_Channel / 1000.0;
+		(fclks->data[vmax0p9_idx].clocks_in_khz / 1000.0f)
+		* ddr4_dram_factor_single_Channel / 1000.0f;
 }
 
 void dcn_bw_update_from_pplib_dcfclks(
@@ -1347,10 +1349,10 @@ void dcn_bw_update_from_pplib_dcfclks(
 	struct dm_pp_clock_levels_with_voltage *dcfclks)
 {
 	if (dcfclks->num_levels >= 3) {
-		dc->dcn_soc->dcfclkv_min0p65 = dcfclks->data[0].clocks_in_khz / 1000.0;
-		dc->dcn_soc->dcfclkv_mid0p72 = dcfclks->data[dcfclks->num_levels - 3].clocks_in_khz / 1000.0;
-		dc->dcn_soc->dcfclkv_nom0p8 = dcfclks->data[dcfclks->num_levels - 2].clocks_in_khz / 1000.0;
-		dc->dcn_soc->dcfclkv_max0p9 = dcfclks->data[dcfclks->num_levels - 1].clocks_in_khz / 1000.0;
+		dc->dcn_soc->dcfclkv_min0p65 = dcfclks->data[0].clocks_in_khz / 1000.0f;
+		dc->dcn_soc->dcfclkv_mid0p72 = dcfclks->data[dcfclks->num_levels - 3].clocks_in_khz / 1000.0f;
+		dc->dcn_soc->dcfclkv_nom0p8 = dcfclks->data[dcfclks->num_levels - 2].clocks_in_khz / 1000.0f;
+		dc->dcn_soc->dcfclkv_max0p9 = dcfclks->data[dcfclks->num_levels - 1].clocks_in_khz / 1000.0f;
 	}
 }
 
@@ -1360,9 +1362,9 @@ void dcn_get_soc_clks(
 	int *min_dcfclk_khz,
 	int *socclk_khz)
 {
-	*min_fclk_khz = dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 * 1000000 / 32;
-	*min_dcfclk_khz = dc->dcn_soc->dcfclkv_min0p65 * 1000;
-	*socclk_khz = dc->dcn_soc->socclk * 1000;
+	*min_fclk_khz = (int)(dc->dcn_soc->fabric_and_dram_bandwidth_vmin0p65 * 1000000.0f / 32.0f);
+	*min_dcfclk_khz = (int)(dc->dcn_soc->dcfclkv_min0p65 * 1000.0f);
+	*socclk_khz = (int)(dc->dcn_soc->socclk * 1000.0f);
 }
 
 void dcn_bw_notify_pplib_of_wm_ranges(
@@ -1391,15 +1393,15 @@ void dcn_bw_notify_pplib_of_wm_ranges(
 	ranges.num_reader_wm_sets = WM_SET_COUNT;
 	ranges.num_writer_wm_sets = WM_SET_COUNT;
 	ranges.reader_wm_sets[0].wm_inst = WM_A;
-	ranges.reader_wm_sets[0].min_drain_clk_mhz = min_dcfclk_khz / 1000;
-	ranges.reader_wm_sets[0].max_drain_clk_mhz = overdrive / 1000;
-	ranges.reader_wm_sets[0].min_fill_clk_mhz = min_fclk_khz / 1000;
-	ranges.reader_wm_sets[0].max_fill_clk_mhz = overdrive / 1000;
+	ranges.reader_wm_sets[0].min_drain_clk_mhz = (uint16_t)(min_dcfclk_khz / 1000);
+	ranges.reader_wm_sets[0].max_drain_clk_mhz = (uint16_t)(overdrive / 1000);
+	ranges.reader_wm_sets[0].min_fill_clk_mhz = (uint16_t)(min_fclk_khz / 1000);
+	ranges.reader_wm_sets[0].max_fill_clk_mhz = (uint16_t)(overdrive / 1000);
 	ranges.writer_wm_sets[0].wm_inst = WM_A;
-	ranges.writer_wm_sets[0].min_fill_clk_mhz = socclk_khz / 1000;
-	ranges.writer_wm_sets[0].max_fill_clk_mhz = overdrive / 1000;
-	ranges.writer_wm_sets[0].min_drain_clk_mhz = min_fclk_khz / 1000;
-	ranges.writer_wm_sets[0].max_drain_clk_mhz = overdrive / 1000;
+	ranges.writer_wm_sets[0].min_fill_clk_mhz = (uint16_t)(socclk_khz / 1000);
+	ranges.writer_wm_sets[0].max_fill_clk_mhz = (uint16_t)(overdrive / 1000);
+	ranges.writer_wm_sets[0].min_drain_clk_mhz = (uint16_t)(min_fclk_khz / 1000);
+	ranges.writer_wm_sets[0].max_drain_clk_mhz = (uint16_t)(overdrive / 1000);
 
 	if (dc->debug.pplib_wm_report_mode == WM_REPORT_OVERRIDE) {
 		ranges.reader_wm_sets[0].wm_inst = WM_A;
@@ -1583,14 +1585,14 @@ void dcn_bw_sync_calcs_and_dml(struct dc *dc)
 	dc->dml.soc.dram_clock_change_latency_us = dc->dcn_soc->dram_clock_change_latency;
 	dc->dml.soc.return_bus_width_bytes = dc->dcn_soc->return_bus_width;
 
-	dc->dml.ip.rob_buffer_size_kbytes = dc->dcn_ip->rob_buffer_size_in_kbyte;
-	dc->dml.ip.det_buffer_size_kbytes = dc->dcn_ip->det_buffer_size_in_kbyte;
-	dc->dml.ip.dpp_output_buffer_pixels = dc->dcn_ip->dpp_output_buffer_pixels;
-	dc->dml.ip.opp_output_buffer_lines = dc->dcn_ip->opp_output_buffer_lines;
-	dc->dml.ip.pixel_chunk_size_kbytes = dc->dcn_ip->pixel_chunk_size_in_kbyte;
+	dc->dml.ip.rob_buffer_size_kbytes = (unsigned int)dc->dcn_ip->rob_buffer_size_in_kbyte;
+	dc->dml.ip.det_buffer_size_kbytes = (unsigned int)dc->dcn_ip->det_buffer_size_in_kbyte;
+	dc->dml.ip.dpp_output_buffer_pixels = (unsigned int)dc->dcn_ip->dpp_output_buffer_pixels;
+	dc->dml.ip.opp_output_buffer_lines = (unsigned int)dc->dcn_ip->opp_output_buffer_lines;
+	dc->dml.ip.pixel_chunk_size_kbytes = (unsigned int)dc->dcn_ip->pixel_chunk_size_in_kbyte;
 	dc->dml.ip.pte_enable = dc->dcn_ip->pte_enable == dcn_bw_yes;
-	dc->dml.ip.pte_chunk_size_kbytes = dc->dcn_ip->pte_chunk_size;
-	dc->dml.ip.meta_chunk_size_kbytes = dc->dcn_ip->meta_chunk_size;
+	dc->dml.ip.pte_chunk_size_kbytes = (unsigned int)dc->dcn_ip->pte_chunk_size;
+	dc->dml.ip.meta_chunk_size_kbytes = (unsigned int)dc->dcn_ip->meta_chunk_size;
 	dc->dml.ip.writeback_chunk_size_kbytes = dc->dcn_ip->writeback_chunk_size;
 	dc->dml.ip.line_buffer_size_bits = dc->dcn_ip->line_buffer_size;
 	dc->dml.ip.max_line_buffer_lines = dc->dcn_ip->max_line_buffer_lines;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
index e82f2d5312118..ed7c989a5b130 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/dcn20_fpu.c
@@ -990,7 +990,8 @@ void dcn20_populate_dml_writeback_from_context(struct dc *dc,
 					       struct resource_context *res_ctx,
 					       display_e2e_pipe_params_st *pipes)
 {
-	int pipe_cnt, i;
+	unsigned int i;
+	int pipe_cnt;
 
 	dc_assert_fp_enabled();
 
@@ -1036,15 +1037,15 @@ void dcn20_fpu_set_wb_arb_params(struct mcif_arb_params *wb_arb_params,
 	dc_assert_fp_enabled();
 
 	for (k = 0; k < sizeof(wb_arb_params->cli_watermark)/sizeof(wb_arb_params->cli_watermark[0]); k++) {
-		wb_arb_params->cli_watermark[k] = get_wm_writeback_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		wb_arb_params->pstate_watermark[k] = get_wm_writeback_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+		wb_arb_params->cli_watermark[k] = (unsigned int)(get_wm_writeback_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+		wb_arb_params->pstate_watermark[k] = (unsigned int)(get_wm_writeback_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
 	}
-	wb_arb_params->time_per_pixel = 16.0 * 1000 / (context->res_ctx.pipe_ctx[i].stream->phy_pix_clk / 1000); /* 4 bit fraction, ms */
+	wb_arb_params->time_per_pixel = (unsigned int)(16.0 * 1000.0 / (context->res_ctx.pipe_ctx[i].stream->phy_pix_clk / 1000)); /* 4 bit fraction, ms */
 }
 
 static bool is_dtbclk_required(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		if (!context->res_ctx.pipe_ctx[i].stream)
 			continue;
@@ -1104,7 +1105,7 @@ static enum dcn_zstate_support_state  decide_zstate_support(struct dc *dc, struc
 }
 
 static void dcn20_adjust_freesync_v_startup(
-		const struct dc_crtc_timing *dc_crtc_timing, int *vstartup_start)
+		const struct dc_crtc_timing *dc_crtc_timing, unsigned int *vstartup_start)
 {
 	struct dc_crtc_timing patched_crtc_timing;
 	uint32_t asic_blank_end   = 0;
@@ -1134,7 +1135,7 @@ static void dcn20_adjust_freesync_v_startup(
 	/* The newVStartUp is 1 line before vsync point */
 	newVstartup = asic_blank_end + 1;
 
-	*vstartup_start = ((newVstartup > *vstartup_start) ? newVstartup : *vstartup_start);
+	*vstartup_start = (((int)newVstartup > *vstartup_start) ? (int)newVstartup : *vstartup_start);
 }
 
 void dcn20_calculate_dlg_params(struct dc *dc,
@@ -1143,23 +1144,24 @@ void dcn20_calculate_dlg_params(struct dc *dc,
 				int pipe_cnt,
 				int vlevel)
 {
-	int i, pipe_idx, active_hubp_count = 0;
+	int pipe_idx, active_hubp_count = 0;
+	unsigned int i;
 
 	dc_assert_fp_enabled();
 
 	/* Writeback MCIF_WB arbitration parameters */
 	dc->res_pool->funcs->set_mcif_arb_params(dc, context, pipes, pipe_cnt);
 
-	context->bw_ctx.bw.dcn.clk.dispclk_khz = context->bw_ctx.dml.vba.DISPCLK * 1000;
-	context->bw_ctx.bw.dcn.clk.dcfclk_khz = context->bw_ctx.dml.vba.DCFCLK * 1000;
-	context->bw_ctx.bw.dcn.clk.socclk_khz = context->bw_ctx.dml.vba.SOCCLK * 1000;
-	context->bw_ctx.bw.dcn.clk.dramclk_khz = context->bw_ctx.dml.vba.DRAMSpeed * 1000 / 16;
+	context->bw_ctx.bw.dcn.clk.dispclk_khz = (int)(context->bw_ctx.dml.vba.DISPCLK * 1000.0);
+	context->bw_ctx.bw.dcn.clk.dcfclk_khz = (int)(context->bw_ctx.dml.vba.DCFCLK * 1000.0);
+	context->bw_ctx.bw.dcn.clk.socclk_khz = (int)(context->bw_ctx.dml.vba.SOCCLK * 1000.0);
+	context->bw_ctx.bw.dcn.clk.dramclk_khz = (int)(context->bw_ctx.dml.vba.DRAMSpeed * 1000.0 / 16.0);
 
-	if (dc->debug.min_dram_clk_khz > context->bw_ctx.bw.dcn.clk.dramclk_khz)
+	if ((int)dc->debug.min_dram_clk_khz > context->bw_ctx.bw.dcn.clk.dramclk_khz)
 		context->bw_ctx.bw.dcn.clk.dramclk_khz = dc->debug.min_dram_clk_khz;
 
-	context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz = context->bw_ctx.dml.vba.DCFCLKDeepSleep * 1000;
-	context->bw_ctx.bw.dcn.clk.fclk_khz = context->bw_ctx.dml.vba.FabricClock * 1000;
+	context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz = (int)(context->bw_ctx.dml.vba.DCFCLKDeepSleep * 1000.0);
+	context->bw_ctx.bw.dcn.clk.fclk_khz = (int)(context->bw_ctx.dml.vba.FabricClock * 1000.0);
 	context->bw_ctx.bw.dcn.clk.p_state_change_support =
 		context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb]
 							!= dm_dram_clock_change_unsupported;
@@ -1173,18 +1175,18 @@ void dcn20_calculate_dlg_params(struct dc *dc,
 
 	context->bw_ctx.bw.dcn.clk.dtbclk_en = is_dtbclk_required(dc, context);
 
-	if (context->bw_ctx.bw.dcn.clk.dispclk_khz < dc->debug.min_disp_clk_khz)
-		context->bw_ctx.bw.dcn.clk.dispclk_khz = dc->debug.min_disp_clk_khz;
+	if ((unsigned int)context->bw_ctx.bw.dcn.clk.dispclk_khz < dc->debug.min_disp_clk_khz)
+		context->bw_ctx.bw.dcn.clk.dispclk_khz = (int)dc->debug.min_disp_clk_khz;
 
 	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
 		if (!context->res_ctx.pipe_ctx[i].stream)
 			continue;
 		if (context->res_ctx.pipe_ctx[i].plane_state)
 			active_hubp_count++;
-		pipes[pipe_idx].pipe.dest.vstartup_start = get_vstartup(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
-		pipes[pipe_idx].pipe.dest.vupdate_offset = get_vupdate_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
-		pipes[pipe_idx].pipe.dest.vupdate_width = get_vupdate_width(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
-		pipes[pipe_idx].pipe.dest.vready_offset = get_vready_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+		pipes[pipe_idx].pipe.dest.vstartup_start = (unsigned int)get_vstartup(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+		pipes[pipe_idx].pipe.dest.vupdate_offset = (unsigned int)get_vupdate_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+		pipes[pipe_idx].pipe.dest.vupdate_width = (unsigned int)get_vupdate_width(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+		pipes[pipe_idx].pipe.dest.vready_offset = (unsigned int)get_vready_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
 
 		if (dc_state_get_pipe_subvp_type(context, &context->res_ctx.pipe_ctx[i]) == SUBVP_PHANTOM) {
 			// Phantom pipe requires that DET_SIZE = 0 and no unbounded requests
@@ -1196,9 +1198,9 @@ void dcn20_calculate_dlg_params(struct dc *dc,
 		}
 
 		if (context->bw_ctx.bw.dcn.clk.dppclk_khz < pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000)
-			context->bw_ctx.bw.dcn.clk.dppclk_khz = pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000;
+			context->bw_ctx.bw.dcn.clk.dppclk_khz = (int)(pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000.0);
 		context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz =
-						pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000;
+						(int)(pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000.0);
 		context->res_ctx.pipe_ctx[i].pipe_dlg_param = pipes[pipe_idx].pipe.dest;
 		if (dc->ctx->dce_version < DCN_VERSION_3_1 &&
 		    context->res_ctx.pipe_ctx[i].stream->adaptive_sync_infopacket.valid)
@@ -1215,8 +1217,8 @@ void dcn20_calculate_dlg_params(struct dc *dc,
 	/*save a original dppclock copy*/
 	context->bw_ctx.bw.dcn.clk.bw_dppclk_khz = context->bw_ctx.bw.dcn.clk.dppclk_khz;
 	context->bw_ctx.bw.dcn.clk.bw_dispclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz;
-	context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = context->bw_ctx.dml.soc.clock_limits[vlevel].dppclk_mhz * 1000;
-	context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = context->bw_ctx.dml.soc.clock_limits[vlevel].dispclk_mhz * 1000;
+	context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = (int)(context->bw_ctx.dml.soc.clock_limits[vlevel].dppclk_mhz * 1000.0);
+	context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = (int)(context->bw_ctx.dml.soc.clock_limits[vlevel].dispclk_mhz * 1000.0);
 
 	context->bw_ctx.bw.dcn.compbuf_size_kb = context->bw_ctx.dml.ip.config_return_buffer_size_in_kbytes
 						- context->bw_ctx.dml.ip.det_buffer_size_kbytes * pipe_idx;
@@ -1251,7 +1253,7 @@ void dcn20_calculate_dlg_params(struct dc *dc,
 
 static void swizzle_to_dml_params(
 		enum swizzle_mode_values swizzle,
-		unsigned int *sw_mode)
+		int *sw_mode)
 {
 	switch (swizzle) {
 	case DC_SW_LINEAR:
@@ -1317,7 +1319,8 @@ int dcn20_populate_dml_pipes_from_context(struct dc *dc,
 					  enum dc_validate_mode validate_mode)
 {
 	(void)validate_mode;
-	int pipe_cnt, i;
+	int pipe_cnt;
+	unsigned int i;
 	bool synchronized_vblank = true;
 	struct resource_context *res_ctx = &context->res_ctx;
 
@@ -1402,11 +1405,12 @@ int dcn20_populate_dml_pipes_from_context(struct dc *dc,
 			timing->h_addressable + timing->h_border_left + timing->h_border_right;
 		pipes[pipe_cnt].pipe.dest.vactive =
 			timing->v_addressable + timing->v_border_top + timing->v_border_bottom;
-		pipes[pipe_cnt].pipe.dest.interlaced = timing->flags.INTERLACE;
+		pipes[pipe_cnt].pipe.dest.interlaced = (unsigned char)timing->flags.INTERLACE;
 		pipes[pipe_cnt].pipe.dest.pixel_rate_mhz = timing->pix_clk_100hz/10000.0;
 		if (timing->timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
 			pipes[pipe_cnt].pipe.dest.pixel_rate_mhz *= 2;
-		pipes[pipe_cnt].pipe.dest.otg_inst = res_ctx->pipe_ctx[i].stream_res.tg->inst;
+		pipes[pipe_cnt].pipe.dest.otg_inst =
+				(unsigned char)res_ctx->pipe_ctx[i].stream_res.tg->inst;
 		pipes[pipe_cnt].dout.dp_lanes = 4;
 		pipes[pipe_cnt].dout.dp_rate = dm_dp_rate_na;
 		pipes[pipe_cnt].dout.is_virtual = 0;
@@ -1735,7 +1739,8 @@ void dcn20_calculate_wm(struct dc *dc, struct dc_state *context,
 			int vlevel,
 			enum dc_validate_mode validate_mode)
 {
-	int pipe_cnt, i, pipe_idx;
+	int pipe_cnt, pipe_idx;
+	unsigned int i;
 
 	dc_assert_fp_enabled();
 
@@ -1798,51 +1803,51 @@ void dcn20_calculate_wm(struct dc *dc, struct dc_state *context,
 		pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[1].dcfclk_mhz;
 		pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[1].socclk_mhz;
 	}
-	context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.b.urgent_latency_ns = (uint32_t)(get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
 
 	if (vlevel < 2) {
 		pipes[0].clks_cfg.voltage = 2;
 		pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[2].dcfclk_mhz;
 		pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[2].socclk_mhz;
 	}
-	context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
 
 	if (vlevel < 3) {
 		pipes[0].clks_cfg.voltage = 3;
 		pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[2].dcfclk_mhz;
 		pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[2].socclk_mhz;
 	}
-	context->bw_ctx.bw.dcn.watermarks.d.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.d.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.d.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.d.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.d.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
 
 	pipes[0].clks_cfg.voltage = vlevel;
 	pipes[0].clks_cfg.dcfclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].dcfclk_mhz;
 	pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz;
-	context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
+	context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000.0);
 }
 
 void dcn20_update_bounding_box(struct dc *dc,
@@ -1853,7 +1858,7 @@ void dcn20_update_bounding_box(struct dc *dc,
 {
 	int num_calculated_states = 0;
 	int min_dcfclk = 0;
-	int i;
+	unsigned int i;
 
 	dc_assert_fp_enabled();
 
@@ -1879,7 +1884,7 @@ void dcn20_update_bounding_box(struct dc *dc,
 		bb->clock_limits[i].dram_speed_mts = uclk_states[i] * 16 / 1000;
 
 		// FCLK:UCLK ratio is 1.08
-		min_fclk_required_by_uclk = div_u64(((unsigned long long)uclk_states[i]) * 1080,
+		min_fclk_required_by_uclk = (int)div_u64(((unsigned long long)uclk_states[i]) * 1080,
 			1000000);
 
 		bb->clock_limits[i].fabricclk_mhz = (min_fclk_required_by_uclk < min_dcfclk) ?
@@ -1914,7 +1919,7 @@ void dcn20_update_bounding_box(struct dc *dc,
 void dcn20_cap_soc_clocks(struct _vcs_dpi_soc_bounding_box_st *bb,
 			  struct pp_smu_nv_clock_table max_clocks)
 {
-	int i;
+	unsigned int i;
 
 	dc_assert_fp_enabled();
 
@@ -2134,8 +2139,8 @@ void dcn20_fpu_set_wm_ranges(int i,
 {
 	dc_assert_fp_enabled();
 
-	ranges->reader_wm_sets[i].min_fill_clk_mhz = (i > 0) ? (loaded_bb->clock_limits[i - 1].dram_speed_mts / 16) + 1 : 0;
-	ranges->reader_wm_sets[i].max_fill_clk_mhz = loaded_bb->clock_limits[i].dram_speed_mts / 16;
+	ranges->reader_wm_sets[i].min_fill_clk_mhz = (uint16_t)((i > 0) ? (loaded_bb->clock_limits[i - 1].dram_speed_mts / 16) + 1 : 0);
+	ranges->reader_wm_sets[i].max_fill_clk_mhz = (uint16_t)(loaded_bb->clock_limits[i].dram_speed_mts / 16);
 }
 
 void dcn20_fpu_adjust_dppclk(struct vba_vars_st *v,
@@ -2158,7 +2163,7 @@ int dcn21_populate_dml_pipes_from_context(struct dc *dc,
 					  enum dc_validate_mode validate_mode)
 {
 	uint32_t pipe_cnt;
-	int i;
+	unsigned int i;
 
 	dc_assert_fp_enabled();
 
@@ -2212,7 +2217,7 @@ static void calculate_wm_set_for_vlevel(int vlevel,
 {
 	double dram_clock_change_latency_cached = dml->soc.dram_clock_change_latency_us;
 
-	ASSERT(vlevel < dml->soc.num_states);
+	ASSERT(vlevel < (int)dml->soc.num_states);
 	/* only pipe 0 is read for voltage and dcf/soc clocks */
 	pipes[0].clks_cfg.voltage = vlevel;
 	pipes[0].clks_cfg.dcfclk_mhz = dml->soc.clock_limits[vlevel].dcfclk_mhz;
@@ -2222,14 +2227,14 @@ static void calculate_wm_set_for_vlevel(int vlevel,
 	dml->soc.sr_exit_time_us = table_entry->sr_exit_time_us;
 	dml->soc.sr_enter_plus_exit_time_us = table_entry->sr_enter_plus_exit_time_us;
 
-	wm_set->urgent_ns = get_wm_urgent(dml, pipes, pipe_cnt) * 1000;
-	wm_set->cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(dml, pipes, pipe_cnt) * 1000;
-	wm_set->cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(dml, pipes, pipe_cnt) * 1000;
-	wm_set->cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(dml, pipes, pipe_cnt) * 1000;
-	wm_set->pte_meta_urgent_ns = get_wm_memory_trip(dml, pipes, pipe_cnt) * 1000;
-	wm_set->frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(dml, pipes, pipe_cnt) * 1000;
-	wm_set->frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(dml, pipes, pipe_cnt) * 1000;
-	wm_set->urgent_latency_ns = get_urgent_latency(dml, pipes, pipe_cnt) * 1000;
+	wm_set->urgent_ns = (uint32_t)(get_wm_urgent(dml, pipes, pipe_cnt) * 1000.0);
+	wm_set->cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(dml, pipes, pipe_cnt) * 1000.0);
+	wm_set->cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(dml, pipes, pipe_cnt) * 1000.0);
+	wm_set->cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(dml, pipes, pipe_cnt) * 1000.0);
+	wm_set->pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(dml, pipes, pipe_cnt) * 1000.0);
+	wm_set->frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(dml, pipes, pipe_cnt) * 1000.0);
+	wm_set->frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(dml, pipes, pipe_cnt) * 1000.0);
+	wm_set->urgent_latency_ns = (uint32_t)(get_urgent_latency(dml, pipes, pipe_cnt) * 1000.0);
 	dml->soc.dram_clock_change_latency_us = dram_clock_change_latency_cached;
 }
 
@@ -2240,7 +2245,8 @@ static void dcn21_calculate_wm(struct dc *dc, struct dc_state *context,
 			int vlevel_req,
 			enum dc_validate_mode validate_mode)
 {
-	int pipe_cnt, i, pipe_idx;
+	int pipe_cnt, pipe_idx;
+	unsigned int i;
 	int vlevel, vlevel_max;
 	struct wm_range_table_entry *table_entry;
 	struct clk_bw_params *bw_params = dc->clk_mgr->bw_params;
@@ -2476,7 +2482,8 @@ void dcn201_populate_dml_writeback_from_context_fpu(struct dc *dc,
 						    struct resource_context *res_ctx,
 						    display_e2e_pipe_params_st *pipes)
 {
-	int pipe_cnt, i, j;
+	int pipe_cnt;
+	unsigned int i, j;
 	double max_calc_writeback_dispclk;
 	double writeback_dispclk;
 	struct writeback_st dout_wb = {0};
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c
index f5f636afe33ce..ac37a2134351d 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c
@@ -326,7 +326,7 @@ static unsigned int dscceComputeDelay(
 		pixelsPerClock = 1;
 
 	//initial transmit delay as per PPS
-	initalXmitDelay = dml_round(rcModelSize / 2.0 / bpp / pixelsPerClock);
+	initalXmitDelay = (unsigned int)dml_round(rcModelSize / 2.0 / bpp / pixelsPerClock);
 
 	//compute ssm delay
 	if (bpc == 8)
@@ -519,13 +519,13 @@ static bool CalculatePrefetchSchedule(
 	double LinesToRequestPrefetchPixelData = 0;
 
 	if (ScalerEnabled)
-		DPPCycles = DPPCLKDelaySubtotal + DPPCLKDelaySCL;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotal + DPPCLKDelaySCL);
 	else
-		DPPCycles = DPPCLKDelaySubtotal + DPPCLKDelaySCLLBOnly;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotal + DPPCLKDelaySCLLBOnly);
 
-	DPPCycles = DPPCycles + DPPCLKDelayCNVCFormater + NumberOfCursors * DPPCLKDelayCNVCCursor;
+	DPPCycles = (unsigned int)(DPPCycles + DPPCLKDelayCNVCFormater + NumberOfCursors * DPPCLKDelayCNVCCursor);
 
-	DISPCLKCycles = DISPCLKDelaySubtotal;
+	DISPCLKCycles = (unsigned int)DISPCLKDelaySubtotal;
 
 	if (DPPCLK == 0.0 || DISPCLK == 0.0)
 		return true;
@@ -545,7 +545,7 @@ static bool CalculatePrefetchSchedule(
 	*DSTYAfterScaler = dml_floor(DSTTotalPixelsAfterScaler / HTotal, 1);
 	*DSTXAfterScaler = DSTTotalPixelsAfterScaler - ((double) (*DSTYAfterScaler * HTotal));
 
-	*VUpdateOffsetPix = dml_ceil(HTotal / 4.0, 1);
+	*VUpdateOffsetPix = (unsigned int)dml_ceil(HTotal / 4.0, 1);
 	TotalRepeaterDelayTime = MaxInterDCNTileRepeaters * (2.0 / DPPCLK + 3.0 / DISPCLK);
 	*VUpdateWidthPix = (14.0 / DCFCLKDeepSleep + 12.0 / DPPCLK + TotalRepeaterDelayTime)
 			* PixelClock;
@@ -574,10 +574,10 @@ static bool CalculatePrefetchSchedule(
 		if (VStartup * LineTime
 				< Tsetup + TWait + UrgentExtraLatency + Tdmbf + Tdmec + Tdmsks) {
 			MyError = true;
-			*VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = (Tsetup + TWait
-					+ UrgentExtraLatency + Tdmbf + Tdmec + Tdmsks) / LineTime;
+			*VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = (unsigned int)((Tsetup + TWait
+					+ UrgentExtraLatency + Tdmbf + Tdmec + Tdmsks) / LineTime);
 		} else
-			*VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = 0.0;
+			*VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = (unsigned int)0.0;
 	} else
 		Tdm = 0;
 
@@ -828,14 +828,13 @@ static double CalculatePrefetchSourceLines(
 
 	if (!mode_lib->vba.IgnoreViewportPositioning) {
 
-		*MaxNumSwath = dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0;
+		*MaxNumSwath = (unsigned int)(dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0);
 
 		if (*VInitPreFill > 1.0)
-			MaxPartialSwath = (unsigned int) (*VInitPreFill - 2) % SwathHeight;
+			   MaxPartialSwath = (unsigned int)((unsigned int)(*VInitPreFill - 2) % SwathHeight);
 		else
-			MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 2)
-					% SwathHeight;
-		MaxPartialSwath = dml_max(1U, MaxPartialSwath);
+			   MaxPartialSwath = (unsigned int)((unsigned int)(*VInitPreFill + SwathHeight - 2) % SwathHeight);
+		MaxPartialSwath = (unsigned int)dml_max(1U, MaxPartialSwath);
 
 	} else {
 
@@ -843,13 +842,12 @@ static double CalculatePrefetchSourceLines(
 			dml_print(
 					"WARNING DML: using viewport y position of 0 even though actual viewport y position is non-zero in prefetch source lines calculation\n");
 
-		*MaxNumSwath = dml_ceil(*VInitPreFill / SwathHeight, 1);
+		*MaxNumSwath = (unsigned int)dml_ceil(*VInitPreFill / SwathHeight, 1);
 
 		if (*VInitPreFill > 1.0)
-			MaxPartialSwath = (unsigned int) (*VInitPreFill - 1) % SwathHeight;
+			   MaxPartialSwath = (unsigned int)((unsigned int)(*VInitPreFill - 1) % SwathHeight);
 		else
-			MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 1)
-					% SwathHeight;
+			   MaxPartialSwath = (unsigned int)((unsigned int)(*VInitPreFill + SwathHeight - 1) % SwathHeight);
 	}
 
 	return *MaxNumSwath * SwathHeight + MaxPartialSwath;
@@ -899,34 +897,34 @@ static unsigned int CalculateVMAndRowBytes(
 		MetaRequestHeight = 8 * BlockHeight256Bytes;
 		MetaRequestWidth = 8 * BlockWidth256Bytes;
 		if (ScanDirection == dm_horz) {
-			*meta_row_height = MetaRequestHeight;
-			MetaSurfWidth = dml_ceil((double) SwathWidth - 1, MetaRequestWidth)
-					+ MetaRequestWidth;
-			*MetaRowByte = MetaSurfWidth * MetaRequestHeight * BytePerPixel / 256.0;
+			   *meta_row_height = (unsigned int)MetaRequestHeight;
+			   MetaSurfWidth = (unsigned int)(dml_ceil((double) SwathWidth - 1, MetaRequestWidth)
+				   + MetaRequestWidth);
+			   *MetaRowByte = (unsigned int)(MetaSurfWidth * MetaRequestHeight * BytePerPixel / 256.0);
 		} else {
-			*meta_row_height = MetaRequestWidth;
-			MetaSurfHeight = dml_ceil((double) SwathWidth - 1, MetaRequestHeight)
-					+ MetaRequestHeight;
-			*MetaRowByte = MetaSurfHeight * MetaRequestWidth * BytePerPixel / 256.0;
+			   *meta_row_height = (unsigned int)MetaRequestWidth;
+			   MetaSurfHeight = (unsigned int)(dml_ceil((double) SwathWidth - 1, MetaRequestHeight)
+				   + MetaRequestHeight);
+			   *MetaRowByte = (unsigned int)(MetaSurfHeight * MetaRequestWidth * BytePerPixel / 256.0);
 		}
 		if (ScanDirection == dm_horz) {
-			DCCMetaSurfaceBytes = DCCMetaPitch
+			DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch
 					* (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes)
 							+ 64 * BlockHeight256Bytes) * BytePerPixel
-					/ 256;
+					/ 256);
 		} else {
-			DCCMetaSurfaceBytes = DCCMetaPitch
+			DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch
 					* (dml_ceil(
 							(double) ViewportHeight - 1,
 							64 * BlockHeight256Bytes)
 							+ 64 * BlockHeight256Bytes) * BytePerPixel
-					/ 256;
+					/ 256);
 		}
 		if (GPUVMEnable == true) {
-			MetaPTEBytesFrame = (dml_ceil(
+			MetaPTEBytesFrame = (unsigned int)((dml_ceil(
 					(double) (DCCMetaSurfaceBytes - VMMPageSize)
 							/ (8 * VMMPageSize),
-					1) + 1) * 64;
+					1) + 1) * 64);
 			MPDEBytesFrame = 128 * (mode_lib->vba.GPUVMMaxPageTableLevels - 1);
 		} else {
 			MetaPTEBytesFrame = 0;
@@ -960,7 +958,7 @@ static unsigned int CalculateVMAndRowBytes(
 	if (GPUVMEnable == true && mode_lib->vba.GPUVMMaxPageTableLevels > 1) {
 		if (ScanDirection == dm_horz) {
 			DPDE0BytesFrame =
-					64
+					(unsigned int)(64
 							* (dml_ceil(
 									((Pitch
 											* (dml_ceil(
@@ -972,10 +970,10 @@ static unsigned int CalculateVMAndRowBytes(
 											- MacroTileSizeBytes)
 											/ (8
 													* 2097152),
-									1) + 1);
+									1) + 1));
 		} else {
 			DPDE0BytesFrame =
-					64
+					(unsigned int)(64
 							* (dml_ceil(
 									((Pitch
 											* (dml_ceil(
@@ -987,7 +985,7 @@ static unsigned int CalculateVMAndRowBytes(
 											- MacroTileSizeBytes)
 											/ (8
 													* 2097152),
-									1) + 1);
+									1) + 1));
 		}
 		ExtraDPDEBytesFrame = 128 * (mode_lib->vba.GPUVMMaxPageTableLevels - 2);
 	} else {
@@ -1007,7 +1005,7 @@ static unsigned int CalculateVMAndRowBytes(
 
 		if (SurfaceTiling == dm_sw_linear) {
 			PixelPTEReqHeight = 1;
-			PixelPTEReqWidth = 8.0 * VMMPageSize / BytePerPixel;
+			PixelPTEReqWidth = (unsigned int)8.0 * VMMPageSize / BytePerPixel;
 			PTERequestSize = 64;
 			FractionOfPTEReturnDrop = 0;
 		} else if (MacroTileSizeBytes == 4096) {
@@ -1036,7 +1034,7 @@ static unsigned int CalculateVMAndRowBytes(
 			EffectivePDEProcessingBufIn64KBReqs = PDEProcessingBufIn64KBReqs;
 
 		if (SurfaceTiling == dm_sw_linear) {
-			*dpte_row_height =
+			*dpte_row_height = (unsigned int)(
 					dml_min(
 							128,
 							1
@@ -1049,21 +1047,21 @@ static unsigned int CalculateVMAndRowBytes(
 																	* 65536.0
 																	/ BytePerPixel)
 															/ Pitch),
-											1));
+										1)));
 			*PixelPTEBytesPerRow = PTERequestSize
-					* (dml_ceil(
+					* (unsigned int)(dml_ceil(
 							(double) (Pitch * *dpte_row_height - 1)
 									/ PixelPTEReqWidth,
 							1) + 1);
 		} else if (ScanDirection == dm_horz) {
 			*dpte_row_height = PixelPTEReqHeight;
 			*PixelPTEBytesPerRow = PTERequestSize
-					* (dml_ceil(((double) SwathWidth - 1) / PixelPTEReqWidth, 1)
+					* (unsigned int)(dml_ceil(((double) SwathWidth - 1) / PixelPTEReqWidth, 1)
 							+ 1);
 		} else {
-			*dpte_row_height = dml_min(PixelPTEReqWidth, *MacroTileWidth);
+			*dpte_row_height = (unsigned int)dml_min(PixelPTEReqWidth, *MacroTileWidth);
 			*PixelPTEBytesPerRow = PTERequestSize
-					* (dml_ceil(
+					* (unsigned int)(dml_ceil(
 							((double) SwathWidth - 1)
 									/ PixelPTEReqHeight,
 							1) + 1);
@@ -1539,10 +1537,10 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 								/ mode_lib->vba.PixelClock[k])
 						/ mode_lib->vba.VRatio[k];
 		if (mode_lib->vba.BytePerPixelDETC[k] > 0) {
-			mode_lib->vba.LinesInDETC[k] = mode_lib->vba.DETBufferSizeC[k]
+			mode_lib->vba.LinesInDETC[k] = (unsigned int)(mode_lib->vba.DETBufferSizeC[k]
 					/ mode_lib->vba.BytePerPixelDETC[k]
-					/ (mode_lib->vba.SwathWidthY[k] / 2);
-			mode_lib->vba.LinesInDETCRoundedDownToSwath[k] = dml_floor(
+					/ (mode_lib->vba.SwathWidthY[k] / 2));
+			mode_lib->vba.LinesInDETCRoundedDownToSwath[k] = (unsigned int)dml_floor(
 					mode_lib->vba.LinesInDETC[k],
 					mode_lib->vba.SwathHeightC[k]);
 			mode_lib->vba.FullDETBufferingTimeC[k] =
@@ -1811,7 +1809,7 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 						dscceComputeDelay(
 								mode_lib->vba.DSCInputBitPerComponent[k],
 								bpp,
-								dml_ceil(
+								(unsigned int)dml_ceil(
 										(double) mode_lib->vba.HActive[k]
 												/ mode_lib->vba.NumberOfDSCSlices[k],
 										1),
@@ -1821,22 +1819,22 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 										mode_lib->vba.OutputFormat[k]);
 			} else {
 				mode_lib->vba.DSCDelay[k] =
-						2
+						(unsigned int)(2
 								* (dscceComputeDelay(
 										mode_lib->vba.DSCInputBitPerComponent[k],
 										bpp,
-										dml_ceil(
+										(unsigned int)dml_ceil(
 												(double) mode_lib->vba.HActive[k]
 														/ mode_lib->vba.NumberOfDSCSlices[k],
 												1),
-										slices / 2.0,
+										(unsigned int)(slices / 2.0),
 										mode_lib->vba.OutputFormat[k])
 										+ dscComputeDelay(
-												mode_lib->vba.OutputFormat[k]));
+												mode_lib->vba.OutputFormat[k])));
 			}
-			mode_lib->vba.DSCDelay[k] = mode_lib->vba.DSCDelay[k]
+				mode_lib->vba.DSCDelay[k] = (unsigned int)(mode_lib->vba.DSCDelay[k]
 					* mode_lib->vba.PixelClock[k]
-					/ mode_lib->vba.PixelClockBackEnd[k];
+						/ mode_lib->vba.PixelClockBackEnd[k]);
 		} else {
 			mode_lib->vba.DSCDelay[k] = 0;
 		}
@@ -1860,8 +1858,8 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 		Calculate256BBlockSizes(
 				mode_lib->vba.SourcePixelFormat[k],
 				mode_lib->vba.SurfaceTiling[k],
-				dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
-				dml_ceil(mode_lib->vba.BytePerPixelDETC[k], 2),
+				(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
+				(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelDETC[k], 2),
 				&mode_lib->vba.BlockHeight256BytesY[k],
 				&mode_lib->vba.BlockHeight256BytesC[k],
 				&mode_lib->vba.BlockWidth256BytesY[k],
@@ -1873,11 +1871,11 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 				mode_lib->vba.BlockWidth256BytesY[k],
 				mode_lib->vba.SourcePixelFormat[k],
 				mode_lib->vba.SurfaceTiling[k],
-				dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
+				(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
 				mode_lib->vba.SourceScan[k],
 				mode_lib->vba.ViewportWidth[k],
 				mode_lib->vba.ViewportHeight[k],
-				mode_lib->vba.SwathWidthY[k],
+				(unsigned int)mode_lib->vba.SwathWidthY[k],
 				mode_lib->vba.GPUVMEnable,
 				mode_lib->vba.VMMPageSize,
 				mode_lib->vba.PTEBufferSizeInRequestsLuma,
@@ -1913,13 +1911,13 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 							mode_lib->vba.BlockWidth256BytesC[k],
 							mode_lib->vba.SourcePixelFormat[k],
 							mode_lib->vba.SurfaceTiling[k],
-							dml_ceil(
+							(unsigned int)dml_ceil(
 									mode_lib->vba.BytePerPixelDETC[k],
 									2),
 							mode_lib->vba.SourceScan[k],
 							mode_lib->vba.ViewportWidth[k] / 2,
 							mode_lib->vba.ViewportHeight[k] / 2,
-							mode_lib->vba.SwathWidthY[k] / 2,
+							(unsigned int)mode_lib->vba.SwathWidthY[k] / 2,
 							mode_lib->vba.GPUVMEnable,
 							mode_lib->vba.VMMPageSize,
 							mode_lib->vba.PTEBufferSizeInRequestsLuma,
@@ -1989,7 +1987,7 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 										mode_lib->vba.WritebackLumaVTaps[k],
 										mode_lib->vba.WritebackChromaHTaps[k],
 										mode_lib->vba.WritebackChromaVTaps[k],
-										mode_lib->vba.WritebackDestinationWidth[k])
+										(unsigned int)mode_lib->vba.WritebackDestinationWidth[k])
 										/ mode_lib->vba.DISPCLK;
 			} else
 				mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k] = 0;
@@ -2008,7 +2006,7 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 													mode_lib->vba.WritebackLumaVTaps[j],
 													mode_lib->vba.WritebackChromaHTaps[j],
 													mode_lib->vba.WritebackChromaVTaps[j],
-													mode_lib->vba.WritebackDestinationWidth[j])
+													(unsigned int)mode_lib->vba.WritebackDestinationWidth[j])
 													/ mode_lib->vba.DISPCLK);
 				}
 			}
@@ -2023,7 +2021,7 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 
 	mode_lib->vba.VStartupLines = 13;
 	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
-		mode_lib->vba.MaxVStartupLines[k] =
+		mode_lib->vba.MaxVStartupLines[k] = (unsigned int)(
 				mode_lib->vba.VTotal[k] - mode_lib->vba.VActive[k]
 						- dml_max(
 								1.0,
@@ -2031,11 +2029,11 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 										mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k]
 												/ (mode_lib->vba.HTotal[k]
 														/ mode_lib->vba.PixelClock[k]),
-										1));
+										1)));
 	}
 
 	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
-		mode_lib->vba.MaximumMaxVStartupLines = dml_max(
+		mode_lib->vba.MaximumMaxVStartupLines = (unsigned int)dml_max(
 				mode_lib->vba.MaximumMaxVStartupLines,
 				mode_lib->vba.MaxVStartupLines[k]);
 
@@ -2111,7 +2109,7 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 									- mode_lib->vba.VActive[k],
 							mode_lib->vba.HTotal[k],
 							mode_lib->vba.MaxInterDCNTileRepeaters,
-							dml_min(
+							(unsigned int)dml_min(
 									mode_lib->vba.VStartupLines,
 									mode_lib->vba.MaxVStartupLines[k]),
 							mode_lib->vba.GPUVMMaxPageTableLevels,
@@ -2123,11 +2121,11 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 							mode_lib->vba.UrgentLatencyPixelDataOnly,
 							mode_lib->vba.UrgentExtraLatency,
 							mode_lib->vba.TCalc,
-							mode_lib->vba.PDEAndMetaPTEBytesFrame[k],
-							mode_lib->vba.MetaRowByte[k],
-							mode_lib->vba.PixelPTEBytesPerRow[k],
+							(unsigned int)mode_lib->vba.PDEAndMetaPTEBytesFrame[k],
+							(unsigned int)mode_lib->vba.MetaRowByte[k],
+							(unsigned int)mode_lib->vba.PixelPTEBytesPerRow[k],
 							mode_lib->vba.PrefetchSourceLinesY[k],
-							mode_lib->vba.SwathWidthY[k],
+							(unsigned int)mode_lib->vba.SwathWidthY[k],
 							mode_lib->vba.BytePerPixelDETY[k],
 							mode_lib->vba.VInitPreFillY[k],
 							mode_lib->vba.MaxNumSwathY[k],
@@ -2157,7 +2155,7 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 							&mode_lib->vba.VUpdateWidthPix[k],
 							&mode_lib->vba.VReadyOffsetPix[k]);
 			if (mode_lib->vba.BlendingAndTiming[k] == k) {
-				mode_lib->vba.VStartup[k] = dml_min(
+				mode_lib->vba.VStartup[k] = (unsigned int)dml_min(
 						mode_lib->vba.VStartupLines,
 						mode_lib->vba.MaxVStartupLines[k]);
 				if (mode_lib->vba.VStartupRequiredWhenNotEnoughTimeForDynamicMetadata
@@ -2166,7 +2164,7 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 							mode_lib->vba.VStartupRequiredWhenNotEnoughTimeForDynamicMetadata;
 				}
 			} else {
-				mode_lib->vba.VStartup[k] =
+				mode_lib->vba.VStartup[k] = (unsigned int)
 						dml_min(
 								mode_lib->vba.VStartupLines,
 								mode_lib->vba.MaxVStartupLines[mode_lib->vba.BlendingAndTiming[k]]);
@@ -2253,10 +2251,10 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 				ImmediateFlipBytes[k] = 0;
 				if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
 						&& mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
-					ImmediateFlipBytes[k] =
+					ImmediateFlipBytes[k] = (unsigned int)(
 							mode_lib->vba.PDEAndMetaPTEBytesFrame[k]
 									+ mode_lib->vba.MetaRowByte[k]
-									+ mode_lib->vba.PixelPTEBytesPerRow[k];
+									+ mode_lib->vba.PixelPTEBytesPerRow[k]);
 				}
 			}
 			mode_lib->vba.TotImmediateFlipBytes = 0;
@@ -2264,8 +2262,8 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 				if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
 						&& mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
 					mode_lib->vba.TotImmediateFlipBytes =
-							mode_lib->vba.TotImmediateFlipBytes
-									+ ImmediateFlipBytes[k];
+							(unsigned int)(mode_lib->vba.TotImmediateFlipBytes
+									+ ImmediateFlipBytes[k]);
 				}
 			}
 			for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
@@ -2284,8 +2282,8 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 						mode_lib->vba.VRatio[k],
 						mode_lib->vba.Tno_bw[k],
 						mode_lib->vba.PDEAndMetaPTEBytesFrame[k],
-						mode_lib->vba.MetaRowByte[k],
-						mode_lib->vba.PixelPTEBytesPerRow[k],
+						(unsigned int)mode_lib->vba.MetaRowByte[k],
+						(unsigned int)mode_lib->vba.PixelPTEBytesPerRow[k],
 						mode_lib->vba.DCCEnable[k],
 						mode_lib->vba.dpte_row_height[k],
 						mode_lib->vba.meta_row_height[k],
@@ -2409,7 +2407,7 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 		double MaxDETBufferingTimeY;
 		double ActiveDRAMClockChangeLatencyMarginY;
 
-		mode_lib->vba.LBLatencyHidingSourceLinesY =
+		mode_lib->vba.LBLatencyHidingSourceLinesY = (unsigned int)(
 				dml_min(
 						mode_lib->vba.MaxLineBufferLines,
 						(unsigned int) dml_floor(
@@ -2419,9 +2417,9 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 												/ dml_max(
 														mode_lib->vba.HRatio[k],
 														1.0)),
-								1)) - (mode_lib->vba.vtaps[k] - 1);
+								1)) -(mode_lib->vba.vtaps[k] - 1));
 
-		mode_lib->vba.LBLatencyHidingSourceLinesC =
+		mode_lib->vba.LBLatencyHidingSourceLinesC = (unsigned int)(
 				dml_min(
 						mode_lib->vba.MaxLineBufferLines,
 						(unsigned int) dml_floor(
@@ -2434,7 +2432,7 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 																/ 2,
 														1.0)),
 								1))
-						- (mode_lib->vba.VTAPsChroma[k] - 1);
+				-(mode_lib->vba.VTAPsChroma[k] - 1));
 
 		EffectiveLBLatencyHidingY = mode_lib->vba.LBLatencyHidingSourceLinesY
 				/ mode_lib->vba.VRatio[k]
@@ -2691,14 +2689,14 @@ static void dml20_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPer
 
 		for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
 			if (mode_lib->vba.BlendingAndTiming[k] == k) {
-				unsigned int Margin = (mode_lib->vba.MaxVStartupLines[k] - mode_lib->vba.VStartup[k])
-						* mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k];
+				unsigned int Margin = (unsigned int)((mode_lib->vba.MaxVStartupLines[k] - mode_lib->vba.VStartup[k])
+						* mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]);
 
 				if (FirstMainPlane) {
 					VStartupMargin = Margin;
 					FirstMainPlane = false;
 				} else
-					VStartupMargin = dml_min(VStartupMargin, Margin);
+					VStartupMargin = (unsigned int)dml_min(VStartupMargin, Margin);
 		}
 
 		if (mode_lib->vba.UseMaximumVStartup) {
@@ -2896,26 +2894,26 @@ static void dml20_DisplayPipeConfiguration(struct display_mode_lib *mode_lib)
 
 		if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
 				<= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
-			mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
-			mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
+			   mode_lib->vba.SwathHeightY[k] = (unsigned int)MaximumSwathHeightY;
+			   mode_lib->vba.SwathHeightC[k] = (unsigned int)MaximumSwathHeightC;
 		} else {
-			mode_lib->vba.SwathHeightY[k] = MinimumSwathHeightY;
-			mode_lib->vba.SwathHeightC[k] = MinimumSwathHeightC;
+			   mode_lib->vba.SwathHeightY[k] = (unsigned int)MinimumSwathHeightY;
+			   mode_lib->vba.SwathHeightC[k] = (unsigned int)MinimumSwathHeightC;
 		}
 
 		if (mode_lib->vba.SwathHeightC[k] == 0) {
-			mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0] * 1024;
-			mode_lib->vba.DETBufferSizeC[k] = 0;
+			   mode_lib->vba.DETBufferSizeY[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0] * 1024);
+			   mode_lib->vba.DETBufferSizeC[k] = 0;
 		} else if (mode_lib->vba.SwathHeightY[k] <= mode_lib->vba.SwathHeightC[k]) {
-			mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
-					* 1024.0 / 2;
-			mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
-					* 1024.0 / 2;
+			   mode_lib->vba.DETBufferSizeY[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0]
+				   * 1024.0 / 2);
+			   mode_lib->vba.DETBufferSizeC[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0]
+				   * 1024.0 / 2);
 		} else {
-			mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
-					* 1024.0 * 2 / 3;
-			mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
-					* 1024.0 / 3;
+			   mode_lib->vba.DETBufferSizeY[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0]
+				   * 1024.0 * 2 / 3);
+			   mode_lib->vba.DETBufferSizeC[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0]
+				   * 1024.0 / 3);
 		}
 	}
 }
@@ -3233,23 +3231,23 @@ static unsigned int TruncToValidBPP(
 				if (DecimalBPP < 6)
 					return BPP_INVALID;
 				else if (DecimalBPP >= 1.5 * DSCInputBitPerComponent - 1.0 / 16)
-					return 1.5 * DSCInputBitPerComponent - 1.0 / 16;
+					return (unsigned int)(1.5 * DSCInputBitPerComponent - 1.0 / 16);
 				else
-					return dml_floor(16 * DecimalBPP, 1) / 16;
+					return (unsigned int)(dml_floor(16 * DecimalBPP, 1) / 16);
 			} else if (Format == dm_n422) {
 				if (DecimalBPP < 7)
 					return BPP_INVALID;
 				else if (DecimalBPP >= 2 * DSCInputBitPerComponent - 1.0 / 16)
-					return 2 * DSCInputBitPerComponent - 1.0 / 16;
+					return (unsigned int)(2 * DSCInputBitPerComponent - 1.0 / 16);
 				else
-					return dml_floor(16 * DecimalBPP, 1) / 16;
+					return (unsigned int)(dml_floor(16 * DecimalBPP, 1) / 16);
 			} else {
 				if (DecimalBPP < 8)
 					return BPP_INVALID;
 				else if (DecimalBPP >= 3 * DSCInputBitPerComponent - 1.0 / 16)
-					return 3 * DSCInputBitPerComponent - 1.0 / 16;
+					return (unsigned int)(3 * DSCInputBitPerComponent - 1.0 / 16);
 				else
-					return dml_floor(16 * DecimalBPP, 1) / 16;
+					return (unsigned int)(dml_floor(16 * DecimalBPP, 1) / 16);
 			}
 		} else if (Format == dm_420) {
 			if (DecimalBPP >= 18)
@@ -3288,8 +3286,8 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 {
 	struct vba_vars_st *locals = &mode_lib->vba;
 
-	int i;
-	unsigned int j, k, m;
+	int idx;
+	unsigned int i, j, k, m;
 
 	/*MODE SUPPORT, VOLTAGE STATE AND SOC CONFIGURATION*/
 
@@ -3752,8 +3750,8 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		Calculate256BBlockSizes(
 				mode_lib->vba.SourcePixelFormat[k],
 				mode_lib->vba.SurfaceTiling[k],
-				dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
-				dml_ceil(locals->BytePerPixelInDETC[k], 2.0),
+				(unsigned int)dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
+				(unsigned int)dml_ceil(locals->BytePerPixelInDETC[k], 2.0),
 				&locals->Read256BlockHeightY[k],
 				&locals->Read256BlockHeightC[k],
 				&locals->Read256BlockWidthY[k],
@@ -3930,7 +3928,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					locals->DISPCLK_DPPCLK_Support[i][j] = false;
 				}
 			}
-			locals->TotalNumberOfActiveDPP[i][j] = 0.0;
+			locals->TotalNumberOfActiveDPP[i][j] = (unsigned int)0.0;
 			for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++)
 				locals->TotalNumberOfActiveDPP[i][j] = locals->TotalNumberOfActiveDPP[i][j] + locals->NoOfDPP[i][j][k];
 			if (j == 1) {
@@ -3985,7 +3983,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							|| mode_lib->vba.PlaneRequiredDISPCLK > mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity)
 						locals->DISPCLK_DPPCLK_Support[i][j] = false;
 				}
-				locals->TotalNumberOfActiveDPP[i][j] = 0.0;
+				locals->TotalNumberOfActiveDPP[i][j] = (unsigned int)0.0;
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++)
 					locals->TotalNumberOfActiveDPP[i][j] = locals->TotalNumberOfActiveDPP[i][j] + locals->NoOfDPP[i][j][k];
 			}
@@ -4239,17 +4237,17 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					|| locals->RequiresDSC[i][k] == false) {
 				mode_lib->vba.slices = 0;
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 3200.0) {
-				mode_lib->vba.slices = dml_ceil(
+				mode_lib->vba.slices = (unsigned int)dml_ceil(
 						mode_lib->vba.PixelClockBackEnd[k] / 400.0,
 						4.0);
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 1360.0) {
-				mode_lib->vba.slices = 8.0;
+				mode_lib->vba.slices = (unsigned int)8.0;
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 680.0) {
-				mode_lib->vba.slices = 4.0;
+				mode_lib->vba.slices = (unsigned int)4.0;
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 340.0) {
-				mode_lib->vba.slices = 2.0;
+				mode_lib->vba.slices = (unsigned int)2.0;
 			} else {
-				mode_lib->vba.slices = 1.0;
+				mode_lib->vba.slices = (unsigned int)1.0;
 			}
 			if (locals->OutputBppPerState[i][k] == BPP_BLENDED_PIPE
 					|| locals->OutputBppPerState[i][k] == BPP_INVALID) {
@@ -4263,7 +4261,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							dscceComputeDelay(
 									mode_lib->vba.DSCInputBitPerComponent[k],
 									mode_lib->vba.bpp,
-									dml_ceil(
+									(unsigned int)dml_ceil(
 											mode_lib->vba.HActive[k]
 													/ mode_lib->vba.slices,
 											1.0),
@@ -4276,7 +4274,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							2.0 * (dscceComputeDelay(
 											mode_lib->vba.DSCInputBitPerComponent[k],
 											mode_lib->vba.bpp,
-											dml_ceil(mode_lib->vba.HActive[k] / mode_lib->vba.slices, 1.0),
+											(unsigned int)dml_ceil(mode_lib->vba.HActive[k] / mode_lib->vba.slices, 1.0),
 											mode_lib->vba.slices / 2,
 											mode_lib->vba.OutputFormat[k])
 									+ dscComputeDelay(mode_lib->vba.OutputFormat[k]));
@@ -4302,9 +4300,9 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		for (j = 0; j < 2; j++) {
 			for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
 				if (locals->ODMCombineEnablePerState[i][k] == dm_odm_combine_mode_2to1)
-					locals->SwathWidthYPerState[i][j][k] = dml_min(locals->SwathWidthYSingleDPP[k], dml_round(locals->HActive[k] / 2 * locals->HRatio[k]));
+					locals->SwathWidthYPerState[i][j][k] = (unsigned int)dml_min(locals->SwathWidthYSingleDPP[k], dml_round(locals->HActive[k] / 2 * locals->HRatio[k]));
 				else
-					locals->SwathWidthYPerState[i][j][k] = locals->SwathWidthYSingleDPP[k] / locals->NoOfDPP[i][j][k];
+					locals->SwathWidthYPerState[i][j][k] = (unsigned int)locals->SwathWidthYSingleDPP[k] / locals->NoOfDPP[i][j][k];
 				locals->SwathWidthGranularityY = 256  / dml_ceil(locals->BytePerPixelInDETY[k], 1) / locals->MaxSwathHeightY[k];
 				locals->RoundedUpMaxSwathSizeBytesY = (dml_ceil(locals->SwathWidthYPerState[i][j][k] - 1, locals->SwathWidthGranularityY)
 						+ locals->SwathWidthGranularityY) * locals->BytePerPixelInDETY[k] * locals->MaxSwathHeightY[k];
@@ -4324,11 +4322,11 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				}
 
 				if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte[0] * 1024.0 / 2) {
-					locals->SwathHeightYPerState[i][j][k] = locals->MaxSwathHeightY[k];
-					locals->SwathHeightCPerState[i][j][k] = locals->MaxSwathHeightC[k];
+					locals->SwathHeightYPerState[i][j][k] = (unsigned int)locals->MaxSwathHeightY[k];
+					locals->SwathHeightCPerState[i][j][k] = (unsigned int)locals->MaxSwathHeightC[k];
 				} else {
-					locals->SwathHeightYPerState[i][j][k] = locals->MinSwathHeightY[k];
-					locals->SwathHeightCPerState[i][j][k] = locals->MinSwathHeightC[k];
+					locals->SwathHeightYPerState[i][j][k] = (unsigned int)locals->MinSwathHeightY[k];
+					locals->SwathHeightCPerState[i][j][k] = (unsigned int)locals->MinSwathHeightC[k];
 				}
 
 				if (locals->BytePerPixelInDETC[k] == 0) {
@@ -4343,16 +4341,16 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
 				}
 
-				locals->EffectiveLBLatencyHidingSourceLinesLuma = dml_min(locals->MaxLineBufferLines,
+				locals->EffectiveLBLatencyHidingSourceLinesLuma = (unsigned int)dml_min(locals->MaxLineBufferLines,
 					dml_floor(locals->LineBufferSize / locals->LBBitPerPixel[k] / (locals->SwathWidthYPerState[i][j][k]
 					/ dml_max(locals->HRatio[k], 1)), 1)) - (locals->vtaps[k] - 1);
 
-				locals->EffectiveLBLatencyHidingSourceLinesChroma =  dml_min(locals->MaxLineBufferLines,
+				locals->EffectiveLBLatencyHidingSourceLinesChroma =  (unsigned int)dml_min(locals->MaxLineBufferLines,
 						dml_floor(locals->LineBufferSize / locals->LBBitPerPixel[k]
 						/ (locals->SwathWidthYPerState[i][j][k] / 2
 						/ dml_max(locals->HRatio[k] / 2, 1)), 1)) - (locals->VTAPsChroma[k] - 1);
 
-				locals->EffectiveDETLBLinesLuma = dml_floor(locals->LinesInDETLuma +  dml_min(
+				locals->EffectiveDETLBLinesLuma = (unsigned int)dml_floor(locals->LinesInDETLuma +  dml_min(
 						locals->LinesInDETLuma * locals->RequiredDISPCLK[i][j] * locals->BytePerPixelInDETY[k] *
 						locals->PSCL_FACTOR[k] / locals->ReturnBWPerState[i][0],
 						locals->EffectiveLBLatencyHidingSourceLinesLuma),
@@ -4507,7 +4505,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						mode_lib->vba.Read256BlockWidthY[k],
 						mode_lib->vba.SourcePixelFormat[k],
 						mode_lib->vba.SurfaceTiling[k],
-						dml_ceil(mode_lib->vba.BytePerPixelInDETY[k], 1.0),
+						(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelInDETY[k], 1.0),
 						mode_lib->vba.SourceScan[k],
 						mode_lib->vba.ViewportWidth[k],
 						mode_lib->vba.ViewportHeight[k],
@@ -4546,17 +4544,17 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							mode_lib->vba.Read256BlockWidthY[k],
 							mode_lib->vba.SourcePixelFormat[k],
 							mode_lib->vba.SurfaceTiling[k],
-							dml_ceil(mode_lib->vba.BytePerPixelInDETC[k], 2.0),
+							(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelInDETC[k], 2.0),
 							mode_lib->vba.SourceScan[k],
-							mode_lib->vba.ViewportWidth[k] / 2.0,
-							mode_lib->vba.ViewportHeight[k] / 2.0,
-							mode_lib->vba.SwathWidthYPerState[i][j][k] / 2.0,
+							(unsigned int)(mode_lib->vba.ViewportWidth[k] / 2.0),
+							(unsigned int)(mode_lib->vba.ViewportHeight[k] / 2.0),
+							(unsigned int)(mode_lib->vba.SwathWidthYPerState[i][j][k] / 2.0),
 							mode_lib->vba.GPUVMEnable,
 							mode_lib->vba.VMMPageSize,
 							mode_lib->vba.PTEBufferSizeInRequestsLuma,
 							mode_lib->vba.PDEProcessingBufIn64KBReqs,
 							mode_lib->vba.PitchC[k],
-							0.0,
+							(unsigned int)0.0,
 							&mode_lib->vba.MacroTileWidthC[k],
 							&mode_lib->vba.MetaRowBytesC,
 							&mode_lib->vba.DPTEBytesPerRowC,
@@ -4575,8 +4573,8 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							&mode_lib->vba.MaxNumSwC[k]);
 				} else {
 					mode_lib->vba.PDEAndMetaPTEBytesPerFrameC = 0.0;
-					mode_lib->vba.MetaRowBytesC = 0.0;
-					mode_lib->vba.DPTEBytesPerRowC = 0.0;
+					mode_lib->vba.MetaRowBytesC = (unsigned int)0.0;
+					mode_lib->vba.DPTEBytesPerRowC = (unsigned int)0.0;
 					locals->PrefetchLinesC[0][0][k] = 0.0;
 					locals->PTEBufferSizeNotExceededC[i][j][k] = true;
 					locals->PTEBufferSizeInRequestsForLuma = mode_lib->vba.PTEBufferSizeInRequestsLuma + mode_lib->vba.PTEBufferSizeInRequestsChroma;
@@ -4632,7 +4630,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 										mode_lib->vba.WritebackLumaVTaps[k],
 										mode_lib->vba.WritebackChromaHTaps[k],
 										mode_lib->vba.WritebackChromaVTaps[k],
-										mode_lib->vba.WritebackDestinationWidth[k]) / locals->RequiredDISPCLK[i][j];
+										(unsigned int)mode_lib->vba.WritebackDestinationWidth[k]) / locals->RequiredDISPCLK[i][j];
 					} else {
 						locals->WritebackDelay[i][k] = 0.0;
 					}
@@ -4649,7 +4647,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 													mode_lib->vba.WritebackLumaVTaps[m],
 													mode_lib->vba.WritebackChromaHTaps[m],
 													mode_lib->vba.WritebackChromaVTaps[m],
-													mode_lib->vba.WritebackDestinationWidth[m]) / locals->RequiredDISPCLK[i][j]);
+													(unsigned int)mode_lib->vba.WritebackDestinationWidth[m]) / locals->RequiredDISPCLK[i][j]);
 						}
 					}
 				}
@@ -4714,7 +4712,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 									mode_lib->vba.RequiredDISPCLK[i][j],
 									mode_lib->vba.PixelClock[k],
 									mode_lib->vba.ProjectedDCFCLKDeepSleep[0][0],
-									mode_lib->vba.DSCDelayPerState[i][k],
+									(unsigned int)mode_lib->vba.DSCDelayPerState[i][k],
 									mode_lib->vba.NoOfDPP[i][j][k],
 									mode_lib->vba.ScalerEnabled[k],
 									mode_lib->vba.NumberOfCursors[k],
@@ -4724,14 +4722,14 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 									mode_lib->vba.DPPCLKDelayCNVCFormater,
 									mode_lib->vba.DPPCLKDelayCNVCCursor,
 									mode_lib->vba.DISPCLKDelaySubtotal,
-									mode_lib->vba.SwathWidthYPerState[i][j][k]
-											/ mode_lib->vba.HRatio[k],
+									(unsigned int)(mode_lib->vba.SwathWidthYPerState[i][j][k]
+										      / mode_lib->vba.HRatio[k]),
 									mode_lib->vba.OutputFormat[k],
 									mode_lib->vba.VTotal[k]
 											- mode_lib->vba.VActive[k],
 									mode_lib->vba.HTotal[k],
 									mode_lib->vba.MaxInterDCNTileRepeaters,
-									mode_lib->vba.MaximumVStartup[0][0][k],
+									(unsigned int)mode_lib->vba.MaximumVStartup[0][0][k],
 									mode_lib->vba.GPUVMMaxPageTableLevels,
 									mode_lib->vba.GPUVMEnable,
 									mode_lib->vba.DynamicMetadataEnable[k],
@@ -4741,9 +4739,9 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 									mode_lib->vba.UrgentLatencyPixelDataOnly,
 									mode_lib->vba.ExtraLatency,
 									mode_lib->vba.TimeCalc,
-									mode_lib->vba.PDEAndMetaPTEBytesPerFrame[0][0][k],
-									mode_lib->vba.MetaRowBytes[0][0][k],
-									mode_lib->vba.DPTEBytesPerRow[0][0][k],
+									(unsigned int)mode_lib->vba.PDEAndMetaPTEBytesPerFrame[0][0][k],
+									(unsigned int)mode_lib->vba.MetaRowBytes[0][0][k],
+									(unsigned int)mode_lib->vba.DPTEBytesPerRow[0][0][k],
 									mode_lib->vba.PrefetchLinesY[0][0][k],
 									mode_lib->vba.SwathWidthYPerState[i][j][k],
 									mode_lib->vba.BytePerPixelInDETY[k],
@@ -4852,16 +4850,16 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 											mode_lib->vba.PrefetchBW[k]);
 				}
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
-					mode_lib->vba.ImmediateFlipBytes[k] = 0.0;
+					mode_lib->vba.ImmediateFlipBytes[k] = (unsigned int)0.0;
 					if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
 							&& mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
 						mode_lib->vba.ImmediateFlipBytes[k] =
-								mode_lib->vba.PDEAndMetaPTEBytesPerFrame[0][0][k]
+								(unsigned int)(mode_lib->vba.PDEAndMetaPTEBytesPerFrame[0][0][k]
 										+ mode_lib->vba.MetaRowBytes[0][0][k]
-										+ mode_lib->vba.DPTEBytesPerRow[0][0][k];
+										+ mode_lib->vba.DPTEBytesPerRow[0][0][k]);
 					}
 				}
-				mode_lib->vba.TotImmediateFlipBytes = 0.0;
+				   mode_lib->vba.TotImmediateFlipBytes = 0;
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
 					if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
 							&& mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
@@ -4887,8 +4885,8 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							mode_lib->vba.VRatio[k],
 							mode_lib->vba.Tno_bw[k],
 							mode_lib->vba.PDEAndMetaPTEBytesPerFrame[0][0][k],
-							mode_lib->vba.MetaRowBytes[0][0][k],
-							mode_lib->vba.DPTEBytesPerRow[0][0][k],
+							(unsigned int)mode_lib->vba.MetaRowBytes[0][0][k],
+							(unsigned int)mode_lib->vba.DPTEBytesPerRow[0][0][k],
 							mode_lib->vba.DCCEnable[k],
 							mode_lib->vba.dpte_row_height[k],
 							mode_lib->vba.meta_row_height[k],
@@ -5018,7 +5016,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	}
 	/*Mode Support, Voltage State and SOC Configuration*/
 
-	for (i = mode_lib->vba.soc.num_states; i >= 0; i--) {
+	for (idx = mode_lib->vba.soc.num_states; idx >= 0; idx--) {
 		for (j = 0; j < 2; j++) {
 			enum dm_validation_status status = DML_VALIDATION_OK;
 
@@ -5026,21 +5024,21 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				status = DML_FAIL_SCALE_RATIO_TAP;
 			} else if (mode_lib->vba.SourceFormatPixelAndScanSupport != true) {
 				status = DML_FAIL_SOURCE_PIXEL_FORMAT;
-			} else if (locals->ViewportSizeSupport[i][0] != true) {
+			} else if (locals->ViewportSizeSupport[idx][0] != true) {
 				status = DML_FAIL_VIEWPORT_SIZE;
-			} else if (locals->DIOSupport[i] != true) {
+			} else if (locals->DIOSupport[idx] != true) {
 				status = DML_FAIL_DIO_SUPPORT;
-			} else if (locals->NotEnoughDSCUnits[i] != false) {
+			} else if (locals->NotEnoughDSCUnits[idx] != false) {
 				status = DML_FAIL_NOT_ENOUGH_DSC;
-			} else if (locals->DSCCLKRequiredMoreThanSupported[i] != false) {
+			} else if (locals->DSCCLKRequiredMoreThanSupported[idx] != false) {
 				status = DML_FAIL_DSC_CLK_REQUIRED;
-			} else if (locals->UrgentLatencySupport[i][j] != true) {
+			} else if (locals->UrgentLatencySupport[idx][j] != true) {
 				status = DML_FAIL_URGENT_LATENCY;
-			} else if (locals->ROBSupport[i][0] != true) {
+			} else if (locals->ROBSupport[idx][0] != true) {
 				status = DML_FAIL_REORDERING_BUFFER;
-			} else if (locals->DISPCLK_DPPCLK_Support[i][j] != true) {
+			} else if (locals->DISPCLK_DPPCLK_Support[idx][j] != true) {
 				status = DML_FAIL_DISPCLK_DPPCLK;
-			} else if (locals->TotalAvailablePipesSupport[i][j] != true) {
+			} else if (locals->TotalAvailablePipesSupport[idx][j] != true) {
 				status = DML_FAIL_TOTAL_AVAILABLE_PIPES;
 			} else if (mode_lib->vba.NumberOfOTGSupport != true) {
 				status = DML_FAIL_NUM_OTG;
@@ -5054,24 +5052,24 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				status = DML_FAIL_CURSOR_SUPPORT;
 			} else if (mode_lib->vba.PitchSupport != true) {
 				status = DML_FAIL_PITCH_SUPPORT;
-			} else if (locals->PrefetchSupported[i][j] != true) {
+			} else if (locals->PrefetchSupported[idx][j] != true) {
 				status = DML_FAIL_PREFETCH_SUPPORT;
-			} else if (locals->TotalVerticalActiveBandwidthSupport[i][0] != true) {
+			} else if (locals->TotalVerticalActiveBandwidthSupport[idx][0] != true) {
 				status = DML_FAIL_TOTAL_V_ACTIVE_BW;
-			} else if (locals->VRatioInPrefetchSupported[i][j] != true) {
+			} else if (locals->VRatioInPrefetchSupported[idx][j] != true) {
 				status = DML_FAIL_V_RATIO_PREFETCH;
-			} else if (locals->PTEBufferSizeNotExceeded[i][j] != true) {
+			} else if (locals->PTEBufferSizeNotExceeded[idx][j] != true) {
 				status = DML_FAIL_PTE_BUFFER_SIZE;
 			} else if (mode_lib->vba.NonsupportedDSCInputBPC != false) {
 				status = DML_FAIL_DSC_INPUT_BPC;
 			}
 
 			if (status == DML_VALIDATION_OK) {
-				locals->ModeSupport[i][j] = true;
+				locals->ModeSupport[idx][j] = true;
 			} else {
-				locals->ModeSupport[i][j] = false;
+				locals->ModeSupport[idx][j] = false;
 			}
-			locals->ValidationStatus[i] = status;
+			locals->ValidationStatus[idx] = status;
 		}
 	}
 	{
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c
index 95b0a35018806..a6b8b5ddd9457 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c
@@ -350,7 +350,7 @@ static unsigned int dscceComputeDelay(
 		pixelsPerClock = 1;
 
 	//initial transmit delay as per PPS
-	initalXmitDelay = dml_round(rcModelSize / 2.0 / bpp / pixelsPerClock);
+	initalXmitDelay = (unsigned int)dml_round(rcModelSize / 2.0 / bpp / pixelsPerClock);
 
 	//compute ssm delay
 	if (bpc == 8)
@@ -510,13 +510,13 @@ static bool CalculateDelayAfterScaler(
 	}
 
 	if (ScalerEnabled)
-		DPPCycles = DPPCLKDelaySubtotal + DPPCLKDelaySCL;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotal + DPPCLKDelaySCL);
 	else
-		DPPCycles = DPPCLKDelaySubtotal + DPPCLKDelaySCLLBOnly;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotal + DPPCLKDelaySCLLBOnly);
 
-	DPPCycles = DPPCycles + DPPCLKDelayCNVCFormater + NumberOfCursors * DPPCLKDelayCNVCCursor;
+	DPPCycles = (unsigned int)(DPPCycles + DPPCLKDelayCNVCFormater + NumberOfCursors * DPPCLKDelayCNVCCursor);
 
-	DISPCLKCycles = DISPCLKDelaySubtotal;
+	DISPCLKCycles = (unsigned int)DISPCLKDelaySubtotal;
 
 	if (DPPCLK == 0.0 || DISPCLK == 0.0)
 		return true;
@@ -609,7 +609,7 @@ static bool CalculatePrefetchSchedule(
 	double TimeForFetchingRowInVBlank = 0;
 	double LinesToRequestPrefetchPixelData = 0;
 
-	*VUpdateOffsetPix = dml_ceil(HTotal / 4.0, 1);
+	*VUpdateOffsetPix = (unsigned int)dml_ceil(HTotal / 4.0, 1);
 	TotalRepeaterDelayTime = MaxInterDCNTileRepeaters * (2.0 / DPPCLK + 3.0 / DISPCLK);
 	*VUpdateWidthPix = (14.0 / DCFCLKDeepSleep + 12.0 / DPPCLK + TotalRepeaterDelayTime)
 			* PixelClock;
@@ -889,14 +889,13 @@ static double CalculatePrefetchSourceLines(
 
 	if (!mode_lib->vba.IgnoreViewportPositioning) {
 
-		*MaxNumSwath = dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0;
+		*MaxNumSwath = (unsigned int)(dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0);
 
 		if (*VInitPreFill > 1.0)
-			MaxPartialSwath = (unsigned int) (*VInitPreFill - 2) % SwathHeight;
+			   MaxPartialSwath = (unsigned int)((unsigned int)(*VInitPreFill - 2) % SwathHeight);
 		else
-			MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 2)
-					% SwathHeight;
-		MaxPartialSwath = dml_max(1U, MaxPartialSwath);
+			   MaxPartialSwath = (unsigned int)((unsigned int)(*VInitPreFill + SwathHeight - 2) % SwathHeight);
+		MaxPartialSwath = (unsigned int)dml_max(1U, MaxPartialSwath);
 
 	} else {
 
@@ -904,13 +903,12 @@ static double CalculatePrefetchSourceLines(
 			dml_print(
 					"WARNING DML: using viewport y position of 0 even though actual viewport y position is non-zero in prefetch source lines calculation\n");
 
-		*MaxNumSwath = dml_ceil(*VInitPreFill / SwathHeight, 1);
+		*MaxNumSwath = (unsigned int)dml_ceil(*VInitPreFill / SwathHeight, 1);
 
 		if (*VInitPreFill > 1.0)
-			MaxPartialSwath = (unsigned int) (*VInitPreFill - 1) % SwathHeight;
+			   MaxPartialSwath = (unsigned int)((unsigned int)(*VInitPreFill - 1) % SwathHeight);
 		else
-			MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 1)
-					% SwathHeight;
+			   MaxPartialSwath = (unsigned int)((unsigned int)(*VInitPreFill + SwathHeight - 1) % SwathHeight);
 	}
 
 	return *MaxNumSwath * SwathHeight + MaxPartialSwath;
@@ -960,34 +958,34 @@ static unsigned int CalculateVMAndRowBytes(
 		MetaRequestHeight = 8 * BlockHeight256Bytes;
 		MetaRequestWidth = 8 * BlockWidth256Bytes;
 		if (ScanDirection == dm_horz) {
-			*meta_row_height = MetaRequestHeight;
-			MetaSurfWidth = dml_ceil((double) SwathWidth - 1, MetaRequestWidth)
-					+ MetaRequestWidth;
-			*MetaRowByte = MetaSurfWidth * MetaRequestHeight * BytePerPixel / 256.0;
+			*meta_row_height = (unsigned int)MetaRequestHeight;
+			MetaSurfWidth = (unsigned int)(dml_ceil((double) SwathWidth - 1, MetaRequestWidth)
+					+ MetaRequestWidth);
+			   *MetaRowByte = (unsigned int)(MetaSurfWidth * MetaRequestHeight * BytePerPixel / 256.0);
 		} else {
-			*meta_row_height = MetaRequestWidth;
-			MetaSurfHeight = dml_ceil((double) SwathWidth - 1, MetaRequestHeight)
-					+ MetaRequestHeight;
-			*MetaRowByte = MetaSurfHeight * MetaRequestWidth * BytePerPixel / 256.0;
+			*meta_row_height = (unsigned int)MetaRequestWidth;
+			MetaSurfHeight = (unsigned int)(dml_ceil((double) SwathWidth - 1, MetaRequestHeight)
+					+ MetaRequestHeight);
+			   *MetaRowByte = (unsigned int)(MetaSurfHeight * MetaRequestWidth * BytePerPixel / 256.0);
 		}
 		if (ScanDirection == dm_horz) {
-			DCCMetaSurfaceBytes = DCCMetaPitch
+			DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch
 					* (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes)
 							+ 64 * BlockHeight256Bytes) * BytePerPixel
-					/ 256;
+					/ 256);
 		} else {
-			DCCMetaSurfaceBytes = DCCMetaPitch
+			DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch
 					* (dml_ceil(
 							(double) ViewportHeight - 1,
 							64 * BlockHeight256Bytes)
 							+ 64 * BlockHeight256Bytes) * BytePerPixel
-					/ 256;
+					/ 256);
 		}
 		if (GPUVMEnable == true) {
-			MetaPTEBytesFrame = (dml_ceil(
+			MetaPTEBytesFrame = (unsigned int)((dml_ceil(
 					(double) (DCCMetaSurfaceBytes - VMMPageSize)
 							/ (8 * VMMPageSize),
-					1) + 1) * 64;
+					1) + 1) * 64);
 			MPDEBytesFrame = 128 * (mode_lib->vba.GPUVMMaxPageTableLevels - 1);
 		} else {
 			MetaPTEBytesFrame = 0;
@@ -1021,7 +1019,7 @@ static unsigned int CalculateVMAndRowBytes(
 	if (GPUVMEnable == true && mode_lib->vba.GPUVMMaxPageTableLevels > 1) {
 		if (ScanDirection == dm_horz) {
 			DPDE0BytesFrame =
-					64
+					(unsigned int)(64
 							* (dml_ceil(
 									((Pitch
 											* (dml_ceil(
@@ -1033,10 +1031,10 @@ static unsigned int CalculateVMAndRowBytes(
 											- MacroTileSizeBytes)
 											/ (8
 													* 2097152),
-									1) + 1);
+									1) + 1));
 		} else {
 			DPDE0BytesFrame =
-					64
+					(unsigned int)(64
 							* (dml_ceil(
 									((Pitch
 											* (dml_ceil(
@@ -1048,7 +1046,7 @@ static unsigned int CalculateVMAndRowBytes(
 											- MacroTileSizeBytes)
 											/ (8
 													* 2097152),
-									1) + 1);
+									1) + 1));
 		}
 		ExtraDPDEBytesFrame = 128 * (mode_lib->vba.GPUVMMaxPageTableLevels - 2);
 	} else {
@@ -1068,7 +1066,7 @@ static unsigned int CalculateVMAndRowBytes(
 
 		if (SurfaceTiling == dm_sw_linear) {
 			PixelPTEReqHeight = 1;
-			PixelPTEReqWidth = 8.0 * VMMPageSize / BytePerPixel;
+			PixelPTEReqWidth = (unsigned int)8.0 * VMMPageSize / BytePerPixel;
 			PTERequestSize = 64;
 			FractionOfPTEReturnDrop = 0;
 		} else if (MacroTileSizeBytes == 4096) {
@@ -1097,7 +1095,7 @@ static unsigned int CalculateVMAndRowBytes(
 			EffectivePDEProcessingBufIn64KBReqs = PDEProcessingBufIn64KBReqs;
 
 		if (SurfaceTiling == dm_sw_linear) {
-			*dpte_row_height =
+			*dpte_row_height = (unsigned int)(
 					dml_min(
 							128,
 							1
@@ -1110,21 +1108,21 @@ static unsigned int CalculateVMAndRowBytes(
 																	* 65536.0
 																	/ BytePerPixel)
 															/ Pitch),
-											1));
+										1)));
 			*PixelPTEBytesPerRow = PTERequestSize
-					* (dml_ceil(
+					* (unsigned int)(dml_ceil(
 							(double) (Pitch * *dpte_row_height - 1)
 									/ PixelPTEReqWidth,
 							1) + 1);
 		} else if (ScanDirection == dm_horz) {
 			*dpte_row_height = PixelPTEReqHeight;
 			*PixelPTEBytesPerRow = PTERequestSize
-					* (dml_ceil(((double) SwathWidth - 1) / PixelPTEReqWidth, 1)
+					* (unsigned int)(dml_ceil(((double) SwathWidth - 1) / PixelPTEReqWidth, 1)
 							+ 1);
 		} else {
-			*dpte_row_height = dml_min(PixelPTEReqWidth, *MacroTileWidth);
+			*dpte_row_height = (unsigned int)dml_min(PixelPTEReqWidth, *MacroTileWidth);
 			*PixelPTEBytesPerRow = PTERequestSize
-					* (dml_ceil(
+					* (unsigned int)(dml_ceil(
 							((double) SwathWidth - 1)
 									/ PixelPTEReqHeight,
 							1) + 1);
@@ -1576,10 +1574,10 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 								/ mode_lib->vba.PixelClock[k])
 						/ mode_lib->vba.VRatio[k];
 		if (mode_lib->vba.BytePerPixelDETC[k] > 0) {
-			mode_lib->vba.LinesInDETC[k] = mode_lib->vba.DETBufferSizeC[k]
+			mode_lib->vba.LinesInDETC[k] = (unsigned int)(mode_lib->vba.DETBufferSizeC[k]
 					/ mode_lib->vba.BytePerPixelDETC[k]
-					/ (mode_lib->vba.SwathWidthY[k] / 2);
-			mode_lib->vba.LinesInDETCRoundedDownToSwath[k] = dml_floor(
+					/ (mode_lib->vba.SwathWidthY[k] / 2));
+			mode_lib->vba.LinesInDETCRoundedDownToSwath[k] = (unsigned int)dml_floor(
 					mode_lib->vba.LinesInDETC[k],
 					mode_lib->vba.SwathHeightC[k]);
 			mode_lib->vba.FullDETBufferingTimeC[k] =
@@ -1848,7 +1846,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 						dscceComputeDelay(
 								mode_lib->vba.DSCInputBitPerComponent[k],
 								bpp,
-								dml_ceil(
+								(unsigned int)dml_ceil(
 										(double) mode_lib->vba.HActive[k]
 												/ mode_lib->vba.NumberOfDSCSlices[k],
 										1),
@@ -1862,18 +1860,18 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 								* (dscceComputeDelay(
 										mode_lib->vba.DSCInputBitPerComponent[k],
 										bpp,
-										dml_ceil(
+										(unsigned int)dml_ceil(
 												(double) mode_lib->vba.HActive[k]
 														/ mode_lib->vba.NumberOfDSCSlices[k],
 												1),
-										slices / 2.0,
+										(unsigned int)(slices / 2.0),
 										mode_lib->vba.OutputFormat[k])
 										+ dscComputeDelay(
 												mode_lib->vba.OutputFormat[k]));
 			}
-			mode_lib->vba.DSCDelay[k] = mode_lib->vba.DSCDelay[k]
+				mode_lib->vba.DSCDelay[k] = (unsigned int)(mode_lib->vba.DSCDelay[k]
 					* mode_lib->vba.PixelClock[k]
-					/ mode_lib->vba.PixelClockBackEnd[k];
+						/ mode_lib->vba.PixelClockBackEnd[k]);
 		} else {
 			mode_lib->vba.DSCDelay[k] = 0;
 		}
@@ -1897,8 +1895,8 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 		Calculate256BBlockSizes(
 				mode_lib->vba.SourcePixelFormat[k],
 				mode_lib->vba.SurfaceTiling[k],
-				dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
-				dml_ceil(mode_lib->vba.BytePerPixelDETC[k], 2),
+				(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
+				(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelDETC[k], 2),
 				&mode_lib->vba.BlockHeight256BytesY[k],
 				&mode_lib->vba.BlockHeight256BytesC[k],
 				&mode_lib->vba.BlockWidth256BytesY[k],
@@ -1910,11 +1908,11 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 				mode_lib->vba.BlockWidth256BytesY[k],
 				mode_lib->vba.SourcePixelFormat[k],
 				mode_lib->vba.SurfaceTiling[k],
-				dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
+				(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelDETY[k], 1),
 				mode_lib->vba.SourceScan[k],
 				mode_lib->vba.ViewportWidth[k],
 				mode_lib->vba.ViewportHeight[k],
-				mode_lib->vba.SwathWidthY[k],
+				(unsigned int)mode_lib->vba.SwathWidthY[k],
 				mode_lib->vba.GPUVMEnable,
 				mode_lib->vba.VMMPageSize,
 				mode_lib->vba.PTEBufferSizeInRequestsLuma,
@@ -1950,13 +1948,13 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 							mode_lib->vba.BlockWidth256BytesC[k],
 							mode_lib->vba.SourcePixelFormat[k],
 							mode_lib->vba.SurfaceTiling[k],
-							dml_ceil(
+							(unsigned int)dml_ceil(
 									mode_lib->vba.BytePerPixelDETC[k],
 									2),
 							mode_lib->vba.SourceScan[k],
 							mode_lib->vba.ViewportWidth[k] / 2,
 							mode_lib->vba.ViewportHeight[k] / 2,
-							mode_lib->vba.SwathWidthY[k] / 2,
+							(unsigned int)mode_lib->vba.SwathWidthY[k] / 2,
 							mode_lib->vba.GPUVMEnable,
 							mode_lib->vba.VMMPageSize,
 							mode_lib->vba.PTEBufferSizeInRequestsLuma,
@@ -2026,7 +2024,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 										mode_lib->vba.WritebackLumaVTaps[k],
 										mode_lib->vba.WritebackChromaHTaps[k],
 										mode_lib->vba.WritebackChromaVTaps[k],
-										mode_lib->vba.WritebackDestinationWidth[k])
+										(unsigned int)mode_lib->vba.WritebackDestinationWidth[k])
 										/ mode_lib->vba.DISPCLK;
 			} else
 				mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k] = 0;
@@ -2045,7 +2043,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 													mode_lib->vba.WritebackLumaVTaps[j],
 													mode_lib->vba.WritebackChromaHTaps[j],
 													mode_lib->vba.WritebackChromaVTaps[j],
-													mode_lib->vba.WritebackDestinationWidth[j])
+													(unsigned int)mode_lib->vba.WritebackDestinationWidth[j])
 													/ mode_lib->vba.DISPCLK);
 				}
 			}
@@ -2060,7 +2058,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 
 	mode_lib->vba.VStartupLines = 13;
 	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
-		mode_lib->vba.MaxVStartupLines[k] =
+		mode_lib->vba.MaxVStartupLines[k] = (unsigned int)(
 				mode_lib->vba.VTotal[k] - mode_lib->vba.VActive[k]
 						- dml_max(
 								1.0,
@@ -2068,11 +2066,11 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 										mode_lib->vba.WritebackDelay[mode_lib->vba.VoltageLevel][k]
 												/ (mode_lib->vba.HTotal[k]
 														/ mode_lib->vba.PixelClock[k]),
-										1));
+										1)));
 	}
 
 	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
-		mode_lib->vba.MaximumMaxVStartupLines = dml_max(
+		mode_lib->vba.MaximumMaxVStartupLines = (unsigned int)dml_max(
 				mode_lib->vba.MaximumMaxVStartupLines,
 				mode_lib->vba.MaxVStartupLines[k]);
 
@@ -2129,8 +2127,8 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 					mode_lib->vba.DisplayPipeLineDeliveryTimeLuma[k], mode_lib->vba.DisplayPipeLineDeliveryTimeChroma[k],
 					mode_lib->vba.DPPCLK[k], mode_lib->vba.DISPCLK, mode_lib->vba.PixelClock[k], mode_lib->vba.DSCDelay[k], mode_lib->vba.DPPPerPlane[k], mode_lib->vba.ScalerEnabled[k], mode_lib->vba.NumberOfCursors[k],
 					mode_lib->vba.DPPCLKDelaySubtotal, mode_lib->vba.DPPCLKDelaySCL, mode_lib->vba.DPPCLKDelaySCLLBOnly, mode_lib->vba.DPPCLKDelayCNVCFormater, mode_lib->vba.DPPCLKDelayCNVCCursor, mode_lib->vba.DISPCLKDelaySubtotal,
-					mode_lib->vba.SwathWidthY[k] / mode_lib->vba.HRatio[k], mode_lib->vba.OutputFormat[k], mode_lib->vba.HTotal[k],
-					mode_lib->vba.SwathWidthSingleDPPY[k], mode_lib->vba.BytePerPixelDETY[k], mode_lib->vba.BytePerPixelDETC[k], mode_lib->vba.SwathHeightY[k], mode_lib->vba.SwathHeightC[k], mode_lib->vba.Interlace[k],
+					(unsigned int)(mode_lib->vba.SwathWidthY[k] / mode_lib->vba.HRatio[k]), mode_lib->vba.OutputFormat[k], mode_lib->vba.HTotal[k],
+					(unsigned int)mode_lib->vba.SwathWidthSingleDPPY[k], mode_lib->vba.BytePerPixelDETY[k], mode_lib->vba.BytePerPixelDETC[k], mode_lib->vba.SwathHeightY[k], mode_lib->vba.SwathHeightC[k], mode_lib->vba.Interlace[k],
 					mode_lib->vba.ProgressiveToInterlaceUnitInOPP, &mode_lib->vba.DSTXAfterScaler[k], &mode_lib->vba.DSTYAfterScaler[k]);
 
 			mode_lib->vba.ErrorResult[k] =
@@ -2146,7 +2144,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 									- mode_lib->vba.VActive[k],
 							mode_lib->vba.HTotal[k],
 							mode_lib->vba.MaxInterDCNTileRepeaters,
-							dml_min(
+							(unsigned int)dml_min(
 									mode_lib->vba.VStartupLines,
 									mode_lib->vba.MaxVStartupLines[k]),
 							mode_lib->vba.GPUVMMaxPageTableLevels,
@@ -2158,11 +2156,11 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 							mode_lib->vba.UrgentLatencyPixelDataOnly,
 							mode_lib->vba.UrgentExtraLatency,
 							mode_lib->vba.TCalc,
-							mode_lib->vba.PDEAndMetaPTEBytesFrame[k],
-							mode_lib->vba.MetaRowByte[k],
-							mode_lib->vba.PixelPTEBytesPerRow[k],
+							(unsigned int)mode_lib->vba.PDEAndMetaPTEBytesFrame[k],
+							(unsigned int)mode_lib->vba.MetaRowByte[k],
+							(unsigned int)mode_lib->vba.PixelPTEBytesPerRow[k],
 							mode_lib->vba.PrefetchSourceLinesY[k],
-							mode_lib->vba.SwathWidthY[k],
+							(unsigned int)mode_lib->vba.SwathWidthY[k],
 							mode_lib->vba.BytePerPixelDETY[k],
 							mode_lib->vba.VInitPreFillY[k],
 							mode_lib->vba.MaxNumSwathY[k],
@@ -2192,7 +2190,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 							&mode_lib->vba.VReadyOffsetPix[k]);
 
 			if (mode_lib->vba.BlendingAndTiming[k] == k) {
-				mode_lib->vba.VStartup[k] = dml_min(
+				mode_lib->vba.VStartup[k] = (unsigned int)dml_min(
 						mode_lib->vba.VStartupLines,
 						mode_lib->vba.MaxVStartupLines[k]);
 				if (mode_lib->vba.VStartupRequiredWhenNotEnoughTimeForDynamicMetadata
@@ -2201,7 +2199,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 							mode_lib->vba.VStartupRequiredWhenNotEnoughTimeForDynamicMetadata;
 				}
 			} else {
-				mode_lib->vba.VStartup[k] =
+				mode_lib->vba.VStartup[k] = (unsigned int)
 						dml_min(
 								mode_lib->vba.VStartupLines,
 								mode_lib->vba.MaxVStartupLines[mode_lib->vba.BlendingAndTiming[k]]);
@@ -2288,10 +2286,10 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 				ImmediateFlipBytes[k] = 0;
 				if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
 						&& mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
-					ImmediateFlipBytes[k] =
+					ImmediateFlipBytes[k] = (unsigned int)(
 							mode_lib->vba.PDEAndMetaPTEBytesFrame[k]
 									+ mode_lib->vba.MetaRowByte[k]
-									+ mode_lib->vba.PixelPTEBytesPerRow[k];
+									+ mode_lib->vba.PixelPTEBytesPerRow[k]);
 				}
 			}
 			mode_lib->vba.TotImmediateFlipBytes = 0;
@@ -2319,8 +2317,8 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 						mode_lib->vba.VRatio[k],
 						mode_lib->vba.Tno_bw[k],
 						mode_lib->vba.PDEAndMetaPTEBytesFrame[k],
-						mode_lib->vba.MetaRowByte[k],
-						mode_lib->vba.PixelPTEBytesPerRow[k],
+						(unsigned int)mode_lib->vba.MetaRowByte[k],
+						(unsigned int)mode_lib->vba.PixelPTEBytesPerRow[k],
 						mode_lib->vba.DCCEnable[k],
 						mode_lib->vba.dpte_row_height[k],
 						mode_lib->vba.meta_row_height[k],
@@ -2444,7 +2442,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 		double MaxDETBufferingTimeY;
 		double ActiveDRAMClockChangeLatencyMarginY;
 
-		mode_lib->vba.LBLatencyHidingSourceLinesY =
+		mode_lib->vba.LBLatencyHidingSourceLinesY = (unsigned int)(
 				dml_min(
 						mode_lib->vba.MaxLineBufferLines,
 						(unsigned int) dml_floor(
@@ -2454,9 +2452,9 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 												/ dml_max(
 														mode_lib->vba.HRatio[k],
 														1.0)),
-								1)) - (mode_lib->vba.vtaps[k] - 1);
+								1)) -(mode_lib->vba.vtaps[k] - 1));
 
-		mode_lib->vba.LBLatencyHidingSourceLinesC =
+		mode_lib->vba.LBLatencyHidingSourceLinesC = (unsigned int)(
 				dml_min(
 						mode_lib->vba.MaxLineBufferLines,
 						(unsigned int) dml_floor(
@@ -2469,7 +2467,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 																/ 2,
 														1.0)),
 								1))
-						- (mode_lib->vba.VTAPsChroma[k] - 1);
+				-(mode_lib->vba.VTAPsChroma[k] - 1));
 
 		EffectiveLBLatencyHidingY = mode_lib->vba.LBLatencyHidingSourceLinesY
 				/ mode_lib->vba.VRatio[k]
@@ -2631,7 +2629,7 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 				&& mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k]
 						< SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank) {
 			SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank =
-					mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k];
+					(float)mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k];
 		}
 	}
 
@@ -2765,14 +2763,14 @@ static void dml20v2_DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndP
 
 		for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
 			if (mode_lib->vba.BlendingAndTiming[k] == k) {
-				unsigned int Margin = (mode_lib->vba.MaxVStartupLines[k] - mode_lib->vba.VStartup[k])
-						* mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k];
+				unsigned int Margin = (unsigned int)((mode_lib->vba.MaxVStartupLines[k] - mode_lib->vba.VStartup[k])
+						* mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]);
 
 				if (FirstMainPlane) {
 					VStartupMargin = Margin;
 					FirstMainPlane = false;
 				} else
-					VStartupMargin = dml_min(VStartupMargin, Margin);
+					VStartupMargin = (unsigned int)dml_min(VStartupMargin, Margin);
 		}
 
 		if (mode_lib->vba.UseMaximumVStartup) {
@@ -2970,26 +2968,26 @@ static void dml20v2_DisplayPipeConfiguration(struct display_mode_lib *mode_lib)
 
 		if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
 				<= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
-			mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
-			mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
+			   mode_lib->vba.SwathHeightY[k] = (unsigned int)MaximumSwathHeightY;
+			   mode_lib->vba.SwathHeightC[k] = (unsigned int)MaximumSwathHeightC;
 		} else {
-			mode_lib->vba.SwathHeightY[k] = MinimumSwathHeightY;
-			mode_lib->vba.SwathHeightC[k] = MinimumSwathHeightC;
+			   mode_lib->vba.SwathHeightY[k] = (unsigned int)MinimumSwathHeightY;
+			   mode_lib->vba.SwathHeightC[k] = (unsigned int)MinimumSwathHeightC;
 		}
 
 		if (mode_lib->vba.SwathHeightC[k] == 0) {
-			mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0] * 1024;
-			mode_lib->vba.DETBufferSizeC[k] = 0;
+			   mode_lib->vba.DETBufferSizeY[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0] * 1024);
+			   mode_lib->vba.DETBufferSizeC[k] = 0;
 		} else if (mode_lib->vba.SwathHeightY[k] <= mode_lib->vba.SwathHeightC[k]) {
-			mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
-					* 1024.0 / 2;
-			mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
-					* 1024.0 / 2;
+			   mode_lib->vba.DETBufferSizeY[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0]
+				   * 1024.0 / 2);
+			   mode_lib->vba.DETBufferSizeC[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0]
+				   * 1024.0 / 2);
 		} else {
-			mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
-					* 1024.0 * 2 / 3;
-			mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
-					* 1024.0 / 3;
+			   mode_lib->vba.DETBufferSizeY[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0]
+				   * 1024.0 * 2 / 3);
+			   mode_lib->vba.DETBufferSizeC[k] = (unsigned int)(mode_lib->vba.DETBufferSizeInKByte[0]
+				   * 1024.0 / 3);
 		}
 	}
 }
@@ -3309,9 +3307,9 @@ static unsigned int TruncToValidBPP(
 					if (DecimalBPP < 6)
 						return BPP_INVALID;
 					else if (DecimalBPP >= 1.5 * DSCInputBitPerComponent - 1.0 / 16.0)
-						return 1.5 * DSCInputBitPerComponent - 1.0 / 16.0;
+						return (unsigned int)(1.5 * DSCInputBitPerComponent - 1.0 / 16.0);
 					else
-						return dml_floor(16 * DecimalBPP, 1) / 16.0;
+						return (unsigned int)(dml_floor(16 * DecimalBPP, 1) / 16.0);
 				} else {
 					if (DecimalBPP < 6
 							|| DesiredBPP < 6
@@ -3319,7 +3317,7 @@ static unsigned int TruncToValidBPP(
 							|| DecimalBPP < DesiredBPP) {
 						return BPP_INVALID;
 					} else {
-						return DesiredBPP;
+						return (unsigned int)DesiredBPP;
 					}
 				}
 			} else if (Format == dm_n422) {
@@ -3327,9 +3325,9 @@ static unsigned int TruncToValidBPP(
 					if (DecimalBPP < 7)
 						return BPP_INVALID;
 					else if (DecimalBPP >= 2 * DSCInputBitPerComponent - 1.0 / 16.0)
-						return 2 * DSCInputBitPerComponent - 1.0 / 16.0;
+						return (unsigned int)(2 * DSCInputBitPerComponent - 1.0 / 16.0);
 					else
-						return dml_floor(16 * DecimalBPP, 1) / 16.0;
+						return (unsigned int)(dml_floor(16 * DecimalBPP, 1) / 16.0);
 				} else {
 					if (DecimalBPP < 7
 							|| DesiredBPP < 7
@@ -3337,7 +3335,7 @@ static unsigned int TruncToValidBPP(
 							|| DecimalBPP < DesiredBPP) {
 						return BPP_INVALID;
 					} else {
-						return DesiredBPP;
+						return (unsigned int)DesiredBPP;
 					}
 				}
 			} else {
@@ -3345,9 +3343,9 @@ static unsigned int TruncToValidBPP(
 					if (DecimalBPP < 8)
 						return BPP_INVALID;
 					else if (DecimalBPP >= 3 * DSCInputBitPerComponent - 1.0 / 16.0)
-						return 3 * DSCInputBitPerComponent - 1.0 / 16.0;
+						return (unsigned int)(3 * DSCInputBitPerComponent - 1.0 / 16.0);
 					else
-						return dml_floor(16 * DecimalBPP, 1) / 16.0;
+						return (unsigned int)(dml_floor(16 * DecimalBPP, 1) / 16.0);
 				} else {
 					if (DecimalBPP < 8
 							|| DesiredBPP < 8
@@ -3355,7 +3353,7 @@ static unsigned int TruncToValidBPP(
 							|| DecimalBPP < DesiredBPP) {
 						return BPP_INVALID;
 					} else {
-						return DesiredBPP;
+						return (unsigned int)DesiredBPP;
 					}
 				}
 			}
@@ -3396,8 +3394,8 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 {
 	struct vba_vars_st *locals = &mode_lib->vba;
 
-	int i;
-	unsigned int j, k, m;
+	int idx;
+	unsigned int i, j, k, m;
 
 	/*MODE SUPPORT, VOLTAGE STATE AND SOC CONFIGURATION*/
 
@@ -3860,8 +3858,8 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 		Calculate256BBlockSizes(
 				mode_lib->vba.SourcePixelFormat[k],
 				mode_lib->vba.SurfaceTiling[k],
-				dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
-				dml_ceil(locals->BytePerPixelInDETC[k], 2.0),
+				(unsigned int)dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
+				(unsigned int)dml_ceil(locals->BytePerPixelInDETC[k], 2.0),
 				&locals->Read256BlockHeightY[k],
 				&locals->Read256BlockHeightC[k],
 				&locals->Read256BlockWidthY[k],
@@ -4045,7 +4043,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 					locals->DISPCLK_DPPCLK_Support[i][j] = false;
 				}
 			}
-			locals->TotalNumberOfActiveDPP[i][j] = 0.0;
+			locals->TotalNumberOfActiveDPP[i][j] = (unsigned int)0.0;
 			for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++)
 				locals->TotalNumberOfActiveDPP[i][j] = locals->TotalNumberOfActiveDPP[i][j] + locals->NoOfDPP[i][j][k];
 			if (j == 1) {
@@ -4100,7 +4098,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 							|| mode_lib->vba.PlaneRequiredDISPCLK > mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity)
 						locals->DISPCLK_DPPCLK_Support[i][j] = false;
 				}
-				locals->TotalNumberOfActiveDPP[i][j] = 0.0;
+				locals->TotalNumberOfActiveDPP[i][j] = (unsigned int)0.0;
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++)
 					locals->TotalNumberOfActiveDPP[i][j] = locals->TotalNumberOfActiveDPP[i][j] + locals->NoOfDPP[i][j][k];
 			}
@@ -4361,17 +4359,17 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 					|| locals->RequiresDSC[i][k] == false) {
 				mode_lib->vba.slices = 0;
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 3200.0) {
-				mode_lib->vba.slices = dml_ceil(
+				mode_lib->vba.slices = (unsigned int)dml_ceil(
 						mode_lib->vba.PixelClockBackEnd[k] / 400.0,
 						4.0);
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 1360.0) {
-				mode_lib->vba.slices = 8.0;
+				mode_lib->vba.slices = (unsigned int)8.0;
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 680.0) {
-				mode_lib->vba.slices = 4.0;
+				mode_lib->vba.slices = (unsigned int)4.0;
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 340.0) {
-				mode_lib->vba.slices = 2.0;
+				mode_lib->vba.slices = (unsigned int)2.0;
 			} else {
-				mode_lib->vba.slices = 1.0;
+				mode_lib->vba.slices = (unsigned int)1.0;
 			}
 			if (locals->OutputBppPerState[i][k] == BPP_BLENDED_PIPE
 					|| locals->OutputBppPerState[i][k] == BPP_INVALID) {
@@ -4385,7 +4383,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 							dscceComputeDelay(
 									mode_lib->vba.DSCInputBitPerComponent[k],
 									mode_lib->vba.bpp,
-									dml_ceil(
+									(unsigned int)dml_ceil(
 											mode_lib->vba.HActive[k]
 													/ mode_lib->vba.slices,
 											1.0),
@@ -4398,7 +4396,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 							2.0 * (dscceComputeDelay(
 											mode_lib->vba.DSCInputBitPerComponent[k],
 											mode_lib->vba.bpp,
-											dml_ceil(mode_lib->vba.HActive[k] / mode_lib->vba.slices, 1.0),
+											(unsigned int)dml_ceil(mode_lib->vba.HActive[k] / mode_lib->vba.slices, 1.0),
 											mode_lib->vba.slices / 2,
 											mode_lib->vba.OutputFormat[k])
 									+ dscComputeDelay(mode_lib->vba.OutputFormat[k]));
@@ -4424,9 +4422,9 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 		for (j = 0; j < 2; j++) {
 			for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
 				if (locals->ODMCombineEnablePerState[i][k] == dm_odm_combine_mode_2to1)
-					locals->SwathWidthYPerState[i][j][k] = dml_min(locals->SwathWidthYSingleDPP[k], dml_round(locals->HActive[k] / 2 * locals->HRatio[k]));
+					locals->SwathWidthYPerState[i][j][k] = (unsigned int)dml_min(locals->SwathWidthYSingleDPP[k], dml_round(locals->HActive[k] / 2 * locals->HRatio[k]));
 				else
-					locals->SwathWidthYPerState[i][j][k] = locals->SwathWidthYSingleDPP[k] / locals->NoOfDPP[i][j][k];
+					locals->SwathWidthYPerState[i][j][k] = (unsigned int)locals->SwathWidthYSingleDPP[k] / locals->NoOfDPP[i][j][k];
 				locals->SwathWidthGranularityY = 256  / dml_ceil(locals->BytePerPixelInDETY[k], 1) / locals->MaxSwathHeightY[k];
 				locals->RoundedUpMaxSwathSizeBytesY = (dml_ceil(locals->SwathWidthYPerState[i][j][k] - 1, locals->SwathWidthGranularityY)
 						+ locals->SwathWidthGranularityY) * locals->BytePerPixelInDETY[k] * locals->MaxSwathHeightY[k];
@@ -4446,11 +4444,11 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 				}
 
 				if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte[0] * 1024.0 / 2) {
-					locals->SwathHeightYPerState[i][j][k] = locals->MaxSwathHeightY[k];
-					locals->SwathHeightCPerState[i][j][k] = locals->MaxSwathHeightC[k];
+					locals->SwathHeightYPerState[i][j][k] = (unsigned int)locals->MaxSwathHeightY[k];
+					locals->SwathHeightCPerState[i][j][k] = (unsigned int)locals->MaxSwathHeightC[k];
 				} else {
-					locals->SwathHeightYPerState[i][j][k] = locals->MinSwathHeightY[k];
-					locals->SwathHeightCPerState[i][j][k] = locals->MinSwathHeightC[k];
+					locals->SwathHeightYPerState[i][j][k] = (unsigned int)locals->MinSwathHeightY[k];
+					locals->SwathHeightCPerState[i][j][k] = (unsigned int)locals->MinSwathHeightC[k];
 				}
 
 				if (locals->BytePerPixelInDETC[k] == 0) {
@@ -4465,16 +4463,16 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 					locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
 				}
 
-				locals->EffectiveLBLatencyHidingSourceLinesLuma = dml_min(locals->MaxLineBufferLines,
+				locals->EffectiveLBLatencyHidingSourceLinesLuma = (unsigned int)dml_min(locals->MaxLineBufferLines,
 					dml_floor(locals->LineBufferSize / locals->LBBitPerPixel[k] / (locals->SwathWidthYPerState[i][j][k]
 					/ dml_max(locals->HRatio[k], 1)), 1)) - (locals->vtaps[k] - 1);
 
-				locals->EffectiveLBLatencyHidingSourceLinesChroma =  dml_min(locals->MaxLineBufferLines,
+				locals->EffectiveLBLatencyHidingSourceLinesChroma =  (unsigned int)dml_min(locals->MaxLineBufferLines,
 						dml_floor(locals->LineBufferSize / locals->LBBitPerPixel[k]
 						/ (locals->SwathWidthYPerState[i][j][k] / 2
 						/ dml_max(locals->HRatio[k] / 2, 1)), 1)) - (locals->VTAPsChroma[k] - 1);
 
-				locals->EffectiveDETLBLinesLuma = dml_floor(locals->LinesInDETLuma +  dml_min(
+				locals->EffectiveDETLBLinesLuma = (unsigned int)dml_floor(locals->LinesInDETLuma +  dml_min(
 						locals->LinesInDETLuma * locals->RequiredDISPCLK[i][j] * locals->BytePerPixelInDETY[k] *
 						locals->PSCL_FACTOR[k] / locals->ReturnBWPerState[i][0],
 						locals->EffectiveLBLatencyHidingSourceLinesLuma),
@@ -4630,7 +4628,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 						mode_lib->vba.Read256BlockWidthY[k],
 						mode_lib->vba.SourcePixelFormat[k],
 						mode_lib->vba.SurfaceTiling[k],
-						dml_ceil(mode_lib->vba.BytePerPixelInDETY[k], 1.0),
+						(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelInDETY[k], 1.0),
 						mode_lib->vba.SourceScan[k],
 						mode_lib->vba.ViewportWidth[k],
 						mode_lib->vba.ViewportHeight[k],
@@ -4669,17 +4667,17 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 							mode_lib->vba.Read256BlockWidthY[k],
 							mode_lib->vba.SourcePixelFormat[k],
 							mode_lib->vba.SurfaceTiling[k],
-							dml_ceil(mode_lib->vba.BytePerPixelInDETC[k], 2.0),
+							(unsigned int)dml_ceil(mode_lib->vba.BytePerPixelInDETC[k], 2.0),
 							mode_lib->vba.SourceScan[k],
-							mode_lib->vba.ViewportWidth[k] / 2.0,
-							mode_lib->vba.ViewportHeight[k] / 2.0,
-							mode_lib->vba.SwathWidthYPerState[i][j][k] / 2.0,
+							(unsigned int)(mode_lib->vba.ViewportWidth[k] / 2.0),
+							(unsigned int)(mode_lib->vba.ViewportHeight[k] / 2.0),
+							(unsigned int)(mode_lib->vba.SwathWidthYPerState[i][j][k] / 2.0),
 							mode_lib->vba.GPUVMEnable,
 							mode_lib->vba.VMMPageSize,
 							mode_lib->vba.PTEBufferSizeInRequestsLuma,
 							mode_lib->vba.PDEProcessingBufIn64KBReqs,
 							mode_lib->vba.PitchC[k],
-							0.0,
+							(unsigned int)0.0,
 							&mode_lib->vba.MacroTileWidthC[k],
 							&mode_lib->vba.MetaRowBytesC,
 							&mode_lib->vba.DPTEBytesPerRowC,
@@ -4698,8 +4696,8 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 							&mode_lib->vba.MaxNumSwC[k]);
 				} else {
 					mode_lib->vba.PDEAndMetaPTEBytesPerFrameC = 0.0;
-					mode_lib->vba.MetaRowBytesC = 0.0;
-					mode_lib->vba.DPTEBytesPerRowC = 0.0;
+					mode_lib->vba.MetaRowBytesC = (unsigned int)0.0;
+					mode_lib->vba.DPTEBytesPerRowC = (unsigned int)0.0;
 					locals->PrefetchLinesC[0][0][k] = 0.0;
 					locals->PTEBufferSizeNotExceededC[i][j][k] = true;
 					locals->PTEBufferSizeInRequestsForLuma = mode_lib->vba.PTEBufferSizeInRequestsLuma + mode_lib->vba.PTEBufferSizeInRequestsChroma;
@@ -4755,7 +4753,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 										mode_lib->vba.WritebackLumaVTaps[k],
 										mode_lib->vba.WritebackChromaHTaps[k],
 										mode_lib->vba.WritebackChromaVTaps[k],
-										mode_lib->vba.WritebackDestinationWidth[k]) / locals->RequiredDISPCLK[i][j];
+										(unsigned int)mode_lib->vba.WritebackDestinationWidth[k]) / locals->RequiredDISPCLK[i][j];
 					} else {
 						locals->WritebackDelay[i][k] = 0.0;
 					}
@@ -4772,7 +4770,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 													mode_lib->vba.WritebackLumaVTaps[m],
 													mode_lib->vba.WritebackChromaHTaps[m],
 													mode_lib->vba.WritebackChromaVTaps[m],
-													mode_lib->vba.WritebackDestinationWidth[m]) / locals->RequiredDISPCLK[i][j]);
+													(unsigned int)mode_lib->vba.WritebackDestinationWidth[m]) / locals->RequiredDISPCLK[i][j]);
 						}
 					}
 				}
@@ -4833,10 +4831,10 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 
 					CalculateDelayAfterScaler(mode_lib, mode_lib->vba.ReturnBWPerState[i][0], mode_lib->vba.ReadBandwidthLuma[k], mode_lib->vba.ReadBandwidthChroma[k], mode_lib->vba.MaxTotalVActiveRDBandwidth,
 						mode_lib->vba.DisplayPipeLineDeliveryTimeLuma[k], mode_lib->vba.DisplayPipeLineDeliveryTimeChroma[k],
-						mode_lib->vba.RequiredDPPCLK[i][j][k], mode_lib->vba.RequiredDISPCLK[i][j], mode_lib->vba.PixelClock[k], mode_lib->vba.DSCDelayPerState[i][k], mode_lib->vba.NoOfDPP[i][j][k], mode_lib->vba.ScalerEnabled[k], mode_lib->vba.NumberOfCursors[k],
+						(unsigned int)mode_lib->vba.RequiredDPPCLK[i][j][k], (unsigned int)mode_lib->vba.RequiredDISPCLK[i][j], mode_lib->vba.PixelClock[k], (unsigned int)mode_lib->vba.DSCDelayPerState[i][k], mode_lib->vba.NoOfDPP[i][j][k], mode_lib->vba.ScalerEnabled[k], mode_lib->vba.NumberOfCursors[k],
 						mode_lib->vba.DPPCLKDelaySubtotal, mode_lib->vba.DPPCLKDelaySCL, mode_lib->vba.DPPCLKDelaySCLLBOnly, mode_lib->vba.DPPCLKDelayCNVCFormater, mode_lib->vba.DPPCLKDelayCNVCCursor, mode_lib->vba.DISPCLKDelaySubtotal,
-						mode_lib->vba.SwathWidthYPerState[i][j][k] / mode_lib->vba.HRatio[k], mode_lib->vba.OutputFormat[k], mode_lib->vba.HTotal[k],
-						mode_lib->vba.SwathWidthYSingleDPP[k], mode_lib->vba.BytePerPixelInDETY[k], mode_lib->vba.BytePerPixelInDETC[k], mode_lib->vba.SwathHeightYThisState[k], mode_lib->vba.SwathHeightCThisState[k], mode_lib->vba.Interlace[k], mode_lib->vba.ProgressiveToInterlaceUnitInOPP,
+						(unsigned int)(mode_lib->vba.SwathWidthYPerState[i][j][k] / mode_lib->vba.HRatio[k]), mode_lib->vba.OutputFormat[k], mode_lib->vba.HTotal[k],
+						(unsigned int)mode_lib->vba.SwathWidthYSingleDPP[k], mode_lib->vba.BytePerPixelInDETY[k], mode_lib->vba.BytePerPixelInDETC[k], mode_lib->vba.SwathHeightYThisState[k], mode_lib->vba.SwathHeightCThisState[k], mode_lib->vba.Interlace[k], mode_lib->vba.ProgressiveToInterlaceUnitInOPP,
 						&mode_lib->vba.DSTXAfterScaler[k], &mode_lib->vba.DSTYAfterScaler[k]);
 
 					mode_lib->vba.IsErrorResult[i][j][k] =
@@ -4852,7 +4850,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 											- mode_lib->vba.VActive[k],
 									mode_lib->vba.HTotal[k],
 									mode_lib->vba.MaxInterDCNTileRepeaters,
-									mode_lib->vba.MaximumVStartup[0][0][k],
+									(unsigned int)mode_lib->vba.MaximumVStartup[0][0][k],
 									mode_lib->vba.GPUVMMaxPageTableLevels,
 									mode_lib->vba.GPUVMEnable,
 									mode_lib->vba.DynamicMetadataEnable[k],
@@ -4862,9 +4860,9 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 									mode_lib->vba.UrgentLatencyPixelDataOnly,
 									mode_lib->vba.ExtraLatency,
 									mode_lib->vba.TimeCalc,
-									mode_lib->vba.PDEAndMetaPTEBytesPerFrame[0][0][k],
-									mode_lib->vba.MetaRowBytes[0][0][k],
-									mode_lib->vba.DPTEBytesPerRow[0][0][k],
+									(unsigned int)mode_lib->vba.PDEAndMetaPTEBytesPerFrame[0][0][k],
+									(unsigned int)mode_lib->vba.MetaRowBytes[0][0][k],
+									(unsigned int)mode_lib->vba.DPTEBytesPerRow[0][0][k],
 									mode_lib->vba.PrefetchLinesY[0][0][k],
 									mode_lib->vba.SwathWidthYPerState[i][j][k],
 									mode_lib->vba.BytePerPixelInDETY[k],
@@ -4972,16 +4970,16 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 											mode_lib->vba.PrefetchBW[k]);
 				}
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
-					mode_lib->vba.ImmediateFlipBytes[k] = 0.0;
+					mode_lib->vba.ImmediateFlipBytes[k] = (unsigned int)0.0;
 					if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
 							&& mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
-						mode_lib->vba.ImmediateFlipBytes[k] =
+						mode_lib->vba.ImmediateFlipBytes[k] = (unsigned int)(
 								mode_lib->vba.PDEAndMetaPTEBytesPerFrame[0][0][k]
 										+ mode_lib->vba.MetaRowBytes[0][0][k]
-										+ mode_lib->vba.DPTEBytesPerRow[0][0][k];
+										+ mode_lib->vba.DPTEBytesPerRow[0][0][k]);
 					}
 				}
-				mode_lib->vba.TotImmediateFlipBytes = 0.0;
+				   mode_lib->vba.TotImmediateFlipBytes = 0;
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
 					if ((mode_lib->vba.SourcePixelFormat[k] != dm_420_8
 							&& mode_lib->vba.SourcePixelFormat[k] != dm_420_10)) {
@@ -5007,8 +5005,8 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 							mode_lib->vba.VRatio[k],
 							mode_lib->vba.Tno_bw[k],
 							mode_lib->vba.PDEAndMetaPTEBytesPerFrame[0][0][k],
-							mode_lib->vba.MetaRowBytes[0][0][k],
-							mode_lib->vba.DPTEBytesPerRow[0][0][k],
+							(unsigned int)mode_lib->vba.MetaRowBytes[0][0][k],
+							(unsigned int)mode_lib->vba.DPTEBytesPerRow[0][0][k],
 							mode_lib->vba.DCCEnable[k],
 							mode_lib->vba.dpte_row_height[k],
 							mode_lib->vba.meta_row_height[k],
@@ -5135,7 +5133,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 	}
 	/*Mode Support, Voltage State and SOC Configuration*/
 
-	for (i = mode_lib->vba.soc.num_states; i >= 0; i--) {
+	for (idx = mode_lib->vba.soc.num_states; idx >= 0; idx--) {
 		for (j = 0; j < 2; j++) {
 			enum dm_validation_status status = DML_VALIDATION_OK;
 
@@ -5143,21 +5141,21 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 				status = DML_FAIL_SCALE_RATIO_TAP;
 			} else if (mode_lib->vba.SourceFormatPixelAndScanSupport != true) {
 				status = DML_FAIL_SOURCE_PIXEL_FORMAT;
-			} else if (locals->ViewportSizeSupport[i][0] != true) {
+			} else if (locals->ViewportSizeSupport[idx][0] != true) {
 				status = DML_FAIL_VIEWPORT_SIZE;
-			} else if (locals->DIOSupport[i] != true) {
+			} else if (locals->DIOSupport[idx] != true) {
 				status = DML_FAIL_DIO_SUPPORT;
-			} else if (locals->NotEnoughDSCUnits[i] != false) {
+			} else if (locals->NotEnoughDSCUnits[idx] != false) {
 				status = DML_FAIL_NOT_ENOUGH_DSC;
-			} else if (locals->DSCCLKRequiredMoreThanSupported[i] != false) {
+			} else if (locals->DSCCLKRequiredMoreThanSupported[idx] != false) {
 				status = DML_FAIL_DSC_CLK_REQUIRED;
-			} else if (locals->UrgentLatencySupport[i][j] != true) {
+			} else if (locals->UrgentLatencySupport[idx][j] != true) {
 				status = DML_FAIL_URGENT_LATENCY;
-			} else if (locals->ROBSupport[i][0] != true) {
+			} else if (locals->ROBSupport[idx][0] != true) {
 				status = DML_FAIL_REORDERING_BUFFER;
-			} else if (locals->DISPCLK_DPPCLK_Support[i][j] != true) {
+			} else if (locals->DISPCLK_DPPCLK_Support[idx][j] != true) {
 				status = DML_FAIL_DISPCLK_DPPCLK;
-			} else if (locals->TotalAvailablePipesSupport[i][j] != true) {
+			} else if (locals->TotalAvailablePipesSupport[idx][j] != true) {
 				status = DML_FAIL_TOTAL_AVAILABLE_PIPES;
 			} else if (mode_lib->vba.NumberOfOTGSupport != true) {
 				status = DML_FAIL_NUM_OTG;
@@ -5171,24 +5169,24 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode
 				status = DML_FAIL_CURSOR_SUPPORT;
 			} else if (mode_lib->vba.PitchSupport != true) {
 				status = DML_FAIL_PITCH_SUPPORT;
-			} else if (locals->PrefetchSupported[i][j] != true) {
+			} else if (locals->PrefetchSupported[idx][j] != true) {
 				status = DML_FAIL_PREFETCH_SUPPORT;
-			} else if (locals->TotalVerticalActiveBandwidthSupport[i][0] != true) {
+			} else if (locals->TotalVerticalActiveBandwidthSupport[idx][0] != true) {
 				status = DML_FAIL_TOTAL_V_ACTIVE_BW;
-			} else if (locals->VRatioInPrefetchSupported[i][j] != true) {
+			} else if (locals->VRatioInPrefetchSupported[idx][j] != true) {
 				status = DML_FAIL_V_RATIO_PREFETCH;
-			} else if (locals->PTEBufferSizeNotExceeded[i][j] != true) {
+			} else if (locals->PTEBufferSizeNotExceeded[idx][j] != true) {
 				status = DML_FAIL_PTE_BUFFER_SIZE;
 			} else if (mode_lib->vba.NonsupportedDSCInputBPC != false) {
 				status = DML_FAIL_DSC_INPUT_BPC;
 			}
 
 			if (status == DML_VALIDATION_OK) {
-				locals->ModeSupport[i][j] = true;
+				locals->ModeSupport[idx][j] = true;
 			} else {
-				locals->ModeSupport[i][j] = false;
+				locals->ModeSupport[idx][j] = false;
 			}
-			locals->ValidationStatus[i] = status;
+			locals->ValidationStatus[idx] = status;
 		}
 	}
 	{
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c
index 591d9618bdc0b..59edfcf27d91d 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20.c
@@ -196,13 +196,13 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib,
 
 	extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_l), &rq_param->sizing.rq_l);
 
-	rq_regs->rq_regs_l.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height),
-			1) - 3;
+	rq_regs->rq_regs_l.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height),
+			1) - 3);
 
 	if (rq_param->yuv420) {
 		extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_c), &rq_param->sizing.rq_c);
-		rq_regs->rq_regs_c.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height),
-				1) - 3;
+		rq_regs->rq_regs_c.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height),
+				1) - 3);
 	}
 
 	rq_regs->rq_regs_l.swath_height = dml_log2(rq_param->dlg.rq_l.swath_height);
@@ -222,11 +222,11 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib,
 	if (rq_param->yuv420) {
 		if ((double) rq_param->misc.rq_l.stored_swath_bytes
 				/ (double) rq_param->misc.rq_c.stored_swath_bytes <= 1.5) {
-			detile_buf_plane1_addr = (detile_buf_size_in_bytes / 2.0 / 64.0); // half to chroma
+			detile_buf_plane1_addr = (unsigned int)(detile_buf_size_in_bytes / 2.0 / 64.0); // half to chroma
 		} else {
-			detile_buf_plane1_addr = dml_round_to_multiple((unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0),
-					256,
-					0) / 64.0; // 2/3 to chroma
+			detile_buf_plane1_addr = (unsigned int)(dml_round_to_multiple((unsigned int)((2.0 * detile_buf_size_in_bytes) / 3.0),
+					  256,
+					  0) / 64.0); // 2/3 to chroma
 		}
 	}
 	rq_regs->plane1_base_address = detile_buf_plane1_addr;
@@ -451,8 +451,8 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
 	// "/2" is like square root
 	// blk is vertical biased
 	if (tiling != dm_sw_linear)
-		log2_blk_height = log2_blk256_height
-				+ dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
+		log2_blk_height = (unsigned int)(log2_blk256_height
+				+ dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1));
 	else
 		log2_blk_height = 0;  // blk height of 1
 
@@ -601,7 +601,7 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
 	if (surf_linear) {
 		unsigned int dpte_row_height;
 
-		log2_dpte_row_height_linear = dml_floor(dml_log2(dml_min(64 * 1024 * pde_buf_entries
+		log2_dpte_row_height_linear = (unsigned int)dml_floor(dml_log2(dml_min(64 * 1024 * pde_buf_entries
 										/ bytes_per_element,
 								dpte_buf_in_pte_reqs
 										* dpte_req_width)
@@ -671,7 +671,7 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
 
 	// since dpte groups are only aligned to dpte_req_width and not dpte_group_width,
 	// the upper bound for the dpte groups per row is as follows.
-	rq_dlg_param->dpte_groups_per_row_ub = dml_ceil((double) dpte_row_width_ub / dpte_group_width,
+	rq_dlg_param->dpte_groups_per_row_ub = (unsigned int)dml_ceil((double) dpte_row_width_ub / dpte_group_width,
 			1);
 }
 
@@ -1018,11 +1018,11 @@ static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 	if (dout->dsc_enable) {
 		double dsc_delay = get_dsc_delay(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
 
-		dispclk_delay_subtotal += dsc_delay;
+		dispclk_delay_subtotal += (unsigned int)dsc_delay;
 	}
 
-	pixel_rate_delay_subtotal = dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
-			+ dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz;
+	pixel_rate_delay_subtotal = (unsigned int)(dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
+			+ dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz);
 
 	vstartup_start = dst->vstartup_start;
 	if (interlaced) {
@@ -1043,7 +1043,7 @@ static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 
 	// TODO: Where is this coming from?
 	if (interlaced)
-		vstartup_start = vstartup_start / 2;
+		vstartup_start = (unsigned int)(vstartup_start / 2);
 
 	// TODO: What if this min_vblank doesn't match the value in the dml_config_settings.cpp?
 	if (vstartup_start >= min_vblank) {
@@ -1062,8 +1062,8 @@ static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 				min_vblank);
 	}
 
-	dst_x_after_scaler = get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
-	dst_y_after_scaler = get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+	dst_x_after_scaler = (unsigned int)get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+	dst_y_after_scaler = (unsigned int)get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
 
 	dml_print("DML_DLG: %s: htotal                                 = %d\n", __func__, htotal);
 	dml_print("DML_DLG: %s: pixel_rate_delay_subtotal              = %d\n",
@@ -1376,57 +1376,57 @@ static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 
 	// Assignment to register structures
 	disp_dlg_regs->dst_y_after_scaler = dst_y_after_scaler; // in terms of line
-	disp_dlg_regs->refcyc_x_after_scaler = dst_x_after_scaler * ref_freq_to_pix_freq; // in terms of refclk
+	disp_dlg_regs->refcyc_x_after_scaler = (unsigned int)(dst_x_after_scaler * ref_freq_to_pix_freq); // in terms of refclk
 	ASSERT(disp_dlg_regs->refcyc_x_after_scaler < (unsigned int) dml_pow(2, 13));
-	disp_dlg_regs->dst_y_prefetch = (unsigned int) (dst_y_prefetch * dml_pow(2, 2));
-	disp_dlg_regs->dst_y_per_vm_vblank = (unsigned int) (dst_y_per_vm_vblank * dml_pow(2, 2));
-	disp_dlg_regs->dst_y_per_row_vblank = (unsigned int) (dst_y_per_row_vblank * dml_pow(2, 2));
-	disp_dlg_regs->dst_y_per_vm_flip = (unsigned int) (dst_y_per_vm_flip * dml_pow(2, 2));
-	disp_dlg_regs->dst_y_per_row_flip = (unsigned int) (dst_y_per_row_flip * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_prefetch = (unsigned int)(dst_y_prefetch * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_vm_vblank = (unsigned int)(dst_y_per_vm_vblank * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_row_vblank = (unsigned int)(dst_y_per_row_vblank * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_vm_flip = (unsigned int)(dst_y_per_vm_flip * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_row_flip = (unsigned int)(dst_y_per_row_flip * dml_pow(2, 2));
 
-	disp_dlg_regs->vratio_prefetch = (unsigned int) (vratio_pre_l * dml_pow(2, 19));
-	disp_dlg_regs->vratio_prefetch_c = (unsigned int) (vratio_pre_c * dml_pow(2, 19));
+	disp_dlg_regs->vratio_prefetch = (unsigned int)(vratio_pre_l * dml_pow(2, 19));
+	disp_dlg_regs->vratio_prefetch_c = (unsigned int)(vratio_pre_c * dml_pow(2, 19));
 
 	disp_dlg_regs->refcyc_per_pte_group_vblank_l =
-			(unsigned int) (dst_y_per_row_vblank * (double) htotal
-					* ref_freq_to_pix_freq / (double) dpte_groups_per_row_ub_l);
+			  (unsigned int)(dst_y_per_row_vblank * (double) htotal
+					    * ref_freq_to_pix_freq / (double) dpte_groups_per_row_ub_l);
 	ASSERT(disp_dlg_regs->refcyc_per_pte_group_vblank_l < (unsigned int) dml_pow(2, 13));
 
 	if (dual_plane) {
-		disp_dlg_regs->refcyc_per_pte_group_vblank_c = (unsigned int) (dst_y_per_row_vblank
-				* (double) htotal * ref_freq_to_pix_freq
-				/ (double) dpte_groups_per_row_ub_c);
+			   disp_dlg_regs->refcyc_per_pte_group_vblank_c = (unsigned int)(dst_y_per_row_vblank
+							   * (double) htotal * ref_freq_to_pix_freq
+							   / (double) dpte_groups_per_row_ub_c);
 		ASSERT(disp_dlg_regs->refcyc_per_pte_group_vblank_c
 						< (unsigned int) dml_pow(2, 13));
 	}
 
 	disp_dlg_regs->refcyc_per_meta_chunk_vblank_l =
-			(unsigned int) (dst_y_per_row_vblank * (double) htotal
-					* ref_freq_to_pix_freq / (double) meta_chunks_per_row_ub_l);
+			  (unsigned int)(dst_y_per_row_vblank * (double) htotal
+					    * ref_freq_to_pix_freq / (double) meta_chunks_per_row_ub_l);
 	ASSERT(disp_dlg_regs->refcyc_per_meta_chunk_vblank_l < (unsigned int) dml_pow(2, 13));
 
 	disp_dlg_regs->refcyc_per_meta_chunk_vblank_c =
 			disp_dlg_regs->refcyc_per_meta_chunk_vblank_l; // dcc for 4:2:0 is not supported in dcn1.0.  assigned to be the same as _l for now
 
-	disp_dlg_regs->refcyc_per_pte_group_flip_l = (unsigned int) (dst_y_per_row_flip * htotal
-			* ref_freq_to_pix_freq) / dpte_groups_per_row_ub_l;
-	disp_dlg_regs->refcyc_per_meta_chunk_flip_l = (unsigned int) (dst_y_per_row_flip * htotal
-			* ref_freq_to_pix_freq) / meta_chunks_per_row_ub_l;
+	disp_dlg_regs->refcyc_per_pte_group_flip_l = (unsigned int)((dst_y_per_row_flip * htotal
+			  * ref_freq_to_pix_freq) / dpte_groups_per_row_ub_l);
+	disp_dlg_regs->refcyc_per_meta_chunk_flip_l = (unsigned int)((dst_y_per_row_flip * htotal
+			  * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_l);
 
 	if (dual_plane) {
-		disp_dlg_regs->refcyc_per_pte_group_flip_c = (unsigned int) (dst_y_per_row_flip
-				* htotal * ref_freq_to_pix_freq) / dpte_groups_per_row_ub_c;
-		disp_dlg_regs->refcyc_per_meta_chunk_flip_c = (unsigned int) (dst_y_per_row_flip
-				* htotal * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_c;
+			   disp_dlg_regs->refcyc_per_pte_group_flip_c = (unsigned int)((dst_y_per_row_flip
+							   * htotal * ref_freq_to_pix_freq) / dpte_groups_per_row_ub_c);
+			   disp_dlg_regs->refcyc_per_meta_chunk_flip_c = (unsigned int)((dst_y_per_row_flip
+							   * htotal * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_c);
 	}
 
-	disp_dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int) ((double) dpte_row_height_l
-			/ (double) vratio_l * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int)(((double) dpte_row_height_l
+			  / (double) vratio_l * dml_pow(2, 2)));
 	ASSERT(disp_dlg_regs->dst_y_per_pte_row_nom_l < (unsigned int) dml_pow(2, 17));
 
 	if (dual_plane) {
-		disp_dlg_regs->dst_y_per_pte_row_nom_c = (unsigned int) ((double) dpte_row_height_c
-				/ (double) vratio_c * dml_pow(2, 2));
+			   disp_dlg_regs->dst_y_per_pte_row_nom_c = (unsigned int)(((double) dpte_row_height_c
+							   / (double) vratio_c * dml_pow(2, 2)));
 		if (disp_dlg_regs->dst_y_per_pte_row_nom_c >= (unsigned int) dml_pow(2, 17)) {
 			dml_print("DML_DLG: %s: Warning dst_y_per_pte_row_nom_c %u larger than supported by register format U15.2 %u\n",
 					__func__,
@@ -1435,38 +1435,38 @@ static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 		}
 	}
 
-	disp_dlg_regs->dst_y_per_meta_row_nom_l = (unsigned int) ((double) meta_row_height_l
-			/ (double) vratio_l * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_meta_row_nom_l = (unsigned int)(((double) meta_row_height_l
+			  / (double) vratio_l * dml_pow(2, 2)));
 	ASSERT(disp_dlg_regs->dst_y_per_meta_row_nom_l < (unsigned int) dml_pow(2, 17));
 
 	disp_dlg_regs->dst_y_per_meta_row_nom_c = disp_dlg_regs->dst_y_per_meta_row_nom_l; // TODO: dcc for 4:2:0 is not supported in dcn1.0.  assigned to be the same as _l for now
 
-	disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int) ((double) dpte_row_height_l
-			/ (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
-			/ (double) dpte_groups_per_row_ub_l);
+	disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)(((double) dpte_row_height_l
+			  / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
+			  / (double) dpte_groups_per_row_ub_l));
 	if (disp_dlg_regs->refcyc_per_pte_group_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_pte_group_nom_l = dml_pow(2, 23) - 1;
-	disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int) ((double) meta_row_height_l
-			/ (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
-			/ (double) meta_chunks_per_row_ub_l);
+		disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
+	disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)(((double) meta_row_height_l
+			  / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
+			  / (double) meta_chunks_per_row_ub_l));
 	if (disp_dlg_regs->refcyc_per_meta_chunk_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (dual_plane) {
-		disp_dlg_regs->refcyc_per_pte_group_nom_c =
-				(unsigned int) ((double) dpte_row_height_c / (double) vratio_c
-						* (double) htotal * ref_freq_to_pix_freq
-						/ (double) dpte_groups_per_row_ub_c);
+			   disp_dlg_regs->refcyc_per_pte_group_nom_c =
+							   (unsigned int)(((double) dpte_row_height_c / (double) vratio_c
+											   * (double) htotal * ref_freq_to_pix_freq
+											   / (double) dpte_groups_per_row_ub_c));
 		if (disp_dlg_regs->refcyc_per_pte_group_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_pte_group_nom_c = dml_pow(2, 23) - 1;
+				disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 
 		// TODO: Is this the right calculation? Does htotal need to be halved?
-		disp_dlg_regs->refcyc_per_meta_chunk_nom_c =
-				(unsigned int) ((double) meta_row_height_c / (double) vratio_c
-						* (double) htotal * ref_freq_to_pix_freq
-						/ (double) meta_chunks_per_row_ub_c);
+			   disp_dlg_regs->refcyc_per_meta_chunk_nom_c =
+							   (unsigned int)(((double) meta_row_height_c / (double) vratio_c
+											   * (double) htotal * ref_freq_to_pix_freq
+											   / (double) meta_chunks_per_row_ub_c));
 		if (disp_dlg_regs->refcyc_per_meta_chunk_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = dml_pow(2, 23) - 1;
+				disp_dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 	}
 
 	disp_dlg_regs->refcyc_per_line_delivery_pre_l = (unsigned int) dml_floor(refcyc_per_line_delivery_pre_l,
@@ -1490,22 +1490,22 @@ static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 
 	disp_dlg_regs->dst_y_delta_drq_limit = 0x7fff; // off
 
-	disp_ttu_regs->refcyc_per_req_delivery_pre_l = (unsigned int) (refcyc_per_req_delivery_pre_l
-			* dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_l = (unsigned int) (refcyc_per_req_delivery_l
-			* dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_pre_c = (unsigned int) (refcyc_per_req_delivery_pre_c
-			* dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_c = (unsigned int) (refcyc_per_req_delivery_c
-			* dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_pre_l = (unsigned int)(refcyc_per_req_delivery_pre_l
+			  * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_l = (unsigned int)(refcyc_per_req_delivery_l
+			  * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_pre_c = (unsigned int)(refcyc_per_req_delivery_pre_c
+			  * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_c = (unsigned int)(refcyc_per_req_delivery_c
+			  * dml_pow(2, 10));
 	disp_ttu_regs->refcyc_per_req_delivery_pre_cur0 =
-			(unsigned int) (refcyc_per_req_delivery_pre_cur0 * dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_cur0 = (unsigned int) (refcyc_per_req_delivery_cur0
-			* dml_pow(2, 10));
+			  (unsigned int)(refcyc_per_req_delivery_pre_cur0 * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_cur0 = (unsigned int)(refcyc_per_req_delivery_cur0
+			  * dml_pow(2, 10));
 	disp_ttu_regs->refcyc_per_req_delivery_pre_cur1 =
-			(unsigned int) (refcyc_per_req_delivery_pre_cur1 * dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_cur1 = (unsigned int) (refcyc_per_req_delivery_cur1
-			* dml_pow(2, 10));
+			  (unsigned int)(refcyc_per_req_delivery_pre_cur1 * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_cur1 = (unsigned int)(refcyc_per_req_delivery_cur1
+			  * dml_pow(2, 10));
 	disp_ttu_regs->qos_level_low_wm = 0;
 	ASSERT(disp_ttu_regs->qos_level_low_wm < dml_pow(2, 14));
 	disp_ttu_regs->qos_level_high_wm = (unsigned int) (4.0 * (double) htotal
@@ -1520,7 +1520,7 @@ static void dml20_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 	disp_ttu_regs->qos_ramp_disable_c = 0;
 	disp_ttu_regs->qos_ramp_disable_cur0 = 0;
 
-	disp_ttu_regs->min_ttu_vblank = min_ttu_vblank * refclk_freq_in_mhz;
+	disp_ttu_regs->min_ttu_vblank = (unsigned int)(min_ttu_vblank * refclk_freq_in_mhz);
 	ASSERT(disp_ttu_regs->min_ttu_vblank < dml_pow(2, 24));
 
 	print__ttu_regs_st(mode_lib, disp_ttu_regs);
@@ -1557,7 +1557,7 @@ void dml20_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	dlg_sys_param.total_flip_bw = get_total_immediate_flip_bw(mode_lib,
 			e2e_pipe_param,
 			num_pipes);
-	dlg_sys_param.total_flip_bytes = get_total_immediate_flip_bytes(mode_lib,
+	dlg_sys_param.total_flip_bytes = (unsigned int)get_total_immediate_flip_bytes(mode_lib,
 			e2e_pipe_param,
 			num_pipes);
 
@@ -1620,7 +1620,7 @@ static void calculate_ttu_cursor(struct display_mode_lib *mode_lib,
 				cur_req_size = 256;
 		}
 
-		cur_req_width = (double) cur_req_size / ((double) cur_bit_per_pixel / 8.0);
+		cur_req_width = (unsigned int)((double) cur_req_size / ((double) cur_bit_per_pixel / 8.0));
 		cur_width_ub = dml_ceil((double) cur_src_width / (double) cur_req_width, 1)
 				* (double) cur_req_width;
 		cur_req_per_width = cur_width_ub / (double) cur_req_width;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c
index d4dddc9d535a2..1d2e477da0322 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn20/display_rq_dlg_calc_20v2.c
@@ -196,13 +196,13 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib,
 
 	extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_l), &rq_param->sizing.rq_l);
 
-	rq_regs->rq_regs_l.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height),
-			1) - 3;
+	rq_regs->rq_regs_l.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height),
+			1) - 3);
 
 	if (rq_param->yuv420) {
 		extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_c), &rq_param->sizing.rq_c);
-		rq_regs->rq_regs_c.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height),
-				1) - 3;
+		rq_regs->rq_regs_c.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height),
+				1) - 3);
 	}
 
 	rq_regs->rq_regs_l.swath_height = dml_log2(rq_param->dlg.rq_l.swath_height);
@@ -222,11 +222,11 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib,
 	if (rq_param->yuv420) {
 		if ((double) rq_param->misc.rq_l.stored_swath_bytes
 				/ (double) rq_param->misc.rq_c.stored_swath_bytes <= 1.5) {
-			detile_buf_plane1_addr = (detile_buf_size_in_bytes / 2.0 / 64.0); // half to chroma
+			detile_buf_plane1_addr = (unsigned int)(detile_buf_size_in_bytes / 2.0 / 64.0); // half to chroma
 		} else {
-			detile_buf_plane1_addr = dml_round_to_multiple((unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0),
-					256,
-					0) / 64.0; // 2/3 to chroma
+			detile_buf_plane1_addr = (unsigned int)(dml_round_to_multiple((unsigned int)((2.0 * detile_buf_size_in_bytes) / 3.0),
+					  256,
+					  0) / 64.0); // 2/3 to chroma
 		}
 	}
 	rq_regs->plane1_base_address = detile_buf_plane1_addr;
@@ -451,8 +451,8 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
 	// "/2" is like square root
 	// blk is vertical biased
 	if (tiling != dm_sw_linear)
-		log2_blk_height = log2_blk256_height
-				+ dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
+		log2_blk_height = (unsigned int)(log2_blk256_height
+				+ dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1));
 	else
 		log2_blk_height = 0;  // blk height of 1
 
@@ -601,7 +601,7 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
 	if (surf_linear) {
 		unsigned int dpte_row_height;
 
-		log2_dpte_row_height_linear = dml_floor(dml_log2(dml_min(64 * 1024 * pde_buf_entries
+		log2_dpte_row_height_linear = (unsigned int)dml_floor(dml_log2(dml_min(64 * 1024 * pde_buf_entries
 										/ bytes_per_element,
 								dpte_buf_in_pte_reqs
 										* dpte_req_width)
@@ -671,7 +671,7 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
 
 	// since dpte groups are only aligned to dpte_req_width and not dpte_group_width,
 	// the upper bound for the dpte groups per row is as follows.
-	rq_dlg_param->dpte_groups_per_row_ub = dml_ceil((double) dpte_row_width_ub / dpte_group_width,
+	rq_dlg_param->dpte_groups_per_row_ub = (unsigned int)dml_ceil((double) dpte_row_width_ub / dpte_group_width,
 			1);
 }
 
@@ -1019,11 +1019,11 @@ static void dml20v2_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 	if (dout->dsc_enable) {
 		double dsc_delay = get_dsc_delay(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
 
-		dispclk_delay_subtotal += dsc_delay;
+		dispclk_delay_subtotal += (unsigned int)dsc_delay;
 	}
 
-	pixel_rate_delay_subtotal = dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
-			+ dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz;
+	pixel_rate_delay_subtotal = (unsigned int)(dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
+			+ dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz);
 
 	vstartup_start = dst->vstartup_start;
 	if (interlaced) {
@@ -1044,7 +1044,7 @@ static void dml20v2_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 
 	// TODO: Where is this coming from?
 	if (interlaced)
-		vstartup_start = vstartup_start / 2;
+		vstartup_start = (unsigned int)(vstartup_start / 2);
 
 	// TODO: What if this min_vblank doesn't match the value in the dml_config_settings.cpp?
 	if (vstartup_start >= min_vblank) {
@@ -1063,8 +1063,8 @@ static void dml20v2_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 				min_vblank);
 	}
 
-	dst_x_after_scaler = get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
-	dst_y_after_scaler = get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+	dst_x_after_scaler = (unsigned int)get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+	dst_y_after_scaler = (unsigned int)get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
 
 	dml_print("DML_DLG: %s: htotal                                 = %d\n", __func__, htotal);
 	dml_print("DML_DLG: %s: pixel_rate_delay_subtotal              = %d\n",
@@ -1377,57 +1377,57 @@ static void dml20v2_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 
 	// Assignment to register structures
 	disp_dlg_regs->dst_y_after_scaler = dst_y_after_scaler; // in terms of line
-	disp_dlg_regs->refcyc_x_after_scaler = dst_x_after_scaler * ref_freq_to_pix_freq; // in terms of refclk
+	disp_dlg_regs->refcyc_x_after_scaler = (unsigned int)(dst_x_after_scaler * ref_freq_to_pix_freq); // in terms of refclk
 	ASSERT(disp_dlg_regs->refcyc_x_after_scaler < (unsigned int) dml_pow(2, 13));
-	disp_dlg_regs->dst_y_prefetch = (unsigned int) (dst_y_prefetch * dml_pow(2, 2));
-	disp_dlg_regs->dst_y_per_vm_vblank = (unsigned int) (dst_y_per_vm_vblank * dml_pow(2, 2));
-	disp_dlg_regs->dst_y_per_row_vblank = (unsigned int) (dst_y_per_row_vblank * dml_pow(2, 2));
-	disp_dlg_regs->dst_y_per_vm_flip = (unsigned int) (dst_y_per_vm_flip * dml_pow(2, 2));
-	disp_dlg_regs->dst_y_per_row_flip = (unsigned int) (dst_y_per_row_flip * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_prefetch = (unsigned int)(dst_y_prefetch * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_vm_vblank = (unsigned int)(dst_y_per_vm_vblank * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_row_vblank = (unsigned int)(dst_y_per_row_vblank * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_vm_flip = (unsigned int)(dst_y_per_vm_flip * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_row_flip = (unsigned int)(dst_y_per_row_flip * dml_pow(2, 2));
 
-	disp_dlg_regs->vratio_prefetch = (unsigned int) (vratio_pre_l * dml_pow(2, 19));
-	disp_dlg_regs->vratio_prefetch_c = (unsigned int) (vratio_pre_c * dml_pow(2, 19));
+	disp_dlg_regs->vratio_prefetch = (unsigned int)(vratio_pre_l * dml_pow(2, 19));
+	disp_dlg_regs->vratio_prefetch_c = (unsigned int)(vratio_pre_c * dml_pow(2, 19));
 
 	disp_dlg_regs->refcyc_per_pte_group_vblank_l =
-			(unsigned int) (dst_y_per_row_vblank * (double) htotal
-					* ref_freq_to_pix_freq / (double) dpte_groups_per_row_ub_l);
+			  (unsigned int)(dst_y_per_row_vblank * (double) htotal
+					    * ref_freq_to_pix_freq / (double) dpte_groups_per_row_ub_l);
 	ASSERT(disp_dlg_regs->refcyc_per_pte_group_vblank_l < (unsigned int) dml_pow(2, 13));
 
 	if (dual_plane) {
-		disp_dlg_regs->refcyc_per_pte_group_vblank_c = (unsigned int) (dst_y_per_row_vblank
-				* (double) htotal * ref_freq_to_pix_freq
-				/ (double) dpte_groups_per_row_ub_c);
+			   disp_dlg_regs->refcyc_per_pte_group_vblank_c = (unsigned int)(dst_y_per_row_vblank
+							   * (double) htotal * ref_freq_to_pix_freq
+							   / (double) dpte_groups_per_row_ub_c);
 		ASSERT(disp_dlg_regs->refcyc_per_pte_group_vblank_c
 						< (unsigned int) dml_pow(2, 13));
 	}
 
 	disp_dlg_regs->refcyc_per_meta_chunk_vblank_l =
-			(unsigned int) (dst_y_per_row_vblank * (double) htotal
-					* ref_freq_to_pix_freq / (double) meta_chunks_per_row_ub_l);
+			  (unsigned int)(dst_y_per_row_vblank * (double) htotal
+					    * ref_freq_to_pix_freq / (double) meta_chunks_per_row_ub_l);
 	ASSERT(disp_dlg_regs->refcyc_per_meta_chunk_vblank_l < (unsigned int) dml_pow(2, 13));
 
 	disp_dlg_regs->refcyc_per_meta_chunk_vblank_c =
 			disp_dlg_regs->refcyc_per_meta_chunk_vblank_l; // dcc for 4:2:0 is not supported in dcn1.0.  assigned to be the same as _l for now
 
-	disp_dlg_regs->refcyc_per_pte_group_flip_l = (unsigned int) (dst_y_per_row_flip * htotal
-			* ref_freq_to_pix_freq) / dpte_groups_per_row_ub_l;
-	disp_dlg_regs->refcyc_per_meta_chunk_flip_l = (unsigned int) (dst_y_per_row_flip * htotal
-			* ref_freq_to_pix_freq) / meta_chunks_per_row_ub_l;
+	disp_dlg_regs->refcyc_per_pte_group_flip_l = (unsigned int)((dst_y_per_row_flip * htotal
+			  * ref_freq_to_pix_freq) / dpte_groups_per_row_ub_l);
+	disp_dlg_regs->refcyc_per_meta_chunk_flip_l = (unsigned int)((dst_y_per_row_flip * htotal
+			  * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_l);
 
 	if (dual_plane) {
-		disp_dlg_regs->refcyc_per_pte_group_flip_c = (unsigned int) (dst_y_per_row_flip
-				* htotal * ref_freq_to_pix_freq) / dpte_groups_per_row_ub_c;
-		disp_dlg_regs->refcyc_per_meta_chunk_flip_c = (unsigned int) (dst_y_per_row_flip
-				* htotal * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_c;
+			   disp_dlg_regs->refcyc_per_pte_group_flip_c = (unsigned int)((dst_y_per_row_flip
+							   * htotal * ref_freq_to_pix_freq) / dpte_groups_per_row_ub_c);
+			   disp_dlg_regs->refcyc_per_meta_chunk_flip_c = (unsigned int)((dst_y_per_row_flip
+							   * htotal * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_c);
 	}
 
-	disp_dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int) ((double) dpte_row_height_l
-			/ (double) vratio_l * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int)(((double) dpte_row_height_l
+			  / (double) vratio_l * dml_pow(2, 2)));
 	ASSERT(disp_dlg_regs->dst_y_per_pte_row_nom_l < (unsigned int) dml_pow(2, 17));
 
 	if (dual_plane) {
-		disp_dlg_regs->dst_y_per_pte_row_nom_c = (unsigned int) ((double) dpte_row_height_c
-				/ (double) vratio_c * dml_pow(2, 2));
+			   disp_dlg_regs->dst_y_per_pte_row_nom_c = (unsigned int)(((double) dpte_row_height_c
+							   / (double) vratio_c * dml_pow(2, 2)));
 		if (disp_dlg_regs->dst_y_per_pte_row_nom_c >= (unsigned int) dml_pow(2, 17)) {
 			dml_print("DML_DLG: %s: Warning dst_y_per_pte_row_nom_c %u larger than supported by register format U15.2 %u\n",
 					__func__,
@@ -1436,38 +1436,38 @@ static void dml20v2_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 		}
 	}
 
-	disp_dlg_regs->dst_y_per_meta_row_nom_l = (unsigned int) ((double) meta_row_height_l
-			/ (double) vratio_l * dml_pow(2, 2));
+	disp_dlg_regs->dst_y_per_meta_row_nom_l = (unsigned int)(((double) meta_row_height_l
+			  / (double) vratio_l * dml_pow(2, 2)));
 	ASSERT(disp_dlg_regs->dst_y_per_meta_row_nom_l < (unsigned int) dml_pow(2, 17));
 
 	disp_dlg_regs->dst_y_per_meta_row_nom_c = disp_dlg_regs->dst_y_per_meta_row_nom_l; // TODO: dcc for 4:2:0 is not supported in dcn1.0.  assigned to be the same as _l for now
 
-	disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int) ((double) dpte_row_height_l
-			/ (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
-			/ (double) dpte_groups_per_row_ub_l);
+	disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)(((double) dpte_row_height_l
+			  / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
+			  / (double) dpte_groups_per_row_ub_l));
 	if (disp_dlg_regs->refcyc_per_pte_group_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_pte_group_nom_l = dml_pow(2, 23) - 1;
-	disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int) ((double) meta_row_height_l
-			/ (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
-			/ (double) meta_chunks_per_row_ub_l);
+		disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
+	disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)(((double) meta_row_height_l
+			  / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
+			  / (double) meta_chunks_per_row_ub_l));
 	if (disp_dlg_regs->refcyc_per_meta_chunk_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (dual_plane) {
-		disp_dlg_regs->refcyc_per_pte_group_nom_c =
-				(unsigned int) ((double) dpte_row_height_c / (double) vratio_c
-						* (double) htotal * ref_freq_to_pix_freq
-						/ (double) dpte_groups_per_row_ub_c);
+			   disp_dlg_regs->refcyc_per_pte_group_nom_c =
+							   (unsigned int)(((double) dpte_row_height_c / (double) vratio_c
+											   * (double) htotal * ref_freq_to_pix_freq
+											   / (double) dpte_groups_per_row_ub_c));
 		if (disp_dlg_regs->refcyc_per_pte_group_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_pte_group_nom_c = dml_pow(2, 23) - 1;
+				disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 
 		// TODO: Is this the right calculation? Does htotal need to be halved?
-		disp_dlg_regs->refcyc_per_meta_chunk_nom_c =
-				(unsigned int) ((double) meta_row_height_c / (double) vratio_c
-						* (double) htotal * ref_freq_to_pix_freq
-						/ (double) meta_chunks_per_row_ub_c);
+			   disp_dlg_regs->refcyc_per_meta_chunk_nom_c =
+							   (unsigned int)(((double) meta_row_height_c / (double) vratio_c
+											   * (double) htotal * ref_freq_to_pix_freq
+											   / (double) meta_chunks_per_row_ub_c));
 		if (disp_dlg_regs->refcyc_per_meta_chunk_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = dml_pow(2, 23) - 1;
+				disp_dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 	}
 
 	disp_dlg_regs->refcyc_per_line_delivery_pre_l = (unsigned int) dml_floor(refcyc_per_line_delivery_pre_l,
@@ -1491,22 +1491,22 @@ static void dml20v2_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 
 	disp_dlg_regs->dst_y_delta_drq_limit = 0x7fff; // off
 
-	disp_ttu_regs->refcyc_per_req_delivery_pre_l = (unsigned int) (refcyc_per_req_delivery_pre_l
-			* dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_l = (unsigned int) (refcyc_per_req_delivery_l
-			* dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_pre_c = (unsigned int) (refcyc_per_req_delivery_pre_c
-			* dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_c = (unsigned int) (refcyc_per_req_delivery_c
-			* dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_pre_l = (unsigned int)(refcyc_per_req_delivery_pre_l
+			  * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_l = (unsigned int)(refcyc_per_req_delivery_l
+			  * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_pre_c = (unsigned int)(refcyc_per_req_delivery_pre_c
+			  * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_c = (unsigned int)(refcyc_per_req_delivery_c
+			  * dml_pow(2, 10));
 	disp_ttu_regs->refcyc_per_req_delivery_pre_cur0 =
-			(unsigned int) (refcyc_per_req_delivery_pre_cur0 * dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_cur0 = (unsigned int) (refcyc_per_req_delivery_cur0
-			* dml_pow(2, 10));
+			  (unsigned int)(refcyc_per_req_delivery_pre_cur0 * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_cur0 = (unsigned int)(refcyc_per_req_delivery_cur0
+			  * dml_pow(2, 10));
 	disp_ttu_regs->refcyc_per_req_delivery_pre_cur1 =
-			(unsigned int) (refcyc_per_req_delivery_pre_cur1 * dml_pow(2, 10));
-	disp_ttu_regs->refcyc_per_req_delivery_cur1 = (unsigned int) (refcyc_per_req_delivery_cur1
-			* dml_pow(2, 10));
+			  (unsigned int)(refcyc_per_req_delivery_pre_cur1 * dml_pow(2, 10));
+	disp_ttu_regs->refcyc_per_req_delivery_cur1 = (unsigned int)(refcyc_per_req_delivery_cur1
+			  * dml_pow(2, 10));
 	disp_ttu_regs->qos_level_low_wm = 0;
 	ASSERT(disp_ttu_regs->qos_level_low_wm < dml_pow(2, 14));
 	disp_ttu_regs->qos_level_high_wm = (unsigned int) (4.0 * (double) htotal
@@ -1521,7 +1521,7 @@ static void dml20v2_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 	disp_ttu_regs->qos_ramp_disable_c = 0;
 	disp_ttu_regs->qos_ramp_disable_cur0 = 0;
 
-	disp_ttu_regs->min_ttu_vblank = min_ttu_vblank * refclk_freq_in_mhz;
+	disp_ttu_regs->min_ttu_vblank = (unsigned int)(min_ttu_vblank * refclk_freq_in_mhz);
 	ASSERT(disp_ttu_regs->min_ttu_vblank < dml_pow(2, 24));
 
 	print__ttu_regs_st(mode_lib, disp_ttu_regs);
@@ -1558,7 +1558,7 @@ void dml20v2_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	dlg_sys_param.total_flip_bw = get_total_immediate_flip_bw(mode_lib,
 			e2e_pipe_param,
 			num_pipes);
-	dlg_sys_param.total_flip_bytes = get_total_immediate_flip_bytes(mode_lib,
+	dlg_sys_param.total_flip_bytes = (unsigned int)get_total_immediate_flip_bytes(mode_lib,
 			e2e_pipe_param,
 			num_pipes);
 
@@ -1621,7 +1621,7 @@ static void calculate_ttu_cursor(struct display_mode_lib *mode_lib,
 				cur_req_size = 256;
 		}
 
-		cur_req_width = (double) cur_req_size / ((double) cur_bit_per_pixel / 8.0);
+		cur_req_width = (unsigned int)((double) cur_req_size / ((double) cur_bit_per_pixel / 8.0));
 		cur_width_ub = dml_ceil((double) cur_src_width / (double) cur_req_width, 1)
 				* (double) cur_req_width;
 		cur_req_per_width = cur_width_ub / (double) cur_req_width;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c
index 11570a0c94272..961b705acfa9d 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c
@@ -296,7 +296,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 		double UrgentOutOfOrderReturn,
 		double ReturnBW,
 		bool GPUVMEnable,
-		int dpte_group_bytes[],
+		unsigned int dpte_group_bytes[],
 		unsigned int MetaChunkSize,
 		double UrgentLatency,
 		double ExtraLatency,
@@ -307,7 +307,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 		double SRExitTime,
 		double SREnterPlusExitTime,
 		double DCFCLKDeepSleep,
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		bool DCCEnable[],
 		double DPPCLK[],
 		double SwathWidthSingleDPPY[],
@@ -346,7 +346,7 @@ static void CalculateDCFCLKDeepSleep(
 		double BytePerPixelDETC[],
 		double VRatio[],
 		double SwathWidthY[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		double HRatio[],
 		double PixelClock[],
 		double PSCL_THROUGHPUT[],
@@ -390,7 +390,7 @@ static void CalculatePixelDeliveryTimes(
 		double                 VRatioPrefetchC[],
 		unsigned int           swath_width_luma_ub[],
 		unsigned int           swath_width_chroma_ub[],
-		int                    DPPPerPlane[],
+		unsigned int           DPPPerPlane[],
 		double                 HRatio[],
 		double                 PixelClock[],
 		double                 PSCL_THROUGHPUT[],
@@ -436,7 +436,7 @@ static void CalculateMetaAndPTETimes(
 		unsigned int           meta_row_height[],
 		unsigned int           meta_req_width[],
 		unsigned int           meta_req_height[],
-		int                   dpte_group_bytes[],
+		unsigned int           dpte_group_bytes[],
 		unsigned int           PTERequestSizeY[],
 		unsigned int           PTERequestSizeC[],
 		unsigned int           PixelPTEReqWidthY[],
@@ -477,8 +477,8 @@ static double CalculateExtraLatency(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
 		int HostVMMaxPageTableLevels,
@@ -521,7 +521,7 @@ static unsigned int dscceComputeDelay(
 		pixelsPerClock = 1;
 
 	//initial transmit delay as per PPS
-	initalXmitDelay = dml_round(rcModelSize / 2.0 / bpp / pixelsPerClock);
+	initalXmitDelay = (unsigned int)dml_round(rcModelSize / 2.0 / bpp / pixelsPerClock);
 
 	//compute ssm delay
 	if (bpc == 8)
@@ -729,22 +729,22 @@ static bool CalculatePrefetchSchedule(
 	}
 
 	if (myPipe->ScalerEnabled)
-		DPPCycles = DPPCLKDelaySubtotal + DPPCLKDelaySCL;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotal + DPPCLKDelaySCL);
 	else
-		DPPCycles = DPPCLKDelaySubtotal + DPPCLKDelaySCLLBOnly;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotal + DPPCLKDelaySCLLBOnly);
 
-	DPPCycles = DPPCycles + DPPCLKDelayCNVCFormater + myPipe->NumberOfCursors * DPPCLKDelayCNVCCursor;
+	DPPCycles = (unsigned int)(DPPCycles + DPPCLKDelayCNVCFormater + myPipe->NumberOfCursors * DPPCLKDelayCNVCCursor);
 
-	DISPCLKCycles = DISPCLKDelaySubtotal;
+	DISPCLKCycles = (unsigned int)(DISPCLKDelaySubtotal);
 
 	if (myPipe->DPPCLK == 0.0 || myPipe->DISPCLK == 0.0)
 		return true;
 
-	*DSTXAfterScaler = DPPCycles * myPipe->PixelClock / myPipe->DPPCLK
-			+ DISPCLKCycles * myPipe->PixelClock / myPipe->DISPCLK + DSCDelay;
+	*DSTXAfterScaler = (unsigned int)(DPPCycles * myPipe->PixelClock / myPipe->DPPCLK
+			  + DISPCLKCycles * myPipe->PixelClock / myPipe->DISPCLK + DSCDelay);
 
-	if (myPipe->DPPPerPlane > 1)
-		*DSTXAfterScaler = *DSTXAfterScaler + ScalerRecoutWidth;
+	       if (myPipe->DPPPerPlane > 1)
+		       *DSTXAfterScaler = (unsigned int)(*DSTXAfterScaler + ScalerRecoutWidth);
 
 	if (OutputFormat == dm_420 || (myPipe->InterlaceEnable && ProgressiveToInterlaceUnitInOPP))
 		*DSTYAfterScaler = 1;
@@ -755,7 +755,7 @@ static bool CalculatePrefetchSchedule(
 	*DSTYAfterScaler = dml_floor(DSTTotalPixelsAfterScaler / myPipe->HTotal, 1);
 	*DSTXAfterScaler = DSTTotalPixelsAfterScaler - ((double) (*DSTYAfterScaler * myPipe->HTotal));
 
-	*VUpdateOffsetPix = dml_ceil(myPipe->HTotal / 4.0, 1);
+	*VUpdateOffsetPix = (unsigned int)(dml_ceil(myPipe->HTotal / 4.0, 1));
 	TotalRepeaterDelayTime = MaxInterDCNTileRepeaters * (2.0 / myPipe->DPPCLK + 3.0 / myPipe->DISPCLK);
 	*VUpdateWidthPix = (14.0 / myPipe->DCFCLKDeepSleep + 12.0 / myPipe->DPPCLK + TotalRepeaterDelayTime)
 			* myPipe->PixelClock;
@@ -783,11 +783,11 @@ static bool CalculatePrefetchSchedule(
 			Tdmsks = Tdmsks / 2;
 		if (VStartup * LineTime
 				< Tsetup + TWait + UrgentExtraLatency + Tdmbf + Tdmec + Tdmsks) {
-			MyError = true;
-			*VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = (Tsetup + TWait
-					+ UrgentExtraLatency + Tdmbf + Tdmec + Tdmsks) / LineTime;
+			MyError = (unsigned int)(true);
+			*VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = (unsigned int)((Tsetup + TWait
+					  + UrgentExtraLatency + Tdmbf + Tdmec + Tdmsks) / LineTime);
 		} else
-			*VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = 0.0;
+			*VStartupRequiredWhenNotEnoughTimeForDynamicMetadata = (unsigned int)(0U);
 	} else
 		Tdm = 0;
 
@@ -808,13 +808,13 @@ static bool CalculatePrefetchSchedule(
 	Tsw_oto = dml_max(PrefetchSourceLinesY, PrefetchSourceLinesC) * LineTime;
 
 	if (myPipe->SourceScan == dm_horz) {
-		*swath_width_luma_ub = dml_ceil(SwathWidthY - 1, myPipe->BlockWidth256BytesY) + myPipe->BlockWidth256BytesY;
+		*swath_width_luma_ub = (unsigned int)(dml_ceil(SwathWidthY - 1, myPipe->BlockWidth256BytesY) + myPipe->BlockWidth256BytesY);
 		if (myPipe->BlockWidth256BytesC > 0)
-			*swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockWidth256BytesC) + myPipe->BlockWidth256BytesC;
+			*swath_width_chroma_ub = (unsigned int)(dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockWidth256BytesC) + myPipe->BlockWidth256BytesC);
 	} else {
-		*swath_width_luma_ub = dml_ceil(SwathWidthY - 1, myPipe->BlockHeight256BytesY) + myPipe->BlockHeight256BytesY;
+		*swath_width_luma_ub = (unsigned int)(dml_ceil(SwathWidthY - 1, myPipe->BlockHeight256BytesY) + myPipe->BlockHeight256BytesY);
 		if (myPipe->BlockHeight256BytesC > 0)
-			*swath_width_chroma_ub = dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockHeight256BytesC) + myPipe->BlockHeight256BytesC;
+			*swath_width_chroma_ub = (unsigned int)(dml_ceil(SwathWidthY / 2 - 1, myPipe->BlockHeight256BytesC) + myPipe->BlockHeight256BytesC);
 	}
 
 	prefetch_bw_oto = (PrefetchSourceLinesY * *swath_width_luma_ub * dml_ceil(BytePerPixelDETY, 1) + PrefetchSourceLinesC * *swath_width_chroma_ub * dml_ceil(BytePerPixelDETC, 2)) / Tsw_oto;
@@ -1228,14 +1228,14 @@ static double CalculatePrefetchSourceLines(
 
 	if (!mode_lib->vba.IgnoreViewportPositioning) {
 
-		*MaxNumSwath = dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0;
+		*MaxNumSwath = (unsigned int)(dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0);
 
 		if (*VInitPreFill > 1.0)
 			MaxPartialSwath = (unsigned int) (*VInitPreFill - 2) % SwathHeight;
 		else
 			MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 2)
 					% SwathHeight;
-		MaxPartialSwath = dml_max(1U, MaxPartialSwath);
+		MaxPartialSwath = (unsigned int)(dml_max(1U, MaxPartialSwath));
 
 	} else {
 
@@ -1243,7 +1243,7 @@ static double CalculatePrefetchSourceLines(
 			dml_print(
 					"WARNING DML: using viewport y position of 0 even though actual viewport y position is non-zero in prefetch source lines calculation\n");
 
-		*MaxNumSwath = dml_ceil(*VInitPreFill / SwathHeight, 1);
+		*MaxNumSwath = (unsigned int)(dml_ceil(*VInitPreFill / SwathHeight, 1));
 
 		if (*VInitPreFill > 1.0)
 			MaxPartialSwath = (unsigned int) (*VInitPreFill - 1) % SwathHeight;
@@ -1252,7 +1252,7 @@ static double CalculatePrefetchSourceLines(
 					% SwathHeight;
 	}
 
-	return *MaxNumSwath * SwathHeight + MaxPartialSwath;
+	return (unsigned int)(*MaxNumSwath * SwathHeight + MaxPartialSwath);
 }
 
 static unsigned int CalculateVMAndRowBytes(
@@ -1307,34 +1307,34 @@ static unsigned int CalculateVMAndRowBytes(
 		*MetaRequestHeight = 8 * BlockHeight256Bytes;
 		*MetaRequestWidth = 8 * BlockWidth256Bytes;
 		if (ScanDirection == dm_horz) {
-			*meta_row_height = *MetaRequestHeight;
-			*meta_row_width = dml_ceil((double) SwathWidth - 1, *MetaRequestWidth)
-					+ *MetaRequestWidth;
-			*MetaRowByte = *meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0;
+			*meta_row_height = (unsigned int)(*MetaRequestHeight);
+			*meta_row_width = (unsigned int)(dml_ceil((double) SwathWidth - 1, *MetaRequestWidth)
+					+ *MetaRequestWidth);
+			*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0);
 		} else {
-			*meta_row_height = *MetaRequestWidth;
-			*meta_row_width = dml_ceil((double) SwathWidth - 1, *MetaRequestHeight)
-					+ *MetaRequestHeight;
-			*MetaRowByte = *meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0;
+			*meta_row_height = (unsigned int)(*MetaRequestWidth);
+			*meta_row_width = (unsigned int)(dml_ceil((double) SwathWidth - 1, *MetaRequestHeight)
+					+ *MetaRequestHeight);
+			*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0);
 		}
 		if (ScanDirection == dm_horz) {
-			DCCMetaSurfaceBytes = DCCMetaPitch
+			DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch
 					* (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes)
 							+ 64 * BlockHeight256Bytes) * BytePerPixel
-					/ 256;
+					/ 256);
 		} else {
-			DCCMetaSurfaceBytes = DCCMetaPitch
+			DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch
 					* (dml_ceil(
 							(double) ViewportHeight - 1,
 							64 * BlockHeight256Bytes)
 							+ 64 * BlockHeight256Bytes) * BytePerPixel
-					/ 256;
+					/ 256);
 		}
 		if (GPUVMEnable == true) {
-			*MetaPTEBytesFrame = (dml_ceil(
+			*MetaPTEBytesFrame = (unsigned int)((dml_ceil(
 					(double) (DCCMetaSurfaceBytes - VMMPageSize)
 							/ (8 * VMMPageSize),
-					1) + 1) * 64;
+					1) + 1) * 64);
 			MPDEBytesFrame = 128 * ((mode_lib->vba.GPUVMMaxPageTableLevels + 1) * (mode_lib->vba.HostVMMaxPageTableLevels + 1) - 2);
 		} else {
 			*MetaPTEBytesFrame = 0;
@@ -1367,9 +1367,9 @@ static unsigned int CalculateVMAndRowBytes(
 
 	if (GPUVMEnable == true && (mode_lib->vba.GPUVMMaxPageTableLevels + 1) * (mode_lib->vba.HostVMMaxPageTableLevels + 1) > 2) {
 		if (ScanDirection == dm_horz) {
-			*DPDE0BytesFrame = 64 * (dml_ceil(((Pitch * (dml_ceil(ViewportHeight - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes) / (8 * 2097152), 1) + 1);
+			*DPDE0BytesFrame = (unsigned int)(64 * (dml_ceil(((Pitch * (dml_ceil(ViewportHeight - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes) / (8 * 2097152), 1) + 1));
 		} else {
-			*DPDE0BytesFrame = 64 * (dml_ceil(((Pitch * (dml_ceil((double) SwathWidth - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes) / (8 * 2097152), 1) + 1);
+			*DPDE0BytesFrame = (unsigned int)(64 * (dml_ceil(((Pitch * (dml_ceil((double) SwathWidth - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes) / (8 * 2097152), 1) + 1));
 		}
 		ExtraDPDEBytesFrame = 128 * ((mode_lib->vba.GPUVMMaxPageTableLevels + 1) * (mode_lib->vba.HostVMMaxPageTableLevels + 1) - 3);
 	} else {
@@ -1390,7 +1390,7 @@ static unsigned int CalculateVMAndRowBytes(
 		if (SurfaceTiling == dm_sw_linear) {
 			PixelPTEReqHeightPTEs = 1;
 			*PixelPTEReqHeight = 1;
-			*PixelPTEReqWidth = 8.0 * VMMPageSize / BytePerPixel;
+			*PixelPTEReqWidth = (unsigned int)(8.0 * VMMPageSize / BytePerPixel);
 			*PTERequestSize = 64;
 			FractionOfPTEReturnDrop = 0;
 		} else if (MacroTileSizeBytes == 4096) {
@@ -1417,20 +1417,20 @@ static unsigned int CalculateVMAndRowBytes(
 		}
 
 		if (SurfaceTiling == dm_sw_linear) {
-			*dpte_row_height = dml_min(128,
+			*dpte_row_height = (unsigned int)dml_min(128,
 					1 << (unsigned int) dml_floor(
 						dml_log2(
 							(double) PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch),
 						1));
-			*dpte_row_width_ub = (dml_ceil((double) (Pitch * *dpte_row_height - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth;
+			*dpte_row_width_ub = (unsigned int)((dml_ceil((double) (Pitch * *dpte_row_height - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth);
 			*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqWidth * *PTERequestSize;
 		} else if (ScanDirection == dm_horz) {
 			*dpte_row_height = *PixelPTEReqHeight;
-			*dpte_row_width_ub = (dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth;
+			*dpte_row_width_ub = (unsigned int)((dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth);
 			*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqWidth * *PTERequestSize;
 		} else {
-			*dpte_row_height = dml_min(*PixelPTEReqWidth, *MacroTileWidth);
-			*dpte_row_width_ub = (dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1) * *PixelPTEReqHeight;
+			*dpte_row_height = (unsigned int)(dml_min(*PixelPTEReqWidth, *MacroTileWidth));
+			*dpte_row_width_ub = (unsigned int)((dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1) * *PixelPTEReqHeight);
 			*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqHeight * *PTERequestSize;
 		}
 		if (*PixelPTEBytesPerRow * (1 - FractionOfPTEReturnDrop)
@@ -1805,7 +1805,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 						dscceComputeDelay(
 								mode_lib->vba.DSCInputBitPerComponent[k],
 								bpp,
-								dml_ceil(
+								(unsigned int)dml_ceil(
 										(double) mode_lib->vba.HActive[k]
 												/ mode_lib->vba.NumberOfDSCSlices[k],
 										1),
@@ -1819,18 +1819,18 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 								* (dscceComputeDelay(
 										mode_lib->vba.DSCInputBitPerComponent[k],
 										bpp,
-										dml_ceil(
+										(unsigned int)dml_ceil(
 												(double) mode_lib->vba.HActive[k]
 														/ mode_lib->vba.NumberOfDSCSlices[k],
 												1),
-										slices / 2.0,
+										(unsigned int)(slices / 2.0),
 										mode_lib->vba.OutputFormat[k])
 										+ dscComputeDelay(
 												mode_lib->vba.OutputFormat[k]));
 			}
-			locals->DSCDelay[k] = locals->DSCDelay[k]
+			locals->DSCDelay[k] = (unsigned int)(locals->DSCDelay[k]
 					* mode_lib->vba.PixelClock[k]
-					/ mode_lib->vba.PixelClockBackEnd[k];
+					/ mode_lib->vba.PixelClockBackEnd[k]);
 		} else {
 			locals->DSCDelay[k] = 0;
 		}
@@ -1856,8 +1856,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 		Calculate256BBlockSizes(
 				mode_lib->vba.SourcePixelFormat[k],
 				mode_lib->vba.SurfaceTiling[k],
-				dml_ceil(locals->BytePerPixelDETY[k], 1),
-				dml_ceil(locals->BytePerPixelDETC[k], 2),
+				(unsigned int)dml_ceil(locals->BytePerPixelDETY[k], 1),
+				(unsigned int)dml_ceil(locals->BytePerPixelDETC[k], 2),
 				&locals->BlockHeight256BytesY[k],
 				&locals->BlockHeight256BytesC[k],
 				&locals->BlockWidth256BytesY[k],
@@ -1886,13 +1886,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 							locals->BlockWidth256BytesC[k],
 							mode_lib->vba.SourcePixelFormat[k],
 							mode_lib->vba.SurfaceTiling[k],
-							dml_ceil(
+							(unsigned int)dml_ceil(
 									locals->BytePerPixelDETC[k],
 									2),
 							mode_lib->vba.SourceScan[k],
-							mode_lib->vba.ViewportWidth[k] / 2,
+							(unsigned int)(mode_lib->vba.ViewportWidth[k] / 2),
 							mode_lib->vba.ViewportHeight[k] / 2,
-							locals->SwathWidthY[k] / 2,
+							(unsigned int)(locals->SwathWidthY[k] / 2),
 							mode_lib->vba.GPUVMEnable,
 							mode_lib->vba.HostVMEnable,
 							mode_lib->vba.HostVMMaxPageTableLevels,
@@ -1945,11 +1945,11 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				locals->BlockWidth256BytesY[k],
 				mode_lib->vba.SourcePixelFormat[k],
 				mode_lib->vba.SurfaceTiling[k],
-				dml_ceil(locals->BytePerPixelDETY[k], 1),
+				(unsigned int)dml_ceil(locals->BytePerPixelDETY[k], 1),
 				mode_lib->vba.SourceScan[k],
 				mode_lib->vba.ViewportWidth[k],
 				mode_lib->vba.ViewportHeight[k],
-				locals->SwathWidthY[k],
+				(unsigned int)locals->SwathWidthY[k],
 				mode_lib->vba.GPUVMEnable,
 				mode_lib->vba.HostVMEnable,
 				mode_lib->vba.HostVMMaxPageTableLevels,
@@ -2053,7 +2053,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 										mode_lib->vba.WritebackLumaVTaps[k],
 										mode_lib->vba.WritebackChromaHTaps[k],
 										mode_lib->vba.WritebackChromaVTaps[k],
-										mode_lib->vba.WritebackDestinationWidth[k])
+												(unsigned int)mode_lib->vba.WritebackDestinationWidth[k])
 										/ mode_lib->vba.DISPCLK;
 			} else
 				locals->WritebackDelay[mode_lib->vba.VoltageLevel][k] = 0;
@@ -2072,7 +2072,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 													mode_lib->vba.WritebackLumaVTaps[j],
 													mode_lib->vba.WritebackChromaHTaps[j],
 													mode_lib->vba.WritebackChromaVTaps[j],
-													mode_lib->vba.WritebackDestinationWidth[j])
+																(unsigned int)mode_lib->vba.WritebackDestinationWidth[j])
 													/ mode_lib->vba.DISPCLK);
 				}
 			}
@@ -2087,11 +2087,11 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 
 	mode_lib->vba.VStartupLines = 13;
 	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
-		locals->MaxVStartupLines[k] = mode_lib->vba.VTotal[k] - mode_lib->vba.VActive[k] - dml_max(1.0, dml_ceil(locals->WritebackDelay[mode_lib->vba.VoltageLevel][k] / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]), 1));
+		locals->MaxVStartupLines[k] = (unsigned int)(mode_lib->vba.VTotal[k] - mode_lib->vba.VActive[k] - dml_max(1.0, dml_ceil(locals->WritebackDelay[mode_lib->vba.VoltageLevel][k] / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]), 1)));
 	}
 
 	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k)
-		locals->MaximumMaxVStartupLines = dml_max(locals->MaximumMaxVStartupLines, locals->MaxVStartupLines[k]);
+		locals->MaximumMaxVStartupLines = (unsigned int)(dml_max(locals->MaximumMaxVStartupLines, locals->MaxVStartupLines[k]));
 
 	// We don't really care to iterate between the various prefetch modes
 	//mode_lib->vba.PrefetchERROR = CalculateMinAndMaxPrefetchMode(mode_lib->vba.AllowDRAMSelfRefreshOrDRAMClockChangeInVblank, &mode_lib->vba.MinPrefetchMode, &mode_lib->vba.MaxPrefetchMode);
@@ -2177,7 +2177,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 									/ mode_lib->vba.HRatio[k]),
 							mode_lib->vba.OutputFormat[k],
 							mode_lib->vba.MaxInterDCNTileRepeaters,
-							dml_min(mode_lib->vba.VStartupLines, locals->MaxVStartupLines[k]),
+							(unsigned int)dml_min(mode_lib->vba.VStartupLines, locals->MaxVStartupLines[k]),
 							locals->MaxVStartupLines[k],
 							mode_lib->vba.GPUVMMaxPageTableLevels,
 							mode_lib->vba.GPUVMEnable,
@@ -2189,11 +2189,11 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 							mode_lib->vba.UrgentLatency,
 							mode_lib->vba.UrgentExtraLatency,
 							mode_lib->vba.TCalc,
-							locals->PDEAndMetaPTEBytesFrame[k],
-							locals->MetaRowByte[k],
-							locals->PixelPTEBytesPerRow[k],
+							(unsigned int)locals->PDEAndMetaPTEBytesFrame[k],
+							(unsigned int)locals->MetaRowByte[k],
+							(unsigned int)locals->PixelPTEBytesPerRow[k],
 							locals->PrefetchSourceLinesY[k],
-							locals->SwathWidthY[k],
+							(unsigned int)locals->SwathWidthY[k],
 							locals->BytePerPixelDETY[k],
 							locals->VInitPreFillY[k],
 							locals->MaxNumSwathY[k],
@@ -2226,7 +2226,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 							&mode_lib->vba.VUpdateWidthPix[k],
 							&mode_lib->vba.VReadyOffsetPix[k]);
 			if (mode_lib->vba.BlendingAndTiming[k] == k) {
-				locals->VStartup[k] = dml_min(
+				locals->VStartup[k] = (unsigned int)dml_min(
 						mode_lib->vba.VStartupLines,
 						locals->MaxVStartupLines[k]);
 				if (locals->VStartupRequiredWhenNotEnoughTimeForDynamicMetadata
@@ -2236,7 +2236,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				}
 			} else {
 				locals->VStartup[k] =
-						dml_min(
+						(unsigned int)dml_min(
 								mode_lib->vba.VStartupLines,
 								locals->MaxVStartupLines[mode_lib->vba.BlendingAndTiming[k]]);
 			}
@@ -2256,13 +2256,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					mode_lib->vba.DETBufferSizeInKByte[0],
 					mode_lib->vba.SwathHeightY[k],
 					mode_lib->vba.SwathHeightC[k],
-					locals->SwathWidthY[k],
+					(unsigned int)locals->SwathWidthY[k],
 					mode_lib->vba.HTotal[k] /
 					mode_lib->vba.PixelClock[k],
 					mode_lib->vba.UrgentLatency,
 					mode_lib->vba.CursorBufferSize,
 					mode_lib->vba.CursorWidth[k][0] + mode_lib->vba.CursorWidth[k][1],
-					dml_max(mode_lib->vba.CursorBPP[k][0], mode_lib->vba.CursorBPP[k][1]),
+					(unsigned int)dml_max(mode_lib->vba.CursorBPP[k][0], mode_lib->vba.CursorBPP[k][1]),
 					mode_lib->vba.VRatio[k],
 					locals->VRatioPrefetchY[k],
 					locals->VRatioPrefetchC[k],
@@ -2333,7 +2333,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 
 			mode_lib->vba.TotImmediateFlipBytes = 0;
 			for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
-				mode_lib->vba.TotImmediateFlipBytes = mode_lib->vba.TotImmediateFlipBytes + locals->PDEAndMetaPTEBytesFrame[k] + locals->MetaRowByte[k] + locals->PixelPTEBytesPerRow[k];
+				mode_lib->vba.TotImmediateFlipBytes = (unsigned int)(mode_lib->vba.TotImmediateFlipBytes + locals->PDEAndMetaPTEBytesFrame[k] + locals->MetaRowByte[k] + locals->PixelPTEBytesPerRow[k]);
 			}
 			for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
 				CalculateFlipSchedule(
@@ -2600,7 +2600,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			locals->BlockHeight256BytesY[k],
 			mode_lib->vba.SwathHeightY[k],
 			mode_lib->vba.SurfaceTiling[k],
-			locals->BytePerPixelDETY[k],
+			(unsigned int)locals->BytePerPixelDETY[k],
 			mode_lib->vba.SourceScan[k],
 			&locals->DCCYMaxUncompressedBlock[k],
 			&locals->DCCYMaxCompressedBlock[k],
@@ -2722,7 +2722,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			mode_lib->vba.FrameTimeForMinFullDETBufferingTime =
 				(double) mode_lib->vba.VTotal[k] * mode_lib->vba.HTotal[k]
 				/ mode_lib->vba.PixelClock[k];
-			locals->BytePerPixelYCriticalPlane = dml_ceil(locals->BytePerPixelDETY[k], 1);
+			locals->BytePerPixelYCriticalPlane = (unsigned int)(dml_ceil(locals->BytePerPixelDETY[k], 1));
 			locals->SwathWidthYCriticalPlane = locals->SwathWidthY[k];
 			locals->LinesToFinishSwathTransferStutterCriticalPlane =
 				mode_lib->vba.SwathHeightY[k] - (locals->LinesInDETY[k] - locals->LinesInDETYRoundedDownToSwath[k]);
@@ -2993,11 +2993,11 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib)
 
 		if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
 				<= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
-			mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
-			mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
+			mode_lib->vba.SwathHeightY[k] = (unsigned int)(MaximumSwathHeightY);
+			mode_lib->vba.SwathHeightC[k] = (unsigned int)(MaximumSwathHeightC);
 		} else {
-			mode_lib->vba.SwathHeightY[k] = MinimumSwathHeightY;
-			mode_lib->vba.SwathHeightC[k] = MinimumSwathHeightC;
+			mode_lib->vba.SwathHeightY[k] = (unsigned int)(MinimumSwathHeightY);
+			mode_lib->vba.SwathHeightC[k] = (unsigned int)(MinimumSwathHeightC);
 		}
 
 		CalculateDETBufferSize(
@@ -3317,9 +3317,9 @@ static unsigned int TruncToValidBPP(
 					if (DecimalBPP < 6)
 						return BPP_INVALID;
 					else if (DecimalBPP >= 1.5 * DSCInputBitPerComponent - 1.0 / 16.0)
-						return 1.5 * DSCInputBitPerComponent - 1.0 / 16.0;
+						return (unsigned int)(1.5 * DSCInputBitPerComponent - 1.0 / 16.0);
 					else
-						return dml_floor(16 * DecimalBPP, 1) / 16.0;
+						return (unsigned int)(dml_floor(16 * DecimalBPP, 1) / 16.0);
 				} else {
 					if (DecimalBPP < 6
 							|| DesiredBPP < 6
@@ -3327,7 +3327,7 @@ static unsigned int TruncToValidBPP(
 							|| DecimalBPP < DesiredBPP) {
 						return BPP_INVALID;
 					} else {
-						return DesiredBPP;
+						return (unsigned int)(DesiredBPP);
 					}
 				}
 			} else if (Format == dm_n422) {
@@ -3335,9 +3335,9 @@ static unsigned int TruncToValidBPP(
 					if (DecimalBPP < 7)
 						return BPP_INVALID;
 					else if (DecimalBPP >= 2 * DSCInputBitPerComponent - 1.0 / 16.0)
-						return 2 * DSCInputBitPerComponent - 1.0 / 16.0;
+						return (unsigned int)(2 * DSCInputBitPerComponent - 1.0 / 16.0);
 					else
-						return dml_floor(16 * DecimalBPP, 1) / 16.0;
+						return (unsigned int)(dml_floor(16 * DecimalBPP, 1) / 16.0);
 				} else {
 					if (DecimalBPP < 7
 							|| DesiredBPP < 7
@@ -3345,7 +3345,7 @@ static unsigned int TruncToValidBPP(
 							|| DecimalBPP < DesiredBPP) {
 						return BPP_INVALID;
 					} else {
-						return DesiredBPP;
+						return (unsigned int)(DesiredBPP);
 					}
 				}
 			} else {
@@ -3353,9 +3353,9 @@ static unsigned int TruncToValidBPP(
 					if (DecimalBPP < 8)
 						return BPP_INVALID;
 					else if (DecimalBPP >= 3 * DSCInputBitPerComponent - 1.0 / 16.0)
-						return 3 * DSCInputBitPerComponent - 1.0 / 16.0;
+						return (unsigned int)(3 * DSCInputBitPerComponent - 1.0 / 16.0);
 					else
-						return dml_floor(16 * DecimalBPP, 1) / 16.0;
+						return (unsigned int)(dml_floor(16 * DecimalBPP, 1) / 16.0);
 				} else {
 					if (DecimalBPP < 8
 							|| DesiredBPP < 8
@@ -3363,7 +3363,7 @@ static unsigned int TruncToValidBPP(
 							|| DecimalBPP < DesiredBPP) {
 						return BPP_INVALID;
 					} else {
-						return DesiredBPP;
+						return (unsigned int)(DesiredBPP);
 					}
 				}
 			}
@@ -3462,18 +3462,18 @@ static noinline void CalculatePrefetchSchedulePerPlane(
 			mode_lib->vba.PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
 			mode_lib->vba.PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
 			&myPipe,
-			locals->DSCDelayPerState[i][k],
+			(unsigned int)locals->DSCDelayPerState[i][k],
 			mode_lib->vba.DPPCLKDelaySubtotal,
 			mode_lib->vba.DPPCLKDelaySCL,
 			mode_lib->vba.DPPCLKDelaySCLLBOnly,
 			mode_lib->vba.DPPCLKDelayCNVCFormater,
 			mode_lib->vba.DPPCLKDelayCNVCCursor,
 			mode_lib->vba.DISPCLKDelaySubtotal,
-			locals->SwathWidthYThisState[k] / mode_lib->vba.HRatio[k],
+			(unsigned int)(locals->SwathWidthYThisState[k] / mode_lib->vba.HRatio[k]),
 			mode_lib->vba.OutputFormat[k],
 			mode_lib->vba.MaxInterDCNTileRepeaters,
-			dml_min(mode_lib->vba.MaxVStartup, locals->MaximumVStartup[0][0][k]),
-			locals->MaximumVStartup[0][0][k],
+			(unsigned int)dml_min(mode_lib->vba.MaxVStartup, locals->MaximumVStartup[0][0][k]),
+			(unsigned int)locals->MaximumVStartup[0][0][k],
 			mode_lib->vba.GPUVMMaxPageTableLevels,
 			mode_lib->vba.GPUVMEnable,
 			&myHostVM,
@@ -3484,11 +3484,11 @@ static noinline void CalculatePrefetchSchedulePerPlane(
 			mode_lib->vba.UrgentLatency,
 			mode_lib->vba.ExtraLatency,
 			mode_lib->vba.TimeCalc,
-			locals->PDEAndMetaPTEBytesPerFrame[0][0][k],
-			locals->MetaRowBytes[0][0][k],
-			locals->DPTEBytesPerRow[0][0][k],
+			(unsigned int)locals->PDEAndMetaPTEBytesPerFrame[0][0][k],
+			(unsigned int)locals->MetaRowBytes[0][0][k],
+			(unsigned int)locals->DPTEBytesPerRow[0][0][k],
 			locals->PrefetchLinesY[0][0][k],
-			locals->SwathWidthYThisState[k],
+			(unsigned int)locals->SwathWidthYThisState[k],
 			locals->BytePerPixelInDETY[k],
 			locals->PrefillY[k],
 			locals->MaxNumSwY[k],
@@ -3525,8 +3525,8 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 {
 	struct vba_vars_st *locals = &mode_lib->vba;
 
-	int i;
-	unsigned int j, k, m;
+	int idx;
+	unsigned int i, j, k, m;
 
 	/*MODE SUPPORT, VOLTAGE STATE AND SOC CONFIGURATION*/
 
@@ -3951,8 +3951,8 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		Calculate256BBlockSizes(
 				mode_lib->vba.SourcePixelFormat[k],
 				mode_lib->vba.SurfaceTiling[k],
-				dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
-				dml_ceil(locals->BytePerPixelInDETC[k], 2.0),
+				(unsigned int)dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
+				(unsigned int)dml_ceil(locals->BytePerPixelInDETC[k], 2.0),
 				&locals->Read256BlockHeightY[k],
 				&locals->Read256BlockHeightC[k],
 				&locals->Read256BlockWidthY[k],
@@ -4141,7 +4141,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					locals->DISPCLK_DPPCLK_Support[i][j] = false;
 				}
 			}
-			locals->TotalNumberOfActiveDPP[i][j] = 0.0;
+			locals->TotalNumberOfActiveDPP[i][j] = (unsigned int)(0.0);
 			for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++)
 				locals->TotalNumberOfActiveDPP[i][j] = locals->TotalNumberOfActiveDPP[i][j] + locals->NoOfDPP[i][j][k];
 			if (j == 1) {
@@ -4196,7 +4196,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							|| mode_lib->vba.PlaneRequiredDISPCLK > mode_lib->vba.MaxDispclkRoundedDownToDFSGranularity)
 						locals->DISPCLK_DPPCLK_Support[i][j] = false;
 				}
-				locals->TotalNumberOfActiveDPP[i][j] = 0.0;
+				locals->TotalNumberOfActiveDPP[i][j] = (unsigned int)(0.0);
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++)
 					locals->TotalNumberOfActiveDPP[i][j] = locals->TotalNumberOfActiveDPP[i][j] + locals->NoOfDPP[i][j][k];
 			}
@@ -4455,19 +4455,19 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				mode_lib->vba.slices = 0;
 			} else if (locals->RequiresDSC[i][k] == 0
 					|| locals->RequiresDSC[i][k] == false) {
-				mode_lib->vba.slices = 0;
+				mode_lib->vba.slices = (unsigned int)(0);
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 3200.0) {
-				mode_lib->vba.slices = dml_ceil(
+				mode_lib->vba.slices = (unsigned int)dml_ceil(
 						mode_lib->vba.PixelClockBackEnd[k] / 400.0,
 						4.0);
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 1360.0) {
-				mode_lib->vba.slices = 8.0;
+				mode_lib->vba.slices = (unsigned int)(8.0);
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 680.0) {
-				mode_lib->vba.slices = 4.0;
+				mode_lib->vba.slices = (unsigned int)(4.0);
 			} else if (mode_lib->vba.PixelClockBackEnd[k] > 340.0) {
-				mode_lib->vba.slices = 2.0;
+				mode_lib->vba.slices = (unsigned int)(2.0);
 			} else {
-				mode_lib->vba.slices = 1.0;
+				mode_lib->vba.slices = (unsigned int)(1.0);
 			}
 			if (locals->OutputBppPerState[i][k] == BPP_BLENDED_PIPE
 					|| locals->OutputBppPerState[i][k] == BPP_INVALID) {
@@ -4481,7 +4481,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							dscceComputeDelay(
 									mode_lib->vba.DSCInputBitPerComponent[k],
 									mode_lib->vba.bpp,
-									dml_ceil(
+									(unsigned int)dml_ceil(
 											mode_lib->vba.HActive[k]
 													/ mode_lib->vba.slices,
 											1.0),
@@ -4494,7 +4494,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							2.0 * (dscceComputeDelay(
 											mode_lib->vba.DSCInputBitPerComponent[k],
 											mode_lib->vba.bpp,
-											dml_ceil(mode_lib->vba.HActive[k] / mode_lib->vba.slices, 1.0),
+									(unsigned int)dml_ceil(mode_lib->vba.HActive[k] / mode_lib->vba.slices, 1.0),
 											mode_lib->vba.slices / 2,
 											mode_lib->vba.OutputFormat[k])
 									+ dscComputeDelay(mode_lib->vba.OutputFormat[k]));
@@ -4569,13 +4569,13 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				}
 				if (mode_lib->vba.RoundedUpMaxSwathSizeBytesY + mode_lib->vba.RoundedUpMaxSwathSizeBytesC
 						<= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
-					locals->SwathHeightYThisState[k] = locals->MaxSwathHeightY[k];
-					locals->SwathHeightCThisState[k] = locals->MaxSwathHeightC[k];
+					locals->SwathHeightYThisState[k] = (unsigned int)(locals->MaxSwathHeightY[k]);
+					locals->SwathHeightCThisState[k] = (unsigned int)(locals->MaxSwathHeightC[k]);
 				} else {
 					locals->SwathHeightYThisState[k] =
-							locals->MinSwathHeightY[k];
+							(unsigned int)locals->MinSwathHeightY[k];
 					locals->SwathHeightCThisState[k] =
-							locals->MinSwathHeightC[k];
+							(unsigned int)locals->MinSwathHeightC[k];
 				}
 			}
 
@@ -4607,11 +4607,11 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							locals->Read256BlockWidthC[k],
 							mode_lib->vba.SourcePixelFormat[k],
 							mode_lib->vba.SurfaceTiling[k],
-							dml_ceil(locals->BytePerPixelInDETC[k], 2.0),
+							(unsigned int)dml_ceil(locals->BytePerPixelInDETC[k], 2.0),
 							mode_lib->vba.SourceScan[k],
-							mode_lib->vba.ViewportWidth[k] / 2.0,
-							mode_lib->vba.ViewportHeight[k] / 2.0,
-							locals->SwathWidthYThisState[k] / 2.0,
+							(unsigned int)(mode_lib->vba.ViewportWidth[k] / 2.0),
+							(unsigned int)(mode_lib->vba.ViewportHeight[k] / 2.0),
+							(unsigned int)(locals->SwathWidthYThisState[k] / 2.0),
 							mode_lib->vba.GPUVMEnable,
 							mode_lib->vba.HostVMEnable,
 							mode_lib->vba.HostVMMaxPageTableLevels,
@@ -4619,7 +4619,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							mode_lib->vba.VMMPageSize,
 							mode_lib->vba.PTEBufferSizeInRequestsChroma,
 							mode_lib->vba.PitchC[k],
-							0.0,
+							0,
 							&locals->MacroTileWidthC[k],
 							&mode_lib->vba.MetaRowBytesC,
 							&mode_lib->vba.DPTEBytesPerRowC,
@@ -4650,8 +4650,8 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					locals->PTEBufferSizeInRequestsForLuma = mode_lib->vba.PTEBufferSizeInRequestsLuma;
 				} else {
 					mode_lib->vba.PDEAndMetaPTEBytesPerFrameC = 0.0;
-					mode_lib->vba.MetaRowBytesC = 0.0;
-					mode_lib->vba.DPTEBytesPerRowC = 0.0;
+					mode_lib->vba.MetaRowBytesC = (unsigned int)(0.0);
+					mode_lib->vba.DPTEBytesPerRowC = (unsigned int)(0.0);
 					locals->PrefetchLinesC[0][0][k] = 0.0;
 					locals->PTEBufferSizeNotExceededC[i][j][k] = true;
 					locals->PTEBufferSizeInRequestsForLuma = mode_lib->vba.PTEBufferSizeInRequestsLuma + mode_lib->vba.PTEBufferSizeInRequestsChroma;
@@ -4663,11 +4663,11 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						locals->Read256BlockWidthY[k],
 						mode_lib->vba.SourcePixelFormat[k],
 						mode_lib->vba.SurfaceTiling[k],
-						dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
+						(unsigned int)dml_ceil(locals->BytePerPixelInDETY[k], 1.0),
 						mode_lib->vba.SourceScan[k],
 						mode_lib->vba.ViewportWidth[k],
 						mode_lib->vba.ViewportHeight[k],
-						locals->SwathWidthYThisState[k],
+						(unsigned int)locals->SwathWidthYThisState[k],
 						mode_lib->vba.GPUVMEnable,
 						mode_lib->vba.HostVMEnable,
 						mode_lib->vba.HostVMMaxPageTableLevels,
@@ -4756,7 +4756,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 										mode_lib->vba.WritebackLumaVTaps[k],
 										mode_lib->vba.WritebackChromaHTaps[k],
 										mode_lib->vba.WritebackChromaVTaps[k],
-										mode_lib->vba.WritebackDestinationWidth[k]) / locals->RequiredDISPCLK[i][j];
+												(unsigned int)mode_lib->vba.WritebackDestinationWidth[k]) / locals->RequiredDISPCLK[i][j];
 					} else {
 						locals->WritebackDelay[i][k] = 0.0;
 					}
@@ -4773,7 +4773,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 													mode_lib->vba.WritebackLumaVTaps[m],
 													mode_lib->vba.WritebackChromaHTaps[m],
 													mode_lib->vba.WritebackChromaVTaps[m],
-													mode_lib->vba.WritebackDestinationWidth[m]) / locals->RequiredDISPCLK[i][j]);
+																(unsigned int)mode_lib->vba.WritebackDestinationWidth[m]) / locals->RequiredDISPCLK[i][j]);
 						}
 					}
 				}
@@ -4809,14 +4809,14 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				mode_lib->vba.MaximumReadBandwidthWithoutPrefetch = 0.0;
 				mode_lib->vba.MaximumReadBandwidthWithPrefetch = 0.0;
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
-					unsigned int m;
+					unsigned int cursor_idx;
 
 					locals->cursor_bw[k] = 0;
 					locals->cursor_bw_pre[k] = 0;
-					for (m = 0; m < mode_lib->vba.NumberOfCursors[k]; m++) {
-						locals->cursor_bw[k] = mode_lib->vba.CursorWidth[k][m] * mode_lib->vba.CursorBPP[k][m]
+					for (cursor_idx = 0; cursor_idx < mode_lib->vba.NumberOfCursors[k]; cursor_idx++) {
+						locals->cursor_bw[k] = mode_lib->vba.CursorWidth[k][cursor_idx] * mode_lib->vba.CursorBPP[k][cursor_idx]
 							/ 8.0 / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]) * mode_lib->vba.VRatio[k];
-						locals->cursor_bw_pre[k] = mode_lib->vba.CursorWidth[k][m] * mode_lib->vba.CursorBPP[k][m]
+						locals->cursor_bw_pre[k] = mode_lib->vba.CursorWidth[k][cursor_idx] * mode_lib->vba.CursorBPP[k][cursor_idx]
 							/ 8.0 / (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]) * locals->VRatioPreY[i][j][k];
 					}
 
@@ -4824,12 +4824,12 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							mode_lib->vba.DETBufferSizeInKByte[0],
 							locals->SwathHeightYThisState[k],
 							locals->SwathHeightCThisState[k],
-							locals->SwathWidthYThisState[k],
+							(unsigned int)locals->SwathWidthYThisState[k],
 							mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k],
 							mode_lib->vba.UrgentLatency,
 							mode_lib->vba.CursorBufferSize,
 							mode_lib->vba.CursorWidth[k][0] + mode_lib->vba.CursorWidth[k][1],
-							dml_max(mode_lib->vba.CursorBPP[k][0], mode_lib->vba.CursorBPP[k][1]),
+							(unsigned int)dml_max(mode_lib->vba.CursorBPP[k][0], mode_lib->vba.CursorBPP[k][1]),
 							mode_lib->vba.VRatio[k],
 							locals->VRatioPreY[i][j][k],
 							locals->VRatioPreC[i][j][k],
@@ -4922,10 +4922,10 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							+ locals->RequiredPrefetchPixelDataBWChroma[i][j][k] * locals->UrgentBurstFactorChromaPre[k]
 							+ locals->cursor_bw_pre[k] * locals->UrgentBurstFactorCursorPre[k]);
 				}
-				mode_lib->vba.TotImmediateFlipBytes = 0.0;
+				mode_lib->vba.TotImmediateFlipBytes = (unsigned int)(0.0);
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
-					mode_lib->vba.TotImmediateFlipBytes = mode_lib->vba.TotImmediateFlipBytes
-						+ locals->PDEAndMetaPTEBytesPerFrame[0][0][k] + locals->MetaRowBytes[0][0][k] + locals->DPTEBytesPerRow[0][0][k];
+					mode_lib->vba.TotImmediateFlipBytes = (unsigned int)(mode_lib->vba.TotImmediateFlipBytes
+							+ locals->PDEAndMetaPTEBytesPerFrame[0][0][k] + locals->MetaRowBytes[0][0][k] + locals->DPTEBytesPerRow[0][0][k]);
 				}
 
 				for (k = 0; k <= mode_lib->vba.NumberOfActivePlanes - 1; k++) {
@@ -5138,7 +5138,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	}
 	/*Mode Support, Voltage State and SOC Configuration*/
 
-	for (i = mode_lib->vba.soc.num_states; i >= 0; i--) {
+	for (idx = mode_lib->vba.soc.num_states; idx >= 0; idx--) {
 		for (j = 0; j < 2; j++) {
 			enum dm_validation_status status = DML_VALIDATION_OK;
 
@@ -5146,19 +5146,19 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				status = DML_FAIL_SCALE_RATIO_TAP;
 			} else if (!mode_lib->vba.SourceFormatPixelAndScanSupport) {
 				status = DML_FAIL_SOURCE_PIXEL_FORMAT;
-			} else if (!locals->ViewportSizeSupport[i][0]) {
+			} else if (!locals->ViewportSizeSupport[idx][0]) {
 				status = DML_FAIL_VIEWPORT_SIZE;
-			} else if (!locals->DIOSupport[i]) {
+			} else if (!locals->DIOSupport[idx]) {
 				status = DML_FAIL_DIO_SUPPORT;
-			} else if (locals->NotEnoughDSCUnits[i]) {
+			} else if (locals->NotEnoughDSCUnits[idx]) {
 				status = DML_FAIL_NOT_ENOUGH_DSC;
-			} else if (locals->DSCCLKRequiredMoreThanSupported[i]) {
+			} else if (locals->DSCCLKRequiredMoreThanSupported[idx]) {
 				status = DML_FAIL_DSC_CLK_REQUIRED;
-			} else if (!locals->ROBSupport[i][0]) {
+			} else if (!locals->ROBSupport[idx][0]) {
 				status = DML_FAIL_REORDERING_BUFFER;
-			} else if (!locals->DISPCLK_DPPCLK_Support[i][j]) {
+			} else if (!locals->DISPCLK_DPPCLK_Support[idx][j]) {
 				status = DML_FAIL_DISPCLK_DPPCLK;
-			} else if (!locals->TotalAvailablePipesSupport[i][j]) {
+			} else if (!locals->TotalAvailablePipesSupport[idx][j]) {
 				status = DML_FAIL_TOTAL_AVAILABLE_PIPES;
 			} else if (!mode_lib->vba.NumberOfOTGSupport) {
 				status = DML_FAIL_NUM_OTG;
@@ -5172,27 +5172,27 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				status = DML_FAIL_CURSOR_SUPPORT;
 			} else if (!mode_lib->vba.PitchSupport) {
 				status = DML_FAIL_PITCH_SUPPORT;
-			} else if (!locals->TotalVerticalActiveBandwidthSupport[i][0]) {
+			} else if (!locals->TotalVerticalActiveBandwidthSupport[idx][0]) {
 				status = DML_FAIL_TOTAL_V_ACTIVE_BW;
-			} else if (!locals->PTEBufferSizeNotExceeded[i][j]) {
+			} else if (!locals->PTEBufferSizeNotExceeded[idx][j]) {
 				status = DML_FAIL_PTE_BUFFER_SIZE;
 			} else if (mode_lib->vba.NonsupportedDSCInputBPC) {
 				status = DML_FAIL_DSC_INPUT_BPC;
 			} else if ((mode_lib->vba.HostVMEnable
-					&& !locals->ImmediateFlipSupportedForState[i][j])) {
+					&& !locals->ImmediateFlipSupportedForState[idx][j])) {
 				status = DML_FAIL_HOST_VM_IMMEDIATE_FLIP;
-			} else if (!locals->PrefetchSupported[i][j]) {
+			} else if (!locals->PrefetchSupported[idx][j]) {
 				status = DML_FAIL_PREFETCH_SUPPORT;
-			} else if (!locals->VRatioInPrefetchSupported[i][j]) {
+			} else if (!locals->VRatioInPrefetchSupported[idx][j]) {
 				status = DML_FAIL_V_RATIO_PREFETCH;
 			}
 
 			if (status == DML_VALIDATION_OK) {
-				locals->ModeSupport[i][j] = true;
+				locals->ModeSupport[idx][j] = true;
 			} else {
-				locals->ModeSupport[i][j] = false;
+				locals->ModeSupport[idx][j] = false;
 			}
-			locals->ValidationStatus[i] = status;
+			locals->ValidationStatus[idx] = status;
 		}
 	}
 	{
@@ -5257,7 +5257,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 		double UrgentOutOfOrderReturn,
 		double ReturnBW,
 		bool GPUVMEnable,
-		int dpte_group_bytes[],
+		unsigned int dpte_group_bytes[],
 		unsigned int MetaChunkSize,
 		double UrgentLatency,
 		double ExtraLatency,
@@ -5268,7 +5268,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 		double SRExitTime,
 		double SREnterPlusExitTime,
 		double DCFCLKDeepSleep,
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		bool DCCEnable[],
 		double DPPCLK[],
 		double SwathWidthSingleDPPY[],
@@ -5373,13 +5373,13 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 
-		mode_lib->vba.LBLatencyHidingSourceLinesY = dml_min((double) MaxLineBufferLines,
+		mode_lib->vba.LBLatencyHidingSourceLinesY = (unsigned int)(dml_min((double) MaxLineBufferLines,
 			dml_floor(LineBufferSize / LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(HRatio[k], 1.0)), 1))
-				- (vtaps[k] - 1);
+				- (vtaps[k] - 1));
 
-		mode_lib->vba.LBLatencyHidingSourceLinesC = dml_min((double) MaxLineBufferLines,
+		mode_lib->vba.LBLatencyHidingSourceLinesC = (unsigned int)(dml_min((double) MaxLineBufferLines,
 			dml_floor(LineBufferSize / LBBitPerPixel[k] / (SwathWidthY[k] / 2 / dml_max(HRatio[k] / 2, 1.0)), 1))
-				- (VTAPsChroma[k] - 1);
+				- (VTAPsChroma[k] - 1));
 
 		EffectiveLBLatencyHidingY = mode_lib->vba.LBLatencyHidingSourceLinesY / VRatio[k]
 				* (HTotal[k] / PixelClock[k]);
@@ -5412,12 +5412,12 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 				&DETBufferSizeC);
 
 		LinesInDETY[k] = (double)DETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k];
-		LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]);
+		LinesInDETYRoundedDownToSwath[k] = (unsigned int)(dml_floor(LinesInDETY[k], SwathHeightY[k]));
 		FullDETBufferingTimeY[k] = LinesInDETYRoundedDownToSwath[k]
 				* (HTotal[k] / PixelClock[k]) / VRatio[k];
 		if (BytePerPixelDETC[k] > 0) {
 			LinesInDETC = (double)DETBufferSizeC / BytePerPixelDETC[k] / (SwathWidthY[k] / 2.0);
-			LinesInDETCRoundedDownToSwath = dml_floor(LinesInDETC, SwathHeightC[k]);
+			LinesInDETCRoundedDownToSwath = (unsigned int)(dml_floor(LinesInDETC, SwathHeightC[k]));
 			FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath
 					* (HTotal[k] / PixelClock[k]) / (VRatio[k] / 2);
 		} else {
@@ -5543,7 +5543,7 @@ static void CalculateDCFCLKDeepSleep(
 		double BytePerPixelDETC[],
 		double VRatio[],
 		double SwathWidthY[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		double HRatio[],
 		double PixelClock[],
 		double PSCL_THROUGHPUT[],
@@ -5749,7 +5749,7 @@ static void CalculatePixelDeliveryTimes(
 		double VRatioPrefetchC[],
 		unsigned int swath_width_luma_ub[],
 		unsigned int swath_width_chroma_ub[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		double HRatio[],
 		double PixelClock[],
 		double PSCL_THROUGHPUT[],
@@ -5870,7 +5870,7 @@ static void CalculateMetaAndPTETimes(
 		unsigned int meta_row_height[],
 		unsigned int meta_req_width[],
 		unsigned int meta_req_height[],
-		int dpte_group_bytes[],
+		unsigned int dpte_group_bytes[],
 		unsigned int PTERequestSizeY[],
 		unsigned int PTERequestSizeC[],
 		unsigned int PixelPTEReqWidthY[],
@@ -5939,10 +5939,10 @@ static void CalculateMetaAndPTETimes(
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		if (DCCEnable[k] == true) {
-			meta_chunk_width = MetaChunkSize * 1024 * 256
-					/ dml_ceil(BytePerPixelDETY[k], 1) / meta_row_height[k];
-			min_meta_chunk_width = MinMetaChunkSizeBytes * 256
-					/ dml_ceil(BytePerPixelDETY[k], 1) / meta_row_height[k];
+			meta_chunk_width = (unsigned int)(MetaChunkSize * 1024 * 256
+					/ dml_ceil(BytePerPixelDETY[k], 1) / meta_row_height[k]);
+			min_meta_chunk_width = (unsigned int)(MinMetaChunkSizeBytes * 256
+					/ dml_ceil(BytePerPixelDETY[k], 1) / meta_row_height[k]);
 			meta_chunk_per_row_int = meta_row_width[k] / meta_chunk_width;
 			meta_row_remainder = meta_row_width[k] % meta_chunk_width;
 			if (SourceScan[k] == dm_horz) {
@@ -5978,7 +5978,7 @@ static void CalculateMetaAndPTETimes(
 				dpte_group_width_luma = dpte_group_bytes[k] / PTERequestSizeY[k]
 						* PixelPTEReqHeightY[k];
 			}
-			dpte_groups_per_row_luma_ub = dml_ceil(
+			dpte_groups_per_row_luma_ub = (unsigned int)dml_ceil(
 					(float) dpte_row_width_luma_ub[k] / dpte_group_width_luma,
 					1);
 			time_per_pte_group_nom_luma[k] = DST_Y_PER_PTE_ROW_NOM_L[k] * HTotal[k]
@@ -6002,7 +6002,7 @@ static void CalculateMetaAndPTETimes(
 							/ PTERequestSizeC[k]
 							* PixelPTEReqHeightC[k];
 				}
-				dpte_groups_per_row_chroma_ub = dml_ceil(
+				dpte_groups_per_row_chroma_ub = (unsigned int)dml_ceil(
 						(float) dpte_row_width_chroma_ub[k]
 								/ dpte_group_width_chroma,
 						1);
@@ -6032,34 +6032,32 @@ static void CalculateMetaAndPTETimes(
 		if (GPUVMEnable == true && (DCCEnable[k] == true || GPUVMMaxPageTableLevels > 1)) {
 			if (DCCEnable[k] == false) {
 				if (BytePerPixelDETC[k] > 0) {
-					num_group_per_lower_vm_stage =
+					num_group_per_lower_vm_stage = (unsigned int)(
 						dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-						+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+						+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 				} else {
-					num_group_per_lower_vm_stage =
-							dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+					num_group_per_lower_vm_stage = (unsigned int)dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
 				}
 			} else {
 				if (GPUVMMaxPageTableLevels == 1) {
 					if (BytePerPixelDETC[k] > 0) {
-						num_group_per_lower_vm_stage =
+						num_group_per_lower_vm_stage = (unsigned int)(
 							dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-							+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+							+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 					} else {
-						num_group_per_lower_vm_stage =
-							dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+						num_group_per_lower_vm_stage = (unsigned int)dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
 					}
 				} else {
 					if (BytePerPixelDETC[k] > 0) {
-						num_group_per_lower_vm_stage =
+						num_group_per_lower_vm_stage = (unsigned int)(
 							dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
 							+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1)
 							+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-							+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+							+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 					} else {
-						num_group_per_lower_vm_stage =
+						num_group_per_lower_vm_stage = (unsigned int)(
 							dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-							+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+							+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1));
 					}
 				}
 			}
@@ -6128,8 +6126,8 @@ static double CalculateExtraLatency(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
 		int HostVMMaxPageTableLevels,
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c
index 8a611b3bec331..189b0157fcf3c 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn21/display_rq_dlg_calc_21.c
@@ -174,19 +174,19 @@ static void extract_rq_regs(
 
 	extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_l), &rq_param->sizing.rq_l);
 
-	rq_regs->rq_regs_l.pte_row_height_linear = dml_floor(
+	rq_regs->rq_regs_l.pte_row_height_linear = (unsigned int)(dml_floor(
 			dml_log2(rq_param->dlg.rq_l.dpte_row_height),
-			1) - 3;
+			1) - 3);
 
 	if (rq_param->yuv420) {
 		extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_c), &rq_param->sizing.rq_c);
-		rq_regs->rq_regs_c.pte_row_height_linear = dml_floor(
+		rq_regs->rq_regs_c.pte_row_height_linear = (unsigned int)(dml_floor(
 				dml_log2(rq_param->dlg.rq_c.dpte_row_height),
-				1) - 3;
+				1) - 3);
 	}
 
-	rq_regs->rq_regs_l.swath_height = dml_log2(rq_param->dlg.rq_l.swath_height);
-	rq_regs->rq_regs_c.swath_height = dml_log2(rq_param->dlg.rq_c.swath_height);
+	rq_regs->rq_regs_l.swath_height = (unsigned int)dml_log2(rq_param->dlg.rq_l.swath_height);
+	rq_regs->rq_regs_c.swath_height = (unsigned int)dml_log2(rq_param->dlg.rq_c.swath_height);
 
 	// FIXME: take the max between luma, chroma chunk size?
 	// okay for now, as we are setting chunk_bytes to 8kb anyways
@@ -202,12 +202,12 @@ static void extract_rq_regs(
 	if (rq_param->yuv420) {
 		if ((double) rq_param->misc.rq_l.stored_swath_bytes
 				/ (double) rq_param->misc.rq_c.stored_swath_bytes <= 1.5) {
-			detile_buf_plane1_addr = (detile_buf_size_in_bytes / 2.0 / 64.0); // half to chroma
+			detile_buf_plane1_addr = (unsigned int)(detile_buf_size_in_bytes / 2.0 / 64.0); // half to chroma
 		} else {
-			detile_buf_plane1_addr = dml_round_to_multiple(
+			detile_buf_plane1_addr = (unsigned int)(dml_round_to_multiple(
 					(unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0),
 					256,
-					0) / 64.0; // 2/3 to chroma
+							0) / 64.0); // 2/3 to chroma
 		}
 	}
 	rq_regs->plane1_base_address = detile_buf_plane1_addr;
@@ -443,8 +443,8 @@ static void get_meta_and_pte_attr(
 	// "/2" is like square root
 	// blk is vertical biased
 	if (tiling != dm_sw_linear)
-		log2_blk_height = log2_blk256_height
-				+ dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
+		log2_blk_height = (unsigned int)(log2_blk256_height
+				+ dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1));
 	else
 		log2_blk_height = 0;  // blk height of 1
 
@@ -599,7 +599,7 @@ static void get_meta_and_pte_attr(
 	if (surf_linear) {
 		unsigned int dpte_row_height;
 
-		log2_dpte_row_height_linear = dml_floor(
+		log2_dpte_row_height_linear = (unsigned int)dml_floor(
 				dml_log2(
 						dml_min(
 								64 * 1024 * pde_buf_entries
@@ -678,7 +678,7 @@ static void get_meta_and_pte_attr(
 
 	// since dpte groups are only aligned to dpte_req_width and not dpte_group_width,
 	// the upper bound for the dpte groups per row is as follows.
-	rq_dlg_param->dpte_groups_per_row_ub = dml_ceil(
+	rq_dlg_param->dpte_groups_per_row_ub = (unsigned int)dml_ceil(
 			(double) dpte_row_width_ub / dpte_group_width,
 			1);
 }
@@ -718,18 +718,18 @@ static void get_surf_rq_param(
 		access_dir = (pipe_param->src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
 		hactive_half  = pipe_param->dest.hactive / 2;
 		if (is_chroma) {
-			full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width;
-			src_hactive_half  = pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_half;
+			full_src_vp_width = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width);
+			src_hactive_half  = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_half);
 		} else {
-			full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width;
-			src_hactive_half  = pipe_param->scale_ratio_depth.hscl_ratio * hactive_half;
+			full_src_vp_width = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width);
+			src_hactive_half  = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio * hactive_half);
 		}
 
 		if (access_dir == 0) {
-			vp_width = dml_min(full_src_vp_width, src_hactive_half);
+			vp_width = (unsigned int)dml_min(full_src_vp_width, src_hactive_half);
 			dml_print("DML_DLG: %s: vp_width = %d\n", __func__, vp_width);
 		} else {
-			vp_height = dml_min(full_src_vp_width, src_hactive_half);
+			vp_height = (unsigned int)dml_min(full_src_vp_width, src_hactive_half);
 			dml_print("DML_DLG: %s: vp_height = %d\n", __func__, vp_height);
 
 		}
@@ -1058,11 +1058,11 @@ static void dml_rq_dlg_get_dlg_params(
 	if (dout->dsc_enable) {
 		double dsc_delay = get_dsc_delay(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
 
-		dispclk_delay_subtotal += dsc_delay;
+		dispclk_delay_subtotal += (unsigned int)dsc_delay;
 	}
 
-	pixel_rate_delay_subtotal = dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
-			+ dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz;
+	pixel_rate_delay_subtotal = (unsigned int)(dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
+			+ dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz);
 
 	vstartup_start = dst->vstartup_start;
 	if (interlaced) {
@@ -1105,8 +1105,8 @@ static void dml_rq_dlg_get_dlg_params(
 				min_vblank);
 	}
 
-	dst_x_after_scaler = get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
-	dst_y_after_scaler = get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+	dst_x_after_scaler = (unsigned int)get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+	dst_y_after_scaler = (unsigned int)get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
 
 	dml_print("DML_DLG: %s: htotal                                 = %d\n", __func__, htotal);
 	dml_print(
@@ -1446,7 +1446,7 @@ static void dml_rq_dlg_get_dlg_params(
 
 	// Assignment to register structures
 	disp_dlg_regs->dst_y_after_scaler = dst_y_after_scaler; // in terms of line
-	disp_dlg_regs->refcyc_x_after_scaler = dst_x_after_scaler * ref_freq_to_pix_freq; // in terms of refclk
+	disp_dlg_regs->refcyc_x_after_scaler = (unsigned int)(dst_x_after_scaler * ref_freq_to_pix_freq); // in terms of refclk
 	ASSERT(disp_dlg_regs->refcyc_x_after_scaler < (unsigned int)dml_pow(2, 13));
 	disp_dlg_regs->dst_y_prefetch = (unsigned int) (dst_y_prefetch * dml_pow(2, 2));
 	disp_dlg_regs->dst_y_per_vm_vblank = (unsigned int) (dst_y_per_vm_vblank * dml_pow(2, 2));
@@ -1506,23 +1506,23 @@ static void dml_rq_dlg_get_dlg_params(
 				* htotal * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_c;
 	}
 
-	disp_dlg_regs->refcyc_per_vm_group_vblank   = get_refcyc_per_vm_group_vblank(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz;
-	disp_dlg_regs->refcyc_per_vm_group_flip     = get_refcyc_per_vm_group_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz;
-	disp_dlg_regs->refcyc_per_vm_req_vblank     = get_refcyc_per_vm_req_vblank(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10);
-	disp_dlg_regs->refcyc_per_vm_req_flip       = get_refcyc_per_vm_req_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10);
+	disp_dlg_regs->refcyc_per_vm_group_vblank   = (unsigned int)(get_refcyc_per_vm_group_vblank(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz);
+	disp_dlg_regs->refcyc_per_vm_group_flip     = (unsigned int)(get_refcyc_per_vm_group_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz);
+	disp_dlg_regs->refcyc_per_vm_req_vblank     = (unsigned int)(get_refcyc_per_vm_req_vblank(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10));
+	disp_dlg_regs->refcyc_per_vm_req_flip       = (unsigned int)(get_refcyc_per_vm_req_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10));
 
 	// Clamp to max for now
 	if (disp_dlg_regs->refcyc_per_vm_group_vblank >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_group_vblank = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_group_vblank = (unsigned int)dml_pow(2, 23) - 1;
 
 	if (disp_dlg_regs->refcyc_per_vm_group_flip >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_group_flip = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_group_flip = (unsigned int)dml_pow(2, 23) - 1;
 
 	if (disp_dlg_regs->refcyc_per_vm_req_vblank >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_req_vblank = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_req_vblank = (unsigned int)dml_pow(2, 23) - 1;
 
 	if (disp_dlg_regs->refcyc_per_vm_req_flip >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_req_flip = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_req_flip = (unsigned int)dml_pow(2, 23) - 1;
 	disp_dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int) ((double) dpte_row_height_l
 			/ (double) vratio_l * dml_pow(2, 2));
 	ASSERT(disp_dlg_regs->dst_y_per_pte_row_nom_l < (unsigned int)dml_pow(2, 17));
@@ -1553,12 +1553,12 @@ static void dml_rq_dlg_get_dlg_params(
 			/ (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
 			/ (double) dpte_groups_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_pte_group_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_pte_group_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)dml_pow(2, 23) - 1;
 	disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int) ((double) meta_row_height_l
 			/ (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
 			/ (double) meta_chunks_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_meta_chunk_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)dml_pow(2, 23) - 1;
 
 	if (dual_plane) {
 		disp_dlg_regs->refcyc_per_pte_group_nom_c =
@@ -1566,7 +1566,7 @@ static void dml_rq_dlg_get_dlg_params(
 						* (double) htotal * ref_freq_to_pix_freq
 						/ (double) dpte_groups_per_row_ub_c);
 		if (disp_dlg_regs->refcyc_per_pte_group_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_pte_group_nom_c = dml_pow(2, 23) - 1;
+			disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int)dml_pow(2, 23) - 1;
 
 		// TODO: Is this the right calculation? Does htotal need to be halved?
 		disp_dlg_regs->refcyc_per_meta_chunk_nom_c =
@@ -1574,7 +1574,7 @@ static void dml_rq_dlg_get_dlg_params(
 						* (double) htotal * ref_freq_to_pix_freq
 						/ (double) meta_chunks_per_row_ub_c);
 		if (disp_dlg_regs->refcyc_per_meta_chunk_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = dml_pow(2, 23) - 1;
+			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int)dml_pow(2, 23) - 1;
 	}
 
 	disp_dlg_regs->refcyc_per_line_delivery_pre_l = (unsigned int) dml_floor(
@@ -1628,7 +1628,7 @@ static void dml_rq_dlg_get_dlg_params(
 	disp_ttu_regs->qos_ramp_disable_c = 0;
 	disp_ttu_regs->qos_ramp_disable_cur0 = 0;
 
-	disp_ttu_regs->min_ttu_vblank = min_ttu_vblank * refclk_freq_in_mhz;
+	disp_ttu_regs->min_ttu_vblank = (unsigned int)(min_ttu_vblank * refclk_freq_in_mhz);
 	ASSERT(disp_ttu_regs->min_ttu_vblank < dml_pow(2, 24));
 
 	print__ttu_regs_st(mode_lib, disp_ttu_regs);
@@ -1668,7 +1668,7 @@ void dml21_rq_dlg_get_dlg_reg(
 			mode_lib,
 			e2e_pipe_param,
 			num_pipes);
-	dlg_sys_param.total_flip_bytes = get_total_immediate_flip_bytes(
+	dlg_sys_param.total_flip_bytes = (unsigned int)get_total_immediate_flip_bytes(
 			mode_lib,
 			e2e_pipe_param,
 			num_pipes);
@@ -1734,7 +1734,7 @@ static void calculate_ttu_cursor(
 				cur_req_size = 256;
 		}
 
-		cur_req_width = (double) cur_req_size / ((double) cur_bit_per_pixel / 8.0);
+		cur_req_width = (unsigned int)((double) cur_req_size / ((double) cur_bit_per_pixel / 8.0));
 		cur_width_ub = dml_ceil((double) cur_src_width / (double) cur_req_width, 1)
 				* (double) cur_req_width;
 		cur_req_per_width = cur_width_ub / (double) cur_req_width;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
index 0cdd60869ce1c..0ba388c6aec1a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/dcn30_fpu.c
@@ -181,7 +181,8 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_0_soc = {
 void dcn30_fpu_populate_dml_writeback_from_context(
 		struct dc *dc, struct resource_context *res_ctx, display_e2e_pipe_params_st *pipes)
 {
-	int pipe_cnt, i, j;
+	int pipe_cnt;
+	unsigned int i, j;
 	double max_calc_writeback_dispclk;
 	double writeback_dispclk;
 	struct writeback_st dout_wb = {0};
@@ -279,11 +280,11 @@ void dcn30_fpu_set_mcif_arb_params(struct mcif_arb_params *wb_arb_params,
 	dc_assert_fp_enabled();
 
 	for (i = 0; i < ARRAY_SIZE(wb_arb_params->cli_watermark); i++) {
-		wb_arb_params->cli_watermark[i] = get_wm_writeback_urgent(dml, pipes, pipe_cnt) * 1000;
-		wb_arb_params->pstate_watermark[i] = get_wm_writeback_dram_clock_change(dml, pipes, pipe_cnt) * 1000;
+		wb_arb_params->cli_watermark[i] = (unsigned int)(get_wm_writeback_urgent(dml, pipes, pipe_cnt) * 1000);
+		wb_arb_params->pstate_watermark[i] = (unsigned int)(get_wm_writeback_dram_clock_change(dml, pipes, pipe_cnt) * 1000);
 	}
 
-	wb_arb_params->dram_speed_change_duration = dml->vba.WritebackAllowDRAMClockChangeEndPosition[cur_pipe] * pipes[0].clks_cfg.refclk_mhz; /* num_clock_cycles = us * MHz */
+	wb_arb_params->dram_speed_change_duration = (unsigned int)(dml->vba.WritebackAllowDRAMClockChangeEndPosition[cur_pipe] * pipes[0].clks_cfg.refclk_mhz); /* num_clock_cycles = us * MHz */
 }
 
 void dcn30_fpu_update_soc_for_wm_a(struct dc *dc, struct dc_state *context)
@@ -308,6 +309,7 @@ void dcn30_fpu_calculate_wm_and_dlg(
 {
 	int maxMpcComb = context->bw_ctx.dml.vba.maxMpcComb;
 	int i, pipe_idx;
+	unsigned int pipe_i, state_i;
 	double dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][maxMpcComb];
 	bool pstate_en = context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][maxMpcComb] != dm_dram_clock_change_unsupported;
 	unsigned int dummy_latency_index = 0;
@@ -367,14 +369,14 @@ void dcn30_fpu_calculate_wm_and_dlg(
 		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_enter_plus_exit_time_us;
 		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_exit_time_us;
 	}
-	context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.urgent_latency_ns = (uint32_t)(get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 
 	pipes[0].clks_cfg.voltage = vlevel;
 	pipes[0].clks_cfg.dcfclk_mhz = dcfclk;
@@ -406,7 +408,7 @@ void dcn30_fpu_calculate_wm_and_dlg(
 	 * pstate latency overridden to 5us
 	 */
 	if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid) {
-		unsigned int min_dram_speed_mts = context->bw_ctx.dml.vba.DRAMSpeed;
+		unsigned int min_dram_speed_mts = (unsigned int)context->bw_ctx.dml.vba.DRAMSpeed;
 		unsigned int min_dram_speed_mts_margin = 160;
 
 		context->bw_ctx.dml.soc.dram_clock_change_latency_us =
@@ -437,14 +439,14 @@ void dcn30_fpu_calculate_wm_and_dlg(
 		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_exit_time_us;
 	}
 
-	context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.urgent_latency_ns = (uint32_t)(get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 
 	if (!pstate_en) {
 		/* The only difference between A and C is p-state latency, if p-state is not supported we want to
@@ -460,23 +462,23 @@ void dcn30_fpu_calculate_wm_and_dlg(
 		 * Set A calculated last so that following calculations are based on Set A
 		 */
 		dc->res_pool->funcs->update_soc_for_wm_a(dc, context);
-		context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+		context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.urgent_latency_ns = (uint32_t)(get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 	}
 
-	context->perf_params.stutter_period_us = context->bw_ctx.dml.vba.StutterPeriod;
+	context->perf_params.stutter_period_us = (unsigned int)context->bw_ctx.dml.vba.StutterPeriod;
 
 	/* Make set D = set A until set D is enabled */
 	context->bw_ctx.bw.dcn.watermarks.d = context->bw_ctx.bw.dcn.watermarks.a;
 
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
-		if (!context->res_ctx.pipe_ctx[i].stream)
+	for (pipe_i = 0, pipe_idx = 0; pipe_i < dc->res_pool->pipe_count; pipe_i++) {
+		if (!context->res_ctx.pipe_ctx[pipe_i].stream)
 			continue;
 
 		pipes[pipe_idx].clks_cfg.dispclk_mhz = get_dispclk_calculated(&context->bw_ctx.dml, pipes, pipe_cnt);
@@ -500,9 +502,9 @@ void dcn30_fpu_calculate_wm_and_dlg(
 			context->bw_ctx.dml.vba.DRAMSpeed <= 1700 &&
 			context->bw_ctx.dml.vba.DRAMSpeed >= 1500) {
 
-		for (i = 0; i < dc->dml.soc.num_states; i++) {
-			if (dc->dml.soc.clock_limits[i].dram_speed_mts > 1700) {
-				context->bw_ctx.dml.vba.DRAMSpeed = dc->dml.soc.clock_limits[i].dram_speed_mts;
+		for (state_i = 0; state_i < dc->dml.soc.num_states; state_i++) {
+			if (dc->dml.soc.clock_limits[state_i].dram_speed_mts > 1700) {
+				context->bw_ctx.dml.vba.DRAMSpeed = dc->dml.soc.clock_limits[state_i].dram_speed_mts;
 				break;
 			}
 		}
@@ -532,13 +534,13 @@ void dcn30_fpu_update_max_clk(struct dc_bounding_box_max_clk *dcn30_bb_max_clk)
 		dc_assert_fp_enabled();
 
 		if (!dcn30_bb_max_clk->max_dcfclk_mhz)
-			dcn30_bb_max_clk->max_dcfclk_mhz = dcn3_0_soc.clock_limits[0].dcfclk_mhz;
+			dcn30_bb_max_clk->max_dcfclk_mhz = (int)dcn3_0_soc.clock_limits[0].dcfclk_mhz;
 		if (!dcn30_bb_max_clk->max_dispclk_mhz)
-			dcn30_bb_max_clk->max_dispclk_mhz = dcn3_0_soc.clock_limits[0].dispclk_mhz;
+			dcn30_bb_max_clk->max_dispclk_mhz = (int)dcn3_0_soc.clock_limits[0].dispclk_mhz;
 		if (!dcn30_bb_max_clk->max_dppclk_mhz)
-			dcn30_bb_max_clk->max_dppclk_mhz = dcn3_0_soc.clock_limits[0].dppclk_mhz;
+			dcn30_bb_max_clk->max_dppclk_mhz = (int)dcn3_0_soc.clock_limits[0].dppclk_mhz;
 		if (!dcn30_bb_max_clk->max_phyclk_mhz)
-			dcn30_bb_max_clk->max_phyclk_mhz = dcn3_0_soc.clock_limits[0].phyclk_mhz;
+			dcn30_bb_max_clk->max_phyclk_mhz = (int)dcn3_0_soc.clock_limits[0].phyclk_mhz;
 }
 
 void dcn30_fpu_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts,
@@ -557,12 +559,12 @@ void dcn30_fpu_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts,
 	bw_from_dram = (bw_from_dram1 < bw_from_dram2) ? bw_from_dram1 : bw_from_dram2;
 
 	if (optimal_fclk)
-		*optimal_fclk = bw_from_dram /
-		(dcn3_0_soc.fabric_datapath_to_dcn_data_return_bytes * (dcn3_0_soc.max_avg_sdp_bw_use_normal_percent / 100));
+		*optimal_fclk = (unsigned int)(bw_from_dram /
+		(dcn3_0_soc.fabric_datapath_to_dcn_data_return_bytes * (dcn3_0_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 
 	if (optimal_dcfclk)
-		*optimal_dcfclk =  bw_from_dram /
-		(dcn3_0_soc.return_bus_width_bytes * (dcn3_0_soc.max_avg_sdp_bw_use_normal_percent / 100));
+		*optimal_dcfclk = (unsigned int)(bw_from_dram /
+		(dcn3_0_soc.return_bus_width_bytes * (dcn3_0_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 }
 
 void dcn30_fpu_update_bw_bounding_box(struct dc *dc,
@@ -662,7 +664,7 @@ void dcn3_fpu_build_wm_range_table(struct clk_mgr *base)
 	double pstate_latency_us = base->ctx->dc->dml.soc.dram_clock_change_latency_us;
 	double sr_exit_time_us = base->ctx->dc->dml.soc.sr_exit_time_us;
 	double sr_enter_plus_exit_time_us = base->ctx->dc->dml.soc.sr_enter_plus_exit_time_us;
-	uint16_t min_uclk_mhz = base->bw_params->clk_table.entries[0].memclk_mhz;
+	uint16_t min_uclk_mhz = (uint16_t)base->bw_params->clk_table.entries[0].memclk_mhz;
 
 	dc_assert_fp_enabled();
 
@@ -719,9 +721,9 @@ void dcn3_fpu_build_wm_range_table(struct clk_mgr *base)
 	base->bw_params->wm_table.nv_entries[WM_D].pmfw_breakdown.max_uclk = 0xFFFF;
 }
 
-void patch_dcn30_soc_bounding_box(struct dc *dc, struct _vcs_dpi_soc_bounding_box_st *dcn3_0_ip)
+void patch_dcn30_soc_bounding_box(struct dc *dc, struct _vcs_dpi_soc_bounding_box_st *soc_bb)
 {
-	(void)dcn3_0_ip;
+	(void)soc_bb;
 	dc_assert_fp_enabled();
 
 	if (dc->ctx->dc_bios->funcs->get_soc_bb_info) {
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
index 634982173190e..f0b1bfb408f65 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c
@@ -285,8 +285,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 static void CalculateDCFCLKDeepSleep(
 		struct display_mode_lib *mode_lib,
 		unsigned int NumberOfActivePlanes,
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		double VRatio[],
 		double VRatioChroma[],
 		double SwathWidthY[],
@@ -327,7 +327,7 @@ static void CalculateUrgentBurstFactor(
 static void UseMinimumDCFCLK(
 		struct display_mode_lib *mode_lib,
 		struct vba_vars_st *v,
-		int MaxPrefetchMode,
+		unsigned int MaxPrefetchMode,
 		int ReorderingBytes);
 
 static void CalculatePixelDeliveryTimes(
@@ -345,7 +345,7 @@ static void CalculatePixelDeliveryTimes(
 		double PSCL_THROUGHPUT[],
 		double PSCL_THROUGHPUT_CHROMA[],
 		double DPPCLK[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
 		unsigned int NumberOfCursors[],
 		unsigned int CursorWidth[][2],
@@ -366,39 +366,39 @@ static void CalculatePixelDeliveryTimes(
 		double CursorRequestDeliveryTimePrefetch[]);
 
 static void CalculateMetaAndPTETimes(
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		bool GPUVMEnable,
 		int MetaChunkSize,
 		int MinMetaChunkSizeBytes,
-		int HTotal[],
+		unsigned int HTotal[],
 		double VRatio[],
 		double VRatioChroma[],
 		double DestinationLinesToRequestRowInVBlank[],
 		double DestinationLinesToRequestRowInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
-		int dpte_row_height[],
-		int dpte_row_height_chroma[],
-		int meta_row_width[],
-		int meta_row_width_chroma[],
-		int meta_row_height[],
-		int meta_row_height_chroma[],
-		int meta_req_width[],
-		int meta_req_width_chroma[],
-		int meta_req_height[],
-		int meta_req_height_chroma[],
-		int dpte_group_bytes[],
-		int PTERequestSizeY[],
-		int PTERequestSizeC[],
-		int PixelPTEReqWidthY[],
-		int PixelPTEReqHeightY[],
-		int PixelPTEReqWidthC[],
-		int PixelPTEReqHeightC[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
+		unsigned int dpte_row_height[],
+		unsigned int dpte_row_height_chroma[],
+		unsigned int meta_row_width[],
+		unsigned int meta_row_width_chroma[],
+		unsigned int meta_row_height[],
+		unsigned int meta_row_height_chroma[],
+		unsigned int meta_req_width[],
+		unsigned int meta_req_width_chroma[],
+		unsigned int meta_req_height[],
+		unsigned int meta_req_height_chroma[],
+		unsigned int dpte_group_bytes[],
+		unsigned int PTERequestSizeY[],
+		unsigned int PTERequestSizeC[],
+		unsigned int PixelPTEReqWidthY[],
+		unsigned int PixelPTEReqHeightY[],
+		unsigned int PixelPTEReqWidthC[],
+		unsigned int PixelPTEReqHeightC[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
 		double DST_Y_PER_PTE_ROW_NOM_L[],
 		double DST_Y_PER_PTE_ROW_NOM_C[],
 		double DST_Y_PER_META_ROW_NOM_L[],
@@ -421,52 +421,52 @@ static void CalculateVMGroupAndRequestTimes(
 		bool GPUVMEnable,
 		unsigned int GPUVMMaxPageTableLevels,
 		unsigned int HTotal[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		double DestinationLinesToRequestVMInVBlank[],
 		double DestinationLinesToRequestVMInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
-		int vm_group_bytes[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
+		unsigned int vm_group_bytes[],
 		unsigned int dpde0_bytes_per_frame_ub_l[],
 		unsigned int dpde0_bytes_per_frame_ub_c[],
-		int meta_pte_bytes_per_frame_ub_l[],
-		int meta_pte_bytes_per_frame_ub_c[],
+		unsigned int meta_pte_bytes_per_frame_ub_l[],
+		unsigned int meta_pte_bytes_per_frame_ub_c[],
 		double TimePerVMGroupVBlank[],
 		double TimePerVMGroupFlip[],
 		double TimePerVMRequestVBlank[],
 		double TimePerVMRequestFlip[]);
 
 static void CalculateStutterEfficiency(
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		long ROBBufferSizeInKByte,
 		double TotalDataReadBandwidth,
 		double DCFCLK,
 		double ReturnBW,
 		double SRExitTime,
 		bool SynchronizedVBlank,
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		unsigned int DETBufferSizeY[],
-		int BytePerPixelY[],
+		unsigned int BytePerPixelY[],
 		double BytePerPixelDETY[],
 		double SwathWidthY[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		double DCCRateLuma[],
 		double DCCRateChroma[],
-		int HTotal[],
-		int VTotal[],
+		unsigned int HTotal[],
+		unsigned int VTotal[],
 		double PixelClock[],
 		double VRatio[],
 		enum scan_direction_class SourceScan[],
-		int BlockHeight256BytesY[],
-		int BlockWidth256BytesY[],
-		int BlockHeight256BytesC[],
-		int BlockWidth256BytesC[],
-		int DCCYMaxUncompressedBlock[],
-		int DCCCMaxUncompressedBlock[],
-		int VActive[],
+		unsigned int BlockHeight256BytesY[],
+		unsigned int BlockWidth256BytesY[],
+		unsigned int BlockHeight256BytesC[],
+		unsigned int BlockWidth256BytesC[],
+		unsigned int DCCYMaxUncompressedBlock[],
+		unsigned int DCCCMaxUncompressedBlock[],
+		unsigned int VActive[],
 		bool DCCEnable[],
 		bool WritebackEnable[],
 		double ReadBandwidthPlaneLuma[],
@@ -479,46 +479,46 @@ static void CalculateStutterEfficiency(
 
 static void CalculateSwathAndDETConfiguration(
 		bool ForceSingleDPP,
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		unsigned int DETBufferSizeInKByte,
 		double MaximumSwathWidthLuma[],
 		double MaximumSwathWidthChroma[],
 		enum scan_direction_class SourceScan[],
 		enum source_format_class SourcePixelFormat[],
 		enum dm_swizzle_mode SurfaceTiling[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BlendingAndTiming[],
-		int BytePerPixY[],
-		int BytePerPixC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
 		double BytePerPixDETY[],
 		double BytePerPixDETC[],
-		int HActive[],
+		unsigned int HActive[],
 		double HRatio[],
 		double HRatioChroma[],
-		int DPPPerPlane[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[],
+		unsigned int DPPPerPlane[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[],
 		double SwathWidth[],
 		double SwathWidthChroma[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		unsigned int DETBufferSizeY[],
 		unsigned int DETBufferSizeC[],
 		bool ViewportSizeSupportPerPlane[],
 		bool *ViewportSizeSupport);
 static void CalculateSwathWidth(
 		bool ForceSingleDPP,
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		enum source_format_class SourcePixelFormat[],
 		enum scan_direction_class SourceScan[],
 		unsigned int ViewportWidth[],
@@ -528,22 +528,22 @@ static void CalculateSwathWidth(
 		unsigned int SurfaceHeightY[],
 		unsigned int SurfaceHeightC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BytePerPixY[],
-		int BytePerPixC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
-		int BlendingAndTiming[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
+		unsigned int BlendingAndTiming[],
 		unsigned int HActive[],
 		double HRatio[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		double SwathWidthSingleDPPY[],
 		double SwathWidthSingleDPPC[],
 		double SwathWidthY[],
 		double SwathWidthC[],
-		int MaximumSwathHeightY[],
-		int MaximumSwathHeightC[],
+		unsigned int MaximumSwathHeightY[],
+		unsigned int MaximumSwathHeightC[],
 		unsigned int swath_width_luma_ub[],
 		unsigned int swath_width_chroma_ub[]);
 static double CalculateExtraLatency(
@@ -558,8 +558,8 @@ static double CalculateExtraLatency(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
 		double HostVMMinPageSize,
@@ -572,9 +572,9 @@ static double CalculateExtraLatencyBytes(
 		int MetaChunkSize,
 		bool GPUVMEnable,
 		bool HostVMEnable,
-		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfActivePlanes,
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
 		double HostVMMinPageSize,
@@ -630,7 +630,7 @@ static unsigned int dscceComputeDelay(
 		pixelsPerClock = 1;
 
 	//initial transmit delay as per PPS
-	initalXmitDelay = dml_round(rcModelSize / 2.0 / BPP / pixelsPerClock);
+	initalXmitDelay = (unsigned int)dml_round(rcModelSize / 2.0 / BPP / pixelsPerClock);
 
 	//compute ssm delay
 	if (bpc == 8)
@@ -866,13 +866,13 @@ static bool CalculatePrefetchSchedule(
 	v->Tdmdl_vm[k] = (v->DynamicMetadataEnable[k] == true && v->DynamicMetadataVMEnabled == true && v->GPUVMEnable == true ? TWait + Tvm_trips : 0);
 
 	if (myPipe->ScalerEnabled)
-		DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCL;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCL);
 	else
-		DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCLLBOnly;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCLLBOnly);
 
-	DPPCycles = DPPCycles + myPipe->NumberOfCursors * v->DPPCLKDelayCNVCCursor;
+	DPPCycles = (unsigned int)(DPPCycles + myPipe->NumberOfCursors * v->DPPCLKDelayCNVCCursor);
 
-	DISPCLKCycles = v->DISPCLKDelaySubtotal;
+	DISPCLKCycles = (unsigned int)v->DISPCLKDelaySubtotal;
 
 	if (myPipe->DPPCLK == 0.0 || myPipe->DISPCLK == 0.0)
 		return true;
@@ -1302,14 +1302,14 @@ static void CalculateDCCConfiguration(
 		horz_div_c = 0;
 
 	if (BytePerPixelC == 0) {
-		swath_buf_size = DETBufferSize / 2 - 2 * 256;
+		swath_buf_size = (long)(DETBufferSize / 2 - 2 * 256);
 		detile_buf_vp_horz_limit = (double) swath_buf_size
 				/ ((double) RequestHeight256ByteLuma * BytePerPixelY
 						/ (1 + horz_div_l));
 		detile_buf_vp_vert_limit = (double) swath_buf_size
 				/ (256.0 / RequestHeight256ByteLuma / (1 + vert_div_l));
 	} else {
-		swath_buf_size = DETBufferSize / 2 - 2 * 2 * 256;
+		swath_buf_size = (long)(DETBufferSize / 2 - 2 * 2 * 256);
 		detile_buf_vp_horz_limit = (double) swath_buf_size
 				/ ((double) RequestHeight256ByteLuma * BytePerPixelY
 						/ (1 + horz_div_l)
@@ -1332,14 +1332,15 @@ static void CalculateDCCConfiguration(
 
 	MAS_vp_horz_limit = 5760;
 	MAS_vp_vert_limit = (BytePerPixelC > 0 ? 2880 : 5760);
-	max_vp_horz_width = dml_min((double) MAS_vp_horz_limit, detile_buf_vp_horz_limit);
-	max_vp_vert_height = dml_min((double) MAS_vp_vert_limit, detile_buf_vp_vert_limit);
+	max_vp_horz_width = (long)dml_min((double) MAS_vp_horz_limit, detile_buf_vp_horz_limit);
+	max_vp_vert_height = (long)dml_min((double) MAS_vp_vert_limit, detile_buf_vp_vert_limit);
 	eff_surf_width_l =
-			(SurfaceWidthLuma > max_vp_horz_width ? max_vp_horz_width : SurfaceWidthLuma);
+			(SurfaceWidthLuma > (unsigned long)max_vp_horz_width ?
+			(unsigned long)max_vp_horz_width : SurfaceWidthLuma);
 	eff_surf_width_c = eff_surf_width_l / (1 + yuv420);
 	eff_surf_height_l = (
-			SurfaceHeightLuma > max_vp_vert_height ?
-					max_vp_vert_height : SurfaceHeightLuma);
+			SurfaceHeightLuma > (unsigned long)max_vp_vert_height ?
+					(unsigned long)max_vp_vert_height : SurfaceHeightLuma);
 	eff_surf_height_c = eff_surf_height_l / (1 + yuv420);
 
 	full_swath_bytes_horz_wc_l = eff_surf_width_l * RequestHeight256ByteLuma * BytePerPixelY;
@@ -1354,10 +1355,10 @@ static void CalculateDCCConfiguration(
 	}
 
 	if (SourcePixelFormat == dm_420_10) {
-		full_swath_bytes_horz_wc_l = dml_ceil(full_swath_bytes_horz_wc_l * 2 / 3, 256);
-		full_swath_bytes_horz_wc_c = dml_ceil(full_swath_bytes_horz_wc_c * 2 / 3, 256);
-		full_swath_bytes_vert_wc_l = dml_ceil(full_swath_bytes_vert_wc_l * 2 / 3, 256);
-		full_swath_bytes_vert_wc_c = dml_ceil(full_swath_bytes_vert_wc_c * 2 / 3, 256);
+		full_swath_bytes_horz_wc_l = (long)dml_ceil(full_swath_bytes_horz_wc_l * 2 / 3, 256);
+		full_swath_bytes_horz_wc_c = (long)dml_ceil(full_swath_bytes_horz_wc_c * 2 / 3, 256);
+		full_swath_bytes_vert_wc_l = (long)dml_ceil(full_swath_bytes_vert_wc_l * 2 / 3, 256);
+		full_swath_bytes_vert_wc_c = (long)dml_ceil(full_swath_bytes_vert_wc_c * 2 / 3, 256);
 	}
 
 	if (2 * full_swath_bytes_horz_wc_l + 2 * full_swath_bytes_horz_wc_c <= DETBufferSize) {
@@ -1537,14 +1538,14 @@ static double CalculatePrefetchSourceLines(
 
 	if (!mode_lib->vba.IgnoreViewportPositioning) {
 
-		*MaxNumSwath = dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0;
+		*MaxNumSwath = (unsigned int)(dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0);
 
 		if (*VInitPreFill > 1.0)
 			MaxPartialSwath = (unsigned int) (*VInitPreFill - 2) % SwathHeight;
 		else
 			MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 2)
 					% SwathHeight;
-		MaxPartialSwath = dml_max(1U, MaxPartialSwath);
+		MaxPartialSwath = (unsigned int)dml_max(1U, MaxPartialSwath);
 
 	} else {
 
@@ -1552,7 +1553,7 @@ static double CalculatePrefetchSourceLines(
 			dml_print(
 					"WARNING DML: using viewport y position of 0 even though actual viewport y position is non-zero in prefetch source lines calculation\n");
 
-		*MaxNumSwath = dml_ceil(*VInitPreFill / SwathHeight, 1);
+		*MaxNumSwath = (unsigned int)dml_ceil(*VInitPreFill / SwathHeight, 1);
 
 		if (*VInitPreFill > 1.0)
 			MaxPartialSwath = (unsigned int) (*VInitPreFill - 1) % SwathHeight;
@@ -1617,9 +1618,9 @@ static unsigned int CalculateVMAndRowBytes(
 		if (HostVMMinPageSize < 2048) {
 			HostVMDynamicLevels = HostVMMaxNonCachedPageTableLevels;
 		} else if (HostVMMinPageSize >= 2048 && HostVMMinPageSize < 1048576) {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
 		} else {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
 		}
 	}
 
@@ -1627,19 +1628,19 @@ static unsigned int CalculateVMAndRowBytes(
 	*MetaRequestWidth = 8 * BlockWidth256Bytes;
 	if (ScanDirection != dm_vert) {
 		*meta_row_height = *MetaRequestHeight;
-		*meta_row_width = dml_ceil((double) SwathWidth - 1, *MetaRequestWidth)
-				+ *MetaRequestWidth;
-		*MetaRowByte = *meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0;
+		*meta_row_width = (unsigned int)(dml_ceil((double) SwathWidth - 1, *MetaRequestWidth)
+				+ *MetaRequestWidth);
+		*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0);
 	} else {
 		*meta_row_height = *MetaRequestWidth;
-		*meta_row_width = dml_ceil((double) SwathWidth - 1, *MetaRequestHeight)
-				+ *MetaRequestHeight;
-		*MetaRowByte = *meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0;
+		*meta_row_width = (unsigned int)(dml_ceil((double) SwathWidth - 1, *MetaRequestHeight)
+				+ *MetaRequestHeight);
+		*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0);
 	}
-	DCCMetaSurfaceBytes = DCCMetaPitch * (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes)
-					+ 64 * BlockHeight256Bytes) * BytePerPixel / 256;
+	DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch * (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes)
+					+ 64 * BlockHeight256Bytes) * BytePerPixel / 256);
 	if (GPUVMEnable == true) {
-		*MetaPTEBytesFrame = (dml_ceil((double) (DCCMetaSurfaceBytes - 4.0 * 1024.0) / (8 * 4.0 * 1024), 1) + 1) * 64;
+		*MetaPTEBytesFrame = (unsigned int)((dml_ceil((double) (DCCMetaSurfaceBytes - 4.0 * 1024.0) / (8 * 4.0 * 1024), 1) + 1) * 64);
 		MPDEBytesFrame = 128 * (mode_lib->vba.GPUVMMaxPageTableLevels - 1);
 	} else {
 		*MetaPTEBytesFrame = 0;
@@ -1663,9 +1664,9 @@ static unsigned int CalculateVMAndRowBytes(
 
 	if (GPUVMEnable == true && mode_lib->vba.GPUVMMaxPageTableLevels > 1) {
 		if (ScanDirection != dm_vert) {
-			*DPDE0BytesFrame = 64 * (dml_ceil(((Pitch * (dml_ceil(ViewportHeight - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes) / (8 * 2097152), 1) + 1);
+			*DPDE0BytesFrame = (unsigned int)(64 * (dml_ceil(((Pitch * (dml_ceil(ViewportHeight - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes) / (8 * 2097152), 1) + 1));
 		} else {
-			*DPDE0BytesFrame = 64 * (dml_ceil(((Pitch * (dml_ceil((double) SwathWidth - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes) / (8 * 2097152), 1) + 1);
+			*DPDE0BytesFrame = (unsigned int)(64 * (dml_ceil(((Pitch * (dml_ceil((double) SwathWidth - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes) / (8 * 2097152), 1) + 1));
 		}
 		ExtraDPDEBytesFrame = 128 * (mode_lib->vba.GPUVMMaxPageTableLevels - 2);
 	} else {
@@ -1683,7 +1684,7 @@ static unsigned int CalculateVMAndRowBytes(
 	if (SurfaceTiling == dm_sw_linear) {
 		PixelPTEReqHeightPTEs = 1;
 		*PixelPTEReqHeight = 1;
-		*PixelPTEReqWidth = 32768.0 / BytePerPixel;
+		*PixelPTEReqWidth = (unsigned int)(32768.0 / BytePerPixel);
 		*PTERequestSize = 64;
 		FractionOfPTEReturnDrop = 0;
 	} else if (GPUVMMinPageSize == 4 && MacroTileSizeBytes > 4096) {
@@ -1704,16 +1705,16 @@ static unsigned int CalculateVMAndRowBytes(
 		if (PTEBufferSizeInRequests == 0)
 			*dpte_row_height = 1;
 		else
-			*dpte_row_height = dml_min(128, 1 << (unsigned int) dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1));
-		*dpte_row_width_ub = (dml_ceil(((double) SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth;
+			*dpte_row_height = (unsigned int)dml_min(128, 1 << (unsigned int) dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1));
+		*dpte_row_width_ub = (unsigned int)((dml_ceil(((double) SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth);
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqWidth * *PTERequestSize;
 	} else if (ScanDirection != dm_vert) {
 		*dpte_row_height = *PixelPTEReqHeight;
-		*dpte_row_width_ub = (dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth;
+		*dpte_row_width_ub = (unsigned int)((dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth);
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqWidth * *PTERequestSize;
 	} else {
-		*dpte_row_height = dml_min(*PixelPTEReqWidth, *MacroTileWidth);
-		*dpte_row_width_ub = (dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1) * *PixelPTEReqHeight;
+		*dpte_row_height = (unsigned int)dml_min(*PixelPTEReqWidth, *MacroTileWidth);
+		*dpte_row_width_ub = (unsigned int)((dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1) * *PixelPTEReqHeight);
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqHeight * *PTERequestSize;
 	}
 	if (*PixelPTEBytesPerRow * (1 - FractionOfPTEReturnDrop)
@@ -1793,9 +1794,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 						v->WritebackHTaps[k],
 						v->WritebackVTaps[k],
 						v->WritebackSourceWidth[k],
-						v->WritebackDestinationWidth[k],
+						(long)v->WritebackDestinationWidth[k],
 						v->HTotal[k],
-						v->WritebackLineBufferSize));
+						(unsigned int)v->WritebackLineBufferSize));
 		}
 	}
 
@@ -2002,7 +2003,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			v->DPPCLK,
 			v->ReadBandwidthPlaneLuma,
 			v->ReadBandwidthPlaneChroma,
-			v->ReturnBusWidth,
+			(int)v->ReturnBusWidth,
 			&v->DCFCLKDeepSleep);
 
 	// DSCCLK
@@ -2038,7 +2039,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			if (v->ODMCombineEnabled[k] == dm_odm_combine_mode_disabled) {
 				v->DSCDelay[k] = dscceComputeDelay(v->DSCInputBitPerComponent[k],
 						BPP,
-						dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
+						(unsigned int)dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
 						v->NumberOfDSCSlices[k],
 						v->OutputFormat[k],
 						v->Output[k])
@@ -2046,21 +2047,21 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			} else if (v->ODMCombineEnabled[k] == dm_odm_combine_mode_2to1) {
 				v->DSCDelay[k] = 2 * dscceComputeDelay(v->DSCInputBitPerComponent[k],
 						BPP,
-						dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
-						v->NumberOfDSCSlices[k] / 2.0,
+						(unsigned int)dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
+						(unsigned int)(v->NumberOfDSCSlices[k] / 2.0),
 						v->OutputFormat[k],
 						v->Output[k])
 					+ dscComputeDelay(v->OutputFormat[k], v->Output[k]);
 			} else {
 				v->DSCDelay[k] = 4 * dscceComputeDelay(v->DSCInputBitPerComponent[k],
 						BPP,
-						dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
-						v->NumberOfDSCSlices[k] / 4.0,
+						(unsigned int)dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
+						(unsigned int)(v->NumberOfDSCSlices[k] / 4.0),
 						v->OutputFormat[k],
 						v->Output[k])
 					+ dscComputeDelay(v->OutputFormat[k], v->Output[k]);
 			}
-			v->DSCDelay[k] = v->DSCDelay[k] * v->PixelClock[k] / v->PixelClockBackEnd[k];
+			v->DSCDelay[k] = (unsigned int)(v->DSCDelay[k] * v->PixelClock[k] / v->PixelClockBackEnd[k]);
 		} else {
 			v->DSCDelay[k] = 0;
 		}
@@ -2102,13 +2103,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					v->SurfaceTiling[k],
 					v->BytePerPixelC[k],
 					v->SourceScan[k],
-					v->SwathWidthC[k],
+					(unsigned int)v->SwathWidthC[k],
 					v->ViewportHeightChroma[k],
 					v->GPUVMEnable,
 					v->HostVMEnable,
 					v->HostVMMaxNonCachedPageTableLevels,
-					v->GPUVMMinPageSize,
-					v->HostVMMinPageSize,
+					(unsigned int)v->GPUVMMinPageSize,
+					(unsigned int)v->HostVMMinPageSize,
 					v->PTEBufferSizeInRequestsForChroma,
 					v->PitchC[k],
 					v->DCCMetaPitchC[k],
@@ -2159,13 +2160,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				v->SurfaceTiling[k],
 				v->BytePerPixelY[k],
 				v->SourceScan[k],
-				v->SwathWidthY[k],
+				(unsigned int)v->SwathWidthY[k],
 				v->ViewportHeight[k],
 				v->GPUVMEnable,
 				v->HostVMEnable,
 				v->HostVMMaxNonCachedPageTableLevels,
-				v->GPUVMMinPageSize,
-				v->HostVMMinPageSize,
+				(unsigned int)v->GPUVMMinPageSize,
+				(unsigned int)v->HostVMMinPageSize,
 				v->PTEBufferSizeInRequestsForLuma,
 				v->PitchY[k],
 				v->DCCMetaPitchY[k],
@@ -2232,13 +2233,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 	}
 
 
-	ReorderBytes = v->NumberOfChannels * dml_max3(
+	ReorderBytes = (long)(v->NumberOfChannels * dml_max3(
 		v->UrgentOutOfOrderReturnPerChannelPixelDataOnly,
 		v->UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData,
-		v->UrgentOutOfOrderReturnPerChannelVMDataOnly);
+		v->UrgentOutOfOrderReturnPerChannelVMDataOnly));
 
 	v->UrgentExtraLatency = CalculateExtraLatency(
-		v->RoundTripPingLatencyCycles,
+		(long)v->RoundTripPingLatencyCycles,
 		ReorderBytes,
 		v->DCFCLK,
 		v->TotalActiveDPP,
@@ -2266,9 +2267,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 									v->WritebackHRatio[k],
 									v->WritebackVRatio[k],
 									v->WritebackVTaps[k],
-									v->WritebackDestinationWidth[k],
-									v->WritebackDestinationHeight[k],
-									v->WritebackSourceHeight[k],
+									(long)v->WritebackDestinationWidth[k],
+									(long)v->WritebackDestinationHeight[k],
+									(long)v->WritebackSourceHeight[k],
 									v->HTotal[k]) / v->DISPCLK;
 			} else
 				v->WritebackDelay[v->VoltageLevel][k] = 0;
@@ -2281,9 +2282,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 											v->WritebackHRatio[j],
 											v->WritebackVRatio[j],
 											v->WritebackVTaps[j],
-											v->WritebackDestinationWidth[j],
-											v->WritebackDestinationHeight[j],
-											v->WritebackSourceHeight[j],
+											(long)v->WritebackDestinationWidth[j],
+											(long)v->WritebackDestinationHeight[j],
+											(long)v->WritebackSourceHeight[j],
 											v->HTotal[k]) / v->DISPCLK);
 				}
 			}
@@ -2296,12 +2297,12 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				v->WritebackDelay[v->VoltageLevel][k] = v->WritebackDelay[v->VoltageLevel][j];
 
 	for (k = 0; k < v->NumberOfActivePlanes; ++k) {
-		v->MaxVStartupLines[k] = v->VTotal[k] - v->VActive[k] - dml_max(1.0, dml_ceil((double) v->WritebackDelay[v->VoltageLevel][k] / (v->HTotal[k] / v->PixelClock[k]), 1));
+		v->MaxVStartupLines[k] = (unsigned int)(v->VTotal[k] - v->VActive[k] - dml_max(1.0, dml_ceil((double) v->WritebackDelay[v->VoltageLevel][k] / (v->HTotal[k] / v->PixelClock[k]), 1)));
 	}
 
 	v->MaximumMaxVStartupLines = 0;
 	for (k = 0; k < v->NumberOfActivePlanes; ++k)
-		v->MaximumMaxVStartupLines = dml_max(v->MaximumMaxVStartupLines, v->MaxVStartupLines[k]);
+		v->MaximumMaxVStartupLines = (unsigned int)dml_max(v->MaximumMaxVStartupLines, v->MaxVStartupLines[k]);
 
 	if (v->DRAMClockChangeLatencyOverride > 0.0) {
 		v->FinalDRAMClockChangeLatency = v->DRAMClockChangeLatencyOverride;
@@ -2354,21 +2355,21 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					&myPipe,
 					v->DSCDelay[k],
 					(unsigned int) (v->SwathWidthY[k] / v->HRatio[k]),
-					dml_min(v->VStartupLines, v->MaxVStartupLines[k]),
+					(unsigned int)dml_min(v->VStartupLines, v->MaxVStartupLines[k]),
 					v->MaxVStartupLines[k],
 					v->UrgentLatency,
 					v->UrgentExtraLatency,
 					v->TCalc,
-					v->PDEAndMetaPTEBytesFrame[k],
-					v->MetaRowByte[k],
-					v->PixelPTEBytesPerRow[k],
+					(unsigned int)v->PDEAndMetaPTEBytesFrame[k],
+					(unsigned int)v->MetaRowByte[k],
+					(unsigned int)v->PixelPTEBytesPerRow[k],
 					v->PrefetchSourceLinesY[k],
-					v->SwathWidthY[k],
+					(unsigned int)v->SwathWidthY[k],
 					v->BytePerPixelY[k],
 					v->VInitPreFillY[k],
 					v->MaxNumSwathY[k],
 					v->PrefetchSourceLinesC[k],
-					v->SwathWidthC[k],
+					(unsigned int)v->SwathWidthC[k],
 					v->VInitPreFillC[k],
 					v->MaxNumSwathC[k],
 					v->swath_width_luma_ub[k],
@@ -2389,17 +2390,17 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				double TotalRepeaterDelayTime = v->MaxInterDCNTileRepeaters * (2 / v->DPPCLK[k] + 3 / v->DISPCLK);
 				v->VUpdateWidthPix[k] = (14 / v->DCFCLKDeepSleep + 12 / v->DPPCLK[k] + TotalRepeaterDelayTime) * v->PixelClock[k];
 				v->VReadyOffsetPix[k] = dml_max(150.0 / v->DPPCLK[k], TotalRepeaterDelayTime + 20 / v->DCFCLKDeepSleep + 10 / v->DPPCLK[k]) * v->PixelClock[k];
-				v->VUpdateOffsetPix[k] = dml_ceil(v->HTotal[k] / 4.0, 1);
-				v->VStartup[k] = dml_min(v->VStartupLines, v->MaxVStartupLines[k]);
+				v->VUpdateOffsetPix[k] = (unsigned int)dml_ceil(v->HTotal[k] / 4.0, 1);
+				v->VStartup[k] = (unsigned int)dml_min(v->VStartupLines, v->MaxVStartupLines[k]);
 			} else {
 				int x = v->BlendingAndTiming[k];
 				double TotalRepeaterDelayTime = v->MaxInterDCNTileRepeaters * (2 / v->DPPCLK[k] + 3 / v->DISPCLK);
 				v->VUpdateWidthPix[k] = (14 / v->DCFCLKDeepSleep + 12 / v->DPPCLK[k] + TotalRepeaterDelayTime) * v->PixelClock[x];
 				v->VReadyOffsetPix[k] = dml_max(150.0 / v->DPPCLK[k], TotalRepeaterDelayTime + 20 / v->DCFCLKDeepSleep + 10 / v->DPPCLK[k]) * v->PixelClock[x];
-				v->VUpdateOffsetPix[k] = dml_ceil(v->HTotal[x] / 4.0, 1);
+				v->VUpdateOffsetPix[k] = (unsigned int)dml_ceil(v->HTotal[x] / 4.0, 1);
 				if (!v->MaxVStartupLines[x])
 					v->MaxVStartupLines[x] = v->MaxVStartupLines[k];
-				v->VStartup[k] = dml_min(v->VStartupLines, v->MaxVStartupLines[x]);
+				v->VStartup[k] = (unsigned int)dml_min(v->VStartupLines, v->MaxVStartupLines[x]);
 			}
 		}
 
@@ -2524,7 +2525,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 
 			v->TotImmediateFlipBytes = 0;
 			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
-				v->TotImmediateFlipBytes = v->TotImmediateFlipBytes + v->DPPPerPlane[k] * (v->PDEAndMetaPTEBytesFrame[k] + v->MetaRowByte[k] + v->PixelPTEBytesPerRow[k]);
+				v->TotImmediateFlipBytes = (unsigned int)(v->TotImmediateFlipBytes + v->DPPPerPlane[k] * (v->PDEAndMetaPTEBytesFrame[k] + v->MetaRowByte[k] + v->PixelPTEBytesPerRow[k]));
 			}
 			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 				CalculateFlipSchedule(
@@ -2887,18 +2888,18 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib)
 	// Display Pipe Configuration
 	double BytePerPixDETY[DC__NUM_DPP__MAX] = { 0 };
 	double BytePerPixDETC[DC__NUM_DPP__MAX] = { 0 };
-	int BytePerPixY[DC__NUM_DPP__MAX] = { 0 };
-	int BytePerPixC[DC__NUM_DPP__MAX] = { 0 };
-	int Read256BytesBlockHeightY[DC__NUM_DPP__MAX] = { 0 };
-	int Read256BytesBlockHeightC[DC__NUM_DPP__MAX] = { 0 };
-	int Read256BytesBlockWidthY[DC__NUM_DPP__MAX] = { 0 };
-	int Read256BytesBlockWidthC[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int BytePerPixY[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int BytePerPixC[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int Read256BytesBlockHeightY[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int Read256BytesBlockHeightC[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int Read256BytesBlockWidthY[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int Read256BytesBlockWidthC[DC__NUM_DPP__MAX] = { 0 };
 	double dummy1[DC__NUM_DPP__MAX] = { 0 };
 	double dummy2[DC__NUM_DPP__MAX] = { 0 };
 	double dummy3[DC__NUM_DPP__MAX] = { 0 };
 	double dummy4[DC__NUM_DPP__MAX] = { 0 };
-	int dummy5[DC__NUM_DPP__MAX] = { 0 };
-	int dummy6[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int dummy5[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int dummy6[DC__NUM_DPP__MAX] = { 0 };
 	bool dummy7[DC__NUM_DPP__MAX] = { 0 };
 	bool dummysinglestring = 0;
 	unsigned int k;
@@ -3380,9 +3381,9 @@ static double TruncToValidBPP(
 void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib)
 {
 	struct vba_vars_st *v = &mode_lib->vba;
-	int MinPrefetchMode, MaxPrefetchMode;
-	int i, start_state;
-	unsigned int j, k, m;
+	unsigned int MinPrefetchMode, MaxPrefetchMode;
+	int idx, start_state;
+	unsigned int i, j, k, m;
 	bool   EnoughWritebackUnits = true;
 	bool   WritebackModeSupport = true;
 	bool   ViewportExceedsSurface = false;
@@ -3590,9 +3591,9 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->WritebackHTaps[k],
 							v->WritebackVTaps[k],
 							v->WritebackSourceWidth[k],
-							v->WritebackDestinationWidth[k],
+							(long)v->WritebackDestinationWidth[k],
 							v->HTotal[k],
-							v->WritebackLineBufferSize));
+							(unsigned int)v->WritebackLineBufferSize));
 		}
 	}
 	for (k = 0; k <= v->NumberOfActivePlanes - 1; k++) {
@@ -3631,15 +3632,15 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		int MaximumSwathWidthSupportChroma = 0;
 
 		if (v->SurfaceTiling[k] == dm_sw_linear) {
-			MaximumSwathWidthSupportLuma = 8192.0;
+			MaximumSwathWidthSupportLuma = (int)8192.0;
 		} else if (v->SourceScan[k] == dm_vert && v->BytePerPixelC[k] > 0) {
-			MaximumSwathWidthSupportLuma = 2880.0;
+			MaximumSwathWidthSupportLuma = (int)2880.0;
 		} else {
-			MaximumSwathWidthSupportLuma = 5760.0;
+			MaximumSwathWidthSupportLuma = (int)5760.0;
 		}
 
 		if (v->SourcePixelFormat[k] == dm_420_8 || v->SourcePixelFormat[k] == dm_420_10 || v->SourcePixelFormat[k] == dm_420_12) {
-			MaximumSwathWidthSupportChroma = MaximumSwathWidthSupportLuma / 2.0;
+			MaximumSwathWidthSupportChroma = (int)(MaximumSwathWidthSupportLuma / 2.0);
 		} else {
 			MaximumSwathWidthSupportChroma = MaximumSwathWidthSupportLuma;
 		}
@@ -3837,7 +3838,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						v->DISPCLK_DPPCLK_Support[i][j] = false;
 					}
 				}
-				v->TotalNumberOfActiveDPP[i][j] = 0.0;
+				v->TotalNumberOfActiveDPP[i][j] = (unsigned int)0.0;
 				for (k = 0; k <= v->NumberOfActivePlanes - 1; k++) {
 					v->TotalNumberOfActiveDPP[i][j] = v->TotalNumberOfActiveDPP[i][j] + v->NoOfDPP[i][j][k];
 				}
@@ -3875,7 +3876,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 		if (v->BlendingAndTiming[k] == k) {
 			if (v->PixelClockBackEnd[k] > 3200) {
-				v->NumberOfDSCSlices[k] = dml_ceil(v->PixelClockBackEnd[k] / 400.0, 4.0);
+				v->NumberOfDSCSlices[k] = (unsigned int)dml_ceil(v->PixelClockBackEnd[k] / 400.0, 4.0);
 			} else if (v->PixelClockBackEnd[k] > 1360) {
 				v->NumberOfDSCSlices[k] = 8;
 			} else if (v->PixelClockBackEnd[k] > 680) {
@@ -3932,7 +3933,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					if (v->PHYCLKPerState[i] >= 270.0) {
 						v->Outbpp = TruncToValidBPP(
 								(1.0 - v->Downspreading / 100.0) * 2700,
-								v->OutputLinkDPLanes[k],
+								(int)v->OutputLinkDPLanes[k],
 								v->HTotal[k],
 								v->HActive[k],
 								v->PixelClockBackEnd[k],
@@ -3952,7 +3953,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 540.0) {
 						v->Outbpp = TruncToValidBPP(
 								(1.0 - v->Downspreading / 100.0) * 5400,
-								v->OutputLinkDPLanes[k],
+								(int)v->OutputLinkDPLanes[k],
 								v->HTotal[k],
 								v->HActive[k],
 								v->PixelClockBackEnd[k],
@@ -3972,7 +3973,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 810.0) {
 						v->Outbpp = TruncToValidBPP(
 								(1.0 - v->Downspreading / 100.0) * 8100,
-								v->OutputLinkDPLanes[k],
+								(int)v->OutputLinkDPLanes[k],
 								v->HTotal[k],
 								v->HActive[k],
 								v->PixelClockBackEnd[k],
@@ -3994,7 +3995,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							}
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 8100,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4073,7 +4074,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					v->DSCDelayPerState[i][k] = dscceComputeDelay(
 							v->DSCInputBitPerComponent[k],
 							v->BPP,
-							dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
+							(unsigned int)dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
 							v->NumberOfDSCSlices[k],
 							v->OutputFormat[k],
 							v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]);
@@ -4082,7 +4083,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							* dscceComputeDelay(
 									v->DSCInputBitPerComponent[k],
 									v->BPP,
-									dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
+									(unsigned int)dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
 									v->NumberOfDSCSlices[k] / 2,
 									v->OutputFormat[k],
 									v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]);
@@ -4091,7 +4092,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							* (dscceComputeDelay(
 									v->DSCInputBitPerComponent[k],
 									v->BPP,
-									dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
+									(unsigned int)dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
 									v->NumberOfDSCSlices[k] / 4,
 									v->OutputFormat[k],
 									v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]));
@@ -4163,8 +4164,8 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 				v->swath_width_luma_ub_all_states[i][j][k] = v->swath_width_luma_ub_this_state[k];
 				v->swath_width_chroma_ub_all_states[i][j][k] = v->swath_width_chroma_ub_this_state[k];
-				v->SwathWidthYAllStates[i][j][k] = v->SwathWidthYThisState[k];
-				v->SwathWidthCAllStates[i][j][k] = v->SwathWidthCThisState[k];
+				v->SwathWidthYAllStates[i][j][k] = (unsigned int)v->SwathWidthYThisState[k];
+				v->SwathWidthCAllStates[i][j][k] = (unsigned int)v->SwathWidthCThisState[k];
 				v->SwathHeightYAllStates[i][j][k] = v->SwathHeightYThisState[k];
 				v->SwathHeightCAllStates[i][j][k] = v->SwathHeightCThisState[k];
 				v->DETBufferSizeYAllStates[i][j][k] = v->DETBufferSizeYThisState[k];
@@ -4186,8 +4187,8 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				v->SwathWidthCThisState[k] = v->SwathWidthCAllStates[i][j][k];
 				v->SwathHeightYThisState[k] = v->SwathHeightYAllStates[i][j][k];
 				v->SwathHeightCThisState[k] = v->SwathHeightCAllStates[i][j][k];
-				v->DETBufferSizeYThisState[k] = v->DETBufferSizeYAllStates[i][j][k];
-				v->DETBufferSizeCThisState[k] = v->DETBufferSizeCAllStates[i][j][k];
+				v->DETBufferSizeYThisState[k] = (unsigned int)v->DETBufferSizeYAllStates[i][j][k];
+				v->DETBufferSizeCThisState[k] = (unsigned int)v->DETBufferSizeCAllStates[i][j][k];
 			}
 
 			v->TotalNumberOfDCCActiveDPP[i][j] = 0;
@@ -4218,16 +4219,16 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->SurfaceTiling[k],
 							v->BytePerPixelC[k],
 							v->SourceScan[k],
-							v->SwathWidthCThisState[k],
+							(unsigned int)v->SwathWidthCThisState[k],
 							v->ViewportHeightChroma[k],
 							v->GPUVMEnable,
 							v->HostVMEnable,
 							v->HostVMMaxNonCachedPageTableLevels,
-							v->GPUVMMinPageSize,
-							v->HostVMMinPageSize,
+							(unsigned int)v->GPUVMMinPageSize,
+							(unsigned int)v->HostVMMinPageSize,
 							v->PTEBufferSizeInRequestsForChroma,
 							v->PitchC[k],
-							0.0,
+							(unsigned int)0,
 							&v->MacroTileWidthC[k],
 							&v->MetaRowBytesC,
 							&v->DPTEBytesPerRowC,
@@ -4260,8 +4261,8 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					v->PTEBufferSizeInRequestsForLuma = v->PTEBufferSizeInRequestsLuma + v->PTEBufferSizeInRequestsChroma;
 					v->PTEBufferSizeInRequestsForChroma = 0;
 					v->PDEAndMetaPTEBytesPerFrameC = 0.0;
-					v->MetaRowBytesC = 0.0;
-					v->DPTEBytesPerRowC = 0.0;
+					v->MetaRowBytesC = (unsigned int)0;
+					v->DPTEBytesPerRowC = (unsigned int)0;
 					v->PrefetchLinesC[i][j][k] = 0.0;
 					v->PTEBufferSizeNotExceededC[i][j][k] = true;
 				}
@@ -4274,13 +4275,13 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						v->SurfaceTiling[k],
 						v->BytePerPixelY[k],
 						v->SourceScan[k],
-						v->SwathWidthYThisState[k],
+						(unsigned int)v->SwathWidthYThisState[k],
 						v->ViewportHeight[k],
 						v->GPUVMEnable,
 						v->HostVMEnable,
 						v->HostVMMaxNonCachedPageTableLevels,
-						v->GPUVMMinPageSize,
-						v->HostVMMinPageSize,
+						(unsigned int)v->GPUVMMinPageSize,
+						(unsigned int)v->HostVMMinPageSize,
 						v->PTEBufferSizeInRequestsForLuma,
 						v->PitchY[k],
 						v->DCCMetaPitchY[k],
@@ -4408,7 +4409,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					v->RequiredDPPCLKThisState,
 					v->ReadBandwidthLuma,
 					v->ReadBandwidthChroma,
-					v->ReturnBusWidth,
+					(int)v->ReturnBusWidth,
 					&v->ProjectedDCFCLKDeepSleep[i][j]);
 		}
 	}
@@ -4426,9 +4427,9 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 										v->WritebackHRatio[k],
 										v->WritebackVRatio[k],
 										v->WritebackVTaps[k],
-										v->WritebackDestinationWidth[k],
-										v->WritebackDestinationHeight[k],
-										v->WritebackSourceHeight[k],
+										(long)v->WritebackDestinationWidth[k],
+										(long)v->WritebackDestinationHeight[k],
+										(long)v->WritebackSourceHeight[k],
 										v->HTotal[k]) / v->RequiredDISPCLK[i][j];
 					} else {
 						v->WritebackDelayTime[k] = 0.0;
@@ -4443,9 +4444,9 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 													v->WritebackHRatio[m],
 													v->WritebackVRatio[m],
 													v->WritebackVTaps[m],
-													v->WritebackDestinationWidth[m],
-													v->WritebackDestinationHeight[m],
-													v->WritebackSourceHeight[m],
+													(long)v->WritebackDestinationWidth[m],
+													(long)v->WritebackDestinationHeight[m],
+													(long)v->WritebackSourceHeight[m],
 													v->HTotal[m]) / v->RequiredDISPCLK[i][j]);
 						}
 					}
@@ -4467,11 +4468,11 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		}
 	}
 
-	ReorderingBytes = v->NumberOfChannels
+	ReorderingBytes = (long)(v->NumberOfChannels
 			* dml_max3(
 					v->UrgentOutOfOrderReturnPerChannelPixelDataOnly,
 					v->UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData,
-					v->UrgentOutOfOrderReturnPerChannelVMDataOnly);
+					v->UrgentOutOfOrderReturnPerChannelVMDataOnly));
 	v->FinalDRAMClockChangeLatency = (v->DRAMClockChangeLatencyOverride > 0 ? v->DRAMClockChangeLatencyOverride : v->DRAMClockChangeLatency);
 
 	for (i = start_state; i < mode_lib->soc.num_states; ++i) {
@@ -4549,7 +4550,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 
 	for (i = start_state; i < mode_lib->soc.num_states; ++i) {
 		for (j = 0; j <= 1; ++j) {
-			int NextPrefetchModeState = MinPrefetchMode;
+			unsigned int NextPrefetchModeState = MinPrefetchMode;
 
 			v->TimeCalc = 24 / v->ProjectedDCFCLKDeepSleep[i][j];
 
@@ -4567,13 +4568,13 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				v->SwathWidthCThisState[k] = v->SwathWidthCAllStates[i][j][k];
 				v->SwathHeightYThisState[k] = v->SwathHeightYAllStates[i][j][k];
 				v->SwathHeightCThisState[k] = v->SwathHeightCAllStates[i][j][k];
-				v->DETBufferSizeYThisState[k] = v->DETBufferSizeYAllStates[i][j][k];
-				v->DETBufferSizeCThisState[k] = v->DETBufferSizeCAllStates[i][j][k];
+				v->DETBufferSizeYThisState[k] = (unsigned int)v->DETBufferSizeYAllStates[i][j][k];
+				v->DETBufferSizeCThisState[k] = (unsigned int)v->DETBufferSizeCAllStates[i][j][k];
 				v->ODMCombineEnabled[k] = v->ODMCombineEnablePerState[i][k];
 			}
 
 			v->ExtraLatency = CalculateExtraLatency(
-					v->RoundTripPingLatencyCycles,
+					(long)v->RoundTripPingLatencyCycles,
 					ReorderingBytes,
 					v->DCFCLKState[i][j],
 					v->TotalNumberOfActiveDPP[i][j],
@@ -4623,23 +4624,23 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							mode_lib,
 							k,
 							&myPipe,
-							v->DSCDelayPerState[i][k],
-							v->SwathWidthYThisState[k] / v->HRatio[k],
-							dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]),
-							v->MaximumVStartup[i][j][k],
+							(unsigned int)v->DSCDelayPerState[i][k],
+							(unsigned int)(v->SwathWidthYThisState[k] / v->HRatio[k]),
+							(unsigned int)dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]),
+							(unsigned int)v->MaximumVStartup[i][j][k],
 							v->UrgLatency[i],
 							v->ExtraLatency,
 							v->TimeCalc,
-							v->PDEAndMetaPTEBytesPerFrame[i][j][k],
-							v->MetaRowBytes[i][j][k],
-							v->DPTEBytesPerRow[i][j][k],
+							(unsigned int)v->PDEAndMetaPTEBytesPerFrame[i][j][k],
+							(unsigned int)v->MetaRowBytes[i][j][k],
+							(unsigned int)v->DPTEBytesPerRow[i][j][k],
 							v->PrefetchLinesY[i][j][k],
-							v->SwathWidthYThisState[k],
+							(unsigned int)v->SwathWidthYThisState[k],
 							v->BytePerPixelY[k],
 							v->PrefillY[k],
 							v->MaxNumSwY[k],
 							v->PrefetchLinesC[i][j][k],
-							v->SwathWidthCThisState[k],
+							(unsigned int)v->SwathWidthCThisState[k],
 							v->PrefillC[k],
 							v->MaxNumSwC[k],
 							v->swath_width_luma_ub_this_state[k],
@@ -4751,10 +4752,10 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 																* v->UrgentBurstFactorChromaPre[k])
 												+ v->cursor_bw_pre[k] * v->UrgentBurstFactorCursorPre[k]);
 					}
-					v->TotImmediateFlipBytes = 0.0;
+					v->TotImmediateFlipBytes = (unsigned int)0;
 					for (k = 0; k <= v->NumberOfActivePlanes - 1; k++) {
-						v->TotImmediateFlipBytes = v->TotImmediateFlipBytes + v->NoOfDPP[i][j][k] * (v->PDEAndMetaPTEBytesPerFrame[i][j][k]
-								+ v->MetaRowBytes[i][j][k] + v->DPTEBytesPerRow[i][j][k]);
+						v->TotImmediateFlipBytes = (unsigned int)(v->TotImmediateFlipBytes + v->NoOfDPP[i][j][k] * (v->PDEAndMetaPTEBytesPerFrame[i][j][k]
+								+ v->MetaRowBytes[i][j][k] + v->DPTEBytesPerRow[i][j][k]));
 					}
 
 					for (k = 0; k <= v->NumberOfActivePlanes - 1; k++) {
@@ -4913,33 +4914,39 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	}
 	/*Mode Support, Voltage State and SOC Configuration*/
 
-	for (i = v->soc.num_states - 1; i >= start_state; i--) {
+	for (idx = v->soc.num_states - 1; idx >= start_state; idx--) {
 		for (j = 0; j < 2; j++) {
-			if (v->ScaleRatioAndTapsSupport == 1 && v->SourceFormatPixelAndScanSupport == 1 && v->ViewportSizeSupport[i][j] == 1
-					&& v->DIOSupport[i] == 1 && v->ODMCombine4To1SupportCheckOK[i] == 1
-					&& v->NotEnoughDSCUnits[i] == 0
-					&& v->DTBCLKRequiredMoreThanSupported[i] == 0
-					&& v->ROBSupport[i][j] == 1 && v->DISPCLK_DPPCLK_Support[i][j] == 1 && v->TotalAvailablePipesSupport[i][j] == 1
+			if (v->ScaleRatioAndTapsSupport == 1 && v->SourceFormatPixelAndScanSupport == 1
+					&& v->ViewportSizeSupport[idx][j] == 1
+					&& v->DIOSupport[idx] == 1 && v->ODMCombine4To1SupportCheckOK[idx] == 1
+					&& v->NotEnoughDSCUnits[idx] == 0
+					&& v->DTBCLKRequiredMoreThanSupported[idx] == 0
+					&& v->ROBSupport[idx][j] == 1 && v->DISPCLK_DPPCLK_Support[idx][j] == 1
+					&& v->TotalAvailablePipesSupport[idx][j] == 1
 					&& EnoughWritebackUnits == 1 && WritebackModeSupport == 1
-					&& v->WritebackLatencySupport == 1 && v->WritebackScaleRatioAndTapsSupport == 1 && v->CursorSupport == 1 && v->PitchSupport == 1
-					&& ViewportExceedsSurface == 0 && v->PrefetchSupported[i][j] == 1 && v->DynamicMetadataSupported[i][j] == 1
-					&& v->TotalVerticalActiveBandwidthSupport[i][j] == 1 && v->VRatioInPrefetchSupported[i][j] == 1
-					&& v->PTEBufferSizeNotExceeded[i][j] == 1 && v->NonsupportedDSCInputBPC == 0
+					&& v->WritebackLatencySupport == 1 && v->WritebackScaleRatioAndTapsSupport == 1
+					&& v->CursorSupport == 1 && v->PitchSupport == 1
+					&& ViewportExceedsSurface == 0 && v->PrefetchSupported[idx][j] == 1
+					&& v->DynamicMetadataSupported[idx][j] == 1
+					&& v->TotalVerticalActiveBandwidthSupport[idx][j] == 1
+					&& v->VRatioInPrefetchSupported[idx][j] == 1
+					&& v->PTEBufferSizeNotExceeded[idx][j] == 1 && v->NonsupportedDSCInputBPC == 0
 					&& ((v->HostVMEnable == 0 && v->ImmediateFlipRequirement[0] != dm_immediate_flip_required)
-							|| v->ImmediateFlipSupportedForState[i][j] == true)) {
-				v->ModeSupport[i][j] = true;
+							|| v->ImmediateFlipSupportedForState[idx][j] == true)) {
+				v->ModeSupport[idx][j] = true;
 			} else {
-				v->ModeSupport[i][j] = false;
+				v->ModeSupport[idx][j] = false;
 			}
 		}
 	}
 	{
 		unsigned int MaximumMPCCombine = 0;
-		for (i = v->soc.num_states; i >= start_state; i--) {
-			if (i == v->soc.num_states || v->ModeSupport[i][0] == true || v->ModeSupport[i][1] == true) {
-				v->VoltageLevel = i;
-				v->ModeIsSupported = v->ModeSupport[i][0] == true || v->ModeSupport[i][1] == true;
-				if (v->ModeSupport[i][1] == true) {
+		for (idx = v->soc.num_states; idx >= start_state; idx--) {
+			if (idx == (int)v->soc.num_states || v->ModeSupport[idx][0] == true
+					|| v->ModeSupport[idx][1] == true) {
+				v->VoltageLevel = idx;
+				v->ModeIsSupported = v->ModeSupport[idx][0] == true || v->ModeSupport[idx][1] == true;
+				if (v->ModeSupport[idx][1] == true) {
 					MaximumMPCCombine = 1;
 				} else {
 					MaximumMPCCombine = 0;
@@ -5038,20 +5045,20 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 
 	for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 
-		v->LBLatencyHidingSourceLinesY = dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1);
+		v->LBLatencyHidingSourceLinesY = (unsigned int)(dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1));
 
-		v->LBLatencyHidingSourceLinesC = dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1);
+		v->LBLatencyHidingSourceLinesC = (unsigned int)(dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1));
 
 		EffectiveLBLatencyHidingY = v->LBLatencyHidingSourceLinesY / v->VRatio[k] * (v->HTotal[k] / v->PixelClock[k]);
 
 		EffectiveLBLatencyHidingC = v->LBLatencyHidingSourceLinesC / v->VRatioChroma[k] * (v->HTotal[k] / v->PixelClock[k]);
 
 		LinesInDETY[k] = (double) DETBufferSizeY[k] / BytePerPixelDETY[k] / SwathWidthY[k];
-		LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]);
+		LinesInDETYRoundedDownToSwath[k] = (unsigned int)dml_floor(LinesInDETY[k], SwathHeightY[k]);
 		FullDETBufferingTimeY[k] = LinesInDETYRoundedDownToSwath[k] * (v->HTotal[k] / v->PixelClock[k]) / v->VRatio[k];
 		if (BytePerPixelDETC[k] > 0) {
 			LinesInDETC = v->DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k];
-			LinesInDETCRoundedDownToSwath = dml_floor(LinesInDETC, SwathHeightC[k]);
+			LinesInDETCRoundedDownToSwath = (unsigned int)dml_floor(LinesInDETC, SwathHeightC[k]);
 			FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath * (v->HTotal[k] / v->PixelClock[k]) / v->VRatioChroma[k];
 		} else {
 			LinesInDETC = 0;
@@ -5146,8 +5153,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 static void CalculateDCFCLKDeepSleep(
 		struct display_mode_lib *mode_lib,
 		unsigned int NumberOfActivePlanes,
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		double VRatio[],
 		double VRatioChroma[],
 		double SwathWidthY[],
@@ -5299,7 +5306,7 @@ static void CalculatePixelDeliveryTimes(
 		double PSCL_THROUGHPUT[],
 		double PSCL_THROUGHPUT_CHROMA[],
 		double DPPCLK[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
 		unsigned int NumberOfCursors[],
 		unsigned int CursorWidth[][2],
@@ -5380,7 +5387,7 @@ static void CalculatePixelDeliveryTimes(
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		int cursor_req_per_width = 0;
-		cursor_req_per_width = dml_ceil(CursorWidth[k][0] * CursorBPP[k][0] / 256 / 8, 1);
+		cursor_req_per_width = (int)dml_ceil(CursorWidth[k][0] * CursorBPP[k][0] / 256 / 8, 1);
 		if (NumberOfCursors[k] > 0) {
 			if (VRatio[k] <= 1) {
 				CursorRequestDeliveryTime[k] = CursorWidth[k][0] / HRatio[k] / PixelClock[k] / cursor_req_per_width;
@@ -5400,39 +5407,39 @@ static void CalculatePixelDeliveryTimes(
 }
 
 static void CalculateMetaAndPTETimes(
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		bool GPUVMEnable,
 		int MetaChunkSize,
 		int MinMetaChunkSizeBytes,
-		int HTotal[],
+		unsigned int HTotal[],
 		double VRatio[],
 		double VRatioChroma[],
 		double DestinationLinesToRequestRowInVBlank[],
 		double DestinationLinesToRequestRowInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
-		int dpte_row_height[],
-		int dpte_row_height_chroma[],
-		int meta_row_width[],
-		int meta_row_width_chroma[],
-		int meta_row_height[],
-		int meta_row_height_chroma[],
-		int meta_req_width[],
-		int meta_req_width_chroma[],
-		int meta_req_height[],
-		int meta_req_height_chroma[],
-		int dpte_group_bytes[],
-		int PTERequestSizeY[],
-		int PTERequestSizeC[],
-		int PixelPTEReqWidthY[],
-		int PixelPTEReqHeightY[],
-		int PixelPTEReqWidthC[],
-		int PixelPTEReqHeightC[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
+		unsigned int dpte_row_height[],
+		unsigned int dpte_row_height_chroma[],
+		unsigned int meta_row_width[],
+		unsigned int meta_row_width_chroma[],
+		unsigned int meta_row_height[],
+		unsigned int meta_row_height_chroma[],
+		unsigned int meta_req_width[],
+		unsigned int meta_req_width_chroma[],
+		unsigned int meta_req_height[],
+		unsigned int meta_req_height_chroma[],
+		unsigned int dpte_group_bytes[],
+		unsigned int PTERequestSizeY[],
+		unsigned int PTERequestSizeC[],
+		unsigned int PixelPTEReqWidthY[],
+		unsigned int PixelPTEReqHeightY[],
+		unsigned int PixelPTEReqWidthC[],
+		unsigned int PixelPTEReqHeightC[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
 		double DST_Y_PER_PTE_ROW_NOM_L[],
 		double DST_Y_PER_PTE_ROW_NOM_C[],
 		double DST_Y_PER_META_ROW_NOM_L[],
@@ -5509,7 +5516,7 @@ static void CalculateMetaAndPTETimes(
 			} else {
 				meta_chunk_width_chroma = MetaChunkSize * 1024 * 256 / BytePerPixelC[k] / meta_row_height_chroma[k];
 				min_meta_chunk_width_chroma = MinMetaChunkSizeBytes * 256 / BytePerPixelC[k] / meta_row_height_chroma[k];
-				meta_chunk_per_row_int_chroma = (double) meta_row_width_chroma[k] / meta_chunk_width_chroma;
+				meta_chunk_per_row_int_chroma = (unsigned int)((double) meta_row_width_chroma[k] / meta_chunk_width_chroma);
 				meta_row_remainder_chroma = meta_row_width_chroma[k] % meta_chunk_width_chroma;
 				if (SourceScan[k] != dm_vert) {
 					meta_chunk_threshold_chroma = 2 * min_meta_chunk_width_chroma - meta_req_width_chroma[k];
@@ -5542,7 +5549,7 @@ static void CalculateMetaAndPTETimes(
 			} else {
 				dpte_group_width_luma = dpte_group_bytes[k] / PTERequestSizeY[k] * PixelPTEReqHeightY[k];
 			}
-			dpte_groups_per_row_luma_ub = dml_ceil(1.0 * dpte_row_width_luma_ub[k] / dpte_group_width_luma, 1);
+			dpte_groups_per_row_luma_ub = (unsigned int)dml_ceil(1.0 * dpte_row_width_luma_ub[k] / dpte_group_width_luma, 1);
 			time_per_pte_group_nom_luma[k] = DST_Y_PER_PTE_ROW_NOM_L[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_luma_ub;
 			time_per_pte_group_vblank_luma[k] = DestinationLinesToRequestRowInVBlank[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_luma_ub;
 			time_per_pte_group_flip_luma[k] = DestinationLinesToRequestRowInImmediateFlip[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_luma_ub;
@@ -5556,7 +5563,7 @@ static void CalculateMetaAndPTETimes(
 				} else {
 					dpte_group_width_chroma = dpte_group_bytes[k] / PTERequestSizeC[k] * PixelPTEReqHeightC[k];
 				}
-				dpte_groups_per_row_chroma_ub = dml_ceil(1.0 * dpte_row_width_chroma_ub[k] / dpte_group_width_chroma, 1);
+				dpte_groups_per_row_chroma_ub = (unsigned int)dml_ceil(1.0 * dpte_row_width_chroma_ub[k] / dpte_group_width_chroma, 1);
 				time_per_pte_group_nom_chroma[k] = DST_Y_PER_PTE_ROW_NOM_C[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_chroma_ub;
 				time_per_pte_group_vblank_chroma[k] = DestinationLinesToRequestRowInVBlank[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_chroma_ub;
 				time_per_pte_group_flip_chroma[k] = DestinationLinesToRequestRowInImmediateFlip[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_chroma_ub;
@@ -5577,18 +5584,18 @@ static void CalculateVMGroupAndRequestTimes(
 		bool GPUVMEnable,
 		unsigned int GPUVMMaxPageTableLevels,
 		unsigned int HTotal[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		double DestinationLinesToRequestVMInVBlank[],
 		double DestinationLinesToRequestVMInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
-		int vm_group_bytes[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
+		unsigned int vm_group_bytes[],
 		unsigned int dpde0_bytes_per_frame_ub_l[],
 		unsigned int dpde0_bytes_per_frame_ub_c[],
-		int meta_pte_bytes_per_frame_ub_l[],
-		int meta_pte_bytes_per_frame_ub_c[],
+		unsigned int meta_pte_bytes_per_frame_ub_l[],
+		unsigned int meta_pte_bytes_per_frame_ub_c[],
 		double TimePerVMGroupVBlank[],
 		double TimePerVMGroupFlip[],
 		double TimePerVMRequestVBlank[],
@@ -5604,32 +5611,32 @@ static void CalculateVMGroupAndRequestTimes(
 		if (GPUVMEnable == true && (DCCEnable[k] == true || GPUVMMaxPageTableLevels > 1)) {
 			if (DCCEnable[k] == false) {
 				if (BytePerPixelC[k] > 0) {
-					num_group_per_lower_vm_stage = dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k])
+					num_group_per_lower_vm_stage = (int)(dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k])
 						/ (double) (vm_group_bytes[k]), 1) + dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k])
-									/ (double) (vm_group_bytes[k]), 1);
+									/ (double) (vm_group_bytes[k]), 1));
 				} else {
-					num_group_per_lower_vm_stage = dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k])
+					num_group_per_lower_vm_stage = (int)dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k])
 							/ (double) (vm_group_bytes[k]), 1);
 				}
 			} else {
 				if (GPUVMMaxPageTableLevels == 1) {
 					if (BytePerPixelC[k] > 0) {
-						num_group_per_lower_vm_stage = dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k])
+						num_group_per_lower_vm_stage = (int)(dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k])
 							/ (double) (vm_group_bytes[k]), 1) + dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k])
-									/ (double) (vm_group_bytes[k]), 1);
+									/ (double) (vm_group_bytes[k]), 1));
 					} else {
-						num_group_per_lower_vm_stage = dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k])
+						num_group_per_lower_vm_stage = (int)dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k])
 							/ (double) (vm_group_bytes[k]), 1);
 					}
 				} else {
 					if (BytePerPixelC[k] > 0) {
-						num_group_per_lower_vm_stage = 2 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+						num_group_per_lower_vm_stage = (int)(2 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
 								+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1)
 								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+								+ (int)dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 					} else {
-						num_group_per_lower_vm_stage = 1 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+						num_group_per_lower_vm_stage = (int)(1 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1));
 					}
 				}
 			}
@@ -5686,34 +5693,34 @@ static void CalculateVMGroupAndRequestTimes(
 }
 
 static void CalculateStutterEfficiency(
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		long ROBBufferSizeInKByte,
 		double TotalDataReadBandwidth,
 		double DCFCLK,
 		double ReturnBW,
 		double SRExitTime,
 		bool SynchronizedVBlank,
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		unsigned int DETBufferSizeY[],
-		int BytePerPixelY[],
+		unsigned int BytePerPixelY[],
 		double BytePerPixelDETY[],
 		double SwathWidthY[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		double DCCRateLuma[],
 		double DCCRateChroma[],
-		int HTotal[],
-		int VTotal[],
+		unsigned int HTotal[],
+		unsigned int VTotal[],
 		double PixelClock[],
 		double VRatio[],
 		enum scan_direction_class SourceScan[],
-		int BlockHeight256BytesY[],
-		int BlockWidth256BytesY[],
-		int BlockHeight256BytesC[],
-		int BlockWidth256BytesC[],
-		int DCCYMaxUncompressedBlock[],
-		int DCCCMaxUncompressedBlock[],
-		int VActive[],
+		unsigned int BlockHeight256BytesY[],
+		unsigned int BlockWidth256BytesY[],
+		unsigned int BlockHeight256BytesC[],
+		unsigned int BlockWidth256BytesC[],
+		unsigned int DCCYMaxUncompressedBlock[],
+		unsigned int DCCCMaxUncompressedBlock[],
+		unsigned int VActive[],
 		bool DCCEnable[],
 		bool WritebackEnable[],
 		double ReadBandwidthPlaneLuma[],
@@ -5840,49 +5847,49 @@ static void CalculateStutterEfficiency(
 
 static void CalculateSwathAndDETConfiguration(
 		bool ForceSingleDPP,
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		unsigned int DETBufferSizeInKByte,
 		double MaximumSwathWidthLuma[],
 		double MaximumSwathWidthChroma[],
 		enum scan_direction_class SourceScan[],
 		enum source_format_class SourcePixelFormat[],
 		enum dm_swizzle_mode SurfaceTiling[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BlendingAndTiming[],
-		int BytePerPixY[],
-		int BytePerPixC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
 		double BytePerPixDETY[],
 		double BytePerPixDETC[],
-		int HActive[],
+		unsigned int HActive[],
 		double HRatio[],
 		double HRatioChroma[],
-		int DPPPerPlane[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[],
+		unsigned int DPPPerPlane[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[],
 		double SwathWidth[],
 		double SwathWidthChroma[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		unsigned int DETBufferSizeY[],
 		unsigned int DETBufferSizeC[],
 		bool ViewportSizeSupportPerPlane[],
 		bool *ViewportSizeSupport)
 {
 	(void)HRatioChroma;
-	int MaximumSwathHeightY[DC__NUM_DPP__MAX] = { 0 };
-	int MaximumSwathHeightC[DC__NUM_DPP__MAX] = { 0 };
-	int MinimumSwathHeightY = 0;
-	int MinimumSwathHeightC = 0;
+	unsigned int MaximumSwathHeightY[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int MaximumSwathHeightC[DC__NUM_DPP__MAX] = { 0 };
+	unsigned int MinimumSwathHeightY = 0;
+	unsigned int MinimumSwathHeightC = 0;
 	long RoundedUpMaxSwathSizeBytesY = 0;
 	long RoundedUpMaxSwathSizeBytesC = 0;
 	long RoundedUpMinSwathSizeBytesY = 0;
@@ -5891,7 +5898,7 @@ static void CalculateSwathAndDETConfiguration(
 	long RoundedUpSwathSizeBytesC = 0;
 	double SwathWidthSingleDPP[DC__NUM_DPP__MAX] = { 0 };
 	double SwathWidthSingleDPPChroma[DC__NUM_DPP__MAX] = { 0 };
-	int k;
+	unsigned int k;
 
 	CalculateSwathWidth(
 			ForceSingleDPP,
@@ -5962,39 +5969,39 @@ static void CalculateSwathAndDETConfiguration(
 			}
 		}
 
-		RoundedUpMaxSwathSizeBytesY = swath_width_luma_ub[k] * BytePerPixDETY[k]
-				* MaximumSwathHeightY[k];
-		RoundedUpMinSwathSizeBytesY = swath_width_luma_ub[k] * BytePerPixDETY[k]
-				* MinimumSwathHeightY;
+		RoundedUpMaxSwathSizeBytesY = (long)(swath_width_luma_ub[k] * BytePerPixDETY[k]
+				* MaximumSwathHeightY[k]);
+		RoundedUpMinSwathSizeBytesY = (long)(swath_width_luma_ub[k] * BytePerPixDETY[k]
+				* MinimumSwathHeightY);
 		if (SourcePixelFormat[k] == dm_420_10) {
-			RoundedUpMaxSwathSizeBytesY = dml_ceil((double) RoundedUpMaxSwathSizeBytesY, 256);
-			RoundedUpMinSwathSizeBytesY = dml_ceil((double) RoundedUpMinSwathSizeBytesY, 256);
+			RoundedUpMaxSwathSizeBytesY = (long)dml_ceil((double) RoundedUpMaxSwathSizeBytesY, 256);
+			RoundedUpMinSwathSizeBytesY = (long)dml_ceil((double) RoundedUpMinSwathSizeBytesY, 256);
 		}
-		RoundedUpMaxSwathSizeBytesC = swath_width_chroma_ub[k] * BytePerPixDETC[k]
-				* MaximumSwathHeightC[k];
-		RoundedUpMinSwathSizeBytesC = swath_width_chroma_ub[k] * BytePerPixDETC[k]
-				* MinimumSwathHeightC;
+		RoundedUpMaxSwathSizeBytesC = (long)(swath_width_chroma_ub[k] * BytePerPixDETC[k]
+				* MaximumSwathHeightC[k]);
+		RoundedUpMinSwathSizeBytesC = (long)(swath_width_chroma_ub[k] * BytePerPixDETC[k]
+				* MinimumSwathHeightC);
 		if (SourcePixelFormat[k] == dm_420_10) {
-			RoundedUpMaxSwathSizeBytesC = dml_ceil(RoundedUpMaxSwathSizeBytesC, 256);
-			RoundedUpMinSwathSizeBytesC = dml_ceil(RoundedUpMinSwathSizeBytesC, 256);
+			RoundedUpMaxSwathSizeBytesC = (long)dml_ceil(RoundedUpMaxSwathSizeBytesC, 256);
+			RoundedUpMinSwathSizeBytesC = (long)dml_ceil(RoundedUpMinSwathSizeBytesC, 256);
 		}
 
 		if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
-				<= DETBufferSizeInKByte * 1024 / 2) {
+				<= (long)DETBufferSizeInKByte * 1024 / 2) {
 			SwathHeightY[k] = MaximumSwathHeightY[k];
 			SwathHeightC[k] = MaximumSwathHeightC[k];
 			RoundedUpSwathSizeBytesY = RoundedUpMaxSwathSizeBytesY;
 			RoundedUpSwathSizeBytesC = RoundedUpMaxSwathSizeBytesC;
 		} else if (RoundedUpMaxSwathSizeBytesY >= 1.5 * RoundedUpMaxSwathSizeBytesC
 				&& RoundedUpMinSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
-						<= DETBufferSizeInKByte * 1024 / 2) {
+						<= (long)DETBufferSizeInKByte * 1024 / 2) {
 			SwathHeightY[k] = MinimumSwathHeightY;
 			SwathHeightC[k] = MaximumSwathHeightC[k];
 			RoundedUpSwathSizeBytesY = RoundedUpMinSwathSizeBytesY;
 			RoundedUpSwathSizeBytesC = RoundedUpMaxSwathSizeBytesC;
 		} else if (RoundedUpMaxSwathSizeBytesY < 1.5 * RoundedUpMaxSwathSizeBytesC
 				&& RoundedUpMaxSwathSizeBytesY + RoundedUpMinSwathSizeBytesC
-						<= DETBufferSizeInKByte * 1024 / 2) {
+						<= (long)DETBufferSizeInKByte * 1024 / 2) {
 			SwathHeightY[k] = MaximumSwathHeightY[k];
 			SwathHeightC[k] = MinimumSwathHeightC;
 			RoundedUpSwathSizeBytesY = RoundedUpMaxSwathSizeBytesY;
@@ -6018,7 +6025,7 @@ static void CalculateSwathAndDETConfiguration(
 		}
 
 		if (RoundedUpMinSwathSizeBytesY + RoundedUpMinSwathSizeBytesC
-				> DETBufferSizeInKByte * 1024 / 2
+				> (long)DETBufferSizeInKByte * 1024 / 2
 				|| SwathWidth[k] > MaximumSwathWidthLuma[k]
 				|| (SwathHeightC[k] > 0
 						&& SwathWidthChroma[k] > MaximumSwathWidthChroma[k])) {
@@ -6032,7 +6039,7 @@ static void CalculateSwathAndDETConfiguration(
 
 static void CalculateSwathWidth(
 		bool ForceSingleDPP,
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		enum source_format_class SourcePixelFormat[],
 		enum scan_direction_class SourceScan[],
 		unsigned int ViewportWidth[],
@@ -6042,22 +6049,22 @@ static void CalculateSwathWidth(
 		unsigned int SurfaceHeightY[],
 		unsigned int SurfaceHeightC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BytePerPixY[],
-		int BytePerPixC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
-		int BlendingAndTiming[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
+		unsigned int BlendingAndTiming[],
 		unsigned int HActive[],
 		double HRatio[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		double SwathWidthSingleDPPY[],
 		double SwathWidthSingleDPPC[],
 		double SwathWidthY[],
 		double SwathWidthC[],
-		int MaximumSwathHeightY[],
-		int MaximumSwathHeightC[],
+		unsigned int MaximumSwathHeightY[],
+		unsigned int MaximumSwathHeightC[],
 		unsigned int swath_width_luma_ub[],
 		unsigned int swath_width_chroma_ub[])
 {
@@ -6107,17 +6114,17 @@ static void CalculateSwathWidth(
 			SwathWidthC[k] = SwathWidthSingleDPPC[k];
 		}
 
-		surface_width_ub_l  = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
-		surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
+		surface_width_ub_l  = (long)dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
+		surface_height_ub_l = (long)dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
 
 		if (SourceScan[k] != dm_vert) {
 			MaximumSwathHeightY[k] = Read256BytesBlockHeightY[k];
 			MaximumSwathHeightC[k] = Read256BytesBlockHeightC[k];
-			swath_width_luma_ub[k] = dml_min(surface_width_ub_l, (long) dml_ceil(SwathWidthY[k] - 1,
+			swath_width_luma_ub[k] = (unsigned int)dml_min(surface_width_ub_l, (long) dml_ceil(SwathWidthY[k] - 1,
 					Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]);
 			if (BytePerPixC[k] > 0) {
-				surface_width_ub_c  = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
-				swath_width_chroma_ub[k] = dml_min(surface_width_ub_c, (long) dml_ceil(SwathWidthC[k] - 1,
+				surface_width_ub_c  = (long)dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
+				swath_width_chroma_ub[k] = (unsigned int)dml_min(surface_width_ub_c, (long) dml_ceil(SwathWidthC[k] - 1,
 						Read256BytesBlockWidthC[k]) + Read256BytesBlockWidthC[k]);
 			} else {
 				swath_width_chroma_ub[k] = 0;
@@ -6125,11 +6132,11 @@ static void CalculateSwathWidth(
 		} else {
 			MaximumSwathHeightY[k] = Read256BytesBlockWidthY[k];
 			MaximumSwathHeightC[k] = Read256BytesBlockWidthC[k];
-			swath_width_luma_ub[k] = dml_min(surface_height_ub_l, (long) dml_ceil(SwathWidthY[k] - 1,
+			swath_width_luma_ub[k] = (unsigned int)dml_min(surface_height_ub_l, (long) dml_ceil(SwathWidthY[k] - 1,
 					Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
 			if (BytePerPixC[k] > 0) {
-				surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
-				swath_width_chroma_ub[k] = dml_min(surface_height_ub_c, (long) dml_ceil(SwathWidthC[k] - 1,
+				surface_height_ub_c = (long)dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
+				swath_width_chroma_ub[k] = (unsigned int)dml_min(surface_height_ub_c, (long) dml_ceil(SwathWidthC[k] - 1,
 						Read256BytesBlockHeightC[k]) + Read256BytesBlockHeightC[k]);
 			} else {
 				swath_width_chroma_ub[k] = 0;
@@ -6150,8 +6157,8 @@ static double CalculateExtraLatency(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
 		double HostVMMinPageSize,
@@ -6185,9 +6192,9 @@ static double CalculateExtraLatencyBytes(
 		int MetaChunkSize,
 		bool GPUVMEnable,
 		bool HostVMEnable,
-		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfActivePlanes,
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyPixelMixedWithVMData,
 		double PercentOfIdealDRAMFabricAndSDPPortBWReceivedAfterUrgLatencyVMDataOnly,
 		double HostVMMinPageSize,
@@ -6203,9 +6210,9 @@ static double CalculateExtraLatencyBytes(
 		if (HostVMMinPageSize < 2048) {
 			HostVMDynamicLevels = HostVMMaxNonCachedPageTableLevels;
 		} else if (HostVMMinPageSize >= 2048 && HostVMMinPageSize < 1048576) {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
+			HostVMDynamicLevels = (int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
 		} else {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
+			HostVMDynamicLevels = (int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
 		}
 	} else {
 		HostVMInefficiencyFactor = 1;
@@ -6244,7 +6251,7 @@ static double CalculateUrgentLatency(
 static noinline_for_stack void UseMinimumDCFCLK(
 		struct display_mode_lib *mode_lib,
 		struct vba_vars_st *v,
-		int MaxPrefetchMode,
+		unsigned int MaxPrefetchMode,
 		int ReorderingBytes)
 {
 	double   NormalEfficiency = 0;
@@ -6269,7 +6276,7 @@ static noinline_for_stack void UseMinimumDCFCLK(
 			double ExtraLatencyBytes = 0;
 			double ExtraLatencyCycles = 0;
 			double DCFCLKRequiredForPeakBandwidth = 0;
-			int NoOfDPPState[DC__NUM_DPP__MAX] = { 0 };
+			unsigned int NoOfDPPState[DC__NUM_DPP__MAX] = { 0 };
 			double MinimumTvmPlus2Tr0 = 0;
 
 			TotalMaxPrefetchFlipDPTERowBandwidth[i][j] = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c
index 472ac5ee165f1..a373df0aebf1f 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn30/display_rq_dlg_calc_30.c
@@ -120,13 +120,13 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib,
 
 	extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_l), &rq_param->sizing.rq_l);
 
-	rq_regs->rq_regs_l.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height),
-		1) - 3;
+	rq_regs->rq_regs_l.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height),
+		1) - 3);
 
 	if (rq_param->yuv420) {
 		extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_c), &rq_param->sizing.rq_c);
-		rq_regs->rq_regs_c.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height),
-			1) - 3;
+		rq_regs->rq_regs_c.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height),
+			1) - 3);
 	}
 
 	rq_regs->rq_regs_l.swath_height = dml_log2(rq_param->dlg.rq_l.swath_height);
@@ -146,11 +146,11 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib,
 	if (rq_param->yuv420) {
 	if ((double)rq_param->misc.rq_l.stored_swath_bytes
 			/ (double)rq_param->misc.rq_c.stored_swath_bytes <= 1.5) {
-			detile_buf_plane1_addr = (detile_buf_size_in_bytes / 2.0 / 64.0); // half to chroma
+			detile_buf_plane1_addr = (unsigned int)(detile_buf_size_in_bytes / 2.0 / 64.0); // half to chroma
 		} else {
-			detile_buf_plane1_addr = dml_round_to_multiple((unsigned int)((2.0 * detile_buf_size_in_bytes) / 3.0),
+			detile_buf_plane1_addr = (unsigned int)(dml_round_to_multiple((unsigned int)((2.0 * detile_buf_size_in_bytes) / 3.0),
 				256,
-				0) / 64.0; // 2/3 to chroma
+				0) / 64.0); // 2/3 to chroma
 		}
 	}
 	rq_regs->plane1_base_address = detile_buf_plane1_addr;
@@ -177,10 +177,10 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib,
 	full_swath_bytes_packed_c = rq_param->misc.rq_c.full_swath_bytes;
 
 	if (rq_param->yuv420_10bpc) {
-		full_swath_bytes_packed_l = dml_round_to_multiple(rq_param->misc.rq_l.full_swath_bytes * 2.0 / 3.0,
+		full_swath_bytes_packed_l = (unsigned int)dml_round_to_multiple((unsigned int)(rq_param->misc.rq_l.full_swath_bytes * 2.0 / 3.0),
 			256,
 			1) + 256;
-		full_swath_bytes_packed_c = dml_round_to_multiple(rq_param->misc.rq_c.full_swath_bytes * 2.0 / 3.0,
+		full_swath_bytes_packed_c = (unsigned int)dml_round_to_multiple((unsigned int)(rq_param->misc.rq_c.full_swath_bytes * 2.0 / 3.0),
 			256,
 			1) + 256;
 	}
@@ -400,8 +400,8 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
 	// "/2" is like square root
 	// blk is vertical biased
 	if (tiling != dm_sw_linear)
-		log2_blk_height = log2_blk256_height
-		+ dml_ceil((double)(log2_blk_bytes - 8) / 2.0, 1);
+		log2_blk_height = (unsigned int)(log2_blk256_height
+		+ dml_ceil((double)(log2_blk_bytes - 8) / 2.0, 1));
 	else
 		log2_blk_height = 0;  // blk height of 1
 
@@ -573,7 +573,7 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
 	if (surf_linear) {
 		unsigned int dpte_row_height = 0;
 
-		log2_dpte_row_height_linear = dml_floor(dml_log2(dpte_buf_in_pte_reqs * dpte_req_width / data_pitch), 1);
+		log2_dpte_row_height_linear = (unsigned int)dml_floor(dml_log2(dpte_buf_in_pte_reqs * dpte_req_width / data_pitch), 1);
 
 		dml_print("DML_DLG: %s: is_chroma                   = %d\n", __func__, is_chroma);
 		dml_print("DML_DLG: %s: dpte_buf_in_pte_reqs        = %d\n", __func__, dpte_buf_in_pte_reqs);
@@ -645,7 +645,7 @@ static void get_meta_and_pte_attr(struct display_mode_lib *mode_lib,
 
 	// since dpte groups are only aligned to dpte_req_width and not dpte_group_width,
 	// the upper bound for the dpte groups per row is as follows.
-	rq_dlg_param->dpte_groups_per_row_ub = dml_ceil((double)dpte_row_width_ub / dpte_group_width,
+	rq_dlg_param->dpte_groups_per_row_ub = (unsigned int)dml_ceil((double)dpte_row_width_ub / dpte_group_width,
 		1);
 }
 
@@ -670,7 +670,7 @@ static void get_surf_rq_param(struct display_mode_lib *mode_lib,
 		vp_height = pipe_param->src.viewport_height_c;
 		data_pitch = pipe_param->src.data_pitch_c;
 		meta_pitch = pipe_param->src.meta_pitch_c;
-		surface_height = pipe_param->src.surface_height_y / 2.0;
+		surface_height = (unsigned int)(pipe_param->src.surface_height_y / 2.0);
 	} else {
 		vp_width = pipe_param->src.viewport_width / ppe;
 		vp_height = pipe_param->src.viewport_height;
@@ -687,18 +687,18 @@ static void get_surf_rq_param(struct display_mode_lib *mode_lib,
 		access_dir = (pipe_param->src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
 		hactive_odm  = pipe_param->dest.hactive / ((unsigned int) pipe_param->dest.odm_combine*2);
 		if (is_chroma) {
-			full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width;
-			src_hactive_odm  = pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_odm;
+			full_src_vp_width = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width);
+			src_hactive_odm  = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_odm);
 		} else {
-			full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width;
-			src_hactive_odm  = pipe_param->scale_ratio_depth.hscl_ratio * hactive_odm;
+			full_src_vp_width = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width);
+			src_hactive_odm  = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio * hactive_odm);
 		}
 
 		if (access_dir == 0) {
-			vp_width = dml_min(full_src_vp_width, src_hactive_odm);
+			vp_width = (unsigned int)dml_min(full_src_vp_width, src_hactive_odm);
 			dml_print("DML_DLG: %s: vp_width = %d\n", __func__, vp_width);
 		} else {
-			vp_height = dml_min(full_src_vp_width, src_hactive_odm);
+			vp_height = (unsigned int)dml_min(full_src_vp_width, src_hactive_odm);
 			dml_print("DML_DLG: %s: vp_height = %d\n", __func__, vp_height);
 		}
 		dml_print("DML_DLG: %s: full_src_vp_width = %d\n", __func__, full_src_vp_width);
@@ -833,7 +833,7 @@ static void calculate_ttu_cursor(struct display_mode_lib *mode_lib,
 				cur_req_size = 256;
 		}
 
-		cur_req_width = (double)cur_req_size / ((double)cur_bit_per_pixel / 8.0);
+		cur_req_width = (unsigned int)((double)cur_req_size / ((double)cur_bit_per_pixel / 8.0));
 		cur_width_ub = dml_ceil((double)cur_src_width / (double)cur_req_width, 1)
 			* (double)cur_req_width;
 		cur_req_per_width = cur_width_ub / (double)cur_req_width;
@@ -1123,11 +1123,11 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 	if (dout->dsc_enable) {
 		double dsc_delay = get_dsc_delay(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
 
-		dispclk_delay_subtotal += dsc_delay;
+		dispclk_delay_subtotal += (unsigned int)dsc_delay;
 	}
 
-	pixel_rate_delay_subtotal = dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
-		+ dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz;
+	pixel_rate_delay_subtotal = (unsigned int)(dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz)
+		+ (unsigned int)(dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz);
 
 	vstartup_start = dst->vstartup_start;
 	if (interlaced) {
@@ -1167,8 +1167,8 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 			min_vblank);
 	}
 
-	dst_x_after_scaler = get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
-	dst_y_after_scaler = get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+	dst_x_after_scaler = (unsigned int)get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+	dst_y_after_scaler = (int)get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
 	if (dst_y_after_scaler < 0)
 		dst_y_after_scaler = 0;
 
@@ -1552,7 +1552,7 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 	// Assignment to register structures
 	disp_dlg_regs->dst_y_after_scaler = dst_y_after_scaler; // in terms of line
 	ASSERT(disp_dlg_regs->dst_y_after_scaler < (unsigned int)8);
-	disp_dlg_regs->refcyc_x_after_scaler = dst_x_after_scaler * ref_freq_to_pix_freq; // in terms of refclk
+	disp_dlg_regs->refcyc_x_after_scaler = (unsigned int)(dst_x_after_scaler * ref_freq_to_pix_freq); // in terms of refclk
 	ASSERT(disp_dlg_regs->refcyc_x_after_scaler < (unsigned int)dml_pow(2, 13));
 	disp_dlg_regs->dst_y_prefetch = (unsigned int)(dst_y_prefetch * dml_pow(2, 2));
 	disp_dlg_regs->dst_y_per_vm_vblank = (unsigned int)(dst_y_per_vm_vblank * dml_pow(2, 2));
@@ -1601,23 +1601,23 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 			* htotal * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_c;
 	}
 
-	disp_dlg_regs->refcyc_per_vm_group_vblank   = get_refcyc_per_vm_group_vblank(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz;
-	disp_dlg_regs->refcyc_per_vm_group_flip	 = get_refcyc_per_vm_group_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz;
-	disp_dlg_regs->refcyc_per_vm_req_vblank	 = get_refcyc_per_vm_req_vblank(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10);
-	disp_dlg_regs->refcyc_per_vm_req_flip	   = get_refcyc_per_vm_req_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10);
+	disp_dlg_regs->refcyc_per_vm_group_vblank   = (unsigned int)(get_refcyc_per_vm_group_vblank(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz);
+	disp_dlg_regs->refcyc_per_vm_group_flip	 = (unsigned int)(get_refcyc_per_vm_group_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz);
+	disp_dlg_regs->refcyc_per_vm_req_vblank	 = (unsigned int)(get_refcyc_per_vm_req_vblank(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10));
+	disp_dlg_regs->refcyc_per_vm_req_flip	   = (unsigned int)(get_refcyc_per_vm_req_flip(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10));
 
 	// Clamp to max for now
 	if (disp_dlg_regs->refcyc_per_vm_group_vblank >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_group_vblank = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_group_vblank = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (disp_dlg_regs->refcyc_per_vm_group_flip >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_group_flip = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_group_flip = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (disp_dlg_regs->refcyc_per_vm_req_vblank >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_req_vblank = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_req_vblank = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (disp_dlg_regs->refcyc_per_vm_req_flip >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_req_flip = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_req_flip = (unsigned int)(dml_pow(2, 23) - 1);
 
 	disp_dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int)((double)dpte_row_height_l
 		/ (double)vratio_l * dml_pow(2, 2));
@@ -1646,12 +1646,12 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 		/ (double)vratio_l * (double)htotal * ref_freq_to_pix_freq
 		/ (double)dpte_groups_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_pte_group_nom_l >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_pte_group_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 	disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)((double)meta_row_height_l
 		/ (double)vratio_l * (double)htotal * ref_freq_to_pix_freq
 		/ (double)meta_chunks_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_meta_chunk_nom_l >= (unsigned int)dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (dual_plane) {
 		disp_dlg_regs->refcyc_per_pte_group_nom_c =
@@ -1659,7 +1659,7 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 				* (double)htotal * ref_freq_to_pix_freq
 				/ (double)dpte_groups_per_row_ub_c);
 		if (disp_dlg_regs->refcyc_per_pte_group_nom_c >= (unsigned int)dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_pte_group_nom_c = dml_pow(2, 23) - 1;
+			disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 
 		// TODO: Is this the right calculation? Does htotal need to be halved?
 		disp_dlg_regs->refcyc_per_meta_chunk_nom_c =
@@ -1667,7 +1667,7 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 				* (double)htotal * ref_freq_to_pix_freq
 				/ (double)meta_chunks_per_row_ub_c);
 		if (disp_dlg_regs->refcyc_per_meta_chunk_nom_c >= (unsigned int)dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = dml_pow(2, 23) - 1;
+			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 	}
 
 	disp_dlg_regs->refcyc_per_line_delivery_pre_l = (unsigned int)dml_floor(refcyc_per_line_delivery_pre_l,
@@ -1721,7 +1721,7 @@ static void dml_rq_dlg_get_dlg_params(struct display_mode_lib *mode_lib,
 	disp_ttu_regs->qos_ramp_disable_c = 0;
 	disp_ttu_regs->qos_ramp_disable_cur0 = 0;
 
-	disp_ttu_regs->min_ttu_vblank = min_ttu_vblank * refclk_freq_in_mhz;
+	disp_ttu_regs->min_ttu_vblank = (unsigned int)(min_ttu_vblank * refclk_freq_in_mhz);
 	ASSERT(disp_ttu_regs->min_ttu_vblank < dml_pow(2, 24));
 
 	print__ttu_regs_st(mode_lib, disp_ttu_regs);
@@ -1755,7 +1755,7 @@ void dml30_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	dlg_sys_param.total_flip_bw = get_total_immediate_flip_bw(mode_lib,
 		e2e_pipe_param,
 		num_pipes);
-	dlg_sys_param.total_flip_bytes = get_total_immediate_flip_bytes(mode_lib,
+	dlg_sys_param.total_flip_bytes = (unsigned int)get_total_immediate_flip_bytes(mode_lib,
 		e2e_pipe_param,
 		num_pipes);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c
index 1aaa77265eede..c4b73acd7140d 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn301/dcn301_fpu.c
@@ -298,7 +298,7 @@ static void calculate_wm_set_for_vlevel(int vlevel,
 {
 	double dram_clock_change_latency_cached = dml->soc.dram_clock_change_latency_us;
 
-	ASSERT(vlevel < dml->soc.num_states);
+	ASSERT(vlevel < (int)dml->soc.num_states);
 	/* only pipe 0 is read for voltage and dcf/soc clocks */
 	pipes[0].clks_cfg.voltage = vlevel;
 	pipes[0].clks_cfg.dcfclk_mhz = dml->soc.clock_limits[vlevel].dcfclk_mhz;
@@ -308,14 +308,14 @@ static void calculate_wm_set_for_vlevel(int vlevel,
 	dml->soc.sr_exit_time_us = table_entry->sr_exit_time_us;
 	dml->soc.sr_enter_plus_exit_time_us = table_entry->sr_enter_plus_exit_time_us;
 
-	wm_set->urgent_ns = get_wm_urgent(dml, pipes, pipe_cnt) * 1000;
-	wm_set->cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(dml, pipes, pipe_cnt) * 1000;
-	wm_set->cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(dml, pipes, pipe_cnt) * 1000;
-	wm_set->cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(dml, pipes, pipe_cnt) * 1000;
-	wm_set->pte_meta_urgent_ns = get_wm_memory_trip(dml, pipes, pipe_cnt) * 1000;
-	wm_set->frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(dml, pipes, pipe_cnt) * 1000;
-	wm_set->frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(dml, pipes, pipe_cnt) * 1000;
-	wm_set->urgent_latency_ns = get_urgent_latency(dml, pipes, pipe_cnt) * 1000;
+	wm_set->urgent_ns = (uint32_t)(get_wm_urgent(dml, pipes, pipe_cnt) * 1000);
+	wm_set->cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(dml, pipes, pipe_cnt) * 1000);
+	wm_set->cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(dml, pipes, pipe_cnt) * 1000);
+	wm_set->cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(dml, pipes, pipe_cnt) * 1000);
+	wm_set->pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(dml, pipes, pipe_cnt) * 1000);
+	wm_set->frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(dml, pipes, pipe_cnt) * 1000);
+	wm_set->frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(dml, pipes, pipe_cnt) * 1000);
+	wm_set->urgent_latency_ns = (uint32_t)(get_urgent_latency(dml, pipes, pipe_cnt) * 1000);
 	dml->soc.dram_clock_change_latency_us = dram_clock_change_latency_cached;
 
 }
@@ -326,7 +326,8 @@ void dcn301_fpu_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_p
 	struct dcn301_resource_pool *pool = TO_DCN301_RES_POOL(dc->res_pool);
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
 	unsigned int i, closest_clk_lvl;
-	int j = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0;
+	int j = 0;
+	unsigned int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
 
 	dc_assert_fp_enabled();
 
@@ -405,8 +406,8 @@ void dcn301_fpu_set_wm_ranges(int i,
 {
 	dc_assert_fp_enabled();
 
-	ranges->reader_wm_sets[i].min_fill_clk_mhz = (i > 0) ? (loaded_bb->clock_limits[i - 1].dram_speed_mts / 16) + 1 : 0;
-	ranges->reader_wm_sets[i].max_fill_clk_mhz = loaded_bb->clock_limits[i].dram_speed_mts / 16;
+	ranges->reader_wm_sets[i].min_fill_clk_mhz = (uint16_t)((i > 0) ? (loaded_bb->clock_limits[i - 1].dram_speed_mts / 16) + 1 : 0);
+	ranges->reader_wm_sets[i].max_fill_clk_mhz = (uint16_t)(loaded_bb->clock_limits[i].dram_speed_mts / 16);
 }
 
 void dcn301_fpu_init_soc_bounding_box(struct bp_soc_bb_info bb_info)
@@ -429,7 +430,7 @@ void dcn301_fpu_calculate_wm_and_dlg(struct dc *dc,
 		int pipe_cnt,
 		int vlevel_req)
 {
-	int i, pipe_idx;
+	unsigned int i, pipe_idx;
 	int vlevel, vlevel_max;
 	struct wm_range_table_entry *table_entry;
 	struct clk_bw_params *bw_params = dc->clk_mgr->bw_params;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn302/dcn302_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn302/dcn302_fpu.c
index 8d7c59ec701dc..90a7b29ebe748 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn302/dcn302_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn302/dcn302_fpu.c
@@ -182,14 +182,14 @@ static void dcn302_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts,
 			bw_from_dram = (bw_from_dram1 < bw_from_dram2) ? bw_from_dram1 : bw_from_dram2;
 
 			if (optimal_fclk)
-				*optimal_fclk = bw_from_dram /
+				*optimal_fclk = (unsigned int)(bw_from_dram /
 				(dcn3_02_soc.fabric_datapath_to_dcn_data_return_bytes *
-				 (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100));
+				 (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 
 			if (optimal_dcfclk)
-				*optimal_dcfclk =  bw_from_dram /
+				*optimal_dcfclk = (unsigned int)(bw_from_dram /
 				(dcn3_02_soc.return_bus_width_bytes *
-				 (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100));
+				 (dcn3_02_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 }
 
 void dcn302_fpu_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params)
@@ -218,7 +218,7 @@ void dcn302_fpu_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_p
 	dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0;
 
 	if (bw_params->clk_table.entries[0].memclk_mhz) {
-		int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0, max_phyclk_mhz = 0;
+		unsigned int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0, max_phyclk_mhz = 0;
 
 		for (i = 0; i < MAX_NUM_DPM_LVL; i++) {
 			if (bw_params->clk_table.entries[i].dcfclk_mhz > max_dcfclk_mhz)
@@ -231,13 +231,13 @@ void dcn302_fpu_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_p
 				max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz;
 		}
 		if (!max_dcfclk_mhz)
-			max_dcfclk_mhz = dcn3_02_soc.clock_limits[0].dcfclk_mhz;
+			max_dcfclk_mhz = (int)dcn3_02_soc.clock_limits[0].dcfclk_mhz;
 		if (!max_dispclk_mhz)
-			max_dispclk_mhz = dcn3_02_soc.clock_limits[0].dispclk_mhz;
+			max_dispclk_mhz = (int)dcn3_02_soc.clock_limits[0].dispclk_mhz;
 		if (!max_dppclk_mhz)
-			max_dppclk_mhz = dcn3_02_soc.clock_limits[0].dppclk_mhz;
+			max_dppclk_mhz = (int)dcn3_02_soc.clock_limits[0].dppclk_mhz;
 		if (!max_phyclk_mhz)
-			max_phyclk_mhz = dcn3_02_soc.clock_limits[0].phyclk_mhz;
+			max_phyclk_mhz = (int)dcn3_02_soc.clock_limits[0].phyclk_mhz;
 
 		if (max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
 			/* If max DCFCLK is greater than the max DCFCLK STA target, insert into the DCFCLK STA target array */
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn303/dcn303_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn303/dcn303_fpu.c
index b5d3fd4c3694e..e42419c7868d2 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn303/dcn303_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn303/dcn303_fpu.c
@@ -178,13 +178,13 @@ static void dcn303_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts,
 	bw_from_dram = (bw_from_dram1 < bw_from_dram2) ? bw_from_dram1 : bw_from_dram2;
 
 	if (optimal_fclk)
-		*optimal_fclk = bw_from_dram /
+		*optimal_fclk = (unsigned int)(bw_from_dram /
 		(dcn3_03_soc.fabric_datapath_to_dcn_data_return_bytes *
-				(dcn3_03_soc.max_avg_sdp_bw_use_normal_percent / 100));
+				(dcn3_03_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 
 	if (optimal_dcfclk)
-		*optimal_dcfclk =  bw_from_dram /
-		(dcn3_03_soc.return_bus_width_bytes * (dcn3_03_soc.max_avg_sdp_bw_use_normal_percent / 100));
+		*optimal_dcfclk = (unsigned int)(bw_from_dram /
+		(dcn3_03_soc.return_bus_width_bytes * (dcn3_03_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 }
 
 
@@ -214,7 +214,7 @@ void dcn303_fpu_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_p
 	dc->dml.soc.dispclk_dppclk_vco_speed_mhz = dc->clk_mgr->dentist_vco_freq_khz / 1000.0;
 
 	if (bw_params->clk_table.entries[0].memclk_mhz) {
-		int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0, max_phyclk_mhz = 0;
+		unsigned int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0, max_phyclk_mhz = 0;
 
 		for (i = 0; i < MAX_NUM_DPM_LVL; i++) {
 			if (bw_params->clk_table.entries[i].dcfclk_mhz > max_dcfclk_mhz)
@@ -227,13 +227,13 @@ void dcn303_fpu_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_p
 				max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz;
 		}
 		if (!max_dcfclk_mhz)
-			max_dcfclk_mhz = dcn3_03_soc.clock_limits[0].dcfclk_mhz;
+			max_dcfclk_mhz = (int)dcn3_03_soc.clock_limits[0].dcfclk_mhz;
 		if (!max_dispclk_mhz)
-			max_dispclk_mhz = dcn3_03_soc.clock_limits[0].dispclk_mhz;
+			max_dispclk_mhz = (int)dcn3_03_soc.clock_limits[0].dispclk_mhz;
 		if (!max_dppclk_mhz)
-			max_dppclk_mhz = dcn3_03_soc.clock_limits[0].dppclk_mhz;
+			max_dppclk_mhz = (int)dcn3_03_soc.clock_limits[0].dppclk_mhz;
 		if (!max_phyclk_mhz)
-			max_phyclk_mhz = dcn3_03_soc.clock_limits[0].phyclk_mhz;
+			max_phyclk_mhz = (int)dcn3_03_soc.clock_limits[0].phyclk_mhz;
 
 		if (max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
 			dcfclk_sta_targets[num_dcfclk_sta_targets] = max_dcfclk_mhz;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
index ad23215da9f86..f9b2da6b2bb86 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/dcn31_fpu.c
@@ -485,9 +485,12 @@ void dcn31_calculate_wm_and_dlg_fp(
 		int pipe_cnt,
 		int vlevel)
 {
-	int i, pipe_idx, total_det = 0, active_hubp_count = 0;
+	int total_det = 0, active_hubp_count = 0;
+	unsigned int i, pipe_idx;
 	double dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
 	uint32_t cstate_enter_plus_exit_z8_ns;
+	uint32_t minimum_z8_residency_time_ns =
+			(uint32_t)dc->debug.minimum_z8_residency_time * 1000U;
 
 	dc_assert_fp_enabled();
 
@@ -499,7 +502,7 @@ void dcn31_calculate_wm_and_dlg_fp(
 	 * Override any clocks that can block S0i3 to min here
 	 */
 	if (pipe_cnt == 0) {
-		context->bw_ctx.bw.dcn.clk.dcfclk_khz = dcfclk; // always should be vlevel 0
+		context->bw_ctx.bw.dcn.clk.dcfclk_khz = (int)dcfclk; // always should be vlevel 0
 		return;
 	}
 
@@ -508,11 +511,11 @@ void dcn31_calculate_wm_and_dlg_fp(
 	pipes[0].clks_cfg.socclk_mhz = context->bw_ctx.dml.soc.clock_limits[vlevel].socclk_mhz;
 
 	cstate_enter_plus_exit_z8_ns =
-		get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+		(uint32_t)(get_wm_z8_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 
 	if (get_stutter_period(&context->bw_ctx.dml, pipes, pipe_cnt) < dc->debug.minimum_z8_residency_time &&
-			cstate_enter_plus_exit_z8_ns < dc->debug.minimum_z8_residency_time * 1000)
-		cstate_enter_plus_exit_z8_ns = dc->debug.minimum_z8_residency_time * 1000;
+				cstate_enter_plus_exit_z8_ns < minimum_z8_residency_time_ns)
+		cstate_enter_plus_exit_z8_ns = minimum_z8_residency_time_ns;
 
 	/* Set A:
 	 * All clocks min required
@@ -520,16 +523,16 @@ void dcn31_calculate_wm_and_dlg_fp(
 	 * Set A calculated last so that following calculations are based on Set A
 	 */
 	dc->res_pool->funcs->update_soc_for_wm_a(dc, context);
-	context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_z8_ns = cstate_enter_plus_exit_z8_ns;
-	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_z8_ns = get_wm_z8_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.a.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_z8_ns = (uint32_t)(get_wm_z8_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.a.urgent_latency_ns = (uint32_t)(get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 	context->bw_ctx.bw.dcn.watermarks.b = context->bw_ctx.bw.dcn.watermarks.a;
 	context->bw_ctx.bw.dcn.watermarks.c = context->bw_ctx.bw.dcn.watermarks.a;
 	context->bw_ctx.bw.dcn.watermarks.d = context->bw_ctx.bw.dcn.watermarks.a;
@@ -578,7 +581,7 @@ void dcn31_calculate_wm_and_dlg_fp(
 			continue;
 
 		context->res_ctx.pipe_ctx[i].det_buffer_size_kb =
-				get_det_buffer_size_kbytes(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+				(int)get_det_buffer_size_kbytes(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
 		if (context->res_ctx.pipe_ctx[i].det_buffer_size_kb > 384)
 			context->res_ctx.pipe_ctx[i].det_buffer_size_kb /= 2;
 		total_det += context->res_ctx.pipe_ctx[i].det_buffer_size_kb;
@@ -592,7 +595,7 @@ void dcn31_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa
 	struct _vcs_dpi_voltage_scaling_st *s = dc->scratch.update_bw_bounding_box.clock_limits;
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
 	unsigned int i, closest_clk_lvl;
-	int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
+	unsigned int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
 	int j;
 
 	dc_assert_fp_enabled();
@@ -668,7 +671,7 @@ void dcn31_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa
 void dcn315_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_params)
 {
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
-	int i, max_dispclk_mhz = 0, max_dppclk_mhz = 0;
+	unsigned int i, max_dispclk_mhz = 0, max_dppclk_mhz = 0;
 
 	dc_assert_fp_enabled();
 
@@ -731,7 +734,7 @@ void dcn316_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p
 	struct _vcs_dpi_voltage_scaling_st *s = dc->scratch.update_bw_bounding_box.clock_limits;
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
 	unsigned int i, closest_clk_lvl;
-	int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
+	unsigned int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
 	int j;
 
 	dc_assert_fp_enabled();
@@ -810,7 +813,7 @@ int dcn_get_max_non_odm_pix_rate_100hz(struct _vcs_dpi_soc_bounding_box_st *soc)
 {
 	dc_assert_fp_enabled();
 
-	return soc->clock_limits[0].dispclk_mhz * 10000.0 / (1.0 + soc->dcn_downspread_percent / 100.0);
+	return (int)(soc->clock_limits[0].dispclk_mhz * 10000.0 / (1.0 + soc->dcn_downspread_percent / 100.0));
 }
 
 int dcn_get_approx_det_segs_required_for_pstate(
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
index 9833467722b93..f9224a4332202 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_mode_vba_31.c
@@ -157,7 +157,7 @@ static bool CalculatePrefetchSchedule(
 		double *Tdmdl_vm,
 		double *Tdmdl,
 		double *TSetup,
-		int *VUpdateOffsetPix,
+		unsigned int *VUpdateOffsetPix,
 		double *VUpdateWidthPix,
 		double *VReadyOffsetPix);
 static double RoundToDFSGranularityUp(double Clock, double VCOSpeed);
@@ -218,19 +218,19 @@ static unsigned int CalculateVMAndRowBytes(
 		unsigned int *MetaRowByte,
 		unsigned int *PixelPTEBytesPerRow,
 		bool *PTEBufferSizeNotExceeded,
-		int *dpte_row_width_ub,
+		unsigned int *dpte_row_width_ub,
 		unsigned int *dpte_row_height,
 		unsigned int *MetaRequestWidth,
 		unsigned int *MetaRequestHeight,
 		unsigned int *meta_row_width,
 		unsigned int *meta_row_height,
-		int *vm_group_bytes,
+		unsigned int *vm_group_bytes,
 		unsigned int *dpte_group_bytes,
 		unsigned int *PixelPTEReqWidth,
 		unsigned int *PixelPTEReqHeight,
 		unsigned int *PTERequestSize,
-		int *DPDE0BytesFrame,
-		int *MetaPTEBytesFrame);
+		unsigned int *DPDE0BytesFrame,
+		unsigned int *MetaPTEBytesFrame);
 static double CalculateTWait(unsigned int PrefetchMode, double DRAMClockChangeLatency, double UrgentLatency, double SREnterPlusExitTime);
 static void CalculateRowBandwidth(
 		bool GPUVMEnable,
@@ -285,7 +285,7 @@ static void CalculateVupdateAndDynamicMetadataParameters(
 		double *Tdmbf,
 		double *Tdmec,
 		double *Tdmsks,
-		int *VUpdateOffsetPix,
+		unsigned int *VUpdateOffsetPix,
 		double *VUpdateWidthPix,
 		double *VReadyOffsetPix);
 
@@ -318,8 +318,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 static void CalculateDCFCLKDeepSleep(
 		struct display_mode_lib *mode_lib,
 		unsigned int NumberOfActivePlanes,
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		double VRatio[],
 		double VRatioChroma[],
 		double SwathWidthY[],
@@ -377,7 +377,7 @@ static void CalculatePixelDeliveryTimes(
 		double PSCL_THROUGHPUT[],
 		double PSCL_THROUGHPUT_CHROMA[],
 		double DPPCLK[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
 		unsigned int NumberOfCursors[],
 		unsigned int CursorWidth[][DC__NUM_CURSOR__MAX],
@@ -402,35 +402,35 @@ static void CalculateMetaAndPTETimes(
 		bool GPUVMEnable,
 		int MetaChunkSize,
 		int MinMetaChunkSizeBytes,
-		int HTotal[],
+		unsigned int HTotal[],
 		double VRatio[],
 		double VRatioChroma[],
 		double DestinationLinesToRequestRowInVBlank[],
 		double DestinationLinesToRequestRowInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
-		int dpte_row_height[],
-		int dpte_row_height_chroma[],
-		int meta_row_width[],
-		int meta_row_width_chroma[],
-		int meta_row_height[],
-		int meta_row_height_chroma[],
-		int meta_req_width[],
-		int meta_req_width_chroma[],
-		int meta_req_height[],
-		int meta_req_height_chroma[],
-		int dpte_group_bytes[],
-		int PTERequestSizeY[],
-		int PTERequestSizeC[],
-		int PixelPTEReqWidthY[],
-		int PixelPTEReqHeightY[],
-		int PixelPTEReqWidthC[],
-		int PixelPTEReqHeightC[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
+		unsigned int dpte_row_height[],
+		unsigned int dpte_row_height_chroma[],
+		unsigned int meta_row_width[],
+		unsigned int meta_row_width_chroma[],
+		unsigned int meta_row_height[],
+		unsigned int meta_row_height_chroma[],
+		unsigned int meta_req_width[],
+		unsigned int meta_req_width_chroma[],
+		unsigned int meta_req_height[],
+		unsigned int meta_req_height_chroma[],
+		unsigned int dpte_group_bytes[],
+		unsigned int PTERequestSizeY[],
+		unsigned int PTERequestSizeC[],
+		unsigned int PixelPTEReqWidthY[],
+		unsigned int PixelPTEReqHeightY[],
+		unsigned int PixelPTEReqWidthC[],
+		unsigned int PixelPTEReqHeightC[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
 		double DST_Y_PER_PTE_ROW_NOM_L[],
 		double DST_Y_PER_PTE_ROW_NOM_C[],
 		double DST_Y_PER_META_ROW_NOM_L[],
@@ -453,18 +453,18 @@ static void CalculateVMGroupAndRequestTimes(
 		bool GPUVMEnable,
 		unsigned int GPUVMMaxPageTableLevels,
 		unsigned int HTotal[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		double DestinationLinesToRequestVMInVBlank[],
 		double DestinationLinesToRequestVMInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
-		int vm_group_bytes[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
+		unsigned int vm_group_bytes[],
 		unsigned int dpde0_bytes_per_frame_ub_l[],
 		unsigned int dpde0_bytes_per_frame_ub_c[],
-		int meta_pte_bytes_per_frame_ub_l[],
-		int meta_pte_bytes_per_frame_ub_c[],
+		unsigned int meta_pte_bytes_per_frame_ub_l[],
+		unsigned int meta_pte_bytes_per_frame_ub_c[],
 		double TimePerVMGroupVBlank[],
 		double TimePerVMGroupFlip[],
 		double TimePerVMRequestVBlank[],
@@ -492,29 +492,29 @@ static void CalculateStutterEfficiency(
 		bool ProgressiveToInterlaceUnitInOPP,
 		bool Interlace[],
 		double MinTTUVBlank[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		unsigned int DETBufferSizeY[],
-		int BytePerPixelY[],
+		unsigned int BytePerPixelY[],
 		double BytePerPixelDETY[],
 		double SwathWidthY[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		double NetDCCRateLuma[],
 		double NetDCCRateChroma[],
 		double DCCFractionOfZeroSizeRequestsLuma[],
 		double DCCFractionOfZeroSizeRequestsChroma[],
-		int HTotal[],
-		int VTotal[],
+		unsigned int HTotal[],
+		unsigned int VTotal[],
 		double PixelClock[],
 		double VRatio[],
 		enum scan_direction_class SourceScan[],
-		int BlockHeight256BytesY[],
-		int BlockWidth256BytesY[],
-		int BlockHeight256BytesC[],
-		int BlockWidth256BytesC[],
-		int DCCYMaxUncompressedBlock[],
-		int DCCCMaxUncompressedBlock[],
-		int VActive[],
+		unsigned int BlockHeight256BytesY[],
+		unsigned int BlockWidth256BytesY[],
+		unsigned int BlockHeight256BytesC[],
+		unsigned int BlockWidth256BytesC[],
+		unsigned int DCCYMaxUncompressedBlock[],
+		unsigned int DCCCMaxUncompressedBlock[],
+		unsigned int VActive[],
 		bool DCCEnable[],
 		bool WritebackEnable[],
 		double ReadBandwidthPlaneLuma[],
@@ -539,32 +539,32 @@ static void CalculateSwathAndDETConfiguration(
 		enum scan_direction_class SourceScan[],
 		enum source_format_class SourcePixelFormat[],
 		enum dm_swizzle_mode SurfaceTiling[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BlendingAndTiming[],
-		int BytePerPixY[],
-		int BytePerPixC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
 		double BytePerPixDETY[],
 		double BytePerPixDETC[],
-		int HActive[],
+		unsigned int HActive[],
 		double HRatio[],
 		double HRatioChroma[],
-		int DPPPerPlane[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[],
+		unsigned int DPPPerPlane[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[],
 		double SwathWidth[],
 		double SwathWidthChroma[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		unsigned int DETBufferSizeY[],
 		unsigned int DETBufferSizeC[],
 		bool ViewportSizeSupportPerPlane[],
@@ -574,31 +574,31 @@ static void CalculateSwathWidth(
 		int NumberOfActivePlanes,
 		enum source_format_class SourcePixelFormat[],
 		enum scan_direction_class SourceScan[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BytePerPixY[],
-		int BytePerPixC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
-		int BlendingAndTiming[],
-		int HActive[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int HActive[],
 		double HRatio[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		double SwathWidthSingleDPPY[],
 		double SwathWidthSingleDPPC[],
 		double SwathWidthY[],
 		double SwathWidthC[],
-		int MaximumSwathHeightY[],
-		int MaximumSwathHeightC[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[]);
+		unsigned int MaximumSwathHeightY[],
+		unsigned int MaximumSwathHeightC[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[]);
 
 static double CalculateExtraLatency(
 		int RoundTripPingLatencyCycles,
@@ -612,8 +612,8 @@ static double CalculateExtraLatency(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double HostVMInefficiencyFactor,
 		double HostVMMinPageSize,
 		int HostVMMaxNonCachedPageTableLevels);
@@ -627,8 +627,8 @@ static double CalculateExtraLatencyBytes(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double HostVMInefficiencyFactor,
 		double HostVMMinPageSize,
 		int HostVMMaxNonCachedPageTableLevels);
@@ -652,7 +652,7 @@ static void CalculateUnboundedRequestAndCompressedBufferSize(
 		int CompressedBufferSegmentSizeInkByteFinal,
 		enum output_encoder_class *Output,
 		bool *UnboundedRequestEnabled,
-		int *CompressedBufferSizeInkByte);
+		unsigned int *CompressedBufferSizeInkByte);
 
 static bool UnboundedRequest(enum unbounded_requesting_policy UseUnboundedRequestingFinal, int TotalNumberOfActiveDPP, bool NoChroma, enum output_encoder_class Output);
 
@@ -700,7 +700,7 @@ static unsigned int dscceComputeDelay(
 		pixelsPerClock = 1;
 
 	//initial transmit delay as per PPS
-	initalXmitDelay = dml_round(rcModelSize / 2.0 / BPP / pixelsPerClock);
+	initalXmitDelay = (unsigned int)dml_round(rcModelSize / 2.0 / BPP / pixelsPerClock);
 
 	//compute ssm delay
 	if (bpc == 8)
@@ -869,7 +869,7 @@ static bool CalculatePrefetchSchedule(
 		double *Tdmdl_vm,
 		double *Tdmdl,
 		double *TSetup,
-		int *VUpdateOffsetPix,
+		unsigned int *VUpdateOffsetPix,
 		double *VUpdateWidthPix,
 		double *VReadyOffsetPix)
 {
@@ -985,13 +985,13 @@ static bool CalculatePrefetchSchedule(
 	*Tdmdl_vm = (DynamicMetadataEnable == true && DynamicMetadataVMEnabled == true && GPUVMEnable == true ? TWait + Tvm_trips : 0);
 
 	if (myPipe->ScalerEnabled)
-		DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCL;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCL);
 	else
-		DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCLLBOnly;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCLLBOnly);
 
-	DPPCycles = DPPCycles + myPipe->NumberOfCursors * DPPCLKDelayCNVCCursor;
+	DPPCycles = (unsigned int)(DPPCycles + myPipe->NumberOfCursors * DPPCLKDelayCNVCCursor);
 
-	DISPCLKCycles = DISPCLKDelaySubtotal;
+	DISPCLKCycles = (unsigned int)DISPCLKDelaySubtotal;
 
 	if (myPipe->DPPCLK == 0.0 || myPipe->DISPCLK == 0.0)
 		return true;
@@ -1510,10 +1510,10 @@ static void CalculateDCCConfiguration(
 	double detile_buf_vp_horz_limit;
 	double detile_buf_vp_vert_limit;
 
-	int MAS_vp_horz_limit;
-	int MAS_vp_vert_limit;
-	int max_vp_horz_width;
-	int max_vp_vert_height;
+	unsigned int MAS_vp_horz_limit;
+	unsigned int MAS_vp_vert_limit;
+	unsigned int max_vp_horz_width;
+	unsigned int max_vp_vert_height;
 	int eff_surf_width_l;
 	int eff_surf_width_c;
 	int eff_surf_height_l;
@@ -1554,11 +1554,11 @@ static void CalculateDCCConfiguration(
 		horz_div_c = 0;
 
 	if (BytePerPixelC == 0) {
-		swath_buf_size = DETBufferSize / 2 - 2 * 256;
+		swath_buf_size = (int)(DETBufferSize / 2 - 2 * 256);
 		detile_buf_vp_horz_limit = (double) swath_buf_size / ((double) RequestHeight256ByteLuma * BytePerPixelY / (1 + horz_div_l));
 		detile_buf_vp_vert_limit = (double) swath_buf_size / (256.0 / RequestHeight256ByteLuma / (1 + vert_div_l));
 	} else {
-		swath_buf_size = DETBufferSize / 2 - 2 * 2 * 256;
+		swath_buf_size = (int)(DETBufferSize / 2 - 2 * 2 * 256);
 		detile_buf_vp_horz_limit = (double) swath_buf_size
 				/ ((double) RequestHeight256ByteLuma * BytePerPixelY / (1 + horz_div_l)
 						+ (double) RequestHeight256ByteChroma * BytePerPixelC / (1 + horz_div_c) / (1 + yuv420));
@@ -1576,8 +1576,8 @@ static void CalculateDCCConfiguration(
 
 	MAS_vp_horz_limit = SourcePixelFormat == dm_rgbe_alpha ? 3840 : 5760;
 	MAS_vp_vert_limit = (BytePerPixelC > 0 ? 2880 : 5760);
-	max_vp_horz_width = dml_min((double) MAS_vp_horz_limit, detile_buf_vp_horz_limit);
-	max_vp_vert_height = dml_min((double) MAS_vp_vert_limit, detile_buf_vp_vert_limit);
+	max_vp_horz_width = (int)dml_min((double) MAS_vp_horz_limit, detile_buf_vp_horz_limit);
+	max_vp_vert_height = (int)dml_min((double) MAS_vp_vert_limit, detile_buf_vp_vert_limit);
 	eff_surf_width_l = (SurfaceWidthLuma > max_vp_horz_width ? max_vp_horz_width : SurfaceWidthLuma);
 	eff_surf_width_c = eff_surf_width_l / (1 + yuv420);
 	eff_surf_height_l = (SurfaceHeightLuma > max_vp_vert_height ? max_vp_vert_height : SurfaceHeightLuma);
@@ -1594,10 +1594,10 @@ static void CalculateDCCConfiguration(
 	}
 
 	if (SourcePixelFormat == dm_420_10) {
-		full_swath_bytes_horz_wc_l = dml_ceil(full_swath_bytes_horz_wc_l * 2 / 3, 256);
-		full_swath_bytes_horz_wc_c = dml_ceil(full_swath_bytes_horz_wc_c * 2 / 3, 256);
-		full_swath_bytes_vert_wc_l = dml_ceil(full_swath_bytes_vert_wc_l * 2 / 3, 256);
-		full_swath_bytes_vert_wc_c = dml_ceil(full_swath_bytes_vert_wc_c * 2 / 3, 256);
+		full_swath_bytes_horz_wc_l = (int)dml_ceil(full_swath_bytes_horz_wc_l * 2 / 3, 256);
+		full_swath_bytes_horz_wc_c = (int)dml_ceil(full_swath_bytes_horz_wc_c * 2 / 3, 256);
+		full_swath_bytes_vert_wc_l = (int)dml_ceil(full_swath_bytes_vert_wc_l * 2 / 3, 256);
+		full_swath_bytes_vert_wc_c = (int)dml_ceil(full_swath_bytes_vert_wc_c * 2 / 3, 256);
 	}
 
 	if (2 * full_swath_bytes_horz_wc_l + 2 * full_swath_bytes_horz_wc_c <= DETBufferSize) {
@@ -1760,20 +1760,20 @@ static double CalculatePrefetchSourceLines(
 
 	if (!v->IgnoreViewportPositioning) {
 
-		*MaxNumSwath = dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0;
+		*MaxNumSwath = (unsigned int)(dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0);
 
 		if (*VInitPreFill > 1.0)
 			MaxPartialSwath = (unsigned int) (*VInitPreFill - 2) % SwathHeight;
 		else
 			MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 2) % SwathHeight;
-		MaxPartialSwath = dml_max(1U, MaxPartialSwath);
+		MaxPartialSwath = (unsigned int)dml_max(1U, MaxPartialSwath);
 
 	} else {
 
 		if (ViewportYStart != 0)
 			dml_print("WARNING DML: using viewport y position of 0 even though actual viewport y position is non-zero in prefetch source lines calculation\n");
 
-		*MaxNumSwath = dml_ceil(*VInitPreFill / SwathHeight, 1);
+		*MaxNumSwath = (unsigned int)dml_ceil(*VInitPreFill / SwathHeight, 1);
 
 		if (*VInitPreFill > 1.0)
 			MaxPartialSwath = (unsigned int) (*VInitPreFill - 1) % SwathHeight;
@@ -1818,19 +1818,19 @@ static unsigned int CalculateVMAndRowBytes(
 		unsigned int *MetaRowByte,
 		unsigned int *PixelPTEBytesPerRow,
 		bool *PTEBufferSizeNotExceeded,
-		int *dpte_row_width_ub,
+		unsigned int *dpte_row_width_ub,
 		unsigned int *dpte_row_height,
 		unsigned int *MetaRequestWidth,
 		unsigned int *MetaRequestHeight,
 		unsigned int *meta_row_width,
 		unsigned int *meta_row_height,
-		int *vm_group_bytes,
+		unsigned int *vm_group_bytes,
 		unsigned int *dpte_group_bytes,
 		unsigned int *PixelPTEReqWidth,
 		unsigned int *PixelPTEReqHeight,
 		unsigned int *PTERequestSize,
-		int *DPDE0BytesFrame,
-		int *MetaPTEBytesFrame)
+		unsigned int *DPDE0BytesFrame,
+		unsigned int *MetaPTEBytesFrame)
 {
 	(void)SourcePixelFormat;
 	struct vba_vars_st *v = &mode_lib->vba;
@@ -1848,9 +1848,9 @@ static unsigned int CalculateVMAndRowBytes(
 		if (HostVMMinPageSize < 2048) {
 			HostVMDynamicLevels = HostVMMaxNonCachedPageTableLevels;
 		} else if (HostVMMinPageSize >= 2048 && HostVMMinPageSize < 1048576) {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
 		} else {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
 		}
 	}
 
@@ -1858,16 +1858,16 @@ static unsigned int CalculateVMAndRowBytes(
 	*MetaRequestWidth = 8 * BlockWidth256Bytes;
 	if (ScanDirection != dm_vert) {
 		*meta_row_height = *MetaRequestHeight;
-		*meta_row_width = dml_ceil((double) SwathWidth - 1, *MetaRequestWidth) + *MetaRequestWidth;
-		*MetaRowByte = *meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0;
+		*meta_row_width = (unsigned int)(dml_ceil((double) SwathWidth - 1, *MetaRequestWidth) + *MetaRequestWidth);
+		*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0);
 	} else {
 		*meta_row_height = *MetaRequestWidth;
-		*meta_row_width = dml_ceil((double) SwathWidth - 1, *MetaRequestHeight) + *MetaRequestHeight;
-		*MetaRowByte = *meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0;
+		*meta_row_width = (unsigned int)(dml_ceil((double) SwathWidth - 1, *MetaRequestHeight) + *MetaRequestHeight);
+		*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0);
 	}
-	DCCMetaSurfaceBytes = DCCMetaPitch * (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes) + 64 * BlockHeight256Bytes) * BytePerPixel / 256;
+	DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch * (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes) + 64 * BlockHeight256Bytes) * BytePerPixel / 256);
 	if (GPUVMEnable == true) {
-		*MetaPTEBytesFrame = (dml_ceil((double) (DCCMetaSurfaceBytes - 4.0 * 1024.0) / (8 * 4.0 * 1024), 1) + 1) * 64;
+		*MetaPTEBytesFrame = (int)((dml_ceil((double) (DCCMetaSurfaceBytes - 4.0 * 1024.0) / (8 * 4.0 * 1024), 1) + 1) * 64);
 		MPDEBytesFrame = 128 * (v->GPUVMMaxPageTableLevels - 1);
 	} else {
 		*MetaPTEBytesFrame = 0;
@@ -1891,17 +1891,17 @@ static unsigned int CalculateVMAndRowBytes(
 
 	if (GPUVMEnable == true && v->GPUVMMaxPageTableLevels > 1) {
 		if (ScanDirection != dm_vert) {
-			*DPDE0BytesFrame = 64
+			*DPDE0BytesFrame = (int)(64
 					* (dml_ceil(
 							((Pitch * (dml_ceil(ViewportHeight - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes)
 									/ (8 * 2097152),
-							1) + 1);
+							1) + 1));
 		} else {
-			*DPDE0BytesFrame = 64
+			*DPDE0BytesFrame = (int)(64
 					* (dml_ceil(
 							((Pitch * (dml_ceil((double) SwathWidth - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes)
 									/ (8 * 2097152),
-							1) + 1);
+							1) + 1));
 		}
 		ExtraDPDEBytesFrame = 128 * (v->GPUVMMaxPageTableLevels - 2);
 	} else {
@@ -1929,7 +1929,7 @@ static unsigned int CalculateVMAndRowBytes(
 	if (SurfaceTiling == dm_sw_linear) {
 		PixelPTEReqHeightPTEs = 1;
 		*PixelPTEReqHeight = 1;
-		*PixelPTEReqWidth = 32768.0 / BytePerPixel;
+		*PixelPTEReqWidth = (unsigned int)(32768.0 / BytePerPixel);
 		*PTERequestSize = 64;
 		FractionOfPTEReturnDrop = 0;
 	} else if (GPUVMMinPageSize == 4 && MacroTileSizeBytes > 4096) {
@@ -1947,16 +1947,16 @@ static unsigned int CalculateVMAndRowBytes(
 	}
 
 	if (SurfaceTiling == dm_sw_linear) {
-		*dpte_row_height = dml_min(128, 1 << (unsigned int) dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1));
-		*dpte_row_width_ub = (dml_ceil((double)(Pitch * *dpte_row_height - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth;
+		*dpte_row_height = (unsigned int)dml_min(128, 1 << (unsigned int) dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1));
+		*dpte_row_width_ub = (int)((dml_ceil((double)(Pitch * *dpte_row_height - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth);
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqWidth * *PTERequestSize;
 	} else if (ScanDirection != dm_vert) {
 		*dpte_row_height = *PixelPTEReqHeight;
-		*dpte_row_width_ub = (dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth;
+		*dpte_row_width_ub = (int)((dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth);
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqWidth * *PTERequestSize;
 	} else {
-		*dpte_row_height = dml_min(*PixelPTEReqWidth, *MacroTileWidth);
-		*dpte_row_width_ub = (dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1) * *PixelPTEReqHeight;
+		*dpte_row_height = (unsigned int)dml_min(*PixelPTEReqWidth, *MacroTileWidth);
+		*dpte_row_width_ub = (int)((dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1) * *PixelPTEReqHeight);
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqHeight * *PTERequestSize;
 	}
 
@@ -2041,9 +2041,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 							v->WritebackHTaps[k],
 							v->WritebackVTaps[k],
 							v->WritebackSourceWidth[k],
-							v->WritebackDestinationWidth[k],
+							(long)v->WritebackDestinationWidth[k],
 							v->HTotal[k],
-							v->WritebackLineBufferSize));
+							(unsigned int)v->WritebackLineBufferSize));
 		}
 	}
 
@@ -2229,7 +2229,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			v->DPPCLK,
 			v->ReadBandwidthPlaneLuma,
 			v->ReadBandwidthPlaneChroma,
-			v->ReturnBusWidth,
+			(int)v->ReturnBusWidth,
 			&v->DCFCLKDeepSleep);
 
 	// DSCCLK
@@ -2266,7 +2266,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				v->DSCDelay[k] = dscceComputeDelay(
 						v->DSCInputBitPerComponent[k],
 						BPP,
-						dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
+						(unsigned int)dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
 						v->NumberOfDSCSlices[k],
 						v->OutputFormat[k],
 						v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]);
@@ -2275,8 +2275,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 						* (dscceComputeDelay(
 								v->DSCInputBitPerComponent[k],
 								BPP,
-								dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
-								v->NumberOfDSCSlices[k] / 2.0,
+								(unsigned int)dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
+								(unsigned int)(v->NumberOfDSCSlices[k] / 2.0),
 								v->OutputFormat[k],
 								v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]));
 			} else {
@@ -2284,12 +2284,12 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 						* (dscceComputeDelay(
 								v->DSCInputBitPerComponent[k],
 								BPP,
-								dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
-								v->NumberOfDSCSlices[k] / 4.0,
+								(unsigned int)dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
+								(unsigned int)(v->NumberOfDSCSlices[k] / 4.0),
 								v->OutputFormat[k],
 								v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]));
 			}
-			v->DSCDelay[k] = v->DSCDelay[k] * v->PixelClock[k] / v->PixelClockBackEnd[k];
+			v->DSCDelay[k] = (unsigned int)(v->DSCDelay[k] * v->PixelClock[k] / v->PixelClockBackEnd[k]);
 		} else {
 			v->DSCDelay[k] = 0;
 		}
@@ -2330,13 +2330,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					v->SurfaceTiling[k],
 					v->BytePerPixelC[k],
 					v->SourceScan[k],
-					v->SwathWidthC[k],
+					(unsigned int)v->SwathWidthC[k],
 					v->ViewportHeightChroma[k],
 					v->GPUVMEnable,
 					v->HostVMEnable,
 					v->HostVMMaxNonCachedPageTableLevels,
-					v->GPUVMMinPageSize,
-					v->HostVMMinPageSize,
+					(unsigned int)v->GPUVMMinPageSize,
+					(unsigned int)v->HostVMMinPageSize,
 					v->PTEBufferSizeInRequestsForChroma,
 					v->PitchC[k],
 					v->DCCMetaPitchC[k],
@@ -2387,13 +2387,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				v->SurfaceTiling[k],
 				v->BytePerPixelY[k],
 				v->SourceScan[k],
-				v->SwathWidthY[k],
+				(unsigned int)v->SwathWidthY[k],
 				v->ViewportHeight[k],
 				v->GPUVMEnable,
 				v->HostVMEnable,
 				v->HostVMMaxNonCachedPageTableLevels,
-				v->GPUVMMinPageSize,
-				v->HostVMMinPageSize,
+				(unsigned int)v->GPUVMMinPageSize,
+				(unsigned int)v->HostVMMinPageSize,
 				v->PTEBufferSizeInRequestsForLuma,
 				v->PitchY[k],
 				v->DCCMetaPitchY[k],
@@ -2459,11 +2459,11 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			NoChromaPlanes = false;
 	}
 
-	ReorderBytes = v->NumberOfChannels
+	ReorderBytes = (int)(v->NumberOfChannels
 			* dml_max3(
 					v->UrgentOutOfOrderReturnPerChannelPixelDataOnly,
 					v->UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData,
-					v->UrgentOutOfOrderReturnPerChannelVMDataOnly);
+					v->UrgentOutOfOrderReturnPerChannelVMDataOnly));
 
 	VMDataOnlyReturnBW = dml_min(
 			dml_min(v->ReturnBusWidth * v->DCFCLK, v->FabricClock * v->FabricDatapathToDCNDataReturn)
@@ -2489,7 +2489,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 		HostVMInefficiencyFactor = v->ReturnBW / VMDataOnlyReturnBW;
 
 	v->UrgentExtraLatency = CalculateExtraLatency(
-			v->RoundTripPingLatencyCycles,
+			(int)v->RoundTripPingLatencyCycles,
 			ReorderBytes,
 			v->DCFCLK,
 			v->TotalActiveDPP,
@@ -2517,9 +2517,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 								v->WritebackHRatio[k],
 								v->WritebackVRatio[k],
 								v->WritebackVTaps[k],
-								v->WritebackDestinationWidth[k],
-								v->WritebackDestinationHeight[k],
-								v->WritebackSourceHeight[k],
+								(int)v->WritebackDestinationWidth[k],
+								(int)v->WritebackDestinationHeight[k],
+								(int)v->WritebackSourceHeight[k],
 								v->HTotal[k]) / v->DISPCLK;
 			} else
 				v->WritebackDelay[v->VoltageLevel][k] = 0;
@@ -2533,9 +2533,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 											v->WritebackHRatio[j],
 											v->WritebackVRatio[j],
 											v->WritebackVTaps[j],
-											v->WritebackDestinationWidth[j],
-											v->WritebackDestinationHeight[j],
-											v->WritebackSourceHeight[j],
+											(int)v->WritebackDestinationWidth[j],
+											(int)v->WritebackDestinationHeight[j],
+											(int)v->WritebackSourceHeight[j],
 											v->HTotal[k]) / v->DISPCLK);
 				}
 			}
@@ -2548,7 +2548,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				v->WritebackDelay[v->VoltageLevel][k] = v->WritebackDelay[v->VoltageLevel][j];
 
 	for (k = 0; k < v->NumberOfActivePlanes; ++k) {
-		v->MaxVStartupLines[k] =
+		v->MaxVStartupLines[k] = (unsigned int)(
 				(v->Interlace[k] && !v->ProgressiveToInterlaceUnitInOPP) ?
 						dml_floor((v->VTotal[k] - v->VActive[k]) / 2.0, 1.0) :
 						v->VTotal[k] - v->VActive[k]
@@ -2557,7 +2557,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 										dml_ceil(
 												(double) v->WritebackDelay[v->VoltageLevel][k]
 														/ (v->HTotal[k] / v->PixelClock[k]),
-												1));
+												1)));
 		if (v->MaxVStartupLines[k] > 1023)
 			v->MaxVStartupLines[k] = 1023;
 
@@ -2570,7 +2570,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 
 	v->MaximumMaxVStartupLines = 0;
 	for (k = 0; k < v->NumberOfActivePlanes; ++k)
-		v->MaximumMaxVStartupLines = dml_max(v->MaximumMaxVStartupLines, v->MaxVStartupLines[k]);
+		v->MaximumMaxVStartupLines = (unsigned int)dml_max(v->MaximumMaxVStartupLines, v->MaxVStartupLines[k]);
 
 	// VBA_DELTA
 	// We don't really care to iterate between the various prefetch modes
@@ -2639,7 +2639,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					(unsigned int) (v->SwathWidthY[k] / v->HRatio[k]),
 					v->OutputFormat[k],
 					v->MaxInterDCNTileRepeaters,
-					dml_min(v->VStartupLines, v->MaxVStartupLines[k]),
+					(unsigned int)dml_min(v->VStartupLines, v->MaxVStartupLines[k]),
 					v->MaxVStartupLines[k],
 					v->GPUVMMaxPageTableLevels,
 					v->GPUVMEnable,
@@ -2653,15 +2653,15 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					v->UrgentLatency,
 					v->UrgentExtraLatency,
 					v->TCalc,
-					v->PDEAndMetaPTEBytesFrame[k],
-					v->MetaRowByte[k],
-					v->PixelPTEBytesPerRow[k],
+					(unsigned int)v->PDEAndMetaPTEBytesFrame[k],
+					(unsigned int)v->MetaRowByte[k],
+					(unsigned int)v->PixelPTEBytesPerRow[k],
 					v->PrefetchSourceLinesY[k],
-					v->SwathWidthY[k],
+					(unsigned int)v->SwathWidthY[k],
 					v->VInitPreFillY[k],
 					v->MaxNumSwathY[k],
 					v->PrefetchSourceLinesC[k],
-					v->SwathWidthC[k],
+					(unsigned int)v->SwathWidthC[k],
 					v->VInitPreFillC[k],
 					v->MaxNumSwathC[k],
 					v->swath_width_luma_ub[k],
@@ -2692,7 +2692,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 #ifdef __DML_VBA_DEBUG__
 			dml_print("DML::%s: k=%0d Prefetch cal result=%0d\n", __func__, k, v->ErrorResult[k]);
 #endif
-			v->VStartup[k] = dml_min(v->VStartupLines, v->MaxVStartupLines[k]);
+			v->VStartup[k] = (unsigned int)dml_min(v->VStartupLines, v->MaxVStartupLines[k]);
 		}
 
 		v->NoEnoughUrgentLatencyHiding = false;
@@ -2846,8 +2846,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 
 			v->TotImmediateFlipBytes = 0;
 			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
-				v->TotImmediateFlipBytes = v->TotImmediateFlipBytes
-						+ v->DPPPerPlane[k] * (v->PDEAndMetaPTEBytesFrame[k] + v->MetaRowByte[k] + v->PixelPTEBytesPerRow[k]);
+				v->TotImmediateFlipBytes = (unsigned int)(v->TotImmediateFlipBytes
+						+ v->DPPPerPlane[k] * (v->PDEAndMetaPTEBytesFrame[k] + v->MetaRowByte[k] + v->PixelPTEBytesPerRow[k]));
 			}
 			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 				CalculateFlipSchedule(
@@ -3159,10 +3159,10 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 
 		isInterlaceTiming = (v->Interlace[k] && !v->ProgressiveToInterlaceUnitInOPP);
 
-		v->MIN_DST_Y_NEXT_START[k] = ((isInterlaceTiming ? dml_floor((v->VTotal[k] - v->VFrontPorch[k]) / 2.0, 1.0) : v->VTotal[k])
+		v->MIN_DST_Y_NEXT_START[k] = (unsigned int)(((isInterlaceTiming ? dml_floor((v->VTotal[k] - v->VFrontPorch[k]) / 2.0, 1.0) : v->VTotal[k])
 				- v->VFrontPorch[k])
 				+ dml_max(1.0, dml_ceil(v->WritebackDelay[v->VoltageLevel][k] / (v->HTotal[k] / v->PixelClock[k]), 1.0))
-				+ dml_floor(4.0 * v->TSetup[k] / (v->HTotal[k] / v->PixelClock[k]), 1.0) / 4.0;
+				+ dml_floor(4.0 * v->TSetup[k] / (v->HTotal[k] / v->PixelClock[k]), 1.0) / 4.0);
 
 		v->VStartup[k] = (isInterlaceTiming ? (2 * v->MaxVStartupLines[k]) : v->MaxVStartupLines[k]);
 
@@ -3278,18 +3278,18 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib)
 	// Display Pipe Configuration
 	double BytePerPixDETY[DC__NUM_DPP__MAX];
 	double BytePerPixDETC[DC__NUM_DPP__MAX];
-	int BytePerPixY[DC__NUM_DPP__MAX];
-	int BytePerPixC[DC__NUM_DPP__MAX];
-	int Read256BytesBlockHeightY[DC__NUM_DPP__MAX];
-	int Read256BytesBlockHeightC[DC__NUM_DPP__MAX];
-	int Read256BytesBlockWidthY[DC__NUM_DPP__MAX];
-	int Read256BytesBlockWidthC[DC__NUM_DPP__MAX];
+	unsigned int BytePerPixY[DC__NUM_DPP__MAX];
+	unsigned int BytePerPixC[DC__NUM_DPP__MAX];
+	unsigned int Read256BytesBlockHeightY[DC__NUM_DPP__MAX];
+	unsigned int Read256BytesBlockHeightC[DC__NUM_DPP__MAX];
+	unsigned int Read256BytesBlockWidthY[DC__NUM_DPP__MAX];
+	unsigned int Read256BytesBlockWidthC[DC__NUM_DPP__MAX];
 	double dummy1[DC__NUM_DPP__MAX];
 	double dummy2[DC__NUM_DPP__MAX];
 	double dummy3[DC__NUM_DPP__MAX];
 	double dummy4[DC__NUM_DPP__MAX];
-	int dummy5[DC__NUM_DPP__MAX];
-	int dummy6[DC__NUM_DPP__MAX];
+	unsigned int dummy5[DC__NUM_DPP__MAX];
+	unsigned int dummy6[DC__NUM_DPP__MAX];
 	bool dummy7[DC__NUM_DPP__MAX];
 	bool dummysinglestring;
 
@@ -3429,7 +3429,7 @@ static void CalculateVupdateAndDynamicMetadataParameters(
 		double *Tdmbf,
 		double *Tdmec,
 		double *Tdmsks,
-		int *VUpdateOffsetPix,
+		unsigned int *VUpdateOffsetPix,
 		double *VUpdateWidthPix,
 		double *VReadyOffsetPix)
 {
@@ -3438,7 +3438,7 @@ static void CalculateVupdateAndDynamicMetadataParameters(
 	TotalRepeaterDelayTime = MaxInterDCNTileRepeaters * (2 / DPPCLK + 3 / DISPCLK);
 	*VUpdateWidthPix = dml_ceil((14.0 / DCFClkDeepSleep + 12.0 / DPPCLK + TotalRepeaterDelayTime) * PixelClock, 1.0);
 	*VReadyOffsetPix = dml_ceil(dml_max(150.0 / DPPCLK, TotalRepeaterDelayTime + 20.0 / DCFClkDeepSleep + 10.0 / DPPCLK) * PixelClock, 1.0);
-	*VUpdateOffsetPix = dml_ceil(HTotal / 4.0, 1);
+	*VUpdateOffsetPix = (unsigned int)dml_ceil(HTotal / 4.0, 1);
 	*TSetup = (*VUpdateOffsetPix + *VUpdateWidthPix + *VReadyOffsetPix) / PixelClock;
 	*Tdmbf = DynamicMetadataTransmittedBytes / 4.0 / DISPCLK;
 	*Tdmec = HTotal / PixelClock;
@@ -3723,17 +3723,17 @@ static noinline void CalculatePrefetchSchedulePerPlane(
 		mode_lib,
 		HostVMInefficiencyFactor,
 		&myPipe,
-		v->DSCDelayPerState[i][k],
+		(unsigned int)v->DSCDelayPerState[i][k],
 		v->DPPCLKDelaySubtotal + v->DPPCLKDelayCNVCFormater,
 		v->DPPCLKDelaySCL,
 		v->DPPCLKDelaySCLLBOnly,
 		v->DPPCLKDelayCNVCCursor,
 		v->DISPCLKDelaySubtotal,
-		v->SwathWidthYThisState[k] / v->HRatio[k],
+		(unsigned int)(v->SwathWidthYThisState[k] / v->HRatio[k]),
 		v->OutputFormat[k],
 		v->MaxInterDCNTileRepeaters,
-		dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]),
-		v->MaximumVStartup[i][j][k],
+		(unsigned int)dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]),
+			(unsigned int)v->MaximumVStartup[i][j][k],
 		v->GPUVMMaxPageTableLevels,
 		v->GPUVMEnable,
 		v->HostVMEnable,
@@ -3746,15 +3746,15 @@ static noinline void CalculatePrefetchSchedulePerPlane(
 		v->UrgLatency[i],
 		v->ExtraLatency,
 		v->TimeCalc,
-		v->PDEAndMetaPTEBytesPerFrame[i][j][k],
-		v->MetaRowBytes[i][j][k],
-		v->DPTEBytesPerRow[i][j][k],
+			(unsigned int)v->PDEAndMetaPTEBytesPerFrame[i][j][k],
+			(unsigned int)v->MetaRowBytes[i][j][k],
+		(unsigned int)v->DPTEBytesPerRow[i][j][k],
 		v->PrefetchLinesY[i][j][k],
-		v->SwathWidthYThisState[k],
+		(unsigned int)v->SwathWidthYThisState[k],
 		v->PrefillY[k],
 		v->MaxNumSwY[k],
 		v->PrefetchLinesC[i][j][k],
-		v->SwathWidthCThisState[k],
+		(unsigned int)v->SwathWidthCThisState[k],
 		v->PrefillC[k],
 		v->MaxNumSwC[k],
 		v->swath_width_luma_ub_this_state[k],
@@ -3783,9 +3783,10 @@ static noinline void CalculatePrefetchSchedulePerPlane(
 		&v->VReadyOffsetPix[k]);
 }
 
-static void PatchDETBufferSizeInKByte(unsigned int NumberOfActivePlanes, int NoOfDPPThisState[], unsigned int config_return_buffer_size_in_kbytes, unsigned int DETBufferSizeInKByte[])
+static void PatchDETBufferSizeInKByte(unsigned int NumberOfActivePlanes, unsigned int NoOfDPPThisState[], unsigned int config_return_buffer_size_in_kbytes, unsigned int DETBufferSizeInKByte[])
 {
-	int i, total_pipes = 0;
+	int total_pipes = 0;
+	unsigned int i;
 	for (i = 0; i < NumberOfActivePlanes; i++)
 		total_pipes += NoOfDPPThisState[i];
 	DETBufferSizeInKByte[0] = ((config_return_buffer_size_in_kbytes - DCN3_15_MIN_COMPBUF_SIZE_KB) / 64 / total_pipes) * 64;
@@ -3800,10 +3801,10 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 {
 	struct vba_vars_st *v = &mode_lib->vba;
 
-	int i, j;
-	unsigned int k, m;
+	int idx;
+	unsigned int i, j, k, m;
 	int ReorderingBytes;
-	int MinPrefetchMode = 0, MaxPrefetchMode = 2;
+	unsigned int MinPrefetchMode = 0, MaxPrefetchMode = 2;
 	bool NoChroma = true;
 	bool EnoughWritebackUnits = true;
 	bool P2IWith420 = false;
@@ -3954,9 +3955,9 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->WritebackHTaps[k],
 							v->WritebackVTaps[k],
 							v->WritebackSourceWidth[k],
-							v->WritebackDestinationWidth[k],
+							(long)v->WritebackDestinationWidth[k],
 							v->HTotal[k],
-							v->WritebackLineBufferSize));
+							(unsigned int)v->WritebackLineBufferSize));
 		}
 	}
 	for (k = 0; k < v->NumberOfActivePlanes; k++) {
@@ -4003,17 +4004,17 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		int MaximumSwathWidthSupportChroma;
 
 		if (v->SurfaceTiling[k] == dm_sw_linear) {
-			MaximumSwathWidthSupportLuma = 8192.0;
+			MaximumSwathWidthSupportLuma = (int)8192.0;
 		} else if (v->SourceScan[k] == dm_vert && v->BytePerPixelC[k] > 0) {
-			MaximumSwathWidthSupportLuma = 2880.0;
+			MaximumSwathWidthSupportLuma = (int)2880.0;
 		} else if (v->SourcePixelFormat[k] == dm_rgbe_alpha) {
-			MaximumSwathWidthSupportLuma = 3840.0;
+			MaximumSwathWidthSupportLuma = (int)3840.0;
 		} else {
-			MaximumSwathWidthSupportLuma = 5760.0;
+			MaximumSwathWidthSupportLuma = (int)5760.0;
 		}
 
 		if (v->SourcePixelFormat[k] == dm_420_8 || v->SourcePixelFormat[k] == dm_420_10 || v->SourcePixelFormat[k] == dm_420_12) {
-			MaximumSwathWidthSupportChroma = MaximumSwathWidthSupportLuma / 2.0;
+			MaximumSwathWidthSupportChroma = (int)(MaximumSwathWidthSupportLuma / 2.0);
 		} else {
 			MaximumSwathWidthSupportChroma = MaximumSwathWidthSupportLuma;
 		}
@@ -4245,7 +4246,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						v->DISPCLK_DPPCLK_Support[i][j] = false;
 					}
 				}
-				v->TotalNumberOfActiveDPP[i][j] = 0.0;
+				v->TotalNumberOfActiveDPP[i][j] = 0;
 				for (k = 0; k < v->NumberOfActivePlanes; k++) {
 					v->TotalNumberOfActiveDPP[i][j] = v->TotalNumberOfActiveDPP[i][j] + v->NoOfDPP[i][j][k];
 				}
@@ -4282,7 +4283,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 		if (v->BlendingAndTiming[k] == k) {
 			if (v->PixelClockBackEnd[k] > 3200) {
-				v->NumberOfDSCSlices[k] = dml_ceil(v->PixelClockBackEnd[k] / 400.0, 4.0);
+				v->NumberOfDSCSlices[k] = (unsigned int)dml_ceil(v->PixelClockBackEnd[k] / 400.0, 4.0);
 			} else if (v->PixelClockBackEnd[k] > 1360) {
 				v->NumberOfDSCSlices[k] = 8;
 			} else if (v->PixelClockBackEnd[k] > 680) {
@@ -4344,7 +4345,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->PHYCLKD18PerState[k] >= 10000.0 / 18.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 10000,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4363,7 +4364,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 								v->LinkDSCEnable = true;
 								v->Outbpp = TruncToValidBPP(
 										(1.0 - v->Downspreading / 100.0) * 10000,
-										v->OutputLinkDPLanes[k],
+										(int)v->OutputLinkDPLanes[k],
 										v->HTotal[k],
 										v->HActive[k],
 										v->PixelClockBackEnd[k],
@@ -4386,7 +4387,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->PHYCLKD18PerState[k] >= 13500.0 / 18.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 13500,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4405,7 +4406,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 								v->LinkDSCEnable = true;
 								v->Outbpp = TruncToValidBPP(
 										(1.0 - v->Downspreading / 100.0) * 13500,
-										v->OutputLinkDPLanes[k],
+										(int)v->OutputLinkDPLanes[k],
 										v->HTotal[k],
 										v->HActive[k],
 										v->PixelClockBackEnd[k],
@@ -4428,7 +4429,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->PHYCLKD18PerState[k] >= 20000.0 / 18.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 20000,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4447,7 +4448,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 								v->LinkDSCEnable = true;
 								v->Outbpp = TruncToValidBPP(
 										(1.0 - v->Downspreading / 100.0) * 20000,
-										v->OutputLinkDPLanes[k],
+										(int)v->OutputLinkDPLanes[k],
 										v->HTotal[k],
 										v->HActive[k],
 										v->PixelClockBackEnd[k],
@@ -4470,7 +4471,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						if (v->PHYCLKPerState[i] >= 270.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 2700,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4490,7 +4491,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 540.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 5400,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4510,7 +4511,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 810.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 8100,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4608,7 +4609,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					v->DSCDelayPerState[i][k] = dscceComputeDelay(
 							v->DSCInputBitPerComponent[k],
 							v->BPP,
-							dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
+							(unsigned int)dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
 							v->NumberOfDSCSlices[k],
 							v->OutputFormat[k],
 							v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]);
@@ -4617,7 +4618,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							* (dscceComputeDelay(
 									v->DSCInputBitPerComponent[k],
 									v->BPP,
-									dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
+									(unsigned int)dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
 									v->NumberOfDSCSlices[k] / 2,
 									v->OutputFormat[k],
 									v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]));
@@ -4626,7 +4627,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							* (dscceComputeDelay(
 									v->DSCInputBitPerComponent[k],
 									v->BPP,
-									dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
+									(unsigned int)dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
 									v->NumberOfDSCSlices[k] / 4,
 									v->OutputFormat[k],
 									v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]));
@@ -4716,14 +4717,14 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					v->RequiredDPPCLKThisState,
 					v->ReadBandwidthLuma,
 					v->ReadBandwidthChroma,
-					v->ReturnBusWidth,
+					(int)v->ReturnBusWidth,
 					&v->ProjectedDCFCLKDeepSleep[i][j]);
 
 			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 				v->swath_width_luma_ub_all_states[i][j][k] = v->swath_width_luma_ub_this_state[k];
 				v->swath_width_chroma_ub_all_states[i][j][k] = v->swath_width_chroma_ub_this_state[k];
-				v->SwathWidthYAllStates[i][j][k] = v->SwathWidthYThisState[k];
-				v->SwathWidthCAllStates[i][j][k] = v->SwathWidthCThisState[k];
+				v->SwathWidthYAllStates[i][j][k] = (unsigned int)v->SwathWidthYThisState[k];
+				v->SwathWidthCAllStates[i][j][k] = (unsigned int)v->SwathWidthCThisState[k];
 				v->SwathHeightYAllStates[i][j][k] = v->SwathHeightYThisState[k];
 				v->SwathHeightCAllStates[i][j][k] = v->SwathHeightCThisState[k];
 				v->DETBufferSizeYAllStates[i][j][k] = v->DETBufferSizeYThisState[k];
@@ -4748,8 +4749,8 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				v->SwathWidthCThisState[k] = v->SwathWidthCAllStates[i][j][k];
 				v->SwathHeightYThisState[k] = v->SwathHeightYAllStates[i][j][k];
 				v->SwathHeightCThisState[k] = v->SwathHeightCAllStates[i][j][k];
-				v->DETBufferSizeYThisState[k] = v->DETBufferSizeYAllStates[i][j][k];
-				v->DETBufferSizeCThisState[k] = v->DETBufferSizeCAllStates[i][j][k];
+				v->DETBufferSizeYThisState[k] = (unsigned int)v->DETBufferSizeYAllStates[i][j][k];
+				v->DETBufferSizeCThisState[k] = (unsigned int)v->DETBufferSizeCAllStates[i][j][k];
 			}
 
 			v->TotalNumberOfDCCActiveDPP[i][j] = 0;
@@ -4782,16 +4783,16 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							v->SurfaceTiling[k],
 							v->BytePerPixelC[k],
 							v->SourceScan[k],
-							v->SwathWidthCThisState[k],
+							(unsigned int)v->SwathWidthCThisState[k],
 							v->ViewportHeightChroma[k],
 							v->GPUVMEnable,
 							v->HostVMEnable,
 							v->HostVMMaxNonCachedPageTableLevels,
-							v->GPUVMMinPageSize,
-							v->HostVMMinPageSize,
+							(unsigned int)v->GPUVMMinPageSize,
+							(unsigned int)v->HostVMMinPageSize,
 							v->PTEBufferSizeInRequestsForChroma,
 							v->PitchC[k],
-							0.0,
+							0,
 							&v->MacroTileWidthC[k],
 							&v->MetaRowBytesC,
 							&v->DPTEBytesPerRowC,
@@ -4824,8 +4825,8 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					v->PTEBufferSizeInRequestsForLuma = v->PTEBufferSizeInRequestsLuma + v->PTEBufferSizeInRequestsChroma;
 					v->PTEBufferSizeInRequestsForChroma = 0;
 					v->PDEAndMetaPTEBytesPerFrameC = 0.0;
-					v->MetaRowBytesC = 0.0;
-					v->DPTEBytesPerRowC = 0.0;
+					v->MetaRowBytesC = 0;
+					v->DPTEBytesPerRowC = 0;
 					v->PrefetchLinesC[i][j][k] = 0.0;
 					v->PTEBufferSizeNotExceededC[i][j][k] = true;
 				}
@@ -4838,13 +4839,13 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						v->SurfaceTiling[k],
 						v->BytePerPixelY[k],
 						v->SourceScan[k],
-						v->SwathWidthYThisState[k],
+						(unsigned int)v->SwathWidthYThisState[k],
 						v->ViewportHeight[k],
 						v->GPUVMEnable,
 						v->HostVMEnable,
 						v->HostVMMaxNonCachedPageTableLevels,
-						v->GPUVMMinPageSize,
-						v->HostVMMinPageSize,
+						(unsigned int)v->GPUVMMinPageSize,
+						(unsigned int)v->HostVMMinPageSize,
 						v->PTEBufferSizeInRequestsForLuma,
 						v->PitchY[k],
 						v->DCCMetaPitchY[k],
@@ -4978,9 +4979,9 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 										v->WritebackHRatio[k],
 										v->WritebackVRatio[k],
 										v->WritebackVTaps[k],
-										v->WritebackDestinationWidth[k],
-										v->WritebackDestinationHeight[k],
-										v->WritebackSourceHeight[k],
+										(int)v->WritebackDestinationWidth[k],
+										(int)v->WritebackDestinationHeight[k],
+										(int)v->WritebackSourceHeight[k],
 										v->HTotal[k]) / v->RequiredDISPCLK[i][j];
 					} else {
 						v->WritebackDelayTime[k] = 0.0;
@@ -4995,9 +4996,9 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 													v->WritebackHRatio[m],
 													v->WritebackVRatio[m],
 													v->WritebackVTaps[m],
-													v->WritebackDestinationWidth[m],
-													v->WritebackDestinationHeight[m],
-													v->WritebackSourceHeight[m],
+													(int)v->WritebackDestinationWidth[m],
+													(int)v->WritebackDestinationHeight[m],
+													(int)v->WritebackSourceHeight[m],
 													v->HTotal[m]) / v->RequiredDISPCLK[i][j]);
 						}
 					}
@@ -5030,11 +5031,11 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		}
 	}
 
-	ReorderingBytes = v->NumberOfChannels
+	ReorderingBytes = (int)(v->NumberOfChannels
 			* dml_max3(
 					v->UrgentOutOfOrderReturnPerChannelPixelDataOnly,
 					v->UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData,
-					v->UrgentOutOfOrderReturnPerChannelVMDataOnly);
+					v->UrgentOutOfOrderReturnPerChannelVMDataOnly));
 
 	for (i = 0; i < v->soc.num_states; ++i) {
 		for (j = 0; j <= 1; ++j) {
@@ -5118,7 +5119,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 			double HostVMInefficiencyFactor = 1;
 			int NextPrefetchModeState = MinPrefetchMode;
 			bool UnboundedRequestEnabledThisState = false;
-			int CompressedBufferSizeInkByteThisState = 0;
+			unsigned int CompressedBufferSizeInkByteThisState = 0;
 			double dummy;
 
 			v->TimeCalc = 24 / v->ProjectedDCFCLKDeepSleep[i][j];
@@ -5137,8 +5138,8 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				v->SwathWidthCThisState[k] = v->SwathWidthCAllStates[i][j][k];
 				v->SwathHeightYThisState[k] = v->SwathHeightYAllStates[i][j][k];
 				v->SwathHeightCThisState[k] = v->SwathHeightCAllStates[i][j][k];
-				v->DETBufferSizeYThisState[k] = v->DETBufferSizeYAllStates[i][j][k];
-				v->DETBufferSizeCThisState[k] = v->DETBufferSizeCAllStates[i][j][k];
+				v->DETBufferSizeYThisState[k] = (unsigned int)v->DETBufferSizeYAllStates[i][j][k];
+				v->DETBufferSizeCThisState[k] = (unsigned int)v->DETBufferSizeCAllStates[i][j][k];
 			}
 
 			VMDataOnlyReturnBWPerState = dml_min(
@@ -5152,7 +5153,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				HostVMInefficiencyFactor = v->ReturnBWPerState[i][j] / VMDataOnlyReturnBWPerState;
 
 			v->ExtraLatency = CalculateExtraLatency(
-					v->RoundTripPingLatencyCycles,
+					(int)v->RoundTripPingLatencyCycles,
 					ReorderingBytes,
 					v->DCFCLKState[i][j],
 					v->TotalNumberOfActiveDPP[i][j],
@@ -5285,11 +5286,11 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 																* v->UrgentBurstFactorChromaPre[k])
 												+ v->cursor_bw_pre[k] * v->UrgentBurstFactorCursorPre[k]);
 					}
-					v->TotImmediateFlipBytes = 0.0;
+					v->TotImmediateFlipBytes = 0;
 					for (k = 0; k < v->NumberOfActivePlanes; k++) {
-						v->TotImmediateFlipBytes = v->TotImmediateFlipBytes
+						v->TotImmediateFlipBytes = (unsigned int)(v->TotImmediateFlipBytes
 								+ v->NoOfDPP[i][j][k] * (v->PDEAndMetaPTEBytesPerFrame[i][j][k] + v->MetaRowBytes[i][j][k]
-								+ v->DPTEBytesPerRow[i][j][k]);
+								+ v->DPTEBytesPerRow[i][j][k]));
 					}
 
 					for (k = 0; k < v->NumberOfActivePlanes; k++) {
@@ -5451,46 +5452,53 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	}
 
 	/*Mode Support, Voltage State and SOC Configuration*/
-	for (i = v->soc.num_states - 1; i >= 0; i--) {
+	for (idx = (int)v->soc.num_states - 1; idx >= 0; idx--) {
 		for (j = 0; j < 2; j++) {
-			if (v->ScaleRatioAndTapsSupport == true && v->SourceFormatPixelAndScanSupport == true && v->ViewportSizeSupport[i][j] == true
-					&& v->LinkCapacitySupport[i] == true && !P2IWith420 && !DSCOnlyIfNecessaryWithBPP
-					&& !DSC422NativeNotSupported && v->ODMCombine4To1SupportCheckOK[i] == true && v->NotEnoughDSCUnits[i] == false
-					&& v->DTBCLKRequiredMoreThanSupported[i] == false
-					&& v->ROBSupport[i][j] == true && v->DISPCLK_DPPCLK_Support[i][j] == true
-					&& v->TotalAvailablePipesSupport[i][j] == true && EnoughWritebackUnits == true
+			if (v->ScaleRatioAndTapsSupport == true && v->SourceFormatPixelAndScanSupport == true
+					&& v->ViewportSizeSupport[idx][j] == true
+					&& v->LinkCapacitySupport[idx] == true && !P2IWith420
+					&& !DSCOnlyIfNecessaryWithBPP
+					&& !DSC422NativeNotSupported && v->ODMCombine4To1SupportCheckOK[idx] == true
+					&& v->NotEnoughDSCUnits[idx] == false
+					&& v->DTBCLKRequiredMoreThanSupported[idx] == false
+					&& v->ROBSupport[idx][j] == true && v->DISPCLK_DPPCLK_Support[idx][j] == true
+					&& v->TotalAvailablePipesSupport[idx][j] == true && EnoughWritebackUnits == true
 					&& v->WritebackLatencySupport == true && v->WritebackScaleRatioAndTapsSupport == true
-					&& v->CursorSupport == true && v->PitchSupport == true && ViewportExceedsSurface == false
-					&& v->PrefetchSupported[i][j] == true && v->DynamicMetadataSupported[i][j] == true
-					&& v->TotalVerticalActiveBandwidthSupport[i][j] == true && v->VRatioInPrefetchSupported[i][j] == true
-					&& v->PTEBufferSizeNotExceeded[i][j] == true && v->NonsupportedDSCInputBPC == false
+					&& v->CursorSupport == true && v->PitchSupport == true
+					&& ViewportExceedsSurface == false
+					&& v->PrefetchSupported[idx][j] == true
+					&& v->DynamicMetadataSupported[idx][j] == true
+					&& v->TotalVerticalActiveBandwidthSupport[idx][j] == true
+					&& v->VRatioInPrefetchSupported[idx][j] == true
+					&& v->PTEBufferSizeNotExceeded[idx][j] == true
+					&& v->NonsupportedDSCInputBPC == false
 					&& ((v->HostVMEnable == false
 					&& v->ImmediateFlipRequirement[0] != dm_immediate_flip_required)
-							|| v->ImmediateFlipSupportedForState[i][j] == true)
+							|| v->ImmediateFlipSupportedForState[idx][j] == true)
 					&& FMTBufferExceeded == false) {
-				v->ModeSupport[i][j] = true;
+				v->ModeSupport[idx][j] = true;
 			} else {
-				v->ModeSupport[i][j] = false;
+				v->ModeSupport[idx][j] = false;
 #ifdef __DML_VBA_DEBUG__
 				if (v->ScaleRatioAndTapsSupport == false)
 					dml_print("DML SUPPORT:     ScaleRatioAndTapsSupport failed");
 				if (v->SourceFormatPixelAndScanSupport == false)
 					dml_print("DML SUPPORT:     SourceFormatPixelAndScanSupport failed");
-				if (v->ViewportSizeSupport[i][j] == false)
+				if (v->ViewportSizeSupport[idx][j] == false)
 					dml_print("DML SUPPORT:     ViewportSizeSupport failed");
-				if (v->LinkCapacitySupport[i] == false)
+				if (v->LinkCapacitySupport[idx] == false)
 					dml_print("DML SUPPORT:     LinkCapacitySupport failed");
-				if (v->ODMCombine4To1SupportCheckOK[i] == false)
+				if (v->ODMCombine4To1SupportCheckOK[idx] == false)
 					dml_print("DML SUPPORT:     DSC422NativeNotSupported failed");
-				if (v->NotEnoughDSCUnits[i] == true)
+				if (v->NotEnoughDSCUnits[idx] == true)
 					dml_print("DML SUPPORT:     NotEnoughDSCUnits");
-				if (v->DTBCLKRequiredMoreThanSupported[i] == true)
+				if (v->DTBCLKRequiredMoreThanSupported[idx] == true)
 					dml_print("DML SUPPORT:     DTBCLKRequiredMoreThanSupported");
-				if (v->ROBSupport[i][j] == false)
+				if (v->ROBSupport[idx][j] == false)
 					dml_print("DML SUPPORT:     ROBSupport failed");
-				if (v->DISPCLK_DPPCLK_Support[i][j] == false)
+				if (v->DISPCLK_DPPCLK_Support[idx][j] == false)
 					dml_print("DML SUPPORT:     DISPCLK_DPPCLK_Support failed");
-				if (v->TotalAvailablePipesSupport[i][j] == false)
+				if (v->TotalAvailablePipesSupport[idx][j] == false)
 					dml_print("DML SUPPORT:     DSC422NativeNotSupported failed");
 				if (EnoughWritebackUnits == false)
 					dml_print("DML SUPPORT:     DSC422NativeNotSupported failed");
@@ -5504,21 +5512,21 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					dml_print("DML SUPPORT:     PitchSupport failed");
 				if (ViewportExceedsSurface == true)
 					dml_print("DML SUPPORT:     ViewportExceedsSurface failed");
-				if (v->PrefetchSupported[i][j] == false)
+				if (v->PrefetchSupported[idx][j] == false)
 					dml_print("DML SUPPORT:     PrefetchSupported failed");
-				if (v->DynamicMetadataSupported[i][j] == false)
+				if (v->DynamicMetadataSupported[idx][j] == false)
 					dml_print("DML SUPPORT:     DSC422NativeNotSupported failed");
-				if (v->TotalVerticalActiveBandwidthSupport[i][j] == false)
+				if (v->TotalVerticalActiveBandwidthSupport[idx][j] == false)
 					dml_print("DML SUPPORT:     TotalVerticalActiveBandwidthSupport failed");
-				if (v->VRatioInPrefetchSupported[i][j] == false)
+				if (v->VRatioInPrefetchSupported[idx][j] == false)
 					dml_print("DML SUPPORT:     VRatioInPrefetchSupported failed");
-				if (v->PTEBufferSizeNotExceeded[i][j] == false)
+				if (v->PTEBufferSizeNotExceeded[idx][j] == false)
 					dml_print("DML SUPPORT:     PTEBufferSizeNotExceeded failed");
 				if (v->NonsupportedDSCInputBPC == true)
 					dml_print("DML SUPPORT:     NonsupportedDSCInputBPC failed");
 				if (!((v->HostVMEnable == false
 					&& v->ImmediateFlipRequirement[0] != dm_immediate_flip_required)
-							|| v->ImmediateFlipSupportedForState[i][j] == true))
+							|| v->ImmediateFlipSupportedForState[idx][j] == true))
 					dml_print("DML SUPPORT:     ImmediateFlipRequirement failed");
 				if (FMTBufferExceeded == true)
 					dml_print("DML SUPPORT:     FMTBufferExceeded failed");
@@ -5529,11 +5537,12 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 
 	{
 		unsigned int MaximumMPCCombine = 0;
-		for (i = v->soc.num_states; i >= 0; i--) {
-			if (i == v->soc.num_states || v->ModeSupport[i][0] == true || v->ModeSupport[i][1] == true) {
-				v->VoltageLevel = i;
-				v->ModeIsSupported = v->ModeSupport[i][0] == true || v->ModeSupport[i][1] == true;
-				if (v->ModeSupport[i][0] == true) {
+		for (idx = (int)v->soc.num_states; idx >= 0; idx--) {
+			if (idx == (int)v->soc.num_states || v->ModeSupport[idx][0] == true
+					|| v->ModeSupport[idx][1] == true) {
+				v->VoltageLevel = idx;
+				v->ModeIsSupported = v->ModeSupport[idx][0] == true || v->ModeSupport[idx][1] == true;
+				if (v->ModeSupport[idx][0] == true) {
 					MaximumMPCCombine = 0;
 				} else {
 					MaximumMPCCombine = 1;
@@ -5541,7 +5550,7 @@ void dml31_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 			}
 		}
 		v->ImmediateFlipSupport = v->ImmediateFlipSupportedForState[v->VoltageLevel][MaximumMPCCombine];
-		for (k = 0; k <= v->NumberOfActivePlanes - 1; k++) {
+		for (k = 0; k < v->NumberOfActivePlanes; k++) {
 			v->MPCCombineEnable[k] = v->MPCCombine[v->VoltageLevel][MaximumMPCCombine][k];
 			v->DPPPerPlane[k] = v->NoOfDPP[v->VoltageLevel][MaximumMPCCombine][k];
 		}
@@ -5599,7 +5608,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 	double SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank;
 	double WritebackDRAMClockChangeLatencyHiding;
 	double TotalPixelBW = 0.0;
-	int k, j;
+	unsigned int k, j;
 
 	v->UrgentWatermark = UrgentLatency + ExtraLatency;
 
@@ -5644,13 +5653,13 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 	for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 		double EffectiveDETBufferSizeY = DETBufferSizeY[k];
 
-		v->LBLatencyHidingSourceLinesY = dml_min(
+		v->LBLatencyHidingSourceLinesY = (unsigned int)(dml_min(
 				(double) v->MaxLineBufferLines,
-				dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1);
+				dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1));
 
-		v->LBLatencyHidingSourceLinesC = dml_min(
+		v->LBLatencyHidingSourceLinesC = (unsigned int)(dml_min(
 				(double) v->MaxLineBufferLines,
-				dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1);
+				dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1));
 
 		EffectiveLBLatencyHidingY = v->LBLatencyHidingSourceLinesY / v->VRatio[k] * (v->HTotal[k] / v->PixelClock[k]);
 
@@ -5662,11 +5671,11 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 		}
 
 		LinesInDETY[k] = (double) EffectiveDETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k];
-		LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]);
+		LinesInDETYRoundedDownToSwath[k] = (unsigned int)dml_floor(LinesInDETY[k], SwathHeightY[k]);
 		FullDETBufferingTimeY = LinesInDETYRoundedDownToSwath[k] * (v->HTotal[k] / v->PixelClock[k]) / v->VRatio[k];
 		if (BytePerPixelDETC[k] > 0) {
 			LinesInDETC = v->DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k];
-			LinesInDETCRoundedDownToSwath = dml_floor(LinesInDETC, SwathHeightC[k]);
+			LinesInDETCRoundedDownToSwath = (unsigned int)dml_floor(LinesInDETC, SwathHeightC[k]);
 			FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath * (v->HTotal[k] / v->PixelClock[k]) / v->VRatioChroma[k];
 		} else {
 			LinesInDETC = 0;
@@ -5765,8 +5774,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 static void CalculateDCFCLKDeepSleep(
 		struct display_mode_lib *mode_lib,
 		unsigned int NumberOfActivePlanes,
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		double VRatio[],
 		double VRatioChroma[],
 		double SwathWidthY[],
@@ -5787,7 +5796,7 @@ static void CalculateDCFCLKDeepSleep(
 	double DisplayPipeLineDeliveryTimeLuma;
 	double DisplayPipeLineDeliveryTimeChroma;
 	double ReadBandwidth = 0.0;
-	int k;
+	unsigned int k;
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 
@@ -5917,7 +5926,7 @@ static void CalculatePixelDeliveryTimes(
 		double PSCL_THROUGHPUT[],
 		double PSCL_THROUGHPUT_CHROMA[],
 		double DPPCLK[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
 		unsigned int NumberOfCursors[],
 		unsigned int CursorWidth[][DC__NUM_CURSOR__MAX],
@@ -5938,7 +5947,7 @@ static void CalculatePixelDeliveryTimes(
 		double CursorRequestDeliveryTimePrefetch[])
 {
 	double req_per_swath_ub;
-	int k;
+	unsigned int k;
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		if (VRatio[k] <= 1) {
@@ -6012,7 +6021,7 @@ static void CalculatePixelDeliveryTimes(
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		int cursor_req_per_width;
-		cursor_req_per_width = dml_ceil(CursorWidth[k][0] * CursorBPP[k][0] / 256 / 8, 1);
+		cursor_req_per_width = (int)dml_ceil(CursorWidth[k][0] * CursorBPP[k][0] / 256 / 8, 1);
 		if (NumberOfCursors[k] > 0) {
 			if (VRatio[k] <= 1) {
 				CursorRequestDeliveryTime[k] = CursorWidth[k][0] / HRatio[k] / PixelClock[k] / cursor_req_per_width;
@@ -6041,35 +6050,35 @@ static void CalculateMetaAndPTETimes(
 		bool GPUVMEnable,
 		int MetaChunkSize,
 		int MinMetaChunkSizeBytes,
-		int HTotal[],
+		unsigned int HTotal[],
 		double VRatio[],
 		double VRatioChroma[],
 		double DestinationLinesToRequestRowInVBlank[],
 		double DestinationLinesToRequestRowInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
-		int dpte_row_height[],
-		int dpte_row_height_chroma[],
-		int meta_row_width[],
-		int meta_row_width_chroma[],
-		int meta_row_height[],
-		int meta_row_height_chroma[],
-		int meta_req_width[],
-		int meta_req_width_chroma[],
-		int meta_req_height[],
-		int meta_req_height_chroma[],
-		int dpte_group_bytes[],
-		int PTERequestSizeY[],
-		int PTERequestSizeC[],
-		int PixelPTEReqWidthY[],
-		int PixelPTEReqHeightY[],
-		int PixelPTEReqWidthC[],
-		int PixelPTEReqHeightC[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
+		unsigned int dpte_row_height[],
+		unsigned int dpte_row_height_chroma[],
+		unsigned int meta_row_width[],
+		unsigned int meta_row_width_chroma[],
+		unsigned int meta_row_height[],
+		unsigned int meta_row_height_chroma[],
+		unsigned int meta_req_width[],
+		unsigned int meta_req_width_chroma[],
+		unsigned int meta_req_height[],
+		unsigned int meta_req_height_chroma[],
+		unsigned int dpte_group_bytes[],
+		unsigned int PTERequestSizeY[],
+		unsigned int PTERequestSizeC[],
+		unsigned int PixelPTEReqWidthY[],
+		unsigned int PixelPTEReqHeightY[],
+		unsigned int PixelPTEReqWidthC[],
+		unsigned int PixelPTEReqHeightC[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
 		double DST_Y_PER_PTE_ROW_NOM_L[],
 		double DST_Y_PER_PTE_ROW_NOM_C[],
 		double DST_Y_PER_META_ROW_NOM_L[],
@@ -6146,7 +6155,7 @@ static void CalculateMetaAndPTETimes(
 			} else {
 				meta_chunk_width_chroma = MetaChunkSize * 1024 * 256 / BytePerPixelC[k] / meta_row_height_chroma[k];
 				min_meta_chunk_width_chroma = MinMetaChunkSizeBytes * 256 / BytePerPixelC[k] / meta_row_height_chroma[k];
-				meta_chunk_per_row_int_chroma = (double) meta_row_width_chroma[k] / meta_chunk_width_chroma;
+				meta_chunk_per_row_int_chroma = (unsigned int)((double) meta_row_width_chroma[k] / meta_chunk_width_chroma);
 				meta_row_remainder_chroma = meta_row_width_chroma[k] % meta_chunk_width_chroma;
 				if (SourceScan[k] != dm_vert) {
 					meta_chunk_threshold_chroma = 2 * min_meta_chunk_width_chroma - meta_req_width_chroma[k];
@@ -6179,7 +6188,7 @@ static void CalculateMetaAndPTETimes(
 			} else {
 				dpte_group_width_luma = dpte_group_bytes[k] / PTERequestSizeY[k] * PixelPTEReqHeightY[k];
 			}
-			dpte_groups_per_row_luma_ub = dml_ceil(1.0 * dpte_row_width_luma_ub[k] / dpte_group_width_luma, 1);
+			dpte_groups_per_row_luma_ub = (unsigned int)dml_ceil(1.0 * dpte_row_width_luma_ub[k] / dpte_group_width_luma, 1);
 			time_per_pte_group_nom_luma[k] = DST_Y_PER_PTE_ROW_NOM_L[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_luma_ub;
 			time_per_pte_group_vblank_luma[k] = DestinationLinesToRequestRowInVBlank[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_luma_ub;
 			time_per_pte_group_flip_luma[k] = DestinationLinesToRequestRowInImmediateFlip[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_luma_ub;
@@ -6193,7 +6202,7 @@ static void CalculateMetaAndPTETimes(
 				} else {
 					dpte_group_width_chroma = dpte_group_bytes[k] / PTERequestSizeC[k] * PixelPTEReqHeightC[k];
 				}
-				dpte_groups_per_row_chroma_ub = dml_ceil(1.0 * dpte_row_width_chroma_ub[k] / dpte_group_width_chroma, 1);
+				dpte_groups_per_row_chroma_ub = (unsigned int)dml_ceil(1.0 * dpte_row_width_chroma_ub[k] / dpte_group_width_chroma, 1);
 				time_per_pte_group_nom_chroma[k] = DST_Y_PER_PTE_ROW_NOM_C[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_chroma_ub;
 				time_per_pte_group_vblank_chroma[k] = DestinationLinesToRequestRowInVBlank[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_chroma_ub;
 				time_per_pte_group_flip_chroma[k] = DestinationLinesToRequestRowInImmediateFlip[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_chroma_ub;
@@ -6214,18 +6223,18 @@ static void CalculateVMGroupAndRequestTimes(
 		bool GPUVMEnable,
 		unsigned int GPUVMMaxPageTableLevels,
 		unsigned int HTotal[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		double DestinationLinesToRequestVMInVBlank[],
 		double DestinationLinesToRequestVMInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
-		int vm_group_bytes[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
+		unsigned int vm_group_bytes[],
 		unsigned int dpde0_bytes_per_frame_ub_l[],
 		unsigned int dpde0_bytes_per_frame_ub_c[],
-		int meta_pte_bytes_per_frame_ub_l[],
-		int meta_pte_bytes_per_frame_ub_c[],
+		unsigned int meta_pte_bytes_per_frame_ub_l[],
+		unsigned int meta_pte_bytes_per_frame_ub_c[],
 		double TimePerVMGroupVBlank[],
 		double TimePerVMGroupFlip[],
 		double TimePerVMRequestVBlank[],
@@ -6235,34 +6244,34 @@ static void CalculateVMGroupAndRequestTimes(
 	(void)dpte_row_width_chroma_ub;
 	int num_group_per_lower_vm_stage;
 	int num_req_per_lower_vm_stage;
-	int k;
+	unsigned int k;
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		if (GPUVMEnable == true && (DCCEnable[k] == true || GPUVMMaxPageTableLevels > 1)) {
 			if (DCCEnable[k] == false) {
 				if (BytePerPixelC[k] > 0) {
-					num_group_per_lower_vm_stage = dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-							+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+					num_group_per_lower_vm_stage = (int)(dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+							+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 				} else {
-					num_group_per_lower_vm_stage = dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+					num_group_per_lower_vm_stage = (int)dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
 				}
 			} else {
 				if (GPUVMMaxPageTableLevels == 1) {
 					if (BytePerPixelC[k] > 0) {
-						num_group_per_lower_vm_stage = dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+						num_group_per_lower_vm_stage = (int)(dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 					} else {
-						num_group_per_lower_vm_stage = dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+						num_group_per_lower_vm_stage = (int)dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
 					}
 				} else {
 					if (BytePerPixelC[k] > 0) {
-						num_group_per_lower_vm_stage = 2 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+						num_group_per_lower_vm_stage = (int)(2 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
 								+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1)
 								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 					} else {
-						num_group_per_lower_vm_stage = 1 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+						num_group_per_lower_vm_stage = (int)(1 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1));
 					}
 				}
 			}
@@ -6333,29 +6342,29 @@ static void CalculateStutterEfficiency(
 		bool ProgressiveToInterlaceUnitInOPP,
 		bool Interlace[],
 		double MinTTUVBlank[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		unsigned int DETBufferSizeY[],
-		int BytePerPixelY[],
+		unsigned int BytePerPixelY[],
 		double BytePerPixelDETY[],
 		double SwathWidthY[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		double NetDCCRateLuma[],
 		double NetDCCRateChroma[],
 		double DCCFractionOfZeroSizeRequestsLuma[],
 		double DCCFractionOfZeroSizeRequestsChroma[],
-		int HTotal[],
-		int VTotal[],
+		unsigned int HTotal[],
+		unsigned int VTotal[],
 		double PixelClock[],
 		double VRatio[],
 		enum scan_direction_class SourceScan[],
-		int BlockHeight256BytesY[],
-		int BlockWidth256BytesY[],
-		int BlockHeight256BytesC[],
-		int BlockWidth256BytesC[],
-		int DCCYMaxUncompressedBlock[],
-		int DCCCMaxUncompressedBlock[],
-		int VActive[],
+		unsigned int BlockHeight256BytesY[],
+		unsigned int BlockWidth256BytesY[],
+		unsigned int BlockHeight256BytesC[],
+		unsigned int BlockWidth256BytesC[],
+		unsigned int DCCYMaxUncompressedBlock[],
+		unsigned int DCCCMaxUncompressedBlock[],
+		unsigned int VActive[],
 		bool DCCEnable[],
 		bool WritebackEnable[],
 		double ReadBandwidthPlaneLuma[],
@@ -6573,8 +6582,8 @@ static void CalculateStutterEfficiency(
 #endif
 		*StutterEfficiencyNotIncludingVBlank = dml_max(0., 1 - (SRExitTime + StutterBurstTime) / *StutterPeriod) * 100;
 		*Z8StutterEfficiencyNotIncludingVBlank = dml_max(0., 1 - (SRExitZ8Time + StutterBurstTime) / *StutterPeriod) * 100;
-		*NumberOfStutterBurstsPerFrame = (*StutterEfficiencyNotIncludingVBlank > 0 ? dml_ceil(VActiveTimeCriticalPlane / *StutterPeriod, 1) : 0);
-		*Z8NumberOfStutterBurstsPerFrame = (*Z8StutterEfficiencyNotIncludingVBlank > 0 ? dml_ceil(VActiveTimeCriticalPlane / *StutterPeriod, 1) : 0);
+		*NumberOfStutterBurstsPerFrame = (int)(*StutterEfficiencyNotIncludingVBlank > 0 ? dml_ceil(VActiveTimeCriticalPlane / *StutterPeriod, 1) : 0);
+		*Z8NumberOfStutterBurstsPerFrame = (int)(*Z8StutterEfficiencyNotIncludingVBlank > 0 ? dml_ceil(VActiveTimeCriticalPlane / *StutterPeriod, 1) : 0);
 	} else {
 		*StutterEfficiencyNotIncludingVBlank = 0.;
 		*Z8StutterEfficiencyNotIncludingVBlank = 0.;
@@ -6640,48 +6649,48 @@ static void CalculateSwathAndDETConfiguration(
 		enum scan_direction_class SourceScan[],
 		enum source_format_class SourcePixelFormat[],
 		enum dm_swizzle_mode SurfaceTiling[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BlendingAndTiming[],
-		int BytePerPixY[],
-		int BytePerPixC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
 		double BytePerPixDETY[],
 		double BytePerPixDETC[],
-		int HActive[],
+		unsigned int HActive[],
 		double HRatio[],
 		double HRatioChroma[],
-		int DPPPerPlane[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[],
+		unsigned int DPPPerPlane[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[],
 		double SwathWidth[],
 		double SwathWidthChroma[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		unsigned int DETBufferSizeY[],
 		unsigned int DETBufferSizeC[],
 		bool ViewportSizeSupportPerPlane[],
 		bool *ViewportSizeSupport)
 {
 	(void)HRatioChroma;
-	int MaximumSwathHeightY[DC__NUM_DPP__MAX];
-	int MaximumSwathHeightC[DC__NUM_DPP__MAX];
-	int MinimumSwathHeightY;
-	int MinimumSwathHeightC;
-	int RoundedUpMaxSwathSizeBytesY;
-	int RoundedUpMaxSwathSizeBytesC;
-	int RoundedUpMinSwathSizeBytesY;
-	int RoundedUpMinSwathSizeBytesC;
-	int RoundedUpSwathSizeBytesY;
-	int RoundedUpSwathSizeBytesC;
+	unsigned int MaximumSwathHeightY[DC__NUM_DPP__MAX];
+	unsigned int MaximumSwathHeightC[DC__NUM_DPP__MAX];
+	unsigned int MinimumSwathHeightY;
+	unsigned int MinimumSwathHeightC;
+	unsigned int RoundedUpMaxSwathSizeBytesY;
+	unsigned int RoundedUpMaxSwathSizeBytesC;
+	unsigned int RoundedUpMinSwathSizeBytesY;
+	unsigned int RoundedUpMinSwathSizeBytesC;
+	unsigned int RoundedUpSwathSizeBytesY;
+	unsigned int RoundedUpSwathSizeBytesC;
 	double SwathWidthSingleDPP[DC__NUM_DPP__MAX];
 	double SwathWidthSingleDPPChroma[DC__NUM_DPP__MAX];
 	int k;
@@ -6755,17 +6764,17 @@ static void CalculateSwathAndDETConfiguration(
 			}
 		}
 
-		RoundedUpMaxSwathSizeBytesY = swath_width_luma_ub[k] * BytePerPixDETY[k] * MaximumSwathHeightY[k];
-		RoundedUpMinSwathSizeBytesY = swath_width_luma_ub[k] * BytePerPixDETY[k] * MinimumSwathHeightY;
+		RoundedUpMaxSwathSizeBytesY = (int)(swath_width_luma_ub[k] * BytePerPixDETY[k] * MaximumSwathHeightY[k]);
+		RoundedUpMinSwathSizeBytesY = (int)(swath_width_luma_ub[k] * BytePerPixDETY[k] * MinimumSwathHeightY);
 		if (SourcePixelFormat[k] == dm_420_10) {
-			RoundedUpMaxSwathSizeBytesY = dml_ceil((double) RoundedUpMaxSwathSizeBytesY, 256);
-			RoundedUpMinSwathSizeBytesY = dml_ceil((double) RoundedUpMinSwathSizeBytesY, 256);
+			RoundedUpMaxSwathSizeBytesY = (int)dml_ceil((double) RoundedUpMaxSwathSizeBytesY, 256);
+			RoundedUpMinSwathSizeBytesY = (int)dml_ceil((double) RoundedUpMinSwathSizeBytesY, 256);
 		}
-		RoundedUpMaxSwathSizeBytesC = swath_width_chroma_ub[k] * BytePerPixDETC[k] * MaximumSwathHeightC[k];
-		RoundedUpMinSwathSizeBytesC = swath_width_chroma_ub[k] * BytePerPixDETC[k] * MinimumSwathHeightC;
+		RoundedUpMaxSwathSizeBytesC = (int)(swath_width_chroma_ub[k] * BytePerPixDETC[k] * MaximumSwathHeightC[k]);
+		RoundedUpMinSwathSizeBytesC = (int)(swath_width_chroma_ub[k] * BytePerPixDETC[k] * MinimumSwathHeightC);
 		if (SourcePixelFormat[k] == dm_420_10) {
-			RoundedUpMaxSwathSizeBytesC = dml_ceil(RoundedUpMaxSwathSizeBytesC, 256);
-			RoundedUpMinSwathSizeBytesC = dml_ceil(RoundedUpMinSwathSizeBytesC, 256);
+			RoundedUpMaxSwathSizeBytesC = (int)dml_ceil(RoundedUpMaxSwathSizeBytesC, 256);
+			RoundedUpMinSwathSizeBytesC = (int)dml_ceil(RoundedUpMinSwathSizeBytesC, 256);
 		}
 
 		if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC <= DETBufferSizeInKByte * 1024 / 2) {
@@ -6794,14 +6803,14 @@ static void CalculateSwathAndDETConfiguration(
 		{
 		double actDETBufferSizeInKByte = dml_ceil(DETBufferSizeInKByte, 64);
 		if (SwathHeightC[k] == 0) {
-			DETBufferSizeY[k] = actDETBufferSizeInKByte * 1024;
+			DETBufferSizeY[k] = (unsigned int)(actDETBufferSizeInKByte * 1024);
 			DETBufferSizeC[k] = 0;
 		} else if (RoundedUpSwathSizeBytesY <= 1.5 * RoundedUpSwathSizeBytesC) {
-			DETBufferSizeY[k] = actDETBufferSizeInKByte * 1024 / 2;
-			DETBufferSizeC[k] = actDETBufferSizeInKByte * 1024 / 2;
+			DETBufferSizeY[k] = (unsigned int)(actDETBufferSizeInKByte * 1024 / 2);
+			DETBufferSizeC[k] = (unsigned int)(actDETBufferSizeInKByte * 1024 / 2);
 		} else {
-			DETBufferSizeY[k] = dml_floor(actDETBufferSizeInKByte * 1024 * 2 / 3, 1024);
-			DETBufferSizeC[k] = actDETBufferSizeInKByte * 1024 / 3;
+			DETBufferSizeY[k] = (unsigned int)dml_floor(actDETBufferSizeInKByte * 1024 * 2 / 3, 1024);
+			DETBufferSizeC[k] = (unsigned int)(actDETBufferSizeInKByte * 1024 / 3);
 		}
 
 		if (RoundedUpMinSwathSizeBytesY + RoundedUpMinSwathSizeBytesC > actDETBufferSizeInKByte * 1024 / 2 || SwathWidth[k] > MaximumSwathWidthLuma[k]
@@ -6820,31 +6829,31 @@ static void CalculateSwathWidth(
 		int NumberOfActivePlanes,
 		enum source_format_class SourcePixelFormat[],
 		enum scan_direction_class SourceScan[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BytePerPixY[],
-		int BytePerPixC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
-		int BlendingAndTiming[],
-		int HActive[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int HActive[],
 		double HRatio[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		double SwathWidthSingleDPPY[],
 		double SwathWidthSingleDPPC[],
 		double SwathWidthY[],
 		double SwathWidthC[],
-		int MaximumSwathHeightY[],
-		int MaximumSwathHeightC[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[])
+		unsigned int MaximumSwathHeightY[],
+		unsigned int MaximumSwathHeightC[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[])
 {
 	(void)BytePerPixY;
 	enum odm_combine_mode MainPlaneODMCombine;
@@ -6901,8 +6910,8 @@ static void CalculateSwathWidth(
 			SwathWidthC[k] = SwathWidthSingleDPPC[k];
 		}
 		{
-		int surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
-		int surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
+		int surface_width_ub_l = (int)dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
+		int surface_height_ub_l = (int)dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
 
 #ifdef __DML_VBA_DEBUG__
 		dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l);
@@ -6911,11 +6920,11 @@ static void CalculateSwathWidth(
 		if (SourceScan[k] != dm_vert) {
 			MaximumSwathHeightY[k] = Read256BytesBlockHeightY[k];
 			MaximumSwathHeightC[k] = Read256BytesBlockHeightC[k];
-			swath_width_luma_ub[k] = dml_min(surface_width_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]);
+			swath_width_luma_ub[k] = (int)dml_min(surface_width_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]);
 			if (BytePerPixC[k] > 0) {
-				int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
+				int surface_width_ub_c = (int)dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
 
-				swath_width_chroma_ub[k] = dml_min(
+				swath_width_chroma_ub[k] = (int)dml_min(
 						surface_width_ub_c,
 						(int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockWidthC[k]) + Read256BytesBlockWidthC[k]);
 			} else {
@@ -6924,11 +6933,11 @@ static void CalculateSwathWidth(
 		} else {
 			MaximumSwathHeightY[k] = Read256BytesBlockWidthY[k];
 			MaximumSwathHeightC[k] = Read256BytesBlockWidthC[k];
-			swath_width_luma_ub[k] = dml_min(surface_height_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
+			swath_width_luma_ub[k] = (int)dml_min(surface_height_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
 			if (BytePerPixC[k] > 0) {
-				int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
+				int surface_height_ub_c = (int)dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
 
-				swath_width_chroma_ub[k] = dml_min(
+				swath_width_chroma_ub[k] = (int)dml_min(
 						surface_height_ub_c,
 						(int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockHeightC[k]) + Read256BytesBlockHeightC[k]);
 			} else {
@@ -6951,8 +6960,8 @@ static double CalculateExtraLatency(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double HostVMInefficiencyFactor,
 		double HostVMMinPageSize,
 		int HostVMMaxNonCachedPageTableLevels)
@@ -6997,8 +7006,8 @@ static double CalculateExtraLatencyBytes(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double HostVMInefficiencyFactor,
 		double HostVMMinPageSize,
 		int HostVMMaxNonCachedPageTableLevels)
@@ -7010,9 +7019,9 @@ static double CalculateExtraLatencyBytes(
 		if (HostVMMinPageSize < 2048) {
 			HostVMDynamicLevels = HostVMMaxNonCachedPageTableLevels;
 		} else if (HostVMMinPageSize >= 2048 && HostVMMinPageSize < 1048576) {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
+			HostVMDynamicLevels = (int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
 		} else {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
+			HostVMDynamicLevels = (int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
 		}
 	} else {
 		HostVMDynamicLevels = 0;
@@ -7052,7 +7061,9 @@ static noinline_for_stack void UseMinimumDCFCLK(
 		int ReorderingBytes)
 {
 	struct vba_vars_st *v = &mode_lib->vba;
-	int dummy1, i, j, k;
+	unsigned int dummy1;
+	unsigned int j, k;
+	unsigned int i;
 	double NormalEfficiency,  dummy2, dummy3;
 	double TotalMaxPrefetchFlipDPTERowBandwidth[DC__VOLTAGE_STATES][2];
 
@@ -7070,7 +7081,7 @@ static noinline_for_stack void UseMinimumDCFCLK(
 			double ExtraLatencyBytes;
 			double ExtraLatencyCycles;
 			double DCFCLKRequiredForPeakBandwidth;
-			int NoOfDPPState[DC__NUM_DPP__MAX];
+			unsigned int NoOfDPPState[DC__NUM_DPP__MAX];
 			double MinimumTvmPlus2Tr0;
 
 			TotalMaxPrefetchFlipDPTERowBandwidth[i][j] = 0;
@@ -7079,9 +7090,8 @@ static noinline_for_stack void UseMinimumDCFCLK(
 						+ v->NoOfDPP[i][j][k] * v->DPTEBytesPerRow[i][j][k] / (15.75 * v->HTotal[k] / v->PixelClock[k]);
 			}
 
-			for (k = 0; k <= v->NumberOfActivePlanes - 1; ++k) {
+			for (k = 0; k < v->NumberOfActivePlanes; ++k)
 				NoOfDPPState[k] = v->NoOfDPP[i][j][k];
-			}
 
 			MinimumTWait = CalculateTWait(MaxPrefetchMode, v->FinalDRAMClockChangeLatency, v->UrgLatency[i], v->SREnterPlusExitTime);
 			NonDPTEBandwidth = v->TotalVActivePixelBandwidth[i][j] + v->TotalVActiveCursorBandwidth[i][j] + v->TotalMetaRowBandwidth[i][j];
@@ -7182,9 +7192,8 @@ static noinline_for_stack void UseMinimumDCFCLK(
 				}
 			}
 			DCFCLKRequiredForPeakBandwidth = 0;
-			for (k = 0; k <= v->NumberOfActivePlanes - 1; ++k) {
+			for (k = 0; k < v->NumberOfActivePlanes; ++k)
 				DCFCLKRequiredForPeakBandwidth = DCFCLKRequiredForPeakBandwidth + DCFCLKRequiredForPeakBandwidthPerPlane[k];
-			}
 			MinimumTvmPlus2Tr0 = v->UrgLatency[i]
 					* (v->GPUVMEnable == true ?
 							(v->HostVMEnable == true ?
@@ -7217,12 +7226,12 @@ static void CalculateUnboundedRequestAndCompressedBufferSize(
 		int CompressedBufferSegmentSizeInkByteFinal,
 		enum output_encoder_class *Output,
 		bool *UnboundedRequestEnabled,
-		int *CompressedBufferSizeInkByte)
+		unsigned int *CompressedBufferSizeInkByte)
 {
 	double actDETBufferSizeInKByte = dml_ceil(DETBufferSizeInKByte, 64);
 
 	*UnboundedRequestEnabled = UnboundedRequest(UseUnboundedRequestingFinal, TotalActiveDPP, NoChromaPlanes, Output[0]);
-	*CompressedBufferSizeInkByte = (
+	*CompressedBufferSizeInkByte = (int)(
 			*UnboundedRequestEnabled == true ?
 					ConfigReturnBufferSizeInKByte - TotalActiveDPP * actDETBufferSizeInKByte :
 					ConfigReturnBufferSizeInKByte - MaxNumDPP * actDETBufferSizeInKByte);
@@ -7235,7 +7244,7 @@ static void CalculateUnboundedRequestAndCompressedBufferSize(
 	dml_print("DML::%s: UseUnboundedRequestingFinal = %d\n", __func__, UseUnboundedRequestingFinal);
 	dml_print("DML::%s: actDETBufferSizeInKByte = %f\n", __func__, actDETBufferSizeInKByte);
 	dml_print("DML::%s: UnboundedRequestEnabled = %d\n", __func__, *UnboundedRequestEnabled);
-	dml_print("DML::%s: CompressedBufferSizeInkByte = %d\n", __func__, *CompressedBufferSizeInkByte);
+	dml_print("DML::%s: CompressedBufferSizeInkByte = %u\n", __func__, *CompressedBufferSizeInkByte);
 #endif
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c
index dfa1bc31eb0a3..ada80b3086ff2 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn31/display_rq_dlg_calc_31.c
@@ -117,11 +117,11 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib, display_rq_regs_s
 
 	extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_l), &rq_param->sizing.rq_l);
 
-	rq_regs->rq_regs_l.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height), 1) - 3;
+	rq_regs->rq_regs_l.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height), 1) - 3);
 
 	if (rq_param->yuv420) {
 		extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_c), &rq_param->sizing.rq_c);
-		rq_regs->rq_regs_c.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height), 1) - 3;
+		rq_regs->rq_regs_c.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height), 1) - 3);
 	}
 
 	rq_regs->rq_regs_l.swath_height = dml_log2(rq_param->dlg.rq_l.swath_height);
@@ -141,12 +141,12 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib, display_rq_regs_s
 	// Note: detile_buf_plane1_addr is in unit of 1KB
 	if (rq_param->yuv420) {
 		if ((double) rq_param->misc.rq_l.stored_swath_bytes / (double) rq_param->misc.rq_c.stored_swath_bytes <= 1.5) {
-			detile_buf_plane1_addr = (detile_buf_size_in_bytes / 2.0 / 1024.0); // half to chroma
+			detile_buf_plane1_addr = (unsigned int)(detile_buf_size_in_bytes / 2.0 / 1024.0); // half to chroma
 #ifdef __DML_RQ_DLG_CALC_DEBUG__
 					dml_print("DML_DLG: %s: detile_buf_plane1_addr = %0d (1/2 to chroma)\n", __func__, detile_buf_plane1_addr);
 #endif
 		} else {
-			detile_buf_plane1_addr = dml_round_to_multiple((unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0), 1024, 0) / 1024.0; // 2/3 to luma
+			detile_buf_plane1_addr = (unsigned int)(dml_round_to_multiple((unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0), 1024, 0) / 1024.0); // 2/3 to luma
 #ifdef __DML_RQ_DLG_CALC_DEBUG__
 					dml_print("DML_DLG: %s: detile_buf_plane1_addr = %0d (1/3 chroma)\n", __func__, detile_buf_plane1_addr);
 #endif
@@ -189,8 +189,8 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib, display_rq_p
 #endif
 
 	if (rq_param->yuv420_10bpc) {
-		full_swath_bytes_packed_l = dml_round_to_multiple(rq_param->misc.rq_l.full_swath_bytes * 2.0 / 3.0, 256, 1) + 256;
-		full_swath_bytes_packed_c = dml_round_to_multiple(rq_param->misc.rq_c.full_swath_bytes * 2.0 / 3.0, 256, 1) + 256;
+		full_swath_bytes_packed_l = (unsigned int)(dml_round_to_multiple((unsigned int)(rq_param->misc.rq_l.full_swath_bytes * 2.0 / 3.0), 256, 1) + 256);
+		full_swath_bytes_packed_c = (unsigned int)(dml_round_to_multiple((unsigned int)(rq_param->misc.rq_c.full_swath_bytes * 2.0 / 3.0), 256, 1) + 256);
 #ifdef __DML_RQ_DLG_CALC_DEBUG__
 		dml_print("DML_DLG: %s: full_swath_bytes_packed_l = %0d (3-2 packing)\n", __func__, full_swath_bytes_packed_l);
 		dml_print("DML_DLG: %s: full_swath_bytes_packed_c = %0d (3-2 packing)\n", __func__, full_swath_bytes_packed_c);
@@ -421,7 +421,7 @@ static void get_meta_and_pte_attr(
 	// "/2" is like square root
 	// blk is vertical biased
 	if (tiling != dm_sw_linear)
-		log2_blk_height = log2_blk256_height + dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
+		log2_blk_height = (unsigned int)(log2_blk256_height + dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1));
 	else
 		log2_blk_height = 0;	// blk height of 1
 
@@ -574,7 +574,7 @@ static void get_meta_and_pte_attr(
 	if (surf_linear) {
 		unsigned int dpte_row_height;
 
-		log2_dpte_row_height_linear = dml_floor(dml_log2(dpte_buf_in_pte_reqs * dpte_req_width / data_pitch), 1);
+		log2_dpte_row_height_linear = (unsigned int)(dml_floor(dml_log2(dpte_buf_in_pte_reqs * dpte_req_width / data_pitch), 1));
 
 		dml_print("DML_DLG: %s: is_chroma                   = %d\n", __func__, is_chroma);
 		dml_print("DML_DLG: %s: dpte_buf_in_pte_reqs        = %d\n", __func__, dpte_buf_in_pte_reqs);
@@ -640,7 +640,7 @@ static void get_meta_and_pte_attr(
 
 	// since dpte groups are only aligned to dpte_req_width and not dpte_group_width,
 	// the upper bound for the dpte groups per row is as follows.
-	rq_dlg_param->dpte_groups_per_row_ub = dml_ceil((double) dpte_row_width_ub / dpte_group_width, 1);
+	rq_dlg_param->dpte_groups_per_row_ub = (unsigned int)dml_ceil((double) dpte_row_width_ub / dpte_group_width, 1);
 }
 
 static void get_surf_rq_param(
@@ -665,7 +665,7 @@ static void get_surf_rq_param(
 		vp_height = pipe_param->src.viewport_height_c;
 		data_pitch = pipe_param->src.data_pitch_c;
 		meta_pitch = pipe_param->src.meta_pitch_c;
-		surface_height = pipe_param->src.surface_height_y / 2.0;
+		surface_height = (unsigned int)(pipe_param->src.surface_height_y / 2.0);
 	} else {
 		vp_width = pipe_param->src.viewport_width / ppe;
 		vp_height = pipe_param->src.viewport_height;
@@ -683,18 +683,18 @@ static void get_surf_rq_param(
 		access_dir = (pipe_param->src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
 		hactive_odm = pipe_param->dest.hactive / ((unsigned int) pipe_param->dest.odm_combine * 2);
 		if (is_chroma) {
-			full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width;
-			src_hactive_odm = pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_odm;
+			full_src_vp_width = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width);
+			src_hactive_odm = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_odm);
 		} else {
-			full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width;
-			src_hactive_odm = pipe_param->scale_ratio_depth.hscl_ratio * hactive_odm;
+			full_src_vp_width = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width);
+			src_hactive_odm = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio * hactive_odm);
 		}
 
 		if (access_dir == 0) {
-			vp_width = dml_min(full_src_vp_width, src_hactive_odm);
+			vp_width = (unsigned int)dml_min(full_src_vp_width, src_hactive_odm);
 			dml_print("DML_DLG: %s: vp_width = %d\n", __func__, vp_width);
 		} else {
-			vp_height = dml_min(full_src_vp_width, src_hactive_odm);
+			vp_height = (unsigned int)dml_min(full_src_vp_width, src_hactive_odm);
 			dml_print("DML_DLG: %s: vp_height = %d\n", __func__, vp_height);
 
 		}
@@ -814,7 +814,7 @@ static void calculate_ttu_cursor(
 				cur_req_size = 256;
 		}
 
-		cur_req_width = (double) cur_req_size / ((double) cur_bit_per_pixel / 8.0);
+		cur_req_width = (unsigned int)((double) cur_req_size / ((double) cur_bit_per_pixel / 8.0));
 		cur_width_ub = dml_ceil((double) cur_src_width / (double) cur_req_width, 1) * (double) cur_req_width;
 		cur_req_per_width = cur_width_ub / (double) cur_req_width;
 		hactive_cur = (double) cur_src_width / hscl_ratio; // FIXME: oswin to think about what to do for cursor
@@ -1051,8 +1051,8 @@ static void dml_rq_dlg_get_dlg_params(
 	if (interlaced)
 		vstartup_start = vstartup_start / 2;
 
-	dst_x_after_scaler = get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
-	dst_y_after_scaler = get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
+	dst_x_after_scaler = (unsigned int)get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
+	dst_y_after_scaler = (unsigned int)get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
 
 	// do some adjustment on the dst_after scaler to account for odm combine mode
 	dml_print("DML_DLG: %s: input dst_x_after_scaler   = %d\n", __func__, dst_x_after_scaler);
@@ -1278,9 +1278,9 @@ static void dml_rq_dlg_get_dlg_params(
 	}
 
 	if (src->dynamic_metadata_enable && src->gpuvm)
-		disp_dlg_regs->refcyc_per_vm_dmdata = get_refcyc_per_vm_dmdata_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
+		disp_dlg_regs->refcyc_per_vm_dmdata = (unsigned int)(get_refcyc_per_vm_dmdata_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz); // From VBA
 
-	disp_dlg_regs->dmdata_dl_delta = get_dmdata_dl_delta_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
+	disp_dlg_regs->dmdata_dl_delta = (unsigned int)(get_dmdata_dl_delta_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz); // From VBA
 
 	// TTU - Luma / Chroma
 	if (access_dir) {  // vertical access
@@ -1386,8 +1386,8 @@ static void dml_rq_dlg_get_dlg_params(
 				src->cur0_src_width,
 				(enum cursor_bpp) (src->cur0_bpp));
 
-		vba__refcyc_per_req_delivery_pre_cur0 = get_refcyc_per_cursor_req_delivery_pre_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
-		vba__refcyc_per_req_delivery_cur0 = get_refcyc_per_cursor_req_delivery_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
+				vba__refcyc_per_req_delivery_pre_cur0 = (float)(get_refcyc_per_cursor_req_delivery_pre_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz); // From VBA
+				vba__refcyc_per_req_delivery_cur0 = (float)(get_refcyc_per_cursor_req_delivery_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz); // From VBA
 
 		dml_print("DML_DLG: %s: refcyc_per_req_delivery_pre_cur0 = %3.2f\n", __func__, refcyc_per_req_delivery_pre_cur0);
 		dml_print("DML_DLG: %s: refcyc_per_req_delivery_cur0     = %3.2f\n", __func__, refcyc_per_req_delivery_cur0);
@@ -1407,7 +1407,7 @@ static void dml_rq_dlg_get_dlg_params(
 	// Assignment to register structures
 	disp_dlg_regs->dst_y_after_scaler = dst_y_after_scaler;	// in terms of line
 	ASSERT(disp_dlg_regs->dst_y_after_scaler < (unsigned int)8);
-	disp_dlg_regs->refcyc_x_after_scaler = dst_x_after_scaler * ref_freq_to_pix_freq;	// in terms of refclk
+	disp_dlg_regs->refcyc_x_after_scaler = (unsigned int)(dst_x_after_scaler * ref_freq_to_pix_freq);	// in terms of refclk
 	ASSERT(disp_dlg_regs->refcyc_x_after_scaler < (unsigned int)dml_pow(2, 13));
 	disp_dlg_regs->dst_y_prefetch = (unsigned int) (dst_y_prefetch * dml_pow(2, 2));
 	disp_dlg_regs->dst_y_per_vm_vblank = (unsigned int) (dst_y_per_vm_vblank * dml_pow(2, 2));
@@ -1444,23 +1444,23 @@ static void dml_rq_dlg_get_dlg_params(
 		disp_dlg_regs->refcyc_per_meta_chunk_flip_c = (unsigned int) (dst_y_per_row_flip * htotal * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_c;
 	}
 
-	disp_dlg_regs->refcyc_per_vm_group_vblank = get_refcyc_per_vm_group_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz;            // From VBA
-	disp_dlg_regs->refcyc_per_vm_group_flip = get_refcyc_per_vm_group_flip_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz;            // From VBA
-	disp_dlg_regs->refcyc_per_vm_req_vblank = get_refcyc_per_vm_req_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10); // From VBA
-	disp_dlg_regs->refcyc_per_vm_req_flip = get_refcyc_per_vm_req_flip_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10);   // From VBA
+	disp_dlg_regs->refcyc_per_vm_group_vblank = (unsigned int)(get_refcyc_per_vm_group_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz);            // From VBA
+	disp_dlg_regs->refcyc_per_vm_group_flip = (unsigned int)(get_refcyc_per_vm_group_flip_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz);            // From VBA
+	disp_dlg_regs->refcyc_per_vm_req_vblank = (unsigned int)(get_refcyc_per_vm_req_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10)); // From VBA
+	disp_dlg_regs->refcyc_per_vm_req_flip = (unsigned int)(get_refcyc_per_vm_req_flip_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10));   // From VBA
 
 	// Clamp to max for now
 	if (disp_dlg_regs->refcyc_per_vm_group_vblank >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_group_vblank = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_group_vblank = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (disp_dlg_regs->refcyc_per_vm_group_flip >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_group_flip = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_group_flip = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (disp_dlg_regs->refcyc_per_vm_req_vblank >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_req_vblank = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_req_vblank = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (disp_dlg_regs->refcyc_per_vm_req_flip >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_req_flip = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_req_flip = (unsigned int)(dml_pow(2, 23) - 1);
 
 	disp_dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int) ((double) dpte_row_height_l / (double) vratio_l * dml_pow(2, 2));
 	ASSERT(disp_dlg_regs->dst_y_per_pte_row_nom_l < (unsigned int)dml_pow(2, 17));
@@ -1484,23 +1484,23 @@ static void dml_rq_dlg_get_dlg_params(
 	disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int) ((double) dpte_row_height_l / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
 			/ (double) dpte_groups_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_pte_group_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_pte_group_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 	disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int) ((double) meta_row_height_l / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
 			/ (double) meta_chunks_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_meta_chunk_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (dual_plane) {
 		disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int) ((double) dpte_row_height_c / (double) vratio_c * (double) htotal * ref_freq_to_pix_freq
 				/ (double) dpte_groups_per_row_ub_c);
 		if (disp_dlg_regs->refcyc_per_pte_group_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_pte_group_nom_c = dml_pow(2, 23) - 1;
+			disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 
 		// TODO: Is this the right calculation? Does htotal need to be halved?
 		disp_dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int) ((double) meta_row_height_c / (double) vratio_c * (double) htotal * ref_freq_to_pix_freq
 				/ (double) meta_chunks_per_row_ub_c);
 		if (disp_dlg_regs->refcyc_per_meta_chunk_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = dml_pow(2, 23) - 1;
+			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 	}
 
 	disp_dlg_regs->refcyc_per_line_delivery_pre_l = (unsigned int) dml_floor(refcyc_per_line_delivery_pre_l, 1);
@@ -1541,7 +1541,7 @@ static void dml_rq_dlg_get_dlg_params(
 	disp_ttu_regs->qos_ramp_disable_c = 0;
 	disp_ttu_regs->qos_ramp_disable_cur0 = 0;
 
-	disp_ttu_regs->min_ttu_vblank = min_ttu_vblank * refclk_freq_in_mhz;
+	disp_ttu_regs->min_ttu_vblank = (unsigned int)(min_ttu_vblank * refclk_freq_in_mhz);
 	ASSERT(disp_ttu_regs->min_ttu_vblank < dml_pow(2, 24));
 
 	print__ttu_regs_st(mode_lib, disp_ttu_regs);
@@ -1572,7 +1572,7 @@ void dml31_rq_dlg_get_dlg_reg(
 	dlg_sys_param.t_mclk_wm_us = get_wm_dram_clock_change(mode_lib, e2e_pipe_param, num_pipes);
 	dlg_sys_param.t_sr_wm_us = get_wm_stutter_enter_exit(mode_lib, e2e_pipe_param, num_pipes);
 	dlg_sys_param.total_flip_bw = get_total_immediate_flip_bw(mode_lib, e2e_pipe_param, num_pipes);
-	dlg_sys_param.total_flip_bytes = get_total_immediate_flip_bytes(mode_lib, e2e_pipe_param, num_pipes);
+	dlg_sys_param.total_flip_bytes = (unsigned int)get_total_immediate_flip_bytes(mode_lib, e2e_pipe_param, num_pipes);
 
 	print__dlg_sys_params_st(mode_lib, &dlg_sys_param);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
index ab016c294ba7e..2f9ae79da7311 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/dcn314_fpu.c
@@ -185,7 +185,7 @@ void dcn314_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p
 	struct _vcs_dpi_voltage_scaling_st *clock_limits =
 		dcn3_14_soc.clock_limits;
 	unsigned int i, closest_clk_lvl;
-	int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
+	unsigned int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
 	int j;
 
 	dc_assert_fp_enabled();
@@ -284,11 +284,11 @@ static bool is_dual_plane(enum surface_pixel_format format)
 static unsigned int micro_sec_to_vert_lines(unsigned int num_us, struct dc_crtc_timing *timing)
 {
 	unsigned int num_lines = 0;
-	unsigned int lines_time_in_ns = 1000.0 *
-			(((float)timing->h_total * 1000.0) /
-			 ((float)timing->pix_clk_100hz / 10.0));
+	double lines_time_in_ns = 1000.0 *
+			(((double)timing->h_total * 1000.0) /
+			 ((double)timing->pix_clk_100hz / 10.0));
 
-	num_lines = dml_ceil(1000.0 * num_us / lines_time_in_ns, 1.0);
+	num_lines = (unsigned int)dml_ceil(1000.0 * num_us / lines_time_in_ns, 1.0);
 
 	return num_lines;
 }
@@ -308,7 +308,8 @@ int dcn314_populate_dml_pipes_from_context_fpu(struct dc *dc, struct dc_state *c
 					       display_e2e_pipe_params_st *pipes,
 					       enum dc_validate_mode validate_mode)
 {
-	int i, pipe_cnt;
+	int pipe_cnt;
+	unsigned int i;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe = 0;
 	bool upscaled = false;
@@ -409,15 +410,15 @@ int dcn314_populate_dml_pipes_from_context_fpu(struct dc *dc, struct dc_state *c
 		context->bw_ctx.dml.ip.odm_combine_4to1_supported = true;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *cur_pipe = &context->res_ctx.pipe_ctx[i];
 
-		if (!pipe->stream)
+		if (!cur_pipe->stream)
 			continue;
 
-		if (pipe->stream->signal == SIGNAL_TYPE_EDP && dc->debug.seamless_boot_odm_combine &&
-				pipe->stream->apply_seamless_boot_optimization) {
+		if (cur_pipe->stream->signal == SIGNAL_TYPE_EDP && dc->debug.seamless_boot_odm_combine &&
+				cur_pipe->stream->apply_seamless_boot_optimization) {
 
-			if (pipe->stream->apply_boot_odm_mode == dm_odm_combine_policy_2to1) {
+			if (cur_pipe->stream->apply_boot_odm_mode == dm_odm_combine_policy_2to1) {
 				context->bw_ctx.dml.vba.ODMCombinePolicy = dm_odm_combine_policy_2to1;
 				break;
 			}
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
index 033fde774337f..dd9dc0c8cb437 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_mode_vba_314.c
@@ -166,7 +166,7 @@ static bool CalculatePrefetchSchedule(
 		double *Tdmdl_vm,
 		double *Tdmdl,
 		double *TSetup,
-		int *VUpdateOffsetPix,
+		unsigned int *VUpdateOffsetPix,
 		double *VUpdateWidthPix,
 		double *VReadyOffsetPix);
 static double RoundToDFSGranularityUp(double Clock, double VCOSpeed);
@@ -227,19 +227,19 @@ static unsigned int CalculateVMAndRowBytes(
 		unsigned int *MetaRowByte,
 		unsigned int *PixelPTEBytesPerRow,
 		bool *PTEBufferSizeNotExceeded,
-		int *dpte_row_width_ub,
+		unsigned int *dpte_row_width_ub,
 		unsigned int *dpte_row_height,
 		unsigned int *MetaRequestWidth,
 		unsigned int *MetaRequestHeight,
 		unsigned int *meta_row_width,
 		unsigned int *meta_row_height,
-		int *vm_group_bytes,
+		unsigned int *vm_group_bytes,
 		unsigned int *dpte_group_bytes,
 		unsigned int *PixelPTEReqWidth,
 		unsigned int *PixelPTEReqHeight,
 		unsigned int *PTERequestSize,
-		int *DPDE0BytesFrame,
-		int *MetaPTEBytesFrame);
+		unsigned int *DPDE0BytesFrame,
+		unsigned int *MetaPTEBytesFrame);
 static double CalculateTWait(unsigned int PrefetchMode, double DRAMClockChangeLatency, double UrgentLatency, double SREnterPlusExitTime);
 static void CalculateRowBandwidth(
 		bool GPUVMEnable,
@@ -294,7 +294,7 @@ static void CalculateVupdateAndDynamicMetadataParameters(
 		double *Tdmbf,
 		double *Tdmec,
 		double *Tdmsks,
-		int *VUpdateOffsetPix,
+		unsigned int *VUpdateOffsetPix,
 		double *VUpdateWidthPix,
 		double *VReadyOffsetPix);
 
@@ -327,8 +327,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 static void CalculateDCFCLKDeepSleep(
 		struct display_mode_lib *mode_lib,
 		unsigned int NumberOfActivePlanes,
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		double VRatio[],
 		double VRatioChroma[],
 		double SwathWidthY[],
@@ -386,7 +386,7 @@ static void CalculatePixelDeliveryTimes(
 		double PSCL_THROUGHPUT[],
 		double PSCL_THROUGHPUT_CHROMA[],
 		double DPPCLK[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
 		unsigned int NumberOfCursors[],
 		unsigned int CursorWidth[][DC__NUM_CURSOR__MAX],
@@ -407,39 +407,39 @@ static void CalculatePixelDeliveryTimes(
 		double CursorRequestDeliveryTimePrefetch[]);
 
 static void CalculateMetaAndPTETimes(
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		bool GPUVMEnable,
 		int MetaChunkSize,
 		int MinMetaChunkSizeBytes,
-		int HTotal[],
+		unsigned int HTotal[],
 		double VRatio[],
 		double VRatioChroma[],
 		double DestinationLinesToRequestRowInVBlank[],
 		double DestinationLinesToRequestRowInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
-		int dpte_row_height[],
-		int dpte_row_height_chroma[],
-		int meta_row_width[],
-		int meta_row_width_chroma[],
-		int meta_row_height[],
-		int meta_row_height_chroma[],
-		int meta_req_width[],
-		int meta_req_width_chroma[],
-		int meta_req_height[],
-		int meta_req_height_chroma[],
-		int dpte_group_bytes[],
-		int PTERequestSizeY[],
-		int PTERequestSizeC[],
-		int PixelPTEReqWidthY[],
-		int PixelPTEReqHeightY[],
-		int PixelPTEReqWidthC[],
-		int PixelPTEReqHeightC[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
+		unsigned int dpte_row_height[],
+		unsigned int dpte_row_height_chroma[],
+		unsigned int meta_row_width[],
+		unsigned int meta_row_width_chroma[],
+		unsigned int meta_row_height[],
+		unsigned int meta_row_height_chroma[],
+		unsigned int meta_req_width[],
+		unsigned int meta_req_width_chroma[],
+		unsigned int meta_req_height[],
+		unsigned int meta_req_height_chroma[],
+		unsigned int dpte_group_bytes[],
+		unsigned int PTERequestSizeY[],
+		unsigned int PTERequestSizeC[],
+		unsigned int PixelPTEReqWidthY[],
+		unsigned int PixelPTEReqHeightY[],
+		unsigned int PixelPTEReqWidthC[],
+		unsigned int PixelPTEReqHeightC[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
 		double DST_Y_PER_PTE_ROW_NOM_L[],
 		double DST_Y_PER_PTE_ROW_NOM_C[],
 		double DST_Y_PER_META_ROW_NOM_L[],
@@ -462,18 +462,18 @@ static void CalculateVMGroupAndRequestTimes(
 		bool GPUVMEnable,
 		unsigned int GPUVMMaxPageTableLevels,
 		unsigned int HTotal[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		double DestinationLinesToRequestVMInVBlank[],
 		double DestinationLinesToRequestVMInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
-		int vm_group_bytes[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
+		unsigned int vm_group_bytes[],
 		unsigned int dpde0_bytes_per_frame_ub_l[],
 		unsigned int dpde0_bytes_per_frame_ub_c[],
-		int meta_pte_bytes_per_frame_ub_l[],
-		int meta_pte_bytes_per_frame_ub_c[],
+		unsigned int meta_pte_bytes_per_frame_ub_l[],
+		unsigned int meta_pte_bytes_per_frame_ub_c[],
 		double TimePerVMGroupVBlank[],
 		double TimePerVMGroupFlip[],
 		double TimePerVMRequestVBlank[],
@@ -501,29 +501,29 @@ static void CalculateStutterEfficiency(
 		bool ProgressiveToInterlaceUnitInOPP,
 		bool Interlace[],
 		double MinTTUVBlank[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		unsigned int DETBufferSizeY[],
-		int BytePerPixelY[],
+		unsigned int BytePerPixelY[],
 		double BytePerPixelDETY[],
 		double SwathWidthY[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		double NetDCCRateLuma[],
 		double NetDCCRateChroma[],
 		double DCCFractionOfZeroSizeRequestsLuma[],
 		double DCCFractionOfZeroSizeRequestsChroma[],
-		int HTotal[],
-		int VTotal[],
+		unsigned int HTotal[],
+		unsigned int VTotal[],
 		double PixelClock[],
 		double VRatio[],
 		enum scan_direction_class SourceScan[],
-		int BlockHeight256BytesY[],
-		int BlockWidth256BytesY[],
-		int BlockHeight256BytesC[],
-		int BlockWidth256BytesC[],
-		int DCCYMaxUncompressedBlock[],
-		int DCCCMaxUncompressedBlock[],
-		int VActive[],
+		unsigned int BlockHeight256BytesY[],
+		unsigned int BlockWidth256BytesY[],
+		unsigned int BlockHeight256BytesC[],
+		unsigned int BlockWidth256BytesC[],
+		unsigned int DCCYMaxUncompressedBlock[],
+		unsigned int DCCCMaxUncompressedBlock[],
+		unsigned int VActive[],
 		bool DCCEnable[],
 		bool WritebackEnable[],
 		double ReadBandwidthPlaneLuma[],
@@ -547,32 +547,32 @@ static void CalculateSwathAndDETConfiguration(
 		enum scan_direction_class SourceScan[],
 		enum source_format_class SourcePixelFormat[],
 		enum dm_swizzle_mode SurfaceTiling[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BlendingAndTiming[],
-		int BytePerPixY[],
-		int BytePerPixC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
 		double BytePerPixDETY[],
 		double BytePerPixDETC[],
-		int HActive[],
+		unsigned int HActive[],
 		double HRatio[],
 		double HRatioChroma[],
-		int DPPPerPlane[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[],
+		unsigned int DPPPerPlane[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[],
 		double SwathWidth[],
 		double SwathWidthChroma[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		unsigned int DETBufferSizeY[],
 		unsigned int DETBufferSizeC[],
 		bool ViewportSizeSupportPerPlane[],
@@ -582,31 +582,31 @@ static void CalculateSwathWidth(
 		int NumberOfActivePlanes,
 		enum source_format_class SourcePixelFormat[],
 		enum scan_direction_class SourceScan[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BytePerPixY[],
-		int BytePerPixC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
-		int BlendingAndTiming[],
-		int HActive[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int HActive[],
 		double HRatio[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		double SwathWidthSingleDPPY[],
 		double SwathWidthSingleDPPC[],
 		double SwathWidthY[],
 		double SwathWidthC[],
-		int MaximumSwathHeightY[],
-		int MaximumSwathHeightC[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[]);
+		unsigned int MaximumSwathHeightY[],
+		unsigned int MaximumSwathHeightC[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[]);
 
 static double CalculateExtraLatency(
 		int RoundTripPingLatencyCycles,
@@ -620,8 +620,8 @@ static double CalculateExtraLatency(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double HostVMInefficiencyFactor,
 		double HostVMMinPageSize,
 		int HostVMMaxNonCachedPageTableLevels);
@@ -635,8 +635,8 @@ static double CalculateExtraLatencyBytes(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double HostVMInefficiencyFactor,
 		double HostVMMinPageSize,
 		int HostVMMaxNonCachedPageTableLevels);
@@ -660,7 +660,7 @@ static void CalculateUnboundedRequestAndCompressedBufferSize(
 		int CompressedBufferSegmentSizeInkByteFinal,
 		enum output_encoder_class *Output,
 		bool *UnboundedRequestEnabled,
-		int *CompressedBufferSizeInkByte);
+		unsigned int *CompressedBufferSizeInkByte);
 
 static bool UnboundedRequest(enum unbounded_requesting_policy UseUnboundedRequestingFinal, int TotalNumberOfActiveDPP, bool NoChroma, enum output_encoder_class Output);
 static unsigned int CalculateMaxVStartup(
@@ -718,7 +718,7 @@ static unsigned int dscceComputeDelay(
 		pixelsPerClock = 1;
 
 	//initial transmit delay as per PPS
-	initalXmitDelay = dml_round(rcModelSize / 2.0 / BPP / pixelsPerClock);
+	initalXmitDelay = (unsigned int)dml_round(rcModelSize / 2.0 / BPP / pixelsPerClock);
 
 	//compute ssm delay
 	if (bpc == 8)
@@ -887,7 +887,7 @@ static bool CalculatePrefetchSchedule(
 		double *Tdmdl_vm,
 		double *Tdmdl,
 		double *TSetup,
-		int *VUpdateOffsetPix,
+		unsigned int *VUpdateOffsetPix,
 		double *VUpdateWidthPix,
 		double *VReadyOffsetPix)
 {
@@ -1003,13 +1003,13 @@ static bool CalculatePrefetchSchedule(
 	*Tdmdl_vm = (DynamicMetadataEnable == true && DynamicMetadataVMEnabled == true && GPUVMEnable == true ? TWait + Tvm_trips : 0);
 
 	if (myPipe->ScalerEnabled)
-		DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCL;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCL);
 	else
-		DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCLLBOnly;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotalPlusCNVCFormater + DPPCLKDelaySCLLBOnly);
 
-	DPPCycles = DPPCycles + myPipe->NumberOfCursors * DPPCLKDelayCNVCCursor;
+	DPPCycles = (unsigned int)(DPPCycles + myPipe->NumberOfCursors * DPPCLKDelayCNVCCursor);
 
-	DISPCLKCycles = DISPCLKDelaySubtotal;
+	DISPCLKCycles = (unsigned int)DISPCLKDelaySubtotal;
 
 	if (myPipe->DPPCLK == 0.0 || myPipe->DISPCLK == 0.0)
 		return true;
@@ -1527,14 +1527,14 @@ static void CalculateDCCConfiguration(
 	double detile_buf_vp_horz_limit;
 	double detile_buf_vp_vert_limit;
 
-	int MAS_vp_horz_limit;
-	int MAS_vp_vert_limit;
-	int max_vp_horz_width;
-	int max_vp_vert_height;
-	int eff_surf_width_l;
-	int eff_surf_width_c;
-	int eff_surf_height_l;
-	int eff_surf_height_c;
+	unsigned int MAS_vp_horz_limit;
+	unsigned int MAS_vp_vert_limit;
+	unsigned int max_vp_horz_width;
+	unsigned int max_vp_vert_height;
+	unsigned int eff_surf_width_l;
+	unsigned int eff_surf_width_c;
+	unsigned int eff_surf_height_l;
+	unsigned int eff_surf_height_c;
 
 	int full_swath_bytes_horz_wc_l;
 	int full_swath_bytes_horz_wc_c;
@@ -1571,11 +1571,11 @@ static void CalculateDCCConfiguration(
 		horz_div_c = 0;
 
 	if (BytePerPixelC == 0) {
-		swath_buf_size = DETBufferSize / 2 - 2 * 256;
+		swath_buf_size = (int)(DETBufferSize / 2 - 2 * 256);
 		detile_buf_vp_horz_limit = (double) swath_buf_size / ((double) RequestHeight256ByteLuma * BytePerPixelY / (1 + horz_div_l));
 		detile_buf_vp_vert_limit = (double) swath_buf_size / (256.0 / RequestHeight256ByteLuma / (1 + vert_div_l));
 	} else {
-		swath_buf_size = DETBufferSize / 2 - 2 * 2 * 256;
+		swath_buf_size = (int)(DETBufferSize / 2 - 2 * 2 * 256);
 		detile_buf_vp_horz_limit = (double) swath_buf_size
 				/ ((double) RequestHeight256ByteLuma * BytePerPixelY / (1 + horz_div_l)
 						+ (double) RequestHeight256ByteChroma * BytePerPixelC / (1 + horz_div_c) / (1 + yuv420));
@@ -1593,8 +1593,8 @@ static void CalculateDCCConfiguration(
 
 	MAS_vp_horz_limit = SourcePixelFormat == dm_rgbe_alpha ? 3840 : 5760;
 	MAS_vp_vert_limit = (BytePerPixelC > 0 ? 2880 : 5760);
-	max_vp_horz_width = dml_min((double) MAS_vp_horz_limit, detile_buf_vp_horz_limit);
-	max_vp_vert_height = dml_min((double) MAS_vp_vert_limit, detile_buf_vp_vert_limit);
+	max_vp_horz_width = (int)dml_min((double) MAS_vp_horz_limit, detile_buf_vp_horz_limit);
+	max_vp_vert_height = (int)dml_min((double) MAS_vp_vert_limit, detile_buf_vp_vert_limit);
 	eff_surf_width_l = (SurfaceWidthLuma > max_vp_horz_width ? max_vp_horz_width : SurfaceWidthLuma);
 	eff_surf_width_c = eff_surf_width_l / (1 + yuv420);
 	eff_surf_height_l = (SurfaceHeightLuma > max_vp_vert_height ? max_vp_vert_height : SurfaceHeightLuma);
@@ -1611,10 +1611,10 @@ static void CalculateDCCConfiguration(
 	}
 
 	if (SourcePixelFormat == dm_420_10) {
-		full_swath_bytes_horz_wc_l = dml_ceil(full_swath_bytes_horz_wc_l * 2 / 3, 256);
-		full_swath_bytes_horz_wc_c = dml_ceil(full_swath_bytes_horz_wc_c * 2 / 3, 256);
-		full_swath_bytes_vert_wc_l = dml_ceil(full_swath_bytes_vert_wc_l * 2 / 3, 256);
-		full_swath_bytes_vert_wc_c = dml_ceil(full_swath_bytes_vert_wc_c * 2 / 3, 256);
+		full_swath_bytes_horz_wc_l = (int)dml_ceil(full_swath_bytes_horz_wc_l * 2 / 3, 256);
+		full_swath_bytes_horz_wc_c = (int)dml_ceil(full_swath_bytes_horz_wc_c * 2 / 3, 256);
+		full_swath_bytes_vert_wc_l = (int)dml_ceil(full_swath_bytes_vert_wc_l * 2 / 3, 256);
+		full_swath_bytes_vert_wc_c = (int)dml_ceil(full_swath_bytes_vert_wc_c * 2 / 3, 256);
 	}
 
 	if (2 * full_swath_bytes_horz_wc_l + 2 * full_swath_bytes_horz_wc_c <= DETBufferSize) {
@@ -1777,20 +1777,20 @@ static double CalculatePrefetchSourceLines(
 
 	if (!v->IgnoreViewportPositioning) {
 
-		*MaxNumSwath = dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0;
+		*MaxNumSwath = (unsigned int)(dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1.0);
 
 		if (*VInitPreFill > 1.0)
 			MaxPartialSwath = (unsigned int) (*VInitPreFill - 2) % SwathHeight;
 		else
 			MaxPartialSwath = (unsigned int) (*VInitPreFill + SwathHeight - 2) % SwathHeight;
-		MaxPartialSwath = dml_max(1U, MaxPartialSwath);
+		MaxPartialSwath = (unsigned int)dml_max(1U, MaxPartialSwath);
 
 	} else {
 
 		if (ViewportYStart != 0)
 			dml_print("WARNING DML: using viewport y position of 0 even though actual viewport y position is non-zero in prefetch source lines calculation\n");
 
-		*MaxNumSwath = dml_ceil(*VInitPreFill / SwathHeight, 1);
+		*MaxNumSwath = (unsigned int)dml_ceil(*VInitPreFill / SwathHeight, 1);
 
 		if (*VInitPreFill > 1.0)
 			MaxPartialSwath = (unsigned int) (*VInitPreFill - 1) % SwathHeight;
@@ -1835,19 +1835,19 @@ static unsigned int CalculateVMAndRowBytes(
 		unsigned int *MetaRowByte,
 		unsigned int *PixelPTEBytesPerRow,
 		bool *PTEBufferSizeNotExceeded,
-		int *dpte_row_width_ub,
+		unsigned int *dpte_row_width_ub,
 		unsigned int *dpte_row_height,
 		unsigned int *MetaRequestWidth,
 		unsigned int *MetaRequestHeight,
 		unsigned int *meta_row_width,
 		unsigned int *meta_row_height,
-		int *vm_group_bytes,
+		unsigned int *vm_group_bytes,
 		unsigned int *dpte_group_bytes,
 		unsigned int *PixelPTEReqWidth,
 		unsigned int *PixelPTEReqHeight,
 		unsigned int *PTERequestSize,
-		int *DPDE0BytesFrame,
-		int *MetaPTEBytesFrame)
+		unsigned int *DPDE0BytesFrame,
+		unsigned int *MetaPTEBytesFrame)
 {
 	(void)SourcePixelFormat;
 	struct vba_vars_st *v = &mode_lib->vba;
@@ -1865,9 +1865,9 @@ static unsigned int CalculateVMAndRowBytes(
 		if (HostVMMinPageSize < 2048) {
 			HostVMDynamicLevels = HostVMMaxNonCachedPageTableLevels;
 		} else if (HostVMMinPageSize >= 2048 && HostVMMinPageSize < 1048576) {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
 		} else {
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
 		}
 	}
 
@@ -1875,16 +1875,16 @@ static unsigned int CalculateVMAndRowBytes(
 	*MetaRequestWidth = 8 * BlockWidth256Bytes;
 	if (ScanDirection != dm_vert) {
 		*meta_row_height = *MetaRequestHeight;
-		*meta_row_width = dml_ceil((double) SwathWidth - 1, *MetaRequestWidth) + *MetaRequestWidth;
-		*MetaRowByte = *meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0;
+		*meta_row_width = (unsigned int)(dml_ceil((double) SwathWidth - 1, *MetaRequestWidth) + *MetaRequestWidth);
+		*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0);
 	} else {
 		*meta_row_height = *MetaRequestWidth;
-		*meta_row_width = dml_ceil((double) SwathWidth - 1, *MetaRequestHeight) + *MetaRequestHeight;
-		*MetaRowByte = *meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0;
+		*meta_row_width = (unsigned int)(dml_ceil((double) SwathWidth - 1, *MetaRequestHeight) + *MetaRequestHeight);
+		*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0);
 	}
-	DCCMetaSurfaceBytes = DCCMetaPitch * (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes) + 64 * BlockHeight256Bytes) * BytePerPixel / 256;
+	DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch * (dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes) + 64 * BlockHeight256Bytes) * BytePerPixel / 256);
 	if (GPUVMEnable == true) {
-		*MetaPTEBytesFrame = (dml_ceil((double) (DCCMetaSurfaceBytes - 4.0 * 1024.0) / (8 * 4.0 * 1024), 1) + 1) * 64;
+		*MetaPTEBytesFrame = (int)((dml_ceil((double) (DCCMetaSurfaceBytes - 4.0 * 1024.0) / (8 * 4.0 * 1024), 1) + 1) * 64);
 		MPDEBytesFrame = 128 * (v->GPUVMMaxPageTableLevels - 1);
 	} else {
 		*MetaPTEBytesFrame = 0;
@@ -1908,17 +1908,17 @@ static unsigned int CalculateVMAndRowBytes(
 
 	if (GPUVMEnable == true && v->GPUVMMaxPageTableLevels > 1) {
 		if (ScanDirection != dm_vert) {
-			*DPDE0BytesFrame = 64
+			*DPDE0BytesFrame = (int)(64
 					* (dml_ceil(
 							((Pitch * (dml_ceil(ViewportHeight - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes)
 									/ (8 * 2097152),
-							1) + 1);
+							1) + 1));
 		} else {
-			*DPDE0BytesFrame = 64
+			*DPDE0BytesFrame = (int)(64
 					* (dml_ceil(
 							((Pitch * (dml_ceil((double) SwathWidth - 1, MacroTileHeight) + MacroTileHeight) * BytePerPixel) - MacroTileSizeBytes)
 									/ (8 * 2097152),
-							1) + 1);
+							1) + 1));
 		}
 		ExtraDPDEBytesFrame = 128 * (v->GPUVMMaxPageTableLevels - 2);
 	} else {
@@ -1946,7 +1946,7 @@ static unsigned int CalculateVMAndRowBytes(
 	if (SurfaceTiling == dm_sw_linear) {
 		PixelPTEReqHeightPTEs = 1;
 		*PixelPTEReqHeight = 1;
-		*PixelPTEReqWidth = 32768.0 / BytePerPixel;
+		*PixelPTEReqWidth = (unsigned int)(32768.0 / BytePerPixel);
 		*PTERequestSize = 64;
 		FractionOfPTEReturnDrop = 0;
 	} else if (GPUVMMinPageSize == 4 && MacroTileSizeBytes > 4096) {
@@ -1964,16 +1964,16 @@ static unsigned int CalculateVMAndRowBytes(
 	}
 
 	if (SurfaceTiling == dm_sw_linear) {
-		*dpte_row_height = dml_min(128, 1 << (unsigned int) dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1));
-		*dpte_row_width_ub = (dml_ceil((double)(Pitch * *dpte_row_height - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth;
+		*dpte_row_height = (unsigned int)dml_min(128, 1 << (unsigned int) dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1));
+		*dpte_row_width_ub = (int)((dml_ceil((double)(Pitch * *dpte_row_height - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth);
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqWidth * *PTERequestSize;
 	} else if (ScanDirection != dm_vert) {
 		*dpte_row_height = *PixelPTEReqHeight;
-		*dpte_row_width_ub = (dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth;
+		*dpte_row_width_ub = (int)((dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth);
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqWidth * *PTERequestSize;
 	} else {
-		*dpte_row_height = dml_min(*PixelPTEReqWidth, *MacroTileWidth);
-		*dpte_row_width_ub = (dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1) * *PixelPTEReqHeight;
+		*dpte_row_height = (unsigned int)dml_min(*PixelPTEReqWidth, *MacroTileWidth);
+		*dpte_row_width_ub = (int)((dml_ceil((double) (SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1) * *PixelPTEReqHeight);
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqHeight * *PTERequestSize;
 	}
 
@@ -2059,9 +2059,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 							v->WritebackHTaps[k],
 							v->WritebackVTaps[k],
 							v->WritebackSourceWidth[k],
-							v->WritebackDestinationWidth[k],
+							(long)v->WritebackDestinationWidth[k],
 							v->HTotal[k],
-							v->WritebackLineBufferSize));
+							(unsigned int)v->WritebackLineBufferSize));
 		}
 	}
 
@@ -2247,7 +2247,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			v->DPPCLK,
 			v->ReadBandwidthPlaneLuma,
 			v->ReadBandwidthPlaneChroma,
-			v->ReturnBusWidth,
+			(int)v->ReturnBusWidth,
 			&v->DCFCLKDeepSleep);
 
 	// DSCCLK
@@ -2284,7 +2284,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				v->DSCDelay[k] = dscceComputeDelay(
 						v->DSCInputBitPerComponent[k],
 						BPP,
-						dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
+						(unsigned int)dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
 						v->NumberOfDSCSlices[k],
 						v->OutputFormat[k],
 						v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]);
@@ -2293,8 +2293,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 						* (dscceComputeDelay(
 								v->DSCInputBitPerComponent[k],
 								BPP,
-								dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
-								v->NumberOfDSCSlices[k] / 2.0,
+								(unsigned int)dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
+								(unsigned int)(v->NumberOfDSCSlices[k] / 2.0),
 								v->OutputFormat[k],
 								v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]));
 			} else {
@@ -2302,13 +2302,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 						* (dscceComputeDelay(
 								v->DSCInputBitPerComponent[k],
 								BPP,
-								dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
-								v->NumberOfDSCSlices[k] / 4.0,
+								(unsigned int)dml_ceil((double) v->HActive[k] / v->NumberOfDSCSlices[k], 1),
+								(unsigned int)(v->NumberOfDSCSlices[k] / 4.0),
 								v->OutputFormat[k],
 								v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]));
 			}
-			v->DSCDelay[k] = v->DSCDelay[k] + (v->HTotal[k] - v->HActive[k]) * dml_ceil((double) v->DSCDelay[k] / v->HActive[k], 1);
-			v->DSCDelay[k] = v->DSCDelay[k] * v->PixelClock[k] / v->PixelClockBackEnd[k];
+			v->DSCDelay[k] = (unsigned int)(v->DSCDelay[k] + (v->HTotal[k] - v->HActive[k]) * dml_ceil((double) v->DSCDelay[k] / v->HActive[k], 1));
+			v->DSCDelay[k] = (unsigned int)(v->DSCDelay[k] * v->PixelClock[k] / v->PixelClockBackEnd[k]);
 		} else {
 			v->DSCDelay[k] = 0;
 		}
@@ -2349,13 +2349,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					v->SurfaceTiling[k],
 					v->BytePerPixelC[k],
 					v->SourceScan[k],
-					v->SwathWidthC[k],
+					(unsigned int)v->SwathWidthC[k],
 					v->ViewportHeightChroma[k],
 					v->GPUVMEnable,
 					v->HostVMEnable,
 					v->HostVMMaxNonCachedPageTableLevels,
-					v->GPUVMMinPageSize,
-					v->HostVMMinPageSize,
+					(unsigned int)v->GPUVMMinPageSize,
+					(unsigned int)v->HostVMMinPageSize,
 					v->PTEBufferSizeInRequestsForChroma,
 					v->PitchC[k],
 					v->DCCMetaPitchC[k],
@@ -2406,13 +2406,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				v->SurfaceTiling[k],
 				v->BytePerPixelY[k],
 				v->SourceScan[k],
-				v->SwathWidthY[k],
+				(unsigned int)v->SwathWidthY[k],
 				v->ViewportHeight[k],
 				v->GPUVMEnable,
 				v->HostVMEnable,
 				v->HostVMMaxNonCachedPageTableLevels,
-				v->GPUVMMinPageSize,
-				v->HostVMMinPageSize,
+				(unsigned int)v->GPUVMMinPageSize,
+				(unsigned int)v->HostVMMinPageSize,
 				v->PTEBufferSizeInRequestsForLuma,
 				v->PitchY[k],
 				v->DCCMetaPitchY[k],
@@ -2478,11 +2478,11 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			NoChromaPlanes = false;
 	}
 
-	ReorderBytes = v->NumberOfChannels
+	ReorderBytes = (int)(v->NumberOfChannels
 			* dml_max3(
 					v->UrgentOutOfOrderReturnPerChannelPixelDataOnly,
 					v->UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData,
-					v->UrgentOutOfOrderReturnPerChannelVMDataOnly);
+					v->UrgentOutOfOrderReturnPerChannelVMDataOnly));
 
 	VMDataOnlyReturnBW = dml_min(
 			dml_min(v->ReturnBusWidth * v->DCFCLK, v->FabricClock * v->FabricDatapathToDCNDataReturn)
@@ -2508,7 +2508,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 		HostVMInefficiencyFactor = v->ReturnBW / VMDataOnlyReturnBW;
 
 	v->UrgentExtraLatency = CalculateExtraLatency(
-			v->RoundTripPingLatencyCycles,
+			(int)v->RoundTripPingLatencyCycles,
 			ReorderBytes,
 			v->DCFCLK,
 			v->TotalActiveDPP,
@@ -2536,9 +2536,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 								v->WritebackHRatio[k],
 								v->WritebackVRatio[k],
 								v->WritebackVTaps[k],
-								v->WritebackDestinationWidth[k],
-								v->WritebackDestinationHeight[k],
-								v->WritebackSourceHeight[k],
+								(int)v->WritebackDestinationWidth[k],
+								(int)v->WritebackDestinationHeight[k],
+								(int)v->WritebackSourceHeight[k],
 								v->HTotal[k]) / v->DISPCLK;
 			} else
 				v->WritebackDelay[v->VoltageLevel][k] = 0;
@@ -2552,9 +2552,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 											v->WritebackHRatio[j],
 											v->WritebackVRatio[j],
 											v->WritebackVTaps[j],
-											v->WritebackDestinationWidth[j],
-											v->WritebackDestinationHeight[j],
-											v->WritebackSourceHeight[j],
+											(int)v->WritebackDestinationWidth[j],
+											(int)v->WritebackDestinationHeight[j],
+											(int)v->WritebackSourceHeight[j],
 											v->HTotal[k]) / v->DISPCLK);
 				}
 			}
@@ -2588,7 +2588,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 
 	v->MaximumMaxVStartupLines = 0;
 	for (k = 0; k < v->NumberOfActivePlanes; ++k)
-		v->MaximumMaxVStartupLines = dml_max(v->MaximumMaxVStartupLines, v->MaxVStartupLines[k]);
+		v->MaximumMaxVStartupLines = (unsigned int)dml_max(v->MaximumMaxVStartupLines, v->MaxVStartupLines[k]);
 
 	// VBA_DELTA
 	// We don't really care to iterate between the various prefetch modes
@@ -2658,7 +2658,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					(unsigned int) (v->SwathWidthY[k] / v->HRatio[k]),
 					v->OutputFormat[k],
 					v->MaxInterDCNTileRepeaters,
-					dml_min(v->VStartupLines, v->MaxVStartupLines[k]),
+					(unsigned int)dml_min(v->VStartupLines, v->MaxVStartupLines[k]),
 					v->MaxVStartupLines[k],
 					v->GPUVMMaxPageTableLevels,
 					v->GPUVMEnable,
@@ -2672,15 +2672,15 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					v->UrgentLatency,
 					v->UrgentExtraLatency,
 					v->TCalc,
-					v->PDEAndMetaPTEBytesFrame[k],
-					v->MetaRowByte[k],
-					v->PixelPTEBytesPerRow[k],
+					(unsigned int)v->PDEAndMetaPTEBytesFrame[k],
+					(unsigned int)v->MetaRowByte[k],
+					(unsigned int)v->PixelPTEBytesPerRow[k],
 					v->PrefetchSourceLinesY[k],
-					v->SwathWidthY[k],
+					(unsigned int)v->SwathWidthY[k],
 					v->VInitPreFillY[k],
 					v->MaxNumSwathY[k],
 					v->PrefetchSourceLinesC[k],
-					v->SwathWidthC[k],
+					(unsigned int)v->SwathWidthC[k],
 					v->VInitPreFillC[k],
 					v->MaxNumSwathC[k],
 					v->swath_width_luma_ub[k],
@@ -2711,7 +2711,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 #ifdef __DML_VBA_DEBUG__
 			dml_print("DML::%s: k=%0d Prefetch cal result=%0d\n", __func__, k, v->ErrorResult[k]);
 #endif
-			v->VStartup[k] = dml_min(v->VStartupLines, v->MaxVStartupLines[k]);
+			v->VStartup[k] = (unsigned int)dml_min(v->VStartupLines, v->MaxVStartupLines[k]);
 		}
 
 		v->NoEnoughUrgentLatencyHiding = false;
@@ -2865,8 +2865,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 
 			v->TotImmediateFlipBytes = 0;
 			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
-				v->TotImmediateFlipBytes = v->TotImmediateFlipBytes
-						+ v->DPPPerPlane[k] * (v->PDEAndMetaPTEBytesFrame[k] + v->MetaRowByte[k] + v->PixelPTEBytesPerRow[k]);
+				v->TotImmediateFlipBytes = (unsigned int)(v->TotImmediateFlipBytes
+						+ v->DPPPerPlane[k] * (v->PDEAndMetaPTEBytesFrame[k] + v->MetaRowByte[k] + v->PixelPTEBytesPerRow[k]));
 			}
 			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 				CalculateFlipSchedule(
@@ -3179,11 +3179,11 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 		isInterlaceTiming = (v->Interlace[k] && !v->ProgressiveToInterlaceUnitInOPP);
 		v->VStartup[k] = (isInterlaceTiming ? (2 * v->MaxVStartupLines[k]) : v->MaxVStartupLines[k]);
 		if (v->Interlace[k] && !v->ProgressiveToInterlaceUnitInOPP) {
-			v->MIN_DST_Y_NEXT_START[k] = dml_floor((v->VTotal[k] - v->VFrontPorch[k] + v->VTotal[k] - v->VActive[k] - v->VStartup[k]) / 2.0, 1.0);
+			v->MIN_DST_Y_NEXT_START[k] = (unsigned int)dml_floor((v->VTotal[k] - v->VFrontPorch[k] + v->VTotal[k] - v->VActive[k] - v->VStartup[k]) / 2.0, 1.0);
 		} else {
 			v->MIN_DST_Y_NEXT_START[k] = v->VTotal[k] - v->VFrontPorch[k] + v->VTotal[k] - v->VActive[k] - v->VStartup[k];
 		}
-		v->MIN_DST_Y_NEXT_START[k] += dml_floor(4.0 * v->TSetup[k] / ((double)v->HTotal[k] / v->PixelClock[k]), 1.0) / 4.0;
+		v->MIN_DST_Y_NEXT_START[k] += (unsigned int)(dml_floor(4.0 * v->TSetup[k] / ((double)v->HTotal[k] / v->PixelClock[k]), 1.0) / 4.0);
 		if (((v->VUpdateOffsetPix[k] + v->VUpdateWidthPix[k] + v->VReadyOffsetPix[k]) / v->HTotal[k])
 				<= (isInterlaceTiming ?
 						dml_floor((v->VTotal[k] - v->VActive[k] - v->VFrontPorch[k] - v->VStartup[k]) / 2.0, 1.0) :
@@ -3297,18 +3297,18 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib)
 	// Display Pipe Configuration
 	double BytePerPixDETY[DC__NUM_DPP__MAX];
 	double BytePerPixDETC[DC__NUM_DPP__MAX];
-	int BytePerPixY[DC__NUM_DPP__MAX];
-	int BytePerPixC[DC__NUM_DPP__MAX];
-	int Read256BytesBlockHeightY[DC__NUM_DPP__MAX];
-	int Read256BytesBlockHeightC[DC__NUM_DPP__MAX];
-	int Read256BytesBlockWidthY[DC__NUM_DPP__MAX];
-	int Read256BytesBlockWidthC[DC__NUM_DPP__MAX];
+	unsigned int BytePerPixY[DC__NUM_DPP__MAX];
+	unsigned int BytePerPixC[DC__NUM_DPP__MAX];
+	unsigned int Read256BytesBlockHeightY[DC__NUM_DPP__MAX];
+	unsigned int Read256BytesBlockHeightC[DC__NUM_DPP__MAX];
+	unsigned int Read256BytesBlockWidthY[DC__NUM_DPP__MAX];
+	unsigned int Read256BytesBlockWidthC[DC__NUM_DPP__MAX];
 	double dummy1[DC__NUM_DPP__MAX];
 	double dummy2[DC__NUM_DPP__MAX];
 	double dummy3[DC__NUM_DPP__MAX];
 	double dummy4[DC__NUM_DPP__MAX];
-	int dummy5[DC__NUM_DPP__MAX];
-	int dummy6[DC__NUM_DPP__MAX];
+	unsigned int dummy5[DC__NUM_DPP__MAX];
+	unsigned int dummy6[DC__NUM_DPP__MAX];
 	bool dummy7[DC__NUM_DPP__MAX];
 	bool dummysinglestring;
 
@@ -3535,7 +3535,7 @@ static void CalculateVupdateAndDynamicMetadataParameters(
 		double *Tdmbf,
 		double *Tdmec,
 		double *Tdmsks,
-		int *VUpdateOffsetPix,
+		unsigned int *VUpdateOffsetPix,
 		double *VUpdateWidthPix,
 		double *VReadyOffsetPix)
 {
@@ -3544,7 +3544,7 @@ static void CalculateVupdateAndDynamicMetadataParameters(
 	TotalRepeaterDelayTime = MaxInterDCNTileRepeaters * (2 / DPPCLK + 3 / DISPCLK);
 	*VUpdateWidthPix = dml_ceil((14.0 / DCFClkDeepSleep + 12.0 / DPPCLK + TotalRepeaterDelayTime) * PixelClock, 1.0);
 	*VReadyOffsetPix = dml_ceil(dml_max(150.0 / DPPCLK, TotalRepeaterDelayTime + 20.0 / DCFClkDeepSleep + 10.0 / DPPCLK) * PixelClock, 1.0);
-	*VUpdateOffsetPix = dml_ceil(HTotal / 4.0, 1);
+	*VUpdateOffsetPix = (unsigned int)dml_ceil(HTotal / 4.0, 1);
 	*TSetup = (*VUpdateOffsetPix + *VUpdateWidthPix + *VReadyOffsetPix) / PixelClock;
 	*Tdmbf = DynamicMetadataTransmittedBytes / 4.0 / DISPCLK;
 	*Tdmec = HTotal / PixelClock;
@@ -3829,17 +3829,17 @@ static noinline void CalculatePrefetchSchedulePerPlane(
 		mode_lib,
 		HostVMInefficiencyFactor,
 		&myPipe,
-		v->DSCDelayPerState[i][k],
+		(unsigned int)v->DSCDelayPerState[i][k],
 		v->DPPCLKDelaySubtotal + v->DPPCLKDelayCNVCFormater,
 		v->DPPCLKDelaySCL,
 		v->DPPCLKDelaySCLLBOnly,
 		v->DPPCLKDelayCNVCCursor,
 		v->DISPCLKDelaySubtotal,
-		v->SwathWidthYThisState[k] / v->HRatio[k],
+		(unsigned int)(v->SwathWidthYThisState[k] / v->HRatio[k]),
 		v->OutputFormat[k],
 		v->MaxInterDCNTileRepeaters,
-		dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]),
-		v->MaximumVStartup[i][j][k],
+		(unsigned int)dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]),
+		(unsigned int)v->MaximumVStartup[i][j][k],
 		v->GPUVMMaxPageTableLevels,
 		v->GPUVMEnable,
 		v->HostVMEnable,
@@ -3852,15 +3852,15 @@ static noinline void CalculatePrefetchSchedulePerPlane(
 		v->UrgLatency[i],
 		v->ExtraLatency,
 		v->TimeCalc,
-		v->PDEAndMetaPTEBytesPerFrame[i][j][k],
-		v->MetaRowBytes[i][j][k],
-		v->DPTEBytesPerRow[i][j][k],
+		(unsigned int)v->PDEAndMetaPTEBytesPerFrame[i][j][k],
+		(unsigned int)v->MetaRowBytes[i][j][k],
+		(unsigned int)v->DPTEBytesPerRow[i][j][k],
 		v->PrefetchLinesY[i][j][k],
-		v->SwathWidthYThisState[k],
+		(unsigned int)v->SwathWidthYThisState[k],
 		v->PrefillY[k],
 		v->MaxNumSwY[k],
 		v->PrefetchLinesC[i][j][k],
-		v->SwathWidthCThisState[k],
+		(unsigned int)v->SwathWidthCThisState[k],
 		v->PrefillC[k],
 		v->MaxNumSwC[k],
 		v->swath_width_luma_ub_this_state[k],
@@ -3893,10 +3893,11 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 {
 	struct vba_vars_st *v = &mode_lib->vba;
 
-	int i, j;
+	int j;
+	unsigned int i;
 	unsigned int k, m;
 	int ReorderingBytes;
-	int MinPrefetchMode = 0, MaxPrefetchMode = 2;
+	unsigned int MinPrefetchMode = 0, MaxPrefetchMode = 2;
 	bool NoChroma = true;
 	bool EnoughWritebackUnits = true;
 	bool P2IWith420 = false;
@@ -4045,9 +4046,9 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 							v->WritebackHTaps[k],
 							v->WritebackVTaps[k],
 							v->WritebackSourceWidth[k],
-							v->WritebackDestinationWidth[k],
+							(long)v->WritebackDestinationWidth[k],
 							v->HTotal[k],
-							v->WritebackLineBufferSize));
+							(unsigned int)v->WritebackLineBufferSize));
 		}
 	}
 	for (k = 0; k < v->NumberOfActivePlanes; k++) {
@@ -4094,17 +4095,17 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 		int MaximumSwathWidthSupportChroma;
 
 		if (v->SurfaceTiling[k] == dm_sw_linear) {
-			MaximumSwathWidthSupportLuma = 8192.0;
+			MaximumSwathWidthSupportLuma = (int)8192.0;
 		} else if (v->SourceScan[k] == dm_vert && v->BytePerPixelC[k] > 0) {
-			MaximumSwathWidthSupportLuma = 2880.0;
+			MaximumSwathWidthSupportLuma = (int)2880.0;
 		} else if (v->SourcePixelFormat[k] == dm_rgbe_alpha) {
-			MaximumSwathWidthSupportLuma = 3840.0;
+			MaximumSwathWidthSupportLuma = (int)3840.0;
 		} else {
-			MaximumSwathWidthSupportLuma = 5760.0;
+			MaximumSwathWidthSupportLuma = (int)5760.0;
 		}
 
 		if (v->SourcePixelFormat[k] == dm_420_8 || v->SourcePixelFormat[k] == dm_420_10 || v->SourcePixelFormat[k] == dm_420_12) {
-			MaximumSwathWidthSupportChroma = MaximumSwathWidthSupportLuma / 2.0;
+			MaximumSwathWidthSupportChroma = (int)(MaximumSwathWidthSupportLuma / 2.0);
 		} else {
 			MaximumSwathWidthSupportChroma = MaximumSwathWidthSupportLuma;
 		}
@@ -4332,7 +4333,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 						v->DISPCLK_DPPCLK_Support[i][j] = false;
 					}
 				}
-				v->TotalNumberOfActiveDPP[i][j] = 0.0;
+				v->TotalNumberOfActiveDPP[i][j] = 0;
 				for (k = 0; k < v->NumberOfActivePlanes; k++) {
 					v->TotalNumberOfActiveDPP[i][j] = v->TotalNumberOfActiveDPP[i][j] + v->NoOfDPP[i][j][k];
 				}
@@ -4369,7 +4370,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 	for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 		if (v->BlendingAndTiming[k] == k) {
 			if (v->PixelClockBackEnd[k] > 3200) {
-				v->NumberOfDSCSlices[k] = dml_ceil(v->PixelClockBackEnd[k] / 400.0, 4.0);
+				v->NumberOfDSCSlices[k] = (unsigned int)dml_ceil(v->PixelClockBackEnd[k] / 400.0, 4.0);
 			} else if (v->PixelClockBackEnd[k] > 1360) {
 				v->NumberOfDSCSlices[k] = 8;
 			} else if (v->PixelClockBackEnd[k] > 680) {
@@ -4431,7 +4432,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 							v->PHYCLKD18PerState[k] >= 10000.0 / 18.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 10000,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4450,7 +4451,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 								v->LinkDSCEnable = true;
 								v->Outbpp = TruncToValidBPP(
 										(1.0 - v->Downspreading / 100.0) * 10000,
-										v->OutputLinkDPLanes[k],
+										(int)v->OutputLinkDPLanes[k],
 										v->HTotal[k],
 										v->HActive[k],
 										v->PixelClockBackEnd[k],
@@ -4473,7 +4474,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 							v->PHYCLKD18PerState[k] >= 13500.0 / 18.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 13500,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4492,7 +4493,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 								v->LinkDSCEnable = true;
 								v->Outbpp = TruncToValidBPP(
 										(1.0 - v->Downspreading / 100.0) * 13500,
-										v->OutputLinkDPLanes[k],
+										(int)v->OutputLinkDPLanes[k],
 										v->HTotal[k],
 										v->HActive[k],
 										v->PixelClockBackEnd[k],
@@ -4515,7 +4516,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 							v->PHYCLKD18PerState[k] >= 20000.0 / 18.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 20000,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4534,7 +4535,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 								v->LinkDSCEnable = true;
 								v->Outbpp = TruncToValidBPP(
 										(1.0 - v->Downspreading / 100.0) * 20000,
-										v->OutputLinkDPLanes[k],
+										(int)v->OutputLinkDPLanes[k],
 										v->HTotal[k],
 										v->HActive[k],
 										v->PixelClockBackEnd[k],
@@ -4557,7 +4558,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 						if (v->PHYCLKPerState[i] >= 270.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 2700,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4577,7 +4578,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 						if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 540.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 5400,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4597,7 +4598,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 						if (v->Outbpp == BPP_INVALID && v->PHYCLKPerState[i] >= 810.0) {
 							v->Outbpp = TruncToValidBPP(
 									(1.0 - v->Downspreading / 100.0) * 8100,
-									v->OutputLinkDPLanes[k],
+									(int)v->OutputLinkDPLanes[k],
 									v->HTotal[k],
 									v->HActive[k],
 									v->PixelClockBackEnd[k],
@@ -4696,7 +4697,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 					v->DSCDelayPerState[i][k] = dscceComputeDelay(
 							v->DSCInputBitPerComponent[k],
 							v->BPP,
-							dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
+							(unsigned int)dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
 							v->NumberOfDSCSlices[k],
 							v->OutputFormat[k],
 							v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]);
@@ -4705,7 +4706,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 							* (dscceComputeDelay(
 									v->DSCInputBitPerComponent[k],
 									v->BPP,
-									dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
+									(unsigned int)dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
 									v->NumberOfDSCSlices[k] / 2,
 									v->OutputFormat[k],
 									v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]));
@@ -4714,7 +4715,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 							* (dscceComputeDelay(
 									v->DSCInputBitPerComponent[k],
 									v->BPP,
-									dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
+									(unsigned int)dml_ceil(1.0 * v->HActive[k] / v->NumberOfDSCSlices[k], 1.0),
 									v->NumberOfDSCSlices[k] / 4,
 									v->OutputFormat[k],
 									v->Output[k]) + dscComputeDelay(v->OutputFormat[k], v->Output[k]));
@@ -4802,14 +4803,14 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 					v->RequiredDPPCLKThisState,
 					v->ReadBandwidthLuma,
 					v->ReadBandwidthChroma,
-					v->ReturnBusWidth,
+					(int)v->ReturnBusWidth,
 					&v->ProjectedDCFCLKDeepSleep[i][j]);
 
 			for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 				v->swath_width_luma_ub_all_states[i][j][k] = v->swath_width_luma_ub_this_state[k];
 				v->swath_width_chroma_ub_all_states[i][j][k] = v->swath_width_chroma_ub_this_state[k];
-				v->SwathWidthYAllStates[i][j][k] = v->SwathWidthYThisState[k];
-				v->SwathWidthCAllStates[i][j][k] = v->SwathWidthCThisState[k];
+				v->SwathWidthYAllStates[i][j][k] = (unsigned int)v->SwathWidthYThisState[k];
+				v->SwathWidthCAllStates[i][j][k] = (unsigned int)v->SwathWidthCThisState[k];
 				v->SwathHeightYAllStates[i][j][k] = v->SwathHeightYThisState[k];
 				v->SwathHeightCAllStates[i][j][k] = v->SwathHeightCThisState[k];
 				v->DETBufferSizeYAllStates[i][j][k] = v->DETBufferSizeYThisState[k];
@@ -4834,8 +4835,8 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 				v->SwathWidthCThisState[k] = v->SwathWidthCAllStates[i][j][k];
 				v->SwathHeightYThisState[k] = v->SwathHeightYAllStates[i][j][k];
 				v->SwathHeightCThisState[k] = v->SwathHeightCAllStates[i][j][k];
-				v->DETBufferSizeYThisState[k] = v->DETBufferSizeYAllStates[i][j][k];
-				v->DETBufferSizeCThisState[k] = v->DETBufferSizeCAllStates[i][j][k];
+				v->DETBufferSizeYThisState[k] = (unsigned int)v->DETBufferSizeYAllStates[i][j][k];
+				v->DETBufferSizeCThisState[k] = (unsigned int)v->DETBufferSizeCAllStates[i][j][k];
 			}
 
 			v->TotalNumberOfDCCActiveDPP[i][j] = 0;
@@ -4868,16 +4869,16 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 							v->SurfaceTiling[k],
 							v->BytePerPixelC[k],
 							v->SourceScan[k],
-							v->SwathWidthCThisState[k],
+							(unsigned int)v->SwathWidthCThisState[k],
 							v->ViewportHeightChroma[k],
 							v->GPUVMEnable,
 							v->HostVMEnable,
 							v->HostVMMaxNonCachedPageTableLevels,
-							v->GPUVMMinPageSize,
-							v->HostVMMinPageSize,
+							(unsigned int)v->GPUVMMinPageSize,
+							(unsigned int)v->HostVMMinPageSize,
 							v->PTEBufferSizeInRequestsForChroma,
 							v->PitchC[k],
-							0.0,
+							0,
 							&v->MacroTileWidthC[k],
 							&v->MetaRowBytesC,
 							&v->DPTEBytesPerRowC,
@@ -4910,8 +4911,8 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 					v->PTEBufferSizeInRequestsForLuma = v->PTEBufferSizeInRequestsLuma + v->PTEBufferSizeInRequestsChroma;
 					v->PTEBufferSizeInRequestsForChroma = 0;
 					v->PDEAndMetaPTEBytesPerFrameC = 0.0;
-					v->MetaRowBytesC = 0.0;
-					v->DPTEBytesPerRowC = 0.0;
+					v->MetaRowBytesC = 0;
+					v->DPTEBytesPerRowC = 0;
 					v->PrefetchLinesC[i][j][k] = 0.0;
 					v->PTEBufferSizeNotExceededC[i][j][k] = true;
 				}
@@ -4924,13 +4925,13 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 						v->SurfaceTiling[k],
 						v->BytePerPixelY[k],
 						v->SourceScan[k],
-						v->SwathWidthYThisState[k],
+						(unsigned int)v->SwathWidthYThisState[k],
 						v->ViewportHeight[k],
 						v->GPUVMEnable,
 						v->HostVMEnable,
 						v->HostVMMaxNonCachedPageTableLevels,
-						v->GPUVMMinPageSize,
-						v->HostVMMinPageSize,
+						(unsigned int)v->GPUVMMinPageSize,
+						(unsigned int)v->HostVMMinPageSize,
 						v->PTEBufferSizeInRequestsForLuma,
 						v->PitchY[k],
 						v->DCCMetaPitchY[k],
@@ -5066,9 +5067,9 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 										v->WritebackHRatio[k],
 										v->WritebackVRatio[k],
 										v->WritebackVTaps[k],
-										v->WritebackDestinationWidth[k],
-										v->WritebackDestinationHeight[k],
-										v->WritebackSourceHeight[k],
+										(int)v->WritebackDestinationWidth[k],
+										(int)v->WritebackDestinationHeight[k],
+										(int)v->WritebackSourceHeight[k],
 										v->HTotal[k]) / v->RequiredDISPCLK[i][j];
 					} else {
 						v->WritebackDelayTime[k] = 0.0;
@@ -5083,9 +5084,9 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 													v->WritebackHRatio[m],
 													v->WritebackVRatio[m],
 													v->WritebackVTaps[m],
-													v->WritebackDestinationWidth[m],
-													v->WritebackDestinationHeight[m],
-													v->WritebackSourceHeight[m],
+													(int)v->WritebackDestinationWidth[m],
+													(int)v->WritebackDestinationHeight[m],
+													(int)v->WritebackSourceHeight[m],
 													v->HTotal[m]) / v->RequiredDISPCLK[i][j]);
 						}
 					}
@@ -5116,11 +5117,11 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 		}
 	}
 
-	ReorderingBytes = v->NumberOfChannels
+	ReorderingBytes = (int)(v->NumberOfChannels
 			* dml_max3(
 					v->UrgentOutOfOrderReturnPerChannelPixelDataOnly,
 					v->UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData,
-					v->UrgentOutOfOrderReturnPerChannelVMDataOnly);
+					v->UrgentOutOfOrderReturnPerChannelVMDataOnly));
 
 	for (i = 0; i < v->soc.num_states; ++i) {
 		for (j = 0; j <= 1; ++j) {
@@ -5204,7 +5205,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 			double HostVMInefficiencyFactor = 1;
 			int NextPrefetchModeState = MinPrefetchMode;
 			bool UnboundedRequestEnabledThisState = false;
-			int CompressedBufferSizeInkByteThisState = 0;
+			unsigned int CompressedBufferSizeInkByteThisState = 0;
 			double dummy;
 
 			v->TimeCalc = 24 / v->ProjectedDCFCLKDeepSleep[i][j];
@@ -5223,8 +5224,8 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 				v->SwathWidthCThisState[k] = v->SwathWidthCAllStates[i][j][k];
 				v->SwathHeightYThisState[k] = v->SwathHeightYAllStates[i][j][k];
 				v->SwathHeightCThisState[k] = v->SwathHeightCAllStates[i][j][k];
-				v->DETBufferSizeYThisState[k] = v->DETBufferSizeYAllStates[i][j][k];
-				v->DETBufferSizeCThisState[k] = v->DETBufferSizeCAllStates[i][j][k];
+				v->DETBufferSizeYThisState[k] = (unsigned int)v->DETBufferSizeYAllStates[i][j][k];
+				v->DETBufferSizeCThisState[k] = (unsigned int)v->DETBufferSizeCAllStates[i][j][k];
 			}
 
 			VMDataOnlyReturnBWPerState = dml_min(
@@ -5238,7 +5239,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 				HostVMInefficiencyFactor = v->ReturnBWPerState[i][j] / VMDataOnlyReturnBWPerState;
 
 			v->ExtraLatency = CalculateExtraLatency(
-					v->RoundTripPingLatencyCycles,
+					(int)v->RoundTripPingLatencyCycles,
 					ReorderingBytes,
 					v->DCFCLKState[i][j],
 					v->TotalNumberOfActiveDPP[i][j],
@@ -5371,11 +5372,11 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 																* v->UrgentBurstFactorChromaPre[k])
 												+ v->cursor_bw_pre[k] * v->UrgentBurstFactorCursorPre[k]);
 					}
-					v->TotImmediateFlipBytes = 0.0;
+					v->TotImmediateFlipBytes = 0;
 					for (k = 0; k < v->NumberOfActivePlanes; k++) {
-						v->TotImmediateFlipBytes = v->TotImmediateFlipBytes
+						v->TotImmediateFlipBytes = (unsigned int)(v->TotImmediateFlipBytes
 								+ v->NoOfDPP[i][j][k] * (v->PDEAndMetaPTEBytesPerFrame[i][j][k] + v->MetaRowBytes[i][j][k]
-								+ v->DPTEBytesPerRow[i][j][k]);
+								+ v->DPTEBytesPerRow[i][j][k]));
 					}
 
 					for (k = 0; k < v->NumberOfActivePlanes; k++) {
@@ -5537,30 +5538,37 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 	}
 
 	/*Mode Support, Voltage State and SOC Configuration*/
-	for (i = v->soc.num_states - 1; i >= 0; i--) {
+	for (int idx = (int)v->soc.num_states - 1; idx >= 0; idx--) {
 		for (j = 0; j < 2; j++) {
-			if (v->ScaleRatioAndTapsSupport == true && v->SourceFormatPixelAndScanSupport == true && v->ViewportSizeSupport[i][j] == true
-					&& v->LinkCapacitySupport[i] == true && !P2IWith420 && !DSCOnlyIfNecessaryWithBPP
-					&& !DSC422NativeNotSupported && v->ODMCombine4To1SupportCheckOK[i] == true && v->NotEnoughDSCUnits[i] == false
-					&& v->DTBCLKRequiredMoreThanSupported[i] == false
-					&& v->ROBSupport[i][j] == true && v->DISPCLK_DPPCLK_Support[i][j] == true
-					&& v->TotalAvailablePipesSupport[i][j] == true && EnoughWritebackUnits == true
+			if (v->ScaleRatioAndTapsSupport == true && v->SourceFormatPixelAndScanSupport == true
+					&& v->ViewportSizeSupport[idx][j] == true
+					&& v->LinkCapacitySupport[idx] == true && !P2IWith420
+					&& !DSCOnlyIfNecessaryWithBPP
+					&& !DSC422NativeNotSupported && v->ODMCombine4To1SupportCheckOK[idx] == true
+					&& v->NotEnoughDSCUnits[idx] == false
+					&& v->DTBCLKRequiredMoreThanSupported[idx] == false
+					&& v->ROBSupport[idx][j] == true && v->DISPCLK_DPPCLK_Support[idx][j] == true
+					&& v->TotalAvailablePipesSupport[idx][j] == true && EnoughWritebackUnits == true
 					&& v->WritebackLatencySupport == true && v->WritebackScaleRatioAndTapsSupport == true
-					&& v->CursorSupport == true && v->PitchSupport == true && ViewportExceedsSurface == false
-					&& v->PrefetchSupported[i][j] == true && v->DynamicMetadataSupported[i][j] == true
-					&& v->TotalVerticalActiveBandwidthSupport[i][j] == true && v->VRatioInPrefetchSupported[i][j] == true
-					&& v->PTEBufferSizeNotExceeded[i][j] == true && v->NonsupportedDSCInputBPC == false
+					&& v->CursorSupport == true && v->PitchSupport == true
+					&& ViewportExceedsSurface == false
+					&& v->PrefetchSupported[idx][j] == true
+					&& v->DynamicMetadataSupported[idx][j] == true
+					&& v->TotalVerticalActiveBandwidthSupport[idx][j] == true
+					&& v->VRatioInPrefetchSupported[idx][j] == true
+					&& v->PTEBufferSizeNotExceeded[idx][j] == true
+					&& v->NonsupportedDSCInputBPC == false
 					&& ((v->HostVMEnable == false
 					&& v->ImmediateFlipRequirement[0] != dm_immediate_flip_required)
-							|| v->ImmediateFlipSupportedForState[i][j] == true)
+							|| v->ImmediateFlipSupportedForState[idx][j] == true)
 					&& FMTBufferExceeded == false) {
-				v->ModeSupport[i][j] = true;
+				v->ModeSupport[idx][j] = true;
 			} else {
-				v->ModeSupport[i][j] = false;
+				v->ModeSupport[idx][j] = false;
 			}
 		}
 	}
-	for (i = v->soc.num_states; i >= 0; i--) {
+	for (int idx = (int)v->soc.num_states; idx >= 0; idx--) {
 		for (j = 0; j < 2; j++) {
 			enum dm_validation_status status = DML_VALIDATION_OK;
 
@@ -5568,7 +5576,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 				status = DML_FAIL_SCALE_RATIO_TAP;
 			} else if (!v->SourceFormatPixelAndScanSupport) {
 				status = DML_FAIL_SOURCE_PIXEL_FORMAT;
-			} else if (!v->ViewportSizeSupport[i][j]) {
+			} else if (!v->ViewportSizeSupport[idx][j]) {
 				status = DML_FAIL_VIEWPORT_SIZE;
 			} else if (P2IWith420) {
 				status = DML_FAIL_P2I_WITH_420;
@@ -5576,15 +5584,15 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 				status = DML_FAIL_DSC_ONLY_IF_NECESSARY_WITH_BPP;
 			} else if (DSC422NativeNotSupported) {
 				status = DML_FAIL_NOT_DSC422_NATIVE;
-			} else if (!v->ODMCombine4To1SupportCheckOK[i]) {
+			} else if (!v->ODMCombine4To1SupportCheckOK[idx]) {
 				status = DML_FAIL_ODM_COMBINE4TO1;
-			} else if (v->NotEnoughDSCUnits[i]) {
+			} else if (v->NotEnoughDSCUnits[idx]) {
 				status = DML_FAIL_NOT_ENOUGH_DSC;
-			} else if (!v->ROBSupport[i][j]) {
+			} else if (!v->ROBSupport[idx][j]) {
 				status = DML_FAIL_REORDERING_BUFFER;
-			} else if (!v->DISPCLK_DPPCLK_Support[i][j]) {
+			} else if (!v->DISPCLK_DPPCLK_Support[idx][j]) {
 				status = DML_FAIL_DISPCLK_DPPCLK;
-			} else if (!v->TotalAvailablePipesSupport[i][j]) {
+			} else if (!v->TotalAvailablePipesSupport[idx][j]) {
 				status = DML_FAIL_TOTAL_AVAILABLE_PIPES;
 			} else if (!EnoughWritebackUnits) {
 				status = DML_FAIL_ENOUGH_WRITEBACK_UNITS;
@@ -5598,36 +5606,36 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 				status = DML_FAIL_PITCH_SUPPORT;
 			} else if (ViewportExceedsSurface) {
 				status = DML_FAIL_VIEWPORT_EXCEEDS_SURFACE;
-			} else if (!v->PrefetchSupported[i][j]) {
+			} else if (!v->PrefetchSupported[idx][j]) {
 				status = DML_FAIL_PREFETCH_SUPPORT;
-			} else if (!v->DynamicMetadataSupported[i][j]) {
+			} else if (!v->DynamicMetadataSupported[idx][j]) {
 				status = DML_FAIL_DYNAMIC_METADATA;
-			} else if (!v->TotalVerticalActiveBandwidthSupport[i][j]) {
+			} else if (!v->TotalVerticalActiveBandwidthSupport[idx][j]) {
 				status = DML_FAIL_TOTAL_V_ACTIVE_BW;
-			} else if (!v->VRatioInPrefetchSupported[i][j]) {
+			} else if (!v->VRatioInPrefetchSupported[idx][j]) {
 				status = DML_FAIL_V_RATIO_PREFETCH;
-			} else if (!v->PTEBufferSizeNotExceeded[i][j]) {
+			} else if (!v->PTEBufferSizeNotExceeded[idx][j]) {
 				status = DML_FAIL_PTE_BUFFER_SIZE;
 			} else if (v->NonsupportedDSCInputBPC) {
 				status = DML_FAIL_DSC_INPUT_BPC;
 			} else if ((v->HostVMEnable
-					&& !v->ImmediateFlipSupportedForState[i][j])) {
+					&& !v->ImmediateFlipSupportedForState[idx][j])) {
 				status = DML_FAIL_HOST_VM_IMMEDIATE_FLIP;
 			} else if (FMTBufferExceeded) {
 				status = DML_FAIL_FMT_BUFFER_EXCEEDED;
 			}
-			mode_lib->vba.ValidationStatus[i] = status;
+			mode_lib->vba.ValidationStatus[idx] = status;
 		}
 	}
 
 	{
 		unsigned int MaximumMPCCombine = 0;
-
-		for (i = v->soc.num_states; i >= 0; i--) {
-			if (i == v->soc.num_states || v->ModeSupport[i][0] == true || v->ModeSupport[i][1] == true) {
-				v->VoltageLevel = i;
-				v->ModeIsSupported = v->ModeSupport[i][0] == true || v->ModeSupport[i][1] == true;
-				if (v->ModeSupport[i][0] == true) {
+		for (int idx = (int)v->soc.num_states; idx >= 0; idx--) {
+			if (idx == (int)v->soc.num_states || v->ModeSupport[idx][0] == true
+					|| v->ModeSupport[idx][1] == true) {
+				v->VoltageLevel = idx;
+				v->ModeIsSupported = v->ModeSupport[idx][0] == true || v->ModeSupport[idx][1] == true;
+				if (v->ModeSupport[idx][0] == true) {
 					MaximumMPCCombine = 0;
 				} else {
 					MaximumMPCCombine = 1;
@@ -5635,7 +5643,7 @@ void dml314_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_
 			}
 		}
 		v->ImmediateFlipSupport = v->ImmediateFlipSupportedForState[v->VoltageLevel][MaximumMPCCombine];
-		for (k = 0; k <= v->NumberOfActivePlanes - 1; k++) {
+		for (k = 0; k < v->NumberOfActivePlanes; k++) {
 			v->MPCCombineEnable[k] = v->MPCCombine[v->VoltageLevel][MaximumMPCCombine][k];
 			v->DPPPerPlane[k] = v->NoOfDPP[v->VoltageLevel][MaximumMPCCombine][k];
 		}
@@ -5693,7 +5701,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 	double SecondMinActiveDRAMClockChangeMarginOneDisplayInVBLank;
 	double WritebackDRAMClockChangeLatencyHiding;
 	double TotalPixelBW = 0.0;
-	int k, j;
+	unsigned int k, j;
 
 	v->UrgentWatermark = UrgentLatency + ExtraLatency;
 
@@ -5738,13 +5746,13 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 	for (k = 0; k < v->NumberOfActivePlanes; ++k) {
 		double EffectiveDETBufferSizeY = DETBufferSizeY[k];
 
-		v->LBLatencyHidingSourceLinesY = dml_min(
+		v->LBLatencyHidingSourceLinesY = (unsigned int)(dml_min(
 				(double) v->MaxLineBufferLines,
-				dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1);
+				dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1));
 
-		v->LBLatencyHidingSourceLinesC = dml_min(
+		v->LBLatencyHidingSourceLinesC = (unsigned int)(dml_min(
 				(double) v->MaxLineBufferLines,
-				dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1);
+				dml_floor(v->LineBufferSize / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1));
 
 		EffectiveLBLatencyHidingY = v->LBLatencyHidingSourceLinesY / v->VRatio[k] * (v->HTotal[k] / v->PixelClock[k]);
 
@@ -5756,11 +5764,11 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 		}
 
 		LinesInDETY[k] = (double) EffectiveDETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k];
-		LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]);
+		LinesInDETYRoundedDownToSwath[k] = (unsigned int)dml_floor(LinesInDETY[k], SwathHeightY[k]);
 		FullDETBufferingTimeY = LinesInDETYRoundedDownToSwath[k] * (v->HTotal[k] / v->PixelClock[k]) / v->VRatio[k];
 		if (BytePerPixelDETC[k] > 0) {
 			LinesInDETC = v->DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k];
-			LinesInDETCRoundedDownToSwath = dml_floor(LinesInDETC, SwathHeightC[k]);
+			LinesInDETCRoundedDownToSwath = (unsigned int)dml_floor(LinesInDETC, SwathHeightC[k]);
 			FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath * (v->HTotal[k] / v->PixelClock[k]) / v->VRatioChroma[k];
 		} else {
 			LinesInDETC = 0;
@@ -5859,8 +5867,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport(
 static void CalculateDCFCLKDeepSleep(
 		struct display_mode_lib *mode_lib,
 		unsigned int NumberOfActivePlanes,
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		double VRatio[],
 		double VRatioChroma[],
 		double SwathWidthY[],
@@ -5881,7 +5889,7 @@ static void CalculateDCFCLKDeepSleep(
 	double DisplayPipeLineDeliveryTimeLuma;
 	double DisplayPipeLineDeliveryTimeChroma;
 	double ReadBandwidth = 0.0;
-	int k;
+	unsigned int k;
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 
@@ -6011,7 +6019,7 @@ static void CalculatePixelDeliveryTimes(
 		double PSCL_THROUGHPUT[],
 		double PSCL_THROUGHPUT_CHROMA[],
 		double DPPCLK[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
 		unsigned int NumberOfCursors[],
 		unsigned int CursorWidth[][DC__NUM_CURSOR__MAX],
@@ -6032,7 +6040,7 @@ static void CalculatePixelDeliveryTimes(
 		double CursorRequestDeliveryTimePrefetch[])
 {
 	double req_per_swath_ub;
-	int k;
+	unsigned int k;
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		if (VRatio[k] <= 1) {
@@ -6107,7 +6115,7 @@ static void CalculatePixelDeliveryTimes(
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		int cursor_req_per_width;
 
-		cursor_req_per_width = dml_ceil(CursorWidth[k][0] * CursorBPP[k][0] / 256 / 8, 1);
+		cursor_req_per_width = (int)dml_ceil(CursorWidth[k][0] * CursorBPP[k][0] / 256 / 8, 1);
 		if (NumberOfCursors[k] > 0) {
 			if (VRatio[k] <= 1) {
 				CursorRequestDeliveryTime[k] = CursorWidth[k][0] / HRatio[k] / PixelClock[k] / cursor_req_per_width;
@@ -6132,39 +6140,39 @@ static void CalculatePixelDeliveryTimes(
 }
 
 static void CalculateMetaAndPTETimes(
-		int NumberOfActivePlanes,
+		unsigned int NumberOfActivePlanes,
 		bool GPUVMEnable,
 		int MetaChunkSize,
 		int MinMetaChunkSizeBytes,
-		int HTotal[],
+		unsigned int HTotal[],
 		double VRatio[],
 		double VRatioChroma[],
 		double DestinationLinesToRequestRowInVBlank[],
 		double DestinationLinesToRequestRowInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int BytePerPixelY[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelY[],
+		unsigned int BytePerPixelC[],
 		enum scan_direction_class SourceScan[],
-		int dpte_row_height[],
-		int dpte_row_height_chroma[],
-		int meta_row_width[],
-		int meta_row_width_chroma[],
-		int meta_row_height[],
-		int meta_row_height_chroma[],
-		int meta_req_width[],
-		int meta_req_width_chroma[],
-		int meta_req_height[],
-		int meta_req_height_chroma[],
-		int dpte_group_bytes[],
-		int PTERequestSizeY[],
-		int PTERequestSizeC[],
-		int PixelPTEReqWidthY[],
-		int PixelPTEReqHeightY[],
-		int PixelPTEReqWidthC[],
-		int PixelPTEReqHeightC[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
+		unsigned int dpte_row_height[],
+		unsigned int dpte_row_height_chroma[],
+		unsigned int meta_row_width[],
+		unsigned int meta_row_width_chroma[],
+		unsigned int meta_row_height[],
+		unsigned int meta_row_height_chroma[],
+		unsigned int meta_req_width[],
+		unsigned int meta_req_width_chroma[],
+		unsigned int meta_req_height[],
+		unsigned int meta_req_height_chroma[],
+		unsigned int dpte_group_bytes[],
+		unsigned int PTERequestSizeY[],
+		unsigned int PTERequestSizeC[],
+		unsigned int PixelPTEReqWidthY[],
+		unsigned int PixelPTEReqHeightY[],
+		unsigned int PixelPTEReqWidthC[],
+		unsigned int PixelPTEReqHeightC[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
 		double DST_Y_PER_PTE_ROW_NOM_L[],
 		double DST_Y_PER_PTE_ROW_NOM_C[],
 		double DST_Y_PER_META_ROW_NOM_L[],
@@ -6198,7 +6206,7 @@ static void CalculateMetaAndPTETimes(
 	unsigned int dpte_groups_per_row_luma_ub;
 	unsigned int dpte_group_width_chroma;
 	unsigned int dpte_groups_per_row_chroma_ub;
-	int k;
+	unsigned int k;
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		DST_Y_PER_PTE_ROW_NOM_L[k] = dpte_row_height[k] / VRatio[k];
@@ -6241,7 +6249,7 @@ static void CalculateMetaAndPTETimes(
 			} else {
 				meta_chunk_width_chroma = MetaChunkSize * 1024 * 256 / BytePerPixelC[k] / meta_row_height_chroma[k];
 				min_meta_chunk_width_chroma = MinMetaChunkSizeBytes * 256 / BytePerPixelC[k] / meta_row_height_chroma[k];
-				meta_chunk_per_row_int_chroma = (double) meta_row_width_chroma[k] / meta_chunk_width_chroma;
+				meta_chunk_per_row_int_chroma = (unsigned int)((double) meta_row_width_chroma[k] / meta_chunk_width_chroma);
 				meta_row_remainder_chroma = meta_row_width_chroma[k] % meta_chunk_width_chroma;
 				if (SourceScan[k] != dm_vert) {
 					meta_chunk_threshold_chroma = 2 * min_meta_chunk_width_chroma - meta_req_width_chroma[k];
@@ -6274,7 +6282,7 @@ static void CalculateMetaAndPTETimes(
 			} else {
 				dpte_group_width_luma = dpte_group_bytes[k] / PTERequestSizeY[k] * PixelPTEReqHeightY[k];
 			}
-			dpte_groups_per_row_luma_ub = dml_ceil(1.0 * dpte_row_width_luma_ub[k] / dpte_group_width_luma, 1);
+			dpte_groups_per_row_luma_ub = (unsigned int)dml_ceil(1.0 * dpte_row_width_luma_ub[k] / dpte_group_width_luma, 1);
 			time_per_pte_group_nom_luma[k] = DST_Y_PER_PTE_ROW_NOM_L[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_luma_ub;
 			time_per_pte_group_vblank_luma[k] = DestinationLinesToRequestRowInVBlank[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_luma_ub;
 			time_per_pte_group_flip_luma[k] = DestinationLinesToRequestRowInImmediateFlip[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_luma_ub;
@@ -6288,7 +6296,7 @@ static void CalculateMetaAndPTETimes(
 				} else {
 					dpte_group_width_chroma = dpte_group_bytes[k] / PTERequestSizeC[k] * PixelPTEReqHeightC[k];
 				}
-				dpte_groups_per_row_chroma_ub = dml_ceil(1.0 * dpte_row_width_chroma_ub[k] / dpte_group_width_chroma, 1);
+				dpte_groups_per_row_chroma_ub = (unsigned int)dml_ceil(1.0 * dpte_row_width_chroma_ub[k] / dpte_group_width_chroma, 1);
 				time_per_pte_group_nom_chroma[k] = DST_Y_PER_PTE_ROW_NOM_C[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_chroma_ub;
 				time_per_pte_group_vblank_chroma[k] = DestinationLinesToRequestRowInVBlank[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_chroma_ub;
 				time_per_pte_group_flip_chroma[k] = DestinationLinesToRequestRowInImmediateFlip[k] * HTotal[k] / PixelClock[k] / dpte_groups_per_row_chroma_ub;
@@ -6309,18 +6317,18 @@ static void CalculateVMGroupAndRequestTimes(
 		bool GPUVMEnable,
 		unsigned int GPUVMMaxPageTableLevels,
 		unsigned int HTotal[],
-		int BytePerPixelC[],
+		unsigned int BytePerPixelC[],
 		double DestinationLinesToRequestVMInVBlank[],
 		double DestinationLinesToRequestVMInImmediateFlip[],
 		bool DCCEnable[],
 		double PixelClock[],
-		int dpte_row_width_luma_ub[],
-		int dpte_row_width_chroma_ub[],
-		int vm_group_bytes[],
+		unsigned int dpte_row_width_luma_ub[],
+		unsigned int dpte_row_width_chroma_ub[],
+		unsigned int vm_group_bytes[],
 		unsigned int dpde0_bytes_per_frame_ub_l[],
 		unsigned int dpde0_bytes_per_frame_ub_c[],
-		int meta_pte_bytes_per_frame_ub_l[],
-		int meta_pte_bytes_per_frame_ub_c[],
+		unsigned int meta_pte_bytes_per_frame_ub_l[],
+		unsigned int meta_pte_bytes_per_frame_ub_c[],
 		double TimePerVMGroupVBlank[],
 		double TimePerVMGroupFlip[],
 		double TimePerVMRequestVBlank[],
@@ -6330,34 +6338,34 @@ static void CalculateVMGroupAndRequestTimes(
 	(void)dpte_row_width_chroma_ub;
 	int num_group_per_lower_vm_stage;
 	int num_req_per_lower_vm_stage;
-	int k;
+	unsigned int k;
 
 	for (k = 0; k < NumberOfActivePlanes; ++k) {
 		if (GPUVMEnable == true && (DCCEnable[k] == true || GPUVMMaxPageTableLevels > 1)) {
 			if (DCCEnable[k] == false) {
 				if (BytePerPixelC[k] > 0) {
-					num_group_per_lower_vm_stage = dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-							+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+					num_group_per_lower_vm_stage = (int)(dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+							+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 				} else {
-					num_group_per_lower_vm_stage = dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+					num_group_per_lower_vm_stage = (int)dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
 				}
 			} else {
 				if (GPUVMMaxPageTableLevels == 1) {
 					if (BytePerPixelC[k] > 0) {
-						num_group_per_lower_vm_stage = dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+						num_group_per_lower_vm_stage = (int)(dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 					} else {
-						num_group_per_lower_vm_stage = dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+						num_group_per_lower_vm_stage = (int)dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
 					}
 				} else {
 					if (BytePerPixelC[k] > 0) {
-						num_group_per_lower_vm_stage = 2 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+						num_group_per_lower_vm_stage = (int)(2 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
 								+ dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1)
 								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1);
+								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) / (double) (vm_group_bytes[k]), 1));
 					} else {
-						num_group_per_lower_vm_stage = 1 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
-								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1);
+						num_group_per_lower_vm_stage = (int)(1 + dml_ceil((double) (dpde0_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1)
+								+ dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) / (double) (vm_group_bytes[k]), 1));
 					}
 				}
 			}
@@ -6428,29 +6436,29 @@ static void CalculateStutterEfficiency(
 		bool ProgressiveToInterlaceUnitInOPP,
 		bool Interlace[],
 		double MinTTUVBlank[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		unsigned int DETBufferSizeY[],
-		int BytePerPixelY[],
+		unsigned int BytePerPixelY[],
 		double BytePerPixelDETY[],
 		double SwathWidthY[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		double NetDCCRateLuma[],
 		double NetDCCRateChroma[],
 		double DCCFractionOfZeroSizeRequestsLuma[],
 		double DCCFractionOfZeroSizeRequestsChroma[],
-		int HTotal[],
-		int VTotal[],
+		unsigned int HTotal[],
+		unsigned int VTotal[],
 		double PixelClock[],
 		double VRatio[],
 		enum scan_direction_class SourceScan[],
-		int BlockHeight256BytesY[],
-		int BlockWidth256BytesY[],
-		int BlockHeight256BytesC[],
-		int BlockWidth256BytesC[],
-		int DCCYMaxUncompressedBlock[],
-		int DCCCMaxUncompressedBlock[],
-		int VActive[],
+		unsigned int BlockHeight256BytesY[],
+		unsigned int BlockWidth256BytesY[],
+		unsigned int BlockHeight256BytesC[],
+		unsigned int BlockWidth256BytesC[],
+		unsigned int DCCYMaxUncompressedBlock[],
+		unsigned int DCCCMaxUncompressedBlock[],
+		unsigned int VActive[],
 		bool DCCEnable[],
 		bool WritebackEnable[],
 		double ReadBandwidthPlaneLuma[],
@@ -6668,8 +6676,8 @@ static void CalculateStutterEfficiency(
 #endif
 		*StutterEfficiencyNotIncludingVBlank = dml_max(0., 1 - (SRExitTime + StutterBurstTime) / *StutterPeriod) * 100;
 		*Z8StutterEfficiencyNotIncludingVBlank = dml_max(0., 1 - (SRExitZ8Time + StutterBurstTime) / *StutterPeriod) * 100;
-		*NumberOfStutterBurstsPerFrame = (*StutterEfficiencyNotIncludingVBlank > 0 ? dml_ceil(VActiveTimeCriticalPlane / *StutterPeriod, 1) : 0);
-		*Z8NumberOfStutterBurstsPerFrame = (*Z8StutterEfficiencyNotIncludingVBlank > 0 ? dml_ceil(VActiveTimeCriticalPlane / *StutterPeriod, 1) : 0);
+		*NumberOfStutterBurstsPerFrame = (int)(*StutterEfficiencyNotIncludingVBlank > 0 ? dml_ceil(VActiveTimeCriticalPlane / *StutterPeriod, 1) : 0);
+		*Z8NumberOfStutterBurstsPerFrame = (int)(*Z8StutterEfficiencyNotIncludingVBlank > 0 ? dml_ceil(VActiveTimeCriticalPlane / *StutterPeriod, 1) : 0);
 	} else {
 		*StutterEfficiencyNotIncludingVBlank = 0.;
 		*Z8StutterEfficiencyNotIncludingVBlank = 0.;
@@ -6734,48 +6742,48 @@ static void CalculateSwathAndDETConfiguration(
 		enum scan_direction_class SourceScan[],
 		enum source_format_class SourcePixelFormat[],
 		enum dm_swizzle_mode SurfaceTiling[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BlendingAndTiming[],
-		int BytePerPixY[],
-		int BytePerPixC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
 		double BytePerPixDETY[],
 		double BytePerPixDETC[],
-		int HActive[],
+		unsigned int HActive[],
 		double HRatio[],
 		double HRatioChroma[],
-		int DPPPerPlane[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[],
+		unsigned int DPPPerPlane[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[],
 		double SwathWidth[],
 		double SwathWidthChroma[],
-		int SwathHeightY[],
-		int SwathHeightC[],
+		unsigned int SwathHeightY[],
+		unsigned int SwathHeightC[],
 		unsigned int DETBufferSizeY[],
 		unsigned int DETBufferSizeC[],
 		bool ViewportSizeSupportPerPlane[],
 		bool *ViewportSizeSupport)
 {
 	(void)HRatioChroma;
-	int MaximumSwathHeightY[DC__NUM_DPP__MAX];
-	int MaximumSwathHeightC[DC__NUM_DPP__MAX];
+	unsigned int MaximumSwathHeightY[DC__NUM_DPP__MAX];
+	unsigned int MaximumSwathHeightC[DC__NUM_DPP__MAX];
 	int MinimumSwathHeightY;
 	int MinimumSwathHeightC;
-	int RoundedUpMaxSwathSizeBytesY;
-	int RoundedUpMaxSwathSizeBytesC;
-	int RoundedUpMinSwathSizeBytesY;
-	int RoundedUpMinSwathSizeBytesC;
-	int RoundedUpSwathSizeBytesY;
-	int RoundedUpSwathSizeBytesC;
+	unsigned int RoundedUpMaxSwathSizeBytesY;
+	unsigned int RoundedUpMaxSwathSizeBytesC;
+	unsigned int RoundedUpMinSwathSizeBytesY;
+	unsigned int RoundedUpMinSwathSizeBytesC;
+	unsigned int RoundedUpSwathSizeBytesY;
+	unsigned int RoundedUpSwathSizeBytesC;
 	double SwathWidthSingleDPP[DC__NUM_DPP__MAX];
 	double SwathWidthSingleDPPChroma[DC__NUM_DPP__MAX];
 	int k;
@@ -6845,17 +6853,17 @@ static void CalculateSwathAndDETConfiguration(
 			}
 		}
 
-		RoundedUpMaxSwathSizeBytesY = swath_width_luma_ub[k] * BytePerPixDETY[k] * MaximumSwathHeightY[k];
-		RoundedUpMinSwathSizeBytesY = swath_width_luma_ub[k] * BytePerPixDETY[k] * MinimumSwathHeightY;
+		RoundedUpMaxSwathSizeBytesY = (int)(swath_width_luma_ub[k] * BytePerPixDETY[k] * MaximumSwathHeightY[k]);
+		RoundedUpMinSwathSizeBytesY = (int)(swath_width_luma_ub[k] * BytePerPixDETY[k] * MinimumSwathHeightY);
 		if (SourcePixelFormat[k] == dm_420_10) {
-			RoundedUpMaxSwathSizeBytesY = dml_ceil((double) RoundedUpMaxSwathSizeBytesY, 256);
-			RoundedUpMinSwathSizeBytesY = dml_ceil((double) RoundedUpMinSwathSizeBytesY, 256);
+			RoundedUpMaxSwathSizeBytesY = (int)dml_ceil((double) RoundedUpMaxSwathSizeBytesY, 256);
+			RoundedUpMinSwathSizeBytesY = (int)dml_ceil((double) RoundedUpMinSwathSizeBytesY, 256);
 		}
-		RoundedUpMaxSwathSizeBytesC = swath_width_chroma_ub[k] * BytePerPixDETC[k] * MaximumSwathHeightC[k];
-		RoundedUpMinSwathSizeBytesC = swath_width_chroma_ub[k] * BytePerPixDETC[k] * MinimumSwathHeightC;
+		RoundedUpMaxSwathSizeBytesC = (int)(swath_width_chroma_ub[k] * BytePerPixDETC[k] * MaximumSwathHeightC[k]);
+		RoundedUpMinSwathSizeBytesC = (int)(swath_width_chroma_ub[k] * BytePerPixDETC[k] * MinimumSwathHeightC);
 		if (SourcePixelFormat[k] == dm_420_10) {
-			RoundedUpMaxSwathSizeBytesC = dml_ceil(RoundedUpMaxSwathSizeBytesC, 256);
-			RoundedUpMinSwathSizeBytesC = dml_ceil(RoundedUpMinSwathSizeBytesC, 256);
+			RoundedUpMaxSwathSizeBytesC = (int)dml_ceil(RoundedUpMaxSwathSizeBytesC, 256);
+			RoundedUpMinSwathSizeBytesC = (int)dml_ceil(RoundedUpMinSwathSizeBytesC, 256);
 		}
 
 		if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC <= DETBufferSizeInKByte * 1024 / 2) {
@@ -6885,14 +6893,14 @@ static void CalculateSwathAndDETConfiguration(
 		double actDETBufferSizeInKByte = dml_ceil(DETBufferSizeInKByte, 64);
 
 		if (SwathHeightC[k] == 0) {
-			DETBufferSizeY[k] = actDETBufferSizeInKByte * 1024;
+			DETBufferSizeY[k] = (unsigned int)(actDETBufferSizeInKByte * 1024);
 			DETBufferSizeC[k] = 0;
 		} else if (RoundedUpSwathSizeBytesY <= 1.5 * RoundedUpSwathSizeBytesC) {
-			DETBufferSizeY[k] = actDETBufferSizeInKByte * 1024 / 2;
-			DETBufferSizeC[k] = actDETBufferSizeInKByte * 1024 / 2;
+			DETBufferSizeY[k] = (unsigned int)(actDETBufferSizeInKByte * 1024 / 2);
+			DETBufferSizeC[k] = (unsigned int)(actDETBufferSizeInKByte * 1024 / 2);
 		} else {
-			DETBufferSizeY[k] = dml_floor(actDETBufferSizeInKByte * 1024 * 2 / 3, 1024);
-			DETBufferSizeC[k] = actDETBufferSizeInKByte * 1024 / 3;
+			DETBufferSizeY[k] = (unsigned int)dml_floor(actDETBufferSizeInKByte * 1024 * 2 / 3, 1024);
+			DETBufferSizeC[k] = (unsigned int)(actDETBufferSizeInKByte * 1024 / 3);
 		}
 
 		if (RoundedUpMinSwathSizeBytesY + RoundedUpMinSwathSizeBytesC > actDETBufferSizeInKByte * 1024 / 2 || SwathWidth[k] > MaximumSwathWidthLuma[k]
@@ -6911,31 +6919,31 @@ static void CalculateSwathWidth(
 		int NumberOfActivePlanes,
 		enum source_format_class SourcePixelFormat[],
 		enum scan_direction_class SourceScan[],
-		int ViewportWidth[],
-		int ViewportHeight[],
-		int SurfaceWidthY[],
-		int SurfaceWidthC[],
-		int SurfaceHeightY[],
-		int SurfaceHeightC[],
+		unsigned int ViewportWidth[],
+		unsigned int ViewportHeight[],
+		unsigned int SurfaceWidthY[],
+		unsigned int SurfaceWidthC[],
+		unsigned int SurfaceHeightY[],
+		unsigned int SurfaceHeightC[],
 		enum odm_combine_mode ODMCombineEnabled[],
-		int BytePerPixY[],
-		int BytePerPixC[],
-		int Read256BytesBlockHeightY[],
-		int Read256BytesBlockHeightC[],
-		int Read256BytesBlockWidthY[],
-		int Read256BytesBlockWidthC[],
-		int BlendingAndTiming[],
-		int HActive[],
+		unsigned int BytePerPixY[],
+		unsigned int BytePerPixC[],
+		unsigned int Read256BytesBlockHeightY[],
+		unsigned int Read256BytesBlockHeightC[],
+		unsigned int Read256BytesBlockWidthY[],
+		unsigned int Read256BytesBlockWidthC[],
+		unsigned int BlendingAndTiming[],
+		unsigned int HActive[],
 		double HRatio[],
-		int DPPPerPlane[],
+		unsigned int DPPPerPlane[],
 		double SwathWidthSingleDPPY[],
 		double SwathWidthSingleDPPC[],
 		double SwathWidthY[],
 		double SwathWidthC[],
-		int MaximumSwathHeightY[],
-		int MaximumSwathHeightC[],
-		int swath_width_luma_ub[],
-		int swath_width_chroma_ub[])
+		unsigned int MaximumSwathHeightY[],
+		unsigned int MaximumSwathHeightC[],
+		unsigned int swath_width_luma_ub[],
+		unsigned int swath_width_chroma_ub[])
 {
 	(void)BytePerPixY;
 	enum odm_combine_mode MainPlaneODMCombine;
@@ -6991,8 +6999,8 @@ static void CalculateSwathWidth(
 			SwathWidthC[k] = SwathWidthSingleDPPC[k];
 		}
 		{
-		int surface_width_ub_l = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
-		int surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
+		int surface_width_ub_l = (int)dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
+		int surface_height_ub_l = (int)dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
 
 #ifdef __DML_VBA_DEBUG__
 		dml_print("DML::%s: k=%d surface_width_ub_l=%0d\n", __func__, k, surface_width_ub_l);
@@ -7001,11 +7009,11 @@ static void CalculateSwathWidth(
 		if (SourceScan[k] != dm_vert) {
 			MaximumSwathHeightY[k] = Read256BytesBlockHeightY[k];
 			MaximumSwathHeightC[k] = Read256BytesBlockHeightC[k];
-			swath_width_luma_ub[k] = dml_min(surface_width_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]);
+			swath_width_luma_ub[k] = (int)dml_min(surface_width_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockWidthY[k]) + Read256BytesBlockWidthY[k]);
 			if (BytePerPixC[k] > 0) {
-				int surface_width_ub_c = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
+				int surface_width_ub_c = (int)dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
 
-				swath_width_chroma_ub[k] = dml_min(
+				swath_width_chroma_ub[k] = (int)dml_min(
 						surface_width_ub_c,
 						(int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockWidthC[k]) + Read256BytesBlockWidthC[k]);
 			} else {
@@ -7014,11 +7022,11 @@ static void CalculateSwathWidth(
 		} else {
 			MaximumSwathHeightY[k] = Read256BytesBlockWidthY[k];
 			MaximumSwathHeightC[k] = Read256BytesBlockWidthC[k];
-			swath_width_luma_ub[k] = dml_min(surface_height_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
+			swath_width_luma_ub[k] = (int)dml_min(surface_height_ub_l, (int) dml_ceil(SwathWidthY[k] - 1, Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
 			if (BytePerPixC[k] > 0) {
-				int surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
+				int surface_height_ub_c = (int)dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
 
-				swath_width_chroma_ub[k] = dml_min(
+				swath_width_chroma_ub[k] = (int)dml_min(
 						surface_height_ub_c,
 						(int) dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockHeightC[k]) + Read256BytesBlockHeightC[k]);
 			} else {
@@ -7041,8 +7049,8 @@ static double CalculateExtraLatency(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double HostVMInefficiencyFactor,
 		double HostVMMinPageSize,
 		int HostVMMaxNonCachedPageTableLevels)
@@ -7087,8 +7095,8 @@ static double CalculateExtraLatencyBytes(
 		bool GPUVMEnable,
 		bool HostVMEnable,
 		int NumberOfActivePlanes,
-		int NumberOfDPP[],
-		int dpte_group_bytes[],
+		unsigned int NumberOfDPP[],
+		unsigned int dpte_group_bytes[],
 		double HostVMInefficiencyFactor,
 		double HostVMMinPageSize,
 		int HostVMMaxNonCachedPageTableLevels)
@@ -7100,9 +7108,9 @@ static double CalculateExtraLatencyBytes(
 		if (HostVMMinPageSize < 2048)
 			HostVMDynamicLevels = HostVMMaxNonCachedPageTableLevels;
 		else if (HostVMMinPageSize >= 2048 && HostVMMinPageSize < 1048576)
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
+			HostVMDynamicLevels = (int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
 		else
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
+			HostVMDynamicLevels = (int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
 	} else {
 		HostVMDynamicLevels = 0;
 	}
@@ -7139,7 +7147,8 @@ static noinline_for_stack void UseMinimumDCFCLK(
 		int ReorderingBytes)
 {
 	struct vba_vars_st *v = &mode_lib->vba;
-	int dummy1, i, j, k;
+	unsigned int dummy1, j;
+	unsigned int i, k;
 	double NormalEfficiency,  dummy2, dummy3;
 	double TotalMaxPrefetchFlipDPTERowBandwidth[DC__VOLTAGE_STATES][2];
 
@@ -7157,7 +7166,7 @@ static noinline_for_stack void UseMinimumDCFCLK(
 			double ExtraLatencyBytes;
 			double ExtraLatencyCycles;
 			double DCFCLKRequiredForPeakBandwidth;
-			int NoOfDPPState[DC__NUM_DPP__MAX];
+			unsigned int NoOfDPPState[DC__NUM_DPP__MAX];
 			double MinimumTvmPlus2Tr0;
 
 			TotalMaxPrefetchFlipDPTERowBandwidth[i][j] = 0;
@@ -7166,7 +7175,7 @@ static noinline_for_stack void UseMinimumDCFCLK(
 						+ v->NoOfDPP[i][j][k] * v->DPTEBytesPerRow[i][j][k] / (15.75 * v->HTotal[k] / v->PixelClock[k]);
 			}
 
-			for (k = 0; k <= v->NumberOfActivePlanes - 1; ++k)
+			for (k = 0; k < v->NumberOfActivePlanes; ++k)
 				NoOfDPPState[k] = v->NoOfDPP[i][j][k];
 
 			MinimumTWait = CalculateTWait(MaxPrefetchMode, v->FinalDRAMClockChangeLatency, v->UrgLatency[i], v->SREnterPlusExitTime);
@@ -7269,7 +7278,7 @@ static noinline_for_stack void UseMinimumDCFCLK(
 				}
 			}
 			DCFCLKRequiredForPeakBandwidth = 0;
-			for (k = 0; k <= v->NumberOfActivePlanes - 1; ++k)
+			for (k = 0; k < v->NumberOfActivePlanes; ++k)
 				DCFCLKRequiredForPeakBandwidth = DCFCLKRequiredForPeakBandwidth + DCFCLKRequiredForPeakBandwidthPerPlane[k];
 
 			MinimumTvmPlus2Tr0 = v->UrgLatency[i]
@@ -7305,12 +7314,12 @@ static void CalculateUnboundedRequestAndCompressedBufferSize(
 		int CompressedBufferSegmentSizeInkByteFinal,
 		enum output_encoder_class *Output,
 		bool *UnboundedRequestEnabled,
-		int *CompressedBufferSizeInkByte)
+		unsigned int *CompressedBufferSizeInkByte)
 {
 	double actDETBufferSizeInKByte = dml_ceil(DETBufferSizeInKByte, 64);
 
 	*UnboundedRequestEnabled = UnboundedRequest(UseUnboundedRequestingFinal, TotalActiveDPP, NoChromaPlanes, Output[0]);
-	*CompressedBufferSizeInkByte = (
+	*CompressedBufferSizeInkByte = (int)(
 			*UnboundedRequestEnabled == true ?
 					ConfigReturnBufferSizeInKByte - TotalActiveDPP * actDETBufferSizeInKByte :
 					ConfigReturnBufferSizeInKByte - MaxNumDPP * actDETBufferSizeInKByte);
@@ -7352,15 +7361,15 @@ static unsigned int CalculateMaxVStartup(
 	unsigned int vblank_size = 0;
 	double line_time_us = HTotal / PixelClock;
 	unsigned int vblank_actual = VTotal - VActive;
-	unsigned int vblank_nom_default_in_line = dml_floor(VBlankNomDefaultUS / line_time_us, 1.0);
+	unsigned int vblank_nom_default_in_line = (unsigned int)dml_floor(VBlankNomDefaultUS / line_time_us, 1.0);
 	unsigned int vblank_nom_input = VBlankNom; //dml_min(VBlankNom, vblank_nom_default_in_line);
 	unsigned int vblank_avail = vblank_nom_input == 0 ? vblank_nom_default_in_line : vblank_nom_input;
 
 	vblank_size = (unsigned int) dml_min(vblank_actual, vblank_avail);
 	if (Interlace && !ProgressiveTointerlaceUnitinOPP)
-		MaxVStartup = dml_floor(vblank_size / 2.0, 1.0);
+		MaxVStartup = (unsigned int)dml_floor(vblank_size / 2.0, 1.0);
 	else
-		MaxVStartup = vblank_size - dml_max(1.0, dml_ceil(WritebackDelayTime / line_time_us, 1.0));
+		MaxVStartup = (unsigned int)(vblank_size - dml_max(1.0, dml_ceil(WritebackDelayTime / line_time_us, 1.0)));
 	if (MaxVStartup > 1023)
 		MaxVStartup = 1023;
 	return MaxVStartup;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_rq_dlg_calc_314.c b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_rq_dlg_calc_314.c
index 40a916c2a9c61..ec745c2d6c818 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_rq_dlg_calc_314.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn314/display_rq_dlg_calc_314.c
@@ -205,11 +205,11 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib, display_rq_regs_s
 
 	extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_l), &rq_param->sizing.rq_l);
 
-	rq_regs->rq_regs_l.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height), 1) - 3;
+	rq_regs->rq_regs_l.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_l.dpte_row_height), 1) - 3);
 
 	if (rq_param->yuv420) {
 		extract_rq_sizing_regs(mode_lib, &(rq_regs->rq_regs_c), &rq_param->sizing.rq_c);
-		rq_regs->rq_regs_c.pte_row_height_linear = dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height), 1) - 3;
+		rq_regs->rq_regs_c.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(rq_param->dlg.rq_c.dpte_row_height), 1) - 3);
 	}
 
 	rq_regs->rq_regs_l.swath_height = dml_log2(rq_param->dlg.rq_l.swath_height);
@@ -229,12 +229,12 @@ static void extract_rq_regs(struct display_mode_lib *mode_lib, display_rq_regs_s
 	// Note: detile_buf_plane1_addr is in unit of 1KB
 	if (rq_param->yuv420) {
 		if ((double) rq_param->misc.rq_l.stored_swath_bytes / (double) rq_param->misc.rq_c.stored_swath_bytes <= 1.5) {
-			detile_buf_plane1_addr = (detile_buf_size_in_bytes / 2.0 / 1024.0); // half to chroma
+			detile_buf_plane1_addr = (unsigned int)(detile_buf_size_in_bytes / 2.0 / 1024.0); // half to chroma
 #ifdef __DML_RQ_DLG_CALC_DEBUG__
 					dml_print("DML_DLG: %s: detile_buf_plane1_addr = %0d (1/2 to chroma)\n", __func__, detile_buf_plane1_addr);
 #endif
 		} else {
-			detile_buf_plane1_addr = dml_round_to_multiple((unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0), 1024, 0) / 1024.0; // 2/3 to luma
+			detile_buf_plane1_addr = (unsigned int)(dml_round_to_multiple((unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0), 1024, 0) / 1024.0); // 2/3 to luma
 #ifdef __DML_RQ_DLG_CALC_DEBUG__
 					dml_print("DML_DLG: %s: detile_buf_plane1_addr = %0d (1/3 chroma)\n", __func__, detile_buf_plane1_addr);
 #endif
@@ -277,8 +277,8 @@ static void handle_det_buf_split(struct display_mode_lib *mode_lib, display_rq_p
 #endif
 
 	if (rq_param->yuv420_10bpc) {
-		full_swath_bytes_packed_l = dml_round_to_multiple(rq_param->misc.rq_l.full_swath_bytes * 2.0 / 3.0, 256, 1) + 256;
-		full_swath_bytes_packed_c = dml_round_to_multiple(rq_param->misc.rq_c.full_swath_bytes * 2.0 / 3.0, 256, 1) + 256;
+		full_swath_bytes_packed_l = (unsigned int)(dml_round_to_multiple((unsigned int)(rq_param->misc.rq_l.full_swath_bytes * 2.0 / 3.0), 256, 1) + 256);
+		full_swath_bytes_packed_c = (unsigned int)(dml_round_to_multiple((unsigned int)(rq_param->misc.rq_c.full_swath_bytes * 2.0 / 3.0), 256, 1) + 256);
 #ifdef __DML_RQ_DLG_CALC_DEBUG__
 		dml_print("DML_DLG: %s: full_swath_bytes_packed_l = %0d (3-2 packing)\n", __func__, full_swath_bytes_packed_l);
 		dml_print("DML_DLG: %s: full_swath_bytes_packed_c = %0d (3-2 packing)\n", __func__, full_swath_bytes_packed_c);
@@ -509,7 +509,7 @@ static void get_meta_and_pte_attr(
 	// "/2" is like square root
 	// blk is vertical biased
 	if (tiling != dm_sw_linear)
-		log2_blk_height = log2_blk256_height + dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
+		log2_blk_height = (unsigned int)(log2_blk256_height + dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1));
 	else
 		log2_blk_height = 0;	// blk height of 1
 
@@ -662,7 +662,7 @@ static void get_meta_and_pte_attr(
 	if (surf_linear) {
 		unsigned int dpte_row_height;
 
-		log2_dpte_row_height_linear = dml_floor(dml_log2(dpte_buf_in_pte_reqs * dpte_req_width / data_pitch), 1);
+		log2_dpte_row_height_linear = (unsigned int)dml_floor(dml_log2(dpte_buf_in_pte_reqs * dpte_req_width / data_pitch), 1);
 
 		dml_print("DML_DLG: %s: is_chroma                   = %d\n", __func__, is_chroma);
 		dml_print("DML_DLG: %s: dpte_buf_in_pte_reqs        = %d\n", __func__, dpte_buf_in_pte_reqs);
@@ -728,7 +728,7 @@ static void get_meta_and_pte_attr(
 
 	// since dpte groups are only aligned to dpte_req_width and not dpte_group_width,
 	// the upper bound for the dpte groups per row is as follows.
-	rq_dlg_param->dpte_groups_per_row_ub = dml_ceil((double) dpte_row_width_ub / dpte_group_width, 1);
+	rq_dlg_param->dpte_groups_per_row_ub = (unsigned int)dml_ceil((double) dpte_row_width_ub / dpte_group_width, 1);
 }
 
 static void get_surf_rq_param(
@@ -753,7 +753,7 @@ static void get_surf_rq_param(
 		vp_height = pipe_param->src.viewport_height_c;
 		data_pitch = pipe_param->src.data_pitch_c;
 		meta_pitch = pipe_param->src.meta_pitch_c;
-		surface_height = pipe_param->src.surface_height_y / 2.0;
+		surface_height = (unsigned int)(pipe_param->src.surface_height_y / 2.0);
 	} else {
 		vp_width = pipe_param->src.viewport_width / ppe;
 		vp_height = pipe_param->src.viewport_height;
@@ -771,18 +771,18 @@ static void get_surf_rq_param(
 		access_dir = (pipe_param->src.source_scan == dm_vert); // vp access direction: horizontal or vertical accessed
 		hactive_odm = pipe_param->dest.hactive / ((unsigned int) pipe_param->dest.odm_combine * 2);
 		if (is_chroma) {
-			full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width;
-			src_hactive_odm = pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_odm;
+			full_src_vp_width = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio_c * pipe_param->dest.full_recout_width);
+			src_hactive_odm = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio_c * hactive_odm);
 		} else {
-			full_src_vp_width = pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width;
-			src_hactive_odm = pipe_param->scale_ratio_depth.hscl_ratio * hactive_odm;
+			full_src_vp_width = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio * pipe_param->dest.full_recout_width);
+			src_hactive_odm = (unsigned int)(pipe_param->scale_ratio_depth.hscl_ratio * hactive_odm);
 		}
 
 		if (access_dir == 0) {
-			vp_width = dml_min(full_src_vp_width, src_hactive_odm);
+			vp_width = (unsigned int)dml_min(full_src_vp_width, src_hactive_odm);
 			dml_print("DML_DLG: %s: vp_width = %d\n", __func__, vp_width);
 		} else {
-			vp_height = dml_min(full_src_vp_width, src_hactive_odm);
+			vp_height = (unsigned int)dml_min(full_src_vp_width, src_hactive_odm);
 			dml_print("DML_DLG: %s: vp_height = %d\n", __func__, vp_height);
 
 		}
@@ -901,7 +901,7 @@ static void calculate_ttu_cursor(
 				cur_req_size = 256;
 		}
 
-		cur_req_width = (double) cur_req_size / ((double) cur_bit_per_pixel / 8.0);
+		cur_req_width = (unsigned int)((double) cur_req_size / ((double) cur_bit_per_pixel / 8.0));
 		cur_width_ub = dml_ceil((double) cur_src_width / (double) cur_req_width, 1) * (double) cur_req_width;
 		cur_req_per_width = cur_width_ub / (double) cur_req_width;
 		hactive_cur = (double) cur_src_width / hscl_ratio; // FIXME: oswin to think about what to do for cursor
@@ -1042,7 +1042,7 @@ static void dml_rq_dlg_get_dlg_params(
 	double refcyc_per_req_delivery_cur0;
 	double refcyc_per_req_delivery_pre_cur1;
 	double refcyc_per_req_delivery_cur1;
-	unsigned int vba__min_dst_y_next_start = get_min_dst_y_next_start(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // FROM VBA
+	unsigned int vba__min_dst_y_next_start = (unsigned int)get_min_dst_y_next_start(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // FROM VBA
 	unsigned int vba__vready_after_vcount0 = get_vready_at_or_after_vsync(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
 
 	float vba__refcyc_per_line_delivery_pre_l = get_refcyc_per_line_delivery_pre_l_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
@@ -1075,7 +1075,7 @@ static void dml_rq_dlg_get_dlg_params(
 	dlg_vblank_start = interlaced ? (vblank_start / 2) : vblank_start;
 	disp_dlg_regs->min_dst_y_next_start_us =
 		(vba__min_dst_y_next_start * dst->hactive) / (unsigned int) dst->pixel_rate_mhz;
-	disp_dlg_regs->min_dst_y_next_start = vba__min_dst_y_next_start * dml_pow(2, 2);
+	disp_dlg_regs->min_dst_y_next_start = (unsigned int)(vba__min_dst_y_next_start * dml_pow(2, 2));
 
 	ASSERT(disp_dlg_regs->min_dst_y_next_start < (unsigned int)dml_pow(2, 18));
 
@@ -1138,8 +1138,8 @@ static void dml_rq_dlg_get_dlg_params(
 	if (interlaced)
 		vstartup_start = vstartup_start / 2;
 
-	dst_x_after_scaler = get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
-	dst_y_after_scaler = get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
+	dst_x_after_scaler = (unsigned int)get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
+	dst_y_after_scaler = (unsigned int)get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
 
 	// do some adjustment on the dst_after scaler to account for odm combine mode
 	dml_print("DML_DLG: %s: input dst_x_after_scaler   = %d\n", __func__, dst_x_after_scaler);
@@ -1366,9 +1366,9 @@ static void dml_rq_dlg_get_dlg_params(
 	}
 
 	if (src->dynamic_metadata_enable && src->gpuvm)
-		disp_dlg_regs->refcyc_per_vm_dmdata = get_refcyc_per_vm_dmdata_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
+		disp_dlg_regs->refcyc_per_vm_dmdata = (unsigned int)(get_refcyc_per_vm_dmdata_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz); // From VBA
 
-	disp_dlg_regs->dmdata_dl_delta = get_dmdata_dl_delta_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
+	disp_dlg_regs->dmdata_dl_delta = (unsigned int)(get_dmdata_dl_delta_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz); // From VBA
 
 	// TTU - Luma / Chroma
 	if (access_dir) {  // vertical access
@@ -1474,8 +1474,8 @@ static void dml_rq_dlg_get_dlg_params(
 				src->cur0_src_width,
 				(enum cursor_bpp) (src->cur0_bpp));
 
-		vba__refcyc_per_req_delivery_pre_cur0 = get_refcyc_per_cursor_req_delivery_pre_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
-		vba__refcyc_per_req_delivery_cur0 = get_refcyc_per_cursor_req_delivery_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
+		vba__refcyc_per_req_delivery_pre_cur0 = (float)(get_refcyc_per_cursor_req_delivery_pre_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz); // From VBA
+		vba__refcyc_per_req_delivery_cur0 = (float)(get_refcyc_per_cursor_req_delivery_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz); // From VBA
 
 		dml_print("DML_DLG: %s: refcyc_per_req_delivery_pre_cur0 = %3.2f\n", __func__, refcyc_per_req_delivery_pre_cur0);
 		dml_print("DML_DLG: %s: refcyc_per_req_delivery_cur0     = %3.2f\n", __func__, refcyc_per_req_delivery_cur0);
@@ -1495,7 +1495,7 @@ static void dml_rq_dlg_get_dlg_params(
 	// Assignment to register structures
 	disp_dlg_regs->dst_y_after_scaler = dst_y_after_scaler;	// in terms of line
 	ASSERT(disp_dlg_regs->dst_y_after_scaler < 8);
-	disp_dlg_regs->refcyc_x_after_scaler = dst_x_after_scaler * ref_freq_to_pix_freq;	// in terms of refclk
+	disp_dlg_regs->refcyc_x_after_scaler = (unsigned int)(dst_x_after_scaler * ref_freq_to_pix_freq);	// in terms of refclk
 	ASSERT(disp_dlg_regs->refcyc_x_after_scaler < (unsigned int)dml_pow(2, 13));
 	disp_dlg_regs->dst_y_prefetch = (unsigned int) (dst_y_prefetch * dml_pow(2, 2));
 	disp_dlg_regs->dst_y_per_vm_vblank = (unsigned int) (dst_y_per_vm_vblank * dml_pow(2, 2));
@@ -1532,23 +1532,23 @@ static void dml_rq_dlg_get_dlg_params(
 		disp_dlg_regs->refcyc_per_meta_chunk_flip_c = (unsigned int) (dst_y_per_row_flip * htotal * ref_freq_to_pix_freq) / meta_chunks_per_row_ub_c;
 	}
 
-	disp_dlg_regs->refcyc_per_vm_group_vblank = get_refcyc_per_vm_group_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz;            // From VBA
-	disp_dlg_regs->refcyc_per_vm_group_flip = get_refcyc_per_vm_group_flip_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz;            // From VBA
-	disp_dlg_regs->refcyc_per_vm_req_vblank = get_refcyc_per_vm_req_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10); // From VBA
-	disp_dlg_regs->refcyc_per_vm_req_flip = get_refcyc_per_vm_req_flip_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10);   // From VBA
+	disp_dlg_regs->refcyc_per_vm_group_vblank = (unsigned int)(get_refcyc_per_vm_group_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz);            // From VBA
+	disp_dlg_regs->refcyc_per_vm_group_flip = (unsigned int)(get_refcyc_per_vm_group_flip_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz);            // From VBA
+	disp_dlg_regs->refcyc_per_vm_req_vblank = (unsigned int)(get_refcyc_per_vm_req_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10)); // From VBA
+	disp_dlg_regs->refcyc_per_vm_req_flip = (unsigned int)(get_refcyc_per_vm_req_flip_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10));   // From VBA
 
 	// Clamp to max for now
 	if (disp_dlg_regs->refcyc_per_vm_group_vblank >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_group_vblank = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_group_vblank = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (disp_dlg_regs->refcyc_per_vm_group_flip >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_group_flip = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_group_flip = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (disp_dlg_regs->refcyc_per_vm_req_vblank >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_req_vblank = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_req_vblank = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (disp_dlg_regs->refcyc_per_vm_req_flip >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_vm_req_flip = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_vm_req_flip = (unsigned int)(dml_pow(2, 23) - 1);
 
 	disp_dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int) ((double) dpte_row_height_l / (double) vratio_l * dml_pow(2, 2));
 	ASSERT(disp_dlg_regs->dst_y_per_pte_row_nom_l < (unsigned int)dml_pow(2, 17));
@@ -1572,23 +1572,23 @@ static void dml_rq_dlg_get_dlg_params(
 	disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int) ((double) dpte_row_height_l / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
 			/ (double) dpte_groups_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_pte_group_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_pte_group_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 	disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int) ((double) meta_row_height_l / (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
 			/ (double) meta_chunks_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_meta_chunk_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (dual_plane) {
 		disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int) ((double) dpte_row_height_c / (double) vratio_c * (double) htotal * ref_freq_to_pix_freq
 				/ (double) dpte_groups_per_row_ub_c);
 		if (disp_dlg_regs->refcyc_per_pte_group_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_pte_group_nom_c = dml_pow(2, 23) - 1;
+			disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 
 		// TODO: Is this the right calculation? Does htotal need to be halved?
 		disp_dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int) ((double) meta_row_height_c / (double) vratio_c * (double) htotal * ref_freq_to_pix_freq
 				/ (double) meta_chunks_per_row_ub_c);
 		if (disp_dlg_regs->refcyc_per_meta_chunk_nom_c >= (unsigned int) dml_pow(2, 23))
-			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = dml_pow(2, 23) - 1;
+			disp_dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 	}
 
 	disp_dlg_regs->refcyc_per_line_delivery_pre_l = (unsigned int) dml_floor(refcyc_per_line_delivery_pre_l, 1);
@@ -1629,7 +1629,7 @@ static void dml_rq_dlg_get_dlg_params(
 	disp_ttu_regs->qos_ramp_disable_c = 0;
 	disp_ttu_regs->qos_ramp_disable_cur0 = 0;
 
-	disp_ttu_regs->min_ttu_vblank = min_ttu_vblank * refclk_freq_in_mhz;
+	disp_ttu_regs->min_ttu_vblank = (unsigned int)(min_ttu_vblank * refclk_freq_in_mhz);
 	ASSERT(disp_ttu_regs->min_ttu_vblank < dml_pow(2, 24));
 
 	print__ttu_regs_st(mode_lib, disp_ttu_regs);
@@ -1660,7 +1660,7 @@ void dml314_rq_dlg_get_dlg_reg(
 	dlg_sys_param.t_mclk_wm_us = get_wm_dram_clock_change(mode_lib, e2e_pipe_param, num_pipes);
 	dlg_sys_param.t_sr_wm_us = get_wm_stutter_enter_exit(mode_lib, e2e_pipe_param, num_pipes);
 	dlg_sys_param.total_flip_bw = get_total_immediate_flip_bw(mode_lib, e2e_pipe_param, num_pipes);
-	dlg_sys_param.total_flip_bytes = get_total_immediate_flip_bytes(mode_lib, e2e_pipe_param, num_pipes);
+	dlg_sys_param.total_flip_bytes = (unsigned int)get_total_immediate_flip_bytes(mode_lib, e2e_pipe_param, num_pipes);
 
 	print__dlg_sys_params_st(mode_lib, &dlg_sys_param);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
index eb199215d298e..1b1ab6a6d53a3 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/dcn32_fpu.c
@@ -191,21 +191,24 @@ void dcn32_build_wm_range_table_fpu(struct clk_mgr_internal *clk_mgr)
 	double sr_exit_time_us = clk_mgr->base.ctx->dc->dml.soc.sr_exit_time_us;
 	double sr_enter_plus_exit_time_us = clk_mgr->base.ctx->dc->dml.soc.sr_enter_plus_exit_time_us;
 	/* For min clocks use as reported by PM FW and report those as min */
-	uint16_t min_uclk_mhz			= clk_mgr->base.bw_params->clk_table.entries[0].memclk_mhz;
-	uint16_t min_dcfclk_mhz			= clk_mgr->base.bw_params->clk_table.entries[0].dcfclk_mhz;
+	uint16_t min_uclk_mhz			= (uint16_t)clk_mgr->base.bw_params->clk_table.entries[0].memclk_mhz;
+	uint16_t min_dcfclk_mhz			= (uint16_t)clk_mgr->base.bw_params->clk_table.entries[0].dcfclk_mhz;
 	uint16_t setb_min_uclk_mhz		= min_uclk_mhz;
-	uint16_t dcfclk_mhz_for_the_second_state = clk_mgr->base.ctx->dc->dml.soc.clock_limits[2].dcfclk_mhz;
+	uint16_t dcfclk_mhz_for_the_second_state =
+			(uint16_t)clk_mgr->base.ctx->dc->dml.soc.clock_limits[2].dcfclk_mhz;
 
 	dc_assert_fp_enabled();
 
 	/* For Set B ranges use min clocks state 2 when available, and report those to PM FW */
-	if (dcfclk_mhz_for_the_second_state)
-		clk_mgr->base.bw_params->wm_table.nv_entries[WM_B].pmfw_breakdown.min_dcfclk = dcfclk_mhz_for_the_second_state;
-	else
-		clk_mgr->base.bw_params->wm_table.nv_entries[WM_B].pmfw_breakdown.min_dcfclk = clk_mgr->base.bw_params->clk_table.entries[0].dcfclk_mhz;
+	if (dcfclk_mhz_for_the_second_state) {
+		clk_mgr->base.bw_params->wm_table.nv_entries[WM_B].pmfw_breakdown.min_dcfclk =
+				(uint16_t)dcfclk_mhz_for_the_second_state;
+	} else
+		clk_mgr->base.bw_params->wm_table.nv_entries[WM_B].pmfw_breakdown.min_dcfclk =
+				(uint16_t)clk_mgr->base.bw_params->clk_table.entries[0].dcfclk_mhz;
 
 	if (clk_mgr->base.bw_params->clk_table.entries[2].memclk_mhz)
-		setb_min_uclk_mhz = clk_mgr->base.bw_params->clk_table.entries[2].memclk_mhz;
+		setb_min_uclk_mhz = (uint16_t)clk_mgr->base.bw_params->clk_table.entries[2].memclk_mhz;
 
 	/* Set A - Normal - default values */
 	clk_mgr->base.bw_params->wm_table.nv_entries[WM_A].valid = true;
@@ -297,7 +300,7 @@ int dcn32_find_dummy_latency_index_for_fw_based_mclk_switch(struct dc *dc,
 				dcn32_subvp_in_use(dc, context))
 			vba->DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] = temp_clock_change_support;
 
-		if (vlevel < context->bw_ctx.dml.vba.soc.num_states &&
+		if (vlevel < (int)context->bw_ctx.dml.vba.soc.num_states &&
 				vba->DRAMClockChangeSupport[vlevel][vba->maxMpcComb] != dm_dram_clock_change_unsupported)
 			break;
 
@@ -347,13 +350,13 @@ void dcn32_helper_populate_phantom_dlg_params(struct dc *dc,
 
 		if (pipe->plane_state && dc_state_get_pipe_subvp_type(context, pipe) == SUBVP_PHANTOM) {
 			pipes[pipe_idx].pipe.dest.vstartup_start =
-				get_vstartup(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+				(unsigned int)get_vstartup(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
 			pipes[pipe_idx].pipe.dest.vupdate_offset =
-				get_vupdate_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+				(unsigned int)get_vupdate_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
 			pipes[pipe_idx].pipe.dest.vupdate_width =
-				get_vupdate_width(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+				(unsigned int)get_vupdate_width(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
 			pipes[pipe_idx].pipe.dest.vready_offset =
-				get_vready_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+				(unsigned int)get_vready_offset(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
 			pipe->pipe_dlg_param = pipes[pipe_idx].pipe.dest;
 		}
 		pipe_idx++;
@@ -367,18 +370,18 @@ static float calculate_net_bw_in_kbytes_sec(struct _vcs_dpi_voltage_scaling_st *
 	float sdp_bw_kbytes_sec;
 	float limiting_bw_kbytes_sec;
 
-	memory_bw_kbytes_sec = entry->dram_speed_mts *
+	memory_bw_kbytes_sec = (float)(entry->dram_speed_mts *
 				dcn3_2_soc.num_chans *
 				dcn3_2_soc.dram_channel_width_bytes *
-				((float)dcn3_2_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100);
+			((float)dcn3_2_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100));
 
-	fabric_bw_kbytes_sec = entry->fabricclk_mhz *
+	fabric_bw_kbytes_sec = (float)(entry->fabricclk_mhz *
 				dcn3_2_soc.return_bus_width_bytes *
-				((float)dcn3_2_soc.pct_ideal_fabric_bw_after_urgent / 100);
+			((float)dcn3_2_soc.pct_ideal_fabric_bw_after_urgent / 100));
 
-	sdp_bw_kbytes_sec = entry->dcfclk_mhz *
+	sdp_bw_kbytes_sec = (float)(entry->dcfclk_mhz *
 				dcn3_2_soc.return_bus_width_bytes *
-				((float)dcn3_2_soc.pct_ideal_sdp_bw_after_urgent / 100);
+			((float)dcn3_2_soc.pct_ideal_sdp_bw_after_urgent / 100));
 
 	limiting_bw_kbytes_sec = memory_bw_kbytes_sec;
 
@@ -394,23 +397,23 @@ static float calculate_net_bw_in_kbytes_sec(struct _vcs_dpi_voltage_scaling_st *
 static void get_optimal_ntuple(struct _vcs_dpi_voltage_scaling_st *entry)
 {
 	if (entry->dcfclk_mhz > 0) {
-		float bw_on_sdp = entry->dcfclk_mhz * dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_sdp_bw_after_urgent / 100);
+		float bw_on_sdp = (float)(entry->dcfclk_mhz * dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_sdp_bw_after_urgent / 100));
 
-		entry->fabricclk_mhz = bw_on_sdp / (dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_fabric_bw_after_urgent / 100));
-		entry->dram_speed_mts = bw_on_sdp / (dcn3_2_soc.num_chans *
-				dcn3_2_soc.dram_channel_width_bytes * ((float)dcn3_2_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100));
+		entry->fabricclk_mhz = (float)(bw_on_sdp / (dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_fabric_bw_after_urgent / 100)));
+		entry->dram_speed_mts = (float)(bw_on_sdp / (dcn3_2_soc.num_chans *
+				dcn3_2_soc.dram_channel_width_bytes * ((float)dcn3_2_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100)));
 	} else if (entry->fabricclk_mhz > 0) {
-		float bw_on_fabric = entry->fabricclk_mhz * dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_fabric_bw_after_urgent / 100);
+		float bw_on_fabric = (float)(entry->fabricclk_mhz * dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_fabric_bw_after_urgent / 100));
 
-		entry->dcfclk_mhz = bw_on_fabric / (dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_sdp_bw_after_urgent / 100));
-		entry->dram_speed_mts = bw_on_fabric / (dcn3_2_soc.num_chans *
-				dcn3_2_soc.dram_channel_width_bytes * ((float)dcn3_2_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100));
+		entry->dcfclk_mhz = (float)(bw_on_fabric / (dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_sdp_bw_after_urgent / 100)));
+		entry->dram_speed_mts = (float)(bw_on_fabric / (dcn3_2_soc.num_chans *
+				dcn3_2_soc.dram_channel_width_bytes * ((float)dcn3_2_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100)));
 	} else if (entry->dram_speed_mts > 0) {
-		float bw_on_dram = entry->dram_speed_mts * dcn3_2_soc.num_chans *
-				dcn3_2_soc.dram_channel_width_bytes * ((float)dcn3_2_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100);
+		float bw_on_dram = (float)(entry->dram_speed_mts * dcn3_2_soc.num_chans *
+				dcn3_2_soc.dram_channel_width_bytes * ((float)dcn3_2_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100));
 
-		entry->fabricclk_mhz = bw_on_dram / (dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_fabric_bw_after_urgent / 100));
-		entry->dcfclk_mhz = bw_on_dram / (dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_sdp_bw_after_urgent / 100));
+		entry->fabricclk_mhz = (float)(bw_on_dram / (dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_fabric_bw_after_urgent / 100)));
+		entry->dcfclk_mhz = (float)(bw_on_dram / (dcn3_2_soc.return_bus_width_bytes * ((float)dcn3_2_soc.pct_ideal_sdp_bw_after_urgent / 100)));
 	}
 }
 
@@ -418,8 +421,8 @@ static void insert_entry_into_table_sorted(struct _vcs_dpi_voltage_scaling_st *t
 				    unsigned int *num_entries,
 				    struct _vcs_dpi_voltage_scaling_st *entry)
 {
-	int i = 0;
-	int index = 0;
+	unsigned int i = 0;
+	unsigned int index = 0;
 
 	dc_assert_fp_enabled();
 
@@ -476,8 +479,8 @@ void dcn32_set_phantom_stream_timing(struct dc *dc,
 	uint32_t phantom_vactive, phantom_bp, pstate_width_fw_delay_lines;
 	unsigned int num_dpp;
 	unsigned int vlevel = context->bw_ctx.dml.vba.VoltageLevel;
-	unsigned int dcfclk = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
-	unsigned int socclk = context->bw_ctx.dml.vba.SOCCLKPerState[vlevel];
+	unsigned int dcfclk = (unsigned int)context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
+	unsigned int socclk = (unsigned int)context->bw_ctx.dml.vba.SOCCLKPerState[vlevel];
 	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
 	struct dc_stream_state *main_stream = ref_pipe->stream;
 
@@ -497,10 +500,10 @@ void dcn32_set_phantom_stream_timing(struct dc *dc,
 	}
 
 	// Calculate lines required for pstate allow width and FW processing delays
-	pstate_width_fw_delay_lines = ((double)(dc->caps.subvp_fw_processing_delay_us +
+	pstate_width_fw_delay_lines = (uint32_t)(((double)(dc->caps.subvp_fw_processing_delay_us +
 			dc->caps.subvp_pstate_allow_width_us) / 1000000) *
 			(ref_pipe->stream->timing.pix_clk_100hz * 100) /
-			(double)ref_pipe->stream->timing.h_total;
+			(double)ref_pipe->stream->timing.h_total);
 
 	// Update clks_cfg for calling into recalculate
 	pipes[0].clks_cfg.voltage = vlevel;
@@ -512,8 +515,8 @@ void dcn32_set_phantom_stream_timing(struct dc *dc,
 	/* Add 16 lines margin to the MALL REGION because SUB_VP_START_LINE must be aligned
 	 * to 2 swaths (i.e. 16 lines)
 	 */
-	phantom_vactive = get_subviewport_lines_needed_in_mall(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx) +
-				pstate_width_fw_delay_lines + dc->caps.subvp_swath_height_margin_lines;
+	phantom_vactive = (uint32_t)(get_subviewport_lines_needed_in_mall(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx) +
+				pstate_width_fw_delay_lines + dc->caps.subvp_swath_height_margin_lines);
 
 	// W/A for DCC corruption with certain high resolution timings.
 	// Determing if pipesplit is used. If so, add meta_row_height to the phantom vactive.
@@ -524,7 +527,7 @@ void dcn32_set_phantom_stream_timing(struct dc *dc,
 	phantom_vactive += dc->debug.subvp_extra_lines;
 
 	// For backporch of phantom pipe, use vstartup of the main pipe
-	phantom_bp = get_vstartup(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+	phantom_bp = (uint32_t)get_vstartup(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
 
 	phantom_stream->dst.y = 0;
 	phantom_stream->dst.height = phantom_vactive;
@@ -534,7 +537,7 @@ void dcn32_set_phantom_stream_timing(struct dc *dc,
 	 * phantom for this case.
 	 */
 	phantom_stream->src.y = 0;
-	phantom_stream->src.height = (double)phantom_vactive * (double)main_stream->src.height / (double)main_stream->dst.height;
+	phantom_stream->src.height = (int)((double)phantom_vactive * (double)main_stream->src.height / (double)main_stream->dst.height);
 
 	phantom_stream->timing.v_addressable = phantom_vactive;
 	phantom_stream->timing.v_front_porch = 1;
@@ -615,9 +618,9 @@ static bool dcn32_assign_subvp_pipe(struct dc *dc,
 			continue;
 
 		// Round up
-		refresh_rate = (pipe->stream->timing.pix_clk_100hz * 100 +
+		refresh_rate = (unsigned int)((pipe->stream->timing.pix_clk_100hz * 100 +
 				pipe->stream->timing.v_total * pipe->stream->timing.h_total - 1)
-				/ (double)(pipe->stream->timing.v_total * pipe->stream->timing.h_total);
+				/ (double)(pipe->stream->timing.v_total * pipe->stream->timing.h_total));
 		/* SubVP pipe candidate requirements:
 		 * - Refresh rate < 120hz
 		 * - Not able to switch in vactive naturally (switching in active means the
@@ -644,8 +647,8 @@ static bool dcn32_assign_subvp_pipe(struct dc *dc,
 			pipe = &context->res_ctx.pipe_ctx[i];
 			if (num_pipes <= free_pipes) {
 				struct dc_stream_state *stream = pipe->stream;
-				unsigned int frame_us = (stream->timing.v_total * stream->timing.h_total /
-						(double)(stream->timing.pix_clk_100hz * 100)) * 1000000;
+				unsigned int frame_us = (unsigned int)((stream->timing.v_total * stream->timing.h_total /
+						(double)(stream->timing.pix_clk_100hz * 100)) * 1000000);
 				if (frame_us > max_frame_time) {
 					*index = i;
 					max_frame_time = frame_us;
@@ -747,10 +750,10 @@ static bool subvp_subvp_schedulable(struct dc *dc, struct dc_state *context)
 					phantom->timing.v_addressable;
 
 			// Round up when calculating microschedule time (+ 1 at the end)
-			time_us = (microschedule_lines * phantom->timing.h_total) /
+			time_us = (uint32_t)((microschedule_lines * phantom->timing.h_total) /
 					(double)(phantom->timing.pix_clk_100hz * 100) * 1000000 +
 						dc->caps.subvp_prefetch_end_to_mall_start_us +
-						dc->caps.subvp_fw_processing_delay_us + 1;
+						dc->caps.subvp_fw_processing_delay_us + 1);
 			if (time_us > max_microschedule_us)
 				max_microschedule_us = time_us;
 
@@ -762,19 +765,19 @@ static bool subvp_subvp_schedulable(struct dc *dc, struct dc_state *context)
 				break;
 		}
 	}
-	vactive1_us = ((subvp_pipes[0]->stream->timing.v_addressable * subvp_pipes[0]->stream->timing.h_total) /
-			(double)(subvp_pipes[0]->stream->timing.pix_clk_100hz * 100)) * 1000000;
-	vactive2_us = ((subvp_pipes[1]->stream->timing.v_addressable * subvp_pipes[1]->stream->timing.h_total) /
-				(double)(subvp_pipes[1]->stream->timing.pix_clk_100hz * 100)) * 1000000;
-	vblank1_us = (((subvp_pipes[0]->stream->timing.v_total - subvp_pipes[0]->stream->timing.v_addressable) *
+	vactive1_us = (int32_t)(((subvp_pipes[0]->stream->timing.v_addressable * subvp_pipes[0]->stream->timing.h_total) /
+			(double)(subvp_pipes[0]->stream->timing.pix_clk_100hz * 100)) * 1000000);
+	vactive2_us = (int32_t)(((subvp_pipes[1]->stream->timing.v_addressable * subvp_pipes[1]->stream->timing.h_total) /
+				(double)(subvp_pipes[1]->stream->timing.pix_clk_100hz * 100)) * 1000000);
+	vblank1_us = (int32_t)((((subvp_pipes[0]->stream->timing.v_total - subvp_pipes[0]->stream->timing.v_addressable) *
 			subvp_pipes[0]->stream->timing.h_total) /
-			(double)(subvp_pipes[0]->stream->timing.pix_clk_100hz * 100)) * 1000000;
-	vblank2_us = (((subvp_pipes[1]->stream->timing.v_total - subvp_pipes[1]->stream->timing.v_addressable) *
+			(double)(subvp_pipes[0]->stream->timing.pix_clk_100hz * 100)) * 1000000);
+	vblank2_us = (int32_t)((((subvp_pipes[1]->stream->timing.v_total - subvp_pipes[1]->stream->timing.v_addressable) *
 			subvp_pipes[1]->stream->timing.h_total) /
-			(double)(subvp_pipes[1]->stream->timing.pix_clk_100hz * 100)) * 1000000;
+			(double)(subvp_pipes[1]->stream->timing.pix_clk_100hz * 100)) * 1000000);
 
-	if ((vactive1_us - vblank2_us) / 2 > max_microschedule_us &&
-	    (vactive2_us - vblank1_us) / 2 > max_microschedule_us)
+	if ((vactive1_us - vblank2_us) / 2 > (int32_t)max_microschedule_us &&
+	    (vactive2_us - vblank1_us) / 2 > (int32_t)max_microschedule_us)
 		return true;
 
 	return false;
@@ -852,19 +855,19 @@ static bool subvp_drr_schedulable(struct dc *dc, struct dc_state *context)
 		main_timing = &pipe->stream->timing;
 		phantom_timing = &phantom_stream->timing;
 		drr_timing = &drr_pipe->stream->timing;
-		prefetch_us = (phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total /
+		prefetch_us = (uint16_t)((phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total /
 				(double)(phantom_timing->pix_clk_100hz * 100) * 1000000 +
-				dc->caps.subvp_prefetch_end_to_mall_start_us;
-		subvp_active_us = main_timing->v_addressable * main_timing->h_total /
-				(double)(main_timing->pix_clk_100hz * 100) * 1000000;
-		drr_frame_us = drr_timing->v_total * drr_timing->h_total /
-				(double)(drr_timing->pix_clk_100hz * 100) * 1000000;
+				dc->caps.subvp_prefetch_end_to_mall_start_us);
+		subvp_active_us = (uint16_t)(main_timing->v_addressable * main_timing->h_total /
+				(double)(main_timing->pix_clk_100hz * 100) * 1000000);
+		drr_frame_us = (uint16_t)(drr_timing->v_total * drr_timing->h_total /
+				(double)(drr_timing->pix_clk_100hz * 100) * 1000000);
 		// P-State allow width and FW delays already included phantom_timing->v_addressable
-		mall_region_us = phantom_timing->v_addressable * phantom_timing->h_total /
-				(double)(phantom_timing->pix_clk_100hz * 100) * 1000000;
+		mall_region_us = (uint16_t)(phantom_timing->v_addressable * phantom_timing->h_total /
+				(double)(phantom_timing->pix_clk_100hz * 100) * 1000000);
 		stretched_drr_us = drr_frame_us + mall_region_us + SUBVP_DRR_MARGIN_US;
-		drr_stretched_vblank_us = (drr_timing->v_total - drr_timing->v_addressable) * drr_timing->h_total /
-				(double)(drr_timing->pix_clk_100hz * 100) * 1000000 + (stretched_drr_us - drr_frame_us);
+		drr_stretched_vblank_us = (uint16_t)((drr_timing->v_total - drr_timing->v_addressable) * drr_timing->h_total /
+				(double)(drr_timing->pix_clk_100hz * 100) * 1000000 + (stretched_drr_us - drr_frame_us));
 		max_vblank_mallregion = drr_stretched_vblank_us > mall_region_us ? drr_stretched_vblank_us : mall_region_us;
 	}
 
@@ -901,7 +904,7 @@ static bool subvp_vblank_schedulable(struct dc *dc, struct dc_state *context)
 	struct pipe_ctx *subvp_pipe = NULL;
 	bool found = false;
 	bool schedulable = false;
-	uint32_t i = 0;
+	uint8_t i = 0;
 	uint8_t vblank_index = 0;
 	uint16_t prefetch_us = 0;
 	uint16_t mall_region_us = 0;
@@ -947,18 +950,18 @@ static bool subvp_vblank_schedulable(struct dc *dc, struct dc_state *context)
 		vblank_timing = &context->res_ctx.pipe_ctx[vblank_index].stream->timing;
 		// Prefetch time is equal to VACTIVE + BP + VSYNC of the phantom pipe
 		// Also include the prefetch end to mallstart delay time
-		prefetch_us = (phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total /
+		prefetch_us = (uint16_t)((phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total /
 				(double)(phantom_timing->pix_clk_100hz * 100) * 1000000 +
-				dc->caps.subvp_prefetch_end_to_mall_start_us;
+				dc->caps.subvp_prefetch_end_to_mall_start_us);
 		// P-State allow width and FW delays already included phantom_timing->v_addressable
-		mall_region_us = phantom_timing->v_addressable * phantom_timing->h_total /
-				(double)(phantom_timing->pix_clk_100hz * 100) * 1000000;
-		vblank_frame_us = vblank_timing->v_total * vblank_timing->h_total /
-				(double)(vblank_timing->pix_clk_100hz * 100) * 1000000;
-		vblank_blank_us =  (vblank_timing->v_total - vblank_timing->v_addressable) * vblank_timing->h_total /
-				(double)(vblank_timing->pix_clk_100hz * 100) * 1000000;
-		subvp_active_us = main_timing->v_addressable * main_timing->h_total /
-				(double)(main_timing->pix_clk_100hz * 100) * 1000000;
+		mall_region_us = (uint16_t)(phantom_timing->v_addressable * phantom_timing->h_total /
+				(double)(phantom_timing->pix_clk_100hz * 100) * 1000000);
+		vblank_frame_us = (uint16_t)(vblank_timing->v_total * vblank_timing->h_total /
+				(double)(vblank_timing->pix_clk_100hz * 100) * 1000000);
+		vblank_blank_us =  (uint16_t)((vblank_timing->v_total - vblank_timing->v_addressable) * vblank_timing->h_total /
+				(double)(vblank_timing->pix_clk_100hz * 100) * 1000000);
+		subvp_active_us = (uint16_t)(main_timing->v_addressable * main_timing->h_total /
+				(double)(main_timing->pix_clk_100hz * 100) * 1000000);
 		max_vblank_mallregion = vblank_blank_us > mall_region_us ? vblank_blank_us : mall_region_us;
 
 		// Schedulable if VACTIVE region of the SubVP pipe can fit the MALL prefetch, VBLANK frame time,
@@ -986,7 +989,7 @@ static bool subvp_subvp_admissable(struct dc *dc,
 				struct dc_state *context)
 {
 	bool result = false;
-	uint32_t i;
+	uint8_t i;
 	uint8_t subvp_count = 0;
 	uint32_t min_refresh = subvp_high_refresh_list.min_refresh, max_refresh = 0;
 	uint64_t refresh_rate = 0;
@@ -1013,8 +1016,8 @@ static bool subvp_subvp_admissable(struct dc *dc,
 	}
 
 	if (subvp_count == 2 && ((min_refresh < 120 && max_refresh < 120) ||
-		(min_refresh >= subvp_high_refresh_list.min_refresh &&
-				max_refresh <= subvp_high_refresh_list.max_refresh)))
+		(min_refresh >= (uint32_t)subvp_high_refresh_list.min_refresh &&
+				max_refresh <= (uint32_t)subvp_high_refresh_list.max_refresh)))
 		result = true;
 
 	return result;
@@ -1092,7 +1095,7 @@ static bool subvp_validate_static_schedulability(struct dc *dc,
 
 static void assign_subvp_index(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	int index = 0;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -1100,7 +1103,7 @@ static void assign_subvp_index(struct dc *dc, struct dc_state *context)
 
 		if (resource_is_pipe_type(pipe_ctx, OTG_MASTER) &&
 				dc_state_get_pipe_subvp_type(context, pipe_ctx) == SUBVP_MAIN) {
-			pipe_ctx->subvp_index = index++;
+			pipe_ctx->subvp_index = (uint8_t)index++;
 		} else {
 			pipe_ctx->subvp_index = 0;
 		}
@@ -1224,7 +1227,8 @@ static bool update_pipe_slice_table_with_split_flags(
 	 */
 	struct pipe_ctx *pipe;
 	bool odm;
-	int dc_pipe_idx, dml_pipe_idx = 0;
+	unsigned int dc_pipe_idx;
+	int dml_pipe_idx = 0;
 	bool updated = false;
 
 	for (dc_pipe_idx = 0;
@@ -1396,7 +1400,7 @@ static void try_odm_power_optimization_and_revalidate(
 		display_e2e_pipe_params_st *pipes,
 		int *split,
 		bool *merge,
-		unsigned int *vlevel,
+		int *vlevel,
 		int pipe_cnt)
 {
 	int i;
@@ -1466,7 +1470,7 @@ static bool dcn32_full_validate_bw_helper(struct dc *dc,
 
 	*vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, *pipe_cnt);
 	/* This may adjust vlevel and maxMpcComb */
-	if (*vlevel < context->bw_ctx.dml.soc.num_states) {
+	if (*vlevel < (int)context->bw_ctx.dml.soc.num_states) {
 		*vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge);
 		vba->VoltageLevel = *vlevel;
 	}
@@ -1527,14 +1531,14 @@ static bool dcn32_full_validate_bw_helper(struct dc *dc,
 			/* Check that vlevel requested supports pstate or not
 			 * if not, select the lowest vlevel that supports it
 			 */
-			for (i = *vlevel; i < context->bw_ctx.dml.soc.num_states; i++) {
+			for (i = *vlevel; i < (int)context->bw_ctx.dml.soc.num_states; i++) {
 				if (vba->DRAMClockChangeSupport[i][vba->maxMpcComb] != dm_dram_clock_change_unsupported) {
 					*vlevel = i;
 					break;
 				}
 			}
 
-			if (*vlevel < context->bw_ctx.dml.soc.num_states
+			if (*vlevel < (int)context->bw_ctx.dml.soc.num_states
 			    && subvp_validate_static_schedulability(dc, context, *vlevel))
 				found_supported_config = true;
 			if (found_supported_config) {
@@ -1566,7 +1570,7 @@ static bool dcn32_full_validate_bw_helper(struct dc *dc,
 
 			*vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, *pipe_cnt);
 			/* This may adjust vlevel and maxMpcComb */
-			if (*vlevel < context->bw_ctx.dml.soc.num_states) {
+			if (*vlevel < (int)context->bw_ctx.dml.soc.num_states) {
 				*vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, *vlevel, split, merge);
 				vba->VoltageLevel = *vlevel;
 			}
@@ -1599,7 +1603,7 @@ static bool dcn32_full_validate_bw_helper(struct dc *dc,
 
 static bool is_dtbclk_required(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		if (!context->res_ctx.pipe_ctx[i].stream)
@@ -1614,7 +1618,8 @@ static void dcn32_calculate_dlg_params(struct dc *dc, struct dc_state *context,
 				       display_e2e_pipe_params_st *pipes,
 				       int pipe_cnt, int vlevel)
 {
-	int i, pipe_idx, active_hubp_count = 0;
+	int pipe_idx, active_hubp_count = 0;
+	unsigned int i;
 	bool usr_retraining_support = false;
 	bool unbounded_req_enabled = false;
 	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
@@ -1624,12 +1629,12 @@ static void dcn32_calculate_dlg_params(struct dc *dc, struct dc_state *context,
 	/* Writeback MCIF_WB arbitration parameters */
 	dc->res_pool->funcs->set_mcif_arb_params(dc, context, pipes, pipe_cnt);
 
-	context->bw_ctx.bw.dcn.clk.dispclk_khz = context->bw_ctx.dml.vba.DISPCLK * 1000;
-	context->bw_ctx.bw.dcn.clk.dcfclk_khz = context->bw_ctx.dml.vba.DCFCLK * 1000;
-	context->bw_ctx.bw.dcn.clk.socclk_khz = context->bw_ctx.dml.vba.SOCCLK * 1000;
-	context->bw_ctx.bw.dcn.clk.dramclk_khz = context->bw_ctx.dml.vba.DRAMSpeed * 1000 / 16;
-	context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz = context->bw_ctx.dml.vba.DCFCLKDeepSleep * 1000;
-	context->bw_ctx.bw.dcn.clk.fclk_khz = context->bw_ctx.dml.vba.FabricClock * 1000;
+	context->bw_ctx.bw.dcn.clk.dispclk_khz = (int)(context->bw_ctx.dml.vba.DISPCLK * 1000);
+	context->bw_ctx.bw.dcn.clk.dcfclk_khz = (int)(context->bw_ctx.dml.vba.DCFCLK * 1000);
+	context->bw_ctx.bw.dcn.clk.socclk_khz = (int)(context->bw_ctx.dml.vba.SOCCLK * 1000);
+	context->bw_ctx.bw.dcn.clk.dramclk_khz = (int)(context->bw_ctx.dml.vba.DRAMSpeed * 1000 / 16);
+	context->bw_ctx.bw.dcn.clk.dcfclk_deep_sleep_khz = (int)(context->bw_ctx.dml.vba.DCFCLKDeepSleep * 1000);
+	context->bw_ctx.bw.dcn.clk.fclk_khz = (int)(context->bw_ctx.dml.vba.FabricClock * 1000);
 	context->bw_ctx.bw.dcn.clk.p_state_change_support =
 			context->bw_ctx.dml.vba.DRAMClockChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb]
 					!= dm_dram_clock_change_unsupported;
@@ -1641,7 +1646,7 @@ static void dcn32_calculate_dlg_params(struct dc *dc, struct dc_state *context,
 
 	context->bw_ctx.bw.dcn.clk.dppclk_khz = 0;
 	context->bw_ctx.bw.dcn.clk.dtbclk_en = is_dtbclk_required(dc, context);
-	context->bw_ctx.bw.dcn.clk.ref_dtbclk_khz = context->bw_ctx.dml.vba.DTBCLKPerState[vlevel] * 1000;
+	context->bw_ctx.bw.dcn.clk.ref_dtbclk_khz = (int)(context->bw_ctx.dml.vba.DTBCLKPerState[vlevel] * 1000);
 	if (context->bw_ctx.dml.vba.FCLKChangeSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb] == dm_fclock_change_unsupported)
 		context->bw_ctx.bw.dcn.clk.fclk_p_state_change_support = false;
 	else
@@ -1650,8 +1655,8 @@ static void dcn32_calculate_dlg_params(struct dc *dc, struct dc_state *context,
 	usr_retraining_support = context->bw_ctx.dml.vba.USRRetrainingSupport[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
 	ASSERT(usr_retraining_support);
 
-	if (context->bw_ctx.bw.dcn.clk.dispclk_khz < dc->debug.min_disp_clk_khz)
-		context->bw_ctx.bw.dcn.clk.dispclk_khz = dc->debug.min_disp_clk_khz;
+	if ((unsigned int)context->bw_ctx.bw.dcn.clk.dispclk_khz < dc->debug.min_disp_clk_khz)
+		context->bw_ctx.bw.dcn.clk.dispclk_khz = (int)dc->debug.min_disp_clk_khz;
 
 	unbounded_req_enabled = get_unbounded_request_enabled(&context->bw_ctx.dml, pipes, pipe_cnt);
 
@@ -1670,13 +1675,13 @@ static void dcn32_calculate_dlg_params(struct dc *dc, struct dc_state *context,
 			continue;
 		if (context->res_ctx.pipe_ctx[i].plane_state)
 			active_hubp_count++;
-		pipes[pipe_idx].pipe.dest.vstartup_start = get_vstartup(&context->bw_ctx.dml, pipes, pipe_cnt,
+		pipes[pipe_idx].pipe.dest.vstartup_start = (unsigned int)get_vstartup(&context->bw_ctx.dml, pipes, pipe_cnt,
 				pipe_idx);
-		pipes[pipe_idx].pipe.dest.vupdate_offset = get_vupdate_offset(&context->bw_ctx.dml, pipes, pipe_cnt,
+		pipes[pipe_idx].pipe.dest.vupdate_offset = (unsigned int)get_vupdate_offset(&context->bw_ctx.dml, pipes, pipe_cnt,
 				pipe_idx);
-		pipes[pipe_idx].pipe.dest.vupdate_width = get_vupdate_width(&context->bw_ctx.dml, pipes, pipe_cnt,
+		pipes[pipe_idx].pipe.dest.vupdate_width = (unsigned int)get_vupdate_width(&context->bw_ctx.dml, pipes, pipe_cnt,
 				pipe_idx);
-		pipes[pipe_idx].pipe.dest.vready_offset = get_vready_offset(&context->bw_ctx.dml, pipes, pipe_cnt,
+		pipes[pipe_idx].pipe.dest.vready_offset = (unsigned int)get_vready_offset(&context->bw_ctx.dml, pipes, pipe_cnt,
 				pipe_idx);
 
 		if (dc_state_get_pipe_subvp_type(context, &context->res_ctx.pipe_ctx[i]) == SUBVP_PHANTOM) {
@@ -1684,20 +1689,20 @@ static void dcn32_calculate_dlg_params(struct dc *dc, struct dc_state *context,
 			context->res_ctx.pipe_ctx[i].det_buffer_size_kb = 0;
 			context->res_ctx.pipe_ctx[i].unbounded_req = false;
 		} else {
-			context->res_ctx.pipe_ctx[i].det_buffer_size_kb = get_det_buffer_size_kbytes(&context->bw_ctx.dml, pipes, pipe_cnt,
+			context->res_ctx.pipe_ctx[i].det_buffer_size_kb = (int)get_det_buffer_size_kbytes(&context->bw_ctx.dml, pipes, pipe_cnt,
 							pipe_idx);
 			context->res_ctx.pipe_ctx[i].unbounded_req = unbounded_req_enabled;
 		}
 
 		if (context->bw_ctx.bw.dcn.clk.dppclk_khz < pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000)
-			context->bw_ctx.bw.dcn.clk.dppclk_khz = pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000;
+			context->bw_ctx.bw.dcn.clk.dppclk_khz = (int)(pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000);
 		if (context->res_ctx.pipe_ctx[i].plane_state)
-			context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz = pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000;
+			context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz = (int)(pipes[pipe_idx].clks_cfg.dppclk_mhz * 1000);
 		else
 			context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz = 0;
 		context->res_ctx.pipe_ctx[i].pipe_dlg_param = pipes[pipe_idx].pipe.dest;
 
-		context->res_ctx.pipe_ctx[i].surface_size_in_mall_bytes = get_surface_size_in_mall(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
+		context->res_ctx.pipe_ctx[i].surface_size_in_mall_bytes = (unsigned int)get_surface_size_in_mall(&context->bw_ctx.dml, pipes, pipe_cnt, pipe_idx);
 
 		if (vba->ActiveDRAMClockChangeLatencyMarginPerState[vba->VoltageLevel][vba->maxMpcComb][vba->pipe_plane[pipe_idx]] > 0)
 			context->res_ctx.pipe_ctx[i].has_vactive_margin = true;
@@ -1740,10 +1745,10 @@ static void dcn32_calculate_dlg_params(struct dc *dc, struct dc_state *context,
 	/*save a original dppclock copy*/
 	context->bw_ctx.bw.dcn.clk.bw_dppclk_khz = context->bw_ctx.bw.dcn.clk.dppclk_khz;
 	context->bw_ctx.bw.dcn.clk.bw_dispclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz;
-	context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = context->bw_ctx.dml.soc.clock_limits[vlevel].dppclk_mhz
-			* 1000;
-	context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = context->bw_ctx.dml.soc.clock_limits[vlevel].dispclk_mhz
-			* 1000;
+	context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = (int)(context->bw_ctx.dml.soc.clock_limits[vlevel].dppclk_mhz
+			* 1000);
+	context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = (int)(context->bw_ctx.dml.soc.clock_limits[vlevel].dispclk_mhz
+			* 1000);
 
 	context->bw_ctx.bw.dcn.clk.num_ways = dcn32_helper_calculate_num_ways_for_subvp(dc, context);
 
@@ -1779,7 +1784,7 @@ static struct pipe_ctx *dcn32_find_split_pipe(
 
 	if (old_index >= 0 && context->res_ctx.pipe_ctx[old_index].stream == NULL) {
 		pipe = &context->res_ctx.pipe_ctx[old_index];
-		pipe->pipe_idx = old_index;
+		pipe->pipe_idx = (uint8_t)old_index;
 	}
 
 	if (!pipe)
@@ -1788,7 +1793,7 @@ static struct pipe_ctx *dcn32_find_split_pipe(
 					&& dc->current_state->res_ctx.pipe_ctx[i].prev_odm_pipe == NULL) {
 				if (context->res_ctx.pipe_ctx[i].stream == NULL) {
 					pipe = &context->res_ctx.pipe_ctx[i];
-					pipe->pipe_idx = i;
+					pipe->pipe_idx = (uint8_t)i;
 					break;
 				}
 			}
@@ -1803,7 +1808,7 @@ static struct pipe_ctx *dcn32_find_split_pipe(
 		for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
 			if (context->res_ctx.pipe_ctx[i].stream == NULL) {
 				pipe = &context->res_ctx.pipe_ctx[i];
-				pipe->pipe_idx = i;
+				pipe->pipe_idx = (uint8_t)i;
 				break;
 			}
 		}
@@ -1846,13 +1851,13 @@ static bool dcn32_split_stream_for_mpc_or_odm(
 
 	*sec_pipe = *pri_pipe;
 
-	sec_pipe->pipe_idx = pipe_idx;
+	sec_pipe->pipe_idx = (uint8_t)pipe_idx;
 	sec_pipe->plane_res.mi = pool->mis[pipe_idx];
 	sec_pipe->plane_res.hubp = pool->hubps[pipe_idx];
 	sec_pipe->plane_res.ipp = pool->ipps[pipe_idx];
 	sec_pipe->plane_res.xfm = pool->transforms[pipe_idx];
 	sec_pipe->plane_res.dpp = pool->dpps[pipe_idx];
-	sec_pipe->plane_res.mpcc_inst = pool->dpps[pipe_idx]->inst;
+	sec_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[pipe_idx]->inst;
 	sec_pipe->stream_res.dsc = NULL;
 	if (odm) {
 		if (pri_pipe->next_odm_pipe) {
@@ -1904,7 +1909,8 @@ static bool dcn32_apply_merge_split_flags_helper(
 		int *split,
 		bool *merge)
 {
-	int i, pipe_idx;
+	int pipe_idx;
+	unsigned int i;
 	bool newly_split[MAX_PIPES] = { false };
 	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
 
@@ -2165,7 +2171,7 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 
 		context->bw_ctx.dml.validate_max_state = false;
 
-		if (vlevel < context->bw_ctx.dml.soc.num_states) {
+		if (vlevel < (int)context->bw_ctx.dml.soc.num_states) {
 			memset(split, 0, sizeof(split));
 			memset(merge, 0, sizeof(merge));
 			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
@@ -2179,7 +2185,7 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 	if (vlevel == context->bw_ctx.dml.soc.num_states)
 		goto validate_fail;
 
-	for (i = 0, pipe_idx = 0; i < dc->res_pool->pipe_count; i++) {
+	for (i = 0, pipe_idx = 0; i < (int)dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
 		struct pipe_ctx *mpo_pipe = pipe->bottom_pipe;
 
@@ -2210,7 +2216,7 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 	if (repopulate_pipes) {
 		int flag_max_mpc_comb = vba->maxMpcComb;
 		int flag_vlevel = vlevel;
-		int i;
+		int j;
 
 		pipe_cnt = dc->res_pool->funcs->populate_dml_pipes(dc, context, pipes, validate_mode);
 		if (!dc->config.enable_windowed_mpo_odm)
@@ -2225,19 +2231,20 @@ bool dcn32_internal_validate_bw(struct dc *dc,
 					dm_prefetch_support_uclk_fclk_and_stutter_if_possible;
 
 		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
+		const int num_states = (int)context->bw_ctx.dml.soc.num_states;
 
-		if (vlevel == context->bw_ctx.dml.soc.num_states) {
+		if (vlevel == num_states) {
 			/* failed after DET size changes */
 			goto validate_fail;
 		} else if (flag_max_mpc_comb == 0 &&
 				flag_max_mpc_comb != context->bw_ctx.dml.vba.maxMpcComb) {
 			/* check the context constructed with pipe split flags is still valid*/
 			bool flags_valid = false;
-			for (i = flag_vlevel; i < context->bw_ctx.dml.soc.num_states; i++) {
-				if (vba->ModeSupport[i][flag_max_mpc_comb]) {
+			for (j = flag_vlevel; j < (int)context->bw_ctx.dml.soc.num_states; j++) {
+				if (vba->ModeSupport[j][flag_max_mpc_comb]) {
 					vba->maxMpcComb = flag_max_mpc_comb;
-					vba->VoltageLevel = i;
-					vlevel = i;
+					vba->VoltageLevel = j;
+					vlevel = j;
 					flags_valid = true;
 					break;
 				}
@@ -2267,7 +2274,8 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 				int pipe_cnt,
 				int vlevel)
 {
-	int i, pipe_idx, vlevel_temp = 0;
+	int pipe_idx, vlevel_temp = 0;
+	unsigned int i;
 	double dcfclk = dcn3_2_soc.clock_limits[0].dcfclk_mhz;
 	double dcfclk_from_validation = context->bw_ctx.dml.vba.DCFCLKState[vlevel][context->bw_ctx.dml.vba.maxMpcComb];
 	double dram_speed_from_validation = context->bw_ctx.dml.vba.DRAMSpeed;
@@ -2276,7 +2284,7 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 			dm_dram_clock_change_unsupported;
 	unsigned int dummy_latency_index = 0;
 	int maxMpcComb = context->bw_ctx.dml.vba.maxMpcComb;
-	unsigned int min_dram_speed_mts = context->bw_ctx.dml.vba.DRAMSpeed;
+	unsigned int min_dram_speed_mts = (unsigned int)context->bw_ctx.dml.vba.DRAMSpeed;
 	bool subvp_in_use = dcn32_subvp_in_use(dc, context);
 	unsigned int min_dram_speed_mts_margin;
 	bool need_fclk_lat_as_dummy = false;
@@ -2405,16 +2413,16 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_enter_plus_exit_time_us;
 		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_B].dml_input.sr_exit_time_us;
 	}
-	context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.fclk_pstate_change_ns = get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.b.usr_retraining_ns = get_usr_retraining_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.b.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.urgent_latency_ns = (uint32_t)(get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.cstate_pstate.fclk_pstate_change_ns = (uint32_t)(get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.b.usr_retraining_ns = (uint32_t)(get_usr_retraining_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 
 	/* Set D:
 	 * All clocks min.
@@ -2469,7 +2477,7 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 	}
 
 	if (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid) {
-		min_dram_speed_mts = dram_speed_from_validation;
+		min_dram_speed_mts = (unsigned int)dram_speed_from_validation;
 		min_dram_speed_mts_margin = 160;
 
 		context->bw_ctx.dml.soc.dram_clock_change_latency_us =
@@ -2501,20 +2509,20 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].dml_input.sr_exit_time_us;
 	}
 
-	context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.c.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.urgent_latency_ns = (uint32_t)(get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 	/* On DCN32/321, PMFW will set PSTATE_CHANGE_TYPE = 1 (FCLK) for UCLK dummy p-state.
 	 * In this case we must program FCLK WM Set C to use the UCLK dummy p-state WM
 	 * value.
 	 */
-	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.fclk_pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-	context->bw_ctx.bw.dcn.watermarks.c.usr_retraining_ns = get_usr_retraining_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+	context->bw_ctx.bw.dcn.watermarks.c.cstate_pstate.fclk_pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+	context->bw_ctx.bw.dcn.watermarks.c.usr_retraining_ns = (uint32_t)(get_usr_retraining_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 
 	if ((!pstate_en) && (dc->clk_mgr->bw_params->wm_table.nv_entries[WM_C].valid)) {
 		/* The only difference between A and C is p-state latency, if p-state is not supported
@@ -2526,7 +2534,7 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 		/* Calculate FCLK p-state change watermark based on FCLK pstate change latency in case
 		 * UCLK p-state is not supported, to avoid underflow in case FCLK pstate is supported
 		 */
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.fclk_pstate_change_ns = get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.fclk_pstate_change_ns = (uint32_t)(get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 	} else {
 		/* Set A:
 		 * All clocks min.
@@ -2540,16 +2548,16 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 		context->bw_ctx.dml.soc.sr_enter_plus_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.sr_enter_plus_exit_time_us;
 		context->bw_ctx.dml.soc.sr_exit_time_us = dc->clk_mgr->bw_params->wm_table.nv_entries[WM_A].dml_input.sr_exit_time_us;
 
-		context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.urgent_latency_ns = get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.fclk_pstate_change_ns = get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
-		context->bw_ctx.bw.dcn.watermarks.a.usr_retraining_ns = get_usr_retraining_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000;
+		context->bw_ctx.bw.dcn.watermarks.a.urgent_ns = (uint32_t)(get_wm_urgent(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(get_wm_stutter_enter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.cstate_exit_ns = (uint32_t)(get_wm_stutter_exit(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.pstate_change_ns = (uint32_t)(get_wm_dram_clock_change(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.pte_meta_urgent_ns = (uint32_t)(get_wm_memory_trip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_nom = (uint32_t)(get_fraction_of_urgent_bandwidth(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.frac_urg_bw_flip = (uint32_t)(get_fraction_of_urgent_bandwidth_imm_flip(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.urgent_latency_ns = (uint32_t)(get_urgent_latency(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.cstate_pstate.fclk_pstate_change_ns = (uint32_t)(get_fclk_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
+		context->bw_ctx.bw.dcn.watermarks.a.usr_retraining_ns = (uint32_t)(get_usr_retraining_watermark(&context->bw_ctx.dml, pipes, pipe_cnt) * 1000);
 	}
 
 	/* Make set D = set A since we do not optimized watermarks for MALL */
@@ -2574,7 +2582,7 @@ void dcn32_calculate_wm_and_dlg_fpu(struct dc *dc, struct dc_state *context,
 		pipe_idx++;
 	}
 
-	context->perf_params.stutter_period_us = context->bw_ctx.dml.vba.StutterPeriod;
+	context->perf_params.stutter_period_us = (unsigned int)context->bw_ctx.dml.vba.StutterPeriod;
 
 	/* for proper prefetch calculations, if dummy lat > fclk lat, use fclk lat = dummy lat */
 	if (need_fclk_lat_as_dummy)
@@ -2608,18 +2616,18 @@ static void dcn32_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts,
 	bw_from_dram = (bw_from_dram1 < bw_from_dram2) ? bw_from_dram1 : bw_from_dram2;
 
 	if (optimal_fclk)
-		*optimal_fclk = bw_from_dram /
-		(dcn3_2_soc.fabric_datapath_to_dcn_data_return_bytes * (dcn3_2_soc.max_avg_sdp_bw_use_normal_percent / 100));
+		*optimal_fclk = (unsigned int)(bw_from_dram /
+		(dcn3_2_soc.fabric_datapath_to_dcn_data_return_bytes * (dcn3_2_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 
 	if (optimal_dcfclk)
-		*optimal_dcfclk =  bw_from_dram /
-		(dcn3_2_soc.return_bus_width_bytes * (dcn3_2_soc.max_avg_sdp_bw_use_normal_percent / 100));
+		*optimal_dcfclk =  (unsigned int)(bw_from_dram /
+		(dcn3_2_soc.return_bus_width_bytes * (dcn3_2_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 }
 
 static void remove_entry_from_table_at_index(struct _vcs_dpi_voltage_scaling_st *table, unsigned int *num_entries,
 		unsigned int index)
 {
-	int i;
+	unsigned int i;
 
 	if (*num_entries == 0)
 		return;
@@ -2660,13 +2668,13 @@ void dcn32_patch_dpm_table(struct clk_bw_params *bw_params)
 	 *  0, will cause it to skip building the clock table.
 	 */
 	if (max_dcfclk_mhz == 0)
-		bw_params->clk_table.entries[0].dcfclk_mhz = dcn3_2_soc.clock_limits[0].dcfclk_mhz;
+		bw_params->clk_table.entries[0].dcfclk_mhz = (unsigned int)dcn3_2_soc.clock_limits[0].dcfclk_mhz;
 	if (max_dispclk_mhz == 0)
-		bw_params->clk_table.entries[0].dispclk_mhz = dcn3_2_soc.clock_limits[0].dispclk_mhz;
+		bw_params->clk_table.entries[0].dispclk_mhz = (unsigned int)dcn3_2_soc.clock_limits[0].dispclk_mhz;
 	if (max_dtbclk_mhz == 0)
-		bw_params->clk_table.entries[0].dtbclk_mhz = dcn3_2_soc.clock_limits[0].dtbclk_mhz;
+		bw_params->clk_table.entries[0].dtbclk_mhz = (unsigned int)dcn3_2_soc.clock_limits[0].dtbclk_mhz;
 	if (max_uclk_mhz == 0)
-		bw_params->clk_table.entries[0].memclk_mhz = dcn3_2_soc.clock_limits[0].dram_speed_mts / 16;
+		bw_params->clk_table.entries[0].memclk_mhz = (unsigned int)(dcn3_2_soc.clock_limits[0].dram_speed_mts / 16);
 }
 
 static void swap_table_entries(struct _vcs_dpi_voltage_scaling_st *first_entry,
@@ -2686,9 +2694,9 @@ static void sort_entries_with_same_bw(struct _vcs_dpi_voltage_scaling_st *table,
 	unsigned int end_index = 0;
 	unsigned int current_bw = 0;
 
-	for (int i = 0; i < (*num_entries - 1); i++) {
+	for (unsigned int i = 0; i < (*num_entries - 1); i++) {
 		if (table[i].net_bw_in_kbytes_sec == table[i+1].net_bw_in_kbytes_sec) {
-			current_bw = table[i].net_bw_in_kbytes_sec;
+			current_bw = (unsigned int)table[i].net_bw_in_kbytes_sec;
 			start_index = i;
 			end_index = ++i;
 
@@ -2697,8 +2705,8 @@ static void sort_entries_with_same_bw(struct _vcs_dpi_voltage_scaling_st *table,
 		}
 
 		if (start_index != end_index) {
-			for (int j = start_index; j < end_index; j++) {
-				for (int k = start_index; k < end_index; k++) {
+			for (unsigned int j = start_index; j < end_index; j++) {
+				for (unsigned int k = start_index; k < end_index; k++) {
 					if (table[k].dcfclk_mhz > table[k+1].dcfclk_mhz)
 						swap_table_entries(&table[k], &table[k+1]);
 				}
@@ -2717,7 +2725,7 @@ static void sort_entries_with_same_bw(struct _vcs_dpi_voltage_scaling_st *table,
  */
 static void remove_inconsistent_entries(struct _vcs_dpi_voltage_scaling_st *table, unsigned int *num_entries)
 {
-	for (int i = 0; i < (*num_entries - 1); i++) {
+	for (unsigned int i = 0; i < (*num_entries - 1); i++) {
 		if (table[i].net_bw_in_kbytes_sec == table[i+1].net_bw_in_kbytes_sec) {
 			if ((table[i].dram_speed_mts > table[i+1].dram_speed_mts) ||
 				(table[i].fabricclk_mhz > table[i+1].fabricclk_mhz))
@@ -2765,7 +2773,8 @@ static int override_max_clk_values(struct clk_limit_table_entry *max_clk_limit,
 static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk_bw_params *bw_params,
 		struct _vcs_dpi_voltage_scaling_st *table, unsigned int *num_entries)
 {
-	int i, j;
+	int i;
+	unsigned int j;
 	struct _vcs_dpi_voltage_scaling_st entry = {0};
 	struct clk_limit_table_entry max_clk_data = {0};
 
@@ -2835,12 +2844,12 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 		max_clk_data.dppclk_mhz = max_clk_data.dispclk_mhz;
 
 	if (max_clk_data.fclk_mhz == 0)
-		max_clk_data.fclk_mhz = max_clk_data.dcfclk_mhz *
+		max_clk_data.fclk_mhz = (unsigned int)(max_clk_data.dcfclk_mhz *
 				dcn3_2_soc.pct_ideal_sdp_bw_after_urgent /
-				dcn3_2_soc.pct_ideal_fabric_bw_after_urgent;
+				dcn3_2_soc.pct_ideal_fabric_bw_after_urgent);
 
 	if (max_clk_data.phyclk_mhz == 0)
-		max_clk_data.phyclk_mhz = dcn3_2_soc.clock_limits[0].phyclk_mhz;
+		max_clk_data.phyclk_mhz = (unsigned int)dcn3_2_soc.clock_limits[0].phyclk_mhz;
 
 	*num_entries = 0;
 	entry.dispclk_mhz = max_clk_data.dispclk_mhz;
@@ -2852,8 +2861,8 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 	entry.phyclk_d32_mhz = dcn3_2_soc.clock_limits[0].phyclk_d32_mhz;
 
 	// Insert all the DCFCLK STAs
-	for (i = 0; i < num_dcfclk_stas; i++) {
-		entry.dcfclk_mhz = dcfclk_sta_targets[i];
+	for (j = 0; j < num_dcfclk_stas; j++) {
+		entry.dcfclk_mhz = dcfclk_sta_targets[j];
 		entry.fabricclk_mhz = 0;
 		entry.dram_speed_mts = 0;
 
@@ -2872,10 +2881,10 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 	insert_entry_into_table_sorted(table, num_entries, &entry);
 
 	// Insert the UCLK DPMS
-	for (i = 0; i < num_uclk_dpms; i++) {
+	for (j = 0; j < num_uclk_dpms; j++) {
 		entry.dcfclk_mhz = 0;
 		entry.fabricclk_mhz = 0;
-		entry.dram_speed_mts = bw_params->clk_table.entries[i].memclk_mhz * 16;
+		entry.dram_speed_mts = bw_params->clk_table.entries[j].memclk_mhz * 16;
 
 		get_optimal_ntuple(&entry);
 		entry.net_bw_in_kbytes_sec = calculate_net_bw_in_kbytes_sec(&entry);
@@ -2884,9 +2893,9 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 
 	// If FCLK is coarse grained, insert individual DPMs.
 	if (num_fclk_dpms > 2) {
-		for (i = 0; i < num_fclk_dpms; i++) {
+		for (j = 0; j < num_fclk_dpms; j++) {
 			entry.dcfclk_mhz = 0;
-			entry.fabricclk_mhz = bw_params->clk_table.entries[i].fclk_mhz;
+			entry.fabricclk_mhz = bw_params->clk_table.entries[j].fclk_mhz;
 			entry.dram_speed_mts = 0;
 
 			get_optimal_ntuple(&entry);
@@ -2976,7 +2985,7 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 
 	// Remove duplicate states, note duplicate states are always neighbouring since table is sorted.
 	i = 0;
-	while (i < *num_entries - 1) {
+	while (i < ((int)*num_entries - 1)) {
 		if (table[i].dcfclk_mhz == table[i + 1].dcfclk_mhz &&
 				table[i].fabricclk_mhz == table[i + 1].fabricclk_mhz &&
 				table[i].dram_speed_mts == table[i + 1].dram_speed_mts)
@@ -3093,9 +3102,12 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa
 			dc->ctx->dc_bios->vram_info.num_chans) * dc->caps.mall_size_per_mem_channel);
 	}
 
-	if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes)
-		dc->dml2_options.bbox_overrides.dram_chanel_width_bytes =
-		dcn3_2_soc.dram_channel_width_bytes = dc->ctx->dc_bios->vram_info.dram_channel_width_bytes;
+	if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes) {
+		unsigned int dram_channel_width_bytes = (unsigned int)dc->ctx->dc_bios->vram_info.dram_channel_width_bytes;
+
+		dc->dml2_options.bbox_overrides.dram_chanel_width_bytes = dram_channel_width_bytes;
+		dcn3_2_soc.dram_channel_width_bytes = dram_channel_width_bytes;
+	}
 
 	/* DML DSC delay factor workaround */
 	dcn3_2_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0;
@@ -3142,13 +3154,13 @@ void dcn32_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_pa
 			if (min_dcfclk > dcfclk_sta_targets[0])
 				dcfclk_sta_targets[0] = min_dcfclk;
 			if (!max_dcfclk_mhz)
-				max_dcfclk_mhz = dcn3_2_soc.clock_limits[0].dcfclk_mhz;
+				max_dcfclk_mhz = (unsigned int)dcn3_2_soc.clock_limits[0].dcfclk_mhz;
 			if (!max_dispclk_mhz)
-				max_dispclk_mhz = dcn3_2_soc.clock_limits[0].dispclk_mhz;
+				max_dispclk_mhz = (unsigned int)dcn3_2_soc.clock_limits[0].dispclk_mhz;
 			if (!max_dppclk_mhz)
-				max_dppclk_mhz = dcn3_2_soc.clock_limits[0].dppclk_mhz;
+				max_dppclk_mhz = (unsigned int)dcn3_2_soc.clock_limits[0].dppclk_mhz;
 			if (!max_phyclk_mhz)
-				max_phyclk_mhz = dcn3_2_soc.clock_limits[0].phyclk_mhz;
+				max_phyclk_mhz = (unsigned int)dcn3_2_soc.clock_limits[0].phyclk_mhz;
 
 			if (max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
 				// If max DCFCLK is greater than the max DCFCLK STA target, insert into the DCFCLK STA target array
@@ -3365,8 +3377,8 @@ bool dcn32_allow_subvp_with_active_margin(struct pipe_ctx *pipe)
 
 		refresh_rate = (pipe->stream->timing.pix_clk_100hz * (uint64_t)100 +
 			(uint64_t)pipe->stream->timing.v_total * pipe->stream->timing.h_total - (uint64_t)1);
-		refresh_rate = div_u64(refresh_rate, pipe->stream->timing.v_total);
-		refresh_rate = div_u64(refresh_rate, pipe->stream->timing.h_total);
+		refresh_rate = (uint32_t)div_u64(refresh_rate, pipe->stream->timing.v_total);
+		refresh_rate = (uint32_t)div_u64(refresh_rate, pipe->stream->timing.h_total);
 
 		if (refresh_rate >= min_refresh && refresh_rate <= max_refresh &&
 				dcn32_check_native_scaling_for_res(pipe, width, height)) {
@@ -3411,9 +3423,9 @@ bool dcn32_allow_subvp_high_refresh_rate(struct dc *dc, struct dc_state *context
 
 		if (!pipe_ctx->stream)
 			continue;
-		refresh_rate = (pipe_ctx->stream->timing.pix_clk_100hz * 100 +
+		refresh_rate = (unsigned int)((pipe_ctx->stream->timing.pix_clk_100hz * 100 +
 				pipe_ctx->stream->timing.v_total * pipe_ctx->stream->timing.h_total - 1)
-						/ (double)(pipe_ctx->stream->timing.v_total * pipe_ctx->stream->timing.h_total);
+						/ (double)(pipe_ctx->stream->timing.v_total * pipe_ctx->stream->timing.h_total));
 
 		if (refresh_rate < min_refresh)
 			min_refresh = refresh_rate;
@@ -3421,9 +3433,9 @@ bool dcn32_allow_subvp_high_refresh_rate(struct dc *dc, struct dc_state *context
 
 	if (!dc->debug.disable_subvp_high_refresh && min_refresh >= subvp_min_refresh && pipe->stream &&
 			pipe->plane_state && !(pipe->stream->vrr_active_variable || pipe->stream->vrr_active_fixed)) {
-		refresh_rate = (pipe->stream->timing.pix_clk_100hz * 100 +
+		refresh_rate = (unsigned int)((pipe->stream->timing.pix_clk_100hz * 100 +
 						pipe->stream->timing.v_total * pipe->stream->timing.h_total - 1)
-						/ (double)(pipe->stream->timing.v_total * pipe->stream->timing.h_total);
+						/ (double)(pipe->stream->timing.v_total * pipe->stream->timing.h_total));
 		if (refresh_rate >= subvp_min_refresh && refresh_rate <= subvp_max_refresh) {
 			for (i = 0; i < SUBVP_HIGH_REFRESH_LIST_LEN; i++) {
 				uint32_t width = subvp_high_refresh_list.res[i].width;
@@ -3543,8 +3555,8 @@ bool dcn32_find_vactive_pipe(struct dc *dc, const struct dc_state *context, stru
 		/* Every plane (apart from the ones driven by the FPO pipes) needs to have active margin
 		 * in order for us to have found a valid "vactive" config for FPO + Vactive
 		 */
-		blank_us = ((pipe->stream->timing.v_total - pipe->stream->timing.v_addressable) * pipe->stream->timing.h_total /
-				(double)(pipe->stream->timing.pix_clk_100hz * 100)) * 1000000;
+		blank_us = (unsigned int)(((pipe->stream->timing.v_total - pipe->stream->timing.v_addressable) * pipe->stream->timing.h_total /
+				(double)(pipe->stream->timing.pix_clk_100hz * 100)) * 1000000);
 		if (vba->ActiveDRAMClockChangeLatencyMarginPerState[vba->VoltageLevel][vba->maxMpcComb][vba->pipe_plane[pipe_idx]] < vactive_margin_req_us ||
 				pipe->stream->vrr_active_variable || pipe->stream->vrr_active_fixed || blank_us >= dc->debug.fpo_vactive_max_blank_us) {
 			vactive_found = false;
@@ -3572,7 +3584,7 @@ void dcn32_override_min_req_memclk(struct dc *dc, struct dc_state *context)
 		if (context->bw_ctx.dml.vba.DRAMSpeed <= dc->clk_mgr->bw_params->clk_table.entries[0].memclk_mhz * 16 &&
 				num_mclk_levels > 1) {
 			context->bw_ctx.dml.vba.DRAMSpeed = dc->clk_mgr->bw_params->clk_table.entries[1].memclk_mhz * 16;
-			context->bw_ctx.bw.dcn.clk.dramclk_khz = context->bw_ctx.dml.vba.DRAMSpeed * 1000 / 16;
+			context->bw_ctx.bw.dcn.clk.dramclk_khz = (int)(context->bw_ctx.dml.vba.DRAMSpeed * 1000 / 16);
 		}
 	}
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c
index 0782a34689a00..505b5a5f212e1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_32.c
@@ -92,8 +92,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 							mode_lib->vba.WritebackHTaps[k],
 							mode_lib->vba.WritebackVTaps[k],
 							mode_lib->vba.WritebackSourceWidth[k],
-							mode_lib->vba.WritebackDestinationWidth[k],
-							mode_lib->vba.HTotal[k], mode_lib->vba.WritebackLineBufferSize,
+							(long)mode_lib->vba.WritebackDestinationWidth[k],
+							mode_lib->vba.HTotal[k], (unsigned int)mode_lib->vba.WritebackLineBufferSize,
 							mode_lib->vba.DISPCLKDPPCLKVCOSpeed));
 		}
 	}
@@ -305,8 +305,8 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					 .dummy_boolean); /* bool *ViewportSizeSupport */
 	}
 
-	v->CompBufReservedSpaceZs     = v->CompBufReservedSpaceKBytes * 1024.0 / 256.0;
-	v->CompBufReservedSpace64B    = v->CompBufReservedSpaceKBytes * 1024.0 / 64.0;
+	v->CompBufReservedSpaceZs     = (unsigned int)(v->CompBufReservedSpaceKBytes * 1024.0 / 256.0);
+	v->CompBufReservedSpace64B    = (unsigned int)(v->CompBufReservedSpaceKBytes * 1024.0 / 64.0);
 
 	// DCFCLK Deep Sleep
 	dml32_CalculateDCFCLKDeepSleep(
@@ -326,7 +326,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			mode_lib->vba.DPPCLK,
 			v->ReadBandwidthSurfaceLuma,
 			v->ReadBandwidthSurfaceChroma,
-			mode_lib->vba.ReturnBusWidth,
+			(int)mode_lib->vba.ReturnBusWidth,
 
 			/* Output */
 			&v->DCFCLKDeepSleep);
@@ -479,7 +479,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 				mode_lib->vba.HostVMMaxNonCachedPageTableLevels,
 				mode_lib->vba.GPUVMMaxPageTableLevels,
 				mode_lib->vba.GPUVMMinPageSizeKBytes,
-				mode_lib->vba.HostVMMinPageSize,
+				(unsigned int)mode_lib->vba.HostVMMinPageSize,
 
 				/* Output */
 				v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.dummy_boolean_array2[0],  // Boolean PTEBufferSizeNotExceeded[]
@@ -528,12 +528,12 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 	}
 
 
-	v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.ReorderBytes = mode_lib->vba.NumberOfChannels
+	v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.ReorderBytes = (unsigned int)(mode_lib->vba.NumberOfChannels
 			* dml_max3(mode_lib->vba.UrgentOutOfOrderReturnPerChannelPixelDataOnly,
 					mode_lib->vba.UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData,
-					mode_lib->vba.UrgentOutOfOrderReturnPerChannelVMDataOnly);
+					mode_lib->vba.UrgentOutOfOrderReturnPerChannelVMDataOnly));
 
-	v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.VMDataOnlyReturnBW = dml32_get_return_bw_mbps_vm_only(
+	v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.VMDataOnlyReturnBW = (unsigned int)dml32_get_return_bw_mbps_vm_only(
 			&mode_lib->vba.soc,
 			mode_lib->vba.VoltageLevel,
 			mode_lib->vba.DCFCLK,
@@ -577,7 +577,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 	}
 
 	v->UrgentExtraLatency = dml32_CalculateExtraLatency(
-			mode_lib->vba.RoundTripPingLatencyCycles,
+			(int)mode_lib->vba.RoundTripPingLatencyCycles,
 			v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.ReorderBytes,
 			mode_lib->vba.DCFCLK,
 			mode_lib->vba.TotalActiveDPP,
@@ -591,7 +591,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			mode_lib->vba.DPPPerPlane,
 			v->dpte_group_bytes,
 			v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.HostVMInefficiencyFactor,
-			mode_lib->vba.HostVMMinPageSize,
+			(unsigned int)mode_lib->vba.HostVMMinPageSize,
 			mode_lib->vba.HostVMMaxNonCachedPageTableLevels);
 
 	mode_lib->vba.TCalc = 24.0 / v->DCFCLKDeepSleep;
@@ -605,9 +605,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 								mode_lib->vba.WritebackHRatio[k],
 								mode_lib->vba.WritebackVRatio[k],
 								mode_lib->vba.WritebackVTaps[k],
-								mode_lib->vba.WritebackDestinationWidth[k],
-								mode_lib->vba.WritebackDestinationHeight[k],
-								mode_lib->vba.WritebackSourceHeight[k],
+								(long)mode_lib->vba.WritebackDestinationWidth[k],
+								(int)mode_lib->vba.WritebackDestinationHeight[k],
+								(int)mode_lib->vba.WritebackSourceHeight[k],
 								mode_lib->vba.HTotal[k]) / mode_lib->vba.DISPCLK;
 			} else
 				v->WritebackDelay[mode_lib->vba.VoltageLevel][k] = 0;
@@ -622,9 +622,9 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 								mode_lib->vba.WritebackHRatio[j],
 								mode_lib->vba.WritebackVRatio[j],
 								mode_lib->vba.WritebackVTaps[j],
-								mode_lib->vba.WritebackDestinationWidth[j],
-								mode_lib->vba.WritebackDestinationHeight[j],
-								mode_lib->vba.WritebackSourceHeight[j],
+								(int)mode_lib->vba.WritebackDestinationWidth[j],
+								(int)mode_lib->vba.WritebackDestinationHeight[j],
+								(int)mode_lib->vba.WritebackSourceHeight[j],
 								mode_lib->vba.HTotal[k]) / mode_lib->vba.DISPCLK);
 				}
 			}
@@ -693,12 +693,12 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 						mode_lib->vba.UseUnboundedRequesting);
 
 	for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) {
-		v->MaxVStartupLines[k] = ((mode_lib->vba.Interlace[k] &&
+		v->MaxVStartupLines[k] = (unsigned int)(((mode_lib->vba.Interlace[k] &&
 				!mode_lib->vba.ProgressiveToInterlaceUnitInOPP) ?
 				dml_floor((mode_lib->vba.VTotal[k] - mode_lib->vba.VActive[k]) / 2.0, 1.0) :
 				mode_lib->vba.VTotal[k] - mode_lib->vba.VActive[k]) - dml_max(1.0,
 				dml_ceil((double) v->WritebackDelay[mode_lib->vba.VoltageLevel][k]
-				/ (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]), 1));
+				/ (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]), 1)));
 
 		// Clamp to max OTG vstartup register limit
 		if (v->MaxVStartupLines[k] > 1023)
@@ -714,7 +714,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 
 	v->MaximumMaxVStartupLines = 0;
 	for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k)
-		v->MaximumMaxVStartupLines = dml_max(v->MaximumMaxVStartupLines, v->MaxVStartupLines[k]);
+		v->MaximumMaxVStartupLines = (unsigned int)dml_max(v->MaximumMaxVStartupLines, v->MaxVStartupLines[k]);
 
 	ImmediateFlipRequirementFinal = false;
 
@@ -789,21 +789,21 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 					&v->dummy_vars.DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerformanceCalculation.myPipe,
 					v->DSCDelay[k],
 					(unsigned int) (v->SwathWidthY[k] / v->HRatio[k]),
-					dml_min(v->VStartupLines, v->MaxVStartupLines[k]),
+					(unsigned int)dml_min(v->VStartupLines, v->MaxVStartupLines[k]),
 					v->MaxVStartupLines[k],
 					v->UrgentLatency,
 					v->UrgentExtraLatency,
 					v->TCalc,
-					v->PDEAndMetaPTEBytesFrame[k],
-					v->MetaRowByte[k],
-					v->PixelPTEBytesPerRow[k],
+					(unsigned int)v->PDEAndMetaPTEBytesFrame[k],
+					(unsigned int)v->MetaRowByte[k],
+					(unsigned int)v->PixelPTEBytesPerRow[k],
 					v->PrefetchSourceLinesY[k],
-					v->SwathWidthY[k],
-					v->VInitPreFillY[k],
+					(unsigned int)v->SwathWidthY[k],
+					(unsigned int)v->VInitPreFillY[k],
 					v->MaxNumSwathY[k],
 					v->PrefetchSourceLinesC[k],
-					v->SwathWidthC[k],
-					v->VInitPreFillC[k],
+					(unsigned int)v->SwathWidthC[k],
+					(unsigned int)v->VInitPreFillC[k],
 					v->MaxNumSwathC[k],
 					v->swath_width_luma_ub[k],
 					v->swath_width_chroma_ub[k],
@@ -839,7 +839,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			dml_print("DML::%s: k=%0d Prefetch calculation errResult=%0d\n",
 					__func__, k, mode_lib->vba.ErrorResult[k]);
 #endif
-			v->VStartup[k] = dml_min(v->VStartupLines, v->MaxVStartupLines[k]);
+			v->VStartup[k] = (unsigned int)dml_min(v->VStartupLines, v->MaxVStartupLines[k]);
 		}
 
 		for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) {
@@ -1019,18 +1019,18 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 			mode_lib->vba.TotImmediateFlipBytes = 0;
 			for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) {
 				if (mode_lib->vba.ImmediateFlipRequirement[k] != dm_immediate_flip_not_required) {
-					mode_lib->vba.TotImmediateFlipBytes = mode_lib->vba.TotImmediateFlipBytes
+					mode_lib->vba.TotImmediateFlipBytes = (unsigned int)(mode_lib->vba.TotImmediateFlipBytes
 							+ mode_lib->vba.DPPPerPlane[k]
 									* (v->PDEAndMetaPTEBytesFrame[k]
-											+ v->MetaRowByte[k]);
+											+ v->MetaRowByte[k]));
 					if (v->use_one_row_for_frame_flip[k][0][0]) {
 						mode_lib->vba.TotImmediateFlipBytes =
-								mode_lib->vba.TotImmediateFlipBytes
-										+ 2 * v->PixelPTEBytesPerRow[k];
+								(unsigned int)(mode_lib->vba.TotImmediateFlipBytes
+										+ 2 * v->PixelPTEBytesPerRow[k]);
 					} else {
 						mode_lib->vba.TotImmediateFlipBytes =
-								mode_lib->vba.TotImmediateFlipBytes
-										+ v->PixelPTEBytesPerRow[k];
+								(unsigned int)(mode_lib->vba.TotImmediateFlipBytes
+										+ v->PixelPTEBytesPerRow[k]);
 					}
 				}
 			}
@@ -1042,10 +1042,10 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 						mode_lib->vba.HostVMEnable,
 						mode_lib->vba.HostVMMaxNonCachedPageTableLevels,
 						mode_lib->vba.GPUVMEnable,
-						mode_lib->vba.HostVMMinPageSize,
-						v->PDEAndMetaPTEBytesFrame[k],
-						v->MetaRowByte[k],
-						v->PixelPTEBytesPerRow[k],
+						(unsigned int)mode_lib->vba.HostVMMinPageSize,
+						(unsigned int)v->PDEAndMetaPTEBytesFrame[k],
+						(unsigned int)v->MetaRowByte[k],
+						(unsigned int)v->PixelPTEBytesPerRow[k],
 						mode_lib->vba.BandwidthAvailableForImmediateFlip,
 						mode_lib->vba.TotImmediateFlipBytes,
 						mode_lib->vba.SourcePixelFormat[k],
@@ -1439,14 +1439,14 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman
 		isInterlaceTiming = (mode_lib->vba.Interlace[k] &&
 				!mode_lib->vba.ProgressiveToInterlaceUnitInOPP);
 
-		v->MIN_DST_Y_NEXT_START[k] = ((isInterlaceTiming ? dml_floor((mode_lib->vba.VTotal[k] -
+		v->MIN_DST_Y_NEXT_START[k] = (unsigned int)(((isInterlaceTiming ? dml_floor((mode_lib->vba.VTotal[k] -
 						mode_lib->vba.VFrontPorch[k]) / 2.0, 1.0) :
 						mode_lib->vba.VTotal[k]) - mode_lib->vba.VFrontPorch[k])
 						+ dml_max(1.0,
 						dml_ceil(v->WritebackDelay[mode_lib->vba.VoltageLevel][k]
 						/ (mode_lib->vba.HTotal[k] / mode_lib->vba.PixelClock[k]), 1.0))
 						+ dml_floor(4.0 * v->TSetup[k] / (mode_lib->vba.HTotal[k]
-						/ mode_lib->vba.PixelClock[k]), 1.0) / 4.0;
+						/ mode_lib->vba.PixelClock[k]), 1.0) / 4.0);
 
 		v->VStartup[k] = (isInterlaceTiming ? (2 * v->MaxVStartupLines[k]) : v->MaxVStartupLines[k]);
 
@@ -1724,7 +1724,8 @@ static void mode_support_configuration(struct vba_vars_st *v,
 void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_lib)
 {
 	struct vba_vars_st *v = &mode_lib->vba;
-	int i, j, start_state;
+	int i, start_state;
+	unsigned int j;
 	unsigned int k, m;
 	unsigned int MaximumMPCCombine;
 	unsigned int NumberOfNonCombinedSurfaceOfMaximumBandwidth;
@@ -1931,7 +1932,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 
 		if (mode_lib->vba.SourcePixelFormat[k] == dm_420_8 || mode_lib->vba.SourcePixelFormat[k] == dm_420_10
 				|| mode_lib->vba.SourcePixelFormat[k] == dm_420_12) {
-			v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.MaximumSwathWidthSupportChroma = v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.MaximumSwathWidthSupportLuma / 2.0;
+			v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.MaximumSwathWidthSupportChroma = (unsigned int)(v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.MaximumSwathWidthSupportLuma / 2.0);
 		} else {
 			v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.MaximumSwathWidthSupportChroma = v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.MaximumSwathWidthSupportLuma;
 		}
@@ -2030,7 +2031,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	mode_lib->vba.MPCCombineMethodIncompatible = v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.MPCCombineMethodAsNeededForPStateChangeAndVoltage
 			&& v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.MPCCombineMethodAsPossible;
 
-	for (i = start_state; i < v->soc.num_states; i++) {
+	for (i = start_state; i < (int)v->soc.num_states; i++) {
 		for (j = 0; j < 2; j++) {
 			mode_lib->vba.TotalNumberOfActiveDPP[i][j] = 0;
 			mode_lib->vba.TotalAvailablePipesSupport[i][j] = true;
@@ -2103,7 +2104,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.ODMModeNoDSC,
 						v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.ODMModeDSC,
 						mode_lib->vba.DSCEnable[k],
-						mode_lib->vba.OutputLinkDPLanes[k],
+						(int)mode_lib->vba.OutputLinkDPLanes[k],
 						mode_lib->vba.OutputLinkDPRate[k],
 
 						/* Output */
@@ -2237,9 +2238,9 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 									mode_lib->vba.WritebackHTaps[k],
 									mode_lib->vba.WritebackVTaps[k],
 									mode_lib->vba.WritebackSourceWidth[k],
-									mode_lib->vba.WritebackDestinationWidth[k],
+									(long)mode_lib->vba.WritebackDestinationWidth[k],
 									mode_lib->vba.HTotal[k],
-									mode_lib->vba.WritebackLineBufferSize,
+									(unsigned int)mode_lib->vba.WritebackLineBufferSize,
 									mode_lib->vba.DISPCLKDPPCLKVCOSpeed));
 				}
 			}
@@ -2307,7 +2308,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		}
 	}
 
-	for (i = start_state; i < v->soc.num_states; ++i) {
+	for (i = start_state; i < (int)v->soc.num_states; ++i) {
 		mode_lib->vba.ExceededMultistreamSlots[i] = false;
 		for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) {
 			if (mode_lib->vba.OutputMultistreamEn[k] == true && mode_lib->vba.OutputMultistreamId[k] == k) {
@@ -2410,7 +2411,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		}
 	}
 
-	for (i = start_state; i < v->soc.num_states; ++i) {
+	for (i = start_state; i < (int)v->soc.num_states; ++i) {
 		mode_lib->vba.DTBCLKRequiredMoreThanSupported[i] = false;
 		for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) {
 			if (mode_lib->vba.BlendingAndTiming[k] == k
@@ -2427,7 +2428,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		}
 	}
 
-	for (i = start_state; i < v->soc.num_states; ++i) {
+	for (i = start_state; i < (int)v->soc.num_states; ++i) {
 		mode_lib->vba.ODMCombine2To1SupportCheckOK[i] = true;
 		mode_lib->vba.ODMCombine4To1SupportCheckOK[i] = true;
 		for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) {
@@ -2445,7 +2446,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		}
 	}
 
-	for (i = start_state; i < v->soc.num_states; i++) {
+	for (i = start_state; i < (int)v->soc.num_states; i++) {
 		mode_lib->vba.DSCCLKRequiredMoreThanSupported[i] = false;
 		for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) {
 			if (mode_lib->vba.BlendingAndTiming[k] == k) {
@@ -2482,7 +2483,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	/* Check DSC Unit and Slices Support */
 	v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.TotalDSCUnitsRequired = 0;
 
-	for (i = start_state; i < v->soc.num_states; ++i) {
+	for (i = start_state; i < (int)v->soc.num_states; ++i) {
 		mode_lib->vba.NotEnoughDSCUnits[i] = false;
 		mode_lib->vba.NotEnoughDSCSlices[i] = false;
 		v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.TotalDSCUnitsRequired = 0;
@@ -2517,7 +2518,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 	}
 
 	/*DSC Delay per state*/
-	for (i = start_state; i < v->soc.num_states; ++i) {
+	for (i = start_state; i < (int)v->soc.num_states; ++i) {
 		for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k) {
 			mode_lib->vba.DSCDelayPerState[i][k] = dml32_DSCDelayRequirement(
 					mode_lib->vba.RequiresDSC[i][k], mode_lib->vba.ODMCombineEnablePerState[i][k],
@@ -2531,7 +2532,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 
 		for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) {
 			for (m = 0; m <= mode_lib->vba.NumberOfActiveSurfaces - 1; m++) {
-				for (j = 0; j <= mode_lib->vba.NumberOfActiveSurfaces - 1; j++) {
+				for (j = 0; j < mode_lib->vba.NumberOfActiveSurfaces; j++) {
 					if (mode_lib->vba.BlendingAndTiming[k] == m &&
 							mode_lib->vba.RequiresDSC[i][m] == true) {
 						mode_lib->vba.DSCDelayPerState[i][k] =
@@ -2622,8 +2623,8 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						mode_lib->vba.swath_width_luma_ub_this_state[k];
 				mode_lib->vba.swath_width_chroma_ub_all_states[i][j][k] =
 						mode_lib->vba.swath_width_chroma_ub_this_state[k];
-				mode_lib->vba.SwathWidthYAllStates[i][j][k] = mode_lib->vba.SwathWidthYThisState[k];
-				mode_lib->vba.SwathWidthCAllStates[i][j][k] = mode_lib->vba.SwathWidthCThisState[k];
+				mode_lib->vba.SwathWidthYAllStates[i][j][k] = (unsigned int)mode_lib->vba.SwathWidthYThisState[k];
+				mode_lib->vba.SwathWidthCAllStates[i][j][k] = (unsigned int)mode_lib->vba.SwathWidthCThisState[k];
 				mode_lib->vba.SwathHeightYAllStates[i][j][k] = mode_lib->vba.SwathHeightYThisState[k];
 				mode_lib->vba.SwathHeightCAllStates[i][j][k] = mode_lib->vba.SwathHeightCThisState[k];
 				mode_lib->vba.UnboundedRequestEnabledAllStates[i][j] =
@@ -2682,7 +2683,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 			mode_lib->vba.SurfaceSizeInMALL,
 			&mode_lib->vba.ExceededMALLSize);
 
-	for (i = start_state; i < v->soc.num_states; i++) {
+	for (i = start_state; i < (int)v->soc.num_states; i++) {
 		for (j = 0; j < 2; j++) {
 			for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) {
 				mode_lib->vba.swath_width_luma_ub_this_state[k] =
@@ -2696,9 +2697,9 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				mode_lib->vba.DETBufferSizeInKByteThisState[k] =
 						mode_lib->vba.DETBufferSizeInKByteAllStates[i][j][k];
 				mode_lib->vba.DETBufferSizeYThisState[k] =
-						mode_lib->vba.DETBufferSizeYAllStates[i][j][k];
+						(unsigned int)mode_lib->vba.DETBufferSizeYAllStates[i][j][k];
 				mode_lib->vba.DETBufferSizeCThisState[k] =
-						mode_lib->vba.DETBufferSizeCAllStates[i][j][k];
+						(unsigned int)mode_lib->vba.DETBufferSizeCAllStates[i][j][k];
 				mode_lib->vba.RequiredDPPCLKThisState[k] = mode_lib->vba.RequiredDPPCLK[i][j][k];
 				mode_lib->vba.NoOfDPPThisState[k] = mode_lib->vba.NoOfDPP[i][j][k];
 			}
@@ -2772,7 +2773,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						mode_lib->vba.HostVMMaxNonCachedPageTableLevels,
 						mode_lib->vba.GPUVMMaxPageTableLevels,
 						mode_lib->vba.GPUVMMinPageSizeKBytes,
-						mode_lib->vba.HostVMMinPageSize,
+						(unsigned int)mode_lib->vba.HostVMMinPageSize,
 
 						/* Output */
 						mode_lib->vba.PTEBufferSizeNotExceededPerState,
@@ -2901,7 +2902,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 					mode_lib->vba.RequiredDPPCLKThisState,
 					mode_lib->vba.ReadBandwidthLuma,
 					mode_lib->vba.ReadBandwidthChroma,
-					mode_lib->vba.ReturnBusWidth,
+					(int)mode_lib->vba.ReturnBusWidth,
 
 					/* Output */
 					&mode_lib->vba.ProjectedDCFCLKDeepSleep[i][j]);
@@ -2921,9 +2922,9 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							mode_lib->vba.WritebackHRatio[k],
 							mode_lib->vba.WritebackVRatio[k],
 							mode_lib->vba.WritebackVTaps[k],
-							mode_lib->vba.WritebackDestinationWidth[k],
-							mode_lib->vba.WritebackDestinationHeight[k],
-							mode_lib->vba.WritebackSourceHeight[k],
+							(long)mode_lib->vba.WritebackDestinationWidth[k],
+							(int)mode_lib->vba.WritebackDestinationHeight[k],
+							(int)mode_lib->vba.WritebackSourceHeight[k],
 							mode_lib->vba.HTotal[k])
 							/ mode_lib->vba.RequiredDISPCLK[i][j];
 					} else {
@@ -2940,9 +2941,9 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 									mode_lib->vba.WritebackHRatio[m],
 									mode_lib->vba.WritebackVRatio[m],
 									mode_lib->vba.WritebackVTaps[m],
-									mode_lib->vba.WritebackDestinationWidth[m],
-									mode_lib->vba.WritebackDestinationHeight[m],
-									mode_lib->vba.WritebackSourceHeight[m],
+									(int)mode_lib->vba.WritebackDestinationWidth[m],
+									(int)mode_lib->vba.WritebackDestinationHeight[m],
+									(int)mode_lib->vba.WritebackSourceHeight[m],
 									mode_lib->vba.HTotal[m]) /
 									mode_lib->vba.RequiredDISPCLK[i][j]);
 						}
@@ -2979,10 +2980,10 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		}
 	}
 
-	v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.ReorderingBytes = mode_lib->vba.NumberOfChannels
+	v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.ReorderingBytes = (unsigned int)(mode_lib->vba.NumberOfChannels
 			* dml_max3(mode_lib->vba.UrgentOutOfOrderReturnPerChannelPixelDataOnly,
 					mode_lib->vba.UrgentOutOfOrderReturnPerChannelPixelMixedWithVMData,
-					mode_lib->vba.UrgentOutOfOrderReturnPerChannelVMDataOnly);
+					mode_lib->vba.UrgentOutOfOrderReturnPerChannelVMDataOnly));
 
 	dml32_CalculateMinAndMaxPrefetchMode(mode_lib->vba.AllowForPStateChangeOrStutterInVBlankFinal,
 			&mode_lib->vba.MinPrefetchMode,
@@ -3060,8 +3061,8 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				mode_lib->vba.DRAMClockChangeLatency,
 				mode_lib->vba.FCLKChangeLatency,
 				mode_lib->vba.SREnterPlusExitTime,
-				mode_lib->vba.ReturnBusWidth,
-				mode_lib->vba.RoundTripPingLatencyCycles,
+				(int)mode_lib->vba.ReturnBusWidth,
+				(int)mode_lib->vba.RoundTripPingLatencyCycles,
 				v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.ReorderingBytes,
 				mode_lib->vba.PixelChunkSizeInKByte,
 				mode_lib->vba.MetaChunkSize,
@@ -3069,7 +3070,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				mode_lib->vba.GPUVMMaxPageTableLevels,
 				mode_lib->vba.HostVMEnable,
 				mode_lib->vba.NumberOfActiveSurfaces,
-				mode_lib->vba.HostVMMinPageSize,
+				(unsigned int)mode_lib->vba.HostVMMinPageSize,
 				mode_lib->vba.HostVMMaxNonCachedPageTableLevels,
 				mode_lib->vba.DynamicMetadataVMEnabled,
 				mode_lib->vba.ImmediateFlipRequiredFinal,
@@ -3187,9 +3188,9 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 				mode_lib->vba.DETBufferSizeInKByteThisState[k] =
 						mode_lib->vba.DETBufferSizeInKByteAllStates[i][j][k];
 				mode_lib->vba.DETBufferSizeYThisState[k] =
-						mode_lib->vba.DETBufferSizeYAllStates[i][j][k];
+						(unsigned int)mode_lib->vba.DETBufferSizeYAllStates[i][j][k];
 				mode_lib->vba.DETBufferSizeCThisState[k] =
-						mode_lib->vba.DETBufferSizeCAllStates[i][j][k];
+						(unsigned int)mode_lib->vba.DETBufferSizeCAllStates[i][j][k];
 			}
 
 			mode_lib->vba.VActiveBandwithSupport[i][j] = dml32_CalculateVActiveBandwithSupport(
@@ -3236,14 +3237,14 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 						/ v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.VMDataOnlyReturnBWPerState;
 
 			mode_lib->vba.ExtraLatency = dml32_CalculateExtraLatency(
-					mode_lib->vba.RoundTripPingLatencyCycles, v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.ReorderingBytes,
+					(int)mode_lib->vba.RoundTripPingLatencyCycles, v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.ReorderingBytes,
 					mode_lib->vba.DCFCLKState[i][j], mode_lib->vba.TotalNumberOfActiveDPP[i][j],
 					mode_lib->vba.PixelChunkSizeInKByte,
 					mode_lib->vba.TotalNumberOfDCCActiveDPP[i][j], mode_lib->vba.MetaChunkSize,
 					mode_lib->vba.ReturnBWPerState[i][j], mode_lib->vba.GPUVMEnable,
 					mode_lib->vba.HostVMEnable, mode_lib->vba.NumberOfActiveSurfaces,
 					mode_lib->vba.NoOfDPPThisState, mode_lib->vba.dpte_group_bytes,
-					v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.HostVMInefficiencyFactor, mode_lib->vba.HostVMMinPageSize,
+					v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.HostVMInefficiencyFactor, (unsigned int)mode_lib->vba.HostVMMinPageSize,
 					mode_lib->vba.HostVMMaxNonCachedPageTableLevels);
 
 			v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.NextPrefetchModeState = mode_lib->vba.MinPrefetchMode;
@@ -3295,23 +3296,23 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							k,
 							v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.HostVMInefficiencyFactor,
 							&v->dummy_vars.dml32_ModeSupportAndSystemConfigurationFull.myPipe,
-							v->DSCDelayPerState[i][k],
-							v->SwathWidthYThisState[k] / v->HRatio[k],
-							dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]),
-							v->MaximumVStartup[i][j][k],
+							(unsigned int)v->DSCDelayPerState[i][k],
+							(unsigned int)(v->SwathWidthYThisState[k] / v->HRatio[k]),
+							(unsigned int)dml_min(v->MaxVStartup, v->MaximumVStartup[i][j][k]),
+							(unsigned int)v->MaximumVStartup[i][j][k],
 							v->UrgLatency[i],
 							v->ExtraLatency,
 							v->TimeCalc,
-							v->PDEAndMetaPTEBytesPerFrame[i][j][k],
-							v->MetaRowBytes[i][j][k],
-							v->DPTEBytesPerRow[i][j][k],
+							(unsigned int)v->PDEAndMetaPTEBytesPerFrame[i][j][k],
+							(unsigned int)v->MetaRowBytes[i][j][k],
+							(unsigned int)v->DPTEBytesPerRow[i][j][k],
 							v->PrefetchLinesY[i][j][k],
-							v->SwathWidthYThisState[k],
-							v->PrefillY[k],
+							(unsigned int)v->SwathWidthYThisState[k],
+							(unsigned int)v->PrefillY[k],
 							v->MaxNumSwY[k],
 							v->PrefetchLinesC[i][j][k],
-							v->SwathWidthCThisState[k],
-							v->PrefillC[k],
+							(unsigned int)v->SwathWidthCThisState[k],
+							(unsigned int)v->PrefillC[k],
 							v->MaxNumSwC[k],
 							v->swath_width_luma_ub_this_state[k],
 							v->swath_width_chroma_ub_this_state[k],
@@ -3451,23 +3452,23 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							mode_lib->vba.UrgentBurstFactorChromaPre,
 							mode_lib->vba.UrgentBurstFactorCursorPre);
 
-					mode_lib->vba.TotImmediateFlipBytes = 0.0;
+					mode_lib->vba.TotImmediateFlipBytes = 0;
 					for (k = 0; k <= mode_lib->vba.NumberOfActiveSurfaces - 1; k++) {
 						if (!(mode_lib->vba.ImmediateFlipRequirement[k] ==
 								dm_immediate_flip_not_required)) {
 							mode_lib->vba.TotImmediateFlipBytes =
-									mode_lib->vba.TotImmediateFlipBytes
+									(unsigned int)(mode_lib->vba.TotImmediateFlipBytes
 								+ mode_lib->vba.NoOfDPP[i][j][k]
 								* mode_lib->vba.PDEAndMetaPTEBytesPerFrame[i][j][k]
-								+ mode_lib->vba.MetaRowBytes[i][j][k];
+								+ mode_lib->vba.MetaRowBytes[i][j][k]);
 							if (mode_lib->vba.use_one_row_for_frame_flip[i][j][k]) {
 								mode_lib->vba.TotImmediateFlipBytes =
-									mode_lib->vba.TotImmediateFlipBytes + 2
-								* mode_lib->vba.DPTEBytesPerRow[i][j][k];
+									(unsigned int)(mode_lib->vba.TotImmediateFlipBytes + 2
+								* mode_lib->vba.DPTEBytesPerRow[i][j][k]);
 							} else {
 								mode_lib->vba.TotImmediateFlipBytes =
-									mode_lib->vba.TotImmediateFlipBytes
-								+ mode_lib->vba.DPTEBytesPerRow[i][j][k];
+									(unsigned int)(mode_lib->vba.TotImmediateFlipBytes
+								+ mode_lib->vba.DPTEBytesPerRow[i][j][k]);
 							}
 						}
 					}
@@ -3480,7 +3481,7 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 							mode_lib->vba.HostVMEnable,
 							mode_lib->vba.HostVMMaxNonCachedPageTableLevels,
 							mode_lib->vba.GPUVMEnable,
-							mode_lib->vba.HostVMMinPageSize,
+							(unsigned int)mode_lib->vba.HostVMMinPageSize,
 							mode_lib->vba.PDEAndMetaPTEBytesPerFrame[i][j][k],
 							mode_lib->vba.MetaRowBytes[i][j][k],
 							mode_lib->vba.DPTEBytesPerRow[i][j][k],
@@ -3722,9 +3723,9 @@ void dml32_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l
 		mode_lib->vba.DETBufferSizeInKByte[k] =
 			mode_lib->vba.DETBufferSizeInKByteAllStates[mode_lib->vba.VoltageLevel][MaximumMPCCombine][k];
 		mode_lib->vba.DETBufferSizeY[k] =
-				mode_lib->vba.DETBufferSizeYAllStates[mode_lib->vba.VoltageLevel][MaximumMPCCombine][k];
+				(unsigned int)mode_lib->vba.DETBufferSizeYAllStates[mode_lib->vba.VoltageLevel][MaximumMPCCombine][k];
 		mode_lib->vba.DETBufferSizeC[k] =
-				mode_lib->vba.DETBufferSizeCAllStates[mode_lib->vba.VoltageLevel][MaximumMPCCombine][k];
+				(unsigned int)mode_lib->vba.DETBufferSizeCAllStates[mode_lib->vba.VoltageLevel][MaximumMPCCombine][k];
 		mode_lib->vba.OutputType[k] = mode_lib->vba.OutputTypePerState[mode_lib->vba.VoltageLevel][k];
 		mode_lib->vba.OutputRate[k] = mode_lib->vba.OutputRatePerState[mode_lib->vba.VoltageLevel][k];
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
index 5e72966a8daf8..15f5248340a78 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.c
@@ -62,7 +62,7 @@ unsigned int dml32_dscceComputeDelay(
 		pixelsPerClock = 1;
 
 	//initial transmit delay as per PPS
-	initalXmitDelay = dml_round(rcModelSize / 2.0 / BPP / pixelsPerClock);
+	initalXmitDelay = (unsigned int)dml_round(rcModelSize / 2.0 / BPP / pixelsPerClock);
 
 	//compute ssm delay
 	if (bpc == 8)
@@ -514,8 +514,8 @@ void dml32_CalculateSwathAndDETConfiguration(
 			swath_width_chroma_ub);
 
 	for (k = 0; k < NumberOfActiveSurfaces; ++k) {
-		RoundedUpMaxSwathSizeBytesY[k] = swath_width_luma_ub[k] * BytePerPixDETY[k] * MaximumSwathHeightY[k];
-		RoundedUpMaxSwathSizeBytesC[k] = swath_width_chroma_ub[k] * BytePerPixDETC[k] * MaximumSwathHeightC[k];
+		RoundedUpMaxSwathSizeBytesY[k] = (unsigned int)(swath_width_luma_ub[k] * BytePerPixDETY[k] * MaximumSwathHeightY[k]);
+		RoundedUpMaxSwathSizeBytesC[k] = (unsigned int)(swath_width_chroma_ub[k] * BytePerPixDETC[k] * MaximumSwathHeightC[k]);
 #ifdef __DML_VBA_DEBUG__
 		dml_print("DML::%s: k=%0d DPPPerSurface = %d\n", __func__, k, DPPPerSurface[k]);
 		dml_print("DML::%s: k=%0d swath_width_luma_ub = %d\n", __func__, k, swath_width_luma_ub[k]);
@@ -531,8 +531,8 @@ void dml32_CalculateSwathAndDETConfiguration(
 #endif
 
 		if (SourcePixelFormat[k] == dm_420_10) {
-			RoundedUpMaxSwathSizeBytesY[k] = dml_ceil((unsigned int) RoundedUpMaxSwathSizeBytesY[k], 256);
-			RoundedUpMaxSwathSizeBytesC[k] = dml_ceil((unsigned int) RoundedUpMaxSwathSizeBytesC[k], 256);
+			RoundedUpMaxSwathSizeBytesY[k] = (unsigned int)dml_ceil((unsigned int) RoundedUpMaxSwathSizeBytesY[k], 256);
+			RoundedUpMaxSwathSizeBytesC[k] = (unsigned int)dml_ceil((unsigned int) RoundedUpMaxSwathSizeBytesC[k], 256);
 		}
 	}
 
@@ -656,7 +656,7 @@ void dml32_CalculateSwathAndDETConfiguration(
 #ifdef __DML_VBA_DEBUG__
 			dml_print("DML::%s: k=%0d 2/3 DET for plane0, 1/3 for plane1\n", __func__, k);
 #endif
-			DETBufferSizeY[k] = dml_floor(DETBufferSizeInKByte[k] * 1024 * 2 / 3, 1024);
+			DETBufferSizeY[k] = (unsigned int)dml_floor(DETBufferSizeInKByte[k] * 1024 * 2 / 3, 1024);
 			DETBufferSizeC[k] = DETBufferSizeInKByte[k] * 1024 - DETBufferSizeY[k];
 		}
 
@@ -786,14 +786,14 @@ void dml32_CalculateSwathWidth(
 			SwathWidthC[k] = SwathWidthdoubleDPPC[k];
 		}
 
-		surface_width_ub_l  = dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
-		surface_height_ub_l = dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
+		surface_width_ub_l  = (unsigned int)dml_ceil(SurfaceWidthY[k], Read256BytesBlockWidthY[k]);
+		surface_height_ub_l = (unsigned int)dml_ceil(SurfaceHeightY[k], Read256BytesBlockHeightY[k]);
 
 		if (!IsVertical(SourceRotation[k])) {
 			MaximumSwathHeightY[k] = Read256BytesBlockHeightY[k];
 			MaximumSwathHeightC[k] = Read256BytesBlockHeightC[k];
 			if (ViewportStationary[k] && DPPPerSurface[k] == 1) {
-				swath_width_luma_ub[k] = dml_min(surface_width_ub_l,
+				swath_width_luma_ub[k] = (unsigned int)dml_min(surface_width_ub_l,
 						dml_floor(ViewportXStart[k] +
 								SwathWidthY[k] +
 								Read256BytesBlockWidthY[k] - 1,
@@ -801,22 +801,22 @@ void dml32_CalculateSwathWidth(
 								dml_floor(ViewportXStart[k],
 								Read256BytesBlockWidthY[k]));
 			} else {
-				swath_width_luma_ub[k] = dml_min(surface_width_ub_l,
+				swath_width_luma_ub[k] = (unsigned int)dml_min(surface_width_ub_l,
 						dml_ceil(SwathWidthY[k] - 1,
 								Read256BytesBlockWidthY[k]) +
 								Read256BytesBlockWidthY[k]);
 			}
 			if (BytePerPixC[k] > 0) {
-				surface_width_ub_c  = dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
+				surface_width_ub_c  = (unsigned int)dml_ceil(SurfaceWidthC[k], Read256BytesBlockWidthC[k]);
 				if (ViewportStationary[k] && DPPPerSurface[k] == 1) {
-					swath_width_chroma_ub[k] = dml_min(surface_width_ub_c,
+					swath_width_chroma_ub[k] = (unsigned int)dml_min(surface_width_ub_c,
 							dml_floor(ViewportXStartC[k] + SwathWidthC[k] +
 									Read256BytesBlockWidthC[k] - 1,
 									Read256BytesBlockWidthC[k]) -
 									dml_floor(ViewportXStartC[k],
 									Read256BytesBlockWidthC[k]));
 				} else {
-					swath_width_chroma_ub[k] = dml_min(surface_width_ub_c,
+					swath_width_chroma_ub[k] = (unsigned int)dml_min(surface_width_ub_c,
 							dml_ceil(SwathWidthC[k] - 1,
 								Read256BytesBlockWidthC[k]) +
 								Read256BytesBlockWidthC[k]);
@@ -829,25 +829,25 @@ void dml32_CalculateSwathWidth(
 			MaximumSwathHeightC[k] = Read256BytesBlockWidthC[k];
 
 			if (ViewportStationary[k] && DPPPerSurface[k] == 1) {
-				swath_width_luma_ub[k] = dml_min(surface_height_ub_l, dml_floor(ViewportYStart[k] +
+				swath_width_luma_ub[k] = (unsigned int)dml_min(surface_height_ub_l, dml_floor(ViewportYStart[k] +
 						SwathWidthY[k] + Read256BytesBlockHeightY[k] - 1,
 						Read256BytesBlockHeightY[k]) -
 						dml_floor(ViewportYStart[k], Read256BytesBlockHeightY[k]));
 			} else {
-				swath_width_luma_ub[k] = dml_min(surface_height_ub_l, dml_ceil(SwathWidthY[k] - 1,
+				swath_width_luma_ub[k] = (unsigned int)dml_min(surface_height_ub_l, dml_ceil(SwathWidthY[k] - 1,
 						Read256BytesBlockHeightY[k]) + Read256BytesBlockHeightY[k]);
 			}
 			if (BytePerPixC[k] > 0) {
-				surface_height_ub_c = dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
+				surface_height_ub_c = (unsigned int)dml_ceil(SurfaceHeightC[k], Read256BytesBlockHeightC[k]);
 				if (ViewportStationary[k] && DPPPerSurface[k] == 1) {
-					swath_width_chroma_ub[k] = dml_min(surface_height_ub_c,
+					swath_width_chroma_ub[k] = (unsigned int)dml_min(surface_height_ub_c,
 							dml_floor(ViewportYStartC[k] + SwathWidthC[k] +
 									Read256BytesBlockHeightC[k] - 1,
 									Read256BytesBlockHeightC[k]) -
 									dml_floor(ViewportYStartC[k],
 											Read256BytesBlockHeightC[k]));
 				} else {
-					swath_width_chroma_ub[k] = dml_min(surface_height_ub_c,
+					swath_width_chroma_ub[k] = (unsigned int)dml_min(surface_height_ub_c,
 							dml_ceil(SwathWidthC[k] - 1, Read256BytesBlockHeightC[k]) +
 							Read256BytesBlockHeightC[k]);
 				}
@@ -956,7 +956,7 @@ void dml32_CalculateDETBufferSize(
 		if (DETSizeOverride[0] > 0) {
 			DETBufferSizeInKByte[0] = DETSizeOverride[0];
 		} else {
-			DETBufferSizeInKByte[0] = dml_max(nomDETInKByte, dml_ceil(2.0 *
+			DETBufferSizeInKByte[0] = (unsigned int)dml_max((unsigned int)nomDETInKByte, dml_ceil(2.0 *
 					((double) RoundedUpMaxSwathSizeBytesY[0] +
 							(double) RoundedUpMaxSwathSizeBytesC[0]) / 1024.0, 64.0));
 		}
@@ -993,7 +993,7 @@ void dml32_CalculateDETBufferSize(
 #endif
 
 			if (minDET_pipe == 0) {
-				minDET_pipe = dml_max(128, dml_ceil(((double)RoundedUpMaxSwathSizeBytesY[k] +
+				minDET_pipe = (unsigned int)dml_max(128, dml_ceil(((double)RoundedUpMaxSwathSizeBytesY[k] +
 						(double)RoundedUpMaxSwathSizeBytesC[k]) / 1024.0, 64));
 #ifdef __DML_VBA_DEBUG__
 				dml_print("DML::%s: k=%0d minDET_pipe = %d (assume each plane take half DET)\n",
@@ -1106,7 +1106,7 @@ void dml32_CalculateDETBufferSize(
 				//dml_print("DML::%s: j=%d, tmp1 = %f\n", __func__, j, tmp1);
 				//dml_print("DML::%s: j=%d, tmp2 = %f\n", __func__, j, tmp2);
 
-				NextDETBufferPieceInKByte = dml_min(
+				NextDETBufferPieceInKByte = (unsigned int)dml_min(
 					dml_round((double) DETBufferSizePoolInKByte *
 						(ReadBandwidthLuma[NextSurfaceToAssignDETPiece] +
 						ReadBandwidthChroma[NextSurfaceToAssignDETPiece]) /
@@ -1379,7 +1379,7 @@ void dml32_CalculateOutputLink(
 			*RequiresFEC = false;
 			*OutBpp = dml32_TruncToValidBPP(dml_min(600, PHYCLKPerState) * 10, 3, HTotal, HActive,
 					PixelClockBackEnd, ForcedOutputLinkBPP, false, Output, OutputFormat,
-					DSCInputBitPerComponent, NumberOfDSCSlices, AudioSampleRate, AudioSampleLayout,
+					DSCInputBitPerComponent, NumberOfDSCSlices, (unsigned int)AudioSampleRate, AudioSampleLayout,
 					ODMModeNoDSC, ODMModeDSC, &dummy);
 			//OutputTypeAndRate = "HDMI";
 			*OutputType = dm_output_type_hdmi;
@@ -1407,7 +1407,7 @@ void dml32_CalculateOutputLink(
 					*OutBpp = dml32_TruncToValidBPP((1 - Downspreading / 100) * 10000,
 							OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 							ForcedOutputLinkBPP, LinkDSCEnable, Output, OutputFormat,
-							DSCInputBitPerComponent, NumberOfDSCSlices, AudioSampleRate,
+							DSCInputBitPerComponent, NumberOfDSCSlices, (unsigned int)AudioSampleRate,
 							AudioSampleLayout, ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 					if (*OutBpp == 0 && PHYCLKD32PerState < 13500.0 / 32 && DSCEnable == true &&
 							ForcedOutputLinkBPP == 0) {
@@ -1417,7 +1417,7 @@ void dml32_CalculateOutputLink(
 								OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 								ForcedOutputLinkBPP, LinkDSCEnable, Output,
 								OutputFormat, DSCInputBitPerComponent,
-								NumberOfDSCSlices, AudioSampleRate, AudioSampleLayout,
+								NumberOfDSCSlices, (unsigned int)AudioSampleRate, AudioSampleLayout,
 								ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 					}
 					//OutputTypeAndRate = Output & " UHBR10";
@@ -1429,7 +1429,7 @@ void dml32_CalculateOutputLink(
 					*OutBpp = dml32_TruncToValidBPP((1 - Downspreading / 100) * 13500,
 							OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 							ForcedOutputLinkBPP, LinkDSCEnable, Output, OutputFormat,
-							DSCInputBitPerComponent, NumberOfDSCSlices, AudioSampleRate,
+							DSCInputBitPerComponent, NumberOfDSCSlices, (unsigned int)AudioSampleRate,
 							AudioSampleLayout, ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 
 					if (*OutBpp == 0 && PHYCLKD32PerState < 20000 / 32 && DSCEnable == true &&
@@ -1440,7 +1440,7 @@ void dml32_CalculateOutputLink(
 								OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 								ForcedOutputLinkBPP, LinkDSCEnable, Output,
 								OutputFormat, DSCInputBitPerComponent,
-								NumberOfDSCSlices, AudioSampleRate, AudioSampleLayout,
+								NumberOfDSCSlices, (unsigned int)AudioSampleRate, AudioSampleLayout,
 								ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 					}
 					//OutputTypeAndRate = Output & " UHBR13p5";
@@ -1452,7 +1452,7 @@ void dml32_CalculateOutputLink(
 					*OutBpp = dml32_TruncToValidBPP((1 - Downspreading / 100) * 20000,
 							OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 							ForcedOutputLinkBPP, LinkDSCEnable, Output, OutputFormat,
-							DSCInputBitPerComponent, NumberOfDSCSlices, AudioSampleRate,
+							DSCInputBitPerComponent, NumberOfDSCSlices, (unsigned int)AudioSampleRate,
 							AudioSampleLayout, ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 					if (*OutBpp == 0 && DSCEnable == true && ForcedOutputLinkBPP == 0) {
 						*RequiresDSC = true;
@@ -1461,7 +1461,7 @@ void dml32_CalculateOutputLink(
 								OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 								ForcedOutputLinkBPP, LinkDSCEnable, Output,
 								OutputFormat, DSCInputBitPerComponent,
-								NumberOfDSCSlices, AudioSampleRate, AudioSampleLayout,
+								NumberOfDSCSlices, (unsigned int)AudioSampleRate, AudioSampleLayout,
 								ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 					}
 					//OutputTypeAndRate = Output & " UHBR20";
@@ -1475,7 +1475,7 @@ void dml32_CalculateOutputLink(
 					*OutBpp = dml32_TruncToValidBPP((1 - Downspreading / 100) * 2700,
 							OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 							ForcedOutputLinkBPP, LinkDSCEnable, Output, OutputFormat,
-							DSCInputBitPerComponent, NumberOfDSCSlices, AudioSampleRate,
+							DSCInputBitPerComponent, NumberOfDSCSlices, (unsigned int)AudioSampleRate,
 							AudioSampleLayout, ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 					if (*OutBpp == 0 && PHYCLKPerState < 540 && DSCEnable == true &&
 							ForcedOutputLinkBPP == 0) {
@@ -1487,7 +1487,7 @@ void dml32_CalculateOutputLink(
 								OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 								ForcedOutputLinkBPP, LinkDSCEnable, Output,
 								OutputFormat, DSCInputBitPerComponent,
-								NumberOfDSCSlices, AudioSampleRate, AudioSampleLayout,
+								NumberOfDSCSlices, (unsigned int)AudioSampleRate, AudioSampleLayout,
 								ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 					}
 					//OutputTypeAndRate = Output & " HBR";
@@ -1499,7 +1499,7 @@ void dml32_CalculateOutputLink(
 					*OutBpp = dml32_TruncToValidBPP((1 - Downspreading / 100) * 5400,
 							OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 							ForcedOutputLinkBPP, LinkDSCEnable, Output, OutputFormat,
-							DSCInputBitPerComponent, NumberOfDSCSlices, AudioSampleRate,
+							DSCInputBitPerComponent, NumberOfDSCSlices, (unsigned int)AudioSampleRate,
 							AudioSampleLayout, ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 
 					if (*OutBpp == 0 && PHYCLKPerState < 810 && DSCEnable == true &&
@@ -1513,7 +1513,7 @@ void dml32_CalculateOutputLink(
 								OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 								ForcedOutputLinkBPP, LinkDSCEnable, Output,
 								OutputFormat, DSCInputBitPerComponent,
-								NumberOfDSCSlices, AudioSampleRate, AudioSampleLayout,
+								NumberOfDSCSlices, (unsigned int)AudioSampleRate, AudioSampleLayout,
 								ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 					}
 					//OutputTypeAndRate = Output & " HBR2";
@@ -1525,7 +1525,7 @@ void dml32_CalculateOutputLink(
 							OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 							ForcedOutputLinkBPP, LinkDSCEnable, Output,
 							OutputFormat, DSCInputBitPerComponent, NumberOfDSCSlices,
-							AudioSampleRate, AudioSampleLayout, ODMModeNoDSC, ODMModeDSC,
+							(unsigned int)AudioSampleRate, AudioSampleLayout, ODMModeNoDSC, ODMModeDSC,
 							RequiredSlots);
 
 					if (*OutBpp == 0 && DSCEnable == true && ForcedOutputLinkBPP == 0) {
@@ -1538,7 +1538,7 @@ void dml32_CalculateOutputLink(
 								OutputLinkDPLanes, HTotal, HActive, PixelClockBackEnd,
 								ForcedOutputLinkBPP, LinkDSCEnable, Output,
 								OutputFormat, DSCInputBitPerComponent,
-								NumberOfDSCSlices, AudioSampleRate, AudioSampleLayout,
+								NumberOfDSCSlices, (unsigned int)AudioSampleRate, AudioSampleLayout,
 								ODMModeNoDSC, ODMModeDSC, RequiredSlots);
 					}
 					//OutputTypeAndRate = Output & " HBR3";
@@ -1654,7 +1654,7 @@ double dml32_TruncToValidBPP(
 			MaxLinkBPP = 2 * MaxLinkBPP;
 	}
 
-	*RequiredSlots = dml_ceil(DesiredBPP / MaxLinkBPP * 64, 1);
+	*RequiredSlots = (unsigned int)dml_ceil(DesiredBPP / MaxLinkBPP * 64, 1);
 
 	if (DesiredBPP == 0) {
 		if (DSCEnable) {
@@ -1734,22 +1734,22 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled,
 	if (DSCEnabled == true && OutputBpp != 0) {
 		if (ODMMode == dm_odm_combine_mode_4to1) {
 			DSCDelayRequirement_val = 4 * (dml32_dscceComputeDelay(DSCInputBitPerComponent, OutputBpp,
-					dml_ceil(HActive / NumberOfDSCSlices, 1), NumberOfDSCSlices / 4,
+					(unsigned int)dml_ceil(HActive / NumberOfDSCSlices, 1), (unsigned int)(NumberOfDSCSlices / 4),
 					OutputFormat, Output) + dml32_dscComputeDelay(OutputFormat, Output));
 		} else if (ODMMode == dm_odm_combine_mode_2to1) {
 			DSCDelayRequirement_val = 2 * (dml32_dscceComputeDelay(DSCInputBitPerComponent, OutputBpp,
-					dml_ceil(HActive / NumberOfDSCSlices, 1), NumberOfDSCSlices / 2,
+					(unsigned int)dml_ceil(HActive / NumberOfDSCSlices, 1), (unsigned int)(NumberOfDSCSlices / 2),
 					OutputFormat, Output) + dml32_dscComputeDelay(OutputFormat, Output));
 		} else {
 			DSCDelayRequirement_val = dml32_dscceComputeDelay(DSCInputBitPerComponent, OutputBpp,
-					dml_ceil(HActive / NumberOfDSCSlices, 1), NumberOfDSCSlices,
+					(unsigned int)dml_ceil(HActive / NumberOfDSCSlices, 1), (unsigned int)NumberOfDSCSlices,
 					OutputFormat, Output) + dml32_dscComputeDelay(OutputFormat, Output);
 		}
 
-		DSCDelayRequirement_val = DSCDelayRequirement_val + (HTotal - HActive) *
-				dml_ceil((double)DSCDelayRequirement_val / HActive, 1);
+		DSCDelayRequirement_val = (unsigned int)(DSCDelayRequirement_val + (HTotal - HActive) *
+				dml_ceil((double)DSCDelayRequirement_val / HActive, 1));
 
-		DSCDelayRequirement_val = DSCDelayRequirement_val * PixelClock / PixelClockBackEnd;
+		DSCDelayRequirement_val = (unsigned int)(DSCDelayRequirement_val * PixelClock / PixelClockBackEnd);
 
 	} else {
 		DSCDelayRequirement_val = 0;
@@ -1765,7 +1765,7 @@ unsigned int dml32_DSCDelayRequirement(bool DSCEnabled,
 	dml_print("DML::%s: DSCDelayRequirement_val = %d\n", __func__, DSCDelayRequirement_val);
 #endif
 
-	return dml_ceil(DSCDelayRequirement_val * dsc_delay_factor_wa, 1);
+	return (unsigned int)dml_ceil(DSCDelayRequirement_val * dsc_delay_factor_wa, 1);
 }
 
 void dml32_CalculateSurfaceSizeInMall(
@@ -1811,16 +1811,16 @@ void dml32_CalculateSurfaceSizeInMall(
 
 	for (k = 0; k < NumberOfActiveSurfaces; ++k) {
 		if (ViewportStationary[k]) {
-			SurfaceSizeInMALL[k] = dml_min(dml_ceil(SurfaceWidthY[k], ReadBlockWidthY[k]),
+			SurfaceSizeInMALL[k] = (unsigned int)(dml_min(dml_ceil(SurfaceWidthY[k], ReadBlockWidthY[k]),
 					dml_floor(ViewportXStartY[k] + ViewportWidthY[k] + ReadBlockWidthY[k] - 1,
 						ReadBlockWidthY[k]) - dml_floor(ViewportXStartY[k],
 						ReadBlockWidthY[k])) * dml_min(dml_ceil(SurfaceHeightY[k],
 						ReadBlockHeightY[k]), dml_floor(ViewportYStartY[k] +
 						ViewportHeightY[k] + ReadBlockHeightY[k] - 1, ReadBlockHeightY[k]) -
-						dml_floor(ViewportYStartY[k], ReadBlockHeightY[k])) * BytesPerPixelY[k];
+						dml_floor(ViewportYStartY[k], ReadBlockHeightY[k])) * BytesPerPixelY[k]);
 
 			if (ReadBlockWidthC[k] > 0) {
-				SurfaceSizeInMALL[k] = SurfaceSizeInMALL[k] +
+				SurfaceSizeInMALL[k] = (unsigned int)(SurfaceSizeInMALL[k] +
 						dml_min(dml_ceil(SurfaceWidthC[k], ReadBlockWidthC[k]),
 							dml_floor(ViewportXStartC[k] + ViewportWidthC[k] +
 							ReadBlockWidthC[k] - 1, ReadBlockWidthC[k]) -
@@ -1829,10 +1829,10 @@ void dml32_CalculateSurfaceSizeInMall(
 							dml_floor(ViewportYStartC[k] + ViewportHeightC[k] +
 							ReadBlockHeightC[k] - 1, ReadBlockHeightC[k]) -
 							dml_floor(ViewportYStartC[k], ReadBlockHeightC[k])) *
-							BytesPerPixelC[k];
+							BytesPerPixelC[k]);
 			}
 			if (DCCEnable[k] == true) {
-				SurfaceSizeInMALL[k] = SurfaceSizeInMALL[k] +
+				SurfaceSizeInMALL[k] = (unsigned int)(SurfaceSizeInMALL[k] +
 						(dml_min(dml_ceil(DCCMetaPitchY[k], 8 * Read256BytesBlockWidthY[k]),
 							dml_floor(ViewportXStartY[k] + ViewportWidthY[k] + 8 *
 							Read256BytesBlockWidthY[k] - 1, 8 * Read256BytesBlockWidthY[k])
@@ -1841,9 +1841,9 @@ void dml32_CalculateSurfaceSizeInMall(
 							Read256BytesBlockHeightY[k]), dml_floor(ViewportYStartY[k] +
 							ViewportHeightY[k] + 8 * Read256BytesBlockHeightY[k] - 1, 8 *
 							Read256BytesBlockHeightY[k]) - dml_floor(ViewportYStartY[k], 8 *
-							Read256BytesBlockHeightY[k])) * BytesPerPixelY[k] / 256) + (64 * 1024);
+							Read256BytesBlockHeightY[k])) *BytesPerPixelY[k] / 256) + (64 * 1024));
 				if (Read256BytesBlockWidthC[k] > 0) {
-					SurfaceSizeInMALL[k] = SurfaceSizeInMALL[k] +
+					SurfaceSizeInMALL[k] = (unsigned int)(SurfaceSizeInMALL[k] +
 							dml_min(dml_ceil(DCCMetaPitchC[k], 8 *
 								Read256BytesBlockWidthC[k]),
 								dml_floor(ViewportXStartC[k] + ViewportWidthC[k] + 8
@@ -1858,41 +1858,41 @@ void dml32_CalculateSurfaceSizeInMall(
 								Read256BytesBlockHeightC[k]) -
 								dml_floor(ViewportYStartC[k], 8 *
 								Read256BytesBlockHeightC[k])) *
-								BytesPerPixelC[k] / 256;
+								BytesPerPixelC[k] / 256);
 				}
 			}
 		} else {
-			SurfaceSizeInMALL[k] = dml_ceil(dml_min(SurfaceWidthY[k], ViewportWidthY[k] +
+			SurfaceSizeInMALL[k] = (unsigned int)(dml_ceil(dml_min(SurfaceWidthY[k], ViewportWidthY[k] +
 					ReadBlockWidthY[k] - 1), ReadBlockWidthY[k]) *
 					dml_ceil(dml_min(SurfaceHeightY[k], ViewportHeightY[k] +
 							ReadBlockHeightY[k] - 1), ReadBlockHeightY[k]) *
-							BytesPerPixelY[k];
+							BytesPerPixelY[k]);
 			if (ReadBlockWidthC[k] > 0) {
-				SurfaceSizeInMALL[k] = SurfaceSizeInMALL[k] +
+				SurfaceSizeInMALL[k] = (unsigned int)(SurfaceSizeInMALL[k] +
 						dml_ceil(dml_min(SurfaceWidthC[k], ViewportWidthC[k] +
 								ReadBlockWidthC[k] - 1), ReadBlockWidthC[k]) *
 						dml_ceil(dml_min(SurfaceHeightC[k], ViewportHeightC[k] +
 								ReadBlockHeightC[k] - 1), ReadBlockHeightC[k]) *
-								BytesPerPixelC[k];
+								BytesPerPixelC[k]);
 			}
 			if (DCCEnable[k] == true) {
-				SurfaceSizeInMALL[k] = SurfaceSizeInMALL[k] +
+				SurfaceSizeInMALL[k] = (unsigned int)(SurfaceSizeInMALL[k] +
 						(dml_ceil(dml_min(DCCMetaPitchY[k], ViewportWidthY[k] + 8 *
 								Read256BytesBlockWidthY[k] - 1), 8 *
 								Read256BytesBlockWidthY[k]) *
 						dml_ceil(dml_min(SurfaceHeightY[k], ViewportHeightY[k] + 8 *
 								Read256BytesBlockHeightY[k] - 1), 8 *
-								Read256BytesBlockHeightY[k]) * BytesPerPixelY[k] / 256) + (64 * 1024);
+								Read256BytesBlockHeightY[k]) * BytesPerPixelY[k] / 256) + (64 * 1024));
 
 				if (Read256BytesBlockWidthC[k] > 0) {
-					SurfaceSizeInMALL[k] = SurfaceSizeInMALL[k] +
+					SurfaceSizeInMALL[k] = (unsigned int)(SurfaceSizeInMALL[k] +
 							dml_ceil(dml_min(DCCMetaPitchC[k], ViewportWidthC[k] + 8 *
 									Read256BytesBlockWidthC[k] - 1), 8 *
 									Read256BytesBlockWidthC[k]) *
 							dml_ceil(dml_min(SurfaceHeightC[k], ViewportHeightC[k] + 8 *
 									Read256BytesBlockHeightC[k] - 1), 8 *
 									Read256BytesBlockHeightC[k]) *
-									BytesPerPixelC[k] / 256;
+									BytesPerPixelC[k] / 256);
 				}
 			}
 		}
@@ -2321,51 +2321,51 @@ unsigned int dml32_CalculateVMAndRowBytes(
 		if (HostVMMinPageSize < 2048)
 			HostVMDynamicLevels = HostVMMaxNonCachedPageTableLevels;
 		else if (HostVMMinPageSize >= 2048 && HostVMMinPageSize < 1048576)
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
 		else
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
 	}
 
 	*MetaRequestHeight = 8 * BlockHeight256Bytes;
 	*MetaRequestWidth = 8 * BlockWidth256Bytes;
 	if (SurfaceTiling == dm_sw_linear) {
 		*meta_row_height = 32;
-		*meta_row_width = dml_floor(ViewportXStart + SwathWidth + *MetaRequestWidth - 1, *MetaRequestWidth)
-				- dml_floor(ViewportXStart, *MetaRequestWidth);
+		*meta_row_width = (unsigned int)(dml_floor(ViewportXStart + SwathWidth + *MetaRequestWidth - 1, *MetaRequestWidth)
+				- dml_floor(ViewportXStart, *MetaRequestWidth));
 	} else if (!IsVertical(SourceRotation)) {
 		*meta_row_height = *MetaRequestHeight;
 		if (ViewportStationary && NumberOfDPPs == 1) {
-			*meta_row_width = dml_floor(ViewportXStart + SwathWidth + *MetaRequestWidth - 1,
-					*MetaRequestWidth) - dml_floor(ViewportXStart, *MetaRequestWidth);
+			*meta_row_width = (unsigned int)(dml_floor(ViewportXStart + SwathWidth + *MetaRequestWidth - 1,
+					*MetaRequestWidth) - dml_floor(ViewportXStart, *MetaRequestWidth));
 		} else {
-			*meta_row_width = dml_ceil(SwathWidth - 1, *MetaRequestWidth) + *MetaRequestWidth;
+			*meta_row_width = (unsigned int)(dml_ceil(SwathWidth - 1, *MetaRequestWidth) + *MetaRequestWidth);
 		}
-		*MetaRowByte = *meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0;
+		*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestHeight * BytePerPixel / 256.0);
 	} else {
 		*meta_row_height = *MetaRequestWidth;
 		if (ViewportStationary && NumberOfDPPs == 1) {
-			*meta_row_width = dml_floor(ViewportYStart + ViewportHeight + *MetaRequestHeight - 1,
-					*MetaRequestHeight) - dml_floor(ViewportYStart, *MetaRequestHeight);
+			*meta_row_width = (unsigned int)(dml_floor(ViewportYStart + ViewportHeight + *MetaRequestHeight - 1,
+					*MetaRequestHeight) - dml_floor(ViewportYStart, *MetaRequestHeight));
 		} else {
-			*meta_row_width = dml_ceil(SwathWidth - 1, *MetaRequestHeight) + *MetaRequestHeight;
+			*meta_row_width = (unsigned int)(dml_ceil(SwathWidth - 1, *MetaRequestHeight) + *MetaRequestHeight);
 		}
-		*MetaRowByte = *meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0;
+		*MetaRowByte = (unsigned int)(*meta_row_width * *MetaRequestWidth * BytePerPixel / 256.0);
 	}
 
 	if (ViewportStationary && (NumberOfDPPs == 1 || !IsVertical(SourceRotation))) {
-		vp_height_meta_ub = dml_floor(ViewportYStart + ViewportHeight + 64 * BlockHeight256Bytes - 1,
-				64 * BlockHeight256Bytes) - dml_floor(ViewportYStart, 64 * BlockHeight256Bytes);
+		vp_height_meta_ub = (unsigned int)(dml_floor(ViewportYStart + ViewportHeight + 64 * BlockHeight256Bytes - 1,
+				64 * BlockHeight256Bytes) - dml_floor(ViewportYStart, 64 * BlockHeight256Bytes));
 	} else if (!IsVertical(SourceRotation)) {
-		vp_height_meta_ub = dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes) + 64 * BlockHeight256Bytes;
+		vp_height_meta_ub = (unsigned int)(dml_ceil(ViewportHeight - 1, 64 * BlockHeight256Bytes) + 64 * BlockHeight256Bytes);
 	} else {
-		vp_height_meta_ub = dml_ceil(SwathWidth - 1, 64 * BlockHeight256Bytes) + 64 * BlockHeight256Bytes;
+		vp_height_meta_ub = (unsigned int)(dml_ceil(SwathWidth - 1, 64 * BlockHeight256Bytes) + 64 * BlockHeight256Bytes);
 	}
 
-	DCCMetaSurfaceBytes = DCCMetaPitch * vp_height_meta_ub * BytePerPixel / 256.0;
+	DCCMetaSurfaceBytes = (unsigned int)(DCCMetaPitch * vp_height_meta_ub * BytePerPixel / 256.0);
 
 	if (GPUVMEnable == true) {
-		*MetaPTEBytesFrame = (dml_ceil((double) (DCCMetaSurfaceBytes - 4.0 * 1024.0) /
-				(8 * 4.0 * 1024), 1) + 1) * 64;
+		*MetaPTEBytesFrame = (unsigned int)((dml_ceil((double) (DCCMetaSurfaceBytes - 4.0 * 1024.0) /
+				(8 * 4.0 * 1024), 1) + 1) * 64);
 		MPDEBytesFrame = 128 * (GPUVMMaxPageTableLevels - 1);
 	} else {
 		*MetaPTEBytesFrame = 0;
@@ -2382,16 +2382,16 @@ unsigned int dml32_CalculateVMAndRowBytes(
 
 	if (GPUVMEnable == true && GPUVMMaxPageTableLevels > 1) {
 		if (ViewportStationary && (NumberOfDPPs == 1 || !IsVertical(SourceRotation))) {
-			vp_height_dpte_ub = dml_floor(ViewportYStart + ViewportHeight +
+			vp_height_dpte_ub = (unsigned int)(dml_floor(ViewportYStart + ViewportHeight +
 					MacroTileHeight - 1, MacroTileHeight) -
-					dml_floor(ViewportYStart, MacroTileHeight);
+					dml_floor(ViewportYStart, MacroTileHeight));
 		} else if (!IsVertical(SourceRotation)) {
-			vp_height_dpte_ub = dml_ceil(ViewportHeight - 1, MacroTileHeight) + MacroTileHeight;
+			vp_height_dpte_ub = (unsigned int)dml_ceil(ViewportHeight - 1, MacroTileHeight) + MacroTileHeight;
 		} else {
-			vp_height_dpte_ub = dml_ceil(SwathWidth - 1, MacroTileHeight) + MacroTileHeight;
+			vp_height_dpte_ub = (unsigned int)dml_ceil(SwathWidth - 1, MacroTileHeight) + MacroTileHeight;
 		}
-		*DPDE0BytesFrame = 64 * (dml_ceil((Pitch * vp_height_dpte_ub * BytePerPixel - MacroTileSizeBytes) /
-				(8 * 2097152), 1) + 1);
+		*DPDE0BytesFrame = (unsigned int)(64 * (dml_ceil((Pitch * vp_height_dpte_ub * BytePerPixel - MacroTileSizeBytes) /
+				(8 * 2097152), 1) + 1));
 		ExtraDPDEBytesFrame = 128 * (GPUVMMaxPageTableLevels - 2);
 	} else {
 		*DPDE0BytesFrame = 0;
@@ -2449,14 +2449,14 @@ unsigned int dml32_CalculateVMAndRowBytes(
 #endif
 
 	*dpte_row_height_one_row_per_frame = vp_height_dpte_ub;
-	*dpte_row_width_ub_one_row_per_frame = (dml_ceil(((double)Pitch * (double)*dpte_row_height_one_row_per_frame /
+	*dpte_row_width_ub_one_row_per_frame = (unsigned int)((dml_ceil(((double)Pitch * (double)*dpte_row_height_one_row_per_frame /
 			(double) *PixelPTEReqHeight - 1) / (double) *PixelPTEReqWidth, 1) + 1) *
-					(double) *PixelPTEReqWidth;
+					(double) *PixelPTEReqWidth);
 	*PixelPTEBytesPerRow_one_row_per_frame = *dpte_row_width_ub_one_row_per_frame / *PixelPTEReqWidth *
 			*PTERequestSize;
 
 	if (SurfaceTiling == dm_sw_linear) {
-		*dpte_row_height = dml_min(128, 1 << (unsigned int) dml_floor(dml_log2(PTEBufferSizeInRequests *
+		*dpte_row_height = (unsigned int)dml_min(128, 1 << (unsigned int) dml_floor(dml_log2(PTEBufferSizeInRequests *
 				*PixelPTEReqWidth / Pitch), 1));
 #ifdef __DML_VBA_DEBUG__
 		dml_print("DML::%s: dpte_row_height = %d (1)\n", __func__,
@@ -2470,9 +2470,9 @@ unsigned int dml32_CalculateVMAndRowBytes(
 						*PixelPTEReqWidth / Pitch), 1));
 		dml_print("DML::%s: dpte_row_height = %d\n", __func__, *dpte_row_height);
 #endif
-		*dpte_row_width_ub = dml_ceil(((double) Pitch * (double) *dpte_row_height - 1),
+		*dpte_row_width_ub = (unsigned int)dml_ceil(((double) Pitch * (double) *dpte_row_height - 1),
 				(double) *PixelPTEReqWidth) + *PixelPTEReqWidth;
-		*PixelPTEBytesPerRow = *dpte_row_width_ub / (double)*PixelPTEReqWidth * (double)*PTERequestSize;
+		*PixelPTEBytesPerRow = (unsigned int)(*dpte_row_width_ub / (double)*PixelPTEReqWidth * (double)*PTERequestSize);
 
 		// VBA_DELTA, VBA doesn't have programming value for pte row height linear.
 		*dpte_row_height_linear = 1 << (unsigned int) dml_floor(dml_log2(PTEBufferSizeInRequests *
@@ -2484,27 +2484,27 @@ unsigned int dml32_CalculateVMAndRowBytes(
 		*dpte_row_height = *PixelPTEReqHeight;
 
 		if (GPUVMMinPageSizeKBytes > 64) {
-			*dpte_row_width_ub = (dml_ceil((Pitch * *dpte_row_height / *PixelPTEReqHeight - 1) /
-					*PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth;
+			*dpte_row_width_ub = (unsigned int)((dml_ceil((Pitch * *dpte_row_height / *PixelPTEReqHeight - 1) /
+					*PixelPTEReqWidth, 1) + 1) * *PixelPTEReqWidth);
 		} else if (ViewportStationary && (NumberOfDPPs == 1)) {
-			*dpte_row_width_ub = dml_floor(ViewportXStart + SwathWidth +
+			*dpte_row_width_ub = (unsigned int)(dml_floor(ViewportXStart + SwathWidth +
 					*PixelPTEReqWidth - 1, *PixelPTEReqWidth) -
-					dml_floor(ViewportXStart, *PixelPTEReqWidth);
+					dml_floor(ViewportXStart, *PixelPTEReqWidth));
 		} else {
-			*dpte_row_width_ub = (dml_ceil((SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) *
-					*PixelPTEReqWidth;
+			*dpte_row_width_ub = (unsigned int)((dml_ceil((SwathWidth - 1) / *PixelPTEReqWidth, 1) + 1) *
+					*PixelPTEReqWidth);
 		}
 
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqWidth * *PTERequestSize;
 	} else {
-		*dpte_row_height = dml_min(*PixelPTEReqWidth, MacroTileWidth);
+		*dpte_row_height = (unsigned int)dml_min(*PixelPTEReqWidth, MacroTileWidth);
 
 		if (ViewportStationary && (NumberOfDPPs == 1)) {
-			*dpte_row_width_ub = dml_floor(ViewportYStart + ViewportHeight + *PixelPTEReqHeight - 1,
-					*PixelPTEReqHeight) - dml_floor(ViewportYStart, *PixelPTEReqHeight);
+			*dpte_row_width_ub = (unsigned int)(dml_floor(ViewportYStart + ViewportHeight + *PixelPTEReqHeight - 1,
+					*PixelPTEReqHeight) - dml_floor(ViewportYStart, *PixelPTEReqHeight));
 		} else {
-			*dpte_row_width_ub = (dml_ceil((SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1)
-					* *PixelPTEReqHeight;
+			*dpte_row_width_ub = (unsigned int)((dml_ceil((SwathWidth - 1) / *PixelPTEReqHeight, 1) + 1)
+					* *PixelPTEReqHeight);
 		}
 
 		*PixelPTEBytesPerRow = *dpte_row_width_ub / *PixelPTEReqHeight * *PTERequestSize;
@@ -2584,16 +2584,16 @@ double dml32_CalculatePrefetchSourceLines(
 		}
 		sw0_tmp = SwathHeight - (vp_start_rot % SwathHeight);
 		if (sw0_tmp < *VInitPreFill)
-			*MaxNumSwath = dml_ceil((*VInitPreFill - sw0_tmp) / SwathHeight, 1) + 1;
+			*MaxNumSwath = (unsigned int)dml_ceil((*VInitPreFill - sw0_tmp) / SwathHeight, 1) + 1;
 		else
 			*MaxNumSwath = 1;
-		MaxPartialSwath = dml_max(1, (unsigned int) (vp_start_rot + *VInitPreFill - 1) % SwathHeight);
+		MaxPartialSwath = (unsigned int)dml_max(1, (unsigned int) (vp_start_rot + *VInitPreFill - 1) % SwathHeight);
 	} else {
-		*MaxNumSwath = dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1;
+		*MaxNumSwath = (unsigned int)dml_ceil((*VInitPreFill - 1.0) / SwathHeight, 1) + 1;
 		if (*VInitPreFill > 1)
-			MaxPartialSwath = dml_max(1, (unsigned int) (*VInitPreFill - 2) % SwathHeight);
+			MaxPartialSwath = (unsigned int)dml_max(1, (unsigned int) (*VInitPreFill - 2) % SwathHeight);
 		else
-			MaxPartialSwath = dml_max(1, (unsigned int) (*VInitPreFill + SwathHeight - 2) % SwathHeight);
+			MaxPartialSwath = (unsigned int)dml_max(1, (unsigned int) (*VInitPreFill + SwathHeight - 2) % SwathHeight);
 	}
 	numLines = *MaxNumSwath * SwathHeight + MaxPartialSwath;
 
@@ -2954,7 +2954,7 @@ void dml32_UseMinimumDCFCLK(
 		unsigned int VTotal[],
 		unsigned int VActive[],
 		unsigned int DynamicMetadataTransmittedBytes[],
-		unsigned int DynamicMetadataLinesBeforeActiveRequired[],
+		int DynamicMetadataLinesBeforeActiveRequired[],
 		bool Interlace[],
 		double RequiredDPPCLKPerSurface[][2][DC__NUM_DPP__MAX],
 		double RequiredDISPCLK[][2],
@@ -3180,9 +3180,9 @@ unsigned int dml32_CalculateExtraLatencyBytes(unsigned int ReorderingBytes,
 		if (HostVMMinPageSize < 2048)
 			HostVMDynamicLevels = HostVMMaxNonCachedPageTableLevels;
 		else if (HostVMMinPageSize >= 2048 && HostVMMinPageSize < 1048576)
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 1);
 		else
-			HostVMDynamicLevels = dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
+			HostVMDynamicLevels = (unsigned int)dml_max(0, (int) HostVMMaxNonCachedPageTableLevels - 2);
 	} else {
 		HostVMDynamicLevels = 0;
 	}
@@ -3196,7 +3196,7 @@ unsigned int dml32_CalculateExtraLatencyBytes(unsigned int ReorderingBytes,
 					(1 + 8 * HostVMDynamicLevels) * HostVMInefficiencyFactor;
 		}
 	}
-	return ret;
+	return (unsigned int)ret;
 }
 
 void dml32_CalculateVUpdateAndDynamicMetadataParameters(
@@ -3228,7 +3228,7 @@ void dml32_CalculateVUpdateAndDynamicMetadataParameters(
 			dml_ceil((14.0 / DCFClkDeepSleep + 12.0 / Dppclk + TotalRepeaterDelayTime) * PixelClock, 1.0);
 	*VReadyOffsetPix  = dml_ceil(dml_max(150.0 / Dppclk,
 			TotalRepeaterDelayTime + 20.0 / DCFClkDeepSleep + 10.0 / Dppclk) * PixelClock, 1.0);
-	*VUpdateOffsetPix = dml_ceil(HTotal / 4.0, 1.0);
+	*VUpdateOffsetPix = (unsigned int)dml_ceil(HTotal / 4.0, 1.0);
 	*TSetup = (*VUpdateOffsetPix + *VUpdateWidthPix + *VReadyOffsetPix) / PixelClock;
 	*Tdmbf = DynamicMetadataTransmittedBytes / 4.0 / Dispclk;
 	*Tdmec = HTotal / PixelClock;
@@ -3491,7 +3491,7 @@ bool dml32_CalculatePrefetchSchedule(
 	double  prefetch_sw_bytes;
 	double  bytes_pp;
 	double  dep_bytes;
-	unsigned int max_vratio_pre = v->MaxVRatioPre;
+	unsigned int max_vratio_pre = (unsigned int)v->MaxVRatioPre;
 	double  min_Lsw;
 	double  Tsw_est1 = 0;
 	double  Tsw_est3 = 0;
@@ -3567,13 +3567,13 @@ bool dml32_CalculatePrefetchSchedule(
 			v->GPUVMEnable == true ? TWait + Tvm_trips : 0);
 
 	if (myPipe->ScalerEnabled)
-		DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCL;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCL);
 	else
-		DPPCycles = DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCLLBOnly;
+		DPPCycles = (unsigned int)(DPPCLKDelaySubtotalPlusCNVCFormater + v->DPPCLKDelaySCLLBOnly);
 
-	DPPCycles = DPPCycles + myPipe->NumberOfCursors * v->DPPCLKDelayCNVCCursor;
+	DPPCycles = (unsigned int)(DPPCycles + myPipe->NumberOfCursors * v->DPPCLKDelayCNVCCursor);
 
-	DISPCLKCycles = v->DISPCLKDelaySubtotal;
+	DISPCLKCycles = (unsigned int)v->DISPCLKDelaySubtotal;
 
 	if (myPipe->Dppclk == 0.0 || myPipe->Dispclk == 0.0)
 		return true;
@@ -4416,8 +4416,8 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport(
 
 	for (k = 0; k < v->NumberOfActiveSurfaces; ++k) {
 
-		LBLatencyHidingSourceLinesY[k] = dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSizeFinal / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1);
-		LBLatencyHidingSourceLinesC[k] = dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSizeFinal / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1);
+		LBLatencyHidingSourceLinesY[k] = (unsigned int)(dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSizeFinal / v->LBBitPerPixel[k] / (SwathWidthY[k] / dml_max(v->HRatio[k], 1.0)), 1)) - (v->vtaps[k] - 1));
+		LBLatencyHidingSourceLinesC[k] = (unsigned int)(dml_min((double) v->MaxLineBufferLines, dml_floor(v->LineBufferSizeFinal / v->LBBitPerPixel[k] / (SwathWidthC[k] / dml_max(v->HRatioChroma[k], 1.0)), 1)) - (v->VTAPsChroma[k] - 1));
 
 
 #ifdef __DML_VBA_DEBUG__
@@ -4440,7 +4440,7 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport(
 		}
 
 		LinesInDETY[k] = (double) EffectiveDETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k];
-		LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]);
+		LinesInDETYRoundedDownToSwath[k] = (unsigned int)dml_floor(LinesInDETY[k], SwathHeightY[k]);
 		FullDETBufferingTimeY = LinesInDETYRoundedDownToSwath[k] * (v->HTotal[k] / v->PixelClock[k]) / v->VRatio[k];
 
 		ActiveClockChangeLatencyHidingY = EffectiveLBLatencyHidingY + FullDETBufferingTimeY
@@ -4454,7 +4454,7 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport(
 
 		if (BytePerPixelDETC[k] > 0) {
 			LinesInDETC[k] = DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k];
-			LinesInDETCRoundedDownToSwath[k] = dml_floor(LinesInDETC[k], SwathHeightC[k]);
+			LinesInDETCRoundedDownToSwath[k] = (unsigned int)dml_floor(LinesInDETC[k], SwathHeightC[k]);
 			FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath[k] * (v->HTotal[k] / v->PixelClock[k])
 					/ v->VRatioChroma[k];
 			ActiveClockChangeLatencyHidingC = EffectiveLBLatencyHidingC + FullDETBufferingTimeC
@@ -4614,9 +4614,9 @@ void dml32_CalculateWatermarksMALLUseAndDRAMSpeedChangeSupport(
 		unsigned int src_y_pstate_c;
 		unsigned int src_y_ahead_l, src_y_ahead_c, sub_vp_lines_l, sub_vp_lines_c;
 
-		dst_y_pstate = dml_ceil((mmSOCParameters.DRAMClockChangeLatency + mmSOCParameters.UrgentLatency) / (v->HTotal[k] / v->PixelClock[k]), 1);
-		src_y_pstate_l = dml_ceil(dst_y_pstate * v->VRatio[k], SwathHeightY[k]);
-		src_y_ahead_l = dml_floor(DETBufferSizeY[k] / BytePerPixelDETY[k] / SwathWidthY[k], SwathHeightY[k]) + LBLatencyHidingSourceLinesY[k];
+		dst_y_pstate = (unsigned int)dml_ceil((mmSOCParameters.DRAMClockChangeLatency + mmSOCParameters.UrgentLatency) / (v->HTotal[k] / v->PixelClock[k]), 1);
+		src_y_pstate_l = (unsigned int)dml_ceil(dst_y_pstate * v->VRatio[k], SwathHeightY[k]);
+		src_y_ahead_l = (unsigned int)(dml_floor(DETBufferSizeY[k] / BytePerPixelDETY[k] / SwathWidthY[k], SwathHeightY[k]) + LBLatencyHidingSourceLinesY[k]);
 		sub_vp_lines_l = src_y_pstate_l + src_y_ahead_l + v->meta_row_height[k];
 
 #ifdef __DML_VBA_DEBUG__
@@ -4634,10 +4634,10 @@ dml_print("DML::%s: k=%d, sub_vp_lines_l    = %d\n", __func__, k, sub_vp_lines_l
 		SubViewportLinesNeededInMALL[k] = sub_vp_lines_l;
 
 		if (BytePerPixelDETC[k] > 0) {
-			src_y_pstate_c = dml_ceil(dst_y_pstate * v->VRatioChroma[k], SwathHeightC[k]);
-			src_y_ahead_c = dml_floor(DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k], SwathHeightC[k]) + LBLatencyHidingSourceLinesC[k];
+			src_y_pstate_c = (unsigned int)dml_ceil(dst_y_pstate * v->VRatioChroma[k], SwathHeightC[k]);
+			src_y_ahead_c = (unsigned int)(dml_floor(DETBufferSizeC[k] / BytePerPixelDETC[k] / SwathWidthC[k], SwathHeightC[k]) + LBLatencyHidingSourceLinesC[k]);
 			sub_vp_lines_c = src_y_pstate_c + src_y_ahead_c + v->meta_row_height_chroma[k];
-			SubViewportLinesNeededInMALL[k] = dml_max(sub_vp_lines_l, sub_vp_lines_c);
+			SubViewportLinesNeededInMALL[k] = (unsigned int)dml_max(sub_vp_lines_l, sub_vp_lines_c);
 
 #ifdef __DML_VBA_DEBUG__
 dml_print("DML::%s: k=%d, src_y_pstate_c            = %d\n", __func__, k, src_y_pstate_c);
@@ -4852,7 +4852,7 @@ void dml32_CalculatePixelDeliveryTimes(
 	for (k = 0; k < NumberOfActiveSurfaces; ++k) {
 		unsigned int cursor_req_per_width;
 
-		cursor_req_per_width = dml_ceil((double) CursorWidth[k][0] * (double) CursorBPP[k][0] /
+		cursor_req_per_width = (unsigned int)dml_ceil((double) CursorWidth[k][0] * (double) CursorBPP[k][0] /
 				256.0 / 8.0, 1.0);
 		if (NumberOfCursors[k] > 0) {
 			if (VRatio[k] <= 1) {
@@ -5000,8 +5000,8 @@ void dml32_CalculateMetaAndPTETimes(
 						meta_row_height_chroma[k];
 				min_meta_chunk_width_chroma = MinMetaChunkSizeBytes * 256 / BytePerPixelC[k] /
 						meta_row_height_chroma[k];
-				meta_chunk_per_row_int_chroma = (double) meta_row_width_chroma[k] /
-						meta_chunk_width_chroma;
+				meta_chunk_per_row_int_chroma = (unsigned int)((double) meta_row_width_chroma[k] /
+						meta_chunk_width_chroma);
 				meta_row_remainder_chroma = meta_row_width_chroma[k] % meta_chunk_width_chroma;
 				if (!IsVertical(SourceRotation[k])) {
 					meta_chunk_threshold_chroma = 2 * min_meta_chunk_width_chroma -
@@ -5035,18 +5035,18 @@ void dml32_CalculateMetaAndPTETimes(
 	for (k = 0; k < NumberOfActiveSurfaces; ++k) {
 		if (GPUVMEnable == true) {
 			if (!IsVertical(SourceRotation[k])) {
-				dpte_group_width_luma = (double) dpte_group_bytes[k] /
-						(double) PTERequestSizeY[k] * PixelPTEReqWidthY[k];
+				dpte_group_width_luma = (unsigned int)((double) dpte_group_bytes[k] /
+						(double) PTERequestSizeY[k] * PixelPTEReqWidthY[k]);
 			} else {
-				dpte_group_width_luma = (double) dpte_group_bytes[k] /
-						(double) PTERequestSizeY[k] * PixelPTEReqHeightY[k];
+				dpte_group_width_luma = (unsigned int)((double) dpte_group_bytes[k] /
+						(double) PTERequestSizeY[k] * PixelPTEReqHeightY[k]);
 			}
 
 			if (use_one_row_for_frame[k]) {
-				dpte_groups_per_row_luma_ub = dml_ceil((double) dpte_row_width_luma_ub[k] /
+				dpte_groups_per_row_luma_ub = (unsigned int)dml_ceil((double) dpte_row_width_luma_ub[k] /
 						(double) dpte_group_width_luma / 2.0, 1.0);
 			} else {
-				dpte_groups_per_row_luma_ub = dml_ceil((double) dpte_row_width_luma_ub[k] /
+				dpte_groups_per_row_luma_ub = (unsigned int)dml_ceil((double) dpte_row_width_luma_ub[k] /
 						(double) dpte_group_width_luma, 1.0);
 			}
 #ifdef __DML_VBA_DEBUG__
@@ -5080,18 +5080,18 @@ void dml32_CalculateMetaAndPTETimes(
 				time_per_pte_group_flip_chroma[k] = 0;
 			} else {
 				if (!IsVertical(SourceRotation[k])) {
-					dpte_group_width_chroma = (double) dpte_group_bytes[k] /
-							(double) PTERequestSizeC[k] * PixelPTEReqWidthC[k];
+					dpte_group_width_chroma = (unsigned int)((double) dpte_group_bytes[k] /
+							(double) PTERequestSizeC[k] * PixelPTEReqWidthC[k]);
 				} else {
-					dpte_group_width_chroma = (double) dpte_group_bytes[k] /
-							(double) PTERequestSizeC[k] * PixelPTEReqHeightC[k];
+					dpte_group_width_chroma = (unsigned int)((double) dpte_group_bytes[k] /
+							(double) PTERequestSizeC[k] * PixelPTEReqHeightC[k]);
 				}
 
 				if (use_one_row_for_frame[k]) {
-					dpte_groups_per_row_chroma_ub = dml_ceil((double) dpte_row_width_chroma_ub[k] /
+					dpte_groups_per_row_chroma_ub = (unsigned int)dml_ceil((double) dpte_row_width_chroma_ub[k] /
 							(double) dpte_group_width_chroma / 2.0, 1.0);
 				} else {
-					dpte_groups_per_row_chroma_ub = dml_ceil((double) dpte_row_width_chroma_ub[k] /
+					dpte_groups_per_row_chroma_ub = (unsigned int)dml_ceil((double) dpte_row_width_chroma_ub[k] /
 							(double) dpte_group_width_chroma, 1.0);
 				}
 #ifdef __DML_VBA_DEBUG__
@@ -5210,32 +5210,32 @@ void dml32_CalculateVMGroupAndRequestTimes(
 		if (GPUVMEnable == true && (DCCEnable[k] == true || GPUVMMaxPageTableLevels > 1)) {
 			if (DCCEnable[k] == false) {
 				if (BytePerPixelC[k] > 0) {
-					num_group_per_lower_vm_stage = dml_ceil(
+					num_group_per_lower_vm_stage = (unsigned int)(dml_ceil(
 							(double) (dpde0_bytes_per_frame_ub_l[k]) /
 							(double) (vm_group_bytes[k]), 1.0) +
 							dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) /
-							(double) (vm_group_bytes[k]), 1.0);
+							(double) (vm_group_bytes[k]), 1.0));
 				} else {
-					num_group_per_lower_vm_stage = dml_ceil(
+					num_group_per_lower_vm_stage = (unsigned int)dml_ceil(
 							(double) (dpde0_bytes_per_frame_ub_l[k]) /
 							(double) (vm_group_bytes[k]), 1.0);
 				}
 			} else {
 				if (GPUVMMaxPageTableLevels == 1) {
 					if (BytePerPixelC[k] > 0) {
-						num_group_per_lower_vm_stage = dml_ceil(
+						num_group_per_lower_vm_stage = (unsigned int)(dml_ceil(
 							(double) (meta_pte_bytes_per_frame_ub_l[k]) /
 							(double) (vm_group_bytes[k]), 1.0) +
 							dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) /
-							(double) (vm_group_bytes[k]), 1.0);
+							(double) (vm_group_bytes[k]), 1.0));
 					} else {
-						num_group_per_lower_vm_stage = dml_ceil(
+						num_group_per_lower_vm_stage = (unsigned int)dml_ceil(
 								(double) (meta_pte_bytes_per_frame_ub_l[k]) /
 								(double) (vm_group_bytes[k]), 1.0);
 					}
 				} else {
 					if (BytePerPixelC[k] > 0) {
-						num_group_per_lower_vm_stage = 2 + dml_ceil(
+						num_group_per_lower_vm_stage = (unsigned int)(2 + dml_ceil(
 							(double) (dpde0_bytes_per_frame_ub_l[k]) /
 							(double) (vm_group_bytes[k]), 1) +
 							dml_ceil((double) (dpde0_bytes_per_frame_ub_c[k]) /
@@ -5243,13 +5243,13 @@ void dml32_CalculateVMGroupAndRequestTimes(
 							dml_ceil((double) (meta_pte_bytes_per_frame_ub_l[k]) /
 							(double) (vm_group_bytes[k]), 1) +
 							dml_ceil((double) (meta_pte_bytes_per_frame_ub_c[k]) /
-							(double) (vm_group_bytes[k]), 1);
+							(double) (vm_group_bytes[k]), 1));
 					} else {
-						num_group_per_lower_vm_stage = 1 + dml_ceil(
+						num_group_per_lower_vm_stage = (unsigned int)(1 + dml_ceil(
 							(double) (dpde0_bytes_per_frame_ub_l[k]) /
 							(double) (vm_group_bytes[k]), 1) + dml_ceil(
 							(double) (meta_pte_bytes_per_frame_ub_l[k]) /
-							(double) (vm_group_bytes[k]), 1);
+							(double) (vm_group_bytes[k]), 1));
 					}
 				}
 			}
@@ -5424,8 +5424,8 @@ void dml32_CalculateDCCConfiguration(
 
 	MAS_vp_horz_limit = SourcePixelFormat == dm_rgbe_alpha ? 3840 : 6144;
 	MAS_vp_vert_limit = SourcePixelFormat == dm_rgbe_alpha ? 3840 : (BytePerPixelY == 8 ? 3072 : 6144);
-	max_vp_horz_width = dml_min((double) MAS_vp_horz_limit, detile_buf_vp_horz_limit);
-	max_vp_vert_height = dml_min((double) MAS_vp_vert_limit, detile_buf_vp_vert_limit);
+	max_vp_horz_width = (unsigned int)dml_min((double) MAS_vp_horz_limit, detile_buf_vp_horz_limit);
+	max_vp_vert_height = (unsigned int)dml_min((double) MAS_vp_vert_limit, detile_buf_vp_vert_limit);
 	eff_surf_width_l =  (SurfaceWidthLuma > max_vp_horz_width ? max_vp_horz_width : SurfaceWidthLuma);
 	eff_surf_width_c =  eff_surf_width_l / (1 + yuv420);
 	eff_surf_height_l =  (SurfaceHeightLuma > max_vp_vert_height ? max_vp_vert_height : SurfaceHeightLuma);
@@ -5442,10 +5442,10 @@ void dml32_CalculateDCCConfiguration(
 	}
 
 	if (SourcePixelFormat == dm_420_10) {
-		full_swath_bytes_horz_wc_l = dml_ceil((double) full_swath_bytes_horz_wc_l * 2.0 / 3.0, 256.0);
-		full_swath_bytes_horz_wc_c = dml_ceil((double) full_swath_bytes_horz_wc_c * 2.0 / 3.0, 256.0);
-		full_swath_bytes_vert_wc_l = dml_ceil((double) full_swath_bytes_vert_wc_l * 2.0 / 3.0, 256.0);
-		full_swath_bytes_vert_wc_c = dml_ceil((double) full_swath_bytes_vert_wc_c * 2.0 / 3.0, 256.0);
+		full_swath_bytes_horz_wc_l = (unsigned int)dml_ceil((double) full_swath_bytes_horz_wc_l * 2.0 / 3.0, 256.0);
+		full_swath_bytes_horz_wc_c = (unsigned int)dml_ceil((double) full_swath_bytes_horz_wc_c * 2.0 / 3.0, 256.0);
+		full_swath_bytes_vert_wc_l = (unsigned int)dml_ceil((double) full_swath_bytes_vert_wc_l * 2.0 / 3.0, 256.0);
+		full_swath_bytes_vert_wc_c = (unsigned int)dml_ceil((double) full_swath_bytes_vert_wc_c * 2.0 / 3.0, 256.0);
 	}
 
 	if (2 * full_swath_bytes_horz_wc_l + 2 * full_swath_bytes_horz_wc_c <= DETBufferSizeForDCC) {
@@ -5631,7 +5631,7 @@ void dml32_CalculateStutterEfficiency(
 		bool   Interlace[],
 		double    MinTTUVBlank[],
 		unsigned int   DPPPerSurface[],
-		unsigned int      DETBufferSizeY[],
+		unsigned int   DETBufferSizeY[],
 		unsigned int   BytePerPixelY[],
 		double    BytePerPixelDETY[],
 		double      SwathWidthY[],
@@ -5663,10 +5663,10 @@ void dml32_CalculateStutterEfficiency(
 		/* Output */
 		double   *StutterEfficiencyNotIncludingVBlank,
 		double   *StutterEfficiency,
-		unsigned int     *NumberOfStutterBurstsPerFrame,
+		int      *NumberOfStutterBurstsPerFrame,
 		double   *Z8StutterEfficiencyNotIncludingVBlank,
 		double   *Z8StutterEfficiency,
-		unsigned int     *Z8NumberOfStutterBurstsPerFrame,
+		int      *Z8NumberOfStutterBurstsPerFrame,
 		double   *StutterPeriod,
 		bool  *DCHUBBUB_ARB_CSTATE_MAX_CAP_MODE)
 {
@@ -5994,10 +5994,10 @@ void dml32_CalculateStutterEfficiency(
 				1 - (SRExitTime + StutterBurstTime) / *StutterPeriod) * 100;
 		*Z8StutterEfficiencyNotIncludingVBlank = dml_max(0.,
 				1 - (SRExitZ8Time + StutterBurstTime) / *StutterPeriod) * 100;
-		*NumberOfStutterBurstsPerFrame = (
+		*NumberOfStutterBurstsPerFrame = (unsigned int)(
 				*StutterEfficiencyNotIncludingVBlank > 0 ?
 						dml_ceil(VActiveTimeCriticalSurface / *StutterPeriod, 1) : 0);
-		*Z8NumberOfStutterBurstsPerFrame = (
+		*Z8NumberOfStutterBurstsPerFrame = (unsigned int)(
 				*Z8StutterEfficiencyNotIncludingVBlank > 0 ?
 						dml_ceil(VActiveTimeCriticalSurface / *StutterPeriod, 1) : 0);
 	} else {
@@ -6073,11 +6073,11 @@ void dml32_CalculateStutterEfficiency(
 	dml_print("DML::%s: Z8NumberOfStutterBurstsPerFrame = %d\n", __func__, *Z8NumberOfStutterBurstsPerFrame);
 #endif
 
-	SwathSizeCriticalSurface = BytePerPixelYCriticalSurface * SwathHeightYCriticalSurface
-			* dml_ceil(SwathWidthYCriticalSurface, BlockWidth256BytesYCriticalSurface);
+	SwathSizeCriticalSurface = (unsigned int)(BytePerPixelYCriticalSurface * SwathHeightYCriticalSurface
+			* dml_ceil(SwathWidthYCriticalSurface, BlockWidth256BytesYCriticalSurface));
 	LastChunkOfSwathSize = SwathSizeCriticalSurface % (PixelChunkSizeInKByte * 1024);
-	MissingPartOfLastSwathOfDETSize = dml_ceil(DETBufferSizeYCriticalSurface, SwathSizeCriticalSurface)
-			- DETBufferSizeYCriticalSurface;
+	MissingPartOfLastSwathOfDETSize = (unsigned int)(dml_ceil(DETBufferSizeYCriticalSurface, SwathSizeCriticalSurface)
+			- DETBufferSizeYCriticalSurface);
 
 	*DCHUBBUB_ARB_CSTATE_MAX_CAP_MODE = !(!UnboundedRequestEnabled && (NumberOfActiveSurfaces == 1)
 			&& doublePlaneCriticalSurface && doublePipeCriticalSurface && (LastChunkOfSwathSize > 0)
@@ -6107,9 +6107,9 @@ void dml32_CalculateMaxDETAndMinCompressedBufferSize(
 	bool     det_buff_size_override_en  = nomDETInKByteOverrideEnable;
 	unsigned int        det_buff_size_override_val = nomDETInKByteOverrideValue;
 
-	*MaxTotalDETInKByte = dml_ceil(((double)ConfigReturnBufferSizeInKByte +
+	*MaxTotalDETInKByte = (unsigned int)dml_ceil(((double)ConfigReturnBufferSizeInKByte +
 			(double) ROBBufferSizeInKByte) * 4.0 / 5.0, 64);
-	*nomDETInKByte = dml_floor((double) *MaxTotalDETInKByte / (double) MaxNumDPP, 64);
+	*nomDETInKByte = (unsigned int)dml_floor((double) *MaxTotalDETInKByte / (double) MaxNumDPP, 64);
 	*MinCompressedBufferSizeInKByte = ConfigReturnBufferSizeInKByte - *MaxTotalDETInKByte;
 
 #ifdef __DML_VBA_DEBUG__
@@ -6318,7 +6318,7 @@ bool dml32_CalculateDETSwathFillLatencyHiding(unsigned int NumberOfActiveSurface
 		enum dm_use_mall_for_pstate_change_mode UsesMALLForPStateChange[],
 		enum unbounded_requesting_policy UseUnboundedRequesting)
 {
-	int k;
+	unsigned int k;
 	double SwathSizeAllSurfaces = 0;
 	double SwathSizeAllSurfacesInFetchTimeUs;
 	double DETSwathLatencyHidingUs;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h
index 5d34735df83db..9ea36f3ff27c1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h
@@ -614,7 +614,7 @@ void dml32_UseMinimumDCFCLK(
 		unsigned int VTotal[],
 		unsigned int VActive[],
 		unsigned int DynamicMetadataTransmittedBytes[],
-		unsigned int DynamicMetadataLinesBeforeActiveRequired[],
+		int DynamicMetadataLinesBeforeActiveRequired[],
 		bool Interlace[],
 		double RequiredDPPCLKPerSurface[][2][DC__NUM_DPP__MAX],
 		double RequiredDISPCLK[][2],
@@ -1013,7 +1013,7 @@ void dml32_CalculateStutterEfficiency(
 		bool   ProgressiveToInterlaceUnitInOPP,
 		bool   Interlace[],
 		double    MinTTUVBlank[],
-		unsigned int   DPPPerSurface[],
+		unsigned int    DPPPerSurface[],
 		unsigned int      DETBufferSizeY[],
 		unsigned int   BytePerPixelY[],
 		double    BytePerPixelDETY[],
@@ -1046,10 +1046,10 @@ void dml32_CalculateStutterEfficiency(
 		/* Output */
 		double   *StutterEfficiencyNotIncludingVBlank,
 		double   *StutterEfficiency,
-		unsigned int     *NumberOfStutterBurstsPerFrame,
+		int      *NumberOfStutterBurstsPerFrame,
 		double   *Z8StutterEfficiencyNotIncludingVBlank,
 		double   *Z8StutterEfficiency,
-		unsigned int     *Z8NumberOfStutterBurstsPerFrame,
+		int      *Z8NumberOfStutterBurstsPerFrame,
 		double   *StutterPeriod,
 		bool  *DCHUBBUB_ARB_CSTATE_MAX_CAP_MODE);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c
index 6c75aa82327ac..bde043373d5d7 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_rq_dlg_calc_32.c
@@ -73,17 +73,17 @@ void dml32_rq_dlg_get_rq_reg(display_rq_regs_st *rq_regs,
 
 	dml_print("DML_DLG::%s: Calculation for pipe[%d] start, num_pipes=%d\n", __func__, pipe_idx, num_pipes);
 
-	pixel_chunk_bytes = get_pixel_chunk_size_in_kbyte(mode_lib, e2e_pipe_param, num_pipes) * 1024; // From VBA
-	min_pixel_chunk_bytes = get_min_pixel_chunk_size_in_byte(mode_lib, e2e_pipe_param, num_pipes); // From VBA
+	pixel_chunk_bytes = (uint32_t)(get_pixel_chunk_size_in_kbyte(mode_lib, e2e_pipe_param, num_pipes) * 1024); // From VBA
+	min_pixel_chunk_bytes = (uint32_t)get_min_pixel_chunk_size_in_byte(mode_lib, e2e_pipe_param, num_pipes); // From VBA
 
 	if (pixel_chunk_bytes == 64 * 1024)
 		min_pixel_chunk_bytes = 0;
 
-	meta_chunk_bytes = get_meta_chunk_size_in_kbyte(mode_lib, e2e_pipe_param, num_pipes) * 1024; // From VBA
-	min_meta_chunk_bytes = get_min_meta_chunk_size_in_byte(mode_lib, e2e_pipe_param, num_pipes); // From VBA
+	meta_chunk_bytes = (uint32_t)(get_meta_chunk_size_in_kbyte(mode_lib, e2e_pipe_param, num_pipes) * 1024); // From VBA
+	min_meta_chunk_bytes = (uint32_t)get_min_meta_chunk_size_in_byte(mode_lib, e2e_pipe_param, num_pipes); // From VBA
 
-	dpte_group_bytes = get_dpte_group_size_in_bytes(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
-	mpte_group_bytes = get_vm_group_size_in_bytes(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
+	dpte_group_bytes = (uint32_t)get_dpte_group_size_in_bytes(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
+	mpte_group_bytes = (uint32_t)get_vm_group_size_in_bytes(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
 
 	p1_pixel_chunk_bytes = pixel_chunk_bytes;
 	p1_min_pixel_chunk_bytes = min_pixel_chunk_bytes;
@@ -93,10 +93,10 @@ void dml32_rq_dlg_get_rq_reg(display_rq_regs_st *rq_regs,
 	p1_mpte_group_bytes = mpte_group_bytes;
 
 	if ((enum source_format_class) src->source_format == dm_rgbe_alpha)
-		p1_pixel_chunk_bytes = get_alpha_pixel_chunk_size_in_kbyte(mode_lib, e2e_pipe_param, num_pipes) * 1024;
+		p1_pixel_chunk_bytes = (uint32_t)(get_alpha_pixel_chunk_size_in_kbyte(mode_lib, e2e_pipe_param, num_pipes) * 1024);
 
-	rq_regs->rq_regs_l.chunk_size = dml_log2(pixel_chunk_bytes) - 10;
-	rq_regs->rq_regs_c.chunk_size = dml_log2(p1_pixel_chunk_bytes) - 10;
+	rq_regs->rq_regs_l.chunk_size = (uint32_t)(dml_log2(pixel_chunk_bytes) - 10);
+	rq_regs->rq_regs_c.chunk_size = (uint32_t)(dml_log2(p1_pixel_chunk_bytes) - 10);
 
 	if (min_pixel_chunk_bytes == 0)
 		rq_regs->rq_regs_l.min_chunk_size = 0;
@@ -108,8 +108,8 @@ void dml32_rq_dlg_get_rq_reg(display_rq_regs_st *rq_regs,
 	else
 		rq_regs->rq_regs_c.min_chunk_size = dml_log2(p1_min_pixel_chunk_bytes) - 8 + 1;
 
-	rq_regs->rq_regs_l.meta_chunk_size = dml_log2(meta_chunk_bytes) - 10;
-	rq_regs->rq_regs_c.meta_chunk_size = dml_log2(p1_meta_chunk_bytes) - 10;
+	rq_regs->rq_regs_l.meta_chunk_size = (uint32_t)(dml_log2(meta_chunk_bytes) - 10);
+	rq_regs->rq_regs_c.meta_chunk_size = (uint32_t)(dml_log2(p1_meta_chunk_bytes) - 10);
 
 	if (min_meta_chunk_bytes == 0)
 		rq_regs->rq_regs_l.min_meta_chunk_size = 0;
@@ -121,32 +121,32 @@ void dml32_rq_dlg_get_rq_reg(display_rq_regs_st *rq_regs,
 	else
 		rq_regs->rq_regs_c.min_meta_chunk_size = dml_log2(p1_min_meta_chunk_bytes) - 6 + 1;
 
-	rq_regs->rq_regs_l.dpte_group_size = dml_log2(dpte_group_bytes) - 6;
+	rq_regs->rq_regs_l.dpte_group_size = (uint32_t)(dml_log2(dpte_group_bytes) - 6);
 	rq_regs->rq_regs_l.mpte_group_size = dml_log2(mpte_group_bytes) - 6;
 	rq_regs->rq_regs_c.dpte_group_size = dml_log2(p1_dpte_group_bytes) - 6;
 	rq_regs->rq_regs_c.mpte_group_size = dml_log2(p1_mpte_group_bytes) - 6;
 
-	detile_buf_size_in_bytes = get_det_buffer_size_kbytes(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * 1024;
+	detile_buf_size_in_bytes = (unsigned int)(get_det_buffer_size_kbytes(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * 1024);
 	detile_buf_plane1_addr = 0;
-	pte_row_height_linear = get_dpte_row_height_linear_l(mode_lib, e2e_pipe_param, num_pipes,
+	pte_row_height_linear = (unsigned int)get_dpte_row_height_linear_l(mode_lib, e2e_pipe_param, num_pipes,
 			pipe_idx);
 
 	if (src->sw_mode == dm_sw_linear)
 		ASSERT(pte_row_height_linear >= 8);
 
-	rq_regs->rq_regs_l.pte_row_height_linear = dml_floor(dml_log2(pte_row_height_linear), 1) - 3;
+	rq_regs->rq_regs_l.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(pte_row_height_linear), 1) - 3);
 
 	if (dual_plane) {
-		unsigned int p1_pte_row_height_linear = get_dpte_row_height_linear_c(mode_lib, e2e_pipe_param,
+		unsigned int p1_pte_row_height_linear = (unsigned int)get_dpte_row_height_linear_c(mode_lib, e2e_pipe_param,
 				num_pipes, pipe_idx);
 		if (src->sw_mode == dm_sw_linear)
 			ASSERT(p1_pte_row_height_linear >= 8);
 
-		rq_regs->rq_regs_c.pte_row_height_linear = dml_floor(dml_log2(p1_pte_row_height_linear), 1) - 3;
+		rq_regs->rq_regs_c.pte_row_height_linear = (unsigned int)(dml_floor(dml_log2(p1_pte_row_height_linear), 1) - 3);
 	}
 
-	rq_regs->rq_regs_l.swath_height = dml_log2(get_swath_height_l(mode_lib, e2e_pipe_param, num_pipes, pipe_idx));
-	rq_regs->rq_regs_c.swath_height = dml_log2(get_swath_height_c(mode_lib, e2e_pipe_param, num_pipes, pipe_idx));
+	rq_regs->rq_regs_l.swath_height = (unsigned int)dml_log2(get_swath_height_l(mode_lib, e2e_pipe_param, num_pipes, pipe_idx));
+	rq_regs->rq_regs_c.swath_height = (unsigned int)dml_log2(get_swath_height_c(mode_lib, e2e_pipe_param, num_pipes, pipe_idx));
 
 	// FIXME: take the max between luma, chroma chunk size?
 	// okay for now, as we are setting pixel_chunk_bytes to 8kb anyways
@@ -168,19 +168,19 @@ void dml32_rq_dlg_get_rq_reg(display_rq_regs_st *rq_regs,
 	// Note: detile_buf_plane1_addr is in unit of 1KB
 	if (dual_plane) {
 		if (is_phantom_pipe) {
-			detile_buf_plane1_addr = ((1024.0 * 1024.0) / 2.0 / 1024.0); // half to chroma
+			detile_buf_plane1_addr = (unsigned int)((1024.0 * 1024.0) / 2.0 / 1024.0); // half to chroma
 		} else {
 			if (stored_swath_l_bytes / stored_swath_c_bytes <= 1.5) {
-				detile_buf_plane1_addr = (detile_buf_size_in_bytes / 2.0 / 1024.0); // half to chroma
+				detile_buf_plane1_addr = (unsigned int)(detile_buf_size_in_bytes / 2.0 / 1024.0); // half to chroma
 #ifdef __DML_RQ_DLG_CALC_DEBUG__
 				dml_print("DML_DLG: %s: detile_buf_plane1_addr = %d (1/2 to chroma)\n",
 						__func__, detile_buf_plane1_addr);
 #endif
 			} else {
-				detile_buf_plane1_addr =
+				detile_buf_plane1_addr = (unsigned int)(
 						dml_round_to_multiple(
 								(unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0),
-								1024, 0) / 1024.0; // 2/3 to luma
+								1024, 0) / 1024.0); // 2/3 to luma
 #ifdef __DML_RQ_DLG_CALC_DEBUG__
 				dml_print("DML_DLG: %s: detile_buf_plane1_addr = %d (1/3 chroma)\n",
 						__func__, detile_buf_plane1_addr);
@@ -276,7 +276,7 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	dlg_regs->dlg_vblank_end = interlaced ? (vblank_end / 2) : vblank_end; // 15 bits
 
 	min_ttu_vblank = get_min_ttu_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx); // From VBA
-	min_dst_y_next_start = get_min_dst_y_next_start(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
+	min_dst_y_next_start = (unsigned int)get_min_dst_y_next_start(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
 
 	dml_print("DML_DLG: %s: min_ttu_vblank (us)    = %3.2f\n", __func__, min_ttu_vblank);
 	dml_print("DML_DLG: %s: min_dst_y_next_start = %d\n", __func__, min_dst_y_next_start);
@@ -284,14 +284,14 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 
 	dual_plane = is_dual_plane((enum source_format_class) (src->source_format));
 
-	vready_after_vcount0 = get_vready_at_or_after_vsync(mode_lib, e2e_pipe_param, num_pipes,
+	vready_after_vcount0 = (unsigned int)get_vready_at_or_after_vsync(mode_lib, e2e_pipe_param, num_pipes,
 			pipe_idx); // From VBA
 	dlg_regs->vready_after_vcount0 = vready_after_vcount0;
 
 	dml_print("DML_DLG: %s: vready_after_vcount0 = %d\n", __func__, dlg_regs->vready_after_vcount0);
 
-	dst_x_after_scaler = dml_ceil(get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx), 1);
-	dst_y_after_scaler = dml_ceil(get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx), 1);
+	dst_x_after_scaler = (unsigned int)dml_ceil(get_dst_x_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx), 1);
+	dst_y_after_scaler = (unsigned int)dml_ceil(get_dst_y_after_scaler(mode_lib, e2e_pipe_param, num_pipes, pipe_idx), 1);
 
 	// do some adjustment on the dst_after scaler to account for odm combine mode
 	dml_print("DML_DLG: %s: input dst_x_after_scaler   = %d\n", __func__, dst_x_after_scaler);
@@ -398,15 +398,15 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	}
 
 	if (src->dynamic_metadata_enable && src->gpuvm)
-		dlg_regs->refcyc_per_vm_dmdata = get_refcyc_per_vm_dmdata_in_us(mode_lib, e2e_pipe_param, num_pipes,
-				pipe_idx) * refclk_freq_in_mhz; // From VBA
+		dlg_regs->refcyc_per_vm_dmdata = (unsigned int)(get_refcyc_per_vm_dmdata_in_us(mode_lib, e2e_pipe_param, num_pipes,
+				pipe_idx) * refclk_freq_in_mhz); // From VBA
 
-	dlg_regs->dmdata_dl_delta = get_dmdata_dl_delta_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx)
-		* refclk_freq_in_mhz; // From VBA
+	dlg_regs->dmdata_dl_delta = (unsigned int)(get_dmdata_dl_delta_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx)
+		* refclk_freq_in_mhz); // From VBA
 
-	refcyc_per_req_delivery_pre_l = get_refcyc_per_req_delivery_pre_l_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz; // From VBA
-	refcyc_per_req_delivery_l = get_refcyc_per_req_delivery_l_in_us(mode_lib, e2e_pipe_param, num_pipes,
-			pipe_idx) * refclk_freq_in_mhz;     // From VBA
+	refcyc_per_req_delivery_pre_l = (unsigned int)(get_refcyc_per_req_delivery_pre_l_in_us(mode_lib, e2e_pipe_param, num_pipes, pipe_idx) * refclk_freq_in_mhz); // From VBA
+	refcyc_per_req_delivery_l = (unsigned int)(get_refcyc_per_req_delivery_l_in_us(mode_lib, e2e_pipe_param, num_pipes,
+			pipe_idx) * refclk_freq_in_mhz);     // From VBA
 
 	dml_print("DML_DLG: %s: refcyc_per_req_delivery_pre_l = %3.2f\n", __func__, refcyc_per_req_delivery_pre_l);
 	dml_print("DML_DLG: %s: refcyc_per_req_delivery_l     = %3.2f\n", __func__, refcyc_per_req_delivery_l);
@@ -437,11 +437,11 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	}
 
 	// Assign to register structures
-	dlg_regs->min_dst_y_next_start = min_dst_y_next_start * dml_pow(2, 2);
+	dlg_regs->min_dst_y_next_start = (unsigned int)(min_dst_y_next_start * dml_pow(2, 2));
 	ASSERT(dlg_regs->min_dst_y_next_start < (unsigned int)dml_pow(2, 18));
 
 	dlg_regs->dst_y_after_scaler = dst_y_after_scaler; // in terms of line
-	dlg_regs->refcyc_x_after_scaler = dst_x_after_scaler * ref_freq_to_pix_freq; // in terms of refclk
+	dlg_regs->refcyc_x_after_scaler = (unsigned int)(dst_x_after_scaler * ref_freq_to_pix_freq); // in terms of refclk
 	dlg_regs->dst_y_prefetch = (unsigned int) (dst_y_prefetch * dml_pow(2, 2));
 	dlg_regs->dst_y_per_vm_vblank = (unsigned int) (dst_y_per_vm_vblank * dml_pow(2, 2));
 	dlg_regs->dst_y_per_row_vblank = (unsigned int) (dst_y_per_row_vblank * dml_pow(2, 2));
@@ -456,14 +456,14 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	dml_print("DML_DLG: %s: dlg_regs->dst_y_per_vm_flip    = 0x%x\n", __func__, dlg_regs->dst_y_per_vm_flip);
 	dml_print("DML_DLG: %s: dlg_regs->dst_y_per_row_flip   = 0x%x\n", __func__, dlg_regs->dst_y_per_row_flip);
 
-	dlg_regs->refcyc_per_vm_group_vblank = get_refcyc_per_vm_group_vblank_in_us(mode_lib, e2e_pipe_param,
-			num_pipes, pipe_idx) * refclk_freq_in_mhz;               // From VBA
-	dlg_regs->refcyc_per_vm_group_flip = get_refcyc_per_vm_group_flip_in_us(mode_lib, e2e_pipe_param, num_pipes,
-			pipe_idx) * refclk_freq_in_mhz;                 // From VBA
-	dlg_regs->refcyc_per_vm_req_vblank = get_refcyc_per_vm_req_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes,
-			pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10);                 // From VBA
-	dlg_regs->refcyc_per_vm_req_flip = get_refcyc_per_vm_req_flip_in_us(mode_lib, e2e_pipe_param, num_pipes,
-			pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10);  // From VBA
+	dlg_regs->refcyc_per_vm_group_vblank = (unsigned int)(get_refcyc_per_vm_group_vblank_in_us(mode_lib, e2e_pipe_param,
+			num_pipes, pipe_idx) * refclk_freq_in_mhz);               // From VBA
+	dlg_regs->refcyc_per_vm_group_flip = (unsigned int)(get_refcyc_per_vm_group_flip_in_us(mode_lib, e2e_pipe_param, num_pipes,
+			pipe_idx) * refclk_freq_in_mhz);                 // From VBA
+	dlg_regs->refcyc_per_vm_req_vblank = (unsigned int)(get_refcyc_per_vm_req_vblank_in_us(mode_lib, e2e_pipe_param, num_pipes,
+			pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10));                 // From VBA
+	dlg_regs->refcyc_per_vm_req_flip = (unsigned int)(get_refcyc_per_vm_req_flip_in_us(mode_lib, e2e_pipe_param, num_pipes,
+			pipe_idx) * refclk_freq_in_mhz * dml_pow(2, 10));  // From VBA
 
 	// From VBA
 	dst_y_per_pte_row_nom_l = get_dst_y_per_pte_row_nom_l(mode_lib, e2e_pipe_param, num_pipes, pipe_idx);
@@ -500,22 +500,22 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	refcyc_per_meta_chunk_flip_c = get_refcyc_per_meta_chunk_flip_c_in_us(mode_lib, e2e_pipe_param,
 			num_pipes, pipe_idx) * refclk_freq_in_mhz;       // From VBA
 
-	dlg_regs->dst_y_per_pte_row_nom_l = dst_y_per_pte_row_nom_l * dml_pow(2, 2);
-	dlg_regs->dst_y_per_pte_row_nom_c = dst_y_per_pte_row_nom_c * dml_pow(2, 2);
-	dlg_regs->dst_y_per_meta_row_nom_l = dst_y_per_meta_row_nom_l * dml_pow(2, 2);
-	dlg_regs->dst_y_per_meta_row_nom_c = dst_y_per_meta_row_nom_c * dml_pow(2, 2);
-	dlg_regs->refcyc_per_pte_group_nom_l = refcyc_per_pte_group_nom_l;
-	dlg_regs->refcyc_per_pte_group_nom_c = refcyc_per_pte_group_nom_c;
-	dlg_regs->refcyc_per_pte_group_vblank_l = refcyc_per_pte_group_vblank_l;
-	dlg_regs->refcyc_per_pte_group_vblank_c = refcyc_per_pte_group_vblank_c;
-	dlg_regs->refcyc_per_pte_group_flip_l = refcyc_per_pte_group_flip_l;
-	dlg_regs->refcyc_per_pte_group_flip_c = refcyc_per_pte_group_flip_c;
-	dlg_regs->refcyc_per_meta_chunk_nom_l = refcyc_per_meta_chunk_nom_l;
-	dlg_regs->refcyc_per_meta_chunk_nom_c = refcyc_per_meta_chunk_nom_c;
-	dlg_regs->refcyc_per_meta_chunk_vblank_l = refcyc_per_meta_chunk_vblank_l;
-	dlg_regs->refcyc_per_meta_chunk_vblank_c = refcyc_per_meta_chunk_vblank_c;
-	dlg_regs->refcyc_per_meta_chunk_flip_l = refcyc_per_meta_chunk_flip_l;
-	dlg_regs->refcyc_per_meta_chunk_flip_c = refcyc_per_meta_chunk_flip_c;
+	dlg_regs->dst_y_per_pte_row_nom_l = (unsigned int)(dst_y_per_pte_row_nom_l * dml_pow(2, 2));
+	dlg_regs->dst_y_per_pte_row_nom_c = (unsigned int)(dst_y_per_pte_row_nom_c * dml_pow(2, 2));
+	dlg_regs->dst_y_per_meta_row_nom_l = (unsigned int)(dst_y_per_meta_row_nom_l * dml_pow(2, 2));
+	dlg_regs->dst_y_per_meta_row_nom_c = (unsigned int)(dst_y_per_meta_row_nom_c * dml_pow(2, 2));
+	dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)refcyc_per_pte_group_nom_l;
+	dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int)refcyc_per_pte_group_nom_c;
+	dlg_regs->refcyc_per_pte_group_vblank_l = (unsigned int)refcyc_per_pte_group_vblank_l;
+	dlg_regs->refcyc_per_pte_group_vblank_c = (unsigned int)refcyc_per_pte_group_vblank_c;
+	dlg_regs->refcyc_per_pte_group_flip_l = (unsigned int)refcyc_per_pte_group_flip_l;
+	dlg_regs->refcyc_per_pte_group_flip_c = (unsigned int)refcyc_per_pte_group_flip_c;
+	dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)refcyc_per_meta_chunk_nom_l;
+	dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int)refcyc_per_meta_chunk_nom_c;
+	dlg_regs->refcyc_per_meta_chunk_vblank_l = (unsigned int)refcyc_per_meta_chunk_vblank_l;
+	dlg_regs->refcyc_per_meta_chunk_vblank_c = (unsigned int)refcyc_per_meta_chunk_vblank_c;
+	dlg_regs->refcyc_per_meta_chunk_flip_l = (unsigned int)refcyc_per_meta_chunk_flip_l;
+	dlg_regs->refcyc_per_meta_chunk_flip_c = (unsigned int)refcyc_per_meta_chunk_flip_c;
 	dlg_regs->refcyc_per_line_delivery_pre_l = (unsigned int) dml_floor(refcyc_per_line_delivery_pre_l, 1);
 	dlg_regs->refcyc_per_line_delivery_l = (unsigned int) dml_floor(refcyc_per_line_delivery_l, 1);
 	dlg_regs->refcyc_per_line_delivery_pre_c = (unsigned int) dml_floor(refcyc_per_line_delivery_pre_c, 1);
@@ -548,7 +548,7 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	ttu_regs->qos_ramp_disable_l = 0;
 	ttu_regs->qos_ramp_disable_c = 0;
 	ttu_regs->qos_ramp_disable_cur0 = 0;
-	ttu_regs->min_ttu_vblank = min_ttu_vblank * refclk_freq_in_mhz;
+	ttu_regs->min_ttu_vblank = (unsigned int)(min_ttu_vblank * refclk_freq_in_mhz);
 
 	// CHECK for HW registers' range, assert or clamp
 	ASSERT(refcyc_per_req_delivery_pre_l < dml_pow(2, 13));
@@ -556,16 +556,16 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	ASSERT(refcyc_per_req_delivery_pre_c < dml_pow(2, 13));
 	ASSERT(refcyc_per_req_delivery_c < dml_pow(2, 13));
 	if (dlg_regs->refcyc_per_vm_group_vblank >= (unsigned int) dml_pow(2, 23))
-		dlg_regs->refcyc_per_vm_group_vblank = dml_pow(2, 23) - 1;
+		dlg_regs->refcyc_per_vm_group_vblank = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (dlg_regs->refcyc_per_vm_group_flip >= (unsigned int) dml_pow(2, 23))
-		dlg_regs->refcyc_per_vm_group_flip = dml_pow(2, 23) - 1;
+		dlg_regs->refcyc_per_vm_group_flip = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (dlg_regs->refcyc_per_vm_req_vblank >= (unsigned int) dml_pow(2, 23))
-		dlg_regs->refcyc_per_vm_req_vblank = dml_pow(2, 23) - 1;
+		dlg_regs->refcyc_per_vm_req_vblank = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (dlg_regs->refcyc_per_vm_req_flip >= (unsigned int) dml_pow(2, 23))
-		dlg_regs->refcyc_per_vm_req_flip = dml_pow(2, 23) - 1;
+		dlg_regs->refcyc_per_vm_req_flip = (unsigned int)(dml_pow(2, 23) - 1);
 
 	ASSERT(dlg_regs->dst_y_after_scaler < (unsigned int) 8);
 	ASSERT(dlg_regs->refcyc_x_after_scaler < (unsigned int)dml_pow(2, 13));
@@ -581,10 +581,10 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	ASSERT(dlg_regs->dst_y_per_meta_row_nom_c < (unsigned int)dml_pow(2, 17));
 
 	if (dlg_regs->refcyc_per_pte_group_nom_l >= (unsigned int) dml_pow(2, 23))
-		dlg_regs->refcyc_per_pte_group_nom_l = dml_pow(2, 23) - 1;
+		dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 	if (dual_plane) {
 		if (dlg_regs->refcyc_per_pte_group_nom_c >= (unsigned int) dml_pow(2, 23))
-			dlg_regs->refcyc_per_pte_group_nom_c = dml_pow(2, 23) - 1;
+			dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 	}
 	ASSERT(dlg_regs->refcyc_per_pte_group_vblank_l < (unsigned int)dml_pow(2, 13));
 	if (dual_plane) {
@@ -592,10 +592,10 @@ void dml32_rq_dlg_get_dlg_reg(struct display_mode_lib *mode_lib,
 	}
 
 	if (dlg_regs->refcyc_per_meta_chunk_nom_l >= (unsigned int) dml_pow(2, 23))
-		dlg_regs->refcyc_per_meta_chunk_nom_l = dml_pow(2, 23) - 1;
+		dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 	if (dual_plane) {
 		if (dlg_regs->refcyc_per_meta_chunk_nom_c >= (unsigned int) dml_pow(2, 23))
-			dlg_regs->refcyc_per_meta_chunk_nom_c = dml_pow(2, 23) - 1;
+			dlg_regs->refcyc_per_meta_chunk_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 	}
 	ASSERT(dlg_regs->refcyc_per_meta_chunk_vblank_l < (unsigned int)dml_pow(2, 13));
 	ASSERT(dlg_regs->refcyc_per_meta_chunk_vblank_c < (unsigned int)dml_pow(2, 13));
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c
index e0a1dc89ce43f..f4d826e0d13ab 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn321/dcn321_fpu.c
@@ -162,20 +162,20 @@ struct _vcs_dpi_soc_bounding_box_st dcn3_21_soc = {
 static void get_optimal_ntuple(struct _vcs_dpi_voltage_scaling_st *entry)
 {
 	if (entry->dcfclk_mhz > 0) {
-		float bw_on_sdp = entry->dcfclk_mhz * dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_sdp_bw_after_urgent / 100);
+		float bw_on_sdp = (float)(entry->dcfclk_mhz * dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_sdp_bw_after_urgent / 100));
 
 		entry->fabricclk_mhz = bw_on_sdp / (dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_fabric_bw_after_urgent / 100));
 		entry->dram_speed_mts = bw_on_sdp / (dcn3_21_soc.num_chans *
 				dcn3_21_soc.dram_channel_width_bytes * ((float)dcn3_21_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100));
 	} else if (entry->fabricclk_mhz > 0) {
-		float bw_on_fabric = entry->fabricclk_mhz * dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_fabric_bw_after_urgent / 100);
+		float bw_on_fabric = (float)(entry->fabricclk_mhz * dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_fabric_bw_after_urgent / 100));
 
 		entry->dcfclk_mhz = bw_on_fabric / (dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_sdp_bw_after_urgent / 100));
 		entry->dram_speed_mts = bw_on_fabric / (dcn3_21_soc.num_chans *
 				dcn3_21_soc.dram_channel_width_bytes * ((float)dcn3_21_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100));
 	} else if (entry->dram_speed_mts > 0) {
-		float bw_on_dram = entry->dram_speed_mts * dcn3_21_soc.num_chans *
-				dcn3_21_soc.dram_channel_width_bytes * ((float)dcn3_21_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100);
+		float bw_on_dram = (float)(entry->dram_speed_mts * dcn3_21_soc.num_chans *
+				dcn3_21_soc.dram_channel_width_bytes * ((float)dcn3_21_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100));
 
 		entry->fabricclk_mhz = bw_on_dram / (dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_fabric_bw_after_urgent / 100));
 		entry->dcfclk_mhz = bw_on_dram / (dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_sdp_bw_after_urgent / 100));
@@ -189,12 +189,12 @@ static float calculate_net_bw_in_kbytes_sec(struct _vcs_dpi_voltage_scaling_st *
 	float sdp_bw_kbytes_sec;
 	float limiting_bw_kbytes_sec;
 
-	memory_bw_kbytes_sec = entry->dram_speed_mts * dcn3_21_soc.num_chans *
-			dcn3_21_soc.dram_channel_width_bytes * ((float)dcn3_21_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100);
+	memory_bw_kbytes_sec = (float)(entry->dram_speed_mts * dcn3_21_soc.num_chans *
+			dcn3_21_soc.dram_channel_width_bytes * ((float)dcn3_21_soc.pct_ideal_dram_sdp_bw_after_urgent_pixel_only / 100));
 
-	fabric_bw_kbytes_sec = entry->fabricclk_mhz * dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_fabric_bw_after_urgent / 100);
+	fabric_bw_kbytes_sec = (float)(entry->fabricclk_mhz * dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_fabric_bw_after_urgent / 100));
 
-	sdp_bw_kbytes_sec = entry->dcfclk_mhz * dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_sdp_bw_after_urgent / 100);
+	sdp_bw_kbytes_sec = (float)(entry->dcfclk_mhz * dcn3_21_soc.return_bus_width_bytes * ((float)dcn3_21_soc.pct_ideal_sdp_bw_after_urgent / 100));
 
 	limiting_bw_kbytes_sec = memory_bw_kbytes_sec;
 
@@ -211,8 +211,8 @@ static void dcn321_insert_entry_into_table_sorted(struct _vcs_dpi_voltage_scalin
 					   unsigned int *num_entries,
 					   struct _vcs_dpi_voltage_scaling_st *entry)
 {
-	int i = 0;
-	int index = 0;
+	unsigned int i = 0;
+	unsigned int index = 0;
 
 	dc_assert_fp_enabled();
 
@@ -237,7 +237,7 @@ static void dcn321_insert_entry_into_table_sorted(struct _vcs_dpi_voltage_scalin
 static void remove_entry_from_table_at_index(struct _vcs_dpi_voltage_scaling_st *table, unsigned int *num_entries,
 		unsigned int index)
 {
-	int i;
+	unsigned int i;
 
 	if (*num_entries == 0)
 		return;
@@ -265,19 +265,19 @@ static void sort_entries_with_same_bw(struct _vcs_dpi_voltage_scaling_st *table,
 	unsigned int end_index = 0;
 	unsigned int current_bw = 0;
 
-	for (int i = 0; i < (*num_entries - 1); i++) {
+	for (unsigned int i = 0; i + 1 < *num_entries; i++) {
 		if (table[i].net_bw_in_kbytes_sec == table[i+1].net_bw_in_kbytes_sec) {
-			current_bw = table[i].net_bw_in_kbytes_sec;
+			current_bw = (unsigned int)table[i].net_bw_in_kbytes_sec;
 			start_index = i;
 			end_index = ++i;
 
-			while ((i < (*num_entries - 1)) && (table[i+1].net_bw_in_kbytes_sec == current_bw))
+			while ((i + 1 < *num_entries) && (table[i+1].net_bw_in_kbytes_sec == current_bw))
 				end_index = ++i;
 		}
 
 		if (start_index != end_index) {
-			for (int j = start_index; j < end_index; j++) {
-				for (int k = start_index; k < end_index; k++) {
+			for (unsigned int j = start_index; j < end_index; j++) {
+				for (unsigned int k = start_index; k < end_index; k++) {
 					if (table[k].dcfclk_mhz > table[k+1].dcfclk_mhz)
 						swap_table_entries(&table[k], &table[k+1]);
 				}
@@ -296,7 +296,7 @@ static void sort_entries_with_same_bw(struct _vcs_dpi_voltage_scaling_st *table,
  */
 static void remove_inconsistent_entries(struct _vcs_dpi_voltage_scaling_st *table, unsigned int *num_entries)
 {
-	for (int i = 0; i < (*num_entries - 1); i++) {
+	for (unsigned int i = 0; i + 1 < *num_entries; i++) {
 		if (table[i].net_bw_in_kbytes_sec == table[i+1].net_bw_in_kbytes_sec) {
 			if ((table[i].dram_speed_mts > table[i+1].dram_speed_mts) ||
 				(table[i].fabricclk_mhz > table[i+1].fabricclk_mhz))
@@ -344,7 +344,8 @@ static int override_max_clk_values(struct clk_limit_table_entry *max_clk_limit,
 static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk_bw_params *bw_params,
 		struct _vcs_dpi_voltage_scaling_st *table, unsigned int *num_entries)
 {
-	int i, j;
+	int i;
+	unsigned int j, k;
 	struct _vcs_dpi_voltage_scaling_st entry = {0};
 	struct clk_limit_table_entry max_clk_data = {0};
 
@@ -414,12 +415,12 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 		max_clk_data.dppclk_mhz = max_clk_data.dispclk_mhz;
 
 	if (max_clk_data.fclk_mhz == 0)
-		max_clk_data.fclk_mhz = max_clk_data.dcfclk_mhz *
+		max_clk_data.fclk_mhz = (unsigned int)(max_clk_data.dcfclk_mhz *
 				dcn3_21_soc.pct_ideal_sdp_bw_after_urgent /
-				dcn3_21_soc.pct_ideal_fabric_bw_after_urgent;
+				dcn3_21_soc.pct_ideal_fabric_bw_after_urgent);
 
 	if (max_clk_data.phyclk_mhz == 0)
-		max_clk_data.phyclk_mhz = dcn3_21_soc.clock_limits[0].phyclk_mhz;
+		max_clk_data.phyclk_mhz = (unsigned int)dcn3_21_soc.clock_limits[0].phyclk_mhz;
 
 	*num_entries = 0;
 	entry.dispclk_mhz = max_clk_data.dispclk_mhz;
@@ -431,8 +432,8 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 	entry.phyclk_d32_mhz = dcn3_21_soc.clock_limits[0].phyclk_d32_mhz;
 
 	// Insert all the DCFCLK STAs
-	for (i = 0; i < num_dcfclk_stas; i++) {
-		entry.dcfclk_mhz = dcfclk_sta_targets[i];
+	for (k = 0; k < num_dcfclk_stas; k++) {
+		entry.dcfclk_mhz = dcfclk_sta_targets[k];
 		entry.fabricclk_mhz = 0;
 		entry.dram_speed_mts = 0;
 
@@ -451,10 +452,10 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 	dcn321_insert_entry_into_table_sorted(table, num_entries, &entry);
 
 	// Insert the UCLK DPMS
-	for (i = 0; i < num_uclk_dpms; i++) {
+	for (k = 0; k < num_uclk_dpms; k++) {
 		entry.dcfclk_mhz = 0;
 		entry.fabricclk_mhz = 0;
-		entry.dram_speed_mts = bw_params->clk_table.entries[i].memclk_mhz * 16;
+		entry.dram_speed_mts = bw_params->clk_table.entries[k].memclk_mhz * 16;
 
 		get_optimal_ntuple(&entry);
 		entry.net_bw_in_kbytes_sec = calculate_net_bw_in_kbytes_sec(&entry);
@@ -463,9 +464,9 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 
 	// If FCLK is coarse grained, insert individual DPMs.
 	if (num_fclk_dpms > 2) {
-		for (i = 0; i < num_fclk_dpms; i++) {
+		for (k = 0; k < num_fclk_dpms; k++) {
 			entry.dcfclk_mhz = 0;
-			entry.fabricclk_mhz = bw_params->clk_table.entries[i].fclk_mhz;
+			entry.fabricclk_mhz = bw_params->clk_table.entries[k].fclk_mhz;
 			entry.dram_speed_mts = 0;
 
 			get_optimal_ntuple(&entry);
@@ -489,7 +490,7 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 	// ratios (by derate, are exact).
 
 	// Remove states that require higher clocks than are supported
-	for (i = *num_entries - 1; i >= 0 ; i--) {
+	for (i = (int)*num_entries - 1; i >= 0 ; i--) {
 		if (table[i].dcfclk_mhz > max_clk_data.dcfclk_mhz ||
 				table[i].fabricclk_mhz > max_clk_data.fclk_mhz ||
 				table[i].dram_speed_mts > max_clk_data.memclk_mhz * 16)
@@ -519,7 +520,7 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 	// coarse grained DPMs and remove duplicates.
 
 	// Round up UCLKs
-	for (i = *num_entries - 1; i >= 0 ; i--) {
+	for (i = (int)*num_entries - 1; i >= 0 ; i--) {
 		for (j = 0; j < num_uclk_dpms; j++) {
 			if (bw_params->clk_table.entries[j].memclk_mhz * 16 >= table[i].dram_speed_mts) {
 				table[i].dram_speed_mts = bw_params->clk_table.entries[j].memclk_mhz * 16;
@@ -530,7 +531,7 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 
 	// If FCLK is coarse grained, round up to next DPMs
 	if (num_fclk_dpms > 2) {
-		for (i = *num_entries - 1; i >= 0 ; i--) {
+		for (i = (int)*num_entries - 1; i >= 0 ; i--) {
 			for (j = 0; j < num_fclk_dpms; j++) {
 				if (bw_params->clk_table.entries[j].fclk_mhz >= table[i].fabricclk_mhz) {
 					table[i].fabricclk_mhz = bw_params->clk_table.entries[j].fclk_mhz;
@@ -541,7 +542,7 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 	}
 	// Otherwise, round up to minimum.
 	else {
-		for (i = *num_entries - 1; i >= 0 ; i--) {
+		for (i = (int)*num_entries - 1; i >= 0 ; i--) {
 			if (table[i].fabricclk_mhz < min_fclk_mhz) {
 				table[i].fabricclk_mhz = min_fclk_mhz;
 			}
@@ -549,7 +550,7 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 	}
 
 	// Round DCFCLKs up to minimum
-	for (i = *num_entries - 1; i >= 0 ; i--) {
+	for (i = (int)*num_entries - 1; i >= 0 ; i--) {
 		if (table[i].dcfclk_mhz < min_dcfclk_mhz) {
 			table[i].dcfclk_mhz = min_dcfclk_mhz;
 		}
@@ -557,7 +558,7 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 
 	// Remove duplicate states, note duplicate states are always neighbouring since table is sorted.
 	i = 0;
-	while (i < *num_entries - 1) {
+	while (i < (int)*num_entries - 1) {
 		if (table[i].dcfclk_mhz == table[i + 1].dcfclk_mhz &&
 				table[i].fabricclk_mhz == table[i + 1].fabricclk_mhz &&
 				table[i].dram_speed_mts == table[i + 1].dram_speed_mts)
@@ -567,9 +568,8 @@ static int build_synthetic_soc_states(bool disable_dc_mode_overwrite, struct clk
 	}
 
 	// Fix up the state indicies
-	for (i = *num_entries - 1; i >= 0 ; i--) {
+	for (i = (int)*num_entries - 1; i >= 0 ; i--)
 		table[i].state = i;
-	}
 
 	return 0;
 }
@@ -588,12 +588,12 @@ static void dcn321_get_optimal_dcfclk_fclk_for_uclk(unsigned int uclk_mts,
 	bw_from_dram = (bw_from_dram1 < bw_from_dram2) ? bw_from_dram1 : bw_from_dram2;
 
 	if (optimal_fclk)
-		*optimal_fclk = bw_from_dram /
-		(dcn3_21_soc.fabric_datapath_to_dcn_data_return_bytes * (dcn3_21_soc.max_avg_sdp_bw_use_normal_percent / 100));
+		*optimal_fclk = (unsigned int)(bw_from_dram /
+		(dcn3_21_soc.fabric_datapath_to_dcn_data_return_bytes * (dcn3_21_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 
 	if (optimal_dcfclk)
-		*optimal_dcfclk =  bw_from_dram /
-		(dcn3_21_soc.return_bus_width_bytes * (dcn3_21_soc.max_avg_sdp_bw_use_normal_percent / 100));
+		*optimal_dcfclk =  (unsigned int)(bw_from_dram /
+		(dcn3_21_soc.return_bus_width_bytes * (dcn3_21_soc.max_avg_sdp_bw_use_normal_percent / 100)));
 }
 
 /** dcn321_update_bw_bounding_box
@@ -688,9 +688,12 @@ void dcn321_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p
 			dc->ctx->dc_bios->vram_info.num_chans) * dc->caps.mall_size_per_mem_channel);
 	}
 
-	if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes)
-		dc->dml2_options.bbox_overrides.dram_chanel_width_bytes =
-		dcn3_21_soc.dram_channel_width_bytes = dc->ctx->dc_bios->vram_info.dram_channel_width_bytes;
+	if (dc->ctx->dc_bios->vram_info.dram_channel_width_bytes) {
+		unsigned int dram_channel_width_bytes = (unsigned int)dc->ctx->dc_bios->vram_info.dram_channel_width_bytes;
+
+		dc->dml2_options.bbox_overrides.dram_chanel_width_bytes = dram_channel_width_bytes;
+		dcn3_21_soc.dram_channel_width_bytes = dram_channel_width_bytes;
+	}
 
 	/* DML DSC delay factor workaround */
 	dcn3_21_ip.dsc_delay_factor_wa = dc->debug.dsc_delay_factor_wa_x1000 / 1000.0;
@@ -729,13 +732,13 @@ void dcn321_update_bw_bounding_box_fpu(struct dc *dc, struct clk_bw_params *bw_p
 				max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz;
 		}
 		if (!max_dcfclk_mhz)
-			max_dcfclk_mhz = dcn3_21_soc.clock_limits[0].dcfclk_mhz;
+			max_dcfclk_mhz = (unsigned int)dcn3_21_soc.clock_limits[0].dcfclk_mhz;
 		if (!max_dispclk_mhz)
-			max_dispclk_mhz = dcn3_21_soc.clock_limits[0].dispclk_mhz;
+			max_dispclk_mhz = (unsigned int)dcn3_21_soc.clock_limits[0].dispclk_mhz;
 		if (!max_dppclk_mhz)
-			max_dppclk_mhz = dcn3_21_soc.clock_limits[0].dppclk_mhz;
+			max_dppclk_mhz = (unsigned int)dcn3_21_soc.clock_limits[0].dppclk_mhz;
 		if (!max_phyclk_mhz)
-			max_phyclk_mhz = dcn3_21_soc.clock_limits[0].phyclk_mhz;
+			max_phyclk_mhz = (unsigned int)dcn3_21_soc.clock_limits[0].phyclk_mhz;
 
 		if (max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
 			// If max DCFCLK is greater than the max DCFCLK STA target, insert into the DCFCLK STA target array
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c
index 7f40048dd67d5..c15fbc18bfdf1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn35/dcn35_fpu.c
@@ -233,7 +233,7 @@ void dcn35_update_bw_bounding_box_fpu(struct dc *dc,
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
 	struct _vcs_dpi_voltage_scaling_st *clock_limits =
 		dc->scratch.update_bw_bounding_box.clock_limits;
-	int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
+	unsigned int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
 
 	dc_assert_fp_enabled();
 
@@ -357,21 +357,21 @@ void dcn35_update_bw_bounding_box_fpu(struct dc *dc,
 			dc->dml2_options.bbox_overrides.clks_table.num_states =
 				clk_table->num_entries;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dcfclk_mhz =
-				clock_limits[i].dcfclk_mhz;
+				(unsigned int)clock_limits[i].dcfclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].fclk_mhz =
-				clock_limits[i].fabricclk_mhz;
+				(unsigned int)clock_limits[i].fabricclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dispclk_mhz =
-				clock_limits[i].dispclk_mhz;
+				(unsigned int)clock_limits[i].dispclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dppclk_mhz =
-				clock_limits[i].dppclk_mhz;
+				(unsigned int)clock_limits[i].dppclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].socclk_mhz =
-				clock_limits[i].socclk_mhz;
+				(unsigned int)clock_limits[i].socclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].memclk_mhz =
 				clk_table->entries[i].memclk_mhz * clk_table->entries[i].wck_ratio;
 
-			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dram_speed_mts = clock_limits[i].dram_speed_mts;
+			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dram_speed_mts = (unsigned int)clock_limits[i].dram_speed_mts;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dtbclk_mhz =
-				clock_limits[i].dtbclk_mhz;
+				(unsigned int)clock_limits[i].dtbclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.num_entries_per_clk.num_dcfclk_levels =
 				clk_table->num_entries;
 			dc->dml2_options.bbox_overrides.clks_table.num_entries_per_clk.num_fclk_levels =
@@ -415,11 +415,11 @@ static bool is_dual_plane(enum surface_pixel_format format)
 static unsigned int micro_sec_to_vert_lines(unsigned int num_us, struct dc_crtc_timing *timing)
 {
 	unsigned int num_lines = 0;
-	unsigned int lines_time_in_ns = 1000.0 *
-			(((float)timing->h_total * 1000.0) /
-			 ((float)timing->pix_clk_100hz / 10.0));
+	double lines_time_in_ns = 1000.0 *
+			(((double)timing->h_total * 1000.0) /
+			 ((double)timing->pix_clk_100hz / 10.0));
 
-	num_lines = dml_ceil(1000.0 * num_us / lines_time_in_ns, 1.0);
+	num_lines = (unsigned int)dml_ceil(1000.0 * num_us / lines_time_in_ns, 1.0);
 
 	return num_lines;
 }
@@ -440,7 +440,8 @@ int dcn35_populate_dml_pipes_from_context_fpu(struct dc *dc,
 					      display_e2e_pipe_params_st *pipes,
 					      enum dc_validate_mode validate_mode)
 {
-	int i, pipe_cnt;
+	unsigned int i;
+	int pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe = 0;
 	bool upscaled = false;
@@ -550,16 +551,16 @@ int dcn35_populate_dml_pipes_from_context_fpu(struct dc *dc,
 	}
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *cur_pipe = &context->res_ctx.pipe_ctx[i];
 
-		if (!pipe->stream)
+		if (!cur_pipe->stream)
 			continue;
 
-		if (pipe->stream->signal == SIGNAL_TYPE_EDP &&
+		if (cur_pipe->stream->signal == SIGNAL_TYPE_EDP &&
 		    dc->debug.seamless_boot_odm_combine &&
-		    pipe->stream->apply_seamless_boot_optimization) {
+		    cur_pipe->stream->apply_seamless_boot_optimization) {
 
-			if (pipe->stream->apply_boot_odm_mode ==
+			if (cur_pipe->stream->apply_boot_odm_mode ==
 			    dm_odm_combine_policy_2to1) {
 				context->bw_ctx.dml.vba.ODMCombinePolicy =
 					dm_odm_combine_policy_2to1;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dcn351/dcn351_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dcn351/dcn351_fpu.c
index 73c2aee57f288..6552b26de845d 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dcn351/dcn351_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dcn351/dcn351_fpu.c
@@ -266,7 +266,7 @@ void dcn351_update_bw_bounding_box_fpu(struct dc *dc,
 	struct clk_limit_table *clk_table = &bw_params->clk_table;
 	struct _vcs_dpi_voltage_scaling_st *clock_limits =
 		dc->scratch.update_bw_bounding_box.clock_limits;
-	int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
+	unsigned int max_dispclk_mhz = 0, max_dppclk_mhz = 0;
 
 	dc_assert_fp_enabled();
 
@@ -390,20 +390,20 @@ void dcn351_update_bw_bounding_box_fpu(struct dc *dc,
 			dc->dml2_options.bbox_overrides.clks_table.num_states =
 				clk_table->num_entries;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dcfclk_mhz =
-				clock_limits[i].dcfclk_mhz;
+				(unsigned int)clock_limits[i].dcfclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].fclk_mhz =
-				clock_limits[i].fabricclk_mhz;
+				(unsigned int)clock_limits[i].fabricclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dispclk_mhz =
-				clock_limits[i].dispclk_mhz;
+				(unsigned int)clock_limits[i].dispclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dppclk_mhz =
-				clock_limits[i].dppclk_mhz;
+				(unsigned int)clock_limits[i].dppclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].socclk_mhz =
-				clock_limits[i].socclk_mhz;
+				(unsigned int)clock_limits[i].socclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].memclk_mhz =
 				clk_table->entries[i].memclk_mhz * clk_table->entries[i].wck_ratio;
-			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dram_speed_mts = clock_limits[i].dram_speed_mts;
+			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dram_speed_mts = (unsigned int)clock_limits[i].dram_speed_mts;
 			dc->dml2_options.bbox_overrides.clks_table.clk_entries[i].dtbclk_mhz =
-				clock_limits[i].dtbclk_mhz;
+				(unsigned int)clock_limits[i].dtbclk_mhz;
 			dc->dml2_options.bbox_overrides.clks_table.num_entries_per_clk.num_dcfclk_levels =
 				clk_table->num_entries;
 			dc->dml2_options.bbox_overrides.clks_table.num_entries_per_clk.num_fclk_levels =
@@ -447,11 +447,11 @@ static bool is_dual_plane(enum surface_pixel_format format)
 static unsigned int micro_sec_to_vert_lines(unsigned int num_us, struct dc_crtc_timing *timing)
 {
 	unsigned int num_lines = 0;
-	unsigned int lines_time_in_ns = 1000.0 *
-			(((float)timing->h_total * 1000.0) /
-			 ((float)timing->pix_clk_100hz / 10.0));
+	double lines_time_in_ns = 1000.0 *
+			(((double)timing->h_total * 1000.0) /
+			 ((double)timing->pix_clk_100hz / 10.0));
 
-	num_lines = dml_ceil(1000.0 * num_us / lines_time_in_ns, 1.0);
+	num_lines = (unsigned int)dml_ceil(1000.0 * num_us / lines_time_in_ns, 1.0);
 
 	return num_lines;
 }
@@ -472,7 +472,8 @@ int dcn351_populate_dml_pipes_from_context_fpu(struct dc *dc,
 					      display_e2e_pipe_params_st *pipes,
 					      enum dc_validate_mode validate_mode)
 {
-	int i, pipe_cnt;
+	int pipe_cnt;
+	unsigned int i;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe = 0;
 	bool upscaled = false;
@@ -582,16 +583,16 @@ int dcn351_populate_dml_pipes_from_context_fpu(struct dc *dc,
 	}
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
+		struct pipe_ctx *cur_pipe = &context->res_ctx.pipe_ctx[i];
 
-		if (!pipe->stream)
+		if (!cur_pipe->stream)
 			continue;
 
-		if (pipe->stream->signal == SIGNAL_TYPE_EDP &&
+		if (cur_pipe->stream->signal == SIGNAL_TYPE_EDP &&
 		    dc->debug.seamless_boot_odm_combine &&
-		    pipe->stream->apply_seamless_boot_optimization) {
+		    cur_pipe->stream->apply_seamless_boot_optimization) {
 
-			if (pipe->stream->apply_boot_odm_mode ==
+			if (cur_pipe->stream->apply_boot_odm_mode ==
 			    dm_odm_combine_policy_2to1) {
 				context->bw_ctx.dml.vba.ODMCombinePolicy =
 					dm_odm_combine_policy_2to1;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
index 85453bbb4f9b1..c42884dc16b2b 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c
@@ -246,7 +246,7 @@ unsigned int get_total_surface_size_in_mall_bytes(
 		unsigned int num_pipes)
 {
 	unsigned int k;
-	unsigned int size = 0.0;
+	unsigned int size = 0;
 	recalculate_params(mode_lib, pipes, num_pipes);
 	for (k = 0; k < mode_lib->vba.NumberOfActiveSurfaces; ++k)
 		size += mode_lib->vba.SurfaceSizeInMALL[k];
@@ -304,7 +304,7 @@ bool get_is_phantom_pipe(struct display_mode_lib *mode_lib, const display_e2e_pi
 static void fetch_socbb_params(struct display_mode_lib *mode_lib)
 {
 	soc_bounding_box_st *soc = &mode_lib->vba.soc;
-	int i;
+	unsigned int i;
 
 	// SOC Bounding Box Parameters
 	mode_lib->vba.ReturnBusWidth = soc->return_bus_width_bytes;
@@ -343,7 +343,7 @@ static void fetch_socbb_params(struct display_mode_lib *mode_lib)
 	mode_lib->vba.FCLKChangeLatency = soc->fclk_change_latency_us;
 	mode_lib->vba.USRRetrainingLatency = soc->usr_retraining_latency_us;
 	mode_lib->vba.SMNLatency = soc->smn_latency_us;
-	mode_lib->vba.MALLAllocatedForDCNFinal = soc->mall_allocated_for_dcn_mbytes;
+	mode_lib->vba.MALLAllocatedForDCNFinal = (unsigned int)soc->mall_allocated_for_dcn_mbytes;
 
 	mode_lib->vba.PercentOfIdealDRAMBWReceivedAfterUrgLatencySTROBE = soc->pct_ideal_dram_bw_after_urgent_strobe;
 	mode_lib->vba.MaxAveragePercentOfIdealFabricBWDisplayCanUseInNormalSystemOperation =
@@ -946,7 +946,7 @@ static void fetch_pipe_params(struct display_mode_lib *mode_lib)
  */
 static void cache_debug_params(struct display_mode_lib *mode_lib)
 {
-	int k = 0;
+	unsigned int k = 0;
 
 	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; k++)
 		mode_lib->vba.CachedActiveDRAMClockChangeLatencyMargin[k] = mode_lib->vba.ActiveDRAMClockChangeLatencyMargin[k];
diff --git a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h
index 07993741f5e62..cde84dfb0953e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h
@@ -472,7 +472,7 @@ struct vba_vars_st {
 	unsigned int VTotal[DC__NUM_DPP__MAX];
 	unsigned int VTotal_Max[DC__NUM_DPP__MAX];
 	unsigned int VTotal_Min[DC__NUM_DPP__MAX];
-	int DPPPerPlane[DC__NUM_DPP__MAX];
+	unsigned int DPPPerPlane[DC__NUM_DPP__MAX];
 	double PixelClock[DC__NUM_DPP__MAX];
 	double PixelClockBackEnd[DC__NUM_DPP__MAX];
 	bool DCCEnable[DC__NUM_DPP__MAX];
@@ -739,7 +739,7 @@ struct vba_vars_st {
 	/* ms locals */
 	double IdealSDPPortBandwidthPerState[DC__VOLTAGE_STATES][2];
 	unsigned int NoOfDPP[DC__VOLTAGE_STATES][2][DC__NUM_DPP__MAX];
-	int NoOfDPPThisState[DC__NUM_DPP__MAX];
+	unsigned int NoOfDPPThisState[DC__NUM_DPP__MAX];
 	enum odm_combine_mode ODMCombineEnablePerState[DC__VOLTAGE_STATES][DC__NUM_DPP__MAX];
 	double SwathWidthYThisState[DC__NUM_DPP__MAX];
 	unsigned int SwathHeightCPerState[DC__VOLTAGE_STATES][2][DC__NUM_DPP__MAX];
@@ -900,7 +900,7 @@ struct vba_vars_st {
 	int PTEBufferSizeInRequestsForChroma;
 
 	// Missing from VBA
-	int dpte_group_bytes_chroma;
+	unsigned int dpte_group_bytes_chroma;
 	unsigned int vm_group_bytes_chroma;
 	double dst_x_after_scaler;
 	double dst_y_after_scaler;
@@ -1100,8 +1100,8 @@ struct vba_vars_st {
 	unsigned int DETBufferSizeCThisState[DC__NUM_DPP__MAX];
 	bool NoUrgentLatencyHiding[DC__NUM_DPP__MAX];
 	bool NoUrgentLatencyHidingPre[DC__NUM_DPP__MAX];
-	int swath_width_luma_ub_this_state[DC__NUM_DPP__MAX];
-	int swath_width_chroma_ub_this_state[DC__NUM_DPP__MAX];
+	unsigned int swath_width_luma_ub_this_state[DC__NUM_DPP__MAX];
+	unsigned int swath_width_chroma_ub_this_state[DC__NUM_DPP__MAX];
 	double UrgLatency[DC__VOLTAGE_STATES];
 	double VActiveCursorBandwidth[DC__VOLTAGE_STATES][2][DC__NUM_DPP__MAX];
 	double VActivePixelBandwidth[DC__VOLTAGE_STATES][2][DC__NUM_DPP__MAX];
@@ -1172,7 +1172,7 @@ struct vba_vars_st {
 	int ConfigReturnBufferSizeInKByte;
 	enum unbounded_requesting_policy UseUnboundedRequesting;
 	int CompressedBufferSegmentSizeInkByte;
-	int CompressedBufferSizeInkByte;
+	unsigned int CompressedBufferSizeInkByte;
 	int MetaFIFOSizeInKEntries;
 	int ZeroSizeBufferEntries;
 	int COMPBUF_RESERVED_SPACE_64B;
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c b/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c
index cf194bcba4551..ff5cb04db9dd3 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dml1_display_rq_dlg_calc.c
@@ -261,12 +261,12 @@ void dml1_extract_rq_regs(
 	if (rq_param->yuv420) {
 		if ((double) rq_param->misc.rq_l.stored_swath_bytes
 				/ (double) rq_param->misc.rq_c.stored_swath_bytes <= 1.5) {
-			detile_buf_plane1_addr = (detile_buf_size_in_bytes / 2.0 / 64.0); /* half to chroma */
+			detile_buf_plane1_addr = (unsigned int)(detile_buf_size_in_bytes / 2.0 / 64.0); /* half to chroma */
 		} else {
-			detile_buf_plane1_addr = dml_round_to_multiple(
+			detile_buf_plane1_addr = (unsigned int)(dml_round_to_multiple(
 					(unsigned int) ((2.0 * detile_buf_size_in_bytes) / 3.0),
 					256,
-					0) / 64.0; /* 2/3 to chroma */
+					0) / 64.0); /* 2/3 to chroma */
 		}
 	}
 	rq_regs->plane1_base_address = detile_buf_plane1_addr;
@@ -447,7 +447,7 @@ static void dml1_rq_dlg_get_row_heights(
 	 */
 	if (tiling != dm_sw_linear)
 		log2_blk_height = log2_blk256_height
-				+ dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
+				+ (unsigned int)dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
 	else
 		log2_blk_height = 0; /* blk height of 1 */
 
@@ -522,7 +522,7 @@ static void dml1_rq_dlg_get_row_heights(
 	 * round the result down to a power of two.
 	 */
 	if (surf_linear) {
-		log2_dpte_row_height_linear = dml_floor(
+		log2_dpte_row_height_linear = (unsigned int)dml_floor(
 				dml_log2(dpte_buf_in_pte_reqs * dpte_req_width / data_pitch),
 				1);
 
@@ -686,7 +686,7 @@ static void get_surf_rq_param(
 	 */
 	if (pipe_src_param->sw_mode != dm_sw_linear)
 		log2_blk_height = log2_blk256_height
-				+ dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
+				+ (unsigned int)dml_ceil((double) (log2_blk_bytes - 8) / 2.0, 1);
 	else
 		log2_blk_height = 0; /* blk height of 1 */
 
@@ -848,7 +848,7 @@ static void get_surf_rq_param(
 	 * round the result down to a power of two.
 	 */
 	if (surf_linear) {
-		log2_dpte_row_height_linear = dml_floor(
+		log2_dpte_row_height_linear = (unsigned int)dml_floor(
 				dml_log2(dpte_buf_in_pte_reqs * dpte_req_width / data_pitch),
 				1);
 
@@ -922,7 +922,7 @@ static void get_surf_rq_param(
 	/* since dpte groups are only aligned to dpte_req_width and not dpte_group_width,
 	 * the upper bound for the dpte groups per row is as follows.
 	 */
-	rq_dlg_param->dpte_groups_per_row_ub = dml_ceil(
+	rq_dlg_param->dpte_groups_per_row_ub = (unsigned int)dml_ceil(
 			(double) dpte_row_width_ub / dpte_group_width,
 			1);
 
@@ -1248,8 +1248,8 @@ void dml1_rq_dlg_get_dlg_params(
 
 	dppclk_delay_subtotal = mode_lib->ip.dppclk_delay_subtotal;
 	dispclk_delay_subtotal = mode_lib->ip.dispclk_delay_subtotal;
-	pixel_rate_delay_subtotal = dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
-			+ dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz;
+	pixel_rate_delay_subtotal = (unsigned int)(dppclk_delay_subtotal * pclk_freq_in_mhz / dppclk_freq_in_mhz
+			+ dispclk_delay_subtotal * pclk_freq_in_mhz / dispclk_freq_in_mhz);
 
 	vstartup_start = e2e_pipe_param->pipe.dest.vstartup_start;
 
@@ -1439,7 +1439,7 @@ void dml1_rq_dlg_get_dlg_params(
 	}
 
 	disp_dlg_regs->dst_y_after_scaler = dst_y_after_scaler; /* in terms of line */
-	disp_dlg_regs->refcyc_x_after_scaler = dst_x_after_scaler * ref_freq_to_pix_freq; /* in terms of refclk */
+	disp_dlg_regs->refcyc_x_after_scaler = (unsigned int)(dst_x_after_scaler * ref_freq_to_pix_freq); /* in terms of refclk */
 	ASSERT(disp_dlg_regs->refcyc_x_after_scaler < (unsigned int) dml_pow(2, 13));
 	DTRACE(
 			"DLG: %s: disp_dlg_regs->dst_y_after_scaler      = 0x%0x",
@@ -1576,19 +1576,19 @@ void dml1_rq_dlg_get_dlg_params(
 			/ (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
 			/ (double) dpte_groups_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_pte_group_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_pte_group_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_pte_group_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 
 	disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int) ((double) dpte_row_height_c
 			/ (double) vratio_c * (double) htotal * ref_freq_to_pix_freq
 			/ (double) dpte_groups_per_row_ub_c);
 	if (disp_dlg_regs->refcyc_per_pte_group_nom_c >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_pte_group_nom_c = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_pte_group_nom_c = (unsigned int)(dml_pow(2, 23) - 1);
 
 	disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int) ((double) meta_row_height_l
 			/ (double) vratio_l * (double) htotal * ref_freq_to_pix_freq
 			/ (double) meta_chunks_per_row_ub_l);
 	if (disp_dlg_regs->refcyc_per_meta_chunk_nom_l >= (unsigned int) dml_pow(2, 23))
-		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = dml_pow(2, 23) - 1;
+		disp_dlg_regs->refcyc_per_meta_chunk_nom_l = (unsigned int)(dml_pow(2, 23) - 1);
 
 	if (mode_422) {
 		swath_width_pixels_ub_l = swath_width_ub_l * 2; /* *2 for 2 pixel per element */
@@ -1844,7 +1844,7 @@ void dml1_rq_dlg_get_dlg_params(
 				cur0_req_size = 256;
 		}
 
-		cur0_req_width = (double) cur0_req_size / ((double) cur0_bit_per_pixel / 8.0);
+		cur0_req_width = (unsigned int)((double) cur0_req_size / ((double) cur0_bit_per_pixel / 8.0));
 		cur0_width_ub = dml_ceil((double) cur0_src_width / (double) cur0_req_width, 1)
 				* (double) cur0_req_width;
 		cur0_req_per_width = cur0_width_ub / (double) cur0_req_width;
@@ -1917,7 +1917,7 @@ void dml1_rq_dlg_get_dlg_params(
 	disp_ttu_regs->qos_ramp_disable_c = 0;
 	disp_ttu_regs->qos_ramp_disable_cur0 = 0;
 
-	disp_ttu_regs->min_ttu_vblank = min_ttu_vblank * refclk_freq_in_mhz;
+	disp_ttu_regs->min_ttu_vblank = (unsigned int)(min_ttu_vblank * refclk_freq_in_mhz);
 	ASSERT(disp_ttu_regs->min_ttu_vblank < dml_pow(2, 24));
 
 	print__ttu_regs_st(mode_lib, disp_ttu_regs);
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
index 6b2ab4ec2b5ff..be6ebf53d9a2b 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
+++ b/drivers/gpu/drm/amd/display/dc/dml/dml_inline_defs.h
@@ -31,7 +31,7 @@
 
 static inline double dml_min(double a, double b)
 {
-	return (double) dcn_bw_min2(a, b);
+	return (double)dcn_bw_min2((float)a, (float)b);
 }
 
 static inline double dml_min3(double a, double b, double c)
@@ -46,7 +46,7 @@ static inline double dml_min4(double a, double b, double c, double d)
 
 static inline double dml_max(double a, double b)
 {
-	return (double) dcn_bw_max2(a, b);
+	return (double)dcn_bw_max2((float)a, (float)b);
 }
 
 static inline double dml_max3(double a, double b, double c)
@@ -68,14 +68,14 @@ static inline double dml_ceil(double a, double granularity)
 {
 	if (granularity == 0)
 		return 0;
-	return (double) dcn_bw_ceil2(a, granularity);
+	return (double)dcn_bw_ceil2((float)a, (float)granularity);
 }
 
 static inline double dml_floor(double a, double granularity)
 {
 	if (granularity == 0)
 		return 0;
-	return (double) dcn_bw_floor2(a, granularity);
+	return (double)dcn_bw_floor2((float)a, (float)granularity);
 }
 
 static inline double dml_round(double a)
@@ -103,31 +103,31 @@ static inline int dml_log2(double x)
 
 static inline double dml_pow(double a, int exp)
 {
-	return (double) dcn_bw_pow(a, exp);
+	return (double)dcn_bw_pow((float)a, (float)exp);
 }
 
 static inline double dml_fmod(double f, int val)
 {
-	return (double) dcn_bw_mod(f, val);
+	return (double)dcn_bw_mod((float)f, (float)val);
 }
 
 static inline double dml_ceil_2(double f)
 {
-	return (double) dcn_bw_ceil2(f, 2);
+	return (double)dcn_bw_ceil2((float)f, 2.0f);
 }
 
 static inline double dml_ceil_ex(double x, double granularity)
 {
 	if (granularity == 0)
 		return 0;
-	return (double) dcn_bw_ceil2(x, granularity);
+	return (double)dcn_bw_ceil2((float)x, (float)granularity);
 }
 
 static inline double dml_floor_ex(double x, double granularity)
 {
 	if (granularity == 0)
 		return 0;
-	return (double) dcn_bw_floor2(x, granularity);
+	return (double)dcn_bw_floor2((float)x, (float)granularity);
 }
 
 static inline unsigned int dml_round_to_multiple(unsigned int num,
diff --git a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
index ef75eb7d5adc3..b8fc27a5361e8 100644
--- a/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/dml/dsc/rc_calc_fpu.c
@@ -96,7 +96,7 @@ static void get_qp_set(qp_set qps, enum colour_mode cm, enum bits_per_comp bpc,
 	if (!table)
 		return;
 
-	index = (bpp - table[0].bpp) * 2;
+	index = (int)((bpp - table[0].bpp) * 2);
 
 	/* requested size is bigger than the table */
 	if (index >= table_size) {
@@ -112,50 +112,50 @@ static void get_ofs_set(qp_set ofs, enum colour_mode mode, float bpp)
 	int   *p = ofs;
 
 	if (mode == CM_444 || mode == CM_RGB) {
-		*p++ = (bpp <=  6) ? (0) : ((((bpp >=  8) && (bpp <= 12))) ? (2) : ((bpp >= 15) ? (10) : ((((bpp > 6) && (bpp < 8))) ? (0 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (2 + dsc_roundf((bpp - 12) * (8 / 3.0))))));
-		*p++ = (bpp <=  6) ? (-2) : ((((bpp >=  8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (8) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (8 / 3.0))))));
-		*p++ = (bpp <=  6) ? (-2) : ((((bpp >=  8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (6) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
-		*p++ = (bpp <=  6) ? (-4) : ((((bpp >=  8) && (bpp <= 12))) ? (-2) : ((bpp >= 15) ? (4) : ((((bpp > 6) && (bpp < 8))) ? (-4 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (-2 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
-		*p++ = (bpp <=  6) ? (-6) : ((((bpp >=  8) && (bpp <= 12))) ? (-4) : ((bpp >= 15) ? (2) : ((((bpp > 6) && (bpp < 8))) ? (-6 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (-4 + dsc_roundf((bpp - 12) * (6 / 3.0))))));
-		*p++ = (bpp <= 12) ? (-6) : ((bpp >= 15) ? (0) : (-6 + dsc_roundf((bpp - 12) * (6 / 3.0))));
-		*p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-2) : (-8 + dsc_roundf((bpp - 12) * (6 / 3.0))));
-		*p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-4) : (-8 + dsc_roundf((bpp - 12) * (4 / 3.0))));
-		*p++ = (bpp <= 12) ? (-8) : ((bpp >= 15) ? (-6) : (-8 + dsc_roundf((bpp - 12) * (2 / 3.0))));
-		*p++ = (bpp <= 12) ? (-10) : ((bpp >= 15) ? (-8) : (-10 + dsc_roundf((bpp - 12) * (2 / 3.0))));
+		*p++ = (int)((bpp <=  6) ? (0) : ((((bpp >=  8) && (bpp <= 12))) ? (2) : ((bpp >= 15) ? (10) : ((((bpp > 6) && (bpp < 8))) ? (0 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (2 + dsc_roundf((bpp - 12) * (8 / 3.0)))))));
+		*p++ = (int)((bpp <=  6) ? (-2) : ((((bpp >=  8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (8) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (8 / 3.0)))))));
+		*p++ = (int)((bpp <=  6) ? (-2) : ((((bpp >=  8) && (bpp <= 12))) ? (0) : ((bpp >= 15) ? (6) : ((((bpp > 6) && (bpp < 8))) ? (-2 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (0 + dsc_roundf((bpp - 12) * (6 / 3.0)))))));
+		*p++ = (int)((bpp <=  6) ? (-4) : ((((bpp >=  8) && (bpp <= 12))) ? (-2) : ((bpp >= 15) ? (4) : ((((bpp > 6) && (bpp < 8))) ? (-4 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (-2 + dsc_roundf((bpp - 12) * (6 / 3.0)))))));
+		*p++ = (int)((bpp <=  6) ? (-6) : ((((bpp >=  8) && (bpp <= 12))) ? (-4) : ((bpp >= 15) ? (2) : ((((bpp > 6) && (bpp < 8))) ? (-6 + dsc_roundf((bpp -  6) * (2 / 2.0))) : (-4 + dsc_roundf((bpp - 12) * (6 / 3.0)))))));
+		*p++ = (int)((bpp <= 12) ? (-6) : ((bpp >= 15) ? (0) : (-6 + dsc_roundf((bpp - 12) * (6 / 3.0)))));
+		*p++ = (int)((bpp <= 12) ? (-8) : ((bpp >= 15) ? (-2) : (-8 + dsc_roundf((bpp - 12) * (6 / 3.0)))));
+		*p++ = (int)((bpp <= 12) ? (-8) : ((bpp >= 15) ? (-4) : (-8 + dsc_roundf((bpp - 12) * (4 / 3.0)))));
+		*p++ = (int)((bpp <= 12) ? (-8) : ((bpp >= 15) ? (-6) : (-8 + dsc_roundf((bpp - 12) * (2 / 3.0)))));
+		*p++ = (int)((bpp <= 12) ? (-10) : ((bpp >= 15) ? (-8) : (-10 + dsc_roundf((bpp - 12) * (2 / 3.0)))));
 		*p++ = -10;
-		*p++ = (bpp <=  6) ? (-12) : ((bpp >=  8) ? (-10) : (-12 + dsc_roundf((bpp -  6) * (2 / 2.0))));
+		*p++ = (int)((bpp <=  6) ? (-12) : ((bpp >=  8) ? (-10) : (-12 + dsc_roundf((bpp -  6) * (2 / 2.0)))));
 		*p++ = -12;
 		*p++ = -12;
 		*p++ = -12;
 	} else if (mode == CM_422) {
-		*p++ = (bpp <=  8) ? (2) : ((bpp >= 10) ? (10) : (2 + dsc_roundf((bpp -  8) * (8 / 2.0))));
-		*p++ = (bpp <=  8) ? (0) : ((bpp >= 10) ? (8) : (0 + dsc_roundf((bpp -  8) * (8 / 2.0))));
-		*p++ = (bpp <=  8) ? (0) : ((bpp >= 10) ? (6) : (0 + dsc_roundf((bpp -  8) * (6 / 2.0))));
-		*p++ = (bpp <=  8) ? (-2) : ((bpp >= 10) ? (4) : (-2 + dsc_roundf((bpp -  8) * (6 / 2.0))));
-		*p++ = (bpp <=  8) ? (-4) : ((bpp >= 10) ? (2) : (-4 + dsc_roundf((bpp -  8) * (6 / 2.0))));
-		*p++ = (bpp <=  8) ? (-6) : ((bpp >= 10) ? (0) : (-6 + dsc_roundf((bpp -  8) * (6 / 2.0))));
-		*p++ = (bpp <=  8) ? (-8) : ((bpp >= 10) ? (-2) : (-8 + dsc_roundf((bpp -  8) * (6 / 2.0))));
-		*p++ = (bpp <=  8) ? (-8) : ((bpp >= 10) ? (-4) : (-8 + dsc_roundf((bpp -  8) * (4 / 2.0))));
-		*p++ = (bpp <=  8) ? (-8) : ((bpp >= 10) ? (-6) : (-8 + dsc_roundf((bpp -  8) * (2 / 2.0))));
-		*p++ = (bpp <=  8) ? (-10) : ((bpp >= 10) ? (-8) : (-10 + dsc_roundf((bpp -  8) * (2 / 2.0))));
+		*p++ = (int)((bpp <=  8) ? (2) : ((bpp >= 10) ? (10) : (2 + dsc_roundf((bpp -  8) * (8 / 2.0)))));
+		*p++ = (int)((bpp <=  8) ? (0) : ((bpp >= 10) ? (8) : (0 + dsc_roundf((bpp -  8) * (8 / 2.0)))));
+		*p++ = (int)((bpp <=  8) ? (0) : ((bpp >= 10) ? (6) : (0 + dsc_roundf((bpp -  8) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  8) ? (-2) : ((bpp >= 10) ? (4) : (-2 + dsc_roundf((bpp -  8) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  8) ? (-4) : ((bpp >= 10) ? (2) : (-4 + dsc_roundf((bpp -  8) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  8) ? (-6) : ((bpp >= 10) ? (0) : (-6 + dsc_roundf((bpp -  8) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  8) ? (-8) : ((bpp >= 10) ? (-2) : (-8 + dsc_roundf((bpp -  8) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  8) ? (-8) : ((bpp >= 10) ? (-4) : (-8 + dsc_roundf((bpp -  8) * (4 / 2.0)))));
+		*p++ = (int)((bpp <=  8) ? (-8) : ((bpp >= 10) ? (-6) : (-8 + dsc_roundf((bpp -  8) * (2 / 2.0)))));
+		*p++ = (int)((bpp <=  8) ? (-10) : ((bpp >= 10) ? (-8) : (-10 + dsc_roundf((bpp -  8) * (2 / 2.0)))));
 		*p++ = -10;
-		*p++ = (bpp <=  6) ? (-12) : ((bpp >= 7) ? (-10) : (-12 + dsc_roundf((bpp -  6) * (2.0 / 1))));
+		*p++ = (int)((bpp <=  6) ? (-12) : ((bpp >= 7) ? (-10) : (-12 + dsc_roundf((bpp -  6) * (2.0 / 1)))));
 		*p++ = -12;
 		*p++ = -12;
 		*p++ = -12;
 	} else {
-		*p++ = (bpp <=  6) ? (2) : ((bpp >=  8) ? (10) : (2 + dsc_roundf((bpp -  6) * (8 / 2.0))));
-		*p++ = (bpp <=  6) ? (0) : ((bpp >=  8) ? (8) : (0 + dsc_roundf((bpp -  6) * (8 / 2.0))));
-		*p++ = (bpp <=  6) ? (0) : ((bpp >=  8) ? (6) : (0 + dsc_roundf((bpp -  6) * (6 / 2.0))));
-		*p++ = (bpp <=  6) ? (-2) : ((bpp >=  8) ? (4) : (-2 + dsc_roundf((bpp -  6) * (6 / 2.0))));
-		*p++ = (bpp <=  6) ? (-4) : ((bpp >=  8) ? (2) : (-4 + dsc_roundf((bpp -  6) * (6 / 2.0))));
-		*p++ = (bpp <=  6) ? (-6) : ((bpp >=  8) ? (0) : (-6 + dsc_roundf((bpp -  6) * (6 / 2.0))));
-		*p++ = (bpp <=  6) ? (-8) : ((bpp >=  8) ? (-2) : (-8 + dsc_roundf((bpp -  6) * (6 / 2.0))));
-		*p++ = (bpp <=  6) ? (-8) : ((bpp >=  8) ? (-4) : (-8 + dsc_roundf((bpp -  6) * (4 / 2.0))));
-		*p++ = (bpp <=  6) ? (-8) : ((bpp >=  8) ? (-6) : (-8 + dsc_roundf((bpp -  6) * (2 / 2.0))));
-		*p++ = (bpp <=  6) ? (-10) : ((bpp >=  8) ? (-8) : (-10 + dsc_roundf((bpp -  6) * (2 / 2.0))));
+		*p++ = (int)((bpp <=  6) ? (2) : ((bpp >=  8) ? (10) : (2 + dsc_roundf((bpp -  6) * (8 / 2.0)))));
+		*p++ = (int)((bpp <=  6) ? (0) : ((bpp >=  8) ? (8) : (0 + dsc_roundf((bpp -  6) * (8 / 2.0)))));
+		*p++ = (int)((bpp <=  6) ? (0) : ((bpp >=  8) ? (6) : (0 + dsc_roundf((bpp -  6) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  6) ? (-2) : ((bpp >=  8) ? (4) : (-2 + dsc_roundf((bpp -  6) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  6) ? (-4) : ((bpp >=  8) ? (2) : (-4 + dsc_roundf((bpp -  6) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  6) ? (-6) : ((bpp >=  8) ? (0) : (-6 + dsc_roundf((bpp -  6) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  6) ? (-8) : ((bpp >=  8) ? (-2) : (-8 + dsc_roundf((bpp -  6) * (6 / 2.0)))));
+		*p++ = (int)((bpp <=  6) ? (-8) : ((bpp >=  8) ? (-4) : (-8 + dsc_roundf((bpp -  6) * (4 / 2.0)))));
+		*p++ = (int)((bpp <=  6) ? (-8) : ((bpp >=  8) ? (-6) : (-8 + dsc_roundf((bpp -  6) * (2 / 2.0)))));
+		*p++ = (int)((bpp <=  6) ? (-10) : ((bpp >=  8) ? (-8) : (-10 + dsc_roundf((bpp -  6) * (2 / 2.0)))));
 		*p++ = -10;
-		*p++ = (bpp <=  4) ? (-12) : ((bpp >=  5) ? (-10) : (-12 + dsc_roundf((bpp -  4) * (2 / 1.0))));
+		*p++ = (int)((bpp <=  4) ? (-12) : ((bpp >=  5) ? (-10) : (-12 + dsc_roundf((bpp -  4) * (2 / 1.0)))));
 		*p++ = -12;
 		*p++ = -12;
 		*p++ = -12;
@@ -179,7 +179,7 @@ void _do_calc_rc_params(struct rc_params *rc,
 
 	dc_assert_fp_enabled();
 
-	bpp = ((float)drm_bpp / 16.0);
+	bpp = (float)((float)drm_bpp / 16.0);
 	/* in native_422 or native_420 modes, the bits_per_pixel is double the
 	 * target bpp (the latter is what calc_rc_params expects)
 	 */
@@ -189,29 +189,29 @@ void _do_calc_rc_params(struct rc_params *rc,
 	rc->rc_quant_incr_limit0 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
 	rc->rc_quant_incr_limit1 = ((bpc == BPC_8) ? 11 : (bpc == BPC_10 ? 15 : 19)) - ((minor_version == 1 && cm == CM_444) ? 1 : 0);
 
-	bpp_group = MODE_SELECT(bpp, bpp * 2.0, bpp * 2.0);
+	bpp_group = MODE_SELECT(bpp, bpp * 2.0f, bpp * 2.0f);
 
 	switch (cm) {
 	case CM_420:
-		rc->initial_fullness_offset = (bpp >=  6) ? (2048) : ((bpp <=  4) ? (6144) : ((((bpp >  4) && (bpp <=  5))) ? (6144 - dsc_roundf((bpp - 4) * (512))) : (5632 - dsc_roundf((bpp -  5) * (3584)))));
+		rc->initial_fullness_offset = (int)((bpp >=  6) ? (2048) : ((bpp <=  4) ? (6144) : ((((bpp >  4) && (bpp <=  5))) ? (6144 - dsc_roundf((bpp - 4) * (512))) : (5632 - dsc_roundf((bpp -  5) * (3584))))));
 		rc->first_line_bpg_offset   = median3(0, (12 + (int) (0.09 *  min(34, slice_height - 8))), (int)((3 * bpc * 3) - (3 * bpp_group)));
 		rc->second_line_bpg_offset  = median3(0, 12, (int)((3 * bpc * 3) - (3 * bpp_group)));
 		break;
 	case CM_422:
-		rc->initial_fullness_offset = (bpp >=  8) ? (2048) : ((bpp <=  7) ? (5632) : (5632 - dsc_roundf((bpp - 7) * (3584))));
+		rc->initial_fullness_offset = (int)((bpp >=  8) ? (2048) : ((bpp <=  7) ? (5632) : (5632 - dsc_roundf((bpp - 7) * (3584)))));
 		rc->first_line_bpg_offset   = median3(0, (12 + (int) (0.09 *  min(34, slice_height - 8))), (int)((3 * bpc * 4) - (3 * bpp_group)));
 		rc->second_line_bpg_offset  = 0;
 		break;
 	case CM_444:
 	case CM_RGB:
-		rc->initial_fullness_offset = (bpp >= 12) ? (2048) : ((bpp <=  8) ? (6144) : ((((bpp >  8) && (bpp <= 10))) ? (6144 - dsc_roundf((bpp - 8) * (512 / 2))) : (5632 - dsc_roundf((bpp - 10) * (3584 / 2)))));
+		rc->initial_fullness_offset = (int)((bpp >= 12) ? (2048) : ((bpp <=  8) ? (6144) : ((((bpp >  8) && (bpp <= 10))) ? (6144 - dsc_roundf((bpp - 8) * (512 / 2))) : (5632 - dsc_roundf((bpp - 10) * (3584 / 2))))));
 		rc->first_line_bpg_offset   = median3(0, (12 + (int) (0.09 *  min(34, slice_height - 8))), (int)(((3 * bpc + (cm == CM_444 ? 0 : 2)) * 3) - (3 * bpp_group)));
 		rc->second_line_bpg_offset  = 0;
 		break;
 	}
 
-	initial_xmit_delay_factor = (cm == CM_444 || cm == CM_RGB) ? 1.0 : 2.0;
-	rc->initial_xmit_delay = dsc_roundf(8192.0/2.0/bpp/initial_xmit_delay_factor);
+	initial_xmit_delay_factor = (float)((cm == CM_444 || cm == CM_RGB) ? 1.0 : 2.0);
+	rc->initial_xmit_delay = (int)dsc_roundf(8192.0/2.0/bpp/initial_xmit_delay_factor);
 
 	if (cm == CM_422 || cm == CM_420)
 		slice_width /= 2;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/README.md b/drivers/gpu/drm/amd/display/dc/dml2_0/README.md
new file mode 100644
index 0000000000000..9e8814fbe52fd
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/README.md
@@ -0,0 +1,31 @@
+# DML2_0 (Display Mode Library 2.0) repository
+
+## Category to Directory Mapping
+
+```yaml
+directory_categories:
+  app_tools:
+    - dml21/build/
+    - dml21/src/dml2_unit_test/
+    - utils/
+
+  driver_hw_dependent:
+    - dml21/inc/bounding_boxes/
+    - dml21/src/dml2_cga/
+    - dml21/src/dml2_core/
+    - dml21/src/dml2_dpmm/
+    - dml21/src/dml2_mcg/
+    - dml21/src/dml2_pmo/
+    - dml21/src/dml2_standalone_libraries/
+    - dml21/src/dml2_utm_soc_bb/
+
+  driver_hw_independent:
+    - ./
+    - dml21/
+    - dml21/inc/
+    - dml21/src/dml2_top/
+    - dml21/src/inc/
+
+  undefined:
+    - .github/
+```
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_core.c b/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_core.c
index 698d62fb9cf72..241406e9e85af 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_core.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_core.c
@@ -1119,10 +1119,10 @@ static dml_bool_t CalculatePrefetchSchedule(struct display_mode_lib_scratch_st *
 	if (p->myPipe->Dppclk == 0.0 || p->myPipe->Dispclk == 0.0)
 		return true;
 
-	*p->DSTXAfterScaler = (dml_uint_t) dml_round(s->DPPCycles * p->myPipe->PixelClock / p->myPipe->Dppclk + s->DISPCLKCycles * p->myPipe->PixelClock / p->myPipe->Dispclk + p->DSCDelay, 1.0);
+	*p->DSTXAfterScaler = (dml_uint_t) dml_round(s->DPPCycles * p->myPipe->PixelClock / p->myPipe->Dppclk + s->DISPCLKCycles * p->myPipe->PixelClock / p->myPipe->Dispclk + p->DSCDelay, true);
 	*p->DSTXAfterScaler = (dml_uint_t) dml_round(*p->DSTXAfterScaler + (p->myPipe->ODMMode != dml_odm_mode_bypass ? 18 : 0) + (p->myPipe->DPPPerSurface - 1) * p->DPP_RECOUT_WIDTH +
 						((p->myPipe->ODMMode == dml_odm_mode_split_1to2 || p->myPipe->ODMMode == dml_odm_mode_mso_1to2) ? (dml_float_t)p->myPipe->HActive / 2.0 : 0) +
-						((p->myPipe->ODMMode == dml_odm_mode_mso_1to4) ? (dml_float_t)p->myPipe->HActive * 3.0 / 4.0 : 0), 1.0);
+						((p->myPipe->ODMMode == dml_odm_mode_mso_1to4) ? (dml_float_t)p->myPipe->HActive * 3.0 / 4.0 : 0), true);
 
 #ifdef __DML_VBA_DEBUG__
 	dml_print("DML::%s: DPPCycles = %u\n", __func__, s->DPPCycles);
@@ -2626,18 +2626,18 @@ static dml_uint_t CalculateVMAndRowBytes(
 	*PixelPTEBytesPerRow_one_row_per_frame = (dml_uint_t)((dml_float_t) *dpte_row_width_ub_one_row_per_frame / (dml_float_t) *PixelPTEReqWidth * *PTERequestSize);
 
 	if (SurfaceTiling == dml_sw_linear) {
-		*dpte_row_height = (dml_uint_t)(dml_min(128, 1 << (dml_uint_t) dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1)));
+		*dpte_row_height = (dml_uint_t)(dml_min(128, (dml_uint_t)dml_pow(2.0, (int)dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1))));
 		dml_print("DML::%s: dpte_row_height term 1 = %u\n", __func__, PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch);
 		dml_print("DML::%s: dpte_row_height term 2 = %f\n", __func__, dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch));
 		dml_print("DML::%s: dpte_row_height term 3 = %f\n", __func__, dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1));
-		dml_print("DML::%s: dpte_row_height term 4 = %u\n", __func__, 1 << (dml_uint_t) dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1));
+		dml_print("DML::%s: dpte_row_height term 4 = %u\n", __func__, (dml_uint_t)dml_pow(2.0, (int)dml_floor(dml_log2(PTEBufferSizeInRequests * *PixelPTEReqWidth / Pitch), 1)));
 		dml_print("DML::%s: dpte_row_height = %u\n", __func__, *dpte_row_height);
 
 		*dpte_row_width_ub = (dml_uint_t)(dml_ceil(((dml_float_t) Pitch * (dml_float_t) *dpte_row_height - 1), (dml_float_t) *PixelPTEReqWidth) + *PixelPTEReqWidth);
 		*PixelPTEBytesPerRow = (dml_uint_t)((dml_float_t) *dpte_row_width_ub / (dml_float_t) *PixelPTEReqWidth * *PTERequestSize);
 
 		// VBA_DELTA, VBA doesn't have programming value for pte row height linear.
-		*dpte_row_height_linear = 1 << (dml_uint_t) dml_floor(dml_log2(PTEBufferSizeInRequests * PixelPTEReqWidth_linear / Pitch), 1);
+		*dpte_row_height_linear = (dml_uint_t)dml_pow(2.0, (int)dml_floor(dml_log2(PTEBufferSizeInRequests * PixelPTEReqWidth_linear / Pitch), 1));
 		if (*dpte_row_height_linear > 128)
 			*dpte_row_height_linear = 128;
 
@@ -4301,7 +4301,7 @@ static void CalculateSwathAndDETConfiguration(struct display_mode_lib_scratch_st
 
 	*p->compbuf_reserved_space_64b = 2 * p->PixelChunkSizeInKByte * 1024 / 64;
 	if (*p->UnboundedRequestEnabled) {
-		*p->compbuf_reserved_space_64b = dml_max(*p->compbuf_reserved_space_64b,
+		*p->compbuf_reserved_space_64b = (dml_uint_t)dml_max(*p->compbuf_reserved_space_64b,
 				(dml_float_t)(p->ROBBufferSizeInKByte * 1024/64)
 				- (dml_float_t)(RoundedUpSwathSizeBytesY[SurfaceDoingUnboundedRequest] * TTUFIFODEPTH / MAXIMUMCOMPRESSION/64));
 	}
@@ -6178,9 +6178,9 @@ static void CalculateImmediateFlipBandwithSupport(
 
 static dml_uint_t MicroSecToVertLines(dml_uint_t num_us, dml_uint_t h_total, dml_float_t pixel_clock)
 {
-	dml_uint_t lines_time_in_ns = 1000.0 * (h_total * 1000.0) / (pixel_clock * 1000.0);
+	dml_uint_t lines_time_in_ns = (dml_uint_t)(1000.0 * (h_total * 1000.0) / (pixel_clock * 1000.0));
 
-	return dml_ceil(1000.0 * num_us / lines_time_in_ns, 1.0);
+	return (dml_uint_t)dml_ceil(1000.0 * num_us / lines_time_in_ns, 1.0);
 }
 
 /// @brief Calculate the maximum vstartup for mode support and mode programming consideration
@@ -6207,9 +6207,9 @@ static dml_uint_t CalculateMaxVStartup(
 	// + 2 is because
 	// 1 -> VStartup_start should be 1 line before VSync
 	// 1 -> always reserve 1 line between start of VBlank to VStartup signal
-	dml_uint_t vblank_nom_vsync_capped = dml_max(vblank_nom_input,
+	dml_uint_t vblank_nom_vsync_capped = (dml_uint_t)dml_max(vblank_nom_input,
 			timing->VTotal[plane_idx] - timing->VActive[plane_idx] - timing->VFrontPorch[plane_idx] + 2);
-	dml_uint_t vblank_nom_max_allowed_capped = dml_min(vblank_nom_vsync_capped, max_allowed_vblank_nom);
+	dml_uint_t vblank_nom_max_allowed_capped = (dml_uint_t)dml_min(vblank_nom_vsync_capped, max_allowed_vblank_nom);
 	dml_uint_t vblank_avail = (vblank_nom_max_allowed_capped == 0) ?
 			vblank_nom_default_in_line : vblank_nom_max_allowed_capped;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_util.c b/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_util.c
index b2fada6c44c36..3939a0d8b8352 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_util.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/display_mode_util.c
@@ -80,7 +80,7 @@ static inline float dcn_bw_pow(float a, float exp)
 	/*ASSERT(exp == (int)exp);*/
 	if ((int)exp == 0)
 		return 1;
-	temp = dcn_bw_pow(a, (int)(exp / 2));
+	temp = dcn_bw_pow(a, (float)(exp / 2));
 	if (((int)exp % 2) == 0) {
 		return temp * temp;
 	} else {
@@ -110,7 +110,7 @@ dml_float_t dml_ceil(dml_float_t x, dml_float_t granularity)
 	if (granularity == 0)
 		return 0;
 	//return (dml_float_t) (ceil(x / granularity) * granularity);
-	return (dml_float_t)dcn_bw_ceil2(x, granularity);
+	return (dml_float_t)dcn_bw_ceil2((float)x, (float)granularity);
 }
 
 dml_float_t dml_floor(dml_float_t x, dml_float_t granularity)
@@ -118,7 +118,7 @@ dml_float_t dml_floor(dml_float_t x, dml_float_t granularity)
 	if (granularity == 0)
 	return 0;
 	//return (dml_float_t) (floor(x / granularity) * granularity);
-	return (dml_float_t)dcn_bw_floor2(x, granularity);
+	return (dml_float_t)dcn_bw_floor2((float)x, (float)granularity);
 }
 
 dml_float_t dml_min(dml_float_t x, dml_float_t y)
@@ -168,12 +168,12 @@ dml_float_t dml_max5(dml_float_t a, dml_float_t b, dml_float_t c, dml_float_t d,
 }
 dml_float_t dml_log(dml_float_t x, dml_float_t base)
 {
-	return (dml_float_t) (_log(x) / _log(base));
+	return (dml_float_t) (_log((float)x) / _log((float)base));
 }
 
 dml_float_t dml_log2(dml_float_t x)
 {
-	return (dml_float_t) (_log(x) / _log(2));
+	return (dml_float_t) (_log((float)x) / _log(2.0f));
 }
 
 dml_float_t dml_round(dml_float_t val, dml_bool_t bankers_rounding)
@@ -184,19 +184,19 @@ dml_float_t dml_round(dml_float_t val, dml_bool_t bankers_rounding)
 //	else {
 //		return round(val);
 		double round_pt = 0.5;
-		double ceil = dml_ceil(val, 1);
-		double floor = dml_floor(val, 1);
+	double ceil = dml_ceil(val, 1.0);
+	double floor = dml_floor(val, 1.0);
 
 		if (val - floor >= round_pt)
-			return ceil;
+			return (dml_float_t)ceil;
 		else
-			return floor;
+			return (dml_float_t)floor;
 //	}
 }
 
 dml_float_t dml_pow(dml_float_t base, int exp)
 {
-	return (dml_float_t) dcn_bw_pow(base, exp);
+	return (dml_float_t) dcn_bw_pow((float)base, (float)exp);
 }
 
 dml_uint_t dml_round_to_multiple(dml_uint_t num, dml_uint_t multiple, dml_bool_t up)
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c
index 476030193f149..25557c99a28e5 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_translation_helper.c
@@ -90,6 +90,8 @@ static void populate_dml21_timing_config_from_stream_state(struct dml2_timing_cf
 		struct pipe_ctx *pipe_ctx,
 		struct dml2_context *dml_ctx)
 {
+	const unsigned int min_v_front_porch = (stream->timing.flags.INTERLACE != 0) ? 2 : 1;
+
 	unsigned int hblank_start, vblank_start;
 	uint64_t min_hardware_refresh_in_uhz;
 	uint32_t pix_clk_100hz;
@@ -97,7 +99,8 @@ static void populate_dml21_timing_config_from_stream_state(struct dml2_timing_cf
 	timing->h_active = stream->timing.h_addressable + stream->timing.h_border_left + stream->timing.h_border_right + pipe_ctx->dsc_padding_params.dsc_hactive_padding;
 	timing->v_active = stream->timing.v_addressable + stream->timing.v_border_bottom + stream->timing.v_border_top;
 	timing->h_front_porch = stream->timing.h_front_porch;
-	timing->v_front_porch = stream->timing.v_front_porch;
+	timing->v_front_porch = stream->timing.v_front_porch > min_v_front_porch ?
+			stream->timing.v_front_porch : min_v_front_porch;
 	timing->pixel_clock_khz = stream->timing.pix_clk_100hz / 10;
 	if (pipe_ctx->dsc_padding_params.dsc_hactive_padding != 0)
 		timing->pixel_clock_khz = pipe_ctx->dsc_padding_params.dsc_pix_clk_100hz / 10;
@@ -116,7 +119,7 @@ static void populate_dml21_timing_config_from_stream_state(struct dml2_timing_cf
 	if (hblank_start < stream->timing.h_addressable)
 		timing->h_blank_end = 0;
 
-	vblank_start = stream->timing.v_total - stream->timing.v_front_porch;
+	vblank_start = timing->v_total - timing->v_front_porch;
 
 	timing->v_blank_end = vblank_start - stream->timing.v_addressable
 		- stream->timing.v_border_top - stream->timing.v_border_bottom;
@@ -610,6 +613,7 @@ static void populate_dml21_plane_config_from_plane_state(struct dml2_context *dm
 
 	plane->composition.viewport.stationary = false;
 
+#ifndef TRIM_CM2
 	if (plane_state->mcm_luts.lut3d_data.lut3d_src == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
 		plane->tdlut.setup_for_tdlut = true;
 
@@ -640,7 +644,39 @@ static void populate_dml21_plane_config_from_plane_state(struct dml2_context *dm
 			break;
 		}
 	}
+#else
+	if (plane_state->cm.flags.bits.lut3d_dma_enable) {
+		plane->tdlut.setup_for_tdlut = true;
+
+		switch (plane_state->cm.lut3d_dma.swizzle) {
+		case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
+		case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
+			plane->tdlut.tdlut_addressing_mode = dml2_tdlut_sw_linear;
+			break;
+		case CM_LUT_1D_PACKED_LINEAR:
+			plane->tdlut.tdlut_addressing_mode = dml2_tdlut_simple_linear;
+			break;
+		}
 
+		switch (plane_state->cm.lut3d_dma.size) {
+		case CM_LUT_SIZE_171717:
+			plane->tdlut.tdlut_width_mode = dml2_tdlut_width_17_cube;
+			break;
+		case CM_LUT_SIZE_333333:
+			plane->tdlut.tdlut_width_mode = dml2_tdlut_width_33_cube;
+			break;
+		// handling when use case and HW support available
+		case CM_LUT_SIZE_454545:
+		case CM_LUT_SIZE_656565:
+			break;
+		case CM_LUT_SIZE_NONE:
+		case CM_LUT_SIZE_999:
+		default:
+			//plane->tdlut.tdlut_width_mode = dml2_tdlut_width_flatten; // dml2_tdlut_width_flatten undefined
+			break;
+		}
+	}
+#endif // TRIM_CM2
 	plane->tdlut.setup_for_tdlut |= dml_ctx->config.force_tdlut_enable;
 
 	plane->dynamic_meta_data.enable = false;
@@ -848,10 +884,10 @@ void dml21_copy_clocks_to_dc_state(struct dml2_context *in_ctx, struct dc_state
 	context->bw_ctx.bw.dcn.clk.socclk_khz = in_ctx->v21.mode_programming.programming->min_clocks.dcn4x.socclk_khz;
 	context->bw_ctx.bw.dcn.clk.subvp_prefetch_dramclk_khz = in_ctx->v21.mode_programming.programming->min_clocks.dcn4x.svp_prefetch_no_throttle.uclk_khz;
 	context->bw_ctx.bw.dcn.clk.subvp_prefetch_fclk_khz = in_ctx->v21.mode_programming.programming->min_clocks.dcn4x.svp_prefetch_no_throttle.fclk_khz;
-	context->bw_ctx.bw.dcn.clk.stutter_efficiency.base_efficiency = in_ctx->v21.mode_programming.programming->stutter.base_percent_efficiency;
-	context->bw_ctx.bw.dcn.clk.stutter_efficiency.low_power_efficiency = in_ctx->v21.mode_programming.programming->stutter.low_power_percent_efficiency;
-	context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_efficiency = in_ctx->v21.mode_programming.programming->informative.power_management.z8.stutter_efficiency;
-	context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_period = in_ctx->v21.mode_programming.programming->informative.power_management.z8.stutter_period;
+	context->bw_ctx.bw.dcn.clk.stutter_efficiency.base_efficiency = (uint8_t)in_ctx->v21.mode_programming.programming->stutter.base_percent_efficiency;
+	context->bw_ctx.bw.dcn.clk.stutter_efficiency.low_power_efficiency = (uint8_t)in_ctx->v21.mode_programming.programming->stutter.low_power_percent_efficiency;
+	context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_efficiency = (uint8_t)in_ctx->v21.mode_programming.programming->informative.power_management.z8.stutter_efficiency;
+	context->bw_ctx.bw.dcn.clk.stutter_efficiency.z8_stutter_period = (int)in_ctx->v21.mode_programming.programming->informative.power_management.z8.stutter_period;
 	context->bw_ctx.bw.dcn.clk.zstate_support = in_ctx->v21.mode_programming.programming->z8_stutter.supported_in_blank; /*ignore meets_eco since it is not used*/
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
index 732de97335fac..835fece1d46a6 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/dml21_utils.c
@@ -267,7 +267,7 @@ static struct dc_stream_state *dml21_add_phantom_stream(struct dml2_context *dml
 	phantom_stream->dst.height = stream_programming->phantom_stream.descriptor.timing.v_active;
 
 	phantom_stream->src.y = 0;
-	phantom_stream->src.height = (double)phantom_stream_descriptor->timing.v_active * (double)main_stream->src.height / (double)main_stream->dst.height;
+	phantom_stream->src.height = (int)((double)phantom_stream_descriptor->timing.v_active * (double)main_stream->src.height / (double)main_stream->dst.height);
 
 	phantom_stream->use_dynamic_meta = false;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
index 98b26116cdc14..dff903a103dbd 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/inc/dml_top_types.h
@@ -19,7 +19,6 @@ enum dml2_project_id {
 	dml2_project_dcn4x_stage1,
 	dml2_project_dcn4x_stage2,
 	dml2_project_dcn4x_stage2_auto_drr_svp,
-	dml2_project_dcn40,
 	dml2_project_dcn42,
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c
index 6cad99c21139b..67e307fa43108 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_core/dml2_core_factory.c
@@ -21,7 +21,6 @@ bool dml2_core_create(enum dml2_project_id project_id, struct dml2_core_instance
 	case dml2_project_dcn4x_stage1:
 		result = false;
 		break;
-	case dml2_project_dcn40:
 	case dml2_project_dcn4x_stage2:
 	case dml2_project_dcn4x_stage2_auto_drr_svp:
 		out->initialize = &core_dcn4_initialize;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c
index 39965ff2e1115..be0517e10104a 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_dpmm/dml2_dpmm_factory.c
@@ -33,7 +33,6 @@ bool dml2_dpmm_create(enum dml2_project_id project_id, struct dml2_dpmm_instance
 		out->map_watermarks = &dummy_map_watermarks;
 		result = true;
 		break;
-	case dml2_project_dcn40:
 	case dml2_project_dcn4x_stage2:
 		out->map_mode_to_soc_dpm = &dpmm_dcn3_map_mode_to_soc_dpm;
 		out->map_watermarks = &dummy_map_watermarks;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c
index 3eaeff39ee792..ef8b75fed9a3e 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_dcn42.c
@@ -22,7 +22,7 @@ static bool build_min_clk_table_coarse_grained(const struct dml2_soc_bb *soc_bb,
 			min_table->dram_bw_table.entries[i].pre_derate_dram_bw_kbps =
 					uclk_to_dram_bw_kbps(soc_bb->clk_table.uclk.clk_values_khz[i], &soc_bb->clk_table.dram_config, soc_bb->clk_table.wck_ratio.clk_values_khz[i]);
 			min_table->dram_bw_table.entries[i].min_uclk_khz = soc_bb->clk_table.uclk.clk_values_khz[i];
-		} else {
+		} else if (soc_bb->clk_table.uclk.num_clk_values > 0) {
 			min_table->dram_bw_table.entries[i].pre_derate_dram_bw_kbps = min_table->dram_bw_table.entries[soc_bb->clk_table.uclk.num_clk_values - 1].pre_derate_dram_bw_kbps;
 			min_table->dram_bw_table.entries[i].min_uclk_khz = soc_bb->clk_table.uclk.clk_values_khz[soc_bb->clk_table.uclk.num_clk_values - 1];
 		}
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c
index fb0b0ac547c72..270283332cc18 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_mcg/dml2_mcg_factory.c
@@ -27,7 +27,6 @@ bool dml2_mcg_create(enum dml2_project_id project_id, struct dml2_mcg_instance *
 		out->build_min_clock_table = &dummy_build_min_clock_table;
 		result = true;
 		break;
-	case dml2_project_dcn40:
 	case dml2_project_dcn4x_stage2:
 	case dml2_project_dcn4x_stage2_auto_drr_svp:
 		out->build_min_clock_table = &mcg_dcn4_build_min_clock_table;
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c
index 83802aac11cd1..af2ba7d08a612 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_pmo/dml2_pmo_factory.c
@@ -3,8 +3,8 @@
 // Copyright 2024 Advanced Micro Devices, Inc.
 
 #include "dml2_pmo_factory.h"
-#include "dml2_pmo_dcn3.h"
 #include "dml2_pmo_dcn4_fams2.h"
+#include "dml2_pmo_dcn3.h"
 #include "dml2_external_lib_deps.h"
 
 static bool dummy_init_for_stutter(struct dml2_pmo_init_for_stutter_in_out *in_out)
@@ -40,7 +40,6 @@ bool dml2_pmo_create(enum dml2_project_id project_id, struct dml2_pmo_instance *
 		out->optimize_dcc_mcache = pmo_dcn4_fams2_optimize_dcc_mcache;
 		result = true;
 		break;
-	case dml2_project_dcn40:
 	case dml2_project_dcn4x_stage2:
 		out->initialize = pmo_dcn3_initialize;
 
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_interfaces.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_interfaces.c
index a6c5031f69c19..04860b6790df1 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_interfaces.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_interfaces.c
@@ -17,7 +17,6 @@ bool dml2_initialize_instance(struct dml2_initialize_instance_in_out *in_out)
 	case dml2_project_dcn4x_stage1:
 	case dml2_project_dcn4x_stage2:
 	case dml2_project_dcn4x_stage2_auto_drr_svp:
-	case dml2_project_dcn40:
 	case dml2_project_dcn42:
 		return dml2_top_soc15_initialize_instance(in_out);
 	case dml2_project_invalid:
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.h b/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.h
deleted file mode 100644
index 14d0ae03dce6d..0000000000000
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml21/src/dml2_top/dml2_top_legacy.h
+++ /dev/null
@@ -1,9 +0,0 @@
-// SPDX-License-Identifier: MIT
-//
-// Copyright 2024 Advanced Micro Devices, Inc.
-
-#ifndef __DML2_TOP_LEGACY_H__
-#define __DML2_TOP_LEGACY_H__
-#include "dml2_internal_shared_types.h"
-bool dml2_top_legacy_initialize_instance(struct dml2_initialize_instance_in_out *in_out);
-#endif /* __DML2_TOP_LEGACY_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c
index 9bbe4e058be7a..4543a60a06839 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_mall_phantom.c
@@ -244,9 +244,9 @@ static bool assign_subvp_pipe(struct dml2_context *ctx, struct dc_state *context
 			continue;
 
 		// Round up
-		refresh_rate = (pipe->stream->timing.pix_clk_100hz * 100 +
+		refresh_rate = (unsigned int)((pipe->stream->timing.pix_clk_100hz * 100 +
 				pipe->stream->timing.v_total * pipe->stream->timing.h_total - 1)
-				/ (double)(pipe->stream->timing.v_total * pipe->stream->timing.h_total);
+				/ (double)(pipe->stream->timing.v_total * pipe->stream->timing.h_total));
 		/* SubVP pipe candidate requirements:
 		 * - Refresh rate < 120hz
 		 * - Not able to switch in vactive naturally (switching in active means the
@@ -264,8 +264,8 @@ static bool assign_subvp_pipe(struct dml2_context *ctx, struct dc_state *context
 			pipe = &context->res_ctx.pipe_ctx[i];
 			if (num_pipes <= free_pipes) {
 				struct dc_stream_state *stream = pipe->stream;
-				unsigned int frame_us = (stream->timing.v_total * stream->timing.h_total /
-						(double)(stream->timing.pix_clk_100hz * 100)) * 1000000;
+				unsigned int frame_us = (unsigned int)((stream->timing.v_total * stream->timing.h_total /
+						(double)(stream->timing.pix_clk_100hz * 100)) * 1000000);
 				if (frame_us > max_frame_time && !stream->ignore_msa_timing_param) {
 					*index = i;
 					max_frame_time = frame_us;
@@ -382,8 +382,8 @@ static bool subvp_subvp_schedulable(struct dml2_context *ctx, struct dc_state *c
 					phantom->timing.v_addressable;
 
 			// Round up when calculating microschedule time (+ 1 at the end)
-			time_us = (microschedule_lines * phantom->timing.h_total) /
-					(double)(phantom->timing.pix_clk_100hz * 100) * 1000000 +
+			time_us = (uint32_t)((microschedule_lines * phantom->timing.h_total) /
+					(double)(phantom->timing.pix_clk_100hz * 100) * 1000000) +
 					ctx->config.svp_pstate.subvp_prefetch_end_to_mall_start_us +
 					ctx->config.svp_pstate.subvp_fw_processing_delay_us + 1;
 			if (time_us > max_microschedule_us)
@@ -402,16 +402,16 @@ static bool subvp_subvp_schedulable(struct dml2_context *ctx, struct dc_state *c
 	if (index < 2 || !subvp_pipes[0] || !subvp_pipes[1])
 		return false;
 
-	vactive1_us = ((subvp_pipes[0]->stream->timing.v_addressable * subvp_pipes[0]->stream->timing.h_total) /
-			(double)(subvp_pipes[0]->stream->timing.pix_clk_100hz * 100)) * 1000000;
-	vactive2_us = ((subvp_pipes[1]->stream->timing.v_addressable * subvp_pipes[1]->stream->timing.h_total) /
-				(double)(subvp_pipes[1]->stream->timing.pix_clk_100hz * 100)) * 1000000;
-	vblank1_us = (((subvp_pipes[0]->stream->timing.v_total - subvp_pipes[0]->stream->timing.v_addressable) *
+	vactive1_us = (int32_t)(((subvp_pipes[0]->stream->timing.v_addressable * subvp_pipes[0]->stream->timing.h_total) /
+			(double)(subvp_pipes[0]->stream->timing.pix_clk_100hz * 100)) * 1000000);
+	vactive2_us = (int32_t)(((subvp_pipes[1]->stream->timing.v_addressable * subvp_pipes[1]->stream->timing.h_total) /
+				(double)(subvp_pipes[1]->stream->timing.pix_clk_100hz * 100)) * 1000000);
+	vblank1_us = (int32_t)(((subvp_pipes[0]->stream->timing.v_total - subvp_pipes[0]->stream->timing.v_addressable) *
 			subvp_pipes[0]->stream->timing.h_total) /
-			(double)(subvp_pipes[0]->stream->timing.pix_clk_100hz * 100)) * 1000000;
-	vblank2_us = (((subvp_pipes[1]->stream->timing.v_total - subvp_pipes[1]->stream->timing.v_addressable) *
+			(double)(subvp_pipes[0]->stream->timing.pix_clk_100hz * 100) * 1000000);
+	vblank2_us = (int32_t)(((subvp_pipes[1]->stream->timing.v_total - subvp_pipes[1]->stream->timing.v_addressable) *
 			subvp_pipes[1]->stream->timing.h_total) /
-			(double)(subvp_pipes[1]->stream->timing.pix_clk_100hz * 100)) * 1000000;
+			(double)(subvp_pipes[1]->stream->timing.pix_clk_100hz * 100) * 1000000);
 
 	if ((vactive1_us - vblank2_us) / 2 > max_microschedule_us &&
 	    (vactive2_us - vblank1_us) / 2 > max_microschedule_us)
@@ -445,13 +445,13 @@ bool dml2_svp_drr_schedulable(struct dml2_context *ctx, struct dc_state *context
 	struct dc_crtc_timing *main_timing = NULL;
 	struct dc_crtc_timing *phantom_timing = NULL;
 	struct dc_stream_state *phantom_stream;
-	int16_t prefetch_us = 0;
-	int16_t mall_region_us = 0;
-	int16_t drr_frame_us = 0;	// nominal frame time
-	int16_t subvp_active_us = 0;
-	int16_t stretched_drr_us = 0;
-	int16_t drr_stretched_vblank_us = 0;
-	int16_t max_vblank_mallregion = 0;
+	int32_t prefetch_us = 0;
+	int32_t mall_region_us = 0;
+	int32_t drr_frame_us = 0;	// nominal frame time
+	int32_t subvp_active_us = 0;
+	int32_t stretched_drr_us = 0;
+	int32_t drr_stretched_vblank_us = 0;
+	int32_t max_vblank_mallregion = 0;
 
 	// Find SubVP pipe
 	for (i = 0; i < ctx->config.dcn_pipe_count; i++) {
@@ -475,19 +475,19 @@ bool dml2_svp_drr_schedulable(struct dml2_context *ctx, struct dc_state *context
 	phantom_stream = ctx->config.svp_pstate.callbacks.get_paired_subvp_stream(context, pipe->stream);
 	main_timing = &pipe->stream->timing;
 	phantom_timing = &phantom_stream->timing;
-	prefetch_us = (phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total /
+	prefetch_us = (int32_t)((phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total /
 			(double)(phantom_timing->pix_clk_100hz * 100) * 1000000 +
-			ctx->config.svp_pstate.subvp_prefetch_end_to_mall_start_us;
-	subvp_active_us = main_timing->v_addressable * main_timing->h_total /
-			(double)(main_timing->pix_clk_100hz * 100) * 1000000;
-	drr_frame_us = drr_timing->v_total * drr_timing->h_total /
-			(double)(drr_timing->pix_clk_100hz * 100) * 1000000;
+			ctx->config.svp_pstate.subvp_prefetch_end_to_mall_start_us);
+	subvp_active_us = (int32_t)(main_timing->v_addressable * main_timing->h_total /
+			(double)(main_timing->pix_clk_100hz * 100) * 1000000);
+	drr_frame_us = (int32_t)(drr_timing->v_total * drr_timing->h_total /
+			(double)(drr_timing->pix_clk_100hz * 100) * 1000000);
 	// P-State allow width and FW delays already included phantom_timing->v_addressable
-	mall_region_us = phantom_timing->v_addressable * phantom_timing->h_total /
-			(double)(phantom_timing->pix_clk_100hz * 100) * 1000000;
+	mall_region_us = (int32_t)(phantom_timing->v_addressable * phantom_timing->h_total /
+			(double)(phantom_timing->pix_clk_100hz * 100) * 1000000);
 	stretched_drr_us = drr_frame_us + mall_region_us + SUBVP_DRR_MARGIN_US;
-	drr_stretched_vblank_us = (drr_timing->v_total - drr_timing->v_addressable) * drr_timing->h_total /
-			(double)(drr_timing->pix_clk_100hz * 100) * 1000000 + (stretched_drr_us - drr_frame_us);
+	drr_stretched_vblank_us = (int32_t)((drr_timing->v_total - drr_timing->v_addressable) * drr_timing->h_total /
+			(double)(drr_timing->pix_clk_100hz * 100) * 1000000 + (stretched_drr_us - drr_frame_us));
 	max_vblank_mallregion = drr_stretched_vblank_us > mall_region_us ? drr_stretched_vblank_us : mall_region_us;
 
 	/* We consider SubVP + DRR schedulable if the stretched frame duration of the DRR display (i.e. the
@@ -526,12 +526,12 @@ static bool subvp_vblank_schedulable(struct dml2_context *ctx, struct dc_state *
 	bool schedulable = false;
 	uint32_t i = 0;
 	uint8_t vblank_index = 0;
-	uint16_t prefetch_us = 0;
-	uint16_t mall_region_us = 0;
-	uint16_t vblank_frame_us = 0;
-	uint16_t subvp_active_us = 0;
-	uint16_t vblank_blank_us = 0;
-	uint16_t max_vblank_mallregion = 0;
+	uint32_t prefetch_us = 0;
+	uint32_t mall_region_us = 0;
+	uint32_t vblank_frame_us = 0;
+	uint32_t subvp_active_us = 0;
+	uint32_t vblank_blank_us = 0;
+	uint32_t max_vblank_mallregion = 0;
 	struct dc_crtc_timing *main_timing = NULL;
 	struct dc_crtc_timing *phantom_timing = NULL;
 	struct dc_crtc_timing *vblank_timing = NULL;
@@ -581,24 +581,27 @@ static bool subvp_vblank_schedulable(struct dml2_context *ctx, struct dc_state *
 		vblank_timing = &context->res_ctx.pipe_ctx[vblank_index].stream->timing;
 		// Prefetch time is equal to VACTIVE + BP + VSYNC of the phantom pipe
 		// Also include the prefetch end to mallstart delay time
-		prefetch_us = (phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total /
+		prefetch_us = (uint32_t)((phantom_timing->v_total - phantom_timing->v_front_porch) * phantom_timing->h_total /
 				(double)(phantom_timing->pix_clk_100hz * 100) * 1000000 +
-				ctx->config.svp_pstate.subvp_prefetch_end_to_mall_start_us;
+				ctx->config.svp_pstate.subvp_prefetch_end_to_mall_start_us);
 		// P-State allow width and FW delays already included phantom_timing->v_addressable
-		mall_region_us = phantom_timing->v_addressable * phantom_timing->h_total /
-				(double)(phantom_timing->pix_clk_100hz * 100) * 1000000;
-		vblank_frame_us = vblank_timing->v_total * vblank_timing->h_total /
-				(double)(vblank_timing->pix_clk_100hz * 100) * 1000000;
-		vblank_blank_us =  (vblank_timing->v_total - vblank_timing->v_addressable) * vblank_timing->h_total /
-				(double)(vblank_timing->pix_clk_100hz * 100) * 1000000;
-		subvp_active_us = main_timing->v_addressable * main_timing->h_total /
-				(double)(main_timing->pix_clk_100hz * 100) * 1000000;
+		mall_region_us = (uint32_t)(phantom_timing->v_addressable * phantom_timing->h_total /
+				(double)(phantom_timing->pix_clk_100hz * 100) * 1000000);
+		vblank_frame_us = (uint32_t)(vblank_timing->v_total * vblank_timing->h_total /
+				(double)(vblank_timing->pix_clk_100hz * 100) * 1000000);
+		vblank_blank_us = (uint32_t)((vblank_timing->v_total - vblank_timing->v_addressable) * vblank_timing->h_total /
+				(double)(vblank_timing->pix_clk_100hz * 100) * 1000000);
+		subvp_active_us = (uint32_t)(main_timing->v_addressable * main_timing->h_total /
+				(double)(main_timing->pix_clk_100hz * 100) * 1000000);
 		max_vblank_mallregion = vblank_blank_us > mall_region_us ? vblank_blank_us : mall_region_us;
+		const uint64_t required_us = (uint64_t)prefetch_us +
+					     (uint64_t)vblank_frame_us +
+					     (uint64_t)max_vblank_mallregion;
 
 		// Schedulable if VACTIVE region of the SubVP pipe can fit the MALL prefetch, VBLANK frame time,
 		// and the max of (VBLANK blanking time, MALL region)
 		// TODO: Possibly add some margin (i.e. the below conditions should be [...] > X instead of [...] > 0)
-		if (subvp_active_us - prefetch_us - vblank_frame_us - max_vblank_mallregion > 0)
+		if ((uint64_t)subvp_active_us > required_us)
 			schedulable = true;
 	}
 	return schedulable;
@@ -694,10 +697,10 @@ static void set_phantom_stream_timing(struct dml2_context *ctx, struct dc_state
 	}
 
 	// Calculate lines required for pstate allow width and FW processing delays
-	pstate_width_fw_delay_lines = ((double)(ctx->config.svp_pstate.subvp_fw_processing_delay_us +
+	pstate_width_fw_delay_lines = (uint32_t)(((double)(ctx->config.svp_pstate.subvp_fw_processing_delay_us +
 			ctx->config.svp_pstate.subvp_pstate_allow_width_us) / 1000000) *
 			(ref_pipe->stream->timing.pix_clk_100hz * 100) /
-			(double)ref_pipe->stream->timing.h_total;
+			(double)ref_pipe->stream->timing.h_total);
 
 	// DML calculation for MALL region doesn't take into account FW delay
 	// and required pstate allow width for multi-display cases
@@ -712,7 +715,7 @@ static void set_phantom_stream_timing(struct dml2_context *ctx, struct dc_state
 	fp_and_sync_width_time = (phantom_stream->timing.v_front_porch + phantom_stream->timing.v_sync_width) * line_time;
 
 	if ((svp_vstartup * line_time) + fp_and_sync_width_time > cvt_rb_vblank_max) {
-		svp_vstartup = (cvt_rb_vblank_max - fp_and_sync_width_time) / line_time;
+		svp_vstartup = (unsigned int)((cvt_rb_vblank_max - fp_and_sync_width_time) / line_time);
 	}
 
 	// For backporch of phantom pipe, use vstartup of the main pipe
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_policy.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_policy.c
index ef693f608d599..ab2964811c5bc 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_policy.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_policy.c
@@ -123,9 +123,9 @@ int dml2_policy_build_synthetic_soc_states(struct dml2_policy_build_synthetic_so
 	struct dml2_policy_build_synthetic_soc_states_params *p)
 {
 	int i, j;
-	unsigned int min_fclk_mhz = p->in_states->state_array[0].fabricclk_mhz;
-	unsigned int min_dcfclk_mhz = p->in_states->state_array[0].dcfclk_mhz;
-	unsigned int min_socclk_mhz = p->in_states->state_array[0].socclk_mhz;
+	unsigned int min_fclk_mhz = (unsigned int)p->in_states->state_array[0].fabricclk_mhz;
+	unsigned int min_dcfclk_mhz = (unsigned int)p->in_states->state_array[0].dcfclk_mhz;
+	unsigned int min_socclk_mhz = (unsigned int)p->in_states->state_array[0].socclk_mhz;
 
 	int max_dcfclk_mhz = 0, max_dispclk_mhz = 0, max_dppclk_mhz = 0,
 		max_phyclk_mhz = 0, max_dtbclk_mhz = 0, max_fclk_mhz = 0,
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c
index 8e0997441ee0e..0d8ff236c6d00 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_translation_helper.c
@@ -508,11 +508,11 @@ void dml2_init_soc_states(struct dml2_context *dml2, const struct dc *in_dc,
 	/* DCFCLK stas values are project specific */
 	if ((dml2->v20.dml_core_ctx.project == dml_project_dcn32) ||
 		(dml2->v20.dml_core_ctx.project == dml_project_dcn321)) {
-		p->dcfclk_stas_mhz[0] = p->in_states->state_array[0].dcfclk_mhz;
+		p->dcfclk_stas_mhz[0] = (int)p->in_states->state_array[0].dcfclk_mhz;
 		p->dcfclk_stas_mhz[1] = 615;
 		p->dcfclk_stas_mhz[2] = 906;
 		p->dcfclk_stas_mhz[3] = 1324;
-		p->dcfclk_stas_mhz[4] = p->in_states->state_array[1].dcfclk_mhz;
+		p->dcfclk_stas_mhz[4] = (int)p->in_states->state_array[1].dcfclk_mhz;
 	} else if (dml2->v20.dml_core_ctx.project != dml_project_dcn35 &&
 			dml2->v20.dml_core_ctx.project != dml_project_dcn36 &&
 			dml2->v20.dml_core_ctx.project != dml_project_dcn351) {
diff --git a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.c b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.c
index 86567e2324150..5ed14f694fb09 100644
--- a/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.c
+++ b/drivers/gpu/drm/amd/display/dc/dml2_0/dml2_utils.c
@@ -255,7 +255,7 @@ static void populate_pipe_ctx_dlg_params_from_dml(struct pipe_ctx *pipe_ctx, str
 	pipe_ctx->pipe_dlg_param.vupdate_width = dml_get_vupdate_width(mode_lib, pipe_idx);
 	pipe_ctx->pipe_dlg_param.vready_offset = dml_get_vready_offset(mode_lib, pipe_idx);
 
-	ASSERT(pipe_ctx->stream_res.tg->inst >= 0 && pipe_ctx->stream_res.tg->inst <= 0xFF);
+	ASSERT(pipe_ctx->stream_res.tg->inst <= 0xFF);
 	pipe_ctx->pipe_dlg_param.otg_inst = (unsigned char)pipe_ctx->stream_res.tg->inst;
 
 	pipe_ctx->pipe_dlg_param.hactive = hactive;
@@ -333,7 +333,7 @@ void dml2_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_state *cont
 		}
 
 		context->bw_ctx.bw.dcn.compbuf_size_kb -= context->res_ctx.pipe_ctx[dc_pipe_ctx_index].det_buffer_size_kb;
-		context->res_ctx.pipe_ctx[dc_pipe_ctx_index].plane_res.bw.dppclk_khz = dml_get_dppclk_calculated(&context->bw_ctx.dml2->v20.dml_core_ctx, dml_pipe_idx) * 1000;
+		context->res_ctx.pipe_ctx[dc_pipe_ctx_index].plane_res.bw.dppclk_khz = (int)(dml_get_dppclk_calculated(&context->bw_ctx.dml2->v20.dml_core_ctx, dml_pipe_idx) * 1000);
 		if (context->bw_ctx.bw.dcn.clk.dppclk_khz < context->res_ctx.pipe_ctx[dc_pipe_ctx_index].plane_res.bw.dppclk_khz)
 			context->bw_ctx.bw.dcn.clk.dppclk_khz = context->res_ctx.pipe_ctx[dc_pipe_ctx_index].plane_res.bw.dppclk_khz;
 
@@ -362,10 +362,10 @@ void dml2_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_state *cont
 	context->bw_ctx.bw.dcn.clk.bw_dppclk_khz = context->bw_ctx.bw.dcn.clk.dppclk_khz;
 	context->bw_ctx.bw.dcn.clk.bw_dispclk_khz = context->bw_ctx.bw.dcn.clk.dispclk_khz;
 
-	context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = in_ctx->v20.dml_core_ctx.states.state_array[in_ctx->v20.scratch.mode_support_params.out_lowest_state_idx].dppclk_mhz
-		* 1000;
-	context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = in_ctx->v20.dml_core_ctx.states.state_array[in_ctx->v20.scratch.mode_support_params.out_lowest_state_idx].dispclk_mhz
-		* 1000;
+	context->bw_ctx.bw.dcn.clk.max_supported_dppclk_khz = (int)(in_ctx->v20.dml_core_ctx.states.state_array[in_ctx->v20.scratch.mode_support_params.out_lowest_state_idx].dppclk_mhz
+		* 1000);
+	context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz = (int)(in_ctx->v20.dml_core_ctx.states.state_array[in_ctx->v20.scratch.mode_support_params.out_lowest_state_idx].dispclk_mhz
+		* 1000);
 
 	if (dc->config.forced_clocks || dc->debug.max_disp_clk) {
 		context->bw_ctx.bw.dcn.clk.bw_dispclk_khz = context->bw_ctx.bw.dcn.clk.max_supported_dispclk_khz;
@@ -375,18 +375,18 @@ void dml2_calculate_rq_and_dlg_params(const struct dc *dc, struct dc_state *cont
 
 void dml2_extract_watermark_set(struct dcn_watermarks *watermark, struct display_mode_lib_st *dml_core_ctx)
 {
-	watermark->urgent_ns = dml_get_wm_urgent(dml_core_ctx) * 1000;
-	watermark->cstate_pstate.cstate_enter_plus_exit_ns = dml_get_wm_stutter_enter_exit(dml_core_ctx) * 1000;
-	watermark->cstate_pstate.cstate_exit_ns = dml_get_wm_stutter_exit(dml_core_ctx) * 1000;
-	watermark->cstate_pstate.pstate_change_ns = dml_get_wm_dram_clock_change(dml_core_ctx) * 1000;
-	watermark->pte_meta_urgent_ns = dml_get_wm_memory_trip(dml_core_ctx) * 1000;
-	watermark->frac_urg_bw_nom = dml_get_fraction_of_urgent_bandwidth(dml_core_ctx) * 1000;
-	watermark->frac_urg_bw_flip = dml_get_fraction_of_urgent_bandwidth_imm_flip(dml_core_ctx) * 1000;
-	watermark->urgent_latency_ns = dml_get_urgent_latency(dml_core_ctx) * 1000;
-	watermark->cstate_pstate.fclk_pstate_change_ns = dml_get_wm_fclk_change(dml_core_ctx) * 1000;
-	watermark->usr_retraining_ns = dml_get_wm_usr_retraining(dml_core_ctx) * 1000;
-	watermark->cstate_pstate.cstate_enter_plus_exit_z8_ns = dml_get_wm_z8_stutter_enter_exit(dml_core_ctx) * 1000;
-	watermark->cstate_pstate.cstate_exit_z8_ns = dml_get_wm_z8_stutter(dml_core_ctx) * 1000;
+	watermark->urgent_ns = (uint32_t)(dml_get_wm_urgent(dml_core_ctx) * 1000);
+	watermark->cstate_pstate.cstate_enter_plus_exit_ns = (uint32_t)(dml_get_wm_stutter_enter_exit(dml_core_ctx) * 1000);
+	watermark->cstate_pstate.cstate_exit_ns = (uint32_t)(dml_get_wm_stutter_exit(dml_core_ctx) * 1000);
+	watermark->cstate_pstate.pstate_change_ns = (uint32_t)(dml_get_wm_dram_clock_change(dml_core_ctx) * 1000);
+	watermark->pte_meta_urgent_ns = (uint32_t)(dml_get_wm_memory_trip(dml_core_ctx) * 1000);
+	watermark->frac_urg_bw_nom = (uint32_t)(dml_get_fraction_of_urgent_bandwidth(dml_core_ctx) * 1000);
+	watermark->frac_urg_bw_flip = (uint32_t)(dml_get_fraction_of_urgent_bandwidth_imm_flip(dml_core_ctx) * 1000);
+	watermark->urgent_latency_ns = (uint32_t)(dml_get_urgent_latency(dml_core_ctx) * 1000);
+	watermark->cstate_pstate.fclk_pstate_change_ns = (uint32_t)(dml_get_wm_fclk_change(dml_core_ctx) * 1000);
+	watermark->usr_retraining_ns = (uint32_t)(dml_get_wm_usr_retraining(dml_core_ctx) * 1000);
+	watermark->cstate_pstate.cstate_enter_plus_exit_z8_ns = (uint32_t)(dml_get_wm_z8_stutter_enter_exit(dml_core_ctx) * 1000);
+	watermark->cstate_pstate.cstate_exit_z8_ns = (uint32_t)(dml_get_wm_z8_stutter(dml_core_ctx) * 1000);
 }
 
 unsigned int dml2_calc_max_scaled_time(
@@ -434,9 +434,9 @@ void dml2_extract_writeback_wm(struct dc_state *context, struct display_mode_lib
 		for (j = 0 ; j < 4; j++) {
 			/*current dml only has one set of watermark, need to follow up*/
 			bw_writeback->mcif_wb_arb[i].cli_watermark[j] =
-					dml_get_wm_writeback_urgent(dml_core_ctx) * 1000;
+					(unsigned int)(dml_get_wm_writeback_urgent(dml_core_ctx) * 1000);
 			bw_writeback->mcif_wb_arb[i].pstate_watermark[j] =
-					dml_get_wm_writeback_dram_clock_change(dml_core_ctx) * 1000;
+					(unsigned int)(dml_get_wm_writeback_dram_clock_change(dml_core_ctx) * 1000);
 		}
 		if (context->res_ctx.pipe_ctx[i].stream->phy_pix_clk != 0) {
 			/* time_per_pixel should be in u6.6 format */
@@ -450,7 +450,7 @@ void dml2_extract_writeback_wm(struct dc_state *context, struct display_mode_lib
 					wbif_mode, 	wb_arb_params->cli_watermark[0]);
 		/*not required any more*/
 		bw_writeback->mcif_wb_arb[i].dram_speed_change_duration =
-			dml_get_wm_writeback_dram_clock_change(dml_core_ctx) * 1000;
+			(unsigned int)(dml_get_wm_writeback_dram_clock_change(dml_core_ctx) * 1000);
 
 	}
 }
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c
index 2bdd063cc1e1d..53b21adc6267d 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_cm.c
@@ -307,7 +307,7 @@ void dpp1_cm_set_output_csc_default(
 {
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
 	const uint16_t *regval = NULL;
-	int arr_size;
+	uint32_t arr_size;
 
 	regval = find_color_matrix(colorspace, &arr_size);
 	if (regval == NULL) {
@@ -824,7 +824,7 @@ void dpp1_program_input_lut(
 		struct dpp *dpp_base,
 		const struct dc_gamma *gamma)
 {
-	int i;
+	unsigned int i;
 	struct dcn10_dpp *dpp = TO_DCN10_DPP(dpp_base);
 	bool rama_occupied = false;
 	uint32_t ram_num;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_dscl.c
index 0d2c9fcd3362c..b9adf8433b36a 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn10/dcn10_dpp_dscl.c
@@ -256,7 +256,7 @@ static void dpp1_dscl_set_scaler_filter(
 	for (phase = 0; phase < (NUM_PHASES / 2 + 1); phase++) {
 		for (pair = 0; pair < tap_pairs; pair++) {
 			even_coef = filter[phase * taps + 2 * pair];
-			if ((pair * 2 + 1) < taps)
+			if ((uint32_t)(pair * 2 + 1) < taps)
 				odd_coef = filter[phase * taps + 2 * pair + 1];
 			else
 				odd_coef = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
index e7880fc61b4ac..2ebc738c37f28 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp.c
@@ -517,11 +517,16 @@ bool dpp3_get_optimal_number_of_taps(
 	else if (max_taps_c < min_taps_c)
 		return false;
 
-	if (scl_data->taps.v_taps > max_taps_y)
-		scl_data->taps.v_taps = max_taps_y;
+	{
+		uint32_t max_supported_taps_y = (uint32_t)max_taps_y;
+		uint32_t max_supported_taps_c = (uint32_t)max_taps_c;
 
-	if (scl_data->taps.v_taps_c > max_taps_c)
-		scl_data->taps.v_taps_c = max_taps_c;
+		if (scl_data->taps.v_taps > max_supported_taps_y)
+			scl_data->taps.v_taps = max_supported_taps_y;
+
+		if (scl_data->taps.v_taps_c > max_supported_taps_c)
+			scl_data->taps.v_taps_c = max_supported_taps_c;
+	}
 
 	if (!dpp->ctx->dc->debug.always_scale) {
 		if (IDENTITY_RATIO(scl_data->ratios.horz)) {
@@ -543,7 +548,7 @@ bool dpp3_get_optimal_number_of_taps(
 
 static void dpp3_deferred_update(struct dpp *dpp_base)
 {
-	int bypass_state;
+	uint32_t bypass_state;
 	struct dcn3_dpp *dpp = TO_DCN30_DPP(dpp_base);
 
 	if (dpp_base->deferred_reg_writes.bits.disable_dscl) {
@@ -598,6 +603,7 @@ static void dpp3_power_on_blnd_lut(
 		if (power_on) {
 			REG_UPDATE(CM_MEM_PWR_CTRL, BLNDGAM_MEM_PWR_FORCE, 0);
 			REG_WAIT(CM_MEM_PWR_STATUS, BLNDGAM_MEM_PWR_STATE, 0, 1, 5);
+			dpp_base->deferred_reg_writes.bits.disable_blnd_lut = false;
 		} else {
 			dpp_base->ctx->dc->optimized_required = true;
 			dpp_base->deferred_reg_writes.bits.disable_blnd_lut = true;
@@ -615,6 +621,7 @@ static void dpp3_power_on_hdr3dlut(
 		if (power_on) {
 			REG_UPDATE(CM_MEM_PWR_CTRL2, HDR3DLUT_MEM_PWR_FORCE, 0);
 			REG_WAIT(CM_MEM_PWR_STATUS2, HDR3DLUT_MEM_PWR_STATE, 0, 1, 5);
+			dpp_base->deferred_reg_writes.bits.disable_3dlut = false;
 		} else {
 			dpp_base->ctx->dc->optimized_required = true;
 			dpp_base->deferred_reg_writes.bits.disable_3dlut = true;
@@ -632,6 +639,7 @@ static void dpp3_power_on_shaper(
 		if (power_on) {
 			REG_UPDATE(CM_MEM_PWR_CTRL2, SHAPER_MEM_PWR_FORCE, 0);
 			REG_WAIT(CM_MEM_PWR_STATUS2, SHAPER_MEM_PWR_STATE, 0, 1, 5);
+			dpp_base->deferred_reg_writes.bits.disable_shaper = false;
 		} else {
 			dpp_base->ctx->dc->optimized_required = true;
 			dpp_base->deferred_reg_writes.bits.disable_shaper = true;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
index 8170a86ad0ea6..e01d8e9672fc9 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn30/dcn30_dpp_cm.c
@@ -136,6 +136,7 @@ static void dpp3_power_on_gamcor_lut(
 			if (dpp_base->ctx->dc->caps.ips_v2_support)
 				REG_UPDATE(CM_MEM_PWR_CTRL, GAMCOR_MEM_PWR_DIS, 1);
 			REG_WAIT(CM_MEM_PWR_STATUS, GAMCOR_MEM_PWR_STATE, 0, 1, 5);
+			dpp_base->deferred_reg_writes.bits.disable_gamcor = false;
 		} else {
 			dpp_base->ctx->dc->optimized_required = true;
 			dpp_base->deferred_reg_writes.bits.disable_gamcor = true;
@@ -317,7 +318,7 @@ void dpp3_set_hdr_multiplier(
 static void program_gamut_remap(
 		struct dcn3_dpp *dpp,
 		const uint16_t *regval,
-		int select)
+		unsigned int select)
 {
 	uint16_t selection = 0;
 	struct color_matrices_reg gam_regs;
@@ -379,7 +380,7 @@ void dpp3_cm_set_gamut_remap(
 {
 	struct dcn3_dpp *dpp = TO_DCN30_DPP(dpp_base);
 	int i = 0;
-	int gamut_mode;
+	uint32_t gamut_mode;
 
 	if (adjust->gamut_adjust_type != GRAPHICS_GAMUT_ADJUST_TYPE_SW)
 		/* Bypass if type is bypass or hw */
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c
index 4f3b48ed86796..340111e3deacb 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_cm.c
@@ -176,7 +176,7 @@ static void dpp401_program_cursor_csc(
 	struct color_matrices_reg cur_matrix_regs;
 	unsigned int i;
 	const uint16_t *regval = NULL;
-	int arr_size = sizeof(dpp_input_csc_matrix) / sizeof(struct dpp_input_csc_matrix);
+	unsigned int arr_size = sizeof(dpp_input_csc_matrix) / sizeof(struct dpp_input_csc_matrix);
 
 	if (color_space < COLOR_SPACE_YCBCR601) {
 		REG_SET(CUR0_MATRIX_MODE, 0, CUR0_MATRIX_MODE, CUR_MATRIX_BYPASS);
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
index 8b7e55e337d35..62994aecf499a 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn401/dcn401_dpp_dscl.c
@@ -161,6 +161,7 @@ static void dpp401_power_on_dscl(
 				REG_WAIT(DSCL_MEM_PWR_STATUS, LUT_MEM_PWR_STATE, 0, 1, 100);
 			} else
 				REG_WAIT(DSCL_MEM_PWR_STATUS, LUT_MEM_PWR_STATE, 0, 1, 5);
+			dpp->base.deferred_reg_writes.bits.disable_dscl = false;
 		} else {
 			if (dpp->base.ctx->dc->debug.enable_mem_low_power.bits.dscl) {
 				dpp->base.ctx->dc->optimized_required = true;
@@ -253,7 +254,7 @@ static void dpp401_dscl_set_scaler_filter(
 	for (phase = 0; phase < (NUM_PHASES / 2 + 1); phase++) {
 		for (pair = 0; pair < tap_pairs; pair++) {
 			even_coef = filter[phase * taps + 2 * pair];
-			if ((pair * 2 + 1) < taps)
+			if ((uint32_t)(pair * 2 + 1) < taps)
 				odd_coef = filter[phase * taps + 2 * pair + 1];
 			else
 				odd_coef = 0;
diff --git a/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c b/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c
index c126fb9d5bfa9..f60e084068c89 100644
--- a/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c
+++ b/drivers/gpu/drm/amd/display/dc/dpp/dcn42/dcn42_dpp.c
@@ -269,10 +269,10 @@ static void dpp42_dpp_setup(
 
 		tbl_entry.color_space = input_color_space;
 
-		if (color_space >= COLOR_SPACE_YCBCR601)
-			select = INPUT_CSC_SELECT_ICSC;
-		else
+		if (dpp3_should_bypass_post_csc_for_colorspace(color_space))
 			select = INPUT_CSC_SELECT_BYPASS;
+		else
+			select = INPUT_CSC_SELECT_ICSC;
 
 		dpp3_program_post_csc(dpp_base, color_space, select,
 			&tbl_entry);
@@ -280,6 +280,14 @@ static void dpp42_dpp_setup(
 		dpp3_program_post_csc(dpp_base, color_space, select, NULL);
 	}
 }
+static void dcn42_dpp_force_disable_cursor(struct dpp *dpp_base)
+{
+	struct dcn401_dpp *dpp = TO_DCN401_DPP(dpp_base);
+
+	/* Force disable cursor */
+	REG_UPDATE(CURSOR0_CONTROL, CUR0_ENABLE, 0);
+	dpp_base->pos.cur0_ctl.bits.cur0_enable = 0;
+}
 
 static struct dpp_funcs dcn42_dpp_funcs = {
 	.dpp_program_gamcor_lut		= dpp3_program_gamcor_lut,
@@ -302,6 +310,7 @@ static struct dpp_funcs dcn42_dpp_funcs = {
 	.dpp_cm_hist_control        = dpp42_dpp_cm_hist_control,
 	.dpp_cm_hist_read           = dpp42_dpp_cm_hist_read,
 	.dpp_read_reg_state			= dpp30_read_reg_state,
+	.dpp_force_disable_cursor	= dcn42_dpp_force_disable_cursor,
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
index 8dfb6dd14eb29..9aa5adb15103d 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c
@@ -858,19 +858,20 @@ static bool decide_dsc_target_bpp_x16(
 		int *target_bpp_x16)
 {
 	struct dc_dsc_bw_range range;
+	uint32_t target_bandwidth_kbps_u = (uint32_t)target_bandwidth_kbps;
 
 	*target_bpp_x16 = 0;
 
 	if (decide_dsc_bandwidth_range(policy->min_target_bpp * 16, policy->max_target_bpp * 16,
 			num_slices_h, dsc_common_caps, timing, link_encoding, &range)) {
-		if (target_bandwidth_kbps >= range.stream_kbps) {
+		if (target_bandwidth_kbps_u >= range.stream_kbps) {
 			if (policy->enable_dsc_when_not_needed || options->force_dsc_when_not_needed)
 				/* enable max bpp even dsc is not needed */
 				*target_bpp_x16 = range.max_target_bpp_x16;
-		} else if (target_bandwidth_kbps >= range.max_kbps) {
+		} else if (target_bandwidth_kbps_u >= range.max_kbps) {
 			/* use max target bpp allowed */
 			*target_bpp_x16 = range.max_target_bpp_x16;
-		} else if (target_bandwidth_kbps >= range.min_kbps) {
+		} else if (target_bandwidth_kbps_u >= range.min_kbps) {
 			/* use target bpp that can take entire target bandwidth */
 			*target_bpp_x16 = compute_bpp_x16_from_target_bandwidth(
 					target_bandwidth_kbps, timing, num_slices_h,
@@ -1053,10 +1054,11 @@ static bool setup_dsc_config(
 	int max_slices_h = 0;
 	int num_slices_h = 0;
 	int pic_width;
+	uint32_t pic_width_u;
 	int slice_width;
 	int target_bpp;
 	int sink_per_slice_throughput_mps;
-	int branch_max_throughput_mps = 0;
+	uint32_t branch_max_throughput_mps = 0;
 	bool is_dsc_possible = false;
 	int pic_height;
 	int slice_height;
@@ -1066,12 +1068,13 @@ static bool setup_dsc_config(
 
 	dc_dsc_get_policy_for_timing(timing, options->max_target_bpp_limit_override_x16, &policy, link_encoding);
 	pic_width = timing->h_addressable + timing->h_border_left + timing->h_border_right;
+	pic_width_u = (uint32_t)pic_width;
 	pic_height = timing->v_addressable + timing->v_border_top + timing->v_border_bottom;
 
 	if (!dsc_sink_caps->is_dsc_supported)
 		goto done;
 
-	if (dsc_sink_caps->branch_max_line_width && dsc_sink_caps->branch_max_line_width < pic_width)
+	if (dsc_sink_caps->branch_max_line_width && dsc_sink_caps->branch_max_line_width < pic_width_u)
 		goto done;
 
 	// Intersect decoder with encoder DSC caps and validate DSC settings
@@ -1333,7 +1336,7 @@ uint32_t dc_dsc_stream_bandwidth_in_kbps(const struct dc_crtc_timing *timing,
 
 uint32_t dc_dsc_stream_bandwidth_overhead_in_kbps(
 		const struct dc_crtc_timing *timing,
-		const int num_slices_h,
+		const uint32_t num_slices_h,
 		const bool is_dp)
 {
 	struct fixed31_32 max_dsc_overhead;
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dcn20/dcn20_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dcn20/dcn20_dsc.c
index 6e1e759462bf9..cede9588bad87 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/dcn20/dcn20_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dcn20/dcn20_dsc.c
@@ -168,8 +168,9 @@ bool dsc2_validate_stream(struct display_stream_compressor *dsc, const struct ds
 {
 	struct dsc_optc_config dsc_optc_cfg;
 	struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
+	uint32_t max_image_width = (uint32_t)dsc20->max_image_width;
 
-	if (dsc_cfg->pic_width > dsc20->max_image_width)
+	if (dsc_cfg->pic_width > max_image_width)
 		return false;
 
 	return dsc_prepare_config(dsc_cfg, &dsc20->reg_vals, &dsc_optc_cfg);
@@ -227,9 +228,9 @@ bool dsc2_get_packed_pps(struct display_stream_compressor *dsc, const struct dsc
 void dsc2_enable(struct display_stream_compressor *dsc, int opp_pipe)
 {
 	struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
-	int dsc_clock_en;
-	int dsc_fw_config;
-	int enabled_opp_pipe;
+	uint32_t dsc_clock_en;
+	uint32_t dsc_fw_config;
+	uint32_t enabled_opp_pipe;
 
 	DC_LOG_DSC("enable DSC %d at opp pipe %d", dsc->inst, opp_pipe);
 
@@ -252,7 +253,7 @@ void dsc2_enable(struct display_stream_compressor *dsc, int opp_pipe)
 void dsc2_disable(struct display_stream_compressor *dsc)
 {
 	struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
-	int dsc_clock_en;
+	uint32_t dsc_clock_en;
 
 	DC_LOG_DSC("disable DSC %d", dsc->inst);
 
@@ -405,20 +406,20 @@ bool dsc_prepare_config(const struct dsc_config *dsc_cfg, struct dsc_reg_values
 	dsc_reg_vals->pixel_format = dsc_dc_pixel_encoding_to_dsc_pixel_format(dsc_cfg->pixel_encoding, dsc_cfg->dc_dsc_cfg.ycbcr422_simple);
 	dsc_reg_vals->num_slices_h = dsc_cfg->dc_dsc_cfg.num_slices_h;
 	dsc_reg_vals->num_slices_v = dsc_cfg->dc_dsc_cfg.num_slices_v;
-	dsc_reg_vals->pps.dsc_version_minor = dsc_cfg->dc_dsc_cfg.version_minor;
-	dsc_reg_vals->pps.pic_width = dsc_cfg->pic_width;
-	dsc_reg_vals->pps.pic_height = dsc_cfg->pic_height;
+	dsc_reg_vals->pps.dsc_version_minor = (u8)dsc_cfg->dc_dsc_cfg.version_minor;
+	dsc_reg_vals->pps.pic_width = (u16)dsc_cfg->pic_width;
+	dsc_reg_vals->pps.pic_height = (u16)dsc_cfg->pic_height;
 	dsc_reg_vals->pps.bits_per_component = dsc_dc_color_depth_to_dsc_bits_per_comp(dsc_cfg->color_depth);
 	dsc_reg_vals->pps.block_pred_enable = dsc_cfg->dc_dsc_cfg.block_pred_enable;
-	dsc_reg_vals->pps.line_buf_depth = dsc_cfg->dc_dsc_cfg.linebuf_depth;
+	dsc_reg_vals->pps.line_buf_depth = (u8)dsc_cfg->dc_dsc_cfg.linebuf_depth;
 	dsc_reg_vals->alternate_ich_encoding_en = dsc_reg_vals->pps.dsc_version_minor == 1 ? 0 : 1;
 	dsc_reg_vals->ich_reset_at_eol = (dsc_cfg->is_odm || dsc_reg_vals->num_slices_h > 1) ? 0xF : 0;
 
 	// Need to find the ceiling value for the slice width
-	dsc_reg_vals->pps.slice_width = (dsc_cfg->pic_width + dsc_cfg->dsc_padding + dsc_cfg->dc_dsc_cfg.num_slices_h - 1) / dsc_cfg->dc_dsc_cfg.num_slices_h;
+	dsc_reg_vals->pps.slice_width = (u16)((dsc_cfg->pic_width + dsc_cfg->dsc_padding + dsc_cfg->dc_dsc_cfg.num_slices_h - 1) / dsc_cfg->dc_dsc_cfg.num_slices_h);
 	// TODO: in addition to validating slice height (pic height must be divisible by slice height),
 	// see what happens when the same condition doesn't apply for slice_width/pic_width.
-	dsc_reg_vals->pps.slice_height = dsc_cfg->pic_height / dsc_cfg->dc_dsc_cfg.num_slices_v;
+	dsc_reg_vals->pps.slice_height = (u16)(dsc_cfg->pic_height / dsc_cfg->dc_dsc_cfg.num_slices_v);
 
 	ASSERT(dsc_reg_vals->pps.slice_height * dsc_cfg->dc_dsc_cfg.num_slices_v == dsc_cfg->pic_height);
 	if (!(dsc_reg_vals->pps.slice_height * dsc_cfg->dc_dsc_cfg.num_slices_v == dsc_cfg->pic_height)) {
@@ -428,9 +429,9 @@ bool dsc_prepare_config(const struct dsc_config *dsc_cfg, struct dsc_reg_values
 
 	dsc_reg_vals->bpp_x32 = dsc_cfg->dc_dsc_cfg.bits_per_pixel << 1;
 	if (dsc_reg_vals->pixel_format == DSC_PIXFMT_NATIVE_YCBCR420 || dsc_reg_vals->pixel_format == DSC_PIXFMT_NATIVE_YCBCR422)
-		dsc_reg_vals->pps.bits_per_pixel = dsc_reg_vals->bpp_x32;
+		dsc_reg_vals->pps.bits_per_pixel = (u16)dsc_reg_vals->bpp_x32;
 	else
-		dsc_reg_vals->pps.bits_per_pixel = dsc_reg_vals->bpp_x32 >> 1;
+		dsc_reg_vals->pps.bits_per_pixel = (u16)(dsc_reg_vals->bpp_x32 >> 1);
 
 	dsc_reg_vals->pps.convert_rgb = dsc_reg_vals->pixel_format == DSC_PIXFMT_RGB ? 1 : 0;
 	dsc_reg_vals->pps.native_422 = (dsc_reg_vals->pixel_format == DSC_PIXFMT_NATIVE_YCBCR422);
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dcn35/dcn35_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dcn35/dcn35_dsc.c
index 17acb64a9d804..d84c3399d3868 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/dcn35/dcn35_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dcn35/dcn35_dsc.c
@@ -79,9 +79,9 @@ void dsc35_construct(struct dcn20_dsc *dsc,
 static void dsc35_enable(struct display_stream_compressor *dsc, int opp_pipe)
 {
 	struct dcn20_dsc *dsc20 = TO_DCN20_DSC(dsc);
-	int dsc_clock_en;
-	int dsc_fw_config;
-	int enabled_opp_pipe;
+	uint32_t dsc_clock_en;
+	uint32_t dsc_fw_config;
+	uint32_t enabled_opp_pipe;
 
 	DC_LOG_DSC("enable DSC %d at opp pipe %d", dsc->inst, opp_pipe);
 
@@ -96,7 +96,7 @@ static void dsc35_enable(struct display_stream_compressor *dsc, int opp_pipe)
 	REG_GET(DSC_TOP_CONTROL, DSC_CLOCK_EN, &dsc_clock_en);
 	REG_GET_2(DSCRM_DSC_FORWARD_CONFIG, DSCRM_DSC_FORWARD_EN, &dsc_fw_config, DSCRM_DSC_OPP_PIPE_SOURCE, &enabled_opp_pipe);
 	if ((dsc_clock_en || dsc_fw_config) && enabled_opp_pipe != opp_pipe) {
-		DC_LOG_DSC("ERROR: DSC %d at opp pipe %d already enabled!", dsc->inst, enabled_opp_pipe);
+		DC_LOG_DSC("ERROR: DSC %d at opp pipe %u already enabled!", dsc->inst, enabled_opp_pipe);
 		ASSERT(0);
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/dcn401/dcn401_dsc.c b/drivers/gpu/drm/amd/display/dc/dsc/dcn401/dcn401_dsc.c
index 41c3b814b6bd9..7495479600469 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/dcn401/dcn401_dsc.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/dcn401/dcn401_dsc.c
@@ -119,8 +119,9 @@ bool dsc401_validate_stream(struct display_stream_compressor *dsc, const struct
 {
 	struct dsc_optc_config dsc_optc_cfg;
 	struct dcn401_dsc *dsc401 = TO_DCN401_DSC(dsc);
+	uint32_t max_image_width = (uint32_t)dsc401->max_image_width;
 
-	if (dsc_cfg->pic_width > dsc401->max_image_width)
+	if (dsc_cfg->pic_width > max_image_width)
 		return false;
 
 	return dsc_prepare_config(dsc_cfg, &dsc401->reg_vals, &dsc_optc_cfg);
@@ -144,16 +145,16 @@ void dsc401_set_config(struct display_stream_compressor *dsc, const struct dsc_c
 void dsc401_enable(struct display_stream_compressor *dsc, int opp_pipe)
 {
 	struct dcn401_dsc *dsc401 = TO_DCN401_DSC(dsc);
-	int dsc_clock_en;
-	int dsc_fw_config;
-	int enabled_opp_pipe;
+	uint32_t dsc_clock_en;
+	uint32_t dsc_fw_config;
+	uint32_t enabled_opp_pipe;
 
 	DC_LOG_DSC("enable DSC %d at opp pipe %d", dsc->inst, opp_pipe);
 
 	REG_GET(DSC_TOP_CONTROL, DSC_CLOCK_EN, &dsc_clock_en);
 	REG_GET_2(DSCRM_DSC_FORWARD_CONFIG, DSCRM_DSC_FORWARD_EN, &dsc_fw_config, DSCRM_DSC_OPP_PIPE_SOURCE, &enabled_opp_pipe);
 	if ((dsc_clock_en || dsc_fw_config) && enabled_opp_pipe != opp_pipe) {
-		DC_LOG_DSC("ERROR: DSC %d at opp pipe %d already enabled!", dsc->inst, enabled_opp_pipe);
+		DC_LOG_DSC("ERROR: DSC %d at opp pipe %u already enabled!", dsc->inst, enabled_opp_pipe);
 		ASSERT(0);
 	}
 
@@ -169,7 +170,7 @@ void dsc401_enable(struct display_stream_compressor *dsc, int opp_pipe)
 void dsc401_disable(struct display_stream_compressor *dsc)
 {
 	struct dcn401_dsc *dsc401 = TO_DCN401_DSC(dsc);
-	int dsc_clock_en;
+	uint32_t dsc_clock_en;
 
 	DC_LOG_DSC("disable DSC %d", dsc->inst);
 
diff --git a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
index 59864130cf83b..6c07d9a87bfe3 100644
--- a/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
+++ b/drivers/gpu/drm/amd/display/dc/dsc/rc_calc_dpi.c
@@ -72,27 +72,27 @@ static void copy_rc_to_cfg(struct drm_dsc_config *dsc_cfg, const struct rc_param
 {
 	int i;
 
-	dsc_cfg->rc_quant_incr_limit0   = rc->rc_quant_incr_limit0;
-	dsc_cfg->rc_quant_incr_limit1   = rc->rc_quant_incr_limit1;
-	dsc_cfg->initial_offset         = rc->initial_fullness_offset;
-	dsc_cfg->initial_xmit_delay     = rc->initial_xmit_delay;
-	dsc_cfg->first_line_bpg_offset  = rc->first_line_bpg_offset;
-	dsc_cfg->second_line_bpg_offset = rc->second_line_bpg_offset;
-	dsc_cfg->flatness_min_qp        = rc->flatness_min_qp;
-	dsc_cfg->flatness_max_qp        = rc->flatness_max_qp;
+	dsc_cfg->rc_quant_incr_limit0   = (u8)rc->rc_quant_incr_limit0;
+	dsc_cfg->rc_quant_incr_limit1   = (u8)rc->rc_quant_incr_limit1;
+	dsc_cfg->initial_offset         = (u16)rc->initial_fullness_offset;
+	dsc_cfg->initial_xmit_delay     = (u16)rc->initial_xmit_delay;
+	dsc_cfg->first_line_bpg_offset  = (u8)rc->first_line_bpg_offset;
+	dsc_cfg->second_line_bpg_offset = (u8)rc->second_line_bpg_offset;
+	dsc_cfg->flatness_min_qp        = (u8)rc->flatness_min_qp;
+	dsc_cfg->flatness_max_qp        = (u8)rc->flatness_max_qp;
 	for (i = 0; i < QP_SET_SIZE; ++i) {
-		dsc_cfg->rc_range_params[i].range_min_qp     = rc->qp_min[i];
-		dsc_cfg->rc_range_params[i].range_max_qp     = rc->qp_max[i];
+		dsc_cfg->rc_range_params[i].range_min_qp     = (u8)rc->qp_min[i];
+		dsc_cfg->rc_range_params[i].range_max_qp     = (u8)rc->qp_max[i];
 		/* Truncate 8-bit signed value to 6-bit signed value */
 		dsc_cfg->rc_range_params[i].range_bpg_offset = 0x3f & rc->ofs[i];
 	}
-	dsc_cfg->rc_model_size    = rc->rc_model_size;
-	dsc_cfg->rc_edge_factor   = rc->rc_edge_factor;
-	dsc_cfg->rc_tgt_offset_high = rc->rc_tgt_offset_hi;
-	dsc_cfg->rc_tgt_offset_low = rc->rc_tgt_offset_lo;
+	dsc_cfg->rc_model_size    = (u16)rc->rc_model_size;
+	dsc_cfg->rc_edge_factor   = (u8)rc->rc_edge_factor;
+	dsc_cfg->rc_tgt_offset_high = (u8)rc->rc_tgt_offset_hi;
+	dsc_cfg->rc_tgt_offset_low = (u8)rc->rc_tgt_offset_lo;
 
 	for (i = 0; i < QP_SET_SIZE - 1; ++i)
-		dsc_cfg->rc_buf_thresh[i] = rc->rc_buf_thresh[i];
+		dsc_cfg->rc_buf_thresh[i] = (u16)rc->rc_buf_thresh[i];
 }
 
 int dscc_compute_dsc_parameters(const struct drm_dsc_config *pps,
@@ -103,7 +103,8 @@ int dscc_compute_dsc_parameters(const struct drm_dsc_config *pps,
 	struct drm_dsc_config   dsc_cfg;
 
 	dsc_params->pps = *pps;
-	dsc_params->pps.initial_scale_value = 8 * rc->rc_model_size / (rc->rc_model_size - rc->initial_fullness_offset);
+	dsc_params->pps.initial_scale_value = (u8)(8 * rc->rc_model_size /
+			(rc->rc_model_size - rc->initial_fullness_offset));
 
 	copy_pps_fields(&dsc_cfg, &dsc_params->pps);
 	copy_rc_to_cfg(&dsc_cfg, rc);
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c
index 86d60986d6692..a76884a069e85 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_ddc.c
@@ -36,7 +36,7 @@
 
 #undef FN
 #define FN(reg_name, field_name) \
-	ddc->shifts->field_name, ddc->masks->field_name
+	gpio_reg_shift(ddc->shifts->field_name), ddc->masks->field_name
 
 #define CTX \
 	ddc->base.base.ctx
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_generic.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_generic.c
index 7f137cb968953..5db83920e3672 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_generic.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_generic.c
@@ -37,7 +37,7 @@
 
 #undef FN
 #define FN(reg_name, field_name) \
-	generic->shifts->field_name, generic->masks->field_name
+	gpio_reg_shift(generic->shifts->field_name), generic->masks->field_name
 
 #define CTX \
 	generic->base.base.ctx
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.c
index f0d4009728973..07651dc7e205a 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.c
@@ -32,7 +32,7 @@
 
 #undef FN
 #define FN(reg_name, field_name) \
-	gpio->regs->field_name ## _shift, gpio->regs->field_name ## _mask
+	gpio_reg_shift(gpio->regs->field_name ## _shift), gpio->regs->field_name ## _mask
 
 #define CTX \
 	gpio->base.ctx
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.h b/drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.h
index bca0cef18ff9c..5b551068cdd68 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.h
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_gpio.h
@@ -141,4 +141,13 @@ enum gpio_result dal_hw_gpio_change_mode(
 void dal_hw_gpio_close(
 	struct hw_gpio_pin *ptr);
 
+/*
+ * Shared helper used by all GPIO register helpers that pass a field shift
+ * (stored as uint32_t) into register functions that expect uint8_t.
+ */
+static inline uint8_t gpio_reg_shift(uint32_t shift)
+{
+	return (uint8_t)shift;
+}
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c b/drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c
index 79e107904e218..2cb960e517ae9 100644
--- a/drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c
+++ b/drivers/gpu/drm/amd/display/dc/gpio/hw_hpd.c
@@ -35,7 +35,7 @@
 
 #undef FN
 #define FN(reg_name, field_name) \
-	hpd->shifts->field_name, hpd->masks->field_name
+	gpio_reg_shift(hpd->shifts->field_name), hpd->masks->field_name
 
 #define CTX \
 	hpd->base.base.ctx
diff --git a/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c
index 92ed130aeaec9..79d6ab8ed71c4 100644
--- a/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c
+++ b/drivers/gpu/drm/amd/display/dc/hpo/dcn31/dcn31_hpo_dp_stream_encoder.c
@@ -468,7 +468,7 @@ static void dcn31_hpo_dp_stream_enc_update_dp_info_packets(
 				&info_frame->spd,
 				true);
 
-	if (info_frame->hdrsmd.valid)
+	if (info_frame->hdrsmd.valid && !info_frame->firmware_controlled_hdr_info_packet)
 		enc->vpg->funcs->update_generic_info_packet(
 				enc->vpg,
 				3,  /* packetIndex */
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c
index 97ef8281a476e..d683d0740c133 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn10/dcn10_hubbub.c
@@ -638,27 +638,27 @@ void hubbub1_update_dchub(
 				SDPIF_FB_BASE, 0x0FFFF);
 
 		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
-				SDPIF_AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
+				SDPIF_AGP_BASE, (uint32_t)(dh_data->zfb_phys_addr_base >> 22));
 
 		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
-				SDPIF_AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
+				SDPIF_AGP_BOT, (uint32_t)(dh_data->zfb_mc_base_addr >> 22));
 
 		REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
-				SDPIF_AGP_TOP, (dh_data->zfb_mc_base_addr +
-						dh_data->zfb_size_in_byte - 1) >> 22);
+				SDPIF_AGP_TOP, (uint32_t)((dh_data->zfb_mc_base_addr +
+						dh_data->zfb_size_in_byte - 1) >> 22));
 		break;
 	case FRAME_BUFFER_MODE_MIXED_ZFB_AND_LOCAL:
 		/*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
 
 		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BASE,
-				SDPIF_AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
+				SDPIF_AGP_BASE, (uint32_t)(dh_data->zfb_phys_addr_base >> 22));
 
 		REG_UPDATE(DCHUBBUB_SDPIF_AGP_BOT,
-				SDPIF_AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
+				SDPIF_AGP_BOT, (uint32_t)(dh_data->zfb_mc_base_addr >> 22));
 
 		REG_UPDATE(DCHUBBUB_SDPIF_AGP_TOP,
-				SDPIF_AGP_TOP, (dh_data->zfb_mc_base_addr +
-						dh_data->zfb_size_in_byte - 1) >> 22);
+				SDPIF_AGP_TOP, (uint32_t)((dh_data->zfb_mc_base_addr +
+						dh_data->zfb_size_in_byte - 1) >> 22));
 		break;
 	case FRAME_BUFFER_MODE_LOCAL_ONLY:
 		/*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c
index 5c6f7ddafd6be..053a08b6d3a3d 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.c
@@ -398,17 +398,17 @@ int hubbub2_init_dchub_sys_ctx(struct hubbub *hubbub,
 	struct dcn_vmid_page_table_config phys_config;
 
 	REG_SET(DCN_VM_FB_LOCATION_BASE, 0,
-			FB_BASE, pa_config->system_aperture.fb_base >> 24);
+			FB_BASE, ADDR_HI24(pa_config->system_aperture.fb_base));
 	REG_SET(DCN_VM_FB_LOCATION_TOP, 0,
-			FB_TOP, pa_config->system_aperture.fb_top >> 24);
+			FB_TOP, ADDR_HI24(pa_config->system_aperture.fb_top));
 	REG_SET(DCN_VM_FB_OFFSET, 0,
-			FB_OFFSET, pa_config->system_aperture.fb_offset >> 24);
+			FB_OFFSET, ADDR_HI24(pa_config->system_aperture.fb_offset));
 	REG_SET(DCN_VM_AGP_BOT, 0,
-			AGP_BOT, pa_config->system_aperture.agp_bot >> 24);
+			AGP_BOT, ADDR_HI24(pa_config->system_aperture.agp_bot));
 	REG_SET(DCN_VM_AGP_TOP, 0,
-			AGP_TOP, pa_config->system_aperture.agp_top >> 24);
+			AGP_TOP, ADDR_HI24(pa_config->system_aperture.agp_top));
 	REG_SET(DCN_VM_AGP_BASE, 0,
-			AGP_BASE, pa_config->system_aperture.agp_base >> 24);
+			AGP_BASE, ADDR_HI24(pa_config->system_aperture.agp_base));
 
 	REG_SET(DCN_VM_PROTECTION_FAULT_DEFAULT_ADDR_MSB, 0,
 			DCN_VM_PROTECTION_FAULT_DEFAULT_ADDR_MSB, (pa_config->page_table_default_page_addr >> 44) & 0xF);
@@ -447,36 +447,36 @@ void hubbub2_update_dchub(struct hubbub *hubbub,
 
 		/*This field defines the 24 MSBs, bits [47:24] of the 48 bit AGP Base*/
 		REG_UPDATE(DCN_VM_AGP_BASE,
-				AGP_BASE, dh_data->zfb_phys_addr_base >> 24);
+				AGP_BASE, ADDR_HI24(dh_data->zfb_phys_addr_base));
 
 		/*This field defines the bottom range of the AGP aperture and represents the 24*/
 		/*MSBs, bits [47:24] of the 48 address bits*/
 		REG_UPDATE(DCN_VM_AGP_BOT,
-				AGP_BOT, dh_data->zfb_mc_base_addr >> 24);
+				AGP_BOT, ADDR_HI24(dh_data->zfb_mc_base_addr));
 
 		/*This field defines the top range of the AGP aperture and represents the 24*/
 		/*MSBs, bits [47:24] of the 48 address bits*/
 		REG_UPDATE(DCN_VM_AGP_TOP,
-				AGP_TOP, (dh_data->zfb_mc_base_addr +
-						dh_data->zfb_size_in_byte - 1) >> 24);
+				AGP_TOP, ADDR_HI24(dh_data->zfb_mc_base_addr +
+						dh_data->zfb_size_in_byte - 1));
 		break;
 	case FRAME_BUFFER_MODE_MIXED_ZFB_AND_LOCAL:
 		/*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
 
 		/*This field defines the 24 MSBs, bits [47:24] of the 48 bit AGP Base*/
 		REG_UPDATE(DCN_VM_AGP_BASE,
-				AGP_BASE, dh_data->zfb_phys_addr_base >> 24);
+				AGP_BASE, ADDR_HI24(dh_data->zfb_phys_addr_base));
 
 		/*This field defines the bottom range of the AGP aperture and represents the 24*/
 		/*MSBs, bits [47:24] of the 48 address bits*/
 		REG_UPDATE(DCN_VM_AGP_BOT,
-				AGP_BOT, dh_data->zfb_mc_base_addr >> 24);
+				AGP_BOT, ADDR_HI24(dh_data->zfb_mc_base_addr));
 
 		/*This field defines the top range of the AGP aperture and represents the 24*/
 		/*MSBs, bits [47:24] of the 48 address bits*/
 		REG_UPDATE(DCN_VM_AGP_TOP,
-				AGP_TOP, (dh_data->zfb_mc_base_addr +
-						dh_data->zfb_size_in_byte - 1) >> 24);
+				AGP_TOP, ADDR_HI24(dh_data->zfb_mc_base_addr +
+						dh_data->zfb_size_in_byte - 1));
 		break;
 	case FRAME_BUFFER_MODE_LOCAL_ONLY:
 		/*Should not touch FB LOCATION (should be done by VBIOS)*/
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.h b/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.h
index 46d8f5c70750a..6223dfaee270d 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.h
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn20/dcn20_hubbub.h
@@ -141,4 +141,7 @@ void hubbub2_wm_read_state(struct hubbub *hubbub,
 void hubbub2_read_state(struct hubbub *hubbub,
 		struct dcn_hubbub_state *hubbub_state);
 
+/* Extract bits [47:24] of a physical address for hardware register fields */
+#define ADDR_HI24(a)  ((uint32_t)((uint64_t)(a) >> 24))
+
 #endif
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c
index e4496ad203b2f..d790d6ee359a2 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn21/dcn21_hubbub.c
@@ -111,17 +111,17 @@ int hubbub21_init_dchub(struct hubbub *hubbub,
 	struct dcn_vmid_page_table_config phys_config;
 
 	REG_SET(DCN_VM_FB_LOCATION_BASE, 0,
-			FB_BASE, pa_config->system_aperture.fb_base >> 24);
+			FB_BASE, ADDR_HI24(pa_config->system_aperture.fb_base));
 	REG_SET(DCN_VM_FB_LOCATION_TOP, 0,
-			FB_TOP, pa_config->system_aperture.fb_top >> 24);
+			FB_TOP, ADDR_HI24(pa_config->system_aperture.fb_top));
 	REG_SET(DCN_VM_FB_OFFSET, 0,
-			FB_OFFSET, pa_config->system_aperture.fb_offset >> 24);
+			FB_OFFSET, ADDR_HI24(pa_config->system_aperture.fb_offset));
 	REG_SET(DCN_VM_AGP_BOT, 0,
-			AGP_BOT, pa_config->system_aperture.agp_bot >> 24);
+			AGP_BOT, ADDR_HI24(pa_config->system_aperture.agp_bot));
 	REG_SET(DCN_VM_AGP_TOP, 0,
-			AGP_TOP, pa_config->system_aperture.agp_top >> 24);
+			AGP_TOP, ADDR_HI24(pa_config->system_aperture.agp_top));
 	REG_SET(DCN_VM_AGP_BASE, 0,
-			AGP_BASE, pa_config->system_aperture.agp_base >> 24);
+			AGP_BASE, ADDR_HI24(pa_config->system_aperture.agp_base));
 
 	if (pa_config->gart_config.page_table_start_addr != pa_config->gart_config.page_table_end_addr) {
 		phys_config.page_table_start_addr = pa_config->gart_config.page_table_start_addr >> 12;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c
index 181a93dc46e69..6a7c1bee57472 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn30/dcn30_hubbub.c
@@ -68,17 +68,17 @@ int hubbub3_init_dchub_sys_ctx(struct hubbub *hubbub,
 	struct dcn_vmid_page_table_config phys_config;
 
 	REG_SET(DCN_VM_FB_LOCATION_BASE, 0,
-			FB_BASE, pa_config->system_aperture.fb_base >> 24);
+			FB_BASE, ADDR_HI24(pa_config->system_aperture.fb_base));
 	REG_SET(DCN_VM_FB_LOCATION_TOP, 0,
-			FB_TOP, pa_config->system_aperture.fb_top >> 24);
+			FB_TOP, ADDR_HI24(pa_config->system_aperture.fb_top));
 	REG_SET(DCN_VM_FB_OFFSET, 0,
-			FB_OFFSET, pa_config->system_aperture.fb_offset >> 24);
+			FB_OFFSET, ADDR_HI24(pa_config->system_aperture.fb_offset));
 	REG_SET(DCN_VM_AGP_BOT, 0,
-			AGP_BOT, pa_config->system_aperture.agp_bot >> 24);
+			AGP_BOT, ADDR_HI24(pa_config->system_aperture.agp_bot));
 	REG_SET(DCN_VM_AGP_TOP, 0,
-			AGP_TOP, pa_config->system_aperture.agp_top >> 24);
+			AGP_TOP, ADDR_HI24(pa_config->system_aperture.agp_top));
 	REG_SET(DCN_VM_AGP_BASE, 0,
-			AGP_BASE, pa_config->system_aperture.agp_base >> 24);
+			AGP_BASE, ADDR_HI24(pa_config->system_aperture.agp_base));
 
 	if (pa_config->gart_config.page_table_start_addr != pa_config->gart_config.page_table_end_addr) {
 		phys_config.page_table_start_addr = pa_config->gart_config.page_table_start_addr >> 12;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
index 3c298192f3596..79cb506be5cbf 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn31/dcn31_hubbub.c
@@ -910,17 +910,17 @@ int hubbub31_init_dchub_sys_ctx(struct hubbub *hubbub,
 	struct dcn_vmid_page_table_config phys_config;
 
 	REG_SET(DCN_VM_FB_LOCATION_BASE, 0,
-			FB_BASE, pa_config->system_aperture.fb_base >> 24);
+			FB_BASE, ADDR_HI24(pa_config->system_aperture.fb_base));
 	REG_SET(DCN_VM_FB_LOCATION_TOP, 0,
-			FB_TOP, pa_config->system_aperture.fb_top >> 24);
+			FB_TOP, ADDR_HI24(pa_config->system_aperture.fb_top));
 	REG_SET(DCN_VM_FB_OFFSET, 0,
-			FB_OFFSET, pa_config->system_aperture.fb_offset >> 24);
+			FB_OFFSET, ADDR_HI24(pa_config->system_aperture.fb_offset));
 	REG_SET(DCN_VM_AGP_BOT, 0,
-			AGP_BOT, pa_config->system_aperture.agp_bot >> 24);
+			AGP_BOT, ADDR_HI24(pa_config->system_aperture.agp_bot));
 	REG_SET(DCN_VM_AGP_TOP, 0,
-			AGP_TOP, pa_config->system_aperture.agp_top >> 24);
+			AGP_TOP, ADDR_HI24(pa_config->system_aperture.agp_top));
 	REG_SET(DCN_VM_AGP_BASE, 0,
-			AGP_BASE, pa_config->system_aperture.agp_base >> 24);
+			AGP_BASE, ADDR_HI24(pa_config->system_aperture.agp_base));
 
 	if (pa_config->gart_config.page_table_start_addr != pa_config->gart_config.page_table_end_addr) {
 		phys_config.page_table_start_addr = pa_config->gart_config.page_table_start_addr >> 12;
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
index 3b9542c08f3d6..bf30a1bb61b75 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn401/dcn401_hubbub.c
@@ -617,7 +617,9 @@ bool hubbub401_dcc_support_swizzle(
 			swizzle_supported = true;
 		break;
 	case DC_ADDR3_SW_64KB_2D:
+	case DC_ADDR3_SW_64KB_2D_Z:
 	case DC_ADDR3_SW_256KB_2D:
+	case DC_ADDR3_SW_256KB_2D_Z:
 		swizzle_supported = true;
 		break;
 	default:
@@ -833,7 +835,7 @@ bool hubbub401_get_dcc_compression_cap(struct hubbub *hubbub,
 		struct dc_surface_dcc_cap *output)
 {
 	struct dc *dc = hubbub->ctx->dc;
-	const unsigned int max_dcc_plane_width = dc->caps.dcc_plane_width_limit;
+	const int max_dcc_plane_width = (int)dc->caps.dcc_plane_width_limit;
 	/* DCN4_Programming_Guide_DCHUB.docx, Section 5.11.2.2 */
 	enum dcc_control dcc_control;
 	unsigned int plane0_bpe, plane1_bpe;
@@ -1198,6 +1200,7 @@ bool dcn401_program_arbiter(struct hubbub *hubbub, struct dml2_display_arb_regs
 
 	bool wm_pending = false;
 	uint32_t temp;
+	bool allow_sdpif_rate_limit_when_cstate_req = arb_regs->allow_sdpif_rate_limit_when_cstate_req != 0;
 
 	/* request backpressure and outstanding return threshold (unused)*/
 	//REG_UPDATE(DCHUBBUB_TIMEOUT_DETECTION_CTRL1, DCHUBBUB_TIMEOUT_REQ_STALL_THRESHOLD, arb_regs->req_stall_threshold);
@@ -1205,8 +1208,8 @@ bool dcn401_program_arbiter(struct hubbub *hubbub, struct dml2_display_arb_regs
 	/* P-State stall threshold */
 	REG_UPDATE(DCHUBBUB_TIMEOUT_DETECTION_CTRL2, DCHUBBUB_TIMEOUT_PSTATE_STALL_THRESHOLD, arb_regs->pstate_stall_threshold);
 
-	if (safe_to_lower || arb_regs->allow_sdpif_rate_limit_when_cstate_req > hubbub2->allow_sdpif_rate_limit_when_cstate_req) {
-		hubbub2->allow_sdpif_rate_limit_when_cstate_req = arb_regs->allow_sdpif_rate_limit_when_cstate_req;
+	if (safe_to_lower || allow_sdpif_rate_limit_when_cstate_req > hubbub2->allow_sdpif_rate_limit_when_cstate_req) {
+		hubbub2->allow_sdpif_rate_limit_when_cstate_req = allow_sdpif_rate_limit_when_cstate_req;
 
 		/* only update the required bits */
 		REG_GET(DCHUBBUB_CTRL_STATUS, DCHUBBUB_HW_DEBUG, &temp);
diff --git a/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c b/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c
index 73b6b0ffcb741..ddfc60e845a49 100644
--- a/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/hubbub/dcn42/dcn42_hubbub.c
@@ -503,6 +503,7 @@ static bool dcn42_program_arbiter(struct hubbub *hubbub, struct dml2_display_arb
 
 	bool wm_pending = false;
 	uint32_t temp;
+	bool allow_sdpif_rate_limit_when_cstate_req = arb_regs->allow_sdpif_rate_limit_when_cstate_req != 0;
 
 	/* request backpressure and outstanding return threshold (unused)*/
 	//REG_UPDATE(DCHUBBUB_TIMEOUT_DETECTION_CTRL1, DCHUBBUB_TIMEOUT_REQ_STALL_THRESHOLD, arb_regs->req_stall_threshold);
@@ -510,8 +511,8 @@ static bool dcn42_program_arbiter(struct hubbub *hubbub, struct dml2_display_arb
 	/* 401 delta: do not update P-State stall threshold (handled by fw) */
 	// REG_UPDATE(DCHUBBUB_TIMEOUT_DETECTION_CTRL2, DCHUBBUB_TIMEOUT_PSTATE_STALL_THRESHOLD, arb_regs->pstate_stall_threshold);
 
-	if (safe_to_lower || arb_regs->allow_sdpif_rate_limit_when_cstate_req > hubbub2->allow_sdpif_rate_limit_when_cstate_req) {
-		hubbub2->allow_sdpif_rate_limit_when_cstate_req = arb_regs->allow_sdpif_rate_limit_when_cstate_req;
+	if (safe_to_lower || allow_sdpif_rate_limit_when_cstate_req > hubbub2->allow_sdpif_rate_limit_when_cstate_req) {
+		hubbub2->allow_sdpif_rate_limit_when_cstate_req = allow_sdpif_rate_limit_when_cstate_req;
 
 		/* only update the required bits */
 		REG_GET(DCHUBBUB_CTRL_STATUS, DCHUBBUB_HW_DEBUG, &temp);
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c
index ceee5165fd6ad..cf2c9f80056eb 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn20/dcn20_hubp.c
@@ -68,10 +68,10 @@ void hubp2_set_vm_system_aperture_settings(struct hubp *hubp,
 			DCN_VM_SYSTEM_APERTURE_DEFAULT_ADDR_LSB, mc_vm_apt_default.low_part);
 
 	REG_SET(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, 0,
-			MC_VM_SYSTEM_APERTURE_LOW_ADDR, mc_vm_apt_low.quad_part);
+			MC_VM_SYSTEM_APERTURE_LOW_ADDR, mc_vm_apt_low.low_part);
 
 	REG_SET(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, 0,
-			MC_VM_SYSTEM_APERTURE_HIGH_ADDR, mc_vm_apt_high.quad_part);
+			MC_VM_SYSTEM_APERTURE_HIGH_ADDR, mc_vm_apt_high.low_part);
 
 	REG_SET_2(DCN_VM_MX_L1_TLB_CNTL, 0,
 			ENABLE_L1_TLB, 1,
@@ -710,7 +710,7 @@ void hubp2_dmdata_load(
 		uint32_t dmdata_sw_size,
 		const uint32_t *dmdata_sw_data)
 {
-	int i;
+	unsigned int i;
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
 	/* load dmdata into HUBP buffer in SW mode */
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c
index 08ea0a1b9e7fe..67828505939a3 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn21/dcn21_hubp.c
@@ -238,10 +238,10 @@ static void hubp21_set_vm_system_aperture_settings(struct hubp *hubp,
 	mc_vm_apt_high.quad_part = apt->sys_high.quad_part >> 18;
 
 	REG_SET(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, 0,
-			MC_VM_SYSTEM_APERTURE_LOW_ADDR, mc_vm_apt_low.quad_part);
+			MC_VM_SYSTEM_APERTURE_LOW_ADDR, mc_vm_apt_low.low_part);
 
 	REG_SET(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, 0,
-			MC_VM_SYSTEM_APERTURE_HIGH_ADDR, mc_vm_apt_high.quad_part);
+			MC_VM_SYSTEM_APERTURE_HIGH_ADDR, mc_vm_apt_high.low_part);
 
 	REG_SET_2(DCN_VM_MX_L1_TLB_CNTL, 0,
 			ENABLE_L1_TLB, 1,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c
index e2708e30eb1ba..3e5ae0eae39f4 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn30/dcn30_hubp.c
@@ -55,10 +55,10 @@ void hubp3_set_vm_system_aperture_settings(struct hubp *hubp,
 	mc_vm_apt_high.quad_part = apt->sys_high.quad_part >> 18;
 
 	REG_SET(DCN_VM_SYSTEM_APERTURE_LOW_ADDR, 0,
-			MC_VM_SYSTEM_APERTURE_LOW_ADDR, mc_vm_apt_low.quad_part);
+			MC_VM_SYSTEM_APERTURE_LOW_ADDR, mc_vm_apt_low.low_part);
 
 	REG_SET(DCN_VM_SYSTEM_APERTURE_HIGH_ADDR, 0,
-			MC_VM_SYSTEM_APERTURE_HIGH_ADDR, mc_vm_apt_high.quad_part);
+			MC_VM_SYSTEM_APERTURE_HIGH_ADDR, mc_vm_apt_high.low_part);
 
 	REG_SET_2(DCN_VM_MX_L1_TLB_CNTL, 0,
 			ENABLE_L1_TLB, 1,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
index 5a816442deee9..3025151283580 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.c
@@ -41,12 +41,12 @@
 	hubp2->hubp_shift->field_name, hubp2->hubp_mask->field_name
 
 void hubp401_program_3dlut_fl_addr(struct hubp *hubp,
-	const struct dc_plane_address *address)
+	const struct dc_plane_address address)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	REG_UPDATE(HUBP_3DLUT_ADDRESS_HIGH, HUBP_3DLUT_ADDRESS_HIGH, address->lut3d.addr.high_part);
-	REG_WRITE(HUBP_3DLUT_ADDRESS_LOW, address->lut3d.addr.low_part);
+	REG_UPDATE(HUBP_3DLUT_ADDRESS_HIGH, HUBP_3DLUT_ADDRESS_HIGH, address.lut3d.addr.high_part);
+	REG_WRITE(HUBP_3DLUT_ADDRESS_LOW, address.lut3d.addr.low_part);
 }
 
 void hubp401_program_3dlut_fl_dlg_param(struct hubp *hubp, int refcyc_per_3dlut_group)
@@ -63,178 +63,105 @@ void hubp401_enable_3dlut_fl(struct hubp *hubp, bool enable)
 	REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_ENABLE, enable ? 1 : 0);
 }
 
-int hubp401_get_3dlut_fl_done(struct hubp *hubp)
+uint32_t hubp401_get_3dlut_fl_done(struct hubp *hubp)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
-	int ret;
+	uint32_t ret;
 
 	REG_GET(HUBP_3DLUT_CONTROL, HUBP_3DLUT_DONE, &ret);
 	return ret;
 }
 
-static void hubp401_get_3dlut_fl_xbar_map(
-		const enum dc_cm_lut_pixel_format format,
-		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_y_g,
-		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cb_b,
-		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cr_r)
+void hubp401_program_3dlut_fl_addressing_mode(struct hubp *hubp, enum hubp_3dlut_fl_addressing_mode addr_mode)
 {
-	switch (format) {
-	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB:
-	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB:
-	case CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10:
-		/* BGRA */
-		*bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47;
-		*bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
-		*bit_slice_cb_b =  hubp_3dlut_fl_crossbar_bit_slice_0_15;
-		break;
-	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB:
-	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB:
-	case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10:
-	default:
-		/* RGBA */
-		*bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
-		*bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
-		*bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
-		break;
-	}
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
+
+	REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_ADDRESSING_MODE, addr_mode);
 }
 
-void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp,
-		const enum dc_cm_lut_pixel_format format)
+void hubp401_program_3dlut_fl_width(struct hubp *hubp, enum hubp_3dlut_fl_width width)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r = 0;
+	REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_WIDTH, width);
+}
 
-	hubp401_get_3dlut_fl_xbar_map(format,
-			&bit_slice_y_g,
-			&bit_slice_cb_b,
-			&bit_slice_cr_r);
+void hubp401_program_3dlut_fl_tmz_protected(struct hubp *hubp, uint8_t protection_bits)
+{
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	REG_UPDATE_3(HUBP_3DLUT_CONTROL,
-			HUBP_3DLUT_CROSSBAR_SELECT_Y_G, bit_slice_y_g,
-			HUBP_3DLUT_CROSSBAR_SELECT_CB_B, bit_slice_cb_b,
-			HUBP_3DLUT_CROSSBAR_SELECT_CR_R, bit_slice_cr_r);
+	REG_UPDATE(HUBP_3DLUT_CONTROL, HUBP_3DLUT_TMZ, protection_bits);
 }
 
-static enum hubp_3dlut_fl_width hubp401_get_3dlut_fl_width(
-		const enum dc_cm_lut_size size,
-		const enum dc_cm_lut_swizzle swizzle)
+void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp,
+			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g,
+			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b,
+			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r)
 {
-	enum hubp_3dlut_fl_width width = 0;
-
-	switch (size) {
-	case CM_LUT_SIZE_333333:
-		ASSERT(swizzle != CM_LUT_1D_PACKED_LINEAR);
-		width = hubp_3dlut_fl_width_33;
-		break;
-	case CM_LUT_SIZE_171717:
-		if (swizzle != CM_LUT_1D_PACKED_LINEAR) {
-			width = hubp_3dlut_fl_width_17;
-		} else {
-			width = hubp_3dlut_fl_width_17_transformed;
-		}
-		break;
-	default:
-		width = 0;
-		break;
-	}
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	return width;
+	REG_UPDATE_3(HUBP_3DLUT_CONTROL,
+			HUBP_3DLUT_CROSSBAR_SELECT_Y_G, bit_slice_y_g,
+			HUBP_3DLUT_CROSSBAR_SELECT_CB_B, bit_slice_cb_b,
+			HUBP_3DLUT_CROSSBAR_SELECT_CR_R, bit_slice_cr_r);
 }
 
-static enum hubp_3dlut_fl_format hubp401_get_3dlut_fl_format(
-		const enum dc_cm_lut_pixel_format dc_format)
+void hubp401_update_3dlut_fl_bias_scale(struct hubp *hubp, uint16_t bias, uint16_t scale)
 {
-	enum hubp_3dlut_fl_format hubp_format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
-
-	switch (dc_format) {
-	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB:
-	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB:
-		hubp_format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
-		break;
-	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB:
-	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB:
-		hubp_format = hubp_3dlut_fl_format_unorm_12lsb_bitslice;
-		break;
-	case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10:
-	case CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10:
-		hubp_format = hubp_3dlut_fl_format_float_fp1_5_10;
-		break;
-	default:
-		BREAK_TO_DEBUGGER();
-		break;
-	}
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	return hubp_format;
+	REG_UPDATE_2(_3DLUT_FL_BIAS_SCALE, HUBP0_3DLUT_FL_BIAS, bias, HUBP0_3DLUT_FL_SCALE, scale);
 }
 
-static enum hubp_3dlut_fl_addressing_mode hubp401_get_3dlut_fl_addr_mode(
-		const enum dc_cm_lut_swizzle swizzle)
+void hubp401_program_3dlut_fl_mode(struct hubp *hubp, enum hubp_3dlut_fl_mode mode)
 {
-	enum hubp_3dlut_fl_addressing_mode addr_mode;
-
-	switch (swizzle) {
-	case CM_LUT_1D_PACKED_LINEAR:
-		addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
-		break;
-	case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
-	case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
-	default:
-		addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
-		break;
-	}
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	return addr_mode;
+	REG_UPDATE(_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_MODE, mode);
 }
 
-static enum hubp_3dlut_fl_mode hubp401_get_3dlut_fl_mode(
-		const enum dc_cm_lut_swizzle swizzle)
+void hubp401_program_3dlut_fl_format(struct hubp *hubp, enum hubp_3dlut_fl_format format)
 {
-	enum hubp_3dlut_fl_mode mode;
-
-	switch (swizzle) {
-	case CM_LUT_3D_SWIZZLE_LINEAR_RGB:
-		mode = hubp_3dlut_fl_mode_native_1;
-		break;
-	case CM_LUT_3D_SWIZZLE_LINEAR_BGR:
-		mode = hubp_3dlut_fl_mode_native_2;
-		break;
-	case CM_LUT_1D_PACKED_LINEAR:
-		mode = hubp_3dlut_fl_mode_transform;
-		break;
-	default:
-		mode = hubp_3dlut_fl_mode_disable;
-		break;
-	}
+	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	return mode;
+	REG_UPDATE(_3DLUT_FL_CONFIG, HUBP0_3DLUT_FL_FORMAT, format);
 }
 
-void hubp401_program_3dlut_fl_config(struct hubp *hubp,
-		const struct dc_3dlut_dma *config)
+void hubp401_program_3dlut_fl_config(
+	struct hubp *hubp,
+	struct hubp_fl_3dlut_config *cfg)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	enum hubp_3dlut_fl_width width = hubp401_get_3dlut_fl_width(config->size, config->swizzle);
-	enum hubp_3dlut_fl_format format = hubp401_get_3dlut_fl_format(config->format);
-	enum hubp_3dlut_fl_addressing_mode addr_mode = hubp401_get_3dlut_fl_addr_mode(config->swizzle);
-	enum hubp_3dlut_fl_mode mode = hubp401_get_3dlut_fl_mode(config->swizzle);
+	uint32_t mpc_width = {(cfg->width == 17) ? 0 : 1};
+	uint32_t width = {cfg->width};
+
+	if (cfg->layout == DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR)
+		width = (cfg->width == 17) ? 4916 : 35940;
 
 	REG_UPDATE_2(_3DLUT_FL_CONFIG,
-			HUBP0_3DLUT_FL_MODE, mode,
-			HUBP0_3DLUT_FL_FORMAT, format);
+		HUBP0_3DLUT_FL_MODE, cfg->mode,
+		HUBP0_3DLUT_FL_FORMAT, cfg->format);
 
 	REG_UPDATE_2(_3DLUT_FL_BIAS_SCALE,
-			HUBP0_3DLUT_FL_BIAS, config->bias,
-			HUBP0_3DLUT_FL_SCALE, config->scale);
-
-	REG_UPDATE_3(HUBP_3DLUT_CONTROL,
-			HUBP_3DLUT_WIDTH, width,
-			HUBP_3DLUT_ADDRESSING_MODE, addr_mode,
-			HUBP_3DLUT_TMZ, config->addr.tmz_surface);
+		HUBP0_3DLUT_FL_BIAS, cfg->bias,
+		HUBP0_3DLUT_FL_SCALE, cfg->scale);
+
+	REG_UPDATE(HUBP_3DLUT_ADDRESS_HIGH,
+		HUBP_3DLUT_ADDRESS_HIGH, cfg->address.lut3d.addr.high_part);
+	REG_UPDATE(HUBP_3DLUT_ADDRESS_LOW,
+		HUBP_3DLUT_ADDRESS_LOW, cfg->address.lut3d.addr.low_part);
+
+	//cross bar
+	REG_UPDATE_8(HUBP_3DLUT_CONTROL,
+		HUBP_3DLUT_MPC_WIDTH, mpc_width,
+		HUBP_3DLUT_WIDTH, width,
+		HUBP_3DLUT_CROSSBAR_SELECT_CR_R, cfg->crossbar_bit_slice_cr_r,
+		HUBP_3DLUT_CROSSBAR_SELECT_Y_G, cfg->crossbar_bit_slice_y_g,
+		HUBP_3DLUT_CROSSBAR_SELECT_CB_B, cfg->crossbar_bit_slice_cb_b,
+		HUBP_3DLUT_ADDRESSING_MODE, cfg->addr_mode,
+		HUBP_3DLUT_TMZ, cfg->protection_bits,
+		HUBP_3DLUT_ENABLE, cfg->enabled ? 1 : 0);
 }
 
 void hubp401_update_mall_sel(struct hubp *hubp, uint32_t mall_sel, bool c_cursor)
@@ -1135,13 +1062,19 @@ static struct hubp_funcs dcn401_hubp_funcs = {
 	.hubp_update_mall_sel = hubp401_update_mall_sel,
 	.hubp_prepare_subvp_buffering = hubp32_prepare_subvp_buffering,
 	.hubp_program_mcache_id_and_split_coordinate = hubp401_program_mcache_id_and_split_coordinate,
+	.hubp_update_3dlut_fl_bias_scale = hubp401_update_3dlut_fl_bias_scale,
+	.hubp_program_3dlut_fl_mode = hubp401_program_3dlut_fl_mode,
+	.hubp_program_3dlut_fl_format = hubp401_program_3dlut_fl_format,
 	.hubp_program_3dlut_fl_addr = hubp401_program_3dlut_fl_addr,
-	.hubp_program_3dlut_fl_config = hubp401_program_3dlut_fl_config,
 	.hubp_program_3dlut_fl_dlg_param = hubp401_program_3dlut_fl_dlg_param,
 	.hubp_enable_3dlut_fl = hubp401_enable_3dlut_fl,
+	.hubp_program_3dlut_fl_addressing_mode = hubp401_program_3dlut_fl_addressing_mode,
+	.hubp_program_3dlut_fl_width = hubp401_program_3dlut_fl_width,
+	.hubp_program_3dlut_fl_tmz_protected = hubp401_program_3dlut_fl_tmz_protected,
 	.hubp_program_3dlut_fl_crossbar = hubp401_program_3dlut_fl_crossbar,
 	.hubp_get_3dlut_fl_done = hubp401_get_3dlut_fl_done,
 	.hubp_clear_tiling = hubp401_clear_tiling,
+	.hubp_program_3dlut_fl_config = hubp401_program_3dlut_fl_config,
 	.hubp_read_reg_state = hubp3_read_reg_state
 };
 
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h
index 043948f64b862..4116834c552dd 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn401/dcn401_hubp.h
@@ -324,21 +324,36 @@ bool hubp401_construct(
 
 void hubp401_init(struct hubp *hubp);
 
-int hubp401_get_3dlut_fl_done(struct hubp *hubp);
+uint32_t hubp401_get_3dlut_fl_done(struct hubp *hubp);
 
 void hubp401_set_unbounded_requesting(struct hubp *hubp, bool enable);
 
+void hubp401_update_3dlut_fl_bias_scale(struct hubp *hubp, uint16_t bias, uint16_t scale);
+
 void hubp401_program_3dlut_fl_crossbar(struct hubp *hubp,
-		const enum dc_cm_lut_pixel_format format);
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g,
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b,
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r);
+
+void hubp401_program_3dlut_fl_tmz_protected(struct hubp *hubp, uint8_t protection_bits);
+
+void hubp401_program_3dlut_fl_width(struct hubp *hubp, enum hubp_3dlut_fl_width width);
+
+void hubp401_program_3dlut_fl_addressing_mode(struct hubp *hubp, enum hubp_3dlut_fl_addressing_mode addr_mode);
 
 void hubp401_enable_3dlut_fl(struct hubp *hubp, bool enable);
 
 void hubp401_program_3dlut_fl_dlg_param(struct hubp *hubp, int refcyc_per_3dlut_group);
 
-void hubp401_program_3dlut_fl_addr(struct hubp *hubp, const struct dc_plane_address *address);
+void hubp401_program_3dlut_fl_addr(struct hubp *hubp, const struct dc_plane_address address);
 
-void hubp401_program_3dlut_fl_config(struct hubp *hubp,
-		const struct dc_3dlut_dma *config);
+void hubp401_program_3dlut_fl_format(struct hubp *hubp, enum hubp_3dlut_fl_format format);
+
+void hubp401_program_3dlut_fl_mode(struct hubp *hubp, enum hubp_3dlut_fl_mode mode);
+
+void hubp401_program_3dlut_fl_config(
+	struct hubp *hubp,
+	struct hubp_fl_3dlut_config *cfg);
 
 void hubp401_clear_tiling(struct hubp *hubp);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
index ad6badcceb12a..e4602c3ddc662 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.c
@@ -311,84 +311,19 @@ static void hubp42_program_surface_config(
 	hubp42_program_pixel_format(hubp, format);
 }
 
-static void hubp42_get_3dlut_fl_xbar_map(
-		const enum dc_cm_lut_pixel_format format,
-		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_y_g,
-		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cb_b,
-		enum hubp_3dlut_fl_crossbar_bit_slice *bit_slice_cr_r)
-{
-	switch (format) {
-	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12MSB:
-	case CM_LUT_PIXEL_FORMAT_BGRA16161616_UNORM_12LSB:
-	case CM_LUT_PIXEL_FORMAT_BGRA16161616_FLOAT_FP1_5_10:
-		/* BGRA */
-		*bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47;
-		*bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
-		*bit_slice_cb_b =  hubp_3dlut_fl_crossbar_bit_slice_0_15;
-		break;
-	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12MSB:
-	case CM_LUT_PIXEL_FORMAT_RGBA16161616_UNORM_12LSB:
-	case CM_LUT_PIXEL_FORMAT_RGBA16161616_FLOAT_FP1_5_10:
-	default:
-		/* RGBA */
-		*bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
-		*bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
-		*bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
-		break;
-	}
-}
-
 void hubp42_program_3dlut_fl_crossbar(struct hubp *hubp,
-		const enum dc_cm_lut_pixel_format format)
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_r,
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_g,
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_b)
 {
 	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
 
-	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_g = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_b = 0;
-	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_r = 0;
-
-	hubp42_get_3dlut_fl_xbar_map(format,
-			&bit_slice_g,
-			&bit_slice_b,
-			&bit_slice_r);
-
 	REG_UPDATE_3(HUBP_3DLUT_CONTROL,
 		HUBP_3DLUT_CROSSBAR_SEL_R, bit_slice_r,
 		HUBP_3DLUT_CROSSBAR_SEL_G, bit_slice_g,
 		HUBP_3DLUT_CROSSBAR_SEL_B, bit_slice_b);
 }
 
-static uint32_t hubp42_get_3dlut_fl_mpc_width(
-		const enum dc_cm_lut_size size)
-{
-	uint32_t width = 0;
-
-	switch (size) {
-	case CM_LUT_SIZE_333333:
-		width = 1;
-		break;
-	case CM_LUT_SIZE_171717:
-	default:
-		width = 0;
-		break;
-	}
-
-	return width;
-}
-
-void hubp42_program_3dlut_fl_config(struct hubp *hubp,
-		const struct dc_3dlut_dma *config)
-{
-	struct dcn20_hubp *hubp2 = TO_DCN20_HUBP(hubp);
-
-	uint32_t mpc_width = hubp42_get_3dlut_fl_mpc_width(config->size);
-
-	REG_UPDATE(HUBP_3DLUT_CONTROL,
-			HUBP_3DLUT_MPC_WIDTH, mpc_width);
-
-	hubp401_program_3dlut_fl_config(hubp, config);
-}
-
 static bool hubp42_program_surface_flip_and_addr(
 	struct hubp *hubp,
 	const struct dc_plane_address *address,
@@ -670,10 +605,15 @@ struct hubp_funcs dcn42_hubp_funcs = {
 	.hubp_set_flip_int = hubp1_set_flip_int,
 	.hubp_in_blank = hubp1_in_blank,
 	.program_extended_blank = hubp31_program_extended_blank_value,
+	.hubp_update_3dlut_fl_bias_scale = hubp401_update_3dlut_fl_bias_scale,
+	.hubp_program_3dlut_fl_mode = hubp401_program_3dlut_fl_mode,
+	.hubp_program_3dlut_fl_format = hubp401_program_3dlut_fl_format,
 	.hubp_program_3dlut_fl_addr = hubp401_program_3dlut_fl_addr,
-	.hubp_program_3dlut_fl_config = hubp42_program_3dlut_fl_config,
 	.hubp_program_3dlut_fl_dlg_param = hubp401_program_3dlut_fl_dlg_param,
 	.hubp_enable_3dlut_fl = hubp401_enable_3dlut_fl,
+	.hubp_program_3dlut_fl_addressing_mode = hubp401_program_3dlut_fl_addressing_mode,
+	.hubp_program_3dlut_fl_width = hubp401_program_3dlut_fl_width,
+	.hubp_program_3dlut_fl_tmz_protected = hubp401_program_3dlut_fl_tmz_protected,
 	.hubp_program_3dlut_fl_crossbar = hubp42_program_3dlut_fl_crossbar,
 	.hubp_get_3dlut_fl_done = hubp401_get_3dlut_fl_done,
 	.hubp_clear_tiling = hubp3_clear_tiling,
diff --git a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h
index 88bb1337ab9d6..f0e614136228c 100644
--- a/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/hubp/dcn42/dcn42_hubp.h
@@ -58,11 +58,11 @@ bool hubp42_construct(
 	const struct dcn_hubp2_shift *hubp_shift,
 	const struct dcn_hubp2_mask *hubp_mask);
 
-void hubp42_program_3dlut_fl_crossbar(struct hubp *hubp,
-		const enum dc_cm_lut_pixel_format format);
-
-void hubp42_program_3dlut_fl_config(struct hubp *hubp,
-		const struct dc_3dlut_dma *config);
+void hubp42_program_3dlut_fl_crossbar(
+	struct hubp *hubp,
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_r,
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_g,
+	enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_b);
 
 void hubp42_read_state(struct hubp *hubp);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
index f0abbb7c2cb24..d81271b870bb2 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c
@@ -504,7 +504,7 @@ dce110_translate_regamma_to_hw_format(const struct dc_transfer_func *output_tf,
 	}
 
 	j = 0;
-	for (k = 0; k < (region_end - region_start); k++) {
+	for (k = 0; k < (uint32_t)(region_end - region_start); k++) {
 		increment = NUMBER_SW_SEGMENTS / (1 << seg_distr[k]);
 		start_index = (region_start + k + MAX_LOW_POINT) *
 				NUMBER_SW_SEGMENTS;
@@ -887,7 +887,7 @@ void dce110_edp_power_control(
 				DC_LOG_HW_RESUME_S3(
 						"%s: remaining_min_edp_poweroff_time_ms=%llu: begin wait.\n",
 						__func__, remaining_min_edp_poweroff_time_ms);
-				msleep(remaining_min_edp_poweroff_time_ms);
+				msleep((unsigned int)remaining_min_edp_poweroff_time_ms);
 				DC_LOG_HW_RESUME_S3(
 						"%s: remaining_min_edp_poweroff_time_ms=%llu: end wait.\n",
 						__func__, remaining_min_edp_poweroff_time_ms);
@@ -912,7 +912,7 @@ void dce110_edp_power_control(
 		cntl.coherent = false;
 		cntl.lanes_number = LANE_COUNT_FOUR;
 		cntl.hpd_sel = link->link_enc->hpd_source;
-		pwrseq_instance = link->panel_cntl->pwrseq_inst;
+		pwrseq_instance = (uint8_t)link->panel_cntl->pwrseq_inst;
 
 		if (ctx->dc->ctx->dmub_srv &&
 				ctx->dc->debug.dmub_command_table) {
@@ -981,7 +981,7 @@ void dce110_edp_wait_for_T12(
 		t12_duration += link->panel_config.pps.extra_t12_ms; // Add extra T12
 
 		if (time_since_edp_poweroff_ms < t12_duration)
-			msleep(t12_duration - time_since_edp_poweroff_ms);
+			msleep((unsigned int)(t12_duration - time_since_edp_poweroff_ms));
 	}
 }
 /*todo: cloned in stream enc, fix*/
@@ -1050,8 +1050,9 @@ void dce110_edp_backlight_control(
 	 */
 	/* dc_service_sleep_in_milliseconds(50); */
 		/*edp 1.2*/
-	if (link->panel_cntl)
-		pwrseq_instance = link->panel_cntl->pwrseq_inst;
+	if (link->panel_cntl) {
+		pwrseq_instance = (uint8_t)link->panel_cntl->pwrseq_inst;
+	}
 
 	if (cntl.action == TRANSMITTER_CONTROL_BACKLIGHT_ON) {
 		if (!link->dc->config.edp_no_power_sequencing)
@@ -1468,11 +1469,11 @@ void build_audio_output(
 	audio_output->crtc_info.pixel_repetition = 1;
 
 	audio_output->crtc_info.interlaced =
-			stream->timing.flags.INTERLACE;
+			(stream->timing.flags.INTERLACE != 0);
 
 	audio_output->crtc_info.refresh_rate =
-		(stream->timing.pix_clk_100hz*100)/
-		(stream->timing.h_total*stream->timing.v_total);
+		(uint16_t)((stream->timing.pix_clk_100hz*100)/
+		(stream->timing.h_total*stream->timing.v_total));
 
 	audio_output->crtc_info.color_depth =
 		stream->timing.display_color_depth;
@@ -1786,20 +1787,22 @@ static void power_down_encoders(struct dc *dc)
 
 	for (i = 0; i < dc->link_count; i++) {
 		struct dc_link *link = dc->links[i];
-		struct link_encoder *link_enc = link->link_enc;
+		struct link_encoder *link_enc = link_enc_cfg_get_link_enc(link);
 		enum signal_type signal = link->connector_signal;
 
 		dc->link_srv->blank_dp_stream(link, false);
 		if (signal != SIGNAL_TYPE_EDP)
 			signal = SIGNAL_TYPE_NONE;
 
-		if (link->ep_type == DISPLAY_ENDPOINT_PHY)
+		if (link->ep_type == DISPLAY_ENDPOINT_PHY && link_enc)
 			link_enc->funcs->disable_output(link_enc, signal);
 
 		if (link->fec_state == dc_link_fec_enabled) {
-			link_enc->funcs->fec_set_enable(link_enc, false);
-			link_enc->funcs->fec_set_ready(link_enc, false);
-			link->fec_state = dc_link_fec_not_ready;
+			if (link_enc && link_enc->funcs->fec_set_enable && link_enc->funcs->fec_set_ready) {
+				link_enc->funcs->fec_set_enable(link_enc, false);
+				link_enc->funcs->fec_set_ready(link_enc, false);
+				link->fec_state = dc_link_fec_not_ready;
+			}
 		}
 
 		link->link_status.link_active = false;
@@ -1809,7 +1812,7 @@ static void power_down_encoders(struct dc *dc)
 
 static void power_down_controllers(struct dc *dc)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
 		dc->res_pool->timing_generators[i]->funcs->disable_crtc(
@@ -1819,7 +1822,7 @@ static void power_down_controllers(struct dc *dc)
 
 static void power_down_clock_sources(struct dc *dc)
 {
-	int i;
+	unsigned int i;
 
 	if (dc->res_pool->dp_clock_source->funcs->cs_power_down(
 		dc->res_pool->dp_clock_source) == false)
@@ -1828,7 +1831,7 @@ static void power_down_clock_sources(struct dc *dc)
 	for (i = 0; i < dc->res_pool->clk_src_count; i++) {
 		if (dc->res_pool->clock_sources[i]->funcs->cs_power_down(
 				dc->res_pool->clock_sources[i]) == false)
-			dm_error("Failed to power down pll! (clk src index=%d)\n", i);
+			dm_error("Failed to power down pll! (clk src index=%u)\n", i);
 	}
 }
 
@@ -1847,7 +1850,7 @@ static void power_down_all_hw_blocks(struct dc *dc)
 static void disable_vga_and_power_gate_all_controllers(
 		struct dc *dc)
 {
-	int i;
+	uint8_t i;
 	struct timing_generator *tg;
 	struct dc_context *ctx = dc->ctx;
 
@@ -1877,7 +1880,7 @@ static void get_edp_streams(struct dc_state *context,
 		struct dc_stream_state **edp_streams,
 		int *edp_stream_num)
 {
-	int i;
+	uint8_t i;
 
 	*edp_stream_num = 0;
 	for (i = 0; i < context->stream_count; i++) {
@@ -1915,13 +1918,13 @@ static void clean_up_dsc_blocks(struct dc *dc)
 	struct stream_encoder *se = NULL;
 	struct dccg *dccg = dc->res_pool->dccg;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
-	int i;
+	unsigned int i;
 
 	if (!dc->caps.is_apu ||
 		dc->ctx->dce_version < DCN_VERSION_3_15)
 		return;
 	/*VBIOS supports dsc starts from dcn315*/
-	for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)dc->res_pool->res_cap->num_dsc; i++) {
 		struct dcn_dsc_state s  = {0};
 
 		dsc = dc->res_pool->dscs[i];
@@ -1999,7 +2002,7 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
 	struct pipe_ctx *pipe_ctx = NULL;
 	struct dce_hwseq *hws = dc->hwseq;
 	int edp_with_sink_num;
-	unsigned int edp_num;
+	unsigned int j, edp_num;
 	int edp_stream_num;
 	int i;
 	bool can_apply_edp_fast_boot = false;
@@ -2019,8 +2022,8 @@ void dce110_enable_accelerated_mode(struct dc *dc, struct dc_state *context)
 
 	/* Check fastboot support, disable on DCE 6-8-10 because of blank screens */
 	if (edp_num && edp_stream_num && dc->ctx->dce_version > DCE_VERSION_10_0) {
-		for (i = 0; i < edp_num; i++) {
-			edp_link = edp_links[i];
+		for (j = 0; j < edp_num; j++) {
+			edp_link = edp_links[j];
 			if (edp_link != edp_streams[0]->link)
 				continue;
 			// enable fastboot if backend is enabled on eDP
@@ -2123,9 +2126,11 @@ static uint32_t compute_pstate_blackout_duration(
 	const struct dc_stream_state *stream)
 {
 	uint32_t total_dest_line_time_ns;
+	int64_t pstate_blackout_duration_ns64;
 	uint32_t pstate_blackout_duration_ns;
 
-	pstate_blackout_duration_ns = 1000 * blackout_duration.value >> 24;
+	pstate_blackout_duration_ns64 = (1000 * blackout_duration.value) >> 24;
+	pstate_blackout_duration_ns = (uint32_t)pstate_blackout_duration_ns64;
 
 	total_dest_line_time_ns = 1000000UL *
 		(stream->timing.h_total * 10) /
@@ -2259,7 +2264,7 @@ static void get_position(struct pipe_ctx **pipe_ctx,
 static void set_static_screen_control(struct pipe_ctx **pipe_ctx,
 		int num_pipes, const struct dc_static_screen_params *params)
 {
-	unsigned int i;
+	int i;
 	unsigned int triggers = 0;
 
 	if (params->triggers.overlay_update)
@@ -2546,7 +2551,7 @@ enum dc_status dce110_apply_ctx_to_hw(
 	struct dce_hwseq *hws = dc->hwseq;
 	struct dc_bios *dcb = dc->ctx->dc_bios;
 	enum dc_status status;
-	int i;
+	unsigned int i;
 	bool was_hpo_acquired = resource_is_hpo_acquired(dc->current_state);
 	bool is_hpo_acquired = resource_is_hpo_acquired(context);
 
@@ -2582,7 +2587,7 @@ enum dc_status dce110_apply_ctx_to_hw(
 		}
 
 		hws->funcs.enable_display_power_gating(
-				dc, i, dc->ctx->dc_bios,
+				dc, (uint8_t)i, dc->ctx->dc_bios,
 				PIPE_GATING_CONTROL_DISABLE);
 	}
 
@@ -2912,7 +2917,7 @@ static void dce110_init_pipes(struct dc *dc, struct dc_state *context)
 
 static void dce110_init_hw(struct dc *dc)
 {
-	int i;
+	unsigned int i;
 	struct dc_bios *bp;
 	struct transform *xfm;
 	struct abm *abm;
@@ -2927,10 +2932,10 @@ static void dce110_init_hw(struct dc *dc)
 		xfm->funcs->transform_reset(xfm);
 
 		hws->funcs.enable_display_power_gating(
-				dc, i, bp,
+				dc, (uint8_t)i, bp,
 				PIPE_GATING_CONTROL_INIT);
 		hws->funcs.enable_display_power_gating(
-				dc, i, bp,
+				dc, (uint8_t)i, bp,
 				PIPE_GATING_CONTROL_DISABLE);
 		hws->funcs.enable_display_pipe_clock_gating(
 			dc->ctx,
@@ -3134,7 +3139,7 @@ static void dce110_apply_ctx_for_surface(
 		int num_planes,
 		struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 
 	if (num_planes == 0)
 		return;
@@ -3188,7 +3193,7 @@ static void dce110_power_down_fe(struct dc *dc, struct dc_state *state, struct p
 		return;
 
 	hws->funcs.enable_display_power_gating(
-		dc, fe_idx, dc->ctx->dc_bios, PIPE_GATING_CONTROL_ENABLE);
+		dc, (uint8_t)fe_idx, dc->ctx->dc_bios, PIPE_GATING_CONTROL_ENABLE);
 
 	dc->res_pool->transforms[fe_idx]->funcs->transform_reset(
 				dc->res_pool->transforms[fe_idx]);
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce120/dce120_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce120/dce120_hwseq.c
index 0689bbf12ad8c..fbe34d1bb39ab 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dce120/dce120_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dce120/dce120_hwseq.c
@@ -208,24 +208,24 @@ static void dce120_update_dchub(
 				FB_BASE, 0x0FFFF);
 
 		REG_UPDATE(DCHUB_AGP_BASE,
-				AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
+				AGP_BASE, (uint32_t)(dh_data->zfb_phys_addr_base >> 22));
 
 		REG_UPDATE(DCHUB_AGP_BOT,
-				AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
+				AGP_BOT, (uint32_t)(dh_data->zfb_mc_base_addr >> 22));
 
 		REG_UPDATE(DCHUB_AGP_TOP,
-				AGP_TOP, (dh_data->zfb_mc_base_addr + dh_data->zfb_size_in_byte - 1) >> 22);
+				AGP_TOP, (uint32_t)((dh_data->zfb_mc_base_addr + dh_data->zfb_size_in_byte - 1) >> 22));
 		break;
 	case FRAME_BUFFER_MODE_MIXED_ZFB_AND_LOCAL:
 		/*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
 		REG_UPDATE(DCHUB_AGP_BASE,
-				AGP_BASE, dh_data->zfb_phys_addr_base >> 22);
+				AGP_BASE, (uint32_t)(dh_data->zfb_phys_addr_base >> 22));
 
 		REG_UPDATE(DCHUB_AGP_BOT,
-				AGP_BOT, dh_data->zfb_mc_base_addr >> 22);
+				AGP_BOT, (uint32_t)(dh_data->zfb_mc_base_addr >> 22));
 
 		REG_UPDATE(DCHUB_AGP_TOP,
-				AGP_TOP, (dh_data->zfb_mc_base_addr + dh_data->zfb_size_in_byte - 1) >> 22);
+				AGP_TOP, (uint32_t)((dh_data->zfb_mc_base_addr + dh_data->zfb_size_in_byte - 1) >> 22));
 		break;
 	case FRAME_BUFFER_MODE_LOCAL_ONLY:
 		/*Should not touch FB LOCATION (done by VBIOS on AsicInit table)*/
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
index 756ce83795381..1733f551248e4 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_hwseq.c
@@ -181,6 +181,7 @@ void dcn10_set_wait_for_update_needed_for_pipe(struct dc *dc, struct pipe_ctx *p
 	uint32_t vupdate_start, vupdate_end;
 	struct crtc_position position;
 	unsigned int vpos, cur_frame;
+	uint32_t max_frame_count;
 
 	if (!pipe_ctx->stream ||
 		!pipe_ctx->stream_res.tg ||
@@ -197,7 +198,8 @@ void dcn10_set_wait_for_update_needed_for_pipe(struct dc *dc, struct pipe_ctx *p
 
 	struct optc *optc1 = DCN10TG_FROM_TG(tg);
 
-	ASSERT(optc1->max_frame_count != 0);
+	max_frame_count = optc1->tg_mask->OTG_FRAME_COUNT >> optc1->tg_shift->OTG_FRAME_COUNT;
+	ASSERT(max_frame_count != 0);
 
 	if (tg->funcs->is_tg_enabled && !tg->funcs->is_tg_enabled(tg))
 		return;
@@ -209,8 +211,8 @@ void dcn10_set_wait_for_update_needed_for_pipe(struct dc *dc, struct pipe_ctx *p
 	if (vpos < vupdate_start) {
 		pipe_ctx->wait_frame_count = cur_frame;
 	} else {
-		if (cur_frame + 1 > optc1->max_frame_count)
-			pipe_ctx->wait_frame_count = cur_frame + 1 - optc1->max_frame_count;
+		if (cur_frame + 1 > max_frame_count)
+			pipe_ctx->wait_frame_count = cur_frame + 1 - max_frame_count;
 		else
 			pipe_ctx->wait_frame_count = cur_frame + 1;
 	}
@@ -225,7 +227,7 @@ void dcn10_lock_all_pipes(struct dc *dc,
 	struct pipe_ctx *pipe_ctx;
 	struct pipe_ctx *old_pipe_ctx;
 	struct timing_generator *tg;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		old_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[i];
@@ -298,7 +300,7 @@ static void dcn10_log_hubp_states(struct dc *dc, void *log_ctx)
 {
 	struct dc_context *dc_ctx = dc->ctx;
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	DTN_INFO(
 		"HUBP:  format  addr_hi  width  height  rot  mir  sw_mode  dcc_en  blank_en  clock_en  ttu_dis  underflow   min_ttu_vblank       qos_low_wm      qos_high_wm\n");
@@ -456,7 +458,7 @@ static void dcn10_log_color_state(struct dc *dc,
 	struct dc_context *dc_ctx = dc->ctx;
 	struct resource_pool *pool = dc->res_pool;
 	bool is_gamut_remap_available = false;
-	int i;
+	unsigned int i;
 
 	DTN_INFO("DPP:    IGAM format    IGAM mode    DGAM mode    RGAM mode"
 		 "  GAMUT adjust  "
@@ -616,7 +618,8 @@ void dcn10_log_hw_state(struct dc *dc,
 {
 	struct dc_context *dc_ctx = dc->ctx;
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	int j;
+	unsigned int i;
 
 	DTN_INFO_BEGIN();
 
@@ -686,8 +689,8 @@ void dcn10_log_hw_state(struct dc *dc,
 	// dcn_dsc_state struct field bytes_per_pixel was renamed to bits_per_pixel
 	// TODO: Update golden log header to reflect this name change
 	DTN_INFO("DSC: CLOCK_EN  SLICE_WIDTH  Bytes_pp\n");
-	for (i = 0; i < pool->res_cap->num_dsc; i++) {
-		struct display_stream_compressor *dsc = pool->dscs[i];
+	for (j = 0; j < pool->res_cap->num_dsc; j++) {
+		struct display_stream_compressor *dsc = pool->dscs[j];
 		struct dcn_dsc_state s = {0};
 
 		dsc->funcs->dsc_read_state(dsc, &s);
@@ -1059,7 +1062,7 @@ static void apply_DEGVIDCN10_253_wa(struct dc *dc)
 {
 	struct dce_hwseq *hws = dc->hwseq;
 	struct hubp *hubp = dc->res_pool->hubps[0];
-	int i;
+	unsigned int i;
 
 	if (dc->debug.disable_stutter)
 		return;
@@ -1089,7 +1092,7 @@ void dcn10_bios_golden_init(struct dc *dc)
 {
 	struct dce_hwseq *hws = dc->hwseq;
 	struct dc_bios *bp = dc->ctx->dc_bios;
-	int i;
+	unsigned int i;
 	bool allow_self_fresh_force_enable = true;
 
 	if (hws->funcs.s0i3_golden_init_wa && hws->funcs.s0i3_golden_init_wa(dc))
@@ -1130,7 +1133,7 @@ static void false_optc_underflow_wa(
 		const struct dc_stream_state *stream,
 		struct timing_generator *tg)
 {
-	int i;
+	unsigned int i;
 	bool underflow;
 
 	if (!dc->hwseq->wa.false_optc_underflow)
@@ -1157,7 +1160,7 @@ static void false_optc_underflow_wa(
 static int calculate_vready_offset_for_group(struct pipe_ctx *pipe)
 {
 	struct pipe_ctx *other_pipe;
-	int vready_offset = pipe->pipe_dlg_param.vready_offset;
+	unsigned int vready_offset = pipe->pipe_dlg_param.vready_offset;
 
 	/* Always use the largest vready_offset of all connected pipes */
 	for (other_pipe = pipe->bottom_pipe; other_pipe != NULL; other_pipe = other_pipe->bottom_pipe) {
@@ -1290,7 +1293,7 @@ static void dcn10_reset_back_end_for_pipe(
 		struct dc_state *context)
 {
 	(void)context;
-	int i;
+	unsigned int i;
 	struct dc_link *link;
 	DC_LOGGER_INIT(dc->ctx);
 	if (pipe_ctx->stream_res.stream_enc == NULL) {
@@ -1574,7 +1577,7 @@ void dcn10_disable_plane(struct dc *dc, struct dc_state *state, struct pipe_ctx
 
 void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	uint8_t i;
 	struct dce_hwseq *hws = dc->hwseq;
 	struct hubbub *hubbub = dc->res_pool->hubbub;
 	bool can_apply_seamless_boot = false;
@@ -1677,7 +1680,7 @@ void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
 
 		pipe_ctx->plane_res.hubp = hubp;
 		pipe_ctx->plane_res.dpp = dpp;
-		pipe_ctx->plane_res.mpcc_inst = dpp->inst;
+		pipe_ctx->plane_res.mpcc_inst = (uint8_t)dpp->inst;
 		hubp->mpcc_id = dpp->inst;
 		hubp->opp_id = OPP_ID_INVALID;
 		hubp->power_gated = false;
@@ -1764,7 +1767,7 @@ void dcn10_init_pipes(struct dc *dc, struct dc_state *context)
 
 void dcn10_init_hw(struct dc *dc)
 {
-	int i;
+	unsigned int i;
 	struct abm *abm = dc->res_pool->abm;
 	struct dmcu *dmcu = dc->res_pool->dmcu;
 	struct dce_hwseq *hws = dc->hwseq;
@@ -2095,22 +2098,22 @@ static void log_tf(struct dc_context *ctx,
 	// DC_LOG_GAMMA is default logging of all hw points
 	// DC_LOG_ALL_GAMMA logs all points, not only hw points
 	// DC_LOG_ALL_TF_POINTS logs all channels of the tf
-	int i = 0;
+	unsigned int i = 0;
 
 	DC_LOG_GAMMA("Gamma Correction TF");
 	DC_LOG_ALL_GAMMA("Logging all tf points...");
 	DC_LOG_ALL_TF_CHANNELS("Logging all channels...");
 
 	for (i = 0; i < hw_points_num; i++) {
-		DC_LOG_GAMMA("R\t%d\t%llu", i, tf->tf_pts.red[i].value);
-		DC_LOG_ALL_TF_CHANNELS("G\t%d\t%llu", i, tf->tf_pts.green[i].value);
-		DC_LOG_ALL_TF_CHANNELS("B\t%d\t%llu", i, tf->tf_pts.blue[i].value);
+		DC_LOG_GAMMA("R\t%u\t%llu", i, tf->tf_pts.red[i].value);
+		DC_LOG_ALL_TF_CHANNELS("G\t%u\t%llu", i, tf->tf_pts.green[i].value);
+		DC_LOG_ALL_TF_CHANNELS("B\t%u\t%llu", i, tf->tf_pts.blue[i].value);
 	}
 
 	for (i = hw_points_num; i < MAX_NUM_HW_POINTS; i++) {
-		DC_LOG_ALL_GAMMA("R\t%d\t%llu", i, tf->tf_pts.red[i].value);
-		DC_LOG_ALL_TF_CHANNELS("G\t%d\t%llu", i, tf->tf_pts.green[i].value);
-		DC_LOG_ALL_TF_CHANNELS("B\t%d\t%llu", i, tf->tf_pts.blue[i].value);
+		DC_LOG_ALL_GAMMA("R\t%u\t%llu", i, tf->tf_pts.red[i].value);
+		DC_LOG_ALL_TF_CHANNELS("G\t%u\t%llu", i, tf->tf_pts.green[i].value);
+		DC_LOG_ALL_TF_CHANNELS("B\t%u\t%llu", i, tf->tf_pts.blue[i].value);
 	}
 }
 
@@ -2258,7 +2261,7 @@ void dcn10_cursor_lock(struct dc *dc, struct pipe_ctx *pipe, bool lock)
 		struct dmub_hw_lock_inst_flags inst_flags = { 0 };
 
 		hw_locks.bits.lock_cursor = 1;
-		inst_flags.opp_inst = pipe->stream_res.opp->inst;
+		inst_flags.opp_inst = (uint8_t)pipe->stream_res.opp->inst;
 
 		dmub_hw_lock_mgr_cmd(dc->ctx->dmub_srv,
 					lock,
@@ -2278,7 +2281,7 @@ static bool wait_for_reset_trigger_to_occur(
 	/* To avoid endless loop we wait at most
 	 * frames_to_wait_on_triggered_reset frames for the reset to occur. */
 	const uint32_t frames_to_wait_on_triggered_reset = 10;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < frames_to_wait_on_triggered_reset; i++) {
 
@@ -2290,7 +2293,7 @@ static bool wait_for_reset_trigger_to_occur(
 		if (tg->funcs->did_triggered_reset_occur(tg)) {
 			rc = true;
 			/* usually occurs at i=1 */
-			DC_SYNC_INFO("GSL: reset occurred at wait count: %d\n",
+			DC_SYNC_INFO("GSL: reset occurred at wait count: %u\n",
 					i);
 			break;
 		}
@@ -2383,7 +2386,7 @@ static uint8_t get_clock_divider(struct pipe_ctx *pipe,
 	}
 	clock_divider *= numpipes;
 
-	return clock_divider;
+	return (uint8_t)clock_divider;
 }
 
 static int dcn10_align_pixel_clocks(struct dc *dc, int group_size,
@@ -2458,7 +2461,7 @@ static int dcn10_align_pixel_clocks(struct dc *dc, int group_size,
 				dc->res_pool->dp_clock_source->funcs->override_dp_pix_clk(
 					dc->res_pool->dp_clock_source,
 					grouped_pipes[i]->stream_res.tg->inst,
-					phase[i], modulo[i]);
+					(unsigned int)phase[i], (unsigned int)modulo[i]);
 				dc->res_pool->dp_clock_source->funcs->get_pixel_clk_frequency_100hz(
 					dc->res_pool->dp_clock_source,
 					grouped_pipes[i]->stream_res.tg->inst, &pclk);
@@ -2484,7 +2487,8 @@ void dcn10_enable_vblanks_synchronization(
 	(void)group_index;
 	struct output_pixel_processor *opp;
 	struct timing_generator *tg;
-	int i, width = 0, height = 0, master;
+	int i, master;
+	uint32_t width = 0, height = 0;
 
 	DC_LOGGER_INIT(dc->ctx);
 
@@ -2550,7 +2554,8 @@ void dcn10_enable_timing_synchronization(
 	(void)group_index;
 	struct output_pixel_processor *opp;
 	struct timing_generator *tg;
-	int i, width = 0, height = 0;
+	int i;
+	uint32_t width = 0, height = 0;
 
 	DC_LOGGER_INIT(dc->ctx);
 
@@ -3263,7 +3268,7 @@ void dcn10_wait_for_pending_cleared(struct dc *dc,
 {
 		struct pipe_ctx *pipe_ctx;
 		struct timing_generator *tg;
-		int i;
+		unsigned int i;
 
 		for (i = 0; i < dc->res_pool->pipe_count; i++) {
 			pipe_ctx = &context->res_ctx.pipe_ctx[i];
@@ -3294,7 +3299,7 @@ void dcn10_post_unlock_program_front_end(
 		struct dc *dc,
 		struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
@@ -3466,7 +3471,7 @@ void dcn10_get_position(struct pipe_ctx **pipe_ctx,
 void dcn10_set_static_screen_control(struct pipe_ctx **pipe_ctx,
 		int num_pipes, const struct dc_static_screen_params *params)
 {
-	unsigned int i;
+	int i;
 	unsigned int triggers = 0;
 
 	if (params->triggers.surface_update)
@@ -3516,7 +3521,7 @@ void dcn10_config_stereo_parameters(
 			}
 		}
 		flags->RIGHT_EYE_POLARITY =\
-				stream->timing.flags.RIGHT_EYE_3D_POLARITY;
+				(stream->timing.flags.RIGHT_EYE_3D_POLARITY != 0);
 		if (timing_3d_format == TIMING_3D_FORMAT_HW_FRAME_PACKING)
 			flags->FRAME_PACKED = 1;
 	}
@@ -3553,7 +3558,7 @@ void dcn10_setup_stereo(struct pipe_ctx *pipe_ctx, struct dc *dc)
 
 static struct hubp *get_hubp_by_inst(struct resource_pool *res_pool, int mpcc_inst)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < res_pool->pipe_count; i++) {
 		if (res_pool->hubps[i]->inst == mpcc_inst)
@@ -3635,8 +3640,8 @@ void dcn10_update_pending_status(struct pipe_ctx *pipe_ctx)
 
 	if (dc->hwseq->wa_state.disallow_self_refresh_during_multi_plane_transition_applied) {
 		struct dce_hwseq *hwseq = dc->hwseq;
-		struct timing_generator *tg = dc->res_pool->timing_generators[0];
-		unsigned int cur_frame = tg->funcs->get_frame_count(tg);
+		struct timing_generator *wa_tg = dc->res_pool->timing_generators[0];
+		unsigned int cur_frame = wa_tg->funcs->get_frame_count(wa_tg);
 
 		if (cur_frame != hwseq->wa_state.disallow_self_refresh_during_multi_plane_transition_applied_on_frame) {
 			struct hubbub *hubbub = dc->res_pool->hubbub;
@@ -3794,7 +3799,7 @@ void dcn10_set_cursor_position(struct pipe_ctx *pipe_ctx)
 	// Swap axis and mirror vertically
 	else if (param.rotation == ROTATION_ANGLE_270) {
 		uint32_t temp_y = pos_cpy.y;
-		int viewport_height =
+		uint32_t viewport_height =
 			pipe_ctx->plane_res.scl_data.viewport.height;
 		int viewport_y =
 			pipe_ctx->plane_res.scl_data.viewport.y;
@@ -4121,7 +4126,7 @@ void dcn10_get_clock(struct dc *dc,
 void dcn10_get_dcc_en_bits(struct dc *dc, int *dcc_en_bits)
 {
 	struct resource_pool *pool = dc->res_pool;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < pool->pipe_count; i++) {
 		struct hubp *hubp = pool->hubps[i];
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c
index 079c226c1097f..b5e82e1901247 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn10/dcn10_init.c
@@ -118,6 +118,8 @@ static const struct hwseq_private_funcs dcn10_private_funcs = {
 	.dsc_pg_control = NULL,
 	.set_hdr_multiplier = dcn10_set_hdr_multiplier,
 	.verify_allow_pstate_change_high = dcn10_verify_allow_pstate_change_high,
+	.wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed,
+	.set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe,
 };
 
 void dcn10_hw_sequencer_construct(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
index 288e4edaa9a2b..07c53a8e73b53 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c
@@ -79,7 +79,7 @@ void dcn20_log_color_state(struct dc *dc,
 	struct dc_context *dc_ctx = dc->ctx;
 	struct resource_pool *pool = dc->res_pool;
 	bool is_gamut_remap_available = false;
-	int i;
+	unsigned int i;
 
 	DTN_INFO("DPP:  DGAM mode  SHAPER mode  3DLUT mode  3DLUT bit depth"
 		 "  3DLUT size  RGAM mode  GAMUT adjust  "
@@ -235,7 +235,7 @@ void dcn20_setup_gsl_group_as_lock(
 
 		group_idx = find_free_gsl_group(dc);
 		ASSERT(group_idx != 0);
-		pipe_ctx->stream_res.gsl_group = group_idx;
+		pipe_ctx->stream_res.gsl_group = (uint8_t)group_idx;
 
 		/* set gsl group reg field and mark resource used */
 		switch (group_idx) {
@@ -400,6 +400,7 @@ void dcn20_init_blank(
 	struct output_pixel_processor *bottom_opp = NULL;
 	uint32_t num_opps, opp_id_src0, opp_id_src1;
 	uint32_t otg_active_width = 0, otg_active_height = 0;
+	unsigned int num_opp_cap = (unsigned int)dc->res_pool->res_cap->num_opp;
 
 	/* program opp dpg blank color */
 	color_space = COLOR_SPACE_SRGB;
@@ -413,7 +414,7 @@ void dcn20_init_blank(
 	/* get the OPTC source */
 	tg->funcs->get_optc_source(tg, &num_opps, &opp_id_src0, &opp_id_src1);
 
-	if (opp_id_src0 >= dc->res_pool->res_cap->num_opp) {
+	if (opp_id_src0 >= num_opp_cap) {
 		ASSERT(false);
 		return;
 	}
@@ -426,7 +427,7 @@ void dcn20_init_blank(
 	if (num_opps == 2) {
 		otg_active_width = otg_active_width / 2;
 
-		if (opp_id_src1 >= dc->res_pool->res_cap->num_opp) {
+		if (opp_id_src1 >= num_opp_cap) {
 			ASSERT(false);
 			return;
 		}
@@ -826,7 +827,7 @@ enum dc_status dcn20_enable_stream_timing(
 	unsigned int event_triggers = 0;
 	int opp_cnt = 1;
 	int opp_inst[MAX_PIPES] = {0};
-	bool interlace = stream->timing.flags.INTERLACE;
+	bool interlace = (stream->timing.flags.INTERLACE != 0);
 	int i;
 	struct mpc_dwb_flow_control flow_control;
 	struct mpc *mpc = dc->res_pool->mpc;
@@ -1405,9 +1406,9 @@ void dcn20_pipe_control_lock(
 	}
 
 	if (flip_immediate && lock) {
-		const int TIMEOUT_FOR_FLIP_PENDING_US = 100000;
+		const unsigned int TIMEOUT_FOR_FLIP_PENDING_US = 100000U;
 		unsigned int polling_interval_us = 1;
-		int i;
+		unsigned int i;
 
 		temp_pipe = pipe;
 		while (temp_pipe) {
@@ -1452,7 +1453,7 @@ void dcn20_pipe_control_lock(
 		struct dmub_hw_lock_inst_flags inst_flags = { 0 };
 
 		hw_locks.bits.lock_pipe = 1;
-		inst_flags.otg_inst =  pipe->stream_res.tg->inst;
+		inst_flags.otg_inst = (uint8_t)pipe->stream_res.tg->inst;
 
 		if (pipe->plane_state != NULL)
 			hw_locks.bits.triple_buffer_lock = pipe->plane_state->triplebuffer_flips;
@@ -1870,7 +1871,7 @@ void dcn20_update_dchubp_dpp(
 static int dcn20_calculate_vready_offset_for_group(struct pipe_ctx *pipe)
 {
 	struct pipe_ctx *other_pipe;
-	int vready_offset = pipe->pipe_dlg_param.vready_offset;
+	unsigned int vready_offset = pipe->pipe_dlg_param.vready_offset;
 
 	/* Always use the largest vready_offset of all connected pipes */
 	for (other_pipe = pipe->bottom_pipe; other_pipe != NULL; other_pipe = other_pipe->bottom_pipe) {
@@ -2038,7 +2039,7 @@ void dcn20_program_front_end_for_ctx(
 		struct dc *dc,
 		struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	unsigned int prev_hubp_count = 0;
 	unsigned int hubp_count = 0;
 	struct dce_hwseq *hws = dc->hwseq;
@@ -2242,7 +2243,7 @@ void dcn20_post_unlock_program_front_end(
 	unsigned int timeout_us = 100000;
 	unsigned int polling_interval_us = 1;
 	struct dce_hwseq *hwseq = dc->hwseq;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++)
 		if (resource_is_pipe_type(&dc->current_state->res_ctx.pipe_ctx[i], OPP_HEAD) &&
@@ -2266,7 +2267,7 @@ void dcn20_post_unlock_program_front_end(
 		if (pipe->plane_state && !pipe->top_pipe && pipe->update_flags.bits.enable &&
 			dc_state_get_pipe_subvp_type(context, pipe) != SUBVP_PHANTOM) {
 			struct hubp *hubp = pipe->plane_res.hubp;
-			int j = 0;
+			unsigned int j = 0;
 
 			for (j = 0; j < timeout_us / polling_interval_us
 				&& hubp->funcs->hubp_is_flip_pending(hubp); j++)
@@ -2285,7 +2286,7 @@ void dcn20_post_unlock_program_front_end(
 		if (resource_is_pipe_type(old_pipe, OTG_MASTER) && resource_is_pipe_type(pipe, OTG_MASTER) &&
 			resource_get_odm_slice_count(old_pipe) < resource_get_odm_slice_count(pipe) &&
 			dc_state_get_pipe_subvp_type(context, pipe) != SUBVP_PHANTOM) {
-			int j = 0;
+			unsigned int j = 0;
 			struct timing_generator *tg = pipe->stream_res.tg;
 
 			if (tg->funcs->get_optc_double_buffer_pending) {
@@ -2420,7 +2421,7 @@ void dcn20_optimize_bandwidth(
 		struct dc_state *context)
 {
 	struct hubbub *hubbub = dc->res_pool->hubbub;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe = &context->res_ctx.pipe_ctx[i];
@@ -2438,10 +2439,13 @@ void dcn20_optimize_bandwidth(
 					dc->res_pool->ref_clocks.dchub_ref_clock_inKhz / 1000,
 					true);
 
-	if (dc->clk_mgr->dc_mode_softmax_enabled)
-		if (dc->clk_mgr->clks.dramclk_khz > dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000 &&
-				context->bw_ctx.bw.dcn.clk.dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000)
+	if (dc->clk_mgr->dc_mode_softmax_enabled) {
+		int softmax_memclk_khz = dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000;
+
+		if (dc->clk_mgr->clks.dramclk_khz > softmax_memclk_khz &&
+				context->bw_ctx.bw.dcn.clk.dramclk_khz <= softmax_memclk_khz)
 			dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
+	}
 
 	/* increase compbuf size */
 	if (hubbub->funcs->program_compbuf_size)
@@ -2478,7 +2482,7 @@ bool dcn20_update_bandwidth(
 		struct dc *dc,
 		struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	struct dce_hwseq *hws = dc->hwseq;
 
 	/* recalculate DML parameters */
@@ -2733,7 +2737,8 @@ void dcn20_update_plane_addr(const struct dc *dc, struct pipe_ctx *pipe_ctx)
 	addr_patched = patch_address_for_sbs_tb_stereo(pipe_ctx, &addr);
 
 	// Call Helper to track VMID use
-	vm_helper_mark_vmid_used(dc->vm_helper, plane_state->address.vmid, pipe_ctx->plane_res.hubp->inst);
+	vm_helper_mark_vmid_used(dc->vm_helper, plane_state->address.vmid,
+			(uint8_t)pipe_ctx->plane_res.hubp->inst);
 
 	pipe_ctx->plane_res.hubp->funcs->hubp_program_surface_flip_and_addr(
 			pipe_ctx->plane_res.hubp,
@@ -3126,7 +3131,7 @@ void dcn20_program_dmdata_engine(struct pipe_ctx *pipe_ctx)
 
 void dcn20_fpga_init_hw(struct dc *dc)
 {
-	int i, j;
+	uint8_t i, j;
 	struct dce_hwseq *hws = dc->hwseq;
 	struct resource_pool *res_pool = dc->res_pool;
 	struct dc_state  *context = dc->current_state;
@@ -3199,7 +3204,7 @@ void dcn20_fpga_init_hw(struct dc *dc)
 
 		pipe_ctx->plane_res.hubp = hubp;
 		pipe_ctx->plane_res.dpp = dpp;
-		pipe_ctx->plane_res.mpcc_inst = dpp->inst;
+		pipe_ctx->plane_res.mpcc_inst = (uint8_t)dpp->inst;
 		hubp->mpcc_id = dpp->inst;
 		hubp->opp_id = OPP_ID_INVALID;
 		hubp->power_gated = false;
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c
index ad253c586ea1a..1797a91b0186e 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_init.c
@@ -136,6 +136,8 @@ static const struct hwseq_private_funcs dcn20_private_funcs = {
 	.dccg_init = dcn20_dccg_init,
 	.set_blend_lut = dcn20_set_blend_lut,
 	.set_shaper_3dlut = dcn20_set_shaper_3dlut,
+	.wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed,
+	.set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe,
 };
 
 void dcn20_hw_sequencer_construct(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c
index 062745389d9a4..476c2112afec1 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn21/dcn21_hwseq.c
@@ -146,10 +146,10 @@ bool dcn21_dmub_abm_set_pipe(struct abm *abm, uint32_t otg_inst,
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.abm_set_pipe.header.type = DMUB_CMD__ABM;
 	cmd.abm_set_pipe.header.sub_type = DMUB_CMD__ABM_SET_PIPE;
-	cmd.abm_set_pipe.abm_set_pipe_data.otg_inst = otg_inst;
-	cmd.abm_set_pipe.abm_set_pipe_data.pwrseq_inst = pwrseq_inst;
-	cmd.abm_set_pipe.abm_set_pipe_data.set_pipe_option = option;
-	cmd.abm_set_pipe.abm_set_pipe_data.panel_inst = panel_inst;
+	cmd.abm_set_pipe.abm_set_pipe_data.otg_inst = (uint8_t)otg_inst;
+	cmd.abm_set_pipe.abm_set_pipe_data.pwrseq_inst = (uint8_t)pwrseq_inst;
+	cmd.abm_set_pipe.abm_set_pipe_data.set_pipe_option = (uint8_t)option;
+	cmd.abm_set_pipe.abm_set_pipe_data.panel_inst = (uint8_t)panel_inst;
 	cmd.abm_set_pipe.abm_set_pipe_data.ramping_boundary = ramping_boundary;
 	cmd.abm_set_pipe.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_pipe_data);
 
@@ -287,7 +287,7 @@ bool dcn21_set_backlight_level(struct pipe_ctx *pipe_ctx,
 bool dcn21_is_abm_supported(struct dc *dc,
 		struct dc_state *context, struct dc_stream_state *stream)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
index 2aa0f1de81037..3c70d685ba65f 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_hwseq.c
@@ -79,7 +79,7 @@ void dcn30_log_color_state(struct dc *dc,
 	struct dc_context *dc_ctx = dc->ctx;
 	struct resource_pool *pool = dc->res_pool;
 	bool is_gamut_remap_available = false;
-	int i;
+	unsigned int i;
 
 	DTN_INFO("DPP:  DGAM ROM  DGAM ROM type  DGAM LUT  SHAPER mode"
 		 "  3DLUT mode  3DLUT bit depth  3DLUT size  RGAM mode"
@@ -262,7 +262,7 @@ static bool dcn30_set_mpc_shaper_3dlut(struct pipe_ctx *pipe_ctx,
 	struct dc *dc = pipe_ctx->stream->ctx->dc;
 	struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc;
 	bool result = false;
-	int acquired_rmu = 0;
+	uint32_t acquired_rmu = 0;
 	int mpcc_id_projected = 0;
 
 	const struct pwl_params *shaper_lut = NULL;
@@ -439,7 +439,7 @@ static void dcn30_set_writeback(
 	ASSERT(wb_info->dwb_pipe_inst < MAX_DWB_PIPES);
 	ASSERT(wb_info->wb_enabled);
 	ASSERT(wb_info->mpcc_inst >= 0);
-	ASSERT(wb_info->mpcc_inst < dc->res_pool->mpcc_count);
+	ASSERT(wb_info->mpcc_inst < (int)dc->res_pool->mpcc_count);
 	mcif_wb = dc->res_pool->mcif_wb[wb_info->dwb_pipe_inst];
 	mcif_buf_params = &wb_info->mcif_buf_params;
 
@@ -581,7 +581,7 @@ void dcn30_program_all_writeback_pipes_in_tree(
 	struct dc_writeback_info wb_info;
 	struct dwbc *dwb;
 	struct dc_stream_status *stream_status = NULL;
-	int i_wb, i_pipe, i_stream;
+	unsigned int i_wb, i_pipe, i_stream;
 	DC_LOG_DWB("%s", __func__);
 
 	ASSERT(stream);
@@ -593,7 +593,9 @@ void dcn30_program_all_writeback_pipes_in_tree(
 	}
 	ASSERT(stream_status);
 
-	ASSERT(stream->num_wb_info <= dc->res_pool->res_cap->num_dwb);
+	// Assert non-negative signed capacity first.
+	ASSERT(dc->res_pool->res_cap->num_dwb >= 0);
+	ASSERT(stream->num_wb_info <= (unsigned int)dc->res_pool->res_cap->num_dwb);
 	/* For each writeback pipe */
 	for (i_wb = 0; i_wb < stream->num_wb_info; i_wb++) {
 
@@ -645,7 +647,7 @@ void dcn30_init_hw(struct dc *dc)
 	struct dce_hwseq *hws = dc->hwseq;
 	struct dc_bios *dcb = dc->ctx->dc_bios;
 	struct resource_pool *res_pool = dc->res_pool;
-	int i;
+	unsigned int i;
 	unsigned int edp_num;
 	uint32_t backlight = MAX_BACKLIGHT_LEVEL;
 	uint32_t user_level = MAX_BACKLIGHT_LEVEL;
@@ -978,7 +980,7 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable)
 							cursor_cache_enable ? &cursor_attr : NULL)) {
 				unsigned int v_total = stream->adjust.v_total_max ?
 						stream->adjust.v_total_max : stream->timing.v_total;
-				unsigned int refresh_hz = div_u64((unsigned long long) stream->timing.pix_clk_100hz *
+				unsigned int refresh_hz = (unsigned int)div_u64((unsigned long long)stream->timing.pix_clk_100hz *
 						100LL, (v_total * stream->timing.h_total));
 
 				/*
@@ -1006,9 +1008,9 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable)
 				unsigned int denom = refresh_hz * 6528;
 				unsigned int stutter_period = dc->current_state->perf_params.stutter_period_us;
 
-				tmr_delay = div_u64(((1000000LL + 2 * stutter_period * refresh_hz) *
+				tmr_delay = (uint32_t)(div_u64(((1000000LL + 2 * stutter_period * refresh_hz) *
 						(100LL + dc->debug.mall_additional_timer_percent) + denom - 1),
-						denom) - 64LL;
+						denom) - 64LL);
 
 				/* In some cases the stutter period is really big (tiny modes) in these
 				 * cases MALL cant be enabled, So skip these cases to avoid a ASSERT()
@@ -1030,9 +1032,9 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable)
 					}
 
 					denom *= 2;
-					tmr_delay = div_u64(((1000000LL + 2 * stutter_period * refresh_hz) *
+					tmr_delay = (uint32_t)(div_u64(((1000000LL + 2 * stutter_period * refresh_hz) *
 							(100LL + dc->debug.mall_additional_timer_percent) + denom - 1),
-							denom) - 64LL;
+							denom) - 64LL);
 				}
 
 				/* Copy HW cursor */
@@ -1062,9 +1064,9 @@ bool dcn30_apply_idle_power_optimizations(struct dc *dc, bool enable)
 					cmd.mall.cursor_copy_src.quad_part = cursor_attr.address.quad_part;
 					cmd.mall.cursor_copy_dst.quad_part =
 							(plane->address.grph.cursor_cache_addr.quad_part + 2047) & ~2047;
-					cmd.mall.cursor_width = cursor_attr.width;
-					cmd.mall.cursor_height = cursor_attr.height;
-					cmd.mall.cursor_pitch = cursor_attr.pitch;
+					cmd.mall.cursor_width = (uint16_t)cursor_attr.width;
+					cmd.mall.cursor_height = (uint16_t)cursor_attr.height;
+					cmd.mall.cursor_pitch = (uint16_t)cursor_attr.pitch;
 
 					dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 
@@ -1197,10 +1199,13 @@ void dcn30_prepare_bandwidth(struct dc *dc,
 		context->bw_ctx.bw.dcn.clk.p_state_change_support = false;
 	}
 
-	if (dc->clk_mgr->dc_mode_softmax_enabled)
-		if (dc->clk_mgr->clks.dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000 &&
-				context->bw_ctx.bw.dcn.clk.dramclk_khz > dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000)
+	if (dc->clk_mgr->dc_mode_softmax_enabled) {
+		int softmax_memclk_khz = dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000;
+
+		if (dc->clk_mgr->clks.dramclk_khz <= softmax_memclk_khz &&
+				context->bw_ctx.bw.dcn.clk.dramclk_khz > softmax_memclk_khz)
 			dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, dc->clk_mgr->bw_params->clk_table.entries[dc->clk_mgr->bw_params->clk_table.num_entries - 1].memclk_mhz);
+	}
 
 	dcn20_prepare_bandwidth(dc, context);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c
index 5cbae0cdda962..9834d30754878 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn30/dcn30_init.c
@@ -145,6 +145,8 @@ static const struct hwseq_private_funcs dcn30_private_funcs = {
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
 	.set_blend_lut = dcn30_set_blend_lut,
 	.set_shaper_3dlut = dcn20_set_shaper_3dlut,
+	.wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed,
+	.set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe,
 };
 
 void dcn30_hw_sequencer_construct(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c
index 33cc48cd01962..a570333aeac18 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn301/dcn301_init.c
@@ -142,6 +142,8 @@ static const struct hwseq_private_funcs dcn301_private_funcs = {
 	.wait_for_blank_complete = dcn20_wait_for_blank_complete,
 	.set_blend_lut = dcn30_set_blend_lut,
 	.set_shaper_3dlut = dcn20_set_shaper_3dlut,
+	.wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed,
+	.set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe,
 };
 
 void dcn301_hw_sequencer_construct(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
index b4afb2bc44934..71643ccf23d92 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_hwseq.c
@@ -72,7 +72,7 @@
 static void enable_memory_low_power(struct dc *dc)
 {
 	struct dce_hwseq *hws = dc->hwseq;
-	int i;
+	unsigned int i;
 
 	if (dc->debug.enable_mem_low_power.bits.dmcu) {
 		// Force ERAM to shutdown if DMCU is not enabled
@@ -116,7 +116,7 @@ void dcn31_init_hw(struct dc *dc)
 	struct resource_pool *res_pool = dc->res_pool;
 	uint32_t backlight = MAX_BACKLIGHT_LEVEL;
 	uint32_t user_level = MAX_BACKLIGHT_LEVEL;
-	int i;
+	unsigned int i;
 
 	if (dc->clk_mgr && dc->clk_mgr->funcs->init_clocks)
 		dc->clk_mgr->funcs->init_clocks(dc->clk_mgr);
@@ -399,6 +399,9 @@ void dcn31_update_info_frame(struct pipe_ctx *pipe_ctx)
 				pipe_ctx->stream_res.hpo_dp_stream_enc,
 				&pipe_ctx->stream_res.encoder_info_frame);
 
+		pipe_ctx->stream_res.encoder_info_frame.firmware_controlled_hdr_info_packet
+			= pipe_ctx->stream->firmware_controlled_hdr_info_packet;
+
 		pipe_ctx->stream_res.hpo_dp_stream_enc->funcs->update_dp_info_packets(
 				pipe_ctx->stream_res.hpo_dp_stream_enc,
 				&pipe_ctx->stream_res.encoder_info_frame);
@@ -409,6 +412,9 @@ void dcn31_update_info_frame(struct pipe_ctx *pipe_ctx)
 				pipe_ctx->stream_res.stream_enc,
 				&pipe_ctx->stream_res.encoder_info_frame);
 
+		pipe_ctx->stream_res.encoder_info_frame.firmware_controlled_hdr_info_packet
+			= pipe_ctx->stream->firmware_controlled_hdr_info_packet;
+
 		pipe_ctx->stream_res.stream_enc->funcs->update_dp_info_packets(
 			pipe_ctx->stream_res.stream_enc,
 			&pipe_ctx->stream_res.encoder_info_frame);
@@ -665,7 +671,7 @@ void dcn31_setup_hpo_hw_control(const struct dce_hwseq *hws, bool enable)
 void dcn31_set_static_screen_control(struct pipe_ctx **pipe_ctx,
 		int num_pipes, const struct dc_static_screen_params *params)
 {
-	unsigned int i;
+	int i;
 	unsigned int triggers = 0;
 
 	if (params->triggers.surface_update)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c
index e56b9a46aecfe..b14e6e60b8786 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn31/dcn31_init.c
@@ -147,6 +147,8 @@ static const struct hwseq_private_funcs dcn31_private_funcs = {
 	.set_blend_lut = dcn30_set_blend_lut,
 	.set_shaper_3dlut = dcn20_set_shaper_3dlut,
 	.setup_hpo_hw_control = dcn31_setup_hpo_hw_control,
+	.wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed,
+	.set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe,
 };
 
 void dcn31_hw_sequencer_construct(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
index 858a06b03b577..09dfbb16dd29a 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_hwseq.c
@@ -147,7 +147,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 
 // Given any pipe_ctx, return the total ODM combine factor, and optionally return
 // the OPPids which are used
-static unsigned int get_odm_config(struct pipe_ctx *pipe_ctx, unsigned int *opp_instances)
+static unsigned int get_odm_config(struct pipe_ctx *pipe_ctx, int *opp_instances)
 {
 	unsigned int opp_count = 1;
 	struct pipe_ctx *odm_pipe;
@@ -211,7 +211,7 @@ void dcn314_update_odm(struct dc *dc, struct dc_state *context, struct pipe_ctx
 	if (pipe_ctx->stream_res.dsc) {
 		struct pipe_ctx *current_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[pipe_ctx->pipe_idx];
 
-		update_dsc_on_stream(pipe_ctx, pipe_ctx->stream->timing.flags.DSC);
+		update_dsc_on_stream(pipe_ctx, pipe_ctx->stream->timing.flags.DSC != 0);
 
 		/* Check if no longer using pipe for ODM, then need to disconnect DSC for that pipe */
 		if (!pipe_ctx->next_odm_pipe && current_pipe_ctx->next_odm_pipe &&
@@ -232,60 +232,60 @@ void dcn314_dsc_pg_control(
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl = 0;
 
-	if (hws->ctx->dc->debug.disable_dsc_power_gate)
-		return;
-
 	if (hws->ctx->dc->debug.root_clock_optimization.bits.dsc &&
 		hws->ctx->dc->res_pool->dccg->funcs->enable_dsc &&
 		power_on)
 		hws->ctx->dc->res_pool->dccg->funcs->enable_dsc(
 			hws->ctx->dc->res_pool->dccg, dsc_inst);
 
-	REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
-	if (org_ip_request_cntl == 0)
-		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
-
-	switch (dsc_inst) {
-	case 0: /* DSC0 */
-		REG_UPDATE(DOMAIN16_PG_CONFIG,
-				DOMAIN_POWER_GATE, power_gate);
-
-		REG_WAIT(DOMAIN16_PG_STATUS,
-				DOMAIN_PGFSM_PWR_STATUS, pwr_status,
-				1, 1000);
-		break;
-	case 1: /* DSC1 */
-		REG_UPDATE(DOMAIN17_PG_CONFIG,
-				DOMAIN_POWER_GATE, power_gate);
-
-		REG_WAIT(DOMAIN17_PG_STATUS,
-				DOMAIN_PGFSM_PWR_STATUS, pwr_status,
-				1, 1000);
-		break;
-	case 2: /* DSC2 */
-		REG_UPDATE(DOMAIN18_PG_CONFIG,
-				DOMAIN_POWER_GATE, power_gate);
-
-		REG_WAIT(DOMAIN18_PG_STATUS,
-				DOMAIN_PGFSM_PWR_STATUS, pwr_status,
-				1, 1000);
-		break;
-	case 3: /* DSC3 */
-		REG_UPDATE(DOMAIN19_PG_CONFIG,
-				DOMAIN_POWER_GATE, power_gate);
+	if (!hws->ctx->dc->debug.disable_dsc_power_gate) {
+
+		REG_GET(DC_IP_REQUEST_CNTL, IP_REQUEST_EN, &org_ip_request_cntl);
+		if (org_ip_request_cntl == 0)
+			REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 1);
+
+		switch (dsc_inst) {
+		case 0: /* DSC0 */
+			REG_UPDATE(DOMAIN16_PG_CONFIG,
+					DOMAIN_POWER_GATE, power_gate);
+
+			REG_WAIT(DOMAIN16_PG_STATUS,
+					DOMAIN_PGFSM_PWR_STATUS, pwr_status,
+					1, 1000);
+			break;
+		case 1: /* DSC1 */
+			REG_UPDATE(DOMAIN17_PG_CONFIG,
+					DOMAIN_POWER_GATE, power_gate);
+
+			REG_WAIT(DOMAIN17_PG_STATUS,
+					DOMAIN_PGFSM_PWR_STATUS, pwr_status,
+					1, 1000);
+			break;
+		case 2: /* DSC2 */
+			REG_UPDATE(DOMAIN18_PG_CONFIG,
+					DOMAIN_POWER_GATE, power_gate);
+
+			REG_WAIT(DOMAIN18_PG_STATUS,
+					DOMAIN_PGFSM_PWR_STATUS, pwr_status,
+					1, 1000);
+			break;
+		case 3: /* DSC3 */
+			REG_UPDATE(DOMAIN19_PG_CONFIG,
+					DOMAIN_POWER_GATE, power_gate);
+
+			REG_WAIT(DOMAIN19_PG_STATUS,
+					DOMAIN_PGFSM_PWR_STATUS, pwr_status,
+					1, 1000);
+			break;
+		default:
+			BREAK_TO_DEBUGGER();
+			break;
+		}
 
-		REG_WAIT(DOMAIN19_PG_STATUS,
-				DOMAIN_PGFSM_PWR_STATUS, pwr_status,
-				1, 1000);
-		break;
-	default:
-		BREAK_TO_DEBUGGER();
-		break;
+		if (org_ip_request_cntl == 0)
+			REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 0);
 	}
 
-	if (org_ip_request_cntl == 0)
-		REG_SET(DC_IP_REQUEST_CNTL, 0, IP_REQUEST_EN, 0);
-
 	if (hws->ctx->dc->debug.root_clock_optimization.bits.dsc) {
 		if (hws->ctx->dc->res_pool->dccg->funcs->disable_dsc && !power_on)
 			hws->ctx->dc->res_pool->dccg->funcs->disable_dsc(
@@ -419,7 +419,7 @@ void dcn314_resync_fifo_dccg_dio(struct dce_hwseq *hws, struct dc *dc, struct dc
 			if (dcn314_is_pipe_dig_fifo_on(pipe))
 				continue;
 			pipe->stream_res.tg->funcs->disable_crtc(pipe->stream_res.tg);
-			reset_sync_context_for_pipe(dc, context, i);
+			reset_sync_context_for_pipe(dc, context, (uint8_t)i);
 			otg_disabled[i] = true;
 		}
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c
index 9900c87b4567f..d782080883ab7 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn314/dcn314_init.c
@@ -154,6 +154,8 @@ static const struct hwseq_private_funcs dcn314_private_funcs = {
 	.setup_hpo_hw_control = dcn31_setup_hpo_hw_control,
 	.calculate_dccg_k1_k2_values = dcn314_calculate_dccg_k1_k2_values,
 	.resync_fifo_dccg_dio = dcn314_resync_fifo_dccg_dio,
+	.wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed,
+	.set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe,
 };
 
 void dcn314_hw_sequencer_construct(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
index b45ceb570a5ca..415b3f875f0d1 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_hwseq.c
@@ -226,7 +226,7 @@ static bool dcn32_check_no_memory_request_for_cab(struct dc *dc)
  */
 static uint32_t dcn32_calculate_cab_allocation(struct dc *dc, struct dc_state *ctx)
 {
-	int i;
+	unsigned int i;
 	uint32_t num_ways = 0;
 	uint32_t mall_ss_size_bytes = 0;
 
@@ -351,7 +351,7 @@ bool dcn32_apply_idle_power_optimizations(struct dc *dc, bool enable)
  */
 void dcn32_commit_subvp_config(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	bool enable_subvp = false;
 
 	if (!dc->ctx || !dc->ctx->dmub_srv)
@@ -607,7 +607,7 @@ bool dcn32_set_output_transfer_func(struct dc *dc,
  */
 void dcn32_update_force_pstate(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 
 	/* Unforce p-state for each pipe if it is not FPO or SubVP.
 	 * For FPO and SubVP, if it's already forced disallow, leave
@@ -674,7 +674,7 @@ void dcn32_update_force_pstate(struct dc *dc, struct dc_state *context)
  */
 void dcn32_update_mall_sel(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	unsigned int num_ways = dcn32_calculate_cab_allocation(dc, context);
 	bool cache_cursor = false;
 
@@ -728,7 +728,7 @@ void dcn32_update_mall_sel(struct dc *dc, struct dc_state *context)
  */
 void dcn32_program_mall_pipe_config(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	struct dce_hwseq *hws = dc->hwseq;
 
 	// Don't force p-state disallow -- can't block dummy p-state
@@ -793,7 +793,7 @@ void dcn32_init_hw(struct dc *dc)
 	struct dce_hwseq *hws = dc->hwseq;
 	struct dc_bios *dcb = dc->ctx->dc_bios;
 	struct resource_pool *res_pool = dc->res_pool;
-	int i;
+	unsigned int i;
 	unsigned int edp_num;
 	uint32_t backlight = MAX_BACKLIGHT_LEVEL;
 	uint32_t user_level = MAX_BACKLIGHT_LEVEL;
@@ -1119,7 +1119,7 @@ void dcn32_update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 * Given any pipe_ctx, return the total ODM combine factor, and optionally return
 * the OPPids which are used
 * */
-static unsigned int get_odm_config(struct pipe_ctx *pipe_ctx, unsigned int *opp_instances)
+static unsigned int get_odm_config(struct pipe_ctx *pipe_ctx, int *opp_instances)
 {
 	unsigned int opp_count = 1;
 	struct pipe_ctx *odm_pipe;
@@ -1175,7 +1175,7 @@ void dcn32_update_odm(struct dc *dc, struct dc_state *context, struct pipe_ctx *
 	if (pipe_ctx->stream_res.dsc) {
 		struct pipe_ctx *current_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[pipe_ctx->pipe_idx];
 
-		dcn32_update_dsc_on_stream(pipe_ctx, pipe_ctx->stream->timing.flags.DSC);
+		dcn32_update_dsc_on_stream(pipe_ctx, pipe_ctx->stream->timing.flags.DSC != 0);
 
 		/* Check if no longer using pipe for ODM, then need to disconnect DSC for that pipe */
 		if (!pipe_ctx->next_odm_pipe && current_pipe_ctx->next_odm_pipe &&
@@ -1277,7 +1277,7 @@ void dcn32_resync_fifo_dccg_dio(struct dce_hwseq *hws, struct dc *dc, struct dc_
 		if ((pipe->stream->dpms_off || dc_is_virtual_signal(pipe->stream->signal))
 			&& dc_state_get_pipe_subvp_type(dc_state, pipe) != SUBVP_PHANTOM) {
 			pipe->stream_res.tg->funcs->disable_crtc(pipe->stream_res.tg);
-			reset_sync_context_for_pipe(dc, context, i);
+			reset_sync_context_for_pipe(dc, context, (uint8_t)i);
 			otg_disabled[i] = true;
 		}
 	}
@@ -1655,6 +1655,7 @@ void dcn32_init_blank(
 	uint32_t num_opps, opp_id_src0, opp_id_src1;
 	uint32_t otg_active_width = 0, otg_active_height = 0;
 	uint32_t i;
+	unsigned int num_opp_cap = (unsigned int)dc->res_pool->res_cap->num_opp;
 
 	/* program opp dpg blank color */
 	color_space = COLOR_SPACE_SRGB;
@@ -1668,12 +1669,12 @@ void dcn32_init_blank(
 	/* get the OPTC source */
 	tg->funcs->get_optc_source(tg, &num_opps, &opp_id_src0, &opp_id_src1);
 
-	if (opp_id_src0 >= dc->res_pool->res_cap->num_opp) {
+	if (opp_id_src0 >= num_opp_cap) {
 		ASSERT(false);
 		return;
 	}
 
-	for (i = 0; i < dc->res_pool->res_cap->num_opp; i++) {
+	for (i = 0; i < num_opp_cap; i++) {
 		if (dc->res_pool->opps[i] != NULL && dc->res_pool->opps[i]->inst == opp_id_src0) {
 			opp = dc->res_pool->opps[i];
 			break;
@@ -1683,11 +1684,11 @@ void dcn32_init_blank(
 	if (num_opps == 2) {
 		otg_active_width = otg_active_width / 2;
 
-		if (opp_id_src1 >= dc->res_pool->res_cap->num_opp) {
+		if (opp_id_src1 >= num_opp_cap) {
 			ASSERT(false);
 			return;
 		}
-		for (i = 0; i < dc->res_pool->res_cap->num_opp; i++) {
+		for (i = 0; i < num_opp_cap; i++) {
 			if (dc->res_pool->opps[i] != NULL && dc->res_pool->opps[i]->inst == opp_id_src1) {
 				bottom_opp = dc->res_pool->opps[i];
 				break;
@@ -1751,7 +1752,7 @@ bool dcn32_is_pipe_topology_transition_seamless(struct dc *dc,
 		const struct dc_state *cur_ctx,
 		const struct dc_state *new_ctx)
 {
-	int i;
+	unsigned int i;
 	const struct pipe_ctx *cur_pipe, *new_pipe;
 	bool is_seamless = true;
 
@@ -1812,10 +1813,13 @@ void dcn32_prepare_bandwidth(struct dc *dc,
 		context->bw_ctx.bw.dcn.clk.p_state_change_support = false;
 	}
 
-	if (dc->clk_mgr->dc_mode_softmax_enabled)
-		if (dc->clk_mgr->clks.dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000 &&
-				context->bw_ctx.bw.dcn.clk.dramclk_khz > dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000)
+	if (dc->clk_mgr->dc_mode_softmax_enabled) {
+		int softmax_memclk_khz = dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000;
+
+		if (dc->clk_mgr->clks.dramclk_khz <= softmax_memclk_khz &&
+				context->bw_ctx.bw.dcn.clk.dramclk_khz > softmax_memclk_khz)
 			dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, dc->clk_mgr->bw_params->clk_table.entries[dc->clk_mgr->bw_params->clk_table.num_entries - 1].memclk_mhz);
+	}
 
 	dcn20_prepare_bandwidth(dc, context);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c
index 849dae18b7384..c68b201047732 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn32/dcn32_init.c
@@ -163,6 +163,8 @@ static const struct hwseq_private_funcs dcn32_private_funcs = {
 	.is_dp_dig_pixel_rate_div_policy = dcn32_is_dp_dig_pixel_rate_div_policy,
 	.apply_single_controller_ctx_to_hw = dce110_apply_single_controller_ctx_to_hw,
 	.reset_back_end_for_pipe = dcn20_reset_back_end_for_pipe,
+	.wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed,
+	.set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe,
 };
 
 void dcn32_hw_sequencer_init_functions(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
index f133b52ea9583..1a0123338dfae 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn35/dcn35_hwseq.c
@@ -145,7 +145,7 @@ void dcn35_init_hw(struct dc *dc)
 	struct resource_pool *res_pool = dc->res_pool;
 	uint32_t backlight = MAX_BACKLIGHT_LEVEL;
 	uint32_t user_level = MAX_BACKLIGHT_LEVEL;
-	int i;
+	unsigned int i;
 
 	print_pg_status(dc, __func__, ": start");
 
@@ -403,7 +403,7 @@ static void update_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable)
 
 // Given any pipe_ctx, return the total ODM combine factor, and optionally return
 // the OPPids which are used
-static unsigned int get_odm_config(struct pipe_ctx *pipe_ctx, unsigned int *opp_instances)
+static unsigned int get_odm_config(struct pipe_ctx *pipe_ctx, int *opp_instances)
 {
 	unsigned int opp_count = 1;
 	struct pipe_ctx *odm_pipe;
@@ -467,7 +467,7 @@ void dcn35_update_odm(struct dc *dc, struct dc_state *context, struct pipe_ctx *
 	if (pipe_ctx->stream_res.dsc) {
 		struct pipe_ctx *current_pipe_ctx = &dc->current_state->res_ctx.pipe_ctx[pipe_ctx->pipe_idx];
 
-		update_dsc_on_stream(pipe_ctx, pipe_ctx->stream->timing.flags.DSC);
+		update_dsc_on_stream(pipe_ctx, pipe_ctx->stream->timing.flags.DSC != 0);
 
 		/* Check if no longer using pipe for ODM, then need to disconnect DSC for that pipe */
 		if (!pipe_ctx->next_odm_pipe && current_pipe_ctx->next_odm_pipe &&
@@ -621,7 +621,7 @@ void dcn35_z10_restore(const struct dc *dc)
 
 void dcn35_init_pipes(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	uint8_t i;
 	struct dce_hwseq *hws = dc->hwseq;
 	struct hubbub *hubbub = dc->res_pool->hubbub;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
@@ -725,7 +725,7 @@ void dcn35_init_pipes(struct dc *dc, struct dc_state *context)
 
 		pipe_ctx->plane_res.hubp = hubp;
 		pipe_ctx->plane_res.dpp = dpp;
-		pipe_ctx->plane_res.mpcc_inst = dpp->inst;
+		pipe_ctx->plane_res.mpcc_inst = (uint8_t)dpp->inst;
 		hubp->mpcc_id = dpp->inst;
 		hubp->opp_id = OPP_ID_INVALID;
 		hubp->power_gated = false;
@@ -904,6 +904,13 @@ void dcn35_disable_plane(struct dc *dc, struct dc_state *state, struct pipe_ctx
 	if (!pipe_ctx->plane_res.hubp || pipe_ctx->plane_res.hubp->power_gated)
 		return;
 
+	if (dc->debug.disable_dpp_power_gate) {
+		/* Workaround for use case when disabled power gating set to 1 */
+		/* Force disable cursor if power gating is disabled */
+		struct dpp *dpp = pipe_ctx->plane_res.dpp;
+		if (dpp && dpp->funcs->dpp_force_disable_cursor)
+			dpp->funcs->dpp_force_disable_cursor(dpp);
+	}
 	if (hws->funcs.plane_atomic_disable)
 		hws->funcs.plane_atomic_disable(dc, pipe_ctx);
 
@@ -923,14 +930,15 @@ void dcn35_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
 	bool hpo_frl_stream_enc_acquired = false;
 	bool hpo_dp_stream_enc_acquired = false;
 	int i = 0, j = 0;
+	unsigned int ui = 0, uj = 0;
 	unsigned int edp_num = 0;
 	struct dc_link *edp_links[MAX_NUM_EDP] = { NULL };
 
 	memset(update_state, 0, sizeof(struct pg_block_update));
 
-	for (i = 0; i < dc->res_pool->hpo_dp_stream_enc_count; i++) {
-		if (context->res_ctx.is_hpo_dp_stream_enc_acquired[i] &&
-				dc->res_pool->hpo_dp_stream_enc[i]) {
+	for (ui = 0; ui < dc->res_pool->hpo_dp_stream_enc_count; ui++) {
+		if (context->res_ctx.is_hpo_dp_stream_enc_acquired[ui] &&
+				dc->res_pool->hpo_dp_stream_enc[ui]) {
 			hpo_dp_stream_enc_acquired = true;
 			break;
 		}
@@ -941,11 +949,11 @@ void dcn35_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
 
 	update_state->pg_res_update[PG_DWB] = true;
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
-		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
+	for (ui = 0; ui < dc->res_pool->pipe_count; ui++) {
+		struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[ui];
 
 		for (j = 0; j < PG_HW_PIPE_RESOURCES_NUM_ELEMENT; j++)
-			update_state->pg_pipe_res_update[j][i] = true;
+			update_state->pg_pipe_res_update[j][ui] = true;
 
 		if (!pipe_ctx)
 			continue;
@@ -968,9 +976,9 @@ void dcn35_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
 				/* All HUBP/DPP instances must be powered if the DSC inst != HUBP inst */
 				if (!pipe_ctx->top_pipe && pipe_ctx->plane_res.hubp &&
 				    pipe_ctx->plane_res.hubp->inst != pipe_ctx->stream_res.dsc->inst) {
-					for (j = 0; j < dc->res_pool->pipe_count; ++j) {
-						update_state->pg_pipe_res_update[PG_HUBP][j] = false;
-						update_state->pg_pipe_res_update[PG_DPP][j] = false;
+					for (uj = 0; uj < dc->res_pool->pipe_count; ++uj) {
+						update_state->pg_pipe_res_update[PG_HUBP][uj] = false;
+						update_state->pg_pipe_res_update[PG_DPP][uj] = false;
 					}
 				}
 			}
@@ -990,10 +998,10 @@ void dcn35_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
 	}
 
 	/*domain24 controls all the otg, mpc, opp, as long as one otg is still up, avoid enabling OTG PG*/
-	for (i = 0; i < dc->res_pool->timing_generator_count; i++) {
-		struct timing_generator *tg = dc->res_pool->timing_generators[i];
+	for (ui = 0; ui < dc->res_pool->timing_generator_count; ui++) {
+		struct timing_generator *tg = dc->res_pool->timing_generators[ui];
 		if (tg && tg->funcs->is_tg_enabled(tg)) {
-			update_state->pg_pipe_res_update[PG_OPTC][i] = false;
+			update_state->pg_pipe_res_update[PG_OPTC][ui] = false;
 			break;
 		}
 	}
@@ -1027,10 +1035,11 @@ void dcn35_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 	bool hpo_frl_stream_enc_acquired = false;
 	bool hpo_dp_stream_enc_acquired = false;
 	int i = 0, j = 0;
+	unsigned int ui = 0, uj = 0;
 
 	memset(update_state, 0, sizeof(struct pg_block_update));
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < (int)dc->res_pool->pipe_count; i++) {
 		struct pipe_ctx *cur_pipe = &dc->current_state->res_ctx.pipe_ctx[i];
 		struct pipe_ctx *new_pipe = &context->res_ctx.pipe_ctx[i];
 
@@ -1104,9 +1113,9 @@ void dcn35_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 		if (dc->links[i]->type != dc_connection_none)
 			update_state->pg_pipe_res_update[PG_PHYSYMCLK][dc->links[i]->link_enc_hw_inst] = true;
 
-	for (i = 0; i < dc->res_pool->hpo_dp_stream_enc_count; i++) {
-		if (context->res_ctx.is_hpo_dp_stream_enc_acquired[i] &&
-				dc->res_pool->hpo_dp_stream_enc[i]) {
+	for (ui = 0; ui < dc->res_pool->hpo_dp_stream_enc_count; ui++) {
+		if (context->res_ctx.is_hpo_dp_stream_enc_acquired[ui] &&
+				dc->res_pool->hpo_dp_stream_enc[ui]) {
 			hpo_dp_stream_enc_acquired = true;
 			break;
 		}
@@ -1119,8 +1128,8 @@ void dcn35_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 		update_state->pg_pipe_res_update[PG_HDMISTREAM][0] = true;
 
 	if (dc->caps.sequential_ono) {
-		for (i = 0; i < dc->res_pool->pipe_count; i++) {
-			struct pipe_ctx *new_pipe = &context->res_ctx.pipe_ctx[i];
+		for (ui = 0; ui < dc->res_pool->pipe_count; ui++) {
+			struct pipe_ctx *new_pipe = &context->res_ctx.pipe_ctx[ui];
 
 			if (new_pipe->stream_res.dsc && !new_pipe->top_pipe &&
 			    update_state->pg_pipe_res_update[PG_DSC][new_pipe->stream_res.dsc->inst]) {
@@ -1130,9 +1139,9 @@ void dcn35_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 				/* All HUBP/DPP instances must be powered if the DSC inst != HUBP inst */
 				if (new_pipe->plane_res.hubp &&
 				    new_pipe->plane_res.hubp->inst != new_pipe->stream_res.dsc->inst) {
-					for (j = 0; j < dc->res_pool->pipe_count; ++j) {
-						update_state->pg_pipe_res_update[PG_HUBP][j] = true;
-						update_state->pg_pipe_res_update[PG_DPP][j] = true;
+					for (uj = 0; uj < dc->res_pool->pipe_count; ++uj) {
+						update_state->pg_pipe_res_update[PG_HUBP][uj] = true;
+						update_state->pg_pipe_res_update[PG_DPP][uj] = true;
 					}
 				}
 			}
@@ -1179,6 +1188,7 @@ void dcn35_hw_block_power_down(struct dc *dc,
 	struct pg_block_update *update_state)
 {
 	int i = 0;
+	unsigned int pipe_idx = 0;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
 
 	if (!pg_cntl)
@@ -1192,11 +1202,11 @@ void dcn35_hw_block_power_down(struct dc *dc,
 	}
 
 	if (!dc->caps.sequential_ono) {
-		for (i = 0; i < dc->res_pool->pipe_count; i++) {
-			if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
-			    update_state->pg_pipe_res_update[PG_DPP][i]) {
+		for (pipe_idx = 0; pipe_idx < dc->res_pool->pipe_count; pipe_idx++) {
+			if (update_state->pg_pipe_res_update[PG_HUBP][pipe_idx] &&
+			    update_state->pg_pipe_res_update[PG_DPP][pipe_idx]) {
 				if (pg_cntl->funcs->hubp_dpp_pg_control)
-					pg_cntl->funcs->hubp_dpp_pg_control(pg_cntl, i, false);
+					pg_cntl->funcs->hubp_dpp_pg_control(pg_cntl, pipe_idx, false);
 			}
 		}
 
@@ -1255,7 +1265,7 @@ void dcn35_hw_block_power_down(struct dc *dc,
 void dcn35_hw_block_power_up(struct dc *dc,
 	struct pg_block_update *update_state)
 {
-	int i = 0;
+	unsigned int i = 0;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
 
 	if (!pg_cntl)
@@ -1268,7 +1278,7 @@ void dcn35_hw_block_power_up(struct dc *dc,
 		pg_cntl->funcs->plane_otg_pg_control(pg_cntl, true);
 
 	if (!dc->caps.sequential_ono) {
-		for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++)
+		for (i = 0; i < (unsigned int)dc->res_pool->res_cap->num_dsc; i++)
 			if (update_state->pg_pipe_res_update[PG_DSC][i]) {
 				if (pg_cntl->funcs->dsc_pg_control)
 					pg_cntl->funcs->dsc_pg_control(pg_cntl, i, true);
@@ -1297,7 +1307,7 @@ void dcn35_hw_block_power_up(struct dc *dc,
 void dcn35_root_clock_control(struct dc *dc,
 	struct pg_block_update *update_state, bool power_on)
 {
-	int i = 0;
+	unsigned int i = 0;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
 
 	if (!pg_cntl)
@@ -1321,7 +1331,7 @@ void dcn35_root_clock_control(struct dc *dc,
 					dc->hwseq->funcs.physymclk_root_clock_control(dc->hwseq, i, power_on);
 
 	}
-	for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)dc->res_pool->res_cap->num_dsc; i++) {
 		if (update_state->pg_pipe_res_update[PG_DSC][i]) {
 			if (power_on) {
 				if (dc->res_pool->dccg->funcs->enable_dsc)
@@ -1442,7 +1452,7 @@ void dcn35_set_drr(struct pipe_ctx **pipe_ctx,
 void dcn35_set_static_screen_control(struct pipe_ctx **pipe_ctx,
 		int num_pipes, const struct dc_static_screen_params *params)
 {
-	unsigned int i;
+	int i;
 	unsigned int triggers = 0;
 
 	if (params->triggers.surface_update)
@@ -1562,7 +1572,8 @@ bool dcn35_is_dp_dig_pixel_rate_div_policy(struct pipe_ctx *pipe_ctx)
  */
 static void dcn35_calc_blocks_to_ungate_for_hw_release(struct dc *dc, struct pg_block_update *update_state)
 {
-	int i = 0, j = 0;
+	unsigned int i = 0;
+	int j = 0;
 
 	memset(update_state, 0, sizeof(struct pg_block_update));
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.c
index 93fe5b262a3d9..8720e47fb33a1 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn351/dcn351_hwseq.c
@@ -152,7 +152,7 @@ void dcn351_hw_block_power_down(struct dc *dc,
 void dcn351_hw_block_power_up(struct dc *dc,
 	struct pg_block_update *update_state)
 {
-	int i = 0;
+	unsigned int i = 0;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
 
 	if (!pg_cntl || dc->debug.ignore_pg)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
index 9768b8f6b6fb6..c6b62870c46eb 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c
@@ -95,6 +95,10 @@ void dcn401_program_gamut_remap(struct pipe_ctx *pipe_ctx)
 	unsigned int mpcc_id = pipe_ctx->plane_res.mpcc_inst;
 	struct mpc *mpc = pipe_ctx->stream_res.opp->ctx->dc->res_pool->mpc;
 
+	//For now assert if location is not pre-blend
+	if (pipe_ctx->plane_state)
+		ASSERT(pipe_ctx->plane_state->mcm_location == MPCC_MOVABLE_CM_LOCATION_BEFORE);
+
 	// program MPCC_MCM_FIRST_GAMUT_REMAP
 	memset(&mpc_adjust, 0, sizeof(mpc_adjust));
 	mpc_adjust.gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_BYPASS;
@@ -139,7 +143,7 @@ void dcn401_init_hw(struct dc *dc)
 	struct dce_hwseq *hws = dc->hwseq;
 	struct dc_bios *dcb = dc->ctx->dc_bios;
 	struct resource_pool *res_pool = dc->res_pool;
-	int i;
+	unsigned int i;
 	unsigned int edp_num;
 	uint32_t backlight = MAX_BACKLIGHT_LEVEL;
 	uint32_t user_level = MAX_BACKLIGHT_LEVEL;
@@ -374,180 +378,298 @@ void dcn401_init_hw(struct dc *dc)
 	}
 }
 
-void dcn401_trigger_3dlut_dma_load(struct dc *dc, struct pipe_ctx *pipe_ctx)
+static void dcn401_get_mcm_lut_xable_from_pipe_ctx(struct dc *dc, struct pipe_ctx *pipe_ctx,
+		enum MCM_LUT_XABLE *shaper_xable,
+		enum MCM_LUT_XABLE *lut3d_xable,
+		enum MCM_LUT_XABLE *lut1d_xable)
 {
-	(void)dc;
-	struct hubp *hubp = pipe_ctx->plane_res.hubp;
+	enum dc_cm2_shaper_3dlut_setting shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL;
+	bool lut1d_enable = false;
+	struct mpc *mpc = dc->res_pool->mpc;
+	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
 
-	if (hubp->funcs->hubp_enable_3dlut_fl) {
-		hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
+	if (!pipe_ctx->plane_state)
+		return;
+	shaper_3dlut_setting = pipe_ctx->plane_state->mcm_shaper_3dlut_setting;
+	lut1d_enable = pipe_ctx->plane_state->mcm_lut1d_enable;
+	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
+	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
+
+	*lut1d_xable = lut1d_enable ? MCM_LUT_ENABLE : MCM_LUT_DISABLE;
+
+	switch (shaper_3dlut_setting) {
+	case DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL:
+		*lut3d_xable = *shaper_xable = MCM_LUT_DISABLE;
+		break;
+	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER:
+		*lut3d_xable = MCM_LUT_DISABLE;
+		*shaper_xable = MCM_LUT_ENABLE;
+		break;
+	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT:
+		*lut3d_xable = *shaper_xable = MCM_LUT_ENABLE;
+		break;
 	}
 }
 
-bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state)
+void dcn401_populate_mcm_luts(struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		struct dc_cm2_func_luts mcm_luts,
+		bool lut_bank_a)
 {
-	struct dc *dc = pipe_ctx->plane_res.hubp->ctx->dc;
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	struct hubp *hubp = pipe_ctx->plane_res.hubp;
-	const struct dc_plane_cm *cm = &plane_state->cm;
 	int mpcc_id = hubp->inst;
 	struct mpc *mpc = dc->res_pool->mpc;
 	union mcm_lut_params m_lut_params;
-	struct dc_3dlut_dma lut3d_dma;
-	bool lut_enable;
-	bool lut_bank_a;
+	enum dc_cm2_transfer_func_source lut3d_src = mcm_luts.lut3d_data.lut3d_src;
+	enum hubp_3dlut_fl_format format = 0;
+	enum hubp_3dlut_fl_mode mode;
+	enum hubp_3dlut_fl_width width = 0;
+	enum hubp_3dlut_fl_addressing_mode addr_mode;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r = 0;
+	enum MCM_LUT_XABLE shaper_xable = MCM_LUT_DISABLE;
+	enum MCM_LUT_XABLE lut3d_xable = MCM_LUT_DISABLE;
+	enum MCM_LUT_XABLE lut1d_xable = MCM_LUT_DISABLE;
 	bool rval;
-	bool result = true;
 
-	/* decide LUT bank based on current in use */
-	mpc->funcs->get_lut_mode(mpc, MCM_LUT_1DLUT, mpcc_id, &lut_enable, &lut_bank_a);
-	if (!lut_enable) {
-		mpc->funcs->get_lut_mode(mpc, MCM_LUT_SHAPER, mpcc_id, &lut_enable, &lut_bank_a);
-	}
-	if (!lut_enable) {
-		mpc->funcs->get_lut_mode(mpc, MCM_LUT_3DLUT, mpcc_id, &lut_enable, &lut_bank_a);
-	}
-
-	/* switch to the next bank */
-	if (lut_enable) {
-		lut_bank_a = !lut_bank_a;
-	}
-
-	/* MCM location fixed to pre-blend */
-	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
+	dcn401_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable);
 
 	/* 1D LUT */
-	lut_enable = cm->flags.bits.blend_enable;
-	memset(&m_lut_params, 0, sizeof(m_lut_params));
-	if (lut_enable) {
-		if (cm->blend_func.type == TF_TYPE_HWPWL)
-			m_lut_params.pwl = &cm->blend_func.pwl;
-		else if (cm->blend_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
-			rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-					&cm->blend_func,
-					&dpp_base->regamma_params,
-					false);
+	if (mcm_luts.lut1d_func) {
+		memset(&m_lut_params, 0, sizeof(m_lut_params));
+		if (mcm_luts.lut1d_func->type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &mcm_luts.lut1d_func->pwl;
+		else if (mcm_luts.lut1d_func->type == TF_TYPE_DISTRIBUTED_POINTS) {
+			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
+					mcm_luts.lut1d_func,
+					&dpp_base->regamma_params, false);
 			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
 		}
-
-		if (!m_lut_params.pwl) {
-			lut_enable = false;
+		if (m_lut_params.pwl) {
+			if (mpc->funcs->populate_lut)
+				mpc->funcs->populate_lut(mpc, MCM_LUT_1DLUT, m_lut_params, lut_bank_a, mpcc_id);
 		}
-	} else {
-		lut_enable = false;
+		if (mpc->funcs->program_lut_mode)
+			mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, lut1d_xable && m_lut_params.pwl, lut_bank_a, mpcc_id);
 	}
 
-	if (mpc->funcs->program_lut_mode)
-		mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, lut_enable, lut_bank_a, CM_LUT_SIZE_NONE, mpcc_id);
-	if (lut_enable && mpc->funcs->populate_lut)
-		mpc->funcs->populate_lut(mpc, MCM_LUT_1DLUT, &m_lut_params, lut_bank_a, mpcc_id);
-
 	/* Shaper */
-	lut_enable = cm->flags.bits.shaper_enable;
-	if (lut_enable) {
+	if (mcm_luts.shaper && mcm_luts.lut3d_data.mpc_3dlut_enable) {
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
-		if (cm->shaper_func.type == TF_TYPE_HWPWL)
-			m_lut_params.pwl = &cm->shaper_func.pwl;
-		else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		if (mcm_luts.shaper->type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &mcm_luts.shaper->pwl;
+		else if (mcm_luts.shaper->type == TF_TYPE_DISTRIBUTED_POINTS) {
 			ASSERT(false);
-			rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
-					&cm->shaper_func,
-					&dpp_base->shaper_params,
-					true);
-			m_lut_params.pwl = rval ? &dpp_base->shaper_params : NULL;
+			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
+					mcm_luts.shaper,
+					&dpp_base->regamma_params, true);
+			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
 		}
-		if (!m_lut_params.pwl) {
-			lut_enable = false;
+		if (m_lut_params.pwl) {
+			if (mpc->funcs->mcm.populate_lut)
+				mpc->funcs->mcm.populate_lut(mpc, m_lut_params, lut_bank_a, mpcc_id);
+			if (mpc->funcs->program_lut_mode)
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_ENABLE, lut_bank_a, mpcc_id);
 		}
-	} else {
-		lut_enable = false;
 	}
 
-	if (mpc->funcs->program_lut_mode)
-		mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, lut_enable, lut_bank_a, CM_LUT_SIZE_NONE, mpcc_id);
-	if (lut_enable && mpc->funcs->populate_lut)
-		mpc->funcs->populate_lut(mpc, MCM_LUT_SHAPER, &m_lut_params, lut_bank_a, mpcc_id);
+	/* 3DLUT */
+	switch (lut3d_src) {
+	case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM:
+		memset(&m_lut_params, 0, sizeof(m_lut_params));
+		if (hubp->funcs->hubp_enable_3dlut_fl)
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
 
-	/* NOTE: Toggling from DMA->Host is not supported atomically as hardware
-	 * blocks writes until 3DLUT FL mode is cleared from HUBP on VUpdate.
-	 * Expectation is either option is used consistently.
-	 */
+		if (mcm_luts.lut3d_data.lut3d_func && mcm_luts.lut3d_data.lut3d_func->state.bits.initialized) {
+			m_lut_params.lut3d = &mcm_luts.lut3d_data.lut3d_func->lut_3d;
+			if (mpc->funcs->populate_lut)
+				mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, m_lut_params, lut_bank_a, mpcc_id);
+			if (mpc->funcs->program_lut_mode)
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a,
+						mpcc_id);
+		}
+		break;
+	case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM:
+		switch (mcm_luts.lut3d_data.gpu_mem_params.size) {
+		case DC_CM2_GPU_MEM_SIZE_333333:
+			if (dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33)
+				width = hubp_3dlut_fl_width_33;
+			break;
+		case DC_CM2_GPU_MEM_SIZE_171717:
+			width = hubp_3dlut_fl_width_17;
+			break;
+		case DC_CM2_GPU_MEM_SIZE_TRANSFORMED:
+			width = hubp_3dlut_fl_width_transformed;
+			break;
+		default:
+			//TODO: handle default case
+			break;
+		}
 
-	/* 3DLUT */
-	lut_enable = cm->flags.bits.lut3d_enable;
-	if (lut_enable && cm->flags.bits.lut3d_dma_enable) {
-		/* Fast (DMA) Load Mode */
-		/* MPC */
-		if (mpc->funcs->program_lut_mode)
-			mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut_enable, lut_bank_a, cm->lut3d_dma.size, mpcc_id);
+		//check for support
+		if (mpc->funcs->mcm.is_config_supported &&
+			!mpc->funcs->mcm.is_config_supported(width))
+			break;
 
-		/* only supports 12 bit */
 		if (mpc->funcs->program_lut_read_write_control)
-			mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, 12, mpcc_id);
+			mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, mpcc_id);
+		if (mpc->funcs->program_lut_mode)
+			mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a, mpcc_id);
 
-		if (mpc->funcs->update_3dlut_fast_load_select)
-			mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, hubp->inst);
+		if (hubp->funcs->hubp_program_3dlut_fl_addr)
+			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, mcm_luts.lut3d_data.gpu_mem_params.addr);
 
-		/* HUBP */
-		if (hubp->funcs->hubp_program_3dlut_fl_config)
-			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &cm->lut3d_dma);
+		if (mpc->funcs->mcm.program_bit_depth)
+			mpc->funcs->mcm.program_bit_depth(mpc, mcm_luts.lut3d_data.gpu_mem_params.bit_depth, mpcc_id);
 
-		if (hubp->funcs->hubp_program_3dlut_fl_crossbar)
-			hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp, cm->lut3d_dma.format);
+		switch (mcm_luts.lut3d_data.gpu_mem_params.layout) {
+		case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB:
+			mode = hubp_3dlut_fl_mode_native_1;
+			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+			break;
+		case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR:
+			mode = hubp_3dlut_fl_mode_native_2;
+			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+			break;
+		case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR:
+			mode = hubp_3dlut_fl_mode_transform;
+			addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
+			break;
+		default:
+			mode = hubp_3dlut_fl_mode_disable;
+			addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+			break;
+		}
+		if (hubp->funcs->hubp_program_3dlut_fl_mode)
+			hubp->funcs->hubp_program_3dlut_fl_mode(hubp, mode);
 
-		if (hubp->funcs->hubp_program_3dlut_fl_addr)
-			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, &cm->lut3d_dma.addr);
+		if (hubp->funcs->hubp_program_3dlut_fl_addressing_mode)
+			hubp->funcs->hubp_program_3dlut_fl_addressing_mode(hubp, addr_mode);
 
-		if (hubp->funcs->hubp_enable_3dlut_fl) {
-			hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
-		} else {
-			/* GPU memory only supports fast load path */
-			BREAK_TO_DEBUGGER();
-			lut_enable = false;
-			result = false;
+		switch (mcm_luts.lut3d_data.gpu_mem_params.format_params.format) {
+		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB:
+			format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
+			break;
+		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB:
+			format = hubp_3dlut_fl_format_unorm_12lsb_bitslice;
+			break;
+		case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10:
+			format = hubp_3dlut_fl_format_float_fp1_5_10;
+			break;
+		}
+		if (hubp->funcs->hubp_program_3dlut_fl_format)
+			hubp->funcs->hubp_program_3dlut_fl_format(hubp, format);
+		if (hubp->funcs->hubp_update_3dlut_fl_bias_scale &&
+				mpc->funcs->mcm.program_bias_scale) {
+			mpc->funcs->mcm.program_bias_scale(mpc,
+				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
+				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale,
+				mpcc_id);
+			hubp->funcs->hubp_update_3dlut_fl_bias_scale(hubp,
+						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
+						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale);
 		}
-	} else {
-		/* Legacy (Host) Load Mode */
-		memset(&m_lut_params, 0, sizeof(m_lut_params));
 
-		if (cm->flags.bits.lut3d_enable && cm->lut3d_func.state.bits.initialized) {
-			m_lut_params.lut3d = &cm->lut3d_func.lut_3d;
-		} else {
-			lut_enable = false;
+		//navi 4x has a bug and r and blue are swapped and need to be worked around here in
+		//TODO: need to make a method for get_xbar per asic OR do the workaround in program_crossbar for 4x
+		switch (mcm_luts.lut3d_data.gpu_mem_params.component_order) {
+		case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA:
+		default:
+			crossbar_bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
+			crossbar_bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+			crossbar_bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
+			break;
 		}
 
-		/* MPC */
-		if (mpc->funcs->program_lut_mode)
-			mpc->funcs->program_lut_mode(mpc,
-					MCM_LUT_3DLUT,
-					lut_enable,
-					lut_bank_a,
-					cm->lut3d_func.lut_3d.use_tetrahedral_9 ? CM_LUT_SIZE_999 : CM_LUT_SIZE_171717,
-					mpcc_id);
-
-		if (lut_enable) {
-			if (mpc->funcs->program_lut_read_write_control)
-				mpc->funcs->program_lut_read_write_control(mpc,
-						MCM_LUT_3DLUT,
-						lut_bank_a,
-						cm->lut3d_func.lut_3d.use_12bits ? 12 : 10,
-						mpcc_id);
+		if (hubp->funcs->hubp_program_3dlut_fl_crossbar)
+			hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp,
+					crossbar_bit_slice_cr_r,
+					crossbar_bit_slice_y_g,
+					crossbar_bit_slice_cb_b);
 
-			if (mpc->funcs->update_3dlut_fast_load_select)
-				mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, 0xf);
+		if (mpc->funcs->mcm.program_lut_read_write_control)
+			mpc->funcs->mcm.program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, true, mpcc_id);
 
-			if (mpc->funcs->populate_lut)
-				mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, &m_lut_params, lut_bank_a, mpcc_id);
-		}
+		if (mpc->funcs->mcm.program_3dlut_size)
+			mpc->funcs->mcm.program_3dlut_size(mpc, width, mpcc_id);
 
-		/* HUBP */
-		memset(&lut3d_dma, 0, sizeof(lut3d_dma));
-		if (hubp->funcs->hubp_program_3dlut_fl_config)
-			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &lut3d_dma);
+		if (mpc->funcs->update_3dlut_fast_load_select)
+			mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, hubp->inst);
 
 		if (hubp->funcs->hubp_enable_3dlut_fl)
-			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
+		else {
+			if (mpc->funcs->program_lut_mode) {
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
+			}
+		}
+		break;
+
+	}
+}
+
+void dcn401_trigger_3dlut_dma_load(struct dc *dc, struct pipe_ctx *pipe_ctx)
+{
+	(void)dc;
+	struct hubp *hubp = pipe_ctx->plane_res.hubp;
+
+	if (hubp->funcs->hubp_enable_3dlut_fl) {
+		hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
+	}
+}
+
+bool dcn401_set_mcm_luts(struct pipe_ctx *pipe_ctx,
+				const struct dc_plane_state *plane_state)
+{
+	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
+	struct dc *dc = pipe_ctx->stream_res.opp->ctx->dc;
+	struct mpc *mpc = dc->res_pool->mpc;
+	bool result;
+	const struct pwl_params *lut_params = NULL;
+	bool rval;
+
+	if (plane_state->mcm_luts.lut3d_data.lut3d_src == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
+		dcn401_populate_mcm_luts(dc, pipe_ctx, plane_state->mcm_luts, plane_state->lut_bank_a);
+		return true;
+	}
+
+	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
+	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
+	// 1D LUT
+	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->blend_tf.pwl;
+	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
+							       &plane_state->blend_tf,
+							       &dpp_base->regamma_params, false);
+		lut_params = rval ? &dpp_base->regamma_params : NULL;
+	}
+	result = mpc->funcs->program_1dlut(mpc, lut_params, mpcc_id);
+	lut_params = NULL;
+
+	// Shaper
+	if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->in_shaper_func.pwl;
+	else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		// TODO: dpp_base replace
+		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
+							       &plane_state->in_shaper_func,
+							       &dpp_base->shaper_params, true);
+		lut_params = rval ? &dpp_base->shaper_params : NULL;
+	}
+	result &= mpc->funcs->program_shaper(mpc, lut_params, mpcc_id);
+
+	// 3D
+	if (mpc->funcs->program_3dlut) {
+		if (plane_state->lut3d_func.state.bits.initialized == 1)
+			result &= mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func.lut_3d, mpcc_id);
+		else
+			result &= mpc->funcs->program_3dlut(mpc, NULL, mpcc_id);
 	}
 
 	return result;
@@ -1157,8 +1279,8 @@ static bool dcn401_check_no_memory_request_for_cab(struct dc *dc)
 
 static uint32_t dcn401_calculate_cab_allocation(struct dc *dc, struct dc_state *ctx)
 {
-	int i;
-	uint8_t num_ways = 0;
+	unsigned int i;
+	uint32_t num_ways = 0;
 	uint32_t mall_ss_size_bytes = 0;
 
 	mall_ss_size_bytes = ctx->bw_ctx.bw.dcn.mall_ss_size_bytes;
@@ -1189,7 +1311,8 @@ static uint32_t dcn401_calculate_cab_allocation(struct dc *dc, struct dc_state *
 bool dcn401_apply_idle_power_optimizations(struct dc *dc, bool enable)
 {
 	union dmub_rb_cmd cmd;
-	uint8_t ways, i;
+	uint32_t ways;
+	uint8_t i;
 	int j;
 	bool mall_ss_unsupported = false;
 	struct dc_plane_state *plane = NULL;
@@ -1242,7 +1365,7 @@ bool dcn401_apply_idle_power_optimizations(struct dc *dc, bool enable)
 			}
 			if (ways <= dc->caps.cache_num_ways && !mall_ss_unsupported) {
 				cmd.cab.header.sub_type = DMUB_CMD__CAB_DCN_SS_FIT_IN_CAB;
-				cmd.cab.cab_alloc_ways = ways;
+				cmd.cab.cab_alloc_ways = (uint8_t)ways;
 				DC_LOG_MALL("cab allocation: %d ways. CAB action: DCN_SS_FIT_IN_CAB\n", ways);
 			} else {
 				cmd.cab.header.sub_type = DMUB_CMD__CAB_DCN_SS_NOT_FIT_IN_CAB;
@@ -1298,10 +1421,13 @@ void dcn401_prepare_bandwidth(struct dc *dc,
 		context->bw_ctx.bw.dcn.clk.p_state_change_support = false;
 	}
 
-	if (dc->clk_mgr->dc_mode_softmax_enabled)
-		if (dc->clk_mgr->clks.dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000 &&
-				context->bw_ctx.bw.dcn.clk.dramclk_khz > dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000)
+	if (dc->clk_mgr->dc_mode_softmax_enabled) {
+		int softmax_memclk_khz = dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000;
+
+		if (dc->clk_mgr->clks.dramclk_khz <= softmax_memclk_khz &&
+				context->bw_ctx.bw.dcn.clk.dramclk_khz > softmax_memclk_khz)
 			dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, dc->clk_mgr->bw_params->clk_table.entries[dc->clk_mgr->bw_params->clk_table.num_entries - 1].memclk_mhz);
+	}
 
 	/* Increase clocks */
 	dc->clk_mgr->funcs->update_clocks(
@@ -1347,7 +1473,7 @@ void dcn401_optimize_bandwidth(
 		struct dc *dc,
 		struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	struct hubbub *hubbub = dc->res_pool->hubbub;
 
 	/* enable fams2 if needed */
@@ -1367,10 +1493,13 @@ void dcn401_optimize_bandwidth(
 		hubbub->funcs->program_arbiter(hubbub, &context->bw_ctx.bw.dcn.arb_regs, true);
 	}
 
-	if (dc->clk_mgr->dc_mode_softmax_enabled)
-		if (dc->clk_mgr->clks.dramclk_khz > dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000 &&
-				context->bw_ctx.bw.dcn.clk.dramclk_khz <= dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000)
+	if (dc->clk_mgr->dc_mode_softmax_enabled) {
+		int softmax_memclk_khz = dc->clk_mgr->bw_params->dc_mode_softmax_memclk * 1000;
+
+		if (dc->clk_mgr->clks.dramclk_khz > softmax_memclk_khz &&
+				context->bw_ctx.bw.dcn.clk.dramclk_khz <= softmax_memclk_khz)
 			dc->clk_mgr->funcs->set_max_memclk(dc->clk_mgr, dc->clk_mgr->bw_params->dc_mode_softmax_memclk);
+	}
 
 	/* increase compbuf size */
 	if (hubbub->funcs->program_compbuf_segments)
@@ -1433,12 +1562,15 @@ void dcn401_dmub_hw_control_lock_fast(union block_sequence_params *params)
 void dcn401_fams2_update_config(struct dc *dc, struct dc_state *context, bool enable)
 {
 	bool fams2_info_required;
+	bool fams2_enabled;
+	bool fams2_legacy_no_fams2;
 
 	if (!dc->ctx || !dc->ctx->dmub_srv || !dc->debug.fams2_config.bits.enable)
 		return;
 
-	fams2_info_required = context->bw_ctx.bw.dcn.fams2_global_config.features.bits.enable;
-	fams2_info_required |= context->bw_ctx.bw.dcn.fams2_global_config.features.bits.legacy_method_no_fams2;
+	fams2_enabled = context->bw_ctx.bw.dcn.fams2_global_config.features.bits.enable != 0u;
+	fams2_legacy_no_fams2 = context->bw_ctx.bw.dcn.fams2_global_config.features.bits.legacy_method_no_fams2 != 0u;
+	fams2_info_required = fams2_enabled || fams2_legacy_no_fams2;
 
 	dc_dmub_srv_fams2_update_config(dc, context, enable && fams2_info_required);
 }
@@ -1470,7 +1602,7 @@ static void update_dsc_for_odm_change(struct dc *dc, struct dc_state *context,
 
 	if (otg_master->stream_res.dsc)
 		dcn32_update_dsc_on_stream(otg_master,
-				otg_master->stream->timing.flags.DSC);
+				otg_master->stream->timing.flags.DSC != 0u);
 	if (old_otg_master && old_otg_master->stream_res.dsc) {
 		for (i = 0; i < old_opp_head_count; i++) {
 			old_pipe = old_opp_heads[i];
@@ -1818,15 +1950,19 @@ void dcn401_perform_3dlut_wa_unlock(struct pipe_ctx *pipe_ctx)
 	 * This is meant to work around a known HW issue where VREADY will cancel the pending 3DLUT_ENABLE signal regardless
 	 * of whether OTG lock is currently being held or not.
 	 */
+	if (!pipe_ctx)
+		return;
+
 	struct pipe_ctx *wa_pipes[MAX_PIPES] = { NULL };
 	struct pipe_ctx *odm_pipe, *mpc_pipe;
 	int i, wa_pipe_ct = 0;
 
 	for (odm_pipe = pipe_ctx; odm_pipe != NULL; odm_pipe = odm_pipe->next_odm_pipe) {
 		for (mpc_pipe = odm_pipe; mpc_pipe != NULL; mpc_pipe = mpc_pipe->bottom_pipe) {
-			if (mpc_pipe->plane_state &&
-					mpc_pipe->plane_state->cm.flags.bits.lut3d_enable &&
-					mpc_pipe->plane_state->cm.flags.bits.lut3d_dma_enable) {
+			if (mpc_pipe->plane_state && mpc_pipe->plane_state->mcm_luts.lut3d_data.lut3d_src
+						== DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM
+					&& mpc_pipe->plane_state->mcm_shaper_3dlut_setting
+						== DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT) {
 				wa_pipes[wa_pipe_ct++] = mpc_pipe;
 			}
 		}
@@ -2337,7 +2473,7 @@ void dcn401_program_front_end_for_ctx(
 	struct dc *dc,
 	struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	unsigned int prev_hubp_count = 0;
 	unsigned int hubp_count = 0;
 	struct dce_hwseq *hws = dc->hwseq;
@@ -2504,7 +2640,7 @@ void dcn401_post_unlock_program_front_end(
 	unsigned int timeout_us = 100000;
 	unsigned int polling_interval_us = 1;
 	struct dce_hwseq *hwseq = dc->hwseq;
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < dc->res_pool->pipe_count; i++)
 		if (resource_is_pipe_type(&dc->current_state->res_ctx.pipe_ctx[i], OPP_HEAD) &&
@@ -2528,7 +2664,7 @@ void dcn401_post_unlock_program_front_end(
 		if (pipe->plane_state && !pipe->top_pipe && pipe->update_flags.bits.enable &&
 			dc_state_get_pipe_subvp_type(context, pipe) != SUBVP_PHANTOM) {
 			struct hubp *hubp = pipe->plane_res.hubp;
-			int j = 0;
+			unsigned int j = 0;
 
 			for (j = 0; j < timeout_us / polling_interval_us
 				&& hubp->funcs->hubp_is_flip_pending(hubp); j++)
@@ -2547,7 +2683,7 @@ void dcn401_post_unlock_program_front_end(
 		if (resource_is_pipe_type(old_pipe, OTG_MASTER) && resource_is_pipe_type(pipe, OTG_MASTER) &&
 			resource_get_odm_slice_count(old_pipe) < resource_get_odm_slice_count(pipe) &&
 			dc_state_get_pipe_subvp_type(context, pipe) != SUBVP_PHANTOM) {
-			int j = 0;
+			unsigned int j = 0;
 			struct timing_generator *tg = pipe->stream_res.tg;
 
 			if (tg->funcs->get_optc_double_buffer_pending) {
@@ -2632,7 +2768,7 @@ bool dcn401_update_bandwidth(
 	struct dc *dc,
 	struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	struct dce_hwseq *hws = dc->hwseq;
 
 	/* recalculate DML parameters */
@@ -3133,9 +3269,10 @@ void dcn401_program_all_writeback_pipes_in_tree_sequence(
 		struct block_sequence_state *seq_state)
 {
 	struct dwbc *dwb;
-	int i_wb, i_pipe;
+	unsigned int i_wb, i_pipe;
+	unsigned int num_dwb_cap = (unsigned int)dc->res_pool->res_cap->num_dwb;
 
-	if (!stream || stream->num_wb_info > dc->res_pool->res_cap->num_dwb)
+	if (!stream || (unsigned int)stream->num_wb_info > num_dwb_cap)
 		return;
 
 	/* For each writeback pipe */
@@ -3297,7 +3434,7 @@ void dcn401_setup_gsl_group_as_lock_sequence(
 
 		group_idx = find_free_gsl_group(dc);
 		ASSERT(group_idx != 0);
-		pipe_ctx->stream_res.gsl_group = group_idx;
+		pipe_ctx->stream_res.gsl_group = (uint8_t)group_idx;
 
 		/* set gsl group reg field and mark resource used */
 		switch (group_idx) {
@@ -3783,7 +3920,7 @@ void dcn401_update_mpcc_sequence(struct dc *dc,
 
 static struct hubp *get_hubp_by_inst(struct resource_pool *res_pool, int mpcc_inst)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < res_pool->pipe_count; i++) {
 		if (res_pool->hubps[i]->inst == mpcc_inst)
@@ -3858,7 +3995,7 @@ void dcn401_set_hdr_multiplier_sequence(struct pipe_ctx *pipe_ctx,
 void dcn401_program_mall_pipe_config_sequence(struct dc *dc, struct dc_state *context,
 		struct block_sequence_state *seq_state)
 {
-	int i;
+	unsigned int i;
 	unsigned int num_ways = dcn401_calculate_cab_allocation(dc, context);
 	bool cache_cursor = false;
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c
index 5d0dfb36f3e1b..0908a791832ba 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_init.c
@@ -167,6 +167,8 @@ static const struct hwseq_private_funcs dcn401_private_funcs = {
 	.perform_3dlut_wa_unlock = dcn401_perform_3dlut_wa_unlock,
 	.program_pipe_sequence = dcn401_program_pipe_sequence,
 	.dc_ip_request_cntl = dcn401_dc_ip_request_cntl,
+	.wait_for_pipe_update_if_needed = dcn10_wait_for_pipe_update_if_needed,
+	.set_wait_for_update_needed_for_pipe = dcn10_set_wait_for_update_needed_for_pipe,
 };
 
 void dcn401_hw_sequencer_init_functions(struct dc *dc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
index 46f2f9833d9e5..cabfac151940f 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.c
@@ -35,7 +35,6 @@
 #include "dc_stream_priv.h"
 #include "dcn35/dcn35_hwseq.h"
 #include "dcn42/dcn42_hwseq.h"
-#include "dce/dmub_hw_lock_mgr.h"
 #include "dio/dcn10/dcn10_dio.h"
 
 #define DC_LOGGER \
@@ -65,7 +64,7 @@ void dcn42_init_hw(struct dc *dc)
 	struct dce_hwseq *hws = dc->hwseq;
 	struct dc_bios *dcb = dc->ctx->dc_bios;
 	struct resource_pool *res_pool = dc->res_pool;
-	int i;
+	unsigned int i;
 	unsigned int edp_num;
 	uint32_t backlight = MAX_BACKLIGHT_LEVEL;
 	uint32_t user_level = MAX_BACKLIGHT_LEVEL;
@@ -394,6 +393,68 @@ void dcn42_program_cm_hist(
 			plane_state->cm_hist_control, plane_state->color_space);
 }
 
+static void dc_get_lut_xbar(
+	enum dc_cm2_gpu_mem_pixel_component_order order,
+	enum hubp_3dlut_fl_crossbar_bit_slice *cr_r,
+	enum hubp_3dlut_fl_crossbar_bit_slice *y_g,
+	enum hubp_3dlut_fl_crossbar_bit_slice *cb_b)
+{
+	switch (order) {
+	case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_RGBA:
+		*cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47;
+		*y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+		*cb_b =  hubp_3dlut_fl_crossbar_bit_slice_0_15;
+		break;
+	case DC_CM2_GPU_MEM_PIXEL_COMPONENT_ORDER_BGRA:
+		*cr_r = hubp_3dlut_fl_crossbar_bit_slice_0_15;
+		*y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+		*cb_b = hubp_3dlut_fl_crossbar_bit_slice_32_47;
+		break;
+	}
+}
+
+static void dc_get_lut_mode(
+	enum dc_cm2_gpu_mem_layout layout,
+	enum hubp_3dlut_fl_mode *mode,
+	enum hubp_3dlut_fl_addressing_mode *addr_mode)
+{
+	switch (layout) {
+	case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB:
+		*mode = hubp_3dlut_fl_mode_native_1;
+		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+		break;
+	case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR:
+		*mode = hubp_3dlut_fl_mode_native_2;
+		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+		break;
+	case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR:
+		*mode = hubp_3dlut_fl_mode_transform;
+		*addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
+		break;
+	default:
+		*mode = hubp_3dlut_fl_mode_disable;
+		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+		break;
+	}
+}
+
+static void dc_get_lut_format(
+	enum dc_cm2_gpu_mem_format dc_format,
+	enum hubp_3dlut_fl_format *format)
+{
+	switch (dc_format) {
+	case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB:
+		*format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
+		break;
+	case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB:
+		*format = hubp_3dlut_fl_format_unorm_12lsb_bitslice;
+		break;
+	case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10:
+		*format = hubp_3dlut_fl_format_float_fp1_5_10;
+		break;
+	}
+}
+
 static bool dc_is_rmcm_3dlut_supported(struct hubp *hubp, struct mpc *mpc)
 {
 	if (mpc->funcs->rmcm.power_on_shaper_3dlut &&
@@ -404,17 +465,119 @@ static bool dc_is_rmcm_3dlut_supported(struct hubp *hubp, struct mpc *mpc)
 	return false;
 }
 
+static bool is_rmcm_3dlut_fl_supported(struct dc *dc, enum dc_cm2_gpu_mem_size size)
+{
+	if (!dc->caps.color.mpc.rmcm_3d_lut_caps.dma_3d_lut)
+		return false;
+	if (size == DC_CM2_GPU_MEM_SIZE_171717)
+		return dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_17 != 0u;
+	else if (size == DC_CM2_GPU_MEM_SIZE_333333)
+		return dc->caps.color.mpc.rmcm_3d_lut_caps.lut_dim_caps.dim_33 != 0u;
+	return false;
+}
+
+static void dcn42_set_mcm_location_post_blend(struct dc *dc, struct pipe_ctx *pipe_ctx, bool bPostBlend)
+{
+	struct mpc *mpc = dc->res_pool->mpc;
+	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
+
+	if (!pipe_ctx->plane_state)
+		return;
+
+	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
+	pipe_ctx->plane_state->mcm_location = (bPostBlend) ?
+											MPCC_MOVABLE_CM_LOCATION_AFTER :
+											MPCC_MOVABLE_CM_LOCATION_BEFORE;
+}
+
+static void dcn42_get_mcm_lut_xable_from_pipe_ctx(struct dc *dc, struct pipe_ctx *pipe_ctx,
+		enum MCM_LUT_XABLE *shaper_xable,
+		enum MCM_LUT_XABLE *lut3d_xable,
+		enum MCM_LUT_XABLE *lut1d_xable)
+{
+	enum dc_cm2_shaper_3dlut_setting shaper_3dlut_setting = DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL;
+	bool lut1d_enable = false;
+	struct mpc *mpc = dc->res_pool->mpc;
+	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
+
+	if (!pipe_ctx->plane_state)
+		return;
+	shaper_3dlut_setting = pipe_ctx->plane_state->mcm_shaper_3dlut_setting;
+	lut1d_enable = pipe_ctx->plane_state->mcm_lut1d_enable;
+	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
+	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
+
+	*lut1d_xable = lut1d_enable ? MCM_LUT_ENABLE : MCM_LUT_DISABLE;
+
+	switch (shaper_3dlut_setting) {
+	case DC_CM2_SHAPER_3DLUT_SETTING_BYPASS_ALL:
+		*lut3d_xable = *shaper_xable = MCM_LUT_DISABLE;
+		break;
+	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER:
+		*lut3d_xable = MCM_LUT_DISABLE;
+		*shaper_xable = MCM_LUT_ENABLE;
+		break;
+	case DC_CM2_SHAPER_3DLUT_SETTING_ENABLE_SHAPER_3DLUT:
+		*lut3d_xable = *shaper_xable = MCM_LUT_ENABLE;
+		break;
+	}
+}
+
+static void fl_get_lut_mode(
+	enum dc_cm2_gpu_mem_layout layout,
+	enum dc_cm2_gpu_mem_size   size,
+	enum hubp_3dlut_fl_mode *mode,
+	enum hubp_3dlut_fl_addressing_mode *addr_mode,
+	enum hubp_3dlut_fl_width *width)
+{
+	*width = hubp_3dlut_fl_width_17;
+
+	if (size == DC_CM2_GPU_MEM_SIZE_333333)
+		*width = hubp_3dlut_fl_width_33;
+
+	switch (layout) {
+	case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_RGB:
+		*mode = hubp_3dlut_fl_mode_native_1;
+		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+		break;
+	case DC_CM2_GPU_MEM_LAYOUT_3D_SWIZZLE_LINEAR_BGR:
+		*mode = hubp_3dlut_fl_mode_native_2;
+		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+		break;
+	case DC_CM2_GPU_MEM_LAYOUT_1D_PACKED_LINEAR:
+		*mode = hubp_3dlut_fl_mode_transform;
+		*addr_mode = hubp_3dlut_fl_addressing_mode_simple_linear;
+		break;
+	default:
+		*mode = hubp_3dlut_fl_mode_disable;
+		*addr_mode = hubp_3dlut_fl_addressing_mode_sw_linear;
+		break;
+	}
+}
+
 bool dcn42_program_rmcm_luts(
 	struct hubp *hubp,
 	struct pipe_ctx *pipe_ctx,
-	const struct dc_plane_cm *cm,
+	enum dc_cm2_transfer_func_source lut3d_src,
+	struct dc_cm2_func_luts *mcm_luts,
 	struct mpc *mpc,
+	bool lut_bank_a,
 	int mpcc_id)
 {
 	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	union mcm_lut_params m_lut_params = {0};
+	enum MCM_LUT_XABLE shaper_xable, lut3d_xable = MCM_LUT_DISABLE, lut1d_xable;
+		enum hubp_3dlut_fl_mode mode;
+	enum hubp_3dlut_fl_addressing_mode addr_mode;
+	enum hubp_3dlut_fl_format format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g = hubp_3dlut_fl_crossbar_bit_slice_16_31;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b = hubp_3dlut_fl_crossbar_bit_slice_0_15;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r = hubp_3dlut_fl_crossbar_bit_slice_32_47;
+	enum hubp_3dlut_fl_width width = hubp_3dlut_fl_width_17;
+
 
 	struct dc *dc = hubp->ctx->dc;
+	struct hubp_fl_3dlut_config fl_config;
 	struct mpc_fl_3dlut_config mpc_fl_config;
 
 	struct dc_stream_state *stream = pipe_ctx->stream;
@@ -422,23 +585,25 @@ bool dcn42_program_rmcm_luts(
 	// true->false when it can be allocated at DI time
 	struct dc_rmcm_3dlut *rmcm_3dlut = dc_stream_get_3dlut_for_stream(dc, stream, false);
 
-	bool lut_bank_a = true; // TODO get from HW
-
 	//check to see current pipe is part of a stream with allocated rmcm 3dlut
 	if (!rmcm_3dlut)
 		return false;
 
+	rmcm_3dlut->protection_bits = mcm_luts->lut3d_data.rmcm_tmz;
+
+	dcn42_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable);
+
 	/* Shaper */
-	if (cm->flags.bits.shaper_enable) {
+	if (mcm_luts->shaper) {
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
 
-		if (cm->shaper_func.type == TF_TYPE_HWPWL) {
-			m_lut_params.pwl = &cm->shaper_func.pwl;
-		} else if (cm->shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		if (mcm_luts->shaper->type == TF_TYPE_HWPWL) {
+			m_lut_params.pwl = &mcm_luts->shaper->pwl;
+		} else if (mcm_luts->shaper->type == TF_TYPE_DISTRIBUTED_POINTS) {
 			ASSERT(false);
 			cm_helper_translate_curve_to_hw_format(
 					dc->ctx,
-					&cm->shaper_func,
+					mcm_luts->shaper,
 					&dpp_base->shaper_params, true);
 			m_lut_params.pwl = &dpp_base->shaper_params;
 		}
@@ -454,21 +619,58 @@ bool dcn42_program_rmcm_luts(
 	}
 
 	/* 3DLUT */
-	if (!cm->flags.bits.lut3d_dma_enable) {
+	switch (lut3d_src) {
+	case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM:
 		memset(&m_lut_params, 0, sizeof(m_lut_params));
 		// Don't know what to do in this case.
-	} else {
-		if (!dc_is_rmcm_3dlut_supported(hubp, mpc))
+		//case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM:
+		break;
+	case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM:
+		fl_get_lut_mode(mcm_luts->lut3d_data.gpu_mem_params.layout,
+				mcm_luts->lut3d_data.gpu_mem_params.size,
+				&mode,
+				&addr_mode,
+				&width);
+
+		if (!dc_is_rmcm_3dlut_supported(hubp, mpc) ||
+			!mpc->funcs->rmcm.is_config_supported(
+				(width == hubp_3dlut_fl_width_17 ||
+				 width == hubp_3dlut_fl_width_transformed) ? 17 : 33))
 			return false;
 
+		// setting native or transformed mode,
+		dc_get_lut_mode(mcm_luts->lut3d_data.gpu_mem_params.layout, &mode, &addr_mode);
+
 		//seems to be only for the MCM
-		mpc_fl_config.enabled			= cm->flags.bits.lut3d_enable;
-		mpc_fl_config.size	            = cm->lut3d_dma.size;
+		dc_get_lut_format(mcm_luts->lut3d_data.gpu_mem_params.format_params.format, &format);
+
+		dc_get_lut_xbar(
+			mcm_luts->lut3d_data.gpu_mem_params.component_order,
+			&crossbar_bit_slice_cr_r,
+			&crossbar_bit_slice_y_g,
+			&crossbar_bit_slice_cb_b);
+
+		fl_config.mode					= mode;
+		fl_config.enabled				= lut3d_xable != MCM_LUT_DISABLE;
+		fl_config.address				= mcm_luts->lut3d_data.gpu_mem_params.addr;
+		fl_config.format				= format;
+		fl_config.crossbar_bit_slice_y_g  = crossbar_bit_slice_y_g;
+		fl_config.crossbar_bit_slice_cb_b = crossbar_bit_slice_cb_b;
+		fl_config.crossbar_bit_slice_cr_r = crossbar_bit_slice_cr_r;
+		fl_config.width				    = width;
+		fl_config.protection_bits		= rmcm_3dlut->protection_bits;
+		fl_config.addr_mode			    = addr_mode;
+		fl_config.layout                = mcm_luts->lut3d_data.gpu_mem_params.layout;
+		fl_config.bias	= mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.bias;
+		fl_config.scale	= mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.scale;
+
+		mpc_fl_config.enabled			= fl_config.enabled;
+		mpc_fl_config.width	            = width;
 		mpc_fl_config.select_lut_bank_a = lut_bank_a;
-		mpc_fl_config.bit_depth		    = 0;
+		mpc_fl_config.bit_depth		    = mcm_luts->lut3d_data.gpu_mem_params.bit_depth;
 		mpc_fl_config.hubp_index		= hubp->inst;
-		mpc_fl_config.bias	= cm->lut3d_dma.bias;
-		mpc_fl_config.scale	= cm->lut3d_dma.scale;
+		mpc_fl_config.bias	= mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.bias;
+		mpc_fl_config.scale	= mcm_luts->lut3d_data.gpu_mem_params.format_params.float_params.scale;
 
 		//1. power down the block
 		mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, false);
@@ -476,49 +678,274 @@ bool dcn42_program_rmcm_luts(
 		//2. program RMCM - 3dlut reg programming
 		mpc->funcs->rmcm.fl_3dlut_configure(mpc, &mpc_fl_config, mpcc_id);
 
-		/* HUBP */
-		if (hubp->funcs->hubp_program_3dlut_fl_config)
-			hubp->funcs->hubp_program_3dlut_fl_config(hubp, &cm->lut3d_dma);
-
-		if (hubp->funcs->hubp_program_3dlut_fl_addr)
-			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, &cm->lut3d_dma.addr);
+		hubp->funcs->hubp_program_3dlut_fl_config(hubp, &fl_config);
 
 		//3. power on the block
 		mpc->funcs->rmcm.power_on_shaper_3dlut(mpc, mpcc_id, true);
+
+		break;
+	default:
+		return false;
 	}
 
 	return true;
 }
 
-bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
-				const struct dc_plane_state *plane_state)
+void dcn42_populate_mcm_luts(struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		struct dc_cm2_func_luts mcm_luts,
+		bool lut_bank_a)
 {
-	struct dc *dc = pipe_ctx->plane_res.hubp->ctx->dc;
+	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
 	struct hubp *hubp = pipe_ctx->plane_res.hubp;
-	const struct dc_plane_cm *cm = &plane_state->cm;
-	struct mpc *mpc = dc->res_pool->mpc;
 	int mpcc_id = hubp->inst;
-	bool result;
-
-	/* MCM */
-	result = dcn401_set_mcm_luts(pipe_ctx, plane_state);
-
-	/* RMCM */
-	if (cm->flags.bits.rmcm_enable && cm->flags.bits.lut3d_dma_enable) {
-		/* TODO - move RMCM to its own block */
+	struct mpc *mpc = dc->res_pool->mpc;
+	union mcm_lut_params m_lut_params;
+	enum dc_cm2_transfer_func_source lut3d_src = mcm_luts.lut3d_data.lut3d_src;
+	enum hubp_3dlut_fl_format format = 0;
+	enum hubp_3dlut_fl_mode mode;
+	enum hubp_3dlut_fl_width width = 0;
+	enum hubp_3dlut_fl_addressing_mode addr_mode;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b = 0;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r = 0;
+	enum MCM_LUT_XABLE shaper_xable = MCM_LUT_DISABLE;
+	enum MCM_LUT_XABLE lut3d_xable = MCM_LUT_DISABLE;
+	enum MCM_LUT_XABLE lut1d_xable = MCM_LUT_DISABLE;
+	bool rval;
+
+	dcn42_get_mcm_lut_xable_from_pipe_ctx(dc, pipe_ctx, &shaper_xable, &lut3d_xable, &lut1d_xable);
+
+	//MCM - setting its location (Before/After) blender
+	//set to post blend (true)
+	dcn42_set_mcm_location_post_blend(
+		dc,
+		pipe_ctx,
+		mcm_luts.lut3d_data.mpc_mcm_post_blend);
+
+	//RMCM - 3dLUT+Shaper
+	if (mcm_luts.lut3d_data.rmcm_3dlut_enable &&
+		is_rmcm_3dlut_fl_supported(dc, mcm_luts.lut3d_data.gpu_mem_params.size)) {
 		dcn42_program_rmcm_luts(
 			hubp,
 			pipe_ctx,
-			cm,
+			lut3d_src,
+			&mcm_luts,
 			mpc,
+			lut_bank_a,
 			mpcc_id);
 	}
 
+	/* 1D LUT */
+	if (mcm_luts.lut1d_func) {
+		memset(&m_lut_params, 0, sizeof(m_lut_params));
+		if (mcm_luts.lut1d_func->type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &mcm_luts.lut1d_func->pwl;
+		else if (mcm_luts.lut1d_func->type == TF_TYPE_DISTRIBUTED_POINTS) {
+			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
+					mcm_luts.lut1d_func,
+					&dpp_base->regamma_params, false);
+			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
+		}
+		if (m_lut_params.pwl) {
+			if (mpc->funcs->populate_lut)
+				mpc->funcs->populate_lut(mpc, MCM_LUT_1DLUT, m_lut_params, lut_bank_a, mpcc_id);
+		}
+		if (mpc->funcs->program_lut_mode)
+			mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, lut1d_xable && m_lut_params.pwl, lut_bank_a, mpcc_id);
+	}
+
+	/* Shaper */
+	if (mcm_luts.shaper && mcm_luts.lut3d_data.mpc_3dlut_enable) {
+		memset(&m_lut_params, 0, sizeof(m_lut_params));
+		if (mcm_luts.shaper->type == TF_TYPE_HWPWL)
+			m_lut_params.pwl = &mcm_luts.shaper->pwl;
+		else if (mcm_luts.shaper->type == TF_TYPE_DISTRIBUTED_POINTS) {
+			ASSERT(false);
+			rval = cm3_helper_translate_curve_to_hw_format(mpc->ctx,
+					mcm_luts.shaper,
+					&dpp_base->regamma_params, true);
+			m_lut_params.pwl = rval ? &dpp_base->regamma_params : NULL;
+		}
+		if (m_lut_params.pwl) {
+			if (mpc->funcs->mcm.populate_lut)
+				mpc->funcs->mcm.populate_lut(mpc, m_lut_params, lut_bank_a, mpcc_id);
+			if (mpc->funcs->program_lut_mode)
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_ENABLE, lut_bank_a, mpcc_id);
+		}
+	}
+
+	/* 3DLUT */
+	switch (lut3d_src) {
+	case DC_CM2_TRANSFER_FUNC_SOURCE_SYSMEM:
+		memset(&m_lut_params, 0, sizeof(m_lut_params));
+		if (hubp->funcs->hubp_enable_3dlut_fl)
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, false);
+
+		if (mcm_luts.lut3d_data.lut3d_func && mcm_luts.lut3d_data.lut3d_func->state.bits.initialized) {
+			m_lut_params.lut3d = &mcm_luts.lut3d_data.lut3d_func->lut_3d;
+			if (mpc->funcs->populate_lut)
+				mpc->funcs->populate_lut(mpc, MCM_LUT_3DLUT, m_lut_params, lut_bank_a, mpcc_id);
+			if (mpc->funcs->program_lut_mode)
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a,
+						mpcc_id);
+		}
+		break;
+	case DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM:
+		switch (mcm_luts.lut3d_data.gpu_mem_params.size) {
+		case DC_CM2_GPU_MEM_SIZE_333333:
+			width = hubp_3dlut_fl_width_33;
+			break;
+		case DC_CM2_GPU_MEM_SIZE_171717:
+			width = hubp_3dlut_fl_width_17;
+			break;
+		case DC_CM2_GPU_MEM_SIZE_TRANSFORMED:
+			width = hubp_3dlut_fl_width_transformed;
+			break;
+		default:
+			//TODO: Handle default case
+			break;
+		}
+
+		//check for support
+		if (mpc->funcs->mcm.is_config_supported &&
+			!mpc->funcs->mcm.is_config_supported(width))
+			break;
+
+		if (mpc->funcs->program_lut_read_write_control)
+			mpc->funcs->program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, mpcc_id);
+		if (mpc->funcs->program_lut_mode)
+			mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, lut3d_xable, lut_bank_a, mpcc_id);
+
+		if (hubp->funcs->hubp_program_3dlut_fl_addr)
+			hubp->funcs->hubp_program_3dlut_fl_addr(hubp, mcm_luts.lut3d_data.gpu_mem_params.addr);
+
+		if (mpc->funcs->mcm.program_bit_depth)
+			mpc->funcs->mcm.program_bit_depth(mpc, mcm_luts.lut3d_data.gpu_mem_params.bit_depth, mpcc_id);
+
+		dc_get_lut_mode(mcm_luts.lut3d_data.gpu_mem_params.layout, &mode, &addr_mode);
+		if (hubp->funcs->hubp_program_3dlut_fl_mode)
+			hubp->funcs->hubp_program_3dlut_fl_mode(hubp, mode);
+
+		if (hubp->funcs->hubp_program_3dlut_fl_addressing_mode)
+			hubp->funcs->hubp_program_3dlut_fl_addressing_mode(hubp, addr_mode);
+
+		switch (mcm_luts.lut3d_data.gpu_mem_params.format_params.format) {
+		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12MSB:
+			format = hubp_3dlut_fl_format_unorm_12msb_bitslice;
+			break;
+		case DC_CM2_GPU_MEM_FORMAT_16161616_UNORM_12LSB:
+			format = hubp_3dlut_fl_format_unorm_12lsb_bitslice;
+			break;
+		case DC_CM2_GPU_MEM_FORMAT_16161616_FLOAT_FP1_5_10:
+			format = hubp_3dlut_fl_format_float_fp1_5_10;
+			break;
+		}
+		if (hubp->funcs->hubp_program_3dlut_fl_format)
+			hubp->funcs->hubp_program_3dlut_fl_format(hubp, format);
+		if (hubp->funcs->hubp_update_3dlut_fl_bias_scale &&
+				mpc->funcs->mcm.program_bias_scale) {
+			mpc->funcs->mcm.program_bias_scale(mpc,
+				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
+				mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale,
+				mpcc_id);
+			hubp->funcs->hubp_update_3dlut_fl_bias_scale(hubp,
+						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.bias,
+						mcm_luts.lut3d_data.gpu_mem_params.format_params.float_params.scale);
+		}
+
+		//navi 4x has a bug and r and blue are swapped and need to be worked around here in
+		//TODO: need to make a method for get_xbar per asic OR do the workaround in program_crossbar for 4x
+		dc_get_lut_xbar(
+			mcm_luts.lut3d_data.gpu_mem_params.component_order,
+			&crossbar_bit_slice_cr_r,
+			&crossbar_bit_slice_y_g,
+			&crossbar_bit_slice_cb_b);
+
+		if (hubp->funcs->hubp_program_3dlut_fl_crossbar)
+			hubp->funcs->hubp_program_3dlut_fl_crossbar(hubp,
+					crossbar_bit_slice_cr_r,
+					crossbar_bit_slice_y_g,
+					crossbar_bit_slice_cb_b);
+
+		if (mpc->funcs->mcm.program_lut_read_write_control)
+			mpc->funcs->mcm.program_lut_read_write_control(mpc, MCM_LUT_3DLUT, lut_bank_a, true, mpcc_id);
+
+		if (mpc->funcs->mcm.program_3dlut_size)
+			mpc->funcs->mcm.program_3dlut_size(mpc, width, mpcc_id);
+
+		if (mpc->funcs->update_3dlut_fast_load_select)
+			mpc->funcs->update_3dlut_fast_load_select(mpc, mpcc_id, hubp->inst);
+
+		if (hubp->funcs->hubp_enable_3dlut_fl)
+			hubp->funcs->hubp_enable_3dlut_fl(hubp, true);
+		else {
+			if (mpc->funcs->program_lut_mode) {
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_SHAPER, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_3DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
+				mpc->funcs->program_lut_mode(mpc, MCM_LUT_1DLUT, MCM_LUT_DISABLE, lut_bank_a, mpcc_id);
+			}
+		}
+		break;
+	}
+}
+
+bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
+				const struct dc_plane_state *plane_state)
+{
+	struct dpp *dpp_base = pipe_ctx->plane_res.dpp;
+	int mpcc_id = pipe_ctx->plane_res.hubp->inst;
+	struct dc *dc = pipe_ctx->stream_res.opp->ctx->dc;
+	struct mpc *mpc = dc->res_pool->mpc;
+	bool result;
+	const struct pwl_params *lut_params = NULL;
+	bool rval;
+
+	if (plane_state->mcm_luts.lut3d_data.lut3d_src == DC_CM2_TRANSFER_FUNC_SOURCE_VIDMEM) {
+		dcn42_populate_mcm_luts(dc, pipe_ctx, plane_state->mcm_luts, plane_state->lut_bank_a);
+		return true;
+	}
+
+	mpc->funcs->set_movable_cm_location(mpc, MPCC_MOVABLE_CM_LOCATION_BEFORE, mpcc_id);
+	pipe_ctx->plane_state->mcm_location = MPCC_MOVABLE_CM_LOCATION_BEFORE;
+	// 1D LUT
+	if (plane_state->blend_tf.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->blend_tf.pwl;
+	else if (plane_state->blend_tf.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
+				&plane_state->blend_tf,
+				&dpp_base->regamma_params, false);
+		lut_params = rval ? &dpp_base->regamma_params : NULL;
+	}
+	result = mpc->funcs->program_1dlut(mpc, lut_params, mpcc_id);
+	lut_params = NULL;
+
+	// Shaper
+	if (plane_state->in_shaper_func.type == TF_TYPE_HWPWL)
+		lut_params = &plane_state->in_shaper_func.pwl;
+	else if (plane_state->in_shaper_func.type == TF_TYPE_DISTRIBUTED_POINTS) {
+		// TODO: dpp_base replace
+		rval = cm3_helper_translate_curve_to_hw_format(plane_state->ctx,
+				&plane_state->in_shaper_func,
+				&dpp_base->shaper_params, true);
+		lut_params = rval ? &dpp_base->shaper_params : NULL;
+	}
+	result &= mpc->funcs->program_shaper(mpc, lut_params, mpcc_id);
+
+	// 3D
+	if (mpc->funcs->program_3dlut) {
+		if (plane_state->lut3d_func.state.bits.initialized == 1)
+			result &= mpc->funcs->program_3dlut(mpc, &plane_state->lut3d_func.lut_3d, mpcc_id);
+		else
+			result &= mpc->funcs->program_3dlut(mpc, NULL, mpcc_id);
+	}
+
 	return result;
 }
 void dcn42_hardware_release(struct dc *dc)
 {
 	dcn35_hardware_release(dc);
+	dc_dmub_srv_release_hw(dc);
 
 }
 static int count_active_streams(const struct dc *dc)
@@ -540,7 +967,8 @@ void dcn42_calc_blocks_to_gate(struct dc *dc, struct dc_state *context,
 {
 	bool hpo_frl_stream_enc_acquired = false;
 	bool hpo_dp_stream_enc_acquired = false;
-	int i = 0, j = 0;
+	unsigned int i = 0;
+	int j = 0;
 
 	memset(update_state, 0, sizeof(struct pg_block_update));
 
@@ -663,7 +1091,8 @@ void dcn42_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 	struct pg_block_update *update_state)
 {
 	bool hpo_dp_stream_enc_acquired = false;
-	int i = 0, j = 0;
+	unsigned int i = 0;
+	int j = 0;
 
 	memset(update_state, 0, sizeof(struct pg_block_update));
 
@@ -795,6 +1224,7 @@ void dcn42_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 	struct pg_block_update *update_state)
 {
 	int i = 0;
+	int pipe_count = dc->res_pool->pipe_count;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
 	bool block_disabled = true;
 
@@ -808,7 +1238,7 @@ void dcn42_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 			pg_cntl->funcs->hpo_pg_control(pg_cntl, false);
 	}
 
-	for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
+	for (i = pipe_count - 1; i >= 0; i--) {
 		if (update_state->pg_pipe_res_update[PG_HUBP][i] &&
 			update_state->pg_pipe_res_update[PG_DPP][i]) {
 			if (pg_cntl->funcs->hubp_dpp_pg_control)
@@ -828,7 +1258,7 @@ void dcn42_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 		}
 	}
 
-	for (i = 0; i < dc->res_pool->pipe_count; i++) {
+	for (i = 0; i < pipe_count; i++) {
 		if (!update_state->pg_pipe_res_update[PG_MPCC][i] ||
 			!update_state->pg_pipe_res_update[PG_OPP][i] ||
 			!update_state->pg_pipe_res_update[PG_OPTC][i]) {
@@ -883,7 +1313,7 @@ void dcn42_calc_blocks_to_ungate(struct dc *dc, struct dc_state *context,
 void dcn42_hw_block_power_up(struct dc *dc,
 	struct pg_block_update *update_state)
 {
-	int i = 0;
+	unsigned int i = 0;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
 	bool block_enabled = false;
 
@@ -917,7 +1347,7 @@ void dcn42_hw_block_power_up(struct dc *dc,
 			pg_cntl->funcs->plane_otg_pg_control(pg_cntl, true);
 	}
 
-	for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)dc->res_pool->res_cap->num_dsc; i++) {
 		if (update_state->pg_pipe_res_update[PG_DSC][i]) {
 			if (pg_cntl->funcs->dsc_pg_control)
 				pg_cntl->funcs->dsc_pg_control(pg_cntl, i, true);
@@ -944,7 +1374,7 @@ void dcn42_hw_block_power_up(struct dc *dc,
 void dcn42_root_clock_control(struct dc *dc,
 	struct pg_block_update *update_state, bool power_on)
 {
-	int i = 0;
+	unsigned int i = 0;
 	struct pg_cntl *pg_cntl = dc->res_pool->pg_cntl;
 
 	if (!pg_cntl)
@@ -968,7 +1398,7 @@ void dcn42_root_clock_control(struct dc *dc,
 					dc->hwseq->funcs.physymclk_root_clock_control(dc->hwseq, i, power_on);
 
 	}
-	for (i = 0; i < dc->res_pool->res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)dc->res_pool->res_cap->num_dsc; i++) {
 		if (update_state->pg_pipe_res_update[PG_DSC][i]) {
 			if (power_on) {
 				if (dc->res_pool->dccg->funcs->enable_dsc)
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
index c4cfeed45b190..0539ee0ffaee3 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn42/dcn42_hwseq.h
@@ -18,11 +18,18 @@ void dcn42_program_cm_hist(
 bool dcn42_set_mcm_luts(struct pipe_ctx *pipe_ctx,
 	const struct dc_plane_state *plane_state);
 
+void dcn42_populate_mcm_luts(struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		struct dc_cm2_func_luts mcm_luts,
+		bool lut_bank_a);
+
 bool dcn42_program_rmcm_luts(
 	struct hubp *hubp,
 	struct pipe_ctx *pipe_ctx,
-	const struct dc_plane_cm *cm,
+	enum dc_cm2_transfer_func_source lut3d_src,
+	struct dc_cm2_func_luts *mcm_luts,
 	struct mpc *mpc,
+	bool lut_bank_a,
 	int mpcc_id);
 void dcn42_hardware_release(struct dc *dc);
 
diff --git a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
index d1dba7ffcd9b3..4b9fcb87e60dc 100644
--- a/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
+++ b/drivers/gpu/drm/amd/display/dc/hwss/hw_sequencer.h
@@ -33,6 +33,7 @@
 #include "inc/core_status.h"
 #include "inc/hw/hw_shared.h"
 #include "dsc/dsc.h"
+#include "link_service_types.h"
 
 struct pipe_ctx;
 struct dc_state;
@@ -52,7 +53,13 @@ struct drr_params;
 struct dc_underflow_debug_data;
 struct dsc_optc_config;
 struct vm_system_aperture_param;
-struct memory_qos;
+struct dc_measured_memory_qos;
+struct stream_encoder;
+struct hpo_dp_stream_encoder;
+struct hpo_frl_stream_encoder;
+struct link_training_settings;
+struct dc_link;
+struct dc_crtc_timing;
 struct subvp_pipe_control_lock_fast_params {
 	struct dc *dc;
 	bool lock;
@@ -91,6 +98,21 @@ struct program_gamut_remap_params {
 	struct pipe_ctx *pipe_ctx;
 };
 
+struct hubp_enable_3dlut_fl_params {
+	struct hubp *hubp;
+};
+
+struct tg_setup_vertical_interrupt0_params {
+	struct timing_generator *tg;
+	uint32_t start_line;
+	uint32_t end_line;
+};
+
+struct update_info_frame_params {
+	struct dc *dc;
+	struct pipe_ctx *pipe_ctx;
+};
+
 struct program_manual_trigger_params {
 	struct pipe_ctx *pipe_ctx;
 };
@@ -707,11 +729,37 @@ struct abort_cursor_offload_update_params {
 	struct pipe_ctx *pipe_ctx;
 };
 
+struct cursor_lock_params {
+	struct dc *dc;
+	struct pipe_ctx *pipe_ctx;
+	bool lock;
+};
+
+struct setup_periodic_interrupt_params {
+	struct dc *dc;
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct send_cursor_info_to_dmu_params {
+	struct pipe_ctx *pipe_ctx;
+	int pipe_idx;
+};
+
 struct set_cursor_attribute_params {
 	struct dc *dc;
 	struct pipe_ctx *pipe_ctx;
 };
 
+struct hubp_set_cursor_attributes_params {
+	struct hubp *hubp;
+	const struct dc_cursor_attributes *attributes;
+};
+
+struct dpp_set_cursor_attributes_params {
+	struct dpp *dpp;
+	struct dc_cursor_attributes *attributes;
+};
+
 struct set_cursor_position_params {
 	struct dc *dc;
 	struct pipe_ctx *pipe_ctx;
@@ -739,6 +787,113 @@ struct phantom_hubp_post_enable_params {
 	struct hubp *hubp;
 };
 
+struct begin_cursor_offload_update_params {
+	struct dc *dc;
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct update_cursor_offload_pipe_params {
+	struct dc *dc;
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct commit_cursor_offload_update_params {
+	struct dc *dc;
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct stream_enc_update_hdmi_info_packets_params {
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num_params {
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct hpo_dp_stream_enc_update_dp_info_packets_params {
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct stream_enc_update_dp_info_packets_sdp_line_num_params {
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct stream_enc_update_dp_info_packets_params {
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct dsc_set_config_simple_params {
+	struct display_stream_compressor *dsc;
+	struct dsc_config dsc_cfg;
+	struct dsc_optc_config dsc_optc_cfg;
+};
+
+struct stream_enc_dp_set_dsc_config_params {
+	struct stream_encoder *stream_enc;
+	const struct dsc_optc_config *dsc_optc_cfg;
+};
+
+struct hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params {
+	struct hpo_dp_stream_encoder *hpo_dp_stream_enc;
+	bool immediate_update;
+	uint8_t *dsc_packed_pps;
+	bool pps_sdp_stream;
+};
+
+struct stream_enc_dp_set_dsc_pps_info_packet_params {
+	struct stream_encoder *stream_enc;
+	bool immediate_update;
+	uint8_t *dsc_packed_pps;
+	bool pps_sdp_stream;
+};
+
+struct dp_trace_source_sequence_params {
+	struct dc_link *link;
+	enum dpcd_source_sequence source;
+};
+
+struct set_dmdata_attributes_params {
+	struct hubp *hubp;
+	struct dc_dmdata_attributes attr;
+};
+
+struct link_increase_mst_payload_params {
+	struct pipe_ctx *pipe_ctx;
+	uint32_t mst_stream_bw;
+};
+
+struct link_reduce_mst_payload_params {
+	struct pipe_ctx *pipe_ctx;
+	uint32_t mst_stream_bw;
+};
+
+struct dp_set_test_pattern_params {
+	struct dc_link *link;
+	enum dp_test_pattern test_pattern;
+	enum dp_test_pattern_color_space test_pattern_color_space;
+	const struct link_training_settings *p_link_settings;
+	const unsigned char *p_custom_pattern;
+	unsigned int cust_pattern_size;
+};
+
+struct link_set_dpms_off_params {
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct disable_audio_stream_params {
+	struct pipe_ctx *pipe_ctx;
+};
+
+struct prepare_bandwidth_params {
+	struct dc *dc;
+	struct dc_state *context;
+};
+
+struct link_set_dpms_on_params {
+	struct dc_state *state;
+	struct pipe_ctx *pipe_ctx;
+};
+
 union block_sequence_params {
 	struct update_plane_addr_params update_plane_addr_params;
 	struct subvp_pipe_control_lock_fast_params subvp_pipe_control_lock_fast_params;
@@ -747,6 +902,9 @@ union block_sequence_params {
 	struct program_triplebuffer_params program_triplebuffer_params;
 	struct set_input_transfer_func_params set_input_transfer_func_params;
 	struct program_gamut_remap_params program_gamut_remap_params;
+	struct hubp_enable_3dlut_fl_params hubp_enable_3dlut_fl_params;
+	struct tg_setup_vertical_interrupt0_params tg_setup_vertical_interrupt0_params;
+	struct update_info_frame_params update_info_frame_params;
 	struct program_manual_trigger_params program_manual_trigger_params;
 	struct send_dmcub_cmd_params send_dmcub_cmd_params;
 	struct setup_dpp_params setup_dpp_params;
@@ -855,12 +1013,38 @@ union block_sequence_params {
 	struct dpp_set_scaler_params dpp_set_scaler_params;
 	struct hubp_mem_program_viewport_params hubp_mem_program_viewport_params;
 	struct abort_cursor_offload_update_params abort_cursor_offload_update_params;
+	struct cursor_lock_params cursor_lock_params;
+	struct setup_periodic_interrupt_params setup_periodic_interrupt_params;
+	struct send_cursor_info_to_dmu_params send_cursor_info_to_dmu_params;
 	struct set_cursor_attribute_params set_cursor_attribute_params;
+	struct hubp_set_cursor_attributes_params hubp_set_cursor_attributes_params;
+	struct dpp_set_cursor_attributes_params dpp_set_cursor_attributes_params;
 	struct set_cursor_position_params set_cursor_position_params;
 	struct set_cursor_sdr_white_level_params set_cursor_sdr_white_level_params;
 	struct program_output_csc_params program_output_csc_params;
 	struct hubp_set_blank_params hubp_set_blank_params;
 	struct phantom_hubp_post_enable_params phantom_hubp_post_enable_params;
+	struct begin_cursor_offload_update_params begin_cursor_offload_update_params;
+	struct update_cursor_offload_pipe_params update_cursor_offload_pipe_params;
+	struct commit_cursor_offload_update_params commit_cursor_offload_update_params;
+	struct stream_enc_update_hdmi_info_packets_params stream_enc_update_hdmi_info_packets_params;
+	struct hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num_params hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num_params;
+	struct hpo_dp_stream_enc_update_dp_info_packets_params hpo_dp_stream_enc_update_dp_info_packets_params;
+	struct stream_enc_update_dp_info_packets_sdp_line_num_params stream_enc_update_dp_info_packets_sdp_line_num_params;
+	struct stream_enc_update_dp_info_packets_params stream_enc_update_dp_info_packets_params;
+	struct dsc_set_config_simple_params dsc_set_config_simple_params;
+	struct stream_enc_dp_set_dsc_config_params stream_enc_dp_set_dsc_config_params;
+	struct hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params hpo_dp_stream_enc_dp_set_dsc_pps_info_packet_params;
+	struct stream_enc_dp_set_dsc_pps_info_packet_params stream_enc_dp_set_dsc_pps_info_packet_params;
+	struct dp_trace_source_sequence_params dp_trace_source_sequence_params;
+	struct set_dmdata_attributes_params set_dmdata_attributes_params;
+	struct link_increase_mst_payload_params link_increase_mst_payload_params;
+	struct link_reduce_mst_payload_params link_reduce_mst_payload_params;
+	struct dp_set_test_pattern_params dp_set_test_pattern_params;
+	struct link_set_dpms_off_params link_set_dpms_off_params;
+	struct disable_audio_stream_params disable_audio_stream_params;
+	struct prepare_bandwidth_params prepare_bandwidth_params;
+	struct link_set_dpms_on_params link_set_dpms_on_params;
 };
 
 enum block_sequence_func {
@@ -871,6 +1055,11 @@ enum block_sequence_func {
 	HUBP_UPDATE_PLANE_ADDR,
 	DPP_SET_INPUT_TRANSFER_FUNC,
 	DPP_PROGRAM_GAMUT_REMAP,
+	HUBP_ENABLE_3DLUT_FL,
+	OTG_SETUP_VERTICAL_INTERRUPT,
+	HWSS_SETUP_PERIODIC_INTERRUPT,
+	HWSS_UPDATE_INFO_FRAME,
+	HUBP_SET_DMDATA_ATTRIBUTES,
 	OPTC_PROGRAM_MANUAL_TRIGGER,
 	DMUB_SEND_DMCUB_CMD,
 	DPP_SETUP_DPP,
@@ -975,13 +1164,37 @@ enum block_sequence_func {
 	DPP_SET_SCALER,
 	HUBP_MEM_PROGRAM_VIEWPORT,
 	ABORT_CURSOR_OFFLOAD_UPDATE,
+	HWSS_CURSOR_LOCK,
+	HWSS_BEGIN_CURSOR_OFFLOAD_UPDATE,
+	HWSS_COMMIT_CURSOR_OFFLOAD_UPDATE,
+	HWSS_UPDATE_CURSOR_OFFLOAD_PIPE,
+	DC_SEND_CURSOR_INFO_TO_DMU,
 	SET_CURSOR_ATTRIBUTE,
+	HUBP_SET_CURSOR_ATTRIBUTES,
+	DPP_SET_CURSOR_ATTRIBUTES,
 	SET_CURSOR_POSITION,
 	SET_CURSOR_SDR_WHITE_LEVEL,
 	PROGRAM_OUTPUT_CSC,
 	HUBP_SET_LEGACY_TILING_COMPAT_LEVEL,
 	HUBP_SET_BLANK,
 	PHANTOM_HUBP_POST_ENABLE,
+	STREAM_ENC_UPDATE_HDMI_INFO_PACKETS,
+	HPO_DP_STREAM_ENC_UPDATE_DP_INFO_PACKETS_SDP_LINE_NUM,
+	HPO_DP_STREAM_ENC_UPDATE_DP_INFO_PACKETS,
+	STREAM_ENC_UPDATE_DP_INFO_PACKETS_SDP_LINE_NUM,
+	STREAM_ENC_UPDATE_DP_INFO_PACKETS,
+	DSC_SET_CONFIG_SIMPLE,
+	STREAM_ENC_DP_SET_DSC_CONFIG,
+	HPO_DP_STREAM_ENC_DP_SET_DSC_PPS_INFO_PACKET,
+	STREAM_ENC_DP_SET_DSC_PPS_INFO_PACKET,
+	LINK_INCREASE_MST_PAYLOAD,
+	LINK_REDUCE_MST_PAYLOAD,
+	DP_TRACE_SOURCE_SEQUENCE,
+	DP_SET_TEST_PATTERN,
+	LINK_SET_DPMS_OFF,
+	DISABLE_AUDIO_STREAM,
+	PREPARE_BANDWIDTH,
+	LINK_SET_DPMS_ON,
 	/* This must be the last value in this enum, add new ones above */
 	HWSS_BLOCK_SEQUENCE_FUNC_COUNT
 };
@@ -1303,12 +1516,12 @@ struct hw_sequencer_funcs {
 	/**
 	 * measure_memory_qos - Measure memory QoS metrics
 	 * @dc: DC structure
-	 * @qos: Pointer to memory_qos struct to populate with measured values
+	 * @qos: Pointer to dc_measured_memory_qos struct to populate with measured values
 	 *
-	 * Populates the provided memory_qos struct with peak bandwidth, average bandwidth,
+	 * Populates the provided dc_measured_memory_qos struct with peak bandwidth, average bandwidth,
 	 * max latency, min latency, and average latency from hardware performance counters.
 	 */
-	void (*measure_memory_qos)(struct dc *dc, struct memory_qos *qos);
+	void (*measure_memory_qos)(struct dc *dc, struct dc_measured_memory_qos *qos);
 
 };
 
@@ -1455,6 +1668,40 @@ void hwss_dsc_calculate_and_set_config(union block_sequence_params *params);
 
 void hwss_dsc_enable_with_opp(union block_sequence_params *params);
 
+void hwss_dsc_set_config_simple(union block_sequence_params *params);
+
+void hwss_stream_enc_update_hdmi_info_packets(union block_sequence_params *params);
+
+void hwss_hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num(union block_sequence_params *params);
+
+void hwss_hpo_dp_stream_enc_update_dp_info_packets(union block_sequence_params *params);
+
+void hwss_stream_enc_update_dp_info_packets_sdp_line_num(union block_sequence_params *params);
+
+void hwss_stream_enc_update_dp_info_packets(union block_sequence_params *params);
+
+void hwss_stream_enc_dp_set_dsc_config(union block_sequence_params *params);
+
+void hwss_hpo_dp_stream_enc_dp_set_dsc_pps_info_packet(union block_sequence_params *params);
+
+void hwss_stream_enc_dp_set_dsc_pps_info_packet(union block_sequence_params *params);
+
+void hwss_set_dmdata_attributes(union block_sequence_params *params);
+
+void hwss_dp_trace_source_sequence(union block_sequence_params *params);
+
+void hwss_link_increase_mst_payload(union block_sequence_params *params);
+
+void hwss_link_reduce_mst_payload(union block_sequence_params *params);
+
+void hwss_dp_set_test_pattern(union block_sequence_params *params);
+
+void hwss_link_set_dpms_off(union block_sequence_params *params);
+
+void hwss_prepare_bandwidth(struct dc *dc, union block_sequence_params *params);
+
+void hwss_link_set_dpms_on(union block_sequence_params *params);
+
 void hwss_program_tg(union block_sequence_params *params);
 
 void hwss_tg_program_global_sync(union block_sequence_params *params);
@@ -1463,6 +1710,12 @@ void hwss_tg_wait_for_state(union block_sequence_params *params);
 
 void hwss_tg_set_vtg_params(union block_sequence_params *params);
 
+void hwss_hubp_enable_3dlut_fl(union block_sequence_params *params);
+
+void hwss_update_info_frame(struct dc *dc, union block_sequence_params *params);
+
+void hwss_tg_setup_vertical_interrupt0(union block_sequence_params *params);
+
 void hwss_tg_setup_vertical_interrupt2(union block_sequence_params *params);
 
 void hwss_dpp_set_hdr_multiplier(union block_sequence_params *params);
@@ -1603,12 +1856,20 @@ void hwss_hubp_mem_program_viewport(union block_sequence_params *params);
 
 void hwss_abort_cursor_offload_update(union block_sequence_params *params);
 
+void hwss_send_cursor_info_to_dmu(union block_sequence_params *params);
+
 void hwss_set_cursor_attribute(union block_sequence_params *params);
 
+void hwss_hubp_set_cursor_attributes(union block_sequence_params *params);
+
+void hwss_dpp_set_cursor_attributes(union block_sequence_params *params);
+
 void hwss_set_cursor_position(union block_sequence_params *params);
 
 void hwss_set_cursor_sdr_white_level(union block_sequence_params *params);
 
+void hwss_program_gamut_remap(union block_sequence_params *params);
+
 void hwss_program_output_csc(union block_sequence_params *params);
 
 void hwss_hubp_set_legacy_tiling_compat_level(union block_sequence_params *params);
@@ -1617,6 +1878,18 @@ void hwss_hubp_set_blank(union block_sequence_params *params);
 
 void hwss_phantom_hubp_post_enable(union block_sequence_params *params);
 
+void hwss_cursor_lock(union block_sequence_params *params);
+
+void hwss_begin_cursor_offload_update(union block_sequence_params *params);
+
+void hwss_commit_cursor_offload_update(union block_sequence_params *params);
+
+void hwss_update_cursor_offload_pipe(union block_sequence_params *params);
+
+void hwss_setup_periodic_interrupt(struct dc *dc, union block_sequence_params *params);
+
+void hwss_disable_audio_stream(struct dc *dc, union block_sequence_params *params);
+
 void hwss_add_optc_pipe_control_lock(struct block_sequence_state *seq_state,
 		struct dc *dc, struct pipe_ctx *pipe_ctx, bool lock);
 
@@ -1695,6 +1968,9 @@ void hwss_add_tg_wait_for_state(struct block_sequence_state *seq_state,
 void hwss_add_tg_set_vtg_params(struct block_sequence_state *seq_state,
 		struct timing_generator *tg, struct dc_crtc_timing *dc_crtc_timing, bool program_fp2);
 
+void hwss_add_vertical_interrupt_setup(struct block_sequence_state *seq_state,
+		struct timing_generator *tg, uint32_t start_line, uint32_t end_line);
+
 void hwss_add_tg_setup_vertical_interrupt2(struct block_sequence_state *seq_state,
 		struct timing_generator *tg, int start_line);
 
@@ -2012,6 +2288,14 @@ void hwss_add_set_cursor_attribute(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx);
 
+void hwss_add_hubp_set_cursor_attributes(struct block_sequence_state *seq_state,
+		struct hubp *hubp,
+		const struct dc_cursor_attributes *attributes);
+
+void hwss_add_dpp_set_cursor_attributes(struct block_sequence_state *seq_state,
+		struct dpp *dpp,
+		struct dc_cursor_attributes *attributes);
+
 void hwss_add_set_cursor_position(struct block_sequence_state *seq_state,
 		struct dc *dc,
 		struct pipe_ctx *pipe_ctx);
@@ -2056,4 +2340,105 @@ void hwss_add_opp_program_left_edge_extra_pixel(struct block_sequence_state *seq
 		enum dc_pixel_encoding pixel_encoding,
 		bool is_otg_master);
 
+void hwss_add_hubp_enable_3dlut_fl(struct block_sequence_state *seq_state,
+		struct hubp *hubp);
+
+void hwss_add_begin_cursor_offload_update(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_cursor_lock(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx,
+		bool lock);
+
+void hwss_add_send_update_cursor_info_to_dmu(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx,
+		int index);
+
+void hwss_add_update_cursor_offload_pipe(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_commit_cursor_offload_update(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_stream_enc_update_hdmi_info_packets(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_hpo_dp_stream_enc_update_dp_info_packets_sdp_line_num(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_hpo_dp_stream_enc_update_dp_info_packets(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_stream_enc_update_dp_info_packets_sdp_line_num(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_stream_enc_update_dp_info_packets(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_dsc_set_config(struct block_sequence_state *seq_state,
+		struct display_stream_compressor *dsc,
+		const struct dsc_config *dsc_cfg,
+		const struct dsc_optc_config *dsc_optc_cfg);
+
+void hwss_add_stream_enc_dp_set_dsc_config(struct block_sequence_state *seq_state,
+		struct stream_encoder *stream_enc,
+		const struct dsc_optc_config *dsc_optc_cfg);
+
+void hwss_add_hpo_dp_stream_enc_dp_set_dsc_pps_info_packet(struct block_sequence_state *seq_state,
+		struct hpo_dp_stream_encoder *hpo_dp_stream_enc,
+		bool immediate_update,
+		uint8_t *dsc_packed_pps,
+		bool pps_sdp_stream);
+
+void hwss_add_stream_enc_dp_set_dsc_pps_info_packet(struct block_sequence_state *seq_state,
+		struct stream_encoder *stream_enc,
+		bool immediate_update,
+		uint8_t *dsc_packed_pps,
+		bool pps_sdp_stream);
+
+void hwss_add_setup_periodic_interrupt(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_dp_trace_source_sequence(struct block_sequence_state *seq_state,
+		struct dc_link *link,
+		enum dpcd_source_sequence source);
+
+void hwss_add_set_dmdata_attributes(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_link_increase_mst_payload(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx,
+		uint32_t mst_stream_bw);
+
+void hwss_add_link_reduce_mst_payload(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx,
+		uint32_t mst_stream_bw);
+
+void hwss_add_dp_set_test_pattern(struct block_sequence_state *seq_state,
+		struct dc_link *link,
+		enum dp_test_pattern test_pattern,
+		enum dp_test_pattern_color_space test_pattern_color_space,
+		const struct link_training_settings *p_link_settings,
+		const unsigned char *p_custom_pattern,
+		unsigned int cust_pattern_size);
+
+void hwss_add_link_set_dpms_off(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_disable_audio_stream(struct block_sequence_state *seq_state,
+		struct pipe_ctx *pipe_ctx);
+
+void hwss_add_prepare_bandwidth(struct block_sequence_state *seq_state,
+		struct dc *dc,
+		struct dc_state *context);
+
+void hwss_add_link_set_dpms_on(struct block_sequence_state *seq_state,
+		struct dc_state *state,
+		struct pipe_ctx *pipe_ctx);
+
 #endif /* __DC_HW_SEQUENCER_H__ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/bw_fixed.h b/drivers/gpu/drm/amd/display/dc/inc/bw_fixed.h
index d1656c9d50dfe..3fdd9a7703341 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/bw_fixed.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/bw_fixed.h
@@ -79,7 +79,7 @@ static inline struct bw_fixed bw_int_to_fixed(int64_t value)
 
 static inline int32_t bw_fixed_to_int(struct bw_fixed value)
 {
-	return BW_FIXED_GET_INTEGER_PART(value.value);
+	return (int32_t)BW_FIXED_GET_INTEGER_PART(value.value);
 }
 
 struct bw_fixed bw_frc_to_fixed(int64_t num, int64_t denum);
@@ -126,7 +126,7 @@ static inline struct bw_fixed bw_div(const struct bw_fixed arg1, const struct bw
 static inline struct bw_fixed bw_mod(const struct bw_fixed arg1, const struct bw_fixed arg2)
 {
 	struct bw_fixed res;
-	div64_u64_rem(arg1.value, arg2.value, (uint64_t *)&res.value);
+	div64_u64_rem((uint64_t)arg1.value, (uint64_t)arg2.value, (uint64_t *)&res.value);
 	return res;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/core_types.h b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
index e960ca9062adc..a99e1937f8ce0 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/core_types.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/core_types.h
@@ -707,7 +707,7 @@ struct dc_bounding_box_max_clk {
 	int max_phyclk_mhz;
 };
 
-struct memory_qos {
+struct dc_measured_memory_qos {
 	uint32_t peak_bw_mbps;
 	uint32_t avg_bw_mbps;
 	uint32_t max_latency_ns;
@@ -715,4 +715,12 @@ struct memory_qos {
 	uint32_t avg_latency_ns;
 };
 
+struct dc_requested_memory_qos {
+	uint32_t bandwidth_lb_in_mbps;
+	uint32_t calculated_avg_bw_in_mbps;
+	uint32_t max_latency_ub_in_ns;
+	uint32_t avg_latency_ub_in_ns;
+	uint32_t max_bw_budget_in_mbps;
+};
+
 #endif /* _CORE_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
index 21224fd6b36d6..dcce81e3d97c9 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/clk_mgr.h
@@ -28,6 +28,7 @@
 #define __DAL_CLK_MGR_H__
 
 #include "dc.h"
+#include "core_types.h"
 #include "dm_pp_smu.h"
 
 /* Constants */
@@ -362,6 +363,27 @@ struct clk_mgr_funcs {
 	uint32_t (*set_smartmux_switch)(struct clk_mgr *clk_mgr, uint32_t pins_to_set);
 
 	unsigned int (*get_max_clock_khz)(struct clk_mgr *clk_mgr_base, enum clk_type clk_type);
+	/**
+	 * override_memory_bandwidth_request - Override the DCN nominal memory
+	 *     bandwidth request sent to PMFW, independent of the current display
+	 *     mode. For debug use only.
+	 * @clk_mgr: clock manager instance
+	 * @bw_kbps: requested bandwidth in kbps; 0 clears the override
+	 *
+	 * Return: capped bandwidth value actually applied (kbps)
+	 */
+	unsigned int (*override_memory_bandwidth_request)(
+			struct clk_mgr *clk_mgr,
+			unsigned int bw_kbps);
+	/**
+	 * get_requested_memory_qos - Retrieve current QoS request from the clock manager's
+	 *     current clock state, reflecting any active bandwidth overrides.
+	 * @clk_mgr: clock manager instance
+	 * @qos: pointer to dc_requested_memory_qos structure to populate
+	 */
+	void (*get_requested_memory_qos)(
+			struct clk_mgr *clk_mgr,
+			struct dc_requested_memory_qos *qos);
 };
 
 struct clk_mgr {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
index 2a5a81d159507..1c18898aa475a 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/hubp.h
@@ -89,7 +89,7 @@ enum hubp_3dlut_fl_addressing_mode {
 enum hubp_3dlut_fl_width {
 	hubp_3dlut_fl_width_17 = 17,
 	hubp_3dlut_fl_width_33 = 33,
-	hubp_3dlut_fl_width_17_transformed    = 4916, //mpc default
+	hubp_3dlut_fl_width_transformed    = 4916, //mpc default
 };
 
 enum hubp_3dlut_fl_crossbar_bit_slice {
@@ -99,6 +99,22 @@ enum hubp_3dlut_fl_crossbar_bit_slice {
 	hubp_3dlut_fl_crossbar_bit_slice_48_63 = 3
 };
 
+struct hubp_fl_3dlut_config {
+	bool enabled;
+	enum hubp_3dlut_fl_width width;
+	enum hubp_3dlut_fl_mode mode;
+	enum hubp_3dlut_fl_format format;
+	uint16_t bias;
+	uint16_t scale;
+	struct dc_plane_address address;
+	enum hubp_3dlut_fl_addressing_mode addr_mode;
+	enum dc_cm2_gpu_mem_layout layout;
+	uint8_t protection_bits;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_y_g;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cb_b;
+	enum hubp_3dlut_fl_crossbar_bit_slice crossbar_bit_slice_cr_r;
+};
+
 struct hubp {
 	const struct hubp_funcs *funcs;
 	struct dc_context *ctx;
@@ -133,7 +149,7 @@ struct surface_flip_registers {
 	uint32_t DCSURF_SECONDARY_META_SURFACE_ADDRESS;
 	uint32_t DCSURF_SECONDARY_SURFACE_ADDRESS_HIGH;
 	uint32_t DCSURF_SECONDARY_SURFACE_ADDRESS;
-	bool tmz_surface;
+	uint8_t tmz_surface;
 	bool immediate;
 	uint8_t vmid;
 	bool grph_stereo;
@@ -273,15 +289,24 @@ struct hubp_funcs {
 
 	void (*hubp_wait_pipe_read_start)(struct hubp *hubp);
 	void (*hubp_program_mcache_id_and_split_coordinate)(struct hubp *hubp, struct dml2_hubp_pipe_mcache_regs *mcache_regs);
+	void (*hubp_update_3dlut_fl_bias_scale)(struct hubp *hubp, uint16_t bias, uint16_t scale);
+	void (*hubp_program_3dlut_fl_mode)(struct hubp *hubp,
+			enum hubp_3dlut_fl_mode mode);
+	void (*hubp_program_3dlut_fl_format)(struct hubp *hubp,
+			enum hubp_3dlut_fl_format format);
 	void (*hubp_program_3dlut_fl_addr)(struct hubp *hubp,
-		const struct dc_plane_address *address);
-	void (*hubp_program_3dlut_fl_config)(struct hubp *hubp,
-		const struct dc_3dlut_dma *config);
+		const struct dc_plane_address address);
 	void (*hubp_program_3dlut_fl_dlg_param)(struct hubp *hubp, int refcyc_per_3dlut_group);
 	void (*hubp_enable_3dlut_fl)(struct hubp *hubp, bool enable);
+	void (*hubp_program_3dlut_fl_addressing_mode)(struct hubp *hubp, enum hubp_3dlut_fl_addressing_mode addr_mode);
+	void (*hubp_program_3dlut_fl_width)(struct hubp *hubp, enum hubp_3dlut_fl_width width);
+	void (*hubp_program_3dlut_fl_tmz_protected)(struct hubp *hubp, uint8_t protection_bits);
 	void (*hubp_program_3dlut_fl_crossbar)(struct hubp *hubp,
-			enum dc_cm_lut_pixel_format format);
-	int (*hubp_get_3dlut_fl_done)(struct hubp *hubp);
+			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_y_g,
+			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cb_b,
+			enum hubp_3dlut_fl_crossbar_bit_slice bit_slice_cr_r);
+	uint32_t (*hubp_get_3dlut_fl_done)(struct hubp *hubp);
+	void (*hubp_program_3dlut_fl_config)(struct hubp *hubp, struct hubp_fl_3dlut_config *cfg);
 	void (*hubp_clear_tiling)(struct hubp *hubp);
 	uint32_t (*hubp_get_current_read_line)(struct hubp *hubp);
 	uint32_t (*hubp_get_det_config_error)(struct hubp *hubp);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
index f5617674bea8e..54eb2eba68bf4 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/mpc.h
@@ -54,7 +54,6 @@
 #include "dc_hw_types.h"
 #include "hw_shared.h"
 #include "transform.h"
-#include "dc_types.h"
 
 #define MAX_MPCC 6
 #define MAX_OPP 6
@@ -102,6 +101,13 @@ enum mpcc_movable_cm_location {
 	MPCC_MOVABLE_CM_LOCATION_AFTER,
 };
 
+enum MCM_LUT_XABLE {
+	MCM_LUT_DISABLE,
+	MCM_LUT_DISABLED = MCM_LUT_DISABLE,
+	MCM_LUT_ENABLE,
+	MCM_LUT_ENABLED = MCM_LUT_ENABLE,
+};
+
 enum MCM_LUT_ID {
 	MCM_LUT_3DLUT,
 	MCM_LUT_1DLUT,
@@ -110,7 +116,7 @@ enum MCM_LUT_ID {
 
 struct mpc_fl_3dlut_config {
 	bool enabled;
-	enum dc_cm_lut_size size;
+	uint16_t width;
 	bool select_lut_bank_a;
 	uint16_t bit_depth;
 	int hubp_index;
@@ -1069,11 +1075,8 @@ struct mpc_funcs {
 	*
 	* void
 	*/
-	void (*populate_lut)(struct mpc *mpc,
-			const enum MCM_LUT_ID id,
-			const union mcm_lut_params *params,
-			const bool lut_bank_a,
-			const int mpcc_id);
+	void (*populate_lut)(struct mpc *mpc, const enum MCM_LUT_ID id, const union mcm_lut_params params,
+			bool lut_bank_a, int mpcc_id);
 
 	/**
 	* @program_lut_read_write_control:
@@ -1084,18 +1087,13 @@ struct mpc_funcs {
 	* - [in/out] mpc - MPC context.
 	* - [in] id
 	* - [in] lut_bank_a
-	* - [in] bit_depth
 	* - [in] mpcc_id
 	*
 	* Return:
 	*
 	* void
 	*/
-	void (*program_lut_read_write_control)(struct mpc *mpc,
-		const enum MCM_LUT_ID id,
-		const bool lut_bank_a,
-		const unsigned int bit_depth,
-		const int mpcc_id);
+	void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id, bool lut_bank_a, int mpcc_id);
 
 	/**
 	* @program_lut_mode:
@@ -1105,44 +1103,33 @@ struct mpc_funcs {
 	* Parameters:
 	* - [in/out] mpc - MPC context.
 	* - [in] id
-	* - [in] enable
+	* - [in] xable
 	* - [in] lut_bank_a
-	* - [in] size
 	* - [in] mpcc_id
 	*
 	* Return:
 	*
 	* void
 	*/
-	void (*program_lut_mode)(struct mpc *mpc,
-			const enum MCM_LUT_ID id,
-			const bool enable,
-			const bool lut_bank_a,
-			const enum dc_cm_lut_size size,
-			const int mpcc_id);
-
+	void (*program_lut_mode)(struct mpc *mpc, const enum MCM_LUT_ID id, const enum MCM_LUT_XABLE xable,
+			bool lut_bank_a, int mpcc_id);
 
 	/**
-	* @get_lut_mode:
-	*
-	* Obtains enablement and ram bank status.
-	*
-	* Parameters:
-	* - [in/out] mpc - MPC context.
-	* - [in] id
-	* - [in] mpcc_id
-	* - [out] enable
-	* - [out] lut_bank_a
-	*
-	* Return:
-	*
-	* void
-	*/
-	void (*get_lut_mode)(struct mpc *mpc,
-			const enum MCM_LUT_ID id,
-			const int mpcc_id,
-			bool *enable,
-			bool *lut_bank_a);
+	 * @mcm:
+	 *
+	 * MPC MCM new HW sequential programming functions
+	 */
+	struct {
+		void (*program_3dlut_size)(struct mpc *mpc, uint32_t width, int mpcc_id);
+		void (*program_bias_scale)(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id);
+		void (*program_bit_depth)(struct mpc *mpc, uint16_t bit_depth, int mpcc_id);
+		bool (*is_config_supported)(uint32_t width);
+		void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id,
+			bool lut_bank_a, bool enabled, int mpcc_id);
+
+		void (*populate_lut)(struct mpc *mpc, const union mcm_lut_params params,
+			bool lut_bank_a, int mpcc_id);
+	} mcm;
 
 	/**
 	 * @rmcm:
@@ -1155,11 +1142,9 @@ struct mpc_funcs {
 		void (*update_3dlut_fast_load_select)(struct mpc *mpc, int mpcc_id, int hubp_idx);
 		void (*program_lut_read_write_control)(struct mpc *mpc, const enum MCM_LUT_ID id,
 			bool lut_bank_a, bool enabled, int mpcc_id);
-		void (*program_lut_mode)(struct mpc *mpc,
-			bool enable,
-			bool lut_bank_a,
-			int mpcc_id);
-		void (*program_3dlut_size)(struct mpc *mpc, const enum dc_cm_lut_size size, int mpcc_id);
+		void (*program_lut_mode)(struct mpc *mpc, const enum MCM_LUT_XABLE xable,
+			bool lut_bank_a, int mpcc_id);
+		void (*program_3dlut_size)(struct mpc *mpc, uint32_t width, int mpcc_id);
 		void (*program_bias_scale)(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id);
 		void (*program_bit_depth)(struct mpc *mpc, uint16_t bit_depth, int mpcc_id);
 		bool (*is_config_supported)(uint32_t width);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h b/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
index e1428a83ecbc3..73cc34ea77263 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/opp.h
@@ -356,8 +356,8 @@ struct opp_funcs {
 
 	void (*opp_program_dpg_dimensions)(
 				struct output_pixel_processor *opp,
-				int width,
-				int height);
+				uint32_t width,
+				uint32_t height);
 
 	bool (*dpg_is_blanked)(
 			struct output_pixel_processor *opp);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h b/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h
index 0d5a8358a7780..7f371cbb35cde 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/optc.h
@@ -68,7 +68,6 @@ struct optc {
 	int pstate_keepout;
 	struct dc_crtc_timing orginal_patched_timing;
 	enum signal_type signal;
-	uint32_t max_frame_count;
 };
 
 void optc1_read_otg_state(struct timing_generator *optc, struct dcn_otg_state *s);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
index 27f950ae45ee5..a9bf960dacbce 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/stream_encoder.h
@@ -93,6 +93,7 @@ struct encoder_info_frame {
 	/* Adaptive Sync SDP*/
 	struct dc_info_packet adaptive_sync;
 	struct enc_sdp_line_num sdp_line_num;
+	bool firmware_controlled_hdr_info_packet;
 };
 
 struct encoder_unblank_param {
diff --git a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
index 2f70bb476c97b..3a80369cde16b 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/hw/timing_generator.h
@@ -32,9 +32,9 @@ struct dc_bios;
 
 /* Contains CRTC vertical/horizontal pixel counters */
 struct crtc_position {
-	int32_t vertical_count;
-	int32_t horizontal_count;
-	int32_t nominal_vcount;
+	uint32_t vertical_count;
+	uint32_t horizontal_count;
+	uint32_t nominal_vcount;
 };
 
 struct dcp_gsl_params {
@@ -321,7 +321,7 @@ struct timing_generator {
 	const struct timing_generator_funcs *funcs;
 	struct dc_bios *bp;
 	struct dc_context *ctx;
-	int inst;
+	uint32_t inst;
 };
 
 struct dc_crtc_timing;
diff --git a/drivers/gpu/drm/amd/display/dc/inc/link_service.h b/drivers/gpu/drm/amd/display/dc/inc/link_service.h
index 57bb82e949427..d0609443af495 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/link_service.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/link_service.h
@@ -110,7 +110,7 @@ struct link_service {
 	struct dc_sink *(*add_remote_sink)(
 			struct dc_link *link,
 			const uint8_t *edid,
-			int len,
+			unsigned int len,
 			struct dc_sink_init_data *init_data);
 	void (*remove_remote_sink)(struct dc_link *link, struct dc_sink *sink);
 	bool (*get_hpd_state)(struct dc_link *link);
diff --git a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
index 26cb1459b743f..7a1ecb8d986f2 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/reg_helper.h
@@ -155,63 +155,63 @@
  * read given register and fill in field value in output parameter */
 #define REG_GET(reg_name, field, val)	\
 		generic_reg_get(CTX, REG(reg_name), \
-				FN(reg_name, field), val)
+				FN(reg_name, field), (uint32_t *)val)
 
 #define REG_GET_2(reg_name, f1, v1, f2, v2)	\
 		generic_reg_get2(CTX, REG(reg_name), \
-				FN(reg_name, f1), v1, \
-				FN(reg_name, f2), v2)
+				FN(reg_name, f1), (uint32_t *)v1, \
+				FN(reg_name, f2), (uint32_t *)v2)
 
 #define REG_GET_3(reg_name, f1, v1, f2, v2, f3, v3)	\
 		generic_reg_get3(CTX, REG(reg_name), \
-				FN(reg_name, f1), v1, \
-				FN(reg_name, f2), v2, \
-				FN(reg_name, f3), v3)
+				FN(reg_name, f1), (uint32_t *)v1, \
+				FN(reg_name, f2), (uint32_t *)v2, \
+				FN(reg_name, f3), (uint32_t *)v3)
 
 #define REG_GET_4(reg_name, f1, v1, f2, v2, f3, v3, f4, v4)	\
 		generic_reg_get4(CTX, REG(reg_name), \
-				FN(reg_name, f1), v1, \
-				FN(reg_name, f2), v2, \
-				FN(reg_name, f3), v3, \
-				FN(reg_name, f4), v4)
+				FN(reg_name, f1), (uint32_t *)v1, \
+				FN(reg_name, f2), (uint32_t *)v2, \
+				FN(reg_name, f3), (uint32_t *)v3, \
+				FN(reg_name, f4), (uint32_t *)v4)
 
 #define REG_GET_5(reg_name, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5)	\
 		generic_reg_get5(CTX, REG(reg_name), \
-				FN(reg_name, f1), v1, \
-				FN(reg_name, f2), v2, \
-				FN(reg_name, f3), v3, \
-				FN(reg_name, f4), v4, \
-				FN(reg_name, f5), v5)
+				FN(reg_name, f1), (uint32_t *)v1, \
+				FN(reg_name, f2), (uint32_t *)v2, \
+				FN(reg_name, f3), (uint32_t *)v3, \
+				FN(reg_name, f4), (uint32_t *)v4, \
+				FN(reg_name, f5), (uint32_t *)v5)
 
 #define REG_GET_6(reg_name, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6)	\
 		generic_reg_get6(CTX, REG(reg_name), \
-				FN(reg_name, f1), v1, \
-				FN(reg_name, f2), v2, \
-				FN(reg_name, f3), v3, \
-				FN(reg_name, f4), v4, \
-				FN(reg_name, f5), v5, \
-				FN(reg_name, f6), v6)
+				FN(reg_name, f1), (uint32_t *)v1, \
+				FN(reg_name, f2), (uint32_t *)v2, \
+				FN(reg_name, f3), (uint32_t *)v3, \
+				FN(reg_name, f4), (uint32_t *)v4, \
+				FN(reg_name, f5), (uint32_t *)v5, \
+				FN(reg_name, f6), (uint32_t *)v6)
 
 #define REG_GET_7(reg_name, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7)	\
 		generic_reg_get7(CTX, REG(reg_name), \
-				FN(reg_name, f1), v1, \
-				FN(reg_name, f2), v2, \
-				FN(reg_name, f3), v3, \
-				FN(reg_name, f4), v4, \
-				FN(reg_name, f5), v5, \
-				FN(reg_name, f6), v6, \
-				FN(reg_name, f7), v7)
+				FN(reg_name, f1), (uint32_t *)v1, \
+				FN(reg_name, f2), (uint32_t *)v2, \
+				FN(reg_name, f3), (uint32_t *)v3, \
+				FN(reg_name, f4), (uint32_t *)v4, \
+				FN(reg_name, f5), (uint32_t *)v5, \
+				FN(reg_name, f6), (uint32_t *)v6, \
+				FN(reg_name, f7), (uint32_t *)v7)
 
 #define REG_GET_8(reg_name, f1, v1, f2, v2, f3, v3, f4, v4, f5, v5, f6, v6, f7, v7, f8, v8)	\
 		generic_reg_get8(CTX, REG(reg_name), \
-				FN(reg_name, f1), v1, \
-				FN(reg_name, f2), v2, \
-				FN(reg_name, f3), v3, \
-				FN(reg_name, f4), v4, \
-				FN(reg_name, f5), v5, \
-				FN(reg_name, f6), v6, \
-				FN(reg_name, f7), v7, \
-				FN(reg_name, f8), v8)
+				FN(reg_name, f1), (uint32_t *)v1, \
+				FN(reg_name, f2), (uint32_t *)v2, \
+				FN(reg_name, f3), (uint32_t *)v3, \
+				FN(reg_name, f4), (uint32_t *)v4, \
+				FN(reg_name, f5), (uint32_t *)v5, \
+				FN(reg_name, f6), (uint32_t *)v6, \
+				FN(reg_name, f7), (uint32_t *)v7, \
+				FN(reg_name, f8), (uint32_t *)v8)
 
 /* macro to poll and wait for a register field to read back given value */
 
diff --git a/drivers/gpu/drm/amd/display/dc/inc/soc_and_ip_translator.h b/drivers/gpu/drm/amd/display/dc/inc/soc_and_ip_translator.h
index 23daf98b8aa88..6a97a3e28bd2c 100644
--- a/drivers/gpu/drm/amd/display/dc/inc/soc_and_ip_translator.h
+++ b/drivers/gpu/drm/amd/display/dc/inc/soc_and_ip_translator.h
@@ -8,9 +8,26 @@
 #include "dc.h"
 #include "dml_top_soc_parameter_types.h"
 
+/* Forward declarations — callers that dereference these structs must include
+ * the full UTM model headers themselves. */
+struct utm_qos_model;
+struct utm_qos_model_dchub_v2;
+
 struct soc_and_ip_translator_funcs {
-	void (*get_soc_bb)(struct dml2_soc_bb *soc_bb, const struct dc *dc, const struct dml2_configuration_options *config);
+	void (*get_soc_bb)(
+			struct dml2_soc_bb *soc_bb,
+			const struct dc *dc,
+			const struct dml2_configuration_options *config);
 	void (*get_ip_caps)(struct dml2_ip_capabilities *dml_ip_caps);
+	/**
+	 * get_utm_qos_model - Return the static UTM QoS model for this DCN
+	 *     generation. Caller provides storage for @qos_model and @dchub.
+	 * @qos_model: output — populated with SoC bounding box and SOP table
+	 * @dchub: output — populated with DCHUB client extension data
+	 */
+	void (*get_utm_qos_model)(
+			struct utm_qos_model *qos_model,
+			struct utm_qos_model_dchub_v2 *dchub);
 };
 
 struct soc_and_ip_translator {
diff --git a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
index 015f3659cf773..8958d286dde1f 100644
--- a/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
+++ b/drivers/gpu/drm/amd/display/dc/irq/dce110/irq_service_dce110.c
@@ -210,7 +210,7 @@ bool dce110_vblank_set(struct irq_service *irq_service,
 			dc_interrupt_to_irq_source(irq_service->ctx->dc,
 						   info->src_id,
 						   info->ext_id);
-	uint8_t pipe_offset = dal_irq_src - IRQ_TYPE_VBLANK;
+	unsigned int pipe_offset = dal_irq_src - IRQ_TYPE_VBLANK;
 
 	struct timing_generator *tg;
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c
index 060460abc3777..ae6ed3a52d53b 100644
--- a/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c
+++ b/drivers/gpu/drm/amd/display/dc/link/accessories/link_dp_cts.c
@@ -199,7 +199,6 @@ static void dp_test_get_audio_test_data(struct dc_link *link, bool disable_video
 	unsigned int channel_count;
 	unsigned int channel = 0;
 	unsigned int modes = 0;
-	unsigned int sampling_rate_in_hz = 0;
 
 	// get audio test mode and test pattern parameters
 	core_link_read_dpcd(
@@ -232,38 +231,10 @@ static void dp_test_get_audio_test_data(struct dc_link *link, bool disable_video
 		}
 	}
 
-	// translate sampling rate
-	switch (dpcd_test_mode.bits.sampling_rate) {
-	case AUDIO_SAMPLING_RATE_32KHZ:
-		sampling_rate_in_hz = 32000;
-		break;
-	case AUDIO_SAMPLING_RATE_44_1KHZ:
-		sampling_rate_in_hz = 44100;
-		break;
-	case AUDIO_SAMPLING_RATE_48KHZ:
-		sampling_rate_in_hz = 48000;
-		break;
-	case AUDIO_SAMPLING_RATE_88_2KHZ:
-		sampling_rate_in_hz = 88200;
-		break;
-	case AUDIO_SAMPLING_RATE_96KHZ:
-		sampling_rate_in_hz = 96000;
-		break;
-	case AUDIO_SAMPLING_RATE_176_4KHZ:
-		sampling_rate_in_hz = 176400;
-		break;
-	case AUDIO_SAMPLING_RATE_192KHZ:
-		sampling_rate_in_hz = 192000;
-		break;
-	default:
-		sampling_rate_in_hz = 0;
-		break;
-	}
-
 	link->audio_test_data.flags.test_requested = 1;
 	link->audio_test_data.flags.disable_video = disable_video;
-	link->audio_test_data.sampling_rate = sampling_rate_in_hz;
-	link->audio_test_data.channel_count = channel_count;
+	link->audio_test_data.sampling_rate = (uint8_t)dpcd_test_mode.bits.sampling_rate;
+	link->audio_test_data.channel_count = (uint8_t)channel_count;
 	link->audio_test_data.pattern_type = test_pattern;
 
 	if (test_pattern == DP_TEST_PATTERN_AUDIO_SAWTOOTH) {
@@ -885,7 +856,7 @@ bool dp_set_test_pattern(
 				struct dmub_hw_lock_inst_flags inst_flags = { 0 };
 
 				hw_locks.bits.lock_dig = 1;
-				inst_flags.dig_inst = pipe_ctx->stream_res.tg->inst;
+				inst_flags.dig_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
 
 				dmub_hw_lock_mgr_cmd(link->ctx->dmub_srv,
 							true,
@@ -933,7 +904,7 @@ bool dp_set_test_pattern(
 				struct dmub_hw_lock_inst_flags inst_flags = { 0 };
 
 				hw_locks.bits.lock_dig = 1;
-				inst_flags.dig_inst = pipe_ctx->stream_res.tg->inst;
+				inst_flags.dig_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
 
 				dmub_hw_lock_mgr_cmd(link->ctx->dmub_srv,
 							false,
diff --git a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c
index dbbedeeed2983..04df75114dd5b 100644
--- a/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c
+++ b/drivers/gpu/drm/amd/display/dc/link/hwss/link_hwss_hpo_dp.c
@@ -63,7 +63,7 @@ void set_hpo_dp_hblank_min_symbol_width(struct pipe_ctx *pipe_ctx,
 		time_slot_in_ms = dc_fixpt_from_fraction(32 * 4, link_bw_in_kbps);
 		mtp_cnt_per_h_blank = dc_fixpt_div(h_blank_in_ms,
 				dc_fixpt_mul_int(time_slot_in_ms, 64));
-		hblank_min_symbol_width = dc_fixpt_floor(
+		hblank_min_symbol_width = (uint16_t)dc_fixpt_floor(
 				dc_fixpt_mul(mtp_cnt_per_h_blank, throttled_vcp_size));
 	}
 
@@ -98,7 +98,7 @@ void setup_hpo_dp_stream_attribute(struct pipe_ctx *pipe_ctx)
 			&stream->timing,
 			stream->output_color_space,
 			stream->use_vsc_sdp_for_colorimetry,
-			stream->timing.flags.DSC,
+			(stream->timing.flags.DSC != 0),
 			false);
 	link->dc->link_srv->dp_trace_source_sequence(link,
 			DPCD_SOURCE_SEQ_AFTER_DP_STREAM_ATTR);
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.c b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
index 794dd6a959183..1ac6a22fecfee 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.c
@@ -290,12 +290,12 @@ static bool i2c_read(
 	struct i2c_payload payloads[2] = {
 		{
 		.write = true,
-		.address = address,
+		.address = (uint8_t)address,
 		.length = 1,
 		.data = &offs_data },
 		{
 		.write = false,
-		.address = address,
+		.address = (uint8_t)address,
 		.length = len,
 		.data = buffer } };
 
@@ -524,7 +524,7 @@ static void read_current_link_settings_on_detect(struct dc_link *link)
 	uint8_t link_rate_set = 0;
 	uint32_t read_dpcd_retry_cnt = 10;
 	enum dc_status status = DC_ERROR_UNEXPECTED;
-	int i;
+	unsigned int i;
 	union max_down_spread max_down_spread = {0};
 
 	// Read DPCD 00101h to find out the number of lanes currently set
@@ -655,7 +655,7 @@ static bool wait_for_entering_dp_alt_mode(struct dc_link *link)
 	unsigned long long enter_timestamp;
 	unsigned long long finish_timestamp;
 	unsigned long long time_taken_in_ns;
-	int tries_taken;
+	unsigned int tries_taken;
 
 	/**
 	 * this function will only exist if we are on dcn21 (is_in_alt_mode is a
@@ -1562,7 +1562,7 @@ static bool link_add_remote_sink_helper(struct dc_link *dc_link, struct dc_sink
 struct dc_sink *link_add_remote_sink(
 		struct dc_link *link,
 		const uint8_t *edid,
-		int len,
+		unsigned int len,
 		struct dc_sink_init_data *init_data)
 {
 	struct dc_sink *dc_sink;
@@ -1619,7 +1619,7 @@ struct dc_sink *link_add_remote_sink(
 
 void link_remove_remote_sink(struct dc_link *link, struct dc_sink *sink)
 {
-	int i;
+	unsigned int i;
 
 	if (!link->sink_count) {
 		BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_detection.h b/drivers/gpu/drm/amd/display/dc/link/link_detection.h
index 1ab29476060b2..e8d29fa1550dc 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_detection.h
+++ b/drivers/gpu/drm/amd/display/dc/link/link_detection.h
@@ -32,7 +32,7 @@ bool link_detect_connection_type(struct dc_link *link,
 struct dc_sink *link_add_remote_sink(
 		struct dc_link *link,
 		const uint8_t *edid,
-		int len,
+		unsigned int len,
 		struct dc_sink_init_data *init_data);
 void link_remove_remote_sink(struct dc_link *link, struct dc_sink *sink);
 bool link_reset_cur_dp_mst_topology(struct dc_link *link);
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
index e12c25896364a..21f64946b993b 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c
@@ -546,24 +546,24 @@ static void update_psp_stream_config(struct pipe_ctx *pipe_ctx, bool dpms_off)
 	config.dig_fe = (uint8_t) pipe_ctx->stream_res.stream_enc->stream_enc_inst;
 
 	/* stream encoder index */
-	config.stream_enc_idx = pipe_ctx->stream_res.stream_enc->id - ENGINE_ID_DIGA;
+	config.stream_enc_idx = (uint8_t)(pipe_ctx->stream_res.stream_enc->id - ENGINE_ID_DIGA);
 	if (dp_is_128b_132b_signal(pipe_ctx))
 		config.stream_enc_idx =
-				pipe_ctx->stream_res.hpo_dp_stream_enc->id - ENGINE_ID_HPO_DP_0;
+				(uint8_t)(pipe_ctx->stream_res.hpo_dp_stream_enc->id - ENGINE_ID_HPO_DP_0);
 
 	/* dig back end */
 	config.dig_be = pipe_ctx->stream->link->link_enc_hw_inst;
 
 	/* link encoder index */
-	config.link_enc_idx = link_enc->transmitter - TRANSMITTER_UNIPHY_A;
+	config.link_enc_idx = (uint8_t)(link_enc->transmitter - TRANSMITTER_UNIPHY_A);
 	if (dp_is_128b_132b_signal(pipe_ctx))
-		config.link_enc_idx = pipe_ctx->link_res.hpo_dp_link_enc->inst;
+		config.link_enc_idx = (uint8_t)pipe_ctx->link_res.hpo_dp_link_enc->inst;
 
 	/* dio output index is dpia index for DPIA endpoint & dcio index by default */
 	if (pipe_ctx->stream->link->ep_type == DISPLAY_ENDPOINT_USB4_DPIA)
-		config.dio_output_idx = pipe_ctx->stream->link->link_id.enum_id - ENUM_ID_1;
+		config.dio_output_idx = (uint8_t)(pipe_ctx->stream->link->link_id.enum_id - ENUM_ID_1);
 	else
-		config.dio_output_idx = link_enc->transmitter - TRANSMITTER_UNIPHY_A;
+		config.dio_output_idx = (uint8_t)(link_enc->transmitter - TRANSMITTER_UNIPHY_A);
 
 
 	/* phy index */
@@ -1411,7 +1411,7 @@ static bool write_128b_132b_sst_payload_allocation_table(
 
 	if (allocate)	{
 		avg_time_slots_per_mtp = link_calculate_sst_avg_time_slots_per_mtp(stream, link);
-		req_slot_count = dc_fixpt_ceil(avg_time_slots_per_mtp);
+		req_slot_count = (uint8_t)dc_fixpt_ceil(avg_time_slots_per_mtp);
 		/// Validation should filter out modes that exceed link BW
 		ASSERT(req_slot_count <= MAX_MTP_SLOT_COUNT);
 		if (req_slot_count > MAX_MTP_SLOT_COUNT)
@@ -1811,7 +1811,7 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx)
 		write_scdc_data(
 			stream->link->ddc,
 			stream->phy_pix_clk,
-			stream->timing.flags.LTE_340MCSC_SCRAMBLE);
+			(stream->timing.flags.LTE_340MCSC_SCRAMBLE != 0));
 
 	memset(&stream->link->cur_link_settings, 0,
 			sizeof(struct dc_link_settings));
@@ -2016,9 +2016,7 @@ static enum dc_status enable_link_dp_mst(
 	return enable_link_dp(state, pipe_ctx);
 }
 
-static enum dc_status enable_link_analog(
-		struct dc_state *state,
-		struct pipe_ctx *pipe_ctx)
+static enum dc_status enable_link_analog(struct pipe_ctx *pipe_ctx)
 {
 	struct dc_link *link = pipe_ctx->stream->link;
 
@@ -2083,7 +2081,7 @@ static enum dc_status enable_link(
 		status = DC_OK;
 		break;
 	case SIGNAL_TYPE_RGB:
-		status = enable_link_analog(state, pipe_ctx);
+		status = enable_link_analog(pipe_ctx);
 		break;
 	case SIGNAL_TYPE_VIRTUAL:
 		status = enable_link_virtual(pipe_ctx);
@@ -2099,10 +2097,10 @@ static enum dc_status enable_link(
 	return status;
 }
 
-static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, int bw)
+static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, int stream_bw)
 {
 	struct dc_link *link = stream->sink->link;
-	int req_bw = bw;
+	int req_bw = stream_bw;
 
 	DC_LOGGER_INIT(link->ctx->logger);
 
@@ -2110,39 +2108,38 @@ static bool allocate_usb4_bandwidth_for_stream(struct dc_stream_state *stream, i
 		return false;
 
 	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
-		int sink_index = 0;
-		int i = 0;
+		int sink_index = -1;
+		unsigned int i = 0;
 
 		for (i = 0; i < link->sink_count; i++) {
 			if (link->remote_sinks[i] == NULL)
 				continue;
 
-			if (stream->sink->sink_id != link->remote_sinks[i]->sink_id)
+			if (stream->sink->sink_id != link->remote_sinks[i]->sink_id) {
+				DC_LOG_DEBUG("%s: add remote_sink=%s, request_bw=%d\n", __func__,
+					(const char *)(&link->remote_sinks[i]->edid_caps.display_name[0]),
+					link->dpia_bw_alloc_config.remote_sink_req_bw[i]);
+
 				req_bw += link->dpia_bw_alloc_config.remote_sink_req_bw[i];
-			else
+			} else
 				sink_index = i;
 		}
 
-		link->dpia_bw_alloc_config.remote_sink_req_bw[sink_index] = bw;
-	}
+		if (sink_index >= 0)
+			link->dpia_bw_alloc_config.remote_sink_req_bw[sink_index] = stream_bw;
+		else
+			DC_LOG_WARNING("%s: stream sink_id=%u not found in remote_sinks[]\n",
+				__func__, stream->sink->sink_id);
 
-	link->dpia_bw_alloc_config.dp_overhead = link_dpia_get_dp_overhead(link);
-	req_bw += link->dpia_bw_alloc_config.dp_overhead;
+		if (req_bw) {
+			link->dpia_bw_alloc_config.dp_overhead = link_dpia_get_dp_overhead(link);
+			req_bw += link->dpia_bw_alloc_config.dp_overhead;
+		} else
+			link->dpia_bw_alloc_config.dp_overhead = 0;
+	}
 
 	link_dp_dpia_allocate_usb4_bandwidth_for_stream(link, req_bw);
 
-	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT_MST) {
-		int i = 0;
-
-		for (i = 0; i < link->sink_count; i++) {
-			if (link->remote_sinks[i] == NULL)
-				continue;
-			DC_LOG_DEBUG("%s, remote_sink=%s, request_bw=%d\n", __func__,
-					(const char *)(&link->remote_sinks[i]->edid_caps.display_name[0]),
-					link->dpia_bw_alloc_config.remote_sink_req_bw[i]);
-		}
-	}
-
 	return true;
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_factory.c b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
index ae4c4ad05baa0..032c97660658a 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_factory.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_factory.c
@@ -381,7 +381,7 @@ static bool encoder_is_external_dp(
 
 static void link_destruct(struct dc_link *link)
 {
-	int i;
+	unsigned int i;
 
 	if (link->ddc)
 		link_destroy_ddc_service(&link->ddc);
@@ -516,7 +516,7 @@ static bool construct_phy(struct dc_link *link,
 	       sizeof(struct dc_link_settings));
 
 	link->link_id =
-		bios->funcs->get_connector_id(bios, init_params->connector_index);
+		bios->funcs->get_connector_id(bios, (uint8_t)init_params->connector_index);
 
 	link->ep_type = DISPLAY_ENDPOINT_PHY;
 
@@ -544,7 +544,7 @@ static bool construct_phy(struct dc_link *link,
 
 	if (bios->funcs->get_disp_connector_caps_info) {
 		bios->funcs->get_disp_connector_caps_info(bios, link->link_id, &disp_connect_caps_info);
-		link->is_internal_display = disp_connect_caps_info.INTERNAL_DISPLAY;
+		link->is_internal_display = (disp_connect_caps_info.INTERNAL_DISPLAY != 0);
 		DC_LOG_DC("BIOS object table - is_internal_display: %d", link->is_internal_display);
 	}
 
@@ -897,7 +897,7 @@ static bool construct_dpia(struct dc_link *link,
 	}
 
 	/* Set dpia port index : 0 to number of dpia ports */
-	link->ddc_hw_inst = init_params->connector_index;
+	link->ddc_hw_inst = (uint8_t)init_params->connector_index;
 
 	// Assign Dpia preferred eng_id
 	if (link->dc->res_pool->funcs->get_preferred_eng_id_dpia)
diff --git a/drivers/gpu/drm/amd/display/dc/link/link_validation.c b/drivers/gpu/drm/amd/display/dc/link/link_validation.c
index eb791285ed06a..ec083e8b75b4e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/link_validation.c
+++ b/drivers/gpu/drm/amd/display/dc/link/link_validation.c
@@ -302,9 +302,11 @@ static bool dp_validate_mode_timing(
 
 	req_bw = dc_bandwidth_in_kbps_from_timing(timing, dc_link_get_highest_encoding_format(link));
 	max_bw = dp_link_bandwidth_kbps(link, link_setting);
+	uint32_t max_uncompressed_pixel_rate_100hz =
+		link->dpcd_caps.max_uncompressed_pixel_rate_cap.bits.max_uncompressed_pixel_rate_cap * 10000U;
 
 	bool is_max_uncompressed_pixel_rate_exceeded = link->dpcd_caps.max_uncompressed_pixel_rate_cap.bits.valid &&
-			timing->pix_clk_100hz > link->dpcd_caps.max_uncompressed_pixel_rate_cap.bits.max_uncompressed_pixel_rate_cap * 10000;
+			timing->pix_clk_100hz > max_uncompressed_pixel_rate_100hz;
 
 	if (is_max_uncompressed_pixel_rate_exceeded && !timing->flags.DSC) {
 		return false;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c
index a66217e54a095..a0ea853f5f645 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_ddc.c
@@ -96,7 +96,7 @@ static void i2c_payloads_add(
 	for (pos = 0; pos < len; pos += payload_size) {
 		struct i2c_payload payload = {
 			.write = write,
-			.address = address,
+			.address = (uint8_t)address,
 			.length = DDC_MIN(payload_size, len - pos),
 			.data = data + pos };
 		dal_vector_append(&payloads->payloads, &payload);
@@ -384,8 +384,7 @@ bool link_query_ddc_data(
 		i2c_payloads_add(
 			&payloads, address, read_size, read_buf, false);
 
-		command.number_of_payloads =
-			i2c_payloads_get_count(&payloads);
+		command.number_of_payloads = (uint8_t)i2c_payloads_get_count(&payloads);
 
 		success = dm_helpers_submit_i2c(
 				ddc->ctx,
@@ -402,12 +401,7 @@ int link_aux_transfer_raw(struct ddc_service *ddc,
 		struct aux_payload *payload,
 		enum aux_return_code_type *operation_result)
 {
-	if (ddc->ctx->dc->debug.enable_dmub_aux_for_legacy_ddc ||
-	    !ddc->ddc_pin) {
-		return dce_aux_transfer_dmub_raw(ddc, payload, operation_result);
-	} else {
-		return dce_aux_transfer_raw(ddc, payload, operation_result);
-	}
+	return dce_aux_transfer_raw(ddc, payload, operation_result);
 }
 
 uint32_t link_get_fixed_vs_pe_retimer_write_address(struct dc_link *link)
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
index 782a45caa13d4..817b4010edcbe 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_capability.c
@@ -1353,7 +1353,7 @@ bool dp_overwrite_extended_receiver_cap(struct dc_link *link)
 	union down_stream_port_count down_strm_port_count;
 	union edp_configuration_cap edp_config_cap;
 
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < read_dpcd_retry_cnt; i++) {
 		status = core_link_read_dpcd(
@@ -1713,7 +1713,7 @@ enum dc_status dp_retrieve_lttpr_cap(struct dc_link *link)
 		CONN_DATA_DETECT(link, lttpr_dpcd_data, sizeof(lttpr_dpcd_data), "LTTPR Caps: ");
 
 		// Identify closest LTTPR to determine if workarounds required for known embedded LTTPR
-		closest_lttpr_offset = dp_get_closest_lttpr_offset(lttpr_count);
+		closest_lttpr_offset = dp_get_closest_lttpr_offset((uint8_t)lttpr_count);
 
 		core_link_read_dpcd(link, (DP_LTTPR_IEEE_OUI + closest_lttpr_offset),
 				link->dpcd_caps.lttpr_caps.lttpr_ieee_oui, sizeof(link->dpcd_caps.lttpr_caps.lttpr_ieee_oui));
@@ -1752,7 +1752,7 @@ static bool retrieve_link_cap(struct dc_link *link)
 	union dp_downstream_port_present ds_port = { 0 };
 	enum dc_status status = DC_ERROR_UNEXPECTED;
 	uint32_t read_dpcd_retry_cnt = 20;
-	int i;
+	unsigned int i;
 	struct dp_sink_hw_fw_revision dp_hw_fw_revision;
 	const uint32_t post_oui_delay = 30; // 30ms
 	bool is_fec_supported = false;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c
index 766b54631c798..da227889e007d 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia.c
@@ -119,7 +119,7 @@ bool dpia_query_hpd_status(struct dc_link *link)
 	/* prepare QUERY_HPD command */
 	cmd.query_hpd.header.type = DMUB_CMD__QUERY_HPD_STATE;
 	cmd.query_hpd.header.payload_bytes = sizeof(cmd.query_hpd.data);
-	cmd.query_hpd.data.instance = link->link_id.enum_id - ENUM_ID_1;
+	cmd.query_hpd.data.instance = (uint8_t)(link->link_id.enum_id - ENUM_ID_1);
 	cmd.query_hpd.data.ch_type = AUX_CHANNEL_DPIA;
 
 	/* Query dpia hpd status from dmub */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c
index c958d3f600c8f..dd854d9926925 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_dpia_bw.c
@@ -112,7 +112,7 @@ static int get_estimated_bw(struct dc_link *link)
 	return bw_estimated_bw * (Kbps_TO_Gbps / link->dpia_bw_alloc_config.bw_granularity);
 }
 
-static int get_non_reduced_max_link_rate(struct dc_link *link)
+static uint8_t get_non_reduced_max_link_rate(struct dc_link *link)
 {
 	uint8_t nrd_max_link_rate = 0;
 
@@ -125,7 +125,7 @@ static int get_non_reduced_max_link_rate(struct dc_link *link)
 	return nrd_max_link_rate;
 }
 
-static int get_non_reduced_max_lane_count(struct dc_link *link)
+static uint8_t get_non_reduced_max_lane_count(struct dc_link *link)
 {
 	uint8_t nrd_max_lane_count = 0;
 
@@ -180,7 +180,7 @@ static void dpia_bw_alloc_unplug(struct dc_link *link)
 
 static void link_dpia_send_bw_alloc_request(struct dc_link *link, int req_bw)
 {
-	uint8_t request_reg_val;
+	uint32_t request_reg_val;
 	uint32_t temp, request_bw;
 
 	if (link->dpia_bw_alloc_config.bw_granularity == 0) {
@@ -197,7 +197,7 @@ static void link_dpia_send_bw_alloc_request(struct dc_link *link, int req_bw)
 
 	request_bw = request_reg_val * (Kbps_TO_Gbps / link->dpia_bw_alloc_config.bw_granularity);
 
-	if (request_bw > link->dpia_bw_alloc_config.estimated_bw) {
+	if (request_bw > (uint32_t)link->dpia_bw_alloc_config.estimated_bw) {
 		DC_LOG_ERROR("%s:  Link[%d]:  Request BW (%d --> %d) > Estimated BW (%d)... Set to Estimated BW!",
 				__func__, link->link_index,
 				req_bw, request_bw, link->dpia_bw_alloc_config.estimated_bw);
@@ -212,8 +212,8 @@ static void link_dpia_send_bw_alloc_request(struct dc_link *link, int req_bw)
 	link->dpia_bw_alloc_config.allocated_bw = request_bw;
 	DC_LOG_DC("%s:  Link[%d]:  Request BW:  %d", __func__, link->link_index, request_bw);
 
-	core_link_write_dpcd(link, REQUESTED_BW,
-		&request_reg_val,
+	uint8_t requested_bw_dpcd = (uint8_t)request_reg_val;
+	core_link_write_dpcd(link, REQUESTED_BW, &requested_bw_dpcd,
 		sizeof(uint8_t));
 }
 
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
index 1860d44f63c1f..54ce768ae6ad9 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_irq_handler.c
@@ -65,7 +65,7 @@ bool dp_parse_link_loss_status(
 	/*1. Check that Link Status changed, before re-training.*/
 
 	/*parse lane status*/
-	for (lane = 0; lane < link->cur_link_settings.lane_count; lane++) {
+	for (lane = 0; lane < (uint32_t)link->cur_link_settings.lane_count; lane++) {
 		/* check status of lanes 0,1
 		 * changed DpcdAddress_Lane01Status (0x202)
 		 */
@@ -223,7 +223,7 @@ static void handle_hpd_irq_vesa_replay_sink(struct dc_link *link)
 	}
 }
 
-static void handle_hpd_irq_replay_sink(struct dc_link *link, bool *need_re_enable)
+static void handle_hpd_irq_replay_sink(struct dc_link *link, bool *need_re_enable, bool *replay_esd_detection_needed)
 {
 	union dpcd_replay_configuration replay_configuration = {0};
 	union dpcd_replay_configuration replay_sink_status = {0};
@@ -311,6 +311,16 @@ static void handle_hpd_irq_replay_sink(struct dc_link *link, bool *need_re_enabl
 			*need_re_enable = true;
 		}
 	}
+
+	if (link->ctx->dc->debug.enable_replay_esd_recovery) {
+		if (!link->replay_settings.replay_allow_active &&
+		    replay_sink_status.bits.SINK_DEVICE_REPLAY_STATUS == 0x7) {
+		    /* If sink device replay status is 0x7 and replay is disabled,
+		     * it means sink is in a bad state and link retraining is needed to recover
+		     */
+		    *replay_esd_detection_needed = true;
+		}
+	}
 }
 
 void dp_handle_link_loss(struct dc_link *link)
@@ -469,6 +479,7 @@ bool dp_handle_hpd_rx_irq(struct dc_link *link,
 	enum dc_status result;
 	bool status = false;
 	bool replay_re_enable_needed = false;
+	bool replay_esd_detection_needed = false;
 
 	if (out_link_loss)
 		*out_link_loss = false;
@@ -482,6 +493,7 @@ bool dp_handle_hpd_rx_irq(struct dc_link *link,
 	DC_LOG_HW_HPD_IRQ("%s: Got short pulse HPD on link %d\n",
 		__func__, link->link_index);
 
+	handle_hpd_irq_replay_sink(link, &replay_re_enable_needed, &replay_esd_detection_needed);
 
 	 /* All the "handle_hpd_irq_xxx()" methods
 		 * should be called only after
@@ -528,8 +540,6 @@ bool dp_handle_hpd_rx_irq(struct dc_link *link,
 		/* PSR-related error was detected and handled */
 		return true;
 
-	handle_hpd_irq_replay_sink(link, &replay_re_enable_needed);
-
 	/* If PSR-related error handled, Main link may be off,
 	 * so do not handle as a normal sink status change interrupt.
 	 */
@@ -552,27 +562,30 @@ bool dp_handle_hpd_rx_irq(struct dc_link *link,
 	 * Downstream port status changed,
 	 * then DM should call DC to do the detection.
 	 * NOTE: Now includes eDP link loss detection and retraining
+	 * Link will be retrained if panel is not EDP or
+	 * Replay ESD recovery is needed.
 	 */
-
-	if (dp_parse_link_loss_status(
-			link,
-			&hpd_irq_dpcd_data)) {
-		/* Connectivity log: link loss */
-		CONN_DATA_LINK_LOSS(link,
-					hpd_irq_dpcd_data.raw,
-					sizeof(hpd_irq_dpcd_data),
-					"Status: ");
-
-		if (defer_handling && has_left_work)
-			*has_left_work = true;
-		else
-			dp_handle_link_loss(link);
-
-		status = false;
-		if (out_link_loss)
-			*out_link_loss = true;
-
-		dp_trace_link_loss_increment(link);
+	if (link->connector_signal != SIGNAL_TYPE_EDP || replay_esd_detection_needed) {
+		if (dp_parse_link_loss_status(
+				link,
+				&hpd_irq_dpcd_data)) {
+			/* Connectivity log: link loss */
+			CONN_DATA_LINK_LOSS(link,
+						hpd_irq_dpcd_data.raw,
+						sizeof(hpd_irq_dpcd_data),
+						"Status: ");
+
+			if (defer_handling && has_left_work)
+				*has_left_work = true;
+			else
+				dp_handle_link_loss(link);
+
+			status = false;
+			if (out_link_loss)
+				*out_link_loss = true;
+
+			dp_trace_link_loss_increment(link);
+		}
 	}
 
 	if (link->dpcd_caps.usb4_dp_tun_info.dp_tun_cap.bits.dp_tunneling) {
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
index 96afce4ffbfa2..d87f87a02d63e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_panel_replay.c
@@ -60,11 +60,10 @@ static void dp_pr_set_static_screen_param(struct dc_link *link)
 		if (dc->current_state->res_ctx.pipe_ctx[i].stream &&
 			dc->current_state->res_ctx.pipe_ctx[i].stream->link == link) {
 			struct dc_stream_state *stream = dc->current_state->res_ctx.pipe_ctx[i].stream;
-			unsigned int vsync_rate_hz = div64_u64(div64_u64(
-											(stream->timing.pix_clk_100hz * (u64)100),
-											stream->timing.v_total),
-											stream->timing.h_total);
-
+			unsigned int vsync_rate_hz = (unsigned int)div64_u64(div64_u64(
+														(stream->timing.pix_clk_100hz * (u64)100),
+														stream->timing.v_total),
+														stream->timing.h_total);
 			params.triggers.cursor_update = true;
 			params.triggers.overlay_update = true;
 			params.triggers.surface_update = true;
@@ -264,7 +263,7 @@ bool dp_pr_enable(struct dc_link *link, bool enable)
 	cmd.pr_enable.header.type = DMUB_CMD__PR;
 	cmd.pr_enable.header.sub_type = DMUB_CMD__PR_ENABLE;
 	cmd.pr_enable.header.payload_bytes = sizeof(struct dmub_cmd_pr_enable_data);
-	cmd.pr_enable.data.panel_inst = panel_inst;
+	cmd.pr_enable.data.panel_inst = (uint8_t)panel_inst;
 	cmd.pr_enable.data.enable = enable ? 1 : 0;
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 
@@ -301,17 +300,17 @@ bool dp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_con
 	cmd.pr_copy_settings.header.type = DMUB_CMD__PR;
 	cmd.pr_copy_settings.header.sub_type = DMUB_CMD__PR_COPY_SETTINGS;
 	cmd.pr_copy_settings.header.payload_bytes = sizeof(struct dmub_cmd_pr_copy_settings_data);
-	cmd.pr_copy_settings.data.panel_inst = panel_inst;
+	cmd.pr_copy_settings.data.panel_inst = (uint8_t)panel_inst;
 	// HW inst
 	cmd.pr_copy_settings.data.aux_inst = replay_context->aux_inst;
 	cmd.pr_copy_settings.data.digbe_inst = replay_context->digbe_inst;
 	cmd.pr_copy_settings.data.digfe_inst = replay_context->digfe_inst;
 	if (pipe_ctx->plane_res.dpp)
-		cmd.pr_copy_settings.data.dpp_inst = pipe_ctx->plane_res.dpp->inst;
+		cmd.pr_copy_settings.data.dpp_inst = (uint8_t)pipe_ctx->plane_res.dpp->inst;
 	else
 		cmd.pr_copy_settings.data.dpp_inst = 0;
 	if (pipe_ctx->stream_res.tg)
-		cmd.pr_copy_settings.data.otg_inst = pipe_ctx->stream_res.tg->inst;
+		cmd.pr_copy_settings.data.otg_inst = (uint8_t)pipe_ctx->stream_res.tg->inst;
 	else
 		cmd.pr_copy_settings.data.otg_inst = 0;
 
@@ -322,6 +321,19 @@ bool dp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_con
 	cmd.pr_copy_settings.data.flags.bitfields.dsc_enable_status = (pipe_ctx->stream->timing.flags.DSC == 1);
 	cmd.pr_copy_settings.data.debug.u32All = link->replay_settings.config.debug_flags;
 
+	// ALPM settings
+	cmd.pr_copy_settings.data.flags.bitfields.alpm_mode = (enum dmub_alpm_mode)link->replay_settings.config.alpm_mode;
+	if (link->replay_settings.config.alpm_mode == DC_ALPM_AUXLESS) {
+		cmd.pr_copy_settings.data.auxless_alpm_data.lfps_setup_ns = (uint16_t)dc->debug.auxless_alpm_lfps_setup_ns;
+		cmd.pr_copy_settings.data.auxless_alpm_data.lfps_period_ns = (uint16_t)dc->debug.auxless_alpm_lfps_period_ns;
+		cmd.pr_copy_settings.data.auxless_alpm_data.lfps_silence_ns = (uint16_t)dc->debug.auxless_alpm_lfps_silence_ns;
+		cmd.pr_copy_settings.data.auxless_alpm_data.lfps_t1_t2_override_us =
+			(uint16_t)dc->debug.auxless_alpm_lfps_t1t2_us;
+		cmd.pr_copy_settings.data.auxless_alpm_data.lfps_t1_t2_offset_us =
+			(uint16_t)dc->debug.auxless_alpm_lfps_t1t2_offset_us;
+		cmd.pr_copy_settings.data.auxless_alpm_data.lttpr_count = link->dc->link_srv->dp_get_lttpr_count(link);
+	}
+
 	cmd.pr_copy_settings.data.su_granularity_needed = link->dpcd_caps.vesa_replay_caps.bits.PR_SU_GRANULARITY_NEEDED;
 	cmd.pr_copy_settings.data.su_x_granularity = link->dpcd_caps.vesa_replay_su_info.pr_su_x_granularity;
 	cmd.pr_copy_settings.data.su_y_granularity = link->dpcd_caps.vesa_replay_su_info.pr_su_y_granularity;
@@ -329,9 +341,9 @@ bool dp_pr_copy_settings(struct dc_link *link, struct replay_context *replay_con
 		link->dpcd_caps.vesa_replay_su_info.pr_su_y_granularity_extended_caps;
 
 	if (pipe_ctx->stream->timing.dsc_cfg.num_slices_v > 0)
-		cmd.pr_copy_settings.data.dsc_slice_height = (pipe_ctx->stream->timing.v_addressable +
+		cmd.pr_copy_settings.data.dsc_slice_height = (uint16_t)((pipe_ctx->stream->timing.v_addressable +
 			pipe_ctx->stream->timing.v_border_top + pipe_ctx->stream->timing.v_border_bottom) /
-			pipe_ctx->stream->timing.dsc_cfg.num_slices_v;
+			pipe_ctx->stream->timing.dsc_cfg.num_slices_v);
 
 	if (dc_is_embedded_signal(link->connector_signal))
 		cmd.pr_copy_settings.data.main_link_activity_option = OPTION_1C;
@@ -358,7 +370,7 @@ bool dp_pr_update_state(struct dc_link *link, struct dmub_cmd_pr_update_state_da
 	cmd.pr_update_state.header.type = DMUB_CMD__PR;
 	cmd.pr_update_state.header.sub_type = DMUB_CMD__PR_UPDATE_STATE;
 	cmd.pr_update_state.header.payload_bytes = sizeof(struct dmub_cmd_pr_update_state_data);
-	cmd.pr_update_state.data.panel_inst = panel_inst;
+	cmd.pr_update_state.data.panel_inst = (uint8_t)panel_inst;
 
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 	return true;
@@ -379,7 +391,7 @@ bool dp_pr_set_general_cmd(struct dc_link *link, struct dmub_cmd_pr_general_cmd_
 	cmd.pr_general_cmd.header.type = DMUB_CMD__PR;
 	cmd.pr_general_cmd.header.sub_type = DMUB_CMD__PR_GENERAL_CMD;
 	cmd.pr_general_cmd.header.payload_bytes = sizeof(struct dmub_cmd_pr_general_cmd_data);
-	cmd.pr_general_cmd.data.panel_inst = panel_inst;
+	cmd.pr_general_cmd.data.panel_inst = (uint8_t)panel_inst;
 
 	dc_wake_and_execute_dmub_cmd(dc->ctx, &cmd, DM_DMUB_WAIT_TYPE_WAIT);
 	return true;
@@ -397,7 +409,7 @@ bool dp_pr_get_state(const struct dc_link *link, uint64_t *state)
 
 	do {
 		// Send gpint command and wait for ack
-		if (!dc_wake_and_execute_gpint(dc->ctx, DMUB_GPINT__GET_REPLAY_STATE, panel_inst,
+		if (!dc_wake_and_execute_gpint(dc->ctx, DMUB_GPINT__GET_REPLAY_STATE, (uint16_t)panel_inst,
 					       &replay_state, DM_DMUB_WAIT_TYPE_WAIT_WITH_REPLY)) {
 			// Return invalid state when GPINT times out
 			replay_state = PR_STATE_INVALID;
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c
index ddff0db4ce706..605bf19dc4f2e 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training.c
@@ -310,7 +310,7 @@ static void maximize_lane_settings(const struct link_training_settings *lt_setti
 	max_requested.FFE_PRESET = lane_settings[0].FFE_PRESET;
 
 	/* Determine what the maximum of the requested settings are*/
-	for (lane = 1; lane < lt_settings->link_settings.lane_count; lane++) {
+	for (lane = 1; lane < (uint32_t)lt_settings->link_settings.lane_count; lane++) {
 		if (lane_settings[lane].VOLTAGE_SWING > max_requested.VOLTAGE_SWING)
 			max_requested.VOLTAGE_SWING = lane_settings[lane].VOLTAGE_SWING;
 
@@ -568,7 +568,7 @@ enum link_training_result dp_check_link_loss_status(
 			sizeof(dpcd_buf));
 
 	/*parse lane status*/
-	for (lane = 0; lane < link->cur_link_settings.lane_count; lane++) {
+	for (lane = 0; lane < (uint32_t)link->cur_link_settings.lane_count; lane++) {
 		/*
 		 * check lanes status
 		 */
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c
index 66d0fb1b9b9d2..4331e032416fc 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_dp_training_8b_10b.c
@@ -132,7 +132,7 @@ void decide_8b_10b_training_settings(
 	 */
 	lt_settings->link_settings.link_spread = link->dp_ss_off ?
 			LINK_SPREAD_DISABLED : LINK_SPREAD_05_DOWNSPREAD_30KHZ;
-	lt_settings->eq_pattern_time = get_eq_training_aux_rd_interval(link, link_setting);
+	lt_settings->eq_pattern_time = (uint16_t)get_eq_training_aux_rd_interval(link, link_setting);
 	lt_settings->pattern_for_cr = decide_cr_training_pattern(link_setting);
 	lt_settings->pattern_for_eq = decide_eq_training_pattern(link, link_res, link_setting);
 	lt_settings->enhanced_framing = 1;
@@ -140,7 +140,7 @@ void decide_8b_10b_training_settings(
 	lt_settings->disallow_per_lane_settings = true;
 	lt_settings->always_match_dpcd_with_hw_lane_settings = true;
 	lt_settings->lttpr_mode = dp_decide_8b_10b_lttpr_mode(link);
-	lt_settings->cr_pattern_time = get_cr_training_aux_rd_interval(link, link_setting, lt_settings->lttpr_mode);
+	lt_settings->cr_pattern_time = (uint16_t)get_cr_training_aux_rd_interval(link, link_setting, lt_settings->lttpr_mode);
 	dp_hw_to_dpcd_lane_settings(lt_settings, lt_settings->hw_lane_settings, lt_settings->dpcd_lane_settings);
 
 	/* Some embedded LTTPRs rely on receiving TPS2 before LT to interop reliably with sensitive VGA dongles
@@ -195,7 +195,7 @@ static void set_link_settings_and_perform_early_tps2_retimer_pre_lt_sequence(str
 	* 6. Begin link training as usual
 	* */
 
-	uint32_t closest_lttpr_address_offset = dp_get_closest_lttpr_offset(lttpr_count);
+	uint32_t closest_lttpr_address_offset = dp_get_closest_lttpr_offset((uint8_t)lttpr_count);
 
 	union dpcd_training_pattern dpcd_pattern = {0};
 
@@ -379,7 +379,7 @@ enum link_training_result perform_8b_10b_channel_equalization_sequence(
 			dpcd_set_lane_settings(link, lt_settings, offset);
 
 		/* 3. wait for receiver to lock-on*/
-		wait_time_microsec = dp_get_eq_aux_rd_interval(link, lt_settings, offset, retries_ch_eq);
+		wait_time_microsec = dp_get_eq_aux_rd_interval(link, lt_settings, offset, (uint8_t)retries_ch_eq);
 
 		dp_wait_for_training_aux_rd_interval(
 				link,
@@ -408,7 +408,7 @@ enum link_training_result perform_8b_10b_channel_equalization_sequence(
 		/* 6. check CHEQ done*/
 		if (dp_is_ch_eq_done(lane_count, dpcd_lane_status) &&
 				dp_is_symbol_locked(lane_count, dpcd_lane_status) &&
-				dp_check_interlane_aligned(dpcd_lane_status_updated, link, retries_ch_eq))
+				dp_check_interlane_aligned(dpcd_lane_status_updated, link, (uint8_t)retries_ch_eq))
 			return LINK_TRAINING_SUCCESS;
 
 		/* 7. update VS/PE/PC2 in lt_settings*/
diff --git a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
index 4a2699a374b7e..e06a9ac652860 100644
--- a/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
+++ b/drivers/gpu/drm/amd/display/dc/link/protocols/link_edp_panel_control.c
@@ -47,9 +47,9 @@
 #define DP_SINK_PR_ENABLE_AND_CONFIGURATION		0x37B
 
 /* Travis */
-static const uint8_t DP_VGA_LVDS_CONVERTER_ID_2[] = "sivarT";
+static const char DP_VGA_LVDS_CONVERTER_ID_2[] = "sivarT";
 /* Nutmeg */
-static const uint8_t DP_VGA_LVDS_CONVERTER_ID_3[] = "dnomlA";
+static const char DP_VGA_LVDS_CONVERTER_ID_3[] = "dnomlA";
 
 void dp_set_panel_mode(struct dc_link *link, enum dp_panel_mode panel_mode)
 {
@@ -462,7 +462,7 @@ bool edp_receiver_ready_T9(struct dc_link *link)
 		do {
 			sinkstatus = 1;
 			result = core_link_read_dpcd(link, DP_SINK_STATUS, &sinkstatus, sizeof(sinkstatus));
-			if (sinkstatus == 0)
+			if (!(sinkstatus & DP_RECEIVE_PORT_0_STATUS))
 				break;
 			if (result != DC_OK)
 				break;
@@ -492,7 +492,7 @@ bool edp_receiver_ready_T7(struct dc_link *link)
 		do {
 			sinkstatus = 0;
 			result = core_link_read_dpcd(link, DP_SINK_STATUS, &sinkstatus, sizeof(sinkstatus));
-			if (sinkstatus == 1)
+			if (sinkstatus & DP_RECEIVE_PORT_0_STATUS)
 				break;
 			if (result != DC_OK)
 				break;
@@ -601,12 +601,12 @@ bool edp_set_psr_allow_active(struct dc_link *link, const bool *allow_active,
 		link->psr_settings.psr_power_opt = *power_opts;
 
 		if (psr != NULL && link->psr_settings.psr_feature_enabled && psr->funcs->psr_set_power_opt)
-			psr->funcs->psr_set_power_opt(psr, link->psr_settings.psr_power_opt, panel_inst);
+			psr->funcs->psr_set_power_opt(psr, link->psr_settings.psr_power_opt, (uint8_t)panel_inst);
 	}
 
 	if (psr != NULL && link->psr_settings.psr_feature_enabled &&
 			force_static && psr->funcs->psr_force_static)
-		psr->funcs->psr_force_static(psr, panel_inst);
+		psr->funcs->psr_force_static(psr, (uint8_t)panel_inst);
 
 	/* Enable or Disable PSR */
 	if (allow_active && link->psr_settings.psr_allow_active != *allow_active) {
@@ -615,9 +615,9 @@ bool edp_set_psr_allow_active(struct dc_link *link, const bool *allow_active,
 		if (!link->psr_settings.psr_allow_active)
 			dc_z10_restore(dc);
 
-		if (psr != NULL && link->psr_settings.psr_feature_enabled) {
-			psr->funcs->psr_enable(psr, link->psr_settings.psr_allow_active, wait, panel_inst);
-		} else if ((dmcu != NULL && dmcu->funcs->is_dmcu_initialized(dmcu)) &&
+		if (psr != NULL && link->psr_settings.psr_feature_enabled)
+			psr->funcs->psr_enable(psr, link->psr_settings.psr_allow_active, wait, (uint8_t)panel_inst);
+		else if ((dmcu != NULL && dmcu->funcs->is_dmcu_initialized(dmcu)) &&
 			link->psr_settings.psr_feature_enabled)
 			dmcu->funcs->set_psr_enable(dmcu, link->psr_settings.psr_allow_active, wait);
 		else
@@ -637,7 +637,7 @@ bool edp_get_psr_state(const struct dc_link *link, enum dc_psr_state *state)
 		return false;
 
 	if (psr != NULL && link->psr_settings.psr_feature_enabled)
-		psr->funcs->psr_get_state(psr, state, panel_inst);
+		psr->funcs->psr_get_state(psr, state, (uint8_t)panel_inst);
 	else if (dmcu != NULL && link->psr_settings.psr_feature_enabled)
 		dmcu->funcs->get_psr_state(dmcu, state);
 
@@ -811,7 +811,7 @@ bool edp_setup_psr(struct dc_link *link,
 	psr_context->smuPhyId = transmitter_to_phy_id(link);
 
 	psr_context->crtcTimingVerticalTotal = stream->timing.v_total;
-	psr_context->vsync_rate_hz = div64_u64(div64_u64((stream->
+	psr_context->vsync_rate_hz = (unsigned int)div64_u64(div64_u64((stream->
 					timing.pix_clk_100hz * (u64)100),
 					stream->timing.v_total),
 					stream->timing.h_total);
@@ -885,7 +885,7 @@ bool edp_setup_psr(struct dc_link *link,
 
 	if (psr) {
 		link->psr_settings.psr_feature_enabled = psr->funcs->psr_copy_settings(psr,
-			link, psr_context, panel_inst);
+			link, psr_context, (uint8_t)panel_inst);
 		link->psr_settings.psr_power_opt = 0;
 		link->psr_settings.psr_allow_active = 0;
 	} else {
@@ -913,7 +913,7 @@ void edp_get_psr_residency(const struct dc_link *link, uint32_t *residency, enum
 
 	// PSR residency measurements only supported on DMCUB
 	if (psr != NULL && link->psr_settings.psr_feature_enabled)
-		psr->funcs->psr_get_residency(psr, residency, panel_inst, mode);
+		psr->funcs->psr_get_residency(psr, residency, (uint8_t)panel_inst, mode);
 	else
 		*residency = 0;
 }
@@ -947,7 +947,7 @@ bool edp_set_replay_allow_active(struct dc_link *link, const bool *allow_active,
 	if (power_opts && link->replay_settings.replay_power_opt_active != *power_opts) {
 		if (replay != NULL && link->replay_settings.replay_feature_enabled &&
 			replay->funcs->replay_set_power_opt) {
-			replay->funcs->replay_set_power_opt(replay, *power_opts, panel_inst);
+			replay->funcs->replay_set_power_opt(replay, *power_opts, (uint8_t)panel_inst);
 			link->replay_settings.replay_power_opt_active = *power_opts;
 		}
 	}
@@ -957,7 +957,7 @@ bool edp_set_replay_allow_active(struct dc_link *link, const bool *allow_active,
 		// TODO: Handle mux change case if force_static is set
 		// If force_static is set, just change the replay_allow_active state directly
 		if (replay != NULL && link->replay_settings.replay_feature_enabled)
-			replay->funcs->replay_enable(replay, *allow_active, wait, panel_inst);
+			replay->funcs->replay_enable(replay, *allow_active, wait, (uint8_t)panel_inst);
 		link->replay_settings.replay_allow_active = *allow_active;
 	}
 
@@ -975,7 +975,7 @@ bool edp_get_replay_state(const struct dc_link *link, uint64_t *state)
 		return false;
 
 	if (replay != NULL && link->replay_settings.replay_feature_enabled)
-		replay->funcs->replay_get_state(replay, &pr_state, panel_inst);
+		replay->funcs->replay_get_state(replay, &pr_state, (uint8_t)panel_inst);
 	*state = pr_state;
 
 	return true;
@@ -1046,12 +1046,11 @@ bool edp_setup_freesync_replay(struct dc_link *link, const struct dc_stream_stat
 	replay_context.os_request_force_ffu = link->replay_settings.config.os_request_force_ffu;
 
 	link->replay_settings.replay_feature_enabled =
-			replay->funcs->replay_copy_settings(replay, link, &replay_context, panel_inst);
+			replay->funcs->replay_copy_settings(replay, link, &replay_context, (uint8_t)panel_inst);
 	if (link->replay_settings.replay_feature_enabled) {
 
 		replay_config.bits.FREESYNC_PANEL_REPLAY_MODE = 1;
-		replay_config.bits.TIMING_DESYNC_ERROR_VERIFICATION =
-			link->replay_settings.config.replay_timing_sync_supported;
+		replay_config.bits.TIMING_DESYNC_ERROR_VERIFICATION = 0;
 		replay_config.bits.STATE_TRANSITION_ERROR_DETECTION = 1;
 		dm_helpers_dp_write_dpcd(link->ctx, link,
 			DP_SINK_PR_ENABLE_AND_CONFIGURATION,
@@ -1096,7 +1095,7 @@ bool edp_send_replay_cmd(struct dc_link *link,
 		return false;
 
 	if (dp_pr_get_panel_inst(dc, link, &panel_inst))
-		cmd_data->panel_inst = panel_inst;
+		cmd_data->panel_inst = (uint8_t)panel_inst;
 	else {
 		DC_LOG_DC("%s(): get edp panel inst fail ", __func__);
 		return false;
@@ -1121,7 +1120,7 @@ bool edp_set_coasting_vtotal(struct dc_link *link, uint32_t coasting_vtotal, uin
 
 	if (coasting_vtotal && (link->replay_settings.coasting_vtotal != coasting_vtotal ||
 		link->replay_settings.frame_skip_number != frame_skip_number)) {
-		replay->funcs->replay_set_coasting_vtotal(replay, coasting_vtotal, panel_inst, frame_skip_number);
+		replay->funcs->replay_set_coasting_vtotal(replay, coasting_vtotal, (uint8_t)panel_inst, frame_skip_number);
 		link->replay_settings.coasting_vtotal = coasting_vtotal;
 		link->replay_settings.frame_skip_number = frame_skip_number;
 	}
@@ -1143,7 +1142,7 @@ bool edp_replay_residency(const struct dc_link *link,
 		return false;
 
 	if (replay != NULL && link->replay_settings.replay_feature_enabled)
-		replay->funcs->replay_residency(replay, panel_inst, residency, is_start, mode);
+		replay->funcs->replay_residency(replay, (uint8_t)panel_inst, residency, is_start, mode);
 	else
 		*residency = 0;
 
@@ -1168,7 +1167,7 @@ bool edp_set_replay_power_opt_and_coasting_vtotal(struct dc_link *link,
 		if (link->replay_settings.replay_feature_enabled &&
 			replay->funcs->replay_set_power_opt_and_coasting_vtotal) {
 			replay->funcs->replay_set_power_opt_and_coasting_vtotal(replay,
-				*power_opts, panel_inst, coasting_vtotal, frame_skip_number);
+				*power_opts, (uint8_t)panel_inst, coasting_vtotal, frame_skip_number);
 			link->replay_settings.replay_power_opt_active = *power_opts;
 			link->replay_settings.coasting_vtotal = coasting_vtotal;
 			link->replay_settings.frame_skip_number = frame_skip_number;
@@ -1211,9 +1210,9 @@ int edp_get_backlight_level(const struct dc_link *link)
 		fw_set_brightness = dmcu->funcs->is_dmcu_initialized(dmcu);
 
 	if (!fw_set_brightness && panel_cntl->funcs->get_current_backlight)
-		return panel_cntl->funcs->get_current_backlight(panel_cntl);
+		return (int)panel_cntl->funcs->get_current_backlight(panel_cntl);
 	else if (abm != NULL && abm->funcs->get_current_backlight != NULL)
-		return (int) abm->funcs->get_current_backlight(abm);
+		return (int)abm->funcs->get_current_backlight(abm);
 	else
 		return DC_ERROR_UNEXPECTED;
 }
@@ -1252,10 +1251,10 @@ static void edp_set_assr_enable(const struct dc *pDC, struct dc_link *link,
 
 	memset(&cmd, 0, sizeof(cmd));
 
-	link_enc_index = link->link_enc->transmitter - TRANSMITTER_UNIPHY_A;
+	link_enc_index = (uint8_t)(link->link_enc->transmitter - TRANSMITTER_UNIPHY_A);
 
 	if (link_res->hpo_dp_link_enc) {
-		link_enc_index = link_res->hpo_dp_link_enc->inst;
+		link_enc_index = (uint8_t)link_res->hpo_dp_link_enc->inst;
 		use_hpo_dp_link_enc = true;
 	}
 
diff --git a/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn20/dcn20_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn20/dcn20_mmhubbub.c
index 2a422e223bf2a..2e0f07ec04e4d 100644
--- a/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn20/dcn20_mmhubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn20/dcn20_mmhubbub.c
@@ -40,8 +40,8 @@
 #define FN(reg_name, field_name) \
 	mcif_wb20->mcif_wb_shift->field_name, mcif_wb20->mcif_wb_mask->field_name
 
-#define MCIF_ADDR(addr) (((unsigned long long)addr & 0xffffffffff) + 0xFE) >> 8
-#define MCIF_ADDR_HIGH(addr) (unsigned long long)addr >> 40
+#define MCIF_ADDR(addr) ((uint32_t)((((unsigned long long)(addr) & 0xffffffffffULL) + 0xFEULL) >> 8))
+#define MCIF_ADDR_HIGH(addr) ((uint32_t)(((unsigned long long)(addr)) >> 40))
 
 /* wbif programming guide:
  * 1. set up wbif parameter:
diff --git a/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn32/dcn32_mmhubbub.c b/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn32/dcn32_mmhubbub.c
index c3b089ba511a0..6b6f80a3bfd57 100644
--- a/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn32/dcn32_mmhubbub.c
+++ b/drivers/gpu/drm/amd/display/dc/mmhubbub/dcn32/dcn32_mmhubbub.c
@@ -40,8 +40,8 @@
 #define FN(reg_name, field_name) \
 	mcif_wb30->mcif_wb_shift->field_name, mcif_wb30->mcif_wb_mask->field_name
 
-#define MCIF_ADDR(addr) (((unsigned long long)addr & 0xffffffffff) + 0xFE) >> 8
-#define MCIF_ADDR_HIGH(addr) (unsigned long long)addr >> 40
+#define MCIF_ADDR(addr) ((uint32_t)((((unsigned long long)(addr) & 0xffffffffffULL) + 0xFEULL) >> 8))
+#define MCIF_ADDR_HIGH(addr) ((uint32_t)(((unsigned long long)(addr)) >> 40))
 
 /* wbif programming guide:
  * 1. set up wbif parameter:
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c
index 27e6532348509..0f2294bedcc33 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn10/dcn10_mpc.c
@@ -375,7 +375,7 @@ void mpc1_mpc_init(struct mpc *mpc)
 void mpc1_mpc_init_single_inst(struct mpc *mpc, unsigned int mpcc_id)
 {
 	struct dcn10_mpc *mpc10 = TO_DCN10_MPC(mpc);
-	int opp_id;
+	uint32_t opp_id;
 
 	REG_GET(MPCC_OPP_ID[mpcc_id], MPCC_OPP_ID, &opp_id);
 
@@ -422,7 +422,7 @@ void mpc1_init_mpcc_list_from_hw(
 
 				if (out_mux == mpcc_id)
 					tree->opp_list = mpcc;
-				if (bot_sel != 0xf && bot_sel < mpc10->num_mpcc) {
+				if (bot_sel != 0xf && bot_sel < (unsigned int)mpc10->num_mpcc) {
 					bot_mpcc_id = bot_sel;
 					REG_GET(MPCC_OPP_ID[bot_mpcc_id],  MPCC_OPP_ID,  &opp_id);
 					REG_GET(MPCC_TOP_SEL[bot_mpcc_id], MPCC_TOP_SEL, &top_sel);
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c
index 4e91e9f6f11a2..d7a07e29d23a0 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn30/dcn30_mpc.c
@@ -1067,7 +1067,7 @@ static void program_gamut_remap(
 		struct dcn30_mpc *mpc30,
 		int mpcc_id,
 		const uint16_t *regval,
-		int select)
+		uint32_t select)
 {
 	uint16_t selection = 0;
 	struct color_matrices_reg gam_regs;
@@ -1129,7 +1129,7 @@ void mpc3_set_gamut_remap(
 {
 	struct dcn30_mpc *mpc30 = TO_DCN30_MPC(mpc);
 	int i = 0;
-	int gamut_mode;
+	uint32_t gamut_mode;
 
 	if (adjust->gamut_adjust_type != GRAPHICS_GAMUT_ADJUST_TYPE_SW)
 		program_gamut_remap(mpc30, mpcc_id, NULL, GAMUT_REMAP_BYPASS);
@@ -1201,7 +1201,7 @@ void mpc3_get_gamut_remap(struct mpc *mpc,
 {
 	struct dcn30_mpc *mpc30 = TO_DCN30_MPC(mpc);
 	uint16_t arr_reg_val[12] = {0};
-	int select;
+	uint32_t select;
 
 	read_gamut_remap(mpc30, mpcc_id, arr_reg_val, &select);
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c
index e0617db2d0c1d..ce1ee2062e41d 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.c
@@ -73,15 +73,56 @@ void mpc401_set_movable_cm_location(struct mpc *mpc, enum mpcc_movable_cm_locati
 	}
 }
 
-void mpc401_populate_lut(struct mpc *mpc,
-		const enum MCM_LUT_ID id,
-		const union mcm_lut_params *params,
-		const bool lut_bank_a,
-		const int mpcc_id)
+static enum dc_lut_mode get3dlut_config(
+			struct mpc *mpc,
+			bool *is_17x17x17,
+			bool *is_12bits_color_channel,
+			int mpcc_id)
+{
+	uint32_t i_mode, i_enable_10bits, lut_size;
+	enum dc_lut_mode mode;
+	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
+
+	REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id],
+			MPCC_MCM_3DLUT_MODE_CURRENT,  &i_mode);
+
+	REG_GET(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
+			MPCC_MCM_3DLUT_30BIT_EN, &i_enable_10bits);
+
+	switch (i_mode) {
+	case 0:
+		mode = LUT_BYPASS;
+		break;
+	case 1:
+		mode = LUT_RAM_A;
+		break;
+	case 2:
+		mode = LUT_RAM_B;
+		break;
+	default:
+		mode = LUT_BYPASS;
+		break;
+	}
+	if (i_enable_10bits > 0)
+		*is_12bits_color_channel = false;
+	else
+		*is_12bits_color_channel = true;
+
+	REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_SIZE, &lut_size);
+
+	if (lut_size == 0)
+		*is_17x17x17 = true;
+	else
+		*is_17x17x17 = false;
+
+	return mode;
+}
+
+void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union mcm_lut_params params, bool lut_bank_a, int mpcc_id)
 {
 	const enum dc_lut_mode next_mode = lut_bank_a ? LUT_RAM_A : LUT_RAM_B;
-	const struct pwl_params *lut1d = params->pwl;
-	const struct pwl_params *lut_shaper = params->pwl;
+	const struct pwl_params *lut1d = params.pwl;
+	const struct pwl_params *lut_shaper = params.pwl;
 	bool is_17x17x17;
 	bool is_12bits_color_channel;
 	const struct dc_rgb *lut0;
@@ -90,7 +131,7 @@ void mpc401_populate_lut(struct mpc *mpc,
 	const struct dc_rgb *lut3;
 	int lut_size0;
 	int lut_size;
-	const struct tetrahedral_params *lut3d = params->lut3d;
+	const struct tetrahedral_params *lut3d = params.lut3d;
 
 	switch (id) {
 	case MCM_LUT_1DLUT:
@@ -133,6 +174,8 @@ void mpc401_populate_lut(struct mpc *mpc,
 
 		mpc32_power_on_shaper_3dlut(mpc, mpcc_id, true);
 
+		get3dlut_config(mpc, &is_17x17x17, &is_12bits_color_channel, mpcc_id);
+
 		is_17x17x17 = !lut3d->use_tetrahedral_9;
 		is_12bits_color_channel = lut3d->use_12bits;
 		if (is_17x17x17) {
@@ -155,6 +198,8 @@ void mpc401_populate_lut(struct mpc *mpc,
 					sizeof(lut3d->tetrahedral_9.lut1[0]);
 			}
 
+		mpc32_select_3dlut_ram(mpc, next_mode,
+					is_12bits_color_channel, mpcc_id);
 		mpc32_select_3dlut_ram_mask(mpc, 0x1, mpcc_id);
 		if (is_12bits_color_channel)
 			mpc32_set3dlut_ram12(mpc, lut0, lut_size0, mpcc_id);
@@ -187,69 +232,46 @@ void mpc401_populate_lut(struct mpc *mpc,
 
 }
 
-static uint32_t mpc401_cm_lut_size_to_3dlut_size(const enum dc_cm_lut_size cm_size)
-{
-	uint32_t size = 0;
-
-	switch (cm_size) {
-	case CM_LUT_SIZE_999:
-		size = 1;
-		break;
-	case CM_LUT_SIZE_171717:
-		size = 0;
-		break;
-	default:
-		/* invalid LUT size */
-		ASSERT(false);
-		size = 0;
-		break;
-	}
-
-	return size;
-}
-
 void mpc401_program_lut_mode(
 		struct mpc *mpc,
 		const enum MCM_LUT_ID id,
-		const bool enable,
-		const bool lut_bank_a,
-		const enum dc_cm_lut_size size,
-		const int mpcc_id)
+		const enum MCM_LUT_XABLE xable,
+		bool lut_bank_a,
+		int mpcc_id)
 {
-	uint32_t lut_size;
 	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
 
 	switch (id) {
 	case MCM_LUT_3DLUT:
-		if (enable) {
-			lut_size = mpc401_cm_lut_size_to_3dlut_size(size);
-			REG_UPDATE_2(MPCC_MCM_3DLUT_MODE[mpcc_id],
-					MPCC_MCM_3DLUT_MODE, lut_bank_a ? 1 : 2,
-					MPCC_MCM_3DLUT_SIZE, lut_size);
-		} else {
-			if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc)
-				mpc32_power_on_shaper_3dlut(mpc, mpcc_id, false);
+		switch (xable) {
+		case MCM_LUT_DISABLE:
 			REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_MODE, 0);
+			break;
+		case MCM_LUT_ENABLE:
+			REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_MODE, lut_bank_a ? 1 : 2);
+			break;
 		}
 		break;
 	case MCM_LUT_SHAPER:
-		if (enable) {
-			REG_UPDATE(MPCC_MCM_SHAPER_CONTROL[mpcc_id], MPCC_MCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2);
-		} else {
-			if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc)
-				mpc32_power_on_shaper_3dlut(mpc, mpcc_id, false);
+		switch (xable) {
+		case MCM_LUT_DISABLE:
 			REG_UPDATE(MPCC_MCM_SHAPER_CONTROL[mpcc_id], MPCC_MCM_SHAPER_LUT_MODE, 0);
+			break;
+		case MCM_LUT_ENABLE:
+			REG_UPDATE(MPCC_MCM_SHAPER_CONTROL[mpcc_id], MPCC_MCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2);
+			break;
 		}
 		break;
 	case MCM_LUT_1DLUT:
-		if (enable) {
-			REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
-					MPCC_MCM_1DLUT_MODE, 2);
-		} else {
-			if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc)
-				mpc32_power_on_blnd_lut(mpc, mpcc_id, false);
+		switch (xable) {
+		case MCM_LUT_DISABLE:
 			REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
 					MPCC_MCM_1DLUT_MODE, 0);
+			break;
+		case MCM_LUT_ENABLE:
+			REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
+					MPCC_MCM_1DLUT_MODE, 2);
+			break;
 		}
 		REG_UPDATE(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
 				MPCC_MCM_1DLUT_SELECT, lut_bank_a ? 0 : 1);
@@ -257,20 +279,14 @@ void mpc401_program_lut_mode(
 	}
 }
 
-void mpc401_program_lut_read_write_control(struct mpc *mpc,
-		const enum MCM_LUT_ID id,
-		const bool lut_bank_a,
-		const unsigned int bit_depth,
-		const int mpcc_id)
+void mpc401_program_lut_read_write_control(struct mpc *mpc, const enum MCM_LUT_ID id, bool lut_bank_a, int mpcc_id)
 {
 	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
 
 	switch (id) {
 	case MCM_LUT_3DLUT:
 		mpc32_select_3dlut_ram_mask(mpc, 0xf, mpcc_id);
-		REG_UPDATE_2(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
-				MPCC_MCM_3DLUT_30BIT_EN, (bit_depth == 10) ? 1 : 0,
-				MPCC_MCM_3DLUT_RAM_SEL, lut_bank_a ? 0 : 1);
+		REG_UPDATE(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPCC_MCM_3DLUT_RAM_SEL, lut_bank_a ? 0 : 1);
 		break;
 	case MCM_LUT_SHAPER:
 		mpc32_configure_shaper_lut(mpc, lut_bank_a, mpcc_id);
@@ -562,44 +578,6 @@ void mpc401_get_gamut_remap(struct mpc *mpc,
 		arr_reg_val, ARRAY_SIZE(arr_reg_val));
 }
 
-void mpc401_get_lut_mode(struct mpc *mpc,
-		const enum MCM_LUT_ID id,
-		const int mpcc_id,
-		bool *enable,
-		bool *lut_bank_a)
-{
-	struct dcn401_mpc *mpc401 = TO_DCN401_MPC(mpc);
-
-	uint32_t lut_mode = 0;
-	uint32_t lut_select = 0;
-
-	*enable = false;
-	*lut_bank_a = true;
-
-	switch (id) {
-	case MCM_LUT_SHAPER:
-		REG_GET(MPCC_MCM_SHAPER_CONTROL[mpcc_id],
-				MPCC_MCM_SHAPER_MODE_CURRENT, &lut_mode);
-		*enable = lut_mode != 0;
-		*lut_bank_a = lut_mode != 2;
-		break;
-	case MCM_LUT_1DLUT:
-		REG_GET_2(MPCC_MCM_1DLUT_CONTROL[mpcc_id],
-				MPCC_MCM_1DLUT_MODE_CURRENT, &lut_mode,
-				MPCC_MCM_1DLUT_SELECT_CURRENT, &lut_select);
-		*enable = lut_mode != 0;
-		*lut_bank_a = lut_mode == 0 || lut_select == 0;
-		break;
-	case MCM_LUT_3DLUT:
-	default:
-		REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id],
-				MPCC_MCM_3DLUT_MODE_CURRENT, &lut_mode);
-		*enable = lut_mode != 0;
-		*lut_bank_a = lut_mode != 2;
-		break;
-	}
-}
-
 static const struct mpc_funcs dcn401_mpc_funcs = {
 	.read_mpcc_state = mpc1_read_mpcc_state,
 	.insert_plane = mpc1_insert_plane,
@@ -638,7 +616,6 @@ static const struct mpc_funcs dcn401_mpc_funcs = {
 	.populate_lut = mpc401_populate_lut,
 	.program_lut_read_write_control = mpc401_program_lut_read_write_control,
 	.program_lut_mode = mpc401_program_lut_mode,
-	.get_lut_mode = mpc401_get_lut_mode,
 };
 
 
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h
index c16560c84453e..6d842d7b95c7e 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn401/dcn401_mpc.h
@@ -206,32 +206,21 @@ void dcn401_mpc_construct(struct dcn401_mpc *mpc401,
 	int num_rmu);
 
 void mpc401_set_movable_cm_location(struct mpc *mpc, enum mpcc_movable_cm_location location, int mpcc_id);
-void mpc401_populate_lut(struct mpc *mpc,
-		const enum MCM_LUT_ID id,
-		const union mcm_lut_params *params,
-		bool lut_bank_a,
-		int mpcc_id);
+void mpc401_populate_lut(struct mpc *mpc, const enum MCM_LUT_ID id, const union mcm_lut_params params,
+		bool lut_bank_a, int mpcc_id);
 
 void mpc401_program_lut_mode(
 		struct mpc *mpc,
 		const enum MCM_LUT_ID id,
-		const bool enable,
-		const bool lut_bank_a,
-		const enum dc_cm_lut_size size,
-		const int mpcc_id);
-
-void mpc401_get_lut_mode(struct mpc *mpc,
-		const enum MCM_LUT_ID id,
-		const int mpcc_id,
-		bool *enable,
-		bool *lut_bank_a);
+		const enum MCM_LUT_XABLE xable,
+		bool lut_bank_a,
+		int mpcc_id);
 
 void mpc401_program_lut_read_write_control(
 		struct mpc *mpc,
 		const enum MCM_LUT_ID id,
-		const bool lut_bank_a,
-		const unsigned int bit_depth,
-		const int mpcc_id);
+		bool lut_bank_a,
+		int mpcc_id);
 
 void mpc401_set_gamut_remap(
 	struct mpc *mpc,
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c
index 507dbdbea6008..38c0e8f96d40e 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.c
@@ -63,6 +63,154 @@ void mpc42_update_blending(
 	mpcc->blnd_cfg = *blnd_cfg;
 }
 
+/* Shaper functions */
+void mpc42_power_on_shaper_3dlut(
+	struct mpc *mpc,
+	uint32_t mpcc_id,
+	bool power_on)
+{
+	uint32_t power_status_shaper = 2;
+	uint32_t power_status_3dlut  = 2;
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+	int max_retries = 10;
+
+	REG_SET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], 0,
+		MPCC_MCM_3DLUT_MEM_PWR_DIS, power_on == true ? 1:0);
+	REG_SET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], 0,
+		MPCC_MCM_SHAPER_MEM_PWR_DIS, power_on == true ? 1:0);
+	/* wait for memory to fully power up */
+	if (power_on && mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc) {
+		REG_WAIT(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_SHAPER_MEM_PWR_STATE, 0, 1, max_retries);
+		REG_WAIT(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_3DLUT_MEM_PWR_STATE, 0, 1, max_retries);
+	}
+
+	/*read status is not mandatory, it is just for debugging*/
+	REG_GET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_SHAPER_MEM_PWR_STATE, &power_status_shaper);
+	REG_GET(MPCC_MCM_MEM_PWR_CTRL[mpcc_id], MPCC_MCM_3DLUT_MEM_PWR_STATE, &power_status_3dlut);
+
+	if (power_status_shaper != 0 && power_on == true)
+		BREAK_TO_DEBUGGER();
+
+	if (power_status_3dlut != 0 && power_on == true)
+		BREAK_TO_DEBUGGER();
+}
+
+void mpc42_configure_shaper_lut(
+	struct mpc *mpc,
+	bool is_ram_a,
+	uint32_t mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	REG_UPDATE(MPCC_MCM_SHAPER_SCALE_G_B[mpcc_id],
+		MPCC_MCM_SHAPER_SCALE_B, 0x7000);
+	REG_UPDATE(MPCC_MCM_SHAPER_SCALE_G_B[mpcc_id],
+		MPCC_MCM_SHAPER_SCALE_G, 0x7000);
+	REG_UPDATE(MPCC_MCM_SHAPER_SCALE_R[mpcc_id],
+		MPCC_MCM_SHAPER_SCALE_R, 0x7000);
+	REG_UPDATE(MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
+			MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK, 7);
+	REG_UPDATE(MPCC_MCM_SHAPER_LUT_WRITE_EN_MASK[mpcc_id],
+			MPCC_MCM_SHAPER_LUT_WRITE_SEL, is_ram_a == true ? 0:1);
+	REG_SET(MPCC_MCM_SHAPER_LUT_INDEX[mpcc_id], 0, MPCC_MCM_SHAPER_LUT_INDEX, 0);
+}
+
+
+void mpc42_program_3dlut_size(struct mpc *mpc, uint32_t width, int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+	uint32_t size = 0xff;
+
+	REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_SIZE, &size);
+
+	REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_SIZE,
+		(width == 33) ? 2 :
+		(width == 17) ? 0 : 2);
+
+	REG_GET(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_SIZE, &size);
+}
+
+void mpc42_program_3dlut_fl_bias_scale(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	REG_UPDATE_2(MPCC_MCM_3DLUT_OUT_OFFSET_R[mpcc_id],
+		MPCC_MCM_3DLUT_OUT_OFFSET_R, bias,
+		MPCC_MCM_3DLUT_OUT_SCALE_R, scale);
+
+	REG_UPDATE_2(MPCC_MCM_3DLUT_OUT_OFFSET_G[mpcc_id],
+		MPCC_MCM_3DLUT_OUT_OFFSET_G, bias,
+		MPCC_MCM_3DLUT_OUT_SCALE_G, scale);
+
+	REG_UPDATE_2(MPCC_MCM_3DLUT_OUT_OFFSET_B[mpcc_id],
+		MPCC_MCM_3DLUT_OUT_OFFSET_B, bias,
+		MPCC_MCM_3DLUT_OUT_SCALE_B, scale);
+}
+
+void mpc42_program_bit_depth(struct mpc *mpc, uint16_t bit_depth, int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	REG_UPDATE(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPCC_MCM_3DLUT_WRITE_EN_MASK, 0xF);
+
+	//program bit_depth
+	REG_UPDATE(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id],
+		MPCC_MCM_3DLUT_30BIT_EN,
+		(bit_depth == 10) ? 1 : 0);
+}
+
+bool mpc42_is_config_supported(uint32_t width)
+{
+	if (width == 17)
+		return true;
+
+	return false;
+}
+
+void mpc42_populate_lut(struct mpc *mpc, const union mcm_lut_params params,
+	bool lut_bank_a, int mpcc_id)
+{
+	const enum dc_lut_mode next_mode = lut_bank_a ? LUT_RAM_A : LUT_RAM_B;
+	const struct pwl_params *lut_shaper = params.pwl;
+
+	if (lut_shaper == NULL)
+		return;
+	if (mpc->ctx->dc->debug.enable_mem_low_power.bits.mpc)
+		mpc42_power_on_shaper_3dlut(mpc, mpcc_id, true);
+
+	mpc42_configure_shaper_lut(mpc, next_mode == LUT_RAM_A, mpcc_id);
+
+	if (next_mode == LUT_RAM_A)
+		mpc32_program_shaper_luta_settings(mpc, lut_shaper, mpcc_id);
+	else
+		mpc32_program_shaper_lutb_settings(mpc, lut_shaper, mpcc_id);
+
+	mpc32_program_shaper_lut(
+			mpc, lut_shaper->rgb_resulted, lut_shaper->hw_points_num, mpcc_id);
+
+	mpc42_power_on_shaper_3dlut(mpc, mpcc_id, false);
+}
+
+void mpc42_program_lut_read_write_control(struct mpc *mpc, const enum MCM_LUT_ID id,
+	bool lut_bank_a, bool enabled, int mpcc_id)
+{
+	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+
+	switch (id) {
+	case MCM_LUT_3DLUT:
+		REG_UPDATE(MPCC_MCM_3DLUT_MODE[mpcc_id], MPCC_MCM_3DLUT_MODE,
+			(!enabled) ? 0 :
+			(lut_bank_a) ? 1 : 2);
+		REG_UPDATE(MPCC_MCM_3DLUT_READ_WRITE_CONTROL[mpcc_id], MPCC_MCM_3DLUT_RAM_SEL, lut_bank_a ? 0 : 1);
+		break;
+	case MCM_LUT_SHAPER:
+		mpc32_configure_shaper_lut(mpc, lut_bank_a, mpcc_id);
+		break;
+	default:
+		break;
+	}
+}
+
 /* RMCM Shaper functions */
 void mpc42_power_on_rmcm_shaper_3dlut(
 	struct mpc *mpc,
@@ -526,47 +674,32 @@ void mpc42_program_rmcm_lut_read_write_control(struct mpc *mpc, const enum MCM_L
 	}
 }
 
-void mpc42_program_lut_mode(struct mpc *mpc,
-	bool enable,
-	bool lut_bank_a,
-	int mpcc_id)
+void mpc42_program_lut_mode(struct mpc *mpc, const enum MCM_LUT_XABLE xable,
+	bool lut_bank_a, int mpcc_id)
 {
 	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
 
-	if (enable) {
-		REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2);
-	} else {
+	switch (xable) {
+	case MCM_LUT_DISABLE:
 		REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, 0);
-	}
-}
-
-static uint32_t mpc42_get_rmcm_3dlut_width(
-		const enum dc_cm_lut_size size)
-{
-	uint32_t width = 0;
-
-	switch (size) {
-	case CM_LUT_SIZE_333333:
-		width = 2;
 		break;
-	case CM_LUT_SIZE_171717:
-	default:
-		width = 0;
+	case MCM_LUT_ENABLE:
+		REG_UPDATE(MPC_RMCM_SHAPER_CONTROL[mpcc_id], MPC_RMCM_SHAPER_LUT_MODE, lut_bank_a ? 1 : 2);
 		break;
 	}
-
-	return width;
 }
 
-void mpc42_program_rmcm_3dlut_size(struct mpc *mpc,
-		const enum dc_cm_lut_size size,
-		int mpcc_id)
+void mpc42_program_rmcm_3dlut_size(struct mpc *mpc, uint32_t width, int mpcc_id)
 {
 	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
-	uint32_t width = mpc42_get_rmcm_3dlut_width(size);
+	uint32_t size = 0xff;
+
+	REG_GET(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_SIZE, &size);
 
-	REG_UPDATE(MPC_RMCM_3DLUT_MODE[mpcc_id],
-			MPC_RMCM_3DLUT_SIZE, width);
+	REG_UPDATE(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_SIZE,
+		(width == 33) ? 2 : 0);
+
+	REG_GET(MPC_RMCM_3DLUT_MODE[mpcc_id], MPC_RMCM_3DLUT_SIZE, &size);
 }
 
 void mpc42_program_rmcm_3dlut_fast_load_bias_scale(struct mpc *mpc, uint16_t bias, uint16_t scale, int mpcc_id)
@@ -598,6 +731,14 @@ void mpc42_program_rmcm_bit_depth(struct mpc *mpc, uint16_t bit_depth, int mpcc_
 		(bit_depth == 10) ? 1 : 0);
 }
 
+bool mpc42_is_rmcm_config_supported(uint32_t width)
+{
+	if (width == 17 || width == 33)
+		return true;
+
+	return false;
+}
+
 void mpc42_set_fl_config(
 	struct mpc *mpc,
 	struct mpc_fl_3dlut_config *cfg,
@@ -605,7 +746,6 @@ void mpc42_set_fl_config(
 {
 	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
 
-	uint32_t width = mpc42_get_rmcm_3dlut_width(cfg->size);
 	/*
 	From: Jie Zhou
 
@@ -646,7 +786,7 @@ void mpc42_set_fl_config(
 
 	//width
 	REG_UPDATE_2(MPC_RMCM_3DLUT_MODE[mpcc_id],
-		MPC_RMCM_3DLUT_SIZE, width,
+		MPC_RMCM_3DLUT_SIZE, (cfg->width == 33) ? 2 : 0,
 		MPC_RMCM_3DLUT_MODE, (!cfg->enabled) ? 0 : (cfg->select_lut_bank_a) ? 1 : 2);
 
 	//connect to hubp
@@ -659,6 +799,182 @@ void mpc42_set_fl_config(
 	REG_UPDATE(MPC_RMCM_CNTL[mpcc_id], MPC_RMCM_CNTL, cfg->enabled ? 0 : 0xF);
 }
 
+//static void rmcm_program_gamut_remap(
+//	struct mpc *mpc,
+//	unsigned int mpcc_id,
+//	const uint16_t *regval,
+//	enum mpcc_gamut_remap_id gamut_remap_block_id,
+//	enum mpcc_gamut_remap_mode_select mode_select)
+//{
+//	struct color_matrices_reg gamut_regs;
+//	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+//
+//	if (gamut_remap_block_id == MPCC_OGAM_GAMUT_REMAP ||
+//		gamut_remap_block_id == MPCC_MCM_FIRST_GAMUT_REMAP ||
+//		gamut_remap_block_id == MPCC_MCM_SECOND_GAMUT_REMAP) {
+//		mpc_program_gamut_remap(mpc, mpcc_id, regval, gamut_remap_block_id, mode_select);
+//		return;
+//	}
+//	if (gamut_remap_block_id == MPCC_OGAM_GAMUT_REMAP) {
+//
+//		if (regval == NULL || mode_select == MPCC_GAMUT_REMAP_MODE_SELECT_0) {
+//			REG_SET(MPC_RMCM_GAMUT_REMAP_MODE[mpcc_id], 0,
+//				MPC_RMCM_GAMUT_REMAP_MODE, mode_select);
+//			return;
+//		}
+//
+//		gamut_regs.shifts.csc_c11 = mpc42->mpc_shift->MPCC_GAMUT_REMAP_C11_A;
+//		gamut_regs.masks.csc_c11 = mpc42->mpc_mask->MPCC_GAMUT_REMAP_C11_A;
+//		gamut_regs.shifts.csc_c12 = mpc42->mpc_shift->MPCC_GAMUT_REMAP_C12_A;
+//		gamut_regs.masks.csc_c12 = mpc42->mpc_mask->MPCC_GAMUT_REMAP_C12_A;
+//
+//		switch (mode_select) {
+//		case MPCC_GAMUT_REMAP_MODE_SELECT_1:
+//			gamut_regs.csc_c11_c12 = REG(MPC_RMCM_GAMUT_REMAP_C11_C12_A[mpcc_id]);
+//			gamut_regs.csc_c33_c34 = REG(MPC_RMCM_GAMUT_REMAP_C33_C34_A[mpcc_id]);
+//			break;
+//		case MPCC_GAMUT_REMAP_MODE_SELECT_2:
+//			gamut_regs.csc_c11_c12 = REG(MPC_RMCM_GAMUT_REMAP_C11_C12_B[mpcc_id]);
+//			gamut_regs.csc_c33_c34 = REG(MPC_RMCM_GAMUT_REMAP_C33_C34_B[mpcc_id]);
+//			break;
+//		default:
+//			break;
+//		}
+//
+//		cm_helper_program_color_matrices(
+//			mpc->ctx,
+//			regval,
+//			&gamut_regs);
+//
+//		//select coefficient set to use, set A (MODE_1) or set B (MODE_2)
+//		REG_SET(MPC_RMCM_GAMUT_REMAP_MODE[mpcc_id], 0, MPC_RMCM_GAMUT_REMAP_MODE, mode_select);
+//	}
+//}
+
+//static bool is_mpc_legacy_gamut_id(enum mpcc_gamut_remap_id gamut_remap_block_id)
+//{
+//	if (gamut_remap_block_id == MPCC_OGAM_GAMUT_REMAP ||
+//		gamut_remap_block_id == MPCC_MCM_FIRST_GAMUT_REMAP ||
+//		gamut_remap_block_id == MPCC_MCM_SECOND_GAMUT_REMAP) {
+//		return true;
+//	}
+//	return false;
+//}
+//static void program_gamut_remap(
+//	struct mpc *mpc,
+//	unsigned int mpcc_id,
+//	const uint16_t *regval,
+//	enum mpcc_gamut_remap_id gamut_remap_block_id,
+//	enum mpcc_gamut_remap_mode_select mode_select)
+//{
+//	if (is_mpc_legacy_gamut_id(gamut_remap_block_id))
+//		mpc_program_gamut_remap(mpc, mpcc_id, regval, gamut_remap_block_id, mode_select);
+//	else
+//		rmcm_program_gamut_remap(mpc, mpcc_id, regval, gamut_remap_block_id, mode_select);
+//}
+
+//void mpc42_set_gamut_remap(
+//	struct mpc *mpc,
+//	int mpcc_id,
+//	const struct mpc_grph_gamut_adjustment *adjust)
+//{
+//	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+//	unsigned int i = 0;
+//	uint32_t mode_select = 0;
+//
+//	if (adjust->gamut_adjust_type != GRAPHICS_GAMUT_ADJUST_TYPE_SW) {
+//		/* Bypass / Disable if type is bypass or hw */
+//		program_gamut_remap(mpc, mpcc_id, NULL,
+//			adjust->mpcc_gamut_remap_block_id, MPCC_GAMUT_REMAP_MODE_SELECT_0);
+//	} else {
+//		struct fixed31_32 arr_matrix[12];
+//		uint16_t arr_reg_val[12];
+//
+//		for (i = 0; i < 12; i++)
+//			arr_matrix[i] = adjust->temperature_matrix[i];
+//
+//		convert_float_matrix(arr_reg_val, arr_matrix, 12);
+//
+//		if (is_mpc_legacy_gamut_id(adjust->mpcc_gamut_remap_block_id))
+//			REG_GET(MPCC_GAMUT_REMAP_MODE[mpcc_id],
+//				MPCC_GAMUT_REMAP_MODE_CURRENT, &mode_select);
+//		else
+//			REG_GET(MPC_RMCM_GAMUT_REMAP_MODE[mpcc_id],
+//				MPC_RMCM_GAMUT_REMAP_MODE_CURRENT, &mode_select);
+//
+//		//If current set in use not set A (MODE_1), then use set A, otherwise use set B
+//		if (mode_select != MPCC_GAMUT_REMAP_MODE_SELECT_1)
+//			mode_select = MPCC_GAMUT_REMAP_MODE_SELECT_1;
+//		else
+//			mode_select = MPCC_GAMUT_REMAP_MODE_SELECT_2;
+//
+//		program_gamut_remap(mpc, mpcc_id, arr_reg_val,
+//			adjust->mpcc_gamut_remap_block_id, mode_select);
+//	}
+//}
+
+//static void read_gamut_remap(struct mpc *mpc,
+//	int mpcc_id,
+//	uint16_t *regval,
+//	enum mpcc_gamut_remap_id gamut_remap_block_id,
+//	uint32_t *mode_select)
+//{
+//	struct color_matrices_reg gamut_regs = {0};
+//	struct dcn42_mpc *mpc42 = TO_DCN42_MPC(mpc);
+//
+//	if (is_mpc_legacy_gamut_id(gamut_remap_block_id)) {
+//		mpc_read_gamut_remap(mpc, mpcc_id, regval, gamut_remap_block_id, mode_select);
+//	}
+//	if (gamut_remap_block_id == MPCC_RMCM_GAMUT_REMAP) {
+//		//current coefficient set in use
+//		REG_GET(MPC_RMCM_GAMUT_REMAP_MODE[mpcc_id], MPC_RMCM_GAMUT_REMAP_MODE, mode_select);
+//
+//		gamut_regs.shifts.csc_c11 = mpc42->mpc_shift->MPCC_GAMUT_REMAP_C11_A;
+//		gamut_regs.masks.csc_c11 = mpc42->mpc_mask->MPCC_GAMUT_REMAP_C11_A;
+//		gamut_regs.shifts.csc_c12 = mpc42->mpc_shift->MPCC_GAMUT_REMAP_C12_A;
+//		gamut_regs.masks.csc_c12 = mpc42->mpc_mask->MPCC_GAMUT_REMAP_C12_A;
+//
+//		switch (*mode_select) {
+//		case MPCC_GAMUT_REMAP_MODE_SELECT_1:
+//			gamut_regs.csc_c11_c12 = REG(MPC_RMCM_GAMUT_REMAP_C11_C12_A[mpcc_id]);
+//			gamut_regs.csc_c33_c34 = REG(MPC_RMCM_GAMUT_REMAP_C33_C34_A[mpcc_id]);
+//			break;
+//		case MPCC_GAMUT_REMAP_MODE_SELECT_2:
+//			gamut_regs.csc_c11_c12 = REG(MPC_RMCM_GAMUT_REMAP_C11_C12_B[mpcc_id]);
+//			gamut_regs.csc_c33_c34 = REG(MPC_RMCM_GAMUT_REMAP_C33_C34_B[mpcc_id]);
+//			break;
+//		default:
+//			break;
+//		}
+//	}
+//
+//	if (*mode_select != MPCC_GAMUT_REMAP_MODE_SELECT_0) {
+//		cm_helper_read_color_matrices(
+//			mpc42->base.ctx,
+//			regval,
+//			&gamut_regs);
+//	}
+//}
+
+//void mpc42_get_gamut_remap(struct mpc *mpc,
+//	int mpcc_id,
+//	struct mpc_grph_gamut_adjustment *adjust)
+//{
+//	uint16_t arr_reg_val[12] = {0};
+//	uint32_t mode_select;
+//
+//	read_gamut_remap(mpc, mpcc_id, arr_reg_val, adjust->mpcc_gamut_remap_block_id, &mode_select);
+//
+//	if (mode_select == MPCC_GAMUT_REMAP_MODE_SELECT_0) {
+//		adjust->gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_BYPASS;
+//		return;
+//	}
+//
+//	adjust->gamut_adjust_type = GRAPHICS_GAMUT_ADJUST_TYPE_SW;
+//	convert_hw_matrix(adjust->temperature_matrix,
+//		arr_reg_val, ARRAY_SIZE(arr_reg_val));
+//}
+
 void mpc42_read_mpcc_state(
 		struct mpc *mpc,
 		int mpcc_inst,
@@ -755,7 +1071,14 @@ static const struct mpc_funcs dcn42_mpc_funcs = {
 	.populate_lut = mpc401_populate_lut,
 	.program_lut_read_write_control = mpc401_program_lut_read_write_control,
 	.program_lut_mode = mpc401_program_lut_mode,
-	.get_lut_mode = mpc401_get_lut_mode,
+	.mcm = {
+		.program_lut_read_write_control = mpc42_program_lut_read_write_control,
+		.program_3dlut_size = mpc42_program_3dlut_size,
+		.program_bias_scale = mpc42_program_3dlut_fl_bias_scale,
+		.program_bit_depth = mpc42_program_bit_depth,
+		.is_config_supported = mpc42_is_config_supported,
+		.populate_lut = mpc42_populate_lut,
+	},
 	.rmcm = {
 		.enable_3dlut_fl = mpc42_enable_3dlut_fl,
 		.update_3dlut_fast_load_select = mpc42_update_3dlut_fast_load_select,
@@ -764,6 +1087,7 @@ static const struct mpc_funcs dcn42_mpc_funcs = {
 		.program_3dlut_size = mpc42_program_rmcm_3dlut_size,
 		.program_bias_scale = mpc42_program_rmcm_3dlut_fast_load_bias_scale,
 		.program_bit_depth = mpc42_program_rmcm_bit_depth,
+		.is_config_supported = mpc42_is_rmcm_config_supported,
 		.power_on_shaper_3dlut = mpc42_power_on_rmcm_shaper_3dlut,
 		.populate_lut = mpc42_populate_rmcm_lut,
 		.fl_3dlut_configure = mpc42_set_fl_config,
diff --git a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h
index a5f7f4f2bb3bc..12a12c28e5537 100644
--- a/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h
+++ b/drivers/gpu/drm/amd/display/dc/mpc/dcn42/dcn42_mpc.h
@@ -884,6 +884,49 @@ void dcn42_mpc_construct(struct dcn42_mpc *mpc401,
 
 void mpc42_init_mpcc(struct mpcc *mpcc, int mpcc_inst);
 
+void mpc42_program_shaper_lutb_settings(
+	struct mpc *mpc,
+	const struct pwl_params *params,
+	uint32_t mpcc_id);
+void mpc42_program_shaper_luta_settings(
+		struct mpc *mpc,
+		const struct pwl_params *params,
+		uint32_t mpcc_id);
+void mpc42_configure_shaper_lut(
+		struct mpc *mpc,
+		bool is_ram_a,
+		uint32_t mpcc_id);
+void mpc42_power_on_shaper_3dlut(
+		struct mpc *mpc,
+		uint32_t mpcc_id,
+		bool power_on);
+void mpc42_program_3dlut_size(
+		struct mpc *mpc,
+		uint32_t width,
+		int mpcc_id);
+void mpc42_program_3dlut_fl_bias_scale(
+		struct mpc *mpc,
+		uint16_t bias,
+		uint16_t scale,
+		int mpcc_id);
+void mpc42_program_bit_depth(
+		struct mpc *mpc,
+		uint16_t bit_depth,
+		int mpcc_id);
+void mpc42_populate_lut(
+		struct mpc *mpc,
+		const union mcm_lut_params params,
+		bool lut_bank_a,
+		int mpcc_id);
+void mpc42_program_lut_read_write_control(
+		struct mpc *mpc,
+		const enum MCM_LUT_ID id,
+		bool lut_bank_a,
+		bool enabled,
+		int mpcc_id);
+
+bool mpc42_is_config_supported(uint32_t width);
+
 /* RMCM */
 void mpc42_program_rmcm_shaper_lut(
 		struct mpc *mpc,
@@ -927,12 +970,12 @@ void mpc42_program_rmcm_lut_read_write_control(
 		int mpcc_id);
 void mpc42_program_lut_mode(
 		struct mpc *mpc,
-		bool enable,
+		const enum MCM_LUT_XABLE xable,
 		bool lut_bank_a,
 		int mpcc_id);
 void mpc42_program_rmcm_3dlut_size(
 		struct mpc *mpc,
-		const enum dc_cm_lut_size size,
+		uint32_t width,
 		int mpcc_id);
 void mpc42_program_rmcm_3dlut_fast_load_bias_scale(
 		struct mpc *mpc,
@@ -944,6 +987,8 @@ void mpc42_program_rmcm_bit_depth(
 		uint16_t bit_depth,
 		int mpcc_id);
 
+bool mpc42_is_rmcm_config_supported(uint32_t width);
+
 void mpc42_set_fl_config(
 	struct mpc *mpc,
 	struct mpc_fl_3dlut_config *cfg,
diff --git a/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c b/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c
index ce826a5be4c71..83730bbe26a8c 100644
--- a/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c
+++ b/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.c
@@ -293,7 +293,7 @@ void opp2_set_disp_pattern_generator(
 
 void opp2_program_dpg_dimensions(
 		struct output_pixel_processor *opp,
-		int width, int height)
+		uint32_t width, uint32_t height)
 {
 	struct dcn20_opp *oppn20 = TO_DCN20_OPP(opp);
 
diff --git a/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.h b/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.h
index fb0c047c1788e..8944fa6c8f79e 100644
--- a/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.h
+++ b/drivers/gpu/drm/amd/display/dc/opp/dcn20/dcn20_opp.h
@@ -156,7 +156,7 @@ void opp2_set_disp_pattern_generator(
 
 void opp2_program_dpg_dimensions(
 		struct output_pixel_processor *opp,
-		int width, int height);
+		uint32_t width, uint32_t height);
 
 bool opp2_dpg_is_blanked(struct output_pixel_processor *opp);
 
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c
index 9e66b9b97c631..e6426ccee2d88 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn10/dcn10_optc.c
@@ -375,7 +375,7 @@ void optc1_set_vtg_params(struct timing_generator *optc,
 	if (REG(OTG_INTERLACE_CONTROL)) {
 		if (patched_crtc_timing.flags.INTERLACE == 1) {
 			v_init = v_init / 2;
-			if ((optc1->vstartup_start/2)*2 > asic_blank_end)
+			if ((uint32_t)((optc1->vstartup_start/2)*2) > asic_blank_end)
 				v_fp2 = v_fp2 / 2;
 		}
 	}
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c
index 39ce4d4a61a1c..c558b1d633f32 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn20/dcn20_optc.c
@@ -305,8 +305,8 @@ static void optc2_align_vblanks(
 	L = div_u64(L, master_h_total);
 	L = div_u64(L, slave_pixel_clock_100Hz);
 	XY = div_u64(L, p);
-	Y = master_v_active - XY - 1;
-	X = div_u64(((XY + 1) * p - L) * master_h_total, p * master_clock_divider);
+	Y = (uint32_t)(master_v_active - XY - 1);
+	X = (uint32_t)div_u64(((XY + 1) * p - L) * master_h_total, p * master_clock_divider);
 
 	/*
 	 * set master OTG to unlock when V/H
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c
index 60e546b69a05a..07895d5f4dfa0 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn32/dcn32_optc.c
@@ -101,7 +101,7 @@ static void optc32_set_odm_combine(struct timing_generator *optc, int *opp_id, i
 void optc32_get_odm_combine_segments(struct timing_generator *tg, int *odm_combine_segments)
 {
 	struct optc *optc1 = DCN10TG_FROM_TG(tg);
-	int segments;
+	uint32_t segments;
 
 	REG_GET(OPTC_DATA_SOURCE_SELECT, OPTC_NUM_OF_INPUT_SEGMENT, &segments);
 
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
index a880e4a6d1659..62f45c156c325 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn35/dcn35_optc.c
@@ -621,7 +621,6 @@ void dcn35_timing_generator_init(struct optc *optc1)
 	optc1->min_v_blank_interlace = 5;
 	optc1->min_h_sync_width = 4;
 	optc1->min_v_sync_width = 1;
-	optc1->max_frame_count = 0xFFFFFF;
 
 	dcn35_timing_generator_set_fgcg(
 		optc1, CTX->dc->debug.enable_fine_grain_clock_gating.bits.optc);
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c
index ed66a2bbb8aeb..a3431ec2f0583 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.c
@@ -283,9 +283,7 @@ void dcn42_timing_generator_init(struct optc *optc1)
 	optc1->min_v_blank_interlace = 5;
 	optc1->min_h_sync_width = 4;
 	optc1->min_v_sync_width = 1;
-	optc1->max_frame_count = 0xFFFFFF;
 
 	dcn35_timing_generator_set_fgcg(
 		optc1, CTX->dc->debug.enable_fine_grain_clock_gating.bits.optc);
 }
-
diff --git a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h
index fc7192f01b336..8e7d65317e7cc 100644
--- a/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h
+++ b/drivers/gpu/drm/amd/display/dc/optc/dcn42/dcn42_optc.h
@@ -164,7 +164,6 @@
 	SF(GSL_SOURCE_SELECT, GSL0_READY_SOURCE_SEL, mask_sh),\
 	SF(GSL_SOURCE_SELECT, GSL1_READY_SOURCE_SEL, mask_sh),\
 	SF(GSL_SOURCE_SELECT, GSL2_READY_SOURCE_SEL, mask_sh),\
-	SF(OTG0_OTG_GLOBAL_CONTROL2, MANUAL_FLOW_CONTROL_SEL, mask_sh),\
 	SF(OTG0_OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, mask_sh),\
 	SF(OTG0_OTG_GSL_WINDOW_X, OTG_GSL_WINDOW_START_X, mask_sh),\
 	SF(OTG0_OTG_GSL_WINDOW_X, OTG_GSL_WINDOW_END_X, mask_sh), \
diff --git a/drivers/gpu/drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c b/drivers/gpu/drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c
index f5bf5940e748c..1d0181534a67f 100644
--- a/drivers/gpu/drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/pg/dcn35/dcn35_pg_cntl.c
@@ -380,7 +380,7 @@ void pg_cntl35_plane_otg_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t power_gate = power_on ? 0 : 1;
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl;
-	int i;
+	unsigned int i;
 	bool block_enabled;
 	bool all_mpcc_disabled = true, all_opp_disabled = true;
 	bool all_optc_disabled = true, all_stream_disabled = true;
@@ -460,7 +460,7 @@ static bool pg_cntl35_mem_status(struct pg_cntl *pg_cntl)
 
 void pg_cntl35_init_pg_status(struct pg_cntl *pg_cntl)
 {
-	int i = 0;
+	unsigned int i = 0;
 	bool block_enabled;
 
 	pg_cntl->pg_res_enable[PG_HPO] = pg_cntl35_hpo_pg_status(pg_cntl);
@@ -494,7 +494,7 @@ void pg_cntl35_init_pg_status(struct pg_cntl *pg_cntl)
 
 static void pg_cntl35_print_pg_status(struct pg_cntl *pg_cntl, const char *debug_func, const char *debug_log)
 {
-	int i = 0;
+	unsigned int i = 0;
 	bool block_enabled = false;
 
 	DC_LOG_DEBUG("%s: %s", debug_func, debug_log);
diff --git a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
index 96290538a889d..729c2b653161d 100644
--- a/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
+++ b/drivers/gpu/drm/amd/display/dc/pg/dcn42/dcn42_pg_cntl.c
@@ -27,9 +27,6 @@ static bool pg_cntl42_dsc_pg_status(struct pg_cntl *pg_cntl, unsigned int dsc_in
 	struct dcn_pg_cntl *pg_cntl_dcn = TO_DCN_PG_CNTL(pg_cntl);
 	uint32_t pwr_status = 0;
 
-	if (pg_cntl->ctx->dc->debug.ignore_pg)
-		return true;
-
 	switch (dsc_inst) {
 	case 0: /* DSC0 */
 		REG_GET(DOMAIN16_PG_STATUS, DOMAIN_PGFSM_PWR_STATUS, &pwr_status);
@@ -64,9 +61,11 @@ void pg_cntl42_dsc_pg_control(struct pg_cntl *pg_cntl, unsigned int dsc_inst, bo
 		pg_cntl->ctx->dc->res_pool->dccg->funcs->enable_dsc(
 				pg_cntl->ctx->dc->res_pool->dccg, dsc_inst);
 
-	if (pg_cntl->ctx->dc->debug.ignore_pg ||
-		pg_cntl->ctx->dc->debug.disable_dsc_power_gate ||
-		pg_cntl->ctx->dc->idle_optimizations_allowed)
+    bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
+			pg_cntl->ctx->dc->debug.disable_dsc_power_gate ||
+			pg_cntl->ctx->dc->idle_optimizations_allowed;
+
+	if (skip_pg && !power_on)
 		return;
 
 	block_enabled = pg_cntl42_dsc_pg_status(pg_cntl, dsc_inst);
@@ -261,11 +260,12 @@ void pg_cntl42_hpo_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t power_forceon;
 	bool block_enabled;
 
-	if (pg_cntl->ctx->dc->debug.ignore_pg ||
-		pg_cntl->ctx->dc->debug.disable_hpo_power_gate ||
-		pg_cntl->ctx->dc->idle_optimizations_allowed)
-		return;
+	bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
+			pg_cntl->ctx->dc->debug.disable_hpo_power_gate ||
+			pg_cntl->ctx->dc->idle_optimizations_allowed;
 
+	if (skip_pg && !power_on)
+		return;
 	block_enabled = pg_cntl42_hpo_pg_status(pg_cntl);
 	if (power_on) {
 		if (block_enabled)
@@ -316,9 +316,11 @@ void pg_cntl42_io_clk_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t power_forceon;
 	bool block_enabled;
 
-	if (pg_cntl->ctx->dc->debug.ignore_pg ||
-		pg_cntl->ctx->dc->idle_optimizations_allowed ||
-		pg_cntl->ctx->dc->debug.disable_io_clk_power_gate)
+	bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
+			pg_cntl->ctx->dc->idle_optimizations_allowed ||
+			pg_cntl->ctx->dc->debug.disable_io_clk_power_gate;
+
+	if (skip_pg && !power_on)
 		return;
 
 	block_enabled = pg_cntl42_io_clk_status(pg_cntl);
@@ -412,9 +414,11 @@ void pg_cntl42_mem_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t power_forceon;
 	bool block_enabled;
 
-	if (pg_cntl->ctx->dc->debug.ignore_pg ||
-		pg_cntl->ctx->dc->idle_optimizations_allowed ||
-		pg_cntl->ctx->dc->debug.disable_mem_power_gate)
+	bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
+			pg_cntl->ctx->dc->idle_optimizations_allowed ||
+			pg_cntl->ctx->dc->debug.disable_mem_power_gate;
+
+	if (skip_pg && !power_on)
 		return;
 
 	block_enabled = pg_cntl42_mem_status(pg_cntl);
@@ -464,9 +468,10 @@ void pg_cntl42_dio_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t org_ip_request_cntl;
 	bool block_enabled;
 
-	if (pg_cntl->ctx->dc->debug.ignore_pg ||
-		pg_cntl->ctx->dc->idle_optimizations_allowed ||
-		pg_cntl->ctx->dc->debug.disable_dio_power_gate)
+	bool skip_pg = pg_cntl->ctx->dc->debug.ignore_pg ||
+			pg_cntl->ctx->dc->idle_optimizations_allowed ||
+			pg_cntl->ctx->dc->debug.disable_dio_power_gate;
+	if (skip_pg && !power_on)
 		return;
 
 	block_enabled = pg_cntl42_dio_pg_status(pg_cntl);
@@ -503,7 +508,7 @@ void pg_cntl42_plane_otg_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 	uint32_t power_gate = power_on ? 0 : 1;
 	uint32_t pwr_status = power_on ? 0 : 2;
 	uint32_t org_ip_request_cntl;
-	int i;
+	unsigned int i;
 	bool block_enabled;
 	bool all_mpcc_disabled = true, all_opp_disabled = true;
 	bool all_optc_disabled = true, all_stream_disabled = true;
@@ -564,7 +569,7 @@ void pg_cntl42_plane_otg_pg_control(struct pg_cntl *pg_cntl, bool power_on)
 
 void pg_cntl42_init_pg_status(struct pg_cntl *pg_cntl)
 {
-	int i = 0;
+	unsigned int i = 0;
 	bool block_enabled;
 
 	pg_cntl->pg_res_enable[PG_HPO] = pg_cntl42_hpo_pg_status(pg_cntl);
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
index caafebe921299..b92d4f378d602 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce100/dce100_resource.c
@@ -804,7 +804,7 @@ static void dce100_resource_destruct(struct dce110_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -869,7 +869,7 @@ enum dc_status dce100_validate_bandwidth(
 	enum dc_validate_mode validate_mode)
 {
 	(void)validate_mode;
-	int i;
+	unsigned int i;
 	bool at_least_one_pipe = false;
 	struct dc_stream_state *stream = NULL;
 	const uint32_t max_pix_clk_khz = max(dc->clk_mgr->clks.max_supported_dispclk_khz, 400000);
@@ -978,7 +978,7 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
 		const struct resource_pool *pool,
 		struct dc_stream_state *stream)
 {
-	int i;
+	unsigned int i;
 	int j = -1;
 	struct dc_link *link = stream->link;
 	enum engine_id preferred_engine = link->link_enc->preferred_engine;
@@ -995,7 +995,7 @@ struct stream_encoder *dce100_find_first_free_match_stream_enc_for_link(
 			/* Store first available for MST second display
 			 * in daisy chain use case
 			 */
-			j = i;
+			j = (int)i;
 			if (pool->stream_enc[i]->id == preferred_engine)
 				return pool->stream_enc[i];
 		}
@@ -1175,7 +1175,7 @@ static bool dce100_resource_construct(
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dce100_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c
index f83acfe7a15ee..19252b25bce02 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce110/dce110_resource.c
@@ -839,7 +839,7 @@ static void dce110_resource_destruct(struct dce110_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1150,7 +1150,7 @@ static struct pipe_ctx *dce110_acquire_underlay(
 	/*pipe_ctx->plane_res.ipp = res_ctx->pool->ipps[underlay_idx];*/
 	pipe_ctx->plane_res.xfm = pool->transforms[underlay_idx];
 	pipe_ctx->stream_res.opp = pool->opps[underlay_idx];
-	pipe_ctx->pipe_idx = underlay_idx;
+	pipe_ctx->pipe_idx = (uint8_t)underlay_idx;
 
 	pipe_ctx->stream = stream;
 
@@ -1160,7 +1160,7 @@ static struct pipe_ctx *dce110_acquire_underlay(
 
 		hws->funcs.enable_display_power_gating(
 				dc,
-				pipe_ctx->stream_res.tg->inst,
+				(uint8_t)pipe_ctx->stream_res.tg->inst,
 				dcb, PIPE_GATING_CONTROL_DISABLE);
 
 		/*
@@ -1213,7 +1213,7 @@ struct stream_encoder *dce110_find_first_free_match_stream_enc_for_link(
 		const struct resource_pool *pool,
 		struct dc_stream_state *stream)
 {
-	int i;
+	unsigned int i;
 	int j = -1;
 	struct dc_link *link = stream->link;
 
@@ -1223,7 +1223,7 @@ struct stream_encoder *dce110_find_first_free_match_stream_enc_for_link(
 			/* Store first available for MST second display
 			 * in daisy chain use case
 			 */
-			j = i;
+			j = (int)i;
 			if (pool->stream_enc[i]->id ==
 					link->link_enc->preferred_engine)
 				return pool->stream_enc[i];
@@ -1492,7 +1492,7 @@ static bool dce110_resource_construct(
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dce110_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
index 458b14e4cb977..1deba53363e55 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce112/dce112_resource.c
@@ -804,7 +804,7 @@ static void dce112_resource_destruct(struct dce110_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1382,7 +1382,7 @@ static bool dce112_resource_construct(
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dce112_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c
index 56bbf9dc16912..38d3b87529b06 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce120/dce120_resource.c
@@ -626,7 +626,7 @@ static void dce120_resource_destruct(struct dce110_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -927,7 +927,7 @@ static void bw_calcs_data_update_from_pplib(struct dc *dc)
 	struct dm_pp_clock_levels_with_latency eng_clks = {0};
 	struct dm_pp_clock_levels_with_latency mem_clks = {0};
 	struct dm_pp_wm_sets_with_clock_ranges clk_ranges = {0};
-	int i;
+	unsigned int i;
 	unsigned int clk;
 	unsigned int latency;
 	/*original logic in dal3*/
@@ -1227,7 +1227,7 @@ static bool dce120_resource_construct(
 		j++;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dce120_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
index 6c00497e9a012..26820d35478a3 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dce80/dce80_resource.c
@@ -854,7 +854,7 @@ static void dce80_resource_destruct(struct dce110_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1058,7 +1058,7 @@ static bool dce80_construct(
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -1258,7 +1258,7 @@ static bool dce81_construct(
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -1456,7 +1456,7 @@ static bool dce83_construct(
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dce80_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
index 0fdebe63d3554..db1b6a1b92e8b 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn10/dcn10_resource.c
@@ -976,7 +976,7 @@ static void dcn10_resource_destruct(struct dcn10_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		kfree(pool->base.hw_i2cs[i]);
@@ -1139,7 +1139,7 @@ static struct pipe_ctx *dcn10_acquire_free_pipe_for_layer(
 	idle_pipe->plane_res.hubp = pool->hubps[idle_pipe->pipe_idx];
 	idle_pipe->plane_res.ipp = pool->ipps[idle_pipe->pipe_idx];
 	idle_pipe->plane_res.dpp = pool->dpps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.mpcc_inst = pool->dpps[idle_pipe->pipe_idx]->inst;
+	idle_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[idle_pipe->pipe_idx]->inst;
 
 	return idle_pipe;
 }
@@ -1181,7 +1181,8 @@ static enum dc_status dcn10_validate_plane(const struct dc_plane_state *plane_st
 {
 	if (plane_state->format >= SURFACE_PIXEL_FORMAT_VIDEO_BEGIN
 			&& caps->max_video_width != 0
-			&& plane_state->src_rect.width > caps->max_video_width)
+			&& plane_state->src_rect.width > 0
+			&& (unsigned int)plane_state->src_rect.width > caps->max_video_width)
 		return DC_FAIL_SURFACE_VALIDATE;
 
 	return DC_OK;
@@ -1266,7 +1267,7 @@ struct stream_encoder *dcn10_find_first_free_match_stream_enc_for_link(
 		const struct resource_pool *pool,
 		struct dc_stream_state *stream)
 {
-	int i;
+	unsigned int i;
 	int j = -1;
 	struct dc_link *link = stream->link;
 
@@ -1278,7 +1279,7 @@ struct stream_encoder *dcn10_find_first_free_match_stream_enc_for_link(
 			 */
 
 			if (pool->stream_enc[i]->id != ENGINE_ID_VIRTUAL)
-				j = i;
+				j = (int)i;
 
 			if (link->ep_type == DISPLAY_ENDPOINT_PHY && pool->stream_enc[i]->id ==
 					link->link_enc->preferred_engine)
@@ -1340,7 +1341,7 @@ static uint32_t read_pipe_fuses(struct dc_context *ctx)
 
 static bool verify_clock_values(struct dm_pp_clock_levels_with_voltage *clks)
 {
-	int i;
+	unsigned int i;
 
 	if (clks->num_levels == 0)
 		return false;
@@ -1358,7 +1359,7 @@ static bool dcn10_resource_construct(
 	struct dc *dc,
 	struct dcn10_resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	int j;
 	struct dc_context *ctx = dc->ctx;
 	uint32_t pipe_fuses = read_pipe_fuses(ctx);
@@ -1653,7 +1654,7 @@ static bool dcn10_resource_construct(
 		j++;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dcn10_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -1735,7 +1736,7 @@ struct resource_pool *dcn10_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn10_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn10_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	kfree(pool);
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
index a99829f239659..95abc4c38eae8 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn20/dcn20_resource.c
@@ -1120,7 +1120,7 @@ static void dcn20_resource_destruct(struct dcn20_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1156,7 +1156,7 @@ static void dcn20_resource_destruct(struct dcn20_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1169,19 +1169,19 @@ static void dcn20_resource_destruct(struct dcn20_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN20_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1415,7 +1415,7 @@ enum dc_status dcn20_add_dsc_to_stream_resource(struct dc *dc,
 		struct dc_stream_state *dc_stream)
 {
 	enum dc_status result = DC_OK;
-	int i;
+	unsigned int i;
 
 	/* Get a DSC if required and available */
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -1523,13 +1523,13 @@ bool dcn20_split_stream_for_odm(
 
 	*next_odm_pipe = *prev_odm_pipe;
 
-	next_odm_pipe->pipe_idx = pipe_idx;
+	next_odm_pipe->pipe_idx = (uint8_t)pipe_idx;
 	next_odm_pipe->plane_res.mi = pool->mis[next_odm_pipe->pipe_idx];
 	next_odm_pipe->plane_res.hubp = pool->hubps[next_odm_pipe->pipe_idx];
 	next_odm_pipe->plane_res.ipp = pool->ipps[next_odm_pipe->pipe_idx];
 	next_odm_pipe->plane_res.xfm = pool->transforms[next_odm_pipe->pipe_idx];
 	next_odm_pipe->plane_res.dpp = pool->dpps[next_odm_pipe->pipe_idx];
-	next_odm_pipe->plane_res.mpcc_inst = pool->dpps[next_odm_pipe->pipe_idx]->inst;
+	next_odm_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[next_odm_pipe->pipe_idx]->inst;
 	next_odm_pipe->stream_res.dsc = NULL;
 	if (prev_odm_pipe->next_odm_pipe && prev_odm_pipe->next_odm_pipe != next_odm_pipe) {
 		next_odm_pipe->next_odm_pipe = prev_odm_pipe->next_odm_pipe;
@@ -1580,13 +1580,13 @@ void dcn20_split_stream_for_mpc(
 	*secondary_pipe = *primary_pipe;
 	secondary_pipe->bottom_pipe = sec_bot_pipe;
 
-	secondary_pipe->pipe_idx = pipe_idx;
+	secondary_pipe->pipe_idx = (uint8_t)pipe_idx;
 	secondary_pipe->plane_res.mi = pool->mis[secondary_pipe->pipe_idx];
 	secondary_pipe->plane_res.hubp = pool->hubps[secondary_pipe->pipe_idx];
 	secondary_pipe->plane_res.ipp = pool->ipps[secondary_pipe->pipe_idx];
 	secondary_pipe->plane_res.xfm = pool->transforms[secondary_pipe->pipe_idx];
 	secondary_pipe->plane_res.dpp = pool->dpps[secondary_pipe->pipe_idx];
-	secondary_pipe->plane_res.mpcc_inst = pool->dpps[secondary_pipe->pipe_idx]->inst;
+	secondary_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[secondary_pipe->pipe_idx]->inst;
 	secondary_pipe->stream_res.dsc = NULL;
 	if (primary_pipe->bottom_pipe && primary_pipe->bottom_pipe != secondary_pipe) {
 		ASSERT(!secondary_pipe->bottom_pipe);
@@ -1636,7 +1636,8 @@ void dcn20_set_mcif_arb_params(
 {
 	enum mmhubbub_wbif_mode wbif_mode;
 	struct mcif_arb_params *wb_arb_params;
-	int i, j, dwb_pipe;
+	int j, dwb_pipe;
+	unsigned int i;
 
 	/* Writeback MCIF_WB arbitration parameters */
 	dwb_pipe = 0;
@@ -1680,7 +1681,7 @@ void dcn20_set_mcif_arb_params(
 
 bool dcn20_validate_dsc(struct dc *dc, struct dc_state *new_ctx)
 {
-	int i;
+	unsigned int i;
 
 	/* Validate DSC config, dsc count validation is already done */
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -1736,7 +1737,7 @@ struct pipe_ctx *dcn20_find_secondary_pipe(struct dc *dc,
 			preferred_pipe_idx = dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].next_odm_pipe->pipe_idx;
 			if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
 				secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-				secondary_pipe->pipe_idx = preferred_pipe_idx;
+				secondary_pipe->pipe_idx = (uint8_t)preferred_pipe_idx;
 			}
 		}
 		if (secondary_pipe == NULL &&
@@ -1744,7 +1745,7 @@ struct pipe_ctx *dcn20_find_secondary_pipe(struct dc *dc,
 			preferred_pipe_idx = dc->current_state->res_ctx.pipe_ctx[primary_pipe->pipe_idx].bottom_pipe->pipe_idx;
 			if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
 				secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-				secondary_pipe->pipe_idx = preferred_pipe_idx;
+				secondary_pipe->pipe_idx = (uint8_t)preferred_pipe_idx;
 			}
 		}
 
@@ -1762,7 +1763,7 @@ struct pipe_ctx *dcn20_find_secondary_pipe(struct dc *dc,
 
 					if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
 						secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-						secondary_pipe->pipe_idx = preferred_pipe_idx;
+						secondary_pipe->pipe_idx = (uint8_t)preferred_pipe_idx;
 						break;
 					}
 				}
@@ -1783,7 +1784,7 @@ struct pipe_ctx *dcn20_find_secondary_pipe(struct dc *dc,
 
 				if (res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) {
 					secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-					secondary_pipe->pipe_idx = preferred_pipe_idx;
+					secondary_pipe->pipe_idx = (uint8_t)preferred_pipe_idx;
 					break;
 				}
 			}
@@ -1797,7 +1798,7 @@ void dcn20_merge_pipes_for_validate(
 		struct dc *dc,
 		struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 
 	/* merge previously split odm pipes since mode support needs to make the decision */
 	for (i = 0; i < dc->res_pool->pipe_count; i++) {
@@ -1864,7 +1865,8 @@ int dcn20_validate_apply_pipe_split_flags(
 		int *split,
 		bool *merge)
 {
-	int i, pipe_idx, vlevel_split;
+	unsigned int i;
+	int pipe_idx, vlevel_split;
 	int plane_count = 0;
 	bool force_split = false;
 	bool avoid_split = dc->debug.pipe_split_policy == MPC_SPLIT_AVOID;
@@ -1897,7 +1899,7 @@ int dcn20_validate_apply_pipe_split_flags(
 				(!pipe->top_pipe || pipe->top_pipe->plane_state != pipe->plane_state))
 			++plane_count;
 	}
-	if (plane_count > dc->res_pool->pipe_count / 2)
+	if ((unsigned int)plane_count > dc->res_pool->pipe_count / 2)
 		avoid_split = true;
 
 	/* W/A: Mode timing with borders may not work well with pipe split, avoid for this corner case */
@@ -1923,12 +1925,12 @@ int dcn20_validate_apply_pipe_split_flags(
 			if (!context->res_ctx.pipe_ctx[i].stream)
 				continue;
 
-			for (vlevel_split = vlevel; vlevel <= context->bw_ctx.dml.soc.num_states; vlevel++)
+			for (vlevel_split = vlevel; (unsigned int)vlevel <= context->bw_ctx.dml.soc.num_states; vlevel++)
 				if (v->NoOfDPP[vlevel][0][pipe_idx] == 1 &&
 						v->ModeSupport[vlevel][0])
 					break;
 			/* Impossible to not split this pipe */
-			if (vlevel > context->bw_ctx.dml.soc.num_states)
+			if ((unsigned int)vlevel > context->bw_ctx.dml.soc.num_states)
 				vlevel = vlevel_split;
 			else
 				max_mpc_comb = 0;
@@ -2064,7 +2066,8 @@ bool dcn20_fast_validate_bw(
 	bool out = false;
 	int split[MAX_PIPES] = { 0 };
 	bool merge[MAX_PIPES] = { false };
-	int pipe_cnt, i, pipe_idx, vlevel;
+	int pipe_cnt, pipe_idx, vlevel;
+	unsigned int i;
 
 	ASSERT(pipes);
 	if (!pipes)
@@ -2083,7 +2086,7 @@ bool dcn20_fast_validate_bw(
 
 	vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
 
-	if (vlevel > context->bw_ctx.dml.soc.num_states)
+	if ((unsigned int)vlevel > context->bw_ctx.dml.soc.num_states)
 		goto validate_fail;
 
 	vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
@@ -2216,7 +2219,7 @@ struct pipe_ctx *dcn20_acquire_free_pipe_for_layer(
 	sec_dpp_pipe->plane_res.hubp = pool->hubps[sec_dpp_pipe->pipe_idx];
 	sec_dpp_pipe->plane_res.ipp = pool->ipps[sec_dpp_pipe->pipe_idx];
 	sec_dpp_pipe->plane_res.dpp = pool->dpps[sec_dpp_pipe->pipe_idx];
-	sec_dpp_pipe->plane_res.mpcc_inst = pool->dpps[sec_dpp_pipe->pipe_idx]->inst;
+	sec_dpp_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[sec_dpp_pipe->pipe_idx]->inst;
 
 	return sec_dpp_pipe;
 }
@@ -2289,7 +2292,7 @@ static const struct resource_funcs dcn20_res_pool_funcs = {
 
 bool dcn20_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -2311,7 +2314,7 @@ bool dcn20_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 
 bool dcn20_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	ASSERT(pipe_count > 0);
@@ -2569,7 +2572,7 @@ static bool dcn20_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; (unsigned int)i < pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2617,12 +2620,12 @@ static bool dcn20_resource_construct(
 
 	if (!dc->debug.disable_pplib_wm_range) {
 		struct pp_smu_wm_range_sets ranges = {0};
-		int i = 0;
+		int j = 0;
 
 		ranges.num_reader_wm_sets = 0;
 
 		if (loaded_bb->num_states == 1) {
-			ranges.reader_wm_sets[0].wm_inst = i;
+			ranges.reader_wm_sets[0].wm_inst = (uint8_t)j;
 			ranges.reader_wm_sets[0].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
 			ranges.reader_wm_sets[0].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
 			ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
@@ -2630,15 +2633,14 @@ static bool dcn20_resource_construct(
 
 			ranges.num_reader_wm_sets = 1;
 		} else if (loaded_bb->num_states > 1) {
-			for (i = 0; i < 4 && i < loaded_bb->num_states; i++) {
-				ranges.reader_wm_sets[i].wm_inst = i;
-				ranges.reader_wm_sets[i].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
-				ranges.reader_wm_sets[i].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
+			for (j = 0; j < 4 && (unsigned int)j < loaded_bb->num_states; j++) {
+				ranges.reader_wm_sets[j].wm_inst = (uint8_t)j;
+				ranges.reader_wm_sets[j].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
+				ranges.reader_wm_sets[j].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
 				DC_FP_START();
-				dcn20_fpu_set_wm_ranges(i, &ranges, loaded_bb);
+				dcn20_fpu_set_wm_ranges(j, &ranges, loaded_bb);
 				DC_FP_END();
-
-				ranges.num_reader_wm_sets = i + 1;
+				ranges.num_reader_wm_sets = j + 1;
 			}
 
 			ranges.reader_wm_sets[0].min_fill_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
@@ -2664,7 +2666,7 @@ static bool dcn20_resource_construct(
 		goto create_fail;
 
 	/* mem input -> ipp -> dpp -> opp -> TG */
-	for (i = 0; i < pool->base.pipe_count; i++) {
+	for (i = 0; (unsigned int)i < pool->base.pipe_count; i++) {
 		pool->base.hubps[i] = dcn20_hubp_create(ctx, i);
 		if (pool->base.hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2792,7 +2794,7 @@ static bool dcn20_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; (unsigned int)i < dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 2;
@@ -2829,7 +2831,7 @@ struct resource_pool *dcn20_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn20_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn20_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
index 84f6d9dc443f5..b653f951e1047 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn21/dcn21_resource.c
@@ -684,7 +684,7 @@ static void dcn21_resource_destruct(struct dcn21_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -719,7 +719,7 @@ static void dcn21_resource_destruct(struct dcn21_resource_pool *pool)
 			dal_irq_service_destroy(&pool->base.irqs);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -732,19 +732,19 @@ static void dcn21_resource_destruct(struct dcn21_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN20_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -804,7 +804,8 @@ bool dcn21_fast_validate_bw(struct dc *dc,
 	bool out = false;
 	int split[MAX_PIPES] = { 0 };
 	bool merge[MAX_PIPES] = { false };
-	int pipe_cnt, i, pipe_idx, vlevel;
+	int pipe_cnt, pipe_idx, vlevel;
+	unsigned int i;
 
 	ASSERT(pipes);
 	if (!pipes)
@@ -829,7 +830,7 @@ bool dcn21_fast_validate_bw(struct dc *dc,
 				dm_allow_self_refresh_and_mclk_switch;
 	vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
 
-	if (vlevel > context->bw_ctx.dml.soc.num_states) {
+	if ((unsigned int)vlevel > context->bw_ctx.dml.soc.num_states) {
 
 		if (allow_self_refresh_only) {
 			/*
@@ -842,7 +843,7 @@ bool dcn21_fast_validate_bw(struct dc *dc,
 			context->bw_ctx.dml.soc.allow_dram_self_refresh_or_dram_clock_change_in_vblank =
 						dm_allow_self_refresh;
 			vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
-			if (vlevel > context->bw_ctx.dml.soc.num_states)
+			if ((unsigned int)vlevel > context->bw_ctx.dml.soc.num_states)
 				goto validate_fail;
 		} else {
 			goto validate_fail;
@@ -1427,7 +1428,8 @@ static bool dcn21_resource_construct(
 	struct dc *dc,
 	struct dcn21_resource_pool *pool)
 {
-	int i, j;
+	unsigned int i;
+	int j;
 	struct dc_context *ctx = dc->ctx;
 	struct irq_service_init_data init_data;
 	uint32_t pipe_fuses = read_pipe_fuses(ctx);
@@ -1659,7 +1661,7 @@ static bool dcn21_resource_construct(
 		j++;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dcn21_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -1703,11 +1705,11 @@ static bool dcn21_resource_construct(
 		goto create_fail;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		pool->base.dscs[i] = dcn21_dsc_create(ctx, i);
 		if (pool->base.dscs[i] == NULL) {
 			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			dm_error("DC: failed to create display stream compressor %u!\n", i);
 			goto create_fail;
 		}
 	}
@@ -1757,7 +1759,7 @@ struct resource_pool *dcn21_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn21_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn21_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
index 4a864689c44f3..4a7f2c5d34e6e 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn30/dcn30_resource.c
@@ -1111,7 +1111,7 @@ static void dcn30_resource_destruct(struct dcn30_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1147,7 +1147,7 @@ static void dcn30_resource_destruct(struct dcn30_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1160,19 +1160,19 @@ static void dcn30_resource_destruct(struct dcn30_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1195,7 +1195,7 @@ static void dcn30_resource_destruct(struct dcn30_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1250,7 +1250,7 @@ static struct hubp *dcn30_hubp_create(
 
 static bool dcn30_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1274,7 +1274,7 @@ static bool dcn30_dwbc_create(struct dc_context *ctx, struct resource_pool *pool
 
 static bool dcn30_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1355,7 +1355,8 @@ int dcn30_populate_dml_pipes_from_context(
 	display_e2e_pipe_params_st *pipes,
 	enum dc_validate_mode validate_mode)
 {
-	int i, pipe_cnt;
+	int pipe_cnt;
+	unsigned int i;
 	struct resource_context *res_ctx = &context->res_ctx;
 
 	DC_FP_START();
@@ -1413,7 +1414,8 @@ void dcn30_set_mcif_arb_params(
 	enum mmhubbub_wbif_mode wbif_mode;
 	struct display_mode_lib *dml = &context->bw_ctx.dml;
 	struct mcif_arb_params *wb_arb_params;
-	int i, j, dwb_pipe;
+	int j, dwb_pipe;
+	unsigned int i;
 
 	/* Writeback MCIF_WB arbitration parameters */
 	dwb_pipe = 0;
@@ -1563,13 +1565,13 @@ static bool dcn30_split_stream_for_mpc_or_odm(
 
 	*sec_pipe = *pri_pipe;
 
-	sec_pipe->pipe_idx = pipe_idx;
+	sec_pipe->pipe_idx = (uint8_t)pipe_idx;
 	sec_pipe->plane_res.mi = pool->mis[pipe_idx];
 	sec_pipe->plane_res.hubp = pool->hubps[pipe_idx];
 	sec_pipe->plane_res.ipp = pool->ipps[pipe_idx];
 	sec_pipe->plane_res.xfm = pool->transforms[pipe_idx];
 	sec_pipe->plane_res.dpp = pool->dpps[pipe_idx];
-	sec_pipe->plane_res.mpcc_inst = pool->dpps[pipe_idx]->inst;
+	sec_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[pipe_idx]->inst;
 	sec_pipe->stream_res.dsc = NULL;
 	if (odm) {
 		if (pri_pipe->next_odm_pipe) {
@@ -1623,7 +1625,7 @@ static struct pipe_ctx *dcn30_find_split_pipe(
 
 	if (old_index >= 0 && context->res_ctx.pipe_ctx[old_index].stream == NULL) {
 		pipe = &context->res_ctx.pipe_ctx[old_index];
-		pipe->pipe_idx = old_index;
+		pipe->pipe_idx = (uint8_t)old_index;
 	}
 
 	if (!pipe)
@@ -1632,7 +1634,7 @@ static struct pipe_ctx *dcn30_find_split_pipe(
 					&& dc->current_state->res_ctx.pipe_ctx[i].prev_odm_pipe == NULL) {
 				if (context->res_ctx.pipe_ctx[i].stream == NULL) {
 					pipe = &context->res_ctx.pipe_ctx[i];
-					pipe->pipe_idx = i;
+					pipe->pipe_idx = (uint8_t)i;
 					break;
 				}
 			}
@@ -1647,7 +1649,7 @@ static struct pipe_ctx *dcn30_find_split_pipe(
 		for (i = dc->res_pool->pipe_count - 1; i >= 0; i--) {
 			if (context->res_ctx.pipe_ctx[i].stream == NULL) {
 				pipe = &context->res_ctx.pipe_ctx[i];
-				pipe->pipe_idx = i;
+				pipe->pipe_idx = (uint8_t)i;
 				break;
 			}
 		}
@@ -1669,7 +1671,8 @@ noinline bool dcn30_internal_validate_bw(
 	int split[MAX_PIPES] = { 0 };
 	bool merge[MAX_PIPES] = { false };
 	bool newly_split[MAX_PIPES] = { false };
-	int pipe_cnt, i, pipe_idx, vlevel = 0;
+	unsigned int i;
+	int pipe_cnt, pipe_idx, vlevel = 0;
 	struct vba_vars_st *vba = &context->bw_ctx.dml.vba;
 
 	ASSERT(pipes);
@@ -1701,7 +1704,7 @@ noinline bool dcn30_internal_validate_bw(
 			dm_allow_self_refresh_and_mclk_switch;
 		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
 		/* This may adjust vlevel and maxMpcComb */
-		if (vlevel < context->bw_ctx.dml.soc.num_states)
+		if ((unsigned int)vlevel < context->bw_ctx.dml.soc.num_states)
 			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
 	}
 	if (allow_self_refresh_only &&
@@ -1719,7 +1722,7 @@ noinline bool dcn30_internal_validate_bw(
 			dm_allow_self_refresh;
 
 		vlevel = dml_get_voltage_level(&context->bw_ctx.dml, pipes, pipe_cnt);
-		if (vlevel < context->bw_ctx.dml.soc.num_states) {
+		if ((unsigned int)vlevel < context->bw_ctx.dml.soc.num_states) {
 			memset(split, 0, sizeof(split));
 			memset(merge, 0, sizeof(merge));
 			vlevel = dcn20_validate_apply_pipe_split_flags(dc, context, vlevel, split, merge);
@@ -2159,13 +2162,13 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params
 	if (bw_params->clk_table.entries[0].memclk_mhz) {
 
 		for (i = 0; i < MAX_NUM_DPM_LVL; i++) {
-			if (bw_params->clk_table.entries[i].dcfclk_mhz > dcn30_bb_max_clk.max_dcfclk_mhz)
+			if (bw_params->clk_table.entries[i].dcfclk_mhz > (unsigned int)dcn30_bb_max_clk.max_dcfclk_mhz)
 				dcn30_bb_max_clk.max_dcfclk_mhz = bw_params->clk_table.entries[i].dcfclk_mhz;
-			if (bw_params->clk_table.entries[i].dispclk_mhz > dcn30_bb_max_clk.max_dispclk_mhz)
+			if (bw_params->clk_table.entries[i].dispclk_mhz > (unsigned int)dcn30_bb_max_clk.max_dispclk_mhz)
 				dcn30_bb_max_clk.max_dispclk_mhz = bw_params->clk_table.entries[i].dispclk_mhz;
-			if (bw_params->clk_table.entries[i].dppclk_mhz > dcn30_bb_max_clk.max_dppclk_mhz)
+			if (bw_params->clk_table.entries[i].dppclk_mhz > (unsigned int)dcn30_bb_max_clk.max_dppclk_mhz)
 				dcn30_bb_max_clk.max_dppclk_mhz = bw_params->clk_table.entries[i].dppclk_mhz;
-			if (bw_params->clk_table.entries[i].phyclk_mhz > dcn30_bb_max_clk.max_phyclk_mhz)
+			if (bw_params->clk_table.entries[i].phyclk_mhz > (unsigned int)dcn30_bb_max_clk.max_phyclk_mhz)
 				dcn30_bb_max_clk.max_phyclk_mhz = bw_params->clk_table.entries[i].phyclk_mhz;
 		}
 
@@ -2173,14 +2176,14 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params
 		dcn30_fpu_update_max_clk(&dcn30_bb_max_clk);
 		DC_FP_END();
 
-		if (dcn30_bb_max_clk.max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
+		if ((unsigned int)dcn30_bb_max_clk.max_dcfclk_mhz > dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
 			// If max DCFCLK is greater than the max DCFCLK STA target, insert into the DCFCLK STA target array
 			dcfclk_sta_targets[num_dcfclk_sta_targets] = dcn30_bb_max_clk.max_dcfclk_mhz;
 			num_dcfclk_sta_targets++;
-		} else if (dcn30_bb_max_clk.max_dcfclk_mhz < dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
+		} else if ((unsigned int)dcn30_bb_max_clk.max_dcfclk_mhz < dcfclk_sta_targets[num_dcfclk_sta_targets-1]) {
 			// If max DCFCLK is less than the max DCFCLK STA target, cap values and remove duplicates
 			for (i = 0; i < num_dcfclk_sta_targets; i++) {
-				if (dcfclk_sta_targets[i] > dcn30_bb_max_clk.max_dcfclk_mhz) {
+				if (dcfclk_sta_targets[i] > (unsigned int)dcn30_bb_max_clk.max_dcfclk_mhz) {
 					dcfclk_sta_targets[i] = dcn30_bb_max_clk.max_dcfclk_mhz;
 					break;
 				}
@@ -2232,7 +2235,7 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params
 				dcfclk_mhz[num_states] = dcfclk_sta_targets[i];
 				dram_speed_mts[num_states++] = optimal_uclk_for_dcfclk_sta_targets[i++];
 			} else {
-				if (j < num_uclk_states && optimal_dcfclk_for_uclk[j] <= dcn30_bb_max_clk.max_dcfclk_mhz) {
+				if (j < num_uclk_states && optimal_dcfclk_for_uclk[j] <= (unsigned int)dcn30_bb_max_clk.max_dcfclk_mhz) {
 					dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j];
 					dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16;
 				} else {
@@ -2247,7 +2250,7 @@ void dcn30_update_bw_bounding_box(struct dc *dc, struct clk_bw_params *bw_params
 		}
 
 		while (j < num_uclk_states && num_states < DC__VOLTAGE_STATES &&
-				optimal_dcfclk_for_uclk[j] <= dcn30_bb_max_clk.max_dcfclk_mhz) {
+				optimal_dcfclk_for_uclk[j] <= (unsigned int)dcn30_bb_max_clk.max_dcfclk_mhz) {
 			dcfclk_mhz[num_states] = optimal_dcfclk_for_uclk[j];
 			dram_speed_mts[num_states++] = bw_params->clk_table.entries[j++].memclk_mhz * 16;
 		}
@@ -2307,7 +2310,7 @@ static bool dcn30_resource_construct(
 	struct dc *dc,
 	struct dcn30_resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	struct dc_context *ctx = dc->ctx;
 	struct irq_service_init_data init_data;
 	struct ddc_service_init_data ddc_init_data = {0};
@@ -2379,7 +2382,7 @@ static bool dcn30_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //3
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut;
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -2529,7 +2532,7 @@ static bool dcn30_resource_construct(
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		pool->base.opps[i] = dcn30_opp_create(ctx, i);
 		if (pool->base.opps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2539,7 +2542,7 @@ static bool dcn30_resource_construct(
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		pool->base.timing_generators[i] = dcn30_timing_generator_create(
 				ctx, i);
 		if (pool->base.timing_generators[i] == NULL) {
@@ -2559,13 +2562,13 @@ static bool dcn30_resource_construct(
 	}
 
 	/* ABM */
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
 				&abm_regs[i],
 				&abm_shift,
 				&abm_mask);
 		if (pool->base.multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			dm_error("DC: failed to create abm for pipe %u!\n", i);
 			BREAK_TO_DEBUGGER();
 			goto create_fail;
 		}
@@ -2578,11 +2581,11 @@ static bool dcn30_resource_construct(
 		goto create_fail;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		pool->base.dscs[i] = dcn30_dsc_create(ctx, i);
 		if (pool->base.dscs[i] == NULL) {
 			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			dm_error("DC: failed to create display stream compressor %u!\n", i);
 			goto create_fail;
 		}
 	}
@@ -2601,7 +2604,7 @@ static bool dcn30_resource_construct(
 	}
 
 	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dcn30_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2669,7 +2672,7 @@ struct resource_pool *dcn30_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn30_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn30_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c
index a74b150cbe9c4..0ecd3a2fcebd1 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn301/dcn301_resource.c
@@ -1082,7 +1082,7 @@ static void dcn301_destruct(struct dcn301_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1118,7 +1118,7 @@ static void dcn301_destruct(struct dcn301_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1131,19 +1131,19 @@ static void dcn301_destruct(struct dcn301_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1166,7 +1166,7 @@ static void dcn301_destruct(struct dcn301_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1182,7 +1182,7 @@ static void dcn301_destruct(struct dcn301_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1210,7 +1210,7 @@ static struct hubp *dcn301_hubp_create(struct dc_context *ctx, uint32_t inst)
 
 static bool dcn301_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1234,7 +1234,7 @@ static bool dcn301_dwbc_create(struct dc_context *ctx, struct resource_pool *poo
 
 static bool dcn301_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1358,7 +1358,7 @@ static void set_wm_ranges(
 		struct _vcs_dpi_soc_bounding_box_st *loaded_bb)
 {
 	struct pp_smu_wm_range_sets ranges = {0};
-	int i;
+	unsigned int i;
 
 	ranges.num_reader_wm_sets = 0;
 
@@ -1372,7 +1372,7 @@ static void set_wm_ranges(
 		ranges.num_reader_wm_sets = 1;
 	} else if (loaded_bb->num_states > 1) {
 		for (i = 0; i < 4 && i < loaded_bb->num_states; i++) {
-			ranges.reader_wm_sets[i].wm_inst = i;
+			ranges.reader_wm_sets[i].wm_inst = (uint8_t)i;
 			ranges.reader_wm_sets[i].min_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MIN;
 			ranges.reader_wm_sets[i].max_drain_clk_mhz = PP_SMU_WM_SET_RANGE_CLK_UNCONSTRAINED_MAX;
 			DC_FP_START();
@@ -1444,7 +1444,8 @@ static bool dcn301_resource_construct(
 	struct dc *dc,
 	struct dcn301_resource_pool *pool)
 {
-	int i, j;
+	int j;
+	unsigned int i;
 	struct dc_context *ctx = dc->ctx;
 	struct irq_service_init_data init_data;
 	uint32_t pipe_fuses = read_pipe_fuses(ctx);
@@ -1506,7 +1507,7 @@ static bool dcn301_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut;
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -1676,13 +1677,13 @@ static bool dcn301_resource_construct(
 
 	/* ABM (or ABMs for NV2x) */
 	/* TODO: */
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		pool->base.multiple_abms[i] = dmub_abm_create(ctx,
 				&abm_regs[i],
 				&abm_shift,
 				&abm_mask);
 		if (pool->base.multiple_abms[i] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			dm_error("DC: failed to create abm for pipe %u!\n", i);
 			BREAK_TO_DEBUGGER();
 			goto create_fail;
 		}
@@ -1696,11 +1697,11 @@ static bool dcn301_resource_construct(
 		goto create_fail;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		pool->base.dscs[i] = dcn301_dsc_create(ctx, i);
 		if (pool->base.dscs[i] == NULL) {
 			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			dm_error("DC: failed to create display stream compressor %u!\n", i);
 			goto create_fail;
 		}
 	}
@@ -1719,7 +1720,7 @@ static bool dcn301_resource_construct(
 	}
 
 	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dcn301_aux_engine_create(ctx, i);
 		if (pool->base.engines[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -1773,7 +1774,7 @@ struct resource_pool *dcn301_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn301_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn301_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
index 539c5aa6bffaa..f7b3947a90915 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn302/dcn302_resource.c
@@ -738,7 +738,7 @@ static const struct dcn30_dwbc_mask dwbc30_mask = {
 
 static bool dcn302_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -773,7 +773,7 @@ static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
 
 static bool dcn302_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1036,7 +1036,7 @@ static void dcn302_resource_destruct(struct resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_dsc; i++) {
 		if (pool->dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->dscs[i]);
 	}
@@ -1071,7 +1071,7 @@ static void dcn302_resource_destruct(struct resource_pool *pool)
 			dal_irq_service_destroy(&pool->irqs);
 	}
 
-	for (i = 0; i < pool->res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_ddc; i++) {
 		if (pool->engines[i] != NULL)
 			dce110_engine_destroy(&pool->engines[i]);
 		if (pool->hw_i2cs[i] != NULL) {
@@ -1084,19 +1084,19 @@ static void dcn302_resource_destruct(struct resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_opp; i++) {
 		if (pool->opps[i] != NULL)
 			pool->opps[i]->funcs->opp_destroy(&pool->opps[i]);
 	}
 
-	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_timing_generator; i++) {
 		if (pool->timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->timing_generators[i]));
 			pool->timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_dwb; i++) {
 		if (pool->dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->dwbc[i]));
 			pool->dwbc[i] = NULL;
@@ -1120,7 +1120,7 @@ static void dcn302_resource_destruct(struct resource_pool *pool)
 	if (pool->dp_clock_source != NULL)
 		dcn20_clock_source_destroy(&pool->dp_clock_source);
 
-	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_mpc_3dlut; i++) {
 		if (pool->mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->mpc_lut[i]);
 			pool->mpc_lut[i] = NULL;
@@ -1300,7 +1300,7 @@ static bool dcn302_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->res_cap->num_mpc_3dlut; //3
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->res_cap->num_mpc_3dlut;
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -1371,7 +1371,7 @@ static bool dcn302_resource_construct(
 					CLOCK_SOURCE_ID_DP_DTO,
 					&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->clk_src_count; i++) {
+	for (i = 0; i < (int)pool->clk_src_count; i++) {
 		if (pool->clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -1416,7 +1416,7 @@ static bool dcn302_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->pipe_count; i++) {
+	for (i = 0; i < (int)pool->pipe_count; i++) {
 		pool->hubps[i] = dcn302_hubp_create(ctx, i);
 		if (pool->hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -1526,7 +1526,7 @@ static bool dcn302_resource_construct(
 
 	dc->caps.max_planes =  pool->pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -1560,7 +1560,7 @@ struct resource_pool *dcn302_create_resource_pool(const struct dc_init_data *ini
 	if (!pool)
 		return NULL;
 
-	if (dcn302_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn302_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return pool;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
index 529eccb4ed3bf..429365f226226 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn303/dcn303_resource.c
@@ -699,7 +699,7 @@ static const struct dcn30_dwbc_mask dwbc30_mask = {
 
 static bool dcn303_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -734,7 +734,7 @@ static const struct dcn30_mmhubbub_mask mcif_wb30_mask = {
 
 static bool dcn303_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -980,7 +980,7 @@ static void dcn303_resource_destruct(struct resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_dsc; i++) {
 		if (pool->dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->dscs[i]);
 	}
@@ -1015,7 +1015,7 @@ static void dcn303_resource_destruct(struct resource_pool *pool)
 			dal_irq_service_destroy(&pool->irqs);
 	}
 
-	for (i = 0; i < pool->res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_ddc; i++) {
 		if (pool->engines[i] != NULL)
 			dce110_engine_destroy(&pool->engines[i]);
 		if (pool->hw_i2cs[i] != NULL) {
@@ -1028,19 +1028,19 @@ static void dcn303_resource_destruct(struct resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_opp; i++) {
 		if (pool->opps[i] != NULL)
 			pool->opps[i]->funcs->opp_destroy(&pool->opps[i]);
 	}
 
-	for (i = 0; i < pool->res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_timing_generator; i++) {
 		if (pool->timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->timing_generators[i]));
 			pool->timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_dwb; i++) {
 		if (pool->dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->dwbc[i]));
 			pool->dwbc[i] = NULL;
@@ -1064,7 +1064,7 @@ static void dcn303_resource_destruct(struct resource_pool *pool)
 	if (pool->dp_clock_source != NULL)
 		dcn20_clock_source_destroy(&pool->dp_clock_source);
 
-	for (i = 0; i < pool->res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->res_cap->num_mpc_3dlut; i++) {
 		if (pool->mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->mpc_lut[i]);
 			pool->mpc_lut[i] = NULL;
@@ -1244,7 +1244,7 @@ static bool dcn303_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->res_cap->num_mpc_3dlut; //3
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->res_cap->num_mpc_3dlut;
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -1303,7 +1303,7 @@ static bool dcn303_resource_construct(
 					CLOCK_SOURCE_ID_DP_DTO,
 					&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->clk_src_count; i++) {
+	for (i = 0; i < (int)pool->clk_src_count; i++) {
 		if (pool->clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -1348,7 +1348,7 @@ static bool dcn303_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->pipe_count; i++) {
+	for (i = 0; i < (int)pool->pipe_count; i++) {
 		pool->hubps[i] = dcn303_hubp_create(ctx, i);
 		if (pool->hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -1458,7 +1458,7 @@ static bool dcn303_resource_construct(
 
 	dc->caps.max_planes =  pool->pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -1492,7 +1492,7 @@ struct resource_pool *dcn303_create_resource_pool(const struct dc_init_data *ini
 	if (!pool)
 		return NULL;
 
-	if (dcn303_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn303_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return pool;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
index 39944d90ea980..df3e35a6a0a58 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn31/dcn31_resource.c
@@ -1146,7 +1146,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1412,7 +1412,7 @@ static void dcn31_resource_destruct(struct dcn31_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1446,7 +1446,7 @@ static void dcn31_resource_destruct(struct dcn31_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1459,19 +1459,19 @@ static void dcn31_resource_destruct(struct dcn31_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1494,7 +1494,7 @@ static void dcn31_resource_destruct(struct dcn31_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1510,7 +1510,7 @@ static void dcn31_resource_destruct(struct dcn31_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1546,7 +1546,7 @@ static struct hubp *dcn31_hubp_create(
 
 static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1570,7 +1570,7 @@ static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool
 
 static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1651,7 +1651,7 @@ int dcn31x_populate_dml_pipes_from_context(struct dc *dc,
 					  enum dc_validate_mode validate_mode)
 {
 	uint32_t pipe_cnt;
-	int i;
+	unsigned int i;
 
 	dc_assert_fp_enabled();
 
@@ -1675,7 +1675,7 @@ int dcn31_populate_dml_pipes_from_context(
 	display_e2e_pipe_params_st *pipes,
 	enum dc_validate_mode validate_mode)
 {
-	int i, pipe_cnt;
+	unsigned int i, pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe = 0;
 	bool upscaled = false;
@@ -1987,7 +1987,7 @@ static bool dcn31_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut; //2
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -2076,7 +2076,7 @@ static bool dcn31_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2115,7 +2115,7 @@ static bool dcn31_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
+	for (i = 0; i < (int)pool->base.pipe_count; i++) {
 		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
 		if (pool->base.hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2252,7 +2252,7 @@ static bool dcn31_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2279,7 +2279,7 @@ struct resource_pool *dcn31_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn31_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn31_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
index 5acc545bbe7f3..74a14c0be20fb 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn314/dcn314_resource.c
@@ -1204,7 +1204,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1471,7 +1471,7 @@ static void dcn314_resource_destruct(struct dcn314_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1504,7 +1504,7 @@ static void dcn314_resource_destruct(struct dcn314_resource_pool *pool)
 			dal_irq_service_destroy(&pool->base.irqs);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1517,19 +1517,19 @@ static void dcn314_resource_destruct(struct dcn314_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1552,7 +1552,7 @@ static void dcn314_resource_destruct(struct dcn314_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1568,7 +1568,7 @@ static void dcn314_resource_destruct(struct dcn314_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1604,7 +1604,7 @@ static struct hubp *dcn31_hubp_create(
 
 static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1628,7 +1628,7 @@ static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool
 
 static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1914,7 +1914,7 @@ static bool dcn314_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut; //2
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -1996,7 +1996,7 @@ static bool dcn314_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2033,7 +2033,7 @@ static bool dcn314_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
+	for (i = 0; i < (int)pool->base.pipe_count; i++) {
 		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
 		if (pool->base.hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2163,7 +2163,7 @@ static bool dcn314_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2191,7 +2191,7 @@ struct resource_pool *dcn314_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn314_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn314_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
index 975e14f3f5fa7..30b2e73abd083 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn315/dcn315_resource.c
@@ -1145,7 +1145,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1413,7 +1413,7 @@ static void dcn315_resource_destruct(struct dcn315_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1447,7 +1447,7 @@ static void dcn315_resource_destruct(struct dcn315_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1460,19 +1460,19 @@ static void dcn315_resource_destruct(struct dcn315_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1495,7 +1495,7 @@ static void dcn315_resource_destruct(struct dcn315_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1511,7 +1511,7 @@ static void dcn315_resource_destruct(struct dcn315_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1547,7 +1547,7 @@ static struct hubp *dcn31_hubp_create(
 
 static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1571,7 +1571,7 @@ static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool
 
 static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1666,7 +1666,7 @@ static int source_format_to_bpp (enum source_format_class SourcePixelFormat)
 
 static bool allow_pixel_rate_crb(struct dc *dc, struct dc_state *context)
 {
-	int i;
+	unsigned int i;
 	struct resource_context *res_ctx = &context->res_ctx;
 
 	/* Only apply for dual stream scenarios with edp*/
@@ -1699,7 +1699,8 @@ static int dcn315_populate_dml_pipes_from_context(
 	display_e2e_pipe_params_st *pipes,
 	enum dc_validate_mode validate_mode)
 {
-	int i, pipe_cnt, crb_idx, crb_pipes;
+	unsigned int i;
+	int pipe_cnt, crb_idx, crb_pipes;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe = NULL;
 	const int max_usable_det = context->bw_ctx.dml.ip.config_return_buffer_size_in_kbytes - DCN3_15_MIN_COMPBUF_SIZE_KB;
@@ -1739,7 +1740,7 @@ static int dcn315_populate_dml_pipes_from_context(
 
 			if (approx_det_segs_required_for_pstate <= 2 * DCN3_15_MAX_DET_SEGS) {
 				bool split_required = approx_det_segs_required_for_pstate > DCN3_15_MAX_DET_SEGS;
-				split_required = split_required || timing->pix_clk_100hz >= dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc);
+				split_required = split_required || (unsigned int)timing->pix_clk_100hz >= (unsigned int)dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc);
 				split_required = split_required || (pipe->plane_state && pipe->plane_state->src_rect.width > 5120);
 
 				/* Minimum 2 segments to allow mpc/odm combine if its used later */
@@ -1788,7 +1789,7 @@ static int dcn315_populate_dml_pipes_from_context(
 				continue;
 			}
 
-			bool split_required = pipe->stream->timing.pix_clk_100hz >= dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc)
+			bool split_required = (unsigned int)pipe->stream->timing.pix_clk_100hz >= (unsigned int)dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc)
 					|| (pipe->plane_state && pipe->plane_state->src_rect.width > 5120);
 
 			if (remaining_det_segs > MIN_RESERVED_DET_SEGS && crb_pipes != 0)
@@ -1825,7 +1826,7 @@ static int dcn315_populate_dml_pipes_from_context(
 					(max_usable_det / DCN3_15_CRB_SEGMENT_SIZE_KB / 4) * DCN3_15_CRB_SEGMENT_SIZE_KB;
 		} else if (!is_dual_plane(pipe->plane_state->format)
 				&& pipe->plane_state->src_rect.width <= 5120
-				&& pipe->stream->timing.pix_clk_100hz < dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc)) {
+				&& (unsigned int)pipe->stream->timing.pix_clk_100hz < (unsigned int)dcn_get_max_non_odm_pix_rate_100hz(&dc->dml.soc)) {
 			/* Limit to 5k max to avoid forced pipe split when there is not enough detile for swath */
 			context->bw_ctx.dml.ip.det_buffer_size_kbytes = 192;
 			pipes[0].pipe.src.unbounded_req_mode = true;
@@ -1957,7 +1958,7 @@ static bool dcn315_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut;
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -2026,7 +2027,7 @@ static bool dcn315_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2065,7 +2066,7 @@ static bool dcn315_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
+	for (i = 0; i < (int)pool->base.pipe_count; i++) {
 		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
 		if (pool->base.hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2192,7 +2193,7 @@ static bool dcn315_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2220,7 +2221,7 @@ struct resource_pool *dcn315_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn315_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn315_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
index 914d91df174ca..0a6dfb44e90d5 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn316/dcn316_resource.c
@@ -1138,7 +1138,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1408,7 +1408,7 @@ static void dcn316_resource_destruct(struct dcn316_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1442,7 +1442,7 @@ static void dcn316_resource_destruct(struct dcn316_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1455,19 +1455,19 @@ static void dcn316_resource_destruct(struct dcn316_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1490,7 +1490,7 @@ static void dcn316_resource_destruct(struct dcn316_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1506,7 +1506,7 @@ static void dcn316_resource_destruct(struct dcn316_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1539,7 +1539,7 @@ static struct hubp *dcn31_hubp_create(
 
 static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1563,7 +1563,7 @@ static bool dcn31_dwbc_create(struct dc_context *ctx, struct resource_pool *pool
 
 static bool dcn31_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1644,7 +1644,8 @@ static int dcn316_populate_dml_pipes_from_context(
 	display_e2e_pipe_params_st *pipes,
 	enum dc_validate_mode validate_mode)
 {
-	int i, pipe_cnt;
+	unsigned int i;
+	int pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe = 0;
 	const int max_usable_det = context->bw_ctx.dml.ip.config_return_buffer_size_in_kbytes - DCN3_16_MIN_COMPBUF_SIZE_KB;
@@ -1833,7 +1834,7 @@ static bool dcn316_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut; //2
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -1900,7 +1901,7 @@ static bool dcn316_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -1939,7 +1940,7 @@ static bool dcn316_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
+	for (i = 0; i < (int)pool->base.pipe_count; i++) {
 		pool->base.hubps[i] = dcn31_hubp_create(ctx, i);
 		if (pool->base.hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2058,7 +2059,7 @@ static bool dcn316_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2086,7 +2087,7 @@ struct resource_pool *dcn316_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn316_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn316_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
index 3751f7a94a059..a6985742c950b 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource.c
@@ -1424,7 +1424,7 @@ static void dcn32_resource_destruct(struct dcn32_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1454,7 +1454,7 @@ static void dcn32_resource_destruct(struct dcn32_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1467,19 +1467,19 @@ static void dcn32_resource_destruct(struct dcn32_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1502,7 +1502,7 @@ static void dcn32_resource_destruct(struct dcn32_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1518,7 +1518,7 @@ static void dcn32_resource_destruct(struct dcn32_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1544,7 +1544,7 @@ static void dcn32_resource_destruct(struct dcn32_resource_pool *pool)
 
 static bool dcn32_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t dwb_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < dwb_count; i++) {
@@ -1572,7 +1572,7 @@ static bool dcn32_dwbc_create(struct dc_context *ctx, struct resource_pool *pool
 
 static bool dcn32_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t dwb_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < dwb_count; i++) {
@@ -1908,7 +1908,8 @@ int dcn32_populate_dml_pipes_from_context(
 	display_e2e_pipe_params_st *pipes,
 	enum dc_validate_mode validate_mode)
 {
-	int i, pipe_cnt;
+	unsigned int i;
+	int pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe = NULL;
 	bool subvp_in_use = false;
@@ -2308,7 +2309,7 @@ static bool dcn32_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -2382,7 +2383,7 @@ static bool dcn32_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2552,7 +2553,7 @@ static bool dcn32_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2625,7 +2626,7 @@ struct resource_pool *dcn32_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn32_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn32_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
@@ -2753,7 +2754,7 @@ static struct pipe_ctx *find_idle_secondary_pipe_check_mpo(
 		if ((res_ctx->pipe_ctx[preferred_pipe_idx].stream == NULL) &&
 			!(next_odm_mpo_pipe && next_odm_mpo_pipe->pipe_idx == preferred_pipe_idx)) {
 			secondary_pipe = &res_ctx->pipe_ctx[preferred_pipe_idx];
-			secondary_pipe->pipe_idx = preferred_pipe_idx;
+			secondary_pipe->pipe_idx = (uint8_t)preferred_pipe_idx;
 		}
 	}
 
@@ -2766,7 +2767,7 @@ static struct pipe_ctx *find_idle_secondary_pipe_check_mpo(
 			if ((res_ctx->pipe_ctx[i].stream == NULL) &&
 				!(next_odm_mpo_pipe && next_odm_mpo_pipe->pipe_idx == i)) {
 				secondary_pipe = &res_ctx->pipe_ctx[i];
-				secondary_pipe->pipe_idx = i;
+				secondary_pipe->pipe_idx = (uint8_t)i;
 				break;
 			}
 		}
@@ -2800,7 +2801,7 @@ static struct pipe_ctx *dcn32_acquire_idle_pipe_for_head_pipe_in_layer(
 	pipe = &old_ctx->pipe_ctx[head_index];
 	if (pipe->bottom_pipe && res_ctx->pipe_ctx[pipe->bottom_pipe->pipe_idx].stream == NULL) {
 		idle_pipe = &res_ctx->pipe_ctx[pipe->bottom_pipe->pipe_idx];
-		idle_pipe->pipe_idx = pipe->bottom_pipe->pipe_idx;
+		idle_pipe->pipe_idx = (uint8_t)pipe->bottom_pipe->pipe_idx;
 	} else {
 		idle_pipe = find_idle_secondary_pipe_check_mpo(res_ctx, pool, head_pipe);
 		if (!idle_pipe)
@@ -2814,7 +2815,7 @@ static struct pipe_ctx *dcn32_acquire_idle_pipe_for_head_pipe_in_layer(
 	idle_pipe->plane_res.hubp = pool->hubps[idle_pipe->pipe_idx];
 	idle_pipe->plane_res.ipp = pool->ipps[idle_pipe->pipe_idx];
 	idle_pipe->plane_res.dpp = pool->dpps[idle_pipe->pipe_idx];
-	idle_pipe->plane_res.mpcc_inst = pool->dpps[idle_pipe->pipe_idx]->inst;
+	idle_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[idle_pipe->pipe_idx]->inst;
 
 	return idle_pipe;
 }
@@ -2865,7 +2866,7 @@ struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_dpp_pipe(
 					pool, opp_head_pipe);
 	if (free_pipe_idx >= 0) {
 		free_pipe = &new_ctx->res_ctx.pipe_ctx[free_pipe_idx];
-		free_pipe->pipe_idx = free_pipe_idx;
+		free_pipe->pipe_idx = (uint8_t)free_pipe_idx;
 		free_pipe->stream = opp_head_pipe->stream;
 		free_pipe->stream_res.tg = opp_head_pipe->stream_res.tg;
 		free_pipe->stream_res.opp = opp_head_pipe->stream_res.opp;
@@ -2874,7 +2875,7 @@ struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_dpp_pipe(
 		free_pipe->plane_res.ipp = pool->ipps[free_pipe->pipe_idx];
 		free_pipe->plane_res.dpp = pool->dpps[free_pipe->pipe_idx];
 		free_pipe->plane_res.mpcc_inst =
-				pool->dpps[free_pipe->pipe_idx]->inst;
+				(uint8_t)pool->dpps[free_pipe->pipe_idx]->inst;
 	} else {
 		ASSERT(opp_head_pipe);
 		free_pipe = NULL;
@@ -2896,7 +2897,7 @@ struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_opp_head(
 
 	if (free_pipe_idx >= 0) {
 		free_pipe = &new_ctx->res_ctx.pipe_ctx[free_pipe_idx];
-		free_pipe->pipe_idx = free_pipe_idx;
+		free_pipe->pipe_idx = (uint8_t)free_pipe_idx;
 		free_pipe->stream = otg_master->stream;
 		free_pipe->stream_res.tg = otg_master->stream_res.tg;
 		free_pipe->stream_res.dsc = NULL;
@@ -2906,7 +2907,7 @@ struct pipe_ctx *dcn32_acquire_free_pipe_as_secondary_opp_head(
 		free_pipe->plane_res.ipp = pool->ipps[free_pipe_idx];
 		free_pipe->plane_res.xfm = pool->transforms[free_pipe_idx];
 		free_pipe->plane_res.dpp = pool->dpps[free_pipe_idx];
-		free_pipe->plane_res.mpcc_inst = pool->dpps[free_pipe_idx]->inst;
+		free_pipe->plane_res.mpcc_inst = (uint8_t)pool->dpps[free_pipe_idx]->inst;
 		free_pipe->dsc_padding_params = otg_master->dsc_padding_params;
 		if (free_pipe->stream->timing.flags.DSC == 1) {
 			dcn20_acquire_dsc(free_pipe->stream->ctx->dc,
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource_helpers.c b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource_helpers.c
index 4808c793590f7..602a0e4e5dc06 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource_helpers.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn32/dcn32_resource_helpers.c
@@ -333,7 +333,7 @@ void dcn32_determine_det_override(struct dc *dc,
 				continue;
 
 			if (context->stream_status[i].plane_count > 0)
-				plane_segments = stream_segments / context->stream_status[i].plane_count;
+				plane_segments = (uint8_t)(stream_segments / context->stream_status[i].plane_count);
 			else
 				plane_segments = stream_segments;
 			for (j = 0; j < dc->res_pool->pipe_count; j++) {
@@ -382,7 +382,8 @@ void dcn32_determine_det_override(struct dc *dc,
 void dcn32_set_det_allocations(struct dc *dc, struct dc_state *context,
 	display_e2e_pipe_params_st *pipes)
 {
-	int i, pipe_cnt;
+	unsigned int i;
+	int pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe = 0;
 	bool disable_unbounded_requesting = dc->debug.disable_z9_mpc || dc->debug.disable_unbounded_requesting;
@@ -751,7 +752,8 @@ bool dcn32_subvp_vblank_admissable(struct dc *dc, struct dc_state *context, int
 void dcn32_update_dml_pipes_odm_policy_based_on_context(struct dc *dc, struct dc_state *context,
 		display_e2e_pipe_params_st *pipes)
 {
-	int i, pipe_cnt;
+	unsigned int i;
+	int pipe_cnt;
 	struct resource_context *res_ctx = &context->res_ctx;
 	struct pipe_ctx *pipe = NULL;
 
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
index c8c81870d50ed..ede21363542a5 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn321/dcn321_resource.c
@@ -1400,7 +1400,7 @@ static void dcn321_resource_destruct(struct dcn321_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1429,7 +1429,7 @@ static void dcn321_resource_destruct(struct dcn321_resource_pool *pool)
 			dal_irq_service_destroy(&pool->base.irqs);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1442,19 +1442,19 @@ static void dcn321_resource_destruct(struct dcn321_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1477,7 +1477,7 @@ static void dcn321_resource_destruct(struct dcn321_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1493,7 +1493,7 @@ static void dcn321_resource_destruct(struct dcn321_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1519,7 +1519,7 @@ static void dcn321_resource_destruct(struct dcn321_resource_pool *pool)
 
 static bool dcn321_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t dwb_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < dwb_count; i++) {
@@ -1547,7 +1547,7 @@ static bool dcn321_dwbc_create(struct dc_context *ctx, struct resource_pool *poo
 
 static bool dcn321_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t dwb_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < dwb_count; i++) {
@@ -1801,7 +1801,7 @@ static bool dcn321_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -1875,7 +1875,7 @@ static bool dcn321_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2040,7 +2040,7 @@ static bool dcn321_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2109,7 +2109,7 @@ struct resource_pool *dcn321_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn321_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn321_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
index 2a1b0b152501c..dcbfe18b484ec 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn35/dcn35_resource.c
@@ -1153,7 +1153,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1483,7 +1483,7 @@ static void dcn35_resource_destruct(struct dcn35_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1513,7 +1513,7 @@ static void dcn35_resource_destruct(struct dcn35_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1526,19 +1526,19 @@ static void dcn35_resource_destruct(struct dcn35_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1561,7 +1561,7 @@ static void dcn35_resource_destruct(struct dcn35_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1577,7 +1577,7 @@ static void dcn35_resource_destruct(struct dcn35_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1634,7 +1634,7 @@ static void dcn35_dwbc_init(struct dcn30_dwbc *dwbc30, struct dc_context *ctx)
 
 static bool dcn35_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1672,7 +1672,7 @@ static void dcn35_mmhubbub_init(struct dcn30_mmhubbub *mcif_wb30,
 
 static bool dcn35_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1948,7 +1948,7 @@ static bool dcn35_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut; //2
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -2037,7 +2037,7 @@ static bool dcn35_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2089,7 +2089,7 @@ static bool dcn35_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
+	for (i = 0; i < (int)pool->base.pipe_count; i++) {
 		pool->base.hubps[i] = dcn35_hubp_create(ctx, i);
 		if (pool->base.hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2221,7 +2221,7 @@ static bool dcn35_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2266,7 +2266,7 @@ struct resource_pool *dcn35_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn35_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn35_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
index bfd1016aa7803..7a6e8e6f0ea61 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn351/dcn351_resource.c
@@ -1133,7 +1133,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1463,7 +1463,7 @@ static void dcn351_resource_destruct(struct dcn351_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1493,7 +1493,7 @@ static void dcn351_resource_destruct(struct dcn351_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1506,19 +1506,19 @@ static void dcn351_resource_destruct(struct dcn351_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1541,7 +1541,7 @@ static void dcn351_resource_destruct(struct dcn351_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1557,7 +1557,7 @@ static void dcn351_resource_destruct(struct dcn351_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1614,7 +1614,7 @@ static void dcn35_dwbc_init(struct dcn30_dwbc *dwbc30, struct dc_context *ctx)
 
 static bool dcn35_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1652,7 +1652,7 @@ static void dcn35_mmhubbub_init(struct dcn30_mmhubbub *mcif_wb30,
 
 static bool dcn35_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1921,7 +1921,7 @@ static bool dcn351_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut; //2
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -2009,7 +2009,7 @@ static bool dcn351_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2061,7 +2061,7 @@ static bool dcn351_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
+	for (i = 0; i < (int)pool->base.pipe_count; i++) {
 		pool->base.hubps[i] = dcn35_hubp_create(ctx, i);
 		if (pool->base.hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2193,7 +2193,7 @@ static bool dcn351_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2239,7 +2239,7 @@ struct resource_pool *dcn351_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn351_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn351_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
index 07a6675a73294..83f2fd6607897 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn36/dcn36_resource.c
@@ -1140,7 +1140,7 @@ static struct link_encoder *dcn31_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if (((unsigned int)eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc_obj(struct dcn20_link_encoder);
@@ -1470,7 +1470,7 @@ static void dcn36_resource_destruct(struct dcn36_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn20_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1500,7 +1500,7 @@ static void dcn36_resource_destruct(struct dcn36_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1513,19 +1513,19 @@ static void dcn36_resource_destruct(struct dcn36_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1548,7 +1548,7 @@ static void dcn36_resource_destruct(struct dcn36_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1564,7 +1564,7 @@ static void dcn36_resource_destruct(struct dcn36_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1621,7 +1621,7 @@ static void dcn35_dwbc_init(struct dcn30_dwbc *dwbc30, struct dc_context *ctx)
 
 static bool dcn35_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1659,7 +1659,7 @@ static void dcn35_mmhubbub_init(struct dcn30_mmhubbub *mcif_wb30,
 
 static bool dcn35_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1918,7 +1918,7 @@ static bool dcn36_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //2
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut; //2
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -2007,7 +2007,7 @@ static bool dcn36_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2059,7 +2059,7 @@ static bool dcn36_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs and TGs */
-	for (i = 0; i < pool->base.pipe_count; i++) {
+	for (i = 0; i < (int)pool->base.pipe_count; i++) {
 		pool->base.hubps[i] = dcn35_hubp_create(ctx, i);
 		if (pool->base.hubps[i] == NULL) {
 			BREAK_TO_DEBUGGER();
@@ -2191,7 +2191,7 @@ static bool dcn36_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2236,7 +2236,7 @@ struct resource_pool *dcn36_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn36_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn36_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c
index cb93bfbe9e9e8..e24d908bcf689 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn401/dcn401_resource.c
@@ -1425,7 +1425,7 @@ static void dcn401_resource_destruct(struct dcn401_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn401_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1455,7 +1455,7 @@ static void dcn401_resource_destruct(struct dcn401_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1468,19 +1468,19 @@ static void dcn401_resource_destruct(struct dcn401_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL)	{
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1503,7 +1503,7 @@ static void dcn401_resource_destruct(struct dcn401_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1519,7 +1519,7 @@ static void dcn401_resource_destruct(struct dcn401_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1545,7 +1545,7 @@ static void dcn401_resource_destruct(struct dcn401_resource_pool *pool)
 
 static bool dcn401_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t dwb_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < dwb_count; i++) {
@@ -1575,7 +1575,7 @@ static bool dcn401_dwbc_create(struct dc_context *ctx, struct resource_pool *poo
 
 static bool dcn401_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t dwb_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < dwb_count; i++) {
@@ -1792,11 +1792,11 @@ static void dcn401_build_pipe_pix_clk_params(struct pipe_ctx *pipe_ctx)
 		pixel_clk_params->dio_se_pix_per_cycle = 2;
 	} else if (dc_is_dp_signal(stream->signal)) {
 		/* round up to nearest power of 2, or max at 8 pixels per cycle */
-		if (pixel_clk_params->requested_pix_clk_100hz > 4 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10) {
+		if ((unsigned int)pixel_clk_params->requested_pix_clk_100hz > (unsigned int)(4 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10)) {
 			pixel_clk_params->dio_se_pix_per_cycle = 8;
-		} else if (pixel_clk_params->requested_pix_clk_100hz > 2 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10) {
+		} else if ((unsigned int)pixel_clk_params->requested_pix_clk_100hz > (unsigned int)(2 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10)) {
 			pixel_clk_params->dio_se_pix_per_cycle = 4;
-		} else if (pixel_clk_params->requested_pix_clk_100hz > stream->ctx->dc->clk_mgr->dprefclk_khz * 10) {
+		} else if ((unsigned int)pixel_clk_params->requested_pix_clk_100hz > (unsigned int)(stream->ctx->dc->clk_mgr->dprefclk_khz * 10)) {
 			pixel_clk_params->dio_se_pix_per_cycle = 2;
 		} else {
 			pixel_clk_params->dio_se_pix_per_cycle = 1;
@@ -1806,10 +1806,10 @@ static void dcn401_build_pipe_pix_clk_params(struct pipe_ctx *pipe_ctx)
 
 int dcn401_get_power_profile(const struct dc_state *context)
 {
-	int uclk_mhz = context->bw_ctx.bw.dcn.clk.dramclk_khz / 1000;
+	unsigned int uclk_mhz = context->bw_ctx.bw.dcn.clk.dramclk_khz / 1000;
 	int dpm_level = 0;
 
-	for (int i = 0; i < context->clk_mgr->bw_params->clk_table.num_entries_per_clk.num_memclk_levels; i++) {
+	for (unsigned int i = 0; i < context->clk_mgr->bw_params->clk_table.num_entries_per_clk.num_memclk_levels; i++) {
 		if (context->clk_mgr->bw_params->clk_table.entries[i].memclk_mhz == 0 ||
 			uclk_mhz < context->clk_mgr->bw_params->clk_table.entries[i].memclk_mhz)
 			break;
@@ -2002,7 +2002,7 @@ static bool dcn401_resource_construct(
 	dc->caps.color.dpp.ocsc = 0;
 
 	dc->caps.color.mpc.gamut_remap = 1;
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut; //4, configurable to be before or after BLND in MPCC
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
 	dc->caps.color.mpc.ogam_rom_caps.bt2020 = 0;
@@ -2096,7 +2096,7 @@ static bool dcn401_resource_construct(
 				CLOCK_SOURCE_ID_DP_DTO,
 				&clk_src_regs[0], true);
 
-	for (i = 0; i < pool->base.clk_src_count; i++) {
+	for (i = 0; i < (int)pool->base.clk_src_count; i++) {
 		if (pool->base.clock_sources[i] == NULL) {
 			dm_error("DC: failed to create clock sources!\n");
 			BREAK_TO_DEBUGGER();
@@ -2258,7 +2258,7 @@ static bool dcn401_resource_construct(
 
 	dc->caps.max_planes =  pool->base.pipe_count;
 
-	for (i = 0; i < dc->caps.max_planes; ++i)
+	for (i = 0; i < (int)dc->caps.max_planes; ++i)
 		dc->caps.planes[i] = plane_cap;
 
 	dc->caps.max_odm_combine_factor = 4;
@@ -2336,7 +2336,7 @@ struct resource_pool *dcn401_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn401_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn401_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
index c0d37f00fed91..1fc457cec2938 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.c
@@ -666,7 +666,6 @@ static const struct resource_caps res_cap_dcn42 = {
 	.num_vmid = 16,
 	.num_mpc_3dlut = 2,
 	.num_dsc = 4,
-	.num_rmcm = 2,
 };
 
 static const struct dc_plane_cap plane_cap = {
@@ -694,9 +693,13 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.force_abm_enable = false,
 	.clock_trace = true,
 	.disable_pplib_clock_request = false,
+	.ignore_pg = false,
 	.disable_dpp_power_gate = true,
 	.disable_hubp_power_gate = true,
 	.disable_optc_power_gate = true,
+	.disable_dsc_power_gate = false,
+	.disable_dio_power_gate = true,
+	.disable_hpo_power_gate = true,
 	.pipe_split_policy = MPC_SPLIT_AVOID,
 	.force_single_disp_pipe_split = false,
 	.disable_dcc = DCC_ENABLE,
@@ -758,11 +761,11 @@ static const struct dc_debug_options debug_defaults_drv = {
 	.min_disp_clk_khz = 50000,
 	.static_screen_wait_frames = 2,
 	.disable_z10 = false,
-	.ignore_pg = true,
 	.disable_stutter_for_wm_program = true,
 	.min_deep_sleep_dcfclk_khz = 8000,
 	.replay_skip_crtc_disabled = true,
 	.psr_skip_crtc_disable = true,
+	.force_odm2to1_for_edp_pixclk_mhz = 550, // Force ODM 2to1 for eDP when pixel clock is above 550MHz
 };
 
 static const struct dc_check_config config_defaults = {
@@ -1414,7 +1417,7 @@ static void dcn42_resource_destruct(struct dcn42_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		if (pool->base.dscs[i] != NULL)
 			dcn42_dsc_destroy(&pool->base.dscs[i]);
 	}
@@ -1443,7 +1446,7 @@ static void dcn42_resource_destruct(struct dcn42_resource_pool *pool)
 			dal_irq_service_destroy(&pool->base.irqs);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		if (pool->base.engines[i] != NULL)
 			dce110_engine_destroy(&pool->base.engines[i]);
 		if (pool->base.hw_i2cs[i] != NULL) {
@@ -1456,19 +1459,19 @@ static void dcn42_resource_destruct(struct dcn42_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_opp; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_opp; i++) {
 		if (pool->base.opps[i] != NULL)
 			pool->base.opps[i]->funcs->opp_destroy(&pool->base.opps[i]);
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.timing_generators[i] != NULL) {
 			kfree(DCN10TG_FROM_TG(pool->base.timing_generators[i]));
 			pool->base.timing_generators[i] = NULL;
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_dwb; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dwb; i++) {
 		if (pool->base.dwbc[i] != NULL) {
 			kfree(TO_DCN30_DWBC(pool->base.dwbc[i]));
 			pool->base.dwbc[i] = NULL;
@@ -1491,7 +1494,7 @@ static void dcn42_resource_destruct(struct dcn42_resource_pool *pool)
 		}
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_mpc_3dlut; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_mpc_3dlut; i++) {
 		if (pool->base.mpc_lut[i] != NULL) {
 			dc_3dlut_func_release(pool->base.mpc_lut[i]);
 			pool->base.mpc_lut[i] = NULL;
@@ -1507,7 +1510,7 @@ static void dcn42_resource_destruct(struct dcn42_resource_pool *pool)
 		pool->base.dp_clock_source = NULL;
 	}
 
-	for (i = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		if (pool->base.multiple_abms[i] != NULL)
 			dce_abm_destroy(&pool->base.multiple_abms[i]);
 	}
@@ -1517,6 +1520,10 @@ static void dcn42_resource_destruct(struct dcn42_resource_pool *pool)
 
 	if (pool->base.pg_cntl != NULL)
 		dcn_pg_cntl_destroy(&pool->base.pg_cntl);
+
+	if (pool->base.replay != NULL)
+		dmub_replay_destroy(&pool->base.replay);
+
 	if (pool->base.dccg != NULL)
 		dcn_dccg_destroy(&pool->base.dccg);
 
@@ -1578,11 +1585,11 @@ static void dcn42_build_pipe_pix_clk_params(struct pipe_ctx *pipe_ctx)
 		pixel_clk_params->dio_se_pix_per_cycle = 2;
 	} else if (dc_is_dp_signal(stream->signal)) {
 		/* round up to nearest power of 2, or max at 8 pixels per cycle */
-		if (pixel_clk_params->requested_pix_clk_100hz > 4 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10) {
+		if (pixel_clk_params->requested_pix_clk_100hz > (uint32_t)(4 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10)) {
 			pixel_clk_params->dio_se_pix_per_cycle = 8;
-		} else if (pixel_clk_params->requested_pix_clk_100hz > 2 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10) {
+		} else if (pixel_clk_params->requested_pix_clk_100hz > (uint32_t)(2 * stream->ctx->dc->clk_mgr->dprefclk_khz * 10)) {
 			pixel_clk_params->dio_se_pix_per_cycle = 4;
-		} else if (pixel_clk_params->requested_pix_clk_100hz > stream->ctx->dc->clk_mgr->dprefclk_khz * 10) {
+		} else if (pixel_clk_params->requested_pix_clk_100hz > (uint32_t)(stream->ctx->dc->clk_mgr->dprefclk_khz * 10)) {
 			pixel_clk_params->dio_se_pix_per_cycle = 2;
 		} else {
 			pixel_clk_params->dio_se_pix_per_cycle = 1;
@@ -1592,7 +1599,7 @@ static void dcn42_build_pipe_pix_clk_params(struct pipe_ctx *pipe_ctx)
 
 static bool dcn42_dwbc_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t dwb_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < dwb_count; i++) {
@@ -1629,7 +1636,7 @@ static void dcn42_mmhubbub_init(struct dcn30_mmhubbub *mcif_wb30,
 
 static bool dcn42_mmhubbub_create(struct dc_context *ctx, struct resource_pool *pool)
 {
-	int i;
+	unsigned int i;
 	uint32_t pipe_count = pool->res_cap->num_dwb;
 
 	for (i = 0; i < pipe_count; i++) {
@@ -1719,9 +1726,12 @@ enum dc_status dcn42_validate_bandwidth(struct dc *dc,
 
 	DC_FP_START();
 
+	dcn42_decide_odm_override(dc, context);
+
 	out = dml2_validate(dc, context, context->bw_ctx.dml2,
 						validate_mode);
 
+
 	if (validate_mode == DC_VALIDATE_MODE_AND_PROGRAMMING) {
 		/*not required for mode enumeration*/
 		dcn42_decide_zstate_support(dc, context);
@@ -1751,7 +1761,7 @@ static struct link_encoder *dcn42_link_enc_create_minimal(
 {
 	struct dcn20_link_encoder *enc20;
 
-	if ((eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
+	if ((unsigned int)(eng_id - ENGINE_ID_DIGA) > ctx->dc->res_pool->res_cap->num_dig_link_enc)
 		return NULL;
 
 	enc20 = kzalloc(sizeof(struct dcn20_link_encoder), GFP_KERNEL);
@@ -1805,8 +1815,6 @@ static struct resource_funcs dcn42_res_pool_funcs = {
 	.get_panel_config_defaults = dcn42_get_panel_config_defaults,
 	.get_preferred_eng_id_dpia = dcn42_get_preferred_eng_id_dpia,
 	.update_soc_for_wm_a = dcn30_update_soc_for_wm_a,
-	.add_phantom_pipes = dcn32_add_phantom_pipes,
-	.calculate_mall_ways_from_bytes = dcn32_calculate_mall_ways_from_bytes,
 	.prepare_mcache_programming = dcn42_prepare_mcache_programming,
 	.build_pipe_pix_clk_params = dcn42_build_pipe_pix_clk_params,
 	.get_power_profile = dcn401_get_power_profile,
@@ -1831,7 +1839,7 @@ static bool dcn42_resource_construct(
 	struct dc *dc,
 	struct dcn42_resource_pool *pool)
 {
-	int i, j;
+	unsigned int i, j;
 	struct dc_context *ctx = dc->ctx;
 	struct irq_service_init_data init_data;
 	uint32_t pipe_fuses;
@@ -1867,7 +1875,7 @@ static bool dcn42_resource_construct(
 	num_pipes = pool->base.res_cap->num_dpp;
 	pipe_fuses = read_pipe_fuses(ctx);
 
-	for (i = 0; i < pool->base.res_cap->num_dpp; i++)
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dpp; i++)
 		if (pipe_fuses & 1 << i)
 			num_pipes--;
 
@@ -1896,27 +1904,7 @@ static bool dcn42_resource_construct(
 	dc->caps.cursor_not_scaled = true;
 	dc->caps.min_horizontal_blanking_period = 80;
 	dc->caps.dmdata_alloc_size = 2048;
-	dc->caps.mall_size_per_mem_channel = 4;
-	/* total size = mall per channel * num channels * 1024 * 1024 */
-	dc->caps.mall_size_total = dc->caps.mall_size_per_mem_channel *
-		dc->ctx->dc_bios->vram_info.num_chans * 1048576;
 	dc->caps.cursor_cache_size = dc->caps.max_cursor_size * dc->caps.max_cursor_size * 8;
-	dc->caps.cache_line_size = 64;
-	dc->caps.cache_num_ways = 16;
-
-	/* Calculate the available MALL space */
-	dc->caps.max_cab_allocation_bytes =
-		dcn32_calc_num_avail_chans_for_mall(dc, dc->ctx->dc_bios->vram_info.num_chans) *
-				dc->caps.mall_size_per_mem_channel * 1024 * 1024;
-	dc->caps.mall_size_total = dc->caps.max_cab_allocation_bytes;
-
-	dc->caps.subvp_fw_processing_delay_us = 15;
-	dc->caps.subvp_drr_max_vblank_margin_us = 40;
-	dc->caps.subvp_prefetch_end_to_mall_start_us = 15;
-	dc->caps.subvp_swath_height_margin_lines = 16;
-	dc->caps.subvp_pstate_allow_width_us = 20;
-	dc->caps.subvp_vertical_int_margin_us = 30;
-	dc->caps.subvp_drr_vblank_start_margin_us = 100; // 100us margin
 
 	dc->caps.max_slave_planes = 2;
 	dc->caps.max_slave_yuv_planes = 2;
@@ -1937,12 +1925,6 @@ static bool dcn42_resource_construct(
 	dc->caps.max_v_total = (1 << 15) - 1;
 	dc->caps.vtotal_limited_by_fp2 = true;
 
-	dc->caps.seamless_odm = true;
-	dc->caps.zstate_support = true;
-	dc->caps.ips_support = true;
-	dc->caps.max_v_total = (1 << 15) - 1;
-	dc->caps.vtotal_limited_by_fp2 = true;
-
 	/* Color pipeline capabilities */
 	dc->caps.color.dpp.dcn_arch = 1;
 	dc->caps.color.dpp.input_lut_shared = 0;
@@ -1969,7 +1951,7 @@ static bool dcn42_resource_construct(
 
 	dc->caps.color.mpc.gamut_remap = 1;
 	//configurable to be before or after BLND in MPCC
-	dc->caps.color.mpc.num_3dluts = pool->base.res_cap->num_mpc_3dlut;
+	dc->caps.color.mpc.num_3dluts = (uint16_t)pool->base.res_cap->num_mpc_3dlut;
 	dc->caps.color.mpc.num_rmcm_3dluts = 2;
 	dc->caps.color.mpc.ogam_ram = 1;
 	dc->caps.color.mpc.ogam_rom_caps.srgb = 0;
@@ -2141,7 +2123,7 @@ static bool dcn42_resource_construct(
 	}
 
 	/* HUBPs, DPPs, OPPs, TGs, ABMs */
-	for (i = 0, j = 0; i < pool->base.res_cap->num_timing_generator; i++) {
+	for (i = 0, j = 0; i < (unsigned int)pool->base.res_cap->num_timing_generator; i++) {
 		/* if pipe is disabled, skip instance of HW pipe,
 		 * i.e, skip ASIC register instance
 		 */
@@ -2185,7 +2167,7 @@ static bool dcn42_resource_construct(
 													  &abm_shift,
 													  &abm_mask);
 		if (pool->base.multiple_abms[j] == NULL) {
-			dm_error("DC: failed to create abm for pipe %d!\n", i);
+			dm_error("DC: failed to create abm for pipe %u!\n", i);
 			BREAK_TO_DEBUGGER();
 			goto create_fail;
 		}
@@ -2220,11 +2202,11 @@ static bool dcn42_resource_construct(
 	}
 
 	/* DSCs */
-	for (i = 0; i < pool->base.res_cap->num_dsc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_dsc; i++) {
 		pool->base.dscs[i] = dcn42_dsc_create(ctx, i);
 		if (pool->base.dscs[i] == NULL) {
 			BREAK_TO_DEBUGGER();
-			dm_error("DC: failed to create display stream compressor %d!\n", i);
+			dm_error("DC: failed to create display stream compressor %u!\n", i);
 			goto create_fail;
 		}
 	}
@@ -2244,7 +2226,7 @@ static bool dcn42_resource_construct(
 	}
 
 	/* AUX and I2C */
-	for (i = 0; i < pool->base.res_cap->num_ddc; i++) {
+	for (i = 0; i < (unsigned int)pool->base.res_cap->num_ddc; i++) {
 		pool->base.engines[i] = dcn42_aux_engine_create(ctx, i);
 
 		if (pool->base.engines[i] == NULL) {
@@ -2302,26 +2284,6 @@ static bool dcn42_resource_construct(
 	resource_init_common_dml2_callbacks(dc, &dc->dml2_options);
 	dc->dml2_options.callbacks.can_support_mclk_switch_using_fw_based_vblank_stretch =
 			&dcn30_can_support_mclk_switch_using_fw_based_vblank_stretch;
-	dc->dml2_options.svp_pstate.callbacks.release_dsc = &dcn20_release_dsc;
-	dc->dml2_options.svp_pstate.callbacks.calculate_mall_ways_from_bytes =
-		pool->base.funcs->calculate_mall_ways_from_bytes;
-
-	dc->dml2_options.svp_pstate.subvp_fw_processing_delay_us = dc->caps.subvp_fw_processing_delay_us;
-	dc->dml2_options.svp_pstate.subvp_prefetch_end_to_mall_start_us = dc->caps.subvp_prefetch_end_to_mall_start_us;
-	dc->dml2_options.svp_pstate.subvp_pstate_allow_width_us = dc->caps.subvp_pstate_allow_width_us;
-	dc->dml2_options.svp_pstate.subvp_swath_height_margin_lines = dc->caps.subvp_swath_height_margin_lines;
-
-	dc->dml2_options.svp_pstate.force_disable_subvp = dc->debug.force_disable_subvp;
-	dc->dml2_options.svp_pstate.force_enable_subvp = dc->debug.force_subvp_mclk_switch;
-
-	dc->dml2_options.mall_cfg.cache_line_size_bytes = dc->caps.cache_line_size;
-	dc->dml2_options.mall_cfg.cache_num_ways = dc->caps.cache_num_ways;
-	dc->dml2_options.mall_cfg.max_cab_allocation_bytes =
-				dc->caps.max_cab_allocation_bytes;
-	dc->dml2_options.mall_cfg.mblk_height_4bpe_pixels = DCN3_2_MBLK_HEIGHT_4BPE;
-	dc->dml2_options.mall_cfg.mblk_height_8bpe_pixels = DCN3_2_MBLK_HEIGHT_8BPE;
-	dc->dml2_options.mall_cfg.mblk_size_bytes = DCN3_2_MALL_MBLK_SIZE_BYTES;
-	dc->dml2_options.mall_cfg.mblk_width_pixels = DCN3_2_MBLK_WIDTH;
 
 	dc->dml2_options.max_segments_per_hubp = 24;
 	dc->dml2_options.det_segment_size = DCN42_CRB_SEGMENT_SIZE_KB;
@@ -2349,7 +2311,7 @@ struct resource_pool *dcn42_create_resource_pool(
 	if (!pool)
 		return NULL;
 
-	if (dcn42_resource_construct(init_data->num_virtual_links, dc, pool))
+	if (dcn42_resource_construct((uint8_t)init_data->num_virtual_links, dc, pool))
 		return &pool->base;
 
 	BREAK_TO_DEBUGGER();
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h
index 60acf0e423d97..fe960542d7bdb 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource.h
@@ -456,6 +456,8 @@
 		SRI_ARR(OTG_V_SYNC_A, OTG, inst), \
 		SRI_ARR(OTG_V_SYNC_A_CNTL, OTG, inst), \
 		SRI_ARR(OTG_CONTROL, OTG, inst), \
+		SRI_ARR(OTG_MASTER_EN, OTG, inst), \
+		SRI_ARR(OTG_LONG_VBLANK_STATUS, OTG, inst), \
 		SRI_ARR(OTG_STEREO_CONTROL, OTG, inst), \
 		SRI_ARR(OTG_3D_STRUCTURE_CONTROL, OTG, inst),                            \
 		SRI_ARR(OTG_STEREO_STATUS, OTG, inst),                                   \
@@ -472,6 +474,11 @@
 		SRI_ARR(OTG_M_CONST_DTO0, OTG, inst),                                    \
 		SRI_ARR(OTG_M_CONST_DTO1, OTG, inst),                                    \
 		SRI_ARR(OTG_CLOCK_CONTROL, OTG, inst),                                   \
+		SRI_ARR(OTG_COUNT_CONTROL, OTG, inst),                                   \
+		SRI_ARR(OTG_COUNT_RESET, OTG, inst),                                   \
+		SRI_ARR(OTG_CRC_SIG_BLUE_CONTROL_MASK, OTG, inst),                      \
+		SRI_ARR(OTG_CRC_SIG_RED_GREEN_MASK, OTG, inst),                      \
+		SRI_ARR(OTG_DRR_TIMING_INT_STATUS, OTG, inst),                      \
 		SRI_ARR(OTG_VERTICAL_INTERRUPT0_CONTROL, OTG, inst),                     \
 		SRI_ARR(OTG_VERTICAL_INTERRUPT0_POSITION, OTG, inst),                    \
 		SRI_ARR(OTG_VERTICAL_INTERRUPT1_CONTROL, OTG, inst),                     \
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.c b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.c
index 33b9775420d3d..ee330559c2337 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.c
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.c
@@ -45,3 +45,25 @@ void dcn42_decide_zstate_support(struct dc *dc, struct dc_state *context)
 	context->bw_ctx.bw.dcn.clk.zstate_support = support;
 
 }
+
+bool dcn42_decide_odm_override(struct dc *dc, struct dc_state *context)
+{
+	bool odm_override = false;
+
+	DC_LOGGER_INIT(dc->ctx->logger);
+	if (dc->ctx->dce_environment == DCE_ENV_DIAG)
+		return false;
+
+	if (context->stream_count == 1 && context->streams[0]->signal == SIGNAL_TYPE_EDP) {
+
+		if (dc->debug.force_odm2to1_for_edp_pixclk_mhz != 0 &&
+			context->streams[0]->timing.pix_clk_100hz > dc->debug.force_odm2to1_for_edp_pixclk_mhz * 10000) {
+			odm_override = true;
+			context->streams[0]->debug.force_odm_combine_segments = 2;
+		}
+		DC_LOG_SMU("odm_override: %d, eDP pixelclock: %d, force_odm2to1_for_edp_pixclk_mhz: %d\n",
+			odm_override, context->streams[0]->timing.pix_clk_100hz / 10000, dc->debug.force_odm2to1_for_edp_pixclk_mhz);
+	}
+	return odm_override;
+}
+
diff --git a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.h b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.h
index e321032205073..aff7be777681d 100644
--- a/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.h
+++ b/drivers/gpu/drm/amd/display/dc/resource/dcn42/dcn42_resource_fpu.h
@@ -29,5 +29,5 @@
 #include "core_types.h"
 
 void dcn42_decide_zstate_support(struct dc *dc, struct dc_state *context);
-
+bool dcn42_decide_odm_override(struct dc *dc, struct dc_state *context);
 #endif /* _DCN42_RESOURCE_FPU_H_ */
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
index 1b397fa7e05c9..89f7ccd7f81f0 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn401/dcn401_soc_and_ip_translator.c
@@ -42,17 +42,17 @@ static void dcn401_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* dcfclk */
 	if (dc_clk_table->num_entries_per_clk.num_dcfclk_levels) {
-		dml_clk_table->dcfclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dcfclk_levels;
+		dml_clk_table->dcfclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_dcfclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->dcfclk.num_clk_values) {
 				if (use_clock_dc_limits && dc_bw_params->dc_mode_limit.dcfclk_mhz &&
 						dc_clk_table->entries[i].dcfclk_mhz > dc_bw_params->dc_mode_limit.dcfclk_mhz) {
 					if (i == 0 || dc_clk_table->entries[i-1].dcfclk_mhz < dc_bw_params->dc_mode_limit.dcfclk_mhz) {
 						dml_clk_table->dcfclk.clk_values_khz[i] = dc_bw_params->dc_mode_limit.dcfclk_mhz * 1000;
-						dml_clk_table->dcfclk.num_clk_values = i + 1;
+						dml_clk_table->dcfclk.num_clk_values = (uint8_t)(i + 1);
 					} else {
 						dml_clk_table->dcfclk.clk_values_khz[i] = 0;
-						dml_clk_table->dcfclk.num_clk_values = i;
+						dml_clk_table->dcfclk.num_clk_values = (uint8_t)i;
 					}
 				} else {
 					dml_clk_table->dcfclk.clk_values_khz[i] = dc_clk_table->entries[i].dcfclk_mhz * 1000;
@@ -65,17 +65,17 @@ static void dcn401_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* fclk */
 	if (dc_clk_table->num_entries_per_clk.num_fclk_levels) {
-		dml_clk_table->fclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_fclk_levels;
+		dml_clk_table->fclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_fclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->fclk.num_clk_values) {
 				if (use_clock_dc_limits && dc_bw_params->dc_mode_limit.fclk_mhz &&
 						dc_clk_table->entries[i].fclk_mhz > dc_bw_params->dc_mode_limit.fclk_mhz) {
 					if (i == 0 || dc_clk_table->entries[i-1].fclk_mhz < dc_bw_params->dc_mode_limit.fclk_mhz) {
 						dml_clk_table->fclk.clk_values_khz[i] = dc_bw_params->dc_mode_limit.fclk_mhz * 1000;
-						dml_clk_table->fclk.num_clk_values = i + 1;
+						dml_clk_table->fclk.num_clk_values = (uint8_t)(i + 1);
 					} else {
 						dml_clk_table->fclk.clk_values_khz[i] = 0;
-						dml_clk_table->fclk.num_clk_values = i;
+						dml_clk_table->fclk.num_clk_values = (uint8_t)i;
 					}
 				} else {
 					dml_clk_table->fclk.clk_values_khz[i] = dc_clk_table->entries[i].fclk_mhz * 1000;
@@ -88,17 +88,17 @@ static void dcn401_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* uclk */
 	if (dc_clk_table->num_entries_per_clk.num_memclk_levels) {
-		dml_clk_table->uclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_memclk_levels;
+		dml_clk_table->uclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_memclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->uclk.num_clk_values) {
 				if (use_clock_dc_limits && dc_bw_params->dc_mode_limit.memclk_mhz &&
 						dc_clk_table->entries[i].memclk_mhz > dc_bw_params->dc_mode_limit.memclk_mhz) {
 					if (i == 0 || dc_clk_table->entries[i-1].memclk_mhz < dc_bw_params->dc_mode_limit.memclk_mhz) {
 						dml_clk_table->uclk.clk_values_khz[i] = dc_bw_params->dc_mode_limit.memclk_mhz * 1000;
-						dml_clk_table->uclk.num_clk_values = i + 1;
+						dml_clk_table->uclk.num_clk_values = (uint8_t)(i + 1);
 					} else {
 						dml_clk_table->uclk.clk_values_khz[i] = 0;
-						dml_clk_table->uclk.num_clk_values = i;
+						dml_clk_table->uclk.num_clk_values = (uint8_t)i;
 					}
 				} else {
 					dml_clk_table->uclk.clk_values_khz[i] = dc_clk_table->entries[i].memclk_mhz * 1000;
@@ -114,17 +114,17 @@ static void dcn401_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* dispclk */
 	if (dc_clk_table->num_entries_per_clk.num_dispclk_levels) {
-		dml_clk_table->dispclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dispclk_levels;
+		dml_clk_table->dispclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_dispclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->dispclk.num_clk_values) {
 				if (use_clock_dc_limits && dc_bw_params->dc_mode_limit.dispclk_mhz &&
 						dc_clk_table->entries[i].dispclk_mhz > dc_bw_params->dc_mode_limit.dispclk_mhz) {
 					if (i == 0 || dc_clk_table->entries[i-1].dispclk_mhz < dc_bw_params->dc_mode_limit.dispclk_mhz) {
 						dml_clk_table->dispclk.clk_values_khz[i] = dc_bw_params->dc_mode_limit.dispclk_mhz * 1000;
-						dml_clk_table->dispclk.num_clk_values = i + 1;
+						dml_clk_table->dispclk.num_clk_values = (uint8_t)(i + 1);
 					} else {
 						dml_clk_table->dispclk.clk_values_khz[i] = 0;
-						dml_clk_table->dispclk.num_clk_values = i;
+						dml_clk_table->dispclk.num_clk_values = (uint8_t)i;
 					}
 				} else {
 					dml_clk_table->dispclk.clk_values_khz[i] = dc_clk_table->entries[i].dispclk_mhz * 1000;
@@ -137,17 +137,17 @@ static void dcn401_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* dppclk */
 	if (dc_clk_table->num_entries_per_clk.num_dppclk_levels) {
-		dml_clk_table->dppclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dppclk_levels;
+		dml_clk_table->dppclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_dppclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->dppclk.num_clk_values) {
 				if (use_clock_dc_limits && dc_bw_params->dc_mode_limit.dppclk_mhz &&
 						dc_clk_table->entries[i].dppclk_mhz > dc_bw_params->dc_mode_limit.dppclk_mhz) {
 					if (i == 0 || dc_clk_table->entries[i-1].dppclk_mhz < dc_bw_params->dc_mode_limit.dppclk_mhz) {
 						dml_clk_table->dppclk.clk_values_khz[i] = dc_bw_params->dc_mode_limit.dppclk_mhz * 1000;
-						dml_clk_table->dppclk.num_clk_values = i + 1;
+						dml_clk_table->dppclk.num_clk_values = (uint8_t)(i + 1);
 					} else {
 						dml_clk_table->dppclk.clk_values_khz[i] = 0;
-						dml_clk_table->dppclk.num_clk_values = i;
+						dml_clk_table->dppclk.num_clk_values = (uint8_t)i;
 					}
 				} else {
 					dml_clk_table->dppclk.clk_values_khz[i] = dc_clk_table->entries[i].dppclk_mhz * 1000;
@@ -160,17 +160,17 @@ static void dcn401_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* dtbclk */
 	if (dc_clk_table->num_entries_per_clk.num_dtbclk_levels) {
-		dml_clk_table->dtbclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dtbclk_levels;
+		dml_clk_table->dtbclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_dtbclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->dtbclk.num_clk_values) {
 				if (use_clock_dc_limits && dc_bw_params->dc_mode_limit.dtbclk_mhz &&
 						dc_clk_table->entries[i].dtbclk_mhz > dc_bw_params->dc_mode_limit.dtbclk_mhz) {
 					if (i == 0 || dc_clk_table->entries[i-1].dtbclk_mhz < dc_bw_params->dc_mode_limit.dtbclk_mhz) {
 						dml_clk_table->dtbclk.clk_values_khz[i] = dc_bw_params->dc_mode_limit.dtbclk_mhz * 1000;
-						dml_clk_table->dtbclk.num_clk_values = i + 1;
+						dml_clk_table->dtbclk.num_clk_values = (uint8_t)(i + 1);
 					} else {
 						dml_clk_table->dtbclk.clk_values_khz[i] = 0;
-						dml_clk_table->dtbclk.num_clk_values = i;
+						dml_clk_table->dtbclk.num_clk_values = (uint8_t)i;
 					}
 				} else {
 					dml_clk_table->dtbclk.clk_values_khz[i] = dc_clk_table->entries[i].dtbclk_mhz * 1000;
@@ -183,17 +183,17 @@ static void dcn401_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* socclk */
 	if (dc_clk_table->num_entries_per_clk.num_socclk_levels) {
-		dml_clk_table->socclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_socclk_levels;
+		dml_clk_table->socclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_socclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->socclk.num_clk_values) {
 				if (use_clock_dc_limits && dc_bw_params->dc_mode_limit.socclk_mhz &&
 						dc_clk_table->entries[i].socclk_mhz > dc_bw_params->dc_mode_limit.socclk_mhz) {
 					if (i == 0 || dc_clk_table->entries[i-1].socclk_mhz < dc_bw_params->dc_mode_limit.socclk_mhz) {
 						dml_clk_table->socclk.clk_values_khz[i] = dc_bw_params->dc_mode_limit.socclk_mhz * 1000;
-						dml_clk_table->socclk.num_clk_values = i + 1;
+						dml_clk_table->socclk.num_clk_values = (uint8_t)(i + 1);
 					} else {
 						dml_clk_table->socclk.clk_values_khz[i] = 0;
-						dml_clk_table->socclk.num_clk_values = i;
+						dml_clk_table->socclk.num_clk_values = (uint8_t)i;
 					}
 				} else {
 					dml_clk_table->socclk.clk_values_khz[i] = dc_clk_table->entries[i].socclk_mhz * 1000;
diff --git a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c
index e723b4d0aff3f..e0802359829e0 100644
--- a/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c
+++ b/drivers/gpu/drm/amd/display/dc/soc_and_ip_translator/dcn42/dcn42_soc_and_ip_translator.c
@@ -47,18 +47,18 @@ static void dcn42_convert_dc_clock_table_to_soc_bb_clock_table(
 	 * for use with dml we need to fill in using an active value aiming for >= 2x DCFCLK
 	 */
 	if (dc_clk_table->num_entries_per_clk.num_fclk_levels && dc_clk_table->num_entries_per_clk.num_dcfclk_levels) {
-		dml_clk_table->fclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dcfclk_levels;
-		dml_clk_table->dcfclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dcfclk_levels;
+		dml_clk_table->fclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_dcfclk_levels;
+		dml_clk_table->dcfclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_dcfclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
-			if (i < dc_clk_table->num_entries_per_clk.num_dcfclk_levels) {
+			if (i < (int)dc_clk_table->num_entries_per_clk.num_dcfclk_levels) {
 				int j, max_fclk = 0;
 
 				dml_clk_table->dcfclk.clk_values_khz[i] = dc_clk_table->entries[i].dcfclk_mhz * 1000;
 				for (j = 0; j < MAX_NUM_DPM_LVL; j++) {
-					if (dc_clk_table->entries[j].fclk_mhz * 1000 > max_fclk)
+					if ((uint32_t)(dc_clk_table->entries[j].fclk_mhz * 1000) > (uint32_t)max_fclk)
 						max_fclk = dc_clk_table->entries[j].fclk_mhz * 1000;
 					dml_clk_table->fclk.clk_values_khz[i] = max_fclk;
-					if (max_fclk >= 2 * dml_clk_table->dcfclk.clk_values_khz[i])
+						if ((uint32_t)max_fclk >= 2 * dml_clk_table->dcfclk.clk_values_khz[i])
 						break;
 				}
 			} else {
@@ -70,7 +70,7 @@ static void dcn42_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* uclk */
 	if (dc_clk_table->num_entries_per_clk.num_memclk_levels) {
-		dml_clk_table->uclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_memclk_levels;
+		dml_clk_table->uclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_memclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->uclk.num_clk_values) {
 				dml_clk_table->uclk.clk_values_khz[i] = dc_clk_table->entries[i].memclk_mhz * 1000;
@@ -84,7 +84,7 @@ static void dcn42_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* dispclk */
 	if (dc_clk_table->num_entries_per_clk.num_dispclk_levels) {
-		dml_clk_table->dispclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dispclk_levels;
+		dml_clk_table->dispclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_dispclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->dispclk.num_clk_values) {
 				dml_clk_table->dispclk.clk_values_khz[i] = dc_clk_table->entries[i].dispclk_mhz * 1000;
@@ -101,7 +101,7 @@ static void dcn42_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* dppclk */
 	if (dc_clk_table->num_entries_per_clk.num_dppclk_levels) {
-		dml_clk_table->dppclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dppclk_levels;
+		dml_clk_table->dppclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_dppclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->dppclk.num_clk_values) {
 				dml_clk_table->dppclk.clk_values_khz[i] = dc_clk_table->entries[i].dppclk_mhz * 1000;
@@ -117,7 +117,7 @@ static void dcn42_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* dtbclk */
 	if (dc_clk_table->num_entries_per_clk.num_dtbclk_levels) {
-		dml_clk_table->dtbclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_dtbclk_levels;
+		dml_clk_table->dtbclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_dtbclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->dtbclk.num_clk_values) {
 				dml_clk_table->dtbclk.clk_values_khz[i] = dc_clk_table->entries[i].dtbclk_mhz * 1000;
@@ -129,7 +129,7 @@ static void dcn42_convert_dc_clock_table_to_soc_bb_clock_table(
 
 	/* socclk */
 	if (dc_clk_table->num_entries_per_clk.num_socclk_levels) {
-		dml_clk_table->socclk.num_clk_values = dc_clk_table->num_entries_per_clk.num_socclk_levels;
+		dml_clk_table->socclk.num_clk_values = (uint8_t)dc_clk_table->num_entries_per_clk.num_socclk_levels;
 		for (i = 0; i < min(DML_MAX_CLK_TABLE_SIZE, MAX_NUM_DPM_LVL); i++) {
 			if (i < dml_clk_table->socclk.num_clk_values) {
 				dml_clk_table->socclk.clk_values_khz[i] = dc_clk_table->entries[i].socclk_mhz * 1000;
diff --git a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
index fe9431cea3e52..2f68438000ea2 100644
--- a/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
+++ b/drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h
@@ -329,7 +329,7 @@ union dmub_addr {
 
 /* Flattened structure containing SOC BB parameters stored in the VBIOS
  * It is not practical to store the entire bounding box in VBIOS since the bounding box struct can gain new parameters.
- * This also prevents alighment issues when new parameters are added to the SoC BB.
+ * This also prevents alignment issues when new parameters are added to the SoC BB.
  * The following parameters should be added since these values can't be obtained elsewhere:
  * -dml2_soc_power_management_parameters
  * -dml2_soc_vmin_clock_limits
@@ -958,7 +958,7 @@ union dmub_fw_meta {
 //==============================================================================
 #if !defined(TENSILICA) && !defined(DMUB_TRACE_ENTRY_DEFINED)
 /**
- * dmub_trace_code_t - firmware trace code, 32-bits
+ * typedef dmub_trace_code_t - firmware trace code, 32-bits
  */
 typedef uint32_t dmub_trace_code_t;
 
@@ -1170,7 +1170,7 @@ enum dmub_shared_state_feature_id {
 };
 
 /**
- * struct dmub_shared_state_ips_fw - Firmware signals for IPS.
+ * struct dmub_shared_state_ips_fw_signals - Firmware signals for IPS.
  */
 union dmub_shared_state_ips_fw_signals {
 	struct {
@@ -1185,7 +1185,7 @@ union dmub_shared_state_ips_fw_signals {
 };
 
 /**
- * struct dmub_shared_state_ips_signals - Firmware signals for IPS.
+ * struct dmub_shared_state_ips_driver_signals - Firmware signals for IPS.
  */
 union dmub_shared_state_ips_driver_signals {
 	struct {
@@ -1247,7 +1247,7 @@ struct dmub_shared_state_ips_driver {
 }; /* 248-bytes, fixed */
 
 /**
- * struct dmub_shared_state_cursor_offload_v1 - Header metadata for cursor offload.
+ * struct dmub_shared_state_cursor_offload_stream_v1 - Header metadata for cursor offload.
  */
 struct dmub_shared_state_cursor_offload_stream_v1 {
 	uint32_t last_write_idx; /**< Last write index */
@@ -1263,14 +1263,14 @@ struct dmub_shared_state_cursor_offload_v1 {
 }; /* 248-bytes, fixed */
 
 /**
- * enum dmub_shared_state_feature_common - Generic payload.
+ * struct dmub_shared_state_feature_common - Generic payload.
  */
 struct dmub_shared_state_feature_common {
 	uint32_t padding[62];
 }; /* 248-bytes, fixed */
 
 /**
- * enum dmub_shared_state_feature_header - Feature description.
+ * struct dmub_shared_state_feature_header - Feature description.
  */
 struct dmub_shared_state_feature_header {
 	uint16_t id; /**< Feature ID */
@@ -1546,8 +1546,9 @@ enum dmub_gpint_command {
 	 * DESC: Gets IPS residency in microseconds
 	 * ARGS: 0 - Return IPS1 residency
 	 *       1 - Return IPS2 residency
-	 *       2 - Return IPS1_RCG residency
+	 *       2 - Return IPS0_RCG residency
 	 *       3 - Return IPS1_ONO2_ON residency
+	 *       4 - Return IPS1_Z8_RETENTION residency
 	 * RETURN: Total residency in microseconds - lower 32 bits
 	 */
 	DMUB_GPINT__GET_IPS_RESIDENCY_DURATION_US_LO = 124,
@@ -1567,17 +1568,18 @@ enum dmub_gpint_command {
 	 * DESC: Gets IPS residency
 	 * ARGS: 0 - Return IPS1 residency
 	 *       1 - Return IPS2 residency
-	 *       2 - Return IPS1_RCG residency
+	 *       2 - Return IPS0_RCG residency
 	 *       3 - Return IPS1_ONO2_ON residency
+	 *       4 - Return IPS1_Z8_RETENTION residency
 	 * RETURN: Total residency in milli-percent.
 	 */
 	DMUB_GPINT__GET_IPS_RESIDENCY_PERCENT = 127,
 	/**
-	 * DESC: Gets IPS1_RCG histogram counts
+	 * DESC: Gets IPS0_RCG histogram counts
 	 * ARGS: Bucket index
 	 * RETURN: Total count for the bucket
 	 */
-	DMUB_GPINT__GET_IPS1_RCG_HISTOGRAM_COUNTER = 128,
+	DMUB_GPINT__GET_IPS0_RCG_HISTOGRAM_COUNTER = 128,
 	/**
 	 * DESC: Gets IPS1_ONO2_ON histogram counts
 	 * ARGS: Bucket index
@@ -1588,8 +1590,9 @@ enum dmub_gpint_command {
 	 * DESC: Gets IPS entry counter during residency measurement
 	 * ARGS: 0 - Return IPS1 entry counts
 	 *       1 - Return IPS2 entry counts
-	 *       2 - Return IPS1_RCG entry counts
-	 *       3 - Return IPS2_ONO2_ON entry counts
+	 *       2 - Return IPS0_RCG entry counts
+	 *       3 - Return IPS1_ONO2_ON entry counts
+	 *       4 - Return IPS1_Z8_RETENTION entry counts
 	 * RETURN: Entry counter for selected IPS mode
 	 */
 	DMUB_GPINT__GET_IPS_RESIDENCY_ENTRY_COUNTER = 130,
@@ -1597,8 +1600,9 @@ enum dmub_gpint_command {
 	 * DESC: Gets IPS inactive residency in microseconds
 	 * ARGS: 0 - Return IPS1_MAX residency
 	 *       1 - Return IPS2 residency
-	 *       2 - Return IPS1_RCG residency
+	 *       2 - Return IPS0_RCG residency
 	 *       3 - Return IPS1_ONO2_ON residency
+	 *       4 - Return IPS1_Z8_RETENTION residency
 	 * RETURN: Total inactive residency in microseconds - lower 32 bits
 	 */
 	DMUB_GPINT__GET_IPS_INACTIVE_RESIDENCY_DURATION_US_LO = 131,
@@ -1606,8 +1610,9 @@ enum dmub_gpint_command {
 	 * DESC: Gets IPS inactive residency in microseconds
 	 * ARGS: 0 - Return IPS1_MAX residency
 	 *       1 - Return IPS2 residency
-	 *       2 - Return IPS1_RCG residency
+	 *       2 - Return IPS0_RCG residency
 	 *       3 - Return IPS1_ONO2_ON residency
+	 *       4 - Return IPS1_Z8_RETENTION residency
 	 * RETURN: Total inactive residency in microseconds - upper 32 bits
 	 */
 	DMUB_GPINT__GET_IPS_INACTIVE_RESIDENCY_DURATION_US_HI = 132,
@@ -1615,8 +1620,9 @@ enum dmub_gpint_command {
 	 * DESC: Gets IPS residency in microseconds
 	 * ARGS: 0 - Return IPS1 residency
 	 *       1 - Return IPS2 residency
-	 *       2 - Return IPS1_RCG residency
+	 *       2 - Return IPS0_RCG residency
 	 *       3 - Return IPS1_ONO2_ON residency
+	 *       4 - Return IPS1_Z8_RETENTION residency
 	 * RETURN: Total residency in microseconds - upper 32 bits
 	 */
 	DMUB_GPINT__GET_IPS_RESIDENCY_DURATION_US_HI = 133,
@@ -1990,7 +1996,7 @@ struct dmub_cmd_read_modify_write_sequence {
 #define DMUB_READ_MODIFY_WRITE_SEQ__MAX 5
 
 /**
- * struct dmub_cmd_read_modify_write_sequence - Read modify write command.
+ * struct dmub_rb_cmd_read_modify_write - Read modify write command.
  */
 struct dmub_rb_cmd_read_modify_write {
 	struct dmub_cmd_header header;  /**< command header */
@@ -2001,7 +2007,7 @@ struct dmub_rb_cmd_read_modify_write {
 };
 
 /*
- * Update a register with specified masks and values sequeunce
+ * Update a register with specified masks and values sequence
  *
  * 60 payload bytes can hold address + up to 7 sets of mask/value combo, each take 2 dword
  *
@@ -2182,7 +2188,7 @@ enum dmub_cmd_cab_type {
 };
 
 /**
- * struct dmub_rb_cmd_cab - CAB command data.
+ * struct dmub_rb_cmd_cab_for_ss - CAB command data.
  */
 struct dmub_rb_cmd_cab_for_ss {
 	struct dmub_cmd_header header;
@@ -2344,24 +2350,24 @@ struct dmub_cmd_lsdma_data {
 			uint32_t dst_mip_max      : 5;
 			uint32_t dst_swizzle_mode : 5;
 			uint32_t dst_mip_id       : 5;
-			uint32_t tmz              : 1;
 			uint32_t dcc              : 1;
+			uint32_t padding1         : 1;
 
 			uint32_t data_format      : 6;
-			uint32_t padding1         : 4;
+			uint32_t tmz              : 4;
 			uint32_t dst_element_size : 3;
 			uint32_t num_type         : 3;
 			uint32_t src_element_size : 3;
 			uint32_t write_compress   : 2;
-			uint32_t cache_policy_dst : 2;
-			uint32_t cache_policy_src : 2;
+			uint32_t cache_policy_dst : 3;
+			uint32_t cache_policy_src : 3;
 			uint32_t read_compress    : 2;
-			uint32_t src_dim          : 2;
-			uint32_t dst_dim          : 2;
+			uint32_t max_com          : 2;
 			uint32_t max_uncom        : 1;
 
-			uint32_t max_com          : 2;
-			uint32_t padding          : 30;
+			uint32_t dst_dim          : 2;
+			uint32_t src_dim          : 2;
+			uint32_t padding          : 28;
 		} tiled_copy_data;
 		struct lsdma_linear_copy_data {
 			uint32_t src_lo;
@@ -2371,11 +2377,13 @@ struct dmub_cmd_lsdma_data {
 			uint32_t dst_hi;
 
 			uint32_t count            : 30;
-			uint32_t cache_policy_dst : 2;
+			uint32_t pad0             : 2;
 
-			uint32_t tmz              : 1;
-			uint32_t cache_policy_src : 2;
-			uint32_t padding          : 29;
+			uint32_t tmz              : 4;
+			uint32_t cache_policy_src : 3;
+			uint32_t cache_policy_dst : 3;
+			uint32_t pad1             : 22;
+			// DCC fields not included because linear mode on display does not support DCC
 		} linear_copy_data;
 		struct lsdma_linear_sub_window_copy_data {
 			uint32_t src_lo;
@@ -2399,11 +2407,13 @@ struct dmub_cmd_lsdma_data {
 			uint32_t src_slice_pitch;
 			uint32_t dst_slice_pitch;
 
-			uint32_t tmz              : 1;
+			uint32_t tmz              : 4;
 			uint32_t element_size     : 3;
 			uint32_t src_cache_policy : 3;
 			uint32_t dst_cache_policy : 3;
-			uint32_t reserved0        : 22;
+			uint32_t reserved0        : 19;
+			// Linear mode on display does not support compression so DCC related fields are not included.
+			// The DCC fields in the command packet will be zero'd at the time of constructing the packet.
 		} linear_sub_window_copy_data;
 		struct lsdma_reg_write_data {
 			uint32_t reg_addr;
@@ -2867,7 +2877,7 @@ struct dmub_cmd_set_pixel_clock_data {
 };
 
 /**
- * struct dmub_cmd_set_pixel_clock_data - Set pixel clock command.
+ * struct dmub_rb_cmd_set_pixel_clock - Set pixel clock command.
  */
 struct dmub_rb_cmd_set_pixel_clock {
 	struct dmub_cmd_header header; /**< header */
@@ -3383,7 +3393,7 @@ struct dpia_bw_allocation_notify_data {
 };
 
 /**
- * union dpia_notify_data_type - DPIA Notification in Outbox command
+ * union dpia_notification_data - DPIA Notification in Outbox command
  */
 union dpia_notification_data {
 	/**
@@ -3439,7 +3449,7 @@ struct dmub_rb_cmd_query_hpd_state {
 };
 
 /**
- * struct dmub_rb_cmd_hpd_sense_notify - HPD sense notification data.
+ * struct dmub_rb_cmd_hpd_sense_notify_data - HPD sense notification data.
  */
 struct dmub_rb_cmd_hpd_sense_notify_data {
 	uint32_t old_hpd_sense_mask; /**< Old HPD sense mask */
@@ -3483,7 +3493,7 @@ enum dmub_cmd_psr_type {
 
 	/**
 	 * Set PSR level.
-	 * PSR level is a 16-bit value dicated by driver that
+	 * PSR level is a 16-bit value dictated by driver that
 	 * will enable/disable different functionality.
 	 */
 	DMUB_CMD__PSR_SET_LEVEL			= 4,
@@ -3640,7 +3650,7 @@ struct dmub_cmd_psr_copy_settings_data {
 	 */
 	union dmub_psr_debug_flags debug;
 	/**
-	 * 16-bit value dicated by driver that will enable/disable different functionality.
+	 * 16-bit value dictated by driver that will enable/disable different functionality.
 	 */
 	uint16_t psr_level;
 	/**
@@ -3680,7 +3690,7 @@ struct dmub_cmd_psr_copy_settings_data {
 	 */
 	uint8_t aux_inst;
 	/**
-	 * Determines if SMU optimzations are enabled/disabled.
+	 * Determines if SMU optimizations are enabled/disabled.
 	 */
 	uint8_t smu_optimizations_en;
 	/**
@@ -3797,7 +3807,7 @@ struct dmub_rb_cmd_psr_copy_settings {
  */
 struct dmub_cmd_psr_set_level_data {
 	/**
-	 * 16-bit value dicated by driver that will enable/disable different functionality.
+	 * 16-bit value dictated by driver that will enable/disable different functionality.
 	 */
 	uint16_t psr_level;
 	/**
@@ -4175,7 +4185,7 @@ struct dmub_rb_cmd_update_cursor_info {
  */
 struct dmub_cmd_psr_set_vtotal_data {
 	/**
-	 * 16-bit value dicated by driver that indicates the vtotal in PSR active requirement when screen idle..
+	 * 16-bit value dictated by driver that indicates the vtotal in PSR active requirement when screen idle..
 	 */
 	uint16_t psr_vtotal_idle;
 	/**
@@ -4189,7 +4199,7 @@ struct dmub_cmd_psr_set_vtotal_data {
 	 */
 	uint8_t panel_inst;
 	/*
-	 * 16-bit value dicated by driver that indicates the vtotal in PSR active requirement when doing SU/FFU.
+	 * 16-bit value dictated by driver that indicates the vtotal in PSR active requirement when doing SU/FFU.
 	 */
 	uint16_t psr_vtotal_su;
 	/**
@@ -4489,7 +4499,7 @@ struct dmub_cmd_replay_copy_settings_data {
 	uint8_t aux_inst;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -4512,7 +4522,7 @@ struct dmub_cmd_replay_copy_settings_data {
 	 */
 	uint8_t dpphy_inst;
 	/**
-	 * Determines if SMU optimzations are enabled/disabled.
+	 * Determines if SMU optimizations are enabled/disabled.
 	 */
 	uint8_t smu_optimizations_en;
 	/**
@@ -4602,7 +4612,7 @@ struct dmub_rb_cmd_smart_power_oled_enable_data {
 	uint8_t enable;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -4641,7 +4651,7 @@ struct dmub_rb_cmd_replay_enable_data {
 	uint8_t enable;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -4689,7 +4699,7 @@ struct dmub_rb_cmd_replay_enable {
 struct dmub_cmd_replay_set_power_opt_data {
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -4709,7 +4719,7 @@ struct dmub_cmd_replay_set_power_opt_data {
 struct dmub_cmd_replay_set_timing_sync_data {
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -4729,7 +4739,7 @@ struct dmub_cmd_replay_set_timing_sync_data {
 struct dmub_cmd_replay_set_pseudo_vtotal {
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -4745,7 +4755,7 @@ struct dmub_cmd_replay_set_pseudo_vtotal {
 struct dmub_cmd_replay_disabled_adaptive_sync_sdp_data {
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -4759,7 +4769,7 @@ struct dmub_cmd_replay_disabled_adaptive_sync_sdp_data {
 struct dmub_cmd_replay_set_general_cmd_data {
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -4796,7 +4806,7 @@ struct dmub_rb_cmd_replay_set_power_opt {
  */
 struct dmub_cmd_replay_set_coasting_vtotal_data {
 	/**
-	 * 16-bit value dicated by driver that indicates the coasting vtotal.
+	 * 16-bit value dictated by driver that indicates the coasting vtotal.
 	 */
 	uint16_t coasting_vtotal;
 	/**
@@ -4805,12 +4815,12 @@ struct dmub_cmd_replay_set_coasting_vtotal_data {
 	uint8_t cmd_version;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
 	/**
-	 * 16-bit value dicated by driver that indicates the coasting vtotal high byte part.
+	 * 16-bit value dictated by driver that indicates the coasting vtotal high byte part.
 	 */
 	uint16_t coasting_vtotal_high;
 	/**
@@ -4913,7 +4923,7 @@ struct dmub_rb_cmd_replay_set_general_cmd {
 struct dmub_cmd_replay_frameupdate_timer_data {
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -4946,7 +4956,7 @@ struct dmub_rb_cmd_replay_set_frameupdate_timer {
 union dmub_replay_cmd_set {
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -5122,7 +5132,7 @@ struct dmub_hw_lock_inst_flags {
 	uint8_t opp_inst;
 	/**
 	 * OTG HW instance for global update lock.
-	 * TODO: Remove, and re-use otg_inst.
+	 * TODO: Remove, and reuse otg_inst.
 	 */
 	uint8_t dig_inst;
 	/**
@@ -6318,7 +6328,7 @@ struct dmub_cmd_cable_id_input {
 };
 
 /**
- * struct dmub_cmd_cable_id_input - Defines the output of DMUB_CMD_GET_USBC_CABLE_ID command.
+ * struct dmub_cmd_cable_id_output - Defines the output of DMUB_CMD_GET_USBC_CABLE_ID command.
  */
 struct dmub_cmd_cable_id_output {
 	uint8_t UHBR10_20_CAPABILITY	:2; /**< b'01 for UHBR10 support, b'10 for both UHBR10 and UHBR20 support */
@@ -6513,7 +6523,7 @@ struct dmub_rb_cmd_assr_enable {
 enum ips_residency_mode {
 	IPS_RESIDENCY__IPS1_MAX,
 	IPS_RESIDENCY__IPS2,
-	IPS_RESIDENCY__IPS1_RCG,
+	IPS_RESIDENCY__IPS1_RCG, // refers to IPS0 + RCG
 	IPS_RESIDENCY__IPS1_ONO2_ON,
 	IPS_RESIDENCY__IPS1_Z8_RETENTION,
 	IPS_RESIDENCY__PG_ONO_LAST_SEEN_IN_IPS,
@@ -6609,7 +6619,7 @@ struct dmub_cmd_pr_enable_data {
 	uint8_t enable;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which replay_state to use
+	 * Panel instance to identify which replay_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -6687,7 +6697,7 @@ struct dmub_cmd_pr_copy_settings_data {
 	uint8_t aux_inst;
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -6696,7 +6706,7 @@ struct dmub_cmd_pr_copy_settings_data {
 	 */
 	uint8_t dpphy_inst;
 	/**
-	 * Determines if SMU optimzations are enabled/disabled.
+	 * Determines if SMU optimizations are enabled/disabled.
 	 */
 	uint8_t smu_optimizations_en;
 	/**
@@ -6704,7 +6714,7 @@ struct dmub_cmd_pr_copy_settings_data {
 	 */
 	uint32_t line_time_in_ns;
 	/*
-	 * Use FSFT afftet pixel clk
+	 * Use FSFT after pixel clk
 	 */
 	uint32_t pix_clk_100hz;
 	/*
@@ -6777,7 +6787,7 @@ union dmub_pr_runtime_flags {
 struct dmub_cmd_pr_update_state_data {
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
@@ -6799,7 +6809,7 @@ struct dmub_cmd_pr_update_state_data {
 struct dmub_cmd_pr_general_cmd_data {
 	/**
 	 * Panel Instance.
-	 * Panel isntance to identify which psr_state to use
+	 * Panel instance to identify which psr_state to use
 	 * Currently the support is only for 0 or 1
 	 */
 	uint8_t panel_inst;
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c
index e5a78df80d72f..3444eb7d7d812 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn401.c
@@ -535,7 +535,7 @@ void dmub_dcn401_send_reg_inbox0_cmd_msg(struct dmub_srv *dmub,
 		union dmub_rb_cmd *cmd)
 {
 	uint32_t *dwords = (uint32_t *)cmd;
-	int32_t payload_size_bytes = cmd->cmd_common.header.payload_bytes;
+	uint32_t payload_size_bytes = cmd->cmd_common.header.payload_bytes;
 	uint32_t msg_index;
 	static_assert(sizeof(*cmd) == 64, "DMUB command size mismatch");
 
diff --git a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
index a09aa19ad3796..e86fcfc749856 100644
--- a/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
+++ b/drivers/gpu/drm/amd/display/dmub/src/dmub_dcn42.c
@@ -429,7 +429,7 @@ void dmub_dcn42_send_reg_inbox0_cmd_msg(struct dmub_srv *dmub,
 		union dmub_rb_cmd *cmd)
 {
 	uint32_t *dwords = (uint32_t *)cmd;
-	int32_t payload_size_bytes = cmd->cmd_common.header.payload_bytes;
+	uint32_t payload_size_bytes = cmd->cmd_common.header.payload_bytes;
 	uint32_t msg_index;
 	static_assert(sizeof(*cmd) == 64, "DMUB command size mismatch");
 
diff --git a/drivers/gpu/drm/amd/display/include/fixed31_32.h b/drivers/gpu/drm/amd/display/include/fixed31_32.h
index 990fa1f19c229..1090933119848 100644
--- a/drivers/gpu/drm/amd/display/include/fixed31_32.h
+++ b/drivers/gpu/drm/amd/display/include/fixed31_32.h
@@ -445,7 +445,7 @@ static inline struct fixed31_32 dc_fixpt_pow(struct fixed31_32 arg1, struct fixe
  */
 static inline int dc_fixpt_floor(struct fixed31_32 arg)
 {
-	unsigned long long arg_value = arg.value > 0 ? arg.value : -arg.value;
+	unsigned long long arg_value = (unsigned long long)(arg.value > 0 ? arg.value : -arg.value);
 
 	if (arg.value >= 0)
 		return (int)(arg_value >> FIXED31_32_BITS_PER_FRACTIONAL_PART);
@@ -459,7 +459,7 @@ static inline int dc_fixpt_floor(struct fixed31_32 arg)
  */
 static inline int dc_fixpt_round(struct fixed31_32 arg)
 {
-	unsigned long long arg_value = arg.value > 0 ? arg.value : -arg.value;
+	unsigned long long arg_value = (unsigned long long)(arg.value > 0 ? arg.value : -arg.value);
 
 	const long long summand = dc_fixpt_half.value;
 
@@ -479,7 +479,7 @@ static inline int dc_fixpt_round(struct fixed31_32 arg)
  */
 static inline int dc_fixpt_ceil(struct fixed31_32 arg)
 {
-	unsigned long long arg_value = arg.value > 0 ? arg.value : -arg.value;
+	unsigned long long arg_value = (unsigned long long)(arg.value > 0 ? arg.value : -arg.value);
 
 	const long long summand = dc_fixpt_one.value -
 		dc_fixpt_epsilon.value;
diff --git a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
index 1f225d0d6c44a..03d88e78165d5 100644
--- a/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
+++ b/drivers/gpu/drm/amd/display/modules/color/color_gamma.c
@@ -364,7 +364,7 @@ void precompute_pq(void)
 /* one-time pre-compute dePQ values - only for max pixel value 125 FP16 */
 void precompute_de_pq(void)
 {
-	int i;
+	uint32_t i;
 	struct fixed31_32  y;
 	uint32_t begin_index, end_index;
 
@@ -690,7 +690,7 @@ static bool find_software_points(
 static bool build_custom_gamma_mapping_coefficients_worker(
 	const struct dc_gamma *ramp,
 	struct pixel_gamma_point *coeff,
-	const struct hw_x_point *coordinates_x,
+	const struct hw_x_point *hw_coordinates_x,
 	const struct gamma_pixel *axis_x,
 	enum channel_name channel,
 	uint32_t number_of_points)
@@ -712,11 +712,11 @@ static bool build_custom_gamma_mapping_coefficients_worker(
 		struct fixed31_32 right_pos;
 
 		if (channel == CHANNEL_NAME_RED)
-			coord_x = coordinates_x[i].regamma_y_red;
+			coord_x = hw_coordinates_x[i].regamma_y_red;
 		else if (channel == CHANNEL_NAME_GREEN)
-			coord_x = coordinates_x[i].regamma_y_green;
+			coord_x = hw_coordinates_x[i].regamma_y_green;
 		else
-			coord_x = coordinates_x[i].regamma_y_blue;
+			coord_x = hw_coordinates_x[i].regamma_y_blue;
 
 		if (!find_software_points(
 			ramp, axis_x, coord_x, channel,
@@ -783,7 +783,7 @@ static struct fixed31_32 calculate_mapped_value(
 	struct pwl_float_data *rgb,
 	const struct pixel_gamma_point *coeff,
 	enum channel_name channel,
-	uint32_t max_index)
+	int32_t max_index)
 {
 	const struct gamma_point *point;
 
@@ -1425,7 +1425,7 @@ static void apply_lut_1d(
 		uint32_t num_hw_points,
 		struct dc_transfer_func_distributed_points *tf_pts)
 {
-	int i = 0;
+	uint32_t i = 0;
 	int color = 0;
 	struct fixed31_32 *regamma_y;
 	struct fixed31_32 norm_y;
@@ -1539,11 +1539,11 @@ static void build_evenly_distributed_points(
 }
 
 static inline void copy_rgb_regamma_to_coordinates_x(
-		struct hw_x_point *coordinates_x,
+		struct hw_x_point *hw_coordinates_x,
 		uint32_t hw_points_num,
 		const struct pwl_float_data_ex *rgb_ex)
 {
-	struct hw_x_point *coords = coordinates_x;
+	struct hw_x_point *coords = hw_coordinates_x;
 	uint32_t i = 0;
 	const struct pwl_float_data_ex *rgb_regamma = rgb_ex;
 
@@ -1562,26 +1562,26 @@ static bool calculate_interpolated_hardware_curve(
 	const struct dc_gamma *ramp,
 	struct pixel_gamma_point *coeff128,
 	struct pwl_float_data *rgb_user,
-	const struct hw_x_point *coordinates_x,
+	const struct hw_x_point *hw_coordinates_x,
 	const struct gamma_pixel *axis_x,
 	uint32_t number_of_points,
 	struct dc_transfer_func_distributed_points *tf_pts)
 {
 
 	const struct pixel_gamma_point *coeff = coeff128;
-	uint32_t max_entries = 3 - 1;
+	int32_t max_entries = 3 - 1;
 
 	uint32_t i = 0;
 
 	for (i = 0; i < 3; i++) {
 		if (!build_custom_gamma_mapping_coefficients_worker(
-				ramp, coeff128, coordinates_x, axis_x, i,
+				ramp, coeff128, hw_coordinates_x, axis_x, i,
 				number_of_points))
 			return false;
 	}
 
 	i = 0;
-	max_entries += ramp->num_entries;
+	max_entries += (int32_t)ramp->num_entries;
 
 	/* TODO: float point case */
 
@@ -1635,7 +1635,7 @@ static bool map_regamma_hw_to_x_user(
 {
 	/* setup to spare calculated ideal regamma values */
 
-	int i = 0;
+	uint32_t i = 0;
 	struct hw_x_point *coords = coords_x;
 	const struct pwl_float_data_ex *regamma = rgb_regamma;
 
@@ -1786,14 +1786,14 @@ bool mod_color_calculate_degamma_params(struct dc_color_caps *dc_caps,
 	if (input_tf->tf == TRANSFER_FUNCTION_PQ) {
 		/* just copy current rgb_regamma into  tf_pts */
 		struct pwl_float_data_ex *curvePt = curve;
-		int i = 0;
+		int j = 0;
 
-		while (i <= MAX_HW_POINTS) {
-			tf_pts->red[i]   = curvePt->r;
-			tf_pts->green[i] = curvePt->g;
-			tf_pts->blue[i]  = curvePt->b;
+		while (j <= MAX_HW_POINTS) {
+			tf_pts->red[j]   = curvePt->r;
+			tf_pts->green[j] = curvePt->g;
+			tf_pts->blue[j]  = curvePt->b;
 			++curvePt;
-			++i;
+			++j;
 		}
 	} else {
 		// clamps to 0-1
diff --git a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c
index 5cb979c2cf8c4..1164fd96b714c 100644
--- a/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c
+++ b/drivers/gpu/drm/amd/display/modules/hdcp/hdcp_log.c
@@ -44,7 +44,7 @@ void mod_hdcp_dump_binary_message(uint8_t *msg, uint32_t msg_size,
 		for (i = 0; i < msg_size; i++) {
 			if (i % bytes_per_line == 0)
 				buf[buf_pos++] = '\n';
-			sprintf(&buf[buf_pos], "%02X ", msg[i]);
+			sprintf((char *)&buf[buf_pos], "%02X ", msg[i]);
 			buf_pos += byte_size;
 		}
 		buf[buf_pos++] = '\0';
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_color_types.h b/drivers/gpu/drm/amd/display/modules/inc/mod_color_types.h
new file mode 100644
index 0000000000000..bf7313df585b6
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_color_types.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef MOD_COLOR_TYPES_H_
+#define MOD_COLOR_TYPES_H_
+
+enum predefined_gamut_type {
+	gamut_type_bt709,
+	gamut_type_bt601,
+	gamut_type_adobe_rgb,
+	gamut_type_srgb,
+	gamut_type_bt2020,
+	gamut_type_dcip3,
+	gamut_type_unknown,
+};
+
+enum predefined_white_point_type {
+	white_point_type_5000k_horizon,
+	white_point_type_6500k_noon,
+	white_point_type_7500k_north_sky,
+	white_point_type_9300k,
+	white_point_type_unknown,
+};
+
+#endif /* MOD_COLOR_TYPES_H_ */
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
index ddd64b7e4c047..11b127eb13d84 100644
--- a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet.h
@@ -27,6 +27,7 @@
 #define MOD_INFO_PACKET_H_
 
 #include "dm_services.h"
+#include "mod_info_packet_types.h"
 #include "mod_shared.h"
 //Forward Declarations
 struct dc_stream_state;
@@ -47,14 +48,6 @@ void mod_build_vsc_infopacket(const struct dc_stream_state *stream,
 void mod_build_hf_vsif_infopacket(const struct dc_stream_state *stream,
 		struct dc_info_packet *info_packet);
 
-enum adaptive_sync_type {
-	ADAPTIVE_SYNC_TYPE_NONE                  = 0,
-	ADAPTIVE_SYNC_TYPE_DP                    = 1,
-	FREESYNC_TYPE_PCON_IN_WHITELIST          = 2,
-	FREESYNC_TYPE_PCON_NOT_IN_WHITELIST      = 3,
-	ADAPTIVE_SYNC_TYPE_EDP                   = 4,
-};
-
 enum adaptive_sync_sdp_version {
 	AS_SDP_VER_0 = 0x0,
 	AS_SDP_VER_1 = 0x1,
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet_types.h b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet_types.h
new file mode 100644
index 0000000000000..30a5259ef36b4
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_info_packet_types.h
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2018 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#ifndef MOD_INFO_PACKET_TYPES_H_
+#define MOD_INFO_PACKET_TYPES_H_
+
+enum adaptive_sync_type {
+	ADAPTIVE_SYNC_TYPE_NONE                  = 0,
+	ADAPTIVE_SYNC_TYPE_DP                    = 1,
+	FREESYNC_TYPE_PCON_IN_WHITELIST          = 2,
+	FREESYNC_TYPE_PCON_NOT_IN_WHITELIST      = 3,
+	ADAPTIVE_SYNC_TYPE_EDP                   = 4,
+};
+
+#endif /* MOD_INFO_PACKET_TYPES_H_ */
diff --git a/drivers/gpu/drm/amd/display/modules/inc/mod_power.h b/drivers/gpu/drm/amd/display/modules/inc/mod_power.h
new file mode 100644
index 0000000000000..f9814cf7bbdb1
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/inc/mod_power.h
@@ -0,0 +1,417 @@
+/* Copyright (c) 2019 Advanced Micro Devices, Inc. All rights reserved. */
+
+#ifndef MODULES_INC_MOD_POWER_H_
+#define MODULES_INC_MOD_POWER_H_
+
+#include "dm_services.h"
+
+struct core_power;
+
+struct mod_power_init_params {
+
+	bool disable_fractional_pwm;
+
+	/* Use nits based brightness instead of brightness percentage
+	 */
+	bool use_nits_based_brightness;
+	unsigned int panel_min_millinits;
+	unsigned int panel_max_millinits;
+
+	unsigned int min_backlight_pwm;
+	unsigned int max_backlight_pwm;
+
+	unsigned int min_abm_backlight;
+	unsigned int num_backlight_levels;
+	bool backlight_ramping_override;
+	unsigned int backlight_ramping_reduction;
+	unsigned int backlight_ramping_start;
+	bool def_varibright_enable;
+	unsigned int def_varibright_level;
+	unsigned int varibright_level;
+	unsigned int abm_config_setting;
+
+	bool allow_psr_smu_optimizations;
+
+	bool allow_psr_multi_disp_optimizations;
+
+	bool use_custom_backlight_caps;
+	unsigned int custom_backlight_caps_config_no;
+	bool use_linear_backlight_curve;
+};
+
+struct mod_power {
+	int dummy;
+};
+
+/* VariBright settings structure */
+struct varibright_info {
+	unsigned int level;
+	bool enable;
+	bool activate;
+};
+
+struct mod_power_psr_context {
+    /* ddc line */
+    unsigned int channel;
+    /* Transmitter id */
+    unsigned int transmitter_id;
+    /* Engine Id is used for Dig Be source select */
+    unsigned int engine_id;
+    /* Controller Id used for Dig Fe source select */
+    unsigned int controller_id;
+    /* Pcie or Uniphy */
+    unsigned int phy_type;
+    /* Physical PHY Id used by SMU interpretation */
+    unsigned int smu_phy_id;
+    /* Vertical total pixels from crtc timing.
+     * This is used for static screen detection.
+     * ie. If we want to detect half a frame,
+     * we use this to determine the hyst lines.
+     */
+    unsigned int crtc_timing_vertical_total;
+    /* PSR supported from panel capabilities and
+     * current display configuration
+     */
+    bool psr_supported_display_config;
+    /* Whether fast link training is supported by the panel */
+    bool psr_exit_link_training_required;
+    /* If RFB setup time is greater than the total VBLANK time,
+     * it is not possible for the sink to capture the video frame
+     * in the same frame the SDP is sent. In this case,
+     * the frame capture indication bit should be set and an extra
+     * static frame should be transmitted to the sink.
+     */
+    bool psr_frame_capture_indication_req;
+    /* Set the last possible line SDP may be transmitted without violating
+     * the RFB setup time or entering the active video frame.
+     */
+    unsigned int sdp_transmit_line_num_deadline;
+    /* The VSync rate in Hz used to calculate the
+     * step size for smooth brightness feature
+     */
+    unsigned int vsync_rate_hz;
+    unsigned int skip_psr_wait_for_pll_lock;
+    unsigned int number_of_controllers;
+    /* Unused, for future use. To indicate that first changed frame from
+     * state3 shouldn't result in psr_inactive, but rather to perform
+     * an automatic single frame rfb_update.
+     */
+    bool rfb_update_auto_en;
+    /* Number of frame before entering static screen */
+    unsigned int timehyst_frames;
+    /* Partial frames before entering static screen */
+    unsigned int hyst_lines;
+    /* # of repeated AUX transaction attempts to make before
+     * indicating failure to the driver
+     */
+    unsigned int aux_repeats;
+    /* Controls hw blocks to power down during PSR active state */
+    unsigned int psr_level;
+    /* Controls additional delay after remote frame capture before
+     * continuing powerd own
+     */
+	unsigned int frame_delay;
+	bool allow_smu_optimizations;
+	bool allow_multi_disp_optimizations;
+	unsigned int line_time_in_us;
+	/* Panel self refresh 2 selective update granularity required */
+	bool su_granularity_required;
+	/* psr2 selective update y granularity capability */
+	uint8_t su_y_granularity;
+	uint8_t rate_control_caps;
+	bool os_request_force_ffu;
+};
+
+enum psr_event {
+	psr_event_invalid = 0x0,
+	psr_event_vsync = 0x1,
+	psr_event_full_screen = 0x2,
+	psr_event_defer_enable = 0x4,
+	psr_event_hw_programming = 0x8,
+	psr_event_test_harness_enable_psr = 0x10,
+	psr_event_test_harness_disable_psr = 0x20,
+	psr_event_mpo_video_selective_update = 0x40,
+	psr_event_edp_panel_off_disable_psr = 0x80,
+	psr_event_dynamic_display_switch = 0x100,
+	psr_event_big_screen_video = 0x200,
+	psr_event_dds_defer_stream_enable = 0x800,
+	psr_event_dynamic_link_rate_control = 0x1000,
+	psr_event_vrr_transition = 0x2000,
+	psr_event_pause = 0x4000,
+	psr_event_immediate_flip = 0x8000,
+	psr_event_os_request_disable = 0x10000,
+	psr_event_os_request_force_ffu = 0x20000,
+	psr_event_os_override_hold = 0x40000,
+	psr_event_crc_window_active = 0x80000,
+};
+
+enum replay_event {
+	replay_event_invalid = 0x0,
+	replay_event_vsync = 0x1,
+	replay_event_full_screen = 0x2,
+	replay_event_mpo_video_selective_update = 0x4,
+	replay_event_big_screen_video = 0x8,
+	replay_event_hw_programming = 0x10,
+	replay_event_edp_panel_off_disable_psr = 0x20,
+	replay_event_general_ui = 0x40,
+	replay_event_vrr = 0x80,
+	replay_event_prepare_vtotal = 0x100,
+	replay_event_test_harness_enable_replay = 0x200,
+	replay_event_test_harness_disable_replay = 0x400,
+	replay_event_test_harness_ultra_sleep = 0x800,
+	replay_event_immediate_flip = 0x1000,
+	replay_event_vrr_transition = 0x2000,
+	replay_event_pause = 0x4000,
+	replay_event_disable_replay_while_DPMS = 0x8000,
+	replay_event_test_harness_mode = 0x10000,
+	replay_event_cursor_updating = 0x20000,
+	replay_event_sleep_resume = 0x40000,
+	replay_event_disable_in_AC = 0x80000,
+	replay_event_disable_replay_while_detect_display = 0x100000,
+	replay_event_disable_replay_while_switching_mux = 0x400000,
+	replay_event_infopacket = 0x800000,
+	replay_event_os_request_disable = 0x1000000,
+	replay_event_os_request_force_ffu = 0x2000000,
+	replay_event_os_override_hold = 0x4000000,
+	replay_event_crc_window_active = 0x8000000,
+};
+
+enum replay_enable_option {
+	pr_enable_option_static_screen = 0x1,
+	pr_enable_option_mpo_video = 0x2,
+	pr_enable_option_full_screen_video = 0x4,
+	pr_enable_option_general_ui = 0x8,
+	pr_enable_option_full_screen = 0x10,
+	pr_enable_option_static_screen_coasting = 0x10000,
+	pr_enable_option_mpo_video_coasting = 0x20000,
+	pr_enable_option_full_screen_video_coasting = 0x40000,
+	pr_enable_option_full_screen_coasting = 0x100000,
+};
+
+struct mod_power *mod_power_create(struct dc *dc,
+		struct mod_power_init_params *init_params,
+		unsigned int edp_num);
+
+void mod_power_destroy(struct mod_power *mod_power);
+
+bool mod_power_hw_init(struct mod_power *mod_power);
+
+bool mod_power_add_stream(struct mod_power *mod_power,
+		struct dc_stream_state *stream, struct psr_caps *caps);
+
+bool mod_power_remove_stream(struct mod_power *mod_power,
+		const struct dc_stream_state *stream);
+
+bool mod_power_replace_stream(struct mod_power *mod_power,
+		const struct dc_stream_state *current_stream,
+		struct dc_stream_state *new_stream,
+		struct psr_caps *new_caps);
+
+bool mod_power_set_backlight_nits(struct mod_power *mod_power,
+		struct dc_stream_state *streams,
+		unsigned int backlight_millinit,
+		unsigned int transition_time_millisec,
+		bool skip_aux,
+		bool is_hdr);
+
+bool mod_power_set_backlight_percent(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millipercent,
+		unsigned int transition_time_millisec,
+		bool is_hdr);
+
+void mod_power_update_backlight(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millipercent);
+
+void mod_power_update_backlight_nits(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millinit);
+
+bool mod_power_get_backlight_pwm(struct mod_power *mod_power,
+		unsigned int *backlight_pwm,
+		unsigned int inst);
+
+bool mod_power_get_backlight_nits(struct mod_power *mod_power,
+		unsigned int *backlight_millinit,
+		unsigned int inst);
+
+bool mod_power_get_backlight_percent(struct mod_power *mod_power,
+		unsigned int *backlight_millipercent,
+		unsigned int inst);
+
+bool mod_power_get_hw_target_backlight_pwm_nits(
+		struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_millinit,
+		unsigned int inst);
+
+bool mod_power_get_hw_target_backlight_pwm_percent(
+		struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_millipercent,
+		unsigned int inst);
+
+bool mod_power_get_hw_target_backlight_pwm(
+		struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_u16_16);
+
+bool mod_power_get_hw_backlight_pwm(
+		struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight);
+
+bool mod_power_get_hw_backlight_pwm_nits(
+		struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_millinit,
+		unsigned int inst);
+
+bool mod_power_get_hw_backlight_aux_nits(
+		struct mod_power *mod_power,
+		struct dc_stream_state **streams, int num_streams,
+		unsigned int *backlight_millinit_avg,
+		unsigned int *backlight_millinit_peak);
+
+bool mod_power_get_hw_backlight_pwm_percent(
+		struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_millipercent,
+		unsigned int inst);
+
+void mod_power_initialize_backlight_caps
+		(struct mod_power *mod_power);
+
+bool mod_power_get_panel_backlight_boundaries
+				(struct mod_power *mod_power,
+				unsigned int *out_min_backlight,
+				unsigned int *out_max_backlight,
+				unsigned int *out_ac_backlight_percent,
+				unsigned int *out_dc_backlight_percent,
+				unsigned int inst);
+
+bool mod_power_set_smooth_brightness(struct mod_power *mod_power,
+		bool enable_brightness,
+		unsigned int inst);
+
+bool mod_power_notify_mode_change(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		bool is_hdr);
+
+bool mod_power_get_varibright_level(struct mod_power *mod_power,
+		unsigned int *varibright_level);
+
+bool mod_power_get_varibright_hw_level(struct mod_power *mod_power,
+		unsigned int *varibright_level);
+
+bool mod_power_get_varibright_default_level(struct mod_power *mod_power,
+		unsigned int *varibright_level);
+
+bool mod_power_get_varibright_enable(struct mod_power *mod_power,
+		bool *varibright_enable);
+
+bool mod_power_varibright_activate(struct mod_power	*mod_power,
+		bool activate, struct dc_stream_update *stream_update);
+
+bool mod_power_varibright_feature_enable(struct mod_power *mod_power,
+		bool enable, struct dc_stream_update *stream_update);
+
+
+bool mod_power_varibright_set_level(struct mod_power *mod_power,
+		unsigned int level, struct dc_stream_update *stream_update);
+
+bool mod_power_varibright_set_hw_level(struct mod_power *mod_power,
+		unsigned int level,	struct dc_stream_update *stream_update);
+
+bool mod_power_is_abm_active(struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int inst);
+
+
+bool mod_power_set_psr_event(struct mod_power *mod_power,
+		struct dc_stream_state *stream, bool set_event,
+		enum psr_event event, bool wait);
+
+bool mod_power_get_psr_event(struct mod_power *mod_power,
+			struct dc_stream_state *stream,
+			unsigned int *active_psr_events);
+
+bool mod_power_get_psr_state(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		enum dc_psr_state *state);
+
+bool mod_power_get_psr_enabled_status(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		bool *psr_enabled);
+
+bool mod_power_set_replay_event(struct mod_power *mod_power,
+	struct dc_stream_state *stream, bool set_event,
+	enum replay_event event, bool wait_for_disable);
+
+bool mod_power_get_replay_event(struct mod_power *mod_power,
+	struct dc_stream_state *stream,
+	unsigned int *active_replay_events);
+
+bool mod_power_get_replay_active_status(const struct dc_stream_state *stream,
+	bool *replay_active);
+
+bool mod_power_replay_set_coasting_vtotal(struct mod_power *mod_power,
+	const struct dc_stream_state *stream,
+	uint32_t coasting_vtotal, uint16_t frame_skip_number);
+
+void mod_power_replay_residency(const struct dc_stream_state *stream,
+	unsigned int *residency, const bool is_start, const bool is_alpm);
+
+bool mod_power_replay_set_power_opt_and_coasting_vtotal(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, unsigned int active_replay_events, uint32_t coasting_vtotal,
+	bool is_ultra_sleep_mode, uint16_t frame_skip_number);
+
+void mod_power_replay_set_timing_sync_supported(struct mod_power *mod_power,
+	const struct dc_stream_state *stream);
+
+void mod_power_replay_disabled_adaptive_sync_sdp(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool force_disabled);
+
+void mod_power_replay_disabled_desync_error_detection(struct mod_power *mod_power,
+	const struct dc_stream_state *stream,  bool force_disabled);
+void mod_power_set_low_rr_activate(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool low_rr_supported);
+
+void mod_power_set_video_conferencing_activate(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool video_conferencing_activate);
+
+void mod_power_set_live_capture_with_cvt_activate(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool live_capture_with_cvt_activate);
+
+void mod_power_set_replay_continuously_resync(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool enable);
+
+void mod_power_set_coasting_vtotal_without_frame_update(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, uint32_t coasting_vtotal);
+
+
+
+void mod_power_psr_residency(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		unsigned int *residency,
+		const uint8_t mode);
+bool mod_power_psr_get_active_psr_events(struct mod_power *mod_power,
+		const struct dc_stream_state *stream, unsigned int *active_psr_events);
+bool mod_power_psr_set_sink_vtotal_in_psr_active(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		uint16_t psr_vtotal_idle,
+		uint16_t psr_vtotal_su);
+
+
+
+bool mod_power_backlight_percent_to_nits(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millipercent,
+		unsigned int *backlight_millinit);
+bool mod_power_backlight_nits_to_percent(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millinit,
+		unsigned int *backlight_millipercent);
+
+#endif /* MODULES_INC_MOD_POWER_H_ */
diff --git a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
index 00473c6284d5c..55c7250f18d84 100644
--- a/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
+++ b/drivers/gpu/drm/amd/display/modules/info_packet/info_packet.c
@@ -246,7 +246,7 @@ void set_vsc_packet_colorimetry_data(
 		break;
 	}
 
-	info_packet->sb[16] = (pixelEncoding << 4) | colorimetryFormat;
+	info_packet->sb[16] = (uint8_t)((pixelEncoding << 4) | colorimetryFormat);
 
 	/* Set color depth */
 	switch (stream->timing.display_color_depth) {
diff --git a/drivers/gpu/drm/amd/display/modules/power/Makefile b/drivers/gpu/drm/amd/display/modules/power/Makefile
index 9d1b22d35ece9..3000f392bdbc3 100644
--- a/drivers/gpu/drm/amd/display/modules/power/Makefile
+++ b/drivers/gpu/drm/amd/display/modules/power/Makefile
@@ -23,9 +23,9 @@
 # Makefile for the 'power' sub-module of DAL.
 #
 
-MOD_POWER = power_helpers.o
+MOD_POWER = power_helpers.o power.o power_abm.o power_psr.o power_replay.o
 
 AMD_DAL_MOD_POWER = $(addprefix $(AMDDALPATH)/modules/power/,$(MOD_POWER))
 #$(info ************  DAL POWER MODULE MAKEFILE ************)
 
-AMD_DISPLAY_FILES += $(AMD_DAL_MOD_POWER)
+AMD_DISPLAY_FILES += $(AMD_DAL_MOD_POWER)
\ No newline at end of file
diff --git a/drivers/gpu/drm/amd/display/modules/power/power.c b/drivers/gpu/drm/amd/display/modules/power/power.c
new file mode 100644
index 0000000000000..1dd65bdf8cc90
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/power/power.c
@@ -0,0 +1,498 @@
+/*
+ * Copyright 2016 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: AMD
+ *
+ */
+
+#include "dm_services.h"
+#include "dc.h"
+#include "mod_power.h"
+#include "core_types.h"
+#include "dmcu.h"
+#include "abm.h"
+#include "power_helpers.h"
+#include "dce/dmub_psr.h"
+#include "dal_asic_id.h"
+#include "link_service.h"
+#include <linux/math.h>
+
+#define DC_TRACE_LEVEL_MESSAGE(...) /* do nothing */
+#define DC_TRACE_LEVEL_MESSAGEP(...) /* do nothing */
+
+#define MOD_POWER_MAX_CONCURRENT_STREAMS 32
+#define SMOOTH_BRIGHTNESS_ADJUSTMENT_TIME_IN_MS 500
+#define LOW_REFRESH_RATE_DURATION_US_UPPER_BOUND 25000
+
+/* If system or panel does not report some sort of brightness percent to nits
+ * mapping, we will use following default values so backlight control using
+ * nits based interfaces will still work, but might not describe panel
+ * correctly. In this case percentage based backlight control should ideally
+ * be used.
+ * Min = 5 nits
+ * Max = 300 nits
+ */
+
+static const unsigned int pwr_default_min_brightness_millinits = 1000;
+static const unsigned int pwr_default_sdr_brightness_millinits = 270000;
+
+static const unsigned int default_ac_backlight_percent   = 100;
+static const unsigned int default_dc_backlight_percent   = 70;
+
+#define MOD_POWER_TO_CORE(mod_power)\
+		container_of(mod_power, struct core_power, mod_public)
+
+/* Given a specific dc_stream* this function finds its equivalent
+ * on the core_freesync->map and returns the corresponding index
+ */
+unsigned int map_index_from_stream(struct core_power *core_power,
+		const struct dc_stream_state *stream)
+{
+	unsigned int index = 0;
+
+	for (index = 0; index < core_power->num_entities; index++) {
+		if (core_power->map[index].stream == stream)
+			return index;
+	}
+	/* Could not find stream requested, this is not trivial, fix when hit*/
+	DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+						WPP_BIT_FLAG_Firmware_PsrState,
+						"map index from stream: ERROR: core_power=%p stream=%p",
+						core_power,
+						stream);
+	ASSERT(false);
+	/* We come here only when we can't map stream index.
+	 * In good cases, this would happen when we attempt to change
+	 * brightness before stream creation, in which case we create a
+	 * dummy stream with index 0.
+	 * With external monitor connected, the index passed from this return
+	 * is 1. Passing anything greater than 0 from here would always point
+	 * to bad memory.
+	 */
+	return 0;
+}
+
+bool mod_power_hw_init(struct mod_power *mod_power)
+{
+	/* Call backlight initialization */
+	return mod_power_hw_init_backlight(mod_power);
+
+	/* Future: Add other HW init here */
+}
+
+struct mod_power *mod_power_create(struct dc *dc,
+		struct mod_power_init_params *init_params,
+		unsigned int edp_num)
+{
+	struct core_power *core_power = NULL;
+	int i = 0;
+	unsigned int abm_max_config = 0;
+	unsigned int inst = 0;
+	bool is_brightness_range_valid = false;
+
+	if (dc == NULL)
+		goto fail_dc_null;
+
+	core_power = kzalloc(sizeof(struct core_power), GFP_KERNEL);
+
+	if (core_power == NULL)
+		goto fail_alloc_context;
+
+	core_power->edp_num = edp_num;
+	core_power->map = kzalloc(sizeof(struct power_entity) * MOD_POWER_MAX_CONCURRENT_STREAMS,
+				  GFP_KERNEL);
+
+	if (core_power->map == NULL)
+		goto fail_alloc_map;
+
+	for (i = 0; i < MOD_POWER_MAX_CONCURRENT_STREAMS; i++) {
+		core_power->map[i].stream = NULL;
+	}
+
+	for (i = 0; i < MOD_POWER_MAX_CONCURRENT_STREAMS; i++) {
+		core_power->map[i].psr_context =
+				kzalloc(sizeof(struct mod_power_psr_context),
+					GFP_KERNEL);
+		if (core_power->map[i].psr_context == NULL)
+			goto fail_construct;
+	}
+
+	core_power->psr_smu_optimizations_support = init_params->allow_psr_smu_optimizations;
+	core_power->multi_disp_optimizations_support = init_params->allow_psr_multi_disp_optimizations;
+
+	for (inst = 0; inst < edp_num; inst++) {
+		core_power->bl_prop[inst].min_abm_backlight =
+				init_params[inst].min_abm_backlight;
+		core_power->bl_prop[inst].disable_fractional_pwm =
+				init_params[inst].disable_fractional_pwm;
+		core_power->bl_prop[inst].use_linear_backlight_curve =
+				init_params[inst].use_linear_backlight_curve;
+		core_power->bl_prop[inst].use_nits_based_brightness =
+				init_params[inst].use_nits_based_brightness;
+		core_power->bl_prop[inst].backlight_ramping_override =
+				init_params[inst].backlight_ramping_override;
+		core_power->bl_prop[inst].backlight_ramping_reduction =
+				init_params[inst].backlight_ramping_reduction;
+		core_power->bl_prop[inst].backlight_ramping_start =
+				init_params[inst].backlight_ramping_start;
+		core_power->bl_prop[inst].use_custom_backlight_caps =
+				init_params[inst].use_custom_backlight_caps;
+		core_power->bl_prop[inst].custom_backlight_caps_config_no =
+				init_params[inst].custom_backlight_caps_config_no;
+
+		// Do not allow less than 101 backlight levels
+		if (init_params[inst].num_backlight_levels < 101)
+			core_power->bl_prop[inst].num_backlight_levels = 101;
+		else
+			core_power->bl_prop[inst].num_backlight_levels =
+				init_params[inst].num_backlight_levels;
+
+		core_power->bl_prop[inst].backlight_lut = (unsigned int *)
+				(kzalloc(sizeof(unsigned int) *
+				core_power->bl_prop[inst].num_backlight_levels, GFP_KERNEL));
+		if (core_power->bl_prop[inst].backlight_lut == NULL)
+			goto fail_alloc_backlight_array;
+	}
+
+	core_power->varibright_prop.varibright_active = false;
+
+	core_power->varibright_prop.varibright_user_enable =
+			init_params->def_varibright_enable;
+
+	// Table of ABM levels here is 1-4, but level 0 also exists as 'off'
+	if (init_params->varibright_level <= abm_defines_max_level) {
+		core_power->varibright_prop.varibright_level =
+			init_params->varibright_level;
+
+	} else {
+		core_power->varibright_prop.varibright_level = 3;
+	}
+	if (init_params->def_varibright_level <= abm_defines_max_level) {
+		core_power->varibright_prop.def_varibright_level =
+			init_params->def_varibright_level;
+	} else {
+		core_power->varibright_prop.def_varibright_level = 3;
+	}
+
+	// ABM used to contain 4 different configs. There is only 3 since ABM 2.3.
+	if ((dc->res_pool->dmcu != NULL) && (dc->res_pool->dmcu->dmcu_version.abm_version < 0x23))
+		abm_max_config = 4;
+	else
+		abm_max_config = 3;
+
+	if (init_params->abm_config_setting < abm_max_config)
+		core_power->varibright_prop.varibright_config_setting =
+			init_params->abm_config_setting;
+	else
+		core_power->varibright_prop.varibright_config_setting = 0;
+
+	for (inst = 0; inst < edp_num; inst++) {
+		core_power->bl_prop[inst].backlight_lut[0] = init_params[inst].min_backlight_pwm;
+		core_power->bl_prop[inst].backlight_lut[
+			core_power->bl_prop[inst].num_backlight_levels-1] =
+				init_params[inst].max_backlight_pwm;
+		core_power->bl_prop[inst].min_backlight_pwm = init_params[inst].min_backlight_pwm;
+		core_power->bl_prop[inst].max_backlight_pwm = init_params[inst].max_backlight_pwm;
+		core_power->bl_prop[inst].ac_backlight_percent =
+				default_ac_backlight_percent;
+		core_power->bl_prop[inst].dc_backlight_percent =
+				default_dc_backlight_percent;
+		core_power->bl_prop[inst].backlight_caps_valid = false;
+
+		if (core_power->bl_prop[inst].use_nits_based_brightness) {
+			core_power->bl_prop[inst].min_brightness_millinits =
+					init_params[inst].panel_min_millinits;
+			core_power->bl_prop[inst].max_brightness_millinits =
+					init_params[inst].panel_max_millinits;
+		} else {
+
+			core_power->bl_prop[inst].min_brightness_millinits =
+					pwr_default_min_brightness_millinits;
+			core_power->bl_prop[inst].max_brightness_millinits =
+					pwr_default_sdr_brightness_millinits;
+		}
+
+		core_power->bl_prop[inst].backlight_range =
+				core_power->bl_prop[inst].max_backlight_pwm-
+				core_power->bl_prop[inst].min_backlight_pwm;
+
+		core_power->bl_prop[inst].nits_range =
+				core_power->bl_prop[inst].max_brightness_millinits -
+				core_power->bl_prop[inst].min_brightness_millinits;
+
+		core_power->bl_state[inst].smooth_brightness_enabled = true;
+	}
+
+	/* Check if at least 1 instance in core_power is populated before failing */
+	for (inst = 0; inst < edp_num; inst++) {
+		if (core_power->bl_prop[inst].nits_range != 0 && core_power->bl_prop[inst].backlight_range != 0) {
+			is_brightness_range_valid = true;
+			break;
+		}
+
+	}
+	if (!is_brightness_range_valid)
+		goto fail_bad_brightness_range;
+
+	core_power->num_entities = 0;
+
+	core_power->dc = dc;
+	for (inst = 0; inst < edp_num; inst++) {
+		initialize_backlight_caps(core_power, inst);
+		core_power->bl_state[inst].backlight_millipercent =
+			core_power->bl_prop[inst].dc_backlight_percent * 1000;
+		core_power->bl_state[inst].backlight_pwm = backlight_millipercent_to_pwm(core_power,
+		core_power->bl_state[inst].backlight_millipercent, inst);
+		core_power->bl_state[inst].backlight_millinit = backlight_millipercent_to_millinit(core_power,
+		core_power->bl_state[inst].backlight_millipercent, inst);
+	}
+
+	return &core_power->mod_public;
+
+fail_bad_brightness_range:
+fail_alloc_backlight_array:
+	for (inst = 0; inst < edp_num; inst++)
+		if (core_power->bl_prop[inst].backlight_lut)
+			kfree(core_power->bl_prop[inst].backlight_lut);
+fail_construct:
+	for (i = 0; i < MOD_POWER_MAX_CONCURRENT_STREAMS; i++) {
+		if (core_power->map[i].psr_context)
+			kfree(core_power->map[i].psr_context);
+	}
+	kfree(core_power->map);
+
+fail_alloc_map:
+	kfree(core_power);
+
+fail_alloc_context:
+fail_dc_null:
+	return NULL;
+}
+
+void mod_power_destroy(struct mod_power *mod_power)
+{
+	if (mod_power != NULL) {
+		unsigned int i;
+		struct core_power *core_power =
+				MOD_POWER_TO_CORE(mod_power);
+
+		for (i = 0; i < MOD_POWER_MAX_CONCURRENT_STREAMS; i++)
+			if (core_power->map[i].psr_context)
+				kfree(core_power->map[i].psr_context);
+
+		for (i = 0; i < core_power->num_entities; i++)
+			if (core_power->map[i].stream)
+				dc_stream_release(core_power->map[i].stream);
+
+		kfree(core_power->map);
+
+		for (i = 0; i < MAX_NUM_EDP; i++)
+			if (core_power->bl_prop[i].backlight_lut)
+				kfree(core_power->bl_prop[i].backlight_lut);
+
+		kfree(core_power);
+	}
+}
+
+bool mod_power_add_stream(struct mod_power *mod_power,
+		struct dc_stream_state *stream, struct psr_caps *caps)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities < MOD_POWER_MAX_CONCURRENT_STREAMS) {
+		dc_stream_retain(stream);
+
+		core_power->map[core_power->num_entities].stream = stream;
+		core_power->map[core_power->num_entities].caps = caps;
+
+		// initialize cached PSR params to something "safe" (something that is
+		// consistent with disabled PSR state)
+		core_power->map[core_power->num_entities].psr_enabled = 0;
+		core_power->map[core_power->num_entities].psr_events = psr_event_vsync;
+		core_power->map[core_power->num_entities].psr_power_opt = 0;
+		core_power->num_entities++;
+		return true;
+	}
+
+	DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+						WPP_BIT_FLAG_Firmware_PsrState,
+						"mod_power: add_stream: ERROR: stream=%p num_entities=%u >= MOD_POWER_MAX_CONCURRENT_STREAMS",
+						stream,
+						core_power->num_entities);
+
+	return false;
+}
+
+bool mod_power_remove_stream(struct mod_power *mod_power,
+		const struct dc_stream_state *stream)
+{
+	unsigned int i = 0;
+	struct core_power *core_power = NULL;
+	unsigned int index = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	if (core_power->num_entities == 0) {
+		/* trying to remove a stream a second time or have not added yet */
+		BREAK_TO_DEBUGGER();
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"mod_power: remove_stream: ERROR: num_entities=0 stream=%p",
+							stream);
+		return false;
+	}
+
+	index = map_index_from_stream(core_power, stream);
+
+	if (index >= core_power->num_entities) {
+		/* trying to remove a stream a second time or have not added yet */
+		BREAK_TO_DEBUGGER();
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"mod_power: remove_stream: ERROR: index=%u >= num_entities=%u stream=%p",
+							index,
+							core_power->num_entities,
+							stream);
+		return false;
+	}
+
+	dc_stream_release(core_power->map[index].stream);
+	core_power->map[index].stream = NULL;
+	/* To remove this entity, shift everything after down */
+	for (i = index; i < core_power->num_entities - 1; i++) {
+		core_power->map[i].stream = core_power->map[i + 1].stream;
+		core_power->map[i].caps = core_power->map[i + 1].caps;
+
+		// copy over cached parameters in case they map to PSR capable display
+		core_power->map[i].psr_enabled = core_power->map[i + 1].psr_enabled;
+		core_power->map[i].psr_events = core_power->map[i + 1].psr_events;
+		core_power->map[i].psr_power_opt = core_power->map[i + 1].psr_power_opt;
+
+		memcpy(core_power->map[i].psr_context, core_power->map[i + 1].psr_context, sizeof(struct mod_power_psr_context));
+		memset(core_power->map[i + 1].psr_context, 0, sizeof(struct mod_power_psr_context));
+	}
+	core_power->num_entities--;
+
+	return true;
+}
+
+/*
+ * Replace_stream should be used when there is a mode set for existing
+ * display target with a valid stream. In this case might need to retain
+ * cached PSR state (events, power opt, en/dis) if we are dealing with PSR
+ * capable display. If mod_power_remove and mod_power_add are used instead,
+ * then stream may be assigned to a different slot and may end up with
+ * wrong cached PSR state. It is hard to tell which PSR events should
+ * persist through mode set or what psr_events should be initialized to, so
+ * it might be better just to retain them all.
+ */
+bool mod_power_replace_stream(struct mod_power *mod_power,
+		const struct dc_stream_state *current_stream,
+		struct dc_stream_state *new_stream,
+		struct psr_caps *new_caps)
+{
+	struct core_power *core_power = NULL;
+	unsigned int index = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	if (core_power->num_entities == 0) {
+		/* no streams exist in the table yet */
+		BREAK_TO_DEBUGGER();
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"mod_power: replace_stream: ERROR: num_entities=0 stream=%p",
+							current_stream);
+		return false;
+	}
+
+	index = map_index_from_stream(core_power, current_stream);
+
+	if (index >= core_power->num_entities) {
+		/* trying to replace a non-existent stream */
+		BREAK_TO_DEBUGGER();
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"mod_power: replace_stream: ERROR: index=%u >= num_entities=%u stream=%p",
+							index,
+							core_power->num_entities,
+							current_stream);
+		return false;
+	}
+
+	dc_stream_release(core_power->map[index].stream);
+	dc_stream_retain(new_stream);
+	core_power->map[index].stream = new_stream;
+	core_power->map[index].caps = new_caps;
+	memset(core_power->map[index].psr_context, 0, sizeof(struct mod_power_psr_context));
+
+	return true;
+}
+bool mod_power_notify_mode_change(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		bool is_hdr)
+{
+	unsigned int stream_index = 0;
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+	struct dc *dc = NULL;
+	unsigned int panel_inst = 0;
+
+	if ((mod_power == NULL) || (stream == NULL))
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	stream_index = map_index_from_stream(core_power, stream);
+
+	if (stream_index >= core_power->num_entities)
+		return false;
+
+	dc = core_power->dc;
+	link = dc_stream_get_link(stream);
+
+	if (link != NULL && dc_get_edp_link_panel_inst(dc, link, &panel_inst)) {
+		ASSERT(link->ddc->ddc_pin->hw_info.ddc_channel <= 0xFF);
+		uint8_t aux_inst = (uint8_t)link->ddc->ddc_pin->hw_info.ddc_channel;
+
+		mod_power_update_backlight_on_mode_change(core_power, link, panel_inst, aux_inst, is_hdr);
+
+		/* Handle PSR notification */
+		mod_power_psr_notify_mode_change(mod_power, stream, link, stream_index);
+
+		/* Handle Replay notification */
+		mod_power_replay_notify_mode_change(mod_power, dc, link, stream, stream_index);
+	}
+
+	return true;
+}
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_abm.c b/drivers/gpu/drm/amd/display/modules/power/power_abm.c
new file mode 100644
index 0000000000000..b94dbb9e64a73
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/power/power_abm.c
@@ -0,0 +1,2062 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dm_services.h"
+#include "dc.h"
+#include "mod_power.h"
+#include "core_types.h"
+#include "dmcu.h"
+#include "abm.h"
+#include "power_helpers.h"
+#include "dce/dmub_psr.h"
+#include "dal_asic_id.h"
+#include "link_service.h"
+#include <linux/math.h>
+
+#define DC_TRACE_LEVEL_MESSAGE(...) /* do nothing */
+#define DC_TRACE_LEVEL_MESSAGEP(...) /* do nothing */
+
+#define DIV_ROUNDUP(a, b) (((a)+((b)/2))/(b))
+#define bswap16_based_on_endian(big_endian, value) \
+	((big_endian) ? cpu_to_be16(value) : cpu_to_le16(value))
+
+/* Possible Min Reduction config from least aggressive to most aggressive
+ *  0    1     2     3     4     5     6     7     8     9     10    11   12
+ * 100  98.0 94.1  94.1  85.1  80.3  75.3  69.4  60.0  57.6  50.2  49.8  40.0 %
+ */
+static const unsigned char min_reduction_table[13] = {
+0xff, 0xfa, 0xf0, 0xf0, 0xd9, 0xcd, 0xc0, 0xb1, 0x99, 0x93, 0x80, 0x82, 0x66};
+
+/* Possible Max Reduction configs from least aggressive to most aggressive
+ *  0    1     2     3     4     5     6     7     8     9     10    11   12
+ * 96.1 89.8 85.1  80.3  69.4  64.7  64.7  50.2  39.6  30.2  30.2  30.2  19.6 %
+ */
+static const unsigned char max_reduction_table[13] = {
+0xf5, 0xe5, 0xd9, 0xcd, 0xb1, 0xa5, 0xa5, 0x80, 0x65, 0x4d, 0x4d, 0x4d, 0x32};
+
+/* Possible ABM 2.2 Min Reduction configs from least aggressive to most aggressive
+ *  0    1     2     3     4     5     6     7     8     9     10    11   12
+ * 100  100   100   100   100   100   100   100  100  92.2  83.1  75.3  75.3 %
+ */
+static const unsigned char min_reduction_table_v_2_2[13] = {
+0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xeb, 0xd4, 0xc0, 0xc0};
+
+/* Possible ABM 2.2 Max Reduction configs from least aggressive to most aggressive
+ *  0    1     2     3     4     5     6     7     8     9     10    11   12
+ * 96.1 89.8 74.9  69.4  64.7  52.2  48.6  39.6  30.2  25.1  19.6  12.5  12.5 %
+ */
+static const unsigned char max_reduction_table_v_2_2[13] = {
+0xf5, 0xe5, 0xbf, 0xb1, 0xa5, 0x85, 0x7c, 0x65, 0x4d, 0x40, 0x32, 0x20, 0x20};
+
+/* Predefined ABM configuration sets. We may have different configuration sets
+ * in order to satisfy different power/quality requirements.
+ */
+static const unsigned char abm_config[abm_defines_max_config][abm_defines_max_level] = {
+/*  ABM Level 1,    ABM Level 2,    ABM Level 3,    ABM Level 4 */
+{       2,              5,              7,              8       },	/* Default - Medium aggressiveness */
+{       2,              5,              8,              11      },	/* Alt #1  - Increased aggressiveness */
+{       0,              2,              4,              8       },	/* Alt #2  - Minimal aggressiveness */
+{       3,              6,              10,             12      },	/* Alt #3  - Super aggressiveness */
+};
+
+struct abm_parameters {
+	unsigned char min_reduction;
+	unsigned char max_reduction;
+	unsigned char bright_pos_gain;
+	unsigned char dark_pos_gain;
+	unsigned char brightness_gain;
+	unsigned char contrast_factor;
+	unsigned char deviation_gain;
+	unsigned char min_knee;
+	unsigned char max_knee;
+	unsigned short blRampReduction;
+	unsigned short blRampStart;
+};
+
+static const struct abm_parameters abm_settings_config0[abm_defines_max_level] = {
+//  min_red  max_red  bright_pos  dark_pos  bright_gain  contrast  dev   min_knee  max_knee  blRed    blStart
+	{0xff,   0xbf,    0x20,       0x00,     0xff,        0x99,     0xb3, 0x40,     0xe0,     0xf777,  0xcccc},
+	{0xde,   0x85,    0x20,       0x00,     0xe0,        0x90,     0xa8, 0x40,     0xc8,     0xf777,  0xcccc},
+	{0xb0,   0x50,    0x20,       0x00,     0xc0,        0x88,     0x78, 0x70,     0xa0,     0xeeee,  0x9999},
+	{0x82,   0x40,    0x20,       0x00,     0x00,        0xb8,     0xb3, 0x70,     0x70,     0xe333,  0xb333},
+};
+
+static const struct abm_parameters abm_settings_config1[abm_defines_max_level] = {
+//  min_red  max_red  bright_pos  dark_pos  bright_gain  contrast  dev   min_knee  max_knee  blRed  blStart
+	{0xf0,   0xd9,    0x20,       0x00,     0x00,        0xff,     0xb3, 0x70,     0x70,     0xcccc,  0xcccc},
+	{0xcd,   0xa5,    0x20,       0x00,     0x00,        0xff,     0xb3, 0x70,     0x70,     0xcccc,  0xcccc},
+	{0x99,   0x65,    0x20,       0x00,     0x00,        0xff,     0xb3, 0x70,     0x70,     0xcccc,  0xcccc},
+	{0x82,   0x4d,    0x20,       0x00,     0x00,        0xff,     0xb3, 0x70,     0x70,     0xcccc,  0xcccc},
+};
+
+static const struct abm_parameters abm_settings_config2[abm_defines_max_level] = {
+//  min_red  max_red  bright_pos  dark_pos  bright_gain  contrast  dev   min_knee  max_knee  blRed    blStart
+	{0xf0,   0xbf,    0x20,       0x00,     0x88,        0x99,     0xb3, 0x40,     0xe0,    0x0000,  0xcccc},
+	{0xd8,   0x85,    0x20,       0x00,     0x70,        0x90,     0xa8, 0x40,     0xc8,    0x0700,  0xb333},
+	{0xb8,   0x58,    0x20,       0x00,     0x64,        0x88,     0x78, 0x70,     0xa0,    0x7000,  0x9999},
+	{0x82,   0x40,    0x20,       0x00,     0x00,        0xb8,     0xb3, 0x70,     0x70,    0xc333,  0xb333},
+};
+
+static const struct abm_parameters * const abm_settings[] = {
+	abm_settings_config0,
+	abm_settings_config1,
+	abm_settings_config2,
+};
+
+static const struct dm_bl_data_point custom_backlight_curve0[] = {
+		{2, 14}, {4, 16}, {6, 18}, {8, 21}, {10, 23}, {12, 26}, {14, 29}, {16, 32}, {18, 35},
+		{20, 38}, {22, 41}, {24, 44}, {26, 48}, {28, 52}, {30, 55}, {32, 59}, {34, 62},
+		{36, 67}, {38, 71}, {40, 75}, {42, 80}, {44, 84}, {46, 88}, {48, 93}, {50, 98},
+		{52, 103}, {54, 108}, {56, 113}, {58, 118}, {60, 123}, {62, 129}, {64, 135}, {66, 140},
+		{68, 146}, {70, 152}, {72, 158}, {74, 164}, {76, 171}, {78, 177}, {80, 183}, {82, 190},
+		{84, 197}, {86, 204}, {88, 211}, {90, 218}, {92, 225}, {94, 232}, {96, 240}, {98, 247}};
+
+struct custom_backlight_profile {
+	uint8_t  ac_level_percentage;
+	uint8_t  dc_level_percentage;
+	uint8_t  min_input_signal;
+	uint8_t  max_input_signal;
+	uint8_t  num_data_points;
+	const struct dm_bl_data_point *data_points;
+};
+
+static const struct custom_backlight_profile custom_backlight_profiles[] = {
+		{100, 32, 12, 255, ARRAY_SIZE(custom_backlight_curve0), custom_backlight_curve0},
+};
+
+#define NUM_AMBI_LEVEL    5
+#define NUM_AGGR_LEVEL    4
+#define NUM_POWER_FN_SEGS 8
+#define NUM_BL_CURVE_SEGS 16
+#define IRAM_SIZE 256
+
+#define IRAM_RESERVE_AREA_START_V2 0xF0  // reserve 0xF0~0xF6 are write by DMCU only
+#define IRAM_RESERVE_AREA_END_V2 0xF6  // reserve 0xF0~0xF6 are write by DMCU only
+
+#define IRAM_RESERVE_AREA_START_V2_2 0xF0  // reserve 0xF0~0xFF are write by DMCU only
+#define IRAM_RESERVE_AREA_END_V2_2 0xFF  // reserve 0xF0~0xFF are write by DMCU only
+
+#pragma pack(push, 1)
+/* NOTE: iRAM is 256B in size */
+struct iram_table_v_2 {
+	/* flags                      */
+	uint16_t min_abm_backlight;					/* 0x00 U16  */
+
+	/* parameters for ABM2.0 algorithm */
+	uint8_t min_reduction[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x02 U0.8 */
+	uint8_t max_reduction[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x16 U0.8 */
+	uint8_t bright_pos_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];	/* 0x2a U2.6 */
+	uint8_t bright_neg_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];	/* 0x3e U2.6 */
+	uint8_t dark_pos_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x52 U2.6 */
+	uint8_t dark_neg_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x66 U2.6 */
+	uint8_t iir_curve[NUM_AMBI_LEVEL];				/* 0x7a U0.8 */
+	uint8_t deviation_gain;						/* 0x7f U0.8 */
+
+	/* parameters for crgb conversion */
+	uint16_t crgb_thresh[NUM_POWER_FN_SEGS];			/* 0x80 U3.13 */
+	uint16_t crgb_offset[NUM_POWER_FN_SEGS];			/* 0x90 U1.15 */
+	uint16_t crgb_slope[NUM_POWER_FN_SEGS];				/* 0xa0 U4.12 */
+
+	/* parameters for custom curve */
+	/* thresholds for brightness --> backlight */
+	uint16_t backlight_thresholds[NUM_BL_CURVE_SEGS];		/* 0xb0 U16.0 */
+	/* offsets for brightness --> backlight */
+	uint16_t backlight_offsets[NUM_BL_CURVE_SEGS];			/* 0xd0 U16.0 */
+
+	/* For reading PSR State directly from IRAM */
+	uint8_t psr_state;						/* 0xf0       */
+	uint8_t dmcu_mcp_interface_version;				/* 0xf1       */
+	uint8_t dmcu_abm_feature_version;				/* 0xf2       */
+	uint8_t dmcu_psr_feature_version;				/* 0xf3       */
+	uint16_t dmcu_version;						/* 0xf4       */
+	uint8_t dmcu_state;						/* 0xf6       */
+
+	uint16_t blRampReduction;					/* 0xf7       */
+	uint16_t blRampStart;						/* 0xf9       */
+	uint8_t dummy5;							/* 0xfb       */
+	uint8_t dummy6;							/* 0xfc       */
+	uint8_t dummy7;							/* 0xfd       */
+	uint8_t dummy8;							/* 0xfe       */
+	uint8_t dummy9;							/* 0xff       */
+};
+
+struct iram_table_v_2_2 {
+	/* flags                      */
+	uint16_t flags;							/* 0x00 U16  */
+
+	/* parameters for ABM2.2 algorithm */
+	uint8_t min_reduction[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x02 U0.8 */
+	uint8_t max_reduction[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x16 U0.8 */
+	uint8_t bright_pos_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];	/* 0x2a U2.6 */
+	uint8_t dark_pos_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x3e U2.6 */
+	uint8_t hybrid_factor[NUM_AGGR_LEVEL];				/* 0x52 U0.8 */
+	uint8_t contrast_factor[NUM_AGGR_LEVEL];			/* 0x56 U0.8 */
+	uint8_t deviation_gain[NUM_AGGR_LEVEL];				/* 0x5a U0.8 */
+	uint8_t iir_curve[NUM_AMBI_LEVEL];				/* 0x5e U0.8 */
+	uint8_t min_knee[NUM_AGGR_LEVEL];				/* 0x63 U0.8 */
+	uint8_t max_knee[NUM_AGGR_LEVEL];				/* 0x67 U0.8 */
+	uint16_t min_abm_backlight;					/* 0x6b U16  */
+	uint8_t pad[19];						/* 0x6d U0.8 */
+
+	/* parameters for crgb conversion */
+	uint16_t crgb_thresh[NUM_POWER_FN_SEGS];			/* 0x80 U3.13 */
+	uint16_t crgb_offset[NUM_POWER_FN_SEGS];			/* 0x90 U1.15 */
+	uint16_t crgb_slope[NUM_POWER_FN_SEGS];				/* 0xa0 U4.12 */
+
+	/* parameters for custom curve */
+	/* thresholds for brightness --> backlight */
+	uint16_t backlight_thresholds[NUM_BL_CURVE_SEGS];		/* 0xb0 U16.0 */
+	/* offsets for brightness --> backlight */
+	uint16_t backlight_offsets[NUM_BL_CURVE_SEGS];			/* 0xd0 U16.0 */
+
+	/* For reading PSR State directly from IRAM */
+	uint8_t psr_state;						/* 0xf0       */
+	uint8_t dmcu_mcp_interface_version;				/* 0xf1       */
+	uint8_t dmcu_abm_feature_version;				/* 0xf2       */
+	uint8_t dmcu_psr_feature_version;				/* 0xf3       */
+	uint16_t dmcu_version;						/* 0xf4       */
+	uint8_t dmcu_state;						/* 0xf6       */
+
+	uint8_t dummy1;							/* 0xf7       */
+	uint8_t dummy2;							/* 0xf8       */
+	uint8_t dummy3;							/* 0xf9       */
+	uint8_t dummy4;							/* 0xfa       */
+	uint8_t dummy5;							/* 0xfb       */
+	uint8_t dummy6;							/* 0xfc       */
+	uint8_t dummy7;							/* 0xfd       */
+	uint8_t dummy8;							/* 0xfe       */
+	uint8_t dummy9;							/* 0xff       */
+};
+#pragma pack(pop)
+
+#define MOD_POWER_MAX_CONCURRENT_STREAMS 32
+#define SMOOTH_BRIGHTNESS_ADJUSTMENT_TIME_IN_MS 500
+
+/* If system or panel does not report some sort of brightness percent to nits
+ * mapping, we will use following default values so backlight control using
+ * nits based interfaces will still work, but might not describe panel
+ * correctly. In this case percentage based backlight control should ideally
+ * be used.
+ * Min = 5 nits
+ * Max = 300 nits
+ */
+
+#define MOD_POWER_TO_CORE(mod_power)\
+		container_of(mod_power, struct core_power, mod_public)
+
+static uint16_t backlight_8_to_16(unsigned int backlight_8bit)
+{
+	return (uint16_t)(backlight_8bit * 0x101);
+}
+
+unsigned int backlight_millipercent_to_millinit(
+		struct core_power *core_power, unsigned int millipercent, unsigned int inst)
+{
+	unsigned int millinit = 0;
+	unsigned long long numerator = 0;
+
+	if (core_power == NULL)
+		return 0;
+
+	numerator = ((unsigned long long)millipercent) *
+				core_power->bl_prop[inst].nits_range;
+	millinit = ((unsigned int)div_u64(numerator, 100000)) +
+			core_power->bl_prop[inst].min_brightness_millinits;
+
+	return millinit;
+}
+
+static unsigned int backlight_millinit_to_millipercent(
+		struct core_power *core_power, unsigned int millinit, unsigned int inst)
+{
+	unsigned int millipercent = 0;
+	unsigned long long numerator = 0;
+
+	if (core_power == NULL)
+		return 0;
+
+	if (millinit <= core_power->bl_prop[inst].min_brightness_millinits)
+		return 0;
+
+	if (millinit >= core_power->bl_prop[inst].max_brightness_millinits)
+		return (100 * 1000);
+
+	numerator = (((unsigned long long)millinit) -
+			core_power->bl_prop[inst].min_brightness_millinits) * 100000;
+	millipercent = ((unsigned int)div_u64(numerator,
+				core_power->bl_prop[inst].nits_range));
+
+	return millipercent;
+}
+
+static unsigned int backlight_pwm_to_millipercent(
+		struct core_power *core_power, unsigned int pwm, unsigned int inst)
+{
+	unsigned int millipercent = 0;
+	unsigned int max_index = 0;
+
+	if (core_power == NULL)
+		return 0;
+
+	if (!core_power->bl_prop[inst].backlight_caps_valid)
+		return 0;
+
+	/* Doesn't really make sense to have one single backlight level
+	 * possible...
+	 */
+	if (core_power->bl_prop[inst].num_backlight_levels < 2)
+		return 0;
+
+	max_index = core_power->bl_prop[inst].num_backlight_levels - 1;
+
+	if (pwm <= core_power->bl_prop[inst].backlight_lut[0])
+		return 0;
+
+	if (pwm > core_power->bl_prop[inst].backlight_lut[max_index])
+		return (100 * 1000);
+
+	/* We need to do a binary search over the array for where the pwm level
+	 * is in the lut. Based on the index we can determine percentage.
+	 */
+	unsigned int min = 0;
+	unsigned int max = max_index;
+	unsigned int mid = 0;
+
+	while (max >= min) {
+		mid = (min + max) / 2; /* floor of half range */
+
+		if (core_power->bl_prop[inst].backlight_lut[mid] < pwm)
+			min = mid + 1;
+		else if (core_power->bl_prop[inst].backlight_lut[mid] > pwm)
+			max = mid - 1;
+		else
+			break;
+	}
+
+	/* In this case, exact match is not found. Check if mid/min/max
+	 * value is actually closer.
+	 */
+	if (max < min) {
+		unsigned int min_delta;
+		unsigned int mid_delta;
+		unsigned int max_delta;
+
+		min_delta = (core_power->bl_prop[inst].backlight_lut[min] > pwm) ?
+				core_power->bl_prop[inst].backlight_lut[min] - pwm :
+				pwm - core_power->bl_prop[inst].backlight_lut[min];
+
+		mid_delta = (core_power->bl_prop[inst].backlight_lut[mid] > pwm) ?
+				core_power->bl_prop[inst].backlight_lut[mid] - pwm :
+				pwm - core_power->bl_prop[inst].backlight_lut[mid];
+
+		max_delta = (core_power->bl_prop[inst].backlight_lut[max] > pwm) ?
+				core_power->bl_prop[inst].backlight_lut[max] - pwm :
+				pwm - core_power->bl_prop[inst].backlight_lut[max];
+
+		if ((min_delta < mid_delta) && (min_delta < max_delta))
+			mid = min;
+
+		if ((max_delta < mid_delta) && (max_delta < min_delta))
+			mid = max;
+	}
+
+	/* No interpolation, just take closest index */
+	millipercent = 1000 * 100 * mid / max_index;
+
+	return millipercent;
+}
+
+static unsigned int backlight_pwm_to_millinit(
+		struct core_power *core_power, unsigned int pwm, unsigned int inst)
+{
+	unsigned int millinit = 0;
+
+	if (core_power == NULL)
+		return 0;
+
+	if (pwm <= core_power->bl_prop[inst].min_backlight_pwm)
+		return core_power->bl_prop[inst].min_brightness_millinits;
+
+	if (pwm >= core_power->bl_prop[inst].max_backlight_pwm)
+		return core_power->bl_prop[inst].max_brightness_millinits;
+
+	millinit = ((unsigned int)div_u64(((unsigned long long)pwm -
+				core_power->bl_prop[inst].min_backlight_pwm) *
+				core_power->bl_prop[inst].nits_range,
+				core_power->bl_prop[inst].backlight_range));
+
+	millinit += core_power->bl_prop[inst].min_brightness_millinits;
+
+	if (millinit > core_power->bl_prop[inst].max_brightness_millinits)
+		millinit = core_power->bl_prop[inst].max_brightness_millinits;
+
+	return millinit;
+}
+
+unsigned int backlight_millipercent_to_pwm(
+		struct core_power *core_power, unsigned int millipercent, unsigned int inst)
+{
+	unsigned int pwm = (unsigned int)-1;
+	unsigned int index = 0;
+
+	if (core_power == NULL)
+		return 0;
+
+	// Bypass the brightness mapping LUT
+	if (core_power->bl_prop->use_linear_backlight_curve) {
+		pwm = core_power->bl_prop[inst].min_backlight_pwm +
+			(unsigned int) div_u64((unsigned long long) millipercent *
+			core_power->bl_prop[inst].backlight_range,
+			100000);
+
+		if (pwm > core_power->bl_prop[inst].max_backlight_pwm)
+			pwm = core_power->bl_prop[inst].max_backlight_pwm;
+
+		return pwm;
+	}
+
+	if (millipercent >= (100 * 1000))
+		return core_power->bl_prop[inst].backlight_lut[core_power->bl_prop[inst].num_backlight_levels - 1];
+
+	/* This will give the floor index. */
+	index = ((core_power->bl_prop[inst].num_backlight_levels - 1) *
+						millipercent) / 100000;
+	/* Null check otherwise eDP doesn't lightup when connected to DP1 */
+	if (core_power->bl_prop[inst].backlight_lut == NULL)
+		return pwm;
+
+	pwm = core_power->bl_prop[inst].backlight_lut[index];
+
+	return pwm;
+}
+
+static unsigned int backlight_millinit_to_pwm(
+		struct core_power *core_power, unsigned int millinit, unsigned int inst)
+{
+	unsigned int pwm = 0;
+
+	if (core_power == NULL)
+		return 0;
+
+	/* For nits based brightness, the signal will be a value
+	 * between the minimum and maximum value.
+	 */
+	if (millinit >= core_power->bl_prop[inst].max_brightness_millinits)
+		return core_power->bl_prop[inst].max_backlight_pwm;
+	else if (millinit <= core_power->bl_prop[inst].min_brightness_millinits)
+		return core_power->bl_prop[inst].min_backlight_pwm;
+
+	pwm = ((unsigned int)div_u64(((unsigned long long)millinit -
+			core_power->bl_prop[inst].min_brightness_millinits) *
+			core_power->bl_prop[inst].backlight_range,
+			core_power->bl_prop[inst].nits_range));
+
+	pwm += core_power->bl_prop[inst].min_backlight_pwm;
+
+	if (pwm > core_power->bl_prop[inst].max_backlight_pwm)
+		pwm = core_power->bl_prop[inst].max_backlight_pwm;
+
+	return pwm;
+}
+
+static bool validate_ext_backlight_caps(
+		struct dm_acpi_atif_backlight_caps *ext_backlight_caps)
+{
+	unsigned int i;
+	unsigned int num_of_data_points = 0;
+	unsigned int last_signal_level = 0;
+	unsigned int last_luminance = 0;
+
+	num_of_data_points = ext_backlight_caps->num_data_points;
+
+	/* Validation rules:
+	 * 1. BIOS should carry customized data points and
+	 * the number of data points should not be larger than 99.
+	 * 2. The max_input_signal should be larger than min_input_signal.
+	 * 3. For each data point:
+	 *	a. luminance should be in ascending order and
+	 *	should not be 0 or 100 since the corresponding signal_level
+	 *	are assigned by min_input_signal and max_input_signal.
+	 *	b. signal_level should be in ascending order and
+	 *	be within the range of min/max_input_signal.
+	 */
+	if (num_of_data_points > BL_DATA_POINTS)
+		return false;
+
+	if (ext_backlight_caps->min_input_signal >= ext_backlight_caps->max_input_signal)
+		return false;
+
+	last_signal_level = ext_backlight_caps->min_input_signal;
+	for (i = 0; i < num_of_data_points; i++) {
+		unsigned int luminance = ext_backlight_caps->data_points[i].luminance;
+		unsigned int signal_level = ext_backlight_caps->data_points[i].signal_level;
+
+		if ((luminance <= last_luminance) || (luminance > BL_DATA_POINTS))
+			return false;
+
+		if ((signal_level <= last_signal_level) || (signal_level >= ext_backlight_caps->max_input_signal))
+			return false;
+
+		last_signal_level = signal_level;
+		last_luminance = luminance;
+	}
+
+	return true;
+}
+
+/* hard coded to default backlight curve. */
+void initialize_backlight_caps(struct core_power *core_power, unsigned int inst)
+{
+	unsigned int i;
+	struct dm_acpi_atif_backlight_caps *ext_backlight_caps = NULL;
+	bool custom_curve_present = false;
+	unsigned int num_levels = 0;
+	struct dc *dc = NULL;
+	enum dm_acpi_display_type acpi_display_type =
+		(inst == 0) ? AcpiDisplayType_LCD1 : AcpiDisplayType_LCD2;
+
+	if (core_power == NULL)
+		return;
+	dc = core_power->dc;
+
+	num_levels = core_power->bl_prop[inst].num_backlight_levels;
+
+	/* Allocate memory for ATIF output
+	 * (do not want to use 256 bytes on the stack)
+	 */
+	ext_backlight_caps = (struct dm_acpi_atif_backlight_caps *)
+		(kzalloc(sizeof(struct dm_acpi_atif_backlight_caps),
+				GFP_KERNEL));
+
+	if (ext_backlight_caps == NULL)
+		return;
+
+	/* Retrieve ACPI extended brightness caps */
+	if (dm_query_extended_brightness_caps
+		(dc->ctx, acpi_display_type, ext_backlight_caps)) {
+		custom_curve_present = validate_ext_backlight_caps(ext_backlight_caps);
+	}
+
+	if (core_power->bl_prop[inst].use_custom_backlight_caps &&
+			fill_custom_backlight_caps(
+					core_power->bl_prop[inst].custom_backlight_caps_config_no,
+					ext_backlight_caps)) {
+		custom_curve_present = validate_ext_backlight_caps(ext_backlight_caps);
+	}
+
+	if (custom_curve_present) {
+		unsigned int index = 1;
+		unsigned int num_of_data_points = ext_backlight_caps->num_data_points;
+
+		core_power->bl_prop[inst].ac_backlight_percent =
+			ext_backlight_caps->ac_level_percentage;
+		core_power->bl_prop[inst].dc_backlight_percent =
+			ext_backlight_caps->dc_level_percentage;
+		core_power->bl_prop[inst].backlight_lut[0] =
+			backlight_8_to_16(
+				ext_backlight_caps->min_input_signal);
+		core_power->bl_prop[inst].backlight_lut[num_levels - 1] =
+			backlight_8_to_16(
+				ext_backlight_caps->max_input_signal);
+
+		/* Filling translation table from data points -
+		 * between every two provided data points we
+		 * lineary interpolate missing values
+		 */
+		for (i = 0; i < num_of_data_points; i++) {
+			unsigned int luminance =
+				ext_backlight_caps->data_points[i].luminance;
+			unsigned int signal_level =
+				backlight_8_to_16(
+					ext_backlight_caps->data_points[i].signal_level);
+
+			/* Since luminance is a percentage, scale it by num_levels*/
+			luminance = (luminance * num_levels) / 101;
+
+			/* Lineary interpolate missing values */
+			if (index < luminance) {
+				unsigned int base_value =
+					core_power->bl_prop[inst].backlight_lut[index-1];
+				unsigned int delta_signal =
+					signal_level - base_value;
+				unsigned int delta_luma =
+					luminance - index + 1;
+				unsigned int step  = delta_signal;
+
+				for (; index < luminance; index++) {
+					core_power->bl_prop[inst].backlight_lut[index] =
+						base_value + (step / delta_luma);
+					step += delta_signal;
+				}
+			}
+
+			/* Now [index == luminance],
+			 * so we can add data point to the translation table
+			 */
+			core_power->bl_prop[inst].backlight_lut[index++] = signal_level;
+		}
+
+		/* Complete the final segment of interpolation -
+		 * between last datapoint and maximum value
+		 */
+		if (index < num_levels - 1) {
+			unsigned int base_value =
+				core_power->bl_prop[inst].backlight_lut[index-1];
+			unsigned int delta_signal =
+				core_power->bl_prop[inst].backlight_lut[num_levels - 1] -
+								base_value;
+			unsigned int delta_luma = num_levels - index;
+			unsigned int step = delta_signal;
+
+			for (; index < num_levels - 1; index++) {
+				core_power->bl_prop[inst].backlight_lut[index] =
+						base_value + (step / delta_luma);
+				step += delta_signal;
+			}
+		}
+	/* Build backlight translation table based on default curve */
+	} else {
+		/* Defines default backlight curve F(x) = A(x*x) + Bx + C.
+		 *
+		 * Backlight curve should always  satisfy:
+		 * F(0) = min, F(100) = max,
+		 * So polynom coefficients are:
+		 * A is 0.0255 - B/100 - min/10000 - (255-max)/10000 =
+		 * (max - min)/10000 - B/100
+		 * B is adjustable factor to modify the curve.
+		 * Bigger B results in less concave curve.
+		 * B range is [0..(max-min)/100]
+		 * C is backlight minimum
+		 */
+		unsigned int backlight_curve_coeff_a_factor =
+				num_levels * num_levels;
+		unsigned int backlight_curve_coeff_b = num_levels;
+		unsigned int delta =
+			core_power->bl_prop[inst].backlight_lut[num_levels - 1] -
+				core_power->bl_prop[inst].backlight_lut[0];
+		unsigned int coeffC = core_power->bl_prop[inst].backlight_lut[0];
+		unsigned int coeffB =
+				(backlight_curve_coeff_b < delta ?
+					backlight_curve_coeff_b : delta);
+		unsigned long long coeffA = delta - coeffB; /* coeffB is B*100 */
+
+		for (i = 1; i < num_levels - 1; i++) {
+			uint64_t lut_val = div_u64(coeffA * i * i, backlight_curve_coeff_a_factor) +
+				div_u64((uint64_t)coeffB * i, backlight_curve_coeff_b) + coeffC;
+
+			ASSERT(lut_val <= 0xFFFFFFFF);
+			core_power->bl_prop[inst].backlight_lut[i] = (unsigned int)lut_val;
+		}
+	}
+
+	if (ext_backlight_caps != NULL)
+		kfree(ext_backlight_caps);
+
+	/* Successfully initialized */
+	core_power->bl_prop[inst].backlight_caps_valid = true;
+}
+
+static void varibright_set_level(struct core_power *core_power)
+{
+	if (!core_power->varibright_prop.varibright_active ||
+		!core_power->varibright_prop.varibright_user_enable)
+		core_power->varibright_prop.varibright_hw_level = 0;
+	else
+		core_power->varibright_prop.varibright_hw_level =
+			core_power->varibright_prop.varibright_level;
+}
+
+bool mod_power_hw_init_backlight(struct mod_power *mod_power)
+{
+	struct core_power *core_power = NULL;
+	struct dc *dc = NULL;
+	struct dmcu *dmcu = NULL;
+	struct dmcu_iram_parameters params;
+	unsigned int i;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	dc = core_power->dc;
+
+	for (i = 0; i < core_power->edp_num; i++) {
+		params.set = core_power->varibright_prop.varibright_config_setting;
+		params.backlight_ramping_override = core_power->bl_prop[i].backlight_ramping_override;
+		params.backlight_ramping_reduction = core_power->bl_prop[i].backlight_ramping_reduction;
+		params.backlight_ramping_start = core_power->bl_prop[i].backlight_ramping_start;
+		params.backlight_lut_array = core_power->bl_prop[i].backlight_lut;
+		params.backlight_lut_array_size = core_power->bl_prop[i].num_backlight_levels;
+		params.min_abm_backlight = core_power->bl_prop[i].min_abm_backlight;
+
+		dmcu = dc->res_pool->dmcu;
+
+		// In the case where abm is implemented on dmcub,
+		// dmcu object will be null.
+		// ABM 2.4 and up are implemented on dmcub.
+		if (dmcu) {
+			//DMCU does not support multiple eDP
+			return dmcu_load_iram(dmcu, params);
+		} else if (dc->ctx->dmub_srv) {
+			if (!dmub_init_abm_config(dc->res_pool, params, i))
+				return false;
+		} else
+			return false;
+	}
+	return true;
+}
+
+void mod_power_update_backlight_on_mode_change(
+    struct core_power *core_power,
+    struct dc_link *link,
+    unsigned int panel_inst,
+    uint8_t aux_inst,
+    bool is_hdr)
+{
+    struct set_backlight_level_params backlight_level_params = { 0 };
+
+		if (link->dpcd_sink_ext_caps.bits.hdr_aux_backlight_control == 1 ||
+			link->dpcd_sink_ext_caps.bits.sdr_aux_backlight_control == 1)
+			dc_link_set_backlight_level_nits(link, core_power->bl_state[panel_inst].isHDR,
+				core_power->bl_state[panel_inst].backlight_millinit, 0);
+
+		backlight_level_params.frame_ramp = 0;
+
+		fill_backlight_level_params(core_power, &backlight_level_params, panel_inst, aux_inst,
+			core_power->bl_state[panel_inst].backlight_pwm, link->backlight_control_type,
+			core_power->bl_state[panel_inst].backlight_millinit, 0, is_hdr);
+
+		dc_link_set_backlight_level(link, &backlight_level_params);
+}
+
+static bool set_backlight_millinits_aux(struct core_power *core_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millinits,
+		unsigned int transition_time_millisec,
+		unsigned int inst)
+{
+	struct dc_link *link = NULL;
+
+	if (core_power == NULL)
+		return false;
+
+	if (stream == NULL)
+		return true;
+
+	link = dc_stream_get_link(stream);
+
+	return dc_link_set_backlight_level_nits(link, core_power->bl_state[inst].isHDR,
+			backlight_millinits, transition_time_millisec);
+}
+
+static bool set_backlight(struct core_power *core_power,
+		struct dc_stream_state *stream,
+		struct set_backlight_level_params *backlight_level_params,
+		unsigned int inst)
+{
+	bool retv = false;
+	unsigned int frame_ramp = 0;
+	unsigned int vsync_rate_hz;
+	union dmcu_abm_set_bl_params params;
+	const struct dc_link *link = NULL;
+	unsigned int backlight_pwm_u16_16 = backlight_level_params->backlight_pwm_u16_16;
+	unsigned int transition_time_millisec = backlight_level_params->transition_time_in_ms;
+
+	if (core_power == NULL)
+		return false;
+
+	core_power->bl_state[inst].backlight_pwm = backlight_pwm_u16_16;
+
+	if (stream == NULL)
+		return true;
+
+	if (stream->link->connector_signal != SIGNAL_TYPE_EDP)
+		return false;
+
+	if (transition_time_millisec != 0) {
+		unsigned int v_total =
+			(stream->adjust.v_total_max == 0) ? stream->timing.v_total : stream->adjust.v_total_max;
+
+		vsync_rate_hz = (unsigned int)div_u64(div_u64((stream->
+			timing.pix_clk_100hz * 100),
+			v_total),
+			stream->timing.h_total);
+
+		if (core_power->bl_state[inst].smooth_brightness_enabled)
+			frame_ramp = ((vsync_rate_hz *
+				transition_time_millisec) + 500) / 1000;
+	}
+
+	core_power->bl_state[inst].frame_ramp = frame_ramp;
+	params.u32All = 0;
+	params.bits.gradual_change = (frame_ramp > 0);
+	params.bits.frame_ramp = frame_ramp;
+	link = dc_stream_get_link(stream);
+
+	mod_power_set_psr_event(&core_power->mod_public, stream, true, psr_event_hw_programming, true);
+	mod_power_set_replay_event(&core_power->mod_public, stream, true, replay_event_hw_programming, true);
+
+	backlight_level_params->frame_ramp = params.u32All;
+	retv = dc_link_set_backlight_level(link, backlight_level_params);
+
+	mod_power_set_psr_event(&core_power->mod_public, stream, false, psr_event_hw_programming, false);
+	mod_power_set_replay_event(&core_power->mod_public, stream, false, replay_event_hw_programming, false);
+
+	return retv;
+}
+
+void fill_backlight_level_params(struct core_power *core_power,
+	struct set_backlight_level_params *backlight_level_params,
+	int panel_inst, uint8_t aux_inst, unsigned int backlight_pwm,
+	enum backlight_control_type backlight_control_type,
+	unsigned int backlight_millinit, unsigned int transition_time_millisec,
+	bool is_hdr)
+{
+	struct pwr_backlight_properties *bl_prop = &core_power->bl_prop[panel_inst];
+
+	backlight_level_params->aux_inst = aux_inst;
+	backlight_level_params->backlight_pwm_u16_16 = backlight_pwm;
+	backlight_level_params->control_type = backlight_control_type;
+	backlight_level_params->backlight_millinits = backlight_millinit;
+	backlight_level_params->transition_time_in_ms = transition_time_millisec;
+	backlight_level_params->min_luminance = bl_prop->min_brightness_millinits;
+	backlight_level_params->max_luminance = bl_prop->max_brightness_millinits;
+	backlight_level_params->min_backlight_pwm = bl_prop->min_backlight_pwm;
+	backlight_level_params->max_backlight_pwm = bl_prop->max_backlight_pwm;
+
+	if (backlight_control_type == BACKLIGHT_CONTROL_AMD_AUX && !is_hdr)
+		backlight_level_params->control_type = BACKLIGHT_CONTROL_PWM;
+}
+
+bool mod_power_set_backlight_nits(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millinit,
+		unsigned int transition_time_millisec,
+		bool skip_aux,
+		bool is_hdr)
+{
+	struct core_power *core_power = NULL;
+	unsigned int backlight_pwm;
+	unsigned int panel_inst = 0;
+	struct set_backlight_level_params backlight_level_params = { 0 };
+	const struct dc_link *link = NULL;
+	uint8_t aux_inst = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	link = dc_stream_get_link(stream);
+
+	ASSERT(link->ddc->ddc_pin->hw_info.ddc_channel <= 0xFF);
+	aux_inst = (uint8_t)link->ddc->ddc_pin->hw_info.ddc_channel;
+
+	if (!dc_get_edp_link_panel_inst(core_power->dc, stream->link, &panel_inst))
+		return false;
+
+	if (!skip_aux) {
+		if (!set_backlight_millinits_aux(core_power, stream,
+						backlight_millinit, transition_time_millisec, panel_inst))
+			return false;
+	}
+// always send both AUX (above) and PWM (below)
+	core_power->bl_state[panel_inst].backlight_millinit = backlight_millinit;
+
+	core_power->bl_state[panel_inst].backlight_millipercent =
+		backlight_millinit_to_millipercent(
+				core_power, backlight_millinit, panel_inst);
+
+	backlight_pwm = backlight_millinit_to_pwm(
+				core_power, backlight_millinit, panel_inst);
+
+	fill_backlight_level_params(core_power, &backlight_level_params, panel_inst, aux_inst, backlight_pwm,
+		link->backlight_control_type, backlight_millinit, transition_time_millisec, is_hdr);
+
+	return set_backlight(core_power, stream,
+			&backlight_level_params, panel_inst);
+}
+
+bool mod_power_backlight_percent_to_nits(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millipercent,
+		unsigned int *backlight_millinit)
+{
+	struct core_power *core_power = NULL;
+	unsigned int inst = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (!dc_get_edp_link_panel_inst(core_power->dc, stream->link, &inst))
+		return false;
+
+	*backlight_millinit = backlight_millipercent_to_millinit(
+			core_power, backlight_millipercent, inst);
+	return true;
+}
+
+bool mod_power_backlight_nits_to_percent(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millinit,
+		unsigned int *backlight_millipercent)
+{
+	struct core_power *core_power = NULL;
+	unsigned int inst = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (!dc_get_edp_link_panel_inst(core_power->dc, stream->link, &inst))
+		return false;
+
+	*backlight_millipercent = backlight_millinit_to_millipercent(
+			core_power, backlight_millinit, inst);
+	return true;
+}
+
+bool mod_power_set_backlight_percent(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millipercent,
+		unsigned int transition_time_millisec,
+		bool is_hdr)
+{
+	struct core_power *core_power = NULL;
+	struct set_backlight_level_params backlight_level_params = { 0 };
+	const struct dc_link *link = NULL;
+	unsigned int backlight_pwm;
+	unsigned int panel_inst = 0;
+	uint8_t aux_inst = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	link = dc_stream_get_link(stream);
+	ASSERT(link->ddc->ddc_pin->hw_info.ddc_channel <= 0xFF);
+	aux_inst = (uint8_t)link->ddc->ddc_pin->hw_info.ddc_channel;
+
+	if (!dc_get_edp_link_panel_inst(core_power->dc, stream->link, &panel_inst))
+		return false;
+	core_power->bl_state[panel_inst].backlight_millipercent = backlight_millipercent;
+
+	core_power->bl_state[panel_inst].backlight_millinit =
+		backlight_millipercent_to_millinit(
+				core_power, backlight_millipercent, panel_inst);
+
+	backlight_pwm = backlight_millipercent_to_pwm(
+				core_power, backlight_millipercent, panel_inst);
+
+	fill_backlight_level_params(core_power, &backlight_level_params, panel_inst,
+		aux_inst, backlight_pwm, link->backlight_control_type,
+		core_power->bl_state[panel_inst].backlight_millinit, transition_time_millisec, is_hdr);
+
+	return set_backlight(core_power, stream,
+			&backlight_level_params, panel_inst);
+}
+
+void mod_power_update_backlight(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millipercent)
+{
+	struct core_power *core_power = NULL;
+	unsigned int inst = 0;
+
+	if (mod_power == NULL)
+		return;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (!dc_get_edp_link_panel_inst(core_power->dc, stream->link, &inst))
+		return;
+	core_power->bl_state[inst].backlight_millipercent = backlight_millipercent;
+
+	core_power->bl_state[inst].backlight_millinit =
+		backlight_millipercent_to_millinit(
+			core_power, backlight_millipercent, inst);
+
+	core_power->bl_state[inst].backlight_pwm = backlight_millipercent_to_pwm(
+		core_power, backlight_millipercent, inst);
+}
+
+void mod_power_update_backlight_nits(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		unsigned int backlight_millinit)
+{
+	struct core_power *core_power = NULL;
+	unsigned int inst = 0;
+
+	if (mod_power == NULL)
+		return;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (!dc_get_edp_link_panel_inst(core_power->dc, stream->link, &inst))
+		return;
+
+	core_power->bl_state[inst].backlight_millinit = backlight_millinit;
+
+	core_power->bl_state[inst].backlight_millipercent = backlight_millinit_to_millipercent(
+		core_power, backlight_millinit, inst);
+	core_power->bl_state[inst].backlight_pwm = backlight_millinit_to_pwm(
+		core_power, backlight_millinit, inst);
+}
+
+bool mod_power_get_backlight_pwm(struct mod_power *mod_power,
+		unsigned int *backlight_pwm,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	*backlight_pwm = core_power->bl_state[inst].backlight_pwm;
+
+	return true;
+}
+
+bool mod_power_get_backlight_nits(struct mod_power *mod_power,
+		unsigned int *backlight_millinit,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	*backlight_millinit = core_power->bl_state[inst].backlight_millinit;
+
+	return true;
+}
+
+bool mod_power_get_backlight_percent(struct mod_power *mod_power,
+		unsigned int *backlight_millipercent,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	*backlight_millipercent = core_power->bl_state[inst].backlight_millipercent;
+
+	return true;
+}
+
+bool mod_power_get_hw_target_backlight_pwm_nits(struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_millinit,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+	unsigned int backlight_u16_16 = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (mod_power_get_hw_target_backlight_pwm(mod_power, link,
+							&backlight_u16_16)) {
+		*backlight_millinit =
+			backlight_pwm_to_millinit(core_power,
+					backlight_u16_16, inst);
+		return true;
+	}
+	return false;
+}
+
+bool mod_power_get_hw_target_backlight_pwm_percent(struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_millipercent,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+	unsigned int backlight_u16_16 = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (mod_power_get_hw_target_backlight_pwm(mod_power, link,
+							&backlight_u16_16)) {
+		*backlight_millipercent =
+			backlight_pwm_to_millipercent(core_power,
+					backlight_u16_16, inst);
+		return true;
+	}
+	return false;
+}
+
+bool mod_power_get_hw_target_backlight_pwm(struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_u16_16)
+{
+	if (mod_power == NULL)
+		return false;
+
+	*backlight_u16_16 = dc_link_get_target_backlight_pwm(link);
+
+	return true;
+}
+
+bool mod_power_get_hw_backlight_pwm_nits(struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_millinit,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+	unsigned int backlight_u16_16 = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (mod_power_get_hw_backlight_pwm(mod_power, link, &backlight_u16_16)) {
+		*backlight_millinit =
+			backlight_pwm_to_millinit(core_power,
+					backlight_u16_16, inst);
+		return true;
+	}
+	return false;
+}
+
+bool mod_power_get_hw_backlight_aux_nits(struct mod_power *mod_power,
+		struct dc_stream_state **streams, int num_streams,
+		unsigned int *backlight_millinit_avg,
+		unsigned int *backlight_millinit_peak)
+{
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+	int stream_index;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power == NULL)
+		return false;
+
+	if (num_streams < 1)
+		return true;
+
+	for (stream_index = 0; stream_index < num_streams; stream_index++)
+		if (streams[stream_index]->link->connector_signal == SIGNAL_TYPE_EDP ||
+				streams[stream_index]->link->connector_signal == SIGNAL_TYPE_DISPLAY_PORT)
+			break;
+
+	if (stream_index == num_streams)
+		return false;
+
+	link = dc_stream_get_link(streams[stream_index]);
+	if (link->dpcd_sink_ext_caps.bits.hdr_aux_backlight_control == 0)
+		return false;
+
+	return dc_link_get_backlight_level_nits(link, backlight_millinit_avg,
+			backlight_millinit_peak);
+}
+
+bool mod_power_get_hw_backlight_pwm_percent(struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_millipercent,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+	unsigned int backlight_u16_16 = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (mod_power_get_hw_backlight_pwm(mod_power, link, &backlight_u16_16)) {
+		*backlight_millipercent =
+			backlight_pwm_to_millipercent(core_power,
+					backlight_u16_16, inst);
+		return true;
+	}
+	return false;
+}
+
+bool mod_power_get_hw_backlight_pwm(struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int *backlight_u16_16)
+{
+	if (mod_power == NULL)
+		return false;
+
+	*backlight_u16_16 = dc_link_get_backlight_level(link);
+
+	return true;
+}
+
+bool mod_power_get_panel_backlight_boundaries(
+				struct mod_power *mod_power,
+				unsigned int *out_min_backlight,
+				unsigned int *out_max_backlight,
+				unsigned int *out_ac_backlight_percent,
+				unsigned int *out_dc_backlight_percent,
+				unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	/* If cache was successfully updated,
+	 * copy the values to output structure and return success
+	 */
+	if (core_power->bl_prop[inst].backlight_caps_valid) {
+		*out_min_backlight = core_power->bl_prop[inst].backlight_lut[0];
+		*out_max_backlight =
+			core_power->bl_prop[inst].backlight_lut[
+				core_power->bl_prop[inst].num_backlight_levels - 1];
+		*out_ac_backlight_percent =
+			core_power->bl_prop[inst].ac_backlight_percent;
+		*out_dc_backlight_percent =
+			core_power->bl_prop[inst].dc_backlight_percent;
+
+		return true;
+	}
+
+	return false;
+}
+
+bool mod_power_set_smooth_brightness(struct mod_power *mod_power,
+		bool enable_brightness,
+		unsigned int inst)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	core_power->bl_state[inst].smooth_brightness_enabled = enable_brightness;
+
+	return true;
+}
+
+bool mod_power_varibright_feature_enable(struct mod_power *mod_power, bool enable,
+		struct dc_stream_update *stream_update)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	core_power->varibright_prop.varibright_user_enable = enable;
+
+	/* find abm hw level to program, and save in stream update */
+	varibright_set_level(core_power);
+	*stream_update->abm_level = core_power->varibright_prop.varibright_hw_level;
+
+	DC_TRACE_LEVEL_MESSAGEP(DAL_TRACE_LEVEL_INFORMATION,
+						WPP_BIT_FLAG_Backlight_ABM,
+						">ABM feature enable: enable=%u su->varibright_level=%u varibright_hw_level=%u",
+						(unsigned int) enable,
+						*stream_update->abm_level,
+						core_power->varibright_prop.varibright_hw_level);
+	return true;
+}
+
+bool mod_power_varibright_activate(struct mod_power *mod_power,
+		bool activate,
+		struct dc_stream_update *stream_update)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	core_power->varibright_prop.varibright_active = activate;
+
+	/* find abm hw level to program, and save in stream update */
+	varibright_set_level(core_power);
+	*stream_update->abm_level = core_power->varibright_prop.varibright_hw_level;
+
+	DC_TRACE_LEVEL_MESSAGEP(DAL_TRACE_LEVEL_INFORMATION,
+						WPP_BIT_FLAG_Backlight_ABM,
+						">ABM activate: activate=%u su->varibright_level=%u",
+						(unsigned int) activate,
+						*stream_update->abm_level);
+	return true;
+}
+bool mod_power_varibright_set_level(struct mod_power *mod_power, unsigned int level,
+		struct dc_stream_update *stream_update)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	core_power->varibright_prop.varibright_level = level;
+	core_power->varibright_prop.varibright_hw_level = level;
+
+	/* find abm hw level to program, and save in stream update */
+	varibright_set_level(core_power);
+	*stream_update->abm_level = core_power->varibright_prop.varibright_hw_level;
+
+	DC_TRACE_LEVEL_MESSAGEP(DAL_TRACE_LEVEL_INFORMATION,
+						WPP_BIT_FLAG_Backlight_ABM,
+						">ABM set level: level=%u -> (varibright_level=%u varibright_hw_level=%u) -> su->varibright_level=%u",
+						level,
+						core_power->varibright_prop.varibright_level,
+						core_power->varibright_prop.varibright_hw_level,
+						*stream_update->abm_level);
+	return true;
+}
+
+bool mod_power_varibright_set_hw_level(struct mod_power *mod_power, unsigned int level,
+		struct dc_stream_update *stream_update)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (level == 0 || level == ABM_LEVEL_IMMEDIATE_DISABLE)
+		core_power->varibright_prop.varibright_active = 0;
+	else
+		core_power->varibright_prop.varibright_active = 1;
+	core_power->varibright_prop.varibright_hw_level = level;
+	*stream_update->abm_level = core_power->varibright_prop.varibright_hw_level;
+
+	DC_TRACE_LEVEL_MESSAGEP(DAL_TRACE_LEVEL_INFORMATION,
+						WPP_BIT_FLAG_Backlight_ABM,
+						">ABM set level: level=%u -> (varibright_level=%u varibright_hw_level=%u) -> su->varibright_level=%u",
+						level,
+						core_power->varibright_prop.varibright_level,
+						core_power->varibright_prop.varibright_hw_level,
+						*stream_update->abm_level);
+	return true;
+}
+
+bool mod_power_get_varibright_level(struct mod_power *mod_power,
+		unsigned int *varibright_level)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	*varibright_level = core_power->varibright_prop.varibright_level;
+
+	DC_TRACE_LEVEL_MESSAGEP(DAL_TRACE_LEVEL_INFORMATION,
+						WPP_BIT_FLAG_Backlight_ABM,
+						">get varibright level: cp->varibright_level=%u",
+						*varibright_level);
+	return true;
+
+}
+
+bool mod_power_get_varibright_hw_level(struct mod_power *mod_power,
+		unsigned int *varibright_level)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	*varibright_level = core_power->varibright_prop.varibright_hw_level;
+	DC_TRACE_LEVEL_MESSAGEP(DAL_TRACE_LEVEL_INFORMATION,
+						WPP_BIT_FLAG_Backlight_ABM,
+						">get varibright HW level: hw_level=%u",
+						*varibright_level);
+	return true;
+}
+
+bool mod_power_get_varibright_default_level(struct mod_power *mod_power,
+		unsigned int *varibright_level)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	*varibright_level = core_power->varibright_prop.def_varibright_level;
+	DC_TRACE_LEVEL_MESSAGEP(DAL_TRACE_LEVEL_INFORMATION,
+						WPP_BIT_FLAG_Backlight_ABM,
+						">get varibright default level: def_varibright_level=%u",
+						*varibright_level);
+	return true;
+}
+
+bool mod_power_get_varibright_enable(struct mod_power *mod_power,
+		bool *varibright_enable)
+{
+	struct core_power *core_power = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	*varibright_enable = core_power->varibright_prop.varibright_user_enable;
+	DC_TRACE_LEVEL_MESSAGEP(DAL_TRACE_LEVEL_INFORMATION,
+				WPP_BIT_FLAG_Backlight_ABM,
+				">get varibright enable state: varibright_user_enable=%u",
+				(unsigned int) (*varibright_enable));
+	return true;
+}
+
+bool mod_power_is_abm_active(struct mod_power *mod_power,
+		const struct dc_link *link,
+		unsigned int inst)
+{
+	unsigned int user_backlight = 0;
+	unsigned int current_backlight = 0;
+	bool is_active = false;
+
+	if (mod_power == NULL)
+		return false;
+
+	mod_power_get_backlight_pwm(mod_power, &user_backlight, inst);
+	mod_power_get_hw_backlight_pwm(mod_power, link,	&current_backlight);
+
+	if (user_backlight != current_backlight)
+		is_active = true;
+	else
+		is_active = false;
+	DC_TRACE_LEVEL_MESSAGEP(DAL_TRACE_LEVEL_INFORMATION,
+						WPP_BIT_FLAG_Backlight_ABM,
+						">get ABM active state: is_active=%u (user_backlight_pwm=%u, current_backlight_pwm=%u)",
+						(unsigned int)is_active,
+						user_backlight,
+						current_backlight);
+	return is_active;
+}
+
+static void fill_backlight_transform_table(struct dmcu_iram_parameters params,
+		struct iram_table_v_2 *table)
+{
+	unsigned int i;
+	unsigned int num_entries = NUM_BL_CURVE_SEGS;
+	unsigned int lut_index;
+
+	table->backlight_thresholds[0] = 0;
+	ASSERT(params.backlight_lut_array[0] <= 0xFFFF);
+	table->backlight_offsets[0] = (uint16_t)params.backlight_lut_array[0];
+	table->backlight_thresholds[num_entries-1] = 0xFFFF;
+	ASSERT(params.backlight_lut_array[params.backlight_lut_array_size - 1] <= 0xFFFF);
+	table->backlight_offsets[num_entries-1] =
+		(uint16_t)params.backlight_lut_array[params.backlight_lut_array_size - 1];
+
+	/* Setup all brightness levels between 0% and 100% exclusive
+	 * Fills brightness-to-backlight transform table. Backlight custom curve
+	 * describes transform from brightness to backlight. It will be defined
+	 * as set of thresholds and set of offsets, together, implying
+	 * extrapolation of custom curve into 16 uniformly spanned linear
+	 * segments.  Each threshold/offset represented by 16 bit entry in
+	 * format U4.10.
+	 */
+	for (i = 1; i+1 < num_entries; i++) {
+		lut_index = (params.backlight_lut_array_size - 1) * i / (num_entries - 1);
+
+		ASSERT(lut_index < params.backlight_lut_array_size);
+
+		unsigned int threshold_val = DIV_ROUNDUP((i * 65536), num_entries);
+		unsigned int offset_val = params.backlight_lut_array[lut_index];
+
+		ASSERT(threshold_val <= 0xFFFF);
+		ASSERT(offset_val <= 0xFFFF);
+
+		table->backlight_thresholds[i] = cpu_to_be16((uint16_t)threshold_val);
+		table->backlight_offsets[i]    = cpu_to_be16((uint16_t)offset_val);
+	}
+}
+
+static void fill_backlight_transform_table_v_2_2(struct dmcu_iram_parameters params,
+		struct iram_table_v_2_2 *table, bool big_endian)
+{
+	unsigned int i;
+	unsigned int num_entries = NUM_BL_CURVE_SEGS;
+	unsigned int lut_index;
+
+	table->backlight_thresholds[0] = 0;
+	ASSERT(params.backlight_lut_array[0] <= 0xFFFF);
+	table->backlight_offsets[0] = (uint16_t)params.backlight_lut_array[0];
+	table->backlight_thresholds[num_entries-1] = 0xFFFF;
+	ASSERT(params.backlight_lut_array[params.backlight_lut_array_size - 1] <= 0xFFFF);
+	table->backlight_offsets[num_entries-1] =
+		(uint16_t)params.backlight_lut_array[params.backlight_lut_array_size - 1];
+
+	/* Setup all brightness levels between 0% and 100% exclusive
+	 * Fills brightness-to-backlight transform table. Backlight custom curve
+	 * describes transform from brightness to backlight. It will be defined
+	 * as set of thresholds and set of offsets, together, implying
+	 * extrapolation of custom curve into 16 uniformly spanned linear
+	 * segments.  Each threshold/offset represented by 16 bit entry in
+	 * format U4.10.
+	 */
+	for (i = 1; i+1 < num_entries; i++) {
+		lut_index = DIV_ROUNDUP((i * params.backlight_lut_array_size), num_entries);
+		ASSERT(lut_index < params.backlight_lut_array_size);
+
+		unsigned int threshold_val = DIV_ROUNDUP((i * 65536), num_entries);
+		unsigned int offset_val = params.backlight_lut_array[lut_index];
+
+		ASSERT(threshold_val <= 0xFFFF);
+		ASSERT(offset_val <= 0xFFFF);
+
+		table->backlight_thresholds[i] = (big_endian) ?
+			cpu_to_be16((uint16_t)threshold_val) : cpu_to_le16((uint16_t)threshold_val);
+		table->backlight_offsets[i] = (big_endian) ?
+			cpu_to_be16((uint16_t)offset_val) : cpu_to_le16((uint16_t)offset_val);
+	}
+}
+
+static void fill_iram_v_2(struct iram_table_v_2 *ram_table, struct dmcu_iram_parameters params)
+{
+	unsigned int set = params.set;
+
+	ram_table->min_abm_backlight =
+			cpu_to_be16(params.min_abm_backlight);
+	ram_table->deviation_gain = 0xb3;
+
+	ram_table->blRampReduction =
+		cpu_to_be16(params.backlight_ramping_reduction);
+	ram_table->blRampStart =
+		cpu_to_be16(params.backlight_ramping_start);
+
+	ram_table->min_reduction[0][0] = min_reduction_table[abm_config[set][0]];
+	ram_table->min_reduction[1][0] = min_reduction_table[abm_config[set][0]];
+	ram_table->min_reduction[2][0] = min_reduction_table[abm_config[set][0]];
+	ram_table->min_reduction[3][0] = min_reduction_table[abm_config[set][0]];
+	ram_table->min_reduction[4][0] = min_reduction_table[abm_config[set][0]];
+	ram_table->max_reduction[0][0] = max_reduction_table[abm_config[set][0]];
+	ram_table->max_reduction[1][0] = max_reduction_table[abm_config[set][0]];
+	ram_table->max_reduction[2][0] = max_reduction_table[abm_config[set][0]];
+	ram_table->max_reduction[3][0] = max_reduction_table[abm_config[set][0]];
+	ram_table->max_reduction[4][0] = max_reduction_table[abm_config[set][0]];
+
+	ram_table->min_reduction[0][1] = min_reduction_table[abm_config[set][1]];
+	ram_table->min_reduction[1][1] = min_reduction_table[abm_config[set][1]];
+	ram_table->min_reduction[2][1] = min_reduction_table[abm_config[set][1]];
+	ram_table->min_reduction[3][1] = min_reduction_table[abm_config[set][1]];
+	ram_table->min_reduction[4][1] = min_reduction_table[abm_config[set][1]];
+	ram_table->max_reduction[0][1] = max_reduction_table[abm_config[set][1]];
+	ram_table->max_reduction[1][1] = max_reduction_table[abm_config[set][1]];
+	ram_table->max_reduction[2][1] = max_reduction_table[abm_config[set][1]];
+	ram_table->max_reduction[3][1] = max_reduction_table[abm_config[set][1]];
+	ram_table->max_reduction[4][1] = max_reduction_table[abm_config[set][1]];
+
+	ram_table->min_reduction[0][2] = min_reduction_table[abm_config[set][2]];
+	ram_table->min_reduction[1][2] = min_reduction_table[abm_config[set][2]];
+	ram_table->min_reduction[2][2] = min_reduction_table[abm_config[set][2]];
+	ram_table->min_reduction[3][2] = min_reduction_table[abm_config[set][2]];
+	ram_table->min_reduction[4][2] = min_reduction_table[abm_config[set][2]];
+	ram_table->max_reduction[0][2] = max_reduction_table[abm_config[set][2]];
+	ram_table->max_reduction[1][2] = max_reduction_table[abm_config[set][2]];
+	ram_table->max_reduction[2][2] = max_reduction_table[abm_config[set][2]];
+	ram_table->max_reduction[3][2] = max_reduction_table[abm_config[set][2]];
+	ram_table->max_reduction[4][2] = max_reduction_table[abm_config[set][2]];
+
+	ram_table->min_reduction[0][3] = min_reduction_table[abm_config[set][3]];
+	ram_table->min_reduction[1][3] = min_reduction_table[abm_config[set][3]];
+	ram_table->min_reduction[2][3] = min_reduction_table[abm_config[set][3]];
+	ram_table->min_reduction[3][3] = min_reduction_table[abm_config[set][3]];
+	ram_table->min_reduction[4][3] = min_reduction_table[abm_config[set][3]];
+	ram_table->max_reduction[0][3] = max_reduction_table[abm_config[set][3]];
+	ram_table->max_reduction[1][3] = max_reduction_table[abm_config[set][3]];
+	ram_table->max_reduction[2][3] = max_reduction_table[abm_config[set][3]];
+	ram_table->max_reduction[3][3] = max_reduction_table[abm_config[set][3]];
+	ram_table->max_reduction[4][3] = max_reduction_table[abm_config[set][3]];
+
+	ram_table->bright_pos_gain[0][0] = 0x20;
+	ram_table->bright_pos_gain[0][1] = 0x20;
+	ram_table->bright_pos_gain[0][2] = 0x20;
+	ram_table->bright_pos_gain[0][3] = 0x20;
+	ram_table->bright_pos_gain[1][0] = 0x20;
+	ram_table->bright_pos_gain[1][1] = 0x20;
+	ram_table->bright_pos_gain[1][2] = 0x20;
+	ram_table->bright_pos_gain[1][3] = 0x20;
+	ram_table->bright_pos_gain[2][0] = 0x20;
+	ram_table->bright_pos_gain[2][1] = 0x20;
+	ram_table->bright_pos_gain[2][2] = 0x20;
+	ram_table->bright_pos_gain[2][3] = 0x20;
+	ram_table->bright_pos_gain[3][0] = 0x20;
+	ram_table->bright_pos_gain[3][1] = 0x20;
+	ram_table->bright_pos_gain[3][2] = 0x20;
+	ram_table->bright_pos_gain[3][3] = 0x20;
+	ram_table->bright_pos_gain[4][0] = 0x20;
+	ram_table->bright_pos_gain[4][1] = 0x20;
+	ram_table->bright_pos_gain[4][2] = 0x20;
+	ram_table->bright_pos_gain[4][3] = 0x20;
+	ram_table->bright_neg_gain[0][0] = 0x00;
+	ram_table->bright_neg_gain[0][1] = 0x00;
+	ram_table->bright_neg_gain[0][2] = 0x00;
+	ram_table->bright_neg_gain[0][3] = 0x00;
+	ram_table->bright_neg_gain[1][0] = 0x00;
+	ram_table->bright_neg_gain[1][1] = 0x00;
+	ram_table->bright_neg_gain[1][2] = 0x00;
+	ram_table->bright_neg_gain[1][3] = 0x00;
+	ram_table->bright_neg_gain[2][0] = 0x00;
+	ram_table->bright_neg_gain[2][1] = 0x00;
+	ram_table->bright_neg_gain[2][2] = 0x00;
+	ram_table->bright_neg_gain[2][3] = 0x00;
+	ram_table->bright_neg_gain[3][0] = 0x00;
+	ram_table->bright_neg_gain[3][1] = 0x00;
+	ram_table->bright_neg_gain[3][2] = 0x00;
+	ram_table->bright_neg_gain[3][3] = 0x00;
+	ram_table->bright_neg_gain[4][0] = 0x00;
+	ram_table->bright_neg_gain[4][1] = 0x00;
+	ram_table->bright_neg_gain[4][2] = 0x00;
+	ram_table->bright_neg_gain[4][3] = 0x00;
+	ram_table->dark_pos_gain[0][0] = 0x00;
+	ram_table->dark_pos_gain[0][1] = 0x00;
+	ram_table->dark_pos_gain[0][2] = 0x00;
+	ram_table->dark_pos_gain[0][3] = 0x00;
+	ram_table->dark_pos_gain[1][0] = 0x00;
+	ram_table->dark_pos_gain[1][1] = 0x00;
+	ram_table->dark_pos_gain[1][2] = 0x00;
+	ram_table->dark_pos_gain[1][3] = 0x00;
+	ram_table->dark_pos_gain[2][0] = 0x00;
+	ram_table->dark_pos_gain[2][1] = 0x00;
+	ram_table->dark_pos_gain[2][2] = 0x00;
+	ram_table->dark_pos_gain[2][3] = 0x00;
+	ram_table->dark_pos_gain[3][0] = 0x00;
+	ram_table->dark_pos_gain[3][1] = 0x00;
+	ram_table->dark_pos_gain[3][2] = 0x00;
+	ram_table->dark_pos_gain[3][3] = 0x00;
+	ram_table->dark_pos_gain[4][0] = 0x00;
+	ram_table->dark_pos_gain[4][1] = 0x00;
+	ram_table->dark_pos_gain[4][2] = 0x00;
+	ram_table->dark_pos_gain[4][3] = 0x00;
+	ram_table->dark_neg_gain[0][0] = 0x00;
+	ram_table->dark_neg_gain[0][1] = 0x00;
+	ram_table->dark_neg_gain[0][2] = 0x00;
+	ram_table->dark_neg_gain[0][3] = 0x00;
+	ram_table->dark_neg_gain[1][0] = 0x00;
+	ram_table->dark_neg_gain[1][1] = 0x00;
+	ram_table->dark_neg_gain[1][2] = 0x00;
+	ram_table->dark_neg_gain[1][3] = 0x00;
+	ram_table->dark_neg_gain[2][0] = 0x00;
+	ram_table->dark_neg_gain[2][1] = 0x00;
+	ram_table->dark_neg_gain[2][2] = 0x00;
+	ram_table->dark_neg_gain[2][3] = 0x00;
+	ram_table->dark_neg_gain[3][0] = 0x00;
+	ram_table->dark_neg_gain[3][1] = 0x00;
+	ram_table->dark_neg_gain[3][2] = 0x00;
+	ram_table->dark_neg_gain[3][3] = 0x00;
+	ram_table->dark_neg_gain[4][0] = 0x00;
+	ram_table->dark_neg_gain[4][1] = 0x00;
+	ram_table->dark_neg_gain[4][2] = 0x00;
+	ram_table->dark_neg_gain[4][3] = 0x00;
+
+	ram_table->iir_curve[0] = 0x65;
+	ram_table->iir_curve[1] = 0x65;
+	ram_table->iir_curve[2] = 0x65;
+	ram_table->iir_curve[3] = 0x65;
+	ram_table->iir_curve[4] = 0x65;
+
+	//Gamma 2.4
+	ram_table->crgb_thresh[0] = cpu_to_be16(0x13b6);
+	ram_table->crgb_thresh[1] = cpu_to_be16(0x1648);
+	ram_table->crgb_thresh[2] = cpu_to_be16(0x18e3);
+	ram_table->crgb_thresh[3] = cpu_to_be16(0x1b41);
+	ram_table->crgb_thresh[4] = cpu_to_be16(0x1d46);
+	ram_table->crgb_thresh[5] = cpu_to_be16(0x1f21);
+	ram_table->crgb_thresh[6] = cpu_to_be16(0x2167);
+	ram_table->crgb_thresh[7] = cpu_to_be16(0x2384);
+	ram_table->crgb_offset[0] = cpu_to_be16(0x2999);
+	ram_table->crgb_offset[1] = cpu_to_be16(0x3999);
+	ram_table->crgb_offset[2] = cpu_to_be16(0x4666);
+	ram_table->crgb_offset[3] = cpu_to_be16(0x5999);
+	ram_table->crgb_offset[4] = cpu_to_be16(0x6333);
+	ram_table->crgb_offset[5] = cpu_to_be16(0x7800);
+	ram_table->crgb_offset[6] = cpu_to_be16(0x8c00);
+	ram_table->crgb_offset[7] = cpu_to_be16(0xa000);
+	ram_table->crgb_slope[0]  = cpu_to_be16(0x3147);
+	ram_table->crgb_slope[1]  = cpu_to_be16(0x2978);
+	ram_table->crgb_slope[2]  = cpu_to_be16(0x23a2);
+	ram_table->crgb_slope[3]  = cpu_to_be16(0x1f55);
+	ram_table->crgb_slope[4]  = cpu_to_be16(0x1c63);
+	ram_table->crgb_slope[5]  = cpu_to_be16(0x1a0f);
+	ram_table->crgb_slope[6]  = cpu_to_be16(0x178d);
+	ram_table->crgb_slope[7]  = cpu_to_be16(0x15ab);
+
+	fill_backlight_transform_table(
+			params, ram_table);
+}
+
+static void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params)
+{
+	unsigned int set = params.set;
+
+	ram_table->flags = 0x0;
+
+	ram_table->min_abm_backlight =
+			cpu_to_be16(params.min_abm_backlight);
+
+	ram_table->deviation_gain[0] = 0xb3;
+	ram_table->deviation_gain[1] = 0xa8;
+	ram_table->deviation_gain[2] = 0x98;
+	ram_table->deviation_gain[3] = 0x68;
+
+	ram_table->min_reduction[0][0] = min_reduction_table_v_2_2[abm_config[set][0]];
+	ram_table->min_reduction[1][0] = min_reduction_table_v_2_2[abm_config[set][0]];
+	ram_table->min_reduction[2][0] = min_reduction_table_v_2_2[abm_config[set][0]];
+	ram_table->min_reduction[3][0] = min_reduction_table_v_2_2[abm_config[set][0]];
+	ram_table->min_reduction[4][0] = min_reduction_table_v_2_2[abm_config[set][0]];
+	ram_table->max_reduction[0][0] = max_reduction_table_v_2_2[abm_config[set][0]];
+	ram_table->max_reduction[1][0] = max_reduction_table_v_2_2[abm_config[set][0]];
+	ram_table->max_reduction[2][0] = max_reduction_table_v_2_2[abm_config[set][0]];
+	ram_table->max_reduction[3][0] = max_reduction_table_v_2_2[abm_config[set][0]];
+	ram_table->max_reduction[4][0] = max_reduction_table_v_2_2[abm_config[set][0]];
+
+	ram_table->min_reduction[0][1] = min_reduction_table_v_2_2[abm_config[set][1]];
+	ram_table->min_reduction[1][1] = min_reduction_table_v_2_2[abm_config[set][1]];
+	ram_table->min_reduction[2][1] = min_reduction_table_v_2_2[abm_config[set][1]];
+	ram_table->min_reduction[3][1] = min_reduction_table_v_2_2[abm_config[set][1]];
+	ram_table->min_reduction[4][1] = min_reduction_table_v_2_2[abm_config[set][1]];
+	ram_table->max_reduction[0][1] = max_reduction_table_v_2_2[abm_config[set][1]];
+	ram_table->max_reduction[1][1] = max_reduction_table_v_2_2[abm_config[set][1]];
+	ram_table->max_reduction[2][1] = max_reduction_table_v_2_2[abm_config[set][1]];
+	ram_table->max_reduction[3][1] = max_reduction_table_v_2_2[abm_config[set][1]];
+	ram_table->max_reduction[4][1] = max_reduction_table_v_2_2[abm_config[set][1]];
+
+	ram_table->min_reduction[0][2] = min_reduction_table_v_2_2[abm_config[set][2]];
+	ram_table->min_reduction[1][2] = min_reduction_table_v_2_2[abm_config[set][2]];
+	ram_table->min_reduction[2][2] = min_reduction_table_v_2_2[abm_config[set][2]];
+	ram_table->min_reduction[3][2] = min_reduction_table_v_2_2[abm_config[set][2]];
+	ram_table->min_reduction[4][2] = min_reduction_table_v_2_2[abm_config[set][2]];
+	ram_table->max_reduction[0][2] = max_reduction_table_v_2_2[abm_config[set][2]];
+	ram_table->max_reduction[1][2] = max_reduction_table_v_2_2[abm_config[set][2]];
+	ram_table->max_reduction[2][2] = max_reduction_table_v_2_2[abm_config[set][2]];
+	ram_table->max_reduction[3][2] = max_reduction_table_v_2_2[abm_config[set][2]];
+	ram_table->max_reduction[4][2] = max_reduction_table_v_2_2[abm_config[set][2]];
+
+	ram_table->min_reduction[0][3] = min_reduction_table_v_2_2[abm_config[set][3]];
+	ram_table->min_reduction[1][3] = min_reduction_table_v_2_2[abm_config[set][3]];
+	ram_table->min_reduction[2][3] = min_reduction_table_v_2_2[abm_config[set][3]];
+	ram_table->min_reduction[3][3] = min_reduction_table_v_2_2[abm_config[set][3]];
+	ram_table->min_reduction[4][3] = min_reduction_table_v_2_2[abm_config[set][3]];
+	ram_table->max_reduction[0][3] = max_reduction_table_v_2_2[abm_config[set][3]];
+	ram_table->max_reduction[1][3] = max_reduction_table_v_2_2[abm_config[set][3]];
+	ram_table->max_reduction[2][3] = max_reduction_table_v_2_2[abm_config[set][3]];
+	ram_table->max_reduction[3][3] = max_reduction_table_v_2_2[abm_config[set][3]];
+	ram_table->max_reduction[4][3] = max_reduction_table_v_2_2[abm_config[set][3]];
+
+	ram_table->bright_pos_gain[0][0] = 0x20;
+	ram_table->bright_pos_gain[0][1] = 0x20;
+	ram_table->bright_pos_gain[0][2] = 0x20;
+	ram_table->bright_pos_gain[0][3] = 0x20;
+	ram_table->bright_pos_gain[1][0] = 0x20;
+	ram_table->bright_pos_gain[1][1] = 0x20;
+	ram_table->bright_pos_gain[1][2] = 0x20;
+	ram_table->bright_pos_gain[1][3] = 0x20;
+	ram_table->bright_pos_gain[2][0] = 0x20;
+	ram_table->bright_pos_gain[2][1] = 0x20;
+	ram_table->bright_pos_gain[2][2] = 0x20;
+	ram_table->bright_pos_gain[2][3] = 0x20;
+	ram_table->bright_pos_gain[3][0] = 0x20;
+	ram_table->bright_pos_gain[3][1] = 0x20;
+	ram_table->bright_pos_gain[3][2] = 0x20;
+	ram_table->bright_pos_gain[3][3] = 0x20;
+	ram_table->bright_pos_gain[4][0] = 0x20;
+	ram_table->bright_pos_gain[4][1] = 0x20;
+	ram_table->bright_pos_gain[4][2] = 0x20;
+	ram_table->bright_pos_gain[4][3] = 0x20;
+
+	ram_table->dark_pos_gain[0][0] = 0x00;
+	ram_table->dark_pos_gain[0][1] = 0x00;
+	ram_table->dark_pos_gain[0][2] = 0x00;
+	ram_table->dark_pos_gain[0][3] = 0x00;
+	ram_table->dark_pos_gain[1][0] = 0x00;
+	ram_table->dark_pos_gain[1][1] = 0x00;
+	ram_table->dark_pos_gain[1][2] = 0x00;
+	ram_table->dark_pos_gain[1][3] = 0x00;
+	ram_table->dark_pos_gain[2][0] = 0x00;
+	ram_table->dark_pos_gain[2][1] = 0x00;
+	ram_table->dark_pos_gain[2][2] = 0x00;
+	ram_table->dark_pos_gain[2][3] = 0x00;
+	ram_table->dark_pos_gain[3][0] = 0x00;
+	ram_table->dark_pos_gain[3][1] = 0x00;
+	ram_table->dark_pos_gain[3][2] = 0x00;
+	ram_table->dark_pos_gain[3][3] = 0x00;
+	ram_table->dark_pos_gain[4][0] = 0x00;
+	ram_table->dark_pos_gain[4][1] = 0x00;
+	ram_table->dark_pos_gain[4][2] = 0x00;
+	ram_table->dark_pos_gain[4][3] = 0x00;
+
+	ram_table->hybrid_factor[0] = 0xff;
+	ram_table->hybrid_factor[1] = 0xff;
+	ram_table->hybrid_factor[2] = 0xff;
+	ram_table->hybrid_factor[3] = 0xc0;
+
+	ram_table->contrast_factor[0] = 0x99;
+	ram_table->contrast_factor[1] = 0x99;
+	ram_table->contrast_factor[2] = 0x90;
+	ram_table->contrast_factor[3] = 0x80;
+
+	ram_table->iir_curve[0] = 0x65;
+	ram_table->iir_curve[1] = 0x65;
+	ram_table->iir_curve[2] = 0x65;
+	ram_table->iir_curve[3] = 0x65;
+	ram_table->iir_curve[4] = 0x65;
+
+	//Gamma 2.2
+	ram_table->crgb_thresh[0] = cpu_to_be16(0x127c);
+	ram_table->crgb_thresh[1] = cpu_to_be16(0x151b);
+	ram_table->crgb_thresh[2] = cpu_to_be16(0x17d5);
+	ram_table->crgb_thresh[3] = cpu_to_be16(0x1a56);
+	ram_table->crgb_thresh[4] = cpu_to_be16(0x1c83);
+	ram_table->crgb_thresh[5] = cpu_to_be16(0x1e72);
+	ram_table->crgb_thresh[6] = cpu_to_be16(0x20f0);
+	ram_table->crgb_thresh[7] = cpu_to_be16(0x232b);
+	ram_table->crgb_offset[0] = cpu_to_be16(0x2999);
+	ram_table->crgb_offset[1] = cpu_to_be16(0x3999);
+	ram_table->crgb_offset[2] = cpu_to_be16(0x4666);
+	ram_table->crgb_offset[3] = cpu_to_be16(0x5999);
+	ram_table->crgb_offset[4] = cpu_to_be16(0x6333);
+	ram_table->crgb_offset[5] = cpu_to_be16(0x7800);
+	ram_table->crgb_offset[6] = cpu_to_be16(0x8c00);
+	ram_table->crgb_offset[7] = cpu_to_be16(0xa000);
+	ram_table->crgb_slope[0]  = cpu_to_be16(0x3609);
+	ram_table->crgb_slope[1]  = cpu_to_be16(0x2dfa);
+	ram_table->crgb_slope[2]  = cpu_to_be16(0x27ea);
+	ram_table->crgb_slope[3]  = cpu_to_be16(0x235d);
+	ram_table->crgb_slope[4]  = cpu_to_be16(0x2042);
+	ram_table->crgb_slope[5]  = cpu_to_be16(0x1dc3);
+	ram_table->crgb_slope[6]  = cpu_to_be16(0x1b1a);
+	ram_table->crgb_slope[7]  = cpu_to_be16(0x1910);
+
+	fill_backlight_transform_table_v_2_2(
+			params, ram_table, true);
+}
+
+static void fill_iram_v_2_3(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params, bool big_endian)
+{
+	unsigned int i, j;
+	unsigned int set = params.set;
+
+	ram_table->flags = 0x0;
+	ram_table->min_abm_backlight = (uint16_t)((big_endian) ?
+		cpu_to_be16(params.min_abm_backlight) :
+		cpu_to_le16(params.min_abm_backlight));
+
+	for (i = 0; i < NUM_AGGR_LEVEL; i++) {
+		ram_table->hybrid_factor[i] = (uint8_t)abm_settings[set][i].brightness_gain;
+		ram_table->contrast_factor[i] = abm_settings[set][i].contrast_factor;
+		ram_table->deviation_gain[i] = abm_settings[set][i].deviation_gain;
+		ram_table->min_knee[i] = abm_settings[set][i].min_knee;
+		ram_table->max_knee[i] = abm_settings[set][i].max_knee;
+
+		for (j = 0; j < NUM_AMBI_LEVEL; j++) {
+			ram_table->min_reduction[j][i] = abm_settings[set][i].min_reduction;
+			ram_table->max_reduction[j][i] = abm_settings[set][i].max_reduction;
+			ram_table->bright_pos_gain[j][i] = abm_settings[set][i].bright_pos_gain;
+			ram_table->dark_pos_gain[j][i] = abm_settings[set][i].dark_pos_gain;
+		}
+	}
+
+	ram_table->iir_curve[0] = 0x65;
+	ram_table->iir_curve[1] = 0x65;
+	ram_table->iir_curve[2] = 0x65;
+	ram_table->iir_curve[3] = 0x65;
+	ram_table->iir_curve[4] = 0x65;
+
+	//Gamma 2.2
+	ram_table->crgb_thresh[0] = bswap16_based_on_endian(big_endian, 0x127c);
+	ram_table->crgb_thresh[1] = bswap16_based_on_endian(big_endian, 0x151b);
+	ram_table->crgb_thresh[2] = bswap16_based_on_endian(big_endian, 0x17d5);
+	ram_table->crgb_thresh[3] = bswap16_based_on_endian(big_endian, 0x1a56);
+	ram_table->crgb_thresh[4] = bswap16_based_on_endian(big_endian, 0x1c83);
+	ram_table->crgb_thresh[5] = bswap16_based_on_endian(big_endian, 0x1e72);
+	ram_table->crgb_thresh[6] = bswap16_based_on_endian(big_endian, 0x20f0);
+	ram_table->crgb_thresh[7] = bswap16_based_on_endian(big_endian, 0x232b);
+	ram_table->crgb_offset[0] = bswap16_based_on_endian(big_endian, 0x2999);
+	ram_table->crgb_offset[1] = bswap16_based_on_endian(big_endian, 0x3999);
+	ram_table->crgb_offset[2] = bswap16_based_on_endian(big_endian, 0x4666);
+	ram_table->crgb_offset[3] = bswap16_based_on_endian(big_endian, 0x5999);
+	ram_table->crgb_offset[4] = bswap16_based_on_endian(big_endian, 0x6333);
+	ram_table->crgb_offset[5] = bswap16_based_on_endian(big_endian, 0x7800);
+	ram_table->crgb_offset[6] = bswap16_based_on_endian(big_endian, 0x8c00);
+	ram_table->crgb_offset[7] = bswap16_based_on_endian(big_endian, 0xa000);
+	ram_table->crgb_slope[0]  = bswap16_based_on_endian(big_endian, 0x3609);
+	ram_table->crgb_slope[1]  = bswap16_based_on_endian(big_endian, 0x2dfa);
+	ram_table->crgb_slope[2]  = bswap16_based_on_endian(big_endian, 0x27ea);
+	ram_table->crgb_slope[3]  = bswap16_based_on_endian(big_endian, 0x235d);
+	ram_table->crgb_slope[4]  = bswap16_based_on_endian(big_endian, 0x2042);
+	ram_table->crgb_slope[5]  = bswap16_based_on_endian(big_endian, 0x1dc3);
+	ram_table->crgb_slope[6]  = bswap16_based_on_endian(big_endian, 0x1b1a);
+	ram_table->crgb_slope[7]  = bswap16_based_on_endian(big_endian, 0x1910);
+
+	fill_backlight_transform_table_v_2_2(
+			params, ram_table, big_endian);
+}
+
+bool dmub_init_abm_config(struct resource_pool *res_pool,
+	struct dmcu_iram_parameters params,
+	unsigned int inst)
+{
+	struct iram_table_v_2_2 ram_table;
+	struct abm_config_table config;
+	unsigned int set = params.set;
+	bool result = false;
+	uint32_t i, j = 0;
+
+	if (res_pool->abm == NULL && res_pool->multiple_abms[inst] == NULL)
+		return false;
+
+	memset(&ram_table, 0, sizeof(ram_table));
+	memset(&config, 0, sizeof(config));
+
+	fill_iram_v_2_3(&ram_table, params, false);
+
+	// We must copy to structure that is aligned to 32-bit
+	for (i = 0; i < NUM_POWER_FN_SEGS; i++) {
+		config.crgb_thresh[i] = ram_table.crgb_thresh[i];
+		config.crgb_offset[i] = ram_table.crgb_offset[i];
+		config.crgb_slope[i] = ram_table.crgb_slope[i];
+	}
+
+	for (i = 0; i < NUM_BL_CURVE_SEGS; i++) {
+		config.backlight_thresholds[i] = ram_table.backlight_thresholds[i];
+		config.backlight_offsets[i] = ram_table.backlight_offsets[i];
+	}
+
+	for (i = 0; i < NUM_AMBI_LEVEL; i++)
+		config.iir_curve[i] = ram_table.iir_curve[i];
+
+	for (i = 0; i < NUM_AMBI_LEVEL; i++) {
+		for (j = 0; j < NUM_AGGR_LEVEL; j++) {
+			config.min_reduction[i][j] = ram_table.min_reduction[i][j];
+			config.max_reduction[i][j] = ram_table.max_reduction[i][j];
+			config.bright_pos_gain[i][j] = ram_table.bright_pos_gain[i][j];
+			config.dark_pos_gain[i][j] = ram_table.dark_pos_gain[i][j];
+		}
+	}
+
+	for (i = 0; i < NUM_AGGR_LEVEL; i++) {
+		config.hybrid_factor[i] = ram_table.hybrid_factor[i];
+		config.contrast_factor[i] = ram_table.contrast_factor[i];
+		config.deviation_gain[i] = ram_table.deviation_gain[i];
+		config.min_knee[i] = ram_table.min_knee[i];
+		config.max_knee[i] = ram_table.max_knee[i];
+	}
+
+	if (params.backlight_ramping_override) {
+
+		ASSERT(params.backlight_ramping_reduction <= 0xFFFF);
+		ASSERT(params.backlight_ramping_start <= 0xFFFF);
+		for (i = 0; i < NUM_AGGR_LEVEL; i++) {
+			config.blRampReduction[i] = (uint16_t)params.backlight_ramping_reduction;
+			config.blRampStart[i]     = (uint16_t)params.backlight_ramping_start;
+		}
+	} else {
+		for (i = 0; i < NUM_AGGR_LEVEL; i++) {
+			config.blRampReduction[i] = abm_settings[set][i].blRampReduction;
+			config.blRampStart[i] = abm_settings[set][i].blRampStart;
+		}
+	}
+
+	config.min_abm_backlight = ram_table.min_abm_backlight;
+
+	if (res_pool->multiple_abms[inst]) {
+		result = res_pool->multiple_abms[inst]->funcs->init_abm_config(
+			res_pool->multiple_abms[inst], (char *)(&config), sizeof(struct abm_config_table), inst);
+	} else
+		result = res_pool->abm->funcs->init_abm_config(
+			res_pool->abm, (char *)(&config), sizeof(struct abm_config_table), 0);
+
+	return result;
+}
+
+bool dmcu_load_iram(struct dmcu *dmcu,
+	struct dmcu_iram_parameters params)
+{
+	unsigned char ram_table[IRAM_SIZE];
+	bool result = false;
+
+	if (dmcu == NULL)
+		return false;
+
+	if (dmcu && !dmcu->funcs->is_dmcu_initialized(dmcu))
+		return true;
+
+	memset(&ram_table, 0, sizeof(ram_table));
+
+	if (dmcu->dmcu_version.abm_version == 0x24) {
+		fill_iram_v_2_3((struct iram_table_v_2_2 *)ram_table, params, true);
+		result = dmcu->funcs->load_iram(dmcu, 0, (char *)(&ram_table),
+						IRAM_RESERVE_AREA_START_V2_2);
+	} else if (dmcu->dmcu_version.abm_version == 0x23) {
+		fill_iram_v_2_3((struct iram_table_v_2_2 *)ram_table, params, true);
+
+		result = dmcu->funcs->load_iram(
+				dmcu, 0, (char *)(&ram_table), IRAM_RESERVE_AREA_START_V2_2);
+	} else if (dmcu->dmcu_version.abm_version == 0x22) {
+		fill_iram_v_2_2((struct iram_table_v_2_2 *)ram_table, params);
+
+		result = dmcu->funcs->load_iram(
+				dmcu, 0, (char *)(&ram_table), IRAM_RESERVE_AREA_START_V2_2);
+	} else {
+		fill_iram_v_2((struct iram_table_v_2 *)ram_table, params);
+
+		result = dmcu->funcs->load_iram(
+				dmcu, 0, (char *)(&ram_table), IRAM_RESERVE_AREA_START_V2);
+
+		if (result)
+			result = dmcu->funcs->load_iram(
+					dmcu, IRAM_RESERVE_AREA_END_V2 + 1,
+					(char *)(&ram_table) + IRAM_RESERVE_AREA_END_V2 + 1,
+					sizeof(ram_table) - IRAM_RESERVE_AREA_END_V2 - 1);
+	}
+
+	return result;
+}
+
+bool fill_custom_backlight_caps(unsigned int config_no, struct dm_acpi_atif_backlight_caps *caps)
+{
+	unsigned int data_points_size;
+	uint64_t caps_size;
+
+	if (config_no >= ARRAY_SIZE(custom_backlight_profiles))
+		return false;
+
+	data_points_size = custom_backlight_profiles[config_no].num_data_points
+			* sizeof(custom_backlight_profiles[config_no].data_points[0]);
+
+	caps_size = sizeof(struct dm_acpi_atif_backlight_caps) - sizeof(caps->data_points) + data_points_size;
+	ASSERT(caps_size <= 0xFFFF);
+	caps->size = (uint16_t)caps_size;
+	caps->flags = 0;
+	caps->error_code = 0;
+	caps->ac_level_percentage = custom_backlight_profiles[config_no].ac_level_percentage;
+	caps->dc_level_percentage = custom_backlight_profiles[config_no].dc_level_percentage;
+	caps->min_input_signal = custom_backlight_profiles[config_no].min_input_signal;
+	caps->max_input_signal = custom_backlight_profiles[config_no].max_input_signal;
+	caps->num_data_points = (uint8_t)custom_backlight_profiles[config_no].num_data_points;
+	memcpy(caps->data_points, custom_backlight_profiles[config_no].data_points, data_points_size);
+	return true;
+}
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
index 5d444e9eb38f5..bf0c5901b4ee2 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
+++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.c
@@ -33,1072 +33,7 @@
 #define bswap16_based_on_endian(big_endian, value) \
 	((big_endian) ? cpu_to_be16(value) : cpu_to_le16(value))
 
-/* Possible Min Reduction config from least aggressive to most aggressive
- *  0    1     2     3     4     5     6     7     8     9     10    11   12
- * 100  98.0 94.1  94.1  85.1  80.3  75.3  69.4  60.0  57.6  50.2  49.8  40.0 %
- */
-static const unsigned char min_reduction_table[13] = {
-0xff, 0xfa, 0xf0, 0xf0, 0xd9, 0xcd, 0xc0, 0xb1, 0x99, 0x93, 0x80, 0x82, 0x66};
-
-/* Possible Max Reduction configs from least aggressive to most aggressive
- *  0    1     2     3     4     5     6     7     8     9     10    11   12
- * 96.1 89.8 85.1  80.3  69.4  64.7  64.7  50.2  39.6  30.2  30.2  30.2  19.6 %
- */
-static const unsigned char max_reduction_table[13] = {
-0xf5, 0xe5, 0xd9, 0xcd, 0xb1, 0xa5, 0xa5, 0x80, 0x65, 0x4d, 0x4d, 0x4d, 0x32};
-
-/* Possible ABM 2.2 Min Reduction configs from least aggressive to most aggressive
- *  0    1     2     3     4     5     6     7     8     9     10    11   12
- * 100  100   100   100   100   100   100   100  100  92.2  83.1  75.3  75.3 %
- */
-static const unsigned char min_reduction_table_v_2_2[13] = {
-0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xeb, 0xd4, 0xc0, 0xc0};
-
-/* Possible ABM 2.2 Max Reduction configs from least aggressive to most aggressive
- *  0    1     2     3     4     5     6     7     8     9     10    11   12
- * 96.1 89.8 74.9  69.4  64.7  52.2  48.6  39.6  30.2  25.1  19.6  12.5  12.5 %
- */
-static const unsigned char max_reduction_table_v_2_2[13] = {
-0xf5, 0xe5, 0xbf, 0xb1, 0xa5, 0x85, 0x7c, 0x65, 0x4d, 0x40, 0x32, 0x20, 0x20};
-
-/* Predefined ABM configuration sets. We may have different configuration sets
- * in order to satisfy different power/quality requirements.
- */
-static const unsigned char abm_config[abm_defines_max_config][abm_defines_max_level] = {
-/*  ABM Level 1,    ABM Level 2,    ABM Level 3,    ABM Level 4 */
-{       2,              5,              7,              8       },	/* Default - Medium aggressiveness */
-{       2,              5,              8,              11      },	/* Alt #1  - Increased aggressiveness */
-{       0,              2,              4,              8       },	/* Alt #2  - Minimal aggressiveness */
-{       3,              6,              10,             12      },	/* Alt #3  - Super aggressiveness */
-};
-
-struct abm_parameters {
-	unsigned char min_reduction;
-	unsigned char max_reduction;
-	unsigned char bright_pos_gain;
-	unsigned char dark_pos_gain;
-	unsigned char brightness_gain;
-	unsigned char contrast_factor;
-	unsigned char deviation_gain;
-	unsigned char min_knee;
-	unsigned char max_knee;
-	unsigned short blRampReduction;
-	unsigned short blRampStart;
-};
-
-static const struct abm_parameters abm_settings_config0[abm_defines_max_level] = {
-//  min_red  max_red  bright_pos  dark_pos  bright_gain  contrast  dev   min_knee  max_knee  blRed    blStart
-	{0xff,   0xbf,    0x20,       0x00,     0xff,        0x99,     0xb3, 0x40,     0xe0,     0xf777,  0xcccc},
-	{0xde,   0x85,    0x20,       0x00,     0xe0,        0x90,     0xa8, 0x40,     0xc8,     0xf777,  0xcccc},
-	{0xb0,   0x50,    0x20,       0x00,     0xc0,        0x88,     0x78, 0x70,     0xa0,     0xeeee,  0x9999},
-	{0x82,   0x40,    0x20,       0x00,     0x00,        0xb8,     0xb3, 0x70,     0x70,     0xe333,  0xb333},
-};
-
-static const struct abm_parameters abm_settings_config1[abm_defines_max_level] = {
-//  min_red  max_red  bright_pos  dark_pos  bright_gain  contrast  dev   min_knee  max_knee  blRed  blStart
-	{0xf0,   0xd9,    0x20,       0x00,     0x00,        0xff,     0xb3, 0x70,     0x70,     0xcccc,  0xcccc},
-	{0xcd,   0xa5,    0x20,       0x00,     0x00,        0xff,     0xb3, 0x70,     0x70,     0xcccc,  0xcccc},
-	{0x99,   0x65,    0x20,       0x00,     0x00,        0xff,     0xb3, 0x70,     0x70,     0xcccc,  0xcccc},
-	{0x82,   0x4d,    0x20,       0x00,     0x00,        0xff,     0xb3, 0x70,     0x70,     0xcccc,  0xcccc},
-};
-
-static const struct abm_parameters abm_settings_config2[abm_defines_max_level] = {
-//  min_red  max_red  bright_pos  dark_pos  bright_gain  contrast  dev   min_knee  max_knee  blRed    blStart
-	{0xf0,   0xbf,    0x20,       0x00,     0x88,        0x99,     0xb3, 0x40,     0xe0,    0x0000,  0xcccc},
-	{0xd8,   0x85,    0x20,       0x00,     0x70,        0x90,     0xa8, 0x40,     0xc8,    0x0700,  0xb333},
-	{0xb8,   0x58,    0x20,       0x00,     0x64,        0x88,     0x78, 0x70,     0xa0,    0x7000,  0x9999},
-	{0x82,   0x40,    0x20,       0x00,     0x00,        0xb8,     0xb3, 0x70,     0x70,    0xc333,  0xb333},
-};
-
-static const struct abm_parameters * const abm_settings[] = {
-	abm_settings_config0,
-	abm_settings_config1,
-	abm_settings_config2,
-};
-
-static const struct dm_bl_data_point custom_backlight_curve0[] = {
-		{2, 14}, {4, 16}, {6, 18}, {8, 21}, {10, 23}, {12, 26}, {14, 29}, {16, 32}, {18, 35},
-		{20, 38}, {22, 41}, {24, 44}, {26, 48}, {28, 52}, {30, 55}, {32, 59}, {34, 62},
-		{36, 67}, {38, 71}, {40, 75}, {42, 80}, {44, 84}, {46, 88}, {48, 93}, {50, 98},
-		{52, 103}, {54, 108}, {56, 113}, {58, 118}, {60, 123}, {62, 129}, {64, 135}, {66, 140},
-		{68, 146}, {70, 152}, {72, 158}, {74, 164}, {76, 171}, {78, 177}, {80, 183}, {82, 190},
-		{84, 197}, {86, 204}, {88, 211}, {90, 218}, {92, 225}, {94, 232}, {96, 240}, {98, 247}};
-
-struct custom_backlight_profile {
-	uint8_t  ac_level_percentage;
-	uint8_t  dc_level_percentage;
-	uint8_t  min_input_signal;
-	uint8_t  max_input_signal;
-	uint8_t  num_data_points;
-	const struct dm_bl_data_point *data_points;
-};
-
-static const struct custom_backlight_profile custom_backlight_profiles[] = {
-		{100, 32, 12, 255, ARRAY_SIZE(custom_backlight_curve0), custom_backlight_curve0},
-};
-
-#define NUM_AMBI_LEVEL    5
-#define NUM_AGGR_LEVEL    4
-#define NUM_POWER_FN_SEGS 8
-#define NUM_BL_CURVE_SEGS 16
-#define IRAM_SIZE 256
-
-#define IRAM_RESERVE_AREA_START_V2 0xF0  // reserve 0xF0~0xF6 are write by DMCU only
-#define IRAM_RESERVE_AREA_END_V2 0xF6  // reserve 0xF0~0xF6 are write by DMCU only
-
-#define IRAM_RESERVE_AREA_START_V2_2 0xF0  // reserve 0xF0~0xFF are write by DMCU only
-#define IRAM_RESERVE_AREA_END_V2_2 0xFF  // reserve 0xF0~0xFF are write by DMCU only
-
-#pragma pack(push, 1)
-/* NOTE: iRAM is 256B in size */
-struct iram_table_v_2 {
-	/* flags                      */
-	uint16_t min_abm_backlight;					/* 0x00 U16  */
-
-	/* parameters for ABM2.0 algorithm */
-	uint8_t min_reduction[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x02 U0.8 */
-	uint8_t max_reduction[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x16 U0.8 */
-	uint8_t bright_pos_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];	/* 0x2a U2.6 */
-	uint8_t bright_neg_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];	/* 0x3e U2.6 */
-	uint8_t dark_pos_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x52 U2.6 */
-	uint8_t dark_neg_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x66 U2.6 */
-	uint8_t iir_curve[NUM_AMBI_LEVEL];				/* 0x7a U0.8 */
-	uint8_t deviation_gain;						/* 0x7f U0.8 */
-
-	/* parameters for crgb conversion */
-	uint16_t crgb_thresh[NUM_POWER_FN_SEGS];			/* 0x80 U3.13 */
-	uint16_t crgb_offset[NUM_POWER_FN_SEGS];			/* 0x90 U1.15 */
-	uint16_t crgb_slope[NUM_POWER_FN_SEGS];				/* 0xa0 U4.12 */
-
-	/* parameters for custom curve */
-	/* thresholds for brightness --> backlight */
-	uint16_t backlight_thresholds[NUM_BL_CURVE_SEGS];		/* 0xb0 U16.0 */
-	/* offsets for brightness --> backlight */
-	uint16_t backlight_offsets[NUM_BL_CURVE_SEGS];			/* 0xd0 U16.0 */
-
-	/* For reading PSR State directly from IRAM */
-	uint8_t psr_state;						/* 0xf0       */
-	uint8_t dmcu_mcp_interface_version;				/* 0xf1       */
-	uint8_t dmcu_abm_feature_version;				/* 0xf2       */
-	uint8_t dmcu_psr_feature_version;				/* 0xf3       */
-	uint16_t dmcu_version;						/* 0xf4       */
-	uint8_t dmcu_state;						/* 0xf6       */
-
-	uint16_t blRampReduction;					/* 0xf7       */
-	uint16_t blRampStart;						/* 0xf9       */
-	uint8_t dummy5;							/* 0xfb       */
-	uint8_t dummy6;							/* 0xfc       */
-	uint8_t dummy7;							/* 0xfd       */
-	uint8_t dummy8;							/* 0xfe       */
-	uint8_t dummy9;							/* 0xff       */
-};
-
-struct iram_table_v_2_2 {
-	/* flags                      */
-	uint16_t flags;							/* 0x00 U16  */
-
-	/* parameters for ABM2.2 algorithm */
-	uint8_t min_reduction[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x02 U0.8 */
-	uint8_t max_reduction[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x16 U0.8 */
-	uint8_t bright_pos_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];	/* 0x2a U2.6 */
-	uint8_t dark_pos_gain[NUM_AMBI_LEVEL][NUM_AGGR_LEVEL];		/* 0x3e U2.6 */
-	uint8_t hybrid_factor[NUM_AGGR_LEVEL];				/* 0x52 U0.8 */
-	uint8_t contrast_factor[NUM_AGGR_LEVEL];			/* 0x56 U0.8 */
-	uint8_t deviation_gain[NUM_AGGR_LEVEL];				/* 0x5a U0.8 */
-	uint8_t iir_curve[NUM_AMBI_LEVEL];				/* 0x5e U0.8 */
-	uint8_t min_knee[NUM_AGGR_LEVEL];				/* 0x63 U0.8 */
-	uint8_t max_knee[NUM_AGGR_LEVEL];				/* 0x67 U0.8 */
-	uint16_t min_abm_backlight;					/* 0x6b U16  */
-	uint8_t pad[19];						/* 0x6d U0.8 */
-
-	/* parameters for crgb conversion */
-	uint16_t crgb_thresh[NUM_POWER_FN_SEGS];			/* 0x80 U3.13 */
-	uint16_t crgb_offset[NUM_POWER_FN_SEGS];			/* 0x90 U1.15 */
-	uint16_t crgb_slope[NUM_POWER_FN_SEGS];				/* 0xa0 U4.12 */
-
-	/* parameters for custom curve */
-	/* thresholds for brightness --> backlight */
-	uint16_t backlight_thresholds[NUM_BL_CURVE_SEGS];		/* 0xb0 U16.0 */
-	/* offsets for brightness --> backlight */
-	uint16_t backlight_offsets[NUM_BL_CURVE_SEGS];			/* 0xd0 U16.0 */
-
-	/* For reading PSR State directly from IRAM */
-	uint8_t psr_state;						/* 0xf0       */
-	uint8_t dmcu_mcp_interface_version;				/* 0xf1       */
-	uint8_t dmcu_abm_feature_version;				/* 0xf2       */
-	uint8_t dmcu_psr_feature_version;				/* 0xf3       */
-	uint16_t dmcu_version;						/* 0xf4       */
-	uint8_t dmcu_state;						/* 0xf6       */
-
-	uint8_t dummy1;							/* 0xf7       */
-	uint8_t dummy2;							/* 0xf8       */
-	uint8_t dummy3;							/* 0xf9       */
-	uint8_t dummy4;							/* 0xfa       */
-	uint8_t dummy5;							/* 0xfb       */
-	uint8_t dummy6;							/* 0xfc       */
-	uint8_t dummy7;							/* 0xfd       */
-	uint8_t dummy8;							/* 0xfe       */
-	uint8_t dummy9;							/* 0xff       */
-};
-#pragma pack(pop)
-
-static void fill_backlight_transform_table(struct dmcu_iram_parameters params,
-		struct iram_table_v_2 *table)
-{
-	unsigned int i;
-	unsigned int num_entries = NUM_BL_CURVE_SEGS;
-	unsigned int lut_index;
-
-	table->backlight_thresholds[0] = 0;
-	ASSERT(params.backlight_lut_array[0] <= 0xFFFF);
-	table->backlight_offsets[0] = (uint16_t)params.backlight_lut_array[0];
-	table->backlight_thresholds[num_entries-1] = 0xFFFF;
-	ASSERT(params.backlight_lut_array[params.backlight_lut_array_size - 1] <= 0xFFFF);
-	table->backlight_offsets[num_entries-1] =
-		(uint16_t)params.backlight_lut_array[params.backlight_lut_array_size - 1];
-
-	/* Setup all brightness levels between 0% and 100% exclusive
-	 * Fills brightness-to-backlight transform table. Backlight custom curve
-	 * describes transform from brightness to backlight. It will be defined
-	 * as set of thresholds and set of offsets, together, implying
-	 * extrapolation of custom curve into 16 uniformly spanned linear
-	 * segments.  Each threshold/offset represented by 16 bit entry in
-	 * format U4.10.
-	 */
-	for (i = 1; i+1 < num_entries; i++) {
-		lut_index = (params.backlight_lut_array_size - 1) * i / (num_entries - 1);
-
-		ASSERT(lut_index < params.backlight_lut_array_size);
-
-		unsigned int threshold_val = DIV_ROUNDUP((i * 65536), num_entries);
-		unsigned int offset_val = params.backlight_lut_array[lut_index];
-
-		ASSERT(threshold_val <= 0xFFFF);
-		ASSERT(offset_val <= 0xFFFF);
-
-		table->backlight_thresholds[i] = cpu_to_be16((uint16_t)threshold_val);
-		table->backlight_offsets[i]    = cpu_to_be16((uint16_t)offset_val);
-	}
-}
-
-static void fill_backlight_transform_table_v_2_2(struct dmcu_iram_parameters params,
-		struct iram_table_v_2_2 *table, bool big_endian)
-{
-	unsigned int i;
-	unsigned int num_entries = NUM_BL_CURVE_SEGS;
-	unsigned int lut_index;
-
-	table->backlight_thresholds[0] = 0;
-	ASSERT(params.backlight_lut_array[0] <= 0xFFFF);
-	table->backlight_offsets[0] = (uint16_t)params.backlight_lut_array[0];
-	table->backlight_thresholds[num_entries-1] = 0xFFFF;
-	ASSERT(params.backlight_lut_array[params.backlight_lut_array_size - 1] <= 0xFFFF);
-	table->backlight_offsets[num_entries-1] =
-		(uint16_t)params.backlight_lut_array[params.backlight_lut_array_size - 1];
-
-	/* Setup all brightness levels between 0% and 100% exclusive
-	 * Fills brightness-to-backlight transform table. Backlight custom curve
-	 * describes transform from brightness to backlight. It will be defined
-	 * as set of thresholds and set of offsets, together, implying
-	 * extrapolation of custom curve into 16 uniformly spanned linear
-	 * segments.  Each threshold/offset represented by 16 bit entry in
-	 * format U4.10.
-	 */
-	for (i = 1; i+1 < num_entries; i++) {
-		lut_index = DIV_ROUNDUP((i * params.backlight_lut_array_size), num_entries);
-		ASSERT(lut_index < params.backlight_lut_array_size);
-
-		unsigned int threshold_val = DIV_ROUNDUP((i * 65536), num_entries);
-		unsigned int offset_val = params.backlight_lut_array[lut_index];
-
-		ASSERT(threshold_val <= 0xFFFF);
-		ASSERT(offset_val <= 0xFFFF);
-
-		table->backlight_thresholds[i] = (big_endian) ?
-			cpu_to_be16((uint16_t)threshold_val) : cpu_to_le16((uint16_t)threshold_val);
-		table->backlight_offsets[i] = (big_endian) ?
-			cpu_to_be16((uint16_t)offset_val) : cpu_to_le16((uint16_t)offset_val);
-	}
-}
-
-static void fill_iram_v_2(struct iram_table_v_2 *ram_table, struct dmcu_iram_parameters params)
-{
-	unsigned int set = params.set;
-
-	ram_table->min_abm_backlight =
-			cpu_to_be16(params.min_abm_backlight);
-	ram_table->deviation_gain = 0xb3;
-
-	ram_table->blRampReduction =
-		cpu_to_be16(params.backlight_ramping_reduction);
-	ram_table->blRampStart =
-		cpu_to_be16(params.backlight_ramping_start);
-
-	ram_table->min_reduction[0][0] = min_reduction_table[abm_config[set][0]];
-	ram_table->min_reduction[1][0] = min_reduction_table[abm_config[set][0]];
-	ram_table->min_reduction[2][0] = min_reduction_table[abm_config[set][0]];
-	ram_table->min_reduction[3][0] = min_reduction_table[abm_config[set][0]];
-	ram_table->min_reduction[4][0] = min_reduction_table[abm_config[set][0]];
-	ram_table->max_reduction[0][0] = max_reduction_table[abm_config[set][0]];
-	ram_table->max_reduction[1][0] = max_reduction_table[abm_config[set][0]];
-	ram_table->max_reduction[2][0] = max_reduction_table[abm_config[set][0]];
-	ram_table->max_reduction[3][0] = max_reduction_table[abm_config[set][0]];
-	ram_table->max_reduction[4][0] = max_reduction_table[abm_config[set][0]];
-
-	ram_table->min_reduction[0][1] = min_reduction_table[abm_config[set][1]];
-	ram_table->min_reduction[1][1] = min_reduction_table[abm_config[set][1]];
-	ram_table->min_reduction[2][1] = min_reduction_table[abm_config[set][1]];
-	ram_table->min_reduction[3][1] = min_reduction_table[abm_config[set][1]];
-	ram_table->min_reduction[4][1] = min_reduction_table[abm_config[set][1]];
-	ram_table->max_reduction[0][1] = max_reduction_table[abm_config[set][1]];
-	ram_table->max_reduction[1][1] = max_reduction_table[abm_config[set][1]];
-	ram_table->max_reduction[2][1] = max_reduction_table[abm_config[set][1]];
-	ram_table->max_reduction[3][1] = max_reduction_table[abm_config[set][1]];
-	ram_table->max_reduction[4][1] = max_reduction_table[abm_config[set][1]];
-
-	ram_table->min_reduction[0][2] = min_reduction_table[abm_config[set][2]];
-	ram_table->min_reduction[1][2] = min_reduction_table[abm_config[set][2]];
-	ram_table->min_reduction[2][2] = min_reduction_table[abm_config[set][2]];
-	ram_table->min_reduction[3][2] = min_reduction_table[abm_config[set][2]];
-	ram_table->min_reduction[4][2] = min_reduction_table[abm_config[set][2]];
-	ram_table->max_reduction[0][2] = max_reduction_table[abm_config[set][2]];
-	ram_table->max_reduction[1][2] = max_reduction_table[abm_config[set][2]];
-	ram_table->max_reduction[2][2] = max_reduction_table[abm_config[set][2]];
-	ram_table->max_reduction[3][2] = max_reduction_table[abm_config[set][2]];
-	ram_table->max_reduction[4][2] = max_reduction_table[abm_config[set][2]];
-
-	ram_table->min_reduction[0][3] = min_reduction_table[abm_config[set][3]];
-	ram_table->min_reduction[1][3] = min_reduction_table[abm_config[set][3]];
-	ram_table->min_reduction[2][3] = min_reduction_table[abm_config[set][3]];
-	ram_table->min_reduction[3][3] = min_reduction_table[abm_config[set][3]];
-	ram_table->min_reduction[4][3] = min_reduction_table[abm_config[set][3]];
-	ram_table->max_reduction[0][3] = max_reduction_table[abm_config[set][3]];
-	ram_table->max_reduction[1][3] = max_reduction_table[abm_config[set][3]];
-	ram_table->max_reduction[2][3] = max_reduction_table[abm_config[set][3]];
-	ram_table->max_reduction[3][3] = max_reduction_table[abm_config[set][3]];
-	ram_table->max_reduction[4][3] = max_reduction_table[abm_config[set][3]];
-
-	ram_table->bright_pos_gain[0][0] = 0x20;
-	ram_table->bright_pos_gain[0][1] = 0x20;
-	ram_table->bright_pos_gain[0][2] = 0x20;
-	ram_table->bright_pos_gain[0][3] = 0x20;
-	ram_table->bright_pos_gain[1][0] = 0x20;
-	ram_table->bright_pos_gain[1][1] = 0x20;
-	ram_table->bright_pos_gain[1][2] = 0x20;
-	ram_table->bright_pos_gain[1][3] = 0x20;
-	ram_table->bright_pos_gain[2][0] = 0x20;
-	ram_table->bright_pos_gain[2][1] = 0x20;
-	ram_table->bright_pos_gain[2][2] = 0x20;
-	ram_table->bright_pos_gain[2][3] = 0x20;
-	ram_table->bright_pos_gain[3][0] = 0x20;
-	ram_table->bright_pos_gain[3][1] = 0x20;
-	ram_table->bright_pos_gain[3][2] = 0x20;
-	ram_table->bright_pos_gain[3][3] = 0x20;
-	ram_table->bright_pos_gain[4][0] = 0x20;
-	ram_table->bright_pos_gain[4][1] = 0x20;
-	ram_table->bright_pos_gain[4][2] = 0x20;
-	ram_table->bright_pos_gain[4][3] = 0x20;
-	ram_table->bright_neg_gain[0][0] = 0x00;
-	ram_table->bright_neg_gain[0][1] = 0x00;
-	ram_table->bright_neg_gain[0][2] = 0x00;
-	ram_table->bright_neg_gain[0][3] = 0x00;
-	ram_table->bright_neg_gain[1][0] = 0x00;
-	ram_table->bright_neg_gain[1][1] = 0x00;
-	ram_table->bright_neg_gain[1][2] = 0x00;
-	ram_table->bright_neg_gain[1][3] = 0x00;
-	ram_table->bright_neg_gain[2][0] = 0x00;
-	ram_table->bright_neg_gain[2][1] = 0x00;
-	ram_table->bright_neg_gain[2][2] = 0x00;
-	ram_table->bright_neg_gain[2][3] = 0x00;
-	ram_table->bright_neg_gain[3][0] = 0x00;
-	ram_table->bright_neg_gain[3][1] = 0x00;
-	ram_table->bright_neg_gain[3][2] = 0x00;
-	ram_table->bright_neg_gain[3][3] = 0x00;
-	ram_table->bright_neg_gain[4][0] = 0x00;
-	ram_table->bright_neg_gain[4][1] = 0x00;
-	ram_table->bright_neg_gain[4][2] = 0x00;
-	ram_table->bright_neg_gain[4][3] = 0x00;
-	ram_table->dark_pos_gain[0][0] = 0x00;
-	ram_table->dark_pos_gain[0][1] = 0x00;
-	ram_table->dark_pos_gain[0][2] = 0x00;
-	ram_table->dark_pos_gain[0][3] = 0x00;
-	ram_table->dark_pos_gain[1][0] = 0x00;
-	ram_table->dark_pos_gain[1][1] = 0x00;
-	ram_table->dark_pos_gain[1][2] = 0x00;
-	ram_table->dark_pos_gain[1][3] = 0x00;
-	ram_table->dark_pos_gain[2][0] = 0x00;
-	ram_table->dark_pos_gain[2][1] = 0x00;
-	ram_table->dark_pos_gain[2][2] = 0x00;
-	ram_table->dark_pos_gain[2][3] = 0x00;
-	ram_table->dark_pos_gain[3][0] = 0x00;
-	ram_table->dark_pos_gain[3][1] = 0x00;
-	ram_table->dark_pos_gain[3][2] = 0x00;
-	ram_table->dark_pos_gain[3][3] = 0x00;
-	ram_table->dark_pos_gain[4][0] = 0x00;
-	ram_table->dark_pos_gain[4][1] = 0x00;
-	ram_table->dark_pos_gain[4][2] = 0x00;
-	ram_table->dark_pos_gain[4][3] = 0x00;
-	ram_table->dark_neg_gain[0][0] = 0x00;
-	ram_table->dark_neg_gain[0][1] = 0x00;
-	ram_table->dark_neg_gain[0][2] = 0x00;
-	ram_table->dark_neg_gain[0][3] = 0x00;
-	ram_table->dark_neg_gain[1][0] = 0x00;
-	ram_table->dark_neg_gain[1][1] = 0x00;
-	ram_table->dark_neg_gain[1][2] = 0x00;
-	ram_table->dark_neg_gain[1][3] = 0x00;
-	ram_table->dark_neg_gain[2][0] = 0x00;
-	ram_table->dark_neg_gain[2][1] = 0x00;
-	ram_table->dark_neg_gain[2][2] = 0x00;
-	ram_table->dark_neg_gain[2][3] = 0x00;
-	ram_table->dark_neg_gain[3][0] = 0x00;
-	ram_table->dark_neg_gain[3][1] = 0x00;
-	ram_table->dark_neg_gain[3][2] = 0x00;
-	ram_table->dark_neg_gain[3][3] = 0x00;
-	ram_table->dark_neg_gain[4][0] = 0x00;
-	ram_table->dark_neg_gain[4][1] = 0x00;
-	ram_table->dark_neg_gain[4][2] = 0x00;
-	ram_table->dark_neg_gain[4][3] = 0x00;
-
-	ram_table->iir_curve[0] = 0x65;
-	ram_table->iir_curve[1] = 0x65;
-	ram_table->iir_curve[2] = 0x65;
-	ram_table->iir_curve[3] = 0x65;
-	ram_table->iir_curve[4] = 0x65;
-
-	//Gamma 2.4
-	ram_table->crgb_thresh[0] = cpu_to_be16(0x13b6);
-	ram_table->crgb_thresh[1] = cpu_to_be16(0x1648);
-	ram_table->crgb_thresh[2] = cpu_to_be16(0x18e3);
-	ram_table->crgb_thresh[3] = cpu_to_be16(0x1b41);
-	ram_table->crgb_thresh[4] = cpu_to_be16(0x1d46);
-	ram_table->crgb_thresh[5] = cpu_to_be16(0x1f21);
-	ram_table->crgb_thresh[6] = cpu_to_be16(0x2167);
-	ram_table->crgb_thresh[7] = cpu_to_be16(0x2384);
-	ram_table->crgb_offset[0] = cpu_to_be16(0x2999);
-	ram_table->crgb_offset[1] = cpu_to_be16(0x3999);
-	ram_table->crgb_offset[2] = cpu_to_be16(0x4666);
-	ram_table->crgb_offset[3] = cpu_to_be16(0x5999);
-	ram_table->crgb_offset[4] = cpu_to_be16(0x6333);
-	ram_table->crgb_offset[5] = cpu_to_be16(0x7800);
-	ram_table->crgb_offset[6] = cpu_to_be16(0x8c00);
-	ram_table->crgb_offset[7] = cpu_to_be16(0xa000);
-	ram_table->crgb_slope[0]  = cpu_to_be16(0x3147);
-	ram_table->crgb_slope[1]  = cpu_to_be16(0x2978);
-	ram_table->crgb_slope[2]  = cpu_to_be16(0x23a2);
-	ram_table->crgb_slope[3]  = cpu_to_be16(0x1f55);
-	ram_table->crgb_slope[4]  = cpu_to_be16(0x1c63);
-	ram_table->crgb_slope[5]  = cpu_to_be16(0x1a0f);
-	ram_table->crgb_slope[6]  = cpu_to_be16(0x178d);
-	ram_table->crgb_slope[7]  = cpu_to_be16(0x15ab);
-
-	fill_backlight_transform_table(
-			params, ram_table);
-}
-
-static void fill_iram_v_2_2(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params)
-{
-	unsigned int set = params.set;
-
-	ram_table->flags = 0x0;
-
-	ram_table->min_abm_backlight =
-			cpu_to_be16(params.min_abm_backlight);
-
-	ram_table->deviation_gain[0] = 0xb3;
-	ram_table->deviation_gain[1] = 0xa8;
-	ram_table->deviation_gain[2] = 0x98;
-	ram_table->deviation_gain[3] = 0x68;
-
-	ram_table->min_reduction[0][0] = min_reduction_table_v_2_2[abm_config[set][0]];
-	ram_table->min_reduction[1][0] = min_reduction_table_v_2_2[abm_config[set][0]];
-	ram_table->min_reduction[2][0] = min_reduction_table_v_2_2[abm_config[set][0]];
-	ram_table->min_reduction[3][0] = min_reduction_table_v_2_2[abm_config[set][0]];
-	ram_table->min_reduction[4][0] = min_reduction_table_v_2_2[abm_config[set][0]];
-	ram_table->max_reduction[0][0] = max_reduction_table_v_2_2[abm_config[set][0]];
-	ram_table->max_reduction[1][0] = max_reduction_table_v_2_2[abm_config[set][0]];
-	ram_table->max_reduction[2][0] = max_reduction_table_v_2_2[abm_config[set][0]];
-	ram_table->max_reduction[3][0] = max_reduction_table_v_2_2[abm_config[set][0]];
-	ram_table->max_reduction[4][0] = max_reduction_table_v_2_2[abm_config[set][0]];
-
-	ram_table->min_reduction[0][1] = min_reduction_table_v_2_2[abm_config[set][1]];
-	ram_table->min_reduction[1][1] = min_reduction_table_v_2_2[abm_config[set][1]];
-	ram_table->min_reduction[2][1] = min_reduction_table_v_2_2[abm_config[set][1]];
-	ram_table->min_reduction[3][1] = min_reduction_table_v_2_2[abm_config[set][1]];
-	ram_table->min_reduction[4][1] = min_reduction_table_v_2_2[abm_config[set][1]];
-	ram_table->max_reduction[0][1] = max_reduction_table_v_2_2[abm_config[set][1]];
-	ram_table->max_reduction[1][1] = max_reduction_table_v_2_2[abm_config[set][1]];
-	ram_table->max_reduction[2][1] = max_reduction_table_v_2_2[abm_config[set][1]];
-	ram_table->max_reduction[3][1] = max_reduction_table_v_2_2[abm_config[set][1]];
-	ram_table->max_reduction[4][1] = max_reduction_table_v_2_2[abm_config[set][1]];
-
-	ram_table->min_reduction[0][2] = min_reduction_table_v_2_2[abm_config[set][2]];
-	ram_table->min_reduction[1][2] = min_reduction_table_v_2_2[abm_config[set][2]];
-	ram_table->min_reduction[2][2] = min_reduction_table_v_2_2[abm_config[set][2]];
-	ram_table->min_reduction[3][2] = min_reduction_table_v_2_2[abm_config[set][2]];
-	ram_table->min_reduction[4][2] = min_reduction_table_v_2_2[abm_config[set][2]];
-	ram_table->max_reduction[0][2] = max_reduction_table_v_2_2[abm_config[set][2]];
-	ram_table->max_reduction[1][2] = max_reduction_table_v_2_2[abm_config[set][2]];
-	ram_table->max_reduction[2][2] = max_reduction_table_v_2_2[abm_config[set][2]];
-	ram_table->max_reduction[3][2] = max_reduction_table_v_2_2[abm_config[set][2]];
-	ram_table->max_reduction[4][2] = max_reduction_table_v_2_2[abm_config[set][2]];
-
-	ram_table->min_reduction[0][3] = min_reduction_table_v_2_2[abm_config[set][3]];
-	ram_table->min_reduction[1][3] = min_reduction_table_v_2_2[abm_config[set][3]];
-	ram_table->min_reduction[2][3] = min_reduction_table_v_2_2[abm_config[set][3]];
-	ram_table->min_reduction[3][3] = min_reduction_table_v_2_2[abm_config[set][3]];
-	ram_table->min_reduction[4][3] = min_reduction_table_v_2_2[abm_config[set][3]];
-	ram_table->max_reduction[0][3] = max_reduction_table_v_2_2[abm_config[set][3]];
-	ram_table->max_reduction[1][3] = max_reduction_table_v_2_2[abm_config[set][3]];
-	ram_table->max_reduction[2][3] = max_reduction_table_v_2_2[abm_config[set][3]];
-	ram_table->max_reduction[3][3] = max_reduction_table_v_2_2[abm_config[set][3]];
-	ram_table->max_reduction[4][3] = max_reduction_table_v_2_2[abm_config[set][3]];
-
-	ram_table->bright_pos_gain[0][0] = 0x20;
-	ram_table->bright_pos_gain[0][1] = 0x20;
-	ram_table->bright_pos_gain[0][2] = 0x20;
-	ram_table->bright_pos_gain[0][3] = 0x20;
-	ram_table->bright_pos_gain[1][0] = 0x20;
-	ram_table->bright_pos_gain[1][1] = 0x20;
-	ram_table->bright_pos_gain[1][2] = 0x20;
-	ram_table->bright_pos_gain[1][3] = 0x20;
-	ram_table->bright_pos_gain[2][0] = 0x20;
-	ram_table->bright_pos_gain[2][1] = 0x20;
-	ram_table->bright_pos_gain[2][2] = 0x20;
-	ram_table->bright_pos_gain[2][3] = 0x20;
-	ram_table->bright_pos_gain[3][0] = 0x20;
-	ram_table->bright_pos_gain[3][1] = 0x20;
-	ram_table->bright_pos_gain[3][2] = 0x20;
-	ram_table->bright_pos_gain[3][3] = 0x20;
-	ram_table->bright_pos_gain[4][0] = 0x20;
-	ram_table->bright_pos_gain[4][1] = 0x20;
-	ram_table->bright_pos_gain[4][2] = 0x20;
-	ram_table->bright_pos_gain[4][3] = 0x20;
-
-	ram_table->dark_pos_gain[0][0] = 0x00;
-	ram_table->dark_pos_gain[0][1] = 0x00;
-	ram_table->dark_pos_gain[0][2] = 0x00;
-	ram_table->dark_pos_gain[0][3] = 0x00;
-	ram_table->dark_pos_gain[1][0] = 0x00;
-	ram_table->dark_pos_gain[1][1] = 0x00;
-	ram_table->dark_pos_gain[1][2] = 0x00;
-	ram_table->dark_pos_gain[1][3] = 0x00;
-	ram_table->dark_pos_gain[2][0] = 0x00;
-	ram_table->dark_pos_gain[2][1] = 0x00;
-	ram_table->dark_pos_gain[2][2] = 0x00;
-	ram_table->dark_pos_gain[2][3] = 0x00;
-	ram_table->dark_pos_gain[3][0] = 0x00;
-	ram_table->dark_pos_gain[3][1] = 0x00;
-	ram_table->dark_pos_gain[3][2] = 0x00;
-	ram_table->dark_pos_gain[3][3] = 0x00;
-	ram_table->dark_pos_gain[4][0] = 0x00;
-	ram_table->dark_pos_gain[4][1] = 0x00;
-	ram_table->dark_pos_gain[4][2] = 0x00;
-	ram_table->dark_pos_gain[4][3] = 0x00;
-
-	ram_table->hybrid_factor[0] = 0xff;
-	ram_table->hybrid_factor[1] = 0xff;
-	ram_table->hybrid_factor[2] = 0xff;
-	ram_table->hybrid_factor[3] = 0xc0;
-
-	ram_table->contrast_factor[0] = 0x99;
-	ram_table->contrast_factor[1] = 0x99;
-	ram_table->contrast_factor[2] = 0x90;
-	ram_table->contrast_factor[3] = 0x80;
-
-	ram_table->iir_curve[0] = 0x65;
-	ram_table->iir_curve[1] = 0x65;
-	ram_table->iir_curve[2] = 0x65;
-	ram_table->iir_curve[3] = 0x65;
-	ram_table->iir_curve[4] = 0x65;
-
-	//Gamma 2.2
-	ram_table->crgb_thresh[0] = cpu_to_be16(0x127c);
-	ram_table->crgb_thresh[1] = cpu_to_be16(0x151b);
-	ram_table->crgb_thresh[2] = cpu_to_be16(0x17d5);
-	ram_table->crgb_thresh[3] = cpu_to_be16(0x1a56);
-	ram_table->crgb_thresh[4] = cpu_to_be16(0x1c83);
-	ram_table->crgb_thresh[5] = cpu_to_be16(0x1e72);
-	ram_table->crgb_thresh[6] = cpu_to_be16(0x20f0);
-	ram_table->crgb_thresh[7] = cpu_to_be16(0x232b);
-	ram_table->crgb_offset[0] = cpu_to_be16(0x2999);
-	ram_table->crgb_offset[1] = cpu_to_be16(0x3999);
-	ram_table->crgb_offset[2] = cpu_to_be16(0x4666);
-	ram_table->crgb_offset[3] = cpu_to_be16(0x5999);
-	ram_table->crgb_offset[4] = cpu_to_be16(0x6333);
-	ram_table->crgb_offset[5] = cpu_to_be16(0x7800);
-	ram_table->crgb_offset[6] = cpu_to_be16(0x8c00);
-	ram_table->crgb_offset[7] = cpu_to_be16(0xa000);
-	ram_table->crgb_slope[0]  = cpu_to_be16(0x3609);
-	ram_table->crgb_slope[1]  = cpu_to_be16(0x2dfa);
-	ram_table->crgb_slope[2]  = cpu_to_be16(0x27ea);
-	ram_table->crgb_slope[3]  = cpu_to_be16(0x235d);
-	ram_table->crgb_slope[4]  = cpu_to_be16(0x2042);
-	ram_table->crgb_slope[5]  = cpu_to_be16(0x1dc3);
-	ram_table->crgb_slope[6]  = cpu_to_be16(0x1b1a);
-	ram_table->crgb_slope[7]  = cpu_to_be16(0x1910);
-
-	fill_backlight_transform_table_v_2_2(
-			params, ram_table, true);
-}
-
-static void fill_iram_v_2_3(struct iram_table_v_2_2 *ram_table, struct dmcu_iram_parameters params, bool big_endian)
-{
-	unsigned int i, j;
-	unsigned int set = params.set;
-
-	ram_table->flags = 0x0;
-	ram_table->min_abm_backlight = (big_endian) ?
-		cpu_to_be16(params.min_abm_backlight) :
-		cpu_to_le16(params.min_abm_backlight);
-
-	for (i = 0; i < NUM_AGGR_LEVEL; i++) {
-		ram_table->hybrid_factor[i] = abm_settings[set][i].brightness_gain;
-		ram_table->contrast_factor[i] = abm_settings[set][i].contrast_factor;
-		ram_table->deviation_gain[i] = abm_settings[set][i].deviation_gain;
-		ram_table->min_knee[i] = abm_settings[set][i].min_knee;
-		ram_table->max_knee[i] = abm_settings[set][i].max_knee;
-
-		for (j = 0; j < NUM_AMBI_LEVEL; j++) {
-			ram_table->min_reduction[j][i] = abm_settings[set][i].min_reduction;
-			ram_table->max_reduction[j][i] = abm_settings[set][i].max_reduction;
-			ram_table->bright_pos_gain[j][i] = abm_settings[set][i].bright_pos_gain;
-			ram_table->dark_pos_gain[j][i] = abm_settings[set][i].dark_pos_gain;
-		}
-	}
-
-	ram_table->iir_curve[0] = 0x65;
-	ram_table->iir_curve[1] = 0x65;
-	ram_table->iir_curve[2] = 0x65;
-	ram_table->iir_curve[3] = 0x65;
-	ram_table->iir_curve[4] = 0x65;
-
-	//Gamma 2.2
-	ram_table->crgb_thresh[0] = bswap16_based_on_endian(big_endian, 0x127c);
-	ram_table->crgb_thresh[1] = bswap16_based_on_endian(big_endian, 0x151b);
-	ram_table->crgb_thresh[2] = bswap16_based_on_endian(big_endian, 0x17d5);
-	ram_table->crgb_thresh[3] = bswap16_based_on_endian(big_endian, 0x1a56);
-	ram_table->crgb_thresh[4] = bswap16_based_on_endian(big_endian, 0x1c83);
-	ram_table->crgb_thresh[5] = bswap16_based_on_endian(big_endian, 0x1e72);
-	ram_table->crgb_thresh[6] = bswap16_based_on_endian(big_endian, 0x20f0);
-	ram_table->crgb_thresh[7] = bswap16_based_on_endian(big_endian, 0x232b);
-	ram_table->crgb_offset[0] = bswap16_based_on_endian(big_endian, 0x2999);
-	ram_table->crgb_offset[1] = bswap16_based_on_endian(big_endian, 0x3999);
-	ram_table->crgb_offset[2] = bswap16_based_on_endian(big_endian, 0x4666);
-	ram_table->crgb_offset[3] = bswap16_based_on_endian(big_endian, 0x5999);
-	ram_table->crgb_offset[4] = bswap16_based_on_endian(big_endian, 0x6333);
-	ram_table->crgb_offset[5] = bswap16_based_on_endian(big_endian, 0x7800);
-	ram_table->crgb_offset[6] = bswap16_based_on_endian(big_endian, 0x8c00);
-	ram_table->crgb_offset[7] = bswap16_based_on_endian(big_endian, 0xa000);
-	ram_table->crgb_slope[0]  = bswap16_based_on_endian(big_endian, 0x3609);
-	ram_table->crgb_slope[1]  = bswap16_based_on_endian(big_endian, 0x2dfa);
-	ram_table->crgb_slope[2]  = bswap16_based_on_endian(big_endian, 0x27ea);
-	ram_table->crgb_slope[3]  = bswap16_based_on_endian(big_endian, 0x235d);
-	ram_table->crgb_slope[4]  = bswap16_based_on_endian(big_endian, 0x2042);
-	ram_table->crgb_slope[5]  = bswap16_based_on_endian(big_endian, 0x1dc3);
-	ram_table->crgb_slope[6]  = bswap16_based_on_endian(big_endian, 0x1b1a);
-	ram_table->crgb_slope[7]  = bswap16_based_on_endian(big_endian, 0x1910);
-
-	fill_backlight_transform_table_v_2_2(
-			params, ram_table, big_endian);
-}
-
-bool dmub_init_abm_config(struct resource_pool *res_pool,
-	struct dmcu_iram_parameters params,
-	unsigned int inst)
-{
-	struct iram_table_v_2_2 ram_table;
-	struct abm_config_table config;
-	unsigned int set = params.set;
-	bool result = false;
-	uint32_t i, j = 0;
-
-	if (res_pool->abm == NULL && res_pool->multiple_abms[inst] == NULL)
-		return false;
-
-	memset(&ram_table, 0, sizeof(ram_table));
-	memset(&config, 0, sizeof(config));
-
-	fill_iram_v_2_3(&ram_table, params, false);
-
-	// We must copy to structure that is aligned to 32-bit
-	for (i = 0; i < NUM_POWER_FN_SEGS; i++) {
-		config.crgb_thresh[i] = ram_table.crgb_thresh[i];
-		config.crgb_offset[i] = ram_table.crgb_offset[i];
-		config.crgb_slope[i] = ram_table.crgb_slope[i];
-	}
-
-	for (i = 0; i < NUM_BL_CURVE_SEGS; i++) {
-		config.backlight_thresholds[i] = ram_table.backlight_thresholds[i];
-		config.backlight_offsets[i] = ram_table.backlight_offsets[i];
-	}
-
-	for (i = 0; i < NUM_AMBI_LEVEL; i++)
-		config.iir_curve[i] = ram_table.iir_curve[i];
-
-	for (i = 0; i < NUM_AMBI_LEVEL; i++) {
-		for (j = 0; j < NUM_AGGR_LEVEL; j++) {
-			config.min_reduction[i][j] = ram_table.min_reduction[i][j];
-			config.max_reduction[i][j] = ram_table.max_reduction[i][j];
-			config.bright_pos_gain[i][j] = ram_table.bright_pos_gain[i][j];
-			config.dark_pos_gain[i][j] = ram_table.dark_pos_gain[i][j];
-		}
-	}
-
-	for (i = 0; i < NUM_AGGR_LEVEL; i++) {
-		config.hybrid_factor[i] = ram_table.hybrid_factor[i];
-		config.contrast_factor[i] = ram_table.contrast_factor[i];
-		config.deviation_gain[i] = ram_table.deviation_gain[i];
-		config.min_knee[i] = ram_table.min_knee[i];
-		config.max_knee[i] = ram_table.max_knee[i];
-	}
-
-	if (params.backlight_ramping_override) {
-
-		ASSERT(params.backlight_ramping_reduction <= 0xFFFF);
-		ASSERT(params.backlight_ramping_start <= 0xFFFF);
-		for (i = 0; i < NUM_AGGR_LEVEL; i++) {
-			config.blRampReduction[i] = (uint16_t)params.backlight_ramping_reduction;
-			config.blRampStart[i]     = (uint16_t)params.backlight_ramping_start;
-		}
-	} else {
-		for (i = 0; i < NUM_AGGR_LEVEL; i++) {
-			config.blRampReduction[i] = abm_settings[set][i].blRampReduction;
-			config.blRampStart[i] = abm_settings[set][i].blRampStart;
-		}
-	}
-
-	config.min_abm_backlight = ram_table.min_abm_backlight;
-
-	if (res_pool->multiple_abms[inst]) {
-		result = res_pool->multiple_abms[inst]->funcs->init_abm_config(
-			res_pool->multiple_abms[inst], (char *)(&config), sizeof(struct abm_config_table), inst);
-	} else
-		result = res_pool->abm->funcs->init_abm_config(
-			res_pool->abm, (char *)(&config), sizeof(struct abm_config_table), 0);
-
-	return result;
-}
-
-bool dmcu_load_iram(struct dmcu *dmcu,
-	struct dmcu_iram_parameters params)
-{
-	unsigned char ram_table[IRAM_SIZE];
-	bool result = false;
-
-	if (dmcu == NULL)
-		return false;
-
-	if (dmcu && !dmcu->funcs->is_dmcu_initialized(dmcu))
-		return true;
-
-	memset(&ram_table, 0, sizeof(ram_table));
-
-	if (dmcu->dmcu_version.abm_version == 0x24) {
-		fill_iram_v_2_3((struct iram_table_v_2_2 *)ram_table, params, true);
-		result = dmcu->funcs->load_iram(dmcu, 0, (char *)(&ram_table),
-						IRAM_RESERVE_AREA_START_V2_2);
-	} else if (dmcu->dmcu_version.abm_version == 0x23) {
-		fill_iram_v_2_3((struct iram_table_v_2_2 *)ram_table, params, true);
-
-		result = dmcu->funcs->load_iram(
-				dmcu, 0, (char *)(&ram_table), IRAM_RESERVE_AREA_START_V2_2);
-	} else if (dmcu->dmcu_version.abm_version == 0x22) {
-		fill_iram_v_2_2((struct iram_table_v_2_2 *)ram_table, params);
-
-		result = dmcu->funcs->load_iram(
-				dmcu, 0, (char *)(&ram_table), IRAM_RESERVE_AREA_START_V2_2);
-	} else {
-		fill_iram_v_2((struct iram_table_v_2 *)ram_table, params);
-
-		result = dmcu->funcs->load_iram(
-				dmcu, 0, (char *)(&ram_table), IRAM_RESERVE_AREA_START_V2);
-
-		if (result)
-			result = dmcu->funcs->load_iram(
-					dmcu, IRAM_RESERVE_AREA_END_V2 + 1,
-					(char *)(&ram_table) + IRAM_RESERVE_AREA_END_V2 + 1,
-					sizeof(ram_table) - IRAM_RESERVE_AREA_END_V2 - 1);
-	}
-
-	return result;
-}
-
-/*
- * is_psr_su_specific_panel() - check if sink is AMD vendor-specific PSR-SU
- * supported eDP device.
- *
- * @link: dc link pointer
- *
- * Return: true if AMDGPU vendor specific PSR-SU eDP panel
- */
-bool is_psr_su_specific_panel(struct dc_link *link)
-{
-	bool isPSRSUSupported = false;
-	struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
-
-	if (dpcd_caps->edp_rev >= DP_EDP_14) {
-		if (dpcd_caps->psr_info.psr_version >= DP_PSR2_WITH_Y_COORD_ET_SUPPORTED)
-			isPSRSUSupported = true;
-		/*
-		 * Some panels will report PSR capabilities over additional DPCD bits.
-		 * Such panels are approved despite reporting only PSR v3, as long as
-		 * the additional bits are reported.
-		 */
-		if (dpcd_caps->sink_dev_id == DP_BRANCH_DEVICE_ID_001CF8) {
-			/*
-			 * This is the temporary workaround to disable PSRSU when system turned on
-			 * DSC function on the sepcific sink.
-			 */
-			if (dpcd_caps->psr_info.psr_version < DP_PSR2_WITH_Y_COORD_IS_SUPPORTED)
-				isPSRSUSupported = false;
-			else if (dpcd_caps->dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT &&
-				((dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x08) ||
-				(dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x07)))
-				isPSRSUSupported = false;
-			else if (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x03)
-				isPSRSUSupported = false;
-			else if (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x01)
-				isPSRSUSupported = false;
-			else if (dpcd_caps->psr_info.force_psrsu_cap == 0x1)
-				isPSRSUSupported = true;
-		}
-	}
-
-	return isPSRSUSupported;
-}
-
-/**
- * mod_power_calc_psr_configs() - calculate/update generic psr configuration fields.
- * @psr_config: [output], psr configuration structure to be updated
- * @link: [input] dc link pointer
- * @stream: [input] dc stream state pointer
- *
- * calculate and update the psr configuration fields that are not DM specific, i.e. such
- * fields which are based on DPCD caps or timing information. To setup PSR in DMUB FW,
- * this helper is assumed to be called before the call of the DC helper dc_link_setup_psr().
- *
- * PSR config fields to be updated within the helper:
- * - psr_rfb_setup_time
- * - psr_sdp_transmit_line_num_deadline
- * - line_time_in_us
- * - su_y_granularity
- * - su_granularity_required
- * - psr_frame_capture_indication_req
- * - psr_exit_link_training_required
- *
- * PSR config fields that are DM specific and NOT updated within the helper:
- * - allow_smu_optimizations
- * - allow_multi_disp_optimizations
- */
-void mod_power_calc_psr_configs(struct psr_config *psr_config,
-		struct dc_link *link,
-		const struct dc_stream_state *stream)
-{
-	unsigned int num_vblank_lines = 0;
-	unsigned int vblank_time_in_us = 0;
-	unsigned int sdp_tx_deadline_in_us = 0;
-	unsigned int line_time_in_us = 0;
-	struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
-	const int psr_setup_time_step_in_us = 55;	/* refer to eDP spec DPCD 0x071h */
-
-	/* timing parameters */
-	num_vblank_lines = stream->timing.v_total -
-			 stream->timing.v_addressable -
-			 stream->timing.v_border_top -
-			 stream->timing.v_border_bottom;
-
-	vblank_time_in_us = (stream->timing.h_total * num_vblank_lines * 1000) / (stream->timing.pix_clk_100hz / 10);
-
-	line_time_in_us = ((stream->timing.h_total * 1000) / (stream->timing.pix_clk_100hz / 10)) + 1;
-
-	/**
-	 * psr configuration fields
-	 *
-	 * as per eDP 1.5 pg. 377 of 459, DPCD 0x071h bits [3:1], psr setup time bits interpreted as below
-	 * 000b <--> 330 us (default)
-	 * 001b <--> 275 us
-	 * 010b <--> 220 us
-	 * 011b <--> 165 us
-	 * 100b <--> 110 us
-	 * 101b <--> 055 us
-	 * 110b <--> 000 us
-	 */
-	psr_config->psr_rfb_setup_time =
-		(6 - dpcd_caps->psr_info.psr_dpcd_caps.bits.PSR_SETUP_TIME) * psr_setup_time_step_in_us;
-
-	if (psr_config->psr_rfb_setup_time > vblank_time_in_us) {
-		link->psr_settings.psr_frame_capture_indication_req = true;
-		link->psr_settings.psr_sdp_transmit_line_num_deadline = num_vblank_lines;
-	} else {
-		sdp_tx_deadline_in_us = vblank_time_in_us - psr_config->psr_rfb_setup_time;
-
-		/* Set the last possible line SDP may be transmitted without violating the RFB setup time */
-		link->psr_settings.psr_frame_capture_indication_req = false;
-		link->psr_settings.psr_sdp_transmit_line_num_deadline = sdp_tx_deadline_in_us / line_time_in_us;
-	}
-
-	psr_config->psr_sdp_transmit_line_num_deadline = link->psr_settings.psr_sdp_transmit_line_num_deadline;
-	psr_config->line_time_in_us = line_time_in_us;
-	psr_config->su_y_granularity = dpcd_caps->psr_info.psr2_su_y_granularity_cap;
-	psr_config->su_granularity_required = dpcd_caps->psr_info.psr_dpcd_caps.bits.SU_GRANULARITY_REQUIRED;
-	psr_config->psr_frame_capture_indication_req = link->psr_settings.psr_frame_capture_indication_req;
-	psr_config->psr_exit_link_training_required =
-		!link->dpcd_caps.psr_info.psr_dpcd_caps.bits.LINK_TRAINING_ON_EXIT_NOT_REQUIRED;
-}
-
-void init_replay_config(struct dc_link *link, struct replay_config *pr_config)
-{
-	link->replay_settings.config = *pr_config;
-}
-
 bool mod_power_only_edp(const struct dc_state *context, const struct dc_stream_state *stream)
 {
 	return context && context->stream_count == 1 && dc_is_embedded_signal(stream->signal);
 }
-
-bool psr_su_set_dsc_slice_height(struct dc *dc, struct dc_link *link,
-			      struct dc_stream_state *stream,
-			      struct psr_config *config)
-{
-	uint16_t pic_height;
-	uint16_t slice_height;
-
-	config->dsc_slice_height = 0;
-	if (!(link->connector_signal & SIGNAL_TYPE_EDP) ||
-	    !dc->caps.edp_dsc_support ||
-	    link->panel_config.dsc.disable_dsc_edp ||
-	    !link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT ||
-	    !stream->timing.dsc_cfg.num_slices_v)
-		return true;
-
-	pic_height = stream->timing.v_addressable +
-		stream->timing.v_border_top + stream->timing.v_border_bottom;
-
-	if (stream->timing.dsc_cfg.num_slices_v == 0)
-		return false;
-
-	slice_height = pic_height / stream->timing.dsc_cfg.num_slices_v;
-	config->dsc_slice_height = slice_height;
-
-	if (slice_height) {
-		if (config->su_y_granularity &&
-		    (slice_height % config->su_y_granularity)) {
-			ASSERT(0);
-			return false;
-		}
-	}
-
-	return true;
-}
-
-void set_replay_frame_skip_number(struct dc_link *link,
-	enum replay_coasting_vtotal_type type,
-	uint32_t coasting_vtotal_refresh_rate_mhz,
-	uint32_t flicker_free_refresh_rate_mhz,
-	bool is_defer)
-{
-	uint32_t *frame_skip_number_array = NULL;
-	uint32_t frame_skip_number = 0;
-
-	if (link == NULL)
-		return;
-
-	if (false == link->replay_settings.config.frame_skip_supported)
-		return;
-
-	if (flicker_free_refresh_rate_mhz == 0 || coasting_vtotal_refresh_rate_mhz == 0)
-		return;
-
-	if (is_defer)
-		frame_skip_number_array = link->replay_settings.defer_frame_skip_number_table;
-	else
-		frame_skip_number_array = link->replay_settings.frame_skip_number_table;
-
-	if (frame_skip_number_array == NULL)
-		return;
-
-	frame_skip_number = coasting_vtotal_refresh_rate_mhz / flicker_free_refresh_rate_mhz;
-
-	if (frame_skip_number >= 1)
-		frame_skip_number_array[type] = frame_skip_number - 1;
-	else
-		frame_skip_number_array[type] = 0;
-}
-
-void set_replay_defer_update_coasting_vtotal(struct dc_link *link,
-	enum replay_coasting_vtotal_type type,
-	uint32_t vtotal)
-{
-	link->replay_settings.defer_update_coasting_vtotal_table[type] = vtotal;
-}
-
-void update_replay_coasting_vtotal_from_defer(struct dc_link *link,
-	enum replay_coasting_vtotal_type type)
-{
-	link->replay_settings.coasting_vtotal_table[type] =
-		link->replay_settings.defer_update_coasting_vtotal_table[type];
-	link->replay_settings.frame_skip_number_table[type] =
-		link->replay_settings.defer_frame_skip_number_table[type];
-}
-
-void set_replay_coasting_vtotal(struct dc_link *link,
-	enum replay_coasting_vtotal_type type,
-	uint32_t vtotal)
-{
-	link->replay_settings.coasting_vtotal_table[type] = vtotal;
-}
-
-void set_replay_low_rr_full_screen_video_src_vtotal(struct dc_link *link, uint16_t vtotal)
-{
-	link->replay_settings.low_rr_full_screen_video_pseudo_vtotal = vtotal;
-}
-
-void calculate_replay_link_off_frame_count(struct dc_link *link,
-	uint16_t vtotal, uint16_t htotal)
-{
-	uint8_t max_link_off_frame_count = 0;
-	uint16_t max_deviation_line = 0,  pixel_deviation_per_line = 0;
-
-	if (!link || link->replay_settings.config.replay_version != DC_FREESYNC_REPLAY)
-		return;
-
-	max_deviation_line = link->dpcd_caps.pr_info.max_deviation_line;
-	pixel_deviation_per_line = link->dpcd_caps.pr_info.pixel_deviation_per_line;
-
-	if (htotal != 0 && vtotal != 0 && pixel_deviation_per_line != 0)
-		max_link_off_frame_count = htotal * max_deviation_line / (pixel_deviation_per_line * vtotal);
-	else
-		ASSERT(0);
-
-	link->replay_settings.link_off_frame_count = max_link_off_frame_count;
-}
-
-bool fill_custom_backlight_caps(unsigned int config_no, struct dm_acpi_atif_backlight_caps *caps)
-{
-	unsigned int data_points_size;
-	uint64_t caps_size;
-
-	if (config_no >= ARRAY_SIZE(custom_backlight_profiles))
-		return false;
-
-	data_points_size = custom_backlight_profiles[config_no].num_data_points
-			* sizeof(custom_backlight_profiles[config_no].data_points[0]);
-
-	caps_size = sizeof(struct dm_acpi_atif_backlight_caps) - sizeof(caps->data_points) + data_points_size;
-	ASSERT(caps_size <= 0xFFFF);
-	caps->size = (uint16_t)caps_size;
-	caps->flags = 0;
-	caps->error_code = 0;
-	caps->ac_level_percentage = custom_backlight_profiles[config_no].ac_level_percentage;
-	caps->dc_level_percentage = custom_backlight_profiles[config_no].dc_level_percentage;
-	caps->min_input_signal = custom_backlight_profiles[config_no].min_input_signal;
-	caps->max_input_signal = custom_backlight_profiles[config_no].max_input_signal;
-	caps->num_data_points = custom_backlight_profiles[config_no].num_data_points;
-	memcpy(caps->data_points, custom_backlight_profiles[config_no].data_points, data_points_size);
-	return true;
-}
-
-void reset_replay_dsync_error_count(struct dc_link *link)
-{
-	link->replay_settings.replay_desync_error_fail_count = 0;
-}
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_helpers.h b/drivers/gpu/drm/amd/display/modules/power/power_helpers.h
index 87d31d9dce5a6..600da3e331261 100644
--- a/drivers/gpu/drm/amd/display/modules/power/power_helpers.h
+++ b/drivers/gpu/drm/amd/display/modules/power/power_helpers.h
@@ -28,6 +28,7 @@
 #include "dc/inc/hw/dmcu.h"
 #include "dc/inc/hw/abm.h"
 #include "dc/inc/core_types.h"
+#include "mod_power.h"
 
 struct resource_pool;
 
@@ -46,7 +47,99 @@ struct dmcu_iram_parameters {
 	unsigned int min_abm_backlight;
 	unsigned int set;
 };
+struct backlight_state {
+	/* HW uses u16.16 format for backlight PWM */
+	unsigned int backlight_pwm;
+	/* DM may call power module to set backlight
+	 * targeting percent brightness
+	 */
+	unsigned int backlight_millipercent;
+	/* DM may call power module to set backlight based on an explicit
+	 * nits value.
+	 */
+	unsigned int backlight_millinit;
+	unsigned int frame_ramp;
+	bool smooth_brightness_enabled;
+	bool isHDR;
+};
+struct power_entity {
+	struct dc_stream_state *stream;
+	struct psr_caps *caps;
+	struct mod_power_psr_context *psr_context;
+
+	/*PSR cached properties*/
+	bool psr_enabled;
+	unsigned int psr_events;
+	unsigned int psr_power_opt;
+	unsigned int replay_events;
+};
+
+struct pwr_backlight_properties {
+	bool use_nits_based_brightness;
+	bool disable_fractional_pwm;
+
+	unsigned int min_abm_backlight;
+	unsigned int num_backlight_levels;
+
+	bool backlight_ramping_override;
+	unsigned int backlight_ramping_reduction;
+	unsigned int backlight_ramping_start;
+
+	/* Backlight cached properties */
+	unsigned int ac_backlight_percent;
+	unsigned int dc_backlight_percent;
+
+	/* backlight LUT stored in HW u16.16 format*/
+	unsigned int *backlight_lut;
+	unsigned int min_backlight_pwm;
+	unsigned int max_backlight_pwm;
+	unsigned int backlight_range;
 
+	/* Describes the panel's min and max luminance in millinits measured
+	 * on full white screen, in min and max backlight settings.
+	 */
+	unsigned int min_brightness_millinits;
+	unsigned int max_brightness_millinits;
+	unsigned int nits_range;
+
+	bool backlight_caps_valid;
+	bool use_custom_backlight_caps;
+	unsigned int custom_backlight_caps_config_no;
+	bool use_linear_backlight_curve;
+};
+
+struct dmcu_varibright_cached_properties {
+	unsigned int varibright_config_setting;
+	unsigned int varibright_level;
+	unsigned int varibright_hw_level;
+	unsigned int def_varibright_level;
+	bool varibright_user_enable;
+	bool varibright_active;
+};
+
+struct core_power {
+	struct mod_power mod_public;
+	struct dc *dc;
+	struct power_entity *map;
+	struct dmcu_varibright_cached_properties varibright_prop;
+	struct pwr_backlight_properties bl_prop[MAX_NUM_EDP];
+	struct backlight_state bl_state[MAX_NUM_EDP];
+	unsigned int edp_num;
+
+	bool psr_smu_optimizations_support;
+	bool multi_disp_optimizations_support;
+
+	unsigned int num_entities;
+};
+
+union dmcu_abm_set_bl_params {
+	struct {
+		unsigned int gradual_change : 1; /* [0:0] */
+		unsigned int reserved : 15; /* [15:1] */
+		unsigned int frame_ramp : 16; /* [31:16] */
+	} bits;
+	unsigned int u32All;
+};
 bool dmcu_load_iram(struct dmcu *dmcu,
 		struct dmcu_iram_parameters params);
 bool dmub_init_abm_config(struct resource_pool *res_pool,
@@ -86,4 +179,33 @@ bool fill_custom_backlight_caps(unsigned int config_no,
 void reset_replay_dsync_error_count(struct dc_link *link);
 void change_replay_to_psr(struct dc_link *link);
 void change_psr_to_replay(struct dc_link *link);
+void initialize_backlight_caps(struct core_power *core_power, unsigned int inst);
+unsigned int backlight_millipercent_to_pwm(
+		struct core_power *core_power, unsigned int millipercent, unsigned int inst);
+unsigned int backlight_millipercent_to_millinit(
+		struct core_power *core_power, unsigned int millipercent, unsigned int inst);
+void fill_backlight_level_params(struct core_power *core_power,
+	struct set_backlight_level_params *backlight_level_params,
+	int panel_inst, uint8_t aux_inst, unsigned int backlight_pwm,
+	enum backlight_control_type backlight_control_type,
+	unsigned int backlight_millinit, unsigned int transition_time_millisec,
+	bool is_hdr);
+bool mod_power_hw_init_backlight(struct mod_power *mod_power);
+void mod_power_update_backlight_on_mode_change(
+    struct core_power *core_power,
+    struct dc_link *link,
+    unsigned int panel_inst,
+    uint8_t aux_inst,
+    bool is_hdr);
+unsigned int map_index_from_stream(struct core_power *core_power,
+		const struct dc_stream_state *stream);
+bool mod_power_psr_notify_mode_change(struct mod_power *mod_power,
+	const struct dc_stream_state *stream,
+	struct dc_link *link,
+	unsigned int stream_index);
+void mod_power_replay_notify_mode_change(struct mod_power *mod_power,
+	struct dc *dc,
+	struct dc_link *link,
+	const struct dc_stream_state *stream,
+	unsigned int stream_index);
 #endif /* MODULES_POWER_POWER_HELPERS_H_ */
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_psr.c b/drivers/gpu/drm/amd/display/modules/power/power_psr.c
new file mode 100644
index 0000000000000..5ecb570c204e1
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/power/power_psr.c
@@ -0,0 +1,665 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dm_services.h"
+#include "dc.h"
+#include "mod_power.h"
+#include "core_types.h"
+#include "dmcu.h"
+#include "abm.h"
+#include "power_helpers.h"
+#include "dce/dmub_psr.h"
+#include "dal_asic_id.h"
+#include "link_service.h"
+#include <linux/math.h>
+
+#define DC_TRACE_LEVEL_MESSAGE(...) /* do nothing */
+#define DC_TRACE_LEVEL_MESSAGEP(...) /* do nothing */
+#include "dc/inc/hw/dmcu.h"
+#include "dc/inc/hw/abm.h"
+#include "dmub_cmd.h"
+
+#define MOD_POWER_TO_CORE(mod_power)\
+		container_of(mod_power, struct core_power, mod_public)
+
+static unsigned int calc_psr_num_static_frames(unsigned int vsync_rate_hz)
+{
+	/* Initialize fail-safe to 2 static frames. */
+	unsigned int num_frames_static = 2;
+
+	/* Calculate number of frames such that at least 30 ms has passed.
+	 * Round up to ensure the static period is not shorter than 30 ms.
+	 */
+	if (vsync_rate_hz != 0)
+		num_frames_static = DIV_ROUND_UP(30000 * vsync_rate_hz, 1000000);
+
+	return num_frames_static;
+}
+
+bool mod_power_psr_notify_mode_change(struct mod_power *mod_power,
+	const struct dc_stream_state *stream,
+	struct dc_link *link,
+	unsigned int stream_index)
+{
+	struct core_power *core_power = NULL;
+	struct dc *dc = NULL;
+	struct psr_config psr_config = {0};
+	struct psr_context psr_context = {0};
+	int active_psr_events = 0;
+
+	if ((mod_power == NULL) || (stream == NULL) || (link == NULL))
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	dc = core_power->dc;
+
+	// NO num_entities check here - already validated by caller
+	// stream_index is passed as validated parameter
+	active_psr_events = core_power->map[stream_index].psr_events;
+
+	/* Calculate PSR configurations */
+	mod_power_calc_psr_configs(&psr_config, link, stream);
+
+	psr_config.psr_exit_link_training_required =
+			core_power->map[stream_index].caps->psr_exit_link_training_required;
+	if (dc->ctx->asic_id.chip_family >= AMDGPU_FAMILY_GC_11_0_1)
+		psr_config.allow_smu_optimizations =
+				core_power->psr_smu_optimizations_support && dc_is_embedded_signal(stream->signal);
+	else
+		psr_config.allow_smu_optimizations =
+				core_power->psr_smu_optimizations_support &&
+				mod_power_only_edp(dc->current_state, stream);
+
+	psr_config.allow_multi_disp_optimizations = core_power->multi_disp_optimizations_support;
+
+	psr_config.rate_control_caps = core_power->map[stream_index].caps->rate_control_caps;
+
+	if (active_psr_events & psr_event_os_request_force_ffu)
+		psr_config.os_request_force_ffu = true;
+
+	/*
+	 * DSC support:
+	 * DSC slice height value must be 'mod' by su_y_granularity.
+	 * According to Panel Vendor, there might be varied conditions to fulfill.
+	 * Right now, DSC slice height value must be multiple of su_y_granularity.
+	 *
+	 * The value of DSC slice height is determined in DSC Driver but it does not
+	 * propagated out here, so we need to calculate it as below 'slice_height'.
+	 */
+	psr_su_set_dsc_slice_height(dc, link,
+				(struct dc_stream_state *) stream,
+				&psr_config);
+
+	dc_link_setup_psr(link, stream, &psr_config, &psr_context);
+
+	return true;
+}
+
+static void mod_power_psr_set_power_opt(struct mod_power *mod_power,
+	struct dc_stream_state *stream,
+	unsigned int active_psr_events,
+	bool psr_enable_request)
+{
+	(void)psr_enable_request;
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+	unsigned int stream_index = 0;
+	unsigned int power_opt = 0;
+
+	if (!stream)
+		return;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	stream_index = map_index_from_stream(core_power, stream);
+	if (!core_power->map[stream_index].caps->psr_version)
+		return;
+
+	link = dc_stream_get_link(stream);
+
+	if (active_psr_events == 0) {
+		/* Static Screen */
+		power_opt |= (psr_power_opt_smu_opt_static_screen | psr_power_opt_z10_static_screen |
+					psr_power_opt_ds_disable_allow);
+	}
+
+	/* psr_power_opt_flag is a configuration parameter into the module that determines
+	 * which optimizations to enable during psr
+	 */
+	power_opt &= core_power->map[stream_index].caps->psr_power_opt_flag;
+	if (core_power->map[stream_index].psr_power_opt != power_opt) {
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_VERBOSE,
+				WPP_BIT_FLAG_Firmware_PsrState,
+				"mod_power set_power_opt: psr_power_opt=0x%04x, power_opt=0x%04x active_psr_events=0x%04x, psr_power_opt_flag=0x%04x",
+				core_power->map[stream_index].psr_power_opt,
+				power_opt,
+				active_psr_events,
+				core_power->map[stream_index].caps->psr_power_opt_flag);
+		dc_link_set_psr_allow_active(link, NULL, false, false, &power_opt);
+		core_power->map[stream_index].psr_power_opt = power_opt;
+	}
+}
+
+static bool set_psr_enable(struct mod_power *mod_power,
+		struct dc_stream_state *stream,
+		bool psr_enable,
+		bool wait,
+		bool force_static)
+{
+	struct core_power *core_power = NULL;
+	enum dc_psr_state state = PSR_STATE0;
+	unsigned int retry_count;
+	const unsigned int max_retry = 1000;
+	struct dc_link *link = NULL;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0) {
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"set psr enable: ERROR: stream=%p num_entities=%u",
+							stream,
+							core_power->num_entities);
+		return false;
+	}
+
+	if (psr_enable)	{
+		unsigned int vsync_rate_hz;
+		struct dc_static_screen_params params = {0};
+
+		vsync_rate_hz = (unsigned int)div_u64(div_u64((
+				stream->timing.pix_clk_100hz * 100),
+				stream->timing.v_total),
+				stream->timing.h_total);
+
+		params.triggers.cursor_update = true;
+		params.triggers.overlay_update = true;
+		params.triggers.surface_update = true;
+		params.num_frames = calc_psr_num_static_frames(vsync_rate_hz);
+
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"set psr enable: CALCS: pix_clk_100hz=%u v_total=%u h_total=%u vsync_rate_hz=%u num_frames=%u",
+							stream->timing.pix_clk_100hz,
+							stream->timing.v_total,
+							stream->timing.h_total,
+							vsync_rate_hz,
+							params.num_frames);
+
+		dc_stream_set_static_screen_params(core_power->dc,
+						   &stream, 1,
+						   &params);
+	}
+
+	link = dc_stream_get_link(stream);
+
+	if (!dc_link_set_psr_allow_active(link, &psr_enable, false, force_static, NULL)) {
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"set psr enable: ERROR: stream=%p link=%p psr_enable=%d",
+							stream,
+							link,
+							psr_enable);
+		return false;
+	}
+
+	if (wait == true) {
+
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"set psr enable: BEGIN WAIT: psr_enable=%d",
+							(int)psr_enable);
+
+		for (retry_count = 0; retry_count <= max_retry; retry_count++) {
+			dc_link_get_psr_state(link, &state);
+			if (psr_enable) {
+				if (state != PSR_STATE0 &&
+						(!force_static || state == PSR_STATE3))
+					break;
+			} else {
+				if (state == PSR_STATE0)
+					break;
+			}
+			udelay(500);
+		}
+
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"set psr enable: END WAIT: psr_enable=%d",
+							(int)psr_enable);
+
+		/* assert if max retry hit */
+		if (retry_count >= max_retry) {
+			ASSERT(0);
+			DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+								WPP_BIT_FLAG_Firmware_PsrState,
+								"set psr enable: ERROR: retry_count=%u: Unexpectedly long wait for PSR state change.",
+								retry_count);
+		}
+	} else {
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_INFORMATION,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"set psr enable: PSR state change initiated (wait=false): psr_enable=%d",
+							(int)psr_enable);
+	}
+
+	return true;
+}
+
+bool mod_power_get_psr_event(struct mod_power *mod_power,
+			struct dc_stream_state *stream,
+			unsigned int *active_psr_events)
+{
+	struct core_power *core_power = NULL;
+	unsigned int stream_index = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	stream_index = map_index_from_stream(core_power, stream);
+
+	if (!core_power->map[stream_index].caps->psr_version)
+		return false;
+
+	*active_psr_events = core_power->map[stream_index].psr_events;
+
+	return true;
+}
+
+bool mod_power_set_psr_event(struct mod_power *mod_power,
+		struct dc_stream_state *stream, bool set_event,
+		enum psr_event event, bool wait)
+{
+	struct core_power *core_power = NULL;
+	unsigned int stream_index = 0;
+	unsigned int active_psr_events = 0;
+	bool psr_enable_request = false;
+	bool force_static = false;
+
+	if (mod_power == NULL || stream == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	stream_index = map_index_from_stream(core_power, stream);
+
+	if (core_power->num_entities == 0) {
+		DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_ERROR,
+							WPP_BIT_FLAG_Firmware_PsrState,
+							"mod_power set_psr_event: ERROR: stream=%p event=%d num_entities=%u",
+							stream,
+							(int)event,
+							core_power->num_entities);
+		return false;
+	}
+
+	if (!core_power->map[stream_index].caps->psr_version)
+		return false;
+
+	if (set_event)
+		core_power->map[stream_index].psr_events |= event;
+	else
+		core_power->map[stream_index].psr_events &= ~event;
+
+	active_psr_events = core_power->map[stream_index].psr_events;
+
+	// ignore other events when we're in forced psr enabled state
+	if (active_psr_events & psr_event_dynamic_display_switch &&
+			event != psr_event_dynamic_display_switch)
+		return false;
+
+	// ignore other events when we're in forced psr enabled state
+	if (active_psr_events & psr_event_os_override_hold &&
+			event != psr_event_os_override_hold)
+		return false;
+
+	// ignore other events when we're in forced psr enabled state
+	// dds events need to be processed while in dynamic_link_rate_control
+	if (active_psr_events & psr_event_dynamic_link_rate_control &&
+			event != psr_event_dynamic_link_rate_control &&
+			event != psr_event_dds_defer_stream_enable &&
+			event != psr_event_dynamic_display_switch)
+		return false;
+
+	if (active_psr_events & (psr_event_test_harness_disable_psr | psr_event_os_request_disable))
+		psr_enable_request = false;
+	else if (active_psr_events & psr_event_pause)
+		psr_enable_request = false;
+	else if (active_psr_events & psr_event_test_harness_enable_psr)
+		psr_enable_request = true;
+	else if (active_psr_events & psr_event_dynamic_display_switch) {
+		psr_enable_request = true;
+		force_static = true;
+	} else if (active_psr_events & psr_event_dynamic_link_rate_control) {
+		psr_enable_request = true;
+		force_static = true;
+	} else if (active_psr_events & psr_event_edp_panel_off_disable_psr)
+		psr_enable_request = false;
+	else if (active_psr_events & (psr_event_hw_programming |
+			psr_event_defer_enable |
+			psr_event_dds_defer_stream_enable |
+			psr_event_vrr_transition |
+			psr_event_immediate_flip))
+		psr_enable_request = false;
+	else if (active_psr_events & psr_event_big_screen_video)
+		psr_enable_request = true;
+	else if (active_psr_events & psr_event_full_screen)
+		psr_enable_request = false;
+	else if (active_psr_events & psr_event_mpo_video_selective_update)
+		psr_enable_request = true;
+	else if (active_psr_events & psr_event_vsync)
+		psr_enable_request = false;
+	else if (active_psr_events & psr_event_crc_window_active)
+		psr_enable_request = false;
+	else
+		psr_enable_request = true;
+
+	DC_TRACE_LEVEL_MESSAGE(DAL_TRACE_LEVEL_VERBOSE,
+						WPP_BIT_FLAG_Firmware_PsrState,
+						"mod_power set_psr_event: before: psr_enabled=%d -> request: set_event=%d event=0x%04x -> result: psr_events=0x%04x psr_enable_request=%d",
+						(int)core_power->map[stream_index].psr_enabled,
+						(int)set_event,
+						(unsigned int)event,
+						(unsigned int)core_power->map[stream_index].psr_events,
+						(int)psr_enable_request);
+	mod_power_psr_set_power_opt(mod_power, stream, active_psr_events, psr_enable_request);
+
+	if (core_power->map[stream_index].psr_enabled != psr_enable_request || force_static) {
+		if (set_psr_enable(mod_power, stream, psr_enable_request, wait, force_static))
+			core_power->map[stream_index].psr_enabled = psr_enable_request;
+	}
+
+	return true;
+}
+
+bool mod_power_get_psr_state(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		enum dc_psr_state *state)
+{
+	struct core_power *core_power = NULL;
+	const struct dc_link *link = NULL;
+
+	if (!stream)
+		return false;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	link = dc_stream_get_link(stream);
+	return dc_link_get_psr_state(link, state);
+}
+
+bool mod_power_get_psr_enabled_status(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		bool *psr_enabled)
+{
+	struct core_power *core_power = NULL;
+	unsigned int stream_index = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	stream_index = map_index_from_stream(core_power, stream);
+
+	if (!core_power->map[stream_index].caps->psr_version)
+		return false;
+
+	*psr_enabled = core_power->map[stream_index].psr_enabled;
+
+	return true;
+}
+
+void mod_power_psr_residency(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		unsigned int *residency,
+		const uint8_t mode)
+{
+	struct core_power *core_power = NULL;
+	const struct dc_link *link = NULL;
+
+	if (!stream)
+		return;
+
+	if (mod_power == NULL)
+		return;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return;
+
+	link = dc_stream_get_link(stream);
+
+	if (link != NULL)
+		link->dc->link_srv->edp_get_psr_residency(link, residency, mode);
+}
+bool mod_power_psr_get_active_psr_events(struct mod_power *mod_power,
+		const struct dc_stream_state *stream, unsigned int *active_psr_events)
+{
+	struct core_power *core_power = NULL;
+	unsigned int stream_index = 0;
+
+	if (!stream)
+		return false;
+
+	if (mod_power == NULL)
+		return false;
+
+	if (active_psr_events == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	stream_index = map_index_from_stream(core_power, stream);
+
+	*active_psr_events = core_power->map[stream_index].psr_events;
+	return true;
+}
+
+bool mod_power_psr_set_sink_vtotal_in_psr_active(struct mod_power *mod_power,
+		const struct dc_stream_state *stream,
+		uint16_t psr_vtotal_idle,
+		uint16_t psr_vtotal_su)
+{
+	struct core_power *core_power = NULL;
+	unsigned int stream_index = 0;
+	const struct dc_link *link = NULL;
+
+	if (!stream)
+		return false;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	stream_index = map_index_from_stream(core_power, stream);
+
+	if (!core_power->map[stream_index].caps->psr_version)
+		return false;
+
+	link = dc_stream_get_link(stream);
+
+	return link->dc->link_srv->edp_set_sink_vtotal_in_psr_active(
+			link, psr_vtotal_idle, psr_vtotal_su);
+}
+/*
+ * is_psr_su_specific_panel() - check if sink is AMD vendor-specific PSR-SU
+ * supported eDP device.
+ *
+ * @link: dc link pointer
+ *
+ * Return: true if AMDGPU vendor specific PSR-SU eDP panel
+ */
+bool is_psr_su_specific_panel(struct dc_link *link)
+{
+	bool isPSRSUSupported = false;
+	struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
+
+	if (dpcd_caps->edp_rev >= DP_EDP_14) {
+		if (dpcd_caps->psr_info.psr_version >= DP_PSR2_WITH_Y_COORD_ET_SUPPORTED)
+			isPSRSUSupported = true;
+		/*
+		 * Some panels will report PSR capabilities over additional DPCD bits.
+		 * Such panels are approved despite reporting only PSR v3, as long as
+		 * the additional bits are reported.
+		 */
+		if (dpcd_caps->sink_dev_id == DP_BRANCH_DEVICE_ID_001CF8) {
+			/*
+			 * This is the temporary workaround to disable PSRSU when system turned on
+			 * DSC function on the sepcific sink.
+			 */
+			if (dpcd_caps->psr_info.psr_version < DP_PSR2_WITH_Y_COORD_IS_SUPPORTED)
+				isPSRSUSupported = false;
+			else if (dpcd_caps->dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT &&
+				((dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x08) ||
+				(dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x07)))
+				isPSRSUSupported = false;
+			else if (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x03)
+				isPSRSUSupported = false;
+			else if (dpcd_caps->sink_dev_id_str[1] == 0x08 && dpcd_caps->sink_dev_id_str[0] == 0x01)
+				isPSRSUSupported = false;
+			else if (dpcd_caps->psr_info.force_psrsu_cap == 0x1)
+				isPSRSUSupported = true;
+		}
+	}
+
+	return isPSRSUSupported;
+}
+
+/**
+ * mod_power_calc_psr_configs() - calculate/update generic psr configuration fields.
+ * @psr_config: [output], psr configuration structure to be updated
+ * @link: [input] dc link pointer
+ * @stream: [input] dc stream state pointer
+ *
+ * calculate and update the psr configuration fields that are not DM specific, i.e. such
+ * fields which are based on DPCD caps or timing information. To setup PSR in DMUB FW,
+ * this helper is assumed to be called before the call of the DC helper dc_link_setup_psr().
+ *
+ * PSR config fields to be updated within the helper:
+ * - psr_rfb_setup_time
+ * - psr_sdp_transmit_line_num_deadline
+ * - line_time_in_us
+ * - su_y_granularity
+ * - su_granularity_required
+ * - psr_frame_capture_indication_req
+ * - psr_exit_link_training_required
+ *
+ * PSR config fields that are DM specific and NOT updated within the helper:
+ * - allow_smu_optimizations
+ * - allow_multi_disp_optimizations
+ */
+void mod_power_calc_psr_configs(struct psr_config *psr_config,
+		struct dc_link *link,
+		const struct dc_stream_state *stream)
+{
+	unsigned int num_vblank_lines = 0;
+	unsigned int vblank_time_in_us = 0;
+	unsigned int sdp_tx_deadline_in_us = 0;
+	unsigned int line_time_in_us = 0;
+	struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
+	const int psr_setup_time_step_in_us = 55;	/* refer to eDP spec DPCD 0x071h */
+
+	/* timing parameters */
+	num_vblank_lines = stream->timing.v_total -
+			 stream->timing.v_addressable -
+			 stream->timing.v_border_top -
+			 stream->timing.v_border_bottom;
+
+	vblank_time_in_us = (stream->timing.h_total * num_vblank_lines * 1000) / (stream->timing.pix_clk_100hz / 10);
+
+	line_time_in_us = ((stream->timing.h_total * 1000) / (stream->timing.pix_clk_100hz / 10)) + 1;
+
+	/**
+	 * psr configuration fields
+	 *
+	 * as per eDP 1.5 pg. 377 of 459, DPCD 0x071h bits [3:1], psr setup time bits interpreted as below
+	 * 000b <--> 330 us (default)
+	 * 001b <--> 275 us
+	 * 010b <--> 220 us
+	 * 011b <--> 165 us
+	 * 100b <--> 110 us
+	 * 101b <--> 055 us
+	 * 110b <--> 000 us
+	 */
+	psr_config->psr_rfb_setup_time =
+		(6 - dpcd_caps->psr_info.psr_dpcd_caps.bits.PSR_SETUP_TIME) * psr_setup_time_step_in_us;
+
+	if (psr_config->psr_rfb_setup_time > vblank_time_in_us) {
+		link->psr_settings.psr_frame_capture_indication_req = true;
+		link->psr_settings.psr_sdp_transmit_line_num_deadline = num_vblank_lines;
+	} else {
+		sdp_tx_deadline_in_us = vblank_time_in_us - psr_config->psr_rfb_setup_time;
+
+		/* Set the last possible line SDP may be transmitted without violating the RFB setup time */
+		link->psr_settings.psr_frame_capture_indication_req = false;
+		link->psr_settings.psr_sdp_transmit_line_num_deadline = sdp_tx_deadline_in_us / line_time_in_us;
+	}
+
+	psr_config->psr_sdp_transmit_line_num_deadline = link->psr_settings.psr_sdp_transmit_line_num_deadline;
+	psr_config->line_time_in_us = line_time_in_us;
+	psr_config->su_y_granularity = dpcd_caps->psr_info.psr2_su_y_granularity_cap;
+	psr_config->su_granularity_required = dpcd_caps->psr_info.psr_dpcd_caps.bits.SU_GRANULARITY_REQUIRED;
+	psr_config->psr_frame_capture_indication_req = link->psr_settings.psr_frame_capture_indication_req;
+	psr_config->psr_exit_link_training_required =
+		!link->dpcd_caps.psr_info.psr_dpcd_caps.bits.LINK_TRAINING_ON_EXIT_NOT_REQUIRED;
+}
+
+bool psr_su_set_dsc_slice_height(struct dc *dc, struct dc_link *link,
+			      struct dc_stream_state *stream,
+			      struct psr_config *config)
+{
+	uint32_t pic_height;
+	uint32_t slice_height;
+
+	config->dsc_slice_height = 0;
+	if (!(link->connector_signal & SIGNAL_TYPE_EDP) ||
+	    !dc->caps.edp_dsc_support ||
+	    link->panel_config.dsc.disable_dsc_edp ||
+	    !link->dpcd_caps.dsc_caps.dsc_basic_caps.fields.dsc_support.DSC_SUPPORT ||
+	    !stream->timing.dsc_cfg.num_slices_v)
+		return true;
+
+	pic_height = stream->timing.v_addressable +
+		stream->timing.v_border_top + stream->timing.v_border_bottom;
+
+	if (stream->timing.dsc_cfg.num_slices_v == 0)
+		return false;
+
+	slice_height = pic_height / stream->timing.dsc_cfg.num_slices_v;
+	config->dsc_slice_height = (uint16_t)slice_height;
+
+	if (slice_height) {
+		if (config->su_y_granularity &&
+		    (slice_height % config->su_y_granularity)) {
+			ASSERT(0);
+			return false;
+		}
+	}
+
+	return true;
+}
diff --git a/drivers/gpu/drm/amd/display/modules/power/power_replay.c b/drivers/gpu/drm/amd/display/modules/power/power_replay.c
new file mode 100644
index 0000000000000..983be9759e748
--- /dev/null
+++ b/drivers/gpu/drm/amd/display/modules/power/power_replay.c
@@ -0,0 +1,911 @@
+// SPDX-License-Identifier: MIT
+//
+// Copyright 2026 Advanced Micro Devices, Inc.
+
+#include "dm_services.h"
+#include "dc.h"
+#include "mod_power.h"
+#include "core_types.h"
+#include "dmcu.h"
+#include "abm.h"
+#include "power_helpers.h"
+#include "dce/dmub_psr.h"
+#include "dal_asic_id.h"
+#include "link_service.h"
+#include <linux/math.h>
+
+#define DC_TRACE_LEVEL_MESSAGE(...) /* do nothing */
+#define DC_TRACE_LEVEL_MESSAGEP(...) /* do nothing */
+#include "power_helpers.h"
+#include "dc/inc/hw/dmcu.h"
+#include "dc/inc/hw/abm.h"
+#include "dc.h"
+#include "core_types.h"
+#include "dmub_cmd.h"
+
+#define MOD_POWER_TO_CORE(mod_power)\
+		container_of(mod_power, struct core_power, mod_public)
+
+#define LOW_REFRESH_RATE_DURATION_US_UPPER_BOUND 25000
+
+static bool mod_power_set_replay_active(struct dc_stream_state *stream,
+	bool replay_active,
+	bool wait,
+	bool force_static)
+{
+	uint64_t state;
+	unsigned int retry_count;
+	const unsigned int max_retry = 1000;
+	struct dc_link *link = NULL;
+
+	if (!stream)
+		return false;
+
+	link = dc_stream_get_link(stream);
+
+	if (!link)
+		return false;
+
+	if (!dc_link_set_replay_allow_active(link, &replay_active, false, force_static, NULL))
+		return false;
+
+	if (wait == true) {
+
+		for (retry_count = 0; retry_count <= max_retry; retry_count++) {
+			dc_link_get_replay_state(link, &state);
+			if (replay_active) {
+				if (state != REPLAY_STATE_0 &&
+					(!force_static || state == REPLAY_STATE_3))
+					break;
+			} else {
+				if (state == REPLAY_STATE_0)
+					break;
+			}
+			udelay(500);
+		}
+
+		/* assert if max retry hit */
+		if (retry_count >= max_retry)
+			ASSERT(0);
+	} else {
+		/* To-do: Add trace log */
+	}
+
+	return true;
+}
+
+static unsigned int mod_power_replay_setup_power_opt(struct dc_link *link,
+	unsigned int active_replay_events, bool is_ultra_sleep_mode)
+{
+	unsigned int power_opt = 0;
+
+	if (is_ultra_sleep_mode) {
+		/* Static Screen */
+		power_opt |= (replay_power_opt_smu_opt_static_screen | replay_power_opt_z10_static_screen);
+	} else if (active_replay_events & replay_event_test_harness_ultra_sleep) {
+		power_opt |= replay_power_opt_z10_static_screen;
+	}
+
+	/* replay_power_opt_flag is a configuration parameter into the module that determines
+	 * which optimizations to enable during replay
+	 */
+	power_opt &= link->replay_settings.config.replay_power_opt_supported;
+
+	return power_opt;
+}
+
+static bool mod_power_replay_set_power_opt(struct mod_power *mod_power,
+	struct dc_stream_state *stream,
+	unsigned int active_replay_events,
+	bool is_ultra_sleep_mode)
+{
+	(void)mod_power;
+	struct dc_link *link = NULL;
+	unsigned int power_opt = 0;
+
+	if (!stream)
+		return false;
+
+	link = dc_stream_get_link(stream);
+
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return false;
+
+	power_opt = mod_power_replay_setup_power_opt(link, active_replay_events, is_ultra_sleep_mode);
+
+	if (!dc_link_set_replay_allow_active(link, NULL, false, false, &power_opt))
+		return false;
+
+	return true;
+}
+
+bool mod_power_get_replay_event(struct mod_power *mod_power,
+	struct dc_stream_state *stream,
+	unsigned int *active_replay_events)
+{
+	struct core_power *core_power = NULL;
+	unsigned int stream_index = 0;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	stream_index = map_index_from_stream(core_power, stream);
+
+	*active_replay_events = core_power->map[stream_index].replay_events;
+
+	return true;
+}
+
+static bool mod_power_update_replay_active_status(unsigned int active_replay_events,
+	struct dc_link *link, uint32_t *coasting_vtotal, bool *is_full_screen_video,
+	bool *is_ultra_sleep_mode, uint16_t *frame_skip_number, bool *is_video_playback)
+{
+	if (!link || !coasting_vtotal || !is_full_screen_video || !is_video_playback)
+		return false;
+
+	// Check coasting_vtotal_table has been updated.
+	if (!link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_STATIC]
+		|| !link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM])
+		return false;
+
+	unsigned int replay_enable_option =
+		link->replay_settings.config.replay_enable_option;
+
+	/* TODO: To support test harness and DDS event */
+
+	*coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM];
+	ASSERT(link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM] <= 0xFFFF);
+	*frame_skip_number = (uint16_t)link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM];
+
+	link->replay_settings.config.replay_timing_sync_supported = false;
+
+	*is_full_screen_video = false;
+
+	*is_ultra_sleep_mode = false;
+
+	*is_video_playback = false;
+
+	/* DSAT test scenario */
+	if (active_replay_events & replay_event_test_harness_mode) {
+		if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS])
+			*coasting_vtotal =
+				link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS];
+		if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) {
+			ASSERT(link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS] <= 0xFFFF);
+			*frame_skip_number =
+				(uint16_t)link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS];
+		}
+
+		/* During the ultra sleep mode testing, disable the timing sync in short vblank mode */
+		if (active_replay_events & (replay_event_test_harness_enable_replay)) {
+			if ((active_replay_events & replay_event_test_harness_ultra_sleep) &&
+				  !link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode)
+				link->replay_settings.config.replay_timing_sync_supported = false;
+			return true;
+		} else
+			return false;
+	} else if (active_replay_events & (replay_event_test_harness_enable_replay)) {
+		if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS])
+			*coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS];
+		if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) {
+			uint32_t frame_skip_val =
+				link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS];
+
+			ASSERT(frame_skip_val <= 0xFFFF);
+			*frame_skip_number = (uint16_t)frame_skip_val;
+		}
+
+		/* During the ultra sleep mode testing, disable the timing sync in short vblank mode */
+		if ((active_replay_events & replay_event_test_harness_ultra_sleep) &&
+			  !link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode) {
+			link->replay_settings.config.replay_timing_sync_supported = false;
+		}
+		return true;
+	} else if (active_replay_events &
+			(replay_event_test_harness_disable_replay | replay_event_os_request_disable)) {
+		// set last set coasting vtotal
+		if (link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS])
+			*coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_TEST_HARNESS];
+		if (link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS]) {
+			uint32_t frame_skip_val =
+				link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_TEST_HARNESS];
+
+			ASSERT(frame_skip_val <= 0xFFFF);
+			*frame_skip_number = (uint16_t)frame_skip_val;
+		}
+		return false;
+	}
+
+	/* Inactive conditions */
+	if (active_replay_events & (replay_event_edp_panel_off_disable_psr |
+			replay_event_hw_programming |
+			replay_event_vrr |
+			replay_event_immediate_flip |
+			replay_event_prepare_vtotal |
+			replay_event_vrr_transition |
+			replay_event_pause |
+			replay_event_disable_replay_while_DPMS |
+			replay_event_sleep_resume |
+			replay_event_disable_in_AC |
+			replay_event_disable_replay_while_detect_display |
+			replay_event_infopacket |
+			replay_event_crc_window_active))
+		return false;
+
+	// Full screen scenario
+	if (active_replay_events & replay_event_full_screen) {
+		if (!(replay_enable_option & pr_enable_option_full_screen))
+			return false;
+	}
+
+	/* Full screen video scenario */
+	if (active_replay_events & replay_event_big_screen_video) {
+
+		link->replay_settings.config.replay_timing_sync_supported = false;
+
+		if (replay_enable_option & pr_enable_option_full_screen_video_coasting) {
+			unsigned int fsn_vid =
+				link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_FULL_SCREEN_VIDEO];
+
+			*coasting_vtotal =
+				link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_FULL_SCREEN_VIDEO];
+			ASSERT(fsn_vid <= 0xFFFF);
+			*frame_skip_number = (uint16_t)fsn_vid;
+		}
+
+		*is_video_playback = true;
+
+		if ((replay_enable_option & pr_enable_option_full_screen_video) &&
+			(replay_enable_option & pr_enable_option_full_screen_video_coasting)) {
+			*is_full_screen_video = true;
+			return true;
+		} else
+			return false;
+	}
+
+	/* MPO video scenario
+	 * Some of the cases may contain a full screen UI layer in MPO video scenario which is
+	 * not the expected case to enable Replay.
+	 */
+	if ((active_replay_events & replay_event_mpo_video_selective_update) &&
+		!(active_replay_events & replay_event_full_screen)) {
+
+		link->replay_settings.config.replay_timing_sync_supported = false;
+
+		if (replay_enable_option & pr_enable_option_mpo_video_coasting) {
+			*coasting_vtotal = link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM];
+			{
+				uint32_t frame_skip_val =
+					link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM];
+
+				ASSERT(frame_skip_val <= 0xFFFF);
+				*frame_skip_number = (uint16_t)frame_skip_val;
+			}
+		}
+
+		*is_video_playback = true;
+
+		if (replay_enable_option & pr_enable_option_mpo_video)
+			return true;
+		else
+			return false;
+	}
+
+	/* Static screen scenario */
+	if (!(active_replay_events & replay_event_vsync)) {
+
+		if (replay_enable_option & pr_enable_option_static_screen_coasting) {
+			// Do not adjust eDP refresh rate if static screen + normal sleep mode
+			if ((!(link->replay_settings.config.replay_power_opt_supported &
+				replay_power_opt_z10_static_screen)) ||
+				(active_replay_events & replay_event_cursor_updating)) {
+				// normal sleep mode
+				*coasting_vtotal =
+					link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_NOM];
+				{
+					uint32_t frame_skip_val =
+						link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_NOM];
+
+					ASSERT(frame_skip_val <= 0xFFFF);
+					*frame_skip_number = (uint16_t)frame_skip_val;
+				}
+			} else {
+				// ultra sleep mode
+				*coasting_vtotal =
+					link->replay_settings.coasting_vtotal_table[PR_COASTING_TYPE_STATIC];
+				{
+					uint32_t frame_skip_val =
+						link->replay_settings.frame_skip_number_table[PR_COASTING_TYPE_STATIC];
+
+					ASSERT(frame_skip_val <= 0xFFFF);
+					*frame_skip_number = (uint16_t)frame_skip_val;
+				}
+				*is_ultra_sleep_mode = true;
+			}
+		}
+
+		if (replay_enable_option & pr_enable_option_static_screen) {
+			if (!link->replay_settings.config.replay_support_fast_resync_in_ultra_sleep_mode)
+				link->replay_settings.config.replay_timing_sync_supported = false;
+			return true;
+		} else
+			return false;
+	}
+
+	/* General UI scenario */
+	if (active_replay_events & replay_event_general_ui) {
+		if (replay_enable_option & pr_enable_option_general_ui)
+			return true;
+		else
+			return false;
+	}
+
+	return false;
+}
+
+bool mod_power_replay_set_coasting_vtotal(struct mod_power *mod_power,
+	const struct dc_stream_state *stream,
+	uint32_t coasting_vtotal,
+	uint16_t frame_skip_number)
+{
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+
+	if (!stream)
+		return false;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return false;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	return link->dc->link_srv->edp_set_coasting_vtotal(link, coasting_vtotal, frame_skip_number);
+}
+
+void mod_power_replay_set_timing_sync_supported(struct mod_power *mod_power,
+	const struct dc_stream_state *stream)
+{
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+	unsigned int stream_index = 0;
+	union dmub_replay_cmd_set cmd_data = { 0 };
+
+	if (!stream || mod_power == NULL)
+		return;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	if (core_power->num_entities == 0)
+		return;
+
+	stream_index = map_index_from_stream(core_power, stream);
+	if (stream_index > core_power->num_entities) //invalid index
+		return;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return;
+
+	cmd_data.sync_data.timing_sync_supported = link->replay_settings.config.replay_timing_sync_supported;
+
+	link->dc->link_srv->edp_send_replay_cmd(link, Replay_Set_Timing_Sync_Supported,
+		&cmd_data);
+}
+
+void mod_power_replay_disabled_adaptive_sync_sdp(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool force_disabled)
+{
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+	unsigned int stream_index = 0;
+	union dmub_replay_cmd_set cmd_data = { 0 };
+
+	if (!stream || mod_power == NULL)
+		return;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	if (core_power->num_entities == 0)
+		return;
+
+	stream_index = map_index_from_stream(core_power, stream);
+	if (stream_index > core_power->num_entities) //invalid index
+		return;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return;
+
+	cmd_data.disabled_adaptive_sync_sdp_data.force_disabled = force_disabled;
+
+	link->dc->link_srv->edp_send_replay_cmd(link, Replay_Disabled_Adaptive_Sync_SDP,
+		&cmd_data);
+}
+
+static void mod_power_replay_set_general_cmd(struct mod_power *mod_power,
+	const struct dc_stream_state *stream,
+	const enum dmub_cmd_replay_general_subtype general_cmd_type,
+	const uint32_t param1, const uint32_t param2)
+{
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+	unsigned int stream_index = 0;
+	union dmub_replay_cmd_set cmd_data = { 0 };
+
+	if (!stream || mod_power == NULL)
+		return;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	if (core_power->num_entities == 0)
+		return;
+
+	stream_index = map_index_from_stream(core_power, stream);
+	if (stream_index > core_power->num_entities) //invalid index
+		return;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return;
+
+	cmd_data.set_general_cmd_data.subtype = general_cmd_type;
+	cmd_data.set_general_cmd_data.param1 = param1;
+	cmd_data.set_general_cmd_data.param2 = param2;
+	link->dc->link_srv->edp_send_replay_cmd(link, Replay_Set_General_Cmd,
+		&cmd_data);
+}
+
+void mod_power_replay_disabled_desync_error_detection(struct mod_power *mod_power,
+	const struct dc_stream_state *stream,  bool force_disabled)
+{
+	mod_power_replay_set_general_cmd(mod_power, stream,
+			REPLAY_GENERAL_CMD_DISABLED_DESYNC_ERROR_DETECTION,
+			force_disabled, 0);
+}
+
+static void mod_power_replay_set_pseudo_vtotal(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, uint16_t vtotal)
+{
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+	unsigned int stream_index = 0;
+	union dmub_replay_cmd_set cmd_data = { 0 };
+
+	if (!stream || mod_power == NULL)
+		return;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	if (core_power->num_entities == 0)
+		return;
+
+	stream_index = map_index_from_stream(core_power, stream);
+	if (stream_index > core_power->num_entities) //invalid index
+		return;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return;
+
+	cmd_data.pseudo_vtotal_data.vtotal = vtotal;
+
+	if (link->replay_settings.last_pseudo_vtotal != vtotal) {
+		link->replay_settings.last_pseudo_vtotal = vtotal;
+		link->dc->link_srv->edp_send_replay_cmd(link, Replay_Set_Pseudo_VTotal, &cmd_data);
+	}
+}
+
+static void mod_power_update_error_status(struct mod_power *mod_power,
+	const struct dc_stream_state *stream)
+{
+	struct dc_link *link = NULL;
+	union replay_debug_flags *pDebug = NULL;
+
+	if (mod_power == NULL || stream == NULL)
+		return;
+
+	link = dc_stream_get_link(stream);
+
+	if (!link)
+		return;
+
+	pDebug = (union replay_debug_flags *)&link->replay_settings.config.debug_flags;
+
+	if (pDebug->bitfields.enable_visual_confirm_debug == 0)
+		return;
+
+	mod_power_replay_set_general_cmd(mod_power, stream,
+		REPLAY_GENERAL_CMD_UPDATE_ERROR_STATUS,
+		link->replay_settings.config.replay_error_status.raw, 0);
+}
+
+void mod_power_set_low_rr_activate(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool low_rr_supported)
+{
+	struct dc_link *link = NULL;
+
+	if (mod_power == NULL || stream == NULL)
+		return;
+
+	link = dc_stream_get_link(stream);
+
+	if (!link)
+		return;
+
+	mod_power_replay_set_general_cmd(mod_power, stream,
+		REPLAY_GENERAL_CMD_SET_LOW_RR_ACTIVATE,
+		low_rr_supported, 0);
+}
+
+void mod_power_set_video_conferencing_activate(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool video_conferencing_activate)
+{
+	struct dc_link *link = NULL;
+
+	if (mod_power == NULL || stream == NULL)
+		return;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return;
+
+	mod_power_replay_set_general_cmd(mod_power, stream,
+		REPLAY_GENERAL_CMD_VIDEO_CONFERENCING,
+		video_conferencing_activate, 0);
+}
+
+void mod_power_set_coasting_vtotal_without_frame_update(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, uint32_t coasting_vtotal)
+{
+	struct dc_link *link = NULL;
+
+	if (mod_power == NULL || stream == NULL)
+		return;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return;
+
+	mod_power_replay_set_general_cmd(mod_power, stream,
+		REPLAY_GENERAL_CMD_SET_COASTING_VTOTAL_WITHOUT_FRAME_UPDATE,
+		coasting_vtotal, 0);
+}
+
+void mod_power_set_replay_continuously_resync(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool enable)
+{
+	struct dc_link *link = NULL;
+
+	if (mod_power == NULL || stream == NULL)
+		return;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return;
+
+	mod_power_replay_set_general_cmd(mod_power, stream,
+		REPLAY_GENERAL_CMD_SET_CONTINUOUSLY_RESYNC,
+		enable, 0);
+}
+
+void mod_power_set_live_capture_with_cvt_activate(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, bool live_capture_with_cvt_activate)
+{
+	struct dc_link *link = NULL;
+
+	if (mod_power == NULL || stream == NULL)
+		return;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return;
+
+	// Check if LIVE_CAPTURE_WITH_CVT bit is enabled in replay optimization config
+	if (!link->replay_settings.config.replay_optimization.bits.LIVE_CAPTURE_WITH_CVT)
+		return;
+
+	if (link->replay_settings.config.live_capture_with_cvt_activated != live_capture_with_cvt_activate) {
+		link->replay_settings.config.live_capture_with_cvt_activated = live_capture_with_cvt_activate;
+		mod_power_replay_set_general_cmd(mod_power, stream,
+			REPLAY_GENERAL_CMD_LIVE_CAPTURE_WITH_CVT,
+			live_capture_with_cvt_activate, 0);
+	}
+}
+
+bool mod_power_set_replay_event(struct mod_power *mod_power,
+	struct dc_stream_state *stream, bool set_event,
+	enum replay_event event, bool wait_for_disable)
+{
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+	unsigned int stream_index = 0;
+	unsigned int active_replay_events = 0;
+	bool replay_active_request = false;
+	bool force_static = false;
+	uint32_t coasting_vtotal = 0;
+	bool current_timing_sync_status = false;
+	bool is_full_screen_video = false;
+	bool is_ultra_sleep_mode = false;
+	unsigned int sink_duration_us = 0;
+	bool low_rr_active = false;
+	uint16_t frame_skip_number = 0;
+	bool is_video_playback = false;
+
+	if (!stream)
+		return false;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	stream_index = map_index_from_stream(core_power, stream);
+
+	if (set_event)
+		core_power->map[stream_index].replay_events |= event;
+	else
+		core_power->map[stream_index].replay_events &= ~event;
+
+	link = dc_stream_get_link(stream);
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return false;
+
+	if ((core_power->map[stream_index].replay_events & replay_event_disable_replay_while_switching_mux) != 0)
+		return false;
+
+	if ((core_power->map[stream_index].replay_events & replay_event_os_override_hold) != 0)
+		return false;
+
+	active_replay_events = core_power->map[stream_index].replay_events;
+
+	current_timing_sync_status =
+		link->replay_settings.config.replay_timing_sync_supported;
+
+	replay_active_request = mod_power_update_replay_active_status(active_replay_events,
+		link, &coasting_vtotal, &is_full_screen_video, &is_ultra_sleep_mode, &frame_skip_number, &is_video_playback);
+
+	if (is_full_screen_video)
+		mod_power_replay_set_pseudo_vtotal(mod_power, stream,
+			link->replay_settings.low_rr_full_screen_video_pseudo_vtotal);
+	else
+		mod_power_replay_set_pseudo_vtotal(mod_power, stream, 0);
+
+	//If timing_sync_status change, then re-enabled set timing_sync_supported value and re-enabled replay
+	if (current_timing_sync_status != link->replay_settings.config.replay_timing_sync_supported)
+		mod_power_replay_set_timing_sync_supported(mod_power, stream);
+
+	if (link->replay_settings.config.low_rr_supported) {
+		sink_duration_us =
+			(unsigned int)(div_u64(((unsigned long long)(coasting_vtotal)
+				* 10000) * stream->timing.h_total,
+					stream->timing.pix_clk_100hz));
+		low_rr_active = sink_duration_us < LOW_REFRESH_RATE_DURATION_US_UPPER_BOUND ? false : true;
+		if (low_rr_active != link->replay_settings.config.low_rr_activated) {
+			mod_power_set_low_rr_activate(mod_power, stream, low_rr_active);
+			link->replay_settings.config.low_rr_activated = low_rr_active;
+		}
+	}
+
+	// The function return fail when
+	// 1. DMUB function is not support (for backward compatible).
+	// 2. active_replay_events or coasting_vtotal is not updated in the same time
+	if (!mod_power_replay_set_power_opt_and_coasting_vtotal(mod_power,
+		stream, active_replay_events, coasting_vtotal, is_ultra_sleep_mode, frame_skip_number)) {
+		if (!mod_power_replay_set_power_opt(mod_power, stream, active_replay_events, is_ultra_sleep_mode))
+			return false;
+
+		if (!mod_power_replay_set_coasting_vtotal(mod_power, stream, coasting_vtotal, frame_skip_number))
+			return false;
+	}
+
+	mod_power_set_live_capture_with_cvt_activate(mod_power, stream, is_video_playback);
+
+	mod_power_update_error_status(mod_power, stream);
+
+	// If Replay is going to be enable (No matter is disable -> enable or enable -> enable), we don't need to wait.
+	// If Replay is going to be disable
+	//     if disable -> disable
+	//         -> Replay DMUB state should be state 0.
+	//            So no matter wait_for_disable is true or not, it should makes no difference.
+	//     if enable -> disable -> We should wait if wait_for_disable is true.
+	if (replay_active_request)
+		wait_for_disable = false;
+
+	if (!mod_power_set_replay_active(stream, replay_active_request, wait_for_disable, force_static))
+		return false;
+
+	return true;
+}
+
+bool mod_power_get_replay_active_status(const struct dc_stream_state *stream,
+	bool *replay_active)
+{
+	const struct dc_link *link = NULL;
+
+	if (!stream)
+		return false;
+
+	link = dc_stream_get_link(stream);
+	*replay_active = link->replay_settings.replay_allow_active;
+
+	return true;
+}
+
+void mod_power_replay_residency(const struct dc_stream_state *stream,
+	unsigned int *residency, const bool is_start, const bool is_alpm)
+{
+	const struct dc_link *link = NULL;
+	enum pr_residency_mode mode;
+
+	if (!stream)
+		return;
+
+	link = dc_stream_get_link(stream);
+
+	if (is_alpm)
+		mode = PR_RESIDENCY_MODE_ALPM;
+	else
+		mode = PR_RESIDENCY_MODE_PHY;
+
+	if (link && link->dc && link->dc->link_srv)
+		link->dc->link_srv->edp_replay_residency(link, residency, is_start, mode);
+}
+
+bool mod_power_replay_set_power_opt_and_coasting_vtotal(struct mod_power *mod_power,
+	const struct dc_stream_state *stream, unsigned int active_replay_events, uint32_t coasting_vtotal,
+	bool is_ultra_sleep_mode, uint16_t frame_skip_number)
+{
+	struct core_power *core_power = NULL;
+	struct dc_link *link = NULL;
+	unsigned int power_opt = 0;
+
+	if (!stream)
+		return false;
+
+	if (mod_power == NULL)
+		return false;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+
+	if (core_power->num_entities == 0)
+		return false;
+
+	link = dc_stream_get_link(stream);
+
+	if (!link || !link->replay_settings.replay_feature_enabled)
+		return false;
+
+	power_opt = mod_power_replay_setup_power_opt(link, active_replay_events, is_ultra_sleep_mode);
+
+	return link->dc->link_srv->edp_set_replay_power_opt_and_coasting_vtotal(link, &power_opt, coasting_vtotal, frame_skip_number);
+}
+
+void mod_power_replay_notify_mode_change(struct mod_power *mod_power,
+	struct dc *dc,
+	struct dc_link *link,
+	const struct dc_stream_state *stream,
+	unsigned int stream_index)
+{
+	struct core_power *core_power = NULL;
+	int active_replay_events = 0;
+
+	if (!mod_power || !dc || !link || !stream)
+		return;
+
+	core_power = MOD_POWER_TO_CORE(mod_power);
+	active_replay_events = core_power->map[stream_index].replay_events;
+
+	link->replay_settings.replay_smu_opt_enable =
+		(link->replay_settings.config.replay_smu_opt_supported &&
+		mod_power_only_edp(dc->current_state, stream));
+
+	if (active_replay_events & replay_event_os_request_force_ffu) {
+		link->replay_settings.config.os_request_force_ffu = true;
+	}
+
+	if (dc_is_embedded_signal(stream->signal))
+		dc->link_srv->dp_setup_replay(link, stream);
+}
+
+void init_replay_config(struct dc_link *link, struct replay_config *pr_config)
+{
+	link->replay_settings.config = *pr_config;
+}
+
+void set_replay_frame_skip_number(struct dc_link *link,
+	enum replay_coasting_vtotal_type type,
+	uint32_t coasting_vtotal_refresh_rate_uhz,
+	uint32_t flicker_free_refresh_rate_uhz,
+	bool is_defer)
+{
+	uint32_t *frame_skip_number_array = NULL;
+	uint32_t frame_skip_number = 0;
+
+	if (link == NULL)
+		return;
+
+	if (link->replay_settings.config.frame_skip_supported == false)
+		return;
+
+	if (flicker_free_refresh_rate_uhz == 0 || coasting_vtotal_refresh_rate_uhz == 0)
+		return;
+
+	if (is_defer)
+		frame_skip_number_array = link->replay_settings.defer_frame_skip_number_table;
+	else
+		frame_skip_number_array = link->replay_settings.frame_skip_number_table;
+
+	if (frame_skip_number_array == NULL)
+		return;
+
+	frame_skip_number = (coasting_vtotal_refresh_rate_uhz + 500000) / flicker_free_refresh_rate_uhz;
+
+	if (frame_skip_number >= 1)
+		frame_skip_number_array[type] = frame_skip_number - 1;
+	else
+		frame_skip_number_array[type] = 0;
+}
+
+void set_replay_defer_update_coasting_vtotal(struct dc_link *link,
+	enum replay_coasting_vtotal_type type,
+	uint32_t vtotal)
+{
+	link->replay_settings.defer_update_coasting_vtotal_table[type] = vtotal;
+}
+
+void update_replay_coasting_vtotal_from_defer(struct dc_link *link,
+	enum replay_coasting_vtotal_type type)
+{
+	link->replay_settings.coasting_vtotal_table[type] =
+		link->replay_settings.defer_update_coasting_vtotal_table[type];
+	link->replay_settings.frame_skip_number_table[type] =
+		link->replay_settings.defer_frame_skip_number_table[type];
+}
+
+void set_replay_coasting_vtotal(struct dc_link *link,
+	enum replay_coasting_vtotal_type type,
+	uint32_t vtotal)
+{
+	link->replay_settings.coasting_vtotal_table[type] = vtotal;
+}
+
+void set_replay_low_rr_full_screen_video_src_vtotal(struct dc_link *link, uint16_t vtotal)
+{
+	link->replay_settings.low_rr_full_screen_video_pseudo_vtotal = vtotal;
+}
+
+void calculate_replay_link_off_frame_count(struct dc_link *link,
+	uint16_t vtotal, uint16_t htotal)
+{
+	uint32_t max_link_off_frame_count = 0;
+	uint16_t max_deviation_line = 0,  pixel_deviation_per_line = 0;
+
+	if (!link || link->replay_settings.config.replay_version != DC_FREESYNC_REPLAY)
+		return;
+
+	max_deviation_line = link->dpcd_caps.pr_info.max_deviation_line;
+	pixel_deviation_per_line = link->dpcd_caps.pr_info.pixel_deviation_per_line;
+
+	if (htotal != 0 && vtotal != 0 && pixel_deviation_per_line != 0)
+		max_link_off_frame_count = htotal * max_deviation_line / (pixel_deviation_per_line * vtotal);
+	else
+		ASSERT(0);
+
+	link->replay_settings.link_off_frame_count = max_link_off_frame_count;
+}
+
+void reset_replay_dsync_error_count(struct dc_link *link)
+{
+	link->replay_settings.replay_desync_error_fail_count = 0;
+}
diff --git a/drivers/gpu/drm/amd/display/modules/vmid/vmid.c b/drivers/gpu/drm/amd/display/modules/vmid/vmid.c
index 179b505f7777c..32617893ffb29 100644
--- a/drivers/gpu/drm/amd/display/modules/vmid/vmid.c
+++ b/drivers/gpu/drm/amd/display/modules/vmid/vmid.c
@@ -56,7 +56,7 @@ static void clear_entry_from_vmid_table(struct core_vmid *core_vmid, unsigned in
 
 static void evict_vmids(struct core_vmid *core_vmid)
 {
-	int i;
+	unsigned int i;
 	int ord_int = dc_get_vmid_use_vector(core_vmid->dc);
 
 	ASSERT(ord_int >= 0 && ord_int <= 0xFFFF);
@@ -72,7 +72,7 @@ static void evict_vmids(struct core_vmid *core_vmid)
 // Return value of -1 indicates vmid table uninitialized or ptb dne in the table
 static int get_existing_vmid_for_ptb(struct core_vmid *core_vmid, uint64_t ptb)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 0; i < core_vmid->num_vmid; i++) {
 		if (core_vmid->ptb_assigned_to_vmid[i] == ptb)
@@ -85,7 +85,7 @@ static int get_existing_vmid_for_ptb(struct core_vmid *core_vmid, uint64_t ptb)
 // Expected to be called only when there's an available vmid
 static int get_next_available_vmid(struct core_vmid *core_vmid)
 {
-	int i;
+	unsigned int i;
 
 	for (i = 1; i < core_vmid->num_vmid; i++) {
 		if (core_vmid->ptb_assigned_to_vmid[i] == 0)
diff --git a/drivers/gpu/drm/amd/include/amd_cper.h b/drivers/gpu/drm/amd/include/amd_cper.h
index a252ee4c7874c..baa0f30b9c4db 100644
--- a/drivers/gpu/drm/amd/include/amd_cper.h
+++ b/drivers/gpu/drm/amd/include/amd_cper.h
@@ -58,7 +58,7 @@
 
 enum cper_error_severity {
 	CPER_SEV_NON_FATAL_UNCORRECTED = 0,
-	CPER_SEV_FATAL                 = 1,
+	CPER_SEV_FATAL_UNCORRECTED     = 1,
 	CPER_SEV_NON_FATAL_CORRECTED   = 2,
 	CPER_SEV_NUM                   = 3,
 
diff --git a/drivers/gpu/drm/amd/include/amdgpu_ptl.h b/drivers/gpu/drm/amd/include/amdgpu_ptl.h
new file mode 100644
index 0000000000000..154b8da3bfa92
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/amdgpu_ptl.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2026 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef __AMDGPU_PTL_H__
+#define __AMDGPU_PTL_H__
+
+enum amdgpu_ptl_fmt {
+	AMDGPU_PTL_FMT_I8   = 0,
+	AMDGPU_PTL_FMT_F16  = 1,
+	AMDGPU_PTL_FMT_BF16 = 2,
+	AMDGPU_PTL_FMT_F32  = 3,
+	AMDGPU_PTL_FMT_F64  = 4,
+	AMDGPU_PTL_FMT_F8   = 5,
+	AMDGPU_PTL_FMT_VECTOR  = 6,
+	AMDGPU_PTL_FMT_INVALID = 7,
+};
+
+enum amdgpu_ptl_disable_source {
+	AMDGPU_PTL_DISABLE_SYSFS = 0,
+	AMDGPU_PTL_DISABLE_PROFILER,
+	AMDGPU_PTL_DISABLE_MAX,
+};
+struct amdgpu_ptl {
+	enum amdgpu_ptl_fmt		fmt1;
+	enum amdgpu_ptl_fmt		fmt2;
+	bool				enabled;
+	bool				hw_supported;
+	bool				permanently_disabled;
+	/* PTL disable reference counting */
+	atomic_t			disable_ref;
+	struct mutex			mutex;
+	DECLARE_BITMAP(disable_bitmap, AMDGPU_PTL_DISABLE_MAX);
+	bool				ptl_sysfs_created;
+};
+
+int amdgpu_ptl_perf_monitor_ctrl(struct amdgpu_device *adev, u32 req_code,
+		u32 *ptl_state,
+		enum amdgpu_ptl_fmt *fmt1,
+		enum amdgpu_ptl_fmt *fmt2);
+
+int amdgpu_ptl_sysfs_init(struct amdgpu_device *adev);
+void amdgpu_ptl_sysfs_fini(struct amdgpu_device *adev);
+
+extern const struct attribute_group amdgpu_ptl_attr_group;
+#endif /* __AMDGPU_PTL_H__ */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h
index f32649047374a..904518791b62c 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_offset.h
@@ -15871,6 +15871,113 @@
 #define mmDC_PERFMON28_PERFMON_LOW                                                                     0x08cf
 #define mmDC_PERFMON28_PERFMON_LOW_BASE_IDX                                                            3
 
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define mmHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define mmHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define mmHDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL                                                        0x08d4
+#define mmHDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL_BASE_IDX                                               3
+#define mmHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define mmHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define mmHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define mmHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define mmHDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL                                                        0x08d4
+#define mmHDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL_BASE_IDX                                               3
+#define mmHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define mmHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define mmHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define mmHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define mmHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define mmHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define mmHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define mmHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define mmHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define mmHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define mmHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define mmHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define mmHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define mmHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define mmHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define mmHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define mmHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define mmHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define mmHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define mmHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define mmHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define mmHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define mmHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define mmHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define mmHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define mmHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define mmHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define mmHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define mmHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define mmHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define mmHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define mmHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define mmHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define mmHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define mmHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define mmHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define mmHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define mmHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define mmHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define mmHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define mmHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define mmHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_MODE                                                                             0x0908
+#define mmHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define mmHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define mmHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define mmHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define mmHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
 // base address: 0x2646c
 #define mmAFMT6_AFMT_VBI_PACKET_CONTROL                                                                0x091c
 #define mmAFMT6_AFMT_VBI_PACKET_CONTROL_BASE_IDX                                                       3
@@ -15934,6 +16041,28 @@
 #define mmVPG6_VPG_MPEG_INFO1                                                                          0x093a
 #define mmVPG6_VPG_MPEG_INFO1_BASE_IDX                                                                 3
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define mmHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define mmHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define mmHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define mmHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define mmHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define mmHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define mmHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define mmHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define mmHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define mmHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define mmHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define mmHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
+// addressBlock: dce_dc_hpo_hpo_top_dispdec
+// base address: 0x0
+#define mmHPO_TOP_CLOCK_CONTROL                                                                        0x0e43
+#define mmHPO_TOP_CLOCK_CONTROL_BASE_IDX                                                               3
 
 // base address: 0x264f0
 #define mmDME6_DME_CONTROL                                                                             0x093c
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h
index 4005c73c2c9fc..92c3ba54effa3 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_0_sh_mask.h
@@ -60370,6 +60370,496 @@
 #define DC_PERFMON28_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
 #define DC_PERFMON28_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
 
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK__SHIFT                                 0x8
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK_MASK                                   0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+//HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_INCREMENT__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_MODULO__SHIFT                    0x8
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_INCREMENT_MASK                   0x000000FFL
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_MODULO_MASK                      0x0000FF00L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
+
+
 //AFMT6_AFMT_VBI_PACKET_CONTROL
 #define AFMT6_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_PACKETS_PER_LINE__SHIFT                                0x10
 #define AFMT6_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_SEND_MAX_PACKETS__SHIFT                                0x18
@@ -60736,10 +61226,120 @@
 #define DME6_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
 #define DME6_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
 
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_hdcp2_hdcp2_dispdec
+//HDCP2_6_HDCP2_CONTROL
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_RESET__SHIFT                                                             0x4
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_TYPE__SHIFT                                                              0x8
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_EESS_WHEN_AVMUTE__SHIFT                                                  0x10
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_ESS_ENABLE__SHIFT                                                        0x14
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_ESS_TYPE__SHIFT                                                          0x18
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_RESET_MASK                                                               0x00000010L
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_TYPE_MASK                                                                0x00000300L
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_EESS_WHEN_AVMUTE_MASK                                                    0x00010000L
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_ESS_ENABLE_MASK                                                          0x00100000L
+#define HDCP2_6_HDCP2_CONTROL__HDCP2_ESS_TYPE_MASK                                                            0x01000000L
+//HDCP2_6_HDCP2_ENABLE_DELAY
+#define HDCP2_6_HDCP2_ENABLE_DELAY__HDCP2_ENABLE_DELAY__SHIFT                                                 0x0
+#define HDCP2_6_HDCP2_ENABLE_DELAY__HDCP2_ENABLE_DELAY_MASK                                                   0x000003FFL
+//HDCP2_6_HDCP2_RIV0
+#define HDCP2_6_HDCP2_RIV0__HDCP2_RIV0__SHIFT                                                                 0x0
+#define HDCP2_6_HDCP2_RIV0__HDCP2_RIV0_MASK                                                                   0xFFFFFFFFL
+//HDCP2_6_HDCP2_RIV1
+#define HDCP2_6_HDCP2_RIV1__HDCP2_RIV1__SHIFT                                                                 0x0
+#define HDCP2_6_HDCP2_RIV1__HDCP2_RIV1_MASK                                                                   0xFFFFFFFFL
+//HDCP2_6_HDCP2_KS_XOR_LC128_0
+#define HDCP2_6_HDCP2_KS_XOR_LC128_0__HDCP2_KS_XOR_LC128_0__SHIFT                                             0x0
+#define HDCP2_6_HDCP2_KS_XOR_LC128_0__HDCP2_KS_XOR_LC128_0_MASK                                               0xFFFFFFFFL
+//HDCP2_6_HDCP2_KS_XOR_LC128_1
+#define HDCP2_6_HDCP2_KS_XOR_LC128_1__HDCP2_KS_XOR_LC128_1__SHIFT                                             0x0
+#define HDCP2_6_HDCP2_KS_XOR_LC128_1__HDCP2_KS_XOR_LC128_1_MASK                                               0xFFFFFFFFL
+//HDCP2_6_HDCP2_KS_XOR_LC128_2
+#define HDCP2_6_HDCP2_KS_XOR_LC128_2__HDCP2_KS_XOR_LC128_2__SHIFT                                             0x0
+#define HDCP2_6_HDCP2_KS_XOR_LC128_2__HDCP2_KS_XOR_LC128_2_MASK                                               0xFFFFFFFFL
+//HDCP2_6_HDCP2_KS_XOR_LC128_3
+#define HDCP2_6_HDCP2_KS_XOR_LC128_3__HDCP2_KS_XOR_LC128_3__SHIFT                                             0x0
+#define HDCP2_6_HDCP2_KS_XOR_LC128_3__HDCP2_KS_XOR_LC128_3_MASK                                               0xFFFFFFFFL
+
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
 // addressBlock: dce_dc_hpo_hpo_top_dispdec
 //HPO_TOP_CLOCK_CONTROL
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_R_GATE_DIS__SHIFT                                                  0x0
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_GATE_DIS__SHIFT                                                    0x1
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_R_GATE_DIS__SHIFT                                                   0x4
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_GATE_DIS__SHIFT                                                     0x5
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_R_GATE_DIS__SHIFT                                            0x8
 #define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_GATE_DIS__SHIFT                                              0x9
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_R_GATE_DIS__SHIFT                                              0xc
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_GATE_DIS__SHIFT                                                0xd
+#define HPO_TOP_CLOCK_CONTROL__HPO_TEST_CLK_SEL__SHIFT                                                        0x10
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_R_GATE_DIS_MASK                                                    0x00000001L
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_GATE_DIS_MASK                                                      0x00000002L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_R_GATE_DIS_MASK                                                     0x00000010L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_GATE_DIS_MASK                                                       0x00000020L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_R_GATE_DIS_MASK                                              0x00000100L
 #define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_GATE_DIS_MASK                                                0x00000200L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_R_GATE_DIS_MASK                                                0x00001000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_GATE_DIS_MASK                                                  0x00002000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_TEST_CLK_SEL_MASK                                                          0x003F0000L
 
 // addressBlock: dce_dc_hpo_hpo_dcperfmon_dc_perfmon_dispdec
 //DC_PERFMON29_PERFCOUNTER_CNTL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_offset.h
index b2962b5ce31e8..47f09e233f476 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_offset.h
@@ -14303,12 +14303,124 @@
 #define mmDC_PERFMON25_PERFMON_HI_BASE_IDX                                                             3
 #define mmDC_PERFMON25_PERFMON_LOW                                                                     0x08cf
 #define mmDC_PERFMON25_PERFMON_LOW_BASE_IDX                                                            3
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define mmHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define mmHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define mmHDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL                                                        0x08d4
+#define mmHDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL_BASE_IDX                                               3
+#define mmHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define mmHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define mmHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define mmHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define mmHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define mmHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define mmHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define mmHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define mmHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define mmHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define mmHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define mmHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define mmHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define mmHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define mmHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define mmHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define mmHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define mmHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define mmHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define mmHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define mmHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define mmHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define mmHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define mmHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define mmHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define mmHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define mmHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define mmHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define mmHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define mmHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define mmHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define mmHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define mmHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define mmHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define mmHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define mmHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define mmHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define mmHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define mmHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define mmHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define mmHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define mmHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_MODE                                                                             0x0908
+#define mmHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define mmHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define mmHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define mmHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define mmHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
+
 // base address: 0x264f0
 #define mmDME5_DME_CONTROL                                                                             0x093c
 #define mmDME5_DME_CONTROL_BASE_IDX                                                                    3
 #define mmDME5_DME_MEMORY_CONTROL                                                                      0x093d
 #define mmDME5_DME_MEMORY_CONTROL_BASE_IDX                                                             3
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define mmHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define mmHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define mmHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define mmHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define mmHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define mmHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define mmHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define mmHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define mmHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define mmHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define mmHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define mmHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_hdcp2_hdcp2_dispdec
 // base address: 0x264f8
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_sh_mask.h
index 7f8f0a6464229..e6e0d1bb96142 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_2_sh_mask.h
@@ -52453,10 +52453,1917 @@
 #define DC_PERFMON25_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
 #define DC_PERFMON25_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
 
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
+//AFMT5_AFMT_ACP
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE__SHIFT                                                                  0x0
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE0__SHIFT                                                  0x8
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE1__SHIFT                                                  0x10
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_MASK                                                                    0x00000003L
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE0_MASK                                                    0x0000FF00L
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE1_MASK                                                    0x00FF0000L
+//AFMT5_AFMT_VBI_PACKET_CONTROL
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_ACP_SOURCE__SHIFT                                                 0xd
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_PACKETS_PER_LINE__SHIFT                                0x10
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_SEND_MAX_PACKETS__SHIFT                                0x18
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_ACP_SOURCE_MASK                                                   0x00002000L
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_PACKETS_PER_LINE_MASK                                  0x001F0000L
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_SEND_MAX_PACKETS_MASK                                  0x01000000L
+//AFMT5_AFMT_AUDIO_PACKET_CONTROL2
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_OVRD__SHIFT                                       0x0
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_SELECT__SHIFT                                     0x1
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_CHANNEL_ENABLE__SHIFT                                    0x8
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_DP_AUDIO_STREAM_ID__SHIFT                                      0x10
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_HBR_ENABLE_OVRD__SHIFT                                         0x18
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_60958_OSF_OVRD__SHIFT                                          0x1c
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_OVRD_MASK                                         0x00000001L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_SELECT_MASK                                       0x00000002L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_CHANNEL_ENABLE_MASK                                      0x0000FF00L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_DP_AUDIO_STREAM_ID_MASK                                        0x00FF0000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_HBR_ENABLE_OVRD_MASK                                           0x01000000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_60958_OSF_OVRD_MASK                                            0x10000000L
+//AFMT5_AFMT_AUDIO_INFO0
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM__SHIFT                                               0x0
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CC__SHIFT                                                     0x8
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CT__SHIFT                                                     0xb
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_OFFSET__SHIFT                                        0x10
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CXT__SHIFT                                                    0x18
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_MASK                                                 0x000000FFL
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CC_MASK                                                       0x00000700L
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CT_MASK                                                       0x00007800L
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_OFFSET_MASK                                          0x00FF0000L
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CXT_MASK                                                      0x1F000000L
+//AFMT5_AFMT_AUDIO_INFO1
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_CA__SHIFT                                                     0x0
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LSV__SHIFT                                                    0xb
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_DM_INH__SHIFT                                                 0xf
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LFEPBL__SHIFT                                                 0x10
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_CA_MASK                                                       0x000000FFL
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LSV_MASK                                                      0x00007800L
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_DM_INH_MASK                                                   0x00008000L
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LFEPBL_MASK                                                   0x00030000L
+//AFMT5_AFMT_60958_0
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_A__SHIFT                                                            0x0
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_B__SHIFT                                                            0x1
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_C__SHIFT                                                            0x2
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_D__SHIFT                                                            0x3
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_MODE__SHIFT                                                         0x6
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CATEGORY_CODE__SHIFT                                                0x8
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_SOURCE_NUMBER__SHIFT                                                0x10
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CHANNEL_NUMBER_L__SHIFT                                             0x14
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_SAMPLING_FREQUENCY__SHIFT                                           0x18
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY__SHIFT                                               0x1c
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_A_MASK                                                              0x00000001L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_B_MASK                                                              0x00000002L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_C_MASK                                                              0x00000004L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_D_MASK                                                              0x00000038L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_MODE_MASK                                                           0x000000C0L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CATEGORY_CODE_MASK                                                  0x0000FF00L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_SOURCE_NUMBER_MASK                                                  0x000F0000L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CHANNEL_NUMBER_L_MASK                                               0x00F00000L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_SAMPLING_FREQUENCY_MASK                                             0x0F000000L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY_MASK                                                 0x30000000L
+//AFMT5_AFMT_60958_1
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_WORD_LENGTH__SHIFT                                                  0x0
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT                                  0x4
+#define AFMT5_AFMT_60958_1__AFMT_60958_VALID_L__SHIFT                                                         0x10
+#define AFMT5_AFMT_60958_1__AFMT_60958_VALID_R__SHIFT                                                         0x12
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_CHANNEL_NUMBER_R__SHIFT                                             0x14
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_WORD_LENGTH_MASK                                                    0x0000000FL
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK                                    0x000000F0L
+#define AFMT5_AFMT_60958_1__AFMT_60958_VALID_L_MASK                                                           0x00010000L
+#define AFMT5_AFMT_60958_1__AFMT_60958_VALID_R_MASK                                                           0x00040000L
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_CHANNEL_NUMBER_R_MASK                                               0x00F00000L
+//AFMT5_AFMT_AUDIO_CRC_CONTROL
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_EN__SHIFT                                                0x0
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CONT__SHIFT                                              0x4
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_SOURCE__SHIFT                                            0x8
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CH_SEL__SHIFT                                            0xc
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_COUNT__SHIFT                                             0x10
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_EN_MASK                                                  0x00000001L
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CONT_MASK                                                0x00000010L
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_SOURCE_MASK                                              0x00000100L
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CH_SEL_MASK                                              0x0000F000L
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_COUNT_MASK                                               0xFFFF0000L
+//AFMT5_AFMT_RAMP_CONTROL0
+#define AFMT5_AFMT_RAMP_CONTROL0__AFMT_RAMP_MAX_COUNT__SHIFT                                                  0x0
+#define AFMT5_AFMT_RAMP_CONTROL0__AFMT_RAMP_DATA_SIGN__SHIFT                                                  0x1f
+#define AFMT5_AFMT_RAMP_CONTROL0__AFMT_RAMP_MAX_COUNT_MASK                                                    0x00FFFFFFL
+#define AFMT5_AFMT_RAMP_CONTROL0__AFMT_RAMP_DATA_SIGN_MASK                                                    0x80000000L
+//AFMT5_AFMT_RAMP_CONTROL1
+#define AFMT5_AFMT_RAMP_CONTROL1__AFMT_RAMP_MIN_COUNT__SHIFT                                                  0x0
+#define AFMT5_AFMT_RAMP_CONTROL1__AFMT_AUDIO_TEST_CH_DISABLE__SHIFT                                           0x18
+#define AFMT5_AFMT_RAMP_CONTROL1__AFMT_RAMP_MIN_COUNT_MASK                                                    0x00FFFFFFL
+#define AFMT5_AFMT_RAMP_CONTROL1__AFMT_AUDIO_TEST_CH_DISABLE_MASK                                             0xFF000000L
+//AFMT5_AFMT_RAMP_CONTROL2
+#define AFMT5_AFMT_RAMP_CONTROL2__AFMT_RAMP_INC_COUNT__SHIFT                                                  0x0
+#define AFMT5_AFMT_RAMP_CONTROL2__AFMT_RAMP_INC_COUNT_MASK                                                    0x00FFFFFFL
+//AFMT5_AFMT_RAMP_CONTROL3
+#define AFMT5_AFMT_RAMP_CONTROL3__AFMT_RAMP_DEC_COUNT__SHIFT                                                  0x0
+#define AFMT5_AFMT_RAMP_CONTROL3__AFMT_RAMP_DEC_COUNT_MASK                                                    0x00FFFFFFL
+//AFMT5_AFMT_60958_2
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_2__SHIFT                                             0x0
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_3__SHIFT                                             0x4
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_4__SHIFT                                             0x8
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_5__SHIFT                                             0xc
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_6__SHIFT                                             0x10
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_7__SHIFT                                             0x14
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_2_MASK                                               0x0000000FL
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_3_MASK                                               0x000000F0L
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_4_MASK                                               0x00000F00L
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_5_MASK                                               0x0000F000L
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_6_MASK                                               0x000F0000L
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_7_MASK                                               0x00F00000L
+//AFMT5_AFMT_AUDIO_CRC_RESULT
+#define AFMT5_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_DONE__SHIFT                                               0x0
+#define AFMT5_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC__SHIFT                                                    0x8
+#define AFMT5_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_DONE_MASK                                                 0x00000001L
+#define AFMT5_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_MASK                                                      0xFFFFFF00L
+//AFMT5_AFMT_STATUS
+#define AFMT5_AFMT_STATUS__AFMT_AUDIO_ENABLE__SHIFT                                                           0x4
+#define AFMT5_AFMT_STATUS__AFMT_AZ_HBR_ENABLE__SHIFT                                                          0x8
+#define AFMT5_AFMT_STATUS__AFMT_AUDIO_FIFO_OVERFLOW__SHIFT                                                    0x18
+#define AFMT5_AFMT_STATUS__AFMT_AZ_AUDIO_ENABLE_CHG__SHIFT                                                    0x1e
+#define AFMT5_AFMT_STATUS__AFMT_AUDIO_ENABLE_MASK                                                             0x00000010L
+#define AFMT5_AFMT_STATUS__AFMT_AZ_HBR_ENABLE_MASK                                                            0x00000100L
+#define AFMT5_AFMT_STATUS__AFMT_AUDIO_FIFO_OVERFLOW_MASK                                                      0x01000000L
+#define AFMT5_AFMT_STATUS__AFMT_AZ_AUDIO_ENABLE_CHG_MASK                                                      0x40000000L
+//AFMT5_AFMT_AUDIO_PACKET_CONTROL
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND__SHIFT                                        0x0
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_DOUBLE_BUFFER_ENABLE__SHIFT                   0x4
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_RESET_FIFO_WHEN_AUDIO_DIS__SHIFT                                0xb
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_EN__SHIFT                                            0xc
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_MODE__SHIFT                                          0xe
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_FIFO_OVERFLOW_ACK__SHIFT                                  0x17
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_CHANNEL_SWAP__SHIFT                                       0x18
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_60958_CS_UPDATE__SHIFT                                          0x1a
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AZ_AUDIO_ENABLE_CHG_ACK__SHIFT                                  0x1e
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_BLANK_TEST_DATA_ON_ENC_ENB__SHIFT                               0x1f
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_MASK                                          0x00000001L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_DOUBLE_BUFFER_ENABLE_MASK                     0x00000010L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_RESET_FIFO_WHEN_AUDIO_DIS_MASK                                  0x00000800L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_EN_MASK                                              0x00001000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_MODE_MASK                                            0x00004000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_FIFO_OVERFLOW_ACK_MASK                                    0x00800000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_CHANNEL_SWAP_MASK                                         0x01000000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_60958_CS_UPDATE_MASK                                            0x04000000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AZ_AUDIO_ENABLE_CHG_ACK_MASK                                    0x40000000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_BLANK_TEST_DATA_ON_ENC_ENB_MASK                                 0x80000000L
+//AFMT5_AFMT_INFOFRAME_CONTROL0
+#define AFMT5_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_SOURCE__SHIFT                                          0x6
+#define AFMT5_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_UPDATE__SHIFT                                          0x7
+#define AFMT5_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_SOURCE_MASK                                            0x00000040L
+#define AFMT5_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_UPDATE_MASK                                            0x00000080L
+//AFMT5_AFMT_AUDIO_SRC_CONTROL
+#define AFMT5_AFMT_AUDIO_SRC_CONTROL__AFMT_AUDIO_SRC_SELECT__SHIFT                                            0x0
+#define AFMT5_AFMT_AUDIO_SRC_CONTROL__AFMT_AUDIO_SRC_SELECT_MASK                                              0x00000007L
+//AFMT5_AFMT_AUDIO_DBG_DTO_CNTL
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_FS_DIV_SEL__SHIFT                                       0x0
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_BASE__SHIFT                                         0x8
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_MULTI__SHIFT                                        0xc
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_DIV__SHIFT                                          0x10
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_FS_DIV_SEL_MASK                                         0x00000007L
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_BASE_MASK                                           0x00000100L
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_MULTI_MASK                                          0x00007000L
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_DIV_MASK                                            0x00070000L
+//AFMT5_AFMT_MEM_PWR
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_DIS__SHIFT                                                           0x0
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_FORCE__SHIFT                                                         0x4
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_STATE__SHIFT                                                         0x8
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_DIS_MASK                                                             0x00000001L
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_FORCE_MASK                                                           0x00000030L
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_STATE_MASK                                                           0x00000300L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_vpg_vpg_dispdec
+//VPG5_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG5_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG5_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG5_VPG_GENERIC_PACKET_DATA
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG5_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG5_VPG_GENERIC_STATUS
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG5_VPG_MEM_PWR
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG5_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG5_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG5_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG5_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG5_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG5_VPG_ISRC1_2_DATA
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+//VPG5_VPG_MPEG_INFO0
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_CHECKSUM__SHIFT                                                    0x0
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB0__SHIFT                                                         0x8
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB1__SHIFT                                                         0x10
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB2__SHIFT                                                         0x18
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_CHECKSUM_MASK                                                      0x000000FFL
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB0_MASK                                                           0x0000FF00L
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB1_MASK                                                           0x00FF0000L
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB2_MASK                                                           0xFF000000L
+//VPG5_VPG_MPEG_INFO1
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_MB3__SHIFT                                                         0x0
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_MF__SHIFT                                                          0x8
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR__SHIFT                                                          0xc
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE__SHIFT                                                      0x10
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_MB3_MASK                                                           0x000000FFL
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_MF_MASK                                                            0x00000300L
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR_MASK                                                            0x00001000L
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE_MASK                                                        0x00010000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dme_dme_dispdec
+//DME5_DME_CONTROL
+#define DME5_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME5_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME5_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME5_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME5_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME5_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME5_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME5_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME5_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME5_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+//DME5_DME_MEMORY_CONTROL
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
+
 // addressBlock: dce_dc_hpo_hpo_top_dispdec
 //HPO_TOP_CLOCK_CONTROL
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_R_GATE_DIS__SHIFT                                                  0x0
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_GATE_DIS__SHIFT                                                    0x1
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_R_GATE_DIS__SHIFT                                                   0x4
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_GATE_DIS__SHIFT                                                     0x5
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_R_GATE_DIS__SHIFT                                            0x8
 #define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_GATE_DIS__SHIFT                                              0x9
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_R_GATE_DIS__SHIFT                                              0xc
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_GATE_DIS__SHIFT                                                0xd
+#define HPO_TOP_CLOCK_CONTROL__HPO_TEST_CLK_SEL__SHIFT                                                        0x10
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_R_GATE_DIS_MASK                                                    0x00000001L
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_GATE_DIS_MASK                                                      0x00000002L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_R_GATE_DIS_MASK                                                     0x00000010L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_GATE_DIS_MASK                                                       0x00000020L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_R_GATE_DIS_MASK                                              0x00000100L
 #define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_GATE_DIS_MASK                                                0x00000200L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_R_GATE_DIS_MASK                                                0x00001000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_GATE_DIS_MASK                                                  0x00002000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_TEST_CLK_SEL_MASK                                                          0x003F0000L
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK__SHIFT                                 0x8
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK_MASK                                   0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+//HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_INCREMENT__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_MODULO__SHIFT                    0x8
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_INCREMENT_MASK                   0x000000FFL
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_MODULO_MASK                      0x0000FF00L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
+//AFMT5_AFMT_ACP
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE__SHIFT                                                                  0x0
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE0__SHIFT                                                  0x8
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE1__SHIFT                                                  0x10
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_MASK                                                                    0x00000003L
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE0_MASK                                                    0x0000FF00L
+#define AFMT5_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE1_MASK                                                    0x00FF0000L
+//AFMT5_AFMT_VBI_PACKET_CONTROL
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_ACP_SOURCE__SHIFT                                                 0xd
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_PACKETS_PER_LINE__SHIFT                                0x10
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_SEND_MAX_PACKETS__SHIFT                                0x18
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_ACP_SOURCE_MASK                                                   0x00002000L
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_PACKETS_PER_LINE_MASK                                  0x001F0000L
+#define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_SEND_MAX_PACKETS_MASK                                  0x01000000L
+//AFMT5_AFMT_AUDIO_PACKET_CONTROL2
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_OVRD__SHIFT                                       0x0
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_SELECT__SHIFT                                     0x1
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_CHANNEL_ENABLE__SHIFT                                    0x8
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_DP_AUDIO_STREAM_ID__SHIFT                                      0x10
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_HBR_ENABLE_OVRD__SHIFT                                         0x18
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_60958_OSF_OVRD__SHIFT                                          0x1c
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_OVRD_MASK                                         0x00000001L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_SELECT_MASK                                       0x00000002L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_CHANNEL_ENABLE_MASK                                      0x0000FF00L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_DP_AUDIO_STREAM_ID_MASK                                        0x00FF0000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_HBR_ENABLE_OVRD_MASK                                           0x01000000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL2__AFMT_60958_OSF_OVRD_MASK                                            0x10000000L
+//AFMT5_AFMT_AUDIO_INFO0
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM__SHIFT                                               0x0
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CC__SHIFT                                                     0x8
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CT__SHIFT                                                     0xb
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_OFFSET__SHIFT                                        0x10
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CXT__SHIFT                                                    0x18
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_MASK                                                 0x000000FFL
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CC_MASK                                                       0x00000700L
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CT_MASK                                                       0x00007800L
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_OFFSET_MASK                                          0x00FF0000L
+#define AFMT5_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CXT_MASK                                                      0x1F000000L
+//AFMT5_AFMT_AUDIO_INFO1
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_CA__SHIFT                                                     0x0
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LSV__SHIFT                                                    0xb
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_DM_INH__SHIFT                                                 0xf
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LFEPBL__SHIFT                                                 0x10
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_CA_MASK                                                       0x000000FFL
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LSV_MASK                                                      0x00007800L
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_DM_INH_MASK                                                   0x00008000L
+#define AFMT5_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LFEPBL_MASK                                                   0x00030000L
+//AFMT5_AFMT_60958_0
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_A__SHIFT                                                            0x0
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_B__SHIFT                                                            0x1
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_C__SHIFT                                                            0x2
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_D__SHIFT                                                            0x3
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_MODE__SHIFT                                                         0x6
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CATEGORY_CODE__SHIFT                                                0x8
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_SOURCE_NUMBER__SHIFT                                                0x10
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CHANNEL_NUMBER_L__SHIFT                                             0x14
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_SAMPLING_FREQUENCY__SHIFT                                           0x18
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY__SHIFT                                               0x1c
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_A_MASK                                                              0x00000001L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_B_MASK                                                              0x00000002L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_C_MASK                                                              0x00000004L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_D_MASK                                                              0x00000038L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_MODE_MASK                                                           0x000000C0L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CATEGORY_CODE_MASK                                                  0x0000FF00L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_SOURCE_NUMBER_MASK                                                  0x000F0000L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CHANNEL_NUMBER_L_MASK                                               0x00F00000L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_SAMPLING_FREQUENCY_MASK                                             0x0F000000L
+#define AFMT5_AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY_MASK                                                 0x30000000L
+//AFMT5_AFMT_60958_1
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_WORD_LENGTH__SHIFT                                                  0x0
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT                                  0x4
+#define AFMT5_AFMT_60958_1__AFMT_60958_VALID_L__SHIFT                                                         0x10
+#define AFMT5_AFMT_60958_1__AFMT_60958_VALID_R__SHIFT                                                         0x12
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_CHANNEL_NUMBER_R__SHIFT                                             0x14
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_WORD_LENGTH_MASK                                                    0x0000000FL
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK                                    0x000000F0L
+#define AFMT5_AFMT_60958_1__AFMT_60958_VALID_L_MASK                                                           0x00010000L
+#define AFMT5_AFMT_60958_1__AFMT_60958_VALID_R_MASK                                                           0x00040000L
+#define AFMT5_AFMT_60958_1__AFMT_60958_CS_CHANNEL_NUMBER_R_MASK                                               0x00F00000L
+//AFMT5_AFMT_AUDIO_CRC_CONTROL
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_EN__SHIFT                                                0x0
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CONT__SHIFT                                              0x4
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_SOURCE__SHIFT                                            0x8
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CH_SEL__SHIFT                                            0xc
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_COUNT__SHIFT                                             0x10
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_EN_MASK                                                  0x00000001L
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CONT_MASK                                                0x00000010L
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_SOURCE_MASK                                              0x00000100L
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CH_SEL_MASK                                              0x0000F000L
+#define AFMT5_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_COUNT_MASK                                               0xFFFF0000L
+//AFMT5_AFMT_RAMP_CONTROL0
+#define AFMT5_AFMT_RAMP_CONTROL0__AFMT_RAMP_MAX_COUNT__SHIFT                                                  0x0
+#define AFMT5_AFMT_RAMP_CONTROL0__AFMT_RAMP_DATA_SIGN__SHIFT                                                  0x1f
+#define AFMT5_AFMT_RAMP_CONTROL0__AFMT_RAMP_MAX_COUNT_MASK                                                    0x00FFFFFFL
+#define AFMT5_AFMT_RAMP_CONTROL0__AFMT_RAMP_DATA_SIGN_MASK                                                    0x80000000L
+//AFMT5_AFMT_RAMP_CONTROL1
+#define AFMT5_AFMT_RAMP_CONTROL1__AFMT_RAMP_MIN_COUNT__SHIFT                                                  0x0
+#define AFMT5_AFMT_RAMP_CONTROL1__AFMT_AUDIO_TEST_CH_DISABLE__SHIFT                                           0x18
+#define AFMT5_AFMT_RAMP_CONTROL1__AFMT_RAMP_MIN_COUNT_MASK                                                    0x00FFFFFFL
+#define AFMT5_AFMT_RAMP_CONTROL1__AFMT_AUDIO_TEST_CH_DISABLE_MASK                                             0xFF000000L
+//AFMT5_AFMT_RAMP_CONTROL2
+#define AFMT5_AFMT_RAMP_CONTROL2__AFMT_RAMP_INC_COUNT__SHIFT                                                  0x0
+#define AFMT5_AFMT_RAMP_CONTROL2__AFMT_RAMP_INC_COUNT_MASK                                                    0x00FFFFFFL
+//AFMT5_AFMT_RAMP_CONTROL3
+#define AFMT5_AFMT_RAMP_CONTROL3__AFMT_RAMP_DEC_COUNT__SHIFT                                                  0x0
+#define AFMT5_AFMT_RAMP_CONTROL3__AFMT_RAMP_DEC_COUNT_MASK                                                    0x00FFFFFFL
+//AFMT5_AFMT_60958_2
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_2__SHIFT                                             0x0
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_3__SHIFT                                             0x4
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_4__SHIFT                                             0x8
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_5__SHIFT                                             0xc
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_6__SHIFT                                             0x10
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_7__SHIFT                                             0x14
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_2_MASK                                               0x0000000FL
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_3_MASK                                               0x000000F0L
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_4_MASK                                               0x00000F00L
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_5_MASK                                               0x0000F000L
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_6_MASK                                               0x000F0000L
+#define AFMT5_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_7_MASK                                               0x00F00000L
+//AFMT5_AFMT_AUDIO_CRC_RESULT
+#define AFMT5_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_DONE__SHIFT                                               0x0
+#define AFMT5_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC__SHIFT                                                    0x8
+#define AFMT5_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_DONE_MASK                                                 0x00000001L
+#define AFMT5_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_MASK                                                      0xFFFFFF00L
+//AFMT5_AFMT_STATUS
+#define AFMT5_AFMT_STATUS__AFMT_AUDIO_ENABLE__SHIFT                                                           0x4
+#define AFMT5_AFMT_STATUS__AFMT_AZ_HBR_ENABLE__SHIFT                                                          0x8
+#define AFMT5_AFMT_STATUS__AFMT_AUDIO_FIFO_OVERFLOW__SHIFT                                                    0x18
+#define AFMT5_AFMT_STATUS__AFMT_AZ_AUDIO_ENABLE_CHG__SHIFT                                                    0x1e
+#define AFMT5_AFMT_STATUS__AFMT_AUDIO_ENABLE_MASK                                                             0x00000010L
+#define AFMT5_AFMT_STATUS__AFMT_AZ_HBR_ENABLE_MASK                                                            0x00000100L
+#define AFMT5_AFMT_STATUS__AFMT_AUDIO_FIFO_OVERFLOW_MASK                                                      0x01000000L
+#define AFMT5_AFMT_STATUS__AFMT_AZ_AUDIO_ENABLE_CHG_MASK                                                      0x40000000L
+//AFMT5_AFMT_AUDIO_PACKET_CONTROL
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND__SHIFT                                        0x0
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_DOUBLE_BUFFER_ENABLE__SHIFT                   0x4
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_RESET_FIFO_WHEN_AUDIO_DIS__SHIFT                                0xb
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_EN__SHIFT                                            0xc
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_MODE__SHIFT                                          0xe
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_FIFO_OVERFLOW_ACK__SHIFT                                  0x17
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_CHANNEL_SWAP__SHIFT                                       0x18
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_60958_CS_UPDATE__SHIFT                                          0x1a
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AZ_AUDIO_ENABLE_CHG_ACK__SHIFT                                  0x1e
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_BLANK_TEST_DATA_ON_ENC_ENB__SHIFT                               0x1f
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_MASK                                          0x00000001L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_DOUBLE_BUFFER_ENABLE_MASK                     0x00000010L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_RESET_FIFO_WHEN_AUDIO_DIS_MASK                                  0x00000800L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_EN_MASK                                              0x00001000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_MODE_MASK                                            0x00004000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_FIFO_OVERFLOW_ACK_MASK                                    0x00800000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_CHANNEL_SWAP_MASK                                         0x01000000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_60958_CS_UPDATE_MASK                                            0x04000000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_AZ_AUDIO_ENABLE_CHG_ACK_MASK                                    0x40000000L
+#define AFMT5_AFMT_AUDIO_PACKET_CONTROL__AFMT_BLANK_TEST_DATA_ON_ENC_ENB_MASK                                 0x80000000L
+//AFMT5_AFMT_INFOFRAME_CONTROL0
+#define AFMT5_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_SOURCE__SHIFT                                          0x6
+#define AFMT5_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_UPDATE__SHIFT                                          0x7
+#define AFMT5_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_SOURCE_MASK                                            0x00000040L
+#define AFMT5_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_UPDATE_MASK                                            0x00000080L
+//AFMT5_AFMT_AUDIO_SRC_CONTROL
+#define AFMT5_AFMT_AUDIO_SRC_CONTROL__AFMT_AUDIO_SRC_SELECT__SHIFT                                            0x0
+#define AFMT5_AFMT_AUDIO_SRC_CONTROL__AFMT_AUDIO_SRC_SELECT_MASK                                              0x00000007L
+//AFMT5_AFMT_AUDIO_DBG_DTO_CNTL
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_FS_DIV_SEL__SHIFT                                       0x0
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_BASE__SHIFT                                         0x8
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_MULTI__SHIFT                                        0xc
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_DIV__SHIFT                                          0x10
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_FS_DIV_SEL_MASK                                         0x00000007L
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_BASE_MASK                                           0x00000100L
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_MULTI_MASK                                          0x00007000L
+#define AFMT5_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_DIV_MASK                                            0x00070000L
+//AFMT5_AFMT_MEM_PWR
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_DIS__SHIFT                                                           0x0
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_FORCE__SHIFT                                                         0x4
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_STATE__SHIFT                                                         0x8
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_DIS_MASK                                                             0x00000001L
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_FORCE_MASK                                                           0x00000030L
+#define AFMT5_AFMT_MEM_PWR__AFMT_MEM_PWR_STATE_MASK                                                           0x00000300L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_vpg_vpg_dispdec
+//VPG5_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG5_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG5_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG5_VPG_GENERIC_PACKET_DATA
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG5_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG5_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG5_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG5_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG5_VPG_GENERIC_STATUS
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG5_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG5_VPG_MEM_PWR
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG5_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG5_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG5_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG5_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG5_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG5_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG5_VPG_ISRC1_2_DATA
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG5_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+//VPG5_VPG_MPEG_INFO0
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_CHECKSUM__SHIFT                                                    0x0
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB0__SHIFT                                                         0x8
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB1__SHIFT                                                         0x10
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB2__SHIFT                                                         0x18
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_CHECKSUM_MASK                                                      0x000000FFL
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB0_MASK                                                           0x0000FF00L
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB1_MASK                                                           0x00FF0000L
+#define VPG5_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB2_MASK                                                           0xFF000000L
+//VPG5_VPG_MPEG_INFO1
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_MB3__SHIFT                                                         0x0
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_MF__SHIFT                                                          0x8
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR__SHIFT                                                          0xc
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE__SHIFT                                                      0x10
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_MB3_MASK                                                           0x000000FFL
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_MF_MASK                                                            0x00000300L
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR_MASK                                                            0x00001000L
+#define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE_MASK                                                        0x00010000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dme_dme_dispdec
+//DME5_DME_CONTROL
+#define DME5_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME5_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME5_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME5_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME5_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME5_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME5_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME5_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME5_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME5_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME5_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+//DME5_DME_MEMORY_CONTROL
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME5_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
+
+// addressBlock: dce_dc_hpo_hpo_top_dispdec
+//HPO_TOP_CLOCK_CONTROL
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_R_GATE_DIS__SHIFT                                                  0x0
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_GATE_DIS__SHIFT                                                    0x1
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_R_GATE_DIS__SHIFT                                                   0x4
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_GATE_DIS__SHIFT                                                     0x5
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_R_GATE_DIS__SHIFT                                            0x8
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_GATE_DIS__SHIFT                                              0x9
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_R_GATE_DIS__SHIFT                                              0xc
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_GATE_DIS__SHIFT                                                0xd
+#define HPO_TOP_CLOCK_CONTROL__HPO_TEST_CLK_SEL__SHIFT                                                        0x10
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_R_GATE_DIS_MASK                                                    0x00000001L
+#define HPO_TOP_CLOCK_CONTROL__HPO_DISPCLK_GATE_DIS_MASK                                                      0x00000002L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_R_GATE_DIS_MASK                                                     0x00000010L
+#define HPO_TOP_CLOCK_CONTROL__HPO_SOCCLK_GATE_DIS_MASK                                                       0x00000020L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_R_GATE_DIS_MASK                                              0x00000100L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMISTREAMCLK_GATE_DIS_MASK                                                0x00000200L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_R_GATE_DIS_MASK                                                0x00001000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_HDMICHARCLK_GATE_DIS_MASK                                                  0x00002000L
+#define HPO_TOP_CLOCK_CONTROL__HPO_TEST_CLK_SEL_MASK                                                          0x003F0000L
+
 
 // addressBlock: dce_dc_hpo_hpo_dcperfmon_dc_perfmon_dispdec
 //DC_PERFMON26_PERFCOUNTER_CNTL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_offset.h
index a3373d1e17362..9b1132bbab07c 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_offset.h
@@ -6838,6 +6838,98 @@
 #define mmDC_PERFMON15_PERFMON_LOW                                                                     0x08cf
 #define mmDC_PERFMON15_PERFMON_LOW_BASE_IDX                                                            3
 
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define mmHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define mmHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define mmHDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL                                                        0x08d4
+#define mmHDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL_BASE_IDX                                               3
+#define mmHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define mmHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define mmHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define mmHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define mmHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define mmHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define mmHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define mmHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define mmHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define mmHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define mmHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define mmHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define mmHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define mmHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define mmHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define mmHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define mmHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define mmHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define mmHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define mmHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define mmHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define mmHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define mmHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define mmHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define mmHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define mmHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define mmHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define mmHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define mmHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define mmHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define mmHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define mmHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define mmHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define mmHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define mmHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define mmHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define mmHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define mmHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define mmHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define mmHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define mmHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define mmHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define mmHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define mmHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define mmHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define mmHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define mmHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define mmHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define mmHDMI_TB_ENC_MODE                                                                             0x0908
+#define mmHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define mmHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define mmHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define mmHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define mmHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 // base address: 0x2646c
@@ -6882,6 +6974,25 @@
 #define mmDME2_DME_MEMORY_CONTROL                                                                      0x093d
 #define mmDME2_DME_MEMORY_CONTROL_BASE_IDX                                                             3
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define mmHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define mmHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define mmHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define mmHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define mmHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define mmHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define mmHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define mmHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define mmHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define mmHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define mmHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define mmHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
 
 // addressBlock: dce_dc_hpo_hpo_top_dispdec
 // base address: 0x0
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_sh_mask.h
index 9549494b65b55..0326604ea9a7b 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_0_3_sh_mask.h
@@ -26316,6 +26316,947 @@
 #define DC_PERFMON15_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
 #define DC_PERFMON15_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
 
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK__SHIFT                                 0x8
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK_MASK                                   0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+//HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_INCREMENT__SHIFT                 0x0
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_MODULO__SHIFT                    0x8
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_INCREMENT_MASK                   0x000000FFL
+#define HDMI_STREAM_ENC_HDMISTREAMCLK_CONTROL__HDMI_STREAM_ENC_HDMISTREAMCLK_MODULO_MASK                      0x0000FF00L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
+//AFMT2_AFMT_ACP
+#define AFMT2_AFMT_ACP__AFMT_ACP_TYPE__SHIFT                                                                  0x0
+#define AFMT2_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE0__SHIFT                                                  0x8
+#define AFMT2_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE1__SHIFT                                                  0x10
+#define AFMT2_AFMT_ACP__AFMT_ACP_TYPE_MASK                                                                    0x00000003L
+#define AFMT2_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE0_MASK                                                    0x0000FF00L
+#define AFMT2_AFMT_ACP__AFMT_ACP_TYPE_DEPENDENT_BYTE1_MASK                                                    0x00FF0000L
+//AFMT2_AFMT_VBI_PACKET_CONTROL
+#define AFMT2_AFMT_VBI_PACKET_CONTROL__AFMT_ACP_SOURCE__SHIFT                                                 0xd
+#define AFMT2_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_PACKETS_PER_LINE__SHIFT                                0x10
+#define AFMT2_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_SEND_MAX_PACKETS__SHIFT                                0x18
+#define AFMT2_AFMT_VBI_PACKET_CONTROL__AFMT_ACP_SOURCE_MASK                                                   0x00002000L
+#define AFMT2_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_PACKETS_PER_LINE_MASK                                  0x001F0000L
+#define AFMT2_AFMT_VBI_PACKET_CONTROL__AFMT_HDMI_AUDIO_SEND_MAX_PACKETS_MASK                                  0x01000000L
+//AFMT2_AFMT_AUDIO_PACKET_CONTROL2
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_OVRD__SHIFT                                       0x0
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_SELECT__SHIFT                                     0x1
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_CHANNEL_ENABLE__SHIFT                                    0x8
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_DP_AUDIO_STREAM_ID__SHIFT                                      0x10
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_HBR_ENABLE_OVRD__SHIFT                                         0x18
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_60958_OSF_OVRD__SHIFT                                          0x1c
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_OVRD_MASK                                         0x00000001L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_LAYOUT_SELECT_MASK                                       0x00000002L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_AUDIO_CHANNEL_ENABLE_MASK                                      0x0000FF00L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_DP_AUDIO_STREAM_ID_MASK                                        0x00FF0000L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_HBR_ENABLE_OVRD_MASK                                           0x01000000L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL2__AFMT_60958_OSF_OVRD_MASK                                            0x10000000L
+//AFMT2_AFMT_AUDIO_INFO0
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM__SHIFT                                               0x0
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CC__SHIFT                                                     0x8
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CT__SHIFT                                                     0xb
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_OFFSET__SHIFT                                        0x10
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CXT__SHIFT                                                    0x18
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_MASK                                                 0x000000FFL
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CC_MASK                                                       0x00000700L
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CT_MASK                                                       0x00007800L
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CHECKSUM_OFFSET_MASK                                          0x00FF0000L
+#define AFMT2_AFMT_AUDIO_INFO0__AFMT_AUDIO_INFO_CXT_MASK                                                      0x1F000000L
+//AFMT2_AFMT_AUDIO_INFO1
+#define AFMT2_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_CA__SHIFT                                                     0x0
+#define AFMT2_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LSV__SHIFT                                                    0xb
+#define AFMT2_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_DM_INH__SHIFT                                                 0xf
+#define AFMT2_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LFEPBL__SHIFT                                                 0x10
+#define AFMT2_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_CA_MASK                                                       0x000000FFL
+#define AFMT2_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LSV_MASK                                                      0x00007800L
+#define AFMT2_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_DM_INH_MASK                                                   0x00008000L
+#define AFMT2_AFMT_AUDIO_INFO1__AFMT_AUDIO_INFO_LFEPBL_MASK                                                   0x00030000L
+//AFMT2_AFMT_60958_0
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_A__SHIFT                                                            0x0
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_B__SHIFT                                                            0x1
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_C__SHIFT                                                            0x2
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_D__SHIFT                                                            0x3
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_MODE__SHIFT                                                         0x6
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_CATEGORY_CODE__SHIFT                                                0x8
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_SOURCE_NUMBER__SHIFT                                                0x10
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_CHANNEL_NUMBER_L__SHIFT                                             0x14
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_SAMPLING_FREQUENCY__SHIFT                                           0x18
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY__SHIFT                                               0x1c
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_A_MASK                                                              0x00000001L
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_B_MASK                                                              0x00000002L
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_C_MASK                                                              0x00000004L
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_D_MASK                                                              0x00000038L
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_MODE_MASK                                                           0x000000C0L
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_CATEGORY_CODE_MASK                                                  0x0000FF00L
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_SOURCE_NUMBER_MASK                                                  0x000F0000L
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_CHANNEL_NUMBER_L_MASK                                               0x00F00000L
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_SAMPLING_FREQUENCY_MASK                                             0x0F000000L
+#define AFMT2_AFMT_60958_0__AFMT_60958_CS_CLOCK_ACCURACY_MASK                                                 0x30000000L
+//AFMT2_AFMT_60958_1
+#define AFMT2_AFMT_60958_1__AFMT_60958_CS_WORD_LENGTH__SHIFT                                                  0x0
+#define AFMT2_AFMT_60958_1__AFMT_60958_CS_ORIGINAL_SAMPLING_FREQUENCY__SHIFT                                  0x4
+#define AFMT2_AFMT_60958_1__AFMT_60958_VALID_L__SHIFT                                                         0x10
+#define AFMT2_AFMT_60958_1__AFMT_60958_VALID_R__SHIFT                                                         0x12
+#define AFMT2_AFMT_60958_1__AFMT_60958_CS_CHANNEL_NUMBER_R__SHIFT                                             0x14
+#define AFMT2_AFMT_60958_1__AFMT_60958_CS_WORD_LENGTH_MASK                                                    0x0000000FL
+#define AFMT2_AFMT_60958_1__AFMT_60958_CS_ORIGINAL_SAMPLING_FREQUENCY_MASK                                    0x000000F0L
+#define AFMT2_AFMT_60958_1__AFMT_60958_VALID_L_MASK                                                           0x00010000L
+#define AFMT2_AFMT_60958_1__AFMT_60958_VALID_R_MASK                                                           0x00040000L
+#define AFMT2_AFMT_60958_1__AFMT_60958_CS_CHANNEL_NUMBER_R_MASK                                               0x00F00000L
+//AFMT2_AFMT_AUDIO_CRC_CONTROL
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_EN__SHIFT                                                0x0
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CONT__SHIFT                                              0x4
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_SOURCE__SHIFT                                            0x8
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CH_SEL__SHIFT                                            0xc
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_COUNT__SHIFT                                             0x10
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_EN_MASK                                                  0x00000001L
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CONT_MASK                                                0x00000010L
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_SOURCE_MASK                                              0x00000100L
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_CH_SEL_MASK                                              0x0000F000L
+#define AFMT2_AFMT_AUDIO_CRC_CONTROL__AFMT_AUDIO_CRC_COUNT_MASK                                               0xFFFF0000L
+//AFMT2_AFMT_RAMP_CONTROL0
+#define AFMT2_AFMT_RAMP_CONTROL0__AFMT_RAMP_MAX_COUNT__SHIFT                                                  0x0
+#define AFMT2_AFMT_RAMP_CONTROL0__AFMT_RAMP_DATA_SIGN__SHIFT                                                  0x1f
+#define AFMT2_AFMT_RAMP_CONTROL0__AFMT_RAMP_MAX_COUNT_MASK                                                    0x00FFFFFFL
+#define AFMT2_AFMT_RAMP_CONTROL0__AFMT_RAMP_DATA_SIGN_MASK                                                    0x80000000L
+//AFMT2_AFMT_RAMP_CONTROL1
+#define AFMT2_AFMT_RAMP_CONTROL1__AFMT_RAMP_MIN_COUNT__SHIFT                                                  0x0
+#define AFMT2_AFMT_RAMP_CONTROL1__AFMT_AUDIO_TEST_CH_DISABLE__SHIFT                                           0x18
+#define AFMT2_AFMT_RAMP_CONTROL1__AFMT_RAMP_MIN_COUNT_MASK                                                    0x00FFFFFFL
+#define AFMT2_AFMT_RAMP_CONTROL1__AFMT_AUDIO_TEST_CH_DISABLE_MASK                                             0xFF000000L
+//AFMT2_AFMT_RAMP_CONTROL2
+#define AFMT2_AFMT_RAMP_CONTROL2__AFMT_RAMP_INC_COUNT__SHIFT                                                  0x0
+#define AFMT2_AFMT_RAMP_CONTROL2__AFMT_RAMP_INC_COUNT_MASK                                                    0x00FFFFFFL
+//AFMT2_AFMT_RAMP_CONTROL3
+#define AFMT2_AFMT_RAMP_CONTROL3__AFMT_RAMP_DEC_COUNT__SHIFT                                                  0x0
+#define AFMT2_AFMT_RAMP_CONTROL3__AFMT_RAMP_DEC_COUNT_MASK                                                    0x00FFFFFFL
+//AFMT2_AFMT_60958_2
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_2__SHIFT                                             0x0
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_3__SHIFT                                             0x4
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_4__SHIFT                                             0x8
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_5__SHIFT                                             0xc
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_6__SHIFT                                             0x10
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_7__SHIFT                                             0x14
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_2_MASK                                               0x0000000FL
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_3_MASK                                               0x000000F0L
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_4_MASK                                               0x00000F00L
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_5_MASK                                               0x0000F000L
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_6_MASK                                               0x000F0000L
+#define AFMT2_AFMT_60958_2__AFMT_60958_CS_CHANNEL_NUMBER_7_MASK                                               0x00F00000L
+//AFMT2_AFMT_AUDIO_CRC_RESULT
+#define AFMT2_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_DONE__SHIFT                                               0x0
+#define AFMT2_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC__SHIFT                                                    0x8
+#define AFMT2_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_DONE_MASK                                                 0x00000001L
+#define AFMT2_AFMT_AUDIO_CRC_RESULT__AFMT_AUDIO_CRC_MASK                                                      0xFFFFFF00L
+//AFMT2_AFMT_STATUS
+#define AFMT2_AFMT_STATUS__AFMT_AUDIO_ENABLE__SHIFT                                                           0x4
+#define AFMT2_AFMT_STATUS__AFMT_AZ_HBR_ENABLE__SHIFT                                                          0x8
+#define AFMT2_AFMT_STATUS__AFMT_AUDIO_FIFO_OVERFLOW__SHIFT                                                    0x18
+#define AFMT2_AFMT_STATUS__AFMT_AZ_AUDIO_ENABLE_CHG__SHIFT                                                    0x1e
+#define AFMT2_AFMT_STATUS__AFMT_AUDIO_ENABLE_MASK                                                             0x00000010L
+#define AFMT2_AFMT_STATUS__AFMT_AZ_HBR_ENABLE_MASK                                                            0x00000100L
+#define AFMT2_AFMT_STATUS__AFMT_AUDIO_FIFO_OVERFLOW_MASK                                                      0x01000000L
+#define AFMT2_AFMT_STATUS__AFMT_AZ_AUDIO_ENABLE_CHG_MASK                                                      0x40000000L
+//AFMT2_AFMT_AUDIO_PACKET_CONTROL
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND__SHIFT                                        0x0
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_DOUBLE_BUFFER_ENABLE__SHIFT                   0x4
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_RESET_FIFO_WHEN_AUDIO_DIS__SHIFT                                0xb
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_EN__SHIFT                                            0xc
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_MODE__SHIFT                                          0xe
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_FIFO_OVERFLOW_ACK__SHIFT                                  0x17
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_CHANNEL_SWAP__SHIFT                                       0x18
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_60958_CS_UPDATE__SHIFT                                          0x1a
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AZ_AUDIO_ENABLE_CHG_ACK__SHIFT                                  0x1e
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_BLANK_TEST_DATA_ON_ENC_ENB__SHIFT                               0x1f
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_MASK                                          0x00000001L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_SAMPLE_SEND_DOUBLE_BUFFER_ENABLE_MASK                     0x00000010L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_RESET_FIFO_WHEN_AUDIO_DIS_MASK                                  0x00000800L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_EN_MASK                                              0x00001000L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_TEST_MODE_MASK                                            0x00004000L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_FIFO_OVERFLOW_ACK_MASK                                    0x00800000L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AUDIO_CHANNEL_SWAP_MASK                                         0x01000000L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_60958_CS_UPDATE_MASK                                            0x04000000L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_AZ_AUDIO_ENABLE_CHG_ACK_MASK                                    0x40000000L
+#define AFMT2_AFMT_AUDIO_PACKET_CONTROL__AFMT_BLANK_TEST_DATA_ON_ENC_ENB_MASK                                 0x80000000L
+//AFMT2_AFMT_INFOFRAME_CONTROL0
+#define AFMT2_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_SOURCE__SHIFT                                          0x6
+#define AFMT2_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_UPDATE__SHIFT                                          0x7
+#define AFMT2_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_SOURCE_MASK                                            0x00000040L
+#define AFMT2_AFMT_INFOFRAME_CONTROL0__AFMT_AUDIO_INFO_UPDATE_MASK                                            0x00000080L
+//AFMT2_AFMT_AUDIO_SRC_CONTROL
+#define AFMT2_AFMT_AUDIO_SRC_CONTROL__AFMT_AUDIO_SRC_SELECT__SHIFT                                            0x0
+#define AFMT2_AFMT_AUDIO_SRC_CONTROL__AFMT_AUDIO_SRC_SELECT_MASK                                              0x00000007L
+//AFMT2_AFMT_AUDIO_DBG_DTO_CNTL
+#define AFMT2_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_FS_DIV_SEL__SHIFT                                       0x0
+#define AFMT2_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_BASE__SHIFT                                         0x8
+#define AFMT2_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_MULTI__SHIFT                                        0xc
+#define AFMT2_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_DIV__SHIFT                                          0x10
+#define AFMT2_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_FS_DIV_SEL_MASK                                         0x00000007L
+#define AFMT2_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_BASE_MASK                                           0x00000100L
+#define AFMT2_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_MULTI_MASK                                          0x00007000L
+#define AFMT2_AFMT_AUDIO_DBG_DTO_CNTL__AFMT_AUDIO_DTO_DBG_DIV_MASK                                            0x00070000L
+//AFMT2_AFMT_MEM_PWR
+#define AFMT2_AFMT_MEM_PWR__AFMT_MEM_PWR_DIS__SHIFT                                                           0x0
+#define AFMT2_AFMT_MEM_PWR__AFMT_MEM_PWR_FORCE__SHIFT                                                         0x4
+#define AFMT2_AFMT_MEM_PWR__AFMT_MEM_PWR_STATE__SHIFT                                                         0x8
+#define AFMT2_AFMT_MEM_PWR__AFMT_MEM_PWR_DIS_MASK                                                             0x00000001L
+#define AFMT2_AFMT_MEM_PWR__AFMT_MEM_PWR_FORCE_MASK                                                           0x00000030L
+#define AFMT2_AFMT_MEM_PWR__AFMT_MEM_PWR_STATE_MASK                                                           0x00000300L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_vpg_vpg_dispdec
+//VPG2_VPG_GENERIC_PACKET_ACCESS_CTRL
+#define VPG2_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX__SHIFT                                    0x0
+#define VPG2_VPG_GENERIC_PACKET_ACCESS_CTRL__VPG_GENERIC_DATA_INDEX_MASK                                      0x000000FFL
+//VPG2_VPG_GENERIC_PACKET_DATA
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0__SHIFT                                           0x0
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1__SHIFT                                           0x8
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2__SHIFT                                           0x10
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3__SHIFT                                           0x18
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE0_MASK                                             0x000000FFL
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE1_MASK                                             0x0000FF00L
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE2_MASK                                             0x00FF0000L
+#define VPG2_VPG_GENERIC_PACKET_DATA__VPG_GENERIC_DATA_BYTE3_MASK                                             0xFF000000L
+//VPG2_VPG_GSP_FRAME_UPDATE_CTRL
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE__SHIFT                                      0x0
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE__SHIFT                                      0x1
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE__SHIFT                                      0x2
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE__SHIFT                                      0x3
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE__SHIFT                                      0x4
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE__SHIFT                                      0x5
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE__SHIFT                                      0x6
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE__SHIFT                                      0x7
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE__SHIFT                                      0x8
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE__SHIFT                                      0x9
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE__SHIFT                                     0xa
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE__SHIFT                                     0xb
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE__SHIFT                                     0xc
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE__SHIFT                                     0xd
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE__SHIFT                                     0xe
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING__SHIFT                              0x10
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING__SHIFT                              0x11
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING__SHIFT                              0x12
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING__SHIFT                              0x13
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING__SHIFT                              0x14
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING__SHIFT                              0x15
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING__SHIFT                              0x16
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING__SHIFT                              0x17
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING__SHIFT                              0x18
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING__SHIFT                              0x19
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING__SHIFT                             0x1a
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING__SHIFT                             0x1b
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING__SHIFT                             0x1c
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING__SHIFT                             0x1d
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING__SHIFT                             0x1e
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_MASK                                        0x00000001L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_MASK                                        0x00000002L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_MASK                                        0x00000004L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_MASK                                        0x00000008L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_MASK                                        0x00000010L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_MASK                                        0x00000020L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_MASK                                        0x00000040L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_MASK                                        0x00000080L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_MASK                                        0x00000100L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_MASK                                        0x00000200L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_MASK                                       0x00000400L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_MASK                                       0x00000800L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_MASK                                       0x00001000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_MASK                                       0x00002000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_MASK                                       0x00004000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC0_FRAME_UPDATE_PENDING_MASK                                0x00010000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC1_FRAME_UPDATE_PENDING_MASK                                0x00020000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC2_FRAME_UPDATE_PENDING_MASK                                0x00040000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC3_FRAME_UPDATE_PENDING_MASK                                0x00080000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC4_FRAME_UPDATE_PENDING_MASK                                0x00100000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC5_FRAME_UPDATE_PENDING_MASK                                0x00200000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC6_FRAME_UPDATE_PENDING_MASK                                0x00400000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC7_FRAME_UPDATE_PENDING_MASK                                0x00800000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC8_FRAME_UPDATE_PENDING_MASK                                0x01000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC9_FRAME_UPDATE_PENDING_MASK                                0x02000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC10_FRAME_UPDATE_PENDING_MASK                               0x04000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC11_FRAME_UPDATE_PENDING_MASK                               0x08000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC12_FRAME_UPDATE_PENDING_MASK                               0x10000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC13_FRAME_UPDATE_PENDING_MASK                               0x20000000L
+#define VPG2_VPG_GSP_FRAME_UPDATE_CTRL__VPG_GENERIC14_FRAME_UPDATE_PENDING_MASK                               0x40000000L
+//VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE__SHIFT                              0x0
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE__SHIFT                              0x1
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE__SHIFT                              0x2
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE__SHIFT                              0x3
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE__SHIFT                              0x4
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE__SHIFT                              0x5
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE__SHIFT                              0x6
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE__SHIFT                              0x7
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE__SHIFT                              0x8
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE__SHIFT                              0x9
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE__SHIFT                             0xa
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE__SHIFT                             0xb
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE__SHIFT                             0xc
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE__SHIFT                             0xd
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE__SHIFT                             0xe
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x10
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x11
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x12
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x13
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x14
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x15
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x16
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x17
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x18
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING__SHIFT                      0x19
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1a
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1b
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1c
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1d
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING__SHIFT                     0x1e
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_MASK                                0x00000001L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_MASK                                0x00000002L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_MASK                                0x00000004L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_MASK                                0x00000008L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_MASK                                0x00000010L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_MASK                                0x00000020L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_MASK                                0x00000040L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_MASK                                0x00000080L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_MASK                                0x00000100L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_MASK                                0x00000200L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_MASK                               0x00000400L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_MASK                               0x00000800L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_MASK                               0x00001000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_MASK                               0x00002000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_MASK                               0x00004000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC0_IMMEDIATE_UPDATE_PENDING_MASK                        0x00010000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC1_IMMEDIATE_UPDATE_PENDING_MASK                        0x00020000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC2_IMMEDIATE_UPDATE_PENDING_MASK                        0x00040000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC3_IMMEDIATE_UPDATE_PENDING_MASK                        0x00080000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC4_IMMEDIATE_UPDATE_PENDING_MASK                        0x00100000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC5_IMMEDIATE_UPDATE_PENDING_MASK                        0x00200000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC6_IMMEDIATE_UPDATE_PENDING_MASK                        0x00400000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC7_IMMEDIATE_UPDATE_PENDING_MASK                        0x00800000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC8_IMMEDIATE_UPDATE_PENDING_MASK                        0x01000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC9_IMMEDIATE_UPDATE_PENDING_MASK                        0x02000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC10_IMMEDIATE_UPDATE_PENDING_MASK                       0x04000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC11_IMMEDIATE_UPDATE_PENDING_MASK                       0x08000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC12_IMMEDIATE_UPDATE_PENDING_MASK                       0x10000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC13_IMMEDIATE_UPDATE_PENDING_MASK                       0x20000000L
+#define VPG2_VPG_GSP_IMMEDIATE_UPDATE_CTRL__VPG_GENERIC14_IMMEDIATE_UPDATE_PENDING_MASK                       0x40000000L
+//VPG2_VPG_GENERIC_STATUS
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS__SHIFT                                               0x0
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED__SHIFT                                          0x1
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR__SHIFT                                              0x4
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_LOCK_STATUS_MASK                                                 0x00000001L
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_OCCURED_MASK                                            0x00000002L
+#define VPG2_VPG_GENERIC_STATUS__VPG_GENERIC_CONFLICT_CLR_MASK                                                0x00000010L
+//VPG2_VPG_MEM_PWR
+#define VPG2_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS__SHIFT                                                  0x0
+#define VPG2_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE__SHIFT                                                    0x4
+#define VPG2_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE__SHIFT                                                        0x8
+#define VPG2_VPG_MEM_PWR__VPG_GSP_MEM_LIGHT_SLEEP_DIS_MASK                                                    0x00000001L
+#define VPG2_VPG_MEM_PWR__VPG_GSP_LIGHT_SLEEP_FORCE_MASK                                                      0x00000010L
+#define VPG2_VPG_MEM_PWR__VPG_GSP_MEM_PWR_STATE_MASK                                                          0x00000100L
+//VPG2_VPG_ISRC1_2_ACCESS_CTRL
+#define VPG2_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX__SHIFT                                           0x0
+#define VPG2_VPG_ISRC1_2_ACCESS_CTRL__VPG_ISRC1_2_DATA_INDEX_MASK                                             0x0000000FL
+//VPG2_VPG_ISRC1_2_DATA
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0__SHIFT                                                     0x0
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1__SHIFT                                                     0x8
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2__SHIFT                                                     0x10
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3__SHIFT                                                     0x18
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE0_MASK                                                       0x000000FFL
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE1_MASK                                                       0x0000FF00L
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE2_MASK                                                       0x00FF0000L
+#define VPG2_VPG_ISRC1_2_DATA__VPG_ISRC_DATA_BYTE3_MASK                                                       0xFF000000L
+//VPG2_VPG_MPEG_INFO0
+#define VPG2_VPG_MPEG_INFO0__VPG_MPEG_INFO_CHECKSUM__SHIFT                                                    0x0
+#define VPG2_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB0__SHIFT                                                         0x8
+#define VPG2_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB1__SHIFT                                                         0x10
+#define VPG2_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB2__SHIFT                                                         0x18
+#define VPG2_VPG_MPEG_INFO0__VPG_MPEG_INFO_CHECKSUM_MASK                                                      0x000000FFL
+#define VPG2_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB0_MASK                                                           0x0000FF00L
+#define VPG2_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB1_MASK                                                           0x00FF0000L
+#define VPG2_VPG_MPEG_INFO0__VPG_MPEG_INFO_MB2_MASK                                                           0xFF000000L
+//VPG2_VPG_MPEG_INFO1
+#define VPG2_VPG_MPEG_INFO1__VPG_MPEG_INFO_MB3__SHIFT                                                         0x0
+#define VPG2_VPG_MPEG_INFO1__VPG_MPEG_INFO_MF__SHIFT                                                          0x8
+#define VPG2_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR__SHIFT                                                          0xc
+#define VPG2_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE__SHIFT                                                      0x10
+#define VPG2_VPG_MPEG_INFO1__VPG_MPEG_INFO_MB3_MASK                                                           0x000000FFL
+#define VPG2_VPG_MPEG_INFO1__VPG_MPEG_INFO_MF_MASK                                                            0x00000300L
+#define VPG2_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR_MASK                                                            0x00001000L
+#define VPG2_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE_MASK                                                        0x00010000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dme_dme_dispdec
+//DME2_DME_CONTROL
+#define DME2_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID__SHIFT                                                   0x0
+#define DME2_DME_CONTROL__METADATA_ENGINE_EN__SHIFT                                                           0x4
+#define DME2_DME_CONTROL__METADATA_STREAM_TYPE__SHIFT                                                         0x8
+#define DME2_DME_CONTROL__METADATA_DB_PENDING__SHIFT                                                          0xc
+#define DME2_DME_CONTROL__METADATA_DB_TAKEN__SHIFT                                                            0xd
+#define DME2_DME_CONTROL__METADATA_DB_TAKEN_CLR__SHIFT                                                        0x10
+#define DME2_DME_CONTROL__METADATA_DB_DISABLE__SHIFT                                                          0x14
+#define DME2_DME_CONTROL__METADATA_HUBP_REQUESTOR_ID_MASK                                                     0x00000007L
+#define DME2_DME_CONTROL__METADATA_ENGINE_EN_MASK                                                             0x00000010L
+#define DME2_DME_CONTROL__METADATA_STREAM_TYPE_MASK                                                           0x00000100L
+#define DME2_DME_CONTROL__METADATA_DB_PENDING_MASK                                                            0x00001000L
+#define DME2_DME_CONTROL__METADATA_DB_TAKEN_MASK                                                              0x00002000L
+#define DME2_DME_CONTROL__METADATA_DB_TAKEN_CLR_MASK                                                          0x00010000L
+#define DME2_DME_CONTROL__METADATA_DB_DISABLE_MASK                                                            0x00100000L
+//DME2_DME_MEMORY_CONTROL
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE__SHIFT                                                     0x0
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS__SHIFT                                                       0x4
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE__SHIFT                                                     0x8
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                                   0xc
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_FORCE_MASK                                                       0x00000003L
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_DIS_MASK                                                         0x00000010L
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
+#define DME2_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
+
+
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
 
 // addressBlock: dce_dc_hpo_hpo_top_dispdec
 //HPO_TOP_CLOCK_CONTROL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_offset.h
index 7fd906f108039..ad8c2dd480f2f 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_offset.h
@@ -12907,9 +12907,41 @@
 #define regDC_PERFMON22_PERFMON_LOW                                                                     0x0e6e
 #define regDC_PERFMON22_PERFMON_LOW_BASE_IDX                                                            3
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define regHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define regHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define regHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define regHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define regHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define regHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define regHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define regHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define regHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define regHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 // base address: 0x2646c
+#define regAFMT5_AFMT_ACP                                                                               0x091b
+#define regAFMT5_AFMT_ACP_BASE_IDX                                                                      3
 #define regAFMT5_AFMT_VBI_PACKET_CONTROL                                                                0x091c
 #define regAFMT5_AFMT_VBI_PACKET_CONTROL_BASE_IDX                                                       3
 #define regAFMT5_AFMT_AUDIO_PACKET_CONTROL2                                                             0x091d
@@ -12981,6 +13013,83 @@
 #define regVPG5_VPG_MPEG_INFO1                                                                          0x093a
 #define regVPG5_VPG_MPEG_INFO1_BASE_IDX                                                                 3
 
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define regHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define regHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define regHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define regHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define regHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define regHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define regHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define regHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define regHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define regHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define regHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define regHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define regHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define regHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define regHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define regHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define regHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define regHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define regHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define regHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define regHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_MODE                                                                             0x0908
+#define regHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define regHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define regHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
 
 // addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
 // base address: 0x1ab8c
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_sh_mask.h
index 07fbfafe6056d..9c570d781c523 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_2_sh_mask.h
@@ -48391,6 +48391,124 @@
 #define DC_PERFMON22_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
 #define DC_PERFMON22_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK__SHIFT                                 0x8
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK_MASK                                   0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 //AFMT5_AFMT_VBI_PACKET_CONTROL
@@ -48765,6 +48883,436 @@
 #define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR_MASK                                                            0x00001000L
 #define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE_MASK                                                        0x00010000L
 
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_DEBUG_DATA
+#define HDMI_TB_ENC_DEBUG_DATA__HDMI_DEBUG_DATA__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DEBUG_DATA__HDMI_DEBUG_DATA_MASK                                                          0xFFFFFFFFL
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
+
 
 // addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
 //DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_4_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_4_offset.h
index 20a6ee7adeef4..3aeb01346c12d 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_4_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_4_offset.h
@@ -14560,6 +14560,96 @@
 #define regDC_PERFMON22_PERFMON_LOW                                                                     0x08cf
 #define regDC_PERFMON22_PERFMON_LOW_BASE_IDX                                                            3
 
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define regHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define regHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define regHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define regHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define regHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define regHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define regHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define regHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define regHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define regHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define regHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define regHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define regHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define regHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define regHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define regHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define regHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define regHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define regHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define regHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define regHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_MODE                                                                             0x0908
+#define regHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define regHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define regHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 // base address: 0x2646c
@@ -14634,6 +14724,24 @@
 #define regDME9_DME_MEMORY_CONTROL                                                                      0x093d
 #define regDME9_DME_MEMORY_CONTROL_BASE_IDX                                                             3
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define regHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define regHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define regHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define regHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define regHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define regHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define regHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define regHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define regHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define regHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
 
 // addressBlock: dce_dc_hpo_hpo_top_dispdec
 // base address: 0x2790c
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_4_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_4_sh_mask.h
index d3d98d43287ca..4fd16249a2db3 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_4_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_4_sh_mask.h
@@ -59876,6 +59876,489 @@
 #define DC_PERFMON22_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
 #define DC_PERFMON22_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
 
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK__SHIFT                                 0x8
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK_MASK                                   0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 //AFMT5_AFMT_VBI_PACKET_CONTROL
@@ -60252,6 +60735,67 @@
 #define DME9_DME_MEMORY_CONTROL__DME_MEM_PWR_STATE_MASK                                                       0x00000300L
 #define DME9_DME_MEMORY_CONTROL__DME_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                                     0x00003000L
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
 
 // addressBlock: dce_dc_hpo_hpo_top_dispdec
 //HPO_TOP_CLOCK_CONTROL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_offset.h
index 16a69d17bb1e1..45ec5cb0f90c1 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_offset.h
@@ -12773,6 +12773,39 @@
 #define regDC_PERFMON22_PERFMON_LOW_BASE_IDX                                                            3
 
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define regHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define regHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define regHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define regHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define regHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define regHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define regHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define regHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define regHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define regHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 // base address: 0x2646c
 #define regAFMT5_AFMT_VBI_PACKET_CONTROL                                                                0x091c
@@ -12845,6 +12878,84 @@
 #define regVPG5_VPG_MPEG_INFO1_BASE_IDX                                                                 3
 
 
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define regHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define regHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define regHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define regHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define regHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define regHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define regHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define regHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define regHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define regHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define regHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define regHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define regHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define regHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define regHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define regHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define regHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define regHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define regHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define regHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define regHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_MODE                                                                             0x0908
+#define regHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define regHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define regHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
+
 // addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
 // base address: 0x1ab8c
 #define regDP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL                                                   0x3623
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_sh_mask.h
index 6473362e39a89..6bc91ae446828 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_5_sh_mask.h
@@ -46664,6 +46664,126 @@
 #define DC_PERFMON22_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
 
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK__SHIFT                                 0x8
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK_MASK                                   0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
+
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 //AFMT5_AFMT_VBI_PACKET_CONTROL
 #define AFMT5_AFMT_VBI_PACKET_CONTROL__AFMT_ACP_SOURCE__SHIFT                                                 0xd
@@ -47041,6 +47161,434 @@
 #define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE_MASK                                                        0x00010000L
 
 
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
+
+
 // addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
 //DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL
 #define DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL__DP_STREAM_ENC_CLOCK_EN__SHIFT                             0x0
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_6_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_6_offset.h
index a05bf8e4f58d9..d289d8356adf1 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_6_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_6_offset.h
@@ -13503,6 +13503,39 @@
 #define regDC_PERFMON22_PERFMON_LOW                                                                     0x0e6e
 #define regDC_PERFMON22_PERFMON_LOW_BASE_IDX                                                            3
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define regHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define regHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define regHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define regHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define regHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define regHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define regHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define regHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define regHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define regHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 // base address: 0x2646c
@@ -13578,6 +13611,84 @@
 #define regVPG5_VPG_MPEG_INFO1_BASE_IDX                                                                 3
 
 
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define regHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define regHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define regHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define regHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define regHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define regHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define regHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define regHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define regHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define regHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define regHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define regHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define regHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define regHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define regHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define regHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define regHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define regHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define regHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define regHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define regHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_MODE                                                                             0x0908
+#define regHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define regHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define regHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
+
 // addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
 // base address: 0x1ab8c
 #define regDP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL                                                   0x3623
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_6_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_6_sh_mask.h
index df84941bbe5be..dd98cd5582cf2 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_6_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_1_6_sh_mask.h
@@ -50009,6 +50009,125 @@
 #define DC_PERFMON22_PERFMON_LOW__PERFMON_LOW__SHIFT                                                          0x0
 #define DC_PERFMON22_PERFMON_LOW__PERFMON_LOW_MASK                                                            0xFFFFFFFFL
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK__SHIFT                                 0x8
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK_MASK                                   0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 //AFMT5_AFMT_VBI_PACKET_CONTROL
@@ -50386,6 +50505,433 @@
 #define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR_MASK                                                            0x00001000L
 #define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE_MASK                                                        0x00010000L
 
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
+
 
 // addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
 //DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_offset.h
index 78cb61d5800ac..bc4524f9b462a 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_offset.h
@@ -12201,6 +12201,38 @@
 #define regDP_STREAM_MAPPER_CONTROL3                                                                    0x0e59
 #define regDP_STREAM_MAPPER_CONTROL3_BASE_IDX                                                           3
 
+// addressBlock: dcn_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define regHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define regHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define regHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define regHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+
+// addressBlock: dcn_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define regHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define regHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define regHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define regHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define regHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define regHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
+
+// addressBlock: dcn_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
 
 // addressBlock: dcn_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 // base address: 0x2646c
@@ -12275,6 +12307,83 @@
 #define regVPG5_VPG_MPEG_INFO1                                                                          0x093a
 #define regVPG5_VPG_MPEG_INFO1_BASE_IDX                                                                 3
 
+// addressBlock: dcn_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define regHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define regHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define regHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define regHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define regHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define regHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define regHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define regHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define regHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define regHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define regHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define regHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define regHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define regHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define regHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define regHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define regHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define regHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define regHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define regHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define regHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_MODE                                                                             0x0908
+#define regHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define regHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define regHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
 
 // addressBlock: dcn_dc_hpo_dp_stream_enc0_dispdec
 // base address: 0x1ab8c
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_sh_mask.h
index c20bf730dc553..a89fe19c4c842 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_0_sh_mask.h
@@ -43567,6 +43567,124 @@
 #define DP_STREAM_MAPPER_CONTROL3__DP_STREAM_LINK_TARGET__SHIFT                                               0x0
 #define DP_STREAM_MAPPER_CONTROL3__DP_STREAM_LINK_TARGET_MASK                                                 0x00000007L
 
+// addressBlock: dcn_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+// addressBlock: dcn_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
+
+// addressBlock: dcn_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK__SHIFT                                 0x8
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK_MASK                                   0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
 
 // addressBlock: dcn_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 //AFMT5_AFMT_VBI_PACKET_CONTROL
@@ -43943,6 +44061,432 @@
 #define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR_MASK                                                            0x00001000L
 #define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE_MASK                                                        0x00010000L
 
+// addressBlock: dcn_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
 
 // addressBlock: dcn_dc_hpo_dp_stream_enc0_dispdec
 //DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_offset.h
index a04b8c32c5649..20deafe52a07a 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_offset.h
@@ -11814,7 +11814,6 @@
 #define regDSCC0_DSCC_TEST_DEBUG_DATA0                                                                  0x303b
 #define regDSCC0_DSCC_TEST_DEBUG_DATA0_BASE_IDX                                                         2
 
-
 // addressBlock: dce_dc_dsc0_dispdec_dsccif_dispdec
 // base address: 0x0
 #define regDSCCIF0_DSCCIF_CONFIG0                                                                       0x3005
@@ -12173,6 +12172,19 @@
 #define regHPO_TOP_HW_CONTROL                                                                           0x0e4a
 #define regHPO_TOP_HW_CONTROL_BASE_IDX                                                                  3
 
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+
 
 // addressBlock: dce_dc_hpo_dp_stream_mapper_dispdec
 // base address: 0x27958
@@ -12185,6 +12197,39 @@
 #define regDP_STREAM_MAPPER_CONTROL3                                                                    0x0e59
 #define regDP_STREAM_MAPPER_CONTROL3_BASE_IDX                                                           3
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+// base address: 0x2656c
+#define regHDMI_LINK_ENC_CONTROL                                                                        0x095b
+#define regHDMI_LINK_ENC_CONTROL_BASE_IDX                                                               3
+#define regHDMI_LINK_ENC_CLK_CTRL                                                                       0x095c
+#define regHDMI_LINK_ENC_CLK_CTRL_BASE_IDX                                                              3
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+// base address: 0x26594
+#define regHDMI_FRL_ENC_CONFIG                                                                          0x0965
+#define regHDMI_FRL_ENC_CONFIG_BASE_IDX                                                                 3
+#define regHDMI_FRL_ENC_CONFIG2                                                                         0x0966
+#define regHDMI_FRL_ENC_CONFIG2_BASE_IDX                                                                3
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS                                                             0x0967
+#define regHDMI_FRL_ENC_METER_BUFFER_STATUS_BASE_IDX                                                    3
+#define regHDMI_FRL_ENC_MEM_CTRL                                                                        0x0968
+#define regHDMI_FRL_ENC_MEM_CTRL_BASE_IDX                                                               3
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+// base address: 0x2634c
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL                                                                0x08d3
+#define regHDMI_STREAM_ENC_CLOCK_CONTROL_BASE_IDX                                                       3
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL                                                            0x08d5
+#define regHDMI_STREAM_ENC_INPUT_MUX_CONTROL_BASE_IDX                                                   3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0                                     0x08d6
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1                                     0x08d7
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1_BASE_IDX                            3
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2                                     0x08d8
+#define regHDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2_BASE_IDX                            3
+
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 // base address: 0x2646c
@@ -12259,6 +12304,83 @@
 #define regVPG5_VPG_MPEG_INFO1                                                                          0x093a
 #define regVPG5_VPG_MPEG_INFO1_BASE_IDX                                                                 3
 
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+// base address: 0x2637c
+#define regHDMI_TB_ENC_CONTROL                                                                          0x08df
+#define regHDMI_TB_ENC_CONTROL_BASE_IDX                                                                 3
+#define regHDMI_TB_ENC_PIXEL_FORMAT                                                                     0x08e0
+#define regHDMI_TB_ENC_PIXEL_FORMAT_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_PACKET_CONTROL                                                                   0x08e1
+#define regHDMI_TB_ENC_PACKET_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL                                                               0x08e2
+#define regHDMI_TB_ENC_ACR_PACKET_CONTROL_BASE_IDX                                                      3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1                                                              0x08e3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL1_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2                                                              0x08e4
+#define regHDMI_TB_ENC_VBI_PACKET_CONTROL2_BASE_IDX                                                     3
+#define regHDMI_TB_ENC_GC_CONTROL                                                                       0x08e5
+#define regHDMI_TB_ENC_GC_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0                                                          0x08e6
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL0_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1                                                          0x08e7
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL1_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2                                                          0x08e8
+#define regHDMI_TB_ENC_GENERIC_PACKET_CONTROL2_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE                                                           0x08e9
+#define regHDMI_TB_ENC_GENERIC_PACKET0_1_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE                                                           0x08ea
+#define regHDMI_TB_ENC_GENERIC_PACKET2_3_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE                                                           0x08eb
+#define regHDMI_TB_ENC_GENERIC_PACKET4_5_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE                                                           0x08ec
+#define regHDMI_TB_ENC_GENERIC_PACKET6_7_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE                                                           0x08ed
+#define regHDMI_TB_ENC_GENERIC_PACKET8_9_LINE_BASE_IDX                                                  3
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE                                                         0x08ee
+#define regHDMI_TB_ENC_GENERIC_PACKET10_11_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE                                                         0x08ef
+#define regHDMI_TB_ENC_GENERIC_PACKET12_13_LINE_BASE_IDX                                                3
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE                                                            0x08f0
+#define regHDMI_TB_ENC_GENERIC_PACKET14_LINE_BASE_IDX                                                   3
+#define regHDMI_TB_ENC_DB_CONTROL                                                                       0x08f1
+#define regHDMI_TB_ENC_DB_CONTROL_BASE_IDX                                                              3
+#define regHDMI_TB_ENC_ACR_32_0                                                                         0x08f2
+#define regHDMI_TB_ENC_ACR_32_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_32_1                                                                         0x08f3
+#define regHDMI_TB_ENC_ACR_32_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_0                                                                         0x08f4
+#define regHDMI_TB_ENC_ACR_44_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_44_1                                                                         0x08f5
+#define regHDMI_TB_ENC_ACR_44_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_0                                                                         0x08f6
+#define regHDMI_TB_ENC_ACR_48_0_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_48_1                                                                         0x08f7
+#define regHDMI_TB_ENC_ACR_48_1_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_ACR_STATUS_0                                                                     0x08f8
+#define regHDMI_TB_ENC_ACR_STATUS_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_ACR_STATUS_1                                                                     0x08f9
+#define regHDMI_TB_ENC_ACR_STATUS_1_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_BUFFER_CONTROL                                                                   0x08fb
+#define regHDMI_TB_ENC_BUFFER_CONTROL_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_MEM_CTRL                                                                         0x08fe
+#define regHDMI_TB_ENC_MEM_CTRL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL                                                          0x08ff
+#define regHDMI_TB_ENC_METADATA_PACKET_CONTROL_BASE_IDX                                                 3
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK                                                                   0x0900
+#define regHDMI_TB_ENC_H_ACTIVE_BLANK_BASE_IDX                                                          3
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK                                                                  0x0901
+#define regHDMI_TB_ENC_HC_ACTIVE_BLANK_BASE_IDX                                                         3
+#define regHDMI_TB_ENC_CRC_CNTL                                                                         0x0903
+#define regHDMI_TB_ENC_CRC_CNTL_BASE_IDX                                                                3
+#define regHDMI_TB_ENC_CRC_RESULT_0                                                                     0x0904
+#define regHDMI_TB_ENC_CRC_RESULT_0_BASE_IDX                                                            3
+#define regHDMI_TB_ENC_MODE                                                                             0x0908
+#define regHDMI_TB_ENC_MODE_BASE_IDX                                                                    3
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS                                                                0x0909
+#define regHDMI_TB_ENC_INPUT_FIFO_STATUS_BASE_IDX                                                       3
+#define regHDMI_TB_ENC_CRC_RESULT_1                                                                     0x090a
+#define regHDMI_TB_ENC_CRC_RESULT_1_BASE_IDX                                                            3
+
 
 // addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
 // base address: 0x1ab8c
diff --git a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_sh_mask.h
index ce773fca621f5..c22929134196f 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/dcn/dcn_3_2_1_sh_mask.h
@@ -43576,6 +43576,125 @@
 #define DP_STREAM_MAPPER_CONTROL3__DP_STREAM_LINK_TARGET__SHIFT                                               0x0
 #define DP_STREAM_MAPPER_CONTROL3__DP_STREAM_LINK_TARGET_MASK                                                 0x00000007L
 
+// addressBlock: dce_dc_hpo_hdmi_link_enc0_dispdec
+//HDMI_LINK_ENC_CONTROL
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE__SHIFT                                                    0x0
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET__SHIFT                                                0x4
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_ENABLE_MASK                                                      0x00000001L
+#define HDMI_LINK_ENC_CONTROL__HDMI_LINK_ENC_SOFT_RESET_MASK                                                  0x00000010L
+//HDMI_LINK_ENC_CLK_CTRL
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN__SHIFT                                                 0x0
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK__SHIFT                                     0x1
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_EN_MASK                                                   0x00000001L
+#define HDMI_LINK_ENC_CLK_CTRL__HDMI_LINK_ENC_CLOCK_ON_HDMICHARCLK_MASK                                       0x00000002L
+
+
+// addressBlock: dce_dc_hpo_hdmi_frl_enc0_dispdec
+//HDMI_FRL_ENC_CONFIG
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT__SHIFT                                                      0x0
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE__SHIFT                                                 0x1
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE__SHIFT                                               0x2
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN__SHIFT                                          0x10
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN__SHIFT                                          0x14
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN__SHIFT                                          0x18
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN__SHIFT                                          0x1c
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE_COUNT_MASK                                                        0x00000001L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_TRAINING_ENABLE_MASK                                                   0x00000002L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_SCRAMBLER_DISABLE_MASK                                                 0x00000004L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE0_TRAINING_PATTERN_MASK                                            0x000F0000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE1_TRAINING_PATTERN_MASK                                            0x00F00000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE2_TRAINING_PATTERN_MASK                                            0x0F000000L
+#define HDMI_FRL_ENC_CONFIG__HDMI_LINK_LANE3_TRAINING_PATTERN_MASK                                            0xF0000000L
+//HDMI_FRL_ENC_CONFIG2
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE__SHIFT                                               0x0
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD__SHIFT                                               0xc
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN__SHIFT                                                  0x18
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE__SHIFT                                            0x19
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL__SHIFT                                            0x1a
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET__SHIFT                                         0x1c
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS__SHIFT                                           0x1d
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS__SHIFT                                   0x1e
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_MASK                                                 0x000001FFL
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_THRESHOLD_MASK                                                 0x001FF000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_CAL_EN_MASK                                                    0x01000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_RC_COMPRESS_DISABLE_MASK                                              0x02000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_FRL_HDMISTREAMCLK_DB_SEL_MASK                                              0x0C000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_MAX_JITTER_VALUE_RESET_MASK                                           0x10000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_JITTER_EXCEED_STATUS_MASK                                             0x20000000L
+#define HDMI_FRL_ENC_CONFIG2__HDMI_LINK_METER_BUFFER_OVERFLOW_STATUS_MASK                                     0x40000000L
+//HDMI_FRL_ENC_METER_BUFFER_STATUS
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL__SHIFT                             0x0
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET__SHIFT                       0x1f
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_MAX_METER_BUFFER_LEVEL_MASK                               0x0000007FL
+#define HDMI_FRL_ENC_METER_BUFFER_STATUS__HDMI_LINK_METER_BUFFER_MAX_LEVEL_RESET_MASK                         0x80000000L
+//HDMI_FRL_ENC_MEM_CTRL
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_FRL_ENC_MEM_CTRL__METERBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
+
+// addressBlock: dce_dc_hpo_hdmi_stream_enc0_dispdec
+//HDMI_STREAM_ENC_CLOCK_CONTROL
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN__SHIFT                                        0x0
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK__SHIFT                                0x4
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK__SHIFT                                 0x8
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK__SHIFT                          0xc
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_EN_MASK                                          0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_DISPCLK_MASK                                  0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_SOCCLK_MASK                                   0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_CONTROL__HDMI_STREAM_ENC_CLOCK_ON_HDMISTREAMCLK_MASK                            0x00001000L
+//HDMI_STREAM_ENC_INPUT_MUX_CONTROL
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL__SHIFT                        0x0
+#define HDMI_STREAM_ENC_INPUT_MUX_CONTROL__HDMI_STREAM_ENC_INPUT_MUX_SOURCE_SEL_MASK                          0x00000007L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE__SHIFT                          0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET__SHIFT                           0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING__SHIFT                  0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE__SHIFT                0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE__SHIFT                        0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE__SHIFT                      0x14
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE__SHIFT             0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR__SHIFT                           0x1c
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ENABLE_MASK                            0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_MASK                             0x00000010L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_PIXEL_ENCODING_MASK                    0x00000300L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ODM_COMBINE_MODE_MASK                  0x00003000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_DSC_MODE_MASK                          0x00030000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_RESET_DONE_MASK                        0x00100000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_VIDEO_STREAM_ACTIVE_MASK               0x01000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL0__FIFO_ERROR_MASK                             0x30000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL__SHIFT             0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE__SHIFT             0x1
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX__SHIFT             0x2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL__SHIFT                 0x4
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL__SHIFT                   0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL__SHIFT                   0x10
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL__SHIFT               0x18
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED__SHIFT                      0x1f
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_USE_OVERWRITE_LEVEL_MASK               0x00000001L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECAL_AVERAGE_MASK               0x00000002L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_FORCE_RECOMP_MINMAX_MASK               0x00000004L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_OVERWRITE_LEVEL_MASK                   0x000003F0L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MINIMUM_LEVEL_MASK                     0x0000F000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_MAXIMUM_LEVEL_MASK                     0x001F0000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CAL_AVERAGE_LEVEL_MASK                 0x3F000000L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL1__FIFO_CALIBRATED_MASK                        0x80000000L
+//HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL__SHIFT                0x0
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC__SHIFT                  0x5
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING__SHIFT                      0x8
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE__SHIFT                      0xc
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_START_LEVEL_MASK                  0x0000001FL
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_READ_CLOCK_SRC_MASK                    0x00000020L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_PENDING_MASK                        0x00000100L
+#define HDMI_STREAM_ENC_CLOCK_RAMP_ADJUSTER_FIFO_STATUS_CONTROL2__FIFO_DB_DISABLE_MASK                        0x00001000L
+
 
 // addressBlock: dce_dc_hpo_hdmi_stream_enc0_afmt_afmt_dispdec
 //AFMT5_AFMT_VBI_PACKET_CONTROL
@@ -43952,6 +44071,435 @@
 #define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_FR_MASK                                                            0x00001000L
 #define VPG5_VPG_MPEG_INFO1__VPG_MPEG_INFO_UPDATE_MASK                                                        0x00010000L
 
+// addressBlock: dce_dc_hpo_hdmi_tb_enc0_dispdec
+//HDMI_TB_ENC_CONTROL
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN__SHIFT                                                            0x0
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET__SHIFT                                                                0x4
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE__SHIFT                                                           0x8
+#define HDMI_TB_ENC_CONTROL__HDMI_TB_ENC_EN_MASK                                                              0x00000001L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_MASK                                                                  0x00000010L
+#define HDMI_TB_ENC_CONTROL__HDMI_RESET_DONE_MASK                                                             0x00000100L
+//HDMI_TB_ENC_PIXEL_FORMAT
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE__SHIFT                                               0x0
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH__SHIFT                                                0x8
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING__SHIFT                                                  0x10
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE__SHIFT                                                        0x18
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_ENABLE_MASK                                                 0x00000001L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DEEP_COLOR_DEPTH_MASK                                                  0x00000300L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_PIXEL_ENCODING_MASK                                                    0x00030000L
+#define HDMI_TB_ENC_PIXEL_FORMAT__HDMI_DSC_MODE_MASK                                                          0x03000000L
+//HDMI_TB_ENC_PACKET_CONTROL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE__SHIFT                                          0x0
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE__SHIFT                                          0x8
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW__SHIFT                                        0xc
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR__SHIFT                                     0x10
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_PACKETS_PER_LINE_MASK                                            0x0000001FL
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_MAX_ISLANDS_PER_LINE_MASK                                            0x00000300L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_OVERFLOW_MASK                                          0x00001000L
+#define HDMI_TB_ENC_PACKET_CONTROL__HDMI_TB_ENC_PACKET_ERROR_CLEAR_MASK                                       0x00010000L
+//HDMI_TB_ENC_ACR_PACKET_CONTROL
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT__SHIFT                                                0x4
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE__SHIFT                                                0x8
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND__SHIFT                                             0xc
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE__SHIFT                                            0x10
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY__SHIFT                                        0x1f
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SELECT_MASK                                                  0x00000030L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_SOURCE_MASK                                                  0x00000100L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUTO_SEND_MASK                                               0x00001000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_N_MULTIPLE_MASK                                              0x00070000L
+#define HDMI_TB_ENC_ACR_PACKET_CONTROL__HDMI_ACR_AUDIO_PRIORITY_MASK                                          0x80000000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND__SHIFT                                                  0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT__SHIFT                                                  0x1
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND__SHIFT                                                0x4
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT__SHIFT                                                0x5
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE__SHIFT                                      0x6
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND__SHIFT                                                 0x8
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE__SHIFT                                       0x9
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND__SHIFT                                          0xc
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT__SHIFT                                          0xd
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE__SHIFT                                0xe
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE__SHIFT                                          0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_SEND_MASK                                                    0x00000001L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_GC_CONT_MASK                                                    0x00000002L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_SEND_MASK                                                  0x00000010L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_CONT_MASK                                                  0x00000020L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ISRC_LINE_REFERENCE_MASK                                        0x00000040L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_SEND_MASK                                                   0x00000100L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_ACP_LINE_REFERENCE_MASK                                         0x00000200L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_SEND_MASK                                            0x00001000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_CONT_MASK                                            0x00002000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_REFERENCE_MASK                                  0x00004000L
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL1__HDMI_AUDIO_INFO_LINE_MASK                                            0x7FFF0000L
+//HDMI_TB_ENC_VBI_PACKET_CONTROL2
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE__SHIFT                                                0x0
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE__SHIFT                                                 0x10
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ISRC_LINE_MASK                                                  0x00007FFFL
+#define HDMI_TB_ENC_VBI_PACKET_CONTROL2__HDMI_ACP_LINE_MASK                                                   0x7FFF0000L
+//HDMI_TB_ENC_GC_CONTROL
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE__SHIFT                                                         0x0
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT__SHIFT                                                    0x2
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE__SHIFT                                                     0x4
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_MASK                                                           0x00000001L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_GC_AVMUTE_CONT_MASK                                                      0x00000004L
+#define HDMI_TB_ENC_GC_CONTROL__HDMI_DEFAULT_PHASE_MASK                                                       0x00000010L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND__SHIFT                                        0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT__SHIFT                                        0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN__SHIFT                                     0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE__SHIFT                              0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND__SHIFT                                        0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT__SHIFT                                        0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN__SHIFT                                     0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE__SHIFT                              0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND__SHIFT                                        0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT__SHIFT                                        0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN__SHIFT                                     0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE__SHIFT                              0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND__SHIFT                                        0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT__SHIFT                                        0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN__SHIFT                                     0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE__SHIFT                              0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND__SHIFT                                        0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT__SHIFT                                        0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN__SHIFT                                     0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE__SHIFT                              0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND__SHIFT                                        0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT__SHIFT                                        0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN__SHIFT                                     0x1e
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE__SHIFT                              0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC0_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC1_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_SEND_MASK                                          0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_CONT_MASK                                          0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LOCK_EN_MASK                                       0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC2_LINE_REFERENCE_MASK                                0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_SEND_MASK                                          0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_CONT_MASK                                          0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LOCK_EN_MASK                                       0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC3_LINE_REFERENCE_MASK                                0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_SEND_MASK                                          0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_CONT_MASK                                          0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LOCK_EN_MASK                                       0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC4_LINE_REFERENCE_MASK                                0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_SEND_MASK                                          0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_CONT_MASK                                          0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LOCK_EN_MASK                                       0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC5_LINE_REFERENCE_MASK                                0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_SEND_MASK                                          0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_CONT_MASK                                          0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LOCK_EN_MASK                                       0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC6_LINE_REFERENCE_MASK                                0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_SEND_MASK                                          0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_CONT_MASK                                          0x20000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LOCK_EN_MASK                                       0x40000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL0__HDMI_GENERIC7_LINE_REFERENCE_MASK                                0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND__SHIFT                                        0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT__SHIFT                                        0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN__SHIFT                                     0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE__SHIFT                              0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND__SHIFT                                        0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT__SHIFT                                        0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN__SHIFT                                     0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE__SHIFT                              0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND__SHIFT                                       0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT__SHIFT                                       0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN__SHIFT                                    0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE__SHIFT                             0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND__SHIFT                                       0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT__SHIFT                                       0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN__SHIFT                                    0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE__SHIFT                             0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND__SHIFT                                       0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT__SHIFT                                       0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN__SHIFT                                    0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE__SHIFT                             0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND__SHIFT                                       0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT__SHIFT                                       0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN__SHIFT                                    0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE__SHIFT                             0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND__SHIFT                                       0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT__SHIFT                                       0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN__SHIFT                                    0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE__SHIFT                             0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_SEND_MASK                                          0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_CONT_MASK                                          0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LOCK_EN_MASK                                       0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC8_LINE_REFERENCE_MASK                                0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_SEND_MASK                                          0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_CONT_MASK                                          0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LOCK_EN_MASK                                       0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC9_LINE_REFERENCE_MASK                                0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_SEND_MASK                                         0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_CONT_MASK                                         0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LOCK_EN_MASK                                      0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC10_LINE_REFERENCE_MASK                               0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_SEND_MASK                                         0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_CONT_MASK                                         0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LOCK_EN_MASK                                      0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC11_LINE_REFERENCE_MASK                               0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_SEND_MASK                                         0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_CONT_MASK                                         0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LOCK_EN_MASK                                      0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC12_LINE_REFERENCE_MASK                               0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_SEND_MASK                                         0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_CONT_MASK                                         0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LOCK_EN_MASK                                      0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC13_LINE_REFERENCE_MASK                               0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_SEND_MASK                                         0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_CONT_MASK                                         0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LOCK_EN_MASK                                      0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL1__HDMI_GENERIC14_LINE_REFERENCE_MASK                               0x08000000L
+//HDMI_TB_ENC_GENERIC_PACKET_CONTROL2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND__SHIFT                              0x0
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING__SHIFT                      0x1
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND__SHIFT                              0x2
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING__SHIFT                      0x3
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND__SHIFT                              0x4
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING__SHIFT                      0x5
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND__SHIFT                              0x6
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING__SHIFT                      0x7
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND__SHIFT                              0x8
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING__SHIFT                      0x9
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND__SHIFT                              0xa
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING__SHIFT                      0xb
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND__SHIFT                              0xc
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING__SHIFT                      0xd
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND__SHIFT                              0xe
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING__SHIFT                      0xf
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND__SHIFT                              0x10
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING__SHIFT                      0x11
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND__SHIFT                              0x12
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING__SHIFT                      0x13
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND__SHIFT                             0x14
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING__SHIFT                     0x15
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND__SHIFT                             0x16
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING__SHIFT                     0x17
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND__SHIFT                             0x18
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING__SHIFT                     0x19
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND__SHIFT                             0x1a
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING__SHIFT                     0x1b
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND__SHIFT                             0x1c
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING__SHIFT                     0x1d
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_MASK                                0x00000001L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC0_IMMEDIATE_SEND_PENDING_MASK                        0x00000002L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_MASK                                0x00000004L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC1_IMMEDIATE_SEND_PENDING_MASK                        0x00000008L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_MASK                                0x00000010L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC2_IMMEDIATE_SEND_PENDING_MASK                        0x00000020L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_MASK                                0x00000040L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC3_IMMEDIATE_SEND_PENDING_MASK                        0x00000080L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_MASK                                0x00000100L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC4_IMMEDIATE_SEND_PENDING_MASK                        0x00000200L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_MASK                                0x00000400L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC5_IMMEDIATE_SEND_PENDING_MASK                        0x00000800L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_MASK                                0x00001000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC6_IMMEDIATE_SEND_PENDING_MASK                        0x00002000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_MASK                                0x00004000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC7_IMMEDIATE_SEND_PENDING_MASK                        0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_MASK                                0x00010000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC8_IMMEDIATE_SEND_PENDING_MASK                        0x00020000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_MASK                                0x00040000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC9_IMMEDIATE_SEND_PENDING_MASK                        0x00080000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_MASK                               0x00100000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC10_IMMEDIATE_SEND_PENDING_MASK                       0x00200000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_MASK                               0x00400000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC11_IMMEDIATE_SEND_PENDING_MASK                       0x00800000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_MASK                               0x01000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC12_IMMEDIATE_SEND_PENDING_MASK                       0x02000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_MASK                               0x04000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC13_IMMEDIATE_SEND_PENDING_MASK                       0x08000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_MASK                               0x10000000L
+#define HDMI_TB_ENC_GENERIC_PACKET_CONTROL2__HDMI_GENERIC14_IMMEDIATE_SEND_PENDING_MASK                       0x20000000L
+//HDMI_TB_ENC_GENERIC_PACKET0_1_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC0_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET0_1_LINE__HDMI_GENERIC1_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET2_3_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC2_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET2_3_LINE__HDMI_GENERIC3_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET4_5_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC4_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET4_5_LINE__HDMI_GENERIC5_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET6_7_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC6_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET6_7_LINE__HDMI_GENERIC7_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET8_9_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE__SHIFT                                         0x10
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP__SHIFT                                          0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC8_EMP_MASK                                            0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_LINE_MASK                                           0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET8_9_LINE__HDMI_GENERIC9_EMP_MASK                                            0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET10_11_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC10_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET10_11_LINE__HDMI_GENERIC11_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET12_13_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE__SHIFT                                      0x0
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP__SHIFT                                       0xf
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE__SHIFT                                      0x10
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP__SHIFT                                       0x1f
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_LINE_MASK                                        0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC12_EMP_MASK                                         0x00008000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_LINE_MASK                                        0x7FFF0000L
+#define HDMI_TB_ENC_GENERIC_PACKET12_13_LINE__HDMI_GENERIC13_EMP_MASK                                         0x80000000L
+//HDMI_TB_ENC_GENERIC_PACKET14_LINE
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE__SHIFT                                         0x0
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP__SHIFT                                          0xf
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_LINE_MASK                                           0x00007FFFL
+#define HDMI_TB_ENC_GENERIC_PACKET14_LINE__HDMI_GENERIC14_EMP_MASK                                            0x00008000L
+//HDMI_TB_ENC_DB_CONTROL
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING__SHIFT                                                        0x0
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE__SHIFT                                                        0xc
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING__SHIFT                                                     0xf
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_PENDING_MASK                                                          0x00000001L
+#define HDMI_TB_ENC_DB_CONTROL__HDMI_DB_DISABLE_MASK                                                          0x00001000L
+#define HDMI_TB_ENC_DB_CONTROL__VUPDATE_DB_PENDING_MASK                                                       0x00008000L
+//HDMI_TB_ENC_ACR_32_0
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_32_0__HDMI_ACR_CTS_32_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_32_1
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_32_1__HDMI_ACR_N_32_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_44_0
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_44_0__HDMI_ACR_CTS_44_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_44_1
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_44_1__HDMI_ACR_N_44_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_48_0
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48__SHIFT                                                          0xc
+#define HDMI_TB_ENC_ACR_48_0__HDMI_ACR_CTS_48_MASK                                                            0xFFFFF000L
+//HDMI_TB_ENC_ACR_48_1
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48__SHIFT                                                            0x0
+#define HDMI_TB_ENC_ACR_48_1__HDMI_ACR_N_48_MASK                                                              0x000FFFFFL
+//HDMI_TB_ENC_ACR_STATUS_0
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS__SHIFT                                                         0xc
+#define HDMI_TB_ENC_ACR_STATUS_0__HDMI_ACR_CTS_MASK                                                           0xFFFFF000L
+//HDMI_TB_ENC_ACR_STATUS_1
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N__SHIFT                                                           0x0
+#define HDMI_TB_ENC_ACR_STATUS_1__HDMI_ACR_N_MASK                                                             0x000FFFFFL
+//HDMI_TB_ENC_BUFFER_CONTROL
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN__SHIFT                              0x0
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN__SHIFT                               0x1
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET__SHIFT                              0x4
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                           0x8
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL__SHIFT                            0x18
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_EN_MASK                                0x00000001L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_EN_MASK                                 0x00000002L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_MAX_MIN_LEVEL_RESET_MASK                                0x00000010L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_BORROWBUFFER_PREFILL_OVERRIDE_LEVEL_MASK                             0x0000FF00L
+#define HDMI_TB_ENC_BUFFER_CONTROL__HDMI_RATE_BUFFER_PREFILL_OVERRIDE_LEVEL_MASK                              0x1F000000L
+//HDMI_TB_ENC_MEM_CTRL
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS__SHIFT                                                 0x0
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE__SHIFT                                               0x1
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE__SHIFT                                               0x4
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE__SHIFT                             0x8
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_DIS_MASK                                                   0x00000001L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_FORCE_MASK                                                 0x00000006L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_PWR_STATE_MASK                                                 0x00000030L
+#define HDMI_TB_ENC_MEM_CTRL__BORROWBUFFER_MEM_DEFAULT_MEM_LOW_POWER_STATE_MASK                               0x00000300L
+
+
+//HDMI_TB_ENC_METADATA_PACKET_CONTROL
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE__SHIFT                               0x0
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE__SHIFT                       0x4
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED__SHIFT                               0x8
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE__SHIFT                                 0x10
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_ENABLE_MASK                                 0x00000001L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_REFERENCE_MASK                         0x00000010L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_MISSED_MASK                                 0x00000100L
+#define HDMI_TB_ENC_METADATA_PACKET_CONTROL__HDMI_METADATA_PACKET_LINE_MASK                                   0xFFFF0000L
+//HDMI_TB_ENC_H_ACTIVE_BLANK
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE__SHIFT                                                      0x0
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK__SHIFT                                                       0x10
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_ACTIVE_MASK                                                        0x00007FFFL
+#define HDMI_TB_ENC_H_ACTIVE_BLANK__HDMI_H_BLANK_MASK                                                         0x7FFF0000L
+//HDMI_TB_ENC_HC_ACTIVE_BLANK
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE__SHIFT                                                    0x0
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK__SHIFT                                                     0x10
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_ACTIVE_MASK                                                      0x00007FFFL
+#define HDMI_TB_ENC_HC_ACTIVE_BLANK__HDMI_HC_BLANK_MASK                                                       0x7FFF0000L
+//HDMI_TB_ENC_CRC_CNTL
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN__SHIFT                                                              0x0
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN__SHIFT                                                         0x1
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE__SHIFT                                                            0x8
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL__SHIFT                                                         0xa
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN__SHIFT                                                    0x10
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE__SHIFT                                                  0x11
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_EN_MASK                                                                0x00000001L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_CONT_EN_MASK                                                           0x00000002L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_TYPE_MASK                                                              0x00000300L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_SRC_SEL_MASK                                                           0x00000C00L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_EN_MASK                                                      0x00010000L
+#define HDMI_TB_ENC_CRC_CNTL__HDMI_CRC_INTERLACE_MODE_MASK                                                    0x00060000L
+//HDMI_TB_ENC_CRC_RESULT_0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1__SHIFT                                                         0x10
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE0_MASK                                                           0x0000FFFFL
+#define HDMI_TB_ENC_CRC_RESULT_0__CRC_TRIBYTE1_MASK                                                           0xFFFF0000L
+//HDMI_TB_ENC_MODE
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE__SHIFT                                                             0x0
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK__SHIFT                                                       0x8
+#define HDMI_TB_ENC_MODE__HDMI_BORROW_MODE_MASK                                                               0x00000003L
+#define HDMI_TB_ENC_MODE__HDMI_SKIP_FIRST_HBLANK_MASK                                                         0x00000100L
+//HDMI_TB_ENC_INPUT_FIFO_STATUS
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR__SHIFT                                                0x0
+#define HDMI_TB_ENC_INPUT_FIFO_STATUS__INPUT_FIFO_ERROR_MASK                                                  0x00000001L
+//HDMI_TB_ENC_CRC_RESULT_1
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2__SHIFT                                                         0x0
+#define HDMI_TB_ENC_CRC_RESULT_1__CRC_TRIBYTE2_MASK                                                           0x0000FFFFL
+
 
 // addressBlock: dce_dc_hpo_dp_stream_enc0_dispdec
 //DP_STREAM_ENC0_DP_STREAM_ENC_CLOCK_CONTROL
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h
index d6a2839b4682a..025e5490e6f12 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_offset.h
@@ -5475,6 +5475,14 @@
 #define regCHA_PERFCOUNTER3_HI_BASE_IDX                                                                 1
 
 
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_perfddec
+// base address: 0x34000
+#define regGC_CANE_PERFCOUNTER0_LO                                                                      0x36c0
+#define regGC_CANE_PERFCOUNTER0_LO_BASE_IDX                                                             1
+#define regGC_CANE_PERFCOUNTER0_HI                                                                      0x36c1
+#define regGC_CANE_PERFCOUNTER0_HI_BASE_IDX                                                             1
+
+
 // addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_perfsdec
 // base address: 0x36000
 #define regCPG_PERFCOUNTER1_SELECT                                                                      0x3800
@@ -5761,6 +5769,16 @@
 #define regCHA_PERFCOUNTER3_SELECT1_BASE_IDX                                                            1
 
 
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_perfsdec
+// base address: 0x36000
+#define regGC_CANE_PERFCOUNTER0_SELECT                                                                  0x3e80
+#define regGC_CANE_PERFCOUNTER0_SELECT_BASE_IDX                                                         1
+#define regGC_CANE_PERFCOUNTER0_SELECT1                                                                 0x3e81
+#define regGC_CANE_PERFCOUNTER0_SELECT1_BASE_IDX                                                        1
+#define regGC_CANE_PERFCOUNTER0_MODE                                                                    0x3e82
+#define regGC_CANE_PERFCOUNTER0_MODE_BASE_IDX                                                           1
+
+
 // addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gdfll_xvmin_xvmin_xvmin_reg_blk
 // base address: 0x3a014
 #define regXVMIN_XVMIN_WR_DATA                                                                          0x4806
@@ -12080,6 +12098,22 @@
 #define regGC_ATC_L2_PERFCOUNTER_HI_BASE_IDX                                                            1
 
 
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcutcl2ffbmprdec
+// base address: 0x353d0
+#define regGCUTCL2_FFBM_PERFCOUNTER_LO                                                                  0x34f4
+#define regGCUTCL2_FFBM_PERFCOUNTER_LO_BASE_IDX                                                         1
+#define regGCUTCL2_FFBM_PERFCOUNTER_HI                                                                  0x34f5
+#define regGCUTCL2_FFBM_PERFCOUNTER_HI_BASE_IDX                                                         1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcutcl2nhttlbprdec
+// base address: 0x353e0
+#define regGCUTCL2_NHTTLB_PERFCOUNTER_LO                                                                0x34f8
+#define regGCUTCL2_NHTTLB_PERFCOUNTER_LO_BASE_IDX                                                       1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER_HI                                                                0x34f9
+#define regGCUTCL2_NHTTLB_PERFCOUNTER_HI_BASE_IDX                                                       1
+
+
 // addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbprdec
 // base address: 0x353f0
 #define regGC_L2TLB_PERFCOUNTER_LO                                                                      0x34fc
@@ -12176,6 +12210,82 @@
 #define regGC_ATC_L2_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                     1
 
 
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcutcl2ffbmpldec
+// base address: 0x37bb0
+#define regGCUTCL2_FFBM_PERFCOUNTER0_CFG                                                                0x3eec
+#define regGCUTCL2_FFBM_PERFCOUNTER0_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER1_CFG                                                                0x3eed
+#define regGCUTCL2_FFBM_PERFCOUNTER1_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER2_CFG                                                                0x3eee
+#define regGCUTCL2_FFBM_PERFCOUNTER2_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER3_CFG                                                                0x3eef
+#define regGCUTCL2_FFBM_PERFCOUNTER3_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER4_CFG                                                                0x3ef0
+#define regGCUTCL2_FFBM_PERFCOUNTER4_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER5_CFG                                                                0x3ef1
+#define regGCUTCL2_FFBM_PERFCOUNTER5_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER6_CFG                                                                0x3ef2
+#define regGCUTCL2_FFBM_PERFCOUNTER6_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER7_CFG                                                                0x3ef3
+#define regGCUTCL2_FFBM_PERFCOUNTER7_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER8_CFG                                                                0x3ef4
+#define regGCUTCL2_FFBM_PERFCOUNTER8_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER9_CFG                                                                0x3ef5
+#define regGCUTCL2_FFBM_PERFCOUNTER9_CFG_BASE_IDX                                                       1
+#define regGCUTCL2_FFBM_PERFCOUNTER10_CFG                                                               0x3ef6
+#define regGCUTCL2_FFBM_PERFCOUNTER10_CFG_BASE_IDX                                                      1
+#define regGCUTCL2_FFBM_PERFCOUNTER11_CFG                                                               0x3ef7
+#define regGCUTCL2_FFBM_PERFCOUNTER11_CFG_BASE_IDX                                                      1
+#define regGCUTCL2_FFBM_PERFCOUNTER12_CFG                                                               0x3ef8
+#define regGCUTCL2_FFBM_PERFCOUNTER12_CFG_BASE_IDX                                                      1
+#define regGCUTCL2_FFBM_PERFCOUNTER13_CFG                                                               0x3ef9
+#define regGCUTCL2_FFBM_PERFCOUNTER13_CFG_BASE_IDX                                                      1
+#define regGCUTCL2_FFBM_PERFCOUNTER14_CFG                                                               0x3efa
+#define regGCUTCL2_FFBM_PERFCOUNTER14_CFG_BASE_IDX                                                      1
+#define regGCUTCL2_FFBM_PERFCOUNTER15_CFG                                                               0x3efb
+#define regGCUTCL2_FFBM_PERFCOUNTER15_CFG_BASE_IDX                                                      1
+#define regGCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL                                                           0x3efc
+#define regGCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                  1
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcutcl2nhttlbpldec
+// base address: 0x37c00
+#define regGCUTCL2_NHTTLB_PERFCOUNTER0_CFG                                                              0x3f00
+#define regGCUTCL2_NHTTLB_PERFCOUNTER0_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER1_CFG                                                              0x3f01
+#define regGCUTCL2_NHTTLB_PERFCOUNTER1_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER2_CFG                                                              0x3f02
+#define regGCUTCL2_NHTTLB_PERFCOUNTER2_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER3_CFG                                                              0x3f03
+#define regGCUTCL2_NHTTLB_PERFCOUNTER3_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER4_CFG                                                              0x3f04
+#define regGCUTCL2_NHTTLB_PERFCOUNTER4_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER5_CFG                                                              0x3f05
+#define regGCUTCL2_NHTTLB_PERFCOUNTER5_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER6_CFG                                                              0x3f06
+#define regGCUTCL2_NHTTLB_PERFCOUNTER6_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER7_CFG                                                              0x3f07
+#define regGCUTCL2_NHTTLB_PERFCOUNTER7_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER8_CFG                                                              0x3f08
+#define regGCUTCL2_NHTTLB_PERFCOUNTER8_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER9_CFG                                                              0x3f09
+#define regGCUTCL2_NHTTLB_PERFCOUNTER9_CFG_BASE_IDX                                                     1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER10_CFG                                                             0x3f0a
+#define regGCUTCL2_NHTTLB_PERFCOUNTER10_CFG_BASE_IDX                                                    1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER11_CFG                                                             0x3f0b
+#define regGCUTCL2_NHTTLB_PERFCOUNTER11_CFG_BASE_IDX                                                    1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER12_CFG                                                             0x3f0c
+#define regGCUTCL2_NHTTLB_PERFCOUNTER12_CFG_BASE_IDX                                                    1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER13_CFG                                                             0x3f0d
+#define regGCUTCL2_NHTTLB_PERFCOUNTER13_CFG_BASE_IDX                                                    1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER14_CFG                                                             0x3f0e
+#define regGCUTCL2_NHTTLB_PERFCOUNTER14_CFG_BASE_IDX                                                    1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER15_CFG                                                             0x3f0f
+#define regGCUTCL2_NHTTLB_PERFCOUNTER15_CFG_BASE_IDX                                                    1
+#define regGCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL                                                         0x3f10
+#define regGCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL_BASE_IDX                                                1
+
+
 // addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbpldec
 // base address: 0x37c50
 #define regGC_L2TLB_PERFCOUNTER0_CFG                                                                    0x3f14
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h
index f606c5cd03c59..1cf890de35cbd 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_0_sh_mask.h
@@ -16951,6 +16951,15 @@
 #define CHA_PERFCOUNTER3_HI__PERFCOUNTER_HI_MASK                                                              0xFFFFFFFFL
 
 
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_perfddec
+//GC_CANE_PERFCOUNTER0_LO
+#define GC_CANE_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                        0x0
+#define GC_CANE_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GC_CANE_PERFCOUNTER0_HI
+#define GC_CANE_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                        0x0
+#define GC_CANE_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                          0xFFFFFFFFL
+
+
 // addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_cpwd_perfsdec
 //CPG_PERFCOUNTER1_SELECT
 #define CPG_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                              0x0
@@ -18123,6 +18132,46 @@
 #define CHA_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                             0xF0000000L
 
 
+// addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_perfsdec
+//GC_CANE_PERFCOUNTER0_SELECT
+#define GC_CANE_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                          0x0
+#define GC_CANE_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                         0xa
+#define GC_CANE_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                         0x14
+#define GC_CANE_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                        0x18
+#define GC_CANE_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                         0x1c
+#define GC_CANE_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                            0x000003FFL
+#define GC_CANE_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                           0x000FFC00L
+#define GC_CANE_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                           0x00F00000L
+#define GC_CANE_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                          0x0F000000L
+#define GC_CANE_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                           0xF0000000L
+//GC_CANE_PERFCOUNTER0_SELECT1
+#define GC_CANE_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                        0x0
+#define GC_CANE_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                        0xa
+#define GC_CANE_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                       0x18
+#define GC_CANE_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                       0x1c
+#define GC_CANE_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                          0x000003FFL
+#define GC_CANE_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                          0x000FFC00L
+#define GC_CANE_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                         0x0F000000L
+#define GC_CANE_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                         0xF0000000L
+//GC_CANE_PERFCOUNTER0_MODE
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_MODE0__SHIFT                                                       0x0
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_MODE1__SHIFT                                                       0x2
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_MODE2__SHIFT                                                       0x4
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_MODE3__SHIFT                                                       0x6
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_VALUE0__SHIFT                                                      0x8
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_VALUE1__SHIFT                                                      0xc
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_VALUE2__SHIFT                                                      0x10
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_VALUE3__SHIFT                                                      0x14
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_MODE0_MASK                                                         0x00000003L
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_MODE1_MASK                                                         0x0000000CL
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_MODE2_MASK                                                         0x00000030L
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_MODE3_MASK                                                         0x000000C0L
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_VALUE0_MASK                                                        0x00000F00L
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_VALUE1_MASK                                                        0x0000F000L
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_VALUE2_MASK                                                        0x000F0000L
+#define GC_CANE_PERFCOUNTER0_MODE__COMPARE_VALUE3_MASK                                                        0x00F00000L
+
+
 // addressBlock: CHIP_XCD_gfxip_xcc_gfx_cpwd_gdfll_xvmin_xvmin_xvmin_reg_blk
 //XVMIN_XVMIN_WR_DATA
 #define XVMIN_XVMIN_WR_DATA__XVMINDATA__SHIFT                                                                 0x0
@@ -43203,6 +43252,28 @@
 #define GC_ATC_L2_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                          0xFFFF0000L
 
 
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcutcl2ffbmprdec
+//GCUTCL2_FFBM_PERFCOUNTER_LO
+#define GCUTCL2_FFBM_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER_LO__COUNTER_LO_MASK                                                          0xFFFFFFFFL
+//GCUTCL2_FFBM_PERFCOUNTER_HI
+#define GCUTCL2_FFBM_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                     0x10
+#define GCUTCL2_FFBM_PERFCOUNTER_HI__COUNTER_HI_MASK                                                          0x0000FFFFL
+#define GCUTCL2_FFBM_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                       0xFFFF0000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcutcl2nhttlbprdec
+//GCUTCL2_NHTTLB_PERFCOUNTER_LO
+#define GCUTCL2_NHTTLB_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER_LO__COUNTER_LO_MASK                                                        0xFFFFFFFFL
+//GCUTCL2_NHTTLB_PERFCOUNTER_HI
+#define GCUTCL2_NHTTLB_PERFCOUNTER_HI__COUNTER_HI__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER_HI__COMPARE_VALUE__SHIFT                                                   0x10
+#define GCUTCL2_NHTTLB_PERFCOUNTER_HI__COUNTER_HI_MASK                                                        0x0000FFFFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER_HI__COMPARE_VALUE_MASK                                                     0xFFFF0000L
+
+
 // addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbprdec
 //GC_L2TLB_PERFCOUNTER_LO
 #define GC_L2TLB_PERFCOUNTER_LO__COUNTER_LO__SHIFT                                                            0x0
@@ -43660,6 +43731,390 @@
 #define GC_ATC_L2_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                            0x04000000L
 
 
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcutcl2ffbmpldec
+//GCUTCL2_FFBM_PERFCOUNTER0_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER0_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER1_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER1_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER2_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER2_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER3_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER3_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER4_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER4_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER5_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER5_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER6_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER6_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER7_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER7_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER8_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER8_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER9_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__PERF_SEL__SHIFT                                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__PERF_SEL_END__SHIFT                                                    0x8
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__PERF_MODE__SHIFT                                                       0x18
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__ENABLE__SHIFT                                                          0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__CLEAR__SHIFT                                                           0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__PERF_SEL_MASK                                                          0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__PERF_SEL_END_MASK                                                      0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__PERF_MODE_MASK                                                         0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__ENABLE_MASK                                                            0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER9_CFG__CLEAR_MASK                                                             0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER10_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__PERF_SEL__SHIFT                                                       0x0
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__PERF_MODE__SHIFT                                                      0x18
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__ENABLE__SHIFT                                                         0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__CLEAR__SHIFT                                                          0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__ENABLE_MASK                                                           0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER10_CFG__CLEAR_MASK                                                            0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER11_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__PERF_SEL__SHIFT                                                       0x0
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__PERF_MODE__SHIFT                                                      0x18
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__ENABLE__SHIFT                                                         0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__CLEAR__SHIFT                                                          0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__ENABLE_MASK                                                           0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER11_CFG__CLEAR_MASK                                                            0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER12_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__PERF_SEL__SHIFT                                                       0x0
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__PERF_MODE__SHIFT                                                      0x18
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__ENABLE__SHIFT                                                         0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__CLEAR__SHIFT                                                          0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__ENABLE_MASK                                                           0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER12_CFG__CLEAR_MASK                                                            0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER13_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__PERF_SEL__SHIFT                                                       0x0
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__PERF_MODE__SHIFT                                                      0x18
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__ENABLE__SHIFT                                                         0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__CLEAR__SHIFT                                                          0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__ENABLE_MASK                                                           0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER13_CFG__CLEAR_MASK                                                            0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER14_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__PERF_SEL__SHIFT                                                       0x0
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__PERF_MODE__SHIFT                                                      0x18
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__ENABLE__SHIFT                                                         0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__CLEAR__SHIFT                                                          0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__ENABLE_MASK                                                           0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER14_CFG__CLEAR_MASK                                                            0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER15_CFG
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__PERF_SEL__SHIFT                                                       0x0
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__PERF_SEL_END__SHIFT                                                   0x8
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__PERF_MODE__SHIFT                                                      0x18
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__ENABLE__SHIFT                                                         0x1c
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__CLEAR__SHIFT                                                          0x1d
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__PERF_SEL_MASK                                                         0x000000FFL
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__PERF_SEL_END_MASK                                                     0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__PERF_MODE_MASK                                                        0x0F000000L
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__ENABLE_MASK                                                           0x10000000L
+#define GCUTCL2_FFBM_PERFCOUNTER15_CFG__CLEAR_MASK                                                            0x20000000L
+//GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                        0x0
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                              0x8
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                               0x10
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                                 0x18
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                  0x19
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                       0x1a
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                          0x0000000FL
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                                0x0000FF00L
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                                 0x00FF0000L
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                   0x01000000L
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                    0x02000000L
+#define GCUTCL2_FFBM_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                         0x04000000L
+
+
+// addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcutcl2nhttlbpldec
+//GCUTCL2_NHTTLB_PERFCOUNTER0_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER0_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER1_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER1_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER2_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER2_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER3_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER3_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER4_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER4_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER5_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER5_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER6_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER6_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER7_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER7_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER8_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER8_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER9_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__PERF_SEL__SHIFT                                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__PERF_SEL_END__SHIFT                                                  0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__PERF_MODE__SHIFT                                                     0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__ENABLE__SHIFT                                                        0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__CLEAR__SHIFT                                                         0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__PERF_SEL_MASK                                                        0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__PERF_SEL_END_MASK                                                    0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__PERF_MODE_MASK                                                       0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__ENABLE_MASK                                                          0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER9_CFG__CLEAR_MASK                                                           0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER10_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__PERF_SEL__SHIFT                                                     0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__PERF_SEL_END__SHIFT                                                 0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__PERF_MODE__SHIFT                                                    0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__ENABLE__SHIFT                                                       0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__CLEAR__SHIFT                                                        0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__PERF_SEL_MASK                                                       0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__PERF_SEL_END_MASK                                                   0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__PERF_MODE_MASK                                                      0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__ENABLE_MASK                                                         0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER10_CFG__CLEAR_MASK                                                          0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER11_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__PERF_SEL__SHIFT                                                     0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__PERF_SEL_END__SHIFT                                                 0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__PERF_MODE__SHIFT                                                    0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__ENABLE__SHIFT                                                       0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__CLEAR__SHIFT                                                        0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__PERF_SEL_MASK                                                       0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__PERF_SEL_END_MASK                                                   0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__PERF_MODE_MASK                                                      0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__ENABLE_MASK                                                         0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER11_CFG__CLEAR_MASK                                                          0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER12_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__PERF_SEL__SHIFT                                                     0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__PERF_SEL_END__SHIFT                                                 0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__PERF_MODE__SHIFT                                                    0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__ENABLE__SHIFT                                                       0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__CLEAR__SHIFT                                                        0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__PERF_SEL_MASK                                                       0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__PERF_SEL_END_MASK                                                   0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__PERF_MODE_MASK                                                      0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__ENABLE_MASK                                                         0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER12_CFG__CLEAR_MASK                                                          0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER13_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__PERF_SEL__SHIFT                                                     0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__PERF_SEL_END__SHIFT                                                 0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__PERF_MODE__SHIFT                                                    0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__ENABLE__SHIFT                                                       0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__CLEAR__SHIFT                                                        0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__PERF_SEL_MASK                                                       0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__PERF_SEL_END_MASK                                                   0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__PERF_MODE_MASK                                                      0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__ENABLE_MASK                                                         0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER13_CFG__CLEAR_MASK                                                          0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER14_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__PERF_SEL__SHIFT                                                     0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__PERF_SEL_END__SHIFT                                                 0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__PERF_MODE__SHIFT                                                    0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__ENABLE__SHIFT                                                       0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__CLEAR__SHIFT                                                        0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__PERF_SEL_MASK                                                       0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__PERF_SEL_END_MASK                                                   0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__PERF_MODE_MASK                                                      0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__ENABLE_MASK                                                         0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER14_CFG__CLEAR_MASK                                                          0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER15_CFG
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__PERF_SEL__SHIFT                                                     0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__PERF_SEL_END__SHIFT                                                 0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__PERF_MODE__SHIFT                                                    0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__ENABLE__SHIFT                                                       0x1c
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__CLEAR__SHIFT                                                        0x1d
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__PERF_SEL_MASK                                                       0x000000FFL
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__PERF_SEL_END_MASK                                                   0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__PERF_MODE_MASK                                                      0x0F000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__ENABLE_MASK                                                         0x10000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER15_CFG__CLEAR_MASK                                                          0x20000000L
+//GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT__SHIFT                                      0x0
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER__SHIFT                                            0x8
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER__SHIFT                                             0x10
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY__SHIFT                                               0x18
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL__SHIFT                                                0x19
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE__SHIFT                                     0x1a
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__PERF_COUNTER_SELECT_MASK                                        0x0000000FL
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__START_TRIGGER_MASK                                              0x0000FF00L
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__STOP_TRIGGER_MASK                                               0x00FF0000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__ENABLE_ANY_MASK                                                 0x01000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__CLEAR_ALL_MASK                                                  0x02000000L
+#define GCUTCL2_NHTTLB_PERFCOUNTER_RSLT_CNTL__STOP_ALL_ON_SATURATE_MASK                                       0x04000000L
+
+
 // addressBlock: CHIP_XCD_gfxip_aigc_gcutcl2_inst0_gcl2tlbpldec
 //GC_L2TLB_PERFCOUNTER0_CFG
 #define GC_L2TLB_PERFCOUNTER0_CFG__PERF_SEL__SHIFT                                                            0x0
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_1_offset.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_1_offset.h
new file mode 100755
index 0000000000000..f10e6168ad54a
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_1_offset.h
@@ -0,0 +1,149 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _gc_12_1_1_OFFSET_HEADER
+#define _gc_12_1_1_OFFSET_HEADER
+
+
+
+// addressBlock: aigc_grbma_grbma_grbmadec
+// base address: 0x18000
+#define regGRBMA_GFX_INDEX                                                                              0x0011
+#define regGRBMA_GFX_INDEX_BASE_IDX                                                                     1
+
+
+// addressBlock: aigc_grbma_grbma_perfddec
+// base address: 0x19200
+#define regGRBMA_PERFCOUNTER0_LO                                                                        0x0480
+#define regGRBMA_PERFCOUNTER0_LO_BASE_IDX                                                               1
+#define regGRBMA_PERFCOUNTER0_HI                                                                        0x0481
+#define regGRBMA_PERFCOUNTER0_HI_BASE_IDX                                                               1
+#define regGRBMA_PERFCOUNTER1_LO                                                                        0x0482
+#define regGRBMA_PERFCOUNTER1_LO_BASE_IDX                                                               1
+#define regGRBMA_PERFCOUNTER1_HI                                                                        0x0483
+#define regGRBMA_PERFCOUNTER1_HI_BASE_IDX                                                               1
+
+
+// addressBlock: aigc_grbma_grbma_perfsdec
+// base address: 0x19300
+#define regGRBMA_PERFCOUNTER0_SELECT                                                                    0x04c0
+#define regGRBMA_PERFCOUNTER0_SELECT_BASE_IDX                                                           1
+#define regGRBMA_PERFCOUNTER1_SELECT                                                                    0x04c1
+#define regGRBMA_PERFCOUNTER1_SELECT_BASE_IDX                                                           1
+#define regAID_PERFMON_CNTL                                                                             0x04c2
+#define regAID_PERFMON_CNTL_BASE_IDX                                                                    1
+
+
+// addressBlock: aigc_gl2x_gfx_se_perfsdec
+// base address: 0x19300
+#define regGL2C_PERFCOUNTER0_SELECT                                                                     0x04e8
+#define regGL2C_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGL2C_PERFCOUNTER0_SELECT1                                                                    0x04e9
+#define regGL2C_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regGL2C_PERFCOUNTER1_SELECT                                                                     0x04ea
+#define regGL2C_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGL2C_PERFCOUNTER1_SELECT1                                                                    0x04eb
+#define regGL2C_PERFCOUNTER1_SELECT1_BASE_IDX                                                           1
+#define regGL2C_PERFCOUNTER2_SELECT                                                                     0x04ec
+#define regGL2C_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGL2C_PERFCOUNTER2_SELECT1                                                                    0x04ed
+#define regGL2C_PERFCOUNTER2_SELECT1_BASE_IDX                                                           1
+#define regGL2C_PERFCOUNTER3_SELECT                                                                     0x04ee
+#define regGL2C_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regGL2C_PERFCOUNTER3_SELECT1                                                                    0x04ef
+#define regGL2C_PERFCOUNTER3_SELECT1_BASE_IDX                                                           1
+#define regGL2A_PERFCOUNTER0_SELECT                                                                     0x04f0
+#define regGL2A_PERFCOUNTER0_SELECT_BASE_IDX                                                            1
+#define regGL2A_PERFCOUNTER0_SELECT1                                                                    0x04f1
+#define regGL2A_PERFCOUNTER0_SELECT1_BASE_IDX                                                           1
+#define regGL2A_PERFCOUNTER1_SELECT                                                                     0x04f2
+#define regGL2A_PERFCOUNTER1_SELECT_BASE_IDX                                                            1
+#define regGL2A_PERFCOUNTER1_SELECT1                                                                    0x04f3
+#define regGL2A_PERFCOUNTER1_SELECT1_BASE_IDX                                                           1
+#define regGL2A_PERFCOUNTER2_SELECT                                                                     0x04f4
+#define regGL2A_PERFCOUNTER2_SELECT_BASE_IDX                                                            1
+#define regGL2A_PERFCOUNTER2_SELECT1                                                                    0x04f5
+#define regGL2A_PERFCOUNTER2_SELECT1_BASE_IDX                                                           1
+#define regGL2A_PERFCOUNTER3_SELECT                                                                     0x04f6
+#define regGL2A_PERFCOUNTER3_SELECT_BASE_IDX                                                            1
+#define regGL2A_PERFCOUNTER3_SELECT1                                                                    0x04f7
+#define regGL2A_PERFCOUNTER3_SELECT1_BASE_IDX                                                           1
+
+
+// addressBlock: aigc_gl2x_gfx_se_perfddec
+// base address: 0x19200
+#define regGL2C_PERFCOUNTER0_LO                                                                         0x04a0
+#define regGL2C_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER0_HI                                                                         0x04a1
+#define regGL2C_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER1_LO                                                                         0x04a2
+#define regGL2C_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER1_HI                                                                         0x04a3
+#define regGL2C_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER2_LO                                                                         0x04a4
+#define regGL2C_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER2_HI                                                                         0x04a5
+#define regGL2C_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER3_LO                                                                         0x04a6
+#define regGL2C_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regGL2C_PERFCOUNTER3_HI                                                                         0x04a7
+#define regGL2C_PERFCOUNTER3_HI_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER0_LO                                                                         0x04a8
+#define regGL2A_PERFCOUNTER0_LO_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER0_HI                                                                         0x04a9
+#define regGL2A_PERFCOUNTER0_HI_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER1_LO                                                                         0x04aa
+#define regGL2A_PERFCOUNTER1_LO_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER1_HI                                                                         0x04ab
+#define regGL2A_PERFCOUNTER1_HI_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER2_LO                                                                         0x04ac
+#define regGL2A_PERFCOUNTER2_LO_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER2_HI                                                                         0x04ad
+#define regGL2A_PERFCOUNTER2_HI_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER3_LO                                                                         0x04ae
+#define regGL2A_PERFCOUNTER3_LO_BASE_IDX                                                                1
+#define regGL2A_PERFCOUNTER3_HI                                                                         0x04af
+#define regGL2A_PERFCOUNTER3_HI_BASE_IDX                                                                1
+
+
+// addressBlock: aigc_gfx_gcea_se_gfx_se_perfsdec
+// base address: 0x19320
+#define regGC_EA_SE_PERFCOUNTER0_SELECT                                                                 0x04c8
+#define regGC_EA_SE_PERFCOUNTER0_SELECT_BASE_IDX                                                        1
+#define regGC_EA_SE_PERFCOUNTER0_SELECT1                                                                0x04c9
+#define regGC_EA_SE_PERFCOUNTER0_SELECT1_BASE_IDX                                                       1
+#define regGC_EA_SE_PERFCOUNTER1_SELECT                                                                 0x04ca
+#define regGC_EA_SE_PERFCOUNTER1_SELECT_BASE_IDX                                                        1
+
+
+// addressBlock: aigc_gfx_gcea_se_gfx_se_perfddec
+// base address: 0x19240
+#define regGC_EA_SE_PERFCOUNTER0_LO                                                                     0x0490
+#define regGC_EA_SE_PERFCOUNTER0_LO_BASE_IDX                                                            1
+#define regGC_EA_SE_PERFCOUNTER0_HI                                                                     0x0491
+#define regGC_EA_SE_PERFCOUNTER0_HI_BASE_IDX                                                            1
+#define regGC_EA_SE_PERFCOUNTER1_LO                                                                     0x0492
+#define regGC_EA_SE_PERFCOUNTER1_LO_BASE_IDX                                                            1
+#define regGC_EA_SE_PERFCOUNTER1_HI                                                                     0x0493
+#define regGC_EA_SE_PERFCOUNTER1_HI_BASE_IDX                                                            1
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_1_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_1_sh_mask.h
new file mode 100755
index 0000000000000..8d09c8150a534
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/gc/gc_12_1_1_sh_mask.h
@@ -0,0 +1,377 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _gc_12_1_1_SH_MASK_HEADER
+#define _gc_12_1_1_SH_MASK_HEADER
+
+
+// addressBlock: aigc_grbma_grbma_grbmadec
+//GRBMA_GFX_INDEX
+#define GRBMA_GFX_INDEX__INSTANCE_INDEX__SHIFT                                                                0x0
+#define GRBMA_GFX_INDEX__SA_INDEX__SHIFT                                                                      0x8
+#define GRBMA_GFX_INDEX__SE_INDEX__SHIFT                                                                      0x10
+#define GRBMA_GFX_INDEX__SA_BROADCAST_WRITES__SHIFT                                                           0x1d
+#define GRBMA_GFX_INDEX__INSTANCE_BROADCAST_WRITES__SHIFT                                                     0x1e
+#define GRBMA_GFX_INDEX__SE_BROADCAST_WRITES__SHIFT                                                           0x1f
+#define GRBMA_GFX_INDEX__INSTANCE_INDEX_MASK                                                                  0x0000007FL
+#define GRBMA_GFX_INDEX__SA_INDEX_MASK                                                                        0x00000300L
+#define GRBMA_GFX_INDEX__SE_INDEX_MASK                                                                        0x000F0000L
+#define GRBMA_GFX_INDEX__SA_BROADCAST_WRITES_MASK                                                             0x20000000L
+#define GRBMA_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK                                                       0x40000000L
+#define GRBMA_GFX_INDEX__SE_BROADCAST_WRITES_MASK                                                             0x80000000L
+
+
+// addressBlock: aigc_grbma_grbma_perfddec
+//GRBMA_PERFCOUNTER0_LO
+#define GRBMA_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GRBMA_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GRBMA_PERFCOUNTER0_HI
+#define GRBMA_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GRBMA_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+//GRBMA_PERFCOUNTER1_LO
+#define GRBMA_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                          0x0
+#define GRBMA_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                            0xFFFFFFFFL
+//GRBMA_PERFCOUNTER1_HI
+#define GRBMA_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                          0x0
+#define GRBMA_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                            0xFFFFFFFFL
+
+
+// addressBlock: aigc_grbma_grbma_perfsdec
+//GRBMA_PERFCOUNTER0_SELECT
+#define GRBMA_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GRBMA_PERFCOUNTER0_SELECT__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                        0x6
+#define GRBMA_PERFCOUNTER0_SELECT__GL1XCC_BUSY_USER_DEFINED_MASK__SHIFT                                       0x7
+#define GRBMA_PERFCOUNTER0_SELECT__PMR_BUSY_USER_DEFINED_MASK__SHIFT                                          0x8
+#define GRBMA_PERFCOUNTER0_SELECT__SC_CLEAN_USER_DEFINED_MASK__SHIFT                                          0x9
+#define GRBMA_PERFCOUNTER0_SELECT__WGS_BUSY_USER_DEFINED_MASK__SHIFT                                          0xa
+#define GRBMA_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                          0xb
+#define GRBMA_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                          0xc
+#define GRBMA_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                           0xd
+#define GRBMA_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                           0xe
+#define GRBMA_PERFCOUNTER0_SELECT__GL2C_BUSY_USER_DEFINED_MASK__SHIFT                                         0xf
+#define GRBMA_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                          0x10
+#define GRBMA_PERFCOUNTER0_SELECT__XCAC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBMA_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                           0x12
+#define GRBMA_PERFCOUNTER0_SELECT__GL2A_BUSY_USER_DEFINED_MASK__SHIFT                                         0x13
+#define GRBMA_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                           0x14
+#define GRBMA_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                           0x15
+#define GRBMA_PERFCOUNTER0_SELECT__EA_LINK_BUSY_USER_DEFINED_MASK__SHIFT                                      0x17
+#define GRBMA_PERFCOUNTER0_SELECT__AIGC_CAC_BUSY_USER_DEFINED_MASK__SHIFT                                     0x18
+#define GRBMA_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                          0x19
+#define GRBMA_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1a
+#define GRBMA_PERFCOUNTER0_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1b
+#define GRBMA_PERFCOUNTER0_SELECT__GE_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1c
+#define GRBMA_PERFCOUNTER0_SELECT__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                        0x1d
+#define GRBMA_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1e
+#define GRBMA_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                              0x0000003FL
+#define GRBMA_PERFCOUNTER0_SELECT__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                          0x00000040L
+#define GRBMA_PERFCOUNTER0_SELECT__GL1XCC_BUSY_USER_DEFINED_MASK_MASK                                         0x00000080L
+#define GRBMA_PERFCOUNTER0_SELECT__PMR_BUSY_USER_DEFINED_MASK_MASK                                            0x00000100L
+#define GRBMA_PERFCOUNTER0_SELECT__SC_CLEAN_USER_DEFINED_MASK_MASK                                            0x00000200L
+#define GRBMA_PERFCOUNTER0_SELECT__WGS_BUSY_USER_DEFINED_MASK_MASK                                            0x00000400L
+#define GRBMA_PERFCOUNTER0_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                            0x00000800L
+#define GRBMA_PERFCOUNTER0_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                            0x00001000L
+#define GRBMA_PERFCOUNTER0_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                             0x00002000L
+#define GRBMA_PERFCOUNTER0_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                             0x00004000L
+#define GRBMA_PERFCOUNTER0_SELECT__GL2C_BUSY_USER_DEFINED_MASK_MASK                                           0x00008000L
+#define GRBMA_PERFCOUNTER0_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                            0x00010000L
+#define GRBMA_PERFCOUNTER0_SELECT__XCAC_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBMA_PERFCOUNTER0_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                             0x00040000L
+#define GRBMA_PERFCOUNTER0_SELECT__GL2A_BUSY_USER_DEFINED_MASK_MASK                                           0x00080000L
+#define GRBMA_PERFCOUNTER0_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                             0x00100000L
+#define GRBMA_PERFCOUNTER0_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                             0x00200000L
+#define GRBMA_PERFCOUNTER0_SELECT__EA_LINK_BUSY_USER_DEFINED_MASK_MASK                                        0x00800000L
+#define GRBMA_PERFCOUNTER0_SELECT__AIGC_CAC_BUSY_USER_DEFINED_MASK_MASK                                       0x01000000L
+#define GRBMA_PERFCOUNTER0_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                            0x02000000L
+#define GRBMA_PERFCOUNTER0_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                            0x04000000L
+#define GRBMA_PERFCOUNTER0_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                            0x08000000L
+#define GRBMA_PERFCOUNTER0_SELECT__GE_BUSY_USER_DEFINED_MASK_MASK                                             0x10000000L
+#define GRBMA_PERFCOUNTER0_SELECT__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                          0x20000000L
+#define GRBMA_PERFCOUNTER0_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                             0x40000000L
+//GRBMA_PERFCOUNTER1_SELECT
+#define GRBMA_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                            0x0
+#define GRBMA_PERFCOUNTER1_SELECT__GL1CC_BUSY_USER_DEFINED_MASK__SHIFT                                        0x6
+#define GRBMA_PERFCOUNTER1_SELECT__GL1XCC_BUSY_USER_DEFINED_MASK__SHIFT                                       0x7
+#define GRBMA_PERFCOUNTER1_SELECT__PMR_BUSY_USER_DEFINED_MASK__SHIFT                                          0x8
+#define GRBMA_PERFCOUNTER1_SELECT__SC_CLEAN_USER_DEFINED_MASK__SHIFT                                          0x9
+#define GRBMA_PERFCOUNTER1_SELECT__WGS_BUSY_USER_DEFINED_MASK__SHIFT                                          0xa
+#define GRBMA_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK__SHIFT                                          0xb
+#define GRBMA_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK__SHIFT                                          0xc
+#define GRBMA_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK__SHIFT                                           0xd
+#define GRBMA_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK__SHIFT                                           0xe
+#define GRBMA_PERFCOUNTER1_SELECT__GL2C_BUSY_USER_DEFINED_MASK__SHIFT                                         0xf
+#define GRBMA_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK__SHIFT                                          0x10
+#define GRBMA_PERFCOUNTER1_SELECT__XCAC_BUSY_USER_DEFINED_MASK__SHIFT                                         0x11
+#define GRBMA_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK__SHIFT                                           0x12
+#define GRBMA_PERFCOUNTER1_SELECT__GL2A_BUSY_USER_DEFINED_MASK__SHIFT                                         0x13
+#define GRBMA_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK__SHIFT                                           0x14
+#define GRBMA_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK__SHIFT                                           0x15
+#define GRBMA_PERFCOUNTER1_SELECT__EA_LINK_BUSY_USER_DEFINED_MASK__SHIFT                                      0x17
+#define GRBMA_PERFCOUNTER1_SELECT__AIGC_CAC_BUSY_USER_DEFINED_MASK__SHIFT                                     0x18
+#define GRBMA_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK__SHIFT                                          0x19
+#define GRBMA_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1a
+#define GRBMA_PERFCOUNTER1_SELECT__TCP_BUSY_USER_DEFINED_MASK__SHIFT                                          0x1b
+#define GRBMA_PERFCOUNTER1_SELECT__GE_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1c
+#define GRBMA_PERFCOUNTER1_SELECT__UTCL1_BUSY_USER_DEFINED_MASK__SHIFT                                        0x1d
+#define GRBMA_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK__SHIFT                                           0x1e
+#define GRBMA_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                              0x0000003FL
+#define GRBMA_PERFCOUNTER1_SELECT__GL1CC_BUSY_USER_DEFINED_MASK_MASK                                          0x00000040L
+#define GRBMA_PERFCOUNTER1_SELECT__GL1XCC_BUSY_USER_DEFINED_MASK_MASK                                         0x00000080L
+#define GRBMA_PERFCOUNTER1_SELECT__PMR_BUSY_USER_DEFINED_MASK_MASK                                            0x00000100L
+#define GRBMA_PERFCOUNTER1_SELECT__SC_CLEAN_USER_DEFINED_MASK_MASK                                            0x00000200L
+#define GRBMA_PERFCOUNTER1_SELECT__WGS_BUSY_USER_DEFINED_MASK_MASK                                            0x00000400L
+#define GRBMA_PERFCOUNTER1_SELECT__DB_CLEAN_USER_DEFINED_MASK_MASK                                            0x00000800L
+#define GRBMA_PERFCOUNTER1_SELECT__CB_CLEAN_USER_DEFINED_MASK_MASK                                            0x00001000L
+#define GRBMA_PERFCOUNTER1_SELECT__TA_BUSY_USER_DEFINED_MASK_MASK                                             0x00002000L
+#define GRBMA_PERFCOUNTER1_SELECT__SX_BUSY_USER_DEFINED_MASK_MASK                                             0x00004000L
+#define GRBMA_PERFCOUNTER1_SELECT__GL2C_BUSY_USER_DEFINED_MASK_MASK                                           0x00008000L
+#define GRBMA_PERFCOUNTER1_SELECT__SPI_BUSY_USER_DEFINED_MASK_MASK                                            0x00010000L
+#define GRBMA_PERFCOUNTER1_SELECT__XCAC_BUSY_USER_DEFINED_MASK_MASK                                           0x00020000L
+#define GRBMA_PERFCOUNTER1_SELECT__PA_BUSY_USER_DEFINED_MASK_MASK                                             0x00040000L
+#define GRBMA_PERFCOUNTER1_SELECT__GL2A_BUSY_USER_DEFINED_MASK_MASK                                           0x00080000L
+#define GRBMA_PERFCOUNTER1_SELECT__DB_BUSY_USER_DEFINED_MASK_MASK                                             0x00100000L
+#define GRBMA_PERFCOUNTER1_SELECT__CB_BUSY_USER_DEFINED_MASK_MASK                                             0x00200000L
+#define GRBMA_PERFCOUNTER1_SELECT__EA_LINK_BUSY_USER_DEFINED_MASK_MASK                                        0x00800000L
+#define GRBMA_PERFCOUNTER1_SELECT__AIGC_CAC_BUSY_USER_DEFINED_MASK_MASK                                       0x01000000L
+#define GRBMA_PERFCOUNTER1_SELECT__BCI_BUSY_USER_DEFINED_MASK_MASK                                            0x02000000L
+#define GRBMA_PERFCOUNTER1_SELECT__RLC_BUSY_USER_DEFINED_MASK_MASK                                            0x04000000L
+#define GRBMA_PERFCOUNTER1_SELECT__TCP_BUSY_USER_DEFINED_MASK_MASK                                            0x08000000L
+#define GRBMA_PERFCOUNTER1_SELECT__GE_BUSY_USER_DEFINED_MASK_MASK                                             0x10000000L
+#define GRBMA_PERFCOUNTER1_SELECT__UTCL1_BUSY_USER_DEFINED_MASK_MASK                                          0x20000000L
+#define GRBMA_PERFCOUNTER1_SELECT__EA_BUSY_USER_DEFINED_MASK_MASK                                             0x40000000L
+//AID_PERFMON_CNTL
+#define AID_PERFMON_CNTL__PERFMON_STATE__SHIFT                                                                0x0
+#define AID_PERFMON_CNTL__SPM_PERFMON_STATE__SHIFT                                                            0x4
+#define AID_PERFMON_CNTL__PERFMON_ENABLE_MODE__SHIFT                                                          0x8
+#define AID_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE__SHIFT                                                        0xa
+#define AID_PERFMON_CNTL__PERFMON_STATE_MASK                                                                  0x0000000FL
+#define AID_PERFMON_CNTL__SPM_PERFMON_STATE_MASK                                                              0x000000F0L
+#define AID_PERFMON_CNTL__PERFMON_ENABLE_MODE_MASK                                                            0x00000300L
+#define AID_PERFMON_CNTL__PERFMON_SAMPLE_ENABLE_MASK                                                          0x00000400L
+
+
+// addressBlock: aigc_gl2x_gfx_se_perfsdec
+//GL2C_PERFCOUNTER0_SELECT
+#define GL2C_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2C_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2C_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2C_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2C_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2C_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2C_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2C_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2C_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2C_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2C_PERFCOUNTER0_SELECT1
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL2C_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL2C_PERFCOUNTER1_SELECT
+#define GL2C_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2C_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2C_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2C_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2C_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2C_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2C_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2C_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2C_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2C_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2C_PERFCOUNTER1_SELECT1
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL2C_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL2C_PERFCOUNTER2_SELECT
+#define GL2C_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2C_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2C_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2C_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2C_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2C_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2C_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2C_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2C_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2C_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2C_PERFCOUNTER2_SELECT1
+#define GL2C_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2C_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL2C_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL2C_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2C_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2C_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL2C_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL2C_PERFCOUNTER3_SELECT
+#define GL2C_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2C_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2C_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2C_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2C_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2C_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2C_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2C_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2C_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2C_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2C_PERFCOUNTER3_SELECT1
+#define GL2C_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2C_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2C_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL2C_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL2C_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2C_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2C_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL2C_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL2A_PERFCOUNTER0_SELECT
+#define GL2A_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2A_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2A_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2A_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2A_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2A_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2A_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2A_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2A_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2A_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2A_PERFCOUNTER0_SELECT1
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL2A_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL2A_PERFCOUNTER1_SELECT
+#define GL2A_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2A_PERFCOUNTER1_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2A_PERFCOUNTER1_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2A_PERFCOUNTER1_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2A_PERFCOUNTER1_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2A_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2A_PERFCOUNTER1_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2A_PERFCOUNTER1_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2A_PERFCOUNTER1_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2A_PERFCOUNTER1_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2A_PERFCOUNTER1_SELECT1
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL2A_PERFCOUNTER1_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL2A_PERFCOUNTER2_SELECT
+#define GL2A_PERFCOUNTER2_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2A_PERFCOUNTER2_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2A_PERFCOUNTER2_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2A_PERFCOUNTER2_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2A_PERFCOUNTER2_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2A_PERFCOUNTER2_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2A_PERFCOUNTER2_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2A_PERFCOUNTER2_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2A_PERFCOUNTER2_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2A_PERFCOUNTER2_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2A_PERFCOUNTER2_SELECT1
+#define GL2A_PERFCOUNTER2_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER2_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2A_PERFCOUNTER2_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL2A_PERFCOUNTER2_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL2A_PERFCOUNTER2_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2A_PERFCOUNTER2_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2A_PERFCOUNTER2_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL2A_PERFCOUNTER2_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+//GL2A_PERFCOUNTER3_SELECT
+#define GL2A_PERFCOUNTER3_SELECT__PERF_SEL__SHIFT                                                             0x0
+#define GL2A_PERFCOUNTER3_SELECT__PERF_SEL1__SHIFT                                                            0xa
+#define GL2A_PERFCOUNTER3_SELECT__CNTR_MODE__SHIFT                                                            0x14
+#define GL2A_PERFCOUNTER3_SELECT__PERF_MODE1__SHIFT                                                           0x18
+#define GL2A_PERFCOUNTER3_SELECT__PERF_MODE__SHIFT                                                            0x1c
+#define GL2A_PERFCOUNTER3_SELECT__PERF_SEL_MASK                                                               0x000003FFL
+#define GL2A_PERFCOUNTER3_SELECT__PERF_SEL1_MASK                                                              0x000FFC00L
+#define GL2A_PERFCOUNTER3_SELECT__CNTR_MODE_MASK                                                              0x00F00000L
+#define GL2A_PERFCOUNTER3_SELECT__PERF_MODE1_MASK                                                             0x0F000000L
+#define GL2A_PERFCOUNTER3_SELECT__PERF_MODE_MASK                                                              0xF0000000L
+//GL2A_PERFCOUNTER3_SELECT1
+#define GL2A_PERFCOUNTER3_SELECT1__PERF_SEL2__SHIFT                                                           0x0
+#define GL2A_PERFCOUNTER3_SELECT1__PERF_SEL3__SHIFT                                                           0xa
+#define GL2A_PERFCOUNTER3_SELECT1__PERF_MODE3__SHIFT                                                          0x18
+#define GL2A_PERFCOUNTER3_SELECT1__PERF_MODE2__SHIFT                                                          0x1c
+#define GL2A_PERFCOUNTER3_SELECT1__PERF_SEL2_MASK                                                             0x000003FFL
+#define GL2A_PERFCOUNTER3_SELECT1__PERF_SEL3_MASK                                                             0x000FFC00L
+#define GL2A_PERFCOUNTER3_SELECT1__PERF_MODE3_MASK                                                            0x0F000000L
+#define GL2A_PERFCOUNTER3_SELECT1__PERF_MODE2_MASK                                                            0xF0000000L
+
+
+// addressBlock: aigc_gfx_gcea_se_gfx_se_perfsdec
+//GC_EA_SE_PERFCOUNTER0_SELECT
+#define GC_EA_SE_PERFCOUNTER0_SELECT__PERF_SEL__SHIFT                                                         0x0
+#define GC_EA_SE_PERFCOUNTER0_SELECT__PERF_SEL1__SHIFT                                                        0xa
+#define GC_EA_SE_PERFCOUNTER0_SELECT__CNTR_MODE__SHIFT                                                        0x14
+#define GC_EA_SE_PERFCOUNTER0_SELECT__PERF_MODE1__SHIFT                                                       0x18
+#define GC_EA_SE_PERFCOUNTER0_SELECT__PERF_MODE__SHIFT                                                        0x1c
+#define GC_EA_SE_PERFCOUNTER0_SELECT__PERF_SEL_MASK                                                           0x000003FFL
+#define GC_EA_SE_PERFCOUNTER0_SELECT__PERF_SEL1_MASK                                                          0x000FFC00L
+#define GC_EA_SE_PERFCOUNTER0_SELECT__CNTR_MODE_MASK                                                          0x00F00000L
+#define GC_EA_SE_PERFCOUNTER0_SELECT__PERF_MODE1_MASK                                                         0x0F000000L
+#define GC_EA_SE_PERFCOUNTER0_SELECT__PERF_MODE_MASK                                                          0xF0000000L
+//GC_EA_SE_PERFCOUNTER0_SELECT1
+#define GC_EA_SE_PERFCOUNTER0_SELECT1__PERF_SEL2__SHIFT                                                       0x0
+#define GC_EA_SE_PERFCOUNTER0_SELECT1__PERF_SEL3__SHIFT                                                       0xa
+#define GC_EA_SE_PERFCOUNTER0_SELECT1__PERF_MODE3__SHIFT                                                      0x18
+#define GC_EA_SE_PERFCOUNTER0_SELECT1__PERF_MODE2__SHIFT                                                      0x1c
+#define GC_EA_SE_PERFCOUNTER0_SELECT1__PERF_SEL2_MASK                                                         0x000003FFL
+#define GC_EA_SE_PERFCOUNTER0_SELECT1__PERF_SEL3_MASK                                                         0x000FFC00L
+#define GC_EA_SE_PERFCOUNTER0_SELECT1__PERF_MODE3_MASK                                                        0x0F000000L
+#define GC_EA_SE_PERFCOUNTER0_SELECT1__PERF_MODE2_MASK                                                        0xF0000000L
+//GC_EA_SE_PERFCOUNTER1_SELECT
+#define GC_EA_SE_PERFCOUNTER1_SELECT__PERF_SEL__SHIFT                                                         0x0
+#define GC_EA_SE_PERFCOUNTER1_SELECT__COUNTER_MODE__SHIFT                                                     0x1c
+#define GC_EA_SE_PERFCOUNTER1_SELECT__PERF_SEL_MASK                                                           0x000003FFL
+#define GC_EA_SE_PERFCOUNTER1_SELECT__COUNTER_MODE_MASK                                                       0xF0000000L
+
+
+// addressBlock: aigc_gfx_gcea_se_gfx_se_perfddec
+//GC_EA_SE_PERFCOUNTER0_LO
+#define GC_EA_SE_PERFCOUNTER0_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GC_EA_SE_PERFCOUNTER0_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GC_EA_SE_PERFCOUNTER0_HI
+#define GC_EA_SE_PERFCOUNTER0_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GC_EA_SE_PERFCOUNTER0_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+//GC_EA_SE_PERFCOUNTER1_LO
+#define GC_EA_SE_PERFCOUNTER1_LO__PERFCOUNTER_LO__SHIFT                                                       0x0
+#define GC_EA_SE_PERFCOUNTER1_LO__PERFCOUNTER_LO_MASK                                                         0xFFFFFFFFL
+//GC_EA_SE_PERFCOUNTER1_HI
+#define GC_EA_SE_PERFCOUNTER1_HI__PERFCOUNTER_HI__SHIFT                                                       0x0
+#define GC_EA_SE_PERFCOUNTER1_HI__PERFCOUNTER_HI_MASK                                                         0xFFFFFFFFL
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_3_2_offset.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_3_2_offset.h
new file mode 100644
index 0000000000000..0b42c1c9c848b
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_3_2_offset.h
@@ -0,0 +1,13149 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _nbio_6_3_2_OFFSET_HEADER
+#define _nbio_6_3_2_OFFSET_HEADER
+
+
+
+// addressBlock: nbif0_nbif0_bif_bx_SYSDEC
+// base address: 0x0
+#define regBIF_BX0_SYSHUB_INDEX_OVLP                                                                    0x0008
+#define regBIF_BX0_SYSHUB_INDEX_OVLP_BASE_IDX                                                           0
+#define regBIF_BX0_SYSHUB_DATA_OVLP                                                                     0x0009
+#define regBIF_BX0_SYSHUB_DATA_OVLP_BASE_IDX                                                            0
+#define regBIF_BX0_SYSHUB_INDEX_HI_OVLP                                                                 0x000a
+#define regBIF_BX0_SYSHUB_INDEX_HI_OVLP_BASE_IDX                                                        0
+#define regBIF_BX0_PCIE_INDEX                                                                           0x000c
+#define regBIF_BX0_PCIE_INDEX_BASE_IDX                                                                  0
+#define regBIF_BX0_PCIE_DATA                                                                            0x000d
+#define regBIF_BX0_PCIE_DATA_BASE_IDX                                                                   0
+#define regBIF_BX0_PCIE_INDEX2                                                                          0x000e
+#define regBIF_BX0_PCIE_INDEX2_BASE_IDX                                                                 0
+#define regBIF_BX0_PCIE_DATA2                                                                           0x000f
+#define regBIF_BX0_PCIE_DATA2_BASE_IDX                                                                  0
+#define regBIF_BX0_PCIE_INDEX_HI                                                                        0x0010
+#define regBIF_BX0_PCIE_INDEX_HI_BASE_IDX                                                               0
+#define regBIF_BX0_PCIE_INDEX2_HI                                                                       0x0011
+#define regBIF_BX0_PCIE_INDEX2_HI_BASE_IDX                                                              0
+#define regBIF_BX0_SBIOS_SCRATCH_0                                                                      0x0034
+#define regBIF_BX0_SBIOS_SCRATCH_0_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_1                                                                      0x0035
+#define regBIF_BX0_SBIOS_SCRATCH_1_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_2                                                                      0x0036
+#define regBIF_BX0_SBIOS_SCRATCH_2_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_3                                                                      0x0037
+#define regBIF_BX0_SBIOS_SCRATCH_3_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_0                                                                       0x0038
+#define regBIF_BX0_BIOS_SCRATCH_0_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_1                                                                       0x0039
+#define regBIF_BX0_BIOS_SCRATCH_1_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_2                                                                       0x003a
+#define regBIF_BX0_BIOS_SCRATCH_2_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_3                                                                       0x003b
+#define regBIF_BX0_BIOS_SCRATCH_3_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_4                                                                       0x003c
+#define regBIF_BX0_BIOS_SCRATCH_4_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_5                                                                       0x003d
+#define regBIF_BX0_BIOS_SCRATCH_5_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_6                                                                       0x003e
+#define regBIF_BX0_BIOS_SCRATCH_6_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_7                                                                       0x003f
+#define regBIF_BX0_BIOS_SCRATCH_7_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_8                                                                       0x0040
+#define regBIF_BX0_BIOS_SCRATCH_8_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_9                                                                       0x0041
+#define regBIF_BX0_BIOS_SCRATCH_9_BASE_IDX                                                              1
+#define regBIF_BX0_BIOS_SCRATCH_10                                                                      0x0042
+#define regBIF_BX0_BIOS_SCRATCH_10_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_11                                                                      0x0043
+#define regBIF_BX0_BIOS_SCRATCH_11_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_12                                                                      0x0044
+#define regBIF_BX0_BIOS_SCRATCH_12_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_13                                                                      0x0045
+#define regBIF_BX0_BIOS_SCRATCH_13_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_14                                                                      0x0046
+#define regBIF_BX0_BIOS_SCRATCH_14_BASE_IDX                                                             1
+#define regBIF_BX0_BIOS_SCRATCH_15                                                                      0x0047
+#define regBIF_BX0_BIOS_SCRATCH_15_BASE_IDX                                                             1
+#define regBIF_BX0_BIF_RLC_INTR_CNTL                                                                    0x004c
+#define regBIF_BX0_BIF_RLC_INTR_CNTL_BASE_IDX                                                           1
+#define regBIF_BX0_BIF_VCE_INTR_CNTL                                                                    0x004d
+#define regBIF_BX0_BIF_VCE_INTR_CNTL_BASE_IDX                                                           1
+#define regBIF_BX0_BIF_UVD_INTR_CNTL                                                                    0x004e
+#define regBIF_BX0_BIF_UVD_INTR_CNTL_BASE_IDX                                                           1
+#define regBIF_BX0_BIF_Engine_INTR_CNTL                                                                 0x004f
+#define regBIF_BX0_BIF_Engine_INTR_CNTL_BASE_IDX                                                        1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR0                                                                0x006c
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR0_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0                                                          0x006d
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR1                                                                0x006e
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR1_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1                                                          0x006f
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR2                                                                0x0070
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR2_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2                                                          0x0071
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR3                                                                0x0072
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR3_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3                                                          0x0073
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR4                                                                0x0074
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR4_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4                                                          0x0075
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR5                                                                0x0076
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR5_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5                                                          0x0077
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR6                                                                0x0078
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR6_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6                                                          0x0079
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR7                                                                0x007a
+#define regBIF_BX0_GFX_MMIOREG_CAM_ADDR7_BASE_IDX                                                       1
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7                                                          0x007b
+#define regBIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7_BASE_IDX                                                 1
+#define regBIF_BX0_GFX_MMIOREG_CAM_CNTL                                                                 0x007c
+#define regBIF_BX0_GFX_MMIOREG_CAM_CNTL_BASE_IDX                                                        1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL                                                             0x007d
+#define regBIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL_BASE_IDX                                                    1
+#define regBIF_BX0_GFX_MMIOREG_CAM_ONE_CPL                                                              0x007e
+#define regBIF_BX0_GFX_MMIOREG_CAM_ONE_CPL_BASE_IDX                                                     1
+#define regBIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL                                                     0x007f
+#define regBIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL_BASE_IDX                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_0                                                                     0x0080
+#define regBIF_BX0_DRIVER_SCRATCH_0_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_1                                                                     0x0081
+#define regBIF_BX0_DRIVER_SCRATCH_1_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_2                                                                     0x0082
+#define regBIF_BX0_DRIVER_SCRATCH_2_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_3                                                                     0x0083
+#define regBIF_BX0_DRIVER_SCRATCH_3_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_4                                                                     0x0084
+#define regBIF_BX0_DRIVER_SCRATCH_4_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_5                                                                     0x0085
+#define regBIF_BX0_DRIVER_SCRATCH_5_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_6                                                                     0x0086
+#define regBIF_BX0_DRIVER_SCRATCH_6_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_7                                                                     0x0087
+#define regBIF_BX0_DRIVER_SCRATCH_7_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_8                                                                     0x0088
+#define regBIF_BX0_DRIVER_SCRATCH_8_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_9                                                                     0x0089
+#define regBIF_BX0_DRIVER_SCRATCH_9_BASE_IDX                                                            1
+#define regBIF_BX0_DRIVER_SCRATCH_10                                                                    0x008a
+#define regBIF_BX0_DRIVER_SCRATCH_10_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_11                                                                    0x008b
+#define regBIF_BX0_DRIVER_SCRATCH_11_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_12                                                                    0x008c
+#define regBIF_BX0_DRIVER_SCRATCH_12_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_13                                                                    0x008d
+#define regBIF_BX0_DRIVER_SCRATCH_13_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_14                                                                    0x008e
+#define regBIF_BX0_DRIVER_SCRATCH_14_BASE_IDX                                                           1
+#define regBIF_BX0_DRIVER_SCRATCH_15                                                                    0x008f
+#define regBIF_BX0_DRIVER_SCRATCH_15_BASE_IDX                                                           1
+#define regBIF_BX0_FW_SCRATCH_0                                                                         0x0090
+#define regBIF_BX0_FW_SCRATCH_0_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_1                                                                         0x0091
+#define regBIF_BX0_FW_SCRATCH_1_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_2                                                                         0x0092
+#define regBIF_BX0_FW_SCRATCH_2_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_3                                                                         0x0093
+#define regBIF_BX0_FW_SCRATCH_3_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_4                                                                         0x0094
+#define regBIF_BX0_FW_SCRATCH_4_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_5                                                                         0x0095
+#define regBIF_BX0_FW_SCRATCH_5_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_6                                                                         0x0096
+#define regBIF_BX0_FW_SCRATCH_6_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_7                                                                         0x0097
+#define regBIF_BX0_FW_SCRATCH_7_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_8                                                                         0x0098
+#define regBIF_BX0_FW_SCRATCH_8_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_9                                                                         0x0099
+#define regBIF_BX0_FW_SCRATCH_9_BASE_IDX                                                                1
+#define regBIF_BX0_FW_SCRATCH_10                                                                        0x009a
+#define regBIF_BX0_FW_SCRATCH_10_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_11                                                                        0x009b
+#define regBIF_BX0_FW_SCRATCH_11_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_12                                                                        0x009c
+#define regBIF_BX0_FW_SCRATCH_12_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_13                                                                        0x009d
+#define regBIF_BX0_FW_SCRATCH_13_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_14                                                                        0x009e
+#define regBIF_BX0_FW_SCRATCH_14_BASE_IDX                                                               1
+#define regBIF_BX0_FW_SCRATCH_15                                                                        0x009f
+#define regBIF_BX0_FW_SCRATCH_15_BASE_IDX                                                               1
+#define regBIF_BX0_SBIOS_SCRATCH_4                                                                      0x00a0
+#define regBIF_BX0_SBIOS_SCRATCH_4_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_5                                                                      0x00a1
+#define regBIF_BX0_SBIOS_SCRATCH_5_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_6                                                                      0x00a2
+#define regBIF_BX0_SBIOS_SCRATCH_6_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_7                                                                      0x00a3
+#define regBIF_BX0_SBIOS_SCRATCH_7_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_8                                                                      0x00a4
+#define regBIF_BX0_SBIOS_SCRATCH_8_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_9                                                                      0x00a5
+#define regBIF_BX0_SBIOS_SCRATCH_9_BASE_IDX                                                             1
+#define regBIF_BX0_SBIOS_SCRATCH_10                                                                     0x00a6
+#define regBIF_BX0_SBIOS_SCRATCH_10_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_11                                                                     0x00a7
+#define regBIF_BX0_SBIOS_SCRATCH_11_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_12                                                                     0x00a8
+#define regBIF_BX0_SBIOS_SCRATCH_12_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_13                                                                     0x00a9
+#define regBIF_BX0_SBIOS_SCRATCH_13_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_14                                                                     0x00aa
+#define regBIF_BX0_SBIOS_SCRATCH_14_BASE_IDX                                                            1
+#define regBIF_BX0_SBIOS_SCRATCH_15                                                                     0x00ab
+#define regBIF_BX0_SBIOS_SCRATCH_15_BASE_IDX                                                            1
+
+
+// addressBlock: nbif0_nbif0_rcc_dwn_dev0_BIFDEC1
+// base address: 0x0
+#define regRCC_DWN_DEV0_0_DN_PCIE_RESERVED                                                              0x0068
+#define regRCC_DWN_DEV0_0_DN_PCIE_RESERVED_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_SCRATCH                                                               0x0069
+#define regRCC_DWN_DEV0_0_DN_PCIE_SCRATCH_BASE_IDX                                                      2
+#define regRCC_DWN_DEV0_0_DN_PCIE_CNTL                                                                  0x006b
+#define regRCC_DWN_DEV0_0_DN_PCIE_CNTL_BASE_IDX                                                         2
+#define regRCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL                                                           0x006c
+#define regRCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                  2
+#define regRCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2                                                              0x006d
+#define regRCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL                                                              0x006e
+#define regRCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL                                                              0x006f
+#define regRCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_F0                                                              0x0070
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_F0_BASE_IDX                                                     2
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC                                                            0x0071
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC_BASE_IDX                                                   2
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2                                                           0x0072
+#define regRCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2_BASE_IDX                                                  2
+
+
+// addressBlock: nbif0_nbif0_rcc_dwnp_dev0_BIFDEC1
+// base address: 0x0
+#define regRCC_DWNP_DEV0_0_PCIE_ERR_CNTL                                                                0x0074
+#define regRCC_DWNP_DEV0_0_PCIE_ERR_CNTL_BASE_IDX                                                       2
+#define regRCC_DWNP_DEV0_0_PCIE_RX_CNTL                                                                 0x0075
+#define regRCC_DWNP_DEV0_0_PCIE_RX_CNTL_BASE_IDX                                                        2
+#define regRCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL                                                           0x0076
+#define regRCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL_BASE_IDX                                                  2
+#define regRCC_DWNP_DEV0_0_PCIE_LC_CNTL2                                                                0x0077
+#define regRCC_DWNP_DEV0_0_PCIE_LC_CNTL2_BASE_IDX                                                       2
+#define regRCC_DWNP_DEV0_0_PCIEP_STRAP_MISC                                                             0x0078
+#define regRCC_DWNP_DEV0_0_PCIEP_STRAP_MISC_BASE_IDX                                                    2
+#define regRCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP                                                         0x0079
+#define regRCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                2
+
+
+// addressBlock: nbif0_nbif0_rcc_ep_dev0_BIFDEC1
+// base address: 0x0
+#define regRCC_EP_DEV0_0_EP_PCIE_SCRATCH                                                                0x004b
+#define regRCC_EP_DEV0_0_EP_PCIE_SCRATCH_BASE_IDX                                                       2
+#define regRCC_EP_DEV0_0_EP_PCIE_CNTL                                                                   0x004d
+#define regRCC_EP_DEV0_0_EP_PCIE_CNTL_BASE_IDX                                                          2
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_CNTL                                                               0x004e
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_STATUS                                                             0x004f
+#define regRCC_EP_DEV0_0_EP_PCIE_INT_STATUS_BASE_IDX                                                    2
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL2                                                               0x0050
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL2_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_BUS_CNTL                                                               0x0051
+#define regRCC_EP_DEV0_0_EP_PCIE_BUS_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_CFG_CNTL                                                               0x0052
+#define regRCC_EP_DEV0_0_EP_PCIE_CFG_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL                                                            0x0054
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0                                               0x0055
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1                                               0x0055
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2                                               0x0055
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3                                               0x0055
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4                                               0x0056
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5                                               0x0056
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6                                               0x0056
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7                                               0x0056
+#define regRCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC                                                             0x0057
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC_BASE_IDX                                                    2
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2                                                            0x0058
+#define regRCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP                                                             0x005a
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                    2
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0x005b
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL                                                            0x005b
+#define regRCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0x005b
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0x005c
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0x005c
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0x005c
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0x005c
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0x005d
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0x005d
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0x005d
+#define regRCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_PME_CONTROL                                                            0x005d
+#define regRCC_EP_DEV0_0_EP_PCIE_PME_CONTROL_BASE_IDX                                                   2
+#define regRCC_EP_DEV0_0_EP_PCIEP_RESERVED                                                              0x005e
+#define regRCC_EP_DEV0_0_EP_PCIEP_RESERVED_BASE_IDX                                                     2
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_CNTL                                                                0x0060
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_CNTL_BASE_IDX                                                       2
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID                                                        0x0061
+#define regRCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                               2
+#define regRCC_EP_DEV0_0_EP_PCIE_ERR_CNTL                                                               0x0062
+#define regRCC_EP_DEV0_0_EP_PCIE_ERR_CNTL_BASE_IDX                                                      2
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL                                                                0x0063
+#define regRCC_EP_DEV0_0_EP_PCIE_RX_CNTL_BASE_IDX                                                       2
+#define regRCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL                                                          0x0064
+#define regRCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                 2
+#define regRCC_EP_DEV0_0_EP_PCIE_DEVICE_CNTL3                                                           0x0065
+#define regRCC_EP_DEV0_0_EP_PCIE_DEVICE_CNTL3_BASE_IDX                                                  2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_pf_SYSPFVFDEC
+// base address: 0x0
+#define regBIF_BX_PF0_MM_INDEX                                                                          0x0000
+#define regBIF_BX_PF0_MM_INDEX_BASE_IDX                                                                 0
+#define regBIF_BX_PF0_MM_DATA                                                                           0x0001
+#define regBIF_BX_PF0_MM_DATA_BASE_IDX                                                                  0
+#define regBIF_BX_PF0_MM_INDEX_HI                                                                       0x0006
+#define regBIF_BX_PF0_MM_INDEX_HI_BASE_IDX                                                              0
+#define regBIF_BX_PF0_RSMU_INDEX                                                                        0x0000
+#define regBIF_BX_PF0_RSMU_INDEX_BASE_IDX                                                               1
+#define regBIF_BX_PF0_RSMU_DATA                                                                         0x0001
+#define regBIF_BX_PF0_RSMU_DATA_BASE_IDX                                                                1
+#define regBIF_BX_PF0_RSMU_INDEX_HI                                                                     0x0002
+#define regBIF_BX_PF0_RSMU_INDEX_HI_BASE_IDX                                                            1
+
+
+// addressBlock: nbif0_nbif0_bif_bx_BIFDEC1
+// base address: 0x0
+#define regBIF_BX0_CC_BIF_BX_STRAP0                                                                     0x00e2
+#define regBIF_BX0_CC_BIF_BX_STRAP0_BASE_IDX                                                            2
+#define regBIF_BX0_CC_BIF_BX_PINSTRAP0                                                                  0x00e4
+#define regBIF_BX0_CC_BIF_BX_PINSTRAP0_BASE_IDX                                                         2
+#define regBIF_BX0_BIF_MM_INDACCESS_CNTL                                                                0x00e6
+#define regBIF_BX0_BIF_MM_INDACCESS_CNTL_BASE_IDX                                                       2
+#define regBIF_BX0_BUS_CNTL                                                                             0x00e7
+#define regBIF_BX0_BUS_CNTL_BASE_IDX                                                                    2
+#define regBIF_BX0_BIF_SCRATCH0                                                                         0x00e8
+#define regBIF_BX0_BIF_SCRATCH0_BASE_IDX                                                                2
+#define regBIF_BX0_BIF_SCRATCH1                                                                         0x00e9
+#define regBIF_BX0_BIF_SCRATCH1_BASE_IDX                                                                2
+#define regBIF_BX0_BX_RESET_EN                                                                          0x00ed
+#define regBIF_BX0_BX_RESET_EN_BASE_IDX                                                                 2
+#define regBIF_BX0_MM_CFGREGS_CNTL                                                                      0x00ee
+#define regBIF_BX0_MM_CFGREGS_CNTL_BASE_IDX                                                             2
+#define regBIF_BX0_BX_RESET_CNTL                                                                        0x00f0
+#define regBIF_BX0_BX_RESET_CNTL_BASE_IDX                                                               2
+#define regBIF_BX0_INTERRUPT_CNTL                                                                       0x00f1
+#define regBIF_BX0_INTERRUPT_CNTL_BASE_IDX                                                              2
+#define regBIF_BX0_INTERRUPT_CNTL2                                                                      0x00f2
+#define regBIF_BX0_INTERRUPT_CNTL2_BASE_IDX                                                             2
+#define regBIF_BX0_CLKREQB_PAD_CNTL                                                                     0x00f8
+#define regBIF_BX0_CLKREQB_PAD_CNTL_BASE_IDX                                                            2
+#define regBIF_BX0_BIF_FEATURES_CONTROL_MISC                                                            0x00fb
+#define regBIF_BX0_BIF_FEATURES_CONTROL_MISC_BASE_IDX                                                   2
+#define regBIF_BX0_HDP_ATOMIC_CONTROL_MISC                                                              0x00fc
+#define regBIF_BX0_HDP_ATOMIC_CONTROL_MISC_BASE_IDX                                                     2
+#define regBIF_BX0_BIF_DOORBELL_CNTL                                                                    0x00fd
+#define regBIF_BX0_BIF_DOORBELL_CNTL_BASE_IDX                                                           2
+#define regBIF_BX0_BIF_DOORBELL_INT_CNTL                                                                0x00fe
+#define regBIF_BX0_BIF_DOORBELL_INT_CNTL_BASE_IDX                                                       2
+#define regBIF_BX0_BIF_FB_EN                                                                            0x0100
+#define regBIF_BX0_BIF_FB_EN_BASE_IDX                                                                   2
+#define regBIF_BX0_BIF_INTR_CNTL                                                                        0x0101
+#define regBIF_BX0_BIF_INTR_CNTL_BASE_IDX                                                               2
+#define regBIF_BX0_BIF_MST_TRANS_PENDING_VF                                                             0x0109
+#define regBIF_BX0_BIF_MST_TRANS_PENDING_VF_BASE_IDX                                                    2
+#define regBIF_BX0_BIF_SLV_TRANS_PENDING_VF                                                             0x010a
+#define regBIF_BX0_BIF_SLV_TRANS_PENDING_VF_BASE_IDX                                                    2
+#define regBIF_BX0_BACO_CNTL                                                                            0x010b
+#define regBIF_BX0_BACO_CNTL_BASE_IDX                                                                   2
+#define regBIF_BX0_BIF_BACO_EXIT_TIME0                                                                  0x010c
+#define regBIF_BX0_BIF_BACO_EXIT_TIME0_BASE_IDX                                                         2
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER1                                                                 0x010d
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER1_BASE_IDX                                                        2
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER2                                                                 0x010e
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER2_BASE_IDX                                                        2
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER3                                                                 0x010f
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER3_BASE_IDX                                                        2
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER4                                                                 0x0110
+#define regBIF_BX0_BIF_BACO_EXIT_TIMER4_BASE_IDX                                                        2
+#define regBIF_BX0_MEM_TYPE_CNTL                                                                        0x0111
+#define regBIF_BX0_MEM_TYPE_CNTL_BASE_IDX                                                               2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_CNTL                                                               0x0113
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_CNTL_BASE_IDX                                                      2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_0                                                                  0x0114
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_0_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_1                                                                  0x0115
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_1_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_2                                                                  0x0116
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_2_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_3                                                                  0x0117
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_3_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_4                                                                  0x0118
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_4_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_5                                                                  0x0119
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_5_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_6                                                                  0x011a
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_6_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_7                                                                  0x011b
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_7_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_8                                                                  0x011c
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_8_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_9                                                                  0x011d
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_9_BASE_IDX                                                         2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_10                                                                 0x011e
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_10_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_11                                                                 0x011f
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_11_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_12                                                                 0x0120
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_12_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_13                                                                 0x0121
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_13_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_14                                                                 0x0122
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_14_BASE_IDX                                                        2
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_15                                                                 0x0123
+#define regBIF_BX0_NBIF_GFX_ADDR_LUT_15_BASE_IDX                                                        2
+#define regBIF_BX0_VF_REGWR_EN                                                                          0x0124
+#define regBIF_BX0_VF_REGWR_EN_BASE_IDX                                                                 2
+#define regBIF_BX0_VF_DOORBELL_EN                                                                       0x0125
+#define regBIF_BX0_VF_DOORBELL_EN_BASE_IDX                                                              2
+#define regBIF_BX0_VF_FB_EN                                                                             0x0126
+#define regBIF_BX0_VF_FB_EN_BASE_IDX                                                                    2
+#define regBIF_BX0_VF_REGWR_STATUS                                                                      0x0127
+#define regBIF_BX0_VF_REGWR_STATUS_BASE_IDX                                                             2
+#define regBIF_BX0_VF_DOORBELL_STATUS                                                                   0x0128
+#define regBIF_BX0_VF_DOORBELL_STATUS_BASE_IDX                                                          2
+#define regBIF_BX0_VF_FB_STATUS                                                                         0x0129
+#define regBIF_BX0_VF_FB_STATUS_BASE_IDX                                                                2
+#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL                                                             0x012d
+#define regBIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL_BASE_IDX                                                    2
+#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL                                                             0x012e
+#define regBIF_BX0_REMAP_HDP_REG_FLUSH_CNTL_BASE_IDX                                                    2
+#define regBIF_BX0_BIF_RB_CNTL                                                                          0x012f
+#define regBIF_BX0_BIF_RB_CNTL_BASE_IDX                                                                 2
+#define regBIF_BX0_BIF_RB_BASE                                                                          0x0130
+#define regBIF_BX0_BIF_RB_BASE_BASE_IDX                                                                 2
+#define regBIF_BX0_BIF_RB_RPTR                                                                          0x0131
+#define regBIF_BX0_BIF_RB_RPTR_BASE_IDX                                                                 2
+#define regBIF_BX0_BIF_RB_WPTR                                                                          0x0132
+#define regBIF_BX0_BIF_RB_WPTR_BASE_IDX                                                                 2
+#define regBIF_BX0_BIF_RB_WPTR_ADDR_HI                                                                  0x0133
+#define regBIF_BX0_BIF_RB_WPTR_ADDR_HI_BASE_IDX                                                         2
+#define regBIF_BX0_BIF_RB_WPTR_ADDR_LO                                                                  0x0134
+#define regBIF_BX0_BIF_RB_WPTR_ADDR_LO_BASE_IDX                                                         2
+#define regBIF_BX0_MAILBOX_INDEX                                                                        0x0135
+#define regBIF_BX0_MAILBOX_INDEX_BASE_IDX                                                               2
+#define regBIF_BX0_BACO_AZ_ENHANCE_CTRL                                                                 0x0141
+#define regBIF_BX0_BACO_AZ_ENHANCE_CTRL_BASE_IDX                                                        2
+#define regBIF_BX0_BIF_MP1_INTR_CTRL                                                                    0x0142
+#define regBIF_BX0_BIF_MP1_INTR_CTRL_BASE_IDX                                                           2
+#define regBIF_BX0_BIF_PERSTB_PAD_CNTL                                                                  0x0145
+#define regBIF_BX0_BIF_PERSTB_PAD_CNTL_BASE_IDX                                                         2
+#define regBIF_BX0_BIF_PX_EN_PAD_CNTL                                                                   0x0146
+#define regBIF_BX0_BIF_PX_EN_PAD_CNTL_BASE_IDX                                                          2
+#define regBIF_BX0_BIF_REFPADKIN_PAD_CNTL                                                               0x0147
+#define regBIF_BX0_BIF_REFPADKIN_PAD_CNTL_BASE_IDX                                                      2
+#define regBIF_BX0_BIF_CLKREQB_PAD_CNTL                                                                 0x0148
+#define regBIF_BX0_BIF_CLKREQB_PAD_CNTL_BASE_IDX                                                        2
+#define regBIF_BX0_BIF_PWRBRK_PAD_CNTL                                                                  0x0149
+#define regBIF_BX0_BIF_PWRBRK_PAD_CNTL_BASE_IDX                                                         2
+#define regBIF_BX0_BIF_GPUIOV_SCH_CFG_SIZE                                                              0x0165
+#define regBIF_BX0_BIF_GPUIOV_SCH_CFG_SIZE_BASE_IDX                                                     2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_BIFDEC1
+// base address: 0x0
+#define regRCC_DEV0_0_RCC_ERR_INT_CNTL                                                                  0x0086
+#define regRCC_DEV0_0_RCC_ERR_INT_CNTL_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_BACO_CNTL_MISC                                                                0x0087
+#define regRCC_DEV0_0_RCC_BACO_CNTL_MISC_BASE_IDX                                                       2
+#define regRCC_DEV0_0_RCC_RESET_EN                                                                      0x0088
+#define regRCC_DEV0_0_RCC_RESET_EN_BASE_IDX                                                             2
+#define regRCC_DEV0_0_RCC_VDM_SUPPORT                                                                   0x0089
+#define regRCC_DEV0_0_RCC_VDM_SUPPORT_BASE_IDX                                                          2
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0                                                            0x008a
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1                                                            0x008b
+#define regRCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_GPUIOV_REGION                                                                 0x008c
+#define regRCC_DEV0_0_RCC_GPUIOV_REGION_BASE_IDX                                                        2
+#define regRCC_DEV0_0_RCC_GPU_HOSTVM_EN                                                                 0x008d
+#define regRCC_DEV0_0_RCC_GPU_HOSTVM_EN_BASE_IDX                                                        2
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL                                                         0x008e
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL_BASE_IDX                                                2
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET                                                   0x008f
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET_BASE_IDX                                          2
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE                                                         0x008f
+#define regRCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE_BASE_IDX                                                2
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE0                                                               0x00be
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE0_BASE_IDX                                                      2
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE1                                                               0x00bf
+#define regRCC_DEV0_0_RCC_PEER_REG_RANGE1_BASE_IDX                                                      2
+#define regRCC_DEV0_0_RCC_BUS_CNTL                                                                      0x00c1
+#define regRCC_DEV0_0_RCC_BUS_CNTL_BASE_IDX                                                             2
+#define regRCC_DEV0_0_RCC_CONFIG_CNTL                                                                   0x00c2
+#define regRCC_DEV0_0_RCC_CONFIG_CNTL_BASE_IDX                                                          2
+#define regRCC_DEV0_0_RCC_CONFIG_F0_BASE                                                                0x00c6
+#define regRCC_DEV0_0_RCC_CONFIG_F0_BASE_BASE_IDX                                                       2
+#define regRCC_DEV0_0_RCC_CONFIG_APER_SIZE                                                              0x00c7
+#define regRCC_DEV0_0_RCC_CONFIG_APER_SIZE_BASE_IDX                                                     2
+#define regRCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE                                                          0x00c8
+#define regRCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE_BASE_IDX                                                 2
+#define regRCC_DEV0_0_RCC_XDMA_LO                                                                       0x00c9
+#define regRCC_DEV0_0_RCC_XDMA_LO_BASE_IDX                                                              2
+#define regRCC_DEV0_0_RCC_XDMA_HI                                                                       0x00ca
+#define regRCC_DEV0_0_RCC_XDMA_HI_BASE_IDX                                                              2
+#define regRCC_DEV0_0_RCC_FEATURES_CONTROL_MISC                                                         0x00cb
+#define regRCC_DEV0_0_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                2
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL1                                                                  0x00cc
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL1_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST0                                                                  0x00cd
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST0_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST1                                                                  0x00ce
+#define regRCC_DEV0_0_RCC_BUSNUM_LIST1_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL2                                                                  0x00cf
+#define regRCC_DEV0_0_RCC_BUSNUM_CNTL2_BASE_IDX                                                         2
+#define regRCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM                                                           0x00d0
+#define regRCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM_BASE_IDX                                                  2
+#define regRCC_DEV0_0_RCC_HOST_BUSNUM                                                                   0x00d1
+#define regRCC_DEV0_0_RCC_HOST_BUSNUM_BASE_IDX                                                          2
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI                                                            0x00d2
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO                                                            0x00d3
+#define regRCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI                                                            0x00d4
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO                                                            0x00d5
+#define regRCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI                                                            0x00d6
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO                                                            0x00d7
+#define regRCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI                                                            0x00d8
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO                                                            0x00d9
+#define regRCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO_BASE_IDX                                                   2
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST0                                                              0x00da
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST0_BASE_IDX                                                     2
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST1                                                              0x00db
+#define regRCC_DEV0_0_RCC_DEVFUNCNUM_LIST1_BASE_IDX                                                     2
+#define regRCC_DEV0_0_RCC_DEV0_LINK_CNTL                                                                0x00dd
+#define regRCC_DEV0_0_RCC_DEV0_LINK_CNTL_BASE_IDX                                                       2
+#define regRCC_DEV0_0_RCC_CMN_LINK_CNTL                                                                 0x00de
+#define regRCC_DEV0_0_RCC_CMN_LINK_CNTL_BASE_IDX                                                        2
+#define regRCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE                                                        0x00df
+#define regRCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                               2
+#define regRCC_DEV0_0_RCC_LTR_LSWITCH_CNTL                                                              0x00e0
+#define regRCC_DEV0_0_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                     2
+#define regRCC_DEV0_0_RCC_MH_ARB_CNTL                                                                   0x00e1
+#define regRCC_DEV0_0_RCC_MH_ARB_CNTL_BASE_IDX                                                          2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO                                                          0x0400
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI                                                          0x0401
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA                                                         0x0402
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                                3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL                                                          0x0403
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO                                                          0x0404
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI                                                          0x0405
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA                                                         0x0406
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                                3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL                                                          0x0407
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO                                                          0x0408
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI                                                          0x0409
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA                                                         0x040a
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                                3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL                                                          0x040b
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO                                                          0x040c
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI                                                          0x040d
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA                                                         0x040e
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                                3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL                                                          0x040f
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_LO                                                          0x0410
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_LO_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_HI                                                          0x0411
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_HI_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT4_MSG_DATA                                                         0x0412
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT4_MSG_DATA_BASE_IDX                                                3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT4_CONTROL                                                          0x0413
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT4_CONTROL_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_LO                                                          0x0414
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_LO_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_HI                                                          0x0415
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_HI_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT5_MSG_DATA                                                         0x0416
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT5_MSG_DATA_BASE_IDX                                                3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT5_CONTROL                                                          0x0417
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT5_CONTROL_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_LO                                                          0x0418
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_LO_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_HI                                                          0x0419
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_HI_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT6_MSG_DATA                                                         0x041a
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT6_MSG_DATA_BASE_IDX                                                3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT6_CONTROL                                                          0x041b
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT6_CONTROL_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_LO                                                          0x041c
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_LO_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_HI                                                          0x041d
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_HI_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT7_MSG_DATA                                                         0x041e
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT7_MSG_DATA_BASE_IDX                                                3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT7_CONTROL                                                          0x041f
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT7_CONTROL_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_LO                                                          0x0420
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_LO_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_HI                                                          0x0421
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_HI_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT8_MSG_DATA                                                         0x0422
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT8_MSG_DATA_BASE_IDX                                                3
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT8_CONTROL                                                          0x0423
+#define regRCC_DEV0_EPF0_GFXMSIX_VECT8_CONTROL_BASE_IDX                                                 3
+#define regRCC_DEV0_EPF0_GFXMSIX_PBA                                                                    0x0800
+#define regRCC_DEV0_EPF0_GFXMSIX_PBA_BASE_IDX                                                           3
+
+
+// addressBlock: nbif0_nbif0_rcc_strap_BIFDEC1
+// base address: 0x0
+#define regRCC_STRAP0_RCC_BIF_STRAP0                                                                    0x0000
+#define regRCC_STRAP0_RCC_BIF_STRAP0_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP1                                                                    0x0001
+#define regRCC_STRAP0_RCC_BIF_STRAP1_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP10                                                                   0x0002
+#define regRCC_STRAP0_RCC_BIF_STRAP10_BASE_IDX                                                          2
+#define regRCC_STRAP0_RCC_BIF_STRAP11                                                                   0x0003
+#define regRCC_STRAP0_RCC_BIF_STRAP11_BASE_IDX                                                          2
+#define regRCC_STRAP0_RCC_BIF_STRAP12                                                                   0x0004
+#define regRCC_STRAP0_RCC_BIF_STRAP12_BASE_IDX                                                          2
+#define regRCC_STRAP0_RCC_BIF_STRAP13                                                                   0x0005
+#define regRCC_STRAP0_RCC_BIF_STRAP13_BASE_IDX                                                          2
+#define regRCC_STRAP0_RCC_BIF_STRAP2                                                                    0x0006
+#define regRCC_STRAP0_RCC_BIF_STRAP2_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP3                                                                    0x0007
+#define regRCC_STRAP0_RCC_BIF_STRAP3_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP4                                                                    0x0008
+#define regRCC_STRAP0_RCC_BIF_STRAP4_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP5                                                                    0x0009
+#define regRCC_STRAP0_RCC_BIF_STRAP5_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP6                                                                    0x000a
+#define regRCC_STRAP0_RCC_BIF_STRAP6_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP7                                                                    0x000b
+#define regRCC_STRAP0_RCC_BIF_STRAP7_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP8                                                                    0x000c
+#define regRCC_STRAP0_RCC_BIF_STRAP8_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_BIF_STRAP9                                                                    0x000d
+#define regRCC_STRAP0_RCC_BIF_STRAP9_BASE_IDX                                                           2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP0                                                              0x000e
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP0_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP1                                                              0x000f
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP1_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP10                                                             0x0010
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP10_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP11                                                             0x0011
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP11_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP12                                                             0x0012
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP12_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP13                                                             0x0013
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP13_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP14                                                             0x0014
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP14_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP15                                                             0x0015
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP15_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP16                                                             0x0016
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP16_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP17                                                             0x0017
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP17_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP18                                                             0x0018
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP18_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP2                                                              0x0019
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP2_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP3                                                              0x001a
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP3_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP4                                                              0x001b
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP4_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP5                                                              0x001c
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP5_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP6                                                              0x001d
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP6_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP7                                                              0x001e
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP7_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP8                                                              0x001f
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP8_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP9                                                              0x0020
+#define regRCC_STRAP0_RCC_DEV0_PORT_STRAP9_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0                                                              0x0021
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP0_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP1                                                              0x0022
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP1_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP13                                                             0x0023
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP13_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP14                                                             0x0024
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP14_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP15                                                             0x0025
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP15_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP16                                                             0x0026
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP16_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP17                                                             0x0027
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP17_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP18                                                             0x0028
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP18_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP19                                                             0x0029
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP19_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP2                                                              0x002a
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP2_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP22                                                             0x002b
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP22_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP23                                                             0x002c
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP23_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP24                                                             0x002d
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP24_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP3                                                              0x002e
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP3_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP4                                                              0x002f
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP4_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP5                                                              0x0030
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP5_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP8                                                              0x0031
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP8_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP9                                                              0x0032
+#define regRCC_STRAP0_RCC_DEV0_EPF0_STRAP9_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP0                                                              0x0033
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP0_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP1                                                              0x0034
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP1_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP10                                                             0x0035
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP10_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP11                                                             0x0036
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP11_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP12                                                             0x0037
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP12_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP13                                                             0x0038
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP13_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP14                                                             0x0039
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP14_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP15                                                             0x003a
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP15_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP16                                                             0x003b
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP16_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP17                                                             0x003c
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP17_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP18                                                             0x003d
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP18_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP19                                                             0x003e
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP19_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP2                                                              0x003f
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP2_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP20                                                             0x0040
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP20_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP21                                                             0x0041
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP21_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP23                                                             0x0042
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP23_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP24                                                             0x0043
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP24_BASE_IDX                                                    2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP3                                                              0x0044
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP3_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP4                                                              0x0045
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP4_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP5                                                              0x0046
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP5_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP6                                                              0x0047
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP6_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP7                                                              0x0048
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP7_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP8                                                              0x0049
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP8_BASE_IDX                                                     2
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP9                                                              0x004a
+#define regRCC_STRAP0_RCC_DEV0_EPF1_STRAP9_BASE_IDX                                                     2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_pf_BIFPFVFDEC1
+// base address: 0x0
+#define regBIF_BX_PF0_BIF_BME_STATUS                                                                    0x00eb
+#define regBIF_BX_PF0_BIF_BME_STATUS_BASE_IDX                                                           2
+#define regBIF_BX_PF0_BIF_ATOMIC_ERR_LOG                                                                0x00ec
+#define regBIF_BX_PF0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                                       2
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                              0x00f3
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                                     2
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                               0x00f4
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                                      2
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL                                                   0x00f5
+#define regBIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                          2
+#define regBIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL                                                      0x00f6
+#define regBIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                             2
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL                                                      0x00f7
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                             2
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                                 0x00f9
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                                        2
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                            0x00fa
+#define regBIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                                   2
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ                                                                 0x0106
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_REQ_BASE_IDX                                                        2
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE                                                                0x0107
+#define regBIF_BX_PF0_GPU_HDP_FLUSH_DONE_BASE_IDX                                                       2
+#define regBIF_BX_PF0_BIF_TRANS_PENDING                                                                 0x0108
+#define regBIF_BX_PF0_BIF_TRANS_PENDING_BASE_IDX                                                        2
+#define regBIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS                                                          0x0112
+#define regBIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                                 2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0                                                            0x0136
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1                                                            0x0137
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2                                                            0x0138
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3                                                            0x0139
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0                                                            0x013a
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1                                                            0x013b
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2                                                            0x013c
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3                                                            0x013d
+#define regBIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                                   2
+#define regBIF_BX_PF0_MAILBOX_CONTROL                                                                   0x013e
+#define regBIF_BX_PF0_MAILBOX_CONTROL_BASE_IDX                                                          2
+#define regBIF_BX_PF0_MAILBOX_INT_CNTL                                                                  0x013f
+#define regBIF_BX_PF0_MAILBOX_INT_CNTL_BASE_IDX                                                         2
+#define regBIF_BX_PF0_BIF_VMHV_MAILBOX                                                                  0x0140
+#define regBIF_BX_PF0_BIF_VMHV_MAILBOX_BASE_IDX                                                         2
+#define regBIF_BX_PF0_PARTITION_COMPUTE_CAP                                                             0x0161
+#define regBIF_BX_PF0_PARTITION_COMPUTE_CAP_BASE_IDX                                                    2
+#define regBIF_BX_PF0_PARTITION_MEM_CAP                                                                 0x0162
+#define regBIF_BX_PF0_PARTITION_MEM_CAP_BASE_IDX                                                        2
+#define regBIF_BX_PF0_PARTITION_COMPUTE_STATUS                                                          0x0163
+#define regBIF_BX_PF0_PARTITION_COMPUTE_STATUS_BASE_IDX                                                 2
+#define regBIF_BX_PF0_PARTITION_MEM_STATUS                                                              0x0164
+#define regBIF_BX_PF0_PARTITION_MEM_STATUS_BASE_IDX                                                     2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_RCC_ERR_LOG                                                                    0x0085
+#define regRCC_DEV0_EPF0_RCC_ERR_LOG_BASE_IDX                                                           2
+#define regRCC_DEV0_EPF0_RCC_DOORBELL_APER_EN                                                           0x00c0
+#define regRCC_DEV0_EPF0_RCC_DOORBELL_APER_EN_BASE_IDX                                                  2
+#define regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE                                                             0x00c3
+#define regRCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE_BASE_IDX                                                    2
+#define regRCC_DEV0_EPF0_RCC_CONFIG_RESERVED                                                            0x00c4
+#define regRCC_DEV0_EPF0_RCC_CONFIG_RESERVED_BASE_IDX                                                   2
+#define regRCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER                                                        0x00c5
+#define regRCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                               2
+
+
+// addressBlock: nbif0_nbif0_gdc_GDCDEC
+// base address: 0x0
+#define regGDC0_NGDC_SDP_PORT_CTRL                                                                      0x0180
+#define regGDC0_NGDC_SDP_PORT_CTRL_BASE_IDX                                                             2
+#define regGDC0_SHUB_REGS_IF_CTL                                                                        0x0181
+#define regGDC0_SHUB_REGS_IF_CTL_BASE_IDX                                                               2
+#define regGDC0_NGDC_MGCG_CTRL                                                                          0x0187
+#define regGDC0_NGDC_MGCG_CTRL_BASE_IDX                                                                 2
+#define regGDC0_S2A_MISC_CNTL                                                                           0x0188
+#define regGDC0_S2A_MISC_CNTL_BASE_IDX                                                                  2
+#define regGDC0_NGDC_MCA_SMN_CTRL0                                                                      0x018b
+#define regGDC0_NGDC_MCA_SMN_CTRL0_BASE_IDX                                                             2
+#define regGDC0_NGDC_EARLY_WAKEUP_CTRL                                                                  0x018c
+#define regGDC0_NGDC_EARLY_WAKEUP_CTRL_BASE_IDX                                                         2
+#define regGDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL                                                             0x0198
+#define regGDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL_BASE_IDX                                                    2
+#define regGDC0_GDC_DMA_SION_PCTRL                                                                      0x0199
+#define regGDC0_GDC_DMA_SION_PCTRL_BASE_IDX                                                             2
+#define regGDC0_NGDC_MP4SDP_CNTL                                                                        0x019b
+#define regGDC0_NGDC_MP4SDP_CNTL_BASE_IDX                                                               2
+#define regGDC0_DOORBELL_VCN_TARGET_VF0                                                                 0x01f3
+#define regGDC0_DOORBELL_VCN_TARGET_VF0_BASE_IDX                                                        2
+#define regGDC0_DOORBELL_VCN_TARGET_VF1                                                                 0x01f3
+#define regGDC0_DOORBELL_VCN_TARGET_VF1_BASE_IDX                                                        2
+#define regGDC0_DOORBELL_VCN_TARGET_VF2                                                                 0x01f4
+#define regGDC0_DOORBELL_VCN_TARGET_VF2_BASE_IDX                                                        2
+#define regGDC0_DOORBELL_VCN_TARGET_VF3                                                                 0x01f4
+#define regGDC0_DOORBELL_VCN_TARGET_VF3_BASE_IDX                                                        2
+#define regGDC0_DOORBELL_VCN_TARGET_VF4                                                                 0x01f5
+#define regGDC0_DOORBELL_VCN_TARGET_VF4_BASE_IDX                                                        2
+#define regGDC0_DOORBELL_VCN_TARGET_VF5                                                                 0x01f5
+#define regGDC0_DOORBELL_VCN_TARGET_VF5_BASE_IDX                                                        2
+#define regGDC0_DOORBELL_VCN_TARGET_VF6                                                                 0x01f6
+#define regGDC0_DOORBELL_VCN_TARGET_VF6_BASE_IDX                                                        2
+#define regGDC0_DOORBELL_VCN_TARGET_VF7                                                                 0x01f6
+#define regGDC0_DOORBELL_VCN_TARGET_VF7_BASE_IDX                                                        2
+#define regGDC0_DOORBELL_ACCESS_EN_PF                                                                   0x01f7
+#define regGDC0_DOORBELL_ACCESS_EN_PF_BASE_IDX                                                          2
+#define regGDC0_DOORBELL_ACCESS_EN_VF0                                                                  0x01f7
+#define regGDC0_DOORBELL_ACCESS_EN_VF0_BASE_IDX                                                         2
+#define regGDC0_DOORBELL_ACCESS_EN_VF1                                                                  0x01f8
+#define regGDC0_DOORBELL_ACCESS_EN_VF1_BASE_IDX                                                         2
+#define regGDC0_DOORBELL_ACCESS_EN_VF2                                                                  0x01f8
+#define regGDC0_DOORBELL_ACCESS_EN_VF2_BASE_IDX                                                         2
+#define regGDC0_DOORBELL_ACCESS_EN_VF3                                                                  0x01f9
+#define regGDC0_DOORBELL_ACCESS_EN_VF3_BASE_IDX                                                         2
+#define regGDC0_DOORBELL_ACCESS_EN_VF4                                                                  0x01f9
+#define regGDC0_DOORBELL_ACCESS_EN_VF4_BASE_IDX                                                         2
+#define regGDC0_DOORBELL_ACCESS_EN_VF5                                                                  0x01fa
+#define regGDC0_DOORBELL_ACCESS_EN_VF5_BASE_IDX                                                         2
+#define regGDC0_DOORBELL_ACCESS_EN_VF6                                                                  0x01fa
+#define regGDC0_DOORBELL_ACCESS_EN_VF6_BASE_IDX                                                         2
+#define regGDC0_DOORBELL_ACCESS_EN_VF7                                                                  0x01fb
+#define regGDC0_DOORBELL_ACCESS_EN_VF7_BASE_IDX                                                         2
+#define regGDC0_NGDC_CNDI_BUS_PORT_CTRL                                                                 0x01fc
+#define regGDC0_NGDC_CNDI_BUS_PORT_CTRL_BASE_IDX                                                        2
+#define regGDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL                                                            0x01fd
+#define regGDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL_BASE_IDX                                                   2
+
+
+// addressBlock: nbif0_nbif0_gdc_s2a_GDCS2A_DEC
+// base address: 0x0
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL                                                           0x01cb
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1                                                          0x01cc
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL                                                           0x01cd
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1                                                          0x01ce
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL                                                           0x01cf
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1                                                          0x01d0
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL                                                           0x01d1
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1                                                          0x01d2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL                                                           0x01d3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1                                                          0x01d4
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL                                                           0x01d5
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1                                                          0x01d6
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL                                                           0x01d7
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1                                                          0x01d8
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL                                                           0x01d9
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1                                                          0x01da
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL                                                           0x01db
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1                                                          0x01dc
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL                                                           0x01dd
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL_BASE_IDX                                                  2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1                                                          0x01de
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL                                                          0x01df
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1                                                         0x01e0
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1_BASE_IDX                                                2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL                                                          0x01e1
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1                                                         0x01e2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1_BASE_IDX                                                2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL                                                          0x01e3
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1                                                         0x01e4
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1_BASE_IDX                                                2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL                                                          0x01e5
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1                                                         0x01e6
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1_BASE_IDX                                                2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL                                                          0x01e7
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1                                                         0x01e8
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1_BASE_IDX                                                2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL                                                          0x01e9
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL_BASE_IDX                                                 2
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1                                                         0x01ea
+#define regGDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1_BASE_IDX                                                2
+#define regGDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG                                                        0x01eb
+#define regGDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG_BASE_IDX                                               2
+#define regGDC_S2A0_NBIF_GFX_DOORBELL_STATUS                                                            0x01ec
+#define regGDC_S2A0_NBIF_GFX_DOORBELL_STATUS_BASE_IDX                                                   2
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VENDOR_ID                                                                  0x0000
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_ID                                                                  0x0002
+#define cfgBIF_CFG_DEV0_EPF0_COMMAND                                                                    0x0004
+#define cfgBIF_CFG_DEV0_EPF0_STATUS                                                                     0x0006
+#define cfgBIF_CFG_DEV0_EPF0_REVISION_ID                                                                0x0008
+#define cfgBIF_CFG_DEV0_EPF0_PROG_INTERFACE                                                             0x0009
+#define cfgBIF_CFG_DEV0_EPF0_SUB_CLASS                                                                  0x000a
+#define cfgBIF_CFG_DEV0_EPF0_BASE_CLASS                                                                 0x000b
+#define cfgBIF_CFG_DEV0_EPF0_CACHE_LINE                                                                 0x000c
+#define cfgBIF_CFG_DEV0_EPF0_LATENCY                                                                    0x000d
+#define cfgBIF_CFG_DEV0_EPF0_HEADER                                                                     0x000e
+#define cfgBIF_CFG_DEV0_EPF0_BIST                                                                       0x000f
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_1                                                                0x0010
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_2                                                                0x0014
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_3                                                                0x0018
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_4                                                                0x001c
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_5                                                                0x0020
+#define cfgBIF_CFG_DEV0_EPF0_BASE_ADDR_6                                                                0x0024
+#define cfgBIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR                                                            0x0028
+#define cfgBIF_CFG_DEV0_EPF0_ADAPTER_ID                                                                 0x002c
+#define cfgBIF_CFG_DEV0_EPF0_ROM_BASE_ADDR                                                              0x0030
+#define cfgBIF_CFG_DEV0_EPF0_CAP_PTR                                                                    0x0034
+#define cfgBIF_CFG_DEV0_EPF0_INTERRUPT_LINE                                                             0x003c
+#define cfgBIF_CFG_DEV0_EPF0_INTERRUPT_PIN                                                              0x003d
+#define cfgBIF_CFG_DEV0_EPF0_MIN_GRANT                                                                  0x003e
+#define cfgBIF_CFG_DEV0_EPF0_MAX_LATENCY                                                                0x003f
+#define cfgVENDOR_CAP_LIST                                                                              0x0048
+#define cfgADAPTER_ID_W                                                                                 0x004c
+#define cfgBIF_CFG_DEV0_EPF0_PMI_CAP_LIST                                                               0x0050
+#define cfgBIF_CFG_DEV0_EPF0_PMI_CAP                                                                    0x0052
+#define cfgBIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL                                                            0x0054
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CAP_LIST                                                              0x0064
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CAP                                                                   0x0066
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CAP                                                                 0x0068
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CNTL                                                                0x006c
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_STATUS                                                              0x006e
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CAP2                                                                0x0088
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_CNTL2                                                               0x008c
+#define cfgBIF_CFG_DEV0_EPF0_DEVICE_STATUS2                                                             0x008e
+#define cfgBIF_CFG_DEV0_EPF0_MSI_CAP_LIST                                                               0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_CNTL                                                               0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO                                                            0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI                                                            0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_DATA                                                               0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA                                                           0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MASK                                                                   0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64                                                            0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64                                                        0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_MSI_MASK_64                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_MSI_PENDING                                                                0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_MSI_PENDING_64                                                             0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_CAP_LIST                                                              0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL                                                              0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_TABLE                                                                 0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_MSIX_PBA                                                                   0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST                                                    0x0600
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x0604
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS                                                     0x0608
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK                                                       0x060c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY                                                   0x0610
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS                                                       0x0614
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK                                                         0x0618
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL                                                      0x061c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0                                                              0x0620
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1                                                              0x0624
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2                                                              0x0628
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3                                                              0x062c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG4                                                              0x063c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0                                                       0x063c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG5                                                              0x0640
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1                                                       0x0640
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG6                                                              0x0644
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2                                                       0x0644
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG7                                                              0x0648
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3                                                       0x0648
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG8                                                              0x064c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG9                                                              0x0650
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG10                                                             0x0654
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG11                                                             0x0658
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG12                                                             0x065c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_HDR_LOG13                                                             0x0660
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                           0x0664
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1                                                    0x0668
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2                                                    0x066c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST                                                      0x06dc
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ACS_CAP                                                               0x06e0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL                                                              0x06e2
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST                                                      0x071c
+#define cfgBIF_CFG_DEV0_EPF0_RTR_DATA1                                                                  0x0720
+#define cfgBIF_CFG_DEV0_EPF0_RTR_DATA2                                                                  0x0724
+#define cfgPCIE_PWR_BUDGET_ENH_CAP_LIST                                                                 0x0728
+#define cfgPCIE_PWR_BUDGET_DATA_SELECT                                                                  0x072c
+#define cfgPCIE_PWR_BUDGET_DATA                                                                         0x0730
+#define cfgPCIE_PWR_BUDGET_CAP                                                                          0x0734
+#define cfgPCIE_DPA_ENH_CAP_LIST                                                                        0x0738
+#define cfgPCIE_DPA_CAP                                                                                 0x073c
+#define cfgPCIE_DPA_LATENCY_INDICATOR                                                                   0x0740
+#define cfgPCIE_DPA_STATUS                                                                              0x0744
+#define cfgPCIE_DPA_CNTL                                                                                0x0746
+#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_0                                                                0x0748
+#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_1                                                                0x0749
+#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_2                                                                0x074a
+#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_3                                                                0x074b
+#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_4                                                                0x074c
+#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_5                                                                0x074d
+#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_6                                                                0x074e
+#define cfgPCIE_DPA_SUBSTATE_PWR_ALLOC_7                                                                0x074f
+#define cfgPCIE_LTR_ENH_CAP_LIST                                                                        0x0768
+#define cfgPCIE_LTR_CAP                                                                                 0x076c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST                                                      0x0770
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ARI_CAP                                                               0x0774
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL                                                              0x0776
+#define cfgPCIE_BAR_ENH_CAP_LIST                                                                        0x0778
+#define cfgPCIE_BAR1_CAP                                                                                0x077c
+#define cfgPCIE_BAR1_CNTL                                                                               0x0780
+#define cfgPCIE_BAR2_CAP                                                                                0x0784
+#define cfgPCIE_BAR2_CNTL                                                                               0x0788
+#define cfgPCIE_BAR3_CAP                                                                                0x078c
+#define cfgPCIE_BAR3_CNTL                                                                               0x0790
+#define cfgPCIE_BAR4_CAP                                                                                0x0794
+#define cfgPCIE_BAR4_CNTL                                                                               0x0798
+#define cfgPCIE_BAR5_CAP                                                                                0x079c
+#define cfgPCIE_BAR5_CNTL                                                                               0x07a0
+#define cfgPCIE_BAR6_CAP                                                                                0x07a4
+#define cfgPCIE_BAR6_CNTL                                                                               0x07a8
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST                                                      0x07ac
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ATS_CAP                                                               0x07b0
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL                                                              0x07b2
+#define cfgPCIE_PAGE_REQ_ENH_CAP_LIST                                                                   0x07b4
+#define cfgPCIE_PAGE_REQ_CNTL                                                                           0x07b8
+#define cfgPCIE_PAGE_REQ_STATUS                                                                         0x07ba
+#define cfgPCIE_OUTSTAND_PAGE_REQ_CAPACITY                                                              0x07bc
+#define cfgPCIE_OUTSTAND_PAGE_REQ_ALLOC                                                                 0x07c0
+#define cfgPCIE_PASID_ENH_CAP_LIST                                                                      0x07c4
+#define cfgPCIE_PASID_CAP                                                                               0x07c8
+#define cfgPCIE_PASID_CNTL                                                                              0x07ca
+#define cfgPCIE_SRIOV_ENH_CAP_LIST                                                                      0x07cc
+#define cfgPCIE_SRIOV_CAP                                                                               0x07d0
+#define cfgPCIE_SRIOV_CONTROL                                                                           0x07d4
+#define cfgPCIE_SRIOV_STATUS                                                                            0x07d6
+#define cfgPCIE_SRIOV_INITIAL_VFS                                                                       0x07d8
+#define cfgPCIE_SRIOV_TOTAL_VFS                                                                         0x07da
+#define cfgPCIE_SRIOV_NUM_VFS                                                                           0x07dc
+#define cfgPCIE_SRIOV_FUNC_DEP_LINK                                                                     0x07de
+#define cfgPCIE_SRIOV_FIRST_VF_OFFSET                                                                   0x07e0
+#define cfgPCIE_SRIOV_VF_STRIDE                                                                         0x07e2
+#define cfgPCIE_SRIOV_VF_DEVICE_ID                                                                      0x07e6
+#define cfgPCIE_SRIOV_SUPPORTED_PAGE_SIZE                                                               0x07e8
+#define cfgPCIE_SRIOV_SYSTEM_PAGE_SIZE                                                                  0x07ec
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_0                                                                    0x07f0
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_1                                                                    0x07f4
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_2                                                                    0x07f8
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_3                                                                    0x07fc
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_4                                                                    0x0800
+#define cfgPCIE_SRIOV_VF_BASE_ADDR_5                                                                    0x0804
+#define cfgPCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET                                                   0x0808
+#define cfgPCIE_VF_RESIZE_BAR_ENH_CAP_LIST                                                              0x08e0
+#define cfgPCIE_VF_RESIZE_BAR1_CAP                                                                      0x08e4
+#define cfgPCIE_VF_RESIZE_BAR1_CNTL                                                                     0x08e8
+#define cfgPCIE_VF_RESIZE_BAR2_CAP                                                                      0x08ec
+#define cfgPCIE_VF_RESIZE_BAR2_CNTL                                                                     0x08f0
+#define cfgPCIE_VF_RESIZE_BAR3_CAP                                                                      0x08f4
+#define cfgPCIE_VF_RESIZE_BAR3_CNTL                                                                     0x08f8
+#define cfgPCIE_VF_RESIZE_BAR4_CAP                                                                      0x08fc
+#define cfgPCIE_VF_RESIZE_BAR4_CNTL                                                                     0x0900
+#define cfgPCIE_VF_RESIZE_BAR5_CAP                                                                      0x0904
+#define cfgPCIE_VF_RESIZE_BAR5_CNTL                                                                     0x0908
+#define cfgPCIE_VF_RESIZE_BAR6_CAP                                                                      0x090c
+#define cfgPCIE_VF_RESIZE_BAR6_CNTL                                                                     0x0910
+#define cfgPCIE_DOE_ENH_CAP_LIST                                                                        0x0914
+#define cfgPCIE_DOE_CAP                                                                                 0x0918
+#define cfgPCIE_DOE_CNTL                                                                                0x091c
+#define cfgPCIE_DOE_STS                                                                                 0x0920
+#define cfgPCIE_DOE_WMBOX                                                                               0x0924
+#define cfgPCIE_DOE_RMBOX                                                                               0x0928
+#define cfgPCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV                                                     0x09ac
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV                                                              0x09b0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW                                                 0x09b4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE                                                  0x09b8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS                                                  0x09bc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL                                                0x09c0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0                                                0x09c4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1                                                0x09c8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2                                                0x09cc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB                                                     0x09d4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION                                                       0x09dc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE                                         0x09e0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0                                                     0x0a04
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1                                                     0x0a08
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2                                                     0x0a0c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3                                                     0x0a10
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4                                                     0x0a14
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW0                                                     0x0a1c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW1                                                     0x0a20
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW2                                                     0x0a24
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW3                                                     0x0a28
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW4                                                     0x0a2c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW5                                                     0x0a30
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW6                                                     0x0a34
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW7                                                     0x0a38
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW8                                                     0x0a3c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW0                                                     0x0a4c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW1                                                     0x0a50
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW2                                                     0x0a54
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW3                                                     0x0a58
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW4                                                     0x0a5c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW5                                                     0x0a60
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW6                                                     0x0a64
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW7                                                     0x0a68
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW8                                                     0x0a6c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW0                                                     0x0a7c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW1                                                     0x0a80
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW2                                                     0x0a84
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW3                                                     0x0a88
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW4                                                     0x0a8c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW5                                                     0x0a90
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW6                                                     0x0a94
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW7                                                     0x0a98
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW8                                                     0x0a9c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW0                                                     0x0aac
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW1                                                     0x0ab0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW2                                                     0x0ab4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW3                                                     0x0ab8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW4                                                     0x0abc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW5                                                     0x0ac0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW6                                                     0x0ac4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW7                                                     0x0ac8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW8                                                     0x0acc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW0                                                     0x0adc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW1                                                     0x0ae0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW2                                                     0x0ae4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW3                                                     0x0ae8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW4                                                     0x0aec
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW5                                                     0x0af0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW6                                                     0x0af4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW7                                                     0x0af8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW8                                                     0x0afc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW0                                                     0x0b0c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW1                                                     0x0b10
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW2                                                     0x0b14
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW3                                                     0x0b18
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW4                                                     0x0b1c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW5                                                     0x0b20
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW6                                                     0x0b24
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW7                                                     0x0b28
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW8                                                     0x0b2c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW0                                                     0x0b3c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW1                                                     0x0b40
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW2                                                     0x0b44
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW3                                                     0x0b48
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW4                                                     0x0b4c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW5                                                     0x0b50
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW6                                                     0x0b54
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW7                                                     0x0b58
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW8                                                     0x0b5c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW0                                                     0x0b6c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW1                                                     0x0b70
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW2                                                     0x0b74
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW3                                                     0x0b78
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW4                                                     0x0b7c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW5                                                     0x0b80
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW6                                                     0x0b84
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW7                                                     0x0b88
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW8                                                     0x0b8c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW0                                                     0x0b9c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW1                                                     0x0ba0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW2                                                     0x0ba4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW3                                                     0x0ba8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW4                                                     0x0bac
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW5                                                     0x0bb0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW6                                                     0x0bb4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW7                                                     0x0bb8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW8                                                     0x0bbc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW0                                                     0x0bcc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW1                                                     0x0bd0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW2                                                     0x0bd4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW3                                                     0x0bd8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW4                                                     0x0bdc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW5                                                     0x0be0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW6                                                     0x0be4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW7                                                     0x0be8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW8                                                     0x0bec
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW0                                                    0x0bfc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW1                                                    0x0c00
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW2                                                    0x0c04
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW3                                                    0x0c08
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW4                                                    0x0c0c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW5                                                    0x0c10
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW6                                                    0x0c14
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW7                                                    0x0c18
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW8                                                    0x0c1c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW0                                                    0x0c2c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW1                                                    0x0c30
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW2                                                    0x0c34
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW3                                                    0x0c38
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW4                                                    0x0c3c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW5                                                    0x0c40
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW6                                                    0x0c44
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW7                                                    0x0c48
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW8                                                    0x0c4c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW0                                                    0x0c5c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW1                                                    0x0c60
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW2                                                    0x0c64
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW3                                                    0x0c68
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW4                                                    0x0c6c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW5                                                    0x0c70
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW6                                                    0x0c74
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW7                                                    0x0c78
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW8                                                    0x0c7c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW0                                                    0x0c8c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW1                                                    0x0c90
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW2                                                    0x0c94
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW3                                                    0x0c98
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW4                                                    0x0c9c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW5                                                    0x0ca0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW6                                                    0x0ca4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW7                                                    0x0ca8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW8                                                    0x0cac
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW0                                                    0x0cbc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW1                                                    0x0cc0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW2                                                    0x0cc4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW3                                                    0x0cc8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW4                                                    0x0ccc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW5                                                    0x0cd0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW6                                                    0x0cd4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW7                                                    0x0cd8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW8                                                    0x0cdc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW0                                                    0x0cec
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW1                                                    0x0cf0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW2                                                    0x0cf4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW3                                                    0x0cf8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW4                                                    0x0cfc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW5                                                    0x0d00
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW6                                                    0x0d04
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW7                                                    0x0d08
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW8                                                    0x0d0c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW0                                                    0x0d1c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW1                                                    0x0d20
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW2                                                    0x0d24
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW3                                                    0x0d28
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW4                                                    0x0d2c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW5                                                    0x0d30
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW6                                                    0x0d34
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW7                                                    0x0d38
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW8                                                    0x0d3c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW0                                                    0x0d4c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW1                                                    0x0d50
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW2                                                    0x0d54
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW3                                                    0x0d58
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW4                                                    0x0d5c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW5                                                    0x0d60
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW6                                                    0x0d64
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW7                                                    0x0d68
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW8                                                    0x0d6c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW0                                                    0x0d7c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW1                                                    0x0d80
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW2                                                    0x0d84
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW3                                                    0x0d88
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW4                                                    0x0d8c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW5                                                    0x0d90
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW6                                                    0x0d94
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW7                                                    0x0d98
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW8                                                    0x0d9c
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW0                                                    0x0dac
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW1                                                    0x0db0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW2                                                    0x0db4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW3                                                    0x0db8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW4                                                    0x0dbc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW5                                                    0x0dc0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW6                                                    0x0dc4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW7                                                    0x0dc8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW8                                                    0x0dcc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE                                          0x0ddc
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE                                         0x0de0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE                                        0x0de4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE                                        0x0de8
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS                                          0x0dec
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS                                         0x0df0
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS                                        0x0df4
+#define cfgPCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS                                        0x0df8
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_bifcfgpciedecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP                                                                   0x0070
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL                                                                  0x0074
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS                                                                0x0076
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP2                                                                  0x0090
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL2                                                                 0x0094
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS2                                                               0x0096
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x0100
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR                                                   0x0104
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1                                                      0x0108
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2                                                      0x010c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST                                                0x0110
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3                                                            0x0114
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS                                                     0x0118
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL                                              0x011c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL                                              0x011e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL                                              0x0120
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL                                              0x0122
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL                                              0x0124
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL                                              0x0126
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL                                              0x0128
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL                                              0x012a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL                                              0x012c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL                                              0x012e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL                                             0x0130
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL                                             0x0132
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL                                             0x0134
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL                                             0x0136
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL                                             0x0138
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL                                             0x013a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST                                                 0x01b8
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP_16GT                                                              0x01bc
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL_16GT                                                             0x01c0
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS_16GT                                                           0x01c4
+#define cfgBIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                          0x01c8
+#define cfgBIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT                                           0x01cc
+#define cfgBIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT                                           0x01d0
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT                                              0x01d8
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT                                              0x01d9
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT                                              0x01da
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT                                              0x01db
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT                                              0x01dc
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT                                              0x01dd
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT                                              0x01de
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT                                              0x01df
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT                                              0x01e0
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT                                              0x01e1
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT                                             0x01e2
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT                                             0x01e3
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT                                             0x01e4
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT                                             0x01e5
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT                                             0x01e6
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT                                             0x01e7
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST                                                 0x01f8
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CAP_32GT                                                              0x01fc
+#define cfgBIF_CFG_DEV0_EPF0_LINK_CNTL_32GT                                                             0x0200
+#define cfgBIF_CFG_DEV0_EPF0_LINK_STATUS_32GT                                                           0x0204
+#define cfgBIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1                                                 0x0208
+#define cfgBIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2                                                 0x020c
+#define cfgBIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1                                              0x0210
+#define cfgBIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2                                              0x0214
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT                                              0x0218
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT                                              0x0219
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT                                              0x021a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT                                              0x021b
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT                                              0x021c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT                                              0x021d
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT                                              0x021e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT                                              0x021f
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT                                              0x0220
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT                                              0x0221
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT                                             0x0222
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT                                             0x0223
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT                                             0x0224
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT                                             0x0225
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT                                             0x0226
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT                                             0x0227
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST                                                       0x0238
+#define cfgBIF_CFG_DEV0_EPF0_AP_CAP                                                                     0x023c
+#define cfgBIF_CFG_DEV0_EPF0_AP_CNTL                                                                    0x0240
+#define cfgBIF_CFG_DEV0_EPF0_AP_DATA1                                                                   0x0244
+#define cfgBIF_CFG_DEV0_EPF0_AP_DATA2                                                                   0x0248
+#define cfgBIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK                                                             0x024c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST                                                0x0250
+#define cfgBIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP                                                         0x0254
+#define cfgBIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS                                                      0x0256
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL                                                 0x0258
+#define cfgBIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS                                               0x025a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL                                                 0x025c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS                                               0x025e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL                                                 0x0260
+#define cfgBIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS                                               0x0262
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL                                                 0x0264
+#define cfgBIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS                                               0x0266
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL                                                 0x0268
+#define cfgBIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS                                               0x026a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL                                                 0x026c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS                                               0x026e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL                                                 0x0270
+#define cfgBIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS                                               0x0272
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL                                                 0x0274
+#define cfgBIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS                                               0x0276
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL                                                 0x0278
+#define cfgBIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS                                               0x027a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL                                                 0x027c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS                                               0x027e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL                                                0x0280
+#define cfgBIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS                                              0x0282
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL                                                0x0284
+#define cfgBIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS                                              0x0286
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL                                                0x0288
+#define cfgBIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS                                              0x028a
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL                                                0x028c
+#define cfgBIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS                                              0x028e
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL                                                0x0290
+#define cfgBIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS                                              0x0292
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL                                                0x0294
+#define cfgBIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS                                              0x0296
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST                                                    0x0400
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST                                                       0x0404
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1                                                      0x0408
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2                                                      0x040c
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL                                                          0x0410
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS                                                        0x0412
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP                                                      0x0414
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL                                                     0x0418
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS                                                   0x041e
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP                                                      0x0420
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL                                                     0x0424
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS                                                   0x042a
+#define cfgBIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST                                                      0x0430
+#define cfgBIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP                                                      0x0434
+#define cfgBIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS                                                   0x0438
+#define cfgBIF_CFG_DEV0_EPF0_IDE_ENH_CAP_LIST                                                           0x0e00
+#define cfgBIF_CFG_DEV0_EPF0_IDE_CAP                                                                    0x0e04
+#define cfgBIF_CFG_DEV0_EPF0_IDE_CNTL                                                                   0x0e08
+#define cfgBIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL                                                     0x0e0c
+#define cfgBIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_STATUS                                                   0x0e10
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CAP                                                 0x0e14
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL                                                0x0e18
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_STATUS                                              0x0e1c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG1                                                 0x0e20
+#define cfgBIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG2                                                 0x0e24
+#define cfgBIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG1                                              0x0e28
+#define cfgBIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG2                                              0x0e2c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG3                                              0x0e30
+#define cfgBIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG1                                              0x0e34
+#define cfgBIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG2                                              0x0e38
+#define cfgBIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG3                                              0x0e3c
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CAP                                                 0x0e40
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL                                                0x0e44
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_STATUS                                              0x0e48
+#define cfgBIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG1                                                 0x0e4c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG2                                                 0x0e50
+#define cfgBIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG1                                              0x0e54
+#define cfgBIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG2                                              0x0e58
+#define cfgBIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG3                                              0x0e5c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG1                                              0x0e60
+#define cfgBIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG2                                              0x0e64
+#define cfgBIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG3                                              0x0e68
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CAP                                                 0x0e6c
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL                                                0x0e70
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_STATUS                                              0x0e74
+#define cfgBIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG1                                                 0x0e78
+#define cfgBIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG2                                                 0x0e7c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG1                                              0x0e80
+#define cfgBIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG2                                              0x0e84
+#define cfgBIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG3                                              0x0e88
+#define cfgBIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG1                                              0x0e8c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG2                                              0x0e90
+#define cfgBIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG3                                              0x0e94
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CAP                                                 0x0e98
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL                                                0x0e9c
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_STATUS                                              0x0ea0
+#define cfgBIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG1                                                 0x0ea4
+#define cfgBIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG2                                                 0x0ea8
+#define cfgBIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG1                                              0x0eac
+#define cfgBIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG2                                              0x0eb0
+#define cfgBIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG3                                              0x0eb4
+#define cfgBIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG1                                              0x0eb8
+#define cfgBIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG2                                              0x0ebc
+#define cfgBIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG3                                              0x0ec0
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CAP                                                 0x0ec4
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL                                                0x0ec8
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_STATUS                                              0x0ecc
+#define cfgBIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG1                                                 0x0ed0
+#define cfgBIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG2                                                 0x0ed4
+#define cfgBIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG1                                              0x0ed8
+#define cfgBIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG2                                              0x0edc
+#define cfgBIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG3                                              0x0ee0
+#define cfgBIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG1                                              0x0ee4
+#define cfgBIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG2                                              0x0ee8
+#define cfgBIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG3                                              0x0eec
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CAP                                                 0x0ef0
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL                                                0x0ef4
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_STATUS                                              0x0ef8
+#define cfgBIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG1                                                 0x0efc
+#define cfgBIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG2                                                 0x0f00
+#define cfgBIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG1                                              0x0f04
+#define cfgBIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG2                                              0x0f08
+#define cfgBIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG3                                              0x0f0c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG1                                              0x0f10
+#define cfgBIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG2                                              0x0f14
+#define cfgBIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG3                                              0x0f18
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CAP                                                 0x0f1c
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL                                                0x0f20
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_STATUS                                              0x0f24
+#define cfgBIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG1                                                 0x0f28
+#define cfgBIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG2                                                 0x0f2c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG1                                              0x0f30
+#define cfgBIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG2                                              0x0f34
+#define cfgBIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG3                                              0x0f38
+#define cfgBIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG1                                              0x0f3c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG2                                              0x0f40
+#define cfgBIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG3                                              0x0f44
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CAP                                                 0x0f48
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL                                                0x0f4c
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_STATUS                                              0x0f50
+#define cfgBIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG1                                                 0x0f54
+#define cfgBIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG2                                                 0x0f58
+#define cfgBIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG1                                              0x0f5c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG2                                              0x0f60
+#define cfgBIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG3                                              0x0f64
+#define cfgBIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG1                                              0x0f68
+#define cfgBIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG2                                              0x0f6c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG3                                              0x0f70
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CAP                                                 0x0f74
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL                                                0x0f78
+#define cfgBIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_STATUS                                              0x0f7c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG1                                                 0x0f80
+#define cfgBIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG2                                                 0x0f84
+#define cfgBIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG1                                              0x0f88
+#define cfgBIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG2                                              0x0f8c
+#define cfgBIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG3                                              0x0f90
+#define cfgBIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG1                                              0x0f94
+#define cfgBIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG2                                              0x0f98
+#define cfgBIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG3                                              0x0f9c
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_rc_bifcfgdecp
+// base address: 0x10100000
+#define regBIF_CFG_DEV0_RC0_VENDOR_ID                                                                   0x0000
+#define regBIF_CFG_DEV0_RC0_VENDOR_ID_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_DEVICE_ID                                                                   0x0000
+#define regBIF_CFG_DEV0_RC0_DEVICE_ID_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_COMMAND                                                                     0x0001
+#define regBIF_CFG_DEV0_RC0_COMMAND_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_STATUS                                                                      0x0001
+#define regBIF_CFG_DEV0_RC0_STATUS_BASE_IDX                                                             5
+#define regBIF_CFG_DEV0_RC0_REVISION_ID                                                                 0x0002
+#define regBIF_CFG_DEV0_RC0_REVISION_ID_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_PROG_INTERFACE                                                              0x0002
+#define regBIF_CFG_DEV0_RC0_PROG_INTERFACE_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_SUB_CLASS                                                                   0x0002
+#define regBIF_CFG_DEV0_RC0_SUB_CLASS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_BASE_CLASS                                                                  0x0002
+#define regBIF_CFG_DEV0_RC0_BASE_CLASS_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_CACHE_LINE                                                                  0x0003
+#define regBIF_CFG_DEV0_RC0_CACHE_LINE_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_LATENCY                                                                     0x0003
+#define regBIF_CFG_DEV0_RC0_LATENCY_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_HEADER                                                                      0x0003
+#define regBIF_CFG_DEV0_RC0_HEADER_BASE_IDX                                                             5
+#define regBIF_CFG_DEV0_RC0_BIST                                                                        0x0003
+#define regBIF_CFG_DEV0_RC0_BIST_BASE_IDX                                                               5
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_1                                                                 0x0004
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_1_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_2                                                                 0x0005
+#define regBIF_CFG_DEV0_RC0_BASE_ADDR_2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY                                                      0x0006
+#define regBIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT                                                               0x0007
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_SECONDARY_STATUS                                                            0x0007
+#define regBIF_CFG_DEV0_RC0_SECONDARY_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_MEM_BASE_LIMIT                                                              0x0008
+#define regBIF_CFG_DEV0_RC0_MEM_BASE_LIMIT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_LIMIT                                                             0x0009
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_LIMIT_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_UPPER                                                             0x000a
+#define regBIF_CFG_DEV0_RC0_PREF_BASE_UPPER_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER                                                            0x000b
+#define regBIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI                                                            0x000c
+#define regBIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_CAP_PTR                                                                     0x000d
+#define regBIF_CFG_DEV0_RC0_CAP_PTR_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_ROM_BASE_ADDR                                                               0x000e
+#define regBIF_CFG_DEV0_RC0_ROM_BASE_ADDR_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_LINE                                                              0x000f
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_LINE_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_PIN                                                               0x000f
+#define regBIF_CFG_DEV0_RC0_INTERRUPT_PIN_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL                                                             0x000f
+#define regBIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL                                                             0x0010
+#define regBIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PMI_CAP_LIST                                                                0x0014
+#define regBIF_CFG_DEV0_RC0_PMI_CAP_LIST_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_PMI_CAP                                                                     0x0014
+#define regBIF_CFG_DEV0_RC0_PMI_CAP_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_PMI_STATUS_CNTL                                                             0x0015
+#define regBIF_CFG_DEV0_RC0_PMI_STATUS_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP_LIST                                                               0x0016
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP                                                                    0x0016
+#define regBIF_CFG_DEV0_RC0_PCIE_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP                                                                  0x0017
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL                                                                 0x0018
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS                                                               0x0018
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_LINK_CAP                                                                    0x0019
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL                                                                   0x001a
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS                                                                 0x001a
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP                                                                    0x001b
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL                                                                   0x001c
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS                                                                 0x001c
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_ROOT_CNTL                                                                   0x001d
+#define regBIF_CFG_DEV0_RC0_ROOT_CNTL_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_ROOT_CAP                                                                    0x001d
+#define regBIF_CFG_DEV0_RC0_ROOT_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_ROOT_STATUS                                                                 0x001e
+#define regBIF_CFG_DEV0_RC0_ROOT_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP2                                                                 0x001f
+#define regBIF_CFG_DEV0_RC0_DEVICE_CAP2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL2                                                                0x0020
+#define regBIF_CFG_DEV0_RC0_DEVICE_CNTL2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS2                                                              0x0020
+#define regBIF_CFG_DEV0_RC0_DEVICE_STATUS2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_LINK_CAP2                                                                   0x0021
+#define regBIF_CFG_DEV0_RC0_LINK_CAP2_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL2                                                                  0x0022
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS2                                                                0x0022
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP2                                                                   0x0023
+#define regBIF_CFG_DEV0_RC0_SLOT_CAP2_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL2                                                                  0x0024
+#define regBIF_CFG_DEV0_RC0_SLOT_CNTL2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS2                                                                0x0024
+#define regBIF_CFG_DEV0_RC0_SLOT_STATUS2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_MSI_CAP_LIST                                                                0x0028
+#define regBIF_CFG_DEV0_RC0_MSI_CAP_LIST_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_CNTL                                                                0x0028
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO                                                             0x0029
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI                                                             0x002a
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA                                                                0x002a
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA                                                            0x002a
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA_64                                                             0x002b
+#define regBIF_CFG_DEV0_RC0_MSI_MSG_DATA_64_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64                                                         0x002b
+#define regBIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_RC0_SSID_CAP_LIST                                                               0x0030
+#define regBIF_CFG_DEV0_RC0_SSID_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_SSID_CAP                                                                    0x0031
+#define regBIF_CFG_DEV0_RC0_SSID_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                           0x0040
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR                                                    0x0041
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1                                                       0x0042
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2                                                       0x0043
+#define regBIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST                                                 0x0044
+#define regBIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3                                                             0x0045
+#define regBIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS                                                      0x0046
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL                                               0x0047
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL                                               0x0047
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL                                               0x0048
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL                                               0x0048
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL                                               0x0049
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL                                               0x0049
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL                                               0x004a
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL                                               0x004a
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL                                               0x004b
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL                                               0x004b
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL                                              0x004c
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL                                              0x004c
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL                                              0x004d
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL                                              0x004d
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL                                              0x004e
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL                                              0x004e
+#define regBIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST                                                  0x006e
+#define regBIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_16GT                                                               0x006f
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_16GT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_16GT                                                              0x0070
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_16GT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_16GT                                                            0x0071
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_16GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                           0x0072
+#define regBIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT                                            0x0073
+#define regBIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT                                            0x0074
+#define regBIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT                                               0x0076
+#define regBIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT                                               0x0076
+#define regBIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT                                               0x0076
+#define regBIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT                                               0x0076
+#define regBIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT                                               0x0077
+#define regBIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT                                               0x0077
+#define regBIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT                                               0x0077
+#define regBIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT                                               0x0077
+#define regBIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT                                               0x0078
+#define regBIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT                                               0x0078
+#define regBIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT                                              0x0078
+#define regBIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT                                              0x0078
+#define regBIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT                                              0x0079
+#define regBIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT                                              0x0079
+#define regBIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT                                              0x0079
+#define regBIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT                                              0x0079
+#define regBIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_PHY_32GT_ENH_CAP_LIST                                                  0x007e
+#define regBIF_CFG_DEV0_RC0_PCIE_PHY_32GT_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_32GT                                                               0x007f
+#define regBIF_CFG_DEV0_RC0_LINK_CAP_32GT_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_32GT                                                              0x0080
+#define regBIF_CFG_DEV0_RC0_LINK_CNTL_32GT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_32GT                                                            0x0081
+#define regBIF_CFG_DEV0_RC0_LINK_STATUS_32GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA1                                                  0x0082
+#define regBIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA2                                                  0x0083
+#define regBIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA1                                               0x0084
+#define regBIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA2                                               0x0085
+#define regBIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_32GT                                               0x0086
+#define regBIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_32GT                                               0x0086
+#define regBIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_32GT                                               0x0086
+#define regBIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_32GT                                               0x0086
+#define regBIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_32GT                                               0x0087
+#define regBIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_32GT                                               0x0087
+#define regBIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_32GT                                               0x0087
+#define regBIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_32GT                                               0x0087
+#define regBIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_32GT                                               0x0088
+#define regBIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_32GT                                               0x0088
+#define regBIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_32GT_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_32GT                                              0x0088
+#define regBIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_32GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_32GT                                              0x0088
+#define regBIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_32GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_32GT                                              0x0089
+#define regBIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_32GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_32GT                                              0x0089
+#define regBIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_32GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_32GT                                              0x0089
+#define regBIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_32GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_32GT                                              0x0089
+#define regBIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_32GT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_AP_ENH_CAP_LIST                                                        0x008e
+#define regBIF_CFG_DEV0_RC0_PCIE_AP_ENH_CAP_LIST_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_AP_CAP                                                                      0x008f
+#define regBIF_CFG_DEV0_RC0_AP_CAP_BASE_IDX                                                             5
+#define regBIF_CFG_DEV0_RC0_AP_CNTL                                                                     0x0090
+#define regBIF_CFG_DEV0_RC0_AP_CNTL_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_AP_DATA1                                                                    0x0091
+#define regBIF_CFG_DEV0_RC0_AP_DATA1_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_AP_DATA2                                                                    0x0092
+#define regBIF_CFG_DEV0_RC0_AP_DATA2_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_AP_SEL_EN_MASK                                                              0x0093
+#define regBIF_CFG_DEV0_RC0_AP_SEL_EN_MASK_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST                                                 0x0094
+#define regBIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_CAP                                                          0x0095
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_CAP_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS                                                       0x0095
+#define regBIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL                                                  0x0096
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS                                                0x0096
+#define regBIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL                                                  0x0097
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS                                                0x0097
+#define regBIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL                                                  0x0098
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS                                                0x0098
+#define regBIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL                                                  0x0099
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS                                                0x0099
+#define regBIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL                                                  0x009a
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS                                                0x009a
+#define regBIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL                                                  0x009b
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS                                                0x009b
+#define regBIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL                                                  0x009c
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS                                                0x009c
+#define regBIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL                                                  0x009d
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS                                                0x009d
+#define regBIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL                                                  0x009e
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS                                                0x009e
+#define regBIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL                                                  0x009f
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS                                                0x009f
+#define regBIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL                                                 0x00a0
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS                                               0x00a0
+#define regBIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL                                                 0x00a1
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS                                               0x00a1
+#define regBIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL                                                 0x00a2
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS                                               0x00a2
+#define regBIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL                                                 0x00a3
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS                                               0x00a3
+#define regBIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL                                                 0x00a4
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS                                               0x00a4
+#define regBIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL                                                 0x00a5
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS                                               0x00a5
+#define regBIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_NULL1_ENH_CAP_LIST                                                     0x0100
+#define regBIF_CFG_DEV0_RC0_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST                                                        0x0101
+#define regBIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1                                                       0x0102
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2                                                       0x0103
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL                                                           0x0104
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS                                                         0x0104
+#define regBIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP                                                       0x0105
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL                                                      0x0106
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS                                                    0x0107
+#define regBIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP                                                       0x0108
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL                                                      0x0109
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS                                                    0x010a
+#define regBIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST                                                       0x010c
+#define regBIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP                                                       0x010d
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS                                                    0x010e
+#define regBIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_NULL2_ENH_CAP_LIST                                                     0x0180
+#define regBIF_CFG_DEV0_RC0_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                               0x0181
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS                                                      0x0182
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK                                                        0x0183
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY                                                    0x0184
+#define regBIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS                                                        0x0185
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK                                                          0x0186
+#define regBIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL                                                       0x0187
+#define regBIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG0                                                               0x0188
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG0_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG1                                                               0x0189
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG1_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG2                                                               0x018a
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG3                                                               0x018b
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG3_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD                                                           0x018c
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS                                                        0x018d
+#define regBIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID                                                             0x018e
+#define regBIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG4                                                               0x018f
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG4_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0                                                        0x018f
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG5                                                               0x0190
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG5_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1                                                        0x0190
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG6                                                               0x0191
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG6_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2                                                        0x0191
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG7                                                               0x0192
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG7_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3                                                        0x0192
+#define regBIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG8                                                               0x0193
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG8_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG9                                                               0x0194
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG9_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG10                                                              0x0195
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG10_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG11                                                              0x0196
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG11_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG12                                                              0x0197
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG12_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG13                                                              0x0198
+#define regBIF_CFG_DEV0_RC0_PCIE_HDR_LOG13_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                            0x0199
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1                                                     0x019a
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2                                                     0x019b
+#define regBIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST                                                       0x01b7
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CAP                                                                0x01b8
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CAP_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CNTL                                                               0x01b8
+#define regBIF_CFG_DEV0_RC0_PCIE_ACS_CNTL_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST                                                       0x01c7
+#define regBIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_RC0_RTR_DATA1                                                                   0x01c8
+#define regBIF_CFG_DEV0_RC0_RTR_DATA1_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_RTR_DATA2                                                                   0x01c9
+#define regBIF_CFG_DEV0_RC0_RTR_DATA2_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_RC0_IDE_ENH_CAP_LIST                                                            0x0380
+#define regBIF_CFG_DEV0_RC0_IDE_ENH_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_RC0_IDE_CAP                                                                     0x0381
+#define regBIF_CFG_DEV0_RC0_IDE_CAP_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_RC0_IDE_CNTL                                                                    0x0382
+#define regBIF_CFG_DEV0_RC0_IDE_CNTL_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL                                                      0x0383
+#define regBIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_STATUS                                                    0x0384
+#define regBIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CAP                                                  0x0385
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL                                                 0x0386
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_STATUS                                               0x0387
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG1                                                  0x0388
+#define regBIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG2                                                  0x0389
+#define regBIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG1                                               0x038a
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG2                                               0x038b
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG3                                               0x038c
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG1                                               0x038d
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG2                                               0x038e
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG3                                               0x038f
+#define regBIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CAP                                                  0x0390
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL                                                 0x0391
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_STATUS                                               0x0392
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG1                                                  0x0393
+#define regBIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG2                                                  0x0394
+#define regBIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG1                                               0x0395
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG2                                               0x0396
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG3                                               0x0397
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG1                                               0x0398
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG2                                               0x0399
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG3                                               0x039a
+#define regBIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CAP                                                  0x039b
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL                                                 0x039c
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_STATUS                                               0x039d
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG1                                                  0x039e
+#define regBIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG2                                                  0x039f
+#define regBIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG1                                               0x03a0
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG2                                               0x03a1
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG3                                               0x03a2
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG1                                               0x03a3
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG2                                               0x03a4
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG3                                               0x03a5
+#define regBIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CAP                                                  0x03a6
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL                                                 0x03a7
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_STATUS                                               0x03a8
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG1                                                  0x03a9
+#define regBIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG2                                                  0x03aa
+#define regBIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG1                                               0x03ab
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG2                                               0x03ac
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG3                                               0x03ad
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG1                                               0x03ae
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG2                                               0x03af
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG3                                               0x03b0
+#define regBIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CAP                                                  0x03b1
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL                                                 0x03b2
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_STATUS                                               0x03b3
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG1                                                  0x03b4
+#define regBIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG2                                                  0x03b5
+#define regBIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG1                                               0x03b6
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG2                                               0x03b7
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG3                                               0x03b8
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG1                                               0x03b9
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG2                                               0x03ba
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG3                                               0x03bb
+#define regBIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CAP                                                  0x03bc
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL                                                 0x03bd
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_STATUS                                               0x03be
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG1                                                  0x03bf
+#define regBIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG2                                                  0x03c0
+#define regBIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG1                                               0x03c1
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG2                                               0x03c2
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG3                                               0x03c3
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG1                                               0x03c4
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG2                                               0x03c5
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG3                                               0x03c6
+#define regBIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CAP                                                  0x03c7
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL                                                 0x03c8
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_STATUS                                               0x03c9
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG1                                                  0x03ca
+#define regBIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG2                                                  0x03cb
+#define regBIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG1                                               0x03cc
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG2                                               0x03cd
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG3                                               0x03ce
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG1                                               0x03cf
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG2                                               0x03d0
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG3                                               0x03d1
+#define regBIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CAP                                                  0x03d2
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL                                                 0x03d3
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_STATUS                                               0x03d4
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG1                                                  0x03d5
+#define regBIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG2                                                  0x03d6
+#define regBIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG1                                               0x03d7
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG2                                               0x03d8
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG3                                               0x03d9
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG1                                               0x03da
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG2                                               0x03db
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG3                                               0x03dc
+#define regBIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CAP                                                  0x03dd
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL                                                 0x03de
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_STATUS                                               0x03df
+#define regBIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_STATUS_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG1                                                  0x03e0
+#define regBIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG2                                                  0x03e1
+#define regBIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG1                                               0x03e2
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG2                                               0x03e3
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG3                                               0x03e4
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG1                                               0x03e5
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG2                                               0x03e6
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG3                                               0x03e7
+#define regBIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                      5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_ID                                                                0x10000
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_ID                                                                0x10000
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_COMMAND                                                                  0x10001
+#define regBIF_CFG_DEV0_EPF0_0_COMMAND_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_STATUS                                                                   0x10001
+#define regBIF_CFG_DEV0_EPF0_0_STATUS_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_0_REVISION_ID                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_0_REVISION_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_PROG_INTERFACE                                                           0x10002
+#define regBIF_CFG_DEV0_EPF0_0_PROG_INTERFACE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_SUB_CLASS                                                                0x10002
+#define regBIF_CFG_DEV0_EPF0_0_SUB_CLASS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_CLASS                                                               0x10002
+#define regBIF_CFG_DEV0_EPF0_0_BASE_CLASS_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_CACHE_LINE                                                               0x10003
+#define regBIF_CFG_DEV0_EPF0_0_CACHE_LINE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LATENCY                                                                  0x10003
+#define regBIF_CFG_DEV0_EPF0_0_LATENCY_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_HEADER                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_0_HEADER_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_0_BIST                                                                     0x10003
+#define regBIF_CFG_DEV0_EPF0_0_BIST_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_1                                                              0x10004
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_2                                                              0x10005
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_3                                                              0x10006
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_4                                                              0x10007
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_5                                                              0x10008
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_6                                                              0x10009
+#define regBIF_CFG_DEV0_EPF0_0_BASE_ADDR_6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR                                                          0x1000a
+#define regBIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID                                                               0x1000b
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR                                                            0x1000c
+#define regBIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_CAP_PTR                                                                  0x1000d
+#define regBIF_CFG_DEV0_EPF0_0_CAP_PTR_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE                                                           0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_MIN_GRANT                                                                0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_MIN_GRANT_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_MAX_LATENCY                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_0_MAX_LATENCY_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST                                                          0x10012
+#define regBIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W                                                             0x10013
+#define regBIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST                                                             0x10014
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP                                                                  0x10014
+#define regBIF_CFG_DEV0_EPF0_0_PMI_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL                                                          0x10015
+#define regBIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST                                                            0x10019
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP                                                                 0x10019
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP                                                               0x1001a
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL                                                              0x1001b
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP2                                                              0x10022
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2                                                             0x10023
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST                                                             0x10028
+#define regBIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL                                                             0x10028
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO                                                          0x10029
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI                                                          0x1002a
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA                                                             0x1002a
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA                                                         0x1002a
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK                                                                 0x1002b
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64                                                          0x1002b
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64                                                      0x1002b
+#define regBIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK_64                                                              0x1002c
+#define regBIF_CFG_DEV0_EPF0_0_MSI_MASK_64_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING                                                              0x1002c
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING_64                                                           0x1002d
+#define regBIF_CFG_DEV0_EPF0_0_MSI_PENDING_64_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST                                                            0x10030
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL                                                            0x10030
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_TABLE                                                               0x10031
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_TABLE_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_PBA                                                                 0x10032
+#define regBIF_CFG_DEV0_EPF0_0_MSIX_PBA_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_NULL2_ENH_CAP_LIST                                                  0x10180
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                            0x10181
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS                                                   0x10182
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK                                                     0x10183
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY                                                 0x10184
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS                                                     0x10185
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK                                                       0x10186
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL                                                    0x10187
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0                                                            0x10188
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1                                                            0x10189
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2                                                            0x1018a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3                                                            0x1018b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG4                                                            0x1018f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0                                                     0x1018f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG5                                                            0x10190
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1                                                     0x10190
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG6                                                            0x10191
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2                                                     0x10191
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG7                                                            0x10192
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG7_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3                                                     0x10192
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG8                                                            0x10193
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG8_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG9                                                            0x10194
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG9_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG10                                                           0x10195
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG10_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG11                                                           0x10196
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG11_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG12                                                           0x10197
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG12_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG13                                                           0x10198
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG13_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                         0x10199
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST_BASE_IDX                                5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1                                                  0x1019a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2                                                  0x1019b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST                                                    0x101b7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP                                                             0x101b8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL                                                            0x101b8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST                                                    0x101c7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_RTR_DATA1                                                                0x101c8
+#define regBIF_CFG_DEV0_EPF0_0_RTR_DATA1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_RTR_DATA2                                                                0x101c9
+#define regBIF_CFG_DEV0_EPF0_0_RTR_DATA2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST                                             0x101ca
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT                                              0x101cb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA                                                     0x101cc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP                                                      0x101cd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST                                                    0x101ce
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP                                                             0x101cf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR                                               0x101d0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS                                                          0x101d1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL                                                            0x101d1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                            0x101d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                            0x101d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                            0x101d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                            0x101d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                            0x101d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                            0x101d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                            0x101d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                            0x101d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST                                                    0x101da
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP                                                             0x101db
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST                                                    0x101dc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP                                                             0x101dd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL                                                            0x101dd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST                                                    0x101de
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP                                                            0x101df
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL                                                           0x101e0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP                                                            0x101e1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL                                                           0x101e2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP                                                            0x101e3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL                                                           0x101e4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP                                                            0x101e5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL                                                           0x101e6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP                                                            0x101e7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL                                                           0x101e8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP                                                            0x101e9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL                                                           0x101ea
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST                                                    0x101eb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP                                                             0x101ec
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL                                                            0x101ec
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST                                               0x101ed
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL                                                       0x101ee
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS                                                     0x101ee
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY                                          0x101ef
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC                                             0x101f0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST                                                  0x101f1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP                                                           0x101f2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL                                                          0x101f2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST                                                  0x101f3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP                                                           0x101f4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL                                                       0x101f5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS                                                        0x101f5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS                                                   0x101f6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS                                                     0x101f6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS                                                       0x101f7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK                                                 0x101f7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET                                               0x101f8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE                                                     0x101f8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID                                                  0x101f9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE                                           0x101fa
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE                                              0x101fb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0                                                0x101fc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1                                                0x101fd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2                                                0x101fe
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3                                                0x101ff
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4                                                0x10200
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5                                                0x10201
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET                               0x10202
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_BASE_IDX                      5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST                                          0x10238
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP                                                  0x10239
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL                                                 0x1023a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP                                                  0x1023b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL                                                 0x1023c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP                                                  0x1023d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL                                                 0x1023e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP                                                  0x1023f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL                                                 0x10240
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP                                                  0x10241
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL                                                 0x10242
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP                                                  0x10243
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL                                                 0x10244
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_ENH_CAP_LIST                                                    0x10245
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP                                                             0x10246
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL                                                            0x10247
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS                                                             0x10248
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_WMBOX                                                           0x10249
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_WMBOX_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_RMBOX                                                           0x1024a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DOE_RMBOX_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV                                 0x1026b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV                                          0x1026c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW                             0x1026d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW_BASE_IDX                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE                              0x1026e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE_BASE_IDX                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS                              0x1026f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS_BASE_IDX                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL                            0x10270
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL_BASE_IDX                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0                            0x10271
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0_BASE_IDX                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1                            0x10272
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1_BASE_IDX                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2                            0x10273
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2_BASE_IDX                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB                                 0x10275
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION                                   0x10277
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION_BASE_IDX                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE                     0x10278
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE_BASE_IDX            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0                                 0x10281
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1                                 0x10282
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2                                 0x10283
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3                                 0x10284
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4                                 0x10285
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW0                                 0x10287
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW1                                 0x10288
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW2                                 0x10289
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW3                                 0x1028a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW4                                 0x1028b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW5                                 0x1028c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW6                                 0x1028d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW7                                 0x1028e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW8                                 0x1028f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW0                                 0x10293
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW1                                 0x10294
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW2                                 0x10295
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW3                                 0x10296
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW4                                 0x10297
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW5                                 0x10298
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW6                                 0x10299
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW7                                 0x1029a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW8                                 0x1029b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW0                                 0x1029f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW1                                 0x102a0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW2                                 0x102a1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW3                                 0x102a2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW4                                 0x102a3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW5                                 0x102a4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW6                                 0x102a5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW7                                 0x102a6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW8                                 0x102a7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW0                                 0x102ab
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW1                                 0x102ac
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW2                                 0x102ad
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW3                                 0x102ae
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW4                                 0x102af
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW5                                 0x102b0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW6                                 0x102b1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW7                                 0x102b2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW8                                 0x102b3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW0                                 0x102b7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW1                                 0x102b8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW2                                 0x102b9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW3                                 0x102ba
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW4                                 0x102bb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW5                                 0x102bc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW6                                 0x102bd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW7                                 0x102be
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW8                                 0x102bf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW0                                 0x102c3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW1                                 0x102c4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW2                                 0x102c5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW3                                 0x102c6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW4                                 0x102c7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW5                                 0x102c8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW6                                 0x102c9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW7                                 0x102ca
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW8                                 0x102cb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW0                                 0x102cf
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW1                                 0x102d0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW2                                 0x102d1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW3                                 0x102d2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW4                                 0x102d3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW5                                 0x102d4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW6                                 0x102d5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW7                                 0x102d6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW8                                 0x102d7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW0                                 0x102db
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW1                                 0x102dc
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW2                                 0x102dd
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW3                                 0x102de
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW4                                 0x102df
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW5                                 0x102e0
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW6                                 0x102e1
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW7                                 0x102e2
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW8                                 0x102e3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW0                                 0x102e7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW1                                 0x102e8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW2                                 0x102e9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW3                                 0x102ea
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW4                                 0x102eb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW5                                 0x102ec
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW6                                 0x102ed
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW7                                 0x102ee
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW8                                 0x102ef
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW0                                 0x102f3
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW0_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW1                                 0x102f4
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW1_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW2                                 0x102f5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW2_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW3                                 0x102f6
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW3_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW4                                 0x102f7
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW4_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW5                                 0x102f8
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW5_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW6                                 0x102f9
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW6_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW7                                 0x102fa
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW7_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW8                                 0x102fb
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW8_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW0                                0x102ff
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW1                                0x10300
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW2                                0x10301
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW3                                0x10302
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW4                                0x10303
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW5                                0x10304
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW6                                0x10305
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW7                                0x10306
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW8                                0x10307
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW0                                0x1030b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW1                                0x1030c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW2                                0x1030d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW3                                0x1030e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW4                                0x1030f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW5                                0x10310
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW6                                0x10311
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW7                                0x10312
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW8                                0x10313
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW0                                0x10317
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW1                                0x10318
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW2                                0x10319
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW3                                0x1031a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW4                                0x1031b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW5                                0x1031c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW6                                0x1031d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW7                                0x1031e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW8                                0x1031f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW0                                0x10323
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW1                                0x10324
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW2                                0x10325
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW3                                0x10326
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW4                                0x10327
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW5                                0x10328
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW6                                0x10329
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW7                                0x1032a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW8                                0x1032b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW0                                0x1032f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW1                                0x10330
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW2                                0x10331
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW3                                0x10332
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW4                                0x10333
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW5                                0x10334
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW6                                0x10335
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW7                                0x10336
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW8                                0x10337
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW0                                0x1033b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW1                                0x1033c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW2                                0x1033d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW3                                0x1033e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW4                                0x1033f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW5                                0x10340
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW6                                0x10341
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW7                                0x10342
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW8                                0x10343
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW0                                0x10347
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW1                                0x10348
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW2                                0x10349
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW3                                0x1034a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW4                                0x1034b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW5                                0x1034c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW6                                0x1034d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW7                                0x1034e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW8                                0x1034f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW0                                0x10353
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW1                                0x10354
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW2                                0x10355
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW3                                0x10356
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW4                                0x10357
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW5                                0x10358
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW6                                0x10359
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW7                                0x1035a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW8                                0x1035b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW0                                0x1035f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW1                                0x10360
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW2                                0x10361
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW3                                0x10362
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW4                                0x10363
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW5                                0x10364
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW6                                0x10365
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW7                                0x10366
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW8                                0x10367
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW0                                0x1036b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW0_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW1                                0x1036c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW1_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW2                                0x1036d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW2_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW3                                0x1036e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW3_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW4                                0x1036f
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW4_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW5                                0x10370
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW5_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW6                                0x10371
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW6_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW7                                0x10372
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW7_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW8                                0x10373
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW8_BASE_IDX                       5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE                      0x10377
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE_BASE_IDX             5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE                     0x10378
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE_BASE_IDX            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE                    0x10379
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE_BASE_IDX           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE                    0x1037a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE_BASE_IDX           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS                      0x1037b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS_BASE_IDX             5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS                     0x1037c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS_BASE_IDX            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS                    0x1037d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS_BASE_IDX           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS                    0x1037e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS_BASE_IDX           5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_bifcfgpciedecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP                                                                 0x1001c
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL                                                                0x1001d
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP2                                                                0x10024
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL2                                                               0x10025
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                        0x10040
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR                                                 0x10041
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1                                                    0x10042
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2                                                    0x10043
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST                                              0x10044
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3                                                          0x10045
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS                                                   0x10046
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL                                            0x10047
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL                                            0x10047
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL                                            0x10048
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL                                            0x10048
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL                                            0x10049
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL                                            0x10049
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL                                            0x1004a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL                                            0x1004a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL                                            0x1004b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL                                            0x1004b
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL                                           0x1004c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL                                           0x1004c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL                                           0x1004d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL                                           0x1004d
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL                                           0x1004e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL                                           0x1004e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST                                               0x1006e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT                                                            0x1006f
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT                                                           0x10070
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT                                                         0x10071
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT                                        0x10072
+#define regBIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                               5
+#define regBIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT                                         0x10073
+#define regBIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                5
+#define regBIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT                                         0x10074
+#define regBIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT_BASE_IDX                                5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT                                            0x10076
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT                                            0x10076
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT                                            0x10076
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT                                            0x10076
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT                                            0x10077
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT                                            0x10077
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT                                            0x10077
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT                                            0x10077
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT                                            0x10078
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT                                            0x10078
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT                                           0x10078
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT                                           0x10078
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT                                           0x10079
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT                                           0x10079
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT                                           0x10079
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT                                           0x10079
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PHY_32GT_ENH_CAP_LIST                                               0x1007e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PHY_32GT_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT                                                            0x1007f
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT                                                           0x10080
+#define regBIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT                                                         0x10081
+#define regBIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA1                                               0x10082
+#define regBIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA2                                               0x10083
+#define regBIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA1                                            0x10084
+#define regBIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA2                                            0x10085
+#define regBIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_32GT                                            0x10086
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_32GT                                            0x10086
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_32GT                                            0x10086
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_32GT                                            0x10086
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_32GT                                            0x10087
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_32GT                                            0x10087
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_32GT                                            0x10087
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_32GT                                            0x10087
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_32GT                                            0x10088
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_32GT                                            0x10088
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_32GT_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_32GT                                           0x10088
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_32GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_32GT                                           0x10088
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_32GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_32GT                                           0x10089
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_32GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_32GT                                           0x10089
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_32GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_32GT                                           0x10089
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_32GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_32GT                                           0x10089
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_32GT_BASE_IDX                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_AP_ENH_CAP_LIST                                                     0x1008e
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_AP_ENH_CAP_LIST_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_AP_CAP                                                                   0x1008f
+#define regBIF_CFG_DEV0_EPF0_0_AP_CAP_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_0_AP_CNTL                                                                  0x10090
+#define regBIF_CFG_DEV0_EPF0_0_AP_CNTL_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_AP_DATA1                                                                 0x10091
+#define regBIF_CFG_DEV0_EPF0_0_AP_DATA1_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_AP_DATA2                                                                 0x10092
+#define regBIF_CFG_DEV0_EPF0_0_AP_DATA2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_AP_SEL_EN_MASK                                                           0x10093
+#define regBIF_CFG_DEV0_EPF0_0_AP_SEL_EN_MASK_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST                                              0x10094
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP                                                       0x10095
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS                                                    0x10095
+#define regBIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL                                               0x10096
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS                                             0x10096
+#define regBIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL                                               0x10097
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS                                             0x10097
+#define regBIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL                                               0x10098
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS                                             0x10098
+#define regBIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL                                               0x10099
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS                                             0x10099
+#define regBIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL                                               0x1009a
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS                                             0x1009a
+#define regBIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL                                               0x1009b
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS                                             0x1009b
+#define regBIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL                                               0x1009c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS                                             0x1009c
+#define regBIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL                                               0x1009d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS                                             0x1009d
+#define regBIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL                                               0x1009e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS                                             0x1009e
+#define regBIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL                                               0x1009f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS                                             0x1009f
+#define regBIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL                                              0x100a0
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS                                            0x100a0
+#define regBIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL                                              0x100a1
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS                                            0x100a1
+#define regBIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL                                              0x100a2
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS                                            0x100a2
+#define regBIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL                                              0x100a3
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS                                            0x100a3
+#define regBIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL                                              0x100a4
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS                                            0x100a4
+#define regBIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL                                              0x100a5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS                                            0x100a5
+#define regBIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_NULL1_ENH_CAP_LIST                                                  0x10100
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST                                                     0x10101
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1                                                    0x10102
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2                                                    0x10103
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL                                                        0x10104
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS                                                      0x10104
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP                                                    0x10105
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL                                                   0x10106
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS                                                 0x10107
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP                                                    0x10108
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL                                                   0x10109
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS                                                 0x1010a
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST                                                    0x1010c
+#define regBIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP                                                    0x1010d
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS                                                 0x1010e
+#define regBIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_ENH_CAP_LIST                                                         0x10380
+#define regBIF_CFG_DEV0_EPF0_0_IDE_ENH_CAP_LIST_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_CAP                                                                  0x10381
+#define regBIF_CFG_DEV0_EPF0_0_IDE_CAP_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_CNTL                                                                 0x10382
+#define regBIF_CFG_DEV0_EPF0_0_IDE_CNTL_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL                                                   0x10383
+#define regBIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_STATUS                                                 0x10384
+#define regBIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CAP                                               0x10385
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CAP_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL                                              0x10386
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_STATUS                                            0x10387
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG1                                               0x10388
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG2                                               0x10389
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG1                                            0x1038a
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG2                                            0x1038b
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG3                                            0x1038c
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG1                                            0x1038d
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG2                                            0x1038e
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG3                                            0x1038f
+#define regBIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CAP                                               0x10390
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CAP_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL                                              0x10391
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_STATUS                                            0x10392
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG1                                               0x10393
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG2                                               0x10394
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG1                                            0x10395
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG2                                            0x10396
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG3                                            0x10397
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG1                                            0x10398
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG2                                            0x10399
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG3                                            0x1039a
+#define regBIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CAP                                               0x1039b
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CAP_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL                                              0x1039c
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_STATUS                                            0x1039d
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG1                                               0x1039e
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG2                                               0x1039f
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG1                                            0x103a0
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG2                                            0x103a1
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG3                                            0x103a2
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG1                                            0x103a3
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG2                                            0x103a4
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG3                                            0x103a5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CAP                                               0x103a6
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CAP_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL                                              0x103a7
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_STATUS                                            0x103a8
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG1                                               0x103a9
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG2                                               0x103aa
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG1                                            0x103ab
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG2                                            0x103ac
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG3                                            0x103ad
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG1                                            0x103ae
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG2                                            0x103af
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG3                                            0x103b0
+#define regBIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CAP                                               0x103b1
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CAP_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL                                              0x103b2
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_STATUS                                            0x103b3
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG1                                               0x103b4
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG2                                               0x103b5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG1                                            0x103b6
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG2                                            0x103b7
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG3                                            0x103b8
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG1                                            0x103b9
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG2                                            0x103ba
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG3                                            0x103bb
+#define regBIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CAP                                               0x103bc
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CAP_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL                                              0x103bd
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_STATUS                                            0x103be
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG1                                               0x103bf
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG2                                               0x103c0
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG1                                            0x103c1
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG2                                            0x103c2
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG3                                            0x103c3
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG1                                            0x103c4
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG2                                            0x103c5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG3                                            0x103c6
+#define regBIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CAP                                               0x103c7
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CAP_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL                                              0x103c8
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_STATUS                                            0x103c9
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG1                                               0x103ca
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG2                                               0x103cb
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG1                                            0x103cc
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG2                                            0x103cd
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG3                                            0x103ce
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG1                                            0x103cf
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG2                                            0x103d0
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG3                                            0x103d1
+#define regBIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CAP                                               0x103d2
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CAP_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL                                              0x103d3
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_STATUS                                            0x103d4
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG1                                               0x103d5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG2                                               0x103d6
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG1                                            0x103d7
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG2                                            0x103d8
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG3                                            0x103d9
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG1                                            0x103da
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG2                                            0x103db
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG3                                            0x103dc
+#define regBIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CAP                                               0x103dd
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CAP_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL                                              0x103de
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_STATUS                                            0x103df
+#define regBIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_STATUS_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG1                                               0x103e0
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG1_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG2                                               0x103e1
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG2_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG1                                            0x103e2
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG2                                            0x103e3
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG3                                            0x103e4
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG3_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG1                                            0x103e5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG1_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG2                                            0x103e6
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG2_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG3                                            0x103e7
+#define regBIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG3_BASE_IDX                                   5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf0_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_VF0_VENDOR_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF0_VENDOR_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF0_COMMAND                                                                0x10001
+#define regBIF_CFG_DEV0_EPF0_VF0_COMMAND_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF0_STATUS                                                                 0x10001
+#define regBIF_CFG_DEV0_EPF0_VF0_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF0_REVISION_ID                                                            0x10002
+#define regBIF_CFG_DEV0_EPF0_VF0_REVISION_ID_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE                                                         0x10002
+#define regBIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF0_SUB_CLASS                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_VF0_SUB_CLASS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_CLASS                                                             0x10002
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_CLASS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF0_CACHE_LINE                                                             0x10003
+#define regBIF_CFG_DEV0_EPF0_VF0_CACHE_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF0_LATENCY                                                                0x10003
+#define regBIF_CFG_DEV0_EPF0_VF0_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF0_HEADER                                                                 0x10003
+#define regBIF_CFG_DEV0_EPF0_VF0_HEADER_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF0_BIST                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_VF0_BIST_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1                                                            0x10004
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2                                                            0x10005
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3                                                            0x10006
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4                                                            0x10007
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5                                                            0x10008
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6                                                            0x10009
+#define regBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR                                                        0x1000a
+#define regBIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID                                                             0x1000b
+#define regBIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR                                                          0x1000c
+#define regBIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_CAP_PTR                                                                0x1000d
+#define regBIF_CFG_DEV0_EPF0_VF0_CAP_PTR_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE                                                         0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN                                                          0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_MIN_GRANT                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF0_MIN_GRANT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST                                                          0x10019
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_CAP                                                               0x10019
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP                                                             0x1001a
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS                                                          0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_CAP                                                               0x1001c
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_CNTL                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_STATUS                                                            0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2                                                            0x10022
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2                                                         0x10023
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_CAP2                                                              0x10024
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2                                                           0x10025
+#define regBIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO                                                        0x10029
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI                                                        0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA                                                           0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA                                                       0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MASK                                                               0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MASK_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64                                                        0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64                                                    0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_PENDING                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64                                                         0x1002d
+#define regBIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE                                                             0x10031
+#define regBIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF0_MSIX_PBA                                                               0x10032
+#define regBIF_CFG_DEV0_EPF0_VF0_MSIX_PBA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x10040
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                             5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR                                               0x10041
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1                                                  0x10042
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2                                                  0x10043
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST                                              0x100b6
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3                                                            0x100b7
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3                                                           0x100b8
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3                                                         0x100b9
+#define regBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST                                                0x10100
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST                                                0x10180
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x10181
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS                                                 0x10182
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK                                                   0x10183
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY                                               0x10184
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS                                                   0x10185
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK                                                     0x10186
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL                                                  0x10187
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0                                                          0x10188
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1                                                          0x10189
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2                                                          0x1018a
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3                                                          0x1018b
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG4                                                          0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG4_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0                                                   0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG5                                                          0x10190
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG5_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1                                                   0x10190
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG6                                                          0x10191
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG6_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2                                                   0x10191
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG7                                                          0x10192
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG7_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3                                                   0x10192
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG8                                                          0x10193
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG8_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG9                                                          0x10194
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG9_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG10                                                         0x10195
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG10_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG11                                                         0x10196
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG11_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG12                                                         0x10197
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG12_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG13                                                         0x10198
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG13_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST                                                  0x101c7
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF0_RTR_DATA1                                                              0x101c8
+#define regBIF_CFG_DEV0_EPF0_VF0_RTR_DATA1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF0_RTR_DATA2                                                              0x101c9
+#define regBIF_CFG_DEV0_EPF0_VF0_RTR_DATA2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST                                                  0x101dc
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP                                                           0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL                                                          0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST                                                  0x101eb
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP                                                           0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL                                                          0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf1_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_VF1_VENDOR_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF1_VENDOR_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF1_COMMAND                                                                0x10001
+#define regBIF_CFG_DEV0_EPF0_VF1_COMMAND_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF1_STATUS                                                                 0x10001
+#define regBIF_CFG_DEV0_EPF0_VF1_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF1_REVISION_ID                                                            0x10002
+#define regBIF_CFG_DEV0_EPF0_VF1_REVISION_ID_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE                                                         0x10002
+#define regBIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF1_SUB_CLASS                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_VF1_SUB_CLASS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_CLASS                                                             0x10002
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_CLASS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF1_CACHE_LINE                                                             0x10003
+#define regBIF_CFG_DEV0_EPF0_VF1_CACHE_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF1_LATENCY                                                                0x10003
+#define regBIF_CFG_DEV0_EPF0_VF1_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF1_HEADER                                                                 0x10003
+#define regBIF_CFG_DEV0_EPF0_VF1_HEADER_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF1_BIST                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_VF1_BIST_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1                                                            0x10004
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2                                                            0x10005
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3                                                            0x10006
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4                                                            0x10007
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5                                                            0x10008
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6                                                            0x10009
+#define regBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR                                                        0x1000a
+#define regBIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID                                                             0x1000b
+#define regBIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR                                                          0x1000c
+#define regBIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_CAP_PTR                                                                0x1000d
+#define regBIF_CFG_DEV0_EPF0_VF1_CAP_PTR_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE                                                         0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN                                                          0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_MIN_GRANT                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF1_MIN_GRANT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST                                                          0x10019
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_CAP                                                               0x10019
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP                                                             0x1001a
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS                                                          0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_CAP                                                               0x1001c
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_CNTL                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_STATUS                                                            0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2                                                            0x10022
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2                                                         0x10023
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_CAP2                                                              0x10024
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2                                                           0x10025
+#define regBIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO                                                        0x10029
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI                                                        0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA                                                           0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA                                                       0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MASK                                                               0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MASK_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64                                                        0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64                                                    0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_PENDING                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64                                                         0x1002d
+#define regBIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE                                                             0x10031
+#define regBIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF1_MSIX_PBA                                                               0x10032
+#define regBIF_CFG_DEV0_EPF0_VF1_MSIX_PBA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x10040
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                             5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR                                               0x10041
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1                                                  0x10042
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2                                                  0x10043
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST                                              0x100b6
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3                                                            0x100b7
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3                                                           0x100b8
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3                                                         0x100b9
+#define regBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST                                                0x10100
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST                                                0x10180
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x10181
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS                                                 0x10182
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK                                                   0x10183
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY                                               0x10184
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS                                                   0x10185
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK                                                     0x10186
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL                                                  0x10187
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0                                                          0x10188
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1                                                          0x10189
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2                                                          0x1018a
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3                                                          0x1018b
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG4                                                          0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG4_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0                                                   0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG5                                                          0x10190
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG5_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1                                                   0x10190
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG6                                                          0x10191
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG6_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2                                                   0x10191
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG7                                                          0x10192
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG7_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3                                                   0x10192
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG8                                                          0x10193
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG8_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG9                                                          0x10194
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG9_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG10                                                         0x10195
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG10_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG11                                                         0x10196
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG11_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG12                                                         0x10197
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG12_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG13                                                         0x10198
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG13_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST                                                  0x101c7
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF1_RTR_DATA1                                                              0x101c8
+#define regBIF_CFG_DEV0_EPF0_VF1_RTR_DATA1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF1_RTR_DATA2                                                              0x101c9
+#define regBIF_CFG_DEV0_EPF0_VF1_RTR_DATA2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST                                                  0x101dc
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP                                                           0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL                                                          0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST                                                  0x101eb
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP                                                           0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL                                                          0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf2_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_VF2_VENDOR_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF2_VENDOR_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF2_COMMAND                                                                0x10001
+#define regBIF_CFG_DEV0_EPF0_VF2_COMMAND_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF2_STATUS                                                                 0x10001
+#define regBIF_CFG_DEV0_EPF0_VF2_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF2_REVISION_ID                                                            0x10002
+#define regBIF_CFG_DEV0_EPF0_VF2_REVISION_ID_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE                                                         0x10002
+#define regBIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF2_SUB_CLASS                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_VF2_SUB_CLASS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_CLASS                                                             0x10002
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_CLASS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF2_CACHE_LINE                                                             0x10003
+#define regBIF_CFG_DEV0_EPF0_VF2_CACHE_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF2_LATENCY                                                                0x10003
+#define regBIF_CFG_DEV0_EPF0_VF2_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF2_HEADER                                                                 0x10003
+#define regBIF_CFG_DEV0_EPF0_VF2_HEADER_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF2_BIST                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_VF2_BIST_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1                                                            0x10004
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2                                                            0x10005
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3                                                            0x10006
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4                                                            0x10007
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5                                                            0x10008
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6                                                            0x10009
+#define regBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR                                                        0x1000a
+#define regBIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID                                                             0x1000b
+#define regBIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR                                                          0x1000c
+#define regBIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_CAP_PTR                                                                0x1000d
+#define regBIF_CFG_DEV0_EPF0_VF2_CAP_PTR_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE                                                         0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN                                                          0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_MIN_GRANT                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF2_MIN_GRANT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST                                                          0x10019
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_CAP                                                               0x10019
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP                                                             0x1001a
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS                                                          0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_CAP                                                               0x1001c
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_CNTL                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_STATUS                                                            0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2                                                            0x10022
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2                                                         0x10023
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_CAP2                                                              0x10024
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2                                                           0x10025
+#define regBIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO                                                        0x10029
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI                                                        0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA                                                           0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA                                                       0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MASK                                                               0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MASK_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64                                                        0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64                                                    0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_PENDING                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64                                                         0x1002d
+#define regBIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE                                                             0x10031
+#define regBIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF2_MSIX_PBA                                                               0x10032
+#define regBIF_CFG_DEV0_EPF0_VF2_MSIX_PBA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x10040
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                             5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR                                               0x10041
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1                                                  0x10042
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2                                                  0x10043
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST                                              0x100b6
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3                                                            0x100b7
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3                                                           0x100b8
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3                                                         0x100b9
+#define regBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST                                                0x10100
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST                                                0x10180
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x10181
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS                                                 0x10182
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK                                                   0x10183
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY                                               0x10184
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS                                                   0x10185
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK                                                     0x10186
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL                                                  0x10187
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0                                                          0x10188
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1                                                          0x10189
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2                                                          0x1018a
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3                                                          0x1018b
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG4                                                          0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG4_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0                                                   0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG5                                                          0x10190
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG5_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1                                                   0x10190
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG6                                                          0x10191
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG6_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2                                                   0x10191
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG7                                                          0x10192
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG7_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3                                                   0x10192
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG8                                                          0x10193
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG8_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG9                                                          0x10194
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG9_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG10                                                         0x10195
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG10_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG11                                                         0x10196
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG11_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG12                                                         0x10197
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG12_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG13                                                         0x10198
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG13_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST                                                  0x101c7
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF2_RTR_DATA1                                                              0x101c8
+#define regBIF_CFG_DEV0_EPF0_VF2_RTR_DATA1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF2_RTR_DATA2                                                              0x101c9
+#define regBIF_CFG_DEV0_EPF0_VF2_RTR_DATA2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST                                                  0x101dc
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP                                                           0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL                                                          0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST                                                  0x101eb
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP                                                           0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL                                                          0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf3_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_VF3_VENDOR_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF3_VENDOR_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF3_COMMAND                                                                0x10001
+#define regBIF_CFG_DEV0_EPF0_VF3_COMMAND_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF3_STATUS                                                                 0x10001
+#define regBIF_CFG_DEV0_EPF0_VF3_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF3_REVISION_ID                                                            0x10002
+#define regBIF_CFG_DEV0_EPF0_VF3_REVISION_ID_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE                                                         0x10002
+#define regBIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF3_SUB_CLASS                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_VF3_SUB_CLASS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_CLASS                                                             0x10002
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_CLASS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF3_CACHE_LINE                                                             0x10003
+#define regBIF_CFG_DEV0_EPF0_VF3_CACHE_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF3_LATENCY                                                                0x10003
+#define regBIF_CFG_DEV0_EPF0_VF3_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF3_HEADER                                                                 0x10003
+#define regBIF_CFG_DEV0_EPF0_VF3_HEADER_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF3_BIST                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_VF3_BIST_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1                                                            0x10004
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2                                                            0x10005
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3                                                            0x10006
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4                                                            0x10007
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5                                                            0x10008
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6                                                            0x10009
+#define regBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR                                                        0x1000a
+#define regBIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID                                                             0x1000b
+#define regBIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR                                                          0x1000c
+#define regBIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_CAP_PTR                                                                0x1000d
+#define regBIF_CFG_DEV0_EPF0_VF3_CAP_PTR_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE                                                         0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN                                                          0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_MIN_GRANT                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF3_MIN_GRANT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST                                                          0x10019
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_CAP                                                               0x10019
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP                                                             0x1001a
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS                                                          0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_CAP                                                               0x1001c
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_CNTL                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_STATUS                                                            0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2                                                            0x10022
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2                                                         0x10023
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_CAP2                                                              0x10024
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2                                                           0x10025
+#define regBIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO                                                        0x10029
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI                                                        0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA                                                           0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA                                                       0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MASK                                                               0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MASK_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64                                                        0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64                                                    0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_PENDING                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64                                                         0x1002d
+#define regBIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE                                                             0x10031
+#define regBIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF3_MSIX_PBA                                                               0x10032
+#define regBIF_CFG_DEV0_EPF0_VF3_MSIX_PBA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x10040
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                             5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR                                               0x10041
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1                                                  0x10042
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2                                                  0x10043
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST                                              0x100b6
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3                                                            0x100b7
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3                                                           0x100b8
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3                                                         0x100b9
+#define regBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST                                                0x10100
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST                                                0x10180
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x10181
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS                                                 0x10182
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK                                                   0x10183
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY                                               0x10184
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS                                                   0x10185
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK                                                     0x10186
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL                                                  0x10187
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0                                                          0x10188
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1                                                          0x10189
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2                                                          0x1018a
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3                                                          0x1018b
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG4                                                          0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG4_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0                                                   0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG5                                                          0x10190
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG5_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1                                                   0x10190
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG6                                                          0x10191
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG6_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2                                                   0x10191
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG7                                                          0x10192
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG7_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3                                                   0x10192
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG8                                                          0x10193
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG8_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG9                                                          0x10194
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG9_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG10                                                         0x10195
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG10_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG11                                                         0x10196
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG11_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG12                                                         0x10197
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG12_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG13                                                         0x10198
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG13_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST                                                  0x101c7
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF3_RTR_DATA1                                                              0x101c8
+#define regBIF_CFG_DEV0_EPF0_VF3_RTR_DATA1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF3_RTR_DATA2                                                              0x101c9
+#define regBIF_CFG_DEV0_EPF0_VF3_RTR_DATA2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST                                                  0x101dc
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP                                                           0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL                                                          0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST                                                  0x101eb
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP                                                           0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL                                                          0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf4_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_VF4_VENDOR_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF4_VENDOR_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF4_COMMAND                                                                0x10001
+#define regBIF_CFG_DEV0_EPF0_VF4_COMMAND_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF4_STATUS                                                                 0x10001
+#define regBIF_CFG_DEV0_EPF0_VF4_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF4_REVISION_ID                                                            0x10002
+#define regBIF_CFG_DEV0_EPF0_VF4_REVISION_ID_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE                                                         0x10002
+#define regBIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF4_SUB_CLASS                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_VF4_SUB_CLASS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_CLASS                                                             0x10002
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_CLASS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF4_CACHE_LINE                                                             0x10003
+#define regBIF_CFG_DEV0_EPF0_VF4_CACHE_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF4_LATENCY                                                                0x10003
+#define regBIF_CFG_DEV0_EPF0_VF4_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF4_HEADER                                                                 0x10003
+#define regBIF_CFG_DEV0_EPF0_VF4_HEADER_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF4_BIST                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_VF4_BIST_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1                                                            0x10004
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2                                                            0x10005
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3                                                            0x10006
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4                                                            0x10007
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5                                                            0x10008
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6                                                            0x10009
+#define regBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR                                                        0x1000a
+#define regBIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID                                                             0x1000b
+#define regBIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR                                                          0x1000c
+#define regBIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_CAP_PTR                                                                0x1000d
+#define regBIF_CFG_DEV0_EPF0_VF4_CAP_PTR_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE                                                         0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN                                                          0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_MIN_GRANT                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF4_MIN_GRANT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST                                                          0x10019
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_CAP                                                               0x10019
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP                                                             0x1001a
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS                                                          0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_CAP                                                               0x1001c
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_CNTL                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_STATUS                                                            0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2                                                            0x10022
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2                                                         0x10023
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_CAP2                                                              0x10024
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2                                                           0x10025
+#define regBIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO                                                        0x10029
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI                                                        0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA                                                           0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA                                                       0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MASK                                                               0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MASK_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64                                                        0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64                                                    0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_PENDING                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64                                                         0x1002d
+#define regBIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE                                                             0x10031
+#define regBIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF4_MSIX_PBA                                                               0x10032
+#define regBIF_CFG_DEV0_EPF0_VF4_MSIX_PBA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x10040
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                             5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR                                               0x10041
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1                                                  0x10042
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2                                                  0x10043
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST                                              0x100b6
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3                                                            0x100b7
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3                                                           0x100b8
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3                                                         0x100b9
+#define regBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST                                                0x10100
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST                                                0x10180
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x10181
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS                                                 0x10182
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK                                                   0x10183
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY                                               0x10184
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS                                                   0x10185
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK                                                     0x10186
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL                                                  0x10187
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0                                                          0x10188
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1                                                          0x10189
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2                                                          0x1018a
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3                                                          0x1018b
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG4                                                          0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG4_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0                                                   0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG5                                                          0x10190
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG5_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1                                                   0x10190
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG6                                                          0x10191
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG6_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2                                                   0x10191
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG7                                                          0x10192
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG7_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3                                                   0x10192
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG8                                                          0x10193
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG8_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG9                                                          0x10194
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG9_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG10                                                         0x10195
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG10_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG11                                                         0x10196
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG11_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG12                                                         0x10197
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG12_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG13                                                         0x10198
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG13_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST                                                  0x101c7
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF4_RTR_DATA1                                                              0x101c8
+#define regBIF_CFG_DEV0_EPF0_VF4_RTR_DATA1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF4_RTR_DATA2                                                              0x101c9
+#define regBIF_CFG_DEV0_EPF0_VF4_RTR_DATA2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST                                                  0x101dc
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP                                                           0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL                                                          0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST                                                  0x101eb
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP                                                           0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL                                                          0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf5_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_VF5_VENDOR_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF5_VENDOR_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF5_COMMAND                                                                0x10001
+#define regBIF_CFG_DEV0_EPF0_VF5_COMMAND_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF5_STATUS                                                                 0x10001
+#define regBIF_CFG_DEV0_EPF0_VF5_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF5_REVISION_ID                                                            0x10002
+#define regBIF_CFG_DEV0_EPF0_VF5_REVISION_ID_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE                                                         0x10002
+#define regBIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF5_SUB_CLASS                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_VF5_SUB_CLASS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_CLASS                                                             0x10002
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_CLASS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF5_CACHE_LINE                                                             0x10003
+#define regBIF_CFG_DEV0_EPF0_VF5_CACHE_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF5_LATENCY                                                                0x10003
+#define regBIF_CFG_DEV0_EPF0_VF5_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF5_HEADER                                                                 0x10003
+#define regBIF_CFG_DEV0_EPF0_VF5_HEADER_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF5_BIST                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_VF5_BIST_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1                                                            0x10004
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2                                                            0x10005
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3                                                            0x10006
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4                                                            0x10007
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5                                                            0x10008
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6                                                            0x10009
+#define regBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR                                                        0x1000a
+#define regBIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID                                                             0x1000b
+#define regBIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR                                                          0x1000c
+#define regBIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_CAP_PTR                                                                0x1000d
+#define regBIF_CFG_DEV0_EPF0_VF5_CAP_PTR_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE                                                         0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN                                                          0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_MIN_GRANT                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF5_MIN_GRANT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST                                                          0x10019
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_CAP                                                               0x10019
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP                                                             0x1001a
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS                                                          0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_CAP                                                               0x1001c
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_CNTL                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_STATUS                                                            0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2                                                            0x10022
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2                                                         0x10023
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_CAP2                                                              0x10024
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2                                                           0x10025
+#define regBIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO                                                        0x10029
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI                                                        0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA                                                           0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA                                                       0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MASK                                                               0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MASK_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64                                                        0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64                                                    0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_PENDING                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64                                                         0x1002d
+#define regBIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE                                                             0x10031
+#define regBIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF5_MSIX_PBA                                                               0x10032
+#define regBIF_CFG_DEV0_EPF0_VF5_MSIX_PBA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x10040
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                             5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR                                               0x10041
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1                                                  0x10042
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2                                                  0x10043
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST                                              0x100b6
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3                                                            0x100b7
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3                                                           0x100b8
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3                                                         0x100b9
+#define regBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST                                                0x10100
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST                                                0x10180
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x10181
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS                                                 0x10182
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK                                                   0x10183
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY                                               0x10184
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS                                                   0x10185
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK                                                     0x10186
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL                                                  0x10187
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0                                                          0x10188
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1                                                          0x10189
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2                                                          0x1018a
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3                                                          0x1018b
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG4                                                          0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG4_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0                                                   0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG5                                                          0x10190
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG5_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1                                                   0x10190
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG6                                                          0x10191
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG6_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2                                                   0x10191
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG7                                                          0x10192
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG7_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3                                                   0x10192
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG8                                                          0x10193
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG8_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG9                                                          0x10194
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG9_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG10                                                         0x10195
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG10_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG11                                                         0x10196
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG11_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG12                                                         0x10197
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG12_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG13                                                         0x10198
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG13_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST                                                  0x101c7
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF5_RTR_DATA1                                                              0x101c8
+#define regBIF_CFG_DEV0_EPF0_VF5_RTR_DATA1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF5_RTR_DATA2                                                              0x101c9
+#define regBIF_CFG_DEV0_EPF0_VF5_RTR_DATA2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST                                                  0x101dc
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP                                                           0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL                                                          0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST                                                  0x101eb
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP                                                           0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL                                                          0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf6_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_VF6_VENDOR_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF6_VENDOR_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF6_COMMAND                                                                0x10001
+#define regBIF_CFG_DEV0_EPF0_VF6_COMMAND_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF6_STATUS                                                                 0x10001
+#define regBIF_CFG_DEV0_EPF0_VF6_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF6_REVISION_ID                                                            0x10002
+#define regBIF_CFG_DEV0_EPF0_VF6_REVISION_ID_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE                                                         0x10002
+#define regBIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF6_SUB_CLASS                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_VF6_SUB_CLASS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_CLASS                                                             0x10002
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_CLASS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF6_CACHE_LINE                                                             0x10003
+#define regBIF_CFG_DEV0_EPF0_VF6_CACHE_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF6_LATENCY                                                                0x10003
+#define regBIF_CFG_DEV0_EPF0_VF6_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF6_HEADER                                                                 0x10003
+#define regBIF_CFG_DEV0_EPF0_VF6_HEADER_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF6_BIST                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_VF6_BIST_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1                                                            0x10004
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2                                                            0x10005
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3                                                            0x10006
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4                                                            0x10007
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5                                                            0x10008
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6                                                            0x10009
+#define regBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR                                                        0x1000a
+#define regBIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID                                                             0x1000b
+#define regBIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR                                                          0x1000c
+#define regBIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_CAP_PTR                                                                0x1000d
+#define regBIF_CFG_DEV0_EPF0_VF6_CAP_PTR_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE                                                         0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN                                                          0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_MIN_GRANT                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF6_MIN_GRANT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST                                                          0x10019
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_CAP                                                               0x10019
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP                                                             0x1001a
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS                                                          0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_CAP                                                               0x1001c
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_CNTL                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_STATUS                                                            0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2                                                            0x10022
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2                                                         0x10023
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_CAP2                                                              0x10024
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2                                                           0x10025
+#define regBIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO                                                        0x10029
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI                                                        0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA                                                           0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA                                                       0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MASK                                                               0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MASK_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64                                                        0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64                                                    0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_PENDING                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64                                                         0x1002d
+#define regBIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE                                                             0x10031
+#define regBIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF6_MSIX_PBA                                                               0x10032
+#define regBIF_CFG_DEV0_EPF0_VF6_MSIX_PBA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x10040
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                             5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR                                               0x10041
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1                                                  0x10042
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2                                                  0x10043
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST                                              0x100b6
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3                                                            0x100b7
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3                                                           0x100b8
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3                                                         0x100b9
+#define regBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST                                                0x10100
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST                                                0x10180
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x10181
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS                                                 0x10182
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK                                                   0x10183
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY                                               0x10184
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS                                                   0x10185
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK                                                     0x10186
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL                                                  0x10187
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0                                                          0x10188
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1                                                          0x10189
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2                                                          0x1018a
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3                                                          0x1018b
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG4                                                          0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG4_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0                                                   0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG5                                                          0x10190
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG5_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1                                                   0x10190
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG6                                                          0x10191
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG6_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2                                                   0x10191
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG7                                                          0x10192
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG7_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3                                                   0x10192
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG8                                                          0x10193
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG8_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG9                                                          0x10194
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG9_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG10                                                         0x10195
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG10_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG11                                                         0x10196
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG11_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG12                                                         0x10197
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG12_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG13                                                         0x10198
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG13_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST                                                  0x101c7
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF6_RTR_DATA1                                                              0x101c8
+#define regBIF_CFG_DEV0_EPF0_VF6_RTR_DATA1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF6_RTR_DATA2                                                              0x101c9
+#define regBIF_CFG_DEV0_EPF0_VF6_RTR_DATA2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST                                                  0x101dc
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP                                                           0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL                                                          0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST                                                  0x101eb
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP                                                           0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL                                                          0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf7_bifcfgdecp
+// base address: 0x10140000
+#define regBIF_CFG_DEV0_EPF0_VF7_VENDOR_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF7_VENDOR_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_ID                                                              0x10000
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF7_COMMAND                                                                0x10001
+#define regBIF_CFG_DEV0_EPF0_VF7_COMMAND_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF7_STATUS                                                                 0x10001
+#define regBIF_CFG_DEV0_EPF0_VF7_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF7_REVISION_ID                                                            0x10002
+#define regBIF_CFG_DEV0_EPF0_VF7_REVISION_ID_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE                                                         0x10002
+#define regBIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF7_SUB_CLASS                                                              0x10002
+#define regBIF_CFG_DEV0_EPF0_VF7_SUB_CLASS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_CLASS                                                             0x10002
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_CLASS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF7_CACHE_LINE                                                             0x10003
+#define regBIF_CFG_DEV0_EPF0_VF7_CACHE_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF7_LATENCY                                                                0x10003
+#define regBIF_CFG_DEV0_EPF0_VF7_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF7_HEADER                                                                 0x10003
+#define regBIF_CFG_DEV0_EPF0_VF7_HEADER_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF0_VF7_BIST                                                                   0x10003
+#define regBIF_CFG_DEV0_EPF0_VF7_BIST_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1                                                            0x10004
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2                                                            0x10005
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3                                                            0x10006
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4                                                            0x10007
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5                                                            0x10008
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6                                                            0x10009
+#define regBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR                                                        0x1000a
+#define regBIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID                                                             0x1000b
+#define regBIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR                                                          0x1000c
+#define regBIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_CAP_PTR                                                                0x1000d
+#define regBIF_CFG_DEV0_EPF0_VF7_CAP_PTR_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE                                                         0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN                                                          0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_MIN_GRANT                                                              0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF7_MIN_GRANT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY                                                            0x1000f
+#define regBIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST                                                          0x10019
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_CAP                                                               0x10019
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP                                                             0x1001a
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL                                                            0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS                                                          0x1001b
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_CAP                                                               0x1001c
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_CNTL                                                              0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_STATUS                                                            0x1001d
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2                                                            0x10022
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2                                                           0x10023
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2                                                         0x10023
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_CAP2                                                              0x10024
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2                                                             0x10025
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2                                                           0x10025
+#define regBIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL                                                           0x10028
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO                                                        0x10029
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI                                                        0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA                                                           0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA                                                       0x1002a
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MASK                                                               0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MASK_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64                                                        0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64                                                    0x1002b
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_PENDING                                                            0x1002c
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64                                                         0x1002d
+#define regBIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL                                                          0x10030
+#define regBIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE                                                             0x10031
+#define regBIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF0_VF7_MSIX_PBA                                                               0x10032
+#define regBIF_CFG_DEV0_EPF0_VF7_MSIX_PBA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x10040
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                             5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR                                               0x10041
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1                                                  0x10042
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2                                                  0x10043
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST                                              0x100b6
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3                                                            0x100b7
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3                                                           0x100b8
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3                                                         0x100b9
+#define regBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST                                                0x10100
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST                                                0x10180
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x10181
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS                                                 0x10182
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK                                                   0x10183
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY                                               0x10184
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS                                                   0x10185
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK                                                     0x10186
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL                                                  0x10187
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0                                                          0x10188
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1                                                          0x10189
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2                                                          0x1018a
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3                                                          0x1018b
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG4                                                          0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG4_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0                                                   0x1018f
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG5                                                          0x10190
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG5_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1                                                   0x10190
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG6                                                          0x10191
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG6_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2                                                   0x10191
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG7                                                          0x10192
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG7_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3                                                   0x10192
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG8                                                          0x10193
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG8_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG9                                                          0x10194
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG9_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG10                                                         0x10195
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG10_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG11                                                         0x10196
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG11_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG12                                                         0x10197
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG12_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG13                                                         0x10198
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG13_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST                                                  0x101c7
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF7_RTR_DATA1                                                              0x101c8
+#define regBIF_CFG_DEV0_EPF0_VF7_RTR_DATA1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF7_RTR_DATA2                                                              0x101c9
+#define regBIF_CFG_DEV0_EPF0_VF7_RTR_DATA2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST                                                  0x101dc
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP                                                           0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL                                                          0x101dd
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST                                                  0x101eb
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP                                                           0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL                                                          0x101ec
+#define regBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf0_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF0_0_BIF_BME_STATUS                                                        0x8e0b
+#define regBIF_BX_DEV0_EPF0_VF0_0_BIF_BME_STATUS_BASE_IDX                                               5
+#define regBIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG                                                    0x8e0c
+#define regBIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x8e13
+#define regBIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         5
+#define regBIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x8e14
+#define regBIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          5
+#define regBIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x8e15
+#define regBIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              5
+#define regBIF_BX_DEV0_EPF0_VF0_0_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x8e16
+#define regBIF_BX_DEV0_EPF0_VF0_0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x8e17
+#define regBIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x8e19
+#define regBIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            5
+#define regBIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x8e1a
+#define regBIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       5
+#define regBIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ                                                     0x8e26
+#define regBIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE                                                    0x8e27
+#define regBIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF0_0_BIF_TRANS_PENDING                                                     0x8e28
+#define regBIF_BX_DEV0_EPF0_VF0_0_BIF_TRANS_PENDING_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF0_0_NBIF_GFX_ADDR_LUT_BYPASS                                              0x8e32
+#define regBIF_BX_DEV0_EPF0_VF0_0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW0                                                0x8e56
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW1                                                0x8e57
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW2                                                0x8e58
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW3                                                0x8e59
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW0                                                0x8e5a
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW1                                                0x8e5b
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW2                                                0x8e5c
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW3                                                0x8e5d
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL                                                       0x8e5e
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL_BASE_IDX                                              5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_INT_CNTL                                                      0x8e5f
+#define regBIF_BX_DEV0_EPF0_VF0_0_MAILBOX_INT_CNTL_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX                                                      0x8e60
+#define regBIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP                                                 0x8e81
+#define regBIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP_BASE_IDX                                        5
+#define regBIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP                                                     0x8e82
+#define regBIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_STATUS                                              0x8e83
+#define regBIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_STATUS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_STATUS                                                  0x8e84
+#define regBIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_STATUS_BASE_IDX                                         5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf0_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF0_0_MM_INDEX                                                              0x8000
+#define regBIF_BX_DEV0_EPF0_VF0_0_MM_INDEX_BASE_IDX                                                     5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MM_DATA                                                               0x8001
+#define regBIF_BX_DEV0_EPF0_VF0_0_MM_DATA_BASE_IDX                                                      5
+#define regBIF_BX_DEV0_EPF0_VF0_0_MM_INDEX_HI                                                           0x8006
+#define regBIF_BX_DEV0_EPF0_VF0_0_MM_INDEX_HI_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf0_BIFPFVFDEC1:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF0_1_BIF_BME_STATUS                                                        0x00eb
+#define regBIF_BX_DEV0_EPF0_VF0_1_BIF_BME_STATUS_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG                                                    0x00ec
+#define regBIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x00f3
+#define regBIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x00f4
+#define regBIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          2
+#define regBIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x00f5
+#define regBIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF0_1_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x00f6
+#define regBIF_BX_DEV0_EPF0_VF0_1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x00f7
+#define regBIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x00f9
+#define regBIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x00fa
+#define regBIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       2
+#define regBIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ                                                     0x0106
+#define regBIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE                                                    0x0107
+#define regBIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF0_1_BIF_TRANS_PENDING                                                     0x0108
+#define regBIF_BX_DEV0_EPF0_VF0_1_BIF_TRANS_PENDING_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF0_1_NBIF_GFX_ADDR_LUT_BYPASS                                              0x0112
+#define regBIF_BX_DEV0_EPF0_VF0_1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW0                                                0x0136
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW1                                                0x0137
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW2                                                0x0138
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW3                                                0x0139
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW0                                                0x013a
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW1                                                0x013b
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW2                                                0x013c
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW3                                                0x013d
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL                                                       0x013e
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_INT_CNTL                                                      0x013f
+#define regBIF_BX_DEV0_EPF0_VF0_1_MAILBOX_INT_CNTL_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX                                                      0x0140
+#define regBIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP                                                 0x0161
+#define regBIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP_BASE_IDX                                        2
+#define regBIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP                                                     0x0162
+#define regBIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_STATUS                                              0x0163
+#define regBIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_STATUS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_STATUS                                                  0x0164
+#define regBIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_STATUS_BASE_IDX                                         2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf0_SYSPFVFDEC:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF0_1_MM_INDEX                                                              0x0000
+#define regBIF_BX_DEV0_EPF0_VF0_1_MM_INDEX_BASE_IDX                                                     0
+#define regBIF_BX_DEV0_EPF0_VF0_1_MM_DATA                                                               0x0001
+#define regBIF_BX_DEV0_EPF0_VF0_1_MM_DATA_BASE_IDX                                                      0
+#define regBIF_BX_DEV0_EPF0_VF0_1_MM_INDEX_HI                                                           0x0006
+#define regBIF_BX_DEV0_EPF0_VF0_1_MM_INDEX_HI_BASE_IDX                                                  0
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf0_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF0_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF0_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf0_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_LO                                                      0x0410
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_HI                                                      0x0411
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_MSG_DATA                                                     0x0412
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_CONTROL                                                      0x0413
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_LO                                                      0x0414
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_HI                                                      0x0415
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_MSG_DATA                                                     0x0416
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_CONTROL                                                      0x0417
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_LO                                                      0x0418
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_HI                                                      0x0419
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_MSG_DATA                                                     0x041a
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_CONTROL                                                      0x041b
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_LO                                                      0x041c
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_HI                                                      0x041d
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_MSG_DATA                                                     0x041e
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_CONTROL                                                      0x041f
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_LO                                                      0x0420
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_HI                                                      0x0421
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_MSG_DATA                                                     0x0422
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_CONTROL                                                      0x0423
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF0_GFXMSIX_PBA_BASE_IDX                                                       3
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf1_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF1_0_BIF_BME_STATUS                                                        0x8e0b
+#define regBIF_BX_DEV0_EPF0_VF1_0_BIF_BME_STATUS_BASE_IDX                                               5
+#define regBIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG                                                    0x8e0c
+#define regBIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x8e13
+#define regBIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         5
+#define regBIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x8e14
+#define regBIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          5
+#define regBIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x8e15
+#define regBIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              5
+#define regBIF_BX_DEV0_EPF0_VF1_0_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x8e16
+#define regBIF_BX_DEV0_EPF0_VF1_0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x8e17
+#define regBIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x8e19
+#define regBIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            5
+#define regBIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x8e1a
+#define regBIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       5
+#define regBIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ                                                     0x8e26
+#define regBIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE                                                    0x8e27
+#define regBIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF1_0_BIF_TRANS_PENDING                                                     0x8e28
+#define regBIF_BX_DEV0_EPF0_VF1_0_BIF_TRANS_PENDING_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF1_0_NBIF_GFX_ADDR_LUT_BYPASS                                              0x8e32
+#define regBIF_BX_DEV0_EPF0_VF1_0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW0                                                0x8e56
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW1                                                0x8e57
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW2                                                0x8e58
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW3                                                0x8e59
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW0                                                0x8e5a
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW1                                                0x8e5b
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW2                                                0x8e5c
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW3                                                0x8e5d
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL                                                       0x8e5e
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL_BASE_IDX                                              5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_INT_CNTL                                                      0x8e5f
+#define regBIF_BX_DEV0_EPF0_VF1_0_MAILBOX_INT_CNTL_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX                                                      0x8e60
+#define regBIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP                                                 0x8e81
+#define regBIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP_BASE_IDX                                        5
+#define regBIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP                                                     0x8e82
+#define regBIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_STATUS                                              0x8e83
+#define regBIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_STATUS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_STATUS                                                  0x8e84
+#define regBIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_STATUS_BASE_IDX                                         5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf1_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF1_0_MM_INDEX                                                              0x8000
+#define regBIF_BX_DEV0_EPF0_VF1_0_MM_INDEX_BASE_IDX                                                     5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MM_DATA                                                               0x8001
+#define regBIF_BX_DEV0_EPF0_VF1_0_MM_DATA_BASE_IDX                                                      5
+#define regBIF_BX_DEV0_EPF0_VF1_0_MM_INDEX_HI                                                           0x8006
+#define regBIF_BX_DEV0_EPF0_VF1_0_MM_INDEX_HI_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf1_BIFPFVFDEC1:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF1_1_BIF_BME_STATUS                                                        0x00eb
+#define regBIF_BX_DEV0_EPF0_VF1_1_BIF_BME_STATUS_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG                                                    0x00ec
+#define regBIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x00f3
+#define regBIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x00f4
+#define regBIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          2
+#define regBIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x00f5
+#define regBIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF1_1_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x00f6
+#define regBIF_BX_DEV0_EPF0_VF1_1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x00f7
+#define regBIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x00f9
+#define regBIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x00fa
+#define regBIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       2
+#define regBIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ                                                     0x0106
+#define regBIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE                                                    0x0107
+#define regBIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF1_1_BIF_TRANS_PENDING                                                     0x0108
+#define regBIF_BX_DEV0_EPF0_VF1_1_BIF_TRANS_PENDING_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF1_1_NBIF_GFX_ADDR_LUT_BYPASS                                              0x0112
+#define regBIF_BX_DEV0_EPF0_VF1_1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW0                                                0x0136
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW1                                                0x0137
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW2                                                0x0138
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW3                                                0x0139
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW0                                                0x013a
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW1                                                0x013b
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW2                                                0x013c
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW3                                                0x013d
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL                                                       0x013e
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_INT_CNTL                                                      0x013f
+#define regBIF_BX_DEV0_EPF0_VF1_1_MAILBOX_INT_CNTL_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX                                                      0x0140
+#define regBIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP                                                 0x0161
+#define regBIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP_BASE_IDX                                        2
+#define regBIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP                                                     0x0162
+#define regBIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_STATUS                                              0x0163
+#define regBIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_STATUS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_STATUS                                                  0x0164
+#define regBIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_STATUS_BASE_IDX                                         2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf1_SYSPFVFDEC:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF1_1_MM_INDEX                                                              0x0000
+#define regBIF_BX_DEV0_EPF0_VF1_1_MM_INDEX_BASE_IDX                                                     0
+#define regBIF_BX_DEV0_EPF0_VF1_1_MM_DATA                                                               0x0001
+#define regBIF_BX_DEV0_EPF0_VF1_1_MM_DATA_BASE_IDX                                                      0
+#define regBIF_BX_DEV0_EPF0_VF1_1_MM_INDEX_HI                                                           0x0006
+#define regBIF_BX_DEV0_EPF0_VF1_1_MM_INDEX_HI_BASE_IDX                                                  0
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf1_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF1_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF1_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf1_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_LO                                                      0x0410
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_HI                                                      0x0411
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_MSG_DATA                                                     0x0412
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_CONTROL                                                      0x0413
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_LO                                                      0x0414
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_HI                                                      0x0415
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_MSG_DATA                                                     0x0416
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_CONTROL                                                      0x0417
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_LO                                                      0x0418
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_HI                                                      0x0419
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_MSG_DATA                                                     0x041a
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_CONTROL                                                      0x041b
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_LO                                                      0x041c
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_HI                                                      0x041d
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_MSG_DATA                                                     0x041e
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_CONTROL                                                      0x041f
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_LO                                                      0x0420
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_HI                                                      0x0421
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_MSG_DATA                                                     0x0422
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_CONTROL                                                      0x0423
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF1_GFXMSIX_PBA_BASE_IDX                                                       3
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf2_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF2_0_BIF_BME_STATUS                                                        0x8e0b
+#define regBIF_BX_DEV0_EPF0_VF2_0_BIF_BME_STATUS_BASE_IDX                                               5
+#define regBIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG                                                    0x8e0c
+#define regBIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x8e13
+#define regBIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         5
+#define regBIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x8e14
+#define regBIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          5
+#define regBIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x8e15
+#define regBIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              5
+#define regBIF_BX_DEV0_EPF0_VF2_0_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x8e16
+#define regBIF_BX_DEV0_EPF0_VF2_0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x8e17
+#define regBIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x8e19
+#define regBIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            5
+#define regBIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x8e1a
+#define regBIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       5
+#define regBIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ                                                     0x8e26
+#define regBIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE                                                    0x8e27
+#define regBIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF2_0_BIF_TRANS_PENDING                                                     0x8e28
+#define regBIF_BX_DEV0_EPF0_VF2_0_BIF_TRANS_PENDING_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF2_0_NBIF_GFX_ADDR_LUT_BYPASS                                              0x8e32
+#define regBIF_BX_DEV0_EPF0_VF2_0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW0                                                0x8e56
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW1                                                0x8e57
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW2                                                0x8e58
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW3                                                0x8e59
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW0                                                0x8e5a
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW1                                                0x8e5b
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW2                                                0x8e5c
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW3                                                0x8e5d
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL                                                       0x8e5e
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL_BASE_IDX                                              5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_INT_CNTL                                                      0x8e5f
+#define regBIF_BX_DEV0_EPF0_VF2_0_MAILBOX_INT_CNTL_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX                                                      0x8e60
+#define regBIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP                                                 0x8e81
+#define regBIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP_BASE_IDX                                        5
+#define regBIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP                                                     0x8e82
+#define regBIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_STATUS                                              0x8e83
+#define regBIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_STATUS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_STATUS                                                  0x8e84
+#define regBIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_STATUS_BASE_IDX                                         5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf2_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF2_0_MM_INDEX                                                              0x8000
+#define regBIF_BX_DEV0_EPF0_VF2_0_MM_INDEX_BASE_IDX                                                     5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MM_DATA                                                               0x8001
+#define regBIF_BX_DEV0_EPF0_VF2_0_MM_DATA_BASE_IDX                                                      5
+#define regBIF_BX_DEV0_EPF0_VF2_0_MM_INDEX_HI                                                           0x8006
+#define regBIF_BX_DEV0_EPF0_VF2_0_MM_INDEX_HI_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf2_BIFPFVFDEC1:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF2_1_BIF_BME_STATUS                                                        0x00eb
+#define regBIF_BX_DEV0_EPF0_VF2_1_BIF_BME_STATUS_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG                                                    0x00ec
+#define regBIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x00f3
+#define regBIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x00f4
+#define regBIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          2
+#define regBIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x00f5
+#define regBIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF2_1_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x00f6
+#define regBIF_BX_DEV0_EPF0_VF2_1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x00f7
+#define regBIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x00f9
+#define regBIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x00fa
+#define regBIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       2
+#define regBIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ                                                     0x0106
+#define regBIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE                                                    0x0107
+#define regBIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF2_1_BIF_TRANS_PENDING                                                     0x0108
+#define regBIF_BX_DEV0_EPF0_VF2_1_BIF_TRANS_PENDING_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF2_1_NBIF_GFX_ADDR_LUT_BYPASS                                              0x0112
+#define regBIF_BX_DEV0_EPF0_VF2_1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW0                                                0x0136
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW1                                                0x0137
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW2                                                0x0138
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW3                                                0x0139
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW0                                                0x013a
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW1                                                0x013b
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW2                                                0x013c
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW3                                                0x013d
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL                                                       0x013e
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_INT_CNTL                                                      0x013f
+#define regBIF_BX_DEV0_EPF0_VF2_1_MAILBOX_INT_CNTL_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX                                                      0x0140
+#define regBIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP                                                 0x0161
+#define regBIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP_BASE_IDX                                        2
+#define regBIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP                                                     0x0162
+#define regBIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_STATUS                                              0x0163
+#define regBIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_STATUS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_STATUS                                                  0x0164
+#define regBIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_STATUS_BASE_IDX                                         2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf2_SYSPFVFDEC:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF2_1_MM_INDEX                                                              0x0000
+#define regBIF_BX_DEV0_EPF0_VF2_1_MM_INDEX_BASE_IDX                                                     0
+#define regBIF_BX_DEV0_EPF0_VF2_1_MM_DATA                                                               0x0001
+#define regBIF_BX_DEV0_EPF0_VF2_1_MM_DATA_BASE_IDX                                                      0
+#define regBIF_BX_DEV0_EPF0_VF2_1_MM_INDEX_HI                                                           0x0006
+#define regBIF_BX_DEV0_EPF0_VF2_1_MM_INDEX_HI_BASE_IDX                                                  0
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf2_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF2_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF2_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf2_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_LO                                                      0x0410
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_HI                                                      0x0411
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_MSG_DATA                                                     0x0412
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_CONTROL                                                      0x0413
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_LO                                                      0x0414
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_HI                                                      0x0415
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_MSG_DATA                                                     0x0416
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_CONTROL                                                      0x0417
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_LO                                                      0x0418
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_HI                                                      0x0419
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_MSG_DATA                                                     0x041a
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_CONTROL                                                      0x041b
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_LO                                                      0x041c
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_HI                                                      0x041d
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_MSG_DATA                                                     0x041e
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_CONTROL                                                      0x041f
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_LO                                                      0x0420
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_HI                                                      0x0421
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_MSG_DATA                                                     0x0422
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_CONTROL                                                      0x0423
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF2_GFXMSIX_PBA_BASE_IDX                                                       3
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf3_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF3_0_BIF_BME_STATUS                                                        0x8e0b
+#define regBIF_BX_DEV0_EPF0_VF3_0_BIF_BME_STATUS_BASE_IDX                                               5
+#define regBIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG                                                    0x8e0c
+#define regBIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x8e13
+#define regBIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         5
+#define regBIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x8e14
+#define regBIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          5
+#define regBIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x8e15
+#define regBIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              5
+#define regBIF_BX_DEV0_EPF0_VF3_0_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x8e16
+#define regBIF_BX_DEV0_EPF0_VF3_0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x8e17
+#define regBIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x8e19
+#define regBIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            5
+#define regBIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x8e1a
+#define regBIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       5
+#define regBIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ                                                     0x8e26
+#define regBIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE                                                    0x8e27
+#define regBIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF3_0_BIF_TRANS_PENDING                                                     0x8e28
+#define regBIF_BX_DEV0_EPF0_VF3_0_BIF_TRANS_PENDING_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF3_0_NBIF_GFX_ADDR_LUT_BYPASS                                              0x8e32
+#define regBIF_BX_DEV0_EPF0_VF3_0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW0                                                0x8e56
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW1                                                0x8e57
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW2                                                0x8e58
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW3                                                0x8e59
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW0                                                0x8e5a
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW1                                                0x8e5b
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW2                                                0x8e5c
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW3                                                0x8e5d
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL                                                       0x8e5e
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL_BASE_IDX                                              5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_INT_CNTL                                                      0x8e5f
+#define regBIF_BX_DEV0_EPF0_VF3_0_MAILBOX_INT_CNTL_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX                                                      0x8e60
+#define regBIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP                                                 0x8e81
+#define regBIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP_BASE_IDX                                        5
+#define regBIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP                                                     0x8e82
+#define regBIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_STATUS                                              0x8e83
+#define regBIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_STATUS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_STATUS                                                  0x8e84
+#define regBIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_STATUS_BASE_IDX                                         5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf3_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF3_0_MM_INDEX                                                              0x8000
+#define regBIF_BX_DEV0_EPF0_VF3_0_MM_INDEX_BASE_IDX                                                     5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MM_DATA                                                               0x8001
+#define regBIF_BX_DEV0_EPF0_VF3_0_MM_DATA_BASE_IDX                                                      5
+#define regBIF_BX_DEV0_EPF0_VF3_0_MM_INDEX_HI                                                           0x8006
+#define regBIF_BX_DEV0_EPF0_VF3_0_MM_INDEX_HI_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf3_BIFPFVFDEC1:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF3_1_BIF_BME_STATUS                                                        0x00eb
+#define regBIF_BX_DEV0_EPF0_VF3_1_BIF_BME_STATUS_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG                                                    0x00ec
+#define regBIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x00f3
+#define regBIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x00f4
+#define regBIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          2
+#define regBIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x00f5
+#define regBIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF3_1_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x00f6
+#define regBIF_BX_DEV0_EPF0_VF3_1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x00f7
+#define regBIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x00f9
+#define regBIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x00fa
+#define regBIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       2
+#define regBIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ                                                     0x0106
+#define regBIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE                                                    0x0107
+#define regBIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF3_1_BIF_TRANS_PENDING                                                     0x0108
+#define regBIF_BX_DEV0_EPF0_VF3_1_BIF_TRANS_PENDING_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF3_1_NBIF_GFX_ADDR_LUT_BYPASS                                              0x0112
+#define regBIF_BX_DEV0_EPF0_VF3_1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW0                                                0x0136
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW1                                                0x0137
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW2                                                0x0138
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW3                                                0x0139
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW0                                                0x013a
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW1                                                0x013b
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW2                                                0x013c
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW3                                                0x013d
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL                                                       0x013e
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_INT_CNTL                                                      0x013f
+#define regBIF_BX_DEV0_EPF0_VF3_1_MAILBOX_INT_CNTL_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX                                                      0x0140
+#define regBIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP                                                 0x0161
+#define regBIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP_BASE_IDX                                        2
+#define regBIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP                                                     0x0162
+#define regBIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_STATUS                                              0x0163
+#define regBIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_STATUS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_STATUS                                                  0x0164
+#define regBIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_STATUS_BASE_IDX                                         2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf3_SYSPFVFDEC:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF3_1_MM_INDEX                                                              0x0000
+#define regBIF_BX_DEV0_EPF0_VF3_1_MM_INDEX_BASE_IDX                                                     0
+#define regBIF_BX_DEV0_EPF0_VF3_1_MM_DATA                                                               0x0001
+#define regBIF_BX_DEV0_EPF0_VF3_1_MM_DATA_BASE_IDX                                                      0
+#define regBIF_BX_DEV0_EPF0_VF3_1_MM_INDEX_HI                                                           0x0006
+#define regBIF_BX_DEV0_EPF0_VF3_1_MM_INDEX_HI_BASE_IDX                                                  0
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf3_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF3_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF3_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf3_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_LO                                                      0x0410
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_HI                                                      0x0411
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_MSG_DATA                                                     0x0412
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_CONTROL                                                      0x0413
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_LO                                                      0x0414
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_HI                                                      0x0415
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_MSG_DATA                                                     0x0416
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_CONTROL                                                      0x0417
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_LO                                                      0x0418
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_HI                                                      0x0419
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_MSG_DATA                                                     0x041a
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_CONTROL                                                      0x041b
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_LO                                                      0x041c
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_HI                                                      0x041d
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_MSG_DATA                                                     0x041e
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_CONTROL                                                      0x041f
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_LO                                                      0x0420
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_HI                                                      0x0421
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_MSG_DATA                                                     0x0422
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_CONTROL                                                      0x0423
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF3_GFXMSIX_PBA_BASE_IDX                                                       3
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf4_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF4_0_BIF_BME_STATUS                                                        0x8e0b
+#define regBIF_BX_DEV0_EPF0_VF4_0_BIF_BME_STATUS_BASE_IDX                                               5
+#define regBIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG                                                    0x8e0c
+#define regBIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x8e13
+#define regBIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         5
+#define regBIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x8e14
+#define regBIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          5
+#define regBIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x8e15
+#define regBIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              5
+#define regBIF_BX_DEV0_EPF0_VF4_0_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x8e16
+#define regBIF_BX_DEV0_EPF0_VF4_0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x8e17
+#define regBIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x8e19
+#define regBIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            5
+#define regBIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x8e1a
+#define regBIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       5
+#define regBIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ                                                     0x8e26
+#define regBIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE                                                    0x8e27
+#define regBIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF4_0_BIF_TRANS_PENDING                                                     0x8e28
+#define regBIF_BX_DEV0_EPF0_VF4_0_BIF_TRANS_PENDING_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF4_0_NBIF_GFX_ADDR_LUT_BYPASS                                              0x8e32
+#define regBIF_BX_DEV0_EPF0_VF4_0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW0                                                0x8e56
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW1                                                0x8e57
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW2                                                0x8e58
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW3                                                0x8e59
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW0                                                0x8e5a
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW1                                                0x8e5b
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW2                                                0x8e5c
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW3                                                0x8e5d
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL                                                       0x8e5e
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL_BASE_IDX                                              5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_INT_CNTL                                                      0x8e5f
+#define regBIF_BX_DEV0_EPF0_VF4_0_MAILBOX_INT_CNTL_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX                                                      0x8e60
+#define regBIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP                                                 0x8e81
+#define regBIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP_BASE_IDX                                        5
+#define regBIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP                                                     0x8e82
+#define regBIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_STATUS                                              0x8e83
+#define regBIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_STATUS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_STATUS                                                  0x8e84
+#define regBIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_STATUS_BASE_IDX                                         5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf4_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF4_0_MM_INDEX                                                              0x8000
+#define regBIF_BX_DEV0_EPF0_VF4_0_MM_INDEX_BASE_IDX                                                     5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MM_DATA                                                               0x8001
+#define regBIF_BX_DEV0_EPF0_VF4_0_MM_DATA_BASE_IDX                                                      5
+#define regBIF_BX_DEV0_EPF0_VF4_0_MM_INDEX_HI                                                           0x8006
+#define regBIF_BX_DEV0_EPF0_VF4_0_MM_INDEX_HI_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf4_BIFPFVFDEC1:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF4_1_BIF_BME_STATUS                                                        0x00eb
+#define regBIF_BX_DEV0_EPF0_VF4_1_BIF_BME_STATUS_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG                                                    0x00ec
+#define regBIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x00f3
+#define regBIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x00f4
+#define regBIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          2
+#define regBIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x00f5
+#define regBIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF4_1_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x00f6
+#define regBIF_BX_DEV0_EPF0_VF4_1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x00f7
+#define regBIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x00f9
+#define regBIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x00fa
+#define regBIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       2
+#define regBIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ                                                     0x0106
+#define regBIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE                                                    0x0107
+#define regBIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF4_1_BIF_TRANS_PENDING                                                     0x0108
+#define regBIF_BX_DEV0_EPF0_VF4_1_BIF_TRANS_PENDING_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF4_1_NBIF_GFX_ADDR_LUT_BYPASS                                              0x0112
+#define regBIF_BX_DEV0_EPF0_VF4_1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW0                                                0x0136
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW1                                                0x0137
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW2                                                0x0138
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW3                                                0x0139
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW0                                                0x013a
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW1                                                0x013b
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW2                                                0x013c
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW3                                                0x013d
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL                                                       0x013e
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_INT_CNTL                                                      0x013f
+#define regBIF_BX_DEV0_EPF0_VF4_1_MAILBOX_INT_CNTL_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX                                                      0x0140
+#define regBIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP                                                 0x0161
+#define regBIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP_BASE_IDX                                        2
+#define regBIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP                                                     0x0162
+#define regBIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_STATUS                                              0x0163
+#define regBIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_STATUS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_STATUS                                                  0x0164
+#define regBIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_STATUS_BASE_IDX                                         2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf4_SYSPFVFDEC:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF4_1_MM_INDEX                                                              0x0000
+#define regBIF_BX_DEV0_EPF0_VF4_1_MM_INDEX_BASE_IDX                                                     0
+#define regBIF_BX_DEV0_EPF0_VF4_1_MM_DATA                                                               0x0001
+#define regBIF_BX_DEV0_EPF0_VF4_1_MM_DATA_BASE_IDX                                                      0
+#define regBIF_BX_DEV0_EPF0_VF4_1_MM_INDEX_HI                                                           0x0006
+#define regBIF_BX_DEV0_EPF0_VF4_1_MM_INDEX_HI_BASE_IDX                                                  0
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf4_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF4_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF4_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf4_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_LO                                                      0x0410
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_HI                                                      0x0411
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_MSG_DATA                                                     0x0412
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_CONTROL                                                      0x0413
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_LO                                                      0x0414
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_HI                                                      0x0415
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_MSG_DATA                                                     0x0416
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_CONTROL                                                      0x0417
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_LO                                                      0x0418
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_HI                                                      0x0419
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_MSG_DATA                                                     0x041a
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_CONTROL                                                      0x041b
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_LO                                                      0x041c
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_HI                                                      0x041d
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_MSG_DATA                                                     0x041e
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_CONTROL                                                      0x041f
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_LO                                                      0x0420
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_HI                                                      0x0421
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_MSG_DATA                                                     0x0422
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_CONTROL                                                      0x0423
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF4_GFXMSIX_PBA_BASE_IDX                                                       3
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf5_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF5_0_BIF_BME_STATUS                                                        0x8e0b
+#define regBIF_BX_DEV0_EPF0_VF5_0_BIF_BME_STATUS_BASE_IDX                                               5
+#define regBIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG                                                    0x8e0c
+#define regBIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x8e13
+#define regBIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         5
+#define regBIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x8e14
+#define regBIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          5
+#define regBIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x8e15
+#define regBIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              5
+#define regBIF_BX_DEV0_EPF0_VF5_0_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x8e16
+#define regBIF_BX_DEV0_EPF0_VF5_0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x8e17
+#define regBIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x8e19
+#define regBIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            5
+#define regBIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x8e1a
+#define regBIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       5
+#define regBIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ                                                     0x8e26
+#define regBIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE                                                    0x8e27
+#define regBIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF5_0_BIF_TRANS_PENDING                                                     0x8e28
+#define regBIF_BX_DEV0_EPF0_VF5_0_BIF_TRANS_PENDING_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF5_0_NBIF_GFX_ADDR_LUT_BYPASS                                              0x8e32
+#define regBIF_BX_DEV0_EPF0_VF5_0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW0                                                0x8e56
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW1                                                0x8e57
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW2                                                0x8e58
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW3                                                0x8e59
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW0                                                0x8e5a
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW1                                                0x8e5b
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW2                                                0x8e5c
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW3                                                0x8e5d
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL                                                       0x8e5e
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL_BASE_IDX                                              5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_INT_CNTL                                                      0x8e5f
+#define regBIF_BX_DEV0_EPF0_VF5_0_MAILBOX_INT_CNTL_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX                                                      0x8e60
+#define regBIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP                                                 0x8e81
+#define regBIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP_BASE_IDX                                        5
+#define regBIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP                                                     0x8e82
+#define regBIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_STATUS                                              0x8e83
+#define regBIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_STATUS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_STATUS                                                  0x8e84
+#define regBIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_STATUS_BASE_IDX                                         5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf5_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF5_0_MM_INDEX                                                              0x8000
+#define regBIF_BX_DEV0_EPF0_VF5_0_MM_INDEX_BASE_IDX                                                     5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MM_DATA                                                               0x8001
+#define regBIF_BX_DEV0_EPF0_VF5_0_MM_DATA_BASE_IDX                                                      5
+#define regBIF_BX_DEV0_EPF0_VF5_0_MM_INDEX_HI                                                           0x8006
+#define regBIF_BX_DEV0_EPF0_VF5_0_MM_INDEX_HI_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf5_BIFPFVFDEC1:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF5_1_BIF_BME_STATUS                                                        0x00eb
+#define regBIF_BX_DEV0_EPF0_VF5_1_BIF_BME_STATUS_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG                                                    0x00ec
+#define regBIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x00f3
+#define regBIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x00f4
+#define regBIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          2
+#define regBIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x00f5
+#define regBIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF5_1_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x00f6
+#define regBIF_BX_DEV0_EPF0_VF5_1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x00f7
+#define regBIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x00f9
+#define regBIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x00fa
+#define regBIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       2
+#define regBIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ                                                     0x0106
+#define regBIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE                                                    0x0107
+#define regBIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF5_1_BIF_TRANS_PENDING                                                     0x0108
+#define regBIF_BX_DEV0_EPF0_VF5_1_BIF_TRANS_PENDING_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF5_1_NBIF_GFX_ADDR_LUT_BYPASS                                              0x0112
+#define regBIF_BX_DEV0_EPF0_VF5_1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW0                                                0x0136
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW1                                                0x0137
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW2                                                0x0138
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW3                                                0x0139
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW0                                                0x013a
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW1                                                0x013b
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW2                                                0x013c
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW3                                                0x013d
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL                                                       0x013e
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_INT_CNTL                                                      0x013f
+#define regBIF_BX_DEV0_EPF0_VF5_1_MAILBOX_INT_CNTL_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX                                                      0x0140
+#define regBIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP                                                 0x0161
+#define regBIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP_BASE_IDX                                        2
+#define regBIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP                                                     0x0162
+#define regBIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_STATUS                                              0x0163
+#define regBIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_STATUS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_STATUS                                                  0x0164
+#define regBIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_STATUS_BASE_IDX                                         2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf5_SYSPFVFDEC:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF5_1_MM_INDEX                                                              0x0000
+#define regBIF_BX_DEV0_EPF0_VF5_1_MM_INDEX_BASE_IDX                                                     0
+#define regBIF_BX_DEV0_EPF0_VF5_1_MM_DATA                                                               0x0001
+#define regBIF_BX_DEV0_EPF0_VF5_1_MM_DATA_BASE_IDX                                                      0
+#define regBIF_BX_DEV0_EPF0_VF5_1_MM_INDEX_HI                                                           0x0006
+#define regBIF_BX_DEV0_EPF0_VF5_1_MM_INDEX_HI_BASE_IDX                                                  0
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf5_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF5_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF5_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf5_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_LO                                                      0x0410
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_HI                                                      0x0411
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_MSG_DATA                                                     0x0412
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_CONTROL                                                      0x0413
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_LO                                                      0x0414
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_HI                                                      0x0415
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_MSG_DATA                                                     0x0416
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_CONTROL                                                      0x0417
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_LO                                                      0x0418
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_HI                                                      0x0419
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_MSG_DATA                                                     0x041a
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_CONTROL                                                      0x041b
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_LO                                                      0x041c
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_HI                                                      0x041d
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_MSG_DATA                                                     0x041e
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_CONTROL                                                      0x041f
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_LO                                                      0x0420
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_HI                                                      0x0421
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_MSG_DATA                                                     0x0422
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_CONTROL                                                      0x0423
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF5_GFXMSIX_PBA_BASE_IDX                                                       3
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf6_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF6_0_BIF_BME_STATUS                                                        0x8e0b
+#define regBIF_BX_DEV0_EPF0_VF6_0_BIF_BME_STATUS_BASE_IDX                                               5
+#define regBIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG                                                    0x8e0c
+#define regBIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x8e13
+#define regBIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         5
+#define regBIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x8e14
+#define regBIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          5
+#define regBIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x8e15
+#define regBIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              5
+#define regBIF_BX_DEV0_EPF0_VF6_0_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x8e16
+#define regBIF_BX_DEV0_EPF0_VF6_0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x8e17
+#define regBIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x8e19
+#define regBIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            5
+#define regBIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x8e1a
+#define regBIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       5
+#define regBIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ                                                     0x8e26
+#define regBIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE                                                    0x8e27
+#define regBIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF6_0_BIF_TRANS_PENDING                                                     0x8e28
+#define regBIF_BX_DEV0_EPF0_VF6_0_BIF_TRANS_PENDING_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF6_0_NBIF_GFX_ADDR_LUT_BYPASS                                              0x8e32
+#define regBIF_BX_DEV0_EPF0_VF6_0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW0                                                0x8e56
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW1                                                0x8e57
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW2                                                0x8e58
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW3                                                0x8e59
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW0                                                0x8e5a
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW1                                                0x8e5b
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW2                                                0x8e5c
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW3                                                0x8e5d
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL                                                       0x8e5e
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL_BASE_IDX                                              5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_INT_CNTL                                                      0x8e5f
+#define regBIF_BX_DEV0_EPF0_VF6_0_MAILBOX_INT_CNTL_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX                                                      0x8e60
+#define regBIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP                                                 0x8e81
+#define regBIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP_BASE_IDX                                        5
+#define regBIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP                                                     0x8e82
+#define regBIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_STATUS                                              0x8e83
+#define regBIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_STATUS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_STATUS                                                  0x8e84
+#define regBIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_STATUS_BASE_IDX                                         5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf6_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF6_0_MM_INDEX                                                              0x8000
+#define regBIF_BX_DEV0_EPF0_VF6_0_MM_INDEX_BASE_IDX                                                     5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MM_DATA                                                               0x8001
+#define regBIF_BX_DEV0_EPF0_VF6_0_MM_DATA_BASE_IDX                                                      5
+#define regBIF_BX_DEV0_EPF0_VF6_0_MM_INDEX_HI                                                           0x8006
+#define regBIF_BX_DEV0_EPF0_VF6_0_MM_INDEX_HI_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf6_BIFPFVFDEC1:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF6_1_BIF_BME_STATUS                                                        0x00eb
+#define regBIF_BX_DEV0_EPF0_VF6_1_BIF_BME_STATUS_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG                                                    0x00ec
+#define regBIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x00f3
+#define regBIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x00f4
+#define regBIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          2
+#define regBIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x00f5
+#define regBIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF6_1_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x00f6
+#define regBIF_BX_DEV0_EPF0_VF6_1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x00f7
+#define regBIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x00f9
+#define regBIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x00fa
+#define regBIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       2
+#define regBIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ                                                     0x0106
+#define regBIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE                                                    0x0107
+#define regBIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF6_1_BIF_TRANS_PENDING                                                     0x0108
+#define regBIF_BX_DEV0_EPF0_VF6_1_BIF_TRANS_PENDING_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF6_1_NBIF_GFX_ADDR_LUT_BYPASS                                              0x0112
+#define regBIF_BX_DEV0_EPF0_VF6_1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW0                                                0x0136
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW1                                                0x0137
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW2                                                0x0138
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW3                                                0x0139
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW0                                                0x013a
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW1                                                0x013b
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW2                                                0x013c
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW3                                                0x013d
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL                                                       0x013e
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_INT_CNTL                                                      0x013f
+#define regBIF_BX_DEV0_EPF0_VF6_1_MAILBOX_INT_CNTL_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX                                                      0x0140
+#define regBIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP                                                 0x0161
+#define regBIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP_BASE_IDX                                        2
+#define regBIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP                                                     0x0162
+#define regBIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_STATUS                                              0x0163
+#define regBIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_STATUS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_STATUS                                                  0x0164
+#define regBIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_STATUS_BASE_IDX                                         2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf6_SYSPFVFDEC:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF6_1_MM_INDEX                                                              0x0000
+#define regBIF_BX_DEV0_EPF0_VF6_1_MM_INDEX_BASE_IDX                                                     0
+#define regBIF_BX_DEV0_EPF0_VF6_1_MM_DATA                                                               0x0001
+#define regBIF_BX_DEV0_EPF0_VF6_1_MM_DATA_BASE_IDX                                                      0
+#define regBIF_BX_DEV0_EPF0_VF6_1_MM_INDEX_HI                                                           0x0006
+#define regBIF_BX_DEV0_EPF0_VF6_1_MM_INDEX_HI_BASE_IDX                                                  0
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf6_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF6_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF6_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf6_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_LO                                                      0x0410
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_HI                                                      0x0411
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_MSG_DATA                                                     0x0412
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_CONTROL                                                      0x0413
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_LO                                                      0x0414
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_HI                                                      0x0415
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_MSG_DATA                                                     0x0416
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_CONTROL                                                      0x0417
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_LO                                                      0x0418
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_HI                                                      0x0419
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_MSG_DATA                                                     0x041a
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_CONTROL                                                      0x041b
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_LO                                                      0x041c
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_HI                                                      0x041d
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_MSG_DATA                                                     0x041e
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_CONTROL                                                      0x041f
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_LO                                                      0x0420
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_HI                                                      0x0421
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_MSG_DATA                                                     0x0422
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_CONTROL                                                      0x0423
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF6_GFXMSIX_PBA_BASE_IDX                                                       3
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf7_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF7_0_BIF_BME_STATUS                                                        0x8e0b
+#define regBIF_BX_DEV0_EPF0_VF7_0_BIF_BME_STATUS_BASE_IDX                                               5
+#define regBIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG                                                    0x8e0c
+#define regBIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x8e13
+#define regBIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         5
+#define regBIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x8e14
+#define regBIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          5
+#define regBIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x8e15
+#define regBIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              5
+#define regBIF_BX_DEV0_EPF0_VF7_0_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x8e16
+#define regBIF_BX_DEV0_EPF0_VF7_0_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x8e17
+#define regBIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 5
+#define regBIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x8e19
+#define regBIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            5
+#define regBIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x8e1a
+#define regBIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       5
+#define regBIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ                                                     0x8e26
+#define regBIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE                                                    0x8e27
+#define regBIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE_BASE_IDX                                           5
+#define regBIF_BX_DEV0_EPF0_VF7_0_BIF_TRANS_PENDING                                                     0x8e28
+#define regBIF_BX_DEV0_EPF0_VF7_0_BIF_TRANS_PENDING_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF7_0_NBIF_GFX_ADDR_LUT_BYPASS                                              0x8e32
+#define regBIF_BX_DEV0_EPF0_VF7_0_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW0                                                0x8e56
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW1                                                0x8e57
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW2                                                0x8e58
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW3                                                0x8e59
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW0                                                0x8e5a
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW1                                                0x8e5b
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW2                                                0x8e5c
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW3                                                0x8e5d
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL                                                       0x8e5e
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL_BASE_IDX                                              5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_INT_CNTL                                                      0x8e5f
+#define regBIF_BX_DEV0_EPF0_VF7_0_MAILBOX_INT_CNTL_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX                                                      0x8e60
+#define regBIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX_BASE_IDX                                             5
+#define regBIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP                                                 0x8e81
+#define regBIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP_BASE_IDX                                        5
+#define regBIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP                                                     0x8e82
+#define regBIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP_BASE_IDX                                            5
+#define regBIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_STATUS                                              0x8e83
+#define regBIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_STATUS_BASE_IDX                                     5
+#define regBIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_STATUS                                                  0x8e84
+#define regBIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_STATUS_BASE_IDX                                         5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf7_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_DEV0_EPF0_VF7_0_MM_INDEX                                                              0x8000
+#define regBIF_BX_DEV0_EPF0_VF7_0_MM_INDEX_BASE_IDX                                                     5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MM_DATA                                                               0x8001
+#define regBIF_BX_DEV0_EPF0_VF7_0_MM_DATA_BASE_IDX                                                      5
+#define regBIF_BX_DEV0_EPF0_VF7_0_MM_INDEX_HI                                                           0x8006
+#define regBIF_BX_DEV0_EPF0_VF7_0_MM_INDEX_HI_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf7_BIFPFVFDEC1:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF7_1_BIF_BME_STATUS                                                        0x00eb
+#define regBIF_BX_DEV0_EPF0_VF7_1_BIF_BME_STATUS_BASE_IDX                                               2
+#define regBIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG                                                    0x00ec
+#define regBIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                  0x00f3
+#define regBIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                         2
+#define regBIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                   0x00f4
+#define regBIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                          2
+#define regBIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_CNTL                                       0x00f5
+#define regBIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                              2
+#define regBIF_BX_DEV0_EPF0_VF7_1_HDP_REG_COHERENCY_FLUSH_CNTL                                          0x00f6
+#define regBIF_BX_DEV0_EPF0_VF7_1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_CNTL                                          0x00f7
+#define regBIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                 2
+#define regBIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                     0x00f9
+#define regBIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                            2
+#define regBIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                0x00fa
+#define regBIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                       2
+#define regBIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ                                                     0x0106
+#define regBIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE                                                    0x0107
+#define regBIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE_BASE_IDX                                           2
+#define regBIF_BX_DEV0_EPF0_VF7_1_BIF_TRANS_PENDING                                                     0x0108
+#define regBIF_BX_DEV0_EPF0_VF7_1_BIF_TRANS_PENDING_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF7_1_NBIF_GFX_ADDR_LUT_BYPASS                                              0x0112
+#define regBIF_BX_DEV0_EPF0_VF7_1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW0                                                0x0136
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW1                                                0x0137
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW2                                                0x0138
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW3                                                0x0139
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW0                                                0x013a
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW1                                                0x013b
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW2                                                0x013c
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW3                                                0x013d
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                       2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL                                                       0x013e
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL_BASE_IDX                                              2
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_INT_CNTL                                                      0x013f
+#define regBIF_BX_DEV0_EPF0_VF7_1_MAILBOX_INT_CNTL_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX                                                      0x0140
+#define regBIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX_BASE_IDX                                             2
+#define regBIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP                                                 0x0161
+#define regBIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP_BASE_IDX                                        2
+#define regBIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP                                                     0x0162
+#define regBIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP_BASE_IDX                                            2
+#define regBIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_STATUS                                              0x0163
+#define regBIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_STATUS_BASE_IDX                                     2
+#define regBIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_STATUS                                                  0x0164
+#define regBIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_STATUS_BASE_IDX                                         2
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf7_SYSPFVFDEC:1
+// base address: 0x0
+#define regBIF_BX_DEV0_EPF0_VF7_1_MM_INDEX                                                              0x0000
+#define regBIF_BX_DEV0_EPF0_VF7_1_MM_INDEX_BASE_IDX                                                     0
+#define regBIF_BX_DEV0_EPF0_VF7_1_MM_DATA                                                               0x0001
+#define regBIF_BX_DEV0_EPF0_VF7_1_MM_DATA_BASE_IDX                                                      0
+#define regBIF_BX_DEV0_EPF0_VF7_1_MM_INDEX_HI                                                           0x0006
+#define regBIF_BX_DEV0_EPF0_VF7_1_MM_INDEX_HI_BASE_IDX                                                  0
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf7_BIFPFVFDEC1
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF7_RCC_ERR_LOG                                                                0x0085
+#define regRCC_DEV0_EPF0_VF7_RCC_ERR_LOG_BASE_IDX                                                       2
+#define regRCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN                                                       0x00c0
+#define regRCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN_BASE_IDX                                              2
+#define regRCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE                                                         0x00c3
+#define regRCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE_BASE_IDX                                                2
+#define regRCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED                                                        0x00c4
+#define regRCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED_BASE_IDX                                               2
+#define regRCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER                                                    0x00c5
+#define regRCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER_BASE_IDX                                           2
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf7_BIFDEC2
+// base address: 0x0
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO                                                      0x0400
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI                                                      0x0401
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA                                                     0x0402
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL                                                      0x0403
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO                                                      0x0404
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI                                                      0x0405
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA                                                     0x0406
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL                                                      0x0407
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO                                                      0x0408
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI                                                      0x0409
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA                                                     0x040a
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL                                                      0x040b
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO                                                      0x040c
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI                                                      0x040d
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA                                                     0x040e
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL                                                      0x040f
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_LO                                                      0x0410
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_HI                                                      0x0411
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_MSG_DATA                                                     0x0412
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_CONTROL                                                      0x0413
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_LO                                                      0x0414
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_HI                                                      0x0415
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_MSG_DATA                                                     0x0416
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_CONTROL                                                      0x0417
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_LO                                                      0x0418
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_HI                                                      0x0419
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_MSG_DATA                                                     0x041a
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_CONTROL                                                      0x041b
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_LO                                                      0x041c
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_HI                                                      0x041d
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_MSG_DATA                                                     0x041e
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_CONTROL                                                      0x041f
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_LO                                                      0x0420
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_LO_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_HI                                                      0x0421
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_HI_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_MSG_DATA                                                     0x0422
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_MSG_DATA_BASE_IDX                                            3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_CONTROL                                                      0x0423
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_CONTROL_BASE_IDX                                             3
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_PBA                                                                0x0800
+#define regRCC_DEV0_EPF0_VF7_GFXMSIX_PBA_BASE_IDX                                                       3
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+// base address: 0x10141000
+#define regBIF_CFG_DEV0_EPF1_VENDOR_ID                                                                  0x10400
+#define regBIF_CFG_DEV0_EPF1_VENDOR_ID_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_ID                                                                  0x10400
+#define regBIF_CFG_DEV0_EPF1_DEVICE_ID_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_COMMAND                                                                    0x10401
+#define regBIF_CFG_DEV0_EPF1_COMMAND_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF1_STATUS                                                                     0x10401
+#define regBIF_CFG_DEV0_EPF1_STATUS_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF1_REVISION_ID                                                                0x10402
+#define regBIF_CFG_DEV0_EPF1_REVISION_ID_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_PROG_INTERFACE                                                             0x10402
+#define regBIF_CFG_DEV0_EPF1_PROG_INTERFACE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_SUB_CLASS                                                                  0x10402
+#define regBIF_CFG_DEV0_EPF1_SUB_CLASS_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_BASE_CLASS                                                                 0x10402
+#define regBIF_CFG_DEV0_EPF1_BASE_CLASS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_CACHE_LINE                                                                 0x10403
+#define regBIF_CFG_DEV0_EPF1_CACHE_LINE_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_LATENCY                                                                    0x10403
+#define regBIF_CFG_DEV0_EPF1_LATENCY_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF1_HEADER                                                                     0x10403
+#define regBIF_CFG_DEV0_EPF1_HEADER_BASE_IDX                                                            5
+#define regBIF_CFG_DEV0_EPF1_BIST                                                                       0x10403
+#define regBIF_CFG_DEV0_EPF1_BIST_BASE_IDX                                                              5
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_1                                                                0x10404
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_1_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_2                                                                0x10405
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_3                                                                0x10406
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_3_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_4                                                                0x10407
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_4_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_5                                                                0x10408
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_5_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_6                                                                0x10409
+#define regBIF_CFG_DEV0_EPF1_BASE_ADDR_6_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR                                                            0x1040a
+#define regBIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_ADAPTER_ID                                                                 0x1040b
+#define regBIF_CFG_DEV0_EPF1_ADAPTER_ID_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_ROM_BASE_ADDR                                                              0x1040c
+#define regBIF_CFG_DEV0_EPF1_ROM_BASE_ADDR_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_CAP_PTR                                                                    0x1040d
+#define regBIF_CFG_DEV0_EPF1_CAP_PTR_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF1_INTERRUPT_LINE                                                             0x1040f
+#define regBIF_CFG_DEV0_EPF1_INTERRUPT_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_INTERRUPT_PIN                                                              0x1040f
+#define regBIF_CFG_DEV0_EPF1_INTERRUPT_PIN_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_MIN_GRANT                                                                  0x1040f
+#define regBIF_CFG_DEV0_EPF1_MIN_GRANT_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_MAX_LATENCY                                                                0x1040f
+#define regBIF_CFG_DEV0_EPF1_MAX_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST                                                            0x10412
+#define regBIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_ADAPTER_ID_W                                                               0x10413
+#define regBIF_CFG_DEV0_EPF1_ADAPTER_ID_W_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_PMI_CAP_LIST                                                               0x10414
+#define regBIF_CFG_DEV0_EPF1_PMI_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_PMI_CAP                                                                    0x10414
+#define regBIF_CFG_DEV0_EPF1_PMI_CAP_BASE_IDX                                                           5
+#define regBIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL                                                            0x10415
+#define regBIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL_BASE_IDX                                                   5
+#define regSBRN                                                                                         0x10418
+#define regSBRN_BASE_IDX                                                                                5
+#define regFLADJ                                                                                        0x10418
+#define regFLADJ_BASE_IDX                                                                               5
+#define regBIF_CFG_DEV0_EPF1_PCIE_CAP_LIST                                                              0x10419
+#define regBIF_CFG_DEV0_EPF1_PCIE_CAP_LIST_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_CAP                                                                   0x10419
+#define regBIF_CFG_DEV0_EPF1_PCIE_CAP_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CAP                                                                 0x1041a
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CAP_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CNTL                                                                0x1041b
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CNTL_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_STATUS                                                              0x1041b
+#define regBIF_CFG_DEV0_EPF1_DEVICE_STATUS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_LINK_CAP                                                                   0x1041c
+#define regBIF_CFG_DEV0_EPF1_LINK_CAP_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF1_LINK_CNTL                                                                  0x1041d
+#define regBIF_CFG_DEV0_EPF1_LINK_CNTL_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_LINK_STATUS                                                                0x1041d
+#define regBIF_CFG_DEV0_EPF1_LINK_STATUS_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CAP2                                                                0x10422
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CAP2_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CNTL2                                                               0x10423
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CNTL2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_STATUS2                                                             0x10423
+#define regBIF_CFG_DEV0_EPF1_DEVICE_STATUS2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_LINK_CAP2                                                                  0x10424
+#define regBIF_CFG_DEV0_EPF1_LINK_CAP2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_LINK_CNTL2                                                                 0x10425
+#define regBIF_CFG_DEV0_EPF1_LINK_CNTL2_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_LINK_STATUS2                                                               0x10425
+#define regBIF_CFG_DEV0_EPF1_LINK_STATUS2_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_MSI_CAP_LIST                                                               0x10428
+#define regBIF_CFG_DEV0_EPF1_MSI_CAP_LIST_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_CNTL                                                               0x10428
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_CNTL_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO                                                            0x10429
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI                                                            0x1042a
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_DATA                                                               0x1042a
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_DATA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA                                                           0x1042a
+#define regBIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_MSI_MASK                                                                   0x1042b
+#define regBIF_CFG_DEV0_EPF1_MSI_MASK_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64                                                            0x1042b
+#define regBIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64                                                        0x1042b
+#define regBIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF1_MSI_MASK_64                                                                0x1042c
+#define regBIF_CFG_DEV0_EPF1_MSI_MASK_64_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_MSI_PENDING                                                                0x1042c
+#define regBIF_CFG_DEV0_EPF1_MSI_PENDING_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_MSI_PENDING_64                                                             0x1042d
+#define regBIF_CFG_DEV0_EPF1_MSI_PENDING_64_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_MSIX_CAP_LIST                                                              0x10430
+#define regBIF_CFG_DEV0_EPF1_MSIX_CAP_LIST_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL                                                              0x10430
+#define regBIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_MSIX_TABLE                                                                 0x10431
+#define regBIF_CFG_DEV0_EPF1_MSIX_TABLE_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_MSIX_PBA                                                                   0x10432
+#define regBIF_CFG_DEV0_EPF1_MSIX_PBA_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                          0x10440
+#define regBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR                                                   0x10441
+#define regBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1                                                      0x10442
+#define regBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2                                                      0x10443
+#define regBIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DEVICE3_ENH_CAP_LIST                                                  0x104b6
+#define regBIF_CFG_DEV0_EPF1_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CAP3                                                                0x104b7
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CAP3_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CNTL3                                                               0x104b8
+#define regBIF_CFG_DEV0_EPF1_DEVICE_CNTL3_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_DEVICE_STATUS3                                                             0x104b9
+#define regBIF_CFG_DEV0_EPF1_DEVICE_STATUS3_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST                                                    0x10500
+#define regBIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST                                                    0x10580
+#define regBIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                              0x10581
+#define regBIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS                                                     0x10582
+#define regBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK                                                       0x10583
+#define regBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY                                                   0x10584
+#define regBIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS                                                       0x10585
+#define regBIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK                                                         0x10586
+#define regBIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL                                                      0x10587
+#define regBIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0                                                              0x10588
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1                                                              0x10589
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2                                                              0x1058a
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3                                                              0x1058b
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG4                                                              0x1058f
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG4_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0                                                       0x1058f
+#define regBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG5                                                              0x10590
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG5_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1                                                       0x10590
+#define regBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG6                                                              0x10591
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG6_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2                                                       0x10591
+#define regBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG7                                                              0x10592
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG7_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3                                                       0x10592
+#define regBIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG8                                                              0x10593
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG8_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG9                                                              0x10594
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG9_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG10                                                             0x10595
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG10_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG11                                                             0x10596
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG11_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG12                                                             0x10597
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG12_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG13                                                             0x10598
+#define regBIF_CFG_DEV0_EPF1_PCIE_HDR_LOG13_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST                                                      0x105b7
+#define regBIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_PCIE_ACS_CAP                                                               0x105b8
+#define regBIF_CFG_DEV0_EPF1_PCIE_ACS_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL                                                              0x105b8
+#define regBIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST                                                      0x105c7
+#define regBIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_RTR_DATA1                                                                  0x105c8
+#define regBIF_CFG_DEV0_EPF1_RTR_DATA1_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_RTR_DATA2                                                                  0x105c9
+#define regBIF_CFG_DEV0_EPF1_RTR_DATA2_BASE_IDX                                                         5
+#define regBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST                                               0x105ca
+#define regBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT                                                0x105cb
+#define regBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA                                                       0x105cc
+#define regBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP                                                        0x105cd
+#define regBIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST                                                      0x105ce
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_CAP                                                               0x105cf
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR                                                 0x105d0
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS                                                            0x105d1
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL                                                              0x105d1
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0                                              0x105d2
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1                                              0x105d2
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2                                              0x105d2
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3                                              0x105d2
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4                                              0x105d3
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5                                              0x105d3
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6                                              0x105d3
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7                                              0x105d3
+#define regBIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST                                                      0x105dc
+#define regBIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_PCIE_ARI_CAP                                                               0x105dd
+#define regBIF_CFG_DEV0_EPF1_PCIE_ARI_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL                                                              0x105dd
+#define regBIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST                                                      0x105de
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST_BASE_IDX                                             5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP                                                              0x105df
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL                                                             0x105e0
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP                                                              0x105e1
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL                                                             0x105e2
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP                                                              0x105e3
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL                                                             0x105e4
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP                                                              0x105e5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL                                                             0x105e6
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP                                                              0x105e7
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL                                                             0x105e8
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP                                                              0x105e9
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL                                                             0x105ea
+#define regBIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST                                                    0x105f1
+#define regBIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_PASID_CAP                                                             0x105f2
+#define regBIF_CFG_DEV0_EPF1_PCIE_PASID_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL                                                            0x105f2
+#define regBIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_ENH_CAP_LIST                                                    0x105f3
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_ENH_CAP_LIST_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP                                                             0x105f4
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL                                                         0x105f5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_STATUS                                                          0x105f5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_INITIAL_VFS                                                     0x105f6
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_INITIAL_VFS_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_TOTAL_VFS                                                       0x105f6
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_TOTAL_VFS_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_NUM_VFS                                                         0x105f7
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_NUM_VFS_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_FUNC_DEP_LINK                                                   0x105f7
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_FUNC_DEP_LINK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_FIRST_VF_OFFSET                                                 0x105f8
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_FIRST_VF_OFFSET_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_STRIDE                                                       0x105f8
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_STRIDE_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_DEVICE_ID                                                    0x105f9
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_DEVICE_ID_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_SUPPORTED_PAGE_SIZE                                             0x105fa
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_SUPPORTED_PAGE_SIZE_BASE_IDX                                    5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_SYSTEM_PAGE_SIZE                                                0x105fb
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_SYSTEM_PAGE_SIZE_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_0                                                  0x105fc
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_0_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_1                                                  0x105fd
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_2                                                  0x105fe
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_3                                                  0x105ff
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_3_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_4                                                  0x10600
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_4_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_5                                                  0x10601
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_5_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET                                 0x10602
+#define regBIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_BASE_IDX                        5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST                                            0x10638
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST_BASE_IDX                                   5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CAP                                                    0x10639
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL                                                   0x1063a
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CAP                                                    0x1063b
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL                                                   0x1063c
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CAP                                                    0x1063d
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL                                                   0x1063e
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CAP                                                    0x1063f
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL                                                   0x10640
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CAP                                                    0x10641
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL                                                   0x10642
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CAP                                                    0x10643
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CAP_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL                                                   0x10644
+#define regBIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL_BASE_IDX                                          5
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf1_vf0_bifcfgdecp
+// base address: 0x10141000
+#define regBIF_CFG_DEV0_EPF1_VF0_VENDOR_ID                                                              0x10400
+#define regBIF_CFG_DEV0_EPF1_VF0_VENDOR_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_ID                                                              0x10400
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_ID_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_VF0_COMMAND                                                                0x10401
+#define regBIF_CFG_DEV0_EPF1_VF0_COMMAND_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_VF0_STATUS                                                                 0x10401
+#define regBIF_CFG_DEV0_EPF1_VF0_STATUS_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_VF0_REVISION_ID                                                            0x10402
+#define regBIF_CFG_DEV0_EPF1_VF0_REVISION_ID_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_PROG_INTERFACE                                                         0x10402
+#define regBIF_CFG_DEV0_EPF1_VF0_PROG_INTERFACE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_VF0_SUB_CLASS                                                              0x10402
+#define regBIF_CFG_DEV0_EPF1_VF0_SUB_CLASS_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_CLASS                                                             0x10402
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_CLASS_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_VF0_CACHE_LINE                                                             0x10403
+#define regBIF_CFG_DEV0_EPF1_VF0_CACHE_LINE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_VF0_LATENCY                                                                0x10403
+#define regBIF_CFG_DEV0_EPF1_VF0_LATENCY_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_VF0_HEADER                                                                 0x10403
+#define regBIF_CFG_DEV0_EPF1_VF0_HEADER_BASE_IDX                                                        5
+#define regBIF_CFG_DEV0_EPF1_VF0_BIST                                                                   0x10403
+#define regBIF_CFG_DEV0_EPF1_VF0_BIST_BASE_IDX                                                          5
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_1                                                            0x10404
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_1_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_2                                                            0x10405
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_3                                                            0x10406
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_4                                                            0x10407
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_4_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_5                                                            0x10408
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_5_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_6                                                            0x10409
+#define regBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_6_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_CARDBUS_CIS_PTR                                                        0x1040a
+#define regBIF_CFG_DEV0_EPF1_VF0_CARDBUS_CIS_PTR_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF1_VF0_ADAPTER_ID                                                             0x1040b
+#define regBIF_CFG_DEV0_EPF1_VF0_ADAPTER_ID_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR                                                          0x1040c
+#define regBIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_CAP_PTR                                                                0x1040d
+#define regBIF_CFG_DEV0_EPF1_VF0_CAP_PTR_BASE_IDX                                                       5
+#define regBIF_CFG_DEV0_EPF1_VF0_INTERRUPT_LINE                                                         0x1040f
+#define regBIF_CFG_DEV0_EPF1_VF0_INTERRUPT_LINE_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_VF0_INTERRUPT_PIN                                                          0x1040f
+#define regBIF_CFG_DEV0_EPF1_VF0_INTERRUPT_PIN_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_MIN_GRANT                                                              0x1040f
+#define regBIF_CFG_DEV0_EPF1_VF0_MIN_GRANT_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_VF0_MAX_LATENCY                                                            0x1040f
+#define regBIF_CFG_DEV0_EPF1_VF0_MAX_LATENCY_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_CAP_LIST                                                          0x10419
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_CAP                                                               0x10419
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP                                                             0x1041a
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL                                                            0x1041b
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS                                                          0x1041b
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_CAP                                                               0x1041c
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_CAP_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_CNTL                                                              0x1041d
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_CNTL_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_STATUS                                                            0x1041d
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_STATUS_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2                                                            0x10422
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2                                                           0x10423
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS2                                                         0x10423
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS2_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_CAP2                                                              0x10424
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_CAP2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2                                                             0x10425
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2                                                           0x10425
+#define regBIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_CAP_LIST                                                           0x10428
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_CAP_LIST_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL                                                           0x10428
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_LO                                                        0x10429
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_LO_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_HI                                                        0x1042a
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_HI_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA                                                           0x1042a
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA                                                       0x1042a
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA_BASE_IDX                                              5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MASK                                                               0x1042b
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MASK_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA_64                                                        0x1042b
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA_64_BASE_IDX                                               5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA_64                                                    0x1042b
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA_64_BASE_IDX                                           5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MASK_64                                                            0x1042c
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_MASK_64_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_PENDING                                                            0x1042c
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_PENDING_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_PENDING_64                                                         0x1042d
+#define regBIF_CFG_DEV0_EPF1_VF0_MSI_PENDING_64_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSIX_CAP_LIST                                                          0x10430
+#define regBIF_CFG_DEV0_EPF1_VF0_MSIX_CAP_LIST_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL                                                          0x10430
+#define regBIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSIX_TABLE                                                             0x10431
+#define regBIF_CFG_DEV0_EPF1_VF0_MSIX_TABLE_BASE_IDX                                                    5
+#define regBIF_CFG_DEV0_EPF1_VF0_MSIX_PBA                                                               0x10432
+#define regBIF_CFG_DEV0_EPF1_VF0_MSIX_PBA_BASE_IDX                                                      5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x10440
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_BASE_IDX                             5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR                                               0x10441
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC1                                                  0x10442
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC1_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC2                                                  0x10443
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC2_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST                                              0x104b6
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST_BASE_IDX                                     5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3                                                            0x104b7
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3_BASE_IDX                                                   5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3                                                           0x104b8
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3                                                         0x104b9
+#define regBIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST                                                0x10500
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST                                                0x10580
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST_BASE_IDX                                       5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x10581
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST_BASE_IDX                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS                                                 0x10582
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS_BASE_IDX                                        5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK                                                   0x10583
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY                                               0x10584
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY_BASE_IDX                                      5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS                                                   0x10585
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK                                                     0x10586
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK_BASE_IDX                                            5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL                                                  0x10587
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG0                                                          0x10588
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG0_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG1                                                          0x10589
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG1_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG2                                                          0x1058a
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG2_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG3                                                          0x1058b
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG3_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG4                                                          0x1058f
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG4_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG0                                                   0x1058f
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG0_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG5                                                          0x10590
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG5_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG1                                                   0x10590
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG1_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG6                                                          0x10591
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG6_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG2                                                   0x10591
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG2_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG7                                                          0x10592
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG7_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG3                                                   0x10592
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG3_BASE_IDX                                          5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG8                                                          0x10593
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG8_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG9                                                          0x10594
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG9_BASE_IDX                                                 5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG10                                                         0x10595
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG10_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG11                                                         0x10596
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG11_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG12                                                         0x10597
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG12_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG13                                                         0x10598
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG13_BASE_IDX                                                5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST                                                  0x105c7
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_VF0_RTR_DATA1                                                              0x105c8
+#define regBIF_CFG_DEV0_EPF1_VF0_RTR_DATA1_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_VF0_RTR_DATA2                                                              0x105c9
+#define regBIF_CFG_DEV0_EPF1_VF0_RTR_DATA2_BASE_IDX                                                     5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST                                                  0x105dc
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST_BASE_IDX                                         5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP                                                           0x105dd
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP_BASE_IDX                                                  5
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL                                                          0x105dd
+#define regBIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL_BASE_IDX                                                 5
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_DEV0_1_RCC_VDM_SUPPORT                                                                   0xc440
+#define regRCC_DEV0_1_RCC_VDM_SUPPORT_BASE_IDX                                                          5
+#define regRCC_DEV0_1_RCC_BUS_CNTL                                                                      0xc441
+#define regRCC_DEV0_1_RCC_BUS_CNTL_BASE_IDX                                                             5
+#define regRCC_DEV0_1_RCC_FEATURES_CONTROL_MISC                                                         0xc442
+#define regRCC_DEV0_1_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                5
+#define regRCC_DEV0_1_RCC_DEV0_LINK_CNTL                                                                0xc443
+#define regRCC_DEV0_1_RCC_DEV0_LINK_CNTL_BASE_IDX                                                       5
+#define regRCC_DEV0_1_RCC_CMN_LINK_CNTL                                                                 0xc444
+#define regRCC_DEV0_1_RCC_CMN_LINK_CNTL_BASE_IDX                                                        5
+#define regRCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE                                                        0xc445
+#define regRCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                               5
+#define regRCC_DEV0_1_RCC_LTR_LSWITCH_CNTL                                                              0xc446
+#define regRCC_DEV0_1_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                     5
+#define regRCC_DEV0_1_RCC_MH_ARB_CNTL                                                                   0xc447
+#define regRCC_DEV0_1_RCC_MH_ARB_CNTL_BASE_IDX                                                          5
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0                                                            0xc448
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1                                                            0xc449
+#define regRCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                   5
+
+
+// addressBlock: nbif0_nbif0_rcc_ep_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_EP_DEV0_1_EP_PCIE_SCRATCH                                                                0xc44c
+#define regRCC_EP_DEV0_1_EP_PCIE_SCRATCH_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_1_EP_PCIE_CNTL                                                                   0xc44e
+#define regRCC_EP_DEV0_1_EP_PCIE_CNTL_BASE_IDX                                                          5
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_CNTL                                                               0xc44f
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_STATUS                                                             0xc450
+#define regRCC_EP_DEV0_1_EP_PCIE_INT_STATUS_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL2                                                               0xc451
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL2_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_BUS_CNTL                                                               0xc452
+#define regRCC_EP_DEV0_1_EP_PCIE_BUS_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_CFG_CNTL                                                               0xc453
+#define regRCC_EP_DEV0_1_EP_PCIE_CFG_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL                                                            0xc454
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC                                                             0xc455
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2                                                            0xc456
+#define regRCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP                                                             0xc457
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0xc458
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL                                                            0xc458
+#define regRCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0xc458
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0xc459
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0xc459
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0xc459
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0xc459
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0xc45a
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0xc45a
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0xc45a
+#define regRCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_PME_CONTROL                                                            0xc45c
+#define regRCC_EP_DEV0_1_EP_PCIE_PME_CONTROL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_1_EP_PCIEP_RESERVED                                                              0xc45d
+#define regRCC_EP_DEV0_1_EP_PCIEP_RESERVED_BASE_IDX                                                     5
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_CNTL                                                                0xc45f
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID                                                        0xc460
+#define regRCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                               5
+#define regRCC_EP_DEV0_1_EP_PCIE_ERR_CNTL                                                               0xc461
+#define regRCC_EP_DEV0_1_EP_PCIE_ERR_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL                                                                0xc462
+#define regRCC_EP_DEV0_1_EP_PCIE_RX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL                                                          0xc463
+#define regRCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                 5
+#define regRCC_EP_DEV0_1_EP_PCIE_DEVICE_CNTL3                                                           0xc464
+#define regRCC_EP_DEV0_1_EP_PCIE_DEVICE_CNTL3_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_rcc_dwn_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_DWN_DEV0_1_DN_PCIE_RESERVED                                                              0xc468
+#define regRCC_DWN_DEV0_1_DN_PCIE_RESERVED_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_SCRATCH                                                               0xc469
+#define regRCC_DWN_DEV0_1_DN_PCIE_SCRATCH_BASE_IDX                                                      5
+#define regRCC_DWN_DEV0_1_DN_PCIE_CNTL                                                                  0xc46b
+#define regRCC_DWN_DEV0_1_DN_PCIE_CNTL_BASE_IDX                                                         5
+#define regRCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL                                                           0xc46c
+#define regRCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                  5
+#define regRCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2                                                              0xc46d
+#define regRCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL                                                              0xc46e
+#define regRCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL                                                              0xc46f
+#define regRCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_F0                                                              0xc470
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_F0_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC                                                            0xc471
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC_BASE_IDX                                                   5
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2                                                           0xc472
+#define regRCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_rcc_dwnp_dev0_RCCPORTDEC
+// base address: 0x10131000
+#define regRCC_DWNP_DEV0_1_PCIE_ERR_CNTL                                                                0xc475
+#define regRCC_DWNP_DEV0_1_PCIE_ERR_CNTL_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_1_PCIE_RX_CNTL                                                                 0xc476
+#define regRCC_DWNP_DEV0_1_PCIE_RX_CNTL_BASE_IDX                                                        5
+#define regRCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL                                                           0xc477
+#define regRCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL_BASE_IDX                                                  5
+#define regRCC_DWNP_DEV0_1_PCIE_LC_CNTL2                                                                0xc478
+#define regRCC_DWNP_DEV0_1_PCIE_LC_CNTL2_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_1_PCIEP_STRAP_MISC                                                             0xc479
+#define regRCC_DWNP_DEV0_1_PCIEP_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP                                                         0xc47a
+#define regRCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                5
+
+
+// addressBlock: nbif0_nbif0_rcc_pfc_amdgfx_RCCPFCDEC
+// base address: 0x10134000
+#define regRCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL                                                              0xd040
+#define regRCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL_BASE_IDX                                                     5
+#define regRCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE                                                           0xd041
+#define regRCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE_BASE_IDX                                                  5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0                                                      0xd042
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1                                                      0xd043
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2                                                      0xd044
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3                                                      0xd045
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4                                                      0xd046
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5                                                      0xd047
+#define regRCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5_BASE_IDX                                             5
+#define regRCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL                                                           0xd048
+#define regRCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL_BASE_IDX                                                  5
+#define regRCC_PFC_AMDGFX_RCC_PFC_MISC_CNTL                                                             0xd049
+#define regRCC_PFC_AMDGFX_RCC_PFC_MISC_CNTL_BASE_IDX                                                    5
+#define regRCC_PFC_AMDGFX_RCC_NFC                                                                       0xd04a
+#define regRCC_PFC_AMDGFX_RCC_NFC_BASE_IDX                                                              5
+
+
+// addressBlock: nbif0_nbif0_rcc_shadow_reg_shadowdec
+// base address: 0x10130000
+#define regSHADOW_COMMAND                                                                               0xc001
+#define regSHADOW_COMMAND_BASE_IDX                                                                      5
+#define regSHADOW_BASE_ADDR_1                                                                           0xc004
+#define regSHADOW_BASE_ADDR_1_BASE_IDX                                                                  5
+#define regSHADOW_BASE_ADDR_2                                                                           0xc005
+#define regSHADOW_BASE_ADDR_2_BASE_IDX                                                                  5
+#define regSHADOW_SUB_BUS_NUMBER_LATENCY                                                                0xc006
+#define regSHADOW_SUB_BUS_NUMBER_LATENCY_BASE_IDX                                                       5
+#define regSHADOW_IO_BASE_LIMIT                                                                         0xc007
+#define regSHADOW_IO_BASE_LIMIT_BASE_IDX                                                                5
+#define regSHADOW_MEM_BASE_LIMIT                                                                        0xc008
+#define regSHADOW_MEM_BASE_LIMIT_BASE_IDX                                                               5
+#define regSHADOW_PREF_BASE_LIMIT                                                                       0xc009
+#define regSHADOW_PREF_BASE_LIMIT_BASE_IDX                                                              5
+#define regSHADOW_PREF_BASE_UPPER                                                                       0xc00a
+#define regSHADOW_PREF_BASE_UPPER_BASE_IDX                                                              5
+#define regSHADOW_PREF_LIMIT_UPPER                                                                      0xc00b
+#define regSHADOW_PREF_LIMIT_UPPER_BASE_IDX                                                             5
+#define regSHADOW_IO_BASE_LIMIT_HI                                                                      0xc00c
+#define regSHADOW_IO_BASE_LIMIT_HI_BASE_IDX                                                             5
+#define regSHADOW_IRQ_BRIDGE_CNTL                                                                       0xc00f
+#define regSHADOW_IRQ_BRIDGE_CNTL_BASE_IDX                                                              5
+#define regSUC_INDEX                                                                                    0xc038
+#define regSUC_INDEX_BASE_IDX                                                                           5
+#define regSUC_DATA                                                                                     0xc039
+#define regSUC_DATA_BASE_IDX                                                                            5
+
+
+// addressBlock: nbif0_nbif0_bif_swus_SUMDEC
+// base address: 0x1013b000
+#define regSUM_INDEX                                                                                    0xec38
+#define regSUM_INDEX_BASE_IDX                                                                           5
+#define regSUM_DATA                                                                                     0xec39
+#define regSUM_DATA_BASE_IDX                                                                            5
+#define regSUM_INDEX_HI                                                                                 0xec3b
+#define regSUM_INDEX_HI_BASE_IDX                                                                        5
+
+
+// addressBlock: nbif0_nbif0_rcc_strap_rcc_strap_internal
+// base address: 0x10100000
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP0                                                              0xc400
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP1                                                              0xc401
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP2                                                              0xc402
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP3                                                              0xc403
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP4                                                              0xc404
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP5                                                              0xc405
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP6                                                              0xc406
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP7                                                              0xc407
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP7_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP8                                                              0xc408
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP9                                                              0xc409
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP10                                                             0xc40a
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP10_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP11                                                             0xc40b
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP11_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP12                                                             0xc40c
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP12_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP13                                                             0xc40d
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP14                                                             0xc40e
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP15                                                             0xc40f
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP15_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP16                                                             0xc410
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP16_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP17                                                             0xc411
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP17_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP18                                                             0xc412
+#define regRCC_STRAP1_RCC_DEV0_PORT_STRAP18_BASE_IDX                                                    5
+#define regRCC_DEV1_PORT_STRAP0                                                                         0xc480
+#define regRCC_DEV1_PORT_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP1                                                                         0xc481
+#define regRCC_DEV1_PORT_STRAP1_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP2                                                                         0xc482
+#define regRCC_DEV1_PORT_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP3                                                                         0xc483
+#define regRCC_DEV1_PORT_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP4                                                                         0xc484
+#define regRCC_DEV1_PORT_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP5                                                                         0xc485
+#define regRCC_DEV1_PORT_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP6                                                                         0xc486
+#define regRCC_DEV1_PORT_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP7                                                                         0xc487
+#define regRCC_DEV1_PORT_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP8                                                                         0xc488
+#define regRCC_DEV1_PORT_STRAP8_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP9                                                                         0xc489
+#define regRCC_DEV1_PORT_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV1_PORT_STRAP10                                                                        0xc48a
+#define regRCC_DEV1_PORT_STRAP10_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP11                                                                        0xc48b
+#define regRCC_DEV1_PORT_STRAP11_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP12                                                                        0xc48c
+#define regRCC_DEV1_PORT_STRAP12_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP13                                                                        0xc48d
+#define regRCC_DEV1_PORT_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP14                                                                        0xc48e
+#define regRCC_DEV1_PORT_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP15                                                                        0xc48f
+#define regRCC_DEV1_PORT_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV1_PORT_STRAP16                                                                        0xc490
+#define regRCC_DEV1_PORT_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP0                                                                         0xc500
+#define regRCC_DEV2_PORT_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP1                                                                         0xc501
+#define regRCC_DEV2_PORT_STRAP1_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP2                                                                         0xc502
+#define regRCC_DEV2_PORT_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP3                                                                         0xc503
+#define regRCC_DEV2_PORT_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP4                                                                         0xc504
+#define regRCC_DEV2_PORT_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP5                                                                         0xc505
+#define regRCC_DEV2_PORT_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP6                                                                         0xc506
+#define regRCC_DEV2_PORT_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP7                                                                         0xc507
+#define regRCC_DEV2_PORT_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP8                                                                         0xc508
+#define regRCC_DEV2_PORT_STRAP8_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP9                                                                         0xc509
+#define regRCC_DEV2_PORT_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV2_PORT_STRAP10                                                                        0xc50a
+#define regRCC_DEV2_PORT_STRAP10_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP11                                                                        0xc50b
+#define regRCC_DEV2_PORT_STRAP11_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP12                                                                        0xc50c
+#define regRCC_DEV2_PORT_STRAP12_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP13                                                                        0xc50d
+#define regRCC_DEV2_PORT_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP14                                                                        0xc50e
+#define regRCC_DEV2_PORT_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP15                                                                        0xc50f
+#define regRCC_DEV2_PORT_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV2_PORT_STRAP16                                                                        0xc510
+#define regRCC_DEV2_PORT_STRAP16_BASE_IDX                                                               5
+#define regRCC_STRAP1_RCC_BIF_STRAP0                                                                    0xc600
+#define regRCC_STRAP1_RCC_BIF_STRAP0_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP1                                                                    0xc601
+#define regRCC_STRAP1_RCC_BIF_STRAP1_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP2                                                                    0xc602
+#define regRCC_STRAP1_RCC_BIF_STRAP2_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP3                                                                    0xc603
+#define regRCC_STRAP1_RCC_BIF_STRAP3_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP4                                                                    0xc604
+#define regRCC_STRAP1_RCC_BIF_STRAP4_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP5                                                                    0xc605
+#define regRCC_STRAP1_RCC_BIF_STRAP5_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP6                                                                    0xc606
+#define regRCC_STRAP1_RCC_BIF_STRAP6_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP7                                                                    0xc607
+#define regRCC_STRAP1_RCC_BIF_STRAP7_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP8                                                                    0xc608
+#define regRCC_STRAP1_RCC_BIF_STRAP8_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP9                                                                    0xc609
+#define regRCC_STRAP1_RCC_BIF_STRAP9_BASE_IDX                                                           5
+#define regRCC_STRAP1_RCC_BIF_STRAP10                                                                   0xc60a
+#define regRCC_STRAP1_RCC_BIF_STRAP10_BASE_IDX                                                          5
+#define regRCC_STRAP1_RCC_BIF_STRAP11                                                                   0xc60b
+#define regRCC_STRAP1_RCC_BIF_STRAP11_BASE_IDX                                                          5
+#define regRCC_STRAP1_RCC_BIF_STRAP12                                                                   0xc60c
+#define regRCC_STRAP1_RCC_BIF_STRAP12_BASE_IDX                                                          5
+#define regRCC_STRAP1_RCC_BIF_STRAP13                                                                   0xc60d
+#define regRCC_STRAP1_RCC_BIF_STRAP13_BASE_IDX                                                          5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0                                                              0xd000
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP1                                                              0xd001
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP2                                                              0xd002
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP3                                                              0xd003
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP4                                                              0xd004
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP5                                                              0xd005
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP8                                                              0xd008
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP9                                                              0xd009
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP13                                                             0xd00d
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP14                                                             0xd00e
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP15                                                             0xd00f
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP15_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP16                                                             0xd010
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP16_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP17                                                             0xd011
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP17_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP18                                                             0xd012
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP18_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP19                                                             0xd013
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP19_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP22                                                             0xd016
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP22_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP23                                                             0xd017
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP23_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP24                                                             0xd018
+#define regRCC_STRAP1_RCC_DEV0_EPF0_STRAP24_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP0                                                              0xd080
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP1                                                              0xd081
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP2                                                              0xd082
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP3                                                              0xd083
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP4                                                              0xd084
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP5                                                              0xd085
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP6                                                              0xd086
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP7                                                              0xd087
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP7_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP8                                                              0xd088
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP9                                                              0xd089
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP10                                                             0xd08a
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP10_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP11                                                             0xd08b
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP11_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP12                                                             0xd08c
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP12_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP13                                                             0xd08d
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP14                                                             0xd08e
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP15                                                             0xd08f
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP15_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP16                                                             0xd090
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP16_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP17                                                             0xd091
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP17_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP18                                                             0xd092
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP18_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP19                                                             0xd093
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP19_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP20                                                             0xd094
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP20_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP21                                                             0xd095
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP21_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP23                                                             0xd097
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP23_BASE_IDX                                                    5
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP24                                                             0xd098
+#define regRCC_STRAP1_RCC_DEV0_EPF1_STRAP24_BASE_IDX                                                    5
+#define regRCC_DEV0_EPF2_STRAP0                                                                         0xd100
+#define regRCC_DEV0_EPF2_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP2                                                                         0xd102
+#define regRCC_DEV0_EPF2_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP3                                                                         0xd103
+#define regRCC_DEV0_EPF2_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP4                                                                         0xd104
+#define regRCC_DEV0_EPF2_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP5                                                                         0xd105
+#define regRCC_DEV0_EPF2_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP6                                                                         0xd106
+#define regRCC_DEV0_EPF2_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP7                                                                         0xd107
+#define regRCC_DEV0_EPF2_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP9                                                                         0xd109
+#define regRCC_DEV0_EPF2_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF2_STRAP10                                                                        0xd10a
+#define regRCC_DEV0_EPF2_STRAP10_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP11                                                                        0xd10b
+#define regRCC_DEV0_EPF2_STRAP11_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP12                                                                        0xd10c
+#define regRCC_DEV0_EPF2_STRAP12_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP13                                                                        0xd10d
+#define regRCC_DEV0_EPF2_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP14                                                                        0xd10e
+#define regRCC_DEV0_EPF2_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP15                                                                        0xd10f
+#define regRCC_DEV0_EPF2_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP16                                                                        0xd110
+#define regRCC_DEV0_EPF2_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF2_STRAP20                                                                        0xd114
+#define regRCC_DEV0_EPF2_STRAP20_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP0                                                                         0xd180
+#define regRCC_DEV0_EPF3_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP2                                                                         0xd182
+#define regRCC_DEV0_EPF3_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP3                                                                         0xd183
+#define regRCC_DEV0_EPF3_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP4                                                                         0xd184
+#define regRCC_DEV0_EPF3_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP5                                                                         0xd185
+#define regRCC_DEV0_EPF3_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP6                                                                         0xd186
+#define regRCC_DEV0_EPF3_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP7                                                                         0xd187
+#define regRCC_DEV0_EPF3_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP9                                                                         0xd189
+#define regRCC_DEV0_EPF3_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF3_STRAP10                                                                        0xd18a
+#define regRCC_DEV0_EPF3_STRAP10_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP11                                                                        0xd18b
+#define regRCC_DEV0_EPF3_STRAP11_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP12                                                                        0xd18c
+#define regRCC_DEV0_EPF3_STRAP12_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP13                                                                        0xd18d
+#define regRCC_DEV0_EPF3_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP14                                                                        0xd18e
+#define regRCC_DEV0_EPF3_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP15                                                                        0xd18f
+#define regRCC_DEV0_EPF3_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP16                                                                        0xd190
+#define regRCC_DEV0_EPF3_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF3_STRAP20                                                                        0xd194
+#define regRCC_DEV0_EPF3_STRAP20_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF4_STRAP0                                                                         0xd200
+#define regRCC_DEV0_EPF4_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP2                                                                         0xd202
+#define regRCC_DEV0_EPF4_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP3                                                                         0xd203
+#define regRCC_DEV0_EPF4_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP4                                                                         0xd204
+#define regRCC_DEV0_EPF4_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP5                                                                         0xd205
+#define regRCC_DEV0_EPF4_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP6                                                                         0xd206
+#define regRCC_DEV0_EPF4_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP7                                                                         0xd207
+#define regRCC_DEV0_EPF4_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP9                                                                         0xd209
+#define regRCC_DEV0_EPF4_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF4_STRAP13                                                                        0xd20d
+#define regRCC_DEV0_EPF4_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF4_STRAP14                                                                        0xd20e
+#define regRCC_DEV0_EPF4_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF4_STRAP15                                                                        0xd20f
+#define regRCC_DEV0_EPF4_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF4_STRAP16                                                                        0xd210
+#define regRCC_DEV0_EPF4_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF5_STRAP0                                                                         0xd280
+#define regRCC_DEV0_EPF5_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP2                                                                         0xd282
+#define regRCC_DEV0_EPF5_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP3                                                                         0xd283
+#define regRCC_DEV0_EPF5_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP4                                                                         0xd284
+#define regRCC_DEV0_EPF5_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP5                                                                         0xd285
+#define regRCC_DEV0_EPF5_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP6                                                                         0xd286
+#define regRCC_DEV0_EPF5_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP7                                                                         0xd287
+#define regRCC_DEV0_EPF5_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP9                                                                         0xd289
+#define regRCC_DEV0_EPF5_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF5_STRAP13                                                                        0xd28d
+#define regRCC_DEV0_EPF5_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF5_STRAP14                                                                        0xd28e
+#define regRCC_DEV0_EPF5_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF5_STRAP15                                                                        0xd28f
+#define regRCC_DEV0_EPF5_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF5_STRAP16                                                                        0xd290
+#define regRCC_DEV0_EPF5_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF6_STRAP0                                                                         0xd300
+#define regRCC_DEV0_EPF6_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP2                                                                         0xd302
+#define regRCC_DEV0_EPF6_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP3                                                                         0xd303
+#define regRCC_DEV0_EPF6_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP4                                                                         0xd304
+#define regRCC_DEV0_EPF6_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP5                                                                         0xd305
+#define regRCC_DEV0_EPF6_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP6                                                                         0xd306
+#define regRCC_DEV0_EPF6_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP7                                                                         0xd307
+#define regRCC_DEV0_EPF6_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP9                                                                         0xd309
+#define regRCC_DEV0_EPF6_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF6_STRAP13                                                                        0xd30d
+#define regRCC_DEV0_EPF6_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF6_STRAP14                                                                        0xd30e
+#define regRCC_DEV0_EPF6_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF6_STRAP15                                                                        0xd30f
+#define regRCC_DEV0_EPF6_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF6_STRAP16                                                                        0xd310
+#define regRCC_DEV0_EPF6_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF7_STRAP0                                                                         0xd380
+#define regRCC_DEV0_EPF7_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP2                                                                         0xd382
+#define regRCC_DEV0_EPF7_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP3                                                                         0xd383
+#define regRCC_DEV0_EPF7_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP4                                                                         0xd384
+#define regRCC_DEV0_EPF7_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP5                                                                         0xd385
+#define regRCC_DEV0_EPF7_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP6                                                                         0xd386
+#define regRCC_DEV0_EPF7_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP9                                                                         0xd389
+#define regRCC_DEV0_EPF7_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV0_EPF7_STRAP13                                                                        0xd38d
+#define regRCC_DEV0_EPF7_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF7_STRAP14                                                                        0xd38e
+#define regRCC_DEV0_EPF7_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF7_STRAP15                                                                        0xd38f
+#define regRCC_DEV0_EPF7_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV0_EPF7_STRAP16                                                                        0xd390
+#define regRCC_DEV0_EPF7_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF0_STRAP0                                                                         0xd400
+#define regRCC_DEV1_EPF0_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP2                                                                         0xd402
+#define regRCC_DEV1_EPF0_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP3                                                                         0xd403
+#define regRCC_DEV1_EPF0_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP4                                                                         0xd404
+#define regRCC_DEV1_EPF0_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP5                                                                         0xd405
+#define regRCC_DEV1_EPF0_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP6                                                                         0xd406
+#define regRCC_DEV1_EPF0_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP9                                                                         0xd409
+#define regRCC_DEV1_EPF0_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF0_STRAP13                                                                        0xd40d
+#define regRCC_DEV1_EPF0_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF0_STRAP14                                                                        0xd40e
+#define regRCC_DEV1_EPF0_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF0_STRAP15                                                                        0xd40f
+#define regRCC_DEV1_EPF0_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF0_STRAP16                                                                        0xd410
+#define regRCC_DEV1_EPF0_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP0                                                                         0xd480
+#define regRCC_DEV1_EPF1_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP1                                                                         0xd481
+#define regRCC_DEV1_EPF1_STRAP1_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP2                                                                         0xd482
+#define regRCC_DEV1_EPF1_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP3                                                                         0xd483
+#define regRCC_DEV1_EPF1_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP4                                                                         0xd484
+#define regRCC_DEV1_EPF1_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP5                                                                         0xd485
+#define regRCC_DEV1_EPF1_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP6                                                                         0xd486
+#define regRCC_DEV1_EPF1_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP7                                                                         0xd487
+#define regRCC_DEV1_EPF1_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP8                                                                         0xd488
+#define regRCC_DEV1_EPF1_STRAP8_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP9                                                                         0xd489
+#define regRCC_DEV1_EPF1_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF1_STRAP13                                                                        0xd48d
+#define regRCC_DEV1_EPF1_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP14                                                                        0xd48e
+#define regRCC_DEV1_EPF1_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP15                                                                        0xd48f
+#define regRCC_DEV1_EPF1_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP16                                                                        0xd490
+#define regRCC_DEV1_EPF1_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP17                                                                        0xd491
+#define regRCC_DEV1_EPF1_STRAP17_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP18                                                                        0xd492
+#define regRCC_DEV1_EPF1_STRAP18_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP19                                                                        0xd493
+#define regRCC_DEV1_EPF1_STRAP19_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP20                                                                        0xd494
+#define regRCC_DEV1_EPF1_STRAP20_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF1_STRAP21                                                                        0xd495
+#define regRCC_DEV1_EPF1_STRAP21_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF2_STRAP0                                                                         0xd500
+#define regRCC_DEV1_EPF2_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP1                                                                         0xd501
+#define regRCC_DEV1_EPF2_STRAP1_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP2                                                                         0xd502
+#define regRCC_DEV1_EPF2_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP3                                                                         0xd503
+#define regRCC_DEV1_EPF2_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP4                                                                         0xd504
+#define regRCC_DEV1_EPF2_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP5                                                                         0xd505
+#define regRCC_DEV1_EPF2_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP6                                                                         0xd506
+#define regRCC_DEV1_EPF2_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP7                                                                         0xd507
+#define regRCC_DEV1_EPF2_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP8                                                                         0xd508
+#define regRCC_DEV1_EPF2_STRAP8_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP9                                                                         0xd509
+#define regRCC_DEV1_EPF2_STRAP9_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF2_STRAP13                                                                        0xd50d
+#define regRCC_DEV1_EPF2_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF2_STRAP14                                                                        0xd50e
+#define regRCC_DEV1_EPF2_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF2_STRAP15                                                                        0xd50f
+#define regRCC_DEV1_EPF2_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF2_STRAP16                                                                        0xd510
+#define regRCC_DEV1_EPF2_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF2_STRAP17                                                                        0xd511
+#define regRCC_DEV1_EPF2_STRAP17_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF2_STRAP18                                                                        0xd512
+#define regRCC_DEV1_EPF2_STRAP18_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF2_STRAP19                                                                        0xd513
+#define regRCC_DEV1_EPF2_STRAP19_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF2_STRAP20                                                                        0xd514
+#define regRCC_DEV1_EPF2_STRAP20_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF2_STRAP21                                                                        0xd515
+#define regRCC_DEV1_EPF2_STRAP21_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF3_STRAP0                                                                         0xd580
+#define regRCC_DEV1_EPF3_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP2                                                                         0xd582
+#define regRCC_DEV1_EPF3_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP3                                                                         0xd583
+#define regRCC_DEV1_EPF3_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP4                                                                         0xd584
+#define regRCC_DEV1_EPF3_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP5                                                                         0xd585
+#define regRCC_DEV1_EPF3_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP6                                                                         0xd586
+#define regRCC_DEV1_EPF3_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP7                                                                         0xd587
+#define regRCC_DEV1_EPF3_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF3_STRAP13                                                                        0xd58d
+#define regRCC_DEV1_EPF3_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF3_STRAP14                                                                        0xd58e
+#define regRCC_DEV1_EPF3_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF3_STRAP15                                                                        0xd58f
+#define regRCC_DEV1_EPF3_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF3_STRAP16                                                                        0xd590
+#define regRCC_DEV1_EPF3_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV1_EPF4_STRAP6                                                                         0xd606
+#define regRCC_DEV1_EPF4_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF5_STRAP6                                                                         0xd686
+#define regRCC_DEV1_EPF5_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV1_EPF6_STRAP6                                                                         0xd706
+#define regRCC_DEV1_EPF6_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP0                                                                         0xd800
+#define regRCC_DEV2_EPF0_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP2                                                                         0xd802
+#define regRCC_DEV2_EPF0_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP3                                                                         0xd803
+#define regRCC_DEV2_EPF0_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP4                                                                         0xd804
+#define regRCC_DEV2_EPF0_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP5                                                                         0xd805
+#define regRCC_DEV2_EPF0_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP6                                                                         0xd806
+#define regRCC_DEV2_EPF0_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP7                                                                         0xd807
+#define regRCC_DEV2_EPF0_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF0_STRAP13                                                                        0xd80d
+#define regRCC_DEV2_EPF0_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF0_STRAP14                                                                        0xd80e
+#define regRCC_DEV2_EPF0_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF0_STRAP15                                                                        0xd80f
+#define regRCC_DEV2_EPF0_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF0_STRAP16                                                                        0xd810
+#define regRCC_DEV2_EPF0_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF0_STRAP18                                                                        0xd812
+#define regRCC_DEV2_EPF0_STRAP18_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF0_STRAP25                                                                        0xd819
+#define regRCC_DEV2_EPF0_STRAP25_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF0_STRAP26                                                                        0xd81a
+#define regRCC_DEV2_EPF0_STRAP26_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF1_STRAP0                                                                         0xd880
+#define regRCC_DEV2_EPF1_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP2                                                                         0xd882
+#define regRCC_DEV2_EPF1_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP3                                                                         0xd883
+#define regRCC_DEV2_EPF1_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP4                                                                         0xd884
+#define regRCC_DEV2_EPF1_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP5                                                                         0xd885
+#define regRCC_DEV2_EPF1_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP6                                                                         0xd886
+#define regRCC_DEV2_EPF1_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP7                                                                         0xd887
+#define regRCC_DEV2_EPF1_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF1_STRAP13                                                                        0xd88d
+#define regRCC_DEV2_EPF1_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF1_STRAP14                                                                        0xd88e
+#define regRCC_DEV2_EPF1_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF1_STRAP15                                                                        0xd88f
+#define regRCC_DEV2_EPF1_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF1_STRAP16                                                                        0xd890
+#define regRCC_DEV2_EPF1_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF2_STRAP0                                                                         0xd900
+#define regRCC_DEV2_EPF2_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP2                                                                         0xd902
+#define regRCC_DEV2_EPF2_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP3                                                                         0xd903
+#define regRCC_DEV2_EPF2_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP4                                                                         0xd904
+#define regRCC_DEV2_EPF2_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP5                                                                         0xd905
+#define regRCC_DEV2_EPF2_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP6                                                                         0xd906
+#define regRCC_DEV2_EPF2_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP7                                                                         0xd907
+#define regRCC_DEV2_EPF2_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF2_STRAP13                                                                        0xd90d
+#define regRCC_DEV2_EPF2_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF2_STRAP14                                                                        0xd90e
+#define regRCC_DEV2_EPF2_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF2_STRAP15                                                                        0xd90f
+#define regRCC_DEV2_EPF2_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF2_STRAP16                                                                        0xd910
+#define regRCC_DEV2_EPF2_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF3_STRAP0                                                                         0xd980
+#define regRCC_DEV2_EPF3_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF3_STRAP2                                                                         0xd982
+#define regRCC_DEV2_EPF3_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF3_STRAP3                                                                         0xd983
+#define regRCC_DEV2_EPF3_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF3_STRAP4                                                                         0xd984
+#define regRCC_DEV2_EPF3_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF3_STRAP5                                                                         0xd985
+#define regRCC_DEV2_EPF3_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF3_STRAP6                                                                         0xd986
+#define regRCC_DEV2_EPF3_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF3_STRAP7                                                                         0xd987
+#define regRCC_DEV2_EPF3_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF3_STRAP13                                                                        0xd98d
+#define regRCC_DEV2_EPF3_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF3_STRAP14                                                                        0xd98e
+#define regRCC_DEV2_EPF3_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF3_STRAP15                                                                        0xd98f
+#define regRCC_DEV2_EPF3_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF3_STRAP16                                                                        0xd990
+#define regRCC_DEV2_EPF3_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF4_STRAP0                                                                         0xda00
+#define regRCC_DEV2_EPF4_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF4_STRAP2                                                                         0xda02
+#define regRCC_DEV2_EPF4_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF4_STRAP3                                                                         0xda03
+#define regRCC_DEV2_EPF4_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF4_STRAP4                                                                         0xda04
+#define regRCC_DEV2_EPF4_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF4_STRAP5                                                                         0xda05
+#define regRCC_DEV2_EPF4_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF4_STRAP6                                                                         0xda06
+#define regRCC_DEV2_EPF4_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF4_STRAP7                                                                         0xda07
+#define regRCC_DEV2_EPF4_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF4_STRAP13                                                                        0xda0d
+#define regRCC_DEV2_EPF4_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF4_STRAP14                                                                        0xda0e
+#define regRCC_DEV2_EPF4_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF4_STRAP15                                                                        0xda0f
+#define regRCC_DEV2_EPF4_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF4_STRAP16                                                                        0xda10
+#define regRCC_DEV2_EPF4_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF5_STRAP0                                                                         0xda80
+#define regRCC_DEV2_EPF5_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF5_STRAP2                                                                         0xda82
+#define regRCC_DEV2_EPF5_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF5_STRAP3                                                                         0xda83
+#define regRCC_DEV2_EPF5_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF5_STRAP4                                                                         0xda84
+#define regRCC_DEV2_EPF5_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF5_STRAP5                                                                         0xda85
+#define regRCC_DEV2_EPF5_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF5_STRAP6                                                                         0xda86
+#define regRCC_DEV2_EPF5_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF5_STRAP7                                                                         0xda87
+#define regRCC_DEV2_EPF5_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF5_STRAP13                                                                        0xda8d
+#define regRCC_DEV2_EPF5_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF5_STRAP14                                                                        0xda8e
+#define regRCC_DEV2_EPF5_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF5_STRAP15                                                                        0xda8f
+#define regRCC_DEV2_EPF5_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF5_STRAP16                                                                        0xda90
+#define regRCC_DEV2_EPF5_STRAP16_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF6_STRAP0                                                                         0xdb00
+#define regRCC_DEV2_EPF6_STRAP0_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF6_STRAP2                                                                         0xdb02
+#define regRCC_DEV2_EPF6_STRAP2_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF6_STRAP3                                                                         0xdb03
+#define regRCC_DEV2_EPF6_STRAP3_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF6_STRAP4                                                                         0xdb04
+#define regRCC_DEV2_EPF6_STRAP4_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF6_STRAP5                                                                         0xdb05
+#define regRCC_DEV2_EPF6_STRAP5_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF6_STRAP6                                                                         0xdb06
+#define regRCC_DEV2_EPF6_STRAP6_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF6_STRAP7                                                                         0xdb07
+#define regRCC_DEV2_EPF6_STRAP7_BASE_IDX                                                                5
+#define regRCC_DEV2_EPF6_STRAP13                                                                        0xdb0d
+#define regRCC_DEV2_EPF6_STRAP13_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF6_STRAP14                                                                        0xdb0e
+#define regRCC_DEV2_EPF6_STRAP14_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF6_STRAP15                                                                        0xdb0f
+#define regRCC_DEV2_EPF6_STRAP15_BASE_IDX                                                               5
+#define regRCC_DEV2_EPF6_STRAP16                                                                        0xdb10
+#define regRCC_DEV2_EPF6_STRAP16_BASE_IDX                                                               5
+
+
+// addressBlock: nbif0_nbif0_bif_rst_bif_rst_regblk
+// base address: 0x10100000
+#define regHARD_RST_CTRL                                                                                0xe000
+#define regHARD_RST_CTRL_BASE_IDX                                                                       5
+#define regSELF_SOFT_RST                                                                                0xe002
+#define regSELF_SOFT_RST_BASE_IDX                                                                       5
+#define regBIF_GFX_DRV_VPU_RST                                                                          0xe003
+#define regBIF_GFX_DRV_VPU_RST_BASE_IDX                                                                 5
+#define regBIF_RST_MISC_CTRL                                                                            0xe004
+#define regBIF_RST_MISC_CTRL_BASE_IDX                                                                   5
+#define regBIF_RST_MISC_CTRL2                                                                           0xe005
+#define regBIF_RST_MISC_CTRL2_BASE_IDX                                                                  5
+#define regBIF_RST_MISC_CTRL3                                                                           0xe006
+#define regBIF_RST_MISC_CTRL3_BASE_IDX                                                                  5
+#define regDEV0_PF0_FLR_RST_CTRL                                                                        0xe008
+#define regDEV0_PF0_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regDEV0_PF1_FLR_RST_CTRL                                                                        0xe009
+#define regDEV0_PF1_FLR_RST_CTRL_BASE_IDX                                                               5
+#define regBIF_INST_RESET_INTR_STS                                                                      0xe010
+#define regBIF_INST_RESET_INTR_STS_BASE_IDX                                                             5
+#define regBIF_PF_FLR_INTR_STS                                                                          0xe011
+#define regBIF_PF_FLR_INTR_STS_BASE_IDX                                                                 5
+#define regBIF_D3HOTD0_INTR_STS                                                                         0xe012
+#define regBIF_D3HOTD0_INTR_STS_BASE_IDX                                                                5
+#define regBIF_POWER_INTR_STS                                                                           0xe014
+#define regBIF_POWER_INTR_STS_BASE_IDX                                                                  5
+#define regBIF_PF_DSTATE_INTR_STS                                                                       0xe015
+#define regBIF_PF_DSTATE_INTR_STS_BASE_IDX                                                              5
+#define regSELF_SOFT_RST_2                                                                              0xe016
+#define regSELF_SOFT_RST_2_BASE_IDX                                                                     5
+#define regSION_SDP_PORT_RST                                                                            0xe017
+#define regSION_SDP_PORT_RST_BASE_IDX                                                                   5
+#define regBIF_INST_RESET_INTR_MASK                                                                     0xe020
+#define regBIF_INST_RESET_INTR_MASK_BASE_IDX                                                            5
+#define regBIF_PF_FLR_INTR_MASK                                                                         0xe021
+#define regBIF_PF_FLR_INTR_MASK_BASE_IDX                                                                5
+#define regBIF_D3HOTD0_INTR_MASK                                                                        0xe022
+#define regBIF_D3HOTD0_INTR_MASK_BASE_IDX                                                               5
+#define regBIF_POWER_INTR_MASK                                                                          0xe024
+#define regBIF_POWER_INTR_MASK_BASE_IDX                                                                 5
+#define regBIF_PF_DSTATE_INTR_MASK                                                                      0xe025
+#define regBIF_PF_DSTATE_INTR_MASK_BASE_IDX                                                             5
+#define regBIF_DEV0_PF0_VF_FLR_INTR_MASK                                                                0xe028
+#define regBIF_DEV0_PF0_VF_FLR_INTR_MASK_BASE_IDX                                                       5
+#define regBIF_DEV0_PF1_VF_FLR_INTR_MASK                                                                0xe029
+#define regBIF_DEV0_PF1_VF_FLR_INTR_MASK_BASE_IDX                                                       5
+#define regBIF_PF_FLR_RST                                                                               0xe040
+#define regBIF_PF_FLR_RST_BASE_IDX                                                                      5
+#define regBIF_PF_FLR_PROTECT                                                                           0xe042
+#define regBIF_PF_FLR_PROTECT_BASE_IDX                                                                  5
+#define regBIF_PF_FLR_PROTECT3                                                                          0xe044
+#define regBIF_PF_FLR_PROTECT3_BASE_IDX                                                                 5
+#define regBIF_DEV0_PF0_DSTATE_VALUE                                                                    0xe050
+#define regBIF_DEV0_PF0_DSTATE_VALUE_BASE_IDX                                                           5
+#define regBIF_DEV0_PF1_DSTATE_VALUE                                                                    0xe051
+#define regBIF_DEV0_PF1_DSTATE_VALUE_BASE_IDX                                                           5
+#define regGFX_RST_CNTL_IND0                                                                            0xe05d
+#define regGFX_RST_CNTL_IND0_BASE_IDX                                                                   5
+#define regGFX_RST_CNTL_SOFT_PF                                                                         0xe05e
+#define regGFX_RST_CNTL_SOFT_PF_BASE_IDX                                                                5
+#define regGFX_RST_CNTL_IND1                                                                            0xe05f
+#define regGFX_RST_CNTL_IND1_BASE_IDX                                                                   5
+#define regGFX_RST_CNTL_IND2                                                                            0xe060
+#define regGFX_RST_CNTL_IND2_BASE_IDX                                                                   5
+#define regGFX_RST_CNTL_IND3                                                                            0xe061
+#define regGFX_RST_CNTL_IND3_BASE_IDX                                                                   5
+#define regGFX_RST_CNTL_IND4                                                                            0xe062
+#define regGFX_RST_CNTL_IND4_BASE_IDX                                                                   5
+#define regGFX_RST_CNTL_IND5                                                                            0xe063
+#define regGFX_RST_CNTL_IND5_BASE_IDX                                                                   5
+#define regGFX_RST_CNTL_IND6                                                                            0xe064
+#define regGFX_RST_CNTL_IND6_BASE_IDX                                                                   5
+#define regGFX_RST_CNTL_IND7                                                                            0xe065
+#define regGFX_RST_CNTL_IND7_BASE_IDX                                                                   5
+#define regGFX_RST_CNTL_IND8                                                                            0xe066
+#define regGFX_RST_CNTL_IND8_BASE_IDX                                                                   5
+#define regDEV0_PF0_D3HOTD0_RST_CTRL                                                                    0xe07e
+#define regDEV0_PF0_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regDEV0_PF1_D3HOTD0_RST_CTRL                                                                    0xe07f
+#define regDEV0_PF1_D3HOTD0_RST_CTRL_BASE_IDX                                                           5
+#define regBIF_DEV0_PF0_VF_FLR_PROTECT                                                                  0xe086
+#define regBIF_DEV0_PF0_VF_FLR_PROTECT_BASE_IDX                                                         5
+#define regBIF_DEV0_PF1_VF_FLR_PROTECT                                                                  0xe087
+#define regBIF_DEV0_PF1_VF_FLR_PROTECT_BASE_IDX                                                         5
+#define regBIF_SRIOVEN_SET_INTR_STS                                                                     0xe088
+#define regBIF_SRIOVEN_SET_INTR_STS_BASE_IDX                                                            5
+#define regBIF_SRIOVEN_CLR_INTR_STS                                                                     0xe089
+#define regBIF_SRIOVEN_CLR_INTR_STS_BASE_IDX                                                            5
+#define regBIF_SRIOVEN_SET_INTR_MASK                                                                    0xe08a
+#define regBIF_SRIOVEN_SET_INTR_MASK_BASE_IDX                                                           5
+#define regBIF_SRIOVEN_CLR_INTR_MASK                                                                    0xe08b
+#define regBIF_SRIOVEN_CLR_INTR_MASK_BASE_IDX                                                           5
+#define regBIF_PORT0_DSTATE_VALUE                                                                       0xe230
+#define regBIF_PORT0_DSTATE_VALUE_BASE_IDX                                                              5
+#define regBIF_DEV0_PF0_RST_VF_FLR_IDLE                                                                 0xe2c4
+#define regBIF_DEV0_PF0_RST_VF_FLR_IDLE_BASE_IDX                                                        5
+#define regBIF_DEV0_PF1_RST_VF_FLR_IDLE                                                                 0xe2c5
+#define regBIF_DEV0_PF1_RST_VF_FLR_IDLE_BASE_IDX                                                        5
+#define regBIF_DEV0_PF0_VF_FLR_INTR_STS                                                                 0xe2c8
+#define regBIF_DEV0_PF0_VF_FLR_INTR_STS_BASE_IDX                                                        5
+#define regBIF_DEV0_PF1_VF_FLR_INTR_STS                                                                 0xe2c9
+#define regBIF_DEV0_PF1_VF_FLR_INTR_STS_BASE_IDX                                                        5
+#define regBIF_DEV0_PF0_VF_FLR_RST                                                                      0xe2cc
+#define regBIF_DEV0_PF0_VF_FLR_RST_BASE_IDX                                                             5
+#define regBIF_DEV0_PF1_VF_FLR_RST                                                                      0xe2cd
+#define regBIF_DEV0_PF1_VF_FLR_RST_BASE_IDX                                                             5
+
+
+// addressBlock: nbif0_nbif0_bif_misc_bif_misc_regblk
+// base address: 0x10100000
+#define regDOE_PRV_CTRL0_DEV0F0                                                                         0xc806
+#define regDOE_PRV_CTRL0_DEV0F0_BASE_IDX                                                                5
+#define regDOE_PRV_CMD_DEV0F0                                                                           0xc807
+#define regDOE_PRV_CMD_DEV0F0_BASE_IDX                                                                  5
+#define regDOE_PRV_WDATA_DEV0F0                                                                         0xc808
+#define regDOE_PRV_WDATA_DEV0F0_BASE_IDX                                                                5
+#define regDOE_PRV_STS_DEV0F0                                                                           0xc809
+#define regDOE_PRV_STS_DEV0F0_BASE_IDX                                                                  5
+#define regDOE_PRV_RDATA_DEV0F0                                                                         0xc80a
+#define regDOE_PRV_RDATA_DEV0F0_BASE_IDX                                                                5
+#define regNBIF_RC_INTR_CNTL                                                                            0xc81b
+#define regNBIF_RC_INTR_CNTL_BASE_IDX                                                                   5
+#define regBIF_DEV0_PF0_TDISP_CNTL                                                                      0xc8c0
+#define regBIF_DEV0_PF0_TDISP_CNTL_BASE_IDX                                                             5
+#define regBIF_DEV0_PF1_TDISP_CNTL                                                                      0xc8c1
+#define regBIF_DEV0_PF1_TDISP_CNTL_BASE_IDX                                                             5
+#define regBIF_DEV0_PF0_VF0_TDISP_CNTL                                                                  0xc8d0
+#define regBIF_DEV0_PF0_VF0_TDISP_CNTL_BASE_IDX                                                         5
+#define regBIF_DEV0_PF0_VF1_TDISP_CNTL                                                                  0xc8d1
+#define regBIF_DEV0_PF0_VF1_TDISP_CNTL_BASE_IDX                                                         5
+#define regBIF_DEV0_PF0_VF2_TDISP_CNTL                                                                  0xc8d2
+#define regBIF_DEV0_PF0_VF2_TDISP_CNTL_BASE_IDX                                                         5
+#define regBIF_DEV0_PF0_VF3_TDISP_CNTL                                                                  0xc8d3
+#define regBIF_DEV0_PF0_VF3_TDISP_CNTL_BASE_IDX                                                         5
+#define regBIF_DEV0_PF0_VF4_TDISP_CNTL                                                                  0xc8d4
+#define regBIF_DEV0_PF0_VF4_TDISP_CNTL_BASE_IDX                                                         5
+#define regBIF_DEV0_PF0_VF5_TDISP_CNTL                                                                  0xc8d5
+#define regBIF_DEV0_PF0_VF5_TDISP_CNTL_BASE_IDX                                                         5
+#define regBIF_DEV0_PF0_VF6_TDISP_CNTL                                                                  0xc8d6
+#define regBIF_DEV0_PF0_VF6_TDISP_CNTL_BASE_IDX                                                         5
+#define regBIF_DEV0_PF0_VF7_TDISP_CNTL                                                                  0xc8d7
+#define regBIF_DEV0_PF0_VF7_TDISP_CNTL_BASE_IDX                                                         5
+#define regBIF_DEV0_PF1_VF0_TDISP_CNTL                                                                  0xc8d8
+#define regBIF_DEV0_PF1_VF0_TDISP_CNTL_BASE_IDX                                                         5
+#define regBIF_TDISP_ERROR_DEV0_PF0_INST0                                                               0xc9d0
+#define regBIF_TDISP_ERROR_DEV0_PF0_INST0_BASE_IDX                                                      5
+#define regBIF_TDISP_ERROR_DEV0_PF1_INST0                                                               0xc9d1
+#define regBIF_TDISP_ERROR_DEV0_PF1_INST0_BASE_IDX                                                      5
+#define regBIF_TDISP_UNLOCKED_DEV0_PF0_INST0                                                            0xc9d8
+#define regBIF_TDISP_UNLOCKED_DEV0_PF0_INST0_BASE_IDX                                                   5
+#define regBIF_TDISP_UNLOCKED_DEV0_PF1_INST0                                                            0xc9d9
+#define regBIF_TDISP_UNLOCKED_DEV0_PF1_INST0_BASE_IDX                                                   5
+#define regBIF_TDISP_MISC_CNTL_DEV0                                                                     0xc9e0
+#define regBIF_TDISP_MISC_CNTL_DEV0_BASE_IDX                                                            5
+#define regBIF_TDISP_MISC_CNTL                                                                          0xc9e4
+#define regBIF_TDISP_MISC_CNTL_BASE_IDX                                                                 5
+#define regBIF_RCC_IDE_TDISP_CNTL                                                                       0xc9e5
+#define regBIF_RCC_IDE_TDISP_CNTL_BASE_IDX                                                              5
+#define regBIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_31                                                        0xca00
+#define regBIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_31_BASE_IDX                                               5
+#define regBIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_VF0_31                                                    0xca01
+#define regBIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_0_BOUND_PF1_VF0_31                                                    0xca02
+#define regBIF_DEV0_SELECTIVE_IDE_0_BOUND_PF1_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_31                                                        0xca03
+#define regBIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_31_BASE_IDX                                               5
+#define regBIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_VF0_31                                                    0xca04
+#define regBIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_1_BOUND_PF1_VF0_31                                                    0xca05
+#define regBIF_DEV0_SELECTIVE_IDE_1_BOUND_PF1_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_31                                                        0xca06
+#define regBIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_31_BASE_IDX                                               5
+#define regBIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_VF0_31                                                    0xca07
+#define regBIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_2_BOUND_PF1_VF0_31                                                    0xca08
+#define regBIF_DEV0_SELECTIVE_IDE_2_BOUND_PF1_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_31                                                        0xca09
+#define regBIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_31_BASE_IDX                                               5
+#define regBIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_VF0_31                                                    0xca0a
+#define regBIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_3_BOUND_PF1_VF0_31                                                    0xca0b
+#define regBIF_DEV0_SELECTIVE_IDE_3_BOUND_PF1_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_31                                                        0xca0c
+#define regBIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_31_BASE_IDX                                               5
+#define regBIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_VF0_31                                                    0xca0d
+#define regBIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_4_BOUND_PF1_VF0_31                                                    0xca0e
+#define regBIF_DEV0_SELECTIVE_IDE_4_BOUND_PF1_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_31                                                        0xca0f
+#define regBIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_31_BASE_IDX                                               5
+#define regBIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_VF0_31                                                    0xca10
+#define regBIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_5_BOUND_PF1_VF0_31                                                    0xca11
+#define regBIF_DEV0_SELECTIVE_IDE_5_BOUND_PF1_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_31                                                        0xca12
+#define regBIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_31_BASE_IDX                                               5
+#define regBIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_VF0_31                                                    0xca13
+#define regBIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_6_BOUND_PF1_VF0_31                                                    0xca14
+#define regBIF_DEV0_SELECTIVE_IDE_6_BOUND_PF1_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_31                                                        0xca15
+#define regBIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_31_BASE_IDX                                               5
+#define regBIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_VF0_31                                                    0xca16
+#define regBIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_7_BOUND_PF1_VF0_31                                                    0xca17
+#define regBIF_DEV0_SELECTIVE_IDE_7_BOUND_PF1_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_31                                                        0xca18
+#define regBIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_31_BASE_IDX                                               5
+#define regBIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_VF0_31                                                    0xca19
+#define regBIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_VF0_31_BASE_IDX                                           5
+#define regBIF_DEV0_SELECTIVE_IDE_8_BOUND_PF1_VF0_31                                                    0xca1a
+#define regBIF_DEV0_SELECTIVE_IDE_8_BOUND_PF1_VF0_31_BASE_IDX                                           5
+#define regBIF_SYSTEM_EN                                                                                0xca1b
+#define regBIF_SYSTEM_EN_BASE_IDX                                                                       5
+#define regBIF_MASK_SYSTEM_EN                                                                           0xca1c
+#define regBIF_MASK_SYSTEM_EN_BASE_IDX                                                                  5
+#define regGPUIOV_SCH0                                                                                  0xcb40
+#define regGPUIOV_SCH0_BASE_IDX                                                                         5
+#define regGPUIOV_SCH1                                                                                  0xcb41
+#define regGPUIOV_SCH1_BASE_IDX                                                                         5
+#define regGPUIOV_SCH2                                                                                  0xcb42
+#define regGPUIOV_SCH2_BASE_IDX                                                                         5
+#define regGPUIOV_SCH3                                                                                  0xcb43
+#define regGPUIOV_SCH3_BASE_IDX                                                                         5
+#define regGPUIOV_SCH4                                                                                  0xcb44
+#define regGPUIOV_SCH4_BASE_IDX                                                                         5
+#define regGPUIOV_SCH5                                                                                  0xcb45
+#define regGPUIOV_SCH5_BASE_IDX                                                                         5
+#define regGPUIOV_SCH6                                                                                  0xcb46
+#define regGPUIOV_SCH6_BASE_IDX                                                                         5
+#define regGPUIOV_SCH7                                                                                  0xcb47
+#define regGPUIOV_SCH7_BASE_IDX                                                                         5
+#define regGPUIOV_SCH8                                                                                  0xcb48
+#define regGPUIOV_SCH8_BASE_IDX                                                                         5
+#define regGPUIOV_SCH9                                                                                  0xcb49
+#define regGPUIOV_SCH9_BASE_IDX                                                                         5
+#define regGPUIOV_SCH10                                                                                 0xcb4a
+#define regGPUIOV_SCH10_BASE_IDX                                                                        5
+#define regGPUIOV_SCH11                                                                                 0xcb4b
+#define regGPUIOV_SCH11_BASE_IDX                                                                        5
+#define regGPUIOV_SCH12                                                                                 0xcb4c
+#define regGPUIOV_SCH12_BASE_IDX                                                                        5
+#define regGPUIOV_SCH13                                                                                 0xcb4d
+#define regGPUIOV_SCH13_BASE_IDX                                                                        5
+#define regGPUIOV_SCH14                                                                                 0xcb4e
+#define regGPUIOV_SCH14_BASE_IDX                                                                        5
+#define regGPUIOV_SCH15                                                                                 0xcb4f
+#define regGPUIOV_SCH15_BASE_IDX                                                                        5
+#define regGPUIOV_SCH16                                                                                 0xcb50
+#define regGPUIOV_SCH16_BASE_IDX                                                                        5
+#define regGPUIOV_SCH17                                                                                 0xcb51
+#define regGPUIOV_SCH17_BASE_IDX                                                                        5
+#define regGPUIOV_SCH18                                                                                 0xcb52
+#define regGPUIOV_SCH18_BASE_IDX                                                                        5
+#define regGPUIOV_SCH19                                                                                 0xcb53
+#define regGPUIOV_SCH19_BASE_IDX                                                                        5
+#define regGPUIOV_MMHUB_P2P_CNTL                                                                        0xcb54
+#define regGPUIOV_MMHUB_P2P_CNTL_BASE_IDX                                                               5
+#define regBIF_AER_INTR_CTRL                                                                            0xcb55
+#define regBIF_AER_INTR_CTRL_BASE_IDX                                                                   5
+#define regBIF_AER_INTR_STS                                                                             0xcb56
+#define regBIF_AER_INTR_STS_BASE_IDX                                                                    5
+#define regBIF_PCIE_MSG_CTRL                                                                            0xcb57
+#define regBIF_PCIE_MSG_CTRL_BASE_IDX                                                                   5
+#define regBIF_PCIE_MSG_HEADER_DW0                                                                      0xcb58
+#define regBIF_PCIE_MSG_HEADER_DW0_BASE_IDX                                                             5
+#define regBIF_PCIE_MSG_HEADER_DW1                                                                      0xcb59
+#define regBIF_PCIE_MSG_HEADER_DW1_BASE_IDX                                                             5
+#define regBIF_PCIE_MSG_HEADER_DW2                                                                      0xcb5a
+#define regBIF_PCIE_MSG_HEADER_DW2_BASE_IDX                                                             5
+#define regBIF_PCIE_MSG_HEADER_DW3                                                                      0xcb5b
+#define regBIF_PCIE_MSG_HEADER_DW3_BASE_IDX                                                             5
+#define regBIF_PCIE_MSG_DATA_DW0                                                                        0xcb5c
+#define regBIF_PCIE_MSG_DATA_DW0_BASE_IDX                                                               5
+#define regBIF_PCIE_MSG_DATA_DW1                                                                        0xcb5d
+#define regBIF_PCIE_MSG_DATA_DW1_BASE_IDX                                                               5
+#define regBIF_PCIE_MSG_DATA_DW2                                                                        0xcb5e
+#define regBIF_PCIE_MSG_DATA_DW2_BASE_IDX                                                               5
+#define regBIF_PCIE_MSG_DATA_DW3                                                                        0xcb5f
+#define regBIF_PCIE_MSG_DATA_DW3_BASE_IDX                                                               5
+#define regBIFC_SEC_AER_SMN_CTRL0                                                                       0xcb60
+#define regBIFC_SEC_AER_SMN_CTRL0_BASE_IDX                                                              5
+#define regBIFC_SEC_AER_SMN_CTRL1                                                                       0xcb61
+#define regBIFC_SEC_AER_SMN_CTRL1_BASE_IDX                                                              5
+#define regBIFC_SEC_AER_SMN_CTRL2                                                                       0xcb62
+#define regBIFC_SEC_AER_SMN_CTRL2_BASE_IDX                                                              5
+#define regBIF_PCIE_ERR_CTRL0                                                                           0xcb63
+#define regBIF_PCIE_ERR_CTRL0_BASE_IDX                                                                  5
+#define regBIF_PCIE_ERR_CTRL1                                                                           0xcb64
+#define regBIF_PCIE_ERR_CTRL1_BASE_IDX                                                                  5
+#define regBIF_PCIE_ERR_CTRL2                                                                           0xcb65
+#define regBIF_PCIE_ERR_CTRL2_BASE_IDX                                                                  5
+#define regBIF_AER_UNCORR_ERR_STATUS                                                                    0xcb66
+#define regBIF_AER_UNCORR_ERR_STATUS_BASE_IDX                                                           5
+#define regBIF_AER_UNCORR_ERR_MASK                                                                      0xcb67
+#define regBIF_AER_UNCORR_ERR_MASK_BASE_IDX                                                             5
+#define regBIF_AER_UNCORR_ERR_SEVERITY                                                                  0xcb68
+#define regBIF_AER_UNCORR_ERR_SEVERITY_BASE_IDX                                                         5
+#define regBIF_AER_CORR_ERR_STATUS                                                                      0xcb69
+#define regBIF_AER_CORR_ERR_STATUS_BASE_IDX                                                             5
+#define regBIF_AER_CORR_ERR_MASK                                                                        0xcb6a
+#define regBIF_AER_CORR_ERR_MASK_BASE_IDX                                                               5
+#define regRCC_EP_STICKY_RESTORE_0                                                                      0xcb6b
+#define regRCC_EP_STICKY_RESTORE_0_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_1                                                                      0xcb6c
+#define regRCC_EP_STICKY_RESTORE_1_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_2                                                                      0xcb6d
+#define regRCC_EP_STICKY_RESTORE_2_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_3                                                                      0xcb6e
+#define regRCC_EP_STICKY_RESTORE_3_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_4                                                                      0xcb6f
+#define regRCC_EP_STICKY_RESTORE_4_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_5                                                                      0xcb70
+#define regRCC_EP_STICKY_RESTORE_5_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_6                                                                      0xcb71
+#define regRCC_EP_STICKY_RESTORE_6_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_7                                                                      0xcb72
+#define regRCC_EP_STICKY_RESTORE_7_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_8                                                                      0xcb73
+#define regRCC_EP_STICKY_RESTORE_8_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_9                                                                      0xcb74
+#define regRCC_EP_STICKY_RESTORE_9_BASE_IDX                                                             5
+#define regRCC_EP_STICKY_RESTORE_10                                                                     0xcb75
+#define regRCC_EP_STICKY_RESTORE_10_BASE_IDX                                                            5
+#define regRCC_EP_STICKY_RESTORE_11                                                                     0xcb76
+#define regRCC_EP_STICKY_RESTORE_11_BASE_IDX                                                            5
+#define regRCC_EP_STICKY_RESTORE_12                                                                     0xcb77
+#define regRCC_EP_STICKY_RESTORE_12_BASE_IDX                                                            5
+#define regRCC_EP_STICKY_RESTORE_13                                                                     0xcb78
+#define regRCC_EP_STICKY_RESTORE_13_BASE_IDX                                                            5
+#define regBIF_DEVICE3_STATUS                                                                           0xcb79
+#define regBIF_DEVICE3_STATUS_BASE_IDX                                                                  5
+#define regBIFC_SEC_MCTP_SMN_CTRL0                                                                      0xcb81
+#define regBIFC_SEC_MCTP_SMN_CTRL0_BASE_IDX                                                             5
+#define regBIFC_SEC_MCTP_SMN_CTRL1                                                                      0xcb82
+#define regBIFC_SEC_MCTP_SMN_CTRL1_BASE_IDX                                                             5
+#define regNBIF_MCTP_GENERIC_CNTL                                                                       0xcc08
+#define regNBIF_MCTP_GENERIC_CNTL_BASE_IDX                                                              5
+#define regNBIF_MCTP_GENERIC_STATUS                                                                     0xcc09
+#define regNBIF_MCTP_GENERIC_STATUS_BASE_IDX                                                            5
+#define regSDPVW_UPDATE_EN                                                                              0xcc22
+#define regSDPVW_UPDATE_EN_BASE_IDX                                                                     5
+#define regREGS_ROM_OFFSET_CTRL                                                                         0xcc23
+#define regREGS_ROM_OFFSET_CTRL_BASE_IDX                                                                5
+#define regBIFL_SEC_RAS_POISON_DATA_DROP_CTRL                                                           0xcc24
+#define regBIFL_SEC_RAS_POISON_DATA_DROP_CTRL_BASE_IDX                                                  5
+#define regREGS_INDEX_DATA_PAIR_CTRL                                                                    0xcc25
+#define regREGS_INDEX_DATA_PAIR_CTRL_BASE_IDX                                                           5
+#define regBIFL_SEC_RAS_RESPONSE_DROP_CTRL                                                              0xcc26
+#define regBIFL_SEC_RAS_RESPONSE_DROP_CTRL_BASE_IDX                                                     5
+#define regBIFC_SEC_MCA_SMN_CTRL0                                                                       0xcc44
+#define regBIFC_SEC_MCA_SMN_CTRL0_BASE_IDX                                                              5
+#define regBIFC_SEC_MCA_SMN_CTRL1                                                                       0xcc45
+#define regBIFC_SEC_MCA_SMN_CTRL1_BASE_IDX                                                              5
+#define regBIFL_SEC_RAS_POISON_DBUG_CTRL                                                                0xcc46
+#define regBIFL_SEC_RAS_POISON_DBUG_CTRL_BASE_IDX                                                       5
+#define regBIFL_SEC_RAS_PARITY_DBUG_CTRL                                                                0xcc47
+#define regBIFL_SEC_RAS_PARITY_DBUG_CTRL_BASE_IDX                                                       5
+#define regBIFL_SEC_RAS_NTB_DBUG_CTRL                                                                   0xcc48
+#define regBIFL_SEC_RAS_NTB_DBUG_CTRL_BASE_IDX                                                          5
+#define regBIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL                                                           0xcc4a
+#define regBIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL_BASE_IDX                                                  5
+#define regBIFL_SEC_RAS_TIMEOUT_DBUG_CTRL                                                               0xcc4c
+#define regBIFL_SEC_RAS_TIMEOUT_DBUG_CTRL_BASE_IDX                                                      5
+#define regBIFL_SEC_RAS_LEAF0_RRESP_POISON_CTRL                                                         0xcc5b
+#define regBIFL_SEC_RAS_LEAF0_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF0_RUSER_POISON_CTRL                                                         0xcc5c
+#define regBIFL_SEC_RAS_LEAF0_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF1_RRESP_POISON_CTRL                                                         0xcc5d
+#define regBIFL_SEC_RAS_LEAF1_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF1_RUSER_POISON_CTRL                                                         0xcc5e
+#define regBIFL_SEC_RAS_LEAF1_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF2_RRESP_POISON_CTRL                                                         0xcc5f
+#define regBIFL_SEC_RAS_LEAF2_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF2_RUSER_POISON_CTRL                                                         0xcc60
+#define regBIFL_SEC_RAS_LEAF2_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF3_RRESP_POISON_CTRL                                                         0xcc61
+#define regBIFL_SEC_RAS_LEAF3_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF3_RUSER_POISON_CTRL                                                         0xcc62
+#define regBIFL_SEC_RAS_LEAF3_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF4_RRESP_POISON_CTRL                                                         0xcc63
+#define regBIFL_SEC_RAS_LEAF4_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF4_RUSER_POISON_CTRL                                                         0xcc64
+#define regBIFL_SEC_RAS_LEAF4_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF5_RRESP_POISON_CTRL                                                         0xcc65
+#define regBIFL_SEC_RAS_LEAF5_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF5_RUSER_POISON_CTRL                                                         0xcc66
+#define regBIFL_SEC_RAS_LEAF5_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF6_RRESP_POISON_CTRL                                                         0xcc67
+#define regBIFL_SEC_RAS_LEAF6_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF6_RUSER_POISON_CTRL                                                         0xcc68
+#define regBIFL_SEC_RAS_LEAF6_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF7_RRESP_POISON_CTRL                                                         0xcc69
+#define regBIFL_SEC_RAS_LEAF7_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF7_RUSER_POISON_CTRL                                                         0xcc6a
+#define regBIFL_SEC_RAS_LEAF7_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF8_RRESP_POISON_CTRL                                                         0xcc6b
+#define regBIFL_SEC_RAS_LEAF8_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF8_RUSER_POISON_CTRL                                                         0xcc6c
+#define regBIFL_SEC_RAS_LEAF8_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF9_RRESP_POISON_CTRL                                                         0xcc6d
+#define regBIFL_SEC_RAS_LEAF9_RRESP_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF9_RUSER_POISON_CTRL                                                         0xcc6e
+#define regBIFL_SEC_RAS_LEAF9_RUSER_POISON_CTRL_BASE_IDX                                                5
+#define regBIFL_SEC_RAS_LEAF10_RRESP_POISON_CTRL                                                        0xcc6f
+#define regBIFL_SEC_RAS_LEAF10_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF10_RUSER_POISON_CTRL                                                        0xcc70
+#define regBIFL_SEC_RAS_LEAF10_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF11_RRESP_POISON_CTRL                                                        0xcc71
+#define regBIFL_SEC_RAS_LEAF11_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF11_RUSER_POISON_CTRL                                                        0xcc72
+#define regBIFL_SEC_RAS_LEAF11_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF12_RRESP_POISON_CTRL                                                        0xcc73
+#define regBIFL_SEC_RAS_LEAF12_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF12_RUSER_POISON_CTRL                                                        0xcc74
+#define regBIFL_SEC_RAS_LEAF12_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF13_RRESP_POISON_CTRL                                                        0xcc75
+#define regBIFL_SEC_RAS_LEAF13_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF13_RUSER_POISON_CTRL                                                        0xcc76
+#define regBIFL_SEC_RAS_LEAF13_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF14_RRESP_POISON_CTRL                                                        0xcc77
+#define regBIFL_SEC_RAS_LEAF14_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF14_RUSER_POISON_CTRL                                                        0xcc78
+#define regBIFL_SEC_RAS_LEAF14_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF15_RRESP_POISON_CTRL                                                        0xcc79
+#define regBIFL_SEC_RAS_LEAF15_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF15_RUSER_POISON_CTRL                                                        0xcc7a
+#define regBIFL_SEC_RAS_LEAF15_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF16_RRESP_POISON_CTRL                                                        0xcc7b
+#define regBIFL_SEC_RAS_LEAF16_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF16_RUSER_POISON_CTRL                                                        0xcc7c
+#define regBIFL_SEC_RAS_LEAF16_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF17_RRESP_POISON_CTRL                                                        0xcc7d
+#define regBIFL_SEC_RAS_LEAF17_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF17_RUSER_POISON_CTRL                                                        0xcc7e
+#define regBIFL_SEC_RAS_LEAF17_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF18_RRESP_POISON_CTRL                                                        0xcc7f
+#define regBIFL_SEC_RAS_LEAF18_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF18_RUSER_POISON_CTRL                                                        0xcc80
+#define regBIFL_SEC_RAS_LEAF18_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF19_RRESP_POISON_CTRL                                                        0xcc81
+#define regBIFL_SEC_RAS_LEAF19_RRESP_POISON_CTRL_BASE_IDX                                               5
+#define regBIFL_SEC_RAS_LEAF19_RUSER_POISON_CTRL                                                        0xcc82
+#define regBIFL_SEC_RAS_LEAF19_RUSER_POISON_CTRL_BASE_IDX                                               5
+#define regNBIF_STRAP_BIOS_CNTL                                                                         0xcc89
+#define regNBIF_STRAP_BIOS_CNTL_BASE_IDX                                                                5
+#define regSMN_MST_NORMAL_SECURITY                                                                      0xcdea
+#define regSMN_MST_NORMAL_SECURITY_BASE_IDX                                                             5
+#define regOVERRIDE_TYPE_FOR_CHAIN_CTRL                                                                 0xcdec
+#define regOVERRIDE_TYPE_FOR_CHAIN_CTRL_BASE_IDX                                                        5
+#define regPROCESSING_HINT_SUPPORT_CNTL                                                                 0xcded
+#define regPROCESSING_HINT_SUPPORT_CNTL_BASE_IDX                                                        5
+#define regDEV0_PF0_VF0_MMIO0_FENCE                                                                     0xce40
+#define regDEV0_PF0_VF0_MMIO0_FENCE_BASE_IDX                                                            5
+#define regDEV0_PF0_VF1_MMIO0_FENCE                                                                     0xce41
+#define regDEV0_PF0_VF1_MMIO0_FENCE_BASE_IDX                                                            5
+#define regDEV0_PF0_VF2_MMIO0_FENCE                                                                     0xce42
+#define regDEV0_PF0_VF2_MMIO0_FENCE_BASE_IDX                                                            5
+#define regDEV0_PF0_VF3_MMIO0_FENCE                                                                     0xce43
+#define regDEV0_PF0_VF3_MMIO0_FENCE_BASE_IDX                                                            5
+#define regDEV0_PF0_VF4_MMIO0_FENCE                                                                     0xce44
+#define regDEV0_PF0_VF4_MMIO0_FENCE_BASE_IDX                                                            5
+#define regDEV0_PF0_VF5_MMIO0_FENCE                                                                     0xce45
+#define regDEV0_PF0_VF5_MMIO0_FENCE_BASE_IDX                                                            5
+#define regDEV0_PF0_VF6_MMIO0_FENCE                                                                     0xce46
+#define regDEV0_PF0_VF6_MMIO0_FENCE_BASE_IDX                                                            5
+#define regDEV0_PF0_VF7_MMIO0_FENCE                                                                     0xce47
+#define regDEV0_PF0_VF7_MMIO0_FENCE_BASE_IDX                                                            5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE0_START                                                         0xce48
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE0_START_BASE_IDX                                                5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE0_END                                                           0xce49
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE0_END_BASE_IDX                                                  5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE1_START                                                         0xce4a
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE1_START_BASE_IDX                                                5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE1_END                                                           0xce4b
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE1_END_BASE_IDX                                                  5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE2_START                                                         0xce4c
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE2_START_BASE_IDX                                                5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE2_END                                                           0xce4d
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE2_END_BASE_IDX                                                  5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE3_START                                                         0xce4e
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE3_START_BASE_IDX                                                5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE3_END                                                           0xce4f
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE3_END_BASE_IDX                                                  5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE4_START                                                         0xce50
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE4_START_BASE_IDX                                                5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE4_END                                                           0xce51
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE4_END_BASE_IDX                                                  5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE5_START                                                         0xce52
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE5_START_BASE_IDX                                                5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE5_END                                                           0xce53
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE5_END_BASE_IDX                                                  5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE6_START                                                         0xce54
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE6_START_BASE_IDX                                                5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE6_END                                                           0xce55
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE6_END_BASE_IDX                                                  5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE7_START                                                         0xce56
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE7_START_BASE_IDX                                                5
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE7_END                                                           0xce57
+#define regDEV0_PF0_VF_MMIO0_FENCE_RANGE7_END_BASE_IDX                                                  5
+#define regDEV0_PF1_VF0_MMIO0_FENCE                                                                     0xce58
+#define regDEV0_PF1_VF0_MMIO0_FENCE_BASE_IDX                                                            5
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE0_START                                                         0xce59
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE0_START_BASE_IDX                                                5
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE0_END                                                           0xce5a
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE0_END_BASE_IDX                                                  5
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE1_START                                                         0xce5b
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE1_START_BASE_IDX                                                5
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE1_END                                                           0xce5c
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE1_END_BASE_IDX                                                  5
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE2_START                                                         0xce5d
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE2_START_BASE_IDX                                                5
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE2_END                                                           0xce5e
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE2_END_BASE_IDX                                                  5
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE3_START                                                         0xce5f
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE3_START_BASE_IDX                                                5
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE3_END                                                           0xce60
+#define regDEV0_PF1_VF_MMIO0_FENCE_RANGE3_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_WR                                                                   0xce61
+#define regDEV0_PF0_PF_MMIO0_FENCE_WR_BASE_IDX                                                          5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RD                                                                   0xce62
+#define regDEV0_PF0_PF_MMIO0_FENCE_RD_BASE_IDX                                                          5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE0_START                                                         0xce63
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE0_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE0_END                                                           0xce64
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE0_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE1_START                                                         0xce65
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE1_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE1_END                                                           0xce66
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE1_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE2_START                                                         0xce67
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE2_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE2_END                                                           0xce68
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE2_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE3_START                                                         0xce69
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE3_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE3_END                                                           0xce6a
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE3_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE4_START                                                         0xce6b
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE4_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE4_END                                                           0xce6c
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE4_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE5_START                                                         0xce6d
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE5_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE5_END                                                           0xce6e
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE5_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE6_START                                                         0xce6f
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE6_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE6_END                                                           0xce70
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE6_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE7_START                                                         0xce71
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE7_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE7_END                                                           0xce72
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE7_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE8_START                                                         0xce73
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE8_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE8_END                                                           0xce74
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE8_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE9_START                                                         0xce75
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE9_START_BASE_IDX                                                5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE9_END                                                           0xce76
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE9_END_BASE_IDX                                                  5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE10_START                                                        0xce77
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE10_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE10_END                                                          0xce78
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE10_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE11_START                                                        0xce79
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE11_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE11_END                                                          0xce7a
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE11_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE12_START                                                        0xce7b
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE12_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE12_END                                                          0xce7c
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE12_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE13_START                                                        0xce7d
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE13_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE13_END                                                          0xce7e
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE13_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE14_START                                                        0xce7f
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE14_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE14_END                                                          0xce80
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE14_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE15_START                                                        0xce81
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE15_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE15_END                                                          0xce82
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE15_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE16_START                                                        0xce83
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE16_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE16_END                                                          0xce84
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE16_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE17_START                                                        0xce85
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE17_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE17_END                                                          0xce86
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE17_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE18_START                                                        0xce87
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE18_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE18_END                                                          0xce88
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE18_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE19_START                                                        0xce89
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE19_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE19_END                                                          0xce8a
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE19_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE20_START                                                        0xce8b
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE20_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE20_END                                                          0xce8c
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE20_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE21_START                                                        0xce8d
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE21_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE21_END                                                          0xce8e
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE21_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE22_START                                                        0xce8f
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE22_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE22_END                                                          0xce90
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE22_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE23_START                                                        0xce91
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE23_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE23_END                                                          0xce92
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE23_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE24_START                                                        0xce93
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE24_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE24_END                                                          0xce94
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE24_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE25_START                                                        0xce95
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE25_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE25_END                                                          0xce96
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE25_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE26_START                                                        0xce97
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE26_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE26_END                                                          0xce98
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE26_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE27_START                                                        0xce99
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE27_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE27_END                                                          0xce9a
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE27_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE28_START                                                        0xce9b
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE28_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE28_END                                                          0xce9c
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE28_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE29_START                                                        0xce9d
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE29_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE29_END                                                          0xce9e
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE29_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE30_START                                                        0xce9f
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE30_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE30_END                                                          0xcea0
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE30_END_BASE_IDX                                                 5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE31_START                                                        0xcea1
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE31_START_BASE_IDX                                               5
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE31_END                                                          0xcea2
+#define regDEV0_PF0_PF_MMIO0_FENCE_RANGE31_END_BASE_IDX                                                 5
+#define regDEV0_PF1_PF_MMIO0_FENCE_WR                                                                   0xcea3
+#define regDEV0_PF1_PF_MMIO0_FENCE_WR_BASE_IDX                                                          5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RD                                                                   0xcea4
+#define regDEV0_PF1_PF_MMIO0_FENCE_RD_BASE_IDX                                                          5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE0_START                                                         0xcea5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE0_START_BASE_IDX                                                5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE0_END                                                           0xcea6
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE0_END_BASE_IDX                                                  5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE1_START                                                         0xcea7
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE1_START_BASE_IDX                                                5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE1_END                                                           0xcea8
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE1_END_BASE_IDX                                                  5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE2_START                                                         0xcea9
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE2_START_BASE_IDX                                                5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE2_END                                                           0xceaa
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE2_END_BASE_IDX                                                  5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE3_START                                                         0xceab
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE3_START_BASE_IDX                                                5
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE3_END                                                           0xceac
+#define regDEV0_PF1_PF_MMIO0_FENCE_RANGE3_END_BASE_IDX                                                  5
+#define regMISC_SCRATCH                                                                                 0xe800
+#define regMISC_SCRATCH_BASE_IDX                                                                        5
+#define regINTR_LINE_POLARITY                                                                           0xe801
+#define regINTR_LINE_POLARITY_BASE_IDX                                                                  5
+#define regINTR_LINE_ENABLE                                                                             0xe802
+#define regINTR_LINE_ENABLE_BASE_IDX                                                                    5
+#define regOUTSTANDING_VC_ALLOC                                                                         0xe803
+#define regOUTSTANDING_VC_ALLOC_BASE_IDX                                                                5
+#define regBIFC_MISC_CTRL0                                                                              0xe804
+#define regBIFC_MISC_CTRL0_BASE_IDX                                                                     5
+#define regBIFC_MISC_CTRL1                                                                              0xe805
+#define regBIFC_MISC_CTRL1_BASE_IDX                                                                     5
+#define regBIFC_BME_ERR_LOG_LB                                                                          0xe806
+#define regBIFC_BME_ERR_LOG_LB_BASE_IDX                                                                 5
+#define regBIFC_LC_TIMER_CTRL                                                                           0xe807
+#define regBIFC_LC_TIMER_CTRL_BASE_IDX                                                                  5
+#define regBIFC_RCCBIH_BME_ERR_LOG0                                                                     0xe808
+#define regBIFC_RCCBIH_BME_ERR_LOG0_BASE_IDX                                                            5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1                                                            0xe80a
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3                                                            0xe80b
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5                                                            0xe80c
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7                                                            0xe80d
+#define regBIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7_BASE_IDX                                                   5
+#define regBIFC_DMA_ATTR_CNTL2_DEV0                                                                     0xe81a
+#define regBIFC_DMA_ATTR_CNTL2_DEV0_BASE_IDX                                                            5
+#define regBIFC_MISC_CTRL2                                                                              0xe822
+#define regBIFC_MISC_CTRL2_BASE_IDX                                                                     5
+#define regBME_DUMMY_CNTL_0                                                                             0xe825
+#define regBME_DUMMY_CNTL_0_BASE_IDX                                                                    5
+#define regBIFC_THT_CNTL                                                                                0xe827
+#define regBIFC_THT_CNTL_BASE_IDX                                                                       5
+#define regBIFC_HSTARB_CNTL                                                                             0xe828
+#define regBIFC_HSTARB_CNTL_BASE_IDX                                                                    5
+#define regBIFC_GSI_CNTL                                                                                0xe829
+#define regBIFC_GSI_CNTL_BASE_IDX                                                                       5
+#define regBIFC_PCIEFUNC_CNTL                                                                           0xe82a
+#define regBIFC_PCIEFUNC_CNTL_BASE_IDX                                                                  5
+#define regBIFC_PASID_CHECK_DIS                                                                         0xe82b
+#define regBIFC_PASID_CHECK_DIS_BASE_IDX                                                                5
+#define regBIFC_SDP_CNTL_0                                                                              0xe82c
+#define regBIFC_SDP_CNTL_0_BASE_IDX                                                                     5
+#define regBIFC_SDP_CNTL_1                                                                              0xe82d
+#define regBIFC_SDP_CNTL_1_BASE_IDX                                                                     5
+#define regBIFC_PASID_STS                                                                               0xe82e
+#define regBIFC_PASID_STS_BASE_IDX                                                                      5
+#define regBIFC_ATHUB_ACT_CNTL                                                                          0xe82f
+#define regBIFC_ATHUB_ACT_CNTL_BASE_IDX                                                                 5
+#define regBIFC_PERF_CNTL_0                                                                             0xe830
+#define regBIFC_PERF_CNTL_0_BASE_IDX                                                                    5
+#define regBIF_MISC_BIFC_PERF_CNTL_1                                                                    0xe831
+#define regBIF_MISC_BIFC_PERF_CNTL_1_BASE_IDX                                                           5
+#define regBIFC_PERF_CNT_MMIO_RD_L32BIT                                                                 0xe832
+#define regBIFC_PERF_CNT_MMIO_RD_L32BIT_BASE_IDX                                                        5
+#define regBIFC_PERF_CNT_MMIO_WR_L32BIT                                                                 0xe833
+#define regBIFC_PERF_CNT_MMIO_WR_L32BIT_BASE_IDX                                                        5
+#define regBIF_MISC_BIFC_PERF_CNT_DMA_RD_L32BIT                                                         0xe834
+#define regBIF_MISC_BIFC_PERF_CNT_DMA_RD_L32BIT_BASE_IDX                                                5
+#define regBIF_MISC_BIFC_PERF_CNT_DMA_WR_L32BIT                                                         0xe835
+#define regBIF_MISC_BIFC_PERF_CNT_DMA_WR_L32BIT_BASE_IDX                                                5
+#define regNBIF_REGIF_ERRSET_CTRL                                                                       0xe836
+#define regNBIF_REGIF_ERRSET_CTRL_BASE_IDX                                                              5
+#define regBIFC_SDP_CNTL_2                                                                              0xe837
+#define regBIFC_SDP_CNTL_2_BASE_IDX                                                                     5
+#define regNBIF_PGMST_CTRL                                                                              0xe838
+#define regNBIF_PGMST_CTRL_BASE_IDX                                                                     5
+#define regNBIF_PGSLV_CTRL                                                                              0xe839
+#define regNBIF_PGSLV_CTRL_BASE_IDX                                                                     5
+#define regNBIF_PG_MISC_CTRL                                                                            0xe83a
+#define regNBIF_PG_MISC_CTRL_BASE_IDX                                                                   5
+#define regSMN_MST_EP_CNTL3                                                                             0xe83c
+#define regSMN_MST_EP_CNTL3_BASE_IDX                                                                    5
+#define regSMN_MST_EP_CNTL4                                                                             0xe83d
+#define regSMN_MST_EP_CNTL4_BASE_IDX                                                                    5
+#define regSMN_MST_CNTL1                                                                                0xe83e
+#define regSMN_MST_CNTL1_BASE_IDX                                                                       5
+#define regSMN_MST_EP_CNTL5                                                                             0xe83f
+#define regSMN_MST_EP_CNTL5_BASE_IDX                                                                    5
+#define regBIF_SELFRING_BUFFER_VID                                                                      0xe840
+#define regBIF_SELFRING_BUFFER_VID_BASE_IDX                                                             5
+#define regBIF_SELFRING_VECTOR_CNTL                                                                     0xe841
+#define regBIF_SELFRING_VECTOR_CNTL_BASE_IDX                                                            5
+#define regHDP_ERR_STATUS_EXT                                                                           0xe842
+#define regHDP_ERR_STATUS_EXT_BASE_IDX                                                                  5
+#define regNBIF_STRAP_WRITE_CTRL                                                                        0xe845
+#define regNBIF_STRAP_WRITE_CTRL_BASE_IDX                                                               5
+#define regNBIF_INTX_DSTATE_MISC_CNTL                                                                   0xe846
+#define regNBIF_INTX_DSTATE_MISC_CNTL_BASE_IDX                                                          5
+#define regNBIF_PENDING_MISC_CNTL                                                                       0xe847
+#define regNBIF_PENDING_MISC_CNTL_BASE_IDX                                                              5
+#define regBIF_GMI_WRR_WEIGHT                                                                           0xe848
+#define regBIF_GMI_WRR_WEIGHT_BASE_IDX                                                                  5
+#define regBIF_GMI_WRR_WEIGHT2                                                                          0xe849
+#define regBIF_GMI_WRR_WEIGHT2_BASE_IDX                                                                 5
+#define regBIF_GMI_WRR_WEIGHT3                                                                          0xe84a
+#define regBIF_GMI_WRR_WEIGHT3_BASE_IDX                                                                 5
+#define regBIF_GMI_WRR_WEIGHT4                                                                          0xe84b
+#define regBIF_GMI_WRR_WEIGHT4_BASE_IDX                                                                 5
+#define regNBIF_PWRBRK_REQUEST                                                                          0xe84c
+#define regNBIF_PWRBRK_REQUEST_BASE_IDX                                                                 5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F0                                                                   0xe850
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F0_BASE_IDX                                                          5
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F1                                                                   0xe851
+#define regBIF_ATOMIC_ERR_LOG_DEV0_F1_BASE_IDX                                                          5
+#define regBIF_DMA_MP4_ERR_LOG                                                                          0xe870
+#define regBIF_DMA_MP4_ERR_LOG_BASE_IDX                                                                 5
+#define regBIF_PASID_ERR_LOG                                                                            0xe871
+#define regBIF_PASID_ERR_LOG_BASE_IDX                                                                   5
+#define regBIF_PASID_ERR_CLR                                                                            0xe872
+#define regBIF_PASID_ERR_CLR_BASE_IDX                                                                   5
+#define regNBIF_VWIRE_CTRL                                                                              0xe880
+#define regNBIF_VWIRE_CTRL_BASE_IDX                                                                     5
+#define regNBIF_MGCG_CTRL_LCLK                                                                          0xe887
+#define regNBIF_MGCG_CTRL_LCLK_BASE_IDX                                                                 5
+#define regNBIF_DS_CTRL_LCLK                                                                            0xe888
+#define regNBIF_DS_CTRL_LCLK_BASE_IDX                                                                   5
+#define regSMN_MST_CNTL0                                                                                0xe889
+#define regSMN_MST_CNTL0_BASE_IDX                                                                       5
+#define regSMN_MST_EP_CNTL1                                                                             0xe88a
+#define regSMN_MST_EP_CNTL1_BASE_IDX                                                                    5
+#define regSMN_MST_EP_CNTL2                                                                             0xe88b
+#define regSMN_MST_EP_CNTL2_BASE_IDX                                                                    5
+#define regNBIF_SDP_VWR_VCHG_DIS_CTRL                                                                   0xe88c
+#define regNBIF_SDP_VWR_VCHG_DIS_CTRL_BASE_IDX                                                          5
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL0                                                                  0xe88d
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL0_BASE_IDX                                                         5
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL1                                                                  0xe88e
+#define regNBIF_SDP_VWR_VCHG_RST_CTRL1_BASE_IDX                                                         5
+#define regNBIF_SDP_VWR_VCHG_TRIG                                                                       0xe88f
+#define regNBIF_SDP_VWR_VCHG_TRIG_BASE_IDX                                                              5
+#define regNBIO_LCLK_DEEPSLEEP_MASK                                                                     0xe890
+#define regNBIO_LCLK_DEEPSLEEP_MASK_BASE_IDX                                                            5
+#define regNBIF_DS_CTRL_USBCLK                                                                          0xe891
+#define regNBIF_DS_CTRL_USBCLK_BASE_IDX                                                                 5
+#define regNBIF_SHUB_TODET_CLIENT_MISC3                                                                 0xe897
+#define regNBIF_SHUB_TODET_CLIENT_MISC3_BASE_IDX                                                        5
+#define regNBIF_SHUB_TODET_CTRL                                                                         0xe898
+#define regNBIF_SHUB_TODET_CTRL_BASE_IDX                                                                5
+#define regNBIF_SHUB_TODET_CLIENT_CTRL                                                                  0xe899
+#define regNBIF_SHUB_TODET_CLIENT_CTRL_BASE_IDX                                                         5
+#define regNBIF_SHUB_TODET_CLIENT_STATUS                                                                0xe89a
+#define regNBIF_SHUB_TODET_CLIENT_STATUS_BASE_IDX                                                       5
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL                                                               0xe89b
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL_BASE_IDX                                                      5
+#define regNBIF_SHUB_TODET_CLIENT_CTRL2                                                                 0xe89c
+#define regNBIF_SHUB_TODET_CLIENT_CTRL2_BASE_IDX                                                        5
+#define regNBIF_SHUB_TODET_CLIENT_STATUS2                                                               0xe89d
+#define regNBIF_SHUB_TODET_CLIENT_STATUS2_BASE_IDX                                                      5
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL2                                                              0xe89e
+#define regNBIF_SHUB_TODET_SYNCFLOOD_CTRL2_BASE_IDX                                                     5
+#define regDMA_SDP_PORT_DEBUG                                                                           0xe8a0
+#define regDMA_SDP_PORT_DEBUG_BASE_IDX                                                                  5
+#define regSDP_PORT_MONITOR_CONTROL                                                                     0xe8a2
+#define regSDP_PORT_MONITOR_CONTROL_BASE_IDX                                                            5
+#define regSDP_PORT_MONITOR_ADDRESS0                                                                    0xe8a3
+#define regSDP_PORT_MONITOR_ADDRESS0_BASE_IDX                                                           5
+#define regSDP_PORT_MONITOR_ADDRESS1                                                                    0xe8a4
+#define regSDP_PORT_MONITOR_ADDRESS1_BASE_IDX                                                           5
+#define regSDP_PORT_MONITOR_UNITID                                                                      0xe8a5
+#define regSDP_PORT_MONITOR_UNITID_BASE_IDX                                                             5
+#define regSDP_PORT_MONITOR_INFO0                                                                       0xe8a6
+#define regSDP_PORT_MONITOR_INFO0_BASE_IDX                                                              5
+#define regSDP_PORT_MONITOR_INFO1                                                                       0xe8a7
+#define regSDP_PORT_MONITOR_INFO1_BASE_IDX                                                              5
+#define regSDP_PORT_MONITOR_INFO2                                                                       0xe8a8
+#define regSDP_PORT_MONITOR_INFO2_BASE_IDX                                                              5
+#define regSDP_PORT_MONITOR_INFO3                                                                       0xe8a9
+#define regSDP_PORT_MONITOR_INFO3_BASE_IDX                                                              5
+#define regSDP_PORT_MONITOR_INFO4                                                                       0xe8aa
+#define regSDP_PORT_MONITOR_INFO4_BASE_IDX                                                              5
+#define regBIFC_BME_ERR_LOG_HB                                                                          0xe8ab
+#define regBIFC_BME_ERR_LOG_HB_BASE_IDX                                                                 5
+#define regBIFC_GFX_INT_MONITOR_MASK                                                                    0xe8ad
+#define regBIFC_GFX_INT_MONITOR_MASK_BASE_IDX                                                           5
+#define regBIFC_GFX_INT_MONITOR_STS                                                                     0xe8ae
+#define regBIFC_GFX_INT_MONITOR_STS_BASE_IDX                                                            5
+#define regBIFC_GFX_INT_MONITOR_CTRL                                                                    0xe8af
+#define regBIFC_GFX_INT_MONITOR_CTRL_BASE_IDX                                                           5
+#define regNBIF_FLUSH_CTRL                                                                              0xe8b1
+#define regNBIF_FLUSH_CTRL_BASE_IDX                                                                     5
+#define regNBIF_GUI_FLUSH_CNTL                                                                          0xe8b3
+#define regNBIF_GUI_FLUSH_CNTL_BASE_IDX                                                                 5
+#define regBIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC                                                            0xe8c0
+#define regBIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC_BASE_IDX                                                   5
+#define regBIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC                                                            0xe8c1
+#define regBIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC_BASE_IDX                                                   5
+#define regBIFC_GMI_SDP_REQ_POOLCRED_ALLOC                                                              0xe8c2
+#define regBIFC_GMI_SDP_REQ_POOLCRED_ALLOC_BASE_IDX                                                     5
+#define regBIFC_GMI_SDP_DAT_POOLCRED_ALLOC                                                              0xe8c3
+#define regBIFC_GMI_SDP_DAT_POOLCRED_ALLOC_BASE_IDX                                                     5
+#define regBIFC_GMI_SST_RDRSP_POOLCRED_ALLOC                                                            0xe8c4
+#define regBIFC_GMI_SST_RDRSP_POOLCRED_ALLOC_BASE_IDX                                                   5
+#define regBIFC_GMI_SST_WRRSP_POOLCRED_ALLOC                                                            0xe8c5
+#define regBIFC_GMI_SST_WRRSP_POOLCRED_ALLOC_BASE_IDX                                                   5
+#define regDISCON_HYSTERESIS_HEAD_CTRL                                                                  0xe8c6
+#define regDISCON_HYSTERESIS_HEAD_CTRL_BASE_IDX                                                         5
+#define regBIFC_GSI_SDP_REQ_POOLCRED_ALLOC                                                              0xe8c7
+#define regBIFC_GSI_SDP_REQ_POOLCRED_ALLOC_BASE_IDX                                                     5
+#define regBIFC_GSI_SDP_DAT_POOLCRED_ALLOC                                                              0xe8c8
+#define regBIFC_GSI_SDP_DAT_POOLCRED_ALLOC_BASE_IDX                                                     5
+#define regBIFC_MCA_SMN_CTRL0                                                                           0xe8cb
+#define regBIFC_MCA_SMN_CTRL0_BASE_IDX                                                                  5
+#define regBIFC_MCTP_SMN_CTRL0                                                                          0xe8ce
+#define regBIFC_MCTP_SMN_CTRL0_BASE_IDX                                                                 5
+#define regNBIF_REGIF_DEBUG_COUNTER_CTRL                                                                0xe8cf
+#define regNBIF_REGIF_DEBUG_COUNTER_CTRL_BASE_IDX                                                       5
+#define regBIFC_EARLY_WAKEUP_CNTL                                                                       0xe8d2
+#define regBIFC_EARLY_WAKEUP_CNTL_BASE_IDX                                                              5
+#define regBIFC_PERF_CNT_MMIO_RD_H16BIT                                                                 0xe8f0
+#define regBIFC_PERF_CNT_MMIO_RD_H16BIT_BASE_IDX                                                        5
+#define regBIFC_PERF_CNT_MMIO_WR_H16BIT                                                                 0xe8f1
+#define regBIFC_PERF_CNT_MMIO_WR_H16BIT_BASE_IDX                                                        5
+#define regBIF_MISC_BIFC_PERF_CNT_DMA_RD_H16BIT                                                         0xe8f2
+#define regBIF_MISC_BIFC_PERF_CNT_DMA_RD_H16BIT_BASE_IDX                                                5
+#define regBIF_MISC_BIFC_PERF_CNT_DMA_WR_H16BIT                                                         0xe8f3
+#define regBIF_MISC_BIFC_PERF_CNT_DMA_WR_H16BIT_BASE_IDX                                                5
+#define regBIF_MISC_NBIF_PERF_COM_COUNT_ENABLE                                                          0xe8f4
+#define regBIF_MISC_NBIF_PERF_COM_COUNT_ENABLE_BASE_IDX                                                 5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_FSM                                                               0xe8f5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_FSM_BASE_IDX                                                      5
+#define regBIF_MISC_NBIF_SDP_PERF_COUNTER                                                               0xe8f6
+#define regBIF_MISC_NBIF_SDP_PERF_COUNTER_BASE_IDX                                                      5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_L32BIT                                                    0xe8f7
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_L32BIT_BASE_IDX                                           5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_L32BIT                                                    0xe8f8
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_L32BIT_BASE_IDX                                           5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_L32BIT                                                    0xe8f9
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_L32BIT_BASE_IDX                                           5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_L32BIT                                                    0xe8fa
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_L32BIT_BASE_IDX                                           5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_H16BIT                                                    0xe8fb
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_H16BIT_BASE_IDX                                           5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_H16BIT                                                    0xe8fc
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_H16BIT_BASE_IDX                                           5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_H16BIT                                                    0xe8fd
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_H16BIT_BASE_IDX                                           5
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_H16BIT                                                    0xe8fe
+#define regBIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_H16BIT_BASE_IDX                                           5
+#define regBIF_MISC_NBIF_BX_PERF_CNT_FSM                                                                0xe8ff
+#define regBIF_MISC_NBIF_BX_PERF_CNT_FSM_BASE_IDX                                                       5
+#define regNBIF_TDISP_ENTER_ERR_CNTL                                                                    0xe905
+#define regNBIF_TDISP_ENTER_ERR_CNTL_BASE_IDX                                                           5
+#define regBIF_UNITID_ERR_LOG                                                                           0xe906
+#define regBIF_UNITID_ERR_LOG_BASE_IDX                                                                  5
+#define regNBIF_COM_COUNT_VALUE                                                                         0xe908
+#define regNBIF_COM_COUNT_VALUE_BASE_IDX                                                                5
+#define regNBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL                                                          0xe909
+#define regNBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL_BASE_IDX                                                 5
+#define regNBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL                                                   0xe90a
+#define regNBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL_BASE_IDX                                          5
+#define regBIF_AER_CNTL                                                                                 0xe92c
+#define regBIF_AER_CNTL_BASE_IDX                                                                        5
+#define regNBIF_IDE_INFO_CNTL                                                                           0xe92d
+#define regNBIF_IDE_INFO_CNTL_BASE_IDX                                                                  5
+#define regBIFC_EXP_MISC_CTRL0                                                                          0xe9c0
+#define regBIFC_EXP_MISC_CTRL0_BASE_IDX                                                                 5
+#define regSECOND_HDP_MEM_FLUSH_CNTL_PF                                                                 0xea80
+#define regSECOND_HDP_MEM_FLUSH_CNTL_PF_BASE_IDX                                                        5
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF0                                                                0xea81
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF0_BASE_IDX                                                       5
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF1                                                                0xea82
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF1_BASE_IDX                                                       5
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF2                                                                0xea83
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF2_BASE_IDX                                                       5
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF3                                                                0xea84
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF3_BASE_IDX                                                       5
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF4                                                                0xea85
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF4_BASE_IDX                                                       5
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF5                                                                0xea86
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF5_BASE_IDX                                                       5
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF6                                                                0xea87
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF6_BASE_IDX                                                       5
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF7                                                                0xea88
+#define regSECOND_HDP_MEM_FLUSH_CNTL_VF7_BASE_IDX                                                       5
+#define regSECOND_HDP_REG_FLUSH_CNTL_PF                                                                 0xea89
+#define regSECOND_HDP_REG_FLUSH_CNTL_PF_BASE_IDX                                                        5
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF0                                                                0xea8a
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF0_BASE_IDX                                                       5
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF1                                                                0xea8b
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF1_BASE_IDX                                                       5
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF2                                                                0xea8c
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF2_BASE_IDX                                                       5
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF3                                                                0xea8d
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF3_BASE_IDX                                                       5
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF4                                                                0xea8e
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF4_BASE_IDX                                                       5
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF5                                                                0xea8f
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF5_BASE_IDX                                                       5
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF6                                                                0xea90
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF6_BASE_IDX                                                       5
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF7                                                                0xea91
+#define regSECOND_HDP_REG_FLUSH_CNTL_VF7_BASE_IDX                                                       5
+#define regGPU_HDP_FLUSH_GFX_FW                                                                         0xf4c0
+#define regGPU_HDP_FLUSH_GFX_FW_BASE_IDX                                                                5
+#define regNBIF_RRMT_CNTL                                                                               0xf500
+#define regNBIF_RRMT_CNTL_BASE_IDX                                                                      5
+#define regRRMT_NBIF_MISC_REG1                                                                          0xf501
+#define regRRMT_NBIF_MISC_REG1_BASE_IDX                                                                 5
+#define regRRMT_LUT_INDEX                                                                               0xf502
+#define regRRMT_LUT_INDEX_BASE_IDX                                                                      5
+#define regRRMT_LUT_DATA                                                                                0xf503
+#define regRRMT_LUT_DATA_BASE_IDX                                                                       5
+
+
+// addressBlock: nbif0_nbif0_bif_misc_pfvf_dev0_epf0_bif_misc_pfvf_regblk
+// base address: 0x10100000
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0                                                 0xf600
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1                                                 0xf601
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2                                                 0xf602
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3                                                 0xf603
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0                                             0xf604
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1                                             0xf605
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2                                             0xf606
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3                                             0xf607
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0                                          0xf608
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0_BASE_IDX                                 5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1                                          0xf609
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1_BASE_IDX                                 5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2                                          0xf60a
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2_BASE_IDX                                 5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3                                          0xf60b
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3_BASE_IDX                                 5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0                                             0xf60c
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1                                             0xf60d
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2                                             0xf60e
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3                                             0xf60f
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0                                                 0xf610
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1                                                 0xf611
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2                                                 0xf612
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3                                                 0xf613
+#define regBIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3_BASE_IDX                                        5
+
+
+// addressBlock: nbif0_nbif0_bif_misc_pfvf_dev0_epf1_bif_misc_pfvf_regblk
+// base address: 0x10100000
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0                                                 0xf619
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1                                                 0xf61a
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2                                                 0xf61b
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3                                                 0xf61c
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0                                             0xf61d
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1                                             0xf61e
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2                                             0xf61f
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3                                             0xf620
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0                                          0xf621
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0_BASE_IDX                                 5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1                                          0xf622
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1_BASE_IDX                                 5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2                                          0xf623
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2_BASE_IDX                                 5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3                                          0xf624
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3_BASE_IDX                                 5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0                                             0xf625
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1                                             0xf626
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2                                             0xf627
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3                                             0xf628
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3_BASE_IDX                                    5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0                                                 0xf629
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1                                                 0xf62a
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2                                                 0xf62b
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2_BASE_IDX                                        5
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3                                                 0xf62c
+#define regBIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3_BASE_IDX                                        5
+
+
+// addressBlock: nbif0_nbif0_bif_trap_bif_trap_regblk
+// base address: 0x10100000
+#define regNBIF_TRAP_NP_STATUS                                                                          0x4000
+#define regNBIF_TRAP_NP_STATUS_BASE_IDX                                                                 5
+#define regNBIF_TRAP_NP_REQUEST0                                                                        0x4001
+#define regNBIF_TRAP_NP_REQUEST0_BASE_IDX                                                               5
+#define regNBIF_TRAP_NP_REQUEST1                                                                        0x4002
+#define regNBIF_TRAP_NP_REQUEST1_BASE_IDX                                                               5
+#define regNBIF_TRAP_NP_REQUEST2                                                                        0x4003
+#define regNBIF_TRAP_NP_REQUEST2_BASE_IDX                                                               5
+#define regNBIF_TRAP_NP_REQUEST3                                                                        0x4004
+#define regNBIF_TRAP_NP_REQUEST3_BASE_IDX                                                               5
+#define regNBIF_TRAP_NP_REQUEST4                                                                        0x4005
+#define regNBIF_TRAP_NP_REQUEST4_BASE_IDX                                                               5
+#define regNBIF_TRAP_NP_REQUEST5                                                                        0x4006
+#define regNBIF_TRAP_NP_REQUEST5_BASE_IDX                                                               5
+#define regNBIF_TRAP_NP_REQUEST6                                                                        0x4007
+#define regNBIF_TRAP_NP_REQUEST6_BASE_IDX                                                               5
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB0                                                               0x4008
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB0_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB1                                                               0x4009
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB1_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB2                                                               0x400a
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB2_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB3                                                               0x400b
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB3_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB4                                                               0x400c
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB4_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB5                                                               0x400d
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB5_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB6                                                               0x400e
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB6_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB7                                                               0x400f
+#define regNBIF_TRAP_NP_REQUEST_DATASTRB7_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_REQUEST_DATA0                                                                   0x4010
+#define regNBIF_TRAP_NP_REQUEST_DATA0_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA1                                                                   0x4011
+#define regNBIF_TRAP_NP_REQUEST_DATA1_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA2                                                                   0x4012
+#define regNBIF_TRAP_NP_REQUEST_DATA2_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA3                                                                   0x4013
+#define regNBIF_TRAP_NP_REQUEST_DATA3_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA4                                                                   0x4014
+#define regNBIF_TRAP_NP_REQUEST_DATA4_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA5                                                                   0x4015
+#define regNBIF_TRAP_NP_REQUEST_DATA5_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA6                                                                   0x4016
+#define regNBIF_TRAP_NP_REQUEST_DATA6_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA7                                                                   0x4017
+#define regNBIF_TRAP_NP_REQUEST_DATA7_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA8                                                                   0x4018
+#define regNBIF_TRAP_NP_REQUEST_DATA8_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA9                                                                   0x4019
+#define regNBIF_TRAP_NP_REQUEST_DATA9_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_REQUEST_DATA10                                                                  0x401a
+#define regNBIF_TRAP_NP_REQUEST_DATA10_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA11                                                                  0x401b
+#define regNBIF_TRAP_NP_REQUEST_DATA11_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA12                                                                  0x401c
+#define regNBIF_TRAP_NP_REQUEST_DATA12_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA13                                                                  0x401d
+#define regNBIF_TRAP_NP_REQUEST_DATA13_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA14                                                                  0x401e
+#define regNBIF_TRAP_NP_REQUEST_DATA14_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA15                                                                  0x401f
+#define regNBIF_TRAP_NP_REQUEST_DATA15_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA16                                                                  0x4020
+#define regNBIF_TRAP_NP_REQUEST_DATA16_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA17                                                                  0x4021
+#define regNBIF_TRAP_NP_REQUEST_DATA17_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA18                                                                  0x4022
+#define regNBIF_TRAP_NP_REQUEST_DATA18_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA19                                                                  0x4023
+#define regNBIF_TRAP_NP_REQUEST_DATA19_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA20                                                                  0x4024
+#define regNBIF_TRAP_NP_REQUEST_DATA20_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA21                                                                  0x4025
+#define regNBIF_TRAP_NP_REQUEST_DATA21_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA22                                                                  0x4026
+#define regNBIF_TRAP_NP_REQUEST_DATA22_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA23                                                                  0x4027
+#define regNBIF_TRAP_NP_REQUEST_DATA23_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA24                                                                  0x4028
+#define regNBIF_TRAP_NP_REQUEST_DATA24_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA25                                                                  0x4029
+#define regNBIF_TRAP_NP_REQUEST_DATA25_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA26                                                                  0x402a
+#define regNBIF_TRAP_NP_REQUEST_DATA26_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA27                                                                  0x402b
+#define regNBIF_TRAP_NP_REQUEST_DATA27_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA28                                                                  0x402c
+#define regNBIF_TRAP_NP_REQUEST_DATA28_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA29                                                                  0x402d
+#define regNBIF_TRAP_NP_REQUEST_DATA29_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA30                                                                  0x402e
+#define regNBIF_TRAP_NP_REQUEST_DATA30_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA31                                                                  0x402f
+#define regNBIF_TRAP_NP_REQUEST_DATA31_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA32                                                                  0x4030
+#define regNBIF_TRAP_NP_REQUEST_DATA32_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA33                                                                  0x4031
+#define regNBIF_TRAP_NP_REQUEST_DATA33_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA34                                                                  0x4032
+#define regNBIF_TRAP_NP_REQUEST_DATA34_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA35                                                                  0x4033
+#define regNBIF_TRAP_NP_REQUEST_DATA35_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA36                                                                  0x4034
+#define regNBIF_TRAP_NP_REQUEST_DATA36_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA37                                                                  0x4035
+#define regNBIF_TRAP_NP_REQUEST_DATA37_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA38                                                                  0x4036
+#define regNBIF_TRAP_NP_REQUEST_DATA38_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA39                                                                  0x4037
+#define regNBIF_TRAP_NP_REQUEST_DATA39_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA40                                                                  0x4038
+#define regNBIF_TRAP_NP_REQUEST_DATA40_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA41                                                                  0x4039
+#define regNBIF_TRAP_NP_REQUEST_DATA41_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA42                                                                  0x403a
+#define regNBIF_TRAP_NP_REQUEST_DATA42_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA43                                                                  0x403b
+#define regNBIF_TRAP_NP_REQUEST_DATA43_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA44                                                                  0x403c
+#define regNBIF_TRAP_NP_REQUEST_DATA44_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA45                                                                  0x403d
+#define regNBIF_TRAP_NP_REQUEST_DATA45_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA46                                                                  0x403e
+#define regNBIF_TRAP_NP_REQUEST_DATA46_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA47                                                                  0x403f
+#define regNBIF_TRAP_NP_REQUEST_DATA47_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA48                                                                  0x4040
+#define regNBIF_TRAP_NP_REQUEST_DATA48_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA49                                                                  0x4041
+#define regNBIF_TRAP_NP_REQUEST_DATA49_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA50                                                                  0x4042
+#define regNBIF_TRAP_NP_REQUEST_DATA50_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA51                                                                  0x4043
+#define regNBIF_TRAP_NP_REQUEST_DATA51_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA52                                                                  0x4044
+#define regNBIF_TRAP_NP_REQUEST_DATA52_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA53                                                                  0x4045
+#define regNBIF_TRAP_NP_REQUEST_DATA53_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA54                                                                  0x4046
+#define regNBIF_TRAP_NP_REQUEST_DATA54_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA55                                                                  0x4047
+#define regNBIF_TRAP_NP_REQUEST_DATA55_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA56                                                                  0x4048
+#define regNBIF_TRAP_NP_REQUEST_DATA56_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA57                                                                  0x4049
+#define regNBIF_TRAP_NP_REQUEST_DATA57_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA58                                                                  0x404a
+#define regNBIF_TRAP_NP_REQUEST_DATA58_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA59                                                                  0x404b
+#define regNBIF_TRAP_NP_REQUEST_DATA59_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA60                                                                  0x404c
+#define regNBIF_TRAP_NP_REQUEST_DATA60_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA61                                                                  0x404d
+#define regNBIF_TRAP_NP_REQUEST_DATA61_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA62                                                                  0x404e
+#define regNBIF_TRAP_NP_REQUEST_DATA62_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_REQUEST_DATA63                                                                  0x404f
+#define regNBIF_TRAP_NP_REQUEST_DATA63_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_CONTROL                                                                0x4050
+#define regNBIF_TRAP_NP_RESPONSE_CONTROL_BASE_IDX                                                       5
+#define regNBIF_TRAP_NP_RESPONSE0                                                                       0x4051
+#define regNBIF_TRAP_NP_RESPONSE0_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_RESPONSE1                                                                       0x4052
+#define regNBIF_TRAP_NP_RESPONSE1_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_RESPONSE2                                                                       0x4053
+#define regNBIF_TRAP_NP_RESPONSE2_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_RESPONSE3                                                                       0x4054
+#define regNBIF_TRAP_NP_RESPONSE3_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_RESPONSE4                                                                       0x4055
+#define regNBIF_TRAP_NP_RESPONSE4_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_RESPONSE5                                                                       0x4056
+#define regNBIF_TRAP_NP_RESPONSE5_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_RESPONSE_DATA0                                                                  0x4057
+#define regNBIF_TRAP_NP_RESPONSE_DATA0_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA1                                                                  0x4058
+#define regNBIF_TRAP_NP_RESPONSE_DATA1_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA2                                                                  0x4059
+#define regNBIF_TRAP_NP_RESPONSE_DATA2_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA3                                                                  0x405a
+#define regNBIF_TRAP_NP_RESPONSE_DATA3_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA4                                                                  0x405b
+#define regNBIF_TRAP_NP_RESPONSE_DATA4_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA5                                                                  0x405c
+#define regNBIF_TRAP_NP_RESPONSE_DATA5_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA6                                                                  0x405d
+#define regNBIF_TRAP_NP_RESPONSE_DATA6_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA7                                                                  0x405e
+#define regNBIF_TRAP_NP_RESPONSE_DATA7_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA8                                                                  0x405f
+#define regNBIF_TRAP_NP_RESPONSE_DATA8_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA9                                                                  0x4060
+#define regNBIF_TRAP_NP_RESPONSE_DATA9_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_RESPONSE_DATA10                                                                 0x4061
+#define regNBIF_TRAP_NP_RESPONSE_DATA10_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA11                                                                 0x4062
+#define regNBIF_TRAP_NP_RESPONSE_DATA11_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA12                                                                 0x4063
+#define regNBIF_TRAP_NP_RESPONSE_DATA12_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA13                                                                 0x4064
+#define regNBIF_TRAP_NP_RESPONSE_DATA13_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA14                                                                 0x4065
+#define regNBIF_TRAP_NP_RESPONSE_DATA14_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA15                                                                 0x4066
+#define regNBIF_TRAP_NP_RESPONSE_DATA15_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA16                                                                 0x4067
+#define regNBIF_TRAP_NP_RESPONSE_DATA16_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA17                                                                 0x4068
+#define regNBIF_TRAP_NP_RESPONSE_DATA17_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA18                                                                 0x4069
+#define regNBIF_TRAP_NP_RESPONSE_DATA18_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA19                                                                 0x406a
+#define regNBIF_TRAP_NP_RESPONSE_DATA19_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA20                                                                 0x406b
+#define regNBIF_TRAP_NP_RESPONSE_DATA20_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA21                                                                 0x406c
+#define regNBIF_TRAP_NP_RESPONSE_DATA21_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA22                                                                 0x406d
+#define regNBIF_TRAP_NP_RESPONSE_DATA22_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA23                                                                 0x406e
+#define regNBIF_TRAP_NP_RESPONSE_DATA23_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA24                                                                 0x406f
+#define regNBIF_TRAP_NP_RESPONSE_DATA24_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA25                                                                 0x4070
+#define regNBIF_TRAP_NP_RESPONSE_DATA25_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA26                                                                 0x4071
+#define regNBIF_TRAP_NP_RESPONSE_DATA26_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA27                                                                 0x4072
+#define regNBIF_TRAP_NP_RESPONSE_DATA27_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA28                                                                 0x4073
+#define regNBIF_TRAP_NP_RESPONSE_DATA28_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA29                                                                 0x4074
+#define regNBIF_TRAP_NP_RESPONSE_DATA29_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA30                                                                 0x4075
+#define regNBIF_TRAP_NP_RESPONSE_DATA30_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA31                                                                 0x4076
+#define regNBIF_TRAP_NP_RESPONSE_DATA31_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA32                                                                 0x4077
+#define regNBIF_TRAP_NP_RESPONSE_DATA32_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA33                                                                 0x4078
+#define regNBIF_TRAP_NP_RESPONSE_DATA33_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA34                                                                 0x4079
+#define regNBIF_TRAP_NP_RESPONSE_DATA34_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA35                                                                 0x407a
+#define regNBIF_TRAP_NP_RESPONSE_DATA35_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA36                                                                 0x407b
+#define regNBIF_TRAP_NP_RESPONSE_DATA36_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA37                                                                 0x407c
+#define regNBIF_TRAP_NP_RESPONSE_DATA37_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA38                                                                 0x407d
+#define regNBIF_TRAP_NP_RESPONSE_DATA38_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA39                                                                 0x407e
+#define regNBIF_TRAP_NP_RESPONSE_DATA39_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA40                                                                 0x407f
+#define regNBIF_TRAP_NP_RESPONSE_DATA40_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA41                                                                 0x4080
+#define regNBIF_TRAP_NP_RESPONSE_DATA41_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA42                                                                 0x4081
+#define regNBIF_TRAP_NP_RESPONSE_DATA42_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA43                                                                 0x4082
+#define regNBIF_TRAP_NP_RESPONSE_DATA43_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA44                                                                 0x4083
+#define regNBIF_TRAP_NP_RESPONSE_DATA44_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA45                                                                 0x4084
+#define regNBIF_TRAP_NP_RESPONSE_DATA45_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA46                                                                 0x4085
+#define regNBIF_TRAP_NP_RESPONSE_DATA46_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA47                                                                 0x4086
+#define regNBIF_TRAP_NP_RESPONSE_DATA47_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA48                                                                 0x4087
+#define regNBIF_TRAP_NP_RESPONSE_DATA48_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA49                                                                 0x4088
+#define regNBIF_TRAP_NP_RESPONSE_DATA49_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA50                                                                 0x4089
+#define regNBIF_TRAP_NP_RESPONSE_DATA50_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA51                                                                 0x408a
+#define regNBIF_TRAP_NP_RESPONSE_DATA51_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA52                                                                 0x408b
+#define regNBIF_TRAP_NP_RESPONSE_DATA52_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA53                                                                 0x408c
+#define regNBIF_TRAP_NP_RESPONSE_DATA53_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA54                                                                 0x408d
+#define regNBIF_TRAP_NP_RESPONSE_DATA54_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA55                                                                 0x408e
+#define regNBIF_TRAP_NP_RESPONSE_DATA55_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA56                                                                 0x408f
+#define regNBIF_TRAP_NP_RESPONSE_DATA56_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA57                                                                 0x4090
+#define regNBIF_TRAP_NP_RESPONSE_DATA57_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA58                                                                 0x4091
+#define regNBIF_TRAP_NP_RESPONSE_DATA58_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA59                                                                 0x4092
+#define regNBIF_TRAP_NP_RESPONSE_DATA59_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA60                                                                 0x4093
+#define regNBIF_TRAP_NP_RESPONSE_DATA60_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA61                                                                 0x4094
+#define regNBIF_TRAP_NP_RESPONSE_DATA61_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA62                                                                 0x4095
+#define regNBIF_TRAP_NP_RESPONSE_DATA62_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_RESPONSE_DATA63                                                                 0x4096
+#define regNBIF_TRAP_NP_RESPONSE_DATA63_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_0_CONTROL0                                                                      0x4097
+#define regNBIF_TRAP_NP_0_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_0_ADDRESS_LO                                                                    0x4098
+#define regNBIF_TRAP_NP_0_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_0_ADDRESS_HI                                                                    0x4099
+#define regNBIF_TRAP_NP_0_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_0_COMMAND                                                                       0x409a
+#define regNBIF_TRAP_NP_0_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_0_ADDRESS_LO_MASK                                                               0x409b
+#define regNBIF_TRAP_NP_0_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_0_ADDRESS_HI_MASK                                                               0x409c
+#define regNBIF_TRAP_NP_0_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_0_COMMAND_MASK                                                                  0x409d
+#define regNBIF_TRAP_NP_0_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_1_CONTROL0                                                                      0x409e
+#define regNBIF_TRAP_NP_1_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_1_ADDRESS_LO                                                                    0x409f
+#define regNBIF_TRAP_NP_1_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_1_ADDRESS_HI                                                                    0x40a0
+#define regNBIF_TRAP_NP_1_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_1_COMMAND                                                                       0x40a1
+#define regNBIF_TRAP_NP_1_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_1_ADDRESS_LO_MASK                                                               0x40a2
+#define regNBIF_TRAP_NP_1_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_1_ADDRESS_HI_MASK                                                               0x40a3
+#define regNBIF_TRAP_NP_1_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_1_COMMAND_MASK                                                                  0x40a4
+#define regNBIF_TRAP_NP_1_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_2_CONTROL0                                                                      0x40a5
+#define regNBIF_TRAP_NP_2_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_2_ADDRESS_LO                                                                    0x40a6
+#define regNBIF_TRAP_NP_2_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_2_ADDRESS_HI                                                                    0x40a7
+#define regNBIF_TRAP_NP_2_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_2_COMMAND                                                                       0x40a8
+#define regNBIF_TRAP_NP_2_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_2_ADDRESS_LO_MASK                                                               0x40a9
+#define regNBIF_TRAP_NP_2_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_2_ADDRESS_HI_MASK                                                               0x40aa
+#define regNBIF_TRAP_NP_2_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_2_COMMAND_MASK                                                                  0x40ab
+#define regNBIF_TRAP_NP_2_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_3_CONTROL0                                                                      0x40ac
+#define regNBIF_TRAP_NP_3_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_3_ADDRESS_LO                                                                    0x40ad
+#define regNBIF_TRAP_NP_3_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_3_ADDRESS_HI                                                                    0x40ae
+#define regNBIF_TRAP_NP_3_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_3_COMMAND                                                                       0x40af
+#define regNBIF_TRAP_NP_3_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_3_ADDRESS_LO_MASK                                                               0x40b0
+#define regNBIF_TRAP_NP_3_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_3_ADDRESS_HI_MASK                                                               0x40b1
+#define regNBIF_TRAP_NP_3_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_3_COMMAND_MASK                                                                  0x40b2
+#define regNBIF_TRAP_NP_3_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_4_CONTROL0                                                                      0x40b3
+#define regNBIF_TRAP_NP_4_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_4_ADDRESS_LO                                                                    0x40b4
+#define regNBIF_TRAP_NP_4_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_4_ADDRESS_HI                                                                    0x40b5
+#define regNBIF_TRAP_NP_4_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_4_COMMAND                                                                       0x40b6
+#define regNBIF_TRAP_NP_4_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_4_ADDRESS_LO_MASK                                                               0x40b7
+#define regNBIF_TRAP_NP_4_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_4_ADDRESS_HI_MASK                                                               0x40b8
+#define regNBIF_TRAP_NP_4_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_4_COMMAND_MASK                                                                  0x40b9
+#define regNBIF_TRAP_NP_4_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_5_CONTROL0                                                                      0x40ba
+#define regNBIF_TRAP_NP_5_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_5_ADDRESS_LO                                                                    0x40bb
+#define regNBIF_TRAP_NP_5_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_5_ADDRESS_HI                                                                    0x40bc
+#define regNBIF_TRAP_NP_5_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_5_COMMAND                                                                       0x40bd
+#define regNBIF_TRAP_NP_5_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_5_ADDRESS_LO_MASK                                                               0x40be
+#define regNBIF_TRAP_NP_5_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_5_ADDRESS_HI_MASK                                                               0x40bf
+#define regNBIF_TRAP_NP_5_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_5_COMMAND_MASK                                                                  0x40c0
+#define regNBIF_TRAP_NP_5_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_6_CONTROL0                                                                      0x40c1
+#define regNBIF_TRAP_NP_6_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_6_ADDRESS_LO                                                                    0x40c2
+#define regNBIF_TRAP_NP_6_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_6_ADDRESS_HI                                                                    0x40c3
+#define regNBIF_TRAP_NP_6_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_6_COMMAND                                                                       0x40c4
+#define regNBIF_TRAP_NP_6_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_6_ADDRESS_LO_MASK                                                               0x40c5
+#define regNBIF_TRAP_NP_6_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_6_ADDRESS_HI_MASK                                                               0x40c6
+#define regNBIF_TRAP_NP_6_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_6_COMMAND_MASK                                                                  0x40c7
+#define regNBIF_TRAP_NP_6_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_7_CONTROL0                                                                      0x40c8
+#define regNBIF_TRAP_NP_7_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_7_ADDRESS_LO                                                                    0x40c9
+#define regNBIF_TRAP_NP_7_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_7_ADDRESS_HI                                                                    0x40ca
+#define regNBIF_TRAP_NP_7_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_7_COMMAND                                                                       0x40cb
+#define regNBIF_TRAP_NP_7_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_7_ADDRESS_LO_MASK                                                               0x40cc
+#define regNBIF_TRAP_NP_7_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_7_ADDRESS_HI_MASK                                                               0x40cd
+#define regNBIF_TRAP_NP_7_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_7_COMMAND_MASK                                                                  0x40ce
+#define regNBIF_TRAP_NP_7_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_8_CONTROL0                                                                      0x40cf
+#define regNBIF_TRAP_NP_8_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_8_ADDRESS_LO                                                                    0x40d0
+#define regNBIF_TRAP_NP_8_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_8_ADDRESS_HI                                                                    0x40d1
+#define regNBIF_TRAP_NP_8_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_8_COMMAND                                                                       0x40d2
+#define regNBIF_TRAP_NP_8_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_8_ADDRESS_LO_MASK                                                               0x40d3
+#define regNBIF_TRAP_NP_8_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_8_ADDRESS_HI_MASK                                                               0x40d4
+#define regNBIF_TRAP_NP_8_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_8_COMMAND_MASK                                                                  0x40d5
+#define regNBIF_TRAP_NP_8_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_9_CONTROL0                                                                      0x40d6
+#define regNBIF_TRAP_NP_9_CONTROL0_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_9_ADDRESS_LO                                                                    0x40d7
+#define regNBIF_TRAP_NP_9_ADDRESS_LO_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_9_ADDRESS_HI                                                                    0x40d8
+#define regNBIF_TRAP_NP_9_ADDRESS_HI_BASE_IDX                                                           5
+#define regNBIF_TRAP_NP_9_COMMAND                                                                       0x40d9
+#define regNBIF_TRAP_NP_9_COMMAND_BASE_IDX                                                              5
+#define regNBIF_TRAP_NP_9_ADDRESS_LO_MASK                                                               0x40da
+#define regNBIF_TRAP_NP_9_ADDRESS_LO_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_9_ADDRESS_HI_MASK                                                               0x40db
+#define regNBIF_TRAP_NP_9_ADDRESS_HI_MASK_BASE_IDX                                                      5
+#define regNBIF_TRAP_NP_9_COMMAND_MASK                                                                  0x40dc
+#define regNBIF_TRAP_NP_9_COMMAND_MASK_BASE_IDX                                                         5
+#define regNBIF_TRAP_NP_10_CONTROL0                                                                     0x40dd
+#define regNBIF_TRAP_NP_10_CONTROL0_BASE_IDX                                                            5
+#define regNBIF_TRAP_NP_10_ADDRESS_LO                                                                   0x40de
+#define regNBIF_TRAP_NP_10_ADDRESS_LO_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_10_ADDRESS_HI                                                                   0x40df
+#define regNBIF_TRAP_NP_10_ADDRESS_HI_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_10_COMMAND                                                                      0x40e0
+#define regNBIF_TRAP_NP_10_COMMAND_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_10_ADDRESS_LO_MASK                                                              0x40e1
+#define regNBIF_TRAP_NP_10_ADDRESS_LO_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_10_ADDRESS_HI_MASK                                                              0x40e2
+#define regNBIF_TRAP_NP_10_ADDRESS_HI_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_10_COMMAND_MASK                                                                 0x40e3
+#define regNBIF_TRAP_NP_10_COMMAND_MASK_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_11_CONTROL0                                                                     0x40e4
+#define regNBIF_TRAP_NP_11_CONTROL0_BASE_IDX                                                            5
+#define regNBIF_TRAP_NP_11_ADDRESS_LO                                                                   0x40e5
+#define regNBIF_TRAP_NP_11_ADDRESS_LO_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_11_ADDRESS_HI                                                                   0x40e6
+#define regNBIF_TRAP_NP_11_ADDRESS_HI_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_11_COMMAND                                                                      0x40e7
+#define regNBIF_TRAP_NP_11_COMMAND_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_11_ADDRESS_LO_MASK                                                              0x40e8
+#define regNBIF_TRAP_NP_11_ADDRESS_LO_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_11_ADDRESS_HI_MASK                                                              0x40e9
+#define regNBIF_TRAP_NP_11_ADDRESS_HI_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_11_COMMAND_MASK                                                                 0x40ea
+#define regNBIF_TRAP_NP_11_COMMAND_MASK_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_12_CONTROL0                                                                     0x40eb
+#define regNBIF_TRAP_NP_12_CONTROL0_BASE_IDX                                                            5
+#define regNBIF_TRAP_NP_12_ADDRESS_LO                                                                   0x40ec
+#define regNBIF_TRAP_NP_12_ADDRESS_LO_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_12_ADDRESS_HI                                                                   0x40ed
+#define regNBIF_TRAP_NP_12_ADDRESS_HI_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_12_COMMAND                                                                      0x40ee
+#define regNBIF_TRAP_NP_12_COMMAND_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_12_ADDRESS_LO_MASK                                                              0x40ef
+#define regNBIF_TRAP_NP_12_ADDRESS_LO_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_12_ADDRESS_HI_MASK                                                              0x40f0
+#define regNBIF_TRAP_NP_12_ADDRESS_HI_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_12_COMMAND_MASK                                                                 0x40f1
+#define regNBIF_TRAP_NP_12_COMMAND_MASK_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_13_CONTROL0                                                                     0x40f2
+#define regNBIF_TRAP_NP_13_CONTROL0_BASE_IDX                                                            5
+#define regNBIF_TRAP_NP_13_ADDRESS_LO                                                                   0x40f3
+#define regNBIF_TRAP_NP_13_ADDRESS_LO_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_13_ADDRESS_HI                                                                   0x40f4
+#define regNBIF_TRAP_NP_13_ADDRESS_HI_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_13_COMMAND                                                                      0x40f5
+#define regNBIF_TRAP_NP_13_COMMAND_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_13_ADDRESS_LO_MASK                                                              0x40f6
+#define regNBIF_TRAP_NP_13_ADDRESS_LO_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_13_ADDRESS_HI_MASK                                                              0x40f7
+#define regNBIF_TRAP_NP_13_ADDRESS_HI_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_13_COMMAND_MASK                                                                 0x40f8
+#define regNBIF_TRAP_NP_13_COMMAND_MASK_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_14_CONTROL0                                                                     0x40f9
+#define regNBIF_TRAP_NP_14_CONTROL0_BASE_IDX                                                            5
+#define regNBIF_TRAP_NP_14_ADDRESS_LO                                                                   0x40fa
+#define regNBIF_TRAP_NP_14_ADDRESS_LO_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_14_ADDRESS_HI                                                                   0x40fb
+#define regNBIF_TRAP_NP_14_ADDRESS_HI_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_14_COMMAND                                                                      0x40fc
+#define regNBIF_TRAP_NP_14_COMMAND_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_14_ADDRESS_LO_MASK                                                              0x40fd
+#define regNBIF_TRAP_NP_14_ADDRESS_LO_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_14_ADDRESS_HI_MASK                                                              0x40fe
+#define regNBIF_TRAP_NP_14_ADDRESS_HI_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_14_COMMAND_MASK                                                                 0x40ff
+#define regNBIF_TRAP_NP_14_COMMAND_MASK_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_15_CONTROL0                                                                     0x4100
+#define regNBIF_TRAP_NP_15_CONTROL0_BASE_IDX                                                            5
+#define regNBIF_TRAP_NP_15_ADDRESS_LO                                                                   0x4101
+#define regNBIF_TRAP_NP_15_ADDRESS_LO_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_15_ADDRESS_HI                                                                   0x4102
+#define regNBIF_TRAP_NP_15_ADDRESS_HI_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_15_COMMAND                                                                      0x4103
+#define regNBIF_TRAP_NP_15_COMMAND_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_15_ADDRESS_LO_MASK                                                              0x4104
+#define regNBIF_TRAP_NP_15_ADDRESS_LO_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_15_ADDRESS_HI_MASK                                                              0x4105
+#define regNBIF_TRAP_NP_15_ADDRESS_HI_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_15_COMMAND_MASK                                                                 0x4106
+#define regNBIF_TRAP_NP_15_COMMAND_MASK_BASE_IDX                                                        5
+#define regNBIF_TRAP_NP_S2_CONTROL0                                                                     0x4107
+#define regNBIF_TRAP_NP_S2_CONTROL0_BASE_IDX                                                            5
+#define regNBIF_TRAP_NP_S2_ADDRESS_LO                                                                   0x4108
+#define regNBIF_TRAP_NP_S2_ADDRESS_LO_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_S2_ADDRESS_HI                                                                   0x4109
+#define regNBIF_TRAP_NP_S2_ADDRESS_HI_BASE_IDX                                                          5
+#define regNBIF_TRAP_NP_S2_COMMAND                                                                      0x410a
+#define regNBIF_TRAP_NP_S2_COMMAND_BASE_IDX                                                             5
+#define regNBIF_TRAP_NP_S2_ADDRESS_LO_MASK                                                              0x410b
+#define regNBIF_TRAP_NP_S2_ADDRESS_LO_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_S2_ADDRESS_HI_MASK                                                              0x410c
+#define regNBIF_TRAP_NP_S2_ADDRESS_HI_MASK_BASE_IDX                                                     5
+#define regNBIF_TRAP_NP_S2_COMMAND_MASK                                                                 0x410d
+#define regNBIF_TRAP_NP_S2_COMMAND_MASK_BASE_IDX                                                        5
+
+
+// addressBlock: nbif0_nbif0_bif_ras_bif_ras_regblk
+// base address: 0x10100000
+#define regBIFL_RAS_CENTRAL_CNTL                                                                        0xe400
+#define regBIFL_RAS_CENTRAL_CNTL_BASE_IDX                                                               5
+#define regBIFL_RAS_CENTRAL_STATUS                                                                      0xe404
+#define regBIFL_RAS_CENTRAL_STATUS_BASE_IDX                                                             5
+#define regBIFL_RAS_LEAF0_CTRL                                                                          0xe408
+#define regBIFL_RAS_LEAF0_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF1_CTRL                                                                          0xe409
+#define regBIFL_RAS_LEAF1_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF2_CTRL                                                                          0xe40a
+#define regBIFL_RAS_LEAF2_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF3_CTRL                                                                          0xe40b
+#define regBIFL_RAS_LEAF3_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF4_CTRL                                                                          0xe40c
+#define regBIFL_RAS_LEAF4_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF5_CTRL                                                                          0xe40d
+#define regBIFL_RAS_LEAF5_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF6_CTRL                                                                          0xe40e
+#define regBIFL_RAS_LEAF6_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF7_CTRL                                                                          0xe40f
+#define regBIFL_RAS_LEAF7_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF8_CTRL                                                                          0xe410
+#define regBIFL_RAS_LEAF8_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF9_CTRL                                                                          0xe411
+#define regBIFL_RAS_LEAF9_CTRL_BASE_IDX                                                                 5
+#define regBIFL_RAS_LEAF10_CTRL                                                                         0xe412
+#define regBIFL_RAS_LEAF10_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF11_CTRL                                                                         0xe413
+#define regBIFL_RAS_LEAF11_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF12_CTRL                                                                         0xe414
+#define regBIFL_RAS_LEAF12_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF13_CTRL                                                                         0xe415
+#define regBIFL_RAS_LEAF13_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF14_CTRL                                                                         0xe416
+#define regBIFL_RAS_LEAF14_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF15_CTRL                                                                         0xe417
+#define regBIFL_RAS_LEAF15_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF16_CTRL                                                                         0xe418
+#define regBIFL_RAS_LEAF16_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF17_CTRL                                                                         0xe419
+#define regBIFL_RAS_LEAF17_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF18_CTRL                                                                         0xe41a
+#define regBIFL_RAS_LEAF18_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF19_CTRL                                                                         0xe41b
+#define regBIFL_RAS_LEAF19_CTRL_BASE_IDX                                                                5
+#define regBIFL_RAS_LEAF2_MISC_CTRL                                                                     0xe42e
+#define regBIFL_RAS_LEAF2_MISC_CTRL_BASE_IDX                                                            5
+#define regBIFL_RAS_LEAF2_MISC_CTRL2                                                                    0xe42f
+#define regBIFL_RAS_LEAF2_MISC_CTRL2_BASE_IDX                                                           5
+#define regBIFL_RAS_LEAF_NTB_CTRL                                                                       0xe430
+#define regBIFL_RAS_LEAF_NTB_CTRL_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF0_STATUS                                                                        0xe431
+#define regBIFL_RAS_LEAF0_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF1_STATUS                                                                        0xe432
+#define regBIFL_RAS_LEAF1_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF2_STATUS                                                                        0xe433
+#define regBIFL_RAS_LEAF2_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF3_STATUS                                                                        0xe434
+#define regBIFL_RAS_LEAF3_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF4_STATUS                                                                        0xe435
+#define regBIFL_RAS_LEAF4_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF5_STATUS                                                                        0xe436
+#define regBIFL_RAS_LEAF5_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF6_STATUS                                                                        0xe437
+#define regBIFL_RAS_LEAF6_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF7_STATUS                                                                        0xe438
+#define regBIFL_RAS_LEAF7_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF8_STATUS                                                                        0xe439
+#define regBIFL_RAS_LEAF8_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF9_STATUS                                                                        0xe43a
+#define regBIFL_RAS_LEAF9_STATUS_BASE_IDX                                                               5
+#define regBIFL_RAS_LEAF10_STATUS                                                                       0xe43b
+#define regBIFL_RAS_LEAF10_STATUS_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF11_STATUS                                                                       0xe43c
+#define regBIFL_RAS_LEAF11_STATUS_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF12_STATUS                                                                       0xe43d
+#define regBIFL_RAS_LEAF12_STATUS_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF13_STATUS                                                                       0xe43e
+#define regBIFL_RAS_LEAF13_STATUS_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF14_STATUS                                                                       0xe43f
+#define regBIFL_RAS_LEAF14_STATUS_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF15_STATUS                                                                       0xe440
+#define regBIFL_RAS_LEAF15_STATUS_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF16_STATUS                                                                       0xe441
+#define regBIFL_RAS_LEAF16_STATUS_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF17_STATUS                                                                       0xe442
+#define regBIFL_RAS_LEAF17_STATUS_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF18_STATUS                                                                       0xe443
+#define regBIFL_RAS_LEAF18_STATUS_BASE_IDX                                                              5
+#define regBIFL_RAS_LEAF19_STATUS                                                                       0xe444
+#define regBIFL_RAS_LEAF19_STATUS_BASE_IDX                                                              5
+#define regBIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP                                                           0xe450
+#define regBIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP_BASE_IDX                                                  5
+#define regBIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP                                                           0xe451
+#define regBIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP_BASE_IDX                                                  5
+#define regBIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP                                                           0xe452
+#define regBIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP_BASE_IDX                                                  5
+#define regBIFL_DFV_POISON_INJ_ATHUB_ORIG_SDP                                                           0xe453
+#define regBIFL_DFV_POISON_INJ_ATHUB_ORIG_SDP_BASE_IDX                                                  5
+#define regBIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP                                                       0xe458
+#define regBIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP_BASE_IDX                                              5
+#define regBIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP                                                       0xe459
+#define regBIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP_BASE_IDX                                              5
+#define regBIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP                                                       0xe45a
+#define regBIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP_BASE_IDX                                              5
+#define regBIFL_DFV_POISON_INJ_CNT_ATHUB_ORIG_SDP                                                       0xe45b
+#define regBIFL_DFV_POISON_INJ_CNT_ATHUB_ORIG_SDP_BASE_IDX                                              5
+#define regBIFL_DFV_POISON_INJ_LOG_IOHUB_ORIG_SDP                                                       0xe460
+#define regBIFL_DFV_POISON_INJ_LOG_IOHUB_ORIG_SDP_BASE_IDX                                              5
+#define regBIFL_DFV_POISON_INJ_LOG_IOHUB_CMPL_SDP                                                       0xe461
+#define regBIFL_DFV_POISON_INJ_LOG_IOHUB_CMPL_SDP_BASE_IDX                                              5
+#define regBIFL_DFV_POISON_INJ_LOG_ATHUB_CMPL_SDP                                                       0xe462
+#define regBIFL_DFV_POISON_INJ_LOG_ATHUB_CMPL_SDP_BASE_IDX                                              5
+#define regBIFL_DFV_POISON_INJ_LOG_ATHUB_ORIG_SDP                                                       0xe463
+#define regBIFL_DFV_POISON_INJ_LOG_ATHUB_ORIG_SDP_BASE_IDX                                              5
+#define regBIFL_IOHUB_RAS_IH_CNTL                                                                       0xe7fe
+#define regBIFL_IOHUB_RAS_IH_CNTL_BASE_IDX                                                              5
+#define regBIFL_RAS_VWR_FROM_IOHUB                                                                      0xe7ff
+#define regBIFL_RAS_VWR_FROM_IOHUB_BASE_IDX                                                             5
+
+
+// addressBlock: nbif0_nbif0_rcc_dwn_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_DWN_DEV0_2_DN_PCIE_RESERVED                                                              0x8d88
+#define regRCC_DWN_DEV0_2_DN_PCIE_RESERVED_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_2_DN_PCIE_SCRATCH                                                               0x8d89
+#define regRCC_DWN_DEV0_2_DN_PCIE_SCRATCH_BASE_IDX                                                      5
+#define regRCC_DWN_DEV0_2_DN_PCIE_CNTL                                                                  0x8d8b
+#define regRCC_DWN_DEV0_2_DN_PCIE_CNTL_BASE_IDX                                                         5
+#define regRCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL                                                           0x8d8c
+#define regRCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL_BASE_IDX                                                  5
+#define regRCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2                                                              0x8d8d
+#define regRCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL                                                              0x8d8e
+#define regRCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL                                                              0x8d8f
+#define regRCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_F0                                                              0x8d90
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_F0_BASE_IDX                                                     5
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC                                                            0x8d91
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC_BASE_IDX                                                   5
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2                                                           0x8d92
+#define regRCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_rcc_dwnp_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_DWNP_DEV0_2_PCIE_ERR_CNTL                                                                0x8d94
+#define regRCC_DWNP_DEV0_2_PCIE_ERR_CNTL_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_2_PCIE_RX_CNTL                                                                 0x8d95
+#define regRCC_DWNP_DEV0_2_PCIE_RX_CNTL_BASE_IDX                                                        5
+#define regRCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL                                                           0x8d96
+#define regRCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL_BASE_IDX                                                  5
+#define regRCC_DWNP_DEV0_2_PCIE_LC_CNTL2                                                                0x8d97
+#define regRCC_DWNP_DEV0_2_PCIE_LC_CNTL2_BASE_IDX                                                       5
+#define regRCC_DWNP_DEV0_2_PCIEP_STRAP_MISC                                                             0x8d98
+#define regRCC_DWNP_DEV0_2_PCIEP_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP                                                         0x8d99
+#define regRCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP_BASE_IDX                                                5
+
+
+// addressBlock: nbif0_nbif0_rcc_ep_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_EP_DEV0_2_EP_PCIE_SCRATCH                                                                0x8d6b
+#define regRCC_EP_DEV0_2_EP_PCIE_SCRATCH_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_2_EP_PCIE_CNTL                                                                   0x8d6d
+#define regRCC_EP_DEV0_2_EP_PCIE_CNTL_BASE_IDX                                                          5
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_CNTL                                                               0x8d6e
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_STATUS                                                             0x8d6f
+#define regRCC_EP_DEV0_2_EP_PCIE_INT_STATUS_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL2                                                               0x8d70
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL2_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_2_EP_PCIE_BUS_CNTL                                                               0x8d71
+#define regRCC_EP_DEV0_2_EP_PCIE_BUS_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_2_EP_PCIE_CFG_CNTL                                                               0x8d72
+#define regRCC_EP_DEV0_2_EP_PCIE_CFG_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL                                                            0x8d74
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0                                               0x8d75
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1                                               0x8d75
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2                                               0x8d75
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3                                               0x8d75
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4                                               0x8d76
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5                                               0x8d76
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6                                               0x8d76
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      5
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7                                               0x8d76
+#define regRCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC                                                             0x8d77
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2                                                            0x8d78
+#define regRCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP                                                             0x8d7a
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP_BASE_IDX                                                    5
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR                                               0x8d7b
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL                                                            0x8d7b
+#define regRCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0                                               0x8d7b
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1                                               0x8d7c
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2                                               0x8d7c
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3                                               0x8d7c
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4                                               0x8d7c
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5                                               0x8d7d
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6                                               0x8d7d
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7                                               0x8d7d
+#define regRCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7_BASE_IDX                                      5
+#define regRCC_EP_DEV0_2_EP_PCIE_PME_CONTROL                                                            0x8d7d
+#define regRCC_EP_DEV0_2_EP_PCIE_PME_CONTROL_BASE_IDX                                                   5
+#define regRCC_EP_DEV0_2_EP_PCIEP_RESERVED                                                              0x8d7e
+#define regRCC_EP_DEV0_2_EP_PCIEP_RESERVED_BASE_IDX                                                     5
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_CNTL                                                                0x8d80
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID                                                        0x8d81
+#define regRCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID_BASE_IDX                                               5
+#define regRCC_EP_DEV0_2_EP_PCIE_ERR_CNTL                                                               0x8d82
+#define regRCC_EP_DEV0_2_EP_PCIE_ERR_CNTL_BASE_IDX                                                      5
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL                                                                0x8d83
+#define regRCC_EP_DEV0_2_EP_PCIE_RX_CNTL_BASE_IDX                                                       5
+#define regRCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL                                                          0x8d84
+#define regRCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL_BASE_IDX                                                 5
+#define regRCC_EP_DEV0_2_EP_PCIE_DEVICE_CNTL3                                                           0x8d85
+#define regRCC_EP_DEV0_2_EP_PCIE_DEVICE_CNTL3_BASE_IDX                                                  5
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_BIFDEC1
+// base address: 0x10120000
+#define regRCC_DEV0_1_RCC_ERR_INT_CNTL                                                                  0x8da6
+#define regRCC_DEV0_1_RCC_ERR_INT_CNTL_BASE_IDX                                                         5
+#define regRCC_DEV0_1_RCC_BACO_CNTL_MISC                                                                0x8da7
+#define regRCC_DEV0_1_RCC_BACO_CNTL_MISC_BASE_IDX                                                       5
+#define regRCC_DEV0_1_RCC_RESET_EN                                                                      0x8da8
+#define regRCC_DEV0_1_RCC_RESET_EN_BASE_IDX                                                             5
+#define regRCC_DEV0_2_RCC_VDM_SUPPORT                                                                   0x8da9
+#define regRCC_DEV0_2_RCC_VDM_SUPPORT_BASE_IDX                                                          5
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0                                                            0x8daa
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0_BASE_IDX                                                   5
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1                                                            0x8dab
+#define regRCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_GPUIOV_REGION                                                                 0x8dac
+#define regRCC_DEV0_1_RCC_GPUIOV_REGION_BASE_IDX                                                        5
+#define regRCC_DEV0_1_RCC_GPU_HOSTVM_EN                                                                 0x8dad
+#define regRCC_DEV0_1_RCC_GPU_HOSTVM_EN_BASE_IDX                                                        5
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL                                                         0x8dae
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL_BASE_IDX                                                5
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET                                                   0x8daf
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET_BASE_IDX                                          5
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE                                                         0x8daf
+#define regRCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE_BASE_IDX                                                5
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE0                                                               0x8dde
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE0_BASE_IDX                                                      5
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE1                                                               0x8ddf
+#define regRCC_DEV0_1_RCC_PEER_REG_RANGE1_BASE_IDX                                                      5
+#define regRCC_DEV0_2_RCC_BUS_CNTL                                                                      0x8de1
+#define regRCC_DEV0_2_RCC_BUS_CNTL_BASE_IDX                                                             5
+#define regRCC_DEV0_1_RCC_CONFIG_CNTL                                                                   0x8de2
+#define regRCC_DEV0_1_RCC_CONFIG_CNTL_BASE_IDX                                                          5
+#define regRCC_DEV0_1_RCC_CONFIG_F0_BASE                                                                0x8de6
+#define regRCC_DEV0_1_RCC_CONFIG_F0_BASE_BASE_IDX                                                       5
+#define regRCC_DEV0_1_RCC_CONFIG_APER_SIZE                                                              0x8de7
+#define regRCC_DEV0_1_RCC_CONFIG_APER_SIZE_BASE_IDX                                                     5
+#define regRCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE                                                          0x8de8
+#define regRCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE_BASE_IDX                                                 5
+#define regRCC_DEV0_1_RCC_XDMA_LO                                                                       0x8de9
+#define regRCC_DEV0_1_RCC_XDMA_LO_BASE_IDX                                                              5
+#define regRCC_DEV0_1_RCC_XDMA_HI                                                                       0x8dea
+#define regRCC_DEV0_1_RCC_XDMA_HI_BASE_IDX                                                              5
+#define regRCC_DEV0_2_RCC_FEATURES_CONTROL_MISC                                                         0x8deb
+#define regRCC_DEV0_2_RCC_FEATURES_CONTROL_MISC_BASE_IDX                                                5
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL1                                                                  0x8dec
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL1_BASE_IDX                                                         5
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST0                                                                  0x8ded
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST0_BASE_IDX                                                         5
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST1                                                                  0x8dee
+#define regRCC_DEV0_1_RCC_BUSNUM_LIST1_BASE_IDX                                                         5
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL2                                                                  0x8def
+#define regRCC_DEV0_1_RCC_BUSNUM_CNTL2_BASE_IDX                                                         5
+#define regRCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM                                                           0x8df0
+#define regRCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM_BASE_IDX                                                  5
+#define regRCC_DEV0_1_RCC_HOST_BUSNUM                                                                   0x8df1
+#define regRCC_DEV0_1_RCC_HOST_BUSNUM_BASE_IDX                                                          5
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI                                                            0x8df2
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO                                                            0x8df3
+#define regRCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI                                                            0x8df4
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO                                                            0x8df5
+#define regRCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI                                                            0x8df6
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO                                                            0x8df7
+#define regRCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI                                                            0x8df8
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO                                                            0x8df9
+#define regRCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO_BASE_IDX                                                   5
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST0                                                              0x8dfa
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST0_BASE_IDX                                                     5
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST1                                                              0x8dfb
+#define regRCC_DEV0_1_RCC_DEVFUNCNUM_LIST1_BASE_IDX                                                     5
+#define regRCC_DEV0_2_RCC_DEV0_LINK_CNTL                                                                0x8dfd
+#define regRCC_DEV0_2_RCC_DEV0_LINK_CNTL_BASE_IDX                                                       5
+#define regRCC_DEV0_2_RCC_CMN_LINK_CNTL                                                                 0x8dfe
+#define regRCC_DEV0_2_RCC_CMN_LINK_CNTL_BASE_IDX                                                        5
+#define regRCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE                                                        0x8dff
+#define regRCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE_BASE_IDX                                               5
+#define regRCC_DEV0_2_RCC_LTR_LSWITCH_CNTL                                                              0x8e00
+#define regRCC_DEV0_2_RCC_LTR_LSWITCH_CNTL_BASE_IDX                                                     5
+#define regRCC_DEV0_2_RCC_MH_ARB_CNTL                                                                   0x8e01
+#define regRCC_DEV0_2_RCC_MH_ARB_CNTL_BASE_IDX                                                          5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_SYSDEC
+// base address: 0x10120000
+#define regBIF_BX1_SYSHUB_INDEX_OVLP                                                                    0x8008
+#define regBIF_BX1_SYSHUB_INDEX_OVLP_BASE_IDX                                                           5
+#define regBIF_BX1_SYSHUB_DATA_OVLP                                                                     0x8009
+#define regBIF_BX1_SYSHUB_DATA_OVLP_BASE_IDX                                                            5
+#define regBIF_BX1_SYSHUB_INDEX_HI_OVLP                                                                 0x800a
+#define regBIF_BX1_SYSHUB_INDEX_HI_OVLP_BASE_IDX                                                        5
+#define regBIF_BX1_PCIE_INDEX                                                                           0x800c
+#define regBIF_BX1_PCIE_INDEX_BASE_IDX                                                                  5
+#define regBIF_BX1_PCIE_DATA                                                                            0x800d
+#define regBIF_BX1_PCIE_DATA_BASE_IDX                                                                   5
+#define regBIF_BX1_PCIE_INDEX2                                                                          0x800e
+#define regBIF_BX1_PCIE_INDEX2_BASE_IDX                                                                 5
+#define regBIF_BX1_PCIE_DATA2                                                                           0x800f
+#define regBIF_BX1_PCIE_DATA2_BASE_IDX                                                                  5
+#define regBIF_BX1_PCIE_INDEX_HI                                                                        0x8010
+#define regBIF_BX1_PCIE_INDEX_HI_BASE_IDX                                                               5
+#define regBIF_BX1_PCIE_INDEX2_HI                                                                       0x8011
+#define regBIF_BX1_PCIE_INDEX2_HI_BASE_IDX                                                              5
+#define regBIF_BX1_SBIOS_SCRATCH_0                                                                      0x8048
+#define regBIF_BX1_SBIOS_SCRATCH_0_BASE_IDX                                                             5
+#define regBIF_BX1_SBIOS_SCRATCH_1                                                                      0x8049
+#define regBIF_BX1_SBIOS_SCRATCH_1_BASE_IDX                                                             5
+#define regBIF_BX1_SBIOS_SCRATCH_2                                                                      0x804a
+#define regBIF_BX1_SBIOS_SCRATCH_2_BASE_IDX                                                             5
+#define regBIF_BX1_SBIOS_SCRATCH_3                                                                      0x804b
+#define regBIF_BX1_SBIOS_SCRATCH_3_BASE_IDX                                                             5
+#define regBIF_BX1_BIOS_SCRATCH_0                                                                       0x804c
+#define regBIF_BX1_BIOS_SCRATCH_0_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_1                                                                       0x804d
+#define regBIF_BX1_BIOS_SCRATCH_1_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_2                                                                       0x804e
+#define regBIF_BX1_BIOS_SCRATCH_2_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_3                                                                       0x804f
+#define regBIF_BX1_BIOS_SCRATCH_3_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_4                                                                       0x8050
+#define regBIF_BX1_BIOS_SCRATCH_4_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_5                                                                       0x8051
+#define regBIF_BX1_BIOS_SCRATCH_5_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_6                                                                       0x8052
+#define regBIF_BX1_BIOS_SCRATCH_6_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_7                                                                       0x8053
+#define regBIF_BX1_BIOS_SCRATCH_7_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_8                                                                       0x8054
+#define regBIF_BX1_BIOS_SCRATCH_8_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_9                                                                       0x8055
+#define regBIF_BX1_BIOS_SCRATCH_9_BASE_IDX                                                              5
+#define regBIF_BX1_BIOS_SCRATCH_10                                                                      0x8056
+#define regBIF_BX1_BIOS_SCRATCH_10_BASE_IDX                                                             5
+#define regBIF_BX1_BIOS_SCRATCH_11                                                                      0x8057
+#define regBIF_BX1_BIOS_SCRATCH_11_BASE_IDX                                                             5
+#define regBIF_BX1_BIOS_SCRATCH_12                                                                      0x8058
+#define regBIF_BX1_BIOS_SCRATCH_12_BASE_IDX                                                             5
+#define regBIF_BX1_BIOS_SCRATCH_13                                                                      0x8059
+#define regBIF_BX1_BIOS_SCRATCH_13_BASE_IDX                                                             5
+#define regBIF_BX1_BIOS_SCRATCH_14                                                                      0x805a
+#define regBIF_BX1_BIOS_SCRATCH_14_BASE_IDX                                                             5
+#define regBIF_BX1_BIOS_SCRATCH_15                                                                      0x805b
+#define regBIF_BX1_BIOS_SCRATCH_15_BASE_IDX                                                             5
+#define regBIF_BX1_BIF_RLC_INTR_CNTL                                                                    0x8060
+#define regBIF_BX1_BIF_RLC_INTR_CNTL_BASE_IDX                                                           5
+#define regBIF_BX1_BIF_VCE_INTR_CNTL                                                                    0x8061
+#define regBIF_BX1_BIF_VCE_INTR_CNTL_BASE_IDX                                                           5
+#define regBIF_BX1_BIF_UVD_INTR_CNTL                                                                    0x8062
+#define regBIF_BX1_BIF_UVD_INTR_CNTL_BASE_IDX                                                           5
+#define regBIF_BX1_BIF_Engine_INTR_CNTL                                                                 0x8063
+#define regBIF_BX1_BIF_Engine_INTR_CNTL_BASE_IDX                                                        5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR0                                                                0x8080
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR0_BASE_IDX                                                       5
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0                                                          0x8081
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0_BASE_IDX                                                 5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR1                                                                0x8082
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR1_BASE_IDX                                                       5
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1                                                          0x8083
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1_BASE_IDX                                                 5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR2                                                                0x8084
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR2_BASE_IDX                                                       5
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2                                                          0x8085
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2_BASE_IDX                                                 5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR3                                                                0x8086
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR3_BASE_IDX                                                       5
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3                                                          0x8087
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3_BASE_IDX                                                 5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR4                                                                0x8088
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR4_BASE_IDX                                                       5
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4                                                          0x8089
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4_BASE_IDX                                                 5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR5                                                                0x808a
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR5_BASE_IDX                                                       5
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5                                                          0x808b
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5_BASE_IDX                                                 5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR6                                                                0x808c
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR6_BASE_IDX                                                       5
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6                                                          0x808d
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6_BASE_IDX                                                 5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR7                                                                0x808e
+#define regBIF_BX1_GFX_MMIOREG_CAM_ADDR7_BASE_IDX                                                       5
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7                                                          0x808f
+#define regBIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7_BASE_IDX                                                 5
+#define regBIF_BX1_GFX_MMIOREG_CAM_CNTL                                                                 0x8090
+#define regBIF_BX1_GFX_MMIOREG_CAM_CNTL_BASE_IDX                                                        5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL                                                             0x8091
+#define regBIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL_BASE_IDX                                                    5
+#define regBIF_BX1_GFX_MMIOREG_CAM_ONE_CPL                                                              0x8092
+#define regBIF_BX1_GFX_MMIOREG_CAM_ONE_CPL_BASE_IDX                                                     5
+#define regBIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL                                                     0x8093
+#define regBIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL_BASE_IDX                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_0                                                                     0x8094
+#define regBIF_BX1_DRIVER_SCRATCH_0_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_1                                                                     0x8095
+#define regBIF_BX1_DRIVER_SCRATCH_1_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_2                                                                     0x8096
+#define regBIF_BX1_DRIVER_SCRATCH_2_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_3                                                                     0x8097
+#define regBIF_BX1_DRIVER_SCRATCH_3_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_4                                                                     0x8098
+#define regBIF_BX1_DRIVER_SCRATCH_4_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_5                                                                     0x8099
+#define regBIF_BX1_DRIVER_SCRATCH_5_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_6                                                                     0x809a
+#define regBIF_BX1_DRIVER_SCRATCH_6_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_7                                                                     0x809b
+#define regBIF_BX1_DRIVER_SCRATCH_7_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_8                                                                     0x809c
+#define regBIF_BX1_DRIVER_SCRATCH_8_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_9                                                                     0x809d
+#define regBIF_BX1_DRIVER_SCRATCH_9_BASE_IDX                                                            5
+#define regBIF_BX1_DRIVER_SCRATCH_10                                                                    0x809e
+#define regBIF_BX1_DRIVER_SCRATCH_10_BASE_IDX                                                           5
+#define regBIF_BX1_DRIVER_SCRATCH_11                                                                    0x809f
+#define regBIF_BX1_DRIVER_SCRATCH_11_BASE_IDX                                                           5
+#define regBIF_BX1_DRIVER_SCRATCH_12                                                                    0x80a0
+#define regBIF_BX1_DRIVER_SCRATCH_12_BASE_IDX                                                           5
+#define regBIF_BX1_DRIVER_SCRATCH_13                                                                    0x80a1
+#define regBIF_BX1_DRIVER_SCRATCH_13_BASE_IDX                                                           5
+#define regBIF_BX1_DRIVER_SCRATCH_14                                                                    0x80a2
+#define regBIF_BX1_DRIVER_SCRATCH_14_BASE_IDX                                                           5
+#define regBIF_BX1_DRIVER_SCRATCH_15                                                                    0x80a3
+#define regBIF_BX1_DRIVER_SCRATCH_15_BASE_IDX                                                           5
+#define regBIF_BX1_FW_SCRATCH_0                                                                         0x80a4
+#define regBIF_BX1_FW_SCRATCH_0_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_1                                                                         0x80a5
+#define regBIF_BX1_FW_SCRATCH_1_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_2                                                                         0x80a6
+#define regBIF_BX1_FW_SCRATCH_2_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_3                                                                         0x80a7
+#define regBIF_BX1_FW_SCRATCH_3_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_4                                                                         0x80a8
+#define regBIF_BX1_FW_SCRATCH_4_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_5                                                                         0x80a9
+#define regBIF_BX1_FW_SCRATCH_5_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_6                                                                         0x80aa
+#define regBIF_BX1_FW_SCRATCH_6_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_7                                                                         0x80ab
+#define regBIF_BX1_FW_SCRATCH_7_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_8                                                                         0x80ac
+#define regBIF_BX1_FW_SCRATCH_8_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_9                                                                         0x80ad
+#define regBIF_BX1_FW_SCRATCH_9_BASE_IDX                                                                5
+#define regBIF_BX1_FW_SCRATCH_10                                                                        0x80ae
+#define regBIF_BX1_FW_SCRATCH_10_BASE_IDX                                                               5
+#define regBIF_BX1_FW_SCRATCH_11                                                                        0x80af
+#define regBIF_BX1_FW_SCRATCH_11_BASE_IDX                                                               5
+#define regBIF_BX1_FW_SCRATCH_12                                                                        0x80b0
+#define regBIF_BX1_FW_SCRATCH_12_BASE_IDX                                                               5
+#define regBIF_BX1_FW_SCRATCH_13                                                                        0x80b1
+#define regBIF_BX1_FW_SCRATCH_13_BASE_IDX                                                               5
+#define regBIF_BX1_FW_SCRATCH_14                                                                        0x80b2
+#define regBIF_BX1_FW_SCRATCH_14_BASE_IDX                                                               5
+#define regBIF_BX1_FW_SCRATCH_15                                                                        0x80b3
+#define regBIF_BX1_FW_SCRATCH_15_BASE_IDX                                                               5
+#define regBIF_BX1_SBIOS_SCRATCH_4                                                                      0x80b4
+#define regBIF_BX1_SBIOS_SCRATCH_4_BASE_IDX                                                             5
+#define regBIF_BX1_SBIOS_SCRATCH_5                                                                      0x80b5
+#define regBIF_BX1_SBIOS_SCRATCH_5_BASE_IDX                                                             5
+#define regBIF_BX1_SBIOS_SCRATCH_6                                                                      0x80b6
+#define regBIF_BX1_SBIOS_SCRATCH_6_BASE_IDX                                                             5
+#define regBIF_BX1_SBIOS_SCRATCH_7                                                                      0x80b7
+#define regBIF_BX1_SBIOS_SCRATCH_7_BASE_IDX                                                             5
+#define regBIF_BX1_SBIOS_SCRATCH_8                                                                      0x80b8
+#define regBIF_BX1_SBIOS_SCRATCH_8_BASE_IDX                                                             5
+#define regBIF_BX1_SBIOS_SCRATCH_9                                                                      0x80b9
+#define regBIF_BX1_SBIOS_SCRATCH_9_BASE_IDX                                                             5
+#define regBIF_BX1_SBIOS_SCRATCH_10                                                                     0x80ba
+#define regBIF_BX1_SBIOS_SCRATCH_10_BASE_IDX                                                            5
+#define regBIF_BX1_SBIOS_SCRATCH_11                                                                     0x80bb
+#define regBIF_BX1_SBIOS_SCRATCH_11_BASE_IDX                                                            5
+#define regBIF_BX1_SBIOS_SCRATCH_12                                                                     0x80bc
+#define regBIF_BX1_SBIOS_SCRATCH_12_BASE_IDX                                                            5
+#define regBIF_BX1_SBIOS_SCRATCH_13                                                                     0x80bd
+#define regBIF_BX1_SBIOS_SCRATCH_13_BASE_IDX                                                            5
+#define regBIF_BX1_SBIOS_SCRATCH_14                                                                     0x80be
+#define regBIF_BX1_SBIOS_SCRATCH_14_BASE_IDX                                                            5
+#define regBIF_BX1_SBIOS_SCRATCH_15                                                                     0x80bf
+#define regBIF_BX1_SBIOS_SCRATCH_15_BASE_IDX                                                            5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_pf_SYSPFVFDEC
+// base address: 0x10120000
+#define regBIF_BX_PF1_MM_INDEX                                                                          0x8000
+#define regBIF_BX_PF1_MM_INDEX_BASE_IDX                                                                 5
+#define regBIF_BX_PF1_MM_DATA                                                                           0x8001
+#define regBIF_BX_PF1_MM_DATA_BASE_IDX                                                                  5
+#define regBIF_BX_PF1_MM_INDEX_HI                                                                       0x8006
+#define regBIF_BX_PF1_MM_INDEX_HI_BASE_IDX                                                              5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_BIFDEC1
+// base address: 0x10120000
+#define regBIF_BX1_CC_BIF_BX_STRAP0                                                                     0x8e02
+#define regBIF_BX1_CC_BIF_BX_STRAP0_BASE_IDX                                                            5
+#define regBIF_BX1_CC_BIF_BX_PINSTRAP0                                                                  0x8e04
+#define regBIF_BX1_CC_BIF_BX_PINSTRAP0_BASE_IDX                                                         5
+#define regBIF_BX1_BIF_MM_INDACCESS_CNTL                                                                0x8e06
+#define regBIF_BX1_BIF_MM_INDACCESS_CNTL_BASE_IDX                                                       5
+#define regBIF_BX1_BUS_CNTL                                                                             0x8e07
+#define regBIF_BX1_BUS_CNTL_BASE_IDX                                                                    5
+#define regBIF_BX1_BIF_SCRATCH0                                                                         0x8e08
+#define regBIF_BX1_BIF_SCRATCH0_BASE_IDX                                                                5
+#define regBIF_BX1_BIF_SCRATCH1                                                                         0x8e09
+#define regBIF_BX1_BIF_SCRATCH1_BASE_IDX                                                                5
+#define regBIF_BX1_BX_RESET_EN                                                                          0x8e0d
+#define regBIF_BX1_BX_RESET_EN_BASE_IDX                                                                 5
+#define regBIF_BX1_MM_CFGREGS_CNTL                                                                      0x8e0e
+#define regBIF_BX1_MM_CFGREGS_CNTL_BASE_IDX                                                             5
+#define regBIF_BX1_BX_RESET_CNTL                                                                        0x8e10
+#define regBIF_BX1_BX_RESET_CNTL_BASE_IDX                                                               5
+#define regBIF_BX1_INTERRUPT_CNTL                                                                       0x8e11
+#define regBIF_BX1_INTERRUPT_CNTL_BASE_IDX                                                              5
+#define regBIF_BX1_INTERRUPT_CNTL2                                                                      0x8e12
+#define regBIF_BX1_INTERRUPT_CNTL2_BASE_IDX                                                             5
+#define regBIF_BX1_CLKREQB_PAD_CNTL                                                                     0x8e18
+#define regBIF_BX1_CLKREQB_PAD_CNTL_BASE_IDX                                                            5
+#define regBIF_BX1_BIF_FEATURES_CONTROL_MISC                                                            0x8e1b
+#define regBIF_BX1_BIF_FEATURES_CONTROL_MISC_BASE_IDX                                                   5
+#define regBIF_BX1_HDP_ATOMIC_CONTROL_MISC                                                              0x8e1c
+#define regBIF_BX1_HDP_ATOMIC_CONTROL_MISC_BASE_IDX                                                     5
+#define regBIF_BX1_BIF_DOORBELL_CNTL                                                                    0x8e1d
+#define regBIF_BX1_BIF_DOORBELL_CNTL_BASE_IDX                                                           5
+#define regBIF_BX1_BIF_DOORBELL_INT_CNTL                                                                0x8e1e
+#define regBIF_BX1_BIF_DOORBELL_INT_CNTL_BASE_IDX                                                       5
+#define regBIF_BX1_BIF_FB_EN                                                                            0x8e20
+#define regBIF_BX1_BIF_FB_EN_BASE_IDX                                                                   5
+#define regBIF_BX1_BIF_INTR_CNTL                                                                        0x8e21
+#define regBIF_BX1_BIF_INTR_CNTL_BASE_IDX                                                               5
+#define regBIF_BX1_BIF_MST_TRANS_PENDING_VF                                                             0x8e29
+#define regBIF_BX1_BIF_MST_TRANS_PENDING_VF_BASE_IDX                                                    5
+#define regBIF_BX1_BIF_SLV_TRANS_PENDING_VF                                                             0x8e2a
+#define regBIF_BX1_BIF_SLV_TRANS_PENDING_VF_BASE_IDX                                                    5
+#define regBIF_BX1_BACO_CNTL                                                                            0x8e2b
+#define regBIF_BX1_BACO_CNTL_BASE_IDX                                                                   5
+#define regBIF_BX1_BIF_BACO_EXIT_TIME0                                                                  0x8e2c
+#define regBIF_BX1_BIF_BACO_EXIT_TIME0_BASE_IDX                                                         5
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER1                                                                 0x8e2d
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER1_BASE_IDX                                                        5
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER2                                                                 0x8e2e
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER2_BASE_IDX                                                        5
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER3                                                                 0x8e2f
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER3_BASE_IDX                                                        5
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER4                                                                 0x8e30
+#define regBIF_BX1_BIF_BACO_EXIT_TIMER4_BASE_IDX                                                        5
+#define regBIF_BX1_MEM_TYPE_CNTL                                                                        0x8e31
+#define regBIF_BX1_MEM_TYPE_CNTL_BASE_IDX                                                               5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_CNTL                                                               0x8e33
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_CNTL_BASE_IDX                                                      5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_0                                                                  0x8e34
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_0_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_1                                                                  0x8e35
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_1_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_2                                                                  0x8e36
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_2_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_3                                                                  0x8e37
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_3_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_4                                                                  0x8e38
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_4_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_5                                                                  0x8e39
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_5_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_6                                                                  0x8e3a
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_6_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_7                                                                  0x8e3b
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_7_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_8                                                                  0x8e3c
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_8_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_9                                                                  0x8e3d
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_9_BASE_IDX                                                         5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_10                                                                 0x8e3e
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_10_BASE_IDX                                                        5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_11                                                                 0x8e3f
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_11_BASE_IDX                                                        5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_12                                                                 0x8e40
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_12_BASE_IDX                                                        5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_13                                                                 0x8e41
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_13_BASE_IDX                                                        5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_14                                                                 0x8e42
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_14_BASE_IDX                                                        5
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_15                                                                 0x8e43
+#define regBIF_BX1_NBIF_GFX_ADDR_LUT_15_BASE_IDX                                                        5
+#define regBIF_BX1_VF_REGWR_EN                                                                          0x8e44
+#define regBIF_BX1_VF_REGWR_EN_BASE_IDX                                                                 5
+#define regBIF_BX1_VF_DOORBELL_EN                                                                       0x8e45
+#define regBIF_BX1_VF_DOORBELL_EN_BASE_IDX                                                              5
+#define regBIF_BX1_VF_FB_EN                                                                             0x8e46
+#define regBIF_BX1_VF_FB_EN_BASE_IDX                                                                    5
+#define regBIF_BX1_VF_REGWR_STATUS                                                                      0x8e47
+#define regBIF_BX1_VF_REGWR_STATUS_BASE_IDX                                                             5
+#define regBIF_BX1_VF_DOORBELL_STATUS                                                                   0x8e48
+#define regBIF_BX1_VF_DOORBELL_STATUS_BASE_IDX                                                          5
+#define regBIF_BX1_VF_FB_STATUS                                                                         0x8e49
+#define regBIF_BX1_VF_FB_STATUS_BASE_IDX                                                                5
+#define regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL                                                             0x8e4d
+#define regBIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL_BASE_IDX                                                    5
+#define regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL                                                             0x8e4e
+#define regBIF_BX1_REMAP_HDP_REG_FLUSH_CNTL_BASE_IDX                                                    5
+#define regBIF_BX1_BIF_RB_CNTL                                                                          0x8e4f
+#define regBIF_BX1_BIF_RB_CNTL_BASE_IDX                                                                 5
+#define regBIF_BX1_BIF_RB_BASE                                                                          0x8e50
+#define regBIF_BX1_BIF_RB_BASE_BASE_IDX                                                                 5
+#define regBIF_BX1_BIF_RB_RPTR                                                                          0x8e51
+#define regBIF_BX1_BIF_RB_RPTR_BASE_IDX                                                                 5
+#define regBIF_BX1_BIF_RB_WPTR                                                                          0x8e52
+#define regBIF_BX1_BIF_RB_WPTR_BASE_IDX                                                                 5
+#define regBIF_BX1_BIF_RB_WPTR_ADDR_HI                                                                  0x8e53
+#define regBIF_BX1_BIF_RB_WPTR_ADDR_HI_BASE_IDX                                                         5
+#define regBIF_BX1_BIF_RB_WPTR_ADDR_LO                                                                  0x8e54
+#define regBIF_BX1_BIF_RB_WPTR_ADDR_LO_BASE_IDX                                                         5
+#define regBIF_BX1_MAILBOX_INDEX                                                                        0x8e55
+#define regBIF_BX1_MAILBOX_INDEX_BASE_IDX                                                               5
+#define regBIF_BX1_BACO_AZ_ENHANCE_CTRL                                                                 0x8e61
+#define regBIF_BX1_BACO_AZ_ENHANCE_CTRL_BASE_IDX                                                        5
+#define regBIF_BX1_BIF_MP1_INTR_CTRL                                                                    0x8e62
+#define regBIF_BX1_BIF_MP1_INTR_CTRL_BASE_IDX                                                           5
+#define regBIF_BX1_BIF_PERSTB_PAD_CNTL                                                                  0x8e65
+#define regBIF_BX1_BIF_PERSTB_PAD_CNTL_BASE_IDX                                                         5
+#define regBIF_BX1_BIF_PX_EN_PAD_CNTL                                                                   0x8e66
+#define regBIF_BX1_BIF_PX_EN_PAD_CNTL_BASE_IDX                                                          5
+#define regBIF_BX1_BIF_REFPADKIN_PAD_CNTL                                                               0x8e67
+#define regBIF_BX1_BIF_REFPADKIN_PAD_CNTL_BASE_IDX                                                      5
+#define regBIF_BX1_BIF_CLKREQB_PAD_CNTL                                                                 0x8e68
+#define regBIF_BX1_BIF_CLKREQB_PAD_CNTL_BASE_IDX                                                        5
+#define regBIF_BX1_BIF_PWRBRK_PAD_CNTL                                                                  0x8e69
+#define regBIF_BX1_BIF_PWRBRK_PAD_CNTL_BASE_IDX                                                         5
+#define regBIF_BX1_BIF_GPUIOV_SCH_CFG_SIZE                                                              0x8e85
+#define regBIF_BX1_BIF_GPUIOV_SCH_CFG_SIZE_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_nbif0_bif_bx_pf_BIFPFVFDEC1
+// base address: 0x10120000
+#define regBIF_BX_PF1_BIF_BME_STATUS                                                                    0x8e0b
+#define regBIF_BX_PF1_BIF_BME_STATUS_BASE_IDX                                                           5
+#define regBIF_BX_PF1_BIF_ATOMIC_ERR_LOG                                                                0x8e0c
+#define regBIF_BX_PF1_BIF_ATOMIC_ERR_LOG_BASE_IDX                                                       5
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH                                              0x8e13
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH_BASE_IDX                                     5
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW                                               0x8e14
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW_BASE_IDX                                      5
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL                                                   0x8e15
+#define regBIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL_BASE_IDX                                          5
+#define regBIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL                                                      0x8e16
+#define regBIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL_BASE_IDX                                             5
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL                                                      0x8e17
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL_BASE_IDX                                             5
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL                                                 0x8e19
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL_BASE_IDX                                        5
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL                                            0x8e1a
+#define regBIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL_BASE_IDX                                   5
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_REQ                                                                 0x8e26
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_REQ_BASE_IDX                                                        5
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_DONE                                                                0x8e27
+#define regBIF_BX_PF1_GPU_HDP_FLUSH_DONE_BASE_IDX                                                       5
+#define regBIF_BX_PF1_BIF_TRANS_PENDING                                                                 0x8e28
+#define regBIF_BX_PF1_BIF_TRANS_PENDING_BASE_IDX                                                        5
+#define regBIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS                                                          0x8e32
+#define regBIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS_BASE_IDX                                                 5
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0                                                            0x8e56
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0_BASE_IDX                                                   5
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1                                                            0x8e57
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1_BASE_IDX                                                   5
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2                                                            0x8e58
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2_BASE_IDX                                                   5
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3                                                            0x8e59
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3_BASE_IDX                                                   5
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0                                                            0x8e5a
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0_BASE_IDX                                                   5
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1                                                            0x8e5b
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1_BASE_IDX                                                   5
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2                                                            0x8e5c
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2_BASE_IDX                                                   5
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3                                                            0x8e5d
+#define regBIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3_BASE_IDX                                                   5
+#define regBIF_BX_PF1_MAILBOX_CONTROL                                                                   0x8e5e
+#define regBIF_BX_PF1_MAILBOX_CONTROL_BASE_IDX                                                          5
+#define regBIF_BX_PF1_MAILBOX_INT_CNTL                                                                  0x8e5f
+#define regBIF_BX_PF1_MAILBOX_INT_CNTL_BASE_IDX                                                         5
+#define regBIF_BX_PF1_BIF_VMHV_MAILBOX                                                                  0x8e60
+#define regBIF_BX_PF1_BIF_VMHV_MAILBOX_BASE_IDX                                                         5
+#define regBIF_BX_PF1_PARTITION_COMPUTE_CAP                                                             0x8e81
+#define regBIF_BX_PF1_PARTITION_COMPUTE_CAP_BASE_IDX                                                    5
+#define regBIF_BX_PF1_PARTITION_MEM_CAP                                                                 0x8e82
+#define regBIF_BX_PF1_PARTITION_MEM_CAP_BASE_IDX                                                        5
+#define regBIF_BX_PF1_PARTITION_COMPUTE_STATUS                                                          0x8e83
+#define regBIF_BX_PF1_PARTITION_COMPUTE_STATUS_BASE_IDX                                                 5
+#define regBIF_BX_PF1_PARTITION_MEM_STATUS                                                              0x8e84
+#define regBIF_BX_PF1_PARTITION_MEM_STATUS_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_nbif0_rcc_strap_BIFDEC1:1
+// base address: 0x10120000
+#define regRCC_STRAP2_RCC_BIF_STRAP0                                                                    0x8d20
+#define regRCC_STRAP2_RCC_BIF_STRAP0_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP1                                                                    0x8d21
+#define regRCC_STRAP2_RCC_BIF_STRAP1_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP10                                                                   0x8d22
+#define regRCC_STRAP2_RCC_BIF_STRAP10_BASE_IDX                                                          5
+#define regRCC_STRAP2_RCC_BIF_STRAP11                                                                   0x8d23
+#define regRCC_STRAP2_RCC_BIF_STRAP11_BASE_IDX                                                          5
+#define regRCC_STRAP2_RCC_BIF_STRAP12                                                                   0x8d24
+#define regRCC_STRAP2_RCC_BIF_STRAP12_BASE_IDX                                                          5
+#define regRCC_STRAP2_RCC_BIF_STRAP13                                                                   0x8d25
+#define regRCC_STRAP2_RCC_BIF_STRAP13_BASE_IDX                                                          5
+#define regRCC_STRAP2_RCC_BIF_STRAP2                                                                    0x8d26
+#define regRCC_STRAP2_RCC_BIF_STRAP2_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP3                                                                    0x8d27
+#define regRCC_STRAP2_RCC_BIF_STRAP3_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP4                                                                    0x8d28
+#define regRCC_STRAP2_RCC_BIF_STRAP4_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP5                                                                    0x8d29
+#define regRCC_STRAP2_RCC_BIF_STRAP5_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP6                                                                    0x8d2a
+#define regRCC_STRAP2_RCC_BIF_STRAP6_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP7                                                                    0x8d2b
+#define regRCC_STRAP2_RCC_BIF_STRAP7_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP8                                                                    0x8d2c
+#define regRCC_STRAP2_RCC_BIF_STRAP8_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_BIF_STRAP9                                                                    0x8d2d
+#define regRCC_STRAP2_RCC_BIF_STRAP9_BASE_IDX                                                           5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP0                                                              0x8d2e
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP1                                                              0x8d2f
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP10                                                             0x8d30
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP10_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP11                                                             0x8d31
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP11_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP12                                                             0x8d32
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP12_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP13                                                             0x8d33
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP14                                                             0x8d34
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP15                                                             0x8d35
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP15_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP16                                                             0x8d36
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP16_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP17                                                             0x8d37
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP17_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP18                                                             0x8d38
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP18_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP2                                                              0x8d39
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP3                                                              0x8d3a
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP4                                                              0x8d3b
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP5                                                              0x8d3c
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP6                                                              0x8d3d
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP7                                                              0x8d3e
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP7_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP8                                                              0x8d3f
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP9                                                              0x8d40
+#define regRCC_STRAP2_RCC_DEV0_PORT_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP0                                                              0x8d41
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP1                                                              0x8d42
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP13                                                             0x8d43
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP14                                                             0x8d44
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP15                                                             0x8d45
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP15_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP16                                                             0x8d46
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP16_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP17                                                             0x8d47
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP17_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP18                                                             0x8d48
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP18_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP19                                                             0x8d49
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP19_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP2                                                              0x8d4a
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP22                                                             0x8d4b
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP22_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP23                                                             0x8d4c
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP23_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP24                                                             0x8d4d
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP24_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP3                                                              0x8d4e
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP4                                                              0x8d4f
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP5                                                              0x8d50
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP8                                                              0x8d51
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP9                                                              0x8d52
+#define regRCC_STRAP2_RCC_DEV0_EPF0_STRAP9_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP0                                                              0x8d53
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP0_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP1                                                              0x8d54
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP1_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP10                                                             0x8d55
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP10_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP11                                                             0x8d56
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP11_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP12                                                             0x8d57
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP12_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP13                                                             0x8d58
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP13_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP14                                                             0x8d59
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP14_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP15                                                             0x8d5a
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP15_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP16                                                             0x8d5b
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP16_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP17                                                             0x8d5c
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP17_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP18                                                             0x8d5d
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP18_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP19                                                             0x8d5e
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP19_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP2                                                              0x8d5f
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP2_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP20                                                             0x8d60
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP20_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP21                                                             0x8d61
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP21_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP23                                                             0x8d62
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP23_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP24                                                             0x8d63
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP24_BASE_IDX                                                    5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP3                                                              0x8d64
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP3_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP4                                                              0x8d65
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP4_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP5                                                              0x8d66
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP5_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP6                                                              0x8d67
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP6_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP7                                                              0x8d68
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP7_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP8                                                              0x8d69
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP8_BASE_IDX                                                     5
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP9                                                              0x8d6a
+#define regRCC_STRAP2_RCC_DEV0_EPF1_STRAP9_BASE_IDX                                                     5
+
+
+// addressBlock: nbif0_hpdma0_hpdma_private_hpdma_private_regblk
+// base address: 0x10100000
+#define regNBIF_SDP_PERF_COUNTER_2                                                                      0x100a
+#define regNBIF_SDP_PERF_COUNTER_2_BASE_IDX                                                             5
+#define regNBIF_SHUB_TODET_CLIENT_STATUS3                                                               0x10d1
+#define regNBIF_SHUB_TODET_CLIENT_STATUS3_BASE_IDX                                                      5
+#define regBIF_AER_ERR_LOG_DEV0_F0                                                                      0x10d5
+#define regBIF_AER_ERR_LOG_DEV0_F0_BASE_IDX                                                             5
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf0                                                                  0x10d6
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf0_BASE_IDX                                                         5
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf1                                                                  0x10d7
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf1_BASE_IDX                                                         5
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf2                                                                  0x10d8
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf2_BASE_IDX                                                         5
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf3                                                                  0x10d9
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf3_BASE_IDX                                                         5
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf4                                                                  0x10da
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf4_BASE_IDX                                                         5
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf5                                                                  0x10db
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf5_BASE_IDX                                                         5
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf6                                                                  0x10dc
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf6_BASE_IDX                                                         5
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf7                                                                  0x10dd
+#define regBIF_AER_ERR_LOG_DEV0_F0_vf7_BASE_IDX                                                         5
+
+
+// addressBlock: nbif0_hphost0_hphost_private_hphost_private_regblk
+// base address: 0x10106000
+#define regNBIF_AON_SVA_DISABLE_SAD                                                                     0x189a
+#define regNBIF_AON_SVA_DISABLE_SAD_BASE_IDX                                                            5
+
+
+// addressBlock: nbif0_nbif0_gdc_dma_sion_SIONDEC
+// base address: 0x1400000
+#define regGDC_DMA_SION_CL0_RdRsp_BurstTarget_REG0                                                      0x4f7400
+#define regGDC_DMA_SION_CL0_RdRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL0_RdRsp_BurstTarget_REG1                                                      0x4f7401
+#define regGDC_DMA_SION_CL0_RdRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL0_RdRsp_TimeSlot_REG0                                                         0x4f7402
+#define regGDC_DMA_SION_CL0_RdRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL0_RdRsp_TimeSlot_REG1                                                         0x4f7403
+#define regGDC_DMA_SION_CL0_RdRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL0_WrRsp_BurstTarget_REG0                                                      0x4f7404
+#define regGDC_DMA_SION_CL0_WrRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL0_WrRsp_BurstTarget_REG1                                                      0x4f7405
+#define regGDC_DMA_SION_CL0_WrRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL0_WrRsp_TimeSlot_REG0                                                         0x4f7406
+#define regGDC_DMA_SION_CL0_WrRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL0_WrRsp_TimeSlot_REG1                                                         0x4f7407
+#define regGDC_DMA_SION_CL0_WrRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL0_Req_BurstTarget_REG0                                                        0x4f7408
+#define regGDC_DMA_SION_CL0_Req_BurstTarget_REG0_BASE_IDX                                               3
+#define regGDC_DMA_SION_CL0_Req_BurstTarget_REG1                                                        0x4f7409
+#define regGDC_DMA_SION_CL0_Req_BurstTarget_REG1_BASE_IDX                                               3
+#define regGDC_DMA_SION_CL0_Req_TimeSlot_REG0                                                           0x4f740a
+#define regGDC_DMA_SION_CL0_Req_TimeSlot_REG0_BASE_IDX                                                  3
+#define regGDC_DMA_SION_CL0_Req_TimeSlot_REG1                                                           0x4f740b
+#define regGDC_DMA_SION_CL0_Req_TimeSlot_REG1_BASE_IDX                                                  3
+#define regGDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG0                                                    0x4f740c
+#define regGDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG0_BASE_IDX                                           3
+#define regGDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG1                                                    0x4f740d
+#define regGDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG1_BASE_IDX                                           3
+#define regGDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG0                                                   0x4f740e
+#define regGDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG0_BASE_IDX                                          3
+#define regGDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG1                                                   0x4f740f
+#define regGDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG1_BASE_IDX                                          3
+#define regGDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG0                                                  0x4f7410
+#define regGDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG1                                                  0x4f7411
+#define regGDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG0                                                  0x4f7412
+#define regGDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG1                                                  0x4f7413
+#define regGDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL1_RdRsp_BurstTarget_REG0                                                      0x4f7414
+#define regGDC_DMA_SION_CL1_RdRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL1_RdRsp_BurstTarget_REG1                                                      0x4f7415
+#define regGDC_DMA_SION_CL1_RdRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL1_RdRsp_TimeSlot_REG0                                                         0x4f7416
+#define regGDC_DMA_SION_CL1_RdRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL1_RdRsp_TimeSlot_REG1                                                         0x4f7417
+#define regGDC_DMA_SION_CL1_RdRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL1_WrRsp_BurstTarget_REG0                                                      0x4f7418
+#define regGDC_DMA_SION_CL1_WrRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL1_WrRsp_BurstTarget_REG1                                                      0x4f7419
+#define regGDC_DMA_SION_CL1_WrRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL1_WrRsp_TimeSlot_REG0                                                         0x4f741a
+#define regGDC_DMA_SION_CL1_WrRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL1_WrRsp_TimeSlot_REG1                                                         0x4f741b
+#define regGDC_DMA_SION_CL1_WrRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL1_Req_BurstTarget_REG0                                                        0x4f741c
+#define regGDC_DMA_SION_CL1_Req_BurstTarget_REG0_BASE_IDX                                               3
+#define regGDC_DMA_SION_CL1_Req_BurstTarget_REG1                                                        0x4f741d
+#define regGDC_DMA_SION_CL1_Req_BurstTarget_REG1_BASE_IDX                                               3
+#define regGDC_DMA_SION_CL1_Req_TimeSlot_REG0                                                           0x4f741e
+#define regGDC_DMA_SION_CL1_Req_TimeSlot_REG0_BASE_IDX                                                  3
+#define regGDC_DMA_SION_CL1_Req_TimeSlot_REG1                                                           0x4f741f
+#define regGDC_DMA_SION_CL1_Req_TimeSlot_REG1_BASE_IDX                                                  3
+#define regGDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG0                                                    0x4f7420
+#define regGDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG0_BASE_IDX                                           3
+#define regGDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG1                                                    0x4f7421
+#define regGDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG1_BASE_IDX                                           3
+#define regGDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG0                                                   0x4f7422
+#define regGDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG0_BASE_IDX                                          3
+#define regGDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG1                                                   0x4f7423
+#define regGDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG1_BASE_IDX                                          3
+#define regGDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG0                                                  0x4f7424
+#define regGDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG1                                                  0x4f7425
+#define regGDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG0                                                  0x4f7426
+#define regGDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG1                                                  0x4f7427
+#define regGDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL2_RdRsp_BurstTarget_REG0                                                      0x4f7428
+#define regGDC_DMA_SION_CL2_RdRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL2_RdRsp_BurstTarget_REG1                                                      0x4f7429
+#define regGDC_DMA_SION_CL2_RdRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL2_RdRsp_TimeSlot_REG0                                                         0x4f742a
+#define regGDC_DMA_SION_CL2_RdRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL2_RdRsp_TimeSlot_REG1                                                         0x4f742b
+#define regGDC_DMA_SION_CL2_RdRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL2_WrRsp_BurstTarget_REG0                                                      0x4f742c
+#define regGDC_DMA_SION_CL2_WrRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL2_WrRsp_BurstTarget_REG1                                                      0x4f742d
+#define regGDC_DMA_SION_CL2_WrRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_DMA_SION_CL2_WrRsp_TimeSlot_REG0                                                         0x4f742e
+#define regGDC_DMA_SION_CL2_WrRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL2_WrRsp_TimeSlot_REG1                                                         0x4f742f
+#define regGDC_DMA_SION_CL2_WrRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_DMA_SION_CL2_Req_BurstTarget_REG0                                                        0x4f7430
+#define regGDC_DMA_SION_CL2_Req_BurstTarget_REG0_BASE_IDX                                               3
+#define regGDC_DMA_SION_CL2_Req_BurstTarget_REG1                                                        0x4f7431
+#define regGDC_DMA_SION_CL2_Req_BurstTarget_REG1_BASE_IDX                                               3
+#define regGDC_DMA_SION_CL2_Req_TimeSlot_REG0                                                           0x4f7432
+#define regGDC_DMA_SION_CL2_Req_TimeSlot_REG0_BASE_IDX                                                  3
+#define regGDC_DMA_SION_CL2_Req_TimeSlot_REG1                                                           0x4f7433
+#define regGDC_DMA_SION_CL2_Req_TimeSlot_REG1_BASE_IDX                                                  3
+#define regGDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG0                                                    0x4f7434
+#define regGDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG0_BASE_IDX                                           3
+#define regGDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG1                                                    0x4f7435
+#define regGDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG1_BASE_IDX                                           3
+#define regGDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG0                                                   0x4f7436
+#define regGDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG0_BASE_IDX                                          3
+#define regGDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG1                                                   0x4f7437
+#define regGDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG1_BASE_IDX                                          3
+#define regGDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG0                                                  0x4f7438
+#define regGDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG1                                                  0x4f7439
+#define regGDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG0                                                  0x4f743a
+#define regGDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG1                                                  0x4f743b
+#define regGDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_DMA_SION_CNTL_REG0                                                                       0x4f743c
+#define regGDC_DMA_SION_CNTL_REG0_BASE_IDX                                                              3
+#define regGDC_DMA_SION_CNTL_REG1                                                                       0x4f743d
+#define regGDC_DMA_SION_CNTL_REG1_BASE_IDX                                                              3
+
+
+// addressBlock: nbif0_nbif0_gdc_hst_sion_SIONDEC
+// base address: 0x1400000
+#define regGDC_HST_SION_CL0_RdRsp_BurstTarget_REG0                                                      0x4f7600
+#define regGDC_HST_SION_CL0_RdRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_HST_SION_CL0_RdRsp_BurstTarget_REG1                                                      0x4f7601
+#define regGDC_HST_SION_CL0_RdRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_HST_SION_CL0_RdRsp_TimeSlot_REG0                                                         0x4f7602
+#define regGDC_HST_SION_CL0_RdRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_HST_SION_CL0_RdRsp_TimeSlot_REG1                                                         0x4f7603
+#define regGDC_HST_SION_CL0_RdRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_HST_SION_CL0_WrRsp_BurstTarget_REG0                                                      0x4f7604
+#define regGDC_HST_SION_CL0_WrRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_HST_SION_CL0_WrRsp_BurstTarget_REG1                                                      0x4f7605
+#define regGDC_HST_SION_CL0_WrRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_HST_SION_CL0_WrRsp_TimeSlot_REG0                                                         0x4f7606
+#define regGDC_HST_SION_CL0_WrRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_HST_SION_CL0_WrRsp_TimeSlot_REG1                                                         0x4f7607
+#define regGDC_HST_SION_CL0_WrRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_HST_SION_CL0_Req_BurstTarget_REG0                                                        0x4f7608
+#define regGDC_HST_SION_CL0_Req_BurstTarget_REG0_BASE_IDX                                               3
+#define regGDC_HST_SION_CL0_Req_BurstTarget_REG1                                                        0x4f7609
+#define regGDC_HST_SION_CL0_Req_BurstTarget_REG1_BASE_IDX                                               3
+#define regGDC_HST_SION_CL0_Req_TimeSlot_REG0                                                           0x4f760a
+#define regGDC_HST_SION_CL0_Req_TimeSlot_REG0_BASE_IDX                                                  3
+#define regGDC_HST_SION_CL0_Req_TimeSlot_REG1                                                           0x4f760b
+#define regGDC_HST_SION_CL0_Req_TimeSlot_REG1_BASE_IDX                                                  3
+#define regGDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG0                                                    0x4f760c
+#define regGDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG0_BASE_IDX                                           3
+#define regGDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG1                                                    0x4f760d
+#define regGDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG1_BASE_IDX                                           3
+#define regGDC_HST_SION_CL0_DataPoolCredit_Alloc_REG0                                                   0x4f760e
+#define regGDC_HST_SION_CL0_DataPoolCredit_Alloc_REG0_BASE_IDX                                          3
+#define regGDC_HST_SION_CL0_DataPoolCredit_Alloc_REG1                                                   0x4f760f
+#define regGDC_HST_SION_CL0_DataPoolCredit_Alloc_REG1_BASE_IDX                                          3
+#define regGDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG0                                                  0x4f7610
+#define regGDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG1                                                  0x4f7611
+#define regGDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG0                                                  0x4f7612
+#define regGDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG1                                                  0x4f7613
+#define regGDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_HST_SION_CL1_RdRsp_BurstTarget_REG0                                                      0x4f7614
+#define regGDC_HST_SION_CL1_RdRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_HST_SION_CL1_RdRsp_BurstTarget_REG1                                                      0x4f7615
+#define regGDC_HST_SION_CL1_RdRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_HST_SION_CL1_RdRsp_TimeSlot_REG0                                                         0x4f7616
+#define regGDC_HST_SION_CL1_RdRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_HST_SION_CL1_RdRsp_TimeSlot_REG1                                                         0x4f7617
+#define regGDC_HST_SION_CL1_RdRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_HST_SION_CL1_WrRsp_BurstTarget_REG0                                                      0x4f7618
+#define regGDC_HST_SION_CL1_WrRsp_BurstTarget_REG0_BASE_IDX                                             3
+#define regGDC_HST_SION_CL1_WrRsp_BurstTarget_REG1                                                      0x4f7619
+#define regGDC_HST_SION_CL1_WrRsp_BurstTarget_REG1_BASE_IDX                                             3
+#define regGDC_HST_SION_CL1_WrRsp_TimeSlot_REG0                                                         0x4f761a
+#define regGDC_HST_SION_CL1_WrRsp_TimeSlot_REG0_BASE_IDX                                                3
+#define regGDC_HST_SION_CL1_WrRsp_TimeSlot_REG1                                                         0x4f761b
+#define regGDC_HST_SION_CL1_WrRsp_TimeSlot_REG1_BASE_IDX                                                3
+#define regGDC_HST_SION_CL1_Req_BurstTarget_REG0                                                        0x4f761c
+#define regGDC_HST_SION_CL1_Req_BurstTarget_REG0_BASE_IDX                                               3
+#define regGDC_HST_SION_CL1_Req_BurstTarget_REG1                                                        0x4f761d
+#define regGDC_HST_SION_CL1_Req_BurstTarget_REG1_BASE_IDX                                               3
+#define regGDC_HST_SION_CL1_Req_TimeSlot_REG0                                                           0x4f761e
+#define regGDC_HST_SION_CL1_Req_TimeSlot_REG0_BASE_IDX                                                  3
+#define regGDC_HST_SION_CL1_Req_TimeSlot_REG1                                                           0x4f761f
+#define regGDC_HST_SION_CL1_Req_TimeSlot_REG1_BASE_IDX                                                  3
+#define regGDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG0                                                    0x4f7620
+#define regGDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG0_BASE_IDX                                           3
+#define regGDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG1                                                    0x4f7621
+#define regGDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG1_BASE_IDX                                           3
+#define regGDC_HST_SION_CL1_DataPoolCredit_Alloc_REG0                                                   0x4f7622
+#define regGDC_HST_SION_CL1_DataPoolCredit_Alloc_REG0_BASE_IDX                                          3
+#define regGDC_HST_SION_CL1_DataPoolCredit_Alloc_REG1                                                   0x4f7623
+#define regGDC_HST_SION_CL1_DataPoolCredit_Alloc_REG1_BASE_IDX                                          3
+#define regGDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG0                                                  0x4f7624
+#define regGDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG1                                                  0x4f7625
+#define regGDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG0                                                  0x4f7626
+#define regGDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG0_BASE_IDX                                         3
+#define regGDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG1                                                  0x4f7627
+#define regGDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG1_BASE_IDX                                         3
+#define regGDC_HST_SION_CNTL_REG0                                                                       0x4f7628
+#define regGDC_HST_SION_CNTL_REG0_BASE_IDX                                                              3
+#define regGDC_HST_SION_CNTL_REG1                                                                       0x4f7629
+#define regGDC_HST_SION_CNTL_REG1_BASE_IDX                                                              3
+
+
+// addressBlock: nbif0_nbif0_gdc_GDCDEC
+// base address: 0x1400000
+#define regGDC1_NGDC_SDP_PORT_CTRL                                                                      0x4f0aa0
+#define regGDC1_NGDC_SDP_PORT_CTRL_BASE_IDX                                                             3
+#define regGDC1_SHUB_REGS_IF_CTL                                                                        0x4f0aa1
+#define regGDC1_SHUB_REGS_IF_CTL_BASE_IDX                                                               3
+#define regGDC1_NGDC_MGCG_CTRL                                                                          0x4f0aa7
+#define regGDC1_NGDC_MGCG_CTRL_BASE_IDX                                                                 3
+#define regGDC1_S2A_MISC_CNTL                                                                           0x4f0aa8
+#define regGDC1_S2A_MISC_CNTL_BASE_IDX                                                                  3
+#define regGDC1_NGDC_MCA_SMN_CTRL0                                                                      0x4f0aab
+#define regGDC1_NGDC_MCA_SMN_CTRL0_BASE_IDX                                                             3
+#define regGDC1_NGDC_EARLY_WAKEUP_CTRL                                                                  0x4f0aac
+#define regGDC1_NGDC_EARLY_WAKEUP_CTRL_BASE_IDX                                                         3
+#define regGDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL                                                             0x4f0ab8
+#define regGDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL_BASE_IDX                                                    3
+#define regGDC1_GDC_DMA_SION_PCTRL                                                                      0x4f0ab9
+#define regGDC1_GDC_DMA_SION_PCTRL_BASE_IDX                                                             3
+#define regGDC1_NGDC_MP4SDP_CNTL                                                                        0x4f0abb
+#define regGDC1_NGDC_MP4SDP_CNTL_BASE_IDX                                                               3
+#define regGDC1_DOORBELL_VCN_TARGET_VF0                                                                 0x4f0b13
+#define regGDC1_DOORBELL_VCN_TARGET_VF0_BASE_IDX                                                        3
+#define regGDC1_DOORBELL_VCN_TARGET_VF1                                                                 0x4f0b13
+#define regGDC1_DOORBELL_VCN_TARGET_VF1_BASE_IDX                                                        3
+#define regGDC1_DOORBELL_VCN_TARGET_VF2                                                                 0x4f0b14
+#define regGDC1_DOORBELL_VCN_TARGET_VF2_BASE_IDX                                                        3
+#define regGDC1_DOORBELL_VCN_TARGET_VF3                                                                 0x4f0b14
+#define regGDC1_DOORBELL_VCN_TARGET_VF3_BASE_IDX                                                        3
+#define regGDC1_DOORBELL_VCN_TARGET_VF4                                                                 0x4f0b15
+#define regGDC1_DOORBELL_VCN_TARGET_VF4_BASE_IDX                                                        3
+#define regGDC1_DOORBELL_VCN_TARGET_VF5                                                                 0x4f0b15
+#define regGDC1_DOORBELL_VCN_TARGET_VF5_BASE_IDX                                                        3
+#define regGDC1_DOORBELL_VCN_TARGET_VF6                                                                 0x4f0b16
+#define regGDC1_DOORBELL_VCN_TARGET_VF6_BASE_IDX                                                        3
+#define regGDC1_DOORBELL_VCN_TARGET_VF7                                                                 0x4f0b16
+#define regGDC1_DOORBELL_VCN_TARGET_VF7_BASE_IDX                                                        3
+#define regGDC1_DOORBELL_ACCESS_EN_PF                                                                   0x4f0b17
+#define regGDC1_DOORBELL_ACCESS_EN_PF_BASE_IDX                                                          3
+#define regGDC1_DOORBELL_ACCESS_EN_VF0                                                                  0x4f0b17
+#define regGDC1_DOORBELL_ACCESS_EN_VF0_BASE_IDX                                                         3
+#define regGDC1_DOORBELL_ACCESS_EN_VF1                                                                  0x4f0b18
+#define regGDC1_DOORBELL_ACCESS_EN_VF1_BASE_IDX                                                         3
+#define regGDC1_DOORBELL_ACCESS_EN_VF2                                                                  0x4f0b18
+#define regGDC1_DOORBELL_ACCESS_EN_VF2_BASE_IDX                                                         3
+#define regGDC1_DOORBELL_ACCESS_EN_VF3                                                                  0x4f0b19
+#define regGDC1_DOORBELL_ACCESS_EN_VF3_BASE_IDX                                                         3
+#define regGDC1_DOORBELL_ACCESS_EN_VF4                                                                  0x4f0b19
+#define regGDC1_DOORBELL_ACCESS_EN_VF4_BASE_IDX                                                         3
+#define regGDC1_DOORBELL_ACCESS_EN_VF5                                                                  0x4f0b1a
+#define regGDC1_DOORBELL_ACCESS_EN_VF5_BASE_IDX                                                         3
+#define regGDC1_DOORBELL_ACCESS_EN_VF6                                                                  0x4f0b1a
+#define regGDC1_DOORBELL_ACCESS_EN_VF6_BASE_IDX                                                         3
+#define regGDC1_DOORBELL_ACCESS_EN_VF7                                                                  0x4f0b1b
+#define regGDC1_DOORBELL_ACCESS_EN_VF7_BASE_IDX                                                         3
+#define regGDC1_NGDC_CNDI_BUS_PORT_CTRL                                                                 0x4f0b1c
+#define regGDC1_NGDC_CNDI_BUS_PORT_CTRL_BASE_IDX                                                        3
+#define regGDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL                                                            0x4f0b1d
+#define regGDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL_BASE_IDX                                                   3
+
+
+// addressBlock: nbif0_nbif0_gdc_ras_gdc_ras_regblk
+// base address: 0x1400000
+#define regGDCSOC_ERR_RSP_CNTL                                                                          0x4f5c00
+#define regGDCSOC_ERR_RSP_CNTL_BASE_IDX                                                                 3
+#define regGDCSOC_RAS_CENTRAL_STATUS                                                                    0x4f5c10
+#define regGDCSOC_RAS_CENTRAL_STATUS_BASE_IDX                                                           3
+#define regGDCSOC_RAS_LEAF0_CTRL                                                                        0x4f5c20
+#define regGDCSOC_RAS_LEAF0_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF1_CTRL                                                                        0x4f5c21
+#define regGDCSOC_RAS_LEAF1_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF2_CTRL                                                                        0x4f5c22
+#define regGDCSOC_RAS_LEAF2_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF3_CTRL                                                                        0x4f5c23
+#define regGDCSOC_RAS_LEAF3_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF4_CTRL                                                                        0x4f5c24
+#define regGDCSOC_RAS_LEAF4_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF5_CTRL                                                                        0x4f5c25
+#define regGDCSOC_RAS_LEAF5_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF6_CTRL                                                                        0x4f5c26
+#define regGDCSOC_RAS_LEAF6_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF7_CTRL                                                                        0x4f5c27
+#define regGDCSOC_RAS_LEAF7_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF8_CTRL                                                                        0x4f5c28
+#define regGDCSOC_RAS_LEAF8_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF9_CTRL                                                                        0x4f5c29
+#define regGDCSOC_RAS_LEAF9_CTRL_BASE_IDX                                                               3
+#define regGDCSOC_RAS_LEAF10_CTRL                                                                       0x4f5c2a
+#define regGDCSOC_RAS_LEAF10_CTRL_BASE_IDX                                                              3
+#define regGDCSOC_RAS_LEAF11_CTRL                                                                       0x4f5c2b
+#define regGDCSOC_RAS_LEAF11_CTRL_BASE_IDX                                                              3
+#define regGDCSOC_RAS_LEAF12_CTRL                                                                       0x4f5c2c
+#define regGDCSOC_RAS_LEAF12_CTRL_BASE_IDX                                                              3
+#define regGDCSOC_RAS_LEAF13_CTRL                                                                       0x4f5c2d
+#define regGDCSOC_RAS_LEAF13_CTRL_BASE_IDX                                                              3
+#define regGDCSOC_RAS_LEAF0_STATUS                                                                      0x4f5c30
+#define regGDCSOC_RAS_LEAF0_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF1_STATUS                                                                      0x4f5c31
+#define regGDCSOC_RAS_LEAF1_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF2_STATUS                                                                      0x4f5c32
+#define regGDCSOC_RAS_LEAF2_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF3_STATUS                                                                      0x4f5c33
+#define regGDCSOC_RAS_LEAF3_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF4_STATUS                                                                      0x4f5c34
+#define regGDCSOC_RAS_LEAF4_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF5_STATUS                                                                      0x4f5c35
+#define regGDCSOC_RAS_LEAF5_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF6_STATUS                                                                      0x4f5c36
+#define regGDCSOC_RAS_LEAF6_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF7_STATUS                                                                      0x4f5c37
+#define regGDCSOC_RAS_LEAF7_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF8_STATUS                                                                      0x4f5c38
+#define regGDCSOC_RAS_LEAF8_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF9_STATUS                                                                      0x4f5c39
+#define regGDCSOC_RAS_LEAF9_STATUS_BASE_IDX                                                             3
+#define regGDCSOC_RAS_LEAF10_STATUS                                                                     0x4f5c3a
+#define regGDCSOC_RAS_LEAF10_STATUS_BASE_IDX                                                            3
+#define regGDCSOC_RAS_LEAF11_STATUS                                                                     0x4f5c3b
+#define regGDCSOC_RAS_LEAF11_STATUS_BASE_IDX                                                            3
+#define regGDCSOC_RAS_LEAF12_STATUS                                                                     0x4f5c3c
+#define regGDCSOC_RAS_LEAF12_STATUS_BASE_IDX                                                            3
+#define regGDCSOC_RAS_LEAF13_STATUS                                                                     0x4f5c3d
+#define regGDCSOC_RAS_LEAF13_STATUS_BASE_IDX                                                            3
+
+
+// addressBlock: nbif0_nbif0_gdc_sec_GDCSEC_DEC
+// base address: 0x1400000
+#define regGDC_SEC_MCA_SMN_CTRL0                                                                        0x4f7001
+#define regGDC_SEC_MCA_SMN_CTRL0_BASE_IDX                                                               3
+#define regGDC_SEC_MCA_SMN_CTRL1                                                                        0x4f7002
+#define regGDC_SEC_MCA_SMN_CTRL1_BASE_IDX                                                               3
+#define regXCD_DOORBELL_FENCE                                                                           0x4f7003
+#define regXCD_DOORBELL_FENCE_BASE_IDX                                                                  3
+#define regSHUB_PWRBRK_DEGLITCH                                                                         0x4f7004
+#define regSHUB_PWRBRK_DEGLITCH_BASE_IDX                                                                3
+#define regSHUB_DIE_CTRL                                                                                0x4f7005
+#define regSHUB_DIE_CTRL_BASE_IDX                                                                       3
+#define regGDC_A2S_ROUTING_TABLE_SECLVL_TRUST_LEVEL                                                     0x4f700a
+#define regGDC_A2S_ROUTING_TABLE_SECLVL_TRUST_LEVEL_BASE_IDX                                            3
+#define regXCD_DOORBELL_FENCE_1                                                                         0x4f700b
+#define regXCD_DOORBELL_FENCE_1_BASE_IDX                                                                3
+#define regNGDC_CNDI_DOORBELL_REQUSER_0                                                                 0x4f700c
+#define regNGDC_CNDI_DOORBELL_REQUSER_0_BASE_IDX                                                        3
+#define regNGDC_CNDI_DOORBELL_REQUSER_1                                                                 0x4f700d
+#define regNGDC_CNDI_DOORBELL_REQUSER_1_BASE_IDX                                                        3
+#define regNGDC_CNDI_DOORBELL_REQUSER_2                                                                 0x4f700e
+#define regNGDC_CNDI_DOORBELL_REQUSER_2_BASE_IDX                                                        3
+#define regNGDC_CNDI_DOORBELL_REQUSER_3                                                                 0x4f700f
+#define regNGDC_CNDI_DOORBELL_REQUSER_3_BASE_IDX                                                        3
+#define regNGDC_CNDI_DOORBELL_REQUSER_4                                                                 0x4f7010
+#define regNGDC_CNDI_DOORBELL_REQUSER_4_BASE_IDX                                                        3
+#define regNGDC_CNDI_DOORBELL_REQUSER_5                                                                 0x4f7011
+#define regNGDC_CNDI_DOORBELL_REQUSER_5_BASE_IDX                                                        3
+#define regNGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0                                                        0x4f7012
+#define regNGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0_BASE_IDX                                               3
+#define regNGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1                                                        0x4f7013
+#define regNGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1_BASE_IDX                                               3
+
+
+// addressBlock: nbif0_nbif0_gdc_rst_GDCRST_DEC
+// base address: 0x1400000
+#define regSHUB_PF_FLR_RST                                                                              0x4f7800
+#define regSHUB_PF_FLR_RST_BASE_IDX                                                                     3
+#define regSHUB_GFX_DRV_VPU_RST                                                                         0x4f7801
+#define regSHUB_GFX_DRV_VPU_RST_BASE_IDX                                                                3
+#define regSHUB_LINK_RESET                                                                              0x4f7802
+#define regSHUB_LINK_RESET_BASE_IDX                                                                     3
+#define regSHUB_HST_NIC400_SW_CTRL_RESET                                                                0x4f7803
+#define regSHUB_HST_NIC400_SW_CTRL_RESET_BASE_IDX                                                       3
+#define regSHUB_DEV0_PF0_VF_FLR_RST                                                                     0x4f7808
+#define regSHUB_DEV0_PF0_VF_FLR_RST_BASE_IDX                                                            3
+#define regSHUB_DEV0_PF1_VF_FLR_RST                                                                     0x4f7809
+#define regSHUB_DEV0_PF1_VF_FLR_RST_BASE_IDX                                                            3
+#define regSHUB_HARD_RST_CTRL                                                                           0x4f7810
+#define regSHUB_HARD_RST_CTRL_BASE_IDX                                                                  3
+#define regSHUB_SOFT_RST_CTRL                                                                           0x4f7811
+#define regSHUB_SOFT_RST_CTRL_BASE_IDX                                                                  3
+#define regSHUB_SDP_PORT_RST                                                                            0x4f7812
+#define regSHUB_SDP_PORT_RST_BASE_IDX                                                                   3
+#define regSHUB_RST_MISC_TRL                                                                            0x4f7813
+#define regSHUB_RST_MISC_TRL_BASE_IDX                                                                   3
+
+
+// addressBlock: nbif0_nbif0_gdc_s2a_GDCS2A_DEC
+// base address: 0x1400000
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL                                                           0x4f0aeb
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1                                                          0x4f0aec
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL                                                           0x4f0aed
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1                                                          0x4f0aee
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL                                                           0x4f0aef
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1                                                          0x4f0af0
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL                                                           0x4f0af1
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1                                                          0x4f0af2
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL                                                           0x4f0af3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1                                                          0x4f0af4
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL                                                           0x4f0af5
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1                                                          0x4f0af6
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL                                                           0x4f0af7
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1                                                          0x4f0af8
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL                                                           0x4f0af9
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1                                                          0x4f0afa
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL                                                           0x4f0afb
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1                                                          0x4f0afc
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL                                                           0x4f0afd
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL_BASE_IDX                                                  3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1                                                          0x4f0afe
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL                                                          0x4f0aff
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1                                                         0x4f0b00
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1_BASE_IDX                                                3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL                                                          0x4f0b01
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1                                                         0x4f0b02
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1_BASE_IDX                                                3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL                                                          0x4f0b03
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1                                                         0x4f0b04
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1_BASE_IDX                                                3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL                                                          0x4f0b05
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1                                                         0x4f0b06
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1_BASE_IDX                                                3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL                                                          0x4f0b07
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1                                                         0x4f0b08
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1_BASE_IDX                                                3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL                                                          0x4f0b09
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL_BASE_IDX                                                 3
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1                                                         0x4f0b0a
+#define regGDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1_BASE_IDX                                                3
+#define regGDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG                                                        0x4f0b0b
+#define regGDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG_BASE_IDX                                               3
+#define regGDC_S2A1_NBIF_GFX_DOORBELL_STATUS                                                            0x4f0b0c
+#define regGDC_S2A1_NBIF_GFX_DOORBELL_STATUS_BASE_IDX                                                   3
+
+
+// addressBlock: nbif0_nbif0_gdc_misc_GDCMISC_DEC
+// base address: 0x1400000
+#define regGDC_DMA_SION_CRED_RST                                                                        0x4f3401
+#define regGDC_DMA_SION_CRED_RST_BASE_IDX                                                               3
+#define regGDC_HST_SION_PER_SDP_PORT_RST                                                                0x4f3402
+#define regGDC_HST_SION_PER_SDP_PORT_RST_BASE_IDX                                                       3
+#define regGDC_HST_SION_PER_SDP_CNTL_REG1                                                               0x4f3403
+#define regGDC_HST_SION_PER_SDP_CNTL_REG1_BASE_IDX                                                      3
+#define regGDC_HST_AXI_STALL_FOR_FLUSH                                                                  0x4f3404
+#define regGDC_HST_AXI_STALL_FOR_FLUSH_BASE_IDX                                                         3
+#define regSYSHUB_SDP_DEBUG_COUNTER_CTRL                                                                0x4f3405
+#define regSYSHUB_SDP_DEBUG_COUNTER_CTRL_BASE_IDX                                                       3
+#define regNBIF_PWRBRK_IN_PAD_CNTL                                                                      0x4f3801
+#define regNBIF_PWRBRK_IN_PAD_CNTL_BASE_IDX                                                             3
+#define regNBIF_PWRBRK_OUT_PAD_CNTL                                                                     0x4f3802
+#define regNBIF_PWRBRK_OUT_PAD_CNTL_BASE_IDX                                                            3
+#define regSHUB_OOB_CRUSH_DUMP_CTRL                                                                     0x4f3803
+#define regSHUB_OOB_CRUSH_DUMP_CTRL_BASE_IDX                                                            3
+#define regSHUB_MP_ERREVENT_MASK_CTRL                                                                   0x4f3804
+#define regSHUB_MP_ERREVENT_MASK_CTRL_BASE_IDX                                                          3
+#define regSHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL                                                          0x4f3805
+#define regSHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL_BASE_IDX                                                 3
+#define regSHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL                                                   0x4f3806
+#define regSHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL_BASE_IDX                                          3
+
+
+// addressBlock: nbif0_nbif0_gdc_sec_misc_GDCSEC_MISC_DEC
+// base address: 0x1400000
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0V3_AXI                                                       0x4f2883
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0V3_AXI_BASE_IDX                                              3
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V3_AXI                                                   0x4f2884
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V3_AXI_BASE_IDX                                          3
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V3_AXI                                                   0x4f2885
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V3_AXI_BASE_IDX                                          3
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0B_AXI                                                        0x4f2886
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0B_AXI_BASE_IDX                                               3
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B_AXI                                                    0x4f2887
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B_AXI_BASE_IDX                                           3
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0B_AXI                                                    0x4f2888
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0B_AXI_BASE_IDX                                           3
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0V4_AXI                                                       0x4f2889
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0V4_AXI_BASE_IDX                                              3
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4_AXI                                                   0x4f288a
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4_AXI_BASE_IDX                                          3
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V4_AXI                                                   0x4f288b
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V4_AXI_BASE_IDX                                          3
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0V4B_AXI                                                      0x4f288c
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0V4B_AXI_BASE_IDX                                             3
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4B_AXI                                                  0x4f288d
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4B_AXI_BASE_IDX                                         3
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V4B_AXI                                                  0x4f288e
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V4B_AXI_BASE_IDX                                         3
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0V7_AXI                                                       0x4f288f
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0V7_AXI_BASE_IDX                                              3
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V7_AXI                                                   0x4f2890
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V7_AXI_BASE_IDX                                          3
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V7_AXI                                                   0x4f2891
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V7_AXI_BASE_IDX                                          3
+#define regGDCSOC_SEC_DFV_POISON_INJ_SHUB_CNDI_ORIG_SDP                                                 0x4f2898
+#define regGDCSOC_SEC_DFV_POISON_INJ_SHUB_CNDI_ORIG_SDP_BASE_IDX                                        3
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_SHUB_CNDI_ORIG_SDP                                             0x4f2899
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_SHUB_CNDI_ORIG_SDP_BASE_IDX                                    3
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_SHUB_CNDI_ORIG_SDP                                             0x4f289a
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_SHUB_CNDI_ORIG_SDP_BASE_IDX                                    3
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0B1_AXI                                                       0x4f28a7
+#define regGDCSOC_SEC_DFV_POISON_INJ_DMACK0B1_AXI_BASE_IDX                                              3
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B1_AXI                                                   0x4f28a8
+#define regGDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B1_AXI_BASE_IDX                                          3
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0B1_AXI                                                   0x4f28a9
+#define regGDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0B1_AXI_BASE_IDX                                          3
+#define regGDCSOC_SEC_RAS_POISON_DBUG_CTRL                                                              0x4f2c00
+#define regGDCSOC_SEC_RAS_POISON_DBUG_CTRL_BASE_IDX                                                     3
+#define regGDCSOC_SEC_RAS_PARITY_DBUG_CTRL                                                              0x4f2c01
+#define regGDCSOC_SEC_RAS_PARITY_DBUG_CTRL_BASE_IDX                                                     3
+#define regGDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL                                                         0x4f2c02
+#define regGDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL_BASE_IDX                                                3
+#define regGDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL                                                             0x4f2c03
+#define regGDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL_BASE_IDX                                                    3
+#define regGDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL                                                         0x4f2c04
+#define regGDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL_BASE_IDX                                                3
+#define regGDCSOC_SEC_RAS_RESPONSE_DROP_CTRL                                                            0x4f2c05
+#define regGDCSOC_SEC_RAS_RESPONSE_DROP_CTRL_BASE_IDX                                                   3
+#define regGDCSOC_SEC_RAS_LEAF0_RRESP_POISON_CTRL                                                       0x4f2c06
+#define regGDCSOC_SEC_RAS_LEAF0_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF0_RUSER_POISON_CTRL                                                       0x4f2c07
+#define regGDCSOC_SEC_RAS_LEAF0_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF1_RRESP_POISON_CTRL                                                       0x4f2c08
+#define regGDCSOC_SEC_RAS_LEAF1_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF1_RUSER_POISON_CTRL                                                       0x4f2c09
+#define regGDCSOC_SEC_RAS_LEAF1_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF2_RRESP_POISON_CTRL                                                       0x4f2c0a
+#define regGDCSOC_SEC_RAS_LEAF2_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF2_RUSER_POISON_CTRL                                                       0x4f2c0b
+#define regGDCSOC_SEC_RAS_LEAF2_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF3_RRESP_POISON_CTRL                                                       0x4f2c0c
+#define regGDCSOC_SEC_RAS_LEAF3_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF3_RUSER_POISON_CTRL                                                       0x4f2c0d
+#define regGDCSOC_SEC_RAS_LEAF3_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF4_RRESP_POISON_CTRL                                                       0x4f2c0e
+#define regGDCSOC_SEC_RAS_LEAF4_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF4_RUSER_POISON_CTRL                                                       0x4f2c0f
+#define regGDCSOC_SEC_RAS_LEAF4_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF5_RRESP_POISON_CTRL                                                       0x4f2c10
+#define regGDCSOC_SEC_RAS_LEAF5_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF5_RUSER_POISON_CTRL                                                       0x4f2c11
+#define regGDCSOC_SEC_RAS_LEAF5_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF6_RRESP_POISON_CTRL                                                       0x4f2c12
+#define regGDCSOC_SEC_RAS_LEAF6_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF6_RUSER_POISON_CTRL                                                       0x4f2c13
+#define regGDCSOC_SEC_RAS_LEAF6_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF7_RRESP_POISON_CTRL                                                       0x4f2c14
+#define regGDCSOC_SEC_RAS_LEAF7_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF7_RUSER_POISON_CTRL                                                       0x4f2c15
+#define regGDCSOC_SEC_RAS_LEAF7_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF8_RRESP_POISON_CTRL                                                       0x4f2c16
+#define regGDCSOC_SEC_RAS_LEAF8_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF8_RUSER_POISON_CTRL                                                       0x4f2c17
+#define regGDCSOC_SEC_RAS_LEAF8_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF9_RRESP_POISON_CTRL                                                       0x4f2c18
+#define regGDCSOC_SEC_RAS_LEAF9_RRESP_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF9_RUSER_POISON_CTRL                                                       0x4f2c19
+#define regGDCSOC_SEC_RAS_LEAF9_RUSER_POISON_CTRL_BASE_IDX                                              3
+#define regGDCSOC_SEC_RAS_LEAF10_RRESP_POISON_CTRL                                                      0x4f2c1a
+#define regGDCSOC_SEC_RAS_LEAF10_RRESP_POISON_CTRL_BASE_IDX                                             3
+#define regGDCSOC_SEC_RAS_LEAF10_RUSER_POISON_CTRL                                                      0x4f2c1b
+#define regGDCSOC_SEC_RAS_LEAF10_RUSER_POISON_CTRL_BASE_IDX                                             3
+#define regGDCSOC_SEC_RAS_LEAF11_RRESP_POISON_CTRL                                                      0x4f2c1c
+#define regGDCSOC_SEC_RAS_LEAF11_RRESP_POISON_CTRL_BASE_IDX                                             3
+#define regGDCSOC_SEC_RAS_LEAF11_RUSER_POISON_CTRL                                                      0x4f2c1d
+#define regGDCSOC_SEC_RAS_LEAF11_RUSER_POISON_CTRL_BASE_IDX                                             3
+#define regGDCSOC_SEC_RAS_LEAF12_RRESP_POISON_CTRL                                                      0x4f2c1e
+#define regGDCSOC_SEC_RAS_LEAF12_RRESP_POISON_CTRL_BASE_IDX                                             3
+#define regGDCSOC_SEC_RAS_LEAF12_RUSER_POISON_CTRL                                                      0x4f2c1f
+#define regGDCSOC_SEC_RAS_LEAF12_RUSER_POISON_CTRL_BASE_IDX                                             3
+#define regGDCSOC_SEC_RAS_LEAF13_RRESP_POISON_CTRL                                                      0x4f2c20
+#define regGDCSOC_SEC_RAS_LEAF13_RRESP_POISON_CTRL_BASE_IDX                                             3
+#define regGDCSOC_SEC_RAS_LEAF13_RUSER_POISON_CTRL                                                      0x4f2c21
+#define regGDCSOC_SEC_RAS_LEAF13_RUSER_POISON_CTRL_BASE_IDX                                             3
+
+
+// addressBlock: nbif0_nbif0_gdc_a2s_GDCA2S_DEC
+// base address: 0x1400000
+#define regA2S_CNTL_SW0                                                                                 0x4f0c40
+#define regA2S_CNTL_SW0_BASE_IDX                                                                        3
+#define regA2S_CNTL_SW1                                                                                 0x4f0c41
+#define regA2S_CNTL_SW1_BASE_IDX                                                                        3
+#define regA2S_CNTL_SW2                                                                                 0x4f0c42
+#define regA2S_CNTL_SW2_BASE_IDX                                                                        3
+#define regA2S_CNTL_SW3                                                                                 0x4f0c43
+#define regA2S_CNTL_SW3_BASE_IDX                                                                        3
+#define regA2S_CNTL_SW4                                                                                 0x4f0c44
+#define regA2S_CNTL_SW4_BASE_IDX                                                                        3
+#define regA2S_CNTL_SW5                                                                                 0x4f0c45
+#define regA2S_CNTL_SW5_BASE_IDX                                                                        3
+#define regA2S_CNTL_2_SW0                                                                               0x4f0c50
+#define regA2S_CNTL_2_SW0_BASE_IDX                                                                      3
+#define regA2S_CNTL_2_SW1                                                                               0x4f0c51
+#define regA2S_CNTL_2_SW1_BASE_IDX                                                                      3
+#define regA2S_CNTL_2_SW2                                                                               0x4f0c52
+#define regA2S_CNTL_2_SW2_BASE_IDX                                                                      3
+#define regA2S_CNTL_2_SW3                                                                               0x4f0c53
+#define regA2S_CNTL_2_SW3_BASE_IDX                                                                      3
+#define regA2S_CNTL_2_SW4                                                                               0x4f0c54
+#define regA2S_CNTL_2_SW4_BASE_IDX                                                                      3
+#define regA2S_CNTL_2_SW5                                                                               0x4f0c55
+#define regA2S_CNTL_2_SW5_BASE_IDX                                                                      3
+#define regA2S_WRSIZEFULL_SUPPORT_DISABLE                                                               0x4f0c5c
+#define regA2S_WRSIZEFULL_SUPPORT_DISABLE_BASE_IDX                                                      3
+#define regROUTING_TABLE_CNTL                                                                           0x4f0c60
+#define regROUTING_TABLE_CNTL_BASE_IDX                                                                  3
+#define regROUTING_TABLE_DATA0                                                                          0x4f0c61
+#define regROUTING_TABLE_DATA0_BASE_IDX                                                                 3
+#define regA2S_ARB_CONTROL_FOR_VC0                                                                      0x4f0c68
+#define regA2S_ARB_CONTROL_FOR_VC0_BASE_IDX                                                             3
+#define regA2S_ARB_CONTROL_FOR_VC1                                                                      0x4f0c69
+#define regA2S_ARB_CONTROL_FOR_VC1_BASE_IDX                                                             3
+#define regA2S_ARB_CONTROL_FOR_VC2                                                                      0x4f0c6a
+#define regA2S_ARB_CONTROL_FOR_VC2_BASE_IDX                                                             3
+#define regA2S_ARB_CONTROL_FOR_VC3                                                                      0x4f0c6b
+#define regA2S_ARB_CONTROL_FOR_VC3_BASE_IDX                                                             3
+#define regA2S_ARB_CONTROL_FOR_VC4                                                                      0x4f0c6c
+#define regA2S_ARB_CONTROL_FOR_VC4_BASE_IDX                                                             3
+#define regA2S_ARB_CONTROL_FOR_VC5                                                                      0x4f0c6d
+#define regA2S_ARB_CONTROL_FOR_VC5_BASE_IDX                                                             3
+#define regA2S_ARB_CONTROL_FOR_VC6                                                                      0x4f0c6e
+#define regA2S_ARB_CONTROL_FOR_VC6_BASE_IDX                                                             3
+#define regA2S_ARB_CONTROL_FOR_VC7                                                                      0x4f0c6f
+#define regA2S_ARB_CONTROL_FOR_VC7_BASE_IDX                                                             3
+#define regA2S_RW_ARB_CNTL                                                                              0x4f0c70
+#define regA2S_RW_ARB_CNTL_BASE_IDX                                                                     3
+#define regA2S_REQ_RSP_TAG_CNTL                                                                         0x4f0c71
+#define regA2S_REQ_RSP_TAG_CNTL_BASE_IDX                                                                3
+#define regA2S_MISC_CNTL                                                                                0x4f0c72
+#define regA2S_MISC_CNTL_BASE_IDX                                                                       3
+#define regA2S_TAG_ALLOC_0                                                                              0x4f0c74
+#define regA2S_TAG_ALLOC_0_BASE_IDX                                                                     3
+#define regA2S_TAG_ALLOC_1                                                                              0x4f0c75
+#define regA2S_TAG_ALLOC_1_BASE_IDX                                                                     3
+#define regA2S_TAG_ALLOC_2                                                                              0x4f0c76
+#define regA2S_TAG_ALLOC_2_BASE_IDX                                                                     3
+#define regA2S_TAG_ALLOC_3                                                                              0x4f0c77
+#define regA2S_TAG_ALLOC_3_BASE_IDX                                                                     3
+#define regSHUB_A2S_TAG_ALLOC_FOR_CHAIN_0                                                               0x4f0c78
+#define regSHUB_A2S_TAG_ALLOC_FOR_CHAIN_0_BASE_IDX                                                      3
+#define regSHUB_A2S_TAG_ALLOC_FOR_CHAIN_1                                                               0x4f0c79
+#define regSHUB_A2S_TAG_ALLOC_FOR_CHAIN_1_BASE_IDX                                                      3
+#define regGDC_SDP_PORT_CTRL                                                                            0x4f0c7c
+#define regGDC_SDP_PORT_CTRL_BASE_IDX                                                                   3
+#define regGDC_A2S_SDP_REQ_POOLCRED_ALLOC                                                               0x4f0c7d
+#define regGDC_A2S_SDP_REQ_POOLCRED_ALLOC_BASE_IDX                                                      3
+#define regGDC_A2S_SDP_DAT_POOLCRED_ALLOC                                                               0x4f0c7e
+#define regGDC_A2S_SDP_DAT_POOLCRED_ALLOC_BASE_IDX                                                      3
+#define regSHUB_PERF_COMMON_COUNTER                                                                     0x4f0c80
+#define regSHUB_PERF_COMMON_COUNTER_BASE_IDX                                                            3
+#define regSHUB_A2S_PERF_CNT_FSM                                                                        0x4f0c81
+#define regSHUB_A2S_PERF_CNT_FSM_BASE_IDX                                                               3
+#define regSHUB_A2S_PERF_CNT_RSP_UNITID                                                                 0x4f0c83
+#define regSHUB_A2S_PERF_CNT_RSP_UNITID_BASE_IDX                                                        3
+#define regSHUB_A2S_PERF_CNT_0_CTRL                                                                     0x4f0c84
+#define regSHUB_A2S_PERF_CNT_0_CTRL_BASE_IDX                                                            3
+#define regSHUB_A2S_PERF_CNT_1_CTRL                                                                     0x4f0c85
+#define regSHUB_A2S_PERF_CNT_1_CTRL_BASE_IDX                                                            3
+#define regSHUB_A2S_PERF_CNT_2_CTRL                                                                     0x4f0c86
+#define regSHUB_A2S_PERF_CNT_2_CTRL_BASE_IDX                                                            3
+#define regSHUB_A2S_PERF_CNT_3_CTRL                                                                     0x4f0c87
+#define regSHUB_A2S_PERF_CNT_3_CTRL_BASE_IDX                                                            3
+#define regSHUB_A2S_PERF_CNT_0_VALUE_L32BIT                                                             0x4f0c88
+#define regSHUB_A2S_PERF_CNT_0_VALUE_L32BIT_BASE_IDX                                                    3
+#define regSHUB_A2S_PERF_CNT_0_VALUE_H16BIT                                                             0x4f0c89
+#define regSHUB_A2S_PERF_CNT_0_VALUE_H16BIT_BASE_IDX                                                    3
+#define regSHUB_A2S_PERF_CNT_1_VALUE_L32BIT                                                             0x4f0c8a
+#define regSHUB_A2S_PERF_CNT_1_VALUE_L32BIT_BASE_IDX                                                    3
+#define regSHUB_A2S_PERF_CNT_1_VALUE_H16BIT                                                             0x4f0c8b
+#define regSHUB_A2S_PERF_CNT_1_VALUE_H16BIT_BASE_IDX                                                    3
+#define regSHUB_A2S_PERF_CNT_2_VALUE_L32BIT                                                             0x4f0c8c
+#define regSHUB_A2S_PERF_CNT_2_VALUE_L32BIT_BASE_IDX                                                    3
+#define regSHUB_A2S_PERF_CNT_2_VALUE_H16BIT                                                             0x4f0c8d
+#define regSHUB_A2S_PERF_CNT_2_VALUE_H16BIT_BASE_IDX                                                    3
+#define regSHUB_A2S_PERF_CNT_3_VALUE_L32BIT                                                             0x4f0c8e
+#define regSHUB_A2S_PERF_CNT_3_VALUE_L32BIT_BASE_IDX                                                    3
+#define regSHUB_A2S_PERF_CNT_3_VALUE_H16BIT                                                             0x4f0c8f
+#define regSHUB_A2S_PERF_CNT_3_VALUE_H16BIT_BASE_IDX                                                    3
+#define regSHUB_TO_ALL_PERF_COUNTER                                                                     0x4f0c90
+#define regSHUB_TO_ALL_PERF_COUNTER_BASE_IDX                                                            3
+#define regGDC_PERF_COUNTER_RESET                                                                       0x4f0c91
+#define regGDC_PERF_COUNTER_RESET_BASE_IDX                                                              3
+#define regSHUB_COM_COUNT_VALUE                                                                         0x4f0c92
+#define regSHUB_COM_COUNT_VALUE_BASE_IDX                                                                3
+#define regA2S_QUEUE_FIFO_ARB_CNTL                                                                      0x4f0c93
+#define regA2S_QUEUE_FIFO_ARB_CNTL_BASE_IDX                                                             3
+#define regWR_A2S_RSP_WRR_CNTL                                                                          0x4f0c94
+#define regWR_A2S_RSP_WRR_CNTL_BASE_IDX                                                                 3
+#define regRD_A2S_RSP_WRR_CNTL                                                                          0x4f0c95
+#define regRD_A2S_RSP_WRR_CNTL_BASE_IDX                                                                 3
+
+
+// addressBlock: nbif0_nbif0_syshub_mmreg_syshubdirect
+// base address: 0x1400000
+#define regSYSHUB_DS_CTRL_SOCCLK                                                                        0x4f3c00
+#define regSYSHUB_DS_CTRL_SOCCLK_BASE_IDX                                                               3
+#define regSYSHUB_DS_CTRL2_SOCCLK                                                                       0x4f3c01
+#define regSYSHUB_DS_CTRL2_SOCCLK_BASE_IDX                                                              3
+#define regSYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK                                                     0x4f3c02
+#define regSYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK_BASE_IDX                                            3
+#define regSYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK                                                        0x4f3c03
+#define regSYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK_BASE_IDX                                               3
+#define regSYSHUB_MGCG_CTRL_SOCCLK                                                                      0x4f3c08
+#define regSYSHUB_MGCG_CTRL_SOCCLK_BASE_IDX                                                             3
+#define regSYSHUB_SCRATCH_SOCCLK                                                                        0x4f3c0a
+#define regSYSHUB_SCRATCH_SOCCLK_BASE_IDX                                                               3
+#define regSYSHUB_HANG_CNTL_SOCCLK                                                                      0x4f3c0c
+#define regSYSHUB_HANG_CNTL_SOCCLK_BASE_IDX                                                             3
+#define regSYSHUB_SELECT_SOCCLK                                                                         0x4f3c0d
+#define regSYSHUB_SELECT_SOCCLK_BASE_IDX                                                                3
+#define regSYSHUB_DS_AW_READY_SOCCLK                                                                    0x4f3c11
+#define regSYSHUB_DS_AW_READY_SOCCLK_BASE_IDX                                                           3
+#define regSYSHUB_DS_AR_READY_SOCCLK                                                                    0x4f3c12
+#define regSYSHUB_DS_AR_READY_SOCCLK_BASE_IDX                                                           3
+#define regSYSHUB_DS_W_READY_SOCCLK                                                                     0x4f3c13
+#define regSYSHUB_DS_W_READY_SOCCLK_BASE_IDX                                                            3
+#define regSYSHUB_DS_R_READY_SOCCLK                                                                     0x4f3c14
+#define regSYSHUB_DS_R_READY_SOCCLK_BASE_IDX                                                            3
+#define regSYSHUB_DS_B_READY_SOCCLK                                                                     0x4f3c15
+#define regSYSHUB_DS_B_READY_SOCCLK_BASE_IDX                                                            3
+#define regSYSHUB_AXI_DEBUG_COUNTER_CNTL_SOCCLK                                                         0x4f3c16
+#define regSYSHUB_AXI_DEBUG_COUNTER_CNTL_SOCCLK_BASE_IDX                                                3
+#define regDMA_CLK0_SW0_SYSHUB_QOS_CNTL                                                                 0x4f3e00
+#define regDMA_CLK0_SW0_SYSHUB_QOS_CNTL_BASE_IDX                                                        3
+#define regDMA_CLK0_SW1_SYSHUB_QOS_CNTL                                                                 0x4f3e01
+#define regDMA_CLK0_SW1_SYSHUB_QOS_CNTL_BASE_IDX                                                        3
+#define regDMA_CLK0_SW0_CL0_CNTL                                                                        0x4f3e40
+#define regDMA_CLK0_SW0_CL0_CNTL_BASE_IDX                                                               3
+#define regDMA_CLK0_SW0_CL1_CNTL                                                                        0x4f3e41
+#define regDMA_CLK0_SW0_CL1_CNTL_BASE_IDX                                                               3
+#define regDMA_CLK0_SW1_CL0_CNTL                                                                        0x4f3e60
+#define regDMA_CLK0_SW1_CL0_CNTL_BASE_IDX                                                               3
+#define regDMA_CLK0_SW1_CL1_CNTL                                                                        0x4f3e61
+#define regDMA_CLK0_SW1_CL1_CNTL_BASE_IDX                                                               3
+#define regDMA_CLK0_SW2_CL0_CNTL                                                                        0x4f3e80
+#define regDMA_CLK0_SW2_CL0_CNTL_BASE_IDX                                                               3
+#define regDMA_CLK0_SW3_CL0_CNTL                                                                        0x4f3ea0
+#define regDMA_CLK0_SW3_CL0_CNTL_BASE_IDX                                                               3
+#define regDMA_CLK0_SW4_CL0_CNTL                                                                        0x4f3ec0
+#define regDMA_CLK0_SW4_CL0_CNTL_BASE_IDX                                                               3
+#define regDMA_CLK0_SW5_CL0_CNTL                                                                        0x4f3ee0
+#define regDMA_CLK0_SW5_CL0_CNTL_BASE_IDX                                                               3
+#define regNIC400_0_AMIB_0_FN_MOD_BM_ISS                                                                0x4f7c02
+#define regNIC400_0_AMIB_0_FN_MOD_BM_ISS_BASE_IDX                                                       3
+#define regNIC400_0_AMIB_0_FN_MOD                                                                       0x4f7c42
+#define regNIC400_0_AMIB_0_FN_MOD_BASE_IDX                                                              3
+#define regNIC400_0_ASIB_0_FN_MOD                                                                       0x4f8842
+#define regNIC400_0_ASIB_0_FN_MOD_BASE_IDX                                                              3
+#define regNIC400_0_ASIB_0_QOS_CNTL                                                                     0x4f8843
+#define regNIC400_0_ASIB_0_QOS_CNTL_BASE_IDX                                                            3
+#define regNIC400_0_ASIB_0_MAX_OT                                                                       0x4f8844
+#define regNIC400_0_ASIB_0_MAX_OT_BASE_IDX                                                              3
+#define regNIC400_0_ASIB_0_MAX_COMB_OT                                                                  0x4f8845
+#define regNIC400_0_ASIB_0_MAX_COMB_OT_BASE_IDX                                                         3
+#define regNIC400_0_ASIB_0_AW_P                                                                         0x4f8846
+#define regNIC400_0_ASIB_0_AW_P_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_0_AW_B                                                                         0x4f8847
+#define regNIC400_0_ASIB_0_AW_B_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_0_AW_R                                                                         0x4f8848
+#define regNIC400_0_ASIB_0_AW_R_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_0_AR_P                                                                         0x4f8849
+#define regNIC400_0_ASIB_0_AR_P_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_0_AR_B                                                                         0x4f884a
+#define regNIC400_0_ASIB_0_AR_B_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_0_AR_R                                                                         0x4f884b
+#define regNIC400_0_ASIB_0_AR_R_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_0_TARGET_FC                                                                    0x4f884c
+#define regNIC400_0_ASIB_0_TARGET_FC_BASE_IDX                                                           3
+#define regNIC400_0_ASIB_0_KI_FC                                                                        0x4f884d
+#define regNIC400_0_ASIB_0_KI_FC_BASE_IDX                                                               3
+#define regNIC400_0_ASIB_0_QOS_RANGE                                                                    0x4f884e
+#define regNIC400_0_ASIB_0_QOS_RANGE_BASE_IDX                                                           3
+#define regNIC400_0_ASIB_1_FN_MOD                                                                       0x4f8c42
+#define regNIC400_0_ASIB_1_FN_MOD_BASE_IDX                                                              3
+#define regNIC400_0_ASIB_1_QOS_CNTL                                                                     0x4f8c43
+#define regNIC400_0_ASIB_1_QOS_CNTL_BASE_IDX                                                            3
+#define regNIC400_0_ASIB_1_MAX_OT                                                                       0x4f8c44
+#define regNIC400_0_ASIB_1_MAX_OT_BASE_IDX                                                              3
+#define regNIC400_0_ASIB_1_MAX_COMB_OT                                                                  0x4f8c45
+#define regNIC400_0_ASIB_1_MAX_COMB_OT_BASE_IDX                                                         3
+#define regNIC400_0_ASIB_1_AW_P                                                                         0x4f8c46
+#define regNIC400_0_ASIB_1_AW_P_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_1_AW_B                                                                         0x4f8c47
+#define regNIC400_0_ASIB_1_AW_B_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_1_AW_R                                                                         0x4f8c48
+#define regNIC400_0_ASIB_1_AW_R_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_1_AR_P                                                                         0x4f8c49
+#define regNIC400_0_ASIB_1_AR_P_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_1_AR_B                                                                         0x4f8c4a
+#define regNIC400_0_ASIB_1_AR_B_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_1_AR_R                                                                         0x4f8c4b
+#define regNIC400_0_ASIB_1_AR_R_BASE_IDX                                                                3
+#define regNIC400_0_ASIB_1_TARGET_FC                                                                    0x4f8c4c
+#define regNIC400_0_ASIB_1_TARGET_FC_BASE_IDX                                                           3
+#define regNIC400_0_ASIB_1_KI_FC                                                                        0x4f8c4d
+#define regNIC400_0_ASIB_1_KI_FC_BASE_IDX                                                               3
+#define regNIC400_0_ASIB_1_QOS_RANGE                                                                    0x4f8c4e
+#define regNIC400_0_ASIB_1_QOS_RANGE_BASE_IDX                                                           3
+#define regNIC400_0_IB_0_FN_MOD                                                                         0x4fb842
+#define regNIC400_0_IB_0_FN_MOD_BASE_IDX                                                                3
+#define regNIC400_1_AMIB_0_FN_MOD_BM_ISS                                                                0x4fbc02
+#define regNIC400_1_AMIB_0_FN_MOD_BM_ISS_BASE_IDX                                                       3
+#define regNIC400_1_AMIB_0_FN_MOD                                                                       0x4fbc42
+#define regNIC400_1_AMIB_0_FN_MOD_BASE_IDX                                                              3
+#define regNIC400_1_ASIB_0_FN_MOD                                                                       0x4fc842
+#define regNIC400_1_ASIB_0_FN_MOD_BASE_IDX                                                              3
+#define regNIC400_1_ASIB_0_QOS_CNTL                                                                     0x4fc843
+#define regNIC400_1_ASIB_0_QOS_CNTL_BASE_IDX                                                            3
+#define regNIC400_1_ASIB_0_MAX_OT                                                                       0x4fc844
+#define regNIC400_1_ASIB_0_MAX_OT_BASE_IDX                                                              3
+#define regNIC400_1_ASIB_0_MAX_COMB_OT                                                                  0x4fc845
+#define regNIC400_1_ASIB_0_MAX_COMB_OT_BASE_IDX                                                         3
+#define regNIC400_1_ASIB_0_AW_P                                                                         0x4fc846
+#define regNIC400_1_ASIB_0_AW_P_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_0_AW_B                                                                         0x4fc847
+#define regNIC400_1_ASIB_0_AW_B_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_0_AW_R                                                                         0x4fc848
+#define regNIC400_1_ASIB_0_AW_R_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_0_AR_P                                                                         0x4fc849
+#define regNIC400_1_ASIB_0_AR_P_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_0_AR_B                                                                         0x4fc84a
+#define regNIC400_1_ASIB_0_AR_B_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_0_AR_R                                                                         0x4fc84b
+#define regNIC400_1_ASIB_0_AR_R_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_0_TARGET_FC                                                                    0x4fc84c
+#define regNIC400_1_ASIB_0_TARGET_FC_BASE_IDX                                                           3
+#define regNIC400_1_ASIB_0_KI_FC                                                                        0x4fc84d
+#define regNIC400_1_ASIB_0_KI_FC_BASE_IDX                                                               3
+#define regNIC400_1_ASIB_0_QOS_RANGE                                                                    0x4fc84e
+#define regNIC400_1_ASIB_0_QOS_RANGE_BASE_IDX                                                           3
+#define regNIC400_1_ASIB_1_FN_MOD                                                                       0x4fcc42
+#define regNIC400_1_ASIB_1_FN_MOD_BASE_IDX                                                              3
+#define regNIC400_1_ASIB_1_QOS_CNTL                                                                     0x4fcc43
+#define regNIC400_1_ASIB_1_QOS_CNTL_BASE_IDX                                                            3
+#define regNIC400_1_ASIB_1_MAX_OT                                                                       0x4fcc44
+#define regNIC400_1_ASIB_1_MAX_OT_BASE_IDX                                                              3
+#define regNIC400_1_ASIB_1_MAX_COMB_OT                                                                  0x4fcc45
+#define regNIC400_1_ASIB_1_MAX_COMB_OT_BASE_IDX                                                         3
+#define regNIC400_1_ASIB_1_AW_P                                                                         0x4fcc46
+#define regNIC400_1_ASIB_1_AW_P_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_1_AW_B                                                                         0x4fcc47
+#define regNIC400_1_ASIB_1_AW_B_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_1_AW_R                                                                         0x4fcc48
+#define regNIC400_1_ASIB_1_AW_R_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_1_AR_P                                                                         0x4fcc49
+#define regNIC400_1_ASIB_1_AR_P_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_1_AR_B                                                                         0x4fcc4a
+#define regNIC400_1_ASIB_1_AR_B_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_1_AR_R                                                                         0x4fcc4b
+#define regNIC400_1_ASIB_1_AR_R_BASE_IDX                                                                3
+#define regNIC400_1_ASIB_1_TARGET_FC                                                                    0x4fcc4c
+#define regNIC400_1_ASIB_1_TARGET_FC_BASE_IDX                                                           3
+#define regNIC400_1_ASIB_1_KI_FC                                                                        0x4fcc4d
+#define regNIC400_1_ASIB_1_KI_FC_BASE_IDX                                                               3
+#define regNIC400_1_ASIB_1_QOS_RANGE                                                                    0x4fcc4e
+#define regNIC400_1_ASIB_1_QOS_RANGE_BASE_IDX                                                           3
+#define regNIC400_1_IB_0_FN_MOD                                                                         0x4ff842
+#define regNIC400_1_IB_0_FN_MOD_BASE_IDX                                                                3
+
+
+// addressBlock: nbif0_nbif0_syshub_mmreg_syshubdec
+// base address: 0x0
+#define regSYSHUB_INDEX                                                                                 0x0008
+#define regSYSHUB_INDEX_BASE_IDX                                                                        0
+#define regSYSHUB_DATA                                                                                  0x0009
+#define regSYSHUB_DATA_BASE_IDX                                                                         0
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_rc_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_RC_VENDOR_ID                                                                    0x0000
+#define cfgBIF_CFG_DEV0_RC_DEVICE_ID                                                                    0x0002
+#define cfgBIF_CFG_DEV0_RC_COMMAND                                                                      0x0004
+#define cfgBIF_CFG_DEV0_RC_STATUS                                                                       0x0006
+#define cfgBIF_CFG_DEV0_RC_REVISION_ID                                                                  0x0008
+#define cfgBIF_CFG_DEV0_RC_PROG_INTERFACE                                                               0x0009
+#define cfgBIF_CFG_DEV0_RC_SUB_CLASS                                                                    0x000a
+#define cfgBIF_CFG_DEV0_RC_BASE_CLASS                                                                   0x000b
+#define cfgBIF_CFG_DEV0_RC_CACHE_LINE                                                                   0x000c
+#define cfgBIF_CFG_DEV0_RC_LATENCY                                                                      0x000d
+#define cfgBIF_CFG_DEV0_RC_HEADER                                                                       0x000e
+#define cfgBIF_CFG_DEV0_RC_BIST                                                                         0x000f
+#define cfgBIF_CFG_DEV0_RC_BASE_ADDR_1                                                                  0x0010
+#define cfgBIF_CFG_DEV0_RC_BASE_ADDR_2                                                                  0x0014
+#define cfgSUB_BUS_NUMBER_LATENCY                                                                       0x0018
+#define cfgIO_BASE_LIMIT                                                                                0x001c
+#define cfgSECONDARY_STATUS                                                                             0x001e
+#define cfgMEM_BASE_LIMIT                                                                               0x0020
+#define cfgPREF_BASE_LIMIT                                                                              0x0024
+#define cfgPREF_BASE_UPPER                                                                              0x0028
+#define cfgPREF_LIMIT_UPPER                                                                             0x002c
+#define cfgIO_BASE_LIMIT_HI                                                                             0x0030
+#define cfgBIF_CFG_DEV0_RC_CAP_PTR                                                                      0x0034
+#define cfgBIF_CFG_DEV0_RC_ROM_BASE_ADDR                                                                0x0038
+#define cfgBIF_CFG_DEV0_RC_INTERRUPT_LINE                                                               0x003c
+#define cfgBIF_CFG_DEV0_RC_INTERRUPT_PIN                                                                0x003d
+#define cfgIRQ_BRIDGE_CNTL                                                                              0x003e
+#define cfgEXT_BRIDGE_CNTL                                                                              0x0040
+#define cfgBIF_CFG_DEV0_RC_PMI_CAP_LIST                                                                 0x0050
+#define cfgBIF_CFG_DEV0_RC_PMI_CAP                                                                      0x0052
+#define cfgBIF_CFG_DEV0_RC_PMI_STATUS_CNTL                                                              0x0054
+#define cfgBIF_CFG_DEV0_RC_PCIE_CAP_LIST                                                                0x0058
+#define cfgBIF_CFG_DEV0_RC_PCIE_CAP                                                                     0x005a
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CAP                                                                   0x005c
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CNTL                                                                  0x0060
+#define cfgBIF_CFG_DEV0_RC_DEVICE_STATUS                                                                0x0062
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP                                                                     0x0064
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL                                                                    0x0068
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS                                                                  0x006a
+#define cfgSLOT_CAP                                                                                     0x006c
+#define cfgSLOT_CNTL                                                                                    0x0070
+#define cfgSLOT_STATUS                                                                                  0x0072
+#define cfgROOT_CNTL                                                                                    0x0074
+#define cfgROOT_CAP                                                                                     0x0076
+#define cfgROOT_STATUS                                                                                  0x0078
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CAP2                                                                  0x007c
+#define cfgBIF_CFG_DEV0_RC_DEVICE_CNTL2                                                                 0x0080
+#define cfgBIF_CFG_DEV0_RC_DEVICE_STATUS2                                                               0x0082
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP2                                                                    0x0084
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL2                                                                   0x0088
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS2                                                                 0x008a
+#define cfgSLOT_CAP2                                                                                    0x008c
+#define cfgSLOT_CNTL2                                                                                   0x0090
+#define cfgSLOT_STATUS2                                                                                 0x0092
+#define cfgBIF_CFG_DEV0_RC_MSI_CAP_LIST                                                                 0x00a0
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_CNTL                                                                 0x00a2
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO                                                              0x00a4
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI                                                              0x00a8
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_DATA                                                                 0x00a8
+#define cfgBIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA                                                             0x00aa
+#define cfgBIF_CFG_DEV0_RC_MSI_MSG_DATA_64                                                              0x00ac
+#define cfgBIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64                                                          0x00ae
+#define cfgSSID_CAP_LIST                                                                                0x00c0
+#define cfgSSID_CAP                                                                                     0x00c4
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                            0x0100
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR                                                     0x0104
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1                                                        0x0108
+#define cfgBIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2                                                        0x010c
+#define cfgBIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST                                                  0x0110
+#define cfgBIF_CFG_DEV0_RC_PCIE_LINK_CNTL3                                                              0x0114
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS                                                       0x0118
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL                                                0x011c
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL                                                0x011e
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL                                                0x0120
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL                                                0x0122
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL                                                0x0124
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL                                                0x0126
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL                                                0x0128
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL                                                0x012a
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL                                                0x012c
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL                                                0x012e
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL                                               0x0130
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL                                               0x0132
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL                                               0x0134
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL                                               0x0136
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL                                               0x0138
+#define cfgBIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL                                               0x013a
+#define cfgBIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST                                                   0x01b8
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP_16GT                                                                0x01bc
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL_16GT                                                               0x01c0
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS_16GT                                                             0x01c4
+#define cfgBIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT                                            0x01c8
+#define cfgBIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT                                             0x01cc
+#define cfgBIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT                                             0x01d0
+#define cfgBIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT                                                0x01d8
+#define cfgBIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT                                                0x01d9
+#define cfgBIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT                                                0x01da
+#define cfgBIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT                                                0x01db
+#define cfgBIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT                                                0x01dc
+#define cfgBIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT                                                0x01dd
+#define cfgBIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT                                                0x01de
+#define cfgBIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT                                                0x01df
+#define cfgBIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT                                                0x01e0
+#define cfgBIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT                                                0x01e1
+#define cfgBIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT                                               0x01e2
+#define cfgBIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT                                               0x01e3
+#define cfgBIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT                                               0x01e4
+#define cfgBIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT                                               0x01e5
+#define cfgBIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT                                               0x01e6
+#define cfgBIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT                                               0x01e7
+#define cfgBIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST                                                   0x01f8
+#define cfgBIF_CFG_DEV0_RC_LINK_CAP_32GT                                                                0x01fc
+#define cfgBIF_CFG_DEV0_RC_LINK_CNTL_32GT                                                               0x0200
+#define cfgBIF_CFG_DEV0_RC_LINK_STATUS_32GT                                                             0x0204
+#define cfgBIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1                                                   0x0208
+#define cfgBIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2                                                   0x020c
+#define cfgBIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1                                                0x0210
+#define cfgBIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2                                                0x0214
+#define cfgBIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT                                                0x0218
+#define cfgBIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT                                                0x0219
+#define cfgBIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT                                                0x021a
+#define cfgBIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT                                                0x021b
+#define cfgBIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT                                                0x021c
+#define cfgBIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT                                                0x021d
+#define cfgBIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT                                                0x021e
+#define cfgBIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT                                                0x021f
+#define cfgBIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT                                                0x0220
+#define cfgBIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT                                                0x0221
+#define cfgBIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT                                               0x0222
+#define cfgBIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT                                               0x0223
+#define cfgBIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT                                               0x0224
+#define cfgBIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT                                               0x0225
+#define cfgBIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT                                               0x0226
+#define cfgBIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT                                               0x0227
+#define cfgBIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST                                                         0x0238
+#define cfgBIF_CFG_DEV0_RC_AP_CAP                                                                       0x023c
+#define cfgBIF_CFG_DEV0_RC_AP_CNTL                                                                      0x0240
+#define cfgBIF_CFG_DEV0_RC_AP_DATA1                                                                     0x0244
+#define cfgBIF_CFG_DEV0_RC_AP_DATA2                                                                     0x0248
+#define cfgBIF_CFG_DEV0_RC_AP_SEL_EN_MASK                                                               0x024c
+#define cfgBIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST                                                  0x0250
+#define cfgBIF_CFG_DEV0_RC_MARGINING_PORT_CAP                                                           0x0254
+#define cfgBIF_CFG_DEV0_RC_MARGINING_PORT_STATUS                                                        0x0256
+#define cfgBIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL                                                   0x0258
+#define cfgBIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS                                                 0x025a
+#define cfgBIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL                                                   0x025c
+#define cfgBIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS                                                 0x025e
+#define cfgBIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL                                                   0x0260
+#define cfgBIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS                                                 0x0262
+#define cfgBIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL                                                   0x0264
+#define cfgBIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS                                                 0x0266
+#define cfgBIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL                                                   0x0268
+#define cfgBIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS                                                 0x026a
+#define cfgBIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL                                                   0x026c
+#define cfgBIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS                                                 0x026e
+#define cfgBIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL                                                   0x0270
+#define cfgBIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS                                                 0x0272
+#define cfgBIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL                                                   0x0274
+#define cfgBIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS                                                 0x0276
+#define cfgBIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL                                                   0x0278
+#define cfgBIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS                                                 0x027a
+#define cfgBIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL                                                   0x027c
+#define cfgBIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS                                                 0x027e
+#define cfgBIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL                                                  0x0280
+#define cfgBIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS                                                0x0282
+#define cfgBIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL                                                  0x0284
+#define cfgBIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS                                                0x0286
+#define cfgBIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL                                                  0x0288
+#define cfgBIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS                                                0x028a
+#define cfgBIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL                                                  0x028c
+#define cfgBIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS                                                0x028e
+#define cfgBIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL                                                  0x0290
+#define cfgBIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS                                                0x0292
+#define cfgBIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL                                                  0x0294
+#define cfgBIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS                                                0x0296
+#define cfgBIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST                                                      0x0400
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST                                                         0x0404
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1                                                        0x0408
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2                                                        0x040c
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL                                                            0x0410
+#define cfgBIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS                                                          0x0412
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP                                                        0x0414
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL                                                       0x0418
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS                                                     0x041e
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP                                                        0x0420
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL                                                       0x0424
+#define cfgBIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS                                                     0x042a
+#define cfgBIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST                                                        0x0430
+#define cfgBIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP                                                        0x0434
+#define cfgBIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS                                                     0x0438
+#define cfgBIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST                                                      0x0600
+#define cfgBIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                                0x0604
+#define cfgBIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS                                                       0x0608
+#define cfgBIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK                                                         0x060c
+#define cfgBIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY                                                     0x0610
+#define cfgBIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS                                                         0x0614
+#define cfgBIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK                                                           0x0618
+#define cfgBIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL                                                        0x061c
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG0                                                                0x0620
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG1                                                                0x0624
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG2                                                                0x0628
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG3                                                                0x062c
+#define cfgPCIE_ROOT_ERR_CMD                                                                            0x0630
+#define cfgPCIE_ROOT_ERR_STATUS                                                                         0x0634
+#define cfgPCIE_ERR_SRC_ID                                                                              0x0638
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG4                                                                0x063c
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0                                                         0x063c
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG5                                                                0x0640
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1                                                         0x0640
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG6                                                                0x0644
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2                                                         0x0644
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG7                                                                0x0648
+#define cfgBIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3                                                         0x0648
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG8                                                                0x064c
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG9                                                                0x0650
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG10                                                               0x0654
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG11                                                               0x0658
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG12                                                               0x065c
+#define cfgBIF_CFG_DEV0_RC_PCIE_HDR_LOG13                                                               0x0660
+#define cfgBIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST                                             0x0664
+#define cfgBIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1                                                      0x0668
+#define cfgBIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2                                                      0x066c
+#define cfgBIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST                                                        0x06dc
+#define cfgBIF_CFG_DEV0_RC_PCIE_ACS_CAP                                                                 0x06e0
+#define cfgBIF_CFG_DEV0_RC_PCIE_ACS_CNTL                                                                0x06e2
+#define cfgBIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST                                                        0x071c
+#define cfgBIF_CFG_DEV0_RC_RTR_DATA1                                                                    0x0720
+#define cfgBIF_CFG_DEV0_RC_RTR_DATA2                                                                    0x0724
+#define cfgBIF_CFG_DEV0_RC_IDE_ENH_CAP_LIST                                                             0x0e00
+#define cfgBIF_CFG_DEV0_RC_IDE_CAP                                                                      0x0e04
+#define cfgBIF_CFG_DEV0_RC_IDE_CNTL                                                                     0x0e08
+#define cfgBIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL                                                       0x0e0c
+#define cfgBIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_STATUS                                                     0x0e10
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CAP                                                   0x0e14
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL                                                  0x0e18
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_STATUS                                                0x0e1c
+#define cfgBIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG1                                                   0x0e20
+#define cfgBIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG2                                                   0x0e24
+#define cfgBIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG1                                                0x0e28
+#define cfgBIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG2                                                0x0e2c
+#define cfgBIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG3                                                0x0e30
+#define cfgBIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG1                                                0x0e34
+#define cfgBIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG2                                                0x0e38
+#define cfgBIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG3                                                0x0e3c
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CAP                                                   0x0e40
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL                                                  0x0e44
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_STATUS                                                0x0e48
+#define cfgBIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG1                                                   0x0e4c
+#define cfgBIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG2                                                   0x0e50
+#define cfgBIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG1                                                0x0e54
+#define cfgBIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG2                                                0x0e58
+#define cfgBIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG3                                                0x0e5c
+#define cfgBIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG1                                                0x0e60
+#define cfgBIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG2                                                0x0e64
+#define cfgBIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG3                                                0x0e68
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CAP                                                   0x0e6c
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL                                                  0x0e70
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_STATUS                                                0x0e74
+#define cfgBIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG1                                                   0x0e78
+#define cfgBIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG2                                                   0x0e7c
+#define cfgBIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG1                                                0x0e80
+#define cfgBIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG2                                                0x0e84
+#define cfgBIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG3                                                0x0e88
+#define cfgBIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG1                                                0x0e8c
+#define cfgBIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG2                                                0x0e90
+#define cfgBIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG3                                                0x0e94
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CAP                                                   0x0e98
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL                                                  0x0e9c
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_STATUS                                                0x0ea0
+#define cfgBIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG1                                                   0x0ea4
+#define cfgBIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG2                                                   0x0ea8
+#define cfgBIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG1                                                0x0eac
+#define cfgBIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG2                                                0x0eb0
+#define cfgBIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG3                                                0x0eb4
+#define cfgBIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG1                                                0x0eb8
+#define cfgBIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG2                                                0x0ebc
+#define cfgBIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG3                                                0x0ec0
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CAP                                                   0x0ec4
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL                                                  0x0ec8
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_STATUS                                                0x0ecc
+#define cfgBIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG1                                                   0x0ed0
+#define cfgBIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG2                                                   0x0ed4
+#define cfgBIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG1                                                0x0ed8
+#define cfgBIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG2                                                0x0edc
+#define cfgBIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG3                                                0x0ee0
+#define cfgBIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG1                                                0x0ee4
+#define cfgBIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG2                                                0x0ee8
+#define cfgBIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG3                                                0x0eec
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CAP                                                   0x0ef0
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL                                                  0x0ef4
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_STATUS                                                0x0ef8
+#define cfgBIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG1                                                   0x0efc
+#define cfgBIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG2                                                   0x0f00
+#define cfgBIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG1                                                0x0f04
+#define cfgBIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG2                                                0x0f08
+#define cfgBIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG3                                                0x0f0c
+#define cfgBIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG1                                                0x0f10
+#define cfgBIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG2                                                0x0f14
+#define cfgBIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG3                                                0x0f18
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CAP                                                   0x0f1c
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL                                                  0x0f20
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_STATUS                                                0x0f24
+#define cfgBIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG1                                                   0x0f28
+#define cfgBIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG2                                                   0x0f2c
+#define cfgBIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG1                                                0x0f30
+#define cfgBIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG2                                                0x0f34
+#define cfgBIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG3                                                0x0f38
+#define cfgBIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG1                                                0x0f3c
+#define cfgBIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG2                                                0x0f40
+#define cfgBIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG3                                                0x0f44
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CAP                                                   0x0f48
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL                                                  0x0f4c
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_STATUS                                                0x0f50
+#define cfgBIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG1                                                   0x0f54
+#define cfgBIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG2                                                   0x0f58
+#define cfgBIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG1                                                0x0f5c
+#define cfgBIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG2                                                0x0f60
+#define cfgBIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG3                                                0x0f64
+#define cfgBIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG1                                                0x0f68
+#define cfgBIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG2                                                0x0f6c
+#define cfgBIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG3                                                0x0f70
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CAP                                                   0x0f74
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL                                                  0x0f78
+#define cfgBIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_STATUS                                                0x0f7c
+#define cfgBIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG1                                                   0x0f80
+#define cfgBIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG2                                                   0x0f84
+#define cfgBIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG1                                                0x0f88
+#define cfgBIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG2                                                0x0f8c
+#define cfgBIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG3                                                0x0f90
+#define cfgBIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG1                                                0x0f94
+#define cfgBIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG2                                                0x0f98
+#define cfgBIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG3                                                0x0f9c
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf0_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF0_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF0_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF0_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF0_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF0_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF0_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF0_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF0_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST                                              0x02d8
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3                                                            0x02dc
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3                                                           0x02e0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3                                                         0x02e4
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST                                                0x0400
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST                                                0x0600
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0604
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS                                                 0x0608
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK                                                   0x060c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY                                               0x0610
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS                                                   0x0614
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK                                                     0x0618
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL                                                  0x061c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0                                                          0x0620
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1                                                          0x0624
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2                                                          0x0628
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3                                                          0x062c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG4                                                          0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0                                                   0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG5                                                          0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1                                                   0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG6                                                          0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2                                                   0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG7                                                          0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3                                                   0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG8                                                          0x064c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG9                                                          0x0650
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG10                                                         0x0654
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG11                                                         0x0658
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG12                                                         0x065c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG13                                                         0x0660
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST                                                  0x071c
+#define cfgBIF_CFG_DEV0_EPF0_VF0_RTR_DATA1                                                              0x0720
+#define cfgBIF_CFG_DEV0_EPF0_VF0_RTR_DATA2                                                              0x0724
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST                                                  0x0770
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP                                                           0x0774
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL                                                          0x0776
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST                                                  0x07ac
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP                                                           0x07b0
+#define cfgBIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL                                                          0x07b2
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf1_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF1_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF1_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF1_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF1_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF1_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF1_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF1_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF1_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST                                              0x02d8
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3                                                            0x02dc
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3                                                           0x02e0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3                                                         0x02e4
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST                                                0x0400
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST                                                0x0600
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0604
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS                                                 0x0608
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK                                                   0x060c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY                                               0x0610
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS                                                   0x0614
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK                                                     0x0618
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL                                                  0x061c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0                                                          0x0620
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1                                                          0x0624
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2                                                          0x0628
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3                                                          0x062c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG4                                                          0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0                                                   0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG5                                                          0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1                                                   0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG6                                                          0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2                                                   0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG7                                                          0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3                                                   0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG8                                                          0x064c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG9                                                          0x0650
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG10                                                         0x0654
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG11                                                         0x0658
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG12                                                         0x065c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG13                                                         0x0660
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST                                                  0x071c
+#define cfgBIF_CFG_DEV0_EPF0_VF1_RTR_DATA1                                                              0x0720
+#define cfgBIF_CFG_DEV0_EPF0_VF1_RTR_DATA2                                                              0x0724
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST                                                  0x0770
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP                                                           0x0774
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL                                                          0x0776
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST                                                  0x07ac
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP                                                           0x07b0
+#define cfgBIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL                                                          0x07b2
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf2_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF2_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF2_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF2_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF2_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF2_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF2_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF2_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF2_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST                                              0x02d8
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3                                                            0x02dc
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3                                                           0x02e0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3                                                         0x02e4
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST                                                0x0400
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST                                                0x0600
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0604
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS                                                 0x0608
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK                                                   0x060c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY                                               0x0610
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS                                                   0x0614
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK                                                     0x0618
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL                                                  0x061c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0                                                          0x0620
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1                                                          0x0624
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2                                                          0x0628
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3                                                          0x062c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG4                                                          0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0                                                   0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG5                                                          0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1                                                   0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG6                                                          0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2                                                   0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG7                                                          0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3                                                   0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG8                                                          0x064c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG9                                                          0x0650
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG10                                                         0x0654
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG11                                                         0x0658
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG12                                                         0x065c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG13                                                         0x0660
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST                                                  0x071c
+#define cfgBIF_CFG_DEV0_EPF0_VF2_RTR_DATA1                                                              0x0720
+#define cfgBIF_CFG_DEV0_EPF0_VF2_RTR_DATA2                                                              0x0724
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST                                                  0x0770
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP                                                           0x0774
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL                                                          0x0776
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST                                                  0x07ac
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP                                                           0x07b0
+#define cfgBIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL                                                          0x07b2
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf3_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF3_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF3_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF3_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF3_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF3_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF3_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF3_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF3_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST                                              0x02d8
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3                                                            0x02dc
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3                                                           0x02e0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3                                                         0x02e4
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST                                                0x0400
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST                                                0x0600
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0604
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS                                                 0x0608
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK                                                   0x060c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY                                               0x0610
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS                                                   0x0614
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK                                                     0x0618
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL                                                  0x061c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0                                                          0x0620
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1                                                          0x0624
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2                                                          0x0628
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3                                                          0x062c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG4                                                          0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0                                                   0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG5                                                          0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1                                                   0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG6                                                          0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2                                                   0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG7                                                          0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3                                                   0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG8                                                          0x064c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG9                                                          0x0650
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG10                                                         0x0654
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG11                                                         0x0658
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG12                                                         0x065c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG13                                                         0x0660
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST                                                  0x071c
+#define cfgBIF_CFG_DEV0_EPF0_VF3_RTR_DATA1                                                              0x0720
+#define cfgBIF_CFG_DEV0_EPF0_VF3_RTR_DATA2                                                              0x0724
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST                                                  0x0770
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP                                                           0x0774
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL                                                          0x0776
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST                                                  0x07ac
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP                                                           0x07b0
+#define cfgBIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL                                                          0x07b2
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf4_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF4_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF4_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF4_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF4_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF4_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF4_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF4_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF4_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST                                              0x02d8
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3                                                            0x02dc
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3                                                           0x02e0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3                                                         0x02e4
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST                                                0x0400
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST                                                0x0600
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0604
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS                                                 0x0608
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK                                                   0x060c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY                                               0x0610
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS                                                   0x0614
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK                                                     0x0618
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL                                                  0x061c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0                                                          0x0620
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1                                                          0x0624
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2                                                          0x0628
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3                                                          0x062c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG4                                                          0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0                                                   0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG5                                                          0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1                                                   0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG6                                                          0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2                                                   0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG7                                                          0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3                                                   0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG8                                                          0x064c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG9                                                          0x0650
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG10                                                         0x0654
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG11                                                         0x0658
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG12                                                         0x065c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG13                                                         0x0660
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST                                                  0x071c
+#define cfgBIF_CFG_DEV0_EPF0_VF4_RTR_DATA1                                                              0x0720
+#define cfgBIF_CFG_DEV0_EPF0_VF4_RTR_DATA2                                                              0x0724
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST                                                  0x0770
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP                                                           0x0774
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL                                                          0x0776
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST                                                  0x07ac
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP                                                           0x07b0
+#define cfgBIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL                                                          0x07b2
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf5_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF5_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF5_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF5_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF5_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF5_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF5_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF5_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF5_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST                                              0x02d8
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3                                                            0x02dc
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3                                                           0x02e0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3                                                         0x02e4
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST                                                0x0400
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST                                                0x0600
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0604
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS                                                 0x0608
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK                                                   0x060c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY                                               0x0610
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS                                                   0x0614
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK                                                     0x0618
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL                                                  0x061c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0                                                          0x0620
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1                                                          0x0624
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2                                                          0x0628
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3                                                          0x062c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG4                                                          0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0                                                   0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG5                                                          0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1                                                   0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG6                                                          0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2                                                   0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG7                                                          0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3                                                   0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG8                                                          0x064c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG9                                                          0x0650
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG10                                                         0x0654
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG11                                                         0x0658
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG12                                                         0x065c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG13                                                         0x0660
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST                                                  0x071c
+#define cfgBIF_CFG_DEV0_EPF0_VF5_RTR_DATA1                                                              0x0720
+#define cfgBIF_CFG_DEV0_EPF0_VF5_RTR_DATA2                                                              0x0724
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST                                                  0x0770
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP                                                           0x0774
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL                                                          0x0776
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST                                                  0x07ac
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP                                                           0x07b0
+#define cfgBIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL                                                          0x07b2
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf6_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF6_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF6_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF6_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF6_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF6_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF6_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF6_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF6_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST                                              0x02d8
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3                                                            0x02dc
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3                                                           0x02e0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3                                                         0x02e4
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST                                                0x0400
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST                                                0x0600
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0604
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS                                                 0x0608
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK                                                   0x060c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY                                               0x0610
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS                                                   0x0614
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK                                                     0x0618
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL                                                  0x061c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0                                                          0x0620
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1                                                          0x0624
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2                                                          0x0628
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3                                                          0x062c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG4                                                          0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0                                                   0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG5                                                          0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1                                                   0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG6                                                          0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2                                                   0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG7                                                          0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3                                                   0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG8                                                          0x064c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG9                                                          0x0650
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG10                                                         0x0654
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG11                                                         0x0658
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG12                                                         0x065c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG13                                                         0x0660
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST                                                  0x071c
+#define cfgBIF_CFG_DEV0_EPF0_VF6_RTR_DATA1                                                              0x0720
+#define cfgBIF_CFG_DEV0_EPF0_VF6_RTR_DATA2                                                              0x0724
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST                                                  0x0770
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP                                                           0x0774
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL                                                          0x0776
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST                                                  0x07ac
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP                                                           0x07b0
+#define cfgBIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL                                                          0x07b2
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf7_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF0_VF7_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF0_VF7_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF0_VF7_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF0_VF7_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF0_VF7_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF0_VF7_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF0_VF7_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF0_VF7_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST                                              0x02d8
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3                                                            0x02dc
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3                                                           0x02e0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3                                                         0x02e4
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST                                                0x0400
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST                                                0x0600
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0604
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS                                                 0x0608
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK                                                   0x060c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY                                               0x0610
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS                                                   0x0614
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK                                                     0x0618
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL                                                  0x061c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0                                                          0x0620
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1                                                          0x0624
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2                                                          0x0628
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3                                                          0x062c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG4                                                          0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0                                                   0x063c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG5                                                          0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1                                                   0x0640
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG6                                                          0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2                                                   0x0644
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG7                                                          0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3                                                   0x0648
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG8                                                          0x064c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG9                                                          0x0650
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG10                                                         0x0654
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG11                                                         0x0658
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG12                                                         0x065c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG13                                                         0x0660
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST                                                  0x071c
+#define cfgBIF_CFG_DEV0_EPF0_VF7_RTR_DATA1                                                              0x0720
+#define cfgBIF_CFG_DEV0_EPF0_VF7_RTR_DATA2                                                              0x0724
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST                                                  0x0770
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP                                                           0x0774
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL                                                          0x0776
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST                                                  0x07ac
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP                                                           0x07b0
+#define cfgBIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL                                                          0x07b2
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf1_vf0_bifcfgdecp
+// base address: 0x0
+#define cfgBIF_CFG_DEV0_EPF1_VF0_VENDOR_ID                                                              0x0000
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_ID                                                              0x0002
+#define cfgBIF_CFG_DEV0_EPF1_VF0_COMMAND                                                                0x0004
+#define cfgBIF_CFG_DEV0_EPF1_VF0_STATUS                                                                 0x0006
+#define cfgBIF_CFG_DEV0_EPF1_VF0_REVISION_ID                                                            0x0008
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PROG_INTERFACE                                                         0x0009
+#define cfgBIF_CFG_DEV0_EPF1_VF0_SUB_CLASS                                                              0x000a
+#define cfgBIF_CFG_DEV0_EPF1_VF0_BASE_CLASS                                                             0x000b
+#define cfgBIF_CFG_DEV0_EPF1_VF0_CACHE_LINE                                                             0x000c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_LATENCY                                                                0x000d
+#define cfgBIF_CFG_DEV0_EPF1_VF0_HEADER                                                                 0x000e
+#define cfgBIF_CFG_DEV0_EPF1_VF0_BIST                                                                   0x000f
+#define cfgBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_1                                                            0x0010
+#define cfgBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_2                                                            0x0014
+#define cfgBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_3                                                            0x0018
+#define cfgBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_4                                                            0x001c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_5                                                            0x0020
+#define cfgBIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_6                                                            0x0024
+#define cfgBIF_CFG_DEV0_EPF1_VF0_CARDBUS_CIS_PTR                                                        0x0028
+#define cfgBIF_CFG_DEV0_EPF1_VF0_ADAPTER_ID                                                             0x002c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR                                                          0x0030
+#define cfgBIF_CFG_DEV0_EPF1_VF0_CAP_PTR                                                                0x0034
+#define cfgBIF_CFG_DEV0_EPF1_VF0_INTERRUPT_LINE                                                         0x003c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_INTERRUPT_PIN                                                          0x003d
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MIN_GRANT                                                              0x003e
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MAX_LATENCY                                                            0x003f
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_CAP_LIST                                                          0x0064
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_CAP                                                               0x0066
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP                                                             0x0068
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL                                                            0x006c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS                                                          0x006e
+#define cfgBIF_CFG_DEV0_EPF1_VF0_LINK_CAP                                                               0x0070
+#define cfgBIF_CFG_DEV0_EPF1_VF0_LINK_CNTL                                                              0x0074
+#define cfgBIF_CFG_DEV0_EPF1_VF0_LINK_STATUS                                                            0x0076
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2                                                            0x0088
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2                                                           0x008c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS2                                                         0x008e
+#define cfgBIF_CFG_DEV0_EPF1_VF0_LINK_CAP2                                                              0x0090
+#define cfgBIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2                                                             0x0094
+#define cfgBIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2                                                           0x0096
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_CAP_LIST                                                           0x00a0
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL                                                           0x00a2
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_LO                                                        0x00a4
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_HI                                                        0x00a8
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA                                                           0x00a8
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA                                                       0x00aa
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_MASK                                                               0x00ac
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA_64                                                        0x00ac
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA_64                                                    0x00ae
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_MASK_64                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_PENDING                                                            0x00b0
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSI_PENDING_64                                                         0x00b4
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSIX_CAP_LIST                                                          0x00c0
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL                                                          0x00c2
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSIX_TABLE                                                             0x00c4
+#define cfgBIF_CFG_DEV0_EPF1_VF0_MSIX_PBA                                                               0x00c8
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST                                      0x0100
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR                                               0x0104
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC1                                                  0x0108
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC2                                                  0x010c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST                                              0x02d8
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3                                                            0x02dc
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3                                                           0x02e0
+#define cfgBIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3                                                         0x02e4
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST                                                0x0400
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST                                                0x0600
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST                                          0x0604
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS                                                 0x0608
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK                                                   0x060c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY                                               0x0610
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS                                                   0x0614
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK                                                     0x0618
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL                                                  0x061c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG0                                                          0x0620
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG1                                                          0x0624
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG2                                                          0x0628
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG3                                                          0x062c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG4                                                          0x063c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG0                                                   0x063c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG5                                                          0x0640
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG1                                                   0x0640
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG6                                                          0x0644
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG2                                                   0x0644
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG7                                                          0x0648
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG3                                                   0x0648
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG8                                                          0x064c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG9                                                          0x0650
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG10                                                         0x0654
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG11                                                         0x0658
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG12                                                         0x065c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG13                                                         0x0660
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST                                                  0x071c
+#define cfgBIF_CFG_DEV0_EPF1_VF0_RTR_DATA1                                                              0x0720
+#define cfgBIF_CFG_DEV0_EPF1_VF0_RTR_DATA2                                                              0x0724
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST                                                  0x0770
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP                                                           0x0774
+#define cfgBIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL                                                          0x0776
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_3_2_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_3_2_sh_mask.h
new file mode 100644
index 0000000000000..9f8f8b3eba020
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/nbio/nbio_6_3_2_sh_mask.h
@@ -0,0 +1,49475 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ */
+#ifndef _nbio_6_3_2_SH_MASK_HEADER
+#define _nbio_6_3_2_SH_MASK_HEADER
+
+
+// addressBlock: nbif0_nbif0_bif_bx_SYSDEC
+//BIF_BX0_SYSHUB_INDEX_OVLP
+#define BIF_BX0_SYSHUB_INDEX_OVLP__SYSHUB_OFFSET__SHIFT                                                       0x0
+#define BIF_BX0_SYSHUB_INDEX_OVLP__SYSHUB_OFFSET_MASK                                                         0x003FFFFFL
+//BIF_BX0_SYSHUB_DATA_OVLP
+#define BIF_BX0_SYSHUB_DATA_OVLP__SYSHUB_DATA__SHIFT                                                          0x0
+#define BIF_BX0_SYSHUB_DATA_OVLP__SYSHUB_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX0_SYSHUB_INDEX_HI_OVLP
+#define BIF_BX0_SYSHUB_INDEX_HI_OVLP__SYSHUB_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX0_SYSHUB_INDEX_HI_OVLP__SYSHUB_OFFSET_HI_MASK                                                   0x0000FFFFL
+//BIF_BX0_PCIE_INDEX
+#define BIF_BX0_PCIE_INDEX__PCIE_INDEX__SHIFT                                                                 0x0
+#define BIF_BX0_PCIE_INDEX__PCIE_INDEX_MASK                                                                   0xFFFFFFFFL
+//BIF_BX0_PCIE_DATA
+#define BIF_BX0_PCIE_DATA__PCIE_DATA__SHIFT                                                                   0x0
+#define BIF_BX0_PCIE_DATA__PCIE_DATA_MASK                                                                     0xFFFFFFFFL
+//BIF_BX0_PCIE_INDEX2
+#define BIF_BX0_PCIE_INDEX2__PCIE_INDEX2__SHIFT                                                               0x0
+#define BIF_BX0_PCIE_INDEX2__PCIE_INDEX2_MASK                                                                 0xFFFFFFFFL
+//BIF_BX0_PCIE_DATA2
+#define BIF_BX0_PCIE_DATA2__PCIE_DATA2__SHIFT                                                                 0x0
+#define BIF_BX0_PCIE_DATA2__PCIE_DATA2_MASK                                                                   0xFFFFFFFFL
+//BIF_BX0_PCIE_INDEX_HI
+#define BIF_BX0_PCIE_INDEX_HI__PCIE_INDEX_HI__SHIFT                                                           0x0
+#define BIF_BX0_PCIE_INDEX_HI__PCIE_INDEX_HI_MASK                                                             0x0000FFFFL
+//BIF_BX0_PCIE_INDEX2_HI
+#define BIF_BX0_PCIE_INDEX2_HI__PCIE_INDEX2_HI__SHIFT                                                         0x0
+#define BIF_BX0_PCIE_INDEX2_HI__PCIE_INDEX2_HI_MASK                                                           0x0000FFFFL
+//BIF_BX0_SBIOS_SCRATCH_0
+#define BIF_BX0_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_1
+#define BIF_BX0_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_2
+#define BIF_BX0_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_3
+#define BIF_BX0_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_0
+#define BIF_BX0_BIOS_SCRATCH_0__BIOS_SCRATCH_0__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_0__BIOS_SCRATCH_0_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_1
+#define BIF_BX0_BIOS_SCRATCH_1__BIOS_SCRATCH_1__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_1__BIOS_SCRATCH_1_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_2
+#define BIF_BX0_BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_3
+#define BIF_BX0_BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_4
+#define BIF_BX0_BIOS_SCRATCH_4__BIOS_SCRATCH_4__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_4__BIOS_SCRATCH_4_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_5
+#define BIF_BX0_BIOS_SCRATCH_5__BIOS_SCRATCH_5__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_5__BIOS_SCRATCH_5_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_6
+#define BIF_BX0_BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_7
+#define BIF_BX0_BIOS_SCRATCH_7__BIOS_SCRATCH_7__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_7__BIOS_SCRATCH_7_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_8
+#define BIF_BX0_BIOS_SCRATCH_8__BIOS_SCRATCH_8__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_8__BIOS_SCRATCH_8_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_9
+#define BIF_BX0_BIOS_SCRATCH_9__BIOS_SCRATCH_9__SHIFT                                                         0x0
+#define BIF_BX0_BIOS_SCRATCH_9__BIOS_SCRATCH_9_MASK                                                           0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_10
+#define BIF_BX0_BIOS_SCRATCH_10__BIOS_SCRATCH_10__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_10__BIOS_SCRATCH_10_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_11
+#define BIF_BX0_BIOS_SCRATCH_11__BIOS_SCRATCH_11__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_11__BIOS_SCRATCH_11_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_12
+#define BIF_BX0_BIOS_SCRATCH_12__BIOS_SCRATCH_12__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_12__BIOS_SCRATCH_12_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_13
+#define BIF_BX0_BIOS_SCRATCH_13__BIOS_SCRATCH_13__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_13__BIOS_SCRATCH_13_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_14
+#define BIF_BX0_BIOS_SCRATCH_14__BIOS_SCRATCH_14__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_14__BIOS_SCRATCH_14_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIOS_SCRATCH_15
+#define BIF_BX0_BIOS_SCRATCH_15__BIOS_SCRATCH_15__SHIFT                                                       0x0
+#define BIF_BX0_BIOS_SCRATCH_15__BIOS_SCRATCH_15_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_BIF_RLC_INTR_CNTL
+//BIF_BX0_BIF_VCE_INTR_CNTL
+//BIF_BX0_BIF_UVD_INTR_CNTL
+//BIF_BX0_BIF_Engine_INTR_CNTL
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_CMD_COMPLETE__SHIFT                                              0x0
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_HANG_SELF_RECOVERED__SHIFT                                       0x1
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_HANG_NEED_FLR__SHIFT                                             0x2
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_VM_BUSY_TRANSITION__SHIFT                                        0x3
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_INST_SEL__SHIFT                                                  0x1b
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_CMD_COMPLETE_MASK                                                0x00000001L
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_HANG_SELF_RECOVERED_MASK                                         0x00000002L
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_HANG_NEED_FLR_MASK                                               0x00000004L
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_VM_BUSY_TRANSITION_MASK                                          0x00000008L
+#define BIF_BX0_BIF_Engine_INTR_CNTL__Engine_INST_SEL_MASK                                                    0xF8000000L
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR1
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR2
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR3
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR4
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR5
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR6
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ADDR7
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7_MASK                                                         0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7__SHIFT                                           0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7_MASK                                             0x000FFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_CNTL
+#define BIF_BX0_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE__SHIFT                                                       0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE_MASK                                                         0x000000FFL
+//BIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL
+#define BIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL__SHIFT                                                 0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL_MASK                                                   0xFFFFFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_ONE_CPL
+#define BIF_BX0_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL__SHIFT                                                   0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL
+#define BIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL__SHIFT                                 0x0
+#define BIF_BX0_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL_MASK                                   0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_0
+#define BIF_BX0_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_1
+#define BIF_BX0_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_2
+#define BIF_BX0_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_3
+#define BIF_BX0_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_4
+#define BIF_BX0_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_5
+#define BIF_BX0_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_6
+#define BIF_BX0_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_7
+#define BIF_BX0_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_8
+#define BIF_BX0_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_9
+#define BIF_BX0_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9__SHIFT                                                     0x0
+#define BIF_BX0_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_10
+#define BIF_BX0_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_11
+#define BIF_BX0_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_12
+#define BIF_BX0_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_13
+#define BIF_BX0_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_14
+#define BIF_BX0_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_DRIVER_SCRATCH_15
+#define BIF_BX0_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15__SHIFT                                                   0x0
+#define BIF_BX0_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15_MASK                                                     0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_0
+#define BIF_BX0_FW_SCRATCH_0__FW_SCRATCH_0__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_0__FW_SCRATCH_0_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_1
+#define BIF_BX0_FW_SCRATCH_1__FW_SCRATCH_1__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_1__FW_SCRATCH_1_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_2
+#define BIF_BX0_FW_SCRATCH_2__FW_SCRATCH_2__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_2__FW_SCRATCH_2_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_3
+#define BIF_BX0_FW_SCRATCH_3__FW_SCRATCH_3__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_3__FW_SCRATCH_3_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_4
+#define BIF_BX0_FW_SCRATCH_4__FW_SCRATCH_4__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_4__FW_SCRATCH_4_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_5
+#define BIF_BX0_FW_SCRATCH_5__FW_SCRATCH_5__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_5__FW_SCRATCH_5_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_6
+#define BIF_BX0_FW_SCRATCH_6__FW_SCRATCH_6__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_6__FW_SCRATCH_6_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_7
+#define BIF_BX0_FW_SCRATCH_7__FW_SCRATCH_7__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_7__FW_SCRATCH_7_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_8
+#define BIF_BX0_FW_SCRATCH_8__FW_SCRATCH_8__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_8__FW_SCRATCH_8_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_9
+#define BIF_BX0_FW_SCRATCH_9__FW_SCRATCH_9__SHIFT                                                             0x0
+#define BIF_BX0_FW_SCRATCH_9__FW_SCRATCH_9_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_10
+#define BIF_BX0_FW_SCRATCH_10__FW_SCRATCH_10__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_10__FW_SCRATCH_10_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_11
+#define BIF_BX0_FW_SCRATCH_11__FW_SCRATCH_11__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_11__FW_SCRATCH_11_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_12
+#define BIF_BX0_FW_SCRATCH_12__FW_SCRATCH_12__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_12__FW_SCRATCH_12_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_13
+#define BIF_BX0_FW_SCRATCH_13__FW_SCRATCH_13__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_13__FW_SCRATCH_13_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_14
+#define BIF_BX0_FW_SCRATCH_14__FW_SCRATCH_14__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_14__FW_SCRATCH_14_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_FW_SCRATCH_15
+#define BIF_BX0_FW_SCRATCH_15__FW_SCRATCH_15__SHIFT                                                           0x0
+#define BIF_BX0_FW_SCRATCH_15__FW_SCRATCH_15_MASK                                                             0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_4
+#define BIF_BX0_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_5
+#define BIF_BX0_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_6
+#define BIF_BX0_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_7
+#define BIF_BX0_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_8
+#define BIF_BX0_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_9
+#define BIF_BX0_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9__SHIFT                                                       0x0
+#define BIF_BX0_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9_MASK                                                         0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_10
+#define BIF_BX0_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_11
+#define BIF_BX0_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_12
+#define BIF_BX0_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_13
+#define BIF_BX0_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_14
+#define BIF_BX0_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14_MASK                                                       0xFFFFFFFFL
+//BIF_BX0_SBIOS_SCRATCH_15
+#define BIF_BX0_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15__SHIFT                                                     0x0
+#define BIF_BX0_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_dwn_dev0_BIFDEC1
+//RCC_DWN_DEV0_0_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_0_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_0_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_0_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_0_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_0_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_0_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_0_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_0_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                 0x5
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+#define RCC_DWN_DEV0_0_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                   0x00000020L
+//RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_0_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: nbif0_nbif0_rcc_dwnp_dev0_BIFDEC1
+//RCC_DWNP_DEV0_0_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_0_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_0_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_0_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                           0x4
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+#define RCC_DWNP_DEV0_0_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                             0x00000010L
+//RCC_DWNP_DEV0_0_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_0_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_0_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_0_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_0_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_0_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_ep_dev0_BIFDEC1
+//RCC_EP_DEV0_0_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_0_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_0_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_0_EP_PCIE_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_0_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_0_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_0_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_0_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_0_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_0_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_0_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                  0x5
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+#define RCC_EP_DEV0_0_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                    0x00000020L
+//RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_0_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_0_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_0_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_0_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_0_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_0_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_0_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_0_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_0_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_0_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_0_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_0_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_0_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_0_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                          0x4
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+#define RCC_EP_DEV0_0_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                            0x00000010L
+//RCC_EP_DEV0_0_EP_PCIE_DEVICE_CNTL3
+#define RCC_EP_DEV0_0_EP_PCIE_DEVICE_CNTL3__SHADOW_F0_DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT          0x2
+#define RCC_EP_DEV0_0_EP_PCIE_DEVICE_CNTL3__SHADOW_F0_DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK            0x00000004L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_pf_SYSPFVFDEC
+//BIF_BX_PF0_MM_INDEX
+#define BIF_BX_PF0_MM_INDEX__MM_OFFSET__SHIFT                                                                 0x0
+#define BIF_BX_PF0_MM_INDEX__MM_APER__SHIFT                                                                   0x1f
+#define BIF_BX_PF0_MM_INDEX__MM_OFFSET_MASK                                                                   0x7FFFFFFFL
+#define BIF_BX_PF0_MM_INDEX__MM_APER_MASK                                                                     0x80000000L
+//BIF_BX_PF0_MM_DATA
+#define BIF_BX_PF0_MM_DATA__MM_DATA__SHIFT                                                                    0x0
+#define BIF_BX_PF0_MM_DATA__MM_DATA_MASK                                                                      0xFFFFFFFFL
+//BIF_BX_PF0_MM_INDEX_HI
+#define BIF_BX_PF0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                           0x0
+#define BIF_BX_PF0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                             0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_INDEX
+#define BIF_BX_PF0_RSMU_INDEX__RSMU_INDEX__SHIFT                                                              0x0
+#define BIF_BX_PF0_RSMU_INDEX__RSMU_INDEX_MASK                                                                0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_DATA
+#define BIF_BX_PF0_RSMU_DATA__RSMU_DATA__SHIFT                                                                0x0
+#define BIF_BX_PF0_RSMU_DATA__RSMU_DATA_MASK                                                                  0xFFFFFFFFL
+//BIF_BX_PF0_RSMU_INDEX_HI
+#define BIF_BX_PF0_RSMU_INDEX_HI__RSMU_INDEX_HI__SHIFT                                                        0x0
+#define BIF_BX_PF0_RSMU_INDEX_HI__RSMU_INDEX_HI_MASK                                                          0x0000FFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_BIFDEC1
+//BIF_BX0_CC_BIF_BX_STRAP0
+#define BIF_BX0_CC_BIF_BX_STRAP0__STRAP_RESERVED__SHIFT                                                       0x19
+#define BIF_BX0_CC_BIF_BX_STRAP0__STRAP_RESERVED_MASK                                                         0xFE000000L
+//BIF_BX0_CC_BIF_BX_PINSTRAP0
+//BIF_BX0_BIF_MM_INDACCESS_CNTL
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__WRITE_DIS__SHIFT                                                       0x0
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS__SHIFT                                                0x1
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__WRITE_DIS_MASK                                                         0x00000001L
+#define BIF_BX0_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS_MASK                                                  0x00000002L
+//BIF_BX0_BUS_CNTL
+#define BIF_BX0_BUS_CNTL__VGA_REG_COHERENCY_DIS__SHIFT                                                        0x6
+#define BIF_BX0_BUS_CNTL__VGA_MEM_COHERENCY_DIS__SHIFT                                                        0x7
+#define BIF_BX0_BUS_CNTL__SET_AZ_TC__SHIFT                                                                    0xa
+#define BIF_BX0_BUS_CNTL__SET_MC_TC__SHIFT                                                                    0xd
+#define BIF_BX0_BUS_CNTL__ZERO_BE_WR_EN__SHIFT                                                                0x10
+#define BIF_BX0_BUS_CNTL__ZERO_BE_RD_EN__SHIFT                                                                0x11
+#define BIF_BX0_BUS_CNTL__RD_STALL_IO_WR__SHIFT                                                               0x12
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS__SHIFT                                              0x18
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS__SHIFT                                          0x19
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS__SHIFT                                         0x1a
+#define BIF_BX0_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS__SHIFT                                        0x1b
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS__SHIFT                                          0x1c
+#define BIF_BX0_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN__SHIFT                                                     0x1d
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_WR_EN__SHIFT                                                          0x1e
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_RD_EN__SHIFT                                                          0x1f
+#define BIF_BX0_BUS_CNTL__VGA_REG_COHERENCY_DIS_MASK                                                          0x00000040L
+#define BIF_BX0_BUS_CNTL__VGA_MEM_COHERENCY_DIS_MASK                                                          0x00000080L
+#define BIF_BX0_BUS_CNTL__SET_AZ_TC_MASK                                                                      0x00001C00L
+#define BIF_BX0_BUS_CNTL__SET_MC_TC_MASK                                                                      0x0000E000L
+#define BIF_BX0_BUS_CNTL__ZERO_BE_WR_EN_MASK                                                                  0x00010000L
+#define BIF_BX0_BUS_CNTL__ZERO_BE_RD_EN_MASK                                                                  0x00020000L
+#define BIF_BX0_BUS_CNTL__RD_STALL_IO_WR_MASK                                                                 0x00040000L
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS_MASK                                                0x01000000L
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS_MASK                                            0x02000000L
+#define BIF_BX0_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS_MASK                                           0x04000000L
+#define BIF_BX0_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS_MASK                                          0x08000000L
+#define BIF_BX0_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS_MASK                                            0x10000000L
+#define BIF_BX0_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN_MASK                                                       0x20000000L
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_WR_EN_MASK                                                            0x40000000L
+#define BIF_BX0_BUS_CNTL__VGAFB_ZERO_BE_RD_EN_MASK                                                            0x80000000L
+//BIF_BX0_BIF_SCRATCH0
+#define BIF_BX0_BIF_SCRATCH0__BIF_SCRATCH0__SHIFT                                                             0x0
+#define BIF_BX0_BIF_SCRATCH0__BIF_SCRATCH0_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_BIF_SCRATCH1
+#define BIF_BX0_BIF_SCRATCH1__BIF_SCRATCH1__SHIFT                                                             0x0
+#define BIF_BX0_BIF_SCRATCH1__BIF_SCRATCH1_MASK                                                               0xFFFFFFFFL
+//BIF_BX0_BX_RESET_EN
+#define BIF_BX0_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN__SHIFT                                                  0x10
+#define BIF_BX0_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN_MASK                                                    0x00010000L
+//BIF_BX0_MM_CFGREGS_CNTL
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL__SHIFT                                                       0x0
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL__SHIFT                                                        0x6
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN__SHIFT                                                       0x1f
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL_MASK                                                         0x00000007L
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL_MASK                                                          0x000000C0L
+#define BIF_BX0_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN_MASK                                                         0x80000000L
+//BIF_BX0_BX_RESET_CNTL
+#define BIF_BX0_BX_RESET_CNTL__LINK_TRAIN_EN__SHIFT                                                           0x0
+#define BIF_BX0_BX_RESET_CNTL__LINK_TRAIN_EN_MASK                                                             0x00000001L
+//BIF_BX0_INTERRUPT_CNTL
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE__SHIFT                                                   0x0
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_EN__SHIFT                                                         0x1
+#define BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN__SHIFT                                                     0x3
+#define BIF_BX0_INTERRUPT_CNTL__IH_INTR_DLY_CNTR__SHIFT                                                       0x4
+#define BIF_BX0_INTERRUPT_CNTL__GEN_IH_INT_EN__SHIFT                                                          0x8
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN__SHIFT                                                 0xf
+#define BIF_BX0_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN__SHIFT                                               0x10
+#define BIF_BX0_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS__SHIFT                                   0x11
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN__SHIFT                                              0x12
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK                                                     0x00000001L
+#define BIF_BX0_INTERRUPT_CNTL__IH_DUMMY_RD_EN_MASK                                                           0x00000002L
+#define BIF_BX0_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK                                                       0x00000008L
+#define BIF_BX0_INTERRUPT_CNTL__IH_INTR_DLY_CNTR_MASK                                                         0x000000F0L
+#define BIF_BX0_INTERRUPT_CNTL__GEN_IH_INT_EN_MASK                                                            0x00000100L
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN_MASK                                                   0x00008000L
+#define BIF_BX0_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN_MASK                                                 0x00010000L
+#define BIF_BX0_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS_MASK                                     0x00020000L
+#define BIF_BX0_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN_MASK                                                0x00040000L
+//BIF_BX0_INTERRUPT_CNTL2
+#define BIF_BX0_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR__SHIFT                                                      0x0
+#define BIF_BX0_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR_MASK                                                        0xFFFFFFFFL
+//BIF_BX0_CLKREQB_PAD_CNTL
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_A__SHIFT                                                        0x0
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL__SHIFT                                                      0x1
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE__SHIFT                                                     0x2
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE__SHIFT                                                    0x3
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0__SHIFT                                                      0x5
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1__SHIFT                                                      0x6
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2__SHIFT                                                      0x7
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3__SHIFT                                                      0x8
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN__SHIFT                                                    0x9
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE__SHIFT                                                     0xa
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN__SHIFT                                                   0xb
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN__SHIFT                                                  0xc
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y__SHIFT                                                        0xd
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_A_MASK                                                          0x00000001L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL_MASK                                                        0x00000002L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE_MASK                                                       0x00000004L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE_MASK                                                      0x00000018L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0_MASK                                                        0x00000020L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1_MASK                                                        0x00000040L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2_MASK                                                        0x00000080L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3_MASK                                                        0x00000100L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN_MASK                                                      0x00000200L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE_MASK                                                       0x00000400L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN_MASK                                                     0x00000800L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN_MASK                                                    0x00001000L
+#define BIF_BX0_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y_MASK                                                          0x00002000L
+//BIF_BX0_BIF_FEATURES_CONTROL_MISC
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS__SHIFT                                          0x0
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS__SHIFT                                          0x1
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS__SHIFT                                          0x2
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS__SHIFT                                          0x3
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_RB_MSI_VEC_NOT_ENABLED_MODE__SHIFT                             0xb
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_RB_SET_OVERFLOW_EN__SHIFT                                      0xc
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS__SHIFT                                          0xd
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS__SHIFT                                       0xe
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BME_HDL_NONVIR_EN__SHIFT                                           0xf
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT__SHIFT                                           0x10
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR__SHIFT                   0x19
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS_MASK                                            0x00000001L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS_MASK                                            0x00000002L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS_MASK                                            0x00000004L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS_MASK                                            0x00000008L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_RB_MSI_VEC_NOT_ENABLED_MODE_MASK                               0x00000800L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BIF_RB_SET_OVERFLOW_EN_MASK                                        0x00001000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS_MASK                                            0x00002000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS_MASK                                         0x00004000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__BME_HDL_NONVIR_EN_MASK                                             0x00008000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT_MASK                                             0x01FF0000L
+#define BIF_BX0_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR_MASK                     0x02000000L
+//BIF_BX0_HDP_ATOMIC_CONTROL_MISC
+#define BIF_BX0_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT__SHIFT                                      0x0
+#define BIF_BX0_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT_MASK                                        0x000000FFL
+//BIF_BX0_BIF_DOORBELL_CNTL
+#define BIF_BX0_BIF_DOORBELL_CNTL__SELF_RING_DIS__SHIFT                                                       0x0
+#define BIF_BX0_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS__SHIFT                                                     0x1
+#define BIF_BX0_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN__SHIFT                                                    0x2
+#define BIF_BX0_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS__SHIFT                                         0x3
+#define BIF_BX0_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN__SHIFT                                                 0x4
+#define BIF_BX0_BIF_DOORBELL_CNTL__SELF_RING_DIS_LOCAL__SHIFT                                                 0x5
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS__SHIFT                                                  0x18
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0__SHIFT                                               0x19
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1__SHIFT                                               0x1a
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2__SHIFT                                               0x1b
+#define BIF_BX0_BIF_DOORBELL_CNTL__SELF_RING_DIS_MASK                                                         0x00000001L
+#define BIF_BX0_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS_MASK                                                       0x00000002L
+#define BIF_BX0_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN_MASK                                                      0x00000004L
+#define BIF_BX0_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS_MASK                                           0x00000008L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN_MASK                                                   0x00000010L
+#define BIF_BX0_BIF_DOORBELL_CNTL__SELF_RING_DIS_LOCAL_MASK                                                   0x00000020L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS_MASK                                                    0x01000000L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0_MASK                                                 0x02000000L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1_MASK                                                 0x04000000L
+#define BIF_BX0_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2_MASK                                                 0x08000000L
+//BIF_BX0_BIF_DOORBELL_INT_CNTL
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS__SHIFT                                       0x0
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS__SHIFT                                      0x1
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS__SHIFT                            0x2
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR__SHIFT                                        0x10
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR__SHIFT                                       0x11
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR__SHIFT                             0x12
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                            0x17
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE__SHIFT                                      0x18
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE__SHIFT                                     0x19
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE__SHIFT                           0x1a
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                               0x1c
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1d
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1e
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                              0x1f
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS_MASK                                         0x00000001L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS_MASK                                        0x00000002L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS_MASK                              0x00000004L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR_MASK                                          0x00010000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR_MASK                                         0x00020000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR_MASK                               0x00040000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE_MASK                              0x00800000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE_MASK                                        0x01000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE_MASK                                       0x02000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE_MASK                             0x04000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE_MASK                                 0x10000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x20000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x40000000L
+#define BIF_BX0_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE_MASK                                0x80000000L
+//BIF_BX0_BIF_FB_EN
+#define BIF_BX0_BIF_FB_EN__FB_READ_EN__SHIFT                                                                  0x0
+#define BIF_BX0_BIF_FB_EN__FB_WRITE_EN__SHIFT                                                                 0x1
+#define BIF_BX0_BIF_FB_EN__FB_READ_EN_MASK                                                                    0x00000001L
+#define BIF_BX0_BIF_FB_EN__FB_WRITE_EN_MASK                                                                   0x00000002L
+//BIF_BX0_BIF_INTR_CNTL
+#define BIF_BX0_BIF_INTR_CNTL__RAS_INTR_VEC_SEL__SHIFT                                                        0x0
+#define BIF_BX0_BIF_INTR_CNTL__RAS_INTR_VEC_SEL_MASK                                                          0x00000001L
+//BIF_BX0_BIF_MST_TRANS_PENDING_VF
+#define BIF_BX0_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX0_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX0_BIF_SLV_TRANS_PENDING_VF
+#define BIF_BX0_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX0_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX0_BACO_CNTL
+#define BIF_BX0_BACO_CNTL__BACO_EN__SHIFT                                                                     0x0
+#define BIF_BX0_BACO_CNTL__BACO_DUMMY_EN__SHIFT                                                               0x2
+#define BIF_BX0_BACO_CNTL__BACO_POWER_OFF__SHIFT                                                              0x3
+#define BIF_BX0_BACO_CNTL__BACO_DSTATE_BYPASS__SHIFT                                                          0x5
+#define BIF_BX0_BACO_CNTL__BACO_RST_INTR_MASK__SHIFT                                                          0x6
+#define BIF_BX0_BACO_CNTL__BACO_MODE__SHIFT                                                                   0x8
+#define BIF_BX0_BACO_CNTL__RCU_BIF_CONFIG_DONE__SHIFT                                                         0x9
+#define BIF_BX0_BACO_CNTL__PWRGOOD_VDDSOC__SHIFT                                                              0x10
+#define BIF_BX0_BACO_CNTL__BACO_AUTO_EXIT__SHIFT                                                              0x1f
+#define BIF_BX0_BACO_CNTL__BACO_EN_MASK                                                                       0x00000001L
+#define BIF_BX0_BACO_CNTL__BACO_DUMMY_EN_MASK                                                                 0x00000004L
+#define BIF_BX0_BACO_CNTL__BACO_POWER_OFF_MASK                                                                0x00000008L
+#define BIF_BX0_BACO_CNTL__BACO_DSTATE_BYPASS_MASK                                                            0x00000020L
+#define BIF_BX0_BACO_CNTL__BACO_RST_INTR_MASK_MASK                                                            0x00000040L
+#define BIF_BX0_BACO_CNTL__BACO_MODE_MASK                                                                     0x00000100L
+#define BIF_BX0_BACO_CNTL__RCU_BIF_CONFIG_DONE_MASK                                                           0x00000200L
+#define BIF_BX0_BACO_CNTL__PWRGOOD_VDDSOC_MASK                                                                0x00010000L
+#define BIF_BX0_BACO_CNTL__BACO_AUTO_EXIT_MASK                                                                0x80000000L
+//BIF_BX0_BIF_BACO_EXIT_TIME0
+#define BIF_BX0_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER__SHIFT                                          0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER_MASK                                            0x000FFFFFL
+//BIF_BX0_BIF_BACO_EXIT_TIMER1
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER__SHIFT                                         0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN__SHIFT                                            0x18
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS__SHIFT                                                 0x1a
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH__SHIFT                                           0x1b
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW__SHIFT                                            0x1c
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL__SHIFT                                                    0x1d
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS__SHIFT                                     0x1f
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER_MASK                                           0x000FFFFFL
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN_MASK                                              0x01000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS_MASK                                                   0x04000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH_MASK                                             0x08000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW_MASK                                              0x10000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL_MASK                                                      0x60000000L
+#define BIF_BX0_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS_MASK                                       0x80000000L
+//BIF_BX0_BIF_BACO_EXIT_TIMER2
+#define BIF_BX0_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER__SHIFT                                         0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER_MASK                                           0x000FFFFFL
+//BIF_BX0_BIF_BACO_EXIT_TIMER3
+#define BIF_BX0_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER__SHIFT                                     0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER_MASK                                       0x000FFFFFL
+//BIF_BX0_BIF_BACO_EXIT_TIMER4
+#define BIF_BX0_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER__SHIFT                                      0x0
+#define BIF_BX0_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER_MASK                                        0x000FFFFFL
+//BIF_BX0_MEM_TYPE_CNTL
+#define BIF_BX0_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3__SHIFT                                                        0x0
+#define BIF_BX0_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3_MASK                                                          0x00000001L
+//BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE__SHIFT                                                     0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE__SHIFT                                                  0x1
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE__SHIFT                                                    0x8
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE_MASK                                                       0x00000001L
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE_MASK                                                    0x00000002L
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE_MASK                                                      0x00000100L
+//BIF_BX0_NBIF_GFX_ADDR_LUT_0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_0__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_0__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_1
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_1__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_1__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_2
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_2__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_2__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_3
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_3__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_3__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_4
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_4__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_4__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_5
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_5__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_5__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_6
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_6__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_6__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_7
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_7__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_7__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_8
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_8__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_8__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_9
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_9__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_9__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_10
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_10__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_10__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_11
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_11__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_11__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_12
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_12__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_12__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_13
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_13__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_13__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_14
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_14__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_14__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_NBIF_GFX_ADDR_LUT_15
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_15__ADDR__SHIFT                                                             0x0
+#define BIF_BX0_NBIF_GFX_ADDR_LUT_15__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX0_VF_REGWR_EN
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF0__SHIFT                                                           0x0
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF1__SHIFT                                                           0x1
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF2__SHIFT                                                           0x2
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF3__SHIFT                                                           0x3
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF4__SHIFT                                                           0x4
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF5__SHIFT                                                           0x5
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF6__SHIFT                                                           0x6
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF7__SHIFT                                                           0x7
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF8__SHIFT                                                           0x8
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF9__SHIFT                                                           0x9
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF10__SHIFT                                                          0xa
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF11__SHIFT                                                          0xb
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF12__SHIFT                                                          0xc
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF13__SHIFT                                                          0xd
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF14__SHIFT                                                          0xe
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF15__SHIFT                                                          0xf
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF16__SHIFT                                                          0x10
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF17__SHIFT                                                          0x11
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF18__SHIFT                                                          0x12
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF19__SHIFT                                                          0x13
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF20__SHIFT                                                          0x14
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF21__SHIFT                                                          0x15
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF22__SHIFT                                                          0x16
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF23__SHIFT                                                          0x17
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF24__SHIFT                                                          0x18
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF25__SHIFT                                                          0x19
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF26__SHIFT                                                          0x1a
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF27__SHIFT                                                          0x1b
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF28__SHIFT                                                          0x1c
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF29__SHIFT                                                          0x1d
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF30__SHIFT                                                          0x1e
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF0_MASK                                                             0x00000001L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF1_MASK                                                             0x00000002L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF2_MASK                                                             0x00000004L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF3_MASK                                                             0x00000008L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF4_MASK                                                             0x00000010L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF5_MASK                                                             0x00000020L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF6_MASK                                                             0x00000040L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF7_MASK                                                             0x00000080L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF8_MASK                                                             0x00000100L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF9_MASK                                                             0x00000200L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF10_MASK                                                            0x00000400L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF11_MASK                                                            0x00000800L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF12_MASK                                                            0x00001000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF13_MASK                                                            0x00002000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF14_MASK                                                            0x00004000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF15_MASK                                                            0x00008000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF16_MASK                                                            0x00010000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF17_MASK                                                            0x00020000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF18_MASK                                                            0x00040000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF19_MASK                                                            0x00080000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF20_MASK                                                            0x00100000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF21_MASK                                                            0x00200000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF22_MASK                                                            0x00400000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF23_MASK                                                            0x00800000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF24_MASK                                                            0x01000000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF25_MASK                                                            0x02000000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF26_MASK                                                            0x04000000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF27_MASK                                                            0x08000000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF28_MASK                                                            0x10000000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF29_MASK                                                            0x20000000L
+#define BIF_BX0_VF_REGWR_EN__VF_REGWR_EN_VF30_MASK                                                            0x40000000L
+//BIF_BX0_VF_DOORBELL_EN
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0__SHIFT                                                     0x0
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1__SHIFT                                                     0x1
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2__SHIFT                                                     0x2
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3__SHIFT                                                     0x3
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4__SHIFT                                                     0x4
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5__SHIFT                                                     0x5
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6__SHIFT                                                     0x6
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7__SHIFT                                                     0x7
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8__SHIFT                                                     0x8
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9__SHIFT                                                     0x9
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10__SHIFT                                                    0xa
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11__SHIFT                                                    0xb
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12__SHIFT                                                    0xc
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13__SHIFT                                                    0xd
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14__SHIFT                                                    0xe
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15__SHIFT                                                    0xf
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16__SHIFT                                                    0x10
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17__SHIFT                                                    0x11
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18__SHIFT                                                    0x12
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19__SHIFT                                                    0x13
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20__SHIFT                                                    0x14
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21__SHIFT                                                    0x15
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22__SHIFT                                                    0x16
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23__SHIFT                                                    0x17
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24__SHIFT                                                    0x18
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25__SHIFT                                                    0x19
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26__SHIFT                                                    0x1a
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27__SHIFT                                                    0x1b
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28__SHIFT                                                    0x1c
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29__SHIFT                                                    0x1d
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30__SHIFT                                                    0x1e
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS__SHIFT                                                 0x1f
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0_MASK                                                       0x00000001L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1_MASK                                                       0x00000002L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2_MASK                                                       0x00000004L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3_MASK                                                       0x00000008L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4_MASK                                                       0x00000010L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5_MASK                                                       0x00000020L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6_MASK                                                       0x00000040L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7_MASK                                                       0x00000080L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8_MASK                                                       0x00000100L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9_MASK                                                       0x00000200L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10_MASK                                                      0x00000400L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11_MASK                                                      0x00000800L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12_MASK                                                      0x00001000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13_MASK                                                      0x00002000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14_MASK                                                      0x00004000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15_MASK                                                      0x00008000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16_MASK                                                      0x00010000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17_MASK                                                      0x00020000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18_MASK                                                      0x00040000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19_MASK                                                      0x00080000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20_MASK                                                      0x00100000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21_MASK                                                      0x00200000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22_MASK                                                      0x00400000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23_MASK                                                      0x00800000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24_MASK                                                      0x01000000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25_MASK                                                      0x02000000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26_MASK                                                      0x04000000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27_MASK                                                      0x08000000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28_MASK                                                      0x10000000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29_MASK                                                      0x20000000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30_MASK                                                      0x40000000L
+#define BIF_BX0_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS_MASK                                                   0x80000000L
+//BIF_BX0_VF_FB_EN
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF0__SHIFT                                                                 0x0
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF1__SHIFT                                                                 0x1
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF2__SHIFT                                                                 0x2
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF3__SHIFT                                                                 0x3
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF4__SHIFT                                                                 0x4
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF5__SHIFT                                                                 0x5
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF6__SHIFT                                                                 0x6
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF7__SHIFT                                                                 0x7
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF8__SHIFT                                                                 0x8
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF9__SHIFT                                                                 0x9
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF10__SHIFT                                                                0xa
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF11__SHIFT                                                                0xb
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF12__SHIFT                                                                0xc
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF13__SHIFT                                                                0xd
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF14__SHIFT                                                                0xe
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF15__SHIFT                                                                0xf
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF16__SHIFT                                                                0x10
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF17__SHIFT                                                                0x11
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF18__SHIFT                                                                0x12
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF19__SHIFT                                                                0x13
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF20__SHIFT                                                                0x14
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF21__SHIFT                                                                0x15
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF22__SHIFT                                                                0x16
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF23__SHIFT                                                                0x17
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF24__SHIFT                                                                0x18
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF25__SHIFT                                                                0x19
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF26__SHIFT                                                                0x1a
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF27__SHIFT                                                                0x1b
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF28__SHIFT                                                                0x1c
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF29__SHIFT                                                                0x1d
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF30__SHIFT                                                                0x1e
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF0_MASK                                                                   0x00000001L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF1_MASK                                                                   0x00000002L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF2_MASK                                                                   0x00000004L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF3_MASK                                                                   0x00000008L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF4_MASK                                                                   0x00000010L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF5_MASK                                                                   0x00000020L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF6_MASK                                                                   0x00000040L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF7_MASK                                                                   0x00000080L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF8_MASK                                                                   0x00000100L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF9_MASK                                                                   0x00000200L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF10_MASK                                                                  0x00000400L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF11_MASK                                                                  0x00000800L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF12_MASK                                                                  0x00001000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF13_MASK                                                                  0x00002000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF14_MASK                                                                  0x00004000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF15_MASK                                                                  0x00008000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF16_MASK                                                                  0x00010000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF17_MASK                                                                  0x00020000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF18_MASK                                                                  0x00040000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF19_MASK                                                                  0x00080000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF20_MASK                                                                  0x00100000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF21_MASK                                                                  0x00200000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF22_MASK                                                                  0x00400000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF23_MASK                                                                  0x00800000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF24_MASK                                                                  0x01000000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF25_MASK                                                                  0x02000000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF26_MASK                                                                  0x04000000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF27_MASK                                                                  0x08000000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF28_MASK                                                                  0x10000000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF29_MASK                                                                  0x20000000L
+#define BIF_BX0_VF_FB_EN__VF_FB_EN_VF30_MASK                                                                  0x40000000L
+//BIF_BX0_VF_REGWR_STATUS
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0__SHIFT                                                   0x0
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1__SHIFT                                                   0x1
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2__SHIFT                                                   0x2
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3__SHIFT                                                   0x3
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4__SHIFT                                                   0x4
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5__SHIFT                                                   0x5
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6__SHIFT                                                   0x6
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7__SHIFT                                                   0x7
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8__SHIFT                                                   0x8
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9__SHIFT                                                   0x9
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10__SHIFT                                                  0xa
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11__SHIFT                                                  0xb
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12__SHIFT                                                  0xc
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13__SHIFT                                                  0xd
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14__SHIFT                                                  0xe
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15__SHIFT                                                  0xf
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16__SHIFT                                                  0x10
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17__SHIFT                                                  0x11
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18__SHIFT                                                  0x12
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19__SHIFT                                                  0x13
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20__SHIFT                                                  0x14
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21__SHIFT                                                  0x15
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22__SHIFT                                                  0x16
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23__SHIFT                                                  0x17
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24__SHIFT                                                  0x18
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25__SHIFT                                                  0x19
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26__SHIFT                                                  0x1a
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27__SHIFT                                                  0x1b
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28__SHIFT                                                  0x1c
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29__SHIFT                                                  0x1d
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30__SHIFT                                                  0x1e
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0_MASK                                                     0x00000001L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1_MASK                                                     0x00000002L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2_MASK                                                     0x00000004L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3_MASK                                                     0x00000008L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4_MASK                                                     0x00000010L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5_MASK                                                     0x00000020L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6_MASK                                                     0x00000040L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7_MASK                                                     0x00000080L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8_MASK                                                     0x00000100L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9_MASK                                                     0x00000200L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10_MASK                                                    0x00000400L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11_MASK                                                    0x00000800L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12_MASK                                                    0x00001000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13_MASK                                                    0x00002000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14_MASK                                                    0x00004000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15_MASK                                                    0x00008000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16_MASK                                                    0x00010000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17_MASK                                                    0x00020000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18_MASK                                                    0x00040000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19_MASK                                                    0x00080000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20_MASK                                                    0x00100000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21_MASK                                                    0x00200000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22_MASK                                                    0x00400000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23_MASK                                                    0x00800000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24_MASK                                                    0x01000000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25_MASK                                                    0x02000000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26_MASK                                                    0x04000000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27_MASK                                                    0x08000000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28_MASK                                                    0x10000000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29_MASK                                                    0x20000000L
+#define BIF_BX0_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30_MASK                                                    0x40000000L
+//BIF_BX0_VF_DOORBELL_STATUS
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0__SHIFT                                             0x0
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1__SHIFT                                             0x1
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2__SHIFT                                             0x2
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3__SHIFT                                             0x3
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4__SHIFT                                             0x4
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5__SHIFT                                             0x5
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6__SHIFT                                             0x6
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7__SHIFT                                             0x7
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8__SHIFT                                             0x8
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9__SHIFT                                             0x9
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10__SHIFT                                            0xa
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11__SHIFT                                            0xb
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12__SHIFT                                            0xc
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13__SHIFT                                            0xd
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14__SHIFT                                            0xe
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15__SHIFT                                            0xf
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16__SHIFT                                            0x10
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17__SHIFT                                            0x11
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18__SHIFT                                            0x12
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19__SHIFT                                            0x13
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20__SHIFT                                            0x14
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21__SHIFT                                            0x15
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22__SHIFT                                            0x16
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23__SHIFT                                            0x17
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24__SHIFT                                            0x18
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25__SHIFT                                            0x19
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26__SHIFT                                            0x1a
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27__SHIFT                                            0x1b
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28__SHIFT                                            0x1c
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29__SHIFT                                            0x1d
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30__SHIFT                                            0x1e
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0_MASK                                               0x00000001L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1_MASK                                               0x00000002L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2_MASK                                               0x00000004L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3_MASK                                               0x00000008L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4_MASK                                               0x00000010L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5_MASK                                               0x00000020L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6_MASK                                               0x00000040L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7_MASK                                               0x00000080L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8_MASK                                               0x00000100L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9_MASK                                               0x00000200L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10_MASK                                              0x00000400L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11_MASK                                              0x00000800L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12_MASK                                              0x00001000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13_MASK                                              0x00002000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14_MASK                                              0x00004000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15_MASK                                              0x00008000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16_MASK                                              0x00010000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17_MASK                                              0x00020000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18_MASK                                              0x00040000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19_MASK                                              0x00080000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20_MASK                                              0x00100000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21_MASK                                              0x00200000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22_MASK                                              0x00400000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23_MASK                                              0x00800000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24_MASK                                              0x01000000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25_MASK                                              0x02000000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26_MASK                                              0x04000000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27_MASK                                              0x08000000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28_MASK                                              0x10000000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29_MASK                                              0x20000000L
+#define BIF_BX0_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30_MASK                                              0x40000000L
+//BIF_BX0_VF_FB_STATUS
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF0__SHIFT                                                         0x0
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF1__SHIFT                                                         0x1
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF2__SHIFT                                                         0x2
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF3__SHIFT                                                         0x3
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF4__SHIFT                                                         0x4
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF5__SHIFT                                                         0x5
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF6__SHIFT                                                         0x6
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF7__SHIFT                                                         0x7
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF8__SHIFT                                                         0x8
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF9__SHIFT                                                         0x9
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF10__SHIFT                                                        0xa
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF11__SHIFT                                                        0xb
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF12__SHIFT                                                        0xc
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF13__SHIFT                                                        0xd
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF14__SHIFT                                                        0xe
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF15__SHIFT                                                        0xf
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF16__SHIFT                                                        0x10
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF17__SHIFT                                                        0x11
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF18__SHIFT                                                        0x12
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF19__SHIFT                                                        0x13
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF20__SHIFT                                                        0x14
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF21__SHIFT                                                        0x15
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF22__SHIFT                                                        0x16
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF23__SHIFT                                                        0x17
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF24__SHIFT                                                        0x18
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF25__SHIFT                                                        0x19
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF26__SHIFT                                                        0x1a
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF27__SHIFT                                                        0x1b
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF28__SHIFT                                                        0x1c
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF29__SHIFT                                                        0x1d
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF30__SHIFT                                                        0x1e
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF0_MASK                                                           0x00000001L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF1_MASK                                                           0x00000002L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF2_MASK                                                           0x00000004L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF3_MASK                                                           0x00000008L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF4_MASK                                                           0x00000010L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF5_MASK                                                           0x00000020L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF6_MASK                                                           0x00000040L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF7_MASK                                                           0x00000080L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF8_MASK                                                           0x00000100L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF9_MASK                                                           0x00000200L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF10_MASK                                                          0x00000400L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF11_MASK                                                          0x00000800L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF12_MASK                                                          0x00001000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF13_MASK                                                          0x00002000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF14_MASK                                                          0x00004000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF15_MASK                                                          0x00008000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF16_MASK                                                          0x00010000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF17_MASK                                                          0x00020000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF18_MASK                                                          0x00040000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF19_MASK                                                          0x00080000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF20_MASK                                                          0x00100000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF21_MASK                                                          0x00200000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF22_MASK                                                          0x00400000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF23_MASK                                                          0x00800000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF24_MASK                                                          0x01000000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF25_MASK                                                          0x02000000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF26_MASK                                                          0x04000000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF27_MASK                                                          0x08000000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF28_MASK                                                          0x10000000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF29_MASK                                                          0x20000000L
+#define BIF_BX0_VF_FB_STATUS__VF_FB_STATUS_VF30_MASK                                                          0x40000000L
+//BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL
+#define BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX0_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL
+#define BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX0_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX0_BIF_RB_CNTL
+#define BIF_BX0_BIF_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define BIF_BX0_BIF_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define BIF_BX0_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT                                                     0x8
+#define BIF_BX0_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER__SHIFT                                                      0x9
+#define BIF_BX0_BIF_RB_CNTL__BIF_RB_TRAN__SHIFT                                                               0x11
+#define BIF_BX0_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL__SHIFT                                                  0x19
+#define BIF_BX0_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY__SHIFT                                                      0x1a
+#define BIF_BX0_BIF_RB_CNTL__RB_INTR_ARB_MODE__SHIFT                                                          0x1d
+#define BIF_BX0_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE__SHIFT                                                     0x1e
+#define BIF_BX0_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT                                                       0x1f
+#define BIF_BX0_BIF_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define BIF_BX0_BIF_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define BIF_BX0_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK                                                       0x00000100L
+#define BIF_BX0_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER_MASK                                                        0x00003E00L
+#define BIF_BX0_BIF_RB_CNTL__BIF_RB_TRAN_MASK                                                                 0x00020000L
+#define BIF_BX0_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL_MASK                                                    0x02000000L
+#define BIF_BX0_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY_MASK                                                        0x1C000000L
+#define BIF_BX0_BIF_RB_CNTL__RB_INTR_ARB_MODE_MASK                                                            0x20000000L
+#define BIF_BX0_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE_MASK                                                       0x40000000L
+#define BIF_BX0_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK                                                         0x80000000L
+//BIF_BX0_BIF_RB_BASE
+#define BIF_BX0_BIF_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define BIF_BX0_BIF_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//BIF_BX0_BIF_RB_RPTR
+#define BIF_BX0_BIF_RB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define BIF_BX0_BIF_RB_RPTR__OFFSET_MASK                                                                      0x0003FFFCL
+//BIF_BX0_BIF_RB_WPTR
+#define BIF_BX0_BIF_RB_WPTR__BIF_RB_OVERFLOW__SHIFT                                                           0x0
+#define BIF_BX0_BIF_RB_WPTR__OFFSET__SHIFT                                                                    0x2
+#define BIF_BX0_BIF_RB_WPTR__BIF_RB_OVERFLOW_MASK                                                             0x00000001L
+#define BIF_BX0_BIF_RB_WPTR__OFFSET_MASK                                                                      0x0003FFFCL
+//BIF_BX0_BIF_RB_WPTR_ADDR_HI
+#define BIF_BX0_BIF_RB_WPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define BIF_BX0_BIF_RB_WPTR_ADDR_HI__ADDR_MASK                                                                0x000000FFL
+//BIF_BX0_BIF_RB_WPTR_ADDR_LO
+#define BIF_BX0_BIF_RB_WPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define BIF_BX0_BIF_RB_WPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//BIF_BX0_MAILBOX_INDEX
+#define BIF_BX0_MAILBOX_INDEX__MAILBOX_INDEX__SHIFT                                                           0x0
+#define BIF_BX0_MAILBOX_INDEX__MAILBOX_INDEX_MASK                                                             0x0000001FL
+//BIF_BX0_BACO_AZ_ENHANCE_CTRL
+#define BIF_BX0_BACO_AZ_ENHANCE_CTRL__GCTL_OFFSET__SHIFT                                                      0x2
+#define BIF_BX0_BACO_AZ_ENHANCE_CTRL__AZ_CRST_WRITTEN_ONE__SHIFT                                              0x10
+#define BIF_BX0_BACO_AZ_ENHANCE_CTRL__AZ_CRST_WRITTEN_ZERO__SHIFT                                             0x11
+#define BIF_BX0_BACO_AZ_ENHANCE_CTRL__AZ_MMIO_RDRSPDATA_FORCE_ZERO__SHIFT                                     0x1f
+#define BIF_BX0_BACO_AZ_ENHANCE_CTRL__GCTL_OFFSET_MASK                                                        0x00003FFCL
+#define BIF_BX0_BACO_AZ_ENHANCE_CTRL__AZ_CRST_WRITTEN_ONE_MASK                                                0x00010000L
+#define BIF_BX0_BACO_AZ_ENHANCE_CTRL__AZ_CRST_WRITTEN_ZERO_MASK                                               0x00020000L
+#define BIF_BX0_BACO_AZ_ENHANCE_CTRL__AZ_MMIO_RDRSPDATA_FORCE_ZERO_MASK                                       0x80000000L
+//BIF_BX0_BIF_MP1_INTR_CTRL
+#define BIF_BX0_BIF_MP1_INTR_CTRL__BACO_EXIT_DONE__SHIFT                                                      0x0
+#define BIF_BX0_BIF_MP1_INTR_CTRL__BACO_EXIT_DONE_MASK                                                        0x00000001L
+//BIF_BX0_BIF_PERSTB_PAD_CNTL
+#define BIF_BX0_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX0_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL_MASK                                                     0x0000FFFFL
+//BIF_BX0_BIF_PX_EN_PAD_CNTL
+#define BIF_BX0_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL__SHIFT                                                     0x0
+#define BIF_BX0_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL_MASK                                                       0x00000FFFL
+//BIF_BX0_BIF_REFPADKIN_PAD_CNTL
+#define BIF_BX0_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL__SHIFT                                             0x0
+#define BIF_BX0_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL_MASK                                               0x000000FFL
+//BIF_BX0_BIF_CLKREQB_PAD_CNTL
+#define BIF_BX0_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL__SHIFT                                                 0x0
+#define BIF_BX0_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_MASK                                                   0x7FFFFFFFL
+//BIF_BX0_BIF_PWRBRK_PAD_CNTL
+#define BIF_BX0_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX0_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL_MASK                                                     0x000000FFL
+//BIF_BX0_BIF_GPUIOV_SCH_CFG_SIZE
+#define BIF_BX0_BIF_GPUIOV_SCH_CFG_SIZE__GPUIOV_SCH_CFG_SIZE__SHIFT                                           0x0
+#define BIF_BX0_BIF_GPUIOV_SCH_CFG_SIZE__GPUIOV_SCH_CFG_SIZE_MASK                                             0x0000000FL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_BIFDEC1
+//RCC_DEV0_0_RCC_ERR_INT_CNTL
+#define RCC_DEV0_0_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN__SHIFT                                0x0
+#define RCC_DEV0_0_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN_MASK                                  0x00000001L
+//RCC_DEV0_0_RCC_BACO_CNTL_MISC
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS__SHIFT                                                 0x0
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS__SHIFT                                                  0x1
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS_MASK                                                   0x00000001L
+#define RCC_DEV0_0_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS_MASK                                                    0x00000002L
+//RCC_DEV0_0_RCC_RESET_EN
+#define RCC_DEV0_0_RCC_RESET_EN__DB_APER_RESET_EN__SHIFT                                                      0xf
+#define RCC_DEV0_0_RCC_RESET_EN__DB_APER_RESET_EN_MASK                                                        0x00008000L
+//RCC_DEV0_0_RCC_VDM_SUPPORT
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_0_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_0_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+//RCC_DEV0_0_RCC_GPUIOV_REGION
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__LFB_REGION__SHIFT                                                       0x0
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__MAX_REGION__SHIFT                                                       0x4
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__LFB_REGION_MASK                                                         0x0000000FL
+#define RCC_DEV0_0_RCC_GPUIOV_REGION__MAX_REGION_MASK                                                         0x000000F0L
+//RCC_DEV0_0_RCC_GPU_HOSTVM_EN
+#define RCC_DEV0_0_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN__SHIFT                                                    0x0
+#define RCC_DEV0_0_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN_MASK                                                      0x00000001L
+//RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE__SHIFT                              0x0
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN__SHIFT                                    0x1
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE_MASK                                0x00000001L
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN_MASK                                      0x00000002L
+//RCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET__SHIFT                        0x0
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET_MASK                          0xFFFFL
+//RCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE__SHIFT                                    0x0
+#define RCC_DEV0_0_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE_MASK                                      0xFFFFL
+//RCC_DEV0_0_RCC_PEER_REG_RANGE0
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE0__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_0_RCC_PEER_REG_RANGE1
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_0_RCC_PEER_REG_RANGE1__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_0_RCC_BUS_CNTL
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_0_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_0_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_0_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_0_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_0_RCC_CONFIG_CNTL
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B__SHIFT                                               0x2
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GRPH_ADRSEL__SHIFT                                                        0x3
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN_MASK                                                       0x00000001L
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B_MASK                                                 0x00000004L
+#define RCC_DEV0_0_RCC_CONFIG_CNTL__GRPH_ADRSEL_MASK                                                          0x00000018L
+//RCC_DEV0_0_RCC_CONFIG_F0_BASE
+#define RCC_DEV0_0_RCC_CONFIG_F0_BASE__F0_BASE__SHIFT                                                         0x0
+#define RCC_DEV0_0_RCC_CONFIG_F0_BASE__F0_BASE_MASK                                                           0xFFFFFFFFL
+//RCC_DEV0_0_RCC_CONFIG_APER_SIZE
+#define RCC_DEV0_0_RCC_CONFIG_APER_SIZE__APER_SIZE__SHIFT                                                     0x0
+#define RCC_DEV0_0_RCC_CONFIG_APER_SIZE__APER_SIZE_MASK                                                       0xFFFFFFFFL
+//RCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE
+#define RCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE__SHIFT                                             0x0
+#define RCC_DEV0_0_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE_MASK                                               0x07FFFFFFL
+//RCC_DEV0_0_RCC_XDMA_LO
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_APER_EN__SHIFT                                                       0x1f
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND_MASK                                                     0x7FFFFFFFL
+#define RCC_DEV0_0_RCC_XDMA_LO__BIF_XDMA_APER_EN_MASK                                                         0x80000000L
+//RCC_DEV0_0_RCC_XDMA_HI
+#define RCC_DEV0_0_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND_MASK                                                     0x7FFFFFFFL
+//RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_0_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_0_RCC_BUSNUM_CNTL1
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL1__ID_MASK__SHIFT                                                           0x0
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL1__ID_MASK_MASK                                                             0x000000FFL
+//RCC_DEV0_0_RCC_BUSNUM_LIST0
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID0__SHIFT                                                               0x0
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID1__SHIFT                                                               0x8
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID2__SHIFT                                                               0x10
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID3__SHIFT                                                               0x18
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID0_MASK                                                                 0x000000FFL
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID1_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID2_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST0__ID3_MASK                                                                 0xFF000000L
+//RCC_DEV0_0_RCC_BUSNUM_LIST1
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID4__SHIFT                                                               0x0
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID5__SHIFT                                                               0x8
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID6__SHIFT                                                               0x10
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID7__SHIFT                                                               0x18
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID4_MASK                                                                 0x000000FFL
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID5_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID6_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_0_RCC_BUSNUM_LIST1__ID7_MASK                                                                 0xFF000000L
+//RCC_DEV0_0_RCC_BUSNUM_CNTL2
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL__SHIFT                                                    0x0
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN__SHIFT                                                     0x8
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__HDPREG_CNTL__SHIFT                                                       0x10
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH__SHIFT                                           0x11
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL_MASK                                                      0x000000FFL
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN_MASK                                                       0x00000100L
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__HDPREG_CNTL_MASK                                                         0x00010000L
+#define RCC_DEV0_0_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH_MASK                                             0x00020000L
+//RCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM
+#define RCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN_MASK                                                     0x00000001L
+//RCC_DEV0_0_RCC_HOST_BUSNUM
+#define RCC_DEV0_0_RCC_HOST_BUSNUM__HOST_ID__SHIFT                                                            0x0
+#define RCC_DEV0_0_RCC_HOST_BUSNUM__HOST_ID_MASK                                                              0x0000FFFFL
+//RCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_0_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1__SHIFT                                                   0x8
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2__SHIFT                                                   0x10
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3__SHIFT                                                   0x18
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0_MASK                                                     0x000000FFL
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1_MASK                                                     0x0000FF00L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2_MASK                                                     0x00FF0000L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3_MASK                                                     0xFF000000L
+//RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4__SHIFT                                                   0x0
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5__SHIFT                                                   0x8
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6__SHIFT                                                   0x10
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7__SHIFT                                                   0x18
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4_MASK                                                     0x000000FFL
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5_MASK                                                     0x0000FF00L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6_MASK                                                     0x00FF0000L
+#define RCC_DEV0_0_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7_MASK                                                     0xFF000000L
+//RCC_DEV0_0_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_0_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_0_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_0_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG__SHIFT                                            0x10
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+#define RCC_DEV0_0_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG_MASK                                              0x00FF0000L
+//RCC_DEV0_0_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_0_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_0_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_0_RCC_MH_ARB_CNTL
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_0_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x0000FFFEL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_BIFDEC2
+//RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT4_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT4_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT5_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT5_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT6_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT6_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT7_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT7_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_LO
+#define RCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                               0x2
+#define RCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//RCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_HI
+#define RCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT8_MSG_DATA
+#define RCC_DEV0_EPF0_GFXMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                                   0xFFFFFFFFL
+//RCC_DEV0_EPF0_GFXMSIX_VECT8_CONTROL
+#define RCC_DEV0_EPF0_GFXMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                                  0x0
+#define RCC_DEV0_EPF0_GFXMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                    0x00000001L
+//RCC_DEV0_EPF0_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                                 0x0
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                                 0x1
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_2__SHIFT                                                 0x2
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_3__SHIFT                                                 0x3
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                                   0x00000001L
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                                   0x00000002L
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_2_MASK                                                   0x00000004L
+#define RCC_DEV0_EPF0_GFXMSIX_PBA__MSIX_PENDING_BITS_3_MASK                                                   0x00000008L
+
+
+// addressBlock: nbif0_nbif0_rcc_strap_BIFDEC1
+//RCC_STRAP0_RCC_BIF_STRAP0
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_AUD_PIN__SHIFT                                                       0x12
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x14
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN__SHIFT                         0x15
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GPUIOV_EN__SHIFT                                                     0x16
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE__SHIFT                                          0x17
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_AUD_PIN_MASK                                                         0x000C0000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00100000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN_MASK                           0x00200000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GPUIOV_EN_MASK                                                       0x00400000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE_MASK                                            0x00800000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP0_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP1
+#define RCC_STRAP0_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP0_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP0_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_FLIT_MODE_SUPPORT__SHIFT                                             0x4
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN__SHIFT                                                  0x8
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN__SHIFT                                                 0x9
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE__SHIFT                                                  0xa
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE__SHIFT                                        0xc
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_FLIT_MODE_SUPPORT_MASK                                               0x00000010L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN_MASK                                                    0x00000100L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN_MASK                                                   0x00000200L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE_MASK                                                    0x00000C00L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE_MASK                                          0x00001000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP0_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP10
+//RCC_STRAP0_RCC_BIF_STRAP11
+//RCC_STRAP0_RCC_BIF_STRAP12
+//RCC_STRAP0_RCC_BIF_STRAP13
+#define RCC_STRAP0_RCC_BIF_STRAP13__STRAP_PCIE_SMN_APER__SHIFT                                                0x0
+#define RCC_STRAP0_RCC_BIF_STRAP13__STRAP_PCIE_SMN_APER_MASK                                                  0x00000FFFL
+//RCC_STRAP0_RCC_BIF_STRAP2
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SPT__SHIFT                                                      0x1
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWDS_SPT__SHIFT                                                      0x2
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS__SHIFT                                            0x3
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS__SHIFT                                            0x4
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP0_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ARI_EN_UP__SHIFT                                                     0x19
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWUS_SPT_MASK                                                        0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWDS_SPT_MASK                                                        0x00000004L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS_MASK                                              0x00000008L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS_MASK                                              0x00000010L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP0_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_ARI_EN_UP_MASK                                                       0x02000000L
+#define RCC_STRAP0_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP3
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP0_RCC_BIF_STRAP4
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP0_RCC_BIF_STRAP5
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN__SHIFT                                 0x1f
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+#define RCC_STRAP0_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN_MASK                                   0x80000000L
+//RCC_STRAP0_RCC_BIF_STRAP6
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_DOE_VERSION_SEL__SHIFT                                               0x3
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE__SHIFT                                               0x5
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0xa
+#define RCC_STRAP0_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6__SHIFT                                                 0xf
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_DOE_VERSION_SEL_MASK                                                 0x00000008L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE_MASK                                                 0x00000020L
+#define RCC_STRAP0_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00007C00L
+#define RCC_STRAP0_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6_MASK                                                   0xFFFF8000L
+//RCC_STRAP0_RCC_BIF_STRAP7
+#define RCC_STRAP0_RCC_BIF_STRAP7__STRAP_GEN6_DIS__SHIFT                                                      0x19
+#define RCC_STRAP0_RCC_BIF_STRAP7__STRAP_BIF_KILL_GEN6__SHIFT                                                 0x1a
+#define RCC_STRAP0_RCC_BIF_STRAP7__STRAP_GEN6_DIS_MASK                                                        0x02000000L
+#define RCC_STRAP0_RCC_BIF_STRAP7__STRAP_BIF_KILL_GEN6_MASK                                                   0x04000000L
+//RCC_STRAP0_RCC_BIF_STRAP8
+//RCC_STRAP0_RCC_BIF_STRAP9
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0__SHIFT                                     0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0__SHIFT                                   0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0_MASK                                       0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0_MASK                                     0x00E00000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0__SHIFT                                   0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0__SHIFT                              0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0__SHIFT                           0x1a
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0_MASK                                     0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0_MASK                                0x03000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0_MASK                             0x0C000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0__SHIFT                               0x1e
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                  0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0_MASK                                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                    0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                     0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                     0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_RECEIVER_L0P_SUPPORTED_DEV0__SHIFT                            0x1a
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_PORT_L0P_EXIT_LATENCY_DEV0__SHIFT                             0x1b
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                       0x01000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                       0x02000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_RECEIVER_L0P_SUPPORTED_DEV0_MASK                              0x04000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP12__STRAP_PORT_L0P_EXIT_LATENCY_DEV0_MASK                               0x38000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0__SHIFT                               0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0__SHIFT                             0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0__SHIFT                                 0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0__SHIFT                          0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0__SHIFT                                         0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0__SHIFT                                      0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0__SHIFT                                   0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DEV0__SHIFT                                            0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RETIMER_L0P_EXIT_LATENCY_DEV0__SHIFT                          0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_DEVICE3_EN_DN_DEV0__SHIFT                                     0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DN_DEV0__SHIFT                                         0xd
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0_MASK                                 0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0_MASK                               0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0_MASK                                   0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0_MASK                            0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0_MASK                                           0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0_MASK                                        0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0_MASK                                     0x00000040L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DEV0_MASK                                              0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_RETIMER_L0P_EXIT_LATENCY_DEV0_MASK                            0x00000E00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_DEVICE3_EN_DN_DEV0_MASK                                       0x00001000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DN_DEV0_MASK                                           0x00002000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP15__STRAP_DSN_CODE_L_DN_DEV0__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP15__STRAP_DSN_CODE_L_DN_DEV0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP16
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP16__STRAP_DSN_CODE_H_DN_DEV0__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP16__STRAP_DSN_CODE_H_DN_DEV0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP17
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_LINK_IDE_STREAM_SUPPORTED_DEV0__SHIFT                         0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_STREAM_SUPPORTED_DEV0__SHIFT                    0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_DEV0__SHIFT                 0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_AGGREGATION_SUPPORTED_DEV0__SHIFT                             0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_PCRC_SUPPORTED_DEV0__SHIFT                                    0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_IDE_KM_PROTOCOL_SUPPORTED_DEV0__SHIFT                         0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_DEV0__SHIFT               0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_DEV0__SHIFT                 0xd
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_DEV0__SHIFT            0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_TEE_LIMITED_SUPPORTED_DEV0__SHIFT                             0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_XT_SUPPORTED_DEV0__SHIFT                                      0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_LINK_IDE_STREAM_SUPPORTED_DEV0_MASK                           0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_STREAM_SUPPORTED_DEV0_MASK                      0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_DEV0_MASK                   0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_AGGREGATION_SUPPORTED_DEV0_MASK                               0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_PCRC_SUPPORTED_DEV0_MASK                                      0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_IDE_KM_PROTOCOL_SUPPORTED_DEV0_MASK                           0x00000040L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_DEV0_MASK                 0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_DEV0_MASK                   0x0000E000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_DEV0_MASK              0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_TEE_LIMITED_SUPPORTED_DEV0_MASK                               0x01000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP17__STRAP_XT_SUPPORTED_DEV0_MASK                                        0x02000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP18__STRAP_NUM_ADDR_ASSOCIATION_REG_BLKS_DEV0__SHIFT                     0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP18__STRAP_NUM_ADDR_ASSOCIATION_REG_BLKS_DEV0_MASK                       0x0000000FL
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0__SHIFT                                         0x1e
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0_MASK                                           0x40000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP0_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0__SHIFT                                   0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0__SHIFT                                0xd
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0_MASK                                0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0_MASK                                     0x00001000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0_MASK                                  0x01FFE000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0__SHIFT                            0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0__SHIFT                            0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0_MASK                              0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0_MASK                              0x000FF000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP19
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F0__SHIFT                          0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F0_MASK                            0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F0_MASK                                  0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0__SHIFT                      0x1
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0__SHIFT                     0x2
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DEVICE3_EN_DEV0_F0__SHIFT                                      0x3
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0__SHIFT                    0x4
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0_MASK                        0x00000002L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0_MASK                       0x00000004L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DEVICE3_EN_DEV0_F0_MASK                                        0x00000008L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0_MASK                      0x00000010L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP22
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0__SHIFT                             0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_MEM_AP_SIZE_DEV0_F0__SHIFT                                    0xc
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_VF_MEM_AP_SIZE_DEV0_F0__SHIFT                                 0x11
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_PF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0__SHIFT                  0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_VF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0__SHIFT                  0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0_MASK                               0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_MEM_AP_SIZE_DEV0_F0_MASK                                      0x0001F000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_VF_MEM_AP_SIZE_DEV0_F0_MASK                                   0x003E0000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_PF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0_MASK                    0x07C00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP22__STRAP_VF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0_MASK                    0xF8000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP23
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP23__STRAP_DSN_CODE_L_DEV0_F0__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP23__STRAP_DSN_CODE_L_DEV0_F0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP24
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP24__STRAP_DSN_CODE_H_DEV0_F0__SHIFT                                     0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP24__STRAP_DSN_CODE_H_DEV0_F0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0__SHIFT             0x19
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0_MASK               0x02000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0__SHIFT                                  0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0_MASK                                    0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0_MASK                                       0x38000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0__SHIFT                                0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0_MASK                                  0x38000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP0_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F0__SHIFT                                  0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+#define RCC_STRAP0_RCC_DEV0_EPF0_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F0_MASK                                    0x10000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP1
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F1__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F1__SHIFT                       0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F1_MASK                                0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F1_MASK                         0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP10
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP11
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP12
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP13
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F1__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F1__SHIFT                                 0x8
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F1__SHIFT                                0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F1__SHIFT                                0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F1_MASK                                   0x000000FFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F1_MASK                                   0x0000FF00L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F1_MASK                                  0x00FF0000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F1_MASK                                  0xFF000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP14__STRAP_VENDOR_ID_DEV0_F1__SHIFT                                      0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP14__STRAP_VENDOR_ID_DEV0_F1_MASK                                        0x0000FFFFL
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP15
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F1__SHIFT                                 0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F1__SHIFT                                  0xc
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_VALID_DEV0_F1__SHIFT                                      0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F1_MASK                                   0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F1_MASK                                    0x00FFF000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_VALID_DEV0_F1_MASK                                        0x01000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP16
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F1__SHIFT                               0xc
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F1_MASK                                     0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F1_MASK                                 0x00FFF000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP17
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F1__SHIFT                              0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_VALID_DEV0_F1__SHIFT                                   0xc
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F1__SHIFT                                0xd
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F1_MASK                                0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_VALID_DEV0_F1_MASK                                     0x00001000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F1_MASK                                  0x01FFE000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F1__SHIFT                            0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F1__SHIFT                            0xc
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F1_MASK                              0x00000FFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F1_MASK                              0x000FF000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP19
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F1__SHIFT                          0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F1__SHIFT                                0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F1_MASK                            0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F1_MASK                                  0xFFFF0000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_SRIOV_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F1__SHIFT                     0x2
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DEVICE3_EN_DEV0_F1__SHIFT                                      0x3
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1__SHIFT                    0x4
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_SRIOV_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F1_MASK                       0x00000004L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DEVICE3_EN_DEV0_F1_MASK                                        0x00000008L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1_MASK                      0x00000010L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F1_MASK                                       0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP20
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP21
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP23
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP24
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1__SHIFT             0x19
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F1__SHIFT                                0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1_MASK               0x02000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F1_MASK                                  0x10000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x1
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1__SHIFT                                  0x2
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1__SHIFT                                        0x8
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x9
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1__SHIFT                                  0xa
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1__SHIFT                                        0x10
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x11
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1__SHIFT                                  0x12
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1__SHIFT                                        0x18
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x19
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1__SHIFT                                        0x1a
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV0_F1_MASK                             0x00000002L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1_MASK                                    0x00000004L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1_MASK                                          0x00000100L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV0_F1_MASK                             0x00000200L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1_MASK                                    0x00000400L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1_MASK                                          0x00010000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_PREFETCHABLE_EN_DEV0_F1_MASK                             0x00020000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1_MASK                                    0x00040000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1_MASK                                          0x01000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_PREFETCHABLE_EN_DEV0_F1_MASK                             0x02000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1_MASK                                          0x04000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_PREFETCHABLE_EN_DEV0_F1_MASK                             0x08000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP7
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP8
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F1__SHIFT                                0x1b
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F1__SHIFT                           0x1e
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F1_MASK                                  0x38000000L
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F1_MASK                             0xC0000000L
+//RCC_STRAP0_RCC_DEV0_EPF1_STRAP9
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F1__SHIFT                           0x0
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F1__SHIFT                                  0x1c
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F1_MASK                             0x0000FFFFL
+#define RCC_STRAP0_RCC_DEV0_EPF1_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F1_MASK                                    0x10000000L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_pf_BIFPFVFDEC1
+//BIF_BX_PF0_BIF_BME_STATUS
+#define BIF_BX_PF0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                                      0x0
+#define BIF_BX_PF0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                                0x10
+#define BIF_BX_PF0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                                        0x00000001L
+#define BIF_BX_PF0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                                  0x00010000L
+//BIF_BX_PF0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                                0x0
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                             0x1
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                                0x2
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                                    0x3
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                          0x10
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                                       0x11
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                          0x12
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                              0x13
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                                  0x00000001L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                               0x00000002L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                                  0x00000004L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                                      0x00000008L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                            0x00010000L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                                         0x00020000L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                            0x00040000L
+#define BIF_BX_PF0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                                0x00080000L
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT          0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK            0xFFFFFFFFL
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT            0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK              0xFFFFFFFFL
+//BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT                      0x0
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT                    0x1
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT                    0x8
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK                        0x00000001L
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK                      0x00000002L
+#define BIF_BX_PF0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK                      0x000FFF00L
+//BIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                          0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                            0x00000001L
+//BIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_PF0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_PF0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                              0x0
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                              0x1
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                              0x2
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                              0x3
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                              0x4
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                              0x5
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                              0x6
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                              0x7
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                              0x8
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                              0x9
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                            0xa
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                            0xb
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                                        0xc
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                                        0xd
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                                        0xe
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                                        0xf
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                                        0x10
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                                        0x11
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                                        0x12
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                                        0x13
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                                        0x14
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                                        0x15
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                                       0x16
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                                       0x17
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                                       0x18
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                                       0x19
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                                       0x1a
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                                       0x1b
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                                       0x1c
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                                       0x1d
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                                       0x1e
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                                       0x1f
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                                0x00000001L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                                0x00000002L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                                0x00000004L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                                0x00000008L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                                0x00000010L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                                0x00000020L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                                0x00000040L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                                0x00000080L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                                0x00000100L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                                0x00000200L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                              0x00000400L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                              0x00000800L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                          0x00001000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                          0x00002000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                          0x00004000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                          0x00008000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                          0x00010000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                          0x00020000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                          0x00040000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                          0x00080000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                          0x00100000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                          0x00200000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                                         0x00400000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                                         0x00800000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                                         0x01000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                                         0x02000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                                         0x04000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                                         0x08000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                                         0x10000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                                         0x20000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                                         0x40000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                                         0x80000000L
+//BIF_BX_PF0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                             0x0
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                             0x1
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                             0x2
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                             0x3
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                             0x4
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                             0x5
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                             0x6
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                             0x7
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                             0x8
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                             0x9
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                           0xa
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                           0xb
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                                       0xc
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                                       0xd
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                                       0xe
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                                       0xf
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                                       0x10
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                                       0x11
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                                       0x12
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                                       0x13
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                                       0x14
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                                       0x15
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                                      0x16
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                                      0x17
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                                      0x18
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                                      0x19
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                                      0x1a
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                                      0x1b
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                                      0x1c
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                                      0x1d
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                                      0x1e
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                                      0x1f
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                               0x00000001L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                               0x00000002L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                               0x00000004L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                               0x00000008L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                               0x00000010L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                               0x00000020L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                               0x00000040L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                               0x00000080L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                               0x00000100L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                               0x00000200L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                             0x00000400L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                             0x00000800L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                                         0x00001000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                                         0x00002000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                                         0x00004000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                                         0x00008000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                                         0x00010000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                                         0x00020000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                                         0x00040000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                                         0x00080000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                                         0x00100000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                                         0x00200000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                                        0x00400000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                                        0x00800000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                                        0x01000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                                        0x02000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                                        0x04000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                                        0x08000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                                        0x10000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                                        0x20000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                                        0x40000000L
+#define BIF_BX_PF0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                                        0x80000000L
+//BIF_BX_PF0_BIF_TRANS_PENDING
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                            0x0
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                            0x1
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                              0x00000001L
+#define BIF_BX_PF0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                              0x00000002L
+//BIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                                0x0
+#define BIF_BX_PF0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                                  0x00000001L
+//BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF0_MAILBOX_CONTROL
+#define BIF_BX_PF0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                                      0x0
+#define BIF_BX_PF0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                                        0x1
+#define BIF_BX_PF0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                                      0x8
+#define BIF_BX_PF0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                                        0x9
+#define BIF_BX_PF0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                                        0x00000001L
+#define BIF_BX_PF0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                          0x00000002L
+#define BIF_BX_PF0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                                        0x00000100L
+#define BIF_BX_PF0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                          0x00000200L
+//BIF_BX_PF0_MAILBOX_INT_CNTL
+#define BIF_BX_PF0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                                      0x0
+#define BIF_BX_PF0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                                        0x1
+#define BIF_BX_PF0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                                        0x00000001L
+#define BIF_BX_PF0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                          0x00000002L
+//BIF_BX_PF0_BIF_VMHV_MAILBOX
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                                      0x0
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                                    0x1
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                                         0x8
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                                        0xf
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                                         0x10
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                                        0x17
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                          0x18
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                          0x19
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                                        0x00000001L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                                      0x00000002L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                           0x00000F00L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                          0x00008000L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                           0x000F0000L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                          0x00800000L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                            0x01000000L
+#define BIF_BX_PF0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                            0x02000000L
+//BIF_BX_PF0_PARTITION_COMPUTE_CAP
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                                  0x0
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                                  0x1
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                                  0x2
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                                  0x3
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                                  0x4
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                             0xa
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                                    0x00000001L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                                    0x00000002L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                                    0x00000004L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                                    0x00000008L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                                    0x00000010L
+#define BIF_BX_PF0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                               0x0003FC00L
+//BIF_BX_PF0_PARTITION_MEM_CAP
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                                     0x0
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                                     0x1
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                                     0x2
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                                     0x3
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                                     0x5
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                                     0x7
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                                       0x00000001L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                                       0x00000002L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                                       0x00000004L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                                       0x00000008L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                                       0x00000020L
+#define BIF_BX_PF0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                                       0x00000080L
+//BIF_BX_PF0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_PF0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                            0x4
+#define BIF_BX_PF0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                              0x000000F0L
+//BIF_BX_PF0_PARTITION_MEM_STATUS
+#define BIF_BX_PF0_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                                 0x0
+#define BIF_BX_PF0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                                      0x4
+#define BIF_BX_PF0_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                                   0x0000000FL
+#define BIF_BX_PF0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                                        0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_BIFPFVFDEC1
+//RCC_DEV0_EPF0_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                                  0x0
+#define RCC_DEV0_EPF0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                         0x1
+#define RCC_DEV0_EPF0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                    0x00000001L
+#define RCC_DEV0_EPF0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                           0x00000002L
+//RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                       0x0
+#define RCC_DEV0_EPF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                         0x00000001L
+//RCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                               0x0
+#define RCC_DEV0_EPF0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                                 0xFFFFFFFFL
+//RCC_DEV0_EPF0_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                              0x1f
+#define RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                           0x00000001L
+#define RCC_DEV0_EPF0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                                0x80000000L
+
+
+// addressBlock: nbif0_nbif0_gdc_GDCDEC
+//GDC0_NGDC_SDP_PORT_CTRL
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS__SHIFT                                                 0x0
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H__SHIFT                                               0x10
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_DIS__SHIFT                                                        0x1f
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_MASK                                                   0x000000FFL
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H_MASK                                                 0x000F0000L
+#define GDC0_NGDC_SDP_PORT_CTRL__SDP_DISCON_DIS_MASK                                                          0x80000000L
+//GDC0_SHUB_REGS_IF_CTL
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS__SHIFT                                0x0
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS__SHIFT                                             0x1
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS_MASK                                  0x00000001L
+#define GDC0_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS_MASK                                               0x00000002L
+//GDC0_NGDC_MGCG_CTRL
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_EN__SHIFT                                                              0x0
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_MODE__SHIFT                                                            0x1
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS__SHIFT                                                      0x2
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS__SHIFT                                                         0xa
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS__SHIFT                                                         0xb
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS__SHIFT                                                         0xc
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS__SHIFT                                                         0xd
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DBG_DIS__SHIFT                                                         0xe
+#define GDC0_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN__SHIFT                                                         0xf
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_EN_MASK                                                                0x00000001L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_MODE_MASK                                                              0x00000002L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS_MASK                                                        0x000003FCL
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS_MASK                                                           0x00000400L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS_MASK                                                           0x00000800L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS_MASK                                                           0x00001000L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS_MASK                                                           0x00002000L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_MGCG_DBG_DIS_MASK                                                           0x00004000L
+#define GDC0_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN_MASK                                                           0x00008000L
+//GDC0_S2A_MISC_CNTL
+#define GDC0_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS__SHIFT                                                         0x3
+#define GDC0_S2A_MISC_CNTL__ATM_ARB_MODE__SHIFT                                                               0x8
+#define GDC0_S2A_MISC_CNTL__RB_ARB_MODE__SHIFT                                                                0xa
+#define GDC0_S2A_MISC_CNTL__HSTR_ARB_MODE__SHIFT                                                              0xc
+#define GDC0_S2A_MISC_CNTL__HDP_PERF_ENH_DIS__SHIFT                                                           0xf
+#define GDC0_S2A_MISC_CNTL__WRSP_ARB_MODE__SHIFT                                                              0x10
+#define GDC0_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS_MASK                                                           0x00000008L
+#define GDC0_S2A_MISC_CNTL__ATM_ARB_MODE_MASK                                                                 0x00000300L
+#define GDC0_S2A_MISC_CNTL__RB_ARB_MODE_MASK                                                                  0x00000C00L
+#define GDC0_S2A_MISC_CNTL__HSTR_ARB_MODE_MASK                                                                0x00003000L
+#define GDC0_S2A_MISC_CNTL__HDP_PERF_ENH_DIS_MASK                                                             0x00008000L
+#define GDC0_S2A_MISC_CNTL__WRSP_ARB_MODE_MASK                                                                0x000F0000L
+//GDC0_NGDC_MCA_SMN_CTRL0
+#define GDC0_NGDC_MCA_SMN_CTRL0__MCA_SMN_POSTED__SHIFT                                                        0x0
+#define GDC0_NGDC_MCA_SMN_CTRL0__MCA_SMN_POSTED_MASK                                                          0x00000001L
+//GDC0_NGDC_EARLY_WAKEUP_CTRL
+#define GDC0_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_ACTIVE__SHIFT                                0x0
+#define GDC0_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_DS_EXIT__SHIFT                               0x1
+#define GDC0_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_ALLOW_AER_ACTIVE__SHIFT                                0x2
+#define GDC0_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_ACTIVE_MASK                                  0x00000001L
+#define GDC0_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_DS_EXIT_MASK                                 0x00000002L
+#define GDC0_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_ALLOW_AER_ACTIVE_MASK                                  0x00000004L
+//GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_BYPASS__SHIFT                                  0x0
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_ACCUM_SEL__SHIFT                               0x1
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_FORCE_EN__SHIFT                                0x2
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_FORCE_DATA__SHIFT                              0x3
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_STATUS_ACCUM_EN__SHIFT                         0x4
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_DATASTATUS_ACCUM_EN__SHIFT                     0x5
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_BYPASS_MASK                                    0x00000001L
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_ACCUM_SEL_MASK                                 0x00000002L
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_FORCE_EN_MASK                                  0x00000004L
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_FORCE_DATA_MASK                                0x00000008L
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_STATUS_ACCUM_EN_MASK                           0x00000010L
+#define GDC0_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_DATASTATUS_ACCUM_EN_MASK                       0x00000020L
+//GDC0_GDC_DMA_SION_PCTRL
+#define GDC0_GDC_DMA_SION_PCTRL__BIF_SDP_DISCON_DIS__SHIFT                                                    0x0
+#define GDC0_GDC_DMA_SION_PCTRL__SYSHUB_SDP_DISCON_DIS__SHIFT                                                 0x3
+#define GDC0_GDC_DMA_SION_PCTRL__INT_SDP_DISCON_DIS__SHIFT                                                    0x6
+#define GDC0_GDC_DMA_SION_PCTRL__BIF_SDP_DISCON_DIS_MASK                                                      0x00000001L
+#define GDC0_GDC_DMA_SION_PCTRL__SYSHUB_SDP_DISCON_DIS_MASK                                                   0x00000008L
+#define GDC0_GDC_DMA_SION_PCTRL__INT_SDP_DISCON_DIS_MASK                                                      0x00000040L
+//GDC0_NGDC_MP4SDP_CNTL
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC0_OVERRIDE_EN__SHIFT                                           0x0
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC0_OVERRIDE_VAL__SHIFT                                          0x1
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC1_OVERRIDE_EN__SHIFT                                           0x4
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC1_OVERRIDE_VAL__SHIFT                                          0x5
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC4_OVERRIDE_EN__SHIFT                                           0x8
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC4_OVERRIDE_VAL__SHIFT                                          0x9
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC0_OVERRIDE_EN_MASK                                             0x00000001L
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC0_OVERRIDE_VAL_MASK                                            0x00000006L
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC1_OVERRIDE_EN_MASK                                             0x00000010L
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC1_OVERRIDE_VAL_MASK                                            0x00000060L
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC4_OVERRIDE_EN_MASK                                             0x00000100L
+#define GDC0_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC4_OVERRIDE_VAL_MASK                                            0x00000600L
+//GDC0_DOORBELL_VCN_TARGET_VF0
+#define GDC0_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN0_TARGET_VF0__SHIFT                                         0x0
+#define GDC0_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN1_TARGET_VF0__SHIFT                                         0x4
+#define GDC0_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN2_TARGET_VF0__SHIFT                                         0x8
+#define GDC0_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN3_TARGET_VF0__SHIFT                                         0xc
+#define GDC0_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN0_TARGET_VF0_MASK                                           0x000FL
+#define GDC0_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN1_TARGET_VF0_MASK                                           0x00F0L
+#define GDC0_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN2_TARGET_VF0_MASK                                           0x0F00L
+#define GDC0_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN3_TARGET_VF0_MASK                                           0xF000L
+//GDC0_DOORBELL_VCN_TARGET_VF1
+#define GDC0_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN0_TARGET_VF1__SHIFT                                         0x0
+#define GDC0_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN1_TARGET_VF1__SHIFT                                         0x4
+#define GDC0_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN2_TARGET_VF1__SHIFT                                         0x8
+#define GDC0_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN3_TARGET_VF1__SHIFT                                         0xc
+#define GDC0_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN0_TARGET_VF1_MASK                                           0x000FL
+#define GDC0_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN1_TARGET_VF1_MASK                                           0x00F0L
+#define GDC0_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN2_TARGET_VF1_MASK                                           0x0F00L
+#define GDC0_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN3_TARGET_VF1_MASK                                           0xF000L
+//GDC0_DOORBELL_VCN_TARGET_VF2
+#define GDC0_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN0_TARGET_VF2__SHIFT                                         0x0
+#define GDC0_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN1_TARGET_VF2__SHIFT                                         0x4
+#define GDC0_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN2_TARGET_VF2__SHIFT                                         0x8
+#define GDC0_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN3_TARGET_VF2__SHIFT                                         0xc
+#define GDC0_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN0_TARGET_VF2_MASK                                           0x000FL
+#define GDC0_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN1_TARGET_VF2_MASK                                           0x00F0L
+#define GDC0_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN2_TARGET_VF2_MASK                                           0x0F00L
+#define GDC0_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN3_TARGET_VF2_MASK                                           0xF000L
+//GDC0_DOORBELL_VCN_TARGET_VF3
+#define GDC0_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN0_TARGET_VF3__SHIFT                                         0x0
+#define GDC0_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN1_TARGET_VF3__SHIFT                                         0x4
+#define GDC0_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN2_TARGET_VF3__SHIFT                                         0x8
+#define GDC0_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN3_TARGET_VF3__SHIFT                                         0xc
+#define GDC0_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN0_TARGET_VF3_MASK                                           0x000FL
+#define GDC0_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN1_TARGET_VF3_MASK                                           0x00F0L
+#define GDC0_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN2_TARGET_VF3_MASK                                           0x0F00L
+#define GDC0_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN3_TARGET_VF3_MASK                                           0xF000L
+//GDC0_DOORBELL_VCN_TARGET_VF4
+#define GDC0_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN0_TARGET_VF4__SHIFT                                         0x0
+#define GDC0_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN1_TARGET_VF4__SHIFT                                         0x4
+#define GDC0_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN2_TARGET_VF4__SHIFT                                         0x8
+#define GDC0_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN3_TARGET_VF4__SHIFT                                         0xc
+#define GDC0_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN0_TARGET_VF4_MASK                                           0x000FL
+#define GDC0_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN1_TARGET_VF4_MASK                                           0x00F0L
+#define GDC0_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN2_TARGET_VF4_MASK                                           0x0F00L
+#define GDC0_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN3_TARGET_VF4_MASK                                           0xF000L
+//GDC0_DOORBELL_VCN_TARGET_VF5
+#define GDC0_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN0_TARGET_VF5__SHIFT                                         0x0
+#define GDC0_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN1_TARGET_VF5__SHIFT                                         0x4
+#define GDC0_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN2_TARGET_VF5__SHIFT                                         0x8
+#define GDC0_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN3_TARGET_VF5__SHIFT                                         0xc
+#define GDC0_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN0_TARGET_VF5_MASK                                           0x000FL
+#define GDC0_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN1_TARGET_VF5_MASK                                           0x00F0L
+#define GDC0_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN2_TARGET_VF5_MASK                                           0x0F00L
+#define GDC0_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN3_TARGET_VF5_MASK                                           0xF000L
+//GDC0_DOORBELL_VCN_TARGET_VF6
+#define GDC0_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN0_TARGET_VF6__SHIFT                                         0x0
+#define GDC0_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN1_TARGET_VF6__SHIFT                                         0x4
+#define GDC0_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN2_TARGET_VF6__SHIFT                                         0x8
+#define GDC0_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN3_TARGET_VF6__SHIFT                                         0xc
+#define GDC0_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN0_TARGET_VF6_MASK                                           0x000FL
+#define GDC0_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN1_TARGET_VF6_MASK                                           0x00F0L
+#define GDC0_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN2_TARGET_VF6_MASK                                           0x0F00L
+#define GDC0_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN3_TARGET_VF6_MASK                                           0xF000L
+//GDC0_DOORBELL_VCN_TARGET_VF7
+#define GDC0_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN0_TARGET_VF7__SHIFT                                         0x0
+#define GDC0_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN1_TARGET_VF7__SHIFT                                         0x4
+#define GDC0_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN2_TARGET_VF7__SHIFT                                         0x8
+#define GDC0_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN3_TARGET_VF7__SHIFT                                         0xc
+#define GDC0_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN0_TARGET_VF7_MASK                                           0x000FL
+#define GDC0_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN1_TARGET_VF7_MASK                                           0x00F0L
+#define GDC0_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN2_TARGET_VF7_MASK                                           0x0F00L
+#define GDC0_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN3_TARGET_VF7_MASK                                           0xF000L
+//GDC0_DOORBELL_ACCESS_EN_PF
+#define GDC0_DOORBELL_ACCESS_EN_PF__DOORBELL_ACCESS_EN_PF__SHIFT                                              0x0
+#define GDC0_DOORBELL_ACCESS_EN_PF__DOORBELL_ACCESS_EN_PF_MASK                                                0x00FFL
+//GDC0_DOORBELL_ACCESS_EN_VF0
+#define GDC0_DOORBELL_ACCESS_EN_VF0__DOORBELL_ACCESS_EN_VF0__SHIFT                                            0x0
+#define GDC0_DOORBELL_ACCESS_EN_VF0__XCD_ACCESS_EN_VF0__SHIFT                                                 0x8
+#define GDC0_DOORBELL_ACCESS_EN_VF0__DOORBELL_ACCESS_EN_VF0_MASK                                              0x00FFL
+#define GDC0_DOORBELL_ACCESS_EN_VF0__XCD_ACCESS_EN_VF0_MASK                                                   0xFF00L
+//GDC0_DOORBELL_ACCESS_EN_VF1
+#define GDC0_DOORBELL_ACCESS_EN_VF1__DOORBELL_ACCESS_EN_VF1__SHIFT                                            0x0
+#define GDC0_DOORBELL_ACCESS_EN_VF1__XCD_ACCESS_EN_VF1__SHIFT                                                 0x8
+#define GDC0_DOORBELL_ACCESS_EN_VF1__DOORBELL_ACCESS_EN_VF1_MASK                                              0x00FFL
+#define GDC0_DOORBELL_ACCESS_EN_VF1__XCD_ACCESS_EN_VF1_MASK                                                   0xFF00L
+//GDC0_DOORBELL_ACCESS_EN_VF2
+#define GDC0_DOORBELL_ACCESS_EN_VF2__DOORBELL_ACCESS_EN_VF2__SHIFT                                            0x0
+#define GDC0_DOORBELL_ACCESS_EN_VF2__XCD_ACCESS_EN_VF2__SHIFT                                                 0x8
+#define GDC0_DOORBELL_ACCESS_EN_VF2__DOORBELL_ACCESS_EN_VF2_MASK                                              0x00FFL
+#define GDC0_DOORBELL_ACCESS_EN_VF2__XCD_ACCESS_EN_VF2_MASK                                                   0xFF00L
+//GDC0_DOORBELL_ACCESS_EN_VF3
+#define GDC0_DOORBELL_ACCESS_EN_VF3__DOORBELL_ACCESS_EN_VF3__SHIFT                                            0x0
+#define GDC0_DOORBELL_ACCESS_EN_VF3__XCD_ACCESS_EN_VF3__SHIFT                                                 0x8
+#define GDC0_DOORBELL_ACCESS_EN_VF3__DOORBELL_ACCESS_EN_VF3_MASK                                              0x00FFL
+#define GDC0_DOORBELL_ACCESS_EN_VF3__XCD_ACCESS_EN_VF3_MASK                                                   0xFF00L
+//GDC0_DOORBELL_ACCESS_EN_VF4
+#define GDC0_DOORBELL_ACCESS_EN_VF4__DOORBELL_ACCESS_EN_VF4__SHIFT                                            0x0
+#define GDC0_DOORBELL_ACCESS_EN_VF4__XCD_ACCESS_EN_VF4__SHIFT                                                 0x8
+#define GDC0_DOORBELL_ACCESS_EN_VF4__DOORBELL_ACCESS_EN_VF4_MASK                                              0x00FFL
+#define GDC0_DOORBELL_ACCESS_EN_VF4__XCD_ACCESS_EN_VF4_MASK                                                   0xFF00L
+//GDC0_DOORBELL_ACCESS_EN_VF5
+#define GDC0_DOORBELL_ACCESS_EN_VF5__DOORBELL_ACCESS_EN_VF5__SHIFT                                            0x0
+#define GDC0_DOORBELL_ACCESS_EN_VF5__XCD_ACCESS_EN_VF5__SHIFT                                                 0x8
+#define GDC0_DOORBELL_ACCESS_EN_VF5__DOORBELL_ACCESS_EN_VF5_MASK                                              0x00FFL
+#define GDC0_DOORBELL_ACCESS_EN_VF5__XCD_ACCESS_EN_VF5_MASK                                                   0xFF00L
+//GDC0_DOORBELL_ACCESS_EN_VF6
+#define GDC0_DOORBELL_ACCESS_EN_VF6__DOORBELL_ACCESS_EN_VF6__SHIFT                                            0x0
+#define GDC0_DOORBELL_ACCESS_EN_VF6__XCD_ACCESS_EN_VF6__SHIFT                                                 0x8
+#define GDC0_DOORBELL_ACCESS_EN_VF6__DOORBELL_ACCESS_EN_VF6_MASK                                              0x00FFL
+#define GDC0_DOORBELL_ACCESS_EN_VF6__XCD_ACCESS_EN_VF6_MASK                                                   0xFF00L
+//GDC0_DOORBELL_ACCESS_EN_VF7
+#define GDC0_DOORBELL_ACCESS_EN_VF7__DOORBELL_ACCESS_EN_VF7__SHIFT                                            0x0
+#define GDC0_DOORBELL_ACCESS_EN_VF7__XCD_ACCESS_EN_VF7__SHIFT                                                 0x8
+#define GDC0_DOORBELL_ACCESS_EN_VF7__DOORBELL_ACCESS_EN_VF7_MASK                                              0x00FFL
+#define GDC0_DOORBELL_ACCESS_EN_VF7__XCD_ACCESS_EN_VF7_MASK                                                   0xFF00L
+//GDC0_NGDC_CNDI_BUS_PORT_CTRL
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_DISCON_HYSTERESIS__SHIFT                                      0x0
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_DISCON_DIS_SOCCLK__SHIFT                                      0xc
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_CMPL_DISCON_HYSTERESIS__SHIFT                                      0xd
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_CMPL_DISCON_DIS_SOCCLK__SHIFT                                      0x19
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_PORT_IDLE__SHIFT                                              0x1a
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__SHUB_CNDI_VDCI_CKEN_MASK__SHIFT                                         0x1b
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_DISCON_HYSTERESIS_MASK                                        0x00000FFFL
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_DISCON_DIS_SOCCLK_MASK                                        0x00001000L
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_CMPL_DISCON_HYSTERESIS_MASK                                        0x01FFE000L
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_CMPL_DISCON_DIS_SOCCLK_MASK                                        0x02000000L
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_PORT_IDLE_MASK                                                0x04000000L
+#define GDC0_NGDC_CNDI_BUS_PORT_CTRL__SHUB_CNDI_VDCI_CKEN_MASK_MASK                                           0x08000000L
+//GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOLCRED_ALLOC_VC0_ALLOC__SHIFT                    0x0
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOLCRED_ALLOC_VC1_ALLOC__SHIFT                    0x4
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOLCRED_ALLOC_VC0_ALLOC__SHIFT                    0x8
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOLCRED_ALLOC_VC1_ALLOC__SHIFT                    0xc
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_ORIG_REQ_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT            0x10
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_ORIG_ORIGDATA_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT       0x11
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT          0x12
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT          0x13
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOLCRED_ALLOC_VC0_ALLOC_MASK                      0x0000000FL
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOLCRED_ALLOC_VC1_ALLOC_MASK                      0x000000F0L
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOLCRED_ALLOC_VC0_ALLOC_MASK                      0x00000F00L
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOLCRED_ALLOC_VC1_ALLOC_MASK                      0x0000F000L
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_ORIG_REQ_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK              0x00010000L
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_ORIG_ORIGDATA_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK         0x00020000L
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK            0x00040000L
+#define GDC0_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK            0x00080000L
+
+
+// addressBlock: nbif0_nbif0_gdc_s2a_GDCS2A_DEC
+//GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE__SHIFT                                  0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID__SHIFT                                    0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID_MASK                                      0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE__SHIFT                                0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID__SHIFT                                  0x1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID_MASK                                    0x0000003EL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A0_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG
+#define GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS__SHIFT                     0x0
+#define GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBE_FENCE_INTR_ENABLE__SHIFT                            0x1
+#define GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS_MASK                       0x00000001L
+#define GDC_S2A0_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBE_FENCE_INTR_ENABLE_MASK                              0x00000002L
+//GDC_S2A0_NBIF_GFX_DOORBELL_STATUS
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT__SHIFT                                      0x0
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_EN__SHIFT                                     0x10
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_ST__SHIFT                                     0x18
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT_MASK                                        0x0000FFFFL
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_EN_MASK                                       0x00010000L
+#define GDC_S2A0_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_ST_MASK                                       0x01000000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF0_COMMAND
+#define BIF_CFG_DEV0_EPF0_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF0_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF0_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF0_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF0_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF0_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF0_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF0_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF0_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF0_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF0_STATUS
+#define BIF_CFG_DEV0_EPF0_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF0_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF0_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF0_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF0_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF0_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF0_LATENCY
+#define BIF_CFG_DEV0_EPF0_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_HEADER
+#define BIF_CFG_DEV0_EPF0_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF0_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF0_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF0_BIST
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF0_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF0_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//VENDOR_CAP_LIST
+#define VENDOR_CAP_LIST__CAP_ID__SHIFT                                                                        0x0
+#define VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                                      0x8
+#define VENDOR_CAP_LIST__LENGTH__SHIFT                                                                        0x10
+#define VENDOR_CAP_LIST__CAP_ID_MASK                                                                          0x000000FFL
+#define VENDOR_CAP_LIST__NEXT_PTR_MASK                                                                        0x0000FF00L
+#define VENDOR_CAP_LIST__LENGTH_MASK                                                                          0x00FF0000L
+//ADAPTER_ID_W
+#define ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                                              0x0
+#define ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                                     0x10
+#define ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                                                0x0000FFFFL
+#define ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF0_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF0_PMI_CAP
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF0_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+#define BIF_CFG_DEV0_EPF0_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                     0x11
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                              0x1d
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                                   0x1e
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__RX_MPS_FIXED_MASK                                                       0x00020000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                                0x20000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                     0x40000000L
+//BIF_CFG_DEV0_EPF0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF0_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                       0x1b
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                                  0x1c
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                                 0x1d
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                                 0x1e
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                         0x08000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                    0x10000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                                   0x20000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                                   0x40000000L
+//BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                       0xd
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                              0x12
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                       0x13
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                         0x0003E000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                                0x00040000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                         0x00F80000L
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG4__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG5__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG6__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG7__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG8__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG9__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG10__TLP_HDR_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG11__TLP_HDR_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG12__TLP_HDR_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_HDR_LOG13__TLP_HDR_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                    0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                   0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                       0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                      0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                     0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV0_EPF0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV0_EPF0_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                                          0x10
+#define PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                         0x14
+#define PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                                             0x0000FFFFL
+#define PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                                            0x000F0000L
+#define PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                                           0xFFF00000L
+//PCIE_PWR_BUDGET_DATA_SELECT
+#define PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                                       0x0
+#define PCIE_PWR_BUDGET_DATA_SELECT__EXTENDED_PWR_BUDGET_ENABLE__SHIFT                                        0x10
+#define PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                                         0x000000FFL
+#define PCIE_PWR_BUDGET_DATA_SELECT__EXTENDED_PWR_BUDGET_ENABLE_MASK                                          0x00010000L
+//PCIE_PWR_BUDGET_DATA
+#define PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                                               0x0
+#define PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                                               0x8
+#define PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                                             0xa
+#define PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                                                 0xd
+#define PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                                     0xf
+#define PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                                               0x12
+#define PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                                                 0x000000FFL
+#define PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                                                 0x00000300L
+#define PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                                               0x00001C00L
+#define PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                                   0x00006000L
+#define PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                                       0x00038000L
+#define PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                                                 0x001C0000L
+//PCIE_PWR_BUDGET_CAP
+#define PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                                          0x0
+#define PCIE_PWR_BUDGET_CAP__EXTENDED_PWR_BUDGET_SUPPORT__SHIFT                                               0x1
+#define PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                                            0x01L
+#define PCIE_PWR_BUDGET_CAP__EXTENDED_PWR_BUDGET_SUPPORT_MASK                                                 0x02L
+//PCIE_DPA_ENH_CAP_LIST
+#define PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                                  0x0
+#define PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                                                 0x10
+#define PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                                0x14
+#define PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                                    0x0000FFFFL
+#define PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                                   0x000F0000L
+#define PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                                  0xFFF00000L
+//PCIE_DPA_CAP
+#define PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                                     0x0
+#define PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                                   0x8
+#define PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                                  0xc
+#define PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                                  0x10
+#define PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                                  0x18
+#define PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                                       0x0000001FL
+#define PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                                     0x00000300L
+#define PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                                    0x00003000L
+#define PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                                    0x00FF0000L
+#define PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                                    0xFF000000L
+//PCIE_DPA_LATENCY_INDICATOR
+#define PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                                           0x0
+#define PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                                             0x000000FFL
+//PCIE_DPA_STATUS
+#define PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                                               0x0
+#define PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                                         0x8
+#define PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                                                 0x001FL
+#define PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                                           0x0100L
+//PCIE_DPA_CNTL
+#define PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                                   0x0
+#define PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                                     0x001FL
+//PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                                              0x0
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                                                0xFFL
+//PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                                              0x0
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                                                0xFFL
+//PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                                              0x0
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                                                0xFFL
+//PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                                              0x0
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                                                0xFFL
+//PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                                              0x0
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                                                0xFFL
+//PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                                              0x0
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                                                0xFFL
+//PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                                              0x0
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                                                0xFFL
+//PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                                              0x0
+#define PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                                                0xFFL
+//PCIE_LTR_ENH_CAP_LIST
+#define PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                                  0x0
+#define PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                                                 0x10
+#define PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                                0x14
+#define PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                                    0x0000FFFFL
+#define PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                                                   0x000F0000L
+#define PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                                  0xFFF00000L
+//PCIE_LTR_CAP
+#define PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                                          0x0
+#define PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                                          0xa
+#define PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                                         0x10
+#define PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                                         0x1a
+#define PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                                            0x000003FFL
+#define PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                                            0x00001C00L
+#define PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                                           0x03FF0000L
+#define PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                                           0x1C000000L
+//BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//PCIE_BAR_ENH_CAP_LIST
+#define PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                                  0x0
+#define PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                                                 0x10
+#define PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                                0x14
+#define PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                                    0x0000FFFFL
+#define PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                                   0x000F0000L
+#define PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                                  0xFFF00000L
+//PCIE_BAR1_CAP
+#define PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                                              0x4
+#define PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                                                0xFFFFFFF0L
+//PCIE_BAR1_CNTL
+#define PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                                      0x0
+#define PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                                  0x5
+#define PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                                       0x8
+#define PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                                       0x10
+#define PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                                        0x00000007L
+#define PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                                    0x000000E0L
+#define PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                                         0x00003F00L
+#define PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                                         0xFFFF0000L
+//PCIE_BAR2_CAP
+#define PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                                              0x4
+#define PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                                                0xFFFFFFF0L
+//PCIE_BAR2_CNTL
+#define PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                                      0x0
+#define PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                                  0x5
+#define PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                                       0x8
+#define PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                                       0x10
+#define PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                                        0x00000007L
+#define PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                                    0x000000E0L
+#define PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                                         0x00003F00L
+#define PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                                         0xFFFF0000L
+//PCIE_BAR3_CAP
+#define PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                                              0x4
+#define PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                                                0xFFFFFFF0L
+//PCIE_BAR3_CNTL
+#define PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                                      0x0
+#define PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                                  0x5
+#define PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                                       0x8
+#define PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                                       0x10
+#define PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                                        0x00000007L
+#define PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                                    0x000000E0L
+#define PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                                         0x00003F00L
+#define PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                                         0xFFFF0000L
+//PCIE_BAR4_CAP
+#define PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                                              0x4
+#define PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                                                0xFFFFFFF0L
+//PCIE_BAR4_CNTL
+#define PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                                      0x0
+#define PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                                  0x5
+#define PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                                       0x8
+#define PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                                       0x10
+#define PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                                        0x00000007L
+#define PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                                    0x000000E0L
+#define PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                                         0x00003F00L
+#define PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                                         0xFFFF0000L
+//PCIE_BAR5_CAP
+#define PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                                              0x4
+#define PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                                                0xFFFFFFF0L
+//PCIE_BAR5_CNTL
+#define PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                                      0x0
+#define PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                                  0x5
+#define PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                                       0x8
+#define PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                                       0x10
+#define PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                                        0x00000007L
+#define PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                                    0x000000E0L
+#define PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                                         0x00003F00L
+#define PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                                         0xFFFF0000L
+//PCIE_BAR6_CAP
+#define PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                                              0x4
+#define PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                                                0xFFFFFFF0L
+//PCIE_BAR6_CNTL
+#define PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                                      0x0
+#define PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                                  0x5
+#define PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                                       0x8
+#define PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                                       0x10
+#define PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                                        0x00000007L
+#define PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                                    0x000000E0L
+#define PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                                         0x00003F00L
+#define PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                                         0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                     0x0200L
+//BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__STU__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__STU_MASK                                                             0x001FL
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                      0x8000L
+//PCIE_PAGE_REQ_ENH_CAP_LIST
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID__SHIFT                                                             0x0
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER__SHIFT                                                            0x10
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                           0x14
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID_MASK                                                               0x0000FFFFL
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER_MASK                                                              0x000F0000L
+#define PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR_MASK                                                             0xFFF00000L
+//PCIE_PAGE_REQ_CNTL
+#define PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT                                                                 0x0
+#define PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT                                                                  0x1
+#define PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK                                                                   0x0001L
+#define PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK                                                                    0x0002L
+//PCIE_PAGE_REQ_STATUS
+#define PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE__SHIFT                                                         0x0
+#define PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX__SHIFT                                            0x1
+#define PCIE_PAGE_REQ_STATUS__STOPPED__SHIFT                                                                  0x8
+#define PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED__SHIFT                                              0xf
+#define PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE_MASK                                                           0x0001L
+#define PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX_MASK                                              0x0002L
+#define PCIE_PAGE_REQ_STATUS__STOPPED_MASK                                                                    0x0100L
+#define PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED_MASK                                                0x8000L
+//PCIE_OUTSTAND_PAGE_REQ_CAPACITY
+#define PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY__SHIFT                                    0x0
+#define PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY_MASK                                      0xFFFFFFFFL
+//PCIE_OUTSTAND_PAGE_REQ_ALLOC
+#define PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT                                          0x0
+#define PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK                                            0xFFFFFFFFL
+//PCIE_PASID_ENH_CAP_LIST
+#define PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                                                0x0
+#define PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                                               0x10
+#define PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                              0x14
+#define PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                                  0x0000FFFFL
+#define PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                                                 0x000F0000L
+#define PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                                                0xFFF00000L
+//PCIE_PASID_CAP
+#define PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                                                 0x1
+#define PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                                      0x2
+#define PCIE_PASID_CAP__TRANSLATED_REQ_WITH_PASID_SUPPORTED__SHIFT                                            0x3
+#define PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                                                0x8
+#define PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                                   0x0002L
+#define PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                                        0x0004L
+#define PCIE_PASID_CAP__TRANSLATED_REQ_WITH_PASID_SUPPORTED_MASK                                              0x0008L
+#define PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                                  0x1F00L
+//PCIE_PASID_CNTL
+#define PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                                  0x0
+#define PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                                   0x1
+#define PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                                              0x2
+#define PCIE_PASID_CNTL__TRANSLATED_REQ_WITH_PASID_ENABLE__SHIFT                                              0x3
+#define PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                                    0x0001L
+#define PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                                     0x0002L
+#define PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                                                0x0004L
+#define PCIE_PASID_CNTL__TRANSLATED_REQ_WITH_PASID_ENABLE_MASK                                                0x0008L
+//PCIE_SRIOV_ENH_CAP_LIST
+#define PCIE_SRIOV_ENH_CAP_LIST__CAP_ID__SHIFT                                                                0x0
+#define PCIE_SRIOV_ENH_CAP_LIST__CAP_VER__SHIFT                                                               0x10
+#define PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                              0x14
+#define PCIE_SRIOV_ENH_CAP_LIST__CAP_ID_MASK                                                                  0x0000FFFFL
+#define PCIE_SRIOV_ENH_CAP_LIST__CAP_VER_MASK                                                                 0x000F0000L
+#define PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR_MASK                                                                0xFFF00000L
+//PCIE_SRIOV_CAP
+#define PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP__SHIFT                                                         0x0
+#define PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED__SHIFT                                              0x1
+#define PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                       0x2
+#define PCIE_SRIOV_CAP__SRIOV_VF_14_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                        0x3
+#define PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM__SHIFT                                                0x15
+#define PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP_MASK                                                           0x00000001L
+#define PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED_MASK                                                0x00000002L
+#define PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                         0x00000004L
+#define PCIE_SRIOV_CAP__SRIOV_VF_14_BIT_TAG_REQUESTER_SUPPORTED_MASK                                          0x00000008L
+#define PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM_MASK                                                  0xFFE00000L
+//PCIE_SRIOV_CONTROL
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE__SHIFT                                                            0x0
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE__SHIFT                                                  0x1
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE__SHIFT                                             0x2
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MSE__SHIFT                                                               0x3
+#define PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY__SHIFT                                                    0x4
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                      0x5
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_14_BIT_TAG_REQUESTER_ENABLE__SHIFT                                       0x6
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE_MASK                                                              0x0001L
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE_MASK                                                    0x0002L
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE_MASK                                               0x0004L
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_MSE_MASK                                                                 0x0008L
+#define PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY_MASK                                                      0x0010L
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                        0x0020L
+#define PCIE_SRIOV_CONTROL__SRIOV_VF_14_BIT_TAG_REQUESTER_ENABLE_MASK                                         0x0040L
+//PCIE_SRIOV_STATUS
+#define PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS__SHIFT                                                   0x0
+#define PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS_MASK                                                     0x0001L
+//PCIE_SRIOV_INITIAL_VFS
+#define PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS__SHIFT                                                      0x0
+#define PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS_MASK                                                        0xFFFFL
+//PCIE_SRIOV_TOTAL_VFS
+#define PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS__SHIFT                                                          0x0
+#define PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS_MASK                                                            0xFFFFL
+//PCIE_SRIOV_NUM_VFS
+#define PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS__SHIFT                                                              0x0
+#define PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS_MASK                                                                0xFFFFL
+//PCIE_SRIOV_FUNC_DEP_LINK
+#define PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK__SHIFT                                                  0x0
+#define PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK_MASK                                                    0xFFL
+//PCIE_SRIOV_FIRST_VF_OFFSET
+#define PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET__SHIFT                                              0x0
+#define PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET_MASK                                                0xFFFFL
+//PCIE_SRIOV_VF_STRIDE
+#define PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE__SHIFT                                                          0x0
+#define PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE_MASK                                                            0xFFFFL
+//PCIE_SRIOV_VF_DEVICE_ID
+#define PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID__SHIFT                                                    0x0
+#define PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID_MASK                                                      0xFFFFL
+//PCIE_SRIOV_SUPPORTED_PAGE_SIZE
+#define PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE__SHIFT                                      0x0
+#define PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE_MASK                                        0xFFFFFFFFL
+//PCIE_SRIOV_SYSTEM_PAGE_SIZE
+#define PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE__SHIFT                                            0x0
+#define PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE_MASK                                              0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_0
+#define PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_1
+#define PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_2
+#define PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_3
+#define PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_4
+#define PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_BASE_ADDR_5
+#define PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR__SHIFT                                                        0x0
+#define PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET
+#define PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR__SHIFT                       0x0
+#define PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SHIFT              0x3
+#define PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR_MASK                         0x00000007L
+#define PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_MASK                0xFFFFFFF8L
+//PCIE_VF_RESIZE_BAR_ENH_CAP_LIST
+#define PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                                       0x10
+#define PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                      0x14
+#define PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                          0x0000FFFFL
+#define PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                         0x000F0000L
+#define PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                        0xFFF00000L
+//PCIE_VF_RESIZE_BAR1_CAP
+#define PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                                                 0x4
+#define PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                                   0xFFFFFFF0L
+//PCIE_VF_RESIZE_BAR1_CNTL
+#define PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX__SHIFT                                                         0x0
+#define PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                                     0x5
+#define PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE__SHIFT                                                          0x8
+#define PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                                          0x10
+#define PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX_MASK                                                           0x00000007L
+#define PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM_MASK                                                       0x000000E0L
+#define PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_MASK                                                            0x00003F00L
+#define PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                                            0xFFFF0000L
+//PCIE_VF_RESIZE_BAR2_CAP
+#define PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                                                 0x4
+#define PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                                   0xFFFFFFF0L
+//PCIE_VF_RESIZE_BAR2_CNTL
+#define PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX__SHIFT                                                         0x0
+#define PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                                     0x5
+#define PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE__SHIFT                                                          0x8
+#define PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                                          0x10
+#define PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX_MASK                                                           0x00000007L
+#define PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM_MASK                                                       0x000000E0L
+#define PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_MASK                                                            0x00003F00L
+#define PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                                            0xFFFF0000L
+//PCIE_VF_RESIZE_BAR3_CAP
+#define PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                                                 0x4
+#define PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                                   0xFFFFFFF0L
+//PCIE_VF_RESIZE_BAR3_CNTL
+#define PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX__SHIFT                                                         0x0
+#define PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                                     0x5
+#define PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE__SHIFT                                                          0x8
+#define PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                                          0x10
+#define PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX_MASK                                                           0x00000007L
+#define PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM_MASK                                                       0x000000E0L
+#define PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_MASK                                                            0x00003F00L
+#define PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                                            0xFFFF0000L
+//PCIE_VF_RESIZE_BAR4_CAP
+#define PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                                                 0x4
+#define PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                                   0xFFFFFFF0L
+//PCIE_VF_RESIZE_BAR4_CNTL
+#define PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX__SHIFT                                                         0x0
+#define PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                                     0x5
+#define PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE__SHIFT                                                          0x8
+#define PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                                          0x10
+#define PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX_MASK                                                           0x00000007L
+#define PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM_MASK                                                       0x000000E0L
+#define PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_MASK                                                            0x00003F00L
+#define PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                                            0xFFFF0000L
+//PCIE_VF_RESIZE_BAR5_CAP
+#define PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                                                 0x4
+#define PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                                   0xFFFFFFF0L
+//PCIE_VF_RESIZE_BAR5_CNTL
+#define PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX__SHIFT                                                         0x0
+#define PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                                     0x5
+#define PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE__SHIFT                                                          0x8
+#define PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                                          0x10
+#define PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX_MASK                                                           0x00000007L
+#define PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM_MASK                                                       0x000000E0L
+#define PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_MASK                                                            0x00003F00L
+#define PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                                            0xFFFF0000L
+//PCIE_VF_RESIZE_BAR6_CAP
+#define PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                                                 0x4
+#define PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                                   0xFFFFFFF0L
+//PCIE_VF_RESIZE_BAR6_CNTL
+#define PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX__SHIFT                                                         0x0
+#define PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                                     0x5
+#define PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE__SHIFT                                                          0x8
+#define PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                                          0x10
+#define PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX_MASK                                                           0x00000007L
+#define PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM_MASK                                                       0x000000E0L
+#define PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_MASK                                                            0x00003F00L
+#define PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                                            0xFFFF0000L
+//PCIE_DOE_ENH_CAP_LIST
+#define PCIE_DOE_ENH_CAP_LIST__CAP_ID__SHIFT                                                                  0x0
+#define PCIE_DOE_ENH_CAP_LIST__CAP_VER__SHIFT                                                                 0x10
+#define PCIE_DOE_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                                0x14
+#define PCIE_DOE_ENH_CAP_LIST__CAP_ID_MASK                                                                    0x0000FFFFL
+#define PCIE_DOE_ENH_CAP_LIST__CAP_VER_MASK                                                                   0x000F0000L
+#define PCIE_DOE_ENH_CAP_LIST__NEXT_PTR_MASK                                                                  0xFFF00000L
+//PCIE_DOE_CAP
+#define PCIE_DOE_CAP__INTR_SUPPORT__SHIFT                                                                     0x0
+#define PCIE_DOE_CAP__DOE_INTR_MSG_NUM__SHIFT                                                                 0x1
+#define PCIE_DOE_CAP__DOE_ATT_SUPPORT__SHIFT                                                                  0xc
+#define PCIE_DOE_CAP__DOE_ASYNC_SUPPORT__SHIFT                                                                0xd
+#define PCIE_DOE_CAP__INTR_SUPPORT_MASK                                                                       0x00000001L
+#define PCIE_DOE_CAP__DOE_INTR_MSG_NUM_MASK                                                                   0x00000FFEL
+#define PCIE_DOE_CAP__DOE_ATT_SUPPORT_MASK                                                                    0x00001000L
+#define PCIE_DOE_CAP__DOE_ASYNC_SUPPORT_MASK                                                                  0x00002000L
+//PCIE_DOE_CNTL
+#define PCIE_DOE_CNTL__DOE_ABORT__SHIFT                                                                       0x0
+#define PCIE_DOE_CNTL__DOE_INTR_EN__SHIFT                                                                     0x1
+#define PCIE_DOE_CNTL__DOE_ATT_NOT_NEED__SHIFT                                                                0x2
+#define PCIE_DOE_CNTL__DOE_ASYNC_MSG_EN__SHIFT                                                                0x3
+#define PCIE_DOE_CNTL__DOE_GO__SHIFT                                                                          0x1f
+#define PCIE_DOE_CNTL__DOE_ABORT_MASK                                                                         0x00000001L
+#define PCIE_DOE_CNTL__DOE_INTR_EN_MASK                                                                       0x00000002L
+#define PCIE_DOE_CNTL__DOE_ATT_NOT_NEED_MASK                                                                  0x00000004L
+#define PCIE_DOE_CNTL__DOE_ASYNC_MSG_EN_MASK                                                                  0x00000008L
+#define PCIE_DOE_CNTL__DOE_GO_MASK                                                                            0x80000000L
+//PCIE_DOE_STS
+#define PCIE_DOE_STS__DOE_BUSY__SHIFT                                                                         0x0
+#define PCIE_DOE_STS__DOE_INTR_STS__SHIFT                                                                     0x1
+#define PCIE_DOE_STS__DOE_ERR__SHIFT                                                                          0x2
+#define PCIE_DOE_STS__DOE_ASYNC_MSG_STS__SHIFT                                                                0x3
+#define PCIE_DOE_STS__DOE_AT_ATTENTION__SHIFT                                                                 0x4
+#define PCIE_DOE_STS__DOE_OBJ_RDY__SHIFT                                                                      0x1f
+#define PCIE_DOE_STS__DOE_BUSY_MASK                                                                           0x00000001L
+#define PCIE_DOE_STS__DOE_INTR_STS_MASK                                                                       0x00000002L
+#define PCIE_DOE_STS__DOE_ERR_MASK                                                                            0x00000004L
+#define PCIE_DOE_STS__DOE_ASYNC_MSG_STS_MASK                                                                  0x00000008L
+#define PCIE_DOE_STS__DOE_AT_ATTENTION_MASK                                                                   0x00000010L
+#define PCIE_DOE_STS__DOE_OBJ_RDY_MASK                                                                        0x80000000L
+//PCIE_DOE_WMBOX
+#define PCIE_DOE_WMBOX__DOE_WMBOX__SHIFT                                                                      0x0
+#define PCIE_DOE_WMBOX__DOE_WMBOX_MASK                                                                        0xFFFFFFFFL
+//PCIE_DOE_RMBOX
+#define PCIE_DOE_RMBOX__DOE_RMBOX__SHIFT                                                                      0x0
+#define PCIE_DOE_RMBOX__DOE_RMBOX_MASK                                                                        0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID__SHIFT                                               0x0
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER__SHIFT                                              0x10
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR__SHIFT                                             0x14
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID_MASK                                                 0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER_MASK                                                0x000F0000L
+#define PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR_MASK                                               0xFFF00000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID__SHIFT                                                       0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV__SHIFT                                                      0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH__SHIFT                                                   0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID_MASK                                                         0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV_MASK                                                        0x000F0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH_MASK                                                     0xFFF00000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW__VF_EN__SHIFT                                            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW__VF_NUM__SHIFT                                           0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW__VF_EN_MASK                                              0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW__VF_NUM_MASK                                             0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN__SHIFT                      0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN__SHIFT                    0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN_MASK                        0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN_MASK                      0x02000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS__SHIFT                  0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS__SHIFT                0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS_MASK                    0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS_MASK                  0x02000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR__SHIFT                                     0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__FUNCTION_ID__SHIFT                                     0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR_MASK                                       0x0001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__FUNCTION_ID_MASK                                       0xFF00L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX__SHIFT                                        0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA__SHIFT                                    0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID__SHIFT                                   0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA__SHIFT                                    0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK__SHIFT                                     0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX_MASK                                          0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA_MASK                                      0x00000F00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID_MASK                                     0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA_MASK                                      0x000F0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK_MASK                                       0x01000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK__SHIFT                                     0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID__SHIFT                                   0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK__SHIFT                                     0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID__SHIFT                                   0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK__SHIFT                                     0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID__SHIFT                                   0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK__SHIFT                                     0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID__SHIFT                                   0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK__SHIFT                                     0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID__SHIFT                                   0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK__SHIFT                                     0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID__SHIFT                                   0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK__SHIFT                                     0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID__SHIFT                                   0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK__SHIFT                                     0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID__SHIFT                                   0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK__SHIFT                                     0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID__SHIFT                                   0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK__SHIFT                                     0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID__SHIFT                                   0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK__SHIFT                                    0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID__SHIFT                                  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK__SHIFT                                    0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID__SHIFT                                  0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK__SHIFT                                    0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID__SHIFT                                  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK__SHIFT                                    0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID__SHIFT                                  0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK__SHIFT                                    0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID__SHIFT                                  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK__SHIFT                                    0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID__SHIFT                                  0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK_MASK                                       0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID_MASK                                     0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK_MASK                                       0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID_MASK                                     0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK_MASK                                       0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID_MASK                                     0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK_MASK                                       0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID_MASK                                     0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK_MASK                                       0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID_MASK                                     0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK_MASK                                       0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID_MASK                                     0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK_MASK                                       0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID_MASK                                     0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK_MASK                                       0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID_MASK                                     0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK_MASK                                       0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID_MASK                                     0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK_MASK                                       0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID_MASK                                     0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK_MASK                                      0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID_MASK                                    0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK_MASK                                      0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID_MASK                                    0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK_MASK                                      0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID_MASK                                    0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK_MASK                                      0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID_MASK                                    0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK_MASK                                      0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID_MASK                                    0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK_MASK                                      0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID_MASK                                    0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK__SHIFT                                    0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID__SHIFT                                  0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK__SHIFT                                    0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID__SHIFT                                  0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK__SHIFT                                    0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID__SHIFT                                  0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK__SHIFT                                    0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID__SHIFT                                  0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK__SHIFT                                    0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID__SHIFT                                  0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK__SHIFT                                    0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID__SHIFT                                  0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK__SHIFT                                    0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID__SHIFT                                  0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK__SHIFT                                    0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID__SHIFT                                  0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK__SHIFT                                    0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID__SHIFT                                  0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK__SHIFT                                    0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID__SHIFT                                  0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK__SHIFT                                    0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID__SHIFT                                  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK__SHIFT                                    0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID__SHIFT                                  0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK__SHIFT                                    0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID__SHIFT                                  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK__SHIFT                                    0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID__SHIFT                                  0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK__SHIFT                                    0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID__SHIFT                                  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK__SHIFT                                      0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID__SHIFT                                    0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK_MASK                                      0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID_MASK                                    0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK_MASK                                      0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID_MASK                                    0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK_MASK                                      0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID_MASK                                    0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK_MASK                                      0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID_MASK                                    0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK_MASK                                      0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID_MASK                                    0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK_MASK                                      0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID_MASK                                    0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK_MASK                                      0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID_MASK                                    0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK_MASK                                      0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID_MASK                                    0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK_MASK                                      0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID_MASK                                    0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK_MASK                                      0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID_MASK                                    0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK_MASK                                      0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID_MASK                                    0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK_MASK                                      0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID_MASK                                    0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK_MASK                                      0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID_MASK                                    0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK_MASK                                      0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID_MASK                                    0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK_MASK                                      0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID_MASK                                    0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK_MASK                                        0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID_MASK                                      0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_AVAILABLE__SHIFT                                   0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_CONSUMED__SHIFT                                    0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_AVAILABLE_MASK                                     0x0000FFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_CONSUMED_MASK                                      0xFFFF0000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__LFB_REGION__SHIFT                                             0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__MAX_REGION__SHIFT                                             0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__LFB_REGION_MASK                                               0x0000000FL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__MAX_REGION_MASK                                               0x000000F0L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_VF__SHIFT                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_PF__SHIFT                  0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_VF_MASK                    0x7FFFFFFFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_PF_MASK                    0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH0_OFFSET__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH1_OFFSET__SHIFT                                          0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH2_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH3_OFFSET__SHIFT                                          0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH0_OFFSET_MASK                                            0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH1_OFFSET_MASK                                            0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH2_OFFSET_MASK                                            0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH3_OFFSET_MASK                                            0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH4_OFFSET__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH5_OFFSET__SHIFT                                          0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH6_OFFSET__SHIFT                                          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH7_OFFSET__SHIFT                                          0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH4_OFFSET_MASK                                            0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH5_OFFSET_MASK                                            0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH6_OFFSET_MASK                                            0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH7_OFFSET_MASK                                            0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH8_OFFSET__SHIFT                                          0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH9_OFFSET__SHIFT                                          0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH10_OFFSET__SHIFT                                         0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH11_OFFSET__SHIFT                                         0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH8_OFFSET_MASK                                            0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH9_OFFSET_MASK                                            0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH10_OFFSET_MASK                                           0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH11_OFFSET_MASK                                           0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH12_OFFSET__SHIFT                                         0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH13_OFFSET__SHIFT                                         0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH14_OFFSET__SHIFT                                         0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH15_OFFSET__SHIFT                                         0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH12_OFFSET_MASK                                           0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH13_OFFSET_MASK                                           0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH14_OFFSET_MASK                                           0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH15_OFFSET_MASK                                           0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH16_OFFSET__SHIFT                                         0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH17_OFFSET__SHIFT                                         0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH18_OFFSET__SHIFT                                         0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH19_OFFSET__SHIFT                                         0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH16_OFFSET_MASK                                           0x000000FFL
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH17_OFFSET_MASK                                           0x0000FF00L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH18_OFFSET_MASK                                           0x00FF0000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH19_OFFSET_MASK                                           0xFF000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW0__DW0__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW0__DW0_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW1__DW1__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW1__DW1_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW2__DW2__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW2__DW2_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW3__DW3__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW3__DW3_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW4__DW4__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW4__DW4_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW5__DW5__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW5__DW5_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW6__DW6__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW6__DW6_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW7__DW7__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW7__DW7_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW8__DW8__SHIFT                                                  0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW8__DW8_MASK                                                    0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW0__DW0__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW0__DW0_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW1__DW1__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW1__DW1_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW2__DW2__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW2__DW2_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW3__DW3__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW3__DW3_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW4__DW4__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW4__DW4_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW5__DW5__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW5__DW5_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW6__DW6__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW6__DW6_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW7__DW7__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW7__DW7_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW8__DW8__SHIFT                                                 0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW8__DW8_MASK                                                   0xFFFFFFFFL
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_CMD_COMPLETE_INTR_EN__SHIFT                0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_HANG_SELF_RECOVERED_INTR_EN__SHIFT         0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_HANG_NEED_FLR_INTR_EN__SHIFT               0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_VM_BUSY_TRANSITION_INTR_EN__SHIFT          0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_CMD_COMPLETE_INTR_EN__SHIFT                0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_HANG_SELF_RECOVERED_INTR_EN__SHIFT         0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_HANG_NEED_FLR_INTR_EN__SHIFT               0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_VM_BUSY_TRANSITION_INTR_EN__SHIFT          0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_CMD_COMPLETE_INTR_EN__SHIFT                0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_HANG_SELF_RECOVERED_INTR_EN__SHIFT         0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_HANG_NEED_FLR_INTR_EN__SHIFT               0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_VM_BUSY_TRANSITION_INTR_EN__SHIFT          0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_CMD_COMPLETE_INTR_EN__SHIFT                0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_HANG_SELF_RECOVERED_INTR_EN__SHIFT         0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_HANG_NEED_FLR_INTR_EN__SHIFT               0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_VM_BUSY_TRANSITION_INTR_EN__SHIFT          0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_CMD_COMPLETE_INTR_EN__SHIFT                0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_HANG_SELF_RECOVERED_INTR_EN__SHIFT         0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_HANG_NEED_FLR_INTR_EN__SHIFT               0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_VM_BUSY_TRANSITION_INTR_EN__SHIFT          0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_CMD_COMPLETE_INTR_EN__SHIFT                0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_HANG_SELF_RECOVERED_INTR_EN__SHIFT         0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_HANG_NEED_FLR_INTR_EN__SHIFT               0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_VM_BUSY_TRANSITION_INTR_EN__SHIFT          0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_CMD_COMPLETE_INTR_EN__SHIFT                0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_HANG_SELF_RECOVERED_INTR_EN__SHIFT         0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_HANG_NEED_FLR_INTR_EN__SHIFT               0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_VM_BUSY_TRANSITION_INTR_EN__SHIFT          0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_CMD_COMPLETE_INTR_EN__SHIFT                0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_HANG_SELF_RECOVERED_INTR_EN__SHIFT         0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_HANG_NEED_FLR_INTR_EN__SHIFT               0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_VM_BUSY_TRANSITION_INTR_EN__SHIFT          0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_CMD_COMPLETE_INTR_EN_MASK                  0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_HANG_SELF_RECOVERED_INTR_EN_MASK           0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_HANG_NEED_FLR_INTR_EN_MASK                 0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_VM_BUSY_TRANSITION_INTR_EN_MASK            0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_CMD_COMPLETE_INTR_EN_MASK                  0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_HANG_SELF_RECOVERED_INTR_EN_MASK           0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_HANG_NEED_FLR_INTR_EN_MASK                 0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_VM_BUSY_TRANSITION_INTR_EN_MASK            0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_CMD_COMPLETE_INTR_EN_MASK                  0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_HANG_SELF_RECOVERED_INTR_EN_MASK           0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_HANG_NEED_FLR_INTR_EN_MASK                 0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_VM_BUSY_TRANSITION_INTR_EN_MASK            0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_CMD_COMPLETE_INTR_EN_MASK                  0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_HANG_SELF_RECOVERED_INTR_EN_MASK           0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_HANG_NEED_FLR_INTR_EN_MASK                 0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_VM_BUSY_TRANSITION_INTR_EN_MASK            0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_CMD_COMPLETE_INTR_EN_MASK                  0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_HANG_SELF_RECOVERED_INTR_EN_MASK           0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_HANG_NEED_FLR_INTR_EN_MASK                 0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_VM_BUSY_TRANSITION_INTR_EN_MASK            0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_CMD_COMPLETE_INTR_EN_MASK                  0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_HANG_SELF_RECOVERED_INTR_EN_MASK           0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_HANG_NEED_FLR_INTR_EN_MASK                 0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_VM_BUSY_TRANSITION_INTR_EN_MASK            0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_CMD_COMPLETE_INTR_EN_MASK                  0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_HANG_SELF_RECOVERED_INTR_EN_MASK           0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_HANG_NEED_FLR_INTR_EN_MASK                 0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_VM_BUSY_TRANSITION_INTR_EN_MASK            0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_CMD_COMPLETE_INTR_EN_MASK                  0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_HANG_SELF_RECOVERED_INTR_EN_MASK           0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_HANG_NEED_FLR_INTR_EN_MASK                 0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_VM_BUSY_TRANSITION_INTR_EN_MASK            0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_CMD_COMPLETE_INTR_EN__SHIFT               0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_HANG_SELF_RECOVERED_INTR_EN__SHIFT        0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_HANG_NEED_FLR_INTR_EN__SHIFT              0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_VM_BUSY_TRANSITION_INTR_EN__SHIFT         0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_CMD_COMPLETE_INTR_EN__SHIFT               0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_HANG_SELF_RECOVERED_INTR_EN__SHIFT        0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_HANG_NEED_FLR_INTR_EN__SHIFT              0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_VM_BUSY_TRANSITION_INTR_EN__SHIFT         0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_CMD_COMPLETE_INTR_EN__SHIFT              0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_HANG_SELF_RECOVERED_INTR_EN__SHIFT       0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_HANG_NEED_FLR_INTR_EN__SHIFT             0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_VM_BUSY_TRANSITION_INTR_EN__SHIFT        0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_CMD_COMPLETE_INTR_EN__SHIFT              0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_HANG_SELF_RECOVERED_INTR_EN__SHIFT       0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_HANG_NEED_FLR_INTR_EN__SHIFT             0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_VM_BUSY_TRANSITION_INTR_EN__SHIFT        0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_CMD_COMPLETE_INTR_EN__SHIFT              0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_HANG_SELF_RECOVERED_INTR_EN__SHIFT       0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_HANG_NEED_FLR_INTR_EN__SHIFT             0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_VM_BUSY_TRANSITION_INTR_EN__SHIFT        0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_CMD_COMPLETE_INTR_EN__SHIFT              0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_HANG_SELF_RECOVERED_INTR_EN__SHIFT       0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_HANG_NEED_FLR_INTR_EN__SHIFT             0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_VM_BUSY_TRANSITION_INTR_EN__SHIFT        0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_CMD_COMPLETE_INTR_EN__SHIFT              0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_HANG_SELF_RECOVERED_INTR_EN__SHIFT       0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_HANG_NEED_FLR_INTR_EN__SHIFT             0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_VM_BUSY_TRANSITION_INTR_EN__SHIFT        0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_CMD_COMPLETE_INTR_EN__SHIFT              0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_HANG_SELF_RECOVERED_INTR_EN__SHIFT       0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_HANG_NEED_FLR_INTR_EN__SHIFT             0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_VM_BUSY_TRANSITION_INTR_EN__SHIFT        0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_CMD_COMPLETE_INTR_EN_MASK                 0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_HANG_SELF_RECOVERED_INTR_EN_MASK          0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_HANG_NEED_FLR_INTR_EN_MASK                0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_VM_BUSY_TRANSITION_INTR_EN_MASK           0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_CMD_COMPLETE_INTR_EN_MASK                 0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_HANG_SELF_RECOVERED_INTR_EN_MASK          0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_HANG_NEED_FLR_INTR_EN_MASK                0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_VM_BUSY_TRANSITION_INTR_EN_MASK           0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_CMD_COMPLETE_INTR_EN_MASK                0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_HANG_SELF_RECOVERED_INTR_EN_MASK         0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_HANG_NEED_FLR_INTR_EN_MASK               0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_VM_BUSY_TRANSITION_INTR_EN_MASK          0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_CMD_COMPLETE_INTR_EN_MASK                0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_HANG_SELF_RECOVERED_INTR_EN_MASK         0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_HANG_NEED_FLR_INTR_EN_MASK               0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_VM_BUSY_TRANSITION_INTR_EN_MASK          0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_CMD_COMPLETE_INTR_EN_MASK                0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_HANG_SELF_RECOVERED_INTR_EN_MASK         0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_HANG_NEED_FLR_INTR_EN_MASK               0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_VM_BUSY_TRANSITION_INTR_EN_MASK          0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_CMD_COMPLETE_INTR_EN_MASK                0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_HANG_SELF_RECOVERED_INTR_EN_MASK         0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_HANG_NEED_FLR_INTR_EN_MASK               0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_VM_BUSY_TRANSITION_INTR_EN_MASK          0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_CMD_COMPLETE_INTR_EN_MASK                0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_HANG_SELF_RECOVERED_INTR_EN_MASK         0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_HANG_NEED_FLR_INTR_EN_MASK               0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_VM_BUSY_TRANSITION_INTR_EN_MASK          0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_CMD_COMPLETE_INTR_EN_MASK                0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_HANG_SELF_RECOVERED_INTR_EN_MASK         0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_HANG_NEED_FLR_INTR_EN_MASK               0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_VM_BUSY_TRANSITION_INTR_EN_MASK          0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_CMD_COMPLETE_INTR_EN__SHIFT             0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_HANG_NEED_FLR_INTR_EN__SHIFT            0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_CMD_COMPLETE_INTR_EN__SHIFT             0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_HANG_NEED_FLR_INTR_EN__SHIFT            0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_CMD_COMPLETE_INTR_EN__SHIFT             0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_HANG_NEED_FLR_INTR_EN__SHIFT            0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_CMD_COMPLETE_INTR_EN__SHIFT             0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_HANG_NEED_FLR_INTR_EN__SHIFT            0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_CMD_COMPLETE_INTR_EN__SHIFT             0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_HANG_NEED_FLR_INTR_EN__SHIFT            0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_CMD_COMPLETE_INTR_EN__SHIFT             0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_HANG_NEED_FLR_INTR_EN__SHIFT            0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_CMD_COMPLETE_INTR_EN__SHIFT             0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_HANG_NEED_FLR_INTR_EN__SHIFT            0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_CMD_COMPLETE_INTR_EN__SHIFT             0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_HANG_NEED_FLR_INTR_EN__SHIFT            0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_CMD_COMPLETE_INTR_EN_MASK               0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_HANG_NEED_FLR_INTR_EN_MASK              0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_CMD_COMPLETE_INTR_EN_MASK               0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_HANG_NEED_FLR_INTR_EN_MASK              0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_CMD_COMPLETE_INTR_EN_MASK               0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_HANG_NEED_FLR_INTR_EN_MASK              0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_CMD_COMPLETE_INTR_EN_MASK               0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_HANG_NEED_FLR_INTR_EN_MASK              0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_CMD_COMPLETE_INTR_EN_MASK               0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_HANG_NEED_FLR_INTR_EN_MASK              0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_CMD_COMPLETE_INTR_EN_MASK               0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_HANG_NEED_FLR_INTR_EN_MASK              0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_CMD_COMPLETE_INTR_EN_MASK               0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_HANG_SELF_RECOVERED_INTR_EN_MASK        0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_HANG_NEED_FLR_INTR_EN_MASK              0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_VM_BUSY_TRANSITION_INTR_EN_MASK         0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_CMD_COMPLETE_INTR_EN_MASK               0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_HANG_SELF_RECOVERED_INTR_EN_MASK        0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_HANG_NEED_FLR_INTR_EN_MASK              0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_VM_BUSY_TRANSITION_INTR_EN_MASK         0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_CMD_COMPLETE_INTR_EN__SHIFT             0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_HANG_NEED_FLR_INTR_EN__SHIFT            0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_CMD_COMPLETE_INTR_EN__SHIFT             0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_HANG_NEED_FLR_INTR_EN__SHIFT            0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_CMD_COMPLETE_INTR_EN__SHIFT             0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_HANG_NEED_FLR_INTR_EN__SHIFT            0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_CMD_COMPLETE_INTR_EN__SHIFT             0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_HANG_NEED_FLR_INTR_EN__SHIFT            0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_CMD_COMPLETE_INTR_EN__SHIFT             0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_HANG_NEED_FLR_INTR_EN__SHIFT            0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_CMD_COMPLETE_INTR_EN__SHIFT             0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_HANG_NEED_FLR_INTR_EN__SHIFT            0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_CMD_COMPLETE_INTR_EN__SHIFT             0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_HANG_NEED_FLR_INTR_EN__SHIFT            0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_CMD_COMPLETE_INTR_EN__SHIFT             0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_HANG_SELF_RECOVERED_INTR_EN__SHIFT      0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_HANG_NEED_FLR_INTR_EN__SHIFT            0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_VM_BUSY_TRANSITION_INTR_EN__SHIFT       0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_CMD_COMPLETE_INTR_EN_MASK               0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_HANG_NEED_FLR_INTR_EN_MASK              0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_CMD_COMPLETE_INTR_EN_MASK               0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_HANG_NEED_FLR_INTR_EN_MASK              0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_CMD_COMPLETE_INTR_EN_MASK               0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_HANG_NEED_FLR_INTR_EN_MASK              0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_CMD_COMPLETE_INTR_EN_MASK               0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_HANG_NEED_FLR_INTR_EN_MASK              0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_CMD_COMPLETE_INTR_EN_MASK               0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_HANG_NEED_FLR_INTR_EN_MASK              0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_CMD_COMPLETE_INTR_EN_MASK               0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_HANG_SELF_RECOVERED_INTR_EN_MASK        0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_HANG_NEED_FLR_INTR_EN_MASK              0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_VM_BUSY_TRANSITION_INTR_EN_MASK         0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_CMD_COMPLETE_INTR_EN_MASK               0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_HANG_SELF_RECOVERED_INTR_EN_MASK        0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_HANG_NEED_FLR_INTR_EN_MASK              0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_VM_BUSY_TRANSITION_INTR_EN_MASK         0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_CMD_COMPLETE_INTR_EN_MASK               0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_HANG_SELF_RECOVERED_INTR_EN_MASK        0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_HANG_NEED_FLR_INTR_EN_MASK              0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_VM_BUSY_TRANSITION_INTR_EN_MASK         0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_CMD_COMPLETE_INTR_STATUS__SHIFT            0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT     0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_HANG_NEED_FLR_INTR_STATUS__SHIFT           0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT      0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_CMD_COMPLETE_INTR_STATUS__SHIFT            0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT     0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_HANG_NEED_FLR_INTR_STATUS__SHIFT           0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT      0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_CMD_COMPLETE_INTR_STATUS__SHIFT            0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT     0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_HANG_NEED_FLR_INTR_STATUS__SHIFT           0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT      0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_CMD_COMPLETE_INTR_STATUS__SHIFT            0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT     0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_HANG_NEED_FLR_INTR_STATUS__SHIFT           0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT      0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_CMD_COMPLETE_INTR_STATUS__SHIFT            0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT     0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_HANG_NEED_FLR_INTR_STATUS__SHIFT           0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT      0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_CMD_COMPLETE_INTR_STATUS__SHIFT            0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT     0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_HANG_NEED_FLR_INTR_STATUS__SHIFT           0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT      0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_CMD_COMPLETE_INTR_STATUS__SHIFT            0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT     0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_HANG_NEED_FLR_INTR_STATUS__SHIFT           0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT      0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_CMD_COMPLETE_INTR_STATUS__SHIFT            0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT     0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_HANG_NEED_FLR_INTR_STATUS__SHIFT           0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT      0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_CMD_COMPLETE_INTR_STATUS_MASK              0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_HANG_SELF_RECOVERED_INTR_STATUS_MASK       0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_HANG_NEED_FLR_INTR_STATUS_MASK             0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_VM_BUSY_TRANSITION_INTR_STATUS_MASK        0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_CMD_COMPLETE_INTR_STATUS_MASK              0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_HANG_SELF_RECOVERED_INTR_STATUS_MASK       0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_HANG_NEED_FLR_INTR_STATUS_MASK             0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_VM_BUSY_TRANSITION_INTR_STATUS_MASK        0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_CMD_COMPLETE_INTR_STATUS_MASK              0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_HANG_SELF_RECOVERED_INTR_STATUS_MASK       0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_HANG_NEED_FLR_INTR_STATUS_MASK             0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_VM_BUSY_TRANSITION_INTR_STATUS_MASK        0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_CMD_COMPLETE_INTR_STATUS_MASK              0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_HANG_SELF_RECOVERED_INTR_STATUS_MASK       0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_HANG_NEED_FLR_INTR_STATUS_MASK             0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_VM_BUSY_TRANSITION_INTR_STATUS_MASK        0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_CMD_COMPLETE_INTR_STATUS_MASK              0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_HANG_SELF_RECOVERED_INTR_STATUS_MASK       0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_HANG_NEED_FLR_INTR_STATUS_MASK             0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_VM_BUSY_TRANSITION_INTR_STATUS_MASK        0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_CMD_COMPLETE_INTR_STATUS_MASK              0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_HANG_SELF_RECOVERED_INTR_STATUS_MASK       0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_HANG_NEED_FLR_INTR_STATUS_MASK             0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_VM_BUSY_TRANSITION_INTR_STATUS_MASK        0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_CMD_COMPLETE_INTR_STATUS_MASK              0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_HANG_SELF_RECOVERED_INTR_STATUS_MASK       0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_HANG_NEED_FLR_INTR_STATUS_MASK             0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_VM_BUSY_TRANSITION_INTR_STATUS_MASK        0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_CMD_COMPLETE_INTR_STATUS_MASK              0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_HANG_SELF_RECOVERED_INTR_STATUS_MASK       0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_HANG_NEED_FLR_INTR_STATUS_MASK             0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_VM_BUSY_TRANSITION_INTR_STATUS_MASK        0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_CMD_COMPLETE_INTR_STATUS__SHIFT           0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT    0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_HANG_NEED_FLR_INTR_STATUS__SHIFT          0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT     0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_CMD_COMPLETE_INTR_STATUS__SHIFT           0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT    0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_HANG_NEED_FLR_INTR_STATUS__SHIFT          0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT     0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_CMD_COMPLETE_INTR_STATUS__SHIFT          0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT   0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_HANG_NEED_FLR_INTR_STATUS__SHIFT         0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT    0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_CMD_COMPLETE_INTR_STATUS__SHIFT          0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT   0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_HANG_NEED_FLR_INTR_STATUS__SHIFT         0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT    0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_CMD_COMPLETE_INTR_STATUS__SHIFT          0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT   0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_HANG_NEED_FLR_INTR_STATUS__SHIFT         0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT    0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_CMD_COMPLETE_INTR_STATUS__SHIFT          0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT   0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_HANG_NEED_FLR_INTR_STATUS__SHIFT         0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT    0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_CMD_COMPLETE_INTR_STATUS__SHIFT          0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT   0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_HANG_NEED_FLR_INTR_STATUS__SHIFT         0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT    0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_CMD_COMPLETE_INTR_STATUS__SHIFT          0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT   0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_HANG_NEED_FLR_INTR_STATUS__SHIFT         0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT    0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_CMD_COMPLETE_INTR_STATUS_MASK             0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_HANG_SELF_RECOVERED_INTR_STATUS_MASK      0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_HANG_NEED_FLR_INTR_STATUS_MASK            0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_VM_BUSY_TRANSITION_INTR_STATUS_MASK       0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_CMD_COMPLETE_INTR_STATUS_MASK             0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_HANG_SELF_RECOVERED_INTR_STATUS_MASK      0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_HANG_NEED_FLR_INTR_STATUS_MASK            0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_VM_BUSY_TRANSITION_INTR_STATUS_MASK       0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_CMD_COMPLETE_INTR_STATUS_MASK            0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_HANG_SELF_RECOVERED_INTR_STATUS_MASK     0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_HANG_NEED_FLR_INTR_STATUS_MASK           0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_VM_BUSY_TRANSITION_INTR_STATUS_MASK      0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_CMD_COMPLETE_INTR_STATUS_MASK            0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_HANG_SELF_RECOVERED_INTR_STATUS_MASK     0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_HANG_NEED_FLR_INTR_STATUS_MASK           0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_VM_BUSY_TRANSITION_INTR_STATUS_MASK      0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_CMD_COMPLETE_INTR_STATUS_MASK            0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_HANG_SELF_RECOVERED_INTR_STATUS_MASK     0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_HANG_NEED_FLR_INTR_STATUS_MASK           0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_VM_BUSY_TRANSITION_INTR_STATUS_MASK      0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_CMD_COMPLETE_INTR_STATUS_MASK            0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_HANG_SELF_RECOVERED_INTR_STATUS_MASK     0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_HANG_NEED_FLR_INTR_STATUS_MASK           0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_VM_BUSY_TRANSITION_INTR_STATUS_MASK      0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_CMD_COMPLETE_INTR_STATUS_MASK            0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_HANG_SELF_RECOVERED_INTR_STATUS_MASK     0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_HANG_NEED_FLR_INTR_STATUS_MASK           0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_VM_BUSY_TRANSITION_INTR_STATUS_MASK      0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_CMD_COMPLETE_INTR_STATUS_MASK            0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_HANG_SELF_RECOVERED_INTR_STATUS_MASK     0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_HANG_NEED_FLR_INTR_STATUS_MASK           0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_VM_BUSY_TRANSITION_INTR_STATUS_MASK      0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_CMD_COMPLETE_INTR_STATUS__SHIFT         0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_CMD_COMPLETE_INTR_STATUS__SHIFT         0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_CMD_COMPLETE_INTR_STATUS__SHIFT         0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_HANG_NEED_FLR_INTR_STATUS__SHIFT        0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_CMD_COMPLETE_INTR_STATUS__SHIFT         0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_HANG_NEED_FLR_INTR_STATUS__SHIFT        0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_CMD_COMPLETE_INTR_STATUS__SHIFT         0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_CMD_COMPLETE_INTR_STATUS__SHIFT         0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_CMD_COMPLETE_INTR_STATUS__SHIFT         0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_CMD_COMPLETE_INTR_STATUS__SHIFT         0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_CMD_COMPLETE_INTR_STATUS_MASK           0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_HANG_NEED_FLR_INTR_STATUS_MASK          0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_CMD_COMPLETE_INTR_STATUS_MASK           0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_HANG_NEED_FLR_INTR_STATUS_MASK          0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_CMD_COMPLETE_INTR_STATUS_MASK           0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_HANG_NEED_FLR_INTR_STATUS_MASK          0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_CMD_COMPLETE_INTR_STATUS_MASK           0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_HANG_NEED_FLR_INTR_STATUS_MASK          0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_CMD_COMPLETE_INTR_STATUS_MASK           0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_HANG_NEED_FLR_INTR_STATUS_MASK          0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_CMD_COMPLETE_INTR_STATUS_MASK           0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_HANG_NEED_FLR_INTR_STATUS_MASK          0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_CMD_COMPLETE_INTR_STATUS_MASK           0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_HANG_NEED_FLR_INTR_STATUS_MASK          0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_CMD_COMPLETE_INTR_STATUS_MASK           0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_HANG_NEED_FLR_INTR_STATUS_MASK          0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x80000000L
+//PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_CMD_COMPLETE_INTR_STATUS__SHIFT         0x0
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x2
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x3
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_CMD_COMPLETE_INTR_STATUS__SHIFT         0x4
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x6
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x7
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_CMD_COMPLETE_INTR_STATUS__SHIFT         0x8
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_HANG_NEED_FLR_INTR_STATUS__SHIFT        0xa
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0xb
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_CMD_COMPLETE_INTR_STATUS__SHIFT         0xc
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_HANG_NEED_FLR_INTR_STATUS__SHIFT        0xe
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0xf
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_CMD_COMPLETE_INTR_STATUS__SHIFT         0x10
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x12
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x13
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_CMD_COMPLETE_INTR_STATUS__SHIFT         0x14
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x16
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x17
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_CMD_COMPLETE_INTR_STATUS__SHIFT         0x18
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x1a
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x1b
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_CMD_COMPLETE_INTR_STATUS__SHIFT         0x1c
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_HANG_NEED_FLR_INTR_STATUS__SHIFT        0x1e
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT   0x1f
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_CMD_COMPLETE_INTR_STATUS_MASK           0x00000001L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00000002L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_HANG_NEED_FLR_INTR_STATUS_MASK          0x00000004L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00000008L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_CMD_COMPLETE_INTR_STATUS_MASK           0x00000010L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00000020L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_HANG_NEED_FLR_INTR_STATUS_MASK          0x00000040L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00000080L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_CMD_COMPLETE_INTR_STATUS_MASK           0x00000100L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00000200L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_HANG_NEED_FLR_INTR_STATUS_MASK          0x00000400L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00000800L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_CMD_COMPLETE_INTR_STATUS_MASK           0x00001000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00002000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_HANG_NEED_FLR_INTR_STATUS_MASK          0x00004000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00008000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_CMD_COMPLETE_INTR_STATUS_MASK           0x00010000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00020000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_HANG_NEED_FLR_INTR_STATUS_MASK          0x00040000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00080000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_CMD_COMPLETE_INTR_STATUS_MASK           0x00100000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x00200000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_HANG_NEED_FLR_INTR_STATUS_MASK          0x00400000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x00800000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_CMD_COMPLETE_INTR_STATUS_MASK           0x01000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x02000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_HANG_NEED_FLR_INTR_STATUS_MASK          0x04000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x08000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_CMD_COMPLETE_INTR_STATUS_MASK           0x10000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_HANG_SELF_RECOVERED_INTR_STATUS_MASK    0x20000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_HANG_NEED_FLR_INTR_STATUS_MASK          0x40000000L
+#define PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_VM_BUSY_TRANSITION_INTR_STATUS_MASK     0x80000000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_bifcfgpciedecp
+//BIF_CFG_DEV0_EPF0_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                  0x00000002L
+#define BIF_CFG_DEV0_EPF0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                       0x0000FE00L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                 0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0xc
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                  0x000FL
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x0070L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                    0x0F00L
+#define BIF_CFG_DEV0_EPF0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x7000L
+//BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_EPF0_LINK_CAP_16GT
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_16GT__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_16GT__RESERVED_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                             0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                             0x00000004L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                             0x00000008L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                               0x00000010L
+//BIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV0_EPF0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV0_EPF0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV0_EPF0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_EPF0_LINK_CAP_32GT
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                             0x9
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                             0xa
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                             0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                               0x00000200L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                               0x00000400L
+#define BIF_CFG_DEV0_EPF0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                0x0000F800L
+//BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                               0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                     0x00000700L
+//BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                      0xa
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                             0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                             0x00000004L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                             0x00000008L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                               0x00000010L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                         0x00000020L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                        0x00000400L
+//BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                           0x3
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                             0x0000FFF8L
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                           0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT            0x18
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                             0x00FFFFFFL
+#define BIF_CFG_DEV0_EPF0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK              0x03000000L
+//BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                      0x00000007L
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                          0x0000FFF8L
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT         0x18
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                          0x00FFFFFFL
+#define BIF_CFG_DEV0_EPF0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK           0x03000000L
+//BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_AP_CAP
+#define BIF_CFG_DEV0_EPF0_AP_CAP__COUNT__SHIFT                                                                0x0
+#define BIF_CFG_DEV0_EPF0_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_AP_CAP__COUNT_MASK                                                                  0x000000FFL
+#define BIF_CFG_DEV0_EPF0_AP_CAP__SEL_EN_SUPPORTED_MASK                                                       0x00000100L
+//BIF_CFG_DEV0_EPF0_AP_CNTL
+#define BIF_CFG_DEV0_EPF0_AP_CNTL__INX_SEL__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF0_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF0_AP_CNTL__INX_SEL_MASK                                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF0_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                        0x00000100L
+//BIF_CFG_DEV0_EPF0_AP_DATA1
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__USAGE_INFOR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__DETAILS__SHIFT                                                            0x5
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__VENDORID__SHIFT                                                           0x10
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__USAGE_INFOR_MASK                                                          0x00000007L
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__DETAILS_MASK                                                              0x0000FFE0L
+#define BIF_CFG_DEV0_EPF0_AP_DATA1__VENDORID_MASK                                                             0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_AP_DATA2
+#define BIF_CFG_DEV0_EPF0_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                   0x00FFFFFFL
+//BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK
+#define BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK__PCIE__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK__OTHERS__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK__PCIE_MASK                                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_AP_SEL_EN_MASK__OTHERS_MASK                                                         0xFFFFFFFEL
+//BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                    0x0001L
+//BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                0x0002L
+//BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                             0xa
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                               0x00000070L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                               0x00000C00L
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                               0x000EL
+//BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                       0x0001L
+//BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                      0x00008000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                          0x003F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                            0x1f
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                        0x000000FEL
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                        0x000E0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                  0x07000000L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                              0x80000000L
+//BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                0x0001L
+#define BIF_CFG_DEV0_EPF0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                               0x0002L
+//BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                   0x1f
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                     0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                     0x80000000L
+//BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                 0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_IDE_ENH_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_IDE_ENH_CAP_LIST__CAP_VER__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF0_IDE_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                   0x14
+#define BIF_CFG_DEV0_EPF0_IDE_ENH_CAP_LIST__CAP_ID_MASK                                                       0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_IDE_ENH_CAP_LIST__CAP_VER_MASK                                                      0x000F0000L
+#define BIF_CFG_DEV0_EPF0_IDE_ENH_CAP_LIST__NEXT_PTR_MASK                                                     0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_CAP
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_LINK_IDE_STREAM_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_STREAM_SUPPORTED__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_AGGREGATION_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_PCRC_SUPPORTED__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_IDE_KM_PROTOCOL_SUPPORTED__SHIFT                                   0x6
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED__SHIFT                         0x7
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_SUPPORTED_ALGORITHMS__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_TEE_LIMITED_SUPPORTED__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_XT_SUPPORTED__SHIFT                                                0x19
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_LINK_IDE_STREAM_SUPPORTED_MASK                                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_STREAM_SUPPORTED_MASK                                0x00000002L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_MASK                             0x00000004L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_MASK                              0x00000008L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_AGGREGATION_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_PCRC_SUPPORTED_MASK                                                0x00000020L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_IDE_KM_PROTOCOL_SUPPORTED_MASK                                     0x00000040L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_MASK                           0x00000080L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_SUPPORTED_ALGORITHMS_MASK                                          0x00001F00L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_MASK                             0x0000E000L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_MASK                        0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_TEE_LIMITED_SUPPORTED_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_IDE_CAP__IDE_CAP_XT_SUPPORTED_MASK                                                  0x02000000L
+//BIF_CFG_DEV0_EPF0_IDE_CNTL
+#define BIF_CFG_DEV0_EPF0_IDE_CNTL__IDE_CNTL_FLOW_THROUGH_IDE_STREAM_ENABLED__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF0_IDE_CNTL__IDE_CNTL_FLOW_THROUGH_IDE_STREAM_ENABLED_MASK                             0x00000004L
+//BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__LINK_IDE_STREAM_ENABLE__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__XT_ENABLE__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                              0x6
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__PCRC_ENABLE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                             0xa
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__TC__SHIFT                                                   0x13
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__STREAM_ID__SHIFT                                            0x18
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__LINK_IDE_STREAM_ENABLE_MASK                                 0x00000001L
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__XT_ENABLE_MASK                                              0x00000002L
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR_MASK                                0x0000000CL
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR_MASK                                 0x00000030L
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL_MASK                                0x000000C0L
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__PCRC_ENABLE_MASK                                            0x00000100L
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                               0x00003C00L
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM_MASK                                     0x0007C000L
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__TC_MASK                                                     0x00380000L
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_CNTL__STREAM_ID_MASK                                              0xFF000000L
+//BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_STATUS
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_STATUS__LINK_IDE_STREAM_STATE__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                              0x1f
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_STATUS__LINK_IDE_STREAM_STATE_MASK                                0x0000000FL
+#define BIF_CFG_DEV0_EPF0_LINK_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                                0x80000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CAP
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                      0x0000000FL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__XT_ENABLE__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                          0x4
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                         0x6
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__PCRC_ENABLE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                    0x9
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                        0xa
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM__SHIFT                              0xe
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TC__SHIFT                                              0x13
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__DEFAULT_STREAM__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TEE_LIMITED_STREAM__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__STREAM_ID__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__XT_ENABLE_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR_MASK                           0x0000000CL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR_MASK                            0x00000030L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__PCRC_ENABLE_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                          0x00003C00L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM_MASK                                0x0007C000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TC_MASK                                                0x00380000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__DEFAULT_STREAM_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__TEE_LIMITED_STREAM_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_CNTL__STREAM_ID_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_STATUS
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                      0x0000000FL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                          0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG2__VALID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG2__VALID_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG2__RID_BASE_MASK                                           0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_0_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_0_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CAP
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                      0x0000000FL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__XT_ENABLE__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                          0x4
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                         0x6
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__PCRC_ENABLE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                    0x9
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                        0xa
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTED_ALGORITHM__SHIFT                              0xe
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TC__SHIFT                                              0x13
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__DEFAULT_STREAM__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TEE_LIMITED_STREAM__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__STREAM_ID__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__XT_ENABLE_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_NPR_MASK                           0x0000000CL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_PR_MASK                            0x00000030L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_CPL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__PCRC_ENABLE_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                          0x00003C00L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTED_ALGORITHM_MASK                                0x0007C000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TC_MASK                                                0x00380000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__DEFAULT_STREAM_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__TEE_LIMITED_STREAM_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_CNTL__STREAM_ID_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_STATUS
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                      0x0000000FL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_1_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                          0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG2__VALID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG2__VALID_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG2__RID_BASE_MASK                                           0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_1_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_1_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CAP
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                      0x0000000FL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__XT_ENABLE__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                          0x4
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                         0x6
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__PCRC_ENABLE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                    0x9
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                        0xa
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTED_ALGORITHM__SHIFT                              0xe
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TC__SHIFT                                              0x13
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__DEFAULT_STREAM__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TEE_LIMITED_STREAM__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__STREAM_ID__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__XT_ENABLE_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_NPR_MASK                           0x0000000CL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_PR_MASK                            0x00000030L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_CPL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__PCRC_ENABLE_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                          0x00003C00L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTED_ALGORITHM_MASK                                0x0007C000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TC_MASK                                                0x00380000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__DEFAULT_STREAM_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__TEE_LIMITED_STREAM_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_CNTL__STREAM_ID_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_STATUS
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                      0x0000000FL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_2_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                          0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG2__VALID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG2__VALID_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG2__RID_BASE_MASK                                           0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_2_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_2_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CAP
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                      0x0000000FL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__XT_ENABLE__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                          0x4
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                         0x6
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__PCRC_ENABLE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                    0x9
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                        0xa
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTED_ALGORITHM__SHIFT                              0xe
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TC__SHIFT                                              0x13
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__DEFAULT_STREAM__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TEE_LIMITED_STREAM__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__STREAM_ID__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__XT_ENABLE_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_NPR_MASK                           0x0000000CL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_PR_MASK                            0x00000030L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_CPL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__PCRC_ENABLE_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                          0x00003C00L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTED_ALGORITHM_MASK                                0x0007C000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TC_MASK                                                0x00380000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__DEFAULT_STREAM_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__TEE_LIMITED_STREAM_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_CNTL__STREAM_ID_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_STATUS
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                      0x0000000FL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_3_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                          0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG2__VALID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG2__VALID_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG2__RID_BASE_MASK                                           0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_3_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_3_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CAP
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                      0x0000000FL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__XT_ENABLE__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                          0x4
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                         0x6
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__PCRC_ENABLE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                    0x9
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                        0xa
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTED_ALGORITHM__SHIFT                              0xe
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TC__SHIFT                                              0x13
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__DEFAULT_STREAM__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TEE_LIMITED_STREAM__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__STREAM_ID__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__XT_ENABLE_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_NPR_MASK                           0x0000000CL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_PR_MASK                            0x00000030L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_CPL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__PCRC_ENABLE_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                          0x00003C00L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTED_ALGORITHM_MASK                                0x0007C000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TC_MASK                                                0x00380000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__DEFAULT_STREAM_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__TEE_LIMITED_STREAM_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_CNTL__STREAM_ID_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_STATUS
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                      0x0000000FL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_4_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                          0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG2__VALID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG2__VALID_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG2__RID_BASE_MASK                                           0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_4_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_4_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CAP
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                      0x0000000FL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__XT_ENABLE__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                          0x4
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                         0x6
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__PCRC_ENABLE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                    0x9
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                        0xa
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTED_ALGORITHM__SHIFT                              0xe
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TC__SHIFT                                              0x13
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__DEFAULT_STREAM__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TEE_LIMITED_STREAM__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__STREAM_ID__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__XT_ENABLE_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_NPR_MASK                           0x0000000CL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_PR_MASK                            0x00000030L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_CPL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__PCRC_ENABLE_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                          0x00003C00L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTED_ALGORITHM_MASK                                0x0007C000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TC_MASK                                                0x00380000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__DEFAULT_STREAM_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__TEE_LIMITED_STREAM_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_CNTL__STREAM_ID_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_STATUS
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                      0x0000000FL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_5_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                          0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG2__VALID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG2__VALID_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG2__RID_BASE_MASK                                           0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_5_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_5_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CAP
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                      0x0000000FL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__XT_ENABLE__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                          0x4
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                         0x6
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__PCRC_ENABLE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                    0x9
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                        0xa
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTED_ALGORITHM__SHIFT                              0xe
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TC__SHIFT                                              0x13
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__DEFAULT_STREAM__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TEE_LIMITED_STREAM__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__STREAM_ID__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__XT_ENABLE_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_NPR_MASK                           0x0000000CL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_PR_MASK                            0x00000030L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_CPL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__PCRC_ENABLE_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                          0x00003C00L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTED_ALGORITHM_MASK                                0x0007C000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TC_MASK                                                0x00380000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__DEFAULT_STREAM_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__TEE_LIMITED_STREAM_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_CNTL__STREAM_ID_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_STATUS
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                      0x0000000FL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_6_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                          0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG2__VALID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG2__VALID_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG2__RID_BASE_MASK                                           0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_6_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_6_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CAP
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                      0x0000000FL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__XT_ENABLE__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                          0x4
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                         0x6
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__PCRC_ENABLE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                    0x9
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                        0xa
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTED_ALGORITHM__SHIFT                              0xe
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TC__SHIFT                                              0x13
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__DEFAULT_STREAM__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TEE_LIMITED_STREAM__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__STREAM_ID__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__XT_ENABLE_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_NPR_MASK                           0x0000000CL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_PR_MASK                            0x00000030L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_CPL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__PCRC_ENABLE_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                          0x00003C00L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTED_ALGORITHM_MASK                                0x0007C000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TC_MASK                                                0x00380000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__DEFAULT_STREAM_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__TEE_LIMITED_STREAM_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_CNTL__STREAM_ID_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_STATUS
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                      0x0000000FL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_7_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                          0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG2__VALID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG2__VALID_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG2__RID_BASE_MASK                                           0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_7_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_7_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CAP
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                      0x0000000FL
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__XT_ENABLE__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                          0x4
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                         0x6
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__PCRC_ENABLE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                    0x9
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                        0xa
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTED_ALGORITHM__SHIFT                              0xe
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TC__SHIFT                                              0x13
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__DEFAULT_STREAM__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TEE_LIMITED_STREAM__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__STREAM_ID__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__XT_ENABLE_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_NPR_MASK                           0x0000000CL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_PR_MASK                            0x00000030L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_CPL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__PCRC_ENABLE_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                      0x00000200L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                          0x00003C00L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTED_ALGORITHM_MASK                                0x0007C000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TC_MASK                                                0x00380000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__DEFAULT_STREAM_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__TEE_LIMITED_STREAM_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_CNTL__STREAM_ID_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_STATUS
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                         0x1f
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                      0x0000000FL
+#define BIF_CFG_DEV0_EPF0_SELECTIVE_IDE_STREAM_8_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                           0x80000000L
+//BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                          0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG2__VALID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG2__VALID_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG2__RID_BASE_MASK                                           0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_8_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                0x8
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                  0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_IDE_8_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                  0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_rc_bifcfgdecp
+//BIF_CFG_DEV0_RC0_VENDOR_ID
+#define BIF_CFG_DEV0_RC0_VENDOR_ID__VENDOR_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_VENDOR_ID__VENDOR_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_DEVICE_ID
+#define BIF_CFG_DEV0_RC0_DEVICE_ID__DEVICE_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_ID__DEVICE_ID_MASK                                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_COMMAND
+#define BIF_CFG_DEV0_RC0_COMMAND__IOEN_DN__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_COMMAND__MEMEN_DN__SHIFT                                                             0x1
+#define BIF_CFG_DEV0_RC0_COMMAND__BUS_MASTER_EN__SHIFT                                                        0x2
+#define BIF_CFG_DEV0_RC0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_RC0_COMMAND__PAL_SNOOP_EN__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_RC0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                0x6
+#define BIF_CFG_DEV0_RC0_COMMAND__AD_STEPPING__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_RC0_COMMAND__SERR_EN__SHIFT                                                              0x8
+#define BIF_CFG_DEV0_RC0_COMMAND__FAST_B2B_EN__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_RC0_COMMAND__INT_DIS__SHIFT                                                              0xa
+#define BIF_CFG_DEV0_RC0_COMMAND__IOEN_DN_MASK                                                                0x0001L
+#define BIF_CFG_DEV0_RC0_COMMAND__MEMEN_DN_MASK                                                               0x0002L
+#define BIF_CFG_DEV0_RC0_COMMAND__BUS_MASTER_EN_MASK                                                          0x0004L
+#define BIF_CFG_DEV0_RC0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV0_RC0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_RC0_COMMAND__PAL_SNOOP_EN_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_RC0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_RC0_COMMAND__AD_STEPPING_MASK                                                            0x0080L
+#define BIF_CFG_DEV0_RC0_COMMAND__SERR_EN_MASK                                                                0x0100L
+#define BIF_CFG_DEV0_RC0_COMMAND__FAST_B2B_EN_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_RC0_COMMAND__INT_DIS_MASK                                                                0x0400L
+//BIF_CFG_DEV0_RC0_STATUS
+#define BIF_CFG_DEV0_RC0_STATUS__IMMEDIATE_READINESS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC0_STATUS__INT_STATUS__SHIFT                                                            0x3
+#define BIF_CFG_DEV0_RC0_STATUS__CAP_LIST__SHIFT                                                              0x4
+#define BIF_CFG_DEV0_RC0_STATUS__PCI_66_CAP__SHIFT                                                            0x5
+#define BIF_CFG_DEV0_RC0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_RC0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                              0x8
+#define BIF_CFG_DEV0_RC0_STATUS__DEVSEL_TIMING__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_RC0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_RC0_STATUS__IMMEDIATE_READINESS_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_RC0_STATUS__INT_STATUS_MASK                                                              0x0008L
+#define BIF_CFG_DEV0_RC0_STATUS__CAP_LIST_MASK                                                                0x0010L
+#define BIF_CFG_DEV0_RC0_STATUS__PCI_66_CAP_MASK                                                              0x0020L
+#define BIF_CFG_DEV0_RC0_STATUS__FAST_BACK_CAPABLE_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_RC0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                0x0100L
+#define BIF_CFG_DEV0_RC0_STATUS__DEVSEL_TIMING_MASK                                                           0x0600L
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_RC0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_RC0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                   0x4000L
+#define BIF_CFG_DEV0_RC0_STATUS__PARITY_ERROR_DETECTED_MASK                                                   0x8000L
+//BIF_CFG_DEV0_RC0_REVISION_ID
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MINOR_REV_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MINOR_REV_ID_MASK                                                       0x0FL
+#define BIF_CFG_DEV0_RC0_REVISION_ID__MAJOR_REV_ID_MASK                                                       0xF0L
+//BIF_CFG_DEV0_RC0_PROG_INTERFACE
+#define BIF_CFG_DEV0_RC0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_PROG_INTERFACE__PROG_INTERFACE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_RC0_SUB_CLASS
+#define BIF_CFG_DEV0_RC0_SUB_CLASS__SUB_CLASS__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_SUB_CLASS__SUB_CLASS_MASK                                                            0xFFL
+//BIF_CFG_DEV0_RC0_BASE_CLASS
+#define BIF_CFG_DEV0_RC0_BASE_CLASS__BASE_CLASS__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_BASE_CLASS__BASE_CLASS_MASK                                                          0xFFL
+//BIF_CFG_DEV0_RC0_CACHE_LINE
+#define BIF_CFG_DEV0_RC0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                     0xFFL
+//BIF_CFG_DEV0_RC0_LATENCY
+#define BIF_CFG_DEV0_RC0_LATENCY__LATENCY_TIMER__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_LATENCY__LATENCY_TIMER_MASK                                                          0xFFL
+//BIF_CFG_DEV0_RC0_HEADER
+#define BIF_CFG_DEV0_RC0_HEADER__HEADER_TYPE__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC0_HEADER__DEVICE_TYPE__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_RC0_HEADER__HEADER_TYPE_MASK                                                             0x7FL
+#define BIF_CFG_DEV0_RC0_HEADER__DEVICE_TYPE_MASK                                                             0x80L
+//BIF_CFG_DEV0_RC0_BIST
+#define BIF_CFG_DEV0_RC0_BIST__BIST_COMP__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC0_BIST__BIST_STRT__SHIFT                                                               0x6
+#define BIF_CFG_DEV0_RC0_BIST__BIST_CAP__SHIFT                                                                0x7
+#define BIF_CFG_DEV0_RC0_BIST__BIST_COMP_MASK                                                                 0x0FL
+#define BIF_CFG_DEV0_RC0_BIST__BIST_STRT_MASK                                                                 0x40L
+#define BIF_CFG_DEV0_RC0_BIST__BIST_CAP_MASK                                                                  0x80L
+//BIF_CFG_DEV0_RC0_BASE_ADDR_1
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_1__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_BASE_ADDR_2
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_BASE_ADDR_2__BASE_ADDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                               0x18
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                           0x0000FF00L
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                             0x00FF0000L
+#define BIF_CFG_DEV0_RC0_SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                 0xFF000000L
+//BIF_CFG_DEV0_RC0_IO_BASE_LIMIT
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                     0x000FL
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_BASE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                    0x0F00L
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT__IO_LIMIT_MASK                                                         0xF000L
+//BIF_CFG_DEV0_RC0_SECONDARY_STATUS
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                           0x7
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                    0x8
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                               0x9
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                         0xb
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                       0xc
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                       0xd
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                       0xe
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PCI_66_CAP_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                             0x0080L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                      0x0100L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                 0x0600L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                           0x0800L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                         0x1000L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                         0x2000L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                         0x4000L
+#define BIF_CFG_DEV0_RC0_SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                         0x8000L
+//BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                0x4
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                   0x0000000FL
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                  0x0000FFF0L
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                             0x0000000FL
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                            0x0000FFF0L
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_RC0_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_PREF_BASE_UPPER
+#define BIF_CFG_DEV0_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER
+#define BIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                              0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                              0x10
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                0xFFFF0000L
+//BIF_CFG_DEV0_RC0_CAP_PTR
+#define BIF_CFG_DEV0_RC0_CAP_PTR__CAP_PTR__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_CAP_PTR__CAP_PTR_MASK                                                                0xFFL
+//BIF_CFG_DEV0_RC0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                          0x1
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                      0xb
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                       0x00000001L
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                            0x0000000EL
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                           0x000000F0L
+#define BIF_CFG_DEV0_RC0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                        0xFFFFF800L
+//BIF_CFG_DEV0_RC0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_RC0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_RC0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_RC0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                    0xFFL
+//BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                      0x1
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                        0x8
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                      0x9
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                         0xa
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                    0xb
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                        0x0002L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                              0x0020L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                          0x0100L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                        0x0200L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                           0x0400L
+#define BIF_CFG_DEV0_RC0_IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                      0x0800L
+//BIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL
+#define BIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                  0x01L
+//BIF_CFG_DEV0_RC0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC0_PMI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC0_PMI_CAP
+#define BIF_CFG_DEV0_RC0_PMI_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_CLOCK__SHIFT                                                            0x3
+#define BIF_CFG_DEV0_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                  0x4
+#define BIF_CFG_DEV0_RC0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_RC0_PMI_CAP__AUX_CURRENT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D1_SUPPORT__SHIFT                                                           0x9
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D2_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_SUPPORT__SHIFT                                                          0xb
+#define BIF_CFG_DEV0_RC0_PMI_CAP__VERSION_MASK                                                                0x0007L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_CLOCK_MASK                                                              0x0008L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                    0x0010L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__AUX_CURRENT_MASK                                                            0x01C0L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D1_SUPPORT_MASK                                                             0x0200L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__D2_SUPPORT_MASK                                                             0x0400L
+#define BIF_CFG_DEV0_RC0_PMI_CAP__PME_SUPPORT_MASK                                                            0xF800L
+//BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                0x16
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                   0x17
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                     0x18
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                    0x00000003L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                  0x00000008L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_EN_MASK                                                         0x00000100L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                    0x00001E00L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                     0x00006000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                     0x00008000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                  0x00400000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                     0x00800000L
+#define BIF_CFG_DEV0_RC0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                       0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_RC0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_RC0_PCIE_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__VERSION_MASK                                                               0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__DEVICE_TYPE_MASK                                                           0x00F0L
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                      0x0100L
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                       0x3E00L
+#define BIF_CFG_DEV0_RC0_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                     0x8000L
+//BIF_CFG_DEV0_RC0_DEVICE_CAP
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                      0x3
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                             0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                          0xf
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                      0x11
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                         0x1a
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                       0x1c
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                               0x1d
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                        0x00000018L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__EXTENDED_TAG_MASK                                                        0x00000020L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                              0x000001C0L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                               0x00000E00L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                            0x00008000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                            0x00010000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__RX_MPS_FIXED_MASK                                                        0x00020000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                           0x03FC0000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                           0x0C000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__FLR_CAPABLE_MASK                                                         0x10000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                                 0x20000000L
+//BIF_CFG_DEV0_RC0_DEVICE_CNTL
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                  0xa
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                      0xb
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                            0xc
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                              0xf
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                   0x00E0L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                    0x0100L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                    0x0200L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                    0x0400L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                        0x0800L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                              0x7000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                0x8000L
+//BIF_CFG_DEV0_RC0_DEVICE_STATUS
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__CORR_ERR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                      0x2
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__AUX_PWR__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                              0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__CORR_ERR_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                    0x0002L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__FATAL_ERR_MASK                                                        0x0004L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__USR_DETECTED_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__AUX_PWR_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                0x0020L
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                    0x0040L
+//BIF_CFG_DEV0_RC0_LINK_CAP
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_SPEED__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_WIDTH__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PM_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                              0x12
+#define BIF_CFG_DEV0_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                         0x13
+#define BIF_CFG_DEV0_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                            0x15
+#define BIF_CFG_DEV0_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                         0x16
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PORT_NUMBER__SHIFT                                                         0x18
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_SPEED_MASK                                                            0x0000000FL
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_WIDTH_MASK                                                            0x000003F0L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PM_SUPPORT_MASK                                                            0x00000C00L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                      0x00007000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                       0x00038000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                0x00040000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                           0x00080000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                           0x00100000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                              0x00200000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                           0x00400000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP__PORT_NUMBER_MASK                                                           0xFF000000L
+//BIF_CFG_DEV0_RC0_LINK_CNTL
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PM_CONTROL__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                       0x2
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_DIS__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                   0x6
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                          0xb
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                              0xe
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PM_CONTROL_MASK                                                           0x0003L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                         0x0004L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_DIS_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__RETRAIN_LINK_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                     0x0040L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__EXTENDED_SYNC_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                            0x0100L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                            0x0400L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                            0x0800L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                0xC000L
+//BIF_CFG_DEV0_RC0_LINK_STATUS
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                            0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_TRAINING__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__DL_ACTIVE__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                        0xe
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                        0xf
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                 0x000FL
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                              0x03F0L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_TRAINING_MASK                                                      0x0800L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__DL_ACTIVE_MASK                                                          0x2000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                          0x4000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                          0x8000L
+//BIF_CFG_DEV0_RC0_SLOT_CAP
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                              0x3
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                               0x4
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                     0x6
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                      0x12
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                   0x13
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                   0x00000001L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                0x00000002L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                    0x00000004L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                0x00000008L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                 0x00000010L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                      0x00000020L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                       0x00000040L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                  0x00007F80L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                  0x00018000L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                         0x00020000L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                        0x00040000L
+#define BIF_CFG_DEV0_RC0_SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                     0xFFF80000L
+//BIF_CFG_DEV0_RC0_SLOT_CNTL
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                              0x2
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                         0x3
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                0x6
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                0xa
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                         0xb
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                0xc
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                        0xd
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                  0xe
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                               0x0001L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                0x0004L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                           0x0008L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                  0x00C0L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                   0x0300L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                  0x0400L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                           0x0800L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                          0x2000L
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                    0x4000L
+//BIF_CFG_DEV0_RC0_SLOT_STATUS
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                0x4
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                     0x7
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                0x0001L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                 0x0004L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                            0x0008L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__COMMAND_COMPLETED_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                              0x0040L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                       0x0080L
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS__DL_STATE_CHANGED_MASK                                                   0x0100L
+//BIF_CFG_DEV0_RC0_ROOT_CNTL
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                 0x0004L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_RC0_ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                           0x0010L
+//BIF_CFG_DEV0_RC0_ROOT_CAP
+#define BIF_CFG_DEV0_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                               0x0001L
+//BIF_CFG_DEV0_RC0_ROOT_STATUS
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_STATUS__SHIFT                                                       0x10
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_PENDING__SHIFT                                                      0x11
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_STATUS_MASK                                                         0x00010000L
+#define BIF_CFG_DEV0_RC0_ROOT_STATUS__PME_PENDING_MASK                                                        0x00020000L
+//BIF_CFG_DEV0_RC0_DEVICE_CAP2
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                        0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                         0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                       0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                       0x7
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                           0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                        0xa
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                               0xc
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                    0xe
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                  0x10
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                  0x11
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                   0x12
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                     0x14
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                     0x15
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                         0x16
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                   0x18
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                    0x1f
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                        0x0000000FL
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                          0x00000010L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                           0x00000020L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                         0x00000080L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                             0x00000200L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                          0x00000400L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                      0x00000800L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                 0x00003000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                      0x0000C000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                     0x000C0000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                       0x00100000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                           0x00C00000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                     0x03000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                      0x04000000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                      0x80000000L
+//BIF_CFG_DEV0_RC0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                        0x7
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                              0x8
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                           0x9
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__LTR_EN__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                    0xb
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                    0xc
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                         0xd
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                 0x000FL
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                 0x0020L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                               0x0040L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                          0x0080L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                0x0100L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                             0x0200L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__LTR_EN_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                      0x0800L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                      0x1000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__OBFF_EN_MASK                                                           0x6000L
+#define BIF_CFG_DEV0_RC0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                       0x8000L
+//BIF_CFG_DEV0_RC0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS2__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_DEVICE_STATUS2__RESERVED_MASK                                                        0xFFFFL
+//BIF_CFG_DEV0_RC0_LINK_CAP2
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                0x8
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                           0x9
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                          0x17
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                          0x18
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                      0x1f
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                 0x000000FEL
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                  0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                             0x0000FE00L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                             0x007F0000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                            0x00800000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                            0x01000000L
+#define BIF_CFG_DEV0_RC0_LINK_CAP2__DRS_SUPPORTED_MASK                                                        0x80000000L
+//BIF_CFG_DEV0_RC0_LINK_CNTL2
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                       0x5
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                              0xa
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                             0xc
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                   0x000FL
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                         0x0020L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                               0x0040L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__XMIT_MARGIN_MASK                                                         0x0380L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                0x0400L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                      0x0800L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                               0xF000L
+//BIF_CFG_DEV0_RC0_LINK_STATUS2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                 0x2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                 0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                   0x5
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                               0x6
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                               0x7
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                            0x8
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                                0xa
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                              0x0001L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                         0x0002L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                   0x0004L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                   0x0008L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                   0x0010L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                     0x0020L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                 0x0080L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                              0x0300L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                  0x0400L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                     0x7000L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                              0x8000L
+//BIF_CFG_DEV0_RC0_SLOT_CAP2
+#define BIF_CFG_DEV0_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                          0x00000001L
+//BIF_CFG_DEV0_RC0_SLOT_CNTL2
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL2__RESERVED__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_SLOT_CNTL2__RESERVED_MASK                                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_SLOT_STATUS2
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS2__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_SLOT_STATUS2__RESERVED_MASK                                                          0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC0_MSI_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EN_MASK                                                            0x0001L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                     0x000EL
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                      0x0070L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                         0x0100L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                              0x0200L
+#define BIF_CFG_DEV0_RC0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                               0x0400L
+//BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                              0x2
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                0xFFFFFFFCL
+//BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA__MSI_DATA_MASK                                                          0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                  0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                    0xFFFFL
+//BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                            0xFFFFL
+//BIF_CFG_DEV0_RC0_SSID_CAP_LIST
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_RC0_SSID_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_RC0_SSID_CAP
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_ID__SHIFT                                                        0x10
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_SSID_CAP__SUBSYSTEM_ID_MASK                                                          0xFFFF0000L
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                   0x14
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                       0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                      0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                     0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                              0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                 0x1
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                      0x9
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                   0x00000002L
+#define BIF_CFG_DEV0_RC0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                        0x0000FE00L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                  0x0000FFFFL
+//BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0xc
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                   0x000FL
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x0070L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                     0x0F00L
+#define BIF_CFG_DEV0_RC0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x7000L
+//BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC0_LINK_CAP_16GT
+#define BIF_CFG_DEV0_RC0_LINK_CAP_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_LINK_CAP_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_16GT__RESERVED__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_16GT__RESERVED_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                            0x1
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                              0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                              0x00000004L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                              0x00000008L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                0x00000010L
+//BIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT          0x0
+#define BIF_CFG_DEV0_RC0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK            0x0000FFFFL
+//BIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV0_RC0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT            0x0
+#define BIF_CFG_DEV0_RC0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK              0x0000FFFFL
+//BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC0_LINK_CAP_32GT
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                              0x9
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                              0xa
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                               0xb
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                              0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                                0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                                0x00000200L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                                0x00000400L
+#define BIF_CFG_DEV0_RC0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                 0x0000F800L
+//BIF_CFG_DEV0_RC0_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                    0x8
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                                0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                      0x00000700L
+//BIF_CFG_DEV0_RC0_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                            0x1
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                    0x8
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                 0x9
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                       0xa
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                              0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                              0x00000004L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                              0x00000008L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                                0x00000010L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                          0x000000C0L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                   0x00000200L
+#define BIF_CFG_DEV0_RC0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                         0x00000400L
+//BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                            0x3
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                          0x00000007L
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                              0x0000FFF8L
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                            0xFFFF0000L
+//BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT             0x18
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                              0x00FFFFFFL
+#define BIF_CFG_DEV0_RC0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK               0x03000000L
+//BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                         0x3
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                       0x10
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                           0x0000FFF8L
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                         0xFFFF0000L
+//BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT          0x18
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                           0x00FFFFFFL
+#define BIF_CFG_DEV0_RC0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK            0x03000000L
+//BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                      0x4
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                        0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                        0xF0L
+//BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                    0x4
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                      0x0FL
+#define BIF_CFG_DEV0_RC0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                      0xF0L
+//BIF_CFG_DEV0_RC0_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC0_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_AP_CAP
+#define BIF_CFG_DEV0_RC0_AP_CAP__COUNT__SHIFT                                                                 0x0
+#define BIF_CFG_DEV0_RC0_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_RC0_AP_CAP__COUNT_MASK                                                                   0x000000FFL
+#define BIF_CFG_DEV0_RC0_AP_CAP__SEL_EN_SUPPORTED_MASK                                                        0x00000100L
+//BIF_CFG_DEV0_RC0_AP_CNTL
+#define BIF_CFG_DEV0_RC0_AP_CNTL__INX_SEL__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC0_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC0_AP_CNTL__INX_SEL_MASK                                                                0x000000FFL
+#define BIF_CFG_DEV0_RC0_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                         0x00000100L
+//BIF_CFG_DEV0_RC0_AP_DATA1
+#define BIF_CFG_DEV0_RC0_AP_DATA1__USAGE_INFOR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_AP_DATA1__DETAILS__SHIFT                                                             0x5
+#define BIF_CFG_DEV0_RC0_AP_DATA1__VENDORID__SHIFT                                                            0x10
+#define BIF_CFG_DEV0_RC0_AP_DATA1__USAGE_INFOR_MASK                                                           0x00000007L
+#define BIF_CFG_DEV0_RC0_AP_DATA1__DETAILS_MASK                                                               0x0000FFE0L
+#define BIF_CFG_DEV0_RC0_AP_DATA1__VENDORID_MASK                                                              0xFFFF0000L
+//BIF_CFG_DEV0_RC0_AP_DATA2
+#define BIF_CFG_DEV0_RC0_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                    0x00FFFFFFL
+//BIF_CFG_DEV0_RC0_AP_SEL_EN_MASK
+#define BIF_CFG_DEV0_RC0_AP_SEL_EN_MASK__PCIE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC0_AP_SEL_EN_MASK__OTHERS__SHIFT                                                        0x1
+#define BIF_CFG_DEV0_RC0_AP_SEL_EN_MASK__PCIE_MASK                                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_AP_SEL_EN_MASK__OTHERS_MASK                                                          0xFFFFFFFEL
+//BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                         0x14
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                             0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                            0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                           0xFFF00000L
+//BIF_CFG_DEV0_RC0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                   0x0
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                     0x0001L
+//BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                        0x0
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                          0x0001L
+#define BIF_CFG_DEV0_RC0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                 0x0002L
+//BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                            0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                0x3
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                             0x8
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                              0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                  0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                  0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                               0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                       0x3
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                       0x6
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                     0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                         0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                         0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                      0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                          0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                              0x6
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                           0x8
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                            0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                             0xFF00L
+//BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                 0x0
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                     0x3
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                     0x6
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                  0x8
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                   0x0007L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                       0x0038L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                       0x0040L
+#define BIF_CFG_DEV0_RC0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                    0xFF00L
+//BIF_CFG_DEV0_RC0_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                              0x10
+#define BIF_CFG_DEV0_RC0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                             0x14
+#define BIF_CFG_DEV0_RC0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                                0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                               0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                              0x4
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                0x8
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                              0xa
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                             0x00000007L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                0x00000070L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                  0x00000300L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                0x00000C00L
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                    0x18
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                      0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                            0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                0x000EL
+//BIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                        0x0001L
+//BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x007F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                         0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                  0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                           0x003F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                    0xFF000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                       0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                 0x18
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                             0x1f
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                         0x000000FEL
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                         0x000E0000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                   0x07000000L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                               0x80000000L
+//BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                              0x1
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                 0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                0x0002L
+//BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                    0x0
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                    0x1f
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                      0x007FFFFFL
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                      0x80000000L
+//BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                  0x007FFFFFL
+#define BIF_CFG_DEV0_RC0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                              0x10
+#define BIF_CFG_DEV0_RC0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                             0x14
+#define BIF_CFG_DEV0_RC0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                                0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                               0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                     0x5
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                         0xd
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                      0x10
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                       0x11
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                        0x12
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                       0x13
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                  0x15
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                 0x16
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                 0x17
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                        0x18
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                         0x19
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                    0x1a
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT                    0x1b
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                               0x1c
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                              0x1d
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                              0x1e
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                          0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                       0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                          0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                           0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                    0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                        0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                         0x00020000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                          0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                         0x00080000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                   0x00100000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                   0x00400000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                   0x00800000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                          0x01000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                           0x02000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                      0x04000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                      0x08000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                                 0x10000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                                0x20000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                                0x40000000L
+//BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                            0x4
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                         0x5
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                            0xc
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                             0xd
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                           0x11
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                            0x12
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                           0x13
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                     0x14
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                      0x15
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                     0x16
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                     0x17
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                            0x18
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                             0x19
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                        0x1a
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                        0x1b
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                                   0x1c
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                                  0x1d
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                                  0x1e
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                              0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                           0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                              0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                               0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                        0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                            0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                             0x00020000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                              0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                             0x00080000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                       0x00100000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                        0x00200000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                       0x00400000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                              0x01000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                               0x02000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                          0x04000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                          0x08000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                     0x10000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                                    0x20000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                                    0x40000000L
+//BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                    0x4
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                 0x5
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                    0xc
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                     0xd
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                              0xf
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                  0x10
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                   0x11
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                    0x12
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                   0x13
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                             0x14
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                              0x15
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                             0x16
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                             0x17
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                    0x18
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                     0x19
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                0x1a
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT                0x1b
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                           0x1c
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                          0x1d
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                          0x1e
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                      0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                   0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                      0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                       0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                0x00008000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                    0x00010000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                     0x00020000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                      0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                     0x00080000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                               0x00100000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                0x00200000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                               0x00400000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                               0x00800000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                      0x01000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                       0x02000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                  0x04000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK                  0x08000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                             0x10000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                            0x20000000L
+#define BIF_CFG_DEV0_RC0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                            0x40000000L
+//BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                             0xc
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                            0xd
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                            0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                           0x00000080L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                0x00000100L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                               0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                              0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                       0x00008000L
+//BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                              0x6
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                             0x7
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                 0xc
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                0xd
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                         0xf
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                               0x00000080L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV0_RC0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                           0x00008000L
+//BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                           0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                     0x9
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                      0xa
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                 0xb
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                         0xc
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                        0xd
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                               0x12
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                        0x13
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                            0x0000001FL
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                             0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                              0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                           0x00000080L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                            0x00000100L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                       0x00000200L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                        0x00000400L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                   0x00000800L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                           0x00001000L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                          0x0003E000L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_RC0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                          0x00F80000L
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                           0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                             0x00000004L
+//BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                      0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                 0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                            0x3
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                 0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                    0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                     0x1b
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                        0x00000002L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                   0x00000004L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                              0x00000008L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                   0x00000020L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                          0x00000180L
+#define BIF_CFG_DEV0_RC0_PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                       0xF8000000L
+//BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                      0xFFFF0000L
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG4__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG5__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG6__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG7__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG8__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG9__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG10__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG11__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG12__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC0_PCIE_HDR_LOG13__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                            0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                            0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                         0x3
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                             0x4
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                              0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                           0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                             0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                 0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                              0x0002L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                              0x0004L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                           0x0008L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                               0x0010L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                0x0020L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                             0x0040L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                               0x0080L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                        0xFF00L
+//BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                        0x2
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                     0x3
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                  0xa
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                0xc
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                             0x0001L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                          0x0004L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                       0x0008L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                         0x0040L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                           0x0080L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0300L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                    0x0C00L
+#define BIF_CFG_DEV0_RC0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                  0x1000L
+//BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                                0x10
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                               0x14
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                   0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                  0x000F0000L
+#define BIF_CFG_DEV0_RC0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                 0xFFF00000L
+//BIF_CFG_DEV0_RC0_RTR_DATA1
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__RESET_TIME__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__DLUP_TIME__SHIFT                                                          0xc
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__RTR_VALID__SHIFT                                                          0x1f
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__RESET_TIME_MASK                                                           0x00000FFFL
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__DLUP_TIME_MASK                                                            0x00FFF000L
+#define BIF_CFG_DEV0_RC0_RTR_DATA1__RTR_VALID_MASK                                                            0x80000000L
+//BIF_CFG_DEV0_RC0_RTR_DATA2
+#define BIF_CFG_DEV0_RC0_RTR_DATA2__FLR_TIME__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_RC0_RTR_DATA2__FLR_TIME_MASK                                                             0x00000FFFL
+#define BIF_CFG_DEV0_RC0_RTR_DATA2__D3HOTD0_TIME_MASK                                                         0x00FFF000L
+//BIF_CFG_DEV0_RC0_IDE_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC0_IDE_ENH_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC0_IDE_ENH_CAP_LIST__CAP_VER__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_RC0_IDE_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                    0x14
+#define BIF_CFG_DEV0_RC0_IDE_ENH_CAP_LIST__CAP_ID_MASK                                                        0x0000FFFFL
+#define BIF_CFG_DEV0_RC0_IDE_ENH_CAP_LIST__CAP_VER_MASK                                                       0x000F0000L
+#define BIF_CFG_DEV0_RC0_IDE_ENH_CAP_LIST__NEXT_PTR_MASK                                                      0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_CAP
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_LINK_IDE_STREAM_SUPPORTED__SHIFT                                    0x0
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_STREAM_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED__SHIFT                            0x2
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED__SHIFT                             0x3
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_AGGREGATION_SUPPORTED__SHIFT                                        0x4
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_PCRC_SUPPORTED__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_IDE_KM_PROTOCOL_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED__SHIFT                          0x7
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_SUPPORTED_ALGORITHMS__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE__SHIFT                            0xd
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED__SHIFT                       0x10
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_TEE_LIMITED_SUPPORTED__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_XT_SUPPORTED__SHIFT                                                 0x19
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_LINK_IDE_STREAM_SUPPORTED_MASK                                      0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_STREAM_SUPPORTED_MASK                                 0x00000002L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_MASK                              0x00000004L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_MASK                               0x00000008L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_AGGREGATION_SUPPORTED_MASK                                          0x00000010L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_PCRC_SUPPORTED_MASK                                                 0x00000020L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_IDE_KM_PROTOCOL_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_MASK                            0x00000080L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_SUPPORTED_ALGORITHMS_MASK                                           0x00001F00L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_MASK                              0x0000E000L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_MASK                         0x00FF0000L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_TEE_LIMITED_SUPPORTED_MASK                                          0x01000000L
+#define BIF_CFG_DEV0_RC0_IDE_CAP__IDE_CAP_XT_SUPPORTED_MASK                                                   0x02000000L
+//BIF_CFG_DEV0_RC0_IDE_CNTL
+#define BIF_CFG_DEV0_RC0_IDE_CNTL__IDE_CNTL_FLOW_THROUGH_IDE_STREAM_ENABLED__SHIFT                            0x2
+#define BIF_CFG_DEV0_RC0_IDE_CNTL__IDE_CNTL_FLOW_THROUGH_IDE_STREAM_ENABLED_MASK                              0x00000004L
+//BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__LINK_IDE_STREAM_ENABLE__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__XT_ENABLE__SHIFT                                             0x1
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                               0x2
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                                0x4
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__PCRC_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                              0xa
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM__SHIFT                                    0xe
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__TC__SHIFT                                                    0x13
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__STREAM_ID__SHIFT                                             0x18
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__LINK_IDE_STREAM_ENABLE_MASK                                  0x00000001L
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__XT_ENABLE_MASK                                               0x00000002L
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR_MASK                                 0x0000000CL
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR_MASK                                  0x00000030L
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL_MASK                                 0x000000C0L
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__PCRC_ENABLE_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                                0x00003C00L
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM_MASK                                      0x0007C000L
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__TC_MASK                                                      0x00380000L
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_CNTL__STREAM_ID_MASK                                               0xFF000000L
+//BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_STATUS
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_STATUS__LINK_IDE_STREAM_STATE__SHIFT                               0x0
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                               0x1f
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_STATUS__LINK_IDE_STREAM_STATE_MASK                                 0x0000000FL
+#define BIF_CFG_DEV0_RC0_LINK_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                                 0x80000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CAP
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                       0x0000000FL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__XT_ENABLE__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                          0x2
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                           0x4
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                          0x6
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__PCRC_ENABLE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                     0x9
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                         0xa
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM__SHIFT                               0xe
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TC__SHIFT                                               0x13
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__DEFAULT_STREAM__SHIFT                                   0x16
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TEE_LIMITED_STREAM__SHIFT                               0x17
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__STREAM_ID__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                        0x00000001L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__XT_ENABLE_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR_MASK                            0x0000000CL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR_MASK                             0x00000030L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL_MASK                            0x000000C0L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__PCRC_ENABLE_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                       0x00000200L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                           0x00003C00L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM_MASK                                 0x0007C000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TC_MASK                                                 0x00380000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__DEFAULT_STREAM_MASK                                     0x00400000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__TEE_LIMITED_STREAM_MASK                                 0x00800000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_CNTL__STREAM_ID_MASK                                          0xFF000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_STATUS
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                       0x0000000FL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                           0x00FFFF00L
+//BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG2__VALID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                      0x18
+#define BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG2__VALID_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG2__RID_BASE_MASK                                            0x00FFFF00L
+#define BIF_CFG_DEV0_RC0_IDE_0_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                        0xFF000000L
+//BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_0_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CAP
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                       0x0000000FL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__XT_ENABLE__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                          0x2
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                           0x4
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                          0x6
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__PCRC_ENABLE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                     0x9
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                         0xa
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTED_ALGORITHM__SHIFT                               0xe
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TC__SHIFT                                               0x13
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__DEFAULT_STREAM__SHIFT                                   0x16
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TEE_LIMITED_STREAM__SHIFT                               0x17
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__STREAM_ID__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                        0x00000001L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__XT_ENABLE_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_NPR_MASK                            0x0000000CL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_PR_MASK                             0x00000030L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_CPL_MASK                            0x000000C0L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__PCRC_ENABLE_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                       0x00000200L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                           0x00003C00L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTED_ALGORITHM_MASK                                 0x0007C000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TC_MASK                                                 0x00380000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__DEFAULT_STREAM_MASK                                     0x00400000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__TEE_LIMITED_STREAM_MASK                                 0x00800000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_CNTL__STREAM_ID_MASK                                          0xFF000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_STATUS
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                       0x0000000FL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_1_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                           0x00FFFF00L
+//BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG2__VALID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                      0x18
+#define BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG2__VALID_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG2__RID_BASE_MASK                                            0x00FFFF00L
+#define BIF_CFG_DEV0_RC0_IDE_1_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                        0xFF000000L
+//BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_1_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CAP
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                       0x0000000FL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__XT_ENABLE__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                          0x2
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                           0x4
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                          0x6
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__PCRC_ENABLE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                     0x9
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                         0xa
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTED_ALGORITHM__SHIFT                               0xe
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TC__SHIFT                                               0x13
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__DEFAULT_STREAM__SHIFT                                   0x16
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TEE_LIMITED_STREAM__SHIFT                               0x17
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__STREAM_ID__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                        0x00000001L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__XT_ENABLE_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_NPR_MASK                            0x0000000CL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_PR_MASK                             0x00000030L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_CPL_MASK                            0x000000C0L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__PCRC_ENABLE_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                       0x00000200L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                           0x00003C00L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTED_ALGORITHM_MASK                                 0x0007C000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TC_MASK                                                 0x00380000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__DEFAULT_STREAM_MASK                                     0x00400000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__TEE_LIMITED_STREAM_MASK                                 0x00800000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_CNTL__STREAM_ID_MASK                                          0xFF000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_STATUS
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                       0x0000000FL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_2_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                           0x00FFFF00L
+//BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG2__VALID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                      0x18
+#define BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG2__VALID_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG2__RID_BASE_MASK                                            0x00FFFF00L
+#define BIF_CFG_DEV0_RC0_IDE_2_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                        0xFF000000L
+//BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_2_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CAP
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                       0x0000000FL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__XT_ENABLE__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                          0x2
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                           0x4
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                          0x6
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__PCRC_ENABLE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                     0x9
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                         0xa
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTED_ALGORITHM__SHIFT                               0xe
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TC__SHIFT                                               0x13
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__DEFAULT_STREAM__SHIFT                                   0x16
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TEE_LIMITED_STREAM__SHIFT                               0x17
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__STREAM_ID__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                        0x00000001L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__XT_ENABLE_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_NPR_MASK                            0x0000000CL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_PR_MASK                             0x00000030L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_CPL_MASK                            0x000000C0L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__PCRC_ENABLE_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                       0x00000200L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                           0x00003C00L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTED_ALGORITHM_MASK                                 0x0007C000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TC_MASK                                                 0x00380000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__DEFAULT_STREAM_MASK                                     0x00400000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__TEE_LIMITED_STREAM_MASK                                 0x00800000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_CNTL__STREAM_ID_MASK                                          0xFF000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_STATUS
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                       0x0000000FL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_3_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                           0x00FFFF00L
+//BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG2__VALID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                      0x18
+#define BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG2__VALID_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG2__RID_BASE_MASK                                            0x00FFFF00L
+#define BIF_CFG_DEV0_RC0_IDE_3_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                        0xFF000000L
+//BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_3_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CAP
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                       0x0000000FL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__XT_ENABLE__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                          0x2
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                           0x4
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                          0x6
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__PCRC_ENABLE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                     0x9
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                         0xa
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTED_ALGORITHM__SHIFT                               0xe
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TC__SHIFT                                               0x13
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__DEFAULT_STREAM__SHIFT                                   0x16
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TEE_LIMITED_STREAM__SHIFT                               0x17
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__STREAM_ID__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                        0x00000001L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__XT_ENABLE_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_NPR_MASK                            0x0000000CL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_PR_MASK                             0x00000030L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_CPL_MASK                            0x000000C0L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__PCRC_ENABLE_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                       0x00000200L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                           0x00003C00L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTED_ALGORITHM_MASK                                 0x0007C000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TC_MASK                                                 0x00380000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__DEFAULT_STREAM_MASK                                     0x00400000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__TEE_LIMITED_STREAM_MASK                                 0x00800000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_CNTL__STREAM_ID_MASK                                          0xFF000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_STATUS
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                       0x0000000FL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_4_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                           0x00FFFF00L
+//BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG2__VALID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                      0x18
+#define BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG2__VALID_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG2__RID_BASE_MASK                                            0x00FFFF00L
+#define BIF_CFG_DEV0_RC0_IDE_4_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                        0xFF000000L
+//BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_4_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CAP
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                       0x0000000FL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__XT_ENABLE__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                          0x2
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                           0x4
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                          0x6
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__PCRC_ENABLE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                     0x9
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                         0xa
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTED_ALGORITHM__SHIFT                               0xe
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TC__SHIFT                                               0x13
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__DEFAULT_STREAM__SHIFT                                   0x16
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TEE_LIMITED_STREAM__SHIFT                               0x17
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__STREAM_ID__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                        0x00000001L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__XT_ENABLE_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_NPR_MASK                            0x0000000CL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_PR_MASK                             0x00000030L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_CPL_MASK                            0x000000C0L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__PCRC_ENABLE_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                       0x00000200L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                           0x00003C00L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTED_ALGORITHM_MASK                                 0x0007C000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TC_MASK                                                 0x00380000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__DEFAULT_STREAM_MASK                                     0x00400000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__TEE_LIMITED_STREAM_MASK                                 0x00800000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_CNTL__STREAM_ID_MASK                                          0xFF000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_STATUS
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                       0x0000000FL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_5_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                           0x00FFFF00L
+//BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG2__VALID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                      0x18
+#define BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG2__VALID_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG2__RID_BASE_MASK                                            0x00FFFF00L
+#define BIF_CFG_DEV0_RC0_IDE_5_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                        0xFF000000L
+//BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_5_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CAP
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                       0x0000000FL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__XT_ENABLE__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                          0x2
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                           0x4
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                          0x6
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__PCRC_ENABLE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                     0x9
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                         0xa
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTED_ALGORITHM__SHIFT                               0xe
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TC__SHIFT                                               0x13
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__DEFAULT_STREAM__SHIFT                                   0x16
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TEE_LIMITED_STREAM__SHIFT                               0x17
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__STREAM_ID__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                        0x00000001L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__XT_ENABLE_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_NPR_MASK                            0x0000000CL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_PR_MASK                             0x00000030L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_CPL_MASK                            0x000000C0L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__PCRC_ENABLE_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                       0x00000200L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                           0x00003C00L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTED_ALGORITHM_MASK                                 0x0007C000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TC_MASK                                                 0x00380000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__DEFAULT_STREAM_MASK                                     0x00400000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__TEE_LIMITED_STREAM_MASK                                 0x00800000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_CNTL__STREAM_ID_MASK                                          0xFF000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_STATUS
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                       0x0000000FL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_6_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                           0x00FFFF00L
+//BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG2__VALID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                      0x18
+#define BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG2__VALID_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG2__RID_BASE_MASK                                            0x00FFFF00L
+#define BIF_CFG_DEV0_RC0_IDE_6_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                        0xFF000000L
+//BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_6_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CAP
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                       0x0000000FL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__XT_ENABLE__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                          0x2
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                           0x4
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                          0x6
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__PCRC_ENABLE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                     0x9
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                         0xa
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTED_ALGORITHM__SHIFT                               0xe
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TC__SHIFT                                               0x13
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__DEFAULT_STREAM__SHIFT                                   0x16
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TEE_LIMITED_STREAM__SHIFT                               0x17
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__STREAM_ID__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                        0x00000001L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__XT_ENABLE_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_NPR_MASK                            0x0000000CL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_PR_MASK                             0x00000030L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_CPL_MASK                            0x000000C0L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__PCRC_ENABLE_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                       0x00000200L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                           0x00003C00L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTED_ALGORITHM_MASK                                 0x0007C000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TC_MASK                                                 0x00380000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__DEFAULT_STREAM_MASK                                     0x00400000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__TEE_LIMITED_STREAM_MASK                                 0x00800000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_CNTL__STREAM_ID_MASK                                          0xFF000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_STATUS
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                       0x0000000FL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_7_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                           0x00FFFF00L
+//BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG2__VALID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                      0x18
+#define BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG2__VALID_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG2__RID_BASE_MASK                                            0x00FFFF00L
+#define BIF_CFG_DEV0_RC0_IDE_7_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                        0xFF000000L
+//BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_7_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CAP
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                       0x0000000FL
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__XT_ENABLE__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                          0x2
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                           0x4
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                          0x6
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__PCRC_ENABLE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                     0x9
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                         0xa
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTED_ALGORITHM__SHIFT                               0xe
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TC__SHIFT                                               0x13
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__DEFAULT_STREAM__SHIFT                                   0x16
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TEE_LIMITED_STREAM__SHIFT                               0x17
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__STREAM_ID__SHIFT                                        0x18
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                        0x00000001L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__XT_ENABLE_MASK                                          0x00000002L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_NPR_MASK                            0x0000000CL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_PR_MASK                             0x00000030L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_CPL_MASK                            0x000000C0L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__PCRC_ENABLE_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                       0x00000200L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                           0x00003C00L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTED_ALGORITHM_MASK                                 0x0007C000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TC_MASK                                                 0x00380000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__DEFAULT_STREAM_MASK                                     0x00400000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__TEE_LIMITED_STREAM_MASK                                 0x00800000L
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_CNTL__STREAM_ID_MASK                                          0xFF000000L
+//BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_STATUS
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                          0x1f
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                       0x0000000FL
+#define BIF_CFG_DEV0_RC0_SELECTIVE_IDE_STREAM_8_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                            0x80000000L
+//BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                         0x8
+#define BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                           0x00FFFF00L
+//BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG2__VALID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                      0x18
+#define BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG2__VALID_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG2__RID_BASE_MASK                                            0x00FFFF00L
+#define BIF_CFG_DEV0_RC0_IDE_8_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                        0xFF000000L
+//BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                 0x8
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                0x14
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                   0x000FFF00L
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                  0xFFFFFFFFL
+//BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC0_IDE_8_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                   0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_ID__VENDOR_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_ID__VENDOR_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_ID__DEVICE_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_ID__DEVICE_ID_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_COMMAND
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__IO_ACCESS_EN__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_ACCESS_EN__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__BUS_MASTER_EN__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PAL_SNOOP_EN__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__AD_STEPPING__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SERR_EN__SHIFT                                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__FAST_B2B_EN__SHIFT                                                       0x9
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__INT_DIS__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__IO_ACCESS_EN_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_ACCESS_EN_MASK                                                       0x0002L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__BUS_MASTER_EN_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                             0x0010L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PAL_SNOOP_EN_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__AD_STEPPING_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__SERR_EN_MASK                                                             0x0100L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__FAST_B2B_EN_MASK                                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_0_COMMAND__INT_DIS_MASK                                                             0x0400L
+//BIF_CFG_DEV0_EPF0_0_STATUS
+#define BIF_CFG_DEV0_EPF0_0_STATUS__IMMEDIATE_READINESS__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_0_STATUS__INT_STATUS__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF0_0_STATUS__CAP_LIST__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PCI_66_CAP__SHIFT                                                         0x5
+#define BIF_CFG_DEV0_EPF0_0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_STATUS__DEVSEL_TIMING__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                0xb
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                              0xd
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_0_STATUS__IMMEDIATE_READINESS_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__INT_STATUS_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__CAP_LIST_MASK                                                             0x0010L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PCI_66_CAP_MASK                                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__FAST_BACK_CAPABLE_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__DEVSEL_TIMING_MASK                                                        0x0600L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                  0x0800L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_TARGET_ABORT_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__RECEIVED_MASTER_ABORT_MASK                                                0x2000L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF0_0_STATUS__PARITY_ERROR_DETECTED_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_0_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MINOR_REV_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MINOR_REV_ID_MASK                                                    0x0FL
+#define BIF_CFG_DEV0_EPF0_0_REVISION_ID__MAJOR_REV_ID_MASK                                                    0xF0L
+//BIF_CFG_DEV0_EPF0_0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PROG_INTERFACE__PROG_INTERFACE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_0_SUB_CLASS__SUB_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_SUB_CLASS__SUB_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_0_BASE_CLASS__BASE_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_CLASS__BASE_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                  0xFFL
+//BIF_CFG_DEV0_EPF0_0_LATENCY
+#define BIF_CFG_DEV0_EPF0_0_LATENCY__LATENCY_TIMER__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_LATENCY__LATENCY_TIMER_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_0_HEADER
+#define BIF_CFG_DEV0_EPF0_0_HEADER__HEADER_TYPE__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_HEADER__DEVICE_TYPE__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_EPF0_0_HEADER__HEADER_TYPE_MASK                                                          0x7FL
+#define BIF_CFG_DEV0_EPF0_0_HEADER__DEVICE_TYPE_MASK                                                          0x80L
+//BIF_CFG_DEV0_EPF0_0_BIST
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_COMP__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_STRT__SHIFT                                                            0x6
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_CAP__SHIFT                                                             0x7
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_COMP_MASK                                                              0x0FL
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_STRT_MASK                                                              0x40L
+#define BIF_CFG_DEV0_EPF0_0_BIST__BIST_CAP_MASK                                                               0x80L
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_1__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_2__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_3__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_4__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_5__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_BASE_ADDR_6__BASE_ADDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                    0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                         0x0000000EL
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                        0x000000F0L
+#define BIF_CFG_DEV0_EPF0_0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                     0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_0_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_0_CAP_PTR__CAP_PTR__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_CAP_PTR__CAP_PTR_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF0_0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_0_MIN_GRANT__MIN_GNT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_MIN_GRANT__MIN_GNT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_0_MAX_LATENCY__MAX_LAT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MAX_LATENCY__MAX_LAT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__LENGTH__SHIFT                                                    0x10
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__CAP_ID_MASK                                                      0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                    0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_VENDOR_CAP_LIST__LENGTH_MASK                                                      0x00FF0000L
+//BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PMI_CAP
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__VERSION__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_CLOCK__SHIFT                                                         0x3
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__AUX_CURRENT__SHIFT                                                       0x6
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D1_SUPPORT__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D2_SUPPORT__SHIFT                                                        0xa
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_SUPPORT__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__VERSION_MASK                                                             0x0007L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_CLOCK_MASK                                                           0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                 0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                   0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__AUX_CURRENT_MASK                                                         0x01C0L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D1_SUPPORT_MASK                                                          0x0200L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__D2_SUPPORT_MASK                                                          0x0400L
+#define BIF_CFG_DEV0_EPF0_0_PMI_CAP__PME_SUPPORT_MASK                                                         0xF800L
+//BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                             0x16
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                0x17
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__POWER_STATE_MASK                                                 0x00000003L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_EN_MASK                                                      0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                 0x00001E00L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                  0x00006000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PME_STATUS_MASK                                                  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                               0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PMI_STATUS_CNTL__PMI_DATA_MASK                                                    0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__VERSION__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__VERSION_MASK                                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__DEVICE_TYPE_MASK                                                        0x00F0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                    0x3E00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                   0x11
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                      0x12
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                      0x1a
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                    0x1c
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                            0x1d
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                                 0x1e
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                     0x00000018L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__EXTENDED_TAG_MASK                                                     0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                           0x000001C0L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                            0x00000E00L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__RX_MPS_FIXED_MASK                                                     0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                        0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                        0x0C000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__FLR_CAPABLE_MASK                                                      0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                              0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                   0x40000000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                  0x2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                               0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                0x0002L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                0x00E0L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                 0x0200L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                           0x7000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL__INITIATE_FLR_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__CORR_ERR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                               0x1
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__USR_DETECTED__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__AUX_PWR__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                               0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__CORR_ERR_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                 0x0002L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__FATAL_ERR_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__USR_DETECTED_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__AUX_PWR_MASK                                                       0x0010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                 0x0040L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                 0xe
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                               0x11
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                0x12
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                  0x15
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x18
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1a
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                     0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                      0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                      0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                          0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                   0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                              0x00003000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                   0x0000C000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                 0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                  0x000C0000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                    0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                        0x00C00000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x03000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                     0x7
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__LTR_EN__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                      0xd
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                              0x0020L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                             0x0100L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                          0x0200L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__LTR_EN_MASK                                                         0x0400L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__OBFF_EN_MASK                                                        0x6000L
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                    0x8000L
+//BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_DEVICE_STATUS2__RESERVED_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__CAP_ID_MASK                                                         0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_MSI_CAP_LIST__NEXT_PTR_MASK                                                       0xFF00L
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                         0x9
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                          0xa
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                  0x000EL
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                   0x0070L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                      0x0100L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                           0x0200L
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                            0x0400L
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA__MSI_DATA_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK__MSI_MASK__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK__MSI_MASK_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                         0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_MASK_64__MSI_MASK_64_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING__MSI_PENDING__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING__MSI_PENDING_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_MSI_PENDING_64__MSI_PENDING_64_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__CAP_ID_MASK                                                        0x00FFL
+#define BIF_CFG_DEV0_EPF0_0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                      0xFF00L
+//BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                              0xe
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                     0xf
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                               0x07FFL
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                0x4000L
+#define BIF_CFG_DEV0_EPF0_0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                       0x8000L
+//BIF_CFG_DEV0_EPF0_0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                   0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                    0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_0_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                     0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                      0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                               0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                     0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                               0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                              0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                              0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                      0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                 0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT                 0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                            0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                           0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                           0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                        0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                   0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                   0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                              0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                             0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                             0x40000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                         0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                          0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                        0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                   0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                         0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                          0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                     0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                     0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                                0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                               0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                               0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                           0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                            0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                       0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                       0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                                 0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                                 0x40000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                 0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                             0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                           0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                 0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                           0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                          0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                          0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT             0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                        0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                       0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                       0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                   0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                   0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                    0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                               0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                             0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                 0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                   0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                            0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                             0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                            0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                            0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                   0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                    0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK               0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK               0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                          0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                         0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                         0x40000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                          0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                            0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                           0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                    0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                    0x00008000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                             0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                      0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                      0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                               0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                        0x00008000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                              0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                      0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                     0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                            0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                     0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                         0x0000001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                        0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                       0x0003E000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                              0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                       0x00F80000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG4__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG5__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG6__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG7__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG8__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG9__TLP_HDR_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG10__TLP_HDR_MASK                                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG11__TLP_HDR_MASK                                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG12__TLP_HDR_MASK                                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_HDR_LOG13__TLP_HDR_MASK                                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                 0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                     0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                    0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                           0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                              0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                           0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                           0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                        0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                            0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                             0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                     0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                      0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                    0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                               0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                             0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                       0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                       0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                    0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                        0x0080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                 0x0C00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                               0x1000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__RESET_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__DLUP_TIME__SHIFT                                                       0xc
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__RTR_VALID__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__RESET_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__DLUP_TIME_MASK                                                         0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA1__RTR_VALID_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA2__FLR_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA2__FLR_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_0_RTR_DATA2__D3HOTD0_TIME_MASK                                                      0x00FFF000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__EXTENDED_PWR_BUDGET_ENABLE__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                     0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA_SELECT__EXTENDED_PWR_BUDGET_ENABLE_MASK                      0x00010000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                 0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                             0x00000300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                           0x00001C00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                               0x00006000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                   0x00038000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                             0x001C0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP__EXTENDED_PWR_BUDGET_SUPPORT__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                        0x01L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PWR_BUDGET_CAP__EXTENDED_PWR_BUDGET_SUPPORT_MASK                             0x02L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                   0x0000001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0x000000FFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                       0x0100L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                 0x001FL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                            0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE__SHIFT                                      0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE__SHIFT                                     0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_VALUE_MASK                                        0x000003FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_S_LATENCY_SCALE_MASK                                        0x00001C00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_VALUE_MASK                                       0x03FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LTR_CAP__LTR_MAX_NS_LATENCY_SCALE_MASK                                       0x1C000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                              0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                            0x0070L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                            0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                     0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                                 0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                             0x001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                   0x0200L
+//BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__STU__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__STU_MASK                                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                     0x0400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_ENABLE__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_RESET__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_ENABLE_MASK                                               0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_CNTL__PRI_RESET_MASK                                                0x0002L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX__SHIFT                        0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__STOPPED__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED__SHIFT                          0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__RESPONSE_FAILURE_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__UNEXPECTED_PAGE_REQ_GRP_INDEX_MASK                          0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__STOPPED_MASK                                                0x0100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PAGE_REQ_STATUS__PRG_RESPONSE_PASID_REQUIRED_MASK                            0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_CAPACITY__OUTSTAND_PAGE_REQ_CAPACITY_MASK                  0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_OUTSTAND_PAGE_REQ_ALLOC__OUTSTAND_PAGE_REQ_ALLOC_MASK                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__TRANSLATED_REQ_WITH_PASID_SUPPORTED__SHIFT                        0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                            0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                               0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__TRANSLATED_REQ_WITH_PASID_SUPPORTED_MASK                          0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                              0x1F00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                          0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__TRANSLATED_REQ_WITH_PASID_ENABLE__SHIFT                          0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                            0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PASID_CNTL__TRANSLATED_REQ_WITH_PASID_ENABLE_MASK                            0x0008L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED__SHIFT                          0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_14_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM__SHIFT                            0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED_MASK                            0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_14_BIT_TAG_REQUESTER_SUPPORTED_MASK                      0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM_MASK                              0xFFE00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE__SHIFT                                           0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_14_BIT_TAG_REQUESTER_ENABLE__SHIFT                   0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE_MASK                                          0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE_MASK                                0x0002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE_MASK                           0x0004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE_MASK                                             0x0008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE_MASK                    0x0020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_CONTROL__SRIOV_VF_14_BIT_TAG_REQUESTER_ENABLE_MASK                     0x0040L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS_MASK                                 0x0001L
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS_MASK                                    0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS_MASK                                        0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS_MASK                                            0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK_MASK                                0xFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET_MASK                            0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE_MASK                                        0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID_MASK                                  0xFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE_MASK                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE_MASK                          0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR_MASK                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR__SHIFT   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR_MASK     0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_MASK  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED_MASK                               0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                 0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM_MASK                                   0x000000E0L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_MASK                                        0x00003F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                        0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DOE_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP__INTR_SUPPORT__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP__DOE_INTR_MSG_NUM__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP__DOE_ATT_SUPPORT__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP__DOE_ASYNC_SUPPORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP__INTR_SUPPORT_MASK                                                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP__DOE_INTR_MSG_NUM_MASK                                               0x00000FFEL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP__DOE_ATT_SUPPORT_MASK                                                0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CAP__DOE_ASYNC_SUPPORT_MASK                                              0x00002000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_ABORT__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_INTR_EN__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_ATT_NOT_NEED__SHIFT                                            0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_ASYNC_MSG_EN__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_GO__SHIFT                                                      0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_ABORT_MASK                                                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_INTR_EN_MASK                                                   0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_ATT_NOT_NEED_MASK                                              0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_ASYNC_MSG_EN_MASK                                              0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_CNTL__DOE_GO_MASK                                                        0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_BUSY__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_INTR_STS__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_ERR__SHIFT                                                      0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_ASYNC_MSG_STS__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_AT_ATTENTION__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_OBJ_RDY__SHIFT                                                  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_BUSY_MASK                                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_INTR_STS_MASK                                                   0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_ERR_MASK                                                        0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_ASYNC_MSG_STS_MASK                                              0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_AT_ATTENTION_MASK                                               0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_STS__DOE_OBJ_RDY_MASK                                                    0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DOE_WMBOX
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_WMBOX__DOE_WMBOX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_WMBOX__DOE_WMBOX_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_DOE_RMBOX
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_RMBOX__DOE_RMBOX__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DOE_RMBOX__DOE_RMBOX_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID__SHIFT                           0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER__SHIFT                          0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR__SHIFT                         0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_ID_MASK                             0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__CAP_VER_MASK                            0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST_GPUIOV__NEXT_PTR_MASK                           0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV__SHIFT                                  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH__SHIFT                               0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_ID_MASK                                     0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_REV_MASK                                    0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV__VSEC_LENGTH_MASK                                 0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW__VF_EN__SHIFT                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW__VF_NUM__SHIFT                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW__VF_EN_MASK                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SRIOV_SHADOW__VF_NUM_MASK                         0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_TRN_ACK_INTR_EN_MASK    0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_ENABLE__HVVM_MAILBOX_RCV_VALID_INTR_EN_MASK  0x02000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_TRN_ACK_INTR_STATUS_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_INTR_STATUS__HVVM_MAILBOX_RCV_VALID_INTR_STATUS_MASK  0x02000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__FUNCTION_ID__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__SOFT_PF_FLR_MASK                   0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_RESET_CONTROL__FUNCTION_ID_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID__SHIFT               0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA__SHIFT                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK__SHIFT                 0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__VF_INDEX_MASK                      0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_DATA_MASK                  0x00000F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__TRN_MSG_VALID_MASK                 0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_DATA_MASK                  0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW0__RCV_MSG_ACK_MASK                   0x01000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID__SHIFT               0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK__SHIFT                 0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID__SHIFT               0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID__SHIFT               0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK__SHIFT                 0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID__SHIFT               0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID__SHIFT               0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK__SHIFT                 0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID__SHIFT               0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK__SHIFT                 0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID__SHIFT               0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK__SHIFT                 0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID__SHIFT               0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK__SHIFT                 0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID__SHIFT               0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK__SHIFT                 0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID__SHIFT               0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK__SHIFT                0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID__SHIFT              0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK__SHIFT                0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID__SHIFT              0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK__SHIFT                0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID__SHIFT              0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK__SHIFT                0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID__SHIFT              0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK__SHIFT                0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID__SHIFT              0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK__SHIFT                0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID__SHIFT              0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_TRN_ACK_MASK                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF0_RCV_VALID_MASK                 0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_TRN_ACK_MASK                   0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF1_RCV_VALID_MASK                 0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_TRN_ACK_MASK                   0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF2_RCV_VALID_MASK                 0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_TRN_ACK_MASK                   0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF3_RCV_VALID_MASK                 0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_TRN_ACK_MASK                   0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF4_RCV_VALID_MASK                 0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_TRN_ACK_MASK                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF5_RCV_VALID_MASK                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_TRN_ACK_MASK                   0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF6_RCV_VALID_MASK                 0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_TRN_ACK_MASK                   0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF7_RCV_VALID_MASK                 0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_TRN_ACK_MASK                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF8_RCV_VALID_MASK                 0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_TRN_ACK_MASK                   0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF9_RCV_VALID_MASK                 0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_TRN_ACK_MASK                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF10_RCV_VALID_MASK                0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_TRN_ACK_MASK                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF11_RCV_VALID_MASK                0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_TRN_ACK_MASK                  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF12_RCV_VALID_MASK                0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_TRN_ACK_MASK                  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF13_RCV_VALID_MASK                0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_TRN_ACK_MASK                  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF14_RCV_VALID_MASK                0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_TRN_ACK_MASK                  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW1__VF15_RCV_VALID_MASK                0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID__SHIFT              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK__SHIFT                0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID__SHIFT              0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK__SHIFT                0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID__SHIFT              0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK__SHIFT                0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID__SHIFT              0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID__SHIFT              0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK__SHIFT                0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID__SHIFT              0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK__SHIFT                0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID__SHIFT              0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK__SHIFT                0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID__SHIFT              0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK__SHIFT                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID__SHIFT              0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK__SHIFT                0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID__SHIFT              0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK__SHIFT                0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID__SHIFT              0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK__SHIFT                0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID__SHIFT              0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK__SHIFT                0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID__SHIFT              0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK__SHIFT                0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID__SHIFT              0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK__SHIFT                0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID__SHIFT              0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK__SHIFT                  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID__SHIFT                0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_TRN_ACK_MASK                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF16_RCV_VALID_MASK                0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_TRN_ACK_MASK                  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF17_RCV_VALID_MASK                0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_TRN_ACK_MASK                  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF18_RCV_VALID_MASK                0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_TRN_ACK_MASK                  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF19_RCV_VALID_MASK                0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_TRN_ACK_MASK                  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF20_RCV_VALID_MASK                0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_TRN_ACK_MASK                  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF21_RCV_VALID_MASK                0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_TRN_ACK_MASK                  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF22_RCV_VALID_MASK                0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_TRN_ACK_MASK                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF23_RCV_VALID_MASK                0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_TRN_ACK_MASK                  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF24_RCV_VALID_MASK                0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_TRN_ACK_MASK                  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF25_RCV_VALID_MASK                0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_TRN_ACK_MASK                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF26_RCV_VALID_MASK                0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_TRN_ACK_MASK                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF27_RCV_VALID_MASK                0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_TRN_ACK_MASK                  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF28_RCV_VALID_MASK                0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_TRN_ACK_MASK                  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF29_RCV_VALID_MASK                0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_TRN_ACK_MASK                  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__VF30_RCV_VALID_MASK                0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_TRN_ACK_MASK                    0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_HVVM_MBOX_DW2__PF_RCV_VALID_MASK                  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_AVAILABLE__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_CONSUMED__SHIFT                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_AVAILABLE_MASK                 0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_TOTAL_FB__TOTAL_FB_CONSUMED_MASK                  0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__LFB_REGION__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__MAX_REGION__SHIFT                         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__LFB_REGION_MASK                           0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_REGION__MAX_REGION_MASK                           0x000000F0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_VF__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_PF__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_VF_MASK  0x7FFFFFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_P2P_OVER_XGMI_ENABLE__P2P_OVER_XGMI_ENABLE_PF_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH0_OFFSET__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH1_OFFSET__SHIFT                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH2_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH3_OFFSET__SHIFT                      0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH0_OFFSET_MASK                        0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH1_OFFSET_MASK                        0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH2_OFFSET_MASK                        0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS0__SCH3_OFFSET_MASK                        0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH4_OFFSET__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH5_OFFSET__SHIFT                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH6_OFFSET__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH7_OFFSET__SHIFT                      0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH4_OFFSET_MASK                        0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH5_OFFSET_MASK                        0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH6_OFFSET_MASK                        0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS1__SCH7_OFFSET_MASK                        0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH8_OFFSET__SHIFT                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH9_OFFSET__SHIFT                      0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH10_OFFSET__SHIFT                     0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH11_OFFSET__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH8_OFFSET_MASK                        0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH9_OFFSET_MASK                        0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH10_OFFSET_MASK                       0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS2__SCH11_OFFSET_MASK                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH12_OFFSET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH13_OFFSET__SHIFT                     0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH14_OFFSET__SHIFT                     0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH15_OFFSET__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH12_OFFSET_MASK                       0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH13_OFFSET_MASK                       0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH14_OFFSET_MASK                       0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS3__SCH15_OFFSET_MASK                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH16_OFFSET__SHIFT                     0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH17_OFFSET__SHIFT                     0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH18_OFFSET__SHIFT                     0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH19_OFFSET__SHIFT                     0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH16_OFFSET_MASK                       0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH17_OFFSET_MASK                       0x0000FF00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH18_OFFSET_MASK                       0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_OFFSETS4__SCH19_OFFSET_MASK                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH0_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH1_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH2_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH3_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH4_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH5_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH6_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH7_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH8_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW0__DW0__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW0__DW0_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW1__DW1__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW1__DW1_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW2__DW2__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW2__DW2_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW3__DW3__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW3__DW3_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW4__DW4__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW4__DW4_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW5__DW5__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW5__DW5_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW6__DW6__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW6__DW6_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW7__DW7__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW7__DW7_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW8__DW8__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH9_DW8__DW8_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH10_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH11_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH12_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH13_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH14_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH15_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH16_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH17_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH18_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW0__DW0__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW0__DW0_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW1__DW1__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW1__DW1_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW2__DW2__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW2__DW2_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW3__DW3__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW3__DW3_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW4__DW4__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW4__DW4_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW5__DW5__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW5__DW5_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW6__DW6__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW6__DW6_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW7__DW7__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW7__DW7_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW8__DW8__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_SCH19_DW8__DW8_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_CMD_COMPLETE_INTR_EN__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_HANG_NEED_FLR_INTR_EN__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_CMD_COMPLETE_INTR_EN__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_HANG_NEED_FLR_INTR_EN__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_CMD_COMPLETE_INTR_EN__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_HANG_NEED_FLR_INTR_EN__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_CMD_COMPLETE_INTR_EN__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_HANG_NEED_FLR_INTR_EN__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_CMD_COMPLETE_INTR_EN__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_HANG_NEED_FLR_INTR_EN__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_CMD_COMPLETE_INTR_EN__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_HANG_NEED_FLR_INTR_EN__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_CMD_COMPLETE_INTR_EN__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_HANG_NEED_FLR_INTR_EN__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_CMD_COMPLETE_INTR_EN__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_HANG_NEED_FLR_INTR_EN__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_CMD_COMPLETE_INTR_EN_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_HANG_NEED_FLR_INTR_EN_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_0_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_CMD_COMPLETE_INTR_EN_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_HANG_NEED_FLR_INTR_EN_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_1_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_CMD_COMPLETE_INTR_EN_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_HANG_NEED_FLR_INTR_EN_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_2_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_CMD_COMPLETE_INTR_EN_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_HANG_NEED_FLR_INTR_EN_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_3_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_CMD_COMPLETE_INTR_EN_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_HANG_NEED_FLR_INTR_EN_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_4_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_CMD_COMPLETE_INTR_EN_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_HANG_NEED_FLR_INTR_EN_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_5_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_CMD_COMPLETE_INTR_EN_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_HANG_SELF_RECOVERED_INTR_EN_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_HANG_NEED_FLR_INTR_EN_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_6_VM_BUSY_TRANSITION_INTR_EN_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_CMD_COMPLETE_INTR_EN_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_HANG_SELF_RECOVERED_INTR_EN_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_HANG_NEED_FLR_INTR_EN_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_ENABLE__ENG_7_VM_BUSY_TRANSITION_INTR_EN_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_CMD_COMPLETE_INTR_EN__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_HANG_NEED_FLR_INTR_EN__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_CMD_COMPLETE_INTR_EN__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_HANG_NEED_FLR_INTR_EN__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_CMD_COMPLETE_INTR_EN__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_HANG_NEED_FLR_INTR_EN__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_CMD_COMPLETE_INTR_EN__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_HANG_NEED_FLR_INTR_EN__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_CMD_COMPLETE_INTR_EN__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_HANG_NEED_FLR_INTR_EN__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_CMD_COMPLETE_INTR_EN__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_HANG_NEED_FLR_INTR_EN__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_CMD_COMPLETE_INTR_EN__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_HANG_NEED_FLR_INTR_EN__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_CMD_COMPLETE_INTR_EN__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_HANG_NEED_FLR_INTR_EN__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_CMD_COMPLETE_INTR_EN_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_HANG_NEED_FLR_INTR_EN_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_8_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_CMD_COMPLETE_INTR_EN_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_HANG_NEED_FLR_INTR_EN_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_9_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_CMD_COMPLETE_INTR_EN_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_HANG_NEED_FLR_INTR_EN_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_10_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_CMD_COMPLETE_INTR_EN_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_HANG_NEED_FLR_INTR_EN_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_11_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_CMD_COMPLETE_INTR_EN_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_HANG_NEED_FLR_INTR_EN_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_12_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_CMD_COMPLETE_INTR_EN_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_HANG_NEED_FLR_INTR_EN_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_13_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_CMD_COMPLETE_INTR_EN_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_HANG_SELF_RECOVERED_INTR_EN_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_HANG_NEED_FLR_INTR_EN_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_14_VM_BUSY_TRANSITION_INTR_EN_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_CMD_COMPLETE_INTR_EN_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_HANG_SELF_RECOVERED_INTR_EN_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_HANG_NEED_FLR_INTR_EN_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_ENABLE__ENG_15_VM_BUSY_TRANSITION_INTR_EN_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_CMD_COMPLETE_INTR_EN__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_HANG_NEED_FLR_INTR_EN__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_CMD_COMPLETE_INTR_EN__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_HANG_NEED_FLR_INTR_EN__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_CMD_COMPLETE_INTR_EN__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_HANG_NEED_FLR_INTR_EN__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_CMD_COMPLETE_INTR_EN__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_HANG_NEED_FLR_INTR_EN__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_CMD_COMPLETE_INTR_EN__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_HANG_NEED_FLR_INTR_EN__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_CMD_COMPLETE_INTR_EN__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_HANG_NEED_FLR_INTR_EN__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_CMD_COMPLETE_INTR_EN__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_HANG_NEED_FLR_INTR_EN__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_CMD_COMPLETE_INTR_EN__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_HANG_NEED_FLR_INTR_EN__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_CMD_COMPLETE_INTR_EN_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_HANG_NEED_FLR_INTR_EN_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_16_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_CMD_COMPLETE_INTR_EN_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_HANG_NEED_FLR_INTR_EN_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_17_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_CMD_COMPLETE_INTR_EN_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_HANG_NEED_FLR_INTR_EN_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_18_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_CMD_COMPLETE_INTR_EN_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_HANG_NEED_FLR_INTR_EN_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_19_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_CMD_COMPLETE_INTR_EN_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_HANG_NEED_FLR_INTR_EN_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_20_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_CMD_COMPLETE_INTR_EN_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_HANG_NEED_FLR_INTR_EN_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_21_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_CMD_COMPLETE_INTR_EN_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_HANG_SELF_RECOVERED_INTR_EN_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_HANG_NEED_FLR_INTR_EN_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_22_VM_BUSY_TRANSITION_INTR_EN_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_CMD_COMPLETE_INTR_EN_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_HANG_SELF_RECOVERED_INTR_EN_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_HANG_NEED_FLR_INTR_EN_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_ENABLE__ENG_23_VM_BUSY_TRANSITION_INTR_EN_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_CMD_COMPLETE_INTR_EN__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_HANG_NEED_FLR_INTR_EN__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_CMD_COMPLETE_INTR_EN__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_HANG_NEED_FLR_INTR_EN__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_CMD_COMPLETE_INTR_EN__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_HANG_NEED_FLR_INTR_EN__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_CMD_COMPLETE_INTR_EN__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_HANG_NEED_FLR_INTR_EN__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_CMD_COMPLETE_INTR_EN__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_HANG_NEED_FLR_INTR_EN__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_CMD_COMPLETE_INTR_EN__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_HANG_NEED_FLR_INTR_EN__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_CMD_COMPLETE_INTR_EN__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_HANG_NEED_FLR_INTR_EN__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_CMD_COMPLETE_INTR_EN__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_HANG_SELF_RECOVERED_INTR_EN__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_HANG_NEED_FLR_INTR_EN__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_VM_BUSY_TRANSITION_INTR_EN__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_CMD_COMPLETE_INTR_EN_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_HANG_NEED_FLR_INTR_EN_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_24_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_CMD_COMPLETE_INTR_EN_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_HANG_NEED_FLR_INTR_EN_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_25_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_CMD_COMPLETE_INTR_EN_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_HANG_NEED_FLR_INTR_EN_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_26_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_CMD_COMPLETE_INTR_EN_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_HANG_NEED_FLR_INTR_EN_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_27_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_CMD_COMPLETE_INTR_EN_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_HANG_NEED_FLR_INTR_EN_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_28_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_CMD_COMPLETE_INTR_EN_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_HANG_SELF_RECOVERED_INTR_EN_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_HANG_NEED_FLR_INTR_EN_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_29_VM_BUSY_TRANSITION_INTR_EN_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_CMD_COMPLETE_INTR_EN_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_HANG_SELF_RECOVERED_INTR_EN_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_HANG_NEED_FLR_INTR_EN_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_30_VM_BUSY_TRANSITION_INTR_EN_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_CMD_COMPLETE_INTR_EN_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_HANG_SELF_RECOVERED_INTR_EN_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_HANG_NEED_FLR_INTR_EN_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_ENABLE__ENG_31_VM_BUSY_TRANSITION_INTR_EN_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_CMD_COMPLETE_INTR_STATUS__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_CMD_COMPLETE_INTR_STATUS__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_CMD_COMPLETE_INTR_STATUS__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_CMD_COMPLETE_INTR_STATUS__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_CMD_COMPLETE_INTR_STATUS__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_CMD_COMPLETE_INTR_STATUS__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_CMD_COMPLETE_INTR_STATUS__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_CMD_COMPLETE_INTR_STATUS__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_CMD_COMPLETE_INTR_STATUS_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_0_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_CMD_COMPLETE_INTR_STATUS_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_1_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_CMD_COMPLETE_INTR_STATUS_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_2_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_CMD_COMPLETE_INTR_STATUS_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_HANG_NEED_FLR_INTR_STATUS_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_3_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_CMD_COMPLETE_INTR_STATUS_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_HANG_NEED_FLR_INTR_STATUS_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_4_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_CMD_COMPLETE_INTR_STATUS_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_HANG_NEED_FLR_INTR_STATUS_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_5_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_CMD_COMPLETE_INTR_STATUS_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_HANG_NEED_FLR_INTR_STATUS_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_6_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_CMD_COMPLETE_INTR_STATUS_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_HANG_NEED_FLR_INTR_STATUS_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_0_7_INTR_STATUS__ENG_7_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_CMD_COMPLETE_INTR_STATUS__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_CMD_COMPLETE_INTR_STATUS__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_CMD_COMPLETE_INTR_STATUS__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_CMD_COMPLETE_INTR_STATUS__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_CMD_COMPLETE_INTR_STATUS__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_CMD_COMPLETE_INTR_STATUS__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_CMD_COMPLETE_INTR_STATUS__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_CMD_COMPLETE_INTR_STATUS__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_CMD_COMPLETE_INTR_STATUS_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_8_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_CMD_COMPLETE_INTR_STATUS_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_9_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_CMD_COMPLETE_INTR_STATUS_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_10_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_CMD_COMPLETE_INTR_STATUS_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_HANG_NEED_FLR_INTR_STATUS_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_11_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_CMD_COMPLETE_INTR_STATUS_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_HANG_NEED_FLR_INTR_STATUS_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_12_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_CMD_COMPLETE_INTR_STATUS_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_HANG_NEED_FLR_INTR_STATUS_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_13_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_CMD_COMPLETE_INTR_STATUS_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_HANG_NEED_FLR_INTR_STATUS_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_14_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_CMD_COMPLETE_INTR_STATUS_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_HANG_NEED_FLR_INTR_STATUS_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_8_15_INTR_STATUS__ENG_15_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_CMD_COMPLETE_INTR_STATUS__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_CMD_COMPLETE_INTR_STATUS__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_CMD_COMPLETE_INTR_STATUS__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_CMD_COMPLETE_INTR_STATUS__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_CMD_COMPLETE_INTR_STATUS__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_CMD_COMPLETE_INTR_STATUS__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_CMD_COMPLETE_INTR_STATUS__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_CMD_COMPLETE_INTR_STATUS__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_CMD_COMPLETE_INTR_STATUS_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_16_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_CMD_COMPLETE_INTR_STATUS_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_17_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_CMD_COMPLETE_INTR_STATUS_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_18_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_CMD_COMPLETE_INTR_STATUS_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_HANG_NEED_FLR_INTR_STATUS_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_19_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_CMD_COMPLETE_INTR_STATUS_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_HANG_NEED_FLR_INTR_STATUS_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_20_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_CMD_COMPLETE_INTR_STATUS_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_HANG_NEED_FLR_INTR_STATUS_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_21_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_CMD_COMPLETE_INTR_STATUS_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_HANG_NEED_FLR_INTR_STATUS_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_22_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_CMD_COMPLETE_INTR_STATUS_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_HANG_NEED_FLR_INTR_STATUS_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_16_23_INTR_STATUS__ENG_23_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_CMD_COMPLETE_INTR_STATUS__SHIFT  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_CMD_COMPLETE_INTR_STATUS__SHIFT  0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x5
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x6
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x7
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_CMD_COMPLETE_INTR_STATUS__SHIFT  0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xb
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_CMD_COMPLETE_INTR_STATUS__SHIFT  0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0xd
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_HANG_NEED_FLR_INTR_STATUS__SHIFT  0xe
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_CMD_COMPLETE_INTR_STATUS__SHIFT  0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x12
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x13
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_CMD_COMPLETE_INTR_STATUS__SHIFT  0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x15
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x16
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x17
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_CMD_COMPLETE_INTR_STATUS__SHIFT  0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x19
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1a
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1b
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_CMD_COMPLETE_INTR_STATUS__SHIFT  0x1c
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_HANG_SELF_RECOVERED_INTR_STATUS__SHIFT  0x1d
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_HANG_NEED_FLR_INTR_STATUS__SHIFT  0x1e
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_VM_BUSY_TRANSITION_INTR_STATUS__SHIFT  0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_CMD_COMPLETE_INTR_STATUS_MASK  0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_24_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_CMD_COMPLETE_INTR_STATUS_MASK  0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_25_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_CMD_COMPLETE_INTR_STATUS_MASK  0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_HANG_NEED_FLR_INTR_STATUS_MASK  0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_26_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00000800L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_CMD_COMPLETE_INTR_STATUS_MASK  0x00001000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00002000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_HANG_NEED_FLR_INTR_STATUS_MASK  0x00004000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_27_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_CMD_COMPLETE_INTR_STATUS_MASK  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00020000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_HANG_NEED_FLR_INTR_STATUS_MASK  0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_28_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_CMD_COMPLETE_INTR_STATUS_MASK  0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_HANG_NEED_FLR_INTR_STATUS_MASK  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_29_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_CMD_COMPLETE_INTR_STATUS_MASK  0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x02000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_HANG_NEED_FLR_INTR_STATUS_MASK  0x04000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_30_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x08000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_CMD_COMPLETE_INTR_STATUS_MASK  0x10000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_HANG_SELF_RECOVERED_INTR_STATUS_MASK  0x20000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_HANG_NEED_FLR_INTR_STATUS_MASK  0x40000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR_GPUIOV_ENG_24_31_INTR_STATUS__ENG_31_VM_BUSY_TRANSITION_INTR_STATUS_MASK  0x80000000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_bifcfgpciedecp
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_SPEED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_WIDTH__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PM_SUPPORT__SHIFT                                                       0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                         0x15
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                      0x16
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PORT_NUMBER__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_SPEED_MASK                                                         0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_WIDTH_MASK                                                         0x000003F0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PM_SUPPORT_MASK                                                         0x00000C00L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                   0x00007000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                    0x00038000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP__PORT_NUMBER_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PM_CONTROL__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_DIS__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                   0x7
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                     0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                       0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                           0xe
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PM_CONTROL_MASK                                                        0x0003L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_DIS_MASK                                                          0x0010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__RETRAIN_LINK_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__EXTENDED_SYNC_MASK                                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                         0x0100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                         0x0400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                         0x0800L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__SRIS_CLOCKING_MASK                                                     0x1000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                 0x2000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                             0xC000L
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_TRAINING__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__DL_ACTIVE__SHIFT                                                     0xd
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                           0x03F0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_TRAINING_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__DL_ACTIVE_MASK                                                       0x2000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                       0x4000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                       0x8000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                       0x17
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                       0x18
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                              0x000000FEL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                          0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                          0x007F0000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                         0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP2__DRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__XMIT_MARGIN_MASK                                                      0x0380L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                            0xF000L
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                              0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                              0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                              0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                           0x0001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                0x0004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                0x0008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                0x0010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                  0x0020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                           0x0300L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                  0x7000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                           0x8000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                    0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                   0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                  0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                              0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                        0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                     0x0000FE00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                               0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT          0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT            0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                 0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK            0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                   0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK              0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT         0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT           0xc
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                0x000FL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK           0x0070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                  0x0F00L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK             0x7000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT__RESERVED__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_16GT__RESERVED_MASK                                                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT__RESERVED__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_16GT__RESERVED_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                         0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                         0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                           0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                 0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                           0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                           0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                             0x00000010L
+//BIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT       0x0
+#define BIF_CFG_DEV0_EPF0_0_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK         0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV0_EPF0_0_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT         0x0
+#define BIF_CFG_DEV0_EPF0_0_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK           0x0000FFFFL
+//BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                           0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                           0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                           0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                        0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                             0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                             0x00000400L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                              0x0000F800L
+//BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                             0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                   0x00000700L
+//BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                               0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                         0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                         0x3
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                           0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                     0x5
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                     0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                              0x9
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                    0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                 0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                           0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                           0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                             0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                       0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                       0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                      0x00000400L
+//BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                       0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                           0x0000FFF8L
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                         0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT          0x18
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                           0x00FFFFFFL
+#define BIF_CFG_DEV0_EPF0_0_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK            0x03000000L
+//BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                    0x10
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                    0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                        0x0000FFF8L
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                      0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT       0x18
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                        0x00FFFFFFL
+#define BIF_CFG_DEV0_EPF0_0_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK         0x03000000L
+//BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                   0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                     0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                     0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                 0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                 0x4
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                   0x0FL
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                   0xF0L
+//BIF_CFG_DEV0_EPF0_0_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                             0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_AP_CAP
+#define BIF_CFG_DEV0_EPF0_0_AP_CAP__COUNT__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_AP_CAP__COUNT_MASK                                                                0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_AP_CAP__SEL_EN_SUPPORTED_MASK                                                     0x00000100L
+//BIF_CFG_DEV0_EPF0_0_AP_CNTL
+#define BIF_CFG_DEV0_EPF0_0_AP_CNTL__INX_SEL__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF0_0_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF0_0_AP_CNTL__INX_SEL_MASK                                                             0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                      0x00000100L
+//BIF_CFG_DEV0_EPF0_0_AP_DATA1
+#define BIF_CFG_DEV0_EPF0_0_AP_DATA1__USAGE_INFOR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_AP_DATA1__DETAILS__SHIFT                                                          0x5
+#define BIF_CFG_DEV0_EPF0_0_AP_DATA1__VENDORID__SHIFT                                                         0x10
+#define BIF_CFG_DEV0_EPF0_0_AP_DATA1__USAGE_INFOR_MASK                                                        0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_AP_DATA1__DETAILS_MASK                                                            0x0000FFE0L
+#define BIF_CFG_DEV0_EPF0_0_AP_DATA1__VENDORID_MASK                                                           0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_0_AP_DATA2
+#define BIF_CFG_DEV0_EPF0_0_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                 0x00FFFFFFL
+//BIF_CFG_DEV0_EPF0_0_AP_SEL_EN_MASK
+#define BIF_CFG_DEV0_EPF0_0_AP_SEL_EN_MASK__PCIE__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_AP_SEL_EN_MASK__OTHERS__SHIFT                                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_AP_SEL_EN_MASK__PCIE_MASK                                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_AP_SEL_EN_MASK__OTHERS_MASK                                                       0xFFFFFFFEL
+//BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                  0x0001L
+//BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                            0x1
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_0_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                              0x0002L
+//BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                             0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                          0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                           0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                               0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                               0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                            0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                    0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                    0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                 0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                  0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                      0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                      0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                   0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                       0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                           0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                           0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                         0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                             0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                             0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                          0xFF00L
+//BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT              0x0
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                  0x3
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                  0x6
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT               0x8
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                0x0007L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                    0x0038L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                    0x0040L
+#define BIF_CFG_DEV0_EPF0_0_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                 0xFF00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                              0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                             0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                 0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                           0x4
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                           0xa
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                          0x00000007L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                             0x00000070L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                             0x00000C00L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                 0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                            0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                   0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                             0x000EL
+//BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                     0x0001L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                  0xf
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                               0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                          0x000000FFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                    0x00008000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                        0x003F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                 0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                    0x11
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                          0x1f
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                        0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                      0x000000FEL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                  0x00010000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                      0x000E0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                0x07000000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                            0x80000000L
+//BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                           0x1
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                              0x0001L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                             0x0002L
+//BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                 0x1f
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                               0x007FFFFFL
+#define BIF_CFG_DEV0_EPF0_0_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_0_IDE_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV0_EPF0_0_IDE_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_0_IDE_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV0_EPF0_0_IDE_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_CAP
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_LINK_IDE_STREAM_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_STREAM_SUPPORTED__SHIFT                            0x1
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED__SHIFT                          0x3
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_AGGREGATION_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_PCRC_SUPPORTED__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_IDE_KM_PROTOCOL_SUPPORTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED__SHIFT                       0x7
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_SUPPORTED_ALGORITHMS__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE__SHIFT                         0xd
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED__SHIFT                    0x10
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_TEE_LIMITED_SUPPORTED__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_XT_SUPPORTED__SHIFT                                              0x19
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_LINK_IDE_STREAM_SUPPORTED_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_STREAM_SUPPORTED_MASK                              0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_MASK                           0x00000004L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_MASK                            0x00000008L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_AGGREGATION_SUPPORTED_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_PCRC_SUPPORTED_MASK                                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_IDE_KM_PROTOCOL_SUPPORTED_MASK                                   0x00000040L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_MASK                         0x00000080L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_SUPPORTED_ALGORITHMS_MASK                                        0x00001F00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_MASK                           0x0000E000L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_MASK                      0x00FF0000L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_TEE_LIMITED_SUPPORTED_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_0_IDE_CAP__IDE_CAP_XT_SUPPORTED_MASK                                                0x02000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_CNTL
+#define BIF_CFG_DEV0_EPF0_0_IDE_CNTL__IDE_CNTL_FLOW_THROUGH_IDE_STREAM_ENABLED__SHIFT                         0x2
+#define BIF_CFG_DEV0_EPF0_0_IDE_CNTL__IDE_CNTL_FLOW_THROUGH_IDE_STREAM_ENABLED_MASK                           0x00000004L
+//BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__LINK_IDE_STREAM_ENABLE__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__XT_ENABLE__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                             0x4
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                            0x6
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__PCRC_ENABLE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                           0xa
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM__SHIFT                                 0xe
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__TC__SHIFT                                                 0x13
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__STREAM_ID__SHIFT                                          0x18
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__LINK_IDE_STREAM_ENABLE_MASK                               0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__XT_ENABLE_MASK                                            0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR_MASK                              0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR_MASK                               0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL_MASK                              0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__PCRC_ENABLE_MASK                                          0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                             0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM_MASK                                   0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__TC_MASK                                                   0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_CNTL__STREAM_ID_MASK                                            0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_STATUS
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_STATUS__LINK_IDE_STREAM_STATE__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                            0x1f
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_STATUS__LINK_IDE_STREAM_STATE_MASK                              0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_LINK_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                              0x80000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CAP
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                    0x0000000FL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__XT_ENABLE__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                        0x4
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__PCRC_ENABLE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                  0x9
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                      0xa
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM__SHIFT                            0xe
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TC__SHIFT                                            0x13
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__DEFAULT_STREAM__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TEE_LIMITED_STREAM__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__STREAM_ID__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__XT_ENABLE_MASK                                       0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR_MASK                         0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR_MASK                          0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__PCRC_ENABLE_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                        0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM_MASK                              0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TC_MASK                                              0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__DEFAULT_STREAM_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__TEE_LIMITED_STREAM_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_CNTL__STREAM_ID_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_STATUS
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                    0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                        0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG2__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG2__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG2__RID_BASE_MASK                                         0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_0_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CAP
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                    0x0000000FL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__XT_ENABLE__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                        0x4
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__PCRC_ENABLE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                  0x9
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                      0xa
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTED_ALGORITHM__SHIFT                            0xe
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TC__SHIFT                                            0x13
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__DEFAULT_STREAM__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TEE_LIMITED_STREAM__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__STREAM_ID__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__XT_ENABLE_MASK                                       0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_NPR_MASK                         0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_PR_MASK                          0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_CPL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__PCRC_ENABLE_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                        0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__SELECTED_ALGORITHM_MASK                              0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TC_MASK                                              0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__DEFAULT_STREAM_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__TEE_LIMITED_STREAM_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_CNTL__STREAM_ID_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_STATUS
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                    0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_1_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                        0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG2__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG2__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG2__RID_BASE_MASK                                         0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_1_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CAP
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                    0x0000000FL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__XT_ENABLE__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                        0x4
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__PCRC_ENABLE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                  0x9
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                      0xa
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTED_ALGORITHM__SHIFT                            0xe
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TC__SHIFT                                            0x13
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__DEFAULT_STREAM__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TEE_LIMITED_STREAM__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__STREAM_ID__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__XT_ENABLE_MASK                                       0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_NPR_MASK                         0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_PR_MASK                          0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_CPL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__PCRC_ENABLE_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                        0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__SELECTED_ALGORITHM_MASK                              0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TC_MASK                                              0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__DEFAULT_STREAM_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__TEE_LIMITED_STREAM_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_CNTL__STREAM_ID_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_STATUS
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                    0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_2_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                        0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG2__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG2__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG2__RID_BASE_MASK                                         0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_2_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CAP
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                    0x0000000FL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__XT_ENABLE__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                        0x4
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__PCRC_ENABLE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                  0x9
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                      0xa
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTED_ALGORITHM__SHIFT                            0xe
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TC__SHIFT                                            0x13
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__DEFAULT_STREAM__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TEE_LIMITED_STREAM__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__STREAM_ID__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__XT_ENABLE_MASK                                       0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_NPR_MASK                         0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_PR_MASK                          0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_CPL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__PCRC_ENABLE_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                        0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__SELECTED_ALGORITHM_MASK                              0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TC_MASK                                              0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__DEFAULT_STREAM_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__TEE_LIMITED_STREAM_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_CNTL__STREAM_ID_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_STATUS
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                    0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_3_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                        0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG2__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG2__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG2__RID_BASE_MASK                                         0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_3_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CAP
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                    0x0000000FL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__XT_ENABLE__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                        0x4
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__PCRC_ENABLE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                  0x9
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                      0xa
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTED_ALGORITHM__SHIFT                            0xe
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TC__SHIFT                                            0x13
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__DEFAULT_STREAM__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TEE_LIMITED_STREAM__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__STREAM_ID__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__XT_ENABLE_MASK                                       0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_NPR_MASK                         0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_PR_MASK                          0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_CPL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__PCRC_ENABLE_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                        0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__SELECTED_ALGORITHM_MASK                              0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TC_MASK                                              0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__DEFAULT_STREAM_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__TEE_LIMITED_STREAM_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_CNTL__STREAM_ID_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_STATUS
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                    0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_4_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                        0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG2__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG2__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG2__RID_BASE_MASK                                         0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_4_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CAP
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                    0x0000000FL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__XT_ENABLE__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                        0x4
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__PCRC_ENABLE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                  0x9
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                      0xa
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTED_ALGORITHM__SHIFT                            0xe
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TC__SHIFT                                            0x13
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__DEFAULT_STREAM__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TEE_LIMITED_STREAM__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__STREAM_ID__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__XT_ENABLE_MASK                                       0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_NPR_MASK                         0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_PR_MASK                          0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_CPL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__PCRC_ENABLE_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                        0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__SELECTED_ALGORITHM_MASK                              0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TC_MASK                                              0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__DEFAULT_STREAM_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__TEE_LIMITED_STREAM_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_CNTL__STREAM_ID_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_STATUS
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                    0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_5_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                        0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG2__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG2__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG2__RID_BASE_MASK                                         0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_5_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CAP
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                    0x0000000FL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__XT_ENABLE__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                        0x4
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__PCRC_ENABLE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                  0x9
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                      0xa
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTED_ALGORITHM__SHIFT                            0xe
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TC__SHIFT                                            0x13
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__DEFAULT_STREAM__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TEE_LIMITED_STREAM__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__STREAM_ID__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__XT_ENABLE_MASK                                       0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_NPR_MASK                         0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_PR_MASK                          0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_CPL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__PCRC_ENABLE_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                        0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__SELECTED_ALGORITHM_MASK                              0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TC_MASK                                              0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__DEFAULT_STREAM_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__TEE_LIMITED_STREAM_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_CNTL__STREAM_ID_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_STATUS
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                    0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_6_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                        0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG2__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG2__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG2__RID_BASE_MASK                                         0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_6_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CAP
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                    0x0000000FL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__XT_ENABLE__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                        0x4
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__PCRC_ENABLE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                  0x9
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                      0xa
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTED_ALGORITHM__SHIFT                            0xe
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TC__SHIFT                                            0x13
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__DEFAULT_STREAM__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TEE_LIMITED_STREAM__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__STREAM_ID__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__XT_ENABLE_MASK                                       0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_NPR_MASK                         0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_PR_MASK                          0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_CPL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__PCRC_ENABLE_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                        0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__SELECTED_ALGORITHM_MASK                              0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TC_MASK                                              0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__DEFAULT_STREAM_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__TEE_LIMITED_STREAM_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_CNTL__STREAM_ID_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_STATUS
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                    0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_7_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                        0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG2__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG2__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG2__RID_BASE_MASK                                         0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_7_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CAP
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                    0x0000000FL
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                   0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__XT_ENABLE__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                        0x4
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                       0x6
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__PCRC_ENABLE__SHIFT                                   0x8
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                  0x9
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                      0xa
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTED_ALGORITHM__SHIFT                            0xe
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TC__SHIFT                                            0x13
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__DEFAULT_STREAM__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TEE_LIMITED_STREAM__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__STREAM_ID__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                     0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__XT_ENABLE_MASK                                       0x00000002L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_NPR_MASK                         0x0000000CL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_PR_MASK                          0x00000030L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_CPL_MASK                         0x000000C0L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__PCRC_ENABLE_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                    0x00000200L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                        0x00003C00L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__SELECTED_ALGORITHM_MASK                              0x0007C000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TC_MASK                                              0x00380000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__DEFAULT_STREAM_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__TEE_LIMITED_STREAM_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_CNTL__STREAM_ID_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_STATUS
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                  0x0
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                       0x1f
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                    0x0000000FL
+#define BIF_CFG_DEV0_EPF0_0_SELECTIVE_IDE_STREAM_8_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                         0x80000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                        0x00FFFF00L
+//BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG2__VALID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                   0x18
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG2__VALID_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG2__RID_BASE_MASK                                         0x00FFFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                              0x8
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                             0x14
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                0x000FFF00L
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                               0xFFF00000L
+//BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                             0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_0_IDE_8_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf0_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF0_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF0_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF0_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF0_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF0_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF0_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF0_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF0_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_HEADER
+#define BIF_CFG_DEV0_EPF0_VF0_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF0_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF0_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF0_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF0_BIST
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF0_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF0_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF0_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF0_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                 0x11
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                          0x1d
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                               0x1e
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__RX_MPS_FIXED_MASK                                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                            0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                 0x40000000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__SRIS_CLOCKING_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                               0x2000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                   0x1
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                     0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                        0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                         0x00000070L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                      0x00000380L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x2
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                      0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__L0P_ENABLE_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                            0x00000070L
+//BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF0_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                       0x00000010L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1b
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                      0x1c
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                     0x1d
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                     0x1e
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK             0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                        0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                       0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                       0x40000000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                          0x12
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                   0x13
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                     0x0003E000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                            0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                     0x00F80000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG4__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG5__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG6__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG7__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG8__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG9__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG10__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG11__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG12__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_HDR_LOG13__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA1__RESET_TIME__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA1__DLUP_TIME__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA1__RTR_VALID__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA1__RESET_TIME_MASK                                                      0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA1__DLUP_TIME_MASK                                                       0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA1__RTR_VALID_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_VF0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA2__FLR_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA2__FLR_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF0_RTR_DATA2__D3HOTD0_TIME_MASK                                                    0x00FFF000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                 0x0200L
+//BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_VF0_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf1_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF1_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF1_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF1_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF1_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF1_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF1_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF1_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF1_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF1_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF1_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF1_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF1_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_HEADER
+#define BIF_CFG_DEV0_EPF0_VF1_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF1_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF1_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF1_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF1_BIST
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF1_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF1_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF1_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF1_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF1_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF1_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                 0x11
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                          0x1d
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                               0x1e
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__RX_MPS_FIXED_MASK                                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                            0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                 0x40000000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__SRIS_CLOCKING_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                               0x2000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF1_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                   0x1
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                     0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                        0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                         0x00000070L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                      0x00000380L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x2
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                      0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__L0P_ENABLE_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                            0x00000070L
+//BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF1_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                       0x00000010L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1b
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                      0x1c
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                     0x1d
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                     0x1e
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK             0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                        0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                       0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                       0x40000000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                          0x12
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                   0x13
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                     0x0003E000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                            0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                     0x00F80000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG4__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG5__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG6__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG7__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG8__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG9__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG10__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG11__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG12__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_HDR_LOG13__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA1__RESET_TIME__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA1__DLUP_TIME__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA1__RTR_VALID__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA1__RESET_TIME_MASK                                                      0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA1__DLUP_TIME_MASK                                                       0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA1__RTR_VALID_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_VF1_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA2__FLR_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA2__FLR_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF1_RTR_DATA2__D3HOTD0_TIME_MASK                                                    0x00FFF000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                 0x0200L
+//BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_VF1_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf2_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF2_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF2_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF2_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF2_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF2_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF2_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF2_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF2_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF2_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF2_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF2_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF2_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_HEADER
+#define BIF_CFG_DEV0_EPF0_VF2_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF2_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF2_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF2_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF2_BIST
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF2_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF2_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF2_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF2_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF2_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF2_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                 0x11
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                          0x1d
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                               0x1e
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__RX_MPS_FIXED_MASK                                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                            0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                 0x40000000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__SRIS_CLOCKING_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                               0x2000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF2_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF2_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                   0x1
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                     0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                        0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                         0x00000070L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                      0x00000380L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x2
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                      0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__L0P_ENABLE_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                            0x00000070L
+//BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF2_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                       0x00000010L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1b
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                      0x1c
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                     0x1d
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                     0x1e
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK             0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                        0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                       0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                       0x40000000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                          0x12
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                   0x13
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                     0x0003E000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                            0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                     0x00F80000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG4__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG5__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG6__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG7__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG8__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG9__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG10__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG11__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG12__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_HDR_LOG13__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA1__RESET_TIME__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA1__DLUP_TIME__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA1__RTR_VALID__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA1__RESET_TIME_MASK                                                      0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA1__DLUP_TIME_MASK                                                       0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA1__RTR_VALID_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_VF2_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA2__FLR_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA2__FLR_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF2_RTR_DATA2__D3HOTD0_TIME_MASK                                                    0x00FFF000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                 0x0200L
+//BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_VF2_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf3_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF3_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF3_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF3_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF3_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF3_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF3_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF3_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF3_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF3_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF3_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF3_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF3_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_HEADER
+#define BIF_CFG_DEV0_EPF0_VF3_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF3_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF3_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF3_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF3_BIST
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF3_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF3_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF3_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF3_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF3_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF3_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                 0x11
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                          0x1d
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                               0x1e
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__RX_MPS_FIXED_MASK                                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                            0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                 0x40000000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__SRIS_CLOCKING_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                               0x2000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF3_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF3_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                   0x1
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                     0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                        0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                         0x00000070L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                      0x00000380L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x2
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                      0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__L0P_ENABLE_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                            0x00000070L
+//BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF3_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                       0x00000010L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1b
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                      0x1c
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                     0x1d
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                     0x1e
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK             0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                        0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                       0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                       0x40000000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                          0x12
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                   0x13
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                     0x0003E000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                            0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                     0x00F80000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG4__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG5__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG6__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG7__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG8__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG9__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG10__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG11__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG12__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_HDR_LOG13__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA1__RESET_TIME__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA1__DLUP_TIME__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA1__RTR_VALID__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA1__RESET_TIME_MASK                                                      0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA1__DLUP_TIME_MASK                                                       0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA1__RTR_VALID_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_VF3_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA2__FLR_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA2__FLR_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF3_RTR_DATA2__D3HOTD0_TIME_MASK                                                    0x00FFF000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                 0x0200L
+//BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_VF3_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf4_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF4_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF4_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF4_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF4_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF4_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF4_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF4_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF4_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF4_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF4_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF4_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF4_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_HEADER
+#define BIF_CFG_DEV0_EPF0_VF4_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF4_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF4_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF4_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF4_BIST
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF4_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF4_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF4_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF4_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF4_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF4_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                 0x11
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                          0x1d
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                               0x1e
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__RX_MPS_FIXED_MASK                                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                            0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                 0x40000000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__SRIS_CLOCKING_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                               0x2000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF4_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF4_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                   0x1
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                     0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                        0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                         0x00000070L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                      0x00000380L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x2
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                      0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__L0P_ENABLE_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                            0x00000070L
+//BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF4_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                       0x00000010L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1b
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                      0x1c
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                     0x1d
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                     0x1e
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK             0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                        0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                       0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                       0x40000000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                          0x12
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                   0x13
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                     0x0003E000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                            0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                     0x00F80000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG4__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG5__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG6__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG7__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG8__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG9__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG10__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG11__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG12__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_HDR_LOG13__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA1__RESET_TIME__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA1__DLUP_TIME__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA1__RTR_VALID__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA1__RESET_TIME_MASK                                                      0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA1__DLUP_TIME_MASK                                                       0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA1__RTR_VALID_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_VF4_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA2__FLR_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA2__FLR_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF4_RTR_DATA2__D3HOTD0_TIME_MASK                                                    0x00FFF000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                 0x0200L
+//BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_VF4_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf5_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF5_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF5_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF5_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF5_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF5_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF5_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF5_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF5_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF5_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF5_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF5_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF5_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_HEADER
+#define BIF_CFG_DEV0_EPF0_VF5_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF5_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF5_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF5_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF5_BIST
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF5_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF5_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF5_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF5_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF5_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF5_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                 0x11
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                          0x1d
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                               0x1e
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__RX_MPS_FIXED_MASK                                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                            0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                 0x40000000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__SRIS_CLOCKING_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                               0x2000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF5_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF5_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                   0x1
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                     0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                        0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                         0x00000070L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                      0x00000380L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x2
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                      0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__L0P_ENABLE_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                            0x00000070L
+//BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF5_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                       0x00000010L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1b
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                      0x1c
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                     0x1d
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                     0x1e
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK             0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                        0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                       0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                       0x40000000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                          0x12
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                   0x13
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                     0x0003E000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                            0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                     0x00F80000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG4__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG5__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG6__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG7__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG8__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG9__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG10__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG11__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG12__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_HDR_LOG13__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA1__RESET_TIME__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA1__DLUP_TIME__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA1__RTR_VALID__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA1__RESET_TIME_MASK                                                      0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA1__DLUP_TIME_MASK                                                       0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA1__RTR_VALID_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_VF5_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA2__FLR_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA2__FLR_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF5_RTR_DATA2__D3HOTD0_TIME_MASK                                                    0x00FFF000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                 0x0200L
+//BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_VF5_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf6_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF6_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF6_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF6_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF6_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF6_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF6_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF6_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF6_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF6_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF6_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF6_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF6_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_HEADER
+#define BIF_CFG_DEV0_EPF0_VF6_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF6_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF6_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF6_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF6_BIST
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF6_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF6_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF6_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF6_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF6_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF6_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                 0x11
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                          0x1d
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                               0x1e
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__RX_MPS_FIXED_MASK                                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                            0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                 0x40000000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__SRIS_CLOCKING_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                               0x2000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF6_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF6_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                   0x1
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                     0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                        0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                         0x00000070L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                      0x00000380L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x2
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                      0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__L0P_ENABLE_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                            0x00000070L
+//BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF6_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                       0x00000010L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1b
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                      0x1c
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                     0x1d
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                     0x1e
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK             0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                        0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                       0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                       0x40000000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                          0x12
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                   0x13
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                     0x0003E000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                            0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                     0x00F80000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG4__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG5__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG6__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG7__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG8__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG9__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG10__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG11__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG12__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_HDR_LOG13__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA1__RESET_TIME__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA1__DLUP_TIME__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA1__RTR_VALID__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA1__RESET_TIME_MASK                                                      0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA1__DLUP_TIME_MASK                                                       0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA1__RTR_VALID_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_VF6_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA2__FLR_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA2__FLR_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF6_RTR_DATA2__D3HOTD0_TIME_MASK                                                    0x00FFF000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                 0x0200L
+//BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_VF6_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf0_vf7_bifcfgdecp
+//BIF_CFG_DEV0_EPF0_VF7_VENDOR_ID
+#define BIF_CFG_DEV0_EPF0_VF7_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_ID
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_COMMAND
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF0_VF7_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF7_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_REVISION_ID
+#define BIF_CFG_DEV0_EPF0_VF7_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF7_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF0_VF7_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF0_VF7_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_SUB_CLASS
+#define BIF_CFG_DEV0_EPF0_VF7_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_CLASS
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_CACHE_LINE
+#define BIF_CFG_DEV0_EPF0_VF7_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF0_VF7_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF7_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_HEADER
+#define BIF_CFG_DEV0_EPF0_VF7_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF7_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF0_VF7_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF0_VF7_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF0_VF7_BIST
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF0_VF7_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF0_VF7_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF0_VF7_CAP_PTR
+#define BIF_CFG_DEV0_EPF0_VF7_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF7_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_MIN_GRANT
+#define BIF_CFG_DEV0_EPF0_VF7_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                 0x11
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                          0x1d
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                               0x1e
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__RX_MPS_FIXED_MASK                                                   0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                            0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                 0x40000000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__SRIS_CLOCKING_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                               0x2000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF0_VF7_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MASK
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF7_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                   0x1
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                     0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                        0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                         0x00000070L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                      0x00000380L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x2
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                      0x00000004L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__L0P_ENABLE_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                            0x00000070L
+//BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                           0x00000008L
+#define BIF_CFG_DEV0_EPF0_VF7_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                       0x00000010L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1b
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                      0x1c
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                     0x1d
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                     0x1e
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK             0x08000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                        0x10000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                       0x20000000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                       0x40000000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                   0xd
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                          0x12
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                   0x13
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                     0x0003E000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                            0x00040000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                     0x00F80000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG4__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG5__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG6__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG7__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG8__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG9__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG10__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG11__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG12__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_HDR_LOG13__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_RTR_DATA1
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA1__RESET_TIME__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA1__DLUP_TIME__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA1__RTR_VALID__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA1__RESET_TIME_MASK                                                      0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA1__DLUP_TIME_MASK                                                       0x00FFF000L
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA1__RTR_VALID_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF0_VF7_RTR_DATA2
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA2__FLR_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA2__FLR_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF0_VF7_RTR_DATA2__D3HOTD0_TIME_MASK                                                    0x00FFF000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST__SHIFT                                       0x5
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED__SHIFT                                0x6
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED__SHIFT                                 0x7
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__INVALIDATE_Q_DEPTH_MASK                                           0x001FL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__PAGE_ALIGNED_REQUEST_MASK                                         0x0020L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__GLOBAL_INVALIDATE_SUPPORTED_MASK                                  0x0040L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__RELAXED_ORDERING_SUPPORTED_MASK                                   0x0080L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CAP__TEE_EXCLUSIVE_ATTR_SUPPORTED_MASK                                 0x0200L
+//BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__STU__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__ATC_ENABLE__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__STU_MASK                                                         0x001FL
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__TEE_EXCLUSIVE_ATTR_ENABLE_MASK                                   0x0400L
+#define BIF_CFG_DEV0_EPF0_VF7_PCIE_ATS_CNTL__ATC_ENABLE_MASK                                                  0x8000L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf0_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF0_0_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF0_0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF0_0_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF0_0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF0_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF0_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF0_0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf0_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF0_0_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF0_0_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF0_0_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF0_0_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF0_0_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF0_0_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_0_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF0_0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF0_0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf0_BIFPFVFDEC1:1
+//BIF_BX_DEV0_EPF0_VF0_1_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF0_1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF0_1_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF0_1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF0_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF0_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF0_1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf0_SYSPFVFDEC:1
+//BIF_BX_DEV0_EPF0_VF0_1_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF0_1_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF0_1_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF0_1_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF0_1_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF0_1_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF0_1_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF0_1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF0_1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf0_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF0_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF0_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF0_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF0_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF0_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF0_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf0_BIFDEC2
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_LO
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_HI
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_MSG_DATA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_CONTROL
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF0_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF0_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf1_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF1_0_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF1_0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF1_0_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF1_0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF1_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF1_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF1_0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf1_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF1_0_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF1_0_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF1_0_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF1_0_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF1_0_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF1_0_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_0_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF1_0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF1_0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf1_BIFPFVFDEC1:1
+//BIF_BX_DEV0_EPF0_VF1_1_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF1_1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF1_1_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF1_1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF1_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF1_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF1_1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf1_SYSPFVFDEC:1
+//BIF_BX_DEV0_EPF0_VF1_1_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF1_1_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF1_1_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF1_1_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF1_1_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF1_1_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF1_1_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF1_1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF1_1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf1_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF1_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF1_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF1_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF1_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF1_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF1_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF1_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF1_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf1_BIFDEC2
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_LO
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_HI
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_MSG_DATA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_CONTROL
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF1_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF1_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf2_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF2_0_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF2_0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF2_0_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF2_0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF2_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF2_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF2_0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf2_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF2_0_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF2_0_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF2_0_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF2_0_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF2_0_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF2_0_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_0_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF2_0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF2_0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf2_BIFPFVFDEC1:1
+//BIF_BX_DEV0_EPF0_VF2_1_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF2_1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF2_1_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF2_1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF2_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF2_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF2_1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf2_SYSPFVFDEC:1
+//BIF_BX_DEV0_EPF0_VF2_1_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF2_1_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF2_1_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF2_1_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF2_1_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF2_1_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF2_1_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF2_1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF2_1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf2_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF2_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF2_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF2_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF2_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF2_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF2_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF2_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF2_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf2_BIFDEC2
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_LO
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_HI
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_MSG_DATA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_CONTROL
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF2_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF2_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf3_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF3_0_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF3_0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF3_0_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF3_0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF3_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF3_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF3_0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf3_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF3_0_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF3_0_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF3_0_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF3_0_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF3_0_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF3_0_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_0_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF3_0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF3_0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf3_BIFPFVFDEC1:1
+//BIF_BX_DEV0_EPF0_VF3_1_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF3_1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF3_1_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF3_1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF3_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF3_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF3_1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf3_SYSPFVFDEC:1
+//BIF_BX_DEV0_EPF0_VF3_1_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF3_1_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF3_1_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF3_1_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF3_1_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF3_1_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF3_1_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF3_1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF3_1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf3_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF3_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF3_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF3_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF3_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF3_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF3_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF3_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF3_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf3_BIFDEC2
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_LO
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_HI
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_MSG_DATA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_CONTROL
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF3_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF3_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf4_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF4_0_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF4_0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF4_0_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF4_0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF4_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF4_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF4_0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf4_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF4_0_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF4_0_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF4_0_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF4_0_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF4_0_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF4_0_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_0_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF4_0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF4_0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf4_BIFPFVFDEC1:1
+//BIF_BX_DEV0_EPF0_VF4_1_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF4_1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF4_1_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF4_1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF4_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF4_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF4_1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf4_SYSPFVFDEC:1
+//BIF_BX_DEV0_EPF0_VF4_1_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF4_1_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF4_1_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF4_1_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF4_1_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF4_1_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF4_1_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF4_1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF4_1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf4_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF4_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF4_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF4_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF4_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF4_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF4_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF4_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF4_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf4_BIFDEC2
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_LO
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_HI
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_MSG_DATA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_CONTROL
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF4_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF4_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf5_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF5_0_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF5_0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF5_0_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF5_0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF5_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF5_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF5_0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf5_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF5_0_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF5_0_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF5_0_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF5_0_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF5_0_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF5_0_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_0_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF5_0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF5_0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf5_BIFPFVFDEC1:1
+//BIF_BX_DEV0_EPF0_VF5_1_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF5_1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF5_1_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF5_1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF5_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF5_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF5_1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf5_SYSPFVFDEC:1
+//BIF_BX_DEV0_EPF0_VF5_1_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF5_1_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF5_1_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF5_1_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF5_1_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF5_1_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF5_1_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF5_1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF5_1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf5_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF5_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF5_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF5_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF5_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF5_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF5_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF5_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF5_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf5_BIFDEC2
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_LO
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_HI
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_MSG_DATA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_CONTROL
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF5_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF5_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf6_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF6_0_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF6_0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF6_0_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF6_0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF6_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF6_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF6_0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf6_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF6_0_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF6_0_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF6_0_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF6_0_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF6_0_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF6_0_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_0_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF6_0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF6_0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf6_BIFPFVFDEC1:1
+//BIF_BX_DEV0_EPF0_VF6_1_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF6_1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF6_1_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF6_1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF6_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF6_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF6_1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf6_SYSPFVFDEC:1
+//BIF_BX_DEV0_EPF0_VF6_1_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF6_1_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF6_1_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF6_1_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF6_1_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF6_1_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF6_1_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF6_1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF6_1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf6_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF6_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF6_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF6_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF6_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF6_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF6_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF6_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF6_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf6_BIFDEC2
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_LO
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_HI
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_MSG_DATA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_CONTROL
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF6_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF6_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf7_BIFPFVFDEC1
+//BIF_BX_DEV0_EPF0_VF7_0_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_0_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF7_0_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF7_0_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF7_0_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF7_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF7_0_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF7_0_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf7_SYSPFVFDEC
+//BIF_BX_DEV0_EPF0_VF7_0_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF7_0_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF7_0_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF7_0_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF7_0_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF7_0_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_0_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF7_0_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF7_0_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf7_BIFPFVFDEC1:1
+//BIF_BX_DEV0_EPF0_VF7_1_BIF_BME_STATUS
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                    0x10
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                      0x00010000L
+//BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                 0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                    0x2
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                        0x3
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                              0x10
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                           0x11
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                              0x12
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                  0x13
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                      0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                      0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                          0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                0x00010000L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                0x00040000L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                    0x00080000L
+//BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT  0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK  0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT          0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT        0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT        0x8
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK          0x000FFF00L
+//BIF_BX_DEV0_EPF0_VF7_1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                        0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                          0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT              0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT    0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK      0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                  0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                  0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                  0x2
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                  0x3
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                  0x4
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                  0x5
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                  0x6
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                  0x7
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                  0x8
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                  0x9
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                0xa
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                0xb
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                            0xc
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                            0xd
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                            0xe
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                            0xf
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                            0x10
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                            0x11
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                            0x12
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                            0x13
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                            0x14
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                            0x15
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                           0x16
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                           0x17
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                           0x18
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                           0x19
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                           0x1a
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                           0x1b
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                           0x1c
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                           0x1d
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                           0x1e
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                           0x1f
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                    0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                    0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                    0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                    0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                    0x00000010L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                    0x00000020L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                    0x00000040L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                    0x00000080L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                    0x00000100L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                    0x00000200L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                  0x00000400L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                  0x00000800L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                              0x00001000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                              0x00002000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                              0x00004000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                              0x00010000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                              0x00020000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                              0x00040000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                              0x00080000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                              0x00100000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                              0x00200000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                             0x00400000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                             0x00800000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                             0x01000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                             0x02000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                             0x04000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                             0x08000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                             0x10000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                             0x20000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                             0x40000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                             0x80000000L
+//BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                 0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                 0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                 0x2
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                 0x3
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                 0x4
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                 0x5
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                 0x6
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                 0x7
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                 0x8
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                 0x9
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                               0xa
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                               0xb
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                           0xc
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                           0xd
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                           0xe
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                           0xf
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                           0x10
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                           0x11
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                           0x12
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                           0x13
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                           0x14
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                           0x15
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                          0x16
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                          0x17
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                          0x18
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                          0x19
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                          0x1a
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                          0x1b
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                          0x1c
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                          0x1d
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                          0x1e
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                          0x1f
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                   0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                   0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                   0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                   0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                   0x00000010L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                   0x00000020L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                   0x00000040L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                   0x00000080L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                   0x00000100L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                   0x00000200L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                 0x00000400L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                 0x00000800L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                             0x00001000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                             0x00002000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                             0x00004000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                             0x00008000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                             0x00010000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                             0x00020000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                             0x00040000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                             0x00080000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                             0x00100000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                             0x00200000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                            0x00400000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                            0x00800000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                            0x01000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                            0x02000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                            0x04000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                            0x08000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                            0x10000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                            0x20000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                            0x40000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                            0x80000000L
+//BIF_BX_DEV0_EPF0_VF7_1_BIF_TRANS_PENDING
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                  0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                  0x00000002L
+//BIF_BX_DEV0_EPF0_VF7_1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_DEV0_EPF0_VF7_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                    0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                      0x00000001L
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                       0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                          0x8
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                            0x9
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                              0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                            0x00000100L
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                              0x00000200L
+//BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_INT_CNTL
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                            0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                              0x00000002L
+//BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                          0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                        0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                             0x8
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                            0xf
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                             0x10
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                            0x17
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                              0x18
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                              0x19
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                            0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                          0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                               0x00000F00L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                              0x00008000L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                               0x000F0000L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                              0x00800000L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                0x01000000L
+#define BIF_BX_DEV0_EPF0_VF7_1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                0x02000000L
+//BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                      0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                      0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                      0x2
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                      0x3
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                      0x4
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                 0xa
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                        0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                        0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                        0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                        0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                        0x00000010L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                   0x0003FC00L
+//BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                         0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                         0x1
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                         0x2
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                         0x3
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                         0x5
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                         0x7
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                           0x00000001L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                           0x00000002L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                           0x00000004L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                           0x00000008L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                           0x00000020L
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                           0x00000080L
+//BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                0x4
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                  0x000000F0L
+//BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_STATUS
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                          0x4
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                       0x0000000FL
+#define BIF_BX_DEV0_EPF0_VF7_1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                            0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_bif_bx_dev0_epf0_vf7_SYSPFVFDEC:1
+//BIF_BX_DEV0_EPF0_VF7_1_MM_INDEX
+#define BIF_BX_DEV0_EPF0_VF7_1_MM_INDEX__MM_OFFSET__SHIFT                                                     0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MM_INDEX__MM_APER__SHIFT                                                       0x1f
+#define BIF_BX_DEV0_EPF0_VF7_1_MM_INDEX__MM_OFFSET_MASK                                                       0x7FFFFFFFL
+#define BIF_BX_DEV0_EPF0_VF7_1_MM_INDEX__MM_APER_MASK                                                         0x80000000L
+//BIF_BX_DEV0_EPF0_VF7_1_MM_DATA
+#define BIF_BX_DEV0_EPF0_VF7_1_MM_DATA__MM_DATA__SHIFT                                                        0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MM_DATA__MM_DATA_MASK                                                          0xFFFFFFFFL
+//BIF_BX_DEV0_EPF0_VF7_1_MM_INDEX_HI
+#define BIF_BX_DEV0_EPF0_VF7_1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                               0x0
+#define BIF_BX_DEV0_EPF0_VF7_1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                 0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf7_BIFPFVFDEC1
+//RCC_DEV0_EPF0_VF7_RCC_ERR_LOG
+#define RCC_DEV0_EPF0_VF7_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS__SHIFT                              0x0
+#define RCC_DEV0_EPF0_VF7_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS__SHIFT                                     0x1
+#define RCC_DEV0_EPF0_VF7_RCC_ERR_LOG__INVALID_REG_ACCESS_IN_SRIOV_STATUS_MASK                                0x00000001L
+#define RCC_DEV0_EPF0_VF7_RCC_ERR_LOG__DOORBELL_READ_ACCESS_STATUS_MASK                                       0x00000002L
+//RCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN
+#define RCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN__SHIFT                                   0x0
+#define RCC_DEV0_EPF0_VF7_RCC_DOORBELL_APER_EN__BIF_DOORBELL_APER_EN_MASK                                     0x00000001L
+//RCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE
+#define RCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_RCC_CONFIG_MEMSIZE__CONFIG_MEMSIZE_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED
+#define RCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED__CONFIG_RESERVED__SHIFT                                         0x0
+#define RCC_DEV0_EPF0_VF7_RCC_CONFIG_RESERVED__CONFIG_RESERVED_MASK                                           0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER
+#define RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER__SHIFT                                     0x0
+#define RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE__SHIFT                                          0x1f
+#define RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER__FUNC_IDENTIFIER_MASK                                       0x00000001L
+#define RCC_DEV0_EPF0_VF7_RCC_IOV_FUNC_IDENTIFIER__IOV_ENABLE_MASK                                            0x80000000L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_epf0_vf7_BIFDEC2
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT0_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT1_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT2_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT3_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT4_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT5_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT6_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT7_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_LO
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO__SHIFT                                           0x2
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_LO__MSG_ADDR_LO_MASK                                             0xFFFFFFFCL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_HI
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI__SHIFT                                           0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_ADDR_HI__MSG_ADDR_HI_MASK                                             0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_MSG_DATA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_MSG_DATA__MSG_DATA__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_MSG_DATA__MSG_DATA_MASK                                               0xFFFFFFFFL
+//RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_CONTROL
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_CONTROL__MASK_BIT__SHIFT                                              0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_VECT8_CONTROL__MASK_BIT_MASK                                                0x00000001L
+//RCC_DEV0_EPF0_VF7_GFXMSIX_PBA
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_PBA__MSIX_PENDING_BITS_0__SHIFT                                             0x0
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_PBA__MSIX_PENDING_BITS_1__SHIFT                                             0x1
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_PBA__MSIX_PENDING_BITS_0_MASK                                               0x00000001L
+#define RCC_DEV0_EPF0_VF7_GFXMSIX_PBA__MSIX_PENDING_BITS_1_MASK                                               0x00000002L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf1_bifcfgdecp
+//BIF_CFG_DEV0_EPF1_VENDOR_ID
+#define BIF_CFG_DEV0_EPF1_VENDOR_ID__VENDOR_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_VENDOR_ID__VENDOR_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF1_DEVICE_ID
+#define BIF_CFG_DEV0_EPF1_DEVICE_ID__DEVICE_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_ID__DEVICE_ID_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_EPF1_COMMAND
+#define BIF_CFG_DEV0_EPF1_COMMAND__IO_ACCESS_EN__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_ACCESS_EN__SHIFT                                                       0x1
+#define BIF_CFG_DEV0_EPF1_COMMAND__BUS_MASTER_EN__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_EPF1_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF1_COMMAND__PAL_SNOOP_EN__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_EPF1_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                               0x6
+#define BIF_CFG_DEV0_EPF1_COMMAND__AD_STEPPING__SHIFT                                                         0x7
+#define BIF_CFG_DEV0_EPF1_COMMAND__SERR_EN__SHIFT                                                             0x8
+#define BIF_CFG_DEV0_EPF1_COMMAND__FAST_B2B_EN__SHIFT                                                         0x9
+#define BIF_CFG_DEV0_EPF1_COMMAND__INT_DIS__SHIFT                                                             0xa
+#define BIF_CFG_DEV0_EPF1_COMMAND__IO_ACCESS_EN_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_ACCESS_EN_MASK                                                         0x0002L
+#define BIF_CFG_DEV0_EPF1_COMMAND__BUS_MASTER_EN_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_EPF1_COMMAND__SPECIAL_CYCLE_EN_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_EPF1_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF1_COMMAND__PAL_SNOOP_EN_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_EPF1_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                 0x0040L
+#define BIF_CFG_DEV0_EPF1_COMMAND__AD_STEPPING_MASK                                                           0x0080L
+#define BIF_CFG_DEV0_EPF1_COMMAND__SERR_EN_MASK                                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_COMMAND__FAST_B2B_EN_MASK                                                           0x0200L
+#define BIF_CFG_DEV0_EPF1_COMMAND__INT_DIS_MASK                                                               0x0400L
+//BIF_CFG_DEV0_EPF1_STATUS
+#define BIF_CFG_DEV0_EPF1_STATUS__IMMEDIATE_READINESS__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_STATUS__INT_STATUS__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF1_STATUS__CAP_LIST__SHIFT                                                             0x4
+#define BIF_CFG_DEV0_EPF1_STATUS__PCI_66_CAP__SHIFT                                                           0x5
+#define BIF_CFG_DEV0_EPF1_STATUS__FAST_BACK_CAPABLE__SHIFT                                                    0x7
+#define BIF_CFG_DEV0_EPF1_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_STATUS__DEVSEL_TIMING__SHIFT                                                        0x9
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                  0xb
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                0xd
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF1_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF1_STATUS__IMMEDIATE_READINESS_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF1_STATUS__INT_STATUS_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF1_STATUS__CAP_LIST_MASK                                                               0x0010L
+#define BIF_CFG_DEV0_EPF1_STATUS__PCI_66_CAP_MASK                                                             0x0020L
+#define BIF_CFG_DEV0_EPF1_STATUS__FAST_BACK_CAPABLE_MASK                                                      0x0080L
+#define BIF_CFG_DEV0_EPF1_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_STATUS__DEVSEL_TIMING_MASK                                                          0x0600L
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNAL_TARGET_ABORT_MASK                                                    0x0800L
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_TARGET_ABORT_MASK                                                  0x1000L
+#define BIF_CFG_DEV0_EPF1_STATUS__RECEIVED_MASTER_ABORT_MASK                                                  0x2000L
+#define BIF_CFG_DEV0_EPF1_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF1_STATUS__PARITY_ERROR_DETECTED_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF1_REVISION_ID
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MINOR_REV_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MAJOR_REV_ID__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MINOR_REV_ID_MASK                                                      0x0FL
+#define BIF_CFG_DEV0_EPF1_REVISION_ID__MAJOR_REV_ID_MASK                                                      0xF0L
+//BIF_CFG_DEV0_EPF1_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF1_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_PROG_INTERFACE__PROG_INTERFACE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF1_SUB_CLASS
+#define BIF_CFG_DEV0_EPF1_SUB_CLASS__SUB_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_SUB_CLASS__SUB_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF1_BASE_CLASS
+#define BIF_CFG_DEV0_EPF1_BASE_CLASS__BASE_CLASS__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_CLASS__BASE_CLASS_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_CACHE_LINE
+#define BIF_CFG_DEV0_EPF1_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                    0xFFL
+//BIF_CFG_DEV0_EPF1_LATENCY
+#define BIF_CFG_DEV0_EPF1_LATENCY__LATENCY_TIMER__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_LATENCY__LATENCY_TIMER_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_HEADER
+#define BIF_CFG_DEV0_EPF1_HEADER__HEADER_TYPE__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF1_HEADER__DEVICE_TYPE__SHIFT                                                          0x7
+#define BIF_CFG_DEV0_EPF1_HEADER__HEADER_TYPE_MASK                                                            0x7FL
+#define BIF_CFG_DEV0_EPF1_HEADER__DEVICE_TYPE_MASK                                                            0x80L
+//BIF_CFG_DEV0_EPF1_BIST
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_COMP__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_STRT__SHIFT                                                              0x6
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_CAP__SHIFT                                                               0x7
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_COMP_MASK                                                                0x0FL
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_STRT_MASK                                                                0x40L
+#define BIF_CFG_DEV0_EPF1_BIST__BIST_CAP_MASK                                                                 0x80L
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_1__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_1__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_2__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_2__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_3__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_3__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_4__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_4__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_5__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_5__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_6__BASE_ADDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_BASE_ADDR_6__BASE_ADDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                     0x10
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                         0x1
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                      0x00000001L
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                           0x0000000EL
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                          0x000000F0L
+#define BIF_CFG_DEV0_EPF1_ROM_BASE_ADDR__BASE_ADDR_MASK                                                       0xFFFFF800L
+//BIF_CFG_DEV0_EPF1_CAP_PTR
+#define BIF_CFG_DEV0_EPF1_CAP_PTR__CAP_PTR__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF1_CAP_PTR__CAP_PTR_MASK                                                               0xFFL
+//BIF_CFG_DEV0_EPF1_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                 0xFFL
+//BIF_CFG_DEV0_EPF1_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                   0xFFL
+//BIF_CFG_DEV0_EPF1_MIN_GRANT
+#define BIF_CFG_DEV0_EPF1_MIN_GRANT__MIN_GNT__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_MIN_GRANT__MIN_GNT_MASK                                                             0xFFL
+//BIF_CFG_DEV0_EPF1_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF1_MAX_LATENCY__MAX_LAT__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_MAX_LATENCY__MAX_LAT_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__CAP_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__NEXT_PTR__SHIFT                                                    0x8
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__LENGTH__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__CAP_ID_MASK                                                        0x000000FFL
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__NEXT_PTR_MASK                                                      0x0000FF00L
+#define BIF_CFG_DEV0_EPF1_VENDOR_CAP_LIST__LENGTH_MASK                                                        0x00FF0000L
+//BIF_CFG_DEV0_EPF1_ADAPTER_ID_W
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_ID__SHIFT                                                   0x10
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_VENDOR_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_ADAPTER_ID_W__SUBSYSTEM_ID_MASK                                                     0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PMI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF1_PMI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF1_PMI_CAP
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__VERSION__SHIFT                                                             0x0
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_CLOCK__SHIFT                                                           0x3
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                 0x4
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                   0x5
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__AUX_CURRENT__SHIFT                                                         0x6
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D1_SUPPORT__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D2_SUPPORT__SHIFT                                                          0xa
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_SUPPORT__SHIFT                                                         0xb
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__VERSION_MASK                                                               0x0007L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_CLOCK_MASK                                                             0x0008L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                   0x0010L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                     0x0020L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__AUX_CURRENT_MASK                                                           0x01C0L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D1_SUPPORT_MASK                                                            0x0200L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__D2_SUPPORT_MASK                                                            0x0400L
+#define BIF_CFG_DEV0_EPF1_PMI_CAP__PME_SUPPORT_MASK                                                           0xF800L
+//BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_EN__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                               0x16
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                  0x17
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__POWER_STATE_MASK                                                   0x00000003L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                 0x00000008L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_EN_MASK                                                        0x00000100L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                   0x00001E00L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                    0x00006000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PME_STATUS_MASK                                                    0x00008000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                 0x00400000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                    0x00800000L
+#define BIF_CFG_DEV0_EPF1_PMI_STATUS_CNTL__PMI_DATA_MASK                                                      0xFF000000L
+//SBRN
+#define SBRN__SBRN__SHIFT                                                                                     0x0
+#define SBRN__SBRN_MASK                                                                                       0xFFL
+//FLADJ
+#define FLADJ__FLADJ__SHIFT                                                                                   0x0
+#define FLADJ__NFC__SHIFT                                                                                     0x6
+#define FLADJ__FLADJ_MASK                                                                                     0x3FL
+#define FLADJ__NFC_MASK                                                                                       0x40L
+//BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF1_PCIE_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__VERSION__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__DEVICE_TYPE__SHIFT                                                        0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__VERSION_MASK                                                              0x000FL
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__DEVICE_TYPE_MASK                                                          0x00F0L
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                      0x3E00L
+#define BIF_CFG_DEV0_EPF1_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                    0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                            0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                         0xf
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                     0x11
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                        0x12
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                        0x1a
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                      0x1c
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                              0x1d
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                                   0x1e
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                0x00000007L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__PHANTOM_FUNC_MASK                                                       0x00000018L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__EXTENDED_TAG_MASK                                                       0x00000020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                             0x000001C0L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                              0x00000E00L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                           0x00008000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__RX_MPS_FIXED_MASK                                                       0x00020000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                          0x03FC0000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                          0x0C000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__FLR_CAPABLE_MASK                                                        0x10000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                                0x20000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                     0x40000000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                0x1
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                    0x2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                 0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__CORR_ERR_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                  0x0002L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                      0x0004L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__USR_REPORT_EN_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                  0x00E0L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                   0x0100L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                   0x0200L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                             0x7000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL__INITIATE_FLR_MASK                                                      0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__CORR_ERR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                 0x1
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__FATAL_ERR__SHIFT                                                     0x2
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__USR_DETECTED__SHIFT                                                  0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__AUX_PWR__SHIFT                                                       0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                 0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__CORR_ERR_MASK                                                        0x0001L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                   0x0002L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__FATAL_ERR_MASK                                                       0x0004L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__USR_DETECTED_MASK                                                    0x0008L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__AUX_PWR_MASK                                                         0x0010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                   0x0040L
+//BIF_CFG_DEV0_EPF1_LINK_CAP
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_SPEED__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_WIDTH__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PM_SUPPORT__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                   0xc
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                        0x13
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                           0x15
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PORT_NUMBER__SHIFT                                                        0x18
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_SPEED_MASK                                                           0x0000000FL
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_WIDTH_MASK                                                           0x000003F0L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PM_SUPPORT_MASK                                                           0x00000C00L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                     0x00007000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__L1_EXIT_LATENCY_MASK                                                      0x00038000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                          0x00100000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                             0x00200000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                          0x00400000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP__PORT_NUMBER_MASK                                                          0xFF000000L
+//BIF_CFG_DEV0_EPF1_LINK_CNTL
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PM_CONTROL__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                      0x2
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_DIS__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__RETRAIN_LINK__SHIFT                                                      0x5
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                  0x6
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                         0xb
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                                 0xd
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                             0xe
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PM_CONTROL_MASK                                                          0x0003L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                        0x0004L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                   0x0008L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_DIS_MASK                                                            0x0010L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__RETRAIN_LINK_MASK                                                        0x0020L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                    0x0040L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__EXTENDED_SYNC_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                           0x0800L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__SRIS_CLOCKING_MASK                                                       0x1000L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                                   0x2000L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                               0xC000L
+//BIF_CFG_DEV0_EPF1_LINK_STATUS
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_TRAINING__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__DL_ACTIVE__SHIFT                                                       0xd
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                             0x03F0L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_TRAINING_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__DL_ACTIVE_MASK                                                         0x2000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                         0x4000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                      0x7
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                       0xa
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                   0xe
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                 0x11
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                  0x12
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                    0x15
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                        0x16
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                  0x18
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                   0x1a
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                   0x1f
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                        0x00000040L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                        0x00000080L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                        0x00000100L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                            0x00000200L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                     0x00000800L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                0x00003000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                     0x0000C000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                   0x00020000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                    0x000C0000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                      0x00200000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                          0x00C00000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                    0x03000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                     0x04000000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                     0x80000000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                              0x5
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                       0x7
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__LTR_EN__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                   0xb
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__OBFF_EN__SHIFT                                                        0xd
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                0x000FL
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                  0x0010L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                0x0020L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                         0x0080L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                            0x0200L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__LTR_EN_MASK                                                           0x0400L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                     0x1000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__OBFF_EN_MASK                                                          0x6000L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                      0x8000L
+//BIF_CFG_DEV0_EPF1_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS2__RESERVED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS2__RESERVED_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF1_LINK_CAP2
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                          0x9
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                          0x10
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                         0x17
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                         0x18
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                0x000000FEL
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                 0x00000100L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                            0x0000FE00L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                           0x00800000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_EPF1_LINK_CAP2__DRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF1_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                 0x4
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                            0x6
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                   0xb
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                   0x0010L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                        0x0020L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                              0x0040L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__XMIT_MARGIN_MASK                                                        0x0380L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                     0x0800L
+#define BIF_CFG_DEV0_EPF1_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                              0xF000L
+//BIF_CFG_DEV0_EPF1_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                      0x1
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                0x2
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                0x3
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                0x4
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                              0x7
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                               0xa
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                  0xc
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                           0xf
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                             0x0001L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                        0x0002L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                  0x0004L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                  0x0008L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                  0x0010L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                0x0080L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                             0x0300L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                    0x7000L
+#define BIF_CFG_DEV0_EPF1_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                             0x8000L
+//BIF_CFG_DEV0_EPF1_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__CAP_ID__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__CAP_ID_MASK                                                           0x00FFL
+#define BIF_CFG_DEV0_EPF1_MSI_CAP_LIST__NEXT_PTR_MASK                                                         0xFF00L
+//BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EN__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                      0x8
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                           0x9
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                            0xa
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EN_MASK                                                           0x0001L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                    0x000EL
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                     0x0070L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_64BIT_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                        0x0100L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                             0x0200L
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                              0x0400L
+//BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                             0x2
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                               0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA__MSI_DATA__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA__MSI_DATA_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                 0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MASK
+#define BIF_CFG_DEV0_EPF1_MSI_MASK__MSI_MASK__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MASK__MSI_MASK_MASK                                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                           0xFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF1_MSI_MASK_64__MSI_MASK_64__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_MSI_MASK_64__MSI_MASK_64_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_PENDING
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING__MSI_PENDING__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING__MSI_PENDING_MASK                                                       0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_MSI_PENDING_64__MSI_PENDING_64_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__CAP_ID__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                      0x8
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__CAP_ID_MASK                                                          0x00FFL
+#define BIF_CFG_DEV0_EPF1_MSIX_CAP_LIST__NEXT_PTR_MASK                                                        0xFF00L
+//BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                                0xe
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                       0xf
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                                 0x07FFL
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                                  0x4000L
+#define BIF_CFG_DEV0_EPF1_MSIX_MSG_CNTL__MSIX_EN_MASK                                                         0x8000L
+//BIF_CFG_DEV0_EPF1_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF1_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_MSIX_PBA
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_BIR_MASK                                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                      0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF1_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                       0x1
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                       0x2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                          0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                         0x00000002L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                         0x00000004L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                            0x00000008L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                             0x00000070L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                          0x00000380L
+//BIF_CFG_DEV0_EPF1_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                        0x2
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                              0x00000001L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                             0x00000002L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                          0x00000004L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__L0P_ENABLE_MASK                                                       0x00000008L
+#define BIF_CFG_DEV0_EPF1_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                                0x00000070L
+//BIF_CFG_DEV0_EPF1_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                             0x00000007L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                               0x00000008L
+#define BIF_CFG_DEV0_EPF1_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                           0x00000010L
+//BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                           0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                        0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                           0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                       0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                          0x11
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                           0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                          0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                     0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                    0x17
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                           0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                            0x19
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                       0x1a
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                       0x1b
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                                  0x1c
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                                 0x1d
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                                 0x1e
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                             0x00000010L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                          0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                             0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                              0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                         0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                           0x00010000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                            0x00020000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                             0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                       0x00200000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                      0x00800000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                             0x01000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                              0x02000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                         0x04000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                         0x08000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                    0x10000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                                   0x20000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                                   0x40000000L
+//BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF1_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                             0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                               0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                        0xe
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                        0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                               0x00000001L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                              0x00000080L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                   0x00000100L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                  0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                 0x00002000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                          0x00004000L
+#define BIF_CFG_DEV0_EPF1_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                          0x00008000L
+//BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                    0x9
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                0xb
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                       0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                              0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                       0x13
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                           0x0000001FL
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                           0x00000100L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                      0x00000200L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                       0x00000400L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                  0x00000800L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                         0x0003E000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                                0x00040000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                         0x00F80000L
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG0__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG1__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG2__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG3__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG4__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG5__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG6__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG7__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                               0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG8__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG9__TLP_HDR_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG10__TLP_HDR_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG11__TLP_HDR_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG12__TLP_HDR_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_HDR_LOG13__TLP_HDR_MASK                                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                           0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                           0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                        0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                             0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                            0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                             0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                             0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                          0x0008L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                               0x0020L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                              0x0080L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                       0xFF00L
+//BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                       0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                    0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                      0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                         0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                         0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                      0x0008L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                          0x0010L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                        0x0040L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                          0x0080L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0300L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                   0x0C00L
+#define BIF_CFG_DEV0_EPF1_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                 0x1000L
+//BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_RTR_DATA1
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__RESET_TIME__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__DLUP_TIME__SHIFT                                                         0xc
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__RTR_VALID__SHIFT                                                         0x1f
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__RESET_TIME_MASK                                                          0x00000FFFL
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__DLUP_TIME_MASK                                                           0x00FFF000L
+#define BIF_CFG_DEV0_EPF1_RTR_DATA1__RTR_VALID_MASK                                                           0x80000000L
+//BIF_CFG_DEV0_EPF1_RTR_DATA2
+#define BIF_CFG_DEV0_EPF1_RTR_DATA2__FLR_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF1_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                      0xc
+#define BIF_CFG_DEV0_EPF1_RTR_DATA2__FLR_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_EPF1_RTR_DATA2__D3HOTD0_TIME_MASK                                                        0x00FFF000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER__SHIFT                                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR__SHIFT                                       0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_ID_MASK                                           0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__CAP_VER_MASK                                          0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_ENH_CAP_LIST__NEXT_PTR_MASK                                         0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__EXTENDED_PWR_BUDGET_ENABLE__SHIFT                      0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__DATA_SELECT_MASK                                       0x000000FFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA_SELECT__EXTENDED_PWR_BUDGET_ENABLE_MASK                        0x00010000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__BASE_POWER__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__DATA_SCALE__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_STATE__SHIFT                                               0xd
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__TYPE__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__POWER_RAIL__SHIFT                                             0x12
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__BASE_POWER_MASK                                               0x000000FFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__DATA_SCALE_MASK                                               0x00000300L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_SUB_STATE_MASK                                             0x00001C00L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__PM_STATE_MASK                                                 0x00006000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__TYPE_MASK                                                     0x00038000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_DATA__POWER_RAIL_MASK                                               0x001C0000L
+//BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP__EXTENDED_PWR_BUDGET_SUPPORT__SHIFT                             0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP__SYSTEM_ALLOCATED_MASK                                          0x01L
+#define BIF_CFG_DEV0_EPF1_PCIE_PWR_BUDGET_CAP__EXTENDED_PWR_BUDGET_SUPPORT_MASK                               0x02L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__SUBSTATE_MAX__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                                0xc
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                                0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                                0x18
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__SUBSTATE_MAX_MASK                                                     0x0000001FL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_UNIT_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                  0x00FF0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                  0xFF000000L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                         0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                           0x000000FFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_STATUS__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_STATUS_MASK                                               0x001FL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_STATUS__SUBSTATE_CNTL_ENABLED_MASK                                         0x0100L
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL__SUBSTATE_CNTL__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_CNTL__SUBSTATE_CNTL_MASK                                                   0x001FL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                              0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                                0xFF00L
+//BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                        0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                          0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                              0x0070L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR1_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR2_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR3_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR4_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR5_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CAP__BAR_SIZE_SUPPORTED_MASK                                              0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_INDEX__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_TOTAL_NUM__SHIFT                                                0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_INDEX_MASK                                                      0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_TOTAL_NUM_MASK                                                  0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_MASK                                                       0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_BAR6_CNTL__BAR_SIZE_SUPPORTED_UPPER_MASK                                       0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED__SHIFT                               0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED__SHIFT                                    0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__TRANSLATED_REQ_WITH_PASID_SUPPORTED__SHIFT                          0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__MAX_PASID_WIDTH__SHIFT                                              0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_EXE_PERMISSION_SUPPORTED_MASK                                 0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__PASID_PRIV_MODE_SUPPORTED_MASK                                      0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__TRANSLATED_REQ_WITH_PASID_SUPPORTED_MASK                            0x0008L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CAP__MAX_PASID_WIDTH_MASK                                                0x1F00L
+//BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE__SHIFT                                 0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__TRANSLATED_REQ_WITH_PASID_ENABLE__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_ENABLE_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_EXE_PERMISSION_ENABLE_MASK                                   0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__PASID_PRIV_MODE_SUPPORTED_ENABLE_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_PASID_CNTL__TRANSLATED_REQ_WITH_PASID_ENABLE_MASK                              0x0008L
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER__SHIFT                                             0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR__SHIFT                                            0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_ENH_CAP_LIST__CAP_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_ENH_CAP_LIST__CAP_VER_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_ENH_CAP_LIST__NEXT_PTR_MASK                                              0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED__SHIFT                            0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                     0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_14_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                      0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM__SHIFT                              0x15
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_CAP_MASK                                         0x00000001L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_ARI_CAP_HIERARCHY_PRESERVED_MASK                              0x00000002L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                       0x00000004L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_14_BIT_TAG_REQUESTER_SUPPORTED_MASK                        0x00000008L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CAP__SRIOV_VF_MIGRATION_INTR_MSG_NUM_MASK                                0xFFE00000L
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE__SHIFT                                0x1
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE__SHIFT                           0x2
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY__SHIFT                                  0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_14_BIT_TAG_REQUESTER_ENABLE__SHIFT                     0x6
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_ENABLE_MASK                                            0x0001L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_ENABLE_MASK                                  0x0002L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MIGRATION_INTR_ENABLE_MASK                             0x0004L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_MSE_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_ARI_CAP_HIERARCHY_MASK                                    0x0010L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_TEN_BIT_TAG_REQUESTER_ENABLE_MASK                      0x0020L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_CONTROL__SRIOV_VF_14_BIT_TAG_REQUESTER_ENABLE_MASK                       0x0040L
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_STATUS
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_STATUS__SRIOV_VF_MIGRATION_STATUS_MASK                                   0x0001L
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_INITIAL_VFS
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_INITIAL_VFS__SRIOV_INITIAL_VFS_MASK                                      0xFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_TOTAL_VFS
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_TOTAL_VFS__SRIOV_TOTAL_VFS_MASK                                          0xFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_NUM_VFS
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_NUM_VFS__SRIOV_NUM_VFS_MASK                                              0xFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_FUNC_DEP_LINK
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_FUNC_DEP_LINK__SRIOV_FUNC_DEP_LINK_MASK                                  0xFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_FIRST_VF_OFFSET
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET__SHIFT                            0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_FIRST_VF_OFFSET__SRIOV_FIRST_VF_OFFSET_MASK                              0xFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_STRIDE
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_STRIDE__SRIOV_VF_STRIDE_MASK                                          0xFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_DEVICE_ID
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID__SHIFT                                  0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_DEVICE_ID__SRIOV_VF_DEVICE_ID_MASK                                    0xFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_SUPPORTED_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE__SHIFT                    0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_SUPPORTED_PAGE_SIZE__SRIOV_SUPPORTED_PAGE_SIZE_MASK                      0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_SYSTEM_PAGE_SIZE
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE__SHIFT                          0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_SYSTEM_PAGE_SIZE__SRIOV_SYSTEM_PAGE_SIZE_MASK                            0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_0__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_1__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_2__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_3__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_4__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_BASE_ADDR_5__VF_BASE_ADDR_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR__SHIFT     0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SHIFT  0x3
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_BIR_MASK       0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET__SRIOV_VF_MIGRATION_STATE_ARRAY_OFFSET_MASK  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR1_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR2_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR3_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR4_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR5_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CAP
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CAP__VF_BAR_SIZE_SUPPORTED_MASK                                 0xFFFFFFF0L
+//BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM__SHIFT                                   0x5
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE__SHIFT                                        0x8
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER__SHIFT                        0x10
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_INDEX_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_TOTAL_NUM_MASK                                     0x000000E0L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_MASK                                          0x00003F00L
+#define BIF_CFG_DEV0_EPF1_PCIE_VF_RESIZE_BAR6_CNTL__VF_BAR_SIZE_SUPPORTED_UPPER_MASK                          0xFFFF0000L
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_epf1_vf0_bifcfgdecp
+//BIF_CFG_DEV0_EPF1_VF0_VENDOR_ID
+#define BIF_CFG_DEV0_EPF1_VF0_VENDOR_ID__VENDOR_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_VENDOR_ID__VENDOR_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_ID
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_ID__DEVICE_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_ID__DEVICE_ID_MASK                                                       0xFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_COMMAND
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__IO_ACCESS_EN__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__MEM_ACCESS_EN__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__BUS_MASTER_EN__SHIFT                                                   0x2
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                         0x4
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__PAL_SNOOP_EN__SHIFT                                                    0x5
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                           0x6
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__AD_STEPPING__SHIFT                                                     0x7
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__SERR_EN__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__FAST_B2B_EN__SHIFT                                                     0x9
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__INT_DIS__SHIFT                                                         0xa
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__IO_ACCESS_EN_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__MEM_ACCESS_EN_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__BUS_MASTER_EN_MASK                                                     0x0004L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__SPECIAL_CYCLE_EN_MASK                                                  0x0008L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                           0x0010L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__PAL_SNOOP_EN_MASK                                                      0x0020L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__PARITY_ERROR_RESPONSE_MASK                                             0x0040L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__AD_STEPPING_MASK                                                       0x0080L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__SERR_EN_MASK                                                           0x0100L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__FAST_B2B_EN_MASK                                                       0x0200L
+#define BIF_CFG_DEV0_EPF1_VF0_COMMAND__INT_DIS_MASK                                                           0x0400L
+//BIF_CFG_DEV0_EPF1_VF0_STATUS
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__IMMEDIATE_READINESS__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__INT_STATUS__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__CAP_LIST__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__PCI_66_CAP__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__FAST_BACK_CAPABLE__SHIFT                                                0x7
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__DEVSEL_TIMING__SHIFT                                                    0x9
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                              0xb
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                            0xc
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                            0xd
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__PARITY_ERROR_DETECTED__SHIFT                                            0xf
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__IMMEDIATE_READINESS_MASK                                                0x0001L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__INT_STATUS_MASK                                                         0x0008L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__CAP_LIST_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__PCI_66_CAP_MASK                                                         0x0020L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__FAST_BACK_CAPABLE_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__DEVSEL_TIMING_MASK                                                      0x0600L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__SIGNAL_TARGET_ABORT_MASK                                                0x0800L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__RECEIVED_TARGET_ABORT_MASK                                              0x1000L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__RECEIVED_MASTER_ABORT_MASK                                              0x2000L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF1_VF0_STATUS__PARITY_ERROR_DETECTED_MASK                                              0x8000L
+//BIF_CFG_DEV0_EPF1_VF0_REVISION_ID
+#define BIF_CFG_DEV0_EPF1_VF0_REVISION_ID__MINOR_REV_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_VF0_REVISION_ID__MAJOR_REV_ID__SHIFT                                                0x4
+#define BIF_CFG_DEV0_EPF1_VF0_REVISION_ID__MINOR_REV_ID_MASK                                                  0x0FL
+#define BIF_CFG_DEV0_EPF1_VF0_REVISION_ID__MAJOR_REV_ID_MASK                                                  0xF0L
+//BIF_CFG_DEV0_EPF1_VF0_PROG_INTERFACE
+#define BIF_CFG_DEV0_EPF1_VF0_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PROG_INTERFACE__PROG_INTERFACE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_SUB_CLASS
+#define BIF_CFG_DEV0_EPF1_VF0_SUB_CLASS__SUB_CLASS__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_SUB_CLASS__SUB_CLASS_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_BASE_CLASS
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_CLASS__BASE_CLASS__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_CLASS__BASE_CLASS_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_CACHE_LINE
+#define BIF_CFG_DEV0_EPF1_VF0_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                              0x0
+#define BIF_CFG_DEV0_EPF1_VF0_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_LATENCY
+#define BIF_CFG_DEV0_EPF1_VF0_LATENCY__LATENCY_TIMER__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_LATENCY__LATENCY_TIMER_MASK                                                     0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_HEADER
+#define BIF_CFG_DEV0_EPF1_VF0_HEADER__HEADER_TYPE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_VF0_HEADER__DEVICE_TYPE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_EPF1_VF0_HEADER__HEADER_TYPE_MASK                                                        0x7FL
+#define BIF_CFG_DEV0_EPF1_VF0_HEADER__DEVICE_TYPE_MASK                                                        0x80L
+//BIF_CFG_DEV0_EPF1_VF0_BIST
+#define BIF_CFG_DEV0_EPF1_VF0_BIST__BIST_COMP__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_EPF1_VF0_BIST__BIST_STRT__SHIFT                                                          0x6
+#define BIF_CFG_DEV0_EPF1_VF0_BIST__BIST_CAP__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_EPF1_VF0_BIST__BIST_COMP_MASK                                                            0x0FL
+#define BIF_CFG_DEV0_EPF1_VF0_BIST__BIST_STRT_MASK                                                            0x40L
+#define BIF_CFG_DEV0_EPF1_VF0_BIST__BIST_CAP_MASK                                                             0x80L
+//BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_1
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_1__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_1__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_2
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_2__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_2__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_3
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_3__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_3__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_4
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_4__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_4__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_5
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_5__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_5__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_6
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_6__BASE_ADDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_BASE_ADDR_6__BASE_ADDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_CARDBUS_CIS_PTR
+#define BIF_CFG_DEV0_EPF1_VF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_VF0_CARDBUS_CIS_PTR__CARDBUS_CIS_PTR_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_ADAPTER_ID
+#define BIF_CFG_DEV0_EPF1_VF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_VF0_ADAPTER_ID__SUBSYSTEM_ID__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_EPF1_VF0_ADAPTER_ID__SUBSYSTEM_VENDOR_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_VF0_ADAPTER_ID__SUBSYSTEM_ID_MASK                                                   0xFFFF0000L
+//BIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                     0x1
+#define BIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                    0x4
+#define BIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                  0x00000001L
+#define BIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                       0x0000000EL
+#define BIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                      0x000000F0L
+#define BIF_CFG_DEV0_EPF1_VF0_ROM_BASE_ADDR__BASE_ADDR_MASK                                                   0xFFFFF800L
+//BIF_CFG_DEV0_EPF1_VF0_CAP_PTR
+#define BIF_CFG_DEV0_EPF1_VF0_CAP_PTR__CAP_PTR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_EPF1_VF0_CAP_PTR__CAP_PTR_MASK                                                           0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_INTERRUPT_LINE
+#define BIF_CFG_DEV0_EPF1_VF0_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_VF0_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                             0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_INTERRUPT_PIN
+#define BIF_CFG_DEV0_EPF1_VF0_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_VF0_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                               0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_MIN_GRANT
+#define BIF_CFG_DEV0_EPF1_VF0_MIN_GRANT__MIN_GNT__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MIN_GRANT__MIN_GNT_MASK                                                         0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_MAX_LATENCY
+#define BIF_CFG_DEV0_EPF1_VF0_MAX_LATENCY__MAX_LAT__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MAX_LATENCY__MAX_LAT_MASK                                                       0xFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__VERSION__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__DEVICE_TYPE__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                               0x8
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                0x9
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                              0xf
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__VERSION_MASK                                                          0x000FL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__DEVICE_TYPE_MASK                                                      0x00F0L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                  0x3E00L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                0x8000L
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                 0x5
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                        0x9
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                     0xf
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                 0x11
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                    0x12
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                    0x1a
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                  0x1c
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                          0x1d
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__TEE_IO_SUPPORT__SHIFT                                               0x1e
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                            0x00000007L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__PHANTOM_FUNC_MASK                                                   0x00000018L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__EXTENDED_TAG_MASK                                                   0x00000020L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                         0x000001C0L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                          0x00000E00L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                       0x00008000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__RX_MPS_FIXED_MASK                                                   0x00020000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                      0x03FC0000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                      0x0C000000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__FLR_CAPABLE_MASK                                                    0x10000000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                            0x20000000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP__TEE_IO_SUPPORT_MASK                                                 0x40000000L
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                            0x1
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                0x2
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                               0x3
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                              0x4
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                            0x5
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                             0x8
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                             0x9
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                             0xa
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                 0xb
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__INITIATE_FLR__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__CORR_ERR_EN_MASK                                                   0x0001L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                              0x0002L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                  0x0004L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__USR_REPORT_EN_MASK                                                 0x0008L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                0x0010L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                              0x00E0L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                               0x0100L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                               0x0200L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                               0x0400L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                   0x0800L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                         0x7000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL__INITIATE_FLR_MASK                                                  0x8000L
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__CORR_ERR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                             0x1
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__FATAL_ERR__SHIFT                                                 0x2
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__USR_DETECTED__SHIFT                                              0x3
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__AUX_PWR__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                         0x5
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                             0x6
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__CORR_ERR_MASK                                                    0x0001L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__NON_FATAL_ERR_MASK                                               0x0002L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__FATAL_ERR_MASK                                                   0x0004L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__USR_DETECTED_MASK                                                0x0008L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__AUX_PWR_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                           0x0020L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                               0x0040L
+//BIF_CFG_DEV0_EPF1_VF0_LINK_CAP
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__LINK_SPEED__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__LINK_WIDTH__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__PM_SUPPORT__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                               0xc
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                0xf
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                         0x12
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                    0x13
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                       0x15
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__PORT_NUMBER__SHIFT                                                    0x18
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__LINK_SPEED_MASK                                                       0x0000000FL
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__LINK_WIDTH_MASK                                                       0x000003F0L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__PM_SUPPORT_MASK                                                       0x00000C00L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                 0x00007000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__L1_EXIT_LATENCY_MASK                                                  0x00038000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                      0x00100000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP__PORT_NUMBER_MASK                                                      0xFF000000L
+//BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__PM_CONTROL__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                  0x2
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                             0x3
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__LINK_DIS__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__RETRAIN_LINK__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                              0x6
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                 0x7
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                   0x9
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                     0xa
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                     0xb
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__SRIS_CLOCKING__SHIFT                                                 0xc
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__FLIT_MODE_DISABLE__SHIFT                                             0xd
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                         0xe
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__PM_CONTROL_MASK                                                      0x0003L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                    0x0004L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                               0x0008L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__LINK_DIS_MASK                                                        0x0010L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__RETRAIN_LINK_MASK                                                    0x0020L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                0x0040L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__EXTENDED_SYNC_MASK                                                   0x0080L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                       0x0100L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                     0x0200L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                       0x0400L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                       0x0800L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__SRIS_CLOCKING_MASK                                                   0x1000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__FLIT_MODE_DISABLE_MASK                                               0x2000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                           0xC000L
+//BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__LINK_TRAINING__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                              0xc
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__DL_ACTIVE__SHIFT                                                   0xd
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                         0x03F0L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__LINK_TRAINING_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                0x1000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__DL_ACTIVE_MASK                                                     0x2000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                     0x4000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                     0x8000L
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                  0x6
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                  0x7
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                   0xa
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                          0xc
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                               0xe
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                             0x11
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                              0x12
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                0x15
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                    0x16
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                              0x18
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                               0x1a
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                               0x1f
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                   0x0000000FL
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                    0x00000040L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                    0x00000080L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                    0x00000100L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                     0x00000400L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                 0x00000800L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                            0x00003000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                 0x0000C000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                               0x00020000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                0x000C0000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                  0x00200000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                      0x00C00000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                0x03000000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                 0x04000000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                 0x80000000L
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                            0x4
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                          0x5
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                   0x7
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                         0x8
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__LTR_EN__SHIFT                                                     0xa
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                               0xb
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__OBFF_EN__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                            0x000FL
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                              0x0010L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                            0x0020L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                     0x0080L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                           0x0100L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                        0x0200L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__LTR_EN_MASK                                                       0x0400L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                 0x0800L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                 0x1000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__OBFF_EN_MASK                                                      0x6000L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                  0x8000L
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS2
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS2__RESERVED__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS2__RESERVED_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                           0x8
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                      0x9
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                      0x10
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                     0x17
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                     0x18
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                 0x1f
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                            0x000000FEL
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                        0x0000FE00L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                        0x007F0000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                       0x00800000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                       0x01000000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CAP2__DRS_SUPPORTED_MASK                                                   0x80000000L
+//BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                             0x4
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                  0x5
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                        0x6
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                         0xa
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                               0xb
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                        0xc
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                              0x000FL
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                               0x0010L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                    0x0020L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                          0x0040L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__XMIT_MARGIN_MASK                                                    0x0380L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                           0x0400L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                 0x0800L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                          0xF000L
+//BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                  0x1
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                            0x2
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                            0x3
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                            0x4
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                              0x5
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                          0x6
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                          0x7
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                       0x8
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                           0xa
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                              0xc
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                       0xf
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                         0x0001L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                    0x0002L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                              0x0004L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                              0x0008L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                              0x0010L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                0x0020L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                            0x0040L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                            0x0080L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                         0x0300L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                             0x0400L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                0x7000L
+#define BIF_CFG_DEV0_EPF1_VF0_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                         0x8000L
+//BIF_CFG_DEV0_EPF1_VF0_MSI_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_CAP_LIST__CAP_ID__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_CAP_LIST__CAP_ID_MASK                                                       0x00FFL
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_CAP_LIST__NEXT_PTR_MASK                                                     0xFF00L
+//BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_EN__SHIFT                                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                              0x1
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                  0x7
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                  0x8
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                       0x9
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                        0xa
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_EN_MASK                                                       0x0001L
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                0x000EL
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                 0x0070L
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_64BIT_MASK                                                    0x0080L
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                    0x0100L
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                         0x0200L
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                          0x0400L
+//BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                         0x2
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                           0xFFFFFFFCL
+//BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA__MSI_DATA__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA__MSI_DATA_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                             0xFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_MSI_MASK
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MASK__MSI_MASK__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MASK__MSI_MASK_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                             0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                               0xFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                       0xFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_MSI_MASK_64
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MASK_64__MSI_MASK_64__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_MASK_64__MSI_MASK_64_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_MSI_PENDING
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_PENDING__MSI_PENDING__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_PENDING__MSI_PENDING_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_MSI_PENDING_64
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_PENDING_64__MSI_PENDING_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSI_PENDING_64__MSI_PENDING_64_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_MSIX_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_CAP_LIST__CAP_ID__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_CAP_LIST__NEXT_PTR__SHIFT                                                  0x8
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_CAP_LIST__CAP_ID_MASK                                                      0x00FFL
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_CAP_LIST__NEXT_PTR_MASK                                                    0xFF00L
+//BIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK__SHIFT                                            0xe
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL__MSIX_EN__SHIFT                                                   0xf
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL__MSIX_TABLE_SIZE_MASK                                             0x07FFL
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL__MSIX_FUNC_MASK_MASK                                              0x4000L
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_MSG_CNTL__MSIX_EN_MASK                                                     0x8000L
+//BIF_CFG_DEV0_EPF1_VF0_MSIX_TABLE
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_TABLE__MSIX_TABLE_BIR__SHIFT                                               0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_TABLE__MSIX_TABLE_OFFSET__SHIFT                                            0x3
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_TABLE__MSIX_TABLE_BIR_MASK                                                 0x00000007L
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_TABLE__MSIX_TABLE_OFFSET_MASK                                              0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_VF0_MSIX_PBA
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_PBA__MSIX_PBA_BIR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_PBA__MSIX_PBA_OFFSET__SHIFT                                                0x3
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_PBA__MSIX_PBA_BIR_MASK                                                     0x00000007L
+#define BIF_CFG_DEV0_EPF1_VF0_MSIX_PBA__MSIX_PBA_OFFSET_MASK                                                  0xFFFFFFF8L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                               0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                              0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                  0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                 0x000F0000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                0xFFF00000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                    0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                           0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                             0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER__SHIFT                                       0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR__SHIFT                                      0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST__CAP_ID_MASK                                          0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST__CAP_VER_MASK                                         0x000F0000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_DEVICE3_ENH_CAP_LIST__NEXT_PTR_MASK                                        0xFFF00000L
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED__SHIFT                              0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED__SHIFT                   0x1
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED__SHIFT                   0x2
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED__SHIFT                                      0x3
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__DMWR_REQUEST_ROUTING_SUPPORTED_MASK                                0x00000001L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_COMPLETER_SUPPORTED_MASK                     0x00000002L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__DEVICE_CAP3_14BIT_TAG_REQUESTER_SUPPORTED_MASK                     0x00000004L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__RECEIVER_L0P_SUPPORTED_MASK                                        0x00000008L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__PORT_L0P_EXIT_LATENCY_MASK                                         0x00000070L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CAP3__RETIMER_L0P_EXIT_LATENCY_MASK                                      0x00000380L
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__DMWR_REQUEST_ENABLE__SHIFT                                        0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING__SHIFT                                       0x1
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT                    0x2
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__L0P_ENABLE__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__TARGET_LINK_WIDTH__SHIFT                                          0x4
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__DMWR_REQUEST_ENABLE_MASK                                          0x00000001L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__DMWR_EGRESS_BLOCKING_MASK                                         0x00000002L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK                      0x00000004L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__L0P_ENABLE_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_CNTL3__TARGET_LINK_WIDTH_MASK                                            0x00000070L
+//BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3__INITIAL_LINK_WIDTH__SHIFT                                       0x0
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3__SEGMENT_CAPTURED__SHIFT                                         0x3
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED__SHIFT                                     0x4
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3__INITIAL_LINK_WIDTH_MASK                                         0x00000007L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3__SEGMENT_CAPTURED_MASK                                           0x00000008L
+#define BIF_CFG_DEV0_EPF1_VF0_DEVICE_STATUS3__REMOTE_L0P_SUPPORTED_MASK                                       0x00000010L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                    0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                   0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                  0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                      0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                     0x000F0000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                    0xFFF00000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                   0x4
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                0x5
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                   0xc
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                    0xd
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                               0xe
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                             0xf
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                 0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                  0x11
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                   0x12
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                  0x13
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                            0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                             0x15
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                            0x16
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                            0x17
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                   0x18
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                    0x19
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT               0x1a
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT               0x1b
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                          0x1c
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                         0x1d
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                         0x1e
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                     0x00000010L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                  0x00000020L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                     0x00001000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                      0x00002000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                 0x00004000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                               0x00008000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                   0x00010000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                    0x00020000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                     0x00040000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                    0x00080000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                              0x00100000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                               0x00200000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                              0x00400000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                              0x00800000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                     0x01000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                      0x02000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                 0x04000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                 0x08000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                            0x10000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                           0x20000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                           0x40000000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                       0x4
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                    0x5
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                       0xc
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                        0xd
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                   0xe
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                 0xf
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                     0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                      0x11
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                       0x12
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                      0x13
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                 0x15
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                0x16
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                0x17
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                       0x18
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                        0x19
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                   0x1a
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                   0x1b
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                              0x1c
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                             0x1d
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                             0x1e
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                         0x00000010L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                      0x00000020L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                         0x00001000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                          0x00002000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                     0x00004000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                   0x00008000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                       0x00010000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                        0x00020000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                         0x00040000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                        0x00080000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                  0x00100000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                   0x00200000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                  0x00400000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                         0x01000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                          0x02000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                     0x04000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                     0x08000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                0x10000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                               0x20000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                               0x40000000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                               0x4
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                            0x5
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                               0xc
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                0xd
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                           0xe
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                         0xf
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                             0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                              0x11
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                               0x12
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                              0x13
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                        0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                         0x15
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                        0x16
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                        0x17
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT               0x18
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                0x19
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1a
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT           0x1b
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                      0x1c
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                     0x1d
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                     0x1e
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                              0x00000020L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                 0x00001000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                  0x00002000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                             0x00004000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                           0x00008000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                               0x00010000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                0x00020000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                 0x00040000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                0x00080000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                          0x00100000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                           0x00200000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                          0x00400000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                          0x00800000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                 0x01000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                  0x02000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK             0x04000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK             0x08000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                        0x10000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                       0x20000000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                       0x40000000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                     0x6
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                         0x8
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                        0xc
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                       0xd
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                0xe
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                0xf
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                           0x00000100L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                          0x00001000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                         0x00002000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                  0x00004000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                  0x00008000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                         0x6
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                        0x7
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                             0x8
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                            0xc
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                           0xd
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                    0xe
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                    0xf
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                           0x00000001L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                           0x00000040L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                               0x00000100L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                              0x00001000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                             0x00002000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                      0x00004000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                      0x00008000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                     0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                      0x5
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                       0x6
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                    0x7
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                     0x8
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                0x9
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                 0xa
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                            0xb
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                    0xc
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                   0xd
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                          0x12
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                   0x13
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                       0x0000001FL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                         0x00000040L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                      0x00000080L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                  0x00000200L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                   0x00000400L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                              0x00000800L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                      0x00001000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                     0x0003E000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                            0x00040000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                     0x00F80000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG0__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG1__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG2__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG3__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG4__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG5__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG6__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG7__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                         0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG8__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG9__TLP_HDR_MASK                                                     0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG10__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG11__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG12__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_HDR_LOG13__TLP_HDR_MASK                                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF1_VF0_RTR_DATA1
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA1__RESET_TIME__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA1__DLUP_TIME__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA1__RTR_VALID__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA1__RESET_TIME_MASK                                                      0x00000FFFL
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA1__DLUP_TIME_MASK                                                       0x00FFF000L
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA1__RTR_VALID_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_EPF1_VF0_RTR_DATA2
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA2__FLR_TIME__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA2__FLR_TIME_MASK                                                        0x00000FFFL
+#define BIF_CFG_DEV0_EPF1_VF0_RTR_DATA2__D3HOTD0_TIME_MASK                                                    0x00FFF000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM__SHIFT                                          0x8
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP__ARI_MFVC_FUNC_GROUPS_CAP_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP__ARI_ACS_FUNC_GROUPS_CAP_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CAP__ARI_NEXT_FUNC_NUM_MASK                                            0xFF00L
+//BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN__SHIFT                                   0x0
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN__SHIFT                                    0x1
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP__SHIFT                                        0x4
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL__ARI_MFVC_FUNC_GROUPS_EN_MASK                                     0x0001L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL__ARI_ACS_FUNC_GROUPS_EN_MASK                                      0x0002L
+#define BIF_CFG_DEV0_EPF1_VF0_PCIE_ARI_CNTL__ARI_FUNCTION_GROUP_MASK                                          0x0070L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_RCCPORTDEC
+//RCC_DEV0_1_RCC_VDM_SUPPORT
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_1_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_1_RCC_BUS_CNTL
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_1_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_1_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_1_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_1_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_1_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_1_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_1_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_1_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_1_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG__SHIFT                                            0x10
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+#define RCC_DEV0_1_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG_MASK                                              0x00FF0000L
+//RCC_DEV0_1_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_1_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_1_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_1_RCC_MH_ARB_CNTL
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_1_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x0000FFFEL
+//RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_1_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+
+
+// addressBlock: nbif0_nbif0_rcc_ep_dev0_RCCPORTDEC
+//RCC_EP_DEV0_1_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_1_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_1_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_1_EP_PCIE_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_1_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_1_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_1_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_1_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_1_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_1_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_1_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                  0x5
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+#define RCC_EP_DEV0_1_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                    0x00000020L
+//RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_1_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_1_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_1_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_1_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_1_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_1_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_1_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_1_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_1_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_1_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_1_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_1_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                          0x4
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+#define RCC_EP_DEV0_1_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                            0x00000010L
+//RCC_EP_DEV0_1_EP_PCIE_DEVICE_CNTL3
+#define RCC_EP_DEV0_1_EP_PCIE_DEVICE_CNTL3__SHADOW_F0_DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT          0x2
+#define RCC_EP_DEV0_1_EP_PCIE_DEVICE_CNTL3__SHADOW_F0_DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK            0x00000004L
+
+
+// addressBlock: nbif0_nbif0_rcc_dwn_dev0_RCCPORTDEC
+//RCC_DWN_DEV0_1_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_1_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_1_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_1_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_1_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_1_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_1_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_1_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_1_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                 0x5
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+#define RCC_DWN_DEV0_1_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                   0x00000020L
+//RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_1_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: nbif0_nbif0_rcc_dwnp_dev0_RCCPORTDEC
+//RCC_DWNP_DEV0_1_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_1_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_1_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_1_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                           0x4
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+#define RCC_DWNP_DEV0_1_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                             0x00000010L
+//RCC_DWNP_DEV0_1_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_1_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_1_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_1_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_1_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_1_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_pfc_amdgfx_RCCPFCDEC
+//RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE__SHIFT                                           0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE__SHIFT                                           0xa
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT__SHIFT                                             0xf
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE__SHIFT                                        0x10
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE__SHIFT                                        0x1a
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT__SHIFT                                          0x1f
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_VALUE_MASK                                             0x000003FFL
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_LATENCY_SCALE_MASK                                             0x00001C00L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__SNOOP_REQUIREMENT_MASK                                               0x00008000L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_VALUE_MASK                                          0x03FF0000L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_LATENCY_SCALE_MASK                                          0x1C000000L
+#define RCC_PFC_AMDGFX_RCC_PFC_LTR_CNTL__NONSNOOP_REQUIREMENT_MASK                                            0x80000000L
+//RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN__SHIFT                                         0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS__SHIFT                                     0x8
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_SENT_FLAG__SHIFT                                              0x9
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_EN_MASK                                           0x00000001L
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_RESTORE_PME_STATUS_MASK                                       0x00000100L
+#define RCC_PFC_AMDGFX_RCC_PFC_PME_RESTORE__PME_SENT_FLAG_MASK                                                0x00000200L
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS__SHIFT                                0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS__SHIFT                            0x1
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS__SHIFT                          0x2
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS__SHIFT                              0x3
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS__SHIFT                                0x4
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS__SHIFT                               0x5
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS__SHIFT                         0x6
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS__SHIFT                  0x7
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_PSN_ERR_STATUS_MASK                                  0x00000001L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_TIMEOUT_STATUS_MASK                              0x00000002L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_CPL_ABORT_ERR_STATUS_MASK                            0x00000004L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNEXP_CPL_STATUS_MASK                                0x00000008L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_MAL_TLP_STATUS_MASK                                  0x00000010L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ECRC_ERR_STATUS_MASK                                 0x00000020L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_UNSUPP_REQ_ERR_STATUS_MASK                           0x00000040L
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_0__RESTORE_ADVISORY_NONFATAL_ERR_STATUS_MASK                    0x00000080L
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_1__RESTORE_TLP_HDR_0_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_2__RESTORE_TLP_HDR_1_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_3__RESTORE_TLP_HDR_2_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3__SHIFT                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_4__RESTORE_TLP_HDR_3_MASK                                       0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX__SHIFT                                    0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_STICKY_RESTORE_5__RESTORE_TLP_PREFIX_MASK                                      0xFFFFFFFFL
+//RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE__SHIFT                                       0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE__SHIFT                                0x3
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_CURRENT_OVERRIDE_MASK                                         0x00000007L
+#define RCC_PFC_AMDGFX_RCC_PFC_AUXPWR_CNTL__AUX_POWER_DETECTED_OVERRIDE_MASK                                  0x00000008L
+//RCC_PFC_AMDGFX_RCC_PFC_MISC_CNTL
+#define RCC_PFC_AMDGFX_RCC_PFC_MISC_CNTL__BURST_EN__SHIFT                                                     0x0
+#define RCC_PFC_AMDGFX_RCC_PFC_MISC_CNTL__BURST_EN_MASK                                                       0x00000007L
+//RCC_PFC_AMDGFX_RCC_NFC
+#define RCC_PFC_AMDGFX_RCC_NFC__NFC_PFV__SHIFT                                                                0x0
+#define RCC_PFC_AMDGFX_RCC_NFC__NFC_PFV_MASK                                                                  0x00000001L
+
+
+// addressBlock: nbif0_nbif0_rcc_shadow_reg_shadowdec
+//SHADOW_COMMAND
+#define SHADOW_COMMAND__IOEN_UP__SHIFT                                                                        0x0
+#define SHADOW_COMMAND__MEMEN_UP__SHIFT                                                                       0x1
+#define SHADOW_COMMAND__IOEN_UP_MASK                                                                          0x0001L
+#define SHADOW_COMMAND__MEMEN_UP_MASK                                                                         0x0002L
+//SHADOW_BASE_ADDR_1
+#define SHADOW_BASE_ADDR_1__BAR1_UP__SHIFT                                                                    0x0
+#define SHADOW_BASE_ADDR_1__BAR1_UP_MASK                                                                      0xFFFFFFFFL
+//SHADOW_BASE_ADDR_2
+#define SHADOW_BASE_ADDR_2__BAR2_UP__SHIFT                                                                    0x0
+#define SHADOW_BASE_ADDR_2__BAR2_UP_MASK                                                                      0xFFFFFFFFL
+//SHADOW_SUB_BUS_NUMBER_LATENCY
+#define SHADOW_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_UP__SHIFT                                                0x8
+#define SHADOW_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_UP__SHIFT                                                  0x10
+#define SHADOW_SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_UP_MASK                                                  0x0000FF00L
+#define SHADOW_SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_UP_MASK                                                    0x00FF0000L
+//SHADOW_IO_BASE_LIMIT
+#define SHADOW_IO_BASE_LIMIT__IO_BASE_UP__SHIFT                                                               0x4
+#define SHADOW_IO_BASE_LIMIT__IO_LIMIT_UP__SHIFT                                                              0xc
+#define SHADOW_IO_BASE_LIMIT__IO_BASE_UP_MASK                                                                 0x00F0L
+#define SHADOW_IO_BASE_LIMIT__IO_LIMIT_UP_MASK                                                                0xF000L
+//SHADOW_MEM_BASE_LIMIT
+#define SHADOW_MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                           0x0
+#define SHADOW_MEM_BASE_LIMIT__MEM_BASE_31_20_UP__SHIFT                                                       0x4
+#define SHADOW_MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                          0x10
+#define SHADOW_MEM_BASE_LIMIT__MEM_LIMIT_31_20_UP__SHIFT                                                      0x14
+#define SHADOW_MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                             0x0000000FL
+#define SHADOW_MEM_BASE_LIMIT__MEM_BASE_31_20_UP_MASK                                                         0x0000FFF0L
+#define SHADOW_MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                            0x000F0000L
+#define SHADOW_MEM_BASE_LIMIT__MEM_LIMIT_31_20_UP_MASK                                                        0xFFF00000L
+//SHADOW_PREF_BASE_LIMIT
+#define SHADOW_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                                     0x0
+#define SHADOW_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_UP__SHIFT                                                 0x4
+#define SHADOW_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                                    0x10
+#define SHADOW_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_UP__SHIFT                                                0x14
+#define SHADOW_PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                                       0x0000000FL
+#define SHADOW_PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_UP_MASK                                                   0x0000FFF0L
+#define SHADOW_PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                                      0x000F0000L
+#define SHADOW_PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_UP_MASK                                                  0xFFF00000L
+//SHADOW_PREF_BASE_UPPER
+#define SHADOW_PREF_BASE_UPPER__PREF_BASE_UPPER_UP__SHIFT                                                     0x0
+#define SHADOW_PREF_BASE_UPPER__PREF_BASE_UPPER_UP_MASK                                                       0xFFFFFFFFL
+//SHADOW_PREF_LIMIT_UPPER
+#define SHADOW_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_UP__SHIFT                                                   0x0
+#define SHADOW_PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_UP_MASK                                                     0xFFFFFFFFL
+//SHADOW_IO_BASE_LIMIT_HI
+#define SHADOW_IO_BASE_LIMIT_HI__IO_BASE_31_16_UP__SHIFT                                                      0x0
+#define SHADOW_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_UP__SHIFT                                                     0x10
+#define SHADOW_IO_BASE_LIMIT_HI__IO_BASE_31_16_UP_MASK                                                        0x0000FFFFL
+#define SHADOW_IO_BASE_LIMIT_HI__IO_LIMIT_31_16_UP_MASK                                                       0xFFFF0000L
+//SHADOW_IRQ_BRIDGE_CNTL
+#define SHADOW_IRQ_BRIDGE_CNTL__ISA_EN_UP__SHIFT                                                              0x2
+#define SHADOW_IRQ_BRIDGE_CNTL__VGA_EN_UP__SHIFT                                                              0x3
+#define SHADOW_IRQ_BRIDGE_CNTL__VGA_DEC_UP__SHIFT                                                             0x4
+#define SHADOW_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_UP__SHIFT                                                 0x6
+#define SHADOW_IRQ_BRIDGE_CNTL__ISA_EN_UP_MASK                                                                0x0004L
+#define SHADOW_IRQ_BRIDGE_CNTL__VGA_EN_UP_MASK                                                                0x0008L
+#define SHADOW_IRQ_BRIDGE_CNTL__VGA_DEC_UP_MASK                                                               0x0010L
+#define SHADOW_IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_UP_MASK                                                   0x0040L
+//SUC_INDEX
+#define SUC_INDEX__SUC_INDEX__SHIFT                                                                           0x0
+#define SUC_INDEX__SUC_INDEX_MASK                                                                             0xFFFFFFFFL
+//SUC_DATA
+#define SUC_DATA__SUC_DATA__SHIFT                                                                             0x0
+#define SUC_DATA__SUC_DATA_MASK                                                                               0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_swus_SUMDEC
+//SUM_INDEX
+#define SUM_INDEX__SUM_INDEX__SHIFT                                                                           0x0
+#define SUM_INDEX__SUM_INDEX_MASK                                                                             0xFFFFFFFFL
+//SUM_DATA
+#define SUM_DATA__SUM_DATA__SHIFT                                                                             0x0
+#define SUM_DATA__SUM_DATA_MASK                                                                               0xFFFFFFFFL
+//SUM_INDEX_HI
+#define SUM_INDEX_HI__SUM_INDEX_HI__SHIFT                                                                     0x0
+#define SUM_INDEX_HI__SUM_INDEX_HI_MASK                                                                       0x0000FFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_strap_rcc_strap_internal
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0__SHIFT                                     0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0__SHIFT                                   0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0_MASK                                       0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0_MASK                                     0x00E00000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0__SHIFT                                         0x1e
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0_MASK                                           0x40000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0__SHIFT                                   0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0__SHIFT                              0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0__SHIFT                           0x1a
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0_MASK                                     0x00100000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0_MASK                                0x03000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0_MASK                             0x0C000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0__SHIFT                               0x1e
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                  0x1f
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0_MASK                                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                    0x80000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                     0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                     0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_RECEIVER_L0P_SUPPORTED_DEV0__SHIFT                            0x1a
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_PORT_L0P_EXIT_LATENCY_DEV0__SHIFT                             0x1b
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                       0x01000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                       0x02000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_RECEIVER_L0P_SUPPORTED_DEV0_MASK                              0x04000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP12__STRAP_PORT_L0P_EXIT_LATENCY_DEV0_MASK                               0x38000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP14
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0__SHIFT                               0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0__SHIFT                             0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0__SHIFT                                 0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0__SHIFT                          0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0__SHIFT                                         0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0__SHIFT                                      0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0__SHIFT                                   0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DEV0__SHIFT                                            0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RETIMER_L0P_EXIT_LATENCY_DEV0__SHIFT                          0x9
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_DEVICE3_EN_DN_DEV0__SHIFT                                     0xc
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DN_DEV0__SHIFT                                         0xd
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0_MASK                                 0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0_MASK                               0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0_MASK                                   0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0_MASK                            0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0_MASK                                           0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0_MASK                                        0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0_MASK                                     0x00000040L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DEV0_MASK                                              0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_RETIMER_L0P_EXIT_LATENCY_DEV0_MASK                            0x00000E00L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_DEVICE3_EN_DN_DEV0_MASK                                       0x00001000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DN_DEV0_MASK                                           0x00002000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP15
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP15__STRAP_DSN_CODE_L_DN_DEV0__SHIFT                                     0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP15__STRAP_DSN_CODE_L_DN_DEV0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP16
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP16__STRAP_DSN_CODE_H_DN_DEV0__SHIFT                                     0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP16__STRAP_DSN_CODE_H_DN_DEV0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP17
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_LINK_IDE_STREAM_SUPPORTED_DEV0__SHIFT                         0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_STREAM_SUPPORTED_DEV0__SHIFT                    0x1
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_DEV0__SHIFT                 0x2
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_AGGREGATION_SUPPORTED_DEV0__SHIFT                             0x4
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_PCRC_SUPPORTED_DEV0__SHIFT                                    0x5
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_IDE_KM_PROTOCOL_SUPPORTED_DEV0__SHIFT                         0x6
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_DEV0__SHIFT               0x7
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_DEV0__SHIFT                 0xd
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_DEV0__SHIFT            0x10
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_TEE_LIMITED_SUPPORTED_DEV0__SHIFT                             0x18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_XT_SUPPORTED_DEV0__SHIFT                                      0x19
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_LINK_IDE_STREAM_SUPPORTED_DEV0_MASK                           0x00000001L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_STREAM_SUPPORTED_DEV0_MASK                      0x00000002L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_DEV0_MASK                   0x00000004L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_AGGREGATION_SUPPORTED_DEV0_MASK                               0x00000010L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_PCRC_SUPPORTED_DEV0_MASK                                      0x00000020L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_IDE_KM_PROTOCOL_SUPPORTED_DEV0_MASK                           0x00000040L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_DEV0_MASK                 0x00000080L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_DEV0_MASK                   0x0000E000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_DEV0_MASK              0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_TEE_LIMITED_SUPPORTED_DEV0_MASK                               0x01000000L
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP17__STRAP_XT_SUPPORTED_DEV0_MASK                                        0x02000000L
+//RCC_STRAP1_RCC_DEV0_PORT_STRAP18
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP18__STRAP_NUM_ADDR_ASSOCIATION_REG_BLKS_DEV0__SHIFT                     0x0
+#define RCC_STRAP1_RCC_DEV0_PORT_STRAP18__STRAP_NUM_ADDR_ASSOCIATION_REG_BLKS_DEV0_MASK                       0x0000000FL
+//RCC_DEV1_PORT_STRAP0
+//RCC_DEV1_PORT_STRAP1
+//RCC_DEV1_PORT_STRAP2
+//RCC_DEV1_PORT_STRAP3
+//RCC_DEV1_PORT_STRAP4
+//RCC_DEV1_PORT_STRAP5
+//RCC_DEV1_PORT_STRAP6
+//RCC_DEV1_PORT_STRAP7
+//RCC_DEV1_PORT_STRAP8
+//RCC_DEV1_PORT_STRAP9
+//RCC_DEV1_PORT_STRAP10
+//RCC_DEV1_PORT_STRAP11
+//RCC_DEV1_PORT_STRAP12
+//RCC_DEV1_PORT_STRAP13
+//RCC_DEV1_PORT_STRAP14
+//RCC_DEV1_PORT_STRAP15
+//RCC_DEV1_PORT_STRAP16
+//RCC_DEV2_PORT_STRAP0
+//RCC_DEV2_PORT_STRAP1
+//RCC_DEV2_PORT_STRAP2
+//RCC_DEV2_PORT_STRAP3
+//RCC_DEV2_PORT_STRAP4
+//RCC_DEV2_PORT_STRAP5
+//RCC_DEV2_PORT_STRAP6
+//RCC_DEV2_PORT_STRAP7
+//RCC_DEV2_PORT_STRAP8
+//RCC_DEV2_PORT_STRAP9
+//RCC_DEV2_PORT_STRAP10
+//RCC_DEV2_PORT_STRAP11
+//RCC_DEV2_PORT_STRAP12
+//RCC_DEV2_PORT_STRAP13
+//RCC_DEV2_PORT_STRAP14
+//RCC_DEV2_PORT_STRAP15
+//RCC_DEV2_PORT_STRAP16
+//RCC_STRAP1_RCC_BIF_STRAP0
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_AUD_PIN__SHIFT                                                       0x12
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x14
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN__SHIFT                         0x15
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GPUIOV_EN__SHIFT                                                     0x16
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE__SHIFT                                          0x17
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_AUD_PIN_MASK                                                         0x000C0000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00100000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN_MASK                           0x00200000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GPUIOV_EN_MASK                                                       0x00400000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE_MASK                                            0x00800000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP1_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP1
+#define RCC_STRAP1_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP1_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP1_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_FLIT_MODE_SUPPORT__SHIFT                                             0x4
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN__SHIFT                                                  0x8
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN__SHIFT                                                 0x9
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE__SHIFT                                                  0xa
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE__SHIFT                                        0xc
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_FLIT_MODE_SUPPORT_MASK                                               0x00000010L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN_MASK                                                    0x00000100L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN_MASK                                                   0x00000200L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE_MASK                                                    0x00000C00L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE_MASK                                          0x00001000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP1_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP2
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWUS_SPT__SHIFT                                                      0x1
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWDS_SPT__SHIFT                                                      0x2
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS__SHIFT                                            0x3
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS__SHIFT                                            0x4
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP1_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ARI_EN_UP__SHIFT                                                     0x19
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWUS_SPT_MASK                                                        0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWDS_SPT_MASK                                                        0x00000004L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS_MASK                                              0x00000008L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS_MASK                                              0x00000010L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP1_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_ARI_EN_UP_MASK                                                       0x02000000L
+#define RCC_STRAP1_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP3
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP1_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP1_RCC_BIF_STRAP4
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP1_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP1_RCC_BIF_STRAP5
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN__SHIFT                                 0x1f
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+#define RCC_STRAP1_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN_MASK                                   0x80000000L
+//RCC_STRAP1_RCC_BIF_STRAP6
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_DOE_VERSION_SEL__SHIFT                                               0x3
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE__SHIFT                                               0x5
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0xa
+#define RCC_STRAP1_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6__SHIFT                                                 0xf
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_DOE_VERSION_SEL_MASK                                                 0x00000008L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE_MASK                                                 0x00000020L
+#define RCC_STRAP1_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00007C00L
+#define RCC_STRAP1_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6_MASK                                                   0xFFFF8000L
+//RCC_STRAP1_RCC_BIF_STRAP7
+#define RCC_STRAP1_RCC_BIF_STRAP7__STRAP_GEN6_DIS__SHIFT                                                      0x19
+#define RCC_STRAP1_RCC_BIF_STRAP7__STRAP_BIF_KILL_GEN6__SHIFT                                                 0x1a
+#define RCC_STRAP1_RCC_BIF_STRAP7__STRAP_GEN6_DIS_MASK                                                        0x02000000L
+#define RCC_STRAP1_RCC_BIF_STRAP7__STRAP_BIF_KILL_GEN6_MASK                                                   0x04000000L
+//RCC_STRAP1_RCC_BIF_STRAP8
+//RCC_STRAP1_RCC_BIF_STRAP9
+//RCC_STRAP1_RCC_BIF_STRAP10
+//RCC_STRAP1_RCC_BIF_STRAP11
+//RCC_STRAP1_RCC_BIF_STRAP12
+//RCC_STRAP1_RCC_BIF_STRAP13
+#define RCC_STRAP1_RCC_BIF_STRAP13__STRAP_PCIE_SMN_APER__SHIFT                                                0x0
+#define RCC_STRAP1_RCC_BIF_STRAP13__STRAP_PCIE_SMN_APER_MASK                                                  0x00000FFFL
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0__SHIFT                      0x1
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0__SHIFT                     0x2
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DEVICE3_EN_DEV0_F0__SHIFT                                      0x3
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0__SHIFT                    0x4
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0_MASK                        0x00000002L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0_MASK                       0x00000004L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DEVICE3_EN_DEV0_F0_MASK                                        0x00000008L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0_MASK                      0x00000010L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0__SHIFT             0x19
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0_MASK               0x02000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0__SHIFT                                  0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0_MASK                                    0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0_MASK                                       0x38000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0__SHIFT                                0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0_MASK                                  0x38000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F0__SHIFT                                  0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F0_MASK                                    0x10000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP17
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0__SHIFT                                   0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0__SHIFT                                0xd
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0_MASK                                0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0_MASK                                     0x00001000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0_MASK                                  0x01FFE000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0__SHIFT                            0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0__SHIFT                            0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0_MASK                              0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0_MASK                              0x000FF000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP19
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F0__SHIFT                          0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F0_MASK                            0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F0_MASK                                  0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP22
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0__SHIFT                             0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_MEM_AP_SIZE_DEV0_F0__SHIFT                                    0xc
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_VF_MEM_AP_SIZE_DEV0_F0__SHIFT                                 0x11
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_PF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0__SHIFT                  0x16
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_VF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0__SHIFT                  0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0_MASK                               0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_MEM_AP_SIZE_DEV0_F0_MASK                                      0x0001F000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_VF_MEM_AP_SIZE_DEV0_F0_MASK                                   0x003E0000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_PF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0_MASK                    0x07C00000L
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP22__STRAP_VF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0_MASK                    0xF8000000L
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP23
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP23__STRAP_DSN_CODE_L_DEV0_F0__SHIFT                                     0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP23__STRAP_DSN_CODE_L_DEV0_F0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP1_RCC_DEV0_EPF0_STRAP24
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP24__STRAP_DSN_CODE_H_DEV0_F0__SHIFT                                     0x0
+#define RCC_STRAP1_RCC_DEV0_EPF0_STRAP24__STRAP_DSN_CODE_H_DEV0_F0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP1
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F1__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F1__SHIFT                       0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F1_MASK                                0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F1_MASK                         0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_SRIOV_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F1__SHIFT                     0x2
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DEVICE3_EN_DEV0_F1__SHIFT                                      0x3
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1__SHIFT                    0x4
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_SRIOV_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F1_MASK                       0x00000004L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DEVICE3_EN_DEV0_F1_MASK                                        0x00000008L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1_MASK                      0x00000010L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F1_MASK                                       0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1__SHIFT             0x19
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F1__SHIFT                                0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1_MASK               0x02000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F1_MASK                                  0x10000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x1
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1__SHIFT                                  0x2
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1__SHIFT                                        0x8
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x9
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1__SHIFT                                  0xa
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1__SHIFT                                        0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x11
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1__SHIFT                                  0x12
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1__SHIFT                                        0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x19
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1__SHIFT                                        0x1a
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV0_F1_MASK                             0x00000002L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1_MASK                                    0x00000004L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1_MASK                                          0x00000100L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV0_F1_MASK                             0x00000200L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1_MASK                                    0x00000400L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1_MASK                                          0x00010000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_PREFETCHABLE_EN_DEV0_F1_MASK                             0x00020000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1_MASK                                    0x00040000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1_MASK                                          0x01000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_PREFETCHABLE_EN_DEV0_F1_MASK                             0x02000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1_MASK                                          0x04000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_PREFETCHABLE_EN_DEV0_F1_MASK                             0x08000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP7
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP8
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F1__SHIFT                                0x1b
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F1__SHIFT                           0x1e
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F1_MASK                                  0x38000000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F1_MASK                             0xC0000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP9
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F1__SHIFT                           0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F1__SHIFT                                  0x1c
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F1_MASK                             0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F1_MASK                                    0x10000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP10
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP11
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP12
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP13
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F1__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F1__SHIFT                                 0x8
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F1__SHIFT                                0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F1__SHIFT                                0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F1_MASK                                   0x000000FFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F1_MASK                                   0x0000FF00L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F1_MASK                                  0x00FF0000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F1_MASK                                  0xFF000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP14
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP14__STRAP_VENDOR_ID_DEV0_F1__SHIFT                                      0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP14__STRAP_VENDOR_ID_DEV0_F1_MASK                                        0x0000FFFFL
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP15
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F1__SHIFT                                 0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F1__SHIFT                                  0xc
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_VALID_DEV0_F1__SHIFT                                      0x18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F1_MASK                                   0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F1_MASK                                    0x00FFF000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_VALID_DEV0_F1_MASK                                        0x01000000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP16
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F1__SHIFT                               0xc
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F1_MASK                                     0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F1_MASK                                 0x00FFF000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP17
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F1__SHIFT                              0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_VALID_DEV0_F1__SHIFT                                   0xc
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F1__SHIFT                                0xd
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F1_MASK                                0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_VALID_DEV0_F1_MASK                                     0x00001000L
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F1_MASK                                  0x01FFE000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP18
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F1__SHIFT                            0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F1__SHIFT                            0xc
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F1_MASK                              0x00000FFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F1_MASK                              0x000FF000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP19
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F1__SHIFT                          0x0
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F1__SHIFT                                0x10
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F1_MASK                            0x0000FFFFL
+#define RCC_STRAP1_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F1_MASK                                  0xFFFF0000L
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP20
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP21
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP23
+//RCC_STRAP1_RCC_DEV0_EPF1_STRAP24
+//RCC_DEV0_EPF2_STRAP0
+//RCC_DEV0_EPF2_STRAP2
+//RCC_DEV0_EPF2_STRAP3
+//RCC_DEV0_EPF2_STRAP4
+//RCC_DEV0_EPF2_STRAP5
+//RCC_DEV0_EPF2_STRAP6
+//RCC_DEV0_EPF2_STRAP7
+//RCC_DEV0_EPF2_STRAP9
+//RCC_DEV0_EPF2_STRAP10
+//RCC_DEV0_EPF2_STRAP11
+//RCC_DEV0_EPF2_STRAP12
+//RCC_DEV0_EPF2_STRAP13
+//RCC_DEV0_EPF2_STRAP14
+//RCC_DEV0_EPF2_STRAP15
+//RCC_DEV0_EPF2_STRAP16
+//RCC_DEV0_EPF2_STRAP20
+//RCC_DEV0_EPF3_STRAP0
+//RCC_DEV0_EPF3_STRAP2
+//RCC_DEV0_EPF3_STRAP3
+//RCC_DEV0_EPF3_STRAP4
+//RCC_DEV0_EPF3_STRAP5
+//RCC_DEV0_EPF3_STRAP6
+//RCC_DEV0_EPF3_STRAP7
+//RCC_DEV0_EPF3_STRAP9
+//RCC_DEV0_EPF3_STRAP10
+//RCC_DEV0_EPF3_STRAP11
+//RCC_DEV0_EPF3_STRAP12
+//RCC_DEV0_EPF3_STRAP13
+//RCC_DEV0_EPF3_STRAP14
+//RCC_DEV0_EPF3_STRAP15
+//RCC_DEV0_EPF3_STRAP16
+//RCC_DEV0_EPF3_STRAP20
+//RCC_DEV0_EPF4_STRAP0
+//RCC_DEV0_EPF4_STRAP2
+//RCC_DEV0_EPF4_STRAP3
+//RCC_DEV0_EPF4_STRAP4
+//RCC_DEV0_EPF4_STRAP5
+//RCC_DEV0_EPF4_STRAP6
+//RCC_DEV0_EPF4_STRAP7
+//RCC_DEV0_EPF4_STRAP9
+//RCC_DEV0_EPF4_STRAP13
+//RCC_DEV0_EPF4_STRAP14
+//RCC_DEV0_EPF4_STRAP15
+//RCC_DEV0_EPF4_STRAP16
+//RCC_DEV0_EPF5_STRAP0
+//RCC_DEV0_EPF5_STRAP2
+//RCC_DEV0_EPF5_STRAP3
+//RCC_DEV0_EPF5_STRAP4
+//RCC_DEV0_EPF5_STRAP5
+//RCC_DEV0_EPF5_STRAP6
+//RCC_DEV0_EPF5_STRAP7
+//RCC_DEV0_EPF5_STRAP9
+//RCC_DEV0_EPF5_STRAP13
+//RCC_DEV0_EPF5_STRAP14
+//RCC_DEV0_EPF5_STRAP15
+//RCC_DEV0_EPF5_STRAP16
+//RCC_DEV0_EPF6_STRAP0
+//RCC_DEV0_EPF6_STRAP2
+//RCC_DEV0_EPF6_STRAP3
+//RCC_DEV0_EPF6_STRAP4
+//RCC_DEV0_EPF6_STRAP5
+//RCC_DEV0_EPF6_STRAP6
+//RCC_DEV0_EPF6_STRAP7
+//RCC_DEV0_EPF6_STRAP9
+//RCC_DEV0_EPF6_STRAP13
+//RCC_DEV0_EPF6_STRAP14
+//RCC_DEV0_EPF6_STRAP15
+//RCC_DEV0_EPF6_STRAP16
+//RCC_DEV0_EPF7_STRAP0
+//RCC_DEV0_EPF7_STRAP2
+//RCC_DEV0_EPF7_STRAP3
+//RCC_DEV0_EPF7_STRAP4
+//RCC_DEV0_EPF7_STRAP5
+//RCC_DEV0_EPF7_STRAP6
+//RCC_DEV0_EPF7_STRAP9
+//RCC_DEV0_EPF7_STRAP13
+//RCC_DEV0_EPF7_STRAP14
+//RCC_DEV0_EPF7_STRAP15
+//RCC_DEV0_EPF7_STRAP16
+//RCC_DEV1_EPF0_STRAP0
+//RCC_DEV1_EPF0_STRAP2
+//RCC_DEV1_EPF0_STRAP3
+//RCC_DEV1_EPF0_STRAP4
+//RCC_DEV1_EPF0_STRAP5
+//RCC_DEV1_EPF0_STRAP6
+//RCC_DEV1_EPF0_STRAP9
+//RCC_DEV1_EPF0_STRAP13
+//RCC_DEV1_EPF0_STRAP14
+//RCC_DEV1_EPF0_STRAP15
+//RCC_DEV1_EPF0_STRAP16
+//RCC_DEV1_EPF1_STRAP0
+//RCC_DEV1_EPF1_STRAP1
+//RCC_DEV1_EPF1_STRAP2
+//RCC_DEV1_EPF1_STRAP3
+//RCC_DEV1_EPF1_STRAP4
+//RCC_DEV1_EPF1_STRAP5
+//RCC_DEV1_EPF1_STRAP6
+//RCC_DEV1_EPF1_STRAP7
+//RCC_DEV1_EPF1_STRAP8
+//RCC_DEV1_EPF1_STRAP9
+//RCC_DEV1_EPF1_STRAP13
+//RCC_DEV1_EPF1_STRAP14
+//RCC_DEV1_EPF1_STRAP15
+//RCC_DEV1_EPF1_STRAP16
+//RCC_DEV1_EPF1_STRAP17
+//RCC_DEV1_EPF1_STRAP18
+//RCC_DEV1_EPF1_STRAP19
+//RCC_DEV1_EPF1_STRAP20
+//RCC_DEV1_EPF1_STRAP21
+//RCC_DEV1_EPF2_STRAP0
+//RCC_DEV1_EPF2_STRAP1
+//RCC_DEV1_EPF2_STRAP2
+//RCC_DEV1_EPF2_STRAP3
+//RCC_DEV1_EPF2_STRAP4
+//RCC_DEV1_EPF2_STRAP5
+//RCC_DEV1_EPF2_STRAP6
+//RCC_DEV1_EPF2_STRAP7
+//RCC_DEV1_EPF2_STRAP8
+//RCC_DEV1_EPF2_STRAP9
+//RCC_DEV1_EPF2_STRAP13
+//RCC_DEV1_EPF2_STRAP14
+//RCC_DEV1_EPF2_STRAP15
+//RCC_DEV1_EPF2_STRAP16
+//RCC_DEV1_EPF2_STRAP17
+//RCC_DEV1_EPF2_STRAP18
+//RCC_DEV1_EPF2_STRAP19
+//RCC_DEV1_EPF2_STRAP20
+//RCC_DEV1_EPF2_STRAP21
+//RCC_DEV1_EPF3_STRAP0
+//RCC_DEV1_EPF3_STRAP2
+//RCC_DEV1_EPF3_STRAP3
+//RCC_DEV1_EPF3_STRAP4
+//RCC_DEV1_EPF3_STRAP5
+//RCC_DEV1_EPF3_STRAP6
+//RCC_DEV1_EPF3_STRAP7
+//RCC_DEV1_EPF3_STRAP13
+//RCC_DEV1_EPF3_STRAP14
+//RCC_DEV1_EPF3_STRAP15
+//RCC_DEV1_EPF3_STRAP16
+//RCC_DEV1_EPF4_STRAP6
+//RCC_DEV1_EPF5_STRAP6
+//RCC_DEV1_EPF6_STRAP6
+//RCC_DEV2_EPF0_STRAP0
+//RCC_DEV2_EPF0_STRAP2
+//RCC_DEV2_EPF0_STRAP3
+//RCC_DEV2_EPF0_STRAP4
+//RCC_DEV2_EPF0_STRAP5
+//RCC_DEV2_EPF0_STRAP6
+//RCC_DEV2_EPF0_STRAP7
+//RCC_DEV2_EPF0_STRAP13
+//RCC_DEV2_EPF0_STRAP14
+//RCC_DEV2_EPF0_STRAP15
+//RCC_DEV2_EPF0_STRAP16
+//RCC_DEV2_EPF0_STRAP18
+//RCC_DEV2_EPF0_STRAP25
+//RCC_DEV2_EPF0_STRAP26
+//RCC_DEV2_EPF1_STRAP0
+//RCC_DEV2_EPF1_STRAP2
+//RCC_DEV2_EPF1_STRAP3
+//RCC_DEV2_EPF1_STRAP4
+//RCC_DEV2_EPF1_STRAP5
+//RCC_DEV2_EPF1_STRAP6
+//RCC_DEV2_EPF1_STRAP7
+//RCC_DEV2_EPF1_STRAP13
+//RCC_DEV2_EPF1_STRAP14
+//RCC_DEV2_EPF1_STRAP15
+//RCC_DEV2_EPF1_STRAP16
+//RCC_DEV2_EPF2_STRAP0
+//RCC_DEV2_EPF2_STRAP2
+//RCC_DEV2_EPF2_STRAP3
+//RCC_DEV2_EPF2_STRAP4
+//RCC_DEV2_EPF2_STRAP5
+//RCC_DEV2_EPF2_STRAP6
+//RCC_DEV2_EPF2_STRAP7
+//RCC_DEV2_EPF2_STRAP13
+//RCC_DEV2_EPF2_STRAP14
+//RCC_DEV2_EPF2_STRAP15
+//RCC_DEV2_EPF2_STRAP16
+//RCC_DEV2_EPF3_STRAP0
+//RCC_DEV2_EPF3_STRAP2
+//RCC_DEV2_EPF3_STRAP3
+//RCC_DEV2_EPF3_STRAP4
+//RCC_DEV2_EPF3_STRAP5
+//RCC_DEV2_EPF3_STRAP6
+//RCC_DEV2_EPF3_STRAP7
+//RCC_DEV2_EPF3_STRAP13
+//RCC_DEV2_EPF3_STRAP14
+//RCC_DEV2_EPF3_STRAP15
+//RCC_DEV2_EPF3_STRAP16
+//RCC_DEV2_EPF4_STRAP0
+//RCC_DEV2_EPF4_STRAP2
+//RCC_DEV2_EPF4_STRAP3
+//RCC_DEV2_EPF4_STRAP4
+//RCC_DEV2_EPF4_STRAP5
+//RCC_DEV2_EPF4_STRAP6
+//RCC_DEV2_EPF4_STRAP7
+//RCC_DEV2_EPF4_STRAP13
+//RCC_DEV2_EPF4_STRAP14
+//RCC_DEV2_EPF4_STRAP15
+//RCC_DEV2_EPF4_STRAP16
+//RCC_DEV2_EPF5_STRAP0
+//RCC_DEV2_EPF5_STRAP2
+//RCC_DEV2_EPF5_STRAP3
+//RCC_DEV2_EPF5_STRAP4
+//RCC_DEV2_EPF5_STRAP5
+//RCC_DEV2_EPF5_STRAP6
+//RCC_DEV2_EPF5_STRAP7
+//RCC_DEV2_EPF5_STRAP13
+//RCC_DEV2_EPF5_STRAP14
+//RCC_DEV2_EPF5_STRAP15
+//RCC_DEV2_EPF5_STRAP16
+//RCC_DEV2_EPF6_STRAP0
+//RCC_DEV2_EPF6_STRAP2
+//RCC_DEV2_EPF6_STRAP3
+//RCC_DEV2_EPF6_STRAP4
+//RCC_DEV2_EPF6_STRAP5
+//RCC_DEV2_EPF6_STRAP6
+//RCC_DEV2_EPF6_STRAP7
+//RCC_DEV2_EPF6_STRAP13
+//RCC_DEV2_EPF6_STRAP14
+//RCC_DEV2_EPF6_STRAP15
+//RCC_DEV2_EPF6_STRAP16
+
+
+// addressBlock: nbif0_nbif0_bif_rst_bif_rst_regblk
+//HARD_RST_CTRL
+#define HARD_RST_CTRL__DSPT_CFG_RST_EN__SHIFT                                                                 0x0
+#define HARD_RST_CTRL__DSPT_CFG_STICKY_RST_EN__SHIFT                                                          0x1
+#define HARD_RST_CTRL__DSPT_PRV_RST_EN__SHIFT                                                                 0x2
+#define HARD_RST_CTRL__DSPT_PRV_STICKY_RST_EN__SHIFT                                                          0x3
+#define HARD_RST_CTRL__EP_CFG_RST_EN__SHIFT                                                                   0x4
+#define HARD_RST_CTRL__EP_CFG_STICKY_RST_EN__SHIFT                                                            0x5
+#define HARD_RST_CTRL__EP_PRV_RST_EN__SHIFT                                                                   0x6
+#define HARD_RST_CTRL__EP_PRV_STICKY_RST_EN__SHIFT                                                            0x7
+#define HARD_RST_CTRL__SDP_PORT_RESET_EN__SHIFT                                                               0x9
+#define HARD_RST_CTRL__SION_AON_RESET_EN__SHIFT                                                               0xa
+#define HARD_RST_CTRL__STRAP_RST_EN__SHIFT                                                                    0x17
+#define HARD_RST_CTRL__SWUS_SHADOW_RST_EN__SHIFT                                                              0x1c
+#define HARD_RST_CTRL__CORE_STICKY_RST_EN__SHIFT                                                              0x1d
+#define HARD_RST_CTRL__RELOAD_STRAP_EN__SHIFT                                                                 0x1e
+#define HARD_RST_CTRL__CORE_RST_EN__SHIFT                                                                     0x1f
+#define HARD_RST_CTRL__DSPT_CFG_RST_EN_MASK                                                                   0x00000001L
+#define HARD_RST_CTRL__DSPT_CFG_STICKY_RST_EN_MASK                                                            0x00000002L
+#define HARD_RST_CTRL__DSPT_PRV_RST_EN_MASK                                                                   0x00000004L
+#define HARD_RST_CTRL__DSPT_PRV_STICKY_RST_EN_MASK                                                            0x00000008L
+#define HARD_RST_CTRL__EP_CFG_RST_EN_MASK                                                                     0x00000010L
+#define HARD_RST_CTRL__EP_CFG_STICKY_RST_EN_MASK                                                              0x00000020L
+#define HARD_RST_CTRL__EP_PRV_RST_EN_MASK                                                                     0x00000040L
+#define HARD_RST_CTRL__EP_PRV_STICKY_RST_EN_MASK                                                              0x00000080L
+#define HARD_RST_CTRL__SDP_PORT_RESET_EN_MASK                                                                 0x00000200L
+#define HARD_RST_CTRL__SION_AON_RESET_EN_MASK                                                                 0x00000400L
+#define HARD_RST_CTRL__STRAP_RST_EN_MASK                                                                      0x00800000L
+#define HARD_RST_CTRL__SWUS_SHADOW_RST_EN_MASK                                                                0x10000000L
+#define HARD_RST_CTRL__CORE_STICKY_RST_EN_MASK                                                                0x20000000L
+#define HARD_RST_CTRL__RELOAD_STRAP_EN_MASK                                                                   0x40000000L
+#define HARD_RST_CTRL__CORE_RST_EN_MASK                                                                       0x80000000L
+//SELF_SOFT_RST
+#define SELF_SOFT_RST__DSPT0_CFG_RST__SHIFT                                                                   0x0
+#define SELF_SOFT_RST__DSPT0_CFG_STICKY_RST__SHIFT                                                            0x1
+#define SELF_SOFT_RST__DSPT0_PRV_RST__SHIFT                                                                   0x2
+#define SELF_SOFT_RST__DSPT0_PRV_STICKY_RST__SHIFT                                                            0x3
+#define SELF_SOFT_RST__EP0_CFG_RST__SHIFT                                                                     0x4
+#define SELF_SOFT_RST__EP0_CFG_STICKY_RST__SHIFT                                                              0x5
+#define SELF_SOFT_RST__EP0_PRV_RST__SHIFT                                                                     0x6
+#define SELF_SOFT_RST__EP0_PRV_STICKY_RST__SHIFT                                                              0x7
+#define SELF_SOFT_RST__PCIE_NBIF_HST_SDP_PORT_RST__SHIFT                                                      0x18
+#define SELF_SOFT_RST__SST_HST_SDP_PORT_RST__SHIFT                                                            0x19
+#define SELF_SOFT_RST__NBIF_PCIE_DMA_SDP_PORT_RST__SHIFT                                                      0x1a
+#define SELF_SOFT_RST__SST_DMA_SDP_PORT_RST__SHIFT                                                            0x1b
+#define SELF_SOFT_RST__SWUS_SHADOW_RST__SHIFT                                                                 0x1c
+#define SELF_SOFT_RST__CORE_STICKY_RST__SHIFT                                                                 0x1d
+#define SELF_SOFT_RST__RELOAD_STRAP__SHIFT                                                                    0x1e
+#define SELF_SOFT_RST__CORE_RST__SHIFT                                                                        0x1f
+#define SELF_SOFT_RST__DSPT0_CFG_RST_MASK                                                                     0x00000001L
+#define SELF_SOFT_RST__DSPT0_CFG_STICKY_RST_MASK                                                              0x00000002L
+#define SELF_SOFT_RST__DSPT0_PRV_RST_MASK                                                                     0x00000004L
+#define SELF_SOFT_RST__DSPT0_PRV_STICKY_RST_MASK                                                              0x00000008L
+#define SELF_SOFT_RST__EP0_CFG_RST_MASK                                                                       0x00000010L
+#define SELF_SOFT_RST__EP0_CFG_STICKY_RST_MASK                                                                0x00000020L
+#define SELF_SOFT_RST__EP0_PRV_RST_MASK                                                                       0x00000040L
+#define SELF_SOFT_RST__EP0_PRV_STICKY_RST_MASK                                                                0x00000080L
+#define SELF_SOFT_RST__PCIE_NBIF_HST_SDP_PORT_RST_MASK                                                        0x01000000L
+#define SELF_SOFT_RST__SST_HST_SDP_PORT_RST_MASK                                                              0x02000000L
+#define SELF_SOFT_RST__NBIF_PCIE_DMA_SDP_PORT_RST_MASK                                                        0x04000000L
+#define SELF_SOFT_RST__SST_DMA_SDP_PORT_RST_MASK                                                              0x08000000L
+#define SELF_SOFT_RST__SWUS_SHADOW_RST_MASK                                                                   0x10000000L
+#define SELF_SOFT_RST__CORE_STICKY_RST_MASK                                                                   0x20000000L
+#define SELF_SOFT_RST__RELOAD_STRAP_MASK                                                                      0x40000000L
+#define SELF_SOFT_RST__CORE_RST_MASK                                                                          0x80000000L
+//BIF_GFX_DRV_VPU_RST
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_RST__SHIFT                                                      0x0
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_FLR_EXC_RST__SHIFT                                              0x1
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_STICKY_RST__SHIFT                                               0x2
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_RST__SHIFT                                                      0x3
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_STICKY_RST__SHIFT                                               0x4
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_RST__SHIFT                                                      0x5
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_STICKY_RST__SHIFT                                               0x6
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_PRV_RST__SHIFT                                                      0x7
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_RST_MASK                                                        0x00000001L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_FLR_EXC_RST_MASK                                                0x00000002L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_CFG_STICKY_RST_MASK                                                 0x00000004L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_RST_MASK                                                        0x00000008L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_PF_PRV_STICKY_RST_MASK                                                 0x00000010L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_RST_MASK                                                        0x00000020L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_CFG_STICKY_RST_MASK                                                 0x00000040L
+#define BIF_GFX_DRV_VPU_RST__DRV_MODE1_VF_PRV_RST_MASK                                                        0x00000080L
+//BIF_RST_MISC_CTRL
+#define BIF_RST_MISC_CTRL__ERRSTATUS_KEPT_IN_PERSTB__SHIFT                                                    0x0
+#define BIF_RST_MISC_CTRL__DRV_RST_MODE__SHIFT                                                                0x2
+#define BIF_RST_MISC_CTRL__DRV_RST_CFG_MASK__SHIFT                                                            0x4
+#define BIF_RST_MISC_CTRL__DRV_RST_BITS_AUTO_CLEAR__SHIFT                                                     0x5
+#define BIF_RST_MISC_CTRL__FLR_RST_BIT_AUTO_CLEAR__SHIFT                                                      0x6
+#define BIF_RST_MISC_CTRL__STRAP_EP_LNK_RST_IOV_EN__SHIFT                                                     0x8
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_MODE__SHIFT                                                          0x9
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_TIMEOUT__SHIFT                                                       0xa
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER_SEL__SHIFT                                                           0xd
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER2_SEL__SHIFT                                                          0xf
+#define BIF_RST_MISC_CTRL__SRIOV_SAVE_VFS_ON_VFENABLE_CLR__SHIFT                                              0x11
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_DIS__SHIFT                                                       0x17
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_RSPSTS__SHIFT                                                    0x18
+#define BIF_RST_MISC_CTRL__HARD_RST_CTRL_SWUS_SHADOW_PRV_RST_EN__SHIFT                                        0x1a
+#define BIF_RST_MISC_CTRL__ERRSTATUS_KEPT_IN_PERSTB_MASK                                                      0x00000001L
+#define BIF_RST_MISC_CTRL__DRV_RST_MODE_MASK                                                                  0x0000000CL
+#define BIF_RST_MISC_CTRL__DRV_RST_CFG_MASK_MASK                                                              0x00000010L
+#define BIF_RST_MISC_CTRL__DRV_RST_BITS_AUTO_CLEAR_MASK                                                       0x00000020L
+#define BIF_RST_MISC_CTRL__FLR_RST_BIT_AUTO_CLEAR_MASK                                                        0x00000040L
+#define BIF_RST_MISC_CTRL__STRAP_EP_LNK_RST_IOV_EN_MASK                                                       0x00000100L
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_MODE_MASK                                                            0x00000200L
+#define BIF_RST_MISC_CTRL__LNK_RST_GRACE_TIMEOUT_MASK                                                         0x00001C00L
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER_SEL_MASK                                                             0x00006000L
+#define BIF_RST_MISC_CTRL__LNK_RST_TIMER2_SEL_MASK                                                            0x00018000L
+#define BIF_RST_MISC_CTRL__SRIOV_SAVE_VFS_ON_VFENABLE_CLR_MASK                                                0x000E0000L
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_DIS_MASK                                                         0x00800000L
+#define BIF_RST_MISC_CTRL__LNK_RST_DMA_DUMMY_RSPSTS_MASK                                                      0x03000000L
+#define BIF_RST_MISC_CTRL__HARD_RST_CTRL_SWUS_SHADOW_PRV_RST_EN_MASK                                          0x04000000L
+//BIF_RST_MISC_CTRL2
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_PROTECT__SHIFT                                                       0x0
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_PROTECT__SHIFT                                                       0x1
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_PROTECT__SHIFT                                                      0x2
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT__SHIFT                                                            0xf
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_TRANS_IDLE__SHIFT                                                    0x10
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_TRANS_IDLE__SHIFT                                                    0x11
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_TRANS_IDLE__SHIFT                                                   0x12
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT_DIS__SHIFT                                                        0x1e
+#define BIF_RST_MISC_CTRL2__ALL_RST_TRANS_IDLE__SHIFT                                                         0x1f
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_PROTECT_MASK                                                         0x00000001L
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_PROTECT_MASK                                                         0x00000002L
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_PROTECT_MASK                                                        0x00000004L
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT_MASK                                                              0x00008000L
+#define BIF_RST_MISC_CTRL2__SWUS_LNK_RST_TRANS_IDLE_MASK                                                      0x00010000L
+#define BIF_RST_MISC_CTRL2__SWDS_LNK_RST_TRANS_IDLE_MASK                                                      0x00020000L
+#define BIF_RST_MISC_CTRL2__ENDP0_LNK_RST_TRANS_IDLE_MASK                                                     0x00040000L
+#define BIF_RST_MISC_CTRL2__ALL_RST_PROTECT_DIS_MASK                                                          0x40000000L
+#define BIF_RST_MISC_CTRL2__ALL_RST_TRANS_IDLE_MASK                                                           0x80000000L
+//BIF_RST_MISC_CTRL3
+#define BIF_RST_MISC_CTRL3__TIMER_SCALE__SHIFT                                                                0x0
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_TIMEOUT__SHIFT                                                        0x4
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_MODE__SHIFT                                                           0x6
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_HARD__SHIFT                                                    0x7
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SOFT__SHIFT                                                    0xa
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SELF__SHIFT                                                    0xd
+#define BIF_RST_MISC_CTRL3__TIMER_SCALE_MASK                                                                  0x0000000FL
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_TIMEOUT_MASK                                                          0x00000030L
+#define BIF_RST_MISC_CTRL3__PME_TURNOFF_MODE_MASK                                                             0x00000040L
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_HARD_MASK                                                      0x00000380L
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SOFT_MASK                                                      0x00001C00L
+#define BIF_RST_MISC_CTRL3__RELOAD_STRAP_DELAY_SELF_MASK                                                      0x0000E000L
+//DEV0_PF0_FLR_RST_CTRL
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_CFG_EN__SHIFT                                         0x5
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_CFG_STICKY_EN__SHIFT                                  0x6
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_PRV_EN__SHIFT                                         0x7
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_EN__SHIFT                                                          0x8
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_FLR_EXC_EN__SHIFT                                                  0x9
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_STICKY_EN__SHIFT                                                   0xa
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PRV_EN__SHIFT                                                          0xb
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PRV_STICKY_EN__SHIFT                                                   0xc
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_VF_CFG_EN__SHIFT                                      0xd
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_VF_CFG_STICKY_EN__SHIFT                               0xe
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_VF_PRV_EN__SHIFT                                      0xf
+#define DEV0_PF0_FLR_RST_CTRL__FLR_TWICE_EN__SHIFT                                                            0x10
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PFCOPY_PRV_EN__SHIFT                                                   0x1f
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF0_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF0_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_CFG_EN_MASK                                           0x00000020L
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_CFG_STICKY_EN_MASK                                    0x00000040L
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_PRV_EN_MASK                                           0x00000080L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_EN_MASK                                                            0x00000100L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_FLR_EXC_EN_MASK                                                    0x00000200L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_CFG_STICKY_EN_MASK                                                     0x00000400L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PRV_EN_MASK                                                            0x00000800L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PRV_STICKY_EN_MASK                                                     0x00001000L
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_VF_CFG_EN_MASK                                        0x00002000L
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_VF_CFG_STICKY_EN_MASK                                 0x00004000L
+#define DEV0_PF0_FLR_RST_CTRL__DEV0_PF0_FLR_RST_CTRL_VF_VF_PRV_EN_MASK                                        0x00008000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_TWICE_EN_MASK                                                              0x00010000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF0_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+#define DEV0_PF0_FLR_RST_CTRL__SOFT_PF_PFCOPY_PRV_EN_MASK                                                     0x80000000L
+//DEV0_PF1_FLR_RST_CTRL
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_EN__SHIFT                                                               0x0
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                       0x1
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                        0x2
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_EN__SHIFT                                                               0x3
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                        0x4
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_CFG_EN__SHIFT                                         0x5
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_CFG_STICKY_EN__SHIFT                                  0x6
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_PRV_EN__SHIFT                                         0x7
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_VF_CFG_EN__SHIFT                                      0xd
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_VF_CFG_STICKY_EN__SHIFT                               0xe
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_VF_PRV_EN__SHIFT                                      0xf
+#define DEV0_PF1_FLR_RST_CTRL__FLR_TWICE_EN__SHIFT                                                            0x10
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_MODE__SHIFT                                                          0x11
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT__SHIFT                                                       0x12
+#define DEV0_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS__SHIFT                                                    0x17
+#define DEV0_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS__SHIFT                                                    0x19
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_EN_MASK                                                                 0x00000001L
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                         0x00000002L
+#define DEV0_PF1_FLR_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                          0x00000004L
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_EN_MASK                                                                 0x00000008L
+#define DEV0_PF1_FLR_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                          0x00000010L
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_CFG_EN_MASK                                           0x00000020L
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_CFG_STICKY_EN_MASK                                    0x00000040L
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_PRV_EN_MASK                                           0x00000080L
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_VF_CFG_EN_MASK                                        0x00002000L
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_VF_CFG_STICKY_EN_MASK                                 0x00004000L
+#define DEV0_PF1_FLR_RST_CTRL__DEV0_PF1_FLR_RST_CTRL_VF_VF_PRV_EN_MASK                                        0x00008000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_TWICE_EN_MASK                                                              0x00010000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_MODE_MASK                                                            0x00020000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_GRACE_TIMEOUT_MASK                                                         0x001C0000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_DMA_DUMMY_RSPSTS_MASK                                                      0x01800000L
+#define DEV0_PF1_FLR_RST_CTRL__FLR_HST_DUMMY_RSPSTS_MASK                                                      0x06000000L
+//BIF_INST_RESET_INTR_STS
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_INTR_STS__SHIFT                                               0x0
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_CFG_ONLY_INTR_STS__SHIFT                                      0x1
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M0_INTR_STS__SHIFT                                                 0x2
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M1_INTR_STS__SHIFT                                                 0x3
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M2_INTR_STS__SHIFT                                                 0x4
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_INTR_STS_MASK                                                 0x00000001L
+#define BIF_INST_RESET_INTR_STS__EP0_LINK_RESET_CFG_ONLY_INTR_STS_MASK                                        0x00000002L
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M0_INTR_STS_MASK                                                   0x00000004L
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M1_INTR_STS_MASK                                                   0x00000008L
+#define BIF_INST_RESET_INTR_STS__DRV_RESET_M2_INTR_STS_MASK                                                   0x00000010L
+//BIF_PF_FLR_INTR_STS
+#define BIF_PF_FLR_INTR_STS__DEV0_PF0_FLR_INTR_STS__SHIFT                                                     0x0
+#define BIF_PF_FLR_INTR_STS__DEV0_PF1_FLR_INTR_STS__SHIFT                                                     0x1
+#define BIF_PF_FLR_INTR_STS__DEV0_PF0_FLR_INTR_STS_MASK                                                       0x00000001L
+#define BIF_PF_FLR_INTR_STS__DEV0_PF1_FLR_INTR_STS_MASK                                                       0x00000002L
+//BIF_D3HOTD0_INTR_STS
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF0_D3HOTD0_INTR_STS__SHIFT                                                0x0
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF1_D3HOTD0_INTR_STS__SHIFT                                                0x1
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF0_D3HOTD0_INTR_STS_MASK                                                  0x00000001L
+#define BIF_D3HOTD0_INTR_STS__DEV0_PF1_D3HOTD0_INTR_STS_MASK                                                  0x00000002L
+//BIF_POWER_INTR_STS
+#define BIF_POWER_INTR_STS__DEV0_PME_TURN_OFF_INTR_STS__SHIFT                                                 0x0
+#define BIF_POWER_INTR_STS__PORT0_DSTATE_INTR_STS__SHIFT                                                      0x10
+#define BIF_POWER_INTR_STS__DEV0_PME_TURN_OFF_INTR_STS_MASK                                                   0x00000001L
+#define BIF_POWER_INTR_STS__PORT0_DSTATE_INTR_STS_MASK                                                        0x00010000L
+//BIF_PF_DSTATE_INTR_STS
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF0_DSTATE_INTR_STS__SHIFT                                               0x0
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF1_DSTATE_INTR_STS__SHIFT                                               0x1
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF2_DSTATE_INTR_STS__SHIFT                                               0x2
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF3_DSTATE_INTR_STS__SHIFT                                               0x3
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF4_DSTATE_INTR_STS__SHIFT                                               0x4
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF5_DSTATE_INTR_STS__SHIFT                                               0x5
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF6_DSTATE_INTR_STS__SHIFT                                               0x6
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF7_DSTATE_INTR_STS__SHIFT                                               0x7
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF0_DSTATE_INTR_STS_MASK                                                 0x00000001L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF1_DSTATE_INTR_STS_MASK                                                 0x00000002L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF2_DSTATE_INTR_STS_MASK                                                 0x00000004L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF3_DSTATE_INTR_STS_MASK                                                 0x00000008L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF4_DSTATE_INTR_STS_MASK                                                 0x00000010L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF5_DSTATE_INTR_STS_MASK                                                 0x00000020L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF6_DSTATE_INTR_STS_MASK                                                 0x00000040L
+#define BIF_PF_DSTATE_INTR_STS__DEV0_PF7_DSTATE_INTR_STS_MASK                                                 0x00000080L
+//SELF_SOFT_RST_2
+#define SELF_SOFT_RST_2__DSPT3_CFG_RST__SHIFT                                                                 0x0
+#define SELF_SOFT_RST_2__DSPT3_CFG_STICKY_RST__SHIFT                                                          0x1
+#define SELF_SOFT_RST_2__DSPT3_PRV_RST__SHIFT                                                                 0x2
+#define SELF_SOFT_RST_2__DSPT3_PRV_STICKY_RST__SHIFT                                                          0x3
+#define SELF_SOFT_RST_2__EP3_CFG_RST__SHIFT                                                                   0x4
+#define SELF_SOFT_RST_2__EP3_CFG_STICKY_RST__SHIFT                                                            0x5
+#define SELF_SOFT_RST_2__EP3_PRV_RST__SHIFT                                                                   0x6
+#define SELF_SOFT_RST_2__EP3_PRV_STICKY_RST__SHIFT                                                            0x7
+#define SELF_SOFT_RST_2__GMISP0_SDP_PORT_RST__SHIFT                                                           0x18
+#define SELF_SOFT_RST_2__STRAP_RST__SHIFT                                                                     0x19
+#define SELF_SOFT_RST_2__SWUS_SHADOW_PRV_RST__SHIFT                                                           0x1a
+#define SELF_SOFT_RST_2__DSPT3_CFG_RST_MASK                                                                   0x00000001L
+#define SELF_SOFT_RST_2__DSPT3_CFG_STICKY_RST_MASK                                                            0x00000002L
+#define SELF_SOFT_RST_2__DSPT3_PRV_RST_MASK                                                                   0x00000004L
+#define SELF_SOFT_RST_2__DSPT3_PRV_STICKY_RST_MASK                                                            0x00000008L
+#define SELF_SOFT_RST_2__EP3_CFG_RST_MASK                                                                     0x00000010L
+#define SELF_SOFT_RST_2__EP3_CFG_STICKY_RST_MASK                                                              0x00000020L
+#define SELF_SOFT_RST_2__EP3_PRV_RST_MASK                                                                     0x00000040L
+#define SELF_SOFT_RST_2__EP3_PRV_STICKY_RST_MASK                                                              0x00000080L
+#define SELF_SOFT_RST_2__GMISP0_SDP_PORT_RST_MASK                                                             0x01000000L
+#define SELF_SOFT_RST_2__STRAP_RST_MASK                                                                       0x02000000L
+#define SELF_SOFT_RST_2__SWUS_SHADOW_PRV_RST_MASK                                                             0x04000000L
+//SION_SDP_PORT_RST
+#define SION_SDP_PORT_RST__SYSHUB_NBIFSION_SDP_PORT_RST__SHIFT                                                0x0
+#define SION_SDP_PORT_RST__NBIFSION_SYSHUB_SDP_PORT_RST__SHIFT                                                0x1
+#define SION_SDP_PORT_RST__BIF_NBIFSION_SDP_PORT_RST__SHIFT                                                   0x2
+#define SION_SDP_PORT_RST__ATDMA_NBIFSOIN_SDP_PORT_RST__SHIFT                                                 0x4
+#define SION_SDP_PORT_RST__INT_NBIFSION_SDP_PORT_RST__SHIFT                                                   0x5
+#define SION_SDP_PORT_RST__MP4SDP_SDP_PORT_RST__SHIFT                                                         0x6
+#define SION_SDP_PORT_RST__NBIFSOIN_ATHUB_SDP_PORT_RST__SHIFT                                                 0x7
+#define SION_SDP_PORT_RST__NTB_HST_SDP_PORT_RST__SHIFT                                                        0x8
+#define SION_SDP_PORT_RST__NTB_DMA_SDP_PORT_RST__SHIFT                                                        0x9
+#define SION_SDP_PORT_RST__ATHUB_HST_SDP_PORT_RST__SHIFT                                                      0xc
+#define SION_SDP_PORT_RST__SION_AON_RST__SHIFT                                                                0x18
+#define SION_SDP_PORT_RST__SYSHUB_NBIFSION_SDP_PORT_RST_MASK                                                  0x00000001L
+#define SION_SDP_PORT_RST__NBIFSION_SYSHUB_SDP_PORT_RST_MASK                                                  0x00000002L
+#define SION_SDP_PORT_RST__BIF_NBIFSION_SDP_PORT_RST_MASK                                                     0x00000004L
+#define SION_SDP_PORT_RST__ATDMA_NBIFSOIN_SDP_PORT_RST_MASK                                                   0x00000010L
+#define SION_SDP_PORT_RST__INT_NBIFSION_SDP_PORT_RST_MASK                                                     0x00000020L
+#define SION_SDP_PORT_RST__MP4SDP_SDP_PORT_RST_MASK                                                           0x00000040L
+#define SION_SDP_PORT_RST__NBIFSOIN_ATHUB_SDP_PORT_RST_MASK                                                   0x00000080L
+#define SION_SDP_PORT_RST__NTB_HST_SDP_PORT_RST_MASK                                                          0x00000100L
+#define SION_SDP_PORT_RST__NTB_DMA_SDP_PORT_RST_MASK                                                          0x00000200L
+#define SION_SDP_PORT_RST__ATHUB_HST_SDP_PORT_RST_MASK                                                        0x00001000L
+#define SION_SDP_PORT_RST__SION_AON_RST_MASK                                                                  0x01000000L
+//BIF_INST_RESET_INTR_MASK
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_INTR_MASK__SHIFT                                             0x0
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_CFG_ONLY_INTR_MASK__SHIFT                                    0x1
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M0_INTR_MASK__SHIFT                                               0x2
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M1_INTR_MASK__SHIFT                                               0x3
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M2_INTR_MASK__SHIFT                                               0x4
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_INTR_MASK_MASK                                               0x00000001L
+#define BIF_INST_RESET_INTR_MASK__EP0_LINK_RESET_CFG_ONLY_INTR_MASK_MASK                                      0x00000002L
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M0_INTR_MASK_MASK                                                 0x00000004L
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M1_INTR_MASK_MASK                                                 0x00000008L
+#define BIF_INST_RESET_INTR_MASK__DRV_RESET_M2_INTR_MASK_MASK                                                 0x00000010L
+//BIF_PF_FLR_INTR_MASK
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF0_FLR_INTR_MASK__SHIFT                                                   0x0
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF1_FLR_INTR_MASK__SHIFT                                                   0x1
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF0_FLR_INTR_MASK_MASK                                                     0x00000001L
+#define BIF_PF_FLR_INTR_MASK__DEV0_PF1_FLR_INTR_MASK_MASK                                                     0x00000002L
+//BIF_D3HOTD0_INTR_MASK
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF0_D3HOTD0_INTR_MASK__SHIFT                                              0x0
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF1_D3HOTD0_INTR_MASK__SHIFT                                              0x1
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF0_D3HOTD0_INTR_MASK_MASK                                                0x00000001L
+#define BIF_D3HOTD0_INTR_MASK__DEV0_PF1_D3HOTD0_INTR_MASK_MASK                                                0x00000002L
+//BIF_POWER_INTR_MASK
+#define BIF_POWER_INTR_MASK__DEV0_PME_TURN_OFF_INTR_MASK__SHIFT                                               0x0
+#define BIF_POWER_INTR_MASK__PORT0_DSTATE_INTR_MASK__SHIFT                                                    0x10
+#define BIF_POWER_INTR_MASK__DEV0_PME_TURN_OFF_INTR_MASK_MASK                                                 0x00000001L
+#define BIF_POWER_INTR_MASK__PORT0_DSTATE_INTR_MASK_MASK                                                      0x00010000L
+//BIF_PF_DSTATE_INTR_MASK
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF0_DSTATE_INTR_MASK__SHIFT                                             0x0
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF1_DSTATE_INTR_MASK__SHIFT                                             0x1
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF2_DSTATE_INTR_MASK__SHIFT                                             0x2
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF3_DSTATE_INTR_MASK__SHIFT                                             0x3
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF4_DSTATE_INTR_MASK__SHIFT                                             0x4
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF5_DSTATE_INTR_MASK__SHIFT                                             0x5
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF6_DSTATE_INTR_MASK__SHIFT                                             0x6
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF7_DSTATE_INTR_MASK__SHIFT                                             0x7
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF0_DSTATE_INTR_MASK_MASK                                               0x00000001L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF1_DSTATE_INTR_MASK_MASK                                               0x00000002L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF2_DSTATE_INTR_MASK_MASK                                               0x00000004L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF3_DSTATE_INTR_MASK_MASK                                               0x00000008L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF4_DSTATE_INTR_MASK_MASK                                               0x00000010L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF5_DSTATE_INTR_MASK_MASK                                               0x00000020L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF6_DSTATE_INTR_MASK_MASK                                               0x00000040L
+#define BIF_PF_DSTATE_INTR_MASK__DEV0_PF7_DSTATE_INTR_MASK_MASK                                               0x00000080L
+//BIF_DEV0_PF0_VF_FLR_INTR_MASK
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF0_FLR_INTR_MASK__SHIFT                                      0x0
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF1_FLR_INTR_MASK__SHIFT                                      0x1
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF2_FLR_INTR_MASK__SHIFT                                      0x2
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF3_FLR_INTR_MASK__SHIFT                                      0x3
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF4_FLR_INTR_MASK__SHIFT                                      0x4
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF5_FLR_INTR_MASK__SHIFT                                      0x5
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF6_FLR_INTR_MASK__SHIFT                                      0x6
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF7_FLR_INTR_MASK__SHIFT                                      0x7
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_SOFTPF_FLR_INTR_MASK__SHIFT                                   0x1f
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF0_FLR_INTR_MASK_MASK                                        0x00000001L
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF1_FLR_INTR_MASK_MASK                                        0x00000002L
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF2_FLR_INTR_MASK_MASK                                        0x00000004L
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF3_FLR_INTR_MASK_MASK                                        0x00000008L
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF4_FLR_INTR_MASK_MASK                                        0x00000010L
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF5_FLR_INTR_MASK_MASK                                        0x00000020L
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF6_FLR_INTR_MASK_MASK                                        0x00000040L
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_VF7_FLR_INTR_MASK_MASK                                        0x00000080L
+#define BIF_DEV0_PF0_VF_FLR_INTR_MASK__DEV0_PF0_SOFTPF_FLR_INTR_MASK_MASK                                     0x80000000L
+//BIF_DEV0_PF1_VF_FLR_INTR_MASK
+#define BIF_DEV0_PF1_VF_FLR_INTR_MASK__DEV0_PF1_VF0_FLR_INTR_MASK__SHIFT                                      0x0
+#define BIF_DEV0_PF1_VF_FLR_INTR_MASK__DEV0_PF1_VF0_FLR_INTR_MASK_MASK                                        0x00000001L
+//BIF_PF_FLR_RST
+#define BIF_PF_FLR_RST__DEV0_PF0_FLR_RST__SHIFT                                                               0x0
+#define BIF_PF_FLR_RST__DEV0_PF1_FLR_RST__SHIFT                                                               0x1
+#define BIF_PF_FLR_RST__DEV0_PF0_FLR_RST_MASK                                                                 0x00000001L
+#define BIF_PF_FLR_RST__DEV0_PF1_FLR_RST_MASK                                                                 0x00000002L
+//BIF_PF_FLR_PROTECT
+#define BIF_PF_FLR_PROTECT__DEV0_PF0_FLR_PROTECT__SHIFT                                                       0x0
+#define BIF_PF_FLR_PROTECT__DEV0_PF1_FLR_PROTECT__SHIFT                                                       0x1
+#define BIF_PF_FLR_PROTECT__DEV0_PF2_FLR_PROTECT__SHIFT                                                       0x2
+#define BIF_PF_FLR_PROTECT__DEV0_PF3_FLR_PROTECT__SHIFT                                                       0x3
+#define BIF_PF_FLR_PROTECT__DEV0_PF4_FLR_PROTECT__SHIFT                                                       0x4
+#define BIF_PF_FLR_PROTECT__DEV0_PF5_FLR_PROTECT__SHIFT                                                       0x5
+#define BIF_PF_FLR_PROTECT__DEV0_PF6_FLR_PROTECT__SHIFT                                                       0x6
+#define BIF_PF_FLR_PROTECT__DEV0_PF7_FLR_PROTECT__SHIFT                                                       0x7
+#define BIF_PF_FLR_PROTECT__DEV0_PF0_TRANS_IDLE__SHIFT                                                        0x10
+#define BIF_PF_FLR_PROTECT__DEV0_PF1_TRANS_IDLE__SHIFT                                                        0x11
+#define BIF_PF_FLR_PROTECT__DEV0_PF2_TRANS_IDLE__SHIFT                                                        0x12
+#define BIF_PF_FLR_PROTECT__DEV0_PF3_TRANS_IDLE__SHIFT                                                        0x13
+#define BIF_PF_FLR_PROTECT__DEV0_PF4_TRANS_IDLE__SHIFT                                                        0x14
+#define BIF_PF_FLR_PROTECT__DEV0_PF5_TRANS_IDLE__SHIFT                                                        0x15
+#define BIF_PF_FLR_PROTECT__DEV0_PF6_TRANS_IDLE__SHIFT                                                        0x16
+#define BIF_PF_FLR_PROTECT__DEV0_PF7_TRANS_IDLE__SHIFT                                                        0x17
+#define BIF_PF_FLR_PROTECT__DEV0_PF0_FLR_PROTECT_MASK                                                         0x00000001L
+#define BIF_PF_FLR_PROTECT__DEV0_PF1_FLR_PROTECT_MASK                                                         0x00000002L
+#define BIF_PF_FLR_PROTECT__DEV0_PF2_FLR_PROTECT_MASK                                                         0x00000004L
+#define BIF_PF_FLR_PROTECT__DEV0_PF3_FLR_PROTECT_MASK                                                         0x00000008L
+#define BIF_PF_FLR_PROTECT__DEV0_PF4_FLR_PROTECT_MASK                                                         0x00000010L
+#define BIF_PF_FLR_PROTECT__DEV0_PF5_FLR_PROTECT_MASK                                                         0x00000020L
+#define BIF_PF_FLR_PROTECT__DEV0_PF6_FLR_PROTECT_MASK                                                         0x00000040L
+#define BIF_PF_FLR_PROTECT__DEV0_PF7_FLR_PROTECT_MASK                                                         0x00000080L
+#define BIF_PF_FLR_PROTECT__DEV0_PF0_TRANS_IDLE_MASK                                                          0x00010000L
+#define BIF_PF_FLR_PROTECT__DEV0_PF1_TRANS_IDLE_MASK                                                          0x00020000L
+#define BIF_PF_FLR_PROTECT__DEV0_PF2_TRANS_IDLE_MASK                                                          0x00040000L
+#define BIF_PF_FLR_PROTECT__DEV0_PF3_TRANS_IDLE_MASK                                                          0x00080000L
+#define BIF_PF_FLR_PROTECT__DEV0_PF4_TRANS_IDLE_MASK                                                          0x00100000L
+#define BIF_PF_FLR_PROTECT__DEV0_PF5_TRANS_IDLE_MASK                                                          0x00200000L
+#define BIF_PF_FLR_PROTECT__DEV0_PF6_TRANS_IDLE_MASK                                                          0x00400000L
+#define BIF_PF_FLR_PROTECT__DEV0_PF7_TRANS_IDLE_MASK                                                          0x00800000L
+//BIF_PF_FLR_PROTECT3
+#define BIF_PF_FLR_PROTECT3__DEV0_PF0_VF_FLR_CTRL__SHIFT                                                      0x0
+#define BIF_PF_FLR_PROTECT3__DEV0_PF1_VF_FLR_CTRL__SHIFT                                                      0x1
+#define BIF_PF_FLR_PROTECT3__DEV0_PF0_VF_FLR_CTRL_MASK                                                        0x00000001L
+#define BIF_PF_FLR_PROTECT3__DEV0_PF1_VF_FLR_CTRL_MASK                                                        0x00000002L
+//BIF_DEV0_PF0_DSTATE_VALUE
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF0_DSTATE_VALUE__DEV0_PF0_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//BIF_DEV0_PF1_DSTATE_VALUE
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_TGT_VALUE__SHIFT                                           0x0
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_NEED_D3TOD0_RESET__SHIFT                                   0x2
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_ACK_VALUE__SHIFT                                           0x10
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_TGT_VALUE_MASK                                             0x00000003L
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_NEED_D3TOD0_RESET_MASK                                     0x00000004L
+#define BIF_DEV0_PF1_DSTATE_VALUE__DEV0_PF1_DSTATE_ACK_VALUE_MASK                                             0x00030000L
+//GFX_RST_CNTL_IND0
+#define GFX_RST_CNTL_IND0__GFX_RST_FINISH_INDICATION__SHIFT                                                   0x0
+#define GFX_RST_CNTL_IND0__SCRATCH_REG__SHIFT                                                                 0x1
+#define GFX_RST_CNTL_IND0__GFX_RST_FINISH_INDICATION_MASK                                                     0x00000001L
+#define GFX_RST_CNTL_IND0__SCRATCH_REG_MASK                                                                   0xFFFFFFFEL
+//GFX_RST_CNTL_SOFT_PF
+#define GFX_RST_CNTL_SOFT_PF__GFX_RST_FINISH_INDICATION__SHIFT                                                0x0
+#define GFX_RST_CNTL_SOFT_PF__SCRATCH_REG__SHIFT                                                              0x1
+#define GFX_RST_CNTL_SOFT_PF__GFX_RST_FINISH_INDICATION_MASK                                                  0x00000001L
+#define GFX_RST_CNTL_SOFT_PF__SCRATCH_REG_MASK                                                                0xFFFFFFFEL
+//GFX_RST_CNTL_IND1
+#define GFX_RST_CNTL_IND1__GFX_RST_FINISH_INDICATION__SHIFT                                                   0x0
+#define GFX_RST_CNTL_IND1__SCRATCH_REG__SHIFT                                                                 0x1
+#define GFX_RST_CNTL_IND1__GFX_RST_FINISH_INDICATION_MASK                                                     0x00000001L
+#define GFX_RST_CNTL_IND1__SCRATCH_REG_MASK                                                                   0xFFFFFFFEL
+//GFX_RST_CNTL_IND2
+#define GFX_RST_CNTL_IND2__GFX_RST_FINISH_INDICATION__SHIFT                                                   0x0
+#define GFX_RST_CNTL_IND2__SCRATCH_REG__SHIFT                                                                 0x1
+#define GFX_RST_CNTL_IND2__GFX_RST_FINISH_INDICATION_MASK                                                     0x00000001L
+#define GFX_RST_CNTL_IND2__SCRATCH_REG_MASK                                                                   0xFFFFFFFEL
+//GFX_RST_CNTL_IND3
+#define GFX_RST_CNTL_IND3__GFX_RST_FINISH_INDICATION__SHIFT                                                   0x0
+#define GFX_RST_CNTL_IND3__SCRATCH_REG__SHIFT                                                                 0x1
+#define GFX_RST_CNTL_IND3__GFX_RST_FINISH_INDICATION_MASK                                                     0x00000001L
+#define GFX_RST_CNTL_IND3__SCRATCH_REG_MASK                                                                   0xFFFFFFFEL
+//GFX_RST_CNTL_IND4
+#define GFX_RST_CNTL_IND4__GFX_RST_FINISH_INDICATION__SHIFT                                                   0x0
+#define GFX_RST_CNTL_IND4__SCRATCH_REG__SHIFT                                                                 0x1
+#define GFX_RST_CNTL_IND4__GFX_RST_FINISH_INDICATION_MASK                                                     0x00000001L
+#define GFX_RST_CNTL_IND4__SCRATCH_REG_MASK                                                                   0xFFFFFFFEL
+//GFX_RST_CNTL_IND5
+#define GFX_RST_CNTL_IND5__GFX_RST_FINISH_INDICATION__SHIFT                                                   0x0
+#define GFX_RST_CNTL_IND5__SCRATCH_REG__SHIFT                                                                 0x1
+#define GFX_RST_CNTL_IND5__GFX_RST_FINISH_INDICATION_MASK                                                     0x00000001L
+#define GFX_RST_CNTL_IND5__SCRATCH_REG_MASK                                                                   0xFFFFFFFEL
+//GFX_RST_CNTL_IND6
+#define GFX_RST_CNTL_IND6__GFX_RST_FINISH_INDICATION__SHIFT                                                   0x0
+#define GFX_RST_CNTL_IND6__SCRATCH_REG__SHIFT                                                                 0x1
+#define GFX_RST_CNTL_IND6__GFX_RST_FINISH_INDICATION_MASK                                                     0x00000001L
+#define GFX_RST_CNTL_IND6__SCRATCH_REG_MASK                                                                   0xFFFFFFFEL
+//GFX_RST_CNTL_IND7
+#define GFX_RST_CNTL_IND7__GFX_RST_FINISH_INDICATION__SHIFT                                                   0x0
+#define GFX_RST_CNTL_IND7__SCRATCH_REG__SHIFT                                                                 0x1
+#define GFX_RST_CNTL_IND7__GFX_RST_FINISH_INDICATION_MASK                                                     0x00000001L
+#define GFX_RST_CNTL_IND7__SCRATCH_REG_MASK                                                                   0xFFFFFFFEL
+//GFX_RST_CNTL_IND8
+#define GFX_RST_CNTL_IND8__GFX_RST_FINISH_INDICATION__SHIFT                                                   0x0
+#define GFX_RST_CNTL_IND8__SCRATCH_REG__SHIFT                                                                 0x1
+#define GFX_RST_CNTL_IND8__GFX_RST_FINISH_INDICATION_MASK                                                     0x00000001L
+#define GFX_RST_CNTL_IND8__SCRATCH_REG_MASK                                                                   0xFFFFFFFEL
+//DEV0_PF0_D3HOTD0_RST_CTRL
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF0_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//DEV0_PF1_D3HOTD0_RST_CTRL
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN__SHIFT                                                           0x0
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN__SHIFT                                                   0x1
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN__SHIFT                                                    0x2
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN__SHIFT                                                           0x3
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN__SHIFT                                                    0x4
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_EN_MASK                                                             0x00000001L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_FLR_EXC_EN_MASK                                                     0x00000002L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_CFG_STICKY_EN_MASK                                                      0x00000004L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_EN_MASK                                                             0x00000008L
+#define DEV0_PF1_D3HOTD0_RST_CTRL__PF_PRV_STICKY_EN_MASK                                                      0x00000010L
+//BIF_DEV0_PF0_VF_FLR_PROTECT
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF0_FLR_PROTECT__SHIFT                                          0x0
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF1_FLR_PROTECT__SHIFT                                          0x1
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF2_FLR_PROTECT__SHIFT                                          0x2
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF3_FLR_PROTECT__SHIFT                                          0x3
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF4_FLR_PROTECT__SHIFT                                          0x4
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF5_FLR_PROTECT__SHIFT                                          0x5
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF6_FLR_PROTECT__SHIFT                                          0x6
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF7_FLR_PROTECT__SHIFT                                          0x7
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_SOFTPF_FLR_PROTECT__SHIFT                                       0x1f
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF0_FLR_PROTECT_MASK                                            0x00000001L
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF1_FLR_PROTECT_MASK                                            0x00000002L
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF2_FLR_PROTECT_MASK                                            0x00000004L
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF3_FLR_PROTECT_MASK                                            0x00000008L
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF4_FLR_PROTECT_MASK                                            0x00000010L
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF5_FLR_PROTECT_MASK                                            0x00000020L
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF6_FLR_PROTECT_MASK                                            0x00000040L
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_VF7_FLR_PROTECT_MASK                                            0x00000080L
+#define BIF_DEV0_PF0_VF_FLR_PROTECT__DEV0_PF0_SOFTPF_FLR_PROTECT_MASK                                         0x80000000L
+//BIF_DEV0_PF1_VF_FLR_PROTECT
+#define BIF_DEV0_PF1_VF_FLR_PROTECT__DEV0_PF1_VF0_FLR_PROTECT__SHIFT                                          0x0
+#define BIF_DEV0_PF1_VF_FLR_PROTECT__DEV0_PF1_VF0_FLR_PROTECT_MASK                                            0x00000001L
+//BIF_SRIOVEN_SET_INTR_STS
+#define BIF_SRIOVEN_SET_INTR_STS__DEV0_PF0_SRIOVEN_SET_INTR_STS__SHIFT                                        0x0
+#define BIF_SRIOVEN_SET_INTR_STS__DEV0_PF1_SRIOVEN_SET_INTR_STS__SHIFT                                        0x1
+#define BIF_SRIOVEN_SET_INTR_STS__DEV0_PF0_SRIOVEN_SET_INTR_STS_MASK                                          0x00000001L
+#define BIF_SRIOVEN_SET_INTR_STS__DEV0_PF1_SRIOVEN_SET_INTR_STS_MASK                                          0x00000002L
+//BIF_SRIOVEN_CLR_INTR_STS
+#define BIF_SRIOVEN_CLR_INTR_STS__DEV0_PF0_SRIOVEN_CLR_INTR_STS__SHIFT                                        0x0
+#define BIF_SRIOVEN_CLR_INTR_STS__DEV0_PF1_SRIOVEN_CLR_INTR_STS__SHIFT                                        0x1
+#define BIF_SRIOVEN_CLR_INTR_STS__DEV0_PF0_SRIOVEN_CLR_INTR_STS_MASK                                          0x00000001L
+#define BIF_SRIOVEN_CLR_INTR_STS__DEV0_PF1_SRIOVEN_CLR_INTR_STS_MASK                                          0x00000002L
+//BIF_SRIOVEN_SET_INTR_MASK
+#define BIF_SRIOVEN_SET_INTR_MASK__DEV0_PF0_SRIOVEN_SET_INTR_MASK__SHIFT                                      0x0
+#define BIF_SRIOVEN_SET_INTR_MASK__DEV0_PF1_SRIOVEN_SET_INTR_MASK__SHIFT                                      0x1
+#define BIF_SRIOVEN_SET_INTR_MASK__DEV0_PF0_SRIOVEN_SET_INTR_MASK_MASK                                        0x00000001L
+#define BIF_SRIOVEN_SET_INTR_MASK__DEV0_PF1_SRIOVEN_SET_INTR_MASK_MASK                                        0x00000002L
+//BIF_SRIOVEN_CLR_INTR_MASK
+#define BIF_SRIOVEN_CLR_INTR_MASK__DEV0_PF0_SRIOVEN_CLR_INTR_MASK__SHIFT                                      0x0
+#define BIF_SRIOVEN_CLR_INTR_MASK__DEV0_PF1_SRIOVEN_CLR_INTR_MASK__SHIFT                                      0x1
+#define BIF_SRIOVEN_CLR_INTR_MASK__DEV0_PF0_SRIOVEN_CLR_INTR_MASK_MASK                                        0x00000001L
+#define BIF_SRIOVEN_CLR_INTR_MASK__DEV0_PF1_SRIOVEN_CLR_INTR_MASK_MASK                                        0x00000002L
+//BIF_PORT0_DSTATE_VALUE
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_TGT_VALUE__SHIFT                                                 0x0
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_ACK_VALUE__SHIFT                                                 0x10
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_TGT_VALUE_MASK                                                   0x00000003L
+#define BIF_PORT0_DSTATE_VALUE__PORT0_DSTATE_ACK_VALUE_MASK                                                   0x00030000L
+//BIF_DEV0_PF0_RST_VF_FLR_IDLE
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF0_TRANS_IDLE__SHIFT                                          0x0
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF1_TRANS_IDLE__SHIFT                                          0x1
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF2_TRANS_IDLE__SHIFT                                          0x2
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF3_TRANS_IDLE__SHIFT                                          0x3
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF4_TRANS_IDLE__SHIFT                                          0x4
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF5_TRANS_IDLE__SHIFT                                          0x5
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF6_TRANS_IDLE__SHIFT                                          0x6
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF7_TRANS_IDLE__SHIFT                                          0x7
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__SOFTPF_TRANS_IDLE__SHIFT                                                0x1f
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF0_TRANS_IDLE_MASK                                            0x00000001L
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF1_TRANS_IDLE_MASK                                            0x00000002L
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF2_TRANS_IDLE_MASK                                            0x00000004L
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF3_TRANS_IDLE_MASK                                            0x00000008L
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF4_TRANS_IDLE_MASK                                            0x00000010L
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF5_TRANS_IDLE_MASK                                            0x00000020L
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF6_TRANS_IDLE_MASK                                            0x00000040L
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__DEV0_PF0_VF7_TRANS_IDLE_MASK                                            0x00000080L
+#define BIF_DEV0_PF0_RST_VF_FLR_IDLE__SOFTPF_TRANS_IDLE_MASK                                                  0x80000000L
+//BIF_DEV0_PF1_RST_VF_FLR_IDLE
+#define BIF_DEV0_PF1_RST_VF_FLR_IDLE__DEV0_PF1_VF0_TRANS_IDLE__SHIFT                                          0x0
+#define BIF_DEV0_PF1_RST_VF_FLR_IDLE__DEV0_PF1_VF0_TRANS_IDLE_MASK                                            0x00000001L
+//BIF_DEV0_PF0_VF_FLR_INTR_STS
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF0_FLR_INTR_STS__SHIFT                                        0x0
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF1_FLR_INTR_STS__SHIFT                                        0x1
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF2_FLR_INTR_STS__SHIFT                                        0x2
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF3_FLR_INTR_STS__SHIFT                                        0x3
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF4_FLR_INTR_STS__SHIFT                                        0x4
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF5_FLR_INTR_STS__SHIFT                                        0x5
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF6_FLR_INTR_STS__SHIFT                                        0x6
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF7_FLR_INTR_STS__SHIFT                                        0x7
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_SOFTPF_FLR_INTR_STS__SHIFT                                     0x1f
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF0_FLR_INTR_STS_MASK                                          0x00000001L
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF1_FLR_INTR_STS_MASK                                          0x00000002L
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF2_FLR_INTR_STS_MASK                                          0x00000004L
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF3_FLR_INTR_STS_MASK                                          0x00000008L
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF4_FLR_INTR_STS_MASK                                          0x00000010L
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF5_FLR_INTR_STS_MASK                                          0x00000020L
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF6_FLR_INTR_STS_MASK                                          0x00000040L
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_VF7_FLR_INTR_STS_MASK                                          0x00000080L
+#define BIF_DEV0_PF0_VF_FLR_INTR_STS__DEV0_PF0_SOFTPF_FLR_INTR_STS_MASK                                       0x80000000L
+//BIF_DEV0_PF1_VF_FLR_INTR_STS
+#define BIF_DEV0_PF1_VF_FLR_INTR_STS__DEV0_PF1_VF0_FLR_INTR_STS__SHIFT                                        0x0
+#define BIF_DEV0_PF1_VF_FLR_INTR_STS__DEV0_PF1_VF0_FLR_INTR_STS_MASK                                          0x00000001L
+//BIF_DEV0_PF0_VF_FLR_RST
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF0_FLR_RST__SHIFT                                                  0x0
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF1_FLR_RST__SHIFT                                                  0x1
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF2_FLR_RST__SHIFT                                                  0x2
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF3_FLR_RST__SHIFT                                                  0x3
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF4_FLR_RST__SHIFT                                                  0x4
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF5_FLR_RST__SHIFT                                                  0x5
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF6_FLR_RST__SHIFT                                                  0x6
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF7_FLR_RST__SHIFT                                                  0x7
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_SOFTPF_FLR_RST__SHIFT                                               0x1f
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF0_FLR_RST_MASK                                                    0x00000001L
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF1_FLR_RST_MASK                                                    0x00000002L
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF2_FLR_RST_MASK                                                    0x00000004L
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF3_FLR_RST_MASK                                                    0x00000008L
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF4_FLR_RST_MASK                                                    0x00000010L
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF5_FLR_RST_MASK                                                    0x00000020L
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF6_FLR_RST_MASK                                                    0x00000040L
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF7_FLR_RST_MASK                                                    0x00000080L
+#define BIF_DEV0_PF0_VF_FLR_RST__DEV0_PF0_SOFTPF_FLR_RST_MASK                                                 0x80000000L
+//BIF_DEV0_PF1_VF_FLR_RST
+#define BIF_DEV0_PF1_VF_FLR_RST__DEV0_PF1_VF0_FLR_RST__SHIFT                                                  0x0
+#define BIF_DEV0_PF1_VF_FLR_RST__DEV0_PF1_VF0_FLR_RST_MASK                                                    0x00000001L
+
+
+// addressBlock: nbif0_nbif0_bif_misc_bif_misc_regblk
+//DOE_PRV_CTRL0_DEV0F0
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_INT_MSG_NUMBER_DEV0F0__SHIFT                                            0x0
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_WRITE_THRESHOLD_DEV0F0__SHIFT                                           0xb
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_READ_THRESHOLD_DEV0F0__SHIFT                                            0x10
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_WOT_INTM_DEV0F0__SHIFT                                                  0x15
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_ROT_INTM_DEV0F0__SHIFT                                                  0x16
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_ABT_INTM_DEV0F0__SHIFT                                                  0x17
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_GO_INTM_DEV0F0__SHIFT                                                   0x18
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_PSN_INTM_DEV0F0__SHIFT                                                  0x19
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_RUDF_INTM_DEV0F0__SHIFT                                                 0x1a
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_INT_MSG_NUMBER_DEV0F0_MASK                                              0x000007FFL
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_WRITE_THRESHOLD_DEV0F0_MASK                                             0x0000F800L
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_READ_THRESHOLD_DEV0F0_MASK                                              0x001F0000L
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_WOT_INTM_DEV0F0_MASK                                                    0x00200000L
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_ROT_INTM_DEV0F0_MASK                                                    0x00400000L
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_ABT_INTM_DEV0F0_MASK                                                    0x00800000L
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_GO_INTM_DEV0F0_MASK                                                     0x01000000L
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_PSN_INTM_DEV0F0_MASK                                                    0x02000000L
+#define DOE_PRV_CTRL0_DEV0F0__DOE_PRV_RUDF_INTM_DEV0F0_MASK                                                   0x04000000L
+//DOE_PRV_CMD_DEV0F0
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_BUSY_CLR_ABORT_DEV0F0__SHIFT                                              0x0
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_ERR_SET_DEV0F0__SHIFT                                                     0x1
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_DATA_REMAIN_DEV0F0__SHIFT                                                 0x2
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_DOE_RST_DEV0F0__SHIFT                                                     0x8
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_BUSY_CLR_NORSP_DEV0F0__SHIFT                                              0x9
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_ASYNC_MSG_STS_DEV0F0__SHIFT                                               0xa
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_BUSY_CLR_ABORT_DEV0F0_MASK                                                0x00000001L
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_ERR_SET_DEV0F0_MASK                                                       0x00000002L
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_DATA_REMAIN_DEV0F0_MASK                                                   0x000000FCL
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_DOE_RST_DEV0F0_MASK                                                       0x00000100L
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_BUSY_CLR_NORSP_DEV0F0_MASK                                                0x00000200L
+#define DOE_PRV_CMD_DEV0F0__DOE_PRV_ASYNC_MSG_STS_DEV0F0_MASK                                                 0x00000400L
+//DOE_PRV_WDATA_DEV0F0
+#define DOE_PRV_WDATA_DEV0F0__DOE_PRV_DATA_PUSH_DEV0F0__SHIFT                                                 0x0
+#define DOE_PRV_WDATA_DEV0F0__DOE_PRV_DATA_PUSH_DEV0F0_MASK                                                   0xFFFFFFFFL
+//DOE_PRV_STS_DEV0F0
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_WRITE_OVER_THRESHOLD_DEV0F0__SHIFT                                        0x0
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_READ_UNDER_THRESHOLD_DEV0F0__SHIFT                                        0x1
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_ABORT_DET_DEV0F0__SHIFT                                                   0x2
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_GO_DET_DEV0F0__SHIFT                                                      0x3
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_PSN_DET_DEV0F0__SHIFT                                                     0x4
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_UNDERFLOW_DET_DEV0F0__SHIFT                                               0x5
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_OCCUP_WMBOX_DEV0F0__SHIFT                                                 0x6
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_VACA_RMBOX_DEV0F0__SHIFT                                                  0xc
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_WRITE_OVER_THRESHOLD_DEV0F0_MASK                                          0x00000001L
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_READ_UNDER_THRESHOLD_DEV0F0_MASK                                          0x00000002L
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_ABORT_DET_DEV0F0_MASK                                                     0x00000004L
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_GO_DET_DEV0F0_MASK                                                        0x00000008L
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_PSN_DET_DEV0F0_MASK                                                       0x00000010L
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_UNDERFLOW_DET_DEV0F0_MASK                                                 0x00000020L
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_OCCUP_WMBOX_DEV0F0_MASK                                                   0x00000FC0L
+#define DOE_PRV_STS_DEV0F0__DOE_PRV_VACA_RMBOX_DEV0F0_MASK                                                    0x0003F000L
+//DOE_PRV_RDATA_DEV0F0
+#define DOE_PRV_RDATA_DEV0F0__DOE_PRV_RDATA_WMBOX_DEV0F0__SHIFT                                               0x0
+#define DOE_PRV_RDATA_DEV0F0__DOE_PRV_RDATA_WMBOX_DEV0F0_MASK                                                 0xFFFFFFFFL
+//NBIF_RC_INTR_CNTL
+#define NBIF_RC_INTR_CNTL__RC_ERR_INTR_TRIG_DEV0__SHIFT                                                       0x0
+#define NBIF_RC_INTR_CNTL__RC_ERR_INTR_TRIG_DEV0_MASK                                                         0x00000001L
+//BIF_DEV0_PF0_TDISP_CNTL
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_TDISP_STATE__SHIFT                                              0x0
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_LOCK_INTERFACE_REQUEST__SHIFT                                   0x4
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_START_INTERFACE_REQUEST__SHIFT                                  0x5
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_STOP_INTERFACE_REQUEST__SHIFT                                   0x7
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_INTERFACE_ERROR__SHIFT                                          0x8
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_TDISP_STATE_MASK                                                0x00000007L
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_LOCK_INTERFACE_REQUEST_MASK                                     0x00000010L
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_START_INTERFACE_REQUEST_MASK                                    0x00000020L
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_STOP_INTERFACE_REQUEST_MASK                                     0x00000080L
+#define BIF_DEV0_PF0_TDISP_CNTL__BIF_DEV0_PF0_INTERFACE_ERROR_MASK                                            0x00000100L
+//BIF_DEV0_PF1_TDISP_CNTL
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_TDISP_STATE__SHIFT                                              0x0
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_LOCK_INTERFACE_REQUEST__SHIFT                                   0x4
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_START_INTERFACE_REQUEST__SHIFT                                  0x5
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_STOP_INTERFACE_REQUEST__SHIFT                                   0x7
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_INTERFACE_ERROR__SHIFT                                          0x8
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_TDISP_STATE_MASK                                                0x00000007L
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_LOCK_INTERFACE_REQUEST_MASK                                     0x00000010L
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_START_INTERFACE_REQUEST_MASK                                    0x00000020L
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_STOP_INTERFACE_REQUEST_MASK                                     0x00000080L
+#define BIF_DEV0_PF1_TDISP_CNTL__BIF_DEV0_PF1_INTERFACE_ERROR_MASK                                            0x00000100L
+//BIF_DEV0_PF0_VF0_TDISP_CNTL
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_TDISP_STATE__SHIFT                                      0x0
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_LOCK_INTERFACE_REQUEST__SHIFT                           0x4
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_START_INTERFACE_REQUEST__SHIFT                          0x5
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_STOP_INTERFACE_REQUEST__SHIFT                           0x7
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_INTERFACE_ERROR__SHIFT                                  0x8
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_TDISP_STATE_MASK                                        0x00000007L
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_LOCK_INTERFACE_REQUEST_MASK                             0x00000010L
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_START_INTERFACE_REQUEST_MASK                            0x00000020L
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_STOP_INTERFACE_REQUEST_MASK                             0x00000080L
+#define BIF_DEV0_PF0_VF0_TDISP_CNTL__BIF_DEV0_PF0_VF0_INTERFACE_ERROR_MASK                                    0x00000100L
+//BIF_DEV0_PF0_VF1_TDISP_CNTL
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_TDISP_STATE__SHIFT                                      0x0
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_LOCK_INTERFACE_REQUEST__SHIFT                           0x4
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_START_INTERFACE_REQUEST__SHIFT                          0x5
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_STOP_INTERFACE_REQUEST__SHIFT                           0x7
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_INTERFACE_ERROR__SHIFT                                  0x8
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_TDISP_STATE_MASK                                        0x00000007L
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_LOCK_INTERFACE_REQUEST_MASK                             0x00000010L
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_START_INTERFACE_REQUEST_MASK                            0x00000020L
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_STOP_INTERFACE_REQUEST_MASK                             0x00000080L
+#define BIF_DEV0_PF0_VF1_TDISP_CNTL__BIF_DEV0_PF0_VF1_INTERFACE_ERROR_MASK                                    0x00000100L
+//BIF_DEV0_PF0_VF2_TDISP_CNTL
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_TDISP_STATE__SHIFT                                      0x0
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_LOCK_INTERFACE_REQUEST__SHIFT                           0x4
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_START_INTERFACE_REQUEST__SHIFT                          0x5
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_STOP_INTERFACE_REQUEST__SHIFT                           0x7
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_INTERFACE_ERROR__SHIFT                                  0x8
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_TDISP_STATE_MASK                                        0x00000007L
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_LOCK_INTERFACE_REQUEST_MASK                             0x00000010L
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_START_INTERFACE_REQUEST_MASK                            0x00000020L
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_STOP_INTERFACE_REQUEST_MASK                             0x00000080L
+#define BIF_DEV0_PF0_VF2_TDISP_CNTL__BIF_DEV0_PF0_VF2_INTERFACE_ERROR_MASK                                    0x00000100L
+//BIF_DEV0_PF0_VF3_TDISP_CNTL
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_TDISP_STATE__SHIFT                                      0x0
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_LOCK_INTERFACE_REQUEST__SHIFT                           0x4
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_START_INTERFACE_REQUEST__SHIFT                          0x5
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_STOP_INTERFACE_REQUEST__SHIFT                           0x7
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_INTERFACE_ERROR__SHIFT                                  0x8
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_TDISP_STATE_MASK                                        0x00000007L
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_LOCK_INTERFACE_REQUEST_MASK                             0x00000010L
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_START_INTERFACE_REQUEST_MASK                            0x00000020L
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_STOP_INTERFACE_REQUEST_MASK                             0x00000080L
+#define BIF_DEV0_PF0_VF3_TDISP_CNTL__BIF_DEV0_PF0_VF3_INTERFACE_ERROR_MASK                                    0x00000100L
+//BIF_DEV0_PF0_VF4_TDISP_CNTL
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_TDISP_STATE__SHIFT                                      0x0
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_LOCK_INTERFACE_REQUEST__SHIFT                           0x4
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_START_INTERFACE_REQUEST__SHIFT                          0x5
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_STOP_INTERFACE_REQUEST__SHIFT                           0x7
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_INTERFACE_ERROR__SHIFT                                  0x8
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_TDISP_STATE_MASK                                        0x00000007L
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_LOCK_INTERFACE_REQUEST_MASK                             0x00000010L
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_START_INTERFACE_REQUEST_MASK                            0x00000020L
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_STOP_INTERFACE_REQUEST_MASK                             0x00000080L
+#define BIF_DEV0_PF0_VF4_TDISP_CNTL__BIF_DEV0_PF0_VF4_INTERFACE_ERROR_MASK                                    0x00000100L
+//BIF_DEV0_PF0_VF5_TDISP_CNTL
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_TDISP_STATE__SHIFT                                      0x0
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_LOCK_INTERFACE_REQUEST__SHIFT                           0x4
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_START_INTERFACE_REQUEST__SHIFT                          0x5
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_STOP_INTERFACE_REQUEST__SHIFT                           0x7
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_INTERFACE_ERROR__SHIFT                                  0x8
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_TDISP_STATE_MASK                                        0x00000007L
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_LOCK_INTERFACE_REQUEST_MASK                             0x00000010L
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_START_INTERFACE_REQUEST_MASK                            0x00000020L
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_STOP_INTERFACE_REQUEST_MASK                             0x00000080L
+#define BIF_DEV0_PF0_VF5_TDISP_CNTL__BIF_DEV0_PF0_VF5_INTERFACE_ERROR_MASK                                    0x00000100L
+//BIF_DEV0_PF0_VF6_TDISP_CNTL
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_TDISP_STATE__SHIFT                                      0x0
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_LOCK_INTERFACE_REQUEST__SHIFT                           0x4
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_START_INTERFACE_REQUEST__SHIFT                          0x5
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_STOP_INTERFACE_REQUEST__SHIFT                           0x7
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_INTERFACE_ERROR__SHIFT                                  0x8
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_TDISP_STATE_MASK                                        0x00000007L
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_LOCK_INTERFACE_REQUEST_MASK                             0x00000010L
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_START_INTERFACE_REQUEST_MASK                            0x00000020L
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_STOP_INTERFACE_REQUEST_MASK                             0x00000080L
+#define BIF_DEV0_PF0_VF6_TDISP_CNTL__BIF_DEV0_PF0_VF6_INTERFACE_ERROR_MASK                                    0x00000100L
+//BIF_DEV0_PF0_VF7_TDISP_CNTL
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_TDISP_STATE__SHIFT                                      0x0
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_LOCK_INTERFACE_REQUEST__SHIFT                           0x4
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_START_INTERFACE_REQUEST__SHIFT                          0x5
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_STOP_INTERFACE_REQUEST__SHIFT                           0x7
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_INTERFACE_ERROR__SHIFT                                  0x8
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_TDISP_STATE_MASK                                        0x00000007L
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_LOCK_INTERFACE_REQUEST_MASK                             0x00000010L
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_START_INTERFACE_REQUEST_MASK                            0x00000020L
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_STOP_INTERFACE_REQUEST_MASK                             0x00000080L
+#define BIF_DEV0_PF0_VF7_TDISP_CNTL__BIF_DEV0_PF0_VF7_INTERFACE_ERROR_MASK                                    0x00000100L
+//BIF_DEV0_PF1_VF0_TDISP_CNTL
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_TDISP_STATE__SHIFT                                      0x0
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_LOCK_INTERFACE_REQUEST__SHIFT                           0x4
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_START_INTERFACE_REQUEST__SHIFT                          0x5
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_STOP_INTERFACE_REQUEST__SHIFT                           0x7
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_INTERFACE_ERROR__SHIFT                                  0x8
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_TDISP_STATE_MASK                                        0x00000007L
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_LOCK_INTERFACE_REQUEST_MASK                             0x00000010L
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_START_INTERFACE_REQUEST_MASK                            0x00000020L
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_STOP_INTERFACE_REQUEST_MASK                             0x00000080L
+#define BIF_DEV0_PF1_VF0_TDISP_CNTL__BIF_DEV0_PF1_VF0_INTERFACE_ERROR_MASK                                    0x00000100L
+//BIF_TDISP_ERROR_DEV0_PF0_INST0
+#define BIF_TDISP_ERROR_DEV0_PF0_INST0__BIF_TDISP_ERROR_DEV0_PF0_INST0__SHIFT                                 0x0
+#define BIF_TDISP_ERROR_DEV0_PF0_INST0__BIF_TDISP_ERROR_DEV0_PF0_INST0_MASK                                   0xFFFFFFFFL
+//BIF_TDISP_ERROR_DEV0_PF1_INST0
+#define BIF_TDISP_ERROR_DEV0_PF1_INST0__BIF_TDISP_ERROR_DEV0_PF1_INST0__SHIFT                                 0x0
+#define BIF_TDISP_ERROR_DEV0_PF1_INST0__BIF_TDISP_ERROR_DEV0_PF1_INST0_MASK                                   0xFFFFFFFFL
+//BIF_TDISP_UNLOCKED_DEV0_PF0_INST0
+#define BIF_TDISP_UNLOCKED_DEV0_PF0_INST0__BIF_TDISP_UNLOCKED_DEV0_PF0_INST0__SHIFT                           0x0
+#define BIF_TDISP_UNLOCKED_DEV0_PF0_INST0__BIF_TDISP_UNLOCKED_DEV0_PF0_INST0_MASK                             0xFFFFFFFFL
+//BIF_TDISP_UNLOCKED_DEV0_PF1_INST0
+#define BIF_TDISP_UNLOCKED_DEV0_PF1_INST0__BIF_TDISP_UNLOCKED_DEV0_PF1_INST0__SHIFT                           0x0
+#define BIF_TDISP_UNLOCKED_DEV0_PF1_INST0__BIF_TDISP_UNLOCKED_DEV0_PF1_INST0_MASK                             0xFFFFFFFFL
+//BIF_TDISP_MISC_CNTL_DEV0
+#define BIF_TDISP_MISC_CNTL_DEV0__BIF_RC_CONFIG_MONITOR_EN_DEV0__SHIFT                                        0x0
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_ERR_NOT_IN_UNLOCK_FOR_VF_DEV0_PF0__SHIFT                            0x5
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_UNLOCK_NOT_IN_ERR_FOR_VF_DEV0_PF0__SHIFT                            0x6
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_ERR_FOR_VF_DEV0_PF0__SHIFT                                          0x7
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_UNLOCK_FOR_VF_DEV0_PF0__SHIFT                                       0x8
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_ERR_NOT_IN_UNLOCK_FOR_VF_DEV0_PF1__SHIFT                            0x9
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_UNLOCK_NOT_IN_ERR_FOR_VF_DEV0_PF1__SHIFT                            0xa
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_ERR_FOR_VF_DEV0_PF1__SHIFT                                          0xb
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_UNLOCK_FOR_VF_DEV0_PF1__SHIFT                                       0xc
+#define BIF_TDISP_MISC_CNTL_DEV0__BIF_RC_CONFIG_MONITOR_EN_DEV0_MASK                                          0x00000001L
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_ERR_NOT_IN_UNLOCK_FOR_VF_DEV0_PF0_MASK                              0x00000020L
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_UNLOCK_NOT_IN_ERR_FOR_VF_DEV0_PF0_MASK                              0x00000040L
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_ERR_FOR_VF_DEV0_PF0_MASK                                            0x00000080L
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_UNLOCK_FOR_VF_DEV0_PF0_MASK                                         0x00000100L
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_ERR_NOT_IN_UNLOCK_FOR_VF_DEV0_PF1_MASK                              0x00000200L
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_UNLOCK_NOT_IN_ERR_FOR_VF_DEV0_PF1_MASK                              0x00000400L
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_ERR_FOR_VF_DEV0_PF1_MASK                                            0x00000800L
+#define BIF_TDISP_MISC_CNTL_DEV0__TRIG_TO_UNLOCK_FOR_VF_DEV0_PF1_MASK                                         0x00001000L
+//BIF_TDISP_MISC_CNTL
+#define BIF_TDISP_MISC_CNTL__BIF_MSI_MSIX_IDE_TBIT_OVERRIDE__SHIFT                                            0x0
+#define BIF_TDISP_MISC_CNTL__REMOTE_TDISP_STATE_VECTOR_CLR__SHIFT                                             0x1
+#define BIF_TDISP_MISC_CNTL__NBIF_GMI_TDISP_REMOTE_TX_EN__SHIFT                                               0x3
+#define BIF_TDISP_MISC_CNTL__BIF_TRUSTED_IO_ERR_Intr_0_MASK__SHIFT                                            0x4
+#define BIF_TDISP_MISC_CNTL__DSPT_CFG_RST_TRIG_TO_UNLOCK_MASK__SHIFT                                          0xa
+#define BIF_TDISP_MISC_CNTL__EP_CFG_RST_TRIG_TO_UNLOCK_MASK__SHIFT                                            0xb
+#define BIF_TDISP_MISC_CNTL__TDISP_ERR_AER_DIS__SHIFT                                                         0xc
+#define BIF_TDISP_MISC_CNTL__TDISP_ERR_PRGR_AER_EN__SHIFT                                                     0xd
+#define BIF_TDISP_MISC_CNTL__BIF_TBIT_FOR_RUN_BIH_MEM__SHIFT                                                  0xe
+#define BIF_TDISP_MISC_CNTL__BIF_XT1_FOR_RUN_BIH_MEM__SHIFT                                                   0xf
+#define BIF_TDISP_MISC_CNTL__BIF_MSI_MSIX_IDE_TBIT_OVERRIDE_MASK                                              0x00000001L
+#define BIF_TDISP_MISC_CNTL__REMOTE_TDISP_STATE_VECTOR_CLR_MASK                                               0x00000002L
+#define BIF_TDISP_MISC_CNTL__NBIF_GMI_TDISP_REMOTE_TX_EN_MASK                                                 0x00000008L
+#define BIF_TDISP_MISC_CNTL__BIF_TRUSTED_IO_ERR_Intr_0_MASK_MASK                                              0x00000010L
+#define BIF_TDISP_MISC_CNTL__DSPT_CFG_RST_TRIG_TO_UNLOCK_MASK_MASK                                            0x00000400L
+#define BIF_TDISP_MISC_CNTL__EP_CFG_RST_TRIG_TO_UNLOCK_MASK_MASK                                              0x00000800L
+#define BIF_TDISP_MISC_CNTL__TDISP_ERR_AER_DIS_MASK                                                           0x00001000L
+#define BIF_TDISP_MISC_CNTL__TDISP_ERR_PRGR_AER_EN_MASK                                                       0x00002000L
+#define BIF_TDISP_MISC_CNTL__BIF_TBIT_FOR_RUN_BIH_MEM_MASK                                                    0x00004000L
+#define BIF_TDISP_MISC_CNTL__BIF_XT1_FOR_RUN_BIH_MEM_MASK                                                     0x00008000L
+//BIF_RCC_IDE_TDISP_CNTL
+#define BIF_RCC_IDE_TDISP_CNTL__BIF_RCC_IDE_TDISP_PRGR_CHECK_EN__SHIFT                                        0x0
+#define BIF_RCC_IDE_TDISP_CNTL__BIF_RCC_IDE_TDISP_PRGR_FOR_UNLOCK_CHECK_EN__SHIFT                             0x1
+#define BIF_RCC_IDE_TDISP_CNTL__BIF_RCC_IDE_CHECK_EN__SHIFT                                                   0x2
+#define BIF_RCC_IDE_TDISP_CNTL__BIF_RCC_IDE_TDISP_PRGR_CHECK_EN_MASK                                          0x00000001L
+#define BIF_RCC_IDE_TDISP_CNTL__BIF_RCC_IDE_TDISP_PRGR_FOR_UNLOCK_CHECK_EN_MASK                               0x00000002L
+#define BIF_RCC_IDE_TDISP_CNTL__BIF_RCC_IDE_CHECK_EN_MASK                                                     0x00000004L
+//BIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_31
+#define BIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF__SHIFT                                  0x0
+#define BIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF_MASK                                    0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_0_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_0_BOUND_PF1_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_0_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_0_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_31
+#define BIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF__SHIFT                                  0x0
+#define BIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF_MASK                                    0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_1_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_1_BOUND_PF1_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_1_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_1_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_31
+#define BIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF__SHIFT                                  0x0
+#define BIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF_MASK                                    0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_2_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_2_BOUND_PF1_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_2_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_2_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_31
+#define BIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF__SHIFT                                  0x0
+#define BIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF_MASK                                    0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_3_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_3_BOUND_PF1_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_3_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_3_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_31
+#define BIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF__SHIFT                                  0x0
+#define BIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF_MASK                                    0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_4_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_4_BOUND_PF1_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_4_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_4_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_31
+#define BIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF__SHIFT                                  0x0
+#define BIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF_MASK                                    0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_5_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_5_BOUND_PF1_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_5_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_5_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_31
+#define BIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF__SHIFT                                  0x0
+#define BIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF_MASK                                    0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_6_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_6_BOUND_PF1_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_6_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_6_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_31
+#define BIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF__SHIFT                                  0x0
+#define BIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF_MASK                                    0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_7_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_7_BOUND_PF1_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_7_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_7_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_31
+#define BIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF__SHIFT                                  0x0
+#define BIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_31__SELECTIVE_IDE_BOUND_PF_MASK                                    0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_8_BOUND_PF0_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_DEV0_SELECTIVE_IDE_8_BOUND_PF1_VF0_31
+#define BIF_DEV0_SELECTIVE_IDE_8_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF__SHIFT                              0x0
+#define BIF_DEV0_SELECTIVE_IDE_8_BOUND_PF1_VF0_31__SELECTIVE_IDE_BOUND_VF_MASK                                0xFFFFFFFFL
+//BIF_SYSTEM_EN
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0__SHIFT                                                           0x0
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF0__SHIFT                                                       0x1
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF1__SHIFT                                                       0x2
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF2__SHIFT                                                       0x3
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF3__SHIFT                                                       0x4
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF4__SHIFT                                                       0x5
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF5__SHIFT                                                       0x6
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF6__SHIFT                                                       0x7
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF7__SHIFT                                                       0x8
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_MASK                                                             0x00000001L
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF0_MASK                                                         0x00000002L
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF1_MASK                                                         0x00000004L
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF2_MASK                                                         0x00000008L
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF3_MASK                                                         0x00000010L
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF4_MASK                                                         0x00000020L
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF5_MASK                                                         0x00000040L
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF6_MASK                                                         0x00000080L
+#define BIF_SYSTEM_EN__BIF_SYSTEM_EN_DEV0_F0_VF7_MASK                                                         0x00000100L
+//BIF_MASK_SYSTEM_EN
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0__SHIFT                                                 0x0
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF0__SHIFT                                             0x1
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF1__SHIFT                                             0x2
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF2__SHIFT                                             0x3
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF3__SHIFT                                             0x4
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF4__SHIFT                                             0x5
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF5__SHIFT                                             0x6
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF6__SHIFT                                             0x7
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF7__SHIFT                                             0x8
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_MASK                                                   0x00000001L
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF0_MASK                                               0x00000002L
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF1_MASK                                               0x00000004L
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF2_MASK                                               0x00000008L
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF3_MASK                                               0x00000010L
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF4_MASK                                               0x00000020L
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF5_MASK                                               0x00000040L
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF6_MASK                                               0x00000080L
+#define BIF_MASK_SYSTEM_EN__BIF_SYSTEM_EN_MASK_DEV0_F0_VF7_MASK                                               0x00000100L
+//GPUIOV_SCH0
+#define GPUIOV_SCH0__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH0__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH1
+#define GPUIOV_SCH1__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH1__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH2
+#define GPUIOV_SCH2__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH2__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH3
+#define GPUIOV_SCH3__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH3__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH4
+#define GPUIOV_SCH4__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH4__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH5
+#define GPUIOV_SCH5__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH5__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH6
+#define GPUIOV_SCH6__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH6__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH7
+#define GPUIOV_SCH7__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH7__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH8
+#define GPUIOV_SCH8__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH8__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH9
+#define GPUIOV_SCH9__REMAP_ADDR__SHIFT                                                                        0x0
+#define GPUIOV_SCH9__REMAP_ADDR_MASK                                                                          0x001FFFFFL
+//GPUIOV_SCH10
+#define GPUIOV_SCH10__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH10__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_SCH11
+#define GPUIOV_SCH11__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH11__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_SCH12
+#define GPUIOV_SCH12__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH12__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_SCH13
+#define GPUIOV_SCH13__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH13__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_SCH14
+#define GPUIOV_SCH14__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH14__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_SCH15
+#define GPUIOV_SCH15__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH15__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_SCH16
+#define GPUIOV_SCH16__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH16__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_SCH17
+#define GPUIOV_SCH17__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH17__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_SCH18
+#define GPUIOV_SCH18__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH18__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_SCH19
+#define GPUIOV_SCH19__REMAP_ADDR__SHIFT                                                                       0x0
+#define GPUIOV_SCH19__REMAP_ADDR_MASK                                                                         0x001FFFFFL
+//GPUIOV_MMHUB_P2P_CNTL
+#define GPUIOV_MMHUB_P2P_CNTL__GPUIOV_MMHUB_P2P_ENABLE_REMAP_ADDR__SHIFT                                      0x0
+#define GPUIOV_MMHUB_P2P_CNTL__GPUIOV_MMHUB_P2P_ENABLE_REMAP_ADDR_MASK                                        0x000FFFFFL
+//BIF_AER_INTR_CTRL
+#define BIF_AER_INTR_CTRL__BIF_AER_INTR_MASK__SHIFT                                                           0x0
+#define BIF_AER_INTR_CTRL__BIF_AER_SMN_POSTED__SHIFT                                                          0x1
+#define BIF_AER_INTR_CTRL__BIF_AER_SMN_WDATA_SEL__SHIFT                                                       0x2
+#define BIF_AER_INTR_CTRL__BIF_AER_INTR_MASK_MASK                                                             0x00000001L
+#define BIF_AER_INTR_CTRL__BIF_AER_SMN_POSTED_MASK                                                            0x00000002L
+#define BIF_AER_INTR_CTRL__BIF_AER_SMN_WDATA_SEL_MASK                                                         0x00000004L
+//BIF_AER_INTR_STS
+#define BIF_AER_INTR_STS__BIF_AER_INTR_STS__SHIFT                                                             0x0
+#define BIF_AER_INTR_STS__BIF_AER_INTR_STS_MASK                                                               0x00000001L
+//BIF_PCIE_MSG_CTRL
+#define BIF_PCIE_MSG_CTRL__BIF_PCIE_MSG_VALID__SHIFT                                                          0x0
+#define BIF_PCIE_MSG_CTRL__BIF_PCIE_MSG_PORTID__SHIFT                                                         0x1
+#define BIF_PCIE_MSG_CTRL__BIF_PCIE_MSG_VF_VALID__SHIFT                                                       0x3
+#define BIF_PCIE_MSG_CTRL__BIF_PCIE_MSG_VFID__SHIFT                                                           0x4
+#define BIF_PCIE_MSG_CTRL__BIF_PCIE_MSG_VALID_MASK                                                            0x00000001L
+#define BIF_PCIE_MSG_CTRL__BIF_PCIE_MSG_PORTID_MASK                                                           0x00000006L
+#define BIF_PCIE_MSG_CTRL__BIF_PCIE_MSG_VF_VALID_MASK                                                         0x00000008L
+#define BIF_PCIE_MSG_CTRL__BIF_PCIE_MSG_VFID_MASK                                                             0x00003FF0L
+//BIF_PCIE_MSG_HEADER_DW0
+#define BIF_PCIE_MSG_HEADER_DW0__BIF_PCIE_MSG_HEADER_DW0__SHIFT                                               0x0
+#define BIF_PCIE_MSG_HEADER_DW0__BIF_PCIE_MSG_HEADER_DW0_MASK                                                 0xFFFFFFFFL
+//BIF_PCIE_MSG_HEADER_DW1
+#define BIF_PCIE_MSG_HEADER_DW1__BIF_PCIE_MSG_HEADER_DW1__SHIFT                                               0x0
+#define BIF_PCIE_MSG_HEADER_DW1__BIF_PCIE_MSG_HEADER_DW1_MASK                                                 0xFFFFFFFFL
+//BIF_PCIE_MSG_HEADER_DW2
+#define BIF_PCIE_MSG_HEADER_DW2__BIF_PCIE_MSG_HEADER_DW2__SHIFT                                               0x0
+#define BIF_PCIE_MSG_HEADER_DW2__BIF_PCIE_MSG_HEADER_DW2_MASK                                                 0xFFFFFFFFL
+//BIF_PCIE_MSG_HEADER_DW3
+#define BIF_PCIE_MSG_HEADER_DW3__BIF_PCIE_MSG_HEADER_DW3__SHIFT                                               0x0
+#define BIF_PCIE_MSG_HEADER_DW3__BIF_PCIE_MSG_HEADER_DW3_MASK                                                 0xFFFFFFFFL
+//BIF_PCIE_MSG_DATA_DW0
+#define BIF_PCIE_MSG_DATA_DW0__BIF_PCIE_MSG_DATA_DW0__SHIFT                                                   0x0
+#define BIF_PCIE_MSG_DATA_DW0__BIF_PCIE_MSG_DATA_DW0_MASK                                                     0xFFFFFFFFL
+//BIF_PCIE_MSG_DATA_DW1
+#define BIF_PCIE_MSG_DATA_DW1__BIF_PCIE_MSG_DATA_DW1__SHIFT                                                   0x0
+#define BIF_PCIE_MSG_DATA_DW1__BIF_PCIE_MSG_DATA_DW1_MASK                                                     0xFFFFFFFFL
+//BIF_PCIE_MSG_DATA_DW2
+#define BIF_PCIE_MSG_DATA_DW2__BIF_PCIE_MSG_DATA_DW2__SHIFT                                                   0x0
+#define BIF_PCIE_MSG_DATA_DW2__BIF_PCIE_MSG_DATA_DW2_MASK                                                     0xFFFFFFFFL
+//BIF_PCIE_MSG_DATA_DW3
+#define BIF_PCIE_MSG_DATA_DW3__BIF_PCIE_MSG_DATA_DW3__SHIFT                                                   0x0
+#define BIF_PCIE_MSG_DATA_DW3__BIF_PCIE_MSG_DATA_DW3_MASK                                                     0xFFFFFFFFL
+//BIFC_SEC_AER_SMN_CTRL0
+#define BIFC_SEC_AER_SMN_CTRL0__BIF_AER_INT_REQ_ADDR__SHIFT                                                   0x0
+#define BIFC_SEC_AER_SMN_CTRL0__BIF_AER_INT_REQ_APER_ID__SHIFT                                                0x14
+#define BIFC_SEC_AER_SMN_CTRL0__BIF_AER_INT_REQ_ADDR_MASK                                                     0x000FFFFFL
+#define BIFC_SEC_AER_SMN_CTRL0__BIF_AER_INT_REQ_APER_ID_MASK                                                  0xFFF00000L
+//BIFC_SEC_AER_SMN_CTRL1
+#define BIFC_SEC_AER_SMN_CTRL1__BIF_AER_INT_REQ_MCM_ADDR__SHIFT                                               0x0
+#define BIFC_SEC_AER_SMN_CTRL1__BIF_AER_SMN_SEC_SEL__SHIFT                                                    0x10
+#define BIFC_SEC_AER_SMN_CTRL1__BIF_AER_INT_REQ_MCM_ADDR_MASK                                                 0x0000FFFFL
+#define BIFC_SEC_AER_SMN_CTRL1__BIF_AER_SMN_SEC_SEL_MASK                                                      0x000F0000L
+//BIFC_SEC_AER_SMN_CTRL2
+#define BIFC_SEC_AER_SMN_CTRL2__BIF_AER_INT_REQ_WDATA__SHIFT                                                  0x0
+#define BIFC_SEC_AER_SMN_CTRL2__BIF_AER_INT_REQ_WDATA_MASK                                                    0xFFFFFFFFL
+//BIF_PCIE_ERR_CTRL0
+#define BIF_PCIE_ERR_CTRL0__BIF_PCIE_ERR_VF_NUM__SHIFT                                                        0x0
+#define BIF_PCIE_ERR_CTRL0__BIF_PCIE_ERR_VF__SHIFT                                                            0x7
+#define BIF_PCIE_ERR_CTRL0__BIF_PCIE_ERR_PF_NUM__SHIFT                                                        0x8
+#define BIF_PCIE_ERR_CTRL0__BIF_PCIE_ERR_VF_NUM_MASK                                                          0x0000007FL
+#define BIF_PCIE_ERR_CTRL0__BIF_PCIE_ERR_VF_MASK                                                              0x00000080L
+#define BIF_PCIE_ERR_CTRL0__BIF_PCIE_ERR_PF_NUM_MASK                                                          0x00000700L
+//BIF_PCIE_ERR_CTRL1
+#define BIF_PCIE_ERR_CTRL1__BIF_CORR_ERR_EN__SHIFT                                                            0x0
+#define BIF_PCIE_ERR_CTRL1__BIF_NON_FATAL_ERR_EN__SHIFT                                                       0x1
+#define BIF_PCIE_ERR_CTRL1__BIF_FATAL_ERR_EN__SHIFT                                                           0x2
+#define BIF_PCIE_ERR_CTRL1__BIF_USR_REPORT_EN__SHIFT                                                          0x3
+#define BIF_PCIE_ERR_CTRL1__BIF_SERR_EN__SHIFT                                                                0x4
+#define BIF_PCIE_ERR_CTRL1__BIF_MASTER_DATA_PARITY_ERROR__SHIFT                                               0x8
+#define BIF_PCIE_ERR_CTRL1__BIF_SIGNALED_SYSTEM_ERROR__SHIFT                                                  0x9
+#define BIF_PCIE_ERR_CTRL1__BIF_PARITY_ERROR_DETECTED__SHIFT                                                  0xa
+#define BIF_PCIE_ERR_CTRL1__BIF_PCIE_DEVICE_STATUS__SHIFT                                                     0x10
+#define BIF_PCIE_ERR_CTRL1__BIF_CORR_ERR_EN_MASK                                                              0x00000001L
+#define BIF_PCIE_ERR_CTRL1__BIF_NON_FATAL_ERR_EN_MASK                                                         0x00000002L
+#define BIF_PCIE_ERR_CTRL1__BIF_FATAL_ERR_EN_MASK                                                             0x00000004L
+#define BIF_PCIE_ERR_CTRL1__BIF_USR_REPORT_EN_MASK                                                            0x00000008L
+#define BIF_PCIE_ERR_CTRL1__BIF_SERR_EN_MASK                                                                  0x00000010L
+#define BIF_PCIE_ERR_CTRL1__BIF_MASTER_DATA_PARITY_ERROR_MASK                                                 0x00000100L
+#define BIF_PCIE_ERR_CTRL1__BIF_SIGNALED_SYSTEM_ERROR_MASK                                                    0x00000200L
+#define BIF_PCIE_ERR_CTRL1__BIF_PARITY_ERROR_DETECTED_MASK                                                    0x00000400L
+#define BIF_PCIE_ERR_CTRL1__BIF_PCIE_DEVICE_STATUS_MASK                                                       0xFFFF0000L
+//BIF_PCIE_ERR_CTRL2
+#define BIF_PCIE_ERR_CTRL2__BIF_AER_FEP__SHIFT                                                                0x0
+#define BIF_PCIE_ERR_CTRL2__BIF_AER_TLP_PREFIX_LOG_PRESENT__SHIFT                                             0xb
+#define BIF_PCIE_ERR_CTRL2__BIF_AER_LOGGED_TLP_SIZE__SHIFT                                                    0x13
+#define BIF_PCIE_ERR_CTRL2__BIF_AER_FEP_MASK                                                                  0x0000001FL
+#define BIF_PCIE_ERR_CTRL2__BIF_AER_TLP_PREFIX_LOG_PRESENT_MASK                                               0x00000800L
+#define BIF_PCIE_ERR_CTRL2__BIF_AER_LOGGED_TLP_SIZE_MASK                                                      0x00F80000L
+//BIF_AER_UNCORR_ERR_STATUS
+#define BIF_AER_UNCORR_ERR_STATUS__BIF_AER_UNCORR_ERR_STATUS__SHIFT                                           0x0
+#define BIF_AER_UNCORR_ERR_STATUS__BIF_AER_UNCORR_ERR_STATUS_MASK                                             0xFFFFFFFFL
+//BIF_AER_UNCORR_ERR_MASK
+#define BIF_AER_UNCORR_ERR_MASK__BIF_AER_UNCORR_ERR_MASK__SHIFT                                               0x0
+#define BIF_AER_UNCORR_ERR_MASK__BIF_AER_UNCORR_ERR_MASK_MASK                                                 0xFFFFFFFFL
+//BIF_AER_UNCORR_ERR_SEVERITY
+#define BIF_AER_UNCORR_ERR_SEVERITY__BIF_AER_UNCORR_ERR_SEVERITY__SHIFT                                       0x0
+#define BIF_AER_UNCORR_ERR_SEVERITY__BIF_AER_UNCORR_ERR_SEVERITY_MASK                                         0xFFFFFFFFL
+//BIF_AER_CORR_ERR_STATUS
+#define BIF_AER_CORR_ERR_STATUS__BIF_AER_CORR_ERR_STATUS__SHIFT                                               0x0
+#define BIF_AER_CORR_ERR_STATUS__BIF_AER_CORR_ERR_STATUS_MASK                                                 0xFFFFFFFFL
+//BIF_AER_CORR_ERR_MASK
+#define BIF_AER_CORR_ERR_MASK__BIF_AER_CORR_ERR_MASK__SHIFT                                                   0x0
+#define BIF_AER_CORR_ERR_MASK__BIF_AER_CORR_ERR_MASK_MASK                                                     0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_0
+#define RCC_EP_STICKY_RESTORE_0__RESTORE_PCIE_TLP_HDR_0__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_0__RESTORE_PCIE_TLP_HDR_0_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_1
+#define RCC_EP_STICKY_RESTORE_1__RESTORE_PCIE_TLP_HDR_1__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_1__RESTORE_PCIE_TLP_HDR_1_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_2
+#define RCC_EP_STICKY_RESTORE_2__RESTORE_PCIE_TLP_HDR_2__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_2__RESTORE_PCIE_TLP_HDR_2_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_3
+#define RCC_EP_STICKY_RESTORE_3__RESTORE_PCIE_TLP_HDR_3__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_3__RESTORE_PCIE_TLP_HDR_3_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_4
+#define RCC_EP_STICKY_RESTORE_4__RESTORE_PCIE_TLP_HDR_4__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_4__RESTORE_PCIE_TLP_HDR_4_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_5
+#define RCC_EP_STICKY_RESTORE_5__RESTORE_PCIE_TLP_HDR_5__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_5__RESTORE_PCIE_TLP_HDR_5_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_6
+#define RCC_EP_STICKY_RESTORE_6__RESTORE_PCIE_TLP_HDR_6__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_6__RESTORE_PCIE_TLP_HDR_6_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_7
+#define RCC_EP_STICKY_RESTORE_7__RESTORE_PCIE_TLP_HDR_7__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_7__RESTORE_PCIE_TLP_HDR_7_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_8
+#define RCC_EP_STICKY_RESTORE_8__RESTORE_PCIE_TLP_HDR_8__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_8__RESTORE_PCIE_TLP_HDR_8_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_9
+#define RCC_EP_STICKY_RESTORE_9__RESTORE_PCIE_TLP_HDR_9__SHIFT                                                0x0
+#define RCC_EP_STICKY_RESTORE_9__RESTORE_PCIE_TLP_HDR_9_MASK                                                  0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_10
+#define RCC_EP_STICKY_RESTORE_10__RESTORE_PCIE_TLP_HDR_10__SHIFT                                              0x0
+#define RCC_EP_STICKY_RESTORE_10__RESTORE_PCIE_TLP_HDR_10_MASK                                                0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_11
+#define RCC_EP_STICKY_RESTORE_11__RESTORE_PCIE_TLP_HDR_11__SHIFT                                              0x0
+#define RCC_EP_STICKY_RESTORE_11__RESTORE_PCIE_TLP_HDR_11_MASK                                                0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_12
+#define RCC_EP_STICKY_RESTORE_12__RESTORE_PCIE_TLP_HDR_12__SHIFT                                              0x0
+#define RCC_EP_STICKY_RESTORE_12__RESTORE_PCIE_TLP_HDR_12_MASK                                                0xFFFFFFFFL
+//RCC_EP_STICKY_RESTORE_13
+#define RCC_EP_STICKY_RESTORE_13__RESTORE_PCIE_TLP_HDR_13__SHIFT                                              0x0
+#define RCC_EP_STICKY_RESTORE_13__RESTORE_PCIE_TLP_HDR_13_MASK                                                0xFFFFFFFFL
+//BIF_DEVICE3_STATUS
+#define BIF_DEVICE3_STATUS__BIF_DEVICE3_INITIAL_LINK_WIDTH__SHIFT                                             0x0
+#define BIF_DEVICE3_STATUS__BIF_DEVICE3_SEGMENT_CAPTURED__SHIFT                                               0x3
+#define BIF_DEVICE3_STATUS__BIF_DEVICE3_REMOTE_L0P_SUPPORTED__SHIFT                                           0x4
+#define BIF_DEVICE3_STATUS__BIF_DEVICE3_INITIAL_LINK_WIDTH_MASK                                               0x00000007L
+#define BIF_DEVICE3_STATUS__BIF_DEVICE3_SEGMENT_CAPTURED_MASK                                                 0x00000008L
+#define BIF_DEVICE3_STATUS__BIF_DEVICE3_REMOTE_L0P_SUPPORTED_MASK                                             0x00000010L
+//BIFC_SEC_MCTP_SMN_CTRL0
+#define BIFC_SEC_MCTP_SMN_CTRL0__MCTP_INT_REQ_ADDR__SHIFT                                                     0x0
+#define BIFC_SEC_MCTP_SMN_CTRL0__MCTP_INT_REQ_APER_ID__SHIFT                                                  0x14
+#define BIFC_SEC_MCTP_SMN_CTRL0__MCTP_INT_REQ_ADDR_MASK                                                       0x000FFFFFL
+#define BIFC_SEC_MCTP_SMN_CTRL0__MCTP_INT_REQ_APER_ID_MASK                                                    0xFFF00000L
+//BIFC_SEC_MCTP_SMN_CTRL1
+#define BIFC_SEC_MCTP_SMN_CTRL1__MCTP_INT_REQ_MCM_ADDR__SHIFT                                                 0x0
+#define BIFC_SEC_MCTP_SMN_CTRL1__MCTP_SMN_SEC_SEL__SHIFT                                                      0x10
+#define BIFC_SEC_MCTP_SMN_CTRL1__MCTP_INT_REQ_MCM_ADDR_MASK                                                   0x0000FFFFL
+#define BIFC_SEC_MCTP_SMN_CTRL1__MCTP_SMN_SEC_SEL_MASK                                                        0x000F0000L
+//NBIF_MCTP_GENERIC_CNTL
+#define NBIF_MCTP_GENERIC_CNTL__NBIF_MCTP_SUPPORT__SHIFT                                                      0x0
+#define NBIF_MCTP_GENERIC_CNTL__NBIF_MCTP_ROUTING_CODE_CHECK__SHIFT                                           0x1
+#define NBIF_MCTP_GENERIC_CNTL__NBIF_MCTP_RBID_CPLID_CNTL__SHIFT                                              0x2
+#define NBIF_MCTP_GENERIC_CNTL__NBIF_MCTP_SUPPORT_MASK                                                        0x00000001L
+#define NBIF_MCTP_GENERIC_CNTL__NBIF_MCTP_ROUTING_CODE_CHECK_MASK                                             0x00000002L
+#define NBIF_MCTP_GENERIC_CNTL__NBIF_MCTP_RBID_CPLID_CNTL_MASK                                                0x00000004L
+//NBIF_MCTP_GENERIC_STATUS
+#define NBIF_MCTP_GENERIC_STATUS__NBIF_MCTP_ERR_FLAG__SHIFT                                                   0x0
+#define NBIF_MCTP_GENERIC_STATUS__NBIF_MCTP_RB_EMPTY_STATUS__SHIFT                                            0x1
+#define NBIF_MCTP_GENERIC_STATUS__NBIF_MCTP_RB_FULL_STATUS__SHIFT                                             0x2
+#define NBIF_MCTP_GENERIC_STATUS__NBIF_MCTP_ERR_FLAG_MASK                                                     0x00000001L
+#define NBIF_MCTP_GENERIC_STATUS__NBIF_MCTP_RB_EMPTY_STATUS_MASK                                              0x00000002L
+#define NBIF_MCTP_GENERIC_STATUS__NBIF_MCTP_RB_FULL_STATUS_MASK                                               0x00000004L
+//SDPVW_UPDATE_EN
+#define SDPVW_UPDATE_EN__SDPVW_UPDATE_EN__SHIFT                                                               0x0
+#define SDPVW_UPDATE_EN__SWUS_SECBUSRST_SET_EN__SHIFT                                                         0x1
+#define SDPVW_UPDATE_EN__SDPVW_UPDATE_EN_MASK                                                                 0x01L
+#define SDPVW_UPDATE_EN__SWUS_SECBUSRST_SET_EN_MASK                                                           0x02L
+//REGS_ROM_OFFSET_CTRL
+#define REGS_ROM_OFFSET_CTRL__ROM_OFFSET__SHIFT                                                               0x0
+#define REGS_ROM_OFFSET_CTRL__ROM_OFFSET_MASK                                                                 0x7FL
+//BIFL_SEC_RAS_POISON_DATA_DROP_CTRL
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF0_POISON_DATA_DROP_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF1_POISON_DATA_DROP_EN__SHIFT                     0x1
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF2_POISON_DATA_DROP_EN__SHIFT                     0x2
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF3_POISON_DATA_DROP_EN__SHIFT                     0x3
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF4_POISON_DATA_DROP_EN__SHIFT                     0x4
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF5_POISON_DATA_DROP_EN__SHIFT                     0x5
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF6_POISON_DATA_DROP_EN__SHIFT                     0x6
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF7_POISON_DATA_DROP_EN__SHIFT                     0x7
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF8_POISON_DATA_DROP_EN__SHIFT                     0x8
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF9_POISON_DATA_DROP_EN__SHIFT                     0x9
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF10_POISON_DATA_DROP_EN__SHIFT                    0xa
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF11_POISON_DATA_DROP_EN__SHIFT                    0xb
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF12_POISON_DATA_DROP_EN__SHIFT                    0xc
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF13_POISON_DATA_DROP_EN__SHIFT                    0xd
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF14_POISON_DATA_DROP_EN__SHIFT                    0xe
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF15_POISON_DATA_DROP_EN__SHIFT                    0xf
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF16_POISON_DATA_DROP_EN__SHIFT                    0x10
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF17_POISON_DATA_DROP_EN__SHIFT                    0x11
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF18_POISON_DATA_DROP_EN__SHIFT                    0x12
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF19_POISON_DATA_DROP_EN__SHIFT                    0x13
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF0_POISON_DATA_DROP_EN_MASK                       0x00000001L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF1_POISON_DATA_DROP_EN_MASK                       0x00000002L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF2_POISON_DATA_DROP_EN_MASK                       0x00000004L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF3_POISON_DATA_DROP_EN_MASK                       0x00000008L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF4_POISON_DATA_DROP_EN_MASK                       0x00000010L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF5_POISON_DATA_DROP_EN_MASK                       0x00000020L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF6_POISON_DATA_DROP_EN_MASK                       0x00000040L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF7_POISON_DATA_DROP_EN_MASK                       0x00000080L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF8_POISON_DATA_DROP_EN_MASK                       0x00000100L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF9_POISON_DATA_DROP_EN_MASK                       0x00000200L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF10_POISON_DATA_DROP_EN_MASK                      0x00000400L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF11_POISON_DATA_DROP_EN_MASK                      0x00000800L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF12_POISON_DATA_DROP_EN_MASK                      0x00001000L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF13_POISON_DATA_DROP_EN_MASK                      0x00002000L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF14_POISON_DATA_DROP_EN_MASK                      0x00004000L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF15_POISON_DATA_DROP_EN_MASK                      0x00008000L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF16_POISON_DATA_DROP_EN_MASK                      0x00010000L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF17_POISON_DATA_DROP_EN_MASK                      0x00020000L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF18_POISON_DATA_DROP_EN_MASK                      0x00040000L
+#define BIFL_SEC_RAS_POISON_DATA_DROP_CTRL__BIFL_SEC_RAS_LEAF19_POISON_DATA_DROP_EN_MASK                      0x00080000L
+//REGS_INDEX_DATA_PAIR_CTRL
+#define REGS_INDEX_DATA_PAIR_CTRL__MM_DATA_DIS__SHIFT                                                         0x0
+#define REGS_INDEX_DATA_PAIR_CTRL__SYSHUB_DATA_DIS__SHIFT                                                     0x1
+#define REGS_INDEX_DATA_PAIR_CTRL__PCIE_DATA_DIS__SHIFT                                                       0x2
+#define REGS_INDEX_DATA_PAIR_CTRL__PCIE_DATA2_DIS__SHIFT                                                      0x3
+#define REGS_INDEX_DATA_PAIR_CTRL__MM_DATA_DIS_MASK                                                           0x00000001L
+#define REGS_INDEX_DATA_PAIR_CTRL__SYSHUB_DATA_DIS_MASK                                                       0x00000002L
+#define REGS_INDEX_DATA_PAIR_CTRL__PCIE_DATA_DIS_MASK                                                         0x00000004L
+#define REGS_INDEX_DATA_PAIR_CTRL__PCIE_DATA2_DIS_MASK                                                        0x00000008L
+//BIFL_SEC_RAS_RESPONSE_DROP_CTRL
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF0_RESPONSE_DROP_EN__SHIFT                           0x0
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF1_RESPONSE_DROP_EN__SHIFT                           0x1
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF2_RESPONSE_DROP_EN__SHIFT                           0x2
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF3_RESPONSE_DROP_EN__SHIFT                           0x3
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF4_RESPONSE_DROP_EN__SHIFT                           0x4
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF5_RESPONSE_DROP_EN__SHIFT                           0x5
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF6_RESPONSE_DROP_EN__SHIFT                           0x6
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF7_RESPONSE_DROP_EN__SHIFT                           0x7
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF8_RESPONSE_DROP_EN__SHIFT                           0x8
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF9_RESPONSE_DROP_EN__SHIFT                           0x9
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF10_RESPONSE_DROP_EN__SHIFT                          0xa
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF11_RESPONSE_DROP_EN__SHIFT                          0xb
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF12_RESPONSE_DROP_EN__SHIFT                          0xc
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF13_RESPONSE_DROP_EN__SHIFT                          0xd
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF14_RESPONSE_DROP_EN__SHIFT                          0xe
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF15_RESPONSE_DROP_EN__SHIFT                          0xf
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF16_RESPONSE_DROP_EN__SHIFT                          0x10
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF17_RESPONSE_DROP_EN__SHIFT                          0x11
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF18_RESPONSE_DROP_EN__SHIFT                          0x12
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF19_RESPONSE_DROP_EN__SHIFT                          0x13
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF0_RESPONSE_DROP_EN_MASK                             0x00000001L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF1_RESPONSE_DROP_EN_MASK                             0x00000002L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF2_RESPONSE_DROP_EN_MASK                             0x00000004L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF3_RESPONSE_DROP_EN_MASK                             0x00000008L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF4_RESPONSE_DROP_EN_MASK                             0x00000010L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF5_RESPONSE_DROP_EN_MASK                             0x00000020L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF6_RESPONSE_DROP_EN_MASK                             0x00000040L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF7_RESPONSE_DROP_EN_MASK                             0x00000080L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF8_RESPONSE_DROP_EN_MASK                             0x00000100L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF9_RESPONSE_DROP_EN_MASK                             0x00000200L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF10_RESPONSE_DROP_EN_MASK                            0x00000400L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF11_RESPONSE_DROP_EN_MASK                            0x00000800L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF12_RESPONSE_DROP_EN_MASK                            0x00001000L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF13_RESPONSE_DROP_EN_MASK                            0x00002000L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF14_RESPONSE_DROP_EN_MASK                            0x00004000L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF15_RESPONSE_DROP_EN_MASK                            0x00008000L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF16_RESPONSE_DROP_EN_MASK                            0x00010000L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF17_RESPONSE_DROP_EN_MASK                            0x00020000L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF18_RESPONSE_DROP_EN_MASK                            0x00040000L
+#define BIFL_SEC_RAS_RESPONSE_DROP_CTRL__BIFL_SEC_RAS_LEAF19_RESPONSE_DROP_EN_MASK                            0x00080000L
+//BIFC_SEC_MCA_SMN_CTRL0
+#define BIFC_SEC_MCA_SMN_CTRL0__MCA_INT_REQ_ADDR__SHIFT                                                       0x0
+#define BIFC_SEC_MCA_SMN_CTRL0__MCA_INT_REQ_APER_ID__SHIFT                                                    0x14
+#define BIFC_SEC_MCA_SMN_CTRL0__MCA_INT_REQ_ADDR_MASK                                                         0x000FFFFFL
+#define BIFC_SEC_MCA_SMN_CTRL0__MCA_INT_REQ_APER_ID_MASK                                                      0xFFF00000L
+//BIFC_SEC_MCA_SMN_CTRL1
+#define BIFC_SEC_MCA_SMN_CTRL1__MCA_INT_REQ_MCM_ADDR__SHIFT                                                   0x0
+#define BIFC_SEC_MCA_SMN_CTRL1__MCA_SMN_SEC_SEL__SHIFT                                                        0x10
+#define BIFC_SEC_MCA_SMN_CTRL1__MCA_INT_REQ_MCM_ADDR_MASK                                                     0x0000FFFFL
+#define BIFC_SEC_MCA_SMN_CTRL1__MCA_SMN_SEC_SEL_MASK                                                          0x000F0000L
+//BIFL_SEC_RAS_POISON_DBUG_CTRL
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF0_CTRL_POISON_DBUG_EN__SHIFT                          0x0
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF1_CTRL_POISON_DBUG_EN__SHIFT                          0x1
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF2_CTRL_POISON_DBUG_EN__SHIFT                          0x2
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF3_CTRL_POISON_DBUG_EN__SHIFT                          0x3
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF4_CTRL_POISON_DBUG_EN__SHIFT                          0x4
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF5_CTRL_POISON_DBUG_EN__SHIFT                          0x5
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF6_CTRL_POISON_DBUG_EN__SHIFT                          0x6
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF7_CTRL_POISON_DBUG_EN__SHIFT                          0x7
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF8_CTRL_POISON_DBUG_EN__SHIFT                          0x8
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF9_CTRL_POISON_DBUG_EN__SHIFT                          0x9
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF10_CTRL_POISON_DBUG_EN__SHIFT                         0xa
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF11_CTRL_POISON_DBUG_EN__SHIFT                         0xb
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF12_CTRL_POISON_DBUG_EN__SHIFT                         0xc
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF13_CTRL_POISON_DBUG_EN__SHIFT                         0xd
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF14_CTRL_POISON_DBUG_EN__SHIFT                         0xe
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF15_CTRL_POISON_DBUG_EN__SHIFT                         0xf
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF16_CTRL_POISON_DBUG_EN__SHIFT                         0x10
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF17_CTRL_POISON_DBUG_EN__SHIFT                         0x11
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF18_CTRL_POISON_DBUG_EN__SHIFT                         0x12
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF19_CTRL_POISON_DBUG_EN__SHIFT                         0x13
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF0_CTRL_POISON_DBUG_EN_MASK                            0x00000001L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF1_CTRL_POISON_DBUG_EN_MASK                            0x00000002L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF2_CTRL_POISON_DBUG_EN_MASK                            0x00000004L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF3_CTRL_POISON_DBUG_EN_MASK                            0x00000008L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF4_CTRL_POISON_DBUG_EN_MASK                            0x00000010L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF5_CTRL_POISON_DBUG_EN_MASK                            0x00000020L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF6_CTRL_POISON_DBUG_EN_MASK                            0x00000040L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF7_CTRL_POISON_DBUG_EN_MASK                            0x00000080L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF8_CTRL_POISON_DBUG_EN_MASK                            0x00000100L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF9_CTRL_POISON_DBUG_EN_MASK                            0x00000200L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF10_CTRL_POISON_DBUG_EN_MASK                           0x00000400L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF11_CTRL_POISON_DBUG_EN_MASK                           0x00000800L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF12_CTRL_POISON_DBUG_EN_MASK                           0x00001000L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF13_CTRL_POISON_DBUG_EN_MASK                           0x00002000L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF14_CTRL_POISON_DBUG_EN_MASK                           0x00004000L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF15_CTRL_POISON_DBUG_EN_MASK                           0x00008000L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF16_CTRL_POISON_DBUG_EN_MASK                           0x00010000L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF17_CTRL_POISON_DBUG_EN_MASK                           0x00020000L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF18_CTRL_POISON_DBUG_EN_MASK                           0x00040000L
+#define BIFL_SEC_RAS_POISON_DBUG_CTRL__BIFL_SEC_RAS_LEAF19_CTRL_POISON_DBUG_EN_MASK                           0x00080000L
+//BIFL_SEC_RAS_PARITY_DBUG_CTRL
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF0_CTRL_PARITY_DBUG_EN__SHIFT                          0x0
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF1_CTRL_PARITY_DBUG_EN__SHIFT                          0x1
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF2_CTRL_PARITY_DBUG_EN__SHIFT                          0x2
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF3_CTRL_PARITY_DBUG_EN__SHIFT                          0x3
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF4_CTRL_PARITY_DBUG_EN__SHIFT                          0x4
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF5_CTRL_PARITY_DBUG_EN__SHIFT                          0x5
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF6_CTRL_PARITY_DBUG_EN__SHIFT                          0x6
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF7_CTRL_PARITY_DBUG_EN__SHIFT                          0x7
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF8_CTRL_PARITY_DBUG_EN__SHIFT                          0x8
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF9_CTRL_PARITY_DBUG_EN__SHIFT                          0x9
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF10_CTRL_PARITY_DBUG_EN__SHIFT                         0xa
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF11_CTRL_PARITY_DBUG_EN__SHIFT                         0xb
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF12_CTRL_PARITY_DBUG_EN__SHIFT                         0xc
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF13_CTRL_PARITY_DBUG_EN__SHIFT                         0xd
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF14_CTRL_PARITY_DBUG_EN__SHIFT                         0xe
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF15_CTRL_PARITY_DBUG_EN__SHIFT                         0xf
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF16_CTRL_PARITY_DBUG_EN__SHIFT                         0x10
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF17_CTRL_PARITY_DBUG_EN__SHIFT                         0x11
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF18_CTRL_PARITY_DBUG_EN__SHIFT                         0x12
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF19_CTRL_PARITY_DBUG_EN__SHIFT                         0x13
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF0_CTRL_PARITY_DBUG_EN_MASK                            0x00000001L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF1_CTRL_PARITY_DBUG_EN_MASK                            0x00000002L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF2_CTRL_PARITY_DBUG_EN_MASK                            0x00000004L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF3_CTRL_PARITY_DBUG_EN_MASK                            0x00000008L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF4_CTRL_PARITY_DBUG_EN_MASK                            0x00000010L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF5_CTRL_PARITY_DBUG_EN_MASK                            0x00000020L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF6_CTRL_PARITY_DBUG_EN_MASK                            0x00000040L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF7_CTRL_PARITY_DBUG_EN_MASK                            0x00000080L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF8_CTRL_PARITY_DBUG_EN_MASK                            0x00000100L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF9_CTRL_PARITY_DBUG_EN_MASK                            0x00000200L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF10_CTRL_PARITY_DBUG_EN_MASK                           0x00000400L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF11_CTRL_PARITY_DBUG_EN_MASK                           0x00000800L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF12_CTRL_PARITY_DBUG_EN_MASK                           0x00001000L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF13_CTRL_PARITY_DBUG_EN_MASK                           0x00002000L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF14_CTRL_PARITY_DBUG_EN_MASK                           0x00004000L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF15_CTRL_PARITY_DBUG_EN_MASK                           0x00008000L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF16_CTRL_PARITY_DBUG_EN_MASK                           0x00010000L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF17_CTRL_PARITY_DBUG_EN_MASK                           0x00020000L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF18_CTRL_PARITY_DBUG_EN_MASK                           0x00040000L
+#define BIFL_SEC_RAS_PARITY_DBUG_CTRL__BIFL_SEC_RAS_LEAF19_CTRL_PARITY_DBUG_EN_MASK                           0x00080000L
+//BIFL_SEC_RAS_NTB_DBUG_CTRL
+#define BIFL_SEC_RAS_NTB_DBUG_CTRL__BIFL_SEC_RAS_LEAF_NTB_CTRL_RCVERREVENT_DBUG_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_NTB_DBUG_CTRL__BIFL_SEC_RAS_LEAF_NTB_CTRL_RCVERREVENT_DBUG_EN_MASK                       0x00000001L
+//BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF0_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x0
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF1_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x1
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF2_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x2
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF3_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x3
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF4_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x4
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF5_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x5
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF6_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x6
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF7_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x7
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF8_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x8
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF9_CTRL_RCVERREVENT_DBUG_EN__SHIFT                0x9
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF10_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0xa
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF11_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0xb
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF12_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0xc
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF13_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0xd
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF14_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0xe
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF15_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0xf
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF16_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0x10
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF17_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0x11
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF18_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0x12
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF19_CTRL_RCVERREVENT_DBUG_EN__SHIFT               0x13
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF0_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000001L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF1_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000002L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF2_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000004L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF3_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000008L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF4_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000010L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF5_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000020L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF6_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000040L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF7_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000080L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF8_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000100L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF9_CTRL_RCVERREVENT_DBUG_EN_MASK                  0x00000200L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF10_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00000400L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF11_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00000800L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF12_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00001000L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF13_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00002000L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF14_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00004000L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF15_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00008000L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF16_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00010000L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF17_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00020000L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF18_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00040000L
+#define BIFL_SEC_RAS_RCVERREVENT_DBUG_CTRL__BIFL_SEC_RAS_LEAF19_CTRL_RCVERREVENT_DBUG_EN_MASK                 0x00080000L
+//BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF0_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x0
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF1_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x1
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF2_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x2
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF3_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x3
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF4_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x4
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF5_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x5
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF6_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x6
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF7_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x7
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF8_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x8
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF9_CTRL_TIMEOUT_DBUG_EN__SHIFT                        0x9
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF10_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0xa
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF11_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0xb
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF12_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0xc
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF13_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0xd
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF14_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0xe
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF15_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0xf
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF16_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0x10
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF17_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0x11
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF18_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0x12
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF19_CTRL_TIMEOUT_DBUG_EN__SHIFT                       0x13
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF0_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000001L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF1_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000002L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF2_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000004L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF3_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000008L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF4_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000010L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF5_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000020L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF6_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000040L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF7_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000080L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF8_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000100L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF9_CTRL_TIMEOUT_DBUG_EN_MASK                          0x00000200L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF10_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00000400L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF11_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00000800L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF12_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00001000L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF13_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00002000L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF14_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00004000L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF15_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00008000L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF16_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00010000L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF17_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00020000L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF18_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00040000L
+#define BIFL_SEC_RAS_TIMEOUT_DBUG_CTRL__BIFL_SEC_RAS_LEAF19_CTRL_TIMEOUT_DBUG_EN_MASK                         0x00080000L
+//BIFL_SEC_RAS_LEAF0_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF0_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF0_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF0_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF0_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF0_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF0_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF0_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF0_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF0_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF1_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF1_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF1_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF1_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF1_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF1_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF1_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF1_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF1_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF1_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF2_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF2_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF2_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF2_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF2_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF2_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF2_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF2_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF2_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF2_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF3_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF3_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF3_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF3_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF3_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF3_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF3_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF3_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF3_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF3_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF4_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF4_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF4_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF4_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF4_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF4_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF4_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF4_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF4_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF4_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF5_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF5_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF5_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF5_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF5_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF5_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF5_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF5_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF5_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF5_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF6_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF6_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF6_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF6_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF6_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF6_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF6_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF6_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF6_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF6_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF7_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF7_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF7_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF7_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF7_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF7_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF7_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF7_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF7_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF7_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF8_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF8_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF8_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF8_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF8_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF8_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF8_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF8_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF8_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF8_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF9_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF9_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF9_RRESP_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF9_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF9_RRESP_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF9_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF9_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF9_RUSER_POISON_EN__SHIFT                       0x0
+#define BIFL_SEC_RAS_LEAF9_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF9_RUSER_POISON_EN_MASK                         0x000000FFL
+//BIFL_SEC_RAS_LEAF10_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF10_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF10_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF10_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF10_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF10_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF10_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF10_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF10_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF10_RUSER_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF11_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF11_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF11_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF11_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF11_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF11_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF11_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF11_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF11_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF11_RUSER_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF12_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF12_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF12_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF12_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF12_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF12_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF12_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF12_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF12_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF12_RUSER_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF13_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF13_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF13_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF13_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF13_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF13_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF13_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF13_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF13_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF13_RUSER_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF14_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF14_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF14_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF14_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF14_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF14_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF14_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF14_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF14_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF14_RUSER_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF15_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF15_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF15_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF15_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF15_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF15_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF15_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF15_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF15_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF15_RUSER_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF16_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF16_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF16_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF16_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF16_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF16_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF16_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF16_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF16_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF16_RUSER_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF17_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF17_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF17_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF17_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF17_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF17_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF17_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF17_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF17_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF17_RUSER_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF18_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF18_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF18_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF18_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF18_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF18_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF18_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF18_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF18_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF18_RUSER_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF19_RRESP_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF19_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF19_RRESP_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF19_RRESP_POISON_CTRL__BIFL_SEC_RAS_LEAF19_RRESP_POISON_EN_MASK                       0x000000FFL
+//BIFL_SEC_RAS_LEAF19_RUSER_POISON_CTRL
+#define BIFL_SEC_RAS_LEAF19_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF19_RUSER_POISON_EN__SHIFT                     0x0
+#define BIFL_SEC_RAS_LEAF19_RUSER_POISON_CTRL__BIFL_SEC_RAS_LEAF19_RUSER_POISON_EN_MASK                       0x000000FFL
+//NBIF_STRAP_BIOS_CNTL
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_BIOS_EN__SHIFT                                                       0x0
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_PCIE_ID_BIOS_EN__SHIFT                                               0x1
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_BIOS_EN_MASK                                                         0x00000001L
+#define NBIF_STRAP_BIOS_CNTL__NBIF_STRAP_PCIE_ID_BIOS_EN_MASK                                                 0x00000002L
+//SMN_MST_NORMAL_SECURITY
+#define SMN_MST_NORMAL_SECURITY__SMN_MST_NORMAL_RECURITY_SEL__SHIFT                                           0x0
+#define SMN_MST_NORMAL_SECURITY__SMN_MST_NORMAL_RECURITY_SEL_MASK                                             0x0000000FL
+//OVERRIDE_TYPE_FOR_CHAIN_CTRL
+#define OVERRIDE_TYPE_FOR_CHAIN_CTRL__OVERRIDE_TYPE_FOR_CHAIN__SHIFT                                          0x0
+#define OVERRIDE_TYPE_FOR_CHAIN_CTRL__OVERRIDE_TYPE_FOR_CHAIN_MASK                                            0x00000001L
+//PROCESSING_HINT_SUPPORT_CNTL
+#define PROCESSING_HINT_SUPPORT_CNTL__PH_MASK_EN__SHIFT                                                       0x0
+#define PROCESSING_HINT_SUPPORT_CNTL__PH_MASK_EN_MASK                                                         0x00000001L
+//DEV0_PF0_VF0_MMIO0_FENCE
+#define DEV0_PF0_VF0_MMIO0_FENCE__WR_ENABLE__SHIFT                                                            0x0
+#define DEV0_PF0_VF0_MMIO0_FENCE__RD_ENABLE__SHIFT                                                            0x10
+#define DEV0_PF0_VF0_MMIO0_FENCE__WR_ENABLE_MASK                                                              0x0000FFFFL
+#define DEV0_PF0_VF0_MMIO0_FENCE__RD_ENABLE_MASK                                                              0xFFFF0000L
+//DEV0_PF0_VF1_MMIO0_FENCE
+#define DEV0_PF0_VF1_MMIO0_FENCE__WR_ENABLE__SHIFT                                                            0x0
+#define DEV0_PF0_VF1_MMIO0_FENCE__RD_ENABLE__SHIFT                                                            0x10
+#define DEV0_PF0_VF1_MMIO0_FENCE__WR_ENABLE_MASK                                                              0x0000FFFFL
+#define DEV0_PF0_VF1_MMIO0_FENCE__RD_ENABLE_MASK                                                              0xFFFF0000L
+//DEV0_PF0_VF2_MMIO0_FENCE
+#define DEV0_PF0_VF2_MMIO0_FENCE__WR_ENABLE__SHIFT                                                            0x0
+#define DEV0_PF0_VF2_MMIO0_FENCE__RD_ENABLE__SHIFT                                                            0x10
+#define DEV0_PF0_VF2_MMIO0_FENCE__WR_ENABLE_MASK                                                              0x0000FFFFL
+#define DEV0_PF0_VF2_MMIO0_FENCE__RD_ENABLE_MASK                                                              0xFFFF0000L
+//DEV0_PF0_VF3_MMIO0_FENCE
+#define DEV0_PF0_VF3_MMIO0_FENCE__WR_ENABLE__SHIFT                                                            0x0
+#define DEV0_PF0_VF3_MMIO0_FENCE__RD_ENABLE__SHIFT                                                            0x10
+#define DEV0_PF0_VF3_MMIO0_FENCE__WR_ENABLE_MASK                                                              0x0000FFFFL
+#define DEV0_PF0_VF3_MMIO0_FENCE__RD_ENABLE_MASK                                                              0xFFFF0000L
+//DEV0_PF0_VF4_MMIO0_FENCE
+#define DEV0_PF0_VF4_MMIO0_FENCE__WR_ENABLE__SHIFT                                                            0x0
+#define DEV0_PF0_VF4_MMIO0_FENCE__RD_ENABLE__SHIFT                                                            0x10
+#define DEV0_PF0_VF4_MMIO0_FENCE__WR_ENABLE_MASK                                                              0x0000FFFFL
+#define DEV0_PF0_VF4_MMIO0_FENCE__RD_ENABLE_MASK                                                              0xFFFF0000L
+//DEV0_PF0_VF5_MMIO0_FENCE
+#define DEV0_PF0_VF5_MMIO0_FENCE__WR_ENABLE__SHIFT                                                            0x0
+#define DEV0_PF0_VF5_MMIO0_FENCE__RD_ENABLE__SHIFT                                                            0x10
+#define DEV0_PF0_VF5_MMIO0_FENCE__WR_ENABLE_MASK                                                              0x0000FFFFL
+#define DEV0_PF0_VF5_MMIO0_FENCE__RD_ENABLE_MASK                                                              0xFFFF0000L
+//DEV0_PF0_VF6_MMIO0_FENCE
+#define DEV0_PF0_VF6_MMIO0_FENCE__WR_ENABLE__SHIFT                                                            0x0
+#define DEV0_PF0_VF6_MMIO0_FENCE__RD_ENABLE__SHIFT                                                            0x10
+#define DEV0_PF0_VF6_MMIO0_FENCE__WR_ENABLE_MASK                                                              0x0000FFFFL
+#define DEV0_PF0_VF6_MMIO0_FENCE__RD_ENABLE_MASK                                                              0xFFFF0000L
+//DEV0_PF0_VF7_MMIO0_FENCE
+#define DEV0_PF0_VF7_MMIO0_FENCE__WR_ENABLE__SHIFT                                                            0x0
+#define DEV0_PF0_VF7_MMIO0_FENCE__RD_ENABLE__SHIFT                                                            0x10
+#define DEV0_PF0_VF7_MMIO0_FENCE__WR_ENABLE_MASK                                                              0x0000FFFFL
+#define DEV0_PF0_VF7_MMIO0_FENCE__RD_ENABLE_MASK                                                              0xFFFF0000L
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE0_START
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE0_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE0_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE0_END
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE0_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE0_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE1_START
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE1_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE1_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE1_END
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE1_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE1_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE2_START
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE2_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE2_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE2_END
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE2_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE2_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE3_START
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE3_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE3_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE3_END
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE3_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE3_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE4_START
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE4_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE4_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE4_END
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE4_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE4_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE5_START
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE5_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE5_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE5_END
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE5_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE5_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE6_START
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE6_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE6_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE6_END
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE6_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE6_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE7_START
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE7_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE7_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_VF_MMIO0_FENCE_RANGE7_END
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE7_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_VF_MMIO0_FENCE_RANGE7_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF1_VF0_MMIO0_FENCE
+#define DEV0_PF1_VF0_MMIO0_FENCE__WR_ENABLE__SHIFT                                                            0x0
+#define DEV0_PF1_VF0_MMIO0_FENCE__RD_ENABLE__SHIFT                                                            0x10
+#define DEV0_PF1_VF0_MMIO0_FENCE__WR_ENABLE_MASK                                                              0x0000FFFFL
+#define DEV0_PF1_VF0_MMIO0_FENCE__RD_ENABLE_MASK                                                              0xFFFF0000L
+//DEV0_PF1_VF_MMIO0_FENCE_RANGE0_START
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE0_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE0_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF1_VF_MMIO0_FENCE_RANGE0_END
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE0_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE0_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF1_VF_MMIO0_FENCE_RANGE1_START
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE1_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE1_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF1_VF_MMIO0_FENCE_RANGE1_END
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE1_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE1_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF1_VF_MMIO0_FENCE_RANGE2_START
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE2_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE2_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF1_VF_MMIO0_FENCE_RANGE2_END
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE2_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE2_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF1_VF_MMIO0_FENCE_RANGE3_START
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE3_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE3_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF1_VF_MMIO0_FENCE_RANGE3_END
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE3_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF1_VF_MMIO0_FENCE_RANGE3_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_WR
+#define DEV0_PF0_PF_MMIO0_FENCE_WR__ENABLE__SHIFT                                                             0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_WR__ENABLE_MASK                                                               0xFFFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RD
+#define DEV0_PF0_PF_MMIO0_FENCE_RD__ENABLE__SHIFT                                                             0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RD__ENABLE_MASK                                                               0xFFFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE0_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE0_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE0_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE0_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE0_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE0_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE1_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE1_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE1_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE1_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE1_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE1_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE2_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE2_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE2_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE2_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE2_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE2_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE3_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE3_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE3_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE3_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE3_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE3_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE4_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE4_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE4_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE4_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE4_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE4_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE5_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE5_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE5_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE5_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE5_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE5_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE6_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE6_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE6_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE6_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE6_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE6_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE7_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE7_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE7_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE7_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE7_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE7_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE8_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE8_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE8_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE8_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE8_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE8_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE9_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE9_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE9_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE9_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE9_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE9_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE10_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE10_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE10_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE10_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE10_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE10_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE11_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE11_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE11_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE11_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE11_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE11_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE12_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE12_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE12_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE12_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE12_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE12_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE13_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE13_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE13_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE13_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE13_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE13_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE14_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE14_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE14_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE14_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE14_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE14_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE15_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE15_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE15_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE15_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE15_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE15_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE16_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE16_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE16_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE16_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE16_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE16_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE17_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE17_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE17_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE17_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE17_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE17_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE18_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE18_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE18_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE18_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE18_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE18_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE19_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE19_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE19_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE19_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE19_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE19_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE20_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE20_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE20_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE20_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE20_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE20_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE21_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE21_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE21_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE21_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE21_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE21_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE22_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE22_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE22_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE22_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE22_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE22_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE23_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE23_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE23_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE23_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE23_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE23_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE24_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE24_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE24_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE24_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE24_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE24_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE25_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE25_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE25_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE25_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE25_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE25_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE26_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE26_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE26_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE26_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE26_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE26_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE27_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE27_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE27_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE27_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE27_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE27_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE28_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE28_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE28_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE28_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE28_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE28_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE29_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE29_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE29_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE29_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE29_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE29_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE30_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE30_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE30_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE30_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE30_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE30_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE31_START
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE31_START__ADDRESS__SHIFT                                                 0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE31_START__ADDRESS_MASK                                                   0x3FFFFFFFL
+//DEV0_PF0_PF_MMIO0_FENCE_RANGE31_END
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE31_END__ADDRESS__SHIFT                                                   0x0
+#define DEV0_PF0_PF_MMIO0_FENCE_RANGE31_END__ADDRESS_MASK                                                     0x3FFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_WR
+#define DEV0_PF1_PF_MMIO0_FENCE_WR__ENABLE__SHIFT                                                             0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_WR__ENABLE_MASK                                                               0xFFFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_RD
+#define DEV0_PF1_PF_MMIO0_FENCE_RD__ENABLE__SHIFT                                                             0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_RD__ENABLE_MASK                                                               0xFFFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_RANGE0_START
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE0_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE0_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_RANGE0_END
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE0_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE0_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_RANGE1_START
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE1_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE1_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_RANGE1_END
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE1_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE1_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_RANGE2_START
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE2_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE2_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_RANGE2_END
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE2_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE2_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_RANGE3_START
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE3_START__ADDRESS__SHIFT                                                  0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE3_START__ADDRESS_MASK                                                    0x3FFFFFFFL
+//DEV0_PF1_PF_MMIO0_FENCE_RANGE3_END
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE3_END__ADDRESS__SHIFT                                                    0x0
+#define DEV0_PF1_PF_MMIO0_FENCE_RANGE3_END__ADDRESS_MASK                                                      0x3FFFFFFFL
+//MISC_SCRATCH
+#define MISC_SCRATCH__MISC_SCRATCH0__SHIFT                                                                    0x0
+#define MISC_SCRATCH__MISC_SCRATCH0_MASK                                                                      0xFFFFFFFFL
+//INTR_LINE_POLARITY
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV0__SHIFT                                                    0x0
+#define INTR_LINE_POLARITY__INTR_LINE_POLARITY_DEV0_MASK                                                      0x000000FFL
+//INTR_LINE_ENABLE
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV0__SHIFT                                                        0x0
+#define INTR_LINE_ENABLE__INTR_LINE_ENABLE_DEV0_MASK                                                          0x000000FFL
+//OUTSTANDING_VC_ALLOC
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC0_ALLOC__SHIFT                                                0x0
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC1_ALLOC__SHIFT                                                0x2
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC2_ALLOC__SHIFT                                                0x4
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC3_ALLOC__SHIFT                                                0x6
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC4_ALLOC__SHIFT                                                0x8
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC5_ALLOC__SHIFT                                                0xa
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC6_ALLOC__SHIFT                                                0xc
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC7_ALLOC__SHIFT                                                0xe
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_THRD__SHIFT                                                     0x10
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC0_ALLOC__SHIFT                                                0x18
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC1_ALLOC__SHIFT                                                0x1a
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_THRD__SHIFT                                                     0x1c
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC0_ALLOC_MASK                                                  0x00000003L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC1_ALLOC_MASK                                                  0x0000000CL
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC2_ALLOC_MASK                                                  0x00000030L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC3_ALLOC_MASK                                                  0x000000C0L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC4_ALLOC_MASK                                                  0x00000300L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC5_ALLOC_MASK                                                  0x00000C00L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC6_ALLOC_MASK                                                  0x00003000L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_VC7_ALLOC_MASK                                                  0x0000C000L
+#define OUTSTANDING_VC_ALLOC__DMA_OUTSTANDING_THRD_MASK                                                       0x000F0000L
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC0_ALLOC_MASK                                                  0x03000000L
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_VC1_ALLOC_MASK                                                  0x0C000000L
+#define OUTSTANDING_VC_ALLOC__HST_OUTSTANDING_THRD_MASK                                                       0xF0000000L
+//BIFC_MISC_CTRL0
+#define BIFC_MISC_CTRL0__VWIRE_TARG_UNITID_CHECK_EN__SHIFT                                                    0x0
+#define BIFC_MISC_CTRL0__VWIRE_SRC_UNITID_CHECK_EN__SHIFT                                                     0x1
+#define BIFC_MISC_CTRL0__REG_ACTIVE_VLINK_L0_EN__SHIFT                                                        0x3
+#define BIFC_MISC_CTRL0__DMA_VC4_NON_DVM_STS__SHIFT                                                           0x4
+#define BIFC_MISC_CTRL0__DMA_CHAIN_BREAK_IN_RCMODE__SHIFT                                                     0x8
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_P__SHIFT                                                          0x9
+#define BIFC_MISC_CTRL0__GSI_SST_ARB_CHAIN_LOCK__SHIFT                                                        0xa
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_FLUSH_EN__SHIFT                                                   0xb
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_NPWR_DIS__SHIFT                                                   0xc
+#define BIFC_MISC_CTRL0__GSI_SET_PRECEEDINGWR_DIS__SHIFT                                                      0xd
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_NP__SHIFT                                                         0xe
+#define BIFC_MISC_CTRL0__HRP_CHAIN_DISABLE__SHIFT                                                             0xf
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_LENGTH_CHK_DIS__SHIFT                                                     0x10
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_FAILED_STS_SEL__SHIFT                                                     0x11
+#define BIFC_MISC_CTRL0__DMA_FORCE_VF_AS_PF_SRIOIVEN_LOW__SHIFT                                               0x12
+#define BIFC_MISC_CTRL0__DMA_ADDR_KEEP_PH__SHIFT                                                              0x13
+#define BIFC_MISC_CTRL0__RCC_GMI_TD_FORCE_ZERO__SHIFT                                                         0x14
+#define BIFC_MISC_CTRL0__HST_FLUSH_DEFER_EN__SHIFT                                                            0x15
+#define BIFC_MISC_CTRL0__HST_FLUSH_CLR_LOCK_EN__SHIFT                                                         0x16
+#define BIFC_MISC_CTRL0__STFETCH_BLOCK_IN_RST__SHIFT                                                          0x17
+#define BIFC_MISC_CTRL0__PCIE_CAPABILITY_PROT_DIS__SHIFT                                                      0x18
+#define BIFC_MISC_CTRL0__ATS_MSG_BLOCK_IN_RST__SHIFT                                                          0x19
+#define BIFC_MISC_CTRL0__DMA_2ND_REQ_DIS__SHIFT                                                               0x1a
+#define BIFC_MISC_CTRL0__PORT_DSTATE_BYPASS_MODE__SHIFT                                                       0x1b
+#define BIFC_MISC_CTRL0__PME_TURNOFF_MODE__SHIFT                                                              0x1c
+#define BIFC_MISC_CTRL0__DMA_ALL_RST_PROTECT_STS_SEL__SHIFT                                                   0x1d
+#define BIFC_MISC_CTRL0__HDP_P2P_DIRECT_ADD_ADJUST__SHIFT                                                     0x1e
+#define BIFC_MISC_CTRL0__PCIESWUS_SELECTION__SHIFT                                                            0x1f
+#define BIFC_MISC_CTRL0__VWIRE_TARG_UNITID_CHECK_EN_MASK                                                      0x00000001L
+#define BIFC_MISC_CTRL0__VWIRE_SRC_UNITID_CHECK_EN_MASK                                                       0x00000006L
+#define BIFC_MISC_CTRL0__REG_ACTIVE_VLINK_L0_EN_MASK                                                          0x00000008L
+#define BIFC_MISC_CTRL0__DMA_VC4_NON_DVM_STS_MASK                                                             0x000000F0L
+#define BIFC_MISC_CTRL0__DMA_CHAIN_BREAK_IN_RCMODE_MASK                                                       0x00000100L
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_P_MASK                                                            0x00000200L
+#define BIFC_MISC_CTRL0__GSI_SST_ARB_CHAIN_LOCK_MASK                                                          0x00000400L
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_FLUSH_EN_MASK                                                     0x00000800L
+#define BIFC_MISC_CTRL0__GSI_RD_SPLIT_STALL_NPWR_DIS_MASK                                                     0x00001000L
+#define BIFC_MISC_CTRL0__GSI_SET_PRECEEDINGWR_DIS_MASK                                                        0x00002000L
+#define BIFC_MISC_CTRL0__HST_ARB_CHAIN_LOCK_NP_MASK                                                           0x00004000L
+#define BIFC_MISC_CTRL0__HRP_CHAIN_DISABLE_MASK                                                               0x00008000L
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_LENGTH_CHK_DIS_MASK                                                       0x00010000L
+#define BIFC_MISC_CTRL0__DMA_ATOMIC_FAILED_STS_SEL_MASK                                                       0x00020000L
+#define BIFC_MISC_CTRL0__DMA_FORCE_VF_AS_PF_SRIOIVEN_LOW_MASK                                                 0x00040000L
+#define BIFC_MISC_CTRL0__DMA_ADDR_KEEP_PH_MASK                                                                0x00080000L
+#define BIFC_MISC_CTRL0__RCC_GMI_TD_FORCE_ZERO_MASK                                                           0x00100000L
+#define BIFC_MISC_CTRL0__HST_FLUSH_DEFER_EN_MASK                                                              0x00200000L
+#define BIFC_MISC_CTRL0__HST_FLUSH_CLR_LOCK_EN_MASK                                                           0x00400000L
+#define BIFC_MISC_CTRL0__STFETCH_BLOCK_IN_RST_MASK                                                            0x00800000L
+#define BIFC_MISC_CTRL0__PCIE_CAPABILITY_PROT_DIS_MASK                                                        0x01000000L
+#define BIFC_MISC_CTRL0__ATS_MSG_BLOCK_IN_RST_MASK                                                            0x02000000L
+#define BIFC_MISC_CTRL0__DMA_2ND_REQ_DIS_MASK                                                                 0x04000000L
+#define BIFC_MISC_CTRL0__PORT_DSTATE_BYPASS_MODE_MASK                                                         0x08000000L
+#define BIFC_MISC_CTRL0__PME_TURNOFF_MODE_MASK                                                                0x10000000L
+#define BIFC_MISC_CTRL0__DMA_ALL_RST_PROTECT_STS_SEL_MASK                                                     0x20000000L
+#define BIFC_MISC_CTRL0__HDP_P2P_DIRECT_ADD_ADJUST_MASK                                                       0x40000000L
+#define BIFC_MISC_CTRL0__PCIESWUS_SELECTION_MASK                                                              0x80000000L
+//BIFC_MISC_CTRL1
+#define BIFC_MISC_CTRL1__THT_HST_CPLD_POISON_REPORT__SHIFT                                                    0x0
+#define BIFC_MISC_CTRL1__DMA_REQ_POISON_REPORT__SHIFT                                                         0x1
+#define BIFC_MISC_CTRL1__DMA_REQ_ACSVIO_REPORT__SHIFT                                                         0x2
+#define BIFC_MISC_CTRL1__DMA_RSP_POISON_CPLD_REPORT__SHIFT                                                    0x3
+#define BIFC_MISC_CTRL1__GSI_SMN_WORST_ERR_STSTUS__SHIFT                                                      0x4
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE1_FOR_ERROR__SHIFT                                           0x5
+#define BIFC_MISC_CTRL1__GSI_RDWR_BALANCE_DIS__SHIFT                                                          0x6
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_DROP__SHIFT                                                        0x7
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_STS__SHIFT                                                      0x8
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_DATASTS__SHIFT                                                  0xa
+#define BIFC_MISC_CTRL1__DROP_OTHER_HT_ADDR_REQ__SHIFT                                                        0xc
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE__SHIFT                                                 0xd
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE_VALUE__SHIFT                                           0xe
+#define BIFC_MISC_CTRL1__UPS_SDP_RDY_TIE1__SHIFT                                                              0xf
+#define BIFC_MISC_CTRL1__GMI_RCC_DN_BME_DROP_DIS__SHIFT                                                       0x10
+#define BIFC_MISC_CTRL1__GMI_RCC_EP_BME_DROP_DIS__SHIFT                                                       0x11
+#define BIFC_MISC_CTRL1__GMI_BIH_DN_BME_DROP_DIS__SHIFT                                                       0x12
+#define BIFC_MISC_CTRL1__GMI_BIH_EP_BME_DROP_DIS__SHIFT                                                       0x13
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE0_FOR_ERROR__SHIFT                                           0x14
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE__SHIFT                                                 0x15
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE_VALUE__SHIFT                                           0x16
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_FOR_AERLOG__SHIFT                                                  0x17
+#define BIFC_MISC_CTRL1__GMI_RDSIZED_REQATTR_MASK__SHIFT                                                      0x18
+#define BIFC_MISC_CTRL1__GMI_RDSIZEDDW_REQATTR_MASK__SHIFT                                                    0x19
+#define BIFC_MISC_CTRL1__GMI_WRSIZED_REQATTR_MASK__SHIFT                                                      0x1a
+#define BIFC_MISC_CTRL1__GMI_WRSIZEDFL_REQATTR_MASK__SHIFT                                                    0x1b
+#define BIFC_MISC_CTRL1__GMI_FORCE_NOT_SEND_NON_BASEVC_RSPCREDIT__SHIFT                                       0x1c
+#define BIFC_MISC_CTRL1__GMI_CPLBUF_EN__SHIFT                                                                 0x1d
+#define BIFC_MISC_CTRL1__GMI_MSG_BLOCKLVL_SEL__SHIFT                                                          0x1e
+#define BIFC_MISC_CTRL1__THT_HST_CPLD_POISON_REPORT_MASK                                                      0x00000001L
+#define BIFC_MISC_CTRL1__DMA_REQ_POISON_REPORT_MASK                                                           0x00000002L
+#define BIFC_MISC_CTRL1__DMA_REQ_ACSVIO_REPORT_MASK                                                           0x00000004L
+#define BIFC_MISC_CTRL1__DMA_RSP_POISON_CPLD_REPORT_MASK                                                      0x00000008L
+#define BIFC_MISC_CTRL1__GSI_SMN_WORST_ERR_STSTUS_MASK                                                        0x00000010L
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE1_FOR_ERROR_MASK                                             0x00000020L
+#define BIFC_MISC_CTRL1__GSI_RDWR_BALANCE_DIS_MASK                                                            0x00000040L
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_DROP_MASK                                                          0x00000080L
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_STS_MASK                                                        0x00000300L
+#define BIFC_MISC_CTRL1__HST_UNSUPPORT_SDPCMD_DATASTS_MASK                                                    0x00000C00L
+#define BIFC_MISC_CTRL1__DROP_OTHER_HT_ADDR_REQ_MASK                                                          0x00001000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE_MASK                                                   0x00002000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTRDRSP_ORDER_FORCE_VALUE_MASK                                             0x00004000L
+#define BIFC_MISC_CTRL1__UPS_SDP_RDY_TIE1_MASK                                                                0x00008000L
+#define BIFC_MISC_CTRL1__GMI_RCC_DN_BME_DROP_DIS_MASK                                                         0x00010000L
+#define BIFC_MISC_CTRL1__GMI_RCC_EP_BME_DROP_DIS_MASK                                                         0x00020000L
+#define BIFC_MISC_CTRL1__GMI_BIH_DN_BME_DROP_DIS_MASK                                                         0x00040000L
+#define BIFC_MISC_CTRL1__GMI_BIH_EP_BME_DROP_DIS_MASK                                                         0x00080000L
+#define BIFC_MISC_CTRL1__GSI_SDP_RDRSP_DATA_FORCE0_FOR_ERROR_MASK                                             0x00100000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE_MASK                                                   0x00200000L
+#define BIFC_MISC_CTRL1__DMAWRREQ_HSTWRRSP_ORDER_FORCE_VALUE_MASK                                             0x00400000L
+#define BIFC_MISC_CTRL1__GMI_ATOMIC_POISON_FOR_AERLOG_MASK                                                    0x00800000L
+#define BIFC_MISC_CTRL1__GMI_RDSIZED_REQATTR_MASK_MASK                                                        0x01000000L
+#define BIFC_MISC_CTRL1__GMI_RDSIZEDDW_REQATTR_MASK_MASK                                                      0x02000000L
+#define BIFC_MISC_CTRL1__GMI_WRSIZED_REQATTR_MASK_MASK                                                        0x04000000L
+#define BIFC_MISC_CTRL1__GMI_WRSIZEDFL_REQATTR_MASK_MASK                                                      0x08000000L
+#define BIFC_MISC_CTRL1__GMI_FORCE_NOT_SEND_NON_BASEVC_RSPCREDIT_MASK                                         0x10000000L
+#define BIFC_MISC_CTRL1__GMI_CPLBUF_EN_MASK                                                                   0x20000000L
+#define BIFC_MISC_CTRL1__GMI_MSG_BLOCKLVL_SEL_MASK                                                            0xC0000000L
+//BIFC_BME_ERR_LOG_LB
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F0__SHIFT                                                    0x0
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F1__SHIFT                                                    0x1
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F0__SHIFT                                              0x10
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F1__SHIFT                                              0x11
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F0_MASK                                                      0x00000001L
+#define BIFC_BME_ERR_LOG_LB__DMA_ON_BME_LOW_DEV0_F1_MASK                                                      0x00000002L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F0_MASK                                                0x00010000L
+#define BIFC_BME_ERR_LOG_LB__CLEAR_DMA_ON_BME_LOW_DEV0_F1_MASK                                                0x00020000L
+//BIFC_LC_TIMER_CTRL
+#define BIFC_LC_TIMER_CTRL__ASPM_IDLE_TIMER_SCALE__SHIFT                                                      0x0
+#define BIFC_LC_TIMER_CTRL__L1_EXIT_TIMER_SCALE__SHIFT                                                        0x10
+#define BIFC_LC_TIMER_CTRL__ASPM_IDLE_TIMER_SCALE_MASK                                                        0x0000FFFFL
+#define BIFC_LC_TIMER_CTRL__L1_EXIT_TIMER_SCALE_MASK                                                          0xFFFF0000L
+//BIFC_RCCBIH_BME_ERR_LOG0
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F0__SHIFT                                            0x0
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F1__SHIFT                                            0x1
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F0__SHIFT                                      0x10
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F1__SHIFT                                      0x11
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F0_MASK                                              0x00000001L
+#define BIFC_RCCBIH_BME_ERR_LOG0__RCCBIH_ON_BME_LOW_DEV0_F1_MASK                                              0x00000002L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F0_MASK                                        0x00010000L
+#define BIFC_RCCBIH_BME_ERR_LOG0__CLEAR_RCCBIH_ON_BME_LOW_DEV0_F1_MASK                                        0x00020000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F0__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F0__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_IDO_DEV0_F0__SHIFT                                      0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F0__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F0__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F0__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F0__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_NONIDO_DEV0_F0__SHIFT                                   0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F1__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F1__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_IDO_DEV0_F1__SHIFT                                      0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F1__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F1__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F1__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F1__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_NONIDO_DEV0_F1__SHIFT                                   0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F0_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F0_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_IDO_DEV0_F0_MASK                                        0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F0_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F0_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F0_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F0_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_NONIDO_DEV0_F0_MASK                                     0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_P_DEV0_F1_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_IDO_OVERIDE_NP_DEV0_F1_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_IDO_DEV0_F1_MASK                                        0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_P_DEV0_F1_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_RO_OVERIDE_NP_DEV0_F1_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_P_DEV0_F1_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__TX_SNR_OVERIDE_NP_DEV0_F1_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F0_F1__BLKLVL_FOR_NONIDO_DEV0_F1_MASK                                     0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F2__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F2__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_IDO_DEV0_F2__SHIFT                                      0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F2__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F2__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F2__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F2__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_NONIDO_DEV0_F2__SHIFT                                   0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F3__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F3__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_IDO_DEV0_F3__SHIFT                                      0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F3__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F3__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F3__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F3__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_NONIDO_DEV0_F3__SHIFT                                   0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F2_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F2_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_IDO_DEV0_F2_MASK                                        0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F2_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F2_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F2_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F2_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_NONIDO_DEV0_F2_MASK                                     0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_P_DEV0_F3_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_IDO_OVERIDE_NP_DEV0_F3_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_IDO_DEV0_F3_MASK                                        0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_P_DEV0_F3_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_RO_OVERIDE_NP_DEV0_F3_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_P_DEV0_F3_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__TX_SNR_OVERIDE_NP_DEV0_F3_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F2_F3__BLKLVL_FOR_NONIDO_DEV0_F3_MASK                                     0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F4__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F4__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_IDO_DEV0_F4__SHIFT                                      0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F4__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F4__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F4__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F4__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_NONIDO_DEV0_F4__SHIFT                                   0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F5__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F5__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_IDO_DEV0_F5__SHIFT                                      0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F5__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F5__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F5__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F5__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_NONIDO_DEV0_F5__SHIFT                                   0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F4_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F4_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_IDO_DEV0_F4_MASK                                        0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F4_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F4_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F4_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F4_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_NONIDO_DEV0_F4_MASK                                     0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_P_DEV0_F5_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_IDO_OVERIDE_NP_DEV0_F5_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_IDO_DEV0_F5_MASK                                        0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_P_DEV0_F5_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_RO_OVERIDE_NP_DEV0_F5_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_P_DEV0_F5_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__TX_SNR_OVERIDE_NP_DEV0_F5_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F4_F5__BLKLVL_FOR_NONIDO_DEV0_F5_MASK                                     0xC0000000L
+//BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F6__SHIFT                                    0x0
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F6__SHIFT                                   0x2
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_IDO_DEV0_F6__SHIFT                                      0x4
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F6__SHIFT                                     0x6
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F6__SHIFT                                    0x8
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F6__SHIFT                                    0xa
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F6__SHIFT                                   0xc
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_NONIDO_DEV0_F6__SHIFT                                   0xe
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F7__SHIFT                                    0x10
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F7__SHIFT                                   0x12
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_IDO_DEV0_F7__SHIFT                                      0x14
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F7__SHIFT                                     0x16
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F7__SHIFT                                    0x18
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F7__SHIFT                                    0x1a
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F7__SHIFT                                   0x1c
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_NONIDO_DEV0_F7__SHIFT                                   0x1e
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F6_MASK                                      0x00000003L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F6_MASK                                     0x0000000CL
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_IDO_DEV0_F6_MASK                                        0x00000030L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F6_MASK                                       0x000000C0L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F6_MASK                                      0x00000300L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F6_MASK                                      0x00000C00L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F6_MASK                                     0x00003000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_NONIDO_DEV0_F6_MASK                                     0x0000C000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_P_DEV0_F7_MASK                                      0x00030000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_IDO_OVERIDE_NP_DEV0_F7_MASK                                     0x000C0000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_IDO_DEV0_F7_MASK                                        0x00300000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_P_DEV0_F7_MASK                                       0x00C00000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_RO_OVERIDE_NP_DEV0_F7_MASK                                      0x03000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_P_DEV0_F7_MASK                                      0x0C000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__TX_SNR_OVERIDE_NP_DEV0_F7_MASK                                     0x30000000L
+#define BIFC_DMA_ATTR_OVERRIDE_DEV0_F6_F7__BLKLVL_FOR_NONIDO_DEV0_F7_MASK                                     0xC0000000L
+//BIFC_DMA_ATTR_CNTL2_DEV0
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F0__SHIFT                               0x0
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F0__SHIFT                                     0x1
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F1__SHIFT                               0x4
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F1__SHIFT                                     0x5
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F2__SHIFT                               0x8
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F2__SHIFT                                     0x9
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F3__SHIFT                               0xc
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F3__SHIFT                                     0xd
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F4__SHIFT                               0x10
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F4__SHIFT                                     0x11
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F5__SHIFT                               0x14
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F5__SHIFT                                     0x15
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F6__SHIFT                               0x18
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F6__SHIFT                                     0x19
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F7__SHIFT                               0x1c
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F7__SHIFT                                     0x1d
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F0_MASK                                 0x00000001L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F0_MASK                                       0x00000002L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F1_MASK                                 0x00000010L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F1_MASK                                       0x00000020L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F2_MASK                                 0x00000100L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F2_MASK                                       0x00000200L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F3_MASK                                 0x00001000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F3_MASK                                       0x00002000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F4_MASK                                 0x00010000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F4_MASK                                       0x00020000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F5_MASK                                 0x00100000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F5_MASK                                       0x00200000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F6_MASK                                 0x01000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F6_MASK                                       0x02000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__BLKLVL_BYPASS_PCIE_IDO_CONTROL_DEV0_F7_MASK                                 0x10000000L
+#define BIFC_DMA_ATTR_CNTL2_DEV0__NBIF_CDMA_ST_OVERRIDE_EN_DEV0_F7_MASK                                       0x20000000L
+//BIFC_MISC_CTRL2
+#define BIFC_MISC_CTRL2__DIH_INTR_STFETCH_BLOCK_IN_LINKDOWN__SHIFT                                            0x0
+#define BIFC_MISC_CTRL2__SLOW_GMI_UPS_RSP_CRED_REL_EN__SHIFT                                                  0x1
+#define BIFC_MISC_CTRL2__DMA_RSP_ECRC_REPORT__SHIFT                                                           0x2
+#define BIFC_MISC_CTRL2__SLFR_IGNORE_DATAERR_EN__SHIFT                                                        0x10
+#define BIFC_MISC_CTRL2__DATAERR_OVERRIDE_SLFR_BYTEEN_EN__SHIFT                                               0x11
+#define BIFC_MISC_CTRL2__PH_SUPPORT__SHIFT                                                                    0x12
+#define BIFC_MISC_CTRL2__GMI_FAIL_REQ_RTS_MASK__SHIFT                                                         0x16
+#define BIFC_MISC_CTRL2__NBIF_AERRPT_BACKPERSURE_EN__SHIFT                                                    0x17
+#define BIFC_MISC_CTRL2__DIH_INTR_STFETCH_BLOCK_IN_LINKDOWN_MASK                                              0x00000001L
+#define BIFC_MISC_CTRL2__SLOW_GMI_UPS_RSP_CRED_REL_EN_MASK                                                    0x00000002L
+#define BIFC_MISC_CTRL2__DMA_RSP_ECRC_REPORT_MASK                                                             0x00000004L
+#define BIFC_MISC_CTRL2__SLFR_IGNORE_DATAERR_EN_MASK                                                          0x00010000L
+#define BIFC_MISC_CTRL2__DATAERR_OVERRIDE_SLFR_BYTEEN_EN_MASK                                                 0x00020000L
+#define BIFC_MISC_CTRL2__PH_SUPPORT_MASK                                                                      0x003C0000L
+#define BIFC_MISC_CTRL2__GMI_FAIL_REQ_RTS_MASK_MASK                                                           0x00400000L
+#define BIFC_MISC_CTRL2__NBIF_AERRPT_BACKPERSURE_EN_MASK                                                      0x00800000L
+//BME_DUMMY_CNTL_0
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F0__SHIFT                                                     0x0
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F1__SHIFT                                                     0x2
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F2__SHIFT                                                     0x4
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F3__SHIFT                                                     0x6
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F4__SHIFT                                                     0x8
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F5__SHIFT                                                     0xa
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F6__SHIFT                                                     0xc
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F7__SHIFT                                                     0xe
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F0_MASK                                                       0x00000003L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F1_MASK                                                       0x0000000CL
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F2_MASK                                                       0x00000030L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F3_MASK                                                       0x000000C0L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F4_MASK                                                       0x00000300L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F5_MASK                                                       0x00000C00L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F6_MASK                                                       0x00003000L
+#define BME_DUMMY_CNTL_0__BME_DUMMY_RSPSTS_DEV0_F7_MASK                                                       0x0000C000L
+//BIFC_THT_CNTL
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_RD_VC0__SHIFT                                                         0x0
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC0__SHIFT                                                         0x4
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC1__SHIFT                                                         0x8
+#define BIFC_THT_CNTL__UR_OVRD_FOR_ECRC_EN__SHIFT                                                             0x10
+#define BIFC_THT_CNTL__THT_NTB_VC0_APER0_ADSC_PUSH_DIS__SHIFT                                                 0x18
+#define BIFC_THT_CNTL__THT_NTB_VC0_OTHAPER_ADSC_PUSH_DIS__SHIFT                                               0x19
+#define BIFC_THT_CNTL__THT_NTB_VC1_APER0_ADSC_PUSH_DIS__SHIFT                                                 0x1a
+#define BIFC_THT_CNTL__THT_NTB_VC1_OTHAPER_ADSC_PUSH_DIS__SHIFT                                               0x1b
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_RD_VC0_MASK                                                           0x0000000FL
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC0_MASK                                                           0x000000F0L
+#define BIFC_THT_CNTL__CREDIT_ALLOC_THT_WR_VC1_MASK                                                           0x00000F00L
+#define BIFC_THT_CNTL__UR_OVRD_FOR_ECRC_EN_MASK                                                               0x00010000L
+#define BIFC_THT_CNTL__THT_NTB_VC0_APER0_ADSC_PUSH_DIS_MASK                                                   0x01000000L
+#define BIFC_THT_CNTL__THT_NTB_VC0_OTHAPER_ADSC_PUSH_DIS_MASK                                                 0x02000000L
+#define BIFC_THT_CNTL__THT_NTB_VC1_APER0_ADSC_PUSH_DIS_MASK                                                   0x04000000L
+#define BIFC_THT_CNTL__THT_NTB_VC1_OTHAPER_ADSC_PUSH_DIS_MASK                                                 0x08000000L
+//BIFC_HSTARB_CNTL
+#define BIFC_HSTARB_CNTL__SLVARB_MODE__SHIFT                                                                  0x0
+#define BIFC_HSTARB_CNTL__CFG_BLOCK_P_EN__SHIFT                                                               0x8
+#define BIFC_HSTARB_CNTL__SLVARB_MODE_MASK                                                                    0x00000003L
+#define BIFC_HSTARB_CNTL__CFG_BLOCK_P_EN_MASK                                                                 0x00000100L
+//BIFC_GSI_CNTL
+#define BIFC_GSI_CNTL__GSI_SDP_RSP_ARB_MODE__SHIFT                                                            0x0
+#define BIFC_GSI_CNTL__GSI_CPL_RSP_ARB_MODE__SHIFT                                                            0x2
+#define BIFC_GSI_CNTL__GSI_CPL_INTERLEAVING_EN__SHIFT                                                         0x6
+#define BIFC_GSI_CNTL__GSI_CPL_PCR_EP_CAUSE_UR_EN__SHIFT                                                      0x7
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_P_EP_CAUSE_UR_EN__SHIFT                                                    0x8
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_NP_EP_CAUSE_UR_EN__SHIFT                                                   0x9
+#define BIFC_GSI_CNTL__GSI_CPL_SST_EP_CAUSE_UR_EN__SHIFT                                                      0xa
+#define BIFC_GSI_CNTL__GSI_SDP_REQ_ARB_MODE__SHIFT                                                            0xb
+#define BIFC_GSI_CNTL__GSI_SMN_REQ_ARB_MODE__SHIFT                                                            0xd
+#define BIFC_GSI_CNTL__GSI_CPL_SST_ATOMIC_EP_CAUSE_UR_EN__SHIFT                                               0xf
+#define BIFC_GSI_CNTL__GSI_SMN_PARITY_CHK_BE_MSK__SHIFT                                                       0x10
+#define BIFC_GSI_CNTL__GSI_SMN_BURST_EN__SHIFT                                                                0x11
+#define BIFC_GSI_CNTL__GSI_SMN_256B_SPLIT_64B_EN__SHIFT                                                       0x12
+#define BIFC_GSI_CNTL__NBIF_UR_HPHIT_TRANS_DIS_EN__SHIFT                                                      0x16
+#define BIFC_GSI_CNTL__SMN_PP_PIPE_ENABLE__SHIFT                                                              0x1b
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_FBFLUSH__SHIFT                                                    0x1c
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPFLUSH__SHIFT                                                   0x1d
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPRD__SHIFT                                                      0x1e
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_ALL__SHIFT                                                        0x1f
+#define BIFC_GSI_CNTL__GSI_SDP_RSP_ARB_MODE_MASK                                                              0x00000003L
+#define BIFC_GSI_CNTL__GSI_CPL_RSP_ARB_MODE_MASK                                                              0x0000003CL
+#define BIFC_GSI_CNTL__GSI_CPL_INTERLEAVING_EN_MASK                                                           0x00000040L
+#define BIFC_GSI_CNTL__GSI_CPL_PCR_EP_CAUSE_UR_EN_MASK                                                        0x00000080L
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_P_EP_CAUSE_UR_EN_MASK                                                      0x00000100L
+#define BIFC_GSI_CNTL__GSI_CPL_SMN_NP_EP_CAUSE_UR_EN_MASK                                                     0x00000200L
+#define BIFC_GSI_CNTL__GSI_CPL_SST_EP_CAUSE_UR_EN_MASK                                                        0x00000400L
+#define BIFC_GSI_CNTL__GSI_SDP_REQ_ARB_MODE_MASK                                                              0x00001800L
+#define BIFC_GSI_CNTL__GSI_SMN_REQ_ARB_MODE_MASK                                                              0x00006000L
+#define BIFC_GSI_CNTL__GSI_CPL_SST_ATOMIC_EP_CAUSE_UR_EN_MASK                                                 0x00008000L
+#define BIFC_GSI_CNTL__GSI_SMN_PARITY_CHK_BE_MSK_MASK                                                         0x00010000L
+#define BIFC_GSI_CNTL__GSI_SMN_BURST_EN_MASK                                                                  0x00020000L
+#define BIFC_GSI_CNTL__GSI_SMN_256B_SPLIT_64B_EN_MASK                                                         0x00040000L
+#define BIFC_GSI_CNTL__NBIF_UR_HPHIT_TRANS_DIS_EN_MASK                                                        0x00400000L
+#define BIFC_GSI_CNTL__SMN_PP_PIPE_ENABLE_MASK                                                                0x08000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_FBFLUSH_MASK                                                      0x10000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPFLUSH_MASK                                                     0x20000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_HDPRD_MASK                                                        0x40000000L
+#define BIFC_GSI_CNTL__HDP_FB_UPLIMIT_COUNT_ALL_MASK                                                          0x80000000L
+//BIFC_PCIEFUNC_CNTL
+#define BIFC_PCIEFUNC_CNTL__DMA_NON_PCIEFUNC_BUSDEVFUNC__SHIFT                                                0x0
+#define BIFC_PCIEFUNC_CNTL__MP1SYSHUBDATA_DRAM_IS_PCIEFUNC__SHIFT                                             0x10
+#define BIFC_PCIEFUNC_CNTL__DMA_NON_PCIEFUNC_BUSDEVFUNC_MASK                                                  0x0000FFFFL
+#define BIFC_PCIEFUNC_CNTL__MP1SYSHUBDATA_DRAM_IS_PCIEFUNC_MASK                                               0x00010000L
+//BIFC_PASID_CHECK_DIS
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F0__SHIFT                                                  0x0
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F1__SHIFT                                                  0x1
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F0_MASK                                                    0x00000001L
+#define BIFC_PASID_CHECK_DIS__PASID_CHECK_DIS_DEV0_F1_MASK                                                    0x00000002L
+//BIFC_SDP_CNTL_0
+#define BIFC_SDP_CNTL_0__HRP_SDP_DISCON_HYSTERESIS__SHIFT                                                     0x0
+#define BIFC_SDP_CNTL_0__GSI_SDP_DISCON_HYSTERESIS__SHIFT                                                     0x8
+#define BIFC_SDP_CNTL_0__GMI_DNS_SDP_DISCON_HYSTERESIS__SHIFT                                                 0x10
+#define BIFC_SDP_CNTL_0__GMI_UPS_SDP_DISCON_HYSTERESIS__SHIFT                                                 0x18
+#define BIFC_SDP_CNTL_0__HRP_SDP_DISCON_HYSTERESIS_MASK                                                       0x000000FFL
+#define BIFC_SDP_CNTL_0__GSI_SDP_DISCON_HYSTERESIS_MASK                                                       0x0000FF00L
+#define BIFC_SDP_CNTL_0__GMI_DNS_SDP_DISCON_HYSTERESIS_MASK                                                   0x00FF0000L
+#define BIFC_SDP_CNTL_0__GMI_UPS_SDP_DISCON_HYSTERESIS_MASK                                                   0xFF000000L
+//BIFC_SDP_CNTL_1
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_DIS__SHIFT                                                            0x0
+#define BIFC_SDP_CNTL_1__GSI_SDP_DISCON_DIS__SHIFT                                                            0x1
+#define BIFC_SDP_CNTL_1__GMI_DNS_SDP_DISCON_DIS__SHIFT                                                        0x2
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_DIS__SHIFT                                                        0x3
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY__SHIFT                                               0x4
+#define BIFC_SDP_CNTL_1__NP_KEEP_GOING_STALL_P__SHIFT                                                         0x5
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY__SHIFT                                           0x7
+#define BIFC_SDP_CNTL_1__ATOMIC_STALL_BY_RDWR_EN__SHIFT                                                       0x8
+#define BIFC_SDP_CNTL_1__POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT                                            0x9
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_DIS_MASK                                                              0x00000001L
+#define BIFC_SDP_CNTL_1__GSI_SDP_DISCON_DIS_MASK                                                              0x00000002L
+#define BIFC_SDP_CNTL_1__GMI_DNS_SDP_DISCON_DIS_MASK                                                          0x00000004L
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_DIS_MASK                                                          0x00000008L
+#define BIFC_SDP_CNTL_1__HRP_SDP_DISCON_VLINK_NONL0_ONLY_MASK                                                 0x00000010L
+#define BIFC_SDP_CNTL_1__NP_KEEP_GOING_STALL_P_MASK                                                           0x00000020L
+#define BIFC_SDP_CNTL_1__GMI_UPS_SDP_DISCON_VLINK_NONL0_ONLY_MASK                                             0x00000080L
+#define BIFC_SDP_CNTL_1__ATOMIC_STALL_BY_RDWR_EN_MASK                                                         0x00000100L
+#define BIFC_SDP_CNTL_1__POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK                                              0x00000200L
+//BIFC_PASID_STS
+#define BIFC_PASID_STS__PASID_STS__SHIFT                                                                      0x0
+#define BIFC_PASID_STS__PASID_STS_MASK                                                                        0x0000000FL
+//BIFC_ATHUB_ACT_CNTL
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_RSP_STS_TYPE__SHIFT                                                0x0
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_SLFR_DATAERR_RSP_STS_TYPE__SHIFT                                           0x3
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_REQ_DROP_DIS__SHIFT                                                0x8
+#define BIFC_ATHUB_ACT_CNTL__GSI_ATHUB_ACT_FLUSH_TRIGGER__SHIFT                                               0x9
+#define BIFC_ATHUB_ACT_CNTL__GMI_ATHUB_ACT_FLUSH_TRIGGER__SHIFT                                               0xa
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_SST_PP_REQ_DROP_EN__SHIFT                                          0xb
+#define BIFC_ATHUB_ACT_CNTL__ERREVENT_MODE1RST_FLUSH_LOG_AER_EN__SHIFT                                        0xc
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_RSP_STS_TYPE_MASK                                                  0x00000007L
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_SLFR_DATAERR_RSP_STS_TYPE_MASK                                             0x00000038L
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_REQ_DROP_DIS_MASK                                                  0x00000100L
+#define BIFC_ATHUB_ACT_CNTL__GSI_ATHUB_ACT_FLUSH_TRIGGER_MASK                                                 0x00000200L
+#define BIFC_ATHUB_ACT_CNTL__GMI_ATHUB_ACT_FLUSH_TRIGGER_MASK                                                 0x00000400L
+#define BIFC_ATHUB_ACT_CNTL__ATHUB_ACT_GSI_SST_PP_REQ_DROP_EN_MASK                                            0x00000800L
+#define BIFC_ATHUB_ACT_CNTL__ERREVENT_MODE1RST_FLUSH_LOG_AER_EN_MASK                                          0x00001000L
+//BIFC_PERF_CNTL_0
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_EN__SHIFT                                                          0x0
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_EN__SHIFT                                                          0x1
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_RESET__SHIFT                                                       0x8
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_RESET__SHIFT                                                       0x9
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_SEL__SHIFT                                                         0x10
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_SEL__SHIFT                                                         0x18
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_EN_MASK                                                            0x00000001L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_EN_MASK                                                            0x00000002L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_RESET_MASK                                                         0x00000100L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_RESET_MASK                                                         0x00000200L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_RD_SEL_MASK                                                           0x007F0000L
+#define BIFC_PERF_CNTL_0__PERF_CNT_MMIO_WR_SEL_MASK                                                           0x7F000000L
+//BIF_MISC_BIFC_PERF_CNTL_1
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_EN__SHIFT                                                  0x0
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_EN__SHIFT                                                  0x1
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_RESET__SHIFT                                               0x4
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_RESET__SHIFT                                               0x5
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_SEL__SHIFT                                                 0x8
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_SEL__SHIFT                                                 0x10
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_EN_MASK                                                    0x00000001L
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_EN_MASK                                                    0x00000002L
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_RESET_MASK                                                 0x00000010L
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_RESET_MASK                                                 0x00000020L
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_RD_SEL_MASK                                                   0x0000FF00L
+#define BIF_MISC_BIFC_PERF_CNTL_1__PERF_CNT_DMA_WR_SEL_MASK                                                   0x03FF0000L
+//BIFC_PERF_CNT_MMIO_RD_L32BIT
+#define BIFC_PERF_CNT_MMIO_RD_L32BIT__PERF_CNT_MMIO_RD_VALUE_L32BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_RD_L32BIT__PERF_CNT_MMIO_RD_VALUE_L32BIT_MASK                                      0xFFFFFFFFL
+//BIFC_PERF_CNT_MMIO_WR_L32BIT
+#define BIFC_PERF_CNT_MMIO_WR_L32BIT__PERF_CNT_MMIO_WR_VALUE_L32BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_WR_L32BIT__PERF_CNT_MMIO_WR_VALUE_L32BIT_MASK                                      0xFFFFFFFFL
+//BIF_MISC_BIFC_PERF_CNT_DMA_RD_L32BIT
+#define BIF_MISC_BIFC_PERF_CNT_DMA_RD_L32BIT__PERF_CNT_DMA_RD_VALUE_L32BIT__SHIFT                             0x0
+#define BIF_MISC_BIFC_PERF_CNT_DMA_RD_L32BIT__PERF_CNT_DMA_RD_VALUE_L32BIT_MASK                               0xFFFFFFFFL
+//BIF_MISC_BIFC_PERF_CNT_DMA_WR_L32BIT
+#define BIF_MISC_BIFC_PERF_CNT_DMA_WR_L32BIT__PERF_CNT_DMA_WR_VALUE_L32BIT__SHIFT                             0x0
+#define BIF_MISC_BIFC_PERF_CNT_DMA_WR_L32BIT__PERF_CNT_DMA_WR_VALUE_L32BIT_MASK                               0xFFFFFFFFL
+//NBIF_REGIF_ERRSET_CTRL
+#define NBIF_REGIF_ERRSET_CTRL__DROP_NONPF_MMREGREQ_SETERR_DIS__SHIFT                                         0x0
+#define NBIF_REGIF_ERRSET_CTRL__DROP_NONPF_MMREGREQ_SETERR_DIS_MASK                                           0x00000001L
+//BIFC_SDP_CNTL_2
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS__SHIFT                                                    0x0
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS_H__SHIFT                                                  0x8
+#define BIFC_SDP_CNTL_2__HRP_SDP_DISCON_HYSTERESIS_H__SHIFT                                                   0x10
+#define BIFC_SDP_CNTL_2__GSI_SDP_DISCON_HYSTERESIS_H__SHIFT                                                   0x18
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS_MASK                                                      0x000000FFL
+#define BIFC_SDP_CNTL_2__SDP_SION_DISCON_HYSTERESIS_H_MASK                                                    0x00000F00L
+#define BIFC_SDP_CNTL_2__HRP_SDP_DISCON_HYSTERESIS_H_MASK                                                     0x000F0000L
+#define BIFC_SDP_CNTL_2__GSI_SDP_DISCON_HYSTERESIS_H_MASK                                                     0x0F000000L
+//NBIF_PGMST_CTRL
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_HYSTERESIS__SHIFT                                                        0x0
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_EN__SHIFT                                                                0x8
+#define NBIF_PGMST_CTRL__NBIF_CFG_IDLENESS_COUNT_EN__SHIFT                                                    0xa
+#define NBIF_PGMST_CTRL__NBIF_CFG_FW_PG_EXIT_EN__SHIFT                                                        0xe
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_HYSTERESIS_MASK                                                          0x000000FFL
+#define NBIF_PGMST_CTRL__NBIF_CFG_PG_EN_MASK                                                                  0x00000100L
+#define NBIF_PGMST_CTRL__NBIF_CFG_IDLENESS_COUNT_EN_MASK                                                      0x00003C00L
+#define NBIF_PGMST_CTRL__NBIF_CFG_FW_PG_EXIT_EN_MASK                                                          0x0000C000L
+//NBIF_PGSLV_CTRL
+#define NBIF_PGSLV_CTRL__NBIF_CFG_IDLE_HYSTERESIS__SHIFT                                                      0x0
+#define NBIF_PGSLV_CTRL__NBIF_CFG_IDLE_HYSTERESIS_MASK                                                        0x0000001FL
+//NBIF_PG_MISC_CTRL
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_0_IDLE_HYSTERESIS__SHIFT                                          0x0
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_1_IDLE_HYSTERESIS__SHIFT                                          0x5
+#define NBIF_PG_MISC_CTRL__NBIF_PG_ENDP_D3_ONLY__SHIFT                                                        0xa
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM1__SHIFT                                                           0xd
+#define NBIF_PG_MISC_CTRL__NBIF_PG_DS_ALLOW_DIS__SHIFT                                                        0xe
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM2__SHIFT                                                           0x10
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_REFCLK_CYCLE_FOR_200NS__SHIFT                                             0x18
+#define NBIF_PG_MISC_CTRL__NBIF_PG_PCIE_NBIF_LD_MASK__SHIFT                                                   0x1e
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_PG_EXIT_OVERRIDE__SHIFT                                                   0x1f
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_0_IDLE_HYSTERESIS_MASK                                            0x0000001FL
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_SHUBCLK_1_IDLE_HYSTERESIS_MASK                                            0x000003E0L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_ENDP_D3_ONLY_MASK                                                          0x00000400L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM1_MASK                                                             0x00002000L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_DS_ALLOW_DIS_MASK                                                          0x00004000L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_CLK_PERM2_MASK                                                             0x00010000L
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_REFCLK_CYCLE_FOR_200NS_MASK                                               0x3F000000L
+#define NBIF_PG_MISC_CTRL__NBIF_PG_PCIE_NBIF_LD_MASK_MASK                                                     0x40000000L
+#define NBIF_PG_MISC_CTRL__NBIF_CFG_PG_EXIT_OVERRIDE_MASK                                                     0x80000000L
+//SMN_MST_EP_CNTL3
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF0__SHIFT                                                0x0
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF1__SHIFT                                                0x1
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF2__SHIFT                                                0x2
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF3__SHIFT                                                0x3
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF4__SHIFT                                                0x4
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF5__SHIFT                                                0x5
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF6__SHIFT                                                0x6
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF7__SHIFT                                                0x7
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF0_MASK                                                  0x00000001L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF1_MASK                                                  0x00000002L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF2_MASK                                                  0x00000004L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF3_MASK                                                  0x00000008L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF4_MASK                                                  0x00000010L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF5_MASK                                                  0x00000020L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF6_MASK                                                  0x00000040L
+#define SMN_MST_EP_CNTL3__SMN_ZERO_BE_WR_EN_EP_DEV0_PF7_MASK                                                  0x00000080L
+//SMN_MST_EP_CNTL4
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF0__SHIFT                                                0x0
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF1__SHIFT                                                0x1
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF2__SHIFT                                                0x2
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF3__SHIFT                                                0x3
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF4__SHIFT                                                0x4
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF5__SHIFT                                                0x5
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF6__SHIFT                                                0x6
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF7__SHIFT                                                0x7
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF0_MASK                                                  0x00000001L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF1_MASK                                                  0x00000002L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF2_MASK                                                  0x00000004L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF3_MASK                                                  0x00000008L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF4_MASK                                                  0x00000010L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF5_MASK                                                  0x00000020L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF6_MASK                                                  0x00000040L
+#define SMN_MST_EP_CNTL4__SMN_ZERO_BE_RD_EN_EP_DEV0_PF7_MASK                                                  0x00000080L
+//SMN_MST_CNTL1
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_UPS__SHIFT                                                    0x0
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV0__SHIFT                                               0x10
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_UPS_MASK                                                      0x00000001L
+#define SMN_MST_CNTL1__SMN_ERRRSP_DATA_ALLF_DIS_DNS_DEV0_MASK                                                 0x00010000L
+//SMN_MST_EP_CNTL5
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF0__SHIFT                                         0x0
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF1__SHIFT                                         0x1
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF2__SHIFT                                         0x2
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF3__SHIFT                                         0x3
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF4__SHIFT                                         0x4
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF5__SHIFT                                         0x5
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF6__SHIFT                                         0x6
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF7__SHIFT                                         0x7
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF0_MASK                                           0x00000001L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF1_MASK                                           0x00000002L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF2_MASK                                           0x00000004L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF3_MASK                                           0x00000008L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF4_MASK                                           0x00000010L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF5_MASK                                           0x00000020L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF6_MASK                                           0x00000040L
+#define SMN_MST_EP_CNTL5__SMN_ERRRSP_DATA_ALLF_DIS_EP_DEV0_PF7_MASK                                           0x00000080L
+//BIF_SELFRING_BUFFER_VID
+#define BIF_SELFRING_BUFFER_VID__DOORBELL_MONITOR_CID__SHIFT                                                  0x0
+#define BIF_SELFRING_BUFFER_VID__RAS_CNTLR_INTR_CID__SHIFT                                                    0x8
+#define BIF_SELFRING_BUFFER_VID__RAS_ATHUB_ERR_EVENT_INTR_CID__SHIFT                                          0x10
+#define BIF_SELFRING_BUFFER_VID__DOORBELL_MONITOR_CID_MASK                                                    0x000000FFL
+#define BIF_SELFRING_BUFFER_VID__RAS_CNTLR_INTR_CID_MASK                                                      0x0000FF00L
+#define BIF_SELFRING_BUFFER_VID__RAS_ATHUB_ERR_EVENT_INTR_CID_MASK                                            0x00FF0000L
+//BIF_SELFRING_VECTOR_CNTL
+#define BIF_SELFRING_VECTOR_CNTL__MISC_DB_MNTR_INTR_DIS__SHIFT                                                0x0
+#define BIF_SELFRING_VECTOR_CNTL__DB_MNTR_TS_FROM__SHIFT                                                      0x1
+#define BIF_SELFRING_VECTOR_CNTL__MISC_DB_MNTR_INTR_DIS_MASK                                                  0x00000001L
+#define BIF_SELFRING_VECTOR_CNTL__DB_MNTR_TS_FROM_MASK                                                        0x00000002L
+//HDP_ERR_STATUS_EXT
+#define HDP_ERR_STATUS_EXT__HDP_ERR_STATUS_WR_EXT__SHIFT                                                      0x0
+#define HDP_ERR_STATUS_EXT__HDP_ERR_STATUS_RD_EXT__SHIFT                                                      0x1
+#define HDP_ERR_STATUS_EXT__HDP_ERR_STATUS_WR_EXT_MASK                                                        0x00000001L
+#define HDP_ERR_STATUS_EXT__HDP_ERR_STATUS_RD_EXT_MASK                                                        0x00000002L
+//NBIF_STRAP_WRITE_CTRL
+#define NBIF_STRAP_WRITE_CTRL__NBIF_STRAP_WRITE_ONCE_ENABLE__SHIFT                                            0x0
+#define NBIF_STRAP_WRITE_CTRL__NBIF_STRAP_WRITE_ONCE_ENABLE_MASK                                              0x00000001L
+//NBIF_INTX_DSTATE_MISC_CNTL
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_EP__SHIFT                                      0x0
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_DN__SHIFT                                      0x1
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_SWUS__SHIFT                                    0x2
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_EP__SHIFT                                         0x3
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_DN__SHIFT                                         0x4
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_EP__SHIFT                                                     0x5
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_DN__SHIFT                                                     0x6
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_SWUS__SHIFT                                                   0x7
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_EP_MASK                                        0x00000001L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_DN_MASK                                        0x00000002L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_DSTATE_CHK_DIS_SWUS_MASK                                      0x00000004L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_EP_MASK                                           0x00000008L
+#define NBIF_INTX_DSTATE_MISC_CNTL__DEASRT_INTX_IN_NOND0_EN_DN_MASK                                           0x00000010L
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_EP_MASK                                                       0x00000020L
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_DN_MASK                                                       0x00000040L
+#define NBIF_INTX_DSTATE_MISC_CNTL__PMI_INT_DIS_SWUS_MASK                                                     0x00000080L
+//NBIF_PENDING_MISC_CNTL
+#define NBIF_PENDING_MISC_CNTL__FLR_MST_PEND_CHK_DIS__SHIFT                                                   0x0
+#define NBIF_PENDING_MISC_CNTL__FLR_SLV_PEND_CHK_DIS__SHIFT                                                   0x1
+#define NBIF_PENDING_MISC_CNTL__FLR_MST_PEND_CHK_DIS_MASK                                                     0x00000001L
+#define NBIF_PENDING_MISC_CNTL__FLR_SLV_PEND_CHK_DIS_MASK                                                     0x00000002L
+//BIF_GMI_WRR_WEIGHT
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_COUNTER_MODE__SHIFT                                               0x1d
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_MODE__SHIFT                                                       0x1e
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_SIZE_MODE__SHIFT                                                  0x1f
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_COUNTER_MODE_MASK                                                 0x20000000L
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_MODE_MASK                                                         0x40000000L
+#define BIF_GMI_WRR_WEIGHT__GMI_REQ_WRR_LRG_SIZE_MODE_MASK                                                    0x80000000L
+//BIF_GMI_WRR_WEIGHT2
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY0_WEIGHT__SHIFT                                                     0x0
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY1_WEIGHT__SHIFT                                                     0x8
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY2_WEIGHT__SHIFT                                                     0x10
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY3_WEIGHT__SHIFT                                                     0x18
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY0_WEIGHT_MASK                                                       0x000000FFL
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY1_WEIGHT_MASK                                                       0x0000FF00L
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY2_WEIGHT_MASK                                                       0x00FF0000L
+#define BIF_GMI_WRR_WEIGHT2__GMI_REQ_ENTRY3_WEIGHT_MASK                                                       0xFF000000L
+//BIF_GMI_WRR_WEIGHT3
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY4_WEIGHT__SHIFT                                                     0x0
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY5_WEIGHT__SHIFT                                                     0x8
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY6_WEIGHT__SHIFT                                                     0x10
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY7_WEIGHT__SHIFT                                                     0x18
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY4_WEIGHT_MASK                                                       0x000000FFL
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY5_WEIGHT_MASK                                                       0x0000FF00L
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY6_WEIGHT_MASK                                                       0x00FF0000L
+#define BIF_GMI_WRR_WEIGHT3__GMI_REQ_ENTRY7_WEIGHT_MASK                                                       0xFF000000L
+//BIF_GMI_WRR_WEIGHT4
+#define BIF_GMI_WRR_WEIGHT4__GMI_REQ_ENTRY8_WEIGHT__SHIFT                                                     0x0
+#define BIF_GMI_WRR_WEIGHT4__GMI_REQ_ENTRY8_WEIGHT_MASK                                                       0x000000FFL
+//NBIF_PWRBRK_REQUEST
+#define NBIF_PWRBRK_REQUEST__NBIF_PWRBRK_REQUEST__SHIFT                                                       0x0
+#define NBIF_PWRBRK_REQUEST__NBIF_PWRBRK_REQUEST_MASK                                                         0x00000001L
+//BIF_ATOMIC_ERR_LOG_DEV0_F0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_OPCODE_DEV0_F0__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_REQEN_LOW_DEV0_F0__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_LENGTH_DEV0_F0__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_NR_DEV0_F0__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_NR_DEV0_F0__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_OPCODE_DEV0_F0_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_REQEN_LOW_DEV0_F0_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_LENGTH_DEV0_F0_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__UR_ATOMIC_NR_DEV0_F0_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_NR_DEV0_F0_MASK                                           0x00080000L
+//BIF_ATOMIC_ERR_LOG_DEV0_F1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_OPCODE_DEV0_F1__SHIFT                                           0x0
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_REQEN_LOW_DEV0_F1__SHIFT                                        0x1
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_LENGTH_DEV0_F1__SHIFT                                           0x2
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_NR_DEV0_F1__SHIFT                                               0x3
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_OPCODE_DEV0_F1__SHIFT                                     0x10
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F1__SHIFT                                  0x11
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_LENGTH_DEV0_F1__SHIFT                                     0x12
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_NR_DEV0_F1__SHIFT                                         0x13
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_OPCODE_DEV0_F1_MASK                                             0x00000001L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_REQEN_LOW_DEV0_F1_MASK                                          0x00000002L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_LENGTH_DEV0_F1_MASK                                             0x00000004L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__UR_ATOMIC_NR_DEV0_F1_MASK                                                 0x00000008L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_OPCODE_DEV0_F1_MASK                                       0x00010000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F1_MASK                                    0x00020000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_LENGTH_DEV0_F1_MASK                                       0x00040000L
+#define BIF_ATOMIC_ERR_LOG_DEV0_F1__CLEAR_UR_ATOMIC_NR_DEV0_F1_MASK                                           0x00080000L
+//BIF_DMA_MP4_ERR_LOG
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_VC4_NON_DVM_ERR__SHIFT                                                    0x0
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_ATOMIC_REQEN_LOW_ERR__SHIFT                                               0x1
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_VC4_NON_DVM_ERR__SHIFT                                              0x10
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_ATOMIC_REQEN_LOW_ERR__SHIFT                                         0x11
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_VC4_NON_DVM_ERR_MASK                                                      0x00000001L
+#define BIF_DMA_MP4_ERR_LOG__MP4SDP_ATOMIC_REQEN_LOW_ERR_MASK                                                 0x00000002L
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_VC4_NON_DVM_ERR_MASK                                                0x00010000L
+#define BIF_DMA_MP4_ERR_LOG__CLEAR_MP4SDP_ATOMIC_REQEN_LOW_ERR_MASK                                           0x00020000L
+//BIF_PASID_ERR_LOG
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F0__SHIFT                                                           0x0
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F1__SHIFT                                                           0x1
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F0_MASK                                                             0x00000001L
+#define BIF_PASID_ERR_LOG__PASID_ERR_DEV0_F1_MASK                                                             0x00000002L
+//BIF_PASID_ERR_CLR
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F0__SHIFT                                                       0x0
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F1__SHIFT                                                       0x1
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F0_MASK                                                         0x00000001L
+#define BIF_PASID_ERR_CLR__PASID_ERR_CLR_DEV0_F1_MASK                                                         0x00000002L
+//NBIF_VWIRE_CTRL
+#define NBIF_VWIRE_CTRL__NBIF_SMN_VWR_DIS__SHIFT                                                              0x0
+#define NBIF_VWIRE_CTRL__SMN_VWR_RESET_DELAY_CNT__SHIFT                                                       0x4
+#define NBIF_VWIRE_CTRL__SMN_VWR_POSTED__SHIFT                                                                0x8
+#define NBIF_VWIRE_CTRL__NBIF_SDP_UPS_VWR_DIS__SHIFT                                                          0x10
+#define NBIF_VWIRE_CTRL__SDP_VWR_RESET_DELAY_CNT__SHIFT                                                       0x14
+#define NBIF_VWIRE_CTRL__SDP_VWR_BLOCKLVL__SHIFT                                                              0x1a
+#define NBIF_VWIRE_CTRL__NBIF_SMN_VWR_DIS_MASK                                                                0x00000001L
+#define NBIF_VWIRE_CTRL__SMN_VWR_RESET_DELAY_CNT_MASK                                                         0x000000F0L
+#define NBIF_VWIRE_CTRL__SMN_VWR_POSTED_MASK                                                                  0x00000100L
+#define NBIF_VWIRE_CTRL__NBIF_SDP_UPS_VWR_DIS_MASK                                                            0x00010000L
+#define NBIF_VWIRE_CTRL__SDP_VWR_RESET_DELAY_CNT_MASK                                                         0x00F00000L
+#define NBIF_VWIRE_CTRL__SDP_VWR_BLOCKLVL_MASK                                                                0x0C000000L
+//NBIF_MGCG_CTRL_LCLK
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_EN_LCLK__SHIFT                                                         0x0
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_MODE_LCLK__SHIFT                                                       0x1
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HYSTERESIS_LCLK__SHIFT                                                 0x2
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HST_DIS_LCLK__SHIFT                                                    0xa
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DMA_DIS_LCLK__SHIFT                                                    0xb
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_REG_DIS_LCLK__SHIFT                                                    0xc
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_AER_DIS_LCLK__SHIFT                                                    0xd
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DBG_DIS_LCLK__SHIFT                                                    0xe
+#define NBIF_MGCG_CTRL_LCLK__NBIF_SRAM_FGCG_EN_LCLK__SHIFT                                                    0xf
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_EN_LCLK_MASK                                                           0x00000001L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_MODE_LCLK_MASK                                                         0x00000002L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HYSTERESIS_LCLK_MASK                                                   0x000003FCL
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_HST_DIS_LCLK_MASK                                                      0x00000400L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DMA_DIS_LCLK_MASK                                                      0x00000800L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_REG_DIS_LCLK_MASK                                                      0x00001000L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_AER_DIS_LCLK_MASK                                                      0x00002000L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_MGCG_DBG_DIS_LCLK_MASK                                                      0x00004000L
+#define NBIF_MGCG_CTRL_LCLK__NBIF_SRAM_FGCG_EN_LCLK_MASK                                                      0x00008000L
+//NBIF_DS_CTRL_LCLK
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_EN__SHIFT                                                             0x0
+#define NBIF_DS_CTRL_LCLK__ATHUB_LCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                           0x1
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_TIMER__SHIFT                                                          0x10
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_EN_MASK                                                               0x00000001L
+#define NBIF_DS_CTRL_LCLK__ATHUB_LCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                             0x00000002L
+#define NBIF_DS_CTRL_LCLK__NBIF_LCLK_DS_TIMER_MASK                                                            0xFFFF0000L
+//SMN_MST_CNTL0
+#define SMN_MST_CNTL0__SMN_ARB_MODE__SHIFT                                                                    0x0
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_UPS__SHIFT                                                           0x8
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_UPS__SHIFT                                                           0x9
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_UPS__SHIFT                                                            0xa
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_UPS__SHIFT                                                      0xb
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV0__SHIFT                                                      0x10
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV0__SHIFT                                                      0x14
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV0__SHIFT                                                       0x18
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV0__SHIFT                                                 0x1c
+#define SMN_MST_CNTL0__SMN_ARB_MODE_MASK                                                                      0x00000003L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_UPS_MASK                                                             0x00000100L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_UPS_MASK                                                             0x00000200L
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_UPS_MASK                                                              0x00000400L
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_UPS_MASK                                                        0x00000800L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_WR_EN_DNS_DEV0_MASK                                                        0x00010000L
+#define SMN_MST_CNTL0__SMN_ZERO_BE_RD_EN_DNS_DEV0_MASK                                                        0x00100000L
+#define SMN_MST_CNTL0__SMN_POST_MASK_EN_DNS_DEV0_MASK                                                         0x01000000L
+#define SMN_MST_CNTL0__MULTI_SMN_TRANS_ID_DIS_DNS_DEV0_MASK                                                   0x10000000L
+//SMN_MST_EP_CNTL1
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF0__SHIFT                                                 0x0
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF1__SHIFT                                                 0x1
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF2__SHIFT                                                 0x2
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF3__SHIFT                                                 0x3
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF4__SHIFT                                                 0x4
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF5__SHIFT                                                 0x5
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF6__SHIFT                                                 0x6
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF7__SHIFT                                                 0x7
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF0_MASK                                                   0x00000001L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF1_MASK                                                   0x00000002L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF2_MASK                                                   0x00000004L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF3_MASK                                                   0x00000008L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF4_MASK                                                   0x00000010L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF5_MASK                                                   0x00000020L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF6_MASK                                                   0x00000040L
+#define SMN_MST_EP_CNTL1__SMN_POST_MASK_EN_EP_DEV0_PF7_MASK                                                   0x00000080L
+//SMN_MST_EP_CNTL2
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF0__SHIFT                                           0x0
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF1__SHIFT                                           0x1
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF2__SHIFT                                           0x2
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF3__SHIFT                                           0x3
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF4__SHIFT                                           0x4
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF5__SHIFT                                           0x5
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF6__SHIFT                                           0x6
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF7__SHIFT                                           0x7
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF0_MASK                                             0x00000001L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF1_MASK                                             0x00000002L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF2_MASK                                             0x00000004L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF3_MASK                                             0x00000008L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF4_MASK                                             0x00000010L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF5_MASK                                             0x00000020L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF6_MASK                                             0x00000040L
+#define SMN_MST_EP_CNTL2__MULTI_SMN_TRANS_ID_DIS_EP_DEV0_PF7_MASK                                             0x00000080L
+//NBIF_SDP_VWR_VCHG_DIS_CTRL
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F0_DIS__SHIFT                                           0x0
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F1_DIS__SHIFT                                           0x1
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F2_DIS__SHIFT                                           0x2
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F3_DIS__SHIFT                                           0x3
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F4_DIS__SHIFT                                           0x4
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F5_DIS__SHIFT                                           0x5
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F6_DIS__SHIFT                                           0x6
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F7_DIS__SHIFT                                           0x7
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_SWDS_P0_DIS__SHIFT                                           0x18
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F0_DIS_MASK                                             0x00000001L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F1_DIS_MASK                                             0x00000002L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F2_DIS_MASK                                             0x00000004L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F3_DIS_MASK                                             0x00000008L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F4_DIS_MASK                                             0x00000010L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F5_DIS_MASK                                             0x00000020L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F6_DIS_MASK                                             0x00000040L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_ENDP_F7_DIS_MASK                                             0x00000080L
+#define NBIF_SDP_VWR_VCHG_DIS_CTRL__SDP_VWR_VCHG_SWDS_P0_DIS_MASK                                             0x01000000L
+//NBIF_SDP_VWR_VCHG_RST_CTRL0
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_EN__SHIFT                                  0x0
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_EN__SHIFT                                  0x1
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_EN__SHIFT                                  0x2
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_EN__SHIFT                                  0x3
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_EN__SHIFT                                  0x4
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_EN__SHIFT                                  0x5
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_EN__SHIFT                                  0x6
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_EN__SHIFT                                  0x7
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_EN__SHIFT                                  0x18
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_EN_MASK                                    0x00000001L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_EN_MASK                                    0x00000002L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_EN_MASK                                    0x00000004L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_EN_MASK                                    0x00000008L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_EN_MASK                                    0x00000010L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_EN_MASK                                    0x00000020L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_EN_MASK                                    0x00000040L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_EN_MASK                                    0x00000080L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL0__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_EN_MASK                                    0x01000000L
+//NBIF_SDP_VWR_VCHG_RST_CTRL1
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_VAL__SHIFT                                 0x0
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_VAL__SHIFT                                 0x1
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_VAL__SHIFT                                 0x2
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_VAL__SHIFT                                 0x3
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_VAL__SHIFT                                 0x4
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_VAL__SHIFT                                 0x5
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_VAL__SHIFT                                 0x6
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_VAL__SHIFT                                 0x7
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_VAL__SHIFT                                 0x18
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F0_RST_OVRD_VAL_MASK                                   0x00000001L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F1_RST_OVRD_VAL_MASK                                   0x00000002L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F2_RST_OVRD_VAL_MASK                                   0x00000004L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F3_RST_OVRD_VAL_MASK                                   0x00000008L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F4_RST_OVRD_VAL_MASK                                   0x00000010L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F5_RST_OVRD_VAL_MASK                                   0x00000020L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F6_RST_OVRD_VAL_MASK                                   0x00000040L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_ENDP_F7_RST_OVRD_VAL_MASK                                   0x00000080L
+#define NBIF_SDP_VWR_VCHG_RST_CTRL1__SDP_VWR_VCHG_SWDS_P0_RST_OVRD_VAL_MASK                                   0x01000000L
+//NBIF_SDP_VWR_VCHG_TRIG
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F0_TRIG__SHIFT                                              0x0
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F1_TRIG__SHIFT                                              0x1
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F2_TRIG__SHIFT                                              0x2
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F3_TRIG__SHIFT                                              0x3
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F4_TRIG__SHIFT                                              0x4
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F5_TRIG__SHIFT                                              0x5
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F6_TRIG__SHIFT                                              0x6
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F7_TRIG__SHIFT                                              0x7
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_SWDS_P0_TRIG__SHIFT                                              0x18
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F0_TRIG_MASK                                                0x00000001L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F1_TRIG_MASK                                                0x00000002L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F2_TRIG_MASK                                                0x00000004L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F3_TRIG_MASK                                                0x00000008L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F4_TRIG_MASK                                                0x00000010L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F5_TRIG_MASK                                                0x00000020L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F6_TRIG_MASK                                                0x00000040L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_ENDP_F7_TRIG_MASK                                                0x00000080L
+#define NBIF_SDP_VWR_VCHG_TRIG__SDP_VWR_VCHG_SWDS_P0_TRIG_MASK                                                0x01000000L
+//NBIO_LCLK_DEEPSLEEP_MASK
+#define NBIO_LCLK_DEEPSLEEP_MASK__LCLK_DS_MASK__SHIFT                                                         0x0
+#define NBIO_LCLK_DEEPSLEEP_MASK__LCLK_DS_MASK_MASK                                                           0xFFFFFFFFL
+//NBIF_DS_CTRL_USBCLK
+#define NBIF_DS_CTRL_USBCLK__NBIF_USBCLK_DS_EN__SHIFT                                                         0x0
+#define NBIF_DS_CTRL_USBCLK__USB_USBCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                         0x1
+#define NBIF_DS_CTRL_USBCLK__NBIF_USBCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                        0x2
+#define NBIF_DS_CTRL_USBCLK__NBIF_USBCLK_DS_EN_MASK                                                           0x00000001L
+#define NBIF_DS_CTRL_USBCLK__USB_USBCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                           0x00000002L
+#define NBIF_DS_CTRL_USBCLK__NBIF_USBCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                          0x00000004L
+//NBIF_SHUB_TODET_CLIENT_MISC3
+#define NBIF_SHUB_TODET_CLIENT_MISC3__NBIF_SHUB_TODET_SLVERR_EN3__SHIFT                                       0x0
+#define NBIF_SHUB_TODET_CLIENT_MISC3__NBIF_SHUB_TODET_SYNCFLOOD_EN3__SHIFT                                    0xa
+#define NBIF_SHUB_TODET_CLIENT_MISC3__NBIF_SHUB_TODET_SLVERR_EN3_MASK                                         0x000003FFL
+#define NBIF_SHUB_TODET_CLIENT_MISC3__NBIF_SHUB_TODET_SYNCFLOOD_EN3_MASK                                      0x000FFC00L
+//NBIF_SHUB_TODET_CTRL
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_EN__SHIFT                                                       0x0
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_AER_LOG_EN__SHIFT                                               0x1
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_TIMER_UNIT__SHIFT                                               0x8
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TIMEOUT_COUNT__SHIFT                                                  0x10
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_EN_MASK                                                         0x00000001L
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_AER_LOG_EN_MASK                                                 0x00000002L
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TODET_TIMER_UNIT_MASK                                                 0x00000700L
+#define NBIF_SHUB_TODET_CTRL__NBIF_SHUB_TIMEOUT_COUNT_MASK                                                    0xFFFF0000L
+//NBIF_SHUB_TODET_CLIENT_CTRL
+#define NBIF_SHUB_TODET_CLIENT_CTRL__NBIF_SHUB_TODET_SLVERR_EN__SHIFT                                         0x0
+#define NBIF_SHUB_TODET_CLIENT_CTRL__NBIF_SHUB_TODET_SLVERR_EN_MASK                                           0xFFFFFFFFL
+//NBIF_SHUB_TODET_CLIENT_STATUS
+#define NBIF_SHUB_TODET_CLIENT_STATUS__NBIF_SHUB_TODET_CLIENT_STATUS__SHIFT                                   0x0
+#define NBIF_SHUB_TODET_CLIENT_STATUS__NBIF_SHUB_TODET_CLIENT_STATUS_MASK                                     0xFFFFFFFFL
+//NBIF_SHUB_TODET_SYNCFLOOD_CTRL
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL__NBIF_SHUB_TODET_SYNCFLOOD_EN__SHIFT                                   0x0
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL__NBIF_SHUB_TODET_SYNCFLOOD_EN_MASK                                     0xFFFFFFFFL
+//NBIF_SHUB_TODET_CLIENT_CTRL2
+#define NBIF_SHUB_TODET_CLIENT_CTRL2__NBIF_SHUB_TODET_SLVERR_EN2__SHIFT                                       0x0
+#define NBIF_SHUB_TODET_CLIENT_CTRL2__NBIF_SHUB_TODET_SLVERR_EN2_MASK                                         0xFFFFFFFFL
+//NBIF_SHUB_TODET_CLIENT_STATUS2
+#define NBIF_SHUB_TODET_CLIENT_STATUS2__NBIF_SHUB_TODET_CLIENT_STATUS2__SHIFT                                 0x0
+#define NBIF_SHUB_TODET_CLIENT_STATUS2__NBIF_SHUB_TODET_CLIENT_STATUS2_MASK                                   0xFFFFFFFFL
+//NBIF_SHUB_TODET_SYNCFLOOD_CTRL2
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL2__NBIF_SHUB_TODET_SYNCFLOOD_EN2__SHIFT                                 0x0
+#define NBIF_SHUB_TODET_SYNCFLOOD_CTRL2__NBIF_SHUB_TODET_SYNCFLOOD_EN2_MASK                                   0xFFFFFFFFL
+//DMA_SDP_PORT_DEBUG
+#define DMA_SDP_PORT_DEBUG__DMA_SDP_PORT_REQ_CREDIT_RDY_FOR_VC__SHIFT                                         0x0
+#define DMA_SDP_PORT_DEBUG__DMA_SDP_PORT_DAT_CREDIT_RDY_FOR_VC__SHIFT                                         0x8
+#define DMA_SDP_PORT_DEBUG__DMA_SDP_PORT_RDRSP_CREDIT_VLD_FOR_VC__SHIFT                                       0x10
+#define DMA_SDP_PORT_DEBUG__DMA_SDP_PORT_WRRSP_CREDIT_VLD_FOR_VC__SHIFT                                       0x18
+#define DMA_SDP_PORT_DEBUG__DMA_SDP_PORT_REQ_CREDIT_RDY_FOR_VC_MASK                                           0x000000FFL
+#define DMA_SDP_PORT_DEBUG__DMA_SDP_PORT_DAT_CREDIT_RDY_FOR_VC_MASK                                           0x0000FF00L
+#define DMA_SDP_PORT_DEBUG__DMA_SDP_PORT_RDRSP_CREDIT_VLD_FOR_VC_MASK                                         0x00FF0000L
+#define DMA_SDP_PORT_DEBUG__DMA_SDP_PORT_WRRSP_CREDIT_VLD_FOR_VC_MASK                                         0xFF000000L
+//SDP_PORT_MONITOR_CONTROL
+#define SDP_PORT_MONITOR_CONTROL__DMA_SDP_PORT_MONITOR_EN__SHIFT                                              0x0
+#define SDP_PORT_MONITOR_CONTROL__HST_SDP_PORT_MONITOR_EN__SHIFT                                              0x1
+#define SDP_PORT_MONITOR_CONTROL__SDP_PORT_MONITOR_ADDRESS_EN__SHIFT                                          0x2
+#define SDP_PORT_MONITOR_CONTROL__SDP_PORT_MONITOR_UNITID_EN__SHIFT                                           0x3
+#define SDP_PORT_MONITOR_CONTROL__DMA_SDP_PORT_MONITOR_EN_MASK                                                0x00000001L
+#define SDP_PORT_MONITOR_CONTROL__HST_SDP_PORT_MONITOR_EN_MASK                                                0x00000002L
+#define SDP_PORT_MONITOR_CONTROL__SDP_PORT_MONITOR_ADDRESS_EN_MASK                                            0x00000004L
+#define SDP_PORT_MONITOR_CONTROL__SDP_PORT_MONITOR_UNITID_EN_MASK                                             0x00000008L
+//SDP_PORT_MONITOR_ADDRESS0
+#define SDP_PORT_MONITOR_ADDRESS0__SDP_PORT_MONITOR_ADDRESS0__SHIFT                                           0x0
+#define SDP_PORT_MONITOR_ADDRESS0__SDP_PORT_MONITOR_ADDRESS0_MASK                                             0xFFFFFFFFL
+//SDP_PORT_MONITOR_ADDRESS1
+#define SDP_PORT_MONITOR_ADDRESS1__SDP_PORT_MONITOR_ADDRESS1__SHIFT                                           0x0
+#define SDP_PORT_MONITOR_ADDRESS1__SDP_PORT_MONITOR_ADDRESS1_MASK                                             0xFFFFFFFFL
+//SDP_PORT_MONITOR_UNITID
+#define SDP_PORT_MONITOR_UNITID__SDP_PORT_MONITOR_UNITID__SHIFT                                               0x0
+#define SDP_PORT_MONITOR_UNITID__SDP_PORT_MONITOR_UNITID_MASK                                                 0x000007FFL
+//SDP_PORT_MONITOR_INFO0
+#define SDP_PORT_MONITOR_INFO0__SDP_PORT_MONITOR_INFO_ADDRESS0__SHIFT                                         0x0
+#define SDP_PORT_MONITOR_INFO0__SDP_PORT_MONITOR_INFO_ADDRESS0_MASK                                           0xFFFFFFFFL
+//SDP_PORT_MONITOR_INFO1
+#define SDP_PORT_MONITOR_INFO1__SDP_PORT_MONITOR_INFO_ADDRESS1__SHIFT                                         0x0
+#define SDP_PORT_MONITOR_INFO1__SDP_PORT_MONITOR_INFO_ADDRESS1_MASK                                           0xFFFFFFFFL
+//SDP_PORT_MONITOR_INFO2
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_BLOCKLEVEL__SHIFT                                       0x0
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_CHAIN__SHIFT                                            0x2
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_CMD__SHIFT                                              0x3
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_IO__SHIFT                                               0x9
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_PASSPW__SHIFT                                           0xa
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_RSPPASSPW__SHIFT                                        0xb
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_SECLEVEL__SHIFT                                         0xc
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_STREAMID__SHIFT                                         0x10
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_BLOCKLEVEL_MASK                                         0x00000003L
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_CHAIN_MASK                                              0x00000004L
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_CMD_MASK                                                0x000001F8L
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_IO_MASK                                                 0x00000200L
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_PASSPW_MASK                                             0x00000400L
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_RSPPASSPW_MASK                                          0x00000800L
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_SECLEVEL_MASK                                           0x0000F000L
+#define SDP_PORT_MONITOR_INFO2__SDP_PORT_MONITOR_INFO_STREAMID_MASK                                           0xFFFF0000L
+//SDP_PORT_MONITOR_INFO3
+#define SDP_PORT_MONITOR_INFO3__SDP_PORT_MONITOR_INFO_TAG__SHIFT                                              0x0
+#define SDP_PORT_MONITOR_INFO3__SDP_PORT_MONITOR_INFO_UNITID__SHIFT                                           0xc
+#define SDP_PORT_MONITOR_INFO3__SDP_PORT_MONITOR_INFO_VC__SHIFT                                               0x17
+#define SDP_PORT_MONITOR_INFO3__SDP_PORT_MONITOR_INFO_LEN__SHIFT                                              0x1a
+#define SDP_PORT_MONITOR_INFO3__SDP_PORT_MONITOR_INFO_TAG_MASK                                                0x00000FFFL
+#define SDP_PORT_MONITOR_INFO3__SDP_PORT_MONITOR_INFO_UNITID_MASK                                             0x007FF000L
+#define SDP_PORT_MONITOR_INFO3__SDP_PORT_MONITOR_INFO_VC_MASK                                                 0x03800000L
+#define SDP_PORT_MONITOR_INFO3__SDP_PORT_MONITOR_INFO_LEN_MASK                                                0xFC000000L
+//SDP_PORT_MONITOR_INFO4
+#define SDP_PORT_MONITOR_INFO4__SDP_PORT_MONITOR_INFO_ATTR__SHIFT                                             0x0
+#define SDP_PORT_MONITOR_INFO4__SDP_PORT_MONITOR_INFO_ATTR_MASK                                               0x000000FFL
+//BIFC_BME_ERR_LOG_HB
+//BIFC_GFX_INT_MONITOR_MASK
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_CTRL_MASK__SHIFT                                       0x0
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_ADDR_MASK__SHIFT                                       0x1
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_UPADDR_MASK__SHIFT                                     0x2
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_DATA_MASK__SHIFT                                       0x3
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_EDATA_MASK__SHIFT                                      0x4
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSIX_ADDR_MASK__SHIFT                                      0x5
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSIX_UPADDR_MASK__SHIFT                                    0x6
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSIX_DATA_MASK__SHIFT                                      0x7
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_VEC_MASK__SHIFT                                            0x8
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_PEND_MASK__SHIFT                                           0x9
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSIX_CTRL_MASK__SHIFT                                      0xa
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_CTRL_MASK_MASK                                         0x00000001L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_ADDR_MASK_MASK                                         0x00000002L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_UPADDR_MASK_MASK                                       0x00000004L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_DATA_MASK_MASK                                         0x00000008L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSI_EDATA_MASK_MASK                                        0x00000010L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSIX_ADDR_MASK_MASK                                        0x00000020L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSIX_UPADDR_MASK_MASK                                      0x00000040L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSIX_DATA_MASK_MASK                                        0x00000080L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_VEC_MASK_MASK                                              0x00000100L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_PEND_MASK_MASK                                             0x00000200L
+#define BIFC_GFX_INT_MONITOR_MASK__GFX_INT_MONITOR_MSIX_CTRL_MASK_MASK                                        0x00000400L
+//BIFC_GFX_INT_MONITOR_STS
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_CTRL_STS__SHIFT                                         0x0
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_ADDR_STS__SHIFT                                         0x1
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_UPADDR_STS__SHIFT                                       0x2
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_DATA_STS__SHIFT                                         0x3
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_EDATA_STS__SHIFT                                        0x4
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSIX_ADDR_STS__SHIFT                                        0x5
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSIX_UPADDR_STS__SHIFT                                      0x6
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSIX_DATA_STS__SHIFT                                        0x7
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_VEC_STS__SHIFT                                              0x8
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_PEND_STS__SHIFT                                             0x9
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSIX_CTRL_STS__SHIFT                                        0xa
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_CTRL_STS_MASK                                           0x00000001L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_ADDR_STS_MASK                                           0x00000002L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_UPADDR_STS_MASK                                         0x00000004L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_DATA_STS_MASK                                           0x00000008L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSI_EDATA_STS_MASK                                          0x00000010L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSIX_ADDR_STS_MASK                                          0x00000020L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSIX_UPADDR_STS_MASK                                        0x00000040L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSIX_DATA_STS_MASK                                          0x00000080L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_VEC_STS_MASK                                                0x00000100L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_PEND_STS_MASK                                               0x00000200L
+#define BIFC_GFX_INT_MONITOR_STS__GFX_INT_MONITOR_MSIX_CTRL_STS_MASK                                          0x00000400L
+//BIFC_GFX_INT_MONITOR_CTRL
+#define BIFC_GFX_INT_MONITOR_CTRL__GFX_INT_MONITOR_VEC_SEL__SHIFT                                             0x0
+#define BIFC_GFX_INT_MONITOR_CTRL__GFX_INT_MONITOR_VEC_SEL_MASK                                               0x0000000FL
+//NBIF_FLUSH_CTRL
+#define NBIF_FLUSH_CTRL__GMI_FLUSH_OT_ENABLE__SHIFT                                                           0x0
+#define NBIF_FLUSH_CTRL__GSI_FLUSH_OT_ENABLE__SHIFT                                                           0x1
+#define NBIF_FLUSH_CTRL__FLUSH_ADD_BUBBLE_EN__SHIFT                                                           0x3
+#define NBIF_FLUSH_CTRL__FLUSH_ADD_BUBBLE_SEL__SHIFT                                                          0x4
+#define NBIF_FLUSH_CTRL__NBIF_GRP1_RW_OT_SWITCH__SHIFT                                                        0x6
+#define NBIF_FLUSH_CTRL__GMI_FLUSH_OT_ENABLE_MASK                                                             0x00000001L
+#define NBIF_FLUSH_CTRL__GSI_FLUSH_OT_ENABLE_MASK                                                             0x00000002L
+#define NBIF_FLUSH_CTRL__FLUSH_ADD_BUBBLE_EN_MASK                                                             0x00000008L
+#define NBIF_FLUSH_CTRL__FLUSH_ADD_BUBBLE_SEL_MASK                                                            0x00000030L
+#define NBIF_FLUSH_CTRL__NBIF_GRP1_RW_OT_SWITCH_MASK                                                          0x00000040L
+//NBIF_GUI_FLUSH_CNTL
+#define NBIF_GUI_FLUSH_CNTL__NBIF_GUI_FLUSH_REQ_MERGE__SHIFT                                                  0x0
+#define NBIF_GUI_FLUSH_CNTL__NBIF_GUI_FLUSH_REQ_MERGE_MASK                                                    0x00000001L
+//BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_HRP_SDP_WRRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+//BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_HRP_SDP_RDRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+//BIFC_GMI_SDP_REQ_POOLCRED_ALLOC
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                     0x0
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                     0x4
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                     0x8
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                     0xc
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                     0x10
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                     0x14
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                     0x18
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                     0x1c
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                       0x0000000FL
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                       0x000000F0L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                       0x00000F00L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                       0x0000F000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                       0x000F0000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                       0x00F00000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                       0x0F000000L
+#define BIFC_GMI_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                       0xF0000000L
+//BIFC_GMI_SDP_DAT_POOLCRED_ALLOC
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                     0x0
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                     0x4
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                     0x8
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                     0xc
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                     0x10
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                     0x14
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                     0x18
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                     0x1c
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                       0x0000000FL
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                       0x000000F0L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                       0x00000F00L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                       0x0000F000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                       0x000F0000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                       0x00F00000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                       0x0F000000L
+#define BIFC_GMI_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                       0xF0000000L
+//BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                   0x8
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                   0xc
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                   0x10
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                   0x14
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                   0x18
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                   0x1c
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                     0x00000F00L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                     0x0000F000L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                     0x000F0000L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                     0x00F00000L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                     0x0F000000L
+#define BIFC_GMI_SST_RDRSP_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                     0xF0000000L
+//BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                   0x0
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                   0x4
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                   0x8
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                   0xc
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                   0x10
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                   0x14
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                   0x18
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                   0x1c
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                     0x0000000FL
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                     0x000000F0L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                     0x00000F00L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                     0x0000F000L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                     0x000F0000L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                     0x00F00000L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                     0x0F000000L
+#define BIFC_GMI_SST_WRRSP_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                     0xF0000000L
+//DISCON_HYSTERESIS_HEAD_CTRL
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_DNS_SDP_DISCON_HYSTERESIS_H__SHIFT                                   0x0
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_UPS_SDP_DISCON_HYSTERESIS_H__SHIFT                                   0x8
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_DNS_SDP_DISCON_HYSTERESIS_H_MASK                                     0x0000000FL
+#define DISCON_HYSTERESIS_HEAD_CTRL__GMI_UPS_SDP_DISCON_HYSTERESIS_H_MASK                                     0x00000F00L
+//BIFC_GSI_SDP_REQ_POOLCRED_ALLOC
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                     0x0
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                     0x4
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                     0x8
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                     0xc
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                     0x10
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                     0x14
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                     0x18
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                     0x1c
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                       0x0000000FL
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                       0x000000F0L
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                       0x00000F00L
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                       0x0000F000L
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                       0x000F0000L
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                       0x00F00000L
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                       0x0F000000L
+#define BIFC_GSI_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                       0xF0000000L
+//BIFC_GSI_SDP_DAT_POOLCRED_ALLOC
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                     0x0
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                     0x4
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                     0x8
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                     0xc
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                     0x10
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                     0x14
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                     0x18
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                     0x1c
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                       0x0000000FL
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                       0x000000F0L
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                       0x00000F00L
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                       0x0000F000L
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                       0x000F0000L
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                       0x00F00000L
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                       0x0F000000L
+#define BIFC_GSI_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                       0xF0000000L
+//BIFC_MCA_SMN_CTRL0
+#define BIFC_MCA_SMN_CTRL0__MCA_SMN_POSTED__SHIFT                                                             0x0
+#define BIFC_MCA_SMN_CTRL0__MCA_SMN_POSTED_MASK                                                               0x00000001L
+//BIFC_MCTP_SMN_CTRL0
+#define BIFC_MCTP_SMN_CTRL0__MCTP_SMN_POSTED__SHIFT                                                           0x0
+#define BIFC_MCTP_SMN_CTRL0__MCTP_SMN_POSTED_MASK                                                             0x00000001L
+//NBIF_REGIF_DEBUG_COUNTER_CTRL
+//BIFC_EARLY_WAKEUP_CNTL
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_ACTIVE__SHIFT                                     0x0
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_DS_EXIT__SHIFT                                    0x1
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_ALLOW_AER_ACTIVE__SHIFT                                     0x2
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_ACTIVE_MASK                                       0x00000001L
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_BY_CLIENT_DS_EXIT_MASK                                      0x00000002L
+#define BIFC_EARLY_WAKEUP_CNTL__NBIF_EARLY_WAKEUP_ALLOW_AER_ACTIVE_MASK                                       0x00000004L
+//BIFC_PERF_CNT_MMIO_RD_H16BIT
+#define BIFC_PERF_CNT_MMIO_RD_H16BIT__PERF_CNT_MMIO_RD_VALUE_H16BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_RD_H16BIT__PERF_CNT_MMIO_RD_VALUE_H16BIT_MASK                                      0x0000FFFFL
+//BIFC_PERF_CNT_MMIO_WR_H16BIT
+#define BIFC_PERF_CNT_MMIO_WR_H16BIT__PERF_CNT_MMIO_WR_VALUE_H16BIT__SHIFT                                    0x0
+#define BIFC_PERF_CNT_MMIO_WR_H16BIT__PERF_CNT_MMIO_WR_VALUE_H16BIT_MASK                                      0x0000FFFFL
+//BIF_MISC_BIFC_PERF_CNT_DMA_RD_H16BIT
+#define BIF_MISC_BIFC_PERF_CNT_DMA_RD_H16BIT__PERF_CNT_DMA_RD_VALUE_H16BIT__SHIFT                             0x0
+#define BIF_MISC_BIFC_PERF_CNT_DMA_RD_H16BIT__PERF_CNT_DMA_RD_VALUE_H16BIT_MASK                               0x0000FFFFL
+//BIF_MISC_BIFC_PERF_CNT_DMA_WR_H16BIT
+#define BIF_MISC_BIFC_PERF_CNT_DMA_WR_H16BIT__PERF_CNT_DMA_WR_VALUE_H16BIT__SHIFT                             0x0
+#define BIF_MISC_BIFC_PERF_CNT_DMA_WR_H16BIT__PERF_CNT_DMA_WR_VALUE_H16BIT_MASK                               0x0000FFFFL
+//BIF_MISC_NBIF_PERF_COM_COUNT_ENABLE
+#define BIF_MISC_NBIF_PERF_COM_COUNT_ENABLE__NBIF_COM_COUNT_ENABLE__SHIFT                                     0x0
+#define BIF_MISC_NBIF_PERF_COM_COUNT_ENABLE__START_COUNT_NOPULS__SHIFT                                        0x3
+#define BIF_MISC_NBIF_PERF_COM_COUNT_ENABLE__LEGACY_OUT_REALTIME_SEL__SHIFT                                   0x4
+#define BIF_MISC_NBIF_PERF_COM_COUNT_ENABLE__PERF_CNT_TRIG_MONI_SEL__SHIFT                                    0x5
+#define BIF_MISC_NBIF_PERF_COM_COUNT_ENABLE__NBIF_COM_COUNT_ENABLE_MASK                                       0x00000001L
+#define BIF_MISC_NBIF_PERF_COM_COUNT_ENABLE__START_COUNT_NOPULS_MASK                                          0x00000008L
+#define BIF_MISC_NBIF_PERF_COM_COUNT_ENABLE__LEGACY_OUT_REALTIME_SEL_MASK                                     0x00000010L
+#define BIF_MISC_NBIF_PERF_COM_COUNT_ENABLE__PERF_CNT_TRIG_MONI_SEL_MASK                                      0x00000060L
+//BIF_MISC_NBIF_SDP_PERF_CNT_FSM
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_GLOBAL_SHADOW_TGL_DELAY_COUNT__SHIFT                              0x0
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_GLOBAL_PERF_RESET_TGL_DELAY_COUNT__SHIFT                          0x4
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_GLOBAL_PERF_RESET_TGL_DELAY_EN__SHIFT                             0x8
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PRE_FLD_GLOBAL_SHADOW_WR__SHIFT                                   0x9
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT_DONE__SHIFT                                              0xa
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT0_RESET__SHIFT                                            0xb
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT1_RESET__SHIFT                                            0xc
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT2_RESET__SHIFT                                            0xd
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT3_RESET__SHIFT                                            0xe
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_REQ_UNIT_ID__SHIFT                                           0xf
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_GLOBAL_SHADOW_TGL_DELAY_COUNT_MASK                                0x0000000FL
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_GLOBAL_PERF_RESET_TGL_DELAY_COUNT_MASK                            0x000000F0L
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_GLOBAL_PERF_RESET_TGL_DELAY_EN_MASK                               0x00000100L
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PRE_FLD_GLOBAL_SHADOW_WR_MASK                                     0x00000200L
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT_DONE_MASK                                                0x00000400L
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT0_RESET_MASK                                              0x00000800L
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT1_RESET_MASK                                              0x00001000L
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT2_RESET_MASK                                              0x00002000L
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_CNT3_RESET_MASK                                              0x00004000L
+#define BIF_MISC_NBIF_SDP_PERF_CNT_FSM__SDP_PERF_REQ_UNIT_ID_MASK                                             0x03FF8000L
+//BIF_MISC_NBIF_SDP_PERF_COUNTER
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_0_EN__SHIFT                                              0x0
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_1_EN__SHIFT                                              0x1
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_2_EN__SHIFT                                              0x2
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_3_EN__SHIFT                                              0x3
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_0_SEL__SHIFT                                             0x4
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_1_SEL__SHIFT                                             0xb
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_2_SEL__SHIFT                                             0x12
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_3_SEL__SHIFT                                             0x19
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_0_EN_MASK                                                0x00000001L
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_1_EN_MASK                                                0x00000002L
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_2_EN_MASK                                                0x00000004L
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_3_EN_MASK                                                0x00000008L
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_0_SEL_MASK                                               0x000007F0L
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_1_SEL_MASK                                               0x0003F800L
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_2_SEL_MASK                                               0x01FC0000L
+#define BIF_MISC_NBIF_SDP_PERF_COUNTER__SDP_PERF_CNT_3_SEL_MASK                                               0xFE000000L
+//BIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_L32BIT
+#define BIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_L32BIT__SDP_PERF_CNT_0_VALUE_L32BIT__SHIFT                         0x0
+#define BIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_L32BIT__SDP_PERF_CNT_0_VALUE_L32BIT_MASK                           0xFFFFFFFFL
+//BIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_L32BIT
+#define BIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_L32BIT__SDP_PERF_CNT_1_VALUE_L32BIT__SHIFT                         0x0
+#define BIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_L32BIT__SDP_PERF_CNT_1_VALUE_L32BIT_MASK                           0xFFFFFFFFL
+//BIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_L32BIT
+#define BIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_L32BIT__SDP_PERF_CNT_2_VALUE_L32BIT__SHIFT                         0x0
+#define BIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_L32BIT__SDP_PERF_CNT_2_VALUE_L32BIT_MASK                           0xFFFFFFFFL
+//BIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_L32BIT
+#define BIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_L32BIT__SDP_PERF_CNT_3_VALUE_L32BIT__SHIFT                         0x0
+#define BIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_L32BIT__SDP_PERF_CNT_3_VALUE_L32BIT_MASK                           0xFFFFFFFFL
+//BIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_H16BIT
+#define BIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_H16BIT__SDP_PERF_CNT_0_VALUE_H16BIT__SHIFT                         0x0
+#define BIF_MISC_NBIF_SDP_PERF_CNT_0_VALUE_H16BIT__SDP_PERF_CNT_0_VALUE_H16BIT_MASK                           0x0000FFFFL
+//BIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_H16BIT
+#define BIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_H16BIT__SDP_PERF_CNT_1_VALUE_H16BIT__SHIFT                         0x0
+#define BIF_MISC_NBIF_SDP_PERF_CNT_1_VALUE_H16BIT__SDP_PERF_CNT_1_VALUE_H16BIT_MASK                           0x0000FFFFL
+//BIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_H16BIT
+#define BIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_H16BIT__SDP_PERF_CNT_2_VALUE_H16BIT__SHIFT                         0x0
+#define BIF_MISC_NBIF_SDP_PERF_CNT_2_VALUE_H16BIT__SDP_PERF_CNT_2_VALUE_H16BIT_MASK                           0x0000FFFFL
+//BIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_H16BIT
+#define BIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_H16BIT__SDP_PERF_CNT_3_VALUE_H16BIT__SHIFT                         0x0
+#define BIF_MISC_NBIF_SDP_PERF_CNT_3_VALUE_H16BIT__SDP_PERF_CNT_3_VALUE_H16BIT_MASK                           0x0000FFFFL
+//BIF_MISC_NBIF_BX_PERF_CNT_FSM
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_SHADOW_TGL_DELAY_COUNT__SHIFT                                0x0
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_PERF_RESET_TGL_DELAY_COUNT__SHIFT                            0x4
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_PERF_RESET_TGL_DELAY_EN__SHIFT                               0x8
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_PRE_FLD_GLOBAL_SHADOW_WR__SHIFT                                     0x9
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_PERF_CNT_DONE__SHIFT                                                0xa
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_SHADOW_TGL_DELAY_COUNT_MASK                                  0x0000000FL
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_PERF_RESET_TGL_DELAY_COUNT_MASK                              0x000000F0L
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_GLOBAL_PERF_RESET_TGL_DELAY_EN_MASK                                 0x00000100L
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_PRE_FLD_GLOBAL_SHADOW_WR_MASK                                       0x00000200L
+#define BIF_MISC_NBIF_BX_PERF_CNT_FSM__BX_PERF_CNT_DONE_MASK                                                  0x00000400L
+//NBIF_TDISP_ENTER_ERR_CNTL
+#define NBIF_TDISP_ENTER_ERR_CNTL__NBIF_TDISP_ENTER_ERR_BY_POISONEDTLP_EN__SHIFT                              0x0
+#define NBIF_TDISP_ENTER_ERR_CNTL__NBIF_TDISP_ENTER_ERR_BY_UR_EN__SHIFT                                       0x1
+#define NBIF_TDISP_ENTER_ERR_CNTL__NBIF_TDISP_ENTER_ERR_BY_CA_EN__SHIFT                                       0x2
+#define NBIF_TDISP_ENTER_ERR_CNTL__NBIF_TDISP_ENTER_ERR_BY_TO_EN__SHIFT                                       0x3
+#define NBIF_TDISP_ENTER_ERR_CNTL__NBIF_TDISP_ENTER_ERR_BY_POISONEDTLP_EN_MASK                                0x00000001L
+#define NBIF_TDISP_ENTER_ERR_CNTL__NBIF_TDISP_ENTER_ERR_BY_UR_EN_MASK                                         0x00000002L
+#define NBIF_TDISP_ENTER_ERR_CNTL__NBIF_TDISP_ENTER_ERR_BY_CA_EN_MASK                                         0x00000004L
+#define NBIF_TDISP_ENTER_ERR_CNTL__NBIF_TDISP_ENTER_ERR_BY_TO_EN_MASK                                         0x00000008L
+//BIF_UNITID_ERR_LOG
+#define BIF_UNITID_ERR_LOG__UR_UNITID_athub__SHIFT                                                            0x0
+#define BIF_UNITID_ERR_LOG__CLEAR_UR_UNITID_athub__SHIFT                                                      0x10
+#define BIF_UNITID_ERR_LOG__UR_UNITID_athub_MASK                                                              0x00000001L
+#define BIF_UNITID_ERR_LOG__CLEAR_UR_UNITID_athub_MASK                                                        0x00010000L
+//NBIF_COM_COUNT_VALUE
+#define NBIF_COM_COUNT_VALUE__NBIF_COM_COUNT_VALUE__SHIFT                                                     0x0
+#define NBIF_COM_COUNT_VALUE__NBIF_COM_COUNT_VALUE_MASK                                                       0xFFFFFFFFL
+//NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT0_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x0
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT1_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x1
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT2_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x2
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT3_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x3
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT4_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x4
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT5_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x5
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT6_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x6
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT7_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x7
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT0_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000001L
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT1_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000002L
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT2_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000004L
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT3_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000008L
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT4_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000010L
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT5_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000020L
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT6_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000040L
+#define NBIF_ERREREVNT_SIDEBAND_RAS_V1_CTRL__NBIF_PORT7_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000080L
+//NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT0_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x0
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT1_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x1
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT2_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x2
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT3_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x3
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT4_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x4
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT5_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x5
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT6_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x6
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT7_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x7
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT0_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000001L
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT1_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000002L
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT2_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000004L
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT3_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000008L
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT4_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000010L
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT5_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000020L
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT6_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000040L
+#define NBIF_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__NBIF_PORT7_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000080L
+//BIF_AER_CNTL
+#define BIF_AER_CNTL__BIF_AER_VF_NUM__SHIFT                                                                   0x0
+#define BIF_AER_CNTL__BIF_AER_PF_NUM__SHIFT                                                                   0x8
+#define BIF_AER_CNTL__BIF_AER_ERR_PTR__SHIFT                                                                  0xb
+#define BIF_AER_CNTL__BIF_AER_HeaderStoreOffset__SHIFT                                                        0x10
+#define BIF_AER_CNTL__BIF_AER_ERR_MSG_TYPE__SHIFT                                                             0x18
+#define BIF_AER_CNTL__BIF_AER_DEVERR__SHIFT                                                                   0x1a
+#define BIF_AER_CNTL__BIF_AER_POSTED__SHIFT                                                                   0x1b
+#define BIF_AER_CNTL__BIF_AER_TOTAL_OF_PF__SHIFT                                                              0x1c
+#define BIF_AER_CNTL__BIF_AER_VF_HIT__SHIFT                                                                   0x1f
+#define BIF_AER_CNTL__BIF_AER_VF_NUM_MASK                                                                     0x000000FFL
+#define BIF_AER_CNTL__BIF_AER_PF_NUM_MASK                                                                     0x00000700L
+#define BIF_AER_CNTL__BIF_AER_ERR_PTR_MASK                                                                    0x0000F800L
+#define BIF_AER_CNTL__BIF_AER_HeaderStoreOffset_MASK                                                          0x00FF0000L
+#define BIF_AER_CNTL__BIF_AER_ERR_MSG_TYPE_MASK                                                               0x03000000L
+#define BIF_AER_CNTL__BIF_AER_DEVERR_MASK                                                                     0x04000000L
+#define BIF_AER_CNTL__BIF_AER_POSTED_MASK                                                                     0x08000000L
+#define BIF_AER_CNTL__BIF_AER_TOTAL_OF_PF_MASK                                                                0x70000000L
+#define BIF_AER_CNTL__BIF_AER_VF_HIT_MASK                                                                     0x80000000L
+//NBIF_IDE_INFO_CNTL
+#define NBIF_IDE_INFO_CNTL__NBIF_PRGR_MSG_IDE_INFO_FROM_DATA_EN__SHIFT                                        0x0
+#define NBIF_IDE_INFO_CNTL__NBIF_PRGR_MSG_IDE_INFO_FROM_DATA_EN_MASK                                          0x00000001L
+//BIFC_EXP_MISC_CTRL0
+#define BIFC_EXP_MISC_CTRL0__VC0_DMA_IOCFG_DIS__SHIFT                                                         0x0
+#define BIFC_EXP_MISC_CTRL0__VC1_DMA_IOCFG_DIS__SHIFT                                                         0x1
+#define BIFC_EXP_MISC_CTRL0__VC2_DMA_IOCFG_DIS__SHIFT                                                         0x2
+#define BIFC_EXP_MISC_CTRL0__VC3_DMA_IOCFG_DIS__SHIFT                                                         0x3
+#define BIFC_EXP_MISC_CTRL0__VC4_DMA_IOCFG_DIS__SHIFT                                                         0x4
+#define BIFC_EXP_MISC_CTRL0__VC5_DMA_IOCFG_DIS__SHIFT                                                         0x5
+#define BIFC_EXP_MISC_CTRL0__VC6_DMA_IOCFG_DIS__SHIFT                                                         0x6
+#define BIFC_EXP_MISC_CTRL0__VC7_DMA_IOCFG_DIS__SHIFT                                                         0x7
+#define BIFC_EXP_MISC_CTRL0__VC0_DMA_IOCFG_DIS_MASK                                                           0x00000001L
+#define BIFC_EXP_MISC_CTRL0__VC1_DMA_IOCFG_DIS_MASK                                                           0x00000002L
+#define BIFC_EXP_MISC_CTRL0__VC2_DMA_IOCFG_DIS_MASK                                                           0x00000004L
+#define BIFC_EXP_MISC_CTRL0__VC3_DMA_IOCFG_DIS_MASK                                                           0x00000008L
+#define BIFC_EXP_MISC_CTRL0__VC4_DMA_IOCFG_DIS_MASK                                                           0x00000010L
+#define BIFC_EXP_MISC_CTRL0__VC5_DMA_IOCFG_DIS_MASK                                                           0x00000020L
+#define BIFC_EXP_MISC_CTRL0__VC6_DMA_IOCFG_DIS_MASK                                                           0x00000040L
+#define BIFC_EXP_MISC_CTRL0__VC7_DMA_IOCFG_DIS_MASK                                                           0x00000080L
+//SECOND_HDP_MEM_FLUSH_CNTL_PF
+#define SECOND_HDP_MEM_FLUSH_CNTL_PF__ADDRESS__SHIFT                                                          0x2
+#define SECOND_HDP_MEM_FLUSH_CNTL_PF__ADDRESS_MASK                                                            0x0007FFFCL
+//SECOND_HDP_MEM_FLUSH_CNTL_VF0
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF0__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF0__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_MEM_FLUSH_CNTL_VF1
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF1__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF1__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_MEM_FLUSH_CNTL_VF2
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF2__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF2__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_MEM_FLUSH_CNTL_VF3
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF3__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF3__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_MEM_FLUSH_CNTL_VF4
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF4__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF4__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_MEM_FLUSH_CNTL_VF5
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF5__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF5__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_MEM_FLUSH_CNTL_VF6
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF6__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF6__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_MEM_FLUSH_CNTL_VF7
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF7__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_MEM_FLUSH_CNTL_VF7__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_REG_FLUSH_CNTL_PF
+#define SECOND_HDP_REG_FLUSH_CNTL_PF__ADDRESS__SHIFT                                                          0x2
+#define SECOND_HDP_REG_FLUSH_CNTL_PF__ADDRESS_MASK                                                            0x0007FFFCL
+//SECOND_HDP_REG_FLUSH_CNTL_VF0
+#define SECOND_HDP_REG_FLUSH_CNTL_VF0__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_REG_FLUSH_CNTL_VF0__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_REG_FLUSH_CNTL_VF1
+#define SECOND_HDP_REG_FLUSH_CNTL_VF1__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_REG_FLUSH_CNTL_VF1__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_REG_FLUSH_CNTL_VF2
+#define SECOND_HDP_REG_FLUSH_CNTL_VF2__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_REG_FLUSH_CNTL_VF2__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_REG_FLUSH_CNTL_VF3
+#define SECOND_HDP_REG_FLUSH_CNTL_VF3__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_REG_FLUSH_CNTL_VF3__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_REG_FLUSH_CNTL_VF4
+#define SECOND_HDP_REG_FLUSH_CNTL_VF4__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_REG_FLUSH_CNTL_VF4__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_REG_FLUSH_CNTL_VF5
+#define SECOND_HDP_REG_FLUSH_CNTL_VF5__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_REG_FLUSH_CNTL_VF5__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_REG_FLUSH_CNTL_VF6
+#define SECOND_HDP_REG_FLUSH_CNTL_VF6__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_REG_FLUSH_CNTL_VF6__ADDRESS_MASK                                                           0x0007FFFCL
+//SECOND_HDP_REG_FLUSH_CNTL_VF7
+#define SECOND_HDP_REG_FLUSH_CNTL_VF7__ADDRESS__SHIFT                                                         0x2
+#define SECOND_HDP_REG_FLUSH_CNTL_VF7__ADDRESS_MASK                                                           0x0007FFFCL
+//GPU_HDP_FLUSH_GFX_FW
+#define GPU_HDP_FLUSH_GFX_FW__CLEAR_ALL_VF_FLUSH_DONE__SHIFT                                                  0x0
+#define GPU_HDP_FLUSH_GFX_FW__CLEAR_ALL_VF_FLUSH_DONE_MASK                                                    0x00000001L
+//NBIF_RRMT_CNTL
+#define NBIF_RRMT_CNTL__PARTITION_MODE__SHIFT                                                                 0x0
+#define NBIF_RRMT_CNTL__AID_DIE_ID__SHIFT                                                                     0x4
+#define NBIF_RRMT_CNTL__RRMT_ENABLE__SHIFT                                                                    0x8
+#define NBIF_RRMT_CNTL__RRMT_Invalid_Address_H__SHIFT                                                         0x10
+#define NBIF_RRMT_CNTL__PARTITION_MODE_MASK                                                                   0x00000007L
+#define NBIF_RRMT_CNTL__AID_DIE_ID_MASK                                                                       0x00000030L
+#define NBIF_RRMT_CNTL__RRMT_ENABLE_MASK                                                                      0x00000100L
+#define NBIF_RRMT_CNTL__RRMT_Invalid_Address_H_MASK                                                           0xFFFF0000L
+//RRMT_NBIF_MISC_REG1
+#define RRMT_NBIF_MISC_REG1__RRMT_Invalid_Address_L__SHIFT                                                    0x0
+#define RRMT_NBIF_MISC_REG1__RRMT_Invalid_Address_L_MASK                                                      0xFFFFFFFFL
+//RRMT_LUT_INDEX
+#define RRMT_LUT_INDEX__RRMT_LUT_INDEX__SHIFT                                                                 0x0
+#define RRMT_LUT_INDEX__RRMT_LUT_INDEX_MASK                                                                   0x000003FFL
+//RRMT_LUT_DATA
+#define RRMT_LUT_DATA__RRMT_LUT_DATA__SHIFT                                                                   0x0
+#define RRMT_LUT_DATA__RRMT_LUT_DATA_MASK                                                                     0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_misc_pfvf_dev0_epf0_bif_misc_pfvf_regblk
+//BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF0__SHIFT                               0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF1__SHIFT                               0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF2__SHIFT                               0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF3__SHIFT                               0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF4__SHIFT                               0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF5__SHIFT                               0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF6__SHIFT                               0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF7__SHIFT                               0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF8__SHIFT                               0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF9__SHIFT                               0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF10__SHIFT                              0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF11__SHIFT                              0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF12__SHIFT                              0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF13__SHIFT                              0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF14__SHIFT                              0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF15__SHIFT                              0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF16__SHIFT                              0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF17__SHIFT                              0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF18__SHIFT                              0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF19__SHIFT                              0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF20__SHIFT                              0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF21__SHIFT                              0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF22__SHIFT                              0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF23__SHIFT                              0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF24__SHIFT                              0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF25__SHIFT                              0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF26__SHIFT                              0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF27__SHIFT                              0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF28__SHIFT                              0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF29__SHIFT                              0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF30__SHIFT                              0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF31__SHIFT                              0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF0_MASK                                 0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF1_MASK                                 0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF2_MASK                                 0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF3_MASK                                 0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF4_MASK                                 0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF5_MASK                                 0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF6_MASK                                 0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF7_MASK                                 0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF8_MASK                                 0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF9_MASK                                 0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF10_MASK                                0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF11_MASK                                0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF12_MASK                                0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF13_MASK                                0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF14_MASK                                0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF15_MASK                                0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF16_MASK                                0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF17_MASK                                0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF18_MASK                                0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF19_MASK                                0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF20_MASK                                0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF21_MASK                                0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF22_MASK                                0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF23_MASK                                0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF24_MASK                                0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF25_MASK                                0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF26_MASK                                0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF27_MASK                                0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF28_MASK                                0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF29_MASK                                0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF30_MASK                                0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF31_MASK                                0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF32__SHIFT                              0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF33__SHIFT                              0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF34__SHIFT                              0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF35__SHIFT                              0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF36__SHIFT                              0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF37__SHIFT                              0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF38__SHIFT                              0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF39__SHIFT                              0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF40__SHIFT                              0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF41__SHIFT                              0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF42__SHIFT                              0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF43__SHIFT                              0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF44__SHIFT                              0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF45__SHIFT                              0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF46__SHIFT                              0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF47__SHIFT                              0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF48__SHIFT                              0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF49__SHIFT                              0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF50__SHIFT                              0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF51__SHIFT                              0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF52__SHIFT                              0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF53__SHIFT                              0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF54__SHIFT                              0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF55__SHIFT                              0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF56__SHIFT                              0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF57__SHIFT                              0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF58__SHIFT                              0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF59__SHIFT                              0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF60__SHIFT                              0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF61__SHIFT                              0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF62__SHIFT                              0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF63__SHIFT                              0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF32_MASK                                0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF33_MASK                                0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF34_MASK                                0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF35_MASK                                0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF36_MASK                                0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF37_MASK                                0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF38_MASK                                0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF39_MASK                                0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF40_MASK                                0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF41_MASK                                0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF42_MASK                                0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF43_MASK                                0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF44_MASK                                0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF45_MASK                                0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF46_MASK                                0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF47_MASK                                0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF48_MASK                                0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF49_MASK                                0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF50_MASK                                0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF51_MASK                                0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF52_MASK                                0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF53_MASK                                0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF54_MASK                                0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF55_MASK                                0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF56_MASK                                0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF57_MASK                                0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF58_MASK                                0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF59_MASK                                0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF60_MASK                                0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF61_MASK                                0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF62_MASK                                0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF63_MASK                                0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF64__SHIFT                              0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF65__SHIFT                              0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF66__SHIFT                              0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF67__SHIFT                              0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF68__SHIFT                              0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF69__SHIFT                              0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF70__SHIFT                              0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF71__SHIFT                              0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF72__SHIFT                              0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF73__SHIFT                              0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF74__SHIFT                              0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF75__SHIFT                              0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF76__SHIFT                              0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF77__SHIFT                              0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF78__SHIFT                              0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF79__SHIFT                              0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF80__SHIFT                              0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF81__SHIFT                              0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF82__SHIFT                              0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF83__SHIFT                              0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF84__SHIFT                              0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF85__SHIFT                              0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF86__SHIFT                              0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF87__SHIFT                              0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF88__SHIFT                              0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF89__SHIFT                              0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF90__SHIFT                              0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF91__SHIFT                              0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF92__SHIFT                              0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF93__SHIFT                              0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF94__SHIFT                              0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF95__SHIFT                              0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF64_MASK                                0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF65_MASK                                0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF66_MASK                                0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF67_MASK                                0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF68_MASK                                0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF69_MASK                                0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF70_MASK                                0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF71_MASK                                0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF72_MASK                                0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF73_MASK                                0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF74_MASK                                0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF75_MASK                                0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF76_MASK                                0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF77_MASK                                0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF78_MASK                                0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF79_MASK                                0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF80_MASK                                0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF81_MASK                                0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF82_MASK                                0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF83_MASK                                0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF84_MASK                                0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF85_MASK                                0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF86_MASK                                0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF87_MASK                                0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF88_MASK                                0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF89_MASK                                0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF90_MASK                                0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF91_MASK                                0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF92_MASK                                0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF93_MASK                                0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF94_MASK                                0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF95_MASK                                0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF96__SHIFT                              0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF97__SHIFT                              0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF98__SHIFT                              0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF99__SHIFT                              0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF100__SHIFT                             0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF101__SHIFT                             0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF102__SHIFT                             0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF103__SHIFT                             0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF104__SHIFT                             0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF105__SHIFT                             0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF106__SHIFT                             0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF107__SHIFT                             0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF108__SHIFT                             0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF109__SHIFT                             0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF110__SHIFT                             0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF111__SHIFT                             0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF112__SHIFT                             0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF113__SHIFT                             0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF114__SHIFT                             0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF115__SHIFT                             0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF116__SHIFT                             0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF117__SHIFT                             0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF118__SHIFT                             0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF119__SHIFT                             0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF120__SHIFT                             0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF121__SHIFT                             0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF122__SHIFT                             0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF123__SHIFT                             0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF124__SHIFT                             0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF125__SHIFT                             0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF126__SHIFT                             0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF127__SHIFT                             0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF96_MASK                                0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF97_MASK                                0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF98_MASK                                0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF99_MASK                                0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF100_MASK                               0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF101_MASK                               0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF102_MASK                               0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF103_MASK                               0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF104_MASK                               0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF105_MASK                               0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF106_MASK                               0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF107_MASK                               0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF108_MASK                               0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF109_MASK                               0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF110_MASK                               0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF111_MASK                               0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF112_MASK                               0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF113_MASK                               0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF114_MASK                               0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF115_MASK                               0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF116_MASK                               0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF117_MASK                               0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF118_MASK                               0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF119_MASK                               0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF120_MASK                               0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF121_MASK                               0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF122_MASK                               0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF123_MASK                               0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF124_MASK                               0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF125_MASK                               0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF126_MASK                               0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF127_MASK                               0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF0__SHIFT                         0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF1__SHIFT                         0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF2__SHIFT                         0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF3__SHIFT                         0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF4__SHIFT                         0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF5__SHIFT                         0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF6__SHIFT                         0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF7__SHIFT                         0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF8__SHIFT                         0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF9__SHIFT                         0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF10__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF11__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF12__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF13__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF14__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF15__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF16__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF17__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF18__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF19__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF20__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF21__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF22__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF23__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF24__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF25__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF26__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF27__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF28__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF29__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF30__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF31__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF0_MASK                           0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF1_MASK                           0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF2_MASK                           0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF3_MASK                           0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF4_MASK                           0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF5_MASK                           0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF6_MASK                           0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF7_MASK                           0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF8_MASK                           0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF9_MASK                           0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF10_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF11_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF12_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF13_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF14_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF15_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF16_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF17_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF18_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF19_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF20_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF21_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF22_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF23_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF24_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF25_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF26_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF27_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF28_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF29_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF30_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF31_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF32__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF33__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF34__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF35__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF36__SHIFT                        0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF37__SHIFT                        0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF38__SHIFT                        0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF39__SHIFT                        0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF40__SHIFT                        0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF41__SHIFT                        0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF42__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF43__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF44__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF45__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF46__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF47__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF48__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF49__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF50__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF51__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF52__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF53__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF54__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF55__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF56__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF57__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF58__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF59__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF60__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF61__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF62__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF63__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF32_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF33_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF34_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF35_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF36_MASK                          0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF37_MASK                          0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF38_MASK                          0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF39_MASK                          0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF40_MASK                          0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF41_MASK                          0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF42_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF43_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF44_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF45_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF46_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF47_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF48_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF49_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF50_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF51_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF52_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF53_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF54_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF55_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF56_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF57_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF58_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF59_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF60_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF61_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF62_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF63_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF64__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF65__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF66__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF67__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF68__SHIFT                        0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF69__SHIFT                        0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF70__SHIFT                        0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF71__SHIFT                        0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF72__SHIFT                        0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF73__SHIFT                        0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF74__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF75__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF76__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF77__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF78__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF79__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF80__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF81__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF82__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF83__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF84__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF85__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF86__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF87__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF88__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF89__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF90__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF91__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF92__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF93__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF94__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF95__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF64_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF65_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF66_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF67_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF68_MASK                          0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF69_MASK                          0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF70_MASK                          0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF71_MASK                          0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF72_MASK                          0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF73_MASK                          0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF74_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF75_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF76_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF77_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF78_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF79_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF80_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF81_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF82_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF83_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF84_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF85_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF86_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF87_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF88_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF89_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF90_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF91_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF92_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF93_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF94_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF95_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF96__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF97__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF98__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF99__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF100__SHIFT                       0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF101__SHIFT                       0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF102__SHIFT                       0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF103__SHIFT                       0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF104__SHIFT                       0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF105__SHIFT                       0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF106__SHIFT                       0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF107__SHIFT                       0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF108__SHIFT                       0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF109__SHIFT                       0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF110__SHIFT                       0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF111__SHIFT                       0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF112__SHIFT                       0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF113__SHIFT                       0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF114__SHIFT                       0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF115__SHIFT                       0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF116__SHIFT                       0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF117__SHIFT                       0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF118__SHIFT                       0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF119__SHIFT                       0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF120__SHIFT                       0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF121__SHIFT                       0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF122__SHIFT                       0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF123__SHIFT                       0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF124__SHIFT                       0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF125__SHIFT                       0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF126__SHIFT                       0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF127__SHIFT                       0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF96_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF97_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF98_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF99_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF100_MASK                         0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF101_MASK                         0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF102_MASK                         0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF103_MASK                         0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF104_MASK                         0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF105_MASK                         0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF106_MASK                         0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF107_MASK                         0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF108_MASK                         0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF109_MASK                         0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF110_MASK                         0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF111_MASK                         0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF112_MASK                         0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF113_MASK                         0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF114_MASK                         0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF115_MASK                         0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF116_MASK                         0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF117_MASK                         0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF118_MASK                         0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF119_MASK                         0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF120_MASK                         0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF121_MASK                         0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF122_MASK                         0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF123_MASK                         0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF124_MASK                         0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF125_MASK                         0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF126_MASK                         0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF127_MASK                         0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF0__SHIFT                   0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF1__SHIFT                   0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF2__SHIFT                   0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF3__SHIFT                   0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF4__SHIFT                   0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF5__SHIFT                   0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF6__SHIFT                   0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF7__SHIFT                   0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF8__SHIFT                   0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF9__SHIFT                   0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF10__SHIFT                  0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF11__SHIFT                  0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF12__SHIFT                  0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF13__SHIFT                  0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF14__SHIFT                  0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF15__SHIFT                  0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF16__SHIFT                  0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF17__SHIFT                  0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF18__SHIFT                  0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF19__SHIFT                  0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF20__SHIFT                  0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF21__SHIFT                  0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF22__SHIFT                  0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF23__SHIFT                  0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF24__SHIFT                  0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF25__SHIFT                  0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF26__SHIFT                  0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF27__SHIFT                  0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF28__SHIFT                  0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF29__SHIFT                  0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF30__SHIFT                  0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF31__SHIFT                  0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF0_MASK                     0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF1_MASK                     0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF2_MASK                     0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF3_MASK                     0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF4_MASK                     0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF5_MASK                     0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF6_MASK                     0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF7_MASK                     0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF8_MASK                     0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF9_MASK                     0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF10_MASK                    0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF11_MASK                    0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF12_MASK                    0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF13_MASK                    0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF14_MASK                    0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF15_MASK                    0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF16_MASK                    0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF17_MASK                    0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF18_MASK                    0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF19_MASK                    0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF20_MASK                    0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF21_MASK                    0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF22_MASK                    0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF23_MASK                    0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF24_MASK                    0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF25_MASK                    0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF26_MASK                    0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF27_MASK                    0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF28_MASK                    0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF29_MASK                    0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF30_MASK                    0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF31_MASK                    0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF32__SHIFT                  0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF33__SHIFT                  0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF34__SHIFT                  0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF35__SHIFT                  0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF36__SHIFT                  0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF37__SHIFT                  0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF38__SHIFT                  0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF39__SHIFT                  0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF40__SHIFT                  0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF41__SHIFT                  0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF42__SHIFT                  0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF43__SHIFT                  0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF44__SHIFT                  0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF45__SHIFT                  0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF46__SHIFT                  0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF47__SHIFT                  0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF48__SHIFT                  0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF49__SHIFT                  0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF50__SHIFT                  0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF51__SHIFT                  0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF52__SHIFT                  0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF53__SHIFT                  0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF54__SHIFT                  0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF55__SHIFT                  0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF56__SHIFT                  0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF57__SHIFT                  0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF58__SHIFT                  0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF59__SHIFT                  0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF60__SHIFT                  0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF61__SHIFT                  0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF62__SHIFT                  0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF63__SHIFT                  0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF32_MASK                    0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF33_MASK                    0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF34_MASK                    0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF35_MASK                    0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF36_MASK                    0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF37_MASK                    0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF38_MASK                    0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF39_MASK                    0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF40_MASK                    0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF41_MASK                    0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF42_MASK                    0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF43_MASK                    0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF44_MASK                    0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF45_MASK                    0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF46_MASK                    0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF47_MASK                    0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF48_MASK                    0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF49_MASK                    0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF50_MASK                    0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF51_MASK                    0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF52_MASK                    0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF53_MASK                    0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF54_MASK                    0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF55_MASK                    0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF56_MASK                    0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF57_MASK                    0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF58_MASK                    0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF59_MASK                    0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF60_MASK                    0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF61_MASK                    0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF62_MASK                    0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF63_MASK                    0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF64__SHIFT                  0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF65__SHIFT                  0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF66__SHIFT                  0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF67__SHIFT                  0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF68__SHIFT                  0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF69__SHIFT                  0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF70__SHIFT                  0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF71__SHIFT                  0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF72__SHIFT                  0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF73__SHIFT                  0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF74__SHIFT                  0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF75__SHIFT                  0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF76__SHIFT                  0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF77__SHIFT                  0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF78__SHIFT                  0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF79__SHIFT                  0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF80__SHIFT                  0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF81__SHIFT                  0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF82__SHIFT                  0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF83__SHIFT                  0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF84__SHIFT                  0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF85__SHIFT                  0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF86__SHIFT                  0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF87__SHIFT                  0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF88__SHIFT                  0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF89__SHIFT                  0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF90__SHIFT                  0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF91__SHIFT                  0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF92__SHIFT                  0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF93__SHIFT                  0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF94__SHIFT                  0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF95__SHIFT                  0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF64_MASK                    0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF65_MASK                    0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF66_MASK                    0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF67_MASK                    0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF68_MASK                    0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF69_MASK                    0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF70_MASK                    0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF71_MASK                    0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF72_MASK                    0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF73_MASK                    0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF74_MASK                    0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF75_MASK                    0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF76_MASK                    0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF77_MASK                    0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF78_MASK                    0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF79_MASK                    0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF80_MASK                    0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF81_MASK                    0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF82_MASK                    0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF83_MASK                    0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF84_MASK                    0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF85_MASK                    0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF86_MASK                    0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF87_MASK                    0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF88_MASK                    0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF89_MASK                    0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF90_MASK                    0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF91_MASK                    0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF92_MASK                    0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF93_MASK                    0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF94_MASK                    0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF95_MASK                    0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF96__SHIFT                  0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF97__SHIFT                  0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF98__SHIFT                  0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF99__SHIFT                  0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF100__SHIFT                 0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF101__SHIFT                 0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF102__SHIFT                 0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF103__SHIFT                 0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF104__SHIFT                 0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF105__SHIFT                 0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF106__SHIFT                 0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF107__SHIFT                 0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF108__SHIFT                 0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF109__SHIFT                 0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF110__SHIFT                 0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF111__SHIFT                 0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF112__SHIFT                 0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF113__SHIFT                 0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF114__SHIFT                 0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF115__SHIFT                 0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF116__SHIFT                 0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF117__SHIFT                 0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF118__SHIFT                 0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF119__SHIFT                 0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF120__SHIFT                 0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF121__SHIFT                 0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF122__SHIFT                 0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF123__SHIFT                 0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF124__SHIFT                 0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF125__SHIFT                 0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF126__SHIFT                 0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF127__SHIFT                 0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF96_MASK                    0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF97_MASK                    0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF98_MASK                    0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF99_MASK                    0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF100_MASK                   0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF101_MASK                   0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF102_MASK                   0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF103_MASK                   0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF104_MASK                   0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF105_MASK                   0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF106_MASK                   0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF107_MASK                   0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF108_MASK                   0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF109_MASK                   0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF110_MASK                   0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF111_MASK                   0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF112_MASK                   0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF113_MASK                   0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF114_MASK                   0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF115_MASK                   0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF116_MASK                   0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF117_MASK                   0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF118_MASK                   0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF119_MASK                   0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF120_MASK                   0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF121_MASK                   0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF122_MASK                   0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF123_MASK                   0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF124_MASK                   0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF125_MASK                   0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF126_MASK                   0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF127_MASK                   0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF0__SHIFT                         0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF1__SHIFT                         0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF2__SHIFT                         0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF3__SHIFT                         0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF4__SHIFT                         0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF5__SHIFT                         0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF6__SHIFT                         0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF7__SHIFT                         0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF8__SHIFT                         0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF9__SHIFT                         0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF10__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF11__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF12__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF13__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF14__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF15__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF16__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF17__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF18__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF19__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF20__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF21__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF22__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF23__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF24__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF25__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF26__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF27__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF28__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF29__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF30__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF31__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF0_MASK                           0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF1_MASK                           0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF2_MASK                           0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF3_MASK                           0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF4_MASK                           0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF5_MASK                           0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF6_MASK                           0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF7_MASK                           0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF8_MASK                           0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF9_MASK                           0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF10_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF11_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF12_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF13_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF14_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF15_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF16_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF17_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF18_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF19_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF20_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF21_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF22_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF23_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF24_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF25_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF26_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF27_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF28_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF29_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF30_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF31_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF32__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF33__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF34__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF35__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF36__SHIFT                        0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF37__SHIFT                        0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF38__SHIFT                        0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF39__SHIFT                        0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF40__SHIFT                        0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF41__SHIFT                        0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF42__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF43__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF44__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF45__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF46__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF47__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF48__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF49__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF50__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF51__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF52__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF53__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF54__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF55__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF56__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF57__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF58__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF59__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF60__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF61__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF62__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF63__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF32_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF33_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF34_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF35_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF36_MASK                          0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF37_MASK                          0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF38_MASK                          0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF39_MASK                          0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF40_MASK                          0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF41_MASK                          0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF42_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF43_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF44_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF45_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF46_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF47_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF48_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF49_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF50_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF51_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF52_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF53_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF54_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF55_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF56_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF57_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF58_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF59_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF60_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF61_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF62_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF63_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF64__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF65__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF66__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF67__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF68__SHIFT                        0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF69__SHIFT                        0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF70__SHIFT                        0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF71__SHIFT                        0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF72__SHIFT                        0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF73__SHIFT                        0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF74__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF75__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF76__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF77__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF78__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF79__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF80__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF81__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF82__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF83__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF84__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF85__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF86__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF87__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF88__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF89__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF90__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF91__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF92__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF93__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF94__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF95__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF64_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF65_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF66_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF67_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF68_MASK                          0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF69_MASK                          0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF70_MASK                          0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF71_MASK                          0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF72_MASK                          0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF73_MASK                          0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF74_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF75_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF76_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF77_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF78_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF79_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF80_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF81_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF82_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF83_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF84_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF85_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF86_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF87_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF88_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF89_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF90_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF91_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF92_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF93_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF94_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF95_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF96__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF97__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF98__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF99__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF100__SHIFT                       0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF101__SHIFT                       0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF102__SHIFT                       0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF103__SHIFT                       0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF104__SHIFT                       0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF105__SHIFT                       0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF106__SHIFT                       0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF107__SHIFT                       0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF108__SHIFT                       0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF109__SHIFT                       0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF110__SHIFT                       0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF111__SHIFT                       0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF112__SHIFT                       0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF113__SHIFT                       0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF114__SHIFT                       0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF115__SHIFT                       0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF116__SHIFT                       0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF117__SHIFT                       0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF118__SHIFT                       0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF119__SHIFT                       0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF120__SHIFT                       0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF121__SHIFT                       0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF122__SHIFT                       0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF123__SHIFT                       0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF124__SHIFT                       0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF125__SHIFT                       0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF126__SHIFT                       0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF127__SHIFT                       0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF96_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF97_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF98_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF99_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF100_MASK                         0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF101_MASK                         0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF102_MASK                         0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF103_MASK                         0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF104_MASK                         0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF105_MASK                         0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF106_MASK                         0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF107_MASK                         0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF108_MASK                         0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF109_MASK                         0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF110_MASK                         0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF111_MASK                         0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF112_MASK                         0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF113_MASK                         0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF114_MASK                         0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF115_MASK                         0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF116_MASK                         0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF117_MASK                         0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF118_MASK                         0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF119_MASK                         0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF120_MASK                         0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF121_MASK                         0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF122_MASK                         0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF123_MASK                         0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF124_MASK                         0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF125_MASK                         0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF126_MASK                         0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF127_MASK                         0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF0__SHIFT                                 0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF1__SHIFT                                 0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF2__SHIFT                                 0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF3__SHIFT                                 0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF4__SHIFT                                 0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF5__SHIFT                                 0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF6__SHIFT                                 0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF7__SHIFT                                 0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF8__SHIFT                                 0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF9__SHIFT                                 0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF10__SHIFT                                0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF11__SHIFT                                0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF12__SHIFT                                0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF13__SHIFT                                0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF14__SHIFT                                0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF15__SHIFT                                0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF16__SHIFT                                0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF17__SHIFT                                0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF18__SHIFT                                0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF19__SHIFT                                0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF20__SHIFT                                0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF21__SHIFT                                0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF22__SHIFT                                0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF23__SHIFT                                0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF24__SHIFT                                0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF25__SHIFT                                0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF26__SHIFT                                0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF27__SHIFT                                0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF28__SHIFT                                0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF29__SHIFT                                0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF30__SHIFT                                0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF31__SHIFT                                0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF0_MASK                                   0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF1_MASK                                   0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF2_MASK                                   0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF3_MASK                                   0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF4_MASK                                   0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF5_MASK                                   0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF6_MASK                                   0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF7_MASK                                   0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF8_MASK                                   0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF9_MASK                                   0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF10_MASK                                  0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF11_MASK                                  0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF12_MASK                                  0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF13_MASK                                  0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF14_MASK                                  0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF15_MASK                                  0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF16_MASK                                  0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF17_MASK                                  0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF18_MASK                                  0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF19_MASK                                  0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF20_MASK                                  0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF21_MASK                                  0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF22_MASK                                  0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF23_MASK                                  0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF24_MASK                                  0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF25_MASK                                  0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF26_MASK                                  0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF27_MASK                                  0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF28_MASK                                  0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF29_MASK                                  0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF30_MASK                                  0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF31_MASK                                  0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF32__SHIFT                                0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF33__SHIFT                                0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF34__SHIFT                                0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF35__SHIFT                                0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF36__SHIFT                                0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF37__SHIFT                                0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF38__SHIFT                                0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF39__SHIFT                                0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF40__SHIFT                                0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF41__SHIFT                                0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF42__SHIFT                                0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF43__SHIFT                                0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF44__SHIFT                                0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF45__SHIFT                                0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF46__SHIFT                                0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF47__SHIFT                                0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF48__SHIFT                                0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF49__SHIFT                                0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF50__SHIFT                                0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF51__SHIFT                                0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF52__SHIFT                                0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF53__SHIFT                                0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF54__SHIFT                                0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF55__SHIFT                                0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF56__SHIFT                                0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF57__SHIFT                                0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF58__SHIFT                                0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF59__SHIFT                                0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF60__SHIFT                                0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF61__SHIFT                                0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF62__SHIFT                                0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF63__SHIFT                                0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF32_MASK                                  0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF33_MASK                                  0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF34_MASK                                  0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF35_MASK                                  0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF36_MASK                                  0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF37_MASK                                  0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF38_MASK                                  0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF39_MASK                                  0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF40_MASK                                  0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF41_MASK                                  0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF42_MASK                                  0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF43_MASK                                  0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF44_MASK                                  0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF45_MASK                                  0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF46_MASK                                  0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF47_MASK                                  0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF48_MASK                                  0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF49_MASK                                  0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF50_MASK                                  0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF51_MASK                                  0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF52_MASK                                  0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF53_MASK                                  0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF54_MASK                                  0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF55_MASK                                  0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF56_MASK                                  0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF57_MASK                                  0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF58_MASK                                  0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF59_MASK                                  0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF60_MASK                                  0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF61_MASK                                  0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF62_MASK                                  0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF63_MASK                                  0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF64__SHIFT                                0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF65__SHIFT                                0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF66__SHIFT                                0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF67__SHIFT                                0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF68__SHIFT                                0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF69__SHIFT                                0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF70__SHIFT                                0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF71__SHIFT                                0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF72__SHIFT                                0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF73__SHIFT                                0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF74__SHIFT                                0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF75__SHIFT                                0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF76__SHIFT                                0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF77__SHIFT                                0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF78__SHIFT                                0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF79__SHIFT                                0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF80__SHIFT                                0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF81__SHIFT                                0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF82__SHIFT                                0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF83__SHIFT                                0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF84__SHIFT                                0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF85__SHIFT                                0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF86__SHIFT                                0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF87__SHIFT                                0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF88__SHIFT                                0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF89__SHIFT                                0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF90__SHIFT                                0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF91__SHIFT                                0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF92__SHIFT                                0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF93__SHIFT                                0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF94__SHIFT                                0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF95__SHIFT                                0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF64_MASK                                  0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF65_MASK                                  0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF66_MASK                                  0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF67_MASK                                  0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF68_MASK                                  0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF69_MASK                                  0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF70_MASK                                  0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF71_MASK                                  0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF72_MASK                                  0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF73_MASK                                  0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF74_MASK                                  0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF75_MASK                                  0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF76_MASK                                  0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF77_MASK                                  0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF78_MASK                                  0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF79_MASK                                  0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF80_MASK                                  0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF81_MASK                                  0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF82_MASK                                  0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF83_MASK                                  0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF84_MASK                                  0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF85_MASK                                  0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF86_MASK                                  0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF87_MASK                                  0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF88_MASK                                  0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF89_MASK                                  0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF90_MASK                                  0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF91_MASK                                  0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF92_MASK                                  0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF93_MASK                                  0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF94_MASK                                  0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF95_MASK                                  0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF96__SHIFT                                0x0
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF97__SHIFT                                0x1
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF98__SHIFT                                0x2
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF99__SHIFT                                0x3
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF100__SHIFT                               0x4
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF101__SHIFT                               0x5
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF102__SHIFT                               0x6
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF103__SHIFT                               0x7
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF104__SHIFT                               0x8
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF105__SHIFT                               0x9
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF106__SHIFT                               0xa
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF107__SHIFT                               0xb
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF108__SHIFT                               0xc
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF109__SHIFT                               0xd
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF110__SHIFT                               0xe
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF111__SHIFT                               0xf
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF112__SHIFT                               0x10
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF113__SHIFT                               0x11
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF114__SHIFT                               0x12
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF115__SHIFT                               0x13
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF116__SHIFT                               0x14
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF117__SHIFT                               0x15
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF118__SHIFT                               0x16
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF119__SHIFT                               0x17
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF120__SHIFT                               0x18
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF121__SHIFT                               0x19
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF122__SHIFT                               0x1a
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF123__SHIFT                               0x1b
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF124__SHIFT                               0x1c
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF125__SHIFT                               0x1d
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF126__SHIFT                               0x1e
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF127__SHIFT                               0x1f
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF96_MASK                                  0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF97_MASK                                  0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF98_MASK                                  0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF99_MASK                                  0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF100_MASK                                 0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF101_MASK                                 0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF102_MASK                                 0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF103_MASK                                 0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF104_MASK                                 0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF105_MASK                                 0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF106_MASK                                 0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF107_MASK                                 0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF108_MASK                                 0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF109_MASK                                 0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF110_MASK                                 0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF111_MASK                                 0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF112_MASK                                 0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF113_MASK                                 0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF114_MASK                                 0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF115_MASK                                 0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF116_MASK                                 0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF117_MASK                                 0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF118_MASK                                 0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF119_MASK                                 0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF120_MASK                                 0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF121_MASK                                 0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF122_MASK                                 0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF123_MASK                                 0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF124_MASK                                 0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF125_MASK                                 0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF126_MASK                                 0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF0_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF127_MASK                                 0x80000000L
+
+
+// addressBlock: nbif0_nbif0_bif_misc_pfvf_dev0_epf1_bif_misc_pfvf_regblk
+//BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF0__SHIFT                               0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF1__SHIFT                               0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF2__SHIFT                               0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF3__SHIFT                               0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF4__SHIFT                               0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF5__SHIFT                               0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF6__SHIFT                               0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF7__SHIFT                               0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF8__SHIFT                               0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF9__SHIFT                               0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF10__SHIFT                              0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF11__SHIFT                              0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF12__SHIFT                              0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF13__SHIFT                              0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF14__SHIFT                              0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF15__SHIFT                              0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF16__SHIFT                              0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF17__SHIFT                              0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF18__SHIFT                              0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF19__SHIFT                              0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF20__SHIFT                              0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF21__SHIFT                              0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF22__SHIFT                              0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF23__SHIFT                              0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF24__SHIFT                              0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF25__SHIFT                              0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF26__SHIFT                              0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF27__SHIFT                              0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF28__SHIFT                              0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF29__SHIFT                              0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF30__SHIFT                              0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF31__SHIFT                              0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF0_MASK                                 0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF1_MASK                                 0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF2_MASK                                 0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF3_MASK                                 0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF4_MASK                                 0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF5_MASK                                 0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF6_MASK                                 0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF7_MASK                                 0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF8_MASK                                 0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF9_MASK                                 0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF10_MASK                                0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF11_MASK                                0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF12_MASK                                0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF13_MASK                                0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF14_MASK                                0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF15_MASK                                0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF16_MASK                                0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF17_MASK                                0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF18_MASK                                0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF19_MASK                                0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF20_MASK                                0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF21_MASK                                0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF22_MASK                                0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF23_MASK                                0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF24_MASK                                0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF25_MASK                                0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF26_MASK                                0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF27_MASK                                0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF28_MASK                                0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF29_MASK                                0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF30_MASK                                0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG0__DMA_ON_BME_LOW_VF31_MASK                                0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF32__SHIFT                              0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF33__SHIFT                              0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF34__SHIFT                              0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF35__SHIFT                              0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF36__SHIFT                              0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF37__SHIFT                              0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF38__SHIFT                              0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF39__SHIFT                              0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF40__SHIFT                              0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF41__SHIFT                              0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF42__SHIFT                              0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF43__SHIFT                              0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF44__SHIFT                              0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF45__SHIFT                              0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF46__SHIFT                              0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF47__SHIFT                              0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF48__SHIFT                              0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF49__SHIFT                              0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF50__SHIFT                              0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF51__SHIFT                              0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF52__SHIFT                              0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF53__SHIFT                              0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF54__SHIFT                              0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF55__SHIFT                              0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF56__SHIFT                              0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF57__SHIFT                              0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF58__SHIFT                              0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF59__SHIFT                              0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF60__SHIFT                              0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF61__SHIFT                              0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF62__SHIFT                              0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF63__SHIFT                              0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF32_MASK                                0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF33_MASK                                0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF34_MASK                                0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF35_MASK                                0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF36_MASK                                0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF37_MASK                                0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF38_MASK                                0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF39_MASK                                0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF40_MASK                                0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF41_MASK                                0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF42_MASK                                0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF43_MASK                                0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF44_MASK                                0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF45_MASK                                0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF46_MASK                                0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF47_MASK                                0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF48_MASK                                0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF49_MASK                                0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF50_MASK                                0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF51_MASK                                0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF52_MASK                                0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF53_MASK                                0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF54_MASK                                0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF55_MASK                                0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF56_MASK                                0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF57_MASK                                0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF58_MASK                                0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF59_MASK                                0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF60_MASK                                0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF61_MASK                                0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF62_MASK                                0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG1__DMA_ON_BME_LOW_VF63_MASK                                0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF64__SHIFT                              0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF65__SHIFT                              0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF66__SHIFT                              0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF67__SHIFT                              0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF68__SHIFT                              0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF69__SHIFT                              0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF70__SHIFT                              0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF71__SHIFT                              0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF72__SHIFT                              0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF73__SHIFT                              0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF74__SHIFT                              0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF75__SHIFT                              0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF76__SHIFT                              0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF77__SHIFT                              0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF78__SHIFT                              0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF79__SHIFT                              0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF80__SHIFT                              0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF81__SHIFT                              0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF82__SHIFT                              0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF83__SHIFT                              0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF84__SHIFT                              0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF85__SHIFT                              0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF86__SHIFT                              0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF87__SHIFT                              0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF88__SHIFT                              0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF89__SHIFT                              0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF90__SHIFT                              0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF91__SHIFT                              0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF92__SHIFT                              0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF93__SHIFT                              0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF94__SHIFT                              0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF95__SHIFT                              0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF64_MASK                                0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF65_MASK                                0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF66_MASK                                0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF67_MASK                                0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF68_MASK                                0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF69_MASK                                0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF70_MASK                                0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF71_MASK                                0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF72_MASK                                0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF73_MASK                                0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF74_MASK                                0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF75_MASK                                0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF76_MASK                                0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF77_MASK                                0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF78_MASK                                0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF79_MASK                                0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF80_MASK                                0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF81_MASK                                0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF82_MASK                                0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF83_MASK                                0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF84_MASK                                0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF85_MASK                                0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF86_MASK                                0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF87_MASK                                0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF88_MASK                                0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF89_MASK                                0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF90_MASK                                0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF91_MASK                                0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF92_MASK                                0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF93_MASK                                0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF94_MASK                                0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG2__DMA_ON_BME_LOW_VF95_MASK                                0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF96__SHIFT                              0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF97__SHIFT                              0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF98__SHIFT                              0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF99__SHIFT                              0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF100__SHIFT                             0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF101__SHIFT                             0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF102__SHIFT                             0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF103__SHIFT                             0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF104__SHIFT                             0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF105__SHIFT                             0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF106__SHIFT                             0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF107__SHIFT                             0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF108__SHIFT                             0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF109__SHIFT                             0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF110__SHIFT                             0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF111__SHIFT                             0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF112__SHIFT                             0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF113__SHIFT                             0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF114__SHIFT                             0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF115__SHIFT                             0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF116__SHIFT                             0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF117__SHIFT                             0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF118__SHIFT                             0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF119__SHIFT                             0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF120__SHIFT                             0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF121__SHIFT                             0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF122__SHIFT                             0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF123__SHIFT                             0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF124__SHIFT                             0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF125__SHIFT                             0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF126__SHIFT                             0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF127__SHIFT                             0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF96_MASK                                0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF97_MASK                                0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF98_MASK                                0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF99_MASK                                0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF100_MASK                               0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF101_MASK                               0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF102_MASK                               0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF103_MASK                               0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF104_MASK                               0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF105_MASK                               0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF106_MASK                               0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF107_MASK                               0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF108_MASK                               0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF109_MASK                               0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF110_MASK                               0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF111_MASK                               0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF112_MASK                               0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF113_MASK                               0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF114_MASK                               0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF115_MASK                               0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF116_MASK                               0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF117_MASK                               0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF118_MASK                               0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF119_MASK                               0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF120_MASK                               0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF121_MASK                               0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF122_MASK                               0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF123_MASK                               0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF124_MASK                               0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF125_MASK                               0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF126_MASK                               0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIFC_VF_BME_ERR_LOG3__DMA_ON_BME_LOW_VF127_MASK                               0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF0__SHIFT                         0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF1__SHIFT                         0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF2__SHIFT                         0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF3__SHIFT                         0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF4__SHIFT                         0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF5__SHIFT                         0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF6__SHIFT                         0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF7__SHIFT                         0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF8__SHIFT                         0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF9__SHIFT                         0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF10__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF11__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF12__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF13__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF14__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF15__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF16__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF17__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF18__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF19__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF20__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF21__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF22__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF23__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF24__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF25__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF26__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF27__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF28__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF29__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF30__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF31__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF0_MASK                           0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF1_MASK                           0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF2_MASK                           0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF3_MASK                           0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF4_MASK                           0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF5_MASK                           0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF6_MASK                           0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF7_MASK                           0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF8_MASK                           0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF9_MASK                           0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF10_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF11_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF12_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF13_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF14_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF15_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF16_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF17_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF18_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF19_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF20_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF21_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF22_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF23_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF24_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF25_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF26_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF27_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF28_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF29_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF30_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE0__UR_ATOMIC_OPCODE_VF31_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF32__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF33__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF34__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF35__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF36__SHIFT                        0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF37__SHIFT                        0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF38__SHIFT                        0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF39__SHIFT                        0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF40__SHIFT                        0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF41__SHIFT                        0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF42__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF43__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF44__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF45__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF46__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF47__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF48__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF49__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF50__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF51__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF52__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF53__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF54__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF55__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF56__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF57__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF58__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF59__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF60__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF61__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF62__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF63__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF32_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF33_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF34_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF35_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF36_MASK                          0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF37_MASK                          0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF38_MASK                          0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF39_MASK                          0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF40_MASK                          0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF41_MASK                          0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF42_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF43_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF44_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF45_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF46_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF47_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF48_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF49_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF50_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF51_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF52_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF53_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF54_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF55_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF56_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF57_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF58_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF59_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF60_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF61_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF62_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE1__UR_ATOMIC_OPCODE_VF63_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF64__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF65__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF66__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF67__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF68__SHIFT                        0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF69__SHIFT                        0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF70__SHIFT                        0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF71__SHIFT                        0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF72__SHIFT                        0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF73__SHIFT                        0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF74__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF75__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF76__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF77__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF78__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF79__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF80__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF81__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF82__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF83__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF84__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF85__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF86__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF87__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF88__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF89__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF90__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF91__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF92__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF93__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF94__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF95__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF64_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF65_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF66_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF67_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF68_MASK                          0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF69_MASK                          0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF70_MASK                          0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF71_MASK                          0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF72_MASK                          0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF73_MASK                          0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF74_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF75_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF76_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF77_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF78_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF79_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF80_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF81_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF82_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF83_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF84_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF85_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF86_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF87_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF88_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF89_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF90_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF91_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF92_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF93_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF94_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE2__UR_ATOMIC_OPCODE_VF95_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF96__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF97__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF98__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF99__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF100__SHIFT                       0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF101__SHIFT                       0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF102__SHIFT                       0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF103__SHIFT                       0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF104__SHIFT                       0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF105__SHIFT                       0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF106__SHIFT                       0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF107__SHIFT                       0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF108__SHIFT                       0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF109__SHIFT                       0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF110__SHIFT                       0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF111__SHIFT                       0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF112__SHIFT                       0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF113__SHIFT                       0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF114__SHIFT                       0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF115__SHIFT                       0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF116__SHIFT                       0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF117__SHIFT                       0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF118__SHIFT                       0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF119__SHIFT                       0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF120__SHIFT                       0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF121__SHIFT                       0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF122__SHIFT                       0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF123__SHIFT                       0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF124__SHIFT                       0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF125__SHIFT                       0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF126__SHIFT                       0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF127__SHIFT                       0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF96_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF97_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF98_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF99_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF100_MASK                         0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF101_MASK                         0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF102_MASK                         0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF103_MASK                         0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF104_MASK                         0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF105_MASK                         0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF106_MASK                         0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF107_MASK                         0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF108_MASK                         0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF109_MASK                         0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF110_MASK                         0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF111_MASK                         0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF112_MASK                         0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF113_MASK                         0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF114_MASK                         0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF115_MASK                         0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF116_MASK                         0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF117_MASK                         0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF118_MASK                         0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF119_MASK                         0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF120_MASK                         0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF121_MASK                         0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF122_MASK                         0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF123_MASK                         0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF124_MASK                         0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF125_MASK                         0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF126_MASK                         0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_OPCODE3__UR_ATOMIC_OPCODE_VF127_MASK                         0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF0__SHIFT                   0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF1__SHIFT                   0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF2__SHIFT                   0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF3__SHIFT                   0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF4__SHIFT                   0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF5__SHIFT                   0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF6__SHIFT                   0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF7__SHIFT                   0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF8__SHIFT                   0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF9__SHIFT                   0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF10__SHIFT                  0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF11__SHIFT                  0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF12__SHIFT                  0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF13__SHIFT                  0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF14__SHIFT                  0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF15__SHIFT                  0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF16__SHIFT                  0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF17__SHIFT                  0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF18__SHIFT                  0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF19__SHIFT                  0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF20__SHIFT                  0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF21__SHIFT                  0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF22__SHIFT                  0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF23__SHIFT                  0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF24__SHIFT                  0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF25__SHIFT                  0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF26__SHIFT                  0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF27__SHIFT                  0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF28__SHIFT                  0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF29__SHIFT                  0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF30__SHIFT                  0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF31__SHIFT                  0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF0_MASK                     0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF1_MASK                     0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF2_MASK                     0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF3_MASK                     0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF4_MASK                     0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF5_MASK                     0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF6_MASK                     0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF7_MASK                     0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF8_MASK                     0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF9_MASK                     0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF10_MASK                    0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF11_MASK                    0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF12_MASK                    0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF13_MASK                    0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF14_MASK                    0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF15_MASK                    0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF16_MASK                    0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF17_MASK                    0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF18_MASK                    0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF19_MASK                    0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF20_MASK                    0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF21_MASK                    0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF22_MASK                    0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF23_MASK                    0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF24_MASK                    0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF25_MASK                    0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF26_MASK                    0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF27_MASK                    0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF28_MASK                    0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF29_MASK                    0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF30_MASK                    0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW0__UR_ATOMIC_REQEN_LOW_VF31_MASK                    0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF32__SHIFT                  0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF33__SHIFT                  0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF34__SHIFT                  0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF35__SHIFT                  0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF36__SHIFT                  0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF37__SHIFT                  0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF38__SHIFT                  0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF39__SHIFT                  0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF40__SHIFT                  0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF41__SHIFT                  0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF42__SHIFT                  0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF43__SHIFT                  0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF44__SHIFT                  0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF45__SHIFT                  0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF46__SHIFT                  0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF47__SHIFT                  0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF48__SHIFT                  0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF49__SHIFT                  0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF50__SHIFT                  0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF51__SHIFT                  0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF52__SHIFT                  0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF53__SHIFT                  0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF54__SHIFT                  0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF55__SHIFT                  0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF56__SHIFT                  0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF57__SHIFT                  0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF58__SHIFT                  0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF59__SHIFT                  0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF60__SHIFT                  0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF61__SHIFT                  0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF62__SHIFT                  0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF63__SHIFT                  0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF32_MASK                    0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF33_MASK                    0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF34_MASK                    0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF35_MASK                    0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF36_MASK                    0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF37_MASK                    0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF38_MASK                    0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF39_MASK                    0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF40_MASK                    0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF41_MASK                    0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF42_MASK                    0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF43_MASK                    0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF44_MASK                    0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF45_MASK                    0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF46_MASK                    0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF47_MASK                    0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF48_MASK                    0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF49_MASK                    0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF50_MASK                    0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF51_MASK                    0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF52_MASK                    0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF53_MASK                    0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF54_MASK                    0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF55_MASK                    0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF56_MASK                    0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF57_MASK                    0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF58_MASK                    0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF59_MASK                    0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF60_MASK                    0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF61_MASK                    0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF62_MASK                    0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW1__UR_ATOMIC_REQEN_LOW_VF63_MASK                    0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF64__SHIFT                  0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF65__SHIFT                  0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF66__SHIFT                  0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF67__SHIFT                  0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF68__SHIFT                  0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF69__SHIFT                  0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF70__SHIFT                  0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF71__SHIFT                  0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF72__SHIFT                  0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF73__SHIFT                  0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF74__SHIFT                  0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF75__SHIFT                  0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF76__SHIFT                  0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF77__SHIFT                  0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF78__SHIFT                  0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF79__SHIFT                  0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF80__SHIFT                  0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF81__SHIFT                  0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF82__SHIFT                  0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF83__SHIFT                  0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF84__SHIFT                  0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF85__SHIFT                  0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF86__SHIFT                  0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF87__SHIFT                  0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF88__SHIFT                  0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF89__SHIFT                  0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF90__SHIFT                  0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF91__SHIFT                  0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF92__SHIFT                  0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF93__SHIFT                  0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF94__SHIFT                  0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF95__SHIFT                  0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF64_MASK                    0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF65_MASK                    0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF66_MASK                    0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF67_MASK                    0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF68_MASK                    0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF69_MASK                    0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF70_MASK                    0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF71_MASK                    0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF72_MASK                    0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF73_MASK                    0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF74_MASK                    0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF75_MASK                    0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF76_MASK                    0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF77_MASK                    0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF78_MASK                    0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF79_MASK                    0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF80_MASK                    0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF81_MASK                    0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF82_MASK                    0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF83_MASK                    0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF84_MASK                    0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF85_MASK                    0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF86_MASK                    0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF87_MASK                    0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF88_MASK                    0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF89_MASK                    0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF90_MASK                    0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF91_MASK                    0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF92_MASK                    0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF93_MASK                    0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF94_MASK                    0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW2__UR_ATOMIC_REQEN_LOW_VF95_MASK                    0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF96__SHIFT                  0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF97__SHIFT                  0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF98__SHIFT                  0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF99__SHIFT                  0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF100__SHIFT                 0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF101__SHIFT                 0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF102__SHIFT                 0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF103__SHIFT                 0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF104__SHIFT                 0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF105__SHIFT                 0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF106__SHIFT                 0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF107__SHIFT                 0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF108__SHIFT                 0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF109__SHIFT                 0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF110__SHIFT                 0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF111__SHIFT                 0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF112__SHIFT                 0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF113__SHIFT                 0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF114__SHIFT                 0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF115__SHIFT                 0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF116__SHIFT                 0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF117__SHIFT                 0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF118__SHIFT                 0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF119__SHIFT                 0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF120__SHIFT                 0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF121__SHIFT                 0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF122__SHIFT                 0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF123__SHIFT                 0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF124__SHIFT                 0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF125__SHIFT                 0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF126__SHIFT                 0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF127__SHIFT                 0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF96_MASK                    0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF97_MASK                    0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF98_MASK                    0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF99_MASK                    0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF100_MASK                   0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF101_MASK                   0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF102_MASK                   0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF103_MASK                   0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF104_MASK                   0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF105_MASK                   0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF106_MASK                   0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF107_MASK                   0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF108_MASK                   0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF109_MASK                   0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF110_MASK                   0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF111_MASK                   0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF112_MASK                   0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF113_MASK                   0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF114_MASK                   0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF115_MASK                   0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF116_MASK                   0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF117_MASK                   0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF118_MASK                   0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF119_MASK                   0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF120_MASK                   0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF121_MASK                   0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF122_MASK                   0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF123_MASK                   0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF124_MASK                   0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF125_MASK                   0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF126_MASK                   0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_REQEN_LOW3__UR_ATOMIC_REQEN_LOW_VF127_MASK                   0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF0__SHIFT                         0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF1__SHIFT                         0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF2__SHIFT                         0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF3__SHIFT                         0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF4__SHIFT                         0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF5__SHIFT                         0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF6__SHIFT                         0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF7__SHIFT                         0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF8__SHIFT                         0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF9__SHIFT                         0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF10__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF11__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF12__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF13__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF14__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF15__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF16__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF17__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF18__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF19__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF20__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF21__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF22__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF23__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF24__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF25__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF26__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF27__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF28__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF29__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF30__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF31__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF0_MASK                           0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF1_MASK                           0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF2_MASK                           0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF3_MASK                           0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF4_MASK                           0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF5_MASK                           0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF6_MASK                           0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF7_MASK                           0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF8_MASK                           0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF9_MASK                           0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF10_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF11_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF12_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF13_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF14_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF15_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF16_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF17_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF18_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF19_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF20_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF21_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF22_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF23_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF24_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF25_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF26_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF27_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF28_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF29_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF30_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH0__UR_ATOMIC_LENGTH_VF31_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF32__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF33__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF34__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF35__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF36__SHIFT                        0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF37__SHIFT                        0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF38__SHIFT                        0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF39__SHIFT                        0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF40__SHIFT                        0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF41__SHIFT                        0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF42__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF43__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF44__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF45__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF46__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF47__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF48__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF49__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF50__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF51__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF52__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF53__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF54__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF55__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF56__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF57__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF58__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF59__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF60__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF61__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF62__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF63__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF32_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF33_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF34_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF35_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF36_MASK                          0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF37_MASK                          0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF38_MASK                          0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF39_MASK                          0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF40_MASK                          0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF41_MASK                          0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF42_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF43_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF44_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF45_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF46_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF47_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF48_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF49_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF50_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF51_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF52_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF53_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF54_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF55_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF56_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF57_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF58_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF59_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF60_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF61_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF62_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH1__UR_ATOMIC_LENGTH_VF63_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF64__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF65__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF66__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF67__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF68__SHIFT                        0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF69__SHIFT                        0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF70__SHIFT                        0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF71__SHIFT                        0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF72__SHIFT                        0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF73__SHIFT                        0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF74__SHIFT                        0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF75__SHIFT                        0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF76__SHIFT                        0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF77__SHIFT                        0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF78__SHIFT                        0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF79__SHIFT                        0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF80__SHIFT                        0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF81__SHIFT                        0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF82__SHIFT                        0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF83__SHIFT                        0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF84__SHIFT                        0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF85__SHIFT                        0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF86__SHIFT                        0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF87__SHIFT                        0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF88__SHIFT                        0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF89__SHIFT                        0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF90__SHIFT                        0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF91__SHIFT                        0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF92__SHIFT                        0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF93__SHIFT                        0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF94__SHIFT                        0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF95__SHIFT                        0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF64_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF65_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF66_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF67_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF68_MASK                          0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF69_MASK                          0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF70_MASK                          0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF71_MASK                          0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF72_MASK                          0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF73_MASK                          0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF74_MASK                          0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF75_MASK                          0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF76_MASK                          0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF77_MASK                          0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF78_MASK                          0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF79_MASK                          0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF80_MASK                          0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF81_MASK                          0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF82_MASK                          0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF83_MASK                          0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF84_MASK                          0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF85_MASK                          0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF86_MASK                          0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF87_MASK                          0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF88_MASK                          0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF89_MASK                          0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF90_MASK                          0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF91_MASK                          0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF92_MASK                          0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF93_MASK                          0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF94_MASK                          0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH2__UR_ATOMIC_LENGTH_VF95_MASK                          0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF96__SHIFT                        0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF97__SHIFT                        0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF98__SHIFT                        0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF99__SHIFT                        0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF100__SHIFT                       0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF101__SHIFT                       0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF102__SHIFT                       0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF103__SHIFT                       0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF104__SHIFT                       0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF105__SHIFT                       0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF106__SHIFT                       0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF107__SHIFT                       0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF108__SHIFT                       0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF109__SHIFT                       0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF110__SHIFT                       0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF111__SHIFT                       0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF112__SHIFT                       0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF113__SHIFT                       0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF114__SHIFT                       0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF115__SHIFT                       0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF116__SHIFT                       0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF117__SHIFT                       0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF118__SHIFT                       0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF119__SHIFT                       0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF120__SHIFT                       0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF121__SHIFT                       0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF122__SHIFT                       0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF123__SHIFT                       0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF124__SHIFT                       0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF125__SHIFT                       0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF126__SHIFT                       0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF127__SHIFT                       0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF96_MASK                          0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF97_MASK                          0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF98_MASK                          0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF99_MASK                          0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF100_MASK                         0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF101_MASK                         0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF102_MASK                         0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF103_MASK                         0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF104_MASK                         0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF105_MASK                         0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF106_MASK                         0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF107_MASK                         0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF108_MASK                         0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF109_MASK                         0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF110_MASK                         0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF111_MASK                         0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF112_MASK                         0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF113_MASK                         0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF114_MASK                         0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF115_MASK                         0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF116_MASK                         0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF117_MASK                         0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF118_MASK                         0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF119_MASK                         0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF120_MASK                         0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF121_MASK                         0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF122_MASK                         0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF123_MASK                         0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF124_MASK                         0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF125_MASK                         0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF126_MASK                         0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_LENGTH3__UR_ATOMIC_LENGTH_VF127_MASK                         0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF0__SHIFT                                 0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF1__SHIFT                                 0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF2__SHIFT                                 0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF3__SHIFT                                 0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF4__SHIFT                                 0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF5__SHIFT                                 0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF6__SHIFT                                 0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF7__SHIFT                                 0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF8__SHIFT                                 0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF9__SHIFT                                 0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF10__SHIFT                                0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF11__SHIFT                                0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF12__SHIFT                                0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF13__SHIFT                                0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF14__SHIFT                                0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF15__SHIFT                                0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF16__SHIFT                                0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF17__SHIFT                                0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF18__SHIFT                                0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF19__SHIFT                                0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF20__SHIFT                                0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF21__SHIFT                                0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF22__SHIFT                                0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF23__SHIFT                                0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF24__SHIFT                                0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF25__SHIFT                                0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF26__SHIFT                                0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF27__SHIFT                                0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF28__SHIFT                                0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF29__SHIFT                                0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF30__SHIFT                                0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF31__SHIFT                                0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF0_MASK                                   0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF1_MASK                                   0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF2_MASK                                   0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF3_MASK                                   0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF4_MASK                                   0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF5_MASK                                   0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF6_MASK                                   0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF7_MASK                                   0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF8_MASK                                   0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF9_MASK                                   0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF10_MASK                                  0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF11_MASK                                  0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF12_MASK                                  0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF13_MASK                                  0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF14_MASK                                  0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF15_MASK                                  0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF16_MASK                                  0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF17_MASK                                  0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF18_MASK                                  0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF19_MASK                                  0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF20_MASK                                  0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF21_MASK                                  0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF22_MASK                                  0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF23_MASK                                  0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF24_MASK                                  0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF25_MASK                                  0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF26_MASK                                  0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF27_MASK                                  0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF28_MASK                                  0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF29_MASK                                  0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF30_MASK                                  0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR0__UR_ATOMIC_NR_VF31_MASK                                  0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF32__SHIFT                                0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF33__SHIFT                                0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF34__SHIFT                                0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF35__SHIFT                                0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF36__SHIFT                                0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF37__SHIFT                                0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF38__SHIFT                                0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF39__SHIFT                                0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF40__SHIFT                                0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF41__SHIFT                                0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF42__SHIFT                                0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF43__SHIFT                                0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF44__SHIFT                                0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF45__SHIFT                                0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF46__SHIFT                                0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF47__SHIFT                                0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF48__SHIFT                                0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF49__SHIFT                                0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF50__SHIFT                                0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF51__SHIFT                                0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF52__SHIFT                                0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF53__SHIFT                                0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF54__SHIFT                                0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF55__SHIFT                                0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF56__SHIFT                                0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF57__SHIFT                                0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF58__SHIFT                                0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF59__SHIFT                                0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF60__SHIFT                                0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF61__SHIFT                                0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF62__SHIFT                                0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF63__SHIFT                                0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF32_MASK                                  0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF33_MASK                                  0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF34_MASK                                  0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF35_MASK                                  0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF36_MASK                                  0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF37_MASK                                  0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF38_MASK                                  0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF39_MASK                                  0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF40_MASK                                  0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF41_MASK                                  0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF42_MASK                                  0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF43_MASK                                  0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF44_MASK                                  0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF45_MASK                                  0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF46_MASK                                  0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF47_MASK                                  0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF48_MASK                                  0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF49_MASK                                  0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF50_MASK                                  0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF51_MASK                                  0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF52_MASK                                  0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF53_MASK                                  0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF54_MASK                                  0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF55_MASK                                  0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF56_MASK                                  0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF57_MASK                                  0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF58_MASK                                  0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF59_MASK                                  0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF60_MASK                                  0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF61_MASK                                  0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF62_MASK                                  0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR1__UR_ATOMIC_NR_VF63_MASK                                  0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF64__SHIFT                                0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF65__SHIFT                                0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF66__SHIFT                                0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF67__SHIFT                                0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF68__SHIFT                                0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF69__SHIFT                                0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF70__SHIFT                                0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF71__SHIFT                                0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF72__SHIFT                                0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF73__SHIFT                                0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF74__SHIFT                                0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF75__SHIFT                                0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF76__SHIFT                                0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF77__SHIFT                                0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF78__SHIFT                                0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF79__SHIFT                                0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF80__SHIFT                                0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF81__SHIFT                                0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF82__SHIFT                                0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF83__SHIFT                                0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF84__SHIFT                                0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF85__SHIFT                                0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF86__SHIFT                                0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF87__SHIFT                                0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF88__SHIFT                                0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF89__SHIFT                                0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF90__SHIFT                                0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF91__SHIFT                                0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF92__SHIFT                                0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF93__SHIFT                                0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF94__SHIFT                                0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF95__SHIFT                                0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF64_MASK                                  0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF65_MASK                                  0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF66_MASK                                  0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF67_MASK                                  0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF68_MASK                                  0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF69_MASK                                  0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF70_MASK                                  0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF71_MASK                                  0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF72_MASK                                  0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF73_MASK                                  0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF74_MASK                                  0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF75_MASK                                  0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF76_MASK                                  0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF77_MASK                                  0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF78_MASK                                  0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF79_MASK                                  0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF80_MASK                                  0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF81_MASK                                  0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF82_MASK                                  0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF83_MASK                                  0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF84_MASK                                  0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF85_MASK                                  0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF86_MASK                                  0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF87_MASK                                  0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF88_MASK                                  0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF89_MASK                                  0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF90_MASK                                  0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF91_MASK                                  0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF92_MASK                                  0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF93_MASK                                  0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF94_MASK                                  0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR2__UR_ATOMIC_NR_VF95_MASK                                  0x80000000L
+//BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF96__SHIFT                                0x0
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF97__SHIFT                                0x1
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF98__SHIFT                                0x2
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF99__SHIFT                                0x3
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF100__SHIFT                               0x4
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF101__SHIFT                               0x5
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF102__SHIFT                               0x6
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF103__SHIFT                               0x7
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF104__SHIFT                               0x8
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF105__SHIFT                               0x9
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF106__SHIFT                               0xa
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF107__SHIFT                               0xb
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF108__SHIFT                               0xc
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF109__SHIFT                               0xd
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF110__SHIFT                               0xe
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF111__SHIFT                               0xf
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF112__SHIFT                               0x10
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF113__SHIFT                               0x11
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF114__SHIFT                               0x12
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF115__SHIFT                               0x13
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF116__SHIFT                               0x14
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF117__SHIFT                               0x15
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF118__SHIFT                               0x16
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF119__SHIFT                               0x17
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF120__SHIFT                               0x18
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF121__SHIFT                               0x19
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF122__SHIFT                               0x1a
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF123__SHIFT                               0x1b
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF124__SHIFT                               0x1c
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF125__SHIFT                               0x1d
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF126__SHIFT                               0x1e
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF127__SHIFT                               0x1f
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF96_MASK                                  0x00000001L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF97_MASK                                  0x00000002L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF98_MASK                                  0x00000004L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF99_MASK                                  0x00000008L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF100_MASK                                 0x00000010L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF101_MASK                                 0x00000020L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF102_MASK                                 0x00000040L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF103_MASK                                 0x00000080L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF104_MASK                                 0x00000100L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF105_MASK                                 0x00000200L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF106_MASK                                 0x00000400L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF107_MASK                                 0x00000800L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF108_MASK                                 0x00001000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF109_MASK                                 0x00002000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF110_MASK                                 0x00004000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF111_MASK                                 0x00008000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF112_MASK                                 0x00010000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF113_MASK                                 0x00020000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF114_MASK                                 0x00040000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF115_MASK                                 0x00080000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF116_MASK                                 0x00100000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF117_MASK                                 0x00200000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF118_MASK                                 0x00400000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF119_MASK                                 0x00800000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF120_MASK                                 0x01000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF121_MASK                                 0x02000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF122_MASK                                 0x04000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF123_MASK                                 0x08000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF124_MASK                                 0x10000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF125_MASK                                 0x20000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF126_MASK                                 0x40000000L
+#define BIF_MISC_PFVF_DEV0_EPF1_BIF_VF_UR_ATOMIC_NR3__UR_ATOMIC_NR_VF127_MASK                                 0x80000000L
+
+
+// addressBlock: nbif0_nbif0_bif_trap_bif_trap_regblk
+//NBIF_TRAP_NP_STATUS
+#define NBIF_TRAP_NP_STATUS__NBIF_TRAP_NP_TrapReqValid__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_STATUS__NBIF_TRAP_NP_TrapNumber__SHIFT                                                   0x8
+#define NBIF_TRAP_NP_STATUS__NBIF_TRAP_NP_TrapS2Vld__SHIFT                                                    0xc
+#define NBIF_TRAP_NP_STATUS__NBIF_TRAP_NP_TrapS2Number__SHIFT                                                 0x10
+#define NBIF_TRAP_NP_STATUS__NBIF_TRAP_NP_TrapReqValid_MASK                                                   0x00000001L
+#define NBIF_TRAP_NP_STATUS__NBIF_TRAP_NP_TrapNumber_MASK                                                     0x00000F00L
+#define NBIF_TRAP_NP_STATUS__NBIF_TRAP_NP_TrapS2Vld_MASK                                                      0x00001000L
+#define NBIF_TRAP_NP_STATUS__NBIF_TRAP_NP_TrapS2Number_MASK                                                   0x01FF0000L
+//NBIF_TRAP_NP_REQUEST0
+#define NBIF_TRAP_NP_REQUEST0__NBIF_TRAP_NP_TrapReqAddrLo__SHIFT                                              0x2
+#define NBIF_TRAP_NP_REQUEST0__NBIF_TRAP_NP_TrapReqAddrLo_MASK                                                0xFFFFFFFCL
+//NBIF_TRAP_NP_REQUEST1
+#define NBIF_TRAP_NP_REQUEST1__NBIF_TRAP_NP_TrapReqAddrHi__SHIFT                                              0x0
+#define NBIF_TRAP_NP_REQUEST1__NBIF_TRAP_NP_TrapReqAddrHi_MASK                                                0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST2
+#define NBIF_TRAP_NP_REQUEST2__NBIF_TRAP_NP_TrapReqCmd__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_REQUEST2__NBIF_TRAP_NP_TrapReqAttr__SHIFT                                                0x8
+#define NBIF_TRAP_NP_REQUEST2__NBIF_TRAP_NP_TrapReqLen__SHIFT                                                 0x10
+#define NBIF_TRAP_NP_REQUEST2__NBIF_TRAP_NP_TrapReqCmd_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_REQUEST2__NBIF_TRAP_NP_TrapReqAttr_MASK                                                  0x0000FF00L
+#define NBIF_TRAP_NP_REQUEST2__NBIF_TRAP_NP_TrapReqLen_MASK                                                   0x003F0000L
+//NBIF_TRAP_NP_REQUEST3
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqVC__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqBlockLevel__SHIFT                                          0x4
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqChain__SHIFT                                               0x6
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqIO__SHIFT                                                  0x7
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqPassPW__SHIFT                                              0x8
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqRspPassPW__SHIFT                                           0x9
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqUnitID__SHIFT                                              0x10
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqVC_MASK                                                    0x00000007L
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqBlockLevel_MASK                                            0x00000030L
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqChain_MASK                                                 0x00000040L
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqIO_MASK                                                    0x00000080L
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqPassPW_MASK                                                0x00000100L
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqRspPassPW_MASK                                             0x00000200L
+#define NBIF_TRAP_NP_REQUEST3__NBIF_TRAP_NP_TrapReqUnitID_MASK                                                0x07FF0000L
+//NBIF_TRAP_NP_REQUEST4
+#define NBIF_TRAP_NP_REQUEST4__NBIF_TRAP_NP_TrapReqSecLevel__SHIFT                                            0x0
+#define NBIF_TRAP_NP_REQUEST4__NBIF_TRAP_NP_TrapReqSecLevel_MASK                                              0x0000000FL
+//NBIF_TRAP_NP_REQUEST5
+#define NBIF_TRAP_NP_REQUEST5__NBIF_TRAP_NP_TrapReqDataVC__SHIFT                                              0x0
+#define NBIF_TRAP_NP_REQUEST5__NBIF_TRAP_NP_TrapReqDataErr__SHIFT                                             0x4
+#define NBIF_TRAP_NP_REQUEST5__NBIF_TRAP_NP_TrapReqDataVC_MASK                                                0x00000007L
+#define NBIF_TRAP_NP_REQUEST5__NBIF_TRAP_NP_TrapReqDataErr_MASK                                               0x000000F0L
+//NBIF_TRAP_NP_REQUEST6
+#define NBIF_TRAP_NP_REQUEST6__NBIF_TRAP_NP_TrapReqDataParity__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST6__NBIF_TRAP_NP_TrapReqDataParity_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATASTRB0
+#define NBIF_TRAP_NP_REQUEST_DATASTRB0__NBIF_TRAP_NP_TrapReqDataBytEn0__SHIFT                                 0x0
+#define NBIF_TRAP_NP_REQUEST_DATASTRB0__NBIF_TRAP_NP_TrapReqDataBytEn0_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATASTRB1
+#define NBIF_TRAP_NP_REQUEST_DATASTRB1__NBIF_TRAP_NP_TrapReqDataBytEn1__SHIFT                                 0x0
+#define NBIF_TRAP_NP_REQUEST_DATASTRB1__NBIF_TRAP_NP_TrapReqDataBytEn1_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATASTRB2
+#define NBIF_TRAP_NP_REQUEST_DATASTRB2__NBIF_TRAP_NP_TrapReqDataBytEn2__SHIFT                                 0x0
+#define NBIF_TRAP_NP_REQUEST_DATASTRB2__NBIF_TRAP_NP_TrapReqDataBytEn2_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATASTRB3
+#define NBIF_TRAP_NP_REQUEST_DATASTRB3__NBIF_TRAP_NP_TrapReqDataBytEn3__SHIFT                                 0x0
+#define NBIF_TRAP_NP_REQUEST_DATASTRB3__NBIF_TRAP_NP_TrapReqDataBytEn3_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATASTRB4
+#define NBIF_TRAP_NP_REQUEST_DATASTRB4__NBIF_TRAP_NP_TrapReqDataBytEn4__SHIFT                                 0x0
+#define NBIF_TRAP_NP_REQUEST_DATASTRB4__NBIF_TRAP_NP_TrapReqDataBytEn4_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATASTRB5
+#define NBIF_TRAP_NP_REQUEST_DATASTRB5__NBIF_TRAP_NP_TrapReqDataBytEn5__SHIFT                                 0x0
+#define NBIF_TRAP_NP_REQUEST_DATASTRB5__NBIF_TRAP_NP_TrapReqDataBytEn5_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATASTRB6
+#define NBIF_TRAP_NP_REQUEST_DATASTRB6__NBIF_TRAP_NP_TrapReqDataBytEn6__SHIFT                                 0x0
+#define NBIF_TRAP_NP_REQUEST_DATASTRB6__NBIF_TRAP_NP_TrapReqDataBytEn6_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATASTRB7
+#define NBIF_TRAP_NP_REQUEST_DATASTRB7__NBIF_TRAP_NP_TrapReqDataBytEn7__SHIFT                                 0x0
+#define NBIF_TRAP_NP_REQUEST_DATASTRB7__NBIF_TRAP_NP_TrapReqDataBytEn7_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA0
+#define NBIF_TRAP_NP_REQUEST_DATA0__NBIF_TRAP_NP_TrapReqData0__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA0__NBIF_TRAP_NP_TrapReqData0_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA1
+#define NBIF_TRAP_NP_REQUEST_DATA1__NBIF_TRAP_NP_TrapReqData1__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA1__NBIF_TRAP_NP_TrapReqData1_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA2
+#define NBIF_TRAP_NP_REQUEST_DATA2__NBIF_TRAP_NP_TrapReqData2__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA2__NBIF_TRAP_NP_TrapReqData2_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA3
+#define NBIF_TRAP_NP_REQUEST_DATA3__NBIF_TRAP_NP_TrapReqData3__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA3__NBIF_TRAP_NP_TrapReqData3_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA4
+#define NBIF_TRAP_NP_REQUEST_DATA4__NBIF_TRAP_NP_TrapReqData4__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA4__NBIF_TRAP_NP_TrapReqData4_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA5
+#define NBIF_TRAP_NP_REQUEST_DATA5__NBIF_TRAP_NP_TrapReqData5__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA5__NBIF_TRAP_NP_TrapReqData5_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA6
+#define NBIF_TRAP_NP_REQUEST_DATA6__NBIF_TRAP_NP_TrapReqData6__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA6__NBIF_TRAP_NP_TrapReqData6_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA7
+#define NBIF_TRAP_NP_REQUEST_DATA7__NBIF_TRAP_NP_TrapReqData7__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA7__NBIF_TRAP_NP_TrapReqData7_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA8
+#define NBIF_TRAP_NP_REQUEST_DATA8__NBIF_TRAP_NP_TrapReqData8__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA8__NBIF_TRAP_NP_TrapReqData8_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA9
+#define NBIF_TRAP_NP_REQUEST_DATA9__NBIF_TRAP_NP_TrapReqData9__SHIFT                                          0x0
+#define NBIF_TRAP_NP_REQUEST_DATA9__NBIF_TRAP_NP_TrapReqData9_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA10
+#define NBIF_TRAP_NP_REQUEST_DATA10__NBIF_TRAP_NP_TrapReqData10__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA10__NBIF_TRAP_NP_TrapReqData10_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA11
+#define NBIF_TRAP_NP_REQUEST_DATA11__NBIF_TRAP_NP_TrapReqData11__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA11__NBIF_TRAP_NP_TrapReqData11_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA12
+#define NBIF_TRAP_NP_REQUEST_DATA12__NBIF_TRAP_NP_TrapReqData12__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA12__NBIF_TRAP_NP_TrapReqData12_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA13
+#define NBIF_TRAP_NP_REQUEST_DATA13__NBIF_TRAP_NP_TrapReqData13__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA13__NBIF_TRAP_NP_TrapReqData13_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA14
+#define NBIF_TRAP_NP_REQUEST_DATA14__NBIF_TRAP_NP_TrapReqData14__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA14__NBIF_TRAP_NP_TrapReqData14_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA15
+#define NBIF_TRAP_NP_REQUEST_DATA15__NBIF_TRAP_NP_TrapReqData15__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA15__NBIF_TRAP_NP_TrapReqData15_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA16
+#define NBIF_TRAP_NP_REQUEST_DATA16__NBIF_TRAP_NP_TrapReqData16__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA16__NBIF_TRAP_NP_TrapReqData16_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA17
+#define NBIF_TRAP_NP_REQUEST_DATA17__NBIF_TRAP_NP_TrapReqData17__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA17__NBIF_TRAP_NP_TrapReqData17_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA18
+#define NBIF_TRAP_NP_REQUEST_DATA18__NBIF_TRAP_NP_TrapReqData18__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA18__NBIF_TRAP_NP_TrapReqData18_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA19
+#define NBIF_TRAP_NP_REQUEST_DATA19__NBIF_TRAP_NP_TrapReqData19__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA19__NBIF_TRAP_NP_TrapReqData19_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA20
+#define NBIF_TRAP_NP_REQUEST_DATA20__NBIF_TRAP_NP_TrapReqData20__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA20__NBIF_TRAP_NP_TrapReqData20_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA21
+#define NBIF_TRAP_NP_REQUEST_DATA21__NBIF_TRAP_NP_TrapReqData21__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA21__NBIF_TRAP_NP_TrapReqData21_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA22
+#define NBIF_TRAP_NP_REQUEST_DATA22__NBIF_TRAP_NP_TrapReqData22__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA22__NBIF_TRAP_NP_TrapReqData22_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA23
+#define NBIF_TRAP_NP_REQUEST_DATA23__NBIF_TRAP_NP_TrapReqData23__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA23__NBIF_TRAP_NP_TrapReqData23_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA24
+#define NBIF_TRAP_NP_REQUEST_DATA24__NBIF_TRAP_NP_TrapReqData24__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA24__NBIF_TRAP_NP_TrapReqData24_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA25
+#define NBIF_TRAP_NP_REQUEST_DATA25__NBIF_TRAP_NP_TrapReqData25__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA25__NBIF_TRAP_NP_TrapReqData25_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA26
+#define NBIF_TRAP_NP_REQUEST_DATA26__NBIF_TRAP_NP_TrapReqData26__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA26__NBIF_TRAP_NP_TrapReqData26_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA27
+#define NBIF_TRAP_NP_REQUEST_DATA27__NBIF_TRAP_NP_TrapReqData27__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA27__NBIF_TRAP_NP_TrapReqData27_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA28
+#define NBIF_TRAP_NP_REQUEST_DATA28__NBIF_TRAP_NP_TrapReqData28__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA28__NBIF_TRAP_NP_TrapReqData28_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA29
+#define NBIF_TRAP_NP_REQUEST_DATA29__NBIF_TRAP_NP_TrapReqData29__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA29__NBIF_TRAP_NP_TrapReqData29_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA30
+#define NBIF_TRAP_NP_REQUEST_DATA30__NBIF_TRAP_NP_TrapReqData30__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA30__NBIF_TRAP_NP_TrapReqData30_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA31
+#define NBIF_TRAP_NP_REQUEST_DATA31__NBIF_TRAP_NP_TrapReqData31__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA31__NBIF_TRAP_NP_TrapReqData31_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA32
+#define NBIF_TRAP_NP_REQUEST_DATA32__NBIF_TRAP_NP_TrapReqData32__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA32__NBIF_TRAP_NP_TrapReqData32_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA33
+#define NBIF_TRAP_NP_REQUEST_DATA33__NBIF_TRAP_NP_TrapReqData33__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA33__NBIF_TRAP_NP_TrapReqData33_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA34
+#define NBIF_TRAP_NP_REQUEST_DATA34__NBIF_TRAP_NP_TrapReqData34__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA34__NBIF_TRAP_NP_TrapReqData34_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA35
+#define NBIF_TRAP_NP_REQUEST_DATA35__NBIF_TRAP_NP_TrapReqData35__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA35__NBIF_TRAP_NP_TrapReqData35_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA36
+#define NBIF_TRAP_NP_REQUEST_DATA36__NBIF_TRAP_NP_TrapReqData36__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA36__NBIF_TRAP_NP_TrapReqData36_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA37
+#define NBIF_TRAP_NP_REQUEST_DATA37__NBIF_TRAP_NP_TrapReqData37__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA37__NBIF_TRAP_NP_TrapReqData37_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA38
+#define NBIF_TRAP_NP_REQUEST_DATA38__NBIF_TRAP_NP_TrapReqData38__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA38__NBIF_TRAP_NP_TrapReqData38_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA39
+#define NBIF_TRAP_NP_REQUEST_DATA39__NBIF_TRAP_NP_TrapReqData39__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA39__NBIF_TRAP_NP_TrapReqData39_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA40
+#define NBIF_TRAP_NP_REQUEST_DATA40__NBIF_TRAP_NP_TrapReqData40__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA40__NBIF_TRAP_NP_TrapReqData40_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA41
+#define NBIF_TRAP_NP_REQUEST_DATA41__NBIF_TRAP_NP_TrapReqData41__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA41__NBIF_TRAP_NP_TrapReqData41_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA42
+#define NBIF_TRAP_NP_REQUEST_DATA42__NBIF_TRAP_NP_TrapReqData42__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA42__NBIF_TRAP_NP_TrapReqData42_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA43
+#define NBIF_TRAP_NP_REQUEST_DATA43__NBIF_TRAP_NP_TrapReqData43__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA43__NBIF_TRAP_NP_TrapReqData43_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA44
+#define NBIF_TRAP_NP_REQUEST_DATA44__NBIF_TRAP_NP_TrapReqData44__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA44__NBIF_TRAP_NP_TrapReqData44_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA45
+#define NBIF_TRAP_NP_REQUEST_DATA45__NBIF_TRAP_NP_TrapReqData45__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA45__NBIF_TRAP_NP_TrapReqData45_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA46
+#define NBIF_TRAP_NP_REQUEST_DATA46__NBIF_TRAP_NP_TrapReqData46__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA46__NBIF_TRAP_NP_TrapReqData46_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA47
+#define NBIF_TRAP_NP_REQUEST_DATA47__NBIF_TRAP_NP_TrapReqData47__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA47__NBIF_TRAP_NP_TrapReqData47_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA48
+#define NBIF_TRAP_NP_REQUEST_DATA48__NBIF_TRAP_NP_TrapReqData48__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA48__NBIF_TRAP_NP_TrapReqData48_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA49
+#define NBIF_TRAP_NP_REQUEST_DATA49__NBIF_TRAP_NP_TrapReqData49__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA49__NBIF_TRAP_NP_TrapReqData49_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA50
+#define NBIF_TRAP_NP_REQUEST_DATA50__NBIF_TRAP_NP_TrapReqData50__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA50__NBIF_TRAP_NP_TrapReqData50_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA51
+#define NBIF_TRAP_NP_REQUEST_DATA51__NBIF_TRAP_NP_TrapReqData51__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA51__NBIF_TRAP_NP_TrapReqData51_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA52
+#define NBIF_TRAP_NP_REQUEST_DATA52__NBIF_TRAP_NP_TrapReqData52__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA52__NBIF_TRAP_NP_TrapReqData52_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA53
+#define NBIF_TRAP_NP_REQUEST_DATA53__NBIF_TRAP_NP_TrapReqData53__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA53__NBIF_TRAP_NP_TrapReqData53_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA54
+#define NBIF_TRAP_NP_REQUEST_DATA54__NBIF_TRAP_NP_TrapReqData54__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA54__NBIF_TRAP_NP_TrapReqData54_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA55
+#define NBIF_TRAP_NP_REQUEST_DATA55__NBIF_TRAP_NP_TrapReqData55__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA55__NBIF_TRAP_NP_TrapReqData55_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA56
+#define NBIF_TRAP_NP_REQUEST_DATA56__NBIF_TRAP_NP_TrapReqData56__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA56__NBIF_TRAP_NP_TrapReqData56_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA57
+#define NBIF_TRAP_NP_REQUEST_DATA57__NBIF_TRAP_NP_TrapReqData57__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA57__NBIF_TRAP_NP_TrapReqData57_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA58
+#define NBIF_TRAP_NP_REQUEST_DATA58__NBIF_TRAP_NP_TrapReqData58__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA58__NBIF_TRAP_NP_TrapReqData58_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA59
+#define NBIF_TRAP_NP_REQUEST_DATA59__NBIF_TRAP_NP_TrapReqData59__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA59__NBIF_TRAP_NP_TrapReqData59_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA60
+#define NBIF_TRAP_NP_REQUEST_DATA60__NBIF_TRAP_NP_TrapReqData60__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA60__NBIF_TRAP_NP_TrapReqData60_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA61
+#define NBIF_TRAP_NP_REQUEST_DATA61__NBIF_TRAP_NP_TrapReqData61__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA61__NBIF_TRAP_NP_TrapReqData61_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA62
+#define NBIF_TRAP_NP_REQUEST_DATA62__NBIF_TRAP_NP_TrapReqData62__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA62__NBIF_TRAP_NP_TrapReqData62_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_REQUEST_DATA63
+#define NBIF_TRAP_NP_REQUEST_DATA63__NBIF_TRAP_NP_TrapReqData63__SHIFT                                        0x0
+#define NBIF_TRAP_NP_REQUEST_DATA63__NBIF_TRAP_NP_TrapReqData63_MASK                                          0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_CONTROL
+#define NBIF_TRAP_NP_RESPONSE_CONTROL__NBIF_TRAP_NP_TrapRspTrigger__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_CONTROL__NBIF_TRAP_NP_TrapRspReqPassthru__SHIFT                                 0x1
+#define NBIF_TRAP_NP_RESPONSE_CONTROL__NBIF_TRAP_NP_TrapRspTrigger_MASK                                       0x00000001L
+#define NBIF_TRAP_NP_RESPONSE_CONTROL__NBIF_TRAP_NP_TrapRspReqPassthru_MASK                                   0x00000002L
+//NBIF_TRAP_NP_RESPONSE0
+#define NBIF_TRAP_NP_RESPONSE0__NBIF_TRAP_NP_TrapRspPassPW__SHIFT                                             0x0
+#define NBIF_TRAP_NP_RESPONSE0__NBIF_TRAP_NP_TrapRspStatus__SHIFT                                             0x4
+#define NBIF_TRAP_NP_RESPONSE0__NBIF_TRAP_NP_TrapRspDataStatus__SHIFT                                         0x10
+#define NBIF_TRAP_NP_RESPONSE0__NBIF_TRAP_NP_TrapRspPassPW_MASK                                               0x00000001L
+#define NBIF_TRAP_NP_RESPONSE0__NBIF_TRAP_NP_TrapRspStatus_MASK                                               0x000000F0L
+#define NBIF_TRAP_NP_RESPONSE0__NBIF_TRAP_NP_TrapRspDataStatus_MASK                                           0x00FF0000L
+//NBIF_TRAP_NP_RESPONSE1
+#define NBIF_TRAP_NP_RESPONSE1__NBIF_TRAP_NP_TrapRspCplid__SHIFT                                              0x0
+#define NBIF_TRAP_NP_RESPONSE1__NBIF_TRAP_NP_TrapRspSrcdata__SHIFT                                            0x10
+#define NBIF_TRAP_NP_RESPONSE1__NBIF_TRAP_NP_TrapRspCplid_MASK                                                0x0000FFFFL
+#define NBIF_TRAP_NP_RESPONSE1__NBIF_TRAP_NP_TrapRspSrcdata_MASK                                              0x003F0000L
+//NBIF_TRAP_NP_RESPONSE2
+#define NBIF_TRAP_NP_RESPONSE2__NBIF_TRAP_NP_TrapRspUser0__SHIFT                                              0x0
+#define NBIF_TRAP_NP_RESPONSE2__NBIF_TRAP_NP_TrapRspUser0_MASK                                                0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE3
+#define NBIF_TRAP_NP_RESPONSE3__NBIF_TRAP_NP_TrapRspUser1__SHIFT                                              0x0
+#define NBIF_TRAP_NP_RESPONSE3__NBIF_TRAP_NP_TrapRspUser1_MASK                                                0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE4
+#define NBIF_TRAP_NP_RESPONSE4__NBIF_TRAP_NP_TrapRspUser2__SHIFT                                              0x0
+#define NBIF_TRAP_NP_RESPONSE4__NBIF_TRAP_NP_TrapRspUser2_MASK                                                0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE5
+#define NBIF_TRAP_NP_RESPONSE5__NBIF_TRAP_NP_TrapRspUser3__SHIFT                                              0x0
+#define NBIF_TRAP_NP_RESPONSE5__NBIF_TRAP_NP_TrapRspUser3_MASK                                                0x00000007L
+//NBIF_TRAP_NP_RESPONSE_DATA0
+#define NBIF_TRAP_NP_RESPONSE_DATA0__NBIF_TRAP_NP_TrapRdRspData0__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA0__NBIF_TRAP_NP_TrapRdRspData0_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA1
+#define NBIF_TRAP_NP_RESPONSE_DATA1__NBIF_TRAP_NP_TrapRdRspData1__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA1__NBIF_TRAP_NP_TrapRdRspData1_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA2
+#define NBIF_TRAP_NP_RESPONSE_DATA2__NBIF_TRAP_NP_TrapRdRspData2__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA2__NBIF_TRAP_NP_TrapRdRspData2_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA3
+#define NBIF_TRAP_NP_RESPONSE_DATA3__NBIF_TRAP_NP_TrapRdRspData3__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA3__NBIF_TRAP_NP_TrapRdRspData3_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA4
+#define NBIF_TRAP_NP_RESPONSE_DATA4__NBIF_TRAP_NP_TrapRdRspData4__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA4__NBIF_TRAP_NP_TrapRdRspData4_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA5
+#define NBIF_TRAP_NP_RESPONSE_DATA5__NBIF_TRAP_NP_TrapRdRspData5__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA5__NBIF_TRAP_NP_TrapRdRspData5_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA6
+#define NBIF_TRAP_NP_RESPONSE_DATA6__NBIF_TRAP_NP_TrapRdRspData6__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA6__NBIF_TRAP_NP_TrapRdRspData6_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA7
+#define NBIF_TRAP_NP_RESPONSE_DATA7__NBIF_TRAP_NP_TrapRdRspData7__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA7__NBIF_TRAP_NP_TrapRdRspData7_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA8
+#define NBIF_TRAP_NP_RESPONSE_DATA8__NBIF_TRAP_NP_TrapRdRspData8__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA8__NBIF_TRAP_NP_TrapRdRspData8_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA9
+#define NBIF_TRAP_NP_RESPONSE_DATA9__NBIF_TRAP_NP_TrapRdRspData9__SHIFT                                       0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA9__NBIF_TRAP_NP_TrapRdRspData9_MASK                                         0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA10
+#define NBIF_TRAP_NP_RESPONSE_DATA10__NBIF_TRAP_NP_TrapRdRspData10__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA10__NBIF_TRAP_NP_TrapRdRspData10_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA11
+#define NBIF_TRAP_NP_RESPONSE_DATA11__NBIF_TRAP_NP_TrapRdRspData11__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA11__NBIF_TRAP_NP_TrapRdRspData11_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA12
+#define NBIF_TRAP_NP_RESPONSE_DATA12__NBIF_TRAP_NP_TrapRdRspData12__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA12__NBIF_TRAP_NP_TrapRdRspData12_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA13
+#define NBIF_TRAP_NP_RESPONSE_DATA13__NBIF_TRAP_NP_TrapRdRspData13__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA13__NBIF_TRAP_NP_TrapRdRspData13_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA14
+#define NBIF_TRAP_NP_RESPONSE_DATA14__NBIF_TRAP_NP_TrapRdRspData14__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA14__NBIF_TRAP_NP_TrapRdRspData14_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA15
+#define NBIF_TRAP_NP_RESPONSE_DATA15__NBIF_TRAP_NP_TrapRdRspData15__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA15__NBIF_TRAP_NP_TrapRdRspData15_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA16
+#define NBIF_TRAP_NP_RESPONSE_DATA16__NBIF_TRAP_NP_TrapRdRspData16__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA16__NBIF_TRAP_NP_TrapRdRspData16_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA17
+#define NBIF_TRAP_NP_RESPONSE_DATA17__NBIF_TRAP_NP_TrapRdRspData17__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA17__NBIF_TRAP_NP_TrapRdRspData17_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA18
+#define NBIF_TRAP_NP_RESPONSE_DATA18__NBIF_TRAP_NP_TrapRdRspData18__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA18__NBIF_TRAP_NP_TrapRdRspData18_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA19
+#define NBIF_TRAP_NP_RESPONSE_DATA19__NBIF_TRAP_NP_TrapRdRspData19__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA19__NBIF_TRAP_NP_TrapRdRspData19_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA20
+#define NBIF_TRAP_NP_RESPONSE_DATA20__NBIF_TRAP_NP_TrapRdRspData20__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA20__NBIF_TRAP_NP_TrapRdRspData20_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA21
+#define NBIF_TRAP_NP_RESPONSE_DATA21__NBIF_TRAP_NP_TrapRdRspData21__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA21__NBIF_TRAP_NP_TrapRdRspData21_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA22
+#define NBIF_TRAP_NP_RESPONSE_DATA22__NBIF_TRAP_NP_TrapRdRspData22__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA22__NBIF_TRAP_NP_TrapRdRspData22_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA23
+#define NBIF_TRAP_NP_RESPONSE_DATA23__NBIF_TRAP_NP_TrapRdRspData23__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA23__NBIF_TRAP_NP_TrapRdRspData23_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA24
+#define NBIF_TRAP_NP_RESPONSE_DATA24__NBIF_TRAP_NP_TrapRdRspData24__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA24__NBIF_TRAP_NP_TrapRdRspData24_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA25
+#define NBIF_TRAP_NP_RESPONSE_DATA25__NBIF_TRAP_NP_TrapRdRspData25__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA25__NBIF_TRAP_NP_TrapRdRspData25_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA26
+#define NBIF_TRAP_NP_RESPONSE_DATA26__NBIF_TRAP_NP_TrapRdRspData26__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA26__NBIF_TRAP_NP_TrapRdRspData26_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA27
+#define NBIF_TRAP_NP_RESPONSE_DATA27__NBIF_TRAP_NP_TrapRdRspData27__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA27__NBIF_TRAP_NP_TrapRdRspData27_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA28
+#define NBIF_TRAP_NP_RESPONSE_DATA28__NBIF_TRAP_NP_TrapRdRspData28__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA28__NBIF_TRAP_NP_TrapRdRspData28_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA29
+#define NBIF_TRAP_NP_RESPONSE_DATA29__NBIF_TRAP_NP_TrapRdRspData29__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA29__NBIF_TRAP_NP_TrapRdRspData29_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA30
+#define NBIF_TRAP_NP_RESPONSE_DATA30__NBIF_TRAP_NP_TrapRdRspData30__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA30__NBIF_TRAP_NP_TrapRdRspData30_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA31
+#define NBIF_TRAP_NP_RESPONSE_DATA31__NBIF_TRAP_NP_TrapRdRspData31__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA31__NBIF_TRAP_NP_TrapRdRspData31_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA32
+#define NBIF_TRAP_NP_RESPONSE_DATA32__NBIF_TRAP_NP_TrapRdRspData32__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA32__NBIF_TRAP_NP_TrapRdRspData32_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA33
+#define NBIF_TRAP_NP_RESPONSE_DATA33__NBIF_TRAP_NP_TrapRdRspData33__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA33__NBIF_TRAP_NP_TrapRdRspData33_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA34
+#define NBIF_TRAP_NP_RESPONSE_DATA34__NBIF_TRAP_NP_TrapRdRspData34__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA34__NBIF_TRAP_NP_TrapRdRspData34_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA35
+#define NBIF_TRAP_NP_RESPONSE_DATA35__NBIF_TRAP_NP_TrapRdRspData35__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA35__NBIF_TRAP_NP_TrapRdRspData35_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA36
+#define NBIF_TRAP_NP_RESPONSE_DATA36__NBIF_TRAP_NP_TrapRdRspData36__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA36__NBIF_TRAP_NP_TrapRdRspData36_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA37
+#define NBIF_TRAP_NP_RESPONSE_DATA37__NBIF_TRAP_NP_TrapRdRspData37__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA37__NBIF_TRAP_NP_TrapRdRspData37_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA38
+#define NBIF_TRAP_NP_RESPONSE_DATA38__NBIF_TRAP_NP_TrapRdRspData38__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA38__NBIF_TRAP_NP_TrapRdRspData38_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA39
+#define NBIF_TRAP_NP_RESPONSE_DATA39__NBIF_TRAP_NP_TrapRdRspData39__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA39__NBIF_TRAP_NP_TrapRdRspData39_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA40
+#define NBIF_TRAP_NP_RESPONSE_DATA40__NBIF_TRAP_NP_TrapRdRspData40__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA40__NBIF_TRAP_NP_TrapRdRspData40_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA41
+#define NBIF_TRAP_NP_RESPONSE_DATA41__NBIF_TRAP_NP_TrapRdRspData41__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA41__NBIF_TRAP_NP_TrapRdRspData41_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA42
+#define NBIF_TRAP_NP_RESPONSE_DATA42__NBIF_TRAP_NP_TrapRdRspData42__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA42__NBIF_TRAP_NP_TrapRdRspData42_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA43
+#define NBIF_TRAP_NP_RESPONSE_DATA43__NBIF_TRAP_NP_TrapRdRspData43__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA43__NBIF_TRAP_NP_TrapRdRspData43_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA44
+#define NBIF_TRAP_NP_RESPONSE_DATA44__NBIF_TRAP_NP_TrapRdRspData44__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA44__NBIF_TRAP_NP_TrapRdRspData44_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA45
+#define NBIF_TRAP_NP_RESPONSE_DATA45__NBIF_TRAP_NP_TrapRdRspData45__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA45__NBIF_TRAP_NP_TrapRdRspData45_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA46
+#define NBIF_TRAP_NP_RESPONSE_DATA46__NBIF_TRAP_NP_TrapRdRspData46__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA46__NBIF_TRAP_NP_TrapRdRspData46_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA47
+#define NBIF_TRAP_NP_RESPONSE_DATA47__NBIF_TRAP_NP_TrapRdRspData47__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA47__NBIF_TRAP_NP_TrapRdRspData47_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA48
+#define NBIF_TRAP_NP_RESPONSE_DATA48__NBIF_TRAP_NP_TrapRdRspData48__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA48__NBIF_TRAP_NP_TrapRdRspData48_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA49
+#define NBIF_TRAP_NP_RESPONSE_DATA49__NBIF_TRAP_NP_TrapRdRspData49__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA49__NBIF_TRAP_NP_TrapRdRspData49_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA50
+#define NBIF_TRAP_NP_RESPONSE_DATA50__NBIF_TRAP_NP_TrapRdRspData50__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA50__NBIF_TRAP_NP_TrapRdRspData50_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA51
+#define NBIF_TRAP_NP_RESPONSE_DATA51__NBIF_TRAP_NP_TrapRdRspData51__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA51__NBIF_TRAP_NP_TrapRdRspData51_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA52
+#define NBIF_TRAP_NP_RESPONSE_DATA52__NBIF_TRAP_NP_TrapRdRspData52__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA52__NBIF_TRAP_NP_TrapRdRspData52_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA53
+#define NBIF_TRAP_NP_RESPONSE_DATA53__NBIF_TRAP_NP_TrapRdRspData53__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA53__NBIF_TRAP_NP_TrapRdRspData53_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA54
+#define NBIF_TRAP_NP_RESPONSE_DATA54__NBIF_TRAP_NP_TrapRdRspData54__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA54__NBIF_TRAP_NP_TrapRdRspData54_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA55
+#define NBIF_TRAP_NP_RESPONSE_DATA55__NBIF_TRAP_NP_TrapRdRspData55__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA55__NBIF_TRAP_NP_TrapRdRspData55_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA56
+#define NBIF_TRAP_NP_RESPONSE_DATA56__NBIF_TRAP_NP_TrapRdRspData56__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA56__NBIF_TRAP_NP_TrapRdRspData56_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA57
+#define NBIF_TRAP_NP_RESPONSE_DATA57__NBIF_TRAP_NP_TrapRdRspData57__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA57__NBIF_TRAP_NP_TrapRdRspData57_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA58
+#define NBIF_TRAP_NP_RESPONSE_DATA58__NBIF_TRAP_NP_TrapRdRspData58__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA58__NBIF_TRAP_NP_TrapRdRspData58_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA59
+#define NBIF_TRAP_NP_RESPONSE_DATA59__NBIF_TRAP_NP_TrapRdRspData59__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA59__NBIF_TRAP_NP_TrapRdRspData59_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA60
+#define NBIF_TRAP_NP_RESPONSE_DATA60__NBIF_TRAP_NP_TrapRdRspData60__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA60__NBIF_TRAP_NP_TrapRdRspData60_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA61
+#define NBIF_TRAP_NP_RESPONSE_DATA61__NBIF_TRAP_NP_TrapRdRspData61__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA61__NBIF_TRAP_NP_TrapRdRspData61_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA62
+#define NBIF_TRAP_NP_RESPONSE_DATA62__NBIF_TRAP_NP_TrapRdRspData62__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA62__NBIF_TRAP_NP_TrapRdRspData62_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_RESPONSE_DATA63
+#define NBIF_TRAP_NP_RESPONSE_DATA63__NBIF_TRAP_NP_TrapRdRspData63__SHIFT                                     0x0
+#define NBIF_TRAP_NP_RESPONSE_DATA63__NBIF_TRAP_NP_TrapRdRspData63_MASK                                       0xFFFFFFFFL
+//NBIF_TRAP_NP_0_CONTROL0
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_0_CONTROL0__NBIF_TRAP_NP_Trap0Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_0_ADDRESS_LO
+#define NBIF_TRAP_NP_0_ADDRESS_LO__NBIF_TRAP_NP_Trap0AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_0_ADDRESS_LO__NBIF_TRAP_NP_Trap0AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_0_ADDRESS_HI
+#define NBIF_TRAP_NP_0_ADDRESS_HI__NBIF_TRAP_NP_Trap0AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_0_ADDRESS_HI__NBIF_TRAP_NP_Trap0AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_0_COMMAND
+#define NBIF_TRAP_NP_0_COMMAND__NBIF_TRAP_NP_Trap0Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_0_COMMAND__NBIF_TRAP_NP_Trap0Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_0_COMMAND__NBIF_TRAP_NP_Trap0Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_0_COMMAND__NBIF_TRAP_NP_Trap0Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_0_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_0_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap0AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_0_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap0AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_0_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_0_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap0AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_0_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap0AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_0_COMMAND_MASK
+#define NBIF_TRAP_NP_0_COMMAND_MASK__NBIF_TRAP_NP_Trap0Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_0_COMMAND_MASK__NBIF_TRAP_NP_Trap0Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_0_COMMAND_MASK__NBIF_TRAP_NP_Trap0Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_0_COMMAND_MASK__NBIF_TRAP_NP_Trap0Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_1_CONTROL0
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_1_CONTROL0__NBIF_TRAP_NP_Trap1Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_1_ADDRESS_LO
+#define NBIF_TRAP_NP_1_ADDRESS_LO__NBIF_TRAP_NP_Trap1AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_1_ADDRESS_LO__NBIF_TRAP_NP_Trap1AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_1_ADDRESS_HI
+#define NBIF_TRAP_NP_1_ADDRESS_HI__NBIF_TRAP_NP_Trap1AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_1_ADDRESS_HI__NBIF_TRAP_NP_Trap1AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_1_COMMAND
+#define NBIF_TRAP_NP_1_COMMAND__NBIF_TRAP_NP_Trap1Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_1_COMMAND__NBIF_TRAP_NP_Trap1Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_1_COMMAND__NBIF_TRAP_NP_Trap1Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_1_COMMAND__NBIF_TRAP_NP_Trap1Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_1_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_1_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap1AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_1_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap1AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_1_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_1_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap1AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_1_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap1AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_1_COMMAND_MASK
+#define NBIF_TRAP_NP_1_COMMAND_MASK__NBIF_TRAP_NP_Trap1Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_1_COMMAND_MASK__NBIF_TRAP_NP_Trap1Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_1_COMMAND_MASK__NBIF_TRAP_NP_Trap1Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_1_COMMAND_MASK__NBIF_TRAP_NP_Trap1Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_2_CONTROL0
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_2_CONTROL0__NBIF_TRAP_NP_Trap2Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_2_ADDRESS_LO
+#define NBIF_TRAP_NP_2_ADDRESS_LO__NBIF_TRAP_NP_Trap2AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_2_ADDRESS_LO__NBIF_TRAP_NP_Trap2AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_2_ADDRESS_HI
+#define NBIF_TRAP_NP_2_ADDRESS_HI__NBIF_TRAP_NP_Trap2AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_2_ADDRESS_HI__NBIF_TRAP_NP_Trap2AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_2_COMMAND
+#define NBIF_TRAP_NP_2_COMMAND__NBIF_TRAP_NP_Trap2Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_2_COMMAND__NBIF_TRAP_NP_Trap2Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_2_COMMAND__NBIF_TRAP_NP_Trap2Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_2_COMMAND__NBIF_TRAP_NP_Trap2Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_2_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_2_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap2AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_2_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap2AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_2_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_2_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap2AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_2_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap2AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_2_COMMAND_MASK
+#define NBIF_TRAP_NP_2_COMMAND_MASK__NBIF_TRAP_NP_Trap2Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_2_COMMAND_MASK__NBIF_TRAP_NP_Trap2Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_2_COMMAND_MASK__NBIF_TRAP_NP_Trap2Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_2_COMMAND_MASK__NBIF_TRAP_NP_Trap2Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_3_CONTROL0
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_3_CONTROL0__NBIF_TRAP_NP_Trap3Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_3_ADDRESS_LO
+#define NBIF_TRAP_NP_3_ADDRESS_LO__NBIF_TRAP_NP_Trap3AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_3_ADDRESS_LO__NBIF_TRAP_NP_Trap3AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_3_ADDRESS_HI
+#define NBIF_TRAP_NP_3_ADDRESS_HI__NBIF_TRAP_NP_Trap3AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_3_ADDRESS_HI__NBIF_TRAP_NP_Trap3AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_3_COMMAND
+#define NBIF_TRAP_NP_3_COMMAND__NBIF_TRAP_NP_Trap3Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_3_COMMAND__NBIF_TRAP_NP_Trap3Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_3_COMMAND__NBIF_TRAP_NP_Trap3Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_3_COMMAND__NBIF_TRAP_NP_Trap3Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_3_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_3_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap3AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_3_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap3AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_3_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_3_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap3AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_3_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap3AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_3_COMMAND_MASK
+#define NBIF_TRAP_NP_3_COMMAND_MASK__NBIF_TRAP_NP_Trap3Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_3_COMMAND_MASK__NBIF_TRAP_NP_Trap3Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_3_COMMAND_MASK__NBIF_TRAP_NP_Trap3Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_3_COMMAND_MASK__NBIF_TRAP_NP_Trap3Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_4_CONTROL0
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_4_CONTROL0__NBIF_TRAP_NP_Trap4Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_4_ADDRESS_LO
+#define NBIF_TRAP_NP_4_ADDRESS_LO__NBIF_TRAP_NP_Trap4AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_4_ADDRESS_LO__NBIF_TRAP_NP_Trap4AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_4_ADDRESS_HI
+#define NBIF_TRAP_NP_4_ADDRESS_HI__NBIF_TRAP_NP_Trap4AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_4_ADDRESS_HI__NBIF_TRAP_NP_Trap4AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_4_COMMAND
+#define NBIF_TRAP_NP_4_COMMAND__NBIF_TRAP_NP_Trap4Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_4_COMMAND__NBIF_TRAP_NP_Trap4Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_4_COMMAND__NBIF_TRAP_NP_Trap4Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_4_COMMAND__NBIF_TRAP_NP_Trap4Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_4_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_4_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap4AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_4_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap4AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_4_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_4_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap4AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_4_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap4AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_4_COMMAND_MASK
+#define NBIF_TRAP_NP_4_COMMAND_MASK__NBIF_TRAP_NP_Trap4Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_4_COMMAND_MASK__NBIF_TRAP_NP_Trap4Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_4_COMMAND_MASK__NBIF_TRAP_NP_Trap4Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_4_COMMAND_MASK__NBIF_TRAP_NP_Trap4Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_5_CONTROL0
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_5_CONTROL0__NBIF_TRAP_NP_Trap5Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_5_ADDRESS_LO
+#define NBIF_TRAP_NP_5_ADDRESS_LO__NBIF_TRAP_NP_Trap5AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_5_ADDRESS_LO__NBIF_TRAP_NP_Trap5AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_5_ADDRESS_HI
+#define NBIF_TRAP_NP_5_ADDRESS_HI__NBIF_TRAP_NP_Trap5AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_5_ADDRESS_HI__NBIF_TRAP_NP_Trap5AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_5_COMMAND
+#define NBIF_TRAP_NP_5_COMMAND__NBIF_TRAP_NP_Trap5Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_5_COMMAND__NBIF_TRAP_NP_Trap5Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_5_COMMAND__NBIF_TRAP_NP_Trap5Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_5_COMMAND__NBIF_TRAP_NP_Trap5Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_5_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_5_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap5AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_5_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap5AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_5_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_5_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap5AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_5_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap5AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_5_COMMAND_MASK
+#define NBIF_TRAP_NP_5_COMMAND_MASK__NBIF_TRAP_NP_Trap5Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_5_COMMAND_MASK__NBIF_TRAP_NP_Trap5Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_5_COMMAND_MASK__NBIF_TRAP_NP_Trap5Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_5_COMMAND_MASK__NBIF_TRAP_NP_Trap5Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_6_CONTROL0
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_6_CONTROL0__NBIF_TRAP_NP_Trap6Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_6_ADDRESS_LO
+#define NBIF_TRAP_NP_6_ADDRESS_LO__NBIF_TRAP_NP_Trap6AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_6_ADDRESS_LO__NBIF_TRAP_NP_Trap6AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_6_ADDRESS_HI
+#define NBIF_TRAP_NP_6_ADDRESS_HI__NBIF_TRAP_NP_Trap6AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_6_ADDRESS_HI__NBIF_TRAP_NP_Trap6AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_6_COMMAND
+#define NBIF_TRAP_NP_6_COMMAND__NBIF_TRAP_NP_Trap6Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_6_COMMAND__NBIF_TRAP_NP_Trap6Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_6_COMMAND__NBIF_TRAP_NP_Trap6Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_6_COMMAND__NBIF_TRAP_NP_Trap6Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_6_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_6_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap6AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_6_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap6AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_6_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_6_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap6AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_6_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap6AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_6_COMMAND_MASK
+#define NBIF_TRAP_NP_6_COMMAND_MASK__NBIF_TRAP_NP_Trap6Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_6_COMMAND_MASK__NBIF_TRAP_NP_Trap6Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_6_COMMAND_MASK__NBIF_TRAP_NP_Trap6Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_6_COMMAND_MASK__NBIF_TRAP_NP_Trap6Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_7_CONTROL0
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_7_CONTROL0__NBIF_TRAP_NP_Trap7Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_7_ADDRESS_LO
+#define NBIF_TRAP_NP_7_ADDRESS_LO__NBIF_TRAP_NP_Trap7AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_7_ADDRESS_LO__NBIF_TRAP_NP_Trap7AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_7_ADDRESS_HI
+#define NBIF_TRAP_NP_7_ADDRESS_HI__NBIF_TRAP_NP_Trap7AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_7_ADDRESS_HI__NBIF_TRAP_NP_Trap7AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_7_COMMAND
+#define NBIF_TRAP_NP_7_COMMAND__NBIF_TRAP_NP_Trap7Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_7_COMMAND__NBIF_TRAP_NP_Trap7Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_7_COMMAND__NBIF_TRAP_NP_Trap7Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_7_COMMAND__NBIF_TRAP_NP_Trap7Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_7_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_7_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap7AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_7_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap7AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_7_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_7_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap7AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_7_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap7AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_7_COMMAND_MASK
+#define NBIF_TRAP_NP_7_COMMAND_MASK__NBIF_TRAP_NP_Trap7Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_7_COMMAND_MASK__NBIF_TRAP_NP_Trap7Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_7_COMMAND_MASK__NBIF_TRAP_NP_Trap7Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_7_COMMAND_MASK__NBIF_TRAP_NP_Trap7Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_8_CONTROL0
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_8_CONTROL0__NBIF_TRAP_NP_Trap8Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_8_ADDRESS_LO
+#define NBIF_TRAP_NP_8_ADDRESS_LO__NBIF_TRAP_NP_Trap8AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_8_ADDRESS_LO__NBIF_TRAP_NP_Trap8AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_8_ADDRESS_HI
+#define NBIF_TRAP_NP_8_ADDRESS_HI__NBIF_TRAP_NP_Trap8AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_8_ADDRESS_HI__NBIF_TRAP_NP_Trap8AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_8_COMMAND
+#define NBIF_TRAP_NP_8_COMMAND__NBIF_TRAP_NP_Trap8Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_8_COMMAND__NBIF_TRAP_NP_Trap8Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_8_COMMAND__NBIF_TRAP_NP_Trap8Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_8_COMMAND__NBIF_TRAP_NP_Trap8Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_8_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_8_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap8AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_8_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap8AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_8_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_8_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap8AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_8_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap8AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_8_COMMAND_MASK
+#define NBIF_TRAP_NP_8_COMMAND_MASK__NBIF_TRAP_NP_Trap8Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_8_COMMAND_MASK__NBIF_TRAP_NP_Trap8Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_8_COMMAND_MASK__NBIF_TRAP_NP_Trap8Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_8_COMMAND_MASK__NBIF_TRAP_NP_Trap8Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_9_CONTROL0
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9En__SHIFT                                                  0x0
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9SMUIntr__SHIFT                                             0x3
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9Stage2Ptr__SHIFT                                           0xe
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9CrossTrigger__SHIFT                                        0x18
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9Stage2En__SHIFT                                            0x1f
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9En_MASK                                                    0x00000001L
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9SMUIntr_MASK                                               0x00000008L
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9Stage2Ptr_MASK                                             0x00FFC000L
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9CrossTrigger_MASK                                          0x0F000000L
+#define NBIF_TRAP_NP_9_CONTROL0__NBIF_TRAP_NP_Trap9Stage2En_MASK                                              0x80000000L
+//NBIF_TRAP_NP_9_ADDRESS_LO
+#define NBIF_TRAP_NP_9_ADDRESS_LO__NBIF_TRAP_NP_Trap9AddrLo__SHIFT                                            0x2
+#define NBIF_TRAP_NP_9_ADDRESS_LO__NBIF_TRAP_NP_Trap9AddrLo_MASK                                              0xFFFFFFFCL
+//NBIF_TRAP_NP_9_ADDRESS_HI
+#define NBIF_TRAP_NP_9_ADDRESS_HI__NBIF_TRAP_NP_Trap9AddrHi__SHIFT                                            0x0
+#define NBIF_TRAP_NP_9_ADDRESS_HI__NBIF_TRAP_NP_Trap9AddrHi_MASK                                              0xFFFFFFFFL
+//NBIF_TRAP_NP_9_COMMAND
+#define NBIF_TRAP_NP_9_COMMAND__NBIF_TRAP_NP_Trap9Cmd0__SHIFT                                                 0x0
+#define NBIF_TRAP_NP_9_COMMAND__NBIF_TRAP_NP_Trap9Cmd1__SHIFT                                                 0x8
+#define NBIF_TRAP_NP_9_COMMAND__NBIF_TRAP_NP_Trap9Cmd0_MASK                                                   0x0000003FL
+#define NBIF_TRAP_NP_9_COMMAND__NBIF_TRAP_NP_Trap9Cmd1_MASK                                                   0x00003F00L
+//NBIF_TRAP_NP_9_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_9_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap9AddrLoMask__SHIFT                                   0x2
+#define NBIF_TRAP_NP_9_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap9AddrLoMask_MASK                                     0xFFFFFFFCL
+//NBIF_TRAP_NP_9_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_9_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap9AddrHiMask__SHIFT                                   0x0
+#define NBIF_TRAP_NP_9_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap9AddrHiMask_MASK                                     0xFFFFFFFFL
+//NBIF_TRAP_NP_9_COMMAND_MASK
+#define NBIF_TRAP_NP_9_COMMAND_MASK__NBIF_TRAP_NP_Trap9Cmd0Mask__SHIFT                                        0x0
+#define NBIF_TRAP_NP_9_COMMAND_MASK__NBIF_TRAP_NP_Trap9Cmd1Mask__SHIFT                                        0x8
+#define NBIF_TRAP_NP_9_COMMAND_MASK__NBIF_TRAP_NP_Trap9Cmd0Mask_MASK                                          0x0000003FL
+#define NBIF_TRAP_NP_9_COMMAND_MASK__NBIF_TRAP_NP_Trap9Cmd1Mask_MASK                                          0x00003F00L
+//NBIF_TRAP_NP_10_CONTROL0
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10En__SHIFT                                                0x0
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10SMUIntr__SHIFT                                           0x3
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10Stage2Ptr__SHIFT                                         0xe
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10CrossTrigger__SHIFT                                      0x18
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10Stage2En__SHIFT                                          0x1f
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10En_MASK                                                  0x00000001L
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10SMUIntr_MASK                                             0x00000008L
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10Stage2Ptr_MASK                                           0x00FFC000L
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10CrossTrigger_MASK                                        0x0F000000L
+#define NBIF_TRAP_NP_10_CONTROL0__NBIF_TRAP_NP_Trap10Stage2En_MASK                                            0x80000000L
+//NBIF_TRAP_NP_10_ADDRESS_LO
+#define NBIF_TRAP_NP_10_ADDRESS_LO__NBIF_TRAP_NP_Trap10AddrLo__SHIFT                                          0x2
+#define NBIF_TRAP_NP_10_ADDRESS_LO__NBIF_TRAP_NP_Trap10AddrLo_MASK                                            0xFFFFFFFCL
+//NBIF_TRAP_NP_10_ADDRESS_HI
+#define NBIF_TRAP_NP_10_ADDRESS_HI__NBIF_TRAP_NP_Trap10AddrHi__SHIFT                                          0x0
+#define NBIF_TRAP_NP_10_ADDRESS_HI__NBIF_TRAP_NP_Trap10AddrHi_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_10_COMMAND
+#define NBIF_TRAP_NP_10_COMMAND__NBIF_TRAP_NP_Trap10Cmd0__SHIFT                                               0x0
+#define NBIF_TRAP_NP_10_COMMAND__NBIF_TRAP_NP_Trap10Cmd1__SHIFT                                               0x8
+#define NBIF_TRAP_NP_10_COMMAND__NBIF_TRAP_NP_Trap10Cmd0_MASK                                                 0x0000003FL
+#define NBIF_TRAP_NP_10_COMMAND__NBIF_TRAP_NP_Trap10Cmd1_MASK                                                 0x00003F00L
+//NBIF_TRAP_NP_10_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_10_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap10AddrLoMask__SHIFT                                 0x2
+#define NBIF_TRAP_NP_10_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap10AddrLoMask_MASK                                   0xFFFFFFFCL
+//NBIF_TRAP_NP_10_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_10_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap10AddrHiMask__SHIFT                                 0x0
+#define NBIF_TRAP_NP_10_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap10AddrHiMask_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_10_COMMAND_MASK
+#define NBIF_TRAP_NP_10_COMMAND_MASK__NBIF_TRAP_NP_Trap10Cmd0Mask__SHIFT                                      0x0
+#define NBIF_TRAP_NP_10_COMMAND_MASK__NBIF_TRAP_NP_Trap10Cmd1Mask__SHIFT                                      0x8
+#define NBIF_TRAP_NP_10_COMMAND_MASK__NBIF_TRAP_NP_Trap10Cmd0Mask_MASK                                        0x0000003FL
+#define NBIF_TRAP_NP_10_COMMAND_MASK__NBIF_TRAP_NP_Trap10Cmd1Mask_MASK                                        0x00003F00L
+//NBIF_TRAP_NP_11_CONTROL0
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11En__SHIFT                                                0x0
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11SMUIntr__SHIFT                                           0x3
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11Stage2Ptr__SHIFT                                         0xe
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11CrossTrigger__SHIFT                                      0x18
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11Stage2En__SHIFT                                          0x1f
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11En_MASK                                                  0x00000001L
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11SMUIntr_MASK                                             0x00000008L
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11Stage2Ptr_MASK                                           0x00FFC000L
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11CrossTrigger_MASK                                        0x0F000000L
+#define NBIF_TRAP_NP_11_CONTROL0__NBIF_TRAP_NP_Trap11Stage2En_MASK                                            0x80000000L
+//NBIF_TRAP_NP_11_ADDRESS_LO
+#define NBIF_TRAP_NP_11_ADDRESS_LO__NBIF_TRAP_NP_Trap11AddrLo__SHIFT                                          0x2
+#define NBIF_TRAP_NP_11_ADDRESS_LO__NBIF_TRAP_NP_Trap11AddrLo_MASK                                            0xFFFFFFFCL
+//NBIF_TRAP_NP_11_ADDRESS_HI
+#define NBIF_TRAP_NP_11_ADDRESS_HI__NBIF_TRAP_NP_Trap11AddrHi__SHIFT                                          0x0
+#define NBIF_TRAP_NP_11_ADDRESS_HI__NBIF_TRAP_NP_Trap11AddrHi_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_11_COMMAND
+#define NBIF_TRAP_NP_11_COMMAND__NBIF_TRAP_NP_Trap11Cmd0__SHIFT                                               0x0
+#define NBIF_TRAP_NP_11_COMMAND__NBIF_TRAP_NP_Trap11Cmd1__SHIFT                                               0x8
+#define NBIF_TRAP_NP_11_COMMAND__NBIF_TRAP_NP_Trap11Cmd0_MASK                                                 0x0000003FL
+#define NBIF_TRAP_NP_11_COMMAND__NBIF_TRAP_NP_Trap11Cmd1_MASK                                                 0x00003F00L
+//NBIF_TRAP_NP_11_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_11_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap11AddrLoMask__SHIFT                                 0x2
+#define NBIF_TRAP_NP_11_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap11AddrLoMask_MASK                                   0xFFFFFFFCL
+//NBIF_TRAP_NP_11_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_11_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap11AddrHiMask__SHIFT                                 0x0
+#define NBIF_TRAP_NP_11_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap11AddrHiMask_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_11_COMMAND_MASK
+#define NBIF_TRAP_NP_11_COMMAND_MASK__NBIF_TRAP_NP_Trap11Cmd0Mask__SHIFT                                      0x0
+#define NBIF_TRAP_NP_11_COMMAND_MASK__NBIF_TRAP_NP_Trap11Cmd1Mask__SHIFT                                      0x8
+#define NBIF_TRAP_NP_11_COMMAND_MASK__NBIF_TRAP_NP_Trap11Cmd0Mask_MASK                                        0x0000003FL
+#define NBIF_TRAP_NP_11_COMMAND_MASK__NBIF_TRAP_NP_Trap11Cmd1Mask_MASK                                        0x00003F00L
+//NBIF_TRAP_NP_12_CONTROL0
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12En__SHIFT                                                0x0
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12SMUIntr__SHIFT                                           0x3
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12Stage2Ptr__SHIFT                                         0xe
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12CrossTrigger__SHIFT                                      0x18
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12Stage2En__SHIFT                                          0x1f
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12En_MASK                                                  0x00000001L
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12SMUIntr_MASK                                             0x00000008L
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12Stage2Ptr_MASK                                           0x00FFC000L
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12CrossTrigger_MASK                                        0x0F000000L
+#define NBIF_TRAP_NP_12_CONTROL0__NBIF_TRAP_NP_Trap12Stage2En_MASK                                            0x80000000L
+//NBIF_TRAP_NP_12_ADDRESS_LO
+#define NBIF_TRAP_NP_12_ADDRESS_LO__NBIF_TRAP_NP_Trap12AddrLo__SHIFT                                          0x2
+#define NBIF_TRAP_NP_12_ADDRESS_LO__NBIF_TRAP_NP_Trap12AddrLo_MASK                                            0xFFFFFFFCL
+//NBIF_TRAP_NP_12_ADDRESS_HI
+#define NBIF_TRAP_NP_12_ADDRESS_HI__NBIF_TRAP_NP_Trap12AddrHi__SHIFT                                          0x0
+#define NBIF_TRAP_NP_12_ADDRESS_HI__NBIF_TRAP_NP_Trap12AddrHi_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_12_COMMAND
+#define NBIF_TRAP_NP_12_COMMAND__NBIF_TRAP_NP_Trap12Cmd0__SHIFT                                               0x0
+#define NBIF_TRAP_NP_12_COMMAND__NBIF_TRAP_NP_Trap12Cmd1__SHIFT                                               0x8
+#define NBIF_TRAP_NP_12_COMMAND__NBIF_TRAP_NP_Trap12Cmd0_MASK                                                 0x0000003FL
+#define NBIF_TRAP_NP_12_COMMAND__NBIF_TRAP_NP_Trap12Cmd1_MASK                                                 0x00003F00L
+//NBIF_TRAP_NP_12_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_12_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap12AddrLoMask__SHIFT                                 0x2
+#define NBIF_TRAP_NP_12_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap12AddrLoMask_MASK                                   0xFFFFFFFCL
+//NBIF_TRAP_NP_12_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_12_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap12AddrHiMask__SHIFT                                 0x0
+#define NBIF_TRAP_NP_12_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap12AddrHiMask_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_12_COMMAND_MASK
+#define NBIF_TRAP_NP_12_COMMAND_MASK__NBIF_TRAP_NP_Trap12Cmd0Mask__SHIFT                                      0x0
+#define NBIF_TRAP_NP_12_COMMAND_MASK__NBIF_TRAP_NP_Trap12Cmd1Mask__SHIFT                                      0x8
+#define NBIF_TRAP_NP_12_COMMAND_MASK__NBIF_TRAP_NP_Trap12Cmd0Mask_MASK                                        0x0000003FL
+#define NBIF_TRAP_NP_12_COMMAND_MASK__NBIF_TRAP_NP_Trap12Cmd1Mask_MASK                                        0x00003F00L
+//NBIF_TRAP_NP_13_CONTROL0
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13En__SHIFT                                                0x0
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13SMUIntr__SHIFT                                           0x3
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13Stage2Ptr__SHIFT                                         0xe
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13CrossTrigger__SHIFT                                      0x18
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13Stage2En__SHIFT                                          0x1f
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13En_MASK                                                  0x00000001L
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13SMUIntr_MASK                                             0x00000008L
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13Stage2Ptr_MASK                                           0x00FFC000L
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13CrossTrigger_MASK                                        0x0F000000L
+#define NBIF_TRAP_NP_13_CONTROL0__NBIF_TRAP_NP_Trap13Stage2En_MASK                                            0x80000000L
+//NBIF_TRAP_NP_13_ADDRESS_LO
+#define NBIF_TRAP_NP_13_ADDRESS_LO__NBIF_TRAP_NP_Trap13AddrLo__SHIFT                                          0x2
+#define NBIF_TRAP_NP_13_ADDRESS_LO__NBIF_TRAP_NP_Trap13AddrLo_MASK                                            0xFFFFFFFCL
+//NBIF_TRAP_NP_13_ADDRESS_HI
+#define NBIF_TRAP_NP_13_ADDRESS_HI__NBIF_TRAP_NP_Trap13AddrHi__SHIFT                                          0x0
+#define NBIF_TRAP_NP_13_ADDRESS_HI__NBIF_TRAP_NP_Trap13AddrHi_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_13_COMMAND
+#define NBIF_TRAP_NP_13_COMMAND__NBIF_TRAP_NP_Trap13Cmd0__SHIFT                                               0x0
+#define NBIF_TRAP_NP_13_COMMAND__NBIF_TRAP_NP_Trap13Cmd1__SHIFT                                               0x8
+#define NBIF_TRAP_NP_13_COMMAND__NBIF_TRAP_NP_Trap13Cmd0_MASK                                                 0x0000003FL
+#define NBIF_TRAP_NP_13_COMMAND__NBIF_TRAP_NP_Trap13Cmd1_MASK                                                 0x00003F00L
+//NBIF_TRAP_NP_13_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_13_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap13AddrLoMask__SHIFT                                 0x2
+#define NBIF_TRAP_NP_13_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap13AddrLoMask_MASK                                   0xFFFFFFFCL
+//NBIF_TRAP_NP_13_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_13_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap13AddrHiMask__SHIFT                                 0x0
+#define NBIF_TRAP_NP_13_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap13AddrHiMask_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_13_COMMAND_MASK
+#define NBIF_TRAP_NP_13_COMMAND_MASK__NBIF_TRAP_NP_Trap13Cmd0Mask__SHIFT                                      0x0
+#define NBIF_TRAP_NP_13_COMMAND_MASK__NBIF_TRAP_NP_Trap13Cmd1Mask__SHIFT                                      0x8
+#define NBIF_TRAP_NP_13_COMMAND_MASK__NBIF_TRAP_NP_Trap13Cmd0Mask_MASK                                        0x0000003FL
+#define NBIF_TRAP_NP_13_COMMAND_MASK__NBIF_TRAP_NP_Trap13Cmd1Mask_MASK                                        0x00003F00L
+//NBIF_TRAP_NP_14_CONTROL0
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14En__SHIFT                                                0x0
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14SMUIntr__SHIFT                                           0x3
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14Stage2Ptr__SHIFT                                         0xe
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14CrossTrigger__SHIFT                                      0x18
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14Stage2En__SHIFT                                          0x1f
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14En_MASK                                                  0x00000001L
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14SMUIntr_MASK                                             0x00000008L
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14Stage2Ptr_MASK                                           0x00FFC000L
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14CrossTrigger_MASK                                        0x0F000000L
+#define NBIF_TRAP_NP_14_CONTROL0__NBIF_TRAP_NP_Trap14Stage2En_MASK                                            0x80000000L
+//NBIF_TRAP_NP_14_ADDRESS_LO
+#define NBIF_TRAP_NP_14_ADDRESS_LO__NBIF_TRAP_NP_Trap14AddrLo__SHIFT                                          0x2
+#define NBIF_TRAP_NP_14_ADDRESS_LO__NBIF_TRAP_NP_Trap14AddrLo_MASK                                            0xFFFFFFFCL
+//NBIF_TRAP_NP_14_ADDRESS_HI
+#define NBIF_TRAP_NP_14_ADDRESS_HI__NBIF_TRAP_NP_Trap14AddrHi__SHIFT                                          0x0
+#define NBIF_TRAP_NP_14_ADDRESS_HI__NBIF_TRAP_NP_Trap14AddrHi_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_14_COMMAND
+#define NBIF_TRAP_NP_14_COMMAND__NBIF_TRAP_NP_Trap14Cmd0__SHIFT                                               0x0
+#define NBIF_TRAP_NP_14_COMMAND__NBIF_TRAP_NP_Trap14Cmd1__SHIFT                                               0x8
+#define NBIF_TRAP_NP_14_COMMAND__NBIF_TRAP_NP_Trap14Cmd0_MASK                                                 0x0000003FL
+#define NBIF_TRAP_NP_14_COMMAND__NBIF_TRAP_NP_Trap14Cmd1_MASK                                                 0x00003F00L
+//NBIF_TRAP_NP_14_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_14_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap14AddrLoMask__SHIFT                                 0x2
+#define NBIF_TRAP_NP_14_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap14AddrLoMask_MASK                                   0xFFFFFFFCL
+//NBIF_TRAP_NP_14_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_14_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap14AddrHiMask__SHIFT                                 0x0
+#define NBIF_TRAP_NP_14_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap14AddrHiMask_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_14_COMMAND_MASK
+#define NBIF_TRAP_NP_14_COMMAND_MASK__NBIF_TRAP_NP_Trap14Cmd0Mask__SHIFT                                      0x0
+#define NBIF_TRAP_NP_14_COMMAND_MASK__NBIF_TRAP_NP_Trap14Cmd1Mask__SHIFT                                      0x8
+#define NBIF_TRAP_NP_14_COMMAND_MASK__NBIF_TRAP_NP_Trap14Cmd0Mask_MASK                                        0x0000003FL
+#define NBIF_TRAP_NP_14_COMMAND_MASK__NBIF_TRAP_NP_Trap14Cmd1Mask_MASK                                        0x00003F00L
+//NBIF_TRAP_NP_15_CONTROL0
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15En__SHIFT                                                0x0
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15SMUIntr__SHIFT                                           0x3
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15Stage2Ptr__SHIFT                                         0xe
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15CrossTrigger__SHIFT                                      0x18
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15Stage2En__SHIFT                                          0x1f
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15En_MASK                                                  0x00000001L
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15SMUIntr_MASK                                             0x00000008L
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15Stage2Ptr_MASK                                           0x00FFC000L
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15CrossTrigger_MASK                                        0x0F000000L
+#define NBIF_TRAP_NP_15_CONTROL0__NBIF_TRAP_NP_Trap15Stage2En_MASK                                            0x80000000L
+//NBIF_TRAP_NP_15_ADDRESS_LO
+#define NBIF_TRAP_NP_15_ADDRESS_LO__NBIF_TRAP_NP_Trap15AddrLo__SHIFT                                          0x2
+#define NBIF_TRAP_NP_15_ADDRESS_LO__NBIF_TRAP_NP_Trap15AddrLo_MASK                                            0xFFFFFFFCL
+//NBIF_TRAP_NP_15_ADDRESS_HI
+#define NBIF_TRAP_NP_15_ADDRESS_HI__NBIF_TRAP_NP_Trap15AddrHi__SHIFT                                          0x0
+#define NBIF_TRAP_NP_15_ADDRESS_HI__NBIF_TRAP_NP_Trap15AddrHi_MASK                                            0xFFFFFFFFL
+//NBIF_TRAP_NP_15_COMMAND
+#define NBIF_TRAP_NP_15_COMMAND__NBIF_TRAP_NP_Trap15Cmd0__SHIFT                                               0x0
+#define NBIF_TRAP_NP_15_COMMAND__NBIF_TRAP_NP_Trap15Cmd1__SHIFT                                               0x8
+#define NBIF_TRAP_NP_15_COMMAND__NBIF_TRAP_NP_Trap15Cmd0_MASK                                                 0x0000003FL
+#define NBIF_TRAP_NP_15_COMMAND__NBIF_TRAP_NP_Trap15Cmd1_MASK                                                 0x00003F00L
+//NBIF_TRAP_NP_15_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_15_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap15AddrLoMask__SHIFT                                 0x2
+#define NBIF_TRAP_NP_15_ADDRESS_LO_MASK__NBIF_TRAP_NP_Trap15AddrLoMask_MASK                                   0xFFFFFFFCL
+//NBIF_TRAP_NP_15_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_15_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap15AddrHiMask__SHIFT                                 0x0
+#define NBIF_TRAP_NP_15_ADDRESS_HI_MASK__NBIF_TRAP_NP_Trap15AddrHiMask_MASK                                   0xFFFFFFFFL
+//NBIF_TRAP_NP_15_COMMAND_MASK
+#define NBIF_TRAP_NP_15_COMMAND_MASK__NBIF_TRAP_NP_Trap15Cmd0Mask__SHIFT                                      0x0
+#define NBIF_TRAP_NP_15_COMMAND_MASK__NBIF_TRAP_NP_Trap15Cmd1Mask__SHIFT                                      0x8
+#define NBIF_TRAP_NP_15_COMMAND_MASK__NBIF_TRAP_NP_Trap15Cmd0Mask_MASK                                        0x0000003FL
+#define NBIF_TRAP_NP_15_COMMAND_MASK__NBIF_TRAP_NP_Trap15Cmd1Mask_MASK                                        0x00003F00L
+//NBIF_TRAP_NP_S2_CONTROL0
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapEn__SHIFT                                               0x0
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapSMUIntr__SHIFT                                          0x3
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapStage2Offset__SHIFT                                     0x4
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapStage2Ptr__SHIFT                                        0xe
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapCrossTrigger__SHIFT                                     0x18
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapStage2En__SHIFT                                         0x1f
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapEn_MASK                                                 0x00000001L
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapSMUIntr_MASK                                            0x00000008L
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapStage2Offset_MASK                                       0x00003FF0L
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapStage2Ptr_MASK                                          0x00FFC000L
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapCrossTrigger_MASK                                       0x0F000000L
+#define NBIF_TRAP_NP_S2_CONTROL0__NBIF_TRAP_NP_S2_TrapStage2En_MASK                                           0x80000000L
+//NBIF_TRAP_NP_S2_ADDRESS_LO
+#define NBIF_TRAP_NP_S2_ADDRESS_LO__NBIF_TRAP_NP_S2_TrapAddrLo__SHIFT                                         0x2
+#define NBIF_TRAP_NP_S2_ADDRESS_LO__NBIF_TRAP_NP_S2_TrapAddrLo_MASK                                           0xFFFFFFFCL
+//NBIF_TRAP_NP_S2_ADDRESS_HI
+#define NBIF_TRAP_NP_S2_ADDRESS_HI__NBIF_TRAP_NP_S2_TrapAddrHi__SHIFT                                         0x0
+#define NBIF_TRAP_NP_S2_ADDRESS_HI__NBIF_TRAP_NP_S2_TrapAddrHi_MASK                                           0xFFFFFFFFL
+//NBIF_TRAP_NP_S2_COMMAND
+#define NBIF_TRAP_NP_S2_COMMAND__NBIF_TRAP_NP_S2_TrapCmd0__SHIFT                                              0x0
+#define NBIF_TRAP_NP_S2_COMMAND__NBIF_TRAP_NP_S2_TrapCmd1__SHIFT                                              0x8
+#define NBIF_TRAP_NP_S2_COMMAND__NBIF_TRAP_NP_S2_TrapCmd0_MASK                                                0x0000003FL
+#define NBIF_TRAP_NP_S2_COMMAND__NBIF_TRAP_NP_S2_TrapCmd1_MASK                                                0x00003F00L
+//NBIF_TRAP_NP_S2_ADDRESS_LO_MASK
+#define NBIF_TRAP_NP_S2_ADDRESS_LO_MASK__NBIF_TRAP_NP_S2_TrapAddrLoMask__SHIFT                                0x2
+#define NBIF_TRAP_NP_S2_ADDRESS_LO_MASK__NBIF_TRAP_NP_S2_TrapAddrLoMask_MASK                                  0xFFFFFFFCL
+//NBIF_TRAP_NP_S2_ADDRESS_HI_MASK
+#define NBIF_TRAP_NP_S2_ADDRESS_HI_MASK__NBIF_TRAP_NP_S2_TrapAddrHiMask__SHIFT                                0x0
+#define NBIF_TRAP_NP_S2_ADDRESS_HI_MASK__NBIF_TRAP_NP_S2_TrapAddrHiMask_MASK                                  0xFFFFFFFFL
+//NBIF_TRAP_NP_S2_COMMAND_MASK
+#define NBIF_TRAP_NP_S2_COMMAND_MASK__NBIF_TRAP_NP_S2_TrapCmd0Mask__SHIFT                                     0x0
+#define NBIF_TRAP_NP_S2_COMMAND_MASK__NBIF_TRAP_NP_S2_TrapCmd1Mask__SHIFT                                     0x8
+#define NBIF_TRAP_NP_S2_COMMAND_MASK__NBIF_TRAP_NP_S2_TrapCmd0Mask_MASK                                       0x0000003FL
+#define NBIF_TRAP_NP_S2_COMMAND_MASK__NBIF_TRAP_NP_S2_TrapCmd1Mask_MASK                                       0x00003F00L
+
+
+// addressBlock: nbif0_nbif0_bif_ras_bif_ras_regblk
+//BIFL_RAS_CENTRAL_CNTL
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_HST_STALL_DIS__SHIFT                                   0x1b
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_STALL_DIS__SHIFT                                       0x1c
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_DIS__SHIFT                                             0x1d
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_INTR_DIS__SHIFT                                                 0x1e
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_LINKDIS_TRIG_EGRESS_STALL_DIS__SHIFT                                      0x1f
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_HST_STALL_DIS_MASK                                     0x08000000L
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_STALL_DIS_MASK                                         0x10000000L
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_ERREVENT_DIS_MASK                                               0x20000000L
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_RAS_CONTL_INTR_DIS_MASK                                                   0x40000000L
+#define BIFL_RAS_CENTRAL_CNTL__BIFL_LINKDIS_TRIG_EGRESS_STALL_DIS_MASK                                        0x80000000L
+//BIFL_RAS_CENTRAL_STATUS
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_L2C_EgStall_det__SHIFT                                                  0x0
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_L2C_ErrEvent_det__SHIFT                                                 0x1
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_C2L_EgStall_det__SHIFT                                                  0x2
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_C2L_ErrEvent_det__SHIFT                                                 0x3
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_RasContller_ErrEvent_Recv__SHIFT                                        0x1d
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_RasContller_Intr_Recv__SHIFT                                            0x1e
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_LinkDis_Recv__SHIFT                                                     0x1f
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_L2C_EgStall_det_MASK                                                    0x00000001L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_L2C_ErrEvent_det_MASK                                                   0x00000002L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_C2L_EgStall_det_MASK                                                    0x00000004L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_C2L_ErrEvent_det_MASK                                                   0x00000008L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_RasContller_ErrEvent_Recv_MASK                                          0x20000000L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_RasContller_Intr_Recv_MASK                                              0x40000000L
+#define BIFL_RAS_CENTRAL_STATUS__BIFL_LinkDis_Recv_MASK                                                       0x80000000L
+//BIFL_RAS_LEAF0_CTRL
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF0_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF0_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF0_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF0_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF0_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF0_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF0_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF0_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF0_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF0_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF0_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF0_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF0_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF0_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF0_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF0_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF0_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF0_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF0_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF0_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF0_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF0_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF0_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF0_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF1_CTRL
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF1_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF1_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF1_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF1_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF1_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF1_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF1_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF1_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF1_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF1_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF1_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF1_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF1_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF1_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF1_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF1_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF1_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF1_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF1_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF1_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF1_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF1_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF1_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF1_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF2_CTRL
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF2_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF2_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF2_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF2_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF2_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF2_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF2_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF2_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF2_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF2_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF2_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF2_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF2_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF2_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF2_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF2_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF2_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF2_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF2_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF2_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF2_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF2_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF2_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF2_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF3_CTRL
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF3_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF3_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF3_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF3_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF3_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF3_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF3_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF3_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF3_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF3_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF3_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF3_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF3_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF3_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF3_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF3_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF3_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF3_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF3_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF3_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF3_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF3_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF3_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF3_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF4_CTRL
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF4_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF4_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF4_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF4_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF4_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF4_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF4_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF4_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF4_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF4_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF4_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF4_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF4_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF4_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF4_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF4_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF4_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF4_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF4_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF4_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF4_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF4_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF4_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF4_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF4_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF4_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF4_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF4_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF4_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF5_CTRL
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF5_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF5_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF5_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF5_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF5_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF5_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF5_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF5_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF5_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF5_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF5_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF5_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF5_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF5_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF5_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF5_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF5_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF5_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF5_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF5_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF5_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF5_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF5_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF5_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF5_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF5_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF5_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF5_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF5_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF6_CTRL
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF6_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF6_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF6_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF6_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF6_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF6_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF6_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF6_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF6_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF6_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF6_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF6_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF6_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF6_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF6_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF6_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF6_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF6_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF6_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF6_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF6_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF6_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF6_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF6_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF6_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF6_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF6_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF6_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF6_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF7_CTRL
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF7_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF7_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF7_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF7_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF7_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF7_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF7_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF7_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF7_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF7_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF7_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF7_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF7_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF7_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF7_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF7_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF7_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF7_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF7_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF7_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF7_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF7_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF7_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF7_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF7_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF7_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF7_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF7_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF7_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF8_CTRL
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF8_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF8_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF8_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF8_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF8_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF8_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF8_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF8_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF8_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF8_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF8_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF8_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF8_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF8_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF8_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF8_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF8_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF8_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF8_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF8_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF8_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF8_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF8_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF8_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF8_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF8_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF8_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF8_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF8_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF9_CTRL
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_DET_EN__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF9_CTRL__POISON_ERREVENT_EN__SHIFT                                                        0x1
+#define BIFL_RAS_LEAF9_CTRL__POISON_STALL_EN__SHIFT                                                           0x2
+#define BIFL_RAS_LEAF9_CTRL__PARITY_ERREVENT_EN__SHIFT                                                        0x3
+#define BIFL_RAS_LEAF9_CTRL__PARITY_STALL_EN__SHIFT                                                           0x4
+#define BIFL_RAS_LEAF9_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                   0x5
+#define BIFL_RAS_LEAF9_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                      0x6
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                          0x8
+#define BIFL_RAS_LEAF9_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                       0x9
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                         0xa
+#define BIFL_RAS_LEAF9_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                      0xb
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                     0x10
+#define BIFL_RAS_LEAF9_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                   0x11
+#define BIFL_RAS_LEAF9_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                   0x12
+#define BIFL_RAS_LEAF9_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                  0x13
+#define BIFL_RAS_LEAF9_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                              0x14
+#define BIFL_RAS_LEAF9_CTRL__UCP_EN__SHIFT                                                                    0x15
+#define BIFL_RAS_LEAF9_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                                0x16
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                 0x17
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_DET_EN_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF9_CTRL__POISON_ERREVENT_EN_MASK                                                          0x00000002L
+#define BIFL_RAS_LEAF9_CTRL__POISON_STALL_EN_MASK                                                             0x00000004L
+#define BIFL_RAS_LEAF9_CTRL__PARITY_ERREVENT_EN_MASK                                                          0x00000008L
+#define BIFL_RAS_LEAF9_CTRL__PARITY_STALL_EN_MASK                                                             0x00000010L
+#define BIFL_RAS_LEAF9_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                     0x00000020L
+#define BIFL_RAS_LEAF9_CTRL__RCVERREVENT_STALL_EN_MASK                                                        0x00000040L
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_GEN_EN_MASK                                                            0x00000100L
+#define BIFL_RAS_LEAF9_CTRL__EGRESS_STALL_GEN_EN_MASK                                                         0x00000200L
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_PROP_EN_MASK                                                           0x00000400L
+#define BIFL_RAS_LEAF9_CTRL__EGRESS_STALL_PROP_EN_MASK                                                        0x00000800L
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                       0x00010000L
+#define BIFL_RAS_LEAF9_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                     0x00020000L
+#define BIFL_RAS_LEAF9_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                     0x00040000L
+#define BIFL_RAS_LEAF9_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                    0x00080000L
+#define BIFL_RAS_LEAF9_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                                0x00100000L
+#define BIFL_RAS_LEAF9_CTRL__UCP_EN_MASK                                                                      0x00200000L
+#define BIFL_RAS_LEAF9_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                  0x00400000L
+#define BIFL_RAS_LEAF9_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                   0x00800000L
+//BIFL_RAS_LEAF10_CTRL
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF10_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF10_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF10_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF10_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF10_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF10_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF10_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF10_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF10_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF10_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF10_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF10_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF10_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF10_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF10_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF10_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF10_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF10_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF10_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF10_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF10_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF10_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF10_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF10_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF10_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF10_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF10_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF10_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF10_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF11_CTRL
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF11_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF11_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF11_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF11_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF11_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF11_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF11_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF11_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF11_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF11_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF11_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF11_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF11_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF11_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF11_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF11_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF11_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF11_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF11_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF11_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF11_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF11_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF11_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF11_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF11_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF11_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF11_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF11_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF11_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF12_CTRL
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF12_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF12_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF12_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF12_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF12_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF12_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF12_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF12_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF12_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF12_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF12_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF12_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF12_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF12_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF12_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF12_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF12_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF12_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF12_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF12_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF12_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF12_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF12_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF12_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF12_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF12_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF12_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF12_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF12_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF13_CTRL
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF13_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF13_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF13_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF13_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF13_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF13_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF13_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF13_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF13_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF13_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF13_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF13_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF13_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF13_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF13_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF13_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF13_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF13_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF13_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF13_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF13_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF13_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF13_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF13_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF13_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF13_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF13_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF13_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF13_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF14_CTRL
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF14_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF14_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF14_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF14_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF14_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF14_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF14_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF14_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF14_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF14_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF14_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF14_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF14_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF14_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF14_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF14_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF14_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF14_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF14_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF14_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF14_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF14_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF14_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF14_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF14_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF14_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF14_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF14_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF14_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF15_CTRL
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF15_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF15_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF15_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF15_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF15_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF15_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF15_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF15_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF15_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF15_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF15_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF15_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF15_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF15_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF15_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF15_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF15_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF15_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF15_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF15_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF15_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF15_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF15_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF15_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF15_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF15_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF15_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF15_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF15_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF16_CTRL
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF16_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF16_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF16_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF16_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF16_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF16_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF16_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF16_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF16_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF16_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF16_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF16_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF16_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF16_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF16_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF16_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF16_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF16_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF16_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF16_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF16_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF16_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF16_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF16_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF16_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF16_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF16_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF16_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF16_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF17_CTRL
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF17_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF17_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF17_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF17_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF17_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF17_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF17_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF17_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF17_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF17_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF17_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF17_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF17_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF17_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF17_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF17_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF17_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF17_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF17_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF17_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF17_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF17_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF17_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF17_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF17_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF17_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF17_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF17_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF17_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF18_CTRL
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF18_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF18_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF18_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF18_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF18_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF18_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF18_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF18_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF18_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF18_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF18_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF18_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF18_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF18_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF18_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF18_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF18_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF18_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF18_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF18_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF18_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF18_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF18_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF18_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF18_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF18_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF18_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF18_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF18_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF19_CTRL
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_DET_EN__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF19_CTRL__POISON_ERREVENT_EN__SHIFT                                                       0x1
+#define BIFL_RAS_LEAF19_CTRL__POISON_STALL_EN__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF19_CTRL__PARITY_ERREVENT_EN__SHIFT                                                       0x3
+#define BIFL_RAS_LEAF19_CTRL__PARITY_STALL_EN__SHIFT                                                          0x4
+#define BIFL_RAS_LEAF19_CTRL__RCVERREVENT_ERREVENT_EN__SHIFT                                                  0x5
+#define BIFL_RAS_LEAF19_CTRL__RCVERREVENT_STALL_EN__SHIFT                                                     0x6
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_GEN_EN__SHIFT                                                         0x8
+#define BIFL_RAS_LEAF19_CTRL__EGRESS_STALL_GEN_EN__SHIFT                                                      0x9
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_PROP_EN__SHIFT                                                        0xa
+#define BIFL_RAS_LEAF19_CTRL__EGRESS_STALL_PROP_EN__SHIFT                                                     0xb
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_RAS_INTR_EN__SHIFT                                                    0x10
+#define BIFL_RAS_LEAF19_CTRL__PARITY_ERREVENT_LOG_MCA__SHIFT                                                  0x11
+#define BIFL_RAS_LEAF19_CTRL__POISON_ERREVENT_LOG_MCA__SHIFT                                                  0x12
+#define BIFL_RAS_LEAF19_CTRL__TIMEOUT_ERREVENT_LOG_MCA__SHIFT                                                 0x13
+#define BIFL_RAS_LEAF19_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                             0x14
+#define BIFL_RAS_LEAF19_CTRL__UCP_EN__SHIFT                                                                   0x15
+#define BIFL_RAS_LEAF19_CTRL__POISON_PROPAGATION_LOG_MCA__SHIFT                                               0x16
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_PARITY_CHECK_EN__SHIFT                                                0x17
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_DET_EN_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF19_CTRL__POISON_ERREVENT_EN_MASK                                                         0x00000002L
+#define BIFL_RAS_LEAF19_CTRL__POISON_STALL_EN_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF19_CTRL__PARITY_ERREVENT_EN_MASK                                                         0x00000008L
+#define BIFL_RAS_LEAF19_CTRL__PARITY_STALL_EN_MASK                                                            0x00000010L
+#define BIFL_RAS_LEAF19_CTRL__RCVERREVENT_ERREVENT_EN_MASK                                                    0x00000020L
+#define BIFL_RAS_LEAF19_CTRL__RCVERREVENT_STALL_EN_MASK                                                       0x00000040L
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_GEN_EN_MASK                                                           0x00000100L
+#define BIFL_RAS_LEAF19_CTRL__EGRESS_STALL_GEN_EN_MASK                                                        0x00000200L
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_PROP_EN_MASK                                                          0x00000400L
+#define BIFL_RAS_LEAF19_CTRL__EGRESS_STALL_PROP_EN_MASK                                                       0x00000800L
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_RAS_INTR_EN_MASK                                                      0x00010000L
+#define BIFL_RAS_LEAF19_CTRL__PARITY_ERREVENT_LOG_MCA_MASK                                                    0x00020000L
+#define BIFL_RAS_LEAF19_CTRL__POISON_ERREVENT_LOG_MCA_MASK                                                    0x00040000L
+#define BIFL_RAS_LEAF19_CTRL__TIMEOUT_ERREVENT_LOG_MCA_MASK                                                   0x00080000L
+#define BIFL_RAS_LEAF19_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                               0x00100000L
+#define BIFL_RAS_LEAF19_CTRL__UCP_EN_MASK                                                                     0x00200000L
+#define BIFL_RAS_LEAF19_CTRL__POISON_PROPAGATION_LOG_MCA_MASK                                                 0x00400000L
+#define BIFL_RAS_LEAF19_CTRL__ERR_EVENT_PARITY_CHECK_EN_MASK                                                  0x00800000L
+//BIFL_RAS_LEAF2_MISC_CTRL
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_HSTRSP_SHUB_DROP_EN__SHIFT           0x0
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_HSTRSP_CDC_DROP_EN__SHIFT            0x1
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_IHINTR_PORT_MASK_DIS__SHIFT          0x8
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_IHINTR_TRANS_MASK_DIS__SHIFT         0x9
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_ATHUB_RAS_ACTION_DIS_DMA_REQ__SHIFT      0xb
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_ATHUB_RAS_ACTION_EN_DMA_REQ_CHAIN_CHK__SHIFT  0xc
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_ATHUB_RAS_ACTION_DIS_DMA_RSP__SHIFT      0xd
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_UNITID_EN__SHIFT  0x10
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_TAG_CONST_EN__SHIFT  0x11
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_HSTRSP_SHUB_DROP_EN_MASK             0x00000001L
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_HSTRSP_CDC_DROP_EN_MASK              0x00000002L
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_IHINTR_PORT_MASK_DIS_MASK            0x00000100L
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_IHINTR_TRANS_MASK_DIS_MASK           0x00000200L
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_ATHUB_RAS_ACTION_DIS_DMA_REQ_MASK        0x00000800L
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_ATHUB_RAS_ACTION_EN_DMA_REQ_CHAIN_CHK_MASK  0x00001000L
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_ATHUB_RAS_ACTION_DIS_DMA_RSP_MASK        0x00002000L
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_UNITID_EN_MASK  0x00010000L
+#define BIFL_RAS_LEAF2_MISC_CTRL__BIFL_RAS_LEAF2_MISC_CTRL_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_TAG_CONST_EN_MASK  0x00020000L
+//BIFL_RAS_LEAF2_MISC_CTRL2
+#define BIFL_RAS_LEAF2_MISC_CTRL2__BIFL_RAS_LEAF2_MISC_CTRL2_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_UNITID__SHIFT  0x0
+#define BIFL_RAS_LEAF2_MISC_CTRL2__BIFL_RAS_LEAF2_MISC_CTRL2_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_TAG__SHIFT    0xb
+#define BIFL_RAS_LEAF2_MISC_CTRL2__BIFL_RAS_LEAF2_MISC_CTRL2_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_TAG_OFFSET__SHIFT  0x15
+#define BIFL_RAS_LEAF2_MISC_CTRL2__BIFL_RAS_LEAF2_MISC_CTRL2_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_UNITID_MASK   0x000007FFL
+#define BIFL_RAS_LEAF2_MISC_CTRL2__BIFL_RAS_LEAF2_MISC_CTRL2_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_TAG_MASK      0x001FF800L
+#define BIFL_RAS_LEAF2_MISC_CTRL2__BIFL_RAS_LEAF2_MISC_CTRL2_ERR_EVENT_RAS_ATHUB_DUMMYCHAIN_REQ_TAG_OFFSET_MASK  0x7FE00000L
+//BIFL_RAS_LEAF_NTB_CTRL
+#define BIFL_RAS_LEAF_NTB_CTRL__RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                                           0x0
+#define BIFL_RAS_LEAF_NTB_CTRL__RCVERREVENT_ERREVENT_LOG_MCA_MASK                                             0x00000001L
+//BIFL_RAS_LEAF0_STATUS
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF0_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF0_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF0_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF0_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF0_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF0_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF0_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF0_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF0_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF1_STATUS
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF1_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF1_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF1_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF1_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF1_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF1_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF1_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF1_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF1_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF2_STATUS
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF2_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF2_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF2_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF2_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF2_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF2_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF2_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF2_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF2_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF3_STATUS
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF3_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF3_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF3_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF3_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF3_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF3_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF3_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF3_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF3_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF4_STATUS
+#define BIFL_RAS_LEAF4_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF4_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF4_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF4_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF4_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF4_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF4_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF4_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF4_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF4_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF4_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF4_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF4_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF4_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF5_STATUS
+#define BIFL_RAS_LEAF5_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF5_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF5_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF5_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF5_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF5_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF5_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF5_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF5_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF5_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF5_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF5_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF5_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF5_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF6_STATUS
+#define BIFL_RAS_LEAF6_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF6_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF6_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF6_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF6_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF6_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF6_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF6_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF6_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF6_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF6_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF6_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF6_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF6_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF7_STATUS
+#define BIFL_RAS_LEAF7_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF7_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF7_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF7_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF7_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF7_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF7_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF7_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF7_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF7_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF7_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF7_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF7_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF7_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF8_STATUS
+#define BIFL_RAS_LEAF8_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF8_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF8_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF8_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF8_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF8_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF8_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF8_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF8_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF8_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF8_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF8_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF8_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF8_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF9_STATUS
+#define BIFL_RAS_LEAF9_STATUS__ERR_EVENT_RECV__SHIFT                                                          0x0
+#define BIFL_RAS_LEAF9_STATUS__POISON_ERR_DET__SHIFT                                                          0x1
+#define BIFL_RAS_LEAF9_STATUS__PARITY_ERR_DET__SHIFT                                                          0x2
+#define BIFL_RAS_LEAF9_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                     0x8
+#define BIFL_RAS_LEAF9_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                                0x9
+#define BIFL_RAS_LEAF9_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                     0xa
+#define BIFL_RAS_LEAF9_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                                0xb
+#define BIFL_RAS_LEAF9_STATUS__ERR_EVENT_RECV_MASK                                                            0x00000001L
+#define BIFL_RAS_LEAF9_STATUS__POISON_ERR_DET_MASK                                                            0x00000002L
+#define BIFL_RAS_LEAF9_STATUS__PARITY_ERR_DET_MASK                                                            0x00000004L
+#define BIFL_RAS_LEAF9_STATUS__ERR_EVENT_GENN_STAT_MASK                                                       0x00000100L
+#define BIFL_RAS_LEAF9_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                  0x00000200L
+#define BIFL_RAS_LEAF9_STATUS__ERR_EVENT_PROP_STAT_MASK                                                       0x00000400L
+#define BIFL_RAS_LEAF9_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                  0x00000800L
+//BIFL_RAS_LEAF10_STATUS
+#define BIFL_RAS_LEAF10_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF10_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF10_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF10_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF10_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF10_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF10_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF10_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF10_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF10_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF10_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF10_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF10_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF10_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_RAS_LEAF11_STATUS
+#define BIFL_RAS_LEAF11_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF11_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF11_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF11_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF11_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF11_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF11_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF11_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF11_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF11_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF11_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF11_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF11_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF11_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_RAS_LEAF12_STATUS
+#define BIFL_RAS_LEAF12_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF12_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF12_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF12_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF12_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF12_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF12_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF12_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF12_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF12_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF12_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF12_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF12_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF12_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_RAS_LEAF13_STATUS
+#define BIFL_RAS_LEAF13_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF13_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF13_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF13_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF13_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF13_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF13_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF13_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF13_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF13_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF13_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF13_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF13_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF13_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_RAS_LEAF14_STATUS
+#define BIFL_RAS_LEAF14_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF14_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF14_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF14_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF14_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF14_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF14_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF14_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF14_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF14_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF14_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF14_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF14_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF14_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_RAS_LEAF15_STATUS
+#define BIFL_RAS_LEAF15_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF15_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF15_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF15_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF15_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF15_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF15_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF15_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF15_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF15_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF15_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF15_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF15_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF15_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_RAS_LEAF16_STATUS
+#define BIFL_RAS_LEAF16_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF16_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF16_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF16_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF16_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF16_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF16_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF16_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF16_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF16_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF16_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF16_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF16_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF16_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_RAS_LEAF17_STATUS
+#define BIFL_RAS_LEAF17_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF17_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF17_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF17_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF17_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF17_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF17_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF17_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF17_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF17_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF17_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF17_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF17_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF17_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_RAS_LEAF18_STATUS
+#define BIFL_RAS_LEAF18_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF18_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF18_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF18_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF18_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF18_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF18_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF18_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF18_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF18_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF18_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF18_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF18_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF18_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_RAS_LEAF19_STATUS
+#define BIFL_RAS_LEAF19_STATUS__ERR_EVENT_RECV__SHIFT                                                         0x0
+#define BIFL_RAS_LEAF19_STATUS__POISON_ERR_DET__SHIFT                                                         0x1
+#define BIFL_RAS_LEAF19_STATUS__PARITY_ERR_DET__SHIFT                                                         0x2
+#define BIFL_RAS_LEAF19_STATUS__ERR_EVENT_GENN_STAT__SHIFT                                                    0x8
+#define BIFL_RAS_LEAF19_STATUS__EGRESS_STALLED_GENN_STAT__SHIFT                                               0x9
+#define BIFL_RAS_LEAF19_STATUS__ERR_EVENT_PROP_STAT__SHIFT                                                    0xa
+#define BIFL_RAS_LEAF19_STATUS__EGRESS_STALLED_PROP_STAT__SHIFT                                               0xb
+#define BIFL_RAS_LEAF19_STATUS__ERR_EVENT_RECV_MASK                                                           0x00000001L
+#define BIFL_RAS_LEAF19_STATUS__POISON_ERR_DET_MASK                                                           0x00000002L
+#define BIFL_RAS_LEAF19_STATUS__PARITY_ERR_DET_MASK                                                           0x00000004L
+#define BIFL_RAS_LEAF19_STATUS__ERR_EVENT_GENN_STAT_MASK                                                      0x00000100L
+#define BIFL_RAS_LEAF19_STATUS__EGRESS_STALLED_GENN_STAT_MASK                                                 0x00000200L
+#define BIFL_RAS_LEAF19_STATUS__ERR_EVENT_PROP_STAT_MASK                                                      0x00000400L
+#define BIFL_RAS_LEAF19_STATUS__EGRESS_STALLED_PROP_STAT_MASK                                                 0x00000800L
+//BIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP
+#define BIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_POISON_INJ__SHIFT                    0x0
+#define BIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_PARITY_INJ__SHIFT                    0x4
+#define BIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP__IOHUB_ORIG_EGRESS_ORIGDATA_POISON_INJ__SHIFT                      0x8
+#define BIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP__IOHUB_ORIG_EGRESS_ORIGDATA_PARITY_INJ__SHIFT                      0xc
+#define BIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_POISON_INJ_MASK                      0x0000000FL
+#define BIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_PARITY_INJ_MASK                      0x000000F0L
+#define BIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP__IOHUB_ORIG_EGRESS_ORIGDATA_POISON_INJ_MASK                        0x00000F00L
+#define BIFL_DFV_POISON_INJ_IOHUB_ORIG_SDP__IOHUB_ORIG_EGRESS_ORIGDATA_PARITY_INJ_MASK                        0x0000F000L
+//BIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP
+#define BIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ__SHIFT                     0x0
+#define BIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ__SHIFT                     0x4
+#define BIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP__IOHUB_CMPL_EGRESS_RDRSPDATA_POISON_INJ__SHIFT                     0x8
+#define BIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP__IOHUB_CMPL_EGRESS_RDRSPDATA_PARITY_INJ__SHIFT                     0xc
+#define BIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_MASK                       0x0000000FL
+#define BIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_MASK                       0x000000F0L
+#define BIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP__IOHUB_CMPL_EGRESS_RDRSPDATA_POISON_INJ_MASK                       0x00000F00L
+#define BIFL_DFV_POISON_INJ_IOHUB_CMPL_SDP__IOHUB_CMPL_EGRESS_RDRSPDATA_PARITY_INJ_MASK                       0x0000F000L
+//BIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP
+#define BIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ__SHIFT                     0x0
+#define BIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ__SHIFT                     0x4
+#define BIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP__ATHUB_CMPL_EGRESS_RDRSPDATA_POISON_INJ__SHIFT                     0x8
+#define BIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP__ATHUB_CMPL_EGRESS_RDRSPDATA_PARITY_INJ__SHIFT                     0xc
+#define BIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_MASK                       0x0000000FL
+#define BIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_MASK                       0x000000F0L
+#define BIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP__ATHUB_CMPL_EGRESS_RDRSPDATA_POISON_INJ_MASK                       0x00000F00L
+#define BIFL_DFV_POISON_INJ_ATHUB_CMPL_SDP__ATHUB_CMPL_EGRESS_RDRSPDATA_PARITY_INJ_MASK                       0x0000F000L
+//BIFL_DFV_POISON_INJ_ATHUB_ORIG_SDP
+#define BIFL_DFV_POISON_INJ_ATHUB_ORIG_SDP__ATHUB_ORIG_EGRESS_ORIGDATA_POISON_INJ__SHIFT                      0x8
+#define BIFL_DFV_POISON_INJ_ATHUB_ORIG_SDP__ATHUB_ORIG_EGRESS_ORIGDATA_PARITY_INJ__SHIFT                      0xc
+#define BIFL_DFV_POISON_INJ_ATHUB_ORIG_SDP__ATHUB_ORIG_EGRESS_ORIGDATA_POISON_INJ_MASK                        0x00000F00L
+#define BIFL_DFV_POISON_INJ_ATHUB_ORIG_SDP__ATHUB_ORIG_EGRESS_ORIGDATA_PARITY_INJ_MASK                        0x0000F000L
+//BIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_POISON_INJ_CNT__SHIFT            0x0
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_PARITY_INJ_CNT__SHIFT            0x4
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP__IOHUB_ORIG_EGRESS_ORIGDATA_POISON_INJ_CNT__SHIFT              0x8
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP__IOHUB_ORIG_EGRESS_ORIGDATA_PARITY_INJ_CNT__SHIFT              0xc
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_POISON_INJ_CNT_MASK              0x0000000FL
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_PARITY_INJ_CNT_MASK              0x000000F0L
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP__IOHUB_ORIG_EGRESS_ORIGDATA_POISON_INJ_CNT_MASK                0x00000F00L
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_ORIG_SDP__IOHUB_ORIG_EGRESS_ORIGDATA_PARITY_INJ_CNT_MASK                0x0000F000L
+//BIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_CNT__SHIFT             0x0
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_CNT__SHIFT             0x4
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP__IOHUB_CMPL_EGRESS_RDRSPDATA_POISON_INJ_CNT__SHIFT             0x8
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP__IOHUB_CMPL_EGRESS_RDRSPDATA_PARITY_INJ_CNT__SHIFT             0xc
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_CNT_MASK               0x0000000FL
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_CNT_MASK               0x000000F0L
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP__IOHUB_CMPL_EGRESS_RDRSPDATA_POISON_INJ_CNT_MASK               0x00000F00L
+#define BIFL_DFV_POISON_INJ_CNT_IOHUB_CMPL_SDP__IOHUB_CMPL_EGRESS_RDRSPDATA_PARITY_INJ_CNT_MASK               0x0000F000L
+//BIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_CNT__SHIFT             0x0
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_CNT__SHIFT             0x4
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP__ATHUB_CMPL_EGRESS_RDRSPDATA_POISON_INJ_CNT__SHIFT             0x8
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP__ATHUB_CMPL_EGRESS_RDRSPDATA_PARITY_INJ_CNT__SHIFT             0xc
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_CNT_MASK               0x0000000FL
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_CNT_MASK               0x000000F0L
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP__ATHUB_CMPL_EGRESS_RDRSPDATA_POISON_INJ_CNT_MASK               0x00000F00L
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_CMPL_SDP__ATHUB_CMPL_EGRESS_RDRSPDATA_PARITY_INJ_CNT_MASK               0x0000F000L
+//BIFL_DFV_POISON_INJ_CNT_ATHUB_ORIG_SDP
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_ORIG_SDP__ATHUB_ORIG_EGRESS_ORIGDATA_POISON_INJ_CNT__SHIFT              0x8
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_ORIG_SDP__ATHUB_ORIG_EGRESS_ORIGDATA_PARITY_INJ_CNT__SHIFT              0xc
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_ORIG_SDP__ATHUB_ORIG_EGRESS_ORIGDATA_POISON_INJ_CNT_MASK                0x00000F00L
+#define BIFL_DFV_POISON_INJ_CNT_ATHUB_ORIG_SDP__ATHUB_ORIG_EGRESS_ORIGDATA_PARITY_INJ_CNT_MASK                0x0000F000L
+//BIFL_DFV_POISON_INJ_LOG_IOHUB_ORIG_SDP
+#define BIFL_DFV_POISON_INJ_LOG_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_POISON_INJ_LOG__SHIFT            0x0
+#define BIFL_DFV_POISON_INJ_LOG_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_PARITY_INJ_LOG__SHIFT            0x4
+#define BIFL_DFV_POISON_INJ_LOG_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_POISON_INJ_LOG_MASK              0x0000000FL
+#define BIFL_DFV_POISON_INJ_LOG_IOHUB_ORIG_SDP__IOHUB_ORIG_INGRESS_RDRSPDATA_PARITY_INJ_LOG_MASK              0x000000F0L
+//BIFL_DFV_POISON_INJ_LOG_IOHUB_CMPL_SDP
+#define BIFL_DFV_POISON_INJ_LOG_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_LOG__SHIFT             0x0
+#define BIFL_DFV_POISON_INJ_LOG_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_LOG__SHIFT             0x4
+#define BIFL_DFV_POISON_INJ_LOG_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_LOG_MASK               0x0000000FL
+#define BIFL_DFV_POISON_INJ_LOG_IOHUB_CMPL_SDP__IOHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_LOG_MASK               0x000000F0L
+//BIFL_DFV_POISON_INJ_LOG_ATHUB_CMPL_SDP
+#define BIFL_DFV_POISON_INJ_LOG_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_LOG__SHIFT             0x0
+#define BIFL_DFV_POISON_INJ_LOG_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_LOG__SHIFT             0x4
+#define BIFL_DFV_POISON_INJ_LOG_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_POISON_INJ_LOG_MASK               0x0000000FL
+#define BIFL_DFV_POISON_INJ_LOG_ATHUB_CMPL_SDP__ATHUB_CMPL_INGRESS_ORIGDATA_PARITY_INJ_LOG_MASK               0x000000F0L
+//BIFL_DFV_POISON_INJ_LOG_ATHUB_ORIG_SDP
+//BIFL_IOHUB_RAS_IH_CNTL
+#define BIFL_IOHUB_RAS_IH_CNTL__BIFL_RAS_IH_INTR_EN__SHIFT                                                    0x0
+#define BIFL_IOHUB_RAS_IH_CNTL__BIFL_RAS_IH_INTR_EN_MASK                                                      0x00000001L
+//BIFL_RAS_VWR_FROM_IOHUB
+#define BIFL_RAS_VWR_FROM_IOHUB__BIFL_RAS_IH_INTR_TRIG__SHIFT                                                 0x0
+#define BIFL_RAS_VWR_FROM_IOHUB__BIFL_RAS_IH_INTR_TRIG_MASK                                                   0x00000001L
+
+
+// addressBlock: nbif0_nbif0_rcc_dwn_dev0_BIFDEC1
+//RCC_DWN_DEV0_2_DN_PCIE_RESERVED
+#define RCC_DWN_DEV0_2_DN_PCIE_RESERVED__PCIE_RESERVED__SHIFT                                                 0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_RESERVED__PCIE_RESERVED_MASK                                                   0xFFFFFFFFL
+//RCC_DWN_DEV0_2_DN_PCIE_SCRATCH
+#define RCC_DWN_DEV0_2_DN_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                     0xFFFFFFFFL
+//RCC_DWN_DEV0_2_DN_PCIE_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__HWINIT_WR_LOCK__SHIFT                                                    0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN__SHIFT                                              0x7
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                              0x1e
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__HWINIT_WR_LOCK_MASK                                                      0x00000001L
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__UR_ERR_REPORT_DIS_DN_MASK                                                0x00000080L
+#define RCC_DWN_DEV0_2_DN_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                0x40000000L
+//RCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE__SHIFT                                0x19
+#define RCC_DWN_DEV0_2_DN_PCIE_CONFIG_CNTL__CI_EXTENDED_TAG_EN_OVERRIDE_MASK                                  0x06000000L
+//RCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2
+#define RCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE__SHIFT                                               0x1c
+#define RCC_DWN_DEV0_2_DN_PCIE_RX_CNTL2__FLR_EXTEND_MODE_MASK                                                 0x70000000L
+//RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                             0x7
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN__SHIFT                                   0x8
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                               0x00000080L
+#define RCC_DWN_DEV0_2_DN_PCIE_BUS_CNTL__AER_CPL_TIMEOUT_RO_DIS_SWDN_MASK                                     0x00000100L
+//RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                      0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                 0x1
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                 0x2
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                 0x3
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                 0x4
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                 0x5
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                        0x00000001L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                   0x00000002L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                   0x00000004L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                   0x00000008L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                   0x00000010L
+#define RCC_DWN_DEV0_2_DN_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                   0x00000020L
+//RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_EN__SHIFT                                                   0x0
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN__SHIFT                                                0x11
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP__SHIFT                                        0x15
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_EN_MASK                                                     0x00000001L
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MC_EN_MASK                                                  0x00020000L
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_F0__STRAP_F0_MSI_MULTI_CAP_MASK                                          0x00E00000L
+//RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN__SHIFT                                             0x18
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                          0x1d
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_CLK_PM_EN_MASK                                               0x01000000L
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                            0x20000000L
+//RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN__SHIFT                                    0x2
+#define RCC_DWN_DEV0_2_DN_PCIE_STRAP_MISC2__STRAP_MSTCPL_TIMEOUT_EN_MASK                                      0x00000004L
+
+
+// addressBlock: nbif0_nbif0_rcc_dwnp_dev0_BIFDEC1
+//RCC_DWNP_DEV0_2_PCIE_ERR_CNTL
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                               0x0
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                             0x8
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                    0xb
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                        0x11
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR__SHIFT                                               0x12
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR__SHIFT                                           0x13
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR__SHIFT                                              0x14
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                 0x00000001L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                               0x00000700L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                      0x00000800L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                          0x00020000L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__ERR_CORR_RCVD_CLR_MASK                                                 0x00040000L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__NONFATAL_ERR_RCVD_CLR_MASK                                             0x00080000L
+#define RCC_DWNP_DEV0_2_PCIE_ERR_CNTL__FATAL_ERR_RCVD_CLR_MASK                                                0x00100000L
+//RCC_DWNP_DEV0_2_PCIE_RX_CNTL
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                        0x8
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN__SHIFT                                              0x9
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                          0x14
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN__SHIFT                                     0x15
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS__SHIFT                                           0x1b
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                          0x00000100L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_DN_MASK                                                0x00000200L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                            0x00100000L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_DN_MASK                                       0x00200000L
+#define RCC_DWNP_DEV0_2_PCIE_RX_CNTL__RX_RCB_FLR_TIMEOUT_DIS_MASK                                             0x08000000L
+//RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                           0x0
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                           0x1
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                           0x2
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                           0x3
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                           0x4
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                             0x00000001L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                             0x00000002L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                             0x00000004L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                             0x00000008L
+#define RCC_DWNP_DEV0_2_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                             0x00000010L
+//RCC_DWNP_DEV0_2_PCIE_LC_CNTL2
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS__SHIFT                               0x0
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS__SHIFT                                     0x1b
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__DL_STATE_CHANGED_NOTIFICATION_DIS_MASK                                 0x00000001L
+#define RCC_DWNP_DEV0_2_PCIE_LC_CNTL2__LC_LINK_BW_NOTIFICATION_DIS_MASK                                       0x08000000L
+//RCC_DWNP_DEV0_2_PCIEP_STRAP_MISC
+#define RCC_DWNP_DEV0_2_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN__SHIFT                                          0xa
+#define RCC_DWNP_DEV0_2_PCIEP_STRAP_MISC__STRAP_MULTI_FUNC_EN_MASK                                            0x00000400L
+//RCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP
+#define RCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP__SHIFT                                     0x0
+#define RCC_DWNP_DEV0_2_LTR_MSG_INFO_FROM_EP__LTR_MSG_INFO_FROM_EP_MASK                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_rcc_ep_dev0_BIFDEC1
+//RCC_EP_DEV0_2_EP_PCIE_SCRATCH
+#define RCC_EP_DEV0_2_EP_PCIE_SCRATCH__PCIE_SCRATCH__SHIFT                                                    0x0
+#define RCC_EP_DEV0_2_EP_PCIE_SCRATCH__PCIE_SCRATCH_MASK                                                      0xFFFFFFFFL
+//RCC_EP_DEV0_2_EP_PCIE_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__UR_ERR_REPORT_DIS__SHIFT                                                  0x7
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS__SHIFT                                            0x8
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR__SHIFT                                               0x1e
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__UR_ERR_REPORT_DIS_MASK                                                    0x00000080L
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__PCIE_MALFORM_ATOMIC_OPS_MASK                                              0x00000100L
+#define RCC_EP_DEV0_2_EP_PCIE_CNTL__RX_IGNORE_LTR_MSG_UR_MASK                                                 0x40000000L
+//RCC_EP_DEV0_2_EP_PCIE_INT_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN__SHIFT                                                0x0
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN__SHIFT                                           0x1
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN__SHIFT                                               0x2
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN__SHIFT                                            0x3
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN__SHIFT                                                0x4
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN__SHIFT                                         0x6
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__CORR_ERR_INT_EN_MASK                                                  0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__NON_FATAL_ERR_INT_EN_MASK                                             0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__FATAL_ERR_INT_EN_MASK                                                 0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__USR_DETECTED_INT_EN_MASK                                              0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__MISC_ERR_INT_EN_MASK                                                  0x00000010L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_CNTL__POWER_STATE_CHG_INT_EN_MASK                                           0x00000040L
+//RCC_EP_DEV0_2_EP_PCIE_INT_STATUS
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS__SHIFT                                          0x0
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS__SHIFT                                     0x1
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS__SHIFT                                         0x2
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS__SHIFT                                      0x3
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS__SHIFT                                          0x4
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS__SHIFT                                   0x6
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0__SHIFT                                0x7
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__CORR_ERR_INT_STATUS_MASK                                            0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__NON_FATAL_ERR_INT_STATUS_MASK                                       0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__FATAL_ERR_INT_STATUS_MASK                                           0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__USR_DETECTED_INT_STATUS_MASK                                        0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__MISC_ERR_INT_STATUS_MASK                                            0x00000010L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_MASK                                     0x00000040L
+#define RCC_EP_DEV0_2_EP_PCIE_INT_STATUS__POWER_STATE_CHG_INT_STATUS_F0_MASK                                  0x00000080L
+//RCC_EP_DEV0_2_EP_PCIE_RX_CNTL2
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR__SHIFT                                   0x0
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL2__RX_IGNORE_EP_INVALIDPASID_UR_MASK                                     0x00000001L
+//RCC_EP_DEV0_2_EP_PCIE_BUS_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS__SHIFT                                              0x7
+#define RCC_EP_DEV0_2_EP_PCIE_BUS_CNTL__IMMEDIATE_PMI_DIS_MASK                                                0x00000080L
+//RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG__SHIFT                                       0x0
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG__SHIFT                                  0x1
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG__SHIFT                                  0x2
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG__SHIFT                                  0x3
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG__SHIFT                                  0x4
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG__SHIFT                                  0x5
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_HIDDEN_REG_MASK                                         0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN2_HIDDEN_REG_MASK                                    0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN3_HIDDEN_REG_MASK                                    0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN4_HIDDEN_REG_MASK                                    0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN5_HIDDEN_REG_MASK                                    0x00000010L
+#define RCC_EP_DEV0_2_EP_PCIE_CFG_CNTL__CFG_EN_DEC_TO_GEN6_HIDDEN_REG_MASK                                    0x00000020L
+//RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE__SHIFT                                      0x0
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE__SHIFT                                       0x3
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT__SHIFT                                      0x6
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE__SHIFT                                     0x7
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE__SHIFT                                      0xa
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT__SHIFT                                     0xd
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0__SHIFT                               0xe
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN__SHIFT                                 0xf
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1__SHIFT                                            0x10
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN__SHIFT                                   0x11
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_SHORT_VALUE_MASK                                        0x00000007L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_LONG_VALUE_MASK                                         0x00000038L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_S_REQUIREMENT_MASK                                        0x00000040L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_SHORT_VALUE_MASK                                       0x00000380L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_LONG_VALUE_MASK                                        0x00001C00L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_NS_REQUIREMENT_MASK                                       0x00002000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_MSG_DIS_IN_PM_NON_D0_MASK                                 0x00004000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_PRIV_RST_LTR_IN_DL_DOWN_MASK                                   0x00008000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__TX_CHK_FC_FOR_L1_MASK                                              0x00010000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_LTR_CNTL__LTR_DSTATE_USING_WDATA_EN_MASK                                     0x00020000L
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_1_PCIE_F1_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN__SHIFT                                           0x1d
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC__STRAP_MST_ADR64_EN_MASK                                             0x20000000L
+//RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED__SHIFT                                         0x4
+#define RCC_EP_DEV0_2_EP_PCIE_STRAP_MISC2__STRAP_TPH_SUPPORTED_MASK                                           0x00000010L
+//RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT__SHIFT                                               0x8
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE__SHIFT                                              0xc
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0__SHIFT                                              0x10
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1__SHIFT                                              0x18
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_UNIT_MASK                                                 0x00000300L
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__PWR_ALLOC_SCALE_MASK                                                0x00003000L
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_0_MASK                                                0x00FF0000L
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CAP__TRANS_LAT_VAL_1_MASK                                                0xFF000000L
+//RCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS__SHIFT                       0x0
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_LATENCY_INDICATOR__TRANS_LAT_INDICATOR_BITS_MASK                         0xFFL
+//RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS__SHIFT                                             0x0
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE__SHIFT                                         0x8
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__SUBSTATE_STATUS_MASK                                               0x001FL
+#define RCC_EP_DEV0_2_EP_PCIE_F0_DPA_CNTL__DPA_COMPLIANCE_MODE_MASK                                           0x0100L
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_0__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_1__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_2__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_3__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_4__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_5__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_6__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC__SHIFT                             0x0
+#define RCC_EP_DEV0_2_PCIE_F0_DPA_SUBSTATE_PWR_ALLOC_7__SUBSTATE_PWR_ALLOC_MASK                               0xFFL
+//RCC_EP_DEV0_2_EP_PCIE_PME_CONTROL
+#define RCC_EP_DEV0_2_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER__SHIFT                                           0x0
+#define RCC_EP_DEV0_2_EP_PCIE_PME_CONTROL__PME_SERVICE_TIMER_MASK                                             0x1FL
+//RCC_EP_DEV0_2_EP_PCIEP_RESERVED
+#define RCC_EP_DEV0_2_EP_PCIEP_RESERVED__PCIEP_RESERVED__SHIFT                                                0x0
+#define RCC_EP_DEV0_2_EP_PCIEP_RESERVED__PCIEP_RESERVED_MASK                                                  0xFFFFFFFFL
+//RCC_EP_DEV0_2_EP_PCIE_TX_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE__SHIFT                                                 0xa
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE__SHIFT                                                  0xc
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS__SHIFT                                                   0x18
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS__SHIFT                                                   0x19
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS__SHIFT                                                   0x1a
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_SNR_OVERRIDE_MASK                                                   0x00000C00L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_RO_OVERRIDE_MASK                                                    0x00003000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F0_TPH_DIS_MASK                                                     0x01000000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F1_TPH_DIS_MASK                                                     0x02000000L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_CNTL__TX_F2_TPH_DIS_MASK                                                     0x04000000L
+//RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION__SHIFT                                0x0
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE__SHIFT                                  0x3
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS__SHIFT                                     0x8
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_FUNCTION_MASK                                  0x00000007L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_DEVICE_MASK                                    0x000000F8L
+#define RCC_EP_DEV0_2_EP_PCIE_TX_REQUESTER_ID__TX_REQUESTER_ID_BUS_MASK                                       0x0000FF00L
+//RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS__SHIFT                                              0x0
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT__SHIFT                                            0x8
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY__SHIFT                                       0x11
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL__SHIFT                               0x12
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED__SHIFT                                   0x18
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED__SHIFT                                   0x19
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED__SHIFT                                   0x1a
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED__SHIFT                                   0x1b
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED__SHIFT                                   0x1c
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED__SHIFT                                   0x1d
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED__SHIFT                                   0x1e
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED__SHIFT                                   0x1f
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__ERR_REPORTING_DIS_MASK                                                0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_TIMEOUT_MASK                                              0x00000700L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__SEND_ERR_MSG_IMMEDIATELY_MASK                                         0x00020000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__STRAP_POISONED_ADVISORY_NONFATAL_MASK                                 0x00040000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F0_TIMER_EXPIRED_MASK                                     0x01000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F1_TIMER_EXPIRED_MASK                                     0x02000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F2_TIMER_EXPIRED_MASK                                     0x04000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F3_TIMER_EXPIRED_MASK                                     0x08000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F4_TIMER_EXPIRED_MASK                                     0x10000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F5_TIMER_EXPIRED_MASK                                     0x20000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F6_TIMER_EXPIRED_MASK                                     0x40000000L
+#define RCC_EP_DEV0_2_EP_PCIE_ERR_CNTL__AER_HDR_LOG_F7_TIMER_EXPIRED_MASK                                     0x80000000L
+//RCC_EP_DEV0_2_EP_PCIE_RX_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR__SHIFT                                       0x8
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR__SHIFT                                                0x9
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS__SHIFT                                         0x14
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR__SHIFT                                       0x15
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR__SHIFT                                         0x16
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR__SHIFT                                      0x18
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR__SHIFT                                          0x19
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_TPH_DIS__SHIFT                                                      0x1a
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAX_PAYLOAD_ERR_MASK                                         0x00000100L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_TC_ERR_MASK                                                  0x00000200L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_PCIE_CPL_TIMEOUT_DIS_MASK                                           0x00100000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_SHORTPREFIX_ERR_MASK                                         0x00200000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_MAXPREFIX_ERR_MASK                                           0x00400000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_INVALIDPASID_ERR_MASK                                        0x01000000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_IGNORE_NOT_PASID_UR_MASK                                            0x02000000L
+#define RCC_EP_DEV0_2_EP_PCIE_RX_CNTL__RX_TPH_DIS_MASK                                                        0x04000000L
+//RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP__SHIFT                                          0x0
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP__SHIFT                                          0x1
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP__SHIFT                                          0x2
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP__SHIFT                                          0x3
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP__SHIFT                                          0x4
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN2_EN_STRAP_MASK                                            0x00000001L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN3_EN_STRAP_MASK                                            0x00000002L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN4_EN_STRAP_MASK                                            0x00000004L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN5_EN_STRAP_MASK                                            0x00000008L
+#define RCC_EP_DEV0_2_EP_PCIE_LC_SPEED_CNTL__LC_GEN6_EN_STRAP_MASK                                            0x00000010L
+//RCC_EP_DEV0_2_EP_PCIE_DEVICE_CNTL3
+#define RCC_EP_DEV0_2_EP_PCIE_DEVICE_CNTL3__SHADOW_F0_DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE__SHIFT          0x2
+#define RCC_EP_DEV0_2_EP_PCIE_DEVICE_CNTL3__SHADOW_F0_DEVICE_CNTL3_14BIT_TAG_REQUESTER_ENABLE_MASK            0x00000004L
+
+
+// addressBlock: nbif0_nbif0_rcc_dev0_BIFDEC1
+//RCC_DEV0_1_RCC_ERR_INT_CNTL
+#define RCC_DEV0_1_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN__SHIFT                                0x0
+#define RCC_DEV0_1_RCC_ERR_INT_CNTL__INVALID_REG_ACCESS_IN_SRIOV_INT_EN_MASK                                  0x00000001L
+//RCC_DEV0_1_RCC_BACO_CNTL_MISC
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS__SHIFT                                                 0x0
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS__SHIFT                                                  0x1
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_ROM_REQ_DIS_MASK                                                   0x00000001L
+#define RCC_DEV0_1_RCC_BACO_CNTL_MISC__BIF_AZ_REQ_DIS_MASK                                                    0x00000002L
+//RCC_DEV0_1_RCC_RESET_EN
+#define RCC_DEV0_1_RCC_RESET_EN__DB_APER_RESET_EN__SHIFT                                                      0xf
+#define RCC_DEV0_1_RCC_RESET_EN__DB_APER_RESET_EN_MASK                                                        0x00008000L
+//RCC_DEV0_2_RCC_VDM_SUPPORT
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__MCTP_SUPPORT__SHIFT                                                       0x0
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__AMPTP_SUPPORT__SHIFT                                                      0x1
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT__SHIFT                                                  0x2
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE__SHIFT                                        0x3
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE__SHIFT                                    0x4
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__MCTP_SUPPORT_MASK                                                         0x00000001L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__AMPTP_SUPPORT_MASK                                                        0x00000002L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__OTHER_VDM_SUPPORT_MASK                                                    0x00000004L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_TO_RC_CHECK_IN_RCMODE_MASK                                          0x00000008L
+#define RCC_DEV0_2_RCC_VDM_SUPPORT__ROUTE_BROADCAST_CHECK_IN_RCMODE_MASK                                      0x00000010L
+//RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED__SHIFT                                 0x0
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING__SHIFT                              0x1
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE__SHIFT                                0x2
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER__SHIFT                                 0x3
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD__SHIFT                           0x4
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS__SHIFT                                  0x5
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET__SHIFT                                 0xb
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS__SHIFT                                 0x12
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET__SHIFT                                0x19
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_VOLTAGE_SUPPORTED_MASK                                   0x00000001L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_LEFTRIGHT_TIMING_MASK                                0x00000002L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_UPDOWN_VOLTAGE_MASK                                  0x00000004L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_IND_ERROR_SAMPLER_MASK                                   0x00000008L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_SAMPLE_REPORTING_METHOD_MASK                             0x00000010L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_TIMING_STEPS_MASK                                    0x000007E0L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_TIMING_OFFSET_MASK                                   0x0003F800L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_NUM_VOLTAGE_STEPS_MASK                                   0x01FC0000L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL0__MARGINING_MAX_VOLTAGE_OFFSET_MASK                                  0xFE000000L
+//RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE__SHIFT                             0x0
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING__SHIFT                              0x6
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES__SHIFT                                         0xc
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT__SHIFT                                      0x11
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_VOLTAGE_MASK                               0x0000003FL
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLING_RATE_TIMING_MASK                                0x00000FC0L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_MAX_LANES_MASK                                           0x0001F000L
+#define RCC_DEV0_2_RCC_MARGIN_PARAM_CNTL1__MARGINING_SAMPLE_COUNT_MASK                                        0x00FE0000L
+//RCC_DEV0_1_RCC_GPUIOV_REGION
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__LFB_REGION__SHIFT                                                       0x0
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__MAX_REGION__SHIFT                                                       0x4
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__LFB_REGION_MASK                                                         0x0000000FL
+#define RCC_DEV0_1_RCC_GPUIOV_REGION__MAX_REGION_MASK                                                         0x000000F0L
+//RCC_DEV0_1_RCC_GPU_HOSTVM_EN
+#define RCC_DEV0_1_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN__SHIFT                                                    0x0
+#define RCC_DEV0_1_RCC_GPU_HOSTVM_EN__GPU_HOSTVM_EN_MASK                                                      0x00000001L
+//RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE__SHIFT                              0x0
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN__SHIFT                                    0x1
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__RCC_CONSOLE_IOV_MODE_ENABLE_MASK                                0x00000001L
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_MODE_CNTL__MULTIOS_IH_SUPPORT_EN_MASK                                      0x00000002L
+//RCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET__SHIFT                        0x0
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_FIRST_VF_OFFSET__CONSOLE_IOV_FIRST_VF_OFFSET_MASK                          0xFFFFL
+//RCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE__SHIFT                                    0x0
+#define RCC_DEV0_1_RCC_CONSOLE_IOV_VF_STRIDE__CONSOLE_IOV_VF_STRIDE_MASK                                      0xFFFFL
+//RCC_DEV0_1_RCC_PEER_REG_RANGE0
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE0__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_1_RCC_PEER_REG_RANGE1
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__START_ADDR__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__END_ADDR__SHIFT                                                       0x10
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__START_ADDR_MASK                                                       0x0000FFFFL
+#define RCC_DEV0_1_RCC_PEER_REG_RANGE1__END_ADDR_MASK                                                         0xFFFF0000L
+//RCC_DEV0_2_RCC_BUS_CNTL
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS__SHIFT                                                            0x2
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS__SHIFT                                                           0x3
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_BM_DIS__SHIFT                                                            0x4
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_DN__SHIFT                                                         0x5
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_DN__SHIFT                                                        0x6
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_UP__SHIFT                                                         0x7
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_UP__SHIFT                                                        0x8
+#define RCC_DEV0_2_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT__SHIFT                                                 0xc
+#define RCC_DEV0_2_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC__SHIFT                                           0xd
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x10
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x11
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x12
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR__SHIFT                                          0x13
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR__SHIFT                                          0x14
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR__SHIFT                                          0x15
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE__SHIFT                                                 0x18
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE__SHIFT                                                 0x19
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE__SHIFT                                            0x1c
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE__SHIFT                                            0x1d
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_MASK                                                              0x00000004L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_MASK                                                             0x00000008L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_BM_DIS_MASK                                                              0x00000010L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_DN_MASK                                                           0x00000020L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_DN_MASK                                                          0x00000040L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_IO_DIS_UP_MASK                                                           0x00000080L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PMI_MEM_DIS_UP_MASK                                                          0x00000100L
+#define RCC_DEV0_2_RCC_BUS_CNTL__ROOT_ERR_LOG_ON_EVENT_MASK                                                   0x00001000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__HOST_CPL_POISONED_LOG_IN_RC_MASK                                             0x00002000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00010000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00020000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_SEC_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00040000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_SIG_CPLCA_WITH_EP_ERR_MASK                                            0x00080000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLCA_WITH_EP_ERR_MASK                                            0x00100000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__DN_PRI_RCV_CPLUR_WITH_EP_ERR_MASK                                            0x00200000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_PAYLOAD_SIZE_MODE_MASK                                                   0x01000000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_PAYLOAD_SIZE_MASK                                                   0x0E000000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__MAX_READ_REQUEST_SIZE_MODE_MASK                                              0x10000000L
+#define RCC_DEV0_2_RCC_BUS_CNTL__PRIV_MAX_READ_REQUEST_SIZE_MASK                                              0xE0000000L
+//RCC_DEV0_1_RCC_CONFIG_CNTL
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B__SHIFT                                               0x2
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GRPH_ADRSEL__SHIFT                                                        0x3
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__CFG_VGA_RAM_EN_MASK                                                       0x00000001L
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GENMO_MONO_ADDRESS_B_MASK                                                 0x00000004L
+#define RCC_DEV0_1_RCC_CONFIG_CNTL__GRPH_ADRSEL_MASK                                                          0x00000018L
+//RCC_DEV0_1_RCC_CONFIG_F0_BASE
+#define RCC_DEV0_1_RCC_CONFIG_F0_BASE__F0_BASE__SHIFT                                                         0x0
+#define RCC_DEV0_1_RCC_CONFIG_F0_BASE__F0_BASE_MASK                                                           0xFFFFFFFFL
+//RCC_DEV0_1_RCC_CONFIG_APER_SIZE
+#define RCC_DEV0_1_RCC_CONFIG_APER_SIZE__APER_SIZE__SHIFT                                                     0x0
+#define RCC_DEV0_1_RCC_CONFIG_APER_SIZE__APER_SIZE_MASK                                                       0xFFFFFFFFL
+//RCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE
+#define RCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE__SHIFT                                             0x0
+#define RCC_DEV0_1_RCC_CONFIG_REG_APER_SIZE__REG_APER_SIZE_MASK                                               0x07FFFFFFL
+//RCC_DEV0_1_RCC_XDMA_LO
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_APER_EN__SHIFT                                                       0x1f
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_LOWER_BOUND_MASK                                                     0x7FFFFFFFL
+#define RCC_DEV0_1_RCC_XDMA_LO__BIF_XDMA_APER_EN_MASK                                                         0x80000000L
+//RCC_DEV0_1_RCC_XDMA_HI
+#define RCC_DEV0_1_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_XDMA_HI__BIF_XDMA_UPPER_BOUND_MASK                                                     0x7FFFFFFFL
+//RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS__SHIFT                                   0x7
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN__SHIFT                                 0x8
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR__SHIFT                                    0x9
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR__SHIFT                                    0xa
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR__SHIFT                                 0xb
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR__SHIFT                                  0xc
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR__SHIFT                                      0xd
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS__SHIFT                      0xe
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS__SHIFT                         0xf
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS__SHIFT                                 0x10
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS__SHIFT                           0x11
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN__SHIFT                               0x12
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS__SHIFT                     0x13
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__INIT_PFFLR_CRS_RET_DIS_MASK                                     0x00000080L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__ATC_PRG_RESP_PASID_UR_EN_MASK                                   0x00000100L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMRD_UR_MASK                                      0x00000200L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_TRANSMWR_UR_MASK                                      0x00000400L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_ATSTRANSREQ_UR_MASK                                   0x00000800L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_PAGEREQMSG_UR_MASK                                    0x00001000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__RX_IGNORE_INVCPL_UR_MASK                                        0x00002000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_X_PENDING_WHEN_DISABLED_DIS_MASK                        0x00004000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CHECK_BME_ON_PENDING_PKT_GEN_DIS_MASK                           0x00008000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__PSN_CHECK_ON_PAYLOAD_DIS_MASK                                   0x00010000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__CLR_MSI_PENDING_ON_MULTIEN_DIS_MASK                             0x00020000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__SET_DEVICE_ERR_FOR_ECRC_EN_MASK                                 0x00040000L
+#define RCC_DEV0_2_RCC_FEATURES_CONTROL_MISC__HOST_POISON_FLAG_CHECK_FOR_CHAIN_DIS_MASK                       0x00080000L
+//RCC_DEV0_1_RCC_BUSNUM_CNTL1
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL1__ID_MASK__SHIFT                                                           0x0
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL1__ID_MASK_MASK                                                             0x000000FFL
+//RCC_DEV0_1_RCC_BUSNUM_LIST0
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID0__SHIFT                                                               0x0
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID1__SHIFT                                                               0x8
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID2__SHIFT                                                               0x10
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID3__SHIFT                                                               0x18
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID0_MASK                                                                 0x000000FFL
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID1_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID2_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST0__ID3_MASK                                                                 0xFF000000L
+//RCC_DEV0_1_RCC_BUSNUM_LIST1
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID4__SHIFT                                                               0x0
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID5__SHIFT                                                               0x8
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID6__SHIFT                                                               0x10
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID7__SHIFT                                                               0x18
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID4_MASK                                                                 0x000000FFL
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID5_MASK                                                                 0x0000FF00L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID6_MASK                                                                 0x00FF0000L
+#define RCC_DEV0_1_RCC_BUSNUM_LIST1__ID7_MASK                                                                 0xFF000000L
+//RCC_DEV0_1_RCC_BUSNUM_CNTL2
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL__SHIFT                                                    0x0
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN__SHIFT                                                     0x8
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__HDPREG_CNTL__SHIFT                                                       0x10
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH__SHIFT                                           0x11
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_SEL_MASK                                                      0x000000FFL
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__AUTOUPDATE_EN_MASK                                                       0x00000100L
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__HDPREG_CNTL_MASK                                                         0x00010000L
+#define RCC_DEV0_1_RCC_BUSNUM_CNTL2__ERROR_MULTIPLE_ID_MATCH_MASK                                             0x00020000L
+//RCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM
+#define RCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_CAPTURE_HOST_BUSNUM__CHECK_EN_MASK                                                     0x00000001L
+//RCC_DEV0_1_RCC_HOST_BUSNUM
+#define RCC_DEV0_1_RCC_HOST_BUSNUM__HOST_ID__SHIFT                                                            0x0
+#define RCC_DEV0_1_RCC_HOST_BUSNUM__HOST_ID_MASK                                                              0x0000FFFFL
+//RCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_HI__PEER0_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER0_FB_OFFSET_LO__PEER0_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_HI__PEER1_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER1_FB_OFFSET_LO__PEER1_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_HI__PEER2_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER2_FB_OFFSET_LO__PEER2_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_HI__PEER3_FB_OFFSET_HI_MASK                                            0x000FFFFFL
+//RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO__SHIFT                                          0x0
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN__SHIFT                                                 0x1f
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_OFFSET_LO_MASK                                            0x000FFFFFL
+#define RCC_DEV0_1_RCC_PEER3_FB_OFFSET_LO__PEER3_FB_EN_MASK                                                   0x80000000L
+//RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1__SHIFT                                                   0x8
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2__SHIFT                                                   0x10
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3__SHIFT                                                   0x18
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID0_MASK                                                     0x000000FFL
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID1_MASK                                                     0x0000FF00L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID2_MASK                                                     0x00FF0000L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST0__DEVFUNC_ID3_MASK                                                     0xFF000000L
+//RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4__SHIFT                                                   0x0
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5__SHIFT                                                   0x8
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6__SHIFT                                                   0x10
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7__SHIFT                                                   0x18
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID4_MASK                                                     0x000000FFL
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID5_MASK                                                     0x0000FF00L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID6_MASK                                                     0x00FF0000L
+#define RCC_DEV0_1_RCC_DEVFUNCNUM_LIST1__DEVFUNC_ID7_MASK                                                     0xFF000000L
+//RCC_DEV0_2_RCC_DEV0_LINK_CNTL
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT__SHIFT                                                  0x0
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY__SHIFT                                                 0x8
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS__SHIFT                                            0x10
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS__SHIFT                                            0x11
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_EXIT_MASK                                                    0x00000001L
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__LINK_DOWN_ENTRY_MASK                                                   0x00000100L
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_SRB_RST_TLS_DIS_MASK                                              0x00010000L
+#define RCC_DEV0_2_RCC_DEV0_LINK_CNTL__SWUS_LDN_RST_TLS_DIS_MASK                                              0x00020000L
+//RCC_DEV0_2_RCC_CMN_LINK_CNTL
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS__SHIFT                                             0x0
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS__SHIFT                                              0x1
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS__SHIFT                                             0x2
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN__SHIFT                                          0x3
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER__SHIFT                                             0x10
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L0S_DIS_MASK                                               0x00000001L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_L1_DIS_MASK                                                0x00000002L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__BLOCK_PME_ON_LDN_DIS_MASK                                               0x00000004L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__PM_L1_IDLE_CHECK_DMA_EN_MASK                                            0x00000008L
+#define RCC_DEV0_2_RCC_CMN_LINK_CNTL__VLINK_IN_L1LTR_TIMER_MASK                                               0xFFFF0000L
+//RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS__SHIFT                                            0x0
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV__SHIFT                                            0x8
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG__SHIFT                                            0x10
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_BUS_MASK                                              0x000000FFL
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_DEV_MASK                                              0x00001F00L
+#define RCC_DEV0_2_RCC_EP_REQUESTERID_RESTORE__EP_REQID_SEG_MASK                                              0x00FF0000L
+//RCC_DEV0_2_RCC_LTR_LSWITCH_CNTL
+#define RCC_DEV0_2_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE__SHIFT                                         0x0
+#define RCC_DEV0_2_RCC_LTR_LSWITCH_CNTL__LSWITCH_LATENCY_VALUE_MASK                                           0x000003FFL
+//RCC_DEV0_2_RCC_MH_ARB_CNTL
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_MODE__SHIFT                                                        0x0
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY__SHIFT                                                0x1
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_MODE_MASK                                                          0x00000001L
+#define RCC_DEV0_2_RCC_MH_ARB_CNTL__MH_ARB_FIX_PRIORITY_MASK                                                  0x0000FFFEL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_SYSDEC
+//BIF_BX1_SYSHUB_INDEX_OVLP
+#define BIF_BX1_SYSHUB_INDEX_OVLP__SYSHUB_OFFSET__SHIFT                                                       0x0
+#define BIF_BX1_SYSHUB_INDEX_OVLP__SYSHUB_OFFSET_MASK                                                         0x003FFFFFL
+//BIF_BX1_SYSHUB_DATA_OVLP
+#define BIF_BX1_SYSHUB_DATA_OVLP__SYSHUB_DATA__SHIFT                                                          0x0
+#define BIF_BX1_SYSHUB_DATA_OVLP__SYSHUB_DATA_MASK                                                            0xFFFFFFFFL
+//BIF_BX1_SYSHUB_INDEX_HI_OVLP
+#define BIF_BX1_SYSHUB_INDEX_HI_OVLP__SYSHUB_OFFSET_HI__SHIFT                                                 0x0
+#define BIF_BX1_SYSHUB_INDEX_HI_OVLP__SYSHUB_OFFSET_HI_MASK                                                   0x0000FFFFL
+//BIF_BX1_PCIE_INDEX
+#define BIF_BX1_PCIE_INDEX__PCIE_INDEX__SHIFT                                                                 0x0
+#define BIF_BX1_PCIE_INDEX__PCIE_INDEX_MASK                                                                   0xFFFFFFFFL
+//BIF_BX1_PCIE_DATA
+#define BIF_BX1_PCIE_DATA__PCIE_DATA__SHIFT                                                                   0x0
+#define BIF_BX1_PCIE_DATA__PCIE_DATA_MASK                                                                     0xFFFFFFFFL
+//BIF_BX1_PCIE_INDEX2
+#define BIF_BX1_PCIE_INDEX2__PCIE_INDEX2__SHIFT                                                               0x0
+#define BIF_BX1_PCIE_INDEX2__PCIE_INDEX2_MASK                                                                 0xFFFFFFFFL
+//BIF_BX1_PCIE_DATA2
+#define BIF_BX1_PCIE_DATA2__PCIE_DATA2__SHIFT                                                                 0x0
+#define BIF_BX1_PCIE_DATA2__PCIE_DATA2_MASK                                                                   0xFFFFFFFFL
+//BIF_BX1_PCIE_INDEX_HI
+#define BIF_BX1_PCIE_INDEX_HI__PCIE_INDEX_HI__SHIFT                                                           0x0
+#define BIF_BX1_PCIE_INDEX_HI__PCIE_INDEX_HI_MASK                                                             0x0000FFFFL
+//BIF_BX1_PCIE_INDEX2_HI
+#define BIF_BX1_PCIE_INDEX2_HI__PCIE_INDEX2_HI__SHIFT                                                         0x0
+#define BIF_BX1_PCIE_INDEX2_HI__PCIE_INDEX2_HI_MASK                                                           0x0000FFFFL
+//BIF_BX1_SBIOS_SCRATCH_0
+#define BIF_BX1_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_0__SBIOS_SCRATCH_0_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_1
+#define BIF_BX1_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_1__SBIOS_SCRATCH_1_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_2
+#define BIF_BX1_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_2__SBIOS_SCRATCH_2_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_3
+#define BIF_BX1_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_3__SBIOS_SCRATCH_3_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_0
+#define BIF_BX1_BIOS_SCRATCH_0__BIOS_SCRATCH_0__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_0__BIOS_SCRATCH_0_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_1
+#define BIF_BX1_BIOS_SCRATCH_1__BIOS_SCRATCH_1__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_1__BIOS_SCRATCH_1_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_2
+#define BIF_BX1_BIOS_SCRATCH_2__BIOS_SCRATCH_2__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_2__BIOS_SCRATCH_2_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_3
+#define BIF_BX1_BIOS_SCRATCH_3__BIOS_SCRATCH_3__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_3__BIOS_SCRATCH_3_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_4
+#define BIF_BX1_BIOS_SCRATCH_4__BIOS_SCRATCH_4__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_4__BIOS_SCRATCH_4_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_5
+#define BIF_BX1_BIOS_SCRATCH_5__BIOS_SCRATCH_5__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_5__BIOS_SCRATCH_5_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_6
+#define BIF_BX1_BIOS_SCRATCH_6__BIOS_SCRATCH_6__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_6__BIOS_SCRATCH_6_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_7
+#define BIF_BX1_BIOS_SCRATCH_7__BIOS_SCRATCH_7__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_7__BIOS_SCRATCH_7_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_8
+#define BIF_BX1_BIOS_SCRATCH_8__BIOS_SCRATCH_8__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_8__BIOS_SCRATCH_8_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_9
+#define BIF_BX1_BIOS_SCRATCH_9__BIOS_SCRATCH_9__SHIFT                                                         0x0
+#define BIF_BX1_BIOS_SCRATCH_9__BIOS_SCRATCH_9_MASK                                                           0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_10
+#define BIF_BX1_BIOS_SCRATCH_10__BIOS_SCRATCH_10__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_10__BIOS_SCRATCH_10_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_11
+#define BIF_BX1_BIOS_SCRATCH_11__BIOS_SCRATCH_11__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_11__BIOS_SCRATCH_11_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_12
+#define BIF_BX1_BIOS_SCRATCH_12__BIOS_SCRATCH_12__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_12__BIOS_SCRATCH_12_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_13
+#define BIF_BX1_BIOS_SCRATCH_13__BIOS_SCRATCH_13__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_13__BIOS_SCRATCH_13_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_14
+#define BIF_BX1_BIOS_SCRATCH_14__BIOS_SCRATCH_14__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_14__BIOS_SCRATCH_14_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIOS_SCRATCH_15
+#define BIF_BX1_BIOS_SCRATCH_15__BIOS_SCRATCH_15__SHIFT                                                       0x0
+#define BIF_BX1_BIOS_SCRATCH_15__BIOS_SCRATCH_15_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_BIF_RLC_INTR_CNTL
+//BIF_BX1_BIF_VCE_INTR_CNTL
+//BIF_BX1_BIF_UVD_INTR_CNTL
+//BIF_BX1_BIF_Engine_INTR_CNTL
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_CMD_COMPLETE__SHIFT                                              0x0
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_HANG_SELF_RECOVERED__SHIFT                                       0x1
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_HANG_NEED_FLR__SHIFT                                             0x2
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_VM_BUSY_TRANSITION__SHIFT                                        0x3
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_INST_SEL__SHIFT                                                  0x1b
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_CMD_COMPLETE_MASK                                                0x00000001L
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_HANG_SELF_RECOVERED_MASK                                         0x00000002L
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_HANG_NEED_FLR_MASK                                               0x00000004L
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_VM_BUSY_TRANSITION_MASK                                          0x00000008L
+#define BIF_BX1_BIF_Engine_INTR_CNTL__Engine_INST_SEL_MASK                                                    0xF8000000L
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR0__CAM_ADDR0_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR0__CAM_REMAP_ADDR0_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR1
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR1__CAM_ADDR1_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR1__CAM_REMAP_ADDR1_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR2
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR2__CAM_ADDR2_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR2__CAM_REMAP_ADDR2_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR3
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR3__CAM_ADDR3_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR3__CAM_REMAP_ADDR3_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR4
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR4__CAM_ADDR4_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR4__CAM_REMAP_ADDR4_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR5
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR5__CAM_ADDR5_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR5__CAM_REMAP_ADDR5_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR6
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR6__CAM_ADDR6_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR6__CAM_REMAP_ADDR6_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ADDR7
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ADDR7__CAM_ADDR7_MASK                                                         0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7__SHIFT                                           0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_REMAP_ADDR7__CAM_REMAP_ADDR7_MASK                                             0x000FFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_CNTL
+#define BIF_BX1_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE__SHIFT                                                       0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_CNTL__CAM_ENABLE_MASK                                                         0x000000FFL
+//BIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL
+#define BIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL__SHIFT                                                 0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ZERO_CPL__CAM_ZERO_CPL_MASK                                                   0xFFFFFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_ONE_CPL
+#define BIF_BX1_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL__SHIFT                                                   0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_ONE_CPL__CAM_ONE_CPL_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL
+#define BIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL__SHIFT                                 0x0
+#define BIF_BX1_GFX_MMIOREG_CAM_PROGRAMMABLE_CPL__CAM_PROGRAMMABLE_CPL_MASK                                   0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_0
+#define BIF_BX1_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_0__DRIVER_SCRATCH_0_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_1
+#define BIF_BX1_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_1__DRIVER_SCRATCH_1_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_2
+#define BIF_BX1_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_2__DRIVER_SCRATCH_2_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_3
+#define BIF_BX1_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_3__DRIVER_SCRATCH_3_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_4
+#define BIF_BX1_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_4__DRIVER_SCRATCH_4_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_5
+#define BIF_BX1_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_5__DRIVER_SCRATCH_5_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_6
+#define BIF_BX1_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_6__DRIVER_SCRATCH_6_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_7
+#define BIF_BX1_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_7__DRIVER_SCRATCH_7_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_8
+#define BIF_BX1_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_8__DRIVER_SCRATCH_8_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_9
+#define BIF_BX1_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9__SHIFT                                                     0x0
+#define BIF_BX1_DRIVER_SCRATCH_9__DRIVER_SCRATCH_9_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_10
+#define BIF_BX1_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_10__DRIVER_SCRATCH_10_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_11
+#define BIF_BX1_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_11__DRIVER_SCRATCH_11_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_12
+#define BIF_BX1_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_12__DRIVER_SCRATCH_12_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_13
+#define BIF_BX1_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_13__DRIVER_SCRATCH_13_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_14
+#define BIF_BX1_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_14__DRIVER_SCRATCH_14_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_DRIVER_SCRATCH_15
+#define BIF_BX1_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15__SHIFT                                                   0x0
+#define BIF_BX1_DRIVER_SCRATCH_15__DRIVER_SCRATCH_15_MASK                                                     0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_0
+#define BIF_BX1_FW_SCRATCH_0__FW_SCRATCH_0__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_0__FW_SCRATCH_0_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_1
+#define BIF_BX1_FW_SCRATCH_1__FW_SCRATCH_1__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_1__FW_SCRATCH_1_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_2
+#define BIF_BX1_FW_SCRATCH_2__FW_SCRATCH_2__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_2__FW_SCRATCH_2_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_3
+#define BIF_BX1_FW_SCRATCH_3__FW_SCRATCH_3__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_3__FW_SCRATCH_3_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_4
+#define BIF_BX1_FW_SCRATCH_4__FW_SCRATCH_4__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_4__FW_SCRATCH_4_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_5
+#define BIF_BX1_FW_SCRATCH_5__FW_SCRATCH_5__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_5__FW_SCRATCH_5_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_6
+#define BIF_BX1_FW_SCRATCH_6__FW_SCRATCH_6__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_6__FW_SCRATCH_6_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_7
+#define BIF_BX1_FW_SCRATCH_7__FW_SCRATCH_7__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_7__FW_SCRATCH_7_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_8
+#define BIF_BX1_FW_SCRATCH_8__FW_SCRATCH_8__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_8__FW_SCRATCH_8_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_9
+#define BIF_BX1_FW_SCRATCH_9__FW_SCRATCH_9__SHIFT                                                             0x0
+#define BIF_BX1_FW_SCRATCH_9__FW_SCRATCH_9_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_10
+#define BIF_BX1_FW_SCRATCH_10__FW_SCRATCH_10__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_10__FW_SCRATCH_10_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_11
+#define BIF_BX1_FW_SCRATCH_11__FW_SCRATCH_11__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_11__FW_SCRATCH_11_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_12
+#define BIF_BX1_FW_SCRATCH_12__FW_SCRATCH_12__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_12__FW_SCRATCH_12_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_13
+#define BIF_BX1_FW_SCRATCH_13__FW_SCRATCH_13__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_13__FW_SCRATCH_13_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_14
+#define BIF_BX1_FW_SCRATCH_14__FW_SCRATCH_14__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_14__FW_SCRATCH_14_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_FW_SCRATCH_15
+#define BIF_BX1_FW_SCRATCH_15__FW_SCRATCH_15__SHIFT                                                           0x0
+#define BIF_BX1_FW_SCRATCH_15__FW_SCRATCH_15_MASK                                                             0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_4
+#define BIF_BX1_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_4__SBIOS_SCRATCH_4_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_5
+#define BIF_BX1_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_5__SBIOS_SCRATCH_5_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_6
+#define BIF_BX1_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_6__SBIOS_SCRATCH_6_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_7
+#define BIF_BX1_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_7__SBIOS_SCRATCH_7_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_8
+#define BIF_BX1_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_8__SBIOS_SCRATCH_8_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_9
+#define BIF_BX1_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9__SHIFT                                                       0x0
+#define BIF_BX1_SBIOS_SCRATCH_9__SBIOS_SCRATCH_9_MASK                                                         0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_10
+#define BIF_BX1_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_10__SBIOS_SCRATCH_10_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_11
+#define BIF_BX1_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_11__SBIOS_SCRATCH_11_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_12
+#define BIF_BX1_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_12__SBIOS_SCRATCH_12_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_13
+#define BIF_BX1_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_13__SBIOS_SCRATCH_13_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_14
+#define BIF_BX1_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_14__SBIOS_SCRATCH_14_MASK                                                       0xFFFFFFFFL
+//BIF_BX1_SBIOS_SCRATCH_15
+#define BIF_BX1_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15__SHIFT                                                     0x0
+#define BIF_BX1_SBIOS_SCRATCH_15__SBIOS_SCRATCH_15_MASK                                                       0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_pf_SYSPFVFDEC
+//BIF_BX_PF1_MM_INDEX
+#define BIF_BX_PF1_MM_INDEX__MM_OFFSET__SHIFT                                                                 0x0
+#define BIF_BX_PF1_MM_INDEX__MM_APER__SHIFT                                                                   0x1f
+#define BIF_BX_PF1_MM_INDEX__MM_OFFSET_MASK                                                                   0x7FFFFFFFL
+#define BIF_BX_PF1_MM_INDEX__MM_APER_MASK                                                                     0x80000000L
+//BIF_BX_PF1_MM_DATA
+#define BIF_BX_PF1_MM_DATA__MM_DATA__SHIFT                                                                    0x0
+#define BIF_BX_PF1_MM_DATA__MM_DATA_MASK                                                                      0xFFFFFFFFL
+//BIF_BX_PF1_MM_INDEX_HI
+#define BIF_BX_PF1_MM_INDEX_HI__MM_OFFSET_HI__SHIFT                                                           0x0
+#define BIF_BX_PF1_MM_INDEX_HI__MM_OFFSET_HI_MASK                                                             0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_BIFDEC1
+//BIF_BX1_CC_BIF_BX_STRAP0
+#define BIF_BX1_CC_BIF_BX_STRAP0__STRAP_RESERVED__SHIFT                                                       0x19
+#define BIF_BX1_CC_BIF_BX_STRAP0__STRAP_RESERVED_MASK                                                         0xFE000000L
+//BIF_BX1_CC_BIF_BX_PINSTRAP0
+//BIF_BX1_BIF_MM_INDACCESS_CNTL
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__WRITE_DIS__SHIFT                                                       0x0
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS__SHIFT                                                0x1
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__WRITE_DIS_MASK                                                         0x00000001L
+#define BIF_BX1_BIF_MM_INDACCESS_CNTL__MM_INDACCESS_DIS_MASK                                                  0x00000002L
+//BIF_BX1_BUS_CNTL
+#define BIF_BX1_BUS_CNTL__VGA_REG_COHERENCY_DIS__SHIFT                                                        0x6
+#define BIF_BX1_BUS_CNTL__VGA_MEM_COHERENCY_DIS__SHIFT                                                        0x7
+#define BIF_BX1_BUS_CNTL__SET_AZ_TC__SHIFT                                                                    0xa
+#define BIF_BX1_BUS_CNTL__SET_MC_TC__SHIFT                                                                    0xd
+#define BIF_BX1_BUS_CNTL__ZERO_BE_WR_EN__SHIFT                                                                0x10
+#define BIF_BX1_BUS_CNTL__ZERO_BE_RD_EN__SHIFT                                                                0x11
+#define BIF_BX1_BUS_CNTL__RD_STALL_IO_WR__SHIFT                                                               0x12
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS__SHIFT                                              0x18
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS__SHIFT                                          0x19
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS__SHIFT                                         0x1a
+#define BIF_BX1_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS__SHIFT                                        0x1b
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS__SHIFT                                          0x1c
+#define BIF_BX1_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN__SHIFT                                                     0x1d
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_WR_EN__SHIFT                                                          0x1e
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_RD_EN__SHIFT                                                          0x1f
+#define BIF_BX1_BUS_CNTL__VGA_REG_COHERENCY_DIS_MASK                                                          0x00000040L
+#define BIF_BX1_BUS_CNTL__VGA_MEM_COHERENCY_DIS_MASK                                                          0x00000080L
+#define BIF_BX1_BUS_CNTL__SET_AZ_TC_MASK                                                                      0x00001C00L
+#define BIF_BX1_BUS_CNTL__SET_MC_TC_MASK                                                                      0x0000E000L
+#define BIF_BX1_BUS_CNTL__ZERO_BE_WR_EN_MASK                                                                  0x00010000L
+#define BIF_BX1_BUS_CNTL__ZERO_BE_RD_EN_MASK                                                                  0x00020000L
+#define BIF_BX1_BUS_CNTL__RD_STALL_IO_WR_MASK                                                                 0x00040000L
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_DOORBELL_DIS_MASK                                                0x01000000L
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_FB_FLUSH_DIS_MASK                                            0x02000000L
+#define BIF_BX1_BUS_CNTL__PRECEEDINGWR_STALL_VGA_REG_FLUSH_DIS_MASK                                           0x04000000L
+#define BIF_BX1_BUS_CNTL__MMDAT_RD_HDP_TRIGGER_HDP_FB_FLUSH_DIS_MASK                                          0x08000000L
+#define BIF_BX1_BUS_CNTL__HDP_FB_FLUSH_STALL_MMDAT_RD_HDP_DIS_MASK                                            0x10000000L
+#define BIF_BX1_BUS_CNTL__HDP_REG_FLUSH_VF_MASK_EN_MASK                                                       0x20000000L
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_WR_EN_MASK                                                            0x40000000L
+#define BIF_BX1_BUS_CNTL__VGAFB_ZERO_BE_RD_EN_MASK                                                            0x80000000L
+//BIF_BX1_BIF_SCRATCH0
+#define BIF_BX1_BIF_SCRATCH0__BIF_SCRATCH0__SHIFT                                                             0x0
+#define BIF_BX1_BIF_SCRATCH0__BIF_SCRATCH0_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_BIF_SCRATCH1
+#define BIF_BX1_BIF_SCRATCH1__BIF_SCRATCH1__SHIFT                                                             0x0
+#define BIF_BX1_BIF_SCRATCH1__BIF_SCRATCH1_MASK                                                               0xFFFFFFFFL
+//BIF_BX1_BX_RESET_EN
+#define BIF_BX1_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN__SHIFT                                                  0x10
+#define BIF_BX1_BX_RESET_EN__RESET_ON_VFENABLE_LOW_EN_MASK                                                    0x00010000L
+//BIF_BX1_MM_CFGREGS_CNTL
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL__SHIFT                                                       0x0
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL__SHIFT                                                        0x6
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN__SHIFT                                                       0x1f
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_FUNC_SEL_MASK                                                         0x00000007L
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_CFG_DEV_SEL_MASK                                                          0x000000C0L
+#define BIF_BX1_MM_CFGREGS_CNTL__MM_WR_TO_CFG_EN_MASK                                                         0x80000000L
+//BIF_BX1_BX_RESET_CNTL
+#define BIF_BX1_BX_RESET_CNTL__LINK_TRAIN_EN__SHIFT                                                           0x0
+#define BIF_BX1_BX_RESET_CNTL__LINK_TRAIN_EN_MASK                                                             0x00000001L
+//BIF_BX1_INTERRUPT_CNTL
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE__SHIFT                                                   0x0
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_EN__SHIFT                                                         0x1
+#define BIF_BX1_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN__SHIFT                                                     0x3
+#define BIF_BX1_INTERRUPT_CNTL__IH_INTR_DLY_CNTR__SHIFT                                                       0x4
+#define BIF_BX1_INTERRUPT_CNTL__GEN_IH_INT_EN__SHIFT                                                          0x8
+#define BIF_BX1_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN__SHIFT                                                 0xf
+#define BIF_BX1_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN__SHIFT                                               0x10
+#define BIF_BX1_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS__SHIFT                                   0x11
+#define BIF_BX1_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN__SHIFT                                              0x12
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_OVERRIDE_MASK                                                     0x00000001L
+#define BIF_BX1_INTERRUPT_CNTL__IH_DUMMY_RD_EN_MASK                                                           0x00000002L
+#define BIF_BX1_INTERRUPT_CNTL__IH_REQ_NONSNOOP_EN_MASK                                                       0x00000008L
+#define BIF_BX1_INTERRUPT_CNTL__IH_INTR_DLY_CNTR_MASK                                                         0x000000F0L
+#define BIF_BX1_INTERRUPT_CNTL__GEN_IH_INT_EN_MASK                                                            0x00000100L
+#define BIF_BX1_INTERRUPT_CNTL__BIF_RB_REQ_NONSNOOP_EN_MASK                                                   0x00008000L
+#define BIF_BX1_INTERRUPT_CNTL__DUMMYRD_BYPASS_IN_MSI_EN_MASK                                                 0x00010000L
+#define BIF_BX1_INTERRUPT_CNTL__ALWAYS_SEND_INTPKT_AFTER_DUMMYRD_DIS_MASK                                     0x00020000L
+#define BIF_BX1_INTERRUPT_CNTL__BIF_RB_REQ_RELAX_ORDER_EN_MASK                                                0x00040000L
+//BIF_BX1_INTERRUPT_CNTL2
+#define BIF_BX1_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR__SHIFT                                                      0x0
+#define BIF_BX1_INTERRUPT_CNTL2__IH_DUMMY_RD_ADDR_MASK                                                        0xFFFFFFFFL
+//BIF_BX1_CLKREQB_PAD_CNTL
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_A__SHIFT                                                        0x0
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL__SHIFT                                                      0x1
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE__SHIFT                                                     0x2
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE__SHIFT                                                    0x3
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0__SHIFT                                                      0x5
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1__SHIFT                                                      0x6
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2__SHIFT                                                      0x7
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3__SHIFT                                                      0x8
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN__SHIFT                                                    0x9
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE__SHIFT                                                     0xa
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN__SHIFT                                                   0xb
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN__SHIFT                                                  0xc
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y__SHIFT                                                        0xd
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_A_MASK                                                          0x00000001L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SEL_MASK                                                        0x00000002L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_MODE_MASK                                                       0x00000004L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SPARE_MASK                                                      0x00000018L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN0_MASK                                                        0x00000020L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN1_MASK                                                        0x00000040L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN2_MASK                                                        0x00000080L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SN3_MASK                                                        0x00000100L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SLEWN_MASK                                                      0x00000200L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_WAKE_MASK                                                       0x00000400L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_SCHMEN_MASK                                                     0x00000800L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_EN_MASK                                                    0x00001000L
+#define BIF_BX1_CLKREQB_PAD_CNTL__CLKREQB_PAD_Y_MASK                                                          0x00002000L
+//BIF_BX1_BIF_FEATURES_CONTROL_MISC
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS__SHIFT                                          0x0
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS__SHIFT                                          0x1
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS__SHIFT                                          0x2
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS__SHIFT                                          0x3
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_RB_MSI_VEC_NOT_ENABLED_MODE__SHIFT                             0xb
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_RB_SET_OVERFLOW_EN__SHIFT                                      0xc
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS__SHIFT                                          0xd
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS__SHIFT                                       0xe
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BME_HDL_NONVIR_EN__SHIFT                                           0xf
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT__SHIFT                                           0x10
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR__SHIFT                   0x19
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__MST_BIF_REQ_EP_DIS_MASK                                            0x00000001L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__SLV_BIF_CPL_EP_DIS_MASK                                            0x00000002L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_SLV_REQ_EP_DIS_MASK                                            0x00000004L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_MST_CPL_EP_DIS_MASK                                            0x00000008L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_RB_MSI_VEC_NOT_ENABLED_MODE_MASK                               0x00000800L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BIF_RB_SET_OVERFLOW_EN_MASK                                        0x00001000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ERR_INT_DIS_MASK                                            0x00002000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__ATOMIC_ONLY_WRITE_DIS_MASK                                         0x00004000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__BME_HDL_NONVIR_EN_MASK                                             0x00008000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__HDP_NP_OSTD_LIMIT_MASK                                             0x01FF0000L
+#define BIF_BX1_BIF_FEATURES_CONTROL_MISC__DOORBELL_SELFRING_GPA_APER_CHK_48BIT_ADDR_MASK                     0x02000000L
+//BIF_BX1_HDP_ATOMIC_CONTROL_MISC
+#define BIF_BX1_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT__SHIFT                                      0x0
+#define BIF_BX1_HDP_ATOMIC_CONTROL_MISC__HDP_NP_ATOMIC_OSTD_LIMIT_MASK                                        0x000000FFL
+//BIF_BX1_BIF_DOORBELL_CNTL
+#define BIF_BX1_BIF_DOORBELL_CNTL__SELF_RING_DIS__SHIFT                                                       0x0
+#define BIF_BX1_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS__SHIFT                                                     0x1
+#define BIF_BX1_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN__SHIFT                                                    0x2
+#define BIF_BX1_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS__SHIFT                                         0x3
+#define BIF_BX1_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN__SHIFT                                                 0x4
+#define BIF_BX1_BIF_DOORBELL_CNTL__SELF_RING_DIS_LOCAL__SHIFT                                                 0x5
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS__SHIFT                                                  0x18
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0__SHIFT                                               0x19
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1__SHIFT                                               0x1a
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2__SHIFT                                               0x1b
+#define BIF_BX1_BIF_DOORBELL_CNTL__SELF_RING_DIS_MASK                                                         0x00000001L
+#define BIF_BX1_BIF_DOORBELL_CNTL__TRANS_CHECK_DIS_MASK                                                       0x00000002L
+#define BIF_BX1_BIF_DOORBELL_CNTL__UNTRANS_LBACK_EN_MASK                                                      0x00000004L
+#define BIF_BX1_BIF_DOORBELL_CNTL__NON_CONSECUTIVE_BE_ZERO_DIS_MASK                                           0x00000008L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DOORBELL_MONITOR_EN_MASK                                                   0x00000010L
+#define BIF_BX1_BIF_DOORBELL_CNTL__SELF_RING_DIS_LOCAL_MASK                                                   0x00000020L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_DIS_MASK                                                    0x01000000L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_0_MASK                                                 0x02000000L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_1_MASK                                                 0x04000000L
+#define BIF_BX1_BIF_DOORBELL_CNTL__DB_MNTR_INTGEN_MODE_2_MASK                                                 0x08000000L
+//BIF_BX1_BIF_DOORBELL_INT_CNTL
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS__SHIFT                                       0x0
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS__SHIFT                                      0x1
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS__SHIFT                            0x2
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR__SHIFT                                        0x10
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR__SHIFT                                       0x11
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR__SHIFT                             0x12
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                            0x17
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE__SHIFT                                      0x18
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE__SHIFT                                     0x19
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE__SHIFT                           0x1a
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                               0x1c
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1d
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE__SHIFT                          0x1e
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE__SHIFT                              0x1f
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_STATUS_MASK                                         0x00000001L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_STATUS_MASK                                        0x00000002L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_STATUS_MASK                              0x00000004L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_CLEAR_MASK                                          0x00010000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_CLEAR_MASK                                         0x00020000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_CLEAR_MASK                               0x00040000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_ERR_EVENT_INTERRUPT_ENABLE_MASK                              0x00800000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__DOORBELL_INTERRUPT_DISABLE_MASK                                        0x01000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_CNTLR_INTERRUPT_DISABLE_MASK                                       0x02000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__RAS_ATHUB_ERR_EVENT_INTERRUPT_DISABLE_MASK                             0x04000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_DB_INTR_STATUS_WHEN_RB_ENABLE_MASK                                 0x10000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_IOH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x20000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__SET_ATH_RAS_INTR_STATUS_WHEN_RB_ENABLE_MASK                            0x40000000L
+#define BIF_BX1_BIF_DOORBELL_INT_CNTL__TIMEOUT_ERR_EVENT_INTERRUPT_ENABLE_MASK                                0x80000000L
+//BIF_BX1_BIF_FB_EN
+#define BIF_BX1_BIF_FB_EN__FB_READ_EN__SHIFT                                                                  0x0
+#define BIF_BX1_BIF_FB_EN__FB_WRITE_EN__SHIFT                                                                 0x1
+#define BIF_BX1_BIF_FB_EN__FB_READ_EN_MASK                                                                    0x00000001L
+#define BIF_BX1_BIF_FB_EN__FB_WRITE_EN_MASK                                                                   0x00000002L
+//BIF_BX1_BIF_INTR_CNTL
+#define BIF_BX1_BIF_INTR_CNTL__RAS_INTR_VEC_SEL__SHIFT                                                        0x0
+#define BIF_BX1_BIF_INTR_CNTL__RAS_INTR_VEC_SEL_MASK                                                          0x00000001L
+//BIF_BX1_BIF_MST_TRANS_PENDING_VF
+#define BIF_BX1_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX1_BIF_MST_TRANS_PENDING_VF__BIF_MST_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX1_BIF_SLV_TRANS_PENDING_VF
+#define BIF_BX1_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING__SHIFT                                        0x0
+#define BIF_BX1_BIF_SLV_TRANS_PENDING_VF__BIF_SLV_TRANS_PENDING_MASK                                          0x7FFFFFFFL
+//BIF_BX1_BACO_CNTL
+#define BIF_BX1_BACO_CNTL__BACO_EN__SHIFT                                                                     0x0
+#define BIF_BX1_BACO_CNTL__BACO_DUMMY_EN__SHIFT                                                               0x2
+#define BIF_BX1_BACO_CNTL__BACO_POWER_OFF__SHIFT                                                              0x3
+#define BIF_BX1_BACO_CNTL__BACO_DSTATE_BYPASS__SHIFT                                                          0x5
+#define BIF_BX1_BACO_CNTL__BACO_RST_INTR_MASK__SHIFT                                                          0x6
+#define BIF_BX1_BACO_CNTL__BACO_MODE__SHIFT                                                                   0x8
+#define BIF_BX1_BACO_CNTL__RCU_BIF_CONFIG_DONE__SHIFT                                                         0x9
+#define BIF_BX1_BACO_CNTL__PWRGOOD_VDDSOC__SHIFT                                                              0x10
+#define BIF_BX1_BACO_CNTL__BACO_AUTO_EXIT__SHIFT                                                              0x1f
+#define BIF_BX1_BACO_CNTL__BACO_EN_MASK                                                                       0x00000001L
+#define BIF_BX1_BACO_CNTL__BACO_DUMMY_EN_MASK                                                                 0x00000004L
+#define BIF_BX1_BACO_CNTL__BACO_POWER_OFF_MASK                                                                0x00000008L
+#define BIF_BX1_BACO_CNTL__BACO_DSTATE_BYPASS_MASK                                                            0x00000020L
+#define BIF_BX1_BACO_CNTL__BACO_RST_INTR_MASK_MASK                                                            0x00000040L
+#define BIF_BX1_BACO_CNTL__BACO_MODE_MASK                                                                     0x00000100L
+#define BIF_BX1_BACO_CNTL__RCU_BIF_CONFIG_DONE_MASK                                                           0x00000200L
+#define BIF_BX1_BACO_CNTL__PWRGOOD_VDDSOC_MASK                                                                0x00010000L
+#define BIF_BX1_BACO_CNTL__BACO_AUTO_EXIT_MASK                                                                0x80000000L
+//BIF_BX1_BIF_BACO_EXIT_TIME0
+#define BIF_BX1_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER__SHIFT                                          0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIME0__BACO_EXIT_PXEN_CLR_TIMER_MASK                                            0x000FFFFFL
+//BIF_BX1_BIF_BACO_EXIT_TIMER1
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER__SHIFT                                         0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN__SHIFT                                            0x18
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS__SHIFT                                                 0x1a
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH__SHIFT                                           0x1b
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW__SHIFT                                            0x1c
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL__SHIFT                                                    0x1d
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS__SHIFT                                     0x1f
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_EXIT_SIDEBAND_TIMER_MASK                                           0x000FFFFFL
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_AUTO_FLUSH_EN_MASK                                              0x01000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_HW_EXIT_DIS_MASK                                                   0x04000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_HIGH_MASK                                             0x08000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__PX_EN_OE_IN_PX_EN_LOW_MASK                                              0x10000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__BACO_MODE_SEL_MASK                                                      0x60000000L
+#define BIF_BX1_BIF_BACO_EXIT_TIMER1__AUTO_BACO_EXIT_CLR_BY_HW_DIS_MASK                                       0x80000000L
+//BIF_BX1_BIF_BACO_EXIT_TIMER2
+#define BIF_BX1_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER__SHIFT                                         0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER2__BACO_EXIT_LCLK_BAK_TIMER_MASK                                           0x000FFFFFL
+//BIF_BX1_BIF_BACO_EXIT_TIMER3
+#define BIF_BX1_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER__SHIFT                                     0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER3__BACO_EXIT_DUMMY_EN_CLR_TIMER_MASK                                       0x000FFFFFL
+//BIF_BX1_BIF_BACO_EXIT_TIMER4
+#define BIF_BX1_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER__SHIFT                                      0x0
+#define BIF_BX1_BIF_BACO_EXIT_TIMER4__BACO_EXIT_BACO_EN_CLR_TIMER_MASK                                        0x000FFFFFL
+//BIF_BX1_MEM_TYPE_CNTL
+#define BIF_BX1_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3__SHIFT                                                        0x0
+#define BIF_BX1_MEM_TYPE_CNTL__BF_MEM_PHY_G5_G3_MASK                                                          0x00000001L
+//BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE__SHIFT                                                     0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE__SHIFT                                                  0x1
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE__SHIFT                                                    0x8
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_ENABLE_MASK                                                       0x00000001L
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__MSI_ADDR_MODE_MASK                                                    0x00000002L
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_CNTL__LUT_BC_MODE_MASK                                                      0x00000100L
+//BIF_BX1_NBIF_GFX_ADDR_LUT_0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_0__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_0__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_1
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_1__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_1__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_2
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_2__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_2__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_3
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_3__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_3__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_4
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_4__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_4__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_5
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_5__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_5__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_6
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_6__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_6__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_7
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_7__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_7__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_8
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_8__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_8__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_9
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_9__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_9__ADDR_MASK                                                                0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_10
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_10__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_10__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_11
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_11__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_11__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_12
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_12__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_12__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_13
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_13__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_13__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_14
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_14__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_14__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_NBIF_GFX_ADDR_LUT_15
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_15__ADDR__SHIFT                                                             0x0
+#define BIF_BX1_NBIF_GFX_ADDR_LUT_15__ADDR_MASK                                                               0x00FFFFFFL
+//BIF_BX1_VF_REGWR_EN
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF0__SHIFT                                                           0x0
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF1__SHIFT                                                           0x1
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF2__SHIFT                                                           0x2
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF3__SHIFT                                                           0x3
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF4__SHIFT                                                           0x4
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF5__SHIFT                                                           0x5
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF6__SHIFT                                                           0x6
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF7__SHIFT                                                           0x7
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF8__SHIFT                                                           0x8
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF9__SHIFT                                                           0x9
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF10__SHIFT                                                          0xa
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF11__SHIFT                                                          0xb
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF12__SHIFT                                                          0xc
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF13__SHIFT                                                          0xd
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF14__SHIFT                                                          0xe
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF15__SHIFT                                                          0xf
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF16__SHIFT                                                          0x10
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF17__SHIFT                                                          0x11
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF18__SHIFT                                                          0x12
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF19__SHIFT                                                          0x13
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF20__SHIFT                                                          0x14
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF21__SHIFT                                                          0x15
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF22__SHIFT                                                          0x16
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF23__SHIFT                                                          0x17
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF24__SHIFT                                                          0x18
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF25__SHIFT                                                          0x19
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF26__SHIFT                                                          0x1a
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF27__SHIFT                                                          0x1b
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF28__SHIFT                                                          0x1c
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF29__SHIFT                                                          0x1d
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF30__SHIFT                                                          0x1e
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF0_MASK                                                             0x00000001L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF1_MASK                                                             0x00000002L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF2_MASK                                                             0x00000004L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF3_MASK                                                             0x00000008L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF4_MASK                                                             0x00000010L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF5_MASK                                                             0x00000020L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF6_MASK                                                             0x00000040L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF7_MASK                                                             0x00000080L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF8_MASK                                                             0x00000100L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF9_MASK                                                             0x00000200L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF10_MASK                                                            0x00000400L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF11_MASK                                                            0x00000800L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF12_MASK                                                            0x00001000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF13_MASK                                                            0x00002000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF14_MASK                                                            0x00004000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF15_MASK                                                            0x00008000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF16_MASK                                                            0x00010000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF17_MASK                                                            0x00020000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF18_MASK                                                            0x00040000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF19_MASK                                                            0x00080000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF20_MASK                                                            0x00100000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF21_MASK                                                            0x00200000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF22_MASK                                                            0x00400000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF23_MASK                                                            0x00800000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF24_MASK                                                            0x01000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF25_MASK                                                            0x02000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF26_MASK                                                            0x04000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF27_MASK                                                            0x08000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF28_MASK                                                            0x10000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF29_MASK                                                            0x20000000L
+#define BIF_BX1_VF_REGWR_EN__VF_REGWR_EN_VF30_MASK                                                            0x40000000L
+//BIF_BX1_VF_DOORBELL_EN
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0__SHIFT                                                     0x0
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1__SHIFT                                                     0x1
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2__SHIFT                                                     0x2
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3__SHIFT                                                     0x3
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4__SHIFT                                                     0x4
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5__SHIFT                                                     0x5
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6__SHIFT                                                     0x6
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7__SHIFT                                                     0x7
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8__SHIFT                                                     0x8
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9__SHIFT                                                     0x9
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10__SHIFT                                                    0xa
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11__SHIFT                                                    0xb
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12__SHIFT                                                    0xc
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13__SHIFT                                                    0xd
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14__SHIFT                                                    0xe
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15__SHIFT                                                    0xf
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16__SHIFT                                                    0x10
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17__SHIFT                                                    0x11
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18__SHIFT                                                    0x12
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19__SHIFT                                                    0x13
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20__SHIFT                                                    0x14
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21__SHIFT                                                    0x15
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22__SHIFT                                                    0x16
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23__SHIFT                                                    0x17
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24__SHIFT                                                    0x18
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25__SHIFT                                                    0x19
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26__SHIFT                                                    0x1a
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27__SHIFT                                                    0x1b
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28__SHIFT                                                    0x1c
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29__SHIFT                                                    0x1d
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30__SHIFT                                                    0x1e
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS__SHIFT                                                 0x1f
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF0_MASK                                                       0x00000001L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF1_MASK                                                       0x00000002L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF2_MASK                                                       0x00000004L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF3_MASK                                                       0x00000008L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF4_MASK                                                       0x00000010L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF5_MASK                                                       0x00000020L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF6_MASK                                                       0x00000040L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF7_MASK                                                       0x00000080L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF8_MASK                                                       0x00000100L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF9_MASK                                                       0x00000200L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF10_MASK                                                      0x00000400L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF11_MASK                                                      0x00000800L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF12_MASK                                                      0x00001000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF13_MASK                                                      0x00002000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF14_MASK                                                      0x00004000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF15_MASK                                                      0x00008000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF16_MASK                                                      0x00010000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF17_MASK                                                      0x00020000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF18_MASK                                                      0x00040000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF19_MASK                                                      0x00080000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF20_MASK                                                      0x00100000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF21_MASK                                                      0x00200000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF22_MASK                                                      0x00400000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF23_MASK                                                      0x00800000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF24_MASK                                                      0x01000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF25_MASK                                                      0x02000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF26_MASK                                                      0x04000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF27_MASK                                                      0x08000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF28_MASK                                                      0x10000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF29_MASK                                                      0x20000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_EN_VF30_MASK                                                      0x40000000L
+#define BIF_BX1_VF_DOORBELL_EN__VF_DOORBELL_RD_LOG_DIS_MASK                                                   0x80000000L
+//BIF_BX1_VF_FB_EN
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF0__SHIFT                                                                 0x0
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF1__SHIFT                                                                 0x1
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF2__SHIFT                                                                 0x2
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF3__SHIFT                                                                 0x3
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF4__SHIFT                                                                 0x4
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF5__SHIFT                                                                 0x5
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF6__SHIFT                                                                 0x6
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF7__SHIFT                                                                 0x7
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF8__SHIFT                                                                 0x8
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF9__SHIFT                                                                 0x9
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF10__SHIFT                                                                0xa
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF11__SHIFT                                                                0xb
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF12__SHIFT                                                                0xc
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF13__SHIFT                                                                0xd
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF14__SHIFT                                                                0xe
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF15__SHIFT                                                                0xf
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF16__SHIFT                                                                0x10
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF17__SHIFT                                                                0x11
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF18__SHIFT                                                                0x12
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF19__SHIFT                                                                0x13
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF20__SHIFT                                                                0x14
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF21__SHIFT                                                                0x15
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF22__SHIFT                                                                0x16
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF23__SHIFT                                                                0x17
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF24__SHIFT                                                                0x18
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF25__SHIFT                                                                0x19
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF26__SHIFT                                                                0x1a
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF27__SHIFT                                                                0x1b
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF28__SHIFT                                                                0x1c
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF29__SHIFT                                                                0x1d
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF30__SHIFT                                                                0x1e
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF0_MASK                                                                   0x00000001L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF1_MASK                                                                   0x00000002L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF2_MASK                                                                   0x00000004L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF3_MASK                                                                   0x00000008L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF4_MASK                                                                   0x00000010L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF5_MASK                                                                   0x00000020L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF6_MASK                                                                   0x00000040L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF7_MASK                                                                   0x00000080L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF8_MASK                                                                   0x00000100L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF9_MASK                                                                   0x00000200L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF10_MASK                                                                  0x00000400L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF11_MASK                                                                  0x00000800L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF12_MASK                                                                  0x00001000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF13_MASK                                                                  0x00002000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF14_MASK                                                                  0x00004000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF15_MASK                                                                  0x00008000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF16_MASK                                                                  0x00010000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF17_MASK                                                                  0x00020000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF18_MASK                                                                  0x00040000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF19_MASK                                                                  0x00080000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF20_MASK                                                                  0x00100000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF21_MASK                                                                  0x00200000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF22_MASK                                                                  0x00400000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF23_MASK                                                                  0x00800000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF24_MASK                                                                  0x01000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF25_MASK                                                                  0x02000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF26_MASK                                                                  0x04000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF27_MASK                                                                  0x08000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF28_MASK                                                                  0x10000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF29_MASK                                                                  0x20000000L
+#define BIF_BX1_VF_FB_EN__VF_FB_EN_VF30_MASK                                                                  0x40000000L
+//BIF_BX1_VF_REGWR_STATUS
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0__SHIFT                                                   0x0
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1__SHIFT                                                   0x1
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2__SHIFT                                                   0x2
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3__SHIFT                                                   0x3
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4__SHIFT                                                   0x4
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5__SHIFT                                                   0x5
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6__SHIFT                                                   0x6
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7__SHIFT                                                   0x7
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8__SHIFT                                                   0x8
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9__SHIFT                                                   0x9
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10__SHIFT                                                  0xa
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11__SHIFT                                                  0xb
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12__SHIFT                                                  0xc
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13__SHIFT                                                  0xd
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14__SHIFT                                                  0xe
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15__SHIFT                                                  0xf
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16__SHIFT                                                  0x10
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17__SHIFT                                                  0x11
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18__SHIFT                                                  0x12
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19__SHIFT                                                  0x13
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20__SHIFT                                                  0x14
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21__SHIFT                                                  0x15
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22__SHIFT                                                  0x16
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23__SHIFT                                                  0x17
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24__SHIFT                                                  0x18
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25__SHIFT                                                  0x19
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26__SHIFT                                                  0x1a
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27__SHIFT                                                  0x1b
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28__SHIFT                                                  0x1c
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29__SHIFT                                                  0x1d
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30__SHIFT                                                  0x1e
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF0_MASK                                                     0x00000001L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF1_MASK                                                     0x00000002L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF2_MASK                                                     0x00000004L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF3_MASK                                                     0x00000008L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF4_MASK                                                     0x00000010L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF5_MASK                                                     0x00000020L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF6_MASK                                                     0x00000040L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF7_MASK                                                     0x00000080L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF8_MASK                                                     0x00000100L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF9_MASK                                                     0x00000200L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF10_MASK                                                    0x00000400L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF11_MASK                                                    0x00000800L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF12_MASK                                                    0x00001000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF13_MASK                                                    0x00002000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF14_MASK                                                    0x00004000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF15_MASK                                                    0x00008000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF16_MASK                                                    0x00010000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF17_MASK                                                    0x00020000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF18_MASK                                                    0x00040000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF19_MASK                                                    0x00080000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF20_MASK                                                    0x00100000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF21_MASK                                                    0x00200000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF22_MASK                                                    0x00400000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF23_MASK                                                    0x00800000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF24_MASK                                                    0x01000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF25_MASK                                                    0x02000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF26_MASK                                                    0x04000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF27_MASK                                                    0x08000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF28_MASK                                                    0x10000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF29_MASK                                                    0x20000000L
+#define BIF_BX1_VF_REGWR_STATUS__VF_REGWR_STATUS_VF30_MASK                                                    0x40000000L
+//BIF_BX1_VF_DOORBELL_STATUS
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0__SHIFT                                             0x0
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1__SHIFT                                             0x1
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2__SHIFT                                             0x2
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3__SHIFT                                             0x3
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4__SHIFT                                             0x4
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5__SHIFT                                             0x5
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6__SHIFT                                             0x6
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7__SHIFT                                             0x7
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8__SHIFT                                             0x8
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9__SHIFT                                             0x9
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10__SHIFT                                            0xa
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11__SHIFT                                            0xb
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12__SHIFT                                            0xc
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13__SHIFT                                            0xd
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14__SHIFT                                            0xe
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15__SHIFT                                            0xf
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16__SHIFT                                            0x10
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17__SHIFT                                            0x11
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18__SHIFT                                            0x12
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19__SHIFT                                            0x13
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20__SHIFT                                            0x14
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21__SHIFT                                            0x15
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22__SHIFT                                            0x16
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23__SHIFT                                            0x17
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24__SHIFT                                            0x18
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25__SHIFT                                            0x19
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26__SHIFT                                            0x1a
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27__SHIFT                                            0x1b
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28__SHIFT                                            0x1c
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29__SHIFT                                            0x1d
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30__SHIFT                                            0x1e
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF0_MASK                                               0x00000001L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF1_MASK                                               0x00000002L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF2_MASK                                               0x00000004L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF3_MASK                                               0x00000008L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF4_MASK                                               0x00000010L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF5_MASK                                               0x00000020L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF6_MASK                                               0x00000040L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF7_MASK                                               0x00000080L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF8_MASK                                               0x00000100L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF9_MASK                                               0x00000200L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF10_MASK                                              0x00000400L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF11_MASK                                              0x00000800L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF12_MASK                                              0x00001000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF13_MASK                                              0x00002000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF14_MASK                                              0x00004000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF15_MASK                                              0x00008000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF16_MASK                                              0x00010000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF17_MASK                                              0x00020000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF18_MASK                                              0x00040000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF19_MASK                                              0x00080000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF20_MASK                                              0x00100000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF21_MASK                                              0x00200000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF22_MASK                                              0x00400000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF23_MASK                                              0x00800000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF24_MASK                                              0x01000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF25_MASK                                              0x02000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF26_MASK                                              0x04000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF27_MASK                                              0x08000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF28_MASK                                              0x10000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF29_MASK                                              0x20000000L
+#define BIF_BX1_VF_DOORBELL_STATUS__VF_DOORBELL_STATUS_VF30_MASK                                              0x40000000L
+//BIF_BX1_VF_FB_STATUS
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF0__SHIFT                                                         0x0
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF1__SHIFT                                                         0x1
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF2__SHIFT                                                         0x2
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF3__SHIFT                                                         0x3
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF4__SHIFT                                                         0x4
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF5__SHIFT                                                         0x5
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF6__SHIFT                                                         0x6
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF7__SHIFT                                                         0x7
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF8__SHIFT                                                         0x8
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF9__SHIFT                                                         0x9
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF10__SHIFT                                                        0xa
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF11__SHIFT                                                        0xb
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF12__SHIFT                                                        0xc
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF13__SHIFT                                                        0xd
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF14__SHIFT                                                        0xe
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF15__SHIFT                                                        0xf
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF16__SHIFT                                                        0x10
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF17__SHIFT                                                        0x11
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF18__SHIFT                                                        0x12
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF19__SHIFT                                                        0x13
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF20__SHIFT                                                        0x14
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF21__SHIFT                                                        0x15
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF22__SHIFT                                                        0x16
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF23__SHIFT                                                        0x17
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF24__SHIFT                                                        0x18
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF25__SHIFT                                                        0x19
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF26__SHIFT                                                        0x1a
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF27__SHIFT                                                        0x1b
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF28__SHIFT                                                        0x1c
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF29__SHIFT                                                        0x1d
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF30__SHIFT                                                        0x1e
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF0_MASK                                                           0x00000001L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF1_MASK                                                           0x00000002L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF2_MASK                                                           0x00000004L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF3_MASK                                                           0x00000008L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF4_MASK                                                           0x00000010L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF5_MASK                                                           0x00000020L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF6_MASK                                                           0x00000040L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF7_MASK                                                           0x00000080L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF8_MASK                                                           0x00000100L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF9_MASK                                                           0x00000200L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF10_MASK                                                          0x00000400L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF11_MASK                                                          0x00000800L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF12_MASK                                                          0x00001000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF13_MASK                                                          0x00002000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF14_MASK                                                          0x00004000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF15_MASK                                                          0x00008000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF16_MASK                                                          0x00010000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF17_MASK                                                          0x00020000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF18_MASK                                                          0x00040000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF19_MASK                                                          0x00080000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF20_MASK                                                          0x00100000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF21_MASK                                                          0x00200000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF22_MASK                                                          0x00400000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF23_MASK                                                          0x00800000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF24_MASK                                                          0x01000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF25_MASK                                                          0x02000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF26_MASK                                                          0x04000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF27_MASK                                                          0x08000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF28_MASK                                                          0x10000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF29_MASK                                                          0x20000000L
+#define BIF_BX1_VF_FB_STATUS__VF_FB_STATUS_VF30_MASK                                                          0x40000000L
+//BIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL
+#define BIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX1_REMAP_HDP_MEM_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX1_REMAP_HDP_REG_FLUSH_CNTL
+#define BIF_BX1_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS__SHIFT                                                      0x2
+#define BIF_BX1_REMAP_HDP_REG_FLUSH_CNTL__ADDRESS_MASK                                                        0x0007FFFCL
+//BIF_BX1_BIF_RB_CNTL
+#define BIF_BX1_BIF_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define BIF_BX1_BIF_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define BIF_BX1_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE__SHIFT                                                     0x8
+#define BIF_BX1_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER__SHIFT                                                      0x9
+#define BIF_BX1_BIF_RB_CNTL__BIF_RB_TRAN__SHIFT                                                               0x11
+#define BIF_BX1_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL__SHIFT                                                  0x19
+#define BIF_BX1_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY__SHIFT                                                      0x1a
+#define BIF_BX1_BIF_RB_CNTL__RB_INTR_ARB_MODE__SHIFT                                                          0x1d
+#define BIF_BX1_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE__SHIFT                                                     0x1e
+#define BIF_BX1_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR__SHIFT                                                       0x1f
+#define BIF_BX1_BIF_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define BIF_BX1_BIF_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define BIF_BX1_BIF_RB_CNTL__WPTR_WRITEBACK_ENABLE_MASK                                                       0x00000100L
+#define BIF_BX1_BIF_RB_CNTL__WPTR_WRITEBACK_TIMER_MASK                                                        0x00003E00L
+#define BIF_BX1_BIF_RB_CNTL__BIF_RB_TRAN_MASK                                                                 0x00020000L
+#define BIF_BX1_BIF_RB_CNTL__DIS_PROTECT_WHEN_RB_FULL_MASK                                                    0x02000000L
+#define BIF_BX1_BIF_RB_CNTL__RB_INTR_FIX_PRIORITY_MASK                                                        0x1C000000L
+#define BIF_BX1_BIF_RB_CNTL__RB_INTR_ARB_MODE_MASK                                                            0x20000000L
+#define BIF_BX1_BIF_RB_CNTL__RB_RST_BY_FLR_DISABLE_MASK                                                       0x40000000L
+#define BIF_BX1_BIF_RB_CNTL__WPTR_OVERFLOW_CLEAR_MASK                                                         0x80000000L
+//BIF_BX1_BIF_RB_BASE
+#define BIF_BX1_BIF_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define BIF_BX1_BIF_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//BIF_BX1_BIF_RB_RPTR
+#define BIF_BX1_BIF_RB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define BIF_BX1_BIF_RB_RPTR__OFFSET_MASK                                                                      0x0003FFFCL
+//BIF_BX1_BIF_RB_WPTR
+#define BIF_BX1_BIF_RB_WPTR__BIF_RB_OVERFLOW__SHIFT                                                           0x0
+#define BIF_BX1_BIF_RB_WPTR__OFFSET__SHIFT                                                                    0x2
+#define BIF_BX1_BIF_RB_WPTR__BIF_RB_OVERFLOW_MASK                                                             0x00000001L
+#define BIF_BX1_BIF_RB_WPTR__OFFSET_MASK                                                                      0x0003FFFCL
+//BIF_BX1_BIF_RB_WPTR_ADDR_HI
+#define BIF_BX1_BIF_RB_WPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define BIF_BX1_BIF_RB_WPTR_ADDR_HI__ADDR_MASK                                                                0x000000FFL
+//BIF_BX1_BIF_RB_WPTR_ADDR_LO
+#define BIF_BX1_BIF_RB_WPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define BIF_BX1_BIF_RB_WPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//BIF_BX1_MAILBOX_INDEX
+#define BIF_BX1_MAILBOX_INDEX__MAILBOX_INDEX__SHIFT                                                           0x0
+#define BIF_BX1_MAILBOX_INDEX__MAILBOX_INDEX_MASK                                                             0x0000001FL
+//BIF_BX1_BACO_AZ_ENHANCE_CTRL
+#define BIF_BX1_BACO_AZ_ENHANCE_CTRL__GCTL_OFFSET__SHIFT                                                      0x2
+#define BIF_BX1_BACO_AZ_ENHANCE_CTRL__AZ_CRST_WRITTEN_ONE__SHIFT                                              0x10
+#define BIF_BX1_BACO_AZ_ENHANCE_CTRL__AZ_CRST_WRITTEN_ZERO__SHIFT                                             0x11
+#define BIF_BX1_BACO_AZ_ENHANCE_CTRL__AZ_MMIO_RDRSPDATA_FORCE_ZERO__SHIFT                                     0x1f
+#define BIF_BX1_BACO_AZ_ENHANCE_CTRL__GCTL_OFFSET_MASK                                                        0x00003FFCL
+#define BIF_BX1_BACO_AZ_ENHANCE_CTRL__AZ_CRST_WRITTEN_ONE_MASK                                                0x00010000L
+#define BIF_BX1_BACO_AZ_ENHANCE_CTRL__AZ_CRST_WRITTEN_ZERO_MASK                                               0x00020000L
+#define BIF_BX1_BACO_AZ_ENHANCE_CTRL__AZ_MMIO_RDRSPDATA_FORCE_ZERO_MASK                                       0x80000000L
+//BIF_BX1_BIF_MP1_INTR_CTRL
+#define BIF_BX1_BIF_MP1_INTR_CTRL__BACO_EXIT_DONE__SHIFT                                                      0x0
+#define BIF_BX1_BIF_MP1_INTR_CTRL__BACO_EXIT_DONE_MASK                                                        0x00000001L
+//BIF_BX1_BIF_PERSTB_PAD_CNTL
+#define BIF_BX1_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX1_BIF_PERSTB_PAD_CNTL__PERSTB_PAD_CNTL_MASK                                                     0x0000FFFFL
+//BIF_BX1_BIF_PX_EN_PAD_CNTL
+#define BIF_BX1_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL__SHIFT                                                     0x0
+#define BIF_BX1_BIF_PX_EN_PAD_CNTL__PX_EN_PAD_CNTL_MASK                                                       0x00000FFFL
+//BIF_BX1_BIF_REFPADKIN_PAD_CNTL
+#define BIF_BX1_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL__SHIFT                                             0x0
+#define BIF_BX1_BIF_REFPADKIN_PAD_CNTL__REFPADKIN_PAD_CNTL_MASK                                               0x000000FFL
+//BIF_BX1_BIF_CLKREQB_PAD_CNTL
+#define BIF_BX1_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL__SHIFT                                                 0x0
+#define BIF_BX1_BIF_CLKREQB_PAD_CNTL__CLKREQB_PAD_CNTL_MASK                                                   0x7FFFFFFFL
+//BIF_BX1_BIF_PWRBRK_PAD_CNTL
+#define BIF_BX1_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL__SHIFT                                                   0x0
+#define BIF_BX1_BIF_PWRBRK_PAD_CNTL__PWRBRK_PAD_CNTL_MASK                                                     0x000000FFL
+//BIF_BX1_BIF_GPUIOV_SCH_CFG_SIZE
+#define BIF_BX1_BIF_GPUIOV_SCH_CFG_SIZE__GPUIOV_SCH_CFG_SIZE__SHIFT                                           0x0
+#define BIF_BX1_BIF_GPUIOV_SCH_CFG_SIZE__GPUIOV_SCH_CFG_SIZE_MASK                                             0x0000000FL
+
+
+// addressBlock: nbif0_nbif0_bif_bx_pf_BIFPFVFDEC1
+//BIF_BX_PF1_BIF_BME_STATUS
+#define BIF_BX_PF1_BIF_BME_STATUS__DMA_ON_BME_LOW__SHIFT                                                      0x0
+#define BIF_BX_PF1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW__SHIFT                                                0x10
+#define BIF_BX_PF1_BIF_BME_STATUS__DMA_ON_BME_LOW_MASK                                                        0x00000001L
+#define BIF_BX_PF1_BIF_BME_STATUS__CLEAR_DMA_ON_BME_LOW_MASK                                                  0x00010000L
+//BIF_BX_PF1_BIF_ATOMIC_ERR_LOG
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE__SHIFT                                                0x0
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW__SHIFT                                             0x1
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH__SHIFT                                                0x2
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR__SHIFT                                                    0x3
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE__SHIFT                                          0x10
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW__SHIFT                                       0x11
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH__SHIFT                                          0x12
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR__SHIFT                                              0x13
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_OPCODE_MASK                                                  0x00000001L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_REQEN_LOW_MASK                                               0x00000002L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_LENGTH_MASK                                                  0x00000004L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__UR_ATOMIC_NR_MASK                                                      0x00000008L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_OPCODE_MASK                                            0x00010000L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_REQEN_LOW_MASK                                         0x00020000L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_LENGTH_MASK                                            0x00040000L
+#define BIF_BX_PF1_BIF_ATOMIC_ERR_LOG__CLEAR_UR_ATOMIC_NR_MASK                                                0x00080000L
+//BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH__SHIFT          0x0
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_HIGH__DOORBELL_SELFRING_GPA_APER_BASE_HIGH_MASK            0xFFFFFFFFL
+//BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW__SHIFT            0x0
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_BASE_LOW__DOORBELL_SELFRING_GPA_APER_BASE_LOW_MASK              0xFFFFFFFFL
+//BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN__SHIFT                      0x0
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE__SHIFT                    0x1
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE__SHIFT                    0x8
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_EN_MASK                        0x00000001L
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_MODE_MASK                      0x00000002L
+#define BIF_BX_PF1_DOORBELL_SELFRING_GPA_APER_CNTL__DOORBELL_SELFRING_GPA_APER_SIZE_MASK                      0x000FFF00L
+//BIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF1_HDP_REG_COHERENCY_FLUSH_CNTL__HDP_REG_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR__SHIFT                                    0x0
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_CNTL__HDP_MEM_FLUSH_ADDR_MASK                                      0x00000001L
+//BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR__SHIFT                          0x0
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_FLUSH_ONLY_CNTL__HDP_MEM_FLUSH_ONLY_ADDR_MASK                            0x00000001L
+//BIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR__SHIFT                0x0
+#define BIF_BX_PF1_HDP_MEM_COHERENCY_INVALIDATE_ONLY_CNTL__HDP_MEM_INVALIDATE_ONLY_ADDR_MASK                  0x00000001L
+//BIF_BX_PF1_GPU_HDP_FLUSH_REQ
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP0__SHIFT                                                              0x0
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP1__SHIFT                                                              0x1
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP2__SHIFT                                                              0x2
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP3__SHIFT                                                              0x3
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP4__SHIFT                                                              0x4
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP5__SHIFT                                                              0x5
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP6__SHIFT                                                              0x6
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP7__SHIFT                                                              0x7
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP8__SHIFT                                                              0x8
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP9__SHIFT                                                              0x9
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA0__SHIFT                                                            0xa
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA1__SHIFT                                                            0xb
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG0__SHIFT                                                        0xc
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG1__SHIFT                                                        0xd
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG2__SHIFT                                                        0xe
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG3__SHIFT                                                        0xf
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG4__SHIFT                                                        0x10
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG5__SHIFT                                                        0x11
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG6__SHIFT                                                        0x12
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG7__SHIFT                                                        0x13
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG8__SHIFT                                                        0x14
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG9__SHIFT                                                        0x15
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG10__SHIFT                                                       0x16
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG11__SHIFT                                                       0x17
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG12__SHIFT                                                       0x18
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG13__SHIFT                                                       0x19
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG14__SHIFT                                                       0x1a
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG15__SHIFT                                                       0x1b
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG16__SHIFT                                                       0x1c
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG17__SHIFT                                                       0x1d
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG18__SHIFT                                                       0x1e
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG19__SHIFT                                                       0x1f
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP0_MASK                                                                0x00000001L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP1_MASK                                                                0x00000002L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP2_MASK                                                                0x00000004L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP3_MASK                                                                0x00000008L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP4_MASK                                                                0x00000010L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP5_MASK                                                                0x00000020L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP6_MASK                                                                0x00000040L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP7_MASK                                                                0x00000080L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP8_MASK                                                                0x00000100L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__CP9_MASK                                                                0x00000200L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA0_MASK                                                              0x00000400L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__SDMA1_MASK                                                              0x00000800L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG0_MASK                                                          0x00001000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG1_MASK                                                          0x00002000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG2_MASK                                                          0x00004000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG3_MASK                                                          0x00008000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG4_MASK                                                          0x00010000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG5_MASK                                                          0x00020000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG6_MASK                                                          0x00040000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG7_MASK                                                          0x00080000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG8_MASK                                                          0x00100000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG9_MASK                                                          0x00200000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG10_MASK                                                         0x00400000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG11_MASK                                                         0x00800000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG12_MASK                                                         0x01000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG13_MASK                                                         0x02000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG14_MASK                                                         0x04000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG15_MASK                                                         0x08000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG16_MASK                                                         0x10000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG17_MASK                                                         0x20000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG18_MASK                                                         0x40000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_REQ__RSVD_ENG19_MASK                                                         0x80000000L
+//BIF_BX_PF1_GPU_HDP_FLUSH_DONE
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP0__SHIFT                                                             0x0
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP1__SHIFT                                                             0x1
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP2__SHIFT                                                             0x2
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP3__SHIFT                                                             0x3
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP4__SHIFT                                                             0x4
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP5__SHIFT                                                             0x5
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP6__SHIFT                                                             0x6
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP7__SHIFT                                                             0x7
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP8__SHIFT                                                             0x8
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP9__SHIFT                                                             0x9
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA0__SHIFT                                                           0xa
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA1__SHIFT                                                           0xb
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG0__SHIFT                                                       0xc
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG1__SHIFT                                                       0xd
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG2__SHIFT                                                       0xe
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG3__SHIFT                                                       0xf
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG4__SHIFT                                                       0x10
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG5__SHIFT                                                       0x11
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG6__SHIFT                                                       0x12
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG7__SHIFT                                                       0x13
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG8__SHIFT                                                       0x14
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG9__SHIFT                                                       0x15
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG10__SHIFT                                                      0x16
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG11__SHIFT                                                      0x17
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG12__SHIFT                                                      0x18
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG13__SHIFT                                                      0x19
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG14__SHIFT                                                      0x1a
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG15__SHIFT                                                      0x1b
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG16__SHIFT                                                      0x1c
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG17__SHIFT                                                      0x1d
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG18__SHIFT                                                      0x1e
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG19__SHIFT                                                      0x1f
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP0_MASK                                                               0x00000001L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP1_MASK                                                               0x00000002L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP2_MASK                                                               0x00000004L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP3_MASK                                                               0x00000008L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP4_MASK                                                               0x00000010L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP5_MASK                                                               0x00000020L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP6_MASK                                                               0x00000040L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP7_MASK                                                               0x00000080L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP8_MASK                                                               0x00000100L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__CP9_MASK                                                               0x00000200L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA0_MASK                                                             0x00000400L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__SDMA1_MASK                                                             0x00000800L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG0_MASK                                                         0x00001000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG1_MASK                                                         0x00002000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG2_MASK                                                         0x00004000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG3_MASK                                                         0x00008000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG4_MASK                                                         0x00010000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG5_MASK                                                         0x00020000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG6_MASK                                                         0x00040000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG7_MASK                                                         0x00080000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG8_MASK                                                         0x00100000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG9_MASK                                                         0x00200000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG10_MASK                                                        0x00400000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG11_MASK                                                        0x00800000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG12_MASK                                                        0x01000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG13_MASK                                                        0x02000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG14_MASK                                                        0x04000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG15_MASK                                                        0x08000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG16_MASK                                                        0x10000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG17_MASK                                                        0x20000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG18_MASK                                                        0x40000000L
+#define BIF_BX_PF1_GPU_HDP_FLUSH_DONE__RSVD_ENG19_MASK                                                        0x80000000L
+//BIF_BX_PF1_BIF_TRANS_PENDING
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING__SHIFT                                            0x0
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING__SHIFT                                            0x1
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_MST_TRANS_PENDING_MASK                                              0x00000001L
+#define BIF_BX_PF1_BIF_TRANS_PENDING__BIF_SLV_TRANS_PENDING_MASK                                              0x00000002L
+//BIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS
+#define BIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS__SHIFT                                                0x0
+#define BIF_BX_PF1_NBIF_GFX_ADDR_LUT_BYPASS__LUT_BYPASS_MASK                                                  0x00000001L
+//BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW0__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW1__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW2__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_TRN_DW3__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW0__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW1__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW2__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA__SHIFT                                                 0x0
+#define BIF_BX_PF1_MAILBOX_MSGBUF_RCV_DW3__MSGBUF_DATA_MASK                                                   0xFFFFFFFFL
+//BIF_BX_PF1_MAILBOX_CONTROL
+#define BIF_BX_PF1_MAILBOX_CONTROL__TRN_MSG_VALID__SHIFT                                                      0x0
+#define BIF_BX_PF1_MAILBOX_CONTROL__TRN_MSG_ACK__SHIFT                                                        0x1
+#define BIF_BX_PF1_MAILBOX_CONTROL__RCV_MSG_VALID__SHIFT                                                      0x8
+#define BIF_BX_PF1_MAILBOX_CONTROL__RCV_MSG_ACK__SHIFT                                                        0x9
+#define BIF_BX_PF1_MAILBOX_CONTROL__TRN_MSG_VALID_MASK                                                        0x00000001L
+#define BIF_BX_PF1_MAILBOX_CONTROL__TRN_MSG_ACK_MASK                                                          0x00000002L
+#define BIF_BX_PF1_MAILBOX_CONTROL__RCV_MSG_VALID_MASK                                                        0x00000100L
+#define BIF_BX_PF1_MAILBOX_CONTROL__RCV_MSG_ACK_MASK                                                          0x00000200L
+//BIF_BX_PF1_MAILBOX_INT_CNTL
+#define BIF_BX_PF1_MAILBOX_INT_CNTL__VALID_INT_EN__SHIFT                                                      0x0
+#define BIF_BX_PF1_MAILBOX_INT_CNTL__ACK_INT_EN__SHIFT                                                        0x1
+#define BIF_BX_PF1_MAILBOX_INT_CNTL__VALID_INT_EN_MASK                                                        0x00000001L
+#define BIF_BX_PF1_MAILBOX_INT_CNTL__ACK_INT_EN_MASK                                                          0x00000002L
+//BIF_BX_PF1_BIF_VMHV_MAILBOX
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN__SHIFT                                      0x0
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN__SHIFT                                    0x1
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA__SHIFT                                         0x8
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID__SHIFT                                        0xf
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA__SHIFT                                         0x10
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID__SHIFT                                        0x17
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK__SHIFT                                          0x18
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK__SHIFT                                          0x19
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_ACK_INTR_EN_MASK                                        0x00000001L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_VALID_INTR_EN_MASK                                      0x00000002L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_DATA_MASK                                           0x00000F00L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_VALID_MASK                                          0x00008000L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_DATA_MASK                                           0x000F0000L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_VALID_MASK                                          0x00800000L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_TRN_MSG_ACK_MASK                                            0x01000000L
+#define BIF_BX_PF1_BIF_VMHV_MAILBOX__VMHV_MAILBOX_RCV_MSG_ACK_MASK                                            0x02000000L
+//BIF_BX_PF1_PARTITION_COMPUTE_CAP
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__SPX_SUPPORT__SHIFT                                                  0x0
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__DPX_SUPPORT__SHIFT                                                  0x1
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__TPX_SUPPORT__SHIFT                                                  0x2
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__QPX_SUPPORT__SHIFT                                                  0x3
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__CPX_SUPPORT__SHIFT                                                  0x4
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY__SHIFT                                             0xa
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__SPX_SUPPORT_MASK                                                    0x00000001L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__DPX_SUPPORT_MASK                                                    0x00000002L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__TPX_SUPPORT_MASK                                                    0x00000004L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__QPX_SUPPORT_MASK                                                    0x00000008L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__CPX_SUPPORT_MASK                                                    0x00000010L
+#define BIF_BX_PF1_PARTITION_COMPUTE_CAP__CPX_AVAILABILITY_MASK                                               0x0003FC00L
+//BIF_BX_PF1_PARTITION_MEM_CAP
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS1_SUPPORT__SHIFT                                                     0x0
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS2_SUPPORT__SHIFT                                                     0x1
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS3_SUPPORT__SHIFT                                                     0x2
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS4_SUPPORT__SHIFT                                                     0x3
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS6_SUPPORT__SHIFT                                                     0x5
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS8_SUPPORT__SHIFT                                                     0x7
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS1_SUPPORT_MASK                                                       0x00000001L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS2_SUPPORT_MASK                                                       0x00000002L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS3_SUPPORT_MASK                                                       0x00000004L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS4_SUPPORT_MASK                                                       0x00000008L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS6_SUPPORT_MASK                                                       0x00000020L
+#define BIF_BX_PF1_PARTITION_MEM_CAP__NPS8_SUPPORT_MASK                                                       0x00000080L
+//BIF_BX_PF1_PARTITION_COMPUTE_STATUS
+#define BIF_BX_PF1_PARTITION_COMPUTE_STATUS__PARTITION_MODE__SHIFT                                            0x4
+#define BIF_BX_PF1_PARTITION_COMPUTE_STATUS__PARTITION_MODE_MASK                                              0x000000F0L
+//BIF_BX_PF1_PARTITION_MEM_STATUS
+#define BIF_BX_PF1_PARTITION_MEM_STATUS__CHANGE_STATUS__SHIFT                                                 0x0
+#define BIF_BX_PF1_PARTITION_MEM_STATUS__NPS_MODE__SHIFT                                                      0x4
+#define BIF_BX_PF1_PARTITION_MEM_STATUS__CHANGE_STATUS_MASK                                                   0x0000000FL
+#define BIF_BX_PF1_PARTITION_MEM_STATUS__NPS_MODE_MASK                                                        0x00000FF0L
+
+
+// addressBlock: nbif0_nbif0_rcc_strap_BIFDEC1:1
+//RCC_STRAP2_RCC_BIF_STRAP0
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN4_DIS__SHIFT                                                      0x0
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT__SHIFT                              0x1
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PX_CAPABLE__SHIFT                                                    0x7
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3__SHIFT                                                 0x8
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN__SHIFT                                  0x9
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR__SHIFT                                         0xa
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN__SHIFT                                     0xb
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR__SHIFT                                              0xc
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR__SHIFT                                             0xd
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_AUD_PIN__SHIFT                                                       0x12
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN__SHIFT                                               0x14
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN__SHIFT                         0x15
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GPUIOV_EN__SHIFT                                                     0x16
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE__SHIFT                                          0x17
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN3_DIS__SHIFT                                                      0x18
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4__SHIFT                                                 0x19
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_QUICKSIM_START__SHIFT                                                0x1a
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS__SHIFT                                   0x1c
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN__SHIFT                                      0x1d
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE__SHIFT                                                   0x1e
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN__SHIFT                                            0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN4_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_EXPANSION_ROM_VALIDATION_SUPPORT_MASK                                0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PX_CAPABLE_MASK                                                      0x00000080L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN3_MASK                                                   0x00000100L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_MSI_FIRST_BE_FULL_PAYLOAD_EN_MASK                                    0x00000200L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NBIF_IGNORE_ERR_INFLR_MASK                                           0x00000400L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_PME_SUPPORT_COMPLIANCE_EN_MASK                                       0x00000800L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_EP_ERR_MASK                                                0x00001000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_RX_IGNORE_MSG_ERR_MASK                                               0x00002000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_AUD_PIN_MASK                                                         0x000C0000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_PIN_MASK                                                 0x00100000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_GFX_IOBAR_DIS_AND_REGBAR_64BIT_EN_MASK                           0x00200000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GPUIOV_EN_MASK                                                       0x00400000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIOS_ROM_EN_OVERRIDE_MASK                                            0x00800000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_GEN3_DIS_MASK                                                        0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIF_KILL_GEN4_MASK                                                   0x02000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_QUICKSIM_START_MASK                                                  0x04000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_NO_RO_ENABLED_P2P_PASSING_MASK                                       0x08000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_IGNORE_LOCAL_PREFIX_UR_SWUS_MASK                                     0x10000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_CFG0_RD_VF_BUSNUM_CHK_EN_MASK                                        0x20000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_BIGAPU_MODE_MASK                                                     0x40000000L
+#define RCC_STRAP2_RCC_BIF_STRAP0__STRAP_LINK_DOWN_RESET_EN_MASK                                              0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP1
+#define RCC_STRAP2_RCC_BIF_STRAP1__FUSESTRAP_VALID__SHIFT                                                     0x0
+#define RCC_STRAP2_RCC_BIF_STRAP1__ROMSTRAP_VALID__SHIFT                                                      0x1
+#define RCC_STRAP2_RCC_BIF_STRAP1__WRITE_DISABLE__SHIFT                                                       0x2
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN__SHIFT                                      0x3
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_FLIT_MODE_SUPPORT__SHIFT                                             0x4
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS__SHIFT                                     0x5
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE__SHIFT                                       0x6
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_READY__SHIFT                                               0x7
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN__SHIFT                                                  0x8
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN__SHIFT                                                 0x9
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE__SHIFT                                                  0xa
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE__SHIFT                                        0xc
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_HWREV_LSB2__SHIFT                                                    0xd
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWREV_LSB2__SHIFT                                                    0xf
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY__SHIFT                                             0x11
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS__SHIFT                                             0x12
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN__SHIFT                                                        0x13
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN__SHIFT                                                   0x14
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGIN_EN__SHIFT                                                     0x15
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN__SHIFT                                             0x16
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN__SHIFT                                     0x17
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE__SHIFT                                             0x1a
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN__SHIFT                                       0x1b
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN_EP__SHIFT                                                     0x1d
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN__SHIFT                                                         0x1e
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN_DN__SHIFT                                                      0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP1__FUSESTRAP_VALID_MASK                                                       0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP1__ROMSTRAP_VALID_MASK                                                        0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP1__WRITE_DISABLE_MASK                                                         0x00000004L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_ECRC_INTERMEDIATE_CHK_EN_MASK                                        0x00000008L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_FLIT_MODE_SUPPORT_MASK                                               0x00000010L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_IGNORE_E2E_PREFIX_UR_SWUS_MASK                                       0x00000020L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_USES_SOFTWARE_MASK                                         0x00000040L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGINING_READY_MASK                                                 0x00000080L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_APER_EN_MASK                                                    0x00000100L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_64BAR_EN_MASK                                                   0x00000200L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_AP_SIZE_MASK                                                    0x00000C00L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWUS_APER_PREFETCHABLE_MASK                                          0x00001000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_HWREV_LSB2_MASK                                                      0x00006000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_SWREV_LSB2_MASK                                                      0x00018000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_LINK_RST_CFG_ONLY_MASK                                               0x00020000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_IOV_LKRST_DIS_MASK                                               0x00040000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN_MASK                                                          0x00080000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_PHY_16GT_EN_MASK                                                     0x00100000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_MARGIN_EN_MASK                                                       0x00200000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_PSN_UR_RPT_EN_MASK                                               0x00400000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_BIF_SLOT_POWER_SUPPORT_EN_MASK                                       0x00800000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GFX_FUNC_LTR_MODE_MASK                                               0x04000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_GSI_SMN_POSTWR_MULTI_EN_MASK                                         0x18000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_DLF_EN_EP_MASK                                                       0x20000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN_MASK                                                           0x40000000L
+#define RCC_STRAP2_RCC_BIF_STRAP1__STRAP_AP_EN_DN_MASK                                                        0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP10
+//RCC_STRAP2_RCC_BIF_STRAP11
+//RCC_STRAP2_RCC_BIF_STRAP12
+//RCC_STRAP2_RCC_BIF_STRAP13
+#define RCC_STRAP2_RCC_BIF_STRAP13__STRAP_PCIE_SMN_APER__SHIFT                                                0x0
+#define RCC_STRAP2_RCC_BIF_STRAP13__STRAP_PCIE_SMN_APER_MASK                                                  0x00000FFFL
+//RCC_STRAP2_RCC_BIF_STRAP2
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE__SHIFT                                     0x0
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWUS_SPT__SHIFT                                                      0x1
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWDS_SPT__SHIFT                                                      0x2
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS__SHIFT                                            0x3
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS__SHIFT                                            0x4
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA__SHIFT                                        0x5
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA__SHIFT                                      0x6
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS__SHIFT                                 0x8
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS__SHIFT                                        0x9
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN__SHIFT                                   0xa
+#define RCC_STRAP2_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2__SHIFT                                                 0xd
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS__SHIFT                                             0xe
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN__SHIFT                                 0xf
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE__SHIFT                                         0x10
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS__SHIFT                                        0x18
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ARI_EN_UP__SHIFT                                                     0x19
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS__SHIFT                            0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PCIESWUS_INDEX_APER_RANGE_MASK                                       0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWUS_SPT_MASK                                                        0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWDS_SPT_MASK                                                        0x00000004L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SUC_IND_ACCESS_DIS_MASK                                              0x00000008L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SUM_IND_ACCESS_DIS_MASK                                              0x00000010L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ENDP_LINKDOWN_DROP_DMA_MASK                                          0x00000020L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_SWITCH_LINKDOWN_DROP_DMA_MASK                                        0x00000040L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GMI_DNS_SDP_CLKREQ_TOGGLE_DIS_MASK                                   0x00000100L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ACS_MSKSEV_EP_HIDE_DIS_MASK                                          0x00000200L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_CFG_PG_FW_INTERLOCK_EXIT_EN_MASK                                     0x00000C00L
+#define RCC_STRAP2_RCC_BIF_STRAP2__RESERVED_BIF_STRAP2_MASK                                                   0x00002000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_LTR_IN_ASPML1_DIS_MASK                                               0x00004000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_GFXAZ_POWERSTATE_INTERLOCK_EN_MASK                                   0x00008000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_CYCLE_MASK                                           0x00FF0000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_PWRBRK_DEGLITCH_BYPASS_MASK                                          0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_ARI_EN_UP_MASK                                                       0x02000000L
+#define RCC_STRAP2_RCC_BIF_STRAP2__STRAP_VLINK_PMETO_LDN_EXIT_BY_LNKRST_DIS_MASK                              0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP3
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER__SHIFT                                         0x0
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER__SHIFT                                       0x10
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_ASPM_IDLE_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP2_RCC_BIF_STRAP3__STRAP_VLINK_PM_L1_ENTRY_TIMER_MASK                                         0xFFFF0000L
+//RCC_STRAP2_RCC_BIF_STRAP4
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER__SHIFT                                          0x0
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER__SHIFT                                           0x10
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L0S_EXIT_TIMER_MASK                                            0x0000FFFFL
+#define RCC_STRAP2_RCC_BIF_STRAP4__STRAP_VLINK_L1_EXIT_TIMER_MASK                                             0xFFFF0000L
+//RCC_STRAP2_RCC_BIF_STRAP5
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER__SHIFT                                         0x0
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN__SHIFT                                      0x10
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN__SHIFT                                   0x11
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS__SHIFT                                    0x12
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS__SHIFT                                        0x13
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS__SHIFT                                            0x15
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE__SHIFT                                         0x16
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE__SHIFT                          0x18
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                0x19
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                 0x1b
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER__SHIFT                                           0x1c
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN__SHIFT                                 0x1f
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ENTRY_TIMER_MASK                                           0x0000FFFFL
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_LDN_EN_MASK                                        0x00010000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_LDN_ON_SWUS_SECRST_EN_MASK                                     0x00020000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_VLINK_ENTER_COMPLIANCE_DIS_MASK                                      0x00040000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_IGNORE_PSN_ON_VDM1_DIS_MASK                                          0x00080000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERR_STATUS_MASK_EN_UPS_MASK                                      0x00100000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_REG_PROTECTION_DIS_MASK                                              0x00200000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_SMN_ERRRSP_DATA_FORCE_MASK                                           0x00C00000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_INTERMEDIATERSP_DATA_ALLF_DATA_FORCE_MASK                            0x01000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_SUPPORTED_MASK                                  0x06000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_EMER_POWER_REDUCTION_INIT_REQ_MASK                                   0x08000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_PWRBRK_STATUS_TIMER_MASK                                             0x70000000L
+#define RCC_STRAP2_RCC_BIF_STRAP5__STRAP_BIF_GFX_REG_APER_REMAPPING_EN_MASK                                   0x80000000L
+//RCC_STRAP2_RCC_BIF_STRAP6
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_GEN5_DIS__SHIFT                                                      0x0
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5__SHIFT                                                 0x1
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN__SHIFT                                                   0x2
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_DOE_VERSION_SEL__SHIFT                                               0x3
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE__SHIFT                                               0x5
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN__SHIFT                                               0xa
+#define RCC_STRAP2_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6__SHIFT                                                 0xf
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_GEN5_DIS_MASK                                                        0x00000001L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_BIF_KILL_GEN5_MASK                                                   0x00000002L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PHY_32GT_EN_MASK                                                     0x00000004L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_DOE_VERSION_SEL_MASK                                                 0x00000008L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_PRODUCTION_MODE_MASK                                                 0x00000020L
+#define RCC_STRAP2_RCC_BIF_STRAP6__STRAP_MEM_AP_SIZE_PIN_MASK                                                 0x00007C00L
+#define RCC_STRAP2_RCC_BIF_STRAP6__RESERVED_BIF_STRAP6_MASK                                                   0xFFFF8000L
+//RCC_STRAP2_RCC_BIF_STRAP7
+#define RCC_STRAP2_RCC_BIF_STRAP7__STRAP_GEN6_DIS__SHIFT                                                      0x19
+#define RCC_STRAP2_RCC_BIF_STRAP7__STRAP_BIF_KILL_GEN6__SHIFT                                                 0x1a
+#define RCC_STRAP2_RCC_BIF_STRAP7__STRAP_GEN6_DIS_MASK                                                        0x02000000L
+#define RCC_STRAP2_RCC_BIF_STRAP7__STRAP_BIF_KILL_GEN6_MASK                                                   0x04000000L
+//RCC_STRAP2_RCC_BIF_STRAP8
+//RCC_STRAP2_RCC_BIF_STRAP9
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0__SHIFT                                0x13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0__SHIFT                                     0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0__SHIFT                                   0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0__SHIFT                            0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0__SHIFT                             0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0__SHIFT                             0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0__SHIFT                                      0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_DEVICE_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ARI_EN_DN_DEV0_MASK                                            0x00010000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_ACS_EN_DN_DEV0_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_AER_EN_DN_DEV0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_CPL_ABORT_ERR_EN_DN_DEV0_MASK                                  0x00080000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_NEGO_GLOBAL_EN_DEV0_MASK                                       0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_INTERRUPT_PIN_DN_DEV0_MASK                                     0x00E00000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_IGNORE_E2E_PREFIX_UR_DN_DEV0_MASK                              0x01000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_PAYLOAD_SUPPORT_DN_DEV0_MASK                               0x0E000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_MAX_LINK_WIDTH_SUPPORT_DEV0_MASK                               0x70000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP0__STRAP_EPF0_DUMMY_EN_DEV0_MASK                                        0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0__SHIFT                                   0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_ID_DN_DEV0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP1__STRAP_SUBSYS_VEN_ID_DN_DEV0_MASK                                     0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0__SHIFT                           0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0__SHIFT                  0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0__SHIFT                         0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0__SHIFT                       0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0__SHIFT                                0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0__SHIFT                                   0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0__SHIFT                              0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0__SHIFT                           0x1a
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DEV0_MASK                                0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_NO_EQ_NEED_SUPPORTED_DN_DEV0_MASK                             0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE1_SUPPORTED_DEV0_MASK                    0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFID_TS_USAGE_MODE2_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODEING_ON_DEV0_MASK                           0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_TRANSMITTER_PRECODE_REQUEST_DEV0_MASK                         0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MODIFIED_TS_INFOR1_DEV0_MASK                                  0x0007FFC0L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_GEN6_COMPLIANCE_DEV0_MASK                                     0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DEV0_MASK                                0x03000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP10__STRAP_MAX_E2E_TLP_PREFIXES_DN_DEV0_MASK                             0x0C000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0__SHIFT                             0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0__SHIFT                                 0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0__SHIFT                                      0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0__SHIFT                                         0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0__SHIFT                               0x1e
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                  0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_MODIFIED_TS_VENDOR_ID_DEV0_MASK                               0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_RESET_TIME_DN_DEV0_MASK                                   0x0FFF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_VALID_DN_DEV0_MASK                                        0x10000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_RTR_EN_DN_DEV0_MASK                                           0x20000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_SDPVW_REG_UPDATE_EN_DEV0_MASK                                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP11__STRAP_VF_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                    0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0__SHIFT                                0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                     0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                     0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_RECEIVER_L0P_SUPPORTED_DEV0__SHIFT                            0x1a
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_PORT_L0P_EXIT_LATENCY_DEV0__SHIFT                             0x1b
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_MODIFIED_TS_INFOR2_DEV0_MASK                                  0x00FFFFFFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                       0x01000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_14BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                       0x02000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_RECEIVER_L0P_SUPPORTED_DEV0_MASK                              0x04000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP12__STRAP_PORT_L0P_EXIT_LATENCY_DEV0_MASK                               0x38000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0__SHIFT                          0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0__SHIFT     0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0__SHIFT                        0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0__SHIFT                               0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_COUNT_DEV0_MASK                            0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_SELECTIVE_ENABLE_SUPPORTED_DEV0_MASK       0x00000100L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_ALTERNATE_PROTOCOL_DETAILS_DEV0_MASK                          0x000FFE00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP13__STRAP_RTR_D3HOTD0_TIME_DN_DEV0_MASK                                 0xFFF00000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0__SHIFT                               0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0__SHIFT                             0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0__SHIFT                                 0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0__SHIFT                          0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0__SHIFT                                         0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0__SHIFT                                      0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0__SHIFT                                   0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DEV0__SHIFT                                            0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RETIMER_L0P_EXIT_LATENCY_DEV0__SHIFT                          0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_DEVICE3_EN_DN_DEV0__SHIFT                                     0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DN_DEV0__SHIFT                                         0xd
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_CTO_LOGGING_SUPPORT_DEV0_MASK                                 0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ACS_ENH_CAPABILITY_DN_DEV0_MASK                               0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_COMMAND_COMPLETED_DEV0_MASK                                   0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_ERR_COR_SUBCLASS_CAPABLE_DEV0_MASK                            0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_DOE_EN_UP_DEV0_MASK                                           0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DEV0_MASK                                        0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RX_MPS_FIXED_DN_DEV0_MASK                                     0x00000040L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DEV0_MASK                                              0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_RETIMER_L0P_EXIT_LATENCY_DEV0_MASK                            0x00000E00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_DEVICE3_EN_DN_DEV0_MASK                                       0x00001000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP14__STRAP_IDE_EN_DN_DEV0_MASK                                           0x00002000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP15__STRAP_DSN_CODE_L_DN_DEV0__SHIFT                                     0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP15__STRAP_DSN_CODE_L_DN_DEV0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP16
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP16__STRAP_DSN_CODE_H_DN_DEV0__SHIFT                                     0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP16__STRAP_DSN_CODE_H_DN_DEV0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP17
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_LINK_IDE_STREAM_SUPPORTED_DEV0__SHIFT                         0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_STREAM_SUPPORTED_DEV0__SHIFT                    0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_DEV0__SHIFT                 0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_DEV0__SHIFT                  0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_AGGREGATION_SUPPORTED_DEV0__SHIFT                             0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_PCRC_SUPPORTED_DEV0__SHIFT                                    0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_IDE_KM_PROTOCOL_SUPPORTED_DEV0__SHIFT                         0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_DEV0__SHIFT               0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_DEV0__SHIFT                 0xd
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_DEV0__SHIFT            0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_TEE_LIMITED_SUPPORTED_DEV0__SHIFT                             0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_XT_SUPPORTED_DEV0__SHIFT                                      0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_LINK_IDE_STREAM_SUPPORTED_DEV0_MASK                           0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_STREAM_SUPPORTED_DEV0_MASK                      0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_DEV0_MASK                   0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_DEV0_MASK                    0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_AGGREGATION_SUPPORTED_DEV0_MASK                               0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_PCRC_SUPPORTED_DEV0_MASK                                      0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_IDE_KM_PROTOCOL_SUPPORTED_DEV0_MASK                           0x00000040L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_DEV0_MASK                 0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_DEV0_MASK                   0x0000E000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_DEV0_MASK              0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_TEE_LIMITED_SUPPORTED_DEV0_MASK                               0x01000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP17__STRAP_XT_SUPPORTED_DEV0_MASK                                        0x02000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP18__STRAP_NUM_ADDR_ASSOCIATION_REG_BLKS_DEV0__SHIFT                     0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP18__STRAP_NUM_ADDR_ASSOCIATION_REG_BLKS_DEV0_MASK                       0x0000000FL
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0__SHIFT                                          0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0__SHIFT                                      0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0__SHIFT                                          0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0__SHIFT                                      0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0__SHIFT                                        0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0__SHIFT                             0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0__SHIFT                                0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0__SHIFT                                    0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0__SHIFT                              0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0__SHIFT                      0xd
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0__SHIFT                                    0xe
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0__SHIFT                                            0xf
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0__SHIFT                                    0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0__SHIFT                                    0x11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0__SHIFT                             0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0__SHIFT                                   0x17
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0__SHIFT                              0x1a
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0__SHIFT                                    0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DE_EMPHASIS_SEL_DN_DEV0_MASK                                   0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_DSN_EN_DN_DEV0_MASK                                            0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_E2E_PREFIX_EN_DEV0_MASK                                        0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECN1P1_EN_DEV0_MASK                                            0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_CHECK_EN_DEV0_MASK                                        0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_ECRC_GEN_EN_DEV0_MASK                                          0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_FMT_SUPPORTED_DEV0_MASK                               0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXTENDED_TAG_ECN_EN_DEV0_MASK                                  0x00000100L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_EXT_VC_COUNT_DN_DEV0_MASK                                      0x00000E00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_FIRST_RCVD_ERR_LOG_DN_DEV0_MASK                                0x00001000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_POISONED_ADVISORY_NONFATAL_DN_DEV0_MASK                        0x00002000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_COMPLIANCE_DEV0_MASK                                      0x00004000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN2_EN_DEV0_MASK                                              0x00008000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN3_COMPLIANCE_DEV0_MASK                                      0x00010000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_GEN4_COMPLIANCE_DEV0_MASK                                      0x00020000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_ACCEPTABLE_LATENCY_DEV0_MASK                               0x00700000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L0S_EXIT_LATENCY_DEV0_MASK                                     0x03800000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_ACCEPTABLE_LATENCY_DEV0_MASK                                0x1C000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP2__STRAP_L1_EXIT_LATENCY_DEV0_MASK                                      0xE0000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0__SHIFT                     0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0__SHIFT                                             0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0__SHIFT                                          0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0__SHIFT                                0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0__SHIFT                                          0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0__SHIFT                                  0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0__SHIFT                                     0x9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0xb
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0xe
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0__SHIFT  0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0__SHIFT  0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0__SHIFT                                         0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0__SHIFT                                      0x1b
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0__SHIFT                                       0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0__SHIFT                                         0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LINK_BW_NOTIFICATION_CAP_DN_EN_DEV0_MASK                       0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DEV0_MASK                                               0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_LTR_EN_DN_DEV0_MASK                                            0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MAX_PAYLOAD_SUPPORT_DEV0_MASK                                  0x00000038L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSI_EN_DN_DEV0_MASK                                            0x00000040L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_MSTCPL_TIMEOUT_EN_DEV0_MASK                                    0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_NO_SOFT_RESET_DN_DEV0_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_OBFF_SUPPORTED_DEV0_MASK                                       0x00000600L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x00003800L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_DOWNSTREAM_PORT_TX_PRESET_DEV0_MASK  0x0003C000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_RX_PRESET_HINT_DEV0_MASK  0x001C0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PCIE_LANE_EQUALIZATION_CNTL_UPSTREAM_PORT_TX_PRESET_DEV0_MASK  0x01E00000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DEV0_MASK                                           0x06000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PM_SUPPORT_DN_DEV0_MASK                                        0x18000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_ATOMIC_EN_DN_DEV0_MASK                                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP3__STRAP_PMC_DSI_DN_DEV0_MASK                                           0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0__SHIFT                              0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0__SHIFT                              0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_0_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_1_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_2_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP4__STRAP_PWR_BUDGET_DATA_8T0_3_DEV0_MASK                                0xFF000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0__SHIFT                        0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0__SHIFT                                 0x11
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0__SHIFT                                  0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0__SHIFT                                           0x13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0__SHIFT                                           0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0__SHIFT                                        0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0__SHIFT                                0x16
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0__SHIFT                           0x17
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0__SHIFT                        0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0__SHIFT                        0x19
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0__SHIFT                     0x1a
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0__SHIFT                         0x1b
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0__SHIFT                          0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0__SHIFT                       0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0__SHIFT                                         0x1e
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0__SHIFT                                            0x1f
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_4_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_DATA_8T0_5_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_PWR_BUDGET_SYSTEM_ALLOCATED_DEV0_MASK                          0x00010000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_64BIT_EN_DN_DEV0_MASK                                   0x00020000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ATOMIC_ROUTING_EN_DEV0_MASK                                    0x00040000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_VC_EN_DN_DEV0_MASK                                             0x00080000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DEV0_MASK                                             0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_TwoVC_EN_DN_DEV0_MASK                                          0x00200000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_LOCAL_DLF_SUPPORTED_DEV0_MASK                                  0x00400000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_SOURCE_VALIDATION_DN_DEV0_MASK                             0x00800000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_TRANSLATION_BLOCKING_DN_DEV0_MASK                          0x01000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_REQUEST_REDIRECT_DN_DEV0_MASK                          0x02000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_COMPLETION_REDIRECT_DN_DEV0_MASK                       0x04000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_UPSTREAM_FORWARDING_DN_DEV0_MASK                           0x08000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_P2P_EGRESS_CONTROL_DN_DEV0_MASK                            0x10000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_ACS_DIRECT_TRANSLATED_P2P_DN_DEV0_MASK                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_MSI_MAP_EN_DEV0_MASK                                           0x40000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP5__STRAP_SSID_EN_DEV0_MASK                                              0x80000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0__SHIFT                                         0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0__SHIFT                         0x1
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0__SHIFT                                    0x2
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x3
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0__SHIFT                          0x4
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0__SHIFT                      0x5
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                      0x6
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0__SHIFT                   0x7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0__SHIFT                              0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0__SHIFT                            0x12
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0__SHIFT                     0x13
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0__SHIFT                                  0x15
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0__SHIFT     0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0__SHIFT     0x1c
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_CFG_CRS_EN_DEV0_MASK                                           0x00000001L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_SMN_ERR_STATUS_MASK_EN_DNS_DEV0_MASK                           0x00000002L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_INTERNAL_ERR_EN_DEV0_MASK                                      0x00000004L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM1_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000008L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_RTM2_PRESENCE_DET_SUPPORT_DEV0_MASK                            0x00000010L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_COMPLETER_SUPPORTED_DEV0_MASK                        0x00000020L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                        0x00000040L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_VF_10BIT_TAG_REQUESTER_SUPPORTED_DEV0_MASK                     0x00000080L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x00000F00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_16GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0x0000F000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TPH_CPLR_SUPPORTED_DN_DEV0_MASK                                0x00030000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_MSI_EXT_MSG_DATA_CAP_DN_DEV0_MASK                              0x00040000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_NO_COMMAND_COMPLETED_SUPPORTED_DEV0_MASK                       0x00080000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_GEN5_COMPLIANCE_DEV0_MASK                                      0x00100000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_TARGET_LINK_SPEED_DEV0_MASK                                    0x00E00000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_DSP_TX_PRESET_DEV0_MASK       0x0F000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP6__STRAP_PCIE_32GT_LANE_EQUALIZATION_CNTL_USP_TX_PRESET_DEV0_MASK       0xF0000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP7
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0__SHIFT                                    0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0__SHIFT                                    0xc
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0__SHIFT                                          0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0__SHIFT                                          0x1d
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_PORT_NUMBER_DEV0_MASK                                          0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MAJOR_REV_ID_DN_DEV0_MASK                                      0x00000F00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_MINOR_REV_ID_DN_DEV0_MASK                                      0x0000F000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_RP_BUSNUM_DEV0_MASK                                            0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_DEVNUM_DEV0_MASK                                            0x1F000000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP7__STRAP_DN_FUNCID_DEV0_MASK                                            0xE0000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0__SHIFT                              0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0__SHIFT                              0x18
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_6_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_7_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_8_DEV0_MASK                                0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP8__STRAP_PWR_BUDGET_DATA_8T0_9_DEV0_MASK                                0xFF000000L
+//RCC_STRAP2_RCC_DEV0_PORT_STRAP9
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0__SHIFT                              0x8
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0__SHIFT                                       0x10
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_a_DEV0_MASK                                0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_PWR_BUDGET_DATA_8T0_b_DEV0_MASK                                0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_PORT_STRAP9__STRAP_VENDOR_ID_DN_DEV0_MASK                                         0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0__SHIFT                                    0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0__SHIFT                                         0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0__SHIFT                           0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0__SHIFT                                      0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0__SHIFT                                      0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_DEVICE_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F0_MASK                                      0x000F0000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_MINOR_REV_ID_DEV0_F0_MASK                                      0x00F00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_ATI_REV_ID_DEV0_F0_MASK                                        0x0F000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_FUNC_EN_DEV0_F0_MASK                                           0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F0_MASK                             0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D1_SUPPORT_DEV0_F0_MASK                                        0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP0__STRAP_D2_SUPPORT_DEV0_F0_MASK                                        0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP1
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0__SHIFT                       0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F0_MASK                                0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F0_MASK                         0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0__SHIFT                                 0x8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0__SHIFT                                0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F0_MASK                                   0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F0_MASK                                   0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F0_MASK                                  0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F0_MASK                                  0xFF000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0__SHIFT                                      0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP14__STRAP_VENDOR_ID_DEV0_F0_MASK                                        0x0000FFFFL
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0__SHIFT                                  0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0__SHIFT                                      0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0__SHIFT                     0x19
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0__SHIFT                       0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F0_MASK                                   0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F0_MASK                                    0x00FFF000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_RTR_VALID_DEV0_F0_MASK                                        0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_INVALIDATE_QUEUE_DEPTH_DEV0_F0_MASK                       0x3E000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP15__STRAP_ATS_PAGE_ALIGNED_REQUEST_DEV0_F0_MASK                         0x40000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0__SHIFT                               0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F0_MASK                                     0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F0_MASK                                 0x00FFF000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0__SHIFT                                   0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0__SHIFT                                0xd
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F0_MASK                                0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_VALID_DEV0_F0_MASK                                     0x00001000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F0_MASK                                  0x01FFE000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0__SHIFT                            0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0__SHIFT                            0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F0_MASK                              0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F0_MASK                              0x000FF000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP19
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F0__SHIFT                          0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F0__SHIFT                                0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F0_MASK                            0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F0_MASK                                  0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP2
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0__SHIFT                      0x1
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0__SHIFT                     0x2
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DEVICE3_EN_DEV0_F0__SHIFT                                      0x3
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0__SHIFT                    0x4
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0__SHIFT                                       0x6
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0__SHIFT                                   0x7
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0__SHIFT                                 0x9
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0__SHIFT                          0xe
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0__SHIFT                                          0xf
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0__SHIFT                                0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0__SHIFT                                          0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0__SHIFT                                           0x17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0__SHIFT                                   0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0__SHIFT                                        0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0__SHIFT                  0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0__SHIFT               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0__SHIFT                       0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_SRIOV_EN_DEV0_F0_MASK                                          0x00000001L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RELAXED_ORDERING_SUPPORTED_DEV0_F0_MASK                        0x00000002L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F0_MASK                       0x00000004L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DEVICE3_EN_DEV0_F0_MASK                                        0x00000008L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F0_MASK                      0x00000010L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_64BAR_DIS_DEV0_F0_MASK                                         0x00000040L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F0_MASK                                     0x00000080L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F0_MASK                                   0x00003E00L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F0_MASK                            0x00004000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ARI_EN_DEV0_F0_MASK                                            0x00008000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_AER_EN_DEV0_F0_MASK                                            0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ACS_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_ATS_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F0_MASK                                  0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DPA_EN_DEV0_F0_MASK                                            0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_DSN_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_VC_EN_DEV0_F0_MASK                                             0x00800000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F0_MASK                                     0x07000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F0_MASK                                       0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EN_DEV0_F0_MASK                                          0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F0_MASK                    0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F0_MASK                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F0_MASK                         0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP22
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0__SHIFT                             0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_MEM_AP_SIZE_DEV0_F0__SHIFT                                    0xc
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_VF_MEM_AP_SIZE_DEV0_F0__SHIFT                                 0x11
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_PF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0__SHIFT                  0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_VF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0__SHIFT                  0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_GPUIOV_VSEC_LENGTH_DEV0_F0_MASK                               0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_MEM_AP_SIZE_DEV0_F0_MASK                                      0x0001F000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_VF_MEM_AP_SIZE_DEV0_F0_MASK                                   0x003E0000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_PF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0_MASK                    0x07C00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP22__STRAP_VF_RESIZABLE_BAR_DEFAULT_SIZE_DEV0_F0_MASK                    0xF8000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP23
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP23__STRAP_DSN_CODE_L_DEV0_F0__SHIFT                                     0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP23__STRAP_DSN_CODE_L_DEV0_F0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP24
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP24__STRAP_DSN_CODE_H_DEV0_F0__SHIFT                                     0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP24__STRAP_DSN_CODE_H_DEV0_F0_MASK                                       0xFFFFFFFFL
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP3
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0__SHIFT                      0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0__SHIFT                              0x13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0__SHIFT                                         0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0__SHIFT                                  0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0__SHIFT                                         0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0__SHIFT             0x19
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0__SHIFT                        0x1a
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0__SHIFT                       0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0__SHIFT                                0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0__SHIFT                                       0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0__SHIFT                               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0__SHIFT                                          0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SUBSYS_ID_DEV0_F0_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F0_MASK                        0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PWR_EN_DEV0_F0_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F0_MASK                                0x00080000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_EN_DEV0_F0_MASK                                           0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_MSIX_TABLE_BIR_DEV0_F0_MASK                                    0x00E00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_PMC_DSI_DEV0_F0_MASK                                           0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F0_MASK               0x02000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F0_MASK                          0x04000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F0_MASK                         0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F0_MASK                                  0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_CLK_PM_EN_DEV0_F0_MASK                                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F0_MASK                                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP3__STRAP_RTR_EN_DEV0_F0_MASK                                            0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP4
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0__SHIFT                                       0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0__SHIFT                                     0x17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0__SHIFT                                   0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0__SHIFT                                  0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_RESERVED_STRAP4_DEV0_F0_MASK                                   0x000003FFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_DOE_EN_DEV0_F0_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_ATOMIC_EN_DEV0_F0_MASK                                         0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_FLR_EN_DEV0_F0_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_PME_SUPPORT_DEV0_F0_MASK                                       0x0F800000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F0_MASK                                     0x70000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F0_MASK                                    0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP5
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0__SHIFT                            0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F0_MASK                                     0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_AUX_CURRENT_DEV0_F0_MASK                                       0x38000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F0_MASK                              0x40000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0__SHIFT                                0x3
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0__SHIFT                                     0x4
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0__SHIFT                                      0x7
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0__SHIFT                                     0xd
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0__SHIFT                           0x10
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0__SHIFT                                  0x17
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0__SHIFT                                0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0__SHIFT                           0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_APER_SIZE_DEV0_F0_MASK                                0x00000007L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_DOORBELL_BAR_DIS_DEV0_F0_MASK                                  0x00000008L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_ROM_AP_SIZE_DEV0_F0_MASK                                       0x00000070L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_IO_BAR_DIS_DEV0_F0_MASK                                        0x00000080L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_LFB_ERRMSG_EN_DEV0_F0_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_REG_AP_SIZE_DEV0_F0_MASK                                       0x0000E000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_DOORBELL_APER_SIZE_DEV0_F0_MASK                             0x00070000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_REG_AP_SIZE_DEV0_F0_MASK                                    0x03800000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F0_MASK                                  0x38000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F0_MASK                             0xC0000000L
+//RCC_STRAP2_RCC_DEV0_EPF0_STRAP9
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0__SHIFT                           0x0
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0__SHIFT                               0x12
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0__SHIFT                        0x13
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0__SHIFT                                 0x14
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0__SHIFT                                    0x15
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0__SHIFT                                 0x16
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0__SHIFT                                 0x18
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F0__SHIFT                                  0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F0_MASK                             0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_BAR_COMPLIANCE_EN_DEV0_F0_MASK                                 0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_NBIF_ROM_BAR_DIS_CHICKEN_DEV0_F0_MASK                          0x00080000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_VF_REG_PROT_DIS_DEV0_F0_MASK                                   0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_ALWAYS_ON_DEV0_F0_MASK                                      0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_FB_CPL_TYPE_SEL_DEV0_F0_MASK                                   0x00C00000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_GPUIOV_VSEC_REV_DEV0_F0_MASK                                   0x0F000000L
+#define RCC_STRAP2_RCC_DEV0_EPF0_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F0_MASK                                    0x10000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1__SHIFT                                    0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1__SHIFT                                    0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1__SHIFT                                         0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1__SHIFT                           0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1__SHIFT                                      0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1__SHIFT                                      0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_DEVICE_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MAJOR_REV_ID_DEV0_F1_MASK                                      0x000F0000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_MINOR_REV_ID_DEV0_F1_MASK                                      0x00F00000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_FUNC_EN_DEV0_F1_MASK                                           0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_LEGACY_DEVICE_TYPE_EN_DEV0_F1_MASK                             0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D1_SUPPORT_DEV0_F1_MASK                                        0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP0__STRAP_D2_SUPPORT_DEV0_F1_MASK                                        0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP1
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F1__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F1__SHIFT                       0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_VF_DEVICE_ID_DEV0_F1_MASK                                0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP1__STRAP_SRIOV_SUPPORTED_PAGE_SIZE_DEV0_F1_MASK                         0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP10
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP11
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP12
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP13
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F1__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F1__SHIFT                                 0x8
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F1__SHIFT                                0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F1__SHIFT                                0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_PIF_DEV0_F1_MASK                                   0x000000FFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_SUB_DEV0_F1_MASK                                   0x0000FF00L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP13__STRAP_CLASS_CODE_BASE_DEV0_F1_MASK                                  0x00FF0000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP13__STRAP_SRIOV_TOTAL_VFS_DEV0_F1_MASK                                  0xFF000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP14__STRAP_VENDOR_ID_DEV0_F1__SHIFT                                      0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP14__STRAP_VENDOR_ID_DEV0_F1_MASK                                        0x0000FFFFL
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP15
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F1__SHIFT                                 0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F1__SHIFT                                  0xc
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_VALID_DEV0_F1__SHIFT                                      0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_RESET_TIME_DEV0_F1_MASK                                   0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_DLUP_TIME_DEV0_F1_MASK                                    0x00FFF000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP15__STRAP_RTR_VALID_DEV0_F1_MASK                                        0x01000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP16
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F1__SHIFT                               0xc
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_FLR_TIME_DEV0_F1_MASK                                     0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP16__STRAP_RTR_D3HOTD0_TIME_DEV0_F1_MASK                                 0x00FFF000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP17
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F1__SHIFT                              0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_VALID_DEV0_F1__SHIFT                                   0xc
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F1__SHIFT                                0xd
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_RESET_TIME_DEV0_F1_MASK                                0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_VALID_DEV0_F1_MASK                                     0x00001000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP17__STRAP_RTR_VF_FLR_TIME_DEV0_F1_MASK                                  0x01FFE000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F1__SHIFT                            0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F1__SHIFT                            0xc
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP18__STRAP_RTR_VF_D3HOTD0_TIME_DEV0_F1_MASK                              0x00000FFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP18__STRAP_SRIOV_FUNC_DEP_LINK_DEV0_F1_MASK                              0x000FF000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP19
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F1__SHIFT                          0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F1__SHIFT                                0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_FIRST_VF_OFFSET_DEV0_F1_MASK                            0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP19__STRAP_SRIOV_VF_STRIDE_DEV0_F1_MASK                                  0xFFFF0000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP2
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_SRIOV_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F1__SHIFT                     0x2
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DEVICE3_EN_DEV0_F1__SHIFT                                      0x3
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1__SHIFT                    0x4
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1__SHIFT                                   0x7
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1__SHIFT                                   0x8
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1__SHIFT                                 0x9
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1__SHIFT                          0xe
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1__SHIFT                                          0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1__SHIFT                                0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1__SHIFT                                          0x15
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1__SHIFT                                   0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1__SHIFT                                        0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1__SHIFT                  0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1__SHIFT               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1__SHIFT                       0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_SRIOV_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ARI_CAP_HIERARCHY_PRESERVED_DEV0_F1_MASK                       0x00000004L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DEVICE3_EN_DEV0_F1_MASK                                        0x00000008L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_TEE_EXCLUSIVE_ATTR_SUPPORTED_DEV0_F1_MASK                      0x00000010L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_NO_SOFT_RESET_DEV0_F1_MASK                                     0x00000080L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_RESIZE_BAR_EN_DEV0_F1_MASK                                     0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MAX_PASID_WIDTH_DEV0_F1_MASK                                   0x00003E00L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_PERVECTOR_MASK_CAP_DEV0_F1_MASK                            0x00004000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_AER_EN_DEV0_F1_MASK                                            0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_ACS_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_CPL_ABORT_ERR_EN_DEV0_F1_MASK                                  0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_DPA_EN_DEV0_F1_MASK                                            0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_MSI_MULTI_CAP_DEV0_F1_MASK                                     0x07000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PAGE_REQ_EN_DEV0_F1_MASK                                       0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EN_DEV0_F1_MASK                                          0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_EXE_PERMISSION_SUPPORTED_DEV0_F1_MASK                    0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_GLOBAL_INVALIDATE_SUPPORTED_DEV0_F1_MASK                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP2__STRAP_PASID_PRIV_MODE_SUPPORTED_DEV0_F1_MASK                         0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP20
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP21
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP23
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP24
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP3
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1__SHIFT                                       0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1__SHIFT                      0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1__SHIFT                                          0x11
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1__SHIFT                                          0x12
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1__SHIFT                              0x13
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1__SHIFT                                         0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1__SHIFT                                         0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1__SHIFT             0x19
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1__SHIFT                        0x1a
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1__SHIFT                       0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F1__SHIFT                                0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1__SHIFT                                       0x1d
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1__SHIFT                               0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1__SHIFT                                          0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SUBSYS_ID_DEV0_F1_MASK                                         0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_POISONED_ADVISORY_NONFATAL_DEV0_F1_MASK                        0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PWR_EN_DEV0_F1_MASK                                            0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_EN_DEV0_F1_MASK                                            0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSI_CLR_PENDING_EN_DEV0_F1_MASK                                0x00080000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_MSIX_EN_DEV0_F1_MASK                                           0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_PMC_DSI_DEV0_F1_MASK                                           0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRANSLATED_REQ_WITH_PASID_SUPPORTED_DEV0_F1_MASK               0x02000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_ALL_MSI_EVENT_SUPPORT_EN_DEV0_F1_MASK                          0x04000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_SMN_ERR_STATUS_MASK_EN_EP_DEV0_F1_MASK                         0x08000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_VF_RESIZE_BAR_EN_DEV0_F1_MASK                                  0x10000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_CLK_PM_EN_DEV0_F1_MASK                                         0x20000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_TRUE_PM_STATUS_EN_DEV0_F1_MASK                                 0x40000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP3__STRAP_RTR_EN_DEV0_F1_MASK                                            0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP4
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1__SHIFT                                 0x14
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1__SHIFT                                       0x15
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1__SHIFT                                          0x16
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1__SHIFT                                     0x17
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1__SHIFT                                   0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1__SHIFT                                  0x1f
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_64BIT_EN_DEV0_F1_MASK                                   0x00100000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_ATOMIC_EN_DEV0_F1_MASK                                         0x00200000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_FLR_EN_DEV0_F1_MASK                                            0x00400000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_PME_SUPPORT_DEV0_F1_MASK                                       0x0F800000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_INTERRUPT_PIN_DEV0_F1_MASK                                     0x70000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP4__STRAP_AUXPWR_SUPPORT_DEV0_F1_MASK                                    0x80000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP5
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1__SHIFT                                   0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1__SHIFT                                     0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1__SHIFT                            0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_SUBSYS_VEN_ID_DEV0_F1_MASK                                     0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_AUX_CURRENT_DEV0_F1_MASK                                       0x38000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP5__STRAP_MSI_EXT_MSG_DATA_CAP_DEV0_F1_MASK                              0x40000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP6
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1__SHIFT                                        0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x1
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1__SHIFT                                  0x2
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1__SHIFT                                        0x8
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x9
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1__SHIFT                                  0xa
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1__SHIFT                                        0x10
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x11
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1__SHIFT                                  0x12
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1__SHIFT                                        0x18
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x19
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1__SHIFT                                        0x1a
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_PREFETCHABLE_EN_DEV0_F1__SHIFT                           0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_EN_DEV0_F1_MASK                                          0x00000001L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_PREFETCHABLE_EN_DEV0_F1_MASK                             0x00000002L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER0_64BAR_EN_DEV0_F1_MASK                                    0x00000004L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_EN_DEV0_F1_MASK                                          0x00000100L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_PREFETCHABLE_EN_DEV0_F1_MASK                             0x00000200L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER1_64BAR_EN_DEV0_F1_MASK                                    0x00000400L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_EN_DEV0_F1_MASK                                          0x00010000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_PREFETCHABLE_EN_DEV0_F1_MASK                             0x00020000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER2_64BAR_EN_DEV0_F1_MASK                                    0x00040000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_EN_DEV0_F1_MASK                                          0x01000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER3_PREFETCHABLE_EN_DEV0_F1_MASK                             0x02000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_EN_DEV0_F1_MASK                                          0x04000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP6__STRAP_APER4_PREFETCHABLE_EN_DEV0_F1_MASK                             0x08000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP7
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP8
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F1__SHIFT                                0x1b
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F1__SHIFT                           0x1e
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP8__STRAP_VF_MSI_MULTI_CAP_DEV0_F1_MASK                                  0x38000000L
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP8__STRAP_SRIOV_VF_MAPPING_MODE_DEV0_F1_MASK                             0xC0000000L
+//RCC_STRAP2_RCC_DEV0_EPF1_STRAP9
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F1__SHIFT                           0x0
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F1__SHIFT                                  0x1c
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP9__STRAP_OUTSTAND_PAGE_REQ_CAP_DEV0_F1_MASK                             0x0000FFFFL
+#define RCC_STRAP2_RCC_DEV0_EPF1_STRAP9__STRAP_TEE_IO_SUPPORT_DEV0_F1_MASK                                    0x10000000L
+
+
+// addressBlock: nbif0_hpdma0_hpdma_private_hpdma_private_regblk
+//NBIF_SDP_PERF_COUNTER_2
+#define NBIF_SDP_PERF_COUNTER_2__SDP_PERF_CNT_0_SEL__SHIFT                                                    0x0
+#define NBIF_SDP_PERF_COUNTER_2__SDP_PERF_CNT_1_SEL__SHIFT                                                    0x8
+#define NBIF_SDP_PERF_COUNTER_2__SDP_PERF_CNT_2_SEL__SHIFT                                                    0x10
+#define NBIF_SDP_PERF_COUNTER_2__SDP_PERF_CNT_3_SEL__SHIFT                                                    0x18
+#define NBIF_SDP_PERF_COUNTER_2__SDP_PERF_CNT_0_SEL_MASK                                                      0x000000FFL
+#define NBIF_SDP_PERF_COUNTER_2__SDP_PERF_CNT_1_SEL_MASK                                                      0x0000FF00L
+#define NBIF_SDP_PERF_COUNTER_2__SDP_PERF_CNT_2_SEL_MASK                                                      0x00FF0000L
+#define NBIF_SDP_PERF_COUNTER_2__SDP_PERF_CNT_3_SEL_MASK                                                      0xFF000000L
+//NBIF_SHUB_TODET_CLIENT_STATUS3
+#define NBIF_SHUB_TODET_CLIENT_STATUS3__NBIF_SHUB_TODET_CLIENT_STATUS3__SHIFT                                 0x0
+#define NBIF_SHUB_TODET_CLIENT_STATUS3__NBIF_SHUB_TODET_CLIENT_STATUS3_MASK                                   0x000003FFL
+//BIF_AER_ERR_LOG_DEV0_F0
+#define BIF_AER_ERR_LOG_DEV0_F0__UR_ATOMIC_OPCODE_DEV0_F0__SHIFT                                              0x0
+#define BIF_AER_ERR_LOG_DEV0_F0__UR_ATOMIC_REQEN_LOW_DEV0_F0__SHIFT                                           0x1
+#define BIF_AER_ERR_LOG_DEV0_F0__UR_ATOMIC_LENGTH_DEV0_F0__SHIFT                                              0x2
+#define BIF_AER_ERR_LOG_DEV0_F0__UR_ATOMIC_NR_DEV0_F0__SHIFT                                                  0x3
+#define BIF_AER_ERR_LOG_DEV0_F0__DMA_ON_BME_LOW_DEV0_F0__SHIFT                                                0x4
+#define BIF_AER_ERR_LOG_DEV0_F0__PASID_ERR_DEV0_F0__SHIFT                                                     0x5
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0__SHIFT                                        0x10
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0__SHIFT                                     0x11
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0__SHIFT                                        0x12
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_NR_DEV0_F0__SHIFT                                            0x13
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_DMA_ON_BME_LOW_DEV0_F0__SHIFT                                          0x14
+#define BIF_AER_ERR_LOG_DEV0_F0__PASID_ERR_CLR_DEV0_F0__SHIFT                                                 0x15
+#define BIF_AER_ERR_LOG_DEV0_F0__UR_ATOMIC_OPCODE_DEV0_F0_MASK                                                0x00000001L
+#define BIF_AER_ERR_LOG_DEV0_F0__UR_ATOMIC_REQEN_LOW_DEV0_F0_MASK                                             0x00000002L
+#define BIF_AER_ERR_LOG_DEV0_F0__UR_ATOMIC_LENGTH_DEV0_F0_MASK                                                0x00000004L
+#define BIF_AER_ERR_LOG_DEV0_F0__UR_ATOMIC_NR_DEV0_F0_MASK                                                    0x00000008L
+#define BIF_AER_ERR_LOG_DEV0_F0__DMA_ON_BME_LOW_DEV0_F0_MASK                                                  0x00000010L
+#define BIF_AER_ERR_LOG_DEV0_F0__PASID_ERR_DEV0_F0_MASK                                                       0x00000020L
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_MASK                                          0x00010000L
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_MASK                                       0x00020000L
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_MASK                                          0x00040000L
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_UR_ATOMIC_NR_DEV0_F0_MASK                                              0x00080000L
+#define BIF_AER_ERR_LOG_DEV0_F0__CLEAR_DMA_ON_BME_LOW_DEV0_F0_MASK                                            0x00100000L
+#define BIF_AER_ERR_LOG_DEV0_F0__PASID_ERR_CLR_DEV0_F0_MASK                                                   0x00200000L
+//BIF_AER_ERR_LOG_DEV0_F0_vf0
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__UR_ATOMIC_OPCODE_DEV0_F0_vf0__SHIFT                                      0x0
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf0__SHIFT                                   0x1
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__UR_ATOMIC_LENGTH_DEV0_F0_vf0__SHIFT                                      0x2
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__UR_ATOMIC_NR_DEV0_F0_vf0__SHIFT                                          0x3
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__DMA_ON_BME_LOW_DEV0_F0_vf0__SHIFT                                        0x4
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf0__SHIFT                                0x10
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf0__SHIFT                             0x11
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf0__SHIFT                                0x12
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf0__SHIFT                                    0x13
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf0__SHIFT                                  0x14
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__UR_ATOMIC_OPCODE_DEV0_F0_vf0_MASK                                        0x00000001L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf0_MASK                                     0x00000002L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__UR_ATOMIC_LENGTH_DEV0_F0_vf0_MASK                                        0x00000004L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__UR_ATOMIC_NR_DEV0_F0_vf0_MASK                                            0x00000008L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__DMA_ON_BME_LOW_DEV0_F0_vf0_MASK                                          0x00000010L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf0_MASK                                  0x00010000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf0_MASK                               0x00020000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf0_MASK                                  0x00040000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf0_MASK                                      0x00080000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf0__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf0_MASK                                    0x00100000L
+//BIF_AER_ERR_LOG_DEV0_F0_vf1
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__UR_ATOMIC_OPCODE_DEV0_F0_vf1__SHIFT                                      0x0
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf1__SHIFT                                   0x1
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__UR_ATOMIC_LENGTH_DEV0_F0_vf1__SHIFT                                      0x2
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__UR_ATOMIC_NR_DEV0_F0_vf1__SHIFT                                          0x3
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__DMA_ON_BME_LOW_DEV0_F0_vf1__SHIFT                                        0x4
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf1__SHIFT                                0x10
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf1__SHIFT                             0x11
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf1__SHIFT                                0x12
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf1__SHIFT                                    0x13
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf1__SHIFT                                  0x14
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__UR_ATOMIC_OPCODE_DEV0_F0_vf1_MASK                                        0x00000001L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf1_MASK                                     0x00000002L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__UR_ATOMIC_LENGTH_DEV0_F0_vf1_MASK                                        0x00000004L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__UR_ATOMIC_NR_DEV0_F0_vf1_MASK                                            0x00000008L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__DMA_ON_BME_LOW_DEV0_F0_vf1_MASK                                          0x00000010L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf1_MASK                                  0x00010000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf1_MASK                               0x00020000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf1_MASK                                  0x00040000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf1_MASK                                      0x00080000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf1__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf1_MASK                                    0x00100000L
+//BIF_AER_ERR_LOG_DEV0_F0_vf2
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__UR_ATOMIC_OPCODE_DEV0_F0_vf2__SHIFT                                      0x0
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf2__SHIFT                                   0x1
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__UR_ATOMIC_LENGTH_DEV0_F0_vf2__SHIFT                                      0x2
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__UR_ATOMIC_NR_DEV0_F0_vf2__SHIFT                                          0x3
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__DMA_ON_BME_LOW_DEV0_F0_vf2__SHIFT                                        0x4
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf2__SHIFT                                0x10
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf2__SHIFT                             0x11
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf2__SHIFT                                0x12
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf2__SHIFT                                    0x13
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf2__SHIFT                                  0x14
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__UR_ATOMIC_OPCODE_DEV0_F0_vf2_MASK                                        0x00000001L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf2_MASK                                     0x00000002L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__UR_ATOMIC_LENGTH_DEV0_F0_vf2_MASK                                        0x00000004L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__UR_ATOMIC_NR_DEV0_F0_vf2_MASK                                            0x00000008L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__DMA_ON_BME_LOW_DEV0_F0_vf2_MASK                                          0x00000010L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf2_MASK                                  0x00010000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf2_MASK                               0x00020000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf2_MASK                                  0x00040000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf2_MASK                                      0x00080000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf2__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf2_MASK                                    0x00100000L
+//BIF_AER_ERR_LOG_DEV0_F0_vf3
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__UR_ATOMIC_OPCODE_DEV0_F0_vf3__SHIFT                                      0x0
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf3__SHIFT                                   0x1
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__UR_ATOMIC_LENGTH_DEV0_F0_vf3__SHIFT                                      0x2
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__UR_ATOMIC_NR_DEV0_F0_vf3__SHIFT                                          0x3
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__DMA_ON_BME_LOW_DEV0_F0_vf3__SHIFT                                        0x4
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf3__SHIFT                                0x10
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf3__SHIFT                             0x11
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf3__SHIFT                                0x12
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf3__SHIFT                                    0x13
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf3__SHIFT                                  0x14
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__UR_ATOMIC_OPCODE_DEV0_F0_vf3_MASK                                        0x00000001L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf3_MASK                                     0x00000002L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__UR_ATOMIC_LENGTH_DEV0_F0_vf3_MASK                                        0x00000004L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__UR_ATOMIC_NR_DEV0_F0_vf3_MASK                                            0x00000008L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__DMA_ON_BME_LOW_DEV0_F0_vf3_MASK                                          0x00000010L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf3_MASK                                  0x00010000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf3_MASK                               0x00020000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf3_MASK                                  0x00040000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf3_MASK                                      0x00080000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf3__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf3_MASK                                    0x00100000L
+//BIF_AER_ERR_LOG_DEV0_F0_vf4
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__UR_ATOMIC_OPCODE_DEV0_F0_vf4__SHIFT                                      0x0
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf4__SHIFT                                   0x1
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__UR_ATOMIC_LENGTH_DEV0_F0_vf4__SHIFT                                      0x2
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__UR_ATOMIC_NR_DEV0_F0_vf4__SHIFT                                          0x3
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__DMA_ON_BME_LOW_DEV0_F0_vf4__SHIFT                                        0x4
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf4__SHIFT                                0x10
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf4__SHIFT                             0x11
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf4__SHIFT                                0x12
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf4__SHIFT                                    0x13
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf4__SHIFT                                  0x14
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__UR_ATOMIC_OPCODE_DEV0_F0_vf4_MASK                                        0x00000001L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf4_MASK                                     0x00000002L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__UR_ATOMIC_LENGTH_DEV0_F0_vf4_MASK                                        0x00000004L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__UR_ATOMIC_NR_DEV0_F0_vf4_MASK                                            0x00000008L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__DMA_ON_BME_LOW_DEV0_F0_vf4_MASK                                          0x00000010L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf4_MASK                                  0x00010000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf4_MASK                               0x00020000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf4_MASK                                  0x00040000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf4_MASK                                      0x00080000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf4__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf4_MASK                                    0x00100000L
+//BIF_AER_ERR_LOG_DEV0_F0_vf5
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__UR_ATOMIC_OPCODE_DEV0_F0_vf5__SHIFT                                      0x0
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf5__SHIFT                                   0x1
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__UR_ATOMIC_LENGTH_DEV0_F0_vf5__SHIFT                                      0x2
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__UR_ATOMIC_NR_DEV0_F0_vf5__SHIFT                                          0x3
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__DMA_ON_BME_LOW_DEV0_F0_vf5__SHIFT                                        0x4
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf5__SHIFT                                0x10
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf5__SHIFT                             0x11
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf5__SHIFT                                0x12
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf5__SHIFT                                    0x13
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf5__SHIFT                                  0x14
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__UR_ATOMIC_OPCODE_DEV0_F0_vf5_MASK                                        0x00000001L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf5_MASK                                     0x00000002L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__UR_ATOMIC_LENGTH_DEV0_F0_vf5_MASK                                        0x00000004L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__UR_ATOMIC_NR_DEV0_F0_vf5_MASK                                            0x00000008L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__DMA_ON_BME_LOW_DEV0_F0_vf5_MASK                                          0x00000010L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf5_MASK                                  0x00010000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf5_MASK                               0x00020000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf5_MASK                                  0x00040000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf5_MASK                                      0x00080000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf5__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf5_MASK                                    0x00100000L
+//BIF_AER_ERR_LOG_DEV0_F0_vf6
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__UR_ATOMIC_OPCODE_DEV0_F0_vf6__SHIFT                                      0x0
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf6__SHIFT                                   0x1
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__UR_ATOMIC_LENGTH_DEV0_F0_vf6__SHIFT                                      0x2
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__UR_ATOMIC_NR_DEV0_F0_vf6__SHIFT                                          0x3
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__DMA_ON_BME_LOW_DEV0_F0_vf6__SHIFT                                        0x4
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf6__SHIFT                                0x10
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf6__SHIFT                             0x11
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf6__SHIFT                                0x12
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf6__SHIFT                                    0x13
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf6__SHIFT                                  0x14
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__UR_ATOMIC_OPCODE_DEV0_F0_vf6_MASK                                        0x00000001L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf6_MASK                                     0x00000002L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__UR_ATOMIC_LENGTH_DEV0_F0_vf6_MASK                                        0x00000004L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__UR_ATOMIC_NR_DEV0_F0_vf6_MASK                                            0x00000008L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__DMA_ON_BME_LOW_DEV0_F0_vf6_MASK                                          0x00000010L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf6_MASK                                  0x00010000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf6_MASK                               0x00020000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf6_MASK                                  0x00040000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf6_MASK                                      0x00080000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf6__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf6_MASK                                    0x00100000L
+//BIF_AER_ERR_LOG_DEV0_F0_vf7
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__UR_ATOMIC_OPCODE_DEV0_F0_vf7__SHIFT                                      0x0
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf7__SHIFT                                   0x1
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__UR_ATOMIC_LENGTH_DEV0_F0_vf7__SHIFT                                      0x2
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__UR_ATOMIC_NR_DEV0_F0_vf7__SHIFT                                          0x3
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__DMA_ON_BME_LOW_DEV0_F0_vf7__SHIFT                                        0x4
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf7__SHIFT                                0x10
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf7__SHIFT                             0x11
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf7__SHIFT                                0x12
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf7__SHIFT                                    0x13
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf7__SHIFT                                  0x14
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__UR_ATOMIC_OPCODE_DEV0_F0_vf7_MASK                                        0x00000001L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__UR_ATOMIC_REQEN_LOW_DEV0_F0_vf7_MASK                                     0x00000002L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__UR_ATOMIC_LENGTH_DEV0_F0_vf7_MASK                                        0x00000004L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__UR_ATOMIC_NR_DEV0_F0_vf7_MASK                                            0x00000008L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__DMA_ON_BME_LOW_DEV0_F0_vf7_MASK                                          0x00000010L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_UR_ATOMIC_OPCODE_DEV0_F0_vf7_MASK                                  0x00010000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_UR_ATOMIC_REQEN_LOW_DEV0_F0_vf7_MASK                               0x00020000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_UR_ATOMIC_LENGTH_DEV0_F0_vf7_MASK                                  0x00040000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_UR_ATOMIC_NR_DEV0_F0_vf7_MASK                                      0x00080000L
+#define BIF_AER_ERR_LOG_DEV0_F0_vf7__CLEAR_DMA_ON_BME_LOW_DEV0_F0_vf7_MASK                                    0x00100000L
+
+
+// addressBlock: nbif0_hphost0_hphost_private_hphost_private_regblk
+//NBIF_AON_SVA_DISABLE_SAD
+#define NBIF_AON_SVA_DISABLE_SAD__NBIF_AON_SVA_DISABLE__SHIFT                                                 0x0
+#define NBIF_AON_SVA_DISABLE_SAD__NBIF_AON_SVA_DISABLE_MASK                                                   0x00000001L
+
+
+// addressBlock: nbif0_nbif0_gdc_dma_sion_SIONDEC
+//GDC_DMA_SION_CL0_RdRsp_BurstTarget_REG0
+#define GDC_DMA_SION_CL0_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_DMA_SION_CL0_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_DMA_SION_CL0_RdRsp_BurstTarget_REG1
+#define GDC_DMA_SION_CL0_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_DMA_SION_CL0_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_DMA_SION_CL0_RdRsp_TimeSlot_REG0
+#define GDC_DMA_SION_CL0_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_DMA_SION_CL0_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_DMA_SION_CL0_RdRsp_TimeSlot_REG1
+#define GDC_DMA_SION_CL0_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_DMA_SION_CL0_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_DMA_SION_CL0_WrRsp_BurstTarget_REG0
+#define GDC_DMA_SION_CL0_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_DMA_SION_CL0_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_DMA_SION_CL0_WrRsp_BurstTarget_REG1
+#define GDC_DMA_SION_CL0_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_DMA_SION_CL0_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_DMA_SION_CL0_WrRsp_TimeSlot_REG0
+#define GDC_DMA_SION_CL0_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_DMA_SION_CL0_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_DMA_SION_CL0_WrRsp_TimeSlot_REG1
+#define GDC_DMA_SION_CL0_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_DMA_SION_CL0_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_DMA_SION_CL0_Req_BurstTarget_REG0
+#define GDC_DMA_SION_CL0_Req_BurstTarget_REG0__Req_BurstTarget_31_0__SHIFT                                    0x0
+#define GDC_DMA_SION_CL0_Req_BurstTarget_REG0__Req_BurstTarget_31_0_MASK                                      0xFFFFFFFFL
+//GDC_DMA_SION_CL0_Req_BurstTarget_REG1
+#define GDC_DMA_SION_CL0_Req_BurstTarget_REG1__Req_BurstTarget_63_32__SHIFT                                   0x0
+#define GDC_DMA_SION_CL0_Req_BurstTarget_REG1__Req_BurstTarget_63_32_MASK                                     0xFFFFFFFFL
+//GDC_DMA_SION_CL0_Req_TimeSlot_REG0
+#define GDC_DMA_SION_CL0_Req_TimeSlot_REG0__Req_TimeSlot_31_0__SHIFT                                          0x0
+#define GDC_DMA_SION_CL0_Req_TimeSlot_REG0__Req_TimeSlot_31_0_MASK                                            0xFFFFFFFFL
+//GDC_DMA_SION_CL0_Req_TimeSlot_REG1
+#define GDC_DMA_SION_CL0_Req_TimeSlot_REG1__Req_TimeSlot_63_32__SHIFT                                         0x0
+#define GDC_DMA_SION_CL0_Req_TimeSlot_REG1__Req_TimeSlot_63_32_MASK                                           0xFFFFFFFFL
+//GDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0__SHIFT                            0x0
+#define GDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0_MASK                              0xFFFFFFFFL
+//GDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32__SHIFT                           0x0
+#define GDC_DMA_SION_CL0_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32_MASK                             0xFFFFFFFFL
+//GDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0__SHIFT                          0x0
+#define GDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0_MASK                            0xFFFFFFFFL
+//GDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32__SHIFT                         0x0
+#define GDC_DMA_SION_CL0_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32_MASK                           0xFFFFFFFFL
+//GDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_DMA_SION_CL0_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_DMA_SION_CL0_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_DMA_SION_CL1_RdRsp_BurstTarget_REG0
+#define GDC_DMA_SION_CL1_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_DMA_SION_CL1_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_DMA_SION_CL1_RdRsp_BurstTarget_REG1
+#define GDC_DMA_SION_CL1_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_DMA_SION_CL1_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_DMA_SION_CL1_RdRsp_TimeSlot_REG0
+#define GDC_DMA_SION_CL1_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_DMA_SION_CL1_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_DMA_SION_CL1_RdRsp_TimeSlot_REG1
+#define GDC_DMA_SION_CL1_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_DMA_SION_CL1_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_DMA_SION_CL1_WrRsp_BurstTarget_REG0
+#define GDC_DMA_SION_CL1_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_DMA_SION_CL1_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_DMA_SION_CL1_WrRsp_BurstTarget_REG1
+#define GDC_DMA_SION_CL1_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_DMA_SION_CL1_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_DMA_SION_CL1_WrRsp_TimeSlot_REG0
+#define GDC_DMA_SION_CL1_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_DMA_SION_CL1_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_DMA_SION_CL1_WrRsp_TimeSlot_REG1
+#define GDC_DMA_SION_CL1_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_DMA_SION_CL1_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_DMA_SION_CL1_Req_BurstTarget_REG0
+#define GDC_DMA_SION_CL1_Req_BurstTarget_REG0__Req_BurstTarget_31_0__SHIFT                                    0x0
+#define GDC_DMA_SION_CL1_Req_BurstTarget_REG0__Req_BurstTarget_31_0_MASK                                      0xFFFFFFFFL
+//GDC_DMA_SION_CL1_Req_BurstTarget_REG1
+#define GDC_DMA_SION_CL1_Req_BurstTarget_REG1__Req_BurstTarget_63_32__SHIFT                                   0x0
+#define GDC_DMA_SION_CL1_Req_BurstTarget_REG1__Req_BurstTarget_63_32_MASK                                     0xFFFFFFFFL
+//GDC_DMA_SION_CL1_Req_TimeSlot_REG0
+#define GDC_DMA_SION_CL1_Req_TimeSlot_REG0__Req_TimeSlot_31_0__SHIFT                                          0x0
+#define GDC_DMA_SION_CL1_Req_TimeSlot_REG0__Req_TimeSlot_31_0_MASK                                            0xFFFFFFFFL
+//GDC_DMA_SION_CL1_Req_TimeSlot_REG1
+#define GDC_DMA_SION_CL1_Req_TimeSlot_REG1__Req_TimeSlot_63_32__SHIFT                                         0x0
+#define GDC_DMA_SION_CL1_Req_TimeSlot_REG1__Req_TimeSlot_63_32_MASK                                           0xFFFFFFFFL
+//GDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0__SHIFT                            0x0
+#define GDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0_MASK                              0xFFFFFFFFL
+//GDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32__SHIFT                           0x0
+#define GDC_DMA_SION_CL1_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32_MASK                             0xFFFFFFFFL
+//GDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0__SHIFT                          0x0
+#define GDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0_MASK                            0xFFFFFFFFL
+//GDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32__SHIFT                         0x0
+#define GDC_DMA_SION_CL1_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32_MASK                           0xFFFFFFFFL
+//GDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_DMA_SION_CL1_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_DMA_SION_CL1_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_DMA_SION_CL2_RdRsp_BurstTarget_REG0
+#define GDC_DMA_SION_CL2_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_DMA_SION_CL2_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_DMA_SION_CL2_RdRsp_BurstTarget_REG1
+#define GDC_DMA_SION_CL2_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_DMA_SION_CL2_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_DMA_SION_CL2_RdRsp_TimeSlot_REG0
+#define GDC_DMA_SION_CL2_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_DMA_SION_CL2_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_DMA_SION_CL2_RdRsp_TimeSlot_REG1
+#define GDC_DMA_SION_CL2_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_DMA_SION_CL2_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_DMA_SION_CL2_WrRsp_BurstTarget_REG0
+#define GDC_DMA_SION_CL2_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_DMA_SION_CL2_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_DMA_SION_CL2_WrRsp_BurstTarget_REG1
+#define GDC_DMA_SION_CL2_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_DMA_SION_CL2_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_DMA_SION_CL2_WrRsp_TimeSlot_REG0
+#define GDC_DMA_SION_CL2_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_DMA_SION_CL2_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_DMA_SION_CL2_WrRsp_TimeSlot_REG1
+#define GDC_DMA_SION_CL2_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_DMA_SION_CL2_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_DMA_SION_CL2_Req_BurstTarget_REG0
+#define GDC_DMA_SION_CL2_Req_BurstTarget_REG0__Req_BurstTarget_31_0__SHIFT                                    0x0
+#define GDC_DMA_SION_CL2_Req_BurstTarget_REG0__Req_BurstTarget_31_0_MASK                                      0xFFFFFFFFL
+//GDC_DMA_SION_CL2_Req_BurstTarget_REG1
+#define GDC_DMA_SION_CL2_Req_BurstTarget_REG1__Req_BurstTarget_63_32__SHIFT                                   0x0
+#define GDC_DMA_SION_CL2_Req_BurstTarget_REG1__Req_BurstTarget_63_32_MASK                                     0xFFFFFFFFL
+//GDC_DMA_SION_CL2_Req_TimeSlot_REG0
+#define GDC_DMA_SION_CL2_Req_TimeSlot_REG0__Req_TimeSlot_31_0__SHIFT                                          0x0
+#define GDC_DMA_SION_CL2_Req_TimeSlot_REG0__Req_TimeSlot_31_0_MASK                                            0xFFFFFFFFL
+//GDC_DMA_SION_CL2_Req_TimeSlot_REG1
+#define GDC_DMA_SION_CL2_Req_TimeSlot_REG1__Req_TimeSlot_63_32__SHIFT                                         0x0
+#define GDC_DMA_SION_CL2_Req_TimeSlot_REG1__Req_TimeSlot_63_32_MASK                                           0xFFFFFFFFL
+//GDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0__SHIFT                            0x0
+#define GDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0_MASK                              0xFFFFFFFFL
+//GDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32__SHIFT                           0x0
+#define GDC_DMA_SION_CL2_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32_MASK                             0xFFFFFFFFL
+//GDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0__SHIFT                          0x0
+#define GDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0_MASK                            0xFFFFFFFFL
+//GDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32__SHIFT                         0x0
+#define GDC_DMA_SION_CL2_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32_MASK                           0xFFFFFFFFL
+//GDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_DMA_SION_CL2_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG0
+#define GDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG1
+#define GDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_DMA_SION_CL2_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_DMA_SION_CNTL_REG0
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK0__SHIFT                    0x0
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK1__SHIFT                    0x1
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK2__SHIFT                    0x2
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK3__SHIFT                    0x3
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK4__SHIFT                    0x4
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK5__SHIFT                    0x5
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK6__SHIFT                    0x6
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK7__SHIFT                    0x7
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK8__SHIFT                    0x8
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK9__SHIFT                    0x9
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK0__SHIFT                    0xa
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK1__SHIFT                    0xb
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK2__SHIFT                    0xc
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK3__SHIFT                    0xd
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK4__SHIFT                    0xe
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK5__SHIFT                    0xf
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK6__SHIFT                    0x10
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK7__SHIFT                    0x11
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK8__SHIFT                    0x12
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK9__SHIFT                    0x13
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK0_MASK                      0x00000001L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK1_MASK                      0x00000002L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK2_MASK                      0x00000004L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK3_MASK                      0x00000008L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK4_MASK                      0x00000010L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK5_MASK                      0x00000020L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK6_MASK                      0x00000040L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK7_MASK                      0x00000080L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK8_MASK                      0x00000100L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK9_MASK                      0x00000200L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK0_MASK                      0x00000400L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK1_MASK                      0x00000800L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK2_MASK                      0x00001000L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK3_MASK                      0x00002000L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK4_MASK                      0x00004000L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK5_MASK                      0x00008000L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK6_MASK                      0x00010000L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK7_MASK                      0x00020000L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK8_MASK                      0x00040000L
+#define GDC_DMA_SION_CNTL_REG0__GDC_DMA_SION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK9_MASK                      0x00080000L
+//GDC_DMA_SION_CNTL_REG1
+#define GDC_DMA_SION_CNTL_REG1__GDC_DMA_SION_LIVELOCK_WATCHDOG_THRESHOLD__SHIFT                               0x0
+#define GDC_DMA_SION_CNTL_REG1__GDC_DMA_SION_CG_OFF_HYSTERESIS__SHIFT                                         0x8
+#define GDC_DMA_SION_CNTL_REG1__GDC_DMA_SION_LIVELOCK_WATCHDOG_THRESHOLD_MASK                                 0x000000FFL
+#define GDC_DMA_SION_CNTL_REG1__GDC_DMA_SION_CG_OFF_HYSTERESIS_MASK                                           0x0000FF00L
+
+
+// addressBlock: nbif0_nbif0_gdc_hst_sion_SIONDEC
+//GDC_HST_SION_CL0_RdRsp_BurstTarget_REG0
+#define GDC_HST_SION_CL0_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_HST_SION_CL0_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_HST_SION_CL0_RdRsp_BurstTarget_REG1
+#define GDC_HST_SION_CL0_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_HST_SION_CL0_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_HST_SION_CL0_RdRsp_TimeSlot_REG0
+#define GDC_HST_SION_CL0_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_HST_SION_CL0_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_HST_SION_CL0_RdRsp_TimeSlot_REG1
+#define GDC_HST_SION_CL0_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_HST_SION_CL0_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_HST_SION_CL0_WrRsp_BurstTarget_REG0
+#define GDC_HST_SION_CL0_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_HST_SION_CL0_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_HST_SION_CL0_WrRsp_BurstTarget_REG1
+#define GDC_HST_SION_CL0_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_HST_SION_CL0_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_HST_SION_CL0_WrRsp_TimeSlot_REG0
+#define GDC_HST_SION_CL0_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_HST_SION_CL0_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_HST_SION_CL0_WrRsp_TimeSlot_REG1
+#define GDC_HST_SION_CL0_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_HST_SION_CL0_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_HST_SION_CL0_Req_BurstTarget_REG0
+#define GDC_HST_SION_CL0_Req_BurstTarget_REG0__Req_BurstTarget_31_0__SHIFT                                    0x0
+#define GDC_HST_SION_CL0_Req_BurstTarget_REG0__Req_BurstTarget_31_0_MASK                                      0xFFFFFFFFL
+//GDC_HST_SION_CL0_Req_BurstTarget_REG1
+#define GDC_HST_SION_CL0_Req_BurstTarget_REG1__Req_BurstTarget_63_32__SHIFT                                   0x0
+#define GDC_HST_SION_CL0_Req_BurstTarget_REG1__Req_BurstTarget_63_32_MASK                                     0xFFFFFFFFL
+//GDC_HST_SION_CL0_Req_TimeSlot_REG0
+#define GDC_HST_SION_CL0_Req_TimeSlot_REG0__Req_TimeSlot_31_0__SHIFT                                          0x0
+#define GDC_HST_SION_CL0_Req_TimeSlot_REG0__Req_TimeSlot_31_0_MASK                                            0xFFFFFFFFL
+//GDC_HST_SION_CL0_Req_TimeSlot_REG1
+#define GDC_HST_SION_CL0_Req_TimeSlot_REG1__Req_TimeSlot_63_32__SHIFT                                         0x0
+#define GDC_HST_SION_CL0_Req_TimeSlot_REG1__Req_TimeSlot_63_32_MASK                                           0xFFFFFFFFL
+//GDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG0
+#define GDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0__SHIFT                            0x0
+#define GDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0_MASK                              0xFFFFFFFFL
+//GDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG1
+#define GDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32__SHIFT                           0x0
+#define GDC_HST_SION_CL0_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32_MASK                             0xFFFFFFFFL
+//GDC_HST_SION_CL0_DataPoolCredit_Alloc_REG0
+#define GDC_HST_SION_CL0_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0__SHIFT                          0x0
+#define GDC_HST_SION_CL0_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0_MASK                            0xFFFFFFFFL
+//GDC_HST_SION_CL0_DataPoolCredit_Alloc_REG1
+#define GDC_HST_SION_CL0_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32__SHIFT                         0x0
+#define GDC_HST_SION_CL0_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32_MASK                           0xFFFFFFFFL
+//GDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG0
+#define GDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG1
+#define GDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_HST_SION_CL0_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG0
+#define GDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG1
+#define GDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_HST_SION_CL0_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_HST_SION_CL1_RdRsp_BurstTarget_REG0
+#define GDC_HST_SION_CL1_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_HST_SION_CL1_RdRsp_BurstTarget_REG0__RdRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_HST_SION_CL1_RdRsp_BurstTarget_REG1
+#define GDC_HST_SION_CL1_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_HST_SION_CL1_RdRsp_BurstTarget_REG1__RdRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_HST_SION_CL1_RdRsp_TimeSlot_REG0
+#define GDC_HST_SION_CL1_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_HST_SION_CL1_RdRsp_TimeSlot_REG0__RdRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_HST_SION_CL1_RdRsp_TimeSlot_REG1
+#define GDC_HST_SION_CL1_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_HST_SION_CL1_RdRsp_TimeSlot_REG1__RdRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_HST_SION_CL1_WrRsp_BurstTarget_REG0
+#define GDC_HST_SION_CL1_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0__SHIFT                                0x0
+#define GDC_HST_SION_CL1_WrRsp_BurstTarget_REG0__WrRsp_BurstTarget_31_0_MASK                                  0xFFFFFFFFL
+//GDC_HST_SION_CL1_WrRsp_BurstTarget_REG1
+#define GDC_HST_SION_CL1_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32__SHIFT                               0x0
+#define GDC_HST_SION_CL1_WrRsp_BurstTarget_REG1__WrRsp_BurstTarget_63_32_MASK                                 0xFFFFFFFFL
+//GDC_HST_SION_CL1_WrRsp_TimeSlot_REG0
+#define GDC_HST_SION_CL1_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0__SHIFT                                      0x0
+#define GDC_HST_SION_CL1_WrRsp_TimeSlot_REG0__WrRsp_TimeSlot_31_0_MASK                                        0xFFFFFFFFL
+//GDC_HST_SION_CL1_WrRsp_TimeSlot_REG1
+#define GDC_HST_SION_CL1_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32__SHIFT                                     0x0
+#define GDC_HST_SION_CL1_WrRsp_TimeSlot_REG1__WrRsp_TimeSlot_63_32_MASK                                       0xFFFFFFFFL
+//GDC_HST_SION_CL1_Req_BurstTarget_REG0
+#define GDC_HST_SION_CL1_Req_BurstTarget_REG0__Req_BurstTarget_31_0__SHIFT                                    0x0
+#define GDC_HST_SION_CL1_Req_BurstTarget_REG0__Req_BurstTarget_31_0_MASK                                      0xFFFFFFFFL
+//GDC_HST_SION_CL1_Req_BurstTarget_REG1
+#define GDC_HST_SION_CL1_Req_BurstTarget_REG1__Req_BurstTarget_63_32__SHIFT                                   0x0
+#define GDC_HST_SION_CL1_Req_BurstTarget_REG1__Req_BurstTarget_63_32_MASK                                     0xFFFFFFFFL
+//GDC_HST_SION_CL1_Req_TimeSlot_REG0
+#define GDC_HST_SION_CL1_Req_TimeSlot_REG0__Req_TimeSlot_31_0__SHIFT                                          0x0
+#define GDC_HST_SION_CL1_Req_TimeSlot_REG0__Req_TimeSlot_31_0_MASK                                            0xFFFFFFFFL
+//GDC_HST_SION_CL1_Req_TimeSlot_REG1
+#define GDC_HST_SION_CL1_Req_TimeSlot_REG1__Req_TimeSlot_63_32__SHIFT                                         0x0
+#define GDC_HST_SION_CL1_Req_TimeSlot_REG1__Req_TimeSlot_63_32_MASK                                           0xFFFFFFFFL
+//GDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG0
+#define GDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0__SHIFT                            0x0
+#define GDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG0__ReqPoolCredit_Alloc_31_0_MASK                              0xFFFFFFFFL
+//GDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG1
+#define GDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32__SHIFT                           0x0
+#define GDC_HST_SION_CL1_ReqPoolCredit_Alloc_REG1__ReqPoolCredit_Alloc_63_32_MASK                             0xFFFFFFFFL
+//GDC_HST_SION_CL1_DataPoolCredit_Alloc_REG0
+#define GDC_HST_SION_CL1_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0__SHIFT                          0x0
+#define GDC_HST_SION_CL1_DataPoolCredit_Alloc_REG0__DataPoolCredit_Alloc_31_0_MASK                            0xFFFFFFFFL
+//GDC_HST_SION_CL1_DataPoolCredit_Alloc_REG1
+#define GDC_HST_SION_CL1_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32__SHIFT                         0x0
+#define GDC_HST_SION_CL1_DataPoolCredit_Alloc_REG1__DataPoolCredit_Alloc_63_32_MASK                           0xFFFFFFFFL
+//GDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG0
+#define GDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG0__RdRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG1
+#define GDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_HST_SION_CL1_RdRspPoolCredit_Alloc_REG1__RdRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG0
+#define GDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0__SHIFT                        0x0
+#define GDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG0__WrRspPoolCredit_Alloc_31_0_MASK                          0xFFFFFFFFL
+//GDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG1
+#define GDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32__SHIFT                       0x0
+#define GDC_HST_SION_CL1_WrRspPoolCredit_Alloc_REG1__WrRspPoolCredit_Alloc_63_32_MASK                         0xFFFFFFFFL
+//GDC_HST_SION_CNTL_REG0
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK0__SHIFT                     0x0
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK1__SHIFT                     0x1
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK2__SHIFT                     0x2
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK3__SHIFT                     0x3
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK4__SHIFT                     0x4
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK5__SHIFT                     0x5
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK6__SHIFT                     0x6
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK7__SHIFT                     0x7
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK8__SHIFT                     0x8
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK9__SHIFT                     0x9
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK0__SHIFT                     0xa
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK1__SHIFT                     0xb
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK2__SHIFT                     0xc
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK3__SHIFT                     0xd
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK4__SHIFT                     0xe
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK5__SHIFT                     0xf
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK6__SHIFT                     0x10
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK7__SHIFT                     0x11
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK8__SHIFT                     0x12
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK9__SHIFT                     0x13
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK0_MASK                       0x00000001L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK1_MASK                       0x00000002L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK2_MASK                       0x00000004L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK3_MASK                       0x00000008L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK4_MASK                       0x00000010L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK5_MASK                       0x00000020L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK6_MASK                       0x00000040L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK7_MASK                       0x00000080L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK8_MASK                       0x00000100L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_0_SOFT_OVERRIDE_CLK9_MASK                       0x00000200L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK0_MASK                       0x00000400L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK1_MASK                       0x00000800L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK2_MASK                       0x00001000L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK3_MASK                       0x00002000L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK4_MASK                       0x00004000L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK5_MASK                       0x00008000L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK6_MASK                       0x00010000L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK7_MASK                       0x00020000L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK8_MASK                       0x00040000L
+#define GDC_HST_SION_CNTL_REG0__GDC_HSTSION_GLUE_CG_LCLK_CTRL_1_SOFT_OVERRIDE_CLK9_MASK                       0x00080000L
+//GDC_HST_SION_CNTL_REG1
+#define GDC_HST_SION_CNTL_REG1__GDC_HSTSION_LIVELOCK_WATCHDOG_THRESHOLD__SHIFT                                0x0
+#define GDC_HST_SION_CNTL_REG1__GDC_HSTSION_CG_OFF_HYSTERESIS__SHIFT                                          0x8
+#define GDC_HST_SION_CNTL_REG1__GDC_HSTSION_LIVE_ACTIVE__SHIFT                                                0x10
+#define GDC_HST_SION_CNTL_REG1__GDC_HSTSION_LIVELOCK_WATCHDOG_THRESHOLD_MASK                                  0x000000FFL
+#define GDC_HST_SION_CNTL_REG1__GDC_HSTSION_CG_OFF_HYSTERESIS_MASK                                            0x0000FF00L
+#define GDC_HST_SION_CNTL_REG1__GDC_HSTSION_LIVE_ACTIVE_MASK                                                  0xFFFF0000L
+
+
+// addressBlock: nbif0_nbif0_gdc_GDCDEC
+//GDC1_NGDC_SDP_PORT_CTRL
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS__SHIFT                                                 0x0
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H__SHIFT                                               0x10
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_DIS__SHIFT                                                        0x1f
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_MASK                                                   0x000000FFL
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_HYSTERESIS_H_MASK                                                 0x000F0000L
+#define GDC1_NGDC_SDP_PORT_CTRL__SDP_DISCON_DIS_MASK                                                          0x80000000L
+//GDC1_SHUB_REGS_IF_CTL
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS__SHIFT                                0x0
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS__SHIFT                                             0x1
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_DROP_NONPF_MMREGREQ_SETERR_DIS_MASK                                  0x00000001L
+#define GDC1_SHUB_REGS_IF_CTL__SHUB_REGS_VF_PROTECTION_DIS_MASK                                               0x00000002L
+//GDC1_NGDC_MGCG_CTRL
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_EN__SHIFT                                                              0x0
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_MODE__SHIFT                                                            0x1
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS__SHIFT                                                      0x2
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS__SHIFT                                                         0xa
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS__SHIFT                                                         0xb
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS__SHIFT                                                         0xc
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS__SHIFT                                                         0xd
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DBG_DIS__SHIFT                                                         0xe
+#define GDC1_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN__SHIFT                                                         0xf
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_EN_MASK                                                                0x00000001L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_MODE_MASK                                                              0x00000002L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HYSTERESIS_MASK                                                        0x000003FCL
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_HST_DIS_MASK                                                           0x00000400L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DMA_DIS_MASK                                                           0x00000800L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_REG_DIS_MASK                                                           0x00001000L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_AER_DIS_MASK                                                           0x00002000L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_MGCG_DBG_DIS_MASK                                                           0x00004000L
+#define GDC1_NGDC_MGCG_CTRL__NGDC_SRAM_FGCG_EN_MASK                                                           0x00008000L
+//GDC1_S2A_MISC_CNTL
+#define GDC1_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS__SHIFT                                                         0x3
+#define GDC1_S2A_MISC_CNTL__ATM_ARB_MODE__SHIFT                                                               0x8
+#define GDC1_S2A_MISC_CNTL__RB_ARB_MODE__SHIFT                                                                0xa
+#define GDC1_S2A_MISC_CNTL__HSTR_ARB_MODE__SHIFT                                                              0xc
+#define GDC1_S2A_MISC_CNTL__HDP_PERF_ENH_DIS__SHIFT                                                           0xf
+#define GDC1_S2A_MISC_CNTL__WRSP_ARB_MODE__SHIFT                                                              0x10
+#define GDC1_S2A_MISC_CNTL__AXI_HST_CPL_EP_DIS_MASK                                                           0x00000008L
+#define GDC1_S2A_MISC_CNTL__ATM_ARB_MODE_MASK                                                                 0x00000300L
+#define GDC1_S2A_MISC_CNTL__RB_ARB_MODE_MASK                                                                  0x00000C00L
+#define GDC1_S2A_MISC_CNTL__HSTR_ARB_MODE_MASK                                                                0x00003000L
+#define GDC1_S2A_MISC_CNTL__HDP_PERF_ENH_DIS_MASK                                                             0x00008000L
+#define GDC1_S2A_MISC_CNTL__WRSP_ARB_MODE_MASK                                                                0x000F0000L
+//GDC1_NGDC_MCA_SMN_CTRL0
+#define GDC1_NGDC_MCA_SMN_CTRL0__MCA_SMN_POSTED__SHIFT                                                        0x0
+#define GDC1_NGDC_MCA_SMN_CTRL0__MCA_SMN_POSTED_MASK                                                          0x00000001L
+//GDC1_NGDC_EARLY_WAKEUP_CTRL
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_ACTIVE__SHIFT                                0x0
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_DS_EXIT__SHIFT                               0x1
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_ALLOW_AER_ACTIVE__SHIFT                                0x2
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_ACTIVE_MASK                                  0x00000001L
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_BY_CLIENT_DS_EXIT_MASK                                 0x00000002L
+#define GDC1_NGDC_EARLY_WAKEUP_CTRL__NGDC_EARLY_WAKEUP_ALLOW_AER_ACTIVE_MASK                                  0x00000004L
+//GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_BYPASS__SHIFT                                  0x0
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_ACCUM_SEL__SHIFT                               0x1
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_FORCE_EN__SHIFT                                0x2
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_FORCE_DATA__SHIFT                              0x3
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_STATUS_ACCUM_EN__SHIFT                         0x4
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_DATASTATUS_ACCUM_EN__SHIFT                     0x5
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_BYPASS_MASK                                    0x00000001L
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_ACCUM_SEL_MASK                                 0x00000002L
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_FORCE_EN_MASK                                  0x00000004L
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_FORCE_DATA_MASK                                0x00000008L
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_STATUS_ACCUM_EN_MASK                           0x00000010L
+#define GDC1_GDC_GDCCLK_S2A_ERR_RSP_CNTL__GDC_GDCCLK_S2A_RDRSP_DATASTATUS_ACCUM_EN_MASK                       0x00000020L
+//GDC1_GDC_DMA_SION_PCTRL
+#define GDC1_GDC_DMA_SION_PCTRL__BIF_SDP_DISCON_DIS__SHIFT                                                    0x0
+#define GDC1_GDC_DMA_SION_PCTRL__SYSHUB_SDP_DISCON_DIS__SHIFT                                                 0x3
+#define GDC1_GDC_DMA_SION_PCTRL__INT_SDP_DISCON_DIS__SHIFT                                                    0x6
+#define GDC1_GDC_DMA_SION_PCTRL__BIF_SDP_DISCON_DIS_MASK                                                      0x00000001L
+#define GDC1_GDC_DMA_SION_PCTRL__SYSHUB_SDP_DISCON_DIS_MASK                                                   0x00000008L
+#define GDC1_GDC_DMA_SION_PCTRL__INT_SDP_DISCON_DIS_MASK                                                      0x00000040L
+//GDC1_NGDC_MP4SDP_CNTL
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC0_OVERRIDE_EN__SHIFT                                           0x0
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC0_OVERRIDE_VAL__SHIFT                                          0x1
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC1_OVERRIDE_EN__SHIFT                                           0x4
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC1_OVERRIDE_VAL__SHIFT                                          0x5
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC4_OVERRIDE_EN__SHIFT                                           0x8
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC4_OVERRIDE_VAL__SHIFT                                          0x9
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC0_OVERRIDE_EN_MASK                                             0x00000001L
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC0_OVERRIDE_VAL_MASK                                            0x00000006L
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC1_OVERRIDE_EN_MASK                                             0x00000010L
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC1_OVERRIDE_VAL_MASK                                            0x00000060L
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC4_OVERRIDE_EN_MASK                                             0x00000100L
+#define GDC1_NGDC_MP4SDP_CNTL__MP4SDP_BLKLVL_VC4_OVERRIDE_VAL_MASK                                            0x00000600L
+//GDC1_DOORBELL_VCN_TARGET_VF0
+#define GDC1_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN0_TARGET_VF0__SHIFT                                         0x0
+#define GDC1_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN1_TARGET_VF0__SHIFT                                         0x4
+#define GDC1_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN2_TARGET_VF0__SHIFT                                         0x8
+#define GDC1_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN3_TARGET_VF0__SHIFT                                         0xc
+#define GDC1_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN0_TARGET_VF0_MASK                                           0x000FL
+#define GDC1_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN1_TARGET_VF0_MASK                                           0x00F0L
+#define GDC1_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN2_TARGET_VF0_MASK                                           0x0F00L
+#define GDC1_DOORBELL_VCN_TARGET_VF0__DOORBELL_VCN3_TARGET_VF0_MASK                                           0xF000L
+//GDC1_DOORBELL_VCN_TARGET_VF1
+#define GDC1_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN0_TARGET_VF1__SHIFT                                         0x0
+#define GDC1_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN1_TARGET_VF1__SHIFT                                         0x4
+#define GDC1_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN2_TARGET_VF1__SHIFT                                         0x8
+#define GDC1_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN3_TARGET_VF1__SHIFT                                         0xc
+#define GDC1_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN0_TARGET_VF1_MASK                                           0x000FL
+#define GDC1_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN1_TARGET_VF1_MASK                                           0x00F0L
+#define GDC1_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN2_TARGET_VF1_MASK                                           0x0F00L
+#define GDC1_DOORBELL_VCN_TARGET_VF1__DOORBELL_VCN3_TARGET_VF1_MASK                                           0xF000L
+//GDC1_DOORBELL_VCN_TARGET_VF2
+#define GDC1_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN0_TARGET_VF2__SHIFT                                         0x0
+#define GDC1_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN1_TARGET_VF2__SHIFT                                         0x4
+#define GDC1_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN2_TARGET_VF2__SHIFT                                         0x8
+#define GDC1_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN3_TARGET_VF2__SHIFT                                         0xc
+#define GDC1_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN0_TARGET_VF2_MASK                                           0x000FL
+#define GDC1_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN1_TARGET_VF2_MASK                                           0x00F0L
+#define GDC1_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN2_TARGET_VF2_MASK                                           0x0F00L
+#define GDC1_DOORBELL_VCN_TARGET_VF2__DOORBELL_VCN3_TARGET_VF2_MASK                                           0xF000L
+//GDC1_DOORBELL_VCN_TARGET_VF3
+#define GDC1_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN0_TARGET_VF3__SHIFT                                         0x0
+#define GDC1_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN1_TARGET_VF3__SHIFT                                         0x4
+#define GDC1_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN2_TARGET_VF3__SHIFT                                         0x8
+#define GDC1_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN3_TARGET_VF3__SHIFT                                         0xc
+#define GDC1_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN0_TARGET_VF3_MASK                                           0x000FL
+#define GDC1_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN1_TARGET_VF3_MASK                                           0x00F0L
+#define GDC1_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN2_TARGET_VF3_MASK                                           0x0F00L
+#define GDC1_DOORBELL_VCN_TARGET_VF3__DOORBELL_VCN3_TARGET_VF3_MASK                                           0xF000L
+//GDC1_DOORBELL_VCN_TARGET_VF4
+#define GDC1_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN0_TARGET_VF4__SHIFT                                         0x0
+#define GDC1_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN1_TARGET_VF4__SHIFT                                         0x4
+#define GDC1_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN2_TARGET_VF4__SHIFT                                         0x8
+#define GDC1_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN3_TARGET_VF4__SHIFT                                         0xc
+#define GDC1_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN0_TARGET_VF4_MASK                                           0x000FL
+#define GDC1_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN1_TARGET_VF4_MASK                                           0x00F0L
+#define GDC1_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN2_TARGET_VF4_MASK                                           0x0F00L
+#define GDC1_DOORBELL_VCN_TARGET_VF4__DOORBELL_VCN3_TARGET_VF4_MASK                                           0xF000L
+//GDC1_DOORBELL_VCN_TARGET_VF5
+#define GDC1_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN0_TARGET_VF5__SHIFT                                         0x0
+#define GDC1_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN1_TARGET_VF5__SHIFT                                         0x4
+#define GDC1_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN2_TARGET_VF5__SHIFT                                         0x8
+#define GDC1_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN3_TARGET_VF5__SHIFT                                         0xc
+#define GDC1_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN0_TARGET_VF5_MASK                                           0x000FL
+#define GDC1_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN1_TARGET_VF5_MASK                                           0x00F0L
+#define GDC1_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN2_TARGET_VF5_MASK                                           0x0F00L
+#define GDC1_DOORBELL_VCN_TARGET_VF5__DOORBELL_VCN3_TARGET_VF5_MASK                                           0xF000L
+//GDC1_DOORBELL_VCN_TARGET_VF6
+#define GDC1_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN0_TARGET_VF6__SHIFT                                         0x0
+#define GDC1_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN1_TARGET_VF6__SHIFT                                         0x4
+#define GDC1_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN2_TARGET_VF6__SHIFT                                         0x8
+#define GDC1_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN3_TARGET_VF6__SHIFT                                         0xc
+#define GDC1_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN0_TARGET_VF6_MASK                                           0x000FL
+#define GDC1_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN1_TARGET_VF6_MASK                                           0x00F0L
+#define GDC1_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN2_TARGET_VF6_MASK                                           0x0F00L
+#define GDC1_DOORBELL_VCN_TARGET_VF6__DOORBELL_VCN3_TARGET_VF6_MASK                                           0xF000L
+//GDC1_DOORBELL_VCN_TARGET_VF7
+#define GDC1_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN0_TARGET_VF7__SHIFT                                         0x0
+#define GDC1_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN1_TARGET_VF7__SHIFT                                         0x4
+#define GDC1_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN2_TARGET_VF7__SHIFT                                         0x8
+#define GDC1_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN3_TARGET_VF7__SHIFT                                         0xc
+#define GDC1_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN0_TARGET_VF7_MASK                                           0x000FL
+#define GDC1_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN1_TARGET_VF7_MASK                                           0x00F0L
+#define GDC1_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN2_TARGET_VF7_MASK                                           0x0F00L
+#define GDC1_DOORBELL_VCN_TARGET_VF7__DOORBELL_VCN3_TARGET_VF7_MASK                                           0xF000L
+//GDC1_DOORBELL_ACCESS_EN_PF
+#define GDC1_DOORBELL_ACCESS_EN_PF__DOORBELL_ACCESS_EN_PF__SHIFT                                              0x0
+#define GDC1_DOORBELL_ACCESS_EN_PF__DOORBELL_ACCESS_EN_PF_MASK                                                0x00FFL
+//GDC1_DOORBELL_ACCESS_EN_VF0
+#define GDC1_DOORBELL_ACCESS_EN_VF0__DOORBELL_ACCESS_EN_VF0__SHIFT                                            0x0
+#define GDC1_DOORBELL_ACCESS_EN_VF0__XCD_ACCESS_EN_VF0__SHIFT                                                 0x8
+#define GDC1_DOORBELL_ACCESS_EN_VF0__DOORBELL_ACCESS_EN_VF0_MASK                                              0x00FFL
+#define GDC1_DOORBELL_ACCESS_EN_VF0__XCD_ACCESS_EN_VF0_MASK                                                   0xFF00L
+//GDC1_DOORBELL_ACCESS_EN_VF1
+#define GDC1_DOORBELL_ACCESS_EN_VF1__DOORBELL_ACCESS_EN_VF1__SHIFT                                            0x0
+#define GDC1_DOORBELL_ACCESS_EN_VF1__XCD_ACCESS_EN_VF1__SHIFT                                                 0x8
+#define GDC1_DOORBELL_ACCESS_EN_VF1__DOORBELL_ACCESS_EN_VF1_MASK                                              0x00FFL
+#define GDC1_DOORBELL_ACCESS_EN_VF1__XCD_ACCESS_EN_VF1_MASK                                                   0xFF00L
+//GDC1_DOORBELL_ACCESS_EN_VF2
+#define GDC1_DOORBELL_ACCESS_EN_VF2__DOORBELL_ACCESS_EN_VF2__SHIFT                                            0x0
+#define GDC1_DOORBELL_ACCESS_EN_VF2__XCD_ACCESS_EN_VF2__SHIFT                                                 0x8
+#define GDC1_DOORBELL_ACCESS_EN_VF2__DOORBELL_ACCESS_EN_VF2_MASK                                              0x00FFL
+#define GDC1_DOORBELL_ACCESS_EN_VF2__XCD_ACCESS_EN_VF2_MASK                                                   0xFF00L
+//GDC1_DOORBELL_ACCESS_EN_VF3
+#define GDC1_DOORBELL_ACCESS_EN_VF3__DOORBELL_ACCESS_EN_VF3__SHIFT                                            0x0
+#define GDC1_DOORBELL_ACCESS_EN_VF3__XCD_ACCESS_EN_VF3__SHIFT                                                 0x8
+#define GDC1_DOORBELL_ACCESS_EN_VF3__DOORBELL_ACCESS_EN_VF3_MASK                                              0x00FFL
+#define GDC1_DOORBELL_ACCESS_EN_VF3__XCD_ACCESS_EN_VF3_MASK                                                   0xFF00L
+//GDC1_DOORBELL_ACCESS_EN_VF4
+#define GDC1_DOORBELL_ACCESS_EN_VF4__DOORBELL_ACCESS_EN_VF4__SHIFT                                            0x0
+#define GDC1_DOORBELL_ACCESS_EN_VF4__XCD_ACCESS_EN_VF4__SHIFT                                                 0x8
+#define GDC1_DOORBELL_ACCESS_EN_VF4__DOORBELL_ACCESS_EN_VF4_MASK                                              0x00FFL
+#define GDC1_DOORBELL_ACCESS_EN_VF4__XCD_ACCESS_EN_VF4_MASK                                                   0xFF00L
+//GDC1_DOORBELL_ACCESS_EN_VF5
+#define GDC1_DOORBELL_ACCESS_EN_VF5__DOORBELL_ACCESS_EN_VF5__SHIFT                                            0x0
+#define GDC1_DOORBELL_ACCESS_EN_VF5__XCD_ACCESS_EN_VF5__SHIFT                                                 0x8
+#define GDC1_DOORBELL_ACCESS_EN_VF5__DOORBELL_ACCESS_EN_VF5_MASK                                              0x00FFL
+#define GDC1_DOORBELL_ACCESS_EN_VF5__XCD_ACCESS_EN_VF5_MASK                                                   0xFF00L
+//GDC1_DOORBELL_ACCESS_EN_VF6
+#define GDC1_DOORBELL_ACCESS_EN_VF6__DOORBELL_ACCESS_EN_VF6__SHIFT                                            0x0
+#define GDC1_DOORBELL_ACCESS_EN_VF6__XCD_ACCESS_EN_VF6__SHIFT                                                 0x8
+#define GDC1_DOORBELL_ACCESS_EN_VF6__DOORBELL_ACCESS_EN_VF6_MASK                                              0x00FFL
+#define GDC1_DOORBELL_ACCESS_EN_VF6__XCD_ACCESS_EN_VF6_MASK                                                   0xFF00L
+//GDC1_DOORBELL_ACCESS_EN_VF7
+#define GDC1_DOORBELL_ACCESS_EN_VF7__DOORBELL_ACCESS_EN_VF7__SHIFT                                            0x0
+#define GDC1_DOORBELL_ACCESS_EN_VF7__XCD_ACCESS_EN_VF7__SHIFT                                                 0x8
+#define GDC1_DOORBELL_ACCESS_EN_VF7__DOORBELL_ACCESS_EN_VF7_MASK                                              0x00FFL
+#define GDC1_DOORBELL_ACCESS_EN_VF7__XCD_ACCESS_EN_VF7_MASK                                                   0xFF00L
+//GDC1_NGDC_CNDI_BUS_PORT_CTRL
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_DISCON_HYSTERESIS__SHIFT                                      0x0
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_DISCON_DIS_SOCCLK__SHIFT                                      0xc
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_CMPL_DISCON_HYSTERESIS__SHIFT                                      0xd
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_CMPL_DISCON_DIS_SOCCLK__SHIFT                                      0x19
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_PORT_IDLE__SHIFT                                              0x1a
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__SHUB_CNDI_VDCI_CKEN_MASK__SHIFT                                         0x1b
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_DISCON_HYSTERESIS_MASK                                        0x00000FFFL
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_DISCON_DIS_SOCCLK_MASK                                        0x00001000L
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_CMPL_DISCON_HYSTERESIS_MASK                                        0x01FFE000L
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_CMPL_DISCON_DIS_SOCCLK_MASK                                        0x02000000L
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__CNDI_ORIG_PORT_IDLE_MASK                                                0x04000000L
+#define GDC1_NGDC_CNDI_BUS_PORT_CTRL__SHUB_CNDI_VDCI_CKEN_MASK_MASK                                           0x08000000L
+//GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOLCRED_ALLOC_VC0_ALLOC__SHIFT                    0x0
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOLCRED_ALLOC_VC1_ALLOC__SHIFT                    0x4
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOLCRED_ALLOC_VC0_ALLOC__SHIFT                    0x8
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOLCRED_ALLOC_VC1_ALLOC__SHIFT                    0xc
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_ORIG_REQ_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT            0x10
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_ORIG_ORIGDATA_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT       0x11
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT          0x12
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT          0x13
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOLCRED_ALLOC_VC0_ALLOC_MASK                      0x0000000FL
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOLCRED_ALLOC_VC1_ALLOC_MASK                      0x000000F0L
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOLCRED_ALLOC_VC0_ALLOC_MASK                      0x00000F00L
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOLCRED_ALLOC_VC1_ALLOC_MASK                      0x0000F000L
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_ORIG_REQ_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK              0x00010000L
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_ORIG_ORIGDATA_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK         0x00020000L
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_WRRSP_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK            0x00040000L
+#define GDC1_NGDC_CNDI_BUS_POOL_CRED_CTRL__CNDI_CMPL_RDRSP_POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK            0x00080000L
+
+
+// addressBlock: nbif0_nbif0_gdc_ras_gdc_ras_regblk
+//GDCSOC_ERR_RSP_CNTL
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_BYPASS__SHIFT                                                       0x0
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_ACCUM_SEL__SHIFT                                                    0x1
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_FORCE_EN__SHIFT                                                     0x2
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_FORCE_DATA__SHIFT                                                   0x3
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_STATUS_ACCUM_EN__SHIFT                                              0x4
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_DATASTATUS_ACCUM_EN__SHIFT                                          0x5
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_BYPASS_MASK                                                         0x00000001L
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_ACCUM_SEL_MASK                                                      0x00000002L
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_FORCE_EN_MASK                                                       0x00000004L
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_FORCE_DATA_MASK                                                     0x00000008L
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_STATUS_ACCUM_EN_MASK                                                0x00000010L
+#define GDCSOC_ERR_RSP_CNTL__GDCSOC_RDRSP_DATASTATUS_ACCUM_EN_MASK                                            0x00000020L
+//GDCSOC_RAS_CENTRAL_STATUS
+#define GDCSOC_RAS_CENTRAL_STATUS__GDCSOC_L2C_EgStall_det__SHIFT                                              0x0
+#define GDCSOC_RAS_CENTRAL_STATUS__GDCSOC_L2C_ErrEvent_det__SHIFT                                             0x1
+#define GDCSOC_RAS_CENTRAL_STATUS__GDCSOC_C2L_EgStall_det__SHIFT                                              0x2
+#define GDCSOC_RAS_CENTRAL_STATUS__GDCSOC_C2L_ErrEvent_det__SHIFT                                             0x3
+#define GDCSOC_RAS_CENTRAL_STATUS__GDCSOC_L2C_EgStall_det_MASK                                                0x00000001L
+#define GDCSOC_RAS_CENTRAL_STATUS__GDCSOC_L2C_ErrEvent_det_MASK                                               0x00000002L
+#define GDCSOC_RAS_CENTRAL_STATUS__GDCSOC_C2L_EgStall_det_MASK                                                0x00000004L
+#define GDCSOC_RAS_CENTRAL_STATUS__GDCSOC_C2L_ErrEvent_det_MASK                                               0x00000008L
+//GDCSOC_RAS_LEAF0_CTRL
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF0_CTRL__GDCSOC_RAS_LEAF0_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF1_CTRL
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF1_CTRL__GDCSOC_RAS_LEAF1_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF2_CTRL
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_RAS_INTR_EN__SHIFT                             0x10
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_RAS_INTR_EN_MASK                               0x00010000L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF2_CTRL__GDCSOC_RAS_LEAF2_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF3_CTRL
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF3_CTRL__GDCSOC_RAS_LEAF3_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF4_CTRL
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF4_CTRL__GDCSOC_RAS_LEAF4_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF5_CTRL
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF5_CTRL__GDCSOC_RAS_LEAF5_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF6_CTRL
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF6_CTRL__GDCSOC_RAS_LEAF6_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF7_CTRL
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF7_CTRL__GDCSOC_RAS_LEAF7_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF8_CTRL
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF8_CTRL__GDCSOC_RAS_LEAF8_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF9_CTRL
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_ERR_EVENT_DET_EN__SHIFT                                  0x0
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_POISON_ERREVENT_EN__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_POISON_STALL_EN__SHIFT                                   0x2
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_PARITY_ERREVENT_EN__SHIFT                                0x3
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_PARITY_STALL_EN__SHIFT                                   0x4
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                           0x5
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_RCVERREVENT_STALL_EN__SHIFT                              0x6
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_ERR_EVENT_GEN_EN__SHIFT                                  0x8
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_EGRESS_STALL_GEN_EN__SHIFT                               0x9
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_ERR_EVENT_PROP_EN__SHIFT                                 0xa
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_EGRESS_STALL_PROP_EN__SHIFT                              0xb
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                           0x11
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                           0x12
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                          0x13
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                      0x14
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_UCP_EN__SHIFT                                            0x15
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                        0x16
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                         0x17
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_ERR_EVENT_DET_EN_MASK                                    0x00000001L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_POISON_ERREVENT_EN_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_POISON_STALL_EN_MASK                                     0x00000004L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_PARITY_ERREVENT_EN_MASK                                  0x00000008L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_PARITY_STALL_EN_MASK                                     0x00000010L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_RCVERREVENT_ERREVENT_EN_MASK                             0x00000020L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_RCVERREVENT_STALL_EN_MASK                                0x00000040L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_ERR_EVENT_GEN_EN_MASK                                    0x00000100L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_EGRESS_STALL_GEN_EN_MASK                                 0x00000200L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_ERR_EVENT_PROP_EN_MASK                                   0x00000400L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_EGRESS_STALL_PROP_EN_MASK                                0x00000800L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                             0x00020000L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_POISON_ERREVENT_LOG_MCA_MASK                             0x00040000L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                            0x00080000L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                        0x00100000L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_UCP_EN_MASK                                              0x00200000L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                          0x00400000L
+#define GDCSOC_RAS_LEAF9_CTRL__GDCSOC_RAS_LEAF9_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                           0x00800000L
+//GDCSOC_RAS_LEAF10_CTRL
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_ERR_EVENT_DET_EN__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_POISON_ERREVENT_EN__SHIFT                              0x1
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_POISON_STALL_EN__SHIFT                                 0x2
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_PARITY_ERREVENT_EN__SHIFT                              0x3
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_PARITY_STALL_EN__SHIFT                                 0x4
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                         0x5
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_RCVERREVENT_STALL_EN__SHIFT                            0x6
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_ERR_EVENT_GEN_EN__SHIFT                                0x8
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_EGRESS_STALL_GEN_EN__SHIFT                             0x9
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_ERR_EVENT_PROP_EN__SHIFT                               0xa
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_EGRESS_STALL_PROP_EN__SHIFT                            0xb
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                         0x11
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                         0x12
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                        0x13
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                    0x14
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_UCP_EN__SHIFT                                          0x15
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                      0x16
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                       0x17
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_ERR_EVENT_DET_EN_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_POISON_ERREVENT_EN_MASK                                0x00000002L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_POISON_STALL_EN_MASK                                   0x00000004L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_PARITY_ERREVENT_EN_MASK                                0x00000008L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_PARITY_STALL_EN_MASK                                   0x00000010L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_RCVERREVENT_ERREVENT_EN_MASK                           0x00000020L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_RCVERREVENT_STALL_EN_MASK                              0x00000040L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_ERR_EVENT_GEN_EN_MASK                                  0x00000100L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_EGRESS_STALL_GEN_EN_MASK                               0x00000200L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_ERR_EVENT_PROP_EN_MASK                                 0x00000400L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_EGRESS_STALL_PROP_EN_MASK                              0x00000800L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                           0x00020000L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_POISON_ERREVENT_LOG_MCA_MASK                           0x00040000L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                          0x00080000L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                      0x00100000L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_UCP_EN_MASK                                            0x00200000L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                        0x00400000L
+#define GDCSOC_RAS_LEAF10_CTRL__GDCSOC_RAS_LEAF10_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                         0x00800000L
+//GDCSOC_RAS_LEAF11_CTRL
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_ERR_EVENT_DET_EN__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_POISON_ERREVENT_EN__SHIFT                              0x1
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_POISON_STALL_EN__SHIFT                                 0x2
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_PARITY_ERREVENT_EN__SHIFT                              0x3
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_PARITY_STALL_EN__SHIFT                                 0x4
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                         0x5
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_RCVERREVENT_STALL_EN__SHIFT                            0x6
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_ERR_EVENT_GEN_EN__SHIFT                                0x8
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_EGRESS_STALL_GEN_EN__SHIFT                             0x9
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_ERR_EVENT_PROP_EN__SHIFT                               0xa
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_EGRESS_STALL_PROP_EN__SHIFT                            0xb
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                         0x11
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                         0x12
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                        0x13
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                    0x14
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_UCP_EN__SHIFT                                          0x15
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                      0x16
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                       0x17
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_ERR_EVENT_DET_EN_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_POISON_ERREVENT_EN_MASK                                0x00000002L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_POISON_STALL_EN_MASK                                   0x00000004L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_PARITY_ERREVENT_EN_MASK                                0x00000008L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_PARITY_STALL_EN_MASK                                   0x00000010L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_RCVERREVENT_ERREVENT_EN_MASK                           0x00000020L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_RCVERREVENT_STALL_EN_MASK                              0x00000040L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_ERR_EVENT_GEN_EN_MASK                                  0x00000100L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_EGRESS_STALL_GEN_EN_MASK                               0x00000200L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_ERR_EVENT_PROP_EN_MASK                                 0x00000400L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_EGRESS_STALL_PROP_EN_MASK                              0x00000800L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                           0x00020000L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_POISON_ERREVENT_LOG_MCA_MASK                           0x00040000L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                          0x00080000L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                      0x00100000L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_UCP_EN_MASK                                            0x00200000L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                        0x00400000L
+#define GDCSOC_RAS_LEAF11_CTRL__GDCSOC_RAS_LEAF11_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                         0x00800000L
+//GDCSOC_RAS_LEAF12_CTRL
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_ERR_EVENT_DET_EN__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_POISON_ERREVENT_EN__SHIFT                              0x1
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_POISON_STALL_EN__SHIFT                                 0x2
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_PARITY_ERREVENT_EN__SHIFT                              0x3
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_PARITY_STALL_EN__SHIFT                                 0x4
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                         0x5
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_RCVERREVENT_STALL_EN__SHIFT                            0x6
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_ERR_EVENT_GEN_EN__SHIFT                                0x8
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_EGRESS_STALL_GEN_EN__SHIFT                             0x9
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_ERR_EVENT_PROP_EN__SHIFT                               0xa
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_EGRESS_STALL_PROP_EN__SHIFT                            0xb
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                         0x11
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                         0x12
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                        0x13
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                    0x14
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_UCP_EN__SHIFT                                          0x15
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                      0x16
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                       0x17
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_ERR_EVENT_DET_EN_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_POISON_ERREVENT_EN_MASK                                0x00000002L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_POISON_STALL_EN_MASK                                   0x00000004L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_PARITY_ERREVENT_EN_MASK                                0x00000008L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_PARITY_STALL_EN_MASK                                   0x00000010L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_RCVERREVENT_ERREVENT_EN_MASK                           0x00000020L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_RCVERREVENT_STALL_EN_MASK                              0x00000040L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_ERR_EVENT_GEN_EN_MASK                                  0x00000100L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_EGRESS_STALL_GEN_EN_MASK                               0x00000200L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_ERR_EVENT_PROP_EN_MASK                                 0x00000400L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_EGRESS_STALL_PROP_EN_MASK                              0x00000800L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                           0x00020000L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_POISON_ERREVENT_LOG_MCA_MASK                           0x00040000L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                          0x00080000L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                      0x00100000L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_UCP_EN_MASK                                            0x00200000L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                        0x00400000L
+#define GDCSOC_RAS_LEAF12_CTRL__GDCSOC_RAS_LEAF12_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                         0x00800000L
+//GDCSOC_RAS_LEAF13_CTRL
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_ERR_EVENT_DET_EN__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_POISON_ERREVENT_EN__SHIFT                              0x1
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_POISON_STALL_EN__SHIFT                                 0x2
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_PARITY_ERREVENT_EN__SHIFT                              0x3
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_PARITY_STALL_EN__SHIFT                                 0x4
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_RCVERREVENT_ERREVENT_EN__SHIFT                         0x5
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_RCVERREVENT_STALL_EN__SHIFT                            0x6
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_ERR_EVENT_GEN_EN__SHIFT                                0x8
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_EGRESS_STALL_GEN_EN__SHIFT                             0x9
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_ERR_EVENT_PROP_EN__SHIFT                               0xa
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_EGRESS_STALL_PROP_EN__SHIFT                            0xb
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_PARITY_ERREVENT_LOG_MCA__SHIFT                         0x11
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_POISON_ERREVENT_LOG_MCA__SHIFT                         0x12
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_TIMEOUT_ERREVENT_LOG_MCA__SHIFT                        0x13
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_RCVERREVENT_ERREVENT_LOG_MCA__SHIFT                    0x14
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_UCP_EN__SHIFT                                          0x15
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_POISON_PROPAGATION_LOG_MCA__SHIFT                      0x16
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_ERR_EVENT_PARITY_CHECK_EN__SHIFT                       0x17
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_ERR_EVENT_DET_EN_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_POISON_ERREVENT_EN_MASK                                0x00000002L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_POISON_STALL_EN_MASK                                   0x00000004L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_PARITY_ERREVENT_EN_MASK                                0x00000008L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_PARITY_STALL_EN_MASK                                   0x00000010L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_RCVERREVENT_ERREVENT_EN_MASK                           0x00000020L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_RCVERREVENT_STALL_EN_MASK                              0x00000040L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_ERR_EVENT_GEN_EN_MASK                                  0x00000100L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_EGRESS_STALL_GEN_EN_MASK                               0x00000200L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_ERR_EVENT_PROP_EN_MASK                                 0x00000400L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_EGRESS_STALL_PROP_EN_MASK                              0x00000800L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_PARITY_ERREVENT_LOG_MCA_MASK                           0x00020000L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_POISON_ERREVENT_LOG_MCA_MASK                           0x00040000L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_TIMEOUT_ERREVENT_LOG_MCA_MASK                          0x00080000L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_RCVERREVENT_ERREVENT_LOG_MCA_MASK                      0x00100000L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_UCP_EN_MASK                                            0x00200000L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_POISON_PROPAGATION_LOG_MCA_MASK                        0x00400000L
+#define GDCSOC_RAS_LEAF13_CTRL__GDCSOC_RAS_LEAF13_CTRL_ERR_EVENT_PARITY_CHECK_EN_MASK                         0x00800000L
+//GDCSOC_RAS_LEAF0_STATUS
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF0_STATUS__GDCSOC_RAS_LEAF0_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF1_STATUS
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF1_STATUS__GDCSOC_RAS_LEAF1_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF2_STATUS
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF2_STATUS__GDCSOC_RAS_LEAF2_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF3_STATUS
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF3_STATUS__GDCSOC_RAS_LEAF3_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF4_STATUS
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF4_STATUS__GDCSOC_RAS_LEAF4_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF5_STATUS
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF5_STATUS__GDCSOC_RAS_LEAF5_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF6_STATUS
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF6_STATUS__GDCSOC_RAS_LEAF6_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF7_STATUS
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF7_STATUS__GDCSOC_RAS_LEAF7_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF8_STATUS
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF8_STATUS__GDCSOC_RAS_LEAF8_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF9_STATUS
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_ERR_EVENT_RECV__SHIFT                                0x0
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_POISON_ERR_DET__SHIFT                                0x1
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_PARITY_ERR_DET__SHIFT                                0x2
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_ERR_EVENT_GENN_STAT__SHIFT                           0x8
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                      0x9
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_ERR_EVENT_PROP_STAT__SHIFT                           0xa
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                      0xb
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_ERR_EVENT_RECV_MASK                                  0x00000001L
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_POISON_ERR_DET_MASK                                  0x00000002L
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_PARITY_ERR_DET_MASK                                  0x00000004L
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_ERR_EVENT_GENN_STAT_MASK                             0x00000100L
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_EGRESS_STALLED_GENN_STAT_MASK                        0x00000200L
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_ERR_EVENT_PROP_STAT_MASK                             0x00000400L
+#define GDCSOC_RAS_LEAF9_STATUS__GDCSOC_RAS_LEAF9_STATUS_EGRESS_STALLED_PROP_STAT_MASK                        0x00000800L
+//GDCSOC_RAS_LEAF10_STATUS
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_ERR_EVENT_RECV__SHIFT                              0x0
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_POISON_ERR_DET__SHIFT                              0x1
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_PARITY_ERR_DET__SHIFT                              0x2
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_ERR_EVENT_GENN_STAT__SHIFT                         0x8
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                    0x9
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_ERR_EVENT_PROP_STAT__SHIFT                         0xa
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                    0xb
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_ERR_EVENT_RECV_MASK                                0x00000001L
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_POISON_ERR_DET_MASK                                0x00000002L
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_PARITY_ERR_DET_MASK                                0x00000004L
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_ERR_EVENT_GENN_STAT_MASK                           0x00000100L
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_EGRESS_STALLED_GENN_STAT_MASK                      0x00000200L
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_ERR_EVENT_PROP_STAT_MASK                           0x00000400L
+#define GDCSOC_RAS_LEAF10_STATUS__GDCSOC_RAS_LEAF10_STATUS_EGRESS_STALLED_PROP_STAT_MASK                      0x00000800L
+//GDCSOC_RAS_LEAF11_STATUS
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_ERR_EVENT_RECV__SHIFT                              0x0
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_POISON_ERR_DET__SHIFT                              0x1
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_PARITY_ERR_DET__SHIFT                              0x2
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_ERR_EVENT_GENN_STAT__SHIFT                         0x8
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                    0x9
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_ERR_EVENT_PROP_STAT__SHIFT                         0xa
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                    0xb
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_ERR_EVENT_RECV_MASK                                0x00000001L
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_POISON_ERR_DET_MASK                                0x00000002L
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_PARITY_ERR_DET_MASK                                0x00000004L
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_ERR_EVENT_GENN_STAT_MASK                           0x00000100L
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_EGRESS_STALLED_GENN_STAT_MASK                      0x00000200L
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_ERR_EVENT_PROP_STAT_MASK                           0x00000400L
+#define GDCSOC_RAS_LEAF11_STATUS__GDCSOC_RAS_LEAF11_STATUS_EGRESS_STALLED_PROP_STAT_MASK                      0x00000800L
+//GDCSOC_RAS_LEAF12_STATUS
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_ERR_EVENT_RECV__SHIFT                              0x0
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_POISON_ERR_DET__SHIFT                              0x1
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_PARITY_ERR_DET__SHIFT                              0x2
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_ERR_EVENT_GENN_STAT__SHIFT                         0x8
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                    0x9
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_ERR_EVENT_PROP_STAT__SHIFT                         0xa
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                    0xb
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_ERR_EVENT_RECV_MASK                                0x00000001L
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_POISON_ERR_DET_MASK                                0x00000002L
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_PARITY_ERR_DET_MASK                                0x00000004L
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_ERR_EVENT_GENN_STAT_MASK                           0x00000100L
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_EGRESS_STALLED_GENN_STAT_MASK                      0x00000200L
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_ERR_EVENT_PROP_STAT_MASK                           0x00000400L
+#define GDCSOC_RAS_LEAF12_STATUS__GDCSOC_RAS_LEAF12_STATUS_EGRESS_STALLED_PROP_STAT_MASK                      0x00000800L
+//GDCSOC_RAS_LEAF13_STATUS
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_ERR_EVENT_RECV__SHIFT                              0x0
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_POISON_ERR_DET__SHIFT                              0x1
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_PARITY_ERR_DET__SHIFT                              0x2
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_ERR_EVENT_GENN_STAT__SHIFT                         0x8
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_EGRESS_STALLED_GENN_STAT__SHIFT                    0x9
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_ERR_EVENT_PROP_STAT__SHIFT                         0xa
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_EGRESS_STALLED_PROP_STAT__SHIFT                    0xb
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_ERR_EVENT_RECV_MASK                                0x00000001L
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_POISON_ERR_DET_MASK                                0x00000002L
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_PARITY_ERR_DET_MASK                                0x00000004L
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_ERR_EVENT_GENN_STAT_MASK                           0x00000100L
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_EGRESS_STALLED_GENN_STAT_MASK                      0x00000200L
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_ERR_EVENT_PROP_STAT_MASK                           0x00000400L
+#define GDCSOC_RAS_LEAF13_STATUS__GDCSOC_RAS_LEAF13_STATUS_EGRESS_STALLED_PROP_STAT_MASK                      0x00000800L
+
+
+// addressBlock: nbif0_nbif0_gdc_sec_GDCSEC_DEC
+//GDC_SEC_MCA_SMN_CTRL0
+#define GDC_SEC_MCA_SMN_CTRL0__MCA_INT_REQ_ADDR__SHIFT                                                        0x0
+#define GDC_SEC_MCA_SMN_CTRL0__MCA_INT_REQ_APER_ID__SHIFT                                                     0x14
+#define GDC_SEC_MCA_SMN_CTRL0__MCA_INT_REQ_ADDR_MASK                                                          0x000FFFFFL
+#define GDC_SEC_MCA_SMN_CTRL0__MCA_INT_REQ_APER_ID_MASK                                                       0xFFF00000L
+//GDC_SEC_MCA_SMN_CTRL1
+#define GDC_SEC_MCA_SMN_CTRL1__MCA_INT_REQ_MCM_ADDR__SHIFT                                                    0x0
+#define GDC_SEC_MCA_SMN_CTRL1__MCA_SMN_SEC_FLAG__SHIFT                                                        0x10
+#define GDC_SEC_MCA_SMN_CTRL1__MCA_INT_REQ_MCM_ADDR_MASK                                                      0x0000FFFFL
+#define GDC_SEC_MCA_SMN_CTRL1__MCA_SMN_SEC_FLAG_MASK                                                          0x000F0000L
+//XCD_DOORBELL_FENCE
+#define XCD_DOORBELL_FENCE__XCD_DOORBELL_FENCE_ONCE_TRIGGER_DIS__SHIFT                                        0x0
+#define XCD_DOORBELL_FENCE__XCD_DOORBELL_FENCE_INTR_ENABLE__SHIFT                                             0x1
+#define XCD_DOORBELL_FENCE__RMOTE_SDMA_SENT__SHIFT                                                            0x8
+#define XCD_DOORBELL_FENCE__SDMA_0_SENT__SHIFT                                                                0x9
+#define XCD_DOORBELL_FENCE__SDMA_1_SENT__SHIFT                                                                0xa
+#define XCD_DOORBELL_FENCE__SDMA_2_SENT__SHIFT                                                                0xb
+#define XCD_DOORBELL_FENCE__SDMA_3_SENT__SHIFT                                                                0xc
+#define XCD_DOORBELL_FENCE__SDMA_4_SENT__SHIFT                                                                0xd
+#define XCD_DOORBELL_FENCE__SDMA_5_SENT__SHIFT                                                                0xe
+#define XCD_DOORBELL_FENCE__SDMA_6_SENT__SHIFT                                                                0xf
+#define XCD_DOORBELL_FENCE__SDMA_7_SENT__SHIFT                                                                0x10
+#define XCD_DOORBELL_FENCE__RMOTE_CP_SENT__SHIFT                                                              0x11
+#define XCD_DOORBELL_FENCE__CP_0_SENT__SHIFT                                                                  0x12
+#define XCD_DOORBELL_FENCE__CP_1_SENT__SHIFT                                                                  0x13
+#define XCD_DOORBELL_FENCE__CP_2_SENT__SHIFT                                                                  0x14
+#define XCD_DOORBELL_FENCE__CP_3_SENT__SHIFT                                                                  0x15
+#define XCD_DOORBELL_FENCE__CP_4_SENT__SHIFT                                                                  0x16
+#define XCD_DOORBELL_FENCE__CP_5_SENT__SHIFT                                                                  0x17
+#define XCD_DOORBELL_FENCE__CP_6_SENT__SHIFT                                                                  0x18
+#define XCD_DOORBELL_FENCE__CP_7_SENT__SHIFT                                                                  0x19
+#define XCD_DOORBELL_FENCE__REMOTE_CLIENT_SENT__SHIFT                                                         0x1a
+#define XCD_DOORBELL_FENCE__REMOTE_CLIENT_CLR_PENDING__SHIFT                                                  0x1b
+#define XCD_DOORBELL_FENCE__SHUB_SLV_MODE__SHIFT                                                              0x1c
+#define XCD_DOORBELL_FENCE__XCD_DOORBELL_FENCE_ONCE_TRIGGER_DIS_MASK                                          0x00000001L
+#define XCD_DOORBELL_FENCE__XCD_DOORBELL_FENCE_INTR_ENABLE_MASK                                               0x00000002L
+#define XCD_DOORBELL_FENCE__RMOTE_SDMA_SENT_MASK                                                              0x00000100L
+#define XCD_DOORBELL_FENCE__SDMA_0_SENT_MASK                                                                  0x00000200L
+#define XCD_DOORBELL_FENCE__SDMA_1_SENT_MASK                                                                  0x00000400L
+#define XCD_DOORBELL_FENCE__SDMA_2_SENT_MASK                                                                  0x00000800L
+#define XCD_DOORBELL_FENCE__SDMA_3_SENT_MASK                                                                  0x00001000L
+#define XCD_DOORBELL_FENCE__SDMA_4_SENT_MASK                                                                  0x00002000L
+#define XCD_DOORBELL_FENCE__SDMA_5_SENT_MASK                                                                  0x00004000L
+#define XCD_DOORBELL_FENCE__SDMA_6_SENT_MASK                                                                  0x00008000L
+#define XCD_DOORBELL_FENCE__SDMA_7_SENT_MASK                                                                  0x00010000L
+#define XCD_DOORBELL_FENCE__RMOTE_CP_SENT_MASK                                                                0x00020000L
+#define XCD_DOORBELL_FENCE__CP_0_SENT_MASK                                                                    0x00040000L
+#define XCD_DOORBELL_FENCE__CP_1_SENT_MASK                                                                    0x00080000L
+#define XCD_DOORBELL_FENCE__CP_2_SENT_MASK                                                                    0x00100000L
+#define XCD_DOORBELL_FENCE__CP_3_SENT_MASK                                                                    0x00200000L
+#define XCD_DOORBELL_FENCE__CP_4_SENT_MASK                                                                    0x00400000L
+#define XCD_DOORBELL_FENCE__CP_5_SENT_MASK                                                                    0x00800000L
+#define XCD_DOORBELL_FENCE__CP_6_SENT_MASK                                                                    0x01000000L
+#define XCD_DOORBELL_FENCE__CP_7_SENT_MASK                                                                    0x02000000L
+#define XCD_DOORBELL_FENCE__REMOTE_CLIENT_SENT_MASK                                                           0x04000000L
+#define XCD_DOORBELL_FENCE__REMOTE_CLIENT_CLR_PENDING_MASK                                                    0x08000000L
+#define XCD_DOORBELL_FENCE__SHUB_SLV_MODE_MASK                                                                0x10000000L
+//SHUB_PWRBRK_DEGLITCH
+#define SHUB_PWRBRK_DEGLITCH__SHUB_PWRBRK_DEGLITCH_CYCLE__SHIFT                                               0x0
+#define SHUB_PWRBRK_DEGLITCH__SHUB_PWRBRK_DEGLITCH_BYPASS__SHIFT                                              0x8
+#define SHUB_PWRBRK_DEGLITCH__SHUB_PWRBRK_DEGLITCH_CYCLE_MASK                                                 0x000000FFL
+#define SHUB_PWRBRK_DEGLITCH__SHUB_PWRBRK_DEGLITCH_BYPASS_MASK                                                0x00000100L
+//SHUB_DIE_CTRL
+#define SHUB_DIE_CTRL__SHUB_MST_DIE__SHIFT                                                                    0x0
+#define SHUB_DIE_CTRL__SHUB_MST_DIE_MASK                                                                      0x00000001L
+//GDC_A2S_ROUTING_TABLE_SECLVL_TRUST_LEVEL
+#define GDC_A2S_ROUTING_TABLE_SECLVL_TRUST_LEVEL__GDC_A2S_ROUTING_TABLE_SECLVL_TRUST_LEVEL__SHIFT             0x0
+#define GDC_A2S_ROUTING_TABLE_SECLVL_TRUST_LEVEL__GDC_A2S_ROUTING_TABLE_SECLVL_TRUST_LEVEL_MASK               0x00000007L
+//XCD_DOORBELL_FENCE_1
+#define XCD_DOORBELL_FENCE_1__XCD_0_DOORBELL_FENCE__SHIFT                                                     0x0
+#define XCD_DOORBELL_FENCE_1__XCD_1_DOORBELL_FENCE__SHIFT                                                     0x1
+#define XCD_DOORBELL_FENCE_1__XCD_2_DOORBELL_FENCE__SHIFT                                                     0x2
+#define XCD_DOORBELL_FENCE_1__XCD_3_DOORBELL_FENCE__SHIFT                                                     0x3
+#define XCD_DOORBELL_FENCE_1__XCD_4_DOORBELL_FENCE__SHIFT                                                     0x4
+#define XCD_DOORBELL_FENCE_1__XCD_5_DOORBELL_FENCE__SHIFT                                                     0x5
+#define XCD_DOORBELL_FENCE_1__XCD_6_DOORBELL_FENCE__SHIFT                                                     0x6
+#define XCD_DOORBELL_FENCE_1__XCD_7_DOORBELL_FENCE__SHIFT                                                     0x7
+#define XCD_DOORBELL_FENCE_1__XCD_0_DOORBELL_DISABLE__SHIFT                                                   0x8
+#define XCD_DOORBELL_FENCE_1__XCD_1_DOORBELL_DISABLE__SHIFT                                                   0x9
+#define XCD_DOORBELL_FENCE_1__XCD_2_DOORBELL_DISABLE__SHIFT                                                   0xa
+#define XCD_DOORBELL_FENCE_1__XCD_3_DOORBELL_DISABLE__SHIFT                                                   0xb
+#define XCD_DOORBELL_FENCE_1__XCD_4_DOORBELL_DISABLE__SHIFT                                                   0xc
+#define XCD_DOORBELL_FENCE_1__XCD_5_DOORBELL_DISABLE__SHIFT                                                   0xd
+#define XCD_DOORBELL_FENCE_1__XCD_6_DOORBELL_DISABLE__SHIFT                                                   0xe
+#define XCD_DOORBELL_FENCE_1__XCD_7_DOORBELL_DISABLE__SHIFT                                                   0xf
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_0_REG__SHIFT                                       0x10
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_1_REG__SHIFT                                       0x11
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_2_REG__SHIFT                                       0x12
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_3_REG__SHIFT                                       0x13
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_4_REG__SHIFT                                       0x14
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_5_REG__SHIFT                                       0x15
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_6_REG__SHIFT                                       0x16
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_7_REG__SHIFT                                       0x17
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_0_REG__SHIFT                                     0x18
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_1_REG__SHIFT                                     0x19
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_2_REG__SHIFT                                     0x1a
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_3_REG__SHIFT                                     0x1b
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_4_REG__SHIFT                                     0x1c
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_5_REG__SHIFT                                     0x1d
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_6_REG__SHIFT                                     0x1e
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_7_REG__SHIFT                                     0x1f
+#define XCD_DOORBELL_FENCE_1__XCD_0_DOORBELL_FENCE_MASK                                                       0x00000001L
+#define XCD_DOORBELL_FENCE_1__XCD_1_DOORBELL_FENCE_MASK                                                       0x00000002L
+#define XCD_DOORBELL_FENCE_1__XCD_2_DOORBELL_FENCE_MASK                                                       0x00000004L
+#define XCD_DOORBELL_FENCE_1__XCD_3_DOORBELL_FENCE_MASK                                                       0x00000008L
+#define XCD_DOORBELL_FENCE_1__XCD_4_DOORBELL_FENCE_MASK                                                       0x00000010L
+#define XCD_DOORBELL_FENCE_1__XCD_5_DOORBELL_FENCE_MASK                                                       0x00000020L
+#define XCD_DOORBELL_FENCE_1__XCD_6_DOORBELL_FENCE_MASK                                                       0x00000040L
+#define XCD_DOORBELL_FENCE_1__XCD_7_DOORBELL_FENCE_MASK                                                       0x00000080L
+#define XCD_DOORBELL_FENCE_1__XCD_0_DOORBELL_DISABLE_MASK                                                     0x00000100L
+#define XCD_DOORBELL_FENCE_1__XCD_1_DOORBELL_DISABLE_MASK                                                     0x00000200L
+#define XCD_DOORBELL_FENCE_1__XCD_2_DOORBELL_DISABLE_MASK                                                     0x00000400L
+#define XCD_DOORBELL_FENCE_1__XCD_3_DOORBELL_DISABLE_MASK                                                     0x00000800L
+#define XCD_DOORBELL_FENCE_1__XCD_4_DOORBELL_DISABLE_MASK                                                     0x00001000L
+#define XCD_DOORBELL_FENCE_1__XCD_5_DOORBELL_DISABLE_MASK                                                     0x00002000L
+#define XCD_DOORBELL_FENCE_1__XCD_6_DOORBELL_DISABLE_MASK                                                     0x00004000L
+#define XCD_DOORBELL_FENCE_1__XCD_7_DOORBELL_DISABLE_MASK                                                     0x00008000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_0_REG_MASK                                         0x00010000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_1_REG_MASK                                         0x00020000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_2_REG_MASK                                         0x00040000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_3_REG_MASK                                         0x00080000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_4_REG_MASK                                         0x00100000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_5_REG_MASK                                         0x00200000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_6_REG_MASK                                         0x00400000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_CP_7_REG_MASK                                         0x00800000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_0_REG_MASK                                       0x01000000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_1_REG_MASK                                       0x02000000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_2_REG_MASK                                       0x04000000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_3_REG_MASK                                       0x08000000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_4_REG_MASK                                       0x10000000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_5_REG_MASK                                       0x20000000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_6_REG_MASK                                       0x40000000L
+#define XCD_DOORBELL_FENCE_1__REMOTE_CLIENT_CLR_PENDING_SDMA_7_REG_MASK                                       0x80000000L
+//NGDC_CNDI_DOORBELL_REQUSER_0
+#define NGDC_CNDI_DOORBELL_REQUSER_0__NGDC_CNDI_IH_ReqUser__SHIFT                                             0x0
+#define NGDC_CNDI_DOORBELL_REQUSER_0__NGDC_CNDI_IH_ReqUser_MASK                                               0x0000003FL
+//NGDC_CNDI_DOORBELL_REQUSER_1
+#define NGDC_CNDI_DOORBELL_REQUSER_1__NGDC_CNDI_VCN0_ReqUser__SHIFT                                           0x0
+#define NGDC_CNDI_DOORBELL_REQUSER_1__NGDC_CNDI_VCN1_ReqUser__SHIFT                                           0x8
+#define NGDC_CNDI_DOORBELL_REQUSER_1__NGDC_CNDI_VCN2_ReqUser__SHIFT                                           0x10
+#define NGDC_CNDI_DOORBELL_REQUSER_1__NGDC_CNDI_VCN3_ReqUser__SHIFT                                           0x18
+#define NGDC_CNDI_DOORBELL_REQUSER_1__NGDC_CNDI_VCN0_ReqUser_MASK                                             0x0000003FL
+#define NGDC_CNDI_DOORBELL_REQUSER_1__NGDC_CNDI_VCN1_ReqUser_MASK                                             0x00003F00L
+#define NGDC_CNDI_DOORBELL_REQUSER_1__NGDC_CNDI_VCN2_ReqUser_MASK                                             0x003F0000L
+#define NGDC_CNDI_DOORBELL_REQUSER_1__NGDC_CNDI_VCN3_ReqUser_MASK                                             0x3F000000L
+//NGDC_CNDI_DOORBELL_REQUSER_2
+#define NGDC_CNDI_DOORBELL_REQUSER_2__NGDC_CNDI_XCD0_SDMA_ReqUser__SHIFT                                      0x0
+#define NGDC_CNDI_DOORBELL_REQUSER_2__NGDC_CNDI_XCD1_SDMA_ReqUser__SHIFT                                      0x8
+#define NGDC_CNDI_DOORBELL_REQUSER_2__NGDC_CNDI_XCD2_SDMA_ReqUser__SHIFT                                      0x10
+#define NGDC_CNDI_DOORBELL_REQUSER_2__NGDC_CNDI_XCD3_SDMA_ReqUser__SHIFT                                      0x18
+#define NGDC_CNDI_DOORBELL_REQUSER_2__NGDC_CNDI_XCD0_SDMA_ReqUser_MASK                                        0x0000003FL
+#define NGDC_CNDI_DOORBELL_REQUSER_2__NGDC_CNDI_XCD1_SDMA_ReqUser_MASK                                        0x00003F00L
+#define NGDC_CNDI_DOORBELL_REQUSER_2__NGDC_CNDI_XCD2_SDMA_ReqUser_MASK                                        0x003F0000L
+#define NGDC_CNDI_DOORBELL_REQUSER_2__NGDC_CNDI_XCD3_SDMA_ReqUser_MASK                                        0x3F000000L
+//NGDC_CNDI_DOORBELL_REQUSER_3
+#define NGDC_CNDI_DOORBELL_REQUSER_3__NGDC_CNDI_XCD4_SDMA_ReqUser__SHIFT                                      0x0
+#define NGDC_CNDI_DOORBELL_REQUSER_3__NGDC_CNDI_XCD5_SDMA_ReqUser__SHIFT                                      0x8
+#define NGDC_CNDI_DOORBELL_REQUSER_3__NGDC_CNDI_XCD6_SDMA_ReqUser__SHIFT                                      0x10
+#define NGDC_CNDI_DOORBELL_REQUSER_3__NGDC_CNDI_XCD7_SDMA_ReqUser__SHIFT                                      0x18
+#define NGDC_CNDI_DOORBELL_REQUSER_3__NGDC_CNDI_XCD4_SDMA_ReqUser_MASK                                        0x0000003FL
+#define NGDC_CNDI_DOORBELL_REQUSER_3__NGDC_CNDI_XCD5_SDMA_ReqUser_MASK                                        0x00003F00L
+#define NGDC_CNDI_DOORBELL_REQUSER_3__NGDC_CNDI_XCD6_SDMA_ReqUser_MASK                                        0x003F0000L
+#define NGDC_CNDI_DOORBELL_REQUSER_3__NGDC_CNDI_XCD7_SDMA_ReqUser_MASK                                        0x3F000000L
+//NGDC_CNDI_DOORBELL_REQUSER_4
+#define NGDC_CNDI_DOORBELL_REQUSER_4__NGDC_CNDI_XCD0_CP_ReqUser__SHIFT                                        0x0
+#define NGDC_CNDI_DOORBELL_REQUSER_4__NGDC_CNDI_XCD1_CP_ReqUser__SHIFT                                        0x8
+#define NGDC_CNDI_DOORBELL_REQUSER_4__NGDC_CNDI_XCD2_CP_ReqUser__SHIFT                                        0x10
+#define NGDC_CNDI_DOORBELL_REQUSER_4__NGDC_CNDI_XCD3_CP_ReqUser__SHIFT                                        0x18
+#define NGDC_CNDI_DOORBELL_REQUSER_4__NGDC_CNDI_XCD0_CP_ReqUser_MASK                                          0x0000003FL
+#define NGDC_CNDI_DOORBELL_REQUSER_4__NGDC_CNDI_XCD1_CP_ReqUser_MASK                                          0x00003F00L
+#define NGDC_CNDI_DOORBELL_REQUSER_4__NGDC_CNDI_XCD2_CP_ReqUser_MASK                                          0x003F0000L
+#define NGDC_CNDI_DOORBELL_REQUSER_4__NGDC_CNDI_XCD3_CP_ReqUser_MASK                                          0x3F000000L
+//NGDC_CNDI_DOORBELL_REQUSER_5
+#define NGDC_CNDI_DOORBELL_REQUSER_5__NGDC_CNDI_XCD4_CP_ReqUser__SHIFT                                        0x0
+#define NGDC_CNDI_DOORBELL_REQUSER_5__NGDC_CNDI_XCD5_CP_ReqUser__SHIFT                                        0x8
+#define NGDC_CNDI_DOORBELL_REQUSER_5__NGDC_CNDI_XCD6_CP_ReqUser__SHIFT                                        0x10
+#define NGDC_CNDI_DOORBELL_REQUSER_5__NGDC_CNDI_XCD7_CP_ReqUser__SHIFT                                        0x18
+#define NGDC_CNDI_DOORBELL_REQUSER_5__NGDC_CNDI_XCD4_CP_ReqUser_MASK                                          0x0000003FL
+#define NGDC_CNDI_DOORBELL_REQUSER_5__NGDC_CNDI_XCD5_CP_ReqUser_MASK                                          0x00003F00L
+#define NGDC_CNDI_DOORBELL_REQUSER_5__NGDC_CNDI_XCD6_CP_ReqUser_MASK                                          0x003F0000L
+#define NGDC_CNDI_DOORBELL_REQUSER_5__NGDC_CNDI_XCD7_CP_ReqUser_MASK                                          0x3F000000L
+//NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_IH_REQADDR_32_33BIT__SHIFT                           0x0
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_VCN0_REQADDR_32_33BIT__SHIFT                         0x2
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_VCN1_REQADDR_32_33BIT__SHIFT                         0x4
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_VCN2_REQADDR_32_33BIT__SHIFT                         0x6
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_VCN3_REQADDR_32_33BIT__SHIFT                         0x8
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD0_SDMA_REQADDR_32_33BIT__SHIFT                    0xa
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD1_SDMA_REQADDR_32_33BIT__SHIFT                    0xc
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD2_SDMA_REQADDR_32_33BIT__SHIFT                    0xe
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD3_SDMA_REQADDR_32_33BIT__SHIFT                    0x10
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD4_SDMA_REQADDR_32_33BIT__SHIFT                    0x12
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD5_SDMA_REQADDR_32_33BIT__SHIFT                    0x14
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD6_SDMA_REQADDR_32_33BIT__SHIFT                    0x16
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD7_SDMA_REQADDR_32_33BIT__SHIFT                    0x18
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_IH_REQADDR_32_33BIT_MASK                             0x00000003L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_VCN0_REQADDR_32_33BIT_MASK                           0x0000000CL
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_VCN1_REQADDR_32_33BIT_MASK                           0x00000030L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_VCN2_REQADDR_32_33BIT_MASK                           0x000000C0L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_VCN3_REQADDR_32_33BIT_MASK                           0x00000300L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD0_SDMA_REQADDR_32_33BIT_MASK                      0x00000C00L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD1_SDMA_REQADDR_32_33BIT_MASK                      0x00003000L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD2_SDMA_REQADDR_32_33BIT_MASK                      0x0000C000L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD3_SDMA_REQADDR_32_33BIT_MASK                      0x00030000L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD4_SDMA_REQADDR_32_33BIT_MASK                      0x000C0000L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD5_SDMA_REQADDR_32_33BIT_MASK                      0x00300000L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD6_SDMA_REQADDR_32_33BIT_MASK                      0x00C00000L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_0__NGDC_CNDI_XCD7_SDMA_REQADDR_32_33BIT_MASK                      0x03000000L
+//NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD0_CP_REQADDR_32_33BIT__SHIFT                      0x0
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD1_CP_REQADDR_32_33BIT__SHIFT                      0x2
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD2_CP_REQADDR_32_33BIT__SHIFT                      0x4
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD3_CP_REQADDR_32_33BIT__SHIFT                      0x6
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD4_CP_REQADDR_32_33BIT__SHIFT                      0x8
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD5_CP_REQADDR_32_33BIT__SHIFT                      0xa
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD6_CP_REQADDR_32_33BIT__SHIFT                      0xc
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD7_CP_REQADDR_32_33BIT__SHIFT                      0xe
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD0_CP_REQADDR_32_33BIT_MASK                        0x00000003L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD1_CP_REQADDR_32_33BIT_MASK                        0x0000000CL
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD2_CP_REQADDR_32_33BIT_MASK                        0x00000030L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD3_CP_REQADDR_32_33BIT_MASK                        0x000000C0L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD4_CP_REQADDR_32_33BIT_MASK                        0x00000300L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD5_CP_REQADDR_32_33BIT_MASK                        0x00000C00L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD6_CP_REQADDR_32_33BIT_MASK                        0x00003000L
+#define NGDC_CNDI_DOORBELL_REQADDR_32_33BIT_1__NGDC_CNDI_XCD7_CP_REQADDR_32_33BIT_MASK                        0x0000C000L
+
+
+// addressBlock: nbif0_nbif0_gdc_rst_GDCRST_DEC
+//SHUB_PF_FLR_RST
+#define SHUB_PF_FLR_RST__DEV0_PF0_FLR_RST__SHIFT                                                              0x0
+#define SHUB_PF_FLR_RST__DEV0_PF1_FLR_RST__SHIFT                                                              0x1
+#define SHUB_PF_FLR_RST__DEV0_PF0_FLR_RST_MASK                                                                0x00000001L
+#define SHUB_PF_FLR_RST__DEV0_PF1_FLR_RST_MASK                                                                0x00000002L
+//SHUB_GFX_DRV_VPU_RST
+#define SHUB_GFX_DRV_VPU_RST__GFX_DRV_MODE1_RST__SHIFT                                                        0x0
+#define SHUB_GFX_DRV_VPU_RST__GFX_DRV_MODE1_RST_MASK                                                          0x00000001L
+//SHUB_LINK_RESET
+#define SHUB_LINK_RESET__LINK_P0_RESET__SHIFT                                                                 0x0
+#define SHUB_LINK_RESET__LINK_P1_RESET__SHIFT                                                                 0x1
+#define SHUB_LINK_RESET__LINK_P2_RESET__SHIFT                                                                 0x2
+#define SHUB_LINK_RESET__LINK_P3_RESET__SHIFT                                                                 0x3
+#define SHUB_LINK_RESET__LINK_P0_RESET_MASK                                                                   0x00000001L
+#define SHUB_LINK_RESET__LINK_P1_RESET_MASK                                                                   0x00000002L
+#define SHUB_LINK_RESET__LINK_P2_RESET_MASK                                                                   0x00000004L
+#define SHUB_LINK_RESET__LINK_P3_RESET_MASK                                                                   0x00000008L
+//SHUB_HST_NIC400_SW_CTRL_RESET
+#define SHUB_HST_NIC400_SW_CTRL_RESET__HST_NIC400_SW_CTRL_RESET__SHIFT                                        0x0
+#define SHUB_HST_NIC400_SW_CTRL_RESET__HST_NIC400_SW_CTRL_RESET_MASK                                          0x00000001L
+//SHUB_DEV0_PF0_VF_FLR_RST
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF0_FLR_RST__SHIFT                                                 0x0
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF1_FLR_RST__SHIFT                                                 0x1
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF2_FLR_RST__SHIFT                                                 0x2
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF3_FLR_RST__SHIFT                                                 0x3
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF4_FLR_RST__SHIFT                                                 0x4
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF5_FLR_RST__SHIFT                                                 0x5
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF6_FLR_RST__SHIFT                                                 0x6
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF7_FLR_RST__SHIFT                                                 0x7
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_SOFTPF_FLR_RST__SHIFT                                              0x1f
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF0_FLR_RST_MASK                                                   0x00000001L
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF1_FLR_RST_MASK                                                   0x00000002L
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF2_FLR_RST_MASK                                                   0x00000004L
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF3_FLR_RST_MASK                                                   0x00000008L
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF4_FLR_RST_MASK                                                   0x00000010L
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF5_FLR_RST_MASK                                                   0x00000020L
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF6_FLR_RST_MASK                                                   0x00000040L
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_VF7_FLR_RST_MASK                                                   0x00000080L
+#define SHUB_DEV0_PF0_VF_FLR_RST__DEV0_PF0_SOFTPF_FLR_RST_MASK                                                0x80000000L
+//SHUB_DEV0_PF1_VF_FLR_RST
+#define SHUB_DEV0_PF1_VF_FLR_RST__DEV0_PF1_VF0_FLR_RST__SHIFT                                                 0x0
+#define SHUB_DEV0_PF1_VF_FLR_RST__DEV0_PF1_VF0_FLR_RST_MASK                                                   0x00000001L
+//SHUB_HARD_RST_CTRL
+#define SHUB_HARD_RST_CTRL__COR_RESET_EN__SHIFT                                                               0x0
+#define SHUB_HARD_RST_CTRL__REG_RESET_EN__SHIFT                                                               0x1
+#define SHUB_HARD_RST_CTRL__STY_RESET_EN__SHIFT                                                               0x2
+#define SHUB_HARD_RST_CTRL__NIC400_RESET_EN__SHIFT                                                            0x3
+#define SHUB_HARD_RST_CTRL__SDP_PORT_RESET_EN__SHIFT                                                          0x4
+#define SHUB_HARD_RST_CTRL__SION_AON_RESET_EN__SHIFT                                                          0x5
+#define SHUB_HARD_RST_CTRL__COR_RESET_EN_MASK                                                                 0x00000001L
+#define SHUB_HARD_RST_CTRL__REG_RESET_EN_MASK                                                                 0x00000002L
+#define SHUB_HARD_RST_CTRL__STY_RESET_EN_MASK                                                                 0x00000004L
+#define SHUB_HARD_RST_CTRL__NIC400_RESET_EN_MASK                                                              0x00000008L
+#define SHUB_HARD_RST_CTRL__SDP_PORT_RESET_EN_MASK                                                            0x00000010L
+#define SHUB_HARD_RST_CTRL__SION_AON_RESET_EN_MASK                                                            0x00000020L
+//SHUB_SOFT_RST_CTRL
+#define SHUB_SOFT_RST_CTRL__COR_RESET_EN__SHIFT                                                               0x0
+#define SHUB_SOFT_RST_CTRL__REG_RESET_EN__SHIFT                                                               0x1
+#define SHUB_SOFT_RST_CTRL__STY_RESET_EN__SHIFT                                                               0x2
+#define SHUB_SOFT_RST_CTRL__NIC400_RESET_EN__SHIFT                                                            0x3
+#define SHUB_SOFT_RST_CTRL__SDP_PORT_RESET_EN__SHIFT                                                          0x4
+#define SHUB_SOFT_RST_CTRL__SION_AON_RESET_EN__SHIFT                                                          0x5
+#define SHUB_SOFT_RST_CTRL__COR_RESET_EN_MASK                                                                 0x00000001L
+#define SHUB_SOFT_RST_CTRL__REG_RESET_EN_MASK                                                                 0x00000002L
+#define SHUB_SOFT_RST_CTRL__STY_RESET_EN_MASK                                                                 0x00000004L
+#define SHUB_SOFT_RST_CTRL__NIC400_RESET_EN_MASK                                                              0x00000008L
+#define SHUB_SOFT_RST_CTRL__SDP_PORT_RESET_EN_MASK                                                            0x00000010L
+#define SHUB_SOFT_RST_CTRL__SION_AON_RESET_EN_MASK                                                            0x00000020L
+//SHUB_SDP_PORT_RST
+#define SHUB_SDP_PORT_RST__A2S_SDP_PORT_RST__SHIFT                                                            0x0
+#define SHUB_SDP_PORT_RST__NBIFSION_BIF_SDP_PORT_RST__SHIFT                                                   0x1
+#define SHUB_SDP_PORT_RST__ATHUB_HST_SDP_PORT_RST__SHIFT                                                      0x2
+#define SHUB_SDP_PORT_RST__ATHUB_DMA_SDP_PORT_RST__SHIFT                                                      0x3
+#define SHUB_SDP_PORT_RST__ATDMA_NBIFSOIN_SDP_PORT_RST__SHIFT                                                 0x4
+#define SHUB_SDP_PORT_RST__INT_NBIFSION_SDP_PORT_RST__SHIFT                                                   0x5
+#define SHUB_SDP_PORT_RST__MP4_SDP_SDP_PORT_RST__SHIFT                                                        0x6
+#define SHUB_SDP_PORT_RST__GDC_HST_SDP_PORT_RST__SHIFT                                                        0x7
+#define SHUB_SDP_PORT_RST__NTB_HST_SDP_PORT_RST__SHIFT                                                        0x8
+#define SHUB_SDP_PORT_RST__NTB_DMA_SDP_PORT_RST__SHIFT                                                        0x9
+#define SHUB_SDP_PORT_RST__DACCFE_NBIF_DMA_SDP_PORT_RST__SHIFT                                                0xa
+#define SHUB_SDP_PORT_RST__DACCBE_NBIF_DMA_SDP_PORT_RST__SHIFT                                                0xb
+#define SHUB_SDP_PORT_RST__DACCFE_HST_SDP_PORT_RST__SHIFT                                                     0xc
+#define SHUB_SDP_PORT_RST__SHUB_CNDI_HST_SDP_PORT_RST__SHIFT                                                  0xd
+#define SHUB_SDP_PORT_RST__HST_ATHUB_HST_SION_SDP_PORT_RST__SHIFT                                             0xe
+#define SHUB_SDP_PORT_RST__SION_AON_RST__SHIFT                                                                0x18
+#define SHUB_SDP_PORT_RST__A2S_SDP_PORT_RST_MASK                                                              0x00000001L
+#define SHUB_SDP_PORT_RST__NBIFSION_BIF_SDP_PORT_RST_MASK                                                     0x00000002L
+#define SHUB_SDP_PORT_RST__ATHUB_HST_SDP_PORT_RST_MASK                                                        0x00000004L
+#define SHUB_SDP_PORT_RST__ATHUB_DMA_SDP_PORT_RST_MASK                                                        0x00000008L
+#define SHUB_SDP_PORT_RST__ATDMA_NBIFSOIN_SDP_PORT_RST_MASK                                                   0x00000010L
+#define SHUB_SDP_PORT_RST__INT_NBIFSION_SDP_PORT_RST_MASK                                                     0x00000020L
+#define SHUB_SDP_PORT_RST__MP4_SDP_SDP_PORT_RST_MASK                                                          0x00000040L
+#define SHUB_SDP_PORT_RST__GDC_HST_SDP_PORT_RST_MASK                                                          0x00000080L
+#define SHUB_SDP_PORT_RST__NTB_HST_SDP_PORT_RST_MASK                                                          0x00000100L
+#define SHUB_SDP_PORT_RST__NTB_DMA_SDP_PORT_RST_MASK                                                          0x00000200L
+#define SHUB_SDP_PORT_RST__DACCFE_NBIF_DMA_SDP_PORT_RST_MASK                                                  0x00000400L
+#define SHUB_SDP_PORT_RST__DACCBE_NBIF_DMA_SDP_PORT_RST_MASK                                                  0x00000800L
+#define SHUB_SDP_PORT_RST__DACCFE_HST_SDP_PORT_RST_MASK                                                       0x00001000L
+#define SHUB_SDP_PORT_RST__SHUB_CNDI_HST_SDP_PORT_RST_MASK                                                    0x00002000L
+#define SHUB_SDP_PORT_RST__HST_ATHUB_HST_SION_SDP_PORT_RST_MASK                                               0x00004000L
+#define SHUB_SDP_PORT_RST__SION_AON_RST_MASK                                                                  0x01000000L
+//SHUB_RST_MISC_TRL
+
+
+// addressBlock: nbif0_nbif0_gdc_s2a_GDCS2A_DEC
+//GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL__S2A_DOORBELL_PORT0_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_0_CTRL1__S2A_DOORBELL_PORT0_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL__S2A_DOORBELL_PORT1_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_1_CTRL1__S2A_DOORBELL_PORT1_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL__S2A_DOORBELL_PORT2_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_2_CTRL1__S2A_DOORBELL_PORT2_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL__S2A_DOORBELL_PORT3_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_3_CTRL1__S2A_DOORBELL_PORT3_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL__S2A_DOORBELL_PORT4_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_4_CTRL1__S2A_DOORBELL_PORT4_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL__S2A_DOORBELL_PORT5_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_5_CTRL1__S2A_DOORBELL_PORT5_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL__S2A_DOORBELL_PORT6_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_6_CTRL1__S2A_DOORBELL_PORT6_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL__S2A_DOORBELL_PORT7_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_7_CTRL1__S2A_DOORBELL_PORT7_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL__S2A_DOORBELL_PORT8_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_8_CTRL1__S2A_DOORBELL_PORT8_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE__SHIFT                                  0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID__SHIFT                                    0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE__SHIFT                            0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET__SHIFT                            0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE__SHIFT                              0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS__SHIFT                       0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET__SHIFT                0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_DROP_EN__SHIFT                                 0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE__SHIFT                      0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_ENABLE_MASK                                    0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWID_MASK                                      0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_FENCE_ENABLE_MASK                              0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_OFFSET_MASK                              0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_RANGE_SIZE_MASK                                0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_64BIT_SUPPORT_DIS_MASK                         0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_NEED_DEDUCT_RANGE_OFFSET_MASK                  0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_DROP_EN_MASK                                   0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL__S2A_DOORBELL_PORT9_AWADDR_31_28_VALUE_MASK                        0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_PORT_TYPE__SHIFT                       0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_DIEID__SHIFT                           0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_PORT_ID__SHIFT                         0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_OVERRIDE_POISON_DATA__SHIFT                   0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_4K_PAGE_DECODE_DISABLE__SHIFT                 0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_PORT_TYPE_MASK                         0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_DIEID_MASK                             0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_TARGET_PORT_ID_MASK                           0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_OVERRIDE_POISON_DATA_MASK                     0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_9_CTRL1__S2A_DOORBELL_PORT9_4K_PAGE_DECODE_DISABLE_MASK                   0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL__S2A_DOORBELL_PORT10_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_10_CTRL1__S2A_DOORBELL_PORT10_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL__S2A_DOORBELL_PORT11_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_11_CTRL1__S2A_DOORBELL_PORT11_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL__S2A_DOORBELL_PORT12_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_12_CTRL1__S2A_DOORBELL_PORT12_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL__S2A_DOORBELL_PORT13_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_13_CTRL1__S2A_DOORBELL_PORT13_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL__S2A_DOORBELL_PORT14_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_14_CTRL1__S2A_DOORBELL_PORT14_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE__SHIFT                                0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID__SHIFT                                  0x1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE__SHIFT                          0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET__SHIFT                          0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE__SHIFT                            0x11
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS__SHIFT                     0x19
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET__SHIFT              0x1a
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_DROP_EN__SHIFT                               0x1b
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE__SHIFT                    0x1c
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_ENABLE_MASK                                  0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWID_MASK                                    0x0000003EL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_FENCE_ENABLE_MASK                            0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_OFFSET_MASK                            0x0001FF80L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_RANGE_SIZE_MASK                              0x01FE0000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_64BIT_SUPPORT_DIS_MASK                       0x02000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_NEED_DEDUCT_RANGE_OFFSET_MASK                0x04000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_DROP_EN_MASK                                 0x08000000L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL__S2A_DOORBELL_PORT15_AWADDR_31_28_VALUE_MASK                      0xF0000000L
+//GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_PORT_TYPE__SHIFT                     0x0
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_DIEID__SHIFT                         0x2
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_PORT_ID__SHIFT                       0x4
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_OVERRIDE_POISON_DATA__SHIFT                 0x6
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_4K_PAGE_DECODE_DISABLE__SHIFT               0x7
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_PORT_TYPE_MASK                       0x00000003L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_DIEID_MASK                           0x0000000CL
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_TARGET_PORT_ID_MASK                         0x00000030L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_OVERRIDE_POISON_DATA_MASK                   0x00000040L
+#define GDC_S2A1_S2A_DOORBELL_ENTRY_15_CTRL1__S2A_DOORBELL_PORT15_4K_PAGE_DECODE_DISABLE_MASK                 0x00000080L
+//GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG
+#define GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS__SHIFT                     0x0
+#define GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBE_FENCE_INTR_ENABLE__SHIFT                            0x1
+#define GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBELL_FENCE_ONCE_TRIGGER_DIS_MASK                       0x00000001L
+#define GDC_S2A1_S2A_DOORBELL_COMMON_CTRL_REG__S2A_DOORBE_FENCE_INTR_ENABLE_MASK                              0x00000002L
+//GDC_S2A1_NBIF_GFX_DOORBELL_STATUS
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT__SHIFT                                      0x0
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_EN__SHIFT                                     0x10
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_ST__SHIFT                                     0x18
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__NBIF_GFX_DOORBELL_SENT_MASK                                        0x0000FFFFL
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_EN_MASK                                       0x00010000L
+#define GDC_S2A1_NBIF_GFX_DOORBELL_STATUS__S2A_DOORBELL_ALL_CLR_ST_MASK                                       0x01000000L
+
+
+// addressBlock: nbif0_nbif0_gdc_misc_GDCMISC_DEC
+//GDC_DMA_SION_CRED_RST
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_BIF_CRED_RST__SHIFT                                               0x0
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_BIF_SDP_TX_DS_MSK__SHIFT                                          0x1
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_BIF_SDP_RX_DS_MSK__SHIFT                                          0x2
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_SYSHUB_CRED_RST__SHIFT                                            0x4
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_SYSHUB_SDP_TX_DS_MSK__SHIFT                                       0x5
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_SYSHUB_SDP_RX_DS_MSK__SHIFT                                       0x6
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_INT_CRED_RST__SHIFT                                               0x8
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_INT_SDP_TX_DS_MSK__SHIFT                                          0x9
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_INT_SDP_RX_DS_MSK__SHIFT                                          0xa
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_BIF_CRED_RST_MASK                                                 0x00000001L
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_BIF_SDP_TX_DS_MSK_MASK                                            0x00000002L
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_BIF_SDP_RX_DS_MSK_MASK                                            0x00000004L
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_SYSHUB_CRED_RST_MASK                                              0x00000010L
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_SYSHUB_SDP_TX_DS_MSK_MASK                                         0x00000020L
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_SYSHUB_SDP_RX_DS_MSK_MASK                                         0x00000040L
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_INT_CRED_RST_MASK                                                 0x00000100L
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_INT_SDP_TX_DS_MSK_MASK                                            0x00000200L
+#define GDC_DMA_SION_CRED_RST__GDC_DMA_SION_INT_SDP_RX_DS_MSK_MASK                                            0x00000400L
+//GDC_HST_SION_PER_SDP_PORT_RST
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_GSI_CRED_RST__SHIFT                                       0x0
+#define GDC_HST_SION_PER_SDP_PORT_RST__NBIF_GSI_SDP_PORT_RST__SHIFT                                           0x1
+#define GDC_HST_SION_PER_SDP_PORT_RST__NBIF_GSI_SDP_DISCON_DIS__SHIFT                                         0x2
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_GSI_SDP_TX_DS_MSK__SHIFT                                  0x3
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_GSI_SDP_RX_DS_MSK__SHIFT                                  0x4
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_HSTHB_DB0_CRED_RST__SHIFT                                 0x5
+#define GDC_HST_SION_PER_SDP_PORT_RST__NBIF_HSTHB_DB0_SDP_PORT_RST__SHIFT                                     0x6
+#define GDC_HST_SION_PER_SDP_PORT_RST__NBIF_HSTHB_DB0_SDP_DISCON_DIS__SHIFT                                   0x7
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_HSTHB_DB0_SDP_TX_DS_MSK__SHIFT                            0x8
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_HSTHB_DB0_SDP_RX_DS_MSK__SHIFT                            0x9
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_GSI_CRED_RST_MASK                                         0x00000001L
+#define GDC_HST_SION_PER_SDP_PORT_RST__NBIF_GSI_SDP_PORT_RST_MASK                                             0x00000002L
+#define GDC_HST_SION_PER_SDP_PORT_RST__NBIF_GSI_SDP_DISCON_DIS_MASK                                           0x00000004L
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_GSI_SDP_TX_DS_MSK_MASK                                    0x00000008L
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_GSI_SDP_RX_DS_MSK_MASK                                    0x00000010L
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_HSTHB_DB0_CRED_RST_MASK                                   0x00000020L
+#define GDC_HST_SION_PER_SDP_PORT_RST__NBIF_HSTHB_DB0_SDP_PORT_RST_MASK                                       0x00000040L
+#define GDC_HST_SION_PER_SDP_PORT_RST__NBIF_HSTHB_DB0_SDP_DISCON_DIS_MASK                                     0x00000080L
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_HSTHB_DB0_SDP_TX_DS_MSK_MASK                              0x00000100L
+#define GDC_HST_SION_PER_SDP_PORT_RST__GDC_HST_SION_HSTHB_DB0_SDP_RX_DS_MSK_MASK                              0x00000200L
+//GDC_HST_SION_PER_SDP_CNTL_REG1
+//GDC_HST_AXI_STALL_FOR_FLUSH
+#define GDC_HST_AXI_STALL_FOR_FLUSH__SHUB_HST_AXI_STALL_FOR_FLUSH_GFX__SHIFT                                  0x0
+#define GDC_HST_AXI_STALL_FOR_FLUSH__SHUB_HST_AXI_STALL_FOR_FLUSH_GFX_MASK                                    0x00000001L
+//SYSHUB_SDP_DEBUG_COUNTER_CTRL
+#define SYSHUB_SDP_DEBUG_COUNTER_CTRL__SHUB_SDP_DEBUG_COUNTER_EN__SHIFT                                       0x0
+#define SYSHUB_SDP_DEBUG_COUNTER_CTRL__SHUB_SDP_DEBUG_COUNTER_EN_MASK                                         0x00000001L
+//NBIF_PWRBRK_IN_PAD_CNTL
+#define NBIF_PWRBRK_IN_PAD_CNTL__PWRBRK_IN_PAD_CNTL_BIT0__SHIFT                                               0x0
+#define NBIF_PWRBRK_IN_PAD_CNTL__PWRBRK_IN_PAD_CNTL__SHIFT                                                    0x1
+#define NBIF_PWRBRK_IN_PAD_CNTL__PWRBRK_IN_PAD_CNTL_BIT0_MASK                                                 0x00000001L
+#define NBIF_PWRBRK_IN_PAD_CNTL__PWRBRK_IN_PAD_CNTL_MASK                                                      0x000000FEL
+//NBIF_PWRBRK_OUT_PAD_CNTL
+#define NBIF_PWRBRK_OUT_PAD_CNTL__PWRBRK_OUT_PAD_CNTL_BIT0__SHIFT                                             0x0
+#define NBIF_PWRBRK_OUT_PAD_CNTL__PWRBRK_OUT_PAD_CNTL__SHIFT                                                  0x1
+#define NBIF_PWRBRK_OUT_PAD_CNTL__PWRBRK_OUT_PAD_CNTL_BIT0_MASK                                               0x00000001L
+#define NBIF_PWRBRK_OUT_PAD_CNTL__PWRBRK_OUT_PAD_CNTL_MASK                                                    0x000000FEL
+//SHUB_OOB_CRUSH_DUMP_CTRL
+#define SHUB_OOB_CRUSH_DUMP_CTRL__REGS_SHUB_OOB_TRIGGER__SHIFT                                                0x0
+#define SHUB_OOB_CRUSH_DUMP_CTRL__REGS_SHUB_OOB_TRIGGER_MASK                                                  0x00000001L
+//SHUB_MP_ERREVENT_MASK_CTRL
+#define SHUB_MP_ERREVENT_MASK_CTRL__SHUB_MP_ERREVENT_MASK_EN__SHIFT                                           0x0
+#define SHUB_MP_ERREVENT_MASK_CTRL__MCA_FATAL_OUT_PROP_EN__SHIFT                                              0x1
+#define SHUB_MP_ERREVENT_MASK_CTRL__SHUB_MP_ERREVENT_MASK_EN_MASK                                             0x00000001L
+#define SHUB_MP_ERREVENT_MASK_CTRL__MCA_FATAL_OUT_PROP_EN_MASK                                                0x00000002L
+//SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT0_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x0
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT1_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x1
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT2_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x2
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT3_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x3
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT4_RAS_V1_GEN_ERREVENT_EN__SHIFT                         0x4
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT0_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000001L
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT1_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000002L
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT2_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000004L
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT3_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000008L
+#define SHUB_ERREREVNT_SIDEBAND_RAS_V1_CTRL__SHUB_PORT4_RAS_V1_GEN_ERREVENT_EN_MASK                           0x00000010L
+//SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT0_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x0
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT1_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x1
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT2_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x2
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT3_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x3
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT4_RECV_ERREVENT_GEN_ERREVENT_EN__SHIFT           0x4
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT0_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000001L
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT1_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000002L
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT2_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000004L
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT3_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000008L
+#define SHUB_ERREREVNT_SIDEBAND_RECV_ERREVENT_CTRL__SHUB_PORT4_RECV_ERREVENT_GEN_ERREVENT_EN_MASK             0x00000010L
+
+
+// addressBlock: nbif0_nbif0_gdc_sec_misc_GDCSEC_MISC_DEC
+//GDCSOC_SEC_DFV_POISON_INJ_DMACK0V3_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0V3_AXI__DMACK0V3_EGRESS_RDATA_POISON_INJ__SHIFT                       0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0V3_AXI__DMACK0V3_EGRESS_RDATA_POISON_INJ_MASK                         0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V3_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V3_AXI__DMACK0V3_EGRESS_RDATA_POISON_INJ_CNT__SHIFT               0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V3_AXI__DMACK0V3_EGRESS_RDATA_POISON_INJ_CNT_MASK                 0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V3_AXI
+//GDCSOC_SEC_DFV_POISON_INJ_DMACK0B_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0B_AXI__DMACK0B_EGRESS_RDATA_POISON_INJ__SHIFT                         0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0B_AXI__DMACK0B_EGRESS_RDATA_POISON_INJ_MASK                           0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B_AXI__DMACK0B_EGRESS_RDATA_POISON_INJ_CNT__SHIFT                 0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B_AXI__DMACK0B_EGRESS_RDATA_POISON_INJ_CNT_MASK                   0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0B_AXI
+//GDCSOC_SEC_DFV_POISON_INJ_DMACK0V4_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0V4_AXI__DMACK0V4_EGRESS_RDATA_POISON_INJ__SHIFT                       0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0V4_AXI__DMACK0V4_EGRESS_RDATA_POISON_INJ_MASK                         0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4_AXI__DMACK0V4_EGRESS_RDATA_POISON_INJ_CNT__SHIFT               0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4_AXI__DMACK0V4_EGRESS_RDATA_POISON_INJ_CNT_MASK                 0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V4_AXI
+//GDCSOC_SEC_DFV_POISON_INJ_DMACK0V4B_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0V4B_AXI__DMACK0V4B_EGRESS_RDATA_POISON_INJ__SHIFT                     0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0V4B_AXI__DMACK0V4B_EGRESS_RDATA_POISON_INJ_MASK                       0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4B_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4B_AXI__DMACK0V4B_EGRESS_RDATA_POISON_INJ_CNT__SHIFT             0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V4B_AXI__DMACK0V4B_EGRESS_RDATA_POISON_INJ_CNT_MASK               0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V4B_AXI
+//GDCSOC_SEC_DFV_POISON_INJ_DMACK0V7_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0V7_AXI__DMACK0V7_EGRESS_RDATA_POISON_INJ__SHIFT                       0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0V7_AXI__DMACK0V7_EGRESS_RDATA_POISON_INJ_MASK                         0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V7_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V7_AXI__DMACK0V7_EGRESS_RDATA_POISON_INJ_CNT__SHIFT               0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0V7_AXI__DMACK0V7_EGRESS_RDATA_POISON_INJ_CNT_MASK                 0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0V7_AXI
+//GDCSOC_SEC_DFV_POISON_INJ_SHUB_CNDI_ORIG_SDP
+#define GDCSOC_SEC_DFV_POISON_INJ_SHUB_CNDI_ORIG_SDP__SHUB_CNDI_ORIG_EGRESS_ORIGDATA_POISON_INJ__SHIFT        0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_SHUB_CNDI_ORIG_SDP__SHUB_CNDI_ORIG_EGRESS_ORIGDATA_PARITY_INJ__SHIFT        0xc
+#define GDCSOC_SEC_DFV_POISON_INJ_SHUB_CNDI_ORIG_SDP__SHUB_CNDI_ORIG_EGRESS_ORIGDATA_POISON_INJ_MASK          0x00000F00L
+#define GDCSOC_SEC_DFV_POISON_INJ_SHUB_CNDI_ORIG_SDP__SHUB_CNDI_ORIG_EGRESS_ORIGDATA_PARITY_INJ_MASK          0x0000F000L
+//GDCSOC_SEC_DFV_POISON_INJ_CNT_SHUB_CNDI_ORIG_SDP
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_SHUB_CNDI_ORIG_SDP__SHUB_CNDI_ORIG_EGRESS_ORIGDATA_POISON_INJ_CNT__SHIFT  0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_SHUB_CNDI_ORIG_SDP__SHUB_CNDI_ORIG_EGRESS_ORIGDATA_PARITY_INJ_CNT__SHIFT  0xc
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_SHUB_CNDI_ORIG_SDP__SHUB_CNDI_ORIG_EGRESS_ORIGDATA_POISON_INJ_CNT_MASK  0x00000F00L
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_SHUB_CNDI_ORIG_SDP__SHUB_CNDI_ORIG_EGRESS_ORIGDATA_PARITY_INJ_CNT_MASK  0x0000F000L
+//GDCSOC_SEC_DFV_POISON_INJ_LOG_SHUB_CNDI_ORIG_SDP
+//GDCSOC_SEC_DFV_POISON_INJ_DMACK0B1_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0B1_AXI__DMACK0B1_EGRESS_RDATA_POISON_INJ__SHIFT                       0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_DMACK0B1_AXI__DMACK0B1_EGRESS_RDATA_POISON_INJ_MASK                         0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B1_AXI
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B1_AXI__DMACK0B1_EGRESS_RDATA_POISON_INJ_CNT__SHIFT               0x8
+#define GDCSOC_SEC_DFV_POISON_INJ_CNT_DMACK0B1_AXI__DMACK0B1_EGRESS_RDATA_POISON_INJ_CNT_MASK                 0x00000F00L
+//GDCSOC_SEC_DFV_POISON_INJ_LOG_DMACK0B1_AXI
+//GDCSOC_SEC_RAS_POISON_DBUG_CTRL
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF0_CTRL_POISON_DBUG_EN__SHIFT                      0x0
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF1_CTRL_POISON_DBUG_EN__SHIFT                      0x1
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF2_CTRL_POISON_DBUG_EN__SHIFT                      0x2
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF3_CTRL_POISON_DBUG_EN__SHIFT                      0x3
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF4_CTRL_POISON_DBUG_EN__SHIFT                      0x4
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF5_CTRL_POISON_DBUG_EN__SHIFT                      0x5
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF6_CTRL_POISON_DBUG_EN__SHIFT                      0x6
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF7_CTRL_POISON_DBUG_EN__SHIFT                      0x7
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF8_CTRL_POISON_DBUG_EN__SHIFT                      0x8
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF9_CTRL_POISON_DBUG_EN__SHIFT                      0x9
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF10_CTRL_POISON_DBUG_EN__SHIFT                     0xa
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF11_CTRL_POISON_DBUG_EN__SHIFT                     0xb
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF12_CTRL_POISON_DBUG_EN__SHIFT                     0xc
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF13_CTRL_POISON_DBUG_EN__SHIFT                     0xd
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF0_CTRL_POISON_DBUG_EN_MASK                        0x00000001L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF1_CTRL_POISON_DBUG_EN_MASK                        0x00000002L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF2_CTRL_POISON_DBUG_EN_MASK                        0x00000004L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF3_CTRL_POISON_DBUG_EN_MASK                        0x00000008L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF4_CTRL_POISON_DBUG_EN_MASK                        0x00000010L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF5_CTRL_POISON_DBUG_EN_MASK                        0x00000020L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF6_CTRL_POISON_DBUG_EN_MASK                        0x00000040L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF7_CTRL_POISON_DBUG_EN_MASK                        0x00000080L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF8_CTRL_POISON_DBUG_EN_MASK                        0x00000100L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF9_CTRL_POISON_DBUG_EN_MASK                        0x00000200L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF10_CTRL_POISON_DBUG_EN_MASK                       0x00000400L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF11_CTRL_POISON_DBUG_EN_MASK                       0x00000800L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF12_CTRL_POISON_DBUG_EN_MASK                       0x00001000L
+#define GDCSOC_SEC_RAS_POISON_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF13_CTRL_POISON_DBUG_EN_MASK                       0x00002000L
+//GDCSOC_SEC_RAS_PARITY_DBUG_CTRL
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF0_CTRL_PARITY_DBUG_EN__SHIFT                      0x0
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF1_CTRL_PARITY_DBUG_EN__SHIFT                      0x1
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF2_CTRL_PARITY_DBUG_EN__SHIFT                      0x2
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF3_CTRL_PARITY_DBUG_EN__SHIFT                      0x3
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF4_CTRL_PARITY_DBUG_EN__SHIFT                      0x4
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF5_CTRL_PARITY_DBUG_EN__SHIFT                      0x5
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF6_CTRL_PARITY_DBUG_EN__SHIFT                      0x6
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF7_CTRL_PARITY_DBUG_EN__SHIFT                      0x7
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF8_CTRL_PARITY_DBUG_EN__SHIFT                      0x8
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF9_CTRL_PARITY_DBUG_EN__SHIFT                      0x9
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF10_CTRL_PARITY_DBUG_EN__SHIFT                     0xa
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF11_CTRL_PARITY_DBUG_EN__SHIFT                     0xb
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF12_CTRL_PARITY_DBUG_EN__SHIFT                     0xc
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF13_CTRL_PARITY_DBUG_EN__SHIFT                     0xd
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF0_CTRL_PARITY_DBUG_EN_MASK                        0x00000001L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF1_CTRL_PARITY_DBUG_EN_MASK                        0x00000002L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF2_CTRL_PARITY_DBUG_EN_MASK                        0x00000004L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF3_CTRL_PARITY_DBUG_EN_MASK                        0x00000008L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF4_CTRL_PARITY_DBUG_EN_MASK                        0x00000010L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF5_CTRL_PARITY_DBUG_EN_MASK                        0x00000020L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF6_CTRL_PARITY_DBUG_EN_MASK                        0x00000040L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF7_CTRL_PARITY_DBUG_EN_MASK                        0x00000080L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF8_CTRL_PARITY_DBUG_EN_MASK                        0x00000100L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF9_CTRL_PARITY_DBUG_EN_MASK                        0x00000200L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF10_CTRL_PARITY_DBUG_EN_MASK                       0x00000400L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF11_CTRL_PARITY_DBUG_EN_MASK                       0x00000800L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF12_CTRL_PARITY_DBUG_EN_MASK                       0x00001000L
+#define GDCSOC_SEC_RAS_PARITY_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF13_CTRL_PARITY_DBUG_EN_MASK                       0x00002000L
+//GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF0_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x0
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF1_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x1
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF2_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x2
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF3_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x3
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF4_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x4
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF5_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x5
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF6_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x6
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF7_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x7
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF8_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x8
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF9_CTRL_RCVERREVENT_DBUG_EN__SHIFT            0x9
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF10_CTRL_RCVERREVENT_DBUG_EN__SHIFT           0xa
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF11_CTRL_RCVERREVENT_DBUG_EN__SHIFT           0xb
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF12_CTRL_RCVERREVENT_DBUG_EN__SHIFT           0xc
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF13_CTRL_RCVERREVENT_DBUG_EN__SHIFT           0xd
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF0_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000001L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF1_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000002L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF2_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000004L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF3_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000008L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF4_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000010L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF5_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000020L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF6_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000040L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF7_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000080L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF8_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000100L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF9_CTRL_RCVERREVENT_DBUG_EN_MASK              0x00000200L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF10_CTRL_RCVERREVENT_DBUG_EN_MASK             0x00000400L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF11_CTRL_RCVERREVENT_DBUG_EN_MASK             0x00000800L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF12_CTRL_RCVERREVENT_DBUG_EN_MASK             0x00001000L
+#define GDCSOC_SEC_RAS_RCVERREVENT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF13_CTRL_RCVERREVENT_DBUG_EN_MASK             0x00002000L
+//GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF0_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x0
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF1_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x1
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF2_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x2
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF3_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x3
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF4_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x4
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF5_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x5
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF6_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x6
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF7_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x7
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF8_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x8
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF9_CTRL_TIMEOUT_DBUG_EN__SHIFT                    0x9
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF10_CTRL_TIMEOUT_DBUG_EN__SHIFT                   0xa
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF11_CTRL_TIMEOUT_DBUG_EN__SHIFT                   0xb
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF12_CTRL_TIMEOUT_DBUG_EN__SHIFT                   0xc
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF13_CTRL_TIMEOUT_DBUG_EN__SHIFT                   0xd
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF0_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000001L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF1_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000002L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF2_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000004L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF3_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000008L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF4_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000010L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF5_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000020L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF6_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000040L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF7_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000080L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF8_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000100L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF9_CTRL_TIMEOUT_DBUG_EN_MASK                      0x00000200L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF10_CTRL_TIMEOUT_DBUG_EN_MASK                     0x00000400L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF11_CTRL_TIMEOUT_DBUG_EN_MASK                     0x00000800L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF12_CTRL_TIMEOUT_DBUG_EN_MASK                     0x00001000L
+#define GDCSOC_SEC_RAS_TIMEOUT_DBUG_CTRL__GDCSOC_SEC_RAS_LEAF13_CTRL_TIMEOUT_DBUG_EN_MASK                     0x00002000L
+//GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF0_POISON_DATA_DROP_EN__SHIFT                 0x0
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF1_POISON_DATA_DROP_EN__SHIFT                 0x1
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF2_POISON_DATA_DROP_EN__SHIFT                 0x2
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF3_POISON_DATA_DROP_EN__SHIFT                 0x3
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF4_POISON_DATA_DROP_EN__SHIFT                 0x4
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF5_POISON_DATA_DROP_EN__SHIFT                 0x5
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF6_POISON_DATA_DROP_EN__SHIFT                 0x6
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF7_POISON_DATA_DROP_EN__SHIFT                 0x7
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF8_POISON_DATA_DROP_EN__SHIFT                 0x8
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF9_POISON_DATA_DROP_EN__SHIFT                 0x9
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF10_POISON_DATA_DROP_EN__SHIFT                0xa
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF11_POISON_DATA_DROP_EN__SHIFT                0xb
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF12_POISON_DATA_DROP_EN__SHIFT                0xc
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF13_POISON_DATA_DROP_EN__SHIFT                0xd
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF0_POISON_DATA_DROP_EN_MASK                   0x00000001L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF1_POISON_DATA_DROP_EN_MASK                   0x00000002L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF2_POISON_DATA_DROP_EN_MASK                   0x00000004L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF3_POISON_DATA_DROP_EN_MASK                   0x00000008L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF4_POISON_DATA_DROP_EN_MASK                   0x00000010L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF5_POISON_DATA_DROP_EN_MASK                   0x00000020L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF6_POISON_DATA_DROP_EN_MASK                   0x00000040L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF7_POISON_DATA_DROP_EN_MASK                   0x00000080L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF8_POISON_DATA_DROP_EN_MASK                   0x00000100L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF9_POISON_DATA_DROP_EN_MASK                   0x00000200L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF10_POISON_DATA_DROP_EN_MASK                  0x00000400L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF11_POISON_DATA_DROP_EN_MASK                  0x00000800L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF12_POISON_DATA_DROP_EN_MASK                  0x00001000L
+#define GDCSOC_SEC_RAS_POISON_DATA_DROP_CTRL__GDCSOC_SEC_RAS_LEAF13_POISON_DATA_DROP_EN_MASK                  0x00002000L
+//GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF0_RESPONSE_DROP_EN__SHIFT                       0x0
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF1_RESPONSE_DROP_EN__SHIFT                       0x1
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF2_RESPONSE_DROP_EN__SHIFT                       0x2
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF3_RESPONSE_DROP_EN__SHIFT                       0x3
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF4_RESPONSE_DROP_EN__SHIFT                       0x4
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF5_RESPONSE_DROP_EN__SHIFT                       0x5
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF6_RESPONSE_DROP_EN__SHIFT                       0x6
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF7_RESPONSE_DROP_EN__SHIFT                       0x7
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF8_RESPONSE_DROP_EN__SHIFT                       0x8
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF9_RESPONSE_DROP_EN__SHIFT                       0x9
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF10_RESPONSE_DROP_EN__SHIFT                      0xa
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF11_RESPONSE_DROP_EN__SHIFT                      0xb
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF12_RESPONSE_DROP_EN__SHIFT                      0xc
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF13_RESPONSE_DROP_EN__SHIFT                      0xd
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF0_RESPONSE_DROP_EN_MASK                         0x00000001L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF1_RESPONSE_DROP_EN_MASK                         0x00000002L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF2_RESPONSE_DROP_EN_MASK                         0x00000004L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF3_RESPONSE_DROP_EN_MASK                         0x00000008L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF4_RESPONSE_DROP_EN_MASK                         0x00000010L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF5_RESPONSE_DROP_EN_MASK                         0x00000020L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF6_RESPONSE_DROP_EN_MASK                         0x00000040L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF7_RESPONSE_DROP_EN_MASK                         0x00000080L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF8_RESPONSE_DROP_EN_MASK                         0x00000100L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF9_RESPONSE_DROP_EN_MASK                         0x00000200L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF10_RESPONSE_DROP_EN_MASK                        0x00000400L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF11_RESPONSE_DROP_EN_MASK                        0x00000800L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF12_RESPONSE_DROP_EN_MASK                        0x00001000L
+#define GDCSOC_SEC_RAS_RESPONSE_DROP_CTRL__GDCSOC_SEC_RAS_LEAF13_RESPONSE_DROP_EN_MASK                        0x00002000L
+//GDCSOC_SEC_RAS_LEAF0_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF0_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF0_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF0_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF0_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF0_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF0_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF0_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF0_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF0_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF1_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF1_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF1_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF1_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF1_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF1_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF1_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF1_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF1_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF1_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF2_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF2_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF2_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF2_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF2_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF2_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF2_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF2_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF2_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF2_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF3_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF3_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF3_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF3_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF3_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF3_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF3_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF3_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF3_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF3_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF4_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF4_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF4_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF4_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF4_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF4_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF4_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF4_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF4_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF4_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF5_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF5_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF5_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF5_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF5_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF5_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF5_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF5_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF5_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF5_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF6_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF6_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF6_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF6_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF6_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF6_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF6_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF6_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF6_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF6_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF7_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF7_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF7_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF7_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF7_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF7_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF7_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF7_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF7_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF7_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF8_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF8_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF8_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF8_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF8_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF8_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF8_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF8_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF8_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF8_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF9_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF9_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF9_RRESP_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF9_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF9_RRESP_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF9_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF9_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF9_RUSER_POISON_EN__SHIFT                   0x0
+#define GDCSOC_SEC_RAS_LEAF9_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF9_RUSER_POISON_EN_MASK                     0x000000FFL
+//GDCSOC_SEC_RAS_LEAF10_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF10_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF10_RRESP_POISON_EN__SHIFT                 0x0
+#define GDCSOC_SEC_RAS_LEAF10_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF10_RRESP_POISON_EN_MASK                   0x000000FFL
+//GDCSOC_SEC_RAS_LEAF10_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF10_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF10_RUSER_POISON_EN__SHIFT                 0x0
+#define GDCSOC_SEC_RAS_LEAF10_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF10_RUSER_POISON_EN_MASK                   0x000000FFL
+//GDCSOC_SEC_RAS_LEAF11_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF11_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF11_RRESP_POISON_EN__SHIFT                 0x0
+#define GDCSOC_SEC_RAS_LEAF11_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF11_RRESP_POISON_EN_MASK                   0x000000FFL
+//GDCSOC_SEC_RAS_LEAF11_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF11_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF11_RUSER_POISON_EN__SHIFT                 0x0
+#define GDCSOC_SEC_RAS_LEAF11_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF11_RUSER_POISON_EN_MASK                   0x000000FFL
+//GDCSOC_SEC_RAS_LEAF12_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF12_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF12_RRESP_POISON_EN__SHIFT                 0x0
+#define GDCSOC_SEC_RAS_LEAF12_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF12_RRESP_POISON_EN_MASK                   0x000000FFL
+//GDCSOC_SEC_RAS_LEAF12_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF12_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF12_RUSER_POISON_EN__SHIFT                 0x0
+#define GDCSOC_SEC_RAS_LEAF12_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF12_RUSER_POISON_EN_MASK                   0x000000FFL
+//GDCSOC_SEC_RAS_LEAF13_RRESP_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF13_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF13_RRESP_POISON_EN__SHIFT                 0x0
+#define GDCSOC_SEC_RAS_LEAF13_RRESP_POISON_CTRL__GDCSOC_SEC_RAS_LEAF13_RRESP_POISON_EN_MASK                   0x000000FFL
+//GDCSOC_SEC_RAS_LEAF13_RUSER_POISON_CTRL
+#define GDCSOC_SEC_RAS_LEAF13_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF13_RUSER_POISON_EN__SHIFT                 0x0
+#define GDCSOC_SEC_RAS_LEAF13_RUSER_POISON_CTRL__GDCSOC_SEC_RAS_LEAF13_RUSER_POISON_EN_MASK                   0x000000FFL
+
+
+// addressBlock: nbif0_nbif0_gdc_a2s_GDCA2S_DEC
+//A2S_CNTL_SW0
+#define A2S_CNTL_SW0__STATIC_VC_ENABLE__SHIFT                                                                 0x0
+#define A2S_CNTL_SW0__STATIC_VC_VALUE__SHIFT                                                                  0x1
+#define A2S_CNTL_SW0__SDP_WR_CHAIN_DIS__SHIFT                                                                 0x9
+#define A2S_CNTL_SW0__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                          0xa
+#define A2S_CNTL_SW0__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                       0xb
+#define A2S_CNTL_SW0__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                      0xc
+#define A2S_CNTL_SW0__WRR_RD_WEIGHT__SHIFT                                                                    0x10
+#define A2S_CNTL_SW0__WRR_WR_WEIGHT__SHIFT                                                                    0x18
+#define A2S_CNTL_SW0__STATIC_VC_ENABLE_MASK                                                                   0x00000001L
+#define A2S_CNTL_SW0__STATIC_VC_VALUE_MASK                                                                    0x0000000EL
+#define A2S_CNTL_SW0__SDP_WR_CHAIN_DIS_MASK                                                                   0x00000200L
+#define A2S_CNTL_SW0__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                            0x00000400L
+#define A2S_CNTL_SW0__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                         0x00000800L
+#define A2S_CNTL_SW0__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                        0x00001000L
+#define A2S_CNTL_SW0__WRR_RD_WEIGHT_MASK                                                                      0x00FF0000L
+#define A2S_CNTL_SW0__WRR_WR_WEIGHT_MASK                                                                      0xFF000000L
+//A2S_CNTL_SW1
+#define A2S_CNTL_SW1__STATIC_VC_ENABLE__SHIFT                                                                 0x0
+#define A2S_CNTL_SW1__STATIC_VC_VALUE__SHIFT                                                                  0x1
+#define A2S_CNTL_SW1__SDP_WR_CHAIN_DIS__SHIFT                                                                 0x9
+#define A2S_CNTL_SW1__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                          0xa
+#define A2S_CNTL_SW1__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                       0xb
+#define A2S_CNTL_SW1__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                      0xc
+#define A2S_CNTL_SW1__WRR_RD_WEIGHT__SHIFT                                                                    0x10
+#define A2S_CNTL_SW1__WRR_WR_WEIGHT__SHIFT                                                                    0x18
+#define A2S_CNTL_SW1__STATIC_VC_ENABLE_MASK                                                                   0x00000001L
+#define A2S_CNTL_SW1__STATIC_VC_VALUE_MASK                                                                    0x0000000EL
+#define A2S_CNTL_SW1__SDP_WR_CHAIN_DIS_MASK                                                                   0x00000200L
+#define A2S_CNTL_SW1__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                            0x00000400L
+#define A2S_CNTL_SW1__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                         0x00000800L
+#define A2S_CNTL_SW1__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                        0x00001000L
+#define A2S_CNTL_SW1__WRR_RD_WEIGHT_MASK                                                                      0x00FF0000L
+#define A2S_CNTL_SW1__WRR_WR_WEIGHT_MASK                                                                      0xFF000000L
+//A2S_CNTL_SW2
+#define A2S_CNTL_SW2__STATIC_VC_ENABLE__SHIFT                                                                 0x0
+#define A2S_CNTL_SW2__STATIC_VC_VALUE__SHIFT                                                                  0x1
+#define A2S_CNTL_SW2__SDP_WR_CHAIN_DIS__SHIFT                                                                 0x9
+#define A2S_CNTL_SW2__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                          0xa
+#define A2S_CNTL_SW2__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                       0xb
+#define A2S_CNTL_SW2__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                      0xc
+#define A2S_CNTL_SW2__WRR_RD_WEIGHT__SHIFT                                                                    0x10
+#define A2S_CNTL_SW2__WRR_WR_WEIGHT__SHIFT                                                                    0x18
+#define A2S_CNTL_SW2__STATIC_VC_ENABLE_MASK                                                                   0x00000001L
+#define A2S_CNTL_SW2__STATIC_VC_VALUE_MASK                                                                    0x0000000EL
+#define A2S_CNTL_SW2__SDP_WR_CHAIN_DIS_MASK                                                                   0x00000200L
+#define A2S_CNTL_SW2__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                            0x00000400L
+#define A2S_CNTL_SW2__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                         0x00000800L
+#define A2S_CNTL_SW2__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                        0x00001000L
+#define A2S_CNTL_SW2__WRR_RD_WEIGHT_MASK                                                                      0x00FF0000L
+#define A2S_CNTL_SW2__WRR_WR_WEIGHT_MASK                                                                      0xFF000000L
+//A2S_CNTL_SW3
+#define A2S_CNTL_SW3__STATIC_VC_ENABLE__SHIFT                                                                 0x0
+#define A2S_CNTL_SW3__STATIC_VC_VALUE__SHIFT                                                                  0x1
+#define A2S_CNTL_SW3__SDP_WR_CHAIN_DIS__SHIFT                                                                 0x9
+#define A2S_CNTL_SW3__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                          0xa
+#define A2S_CNTL_SW3__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                       0xb
+#define A2S_CNTL_SW3__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                      0xc
+#define A2S_CNTL_SW3__WRR_RD_WEIGHT__SHIFT                                                                    0x10
+#define A2S_CNTL_SW3__WRR_WR_WEIGHT__SHIFT                                                                    0x18
+#define A2S_CNTL_SW3__STATIC_VC_ENABLE_MASK                                                                   0x00000001L
+#define A2S_CNTL_SW3__STATIC_VC_VALUE_MASK                                                                    0x0000000EL
+#define A2S_CNTL_SW3__SDP_WR_CHAIN_DIS_MASK                                                                   0x00000200L
+#define A2S_CNTL_SW3__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                            0x00000400L
+#define A2S_CNTL_SW3__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                         0x00000800L
+#define A2S_CNTL_SW3__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                        0x00001000L
+#define A2S_CNTL_SW3__WRR_RD_WEIGHT_MASK                                                                      0x00FF0000L
+#define A2S_CNTL_SW3__WRR_WR_WEIGHT_MASK                                                                      0xFF000000L
+//A2S_CNTL_SW4
+#define A2S_CNTL_SW4__STATIC_VC_ENABLE__SHIFT                                                                 0x0
+#define A2S_CNTL_SW4__STATIC_VC_VALUE__SHIFT                                                                  0x1
+#define A2S_CNTL_SW4__SDP_WR_CHAIN_DIS__SHIFT                                                                 0x9
+#define A2S_CNTL_SW4__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                          0xa
+#define A2S_CNTL_SW4__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                       0xb
+#define A2S_CNTL_SW4__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                      0xc
+#define A2S_CNTL_SW4__WRR_RD_WEIGHT__SHIFT                                                                    0x10
+#define A2S_CNTL_SW4__WRR_WR_WEIGHT__SHIFT                                                                    0x18
+#define A2S_CNTL_SW4__STATIC_VC_ENABLE_MASK                                                                   0x00000001L
+#define A2S_CNTL_SW4__STATIC_VC_VALUE_MASK                                                                    0x0000000EL
+#define A2S_CNTL_SW4__SDP_WR_CHAIN_DIS_MASK                                                                   0x00000200L
+#define A2S_CNTL_SW4__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                            0x00000400L
+#define A2S_CNTL_SW4__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                         0x00000800L
+#define A2S_CNTL_SW4__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                        0x00001000L
+#define A2S_CNTL_SW4__WRR_RD_WEIGHT_MASK                                                                      0x00FF0000L
+#define A2S_CNTL_SW4__WRR_WR_WEIGHT_MASK                                                                      0xFF000000L
+//A2S_CNTL_SW5
+#define A2S_CNTL_SW5__STATIC_VC_ENABLE__SHIFT                                                                 0x0
+#define A2S_CNTL_SW5__STATIC_VC_VALUE__SHIFT                                                                  0x1
+#define A2S_CNTL_SW5__SDP_WR_CHAIN_DIS__SHIFT                                                                 0x9
+#define A2S_CNTL_SW5__WR_TAG_FOR_CHAIN_ENABLE__SHIFT                                                          0xa
+#define A2S_CNTL_SW5__WR_TAG_FOR_NONCHAIN_ENABLE__SHIFT                                                       0xb
+#define A2S_CNTL_SW5__SDP_DYNAMIC_VC_WR_CHAIN_DIS__SHIFT                                                      0xc
+#define A2S_CNTL_SW5__WRR_RD_WEIGHT__SHIFT                                                                    0x10
+#define A2S_CNTL_SW5__WRR_WR_WEIGHT__SHIFT                                                                    0x18
+#define A2S_CNTL_SW5__STATIC_VC_ENABLE_MASK                                                                   0x00000001L
+#define A2S_CNTL_SW5__STATIC_VC_VALUE_MASK                                                                    0x0000000EL
+#define A2S_CNTL_SW5__SDP_WR_CHAIN_DIS_MASK                                                                   0x00000200L
+#define A2S_CNTL_SW5__WR_TAG_FOR_CHAIN_ENABLE_MASK                                                            0x00000400L
+#define A2S_CNTL_SW5__WR_TAG_FOR_NONCHAIN_ENABLE_MASK                                                         0x00000800L
+#define A2S_CNTL_SW5__SDP_DYNAMIC_VC_WR_CHAIN_DIS_MASK                                                        0x00001000L
+#define A2S_CNTL_SW5__WRR_RD_WEIGHT_MASK                                                                      0x00FF0000L
+#define A2S_CNTL_SW5__WRR_WR_WEIGHT_MASK                                                                      0xFF000000L
+//A2S_CNTL_2_SW0
+#define A2S_CNTL_2_SW0__SDP_LARGE_WR_CHAIN_DIS__SHIFT                                                         0x0
+#define A2S_CNTL_2_SW0__SDP_LARGE_WR_CHAIN_DIS_MASK                                                           0x00000001L
+//A2S_CNTL_2_SW1
+#define A2S_CNTL_2_SW1__SDP_LARGE_WR_CHAIN_DIS__SHIFT                                                         0x0
+#define A2S_CNTL_2_SW1__SDP_LARGE_WR_CHAIN_DIS_MASK                                                           0x00000001L
+//A2S_CNTL_2_SW2
+#define A2S_CNTL_2_SW2__SDP_LARGE_WR_CHAIN_DIS__SHIFT                                                         0x0
+#define A2S_CNTL_2_SW2__SDP_LARGE_WR_CHAIN_DIS_MASK                                                           0x00000001L
+//A2S_CNTL_2_SW3
+#define A2S_CNTL_2_SW3__SDP_LARGE_WR_CHAIN_DIS__SHIFT                                                         0x0
+#define A2S_CNTL_2_SW3__SDP_LARGE_WR_CHAIN_DIS_MASK                                                           0x00000001L
+//A2S_CNTL_2_SW4
+#define A2S_CNTL_2_SW4__SDP_LARGE_WR_CHAIN_DIS__SHIFT                                                         0x0
+#define A2S_CNTL_2_SW4__SDP_LARGE_WR_CHAIN_DIS_MASK                                                           0x00000001L
+//A2S_CNTL_2_SW5
+#define A2S_CNTL_2_SW5__SDP_LARGE_WR_CHAIN_DIS__SHIFT                                                         0x0
+#define A2S_CNTL_2_SW5__SDP_LARGE_WR_CHAIN_DIS_MASK                                                           0x00000001L
+//A2S_WRSIZEFULL_SUPPORT_DISABLE
+#define A2S_WRSIZEFULL_SUPPORT_DISABLE__SHUB_A2S_WRSIZEFULL_SUPPORT_DISABLE__SHIFT                            0x0
+#define A2S_WRSIZEFULL_SUPPORT_DISABLE__SHUB_A2S_WRSIZEFULL_SUPPORT_DISABLE_MASK                              0x00000001L
+//ROUTING_TABLE_CNTL
+#define ROUTING_TABLE_CNTL__TABLE_INDEX__SHIFT                                                                0x0
+#define ROUTING_TABLE_CNTL__TABLE_INDEX_MASK                                                                  0x0000FFFFL
+//ROUTING_TABLE_DATA0
+#define ROUTING_TABLE_DATA0__TABLE_DATA0__SHIFT                                                               0x0
+#define ROUTING_TABLE_DATA0__TABLE_DATA0_MASK                                                                 0xFFFFFFFFL
+//A2S_ARB_CONTROL_FOR_VC0
+#define A2S_ARB_CONTROL_FOR_VC0__VC_RD_WEIGHT__SHIFT                                                          0x0
+#define A2S_ARB_CONTROL_FOR_VC0__VC_WR_WEIGHT__SHIFT                                                          0x8
+#define A2S_ARB_CONTROL_FOR_VC0__VC_RW_WEIGHT__SHIFT                                                          0x10
+#define A2S_ARB_CONTROL_FOR_VC0__VC_LRG_MODE_RW__SHIFT                                                        0x18
+#define A2S_ARB_CONTROL_FOR_VC0__VC_LRG_SIZE_MODE_RW__SHIFT                                                   0x19
+#define A2S_ARB_CONTROL_FOR_VC0__VC_LRG_COUNTER_MODE_RW__SHIFT                                                0x1a
+#define A2S_ARB_CONTROL_FOR_VC0__VC_RD_WEIGHT_MASK                                                            0x000000FFL
+#define A2S_ARB_CONTROL_FOR_VC0__VC_WR_WEIGHT_MASK                                                            0x0000FF00L
+#define A2S_ARB_CONTROL_FOR_VC0__VC_RW_WEIGHT_MASK                                                            0x00FF0000L
+#define A2S_ARB_CONTROL_FOR_VC0__VC_LRG_MODE_RW_MASK                                                          0x01000000L
+#define A2S_ARB_CONTROL_FOR_VC0__VC_LRG_SIZE_MODE_RW_MASK                                                     0x02000000L
+#define A2S_ARB_CONTROL_FOR_VC0__VC_LRG_COUNTER_MODE_RW_MASK                                                  0x04000000L
+//A2S_ARB_CONTROL_FOR_VC1
+#define A2S_ARB_CONTROL_FOR_VC1__VC_RD_WEIGHT__SHIFT                                                          0x0
+#define A2S_ARB_CONTROL_FOR_VC1__VC_WR_WEIGHT__SHIFT                                                          0x8
+#define A2S_ARB_CONTROL_FOR_VC1__VC_RW_WEIGHT__SHIFT                                                          0x10
+#define A2S_ARB_CONTROL_FOR_VC1__VC_LRG_MODE_RW__SHIFT                                                        0x18
+#define A2S_ARB_CONTROL_FOR_VC1__VC_LRG_SIZE_MODE_RW__SHIFT                                                   0x19
+#define A2S_ARB_CONTROL_FOR_VC1__VC_LRG_COUNTER_MODE_RW__SHIFT                                                0x1a
+#define A2S_ARB_CONTROL_FOR_VC1__VC_RD_WEIGHT_MASK                                                            0x000000FFL
+#define A2S_ARB_CONTROL_FOR_VC1__VC_WR_WEIGHT_MASK                                                            0x0000FF00L
+#define A2S_ARB_CONTROL_FOR_VC1__VC_RW_WEIGHT_MASK                                                            0x00FF0000L
+#define A2S_ARB_CONTROL_FOR_VC1__VC_LRG_MODE_RW_MASK                                                          0x01000000L
+#define A2S_ARB_CONTROL_FOR_VC1__VC_LRG_SIZE_MODE_RW_MASK                                                     0x02000000L
+#define A2S_ARB_CONTROL_FOR_VC1__VC_LRG_COUNTER_MODE_RW_MASK                                                  0x04000000L
+//A2S_ARB_CONTROL_FOR_VC2
+#define A2S_ARB_CONTROL_FOR_VC2__VC_RD_WEIGHT__SHIFT                                                          0x0
+#define A2S_ARB_CONTROL_FOR_VC2__VC_WR_WEIGHT__SHIFT                                                          0x8
+#define A2S_ARB_CONTROL_FOR_VC2__VC_RW_WEIGHT__SHIFT                                                          0x10
+#define A2S_ARB_CONTROL_FOR_VC2__VC_LRG_MODE_RW__SHIFT                                                        0x18
+#define A2S_ARB_CONTROL_FOR_VC2__VC_LRG_SIZE_MODE_RW__SHIFT                                                   0x19
+#define A2S_ARB_CONTROL_FOR_VC2__VC_LRG_COUNTER_MODE_RW__SHIFT                                                0x1a
+#define A2S_ARB_CONTROL_FOR_VC2__VC_RD_WEIGHT_MASK                                                            0x000000FFL
+#define A2S_ARB_CONTROL_FOR_VC2__VC_WR_WEIGHT_MASK                                                            0x0000FF00L
+#define A2S_ARB_CONTROL_FOR_VC2__VC_RW_WEIGHT_MASK                                                            0x00FF0000L
+#define A2S_ARB_CONTROL_FOR_VC2__VC_LRG_MODE_RW_MASK                                                          0x01000000L
+#define A2S_ARB_CONTROL_FOR_VC2__VC_LRG_SIZE_MODE_RW_MASK                                                     0x02000000L
+#define A2S_ARB_CONTROL_FOR_VC2__VC_LRG_COUNTER_MODE_RW_MASK                                                  0x04000000L
+//A2S_ARB_CONTROL_FOR_VC3
+#define A2S_ARB_CONTROL_FOR_VC3__VC_RD_WEIGHT__SHIFT                                                          0x0
+#define A2S_ARB_CONTROL_FOR_VC3__VC_WR_WEIGHT__SHIFT                                                          0x8
+#define A2S_ARB_CONTROL_FOR_VC3__VC_RW_WEIGHT__SHIFT                                                          0x10
+#define A2S_ARB_CONTROL_FOR_VC3__VC_LRG_MODE_RW__SHIFT                                                        0x18
+#define A2S_ARB_CONTROL_FOR_VC3__VC_LRG_SIZE_MODE_RW__SHIFT                                                   0x19
+#define A2S_ARB_CONTROL_FOR_VC3__VC_LRG_COUNTER_MODE_RW__SHIFT                                                0x1a
+#define A2S_ARB_CONTROL_FOR_VC3__VC_RD_WEIGHT_MASK                                                            0x000000FFL
+#define A2S_ARB_CONTROL_FOR_VC3__VC_WR_WEIGHT_MASK                                                            0x0000FF00L
+#define A2S_ARB_CONTROL_FOR_VC3__VC_RW_WEIGHT_MASK                                                            0x00FF0000L
+#define A2S_ARB_CONTROL_FOR_VC3__VC_LRG_MODE_RW_MASK                                                          0x01000000L
+#define A2S_ARB_CONTROL_FOR_VC3__VC_LRG_SIZE_MODE_RW_MASK                                                     0x02000000L
+#define A2S_ARB_CONTROL_FOR_VC3__VC_LRG_COUNTER_MODE_RW_MASK                                                  0x04000000L
+//A2S_ARB_CONTROL_FOR_VC4
+#define A2S_ARB_CONTROL_FOR_VC4__VC_RD_WEIGHT__SHIFT                                                          0x0
+#define A2S_ARB_CONTROL_FOR_VC4__VC_WR_WEIGHT__SHIFT                                                          0x8
+#define A2S_ARB_CONTROL_FOR_VC4__VC_RW_WEIGHT__SHIFT                                                          0x10
+#define A2S_ARB_CONTROL_FOR_VC4__VC_LRG_MODE_RW__SHIFT                                                        0x18
+#define A2S_ARB_CONTROL_FOR_VC4__VC_LRG_SIZE_MODE_RW__SHIFT                                                   0x19
+#define A2S_ARB_CONTROL_FOR_VC4__VC_LRG_COUNTER_MODE_RW__SHIFT                                                0x1a
+#define A2S_ARB_CONTROL_FOR_VC4__VC_RD_WEIGHT_MASK                                                            0x000000FFL
+#define A2S_ARB_CONTROL_FOR_VC4__VC_WR_WEIGHT_MASK                                                            0x0000FF00L
+#define A2S_ARB_CONTROL_FOR_VC4__VC_RW_WEIGHT_MASK                                                            0x00FF0000L
+#define A2S_ARB_CONTROL_FOR_VC4__VC_LRG_MODE_RW_MASK                                                          0x01000000L
+#define A2S_ARB_CONTROL_FOR_VC4__VC_LRG_SIZE_MODE_RW_MASK                                                     0x02000000L
+#define A2S_ARB_CONTROL_FOR_VC4__VC_LRG_COUNTER_MODE_RW_MASK                                                  0x04000000L
+//A2S_ARB_CONTROL_FOR_VC5
+#define A2S_ARB_CONTROL_FOR_VC5__VC_RD_WEIGHT__SHIFT                                                          0x0
+#define A2S_ARB_CONTROL_FOR_VC5__VC_WR_WEIGHT__SHIFT                                                          0x8
+#define A2S_ARB_CONTROL_FOR_VC5__VC_RW_WEIGHT__SHIFT                                                          0x10
+#define A2S_ARB_CONTROL_FOR_VC5__VC_LRG_MODE_RW__SHIFT                                                        0x18
+#define A2S_ARB_CONTROL_FOR_VC5__VC_LRG_SIZE_MODE_RW__SHIFT                                                   0x19
+#define A2S_ARB_CONTROL_FOR_VC5__VC_LRG_COUNTER_MODE_RW__SHIFT                                                0x1a
+#define A2S_ARB_CONTROL_FOR_VC5__VC_RD_WEIGHT_MASK                                                            0x000000FFL
+#define A2S_ARB_CONTROL_FOR_VC5__VC_WR_WEIGHT_MASK                                                            0x0000FF00L
+#define A2S_ARB_CONTROL_FOR_VC5__VC_RW_WEIGHT_MASK                                                            0x00FF0000L
+#define A2S_ARB_CONTROL_FOR_VC5__VC_LRG_MODE_RW_MASK                                                          0x01000000L
+#define A2S_ARB_CONTROL_FOR_VC5__VC_LRG_SIZE_MODE_RW_MASK                                                     0x02000000L
+#define A2S_ARB_CONTROL_FOR_VC5__VC_LRG_COUNTER_MODE_RW_MASK                                                  0x04000000L
+//A2S_ARB_CONTROL_FOR_VC6
+#define A2S_ARB_CONTROL_FOR_VC6__VC_RD_WEIGHT__SHIFT                                                          0x0
+#define A2S_ARB_CONTROL_FOR_VC6__VC_WR_WEIGHT__SHIFT                                                          0x8
+#define A2S_ARB_CONTROL_FOR_VC6__VC_RW_WEIGHT__SHIFT                                                          0x10
+#define A2S_ARB_CONTROL_FOR_VC6__VC_LRG_MODE_RW__SHIFT                                                        0x18
+#define A2S_ARB_CONTROL_FOR_VC6__VC_LRG_SIZE_MODE_RW__SHIFT                                                   0x19
+#define A2S_ARB_CONTROL_FOR_VC6__VC_LRG_COUNTER_MODE_RW__SHIFT                                                0x1a
+#define A2S_ARB_CONTROL_FOR_VC6__VC_RD_WEIGHT_MASK                                                            0x000000FFL
+#define A2S_ARB_CONTROL_FOR_VC6__VC_WR_WEIGHT_MASK                                                            0x0000FF00L
+#define A2S_ARB_CONTROL_FOR_VC6__VC_RW_WEIGHT_MASK                                                            0x00FF0000L
+#define A2S_ARB_CONTROL_FOR_VC6__VC_LRG_MODE_RW_MASK                                                          0x01000000L
+#define A2S_ARB_CONTROL_FOR_VC6__VC_LRG_SIZE_MODE_RW_MASK                                                     0x02000000L
+#define A2S_ARB_CONTROL_FOR_VC6__VC_LRG_COUNTER_MODE_RW_MASK                                                  0x04000000L
+//A2S_ARB_CONTROL_FOR_VC7
+#define A2S_ARB_CONTROL_FOR_VC7__VC_RD_WEIGHT__SHIFT                                                          0x0
+#define A2S_ARB_CONTROL_FOR_VC7__VC_WR_WEIGHT__SHIFT                                                          0x8
+#define A2S_ARB_CONTROL_FOR_VC7__VC_RW_WEIGHT__SHIFT                                                          0x10
+#define A2S_ARB_CONTROL_FOR_VC7__VC_LRG_MODE_RW__SHIFT                                                        0x18
+#define A2S_ARB_CONTROL_FOR_VC7__VC_LRG_SIZE_MODE_RW__SHIFT                                                   0x19
+#define A2S_ARB_CONTROL_FOR_VC7__VC_LRG_COUNTER_MODE_RW__SHIFT                                                0x1a
+#define A2S_ARB_CONTROL_FOR_VC7__VC_RD_WEIGHT_MASK                                                            0x000000FFL
+#define A2S_ARB_CONTROL_FOR_VC7__VC_WR_WEIGHT_MASK                                                            0x0000FF00L
+#define A2S_ARB_CONTROL_FOR_VC7__VC_RW_WEIGHT_MASK                                                            0x00FF0000L
+#define A2S_ARB_CONTROL_FOR_VC7__VC_LRG_MODE_RW_MASK                                                          0x01000000L
+#define A2S_ARB_CONTROL_FOR_VC7__VC_LRG_SIZE_MODE_RW_MASK                                                     0x02000000L
+#define A2S_ARB_CONTROL_FOR_VC7__VC_LRG_COUNTER_MODE_RW_MASK                                                  0x04000000L
+//A2S_RW_ARB_CNTL
+#define A2S_RW_ARB_CNTL__A2S_WRR_WEIGHT_RD__SHIFT                                                             0x0
+#define A2S_RW_ARB_CNTL__A2S_WRR_WEIGHT_WR__SHIFT                                                             0x8
+#define A2S_RW_ARB_CNTL__A2S_LRG_MODE_RW__SHIFT                                                               0x10
+#define A2S_RW_ARB_CNTL__A2S_LRG_SIZE_MODE_RW__SHIFT                                                          0x11
+#define A2S_RW_ARB_CNTL__A2S_LRG_COUNTER_MODE_RW__SHIFT                                                       0x12
+#define A2S_RW_ARB_CNTL__A2S_WRR_WEIGHT_RD_MASK                                                               0x000000FFL
+#define A2S_RW_ARB_CNTL__A2S_WRR_WEIGHT_WR_MASK                                                               0x0000FF00L
+#define A2S_RW_ARB_CNTL__A2S_LRG_MODE_RW_MASK                                                                 0x00010000L
+#define A2S_RW_ARB_CNTL__A2S_LRG_SIZE_MODE_RW_MASK                                                            0x00020000L
+#define A2S_RW_ARB_CNTL__A2S_LRG_COUNTER_MODE_RW_MASK                                                         0x00040000L
+//A2S_REQ_RSP_TAG_CNTL
+#define A2S_REQ_RSP_TAG_CNTL__RD_WRR_LRG_MODE__SHIFT                                                          0x0
+#define A2S_REQ_RSP_TAG_CNTL__RD_WRR_LRG_SIZE_MODE__SHIFT                                                     0x1
+#define A2S_REQ_RSP_TAG_CNTL__RD_WRR_LRG_COUNTER_MODE__SHIFT                                                  0x2
+#define A2S_REQ_RSP_TAG_CNTL__RD_A2S_RSP_ARBMODE__SHIFT                                                       0x3
+#define A2S_REQ_RSP_TAG_CNTL__WR_WRR_LRG_MODE__SHIFT                                                          0x10
+#define A2S_REQ_RSP_TAG_CNTL__WR_WRR_LRG_SIZE_MODE__SHIFT                                                     0x11
+#define A2S_REQ_RSP_TAG_CNTL__WR_WRR_LRG_COUNTER_MODE__SHIFT                                                  0x12
+#define A2S_REQ_RSP_TAG_CNTL__WR_A2S_RSP_ARBMODE__SHIFT                                                       0x13
+#define A2S_REQ_RSP_TAG_CNTL__RD_WRR_LRG_MODE_MASK                                                            0x00000001L
+#define A2S_REQ_RSP_TAG_CNTL__RD_WRR_LRG_SIZE_MODE_MASK                                                       0x00000002L
+#define A2S_REQ_RSP_TAG_CNTL__RD_WRR_LRG_COUNTER_MODE_MASK                                                    0x00000004L
+#define A2S_REQ_RSP_TAG_CNTL__RD_A2S_RSP_ARBMODE_MASK                                                         0x00000008L
+#define A2S_REQ_RSP_TAG_CNTL__WR_WRR_LRG_MODE_MASK                                                            0x00010000L
+#define A2S_REQ_RSP_TAG_CNTL__WR_WRR_LRG_SIZE_MODE_MASK                                                       0x00020000L
+#define A2S_REQ_RSP_TAG_CNTL__WR_WRR_LRG_COUNTER_MODE_MASK                                                    0x00040000L
+#define A2S_REQ_RSP_TAG_CNTL__WR_A2S_RSP_ARBMODE_MASK                                                         0x00080000L
+//A2S_MISC_CNTL
+#define A2S_MISC_CNTL__BLKLVL_FOR_MSG__SHIFT                                                                  0x0
+#define A2S_MISC_CNTL__WRRSP_ACCUM_SEL__SHIFT                                                                 0x6
+#define A2S_MISC_CNTL__BLKLVL_FOR_MSG_MASK                                                                    0x00000003L
+#define A2S_MISC_CNTL__WRRSP_ACCUM_SEL_MASK                                                                   0x00000040L
+//A2S_TAG_ALLOC_0
+#define A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_WR__SHIFT                                                          0x0
+#define A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_RD__SHIFT                                                          0x8
+#define A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC1_WR__SHIFT                                                          0x10
+#define A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_WR_MASK                                                            0x000000FFL
+#define A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC0_RD_MASK                                                            0x0000FF00L
+#define A2S_TAG_ALLOC_0__TAG_ALLOC_FOR_VC1_WR_MASK                                                            0x00FF0000L
+//A2S_TAG_ALLOC_1
+#define A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC3_WR__SHIFT                                                          0x0
+#define A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_WR__SHIFT                                                          0x10
+#define A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_RD__SHIFT                                                          0x18
+#define A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC3_WR_MASK                                                            0x000000FFL
+#define A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_WR_MASK                                                            0x00FF0000L
+#define A2S_TAG_ALLOC_1__TAG_ALLOC_FOR_VC7_RD_MASK                                                            0xFF000000L
+//A2S_TAG_ALLOC_2
+#define A2S_TAG_ALLOC_2__TAG_ALLOC_FOR_VC5_WR__SHIFT                                                          0x0
+#define A2S_TAG_ALLOC_2__TAG_ALLOC_FOR_VC5_RD__SHIFT                                                          0x8
+#define A2S_TAG_ALLOC_2__TAG_ALLOC_FOR_VC6_WR__SHIFT                                                          0x10
+#define A2S_TAG_ALLOC_2__TAG_ALLOC_FOR_VC6_RD__SHIFT                                                          0x18
+#define A2S_TAG_ALLOC_2__TAG_ALLOC_FOR_VC5_WR_MASK                                                            0x000000FFL
+#define A2S_TAG_ALLOC_2__TAG_ALLOC_FOR_VC5_RD_MASK                                                            0x0000FF00L
+#define A2S_TAG_ALLOC_2__TAG_ALLOC_FOR_VC6_WR_MASK                                                            0x00FF0000L
+#define A2S_TAG_ALLOC_2__TAG_ALLOC_FOR_VC6_RD_MASK                                                            0xFF000000L
+//A2S_TAG_ALLOC_3
+#define A2S_TAG_ALLOC_3__TAG_ALLOC_FOR_VC4_WR__SHIFT                                                          0x0
+#define A2S_TAG_ALLOC_3__TAG_ALLOC_FOR_VC4_RD__SHIFT                                                          0x8
+#define A2S_TAG_ALLOC_3__TAG_ALLOC_FOR_VC2_WR__SHIFT                                                          0x10
+#define A2S_TAG_ALLOC_3__TAG_ALLOC_FOR_VC2_RD__SHIFT                                                          0x18
+#define A2S_TAG_ALLOC_3__TAG_ALLOC_FOR_VC4_WR_MASK                                                            0x000000FFL
+#define A2S_TAG_ALLOC_3__TAG_ALLOC_FOR_VC4_RD_MASK                                                            0x0000FF00L
+#define A2S_TAG_ALLOC_3__TAG_ALLOC_FOR_VC2_WR_MASK                                                            0x00FF0000L
+#define A2S_TAG_ALLOC_3__TAG_ALLOC_FOR_VC2_RD_MASK                                                            0xFF000000L
+//SHUB_A2S_TAG_ALLOC_FOR_CHAIN_0
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_0__TAG_ALLOC_FOR_VC0_CHAIN__SHIFT                                        0x0
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_0__TAG_ALLOC_FOR_VC1_CHAIN__SHIFT                                        0x8
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_0__TAG_ALLOC_FOR_VC2_CHAIN__SHIFT                                        0x10
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_0__TAG_ALLOC_FOR_VC3_CHAIN__SHIFT                                        0x18
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_0__TAG_ALLOC_FOR_VC0_CHAIN_MASK                                          0x000000FFL
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_0__TAG_ALLOC_FOR_VC1_CHAIN_MASK                                          0x0000FF00L
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_0__TAG_ALLOC_FOR_VC2_CHAIN_MASK                                          0x00FF0000L
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_0__TAG_ALLOC_FOR_VC3_CHAIN_MASK                                          0xFF000000L
+//SHUB_A2S_TAG_ALLOC_FOR_CHAIN_1
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_1__TAG_ALLOC_FOR_VC4_CHAIN__SHIFT                                        0x0
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_1__TAG_ALLOC_FOR_VC5_CHAIN__SHIFT                                        0x8
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_1__TAG_ALLOC_FOR_VC6_CHAIN__SHIFT                                        0x10
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_1__TAG_ALLOC_FOR_VC7_CHAIN__SHIFT                                        0x18
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_1__TAG_ALLOC_FOR_VC4_CHAIN_MASK                                          0x000000FFL
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_1__TAG_ALLOC_FOR_VC5_CHAIN_MASK                                          0x0000FF00L
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_1__TAG_ALLOC_FOR_VC6_CHAIN_MASK                                          0x00FF0000L
+#define SHUB_A2S_TAG_ALLOC_FOR_CHAIN_1__TAG_ALLOC_FOR_VC7_CHAIN_MASK                                          0xFF000000L
+//GDC_SDP_PORT_CTRL
+#define GDC_SDP_PORT_CTRL__POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC__SHIFT                                          0x0
+#define GDC_SDP_PORT_CTRL__POOL_CREDIT_ALLOC_OVERRIDE_DYNAMIC_MASK                                            0x00000001L
+//GDC_A2S_SDP_REQ_POOLCRED_ALLOC
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                      0x0
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                      0x4
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                      0x8
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                      0xc
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                      0x10
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                      0x14
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                      0x18
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                      0x1c
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                        0x0000000FL
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                        0x000000F0L
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                        0x00000F00L
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                        0x0000F000L
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                        0x000F0000L
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                        0x00F00000L
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                        0x0F000000L
+#define GDC_A2S_SDP_REQ_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                        0xF0000000L
+//GDC_A2S_SDP_DAT_POOLCRED_ALLOC
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC__SHIFT                                                      0x0
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC__SHIFT                                                      0x4
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC__SHIFT                                                      0x8
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC__SHIFT                                                      0xc
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC__SHIFT                                                      0x10
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC__SHIFT                                                      0x14
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC__SHIFT                                                      0x18
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC__SHIFT                                                      0x1c
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC0_ALLOC_MASK                                                        0x0000000FL
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC1_ALLOC_MASK                                                        0x000000F0L
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC2_ALLOC_MASK                                                        0x00000F00L
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC3_ALLOC_MASK                                                        0x0000F000L
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC4_ALLOC_MASK                                                        0x000F0000L
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC5_ALLOC_MASK                                                        0x00F00000L
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC6_ALLOC_MASK                                                        0x0F000000L
+#define GDC_A2S_SDP_DAT_POOLCRED_ALLOC__VC7_ALLOC_MASK                                                        0xF0000000L
+//SHUB_PERF_COMMON_COUNTER
+#define SHUB_PERF_COMMON_COUNTER__SHUB_COM_COUNT_ENBALE__SHIFT                                                0x0
+#define SHUB_PERF_COMMON_COUNTER__SHUB_COM_COUNT_ENBALE_MASK                                                  0x00000001L
+//SHUB_A2S_PERF_CNT_FSM
+#define SHUB_A2S_PERF_CNT_FSM__A2S_GLOBAL_SHADOW_TGL_DELAY_COUNT__SHIFT                                       0x0
+#define SHUB_A2S_PERF_CNT_FSM__A2S_GLOBAL_PERF_RESET_TGL_DELAY_COUNT__SHIFT                                   0x4
+#define SHUB_A2S_PERF_CNT_FSM__A2S_GLOBAL_PERF_RESET_TGL_DELAY_EN__SHIFT                                      0x8
+#define SHUB_A2S_PERF_CNT_FSM__A2S_PRE_FLD_GLOBAL_SHADOW_WR__SHIFT                                            0x9
+#define SHUB_A2S_PERF_CNT_FSM__A2S_PERF_CNT_DONE__SHIFT                                                       0xa
+#define SHUB_A2S_PERF_CNT_FSM__A2S_GLOBAL_SHADOW_TGL_DELAY_COUNT_MASK                                         0x0000000FL
+#define SHUB_A2S_PERF_CNT_FSM__A2S_GLOBAL_PERF_RESET_TGL_DELAY_COUNT_MASK                                     0x000000F0L
+#define SHUB_A2S_PERF_CNT_FSM__A2S_GLOBAL_PERF_RESET_TGL_DELAY_EN_MASK                                        0x00000100L
+#define SHUB_A2S_PERF_CNT_FSM__A2S_PRE_FLD_GLOBAL_SHADOW_WR_MASK                                              0x00000200L
+#define SHUB_A2S_PERF_CNT_FSM__A2S_PERF_CNT_DONE_MASK                                                         0x00000400L
+//SHUB_A2S_PERF_CNT_RSP_UNITID
+#define SHUB_A2S_PERF_CNT_RSP_UNITID__MONITOR_WRRSP_UNITID__SHIFT                                             0x0
+#define SHUB_A2S_PERF_CNT_RSP_UNITID__MONITOR_WRRSP_UNITID_EN__SHIFT                                          0xf
+#define SHUB_A2S_PERF_CNT_RSP_UNITID__MONITOR_RDRSP_UNITID__SHIFT                                             0x10
+#define SHUB_A2S_PERF_CNT_RSP_UNITID__MONITOR_RDRSP_UNITID_EN__SHIFT                                          0x1f
+#define SHUB_A2S_PERF_CNT_RSP_UNITID__MONITOR_WRRSP_UNITID_MASK                                               0x000007FFL
+#define SHUB_A2S_PERF_CNT_RSP_UNITID__MONITOR_WRRSP_UNITID_EN_MASK                                            0x00008000L
+#define SHUB_A2S_PERF_CNT_RSP_UNITID__MONITOR_RDRSP_UNITID_MASK                                               0x07FF0000L
+#define SHUB_A2S_PERF_CNT_RSP_UNITID__MONITOR_RDRSP_UNITID_EN_MASK                                            0x80000000L
+//SHUB_A2S_PERF_CNT_0_CTRL
+#define SHUB_A2S_PERF_CNT_0_CTRL__A2S_PERF_CNT_0_SEL__SHIFT                                                   0x0
+#define SHUB_A2S_PERF_CNT_0_CTRL__A2S_PERF_CNT_0_EN__SHIFT                                                    0x1f
+#define SHUB_A2S_PERF_CNT_0_CTRL__A2S_PERF_CNT_0_SEL_MASK                                                     0x7FFFFFFFL
+#define SHUB_A2S_PERF_CNT_0_CTRL__A2S_PERF_CNT_0_EN_MASK                                                      0x80000000L
+//SHUB_A2S_PERF_CNT_1_CTRL
+#define SHUB_A2S_PERF_CNT_1_CTRL__A2S_PERF_CNT_1_SEL__SHIFT                                                   0x0
+#define SHUB_A2S_PERF_CNT_1_CTRL__A2S_PERF_CNT_1_EN__SHIFT                                                    0x1f
+#define SHUB_A2S_PERF_CNT_1_CTRL__A2S_PERF_CNT_1_SEL_MASK                                                     0x7FFFFFFFL
+#define SHUB_A2S_PERF_CNT_1_CTRL__A2S_PERF_CNT_1_EN_MASK                                                      0x80000000L
+//SHUB_A2S_PERF_CNT_2_CTRL
+#define SHUB_A2S_PERF_CNT_2_CTRL__A2S_PERF_CNT_2_SEL__SHIFT                                                   0x0
+#define SHUB_A2S_PERF_CNT_2_CTRL__A2S_PERF_CNT_2_EN__SHIFT                                                    0x1f
+#define SHUB_A2S_PERF_CNT_2_CTRL__A2S_PERF_CNT_2_SEL_MASK                                                     0x7FFFFFFFL
+#define SHUB_A2S_PERF_CNT_2_CTRL__A2S_PERF_CNT_2_EN_MASK                                                      0x80000000L
+//SHUB_A2S_PERF_CNT_3_CTRL
+#define SHUB_A2S_PERF_CNT_3_CTRL__A2S_PERF_CNT_3_SEL__SHIFT                                                   0x0
+#define SHUB_A2S_PERF_CNT_3_CTRL__A2S_PERF_CNT_3_EN__SHIFT                                                    0x1f
+#define SHUB_A2S_PERF_CNT_3_CTRL__A2S_PERF_CNT_3_SEL_MASK                                                     0x7FFFFFFFL
+#define SHUB_A2S_PERF_CNT_3_CTRL__A2S_PERF_CNT_3_EN_MASK                                                      0x80000000L
+//SHUB_A2S_PERF_CNT_0_VALUE_L32BIT
+#define SHUB_A2S_PERF_CNT_0_VALUE_L32BIT__A2S_PERF_CNT_0_VALUE_L32BIT__SHIFT                                  0x0
+#define SHUB_A2S_PERF_CNT_0_VALUE_L32BIT__A2S_PERF_CNT_0_VALUE_L32BIT_MASK                                    0xFFFFFFFFL
+//SHUB_A2S_PERF_CNT_0_VALUE_H16BIT
+#define SHUB_A2S_PERF_CNT_0_VALUE_H16BIT__A2S_PERF_CNT_0_VALUE_H16BIT__SHIFT                                  0x0
+#define SHUB_A2S_PERF_CNT_0_VALUE_H16BIT__A2S_PERF_CNT_0_VALUE_H16BIT_MASK                                    0xFFFFL
+//SHUB_A2S_PERF_CNT_1_VALUE_L32BIT
+#define SHUB_A2S_PERF_CNT_1_VALUE_L32BIT__A2S_PERF_CNT_1_VALUE_L32BIT__SHIFT                                  0x0
+#define SHUB_A2S_PERF_CNT_1_VALUE_L32BIT__A2S_PERF_CNT_1_VALUE_L32BIT_MASK                                    0xFFFFFFFFL
+//SHUB_A2S_PERF_CNT_1_VALUE_H16BIT
+#define SHUB_A2S_PERF_CNT_1_VALUE_H16BIT__A2S_PERF_CNT_1_VALUE_H16BIT__SHIFT                                  0x0
+#define SHUB_A2S_PERF_CNT_1_VALUE_H16BIT__A2S_PERF_CNT_1_VALUE_H16BIT_MASK                                    0xFFFFL
+//SHUB_A2S_PERF_CNT_2_VALUE_L32BIT
+#define SHUB_A2S_PERF_CNT_2_VALUE_L32BIT__A2S_PERF_CNT_2_VALUE_L32BIT__SHIFT                                  0x0
+#define SHUB_A2S_PERF_CNT_2_VALUE_L32BIT__A2S_PERF_CNT_2_VALUE_L32BIT_MASK                                    0xFFFFFFFFL
+//SHUB_A2S_PERF_CNT_2_VALUE_H16BIT
+#define SHUB_A2S_PERF_CNT_2_VALUE_H16BIT__A2S_PERF_CNT_2_VALUE_H16BIT__SHIFT                                  0x0
+#define SHUB_A2S_PERF_CNT_2_VALUE_H16BIT__A2S_PERF_CNT_2_VALUE_H16BIT_MASK                                    0xFFFFL
+//SHUB_A2S_PERF_CNT_3_VALUE_L32BIT
+#define SHUB_A2S_PERF_CNT_3_VALUE_L32BIT__A2S_PERF_CNT_3_VALUE_L32BIT__SHIFT                                  0x0
+#define SHUB_A2S_PERF_CNT_3_VALUE_L32BIT__A2S_PERF_CNT_3_VALUE_L32BIT_MASK                                    0xFFFFFFFFL
+//SHUB_A2S_PERF_CNT_3_VALUE_H16BIT
+#define SHUB_A2S_PERF_CNT_3_VALUE_H16BIT__A2S_PERF_CNT_3_VALUE_H16BIT__SHIFT                                  0x0
+#define SHUB_A2S_PERF_CNT_3_VALUE_H16BIT__A2S_PERF_CNT_3_VALUE_H16BIT_MASK                                    0xFFFFL
+//SHUB_TO_ALL_PERF_COUNTER
+#define SHUB_TO_ALL_PERF_COUNTER__START_COUNT_NOPULS__SHIFT                                                   0x0
+#define SHUB_TO_ALL_PERF_COUNTER__LEGACY_OUT_REALTIME_SEL__SHIFT                                              0x1
+#define SHUB_TO_ALL_PERF_COUNTER__PERF_CNT_TRIG_MONI_SEL__SHIFT                                               0x2
+#define SHUB_TO_ALL_PERF_COUNTER__START_COUNT_NOPULS_MASK                                                     0x00000001L
+#define SHUB_TO_ALL_PERF_COUNTER__LEGACY_OUT_REALTIME_SEL_MASK                                                0x00000002L
+#define SHUB_TO_ALL_PERF_COUNTER__PERF_CNT_TRIG_MONI_SEL_MASK                                                 0x0000000CL
+//GDC_PERF_COUNTER_RESET
+#define GDC_PERF_COUNTER_RESET__GDC_PERF_CNT0_RESET__SHIFT                                                    0x0
+#define GDC_PERF_COUNTER_RESET__GDC_PERF_CNT1_RESET__SHIFT                                                    0x1
+#define GDC_PERF_COUNTER_RESET__GDC_PERF_CNT2_RESET__SHIFT                                                    0x2
+#define GDC_PERF_COUNTER_RESET__GDC_PERF_CNT3_RESET__SHIFT                                                    0x3
+#define GDC_PERF_COUNTER_RESET__GDC_PERF_CNT0_RESET_MASK                                                      0x00000001L
+#define GDC_PERF_COUNTER_RESET__GDC_PERF_CNT1_RESET_MASK                                                      0x00000002L
+#define GDC_PERF_COUNTER_RESET__GDC_PERF_CNT2_RESET_MASK                                                      0x00000004L
+#define GDC_PERF_COUNTER_RESET__GDC_PERF_CNT3_RESET_MASK                                                      0x00000008L
+//SHUB_COM_COUNT_VALUE
+#define SHUB_COM_COUNT_VALUE__SHUB_COM_COUNT_VALUE__SHIFT                                                     0x0
+#define SHUB_COM_COUNT_VALUE__SHUB_COM_COUNT_VALUE_MASK                                                       0xFFFFFFFFL
+//A2S_QUEUE_FIFO_ARB_CNTL
+#define A2S_QUEUE_FIFO_ARB_CNTL__WR_QUEUE_FIFO_POP_ARB_PRIORITY__SHIFT                                        0x0
+#define A2S_QUEUE_FIFO_ARB_CNTL__RD_QUEUE_FIFO_POP_ARB_PRIORITY__SHIFT                                        0xa
+#define A2S_QUEUE_FIFO_ARB_CNTL__WR_QUEUE_FIFO_POP_ARB_MODE__SHIFT                                            0x14
+#define A2S_QUEUE_FIFO_ARB_CNTL__RD_QUEUE_FIFO_POP_ARB_MODE__SHIFT                                            0x15
+#define A2S_QUEUE_FIFO_ARB_CNTL__WR_QUEUE_FIFO_POP_ARB_PRIORITY_MASK                                          0x000003FFL
+#define A2S_QUEUE_FIFO_ARB_CNTL__RD_QUEUE_FIFO_POP_ARB_PRIORITY_MASK                                          0x000FFC00L
+#define A2S_QUEUE_FIFO_ARB_CNTL__WR_QUEUE_FIFO_POP_ARB_MODE_MASK                                              0x00100000L
+#define A2S_QUEUE_FIFO_ARB_CNTL__RD_QUEUE_FIFO_POP_ARB_MODE_MASK                                              0x00200000L
+//WR_A2S_RSP_WRR_CNTL
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE0__SHIFT                                                    0x0
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE1__SHIFT                                                    0x4
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE2__SHIFT                                                    0x8
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE3__SHIFT                                                    0xc
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE4__SHIFT                                                    0x10
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE5__SHIFT                                                    0x14
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE6__SHIFT                                                    0x18
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE7__SHIFT                                                    0x1c
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE0_MASK                                                      0x0000000FL
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE1_MASK                                                      0x000000F0L
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE2_MASK                                                      0x00000F00L
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE3_MASK                                                      0x0000F000L
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE4_MASK                                                      0x000F0000L
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE5_MASK                                                      0x00F00000L
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE6_MASK                                                      0x0F000000L
+#define WR_A2S_RSP_WRR_CNTL__WR_A2S_WRR_SIZE_QUEUE7_MASK                                                      0xF0000000L
+//RD_A2S_RSP_WRR_CNTL
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE0__SHIFT                                                    0x0
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE1__SHIFT                                                    0x4
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE2__SHIFT                                                    0x8
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE3__SHIFT                                                    0xc
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE4__SHIFT                                                    0x10
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE5__SHIFT                                                    0x14
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE6__SHIFT                                                    0x18
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE7__SHIFT                                                    0x1c
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE0_MASK                                                      0x0000000FL
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE1_MASK                                                      0x000000F0L
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE2_MASK                                                      0x00000F00L
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE3_MASK                                                      0x0000F000L
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE4_MASK                                                      0x000F0000L
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE5_MASK                                                      0x00F00000L
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE6_MASK                                                      0x0F000000L
+#define RD_A2S_RSP_WRR_CNTL__RD_A2S_WRR_SIZE_QUEUE7_MASK                                                      0xF0000000L
+
+
+// addressBlock: nbif0_nbif0_syshub_mmreg_syshubdirect
+//SYSHUB_DS_CTRL_SOCCLK
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL0_SOCCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                   0x0
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL1_SOCCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                   0x1
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL2_SOCCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                   0x2
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL3_SOCCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                   0x3
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL4_SOCCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                   0x4
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL5_SOCCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                   0x5
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL6_SOCCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                   0x6
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL7_SOCCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                   0x7
+#define SYSHUB_DS_CTRL_SOCCLK__SYSHUB_SOCCLK_DEEPSLEEP_ALLOW_ENABLE__SHIFT                                    0x1c
+#define SYSHUB_DS_CTRL_SOCCLK__SYSHUB_SOCCLK_DS_EN__SHIFT                                                     0x1f
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL0_SOCCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                     0x00000001L
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL1_SOCCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                     0x00000002L
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL2_SOCCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                     0x00000004L
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL3_SOCCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                     0x00000008L
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL4_SOCCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                     0x00000010L
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL5_SOCCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                     0x00000020L
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL6_SOCCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                     0x00000040L
+#define SYSHUB_DS_CTRL_SOCCLK__DMA_CL7_SOCCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                     0x00000080L
+#define SYSHUB_DS_CTRL_SOCCLK__SYSHUB_SOCCLK_DEEPSLEEP_ALLOW_ENABLE_MASK                                      0x10000000L
+#define SYSHUB_DS_CTRL_SOCCLK__SYSHUB_SOCCLK_DS_EN_MASK                                                       0x80000000L
+//SYSHUB_DS_CTRL2_SOCCLK
+#define SYSHUB_DS_CTRL2_SOCCLK__SYSHUB_SOCCLK_DS_TIMER__SHIFT                                                 0x0
+#define SYSHUB_DS_CTRL2_SOCCLK__SYSHUB_SOCCLK_DS_TIMER_MASK                                                   0x0000FFFFL
+//SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW0_bypass_en__SHIFT                 0x10
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW1_bypass_en__SHIFT                 0x11
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW2_bypass_en__SHIFT                 0x12
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW3_bypass_en__SHIFT                 0x13
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW4_bypass_en__SHIFT                 0x14
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW5_bypass_en__SHIFT                 0x15
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_DYNAMIC_BYPASS_EN_socclk__SHIFT                      0x1f
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW0_bypass_en_MASK                   0x00010000L
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW1_bypass_en_MASK                   0x00020000L
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW2_bypass_en_MASK                   0x00040000L
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW3_bypass_en_MASK                   0x00080000L
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW4_bypass_en_MASK                   0x00100000L
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW5_bypass_en_MASK                   0x00200000L
+#define SYSHUB_BGEN_ENHANCEMENT_BYPASS_EN_SOCCLK__SYSHUB_DYNAMIC_BYPASS_EN_socclk_MASK                        0x80000000L
+//SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW0_imm_en__SHIFT                       0x10
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW1_imm_en__SHIFT                       0x11
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW2_imm_en__SHIFT                       0x12
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW3_imm_en__SHIFT                       0x13
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW4_imm_en__SHIFT                       0x14
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW5_imm_en__SHIFT                       0x15
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW0_imm_en_MASK                         0x00010000L
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW1_imm_en_MASK                         0x00020000L
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW2_imm_en_MASK                         0x00040000L
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW3_imm_en_MASK                         0x00080000L
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW4_imm_en_MASK                         0x00100000L
+#define SYSHUB_BGEN_ENHANCEMENT_IMM_EN_SOCCLK__SYSHUB_bgen_socclk_DMA_SW5_imm_en_MASK                         0x00200000L
+//SYSHUB_MGCG_CTRL_SOCCLK
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_EN_SOCCLK__SHIFT                                                 0x0
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_MODE_SOCCLK__SHIFT                                               0x1
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_HYSTERESIS_SOCCLK__SHIFT                                         0x2
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_HST_DIS_SOCCLK__SHIFT                                            0xa
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_DMA_DIS_SOCCLK__SHIFT                                            0xb
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_REG_DIS_SOCCLK__SHIFT                                            0xc
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_AER_DIS_SOCCLK__SHIFT                                            0xd
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_DBG_DIS_SOCCLK__SHIFT                                            0xe
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_EN_SOCCLK_MASK                                                   0x00000001L
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_MODE_SOCCLK_MASK                                                 0x00000002L
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_HYSTERESIS_SOCCLK_MASK                                           0x000003FCL
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_HST_DIS_SOCCLK_MASK                                              0x00000400L
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_DMA_DIS_SOCCLK_MASK                                              0x00000800L
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_REG_DIS_SOCCLK_MASK                                              0x00001000L
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_AER_DIS_SOCCLK_MASK                                              0x00002000L
+#define SYSHUB_MGCG_CTRL_SOCCLK__SYSHUB_MGCG_DBG_DIS_SOCCLK_MASK                                              0x00004000L
+//SYSHUB_SCRATCH_SOCCLK
+#define SYSHUB_SCRATCH_SOCCLK__SCRATCH_SOCCLK__SHIFT                                                          0x0
+#define SYSHUB_SCRATCH_SOCCLK__SCRATCH_SOCCLK_MASK                                                            0xFFFFFFFFL
+//SYSHUB_HANG_CNTL_SOCCLK
+//SYSHUB_SELECT_SOCCLK
+#define SYSHUB_SELECT_SOCCLK__SELECT_USB0__SHIFT                                                              0x0
+#define SYSHUB_SELECT_SOCCLK__SELECT_USB0_MASK                                                                0x00000001L
+//SYSHUB_DS_AW_READY_SOCCLK
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL0_SOCCLK_AW_READY_DIS_ENABLE__SHIFT                                  0x0
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL1_SOCCLK_AW_READY_DIS_ENABLE__SHIFT                                  0x1
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL2_SOCCLK_AW_READY_DIS_ENABLE__SHIFT                                  0x2
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL3_SOCCLK_AW_READY_DIS_ENABLE__SHIFT                                  0x3
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL4_SOCCLK_AW_READY_DIS_ENABLE__SHIFT                                  0x4
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL5_SOCCLK_AW_READY_DIS_ENABLE__SHIFT                                  0x5
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL6_SOCCLK_AW_READY_DIS_ENABLE__SHIFT                                  0x6
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL7_SOCCLK_AW_READY_DIS_ENABLE__SHIFT                                  0x7
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL0_SOCCLK_AW_READY_DIS_ENABLE_MASK                                    0x00000001L
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL1_SOCCLK_AW_READY_DIS_ENABLE_MASK                                    0x00000002L
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL2_SOCCLK_AW_READY_DIS_ENABLE_MASK                                    0x00000004L
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL3_SOCCLK_AW_READY_DIS_ENABLE_MASK                                    0x00000008L
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL4_SOCCLK_AW_READY_DIS_ENABLE_MASK                                    0x00000010L
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL5_SOCCLK_AW_READY_DIS_ENABLE_MASK                                    0x00000020L
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL6_SOCCLK_AW_READY_DIS_ENABLE_MASK                                    0x00000040L
+#define SYSHUB_DS_AW_READY_SOCCLK__DMA_CL7_SOCCLK_AW_READY_DIS_ENABLE_MASK                                    0x00000080L
+//SYSHUB_DS_AR_READY_SOCCLK
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL0_SOCCLK_AR_READY_DIS_ENABLE__SHIFT                                  0x0
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL1_SOCCLK_AR_READY_DIS_ENABLE__SHIFT                                  0x1
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL2_SOCCLK_AR_READY_DIS_ENABLE__SHIFT                                  0x2
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL3_SOCCLK_AR_READY_DIS_ENABLE__SHIFT                                  0x3
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL4_SOCCLK_AR_READY_DIS_ENABLE__SHIFT                                  0x4
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL5_SOCCLK_AR_READY_DIS_ENABLE__SHIFT                                  0x5
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL6_SOCCLK_AR_READY_DIS_ENABLE__SHIFT                                  0x6
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL7_SOCCLK_AR_READY_DIS_ENABLE__SHIFT                                  0x7
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL0_SOCCLK_AR_READY_DIS_ENABLE_MASK                                    0x00000001L
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL1_SOCCLK_AR_READY_DIS_ENABLE_MASK                                    0x00000002L
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL2_SOCCLK_AR_READY_DIS_ENABLE_MASK                                    0x00000004L
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL3_SOCCLK_AR_READY_DIS_ENABLE_MASK                                    0x00000008L
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL4_SOCCLK_AR_READY_DIS_ENABLE_MASK                                    0x00000010L
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL5_SOCCLK_AR_READY_DIS_ENABLE_MASK                                    0x00000020L
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL6_SOCCLK_AR_READY_DIS_ENABLE_MASK                                    0x00000040L
+#define SYSHUB_DS_AR_READY_SOCCLK__DMA_CL7_SOCCLK_AR_READY_DIS_ENABLE_MASK                                    0x00000080L
+//SYSHUB_DS_W_READY_SOCCLK
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL0_SOCCLK_W_READY_DIS_ENABLE__SHIFT                                    0x0
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL1_SOCCLK_W_READY_DIS_ENABLE__SHIFT                                    0x1
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL2_SOCCLK_W_READY_DIS_ENABLE__SHIFT                                    0x2
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL3_SOCCLK_W_READY_DIS_ENABLE__SHIFT                                    0x3
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL4_SOCCLK_W_READY_DIS_ENABLE__SHIFT                                    0x4
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL5_SOCCLK_W_READY_DIS_ENABLE__SHIFT                                    0x5
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL6_SOCCLK_W_READY_DIS_ENABLE__SHIFT                                    0x6
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL7_SOCCLK_W_READY_DIS_ENABLE__SHIFT                                    0x7
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL0_SOCCLK_W_READY_DIS_ENABLE_MASK                                      0x00000001L
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL1_SOCCLK_W_READY_DIS_ENABLE_MASK                                      0x00000002L
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL2_SOCCLK_W_READY_DIS_ENABLE_MASK                                      0x00000004L
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL3_SOCCLK_W_READY_DIS_ENABLE_MASK                                      0x00000008L
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL4_SOCCLK_W_READY_DIS_ENABLE_MASK                                      0x00000010L
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL5_SOCCLK_W_READY_DIS_ENABLE_MASK                                      0x00000020L
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL6_SOCCLK_W_READY_DIS_ENABLE_MASK                                      0x00000040L
+#define SYSHUB_DS_W_READY_SOCCLK__DMA_CL7_SOCCLK_W_READY_DIS_ENABLE_MASK                                      0x00000080L
+//SYSHUB_DS_R_READY_SOCCLK
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL0_SOCCLK_R_READY_DIS_ENABLE__SHIFT                                    0x0
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL1_SOCCLK_R_READY_DIS_ENABLE__SHIFT                                    0x1
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL2_SOCCLK_R_READY_DIS_ENABLE__SHIFT                                    0x2
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL3_SOCCLK_R_READY_DIS_ENABLE__SHIFT                                    0x3
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL4_SOCCLK_R_READY_DIS_ENABLE__SHIFT                                    0x4
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL5_SOCCLK_R_READY_DIS_ENABLE__SHIFT                                    0x5
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL6_SOCCLK_R_READY_DIS_ENABLE__SHIFT                                    0x6
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL7_SOCCLK_R_READY_DIS_ENABLE__SHIFT                                    0x7
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL0_SOCCLK_R_READY_DIS_ENABLE_MASK                                      0x00000001L
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL1_SOCCLK_R_READY_DIS_ENABLE_MASK                                      0x00000002L
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL2_SOCCLK_R_READY_DIS_ENABLE_MASK                                      0x00000004L
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL3_SOCCLK_R_READY_DIS_ENABLE_MASK                                      0x00000008L
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL4_SOCCLK_R_READY_DIS_ENABLE_MASK                                      0x00000010L
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL5_SOCCLK_R_READY_DIS_ENABLE_MASK                                      0x00000020L
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL6_SOCCLK_R_READY_DIS_ENABLE_MASK                                      0x00000040L
+#define SYSHUB_DS_R_READY_SOCCLK__DMA_CL7_SOCCLK_R_READY_DIS_ENABLE_MASK                                      0x00000080L
+//SYSHUB_DS_B_READY_SOCCLK
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL0_SOCCLK_B_READY_DIS_ENABLE__SHIFT                                    0x0
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL1_SOCCLK_B_READY_DIS_ENABLE__SHIFT                                    0x1
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL2_SOCCLK_B_READY_DIS_ENABLE__SHIFT                                    0x2
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL3_SOCCLK_B_READY_DIS_ENABLE__SHIFT                                    0x3
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL4_SOCCLK_B_READY_DIS_ENABLE__SHIFT                                    0x4
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL5_SOCCLK_B_READY_DIS_ENABLE__SHIFT                                    0x5
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL6_SOCCLK_B_READY_DIS_ENABLE__SHIFT                                    0x6
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL7_SOCCLK_B_READY_DIS_ENABLE__SHIFT                                    0x7
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL0_SOCCLK_B_READY_DIS_ENABLE_MASK                                      0x00000001L
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL1_SOCCLK_B_READY_DIS_ENABLE_MASK                                      0x00000002L
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL2_SOCCLK_B_READY_DIS_ENABLE_MASK                                      0x00000004L
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL3_SOCCLK_B_READY_DIS_ENABLE_MASK                                      0x00000008L
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL4_SOCCLK_B_READY_DIS_ENABLE_MASK                                      0x00000010L
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL5_SOCCLK_B_READY_DIS_ENABLE_MASK                                      0x00000020L
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL6_SOCCLK_B_READY_DIS_ENABLE_MASK                                      0x00000040L
+#define SYSHUB_DS_B_READY_SOCCLK__DMA_CL7_SOCCLK_B_READY_DIS_ENABLE_MASK                                      0x00000080L
+//SYSHUB_AXI_DEBUG_COUNTER_CNTL_SOCCLK
+#define SYSHUB_AXI_DEBUG_COUNTER_CNTL_SOCCLK__HST_SOCCLK_AXI_DEBUG_COUNTER_EN__SHIFT                          0x0
+#define SYSHUB_AXI_DEBUG_COUNTER_CNTL_SOCCLK__DMA_SOCCLK_AXI_DEBUG_COUNTER_EN__SHIFT                          0x1
+#define SYSHUB_AXI_DEBUG_COUNTER_CNTL_SOCCLK__HST_SOCCLK_AXI_DEBUG_COUNTER_EN_MASK                            0x00000001L
+#define SYSHUB_AXI_DEBUG_COUNTER_CNTL_SOCCLK__DMA_SOCCLK_AXI_DEBUG_COUNTER_EN_MASK                            0x00000002L
+//DMA_CLK0_SW0_SYSHUB_QOS_CNTL
+#define DMA_CLK0_SW0_SYSHUB_QOS_CNTL__QOS_CNTL_MODE__SHIFT                                                    0x0
+#define DMA_CLK0_SW0_SYSHUB_QOS_CNTL__QOS_MAX_VALUE__SHIFT                                                    0x1
+#define DMA_CLK0_SW0_SYSHUB_QOS_CNTL__QOS_MIN_VALUE__SHIFT                                                    0x5
+#define DMA_CLK0_SW0_SYSHUB_QOS_CNTL__QOS_CNTL_MODE_MASK                                                      0x00000001L
+#define DMA_CLK0_SW0_SYSHUB_QOS_CNTL__QOS_MAX_VALUE_MASK                                                      0x0000001EL
+#define DMA_CLK0_SW0_SYSHUB_QOS_CNTL__QOS_MIN_VALUE_MASK                                                      0x000001E0L
+//DMA_CLK0_SW1_SYSHUB_QOS_CNTL
+#define DMA_CLK0_SW1_SYSHUB_QOS_CNTL__QOS_CNTL_MODE__SHIFT                                                    0x0
+#define DMA_CLK0_SW1_SYSHUB_QOS_CNTL__QOS_MAX_VALUE__SHIFT                                                    0x1
+#define DMA_CLK0_SW1_SYSHUB_QOS_CNTL__QOS_MIN_VALUE__SHIFT                                                    0x5
+#define DMA_CLK0_SW1_SYSHUB_QOS_CNTL__QOS_CNTL_MODE_MASK                                                      0x00000001L
+#define DMA_CLK0_SW1_SYSHUB_QOS_CNTL__QOS_MAX_VALUE_MASK                                                      0x0000001EL
+#define DMA_CLK0_SW1_SYSHUB_QOS_CNTL__QOS_MIN_VALUE_MASK                                                      0x000001E0L
+//DMA_CLK0_SW0_CL0_CNTL
+#define DMA_CLK0_SW0_CL0_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW0_CL0_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW0_CL0_CNTL__DEBUG_RS_RESET_EN__SHIFT                                                       0x2
+#define DMA_CLK0_SW0_CL0_CNTL__QOS_STATIC_OVERRIDE_EN__SHIFT                                                  0x8
+#define DMA_CLK0_SW0_CL0_CNTL__QOS_STATIC_OVERRIDE_VALUE__SHIFT                                               0x9
+#define DMA_CLK0_SW0_CL0_CNTL__READ_WRR_WEIGHT__SHIFT                                                         0x10
+#define DMA_CLK0_SW0_CL0_CNTL__WRITE_WRR_WEIGHT__SHIFT                                                        0x18
+#define DMA_CLK0_SW0_CL0_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW0_CL0_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW0_CL0_CNTL__DEBUG_RS_RESET_EN_MASK                                                         0x00000004L
+#define DMA_CLK0_SW0_CL0_CNTL__QOS_STATIC_OVERRIDE_EN_MASK                                                    0x00000100L
+#define DMA_CLK0_SW0_CL0_CNTL__QOS_STATIC_OVERRIDE_VALUE_MASK                                                 0x00001E00L
+#define DMA_CLK0_SW0_CL0_CNTL__READ_WRR_WEIGHT_MASK                                                           0x00FF0000L
+#define DMA_CLK0_SW0_CL0_CNTL__WRITE_WRR_WEIGHT_MASK                                                          0xFF000000L
+//DMA_CLK0_SW0_CL1_CNTL
+#define DMA_CLK0_SW0_CL1_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW0_CL1_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW0_CL1_CNTL__DEBUG_RS_RESET_EN__SHIFT                                                       0x2
+#define DMA_CLK0_SW0_CL1_CNTL__QOS_STATIC_OVERRIDE_EN__SHIFT                                                  0x8
+#define DMA_CLK0_SW0_CL1_CNTL__QOS_STATIC_OVERRIDE_VALUE__SHIFT                                               0x9
+#define DMA_CLK0_SW0_CL1_CNTL__READ_WRR_WEIGHT__SHIFT                                                         0x10
+#define DMA_CLK0_SW0_CL1_CNTL__WRITE_WRR_WEIGHT__SHIFT                                                        0x18
+#define DMA_CLK0_SW0_CL1_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW0_CL1_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW0_CL1_CNTL__DEBUG_RS_RESET_EN_MASK                                                         0x00000004L
+#define DMA_CLK0_SW0_CL1_CNTL__QOS_STATIC_OVERRIDE_EN_MASK                                                    0x00000100L
+#define DMA_CLK0_SW0_CL1_CNTL__QOS_STATIC_OVERRIDE_VALUE_MASK                                                 0x00001E00L
+#define DMA_CLK0_SW0_CL1_CNTL__READ_WRR_WEIGHT_MASK                                                           0x00FF0000L
+#define DMA_CLK0_SW0_CL1_CNTL__WRITE_WRR_WEIGHT_MASK                                                          0xFF000000L
+//DMA_CLK0_SW1_CL0_CNTL
+#define DMA_CLK0_SW1_CL0_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW1_CL0_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW1_CL0_CNTL__DEBUG_RS_RESET_EN__SHIFT                                                       0x2
+#define DMA_CLK0_SW1_CL0_CNTL__QOS_STATIC_OVERRIDE_EN__SHIFT                                                  0x8
+#define DMA_CLK0_SW1_CL0_CNTL__QOS_STATIC_OVERRIDE_VALUE__SHIFT                                               0x9
+#define DMA_CLK0_SW1_CL0_CNTL__READ_WRR_WEIGHT__SHIFT                                                         0x10
+#define DMA_CLK0_SW1_CL0_CNTL__WRITE_WRR_WEIGHT__SHIFT                                                        0x18
+#define DMA_CLK0_SW1_CL0_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW1_CL0_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW1_CL0_CNTL__DEBUG_RS_RESET_EN_MASK                                                         0x00000004L
+#define DMA_CLK0_SW1_CL0_CNTL__QOS_STATIC_OVERRIDE_EN_MASK                                                    0x00000100L
+#define DMA_CLK0_SW1_CL0_CNTL__QOS_STATIC_OVERRIDE_VALUE_MASK                                                 0x00001E00L
+#define DMA_CLK0_SW1_CL0_CNTL__READ_WRR_WEIGHT_MASK                                                           0x00FF0000L
+#define DMA_CLK0_SW1_CL0_CNTL__WRITE_WRR_WEIGHT_MASK                                                          0xFF000000L
+//DMA_CLK0_SW1_CL1_CNTL
+#define DMA_CLK0_SW1_CL1_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW1_CL1_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW1_CL1_CNTL__DEBUG_RS_RESET_EN__SHIFT                                                       0x2
+#define DMA_CLK0_SW1_CL1_CNTL__QOS_STATIC_OVERRIDE_EN__SHIFT                                                  0x8
+#define DMA_CLK0_SW1_CL1_CNTL__QOS_STATIC_OVERRIDE_VALUE__SHIFT                                               0x9
+#define DMA_CLK0_SW1_CL1_CNTL__READ_WRR_WEIGHT__SHIFT                                                         0x10
+#define DMA_CLK0_SW1_CL1_CNTL__WRITE_WRR_WEIGHT__SHIFT                                                        0x18
+#define DMA_CLK0_SW1_CL1_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW1_CL1_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW1_CL1_CNTL__DEBUG_RS_RESET_EN_MASK                                                         0x00000004L
+#define DMA_CLK0_SW1_CL1_CNTL__QOS_STATIC_OVERRIDE_EN_MASK                                                    0x00000100L
+#define DMA_CLK0_SW1_CL1_CNTL__QOS_STATIC_OVERRIDE_VALUE_MASK                                                 0x00001E00L
+#define DMA_CLK0_SW1_CL1_CNTL__READ_WRR_WEIGHT_MASK                                                           0x00FF0000L
+#define DMA_CLK0_SW1_CL1_CNTL__WRITE_WRR_WEIGHT_MASK                                                          0xFF000000L
+//DMA_CLK0_SW2_CL0_CNTL
+#define DMA_CLK0_SW2_CL0_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW2_CL0_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW2_CL0_CNTL__DEBUG_RS_RESET_EN__SHIFT                                                       0x2
+#define DMA_CLK0_SW2_CL0_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW2_CL0_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW2_CL0_CNTL__DEBUG_RS_RESET_EN_MASK                                                         0x00000004L
+//DMA_CLK0_SW3_CL0_CNTL
+#define DMA_CLK0_SW3_CL0_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW3_CL0_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW3_CL0_CNTL__DEBUG_RS_RESET_EN__SHIFT                                                       0x2
+#define DMA_CLK0_SW3_CL0_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW3_CL0_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW3_CL0_CNTL__DEBUG_RS_RESET_EN_MASK                                                         0x00000004L
+//DMA_CLK0_SW4_CL0_CNTL
+#define DMA_CLK0_SW4_CL0_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW4_CL0_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW4_CL0_CNTL__DEBUG_RS_RESET_EN__SHIFT                                                       0x2
+#define DMA_CLK0_SW4_CL0_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW4_CL0_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW4_CL0_CNTL__DEBUG_RS_RESET_EN_MASK                                                         0x00000004L
+//DMA_CLK0_SW5_CL0_CNTL
+#define DMA_CLK0_SW5_CL0_CNTL__FLR_ON_RS_RESET_EN__SHIFT                                                      0x0
+#define DMA_CLK0_SW5_CL0_CNTL__LKRST_ON_RS_RESET_EN__SHIFT                                                    0x1
+#define DMA_CLK0_SW5_CL0_CNTL__DEBUG_RS_RESET_EN__SHIFT                                                       0x2
+#define DMA_CLK0_SW5_CL0_CNTL__FLR_ON_RS_RESET_EN_MASK                                                        0x00000001L
+#define DMA_CLK0_SW5_CL0_CNTL__LKRST_ON_RS_RESET_EN_MASK                                                      0x00000002L
+#define DMA_CLK0_SW5_CL0_CNTL__DEBUG_RS_RESET_EN_MASK                                                         0x00000004L
+//NIC400_0_AMIB_0_FN_MOD_BM_ISS
+#define NIC400_0_AMIB_0_FN_MOD_BM_ISS__read_iss_override__SHIFT                                               0x0
+#define NIC400_0_AMIB_0_FN_MOD_BM_ISS__write_iss_override__SHIFT                                              0x1
+#define NIC400_0_AMIB_0_FN_MOD_BM_ISS__read_iss_override_MASK                                                 0x00000001L
+#define NIC400_0_AMIB_0_FN_MOD_BM_ISS__write_iss_override_MASK                                                0x00000002L
+//NIC400_0_AMIB_0_FN_MOD
+#define NIC400_0_AMIB_0_FN_MOD__read_iss_override__SHIFT                                                      0x0
+#define NIC400_0_AMIB_0_FN_MOD__write_iss_override__SHIFT                                                     0x1
+#define NIC400_0_AMIB_0_FN_MOD__read_iss_override_MASK                                                        0x00000001L
+#define NIC400_0_AMIB_0_FN_MOD__write_iss_override_MASK                                                       0x00000002L
+//NIC400_0_ASIB_0_FN_MOD
+#define NIC400_0_ASIB_0_FN_MOD__read_iss_override__SHIFT                                                      0x0
+#define NIC400_0_ASIB_0_FN_MOD__write_iss_override__SHIFT                                                     0x1
+#define NIC400_0_ASIB_0_FN_MOD__read_iss_override_MASK                                                        0x00000001L
+#define NIC400_0_ASIB_0_FN_MOD__write_iss_override_MASK                                                       0x00000002L
+//NIC400_0_ASIB_0_QOS_CNTL
+#define NIC400_0_ASIB_0_QOS_CNTL__en_aw_rate__SHIFT                                                           0x0
+#define NIC400_0_ASIB_0_QOS_CNTL__en_ar_rate__SHIFT                                                           0x1
+#define NIC400_0_ASIB_0_QOS_CNTL__en_awar_rate__SHIFT                                                         0x2
+#define NIC400_0_ASIB_0_QOS_CNTL__en_aw_fc__SHIFT                                                             0x3
+#define NIC400_0_ASIB_0_QOS_CNTL__en_ar_fc__SHIFT                                                             0x4
+#define NIC400_0_ASIB_0_QOS_CNTL__en_aw_ot__SHIFT                                                             0x5
+#define NIC400_0_ASIB_0_QOS_CNTL__en_ar_ot__SHIFT                                                             0x6
+#define NIC400_0_ASIB_0_QOS_CNTL__en_awar_ot__SHIFT                                                           0x7
+#define NIC400_0_ASIB_0_QOS_CNTL__mode_aw_fc__SHIFT                                                           0x10
+#define NIC400_0_ASIB_0_QOS_CNTL__mode_ar_fc__SHIFT                                                           0x14
+#define NIC400_0_ASIB_0_QOS_CNTL__en_aw_rate_MASK                                                             0x00000001L
+#define NIC400_0_ASIB_0_QOS_CNTL__en_ar_rate_MASK                                                             0x00000002L
+#define NIC400_0_ASIB_0_QOS_CNTL__en_awar_rate_MASK                                                           0x00000004L
+#define NIC400_0_ASIB_0_QOS_CNTL__en_aw_fc_MASK                                                               0x00000008L
+#define NIC400_0_ASIB_0_QOS_CNTL__en_ar_fc_MASK                                                               0x00000010L
+#define NIC400_0_ASIB_0_QOS_CNTL__en_aw_ot_MASK                                                               0x00000020L
+#define NIC400_0_ASIB_0_QOS_CNTL__en_ar_ot_MASK                                                               0x00000040L
+#define NIC400_0_ASIB_0_QOS_CNTL__en_awar_ot_MASK                                                             0x00000080L
+#define NIC400_0_ASIB_0_QOS_CNTL__mode_aw_fc_MASK                                                             0x00010000L
+#define NIC400_0_ASIB_0_QOS_CNTL__mode_ar_fc_MASK                                                             0x00100000L
+//NIC400_0_ASIB_0_MAX_OT
+#define NIC400_0_ASIB_0_MAX_OT__aw_max_otf__SHIFT                                                             0x0
+#define NIC400_0_ASIB_0_MAX_OT__aw_max_oti__SHIFT                                                             0x8
+#define NIC400_0_ASIB_0_MAX_OT__ar_max_otf__SHIFT                                                             0x10
+#define NIC400_0_ASIB_0_MAX_OT__ar_max_oti__SHIFT                                                             0x18
+#define NIC400_0_ASIB_0_MAX_OT__aw_max_otf_MASK                                                               0x000000FFL
+#define NIC400_0_ASIB_0_MAX_OT__aw_max_oti_MASK                                                               0x00003F00L
+#define NIC400_0_ASIB_0_MAX_OT__ar_max_otf_MASK                                                               0x00FF0000L
+#define NIC400_0_ASIB_0_MAX_OT__ar_max_oti_MASK                                                               0x3F000000L
+//NIC400_0_ASIB_0_MAX_COMB_OT
+#define NIC400_0_ASIB_0_MAX_COMB_OT__awar_max_otf__SHIFT                                                      0x0
+#define NIC400_0_ASIB_0_MAX_COMB_OT__awar_max_oti__SHIFT                                                      0x8
+#define NIC400_0_ASIB_0_MAX_COMB_OT__awar_max_otf_MASK                                                        0x000000FFL
+#define NIC400_0_ASIB_0_MAX_COMB_OT__awar_max_oti_MASK                                                        0x00007F00L
+//NIC400_0_ASIB_0_AW_P
+#define NIC400_0_ASIB_0_AW_P__aw_p__SHIFT                                                                     0x18
+#define NIC400_0_ASIB_0_AW_P__aw_p_MASK                                                                       0xFF000000L
+//NIC400_0_ASIB_0_AW_B
+#define NIC400_0_ASIB_0_AW_B__aw_b__SHIFT                                                                     0x0
+#define NIC400_0_ASIB_0_AW_B__aw_b_MASK                                                                       0x0000FFFFL
+//NIC400_0_ASIB_0_AW_R
+#define NIC400_0_ASIB_0_AW_R__aw_r__SHIFT                                                                     0x14
+#define NIC400_0_ASIB_0_AW_R__aw_r_MASK                                                                       0xFFF00000L
+//NIC400_0_ASIB_0_AR_P
+#define NIC400_0_ASIB_0_AR_P__ar_p__SHIFT                                                                     0x18
+#define NIC400_0_ASIB_0_AR_P__ar_p_MASK                                                                       0xFF000000L
+//NIC400_0_ASIB_0_AR_B
+#define NIC400_0_ASIB_0_AR_B__ar_b__SHIFT                                                                     0x0
+#define NIC400_0_ASIB_0_AR_B__ar_b_MASK                                                                       0x0000FFFFL
+//NIC400_0_ASIB_0_AR_R
+#define NIC400_0_ASIB_0_AR_R__ar_r__SHIFT                                                                     0x14
+#define NIC400_0_ASIB_0_AR_R__ar_r_MASK                                                                       0xFFF00000L
+//NIC400_0_ASIB_0_TARGET_FC
+#define NIC400_0_ASIB_0_TARGET_FC__aw_tgt_latency__SHIFT                                                      0x0
+#define NIC400_0_ASIB_0_TARGET_FC__ar_tgt_latency__SHIFT                                                      0x10
+#define NIC400_0_ASIB_0_TARGET_FC__aw_tgt_latency_MASK                                                        0x00000FFFL
+#define NIC400_0_ASIB_0_TARGET_FC__ar_tgt_latency_MASK                                                        0x0FFF0000L
+//NIC400_0_ASIB_0_KI_FC
+#define NIC400_0_ASIB_0_KI_FC__aw_tgt_latency__SHIFT                                                          0x0
+#define NIC400_0_ASIB_0_KI_FC__ar_tgt_latency__SHIFT                                                          0x8
+#define NIC400_0_ASIB_0_KI_FC__aw_tgt_latency_MASK                                                            0x00000007L
+#define NIC400_0_ASIB_0_KI_FC__ar_tgt_latency_MASK                                                            0x00000700L
+//NIC400_0_ASIB_0_QOS_RANGE
+#define NIC400_0_ASIB_0_QOS_RANGE__aw_min_qos__SHIFT                                                          0x0
+#define NIC400_0_ASIB_0_QOS_RANGE__aw_max_qos__SHIFT                                                          0x8
+#define NIC400_0_ASIB_0_QOS_RANGE__ar_min_qos__SHIFT                                                          0x10
+#define NIC400_0_ASIB_0_QOS_RANGE__ar_max_qos__SHIFT                                                          0x18
+#define NIC400_0_ASIB_0_QOS_RANGE__aw_min_qos_MASK                                                            0x0000000FL
+#define NIC400_0_ASIB_0_QOS_RANGE__aw_max_qos_MASK                                                            0x00000F00L
+#define NIC400_0_ASIB_0_QOS_RANGE__ar_min_qos_MASK                                                            0x000F0000L
+#define NIC400_0_ASIB_0_QOS_RANGE__ar_max_qos_MASK                                                            0x0F000000L
+//NIC400_0_ASIB_1_FN_MOD
+#define NIC400_0_ASIB_1_FN_MOD__read_iss_override__SHIFT                                                      0x0
+#define NIC400_0_ASIB_1_FN_MOD__write_iss_override__SHIFT                                                     0x1
+#define NIC400_0_ASIB_1_FN_MOD__read_iss_override_MASK                                                        0x00000001L
+#define NIC400_0_ASIB_1_FN_MOD__write_iss_override_MASK                                                       0x00000002L
+//NIC400_0_ASIB_1_QOS_CNTL
+#define NIC400_0_ASIB_1_QOS_CNTL__en_aw_rate__SHIFT                                                           0x0
+#define NIC400_0_ASIB_1_QOS_CNTL__en_ar_rate__SHIFT                                                           0x1
+#define NIC400_0_ASIB_1_QOS_CNTL__en_awar_rate__SHIFT                                                         0x2
+#define NIC400_0_ASIB_1_QOS_CNTL__en_aw_fc__SHIFT                                                             0x3
+#define NIC400_0_ASIB_1_QOS_CNTL__en_ar_fc__SHIFT                                                             0x4
+#define NIC400_0_ASIB_1_QOS_CNTL__en_aw_ot__SHIFT                                                             0x5
+#define NIC400_0_ASIB_1_QOS_CNTL__en_ar_ot__SHIFT                                                             0x6
+#define NIC400_0_ASIB_1_QOS_CNTL__en_awar_ot__SHIFT                                                           0x7
+#define NIC400_0_ASIB_1_QOS_CNTL__mode_aw_fc__SHIFT                                                           0x10
+#define NIC400_0_ASIB_1_QOS_CNTL__mode_ar_fc__SHIFT                                                           0x14
+#define NIC400_0_ASIB_1_QOS_CNTL__en_aw_rate_MASK                                                             0x00000001L
+#define NIC400_0_ASIB_1_QOS_CNTL__en_ar_rate_MASK                                                             0x00000002L
+#define NIC400_0_ASIB_1_QOS_CNTL__en_awar_rate_MASK                                                           0x00000004L
+#define NIC400_0_ASIB_1_QOS_CNTL__en_aw_fc_MASK                                                               0x00000008L
+#define NIC400_0_ASIB_1_QOS_CNTL__en_ar_fc_MASK                                                               0x00000010L
+#define NIC400_0_ASIB_1_QOS_CNTL__en_aw_ot_MASK                                                               0x00000020L
+#define NIC400_0_ASIB_1_QOS_CNTL__en_ar_ot_MASK                                                               0x00000040L
+#define NIC400_0_ASIB_1_QOS_CNTL__en_awar_ot_MASK                                                             0x00000080L
+#define NIC400_0_ASIB_1_QOS_CNTL__mode_aw_fc_MASK                                                             0x00010000L
+#define NIC400_0_ASIB_1_QOS_CNTL__mode_ar_fc_MASK                                                             0x00100000L
+//NIC400_0_ASIB_1_MAX_OT
+#define NIC400_0_ASIB_1_MAX_OT__aw_max_otf__SHIFT                                                             0x0
+#define NIC400_0_ASIB_1_MAX_OT__aw_max_oti__SHIFT                                                             0x8
+#define NIC400_0_ASIB_1_MAX_OT__ar_max_otf__SHIFT                                                             0x10
+#define NIC400_0_ASIB_1_MAX_OT__ar_max_oti__SHIFT                                                             0x18
+#define NIC400_0_ASIB_1_MAX_OT__aw_max_otf_MASK                                                               0x000000FFL
+#define NIC400_0_ASIB_1_MAX_OT__aw_max_oti_MASK                                                               0x00003F00L
+#define NIC400_0_ASIB_1_MAX_OT__ar_max_otf_MASK                                                               0x00FF0000L
+#define NIC400_0_ASIB_1_MAX_OT__ar_max_oti_MASK                                                               0x3F000000L
+//NIC400_0_ASIB_1_MAX_COMB_OT
+#define NIC400_0_ASIB_1_MAX_COMB_OT__awar_max_otf__SHIFT                                                      0x0
+#define NIC400_0_ASIB_1_MAX_COMB_OT__awar_max_oti__SHIFT                                                      0x8
+#define NIC400_0_ASIB_1_MAX_COMB_OT__awar_max_otf_MASK                                                        0x000000FFL
+#define NIC400_0_ASIB_1_MAX_COMB_OT__awar_max_oti_MASK                                                        0x00007F00L
+//NIC400_0_ASIB_1_AW_P
+#define NIC400_0_ASIB_1_AW_P__aw_p__SHIFT                                                                     0x18
+#define NIC400_0_ASIB_1_AW_P__aw_p_MASK                                                                       0xFF000000L
+//NIC400_0_ASIB_1_AW_B
+#define NIC400_0_ASIB_1_AW_B__aw_b__SHIFT                                                                     0x0
+#define NIC400_0_ASIB_1_AW_B__aw_b_MASK                                                                       0x0000FFFFL
+//NIC400_0_ASIB_1_AW_R
+#define NIC400_0_ASIB_1_AW_R__aw_r__SHIFT                                                                     0x14
+#define NIC400_0_ASIB_1_AW_R__aw_r_MASK                                                                       0xFFF00000L
+//NIC400_0_ASIB_1_AR_P
+#define NIC400_0_ASIB_1_AR_P__ar_p__SHIFT                                                                     0x18
+#define NIC400_0_ASIB_1_AR_P__ar_p_MASK                                                                       0xFF000000L
+//NIC400_0_ASIB_1_AR_B
+#define NIC400_0_ASIB_1_AR_B__ar_b__SHIFT                                                                     0x0
+#define NIC400_0_ASIB_1_AR_B__ar_b_MASK                                                                       0x0000FFFFL
+//NIC400_0_ASIB_1_AR_R
+#define NIC400_0_ASIB_1_AR_R__ar_r__SHIFT                                                                     0x14
+#define NIC400_0_ASIB_1_AR_R__ar_r_MASK                                                                       0xFFF00000L
+//NIC400_0_ASIB_1_TARGET_FC
+#define NIC400_0_ASIB_1_TARGET_FC__aw_tgt_latency__SHIFT                                                      0x0
+#define NIC400_0_ASIB_1_TARGET_FC__ar_tgt_latency__SHIFT                                                      0x10
+#define NIC400_0_ASIB_1_TARGET_FC__aw_tgt_latency_MASK                                                        0x00000FFFL
+#define NIC400_0_ASIB_1_TARGET_FC__ar_tgt_latency_MASK                                                        0x0FFF0000L
+//NIC400_0_ASIB_1_KI_FC
+#define NIC400_0_ASIB_1_KI_FC__aw_tgt_latency__SHIFT                                                          0x0
+#define NIC400_0_ASIB_1_KI_FC__ar_tgt_latency__SHIFT                                                          0x8
+#define NIC400_0_ASIB_1_KI_FC__aw_tgt_latency_MASK                                                            0x00000007L
+#define NIC400_0_ASIB_1_KI_FC__ar_tgt_latency_MASK                                                            0x00000700L
+//NIC400_0_ASIB_1_QOS_RANGE
+#define NIC400_0_ASIB_1_QOS_RANGE__aw_min_qos__SHIFT                                                          0x0
+#define NIC400_0_ASIB_1_QOS_RANGE__aw_max_qos__SHIFT                                                          0x8
+#define NIC400_0_ASIB_1_QOS_RANGE__ar_min_qos__SHIFT                                                          0x10
+#define NIC400_0_ASIB_1_QOS_RANGE__ar_max_qos__SHIFT                                                          0x18
+#define NIC400_0_ASIB_1_QOS_RANGE__aw_min_qos_MASK                                                            0x0000000FL
+#define NIC400_0_ASIB_1_QOS_RANGE__aw_max_qos_MASK                                                            0x00000F00L
+#define NIC400_0_ASIB_1_QOS_RANGE__ar_min_qos_MASK                                                            0x000F0000L
+#define NIC400_0_ASIB_1_QOS_RANGE__ar_max_qos_MASK                                                            0x0F000000L
+//NIC400_0_IB_0_FN_MOD
+#define NIC400_0_IB_0_FN_MOD__read_iss_override__SHIFT                                                        0x0
+#define NIC400_0_IB_0_FN_MOD__write_iss_override__SHIFT                                                       0x1
+#define NIC400_0_IB_0_FN_MOD__read_iss_override_MASK                                                          0x00000001L
+#define NIC400_0_IB_0_FN_MOD__write_iss_override_MASK                                                         0x00000002L
+//NIC400_1_AMIB_0_FN_MOD_BM_ISS
+#define NIC400_1_AMIB_0_FN_MOD_BM_ISS__read_iss_override__SHIFT                                               0x0
+#define NIC400_1_AMIB_0_FN_MOD_BM_ISS__write_iss_override__SHIFT                                              0x1
+#define NIC400_1_AMIB_0_FN_MOD_BM_ISS__read_iss_override_MASK                                                 0x00000001L
+#define NIC400_1_AMIB_0_FN_MOD_BM_ISS__write_iss_override_MASK                                                0x00000002L
+//NIC400_1_AMIB_0_FN_MOD
+#define NIC400_1_AMIB_0_FN_MOD__read_iss_override__SHIFT                                                      0x0
+#define NIC400_1_AMIB_0_FN_MOD__write_iss_override__SHIFT                                                     0x1
+#define NIC400_1_AMIB_0_FN_MOD__read_iss_override_MASK                                                        0x00000001L
+#define NIC400_1_AMIB_0_FN_MOD__write_iss_override_MASK                                                       0x00000002L
+//NIC400_1_ASIB_0_FN_MOD
+#define NIC400_1_ASIB_0_FN_MOD__read_iss_override__SHIFT                                                      0x0
+#define NIC400_1_ASIB_0_FN_MOD__write_iss_override__SHIFT                                                     0x1
+#define NIC400_1_ASIB_0_FN_MOD__read_iss_override_MASK                                                        0x00000001L
+#define NIC400_1_ASIB_0_FN_MOD__write_iss_override_MASK                                                       0x00000002L
+//NIC400_1_ASIB_0_QOS_CNTL
+#define NIC400_1_ASIB_0_QOS_CNTL__en_aw_rate__SHIFT                                                           0x0
+#define NIC400_1_ASIB_0_QOS_CNTL__en_ar_rate__SHIFT                                                           0x1
+#define NIC400_1_ASIB_0_QOS_CNTL__en_awar_rate__SHIFT                                                         0x2
+#define NIC400_1_ASIB_0_QOS_CNTL__en_aw_fc__SHIFT                                                             0x3
+#define NIC400_1_ASIB_0_QOS_CNTL__en_ar_fc__SHIFT                                                             0x4
+#define NIC400_1_ASIB_0_QOS_CNTL__en_aw_ot__SHIFT                                                             0x5
+#define NIC400_1_ASIB_0_QOS_CNTL__en_ar_ot__SHIFT                                                             0x6
+#define NIC400_1_ASIB_0_QOS_CNTL__en_awar_ot__SHIFT                                                           0x7
+#define NIC400_1_ASIB_0_QOS_CNTL__mode_aw_fc__SHIFT                                                           0x10
+#define NIC400_1_ASIB_0_QOS_CNTL__mode_ar_fc__SHIFT                                                           0x14
+#define NIC400_1_ASIB_0_QOS_CNTL__en_aw_rate_MASK                                                             0x00000001L
+#define NIC400_1_ASIB_0_QOS_CNTL__en_ar_rate_MASK                                                             0x00000002L
+#define NIC400_1_ASIB_0_QOS_CNTL__en_awar_rate_MASK                                                           0x00000004L
+#define NIC400_1_ASIB_0_QOS_CNTL__en_aw_fc_MASK                                                               0x00000008L
+#define NIC400_1_ASIB_0_QOS_CNTL__en_ar_fc_MASK                                                               0x00000010L
+#define NIC400_1_ASIB_0_QOS_CNTL__en_aw_ot_MASK                                                               0x00000020L
+#define NIC400_1_ASIB_0_QOS_CNTL__en_ar_ot_MASK                                                               0x00000040L
+#define NIC400_1_ASIB_0_QOS_CNTL__en_awar_ot_MASK                                                             0x00000080L
+#define NIC400_1_ASIB_0_QOS_CNTL__mode_aw_fc_MASK                                                             0x00010000L
+#define NIC400_1_ASIB_0_QOS_CNTL__mode_ar_fc_MASK                                                             0x00100000L
+//NIC400_1_ASIB_0_MAX_OT
+#define NIC400_1_ASIB_0_MAX_OT__aw_max_otf__SHIFT                                                             0x0
+#define NIC400_1_ASIB_0_MAX_OT__aw_max_oti__SHIFT                                                             0x8
+#define NIC400_1_ASIB_0_MAX_OT__ar_max_otf__SHIFT                                                             0x10
+#define NIC400_1_ASIB_0_MAX_OT__ar_max_oti__SHIFT                                                             0x18
+#define NIC400_1_ASIB_0_MAX_OT__aw_max_otf_MASK                                                               0x000000FFL
+#define NIC400_1_ASIB_0_MAX_OT__aw_max_oti_MASK                                                               0x00003F00L
+#define NIC400_1_ASIB_0_MAX_OT__ar_max_otf_MASK                                                               0x00FF0000L
+#define NIC400_1_ASIB_0_MAX_OT__ar_max_oti_MASK                                                               0x3F000000L
+//NIC400_1_ASIB_0_MAX_COMB_OT
+#define NIC400_1_ASIB_0_MAX_COMB_OT__awar_max_otf__SHIFT                                                      0x0
+#define NIC400_1_ASIB_0_MAX_COMB_OT__awar_max_oti__SHIFT                                                      0x8
+#define NIC400_1_ASIB_0_MAX_COMB_OT__awar_max_otf_MASK                                                        0x000000FFL
+#define NIC400_1_ASIB_0_MAX_COMB_OT__awar_max_oti_MASK                                                        0x00007F00L
+//NIC400_1_ASIB_0_AW_P
+#define NIC400_1_ASIB_0_AW_P__aw_p__SHIFT                                                                     0x18
+#define NIC400_1_ASIB_0_AW_P__aw_p_MASK                                                                       0xFF000000L
+//NIC400_1_ASIB_0_AW_B
+#define NIC400_1_ASIB_0_AW_B__aw_b__SHIFT                                                                     0x0
+#define NIC400_1_ASIB_0_AW_B__aw_b_MASK                                                                       0x0000FFFFL
+//NIC400_1_ASIB_0_AW_R
+#define NIC400_1_ASIB_0_AW_R__aw_r__SHIFT                                                                     0x14
+#define NIC400_1_ASIB_0_AW_R__aw_r_MASK                                                                       0xFFF00000L
+//NIC400_1_ASIB_0_AR_P
+#define NIC400_1_ASIB_0_AR_P__ar_p__SHIFT                                                                     0x18
+#define NIC400_1_ASIB_0_AR_P__ar_p_MASK                                                                       0xFF000000L
+//NIC400_1_ASIB_0_AR_B
+#define NIC400_1_ASIB_0_AR_B__ar_b__SHIFT                                                                     0x0
+#define NIC400_1_ASIB_0_AR_B__ar_b_MASK                                                                       0x0000FFFFL
+//NIC400_1_ASIB_0_AR_R
+#define NIC400_1_ASIB_0_AR_R__ar_r__SHIFT                                                                     0x14
+#define NIC400_1_ASIB_0_AR_R__ar_r_MASK                                                                       0xFFF00000L
+//NIC400_1_ASIB_0_TARGET_FC
+#define NIC400_1_ASIB_0_TARGET_FC__aw_tgt_latency__SHIFT                                                      0x0
+#define NIC400_1_ASIB_0_TARGET_FC__ar_tgt_latency__SHIFT                                                      0x10
+#define NIC400_1_ASIB_0_TARGET_FC__aw_tgt_latency_MASK                                                        0x00000FFFL
+#define NIC400_1_ASIB_0_TARGET_FC__ar_tgt_latency_MASK                                                        0x0FFF0000L
+//NIC400_1_ASIB_0_KI_FC
+#define NIC400_1_ASIB_0_KI_FC__aw_tgt_latency__SHIFT                                                          0x0
+#define NIC400_1_ASIB_0_KI_FC__ar_tgt_latency__SHIFT                                                          0x8
+#define NIC400_1_ASIB_0_KI_FC__aw_tgt_latency_MASK                                                            0x00000007L
+#define NIC400_1_ASIB_0_KI_FC__ar_tgt_latency_MASK                                                            0x00000700L
+//NIC400_1_ASIB_0_QOS_RANGE
+#define NIC400_1_ASIB_0_QOS_RANGE__aw_min_qos__SHIFT                                                          0x0
+#define NIC400_1_ASIB_0_QOS_RANGE__aw_max_qos__SHIFT                                                          0x8
+#define NIC400_1_ASIB_0_QOS_RANGE__ar_min_qos__SHIFT                                                          0x10
+#define NIC400_1_ASIB_0_QOS_RANGE__ar_max_qos__SHIFT                                                          0x18
+#define NIC400_1_ASIB_0_QOS_RANGE__aw_min_qos_MASK                                                            0x0000000FL
+#define NIC400_1_ASIB_0_QOS_RANGE__aw_max_qos_MASK                                                            0x00000F00L
+#define NIC400_1_ASIB_0_QOS_RANGE__ar_min_qos_MASK                                                            0x000F0000L
+#define NIC400_1_ASIB_0_QOS_RANGE__ar_max_qos_MASK                                                            0x0F000000L
+//NIC400_1_ASIB_1_FN_MOD
+#define NIC400_1_ASIB_1_FN_MOD__read_iss_override__SHIFT                                                      0x0
+#define NIC400_1_ASIB_1_FN_MOD__write_iss_override__SHIFT                                                     0x1
+#define NIC400_1_ASIB_1_FN_MOD__read_iss_override_MASK                                                        0x00000001L
+#define NIC400_1_ASIB_1_FN_MOD__write_iss_override_MASK                                                       0x00000002L
+//NIC400_1_ASIB_1_QOS_CNTL
+#define NIC400_1_ASIB_1_QOS_CNTL__en_aw_rate__SHIFT                                                           0x0
+#define NIC400_1_ASIB_1_QOS_CNTL__en_ar_rate__SHIFT                                                           0x1
+#define NIC400_1_ASIB_1_QOS_CNTL__en_awar_rate__SHIFT                                                         0x2
+#define NIC400_1_ASIB_1_QOS_CNTL__en_aw_fc__SHIFT                                                             0x3
+#define NIC400_1_ASIB_1_QOS_CNTL__en_ar_fc__SHIFT                                                             0x4
+#define NIC400_1_ASIB_1_QOS_CNTL__en_aw_ot__SHIFT                                                             0x5
+#define NIC400_1_ASIB_1_QOS_CNTL__en_ar_ot__SHIFT                                                             0x6
+#define NIC400_1_ASIB_1_QOS_CNTL__en_awar_ot__SHIFT                                                           0x7
+#define NIC400_1_ASIB_1_QOS_CNTL__mode_aw_fc__SHIFT                                                           0x10
+#define NIC400_1_ASIB_1_QOS_CNTL__mode_ar_fc__SHIFT                                                           0x14
+#define NIC400_1_ASIB_1_QOS_CNTL__en_aw_rate_MASK                                                             0x00000001L
+#define NIC400_1_ASIB_1_QOS_CNTL__en_ar_rate_MASK                                                             0x00000002L
+#define NIC400_1_ASIB_1_QOS_CNTL__en_awar_rate_MASK                                                           0x00000004L
+#define NIC400_1_ASIB_1_QOS_CNTL__en_aw_fc_MASK                                                               0x00000008L
+#define NIC400_1_ASIB_1_QOS_CNTL__en_ar_fc_MASK                                                               0x00000010L
+#define NIC400_1_ASIB_1_QOS_CNTL__en_aw_ot_MASK                                                               0x00000020L
+#define NIC400_1_ASIB_1_QOS_CNTL__en_ar_ot_MASK                                                               0x00000040L
+#define NIC400_1_ASIB_1_QOS_CNTL__en_awar_ot_MASK                                                             0x00000080L
+#define NIC400_1_ASIB_1_QOS_CNTL__mode_aw_fc_MASK                                                             0x00010000L
+#define NIC400_1_ASIB_1_QOS_CNTL__mode_ar_fc_MASK                                                             0x00100000L
+//NIC400_1_ASIB_1_MAX_OT
+#define NIC400_1_ASIB_1_MAX_OT__aw_max_otf__SHIFT                                                             0x0
+#define NIC400_1_ASIB_1_MAX_OT__aw_max_oti__SHIFT                                                             0x8
+#define NIC400_1_ASIB_1_MAX_OT__ar_max_otf__SHIFT                                                             0x10
+#define NIC400_1_ASIB_1_MAX_OT__ar_max_oti__SHIFT                                                             0x18
+#define NIC400_1_ASIB_1_MAX_OT__aw_max_otf_MASK                                                               0x000000FFL
+#define NIC400_1_ASIB_1_MAX_OT__aw_max_oti_MASK                                                               0x00003F00L
+#define NIC400_1_ASIB_1_MAX_OT__ar_max_otf_MASK                                                               0x00FF0000L
+#define NIC400_1_ASIB_1_MAX_OT__ar_max_oti_MASK                                                               0x3F000000L
+//NIC400_1_ASIB_1_MAX_COMB_OT
+#define NIC400_1_ASIB_1_MAX_COMB_OT__awar_max_otf__SHIFT                                                      0x0
+#define NIC400_1_ASIB_1_MAX_COMB_OT__awar_max_oti__SHIFT                                                      0x8
+#define NIC400_1_ASIB_1_MAX_COMB_OT__awar_max_otf_MASK                                                        0x000000FFL
+#define NIC400_1_ASIB_1_MAX_COMB_OT__awar_max_oti_MASK                                                        0x00007F00L
+//NIC400_1_ASIB_1_AW_P
+#define NIC400_1_ASIB_1_AW_P__aw_p__SHIFT                                                                     0x18
+#define NIC400_1_ASIB_1_AW_P__aw_p_MASK                                                                       0xFF000000L
+//NIC400_1_ASIB_1_AW_B
+#define NIC400_1_ASIB_1_AW_B__aw_b__SHIFT                                                                     0x0
+#define NIC400_1_ASIB_1_AW_B__aw_b_MASK                                                                       0x0000FFFFL
+//NIC400_1_ASIB_1_AW_R
+#define NIC400_1_ASIB_1_AW_R__aw_r__SHIFT                                                                     0x14
+#define NIC400_1_ASIB_1_AW_R__aw_r_MASK                                                                       0xFFF00000L
+//NIC400_1_ASIB_1_AR_P
+#define NIC400_1_ASIB_1_AR_P__ar_p__SHIFT                                                                     0x18
+#define NIC400_1_ASIB_1_AR_P__ar_p_MASK                                                                       0xFF000000L
+//NIC400_1_ASIB_1_AR_B
+#define NIC400_1_ASIB_1_AR_B__ar_b__SHIFT                                                                     0x0
+#define NIC400_1_ASIB_1_AR_B__ar_b_MASK                                                                       0x0000FFFFL
+//NIC400_1_ASIB_1_AR_R
+#define NIC400_1_ASIB_1_AR_R__ar_r__SHIFT                                                                     0x14
+#define NIC400_1_ASIB_1_AR_R__ar_r_MASK                                                                       0xFFF00000L
+//NIC400_1_ASIB_1_TARGET_FC
+#define NIC400_1_ASIB_1_TARGET_FC__aw_tgt_latency__SHIFT                                                      0x0
+#define NIC400_1_ASIB_1_TARGET_FC__ar_tgt_latency__SHIFT                                                      0x10
+#define NIC400_1_ASIB_1_TARGET_FC__aw_tgt_latency_MASK                                                        0x00000FFFL
+#define NIC400_1_ASIB_1_TARGET_FC__ar_tgt_latency_MASK                                                        0x0FFF0000L
+//NIC400_1_ASIB_1_KI_FC
+#define NIC400_1_ASIB_1_KI_FC__aw_tgt_latency__SHIFT                                                          0x0
+#define NIC400_1_ASIB_1_KI_FC__ar_tgt_latency__SHIFT                                                          0x8
+#define NIC400_1_ASIB_1_KI_FC__aw_tgt_latency_MASK                                                            0x00000007L
+#define NIC400_1_ASIB_1_KI_FC__ar_tgt_latency_MASK                                                            0x00000700L
+//NIC400_1_ASIB_1_QOS_RANGE
+#define NIC400_1_ASIB_1_QOS_RANGE__aw_min_qos__SHIFT                                                          0x0
+#define NIC400_1_ASIB_1_QOS_RANGE__aw_max_qos__SHIFT                                                          0x8
+#define NIC400_1_ASIB_1_QOS_RANGE__ar_min_qos__SHIFT                                                          0x10
+#define NIC400_1_ASIB_1_QOS_RANGE__ar_max_qos__SHIFT                                                          0x18
+#define NIC400_1_ASIB_1_QOS_RANGE__aw_min_qos_MASK                                                            0x0000000FL
+#define NIC400_1_ASIB_1_QOS_RANGE__aw_max_qos_MASK                                                            0x00000F00L
+#define NIC400_1_ASIB_1_QOS_RANGE__ar_min_qos_MASK                                                            0x000F0000L
+#define NIC400_1_ASIB_1_QOS_RANGE__ar_max_qos_MASK                                                            0x0F000000L
+//NIC400_1_IB_0_FN_MOD
+#define NIC400_1_IB_0_FN_MOD__read_iss_override__SHIFT                                                        0x0
+#define NIC400_1_IB_0_FN_MOD__write_iss_override__SHIFT                                                       0x1
+#define NIC400_1_IB_0_FN_MOD__read_iss_override_MASK                                                          0x00000001L
+#define NIC400_1_IB_0_FN_MOD__write_iss_override_MASK                                                         0x00000002L
+
+
+// addressBlock: nbif0_nbif0_syshub_mmreg_syshubdec
+//SYSHUB_INDEX
+#define SYSHUB_INDEX__INDEX__SHIFT                                                                            0x0
+#define SYSHUB_INDEX__INDEX_MASK                                                                              0xFFFFFFFFL
+//SYSHUB_DATA
+#define SYSHUB_DATA__DATA__SHIFT                                                                              0x0
+#define SYSHUB_DATA__DATA_MASK                                                                                0xFFFFFFFFL
+
+
+// addressBlock: nbif0_nbif0_bif_cfg_dev0_rc_bifcfgdecp
+//BIF_CFG_DEV0_RC_VENDOR_ID
+#define BIF_CFG_DEV0_RC_VENDOR_ID__VENDOR_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_VENDOR_ID__VENDOR_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV0_RC_DEVICE_ID
+#define BIF_CFG_DEV0_RC_DEVICE_ID__DEVICE_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_DEVICE_ID__DEVICE_ID_MASK                                                             0xFFFFL
+//BIF_CFG_DEV0_RC_COMMAND
+#define BIF_CFG_DEV0_RC_COMMAND__IOEN_DN__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_COMMAND__MEMEN_DN__SHIFT                                                              0x1
+#define BIF_CFG_DEV0_RC_COMMAND__BUS_MASTER_EN__SHIFT                                                         0x2
+#define BIF_CFG_DEV0_RC_COMMAND__SPECIAL_CYCLE_EN__SHIFT                                                      0x3
+#define BIF_CFG_DEV0_RC_COMMAND__MEM_WRITE_INVALIDATE_EN__SHIFT                                               0x4
+#define BIF_CFG_DEV0_RC_COMMAND__PAL_SNOOP_EN__SHIFT                                                          0x5
+#define BIF_CFG_DEV0_RC_COMMAND__PARITY_ERROR_RESPONSE__SHIFT                                                 0x6
+#define BIF_CFG_DEV0_RC_COMMAND__AD_STEPPING__SHIFT                                                           0x7
+#define BIF_CFG_DEV0_RC_COMMAND__SERR_EN__SHIFT                                                               0x8
+#define BIF_CFG_DEV0_RC_COMMAND__FAST_B2B_EN__SHIFT                                                           0x9
+#define BIF_CFG_DEV0_RC_COMMAND__INT_DIS__SHIFT                                                               0xa
+#define BIF_CFG_DEV0_RC_COMMAND__IOEN_DN_MASK                                                                 0x0001L
+#define BIF_CFG_DEV0_RC_COMMAND__MEMEN_DN_MASK                                                                0x0002L
+#define BIF_CFG_DEV0_RC_COMMAND__BUS_MASTER_EN_MASK                                                           0x0004L
+#define BIF_CFG_DEV0_RC_COMMAND__SPECIAL_CYCLE_EN_MASK                                                        0x0008L
+#define BIF_CFG_DEV0_RC_COMMAND__MEM_WRITE_INVALIDATE_EN_MASK                                                 0x0010L
+#define BIF_CFG_DEV0_RC_COMMAND__PAL_SNOOP_EN_MASK                                                            0x0020L
+#define BIF_CFG_DEV0_RC_COMMAND__PARITY_ERROR_RESPONSE_MASK                                                   0x0040L
+#define BIF_CFG_DEV0_RC_COMMAND__AD_STEPPING_MASK                                                             0x0080L
+#define BIF_CFG_DEV0_RC_COMMAND__SERR_EN_MASK                                                                 0x0100L
+#define BIF_CFG_DEV0_RC_COMMAND__FAST_B2B_EN_MASK                                                             0x0200L
+#define BIF_CFG_DEV0_RC_COMMAND__INT_DIS_MASK                                                                 0x0400L
+//BIF_CFG_DEV0_RC_STATUS
+#define BIF_CFG_DEV0_RC_STATUS__IMMEDIATE_READINESS__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_RC_STATUS__INT_STATUS__SHIFT                                                             0x3
+#define BIF_CFG_DEV0_RC_STATUS__CAP_LIST__SHIFT                                                               0x4
+#define BIF_CFG_DEV0_RC_STATUS__PCI_66_CAP__SHIFT                                                             0x5
+#define BIF_CFG_DEV0_RC_STATUS__FAST_BACK_CAPABLE__SHIFT                                                      0x7
+#define BIF_CFG_DEV0_RC_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                               0x8
+#define BIF_CFG_DEV0_RC_STATUS__DEVSEL_TIMING__SHIFT                                                          0x9
+#define BIF_CFG_DEV0_RC_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                    0xb
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                  0xc
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                  0xd
+#define BIF_CFG_DEV0_RC_STATUS__SIGNALED_SYSTEM_ERROR__SHIFT                                                  0xe
+#define BIF_CFG_DEV0_RC_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                  0xf
+#define BIF_CFG_DEV0_RC_STATUS__IMMEDIATE_READINESS_MASK                                                      0x0001L
+#define BIF_CFG_DEV0_RC_STATUS__INT_STATUS_MASK                                                               0x0008L
+#define BIF_CFG_DEV0_RC_STATUS__CAP_LIST_MASK                                                                 0x0010L
+#define BIF_CFG_DEV0_RC_STATUS__PCI_66_CAP_MASK                                                               0x0020L
+#define BIF_CFG_DEV0_RC_STATUS__FAST_BACK_CAPABLE_MASK                                                        0x0080L
+#define BIF_CFG_DEV0_RC_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_RC_STATUS__DEVSEL_TIMING_MASK                                                            0x0600L
+#define BIF_CFG_DEV0_RC_STATUS__SIGNAL_TARGET_ABORT_MASK                                                      0x0800L
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_TARGET_ABORT_MASK                                                    0x1000L
+#define BIF_CFG_DEV0_RC_STATUS__RECEIVED_MASTER_ABORT_MASK                                                    0x2000L
+#define BIF_CFG_DEV0_RC_STATUS__SIGNALED_SYSTEM_ERROR_MASK                                                    0x4000L
+#define BIF_CFG_DEV0_RC_STATUS__PARITY_ERROR_DETECTED_MASK                                                    0x8000L
+//BIF_CFG_DEV0_RC_REVISION_ID
+#define BIF_CFG_DEV0_RC_REVISION_ID__MINOR_REV_ID__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC_REVISION_ID__MAJOR_REV_ID__SHIFT                                                      0x4
+#define BIF_CFG_DEV0_RC_REVISION_ID__MINOR_REV_ID_MASK                                                        0x0FL
+#define BIF_CFG_DEV0_RC_REVISION_ID__MAJOR_REV_ID_MASK                                                        0xF0L
+//BIF_CFG_DEV0_RC_PROG_INTERFACE
+#define BIF_CFG_DEV0_RC_PROG_INTERFACE__PROG_INTERFACE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_PROG_INTERFACE__PROG_INTERFACE_MASK                                                   0xFFL
+//BIF_CFG_DEV0_RC_SUB_CLASS
+#define BIF_CFG_DEV0_RC_SUB_CLASS__SUB_CLASS__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_SUB_CLASS__SUB_CLASS_MASK                                                             0xFFL
+//BIF_CFG_DEV0_RC_BASE_CLASS
+#define BIF_CFG_DEV0_RC_BASE_CLASS__BASE_CLASS__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_BASE_CLASS__BASE_CLASS_MASK                                                           0xFFL
+//BIF_CFG_DEV0_RC_CACHE_LINE
+#define BIF_CFG_DEV0_RC_CACHE_LINE__CACHE_LINE_SIZE__SHIFT                                                    0x0
+#define BIF_CFG_DEV0_RC_CACHE_LINE__CACHE_LINE_SIZE_MASK                                                      0xFFL
+//BIF_CFG_DEV0_RC_LATENCY
+#define BIF_CFG_DEV0_RC_LATENCY__LATENCY_TIMER__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_LATENCY__LATENCY_TIMER_MASK                                                           0xFFL
+//BIF_CFG_DEV0_RC_HEADER
+#define BIF_CFG_DEV0_RC_HEADER__HEADER_TYPE__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_RC_HEADER__DEVICE_TYPE__SHIFT                                                            0x7
+#define BIF_CFG_DEV0_RC_HEADER__HEADER_TYPE_MASK                                                              0x7FL
+#define BIF_CFG_DEV0_RC_HEADER__DEVICE_TYPE_MASK                                                              0x80L
+//BIF_CFG_DEV0_RC_BIST
+#define BIF_CFG_DEV0_RC_BIST__BIST_COMP__SHIFT                                                                0x0
+#define BIF_CFG_DEV0_RC_BIST__BIST_STRT__SHIFT                                                                0x6
+#define BIF_CFG_DEV0_RC_BIST__BIST_CAP__SHIFT                                                                 0x7
+#define BIF_CFG_DEV0_RC_BIST__BIST_COMP_MASK                                                                  0x0FL
+#define BIF_CFG_DEV0_RC_BIST__BIST_STRT_MASK                                                                  0x40L
+#define BIF_CFG_DEV0_RC_BIST__BIST_CAP_MASK                                                                   0x80L
+//BIF_CFG_DEV0_RC_BASE_ADDR_1
+#define BIF_CFG_DEV0_RC_BASE_ADDR_1__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_BASE_ADDR_1__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_BASE_ADDR_2
+#define BIF_CFG_DEV0_RC_BASE_ADDR_2__BASE_ADDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_BASE_ADDR_2__BASE_ADDR_MASK                                                           0xFFFFFFFFL
+//SUB_BUS_NUMBER_LATENCY
+#define SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS__SHIFT                                                            0x0
+#define SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS__SHIFT                                                          0x8
+#define SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM__SHIFT                                                            0x10
+#define SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER__SHIFT                                                0x18
+#define SUB_BUS_NUMBER_LATENCY__PRIMARY_BUS_MASK                                                              0x000000FFL
+#define SUB_BUS_NUMBER_LATENCY__SECONDARY_BUS_MASK                                                            0x0000FF00L
+#define SUB_BUS_NUMBER_LATENCY__SUB_BUS_NUM_MASK                                                              0x00FF0000L
+#define SUB_BUS_NUMBER_LATENCY__SECONDARY_LATENCY_TIMER_MASK                                                  0xFF000000L
+//IO_BASE_LIMIT
+#define IO_BASE_LIMIT__IO_BASE_TYPE__SHIFT                                                                    0x0
+#define IO_BASE_LIMIT__IO_BASE__SHIFT                                                                         0x4
+#define IO_BASE_LIMIT__IO_LIMIT_TYPE__SHIFT                                                                   0x8
+#define IO_BASE_LIMIT__IO_LIMIT__SHIFT                                                                        0xc
+#define IO_BASE_LIMIT__IO_BASE_TYPE_MASK                                                                      0x000FL
+#define IO_BASE_LIMIT__IO_BASE_MASK                                                                           0x00F0L
+#define IO_BASE_LIMIT__IO_LIMIT_TYPE_MASK                                                                     0x0F00L
+#define IO_BASE_LIMIT__IO_LIMIT_MASK                                                                          0xF000L
+//SECONDARY_STATUS
+#define SECONDARY_STATUS__PCI_66_CAP__SHIFT                                                                   0x5
+#define SECONDARY_STATUS__FAST_BACK_CAPABLE__SHIFT                                                            0x7
+#define SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR__SHIFT                                                     0x8
+#define SECONDARY_STATUS__DEVSEL_TIMING__SHIFT                                                                0x9
+#define SECONDARY_STATUS__SIGNAL_TARGET_ABORT__SHIFT                                                          0xb
+#define SECONDARY_STATUS__RECEIVED_TARGET_ABORT__SHIFT                                                        0xc
+#define SECONDARY_STATUS__RECEIVED_MASTER_ABORT__SHIFT                                                        0xd
+#define SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR__SHIFT                                                        0xe
+#define SECONDARY_STATUS__PARITY_ERROR_DETECTED__SHIFT                                                        0xf
+#define SECONDARY_STATUS__PCI_66_CAP_MASK                                                                     0x0020L
+#define SECONDARY_STATUS__FAST_BACK_CAPABLE_MASK                                                              0x0080L
+#define SECONDARY_STATUS__MASTER_DATA_PARITY_ERROR_MASK                                                       0x0100L
+#define SECONDARY_STATUS__DEVSEL_TIMING_MASK                                                                  0x0600L
+#define SECONDARY_STATUS__SIGNAL_TARGET_ABORT_MASK                                                            0x0800L
+#define SECONDARY_STATUS__RECEIVED_TARGET_ABORT_MASK                                                          0x1000L
+#define SECONDARY_STATUS__RECEIVED_MASTER_ABORT_MASK                                                          0x2000L
+#define SECONDARY_STATUS__RECEIVED_SYSTEM_ERROR_MASK                                                          0x4000L
+#define SECONDARY_STATUS__PARITY_ERROR_DETECTED_MASK                                                          0x8000L
+//MEM_BASE_LIMIT
+#define MEM_BASE_LIMIT__MEM_BASE_TYPE__SHIFT                                                                  0x0
+#define MEM_BASE_LIMIT__MEM_BASE_31_20__SHIFT                                                                 0x4
+#define MEM_BASE_LIMIT__MEM_LIMIT_TYPE__SHIFT                                                                 0x10
+#define MEM_BASE_LIMIT__MEM_LIMIT_31_20__SHIFT                                                                0x14
+#define MEM_BASE_LIMIT__MEM_BASE_TYPE_MASK                                                                    0x0000000FL
+#define MEM_BASE_LIMIT__MEM_BASE_31_20_MASK                                                                   0x0000FFF0L
+#define MEM_BASE_LIMIT__MEM_LIMIT_TYPE_MASK                                                                   0x000F0000L
+#define MEM_BASE_LIMIT__MEM_LIMIT_31_20_MASK                                                                  0xFFF00000L
+//PREF_BASE_LIMIT
+#define PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE__SHIFT                                                            0x0
+#define PREF_BASE_LIMIT__PREF_MEM_BASE_31_20__SHIFT                                                           0x4
+#define PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE__SHIFT                                                           0x10
+#define PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20__SHIFT                                                          0x14
+#define PREF_BASE_LIMIT__PREF_MEM_BASE_TYPE_MASK                                                              0x0000000FL
+#define PREF_BASE_LIMIT__PREF_MEM_BASE_31_20_MASK                                                             0x0000FFF0L
+#define PREF_BASE_LIMIT__PREF_MEM_LIMIT_TYPE_MASK                                                             0x000F0000L
+#define PREF_BASE_LIMIT__PREF_MEM_LIMIT_31_20_MASK                                                            0xFFF00000L
+//PREF_BASE_UPPER
+#define PREF_BASE_UPPER__PREF_BASE_UPPER__SHIFT                                                               0x0
+#define PREF_BASE_UPPER__PREF_BASE_UPPER_MASK                                                                 0xFFFFFFFFL
+//PREF_LIMIT_UPPER
+#define PREF_LIMIT_UPPER__PREF_LIMIT_UPPER__SHIFT                                                             0x0
+#define PREF_LIMIT_UPPER__PREF_LIMIT_UPPER_MASK                                                               0xFFFFFFFFL
+//IO_BASE_LIMIT_HI
+#define IO_BASE_LIMIT_HI__IO_BASE_31_16__SHIFT                                                                0x0
+#define IO_BASE_LIMIT_HI__IO_LIMIT_31_16__SHIFT                                                               0x10
+#define IO_BASE_LIMIT_HI__IO_BASE_31_16_MASK                                                                  0x0000FFFFL
+#define IO_BASE_LIMIT_HI__IO_LIMIT_31_16_MASK                                                                 0xFFFF0000L
+//BIF_CFG_DEV0_RC_CAP_PTR
+#define BIF_CFG_DEV0_RC_CAP_PTR__CAP_PTR__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_CAP_PTR__CAP_PTR_MASK                                                                 0xFFL
+//BIF_CFG_DEV0_RC_ROM_BASE_ADDR
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_ENABLE__SHIFT                                                      0x0
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS__SHIFT                                           0x1
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__BASE_ADDR__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_ENABLE_MASK                                                        0x00000001L
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_STATUS_MASK                                             0x0000000EL
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__ROM_VALIDATION_DETAILS_MASK                                            0x000000F0L
+#define BIF_CFG_DEV0_RC_ROM_BASE_ADDR__BASE_ADDR_MASK                                                         0xFFFFF800L
+//BIF_CFG_DEV0_RC_INTERRUPT_LINE
+#define BIF_CFG_DEV0_RC_INTERRUPT_LINE__INTERRUPT_LINE__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_INTERRUPT_LINE__INTERRUPT_LINE_MASK                                                   0xFFL
+//BIF_CFG_DEV0_RC_INTERRUPT_PIN
+#define BIF_CFG_DEV0_RC_INTERRUPT_PIN__INTERRUPT_PIN__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_INTERRUPT_PIN__INTERRUPT_PIN_MASK                                                     0xFFL
+//IRQ_BRIDGE_CNTL
+#define IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN__SHIFT                                                            0x0
+#define IRQ_BRIDGE_CNTL__SERR_EN__SHIFT                                                                       0x1
+#define IRQ_BRIDGE_CNTL__ISA_EN__SHIFT                                                                        0x2
+#define IRQ_BRIDGE_CNTL__VGA_EN__SHIFT                                                                        0x3
+#define IRQ_BRIDGE_CNTL__VGA_DEC__SHIFT                                                                       0x4
+#define IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE__SHIFT                                                             0x5
+#define IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET__SHIFT                                                           0x6
+#define IRQ_BRIDGE_CNTL__FAST_B2B_EN__SHIFT                                                                   0x7
+#define IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER__SHIFT                                                         0x8
+#define IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER__SHIFT                                                       0x9
+#define IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS__SHIFT                                                          0xa
+#define IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE__SHIFT                                                     0xb
+#define IRQ_BRIDGE_CNTL__PARITY_RESPONSE_EN_MASK                                                              0x0001L
+#define IRQ_BRIDGE_CNTL__SERR_EN_MASK                                                                         0x0002L
+#define IRQ_BRIDGE_CNTL__ISA_EN_MASK                                                                          0x0004L
+#define IRQ_BRIDGE_CNTL__VGA_EN_MASK                                                                          0x0008L
+#define IRQ_BRIDGE_CNTL__VGA_DEC_MASK                                                                         0x0010L
+#define IRQ_BRIDGE_CNTL__MASTER_ABORT_MODE_MASK                                                               0x0020L
+#define IRQ_BRIDGE_CNTL__SECONDARY_BUS_RESET_MASK                                                             0x0040L
+#define IRQ_BRIDGE_CNTL__FAST_B2B_EN_MASK                                                                     0x0080L
+#define IRQ_BRIDGE_CNTL__PRIMARY_DISCARD_TIMER_MASK                                                           0x0100L
+#define IRQ_BRIDGE_CNTL__SECONDARY_DISCARD_TIMER_MASK                                                         0x0200L
+#define IRQ_BRIDGE_CNTL__DISCARD_TIMER_STATUS_MASK                                                            0x0400L
+#define IRQ_BRIDGE_CNTL__DISCARD_TIMER_SERR_ENABLE_MASK                                                       0x0800L
+//EXT_BRIDGE_CNTL
+#define EXT_BRIDGE_CNTL__IO_PORT_80_EN__SHIFT                                                                 0x0
+#define EXT_BRIDGE_CNTL__IO_PORT_80_EN_MASK                                                                   0x01L
+//BIF_CFG_DEV0_RC_PMI_CAP_LIST
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV0_RC_PMI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV0_RC_PMI_CAP
+#define BIF_CFG_DEV0_RC_PMI_CAP__VERSION__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_CLOCK__SHIFT                                                             0x3
+#define BIF_CFG_DEV0_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0__SHIFT                                   0x4
+#define BIF_CFG_DEV0_RC_PMI_CAP__DEV_SPECIFIC_INIT__SHIFT                                                     0x5
+#define BIF_CFG_DEV0_RC_PMI_CAP__AUX_CURRENT__SHIFT                                                           0x6
+#define BIF_CFG_DEV0_RC_PMI_CAP__D1_SUPPORT__SHIFT                                                            0x9
+#define BIF_CFG_DEV0_RC_PMI_CAP__D2_SUPPORT__SHIFT                                                            0xa
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_SUPPORT__SHIFT                                                           0xb
+#define BIF_CFG_DEV0_RC_PMI_CAP__VERSION_MASK                                                                 0x0007L
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_CLOCK_MASK                                                               0x0008L
+#define BIF_CFG_DEV0_RC_PMI_CAP__IMMEDIATE_READINESS_ON_RETURN_TO_D0_MASK                                     0x0010L
+#define BIF_CFG_DEV0_RC_PMI_CAP__DEV_SPECIFIC_INIT_MASK                                                       0x0020L
+#define BIF_CFG_DEV0_RC_PMI_CAP__AUX_CURRENT_MASK                                                             0x01C0L
+#define BIF_CFG_DEV0_RC_PMI_CAP__D1_SUPPORT_MASK                                                              0x0200L
+#define BIF_CFG_DEV0_RC_PMI_CAP__D2_SUPPORT_MASK                                                              0x0400L
+#define BIF_CFG_DEV0_RC_PMI_CAP__PME_SUPPORT_MASK                                                             0xF800L
+//BIF_CFG_DEV0_RC_PMI_STATUS_CNTL
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__POWER_STATE__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__NO_SOFT_RESET__SHIFT                                                 0x3
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_EN__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SELECT__SHIFT                                                   0x9
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SCALE__SHIFT                                                    0xd
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_STATUS__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT__SHIFT                                                 0x16
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__BUS_PWR_EN__SHIFT                                                    0x17
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PMI_DATA__SHIFT                                                      0x18
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__POWER_STATE_MASK                                                     0x00000003L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__NO_SOFT_RESET_MASK                                                   0x00000008L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_EN_MASK                                                          0x00000100L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SELECT_MASK                                                     0x00001E00L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__DATA_SCALE_MASK                                                      0x00006000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PME_STATUS_MASK                                                      0x00008000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__B2_B3_SUPPORT_MASK                                                   0x00400000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__BUS_PWR_EN_MASK                                                      0x00800000L
+#define BIF_CFG_DEV0_RC_PMI_STATUS_CNTL__PMI_DATA_MASK                                                        0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__CAP_ID__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__NEXT_PTR__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__CAP_ID_MASK                                                            0x00FFL
+#define BIF_CFG_DEV0_RC_PCIE_CAP_LIST__NEXT_PTR_MASK                                                          0xFF00L
+//BIF_CFG_DEV0_RC_PCIE_CAP
+#define BIF_CFG_DEV0_RC_PCIE_CAP__VERSION__SHIFT                                                              0x0
+#define BIF_CFG_DEV0_RC_PCIE_CAP__DEVICE_TYPE__SHIFT                                                          0x4
+#define BIF_CFG_DEV0_RC_PCIE_CAP__SLOT_IMPLEMENTED__SHIFT                                                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_CAP__INT_MESSAGE_NUM__SHIFT                                                      0x9
+#define BIF_CFG_DEV0_RC_PCIE_CAP__FLIT_MODE_SUPPORT__SHIFT                                                    0xf
+#define BIF_CFG_DEV0_RC_PCIE_CAP__VERSION_MASK                                                                0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_CAP__DEVICE_TYPE_MASK                                                            0x00F0L
+#define BIF_CFG_DEV0_RC_PCIE_CAP__SLOT_IMPLEMENTED_MASK                                                       0x0100L
+#define BIF_CFG_DEV0_RC_PCIE_CAP__INT_MESSAGE_NUM_MASK                                                        0x3E00L
+#define BIF_CFG_DEV0_RC_PCIE_CAP__FLIT_MODE_SUPPORT_MASK                                                      0x8000L
+//BIF_CFG_DEV0_RC_DEVICE_CAP
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__PHANTOM_FUNC__SHIFT                                                       0x3
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__EXTENDED_TAG__SHIFT                                                       0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY__SHIFT                                              0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING__SHIFT                                           0xf
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__RX_MPS_FIXED__SHIFT                                                       0x11
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT__SHIFT                                          0x12
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE__SHIFT                                          0x1a
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__FLR_CAPABLE__SHIFT                                                        0x1c
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__MIXED_MPS_SUPPORTED__SHIFT                                                0x1d
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__MAX_PAYLOAD_SUPPORT_MASK                                                  0x00000007L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__PHANTOM_FUNC_MASK                                                         0x00000018L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__EXTENDED_TAG_MASK                                                         0x00000020L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L0S_ACCEPTABLE_LATENCY_MASK                                               0x000001C0L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__L1_ACCEPTABLE_LATENCY_MASK                                                0x00000E00L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ROLE_BASED_ERR_REPORTING_MASK                                             0x00008000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__ERR_COR_SUBCLASS_CAPABLE_MASK                                             0x00010000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__RX_MPS_FIXED_MASK                                                         0x00020000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_LIMIT_MASK                                            0x03FC0000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__CAPTURED_SLOT_POWER_SCALE_MASK                                            0x0C000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__FLR_CAPABLE_MASK                                                          0x10000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP__MIXED_MPS_SUPPORTED_MASK                                                  0x20000000L
+//BIF_CFG_DEV0_RC_DEVICE_CNTL
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__CORR_ERR_EN__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NON_FATAL_ERR_EN__SHIFT                                                  0x1
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__FATAL_ERR_EN__SHIFT                                                      0x2
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__USR_REPORT_EN__SHIFT                                                     0x3
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__RELAXED_ORD_EN__SHIFT                                                    0x4
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE__SHIFT                                                  0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__EXTENDED_TAG_EN__SHIFT                                                   0x8
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__PHANTOM_FUNC_EN__SHIFT                                                   0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__AUX_POWER_PM_EN__SHIFT                                                   0xa
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NO_SNOOP_EN__SHIFT                                                       0xb
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE__SHIFT                                             0xc
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN__SHIFT                                               0xf
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__CORR_ERR_EN_MASK                                                         0x0001L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NON_FATAL_ERR_EN_MASK                                                    0x0002L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__FATAL_ERR_EN_MASK                                                        0x0004L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__USR_REPORT_EN_MASK                                                       0x0008L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__RELAXED_ORD_EN_MASK                                                      0x0010L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_PAYLOAD_SIZE_MASK                                                    0x00E0L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__EXTENDED_TAG_EN_MASK                                                     0x0100L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__PHANTOM_FUNC_EN_MASK                                                     0x0200L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__AUX_POWER_PM_EN_MASK                                                     0x0400L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__NO_SNOOP_EN_MASK                                                         0x0800L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__MAX_READ_REQUEST_SIZE_MASK                                               0x7000L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL__BRIDGE_CFG_RETRY_EN_MASK                                                 0x8000L
+//BIF_CFG_DEV0_RC_DEVICE_STATUS
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__CORR_ERR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__NON_FATAL_ERR__SHIFT                                                   0x1
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__FATAL_ERR__SHIFT                                                       0x2
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__USR_DETECTED__SHIFT                                                    0x3
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__AUX_PWR__SHIFT                                                         0x4
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__TRANSACTIONS_PEND__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED__SHIFT                                   0x6
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__CORR_ERR_MASK                                                          0x0001L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__NON_FATAL_ERR_MASK                                                     0x0002L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__FATAL_ERR_MASK                                                         0x0004L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__USR_DETECTED_MASK                                                      0x0008L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__AUX_PWR_MASK                                                           0x0010L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__TRANSACTIONS_PEND_MASK                                                 0x0020L
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS__EMER_POWER_REDUCTION_DETECTED_MASK                                     0x0040L
+//BIF_CFG_DEV0_RC_LINK_CAP
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_SPEED__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_WIDTH__SHIFT                                                           0x4
+#define BIF_CFG_DEV0_RC_LINK_CAP__PM_SUPPORT__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_RC_LINK_CAP__L0S_EXIT_LATENCY__SHIFT                                                     0xc
+#define BIF_CFG_DEV0_RC_LINK_CAP__L1_EXIT_LATENCY__SHIFT                                                      0xf
+#define BIF_CFG_DEV0_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT__SHIFT                                               0x12
+#define BIF_CFG_DEV0_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING__SHIFT                                          0x13
+#define BIF_CFG_DEV0_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP__SHIFT                                             0x15
+#define BIF_CFG_DEV0_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE__SHIFT                                          0x16
+#define BIF_CFG_DEV0_RC_LINK_CAP__PORT_NUMBER__SHIFT                                                          0x18
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_SPEED_MASK                                                             0x0000000FL
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_WIDTH_MASK                                                             0x000003F0L
+#define BIF_CFG_DEV0_RC_LINK_CAP__PM_SUPPORT_MASK                                                             0x00000C00L
+#define BIF_CFG_DEV0_RC_LINK_CAP__L0S_EXIT_LATENCY_MASK                                                       0x00007000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__L1_EXIT_LATENCY_MASK                                                        0x00038000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__CLOCK_POWER_MANAGEMENT_MASK                                                 0x00040000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__SURPRISE_DOWN_ERR_REPORTING_MASK                                            0x00080000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__DL_ACTIVE_REPORTING_CAPABLE_MASK                                            0x00100000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__LINK_BW_NOTIFICATION_CAP_MASK                                               0x00200000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__ASPM_OPTIONALITY_COMPLIANCE_MASK                                            0x00400000L
+#define BIF_CFG_DEV0_RC_LINK_CAP__PORT_NUMBER_MASK                                                            0xFF000000L
+//BIF_CFG_DEV0_RC_LINK_CNTL
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PM_CONTROL__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B__SHIFT                                        0x2
+#define BIF_CFG_DEV0_RC_LINK_CNTL__READ_CPL_BOUNDARY__SHIFT                                                   0x3
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_DIS__SHIFT                                                            0x4
+#define BIF_CFG_DEV0_RC_LINK_CNTL__RETRAIN_LINK__SHIFT                                                        0x5
+#define BIF_CFG_DEV0_RC_LINK_CNTL__COMMON_CLOCK_CFG__SHIFT                                                    0x6
+#define BIF_CFG_DEV0_RC_LINK_CNTL__EXTENDED_SYNC__SHIFT                                                       0x7
+#define BIF_CFG_DEV0_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE__SHIFT                                         0x9
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN__SHIFT                                           0xa
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN__SHIFT                                           0xb
+#define BIF_CFG_DEV0_RC_LINK_CNTL__DRS_SIGNALING_CONTROL__SHIFT                                               0xe
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PM_CONTROL_MASK                                                            0x0003L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__PTM_PROP_DELAY_ADAPT_INTER_B_MASK                                          0x0004L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__READ_CPL_BOUNDARY_MASK                                                     0x0008L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_DIS_MASK                                                              0x0010L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__RETRAIN_LINK_MASK                                                          0x0020L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__COMMON_CLOCK_CFG_MASK                                                      0x0040L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__EXTENDED_SYNC_MASK                                                         0x0080L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__CLOCK_POWER_MANAGEMENT_EN_MASK                                             0x0100L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__HW_AUTONOMOUS_WIDTH_DISABLE_MASK                                           0x0200L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_BW_MANAGEMENT_INT_EN_MASK                                             0x0400L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__LINK_AUTONOMOUS_BW_INT_EN_MASK                                             0x0800L
+#define BIF_CFG_DEV0_RC_LINK_CNTL__DRS_SIGNALING_CONTROL_MASK                                                 0xC000L
+//BIF_CFG_DEV0_RC_LINK_STATUS
+#define BIF_CFG_DEV0_RC_LINK_STATUS__CURRENT_LINK_SPEED__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH__SHIFT                                             0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_TRAINING__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_RC_LINK_STATUS__SLOT_CLOCK_CFG__SHIFT                                                    0xc
+#define BIF_CFG_DEV0_RC_LINK_STATUS__DL_ACTIVE__SHIFT                                                         0xd
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS__SHIFT                                         0xe
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS__SHIFT                                         0xf
+#define BIF_CFG_DEV0_RC_LINK_STATUS__CURRENT_LINK_SPEED_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_RC_LINK_STATUS__NEGOTIATED_LINK_WIDTH_MASK                                               0x03F0L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_TRAINING_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__SLOT_CLOCK_CFG_MASK                                                      0x1000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__DL_ACTIVE_MASK                                                           0x2000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_BW_MANAGEMENT_STATUS_MASK                                           0x4000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS__LINK_AUTONOMOUS_BW_STATUS_MASK                                           0x8000L
+//SLOT_CAP
+#define SLOT_CAP__ATTN_BUTTON_PRESENT__SHIFT                                                                  0x0
+#define SLOT_CAP__PWR_CONTROLLER_PRESENT__SHIFT                                                               0x1
+#define SLOT_CAP__MRL_SENSOR_PRESENT__SHIFT                                                                   0x2
+#define SLOT_CAP__ATTN_INDICATOR_PRESENT__SHIFT                                                               0x3
+#define SLOT_CAP__PWR_INDICATOR_PRESENT__SHIFT                                                                0x4
+#define SLOT_CAP__HOTPLUG_SURPRISE__SHIFT                                                                     0x5
+#define SLOT_CAP__HOTPLUG_CAPABLE__SHIFT                                                                      0x6
+#define SLOT_CAP__SLOT_PWR_LIMIT_VALUE__SHIFT                                                                 0x7
+#define SLOT_CAP__SLOT_PWR_LIMIT_SCALE__SHIFT                                                                 0xf
+#define SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT__SHIFT                                                        0x11
+#define SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED__SHIFT                                                       0x12
+#define SLOT_CAP__PHYSICAL_SLOT_NUM__SHIFT                                                                    0x13
+#define SLOT_CAP__ATTN_BUTTON_PRESENT_MASK                                                                    0x00000001L
+#define SLOT_CAP__PWR_CONTROLLER_PRESENT_MASK                                                                 0x00000002L
+#define SLOT_CAP__MRL_SENSOR_PRESENT_MASK                                                                     0x00000004L
+#define SLOT_CAP__ATTN_INDICATOR_PRESENT_MASK                                                                 0x00000008L
+#define SLOT_CAP__PWR_INDICATOR_PRESENT_MASK                                                                  0x00000010L
+#define SLOT_CAP__HOTPLUG_SURPRISE_MASK                                                                       0x00000020L
+#define SLOT_CAP__HOTPLUG_CAPABLE_MASK                                                                        0x00000040L
+#define SLOT_CAP__SLOT_PWR_LIMIT_VALUE_MASK                                                                   0x00007F80L
+#define SLOT_CAP__SLOT_PWR_LIMIT_SCALE_MASK                                                                   0x00018000L
+#define SLOT_CAP__ELECTROMECH_INTERLOCK_PRESENT_MASK                                                          0x00020000L
+#define SLOT_CAP__NO_COMMAND_COMPLETED_SUPPORTED_MASK                                                         0x00040000L
+#define SLOT_CAP__PHYSICAL_SLOT_NUM_MASK                                                                      0xFFF80000L
+//SLOT_CNTL
+#define SLOT_CNTL__ATTN_BUTTON_PRESSED_EN__SHIFT                                                              0x0
+#define SLOT_CNTL__PWR_FAULT_DETECTED_EN__SHIFT                                                               0x1
+#define SLOT_CNTL__MRL_SENSOR_CHANGED_EN__SHIFT                                                               0x2
+#define SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN__SHIFT                                                          0x3
+#define SLOT_CNTL__COMMAND_COMPLETED_INTR_EN__SHIFT                                                           0x4
+#define SLOT_CNTL__HOTPLUG_INTR_EN__SHIFT                                                                     0x5
+#define SLOT_CNTL__ATTN_INDICATOR_CNTL__SHIFT                                                                 0x6
+#define SLOT_CNTL__PWR_INDICATOR_CNTL__SHIFT                                                                  0x8
+#define SLOT_CNTL__PWR_CONTROLLER_CNTL__SHIFT                                                                 0xa
+#define SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL__SHIFT                                                          0xb
+#define SLOT_CNTL__DL_STATE_CHANGED_EN__SHIFT                                                                 0xc
+#define SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE__SHIFT                                                         0xd
+#define SLOT_CNTL__INBAND_PD_DISABLE__SHIFT                                                                   0xe
+#define SLOT_CNTL__ATTN_BUTTON_PRESSED_EN_MASK                                                                0x0001L
+#define SLOT_CNTL__PWR_FAULT_DETECTED_EN_MASK                                                                 0x0002L
+#define SLOT_CNTL__MRL_SENSOR_CHANGED_EN_MASK                                                                 0x0004L
+#define SLOT_CNTL__PRESENCE_DETECT_CHANGED_EN_MASK                                                            0x0008L
+#define SLOT_CNTL__COMMAND_COMPLETED_INTR_EN_MASK                                                             0x0010L
+#define SLOT_CNTL__HOTPLUG_INTR_EN_MASK                                                                       0x0020L
+#define SLOT_CNTL__ATTN_INDICATOR_CNTL_MASK                                                                   0x00C0L
+#define SLOT_CNTL__PWR_INDICATOR_CNTL_MASK                                                                    0x0300L
+#define SLOT_CNTL__PWR_CONTROLLER_CNTL_MASK                                                                   0x0400L
+#define SLOT_CNTL__ELECTROMECH_INTERLOCK_CNTL_MASK                                                            0x0800L
+#define SLOT_CNTL__DL_STATE_CHANGED_EN_MASK                                                                   0x1000L
+#define SLOT_CNTL__AUTO_SLOT_PWR_LIMIT_DISABLE_MASK                                                           0x2000L
+#define SLOT_CNTL__INBAND_PD_DISABLE_MASK                                                                     0x4000L
+//SLOT_STATUS
+#define SLOT_STATUS__ATTN_BUTTON_PRESSED__SHIFT                                                               0x0
+#define SLOT_STATUS__PWR_FAULT_DETECTED__SHIFT                                                                0x1
+#define SLOT_STATUS__MRL_SENSOR_CHANGED__SHIFT                                                                0x2
+#define SLOT_STATUS__PRESENCE_DETECT_CHANGED__SHIFT                                                           0x3
+#define SLOT_STATUS__COMMAND_COMPLETED__SHIFT                                                                 0x4
+#define SLOT_STATUS__MRL_SENSOR_STATE__SHIFT                                                                  0x5
+#define SLOT_STATUS__PRESENCE_DETECT_STATE__SHIFT                                                             0x6
+#define SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS__SHIFT                                                      0x7
+#define SLOT_STATUS__DL_STATE_CHANGED__SHIFT                                                                  0x8
+#define SLOT_STATUS__ATTN_BUTTON_PRESSED_MASK                                                                 0x0001L
+#define SLOT_STATUS__PWR_FAULT_DETECTED_MASK                                                                  0x0002L
+#define SLOT_STATUS__MRL_SENSOR_CHANGED_MASK                                                                  0x0004L
+#define SLOT_STATUS__PRESENCE_DETECT_CHANGED_MASK                                                             0x0008L
+#define SLOT_STATUS__COMMAND_COMPLETED_MASK                                                                   0x0010L
+#define SLOT_STATUS__MRL_SENSOR_STATE_MASK                                                                    0x0020L
+#define SLOT_STATUS__PRESENCE_DETECT_STATE_MASK                                                               0x0040L
+#define SLOT_STATUS__ELECTROMECH_INTERLOCK_STATUS_MASK                                                        0x0080L
+#define SLOT_STATUS__DL_STATE_CHANGED_MASK                                                                    0x0100L
+//ROOT_CNTL
+#define ROOT_CNTL__SERR_ON_CORR_ERR_EN__SHIFT                                                                 0x0
+#define ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN__SHIFT                                                             0x1
+#define ROOT_CNTL__SERR_ON_FATAL_ERR_EN__SHIFT                                                                0x2
+#define ROOT_CNTL__PM_INTERRUPT_EN__SHIFT                                                                     0x3
+#define ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN__SHIFT                                                          0x4
+#define ROOT_CNTL__SERR_ON_CORR_ERR_EN_MASK                                                                   0x0001L
+#define ROOT_CNTL__SERR_ON_NONFATAL_ERR_EN_MASK                                                               0x0002L
+#define ROOT_CNTL__SERR_ON_FATAL_ERR_EN_MASK                                                                  0x0004L
+#define ROOT_CNTL__PM_INTERRUPT_EN_MASK                                                                       0x0008L
+#define ROOT_CNTL__CRS_SOFTWARE_VISIBILITY_EN_MASK                                                            0x0010L
+//ROOT_CAP
+#define ROOT_CAP__CRS_SOFTWARE_VISIBILITY__SHIFT                                                              0x0
+#define ROOT_CAP__CRS_SOFTWARE_VISIBILITY_MASK                                                                0x0001L
+//ROOT_STATUS
+#define ROOT_STATUS__PME_REQUESTOR_ID__SHIFT                                                                  0x0
+#define ROOT_STATUS__PME_STATUS__SHIFT                                                                        0x10
+#define ROOT_STATUS__PME_PENDING__SHIFT                                                                       0x11
+#define ROOT_STATUS__PME_REQUESTOR_ID_MASK                                                                    0x0000FFFFL
+#define ROOT_STATUS__PME_STATUS_MASK                                                                          0x00010000L
+#define ROOT_STATUS__PME_PENDING_MASK                                                                         0x00020000L
+//BIF_CFG_DEV0_RC_DEVICE_CAP2
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED__SHIFT                                       0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED__SHIFT                                          0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED__SHIFT                                        0x6
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED__SHIFT                                        0x7
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED__SHIFT                                        0x8
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING__SHIFT                                         0xa
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LTR_SUPPORTED__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED__SHIFT                                                0xc
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LN_SYSTEM_CLS__SHIFT                                                     0xe
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED__SHIFT                                   0x11
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__OBFF_SUPPORTED__SHIFT                                                    0x12
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED__SHIFT                                      0x14
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED__SHIFT                                      0x15
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES__SHIFT                                          0x16
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED__SHIFT                                    0x18
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ__SHIFT                                     0x1a
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__FRS_SUPPORTED__SHIFT                                                     0x1f
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_RANGE_SUPPORTED_MASK                                         0x0000000FL
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CPL_TIMEOUT_DIS_SUPPORTED_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ARI_FORWARDING_SUPPORTED_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_ROUTING_SUPPORTED_MASK                                          0x00000040L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_32CMPLT_SUPPORTED_MASK                                          0x00000080L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__ATOMICOP_64CMPLT_SUPPORTED_MASK                                          0x00000100L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__CAS128_CMPLT_SUPPORTED_MASK                                              0x00000200L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__NO_RO_ENABLED_P2P_PASSING_MASK                                           0x00000400L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LTR_SUPPORTED_MASK                                                       0x00000800L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TPH_CPLR_SUPPORTED_MASK                                                  0x00003000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__LN_SYSTEM_CLS_MASK                                                       0x0000C000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_COMPLETER_SUPPORTED_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__TEN_BIT_TAG_REQUESTER_SUPPORTED_MASK                                     0x00020000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__OBFF_SUPPORTED_MASK                                                      0x000C0000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EXTENDED_FMT_FIELD_SUPPORTED_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__END_END_TLP_PREFIX_SUPPORTED_MASK                                        0x00200000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__MAX_END_END_TLP_PREFIXES_MASK                                            0x00C00000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_SUPPORTED_MASK                                      0x03000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__EMER_POWER_REDUCTION_INIT_REQ_MASK                                       0x04000000L
+#define BIF_CFG_DEV0_RC_DEVICE_CAP2__FRS_SUPPORTED_MASK                                                       0x80000000L
+//BIF_CFG_DEV0_RC_DEVICE_CNTL2
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS__SHIFT                                                  0x4
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ARI_FORWARDING_EN__SHIFT                                                0x5
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN__SHIFT                                              0x6
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING__SHIFT                                         0x7
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE__SHIFT                                               0x8
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__LTR_EN__SHIFT                                                           0xa
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST__SHIFT                                     0xb
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE__SHIFT                                     0xc
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__OBFF_EN__SHIFT                                                          0xd
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_VALUE_MASK                                                  0x000FL
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__CPL_TIMEOUT_DIS_MASK                                                    0x0010L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ARI_FORWARDING_EN_MASK                                                  0x0020L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_REQUEST_EN_MASK                                                0x0040L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__ATOMICOP_EGRESS_BLOCKING_MASK                                           0x0080L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_REQUEST_ENABLE_MASK                                                 0x0100L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__IDO_COMPLETION_ENABLE_MASK                                              0x0200L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__LTR_EN_MASK                                                             0x0400L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__EMER_POWER_REDUCTION_REQUEST_MASK                                       0x0800L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__TEN_BIT_TAG_REQUESTER_ENABLE_MASK                                       0x1000L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__OBFF_EN_MASK                                                            0x6000L
+#define BIF_CFG_DEV0_RC_DEVICE_CNTL2__END_END_TLP_PREFIX_BLOCKING_MASK                                        0x8000L
+//BIF_CFG_DEV0_RC_DEVICE_STATUS2
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS2__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_DEVICE_STATUS2__RESERVED_MASK                                                         0xFFFFL
+//BIF_CFG_DEV0_RC_LINK_CAP2
+#define BIF_CFG_DEV0_RC_LINK_CAP2__SUPPORTED_LINK_SPEED__SHIFT                                                0x1
+#define BIF_CFG_DEV0_RC_LINK_CAP2__CROSSLINK_SUPPORTED__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT__SHIFT                                            0x9
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT__SHIFT                                           0x17
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT__SHIFT                                           0x18
+#define BIF_CFG_DEV0_RC_LINK_CAP2__DRS_SUPPORTED__SHIFT                                                       0x1f
+#define BIF_CFG_DEV0_RC_LINK_CAP2__SUPPORTED_LINK_SPEED_MASK                                                  0x000000FEL
+#define BIF_CFG_DEV0_RC_LINK_CAP2__CROSSLINK_SUPPORTED_MASK                                                   0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_GEN_SUPPORT_MASK                                              0x0000FE00L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__LOWER_SKP_OS_RCV_SUPPORT_MASK                                              0x007F0000L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM1_PRESENCE_DET_SUPPORT_MASK                                             0x00800000L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__RTM2_PRESENCE_DET_SUPPORT_MASK                                             0x01000000L
+#define BIF_CFG_DEV0_RC_LINK_CAP2__DRS_SUPPORTED_MASK                                                         0x80000000L
+//BIF_CFG_DEV0_RC_LINK_CNTL2
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__TARGET_LINK_SPEED__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_COMPLIANCE__SHIFT                                                   0x4
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE__SHIFT                                        0x5
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS__SHIFT                                              0x6
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__XMIT_MARGIN__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE__SHIFT                                               0xa
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_SOS__SHIFT                                                     0xb
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS__SHIFT                                              0xc
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__TARGET_LINK_SPEED_MASK                                                    0x000FL
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_COMPLIANCE_MASK                                                     0x0010L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__HW_AUTONOMOUS_SPEED_DISABLE_MASK                                          0x0020L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__SELECTABLE_DEEMPHASIS_MASK                                                0x0040L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__XMIT_MARGIN_MASK                                                          0x0380L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__ENTER_MOD_COMPLIANCE_MASK                                                 0x0400L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_SOS_MASK                                                       0x0800L
+#define BIF_CFG_DEV0_RC_LINK_CNTL2__COMPLIANCE_DEEMPHASIS_MASK                                                0xF000L
+//BIF_CFG_DEV0_RC_LINK_STATUS2
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT__SHIFT                                  0x2
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT__SHIFT                                  0x3
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT__SHIFT                                  0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT__SHIFT                                    0x5
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM1_PRESENCE_DET__SHIFT                                                0x6
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM2_PRESENCE_DET__SHIFT                                                0x7
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CROSSLINK_RESOLUTION__SHIFT                                             0x8
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__FLIT_MODE_STATUS__SHIFT                                                 0xa
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE__SHIFT                                    0xc
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED__SHIFT                                             0xf
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CUR_DEEMPHASIS_LEVEL_MASK                                               0x0001L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_COMPLETE_8GT_MASK                                          0x0002L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE1_SUCCESS_8GT_MASK                                    0x0004L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE2_SUCCESS_8GT_MASK                                    0x0008L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__EQUALIZATION_PHASE3_SUCCESS_8GT_MASK                                    0x0010L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__LINK_EQUALIZATION_REQUEST_8GT_MASK                                      0x0020L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM1_PRESENCE_DET_MASK                                                  0x0040L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__RTM2_PRESENCE_DET_MASK                                                  0x0080L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__CROSSLINK_RESOLUTION_MASK                                               0x0300L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__FLIT_MODE_STATUS_MASK                                                   0x0400L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DOWNSTREAM_COMPONENT_PRESENCE_MASK                                      0x7000L
+#define BIF_CFG_DEV0_RC_LINK_STATUS2__DRS_MESSAGE_RECEIVED_MASK                                               0x8000L
+//SLOT_CAP2
+#define SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED__SHIFT                                                         0x0
+#define SLOT_CAP2__INBAND_PD_DISABLE_SUPPORTED_MASK                                                           0x00000001L
+//SLOT_CNTL2
+#define SLOT_CNTL2__RESERVED__SHIFT                                                                           0x0
+#define SLOT_CNTL2__RESERVED_MASK                                                                             0xFFFFL
+//SLOT_STATUS2
+#define SLOT_STATUS2__RESERVED__SHIFT                                                                         0x0
+#define SLOT_STATUS2__RESERVED_MASK                                                                           0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_CAP_LIST
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__CAP_ID__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__NEXT_PTR__SHIFT                                                         0x8
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__CAP_ID_MASK                                                             0x00FFL
+#define BIF_CFG_DEV0_RC_MSI_CAP_LIST__NEXT_PTR_MASK                                                           0xFF00L
+//BIF_CFG_DEV0_RC_MSI_MSG_CNTL
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EN__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_CAP__SHIFT                                                    0x1
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_EN__SHIFT                                                     0x4
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_64BIT__SHIFT                                                        0x7
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP__SHIFT                                        0x8
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP__SHIFT                                             0x9
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN__SHIFT                                              0xa
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EN_MASK                                                             0x0001L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_CAP_MASK                                                      0x000EL
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_MULTI_EN_MASK                                                       0x0070L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_64BIT_MASK                                                          0x0080L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_PERVECTOR_MASKING_CAP_MASK                                          0x0100L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_CAP_MASK                                               0x0200L
+#define BIF_CFG_DEV0_RC_MSI_MSG_CNTL__MSI_EXT_MSG_DATA_EN_MASK                                                0x0400L
+//BIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO__SHIFT                                               0x2
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_LO__MSI_MSG_ADDR_LO_MASK                                                 0xFFFFFFFCL
+//BIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_ADDR_HI__MSI_MSG_ADDR_HI_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_MSI_MSG_DATA
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA__MSI_DATA__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA__MSI_DATA_MASK                                                           0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA__MSI_EXT_DATA_MASK                                                   0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_MSG_DATA_64
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA_64__MSI_DATA_64__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_MSI_MSG_DATA_64__MSI_DATA_64_MASK                                                     0xFFFFL
+//BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_MSI_EXT_MSG_DATA_64__MSI_EXT_DATA_64_MASK                                             0xFFFFL
+//SSID_CAP_LIST
+#define SSID_CAP_LIST__CAP_ID__SHIFT                                                                          0x0
+#define SSID_CAP_LIST__NEXT_PTR__SHIFT                                                                        0x8
+#define SSID_CAP_LIST__CAP_ID_MASK                                                                            0x00FFL
+#define SSID_CAP_LIST__NEXT_PTR_MASK                                                                          0xFF00L
+//SSID_CAP
+#define SSID_CAP__SUBSYSTEM_VENDOR_ID__SHIFT                                                                  0x0
+#define SSID_CAP__SUBSYSTEM_ID__SHIFT                                                                         0x10
+#define SSID_CAP__SUBSYSTEM_VENDOR_ID_MASK                                                                    0x0000FFFFL
+#define SSID_CAP__SUBSYSTEM_ID_MASK                                                                           0xFFFF0000L
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER__SHIFT                                     0x10
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                    0x14
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_ID_MASK                                        0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__CAP_VER_MASK                                       0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_ENH_CAP_LIST__NEXT_PTR_MASK                                      0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV__SHIFT                                             0x10
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_ID_MASK                                                0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_REV_MASK                                               0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC_HDR__VSEC_LENGTH_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC1__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH__SHIFT                                                 0x0
+#define BIF_CFG_DEV0_RC_PCIE_VENDOR_SPECIFIC2__SCRATCH_MASK                                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_SECONDARY_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN__SHIFT                                  0x1
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN__SHIFT                                       0x9
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__PERFORM_EQUALIZATION_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__LINK_EQUALIZATION_REQ_INT_EN_MASK                                    0x00000002L
+#define BIF_CFG_DEV0_RC_PCIE_LINK_CNTL3__ENABLE_LOWER_SKP_OS_GEN_MASK                                         0x0000FE00L
+//BIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_ERROR_STATUS__LANE_ERROR_STATUS_BITS_MASK                                   0x0000FFFFL
+//BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_0_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_1_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_2_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_3_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_4_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_5_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_6_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_7_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_8_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT              0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT                0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                     0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                       0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_9_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                  0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_10_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_11_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_12_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_13_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_14_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT             0x4
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET__SHIFT                    0x8
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT__SHIFT               0xc
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_TX_PRESET_MASK                    0x000FL
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__DOWNSTREAM_PORT_8GT_RX_PRESET_HINT_MASK               0x0070L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_TX_PRESET_MASK                      0x0F00L
+#define BIF_CFG_DEV0_RC_PCIE_LANE_15_EQUALIZATION_CNTL__UPSTREAM_PORT_8GT_RX_PRESET_HINT_MASK                 0x7000L
+//BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                           0x14
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__CAP_VER_MASK                                              0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_PHY_16GT_ENH_CAP_LIST__NEXT_PTR_MASK                                             0xFFF00000L
+//BIF_CFG_DEV0_RC_LINK_CAP_16GT
+#define BIF_CFG_DEV0_RC_LINK_CAP_16GT__RESERVED__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_LINK_CAP_16GT__RESERVED_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_LINK_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LINK_CNTL_16GT__RESERVED__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL_16GT__RESERVED_MASK                                                         0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_LINK_STATUS_16GT
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT__SHIFT                             0x1
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT__SHIFT                             0x2
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT__SHIFT                             0x3
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_COMPLETE_16GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE1_SUCCESS_16GT_MASK                               0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE2_SUCCESS_16GT_MASK                               0x00000004L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__EQUALIZATION_PHASE3_SUCCESS_16GT_MASK                               0x00000008L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_16GT__LINK_EQUALIZATION_REQUEST_16GT_MASK                                 0x00000010L
+//BIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS__SHIFT           0x0
+#define BIF_CFG_DEV0_RC_LOCAL_PARITY_MISMATCH_STATUS_16GT__LOCAL_PARITY_MISMATCH_STATUS_BITS_MASK             0x0000FFFFL
+//BIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV0_RC_RTM1_PARITY_MISMATCH_STATUS_16GT__RTM1_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT
+#define BIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS__SHIFT             0x0
+#define BIF_CFG_DEV0_RC_RTM2_PARITY_MISMATCH_STATUS_16GT__RTM2_PARITY_MISMATCH_STATUS_BITS_MASK               0x0000FFFFL
+//BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_16GT__LANE_0_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_16GT__LANE_1_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_16GT__LANE_2_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_16GT__LANE_3_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_16GT__LANE_4_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_16GT__LANE_5_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_16GT__LANE_6_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_16GT__LANE_7_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_16GT__LANE_8_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_DSP_16GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_16GT__LANE_9_USP_16GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_16GT__LANE_10_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_16GT__LANE_11_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_16GT__LANE_12_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_16GT__LANE_13_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_16GT__LANE_14_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_DSP_16GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_16GT__LANE_15_USP_16GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER__SHIFT                                            0x10
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                           0x14
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_ID_MASK                                               0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__CAP_VER_MASK                                              0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_PHY_32GT_ENH_CAP_LIST__NEXT_PTR_MASK                                             0xFFF00000L
+//BIF_CFG_DEV0_RC_LINK_CAP_32GT
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED__SHIFT                                          0x1
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED__SHIFT                               0x8
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED__SHIFT                               0x9
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED__SHIFT                               0xa
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES__SHIFT                                0xb
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__EQ_BYPASS_TO_HIGHEST_RATE_SUPPORTED_MASK                               0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__NO_EQ_NEEDED_SUPPORTED_MASK                                            0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE0_SUPPORTED_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE1_SUPPORTED_MASK                                 0x00000200L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_USAGE_MODE2_SUPPORTED_MASK                                 0x00000400L
+#define BIF_CFG_DEV0_RC_LINK_CAP_32GT__MODIFIED_TS_RESERVED_USAGE_MODES_MASK                                  0x0000F800L
+//BIF_CFG_DEV0_RC_LINK_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL__SHIFT                                     0x8
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__EQ_BYPASS_TO_HIGHEST_RATE_DIS_MASK                                    0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__NO_EQ_NEEDED_DIS_MASK                                                 0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_CNTL_32GT__MODIFIED_TS_USAGE_MODE_SEL_MASK                                       0x00000700L
+//BIF_CFG_DEV0_RC_LINK_STATUS_32GT
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT__SHIFT                                   0x0
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT__SHIFT                             0x1
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT__SHIFT                             0x2
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT__SHIFT                             0x3
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED__SHIFT                                         0x5
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL__SHIFT                         0x6
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON__SHIFT                                     0x8
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST__SHIFT                                  0x9
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED__SHIFT                                        0xa
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_COMPLETE_32GT_MASK                                     0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE1_SUCCESS_32GT_MASK                               0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE2_SUCCESS_32GT_MASK                               0x00000004L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__EQUALIZATION_PHASE3_SUCCESS_32GT_MASK                               0x00000008L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__LINK_EQUALIZATION_REQUEST_32GT_MASK                                 0x00000010L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__MODIFIED_TS_RECEIVED_MASK                                           0x00000020L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__RECEIVED_ENHANCED_LINK_BEHAVIOR_CNTL_MASK                           0x000000C0L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODING_ON_MASK                                       0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__TRANSMITTER_PRECODE_REQUEST_MASK                                    0x00000200L
+#define BIF_CFG_DEV0_RC_LINK_STATUS_32GT__NO_EQ_NEEDED_RECEIVED_MASK                                          0x00000400L
+//BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                             0x3
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                           0x00000007L
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__INFOR1_MASK                                               0x0000FFF8L
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA1__VENDORID_MASK                                             0xFFFF0000L
+//BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                             0x0
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT              0x18
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2__INFOR2_MASK                                               0x00FFFFFFL
+#define BIF_CFG_DEV0_RC_RECEIVED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK                0x03000000L
+//BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID__SHIFT                                        0x10
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__USAGE_MODE_MASK                                        0x00000007L
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__INFOR1_MASK                                            0x0000FFF8L
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA1__VENDORID_MASK                                          0xFFFF0000L
+//BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS__SHIFT           0x18
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2__INFOR2_MASK                                            0x00FFFFFFL
+#define BIF_CFG_DEV0_RC_TRANSMITTED_MODIFIED_TS_DATA2__ALTERNATE_PROTOCOL_NEGOTIATION_STATUS_MASK             0x03000000L
+//BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_0_EQUALIZATION_CNTL_32GT__LANE_0_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_1_EQUALIZATION_CNTL_32GT__LANE_1_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_2_EQUALIZATION_CNTL_32GT__LANE_2_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_3_EQUALIZATION_CNTL_32GT__LANE_3_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_4_EQUALIZATION_CNTL_32GT__LANE_4_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_5_EQUALIZATION_CNTL_32GT__LANE_5_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_6_EQUALIZATION_CNTL_32GT__LANE_6_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_7_EQUALIZATION_CNTL_32GT__LANE_7_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_8_EQUALIZATION_CNTL_32GT__LANE_8_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET__SHIFT                       0x4
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_DSP_32GT_TX_PRESET_MASK                         0x0FL
+#define BIF_CFG_DEV0_RC_LANE_9_EQUALIZATION_CNTL_32GT__LANE_9_USP_32GT_TX_PRESET_MASK                         0xF0L
+//BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_10_EQUALIZATION_CNTL_32GT__LANE_10_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_11_EQUALIZATION_CNTL_32GT__LANE_11_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_12_EQUALIZATION_CNTL_32GT__LANE_12_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_13_EQUALIZATION_CNTL_32GT__LANE_13_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_14_EQUALIZATION_CNTL_32GT__LANE_14_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET__SHIFT                     0x0
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET__SHIFT                     0x4
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_DSP_32GT_TX_PRESET_MASK                       0x0FL
+#define BIF_CFG_DEV0_RC_LANE_15_EQUALIZATION_CNTL_32GT__LANE_15_USP_32GT_TX_PRESET_MASK                       0xF0L
+//BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_AP_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_AP_CAP
+#define BIF_CFG_DEV0_RC_AP_CAP__COUNT__SHIFT                                                                  0x0
+#define BIF_CFG_DEV0_RC_AP_CAP__SEL_EN_SUPPORTED__SHIFT                                                       0x8
+#define BIF_CFG_DEV0_RC_AP_CAP__COUNT_MASK                                                                    0x000000FFL
+#define BIF_CFG_DEV0_RC_AP_CAP__SEL_EN_SUPPORTED_MASK                                                         0x00000100L
+//BIF_CFG_DEV0_RC_AP_CNTL
+#define BIF_CFG_DEV0_RC_AP_CNTL__INX_SEL__SHIFT                                                               0x0
+#define BIF_CFG_DEV0_RC_AP_CNTL__NEGO_GLOBAL_EN__SHIFT                                                        0x8
+#define BIF_CFG_DEV0_RC_AP_CNTL__INX_SEL_MASK                                                                 0x000000FFL
+#define BIF_CFG_DEV0_RC_AP_CNTL__NEGO_GLOBAL_EN_MASK                                                          0x00000100L
+//BIF_CFG_DEV0_RC_AP_DATA1
+#define BIF_CFG_DEV0_RC_AP_DATA1__USAGE_INFOR__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_AP_DATA1__DETAILS__SHIFT                                                              0x5
+#define BIF_CFG_DEV0_RC_AP_DATA1__VENDORID__SHIFT                                                             0x10
+#define BIF_CFG_DEV0_RC_AP_DATA1__USAGE_INFOR_MASK                                                            0x00000007L
+#define BIF_CFG_DEV0_RC_AP_DATA1__DETAILS_MASK                                                                0x0000FFE0L
+#define BIF_CFG_DEV0_RC_AP_DATA1__VENDORID_MASK                                                               0xFFFF0000L
+//BIF_CFG_DEV0_RC_AP_DATA2
+#define BIF_CFG_DEV0_RC_AP_DATA2__MODIFIED_TS_INFOR2__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_AP_DATA2__MODIFIED_TS_INFOR2_MASK                                                     0x00FFFFFFL
+//BIF_CFG_DEV0_RC_AP_SEL_EN_MASK
+#define BIF_CFG_DEV0_RC_AP_SEL_EN_MASK__PCIE__SHIFT                                                           0x0
+#define BIF_CFG_DEV0_RC_AP_SEL_EN_MASK__OTHERS__SHIFT                                                         0x1
+#define BIF_CFG_DEV0_RC_AP_SEL_EN_MASK__PCIE_MASK                                                             0x00000001L
+#define BIF_CFG_DEV0_RC_AP_SEL_EN_MASK__OTHERS_MASK                                                           0xFFFFFFFEL
+//BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR__SHIFT                                          0x14
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_ID_MASK                                              0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__CAP_VER_MASK                                             0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_MARGINING_ENH_CAP_LIST__NEXT_PTR_MASK                                            0xFFF00000L
+//BIF_CFG_DEV0_RC_MARGINING_PORT_CAP
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE__SHIFT                                    0x0
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_CAP__MARGINING_USES_SOFTWARE_MASK                                      0x0001L
+//BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_READY__SHIFT                                         0x0
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY__SHIFT                                0x1
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_READY_MASK                                           0x0001L
+#define BIF_CFG_DEV0_RC_MARGINING_PORT_STATUS__MARGINING_SOFTWARE_READY_MASK                                  0x0002L
+//BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_CNTL__LANE_0_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_0_MARGINING_LANE_STATUS__LANE_0_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_CNTL__LANE_1_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_1_MARGINING_LANE_STATUS__LANE_1_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_CNTL__LANE_2_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_2_MARGINING_LANE_STATUS__LANE_2_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_CNTL__LANE_3_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_3_MARGINING_LANE_STATUS__LANE_3_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_CNTL__LANE_4_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_4_MARGINING_LANE_STATUS__LANE_4_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_CNTL__LANE_5_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_5_MARGINING_LANE_STATUS__LANE_5_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_CNTL__LANE_6_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_6_MARGINING_LANE_STATUS__LANE_6_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_CNTL__LANE_7_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_7_MARGINING_LANE_STATUS__LANE_7_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_CNTL__LANE_8_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_8_MARGINING_LANE_STATUS__LANE_8_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER__SHIFT                             0x0
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE__SHIFT                                 0x3
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL__SHIFT                                 0x6
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD__SHIFT                              0x8
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_RECEIVER_NUMBER_MASK                               0x0007L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_TYPE_MASK                                   0x0038L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_USAGE_MODEL_MASK                                   0x0040L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_CNTL__LANE_9_MARGIN_PAYLOAD_MASK                                0xFF00L
+//BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS__SHIFT                    0x0
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS__SHIFT                        0x3
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS__SHIFT                        0x6
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS__SHIFT                     0x8
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_RECEIVER_NUMBER_STATUS_MASK                      0x0007L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_TYPE_STATUS_MASK                          0x0038L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_USAGE_MODEL_STATUS_MASK                          0x0040L
+#define BIF_CFG_DEV0_RC_LANE_9_MARGINING_LANE_STATUS__LANE_9_MARGIN_PAYLOAD_STATUS_MASK                       0xFF00L
+//BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_CNTL__LANE_10_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_10_MARGINING_LANE_STATUS__LANE_10_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_CNTL__LANE_11_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_11_MARGINING_LANE_STATUS__LANE_11_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_CNTL__LANE_12_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_12_MARGINING_LANE_STATUS__LANE_12_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_CNTL__LANE_13_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_13_MARGINING_LANE_STATUS__LANE_13_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_CNTL__LANE_14_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_14_MARGINING_LANE_STATUS__LANE_14_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER__SHIFT                           0x0
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE__SHIFT                               0x3
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL__SHIFT                               0x6
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD__SHIFT                            0x8
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_RECEIVER_NUMBER_MASK                             0x0007L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_TYPE_MASK                                 0x0038L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_USAGE_MODEL_MASK                                 0x0040L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_CNTL__LANE_15_MARGIN_PAYLOAD_MASK                              0xFF00L
+//BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS__SHIFT                  0x0
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS__SHIFT                      0x3
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS__SHIFT                      0x6
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS__SHIFT                   0x8
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_RECEIVER_NUMBER_STATUS_MASK                    0x0007L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_TYPE_STATUS_MASK                        0x0038L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_USAGE_MODEL_STATUS_MASK                        0x0040L
+#define BIF_CFG_DEV0_RC_LANE_15_MARGINING_LANE_STATUS__LANE_15_MARGIN_PAYLOAD_STATUS_MASK                     0xFF00L
+//BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_NULL1_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID__SHIFT                                                   0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER__SHIFT                                                  0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                 0x14
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_ID_MASK                                                     0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__CAP_VER_MASK                                                    0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC_ENH_CAP_LIST__NEXT_PTR_MASK                                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT__SHIFT                               0x4
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK__SHIFT                                                 0x8
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE__SHIFT                               0xa
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__EXT_VC_COUNT_MASK                                              0x00000007L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__LOW_PRIORITY_EXT_VC_COUNT_MASK                                 0x00000070L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__REF_CLK_MASK                                                   0x00000300L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG1__PORT_ARB_TABLE_ENTRY_SIZE_MASK                                 0x00000C00L
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET__SHIFT                                     0x18
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_CAP_MASK                                                0x000000FFL
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CAP_REG2__VC_ARB_TABLE_OFFSET_MASK                                       0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT__SHIFT                                               0x1
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__LOAD_VC_ARB_TABLE_MASK                                             0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_CNTL__VC_ARB_SELECT_MASK                                                 0x000EL
+//BIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS__SHIFT                                       0x0
+#define BIF_CFG_DEV0_RC_PCIE_PORT_VC_STATUS__VC_ARB_TABLE_STATUS_MASK                                         0x0001L
+//BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x007F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_VC0_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS__SHIFT                                          0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET__SHIFT                                   0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_CAP_MASK                                              0x000000FFL
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__REJECT_SNOOP_TRANS_MASK                                        0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__MAX_TIME_SLOTS_MASK                                            0x003F0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CAP__PORT_ARB_TABLE_OFFSET_MASK                                     0xFF000000L
+//BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7__SHIFT                                        0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE__SHIFT                                    0x10
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID__SHIFT                                                  0x18
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE__SHIFT                                              0x1f
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC0_MASK                                            0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__TC_VC_MAP_TC1_7_MASK                                          0x000000FEL
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__LOAD_PORT_ARB_TABLE_MASK                                      0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__PORT_ARB_SELECT_MASK                                          0x000E0000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ID_MASK                                                    0x07000000L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_CNTL__VC_ENABLE_MASK                                                0x80000000L
+//BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING__SHIFT                               0x1
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__PORT_ARB_TABLE_STATUS_MASK                                  0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_VC1_RESOURCE_STATUS__VC_NEGOTIATION_PENDING_MASK                                 0x0002L
+//BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_DLF_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE__SHIFT                                     0x1f
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__LOCAL_DLF_SUPPORTED_MASK                                       0x007FFFFFL
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_CAP__DLF_EXCHANGE_ENABLE_MASK                                       0x80000000L
+//BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_MASK                                   0x007FFFFFL
+#define BIF_CFG_DEV0_RC_DATA_LINK_FEATURE_STATUS__REMOTE_DLF_SUPPORTED_VALID_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_ID__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_VER__SHIFT                                               0x10
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR__SHIFT                                              0x14
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_ID_MASK                                                  0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__CAP_VER_MASK                                                 0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_NULL2_ENH_CAP_LIST__NEXT_PTR_MASK                                                0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID__SHIFT                                          0x0
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER__SHIFT                                         0x10
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR__SHIFT                                        0x14
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_ID_MASK                                            0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__CAP_VER_MASK                                           0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_RPT_ENH_CAP_LIST__NEXT_PTR_MASK                                          0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS__SHIFT                                      0x5
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS__SHIFT                                         0xc
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS__SHIFT                                          0xd
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS__SHIFT                                     0xe
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS__SHIFT                                   0xf
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS__SHIFT                                       0x10
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS__SHIFT                                        0x11
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS__SHIFT                                         0x12
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS__SHIFT                                        0x13
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS__SHIFT                                  0x14
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS__SHIFT                                   0x15
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS__SHIFT                                  0x16
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS__SHIFT                                  0x17
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS__SHIFT                         0x18
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS__SHIFT                          0x19
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS__SHIFT                     0x1a
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS__SHIFT                     0x1b
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS__SHIFT                                0x1c
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS__SHIFT                               0x1d
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS__SHIFT                               0x1e
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__DLP_ERR_STATUS_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__SURPDN_ERR_STATUS_MASK                                        0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__PSN_ERR_STATUS_MASK                                           0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__FC_ERR_STATUS_MASK                                            0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_TIMEOUT_STATUS_MASK                                       0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__CPL_ABORT_ERR_STATUS_MASK                                     0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNEXP_CPL_STATUS_MASK                                         0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__RCV_OVFL_STATUS_MASK                                          0x00020000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MAL_TLP_STATUS_MASK                                           0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ECRC_ERR_STATUS_MASK                                          0x00080000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNSUPP_REQ_ERR_STATUS_MASK                                    0x00100000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ACS_VIOLATION_STATUS_MASK                                     0x00200000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__UNCORR_INT_ERR_STATUS_MASK                                    0x00400000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MC_BLOCKED_TLP_STATUS_MASK                                    0x00800000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__ATOMICOP_EGRESS_BLOCKED_STATUS_MASK                           0x01000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__TLP_PREFIX_BLOCKED_ERR_STATUS_MASK                            0x02000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__POISONED_TLP_EGRESS_BLOCKED_STATUS_MASK                       0x04000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__DMWR_REQUEST_EGRESS_BLOCKED_STATUS_MASK                       0x08000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__IDE_CHECK_FAILED_STATUS_MASK                                  0x10000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__MISROUTED_IDE_TLP_STATUS_MASK                                 0x20000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_STATUS__PCRC_CHECK_FAILED_STATUS_MASK                                 0x40000000L
+//BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK__SHIFT                                             0x4
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK__SHIFT                                          0x5
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK__SHIFT                                             0xc
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK__SHIFT                                              0xd
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK__SHIFT                                         0xe
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK__SHIFT                                       0xf
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK__SHIFT                                           0x10
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK__SHIFT                                            0x11
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK__SHIFT                                             0x12
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK__SHIFT                                            0x13
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK__SHIFT                                      0x14
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK__SHIFT                                       0x15
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK__SHIFT                                      0x16
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK__SHIFT                                      0x17
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK__SHIFT                             0x18
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK__SHIFT                              0x19
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK__SHIFT                         0x1a
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK__SHIFT                         0x1b
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK__SHIFT                                    0x1c
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK__SHIFT                                   0x1d
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK__SHIFT                                   0x1e
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__DLP_ERR_MASK_MASK                                               0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__SURPDN_ERR_MASK_MASK                                            0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__PSN_ERR_MASK_MASK                                               0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__FC_ERR_MASK_MASK                                                0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_TIMEOUT_MASK_MASK                                           0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__CPL_ABORT_ERR_MASK_MASK                                         0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNEXP_CPL_MASK_MASK                                             0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__RCV_OVFL_MASK_MASK                                              0x00020000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MAL_TLP_MASK_MASK                                               0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ECRC_ERR_MASK_MASK                                              0x00080000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNSUPP_REQ_ERR_MASK_MASK                                        0x00100000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ACS_VIOLATION_MASK_MASK                                         0x00200000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__UNCORR_INT_ERR_MASK_MASK                                        0x00400000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MC_BLOCKED_TLP_MASK_MASK                                        0x00800000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__ATOMICOP_EGRESS_BLOCKED_MASK_MASK                               0x01000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__TLP_PREFIX_BLOCKED_ERR_MASK_MASK                                0x02000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__POISONED_TLP_EGRESS_BLOCKED_MASK_MASK                           0x04000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__DMWR_REQUEST_EGRESS_BLOCKED_MASK_MASK                           0x08000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__IDE_CHECK_FAILED_MASK_MASK                                      0x10000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__MISROUTED_IDE_TLP_MASK_MASK                                     0x20000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_MASK__PCRC_CHECK_FAILED_MASK_MASK                                     0x40000000L
+//BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY__SHIFT                                     0x4
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY__SHIFT                                  0x5
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY__SHIFT                                     0xc
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY__SHIFT                                      0xd
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY__SHIFT                                 0xe
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY__SHIFT                               0xf
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY__SHIFT                                   0x10
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY__SHIFT                                    0x11
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY__SHIFT                                     0x12
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY__SHIFT                                    0x13
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY__SHIFT                              0x14
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY__SHIFT                               0x15
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY__SHIFT                              0x16
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY__SHIFT                              0x17
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY__SHIFT                     0x18
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY__SHIFT                      0x19
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x1a
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY__SHIFT                 0x1b
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY__SHIFT                            0x1c
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY__SHIFT                           0x1d
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY__SHIFT                           0x1e
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__DLP_ERR_SEVERITY_MASK                                       0x00000010L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__SURPDN_ERR_SEVERITY_MASK                                    0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__PSN_ERR_SEVERITY_MASK                                       0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__FC_ERR_SEVERITY_MASK                                        0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_TIMEOUT_SEVERITY_MASK                                   0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__CPL_ABORT_ERR_SEVERITY_MASK                                 0x00008000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNEXP_CPL_SEVERITY_MASK                                     0x00010000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__RCV_OVFL_SEVERITY_MASK                                      0x00020000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MAL_TLP_SEVERITY_MASK                                       0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ECRC_ERR_SEVERITY_MASK                                      0x00080000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNSUPP_REQ_ERR_SEVERITY_MASK                                0x00100000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ACS_VIOLATION_SEVERITY_MASK                                 0x00200000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__UNCORR_INT_ERR_SEVERITY_MASK                                0x00400000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MC_BLOCKED_TLP_SEVERITY_MASK                                0x00800000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__ATOMICOP_EGRESS_BLOCKED_SEVERITY_MASK                       0x01000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__TLP_PREFIX_BLOCKED_ERR_SEVERITY_MASK                        0x02000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__POISONED_TLP_EGRESS_BLOCKED_SEVERITY_MASK                   0x04000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__DMWR_REQUEST_EGRESS_BLOCKED_SEVERITY_MASK                   0x08000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__IDE_CHECK_FAILED_SEVERITY_MASK                              0x10000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__MISROUTED_IDE_TLP_SEVERITY_MASK                             0x20000000L
+#define BIF_CFG_DEV0_RC_PCIE_UNCORR_ERR_SEVERITY__PCRC_CHECK_FAILED_SEVERITY_MASK                             0x40000000L
+//BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS__SHIFT                                           0x6
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS__SHIFT                                          0x7
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS__SHIFT                               0x8
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS__SHIFT                              0xc
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS__SHIFT                             0xd
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS__SHIFT                                      0xe
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS__SHIFT                                      0xf
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__RCV_ERR_STATUS_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_TLP_STATUS_MASK                                             0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__BAD_DLLP_STATUS_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_NUM_ROLLOVER_STATUS_MASK                                 0x00000100L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__REPLAY_TIMER_TIMEOUT_STATUS_MASK                                0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__ADVISORY_NONFATAL_ERR_STATUS_MASK                               0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__CORR_INT_ERR_STATUS_MASK                                        0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_STATUS__HDR_LOG_OVFL_STATUS_MASK                                        0x00008000L
+//BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK__SHIFT                                               0x6
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK__SHIFT                                              0x7
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK__SHIFT                                   0x8
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK__SHIFT                                  0xc
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK__SHIFT                                 0xd
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK__SHIFT                                          0xe
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK__SHIFT                                          0xf
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__RCV_ERR_MASK_MASK                                                 0x00000001L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_TLP_MASK_MASK                                                 0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__BAD_DLLP_MASK_MASK                                                0x00000080L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_NUM_ROLLOVER_MASK_MASK                                     0x00000100L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__REPLAY_TIMER_TIMEOUT_MASK_MASK                                    0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__ADVISORY_NONFATAL_ERR_MASK_MASK                                   0x00002000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__CORR_INT_ERR_MASK_MASK                                            0x00004000L
+#define BIF_CFG_DEV0_RC_PCIE_CORR_ERR_MASK__HDR_LOG_OVFL_MASK_MASK                                            0x00008000L
+//BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP__SHIFT                                            0x5
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN__SHIFT                                             0x6
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP__SHIFT                                          0x7
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP__SHIFT                                      0x9
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN__SHIFT                                       0xa
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT__SHIFT                                  0xb
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE__SHIFT                          0xc
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE__SHIFT                                         0xd
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE__SHIFT                                0x12
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE__SHIFT                                         0x13
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__FIRST_ERR_PTR_MASK                                             0x0000001FL
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_CAP_MASK                                              0x00000020L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_GEN_EN_MASK                                               0x00000040L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_CAP_MASK                                            0x00000080L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__ECRC_CHECK_EN_MASK                                             0x00000100L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_CAP_MASK                                        0x00000200L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__MULTI_HDR_RECD_EN_MASK                                         0x00000400L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__TLP_PREFIX_LOG_PRESENT_MASK                                    0x00000800L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__COMPLETION_TIMEOUT_LOG_CAPABLE_MASK                            0x00001000L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__HEADER_LOG_SIZE_MASK                                           0x0003E000L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_WAS_FLIT_MODE_MASK                                  0x00040000L
+#define BIF_CFG_DEV0_RC_PCIE_ADV_ERR_CAP_CNTL__LOGGED_TLP_SIZE_MASK                                           0x00F80000L
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG0__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG0__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG1
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG1__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG1__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG2
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG2__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG2__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG3
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG3__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG3__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//PCIE_ROOT_ERR_CMD
+#define PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN__SHIFT                                                             0x0
+#define PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN__SHIFT                                                         0x1
+#define PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN__SHIFT                                                            0x2
+#define PCIE_ROOT_ERR_CMD__CORR_ERR_REP_EN_MASK                                                               0x00000001L
+#define PCIE_ROOT_ERR_CMD__NONFATAL_ERR_REP_EN_MASK                                                           0x00000002L
+#define PCIE_ROOT_ERR_CMD__FATAL_ERR_REP_EN_MASK                                                              0x00000004L
+//PCIE_ROOT_ERR_STATUS
+#define PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD__SHIFT                                                            0x0
+#define PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD__SHIFT                                                       0x1
+#define PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD__SHIFT                                                  0x2
+#define PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD__SHIFT                                             0x3
+#define PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL__SHIFT                                                0x4
+#define PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD__SHIFT                                                  0x5
+#define PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD__SHIFT                                                     0x6
+#define PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS__SHIFT                                                         0x7
+#define PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM__SHIFT                                                      0x1b
+#define PCIE_ROOT_ERR_STATUS__ERR_CORR_RCVD_MASK                                                              0x00000001L
+#define PCIE_ROOT_ERR_STATUS__MULT_ERR_CORR_RCVD_MASK                                                         0x00000002L
+#define PCIE_ROOT_ERR_STATUS__ERR_FATAL_NONFATAL_RCVD_MASK                                                    0x00000004L
+#define PCIE_ROOT_ERR_STATUS__MULT_ERR_FATAL_NONFATAL_RCVD_MASK                                               0x00000008L
+#define PCIE_ROOT_ERR_STATUS__FIRST_UNCORRECTABLE_FATAL_MASK                                                  0x00000010L
+#define PCIE_ROOT_ERR_STATUS__NONFATAL_ERROR_MSG_RCVD_MASK                                                    0x00000020L
+#define PCIE_ROOT_ERR_STATUS__FATAL_ERROR_MSG_RCVD_MASK                                                       0x00000040L
+#define PCIE_ROOT_ERR_STATUS__ERR_COR_SUBCLASS_MASK                                                           0x00000180L
+#define PCIE_ROOT_ERR_STATUS__ADV_ERR_INT_MSG_NUM_MASK                                                        0xF8000000L
+//PCIE_ERR_SRC_ID
+#define PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID__SHIFT                                                               0x0
+#define PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID__SHIFT                                                     0x10
+#define PCIE_ERR_SRC_ID__ERR_CORR_SRC_ID_MASK                                                                 0x0000FFFFL
+#define PCIE_ERR_SRC_ID__ERR_FATAL_NONFATAL_SRC_ID_MASK                                                       0xFFFF0000L
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG4
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG4__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG4__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG0__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG5
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG5__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG5__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG1__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG6
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG6__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG6__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG2__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG7
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG7__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG7__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX__SHIFT                                               0x0
+#define BIF_CFG_DEV0_RC_PCIE_TLP_PREFIX_LOG3__TLP_PREFIX_MASK                                                 0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG8
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG8__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG8__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG9
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG9__TLP_HDR__SHIFT                                                         0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG9__TLP_HDR_MASK                                                           0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG10
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG10__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG10__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG11
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG11__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG11__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG12
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG12__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG12__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_HDR_LOG13
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG13__TLP_HDR__SHIFT                                                        0x0
+#define BIF_CFG_DEV0_RC_PCIE_HDR_LOG13__TLP_HDR_MASK                                                          0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID__SHIFT                                       0x0
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER__SHIFT                                      0x10
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR__SHIFT                                     0x14
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_ID_MASK                                         0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__CAP_VER_MASK                                        0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_ENH_CAP_LIST__NEXT_PTR_MASK                                       0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW1__SERIAL_NUMBER_LO_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI__SHIFT                                      0x0
+#define BIF_CFG_DEV0_RC_PCIE_DEV_SERIAL_NUM_DW2__SERIAL_NUMBER_HI_MASK                                        0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_PCIE_ACS_CAP
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__SOURCE_VALIDATION__SHIFT                                                0x0
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING__SHIFT                                             0x1
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT__SHIFT                                             0x2
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT__SHIFT                                          0x3
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING__SHIFT                                              0x4
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL__SHIFT                                               0x5
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P__SHIFT                                            0x6
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY__SHIFT                                              0x7
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__SOURCE_VALIDATION_MASK                                                  0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__TRANSLATION_BLOCKING_MASK                                               0x0002L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_REQUEST_REDIRECT_MASK                                               0x0004L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_COMPLETION_REDIRECT_MASK                                            0x0008L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__UPSTREAM_FORWARDING_MASK                                                0x0010L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__P2P_EGRESS_CONTROL_MASK                                                 0x0020L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__DIRECT_TRANSLATED_P2P_MASK                                              0x0040L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__ENHANCED_CAPABILITY_MASK                                                0x0080L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CAP__EGRESS_CONTROL_VECTOR_SIZE_MASK                                         0xFF00L
+//BIF_CFG_DEV0_RC_PCIE_ACS_CNTL
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN__SHIFT                                            0x0
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN__SHIFT                                         0x2
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN__SHIFT                                      0x3
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN__SHIFT                                          0x4
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN__SHIFT                                           0x5
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN__SHIFT                                        0x6
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN__SHIFT                                          0x7
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0x8
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL__SHIFT                                   0xa
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL__SHIFT                                 0xc
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__SOURCE_VALIDATION_EN_MASK                                              0x0001L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__TRANSLATION_BLOCKING_EN_MASK                                           0x0002L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_REQUEST_REDIRECT_EN_MASK                                           0x0004L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_COMPLETION_REDIRECT_EN_MASK                                        0x0008L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UPSTREAM_FORWARDING_EN_MASK                                            0x0010L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__P2P_EGRESS_CONTROL_EN_MASK                                             0x0020L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DIRECT_TRANSLATED_P2P_EN_MASK                                          0x0040L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__IO_REQUEST_BLOCKING_EN_MASK                                            0x0080L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__DSP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0300L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__USP_MEMORY_TARGET_ACCESS_CNTL_MASK                                     0x0C00L
+#define BIF_CFG_DEV0_RC_PCIE_ACS_CNTL__UNCLAIMED_REQUEST_REDIRECT_CNTL_MASK                                   0x1000L
+//BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__CAP_ID__SHIFT                                                  0x0
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__CAP_VER__SHIFT                                                 0x10
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                0x14
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__CAP_ID_MASK                                                    0x0000FFFFL
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__CAP_VER_MASK                                                   0x000F0000L
+#define BIF_CFG_DEV0_RC_PCIE_RTR_ENH_CAP_LIST__NEXT_PTR_MASK                                                  0xFFF00000L
+//BIF_CFG_DEV0_RC_RTR_DATA1
+#define BIF_CFG_DEV0_RC_RTR_DATA1__RESET_TIME__SHIFT                                                          0x0
+#define BIF_CFG_DEV0_RC_RTR_DATA1__DLUP_TIME__SHIFT                                                           0xc
+#define BIF_CFG_DEV0_RC_RTR_DATA1__RTR_VALID__SHIFT                                                           0x1f
+#define BIF_CFG_DEV0_RC_RTR_DATA1__RESET_TIME_MASK                                                            0x00000FFFL
+#define BIF_CFG_DEV0_RC_RTR_DATA1__DLUP_TIME_MASK                                                             0x00FFF000L
+#define BIF_CFG_DEV0_RC_RTR_DATA1__RTR_VALID_MASK                                                             0x80000000L
+//BIF_CFG_DEV0_RC_RTR_DATA2
+#define BIF_CFG_DEV0_RC_RTR_DATA2__FLR_TIME__SHIFT                                                            0x0
+#define BIF_CFG_DEV0_RC_RTR_DATA2__D3HOTD0_TIME__SHIFT                                                        0xc
+#define BIF_CFG_DEV0_RC_RTR_DATA2__FLR_TIME_MASK                                                              0x00000FFFL
+#define BIF_CFG_DEV0_RC_RTR_DATA2__D3HOTD0_TIME_MASK                                                          0x00FFF000L
+//BIF_CFG_DEV0_RC_IDE_ENH_CAP_LIST
+#define BIF_CFG_DEV0_RC_IDE_ENH_CAP_LIST__CAP_ID__SHIFT                                                       0x0
+#define BIF_CFG_DEV0_RC_IDE_ENH_CAP_LIST__CAP_VER__SHIFT                                                      0x10
+#define BIF_CFG_DEV0_RC_IDE_ENH_CAP_LIST__NEXT_PTR__SHIFT                                                     0x14
+#define BIF_CFG_DEV0_RC_IDE_ENH_CAP_LIST__CAP_ID_MASK                                                         0x0000FFFFL
+#define BIF_CFG_DEV0_RC_IDE_ENH_CAP_LIST__CAP_VER_MASK                                                        0x000F0000L
+#define BIF_CFG_DEV0_RC_IDE_ENH_CAP_LIST__NEXT_PTR_MASK                                                       0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_CAP
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_LINK_IDE_STREAM_SUPPORTED__SHIFT                                     0x0
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_SELECTIVE_IDE_STREAM_SUPPORTED__SHIFT                                0x1
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED__SHIFT                             0x2
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED__SHIFT                              0x3
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_AGGREGATION_SUPPORTED__SHIFT                                         0x4
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_PCRC_SUPPORTED__SHIFT                                                0x5
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_IDE_KM_PROTOCOL_SUPPORTED__SHIFT                                     0x6
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED__SHIFT                           0x7
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_SUPPORTED_ALGORITHMS__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE__SHIFT                             0xd
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED__SHIFT                        0x10
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_TEE_LIMITED_SUPPORTED__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_XT_SUPPORTED__SHIFT                                                  0x19
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_LINK_IDE_STREAM_SUPPORTED_MASK                                       0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_SELECTIVE_IDE_STREAM_SUPPORTED_MASK                                  0x00000002L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_FLOW_THROUGH_IDE_STREAM_SUPPORTED_MASK                               0x00000004L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_PARTIAL_HDR_ENCRYPTION_SUPPORTED_MASK                                0x00000008L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_AGGREGATION_SUPPORTED_MASK                                           0x00000010L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_PCRC_SUPPORTED_MASK                                                  0x00000020L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_IDE_KM_PROTOCOL_SUPPORTED_MASK                                       0x00000040L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_SELECTIVE_IDE_FOR_CFG_REQ_SUPPORTED_MASK                             0x00000080L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_SUPPORTED_ALGORITHMS_MASK                                            0x00001F00L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_NUM_OF_TCS_SUPPORTED_FOR_LINK_IDE_MASK                               0x0000E000L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_NUM_OF_SELECTIVE_IDE_STREAMS_SUPPORTED_MASK                          0x00FF0000L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_TEE_LIMITED_SUPPORTED_MASK                                           0x01000000L
+#define BIF_CFG_DEV0_RC_IDE_CAP__IDE_CAP_XT_SUPPORTED_MASK                                                    0x02000000L
+//BIF_CFG_DEV0_RC_IDE_CNTL
+#define BIF_CFG_DEV0_RC_IDE_CNTL__IDE_CNTL_FLOW_THROUGH_IDE_STREAM_ENABLED__SHIFT                             0x2
+#define BIF_CFG_DEV0_RC_IDE_CNTL__IDE_CNTL_FLOW_THROUGH_IDE_STREAM_ENABLED_MASK                               0x00000004L
+//BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__LINK_IDE_STREAM_ENABLE__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__XT_ENABLE__SHIFT                                              0x1
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                                0x2
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                                 0x4
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                                0x6
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__PCRC_ENABLE__SHIFT                                            0x8
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                               0xa
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM__SHIFT                                     0xe
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__TC__SHIFT                                                     0x13
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__STREAM_ID__SHIFT                                              0x18
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__LINK_IDE_STREAM_ENABLE_MASK                                   0x00000001L
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__XT_ENABLE_MASK                                                0x00000002L
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR_MASK                                  0x0000000CL
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR_MASK                                   0x00000030L
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL_MASK                                  0x000000C0L
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__PCRC_ENABLE_MASK                                              0x00000100L
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                                 0x00003C00L
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM_MASK                                       0x0007C000L
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__TC_MASK                                                       0x00380000L
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_CNTL__STREAM_ID_MASK                                                0xFF000000L
+//BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_STATUS
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_STATUS__LINK_IDE_STREAM_STATE__SHIFT                                0x0
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                                0x1f
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_STATUS__LINK_IDE_STREAM_STATE_MASK                                  0x0000000FL
+#define BIF_CFG_DEV0_RC_LINK_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                                  0x80000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CAP
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                        0x0000000FL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__XT_ENABLE__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                           0x2
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                            0x4
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                           0x6
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__PCRC_ENABLE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                      0x9
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                          0xa
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TC__SHIFT                                                0x13
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__DEFAULT_STREAM__SHIFT                                    0x16
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TEE_LIMITED_STREAM__SHIFT                                0x17
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__STREAM_ID__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                         0x00000001L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__XT_ENABLE_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_NPR_MASK                             0x0000000CL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_PR_MASK                              0x00000030L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TX_AGGREGATION_MODE_CPL_MASK                             0x000000C0L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__PCRC_ENABLE_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                        0x00000200L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                            0x00003C00L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__SELECTED_ALGORITHM_MASK                                  0x0007C000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TC_MASK                                                  0x00380000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__DEFAULT_STREAM_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__TEE_LIMITED_STREAM_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_CNTL__STREAM_ID_MASK                                           0xFF000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_STATUS
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                        0x0000000FL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_0_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                            0x00FFFF00L
+//BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG2__VALID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                       0x18
+#define BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG2__VALID_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG2__RID_BASE_MASK                                             0x00FFFF00L
+#define BIF_CFG_DEV0_RC_IDE_0_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_0_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CAP
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                        0x0000000FL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__XT_ENABLE__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                           0x2
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                            0x4
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                           0x6
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__PCRC_ENABLE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                      0x9
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                          0xa
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__SELECTED_ALGORITHM__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TC__SHIFT                                                0x13
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__DEFAULT_STREAM__SHIFT                                    0x16
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TEE_LIMITED_STREAM__SHIFT                                0x17
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__STREAM_ID__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                         0x00000001L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__XT_ENABLE_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_NPR_MASK                             0x0000000CL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_PR_MASK                              0x00000030L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TX_AGGREGATION_MODE_CPL_MASK                             0x000000C0L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__PCRC_ENABLE_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                        0x00000200L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                            0x00003C00L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__SELECTED_ALGORITHM_MASK                                  0x0007C000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TC_MASK                                                  0x00380000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__DEFAULT_STREAM_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__TEE_LIMITED_STREAM_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_CNTL__STREAM_ID_MASK                                           0xFF000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_STATUS
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                        0x0000000FL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_1_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                            0x00FFFF00L
+//BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG2__VALID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                       0x18
+#define BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG2__VALID_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG2__RID_BASE_MASK                                             0x00FFFF00L
+#define BIF_CFG_DEV0_RC_IDE_1_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_1_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CAP
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                        0x0000000FL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__XT_ENABLE__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                           0x2
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                            0x4
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                           0x6
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__PCRC_ENABLE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                      0x9
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                          0xa
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__SELECTED_ALGORITHM__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TC__SHIFT                                                0x13
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__DEFAULT_STREAM__SHIFT                                    0x16
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TEE_LIMITED_STREAM__SHIFT                                0x17
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__STREAM_ID__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                         0x00000001L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__XT_ENABLE_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_NPR_MASK                             0x0000000CL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_PR_MASK                              0x00000030L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TX_AGGREGATION_MODE_CPL_MASK                             0x000000C0L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__PCRC_ENABLE_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                        0x00000200L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                            0x00003C00L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__SELECTED_ALGORITHM_MASK                                  0x0007C000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TC_MASK                                                  0x00380000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__DEFAULT_STREAM_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__TEE_LIMITED_STREAM_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_CNTL__STREAM_ID_MASK                                           0xFF000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_STATUS
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                        0x0000000FL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_2_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                            0x00FFFF00L
+//BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG2__VALID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                       0x18
+#define BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG2__VALID_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG2__RID_BASE_MASK                                             0x00FFFF00L
+#define BIF_CFG_DEV0_RC_IDE_2_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_2_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CAP
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                        0x0000000FL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__XT_ENABLE__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                           0x2
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                            0x4
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                           0x6
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__PCRC_ENABLE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                      0x9
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                          0xa
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__SELECTED_ALGORITHM__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TC__SHIFT                                                0x13
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__DEFAULT_STREAM__SHIFT                                    0x16
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TEE_LIMITED_STREAM__SHIFT                                0x17
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__STREAM_ID__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                         0x00000001L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__XT_ENABLE_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_NPR_MASK                             0x0000000CL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_PR_MASK                              0x00000030L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TX_AGGREGATION_MODE_CPL_MASK                             0x000000C0L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__PCRC_ENABLE_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                        0x00000200L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                            0x00003C00L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__SELECTED_ALGORITHM_MASK                                  0x0007C000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TC_MASK                                                  0x00380000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__DEFAULT_STREAM_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__TEE_LIMITED_STREAM_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_CNTL__STREAM_ID_MASK                                           0xFF000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_STATUS
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                        0x0000000FL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_3_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                            0x00FFFF00L
+//BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG2__VALID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                       0x18
+#define BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG2__VALID_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG2__RID_BASE_MASK                                             0x00FFFF00L
+#define BIF_CFG_DEV0_RC_IDE_3_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_3_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CAP
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                        0x0000000FL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__XT_ENABLE__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                           0x2
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                            0x4
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                           0x6
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__PCRC_ENABLE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                      0x9
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                          0xa
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__SELECTED_ALGORITHM__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TC__SHIFT                                                0x13
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__DEFAULT_STREAM__SHIFT                                    0x16
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TEE_LIMITED_STREAM__SHIFT                                0x17
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__STREAM_ID__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                         0x00000001L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__XT_ENABLE_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_NPR_MASK                             0x0000000CL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_PR_MASK                              0x00000030L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TX_AGGREGATION_MODE_CPL_MASK                             0x000000C0L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__PCRC_ENABLE_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                        0x00000200L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                            0x00003C00L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__SELECTED_ALGORITHM_MASK                                  0x0007C000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TC_MASK                                                  0x00380000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__DEFAULT_STREAM_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__TEE_LIMITED_STREAM_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_CNTL__STREAM_ID_MASK                                           0xFF000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_STATUS
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                        0x0000000FL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_4_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                            0x00FFFF00L
+//BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG2__VALID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                       0x18
+#define BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG2__VALID_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG2__RID_BASE_MASK                                             0x00FFFF00L
+#define BIF_CFG_DEV0_RC_IDE_4_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_4_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CAP
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                        0x0000000FL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__XT_ENABLE__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                           0x2
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                            0x4
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                           0x6
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__PCRC_ENABLE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                      0x9
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                          0xa
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__SELECTED_ALGORITHM__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TC__SHIFT                                                0x13
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__DEFAULT_STREAM__SHIFT                                    0x16
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TEE_LIMITED_STREAM__SHIFT                                0x17
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__STREAM_ID__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                         0x00000001L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__XT_ENABLE_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_NPR_MASK                             0x0000000CL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_PR_MASK                              0x00000030L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TX_AGGREGATION_MODE_CPL_MASK                             0x000000C0L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__PCRC_ENABLE_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                        0x00000200L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                            0x00003C00L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__SELECTED_ALGORITHM_MASK                                  0x0007C000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TC_MASK                                                  0x00380000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__DEFAULT_STREAM_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__TEE_LIMITED_STREAM_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_CNTL__STREAM_ID_MASK                                           0xFF000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_STATUS
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                        0x0000000FL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_5_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                            0x00FFFF00L
+//BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG2__VALID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                       0x18
+#define BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG2__VALID_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG2__RID_BASE_MASK                                             0x00FFFF00L
+#define BIF_CFG_DEV0_RC_IDE_5_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_5_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CAP
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                        0x0000000FL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__XT_ENABLE__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                           0x2
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                            0x4
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                           0x6
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__PCRC_ENABLE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                      0x9
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                          0xa
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__SELECTED_ALGORITHM__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TC__SHIFT                                                0x13
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__DEFAULT_STREAM__SHIFT                                    0x16
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TEE_LIMITED_STREAM__SHIFT                                0x17
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__STREAM_ID__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                         0x00000001L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__XT_ENABLE_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_NPR_MASK                             0x0000000CL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_PR_MASK                              0x00000030L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TX_AGGREGATION_MODE_CPL_MASK                             0x000000C0L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__PCRC_ENABLE_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                        0x00000200L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                            0x00003C00L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__SELECTED_ALGORITHM_MASK                                  0x0007C000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TC_MASK                                                  0x00380000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__DEFAULT_STREAM_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__TEE_LIMITED_STREAM_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_CNTL__STREAM_ID_MASK                                           0xFF000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_STATUS
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                        0x0000000FL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_6_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                            0x00FFFF00L
+//BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG2__VALID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                       0x18
+#define BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG2__VALID_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG2__RID_BASE_MASK                                             0x00FFFF00L
+#define BIF_CFG_DEV0_RC_IDE_6_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_6_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CAP
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                        0x0000000FL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__XT_ENABLE__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                           0x2
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                            0x4
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                           0x6
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__PCRC_ENABLE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                      0x9
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                          0xa
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__SELECTED_ALGORITHM__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TC__SHIFT                                                0x13
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__DEFAULT_STREAM__SHIFT                                    0x16
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TEE_LIMITED_STREAM__SHIFT                                0x17
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__STREAM_ID__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                         0x00000001L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__XT_ENABLE_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_NPR_MASK                             0x0000000CL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_PR_MASK                              0x00000030L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TX_AGGREGATION_MODE_CPL_MASK                             0x000000C0L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__PCRC_ENABLE_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                        0x00000200L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                            0x00003C00L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__SELECTED_ALGORITHM_MASK                                  0x0007C000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TC_MASK                                                  0x00380000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__DEFAULT_STREAM_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__TEE_LIMITED_STREAM_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_CNTL__STREAM_ID_MASK                                           0xFF000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_STATUS
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                        0x0000000FL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_7_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                            0x00FFFF00L
+//BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG2__VALID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                       0x18
+#define BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG2__VALID_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG2__RID_BASE_MASK                                             0x00FFFF00L
+#define BIF_CFG_DEV0_RC_IDE_7_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_7_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CAP
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CAP__NUM_ADDR_ASSOCIATION_REG_BLKS_MASK                        0x0000000FL
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_STREAM_ENABLE__SHIFT                       0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__XT_ENABLE__SHIFT                                         0x1
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_NPR__SHIFT                           0x2
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_PR__SHIFT                            0x4
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_CPL__SHIFT                           0x6
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__PCRC_ENABLE__SHIFT                                       0x8
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN__SHIFT                      0x9
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__PARTIAL_HDR_ENCRYPT_MODE__SHIFT                          0xa
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__SELECTED_ALGORITHM__SHIFT                                0xe
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TC__SHIFT                                                0x13
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__DEFAULT_STREAM__SHIFT                                    0x16
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TEE_LIMITED_STREAM__SHIFT                                0x17
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__STREAM_ID__SHIFT                                         0x18
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_STREAM_ENABLE_MASK                         0x00000001L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__XT_ENABLE_MASK                                           0x00000002L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_NPR_MASK                             0x0000000CL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_PR_MASK                              0x00000030L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TX_AGGREGATION_MODE_CPL_MASK                             0x000000C0L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__PCRC_ENABLE_MASK                                         0x00000100L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__SELECTIVE_IDE_FOR_CFG_REQ_EN_MASK                        0x00000200L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__PARTIAL_HDR_ENCRYPT_MODE_MASK                            0x00003C00L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__SELECTED_ALGORITHM_MASK                                  0x0007C000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TC_MASK                                                  0x00380000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__DEFAULT_STREAM_MASK                                      0x00400000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__TEE_LIMITED_STREAM_MASK                                  0x00800000L
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_CNTL__STREAM_ID_MASK                                           0xFF000000L
+//BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_STATUS
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_STATUS__SELECTIVE_IDE_STREAM_STATE__SHIFT                      0x0
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_STATUS__RECEIVED_IDE_FAIL_MSG__SHIFT                           0x1f
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_STATUS__SELECTIVE_IDE_STREAM_STATE_MASK                        0x0000000FL
+#define BIF_CFG_DEV0_RC_SELECTIVE_IDE_STREAM_8_STATUS__RECEIVED_IDE_FAIL_MSG_MASK                             0x80000000L
+//BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG1
+#define BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG1__RID_LIMIT__SHIFT                                          0x8
+#define BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG1__RID_LIMIT_MASK                                            0x00FFFF00L
+//BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG2
+#define BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG2__VALID__SHIFT                                              0x0
+#define BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG2__RID_BASE__SHIFT                                           0x8
+#define BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG2__SEGMENT_BASE__SHIFT                                       0x18
+#define BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG2__VALID_MASK                                                0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG2__RID_BASE_MASK                                             0x00FFFF00L
+#define BIF_CFG_DEV0_RC_IDE_8_RID_ASSOCIATION_REG2__SEGMENT_BASE_MASK                                         0xFF000000L
+//BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG1
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG2
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG3
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_0_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG1
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG1__VALID__SHIFT                                           0x0
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER__SHIFT                                  0x8
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER__SHIFT                                 0x14
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG1__VALID_MASK                                             0x00000001L
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_BASE_LOWER_MASK                                    0x000FFF00L
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG1__MEM_LIMIT_LOWER_MASK                                   0xFFF00000L
+//BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG2
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER__SHIFT                                 0x0
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG2__MEM_LIMIT_UPPER_MASK                                   0xFFFFFFFFL
+//BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG3
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER__SHIFT                                  0x0
+#define BIF_CFG_DEV0_RC_IDE_8_ADDR_ASSOCIATION_1_REG3__MEM_BASE_UPPER_MASK                                    0xFFFFFFFFL
+
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vpe/vpe_2_0_0_offset.h b/drivers/gpu/drm/amd/include/asic_reg/vpe/vpe_2_0_0_offset.h
new file mode 100644
index 0000000000000..d815f560dacc4
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/vpe/vpe_2_0_0_offset.h
@@ -0,0 +1,1041 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _vpe_2_0_0_OFFSET_HEADER
+#define _vpe_2_0_0_OFFSET_HEADER
+
+
+
+// addressBlock: vpe_vpec_vpecdec
+// base address: 0x46000
+#define regVPEC_DEC_START                                                                               0x0000
+#define regVPEC_DEC_START_BASE_IDX                                                                      0
+#define regVPEC_UCODE_ADDR                                                                              0x0001
+#define regVPEC_UCODE_ADDR_BASE_IDX                                                                     0
+#define regVPEC_UCODE_DATA                                                                              0x0002
+#define regVPEC_UCODE_DATA_BASE_IDX                                                                     0
+#define regVPEC_F32_CNTL                                                                                0x0003
+#define regVPEC_F32_CNTL_BASE_IDX                                                                       0
+#define regVPEC_MMHUB_CNTL                                                                              0x0004
+#define regVPEC_MMHUB_CNTL_BASE_IDX                                                                     0
+#define regVPEC_MMHUB_TRUSTLVL                                                                          0x0005
+#define regVPEC_MMHUB_TRUSTLVL_BASE_IDX                                                                 0
+#define regVPEC_VPEP_CTRL                                                                               0x0010
+#define regVPEC_VPEP_CTRL_BASE_IDX                                                                      0
+#define regVPEC_CLK_CTRL                                                                                0x0011
+#define regVPEC_CLK_CTRL_BASE_IDX                                                                       0
+#define regVPEC_COLLABORATE_CNTL                                                                        0x0012
+#define regVPEC_COLLABORATE_CNTL_BASE_IDX                                                               0
+#define regVPEC_COLLABORATE_CFG                                                                         0x0013
+#define regVPEC_COLLABORATE_CFG_BASE_IDX                                                                0
+#define regVPEC_POWER_CNTL                                                                              0x0014
+#define regVPEC_POWER_CNTL_BASE_IDX                                                                     0
+#define regVPEC_ZPR_CNTL                                                                                0x0015
+#define regVPEC_ZPR_CNTL_BASE_IDX                                                                       0
+#define regVPEC_CNTL                                                                                    0x0016
+#define regVPEC_CNTL_BASE_IDX                                                                           0
+#define regVPEC_CNTL_DCC                                                                                0x0017
+#define regVPEC_CNTL_DCC_BASE_IDX                                                                       0
+#define regVPEC_CE_OP_MULTI_64B_BURST                                                                   0x0018
+#define regVPEC_CE_OP_MULTI_64B_BURST_BASE_IDX                                                          0
+#define regVPEC_CNTL1                                                                                   0x0019
+#define regVPEC_CNTL1_BASE_IDX                                                                          0
+#define regVPEC_CNTL2                                                                                   0x001a
+#define regVPEC_CNTL2_BASE_IDX                                                                          0
+#define regVPEC_GB_ADDR_CONFIG                                                                          0x001b
+#define regVPEC_GB_ADDR_CONFIG_BASE_IDX                                                                 0
+#define regVPEC_GB_ADDR_CONFIG_READ                                                                     0x001c
+#define regVPEC_GB_ADDR_CONFIG_READ_BASE_IDX                                                            0
+#define regVPEC_GB_ADDR_CONFIG_META                                                                     0x001d
+#define regVPEC_GB_ADDR_CONFIG_META_BASE_IDX                                                            0
+#define regVPEC_PROCESS_QUANTUM0                                                                        0x001e
+#define regVPEC_PROCESS_QUANTUM0_BASE_IDX                                                               0
+#define regVPEC_PROCESS_QUANTUM1                                                                        0x001f
+#define regVPEC_PROCESS_QUANTUM1_BASE_IDX                                                               0
+#define regVPEC_CONTEXT_SWITCH_THRESHOLD                                                                0x0020
+#define regVPEC_CONTEXT_SWITCH_THRESHOLD_BASE_IDX                                                       0
+#define regVPEC_GLOBAL_QUANTUM                                                                          0x0021
+#define regVPEC_GLOBAL_QUANTUM_BASE_IDX                                                                 0
+#define regVPEC_WATCHDOG_CNTL                                                                           0x0025
+#define regVPEC_WATCHDOG_CNTL_BASE_IDX                                                                  0
+#define regVPEC_ATOMIC_CNTL                                                                             0x0026
+#define regVPEC_ATOMIC_CNTL_BASE_IDX                                                                    0
+#define regVPEC_UCODE_VERSION                                                                           0x0027
+#define regVPEC_UCODE_VERSION_BASE_IDX                                                                  0
+#define regVPEC_MEMREQ_BURST_CNTL                                                                       0x0028
+#define regVPEC_MEMREQ_BURST_CNTL_BASE_IDX                                                              0
+#define regVPEC_TIMESTAMP_CNTL                                                                          0x0029
+#define regVPEC_TIMESTAMP_CNTL_BASE_IDX                                                                 0
+#define regVPEC_GLOBAL_TIMESTAMP_LO                                                                     0x002a
+#define regVPEC_GLOBAL_TIMESTAMP_LO_BASE_IDX                                                            0
+#define regVPEC_GLOBAL_TIMESTAMP_HI                                                                     0x002b
+#define regVPEC_GLOBAL_TIMESTAMP_HI_BASE_IDX                                                            0
+#define regVPEC_FREEZE                                                                                  0x002c
+#define regVPEC_FREEZE_BASE_IDX                                                                         0
+#define regVPEC_CE_CTRL                                                                                 0x002d
+#define regVPEC_CE_CTRL_BASE_IDX                                                                        0
+#define regVPEC_RELAX_ORDERING_LUT                                                                      0x002e
+#define regVPEC_RELAX_ORDERING_LUT_BASE_IDX                                                             0
+#define regVPEC_CREDIT_CNTL                                                                             0x002f
+#define regVPEC_CREDIT_CNTL_BASE_IDX                                                                    0
+#define regVPEC_SCRATCH_RAM_DATA                                                                        0x0030
+#define regVPEC_SCRATCH_RAM_DATA_BASE_IDX                                                               0
+#define regVPEC_SCRATCH_RAM_ADDR                                                                        0x0031
+#define regVPEC_SCRATCH_RAM_ADDR_BASE_IDX                                                               0
+#define regVPEC_QUEUE_RESET_REQ                                                                         0x0032
+#define regVPEC_QUEUE_RESET_REQ_BASE_IDX                                                                0
+#define regVPEC_MAILBOX0                                                                                0x0040
+#define regVPEC_MAILBOX0_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX1                                                                                0x0041
+#define regVPEC_MAILBOX1_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX2                                                                                0x0042
+#define regVPEC_MAILBOX2_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX3                                                                                0x0043
+#define regVPEC_MAILBOX3_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX4                                                                                0x0044
+#define regVPEC_MAILBOX4_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX5                                                                                0x0045
+#define regVPEC_MAILBOX5_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX6                                                                                0x0046
+#define regVPEC_MAILBOX6_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX7                                                                                0x0047
+#define regVPEC_MAILBOX7_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX8                                                                                0x0048
+#define regVPEC_MAILBOX8_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX9                                                                                0x0049
+#define regVPEC_MAILBOX9_BASE_IDX                                                                       0
+#define regVPEC_MAILBOX10                                                                               0x004a
+#define regVPEC_MAILBOX10_BASE_IDX                                                                      0
+#define regVPEC_MAILBOX11                                                                               0x004b
+#define regVPEC_MAILBOX11_BASE_IDX                                                                      0
+#define regVPEC_MAILBOX12                                                                               0x004c
+#define regVPEC_MAILBOX12_BASE_IDX                                                                      0
+#define regVPEC_MAILBOX13                                                                               0x004d
+#define regVPEC_MAILBOX13_BASE_IDX                                                                      0
+#define regVPEC_MAILBOX14                                                                               0x004e
+#define regVPEC_MAILBOX14_BASE_IDX                                                                      0
+#define regVPEC_MAILBOX15                                                                               0x004f
+#define regVPEC_MAILBOX15_BASE_IDX                                                                      0
+#define regVPEC_PUB_DUMMY0                                                                              0x0050
+#define regVPEC_PUB_DUMMY0_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY1                                                                              0x0051
+#define regVPEC_PUB_DUMMY1_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY2                                                                              0x0052
+#define regVPEC_PUB_DUMMY2_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY3                                                                              0x0053
+#define regVPEC_PUB_DUMMY3_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY4                                                                              0x0054
+#define regVPEC_PUB_DUMMY4_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY5                                                                              0x0055
+#define regVPEC_PUB_DUMMY5_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY6                                                                              0x0056
+#define regVPEC_PUB_DUMMY6_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY7                                                                              0x0057
+#define regVPEC_PUB_DUMMY7_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY8                                                                              0x0058
+#define regVPEC_PUB_DUMMY8_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY9                                                                              0x0059
+#define regVPEC_PUB_DUMMY9_BASE_IDX                                                                     0
+#define regVPEC_PUB_DUMMY10                                                                             0x005a
+#define regVPEC_PUB_DUMMY10_BASE_IDX                                                                    0
+#define regVPEC_PUB_DUMMY11                                                                             0x005b
+#define regVPEC_PUB_DUMMY11_BASE_IDX                                                                    0
+#define regVPEC_UCODE1_CHECKSUM                                                                         0x005c
+#define regVPEC_UCODE1_CHECKSUM_BASE_IDX                                                                0
+#define regVPEC_VERSION                                                                                 0x005d
+#define regVPEC_VERSION_BASE_IDX                                                                        0
+#define regVPEC_UCODE_CHECKSUM                                                                          0x005e
+#define regVPEC_UCODE_CHECKSUM_BASE_IDX                                                                 0
+#define regVPEC_RB_RPTR_FETCH                                                                           0x005f
+#define regVPEC_RB_RPTR_FETCH_BASE_IDX                                                                  0
+#define regVPEC_RB_RPTR_FETCH_HI                                                                        0x0060
+#define regVPEC_RB_RPTR_FETCH_HI_BASE_IDX                                                               0
+#define regVPEC_IB_OFFSET_FETCH                                                                         0x0061
+#define regVPEC_IB_OFFSET_FETCH_BASE_IDX                                                                0
+#define regVPEC_CMDIB_OFFSET_FETCH                                                                      0x0062
+#define regVPEC_CMDIB_OFFSET_FETCH_BASE_IDX                                                             0
+#define regVPEC_3DLUTIB_OFFSET_FETCH                                                                    0x0063
+#define regVPEC_3DLUTIB_OFFSET_FETCH_BASE_IDX                                                           0
+#define regVPEC_ATOMIC_PREOP_LO                                                                         0x0064
+#define regVPEC_ATOMIC_PREOP_LO_BASE_IDX                                                                0
+#define regVPEC_ATOMIC_PREOP_HI                                                                         0x0065
+#define regVPEC_ATOMIC_PREOP_HI_BASE_IDX                                                                0
+#define regVPEC_CE_BUSY                                                                                 0x0066
+#define regVPEC_CE_BUSY_BASE_IDX                                                                        0
+#define regVPEC_F32_COUNTER                                                                             0x0067
+#define regVPEC_F32_COUNTER_BASE_IDX                                                                    0
+#define regVPEC_HOLE_ADDR_LO                                                                            0x0068
+#define regVPEC_HOLE_ADDR_LO_BASE_IDX                                                                   0
+#define regVPEC_HOLE_ADDR_HI                                                                            0x0069
+#define regVPEC_HOLE_ADDR_HI_BASE_IDX                                                                   0
+#define regVPEC_ERROR_LOG                                                                               0x006a
+#define regVPEC_ERROR_LOG_BASE_IDX                                                                      0
+#define regVPEC_INT_STATUS                                                                              0x006b
+#define regVPEC_INT_STATUS_BASE_IDX                                                                     0
+#define regVPEC_STATUS                                                                                  0x006c
+#define regVPEC_STATUS_BASE_IDX                                                                         0
+#define regVPEC_STATUS1                                                                                 0x006d
+#define regVPEC_STATUS1_BASE_IDX                                                                        0
+#define regVPEC_STATUS2                                                                                 0x006e
+#define regVPEC_STATUS2_BASE_IDX                                                                        0
+#define regVPEC_STATUS3                                                                                 0x006f
+#define regVPEC_STATUS3_BASE_IDX                                                                        0
+#define regVPEC_STATUS4                                                                                 0x0070
+#define regVPEC_STATUS4_BASE_IDX                                                                        0
+#define regVPEC_STATUS5                                                                                 0x0071
+#define regVPEC_STATUS5_BASE_IDX                                                                        0
+#define regVPEC_STATUS6                                                                                 0x0072
+#define regVPEC_STATUS6_BASE_IDX                                                                        0
+#define regVPEC_STATUS7                                                                                 0x0073
+#define regVPEC_STATUS7_BASE_IDX                                                                        0
+#define regVPEC_STATUS8                                                                                 0x0074
+#define regVPEC_STATUS8_BASE_IDX                                                                        0
+#define regVPEC_STATUS9                                                                                 0x0075
+#define regVPEC_STATUS9_BASE_IDX                                                                        0
+#define regVPEC_STATUS10                                                                                0x0076
+#define regVPEC_STATUS10_BASE_IDX                                                                       0
+#define regVPEC_STATUS_DCC                                                                              0x0077
+#define regVPEC_STATUS_DCC_BASE_IDX                                                                     0
+#define regVPEC_STATUS11                                                                                0x0078
+#define regVPEC_STATUS11_BASE_IDX                                                                       0
+#define regVPEC_INST                                                                                    0x0079
+#define regVPEC_INST_BASE_IDX                                                                           0
+#define regVPEC_QUEUE_STATUS0                                                                           0x007a
+#define regVPEC_QUEUE_STATUS0_BASE_IDX                                                                  0
+#define regVPEC_QUEUE_HANG_STATUS                                                                       0x007b
+#define regVPEC_QUEUE_HANG_STATUS_BASE_IDX                                                              0
+#define regVPEC_DPM_IDLE_TIME                                                                           0x007c
+#define regVPEC_DPM_IDLE_TIME_BASE_IDX                                                                  0
+#define regVPEC_DPM_BUSY_TIME                                                                           0x007d
+#define regVPEC_DPM_BUSY_TIME_BASE_IDX                                                                  0
+#define regVPEC_DPM_IDLE_START_LO                                                                       0x007e
+#define regVPEC_DPM_IDLE_START_LO_BASE_IDX                                                              0
+#define regVPEC_DPM_IDLE_START_HI                                                                       0x007f
+#define regVPEC_DPM_IDLE_START_HI_BASE_IDX                                                              0
+#define regVPEC_DPM_BUSY_START_LO                                                                       0x0080
+#define regVPEC_DPM_BUSY_START_LO_BASE_IDX                                                              0
+#define regVPEC_DPM_BUSY_START_HI                                                                       0x0081
+#define regVPEC_DPM_BUSY_START_HI_BASE_IDX                                                              0
+#define regVPEC_DPM_LAST_REQ_TIMESTAMP                                                                  0x0082
+#define regVPEC_DPM_LAST_REQ_TIMESTAMP_BASE_IDX                                                         0
+#define regVPEC_DPM_NEW_JOB_DUMMY3                                                                      0x0083
+#define regVPEC_DPM_NEW_JOB_DUMMY3_BASE_IDX                                                             0
+#define regVPEC_DPM_STATE                                                                               0x0084
+#define regVPEC_DPM_STATE_BASE_IDX                                                                      0
+#define regVPEC_DPM0_FREQ                                                                               0x0085
+#define regVPEC_DPM0_FREQ_BASE_IDX                                                                      0
+#define regVPEC_DPM1_FREQ                                                                               0x0086
+#define regVPEC_DPM1_FREQ_BASE_IDX                                                                      0
+#define regVPEC_DPM2_FREQ                                                                               0x0087
+#define regVPEC_DPM2_FREQ_BASE_IDX                                                                      0
+#define regVPEC_DPM3_FREQ                                                                               0x0088
+#define regVPEC_DPM3_FREQ_BASE_IDX                                                                      0
+#define regVPEC_DPM_THRESHOLD_SKIP                                                                      0x0089
+#define regVPEC_DPM_THRESHOLD_SKIP_BASE_IDX                                                             0
+#define regVPEC_DPM_THRESHOLD_BUSY_OVERFLOW                                                             0x008a
+#define regVPEC_DPM_THRESHOLD_BUSY_OVERFLOW_BASE_IDX                                                    0
+#define regVPEC_DPM_CALC_BUSY_IN_POSTPROCESS                                                            0x008b
+#define regVPEC_DPM_CALC_BUSY_IN_POSTPROCESS_BASE_IDX                                                   0
+#define regVPEC_DPM_IN_CHECKIDLE_LOOP                                                                   0x008c
+#define regVPEC_DPM_IN_CHECKIDLE_LOOP_BASE_IDX                                                          0
+#define regVPEC_DPM_THRESHOLD_IDLE_OVERFLOW                                                             0x008d
+#define regVPEC_DPM_THRESHOLD_IDLE_OVERFLOW_BASE_IDX                                                    0
+#define regVPEC_DPM_BUSY_CLAMP_COUNT                                                                    0x008e
+#define regVPEC_DPM_BUSY_CLAMP_COUNT_BASE_IDX                                                           0
+#define regVPEC_DPM_IDLE_CLAMP_COUNT                                                                    0x008f
+#define regVPEC_DPM_IDLE_CLAMP_COUNT_BASE_IDX                                                           0
+#define regVPEC_PG_CNTL                                                                                 0x00b8
+#define regVPEC_PG_CNTL_BASE_IDX                                                                        0
+#define regVPEC_PG_STATUS                                                                               0x00b9
+#define regVPEC_PG_STATUS_BASE_IDX                                                                      0
+#define regVPEC_CLOCK_GATING_STATUS                                                                     0x00ba
+#define regVPEC_CLOCK_GATING_STATUS_BASE_IDX                                                            0
+#define regVPEC_QUEUE0_RB_CNTL                                                                          0x00c0
+#define regVPEC_QUEUE0_RB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE0_SCHEDULE_CNTL                                                                    0x00c1
+#define regVPEC_QUEUE0_SCHEDULE_CNTL_BASE_IDX                                                           0
+#define regVPEC_QUEUE0_RB_BASE                                                                          0x00c2
+#define regVPEC_QUEUE0_RB_BASE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE0_RB_BASE_HI                                                                       0x00c3
+#define regVPEC_QUEUE0_RB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE0_RB_RPTR                                                                          0x00c4
+#define regVPEC_QUEUE0_RB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE0_RB_RPTR_HI                                                                       0x00c5
+#define regVPEC_QUEUE0_RB_RPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE0_RB_WPTR                                                                          0x00c6
+#define regVPEC_QUEUE0_RB_WPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE0_RB_WPTR_HI                                                                       0x00c7
+#define regVPEC_QUEUE0_RB_WPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE0_RB_RPTR_ADDR_HI                                                                  0x00c8
+#define regVPEC_QUEUE0_RB_RPTR_ADDR_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE0_RB_RPTR_ADDR_LO                                                                  0x00c9
+#define regVPEC_QUEUE0_RB_RPTR_ADDR_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE0_RB_AQL_CNTL                                                                      0x00ca
+#define regVPEC_QUEUE0_RB_AQL_CNTL_BASE_IDX                                                             0
+#define regVPEC_QUEUE0_MINOR_PTR_UPDATE                                                                 0x00cb
+#define regVPEC_QUEUE0_MINOR_PTR_UPDATE_BASE_IDX                                                        0
+#define regVPEC_QUEUE0_CD_INFO                                                                          0x00cc
+#define regVPEC_QUEUE0_CD_INFO_BASE_IDX                                                                 0
+#define regVPEC_QUEUE0_RB_PREEMPT                                                                       0x00cd
+#define regVPEC_QUEUE0_RB_PREEMPT_BASE_IDX                                                              0
+#define regVPEC_QUEUE0_SKIP_CNTL                                                                        0x00ce
+#define regVPEC_QUEUE0_SKIP_CNTL_BASE_IDX                                                               0
+#define regVPEC_QUEUE0_DOORBELL                                                                         0x00cf
+#define regVPEC_QUEUE0_DOORBELL_BASE_IDX                                                                0
+#define regVPEC_QUEUE0_DOORBELL_OFFSET                                                                  0x00d0
+#define regVPEC_QUEUE0_DOORBELL_OFFSET_BASE_IDX                                                         0
+#define regVPEC_QUEUE0_DUMMY0                                                                           0x00d1
+#define regVPEC_QUEUE0_DUMMY0_BASE_IDX                                                                  0
+#define regVPEC_QUEUE0_DUMMY1                                                                           0x00d2
+#define regVPEC_QUEUE0_DUMMY1_BASE_IDX                                                                  0
+#define regVPEC_QUEUE0_DUMMY2                                                                           0x00d3
+#define regVPEC_QUEUE0_DUMMY2_BASE_IDX                                                                  0
+#define regVPEC_QUEUE0_DUMMY3                                                                           0x00d4
+#define regVPEC_QUEUE0_DUMMY3_BASE_IDX                                                                  0
+#define regVPEC_QUEUE0_DUMMY4                                                                           0x00d5
+#define regVPEC_QUEUE0_DUMMY4_BASE_IDX                                                                  0
+#define regVPEC_QUEUE0_IB_CNTL                                                                          0x00ec
+#define regVPEC_QUEUE0_IB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE0_IB_RPTR                                                                          0x00ed
+#define regVPEC_QUEUE0_IB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE0_IB_OFFSET                                                                        0x00ee
+#define regVPEC_QUEUE0_IB_OFFSET_BASE_IDX                                                               0
+#define regVPEC_QUEUE0_IB_BASE_LO                                                                       0x00ef
+#define regVPEC_QUEUE0_IB_BASE_LO_BASE_IDX                                                              0
+#define regVPEC_QUEUE0_IB_BASE_HI                                                                       0x00f0
+#define regVPEC_QUEUE0_IB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE0_IB_SIZE                                                                          0x00f1
+#define regVPEC_QUEUE0_IB_SIZE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE0_CMDIB_CNTL                                                                       0x00f2
+#define regVPEC_QUEUE0_CMDIB_CNTL_BASE_IDX                                                              0
+#define regVPEC_QUEUE0_CMDIB_RPTR                                                                       0x00f3
+#define regVPEC_QUEUE0_CMDIB_RPTR_BASE_IDX                                                              0
+#define regVPEC_QUEUE0_CMDIB_OFFSET                                                                     0x00f4
+#define regVPEC_QUEUE0_CMDIB_OFFSET_BASE_IDX                                                            0
+#define regVPEC_QUEUE0_CMDIB_BASE_LO                                                                    0x00f5
+#define regVPEC_QUEUE0_CMDIB_BASE_LO_BASE_IDX                                                           0
+#define regVPEC_QUEUE0_CMDIB_BASE_HI                                                                    0x00f6
+#define regVPEC_QUEUE0_CMDIB_BASE_HI_BASE_IDX                                                           0
+#define regVPEC_QUEUE0_CMDIB_SIZE                                                                       0x00f7
+#define regVPEC_QUEUE0_CMDIB_SIZE_BASE_IDX                                                              0
+#define regVPEC_QUEUE0_3DLUTIB_CNTL                                                                     0x00f8
+#define regVPEC_QUEUE0_3DLUTIB_CNTL_BASE_IDX                                                            0
+#define regVPEC_QUEUE0_3DLUTIB_RPTR                                                                     0x00f9
+#define regVPEC_QUEUE0_3DLUTIB_RPTR_BASE_IDX                                                            0
+#define regVPEC_QUEUE0_3DLUTIB_OFFSET                                                                   0x00fa
+#define regVPEC_QUEUE0_3DLUTIB_OFFSET_BASE_IDX                                                          0
+#define regVPEC_QUEUE0_3DLUTIB_BASE_LO                                                                  0x00fb
+#define regVPEC_QUEUE0_3DLUTIB_BASE_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE0_3DLUTIB_BASE_HI                                                                  0x00fc
+#define regVPEC_QUEUE0_3DLUTIB_BASE_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE0_3DLUTIB_SIZE                                                                     0x00fd
+#define regVPEC_QUEUE0_3DLUTIB_SIZE_BASE_IDX                                                            0
+#define regVPEC_QUEUE0_CSA_ADDR_LO                                                                      0x00fe
+#define regVPEC_QUEUE0_CSA_ADDR_LO_BASE_IDX                                                             0
+#define regVPEC_QUEUE0_CSA_ADDR_HI                                                                      0x00ff
+#define regVPEC_QUEUE0_CSA_ADDR_HI_BASE_IDX                                                             0
+#define regVPEC_QUEUE0_CONTEXT_STATUS                                                                   0x0100
+#define regVPEC_QUEUE0_CONTEXT_STATUS_BASE_IDX                                                          0
+#define regVPEC_QUEUE0_DOORBELL_LOG                                                                     0x0101
+#define regVPEC_QUEUE0_DOORBELL_LOG_BASE_IDX                                                            0
+#define regVPEC_QUEUE0_IB_SUB_REMAIN                                                                    0x0102
+#define regVPEC_QUEUE0_IB_SUB_REMAIN_BASE_IDX                                                           0
+#define regVPEC_QUEUE0_PREEMPT                                                                          0x0103
+#define regVPEC_QUEUE0_PREEMPT_BASE_IDX                                                                 0
+#define regVPEC_QUEUE0_LOG0BUFFER_CFG                                                                   0x0104
+#define regVPEC_QUEUE0_LOG0BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE0_LOG1BUFFER_CFG                                                                   0x0105
+#define regVPEC_QUEUE0_LOG1BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE1_RB_CNTL                                                                          0x0118
+#define regVPEC_QUEUE1_RB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE1_SCHEDULE_CNTL                                                                    0x0119
+#define regVPEC_QUEUE1_SCHEDULE_CNTL_BASE_IDX                                                           0
+#define regVPEC_QUEUE1_RB_BASE                                                                          0x011a
+#define regVPEC_QUEUE1_RB_BASE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE1_RB_BASE_HI                                                                       0x011b
+#define regVPEC_QUEUE1_RB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE1_RB_RPTR                                                                          0x011c
+#define regVPEC_QUEUE1_RB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE1_RB_RPTR_HI                                                                       0x011d
+#define regVPEC_QUEUE1_RB_RPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE1_RB_WPTR                                                                          0x011e
+#define regVPEC_QUEUE1_RB_WPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE1_RB_WPTR_HI                                                                       0x011f
+#define regVPEC_QUEUE1_RB_WPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE1_RB_RPTR_ADDR_HI                                                                  0x0120
+#define regVPEC_QUEUE1_RB_RPTR_ADDR_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE1_RB_RPTR_ADDR_LO                                                                  0x0121
+#define regVPEC_QUEUE1_RB_RPTR_ADDR_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE1_RB_AQL_CNTL                                                                      0x0122
+#define regVPEC_QUEUE1_RB_AQL_CNTL_BASE_IDX                                                             0
+#define regVPEC_QUEUE1_MINOR_PTR_UPDATE                                                                 0x0123
+#define regVPEC_QUEUE1_MINOR_PTR_UPDATE_BASE_IDX                                                        0
+#define regVPEC_QUEUE1_CD_INFO                                                                          0x0124
+#define regVPEC_QUEUE1_CD_INFO_BASE_IDX                                                                 0
+#define regVPEC_QUEUE1_RB_PREEMPT                                                                       0x0125
+#define regVPEC_QUEUE1_RB_PREEMPT_BASE_IDX                                                              0
+#define regVPEC_QUEUE1_SKIP_CNTL                                                                        0x0126
+#define regVPEC_QUEUE1_SKIP_CNTL_BASE_IDX                                                               0
+#define regVPEC_QUEUE1_DOORBELL                                                                         0x0127
+#define regVPEC_QUEUE1_DOORBELL_BASE_IDX                                                                0
+#define regVPEC_QUEUE1_DOORBELL_OFFSET                                                                  0x0128
+#define regVPEC_QUEUE1_DOORBELL_OFFSET_BASE_IDX                                                         0
+#define regVPEC_QUEUE1_DUMMY0                                                                           0x0129
+#define regVPEC_QUEUE1_DUMMY0_BASE_IDX                                                                  0
+#define regVPEC_QUEUE1_DUMMY1                                                                           0x012a
+#define regVPEC_QUEUE1_DUMMY1_BASE_IDX                                                                  0
+#define regVPEC_QUEUE1_DUMMY2                                                                           0x012b
+#define regVPEC_QUEUE1_DUMMY2_BASE_IDX                                                                  0
+#define regVPEC_QUEUE1_DUMMY3                                                                           0x012c
+#define regVPEC_QUEUE1_DUMMY3_BASE_IDX                                                                  0
+#define regVPEC_QUEUE1_DUMMY4                                                                           0x012d
+#define regVPEC_QUEUE1_DUMMY4_BASE_IDX                                                                  0
+#define regVPEC_QUEUE1_IB_CNTL                                                                          0x0144
+#define regVPEC_QUEUE1_IB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE1_IB_RPTR                                                                          0x0145
+#define regVPEC_QUEUE1_IB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE1_IB_OFFSET                                                                        0x0146
+#define regVPEC_QUEUE1_IB_OFFSET_BASE_IDX                                                               0
+#define regVPEC_QUEUE1_IB_BASE_LO                                                                       0x0147
+#define regVPEC_QUEUE1_IB_BASE_LO_BASE_IDX                                                              0
+#define regVPEC_QUEUE1_IB_BASE_HI                                                                       0x0148
+#define regVPEC_QUEUE1_IB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE1_IB_SIZE                                                                          0x0149
+#define regVPEC_QUEUE1_IB_SIZE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE1_CMDIB_CNTL                                                                       0x014a
+#define regVPEC_QUEUE1_CMDIB_CNTL_BASE_IDX                                                              0
+#define regVPEC_QUEUE1_CMDIB_RPTR                                                                       0x014b
+#define regVPEC_QUEUE1_CMDIB_RPTR_BASE_IDX                                                              0
+#define regVPEC_QUEUE1_CMDIB_OFFSET                                                                     0x014c
+#define regVPEC_QUEUE1_CMDIB_OFFSET_BASE_IDX                                                            0
+#define regVPEC_QUEUE1_CMDIB_BASE_LO                                                                    0x014d
+#define regVPEC_QUEUE1_CMDIB_BASE_LO_BASE_IDX                                                           0
+#define regVPEC_QUEUE1_CMDIB_BASE_HI                                                                    0x014e
+#define regVPEC_QUEUE1_CMDIB_BASE_HI_BASE_IDX                                                           0
+#define regVPEC_QUEUE1_CMDIB_SIZE                                                                       0x014f
+#define regVPEC_QUEUE1_CMDIB_SIZE_BASE_IDX                                                              0
+#define regVPEC_QUEUE1_3DLUTIB_CNTL                                                                     0x0150
+#define regVPEC_QUEUE1_3DLUTIB_CNTL_BASE_IDX                                                            0
+#define regVPEC_QUEUE1_3DLUTIB_RPTR                                                                     0x0151
+#define regVPEC_QUEUE1_3DLUTIB_RPTR_BASE_IDX                                                            0
+#define regVPEC_QUEUE1_3DLUTIB_OFFSET                                                                   0x0152
+#define regVPEC_QUEUE1_3DLUTIB_OFFSET_BASE_IDX                                                          0
+#define regVPEC_QUEUE1_3DLUTIB_BASE_LO                                                                  0x0153
+#define regVPEC_QUEUE1_3DLUTIB_BASE_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE1_3DLUTIB_BASE_HI                                                                  0x0154
+#define regVPEC_QUEUE1_3DLUTIB_BASE_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE1_3DLUTIB_SIZE                                                                     0x0155
+#define regVPEC_QUEUE1_3DLUTIB_SIZE_BASE_IDX                                                            0
+#define regVPEC_QUEUE1_CSA_ADDR_LO                                                                      0x0156
+#define regVPEC_QUEUE1_CSA_ADDR_LO_BASE_IDX                                                             0
+#define regVPEC_QUEUE1_CSA_ADDR_HI                                                                      0x0157
+#define regVPEC_QUEUE1_CSA_ADDR_HI_BASE_IDX                                                             0
+#define regVPEC_QUEUE1_CONTEXT_STATUS                                                                   0x0158
+#define regVPEC_QUEUE1_CONTEXT_STATUS_BASE_IDX                                                          0
+#define regVPEC_QUEUE1_DOORBELL_LOG                                                                     0x0159
+#define regVPEC_QUEUE1_DOORBELL_LOG_BASE_IDX                                                            0
+#define regVPEC_QUEUE1_IB_SUB_REMAIN                                                                    0x015a
+#define regVPEC_QUEUE1_IB_SUB_REMAIN_BASE_IDX                                                           0
+#define regVPEC_QUEUE1_PREEMPT                                                                          0x015b
+#define regVPEC_QUEUE1_PREEMPT_BASE_IDX                                                                 0
+#define regVPEC_QUEUE1_LOG0BUFFER_CFG                                                                   0x015c
+#define regVPEC_QUEUE1_LOG0BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE1_LOG1BUFFER_CFG                                                                   0x015d
+#define regVPEC_QUEUE1_LOG1BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE2_RB_CNTL                                                                          0x0170
+#define regVPEC_QUEUE2_RB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE2_SCHEDULE_CNTL                                                                    0x0171
+#define regVPEC_QUEUE2_SCHEDULE_CNTL_BASE_IDX                                                           0
+#define regVPEC_QUEUE2_RB_BASE                                                                          0x0172
+#define regVPEC_QUEUE2_RB_BASE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE2_RB_BASE_HI                                                                       0x0173
+#define regVPEC_QUEUE2_RB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE2_RB_RPTR                                                                          0x0174
+#define regVPEC_QUEUE2_RB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE2_RB_RPTR_HI                                                                       0x0175
+#define regVPEC_QUEUE2_RB_RPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE2_RB_WPTR                                                                          0x0176
+#define regVPEC_QUEUE2_RB_WPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE2_RB_WPTR_HI                                                                       0x0177
+#define regVPEC_QUEUE2_RB_WPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE2_RB_RPTR_ADDR_HI                                                                  0x0178
+#define regVPEC_QUEUE2_RB_RPTR_ADDR_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE2_RB_RPTR_ADDR_LO                                                                  0x0179
+#define regVPEC_QUEUE2_RB_RPTR_ADDR_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE2_RB_AQL_CNTL                                                                      0x017a
+#define regVPEC_QUEUE2_RB_AQL_CNTL_BASE_IDX                                                             0
+#define regVPEC_QUEUE2_MINOR_PTR_UPDATE                                                                 0x017b
+#define regVPEC_QUEUE2_MINOR_PTR_UPDATE_BASE_IDX                                                        0
+#define regVPEC_QUEUE2_CD_INFO                                                                          0x017c
+#define regVPEC_QUEUE2_CD_INFO_BASE_IDX                                                                 0
+#define regVPEC_QUEUE2_RB_PREEMPT                                                                       0x017d
+#define regVPEC_QUEUE2_RB_PREEMPT_BASE_IDX                                                              0
+#define regVPEC_QUEUE2_SKIP_CNTL                                                                        0x017e
+#define regVPEC_QUEUE2_SKIP_CNTL_BASE_IDX                                                               0
+#define regVPEC_QUEUE2_DOORBELL                                                                         0x017f
+#define regVPEC_QUEUE2_DOORBELL_BASE_IDX                                                                0
+#define regVPEC_QUEUE2_DOORBELL_OFFSET                                                                  0x0180
+#define regVPEC_QUEUE2_DOORBELL_OFFSET_BASE_IDX                                                         0
+#define regVPEC_QUEUE2_DUMMY0                                                                           0x0181
+#define regVPEC_QUEUE2_DUMMY0_BASE_IDX                                                                  0
+#define regVPEC_QUEUE2_DUMMY1                                                                           0x0182
+#define regVPEC_QUEUE2_DUMMY1_BASE_IDX                                                                  0
+#define regVPEC_QUEUE2_DUMMY2                                                                           0x0183
+#define regVPEC_QUEUE2_DUMMY2_BASE_IDX                                                                  0
+#define regVPEC_QUEUE2_DUMMY3                                                                           0x0184
+#define regVPEC_QUEUE2_DUMMY3_BASE_IDX                                                                  0
+#define regVPEC_QUEUE2_DUMMY4                                                                           0x0185
+#define regVPEC_QUEUE2_DUMMY4_BASE_IDX                                                                  0
+#define regVPEC_QUEUE2_IB_CNTL                                                                          0x019c
+#define regVPEC_QUEUE2_IB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE2_IB_RPTR                                                                          0x019d
+#define regVPEC_QUEUE2_IB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE2_IB_OFFSET                                                                        0x019e
+#define regVPEC_QUEUE2_IB_OFFSET_BASE_IDX                                                               0
+#define regVPEC_QUEUE2_IB_BASE_LO                                                                       0x019f
+#define regVPEC_QUEUE2_IB_BASE_LO_BASE_IDX                                                              0
+#define regVPEC_QUEUE2_IB_BASE_HI                                                                       0x01a0
+#define regVPEC_QUEUE2_IB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE2_IB_SIZE                                                                          0x01a1
+#define regVPEC_QUEUE2_IB_SIZE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE2_CMDIB_CNTL                                                                       0x01a2
+#define regVPEC_QUEUE2_CMDIB_CNTL_BASE_IDX                                                              0
+#define regVPEC_QUEUE2_CMDIB_RPTR                                                                       0x01a3
+#define regVPEC_QUEUE2_CMDIB_RPTR_BASE_IDX                                                              0
+#define regVPEC_QUEUE2_CMDIB_OFFSET                                                                     0x01a4
+#define regVPEC_QUEUE2_CMDIB_OFFSET_BASE_IDX                                                            0
+#define regVPEC_QUEUE2_CMDIB_BASE_LO                                                                    0x01a5
+#define regVPEC_QUEUE2_CMDIB_BASE_LO_BASE_IDX                                                           0
+#define regVPEC_QUEUE2_CMDIB_BASE_HI                                                                    0x01a6
+#define regVPEC_QUEUE2_CMDIB_BASE_HI_BASE_IDX                                                           0
+#define regVPEC_QUEUE2_CMDIB_SIZE                                                                       0x01a7
+#define regVPEC_QUEUE2_CMDIB_SIZE_BASE_IDX                                                              0
+#define regVPEC_QUEUE2_3DLUTIB_CNTL                                                                     0x01a8
+#define regVPEC_QUEUE2_3DLUTIB_CNTL_BASE_IDX                                                            0
+#define regVPEC_QUEUE2_3DLUTIB_RPTR                                                                     0x01a9
+#define regVPEC_QUEUE2_3DLUTIB_RPTR_BASE_IDX                                                            0
+#define regVPEC_QUEUE2_3DLUTIB_OFFSET                                                                   0x01aa
+#define regVPEC_QUEUE2_3DLUTIB_OFFSET_BASE_IDX                                                          0
+#define regVPEC_QUEUE2_3DLUTIB_BASE_LO                                                                  0x01ab
+#define regVPEC_QUEUE2_3DLUTIB_BASE_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE2_3DLUTIB_BASE_HI                                                                  0x01ac
+#define regVPEC_QUEUE2_3DLUTIB_BASE_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE2_3DLUTIB_SIZE                                                                     0x01ad
+#define regVPEC_QUEUE2_3DLUTIB_SIZE_BASE_IDX                                                            0
+#define regVPEC_QUEUE2_CSA_ADDR_LO                                                                      0x01ae
+#define regVPEC_QUEUE2_CSA_ADDR_LO_BASE_IDX                                                             0
+#define regVPEC_QUEUE2_CSA_ADDR_HI                                                                      0x01af
+#define regVPEC_QUEUE2_CSA_ADDR_HI_BASE_IDX                                                             0
+#define regVPEC_QUEUE2_CONTEXT_STATUS                                                                   0x01b0
+#define regVPEC_QUEUE2_CONTEXT_STATUS_BASE_IDX                                                          0
+#define regVPEC_QUEUE2_DOORBELL_LOG                                                                     0x01b1
+#define regVPEC_QUEUE2_DOORBELL_LOG_BASE_IDX                                                            0
+#define regVPEC_QUEUE2_IB_SUB_REMAIN                                                                    0x01b2
+#define regVPEC_QUEUE2_IB_SUB_REMAIN_BASE_IDX                                                           0
+#define regVPEC_QUEUE2_PREEMPT                                                                          0x01b3
+#define regVPEC_QUEUE2_PREEMPT_BASE_IDX                                                                 0
+#define regVPEC_QUEUE2_LOG0BUFFER_CFG                                                                   0x01b4
+#define regVPEC_QUEUE2_LOG0BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE2_LOG1BUFFER_CFG                                                                   0x01b5
+#define regVPEC_QUEUE2_LOG1BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE3_RB_CNTL                                                                          0x01c8
+#define regVPEC_QUEUE3_RB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE3_SCHEDULE_CNTL                                                                    0x01c9
+#define regVPEC_QUEUE3_SCHEDULE_CNTL_BASE_IDX                                                           0
+#define regVPEC_QUEUE3_RB_BASE                                                                          0x01ca
+#define regVPEC_QUEUE3_RB_BASE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE3_RB_BASE_HI                                                                       0x01cb
+#define regVPEC_QUEUE3_RB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE3_RB_RPTR                                                                          0x01cc
+#define regVPEC_QUEUE3_RB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE3_RB_RPTR_HI                                                                       0x01cd
+#define regVPEC_QUEUE3_RB_RPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE3_RB_WPTR                                                                          0x01ce
+#define regVPEC_QUEUE3_RB_WPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE3_RB_WPTR_HI                                                                       0x01cf
+#define regVPEC_QUEUE3_RB_WPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE3_RB_RPTR_ADDR_HI                                                                  0x01d0
+#define regVPEC_QUEUE3_RB_RPTR_ADDR_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE3_RB_RPTR_ADDR_LO                                                                  0x01d1
+#define regVPEC_QUEUE3_RB_RPTR_ADDR_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE3_RB_AQL_CNTL                                                                      0x01d2
+#define regVPEC_QUEUE3_RB_AQL_CNTL_BASE_IDX                                                             0
+#define regVPEC_QUEUE3_MINOR_PTR_UPDATE                                                                 0x01d3
+#define regVPEC_QUEUE3_MINOR_PTR_UPDATE_BASE_IDX                                                        0
+#define regVPEC_QUEUE3_CD_INFO                                                                          0x01d4
+#define regVPEC_QUEUE3_CD_INFO_BASE_IDX                                                                 0
+#define regVPEC_QUEUE3_RB_PREEMPT                                                                       0x01d5
+#define regVPEC_QUEUE3_RB_PREEMPT_BASE_IDX                                                              0
+#define regVPEC_QUEUE3_SKIP_CNTL                                                                        0x01d6
+#define regVPEC_QUEUE3_SKIP_CNTL_BASE_IDX                                                               0
+#define regVPEC_QUEUE3_DOORBELL                                                                         0x01d7
+#define regVPEC_QUEUE3_DOORBELL_BASE_IDX                                                                0
+#define regVPEC_QUEUE3_DOORBELL_OFFSET                                                                  0x01d8
+#define regVPEC_QUEUE3_DOORBELL_OFFSET_BASE_IDX                                                         0
+#define regVPEC_QUEUE3_DUMMY0                                                                           0x01d9
+#define regVPEC_QUEUE3_DUMMY0_BASE_IDX                                                                  0
+#define regVPEC_QUEUE3_DUMMY1                                                                           0x01da
+#define regVPEC_QUEUE3_DUMMY1_BASE_IDX                                                                  0
+#define regVPEC_QUEUE3_DUMMY2                                                                           0x01db
+#define regVPEC_QUEUE3_DUMMY2_BASE_IDX                                                                  0
+#define regVPEC_QUEUE3_DUMMY3                                                                           0x01dc
+#define regVPEC_QUEUE3_DUMMY3_BASE_IDX                                                                  0
+#define regVPEC_QUEUE3_DUMMY4                                                                           0x01dd
+#define regVPEC_QUEUE3_DUMMY4_BASE_IDX                                                                  0
+#define regVPEC_QUEUE3_IB_CNTL                                                                          0x01f4
+#define regVPEC_QUEUE3_IB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE3_IB_RPTR                                                                          0x01f5
+#define regVPEC_QUEUE3_IB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE3_IB_OFFSET                                                                        0x01f6
+#define regVPEC_QUEUE3_IB_OFFSET_BASE_IDX                                                               0
+#define regVPEC_QUEUE3_IB_BASE_LO                                                                       0x01f7
+#define regVPEC_QUEUE3_IB_BASE_LO_BASE_IDX                                                              0
+#define regVPEC_QUEUE3_IB_BASE_HI                                                                       0x01f8
+#define regVPEC_QUEUE3_IB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE3_IB_SIZE                                                                          0x01f9
+#define regVPEC_QUEUE3_IB_SIZE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE3_CMDIB_CNTL                                                                       0x01fa
+#define regVPEC_QUEUE3_CMDIB_CNTL_BASE_IDX                                                              0
+#define regVPEC_QUEUE3_CMDIB_RPTR                                                                       0x01fb
+#define regVPEC_QUEUE3_CMDIB_RPTR_BASE_IDX                                                              0
+#define regVPEC_QUEUE3_CMDIB_OFFSET                                                                     0x01fc
+#define regVPEC_QUEUE3_CMDIB_OFFSET_BASE_IDX                                                            0
+#define regVPEC_QUEUE3_CMDIB_BASE_LO                                                                    0x01fd
+#define regVPEC_QUEUE3_CMDIB_BASE_LO_BASE_IDX                                                           0
+#define regVPEC_QUEUE3_CMDIB_BASE_HI                                                                    0x01fe
+#define regVPEC_QUEUE3_CMDIB_BASE_HI_BASE_IDX                                                           0
+#define regVPEC_QUEUE3_CMDIB_SIZE                                                                       0x01ff
+#define regVPEC_QUEUE3_CMDIB_SIZE_BASE_IDX                                                              0
+#define regVPEC_QUEUE3_3DLUTIB_CNTL                                                                     0x0200
+#define regVPEC_QUEUE3_3DLUTIB_CNTL_BASE_IDX                                                            0
+#define regVPEC_QUEUE3_3DLUTIB_RPTR                                                                     0x0201
+#define regVPEC_QUEUE3_3DLUTIB_RPTR_BASE_IDX                                                            0
+#define regVPEC_QUEUE3_3DLUTIB_OFFSET                                                                   0x0202
+#define regVPEC_QUEUE3_3DLUTIB_OFFSET_BASE_IDX                                                          0
+#define regVPEC_QUEUE3_3DLUTIB_BASE_LO                                                                  0x0203
+#define regVPEC_QUEUE3_3DLUTIB_BASE_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE3_3DLUTIB_BASE_HI                                                                  0x0204
+#define regVPEC_QUEUE3_3DLUTIB_BASE_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE3_3DLUTIB_SIZE                                                                     0x0205
+#define regVPEC_QUEUE3_3DLUTIB_SIZE_BASE_IDX                                                            0
+#define regVPEC_QUEUE3_CSA_ADDR_LO                                                                      0x0206
+#define regVPEC_QUEUE3_CSA_ADDR_LO_BASE_IDX                                                             0
+#define regVPEC_QUEUE3_CSA_ADDR_HI                                                                      0x0207
+#define regVPEC_QUEUE3_CSA_ADDR_HI_BASE_IDX                                                             0
+#define regVPEC_QUEUE3_CONTEXT_STATUS                                                                   0x0208
+#define regVPEC_QUEUE3_CONTEXT_STATUS_BASE_IDX                                                          0
+#define regVPEC_QUEUE3_DOORBELL_LOG                                                                     0x0209
+#define regVPEC_QUEUE3_DOORBELL_LOG_BASE_IDX                                                            0
+#define regVPEC_QUEUE3_IB_SUB_REMAIN                                                                    0x020a
+#define regVPEC_QUEUE3_IB_SUB_REMAIN_BASE_IDX                                                           0
+#define regVPEC_QUEUE3_PREEMPT                                                                          0x020b
+#define regVPEC_QUEUE3_PREEMPT_BASE_IDX                                                                 0
+#define regVPEC_QUEUE3_LOG0BUFFER_CFG                                                                   0x020c
+#define regVPEC_QUEUE3_LOG0BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE3_LOG1BUFFER_CFG                                                                   0x020d
+#define regVPEC_QUEUE3_LOG1BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE4_RB_CNTL                                                                          0x0220
+#define regVPEC_QUEUE4_RB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE4_SCHEDULE_CNTL                                                                    0x0221
+#define regVPEC_QUEUE4_SCHEDULE_CNTL_BASE_IDX                                                           0
+#define regVPEC_QUEUE4_RB_BASE                                                                          0x0222
+#define regVPEC_QUEUE4_RB_BASE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE4_RB_BASE_HI                                                                       0x0223
+#define regVPEC_QUEUE4_RB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE4_RB_RPTR                                                                          0x0224
+#define regVPEC_QUEUE4_RB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE4_RB_RPTR_HI                                                                       0x0225
+#define regVPEC_QUEUE4_RB_RPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE4_RB_WPTR                                                                          0x0226
+#define regVPEC_QUEUE4_RB_WPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE4_RB_WPTR_HI                                                                       0x0227
+#define regVPEC_QUEUE4_RB_WPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE4_RB_RPTR_ADDR_HI                                                                  0x0228
+#define regVPEC_QUEUE4_RB_RPTR_ADDR_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE4_RB_RPTR_ADDR_LO                                                                  0x0229
+#define regVPEC_QUEUE4_RB_RPTR_ADDR_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE4_RB_AQL_CNTL                                                                      0x022a
+#define regVPEC_QUEUE4_RB_AQL_CNTL_BASE_IDX                                                             0
+#define regVPEC_QUEUE4_MINOR_PTR_UPDATE                                                                 0x022b
+#define regVPEC_QUEUE4_MINOR_PTR_UPDATE_BASE_IDX                                                        0
+#define regVPEC_QUEUE4_CD_INFO                                                                          0x022c
+#define regVPEC_QUEUE4_CD_INFO_BASE_IDX                                                                 0
+#define regVPEC_QUEUE4_RB_PREEMPT                                                                       0x022d
+#define regVPEC_QUEUE4_RB_PREEMPT_BASE_IDX                                                              0
+#define regVPEC_QUEUE4_SKIP_CNTL                                                                        0x022e
+#define regVPEC_QUEUE4_SKIP_CNTL_BASE_IDX                                                               0
+#define regVPEC_QUEUE4_DOORBELL                                                                         0x022f
+#define regVPEC_QUEUE4_DOORBELL_BASE_IDX                                                                0
+#define regVPEC_QUEUE4_DOORBELL_OFFSET                                                                  0x0230
+#define regVPEC_QUEUE4_DOORBELL_OFFSET_BASE_IDX                                                         0
+#define regVPEC_QUEUE4_DUMMY0                                                                           0x0231
+#define regVPEC_QUEUE4_DUMMY0_BASE_IDX                                                                  0
+#define regVPEC_QUEUE4_DUMMY1                                                                           0x0232
+#define regVPEC_QUEUE4_DUMMY1_BASE_IDX                                                                  0
+#define regVPEC_QUEUE4_DUMMY2                                                                           0x0233
+#define regVPEC_QUEUE4_DUMMY2_BASE_IDX                                                                  0
+#define regVPEC_QUEUE4_DUMMY3                                                                           0x0234
+#define regVPEC_QUEUE4_DUMMY3_BASE_IDX                                                                  0
+#define regVPEC_QUEUE4_DUMMY4                                                                           0x0235
+#define regVPEC_QUEUE4_DUMMY4_BASE_IDX                                                                  0
+#define regVPEC_QUEUE4_IB_CNTL                                                                          0x024c
+#define regVPEC_QUEUE4_IB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE4_IB_RPTR                                                                          0x024d
+#define regVPEC_QUEUE4_IB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE4_IB_OFFSET                                                                        0x024e
+#define regVPEC_QUEUE4_IB_OFFSET_BASE_IDX                                                               0
+#define regVPEC_QUEUE4_IB_BASE_LO                                                                       0x024f
+#define regVPEC_QUEUE4_IB_BASE_LO_BASE_IDX                                                              0
+#define regVPEC_QUEUE4_IB_BASE_HI                                                                       0x0250
+#define regVPEC_QUEUE4_IB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE4_IB_SIZE                                                                          0x0251
+#define regVPEC_QUEUE4_IB_SIZE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE4_CMDIB_CNTL                                                                       0x0252
+#define regVPEC_QUEUE4_CMDIB_CNTL_BASE_IDX                                                              0
+#define regVPEC_QUEUE4_CMDIB_RPTR                                                                       0x0253
+#define regVPEC_QUEUE4_CMDIB_RPTR_BASE_IDX                                                              0
+#define regVPEC_QUEUE4_CMDIB_OFFSET                                                                     0x0254
+#define regVPEC_QUEUE4_CMDIB_OFFSET_BASE_IDX                                                            0
+#define regVPEC_QUEUE4_CMDIB_BASE_LO                                                                    0x0255
+#define regVPEC_QUEUE4_CMDIB_BASE_LO_BASE_IDX                                                           0
+#define regVPEC_QUEUE4_CMDIB_BASE_HI                                                                    0x0256
+#define regVPEC_QUEUE4_CMDIB_BASE_HI_BASE_IDX                                                           0
+#define regVPEC_QUEUE4_CMDIB_SIZE                                                                       0x0257
+#define regVPEC_QUEUE4_CMDIB_SIZE_BASE_IDX                                                              0
+#define regVPEC_QUEUE4_3DLUTIB_CNTL                                                                     0x0258
+#define regVPEC_QUEUE4_3DLUTIB_CNTL_BASE_IDX                                                            0
+#define regVPEC_QUEUE4_3DLUTIB_RPTR                                                                     0x0259
+#define regVPEC_QUEUE4_3DLUTIB_RPTR_BASE_IDX                                                            0
+#define regVPEC_QUEUE4_3DLUTIB_OFFSET                                                                   0x025a
+#define regVPEC_QUEUE4_3DLUTIB_OFFSET_BASE_IDX                                                          0
+#define regVPEC_QUEUE4_3DLUTIB_BASE_LO                                                                  0x025b
+#define regVPEC_QUEUE4_3DLUTIB_BASE_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE4_3DLUTIB_BASE_HI                                                                  0x025c
+#define regVPEC_QUEUE4_3DLUTIB_BASE_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE4_3DLUTIB_SIZE                                                                     0x025d
+#define regVPEC_QUEUE4_3DLUTIB_SIZE_BASE_IDX                                                            0
+#define regVPEC_QUEUE4_CSA_ADDR_LO                                                                      0x025e
+#define regVPEC_QUEUE4_CSA_ADDR_LO_BASE_IDX                                                             0
+#define regVPEC_QUEUE4_CSA_ADDR_HI                                                                      0x025f
+#define regVPEC_QUEUE4_CSA_ADDR_HI_BASE_IDX                                                             0
+#define regVPEC_QUEUE4_CONTEXT_STATUS                                                                   0x0260
+#define regVPEC_QUEUE4_CONTEXT_STATUS_BASE_IDX                                                          0
+#define regVPEC_QUEUE4_DOORBELL_LOG                                                                     0x0261
+#define regVPEC_QUEUE4_DOORBELL_LOG_BASE_IDX                                                            0
+#define regVPEC_QUEUE4_IB_SUB_REMAIN                                                                    0x0262
+#define regVPEC_QUEUE4_IB_SUB_REMAIN_BASE_IDX                                                           0
+#define regVPEC_QUEUE4_PREEMPT                                                                          0x0263
+#define regVPEC_QUEUE4_PREEMPT_BASE_IDX                                                                 0
+#define regVPEC_QUEUE4_LOG0BUFFER_CFG                                                                   0x0264
+#define regVPEC_QUEUE4_LOG0BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE4_LOG1BUFFER_CFG                                                                   0x0265
+#define regVPEC_QUEUE4_LOG1BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE5_RB_CNTL                                                                          0x0278
+#define regVPEC_QUEUE5_RB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE5_SCHEDULE_CNTL                                                                    0x0279
+#define regVPEC_QUEUE5_SCHEDULE_CNTL_BASE_IDX                                                           0
+#define regVPEC_QUEUE5_RB_BASE                                                                          0x027a
+#define regVPEC_QUEUE5_RB_BASE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE5_RB_BASE_HI                                                                       0x027b
+#define regVPEC_QUEUE5_RB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE5_RB_RPTR                                                                          0x027c
+#define regVPEC_QUEUE5_RB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE5_RB_RPTR_HI                                                                       0x027d
+#define regVPEC_QUEUE5_RB_RPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE5_RB_WPTR                                                                          0x027e
+#define regVPEC_QUEUE5_RB_WPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE5_RB_WPTR_HI                                                                       0x027f
+#define regVPEC_QUEUE5_RB_WPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE5_RB_RPTR_ADDR_HI                                                                  0x0280
+#define regVPEC_QUEUE5_RB_RPTR_ADDR_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE5_RB_RPTR_ADDR_LO                                                                  0x0281
+#define regVPEC_QUEUE5_RB_RPTR_ADDR_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE5_RB_AQL_CNTL                                                                      0x0282
+#define regVPEC_QUEUE5_RB_AQL_CNTL_BASE_IDX                                                             0
+#define regVPEC_QUEUE5_MINOR_PTR_UPDATE                                                                 0x0283
+#define regVPEC_QUEUE5_MINOR_PTR_UPDATE_BASE_IDX                                                        0
+#define regVPEC_QUEUE5_CD_INFO                                                                          0x0284
+#define regVPEC_QUEUE5_CD_INFO_BASE_IDX                                                                 0
+#define regVPEC_QUEUE5_RB_PREEMPT                                                                       0x0285
+#define regVPEC_QUEUE5_RB_PREEMPT_BASE_IDX                                                              0
+#define regVPEC_QUEUE5_SKIP_CNTL                                                                        0x0286
+#define regVPEC_QUEUE5_SKIP_CNTL_BASE_IDX                                                               0
+#define regVPEC_QUEUE5_DOORBELL                                                                         0x0287
+#define regVPEC_QUEUE5_DOORBELL_BASE_IDX                                                                0
+#define regVPEC_QUEUE5_DOORBELL_OFFSET                                                                  0x0288
+#define regVPEC_QUEUE5_DOORBELL_OFFSET_BASE_IDX                                                         0
+#define regVPEC_QUEUE5_DUMMY0                                                                           0x0289
+#define regVPEC_QUEUE5_DUMMY0_BASE_IDX                                                                  0
+#define regVPEC_QUEUE5_DUMMY1                                                                           0x028a
+#define regVPEC_QUEUE5_DUMMY1_BASE_IDX                                                                  0
+#define regVPEC_QUEUE5_DUMMY2                                                                           0x028b
+#define regVPEC_QUEUE5_DUMMY2_BASE_IDX                                                                  0
+#define regVPEC_QUEUE5_DUMMY3                                                                           0x028c
+#define regVPEC_QUEUE5_DUMMY3_BASE_IDX                                                                  0
+#define regVPEC_QUEUE5_DUMMY4                                                                           0x028d
+#define regVPEC_QUEUE5_DUMMY4_BASE_IDX                                                                  0
+#define regVPEC_QUEUE5_IB_CNTL                                                                          0x02a4
+#define regVPEC_QUEUE5_IB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE5_IB_RPTR                                                                          0x02a5
+#define regVPEC_QUEUE5_IB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE5_IB_OFFSET                                                                        0x02a6
+#define regVPEC_QUEUE5_IB_OFFSET_BASE_IDX                                                               0
+#define regVPEC_QUEUE5_IB_BASE_LO                                                                       0x02a7
+#define regVPEC_QUEUE5_IB_BASE_LO_BASE_IDX                                                              0
+#define regVPEC_QUEUE5_IB_BASE_HI                                                                       0x02a8
+#define regVPEC_QUEUE5_IB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE5_IB_SIZE                                                                          0x02a9
+#define regVPEC_QUEUE5_IB_SIZE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE5_CMDIB_CNTL                                                                       0x02aa
+#define regVPEC_QUEUE5_CMDIB_CNTL_BASE_IDX                                                              0
+#define regVPEC_QUEUE5_CMDIB_RPTR                                                                       0x02ab
+#define regVPEC_QUEUE5_CMDIB_RPTR_BASE_IDX                                                              0
+#define regVPEC_QUEUE5_CMDIB_OFFSET                                                                     0x02ac
+#define regVPEC_QUEUE5_CMDIB_OFFSET_BASE_IDX                                                            0
+#define regVPEC_QUEUE5_CMDIB_BASE_LO                                                                    0x02ad
+#define regVPEC_QUEUE5_CMDIB_BASE_LO_BASE_IDX                                                           0
+#define regVPEC_QUEUE5_CMDIB_BASE_HI                                                                    0x02ae
+#define regVPEC_QUEUE5_CMDIB_BASE_HI_BASE_IDX                                                           0
+#define regVPEC_QUEUE5_CMDIB_SIZE                                                                       0x02af
+#define regVPEC_QUEUE5_CMDIB_SIZE_BASE_IDX                                                              0
+#define regVPEC_QUEUE5_3DLUTIB_CNTL                                                                     0x02b0
+#define regVPEC_QUEUE5_3DLUTIB_CNTL_BASE_IDX                                                            0
+#define regVPEC_QUEUE5_3DLUTIB_RPTR                                                                     0x02b1
+#define regVPEC_QUEUE5_3DLUTIB_RPTR_BASE_IDX                                                            0
+#define regVPEC_QUEUE5_3DLUTIB_OFFSET                                                                   0x02b2
+#define regVPEC_QUEUE5_3DLUTIB_OFFSET_BASE_IDX                                                          0
+#define regVPEC_QUEUE5_3DLUTIB_BASE_LO                                                                  0x02b3
+#define regVPEC_QUEUE5_3DLUTIB_BASE_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE5_3DLUTIB_BASE_HI                                                                  0x02b4
+#define regVPEC_QUEUE5_3DLUTIB_BASE_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE5_3DLUTIB_SIZE                                                                     0x02b5
+#define regVPEC_QUEUE5_3DLUTIB_SIZE_BASE_IDX                                                            0
+#define regVPEC_QUEUE5_CSA_ADDR_LO                                                                      0x02b6
+#define regVPEC_QUEUE5_CSA_ADDR_LO_BASE_IDX                                                             0
+#define regVPEC_QUEUE5_CSA_ADDR_HI                                                                      0x02b7
+#define regVPEC_QUEUE5_CSA_ADDR_HI_BASE_IDX                                                             0
+#define regVPEC_QUEUE5_CONTEXT_STATUS                                                                   0x02b8
+#define regVPEC_QUEUE5_CONTEXT_STATUS_BASE_IDX                                                          0
+#define regVPEC_QUEUE5_DOORBELL_LOG                                                                     0x02b9
+#define regVPEC_QUEUE5_DOORBELL_LOG_BASE_IDX                                                            0
+#define regVPEC_QUEUE5_IB_SUB_REMAIN                                                                    0x02ba
+#define regVPEC_QUEUE5_IB_SUB_REMAIN_BASE_IDX                                                           0
+#define regVPEC_QUEUE5_PREEMPT                                                                          0x02bb
+#define regVPEC_QUEUE5_PREEMPT_BASE_IDX                                                                 0
+#define regVPEC_QUEUE5_LOG0BUFFER_CFG                                                                   0x02bc
+#define regVPEC_QUEUE5_LOG0BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE5_LOG1BUFFER_CFG                                                                   0x02bd
+#define regVPEC_QUEUE5_LOG1BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE6_RB_CNTL                                                                          0x02d0
+#define regVPEC_QUEUE6_RB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE6_SCHEDULE_CNTL                                                                    0x02d1
+#define regVPEC_QUEUE6_SCHEDULE_CNTL_BASE_IDX                                                           0
+#define regVPEC_QUEUE6_RB_BASE                                                                          0x02d2
+#define regVPEC_QUEUE6_RB_BASE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE6_RB_BASE_HI                                                                       0x02d3
+#define regVPEC_QUEUE6_RB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE6_RB_RPTR                                                                          0x02d4
+#define regVPEC_QUEUE6_RB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE6_RB_RPTR_HI                                                                       0x02d5
+#define regVPEC_QUEUE6_RB_RPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE6_RB_WPTR                                                                          0x02d6
+#define regVPEC_QUEUE6_RB_WPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE6_RB_WPTR_HI                                                                       0x02d7
+#define regVPEC_QUEUE6_RB_WPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE6_RB_RPTR_ADDR_HI                                                                  0x02d8
+#define regVPEC_QUEUE6_RB_RPTR_ADDR_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE6_RB_RPTR_ADDR_LO                                                                  0x02d9
+#define regVPEC_QUEUE6_RB_RPTR_ADDR_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE6_RB_AQL_CNTL                                                                      0x02da
+#define regVPEC_QUEUE6_RB_AQL_CNTL_BASE_IDX                                                             0
+#define regVPEC_QUEUE6_MINOR_PTR_UPDATE                                                                 0x02db
+#define regVPEC_QUEUE6_MINOR_PTR_UPDATE_BASE_IDX                                                        0
+#define regVPEC_QUEUE6_CD_INFO                                                                          0x02dc
+#define regVPEC_QUEUE6_CD_INFO_BASE_IDX                                                                 0
+#define regVPEC_QUEUE6_RB_PREEMPT                                                                       0x02dd
+#define regVPEC_QUEUE6_RB_PREEMPT_BASE_IDX                                                              0
+#define regVPEC_QUEUE6_SKIP_CNTL                                                                        0x02de
+#define regVPEC_QUEUE6_SKIP_CNTL_BASE_IDX                                                               0
+#define regVPEC_QUEUE6_DOORBELL                                                                         0x02df
+#define regVPEC_QUEUE6_DOORBELL_BASE_IDX                                                                0
+#define regVPEC_QUEUE6_DOORBELL_OFFSET                                                                  0x02e0
+#define regVPEC_QUEUE6_DOORBELL_OFFSET_BASE_IDX                                                         0
+#define regVPEC_QUEUE6_DUMMY0                                                                           0x02e1
+#define regVPEC_QUEUE6_DUMMY0_BASE_IDX                                                                  0
+#define regVPEC_QUEUE6_DUMMY1                                                                           0x02e2
+#define regVPEC_QUEUE6_DUMMY1_BASE_IDX                                                                  0
+#define regVPEC_QUEUE6_DUMMY2                                                                           0x02e3
+#define regVPEC_QUEUE6_DUMMY2_BASE_IDX                                                                  0
+#define regVPEC_QUEUE6_DUMMY3                                                                           0x02e4
+#define regVPEC_QUEUE6_DUMMY3_BASE_IDX                                                                  0
+#define regVPEC_QUEUE6_DUMMY4                                                                           0x02e5
+#define regVPEC_QUEUE6_DUMMY4_BASE_IDX                                                                  0
+#define regVPEC_QUEUE6_IB_CNTL                                                                          0x02fc
+#define regVPEC_QUEUE6_IB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE6_IB_RPTR                                                                          0x02fd
+#define regVPEC_QUEUE6_IB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE6_IB_OFFSET                                                                        0x02fe
+#define regVPEC_QUEUE6_IB_OFFSET_BASE_IDX                                                               0
+#define regVPEC_QUEUE6_IB_BASE_LO                                                                       0x02ff
+#define regVPEC_QUEUE6_IB_BASE_LO_BASE_IDX                                                              0
+#define regVPEC_QUEUE6_IB_BASE_HI                                                                       0x0300
+#define regVPEC_QUEUE6_IB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE6_IB_SIZE                                                                          0x0301
+#define regVPEC_QUEUE6_IB_SIZE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE6_CMDIB_CNTL                                                                       0x0302
+#define regVPEC_QUEUE6_CMDIB_CNTL_BASE_IDX                                                              0
+#define regVPEC_QUEUE6_CMDIB_RPTR                                                                       0x0303
+#define regVPEC_QUEUE6_CMDIB_RPTR_BASE_IDX                                                              0
+#define regVPEC_QUEUE6_CMDIB_OFFSET                                                                     0x0304
+#define regVPEC_QUEUE6_CMDIB_OFFSET_BASE_IDX                                                            0
+#define regVPEC_QUEUE6_CMDIB_BASE_LO                                                                    0x0305
+#define regVPEC_QUEUE6_CMDIB_BASE_LO_BASE_IDX                                                           0
+#define regVPEC_QUEUE6_CMDIB_BASE_HI                                                                    0x0306
+#define regVPEC_QUEUE6_CMDIB_BASE_HI_BASE_IDX                                                           0
+#define regVPEC_QUEUE6_CMDIB_SIZE                                                                       0x0307
+#define regVPEC_QUEUE6_CMDIB_SIZE_BASE_IDX                                                              0
+#define regVPEC_QUEUE6_3DLUTIB_CNTL                                                                     0x0308
+#define regVPEC_QUEUE6_3DLUTIB_CNTL_BASE_IDX                                                            0
+#define regVPEC_QUEUE6_3DLUTIB_RPTR                                                                     0x0309
+#define regVPEC_QUEUE6_3DLUTIB_RPTR_BASE_IDX                                                            0
+#define regVPEC_QUEUE6_3DLUTIB_OFFSET                                                                   0x030a
+#define regVPEC_QUEUE6_3DLUTIB_OFFSET_BASE_IDX                                                          0
+#define regVPEC_QUEUE6_3DLUTIB_BASE_LO                                                                  0x030b
+#define regVPEC_QUEUE6_3DLUTIB_BASE_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE6_3DLUTIB_BASE_HI                                                                  0x030c
+#define regVPEC_QUEUE6_3DLUTIB_BASE_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE6_3DLUTIB_SIZE                                                                     0x030d
+#define regVPEC_QUEUE6_3DLUTIB_SIZE_BASE_IDX                                                            0
+#define regVPEC_QUEUE6_CSA_ADDR_LO                                                                      0x030e
+#define regVPEC_QUEUE6_CSA_ADDR_LO_BASE_IDX                                                             0
+#define regVPEC_QUEUE6_CSA_ADDR_HI                                                                      0x030f
+#define regVPEC_QUEUE6_CSA_ADDR_HI_BASE_IDX                                                             0
+#define regVPEC_QUEUE6_CONTEXT_STATUS                                                                   0x0310
+#define regVPEC_QUEUE6_CONTEXT_STATUS_BASE_IDX                                                          0
+#define regVPEC_QUEUE6_DOORBELL_LOG                                                                     0x0311
+#define regVPEC_QUEUE6_DOORBELL_LOG_BASE_IDX                                                            0
+#define regVPEC_QUEUE6_IB_SUB_REMAIN                                                                    0x0312
+#define regVPEC_QUEUE6_IB_SUB_REMAIN_BASE_IDX                                                           0
+#define regVPEC_QUEUE6_PREEMPT                                                                          0x0313
+#define regVPEC_QUEUE6_PREEMPT_BASE_IDX                                                                 0
+#define regVPEC_QUEUE6_LOG0BUFFER_CFG                                                                   0x0314
+#define regVPEC_QUEUE6_LOG0BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE6_LOG1BUFFER_CFG                                                                   0x0315
+#define regVPEC_QUEUE6_LOG1BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE7_RB_CNTL                                                                          0x0328
+#define regVPEC_QUEUE7_RB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE7_SCHEDULE_CNTL                                                                    0x0329
+#define regVPEC_QUEUE7_SCHEDULE_CNTL_BASE_IDX                                                           0
+#define regVPEC_QUEUE7_RB_BASE                                                                          0x032a
+#define regVPEC_QUEUE7_RB_BASE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE7_RB_BASE_HI                                                                       0x032b
+#define regVPEC_QUEUE7_RB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE7_RB_RPTR                                                                          0x032c
+#define regVPEC_QUEUE7_RB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE7_RB_RPTR_HI                                                                       0x032d
+#define regVPEC_QUEUE7_RB_RPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE7_RB_WPTR                                                                          0x032e
+#define regVPEC_QUEUE7_RB_WPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE7_RB_WPTR_HI                                                                       0x032f
+#define regVPEC_QUEUE7_RB_WPTR_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE7_RB_RPTR_ADDR_HI                                                                  0x0330
+#define regVPEC_QUEUE7_RB_RPTR_ADDR_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE7_RB_RPTR_ADDR_LO                                                                  0x0331
+#define regVPEC_QUEUE7_RB_RPTR_ADDR_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE7_RB_AQL_CNTL                                                                      0x0332
+#define regVPEC_QUEUE7_RB_AQL_CNTL_BASE_IDX                                                             0
+#define regVPEC_QUEUE7_MINOR_PTR_UPDATE                                                                 0x0333
+#define regVPEC_QUEUE7_MINOR_PTR_UPDATE_BASE_IDX                                                        0
+#define regVPEC_QUEUE7_CD_INFO                                                                          0x0334
+#define regVPEC_QUEUE7_CD_INFO_BASE_IDX                                                                 0
+#define regVPEC_QUEUE7_RB_PREEMPT                                                                       0x0335
+#define regVPEC_QUEUE7_RB_PREEMPT_BASE_IDX                                                              0
+#define regVPEC_QUEUE7_SKIP_CNTL                                                                        0x0336
+#define regVPEC_QUEUE7_SKIP_CNTL_BASE_IDX                                                               0
+#define regVPEC_QUEUE7_DOORBELL                                                                         0x0337
+#define regVPEC_QUEUE7_DOORBELL_BASE_IDX                                                                0
+#define regVPEC_QUEUE7_DOORBELL_OFFSET                                                                  0x0338
+#define regVPEC_QUEUE7_DOORBELL_OFFSET_BASE_IDX                                                         0
+#define regVPEC_QUEUE7_DUMMY0                                                                           0x0339
+#define regVPEC_QUEUE7_DUMMY0_BASE_IDX                                                                  0
+#define regVPEC_QUEUE7_DUMMY1                                                                           0x033a
+#define regVPEC_QUEUE7_DUMMY1_BASE_IDX                                                                  0
+#define regVPEC_QUEUE7_DUMMY2                                                                           0x033b
+#define regVPEC_QUEUE7_DUMMY2_BASE_IDX                                                                  0
+#define regVPEC_QUEUE7_DUMMY3                                                                           0x033c
+#define regVPEC_QUEUE7_DUMMY3_BASE_IDX                                                                  0
+#define regVPEC_QUEUE7_DUMMY4                                                                           0x033d
+#define regVPEC_QUEUE7_DUMMY4_BASE_IDX                                                                  0
+#define regVPEC_QUEUE7_IB_CNTL                                                                          0x0354
+#define regVPEC_QUEUE7_IB_CNTL_BASE_IDX                                                                 0
+#define regVPEC_QUEUE7_IB_RPTR                                                                          0x0355
+#define regVPEC_QUEUE7_IB_RPTR_BASE_IDX                                                                 0
+#define regVPEC_QUEUE7_IB_OFFSET                                                                        0x0356
+#define regVPEC_QUEUE7_IB_OFFSET_BASE_IDX                                                               0
+#define regVPEC_QUEUE7_IB_BASE_LO                                                                       0x0357
+#define regVPEC_QUEUE7_IB_BASE_LO_BASE_IDX                                                              0
+#define regVPEC_QUEUE7_IB_BASE_HI                                                                       0x0358
+#define regVPEC_QUEUE7_IB_BASE_HI_BASE_IDX                                                              0
+#define regVPEC_QUEUE7_IB_SIZE                                                                          0x0359
+#define regVPEC_QUEUE7_IB_SIZE_BASE_IDX                                                                 0
+#define regVPEC_QUEUE7_CMDIB_CNTL                                                                       0x035a
+#define regVPEC_QUEUE7_CMDIB_CNTL_BASE_IDX                                                              0
+#define regVPEC_QUEUE7_CMDIB_RPTR                                                                       0x035b
+#define regVPEC_QUEUE7_CMDIB_RPTR_BASE_IDX                                                              0
+#define regVPEC_QUEUE7_CMDIB_OFFSET                                                                     0x035c
+#define regVPEC_QUEUE7_CMDIB_OFFSET_BASE_IDX                                                            0
+#define regVPEC_QUEUE7_CMDIB_BASE_LO                                                                    0x035d
+#define regVPEC_QUEUE7_CMDIB_BASE_LO_BASE_IDX                                                           0
+#define regVPEC_QUEUE7_CMDIB_BASE_HI                                                                    0x035e
+#define regVPEC_QUEUE7_CMDIB_BASE_HI_BASE_IDX                                                           0
+#define regVPEC_QUEUE7_CMDIB_SIZE                                                                       0x035f
+#define regVPEC_QUEUE7_CMDIB_SIZE_BASE_IDX                                                              0
+#define regVPEC_QUEUE7_3DLUTIB_CNTL                                                                     0x0360
+#define regVPEC_QUEUE7_3DLUTIB_CNTL_BASE_IDX                                                            0
+#define regVPEC_QUEUE7_3DLUTIB_RPTR                                                                     0x0361
+#define regVPEC_QUEUE7_3DLUTIB_RPTR_BASE_IDX                                                            0
+#define regVPEC_QUEUE7_3DLUTIB_OFFSET                                                                   0x0362
+#define regVPEC_QUEUE7_3DLUTIB_OFFSET_BASE_IDX                                                          0
+#define regVPEC_QUEUE7_3DLUTIB_BASE_LO                                                                  0x0363
+#define regVPEC_QUEUE7_3DLUTIB_BASE_LO_BASE_IDX                                                         0
+#define regVPEC_QUEUE7_3DLUTIB_BASE_HI                                                                  0x0364
+#define regVPEC_QUEUE7_3DLUTIB_BASE_HI_BASE_IDX                                                         0
+#define regVPEC_QUEUE7_3DLUTIB_SIZE                                                                     0x0365
+#define regVPEC_QUEUE7_3DLUTIB_SIZE_BASE_IDX                                                            0
+#define regVPEC_QUEUE7_CSA_ADDR_LO                                                                      0x0366
+#define regVPEC_QUEUE7_CSA_ADDR_LO_BASE_IDX                                                             0
+#define regVPEC_QUEUE7_CSA_ADDR_HI                                                                      0x0367
+#define regVPEC_QUEUE7_CSA_ADDR_HI_BASE_IDX                                                             0
+#define regVPEC_QUEUE7_CONTEXT_STATUS                                                                   0x0368
+#define regVPEC_QUEUE7_CONTEXT_STATUS_BASE_IDX                                                          0
+#define regVPEC_QUEUE7_DOORBELL_LOG                                                                     0x0369
+#define regVPEC_QUEUE7_DOORBELL_LOG_BASE_IDX                                                            0
+#define regVPEC_QUEUE7_IB_SUB_REMAIN                                                                    0x036a
+#define regVPEC_QUEUE7_IB_SUB_REMAIN_BASE_IDX                                                           0
+#define regVPEC_QUEUE7_PREEMPT                                                                          0x036b
+#define regVPEC_QUEUE7_PREEMPT_BASE_IDX                                                                 0
+#define regVPEC_QUEUE7_LOG0BUFFER_CFG                                                                   0x036c
+#define regVPEC_QUEUE7_LOG0BUFFER_CFG_BASE_IDX                                                          0
+#define regVPEC_QUEUE7_LOG1BUFFER_CFG                                                                   0x036d
+#define regVPEC_QUEUE7_LOG1BUFFER_CFG_BASE_IDX                                                          0
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/asic_reg/vpe/vpe_2_0_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/vpe/vpe_2_0_0_sh_mask.h
new file mode 100644
index 0000000000000..7e0e56d14e671
--- /dev/null
+++ b/drivers/gpu/drm/amd/include/asic_reg/vpe/vpe_2_0_0_sh_mask.h
@@ -0,0 +1,3162 @@
+/*
+ * Copyright 2025 Advanced Micro Devices, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _vpe_2_0_0_SH_MASK_HEADER
+#define _vpe_2_0_0_SH_MASK_HEADER
+
+
+// addressBlock: vpe_vpec_vpecdec
+//VPEC_DEC_START
+#define VPEC_DEC_START__START__SHIFT                                                                          0x0
+#define VPEC_DEC_START__START_MASK                                                                            0xFFFFFFFFL
+//VPEC_UCODE_ADDR
+#define VPEC_UCODE_ADDR__VALUE__SHIFT                                                                         0x0
+#define VPEC_UCODE_ADDR__THID__SHIFT                                                                          0xf
+#define VPEC_UCODE_ADDR__VALUE_MASK                                                                           0x00001FFFL
+#define VPEC_UCODE_ADDR__THID_MASK                                                                            0x00008000L
+//VPEC_UCODE_DATA
+#define VPEC_UCODE_DATA__VALUE__SHIFT                                                                         0x0
+#define VPEC_UCODE_DATA__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_F32_CNTL
+#define VPEC_F32_CNTL__HALT__SHIFT                                                                            0x0
+#define VPEC_F32_CNTL__DBG_SELECT_BITS__SHIFT                                                                 0x2
+#define VPEC_F32_CNTL__TH0_CHECKSUM_CLR__SHIFT                                                                0x8
+#define VPEC_F32_CNTL__TH0_RESET__SHIFT                                                                       0x9
+#define VPEC_F32_CNTL__TH0_ENABLE__SHIFT                                                                      0xa
+#define VPEC_F32_CNTL__TH1_CHECKSUM_CLR__SHIFT                                                                0xc
+#define VPEC_F32_CNTL__TH1_RESET__SHIFT                                                                       0xd
+#define VPEC_F32_CNTL__TH1_ENABLE__SHIFT                                                                      0xe
+#define VPEC_F32_CNTL__TH0_PRIORITY__SHIFT                                                                    0x10
+#define VPEC_F32_CNTL__TH1_PRIORITY__SHIFT                                                                    0x18
+#define VPEC_F32_CNTL__HALT_MASK                                                                              0x00000001L
+#define VPEC_F32_CNTL__DBG_SELECT_BITS_MASK                                                                   0x000000FCL
+#define VPEC_F32_CNTL__TH0_CHECKSUM_CLR_MASK                                                                  0x00000100L
+#define VPEC_F32_CNTL__TH0_RESET_MASK                                                                         0x00000200L
+#define VPEC_F32_CNTL__TH0_ENABLE_MASK                                                                        0x00000400L
+#define VPEC_F32_CNTL__TH1_CHECKSUM_CLR_MASK                                                                  0x00001000L
+#define VPEC_F32_CNTL__TH1_RESET_MASK                                                                         0x00002000L
+#define VPEC_F32_CNTL__TH1_ENABLE_MASK                                                                        0x00004000L
+#define VPEC_F32_CNTL__TH0_PRIORITY_MASK                                                                      0x00FF0000L
+#define VPEC_F32_CNTL__TH1_PRIORITY_MASK                                                                      0xFF000000L
+//VPEC_MMHUB_CNTL
+#define VPEC_MMHUB_CNTL__UNIT_ID__SHIFT                                                                       0x0
+#define VPEC_MMHUB_CNTL__UNIT_ID_MASK                                                                         0x0000003FL
+//VPEC_MMHUB_TRUSTLVL
+#define VPEC_MMHUB_TRUSTLVL__SECLVL0__SHIFT                                                                   0x0
+#define VPEC_MMHUB_TRUSTLVL__SECLVL1__SHIFT                                                                   0x4
+#define VPEC_MMHUB_TRUSTLVL__SECLVL2__SHIFT                                                                   0x8
+#define VPEC_MMHUB_TRUSTLVL__SECLVL3__SHIFT                                                                   0xc
+#define VPEC_MMHUB_TRUSTLVL__SECLVL4__SHIFT                                                                   0x10
+#define VPEC_MMHUB_TRUSTLVL__SECLVL5__SHIFT                                                                   0x14
+#define VPEC_MMHUB_TRUSTLVL__SECLVL6__SHIFT                                                                   0x18
+#define VPEC_MMHUB_TRUSTLVL__SECLVL7__SHIFT                                                                   0x1c
+#define VPEC_MMHUB_TRUSTLVL__SECLVL0_MASK                                                                     0x0000000FL
+#define VPEC_MMHUB_TRUSTLVL__SECLVL1_MASK                                                                     0x000000F0L
+#define VPEC_MMHUB_TRUSTLVL__SECLVL2_MASK                                                                     0x00000F00L
+#define VPEC_MMHUB_TRUSTLVL__SECLVL3_MASK                                                                     0x0000F000L
+#define VPEC_MMHUB_TRUSTLVL__SECLVL4_MASK                                                                     0x000F0000L
+#define VPEC_MMHUB_TRUSTLVL__SECLVL5_MASK                                                                     0x00F00000L
+#define VPEC_MMHUB_TRUSTLVL__SECLVL6_MASK                                                                     0x0F000000L
+#define VPEC_MMHUB_TRUSTLVL__SECLVL7_MASK                                                                     0xF0000000L
+//VPEC_VPEP_CTRL
+#define VPEC_VPEP_CTRL__VPEP_SOCCLK_EN__SHIFT                                                                 0x0
+#define VPEC_VPEP_CTRL__VPEP_SW_RESETB__SHIFT                                                                 0x1
+#define VPEC_VPEP_CTRL__RESERVED__SHIFT                                                                       0x2
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S0P0__SHIFT                                                     0x16
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S0P1__SHIFT                                                     0x17
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S0P2__SHIFT                                                     0x18
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S1P0__SHIFT                                                     0x19
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S1P1__SHIFT                                                     0x1a
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S1P2__SHIFT                                                     0x1b
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_3DLUT__SHIFT                                                    0x1c
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_VPEC_VPEP_REG_FGCLKEN__SHIFT                                            0x1d
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_VPEP_SOCCLK__SHIFT                                                      0x1e
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_VPECLK__SHIFT                                                           0x1f
+#define VPEC_VPEP_CTRL__VPEP_SOCCLK_EN_MASK                                                                   0x00000001L
+#define VPEC_VPEP_CTRL__VPEP_SW_RESETB_MASK                                                                   0x00000002L
+#define VPEC_VPEP_CTRL__RESERVED_MASK                                                                         0x003FFFFCL
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S0P0_MASK                                                       0x00400000L
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S0P1_MASK                                                       0x00800000L
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S0P2_MASK                                                       0x01000000L
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S1P0_MASK                                                       0x02000000L
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S1P1_MASK                                                       0x04000000L
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_S1P2_MASK                                                       0x08000000L
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_FGCLKEN_3DLUT_MASK                                                      0x10000000L
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_VPEC_VPEP_REG_FGCLKEN_MASK                                              0x20000000L
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_VPEP_SOCCLK_MASK                                                        0x40000000L
+#define VPEC_VPEP_CTRL__SOFT_OVERRIDE_VPECLK_MASK                                                             0x80000000L
+//VPEC_CLK_CTRL
+#define VPEC_CLK_CTRL__VPECLK_EN__SHIFT                                                                       0x1
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE0_CLK__SHIFT                                                      0x8
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE1_CLK__SHIFT                                                      0x9
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE2_CLK__SHIFT                                                      0xa
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE3_CLK__SHIFT                                                      0xb
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE4_CLK__SHIFT                                                      0xc
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE5_CLK__SHIFT                                                      0xd
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE0_CLK__SHIFT                                                      0x10
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE1_CLK__SHIFT                                                      0x11
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE2_CLK__SHIFT                                                      0x12
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE3_CLK__SHIFT                                                      0x13
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE4_CLK__SHIFT                                                      0x14
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE5_CLK__SHIFT                                                      0x15
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE6_CLK__SHIFT                                                      0x16
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE7_CLK__SHIFT                                                      0x17
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE8_CLK__SHIFT                                                      0x18
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE9_CLK__SHIFT                                                      0x19
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_PERF_CLK__SHIFT                                                          0x1b
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_CE_CLK__SHIFT                                                            0x1c
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_F32_CLK__SHIFT                                                           0x1d
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_DYN_CLK__SHIFT                                                           0x1e
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_REG_CLK__SHIFT                                                           0x1f
+#define VPEC_CLK_CTRL__VPECLK_EN_MASK                                                                         0x00000002L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE0_CLK_MASK                                                        0x00000100L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE1_CLK_MASK                                                        0x00000200L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE2_CLK_MASK                                                        0x00000400L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE3_CLK_MASK                                                        0x00000800L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE4_CLK_MASK                                                        0x00001000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_IP_PIPE5_CLK_MASK                                                        0x00002000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE0_CLK_MASK                                                        0x00010000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE1_CLK_MASK                                                        0x00020000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE2_CLK_MASK                                                        0x00040000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE3_CLK_MASK                                                        0x00080000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE4_CLK_MASK                                                        0x00100000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE5_CLK_MASK                                                        0x00200000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE6_CLK_MASK                                                        0x00400000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE7_CLK_MASK                                                        0x00800000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE8_CLK_MASK                                                        0x01000000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_OP_PIPE9_CLK_MASK                                                        0x02000000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_PERF_CLK_MASK                                                            0x08000000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_CE_CLK_MASK                                                              0x10000000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_F32_CLK_MASK                                                             0x20000000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_DYN_CLK_MASK                                                             0x40000000L
+#define VPEC_CLK_CTRL__SOFT_OVERRIDE_REG_CLK_MASK                                                             0x80000000L
+//VPEC_COLLABORATE_CNTL
+#define VPEC_COLLABORATE_CNTL__COLLABORATE_MODE_EN__SHIFT                                                     0x0
+#define VPEC_COLLABORATE_CNTL__COLLABORATE_MODE_EN_MASK                                                       0x00000001L
+//VPEC_COLLABORATE_CFG
+#define VPEC_COLLABORATE_CFG__MASTER_ID__SHIFT                                                                0x0
+#define VPEC_COLLABORATE_CFG__MASTER_EN__SHIFT                                                                0x3
+#define VPEC_COLLABORATE_CFG__SLAVE0_ID__SHIFT                                                                0x4
+#define VPEC_COLLABORATE_CFG__SLAVE0_EN__SHIFT                                                                0x7
+#define VPEC_COLLABORATE_CFG__MASTER_ID_MASK                                                                  0x00000007L
+#define VPEC_COLLABORATE_CFG__MASTER_EN_MASK                                                                  0x00000008L
+#define VPEC_COLLABORATE_CFG__SLAVE0_ID_MASK                                                                  0x00000070L
+#define VPEC_COLLABORATE_CFG__SLAVE0_EN_MASK                                                                  0x00000080L
+//VPEC_POWER_CNTL
+#define VPEC_POWER_CNTL__LS_ENABLE__SHIFT                                                                     0x0
+#define VPEC_POWER_CNTL__UCODE_SRAM_DS_EN__SHIFT                                                              0x1
+#define VPEC_POWER_CNTL__FISO__SHIFT                                                                          0x2
+#define VPEC_POWER_CNTL__UCODE_SRAM_POWER_UP_RECOVER_DELAY__SHIFT                                             0x8
+#define VPEC_POWER_CNTL__UCODE_SRAM_POWER_STATUS_CHANGE_WAKEUP_TIME__SHIFT                                    0xf
+#define VPEC_POWER_CNTL__UCODE_SRAM_POWER_STATUS_CHANGE_CLK_FORCE__SHIFT                                      0x12
+#define VPEC_POWER_CNTL__SRAM_POWER_LS_CHANGE_DELAY__SHIFT                                                    0x14
+#define VPEC_POWER_CNTL__SRAM_POWER_LS_CHANGE_CLK_FORCE__SHIFT                                                0x17
+#define VPEC_POWER_CNTL__SRAM_POWER_LS_CHANGE_CLK_FORCE_DELAY__SHIFT                                          0x18
+#define VPEC_POWER_CNTL__LS_ENABLE_MASK                                                                       0x00000001L
+#define VPEC_POWER_CNTL__UCODE_SRAM_DS_EN_MASK                                                                0x00000002L
+#define VPEC_POWER_CNTL__FISO_MASK                                                                            0x00000004L
+#define VPEC_POWER_CNTL__UCODE_SRAM_POWER_UP_RECOVER_DELAY_MASK                                               0x00007F00L
+#define VPEC_POWER_CNTL__UCODE_SRAM_POWER_STATUS_CHANGE_WAKEUP_TIME_MASK                                      0x00038000L
+#define VPEC_POWER_CNTL__UCODE_SRAM_POWER_STATUS_CHANGE_CLK_FORCE_MASK                                        0x00040000L
+#define VPEC_POWER_CNTL__SRAM_POWER_LS_CHANGE_DELAY_MASK                                                      0x00700000L
+#define VPEC_POWER_CNTL__SRAM_POWER_LS_CHANGE_CLK_FORCE_MASK                                                  0x00800000L
+#define VPEC_POWER_CNTL__SRAM_POWER_LS_CHANGE_CLK_FORCE_DELAY_MASK                                            0x03000000L
+//VPEC_ZPR_CNTL
+#define VPEC_ZPR_CNTL__CLK_UNGATE_DELAY__SHIFT                                                                0x0
+#define VPEC_ZPR_CNTL__RESERVED__SHIFT                                                                        0x8
+#define VPEC_ZPR_CNTL__CLK_UNGATE_DELAY_MASK                                                                  0x000000FFL
+#define VPEC_ZPR_CNTL__RESERVED_MASK                                                                          0xFFFFFF00L
+//VPEC_CNTL
+#define VPEC_CNTL__TRAP_ENABLE__SHIFT                                                                         0x0
+#define VPEC_CNTL__RESERVED_2_2__SHIFT                                                                        0x2
+#define VPEC_CNTL__DATA_SWAP__SHIFT                                                                           0x3
+#define VPEC_CNTL__FENCE_SWAP_ENABLE__SHIFT                                                                   0x5
+#define VPEC_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                               0x6
+#define VPEC_CNTL__MIDCMD_EXPIRE_ENABLE__SHIFT                                                                0x9
+#define VPEC_CNTL__UMSCH_INT_ENABLE__SHIFT                                                                    0xa
+#define VPEC_CNTL__RESERVED_13_11__SHIFT                                                                      0xb
+#define VPEC_CNTL__NACK_GEN_ERR_INT_ENABLE__SHIFT                                                             0xe
+#define VPEC_CNTL__NACK_PRT_INT_ENABLE__SHIFT                                                                 0xf
+#define VPEC_CNTL__RESERVED_16_16__SHIFT                                                                      0x10
+#define VPEC_CNTL__MIDCMD_WORLDSWITCH_ENABLE__SHIFT                                                           0x11
+#define VPEC_CNTL__RESERVED_19_19__SHIFT                                                                      0x13
+#define VPEC_CNTL__CTXEMPTY_INT_ENABLE__SHIFT                                                                 0x1c
+#define VPEC_CNTL__FROZEN_INT_ENABLE__SHIFT                                                                   0x1d
+#define VPEC_CNTL__IB_PREEMPT_INT_ENABLE__SHIFT                                                               0x1e
+#define VPEC_CNTL__RB_PREEMPT_INT_ENABLE__SHIFT                                                               0x1f
+#define VPEC_CNTL__TRAP_ENABLE_MASK                                                                           0x00000001L
+#define VPEC_CNTL__RESERVED_2_2_MASK                                                                          0x00000004L
+#define VPEC_CNTL__DATA_SWAP_MASK                                                                             0x00000018L
+#define VPEC_CNTL__FENCE_SWAP_ENABLE_MASK                                                                     0x00000020L
+#define VPEC_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                                 0x00000040L
+#define VPEC_CNTL__MIDCMD_EXPIRE_ENABLE_MASK                                                                  0x00000200L
+#define VPEC_CNTL__UMSCH_INT_ENABLE_MASK                                                                      0x00000400L
+#define VPEC_CNTL__RESERVED_13_11_MASK                                                                        0x00003800L
+#define VPEC_CNTL__NACK_GEN_ERR_INT_ENABLE_MASK                                                               0x00004000L
+#define VPEC_CNTL__NACK_PRT_INT_ENABLE_MASK                                                                   0x00008000L
+#define VPEC_CNTL__RESERVED_16_16_MASK                                                                        0x00010000L
+#define VPEC_CNTL__MIDCMD_WORLDSWITCH_ENABLE_MASK                                                             0x00020000L
+#define VPEC_CNTL__RESERVED_19_19_MASK                                                                        0x00080000L
+#define VPEC_CNTL__CTXEMPTY_INT_ENABLE_MASK                                                                   0x10000000L
+#define VPEC_CNTL__FROZEN_INT_ENABLE_MASK                                                                     0x20000000L
+#define VPEC_CNTL__IB_PREEMPT_INT_ENABLE_MASK                                                                 0x40000000L
+#define VPEC_CNTL__RB_PREEMPT_INT_ENABLE_MASK                                                                 0x80000000L
+//VPEC_CNTL_DCC
+#define VPEC_CNTL_DCC__WDCC_COMP_MODE__SHIFT                                                                  0x0
+#define VPEC_CNTL_DCC__RESERVED_3_2__SHIFT                                                                    0x2
+#define VPEC_CNTL_DCC__WDCC_MICRO_TILE_MODE__SHIFT                                                            0x4
+#define VPEC_CNTL_DCC__RESERVED_7_6__SHIFT                                                                    0x6
+#define VPEC_CNTL_DCC__WDCC_DATA_FORMAT__SHIFT                                                                0x8
+#define VPEC_CNTL_DCC__RESERVED_15_13__SHIFT                                                                  0xd
+#define VPEC_CNTL_DCC__WDCC_NUM_FORMAT_EN__SHIFT                                                              0x10
+#define VPEC_CNTL_DCC__RESERVED_19_17__SHIFT                                                                  0x11
+#define VPEC_CNTL_DCC__WDCC_NUM_TYPE__SHIFT                                                                   0x14
+#define VPEC_CNTL_DCC__RESERVED_23_23__SHIFT                                                                  0x17
+#define VPEC_CNTL_DCC__WDCC_MAX_UNCOMP_SIZE__SHIFT                                                            0x18
+#define VPEC_CNTL_DCC__WDCC_MAX_COMP_SIZE__SHIFT                                                              0x19
+#define VPEC_CNTL_DCC__RESERVED_30_27__SHIFT                                                                  0x1b
+#define VPEC_CNTL_DCC__RDCC_COMP_MODE__SHIFT                                                                  0x1f
+#define VPEC_CNTL_DCC__WDCC_COMP_MODE_MASK                                                                    0x00000003L
+#define VPEC_CNTL_DCC__RESERVED_3_2_MASK                                                                      0x0000000CL
+#define VPEC_CNTL_DCC__WDCC_MICRO_TILE_MODE_MASK                                                              0x00000030L
+#define VPEC_CNTL_DCC__RESERVED_7_6_MASK                                                                      0x000000C0L
+#define VPEC_CNTL_DCC__WDCC_DATA_FORMAT_MASK                                                                  0x00001F00L
+#define VPEC_CNTL_DCC__RESERVED_15_13_MASK                                                                    0x0000E000L
+#define VPEC_CNTL_DCC__WDCC_NUM_FORMAT_EN_MASK                                                                0x00010000L
+#define VPEC_CNTL_DCC__RESERVED_19_17_MASK                                                                    0x000E0000L
+#define VPEC_CNTL_DCC__WDCC_NUM_TYPE_MASK                                                                     0x00700000L
+#define VPEC_CNTL_DCC__RESERVED_23_23_MASK                                                                    0x00800000L
+#define VPEC_CNTL_DCC__WDCC_MAX_UNCOMP_SIZE_MASK                                                              0x01000000L
+#define VPEC_CNTL_DCC__WDCC_MAX_COMP_SIZE_MASK                                                                0x06000000L
+#define VPEC_CNTL_DCC__RESERVED_30_27_MASK                                                                    0x78000000L
+#define VPEC_CNTL_DCC__RDCC_COMP_MODE_MASK                                                                    0x80000000L
+//VPEC_CE_OP_MULTI_64B_BURST
+#define VPEC_CE_OP_MULTI_64B_BURST__EN__SHIFT                                                                 0x0
+#define VPEC_CE_OP_MULTI_64B_BURST__RESERVED_3_1__SHIFT                                                       0x1
+#define VPEC_CE_OP_MULTI_64B_BURST__LAZY_TIMER_DLY__SHIFT                                                     0x4
+#define VPEC_CE_OP_MULTI_64B_BURST__NUM_64B_BURST_ALLOWED__SHIFT                                              0xa
+#define VPEC_CE_OP_MULTI_64B_BURST__RESERVED_31_12__SHIFT                                                     0xc
+#define VPEC_CE_OP_MULTI_64B_BURST__EN_MASK                                                                   0x00000001L
+#define VPEC_CE_OP_MULTI_64B_BURST__RESERVED_3_1_MASK                                                         0x0000000EL
+#define VPEC_CE_OP_MULTI_64B_BURST__LAZY_TIMER_DLY_MASK                                                       0x000003F0L
+#define VPEC_CE_OP_MULTI_64B_BURST__NUM_64B_BURST_ALLOWED_MASK                                                0x00000C00L
+#define VPEC_CE_OP_MULTI_64B_BURST__RESERVED_31_12_MASK                                                       0xFFFFF000L
+//VPEC_CNTL1
+#define VPEC_CNTL1__RESERVED_3_1__SHIFT                                                                       0x1
+#define VPEC_CNTL1__SRBM_POLL_RETRYING__SHIFT                                                                 0x5
+#define VPEC_CNTL1__RESERVED_23_10__SHIFT                                                                     0xa
+#define VPEC_CNTL1__CG_STATUS_OUTPUT__SHIFT                                                                   0x18
+#define VPEC_CNTL1__SW_FREEZE_ENABLE__SHIFT                                                                   0x19
+#define VPEC_CNTL1__VPEP_CONFIG_INVALID_CHECK_ENABLE__SHIFT                                                   0x1a
+#define VPEC_CNTL1__RSMU_ACCESS_OFF_VPEP_RETURN_ERROR_ENABLE__SHIFT                                           0x1b
+#define VPEC_CNTL1__RSMU_ACCESS_OFF_VPEP_REPORT_ERROR_ENABLE__SHIFT                                           0x1c
+#define VPEC_CNTL1__RESERVED__SHIFT                                                                           0x1d
+#define VPEC_CNTL1__RESERVED_3_1_MASK                                                                         0x0000000EL
+#define VPEC_CNTL1__SRBM_POLL_RETRYING_MASK                                                                   0x00000020L
+#define VPEC_CNTL1__RESERVED_23_10_MASK                                                                       0x00FFFC00L
+#define VPEC_CNTL1__CG_STATUS_OUTPUT_MASK                                                                     0x01000000L
+#define VPEC_CNTL1__SW_FREEZE_ENABLE_MASK                                                                     0x02000000L
+#define VPEC_CNTL1__VPEP_CONFIG_INVALID_CHECK_ENABLE_MASK                                                     0x04000000L
+#define VPEC_CNTL1__RSMU_ACCESS_OFF_VPEP_RETURN_ERROR_ENABLE_MASK                                             0x08000000L
+#define VPEC_CNTL1__RSMU_ACCESS_OFF_VPEP_REPORT_ERROR_ENABLE_MASK                                             0x10000000L
+#define VPEC_CNTL1__RESERVED_MASK                                                                             0xE0000000L
+//VPEC_CNTL2
+#define VPEC_CNTL2__F32_CMD_PROC_DELAY__SHIFT                                                                 0x0
+#define VPEC_CNTL2__F32_SEND_POSTCODE_EN__SHIFT                                                               0x4
+#define VPEC_CNTL2__UCODE_BUF_DS_EN__SHIFT                                                                    0x6
+#define VPEC_CNTL2__UCODE_SELFLOAD_THREAD_OVERLAP__SHIFT                                                      0x7
+#define VPEC_CNTL2__LUTIB_FIFO_WATERMARK__SHIFT                                                               0x8
+#define VPEC_CNTL2__CMDIB_FIFO_WATERMARK__SHIFT                                                               0xa
+#define VPEC_CNTL2__RESERVED_14_12__SHIFT                                                                     0xc
+#define VPEC_CNTL2__IMPROVE_CE_IP_ARBITER__SHIFT                                                              0xf
+#define VPEC_CNTL2__RB_FIFO_WATERMARK__SHIFT                                                                  0x10
+#define VPEC_CNTL2__IB_FIFO_WATERMARK__SHIFT                                                                  0x12
+#define VPEC_CNTL2__RESERVED_22_20__SHIFT                                                                     0x14
+#define VPEC_CNTL2__CH_RD_WATERMARK__SHIFT                                                                    0x17
+#define VPEC_CNTL2__CH_WR_WATERMARK__SHIFT                                                                    0x19
+#define VPEC_CNTL2__CH_WR_WATERMARK_LSB__SHIFT                                                                0x1e
+#define VPEC_CNTL2__F32_CMD_PROC_DELAY_MASK                                                                   0x0000000FL
+#define VPEC_CNTL2__F32_SEND_POSTCODE_EN_MASK                                                                 0x00000010L
+#define VPEC_CNTL2__UCODE_BUF_DS_EN_MASK                                                                      0x00000040L
+#define VPEC_CNTL2__UCODE_SELFLOAD_THREAD_OVERLAP_MASK                                                        0x00000080L
+#define VPEC_CNTL2__LUTIB_FIFO_WATERMARK_MASK                                                                 0x00000300L
+#define VPEC_CNTL2__CMDIB_FIFO_WATERMARK_MASK                                                                 0x00000C00L
+#define VPEC_CNTL2__RESERVED_14_12_MASK                                                                       0x00007000L
+#define VPEC_CNTL2__IMPROVE_CE_IP_ARBITER_MASK                                                                0x00008000L
+#define VPEC_CNTL2__RB_FIFO_WATERMARK_MASK                                                                    0x00030000L
+#define VPEC_CNTL2__IB_FIFO_WATERMARK_MASK                                                                    0x000C0000L
+#define VPEC_CNTL2__RESERVED_22_20_MASK                                                                       0x00700000L
+#define VPEC_CNTL2__CH_RD_WATERMARK_MASK                                                                      0x01800000L
+#define VPEC_CNTL2__CH_WR_WATERMARK_MASK                                                                      0x3E000000L
+#define VPEC_CNTL2__CH_WR_WATERMARK_LSB_MASK                                                                  0x40000000L
+//VPEC_GB_ADDR_CONFIG
+#define VPEC_GB_ADDR_CONFIG__NUM_PIPES__SHIFT                                                                 0x0
+#define VPEC_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE__SHIFT                                                      0x3
+#define VPEC_GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS__SHIFT                                                      0x6
+#define VPEC_GB_ADDR_CONFIG__NUM_PKRS__SHIFT                                                                  0x8
+#define VPEC_GB_ADDR_CONFIG__NUM_SHADER_ENGINES__SHIFT                                                        0x13
+#define VPEC_GB_ADDR_CONFIG__NUM_RB_PER_SE__SHIFT                                                             0x1a
+#define VPEC_GB_ADDR_CONFIG__NUM_PIPES_MASK                                                                   0x00000007L
+#define VPEC_GB_ADDR_CONFIG__PIPE_INTERLEAVE_SIZE_MASK                                                        0x00000038L
+#define VPEC_GB_ADDR_CONFIG__MAX_COMPRESSED_FRAGS_MASK                                                        0x000000C0L
+#define VPEC_GB_ADDR_CONFIG__NUM_PKRS_MASK                                                                    0x00000700L
+#define VPEC_GB_ADDR_CONFIG__NUM_SHADER_ENGINES_MASK                                                          0x00180000L
+#define VPEC_GB_ADDR_CONFIG__NUM_RB_PER_SE_MASK                                                               0x0C000000L
+//VPEC_GB_ADDR_CONFIG_READ
+#define VPEC_GB_ADDR_CONFIG_READ__NUM_PIPES__SHIFT                                                            0x0
+#define VPEC_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE__SHIFT                                                 0x3
+#define VPEC_GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS__SHIFT                                                 0x6
+#define VPEC_GB_ADDR_CONFIG_READ__NUM_PKRS__SHIFT                                                             0x8
+#define VPEC_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES__SHIFT                                                   0x13
+#define VPEC_GB_ADDR_CONFIG_READ__NUM_RB_PER_SE__SHIFT                                                        0x1a
+#define VPEC_GB_ADDR_CONFIG_READ__NUM_PIPES_MASK                                                              0x00000007L
+#define VPEC_GB_ADDR_CONFIG_READ__PIPE_INTERLEAVE_SIZE_MASK                                                   0x00000038L
+#define VPEC_GB_ADDR_CONFIG_READ__MAX_COMPRESSED_FRAGS_MASK                                                   0x000000C0L
+#define VPEC_GB_ADDR_CONFIG_READ__NUM_PKRS_MASK                                                               0x00000700L
+#define VPEC_GB_ADDR_CONFIG_READ__NUM_SHADER_ENGINES_MASK                                                     0x00180000L
+#define VPEC_GB_ADDR_CONFIG_READ__NUM_RB_PER_SE_MASK                                                          0x0C000000L
+//VPEC_GB_ADDR_CONFIG_META
+#define VPEC_GB_ADDR_CONFIG_META__NUM_PIPES__SHIFT                                                            0x0
+#define VPEC_GB_ADDR_CONFIG_META__PIPE_INTERLEAVE_SIZE__SHIFT                                                 0x3
+#define VPEC_GB_ADDR_CONFIG_META__MAX_COMPRESSED_FRAGS__SHIFT                                                 0x6
+#define VPEC_GB_ADDR_CONFIG_META__NUM_PKRS__SHIFT                                                             0x8
+#define VPEC_GB_ADDR_CONFIG_META__NUM_SHADER_ENGINES__SHIFT                                                   0x13
+#define VPEC_GB_ADDR_CONFIG_META__NUM_RB_PER_SE__SHIFT                                                        0x1a
+#define VPEC_GB_ADDR_CONFIG_META__NUM_PIPES_MASK                                                              0x00000007L
+#define VPEC_GB_ADDR_CONFIG_META__PIPE_INTERLEAVE_SIZE_MASK                                                   0x00000038L
+#define VPEC_GB_ADDR_CONFIG_META__MAX_COMPRESSED_FRAGS_MASK                                                   0x000000C0L
+#define VPEC_GB_ADDR_CONFIG_META__NUM_PKRS_MASK                                                               0x00000700L
+#define VPEC_GB_ADDR_CONFIG_META__NUM_SHADER_ENGINES_MASK                                                     0x00180000L
+#define VPEC_GB_ADDR_CONFIG_META__NUM_RB_PER_SE_MASK                                                          0x0C000000L
+//VPEC_PROCESS_QUANTUM0
+#define VPEC_PROCESS_QUANTUM0__PROCESS0_QUANTUM__SHIFT                                                        0x0
+#define VPEC_PROCESS_QUANTUM0__PROCESS1_QUANTUM__SHIFT                                                        0x8
+#define VPEC_PROCESS_QUANTUM0__PROCESS2_QUANTUM__SHIFT                                                        0x10
+#define VPEC_PROCESS_QUANTUM0__PROCESS3_QUANTUM__SHIFT                                                        0x18
+#define VPEC_PROCESS_QUANTUM0__PROCESS0_QUANTUM_MASK                                                          0x000000FFL
+#define VPEC_PROCESS_QUANTUM0__PROCESS1_QUANTUM_MASK                                                          0x0000FF00L
+#define VPEC_PROCESS_QUANTUM0__PROCESS2_QUANTUM_MASK                                                          0x00FF0000L
+#define VPEC_PROCESS_QUANTUM0__PROCESS3_QUANTUM_MASK                                                          0xFF000000L
+//VPEC_PROCESS_QUANTUM1
+#define VPEC_PROCESS_QUANTUM1__PROCESS4_QUANTUM__SHIFT                                                        0x0
+#define VPEC_PROCESS_QUANTUM1__PROCESS5_QUANTUM__SHIFT                                                        0x8
+#define VPEC_PROCESS_QUANTUM1__PROCESS6_QUANTUM__SHIFT                                                        0x10
+#define VPEC_PROCESS_QUANTUM1__PROCESS7_QUANTUM__SHIFT                                                        0x18
+#define VPEC_PROCESS_QUANTUM1__PROCESS4_QUANTUM_MASK                                                          0x000000FFL
+#define VPEC_PROCESS_QUANTUM1__PROCESS5_QUANTUM_MASK                                                          0x0000FF00L
+#define VPEC_PROCESS_QUANTUM1__PROCESS6_QUANTUM_MASK                                                          0x00FF0000L
+#define VPEC_PROCESS_QUANTUM1__PROCESS7_QUANTUM_MASK                                                          0xFF000000L
+//VPEC_CONTEXT_SWITCH_THRESHOLD
+#define VPEC_CONTEXT_SWITCH_THRESHOLD__REALTIME_THRESHOLD__SHIFT                                              0x0
+#define VPEC_CONTEXT_SWITCH_THRESHOLD__FOCUS_THRESHOLD__SHIFT                                                 0x2
+#define VPEC_CONTEXT_SWITCH_THRESHOLD__NORMAL_THRESHOLD__SHIFT                                                0x4
+#define VPEC_CONTEXT_SWITCH_THRESHOLD__IDLE_THRESHOLD__SHIFT                                                  0x6
+#define VPEC_CONTEXT_SWITCH_THRESHOLD__REALTIME_THRESHOLD_MASK                                                0x00000003L
+#define VPEC_CONTEXT_SWITCH_THRESHOLD__FOCUS_THRESHOLD_MASK                                                   0x0000000CL
+#define VPEC_CONTEXT_SWITCH_THRESHOLD__NORMAL_THRESHOLD_MASK                                                  0x00000030L
+#define VPEC_CONTEXT_SWITCH_THRESHOLD__IDLE_THRESHOLD_MASK                                                    0x000000C0L
+//VPEC_GLOBAL_QUANTUM
+#define VPEC_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM__SHIFT                                                      0x0
+#define VPEC_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM__SHIFT                                                     0x8
+#define VPEC_GLOBAL_QUANTUM__GLOBAL_FOCUS_QUANTUM_MASK                                                        0x000000FFL
+#define VPEC_GLOBAL_QUANTUM__GLOBAL_NORMAL_QUANTUM_MASK                                                       0x0000FF00L
+//VPEC_WATCHDOG_CNTL
+#define VPEC_WATCHDOG_CNTL__QUEUE_HANG_COUNT__SHIFT                                                           0x0
+#define VPEC_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT__SHIFT                                                          0x8
+#define VPEC_WATCHDOG_CNTL__QUEUE_HANG_COUNT_MASK                                                             0x000000FFL
+#define VPEC_WATCHDOG_CNTL__CMD_TIMEOUT_COUNT_MASK                                                            0x0000FF00L
+//VPEC_ATOMIC_CNTL
+#define VPEC_ATOMIC_CNTL__LOOP_TIMER__SHIFT                                                                   0x0
+#define VPEC_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE__SHIFT                                                        0x1f
+#define VPEC_ATOMIC_CNTL__LOOP_TIMER_MASK                                                                     0x7FFFFFFFL
+#define VPEC_ATOMIC_CNTL__ATOMIC_RTN_INT_ENABLE_MASK                                                          0x80000000L
+//VPEC_UCODE_VERSION
+#define VPEC_UCODE_VERSION__T0_UCODE_VERSION__SHIFT                                                           0x0
+#define VPEC_UCODE_VERSION__T1_UCODE_VERSION__SHIFT                                                           0x10
+#define VPEC_UCODE_VERSION__T0_UCODE_VERSION_MASK                                                             0x0000FFFFL
+#define VPEC_UCODE_VERSION__T1_UCODE_VERSION_MASK                                                             0xFFFF0000L
+//VPEC_MEMREQ_BURST_CNTL
+#define VPEC_MEMREQ_BURST_CNTL__DATA_RD_BURST__SHIFT                                                          0x0
+#define VPEC_MEMREQ_BURST_CNTL__DATA_WR_BURST__SHIFT                                                          0x2
+#define VPEC_MEMREQ_BURST_CNTL__RB_RD_BURST__SHIFT                                                            0x4
+#define VPEC_MEMREQ_BURST_CNTL__IB_RD_BURST__SHIFT                                                            0x6
+#define VPEC_MEMREQ_BURST_CNTL__WR_BURST_WAIT_CYCLE__SHIFT                                                    0x8
+#define VPEC_MEMREQ_BURST_CNTL__DATA_RD_BURST_MASK                                                            0x00000003L
+#define VPEC_MEMREQ_BURST_CNTL__DATA_WR_BURST_MASK                                                            0x0000000CL
+#define VPEC_MEMREQ_BURST_CNTL__RB_RD_BURST_MASK                                                              0x00000030L
+#define VPEC_MEMREQ_BURST_CNTL__IB_RD_BURST_MASK                                                              0x000000C0L
+#define VPEC_MEMREQ_BURST_CNTL__WR_BURST_WAIT_CYCLE_MASK                                                      0x00000700L
+//VPEC_TIMESTAMP_CNTL
+#define VPEC_TIMESTAMP_CNTL__CAPTURE__SHIFT                                                                   0x0
+#define VPEC_TIMESTAMP_CNTL__CAPTURE_MASK                                                                     0x00000001L
+//VPEC_GLOBAL_TIMESTAMP_LO
+#define VPEC_GLOBAL_TIMESTAMP_LO__DATA__SHIFT                                                                 0x0
+#define VPEC_GLOBAL_TIMESTAMP_LO__DATA_MASK                                                                   0xFFFFFFFFL
+//VPEC_GLOBAL_TIMESTAMP_HI
+#define VPEC_GLOBAL_TIMESTAMP_HI__DATA__SHIFT                                                                 0x0
+#define VPEC_GLOBAL_TIMESTAMP_HI__DATA_MASK                                                                   0xFFFFFFFFL
+//VPEC_FREEZE
+#define VPEC_FREEZE__PREEMPT__SHIFT                                                                           0x0
+#define VPEC_FREEZE__FREEZE__SHIFT                                                                            0x4
+#define VPEC_FREEZE__FROZEN__SHIFT                                                                            0x5
+#define VPEC_FREEZE__F32_FREEZE__SHIFT                                                                        0x6
+#define VPEC_FREEZE__PREEMPT_MASK                                                                             0x00000001L
+#define VPEC_FREEZE__FREEZE_MASK                                                                              0x00000010L
+#define VPEC_FREEZE__FROZEN_MASK                                                                              0x00000020L
+#define VPEC_FREEZE__F32_FREEZE_MASK                                                                          0x00000040L
+//VPEC_CE_CTRL
+#define VPEC_CE_CTRL__RD_LUT_WATERMARK__SHIFT                                                                 0x0
+#define VPEC_CE_CTRL__RD_LUT_DEPTH__SHIFT                                                                     0x3
+#define VPEC_CE_CTRL__WR_AFIFO_WATERMARK__SHIFT                                                               0x5
+#define VPEC_CE_CTRL__RESERVED__SHIFT                                                                         0x8
+#define VPEC_CE_CTRL__RD_LUT_WATERMARK_MASK                                                                   0x00000007L
+#define VPEC_CE_CTRL__RD_LUT_DEPTH_MASK                                                                       0x00000018L
+#define VPEC_CE_CTRL__WR_AFIFO_WATERMARK_MASK                                                                 0x000000E0L
+#define VPEC_CE_CTRL__RESERVED_MASK                                                                           0xFFFFFF00L
+//VPEC_RELAX_ORDERING_LUT
+#define VPEC_RELAX_ORDERING_LUT__RESERVED0__SHIFT                                                             0x0
+#define VPEC_RELAX_ORDERING_LUT__VPE__SHIFT                                                                   0x1
+#define VPEC_RELAX_ORDERING_LUT__RESERVED_2_2__SHIFT                                                          0x2
+#define VPEC_RELAX_ORDERING_LUT__RESERVED3__SHIFT                                                             0x3
+#define VPEC_RELAX_ORDERING_LUT__RESERVED4__SHIFT                                                             0x4
+#define VPEC_RELAX_ORDERING_LUT__FENCE__SHIFT                                                                 0x5
+#define VPEC_RELAX_ORDERING_LUT__RESERVED76__SHIFT                                                            0x6
+#define VPEC_RELAX_ORDERING_LUT__POLL_MEM__SHIFT                                                              0x8
+#define VPEC_RELAX_ORDERING_LUT__COND_EXE__SHIFT                                                              0x9
+#define VPEC_RELAX_ORDERING_LUT__ATOMIC__SHIFT                                                                0xa
+#define VPEC_RELAX_ORDERING_LUT__RESERVED_11_11__SHIFT                                                        0xb
+#define VPEC_RELAX_ORDERING_LUT__RESERVED_12_12__SHIFT                                                        0xc
+#define VPEC_RELAX_ORDERING_LUT__TIMESTAMP__SHIFT                                                             0xd
+#define VPEC_RELAX_ORDERING_LUT__NATIVE_FENCE__SHIFT                                                          0xe
+#define VPEC_RELAX_ORDERING_LUT__RESERVED__SHIFT                                                              0xf
+#define VPEC_RELAX_ORDERING_LUT__WORLD_SWITCH__SHIFT                                                          0x1b
+#define VPEC_RELAX_ORDERING_LUT__RPTR_WRB__SHIFT                                                              0x1c
+#define VPEC_RELAX_ORDERING_LUT__RESERVED_29_29__SHIFT                                                        0x1d
+#define VPEC_RELAX_ORDERING_LUT__IB_FETCH__SHIFT                                                              0x1e
+#define VPEC_RELAX_ORDERING_LUT__RB_FETCH__SHIFT                                                              0x1f
+#define VPEC_RELAX_ORDERING_LUT__RESERVED0_MASK                                                               0x00000001L
+#define VPEC_RELAX_ORDERING_LUT__VPE_MASK                                                                     0x00000002L
+#define VPEC_RELAX_ORDERING_LUT__RESERVED_2_2_MASK                                                            0x00000004L
+#define VPEC_RELAX_ORDERING_LUT__RESERVED3_MASK                                                               0x00000008L
+#define VPEC_RELAX_ORDERING_LUT__RESERVED4_MASK                                                               0x00000010L
+#define VPEC_RELAX_ORDERING_LUT__FENCE_MASK                                                                   0x00000020L
+#define VPEC_RELAX_ORDERING_LUT__RESERVED76_MASK                                                              0x000000C0L
+#define VPEC_RELAX_ORDERING_LUT__POLL_MEM_MASK                                                                0x00000100L
+#define VPEC_RELAX_ORDERING_LUT__COND_EXE_MASK                                                                0x00000200L
+#define VPEC_RELAX_ORDERING_LUT__ATOMIC_MASK                                                                  0x00000400L
+#define VPEC_RELAX_ORDERING_LUT__RESERVED_11_11_MASK                                                          0x00000800L
+#define VPEC_RELAX_ORDERING_LUT__RESERVED_12_12_MASK                                                          0x00001000L
+#define VPEC_RELAX_ORDERING_LUT__TIMESTAMP_MASK                                                               0x00002000L
+#define VPEC_RELAX_ORDERING_LUT__NATIVE_FENCE_MASK                                                            0x00004000L
+#define VPEC_RELAX_ORDERING_LUT__RESERVED_MASK                                                                0x07FF8000L
+#define VPEC_RELAX_ORDERING_LUT__WORLD_SWITCH_MASK                                                            0x08000000L
+#define VPEC_RELAX_ORDERING_LUT__RPTR_WRB_MASK                                                                0x10000000L
+#define VPEC_RELAX_ORDERING_LUT__RESERVED_29_29_MASK                                                          0x20000000L
+#define VPEC_RELAX_ORDERING_LUT__IB_FETCH_MASK                                                                0x40000000L
+#define VPEC_RELAX_ORDERING_LUT__RB_FETCH_MASK                                                                0x80000000L
+//VPEC_CREDIT_CNTL
+#define VPEC_CREDIT_CNTL__DRM_CREDIT__SHIFT                                                                   0x0
+#define VPEC_CREDIT_CNTL__MC_WRREQ_CREDIT__SHIFT                                                              0x7
+#define VPEC_CREDIT_CNTL__MC_RDREQ_CREDIT__SHIFT                                                              0xd
+#define VPEC_CREDIT_CNTL__DRM_CREDIT_MASK                                                                     0x0000007FL
+#define VPEC_CREDIT_CNTL__MC_WRREQ_CREDIT_MASK                                                                0x00001F80L
+#define VPEC_CREDIT_CNTL__MC_RDREQ_CREDIT_MASK                                                                0x0007E000L
+//VPEC_SCRATCH_RAM_DATA
+#define VPEC_SCRATCH_RAM_DATA__DATA__SHIFT                                                                    0x0
+#define VPEC_SCRATCH_RAM_DATA__DATA_MASK                                                                      0xFFFFFFFFL
+//VPEC_SCRATCH_RAM_ADDR
+#define VPEC_SCRATCH_RAM_ADDR__ADDR__SHIFT                                                                    0x0
+#define VPEC_SCRATCH_RAM_ADDR__ADDR_MASK                                                                      0x000000FFL
+//VPEC_QUEUE_RESET_REQ
+#define VPEC_QUEUE_RESET_REQ__QUEUE0_RESET__SHIFT                                                             0x0
+#define VPEC_QUEUE_RESET_REQ__QUEUE1_RESET__SHIFT                                                             0x1
+#define VPEC_QUEUE_RESET_REQ__QUEUE2_RESET__SHIFT                                                             0x2
+#define VPEC_QUEUE_RESET_REQ__QUEUE3_RESET__SHIFT                                                             0x3
+#define VPEC_QUEUE_RESET_REQ__QUEUE4_RESET__SHIFT                                                             0x4
+#define VPEC_QUEUE_RESET_REQ__QUEUE5_RESET__SHIFT                                                             0x5
+#define VPEC_QUEUE_RESET_REQ__QUEUE6_RESET__SHIFT                                                             0x6
+#define VPEC_QUEUE_RESET_REQ__QUEUE7_RESET__SHIFT                                                             0x7
+#define VPEC_QUEUE_RESET_REQ__RESERVED__SHIFT                                                                 0x8
+#define VPEC_QUEUE_RESET_REQ__QUEUE0_RESET_MASK                                                               0x00000001L
+#define VPEC_QUEUE_RESET_REQ__QUEUE1_RESET_MASK                                                               0x00000002L
+#define VPEC_QUEUE_RESET_REQ__QUEUE2_RESET_MASK                                                               0x00000004L
+#define VPEC_QUEUE_RESET_REQ__QUEUE3_RESET_MASK                                                               0x00000008L
+#define VPEC_QUEUE_RESET_REQ__QUEUE4_RESET_MASK                                                               0x00000010L
+#define VPEC_QUEUE_RESET_REQ__QUEUE5_RESET_MASK                                                               0x00000020L
+#define VPEC_QUEUE_RESET_REQ__QUEUE6_RESET_MASK                                                               0x00000040L
+#define VPEC_QUEUE_RESET_REQ__QUEUE7_RESET_MASK                                                               0x00000080L
+#define VPEC_QUEUE_RESET_REQ__RESERVED_MASK                                                                   0xFFFFFF00L
+//VPEC_MAILBOX0
+#define VPEC_MAILBOX0__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX0__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX1
+#define VPEC_MAILBOX1__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX1__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX2
+#define VPEC_MAILBOX2__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX2__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX3
+#define VPEC_MAILBOX3__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX3__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX4
+#define VPEC_MAILBOX4__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX4__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX5
+#define VPEC_MAILBOX5__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX5__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX6
+#define VPEC_MAILBOX6__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX6__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX7
+#define VPEC_MAILBOX7__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX7__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX8
+#define VPEC_MAILBOX8__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX8__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX9
+#define VPEC_MAILBOX9__VALUE__SHIFT                                                                           0x0
+#define VPEC_MAILBOX9__VALUE_MASK                                                                             0xFFFFFFFFL
+//VPEC_MAILBOX10
+#define VPEC_MAILBOX10__VALUE__SHIFT                                                                          0x0
+#define VPEC_MAILBOX10__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_MAILBOX11
+#define VPEC_MAILBOX11__VALUE__SHIFT                                                                          0x0
+#define VPEC_MAILBOX11__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_MAILBOX12
+#define VPEC_MAILBOX12__VALUE__SHIFT                                                                          0x0
+#define VPEC_MAILBOX12__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_MAILBOX13
+#define VPEC_MAILBOX13__VALUE__SHIFT                                                                          0x0
+#define VPEC_MAILBOX13__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_MAILBOX14
+#define VPEC_MAILBOX14__VALUE__SHIFT                                                                          0x0
+#define VPEC_MAILBOX14__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_MAILBOX15
+#define VPEC_MAILBOX15__VALUE__SHIFT                                                                          0x0
+#define VPEC_MAILBOX15__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_PUB_DUMMY0
+#define VPEC_PUB_DUMMY0__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY0__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY1
+#define VPEC_PUB_DUMMY1__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY1__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY2
+#define VPEC_PUB_DUMMY2__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY2__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY3
+#define VPEC_PUB_DUMMY3__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY3__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY4
+#define VPEC_PUB_DUMMY4__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY4__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY5
+#define VPEC_PUB_DUMMY5__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY5__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY6
+#define VPEC_PUB_DUMMY6__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY6__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY7
+#define VPEC_PUB_DUMMY7__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY7__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY8
+#define VPEC_PUB_DUMMY8__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY8__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY9
+#define VPEC_PUB_DUMMY9__VALUE__SHIFT                                                                         0x0
+#define VPEC_PUB_DUMMY9__VALUE_MASK                                                                           0xFFFFFFFFL
+//VPEC_PUB_DUMMY10
+#define VPEC_PUB_DUMMY10__VALUE__SHIFT                                                                        0x0
+#define VPEC_PUB_DUMMY10__VALUE_MASK                                                                          0xFFFFFFFFL
+//VPEC_PUB_DUMMY11
+#define VPEC_PUB_DUMMY11__VALUE__SHIFT                                                                        0x0
+#define VPEC_PUB_DUMMY11__VALUE_MASK                                                                          0xFFFFFFFFL
+//VPEC_UCODE1_CHECKSUM
+#define VPEC_UCODE1_CHECKSUM__DATA__SHIFT                                                                     0x0
+#define VPEC_UCODE1_CHECKSUM__DATA_MASK                                                                       0xFFFFFFFFL
+//VPEC_VERSION
+#define VPEC_VERSION__MINVER__SHIFT                                                                           0x0
+#define VPEC_VERSION__MAJVER__SHIFT                                                                           0x8
+#define VPEC_VERSION__REV__SHIFT                                                                              0x10
+#define VPEC_VERSION__MINVER_MASK                                                                             0x0000007FL
+#define VPEC_VERSION__MAJVER_MASK                                                                             0x00007F00L
+#define VPEC_VERSION__REV_MASK                                                                                0x003F0000L
+//VPEC_UCODE_CHECKSUM
+#define VPEC_UCODE_CHECKSUM__DATA__SHIFT                                                                      0x0
+#define VPEC_UCODE_CHECKSUM__DATA_MASK                                                                        0xFFFFFFFFL
+//VPEC_RB_RPTR_FETCH
+#define VPEC_RB_RPTR_FETCH__OFFSET__SHIFT                                                                     0x2
+#define VPEC_RB_RPTR_FETCH__OFFSET_MASK                                                                       0xFFFFFFFCL
+//VPEC_RB_RPTR_FETCH_HI
+#define VPEC_RB_RPTR_FETCH_HI__OFFSET__SHIFT                                                                  0x0
+#define VPEC_RB_RPTR_FETCH_HI__OFFSET_MASK                                                                    0xFFFFFFFFL
+//VPEC_IB_OFFSET_FETCH
+#define VPEC_IB_OFFSET_FETCH__OFFSET__SHIFT                                                                   0x2
+#define VPEC_IB_OFFSET_FETCH__OFFSET_MASK                                                                     0x003FFFFCL
+//VPEC_CMDIB_OFFSET_FETCH
+#define VPEC_CMDIB_OFFSET_FETCH__OFFSET__SHIFT                                                                0x2
+#define VPEC_CMDIB_OFFSET_FETCH__OFFSET_MASK                                                                  0x003FFFFCL
+//VPEC_3DLUTIB_OFFSET_FETCH
+#define VPEC_3DLUTIB_OFFSET_FETCH__OFFSET__SHIFT                                                              0x2
+#define VPEC_3DLUTIB_OFFSET_FETCH__OFFSET_MASK                                                                0x003FFFFCL
+//VPEC_ATOMIC_PREOP_LO
+#define VPEC_ATOMIC_PREOP_LO__DATA__SHIFT                                                                     0x0
+#define VPEC_ATOMIC_PREOP_LO__DATA_MASK                                                                       0xFFFFFFFFL
+//VPEC_ATOMIC_PREOP_HI
+#define VPEC_ATOMIC_PREOP_HI__DATA__SHIFT                                                                     0x0
+#define VPEC_ATOMIC_PREOP_HI__DATA_MASK                                                                       0xFFFFFFFFL
+//VPEC_CE_BUSY
+#define VPEC_CE_BUSY__CE_IP_PIPE0_BUSY__SHIFT                                                                 0x0
+#define VPEC_CE_BUSY__CE_IP_PIPE1_BUSY__SHIFT                                                                 0x1
+#define VPEC_CE_BUSY__CE_IP_PIPE2_BUSY__SHIFT                                                                 0x2
+#define VPEC_CE_BUSY__CE_IP_PIPE3_BUSY__SHIFT                                                                 0x3
+#define VPEC_CE_BUSY__CE_IP_PIPE4_BUSY__SHIFT                                                                 0x4
+#define VPEC_CE_BUSY__CE_IP_PIPE5_BUSY__SHIFT                                                                 0x5
+#define VPEC_CE_BUSY__CE_OP_PIPE0_BUSY__SHIFT                                                                 0x10
+#define VPEC_CE_BUSY__CE_OP_PIPE1_BUSY__SHIFT                                                                 0x11
+#define VPEC_CE_BUSY__CE_OP_PIPE2_BUSY__SHIFT                                                                 0x12
+#define VPEC_CE_BUSY__CE_OP_PIPE3_BUSY__SHIFT                                                                 0x13
+#define VPEC_CE_BUSY__CE_OP_PIPE4_BUSY__SHIFT                                                                 0x14
+#define VPEC_CE_BUSY__CE_OP_PIPE5_BUSY__SHIFT                                                                 0x15
+#define VPEC_CE_BUSY__CE_OP_PIPE6_BUSY__SHIFT                                                                 0x16
+#define VPEC_CE_BUSY__CE_OP_PIPE7_BUSY__SHIFT                                                                 0x17
+#define VPEC_CE_BUSY__CE_OP_PIPE8_BUSY__SHIFT                                                                 0x18
+#define VPEC_CE_BUSY__CE_OP_PIPE9_BUSY__SHIFT                                                                 0x19
+#define VPEC_CE_BUSY__CE_IP_PIPE0_BUSY_MASK                                                                   0x00000001L
+#define VPEC_CE_BUSY__CE_IP_PIPE1_BUSY_MASK                                                                   0x00000002L
+#define VPEC_CE_BUSY__CE_IP_PIPE2_BUSY_MASK                                                                   0x00000004L
+#define VPEC_CE_BUSY__CE_IP_PIPE3_BUSY_MASK                                                                   0x00000008L
+#define VPEC_CE_BUSY__CE_IP_PIPE4_BUSY_MASK                                                                   0x00000010L
+#define VPEC_CE_BUSY__CE_IP_PIPE5_BUSY_MASK                                                                   0x00000020L
+#define VPEC_CE_BUSY__CE_OP_PIPE0_BUSY_MASK                                                                   0x00010000L
+#define VPEC_CE_BUSY__CE_OP_PIPE1_BUSY_MASK                                                                   0x00020000L
+#define VPEC_CE_BUSY__CE_OP_PIPE2_BUSY_MASK                                                                   0x00040000L
+#define VPEC_CE_BUSY__CE_OP_PIPE3_BUSY_MASK                                                                   0x00080000L
+#define VPEC_CE_BUSY__CE_OP_PIPE4_BUSY_MASK                                                                   0x00100000L
+#define VPEC_CE_BUSY__CE_OP_PIPE5_BUSY_MASK                                                                   0x00200000L
+#define VPEC_CE_BUSY__CE_OP_PIPE6_BUSY_MASK                                                                   0x00400000L
+#define VPEC_CE_BUSY__CE_OP_PIPE7_BUSY_MASK                                                                   0x00800000L
+#define VPEC_CE_BUSY__CE_OP_PIPE8_BUSY_MASK                                                                   0x01000000L
+#define VPEC_CE_BUSY__CE_OP_PIPE9_BUSY_MASK                                                                   0x02000000L
+//VPEC_F32_COUNTER
+#define VPEC_F32_COUNTER__VALUE__SHIFT                                                                        0x0
+#define VPEC_F32_COUNTER__VALUE_MASK                                                                          0xFFFFFFFFL
+//VPEC_HOLE_ADDR_LO
+#define VPEC_HOLE_ADDR_LO__VALUE__SHIFT                                                                       0x0
+#define VPEC_HOLE_ADDR_LO__VALUE_MASK                                                                         0xFFFFFFFFL
+//VPEC_HOLE_ADDR_HI
+#define VPEC_HOLE_ADDR_HI__VALUE__SHIFT                                                                       0x0
+#define VPEC_HOLE_ADDR_HI__VALUE_MASK                                                                         0xFFFFFFFFL
+//VPEC_ERROR_LOG
+#define VPEC_ERROR_LOG__OVERRIDE__SHIFT                                                                       0x0
+#define VPEC_ERROR_LOG__STATUS__SHIFT                                                                         0x10
+#define VPEC_ERROR_LOG__OVERRIDE_MASK                                                                         0x0000FFFFL
+#define VPEC_ERROR_LOG__STATUS_MASK                                                                           0xFFFF0000L
+//VPEC_INT_STATUS
+#define VPEC_INT_STATUS__DATA__SHIFT                                                                          0x0
+#define VPEC_INT_STATUS__DATA_MASK                                                                            0xFFFFFFFFL
+//VPEC_STATUS
+#define VPEC_STATUS__IDLE__SHIFT                                                                              0x0
+#define VPEC_STATUS__REG_IDLE__SHIFT                                                                          0x1
+#define VPEC_STATUS__RB_EMPTY__SHIFT                                                                          0x2
+#define VPEC_STATUS__RB_FULL__SHIFT                                                                           0x3
+#define VPEC_STATUS__RB_CMD_IDLE__SHIFT                                                                       0x4
+#define VPEC_STATUS__RB_CMD_FULL__SHIFT                                                                       0x5
+#define VPEC_STATUS__IB_CMD_IDLE__SHIFT                                                                       0x6
+#define VPEC_STATUS__IB_CMD_FULL__SHIFT                                                                       0x7
+#define VPEC_STATUS__BLOCK_IDLE__SHIFT                                                                        0x8
+#define VPEC_STATUS__INSIDE_VPEP_CONFIG__SHIFT                                                                0x9
+#define VPEC_STATUS__EX_IDLE__SHIFT                                                                           0xa
+#define VPEC_STATUS__INSIDE_VPEP_3DLUT_CONFIG__SHIFT                                                          0xb
+#define VPEC_STATUS__PACKET_READY__SHIFT                                                                      0xc
+#define VPEC_STATUS__MC_WR_IDLE__SHIFT                                                                        0xd
+#define VPEC_STATUS__SRBM_IDLE__SHIFT                                                                         0xe
+#define VPEC_STATUS__CONTEXT_EMPTY__SHIFT                                                                     0xf
+#define VPEC_STATUS__INSIDE_IB__SHIFT                                                                         0x10
+#define VPEC_STATUS__RB_MC_RREQ_IDLE__SHIFT                                                                   0x11
+#define VPEC_STATUS__IB_MC_RREQ_IDLE__SHIFT                                                                   0x12
+#define VPEC_STATUS__MC_RD_IDLE__SHIFT                                                                        0x13
+#define VPEC_STATUS__DELTA_RPTR_EMPTY__SHIFT                                                                  0x14
+#define VPEC_STATUS__MC_RD_RET_STALL__SHIFT                                                                   0x15
+#define VPEC_STATUS__LUTIB_CMD_IDLE__SHIFT                                                                    0x16
+#define VPEC_STATUS__LUTIB_CMD_FULL__SHIFT                                                                    0x17
+#define VPEC_STATUS__CMDIB_MC_RREQ_IDLE__SHIFT                                                                0x18
+#define VPEC_STATUS__PREV_CMD_IDLE__SHIFT                                                                     0x19
+#define VPEC_STATUS__CMDIB_CMD_IDLE__SHIFT                                                                    0x1a
+#define VPEC_STATUS__CMDIB_CMD_FULL__SHIFT                                                                    0x1b
+#define VPEC_STATUS__RESERVED_29_28__SHIFT                                                                    0x1c
+#define VPEC_STATUS__INT_IDLE__SHIFT                                                                          0x1e
+#define VPEC_STATUS__INT_REQ_STALL__SHIFT                                                                     0x1f
+#define VPEC_STATUS__IDLE_MASK                                                                                0x00000001L
+#define VPEC_STATUS__REG_IDLE_MASK                                                                            0x00000002L
+#define VPEC_STATUS__RB_EMPTY_MASK                                                                            0x00000004L
+#define VPEC_STATUS__RB_FULL_MASK                                                                             0x00000008L
+#define VPEC_STATUS__RB_CMD_IDLE_MASK                                                                         0x00000010L
+#define VPEC_STATUS__RB_CMD_FULL_MASK                                                                         0x00000020L
+#define VPEC_STATUS__IB_CMD_IDLE_MASK                                                                         0x00000040L
+#define VPEC_STATUS__IB_CMD_FULL_MASK                                                                         0x00000080L
+#define VPEC_STATUS__BLOCK_IDLE_MASK                                                                          0x00000100L
+#define VPEC_STATUS__INSIDE_VPEP_CONFIG_MASK                                                                  0x00000200L
+#define VPEC_STATUS__EX_IDLE_MASK                                                                             0x00000400L
+#define VPEC_STATUS__INSIDE_VPEP_3DLUT_CONFIG_MASK                                                            0x00000800L
+#define VPEC_STATUS__PACKET_READY_MASK                                                                        0x00001000L
+#define VPEC_STATUS__MC_WR_IDLE_MASK                                                                          0x00002000L
+#define VPEC_STATUS__SRBM_IDLE_MASK                                                                           0x00004000L
+#define VPEC_STATUS__CONTEXT_EMPTY_MASK                                                                       0x00008000L
+#define VPEC_STATUS__INSIDE_IB_MASK                                                                           0x00010000L
+#define VPEC_STATUS__RB_MC_RREQ_IDLE_MASK                                                                     0x00020000L
+#define VPEC_STATUS__IB_MC_RREQ_IDLE_MASK                                                                     0x00040000L
+#define VPEC_STATUS__MC_RD_IDLE_MASK                                                                          0x00080000L
+#define VPEC_STATUS__DELTA_RPTR_EMPTY_MASK                                                                    0x00100000L
+#define VPEC_STATUS__MC_RD_RET_STALL_MASK                                                                     0x00200000L
+#define VPEC_STATUS__LUTIB_CMD_IDLE_MASK                                                                      0x00400000L
+#define VPEC_STATUS__LUTIB_CMD_FULL_MASK                                                                      0x00800000L
+#define VPEC_STATUS__CMDIB_MC_RREQ_IDLE_MASK                                                                  0x01000000L
+#define VPEC_STATUS__PREV_CMD_IDLE_MASK                                                                       0x02000000L
+#define VPEC_STATUS__CMDIB_CMD_IDLE_MASK                                                                      0x04000000L
+#define VPEC_STATUS__CMDIB_CMD_FULL_MASK                                                                      0x08000000L
+#define VPEC_STATUS__RESERVED_29_28_MASK                                                                      0x30000000L
+#define VPEC_STATUS__INT_IDLE_MASK                                                                            0x40000000L
+#define VPEC_STATUS__INT_REQ_STALL_MASK                                                                       0x80000000L
+//VPEC_STATUS1
+#define VPEC_STATUS1__EX_START__SHIFT                                                                         0x0
+#define VPEC_STATUS1__VPEC_IDLE__SHIFT                                                                        0x1
+#define VPEC_STATUS1__RESERVED_31_2__SHIFT                                                                    0x2
+#define VPEC_STATUS1__EX_START_MASK                                                                           0x00000001L
+#define VPEC_STATUS1__VPEC_IDLE_MASK                                                                          0x00000002L
+#define VPEC_STATUS1__RESERVED_31_2_MASK                                                                      0xFFFFFFFCL
+//VPEC_STATUS2
+#define VPEC_STATUS2__ID__SHIFT                                                                               0x0
+#define VPEC_STATUS2__TH0F32_INSTR_PTR__SHIFT                                                                 0x2
+#define VPEC_STATUS2__CMD_OP__SHIFT                                                                           0x10
+#define VPEC_STATUS2__ID_MASK                                                                                 0x00000003L
+#define VPEC_STATUS2__TH0F32_INSTR_PTR_MASK                                                                   0x0000FFFCL
+#define VPEC_STATUS2__CMD_OP_MASK                                                                             0xFFFF0000L
+//VPEC_STATUS3
+#define VPEC_STATUS3__RESERVED_15_0__SHIFT                                                                    0x0
+#define VPEC_STATUS3__RESERVED_19_16__SHIFT                                                                   0x10
+#define VPEC_STATUS3__EXCEPTION_IDLE__SHIFT                                                                   0x14
+#define VPEC_STATUS3__RESERVED_21_21__SHIFT                                                                   0x15
+#define VPEC_STATUS3__RESERVED_22_22__SHIFT                                                                   0x16
+#define VPEC_STATUS3__RESERVED_23_23__SHIFT                                                                   0x17
+#define VPEC_STATUS3__RESERVED_24_24__SHIFT                                                                   0x18
+#define VPEC_STATUS3__RESERVED_25_25__SHIFT                                                                   0x19
+#define VPEC_STATUS3__INT_QUEUE_ID__SHIFT                                                                     0x1a
+#define VPEC_STATUS3__RESERVED_31_30__SHIFT                                                                   0x1e
+#define VPEC_STATUS3__RESERVED_15_0_MASK                                                                      0x0000FFFFL
+#define VPEC_STATUS3__RESERVED_19_16_MASK                                                                     0x000F0000L
+#define VPEC_STATUS3__EXCEPTION_IDLE_MASK                                                                     0x00100000L
+#define VPEC_STATUS3__RESERVED_21_21_MASK                                                                     0x00200000L
+#define VPEC_STATUS3__RESERVED_22_22_MASK                                                                     0x00400000L
+#define VPEC_STATUS3__RESERVED_23_23_MASK                                                                     0x00800000L
+#define VPEC_STATUS3__RESERVED_24_24_MASK                                                                     0x01000000L
+#define VPEC_STATUS3__RESERVED_25_25_MASK                                                                     0x02000000L
+#define VPEC_STATUS3__INT_QUEUE_ID_MASK                                                                       0x3C000000L
+#define VPEC_STATUS3__RESERVED_31_30_MASK                                                                     0xC0000000L
+//VPEC_STATUS4
+#define VPEC_STATUS4__IDLE__SHIFT                                                                             0x0
+#define VPEC_STATUS4__IH_OUTSTANDING__SHIFT                                                                   0x2
+#define VPEC_STATUS4__RESERVED_3_3__SHIFT                                                                     0x3
+#define VPEC_STATUS4__CH_RD_OUTSTANDING__SHIFT                                                                0x4
+#define VPEC_STATUS4__CH_WR_OUTSTANDING__SHIFT                                                                0x5
+#define VPEC_STATUS4__RESERVED_6_6__SHIFT                                                                     0x6
+#define VPEC_STATUS4__RESERVED_7_7__SHIFT                                                                     0x7
+#define VPEC_STATUS4__RESERVED_8_8__SHIFT                                                                     0x8
+#define VPEC_STATUS4__RESERVED_9_9__SHIFT                                                                     0x9
+#define VPEC_STATUS4__REG_POLLING__SHIFT                                                                      0xa
+#define VPEC_STATUS4__MEM_POLLING__SHIFT                                                                      0xb
+#define VPEC_STATUS4__VPEP_REG_RD_OUTSTANDING__SHIFT                                                          0xc
+#define VPEC_STATUS4__VPEP_REG_WR_OUTSTANDING__SHIFT                                                          0xd
+#define VPEC_STATUS4__RESERVED_15_14__SHIFT                                                                   0xe
+#define VPEC_STATUS4__ACTIVE_QUEUE_ID__SHIFT                                                                  0x10
+#define VPEC_STATUS4__RESERVED_27_20__SHIFT                                                                   0x14
+#define VPEC_STATUS4__IDLE_MASK                                                                               0x00000001L
+#define VPEC_STATUS4__IH_OUTSTANDING_MASK                                                                     0x00000004L
+#define VPEC_STATUS4__RESERVED_3_3_MASK                                                                       0x00000008L
+#define VPEC_STATUS4__CH_RD_OUTSTANDING_MASK                                                                  0x00000010L
+#define VPEC_STATUS4__CH_WR_OUTSTANDING_MASK                                                                  0x00000020L
+#define VPEC_STATUS4__RESERVED_6_6_MASK                                                                       0x00000040L
+#define VPEC_STATUS4__RESERVED_7_7_MASK                                                                       0x00000080L
+#define VPEC_STATUS4__RESERVED_8_8_MASK                                                                       0x00000100L
+#define VPEC_STATUS4__RESERVED_9_9_MASK                                                                       0x00000200L
+#define VPEC_STATUS4__REG_POLLING_MASK                                                                        0x00000400L
+#define VPEC_STATUS4__MEM_POLLING_MASK                                                                        0x00000800L
+#define VPEC_STATUS4__VPEP_REG_RD_OUTSTANDING_MASK                                                            0x00001000L
+#define VPEC_STATUS4__VPEP_REG_WR_OUTSTANDING_MASK                                                            0x00002000L
+#define VPEC_STATUS4__RESERVED_15_14_MASK                                                                     0x0000C000L
+#define VPEC_STATUS4__ACTIVE_QUEUE_ID_MASK                                                                    0x000F0000L
+#define VPEC_STATUS4__RESERVED_27_20_MASK                                                                     0x0FF00000L
+//VPEC_STATUS5
+#define VPEC_STATUS5__QUEUE0_RB_ENABLE_STATUS__SHIFT                                                          0x0
+#define VPEC_STATUS5__QUEUE1_RB_ENABLE_STATUS__SHIFT                                                          0x1
+#define VPEC_STATUS5__QUEUE2_RB_ENABLE_STATUS__SHIFT                                                          0x2
+#define VPEC_STATUS5__QUEUE3_RB_ENABLE_STATUS__SHIFT                                                          0x3
+#define VPEC_STATUS5__QUEUE4_RB_ENABLE_STATUS__SHIFT                                                          0x4
+#define VPEC_STATUS5__QUEUE5_RB_ENABLE_STATUS__SHIFT                                                          0x5
+#define VPEC_STATUS5__QUEUE6_RB_ENABLE_STATUS__SHIFT                                                          0x6
+#define VPEC_STATUS5__QUEUE7_RB_ENABLE_STATUS__SHIFT                                                          0x7
+#define VPEC_STATUS5__RESERVED_27_16__SHIFT                                                                   0x10
+#define VPEC_STATUS5__QUEUE0_RB_ENABLE_STATUS_MASK                                                            0x00000001L
+#define VPEC_STATUS5__QUEUE1_RB_ENABLE_STATUS_MASK                                                            0x00000002L
+#define VPEC_STATUS5__QUEUE2_RB_ENABLE_STATUS_MASK                                                            0x00000004L
+#define VPEC_STATUS5__QUEUE3_RB_ENABLE_STATUS_MASK                                                            0x00000008L
+#define VPEC_STATUS5__QUEUE4_RB_ENABLE_STATUS_MASK                                                            0x00000010L
+#define VPEC_STATUS5__QUEUE5_RB_ENABLE_STATUS_MASK                                                            0x00000020L
+#define VPEC_STATUS5__QUEUE6_RB_ENABLE_STATUS_MASK                                                            0x00000040L
+#define VPEC_STATUS5__QUEUE7_RB_ENABLE_STATUS_MASK                                                            0x00000080L
+#define VPEC_STATUS5__RESERVED_27_16_MASK                                                                     0x000F0000L
+//VPEC_STATUS6
+#define VPEC_STATUS6__ID__SHIFT                                                                               0x0
+#define VPEC_STATUS6__TH1F32_INSTR_PTR__SHIFT                                                                 0x2
+#define VPEC_STATUS6__TH1_EXCEPTION__SHIFT                                                                    0x10
+#define VPEC_STATUS6__ID_MASK                                                                                 0x00000003L
+#define VPEC_STATUS6__TH1F32_INSTR_PTR_MASK                                                                   0x0000FFFCL
+#define VPEC_STATUS6__TH1_EXCEPTION_MASK                                                                      0xFFFF0000L
+//VPEC_STATUS7
+#define VPEC_STATUS7__TH0_DBG_STATUS__SHIFT                                                                   0x0
+#define VPEC_STATUS7__TH0_DBG_STATUS_MASK                                                                     0xFFFFFFFFL
+//VPEC_STATUS8
+#define VPEC_STATUS8__CE_IP0_WREQ_IDLE__SHIFT                                                                 0x0
+#define VPEC_STATUS8__CE_IP0_WR_IDLE__SHIFT                                                                   0x1
+#define VPEC_STATUS8__CE_IP0_SPLIT_RD_IDLE__SHIFT                                                             0x2
+#define VPEC_STATUS8__CE_IP0_SPLIT_WR_IDLE__SHIFT                                                             0x3
+#define VPEC_STATUS8__CE_IP0_RREQ_IDLE__SHIFT                                                                 0x4
+#define VPEC_STATUS8__CE_IP0_OUT_IDLE__SHIFT                                                                  0x5
+#define VPEC_STATUS8__CE_IP0_IN_IDLE__SHIFT                                                                   0x6
+#define VPEC_STATUS8__CE_IP0_DST_IDLE__SHIFT                                                                  0x7
+#define VPEC_STATUS8__CE_IP0_CMD_IDLE__SHIFT                                                                  0x8
+#define VPEC_STATUS8__CE_IP1_WREQ_IDLE__SHIFT                                                                 0x9
+#define VPEC_STATUS8__CE_IP1_WR_IDLE__SHIFT                                                                   0xa
+#define VPEC_STATUS8__CE_IP1_SPLIT_RD_IDLE__SHIFT                                                             0xb
+#define VPEC_STATUS8__CE_IP1_SPLIT_WR_IDLE__SHIFT                                                             0xc
+#define VPEC_STATUS8__CE_IP1_RREQ_IDLE__SHIFT                                                                 0xd
+#define VPEC_STATUS8__CE_IP1_OUT_IDLE__SHIFT                                                                  0xe
+#define VPEC_STATUS8__CE_IP1_IN_IDLE__SHIFT                                                                   0xf
+#define VPEC_STATUS8__CE_IP1_DST_IDLE__SHIFT                                                                  0x10
+#define VPEC_STATUS8__CE_IP1_CMD_IDLE__SHIFT                                                                  0x11
+#define VPEC_STATUS8__CE_IP0_AFIFO_FULL__SHIFT                                                                0x12
+#define VPEC_STATUS8__CE_IP0_CMD_INFO_FULL__SHIFT                                                             0x13
+#define VPEC_STATUS8__CE_IP0_CMD_INFO1_FULL__SHIFT                                                            0x14
+#define VPEC_STATUS8__CE_IP1_AFIFO_FULL__SHIFT                                                                0x15
+#define VPEC_STATUS8__CE_IP1_CMD_INFO_FULL__SHIFT                                                             0x16
+#define VPEC_STATUS8__CE_IP1_CMD_INFO1_FULL__SHIFT                                                            0x17
+#define VPEC_STATUS8__CE_IP0_WR_STALL__SHIFT                                                                  0x18
+#define VPEC_STATUS8__CE_IP1_WR_STALL__SHIFT                                                                  0x19
+#define VPEC_STATUS8__CE_IP0_RD_STALL__SHIFT                                                                  0x1a
+#define VPEC_STATUS8__CE_IP1_RD_STALL__SHIFT                                                                  0x1b
+#define VPEC_STATUS8__RESERVED_31_28__SHIFT                                                                   0x1c
+#define VPEC_STATUS8__CE_IP0_WREQ_IDLE_MASK                                                                   0x00000001L
+#define VPEC_STATUS8__CE_IP0_WR_IDLE_MASK                                                                     0x00000002L
+#define VPEC_STATUS8__CE_IP0_SPLIT_RD_IDLE_MASK                                                               0x00000004L
+#define VPEC_STATUS8__CE_IP0_SPLIT_WR_IDLE_MASK                                                               0x00000008L
+#define VPEC_STATUS8__CE_IP0_RREQ_IDLE_MASK                                                                   0x00000010L
+#define VPEC_STATUS8__CE_IP0_OUT_IDLE_MASK                                                                    0x00000020L
+#define VPEC_STATUS8__CE_IP0_IN_IDLE_MASK                                                                     0x00000040L
+#define VPEC_STATUS8__CE_IP0_DST_IDLE_MASK                                                                    0x00000080L
+#define VPEC_STATUS8__CE_IP0_CMD_IDLE_MASK                                                                    0x00000100L
+#define VPEC_STATUS8__CE_IP1_WREQ_IDLE_MASK                                                                   0x00000200L
+#define VPEC_STATUS8__CE_IP1_WR_IDLE_MASK                                                                     0x00000400L
+#define VPEC_STATUS8__CE_IP1_SPLIT_RD_IDLE_MASK                                                               0x00000800L
+#define VPEC_STATUS8__CE_IP1_SPLIT_WR_IDLE_MASK                                                               0x00001000L
+#define VPEC_STATUS8__CE_IP1_RREQ_IDLE_MASK                                                                   0x00002000L
+#define VPEC_STATUS8__CE_IP1_OUT_IDLE_MASK                                                                    0x00004000L
+#define VPEC_STATUS8__CE_IP1_IN_IDLE_MASK                                                                     0x00008000L
+#define VPEC_STATUS8__CE_IP1_DST_IDLE_MASK                                                                    0x00010000L
+#define VPEC_STATUS8__CE_IP1_CMD_IDLE_MASK                                                                    0x00020000L
+#define VPEC_STATUS8__CE_IP0_AFIFO_FULL_MASK                                                                  0x00040000L
+#define VPEC_STATUS8__CE_IP0_CMD_INFO_FULL_MASK                                                               0x00080000L
+#define VPEC_STATUS8__CE_IP0_CMD_INFO1_FULL_MASK                                                              0x00100000L
+#define VPEC_STATUS8__CE_IP1_AFIFO_FULL_MASK                                                                  0x00200000L
+#define VPEC_STATUS8__CE_IP1_CMD_INFO_FULL_MASK                                                               0x00400000L
+#define VPEC_STATUS8__CE_IP1_CMD_INFO1_FULL_MASK                                                              0x00800000L
+#define VPEC_STATUS8__CE_IP0_WR_STALL_MASK                                                                    0x01000000L
+#define VPEC_STATUS8__CE_IP1_WR_STALL_MASK                                                                    0x02000000L
+#define VPEC_STATUS8__CE_IP0_RD_STALL_MASK                                                                    0x04000000L
+#define VPEC_STATUS8__CE_IP1_RD_STALL_MASK                                                                    0x08000000L
+#define VPEC_STATUS8__RESERVED_31_28_MASK                                                                     0xF0000000L
+//VPEC_STATUS9
+#define VPEC_STATUS9__CE_IP2_WREQ_IDLE__SHIFT                                                                 0x0
+#define VPEC_STATUS9__CE_IP2_WR_IDLE__SHIFT                                                                   0x1
+#define VPEC_STATUS9__CE_IP2_SPLIT_RD_IDLE__SHIFT                                                             0x2
+#define VPEC_STATUS9__CE_IP2_SPLIT_WR_IDLE__SHIFT                                                             0x3
+#define VPEC_STATUS9__CE_IP2_RREQ_IDLE__SHIFT                                                                 0x4
+#define VPEC_STATUS9__CE_IP2_OUT_IDLE__SHIFT                                                                  0x5
+#define VPEC_STATUS9__CE_IP2_IN_IDLE__SHIFT                                                                   0x6
+#define VPEC_STATUS9__CE_IP2_DST_IDLE__SHIFT                                                                  0x7
+#define VPEC_STATUS9__CE_IP2_CMD_IDLE__SHIFT                                                                  0x8
+#define VPEC_STATUS9__CE_IP3_WREQ_IDLE__SHIFT                                                                 0x9
+#define VPEC_STATUS9__CE_IP3_WR_IDLE__SHIFT                                                                   0xa
+#define VPEC_STATUS9__CE_IP3_SPLIT_RD_IDLE__SHIFT                                                             0xb
+#define VPEC_STATUS9__CE_IP3_SPLIT_WR_IDLE__SHIFT                                                             0xc
+#define VPEC_STATUS9__CE_IP3_RREQ_IDLE__SHIFT                                                                 0xd
+#define VPEC_STATUS9__CE_IP3_OUT_IDLE__SHIFT                                                                  0xe
+#define VPEC_STATUS9__CE_IP3_IN_IDLE__SHIFT                                                                   0xf
+#define VPEC_STATUS9__CE_IP3_DST_IDLE__SHIFT                                                                  0x10
+#define VPEC_STATUS9__CE_IP3_CMD_IDLE__SHIFT                                                                  0x11
+#define VPEC_STATUS9__CE_IP2_AFIFO_FULL__SHIFT                                                                0x12
+#define VPEC_STATUS9__CE_IP2_CMD_INFO_FULL__SHIFT                                                             0x13
+#define VPEC_STATUS9__CE_IP2_CMD_INFO1_FULL__SHIFT                                                            0x14
+#define VPEC_STATUS9__CE_IP3_AFIFO_FULL__SHIFT                                                                0x15
+#define VPEC_STATUS9__CE_IP3_CMD_INFO_FULL__SHIFT                                                             0x16
+#define VPEC_STATUS9__CE_IP3_CMD_INFO1_FULL__SHIFT                                                            0x17
+#define VPEC_STATUS9__CE_IP2_WR_STALL__SHIFT                                                                  0x18
+#define VPEC_STATUS9__CE_IP3_WR_STALL__SHIFT                                                                  0x19
+#define VPEC_STATUS9__CE_IP2_RD_STALL__SHIFT                                                                  0x1a
+#define VPEC_STATUS9__CE_IP3_RD_STALL__SHIFT                                                                  0x1b
+#define VPEC_STATUS9__RESERVED_31_28__SHIFT                                                                   0x1c
+#define VPEC_STATUS9__CE_IP2_WREQ_IDLE_MASK                                                                   0x00000001L
+#define VPEC_STATUS9__CE_IP2_WR_IDLE_MASK                                                                     0x00000002L
+#define VPEC_STATUS9__CE_IP2_SPLIT_RD_IDLE_MASK                                                               0x00000004L
+#define VPEC_STATUS9__CE_IP2_SPLIT_WR_IDLE_MASK                                                               0x00000008L
+#define VPEC_STATUS9__CE_IP2_RREQ_IDLE_MASK                                                                   0x00000010L
+#define VPEC_STATUS9__CE_IP2_OUT_IDLE_MASK                                                                    0x00000020L
+#define VPEC_STATUS9__CE_IP2_IN_IDLE_MASK                                                                     0x00000040L
+#define VPEC_STATUS9__CE_IP2_DST_IDLE_MASK                                                                    0x00000080L
+#define VPEC_STATUS9__CE_IP2_CMD_IDLE_MASK                                                                    0x00000100L
+#define VPEC_STATUS9__CE_IP3_WREQ_IDLE_MASK                                                                   0x00000200L
+#define VPEC_STATUS9__CE_IP3_WR_IDLE_MASK                                                                     0x00000400L
+#define VPEC_STATUS9__CE_IP3_SPLIT_RD_IDLE_MASK                                                               0x00000800L
+#define VPEC_STATUS9__CE_IP3_SPLIT_WR_IDLE_MASK                                                               0x00001000L
+#define VPEC_STATUS9__CE_IP3_RREQ_IDLE_MASK                                                                   0x00002000L
+#define VPEC_STATUS9__CE_IP3_OUT_IDLE_MASK                                                                    0x00004000L
+#define VPEC_STATUS9__CE_IP3_IN_IDLE_MASK                                                                     0x00008000L
+#define VPEC_STATUS9__CE_IP3_DST_IDLE_MASK                                                                    0x00010000L
+#define VPEC_STATUS9__CE_IP3_CMD_IDLE_MASK                                                                    0x00020000L
+#define VPEC_STATUS9__CE_IP2_AFIFO_FULL_MASK                                                                  0x00040000L
+#define VPEC_STATUS9__CE_IP2_CMD_INFO_FULL_MASK                                                               0x00080000L
+#define VPEC_STATUS9__CE_IP2_CMD_INFO1_FULL_MASK                                                              0x00100000L
+#define VPEC_STATUS9__CE_IP3_AFIFO_FULL_MASK                                                                  0x00200000L
+#define VPEC_STATUS9__CE_IP3_CMD_INFO_FULL_MASK                                                               0x00400000L
+#define VPEC_STATUS9__CE_IP3_CMD_INFO1_FULL_MASK                                                              0x00800000L
+#define VPEC_STATUS9__CE_IP2_WR_STALL_MASK                                                                    0x01000000L
+#define VPEC_STATUS9__CE_IP3_WR_STALL_MASK                                                                    0x02000000L
+#define VPEC_STATUS9__CE_IP2_RD_STALL_MASK                                                                    0x04000000L
+#define VPEC_STATUS9__CE_IP3_RD_STALL_MASK                                                                    0x08000000L
+#define VPEC_STATUS9__RESERVED_31_28_MASK                                                                     0xF0000000L
+//VPEC_STATUS10
+#define VPEC_STATUS10__CE_OP0_WR_IDLE__SHIFT                                                                  0x0
+#define VPEC_STATUS10__CE_OP0_CMD_IDLE__SHIFT                                                                 0x1
+#define VPEC_STATUS10__CE_OP1_WR_IDLE__SHIFT                                                                  0x2
+#define VPEC_STATUS10__CE_OP1_CMD_IDLE__SHIFT                                                                 0x3
+#define VPEC_STATUS10__CE_OP2_WR_IDLE__SHIFT                                                                  0x4
+#define VPEC_STATUS10__CE_OP2_CMD_IDLE__SHIFT                                                                 0x5
+#define VPEC_STATUS10__CE_OP3_WR_IDLE__SHIFT                                                                  0x6
+#define VPEC_STATUS10__CE_OP3_CMD_IDLE__SHIFT                                                                 0x7
+#define VPEC_STATUS10__CE_OP4_WR_IDLE__SHIFT                                                                  0x8
+#define VPEC_STATUS10__CE_OP4_CMD_IDLE__SHIFT                                                                 0x9
+#define VPEC_STATUS10__CE_OP5_WR_IDLE__SHIFT                                                                  0xa
+#define VPEC_STATUS10__CE_OP5_CMD_IDLE__SHIFT                                                                 0xb
+#define VPEC_STATUS10__CE_OP6_WR_IDLE__SHIFT                                                                  0xc
+#define VPEC_STATUS10__CE_OP6_CMD_IDLE__SHIFT                                                                 0xd
+#define VPEC_STATUS10__CE_OP7_WR_IDLE__SHIFT                                                                  0xe
+#define VPEC_STATUS10__CE_OP7_CMD_IDLE__SHIFT                                                                 0xf
+#define VPEC_STATUS10__CE_OP8_WR_IDLE__SHIFT                                                                  0x10
+#define VPEC_STATUS10__CE_OP8_CMD_IDLE__SHIFT                                                                 0x11
+#define VPEC_STATUS10__CE_OP9_WR_IDLE__SHIFT                                                                  0x12
+#define VPEC_STATUS10__CE_OP9_CMD_IDLE__SHIFT                                                                 0x13
+#define VPEC_STATUS10__RESERVED_31_28__SHIFT                                                                  0x1c
+#define VPEC_STATUS10__CE_OP0_WR_IDLE_MASK                                                                    0x00000001L
+#define VPEC_STATUS10__CE_OP0_CMD_IDLE_MASK                                                                   0x00000002L
+#define VPEC_STATUS10__CE_OP1_WR_IDLE_MASK                                                                    0x00000004L
+#define VPEC_STATUS10__CE_OP1_CMD_IDLE_MASK                                                                   0x00000008L
+#define VPEC_STATUS10__CE_OP2_WR_IDLE_MASK                                                                    0x00000010L
+#define VPEC_STATUS10__CE_OP2_CMD_IDLE_MASK                                                                   0x00000020L
+#define VPEC_STATUS10__CE_OP3_WR_IDLE_MASK                                                                    0x00000040L
+#define VPEC_STATUS10__CE_OP3_CMD_IDLE_MASK                                                                   0x00000080L
+#define VPEC_STATUS10__CE_OP4_WR_IDLE_MASK                                                                    0x00000100L
+#define VPEC_STATUS10__CE_OP4_CMD_IDLE_MASK                                                                   0x00000200L
+#define VPEC_STATUS10__CE_OP5_WR_IDLE_MASK                                                                    0x00000400L
+#define VPEC_STATUS10__CE_OP5_CMD_IDLE_MASK                                                                   0x00000800L
+#define VPEC_STATUS10__CE_OP6_WR_IDLE_MASK                                                                    0x00001000L
+#define VPEC_STATUS10__CE_OP6_CMD_IDLE_MASK                                                                   0x00002000L
+#define VPEC_STATUS10__CE_OP7_WR_IDLE_MASK                                                                    0x00004000L
+#define VPEC_STATUS10__CE_OP7_CMD_IDLE_MASK                                                                   0x00008000L
+#define VPEC_STATUS10__CE_OP8_WR_IDLE_MASK                                                                    0x00010000L
+#define VPEC_STATUS10__CE_OP8_CMD_IDLE_MASK                                                                   0x00020000L
+#define VPEC_STATUS10__CE_OP9_WR_IDLE_MASK                                                                    0x00040000L
+#define VPEC_STATUS10__CE_OP9_CMD_IDLE_MASK                                                                   0x00080000L
+#define VPEC_STATUS10__RESERVED_31_28_MASK                                                                    0xF0000000L
+//VPEC_STATUS_DCC
+#define VPEC_STATUS_DCC__CE_IP0_MRQ_IDLE__SHIFT                                                               0x0
+#define VPEC_STATUS_DCC__CE_IP0_DCCP_IDLE__SHIFT                                                              0x1
+#define VPEC_STATUS_DCC__CE_IP0_DCC_RET_IDLE__SHIFT                                                           0x2
+#define VPEC_STATUS_DCC__CE_IP1_MRQ_IDLE__SHIFT                                                               0x3
+#define VPEC_STATUS_DCC__CE_IP1_DCCP_IDLE__SHIFT                                                              0x4
+#define VPEC_STATUS_DCC__CE_IP1_DCC_RET_IDLE__SHIFT                                                           0x5
+#define VPEC_STATUS_DCC__CE_IP2_MRQ_IDLE__SHIFT                                                               0x6
+#define VPEC_STATUS_DCC__CE_IP2_DCCP_IDLE__SHIFT                                                              0x7
+#define VPEC_STATUS_DCC__CE_IP2_DCC_RET_IDLE__SHIFT                                                           0x8
+#define VPEC_STATUS_DCC__CE_IP3_MRQ_IDLE__SHIFT                                                               0x9
+#define VPEC_STATUS_DCC__CE_IP3_DCCP_IDLE__SHIFT                                                              0xa
+#define VPEC_STATUS_DCC__CE_IP3_DCC_RET_IDLE__SHIFT                                                           0xb
+#define VPEC_STATUS_DCC__CE_IP4_MRQ_IDLE__SHIFT                                                               0xc
+#define VPEC_STATUS_DCC__CE_IP4_DCCP_IDLE__SHIFT                                                              0xd
+#define VPEC_STATUS_DCC__CE_IP4_DCC_RET_IDLE__SHIFT                                                           0xe
+#define VPEC_STATUS_DCC__CE_IP5_MRQ_IDLE__SHIFT                                                               0xf
+#define VPEC_STATUS_DCC__CE_IP5_DCCP_IDLE__SHIFT                                                              0x10
+#define VPEC_STATUS_DCC__CE_IP5_DCC_RET_IDLE__SHIFT                                                           0x11
+#define VPEC_STATUS_DCC__RESERVED_31_18__SHIFT                                                                0x12
+#define VPEC_STATUS_DCC__CE_IP0_MRQ_IDLE_MASK                                                                 0x00000001L
+#define VPEC_STATUS_DCC__CE_IP0_DCCP_IDLE_MASK                                                                0x00000002L
+#define VPEC_STATUS_DCC__CE_IP0_DCC_RET_IDLE_MASK                                                             0x00000004L
+#define VPEC_STATUS_DCC__CE_IP1_MRQ_IDLE_MASK                                                                 0x00000008L
+#define VPEC_STATUS_DCC__CE_IP1_DCCP_IDLE_MASK                                                                0x00000010L
+#define VPEC_STATUS_DCC__CE_IP1_DCC_RET_IDLE_MASK                                                             0x00000020L
+#define VPEC_STATUS_DCC__CE_IP2_MRQ_IDLE_MASK                                                                 0x00000040L
+#define VPEC_STATUS_DCC__CE_IP2_DCCP_IDLE_MASK                                                                0x00000080L
+#define VPEC_STATUS_DCC__CE_IP2_DCC_RET_IDLE_MASK                                                             0x00000100L
+#define VPEC_STATUS_DCC__CE_IP3_MRQ_IDLE_MASK                                                                 0x00000200L
+#define VPEC_STATUS_DCC__CE_IP3_DCCP_IDLE_MASK                                                                0x00000400L
+#define VPEC_STATUS_DCC__CE_IP3_DCC_RET_IDLE_MASK                                                             0x00000800L
+#define VPEC_STATUS_DCC__CE_IP4_MRQ_IDLE_MASK                                                                 0x00001000L
+#define VPEC_STATUS_DCC__CE_IP4_DCCP_IDLE_MASK                                                                0x00002000L
+#define VPEC_STATUS_DCC__CE_IP4_DCC_RET_IDLE_MASK                                                             0x00004000L
+#define VPEC_STATUS_DCC__CE_IP5_MRQ_IDLE_MASK                                                                 0x00008000L
+#define VPEC_STATUS_DCC__CE_IP5_DCCP_IDLE_MASK                                                                0x00010000L
+#define VPEC_STATUS_DCC__CE_IP5_DCC_RET_IDLE_MASK                                                             0x00020000L
+#define VPEC_STATUS_DCC__RESERVED_31_18_MASK                                                                  0xFFFC0000L
+//VPEC_STATUS11
+#define VPEC_STATUS11__CE_IP4_WREQ_IDLE__SHIFT                                                                0x0
+#define VPEC_STATUS11__CE_IP4_WR_IDLE__SHIFT                                                                  0x1
+#define VPEC_STATUS11__CE_IP4_SPLIT_RD_IDLE__SHIFT                                                            0x2
+#define VPEC_STATUS11__CE_IP4_SPLIT_WR_IDLE__SHIFT                                                            0x3
+#define VPEC_STATUS11__CE_IP4_RREQ_IDLE__SHIFT                                                                0x4
+#define VPEC_STATUS11__CE_IP4_OUT_IDLE__SHIFT                                                                 0x5
+#define VPEC_STATUS11__CE_IP4_IN_IDLE__SHIFT                                                                  0x6
+#define VPEC_STATUS11__CE_IP4_DST_IDLE__SHIFT                                                                 0x7
+#define VPEC_STATUS11__CE_IP4_CMD_IDLE__SHIFT                                                                 0x8
+#define VPEC_STATUS11__CE_IP5_WREQ_IDLE__SHIFT                                                                0x9
+#define VPEC_STATUS11__CE_IP5_WR_IDLE__SHIFT                                                                  0xa
+#define VPEC_STATUS11__CE_IP5_SPLIT_RD_IDLE__SHIFT                                                            0xb
+#define VPEC_STATUS11__CE_IP5_SPLIT_WR_IDLE__SHIFT                                                            0xc
+#define VPEC_STATUS11__CE_IP5_RREQ_IDLE__SHIFT                                                                0xd
+#define VPEC_STATUS11__CE_IP5_OUT_IDLE__SHIFT                                                                 0xe
+#define VPEC_STATUS11__CE_IP5_IN_IDLE__SHIFT                                                                  0xf
+#define VPEC_STATUS11__CE_IP5_DST_IDLE__SHIFT                                                                 0x10
+#define VPEC_STATUS11__CE_IP5_CMD_IDLE__SHIFT                                                                 0x11
+#define VPEC_STATUS11__CE_IP4_AFIFO_FULL__SHIFT                                                               0x12
+#define VPEC_STATUS11__CE_IP4_CMD_INFO_FULL__SHIFT                                                            0x13
+#define VPEC_STATUS11__CE_IP4_CMD_INFO1_FULL__SHIFT                                                           0x14
+#define VPEC_STATUS11__CE_IP5_AFIFO_FULL__SHIFT                                                               0x15
+#define VPEC_STATUS11__CE_IP5_CMD_INFO_FULL__SHIFT                                                            0x16
+#define VPEC_STATUS11__CE_IP5_CMD_INFO1_FULL__SHIFT                                                           0x17
+#define VPEC_STATUS11__CE_IP4_WR_STALL__SHIFT                                                                 0x18
+#define VPEC_STATUS11__CE_IP5_WR_STALL__SHIFT                                                                 0x19
+#define VPEC_STATUS11__CE_IP4_RD_STALL__SHIFT                                                                 0x1a
+#define VPEC_STATUS11__CE_IP5_RD_STALL__SHIFT                                                                 0x1b
+#define VPEC_STATUS11__RESERVED_31_28__SHIFT                                                                  0x1c
+#define VPEC_STATUS11__CE_IP4_WREQ_IDLE_MASK                                                                  0x00000001L
+#define VPEC_STATUS11__CE_IP4_WR_IDLE_MASK                                                                    0x00000002L
+#define VPEC_STATUS11__CE_IP4_SPLIT_RD_IDLE_MASK                                                              0x00000004L
+#define VPEC_STATUS11__CE_IP4_SPLIT_WR_IDLE_MASK                                                              0x00000008L
+#define VPEC_STATUS11__CE_IP4_RREQ_IDLE_MASK                                                                  0x00000010L
+#define VPEC_STATUS11__CE_IP4_OUT_IDLE_MASK                                                                   0x00000020L
+#define VPEC_STATUS11__CE_IP4_IN_IDLE_MASK                                                                    0x00000040L
+#define VPEC_STATUS11__CE_IP4_DST_IDLE_MASK                                                                   0x00000080L
+#define VPEC_STATUS11__CE_IP4_CMD_IDLE_MASK                                                                   0x00000100L
+#define VPEC_STATUS11__CE_IP5_WREQ_IDLE_MASK                                                                  0x00000200L
+#define VPEC_STATUS11__CE_IP5_WR_IDLE_MASK                                                                    0x00000400L
+#define VPEC_STATUS11__CE_IP5_SPLIT_RD_IDLE_MASK                                                              0x00000800L
+#define VPEC_STATUS11__CE_IP5_SPLIT_WR_IDLE_MASK                                                              0x00001000L
+#define VPEC_STATUS11__CE_IP5_RREQ_IDLE_MASK                                                                  0x00002000L
+#define VPEC_STATUS11__CE_IP5_OUT_IDLE_MASK                                                                   0x00004000L
+#define VPEC_STATUS11__CE_IP5_IN_IDLE_MASK                                                                    0x00008000L
+#define VPEC_STATUS11__CE_IP5_DST_IDLE_MASK                                                                   0x00010000L
+#define VPEC_STATUS11__CE_IP5_CMD_IDLE_MASK                                                                   0x00020000L
+#define VPEC_STATUS11__CE_IP4_AFIFO_FULL_MASK                                                                 0x00040000L
+#define VPEC_STATUS11__CE_IP4_CMD_INFO_FULL_MASK                                                              0x00080000L
+#define VPEC_STATUS11__CE_IP4_CMD_INFO1_FULL_MASK                                                             0x00100000L
+#define VPEC_STATUS11__CE_IP5_AFIFO_FULL_MASK                                                                 0x00200000L
+#define VPEC_STATUS11__CE_IP5_CMD_INFO_FULL_MASK                                                              0x00400000L
+#define VPEC_STATUS11__CE_IP5_CMD_INFO1_FULL_MASK                                                             0x00800000L
+#define VPEC_STATUS11__CE_IP4_WR_STALL_MASK                                                                   0x01000000L
+#define VPEC_STATUS11__CE_IP5_WR_STALL_MASK                                                                   0x02000000L
+#define VPEC_STATUS11__CE_IP4_RD_STALL_MASK                                                                   0x04000000L
+#define VPEC_STATUS11__CE_IP5_RD_STALL_MASK                                                                   0x08000000L
+#define VPEC_STATUS11__RESERVED_31_28_MASK                                                                    0xF0000000L
+//VPEC_INST
+#define VPEC_INST__ID__SHIFT                                                                                  0x0
+#define VPEC_INST__RESERVED__SHIFT                                                                            0x3
+#define VPEC_INST__ID_MASK                                                                                    0x00000007L
+#define VPEC_INST__RESERVED_MASK                                                                              0xFFFFFFF8L
+//VPEC_QUEUE_STATUS0
+#define VPEC_QUEUE_STATUS0__QUEUE0_STATUS__SHIFT                                                              0x0
+#define VPEC_QUEUE_STATUS0__QUEUE1_STATUS__SHIFT                                                              0x4
+#define VPEC_QUEUE_STATUS0__QUEUE2_STATUS__SHIFT                                                              0x8
+#define VPEC_QUEUE_STATUS0__QUEUE3_STATUS__SHIFT                                                              0xc
+#define VPEC_QUEUE_STATUS0__QUEUE4_STATUS__SHIFT                                                              0x10
+#define VPEC_QUEUE_STATUS0__QUEUE5_STATUS__SHIFT                                                              0x14
+#define VPEC_QUEUE_STATUS0__QUEUE6_STATUS__SHIFT                                                              0x18
+#define VPEC_QUEUE_STATUS0__QUEUE7_STATUS__SHIFT                                                              0x1c
+#define VPEC_QUEUE_STATUS0__QUEUE0_STATUS_MASK                                                                0x0000000FL
+#define VPEC_QUEUE_STATUS0__QUEUE1_STATUS_MASK                                                                0x000000F0L
+#define VPEC_QUEUE_STATUS0__QUEUE2_STATUS_MASK                                                                0x00000F00L
+#define VPEC_QUEUE_STATUS0__QUEUE3_STATUS_MASK                                                                0x0000F000L
+#define VPEC_QUEUE_STATUS0__QUEUE4_STATUS_MASK                                                                0x000F0000L
+#define VPEC_QUEUE_STATUS0__QUEUE5_STATUS_MASK                                                                0x00F00000L
+#define VPEC_QUEUE_STATUS0__QUEUE6_STATUS_MASK                                                                0x0F000000L
+#define VPEC_QUEUE_STATUS0__QUEUE7_STATUS_MASK                                                                0xF0000000L
+//VPEC_QUEUE_HANG_STATUS
+#define VPEC_QUEUE_HANG_STATUS__F30T0_HANG__SHIFT                                                             0x0
+#define VPEC_QUEUE_HANG_STATUS__CE_HANG__SHIFT                                                                0x1
+#define VPEC_QUEUE_HANG_STATUS__EOF_MISMATCH__SHIFT                                                           0x2
+#define VPEC_QUEUE_HANG_STATUS__INVALID_PKT_FIELD__SHIFT                                                      0x3
+#define VPEC_QUEUE_HANG_STATUS__INVALID_VPEP_CONFIG_ADDR__SHIFT                                               0x4
+#define VPEC_QUEUE_HANG_STATUS__F32_ACCESS_OFF_VPDPP1__SHIFT                                                  0x5
+#define VPEC_QUEUE_HANG_STATUS__RSMU_ACCESS_OFF_VPDPP1__SHIFT                                                 0x6
+#define VPEC_QUEUE_HANG_STATUS__EOH_MISMATCH__SHIFT                                                           0x7
+#define VPEC_QUEUE_HANG_STATUS__F30T0_HANG_MASK                                                               0x00000001L
+#define VPEC_QUEUE_HANG_STATUS__CE_HANG_MASK                                                                  0x00000002L
+#define VPEC_QUEUE_HANG_STATUS__EOF_MISMATCH_MASK                                                             0x00000004L
+#define VPEC_QUEUE_HANG_STATUS__INVALID_PKT_FIELD_MASK                                                        0x00000008L
+#define VPEC_QUEUE_HANG_STATUS__INVALID_VPEP_CONFIG_ADDR_MASK                                                 0x00000010L
+#define VPEC_QUEUE_HANG_STATUS__F32_ACCESS_OFF_VPDPP1_MASK                                                    0x00000020L
+#define VPEC_QUEUE_HANG_STATUS__RSMU_ACCESS_OFF_VPDPP1_MASK                                                   0x00000040L
+#define VPEC_QUEUE_HANG_STATUS__EOH_MISMATCH_MASK                                                             0x00000080L
+//VPEC_DPM_IDLE_TIME
+#define VPEC_DPM_IDLE_TIME__VALUE__SHIFT                                                                      0x0
+#define VPEC_DPM_IDLE_TIME__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_DPM_BUSY_TIME
+#define VPEC_DPM_BUSY_TIME__VALUE__SHIFT                                                                      0x0
+#define VPEC_DPM_BUSY_TIME__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_DPM_IDLE_START_LO
+#define VPEC_DPM_IDLE_START_LO__VALUE__SHIFT                                                                  0x0
+#define VPEC_DPM_IDLE_START_LO__VALUE_MASK                                                                    0xFFFFFFFFL
+//VPEC_DPM_IDLE_START_HI
+#define VPEC_DPM_IDLE_START_HI__VALUE__SHIFT                                                                  0x0
+#define VPEC_DPM_IDLE_START_HI__VALUE_MASK                                                                    0xFFFFFFFFL
+//VPEC_DPM_BUSY_START_LO
+#define VPEC_DPM_BUSY_START_LO__VALUE__SHIFT                                                                  0x0
+#define VPEC_DPM_BUSY_START_LO__VALUE_MASK                                                                    0xFFFFFFFFL
+//VPEC_DPM_BUSY_START_HI
+#define VPEC_DPM_BUSY_START_HI__VALUE__SHIFT                                                                  0x0
+#define VPEC_DPM_BUSY_START_HI__VALUE_MASK                                                                    0xFFFFFFFFL
+//VPEC_DPM_LAST_REQ_TIMESTAMP
+#define VPEC_DPM_LAST_REQ_TIMESTAMP__VALUE__SHIFT                                                             0x0
+#define VPEC_DPM_LAST_REQ_TIMESTAMP__VALUE_MASK                                                               0xFFFFFFFFL
+//VPEC_DPM_NEW_JOB_DUMMY3
+#define VPEC_DPM_NEW_JOB_DUMMY3__VALUE__SHIFT                                                                 0x0
+#define VPEC_DPM_NEW_JOB_DUMMY3__VALUE_MASK                                                                   0xFFFFFFFFL
+//VPEC_DPM_STATE
+#define VPEC_DPM_STATE__VALUE__SHIFT                                                                          0x0
+#define VPEC_DPM_STATE__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_DPM0_FREQ
+#define VPEC_DPM0_FREQ__VALUE__SHIFT                                                                          0x0
+#define VPEC_DPM0_FREQ__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_DPM1_FREQ
+#define VPEC_DPM1_FREQ__VALUE__SHIFT                                                                          0x0
+#define VPEC_DPM1_FREQ__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_DPM2_FREQ
+#define VPEC_DPM2_FREQ__VALUE__SHIFT                                                                          0x0
+#define VPEC_DPM2_FREQ__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_DPM3_FREQ
+#define VPEC_DPM3_FREQ__VALUE__SHIFT                                                                          0x0
+#define VPEC_DPM3_FREQ__VALUE_MASK                                                                            0xFFFFFFFFL
+//VPEC_DPM_THRESHOLD_SKIP
+#define VPEC_DPM_THRESHOLD_SKIP__VALUE__SHIFT                                                                 0x0
+#define VPEC_DPM_THRESHOLD_SKIP__VALUE_MASK                                                                   0xFFFFFFFFL
+//VPEC_DPM_THRESHOLD_BUSY_OVERFLOW
+#define VPEC_DPM_THRESHOLD_BUSY_OVERFLOW__VALUE__SHIFT                                                        0x0
+#define VPEC_DPM_THRESHOLD_BUSY_OVERFLOW__VALUE_MASK                                                          0xFFFFFFFFL
+//VPEC_DPM_CALC_BUSY_IN_POSTPROCESS
+#define VPEC_DPM_CALC_BUSY_IN_POSTPROCESS__VALUE__SHIFT                                                       0x0
+#define VPEC_DPM_CALC_BUSY_IN_POSTPROCESS__VALUE_MASK                                                         0xFFFFFFFFL
+//VPEC_DPM_IN_CHECKIDLE_LOOP
+#define VPEC_DPM_IN_CHECKIDLE_LOOP__VALUE__SHIFT                                                              0x0
+#define VPEC_DPM_IN_CHECKIDLE_LOOP__VALUE_MASK                                                                0xFFFFFFFFL
+//VPEC_DPM_THRESHOLD_IDLE_OVERFLOW
+#define VPEC_DPM_THRESHOLD_IDLE_OVERFLOW__VALUE__SHIFT                                                        0x0
+#define VPEC_DPM_THRESHOLD_IDLE_OVERFLOW__VALUE_MASK                                                          0xFFFFFFFFL
+//VPEC_DPM_BUSY_CLAMP_COUNT
+#define VPEC_DPM_BUSY_CLAMP_COUNT__VALUE__SHIFT                                                               0x0
+#define VPEC_DPM_BUSY_CLAMP_COUNT__VALUE_MASK                                                                 0xFFFFFFFFL
+//VPEC_DPM_IDLE_CLAMP_COUNT
+#define VPEC_DPM_IDLE_CLAMP_COUNT__VALUE__SHIFT                                                               0x0
+#define VPEC_DPM_IDLE_CLAMP_COUNT__VALUE_MASK                                                                 0xFFFFFFFFL
+//VPEC_PG_CNTL
+#define VPEC_PG_CNTL__PG_EN__SHIFT                                                                            0x0
+#define VPEC_PG_CNTL__PG_HYSTERESIS__SHIFT                                                                    0x1
+#define VPEC_PG_CNTL__PG1_EN__SHIFT                                                                           0x8
+#define VPEC_PG_CNTL__PG1_HYSTERESIS__SHIFT                                                                   0x9
+#define VPEC_PG_CNTL__ZSTATES_ENABLE__SHIFT                                                                   0x10
+#define VPEC_PG_CNTL__ZSTATES_HYSTERESIS__SHIFT                                                               0x11
+#define VPEC_PG_CNTL__FENCE_HYSTERESIS__SHIFT                                                                 0x18
+#define VPEC_PG_CNTL__CHECK_RSMU_UPON_POWER_UP__SHIFT                                                         0x1c
+#define VPEC_PG_CNTL__PG_EN_MASK                                                                              0x00000001L
+#define VPEC_PG_CNTL__PG_HYSTERESIS_MASK                                                                      0x0000003EL
+#define VPEC_PG_CNTL__PG1_EN_MASK                                                                             0x00000100L
+#define VPEC_PG_CNTL__PG1_HYSTERESIS_MASK                                                                     0x00003E00L
+#define VPEC_PG_CNTL__ZSTATES_ENABLE_MASK                                                                     0x00010000L
+#define VPEC_PG_CNTL__ZSTATES_HYSTERESIS_MASK                                                                 0x003E0000L
+#define VPEC_PG_CNTL__FENCE_HYSTERESIS_MASK                                                                   0x0F000000L
+#define VPEC_PG_CNTL__CHECK_RSMU_UPON_POWER_UP_MASK                                                           0x10000000L
+//VPEC_PG_STATUS
+#define VPEC_PG_STATUS__PG_STATUS__SHIFT                                                                      0x0
+#define VPEC_PG_STATUS__PG1_STATUS__SHIFT                                                                     0x2
+#define VPEC_PG_STATUS__PG_STATUS_MASK                                                                        0x00000003L
+#define VPEC_PG_STATUS__PG1_STATUS_MASK                                                                       0x0000000CL
+//VPEC_CLOCK_GATING_STATUS
+#define VPEC_CLOCK_GATING_STATUS__DYN_CLK_GATE_STATUS__SHIFT                                                  0x0
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE0_CLK_GATE_STATUS__SHIFT                                             0x2
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE1_CLK_GATE_STATUS__SHIFT                                             0x3
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE2_CLK_GATE_STATUS__SHIFT                                             0x4
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE3_CLK_GATE_STATUS__SHIFT                                             0x5
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE0_CLK_GATE_STATUS__SHIFT                                             0x6
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE1_CLK_GATE_STATUS__SHIFT                                             0x7
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE2_CLK_GATE_STATUS__SHIFT                                             0x8
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE3_CLK_GATE_STATUS__SHIFT                                             0x9
+#define VPEC_CLOCK_GATING_STATUS__REG_CLK_GATE_STATUS__SHIFT                                                  0xa
+#define VPEC_CLOCK_GATING_STATUS__F32_CLK_GATE_STATUS__SHIFT                                                  0xb
+#define VPEC_CLOCK_GATING_STATUS__USRAM_CLK_GATE_STATUS__SHIFT                                                0xc
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE4_CLK_GATE_STATUS__SHIFT                                             0xd
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE5_CLK_GATE_STATUS__SHIFT                                             0xe
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE6_CLK_GATE_STATUS__SHIFT                                             0xf
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE7_CLK_GATE_STATUS__SHIFT                                             0x10
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE4_CLK_GATE_STATUS__SHIFT                                             0x11
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE5_CLK_GATE_STATUS__SHIFT                                             0x12
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE8_CLK_GATE_STATUS__SHIFT                                             0x13
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE9_CLK_GATE_STATUS__SHIFT                                             0x14
+#define VPEC_CLOCK_GATING_STATUS__DYN_CLK_GATE_STATUS_MASK                                                    0x00000001L
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE0_CLK_GATE_STATUS_MASK                                               0x00000004L
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE1_CLK_GATE_STATUS_MASK                                               0x00000008L
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE2_CLK_GATE_STATUS_MASK                                               0x00000010L
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE3_CLK_GATE_STATUS_MASK                                               0x00000020L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE0_CLK_GATE_STATUS_MASK                                               0x00000040L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE1_CLK_GATE_STATUS_MASK                                               0x00000080L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE2_CLK_GATE_STATUS_MASK                                               0x00000100L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE3_CLK_GATE_STATUS_MASK                                               0x00000200L
+#define VPEC_CLOCK_GATING_STATUS__REG_CLK_GATE_STATUS_MASK                                                    0x00000400L
+#define VPEC_CLOCK_GATING_STATUS__F32_CLK_GATE_STATUS_MASK                                                    0x00000800L
+#define VPEC_CLOCK_GATING_STATUS__USRAM_CLK_GATE_STATUS_MASK                                                  0x00001000L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE4_CLK_GATE_STATUS_MASK                                               0x00002000L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE5_CLK_GATE_STATUS_MASK                                               0x00004000L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE6_CLK_GATE_STATUS_MASK                                               0x00008000L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE7_CLK_GATE_STATUS_MASK                                               0x00010000L
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE4_CLK_GATE_STATUS_MASK                                               0x00020000L
+#define VPEC_CLOCK_GATING_STATUS__IP_PIPE5_CLK_GATE_STATUS_MASK                                               0x00040000L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE8_CLK_GATE_STATUS_MASK                                               0x00080000L
+#define VPEC_CLOCK_GATING_STATUS__OP_PIPE9_CLK_GATE_STATUS_MASK                                               0x00100000L
+//VPEC_QUEUE0_RB_CNTL
+#define VPEC_QUEUE0_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE0_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define VPEC_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                          0x8
+#define VPEC_QUEUE0_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                            0x9
+#define VPEC_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                     0xa
+#define VPEC_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                      0xb
+#define VPEC_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                     0xc
+#define VPEC_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                0xd
+#define VPEC_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                      0x10
+#define VPEC_QUEUE0_RB_CNTL__RB_PRIV__SHIFT                                                                   0x17
+#define VPEC_QUEUE0_RB_CNTL__RB_VMID__SHIFT                                                                   0x18
+#define VPEC_QUEUE0_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE0_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define VPEC_QUEUE0_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                            0x00000100L
+#define VPEC_QUEUE0_RB_CNTL__RB_SWAP_ENABLE_MASK                                                              0x00000200L
+#define VPEC_QUEUE0_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                       0x00000400L
+#define VPEC_QUEUE0_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                        0x00000800L
+#define VPEC_QUEUE0_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                       0x00001000L
+#define VPEC_QUEUE0_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                  0x00002000L
+#define VPEC_QUEUE0_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                        0x001F0000L
+#define VPEC_QUEUE0_RB_CNTL__RB_PRIV_MASK                                                                     0x00800000L
+#define VPEC_QUEUE0_RB_CNTL__RB_VMID_MASK                                                                     0x0F000000L
+//VPEC_QUEUE0_SCHEDULE_CNTL
+#define VPEC_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                           0x0
+#define VPEC_QUEUE0_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                          0x2
+#define VPEC_QUEUE0_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                            0x6
+#define VPEC_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                     0x8
+#define VPEC_QUEUE0_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                             0x00000003L
+#define VPEC_QUEUE0_SCHEDULE_CNTL__PROCESS_ID_MASK                                                            0x0000001CL
+#define VPEC_QUEUE0_SCHEDULE_CNTL__LOCAL_ID_MASK                                                              0x000000C0L
+#define VPEC_QUEUE0_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                       0x0000FF00L
+//VPEC_QUEUE0_RB_BASE
+#define VPEC_QUEUE0_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define VPEC_QUEUE0_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE0_RB_BASE_HI
+#define VPEC_QUEUE0_RB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE0_RB_BASE_HI__ADDR_MASK                                                                     0x00FFFFFFL
+//VPEC_QUEUE0_RB_RPTR
+#define VPEC_QUEUE0_RB_RPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE0_RB_RPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE0_RB_RPTR_HI
+#define VPEC_QUEUE0_RB_RPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE0_RB_RPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE0_RB_WPTR
+#define VPEC_QUEUE0_RB_WPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE0_RB_WPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE0_RB_WPTR_HI
+#define VPEC_QUEUE0_RB_WPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE0_RB_WPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE0_RB_RPTR_ADDR_HI
+#define VPEC_QUEUE0_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE0_RB_RPTR_ADDR_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE0_RB_RPTR_ADDR_LO
+#define VPEC_QUEUE0_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define VPEC_QUEUE0_RB_RPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//VPEC_QUEUE0_RB_AQL_CNTL
+#define VPEC_QUEUE0_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                       0x1
+#define VPEC_QUEUE0_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                           0x8
+#define VPEC_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                 0x10
+#define VPEC_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                           0x11
+#define VPEC_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                        0x12
+#define VPEC_QUEUE0_RB_AQL_CNTL__AQL_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE0_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                         0x000000FEL
+#define VPEC_QUEUE0_RB_AQL_CNTL__PACKET_STEP_MASK                                                             0x0000FF00L
+#define VPEC_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                   0x00010000L
+#define VPEC_QUEUE0_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                             0x00020000L
+#define VPEC_QUEUE0_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                          0x00040000L
+//VPEC_QUEUE0_MINOR_PTR_UPDATE
+#define VPEC_QUEUE0_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                           0x0
+#define VPEC_QUEUE0_MINOR_PTR_UPDATE__ENABLE_MASK                                                             0x00000001L
+//VPEC_QUEUE0_CD_INFO
+#define VPEC_QUEUE0_CD_INFO__CD_INFO__SHIFT                                                                   0x0
+#define VPEC_QUEUE0_CD_INFO__CD_INFO_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE0_RB_PREEMPT
+#define VPEC_QUEUE0_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                            0x0
+#define VPEC_QUEUE0_RB_PREEMPT__PREEMPT_REQ_MASK                                                              0x00000001L
+//VPEC_QUEUE0_SKIP_CNTL
+#define VPEC_QUEUE0_SKIP_CNTL__SKIP_COUNT__SHIFT                                                              0x0
+#define VPEC_QUEUE0_SKIP_CNTL__SKIP_COUNT_MASK                                                                0x000FFFFFL
+//VPEC_QUEUE0_DOORBELL
+#define VPEC_QUEUE0_DOORBELL__ENABLE__SHIFT                                                                   0x1c
+#define VPEC_QUEUE0_DOORBELL__CAPTURED__SHIFT                                                                 0x1e
+#define VPEC_QUEUE0_DOORBELL__ENABLE_MASK                                                                     0x10000000L
+#define VPEC_QUEUE0_DOORBELL__CAPTURED_MASK                                                                   0x40000000L
+//VPEC_QUEUE0_DOORBELL_OFFSET
+#define VPEC_QUEUE0_DOORBELL_OFFSET__OFFSET__SHIFT                                                            0x2
+#define VPEC_QUEUE0_DOORBELL_OFFSET__OFFSET_MASK                                                              0x0FFFFFFCL
+//VPEC_QUEUE0_DUMMY0
+#define VPEC_QUEUE0_DUMMY0__DUMMY__SHIFT                                                                      0x0
+#define VPEC_QUEUE0_DUMMY0__DUMMY_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE0_DUMMY1
+#define VPEC_QUEUE0_DUMMY1__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE0_DUMMY1__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE0_DUMMY2
+#define VPEC_QUEUE0_DUMMY2__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE0_DUMMY2__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE0_DUMMY3
+#define VPEC_QUEUE0_DUMMY3__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE0_DUMMY3__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE0_DUMMY4
+#define VPEC_QUEUE0_DUMMY4__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE0_DUMMY4__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE0_IB_CNTL
+#define VPEC_QUEUE0_IB_CNTL__IB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                          0x8
+#define VPEC_QUEUE0_IB_CNTL__CMD_VMID__SHIFT                                                                  0x10
+#define VPEC_QUEUE0_IB_CNTL__IB_PRIV__SHIFT                                                                   0x1f
+#define VPEC_QUEUE0_IB_CNTL__IB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE0_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                            0x00000100L
+#define VPEC_QUEUE0_IB_CNTL__CMD_VMID_MASK                                                                    0x000F0000L
+#define VPEC_QUEUE0_IB_CNTL__IB_PRIV_MASK                                                                     0x80000000L
+//VPEC_QUEUE0_IB_RPTR
+#define VPEC_QUEUE0_IB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define VPEC_QUEUE0_IB_RPTR__OFFSET_MASK                                                                      0x003FFFFCL
+//VPEC_QUEUE0_IB_OFFSET
+#define VPEC_QUEUE0_IB_OFFSET__OFFSET__SHIFT                                                                  0x2
+#define VPEC_QUEUE0_IB_OFFSET__OFFSET_MASK                                                                    0x003FFFFCL
+//VPEC_QUEUE0_IB_BASE_LO
+#define VPEC_QUEUE0_IB_BASE_LO__ADDR__SHIFT                                                                   0x5
+#define VPEC_QUEUE0_IB_BASE_LO__ADDR_MASK                                                                     0xFFFFFFE0L
+//VPEC_QUEUE0_IB_BASE_HI
+#define VPEC_QUEUE0_IB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE0_IB_BASE_HI__ADDR_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE0_IB_SIZE
+#define VPEC_QUEUE0_IB_SIZE__SIZE__SHIFT                                                                      0x0
+#define VPEC_QUEUE0_IB_SIZE__SIZE_MASK                                                                        0x000FFFFFL
+//VPEC_QUEUE0_CMDIB_CNTL
+#define VPEC_QUEUE0_CMDIB_CNTL__IB_ENABLE__SHIFT                                                              0x0
+#define VPEC_QUEUE0_CMDIB_CNTL__IB_SWAP_ENABLE__SHIFT                                                         0x4
+#define VPEC_QUEUE0_CMDIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                       0x8
+#define VPEC_QUEUE0_CMDIB_CNTL__CMD_VMID__SHIFT                                                               0x10
+#define VPEC_QUEUE0_CMDIB_CNTL__IB_PRIV__SHIFT                                                                0x1f
+#define VPEC_QUEUE0_CMDIB_CNTL__IB_ENABLE_MASK                                                                0x00000001L
+#define VPEC_QUEUE0_CMDIB_CNTL__IB_SWAP_ENABLE_MASK                                                           0x00000010L
+#define VPEC_QUEUE0_CMDIB_CNTL__SWITCH_INSIDE_IB_MASK                                                         0x00000100L
+#define VPEC_QUEUE0_CMDIB_CNTL__CMD_VMID_MASK                                                                 0x000F0000L
+#define VPEC_QUEUE0_CMDIB_CNTL__IB_PRIV_MASK                                                                  0x80000000L
+//VPEC_QUEUE0_CMDIB_RPTR
+#define VPEC_QUEUE0_CMDIB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define VPEC_QUEUE0_CMDIB_RPTR__OFFSET_MASK                                                                   0x003FFFFCL
+//VPEC_QUEUE0_CMDIB_OFFSET
+#define VPEC_QUEUE0_CMDIB_OFFSET__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE0_CMDIB_OFFSET__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE0_CMDIB_BASE_LO
+#define VPEC_QUEUE0_CMDIB_BASE_LO__ADDR__SHIFT                                                                0x5
+#define VPEC_QUEUE0_CMDIB_BASE_LO__ADDR_MASK                                                                  0xFFFFFFE0L
+//VPEC_QUEUE0_CMDIB_BASE_HI
+#define VPEC_QUEUE0_CMDIB_BASE_HI__ADDR__SHIFT                                                                0x0
+#define VPEC_QUEUE0_CMDIB_BASE_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//VPEC_QUEUE0_CMDIB_SIZE
+#define VPEC_QUEUE0_CMDIB_SIZE__SIZE__SHIFT                                                                   0x0
+#define VPEC_QUEUE0_CMDIB_SIZE__SIZE_MASK                                                                     0x000FFFFFL
+//VPEC_QUEUE0_3DLUTIB_CNTL
+#define VPEC_QUEUE0_3DLUTIB_CNTL__IB_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE0_3DLUTIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                     0x8
+#define VPEC_QUEUE0_3DLUTIB_CNTL__CMD_VMID__SHIFT                                                             0x10
+#define VPEC_QUEUE0_3DLUTIB_CNTL__IB_PRIV__SHIFT                                                              0x1f
+#define VPEC_QUEUE0_3DLUTIB_CNTL__IB_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE0_3DLUTIB_CNTL__SWITCH_INSIDE_IB_MASK                                                       0x00000100L
+#define VPEC_QUEUE0_3DLUTIB_CNTL__CMD_VMID_MASK                                                               0x000F0000L
+#define VPEC_QUEUE0_3DLUTIB_CNTL__IB_PRIV_MASK                                                                0x80000000L
+//VPEC_QUEUE0_3DLUTIB_RPTR
+#define VPEC_QUEUE0_3DLUTIB_RPTR__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE0_3DLUTIB_RPTR__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE0_3DLUTIB_OFFSET
+#define VPEC_QUEUE0_3DLUTIB_OFFSET__OFFSET__SHIFT                                                             0x2
+#define VPEC_QUEUE0_3DLUTIB_OFFSET__OFFSET_MASK                                                               0x003FFFFCL
+//VPEC_QUEUE0_3DLUTIB_BASE_LO
+#define VPEC_QUEUE0_3DLUTIB_BASE_LO__ADDR__SHIFT                                                              0x5
+#define VPEC_QUEUE0_3DLUTIB_BASE_LO__ADDR_MASK                                                                0xFFFFFFE0L
+//VPEC_QUEUE0_3DLUTIB_BASE_HI
+#define VPEC_QUEUE0_3DLUTIB_BASE_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE0_3DLUTIB_BASE_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE0_3DLUTIB_SIZE
+#define VPEC_QUEUE0_3DLUTIB_SIZE__SIZE__SHIFT                                                                 0x0
+#define VPEC_QUEUE0_3DLUTIB_SIZE__SIZE_MASK                                                                   0x000FFFFFL
+//VPEC_QUEUE0_CSA_ADDR_LO
+#define VPEC_QUEUE0_CSA_ADDR_LO__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE0_CSA_ADDR_LO__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE0_CSA_ADDR_HI
+#define VPEC_QUEUE0_CSA_ADDR_HI__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE0_CSA_ADDR_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE0_CONTEXT_STATUS
+#define VPEC_QUEUE0_CONTEXT_STATUS__SELECTED__SHIFT                                                           0x0
+#define VPEC_QUEUE0_CONTEXT_STATUS__USE_IB__SHIFT                                                             0x1
+#define VPEC_QUEUE0_CONTEXT_STATUS__IDLE__SHIFT                                                               0x2
+#define VPEC_QUEUE0_CONTEXT_STATUS__EXPIRED__SHIFT                                                            0x3
+#define VPEC_QUEUE0_CONTEXT_STATUS__EXCEPTION__SHIFT                                                          0x4
+#define VPEC_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                         0x7
+#define VPEC_QUEUE0_CONTEXT_STATUS__USE_3DLUTIB__SHIFT                                                        0x8
+#define VPEC_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                    0xa
+#define VPEC_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                       0xb
+#define VPEC_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                0xc
+#define VPEC_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                             0x10
+#define VPEC_QUEUE0_CONTEXT_STATUS__SELECTED_MASK                                                             0x00000001L
+#define VPEC_QUEUE0_CONTEXT_STATUS__USE_IB_MASK                                                               0x00000002L
+#define VPEC_QUEUE0_CONTEXT_STATUS__IDLE_MASK                                                                 0x00000004L
+#define VPEC_QUEUE0_CONTEXT_STATUS__EXPIRED_MASK                                                              0x00000008L
+#define VPEC_QUEUE0_CONTEXT_STATUS__EXCEPTION_MASK                                                            0x00000070L
+#define VPEC_QUEUE0_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                           0x00000080L
+#define VPEC_QUEUE0_CONTEXT_STATUS__USE_3DLUTIB_MASK                                                          0x00000100L
+#define VPEC_QUEUE0_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                      0x00000400L
+#define VPEC_QUEUE0_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                         0x00000800L
+#define VPEC_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                  0x00001000L
+#define VPEC_QUEUE0_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                               0x00FF0000L
+//VPEC_QUEUE0_DOORBELL_LOG
+#define VPEC_QUEUE0_DOORBELL_LOG__BE_ERROR__SHIFT                                                             0x0
+#define VPEC_QUEUE0_DOORBELL_LOG__DATA__SHIFT                                                                 0x2
+#define VPEC_QUEUE0_DOORBELL_LOG__BE_ERROR_MASK                                                               0x00000001L
+#define VPEC_QUEUE0_DOORBELL_LOG__DATA_MASK                                                                   0xFFFFFFFCL
+//VPEC_QUEUE0_IB_SUB_REMAIN
+#define VPEC_QUEUE0_IB_SUB_REMAIN__SIZE__SHIFT                                                                0x0
+#define VPEC_QUEUE0_IB_SUB_REMAIN__SIZE_MASK                                                                  0x00003FFFL
+//VPEC_QUEUE0_PREEMPT
+#define VPEC_QUEUE0_PREEMPT__IB_PREEMPT__SHIFT                                                                0x0
+#define VPEC_QUEUE0_PREEMPT__IB_PREEMPT_MASK                                                                  0x00000001L
+//VPEC_QUEUE0_LOG0BUFFER_CFG
+#define VPEC_QUEUE0_LOG0BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE0_LOG0BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE0_LOG0BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE0_LOG0BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE0_LOG0BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE0_LOG0BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE0_LOG0BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE0_LOG0BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE0_LOG1BUFFER_CFG
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__PARTIAL_ENTRY__SHIFT                                                      0x1
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__PARTIAL_ENTRY_MASK                                                        0x00000002L
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE0_LOG1BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE1_RB_CNTL
+#define VPEC_QUEUE1_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE1_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define VPEC_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                          0x8
+#define VPEC_QUEUE1_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                            0x9
+#define VPEC_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                     0xa
+#define VPEC_QUEUE1_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                      0xb
+#define VPEC_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                     0xc
+#define VPEC_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                0xd
+#define VPEC_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                      0x10
+#define VPEC_QUEUE1_RB_CNTL__RB_PRIV__SHIFT                                                                   0x17
+#define VPEC_QUEUE1_RB_CNTL__RB_VMID__SHIFT                                                                   0x18
+#define VPEC_QUEUE1_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE1_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define VPEC_QUEUE1_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                            0x00000100L
+#define VPEC_QUEUE1_RB_CNTL__RB_SWAP_ENABLE_MASK                                                              0x00000200L
+#define VPEC_QUEUE1_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                       0x00000400L
+#define VPEC_QUEUE1_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                        0x00000800L
+#define VPEC_QUEUE1_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                       0x00001000L
+#define VPEC_QUEUE1_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                  0x00002000L
+#define VPEC_QUEUE1_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                        0x001F0000L
+#define VPEC_QUEUE1_RB_CNTL__RB_PRIV_MASK                                                                     0x00800000L
+#define VPEC_QUEUE1_RB_CNTL__RB_VMID_MASK                                                                     0x0F000000L
+//VPEC_QUEUE1_SCHEDULE_CNTL
+#define VPEC_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                           0x0
+#define VPEC_QUEUE1_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                          0x2
+#define VPEC_QUEUE1_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                            0x6
+#define VPEC_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                     0x8
+#define VPEC_QUEUE1_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                             0x00000003L
+#define VPEC_QUEUE1_SCHEDULE_CNTL__PROCESS_ID_MASK                                                            0x0000001CL
+#define VPEC_QUEUE1_SCHEDULE_CNTL__LOCAL_ID_MASK                                                              0x000000C0L
+#define VPEC_QUEUE1_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                       0x0000FF00L
+//VPEC_QUEUE1_RB_BASE
+#define VPEC_QUEUE1_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define VPEC_QUEUE1_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE1_RB_BASE_HI
+#define VPEC_QUEUE1_RB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE1_RB_BASE_HI__ADDR_MASK                                                                     0x00FFFFFFL
+//VPEC_QUEUE1_RB_RPTR
+#define VPEC_QUEUE1_RB_RPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE1_RB_RPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE1_RB_RPTR_HI
+#define VPEC_QUEUE1_RB_RPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE1_RB_RPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE1_RB_WPTR
+#define VPEC_QUEUE1_RB_WPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE1_RB_WPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE1_RB_WPTR_HI
+#define VPEC_QUEUE1_RB_WPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE1_RB_WPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE1_RB_RPTR_ADDR_HI
+#define VPEC_QUEUE1_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE1_RB_RPTR_ADDR_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE1_RB_RPTR_ADDR_LO
+#define VPEC_QUEUE1_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define VPEC_QUEUE1_RB_RPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//VPEC_QUEUE1_RB_AQL_CNTL
+#define VPEC_QUEUE1_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                       0x1
+#define VPEC_QUEUE1_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                           0x8
+#define VPEC_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                 0x10
+#define VPEC_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                           0x11
+#define VPEC_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                        0x12
+#define VPEC_QUEUE1_RB_AQL_CNTL__AQL_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE1_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                         0x000000FEL
+#define VPEC_QUEUE1_RB_AQL_CNTL__PACKET_STEP_MASK                                                             0x0000FF00L
+#define VPEC_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                   0x00010000L
+#define VPEC_QUEUE1_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                             0x00020000L
+#define VPEC_QUEUE1_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                          0x00040000L
+//VPEC_QUEUE1_MINOR_PTR_UPDATE
+#define VPEC_QUEUE1_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                           0x0
+#define VPEC_QUEUE1_MINOR_PTR_UPDATE__ENABLE_MASK                                                             0x00000001L
+//VPEC_QUEUE1_CD_INFO
+#define VPEC_QUEUE1_CD_INFO__CD_INFO__SHIFT                                                                   0x0
+#define VPEC_QUEUE1_CD_INFO__CD_INFO_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE1_RB_PREEMPT
+#define VPEC_QUEUE1_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                            0x0
+#define VPEC_QUEUE1_RB_PREEMPT__PREEMPT_REQ_MASK                                                              0x00000001L
+//VPEC_QUEUE1_SKIP_CNTL
+#define VPEC_QUEUE1_SKIP_CNTL__SKIP_COUNT__SHIFT                                                              0x0
+#define VPEC_QUEUE1_SKIP_CNTL__SKIP_COUNT_MASK                                                                0x000FFFFFL
+//VPEC_QUEUE1_DOORBELL
+#define VPEC_QUEUE1_DOORBELL__ENABLE__SHIFT                                                                   0x1c
+#define VPEC_QUEUE1_DOORBELL__CAPTURED__SHIFT                                                                 0x1e
+#define VPEC_QUEUE1_DOORBELL__ENABLE_MASK                                                                     0x10000000L
+#define VPEC_QUEUE1_DOORBELL__CAPTURED_MASK                                                                   0x40000000L
+//VPEC_QUEUE1_DOORBELL_OFFSET
+#define VPEC_QUEUE1_DOORBELL_OFFSET__OFFSET__SHIFT                                                            0x2
+#define VPEC_QUEUE1_DOORBELL_OFFSET__OFFSET_MASK                                                              0x0FFFFFFCL
+//VPEC_QUEUE1_DUMMY0
+#define VPEC_QUEUE1_DUMMY0__DUMMY__SHIFT                                                                      0x0
+#define VPEC_QUEUE1_DUMMY0__DUMMY_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE1_DUMMY1
+#define VPEC_QUEUE1_DUMMY1__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE1_DUMMY1__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE1_DUMMY2
+#define VPEC_QUEUE1_DUMMY2__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE1_DUMMY2__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE1_DUMMY3
+#define VPEC_QUEUE1_DUMMY3__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE1_DUMMY3__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE1_DUMMY4
+#define VPEC_QUEUE1_DUMMY4__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE1_DUMMY4__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE1_IB_CNTL
+#define VPEC_QUEUE1_IB_CNTL__IB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                          0x8
+#define VPEC_QUEUE1_IB_CNTL__CMD_VMID__SHIFT                                                                  0x10
+#define VPEC_QUEUE1_IB_CNTL__IB_PRIV__SHIFT                                                                   0x1f
+#define VPEC_QUEUE1_IB_CNTL__IB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE1_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                            0x00000100L
+#define VPEC_QUEUE1_IB_CNTL__CMD_VMID_MASK                                                                    0x000F0000L
+#define VPEC_QUEUE1_IB_CNTL__IB_PRIV_MASK                                                                     0x80000000L
+//VPEC_QUEUE1_IB_RPTR
+#define VPEC_QUEUE1_IB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define VPEC_QUEUE1_IB_RPTR__OFFSET_MASK                                                                      0x003FFFFCL
+//VPEC_QUEUE1_IB_OFFSET
+#define VPEC_QUEUE1_IB_OFFSET__OFFSET__SHIFT                                                                  0x2
+#define VPEC_QUEUE1_IB_OFFSET__OFFSET_MASK                                                                    0x003FFFFCL
+//VPEC_QUEUE1_IB_BASE_LO
+#define VPEC_QUEUE1_IB_BASE_LO__ADDR__SHIFT                                                                   0x5
+#define VPEC_QUEUE1_IB_BASE_LO__ADDR_MASK                                                                     0xFFFFFFE0L
+//VPEC_QUEUE1_IB_BASE_HI
+#define VPEC_QUEUE1_IB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE1_IB_BASE_HI__ADDR_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE1_IB_SIZE
+#define VPEC_QUEUE1_IB_SIZE__SIZE__SHIFT                                                                      0x0
+#define VPEC_QUEUE1_IB_SIZE__SIZE_MASK                                                                        0x000FFFFFL
+//VPEC_QUEUE1_CMDIB_CNTL
+#define VPEC_QUEUE1_CMDIB_CNTL__IB_ENABLE__SHIFT                                                              0x0
+#define VPEC_QUEUE1_CMDIB_CNTL__IB_SWAP_ENABLE__SHIFT                                                         0x4
+#define VPEC_QUEUE1_CMDIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                       0x8
+#define VPEC_QUEUE1_CMDIB_CNTL__CMD_VMID__SHIFT                                                               0x10
+#define VPEC_QUEUE1_CMDIB_CNTL__IB_PRIV__SHIFT                                                                0x1f
+#define VPEC_QUEUE1_CMDIB_CNTL__IB_ENABLE_MASK                                                                0x00000001L
+#define VPEC_QUEUE1_CMDIB_CNTL__IB_SWAP_ENABLE_MASK                                                           0x00000010L
+#define VPEC_QUEUE1_CMDIB_CNTL__SWITCH_INSIDE_IB_MASK                                                         0x00000100L
+#define VPEC_QUEUE1_CMDIB_CNTL__CMD_VMID_MASK                                                                 0x000F0000L
+#define VPEC_QUEUE1_CMDIB_CNTL__IB_PRIV_MASK                                                                  0x80000000L
+//VPEC_QUEUE1_CMDIB_RPTR
+#define VPEC_QUEUE1_CMDIB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define VPEC_QUEUE1_CMDIB_RPTR__OFFSET_MASK                                                                   0x003FFFFCL
+//VPEC_QUEUE1_CMDIB_OFFSET
+#define VPEC_QUEUE1_CMDIB_OFFSET__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE1_CMDIB_OFFSET__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE1_CMDIB_BASE_LO
+#define VPEC_QUEUE1_CMDIB_BASE_LO__ADDR__SHIFT                                                                0x5
+#define VPEC_QUEUE1_CMDIB_BASE_LO__ADDR_MASK                                                                  0xFFFFFFE0L
+//VPEC_QUEUE1_CMDIB_BASE_HI
+#define VPEC_QUEUE1_CMDIB_BASE_HI__ADDR__SHIFT                                                                0x0
+#define VPEC_QUEUE1_CMDIB_BASE_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//VPEC_QUEUE1_CMDIB_SIZE
+#define VPEC_QUEUE1_CMDIB_SIZE__SIZE__SHIFT                                                                   0x0
+#define VPEC_QUEUE1_CMDIB_SIZE__SIZE_MASK                                                                     0x000FFFFFL
+//VPEC_QUEUE1_3DLUTIB_CNTL
+#define VPEC_QUEUE1_3DLUTIB_CNTL__IB_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE1_3DLUTIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                     0x8
+#define VPEC_QUEUE1_3DLUTIB_CNTL__CMD_VMID__SHIFT                                                             0x10
+#define VPEC_QUEUE1_3DLUTIB_CNTL__IB_PRIV__SHIFT                                                              0x1f
+#define VPEC_QUEUE1_3DLUTIB_CNTL__IB_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE1_3DLUTIB_CNTL__SWITCH_INSIDE_IB_MASK                                                       0x00000100L
+#define VPEC_QUEUE1_3DLUTIB_CNTL__CMD_VMID_MASK                                                               0x000F0000L
+#define VPEC_QUEUE1_3DLUTIB_CNTL__IB_PRIV_MASK                                                                0x80000000L
+//VPEC_QUEUE1_3DLUTIB_RPTR
+#define VPEC_QUEUE1_3DLUTIB_RPTR__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE1_3DLUTIB_RPTR__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE1_3DLUTIB_OFFSET
+#define VPEC_QUEUE1_3DLUTIB_OFFSET__OFFSET__SHIFT                                                             0x2
+#define VPEC_QUEUE1_3DLUTIB_OFFSET__OFFSET_MASK                                                               0x003FFFFCL
+//VPEC_QUEUE1_3DLUTIB_BASE_LO
+#define VPEC_QUEUE1_3DLUTIB_BASE_LO__ADDR__SHIFT                                                              0x5
+#define VPEC_QUEUE1_3DLUTIB_BASE_LO__ADDR_MASK                                                                0xFFFFFFE0L
+//VPEC_QUEUE1_3DLUTIB_BASE_HI
+#define VPEC_QUEUE1_3DLUTIB_BASE_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE1_3DLUTIB_BASE_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE1_3DLUTIB_SIZE
+#define VPEC_QUEUE1_3DLUTIB_SIZE__SIZE__SHIFT                                                                 0x0
+#define VPEC_QUEUE1_3DLUTIB_SIZE__SIZE_MASK                                                                   0x000FFFFFL
+//VPEC_QUEUE1_CSA_ADDR_LO
+#define VPEC_QUEUE1_CSA_ADDR_LO__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE1_CSA_ADDR_LO__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE1_CSA_ADDR_HI
+#define VPEC_QUEUE1_CSA_ADDR_HI__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE1_CSA_ADDR_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE1_CONTEXT_STATUS
+#define VPEC_QUEUE1_CONTEXT_STATUS__SELECTED__SHIFT                                                           0x0
+#define VPEC_QUEUE1_CONTEXT_STATUS__USE_IB__SHIFT                                                             0x1
+#define VPEC_QUEUE1_CONTEXT_STATUS__IDLE__SHIFT                                                               0x2
+#define VPEC_QUEUE1_CONTEXT_STATUS__EXPIRED__SHIFT                                                            0x3
+#define VPEC_QUEUE1_CONTEXT_STATUS__EXCEPTION__SHIFT                                                          0x4
+#define VPEC_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                         0x7
+#define VPEC_QUEUE1_CONTEXT_STATUS__USE_3DLUTIB__SHIFT                                                        0x8
+#define VPEC_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                    0xa
+#define VPEC_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                       0xb
+#define VPEC_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                0xc
+#define VPEC_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                             0x10
+#define VPEC_QUEUE1_CONTEXT_STATUS__SELECTED_MASK                                                             0x00000001L
+#define VPEC_QUEUE1_CONTEXT_STATUS__USE_IB_MASK                                                               0x00000002L
+#define VPEC_QUEUE1_CONTEXT_STATUS__IDLE_MASK                                                                 0x00000004L
+#define VPEC_QUEUE1_CONTEXT_STATUS__EXPIRED_MASK                                                              0x00000008L
+#define VPEC_QUEUE1_CONTEXT_STATUS__EXCEPTION_MASK                                                            0x00000070L
+#define VPEC_QUEUE1_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                           0x00000080L
+#define VPEC_QUEUE1_CONTEXT_STATUS__USE_3DLUTIB_MASK                                                          0x00000100L
+#define VPEC_QUEUE1_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                      0x00000400L
+#define VPEC_QUEUE1_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                         0x00000800L
+#define VPEC_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                  0x00001000L
+#define VPEC_QUEUE1_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                               0x00FF0000L
+//VPEC_QUEUE1_DOORBELL_LOG
+#define VPEC_QUEUE1_DOORBELL_LOG__BE_ERROR__SHIFT                                                             0x0
+#define VPEC_QUEUE1_DOORBELL_LOG__DATA__SHIFT                                                                 0x2
+#define VPEC_QUEUE1_DOORBELL_LOG__BE_ERROR_MASK                                                               0x00000001L
+#define VPEC_QUEUE1_DOORBELL_LOG__DATA_MASK                                                                   0xFFFFFFFCL
+//VPEC_QUEUE1_IB_SUB_REMAIN
+#define VPEC_QUEUE1_IB_SUB_REMAIN__SIZE__SHIFT                                                                0x0
+#define VPEC_QUEUE1_IB_SUB_REMAIN__SIZE_MASK                                                                  0x00003FFFL
+//VPEC_QUEUE1_PREEMPT
+#define VPEC_QUEUE1_PREEMPT__IB_PREEMPT__SHIFT                                                                0x0
+#define VPEC_QUEUE1_PREEMPT__IB_PREEMPT_MASK                                                                  0x00000001L
+//VPEC_QUEUE1_LOG0BUFFER_CFG
+#define VPEC_QUEUE1_LOG0BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE1_LOG0BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE1_LOG0BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE1_LOG0BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE1_LOG0BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE1_LOG0BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE1_LOG0BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE1_LOG0BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE1_LOG1BUFFER_CFG
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__PARTIAL_ENTRY__SHIFT                                                      0x1
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__PARTIAL_ENTRY_MASK                                                        0x00000002L
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE1_LOG1BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE2_RB_CNTL
+#define VPEC_QUEUE2_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE2_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define VPEC_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                          0x8
+#define VPEC_QUEUE2_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                            0x9
+#define VPEC_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                     0xa
+#define VPEC_QUEUE2_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                      0xb
+#define VPEC_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                     0xc
+#define VPEC_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                0xd
+#define VPEC_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                      0x10
+#define VPEC_QUEUE2_RB_CNTL__RB_PRIV__SHIFT                                                                   0x17
+#define VPEC_QUEUE2_RB_CNTL__RB_VMID__SHIFT                                                                   0x18
+#define VPEC_QUEUE2_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE2_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define VPEC_QUEUE2_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                            0x00000100L
+#define VPEC_QUEUE2_RB_CNTL__RB_SWAP_ENABLE_MASK                                                              0x00000200L
+#define VPEC_QUEUE2_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                       0x00000400L
+#define VPEC_QUEUE2_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                        0x00000800L
+#define VPEC_QUEUE2_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                       0x00001000L
+#define VPEC_QUEUE2_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                  0x00002000L
+#define VPEC_QUEUE2_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                        0x001F0000L
+#define VPEC_QUEUE2_RB_CNTL__RB_PRIV_MASK                                                                     0x00800000L
+#define VPEC_QUEUE2_RB_CNTL__RB_VMID_MASK                                                                     0x0F000000L
+//VPEC_QUEUE2_SCHEDULE_CNTL
+#define VPEC_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                           0x0
+#define VPEC_QUEUE2_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                          0x2
+#define VPEC_QUEUE2_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                            0x6
+#define VPEC_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                     0x8
+#define VPEC_QUEUE2_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                             0x00000003L
+#define VPEC_QUEUE2_SCHEDULE_CNTL__PROCESS_ID_MASK                                                            0x0000001CL
+#define VPEC_QUEUE2_SCHEDULE_CNTL__LOCAL_ID_MASK                                                              0x000000C0L
+#define VPEC_QUEUE2_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                       0x0000FF00L
+//VPEC_QUEUE2_RB_BASE
+#define VPEC_QUEUE2_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define VPEC_QUEUE2_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE2_RB_BASE_HI
+#define VPEC_QUEUE2_RB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE2_RB_BASE_HI__ADDR_MASK                                                                     0x00FFFFFFL
+//VPEC_QUEUE2_RB_RPTR
+#define VPEC_QUEUE2_RB_RPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE2_RB_RPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE2_RB_RPTR_HI
+#define VPEC_QUEUE2_RB_RPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE2_RB_RPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE2_RB_WPTR
+#define VPEC_QUEUE2_RB_WPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE2_RB_WPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE2_RB_WPTR_HI
+#define VPEC_QUEUE2_RB_WPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE2_RB_WPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE2_RB_RPTR_ADDR_HI
+#define VPEC_QUEUE2_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE2_RB_RPTR_ADDR_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE2_RB_RPTR_ADDR_LO
+#define VPEC_QUEUE2_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define VPEC_QUEUE2_RB_RPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//VPEC_QUEUE2_RB_AQL_CNTL
+#define VPEC_QUEUE2_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                       0x1
+#define VPEC_QUEUE2_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                           0x8
+#define VPEC_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                 0x10
+#define VPEC_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                           0x11
+#define VPEC_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                        0x12
+#define VPEC_QUEUE2_RB_AQL_CNTL__AQL_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE2_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                         0x000000FEL
+#define VPEC_QUEUE2_RB_AQL_CNTL__PACKET_STEP_MASK                                                             0x0000FF00L
+#define VPEC_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                   0x00010000L
+#define VPEC_QUEUE2_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                             0x00020000L
+#define VPEC_QUEUE2_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                          0x00040000L
+//VPEC_QUEUE2_MINOR_PTR_UPDATE
+#define VPEC_QUEUE2_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                           0x0
+#define VPEC_QUEUE2_MINOR_PTR_UPDATE__ENABLE_MASK                                                             0x00000001L
+//VPEC_QUEUE2_CD_INFO
+#define VPEC_QUEUE2_CD_INFO__CD_INFO__SHIFT                                                                   0x0
+#define VPEC_QUEUE2_CD_INFO__CD_INFO_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE2_RB_PREEMPT
+#define VPEC_QUEUE2_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                            0x0
+#define VPEC_QUEUE2_RB_PREEMPT__PREEMPT_REQ_MASK                                                              0x00000001L
+//VPEC_QUEUE2_SKIP_CNTL
+#define VPEC_QUEUE2_SKIP_CNTL__SKIP_COUNT__SHIFT                                                              0x0
+#define VPEC_QUEUE2_SKIP_CNTL__SKIP_COUNT_MASK                                                                0x000FFFFFL
+//VPEC_QUEUE2_DOORBELL
+#define VPEC_QUEUE2_DOORBELL__ENABLE__SHIFT                                                                   0x1c
+#define VPEC_QUEUE2_DOORBELL__CAPTURED__SHIFT                                                                 0x1e
+#define VPEC_QUEUE2_DOORBELL__ENABLE_MASK                                                                     0x10000000L
+#define VPEC_QUEUE2_DOORBELL__CAPTURED_MASK                                                                   0x40000000L
+//VPEC_QUEUE2_DOORBELL_OFFSET
+#define VPEC_QUEUE2_DOORBELL_OFFSET__OFFSET__SHIFT                                                            0x2
+#define VPEC_QUEUE2_DOORBELL_OFFSET__OFFSET_MASK                                                              0x0FFFFFFCL
+//VPEC_QUEUE2_DUMMY0
+#define VPEC_QUEUE2_DUMMY0__DUMMY__SHIFT                                                                      0x0
+#define VPEC_QUEUE2_DUMMY0__DUMMY_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE2_DUMMY1
+#define VPEC_QUEUE2_DUMMY1__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE2_DUMMY1__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE2_DUMMY2
+#define VPEC_QUEUE2_DUMMY2__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE2_DUMMY2__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE2_DUMMY3
+#define VPEC_QUEUE2_DUMMY3__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE2_DUMMY3__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE2_DUMMY4
+#define VPEC_QUEUE2_DUMMY4__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE2_DUMMY4__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE2_IB_CNTL
+#define VPEC_QUEUE2_IB_CNTL__IB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                          0x8
+#define VPEC_QUEUE2_IB_CNTL__CMD_VMID__SHIFT                                                                  0x10
+#define VPEC_QUEUE2_IB_CNTL__IB_PRIV__SHIFT                                                                   0x1f
+#define VPEC_QUEUE2_IB_CNTL__IB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE2_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                            0x00000100L
+#define VPEC_QUEUE2_IB_CNTL__CMD_VMID_MASK                                                                    0x000F0000L
+#define VPEC_QUEUE2_IB_CNTL__IB_PRIV_MASK                                                                     0x80000000L
+//VPEC_QUEUE2_IB_RPTR
+#define VPEC_QUEUE2_IB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define VPEC_QUEUE2_IB_RPTR__OFFSET_MASK                                                                      0x003FFFFCL
+//VPEC_QUEUE2_IB_OFFSET
+#define VPEC_QUEUE2_IB_OFFSET__OFFSET__SHIFT                                                                  0x2
+#define VPEC_QUEUE2_IB_OFFSET__OFFSET_MASK                                                                    0x003FFFFCL
+//VPEC_QUEUE2_IB_BASE_LO
+#define VPEC_QUEUE2_IB_BASE_LO__ADDR__SHIFT                                                                   0x5
+#define VPEC_QUEUE2_IB_BASE_LO__ADDR_MASK                                                                     0xFFFFFFE0L
+//VPEC_QUEUE2_IB_BASE_HI
+#define VPEC_QUEUE2_IB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE2_IB_BASE_HI__ADDR_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE2_IB_SIZE
+#define VPEC_QUEUE2_IB_SIZE__SIZE__SHIFT                                                                      0x0
+#define VPEC_QUEUE2_IB_SIZE__SIZE_MASK                                                                        0x000FFFFFL
+//VPEC_QUEUE2_CMDIB_CNTL
+#define VPEC_QUEUE2_CMDIB_CNTL__IB_ENABLE__SHIFT                                                              0x0
+#define VPEC_QUEUE2_CMDIB_CNTL__IB_SWAP_ENABLE__SHIFT                                                         0x4
+#define VPEC_QUEUE2_CMDIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                       0x8
+#define VPEC_QUEUE2_CMDIB_CNTL__CMD_VMID__SHIFT                                                               0x10
+#define VPEC_QUEUE2_CMDIB_CNTL__IB_PRIV__SHIFT                                                                0x1f
+#define VPEC_QUEUE2_CMDIB_CNTL__IB_ENABLE_MASK                                                                0x00000001L
+#define VPEC_QUEUE2_CMDIB_CNTL__IB_SWAP_ENABLE_MASK                                                           0x00000010L
+#define VPEC_QUEUE2_CMDIB_CNTL__SWITCH_INSIDE_IB_MASK                                                         0x00000100L
+#define VPEC_QUEUE2_CMDIB_CNTL__CMD_VMID_MASK                                                                 0x000F0000L
+#define VPEC_QUEUE2_CMDIB_CNTL__IB_PRIV_MASK                                                                  0x80000000L
+//VPEC_QUEUE2_CMDIB_RPTR
+#define VPEC_QUEUE2_CMDIB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define VPEC_QUEUE2_CMDIB_RPTR__OFFSET_MASK                                                                   0x003FFFFCL
+//VPEC_QUEUE2_CMDIB_OFFSET
+#define VPEC_QUEUE2_CMDIB_OFFSET__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE2_CMDIB_OFFSET__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE2_CMDIB_BASE_LO
+#define VPEC_QUEUE2_CMDIB_BASE_LO__ADDR__SHIFT                                                                0x5
+#define VPEC_QUEUE2_CMDIB_BASE_LO__ADDR_MASK                                                                  0xFFFFFFE0L
+//VPEC_QUEUE2_CMDIB_BASE_HI
+#define VPEC_QUEUE2_CMDIB_BASE_HI__ADDR__SHIFT                                                                0x0
+#define VPEC_QUEUE2_CMDIB_BASE_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//VPEC_QUEUE2_CMDIB_SIZE
+#define VPEC_QUEUE2_CMDIB_SIZE__SIZE__SHIFT                                                                   0x0
+#define VPEC_QUEUE2_CMDIB_SIZE__SIZE_MASK                                                                     0x000FFFFFL
+//VPEC_QUEUE2_3DLUTIB_CNTL
+#define VPEC_QUEUE2_3DLUTIB_CNTL__IB_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE2_3DLUTIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                     0x8
+#define VPEC_QUEUE2_3DLUTIB_CNTL__CMD_VMID__SHIFT                                                             0x10
+#define VPEC_QUEUE2_3DLUTIB_CNTL__IB_PRIV__SHIFT                                                              0x1f
+#define VPEC_QUEUE2_3DLUTIB_CNTL__IB_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE2_3DLUTIB_CNTL__SWITCH_INSIDE_IB_MASK                                                       0x00000100L
+#define VPEC_QUEUE2_3DLUTIB_CNTL__CMD_VMID_MASK                                                               0x000F0000L
+#define VPEC_QUEUE2_3DLUTIB_CNTL__IB_PRIV_MASK                                                                0x80000000L
+//VPEC_QUEUE2_3DLUTIB_RPTR
+#define VPEC_QUEUE2_3DLUTIB_RPTR__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE2_3DLUTIB_RPTR__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE2_3DLUTIB_OFFSET
+#define VPEC_QUEUE2_3DLUTIB_OFFSET__OFFSET__SHIFT                                                             0x2
+#define VPEC_QUEUE2_3DLUTIB_OFFSET__OFFSET_MASK                                                               0x003FFFFCL
+//VPEC_QUEUE2_3DLUTIB_BASE_LO
+#define VPEC_QUEUE2_3DLUTIB_BASE_LO__ADDR__SHIFT                                                              0x5
+#define VPEC_QUEUE2_3DLUTIB_BASE_LO__ADDR_MASK                                                                0xFFFFFFE0L
+//VPEC_QUEUE2_3DLUTIB_BASE_HI
+#define VPEC_QUEUE2_3DLUTIB_BASE_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE2_3DLUTIB_BASE_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE2_3DLUTIB_SIZE
+#define VPEC_QUEUE2_3DLUTIB_SIZE__SIZE__SHIFT                                                                 0x0
+#define VPEC_QUEUE2_3DLUTIB_SIZE__SIZE_MASK                                                                   0x000FFFFFL
+//VPEC_QUEUE2_CSA_ADDR_LO
+#define VPEC_QUEUE2_CSA_ADDR_LO__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE2_CSA_ADDR_LO__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE2_CSA_ADDR_HI
+#define VPEC_QUEUE2_CSA_ADDR_HI__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE2_CSA_ADDR_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE2_CONTEXT_STATUS
+#define VPEC_QUEUE2_CONTEXT_STATUS__SELECTED__SHIFT                                                           0x0
+#define VPEC_QUEUE2_CONTEXT_STATUS__USE_IB__SHIFT                                                             0x1
+#define VPEC_QUEUE2_CONTEXT_STATUS__IDLE__SHIFT                                                               0x2
+#define VPEC_QUEUE2_CONTEXT_STATUS__EXPIRED__SHIFT                                                            0x3
+#define VPEC_QUEUE2_CONTEXT_STATUS__EXCEPTION__SHIFT                                                          0x4
+#define VPEC_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                         0x7
+#define VPEC_QUEUE2_CONTEXT_STATUS__USE_3DLUTIB__SHIFT                                                        0x8
+#define VPEC_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                    0xa
+#define VPEC_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                       0xb
+#define VPEC_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                0xc
+#define VPEC_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                             0x10
+#define VPEC_QUEUE2_CONTEXT_STATUS__SELECTED_MASK                                                             0x00000001L
+#define VPEC_QUEUE2_CONTEXT_STATUS__USE_IB_MASK                                                               0x00000002L
+#define VPEC_QUEUE2_CONTEXT_STATUS__IDLE_MASK                                                                 0x00000004L
+#define VPEC_QUEUE2_CONTEXT_STATUS__EXPIRED_MASK                                                              0x00000008L
+#define VPEC_QUEUE2_CONTEXT_STATUS__EXCEPTION_MASK                                                            0x00000070L
+#define VPEC_QUEUE2_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                           0x00000080L
+#define VPEC_QUEUE2_CONTEXT_STATUS__USE_3DLUTIB_MASK                                                          0x00000100L
+#define VPEC_QUEUE2_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                      0x00000400L
+#define VPEC_QUEUE2_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                         0x00000800L
+#define VPEC_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                  0x00001000L
+#define VPEC_QUEUE2_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                               0x00FF0000L
+//VPEC_QUEUE2_DOORBELL_LOG
+#define VPEC_QUEUE2_DOORBELL_LOG__BE_ERROR__SHIFT                                                             0x0
+#define VPEC_QUEUE2_DOORBELL_LOG__DATA__SHIFT                                                                 0x2
+#define VPEC_QUEUE2_DOORBELL_LOG__BE_ERROR_MASK                                                               0x00000001L
+#define VPEC_QUEUE2_DOORBELL_LOG__DATA_MASK                                                                   0xFFFFFFFCL
+//VPEC_QUEUE2_IB_SUB_REMAIN
+#define VPEC_QUEUE2_IB_SUB_REMAIN__SIZE__SHIFT                                                                0x0
+#define VPEC_QUEUE2_IB_SUB_REMAIN__SIZE_MASK                                                                  0x00003FFFL
+//VPEC_QUEUE2_PREEMPT
+#define VPEC_QUEUE2_PREEMPT__IB_PREEMPT__SHIFT                                                                0x0
+#define VPEC_QUEUE2_PREEMPT__IB_PREEMPT_MASK                                                                  0x00000001L
+//VPEC_QUEUE2_LOG0BUFFER_CFG
+#define VPEC_QUEUE2_LOG0BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE2_LOG0BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE2_LOG0BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE2_LOG0BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE2_LOG0BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE2_LOG0BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE2_LOG0BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE2_LOG0BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE2_LOG1BUFFER_CFG
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__PARTIAL_ENTRY__SHIFT                                                      0x1
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__PARTIAL_ENTRY_MASK                                                        0x00000002L
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE2_LOG1BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE3_RB_CNTL
+#define VPEC_QUEUE3_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE3_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define VPEC_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                          0x8
+#define VPEC_QUEUE3_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                            0x9
+#define VPEC_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                     0xa
+#define VPEC_QUEUE3_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                      0xb
+#define VPEC_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                     0xc
+#define VPEC_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                0xd
+#define VPEC_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                      0x10
+#define VPEC_QUEUE3_RB_CNTL__RB_PRIV__SHIFT                                                                   0x17
+#define VPEC_QUEUE3_RB_CNTL__RB_VMID__SHIFT                                                                   0x18
+#define VPEC_QUEUE3_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE3_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define VPEC_QUEUE3_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                            0x00000100L
+#define VPEC_QUEUE3_RB_CNTL__RB_SWAP_ENABLE_MASK                                                              0x00000200L
+#define VPEC_QUEUE3_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                       0x00000400L
+#define VPEC_QUEUE3_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                        0x00000800L
+#define VPEC_QUEUE3_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                       0x00001000L
+#define VPEC_QUEUE3_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                  0x00002000L
+#define VPEC_QUEUE3_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                        0x001F0000L
+#define VPEC_QUEUE3_RB_CNTL__RB_PRIV_MASK                                                                     0x00800000L
+#define VPEC_QUEUE3_RB_CNTL__RB_VMID_MASK                                                                     0x0F000000L
+//VPEC_QUEUE3_SCHEDULE_CNTL
+#define VPEC_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                           0x0
+#define VPEC_QUEUE3_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                          0x2
+#define VPEC_QUEUE3_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                            0x6
+#define VPEC_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                     0x8
+#define VPEC_QUEUE3_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                             0x00000003L
+#define VPEC_QUEUE3_SCHEDULE_CNTL__PROCESS_ID_MASK                                                            0x0000001CL
+#define VPEC_QUEUE3_SCHEDULE_CNTL__LOCAL_ID_MASK                                                              0x000000C0L
+#define VPEC_QUEUE3_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                       0x0000FF00L
+//VPEC_QUEUE3_RB_BASE
+#define VPEC_QUEUE3_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define VPEC_QUEUE3_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE3_RB_BASE_HI
+#define VPEC_QUEUE3_RB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE3_RB_BASE_HI__ADDR_MASK                                                                     0x00FFFFFFL
+//VPEC_QUEUE3_RB_RPTR
+#define VPEC_QUEUE3_RB_RPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE3_RB_RPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE3_RB_RPTR_HI
+#define VPEC_QUEUE3_RB_RPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE3_RB_RPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE3_RB_WPTR
+#define VPEC_QUEUE3_RB_WPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE3_RB_WPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE3_RB_WPTR_HI
+#define VPEC_QUEUE3_RB_WPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE3_RB_WPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE3_RB_RPTR_ADDR_HI
+#define VPEC_QUEUE3_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE3_RB_RPTR_ADDR_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE3_RB_RPTR_ADDR_LO
+#define VPEC_QUEUE3_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define VPEC_QUEUE3_RB_RPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//VPEC_QUEUE3_RB_AQL_CNTL
+#define VPEC_QUEUE3_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                       0x1
+#define VPEC_QUEUE3_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                           0x8
+#define VPEC_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                 0x10
+#define VPEC_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                           0x11
+#define VPEC_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                        0x12
+#define VPEC_QUEUE3_RB_AQL_CNTL__AQL_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE3_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                         0x000000FEL
+#define VPEC_QUEUE3_RB_AQL_CNTL__PACKET_STEP_MASK                                                             0x0000FF00L
+#define VPEC_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                   0x00010000L
+#define VPEC_QUEUE3_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                             0x00020000L
+#define VPEC_QUEUE3_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                          0x00040000L
+//VPEC_QUEUE3_MINOR_PTR_UPDATE
+#define VPEC_QUEUE3_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                           0x0
+#define VPEC_QUEUE3_MINOR_PTR_UPDATE__ENABLE_MASK                                                             0x00000001L
+//VPEC_QUEUE3_CD_INFO
+#define VPEC_QUEUE3_CD_INFO__CD_INFO__SHIFT                                                                   0x0
+#define VPEC_QUEUE3_CD_INFO__CD_INFO_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE3_RB_PREEMPT
+#define VPEC_QUEUE3_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                            0x0
+#define VPEC_QUEUE3_RB_PREEMPT__PREEMPT_REQ_MASK                                                              0x00000001L
+//VPEC_QUEUE3_SKIP_CNTL
+#define VPEC_QUEUE3_SKIP_CNTL__SKIP_COUNT__SHIFT                                                              0x0
+#define VPEC_QUEUE3_SKIP_CNTL__SKIP_COUNT_MASK                                                                0x000FFFFFL
+//VPEC_QUEUE3_DOORBELL
+#define VPEC_QUEUE3_DOORBELL__ENABLE__SHIFT                                                                   0x1c
+#define VPEC_QUEUE3_DOORBELL__CAPTURED__SHIFT                                                                 0x1e
+#define VPEC_QUEUE3_DOORBELL__ENABLE_MASK                                                                     0x10000000L
+#define VPEC_QUEUE3_DOORBELL__CAPTURED_MASK                                                                   0x40000000L
+//VPEC_QUEUE3_DOORBELL_OFFSET
+#define VPEC_QUEUE3_DOORBELL_OFFSET__OFFSET__SHIFT                                                            0x2
+#define VPEC_QUEUE3_DOORBELL_OFFSET__OFFSET_MASK                                                              0x0FFFFFFCL
+//VPEC_QUEUE3_DUMMY0
+#define VPEC_QUEUE3_DUMMY0__DUMMY__SHIFT                                                                      0x0
+#define VPEC_QUEUE3_DUMMY0__DUMMY_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE3_DUMMY1
+#define VPEC_QUEUE3_DUMMY1__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE3_DUMMY1__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE3_DUMMY2
+#define VPEC_QUEUE3_DUMMY2__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE3_DUMMY2__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE3_DUMMY3
+#define VPEC_QUEUE3_DUMMY3__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE3_DUMMY3__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE3_DUMMY4
+#define VPEC_QUEUE3_DUMMY4__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE3_DUMMY4__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE3_IB_CNTL
+#define VPEC_QUEUE3_IB_CNTL__IB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                          0x8
+#define VPEC_QUEUE3_IB_CNTL__CMD_VMID__SHIFT                                                                  0x10
+#define VPEC_QUEUE3_IB_CNTL__IB_PRIV__SHIFT                                                                   0x1f
+#define VPEC_QUEUE3_IB_CNTL__IB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE3_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                            0x00000100L
+#define VPEC_QUEUE3_IB_CNTL__CMD_VMID_MASK                                                                    0x000F0000L
+#define VPEC_QUEUE3_IB_CNTL__IB_PRIV_MASK                                                                     0x80000000L
+//VPEC_QUEUE3_IB_RPTR
+#define VPEC_QUEUE3_IB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define VPEC_QUEUE3_IB_RPTR__OFFSET_MASK                                                                      0x003FFFFCL
+//VPEC_QUEUE3_IB_OFFSET
+#define VPEC_QUEUE3_IB_OFFSET__OFFSET__SHIFT                                                                  0x2
+#define VPEC_QUEUE3_IB_OFFSET__OFFSET_MASK                                                                    0x003FFFFCL
+//VPEC_QUEUE3_IB_BASE_LO
+#define VPEC_QUEUE3_IB_BASE_LO__ADDR__SHIFT                                                                   0x5
+#define VPEC_QUEUE3_IB_BASE_LO__ADDR_MASK                                                                     0xFFFFFFE0L
+//VPEC_QUEUE3_IB_BASE_HI
+#define VPEC_QUEUE3_IB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE3_IB_BASE_HI__ADDR_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE3_IB_SIZE
+#define VPEC_QUEUE3_IB_SIZE__SIZE__SHIFT                                                                      0x0
+#define VPEC_QUEUE3_IB_SIZE__SIZE_MASK                                                                        0x000FFFFFL
+//VPEC_QUEUE3_CMDIB_CNTL
+#define VPEC_QUEUE3_CMDIB_CNTL__IB_ENABLE__SHIFT                                                              0x0
+#define VPEC_QUEUE3_CMDIB_CNTL__IB_SWAP_ENABLE__SHIFT                                                         0x4
+#define VPEC_QUEUE3_CMDIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                       0x8
+#define VPEC_QUEUE3_CMDIB_CNTL__CMD_VMID__SHIFT                                                               0x10
+#define VPEC_QUEUE3_CMDIB_CNTL__IB_PRIV__SHIFT                                                                0x1f
+#define VPEC_QUEUE3_CMDIB_CNTL__IB_ENABLE_MASK                                                                0x00000001L
+#define VPEC_QUEUE3_CMDIB_CNTL__IB_SWAP_ENABLE_MASK                                                           0x00000010L
+#define VPEC_QUEUE3_CMDIB_CNTL__SWITCH_INSIDE_IB_MASK                                                         0x00000100L
+#define VPEC_QUEUE3_CMDIB_CNTL__CMD_VMID_MASK                                                                 0x000F0000L
+#define VPEC_QUEUE3_CMDIB_CNTL__IB_PRIV_MASK                                                                  0x80000000L
+//VPEC_QUEUE3_CMDIB_RPTR
+#define VPEC_QUEUE3_CMDIB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define VPEC_QUEUE3_CMDIB_RPTR__OFFSET_MASK                                                                   0x003FFFFCL
+//VPEC_QUEUE3_CMDIB_OFFSET
+#define VPEC_QUEUE3_CMDIB_OFFSET__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE3_CMDIB_OFFSET__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE3_CMDIB_BASE_LO
+#define VPEC_QUEUE3_CMDIB_BASE_LO__ADDR__SHIFT                                                                0x5
+#define VPEC_QUEUE3_CMDIB_BASE_LO__ADDR_MASK                                                                  0xFFFFFFE0L
+//VPEC_QUEUE3_CMDIB_BASE_HI
+#define VPEC_QUEUE3_CMDIB_BASE_HI__ADDR__SHIFT                                                                0x0
+#define VPEC_QUEUE3_CMDIB_BASE_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//VPEC_QUEUE3_CMDIB_SIZE
+#define VPEC_QUEUE3_CMDIB_SIZE__SIZE__SHIFT                                                                   0x0
+#define VPEC_QUEUE3_CMDIB_SIZE__SIZE_MASK                                                                     0x000FFFFFL
+//VPEC_QUEUE3_3DLUTIB_CNTL
+#define VPEC_QUEUE3_3DLUTIB_CNTL__IB_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE3_3DLUTIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                     0x8
+#define VPEC_QUEUE3_3DLUTIB_CNTL__CMD_VMID__SHIFT                                                             0x10
+#define VPEC_QUEUE3_3DLUTIB_CNTL__IB_PRIV__SHIFT                                                              0x1f
+#define VPEC_QUEUE3_3DLUTIB_CNTL__IB_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE3_3DLUTIB_CNTL__SWITCH_INSIDE_IB_MASK                                                       0x00000100L
+#define VPEC_QUEUE3_3DLUTIB_CNTL__CMD_VMID_MASK                                                               0x000F0000L
+#define VPEC_QUEUE3_3DLUTIB_CNTL__IB_PRIV_MASK                                                                0x80000000L
+//VPEC_QUEUE3_3DLUTIB_RPTR
+#define VPEC_QUEUE3_3DLUTIB_RPTR__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE3_3DLUTIB_RPTR__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE3_3DLUTIB_OFFSET
+#define VPEC_QUEUE3_3DLUTIB_OFFSET__OFFSET__SHIFT                                                             0x2
+#define VPEC_QUEUE3_3DLUTIB_OFFSET__OFFSET_MASK                                                               0x003FFFFCL
+//VPEC_QUEUE3_3DLUTIB_BASE_LO
+#define VPEC_QUEUE3_3DLUTIB_BASE_LO__ADDR__SHIFT                                                              0x5
+#define VPEC_QUEUE3_3DLUTIB_BASE_LO__ADDR_MASK                                                                0xFFFFFFE0L
+//VPEC_QUEUE3_3DLUTIB_BASE_HI
+#define VPEC_QUEUE3_3DLUTIB_BASE_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE3_3DLUTIB_BASE_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE3_3DLUTIB_SIZE
+#define VPEC_QUEUE3_3DLUTIB_SIZE__SIZE__SHIFT                                                                 0x0
+#define VPEC_QUEUE3_3DLUTIB_SIZE__SIZE_MASK                                                                   0x000FFFFFL
+//VPEC_QUEUE3_CSA_ADDR_LO
+#define VPEC_QUEUE3_CSA_ADDR_LO__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE3_CSA_ADDR_LO__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE3_CSA_ADDR_HI
+#define VPEC_QUEUE3_CSA_ADDR_HI__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE3_CSA_ADDR_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE3_CONTEXT_STATUS
+#define VPEC_QUEUE3_CONTEXT_STATUS__SELECTED__SHIFT                                                           0x0
+#define VPEC_QUEUE3_CONTEXT_STATUS__USE_IB__SHIFT                                                             0x1
+#define VPEC_QUEUE3_CONTEXT_STATUS__IDLE__SHIFT                                                               0x2
+#define VPEC_QUEUE3_CONTEXT_STATUS__EXPIRED__SHIFT                                                            0x3
+#define VPEC_QUEUE3_CONTEXT_STATUS__EXCEPTION__SHIFT                                                          0x4
+#define VPEC_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                         0x7
+#define VPEC_QUEUE3_CONTEXT_STATUS__USE_3DLUTIB__SHIFT                                                        0x8
+#define VPEC_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                    0xa
+#define VPEC_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                       0xb
+#define VPEC_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                0xc
+#define VPEC_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                             0x10
+#define VPEC_QUEUE3_CONTEXT_STATUS__SELECTED_MASK                                                             0x00000001L
+#define VPEC_QUEUE3_CONTEXT_STATUS__USE_IB_MASK                                                               0x00000002L
+#define VPEC_QUEUE3_CONTEXT_STATUS__IDLE_MASK                                                                 0x00000004L
+#define VPEC_QUEUE3_CONTEXT_STATUS__EXPIRED_MASK                                                              0x00000008L
+#define VPEC_QUEUE3_CONTEXT_STATUS__EXCEPTION_MASK                                                            0x00000070L
+#define VPEC_QUEUE3_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                           0x00000080L
+#define VPEC_QUEUE3_CONTEXT_STATUS__USE_3DLUTIB_MASK                                                          0x00000100L
+#define VPEC_QUEUE3_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                      0x00000400L
+#define VPEC_QUEUE3_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                         0x00000800L
+#define VPEC_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                  0x00001000L
+#define VPEC_QUEUE3_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                               0x00FF0000L
+//VPEC_QUEUE3_DOORBELL_LOG
+#define VPEC_QUEUE3_DOORBELL_LOG__BE_ERROR__SHIFT                                                             0x0
+#define VPEC_QUEUE3_DOORBELL_LOG__DATA__SHIFT                                                                 0x2
+#define VPEC_QUEUE3_DOORBELL_LOG__BE_ERROR_MASK                                                               0x00000001L
+#define VPEC_QUEUE3_DOORBELL_LOG__DATA_MASK                                                                   0xFFFFFFFCL
+//VPEC_QUEUE3_IB_SUB_REMAIN
+#define VPEC_QUEUE3_IB_SUB_REMAIN__SIZE__SHIFT                                                                0x0
+#define VPEC_QUEUE3_IB_SUB_REMAIN__SIZE_MASK                                                                  0x00003FFFL
+//VPEC_QUEUE3_PREEMPT
+#define VPEC_QUEUE3_PREEMPT__IB_PREEMPT__SHIFT                                                                0x0
+#define VPEC_QUEUE3_PREEMPT__IB_PREEMPT_MASK                                                                  0x00000001L
+//VPEC_QUEUE3_LOG0BUFFER_CFG
+#define VPEC_QUEUE3_LOG0BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE3_LOG0BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE3_LOG0BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE3_LOG0BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE3_LOG0BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE3_LOG0BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE3_LOG0BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE3_LOG0BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE3_LOG1BUFFER_CFG
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__PARTIAL_ENTRY__SHIFT                                                      0x1
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__PARTIAL_ENTRY_MASK                                                        0x00000002L
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE3_LOG1BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE4_RB_CNTL
+#define VPEC_QUEUE4_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE4_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define VPEC_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                          0x8
+#define VPEC_QUEUE4_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                            0x9
+#define VPEC_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                     0xa
+#define VPEC_QUEUE4_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                      0xb
+#define VPEC_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                     0xc
+#define VPEC_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                0xd
+#define VPEC_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                      0x10
+#define VPEC_QUEUE4_RB_CNTL__RB_PRIV__SHIFT                                                                   0x17
+#define VPEC_QUEUE4_RB_CNTL__RB_VMID__SHIFT                                                                   0x18
+#define VPEC_QUEUE4_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE4_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define VPEC_QUEUE4_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                            0x00000100L
+#define VPEC_QUEUE4_RB_CNTL__RB_SWAP_ENABLE_MASK                                                              0x00000200L
+#define VPEC_QUEUE4_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                       0x00000400L
+#define VPEC_QUEUE4_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                        0x00000800L
+#define VPEC_QUEUE4_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                       0x00001000L
+#define VPEC_QUEUE4_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                  0x00002000L
+#define VPEC_QUEUE4_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                        0x001F0000L
+#define VPEC_QUEUE4_RB_CNTL__RB_PRIV_MASK                                                                     0x00800000L
+#define VPEC_QUEUE4_RB_CNTL__RB_VMID_MASK                                                                     0x0F000000L
+//VPEC_QUEUE4_SCHEDULE_CNTL
+#define VPEC_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                           0x0
+#define VPEC_QUEUE4_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                          0x2
+#define VPEC_QUEUE4_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                            0x6
+#define VPEC_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                     0x8
+#define VPEC_QUEUE4_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                             0x00000003L
+#define VPEC_QUEUE4_SCHEDULE_CNTL__PROCESS_ID_MASK                                                            0x0000001CL
+#define VPEC_QUEUE4_SCHEDULE_CNTL__LOCAL_ID_MASK                                                              0x000000C0L
+#define VPEC_QUEUE4_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                       0x0000FF00L
+//VPEC_QUEUE4_RB_BASE
+#define VPEC_QUEUE4_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define VPEC_QUEUE4_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE4_RB_BASE_HI
+#define VPEC_QUEUE4_RB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE4_RB_BASE_HI__ADDR_MASK                                                                     0x00FFFFFFL
+//VPEC_QUEUE4_RB_RPTR
+#define VPEC_QUEUE4_RB_RPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE4_RB_RPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE4_RB_RPTR_HI
+#define VPEC_QUEUE4_RB_RPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE4_RB_RPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE4_RB_WPTR
+#define VPEC_QUEUE4_RB_WPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE4_RB_WPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE4_RB_WPTR_HI
+#define VPEC_QUEUE4_RB_WPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE4_RB_WPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE4_RB_RPTR_ADDR_HI
+#define VPEC_QUEUE4_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE4_RB_RPTR_ADDR_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE4_RB_RPTR_ADDR_LO
+#define VPEC_QUEUE4_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define VPEC_QUEUE4_RB_RPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//VPEC_QUEUE4_RB_AQL_CNTL
+#define VPEC_QUEUE4_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                       0x1
+#define VPEC_QUEUE4_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                           0x8
+#define VPEC_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                 0x10
+#define VPEC_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                           0x11
+#define VPEC_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                        0x12
+#define VPEC_QUEUE4_RB_AQL_CNTL__AQL_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE4_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                         0x000000FEL
+#define VPEC_QUEUE4_RB_AQL_CNTL__PACKET_STEP_MASK                                                             0x0000FF00L
+#define VPEC_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                   0x00010000L
+#define VPEC_QUEUE4_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                             0x00020000L
+#define VPEC_QUEUE4_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                          0x00040000L
+//VPEC_QUEUE4_MINOR_PTR_UPDATE
+#define VPEC_QUEUE4_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                           0x0
+#define VPEC_QUEUE4_MINOR_PTR_UPDATE__ENABLE_MASK                                                             0x00000001L
+//VPEC_QUEUE4_CD_INFO
+#define VPEC_QUEUE4_CD_INFO__CD_INFO__SHIFT                                                                   0x0
+#define VPEC_QUEUE4_CD_INFO__CD_INFO_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE4_RB_PREEMPT
+#define VPEC_QUEUE4_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                            0x0
+#define VPEC_QUEUE4_RB_PREEMPT__PREEMPT_REQ_MASK                                                              0x00000001L
+//VPEC_QUEUE4_SKIP_CNTL
+#define VPEC_QUEUE4_SKIP_CNTL__SKIP_COUNT__SHIFT                                                              0x0
+#define VPEC_QUEUE4_SKIP_CNTL__SKIP_COUNT_MASK                                                                0x000FFFFFL
+//VPEC_QUEUE4_DOORBELL
+#define VPEC_QUEUE4_DOORBELL__ENABLE__SHIFT                                                                   0x1c
+#define VPEC_QUEUE4_DOORBELL__CAPTURED__SHIFT                                                                 0x1e
+#define VPEC_QUEUE4_DOORBELL__ENABLE_MASK                                                                     0x10000000L
+#define VPEC_QUEUE4_DOORBELL__CAPTURED_MASK                                                                   0x40000000L
+//VPEC_QUEUE4_DOORBELL_OFFSET
+#define VPEC_QUEUE4_DOORBELL_OFFSET__OFFSET__SHIFT                                                            0x2
+#define VPEC_QUEUE4_DOORBELL_OFFSET__OFFSET_MASK                                                              0x0FFFFFFCL
+//VPEC_QUEUE4_DUMMY0
+#define VPEC_QUEUE4_DUMMY0__DUMMY__SHIFT                                                                      0x0
+#define VPEC_QUEUE4_DUMMY0__DUMMY_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE4_DUMMY1
+#define VPEC_QUEUE4_DUMMY1__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE4_DUMMY1__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE4_DUMMY2
+#define VPEC_QUEUE4_DUMMY2__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE4_DUMMY2__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE4_DUMMY3
+#define VPEC_QUEUE4_DUMMY3__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE4_DUMMY3__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE4_DUMMY4
+#define VPEC_QUEUE4_DUMMY4__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE4_DUMMY4__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE4_IB_CNTL
+#define VPEC_QUEUE4_IB_CNTL__IB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                          0x8
+#define VPEC_QUEUE4_IB_CNTL__CMD_VMID__SHIFT                                                                  0x10
+#define VPEC_QUEUE4_IB_CNTL__IB_PRIV__SHIFT                                                                   0x1f
+#define VPEC_QUEUE4_IB_CNTL__IB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE4_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                            0x00000100L
+#define VPEC_QUEUE4_IB_CNTL__CMD_VMID_MASK                                                                    0x000F0000L
+#define VPEC_QUEUE4_IB_CNTL__IB_PRIV_MASK                                                                     0x80000000L
+//VPEC_QUEUE4_IB_RPTR
+#define VPEC_QUEUE4_IB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define VPEC_QUEUE4_IB_RPTR__OFFSET_MASK                                                                      0x003FFFFCL
+//VPEC_QUEUE4_IB_OFFSET
+#define VPEC_QUEUE4_IB_OFFSET__OFFSET__SHIFT                                                                  0x2
+#define VPEC_QUEUE4_IB_OFFSET__OFFSET_MASK                                                                    0x003FFFFCL
+//VPEC_QUEUE4_IB_BASE_LO
+#define VPEC_QUEUE4_IB_BASE_LO__ADDR__SHIFT                                                                   0x5
+#define VPEC_QUEUE4_IB_BASE_LO__ADDR_MASK                                                                     0xFFFFFFE0L
+//VPEC_QUEUE4_IB_BASE_HI
+#define VPEC_QUEUE4_IB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE4_IB_BASE_HI__ADDR_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE4_IB_SIZE
+#define VPEC_QUEUE4_IB_SIZE__SIZE__SHIFT                                                                      0x0
+#define VPEC_QUEUE4_IB_SIZE__SIZE_MASK                                                                        0x000FFFFFL
+//VPEC_QUEUE4_CMDIB_CNTL
+#define VPEC_QUEUE4_CMDIB_CNTL__IB_ENABLE__SHIFT                                                              0x0
+#define VPEC_QUEUE4_CMDIB_CNTL__IB_SWAP_ENABLE__SHIFT                                                         0x4
+#define VPEC_QUEUE4_CMDIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                       0x8
+#define VPEC_QUEUE4_CMDIB_CNTL__CMD_VMID__SHIFT                                                               0x10
+#define VPEC_QUEUE4_CMDIB_CNTL__IB_PRIV__SHIFT                                                                0x1f
+#define VPEC_QUEUE4_CMDIB_CNTL__IB_ENABLE_MASK                                                                0x00000001L
+#define VPEC_QUEUE4_CMDIB_CNTL__IB_SWAP_ENABLE_MASK                                                           0x00000010L
+#define VPEC_QUEUE4_CMDIB_CNTL__SWITCH_INSIDE_IB_MASK                                                         0x00000100L
+#define VPEC_QUEUE4_CMDIB_CNTL__CMD_VMID_MASK                                                                 0x000F0000L
+#define VPEC_QUEUE4_CMDIB_CNTL__IB_PRIV_MASK                                                                  0x80000000L
+//VPEC_QUEUE4_CMDIB_RPTR
+#define VPEC_QUEUE4_CMDIB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define VPEC_QUEUE4_CMDIB_RPTR__OFFSET_MASK                                                                   0x003FFFFCL
+//VPEC_QUEUE4_CMDIB_OFFSET
+#define VPEC_QUEUE4_CMDIB_OFFSET__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE4_CMDIB_OFFSET__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE4_CMDIB_BASE_LO
+#define VPEC_QUEUE4_CMDIB_BASE_LO__ADDR__SHIFT                                                                0x5
+#define VPEC_QUEUE4_CMDIB_BASE_LO__ADDR_MASK                                                                  0xFFFFFFE0L
+//VPEC_QUEUE4_CMDIB_BASE_HI
+#define VPEC_QUEUE4_CMDIB_BASE_HI__ADDR__SHIFT                                                                0x0
+#define VPEC_QUEUE4_CMDIB_BASE_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//VPEC_QUEUE4_CMDIB_SIZE
+#define VPEC_QUEUE4_CMDIB_SIZE__SIZE__SHIFT                                                                   0x0
+#define VPEC_QUEUE4_CMDIB_SIZE__SIZE_MASK                                                                     0x000FFFFFL
+//VPEC_QUEUE4_3DLUTIB_CNTL
+#define VPEC_QUEUE4_3DLUTIB_CNTL__IB_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE4_3DLUTIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                     0x8
+#define VPEC_QUEUE4_3DLUTIB_CNTL__CMD_VMID__SHIFT                                                             0x10
+#define VPEC_QUEUE4_3DLUTIB_CNTL__IB_PRIV__SHIFT                                                              0x1f
+#define VPEC_QUEUE4_3DLUTIB_CNTL__IB_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE4_3DLUTIB_CNTL__SWITCH_INSIDE_IB_MASK                                                       0x00000100L
+#define VPEC_QUEUE4_3DLUTIB_CNTL__CMD_VMID_MASK                                                               0x000F0000L
+#define VPEC_QUEUE4_3DLUTIB_CNTL__IB_PRIV_MASK                                                                0x80000000L
+//VPEC_QUEUE4_3DLUTIB_RPTR
+#define VPEC_QUEUE4_3DLUTIB_RPTR__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE4_3DLUTIB_RPTR__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE4_3DLUTIB_OFFSET
+#define VPEC_QUEUE4_3DLUTIB_OFFSET__OFFSET__SHIFT                                                             0x2
+#define VPEC_QUEUE4_3DLUTIB_OFFSET__OFFSET_MASK                                                               0x003FFFFCL
+//VPEC_QUEUE4_3DLUTIB_BASE_LO
+#define VPEC_QUEUE4_3DLUTIB_BASE_LO__ADDR__SHIFT                                                              0x5
+#define VPEC_QUEUE4_3DLUTIB_BASE_LO__ADDR_MASK                                                                0xFFFFFFE0L
+//VPEC_QUEUE4_3DLUTIB_BASE_HI
+#define VPEC_QUEUE4_3DLUTIB_BASE_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE4_3DLUTIB_BASE_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE4_3DLUTIB_SIZE
+#define VPEC_QUEUE4_3DLUTIB_SIZE__SIZE__SHIFT                                                                 0x0
+#define VPEC_QUEUE4_3DLUTIB_SIZE__SIZE_MASK                                                                   0x000FFFFFL
+//VPEC_QUEUE4_CSA_ADDR_LO
+#define VPEC_QUEUE4_CSA_ADDR_LO__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE4_CSA_ADDR_LO__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE4_CSA_ADDR_HI
+#define VPEC_QUEUE4_CSA_ADDR_HI__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE4_CSA_ADDR_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE4_CONTEXT_STATUS
+#define VPEC_QUEUE4_CONTEXT_STATUS__SELECTED__SHIFT                                                           0x0
+#define VPEC_QUEUE4_CONTEXT_STATUS__USE_IB__SHIFT                                                             0x1
+#define VPEC_QUEUE4_CONTEXT_STATUS__IDLE__SHIFT                                                               0x2
+#define VPEC_QUEUE4_CONTEXT_STATUS__EXPIRED__SHIFT                                                            0x3
+#define VPEC_QUEUE4_CONTEXT_STATUS__EXCEPTION__SHIFT                                                          0x4
+#define VPEC_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                         0x7
+#define VPEC_QUEUE4_CONTEXT_STATUS__USE_3DLUTIB__SHIFT                                                        0x8
+#define VPEC_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                    0xa
+#define VPEC_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                       0xb
+#define VPEC_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                0xc
+#define VPEC_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                             0x10
+#define VPEC_QUEUE4_CONTEXT_STATUS__SELECTED_MASK                                                             0x00000001L
+#define VPEC_QUEUE4_CONTEXT_STATUS__USE_IB_MASK                                                               0x00000002L
+#define VPEC_QUEUE4_CONTEXT_STATUS__IDLE_MASK                                                                 0x00000004L
+#define VPEC_QUEUE4_CONTEXT_STATUS__EXPIRED_MASK                                                              0x00000008L
+#define VPEC_QUEUE4_CONTEXT_STATUS__EXCEPTION_MASK                                                            0x00000070L
+#define VPEC_QUEUE4_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                           0x00000080L
+#define VPEC_QUEUE4_CONTEXT_STATUS__USE_3DLUTIB_MASK                                                          0x00000100L
+#define VPEC_QUEUE4_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                      0x00000400L
+#define VPEC_QUEUE4_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                         0x00000800L
+#define VPEC_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                  0x00001000L
+#define VPEC_QUEUE4_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                               0x00FF0000L
+//VPEC_QUEUE4_DOORBELL_LOG
+#define VPEC_QUEUE4_DOORBELL_LOG__BE_ERROR__SHIFT                                                             0x0
+#define VPEC_QUEUE4_DOORBELL_LOG__DATA__SHIFT                                                                 0x2
+#define VPEC_QUEUE4_DOORBELL_LOG__BE_ERROR_MASK                                                               0x00000001L
+#define VPEC_QUEUE4_DOORBELL_LOG__DATA_MASK                                                                   0xFFFFFFFCL
+//VPEC_QUEUE4_IB_SUB_REMAIN
+#define VPEC_QUEUE4_IB_SUB_REMAIN__SIZE__SHIFT                                                                0x0
+#define VPEC_QUEUE4_IB_SUB_REMAIN__SIZE_MASK                                                                  0x00003FFFL
+//VPEC_QUEUE4_PREEMPT
+#define VPEC_QUEUE4_PREEMPT__IB_PREEMPT__SHIFT                                                                0x0
+#define VPEC_QUEUE4_PREEMPT__IB_PREEMPT_MASK                                                                  0x00000001L
+//VPEC_QUEUE4_LOG0BUFFER_CFG
+#define VPEC_QUEUE4_LOG0BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE4_LOG0BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE4_LOG0BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE4_LOG0BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE4_LOG0BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE4_LOG0BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE4_LOG0BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE4_LOG0BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE4_LOG1BUFFER_CFG
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__PARTIAL_ENTRY__SHIFT                                                      0x1
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__PARTIAL_ENTRY_MASK                                                        0x00000002L
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE4_LOG1BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE5_RB_CNTL
+#define VPEC_QUEUE5_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE5_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define VPEC_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                          0x8
+#define VPEC_QUEUE5_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                            0x9
+#define VPEC_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                     0xa
+#define VPEC_QUEUE5_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                      0xb
+#define VPEC_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                     0xc
+#define VPEC_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                0xd
+#define VPEC_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                      0x10
+#define VPEC_QUEUE5_RB_CNTL__RB_PRIV__SHIFT                                                                   0x17
+#define VPEC_QUEUE5_RB_CNTL__RB_VMID__SHIFT                                                                   0x18
+#define VPEC_QUEUE5_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE5_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define VPEC_QUEUE5_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                            0x00000100L
+#define VPEC_QUEUE5_RB_CNTL__RB_SWAP_ENABLE_MASK                                                              0x00000200L
+#define VPEC_QUEUE5_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                       0x00000400L
+#define VPEC_QUEUE5_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                        0x00000800L
+#define VPEC_QUEUE5_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                       0x00001000L
+#define VPEC_QUEUE5_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                  0x00002000L
+#define VPEC_QUEUE5_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                        0x001F0000L
+#define VPEC_QUEUE5_RB_CNTL__RB_PRIV_MASK                                                                     0x00800000L
+#define VPEC_QUEUE5_RB_CNTL__RB_VMID_MASK                                                                     0x0F000000L
+//VPEC_QUEUE5_SCHEDULE_CNTL
+#define VPEC_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                           0x0
+#define VPEC_QUEUE5_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                          0x2
+#define VPEC_QUEUE5_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                            0x6
+#define VPEC_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                     0x8
+#define VPEC_QUEUE5_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                             0x00000003L
+#define VPEC_QUEUE5_SCHEDULE_CNTL__PROCESS_ID_MASK                                                            0x0000001CL
+#define VPEC_QUEUE5_SCHEDULE_CNTL__LOCAL_ID_MASK                                                              0x000000C0L
+#define VPEC_QUEUE5_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                       0x0000FF00L
+//VPEC_QUEUE5_RB_BASE
+#define VPEC_QUEUE5_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define VPEC_QUEUE5_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE5_RB_BASE_HI
+#define VPEC_QUEUE5_RB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE5_RB_BASE_HI__ADDR_MASK                                                                     0x00FFFFFFL
+//VPEC_QUEUE5_RB_RPTR
+#define VPEC_QUEUE5_RB_RPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE5_RB_RPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE5_RB_RPTR_HI
+#define VPEC_QUEUE5_RB_RPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE5_RB_RPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE5_RB_WPTR
+#define VPEC_QUEUE5_RB_WPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE5_RB_WPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE5_RB_WPTR_HI
+#define VPEC_QUEUE5_RB_WPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE5_RB_WPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE5_RB_RPTR_ADDR_HI
+#define VPEC_QUEUE5_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE5_RB_RPTR_ADDR_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE5_RB_RPTR_ADDR_LO
+#define VPEC_QUEUE5_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define VPEC_QUEUE5_RB_RPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//VPEC_QUEUE5_RB_AQL_CNTL
+#define VPEC_QUEUE5_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                       0x1
+#define VPEC_QUEUE5_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                           0x8
+#define VPEC_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                 0x10
+#define VPEC_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                           0x11
+#define VPEC_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                        0x12
+#define VPEC_QUEUE5_RB_AQL_CNTL__AQL_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE5_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                         0x000000FEL
+#define VPEC_QUEUE5_RB_AQL_CNTL__PACKET_STEP_MASK                                                             0x0000FF00L
+#define VPEC_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                   0x00010000L
+#define VPEC_QUEUE5_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                             0x00020000L
+#define VPEC_QUEUE5_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                          0x00040000L
+//VPEC_QUEUE5_MINOR_PTR_UPDATE
+#define VPEC_QUEUE5_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                           0x0
+#define VPEC_QUEUE5_MINOR_PTR_UPDATE__ENABLE_MASK                                                             0x00000001L
+//VPEC_QUEUE5_CD_INFO
+#define VPEC_QUEUE5_CD_INFO__CD_INFO__SHIFT                                                                   0x0
+#define VPEC_QUEUE5_CD_INFO__CD_INFO_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE5_RB_PREEMPT
+#define VPEC_QUEUE5_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                            0x0
+#define VPEC_QUEUE5_RB_PREEMPT__PREEMPT_REQ_MASK                                                              0x00000001L
+//VPEC_QUEUE5_SKIP_CNTL
+#define VPEC_QUEUE5_SKIP_CNTL__SKIP_COUNT__SHIFT                                                              0x0
+#define VPEC_QUEUE5_SKIP_CNTL__SKIP_COUNT_MASK                                                                0x000FFFFFL
+//VPEC_QUEUE5_DOORBELL
+#define VPEC_QUEUE5_DOORBELL__ENABLE__SHIFT                                                                   0x1c
+#define VPEC_QUEUE5_DOORBELL__CAPTURED__SHIFT                                                                 0x1e
+#define VPEC_QUEUE5_DOORBELL__ENABLE_MASK                                                                     0x10000000L
+#define VPEC_QUEUE5_DOORBELL__CAPTURED_MASK                                                                   0x40000000L
+//VPEC_QUEUE5_DOORBELL_OFFSET
+#define VPEC_QUEUE5_DOORBELL_OFFSET__OFFSET__SHIFT                                                            0x2
+#define VPEC_QUEUE5_DOORBELL_OFFSET__OFFSET_MASK                                                              0x0FFFFFFCL
+//VPEC_QUEUE5_DUMMY0
+#define VPEC_QUEUE5_DUMMY0__DUMMY__SHIFT                                                                      0x0
+#define VPEC_QUEUE5_DUMMY0__DUMMY_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE5_DUMMY1
+#define VPEC_QUEUE5_DUMMY1__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE5_DUMMY1__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE5_DUMMY2
+#define VPEC_QUEUE5_DUMMY2__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE5_DUMMY2__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE5_DUMMY3
+#define VPEC_QUEUE5_DUMMY3__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE5_DUMMY3__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE5_DUMMY4
+#define VPEC_QUEUE5_DUMMY4__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE5_DUMMY4__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE5_IB_CNTL
+#define VPEC_QUEUE5_IB_CNTL__IB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                          0x8
+#define VPEC_QUEUE5_IB_CNTL__CMD_VMID__SHIFT                                                                  0x10
+#define VPEC_QUEUE5_IB_CNTL__IB_PRIV__SHIFT                                                                   0x1f
+#define VPEC_QUEUE5_IB_CNTL__IB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE5_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                            0x00000100L
+#define VPEC_QUEUE5_IB_CNTL__CMD_VMID_MASK                                                                    0x000F0000L
+#define VPEC_QUEUE5_IB_CNTL__IB_PRIV_MASK                                                                     0x80000000L
+//VPEC_QUEUE5_IB_RPTR
+#define VPEC_QUEUE5_IB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define VPEC_QUEUE5_IB_RPTR__OFFSET_MASK                                                                      0x003FFFFCL
+//VPEC_QUEUE5_IB_OFFSET
+#define VPEC_QUEUE5_IB_OFFSET__OFFSET__SHIFT                                                                  0x2
+#define VPEC_QUEUE5_IB_OFFSET__OFFSET_MASK                                                                    0x003FFFFCL
+//VPEC_QUEUE5_IB_BASE_LO
+#define VPEC_QUEUE5_IB_BASE_LO__ADDR__SHIFT                                                                   0x5
+#define VPEC_QUEUE5_IB_BASE_LO__ADDR_MASK                                                                     0xFFFFFFE0L
+//VPEC_QUEUE5_IB_BASE_HI
+#define VPEC_QUEUE5_IB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE5_IB_BASE_HI__ADDR_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE5_IB_SIZE
+#define VPEC_QUEUE5_IB_SIZE__SIZE__SHIFT                                                                      0x0
+#define VPEC_QUEUE5_IB_SIZE__SIZE_MASK                                                                        0x000FFFFFL
+//VPEC_QUEUE5_CMDIB_CNTL
+#define VPEC_QUEUE5_CMDIB_CNTL__IB_ENABLE__SHIFT                                                              0x0
+#define VPEC_QUEUE5_CMDIB_CNTL__IB_SWAP_ENABLE__SHIFT                                                         0x4
+#define VPEC_QUEUE5_CMDIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                       0x8
+#define VPEC_QUEUE5_CMDIB_CNTL__CMD_VMID__SHIFT                                                               0x10
+#define VPEC_QUEUE5_CMDIB_CNTL__IB_PRIV__SHIFT                                                                0x1f
+#define VPEC_QUEUE5_CMDIB_CNTL__IB_ENABLE_MASK                                                                0x00000001L
+#define VPEC_QUEUE5_CMDIB_CNTL__IB_SWAP_ENABLE_MASK                                                           0x00000010L
+#define VPEC_QUEUE5_CMDIB_CNTL__SWITCH_INSIDE_IB_MASK                                                         0x00000100L
+#define VPEC_QUEUE5_CMDIB_CNTL__CMD_VMID_MASK                                                                 0x000F0000L
+#define VPEC_QUEUE5_CMDIB_CNTL__IB_PRIV_MASK                                                                  0x80000000L
+//VPEC_QUEUE5_CMDIB_RPTR
+#define VPEC_QUEUE5_CMDIB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define VPEC_QUEUE5_CMDIB_RPTR__OFFSET_MASK                                                                   0x003FFFFCL
+//VPEC_QUEUE5_CMDIB_OFFSET
+#define VPEC_QUEUE5_CMDIB_OFFSET__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE5_CMDIB_OFFSET__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE5_CMDIB_BASE_LO
+#define VPEC_QUEUE5_CMDIB_BASE_LO__ADDR__SHIFT                                                                0x5
+#define VPEC_QUEUE5_CMDIB_BASE_LO__ADDR_MASK                                                                  0xFFFFFFE0L
+//VPEC_QUEUE5_CMDIB_BASE_HI
+#define VPEC_QUEUE5_CMDIB_BASE_HI__ADDR__SHIFT                                                                0x0
+#define VPEC_QUEUE5_CMDIB_BASE_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//VPEC_QUEUE5_CMDIB_SIZE
+#define VPEC_QUEUE5_CMDIB_SIZE__SIZE__SHIFT                                                                   0x0
+#define VPEC_QUEUE5_CMDIB_SIZE__SIZE_MASK                                                                     0x000FFFFFL
+//VPEC_QUEUE5_3DLUTIB_CNTL
+#define VPEC_QUEUE5_3DLUTIB_CNTL__IB_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE5_3DLUTIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                     0x8
+#define VPEC_QUEUE5_3DLUTIB_CNTL__CMD_VMID__SHIFT                                                             0x10
+#define VPEC_QUEUE5_3DLUTIB_CNTL__IB_PRIV__SHIFT                                                              0x1f
+#define VPEC_QUEUE5_3DLUTIB_CNTL__IB_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE5_3DLUTIB_CNTL__SWITCH_INSIDE_IB_MASK                                                       0x00000100L
+#define VPEC_QUEUE5_3DLUTIB_CNTL__CMD_VMID_MASK                                                               0x000F0000L
+#define VPEC_QUEUE5_3DLUTIB_CNTL__IB_PRIV_MASK                                                                0x80000000L
+//VPEC_QUEUE5_3DLUTIB_RPTR
+#define VPEC_QUEUE5_3DLUTIB_RPTR__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE5_3DLUTIB_RPTR__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE5_3DLUTIB_OFFSET
+#define VPEC_QUEUE5_3DLUTIB_OFFSET__OFFSET__SHIFT                                                             0x2
+#define VPEC_QUEUE5_3DLUTIB_OFFSET__OFFSET_MASK                                                               0x003FFFFCL
+//VPEC_QUEUE5_3DLUTIB_BASE_LO
+#define VPEC_QUEUE5_3DLUTIB_BASE_LO__ADDR__SHIFT                                                              0x5
+#define VPEC_QUEUE5_3DLUTIB_BASE_LO__ADDR_MASK                                                                0xFFFFFFE0L
+//VPEC_QUEUE5_3DLUTIB_BASE_HI
+#define VPEC_QUEUE5_3DLUTIB_BASE_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE5_3DLUTIB_BASE_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE5_3DLUTIB_SIZE
+#define VPEC_QUEUE5_3DLUTIB_SIZE__SIZE__SHIFT                                                                 0x0
+#define VPEC_QUEUE5_3DLUTIB_SIZE__SIZE_MASK                                                                   0x000FFFFFL
+//VPEC_QUEUE5_CSA_ADDR_LO
+#define VPEC_QUEUE5_CSA_ADDR_LO__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE5_CSA_ADDR_LO__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE5_CSA_ADDR_HI
+#define VPEC_QUEUE5_CSA_ADDR_HI__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE5_CSA_ADDR_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE5_CONTEXT_STATUS
+#define VPEC_QUEUE5_CONTEXT_STATUS__SELECTED__SHIFT                                                           0x0
+#define VPEC_QUEUE5_CONTEXT_STATUS__USE_IB__SHIFT                                                             0x1
+#define VPEC_QUEUE5_CONTEXT_STATUS__IDLE__SHIFT                                                               0x2
+#define VPEC_QUEUE5_CONTEXT_STATUS__EXPIRED__SHIFT                                                            0x3
+#define VPEC_QUEUE5_CONTEXT_STATUS__EXCEPTION__SHIFT                                                          0x4
+#define VPEC_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                         0x7
+#define VPEC_QUEUE5_CONTEXT_STATUS__USE_3DLUTIB__SHIFT                                                        0x8
+#define VPEC_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                    0xa
+#define VPEC_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                       0xb
+#define VPEC_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                0xc
+#define VPEC_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                             0x10
+#define VPEC_QUEUE5_CONTEXT_STATUS__SELECTED_MASK                                                             0x00000001L
+#define VPEC_QUEUE5_CONTEXT_STATUS__USE_IB_MASK                                                               0x00000002L
+#define VPEC_QUEUE5_CONTEXT_STATUS__IDLE_MASK                                                                 0x00000004L
+#define VPEC_QUEUE5_CONTEXT_STATUS__EXPIRED_MASK                                                              0x00000008L
+#define VPEC_QUEUE5_CONTEXT_STATUS__EXCEPTION_MASK                                                            0x00000070L
+#define VPEC_QUEUE5_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                           0x00000080L
+#define VPEC_QUEUE5_CONTEXT_STATUS__USE_3DLUTIB_MASK                                                          0x00000100L
+#define VPEC_QUEUE5_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                      0x00000400L
+#define VPEC_QUEUE5_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                         0x00000800L
+#define VPEC_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                  0x00001000L
+#define VPEC_QUEUE5_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                               0x00FF0000L
+//VPEC_QUEUE5_DOORBELL_LOG
+#define VPEC_QUEUE5_DOORBELL_LOG__BE_ERROR__SHIFT                                                             0x0
+#define VPEC_QUEUE5_DOORBELL_LOG__DATA__SHIFT                                                                 0x2
+#define VPEC_QUEUE5_DOORBELL_LOG__BE_ERROR_MASK                                                               0x00000001L
+#define VPEC_QUEUE5_DOORBELL_LOG__DATA_MASK                                                                   0xFFFFFFFCL
+//VPEC_QUEUE5_IB_SUB_REMAIN
+#define VPEC_QUEUE5_IB_SUB_REMAIN__SIZE__SHIFT                                                                0x0
+#define VPEC_QUEUE5_IB_SUB_REMAIN__SIZE_MASK                                                                  0x00003FFFL
+//VPEC_QUEUE5_PREEMPT
+#define VPEC_QUEUE5_PREEMPT__IB_PREEMPT__SHIFT                                                                0x0
+#define VPEC_QUEUE5_PREEMPT__IB_PREEMPT_MASK                                                                  0x00000001L
+//VPEC_QUEUE5_LOG0BUFFER_CFG
+#define VPEC_QUEUE5_LOG0BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE5_LOG0BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE5_LOG0BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE5_LOG0BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE5_LOG0BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE5_LOG0BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE5_LOG0BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE5_LOG0BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE5_LOG1BUFFER_CFG
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__PARTIAL_ENTRY__SHIFT                                                      0x1
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__PARTIAL_ENTRY_MASK                                                        0x00000002L
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE5_LOG1BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE6_RB_CNTL
+#define VPEC_QUEUE6_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE6_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define VPEC_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                          0x8
+#define VPEC_QUEUE6_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                            0x9
+#define VPEC_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                     0xa
+#define VPEC_QUEUE6_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                      0xb
+#define VPEC_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                     0xc
+#define VPEC_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                0xd
+#define VPEC_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                      0x10
+#define VPEC_QUEUE6_RB_CNTL__RB_PRIV__SHIFT                                                                   0x17
+#define VPEC_QUEUE6_RB_CNTL__RB_VMID__SHIFT                                                                   0x18
+#define VPEC_QUEUE6_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE6_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define VPEC_QUEUE6_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                            0x00000100L
+#define VPEC_QUEUE6_RB_CNTL__RB_SWAP_ENABLE_MASK                                                              0x00000200L
+#define VPEC_QUEUE6_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                       0x00000400L
+#define VPEC_QUEUE6_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                        0x00000800L
+#define VPEC_QUEUE6_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                       0x00001000L
+#define VPEC_QUEUE6_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                  0x00002000L
+#define VPEC_QUEUE6_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                        0x001F0000L
+#define VPEC_QUEUE6_RB_CNTL__RB_PRIV_MASK                                                                     0x00800000L
+#define VPEC_QUEUE6_RB_CNTL__RB_VMID_MASK                                                                     0x0F000000L
+//VPEC_QUEUE6_SCHEDULE_CNTL
+#define VPEC_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                           0x0
+#define VPEC_QUEUE6_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                          0x2
+#define VPEC_QUEUE6_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                            0x6
+#define VPEC_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                     0x8
+#define VPEC_QUEUE6_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                             0x00000003L
+#define VPEC_QUEUE6_SCHEDULE_CNTL__PROCESS_ID_MASK                                                            0x0000001CL
+#define VPEC_QUEUE6_SCHEDULE_CNTL__LOCAL_ID_MASK                                                              0x000000C0L
+#define VPEC_QUEUE6_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                       0x0000FF00L
+//VPEC_QUEUE6_RB_BASE
+#define VPEC_QUEUE6_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define VPEC_QUEUE6_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE6_RB_BASE_HI
+#define VPEC_QUEUE6_RB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE6_RB_BASE_HI__ADDR_MASK                                                                     0x00FFFFFFL
+//VPEC_QUEUE6_RB_RPTR
+#define VPEC_QUEUE6_RB_RPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE6_RB_RPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE6_RB_RPTR_HI
+#define VPEC_QUEUE6_RB_RPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE6_RB_RPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE6_RB_WPTR
+#define VPEC_QUEUE6_RB_WPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE6_RB_WPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE6_RB_WPTR_HI
+#define VPEC_QUEUE6_RB_WPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE6_RB_WPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE6_RB_RPTR_ADDR_HI
+#define VPEC_QUEUE6_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE6_RB_RPTR_ADDR_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE6_RB_RPTR_ADDR_LO
+#define VPEC_QUEUE6_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define VPEC_QUEUE6_RB_RPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//VPEC_QUEUE6_RB_AQL_CNTL
+#define VPEC_QUEUE6_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                       0x1
+#define VPEC_QUEUE6_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                           0x8
+#define VPEC_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                 0x10
+#define VPEC_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                           0x11
+#define VPEC_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                        0x12
+#define VPEC_QUEUE6_RB_AQL_CNTL__AQL_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE6_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                         0x000000FEL
+#define VPEC_QUEUE6_RB_AQL_CNTL__PACKET_STEP_MASK                                                             0x0000FF00L
+#define VPEC_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                   0x00010000L
+#define VPEC_QUEUE6_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                             0x00020000L
+#define VPEC_QUEUE6_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                          0x00040000L
+//VPEC_QUEUE6_MINOR_PTR_UPDATE
+#define VPEC_QUEUE6_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                           0x0
+#define VPEC_QUEUE6_MINOR_PTR_UPDATE__ENABLE_MASK                                                             0x00000001L
+//VPEC_QUEUE6_CD_INFO
+#define VPEC_QUEUE6_CD_INFO__CD_INFO__SHIFT                                                                   0x0
+#define VPEC_QUEUE6_CD_INFO__CD_INFO_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE6_RB_PREEMPT
+#define VPEC_QUEUE6_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                            0x0
+#define VPEC_QUEUE6_RB_PREEMPT__PREEMPT_REQ_MASK                                                              0x00000001L
+//VPEC_QUEUE6_SKIP_CNTL
+#define VPEC_QUEUE6_SKIP_CNTL__SKIP_COUNT__SHIFT                                                              0x0
+#define VPEC_QUEUE6_SKIP_CNTL__SKIP_COUNT_MASK                                                                0x000FFFFFL
+//VPEC_QUEUE6_DOORBELL
+#define VPEC_QUEUE6_DOORBELL__ENABLE__SHIFT                                                                   0x1c
+#define VPEC_QUEUE6_DOORBELL__CAPTURED__SHIFT                                                                 0x1e
+#define VPEC_QUEUE6_DOORBELL__ENABLE_MASK                                                                     0x10000000L
+#define VPEC_QUEUE6_DOORBELL__CAPTURED_MASK                                                                   0x40000000L
+//VPEC_QUEUE6_DOORBELL_OFFSET
+#define VPEC_QUEUE6_DOORBELL_OFFSET__OFFSET__SHIFT                                                            0x2
+#define VPEC_QUEUE6_DOORBELL_OFFSET__OFFSET_MASK                                                              0x0FFFFFFCL
+//VPEC_QUEUE6_DUMMY0
+#define VPEC_QUEUE6_DUMMY0__DUMMY__SHIFT                                                                      0x0
+#define VPEC_QUEUE6_DUMMY0__DUMMY_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE6_DUMMY1
+#define VPEC_QUEUE6_DUMMY1__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE6_DUMMY1__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE6_DUMMY2
+#define VPEC_QUEUE6_DUMMY2__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE6_DUMMY2__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE6_DUMMY3
+#define VPEC_QUEUE6_DUMMY3__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE6_DUMMY3__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE6_DUMMY4
+#define VPEC_QUEUE6_DUMMY4__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE6_DUMMY4__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE6_IB_CNTL
+#define VPEC_QUEUE6_IB_CNTL__IB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                          0x8
+#define VPEC_QUEUE6_IB_CNTL__CMD_VMID__SHIFT                                                                  0x10
+#define VPEC_QUEUE6_IB_CNTL__IB_PRIV__SHIFT                                                                   0x1f
+#define VPEC_QUEUE6_IB_CNTL__IB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE6_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                            0x00000100L
+#define VPEC_QUEUE6_IB_CNTL__CMD_VMID_MASK                                                                    0x000F0000L
+#define VPEC_QUEUE6_IB_CNTL__IB_PRIV_MASK                                                                     0x80000000L
+//VPEC_QUEUE6_IB_RPTR
+#define VPEC_QUEUE6_IB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define VPEC_QUEUE6_IB_RPTR__OFFSET_MASK                                                                      0x003FFFFCL
+//VPEC_QUEUE6_IB_OFFSET
+#define VPEC_QUEUE6_IB_OFFSET__OFFSET__SHIFT                                                                  0x2
+#define VPEC_QUEUE6_IB_OFFSET__OFFSET_MASK                                                                    0x003FFFFCL
+//VPEC_QUEUE6_IB_BASE_LO
+#define VPEC_QUEUE6_IB_BASE_LO__ADDR__SHIFT                                                                   0x5
+#define VPEC_QUEUE6_IB_BASE_LO__ADDR_MASK                                                                     0xFFFFFFE0L
+//VPEC_QUEUE6_IB_BASE_HI
+#define VPEC_QUEUE6_IB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE6_IB_BASE_HI__ADDR_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE6_IB_SIZE
+#define VPEC_QUEUE6_IB_SIZE__SIZE__SHIFT                                                                      0x0
+#define VPEC_QUEUE6_IB_SIZE__SIZE_MASK                                                                        0x000FFFFFL
+//VPEC_QUEUE6_CMDIB_CNTL
+#define VPEC_QUEUE6_CMDIB_CNTL__IB_ENABLE__SHIFT                                                              0x0
+#define VPEC_QUEUE6_CMDIB_CNTL__IB_SWAP_ENABLE__SHIFT                                                         0x4
+#define VPEC_QUEUE6_CMDIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                       0x8
+#define VPEC_QUEUE6_CMDIB_CNTL__CMD_VMID__SHIFT                                                               0x10
+#define VPEC_QUEUE6_CMDIB_CNTL__IB_PRIV__SHIFT                                                                0x1f
+#define VPEC_QUEUE6_CMDIB_CNTL__IB_ENABLE_MASK                                                                0x00000001L
+#define VPEC_QUEUE6_CMDIB_CNTL__IB_SWAP_ENABLE_MASK                                                           0x00000010L
+#define VPEC_QUEUE6_CMDIB_CNTL__SWITCH_INSIDE_IB_MASK                                                         0x00000100L
+#define VPEC_QUEUE6_CMDIB_CNTL__CMD_VMID_MASK                                                                 0x000F0000L
+#define VPEC_QUEUE6_CMDIB_CNTL__IB_PRIV_MASK                                                                  0x80000000L
+//VPEC_QUEUE6_CMDIB_RPTR
+#define VPEC_QUEUE6_CMDIB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define VPEC_QUEUE6_CMDIB_RPTR__OFFSET_MASK                                                                   0x003FFFFCL
+//VPEC_QUEUE6_CMDIB_OFFSET
+#define VPEC_QUEUE6_CMDIB_OFFSET__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE6_CMDIB_OFFSET__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE6_CMDIB_BASE_LO
+#define VPEC_QUEUE6_CMDIB_BASE_LO__ADDR__SHIFT                                                                0x5
+#define VPEC_QUEUE6_CMDIB_BASE_LO__ADDR_MASK                                                                  0xFFFFFFE0L
+//VPEC_QUEUE6_CMDIB_BASE_HI
+#define VPEC_QUEUE6_CMDIB_BASE_HI__ADDR__SHIFT                                                                0x0
+#define VPEC_QUEUE6_CMDIB_BASE_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//VPEC_QUEUE6_CMDIB_SIZE
+#define VPEC_QUEUE6_CMDIB_SIZE__SIZE__SHIFT                                                                   0x0
+#define VPEC_QUEUE6_CMDIB_SIZE__SIZE_MASK                                                                     0x000FFFFFL
+//VPEC_QUEUE6_3DLUTIB_CNTL
+#define VPEC_QUEUE6_3DLUTIB_CNTL__IB_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE6_3DLUTIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                     0x8
+#define VPEC_QUEUE6_3DLUTIB_CNTL__CMD_VMID__SHIFT                                                             0x10
+#define VPEC_QUEUE6_3DLUTIB_CNTL__IB_PRIV__SHIFT                                                              0x1f
+#define VPEC_QUEUE6_3DLUTIB_CNTL__IB_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE6_3DLUTIB_CNTL__SWITCH_INSIDE_IB_MASK                                                       0x00000100L
+#define VPEC_QUEUE6_3DLUTIB_CNTL__CMD_VMID_MASK                                                               0x000F0000L
+#define VPEC_QUEUE6_3DLUTIB_CNTL__IB_PRIV_MASK                                                                0x80000000L
+//VPEC_QUEUE6_3DLUTIB_RPTR
+#define VPEC_QUEUE6_3DLUTIB_RPTR__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE6_3DLUTIB_RPTR__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE6_3DLUTIB_OFFSET
+#define VPEC_QUEUE6_3DLUTIB_OFFSET__OFFSET__SHIFT                                                             0x2
+#define VPEC_QUEUE6_3DLUTIB_OFFSET__OFFSET_MASK                                                               0x003FFFFCL
+//VPEC_QUEUE6_3DLUTIB_BASE_LO
+#define VPEC_QUEUE6_3DLUTIB_BASE_LO__ADDR__SHIFT                                                              0x5
+#define VPEC_QUEUE6_3DLUTIB_BASE_LO__ADDR_MASK                                                                0xFFFFFFE0L
+//VPEC_QUEUE6_3DLUTIB_BASE_HI
+#define VPEC_QUEUE6_3DLUTIB_BASE_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE6_3DLUTIB_BASE_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE6_3DLUTIB_SIZE
+#define VPEC_QUEUE6_3DLUTIB_SIZE__SIZE__SHIFT                                                                 0x0
+#define VPEC_QUEUE6_3DLUTIB_SIZE__SIZE_MASK                                                                   0x000FFFFFL
+//VPEC_QUEUE6_CSA_ADDR_LO
+#define VPEC_QUEUE6_CSA_ADDR_LO__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE6_CSA_ADDR_LO__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE6_CSA_ADDR_HI
+#define VPEC_QUEUE6_CSA_ADDR_HI__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE6_CSA_ADDR_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE6_CONTEXT_STATUS
+#define VPEC_QUEUE6_CONTEXT_STATUS__SELECTED__SHIFT                                                           0x0
+#define VPEC_QUEUE6_CONTEXT_STATUS__USE_IB__SHIFT                                                             0x1
+#define VPEC_QUEUE6_CONTEXT_STATUS__IDLE__SHIFT                                                               0x2
+#define VPEC_QUEUE6_CONTEXT_STATUS__EXPIRED__SHIFT                                                            0x3
+#define VPEC_QUEUE6_CONTEXT_STATUS__EXCEPTION__SHIFT                                                          0x4
+#define VPEC_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                         0x7
+#define VPEC_QUEUE6_CONTEXT_STATUS__USE_3DLUTIB__SHIFT                                                        0x8
+#define VPEC_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                    0xa
+#define VPEC_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                       0xb
+#define VPEC_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                0xc
+#define VPEC_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                             0x10
+#define VPEC_QUEUE6_CONTEXT_STATUS__SELECTED_MASK                                                             0x00000001L
+#define VPEC_QUEUE6_CONTEXT_STATUS__USE_IB_MASK                                                               0x00000002L
+#define VPEC_QUEUE6_CONTEXT_STATUS__IDLE_MASK                                                                 0x00000004L
+#define VPEC_QUEUE6_CONTEXT_STATUS__EXPIRED_MASK                                                              0x00000008L
+#define VPEC_QUEUE6_CONTEXT_STATUS__EXCEPTION_MASK                                                            0x00000070L
+#define VPEC_QUEUE6_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                           0x00000080L
+#define VPEC_QUEUE6_CONTEXT_STATUS__USE_3DLUTIB_MASK                                                          0x00000100L
+#define VPEC_QUEUE6_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                      0x00000400L
+#define VPEC_QUEUE6_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                         0x00000800L
+#define VPEC_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                  0x00001000L
+#define VPEC_QUEUE6_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                               0x00FF0000L
+//VPEC_QUEUE6_DOORBELL_LOG
+#define VPEC_QUEUE6_DOORBELL_LOG__BE_ERROR__SHIFT                                                             0x0
+#define VPEC_QUEUE6_DOORBELL_LOG__DATA__SHIFT                                                                 0x2
+#define VPEC_QUEUE6_DOORBELL_LOG__BE_ERROR_MASK                                                               0x00000001L
+#define VPEC_QUEUE6_DOORBELL_LOG__DATA_MASK                                                                   0xFFFFFFFCL
+//VPEC_QUEUE6_IB_SUB_REMAIN
+#define VPEC_QUEUE6_IB_SUB_REMAIN__SIZE__SHIFT                                                                0x0
+#define VPEC_QUEUE6_IB_SUB_REMAIN__SIZE_MASK                                                                  0x00003FFFL
+//VPEC_QUEUE6_PREEMPT
+#define VPEC_QUEUE6_PREEMPT__IB_PREEMPT__SHIFT                                                                0x0
+#define VPEC_QUEUE6_PREEMPT__IB_PREEMPT_MASK                                                                  0x00000001L
+//VPEC_QUEUE6_LOG0BUFFER_CFG
+#define VPEC_QUEUE6_LOG0BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE6_LOG0BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE6_LOG0BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE6_LOG0BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE6_LOG0BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE6_LOG0BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE6_LOG0BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE6_LOG0BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE6_LOG1BUFFER_CFG
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__PARTIAL_ENTRY__SHIFT                                                      0x1
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__PARTIAL_ENTRY_MASK                                                        0x00000002L
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE6_LOG1BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE7_RB_CNTL
+#define VPEC_QUEUE7_RB_CNTL__RB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE7_RB_CNTL__RB_SIZE__SHIFT                                                                   0x1
+#define VPEC_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE__SHIFT                                                          0x8
+#define VPEC_QUEUE7_RB_CNTL__RB_SWAP_ENABLE__SHIFT                                                            0x9
+#define VPEC_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE__SHIFT                                                     0xa
+#define VPEC_QUEUE7_RB_CNTL__F32_WPTR_POLL_ENABLE__SHIFT                                                      0xb
+#define VPEC_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE__SHIFT                                                     0xc
+#define VPEC_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE__SHIFT                                                0xd
+#define VPEC_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER__SHIFT                                                      0x10
+#define VPEC_QUEUE7_RB_CNTL__RB_PRIV__SHIFT                                                                   0x17
+#define VPEC_QUEUE7_RB_CNTL__RB_VMID__SHIFT                                                                   0x18
+#define VPEC_QUEUE7_RB_CNTL__RB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE7_RB_CNTL__RB_SIZE_MASK                                                                     0x0000003EL
+#define VPEC_QUEUE7_RB_CNTL__WPTR_POLL_ENABLE_MASK                                                            0x00000100L
+#define VPEC_QUEUE7_RB_CNTL__RB_SWAP_ENABLE_MASK                                                              0x00000200L
+#define VPEC_QUEUE7_RB_CNTL__WPTR_POLL_SWAP_ENABLE_MASK                                                       0x00000400L
+#define VPEC_QUEUE7_RB_CNTL__F32_WPTR_POLL_ENABLE_MASK                                                        0x00000800L
+#define VPEC_QUEUE7_RB_CNTL__RPTR_WRITEBACK_ENABLE_MASK                                                       0x00001000L
+#define VPEC_QUEUE7_RB_CNTL__RPTR_WRITEBACK_SWAP_ENABLE_MASK                                                  0x00002000L
+#define VPEC_QUEUE7_RB_CNTL__RPTR_WRITEBACK_TIMER_MASK                                                        0x001F0000L
+#define VPEC_QUEUE7_RB_CNTL__RB_PRIV_MASK                                                                     0x00800000L
+#define VPEC_QUEUE7_RB_CNTL__RB_VMID_MASK                                                                     0x0F000000L
+//VPEC_QUEUE7_SCHEDULE_CNTL
+#define VPEC_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID__SHIFT                                                           0x0
+#define VPEC_QUEUE7_SCHEDULE_CNTL__PROCESS_ID__SHIFT                                                          0x2
+#define VPEC_QUEUE7_SCHEDULE_CNTL__LOCAL_ID__SHIFT                                                            0x6
+#define VPEC_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM__SHIFT                                                     0x8
+#define VPEC_QUEUE7_SCHEDULE_CNTL__GLOBAL_ID_MASK                                                             0x00000003L
+#define VPEC_QUEUE7_SCHEDULE_CNTL__PROCESS_ID_MASK                                                            0x0000001CL
+#define VPEC_QUEUE7_SCHEDULE_CNTL__LOCAL_ID_MASK                                                              0x000000C0L
+#define VPEC_QUEUE7_SCHEDULE_CNTL__CONTEXT_QUANTUM_MASK                                                       0x0000FF00L
+//VPEC_QUEUE7_RB_BASE
+#define VPEC_QUEUE7_RB_BASE__ADDR__SHIFT                                                                      0x0
+#define VPEC_QUEUE7_RB_BASE__ADDR_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE7_RB_BASE_HI
+#define VPEC_QUEUE7_RB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE7_RB_BASE_HI__ADDR_MASK                                                                     0x00FFFFFFL
+//VPEC_QUEUE7_RB_RPTR
+#define VPEC_QUEUE7_RB_RPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE7_RB_RPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE7_RB_RPTR_HI
+#define VPEC_QUEUE7_RB_RPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE7_RB_RPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE7_RB_WPTR
+#define VPEC_QUEUE7_RB_WPTR__OFFSET__SHIFT                                                                    0x0
+#define VPEC_QUEUE7_RB_WPTR__OFFSET_MASK                                                                      0xFFFFFFFFL
+//VPEC_QUEUE7_RB_WPTR_HI
+#define VPEC_QUEUE7_RB_WPTR_HI__OFFSET__SHIFT                                                                 0x0
+#define VPEC_QUEUE7_RB_WPTR_HI__OFFSET_MASK                                                                   0xFFFFFFFFL
+//VPEC_QUEUE7_RB_RPTR_ADDR_HI
+#define VPEC_QUEUE7_RB_RPTR_ADDR_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE7_RB_RPTR_ADDR_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE7_RB_RPTR_ADDR_LO
+#define VPEC_QUEUE7_RB_RPTR_ADDR_LO__ADDR__SHIFT                                                              0x2
+#define VPEC_QUEUE7_RB_RPTR_ADDR_LO__ADDR_MASK                                                                0xFFFFFFFCL
+//VPEC_QUEUE7_RB_AQL_CNTL
+#define VPEC_QUEUE7_RB_AQL_CNTL__AQL_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE__SHIFT                                                       0x1
+#define VPEC_QUEUE7_RB_AQL_CNTL__PACKET_STEP__SHIFT                                                           0x8
+#define VPEC_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE__SHIFT                                                 0x10
+#define VPEC_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE__SHIFT                                           0x11
+#define VPEC_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE__SHIFT                                                        0x12
+#define VPEC_QUEUE7_RB_AQL_CNTL__AQL_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE7_RB_AQL_CNTL__AQL_PACKET_SIZE_MASK                                                         0x000000FEL
+#define VPEC_QUEUE7_RB_AQL_CNTL__PACKET_STEP_MASK                                                             0x0000FF00L
+#define VPEC_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_ENABLE_MASK                                                   0x00010000L
+#define VPEC_QUEUE7_RB_AQL_CNTL__MIDCMD_PREEMPT_DATA_RESTORE_MASK                                             0x00020000L
+#define VPEC_QUEUE7_RB_AQL_CNTL__OVERLAP_ENABLE_MASK                                                          0x00040000L
+//VPEC_QUEUE7_MINOR_PTR_UPDATE
+#define VPEC_QUEUE7_MINOR_PTR_UPDATE__ENABLE__SHIFT                                                           0x0
+#define VPEC_QUEUE7_MINOR_PTR_UPDATE__ENABLE_MASK                                                             0x00000001L
+//VPEC_QUEUE7_CD_INFO
+#define VPEC_QUEUE7_CD_INFO__CD_INFO__SHIFT                                                                   0x0
+#define VPEC_QUEUE7_CD_INFO__CD_INFO_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE7_RB_PREEMPT
+#define VPEC_QUEUE7_RB_PREEMPT__PREEMPT_REQ__SHIFT                                                            0x0
+#define VPEC_QUEUE7_RB_PREEMPT__PREEMPT_REQ_MASK                                                              0x00000001L
+//VPEC_QUEUE7_SKIP_CNTL
+#define VPEC_QUEUE7_SKIP_CNTL__SKIP_COUNT__SHIFT                                                              0x0
+#define VPEC_QUEUE7_SKIP_CNTL__SKIP_COUNT_MASK                                                                0x000FFFFFL
+//VPEC_QUEUE7_DOORBELL
+#define VPEC_QUEUE7_DOORBELL__ENABLE__SHIFT                                                                   0x1c
+#define VPEC_QUEUE7_DOORBELL__CAPTURED__SHIFT                                                                 0x1e
+#define VPEC_QUEUE7_DOORBELL__ENABLE_MASK                                                                     0x10000000L
+#define VPEC_QUEUE7_DOORBELL__CAPTURED_MASK                                                                   0x40000000L
+//VPEC_QUEUE7_DOORBELL_OFFSET
+#define VPEC_QUEUE7_DOORBELL_OFFSET__OFFSET__SHIFT                                                            0x2
+#define VPEC_QUEUE7_DOORBELL_OFFSET__OFFSET_MASK                                                              0x0FFFFFFCL
+//VPEC_QUEUE7_DUMMY0
+#define VPEC_QUEUE7_DUMMY0__DUMMY__SHIFT                                                                      0x0
+#define VPEC_QUEUE7_DUMMY0__DUMMY_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE7_DUMMY1
+#define VPEC_QUEUE7_DUMMY1__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE7_DUMMY1__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE7_DUMMY2
+#define VPEC_QUEUE7_DUMMY2__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE7_DUMMY2__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE7_DUMMY3
+#define VPEC_QUEUE7_DUMMY3__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE7_DUMMY3__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE7_DUMMY4
+#define VPEC_QUEUE7_DUMMY4__VALUE__SHIFT                                                                      0x0
+#define VPEC_QUEUE7_DUMMY4__VALUE_MASK                                                                        0xFFFFFFFFL
+//VPEC_QUEUE7_IB_CNTL
+#define VPEC_QUEUE7_IB_CNTL__IB_ENABLE__SHIFT                                                                 0x0
+#define VPEC_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                          0x8
+#define VPEC_QUEUE7_IB_CNTL__CMD_VMID__SHIFT                                                                  0x10
+#define VPEC_QUEUE7_IB_CNTL__IB_PRIV__SHIFT                                                                   0x1f
+#define VPEC_QUEUE7_IB_CNTL__IB_ENABLE_MASK                                                                   0x00000001L
+#define VPEC_QUEUE7_IB_CNTL__SWITCH_INSIDE_IB_MASK                                                            0x00000100L
+#define VPEC_QUEUE7_IB_CNTL__CMD_VMID_MASK                                                                    0x000F0000L
+#define VPEC_QUEUE7_IB_CNTL__IB_PRIV_MASK                                                                     0x80000000L
+//VPEC_QUEUE7_IB_RPTR
+#define VPEC_QUEUE7_IB_RPTR__OFFSET__SHIFT                                                                    0x2
+#define VPEC_QUEUE7_IB_RPTR__OFFSET_MASK                                                                      0x003FFFFCL
+//VPEC_QUEUE7_IB_OFFSET
+#define VPEC_QUEUE7_IB_OFFSET__OFFSET__SHIFT                                                                  0x2
+#define VPEC_QUEUE7_IB_OFFSET__OFFSET_MASK                                                                    0x003FFFFCL
+//VPEC_QUEUE7_IB_BASE_LO
+#define VPEC_QUEUE7_IB_BASE_LO__ADDR__SHIFT                                                                   0x5
+#define VPEC_QUEUE7_IB_BASE_LO__ADDR_MASK                                                                     0xFFFFFFE0L
+//VPEC_QUEUE7_IB_BASE_HI
+#define VPEC_QUEUE7_IB_BASE_HI__ADDR__SHIFT                                                                   0x0
+#define VPEC_QUEUE7_IB_BASE_HI__ADDR_MASK                                                                     0xFFFFFFFFL
+//VPEC_QUEUE7_IB_SIZE
+#define VPEC_QUEUE7_IB_SIZE__SIZE__SHIFT                                                                      0x0
+#define VPEC_QUEUE7_IB_SIZE__SIZE_MASK                                                                        0x000FFFFFL
+//VPEC_QUEUE7_CMDIB_CNTL
+#define VPEC_QUEUE7_CMDIB_CNTL__IB_ENABLE__SHIFT                                                              0x0
+#define VPEC_QUEUE7_CMDIB_CNTL__IB_SWAP_ENABLE__SHIFT                                                         0x4
+#define VPEC_QUEUE7_CMDIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                       0x8
+#define VPEC_QUEUE7_CMDIB_CNTL__CMD_VMID__SHIFT                                                               0x10
+#define VPEC_QUEUE7_CMDIB_CNTL__IB_PRIV__SHIFT                                                                0x1f
+#define VPEC_QUEUE7_CMDIB_CNTL__IB_ENABLE_MASK                                                                0x00000001L
+#define VPEC_QUEUE7_CMDIB_CNTL__IB_SWAP_ENABLE_MASK                                                           0x00000010L
+#define VPEC_QUEUE7_CMDIB_CNTL__SWITCH_INSIDE_IB_MASK                                                         0x00000100L
+#define VPEC_QUEUE7_CMDIB_CNTL__CMD_VMID_MASK                                                                 0x000F0000L
+#define VPEC_QUEUE7_CMDIB_CNTL__IB_PRIV_MASK                                                                  0x80000000L
+//VPEC_QUEUE7_CMDIB_RPTR
+#define VPEC_QUEUE7_CMDIB_RPTR__OFFSET__SHIFT                                                                 0x2
+#define VPEC_QUEUE7_CMDIB_RPTR__OFFSET_MASK                                                                   0x003FFFFCL
+//VPEC_QUEUE7_CMDIB_OFFSET
+#define VPEC_QUEUE7_CMDIB_OFFSET__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE7_CMDIB_OFFSET__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE7_CMDIB_BASE_LO
+#define VPEC_QUEUE7_CMDIB_BASE_LO__ADDR__SHIFT                                                                0x5
+#define VPEC_QUEUE7_CMDIB_BASE_LO__ADDR_MASK                                                                  0xFFFFFFE0L
+//VPEC_QUEUE7_CMDIB_BASE_HI
+#define VPEC_QUEUE7_CMDIB_BASE_HI__ADDR__SHIFT                                                                0x0
+#define VPEC_QUEUE7_CMDIB_BASE_HI__ADDR_MASK                                                                  0xFFFFFFFFL
+//VPEC_QUEUE7_CMDIB_SIZE
+#define VPEC_QUEUE7_CMDIB_SIZE__SIZE__SHIFT                                                                   0x0
+#define VPEC_QUEUE7_CMDIB_SIZE__SIZE_MASK                                                                     0x000FFFFFL
+//VPEC_QUEUE7_3DLUTIB_CNTL
+#define VPEC_QUEUE7_3DLUTIB_CNTL__IB_ENABLE__SHIFT                                                            0x0
+#define VPEC_QUEUE7_3DLUTIB_CNTL__SWITCH_INSIDE_IB__SHIFT                                                     0x8
+#define VPEC_QUEUE7_3DLUTIB_CNTL__CMD_VMID__SHIFT                                                             0x10
+#define VPEC_QUEUE7_3DLUTIB_CNTL__IB_PRIV__SHIFT                                                              0x1f
+#define VPEC_QUEUE7_3DLUTIB_CNTL__IB_ENABLE_MASK                                                              0x00000001L
+#define VPEC_QUEUE7_3DLUTIB_CNTL__SWITCH_INSIDE_IB_MASK                                                       0x00000100L
+#define VPEC_QUEUE7_3DLUTIB_CNTL__CMD_VMID_MASK                                                               0x000F0000L
+#define VPEC_QUEUE7_3DLUTIB_CNTL__IB_PRIV_MASK                                                                0x80000000L
+//VPEC_QUEUE7_3DLUTIB_RPTR
+#define VPEC_QUEUE7_3DLUTIB_RPTR__OFFSET__SHIFT                                                               0x2
+#define VPEC_QUEUE7_3DLUTIB_RPTR__OFFSET_MASK                                                                 0x003FFFFCL
+//VPEC_QUEUE7_3DLUTIB_OFFSET
+#define VPEC_QUEUE7_3DLUTIB_OFFSET__OFFSET__SHIFT                                                             0x2
+#define VPEC_QUEUE7_3DLUTIB_OFFSET__OFFSET_MASK                                                               0x003FFFFCL
+//VPEC_QUEUE7_3DLUTIB_BASE_LO
+#define VPEC_QUEUE7_3DLUTIB_BASE_LO__ADDR__SHIFT                                                              0x5
+#define VPEC_QUEUE7_3DLUTIB_BASE_LO__ADDR_MASK                                                                0xFFFFFFE0L
+//VPEC_QUEUE7_3DLUTIB_BASE_HI
+#define VPEC_QUEUE7_3DLUTIB_BASE_HI__ADDR__SHIFT                                                              0x0
+#define VPEC_QUEUE7_3DLUTIB_BASE_HI__ADDR_MASK                                                                0xFFFFFFFFL
+//VPEC_QUEUE7_3DLUTIB_SIZE
+#define VPEC_QUEUE7_3DLUTIB_SIZE__SIZE__SHIFT                                                                 0x0
+#define VPEC_QUEUE7_3DLUTIB_SIZE__SIZE_MASK                                                                   0x000FFFFFL
+//VPEC_QUEUE7_CSA_ADDR_LO
+#define VPEC_QUEUE7_CSA_ADDR_LO__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE7_CSA_ADDR_LO__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE7_CSA_ADDR_HI
+#define VPEC_QUEUE7_CSA_ADDR_HI__ADDR__SHIFT                                                                  0x0
+#define VPEC_QUEUE7_CSA_ADDR_HI__ADDR_MASK                                                                    0xFFFFFFFFL
+//VPEC_QUEUE7_CONTEXT_STATUS
+#define VPEC_QUEUE7_CONTEXT_STATUS__SELECTED__SHIFT                                                           0x0
+#define VPEC_QUEUE7_CONTEXT_STATUS__USE_IB__SHIFT                                                             0x1
+#define VPEC_QUEUE7_CONTEXT_STATUS__IDLE__SHIFT                                                               0x2
+#define VPEC_QUEUE7_CONTEXT_STATUS__EXPIRED__SHIFT                                                            0x3
+#define VPEC_QUEUE7_CONTEXT_STATUS__EXCEPTION__SHIFT                                                          0x4
+#define VPEC_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE__SHIFT                                                         0x7
+#define VPEC_QUEUE7_CONTEXT_STATUS__USE_3DLUTIB__SHIFT                                                        0x8
+#define VPEC_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE__SHIFT                                                    0xa
+#define VPEC_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE__SHIFT                                                       0xb
+#define VPEC_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING__SHIFT                                                0xc
+#define VPEC_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT__SHIFT                                             0x10
+#define VPEC_QUEUE7_CONTEXT_STATUS__SELECTED_MASK                                                             0x00000001L
+#define VPEC_QUEUE7_CONTEXT_STATUS__USE_IB_MASK                                                               0x00000002L
+#define VPEC_QUEUE7_CONTEXT_STATUS__IDLE_MASK                                                                 0x00000004L
+#define VPEC_QUEUE7_CONTEXT_STATUS__EXPIRED_MASK                                                              0x00000008L
+#define VPEC_QUEUE7_CONTEXT_STATUS__EXCEPTION_MASK                                                            0x00000070L
+#define VPEC_QUEUE7_CONTEXT_STATUS__CTXSW_ABLE_MASK                                                           0x00000080L
+#define VPEC_QUEUE7_CONTEXT_STATUS__USE_3DLUTIB_MASK                                                          0x00000100L
+#define VPEC_QUEUE7_CONTEXT_STATUS__PREEMPT_DISABLE_MASK                                                      0x00000400L
+#define VPEC_QUEUE7_CONTEXT_STATUS__RPTR_WB_IDLE_MASK                                                         0x00000800L
+#define VPEC_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_PENDING_MASK                                                  0x00001000L
+#define VPEC_QUEUE7_CONTEXT_STATUS__WPTR_UPDATE_FAIL_COUNT_MASK                                               0x00FF0000L
+//VPEC_QUEUE7_DOORBELL_LOG
+#define VPEC_QUEUE7_DOORBELL_LOG__BE_ERROR__SHIFT                                                             0x0
+#define VPEC_QUEUE7_DOORBELL_LOG__DATA__SHIFT                                                                 0x2
+#define VPEC_QUEUE7_DOORBELL_LOG__BE_ERROR_MASK                                                               0x00000001L
+#define VPEC_QUEUE7_DOORBELL_LOG__DATA_MASK                                                                   0xFFFFFFFCL
+//VPEC_QUEUE7_IB_SUB_REMAIN
+#define VPEC_QUEUE7_IB_SUB_REMAIN__SIZE__SHIFT                                                                0x0
+#define VPEC_QUEUE7_IB_SUB_REMAIN__SIZE_MASK                                                                  0x00003FFFL
+//VPEC_QUEUE7_PREEMPT
+#define VPEC_QUEUE7_PREEMPT__IB_PREEMPT__SHIFT                                                                0x0
+#define VPEC_QUEUE7_PREEMPT__IB_PREEMPT_MASK                                                                  0x00000001L
+//VPEC_QUEUE7_LOG0BUFFER_CFG
+#define VPEC_QUEUE7_LOG0BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE7_LOG0BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE7_LOG0BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE7_LOG0BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE7_LOG0BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE7_LOG0BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE7_LOG0BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE7_LOG0BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+//VPEC_QUEUE7_LOG1BUFFER_CFG
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__ENABLE__SHIFT                                                             0x0
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__PARTIAL_ENTRY__SHIFT                                                      0x1
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__FIRST_FREE_ENTRY__SHIFT                                                   0x4
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__LAST_FREE_ENTRY__SHIFT                                                    0xc
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__RESERVED__SHIFT                                                           0x14
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__ENABLE_MASK                                                               0x00000001L
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__PARTIAL_ENTRY_MASK                                                        0x00000002L
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__FIRST_FREE_ENTRY_MASK                                                     0x00000FF0L
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__LAST_FREE_ENTRY_MASK                                                      0x000FF000L
+#define VPEC_QUEUE7_LOG1BUFFER_CFG__RESERVED_MASK                                                             0xFFF00000L
+
+#endif
diff --git a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
index 9aba8596faa7e..44e225e097d0c 100644
--- a/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
+++ b/drivers/gpu/drm/amd/include/kgd_kfd_interface.h
@@ -33,6 +33,7 @@
 #include <linux/dma-fence.h>
 #include "amdgpu_irq.h"
 #include "amdgpu_gfx.h"
+#include "amdgpu_ptl.h"
 
 struct pci_dev;
 struct amdgpu_device;
@@ -333,6 +334,11 @@ struct kfd2kgd_calls {
 			      uint32_t inst, unsigned int utimeout);
 	uint32_t (*hqd_sdma_get_doorbell)(struct amdgpu_device *adev,
 					  int engine, int queue);
+	uint32_t (*ptl_ctrl)(struct amdgpu_device *adev,
+			     uint32_t cmd,
+			     uint32_t *ptl_state,
+			     enum amdgpu_ptl_fmt *fmt1,
+			     enum amdgpu_ptl_fmt *fmt2);
 };
 
 #endif	/* KGD_KFD_INTERFACE_H_INCLUDED */
diff --git a/drivers/gpu/drm/amd/pm/amdgpu_pm.c b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
index 736304e73ca4d..bb11f8bb7bd4b 100644
--- a/drivers/gpu/drm/amd/pm/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/pm/amdgpu_pm.c
@@ -2034,15 +2034,13 @@ static int pp_dpm_clk_default_attr_update(struct amdgpu_device *adev, struct amd
 		       gc_ver == IP_VERSION(11, 0, 3)) && adev->vcn.num_vcn_inst >= 2))
 			*states = ATTR_STATE_UNSUPPORTED;
 	} else if (DEVICE_ATTR_IS(pp_dpm_pcie)) {
-		if (gc_ver == IP_VERSION(9, 4, 2) ||
-		    amdgpu_is_multi_aid(adev))
+		if (amdgpu_is_multi_aid(adev))
 			*states = ATTR_STATE_UNSUPPORTED;
 	}
 
 	switch (gc_ver) {
 	case IP_VERSION(9, 4, 1):
-	case IP_VERSION(9, 4, 2):
-		/* the Mi series card does not support standalone mclk/socclk/fclk level setting */
+		/* Arcturus does not support standalone mclk/socclk/fclk level setting */
 		if (DEVICE_ATTR_IS(pp_dpm_mclk) ||
 		    DEVICE_ATTR_IS(pp_dpm_socclk) ||
 		    DEVICE_ATTR_IS(pp_dpm_fclk)) {
@@ -2050,6 +2048,19 @@ static int pp_dpm_clk_default_attr_update(struct amdgpu_device *adev, struct amd
 			dev_attr->store = NULL;
 		}
 		break;
+	case IP_VERSION(9, 4, 2):
+		if (DEVICE_ATTR_IS(pp_dpm_mclk) ||
+		    DEVICE_ATTR_IS(pp_dpm_socclk)) {
+			/* Aldebaran mclk/socclk DPM only supports voltage control,
+			 * not allow to set dpm level directly */
+			dev_attr->attr.mode &= ~S_IWUGO;
+			dev_attr->store = NULL;
+		} else if (DEVICE_ATTR_IS(pp_dpm_fclk) ||
+			   DEVICE_ATTR_IS(pp_dpm_pcie)) {
+			/* Aldebaran does not support fclk/pcie dpm */
+			*states = ATTR_STATE_UNSUPPORTED;
+		}
+		break;
 	default:
 		break;
 	}
@@ -2505,12 +2516,12 @@ static ssize_t amdgpu_set_pm_policy_attr(struct device *dev,
 		.dev_attr = __ATTR(_name, 0644, amdgpu_get_pm_policy_attr, \
 				   amdgpu_set_pm_policy_attr),             \
 		.id = PP_PM_POLICY_##_id,                                  \
-	};
+	}
 
 #define AMDGPU_PM_POLICY_ATTR_VAR(_name) pm_policy_attr_##_name.dev_attr.attr
 
-AMDGPU_PM_POLICY_ATTR(soc_pstate, SOC_PSTATE)
-AMDGPU_PM_POLICY_ATTR(xgmi_plpd, XGMI_PLPD)
+AMDGPU_PM_POLICY_ATTR(soc_pstate, SOC_PSTATE);
+AMDGPU_PM_POLICY_ATTR(xgmi_plpd, XGMI_PLPD);
 
 static struct attribute *pm_policy_attrs[] = {
 	&AMDGPU_PM_POLICY_ATTR_VAR(soc_pstate),
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
index 0df8c05a7fce7..b123881344890 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/smu_v13_0_6_ppt.c
@@ -783,6 +783,10 @@ int smu_v13_0_6_get_metrics_table(struct smu_context *smu, void *metrics_table,
 		if (ret)
 			return ret;
 
+		if (!memchr_inv(smu_table->metrics_table, 0xff,
+				min(16, table_size)))
+			return -EHWPOISON;
+
 		smu_table->metrics_time = jiffies;
 	}
 
@@ -2071,9 +2075,9 @@ static int smu_v13_0_6_set_soft_freq_limited_range(struct smu_context *smu,
 		return -EINVAL;
 
 	if (smu_dpm->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) {
-		if (min >= max) {
+		if (min > max) {
 			dev_err(smu->adev->dev,
-				"Minimum clk should be less than the maximum allowed clock\n");
+				"Minimum clk should be less/equal to the maximum allowed clock\n");
 			return -EINVAL;
 		}
 
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
index c3cb368138064..a1318409e4b5e 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0.c
@@ -435,10 +435,12 @@ int smu_v15_0_fini_smc_tables(struct smu_context *smu)
 	smu_table->watermarks_table = NULL;
 	smu_table->metrics_time = 0;
 
+	kfree(smu_dpm->dpm_policies);
 	kfree(smu_dpm->dpm_context);
 	kfree(smu_dpm->golden_dpm_context);
 	kfree(smu_dpm->dpm_current_power_state);
 	kfree(smu_dpm->dpm_request_power_state);
+	smu_dpm->dpm_policies = NULL;
 	smu_dpm->dpm_context = NULL;
 	smu_dpm->golden_dpm_context = NULL;
 	smu_dpm->dpm_context_size = 0;
@@ -589,52 +591,47 @@ int smu_v15_0_notify_memory_pool_location(struct smu_context *smu)
 {
 	struct smu_table_context *smu_table = &smu->smu_table;
 	struct smu_table *memory_pool = &smu_table->memory_pool;
-	struct smu_msg_args args = {
-		.msg = SMU_MSG_DramLogSetDramAddr,
-		.num_args = 3,
-		.num_out_args = 0,
-	};
+	uint32_t params[3];
 
 	if (memory_pool->size == 0 || memory_pool->cpu_addr == NULL)
 		return 0;
 
 	/* SMU_MSG_DramLogSetDramAddr: ARG0=low, ARG1=high, ARG2=size */
-	args.args[0] = lower_32_bits(memory_pool->mc_address);
-	args.args[1] = upper_32_bits(memory_pool->mc_address);
-	args.args[2] = (u32)memory_pool->size;
-
-	return smu->msg_ctl.ops->send_msg(&smu->msg_ctl, &args);
+	params[0] = lower_32_bits(memory_pool->mc_address);
+	params[1] = upper_32_bits(memory_pool->mc_address);
+	params[2] = (u32)memory_pool->size;
+
+	return smu_cmn_send_smc_msg_with_params(smu,
+						SMU_MSG_DramLogSetDramAddr,
+						params, ARRAY_SIZE(params),
+						NULL, 0);
 }
 
 int smu_v15_0_set_driver_table_location(struct smu_context *smu)
 {
 	struct smu_table *driver_table = &smu->smu_table.driver_table;
-	struct smu_msg_args args = {
-		.msg = SMU_MSG_SetDriverDramAddr,
-		.num_args = 2,
-		.num_out_args = 0,
+	const uint32_t params[] = {
+		lower_32_bits(driver_table->mc_address),
+		upper_32_bits(driver_table->mc_address),
 	};
 
-	args.args[0] = lower_32_bits(driver_table->mc_address);
-	args.args[1] = upper_32_bits(driver_table->mc_address);
-
-	return smu->msg_ctl.ops->send_msg(&smu->msg_ctl, &args);
+	return smu_cmn_send_smc_msg_with_params(smu, SMU_MSG_SetDriverDramAddr,
+						params, ARRAY_SIZE(params),
+						NULL, 0);
 }
 
 int smu_v15_0_set_tool_table_location(struct smu_context *smu)
 {
 	struct smu_table *tool_table = &smu->smu_table.tables[SMU_TABLE_PMSTATUSLOG];
-	struct smu_msg_args args = {
-		.msg = SMU_MSG_SetToolsDramAddr,
-		.num_args = 2,
-		.num_out_args = 0,
+	const uint32_t params[] = {
+		lower_32_bits(tool_table->mc_address),
+		upper_32_bits(tool_table->mc_address),
 	};
 
 	/* SMU_MSG_SetToolsDramAddr: ARG0=low, ARG1=high */
-	args.args[0] = lower_32_bits(tool_table->mc_address);
-	args.args[1] = upper_32_bits(tool_table->mc_address);
-
-	return smu->msg_ctl.ops->send_msg(&smu->msg_ctl, &args);
+	return smu_cmn_send_smc_msg_with_params(smu, SMU_MSG_SetToolsDramAddr,
+						params, ARRAY_SIZE(params),
+						NULL, 0);
 }
 
 int smu_v15_0_set_allowed_mask(struct smu_context *smu)
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c
index 8d092c347076e..fb1145691410f 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_0_ppt.c
@@ -248,7 +248,7 @@ static int smu_v15_0_0_update_table(struct smu_context *smu,
 	uint64_t address;
 	uint32_t table_size;
 	int ret;
-	struct smu_msg_ctl *ctl = &smu->msg_ctl;
+	uint32_t params[3];
 
 	if (!table_data || table_index >= SMU_TABLE_COUNT || table_id < 0)
 		return -EINVAL;
@@ -265,20 +265,15 @@ static int smu_v15_0_0_update_table(struct smu_context *smu,
 	}
 
 	address = table->mc_address;
+	params[0] = table_id;
+	params[1] = (uint32_t)lower_32_bits(address);
+	params[2] = (uint32_t)upper_32_bits(address);
 
-	struct smu_msg_args args = {
-		.msg = drv2smu ?
-				SMU_MSG_TransferTableDram2Smu :
-				SMU_MSG_TransferTableSmu2Dram,
-		.num_args = 3,
-		.num_out_args = 0,
-	};
-
-	args.args[0] = table_id;
-	args.args[1] = (uint32_t)lower_32_bits(address);
-	args.args[2] = (uint32_t)upper_32_bits(address);
-
-	ret = ctl->ops->send_msg(ctl, &args);
+	ret = smu_cmn_send_smc_msg_with_params(smu,
+					       drv2smu ? SMU_MSG_TransferTableDram2Smu :
+					       SMU_MSG_TransferTableSmu2Dram,
+					       params, ARRAY_SIZE(params),
+					       NULL, 0);
 
 	if (ret)
 		return ret;
@@ -535,22 +530,19 @@ static int smu_v15_0_0_read_sensor(struct smu_context *smu,
 static int smu_v15_0_0_get_enabled_mask(struct smu_context *smu,
 					struct smu_feature_bits *feature_mask)
 {
+	uint32_t out[2];
 	int ret;
-	struct smu_msg_ctl *ctl = &smu->msg_ctl;
 
 	if (!feature_mask)
 		return -EINVAL;
 
-	struct smu_msg_args args = {
-		.msg = SMU_MSG_GetEnabledSmuFeatures,
-		.num_args = 0,
-		.num_out_args = 2,
-	};
-
-	ret = ctl->ops->send_msg(ctl, &args);
+	ret = smu_cmn_send_smc_msg_with_params(smu,
+					       SMU_MSG_GetEnabledSmuFeatures,
+					       NULL, 0, out,
+					       ARRAY_SIZE(out));
 
 	if (!ret)
-		smu_feature_bits_from_arr32(feature_mask, args.out_args,
+		smu_feature_bits_from_arr32(feature_mask, out,
 					    SMU_FEATURE_NUM_DEFAULT);
 
 	return ret;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
index db85186f2d666..2105a1d7bb34b 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.c
@@ -63,6 +63,8 @@
 	for ((stack_idx) = 0; (umc_mask); \
 	     (umc_mask) >>= 4, (stack_idx)++) \
 
+#define NUM_JPEG_RINGS_FW	10
+
 #define to_amdgpu_device(x) (container_of(x, struct amdgpu_device, pm.smu_i2c))
 
 #define SMU_15_0_8_FEA_MAP(smu_feature, smu_15_0_8_feature)                    \
@@ -171,6 +173,13 @@ static const struct cmn2asic_mapping smu_v15_0_8_table_map[SMU_TABLE_COUNT] = {
 	TAB_MAP(I2C_COMMANDS),
 };
 
+static const uint8_t smu_v15_0_8_throttler_map[] = {
+	[THROTTLER_PROCHOT_BIT] = (SMU_THROTTLER_PROCHOT_GFX_BIT),
+	[THROTTLER_THERMAL_SOCKET_BIT] = (SMU_THROTTLER_TEMP_GPU_BIT),
+	[THROTTLER_THERMAL_VR_BIT] = (SMU_THROTTLER_TEMP_VR_GFX_BIT),
+	[THROTTLER_THERMAL_HBM_BIT] = (SMU_THROTTLER_TEMP_MEM_BIT),
+};
+
 static size_t smu_v15_0_8_get_system_metrics_size(void)
 {
 	return sizeof(SystemMetricsTable_t);
@@ -1062,38 +1071,30 @@ static int smu_v15_0_8_set_driver_pptable(struct smu_context *smu)
 		smu_table->tables[SMU_TABLE_SMU_METRICS].version =
 			table_version;
 
-		pptable->MaxSocketPowerLimit =
-			SMUQ10_ROUND(static_metrics->MaxSocketPowerLimit);
-		pptable->MaxGfxclkFrequency =
-			SMUQ10_ROUND(static_metrics->MaxGfxclkFrequency);
-		pptable->MinGfxclkFrequency =
-			SMUQ10_ROUND(static_metrics->MinGfxclkFrequency);
-		pptable->MaxFclkFrequency =
-			SMUQ10_ROUND(static_metrics->MaxFclkFrequency);
-		pptable->MinFclkFrequency =
-			SMUQ10_ROUND(static_metrics->MinFclkFrequency);
-		pptable->MaxGl2clkFrequency =
-			SMUQ10_ROUND(static_metrics->MaxGl2clkFrequency);
-		pptable->MinGl2clkFrequency =
-			SMUQ10_ROUND(static_metrics->MinGl2clkFrequency);
+		pptable->MaxSocketPowerLimit = static_metrics->MaxSocketPowerLimit;
+		pptable->MaxGfxclkFrequency = static_metrics->MaxGfxclkFrequency;
+		pptable->MinGfxclkFrequency = static_metrics->MinGfxclkFrequency;
+		pptable->MaxFclkFrequency = static_metrics->MaxFclkFrequency;
+		pptable->MinFclkFrequency = static_metrics->MinFclkFrequency;
+		pptable->MaxGl2clkFrequency = static_metrics->MaxGl2clkFrequency;
+		pptable->MinGl2clkFrequency = static_metrics->MinGl2clkFrequency;
 
 		for (i = 0; i < ARRAY_SIZE(static_metrics->UclkFrequencyTable); ++i)
-			pptable->UclkFrequencyTable[i] =
-				SMUQ10_ROUND(static_metrics->UclkFrequencyTable[i]);
-
-		pptable->SocclkFrequency = SMUQ10_ROUND(static_metrics->SocclkFrequency);
-		pptable->LclkFrequency = SMUQ10_ROUND(static_metrics->LclkFrequency);
-		pptable->VclkFrequency = SMUQ10_ROUND(static_metrics->VclkFrequency);
-		pptable->DclkFrequency = SMUQ10_ROUND(static_metrics->DclkFrequency);
-
-		pptable->CTFLimitMID = SMUQ10_ROUND(static_metrics->CTFLimit_MID);
-		pptable->CTFLimitAID = SMUQ10_ROUND(static_metrics->CTFLimit_AID);
-		pptable->CTFLimitXCD = SMUQ10_ROUND(static_metrics->CTFLimit_XCD);
-		pptable->CTFLimitHBM = SMUQ10_ROUND(static_metrics->CTFLimit_HBM);
-		pptable->ThermalLimitMID = SMUQ10_ROUND(static_metrics->ThermalLimit_MID);
-		pptable->ThermalLimitAID = SMUQ10_ROUND(static_metrics->ThermalLimit_AID);
-		pptable->ThermalLimitXCD = SMUQ10_ROUND(static_metrics->ThermalLimit_XCD);
-		pptable->ThermalLimitHBM = SMUQ10_ROUND(static_metrics->ThermalLimit_HBM);
+			pptable->UclkFrequencyTable[i] = static_metrics->UclkFrequencyTable[i];
+
+		pptable->SocclkFrequency = static_metrics->SocclkFrequency;
+		pptable->LclkFrequency = static_metrics->LclkFrequency;
+		pptable->VclkFrequency = static_metrics->VclkFrequency;
+		pptable->DclkFrequency = static_metrics->DclkFrequency;
+
+		pptable->CTFLimitMID = static_metrics->CTFLimit_MID;
+		pptable->CTFLimitAID = static_metrics->CTFLimit_AID;
+		pptable->CTFLimitXCD = static_metrics->CTFLimit_XCD;
+		pptable->CTFLimitHBM = static_metrics->CTFLimit_HBM;
+		pptable->ThermalLimitMID = static_metrics->ThermalLimit_MID;
+		pptable->ThermalLimitAID = static_metrics->ThermalLimit_AID;
+		pptable->ThermalLimitXCD = static_metrics->ThermalLimit_XCD;
+		pptable->ThermalLimitHBM = static_metrics->ThermalLimit_HBM;
 
 		/* use MID0 serial number by default */
 		pptable->PublicSerialNumberMID =
@@ -1312,23 +1313,21 @@ static int smu_v15_0_8_system_features_control(struct smu_context *smu,
 static int smu_v15_0_8_get_enabled_mask(struct smu_context *smu,
 					struct smu_feature_bits *feature_mask)
 {
-	struct smu_msg_args args = {
-		.msg = SMU_MSG_GetEnabledSmuFeatures,
-		.num_args = 0,
-		.num_out_args = 2,
-	};
+	uint32_t out[2];
 	int ret;
 
 	if (!feature_mask)
 		return -EINVAL;
 
-	ret = smu->msg_ctl.ops->send_msg(&smu->msg_ctl, &args);
+	ret = smu_cmn_send_smc_msg_with_params(smu,
+					       SMU_MSG_GetEnabledSmuFeatures,
+					       NULL, 0,
+					       out, ARRAY_SIZE(out));
 
 	if (ret)
 		return ret;
 
-	smu_feature_bits_from_arr32(feature_mask, args.out_args,
-				    SMU_FEATURE_NUM_DEFAULT);
+	smu_feature_bits_from_arr32(feature_mask, out, SMU_FEATURE_NUM_DEFAULT);
 
 	return 0;
 }
@@ -1601,6 +1600,81 @@ static ssize_t smu_v15_0_8_get_temp_metrics(struct smu_context *smu,
 	}
 }
 
+static ssize_t smu_v15_0_8_get_xcp_metrics(struct smu_context *smu, int xcp_id,
+					   void *table)
+{
+	struct smu_table_context *smu_table = &smu->smu_table;
+	const u8 num_jpeg_rings = NUM_JPEG_RINGS_FW;
+	struct smu_v15_0_8_partition_metrics *xcp_metrics;
+	MetricsTable_t *metrics;
+	struct amdgpu_device *adev = smu->adev;
+	int ret, inst, i, j, k, idx;
+	struct amdgpu_xcp *xcp;
+	u32 inst_mask;
+
+	if (!table)
+		return sizeof(*xcp_metrics);
+
+	for_each_xcp(adev->xcp_mgr, xcp, i) {
+		if (xcp->id == xcp_id)
+			break;
+	}
+	if (i == adev->xcp_mgr->num_xcps)
+		return -EINVAL;
+
+	xcp_metrics = (struct smu_v15_0_8_partition_metrics *)table;
+	smu_v15_0_8_partition_metrics_init(xcp_metrics, 1, 1);
+
+	ret = smu_v15_0_8_get_metrics_table_internal(smu, 1, NULL);
+	if (ret)
+		return ret;
+
+	metrics = (MetricsTable_t *)smu_table->metrics_table;
+
+	amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_VCN, &inst_mask);
+	idx = 0;
+	for_each_inst(k, inst_mask) {
+		/* Both JPEG and VCN has same instances */
+		inst = GET_INST(VCN, k);
+		for (j = 0; j < num_jpeg_rings; ++j) {
+			xcp_metrics->jpeg_busy[(idx * num_jpeg_rings) + j] =
+				SMUQ10_ROUND(metrics->JpegBusy[(inst * num_jpeg_rings) + j]);
+		}
+		xcp_metrics->vcn_busy[idx] =
+			SMUQ10_ROUND(metrics->VcnBusy[inst]);
+
+		xcp_metrics->current_vclk[idx] = SMUQ10_ROUND(metrics->VclkFrequency[inst]);
+		xcp_metrics->current_dclk[idx] = SMUQ10_ROUND(metrics->DclkFrequency[inst]);
+
+		idx++;
+	}
+
+	amdgpu_xcp_get_inst_details(xcp, AMDGPU_XCP_GFX, &inst_mask);
+	idx = 0;
+	for_each_inst(k, inst_mask) {
+		inst = GET_INST(GC, k);
+		xcp_metrics->current_gfxclk[idx] =
+			SMUQ10_ROUND(metrics->GfxclkFrequency[inst]);
+
+		xcp_metrics->gfx_busy_inst[idx] = SMUQ10_ROUND(metrics->GfxBusy[inst]);
+		xcp_metrics->gfx_busy_acc[idx] = SMUQ10_ROUND(metrics->GfxBusyAcc[inst]);
+		xcp_metrics->gfx_below_host_limit_ppt_acc[idx] =
+			SMUQ10_ROUND(metrics->GfxclkBelowHostLimitPptAcc[inst]);
+		xcp_metrics->gfx_below_host_limit_thm_acc[idx] =
+			SMUQ10_ROUND(metrics->GfxclkBelowHostLimitThmAcc[inst]);
+		xcp_metrics->gfx_low_utilization_acc[idx] =
+			SMUQ10_ROUND(metrics->GfxclkLowUtilizationAcc[inst]);
+		xcp_metrics->gfx_below_host_limit_total_acc[idx] =
+			SMUQ10_ROUND(metrics->GfxclkBelowHostLimitTotalAcc[inst]);
+		idx++;
+	}
+
+	xcp_metrics->accumulation_counter = metrics->AccumulationCounter;
+	xcp_metrics->firmware_timestamp = metrics->Timestamp;
+
+	return sizeof(*xcp_metrics);
+}
+
 static ssize_t smu_v15_0_8_get_gpu_metrics(struct smu_context *smu, void **table)
 {
 	struct smu_table_context *smu_table = &smu->smu_table;
@@ -1627,17 +1701,23 @@ static ssize_t smu_v15_0_8_get_gpu_metrics(struct smu_context *smu, void **table
 	if (adev->umc.active_mask) {
 		u64 mask = adev->umc.active_mask;
 		int out_idx = 0;
+		u16 max_hbm_temp = 0;
 		int stack_idx;
 
-		if (unlikely(hweight64(mask)/4 > SMU_15_0_8_MAX_HBM_STACKS))
+		if (unlikely(hweight64(mask) / 4 > SMU_15_0_8_MAX_HBM_STACKS)) {
 			dev_warn(adev->dev, "Invalid umc mask %lld\n", mask);
-		else  {
+		} else  {
 			for_each_hbm_stack(stack_idx, mask) {
+				u16 temp;
+
 				if (!hbm_stack_mask_valid(mask))
 					continue;
-				gpu_metrics->temperature_hbm[out_idx++] =
-					SMUQ10_ROUND(metrics->HbmTemperature[stack_idx]);
+				temp = SMUQ10_ROUND(metrics->HbmTemperature[stack_idx]);
+				gpu_metrics->temperature_hbm[out_idx++] = temp;
+				if (temp > max_hbm_temp)
+					max_hbm_temp = temp;
 			}
+			gpu_metrics->temperature_mem = max_hbm_temp;
 		}
 	}
 
@@ -2199,6 +2279,71 @@ static int smu_v15_0_8_get_ppt_limit(struct smu_context *smu,
 	return -EOPNOTSUPP;
 }
 
+static uint32_t smu_v15_0_8_get_throttler_status(struct smu_context *smu)
+{
+	struct smu_power_context *smu_power = &smu->smu_power;
+	struct smu_15_0_power_context *power_context = smu_power->power_context;
+	uint32_t throttler_status = 0;
+
+	throttler_status = atomic_read(&power_context->throttle_status);
+	dev_dbg(smu->adev->dev, "SMU Throttler status: %u", throttler_status);
+
+	return throttler_status;
+}
+
+static const char *const throttling_logging_label[] = {
+	[THROTTLER_PROCHOT_BIT] = "Prochot",
+	[THROTTLER_THERMAL_SOCKET_BIT] = "SOC",
+	[THROTTLER_THERMAL_VR_BIT] = "VR",
+	[THROTTLER_THERMAL_HBM_BIT] = "HBM"
+};
+
+static void smu_v15_0_8_log_thermal_throttling_event(struct smu_context *smu)
+{
+	int throttler_idx, throttling_events = 0, buf_idx = 0;
+	struct amdgpu_device *adev = smu->adev;
+	uint32_t throttler_status;
+	char log_buf[256];
+
+	throttler_status = smu_v15_0_8_get_throttler_status(smu);
+	if (!throttler_status)
+		return;
+
+	memset(log_buf, 0, sizeof(log_buf));
+	for (throttler_idx = 0;
+	     throttler_idx < ARRAY_SIZE(throttling_logging_label);
+	     throttler_idx++) {
+		if (throttler_status & (1U << throttler_idx)) {
+			throttling_events++;
+			buf_idx += snprintf(
+				log_buf + buf_idx, sizeof(log_buf) - buf_idx,
+				"%s%s", throttling_events > 1 ? " and " : "",
+				throttling_logging_label[throttler_idx]);
+			if (buf_idx >= sizeof(log_buf)) {
+				dev_err(adev->dev, "buffer overflow!\n");
+				log_buf[sizeof(log_buf) - 1] = '\0';
+				break;
+			}
+		}
+	}
+
+	dev_warn(adev->dev,
+		 "WARN: GPU is throttled, expect performance decrease. %s.\n",
+		 log_buf);
+	kgd2kfd_smi_event_throttle(
+		smu->adev->kfd.dev,
+		smu_cmn_get_indep_throttler_status(throttler_status,
+						   smu_v15_0_8_throttler_map));
+}
+
+static int smu_v15_0_8_enable_thermal_alert(struct smu_context *smu)
+{
+	if (!smu->irq_source.num_types)
+		return 0;
+
+	return amdgpu_irq_get(smu->adev, &smu->irq_source, 0);
+}
+
 static const struct pptable_funcs smu_v15_0_8_ppt_funcs = {
 	.init_allowed_features = smu_v15_0_8_init_allowed_features,
 	.set_default_dpm_table = smu_v15_0_8_set_default_dpm_table,
@@ -2216,10 +2361,14 @@ static const struct pptable_funcs smu_v15_0_8_ppt_funcs = {
 	.get_enabled_mask = smu_v15_0_8_get_enabled_mask,
 	.feature_is_enabled = smu_cmn_feature_is_enabled,
 	.register_irq_handler = smu_v15_0_8_register_irq_handler,
+	.enable_thermal_alert = smu_v15_0_8_enable_thermal_alert,
+	.disable_thermal_alert = smu_v15_0_disable_thermal_alert,
+	.log_thermal_throttling_event = smu_v15_0_8_log_thermal_throttling_event,
 	.setup_pptable = smu_v15_0_8_setup_pptable,
 	.get_pp_feature_mask = smu_cmn_get_pp_feature_mask,
 	.wait_for_event = smu_v15_0_wait_for_event,
 	.get_pm_metrics = smu_v15_0_8_get_pm_metrics,
+	.get_xcp_metrics = smu_v15_0_8_get_xcp_metrics,
 	.mode2_reset = smu_v15_0_8_mode2_reset,
 	.get_dpm_ultimate_freq = smu_v15_0_8_get_dpm_ultimate_freq,
 	.get_gpu_metrics = smu_v15_0_8_get_gpu_metrics,
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h
index 398ce44821741..4dfc40aaffcb4 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu15/smu_v15_0_8_ppt.h
@@ -309,5 +309,39 @@ DECLARE_SMU_METRICS_CLASS(smu_v15_0_8_gpuboard_temp_metrics,
 
 DECLARE_SMU_METRICS_CLASS(smu_v15_0_8_baseboard_temp_metrics,
 			  SMU_15_0_8_BASEBOARD_TEMP_METRICS_FIELDS);
+#define SMU_15_0_8_PARTITION_METRICS_FIELDS(SMU_SCALAR, SMU_ARRAY)             \
+	SMU_ARRAY(SMU_MATTR(CURRENT_GFXCLK), SMU_MUNIT(CLOCK_1),               \
+		  SMU_MTYPE(U16), current_gfxclk, SMU_15_0_8_MAX_XCC);         \
+	SMU_ARRAY(SMU_MATTR(CURRENT_VCLK0), SMU_MUNIT(CLOCK_1),                \
+		  SMU_MTYPE(U16), current_vclk, SMU_15_0_8_MAX_VCN);          \
+	SMU_ARRAY(SMU_MATTR(CURRENT_DCLK0), SMU_MUNIT(CLOCK_1),                \
+		  SMU_MTYPE(U16), current_dclk, SMU_15_0_8_MAX_VCN);          \
+	SMU_ARRAY(SMU_MATTR(GFX_BUSY_INST), SMU_MUNIT(PERCENT),                \
+		  SMU_MTYPE(U8), gfx_busy_inst, SMU_15_0_8_MAX_XCC);          \
+	SMU_ARRAY(SMU_MATTR(JPEG_BUSY), SMU_MUNIT(PERCENT), SMU_MTYPE(U8),    \
+		  jpeg_busy, SMU_15_0_8_MAX_JPEG);                             \
+	SMU_ARRAY(SMU_MATTR(VCN_BUSY), SMU_MUNIT(PERCENT), SMU_MTYPE(U8),     \
+		  vcn_busy, SMU_15_0_8_MAX_VCN);                               \
+	SMU_ARRAY(SMU_MATTR(GFX_BUSY_ACC), SMU_MUNIT(NONE), SMU_MTYPE(U64),    \
+		  gfx_busy_acc, SMU_15_0_8_MAX_XCC);                           \
+	SMU_ARRAY(SMU_MATTR(GFX_BELOW_HOST_LIMIT_PPT_ACC), SMU_MUNIT(NONE),    \
+		  SMU_MTYPE(U64), gfx_below_host_limit_ppt_acc,                \
+		  SMU_15_0_8_MAX_XCC);                                         \
+	SMU_ARRAY(SMU_MATTR(GFX_BELOW_HOST_LIMIT_THM_ACC), SMU_MUNIT(NONE),    \
+		  SMU_MTYPE(U64), gfx_below_host_limit_thm_acc,                \
+		  SMU_15_0_8_MAX_XCC);                                         \
+	SMU_ARRAY(SMU_MATTR(GFX_LOW_UTILIZATION_ACC), SMU_MUNIT(NONE),         \
+		  SMU_MTYPE(U64), gfx_low_utilization_acc,                     \
+		  SMU_15_0_8_MAX_XCC);                                         \
+	SMU_ARRAY(SMU_MATTR(GFX_BELOW_HOST_LIMIT_TOTAL_ACC), SMU_MUNIT(NONE),  \
+		  SMU_MTYPE(U64), gfx_below_host_limit_total_acc,              \
+		  SMU_15_0_8_MAX_XCC);					       \
+	SMU_SCALAR(SMU_MATTR(ACCUMULATION_COUNTER), SMU_MUNIT(NONE),           \
+		   SMU_MTYPE(U64), accumulation_counter);                      \
+	SMU_SCALAR(SMU_MATTR(FIRMWARE_TIMESTAMP), SMU_MUNIT(TIME_2),           \
+		   SMU_MTYPE(U64), firmware_timestamp);
+
+DECLARE_SMU_METRICS_CLASS(smu_v15_0_8_partition_metrics,
+			  SMU_15_0_8_PARTITION_METRICS_FIELDS);
 #endif
 #endif
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
index 90c7127beabff..546e64e3ba9c8 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c
@@ -127,6 +127,61 @@ int smu_cmn_wait_for_response(struct smu_context *smu)
 	return smu_msg_wait_response(&smu->msg_ctl, 0);
 }
 
+/**
+ * smu_cmn_send_smc_msg_with_params_ext - send an SMU message with 0..N args
+ * @smu: pointer to an SMU context
+ * @msg: message to send
+ * @params: optional input argument array
+ * @num_params: number of input arguments in @params
+ * @read_args: optional output argument array
+ * @num_read_args: number of output arguments to read back
+ * @flags: message flags (SMU_MSG_FLAG_*)
+ * @timeout: per-message timeout in us (0 = use default)
+ *
+ * This helper keeps the raw protocol semantics of struct smu_msg_args while
+ * hiding the per-call boilerplate. It is intended for true multi-parameter
+ * messages. Legacy wrappers such as smu_cmn_send_smc_msg() retain their
+ * existing single-zero-parameter behavior for compatibility.
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+int smu_cmn_send_smc_msg_with_params_ext(struct smu_context *smu,
+					 enum smu_message_type msg,
+					 const uint32_t *params,
+					 size_t num_params,
+					 uint32_t *read_args,
+					 size_t num_read_args,
+					 uint32_t flags,
+					 uint32_t timeout)
+{
+	struct smu_msg_ctl *ctl = &smu->msg_ctl;
+	struct smu_msg_args args = {
+		.msg = msg,
+		.num_args = num_params,
+		.num_out_args = num_read_args,
+		.flags = flags,
+		.timeout = timeout,
+	};
+	int ret;
+
+	if ((num_params && !params) || (num_read_args && !read_args))
+		return -EINVAL;
+
+	if (num_params > SMU_MSG_MAX_ARGS || num_read_args > SMU_MSG_MAX_ARGS)
+		return -EINVAL;
+
+	if (num_params)
+		memcpy(args.args, params, num_params * sizeof(*params));
+
+	ret = ctl->ops->send_msg(ctl, &args);
+
+	if (num_read_args)
+		memcpy(read_args, args.out_args,
+		       num_read_args * sizeof(*read_args));
+
+	return ret;
+}
+
 /**
  * smu_cmn_send_smc_msg_with_param -- send a message with parameter
  * @smu: pointer to an SMU context
@@ -164,23 +219,9 @@ int smu_cmn_send_smc_msg_with_param(struct smu_context *smu,
 				    uint32_t param,
 				    uint32_t *read_arg)
 {
-	struct smu_msg_ctl *ctl = &smu->msg_ctl;
-	struct smu_msg_args args = {
-		.msg = msg,
-		.args[0] = param,
-		.num_args = 1,
-		.num_out_args = read_arg ? 1 : 0,
-		.flags = 0,
-		.timeout = 0,
-	};
-	int ret;
-
-	ret = ctl->ops->send_msg(ctl, &args);
-
-	if (read_arg)
-		*read_arg = args.out_args[0];
-
-	return ret;
+	return smu_cmn_send_smc_msg_with_params(smu, msg,
+						&param, 1,
+						read_arg, read_arg ? 1 : 0);
 }
 
 int smu_cmn_send_smc_msg(struct smu_context *smu,
@@ -1071,13 +1112,12 @@ int smu_cmn_update_table_read_arg(struct smu_context *smu,
 	struct amdgpu_device *adev = smu->adev;
 	struct smu_table_context *smu_table = &smu->smu_table;
 	struct smu_table *table = &smu_table->driver_table;
-	struct smu_msg_ctl *ctl = &smu->msg_ctl;
-	struct smu_msg_args args;
 	int table_id = smu_cmn_to_asic_specific_index(smu,
 						      CMN2ASIC_MAPPING_TABLE,
 						      table_index);
 	uint32_t table_size;
 	int ret = 0;
+	uint32_t param;
 
 	if (!table_data || table_index >= SMU_TABLE_COUNT || table_id < 0)
 		return -EINVAL;
@@ -1093,18 +1133,14 @@ int smu_cmn_update_table_read_arg(struct smu_context *smu,
 		amdgpu_hdp_flush(adev, NULL);
 	}
 
-	args.msg = drv2smu ? SMU_MSG_TransferTableDram2Smu : SMU_MSG_TransferTableSmu2Dram;
-	args.args[0] = ((argument & 0xFFFF) << 16) | (table_id  & 0xffff);
-	args.num_args = 1;
-	args.out_args[0] = 0;
-	args.num_out_args = read_arg ? 1 : 0;
-	args.flags = read_arg ? SMU_MSG_FLAG_FORCE_READ_ARG : 0;
-	args.timeout = 0;
-
-	ret = ctl->ops->send_msg(ctl, &args);
+	param = ((argument & 0xFFFF) << 16) | (table_id & 0xffff);
 
-	if (read_arg)
-		*read_arg = args.out_args[0];
+	ret = smu_cmn_send_smc_msg_with_params_ext(
+		smu,
+		drv2smu ? SMU_MSG_TransferTableDram2Smu :
+			  SMU_MSG_TransferTableSmu2Dram,
+		&param, 1, read_arg, read_arg ? 1 : 0,
+		read_arg ? SMU_MSG_FLAG_FORCE_READ_ARG : 0, 0);
 
 	if (ret)
 		return ret;
diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
index c6ac0e876aeab..0e119965ce131 100644
--- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
+++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h
@@ -118,6 +118,28 @@ int smu_cmn_send_smc_msg_with_param(struct smu_context *smu,
 				    uint32_t param,
 				    uint32_t *read_arg);
 
+int smu_cmn_send_smc_msg_with_params_ext(struct smu_context *smu,
+					 enum smu_message_type msg,
+					 const uint32_t *params,
+					 size_t num_params,
+					 uint32_t *read_args,
+					 size_t num_read_args,
+					 uint32_t flags,
+					 uint32_t timeout);
+
+static inline int smu_cmn_send_smc_msg_with_params(struct smu_context *smu,
+						   enum smu_message_type msg,
+						   const uint32_t *params,
+						   size_t num_params,
+						   uint32_t *read_args,
+						   size_t num_read_args)
+{
+	return smu_cmn_send_smc_msg_with_params_ext(smu, msg,
+						    params, num_params,
+						    read_args, num_read_args,
+						    0, 0);
+}
+
 int smu_cmn_send_smc_msg(struct smu_context *smu,
 			 enum smu_message_type msg,
 			 uint32_t *read_arg);
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
index cb6498c308345..c22e53e842073 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_cmd.c
@@ -139,7 +139,8 @@ static int amdgpu_ras_get_ras_safe_fb_addr_ranges(struct ras_core_context *ras_c
 	struct amdgpu_mem_partition_info *mem_ranges;
 	uint32_t i = 0;
 
-	if (cmd->input_size != sizeof(*input_data))
+	if ((cmd->input_size != sizeof(*input_data)) ||
+		(cmd->output_buf_size < sizeof(*ranges)))
 		return RAS_CMD__ERROR_INVALID_INPUT_DATA;
 
 	mem_ranges = adev->gmc.mem_partitions;
@@ -207,7 +208,8 @@ static int amdgpu_ras_translate_fb_address(struct ras_core_context *ras_core,
 			(struct ras_cmd_translate_fb_address_rsp *)cmd->output_buff_raw;
 	int ret = RAS_CMD__ERROR_GENERIC;
 
-	if (cmd->input_size != sizeof(struct ras_cmd_translate_fb_address_req))
+	if ((cmd->input_size != sizeof(struct ras_cmd_translate_fb_address_req)) ||
+		(cmd->output_buf_size < sizeof(*rsp_buff)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
 	if ((req_buff->src_addr_type >= RAS_FB_ADDR_UNKNOWN) ||
@@ -279,7 +281,7 @@ int amdgpu_ras_submit_cmd(struct ras_core_context *ras_core, struct ras_cmd_ctx
 
 	cmd->cmd_res = res;
 
-	if (cmd->output_size > cmd->output_buf_size) {
+	if (!res && (cmd->output_size > cmd->output_buf_size)) {
 		RAS_DEV_ERR(cmd_core->dev,
 			"Output size 0x%x exceeds output buffer size 0x%x!\n",
 			cmd->output_size, cmd->output_buf_size);
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
index ad8862d43263c..a22d1aebbeb94 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_ras_mgr.c
@@ -291,6 +291,8 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block)
 	con->uniras_enabled = false;
 
 	if (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) ||
+	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) ||
+	    amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) ||
 	    adev->debug_enable_ras_aca)
 		con->uniras_enabled = true;
 	else
@@ -307,13 +309,17 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block)
 	if (!ras_mgr->ras_core) {
 		RAS_DEV_ERR(adev, "Failed to create ras core!\n");
 		ret = -EINVAL;
-		goto err;
+		goto err1;
 	}
 
 	ras_mgr->ras_core->dev = adev;
 
 	amdgpu_ras_process_init(adev);
-	ras_core_sw_init(ras_mgr->ras_core);
+	ret = ras_core_sw_init(ras_mgr->ras_core);
+	if (ret) {
+		RAS_DEV_ERR(adev, "ras_core_sw_init failed! ret:%d\n", ret);
+		goto err2;
+	}
 	amdgpu_ras_mgr_init_event_mgr(ras_mgr->ras_core);
 
 	if (amdgpu_sriov_vf(adev)) {
@@ -321,14 +327,22 @@ static int amdgpu_ras_mgr_sw_init(struct amdgpu_ip_block *ip_block)
 		if (ret) {
 			RAS_DEV_ERR(adev,
 				"Virt ras sw_init failed! ret:%d\n", ret);
-			goto err;
+			goto err3;
 		}
 	}
 
 	return 0;
 
-err:
+err3:
+	if (ras_mgr->ras_core)
+		ras_core_sw_fini(ras_mgr->ras_core);
+err2:
+	amdgpu_ras_process_fini(adev);
+	if (ras_mgr->ras_core)
+		ras_core_destroy(ras_mgr->ras_core);
+err1:
 	kfree(ras_mgr);
+	con->ras_mgr = NULL;
 	return ret;
 }
 
@@ -496,8 +510,10 @@ uint64_t amdgpu_ras_mgr_gen_ras_event_seqno(struct amdgpu_device *adev,
 	if ((seqno_type == RAS_SEQNO_TYPE_DE) ||
 	    (seqno_type == RAS_SEQNO_TYPE_POISON_CONSUMPTION)) {
 		ret = ras_core_put_seqno(ras_mgr->ras_core, seqno_type, seq_no);
-		if (ret)
+		if (ret) {
 			RAS_DEV_WARN(adev, "There are too many ras interrupts!");
+			return 0;
+		}
 	}
 
 	return seq_no;
diff --git a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
index fb4d375e87b23..c2761f3d06a0a 100644
--- a/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
+++ b/drivers/gpu/drm/amd/ras/ras_mgr/amdgpu_virt_ras_cmd.c
@@ -192,8 +192,17 @@ static int amdgpu_virt_ras_get_cper_snapshot(struct ras_core_context *ras_core,
 	return RAS_CMD__SUCCESS;
 }
 
+static bool amdgpu_virt_ras_check_batch_cached(struct ras_cmd_batch_trace_record_rsp *rsp,
+				       uint64_t batch_id)
+{
+	return rsp->real_batch_num &&
+	       rsp->real_batch_num <= RAS_CMD_MAX_BATCH_NUM &&
+	       batch_id >= rsp->start_batch_id &&
+	       (batch_id - rsp->start_batch_id) < rsp->real_batch_num;
+}
+
 static int amdgpu_virt_ras_get_batch_records(struct ras_core_context *ras_core, uint64_t batch_id,
-			struct ras_log_info **trace_arr, uint32_t arr_num,
+			struct ras_log_info *trace_arr, uint32_t arr_num,
 			struct ras_cmd_batch_trace_record_rsp *rsp_cache)
 {
 	struct ras_cmd_batch_trace_record_req req = {
@@ -204,27 +213,34 @@ static int amdgpu_virt_ras_get_batch_records(struct ras_core_context *ras_core,
 	struct batch_ras_trace_info *batch;
 	int ret = 0;
 	uint32_t i;
+	uint32_t idx;
 
-	if (!rsp->real_batch_num || (batch_id < rsp->start_batch_id) ||
-		(batch_id >=  (rsp->start_batch_id + rsp->real_batch_num))) {
-
+	if (!amdgpu_virt_ras_check_batch_cached(rsp, batch_id)) {
 		memset(rsp, 0, sizeof(*rsp));
 		ret = amdgpu_virt_ras_send_remote_cmd(ras_core, RAS_CMD__GET_BATCH_TRACE_RECORD,
 			&req, sizeof(req), rsp, sizeof(*rsp));
 		if (ret)
 			return -EPIPE;
-	}
 
-	batch = &rsp->batchs[batch_id - rsp->start_batch_id];
-	if (batch_id != batch->batch_id)
-		return -ENODATA;
+		if (!amdgpu_virt_ras_check_batch_cached(rsp, batch_id)) {
+			memset(rsp, 0, sizeof(*rsp));
+			return -EIO;
+		}
+	}
 
-	for (i = 0; i < batch->trace_num; i++) {
-		if (i >= arr_num)
-			break;
-		trace_arr[i] = &rsp->records[batch->offset + i];
+	idx = (uint32_t)(batch_id - rsp->start_batch_id);
+	batch = &rsp->batchs[idx];
+	if (batch_id != batch->batch_id ||
+	    batch->trace_num > MAX_RECORD_PER_BATCH ||
+	    (uint32_t)batch->offset + batch->trace_num > RAS_CMD_MAX_TRACE_NUM) {
+		memset(rsp, 0, sizeof(*rsp));
+		return -EIO;
 	}
 
+	for (i = 0; i < batch->trace_num && i < arr_num; i++)
+		memcpy(&trace_arr[i],
+			&rsp->records[batch->offset + i], sizeof(*trace_arr));
+
 	return i;
 }
 
@@ -240,19 +256,22 @@ static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core,
 		(struct ras_cmd_cper_record_rsp *)cmd->output_buff_raw;
 	struct ras_log_batch_overview *overview = &virt_ras->batch_mgr.batch_overview;
 	struct ras_cmd_batch_trace_record_rsp *rsp_cache = &virt_ras->batch_mgr.batch_trace;
-	struct ras_log_info **trace;
+	struct ras_log_info *trace;
+	uint32_t trace_count = MAX_RECORD_PER_BATCH;
 	uint32_t offset = 0, real_data_len = 0;
 	uint64_t batch_id;
 	uint8_t *out_buf;
 	int ret = 0, i, count;
 
-	if (cmd->input_size != sizeof(struct ras_cmd_cper_record_req))
+	if (cmd->input_size != sizeof(struct ras_cmd_cper_record_req) ||
+		(cmd->output_buf_size < sizeof(*rsp)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
-	if (!req->buf_size || !req->buf_ptr || !req->cper_num)
+	if (!req->buf_size || !req->buf_ptr || !req->cper_num ||
+	    req->buf_size > RAS_CMD_MAX_CPER_BUF_SZ)
 		return RAS_CMD__ERROR_INVALID_INPUT_DATA;
 
-	trace = kzalloc_objs(*trace, MAX_RECORD_PER_BATCH);
+	trace = kzalloc_objs(*trace, trace_count);
 	if (!trace)
 		return RAS_CMD__ERROR_GENERIC;
 
@@ -269,7 +288,7 @@ static int amdgpu_virt_ras_get_cper_records(struct ras_core_context *ras_core,
 		if (batch_id >= overview->last_batch_id)
 			break;
 		count = amdgpu_virt_ras_get_batch_records(ras_core, batch_id,
-							  trace, MAX_RECORD_PER_BATCH,
+							  trace, trace_count,
 							  rsp_cache);
 		if (count > 0) {
 			ret = ras_cper_generate_cper(ras_core, trace, count,
@@ -455,7 +474,7 @@ int amdgpu_virt_ras_handle_cmd(struct ras_core_context *ras_core,
 
 	cmd->cmd_res = res;
 
-	if (cmd->output_size > cmd->output_buf_size) {
+	if (!res && (cmd->output_size > cmd->output_buf_size)) {
 		RAS_DEV_ERR(ras_core->dev,
 			"Output data size 0x%x exceeds buffer size 0x%x!\n",
 			cmd->output_size, cmd->output_buf_size);
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras.h b/drivers/gpu/drm/amd/ras/rascore/ras.h
index 6449d7b8627d6..c059fcebaf005 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras.h
@@ -329,7 +329,6 @@ struct ras_core_context {
 	struct ras_gfx ras_gfx;
 	struct ras_mp1 ras_mp1;
 	struct ras_process ras_proc;
-	struct ras_cmd_mgr ras_cmd;
 	struct ras_log_ring ras_log_ring;
 
 	const struct ras_sys_func *sys_fn;
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c
index 4f89810d85a15..abd004eee95e7 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.c
@@ -28,20 +28,6 @@
 #define RAS_CMD_MINOR_VERSION 0
 #define RAS_CMD_VERSION  (((RAS_CMD_MAJOR_VERSION) << 10) | (RAS_CMD_MINOR_VERSION))
 
-static int ras_cmd_add_device(struct ras_core_context *ras_core)
-{
-	INIT_LIST_HEAD(&ras_core->ras_cmd.head);
-	ras_core->ras_cmd.ras_core = ras_core;
-	ras_core->ras_cmd.dev_handle = (uintptr_t)ras_core ^ RAS_CMD_DEV_HANDLE_MAGIC;
-	return 0;
-}
-
-static int ras_cmd_remove_device(struct ras_core_context *ras_core)
-{
-	memset(&ras_core->ras_cmd, 0, sizeof(ras_core->ras_cmd));
-	return 0;
-}
-
 static int ras_get_block_ecc_info(struct ras_core_context *ras_core,
 				struct ras_cmd_ctx *cmd, void *data)
 {
@@ -52,7 +38,8 @@ static int ras_get_block_ecc_info(struct ras_core_context *ras_core,
 	struct ras_ecc_count err_data;
 	int ret;
 
-	if (cmd->input_size != sizeof(struct ras_cmd_block_ecc_info_req))
+	if ((cmd->input_size != sizeof(struct ras_cmd_block_ecc_info_req)) ||
+		(cmd->output_buf_size < sizeof(*output_data)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
 	memset(&err_data, 0, sizeof(err_data));
@@ -87,25 +74,31 @@ static int ras_cmd_get_group_bad_pages(struct ras_core_context *ras_core,
 	struct eeprom_umc_record record;
 	struct ras_cmd_bad_page_record *ras_cmd_record;
 	uint32_t i = 0, bp_cnt = 0, group_cnt = 0;
+	int ret = RAS_CMD__SUCCESS;
 
 	output_data->bp_in_group = 0;
 	output_data->group_index = 0;
 
+	mutex_lock(&ras_core->ras_umc.umc_lock);
 	bp_cnt = ras_umc_get_badpage_count(ras_core);
 	if (bp_cnt) {
 		output_data->group_index = group_index;
 		group_cnt = bp_cnt / RAS_CMD_MAX_BAD_PAGES_PER_GROUP
 			+ ((bp_cnt % RAS_CMD_MAX_BAD_PAGES_PER_GROUP) ? 1 : 0);
 
-		if (group_index >= group_cnt)
-			return RAS_CMD__ERROR_INVALID_INPUT_DATA;
+		if (group_index >= group_cnt) {
+			ret = RAS_CMD__ERROR_INVALID_INPUT_DATA;
+			goto out;
+		}
 
 		i = group_index * RAS_CMD_MAX_BAD_PAGES_PER_GROUP;
 		for (;
 		   i < bp_cnt && output_data->bp_in_group < RAS_CMD_MAX_BAD_PAGES_PER_GROUP;
 		   i++) {
-			if (ras_umc_get_badpage_record(ras_core, i, &record))
-				return RAS_CMD__ERROR_GENERIC;
+			if (ras_umc_get_badpage_record(ras_core, i, &record)) {
+				ret = RAS_CMD__ERROR_GENERIC;
+				goto out;
+			}
 
 			ras_cmd_record = &output_data->records[i % RAS_CMD_MAX_BAD_PAGES_PER_GROUP];
 
@@ -115,7 +108,10 @@ static int ras_cmd_get_group_bad_pages(struct ras_core_context *ras_core,
 		}
 	}
 	output_data->bp_total_cnt = bp_cnt;
-	return RAS_CMD__SUCCESS;
+
+out:
+	mutex_unlock(&ras_core->ras_umc.umc_lock);
+	return ret;
 }
 
 static int ras_cmd_get_bad_pages(struct ras_core_context *ras_core,
@@ -127,7 +123,8 @@ static int ras_cmd_get_bad_pages(struct ras_core_context *ras_core,
 			(struct ras_cmd_bad_pages_info_rsp *)cmd->output_buff_raw;
 	int ret;
 
-	if (cmd->input_size != sizeof(struct ras_cmd_bad_pages_info_req))
+	if ((cmd->input_size != sizeof(struct ras_cmd_bad_pages_info_req)) ||
+		(cmd->output_buf_size < sizeof(*output_data)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
 	ret = ras_cmd_get_group_bad_pages(ras_core, input_data->group_index, output_data);
@@ -182,7 +179,8 @@ static int ras_cmd_get_cper_snapshot(struct ras_core_context *ras_core,
 			(struct ras_cmd_cper_snapshot_rsp *)cmd->output_buff_raw;
 	struct ras_log_batch_overview overview;
 
-	if (cmd->input_size != sizeof(struct ras_cmd_cper_snapshot_req))
+	if ((cmd->input_size != sizeof(struct ras_cmd_cper_snapshot_req)) ||
+		(cmd->output_buf_size < sizeof(*output_data)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
 	ras_log_ring_get_batch_overview(ras_core, &overview);
@@ -204,23 +202,32 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core,
 			(struct ras_cmd_cper_record_req *)cmd->input_buff_raw;
 	struct ras_cmd_cper_record_rsp *rsp =
 			(struct ras_cmd_cper_record_rsp *)cmd->output_buff_raw;
-	struct ras_log_info *trace[MAX_RECORD_PER_BATCH] = {0};
+	struct ras_log_info *trace = NULL;
+	uint32_t trace_count = MAX_RECORD_PER_BATCH;
 	struct ras_log_batch_overview overview;
 	uint32_t offset = 0, real_data_len = 0;
 	uint64_t batch_id;
-	uint8_t *buffer;
+	uint8_t *buffer = NULL;
 	int ret = 0, i, count;
 
-	if (cmd->input_size != sizeof(struct ras_cmd_cper_record_req))
+	if ((cmd->input_size != sizeof(struct ras_cmd_cper_record_req)) ||
+		(cmd->output_buf_size < sizeof(*rsp)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
-	if (!req->buf_size || !req->buf_ptr || !req->cper_num)
+	if (!req->buf_size || !req->buf_ptr || !req->cper_num ||
+	    req->buf_size > RAS_CMD_MAX_CPER_BUF_SZ)
 		return RAS_CMD__ERROR_INVALID_INPUT_DATA;
 
 	buffer = kzalloc(req->buf_size, GFP_KERNEL);
 	if (!buffer)
 		return RAS_CMD__ERROR_GENERIC;
 
+	trace = kcalloc(trace_count, sizeof(*trace), GFP_KERNEL);
+	if (!trace) {
+		ret = RAS_CMD__ERROR_GENERIC;
+		goto out;
+	}
+
 	ras_log_ring_get_batch_overview(ras_core, &overview);
 	for (i = 0; i < req->cper_num; i++) {
 		batch_id = req->cper_start_id + i;
@@ -228,7 +235,7 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core,
 			break;
 
 		count = ras_log_ring_get_batch_records(ras_core, batch_id, trace,
-					ARRAY_SIZE(trace));
+					trace_count);
 		if (count > 0) {
 			ret = ras_cper_generate_cper(ras_core, trace, count,
 					&buffer[offset], req->buf_size - offset, &real_data_len);
@@ -241,8 +248,8 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core,
 
 	if ((ret && (ret != -ENOMEM)) ||
 		copy_to_user(u64_to_user_ptr(req->buf_ptr), buffer, offset)) {
-		kfree(buffer);
-		return RAS_CMD__ERROR_GENERIC;
+		ret = RAS_CMD__ERROR_GENERIC;
+		goto out;
 	}
 
 	rsp->real_data_size = offset;
@@ -251,10 +258,12 @@ static int ras_cmd_get_cper_records(struct ras_core_context *ras_core,
 	rsp->version = 0;
 
 	cmd->output_size = sizeof(struct ras_cmd_cper_record_rsp);
+	ret = RAS_CMD__SUCCESS;
 
+out:
+	kfree(trace);
 	kfree(buffer);
-
-	return RAS_CMD__SUCCESS;
+	return ret;
 }
 
 static int ras_cmd_get_batch_trace_snapshot(struct ras_core_context *ras_core,
@@ -265,7 +274,8 @@ static int ras_cmd_get_batch_trace_snapshot(struct ras_core_context *ras_core,
 	struct ras_log_batch_overview overview;
 
 
-	if (cmd->input_size != sizeof(struct ras_cmd_batch_trace_snapshot_req))
+	if ((cmd->input_size != sizeof(struct ras_cmd_batch_trace_snapshot_req)) ||
+		(cmd->output_buf_size < sizeof(*rsp)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
 	ras_log_ring_get_batch_overview(ras_core, &overview);
@@ -287,13 +297,15 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core,
 	struct ras_cmd_batch_trace_record_rsp *output_data =
 			(struct ras_cmd_batch_trace_record_rsp *)cmd->output_buff_raw;
 	struct ras_log_batch_overview overview;
-	struct ras_log_info *trace_arry[MAX_RECORD_PER_BATCH] = {0};
+	struct ras_log_info *trace_arry = NULL;
+	uint32_t trace_count = MAX_RECORD_PER_BATCH;
 	struct ras_log_info *record;
 	int i, j, count = 0, offset = 0;
 	uint64_t id;
 	bool completed = false;
 
-	if (cmd->input_size != sizeof(struct ras_cmd_batch_trace_record_req))
+	if ((cmd->input_size != sizeof(struct ras_cmd_batch_trace_record_req)) ||
+		(cmd->output_buf_size < sizeof(*output_data)))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
 	if ((!input_data->batch_num) || (input_data->batch_num > RAS_CMD_MAX_BATCH_NUM))
@@ -304,6 +316,10 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core,
 	    (input_data->start_batch_id >= overview.last_batch_id))
 		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
 
+	trace_arry = kcalloc(trace_count, sizeof(*trace_arry), GFP_KERNEL);
+	if (!trace_arry)
+		return RAS_CMD__ERROR_GENERIC;
+
 	for (i = 0; i < input_data->batch_num; i++) {
 		id = input_data->start_batch_id + i;
 		if (id >= overview.last_batch_id) {
@@ -312,17 +328,17 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core,
 		}
 
 		count = ras_log_ring_get_batch_records(ras_core,
-					id, trace_arry, ARRAY_SIZE(trace_arry));
+					id, trace_arry, trace_count);
 		if (count > 0) {
 			if ((offset + count) > RAS_CMD_MAX_TRACE_NUM)
 				break;
 			for (j = 0; j < count; j++) {
 				record = &output_data->records[offset + j];
-				record->seqno = trace_arry[j]->seqno;
-				record->timestamp = trace_arry[j]->timestamp;
-				record->event = trace_arry[j]->event;
+				record->seqno = trace_arry[j].seqno;
+				record->timestamp = trace_arry[j].timestamp;
+				record->event = trace_arry[j].event;
 				memcpy(&record->aca_reg,
-					&trace_arry[j]->aca_reg, sizeof(trace_arry[j]->aca_reg));
+					&trace_arry[j].aca_reg, sizeof(trace_arry[j].aca_reg));
 			}
 		} else {
 			count = 0;
@@ -341,6 +357,8 @@ static int ras_cmd_get_batch_trace_records(struct ras_core_context *ras_core,
 
 	cmd->output_size = sizeof(struct ras_cmd_batch_trace_record_rsp);
 
+	kfree(trace_arry);
+
 	return RAS_CMD__SUCCESS;
 }
 
@@ -420,6 +438,10 @@ static int ras_cmd_inject_error(struct ras_core_context *ras_core,
 		.value = req->method,
 	};
 
+	if ((cmd->input_size != sizeof(*req)) ||
+		(cmd->output_buf_size < sizeof(*output_data)))
+		return RAS_CMD__ERROR_INVALID_INPUT_SIZE;
+
 	ret = ras_psp_trigger_error(ras_core, &block_info, req->instance_mask);
 	if (!ret) {
 		output_data->version = 0;
@@ -466,12 +488,11 @@ int rascore_handle_cmd(struct ras_core_context *ras_core,
 
 int ras_cmd_init(struct ras_core_context *ras_core)
 {
-	return ras_cmd_add_device(ras_core);
+	return 0;
 }
 
 int ras_cmd_fini(struct ras_core_context *ras_core)
 {
-	ras_cmd_remove_device(ras_core);
 	return 0;
 }
 
@@ -520,8 +541,3 @@ int ras_cmd_translate_bank_to_soc_pa(struct ras_core_context *ras_core,
 
 	return ras_umc_translate_soc_pa_and_bank(ras_core, soc_pa, &umc_bank, true);
 }
-
-uint64_t ras_cmd_get_dev_handle(struct ras_core_context *ras_core)
-{
-	return ras_core->ras_cmd.dev_handle;
-}
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h
index 7ea35a0289874..c376e074e15da 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_cmd.h
@@ -29,8 +29,6 @@
 #include "ras_log_ring.h"
 #include "ras_cper.h"
 
-#define RAS_CMD_DEV_HANDLE_MAGIC 0xFEEDAD00UL
-
 #define RAS_CMD_MAX_IN_SIZE 256
 #define RAS_CMD_MAX_GPU_NUM 32
 #define RAS_CMD_MAX_BAD_PAGES_PER_GROUP 32
@@ -107,12 +105,6 @@ enum ras_error_type {
 struct ras_core_context;
 struct ras_cmd_ctx;
 
-struct ras_cmd_mgr {
-	struct list_head head;
-	struct ras_core_context *ras_core;
-	uint64_t dev_handle;
-};
-
 struct ras_cmd_func_map {
 	uint32_t cmd_id;
 	int (*func)(struct ras_core_context *ras_core,
@@ -405,6 +397,9 @@ struct batch_ras_trace_info {
 
 #define RAS_CMD_MAX_BATCH_NUM  300
 #define RAS_CMD_MAX_TRACE_NUM  300
+
+/* Upper bounds for RAS_CMD__GET_CPER_RECORD to limit kernel allocations and work. */
+#define RAS_CMD_MAX_CPER_BUF_SZ	(2 * 1024U * 1024U) /* 2 MiB */
 struct ras_cmd_batch_trace_record_rsp {
 	uint32_t version;
 	uint16_t real_batch_num;
@@ -478,7 +473,6 @@ struct ras_cmd_blocks_ecc_rsp {
 int ras_cmd_init(struct ras_core_context *ras_core);
 int ras_cmd_fini(struct ras_core_context *ras_core);
 int rascore_handle_cmd(struct ras_core_context *ras_core, struct ras_cmd_ctx *cmd, void *data);
-uint64_t ras_cmd_get_dev_handle(struct ras_core_context *ras_core);
 int ras_cmd_query_interface_info(struct ras_core_context *ras_core,
 	struct ras_query_interface_info_rsp *rsp);
 int ras_cmd_translate_soc_pa_to_bank(struct ras_core_context *ras_core,
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_core.c b/drivers/gpu/drm/amd/ras/rascore/ras_core.c
index 49b3aa7489ff4..62d124a3eeac2 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_core.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_core.c
@@ -196,9 +196,9 @@ uint64_t ras_core_gen_seqno(struct ras_core_context *ras_core,
 {
 	uint64_t seqno = 0;
 
-	if (ras_core->sys_fn &&
-		ras_core->sys_fn->gen_seqno)
-		ras_core->sys_fn->gen_seqno(ras_core, type, &seqno);
+	if (ras_core->sys_fn && ras_core->sys_fn->gen_seqno &&
+	    ras_core->sys_fn->gen_seqno(ras_core, type, &seqno))
+		return 0;
 
 	return seqno;
 }
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cper.c b/drivers/gpu/drm/amd/ras/rascore/ras_cper.c
index 0fc7522b7ab6b..6e93a13bbc4ce 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_cper.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_cper.c
@@ -175,14 +175,14 @@ static int fill_section_runtime(struct ras_core_context *ras_core,
 }
 
 static int cper_generate_runtime_record(struct ras_core_context *ras_core,
-	struct cper_section_hdr *hdr, struct ras_log_info **trace_arr, uint32_t arr_num,
+	struct cper_section_hdr *hdr, struct ras_log_info *trace_arr, uint32_t arr_num,
 		enum ras_cper_severity sev)
 {
 	struct cper_section_descriptor *descriptor;
 	struct cper_section_runtime *runtime;
 	int i;
 
-	fill_section_hdr(ras_core, hdr, RAS_CPER_TYPE_RUNTIME, sev, trace_arr[0]);
+	fill_section_hdr(ras_core, hdr, RAS_CPER_TYPE_RUNTIME, sev, &trace_arr[0]);
 	hdr->record_length =  RAS_HDR_LEN + ((RAS_SEC_DESC_LEN + RAS_NONSTD_SEC_LEN) * arr_num);
 	hdr->sec_cnt = arr_num;
 	for (i = 0; i < arr_num; i++) {
@@ -194,21 +194,21 @@ static int cper_generate_runtime_record(struct ras_core_context *ras_core,
 		fill_section_descriptor(ras_core, descriptor, sev, RUNTIME,
 			RAS_NONSTD_SEC_OFFSET(hdr->sec_cnt, i),
 			sizeof(struct cper_section_runtime));
-		fill_section_runtime(ras_core, runtime, trace_arr[i], sev);
+		fill_section_runtime(ras_core, runtime, &trace_arr[i], sev);
 	}
 
 	return 0;
 }
 
 static int cper_generate_fatal_record(struct ras_core_context *ras_core,
-	uint8_t *buffer, struct ras_log_info **trace_arr, uint32_t arr_num)
+	uint8_t *buffer, struct ras_log_info *trace_arr, uint32_t arr_num)
 {
 	struct ras_cper_fatal_record record = {0};
 	int i = 0;
 
 	for (i = 0; i < arr_num; i++) {
 		fill_section_hdr(ras_core, &record.hdr, RAS_CPER_TYPE_FATAL,
-				 RAS_CPER_SEV_FATAL_UE, trace_arr[i]);
+				 RAS_CPER_SEV_FATAL_UE, &trace_arr[i]);
 		record.hdr.record_length =  RAS_HDR_LEN + RAS_SEC_DESC_LEN + RAS_FATAL_SEC_LEN;
 		record.hdr.sec_cnt = 1;
 
@@ -216,7 +216,7 @@ static int cper_generate_fatal_record(struct ras_core_context *ras_core,
 					CRASHDUMP, offsetof(struct ras_cper_fatal_record, fatal),
 					sizeof(struct cper_section_fatal));
 
-		fill_section_fatal(ras_core, &record.fatal, trace_arr[i]);
+		fill_section_fatal(ras_core, &record.fatal, &trace_arr[i]);
 
 		memcpy(buffer + (i * record.hdr.record_length),
 				&record, record.hdr.record_length);
@@ -271,7 +271,7 @@ static enum ras_cper_type cper_ras_log_event_to_cper_type(enum ras_log_event eve
 }
 
 int ras_cper_generate_cper(struct ras_core_context *ras_core,
-		struct ras_log_info **trace_list, uint32_t count,
+		struct ras_log_info *trace_list, uint32_t count,
 		uint8_t *buf, uint32_t buf_len, uint32_t *real_data_len)
 {
 	uint8_t *buffer = buf;
@@ -281,14 +281,14 @@ int ras_cper_generate_cper(struct ras_core_context *ras_core,
 
 	/* All the batch traces share the same event */
 	record_size = cper_get_record_size(
-			cper_ras_log_event_to_cper_type(trace_list[0]->event), count);
+			cper_ras_log_event_to_cper_type(trace_list[0].event), count);
 
 	if ((record_size + saved_size) > buf_size)
 		return -ENOMEM;
 
 	hdr = (struct cper_section_hdr *)(buffer + saved_size);
 
-	switch (trace_list[0]->event) {
+	switch (trace_list[0].event) {
 	case RAS_LOG_EVENT_RMA:
 		cper_generate_runtime_record(ras_core, hdr, trace_list, count, RAS_CPER_SEV_RMA);
 		break;
@@ -304,7 +304,7 @@ int ras_cper_generate_cper(struct ras_core_context *ras_core,
 		cper_generate_fatal_record(ras_core, buffer + saved_size, trace_list, count);
 		break;
 	default:
-		RAS_DEV_WARN(ras_core->dev, "Unprocessed trace event: %d\n", trace_list[0]->event);
+		RAS_DEV_WARN(ras_core->dev, "Unprocessed trace event: %d\n", trace_list[0].event);
 		break;
 	}
 
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_cper.h b/drivers/gpu/drm/amd/ras/rascore/ras_cper.h
index 076c1883c1ce4..e4e3615ecc2e0 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_cper.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_cper.h
@@ -299,6 +299,6 @@ struct ras_cper_fatal_record {
 struct ras_core_context;
 struct ras_log_info;
 int ras_cper_generate_cper(struct ras_core_context *ras_core,
-		struct ras_log_info **trace_list, uint32_t count,
+		struct ras_log_info *trace_list, uint32_t count,
 		uint8_t *buf, uint32_t buf_len, uint32_t *real_data_len);
 #endif
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c
index 65c1812a10fb1..34b798f45af21 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom.c
@@ -1163,6 +1163,13 @@ static int __check_ras_table_status(struct ras_core_context *ras_core)
 	}
 
 	control->ras_fri = RAS_OFFSET_TO_INDEX(control, hdr->first_rec_offset);
+	if (hdr->first_rec_offset < control->ras_record_offset ||
+	    control->ras_fri >= control->ras_max_record_count) {
+		RAS_DEV_ERR(ras_core->dev,
+			"RAS header invalid, ras_fri: %u, first_rec_offset:0x%x",
+			control->ras_fri, hdr->first_rec_offset);
+		return -EINVAL;
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c
index 29001e606d1bc..f5fa80db91fb1 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_eeprom_fw.c
@@ -270,6 +270,7 @@ int ras_fw_eeprom_read_idx(struct ras_core_context *ras_core,
 	struct ras_fw_eeprom_control *control = &ras_core->ras_fw_eeprom;
 	int i, ret, end_idx;
 	u64 mca, ipid, ts;
+	u32 cu, mem_channel, mcumc_id;
 
 	if (!ras_core->ras_umc.ip_func ||
 	    !ras_core->ras_umc.ip_func->mca_ipid_parse)
@@ -299,9 +300,10 @@ int ras_fw_eeprom_read_idx(struct ras_core_context *ras_core,
 			record_umc[i - rec_idx].err_type = RAS_EEPROM_ERR_NON_RECOVERABLE;
 
 			ras_core->ras_umc.ip_func->mca_ipid_parse(ras_core, ipid,
-				(uint32_t *)&(record_umc[i - rec_idx].cu),
-				(uint32_t *)&(record_umc[i - rec_idx].mem_channel),
-				(uint32_t *)&(record_umc[i - rec_idx].mcumc_id), NULL);
+				&cu, &mem_channel, &mcumc_id, NULL);
+			record_umc[i - rec_idx].cu = (u8)cu;
+			record_umc[i - rec_idx].mem_channel = (u8)mem_channel;
+			record_umc[i - rec_idx].mcumc_id = (u8)mcumc_id;
 
 			/* update bad channel bitmap */
 			if ((record_umc[i - rec_idx].mem_channel < BITS_PER_TYPE(control->bad_channel_bitmap)) &&
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.c b/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.c
index f10e856f3c8cf..c37343db39dfd 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.c
@@ -265,8 +265,8 @@ void ras_log_ring_add_log_event(struct ras_core_context *ras_core,
 	ras_log_ring_add_data(ras_core, log, batch_tag);
 }
 
-static struct ras_log_info *ras_log_ring_lookup_data(struct ras_core_context *ras_core,
-					uint64_t idx)
+static int ras_log_ring_lookup_data(struct ras_core_context *ras_core,
+					uint64_t idx, struct ras_log_info *log)
 {
 	struct ras_log_ring *log_ring = &ras_core->ras_log_ring;
 	unsigned long flags = 0;
@@ -274,30 +274,27 @@ static struct ras_log_info *ras_log_ring_lookup_data(struct ras_core_context *ra
 
 	spin_lock_irqsave(&log_ring->spin_lock, flags);
 	data = radix_tree_lookup(&log_ring->ras_log_root, idx);
+	if (data)
+		memcpy(log, data, sizeof(*log));
 	spin_unlock_irqrestore(&log_ring->spin_lock, flags);
 
-	return (struct ras_log_info *)data;
+	return data ? 0 : -ENODATA;
 }
 
 int ras_log_ring_get_batch_records(struct ras_core_context *ras_core, uint64_t batch_id,
-		struct ras_log_info **log_arr, uint32_t arr_num)
+		struct ras_log_info *log_arr, uint32_t arr_num)
 {
 	struct ras_log_ring *log_ring = &ras_core->ras_log_ring;
 	uint32_t i, idx, count = 0;
-	void *data;
 
-	if ((batch_id >= log_ring->mono_upward_batch_id) ||
+	if (!log_arr || !arr_num || (batch_id >= log_ring->mono_upward_batch_id) ||
 		(batch_id < log_ring->last_del_batch_id))
 		return -EINVAL;
 
-	for (i = 0; i < MAX_RECORD_PER_BATCH; i++) {
+	for (i = 0; i < MAX_RECORD_PER_BATCH && i < arr_num; i++) {
 		idx = BATCH_IDX_TO_TREE_IDX(batch_id, i);
-		data = ras_log_ring_lookup_data(ras_core, idx);
-		if (data) {
-			log_arr[count++] = data;
-			if (count >= arr_num)
-				break;
-		}
+		if (!ras_log_ring_lookup_data(ras_core, idx, &log_arr[count]))
+			count++;
 	}
 
 	return count;
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.h b/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.h
index 0ff6cc35678dd..cb66beaa9f432 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.h
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_log_ring.h
@@ -86,7 +86,7 @@ void ras_log_ring_add_log_event(struct ras_core_context *ras_core,
 		enum ras_log_event event, void *data, struct ras_log_batch_tag *tag);
 
 int ras_log_ring_get_batch_records(struct ras_core_context *ras_core, uint64_t batch_idx,
-		struct ras_log_info **log_arr, uint32_t arr_num);
+		struct ras_log_info *log_arr, uint32_t arr_num);
 
 int ras_log_ring_get_batch_overview(struct ras_core_context *ras_core,
 		struct ras_log_batch_overview *overview);
diff --git a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
index c7ae005ea7052..91dd730de3cec 100644
--- a/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
+++ b/drivers/gpu/drm/amd/ras/rascore/ras_umc.c
@@ -391,9 +391,8 @@ static int ras_umc_update_eeprom_ram_data(struct ras_core_context *ras_core,
 			data->space_left--;
 		}
 	} else {
-		memcpy(&data->bps[data->count], bps, sizeof(*data->bps));
-		data->count++;
-		data->space_left--;
+		RAS_DEV_ERR(ras_core->dev, "Failed to convert record to nps pages!");
+		return -EINVAL;
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
index 714af5c889d74..e8cb782a6f8ed 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_crtc.c
@@ -75,7 +75,7 @@ static void komeda_crtc_update_clock_ratio(struct komeda_crtc_state *kcrtc_st)
  */
 static int
 komeda_crtc_atomic_check(struct drm_crtc *crtc,
-			 struct drm_atomic_state *state)
+			 struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -279,7 +279,7 @@ komeda_crtc_do_flush(struct drm_crtc *crtc,
 
 static void
 komeda_crtc_atomic_enable(struct drm_crtc *crtc,
-			  struct drm_atomic_state *state)
+			  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old = drm_atomic_get_old_crtc_state(state,
 								   crtc);
@@ -325,7 +325,7 @@ komeda_crtc_flush_and_wait_for_flip_done(struct komeda_crtc *kcrtc,
 
 static void
 komeda_crtc_atomic_disable(struct drm_crtc *crtc,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old = drm_atomic_get_old_crtc_state(state,
 								   crtc);
@@ -387,7 +387,7 @@ komeda_crtc_atomic_disable(struct drm_crtc *crtc,
 
 static void
 komeda_crtc_atomic_flush(struct drm_crtc *crtc,
-			 struct drm_atomic_state *state)
+			 struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_kms.c b/drivers/gpu/drm/arm/display/komeda/komeda_kms.c
index 6ed5040991887..faa8b041373a7 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_kms.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_kms.c
@@ -67,7 +67,7 @@ static const struct drm_driver komeda_kms_driver = {
 	.minor = 1,
 };
 
-static void komeda_kms_atomic_commit_hw_done(struct drm_atomic_state *state)
+static void komeda_kms_atomic_commit_hw_done(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct komeda_kms_dev *kms = to_kdev(dev);
@@ -86,7 +86,7 @@ static void komeda_kms_atomic_commit_hw_done(struct drm_atomic_state *state)
 	drm_atomic_helper_commit_hw_done(state);
 }
 
-static void komeda_kms_commit_tail(struct drm_atomic_state *old_state)
+static void komeda_kms_commit_tail(struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = old_state->dev;
 	bool fence_cookie = dma_fence_begin_signalling();
@@ -153,7 +153,7 @@ static int komeda_plane_state_list_add(struct drm_plane_state *plane_st,
 static int komeda_crtc_normalize_zpos(struct drm_crtc *crtc,
 				      struct drm_crtc_state *crtc_st)
 {
-	struct drm_atomic_state *state = crtc_st->state;
+	struct drm_atomic_commit *state = crtc_st->state;
 	struct komeda_crtc *kcrtc = to_kcrtc(crtc);
 	struct komeda_crtc_state *kcrtc_st = to_kcrtc_st(crtc_st);
 	struct komeda_plane_state *kplane_st;
@@ -216,7 +216,7 @@ static int komeda_crtc_normalize_zpos(struct drm_crtc *crtc,
 }
 
 static int komeda_kms_check(struct drm_device *dev,
-			    struct drm_atomic_state *state)
+			    struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_st;
@@ -228,7 +228,7 @@ static int komeda_kms_check(struct drm_device *dev,
 
 	/* Komeda need to re-calculate resource assumption in every commit
 	 * so need to add all affected_planes (even unchanged) to
-	 * drm_atomic_state.
+	 * drm_atomic_commit.
 	 */
 	for_each_new_crtc_in_state(state, crtc, new_crtc_st, i) {
 		err = drm_atomic_add_affected_planes(state, crtc);
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h
index bbee6da43164f..955f9f683ce31 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline.h
@@ -155,7 +155,7 @@ struct komeda_component_output {
  * @komeda_scaler_state
  */
 struct komeda_component_state {
-	/** @obj: tracking component_state by drm_atomic_state */
+	/** @obj: tracking component_state by drm_atomic_commit */
 	struct drm_private_state obj;
 	/** @component: backpointer to the component */
 	struct komeda_component *component;
@@ -388,7 +388,7 @@ struct komeda_pipeline_funcs {
  * Represent a complete display pipeline and hold all functional components.
  */
 struct komeda_pipeline {
-	/** @obj: link pipeline as private obj of drm_atomic_state */
+	/** @obj: link pipeline as private obj of drm_atomic_commit */
 	struct drm_private_obj obj;
 	/** @mdev: the parent komeda_dev */
 	struct komeda_dev *mdev;
@@ -448,10 +448,10 @@ struct komeda_pipeline {
  *
  * NOTE:
  * Unlike the pipeline, pipeline_state doesn’t gather any component_state
- * into it. It because all component will be managed by drm_atomic_state.
+ * into it. It because all component will be managed by drm_atomic_commit.
  */
 struct komeda_pipeline_state {
-	/** @obj: tracking pipeline_state by drm_atomic_state */
+	/** @obj: tracking pipeline_state by drm_atomic_commit */
 	struct drm_private_state obj;
 	/** @pipe: backpointer to the pipeline */
 	struct komeda_pipeline *pipe;
@@ -555,11 +555,11 @@ int komeda_release_unclaimed_resources(struct komeda_pipeline *pipe,
 
 struct komeda_pipeline_state *
 komeda_pipeline_get_old_state(struct komeda_pipeline *pipe,
-			      struct drm_atomic_state *state);
+			      struct drm_atomic_commit *state);
 bool komeda_pipeline_disable(struct komeda_pipeline *pipe,
-			     struct drm_atomic_state *old_state);
+			     struct drm_atomic_commit *old_state);
 void komeda_pipeline_update(struct komeda_pipeline *pipe,
-			    struct drm_atomic_state *old_state);
+			    struct drm_atomic_commit *old_state);
 
 void komeda_complete_data_flow_cfg(struct komeda_layer *layer,
 				   struct komeda_data_flow_cfg *dflow,
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c
index 6f9b10cc831ff..457b21397014c 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_pipeline_state.c
@@ -22,7 +22,7 @@ static inline bool is_switching_user(void *old, void *new)
 
 static struct komeda_pipeline_state *
 komeda_pipeline_get_state(struct komeda_pipeline *pipe,
-			  struct drm_atomic_state *state)
+			  struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv_st;
 
@@ -35,7 +35,7 @@ komeda_pipeline_get_state(struct komeda_pipeline *pipe,
 
 struct komeda_pipeline_state *
 komeda_pipeline_get_old_state(struct komeda_pipeline *pipe,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv_st;
 
@@ -47,7 +47,7 @@ komeda_pipeline_get_old_state(struct komeda_pipeline *pipe,
 
 static struct komeda_pipeline_state *
 komeda_pipeline_get_new_state(struct komeda_pipeline *pipe,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv_st;
 
@@ -60,7 +60,7 @@ komeda_pipeline_get_new_state(struct komeda_pipeline *pipe,
 /* Assign pipeline for crtc */
 static struct komeda_pipeline_state *
 komeda_pipeline_get_state_and_set_crtc(struct komeda_pipeline *pipe,
-				       struct drm_atomic_state *state,
+				       struct drm_atomic_commit *state,
 				       struct drm_crtc *crtc)
 {
 	struct komeda_pipeline_state *st;
@@ -97,7 +97,7 @@ komeda_pipeline_get_state_and_set_crtc(struct komeda_pipeline *pipe,
 
 static struct komeda_component_state *
 komeda_component_get_state(struct komeda_component *c,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv_st;
 
@@ -112,7 +112,7 @@ komeda_component_get_state(struct komeda_component *c,
 
 static struct komeda_component_state *
 komeda_component_get_old_state(struct komeda_component *c,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv_st;
 
@@ -149,7 +149,7 @@ komeda_component_get_old_state(struct komeda_component *c,
  */
 static struct komeda_component_state *
 komeda_component_get_state_and_set_user(struct komeda_component *c,
-					struct drm_atomic_state *state,
+					struct drm_atomic_commit *state,
 					void *user,
 					struct drm_crtc *crtc)
 {
@@ -253,7 +253,7 @@ komeda_component_validate_private(struct komeda_component *c,
 /* Get current available scaler from the component->supported_outputs */
 static struct komeda_scaler *
 komeda_component_get_avail_scaler(struct komeda_component *c,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct komeda_pipeline_state *pipe_st;
 	u32 avail_scalers;
@@ -505,7 +505,7 @@ komeda_scaler_validate(void *user,
 		       struct komeda_crtc_state *kcrtc_st,
 		       struct komeda_data_flow_cfg *dflow)
 {
-	struct drm_atomic_state *drm_st = kcrtc_st->base.state;
+	struct drm_atomic_commit *drm_st = kcrtc_st->base.state;
 	struct komeda_component_state *c_st;
 	struct komeda_scaler_state *st;
 	struct komeda_scaler *scaler;
@@ -669,7 +669,7 @@ komeda_compiz_set_input(struct komeda_compiz *compiz,
 			struct komeda_crtc_state *kcrtc_st,
 			struct komeda_data_flow_cfg *dflow)
 {
-	struct drm_atomic_state *drm_st = kcrtc_st->base.state;
+	struct drm_atomic_commit *drm_st = kcrtc_st->base.state;
 	struct komeda_component_state *c_st, *old_st;
 	struct komeda_compiz_input_cfg *cin;
 	u16 compiz_w, compiz_h;
@@ -1227,7 +1227,7 @@ static int
 komeda_pipeline_unbound_components(struct komeda_pipeline *pipe,
 				   struct komeda_pipeline_state *new)
 {
-	struct drm_atomic_state *drm_st = new->obj.state;
+	struct drm_atomic_commit *drm_st = new->obj.state;
 	struct komeda_pipeline_state *old = priv_to_pipe_st(pipe->obj.state);
 	struct komeda_component_state *c_st;
 	struct komeda_component *c;
@@ -1255,7 +1255,7 @@ komeda_pipeline_unbound_components(struct komeda_pipeline *pipe,
 int komeda_release_unclaimed_resources(struct komeda_pipeline *pipe,
 				       struct komeda_crtc_state *kcrtc_st)
 {
-	struct drm_atomic_state *drm_st = kcrtc_st->base.state;
+	struct drm_atomic_commit *drm_st = kcrtc_st->base.state;
 	struct komeda_pipeline_state *st;
 
 	/* ignore the pipeline which is not affected */
@@ -1285,7 +1285,7 @@ int komeda_release_unclaimed_resources(struct komeda_pipeline *pipe,
  * false: disable is complete.
  */
 bool komeda_pipeline_disable(struct komeda_pipeline *pipe,
-			     struct drm_atomic_state *old_state)
+			     struct drm_atomic_commit *old_state)
 {
 	struct komeda_pipeline_state *old;
 	struct komeda_component *c;
@@ -1330,7 +1330,7 @@ bool komeda_pipeline_disable(struct komeda_pipeline *pipe,
 }
 
 void komeda_pipeline_update(struct komeda_pipeline *pipe,
-			    struct drm_atomic_state *old_state)
+			    struct drm_atomic_commit *old_state)
 {
 	struct komeda_pipeline_state *new = priv_to_pipe_st(pipe->obj.state);
 	struct komeda_pipeline_state *old;
diff --git a/drivers/gpu/drm/arm/display/komeda/komeda_plane.c b/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
index 835c11fdd7ff5..57a4a0c7d404a 100644
--- a/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
+++ b/drivers/gpu/drm/arm/display/komeda/komeda_plane.c
@@ -69,7 +69,7 @@ komeda_plane_init_data_flow(struct drm_plane_state *st,
  */
 static int
 komeda_plane_atomic_check(struct drm_plane *plane,
-			  struct drm_atomic_state *state)
+			  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -116,7 +116,7 @@ komeda_plane_atomic_check(struct drm_plane *plane,
  */
 static void
 komeda_plane_atomic_update(struct drm_plane *plane,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 }
 
diff --git a/drivers/gpu/drm/arm/hdlcd_crtc.c b/drivers/gpu/drm/arm/hdlcd_crtc.c
index 4b4a08cb396dd..8565749a3e48d 100644
--- a/drivers/gpu/drm/arm/hdlcd_crtc.c
+++ b/drivers/gpu/drm/arm/hdlcd_crtc.c
@@ -175,7 +175,7 @@ static void hdlcd_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static void hdlcd_crtc_atomic_enable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct hdlcd_drm_private *hdlcd = crtc_to_hdlcd_priv(crtc);
 
@@ -186,7 +186,7 @@ static void hdlcd_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void hdlcd_crtc_atomic_disable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct hdlcd_drm_private *hdlcd = crtc_to_hdlcd_priv(crtc);
 
@@ -212,7 +212,7 @@ static enum drm_mode_status hdlcd_crtc_mode_valid(struct drm_crtc *crtc,
 }
 
 static void hdlcd_crtc_atomic_begin(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_pending_vblank_event *event = crtc->state->event;
 
@@ -236,7 +236,7 @@ static const struct drm_crtc_helper_funcs hdlcd_crtc_helper_funcs = {
 };
 
 static int hdlcd_plane_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -267,7 +267,7 @@ static int hdlcd_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void hdlcd_plane_atomic_update(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
diff --git a/drivers/gpu/drm/arm/malidp_crtc.c b/drivers/gpu/drm/arm/malidp_crtc.c
index 18e6157b10474..ebe8e10787772 100644
--- a/drivers/gpu/drm/arm/malidp_crtc.c
+++ b/drivers/gpu/drm/arm/malidp_crtc.c
@@ -47,7 +47,7 @@ static enum drm_mode_status malidp_crtc_mode_valid(struct drm_crtc *crtc,
 }
 
 static void malidp_crtc_atomic_enable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct malidp_drm *malidp = crtc_to_malidp_device(crtc);
 	struct malidp_hw_device *hwdev = malidp->dev;
@@ -71,7 +71,7 @@ static void malidp_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void malidp_crtc_atomic_disable(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 									 crtc);
@@ -334,7 +334,7 @@ static int malidp_crtc_atomic_check_scaling(struct drm_crtc *crtc,
 }
 
 static int malidp_crtc_atomic_check(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
index b765f6c9eea4e..9abe800f598ac 100644
--- a/drivers/gpu/drm/arm/malidp_drv.c
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -190,7 +190,7 @@ static int malidp_set_and_wait_config_valid(struct drm_device *drm)
 	return (ret > 0) ? 0 : -ETIMEDOUT;
 }
 
-static void malidp_atomic_commit_hw_done(struct drm_atomic_state *state)
+static void malidp_atomic_commit_hw_done(struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = state->dev;
 	struct malidp_drm *malidp = drm_to_malidp(drm);
@@ -231,7 +231,7 @@ static void malidp_atomic_commit_hw_done(struct drm_atomic_state *state)
 	drm_atomic_helper_commit_hw_done(state);
 }
 
-static void malidp_atomic_commit_tail(struct drm_atomic_state *state)
+static void malidp_atomic_commit_tail(struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = state->dev;
 	struct malidp_drm *malidp = drm_to_malidp(drm);
diff --git a/drivers/gpu/drm/arm/malidp_mw.c b/drivers/gpu/drm/arm/malidp_mw.c
index fad3438420386..f8be65119e3b0 100644
--- a/drivers/gpu/drm/arm/malidp_mw.c
+++ b/drivers/gpu/drm/arm/malidp_mw.c
@@ -237,7 +237,7 @@ int malidp_mw_connector_init(struct drm_device *drm)
 }
 
 void malidp_mw_atomic_commit(struct drm_device *drm,
-			     struct drm_atomic_state *old_state)
+			     struct drm_atomic_commit *old_state)
 {
 	struct malidp_drm *malidp = drm_to_malidp(drm);
 	struct drm_writeback_connector *mw_conn = &malidp->mw_connector;
diff --git a/drivers/gpu/drm/arm/malidp_mw.h b/drivers/gpu/drm/arm/malidp_mw.h
index 19a007676a1db..53a2d6a956a95 100644
--- a/drivers/gpu/drm/arm/malidp_mw.h
+++ b/drivers/gpu/drm/arm/malidp_mw.h
@@ -10,5 +10,5 @@
 
 int malidp_mw_connector_init(struct drm_device *drm);
 void malidp_mw_atomic_commit(struct drm_device *drm,
-			     struct drm_atomic_state *old_state);
+			     struct drm_atomic_commit *old_state);
 #endif
diff --git a/drivers/gpu/drm/arm/malidp_planes.c b/drivers/gpu/drm/arm/malidp_planes.c
index 4b98763225012..60b3f2bb153b5 100644
--- a/drivers/gpu/drm/arm/malidp_planes.c
+++ b/drivers/gpu/drm/arm/malidp_planes.c
@@ -490,7 +490,7 @@ static void malidp_de_prefetch_settings(struct malidp_plane *mp,
 }
 
 static int malidp_de_plane_check(struct drm_plane *plane,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -779,7 +779,7 @@ static void malidp_de_set_plane_afbc(struct drm_plane *plane)
 }
 
 static void malidp_de_plane_update(struct drm_plane *plane,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -900,7 +900,7 @@ static void malidp_de_plane_update(struct drm_plane *plane,
 }
 
 static void malidp_de_plane_disable(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct malidp_plane *mp = to_malidp_plane(plane);
 
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index d40bb9cf9ff1f..5e006212fea0d 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -414,7 +414,7 @@ static void armada_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static int armada_drm_crtc_atomic_check(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -430,7 +430,7 @@ static int armada_drm_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void armada_drm_crtc_atomic_begin(struct drm_crtc *crtc,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -446,7 +446,7 @@ static void armada_drm_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void armada_drm_crtc_atomic_flush(struct drm_crtc *crtc,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -474,7 +474,7 @@ static void armada_drm_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static void armada_drm_crtc_atomic_disable(struct drm_crtc *crtc,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 									 crtc);
@@ -512,7 +512,7 @@ static void armada_drm_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void armada_drm_crtc_atomic_enable(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 									 crtc);
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index 361fdcece4b0f..3da0b857ef95a 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -69,7 +69,7 @@ static inline u32 armada_csc(struct drm_plane_state *state)
 
 /* === Plane support === */
 static void armada_drm_overlay_plane_atomic_update(struct drm_plane *plane,
-	struct drm_atomic_state *state)
+	struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -220,7 +220,7 @@ static void armada_drm_overlay_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void armada_drm_overlay_plane_atomic_disable(struct drm_plane *plane,
-	struct drm_atomic_state *state)
+	struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -262,7 +262,7 @@ armada_overlay_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 	uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h,
 	struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_plane_state *plane_state;
 	int ret = 0;
 
@@ -270,7 +270,7 @@ armada_overlay_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 				 crtc_x, crtc_y, crtc_w, crtc_h,
 				 src_x, src_y, src_w, src_h);
 
-	state = drm_atomic_state_alloc(plane->dev);
+	state = drm_atomic_commit_alloc(plane->dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -297,7 +297,7 @@ armada_overlay_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
 
 	ret = drm_atomic_nonblocking_commit(state);
 fail:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/armada/armada_plane.c b/drivers/gpu/drm/armada/armada_plane.c
index eba12bba1e098..fe46a8e4508a2 100644
--- a/drivers/gpu/drm/armada/armada_plane.c
+++ b/drivers/gpu/drm/armada/armada_plane.c
@@ -80,7 +80,7 @@ void armada_drm_plane_calc(struct drm_plane_state *state, u32 addrs[2][3],
 }
 
 int armada_drm_plane_atomic_check(struct drm_plane *plane,
-	struct drm_atomic_state *state)
+	struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -130,7 +130,7 @@ int armada_drm_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void armada_drm_primary_plane_atomic_update(struct drm_plane *plane,
-	struct drm_atomic_state *state)
+	struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -220,7 +220,7 @@ static void armada_drm_primary_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void armada_drm_primary_plane_atomic_disable(struct drm_plane *plane,
-	struct drm_atomic_state *state)
+	struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/armada/armada_plane.h b/drivers/gpu/drm/armada/armada_plane.h
index 368415c609a65..69bef09ea0289 100644
--- a/drivers/gpu/drm/armada/armada_plane.h
+++ b/drivers/gpu/drm/armada/armada_plane.h
@@ -24,7 +24,7 @@ void armada_drm_plane_calc(struct drm_plane_state *state, u32 addrs[2][3],
 void armada_drm_plane_cleanup_fb(struct drm_plane *plane,
 	struct drm_plane_state *old_state);
 int armada_drm_plane_atomic_check(struct drm_plane *plane,
-	struct drm_atomic_state *state);
+	struct drm_atomic_commit *state);
 void armada_plane_reset(struct drm_plane *plane);
 struct drm_plane_state *armada_plane_duplicate_state(struct drm_plane *plane);
 void armada_plane_destroy_state(struct drm_plane *plane,
diff --git a/drivers/gpu/drm/ast/ast_2000.c b/drivers/gpu/drm/ast/ast_2000.c
index fa3bc23ce0985..4cf951b3533d3 100644
--- a/drivers/gpu/drm/ast/ast_2000.c
+++ b/drivers/gpu/drm/ast/ast_2000.c
@@ -69,64 +69,59 @@ void ast_2000_set_def_ext_reg(struct ast_device *ast)
 }
 
 static const struct ast_dramstruct ast2000_dram_table_data[] = {
-	{ 0x0108, 0x00000000 },
-	{ 0x0120, 0x00004a21 },
+	AST_DRAMSTRUCT_REG(AST_REG_MCR108, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR120, 0x00004a21),
 	AST_DRAMSTRUCT_UDELAY(67u),
-	{ 0x0000, 0xFFFFFFFF },
-	AST_DRAMSTRUCT_INIT(DRAM_TYPE, 0x00000089),
-	{ 0x0008, 0x22331353 },
-	{ 0x000C, 0x0d07000b },
-	{ 0x0010, 0x11113333 },
-	{ 0x0020, 0x00110350 },
-	{ 0x0028, 0x1e0828f0 },
-	{ 0x0024, 0x00000001 },
-	{ 0x001C, 0x00000000 },
-	{ 0x0014, 0x00000003 },
+	AST_DRAMSTRUCT_REG(AST_REG_MCR00, 0xffffffff), /* FIXME: This locks the MCR registers. */
+	AST_DRAMSTRUCT_REG(AST_REG_MCR04, 0x00000089), /* DRAM type */
+	AST_DRAMSTRUCT_REG(AST_REG_MCR08, 0x22331353),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR0C, 0x0d07000b),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR10, 0x11113333),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR20, 0x00110350),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x1e0828f0),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR24, 0x00000001),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR1C, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR14, 0x00000003),
 	AST_DRAMSTRUCT_UDELAY(67u),
-	{ 0x0018, 0x00000131 },
-	{ 0x0014, 0x00000001 },
+	AST_DRAMSTRUCT_REG(AST_REG_MCR18, 0x00000131),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR14, 0x00000001),
 	AST_DRAMSTRUCT_UDELAY(67u),
-	{ 0x0018, 0x00000031 },
-	{ 0x0014, 0x00000001 },
+	AST_DRAMSTRUCT_REG(AST_REG_MCR18, 0x00000031),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR14, 0x00000001),
 	AST_DRAMSTRUCT_UDELAY(67u),
-	{ 0x0028, 0x1e0828f1 },
-	{ 0x0024, 0x00000003 },
-	{ 0x002C, 0x1f0f28fb },
-	{ 0x0030, 0xFFFFFE01 },
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x1e0828f1),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR24, 0x00000003),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR2C, 0x1f0f28fb),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR30, 0xfffffe01),
 	AST_DRAMSTRUCT_INVALID,
 };
 
 static void ast_post_chip_2000(struct ast_device *ast)
 {
 	u8 j;
-	u32 temp, i;
-	const struct ast_dramstruct *dram_reg_info;
+	u32 i;
 
 	j = ast_get_index_reg_mask(ast, AST_IO_VGACRI, 0xd0, 0xff);
 
 	if ((j & 0x80) == 0) { /* VGA only */
-		dram_reg_info = ast2000_dram_table_data;
-		ast_write32(ast, 0xf004, 0x1e6e0000);
-		ast_write32(ast, 0xf000, 0x1);
-		ast_write32(ast, 0x10100, 0xa8);
+		const struct ast_dramstruct *dram_reg_info = ast2000_dram_table_data;
+		u32 mcr140;
 
-		do {
-			;
-		} while (ast_read32(ast, 0x10100) != 0xa8);
+		ast_moutdwm_poll(ast, AST_REG_MCR100, 0xa8, 0xa8);
 
 		while (!AST_DRAMSTRUCT_IS(dram_reg_info, INVALID)) {
 			if (AST_DRAMSTRUCT_IS(dram_reg_info, UDELAY)) {
 				for (i = 0; i < 15; i++)
 					udelay(dram_reg_info->data);
 			} else {
-				ast_write32(ast, 0x10000 + dram_reg_info->index,
-					    dram_reg_info->data);
+				ast_moutdwm(ast, dram_reg_info->index, dram_reg_info->data);
 			}
 			dram_reg_info++;
 		}
 
-		temp = ast_read32(ast, 0x10140);
-		ast_write32(ast, 0x10140, temp | 0x40);
+		mcr140 = ast_mindwm(ast, AST_REG_MCR140);
+		mcr140 |= 0x00000040;
+		ast_moutdwm(ast, AST_REG_MCR140, mcr140);
 	}
 
 	/* wait ready */
diff --git a/drivers/gpu/drm/ast/ast_2100.c b/drivers/gpu/drm/ast/ast_2100.c
index 05aeb0624d414..5cf8ad9c34463 100644
--- a/drivers/gpu/drm/ast/ast_2100.c
+++ b/drivers/gpu/drm/ast/ast_2100.c
@@ -40,21 +40,19 @@
 
 static enum ast_dram_layout ast_2100_get_dram_layout_p2a(struct ast_device *ast)
 {
-	u32 mcr_cfg;
+	u32 mcr04;
 	enum ast_dram_layout dram_layout;
 
-	ast_write32(ast, 0xf004, 0x1e6e0000);
-	ast_write32(ast, 0xf000, 0x1);
-	mcr_cfg = ast_read32(ast, 0x10004);
+	mcr04 = ast_mindwm(ast, AST_REG_MCR04);
 
-	switch (mcr_cfg & 0x0c) {
+	switch (mcr04 & GENMASK(3, 2)) {
 	case 0:
 	case 4:
 	default:
 		dram_layout = AST_DRAM_512Mx16;
 		break;
 	case 8:
-		if (mcr_cfg & 0x40)
+		if (mcr04 & 0x40)
 			dram_layout = AST_DRAM_1Gx16;
 		else
 			dram_layout = AST_DRAM_512Mx32;
@@ -72,108 +70,108 @@ static enum ast_dram_layout ast_2100_get_dram_layout_p2a(struct ast_device *ast)
  */
 
 static const struct ast_dramstruct ast1100_dram_table_data[] = {
-	{ 0x2000, 0x1688a8a8 },
-	{ 0x2020, 0x000041f0 },
+	AST_DRAMSTRUCT_REG(AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY),
+	AST_DRAMSTRUCT_REG(AST_REG_SCU020, 0x000041f0),
 	AST_DRAMSTRUCT_UDELAY(67u),
-	{ 0x0000, 0xfc600309 },
-	{ 0x006C, 0x00909090 },
-	{ 0x0064, 0x00050000 },
-	AST_DRAMSTRUCT_INIT(DRAM_TYPE, 0x00000585),
-	{ 0x0008, 0x0011030f },
-	{ 0x0010, 0x22201724 },
-	{ 0x0018, 0x1e29011a },
-	{ 0x0020, 0x00c82222 },
-	{ 0x0014, 0x01001523 },
-	{ 0x001C, 0x1024010d },
-	{ 0x0024, 0x00cb2522 },
-	{ 0x0038, 0xffffff82 },
-	{ 0x003C, 0x00000000 },
-	{ 0x0040, 0x00000000 },
-	{ 0x0044, 0x00000000 },
-	{ 0x0048, 0x00000000 },
-	{ 0x004C, 0x00000000 },
-	{ 0x0050, 0x00000000 },
-	{ 0x0054, 0x00000000 },
-	{ 0x0058, 0x00000000 },
-	{ 0x005C, 0x00000000 },
-	{ 0x0060, 0x032aa02a },
-	{ 0x0064, 0x002d3000 },
-	{ 0x0068, 0x00000000 },
-	{ 0x0070, 0x00000000 },
-	{ 0x0074, 0x00000000 },
-	{ 0x0078, 0x00000000 },
-	{ 0x007C, 0x00000000 },
-	{ 0x0034, 0x00000001 },
+	AST_DRAMSTRUCT_REG(AST_REG_MCR00, AST_REG_MCR00_PROTECTION_KEY),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR6C, 0x00909090),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR64, 0x00050000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR04, 0x00000585), // DRAM type
+	AST_DRAMSTRUCT_REG(AST_REG_MCR08, 0x0011030f),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR10, 0x22201724),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR18, 0x1e29011a),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR20, 0x00c82222),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR14, 0x01001523),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR1C, 0x1024010d),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR24, 0x00cb2522),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR38, 0xffffff82),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR3C, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR40, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR44, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR48, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR4C, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR50, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR54, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR58, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR5C, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR60, 0x032aa02a),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR64, 0x002d3000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR68, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR70, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR74, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR78, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR7C, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR34, 0x00000001),
 	AST_DRAMSTRUCT_UDELAY(67u),
-	{ 0x002C, 0x00000732 },
-	{ 0x0030, 0x00000040 },
-	{ 0x0028, 0x00000005 },
-	{ 0x0028, 0x00000007 },
-	{ 0x0028, 0x00000003 },
-	{ 0x0028, 0x00000001 },
-	{ 0x000C, 0x00005a08 },
-	{ 0x002C, 0x00000632 },
-	{ 0x0028, 0x00000001 },
-	{ 0x0030, 0x000003c0 },
-	{ 0x0028, 0x00000003 },
-	{ 0x0030, 0x00000040 },
-	{ 0x0028, 0x00000003 },
-	{ 0x000C, 0x00005a21 },
-	{ 0x0034, 0x00007c03 },
-	{ 0x0120, 0x00004c41 },
+	AST_DRAMSTRUCT_REG(AST_REG_MCR2C, 0x00000732),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR30, 0x00000040),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000005),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000007),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000003),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000001),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR0C, 0x00005a08),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR2C, 0x00000632),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000001),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR30, 0x000003c0),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000003),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR30, 0x00000040),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000003),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR0C, 0x00005a21),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR34, 0x00007c03),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR120, 0x00004c41),
 	AST_DRAMSTRUCT_INVALID,
 };
 
 static const struct ast_dramstruct ast2100_dram_table_data[] = {
-	{ 0x2000, 0x1688a8a8 },
-	{ 0x2020, 0x00004120 },
+	AST_DRAMSTRUCT_REG(AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY),
+	AST_DRAMSTRUCT_REG(AST_REG_SCU020, 0x00004120),
 	AST_DRAMSTRUCT_UDELAY(67u),
-	{ 0x0000, 0xfc600309 },
-	{ 0x006C, 0x00909090 },
-	{ 0x0064, 0x00070000 },
-	AST_DRAMSTRUCT_INIT(DRAM_TYPE, 0x00000489),
-	{ 0x0008, 0x0011030f },
-	{ 0x0010, 0x32302926 },
-	{ 0x0018, 0x274c0122 },
-	{ 0x0020, 0x00ce2222 },
-	{ 0x0014, 0x01001523 },
-	{ 0x001C, 0x1024010d },
-	{ 0x0024, 0x00cb2522 },
-	{ 0x0038, 0xffffff82 },
-	{ 0x003C, 0x00000000 },
-	{ 0x0040, 0x00000000 },
-	{ 0x0044, 0x00000000 },
-	{ 0x0048, 0x00000000 },
-	{ 0x004C, 0x00000000 },
-	{ 0x0050, 0x00000000 },
-	{ 0x0054, 0x00000000 },
-	{ 0x0058, 0x00000000 },
-	{ 0x005C, 0x00000000 },
-	{ 0x0060, 0x0f2aa02a },
-	{ 0x0064, 0x003f3005 },
-	{ 0x0068, 0x02020202 },
-	{ 0x0070, 0x00000000 },
-	{ 0x0074, 0x00000000 },
-	{ 0x0078, 0x00000000 },
-	{ 0x007C, 0x00000000 },
-	{ 0x0034, 0x00000001 },
+	AST_DRAMSTRUCT_REG(AST_REG_MCR00, AST_REG_MCR00_PROTECTION_KEY),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR6C, 0x00909090),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR64, 0x00070000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR04, 0x00000489), // DRAM type
+	AST_DRAMSTRUCT_REG(AST_REG_MCR08, 0x0011030f),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR10, 0x32302926),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR18, 0x274c0122),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR20, 0x00ce2222),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR14, 0x01001523),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR1C, 0x1024010d),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR24, 0x00cb2522),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR38, 0xffffff82),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR3C, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR40, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR44, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR48, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR4C, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR50, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR54, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR58, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR5C, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR60, 0x0f2aa02a),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR64, 0x003f3005),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR68, 0x02020202),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR70, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR74, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR78, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR7C, 0x00000000),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR34, 0x00000001),
 	AST_DRAMSTRUCT_UDELAY(67u),
-	{ 0x002C, 0x00000942 },
-	{ 0x0030, 0x00000040 },
-	{ 0x0028, 0x00000005 },
-	{ 0x0028, 0x00000007 },
-	{ 0x0028, 0x00000003 },
-	{ 0x0028, 0x00000001 },
-	{ 0x000C, 0x00005a08 },
-	{ 0x002C, 0x00000842 },
-	{ 0x0028, 0x00000001 },
-	{ 0x0030, 0x000003c0 },
-	{ 0x0028, 0x00000003 },
-	{ 0x0030, 0x00000040 },
-	{ 0x0028, 0x00000003 },
-	{ 0x000C, 0x00005a21 },
-	{ 0x0034, 0x00007c03 },
-	{ 0x0120, 0x00005061 },
+	AST_DRAMSTRUCT_REG(AST_REG_MCR2C, 0x00000942),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR30, 0x00000040),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000005),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000007),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000003),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000001),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR0C, 0x00005a08),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR2C, 0x00000842),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000001),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR30, 0x000003c0),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000003),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR30, 0x00000040),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR28, 0x00000003),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR0C, 0x00005a21),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR34, 0x00007c03),
+	AST_DRAMSTRUCT_REG(AST_REG_MCR120, 0x00005061),
 	AST_DRAMSTRUCT_INVALID,
 };
 
@@ -209,28 +207,28 @@ static u32 mmctestburst2_ast2150(struct ast_device *ast, u32 datagen)
 {
 	u32 data, timeout;
 
-	ast_moutdwm(ast, 0x1e6e0070, 0x00000000);
-	ast_moutdwm(ast, 0x1e6e0070, 0x00000001 | (datagen << 3));
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000001 | (datagen << 3));
 	timeout = 0;
 	do {
-		data = ast_mindwm(ast, 0x1e6e0070) & 0x40;
+		data = ast_mindwm(ast, AST_REG_MCR70) & 0x40;
 		if (++timeout > TIMEOUT_AST2150) {
-			ast_moutdwm(ast, 0x1e6e0070, 0x00000000);
+			ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
 			return 0xffffffff;
 		}
 	} while (!data);
-	ast_moutdwm(ast, 0x1e6e0070, 0x00000000);
-	ast_moutdwm(ast, 0x1e6e0070, 0x00000003 | (datagen << 3));
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000003 | (datagen << 3));
 	timeout = 0;
 	do {
-		data = ast_mindwm(ast, 0x1e6e0070) & 0x40;
+		data = ast_mindwm(ast, AST_REG_MCR70) & 0x40;
 		if (++timeout > TIMEOUT_AST2150) {
-			ast_moutdwm(ast, 0x1e6e0070, 0x00000000);
+			ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
 			return 0xffffffff;
 		}
 	} while (!data);
-	data = (ast_mindwm(ast, 0x1e6e0070) & 0x80) >> 7;
-	ast_moutdwm(ast, 0x1e6e0070, 0x00000000);
+	data = (ast_mindwm(ast, AST_REG_MCR70) & 0x80) >> 7;
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
 	return data;
 }
 
@@ -249,7 +247,7 @@ static int cbrscan_ast2150(struct ast_device *ast, int busw)
 	u32 patcnt, loop;
 
 	for (patcnt = 0; patcnt < CBR_PATNUM_AST2150; patcnt++) {
-		ast_moutdwm(ast, 0x1e6e007c, pattern_AST2150[patcnt]);
+		ast_moutdwm(ast, AST_REG_MCR7C, pattern_AST2150[patcnt]);
 		for (loop = 0; loop < CBR_PASSNUM_AST2150; loop++) {
 			if (cbrtest_ast2150(ast))
 				break;
@@ -276,7 +274,7 @@ static void cbrdlli_ast2150(struct ast_device *ast, int busw)
 	passcnt = 0;
 
 	for (dlli = 0; dlli < 100; dlli++) {
-		ast_moutdwm(ast, 0x1e6e0068, dlli | (dlli << 8) | (dlli << 16) | (dlli << 24));
+		ast_moutdwm(ast, AST_REG_MCR68, dlli | (dlli << 8) | (dlli << 16) | (dlli << 24));
 		data = cbrscan_ast2150(ast, busw);
 		if (data != 0) {
 			if (data & 0x1) {
@@ -294,78 +292,82 @@ static void cbrdlli_ast2150(struct ast_device *ast, int busw)
 		goto cbr_start;
 
 	dlli = dll_min[0] + (((dll_max[0] - dll_min[0]) * 7) >> 4);
-	ast_moutdwm(ast, 0x1e6e0068, dlli | (dlli << 8) | (dlli << 16) | (dlli << 24));
+	ast_moutdwm(ast, AST_REG_MCR68, dlli | (dlli << 8) | (dlli << 16) | (dlli << 24));
 }
 
 static void ast_post_chip_2100(struct ast_device *ast)
 {
 	u8 j;
-	u32 data, temp, i;
-	const struct ast_dramstruct *dram_reg_info;
-	enum ast_dram_layout dram_layout  = ast_2100_get_dram_layout_p2a(ast);
+	u32 i;
+	enum ast_dram_layout dram_layout = ast_2100_get_dram_layout_p2a(ast);
+	u32 mcr120;
+	u32 scu00c, scu040;
 
 	j = ast_get_index_reg_mask(ast, AST_IO_VGACRI, 0xd0, 0xff);
 
 	if ((j & 0x80) == 0) { /* VGA only */
+		const struct ast_dramstruct *dram_reg_info;
+
 		if (ast->chip == AST2100 || ast->chip == AST2200)
 			dram_reg_info = ast2100_dram_table_data;
 		else
 			dram_reg_info = ast1100_dram_table_data;
 
-		ast_write32(ast, 0xf004, 0x1e6e0000);
-		ast_write32(ast, 0xf000, 0x1);
-		ast_write32(ast, 0x12000, 0x1688A8A8);
-		do {
-			;
-		} while (ast_read32(ast, 0x12000) != 0x01);
-
-		ast_write32(ast, 0x10000, 0xfc600309);
-		do {
-			;
-		} while (ast_read32(ast, 0x10000) != 0x01);
+		ast_moutdwm_poll(ast, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY, 0x01);
+		ast_moutdwm_poll(ast, AST_REG_MCR00, AST_REG_MCR00_PROTECTION_KEY, 0x01);
 
 		while (!AST_DRAMSTRUCT_IS(dram_reg_info, INVALID)) {
 			if (AST_DRAMSTRUCT_IS(dram_reg_info, UDELAY)) {
 				for (i = 0; i < 15; i++)
 					udelay(dram_reg_info->data);
-			} else if (AST_DRAMSTRUCT_IS(dram_reg_info, DRAM_TYPE)) {
+			} else if (AST_DRAMSTRUCT_IS_REG(dram_reg_info, AST_REG_MCR04)) {
+				u32 mcr04;
+				u32 scu070;
+
 				switch (dram_layout) {
 				case AST_DRAM_1Gx16:
-					data = 0x00000d89;
+					mcr04 = 0x00000d89;
 					break;
 				case AST_DRAM_1Gx32:
-					data = 0x00000c8d;
+					mcr04 = 0x00000c8d;
 					break;
 				default:
-					data = dram_reg_info->data;
+					mcr04 = dram_reg_info->data;
 					break;
 				}
 
-				temp = ast_read32(ast, 0x12070);
-				temp &= 0xc;
-				temp <<= 2;
-				ast_write32(ast, 0x10000 + dram_reg_info->index, data | temp);
+				/*
+				 * FIXME: There might be bits already in MCR04[5:4]. Should
+				 *        we only do this in the default case?
+				 */
+				scu070 = ast_mindwm(ast, AST_REG_SCU070);
+				mcr04 |= (scu070 & GENMASK(3, 2)) << 2;
+
+				ast_moutdwm(ast, dram_reg_info->index, mcr04);
 			} else {
-				ast_write32(ast, 0x10000 + dram_reg_info->index,
-					    dram_reg_info->data);
+				ast_moutdwm(ast, dram_reg_info->index, dram_reg_info->data);
 			}
 			dram_reg_info++;
 		}
 
 		/* AST 2100/2150 DRAM calibration */
-		data = ast_read32(ast, 0x10120);
-		if (data == 0x5061) { /* 266Mhz */
-			data = ast_read32(ast, 0x10004);
-			if (data & 0x40)
+		mcr120 = ast_mindwm(ast, AST_REG_MCR120);
+		if (mcr120 == 0x5061) { /* 266Mhz */
+			u32 mcr04 = ast_mindwm(ast, AST_REG_MCR04);
+
+			if (mcr04 & 0x40)
 				cbrdlli_ast2150(ast, 16); /* 16 bits */
 			else
 				cbrdlli_ast2150(ast, 32); /* 32 bits */
 		}
 
-		temp = ast_read32(ast, 0x1200c);
-		ast_write32(ast, 0x1200c, temp & 0xfffffffd);
-		temp = ast_read32(ast, 0x12040);
-		ast_write32(ast, 0x12040, temp | 0x40);
+		scu00c = ast_mindwm(ast, AST_REG_SCU00C);
+		scu00c &= 0xfffffffd;
+		ast_moutdwm(ast, AST_REG_SCU00C, scu00c);
+
+		scu040 = ast_mindwm(ast, AST_REG_SCU040);
+		scu040 |= 0x00000040;
+		ast_moutdwm(ast, AST_REG_SCU040, scu040);
 	}
 
 	/* wait ready */
diff --git a/drivers/gpu/drm/ast/ast_2300.c b/drivers/gpu/drm/ast/ast_2300.c
index 5f50d9f91ffd6..cbeac6990c272 100644
--- a/drivers/gpu/drm/ast/ast_2300.c
+++ b/drivers/gpu/drm/ast/ast_2300.c
@@ -129,19 +129,19 @@ static u32 mmc_test2(struct ast_device *ast, u32 datagen, u8 test_ctl)
 {
 	u32 data, timeout;
 
-	ast_moutdwm(ast, 0x1e6e0070, 0x00000000);
-	ast_moutdwm(ast, 0x1e6e0070, (datagen << 3) | test_ctl);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR70, (datagen << 3) | test_ctl);
 	timeout = 0;
 	do {
-		data = ast_mindwm(ast, 0x1e6e0070) & 0x1000;
+		data = ast_mindwm(ast, AST_REG_MCR70) & 0x1000;
 		if (++timeout > TIMEOUT) {
-			ast_moutdwm(ast, 0x1e6e0070, 0x0);
+			ast_moutdwm(ast, AST_REG_MCR70, 0x0);
 			return 0xffffffff;
 		}
 	} while (!data);
-	data = ast_mindwm(ast, 0x1e6e0078);
+	data = ast_mindwm(ast, AST_REG_MCR78);
 	data = (data | (data >> 16)) & 0xffff;
-	ast_moutdwm(ast, 0x1e6e0070, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
 	return data;
 }
 
@@ -186,7 +186,7 @@ static int cbr_scan(struct ast_device *ast)
 
 	data2 = 3;
 	for (patcnt = 0; patcnt < CBR_PATNUM; patcnt++) {
-		ast_moutdwm(ast, 0x1e6e007c, pattern[patcnt]);
+		ast_moutdwm(ast, AST_REG_MCR7C, pattern[patcnt]);
 		for (loop = 0; loop < CBR_PASSNUM2; loop++) {
 			data = cbr_test(ast);
 			if (data != 0) {
@@ -222,7 +222,7 @@ static u32 cbr_scan2(struct ast_device *ast)
 
 	data2 = 0xffff;
 	for (patcnt = 0; patcnt < CBR_PATNUM; patcnt++) {
-		ast_moutdwm(ast, 0x1e6e007c, pattern[patcnt]);
+		ast_moutdwm(ast, AST_REG_MCR7C, pattern[patcnt]);
 		for (loop = 0; loop < CBR_PASSNUM2; loop++) {
 			data = cbr_test2(ast);
 			if (data != 0) {
@@ -252,7 +252,7 @@ static bool cbr_scan3(struct ast_device *ast)
 	u32 patcnt, loop;
 
 	for (patcnt = 0; patcnt < CBR_PATNUM; patcnt++) {
-		ast_moutdwm(ast, 0x1e6e007c, pattern[patcnt]);
+		ast_moutdwm(ast, AST_REG_MCR7C, pattern[patcnt]);
 		for (loop = 0; loop < 2; loop++) {
 			if (cbr_test3(ast))
 				break;
@@ -274,8 +274,8 @@ static bool finetuneDQI_L(struct ast_device *ast, struct ast2300_dram_param *par
 	}
 	passcnt = 0;
 	for (dlli = 0; dlli < 76; dlli++) {
-		ast_moutdwm(ast, 0x1E6E0068, 0x00001400 | (dlli << 16) | (dlli << 24));
-		ast_moutdwm(ast, 0x1E6E0074, CBR_SIZE1);
+		ast_moutdwm(ast, AST_REG_MCR68, 0x00001400 | (dlli << 16) | (dlli << 24));
+		ast_moutdwm(ast, AST_REG_MCR74, CBR_SIZE1);
 		data = cbr_scan2(ast);
 		if (data != 0) {
 			mask = 0x00010001;
@@ -330,7 +330,7 @@ static bool finetuneDQI_L(struct ast_device *ast, struct ast2300_dram_param *par
 			data |= dlli << 21;
 		}
 	}
-	ast_moutdwm(ast, 0x1E6E0080, data);
+	ast_moutdwm(ast, AST_REG_MCR80, data);
 
 	data = 0;
 	for (cnt = 8; cnt < 16; cnt++) {
@@ -354,7 +354,7 @@ static bool finetuneDQI_L(struct ast_device *ast, struct ast2300_dram_param *par
 			data |= dlli << 21;
 		}
 	}
-	ast_moutdwm(ast, 0x1E6E0084, data);
+	ast_moutdwm(ast, AST_REG_MCR84, data);
 	return status;
 } /* finetuneDQI_L */
 
@@ -367,10 +367,10 @@ static void finetuneDQSI(struct ast_device *ast)
 	char tag[2][76];
 
 	/* Disable DQI CBR */
-	reg_mcr0c  = ast_mindwm(ast, 0x1E6E000C);
-	reg_mcr18  = ast_mindwm(ast, 0x1E6E0018);
+	reg_mcr0c  = ast_mindwm(ast, AST_REG_MCR0C);
+	reg_mcr18  = ast_mindwm(ast, AST_REG_MCR18);
 	reg_mcr18 &= 0x0000ffff;
-	ast_moutdwm(ast, 0x1E6E0018, reg_mcr18);
+	ast_moutdwm(ast, AST_REG_MCR18, reg_mcr18);
 
 	for (dlli = 0; dlli < 76; dlli++) {
 		tag[0][dlli] = 0x0;
@@ -386,14 +386,14 @@ static void finetuneDQSI(struct ast_device *ast)
 		passcnt[0] = 0;
 		passcnt[1] = 0;
 		for (dqsip = 0; dqsip < 2; dqsip++) {
-			ast_moutdwm(ast, 0x1E6E000C, 0);
-			ast_moutdwm(ast, 0x1E6E0018, reg_mcr18 | (dqidly << 16) | (dqsip << 23));
-			ast_moutdwm(ast, 0x1E6E000C, reg_mcr0c);
+			ast_moutdwm(ast, AST_REG_MCR0C, 0);
+			ast_moutdwm(ast, AST_REG_MCR18, reg_mcr18 | (dqidly << 16) | (dqsip << 23));
+			ast_moutdwm(ast, AST_REG_MCR0C, reg_mcr0c);
 			for (dlli = 0; dlli < 76; dlli++) {
-				ast_moutdwm(ast, 0x1E6E0068,
+				ast_moutdwm(ast, AST_REG_MCR68,
 					    0x00001300 | (dlli << 16) | (dlli << 24));
-				ast_moutdwm(ast, 0x1E6E0070, 0);
-				ast_moutdwm(ast, 0x1E6E0074, CBR_SIZE0);
+				ast_moutdwm(ast, AST_REG_MCR70, 0);
+				ast_moutdwm(ast, AST_REG_MCR74, CBR_SIZE0);
 				if (cbr_scan3(ast)) {
 					if (dlli == 0)
 						break;
@@ -457,7 +457,7 @@ static void finetuneDQSI(struct ast_device *ast)
 		}
 	}
 	reg_mcr18 = reg_mcr18 | (g_dqidly << 16) | (g_dqsip << 23);
-	ast_moutdwm(ast, 0x1E6E0018, reg_mcr18);
+	ast_moutdwm(ast, AST_REG_MCR18, reg_mcr18);
 }
 
 static bool cbr_dll2(struct ast_device *ast, struct ast2300_dram_param *param)
@@ -476,8 +476,8 @@ static bool cbr_dll2(struct ast_device *ast, struct ast2300_dram_param *param)
 	dllmax[1] = 0x0;
 	passcnt = 0;
 	for (dlli = 0; dlli < 76; dlli++) {
-		ast_moutdwm(ast, 0x1E6E0068, 0x00001300 | (dlli << 16) | (dlli << 24));
-		ast_moutdwm(ast, 0x1E6E0074, CBR_SIZE2);
+		ast_moutdwm(ast, AST_REG_MCR68, 0x00001300 | (dlli << 16) | (dlli << 24));
+		ast_moutdwm(ast, AST_REG_MCR74, CBR_SIZE2);
 		data = cbr_scan(ast);
 		if (data != 0) {
 			if (data & 0x1) {
@@ -508,7 +508,7 @@ static bool cbr_dll2(struct ast_device *ast, struct ast2300_dram_param *param)
 	dlli  = (dllmin[1] + dllmax[1]) >> 1;
 	dlli <<= 8;
 	dlli += (dllmin[0] + dllmax[0]) >> 1;
-	ast_moutdwm(ast, 0x1E6E0068, ast_mindwm(ast, 0x1E720058) | (dlli << 16));
+	ast_moutdwm(ast, AST_REG_MCR68, ast_mindwm(ast, AST_REG_A2P58) | (dlli << 16));
 	return status;
 } /* CBRDLL2 */
 
@@ -516,10 +516,10 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 {
 	u32 trap, trap_AC2, trap_MRS;
 
-	ast_moutdwm(ast, 0x1E6E2000, 0x1688A8A8);
+	ast_moutdwm(ast, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY);
 
 	/* Ger trap info */
-	trap = (ast_mindwm(ast, 0x1E6E2070) >> 25) & 0x3;
+	trap = (ast_mindwm(ast, AST_REG_SCU070) >> 25) & 0x3;
 	trap_AC2  = 0x00020000 + (trap << 16);
 	trap_AC2 |= 0x00300000 + ((trap & 0x2) << 19);
 	trap_MRS  = 0x00000010 + (trap << 4);
@@ -533,7 +533,7 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 
 	switch (param->dram_freq) {
 	case 336:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0190);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0190);
 		param->wodt          = 0;
 		param->reg_AC1       = 0x22202725;
 		param->reg_AC2       = 0xAA007613 | trap_AC2;
@@ -561,7 +561,7 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 		break;
 	default:
 	case 396:
-		ast_moutdwm(ast, 0x1E6E2020, 0x03F1);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x03F1);
 		param->wodt          = 1;
 		param->reg_AC1       = 0x33302825;
 		param->reg_AC2       = 0xCC009617 | trap_AC2;
@@ -591,7 +591,7 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 		break;
 
 	case 408:
-		ast_moutdwm(ast, 0x1E6E2020, 0x01F0);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x01F0);
 		param->wodt          = 1;
 		param->reg_AC1       = 0x33302825;
 		param->reg_AC2       = 0xCC009617 | trap_AC2;
@@ -621,7 +621,7 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 
 		break;
 	case 456:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0230);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0230);
 		param->wodt          = 0;
 		param->reg_AC1       = 0x33302926;
 		param->reg_AC2       = 0xCD44961A;
@@ -635,7 +635,7 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 4;
 		break;
 	case 504:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0270);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0270);
 		param->wodt          = 1;
 		param->reg_AC1       = 0x33302926;
 		param->reg_AC2       = 0xDE44A61D;
@@ -649,7 +649,7 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 4;
 		break;
 	case 528:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0290);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0290);
 		param->wodt          = 1;
 		param->rodt          = 1;
 		param->reg_AC1       = 0x33302926;
@@ -665,7 +665,7 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 3;
 		break;
 	case 576:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0140);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0140);
 		param->reg_MADJ      = 0x00136868;
 		param->reg_SADJ      = 0x00004534;
 		param->wodt          = 1;
@@ -683,7 +683,7 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 3;
 		break;
 	case 600:
-		ast_moutdwm(ast, 0x1E6E2020, 0x02E1);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x02E1);
 		param->reg_MADJ      = 0x00136868;
 		param->reg_SADJ      = 0x00004534;
 		param->wodt          = 1;
@@ -701,7 +701,7 @@ static void get_ddr3_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 3;
 		break;
 	case 624:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0160);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0160);
 		param->reg_MADJ      = 0x00136868;
 		param->reg_SADJ      = 0x00004534;
 		param->wodt          = 1;
@@ -758,115 +758,115 @@ static void ddr3_init(struct ast_device *ast, struct ast2300_dram_param *param)
 	u32 data, data2, retry = 0;
 
 ddr3_init_start:
-	ast_moutdwm(ast, 0x1E6E0000, 0xFC600309);
-	ast_moutdwm(ast, 0x1E6E0018, 0x00000100);
-	ast_moutdwm(ast, 0x1E6E0024, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0034, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR00, AST_REG_MCR00_PROTECTION_KEY);
+	ast_moutdwm(ast, AST_REG_MCR18, 0x00000100);
+	ast_moutdwm(ast, AST_REG_MCR24, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x00000000);
 	udelay(10);
-	ast_moutdwm(ast, 0x1E6E0064, param->reg_MADJ);
-	ast_moutdwm(ast, 0x1E6E0068, param->reg_SADJ);
+	ast_moutdwm(ast, AST_REG_MCR64, param->reg_MADJ);
+	ast_moutdwm(ast, AST_REG_MCR68, param->reg_SADJ);
 	udelay(10);
-	ast_moutdwm(ast, 0x1E6E0064, param->reg_MADJ | 0xC0000);
+	ast_moutdwm(ast, AST_REG_MCR64, param->reg_MADJ | 0xC0000);
 	udelay(10);
 
-	ast_moutdwm(ast, 0x1E6E0004, param->dram_config);
-	ast_moutdwm(ast, 0x1E6E0008, 0x90040f);
-	ast_moutdwm(ast, 0x1E6E0010, param->reg_AC1);
-	ast_moutdwm(ast, 0x1E6E0014, param->reg_AC2);
-	ast_moutdwm(ast, 0x1E6E0020, param->reg_DQSIC);
-	ast_moutdwm(ast, 0x1E6E0080, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0084, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0088, param->reg_DQIDLY);
-	ast_moutdwm(ast, 0x1E6E0018, 0x4000A170);
-	ast_moutdwm(ast, 0x1E6E0018, 0x00002370);
-	ast_moutdwm(ast, 0x1E6E0038, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0040, 0xFF444444);
-	ast_moutdwm(ast, 0x1E6E0044, 0x22222222);
-	ast_moutdwm(ast, 0x1E6E0048, 0x22222222);
-	ast_moutdwm(ast, 0x1E6E004C, 0x00000002);
-	ast_moutdwm(ast, 0x1E6E0050, 0x80000000);
-	ast_moutdwm(ast, 0x1E6E0050, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0054, 0);
-	ast_moutdwm(ast, 0x1E6E0060, param->reg_DRV);
-	ast_moutdwm(ast, 0x1E6E006C, param->reg_IOZ);
-	ast_moutdwm(ast, 0x1E6E0070, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0074, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0078, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E007C, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR04, param->dram_config);
+	ast_moutdwm(ast, AST_REG_MCR08, 0x90040f);
+	ast_moutdwm(ast, AST_REG_MCR10, param->reg_AC1);
+	ast_moutdwm(ast, AST_REG_MCR14, param->reg_AC2);
+	ast_moutdwm(ast, AST_REG_MCR20, param->reg_DQSIC);
+	ast_moutdwm(ast, AST_REG_MCR80, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR84, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR88, param->reg_DQIDLY);
+	ast_moutdwm(ast, AST_REG_MCR18, 0x4000A170);
+	ast_moutdwm(ast, AST_REG_MCR18, 0x00002370);
+	ast_moutdwm(ast, AST_REG_MCR38, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR40, 0xFF444444);
+	ast_moutdwm(ast, AST_REG_MCR44, 0x22222222);
+	ast_moutdwm(ast, AST_REG_MCR48, 0x22222222);
+	ast_moutdwm(ast, AST_REG_MCR4C, 0x00000002);
+	ast_moutdwm(ast, AST_REG_MCR50, 0x80000000);
+	ast_moutdwm(ast, AST_REG_MCR50, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR54, 0);
+	ast_moutdwm(ast, AST_REG_MCR60, param->reg_DRV);
+	ast_moutdwm(ast, AST_REG_MCR6C, param->reg_IOZ);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR74, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR78, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR7C, 0x00000000);
 	/* Wait MCLK2X lock to MCLK */
 	do {
-		data = ast_mindwm(ast, 0x1E6E001C);
+		data = ast_mindwm(ast, AST_REG_MCR1C);
 	} while (!(data & 0x08000000));
-	data = ast_mindwm(ast, 0x1E6E001C);
+	data = ast_mindwm(ast, AST_REG_MCR1C);
 	data = (data >> 8) & 0xff;
 	while ((data & 0x08) || ((data & 0x7) < 2) || (data < 4)) {
-		data2 = (ast_mindwm(ast, 0x1E6E0064) & 0xfff3ffff) + 4;
+		data2 = (ast_mindwm(ast, AST_REG_MCR64) & 0xfff3ffff) + 4;
 		if ((data2 & 0xff) > param->madj_max)
 			break;
-		ast_moutdwm(ast, 0x1E6E0064, data2);
+		ast_moutdwm(ast, AST_REG_MCR64, data2);
 		if (data2 & 0x00100000)
 			data2 = ((data2 & 0xff) >> 3) + 3;
 		else
 			data2 = ((data2 & 0xff) >> 2) + 5;
-		data = ast_mindwm(ast, 0x1E6E0068) & 0xffff00ff;
+		data = ast_mindwm(ast, AST_REG_MCR68) & 0xffff00ff;
 		data2 += data & 0xff;
 		data = data | (data2 << 8);
-		ast_moutdwm(ast, 0x1E6E0068, data);
+		ast_moutdwm(ast, AST_REG_MCR68, data);
 		udelay(10);
-		ast_moutdwm(ast, 0x1E6E0064, ast_mindwm(ast, 0x1E6E0064) | 0xC0000);
+		ast_moutdwm(ast, AST_REG_MCR64, ast_mindwm(ast, AST_REG_MCR64) | 0xC0000);
 		udelay(10);
-		data = ast_mindwm(ast, 0x1E6E0018) & 0xfffff1ff;
-		ast_moutdwm(ast, 0x1E6E0018, data);
+		data = ast_mindwm(ast, AST_REG_MCR18) & 0xfffff1ff;
+		ast_moutdwm(ast, AST_REG_MCR18, data);
 		data = data | 0x200;
-		ast_moutdwm(ast, 0x1E6E0018, data);
+		ast_moutdwm(ast, AST_REG_MCR18, data);
 		do {
-			data = ast_mindwm(ast, 0x1E6E001C);
+			data = ast_mindwm(ast, AST_REG_MCR1C);
 		} while (!(data & 0x08000000));
 
-		data = ast_mindwm(ast, 0x1E6E001C);
+		data = ast_mindwm(ast, AST_REG_MCR1C);
 		data = (data >> 8) & 0xff;
 	}
-	ast_moutdwm(ast, 0x1E720058, ast_mindwm(ast, 0x1E6E0068) & 0xffff);
-	data = ast_mindwm(ast, 0x1E6E0018) | 0xC00;
-	ast_moutdwm(ast, 0x1E6E0018, data);
+	ast_moutdwm(ast, AST_REG_A2P58, ast_mindwm(ast, AST_REG_MCR68) & 0xffff);
+	data = ast_mindwm(ast, AST_REG_MCR18) | 0xC00;
+	ast_moutdwm(ast, AST_REG_MCR18, data);
 
-	ast_moutdwm(ast, 0x1E6E0034, 0x00000001);
-	ast_moutdwm(ast, 0x1E6E000C, 0x00000040);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x00000001);
+	ast_moutdwm(ast, AST_REG_MCR0C, 0x00000040);
 	udelay(50);
 	/* Mode Register Setting */
-	ast_moutdwm(ast, 0x1E6E002C, param->reg_MRS | 0x100);
-	ast_moutdwm(ast, 0x1E6E0030, param->reg_EMRS);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000005);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000007);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000003);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000001);
-	ast_moutdwm(ast, 0x1E6E002C, param->reg_MRS);
-	ast_moutdwm(ast, 0x1E6E000C, 0x00005C08);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000001);
-
-	ast_moutdwm(ast, 0x1E6E000C, 0x00005C01);
+	ast_moutdwm(ast, AST_REG_MCR2C, param->reg_MRS | 0x100);
+	ast_moutdwm(ast, AST_REG_MCR30, param->reg_EMRS);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000005);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000007);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000003);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000001);
+	ast_moutdwm(ast, AST_REG_MCR2C, param->reg_MRS);
+	ast_moutdwm(ast, AST_REG_MCR0C, 0x00005C08);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000001);
+
+	ast_moutdwm(ast, AST_REG_MCR0C, 0x00005C01);
 	data = 0;
 	if (param->wodt)
 		data = 0x300;
 	if (param->rodt)
 		data = data | 0x3000 | ((param->reg_AC2 & 0x60000) >> 3);
-	ast_moutdwm(ast, 0x1E6E0034, data | 0x3);
+	ast_moutdwm(ast, AST_REG_MCR34, data | 0x3);
 
 	/* Calibrate the DQSI delay */
 	if ((cbr_dll2(ast, param) == false) && (retry++ < 10))
 		goto ddr3_init_start;
 
-	ast_moutdwm(ast, 0x1E6E0120, param->reg_FREQ);
+	ast_moutdwm(ast, AST_REG_MCR120, param->reg_FREQ);
 	/* ECC Memory Initialization */
 #ifdef ECC
-	ast_moutdwm(ast, 0x1E6E007C, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0070, 0x221);
+	ast_moutdwm(ast, AST_REG_MCR7C, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x221);
 	do {
-		data = ast_mindwm(ast, 0x1E6E0070);
+		data = ast_mindwm(ast, AST_REG_MCR70);
 	} while (!(data & 0x00001000));
-	ast_moutdwm(ast, 0x1E6E0070, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0050, 0x80000000);
-	ast_moutdwm(ast, 0x1E6E0050, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR50, 0x80000000);
+	ast_moutdwm(ast, AST_REG_MCR50, 0x00000000);
 #endif
 }
 
@@ -874,10 +874,10 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 {
 	u32 trap, trap_AC2, trap_MRS;
 
-	ast_moutdwm(ast, 0x1E6E2000, 0x1688A8A8);
+	ast_moutdwm(ast, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY);
 
 	/* Ger trap info */
-	trap = (ast_mindwm(ast, 0x1E6E2070) >> 25) & 0x3;
+	trap = (ast_mindwm(ast, AST_REG_SCU070) >> 25) & 0x3;
 	trap_AC2  = (trap << 20) | (trap << 16);
 	trap_AC2 += 0x00110000;
 	trap_MRS  = 0x00000040 | (trap << 4);
@@ -890,7 +890,7 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 
 	switch (param->dram_freq) {
 	case 264:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0130);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0130);
 		param->wodt          = 0;
 		param->reg_AC1       = 0x11101513;
 		param->reg_AC2       = 0x78117011;
@@ -905,7 +905,7 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 3;
 		break;
 	case 336:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0190);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0190);
 		param->wodt          = 1;
 		param->reg_AC1       = 0x22202613;
 		param->reg_AC2       = 0xAA009016 | trap_AC2;
@@ -936,7 +936,7 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 		break;
 	default:
 	case 396:
-		ast_moutdwm(ast, 0x1E6E2020, 0x03F1);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x03F1);
 		param->wodt          = 1;
 		param->rodt          = 0;
 		param->reg_AC1       = 0x33302714;
@@ -970,7 +970,7 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 		break;
 
 	case 408:
-		ast_moutdwm(ast, 0x1E6E2020, 0x01F0);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x01F0);
 		param->wodt          = 1;
 		param->rodt          = 0;
 		param->reg_AC1       = 0x33302714;
@@ -1003,7 +1003,7 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 
 		break;
 	case 456:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0230);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0230);
 		param->wodt          = 0;
 		param->reg_AC1       = 0x33302815;
 		param->reg_AC2       = 0xCD44B01E;
@@ -1018,7 +1018,7 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 3;
 		break;
 	case 504:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0261);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0261);
 		param->wodt          = 1;
 		param->rodt          = 1;
 		param->reg_AC1       = 0x33302815;
@@ -1034,7 +1034,7 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 3;
 		break;
 	case 528:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0120);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0120);
 		param->wodt          = 1;
 		param->rodt          = 1;
 		param->reg_AC1       = 0x33302815;
@@ -1050,7 +1050,7 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 3;
 		break;
 	case 552:
-		ast_moutdwm(ast, 0x1E6E2020, 0x02A1);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x02A1);
 		param->wodt          = 1;
 		param->rodt          = 1;
 		param->reg_AC1       = 0x43402915;
@@ -1066,7 +1066,7 @@ static void get_ddr2_info(struct ast_device *ast, struct ast2300_dram_param *par
 		param->dll2_finetune_step = 3;
 		break;
 	case 576:
-		ast_moutdwm(ast, 0x1E6E2020, 0x0140);
+		ast_moutdwm(ast, AST_REG_SCU020, 0x0140);
 		param->wodt          = 1;
 		param->rodt          = 1;
 		param->reg_AC1       = 0x43402915;
@@ -1121,104 +1121,104 @@ static void ddr2_init(struct ast_device *ast, struct ast2300_dram_param *param)
 	u32 data, data2, retry = 0;
 
 ddr2_init_start:
-	ast_moutdwm(ast, 0x1E6E0000, 0xFC600309);
-	ast_moutdwm(ast, 0x1E6E0018, 0x00000100);
-	ast_moutdwm(ast, 0x1E6E0024, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0064, param->reg_MADJ);
-	ast_moutdwm(ast, 0x1E6E0068, param->reg_SADJ);
+	ast_moutdwm(ast, AST_REG_MCR00, AST_REG_MCR00_PROTECTION_KEY);
+	ast_moutdwm(ast, AST_REG_MCR18, 0x00000100);
+	ast_moutdwm(ast, AST_REG_MCR24, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR64, param->reg_MADJ);
+	ast_moutdwm(ast, AST_REG_MCR68, param->reg_SADJ);
 	udelay(10);
-	ast_moutdwm(ast, 0x1E6E0064, param->reg_MADJ | 0xC0000);
+	ast_moutdwm(ast, AST_REG_MCR64, param->reg_MADJ | 0xC0000);
 	udelay(10);
 
-	ast_moutdwm(ast, 0x1E6E0004, param->dram_config);
-	ast_moutdwm(ast, 0x1E6E0008, 0x90040f);
-	ast_moutdwm(ast, 0x1E6E0010, param->reg_AC1);
-	ast_moutdwm(ast, 0x1E6E0014, param->reg_AC2);
-	ast_moutdwm(ast, 0x1E6E0020, param->reg_DQSIC);
-	ast_moutdwm(ast, 0x1E6E0080, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0084, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0088, param->reg_DQIDLY);
-	ast_moutdwm(ast, 0x1E6E0018, 0x4000A130);
-	ast_moutdwm(ast, 0x1E6E0018, 0x00002330);
-	ast_moutdwm(ast, 0x1E6E0038, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0040, 0xFF808000);
-	ast_moutdwm(ast, 0x1E6E0044, 0x88848466);
-	ast_moutdwm(ast, 0x1E6E0048, 0x44440008);
-	ast_moutdwm(ast, 0x1E6E004C, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0050, 0x80000000);
-	ast_moutdwm(ast, 0x1E6E0050, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0054, 0);
-	ast_moutdwm(ast, 0x1E6E0060, param->reg_DRV);
-	ast_moutdwm(ast, 0x1E6E006C, param->reg_IOZ);
-	ast_moutdwm(ast, 0x1E6E0070, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0074, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0078, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E007C, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR04, param->dram_config);
+	ast_moutdwm(ast, AST_REG_MCR08, 0x90040f);
+	ast_moutdwm(ast, AST_REG_MCR10, param->reg_AC1);
+	ast_moutdwm(ast, AST_REG_MCR14, param->reg_AC2);
+	ast_moutdwm(ast, AST_REG_MCR20, param->reg_DQSIC);
+	ast_moutdwm(ast, AST_REG_MCR80, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR84, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR88, param->reg_DQIDLY);
+	ast_moutdwm(ast, AST_REG_MCR18, 0x4000A130);
+	ast_moutdwm(ast, AST_REG_MCR18, 0x00002330);
+	ast_moutdwm(ast, AST_REG_MCR38, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR40, 0xFF808000);
+	ast_moutdwm(ast, AST_REG_MCR44, 0x88848466);
+	ast_moutdwm(ast, AST_REG_MCR48, 0x44440008);
+	ast_moutdwm(ast, AST_REG_MCR4C, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR50, 0x80000000);
+	ast_moutdwm(ast, AST_REG_MCR50, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR54, 0);
+	ast_moutdwm(ast, AST_REG_MCR60, param->reg_DRV);
+	ast_moutdwm(ast, AST_REG_MCR6C, param->reg_IOZ);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR74, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR78, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR7C, 0x00000000);
 
 	/* Wait MCLK2X lock to MCLK */
 	do {
-		data = ast_mindwm(ast, 0x1E6E001C);
+		data = ast_mindwm(ast, AST_REG_MCR1C);
 	} while (!(data & 0x08000000));
-	data = ast_mindwm(ast, 0x1E6E001C);
+	data = ast_mindwm(ast, AST_REG_MCR1C);
 	data = (data >> 8) & 0xff;
 	while ((data & 0x08) || ((data & 0x7) < 2) || (data < 4)) {
-		data2 = (ast_mindwm(ast, 0x1E6E0064) & 0xfff3ffff) + 4;
+		data2 = (ast_mindwm(ast, AST_REG_MCR64) & 0xfff3ffff) + 4;
 		if ((data2 & 0xff) > param->madj_max)
 			break;
-		ast_moutdwm(ast, 0x1E6E0064, data2);
+		ast_moutdwm(ast, AST_REG_MCR64, data2);
 		if (data2 & 0x00100000)
 			data2 = ((data2 & 0xff) >> 3) + 3;
 		else
 			data2 = ((data2 & 0xff) >> 2) + 5;
-		data = ast_mindwm(ast, 0x1E6E0068) & 0xffff00ff;
+		data = ast_mindwm(ast, AST_REG_MCR68) & 0xffff00ff;
 		data2 += data & 0xff;
 		data = data | (data2 << 8);
-		ast_moutdwm(ast, 0x1E6E0068, data);
+		ast_moutdwm(ast, AST_REG_MCR68, data);
 		udelay(10);
-		ast_moutdwm(ast, 0x1E6E0064, ast_mindwm(ast, 0x1E6E0064) | 0xC0000);
+		ast_moutdwm(ast, AST_REG_MCR64, ast_mindwm(ast, AST_REG_MCR64) | 0xC0000);
 		udelay(10);
-		data = ast_mindwm(ast, 0x1E6E0018) & 0xfffff1ff;
-		ast_moutdwm(ast, 0x1E6E0018, data);
+		data = ast_mindwm(ast, AST_REG_MCR18) & 0xfffff1ff;
+		ast_moutdwm(ast, AST_REG_MCR18, data);
 		data = data | 0x200;
-		ast_moutdwm(ast, 0x1E6E0018, data);
+		ast_moutdwm(ast, AST_REG_MCR18, data);
 		do {
-			data = ast_mindwm(ast, 0x1E6E001C);
+			data = ast_mindwm(ast, AST_REG_MCR1C);
 		} while (!(data & 0x08000000));
 
-		data = ast_mindwm(ast, 0x1E6E001C);
+		data = ast_mindwm(ast, AST_REG_MCR1C);
 		data = (data >> 8) & 0xff;
 	}
-	ast_moutdwm(ast, 0x1E720058, ast_mindwm(ast, 0x1E6E0008) & 0xffff);
-	data = ast_mindwm(ast, 0x1E6E0018) | 0xC00;
-	ast_moutdwm(ast, 0x1E6E0018, data);
+	ast_moutdwm(ast, AST_REG_A2P58, ast_mindwm(ast, AST_REG_MCR08) & 0xffff);
+	data = ast_mindwm(ast, AST_REG_MCR18) | 0xC00;
+	ast_moutdwm(ast, AST_REG_MCR18, data);
 
-	ast_moutdwm(ast, 0x1E6E0034, 0x00000001);
-	ast_moutdwm(ast, 0x1E6E000C, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x00000001);
+	ast_moutdwm(ast, AST_REG_MCR0C, 0x00000000);
 	udelay(50);
 	/* Mode Register Setting */
-	ast_moutdwm(ast, 0x1E6E002C, param->reg_MRS | 0x100);
-	ast_moutdwm(ast, 0x1E6E0030, param->reg_EMRS);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000005);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000007);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000003);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000001);
-
-	ast_moutdwm(ast, 0x1E6E000C, 0x00005C08);
-	ast_moutdwm(ast, 0x1E6E002C, param->reg_MRS);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000001);
-	ast_moutdwm(ast, 0x1E6E0030, param->reg_EMRS | 0x380);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000003);
-	ast_moutdwm(ast, 0x1E6E0030, param->reg_EMRS);
-	ast_moutdwm(ast, 0x1E6E0028, 0x00000003);
-
-	ast_moutdwm(ast, 0x1E6E000C, 0x7FFF5C01);
+	ast_moutdwm(ast, AST_REG_MCR2C, param->reg_MRS | 0x100);
+	ast_moutdwm(ast, AST_REG_MCR30, param->reg_EMRS);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000005);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000007);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000003);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000001);
+
+	ast_moutdwm(ast, AST_REG_MCR0C, 0x00005C08);
+	ast_moutdwm(ast, AST_REG_MCR2C, param->reg_MRS);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000001);
+	ast_moutdwm(ast, AST_REG_MCR30, param->reg_EMRS | 0x380);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000003);
+	ast_moutdwm(ast, AST_REG_MCR30, param->reg_EMRS);
+	ast_moutdwm(ast, AST_REG_MCR28, 0x00000003);
+
+	ast_moutdwm(ast, AST_REG_MCR0C, 0x7FFF5C01);
 	data = 0;
 	if (param->wodt)
 		data = 0x500;
 	if (param->rodt)
 		data = data | 0x3000 | ((param->reg_AC2 & 0x60000) >> 3);
-	ast_moutdwm(ast, 0x1E6E0034, data | 0x3);
-	ast_moutdwm(ast, 0x1E6E0120, param->reg_FREQ);
+	ast_moutdwm(ast, AST_REG_MCR34, data | 0x3);
+	ast_moutdwm(ast, AST_REG_MCR120, param->reg_FREQ);
 
 	/* Calibrate the DQSI delay */
 	if ((cbr_dll2(ast, param) == false) && (retry++ < 10))
@@ -1226,14 +1226,14 @@ static void ddr2_init(struct ast_device *ast, struct ast2300_dram_param *param)
 
 	/* ECC Memory Initialization */
 #ifdef ECC
-	ast_moutdwm(ast, 0x1E6E007C, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0070, 0x221);
+	ast_moutdwm(ast, AST_REG_MCR7C, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x221);
 	do {
-		data = ast_mindwm(ast, 0x1E6E0070);
+		data = ast_mindwm(ast, AST_REG_MCR70);
 	} while (!(data & 0x00001000));
-	ast_moutdwm(ast, 0x1E6E0070, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0050, 0x80000000);
-	ast_moutdwm(ast, 0x1E6E0050, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR50, 0x80000000);
+	ast_moutdwm(ast, AST_REG_MCR50, 0x00000000);
 #endif
 }
 
@@ -1245,26 +1245,19 @@ static void ast_post_chip_2300(struct ast_device *ast)
 
 	reg = ast_get_index_reg_mask(ast, AST_IO_VGACRI, 0xd0, 0xff);
 	if ((reg & 0x80) == 0) {/* vga only */
-		ast_write32(ast, 0xf004, 0x1e6e0000);
-		ast_write32(ast, 0xf000, 0x1);
-		ast_write32(ast, 0x12000, 0x1688a8a8);
-		do {
-			;
-		} while (ast_read32(ast, 0x12000) != 0x1);
+		u32 scu008, scu040;
 
-		ast_write32(ast, 0x10000, 0xfc600309);
-		do {
-			;
-		} while (ast_read32(ast, 0x10000) != 0x1);
+		ast_moutdwm_poll(ast, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY, 0x01);
+		ast_moutdwm_poll(ast, AST_REG_MCR00, AST_REG_MCR00_PROTECTION_KEY, 0x01);
 
 		/* Slow down CPU/AHB CLK in VGA only mode */
-		temp = ast_read32(ast, 0x12008);
-		temp |= 0x73;
-		ast_write32(ast, 0x12008, temp);
+		scu008 = ast_mindwm(ast, AST_REG_SCU008);
+		scu008 |= 0x00000073;
+		ast_moutdwm(ast, AST_REG_SCU008, scu008);
 
 		param.dram_freq = 396;
 		param.dram_type = AST_DDR3;
-		temp = ast_mindwm(ast, 0x1e6e2070);
+		temp = ast_mindwm(ast, AST_REG_SCU070);
 		if (temp & 0x01000000)
 			param.dram_type = AST_DDR2;
 		switch (temp & 0x18000000) {
@@ -1306,8 +1299,9 @@ static void ast_post_chip_2300(struct ast_device *ast)
 			ddr2_init(ast, &param);
 		}
 
-		temp = ast_mindwm(ast, 0x1e6e2040);
-		ast_moutdwm(ast, 0x1e6e2040, temp | 0x40);
+		scu040 = ast_mindwm(ast, AST_REG_SCU040);
+		scu040 |= 0x00000040;
+		ast_moutdwm(ast, AST_REG_SCU040, scu040);
 	}
 
 	/* wait ready */
diff --git a/drivers/gpu/drm/ast/ast_2500.c b/drivers/gpu/drm/ast/ast_2500.c
index 2a52af0ded56c..03a67ec9684bb 100644
--- a/drivers/gpu/drm/ast/ast_2500.c
+++ b/drivers/gpu/drm/ast/ast_2500.c
@@ -109,12 +109,12 @@ void ast_2500_patch_ahb(void __iomem *regs)
 	u32 data;
 
 	/* Clear bus lock condition */
-	__ast_moutdwm(regs, 0x1e600000, 0xAEED1A03);
-	__ast_moutdwm(regs, 0x1e600084, 0x00010000);
-	__ast_moutdwm(regs, 0x1e600088, 0x00000000);
-	__ast_moutdwm(regs, 0x1e6e2000, 0x1688A8A8);
+	__ast_moutdwm(regs, AST_REG_AHBC00, AST_REG_AHBC00_PROTECT_KEY);
+	__ast_moutdwm(regs, AST_REG_AHBC84, 0x00010000);
+	__ast_moutdwm(regs, AST_REG_AHBC88, 0x00000000);
+	__ast_moutdwm(regs, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY);
 
-	data = __ast_mindwm(regs, 0x1e6e2070);
+	data = __ast_mindwm(regs, AST_REG_SCU070);
 	if (data & 0x08000000) { /* check fast reset */
 		/*
 		 * If "Fast restet" is enabled for ARM-ICE debugger,
@@ -127,18 +127,18 @@ void ast_2500_patch_ahb(void __iomem *regs)
 		 *	[1]:= 1:WDT will be cleeared and disabled after timeout occurs
 		 *	[0]:= 1:WDT enable
 		 */
-		__ast_moutdwm(regs, 0x1E785004, 0x00000010);
-		__ast_moutdwm(regs, 0x1E785008, 0x00004755);
-		__ast_moutdwm(regs, 0x1E78500c, 0x00000033);
+		__ast_moutdwm(regs, AST_REG_WDT04(0), 0x00000010);
+		__ast_moutdwm(regs, AST_REG_WDT08(0), 0x00004755);
+		__ast_moutdwm(regs, AST_REG_WDT0C(0), 0x00000033);
 		udelay(1000);
 	}
 
 	do {
-		__ast_moutdwm(regs, 0x1e6e2000, 0x1688A8A8);
-		data = __ast_mindwm(regs, 0x1e6e2000);
+		__ast_moutdwm(regs, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY);
+		data = __ast_mindwm(regs, AST_REG_SCU000);
 	} while (data != 1);
 
-	__ast_moutdwm(regs, 0x1e6e207c, 0x08000000); /* clear fast reset */
+	__ast_moutdwm(regs, AST_REG_SCU07C, 0x08000000); /* clear fast reset */
 }
 
 static bool mmc_test_single_2500(struct ast_device *ast, u32 datagen)
@@ -148,8 +148,8 @@ static bool mmc_test_single_2500(struct ast_device *ast, u32 datagen)
 
 static bool cbr_test_2500(struct ast_device *ast)
 {
-	ast_moutdwm(ast, 0x1E6E0074, 0x0000FFFF);
-	ast_moutdwm(ast, 0x1E6E007C, 0xFF00FF00);
+	ast_moutdwm(ast, AST_REG_MCR74, 0x0000FFFF);
+	ast_moutdwm(ast, AST_REG_MCR7C, 0xFF00FF00);
 	if (!mmc_test_burst(ast, 0))
 		return false;
 	if (!mmc_test_single_2500(ast, 0))
@@ -159,8 +159,8 @@ static bool cbr_test_2500(struct ast_device *ast)
 
 static bool ddr_test_2500(struct ast_device *ast)
 {
-	ast_moutdwm(ast, 0x1E6E0074, 0x0000FFFF);
-	ast_moutdwm(ast, 0x1E6E007C, 0xFF00FF00);
+	ast_moutdwm(ast, AST_REG_MCR74, 0x0000FFFF);
+	ast_moutdwm(ast, AST_REG_MCR7C, 0xFF00FF00);
 	if (!mmc_test_burst(ast, 0))
 		return false;
 	if (!mmc_test_burst(ast, 1))
@@ -176,25 +176,25 @@ static bool ddr_test_2500(struct ast_device *ast)
 
 static void ddr_init_common_2500(struct ast_device *ast)
 {
-	ast_moutdwm(ast, 0x1E6E0034, 0x00020080);
-	ast_moutdwm(ast, 0x1E6E0008, 0x2003000F);
-	ast_moutdwm(ast, 0x1E6E0038, 0x00000FFF);
-	ast_moutdwm(ast, 0x1E6E0040, 0x88448844);
-	ast_moutdwm(ast, 0x1E6E0044, 0x24422288);
-	ast_moutdwm(ast, 0x1E6E0048, 0x22222222);
-	ast_moutdwm(ast, 0x1E6E004C, 0x22222222);
-	ast_moutdwm(ast, 0x1E6E0050, 0x80000000);
-	ast_moutdwm(ast, 0x1E6E0208, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0218, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0220, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0228, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0230, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E02A8, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E02B0, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0240, 0x86000000);
-	ast_moutdwm(ast, 0x1E6E0244, 0x00008600);
-	ast_moutdwm(ast, 0x1E6E0248, 0x80000000);
-	ast_moutdwm(ast, 0x1E6E024C, 0x80808080);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x00020080);
+	ast_moutdwm(ast, AST_REG_MCR08, 0x2003000F);
+	ast_moutdwm(ast, AST_REG_MCR38, 0x00000FFF);
+	ast_moutdwm(ast, AST_REG_MCR40, 0x88448844);
+	ast_moutdwm(ast, AST_REG_MCR44, 0x24422288);
+	ast_moutdwm(ast, AST_REG_MCR48, 0x22222222);
+	ast_moutdwm(ast, AST_REG_MCR4C, 0x22222222);
+	ast_moutdwm(ast, AST_REG_MCR50, 0x80000000);
+	ast_moutdwm(ast, AST_REG_MCR208, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR218, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR220, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR228, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR230, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR2A8, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR2B0, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR240, 0x86000000);
+	ast_moutdwm(ast, AST_REG_MCR244, 0x00008600);
+	ast_moutdwm(ast, AST_REG_MCR248, 0x80000000);
+	ast_moutdwm(ast, AST_REG_MCR24C, 0x80808080);
 }
 
 static void ddr_phy_init_2500(struct ast_device *ast)
@@ -202,29 +202,32 @@ static void ddr_phy_init_2500(struct ast_device *ast)
 	u32 data, pass, timecnt;
 
 	pass = 0;
-	ast_moutdwm(ast, 0x1E6E0060, 0x00000005);
+	ast_moutdwm(ast, AST_REG_MCR60, 0x00000005);
 	while (!pass) {
 		for (timecnt = 0; timecnt < TIMEOUT; timecnt++) {
-			data = ast_mindwm(ast, 0x1E6E0060) & 0x1;
+			data = ast_mindwm(ast, AST_REG_MCR60) & 0x1;
 			if (!data)
 				break;
 		}
 		if (timecnt != TIMEOUT) {
-			data = ast_mindwm(ast, 0x1E6E0300) & 0x000A0000;
+			data = ast_mindwm(ast, AST_REG_MCR300) & 0x000A0000;
 			if (!data)
 				pass = 1;
 		}
 		if (!pass) {
-			ast_moutdwm(ast, 0x1E6E0060, 0x00000000);
+			ast_moutdwm(ast, AST_REG_MCR60, 0x00000000);
 			udelay(10); /* delay 10 us */
-			ast_moutdwm(ast, 0x1E6E0060, 0x00000005);
+			ast_moutdwm(ast, AST_REG_MCR60, 0x00000005);
 		}
 	}
 
-	ast_moutdwm(ast, 0x1E6E0060, 0x00000006);
+	ast_moutdwm(ast, AST_REG_MCR60, 0x00000006);
 }
 
 /*
+ * TODO: Review and fix the comments. The function below only detects
+ *       up to 1 GiB of SDRAM.
+ *
  * Check DRAM Size
  * 1Gb : 0x80000000 ~ 0x87FFFFFF
  * 2Gb : 0x80000000 ~ 0x8FFFFFFF
@@ -235,125 +238,123 @@ static void check_dram_size_2500(struct ast_device *ast, u32 tRFC)
 {
 	u32 reg_04, reg_14;
 
-	reg_04 = ast_mindwm(ast, 0x1E6E0004) & 0xfffffffc;
-	reg_14 = ast_mindwm(ast, 0x1E6E0014) & 0xffffff00;
+	reg_04 = ast_mindwm(ast, AST_REG_MCR04) & 0xfffffffc;
+	reg_14 = ast_mindwm(ast, AST_REG_MCR14) & 0xffffff00;
 
-	ast_moutdwm(ast, 0xA0100000, 0x41424344);
-	ast_moutdwm(ast, 0x90100000, 0x35363738);
-	ast_moutdwm(ast, 0x88100000, 0x292A2B2C);
-	ast_moutdwm(ast, 0x80100000, 0x1D1E1F10);
+	ast_moutdwm(ast, AST_SDRAM(0x20100000), 0x41424344);
+	ast_moutdwm(ast, AST_SDRAM(0x10100000), 0x35363738);
+	ast_moutdwm(ast, AST_SDRAM(0x08100000), 0x292A2B2C);
+	ast_moutdwm(ast, AST_SDRAM(0x00100000), 0x1D1E1F10);
 
 	/* Check 8Gbit */
-	if (ast_mindwm(ast, 0xA0100000) == 0x41424344) {
+	if (ast_mindwm(ast, AST_SDRAM(0x20100000)) == 0x41424344) {
 		reg_04 |= 0x03;
 		reg_14 |= (tRFC >> 24) & 0xFF;
 		/* Check 4Gbit */
-	} else if (ast_mindwm(ast, 0x90100000) == 0x35363738) {
+	} else if (ast_mindwm(ast, AST_SDRAM(0x10100000)) == 0x35363738) {
 		reg_04 |= 0x02;
 		reg_14 |= (tRFC >> 16) & 0xFF;
 		/* Check 2Gbit */
-	} else if (ast_mindwm(ast, 0x88100000) == 0x292A2B2C) {
+	} else if (ast_mindwm(ast, AST_SDRAM(0x08100000)) == 0x292A2B2C) {
 		reg_04 |= 0x01;
 		reg_14 |= (tRFC >> 8) & 0xFF;
 	} else {
 		reg_14 |= tRFC & 0xFF;
 	}
-	ast_moutdwm(ast, 0x1E6E0004, reg_04);
-	ast_moutdwm(ast, 0x1E6E0014, reg_14);
+	ast_moutdwm(ast, AST_REG_MCR04, reg_04);
+	ast_moutdwm(ast, AST_REG_MCR14, reg_14);
 }
 
 static void enable_cache_2500(struct ast_device *ast)
 {
 	u32 reg_04, data;
 
-	reg_04 = ast_mindwm(ast, 0x1E6E0004);
-	ast_moutdwm(ast, 0x1E6E0004, reg_04 | 0x1000);
+	reg_04 = ast_mindwm(ast, AST_REG_MCR04);
+	ast_moutdwm(ast, AST_REG_MCR04, reg_04 | 0x1000);
 
 	do
-		data = ast_mindwm(ast, 0x1E6E0004);
+		data = ast_mindwm(ast, AST_REG_MCR04);
 	while (!(data & 0x80000));
-	ast_moutdwm(ast, 0x1E6E0004, reg_04 | 0x400);
+	ast_moutdwm(ast, AST_REG_MCR04, reg_04 | 0x400);
 }
 
 static void set_mpll_2500(struct ast_device *ast)
 {
-	u32 addr, data, param;
+	u32 mcr, data, param;
 
 	/* Reset MMC */
-	ast_moutdwm(ast, 0x1E6E0000, 0xFC600309);
-	ast_moutdwm(ast, 0x1E6E0034, 0x00020080);
-	for (addr = 0x1e6e0004; addr < 0x1e6e0090;) {
-		ast_moutdwm(ast, addr, 0x0);
-		addr += 4;
-	}
-	ast_moutdwm(ast, 0x1E6E0034, 0x00020000);
-
-	ast_moutdwm(ast, 0x1E6E2000, 0x1688A8A8);
-	data = ast_mindwm(ast, 0x1E6E2070) & 0x00800000;
+	ast_moutdwm(ast, AST_REG_MCR00, AST_REG_MCR00_PROTECTION_KEY);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x00020080);
+	for (mcr = AST_REG_MCR04; mcr <= AST_REG_MCR8C; mcr += 4)
+		ast_moutdwm(ast, mcr, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x00020000);
+
+	ast_moutdwm(ast, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY);
+	data = ast_mindwm(ast, AST_REG_SCU070) & 0x00800000;
 	if (data) {
 		/* CLKIN = 25MHz */
 		param = 0x930023E0;
-		ast_moutdwm(ast, 0x1E6E2160, 0x00011320);
+		ast_moutdwm(ast, AST_REG_SCU160, 0x00011320);
 	} else {
 		/* CLKIN = 24MHz */
 		param = 0x93002400;
 	}
-	ast_moutdwm(ast, 0x1E6E2020, param);
+	ast_moutdwm(ast, AST_REG_SCU020, param);
 	udelay(100);
 }
 
 static void reset_mmc_2500(struct ast_device *ast)
 {
-	ast_moutdwm(ast, 0x1E78505C, 0x00000004);
-	ast_moutdwm(ast, 0x1E785044, 0x00000001);
-	ast_moutdwm(ast, 0x1E785048, 0x00004755);
-	ast_moutdwm(ast, 0x1E78504C, 0x00000013);
+	ast_moutdwm(ast, AST_REG_WDT1C(1), 0x00000004);
+	ast_moutdwm(ast, AST_REG_WDT04(1), 0x00000001);
+	ast_moutdwm(ast, AST_REG_WDT08(1), 0x00004755);
+	ast_moutdwm(ast, AST_REG_WDT0C(1), 0x00000013);
 	mdelay(100);
-	ast_moutdwm(ast, 0x1E785054, 0x00000077);
-	ast_moutdwm(ast, 0x1E6E0000, 0xFC600309);
+	ast_moutdwm(ast, AST_REG_WDT14(1), 0x00000077);
+	ast_moutdwm(ast, AST_REG_MCR00, AST_REG_MCR00_PROTECTION_KEY);
 }
 
 static void ddr3_init_2500(struct ast_device *ast, const u32 *ddr_table)
 {
-	ast_moutdwm(ast, 0x1E6E0004, 0x00000303);
-	ast_moutdwm(ast, 0x1E6E0010, ddr_table[REGIDX_010]);
-	ast_moutdwm(ast, 0x1E6E0014, ddr_table[REGIDX_014]);
-	ast_moutdwm(ast, 0x1E6E0018, ddr_table[REGIDX_018]);
-	ast_moutdwm(ast, 0x1E6E0020, ddr_table[REGIDX_020]);	     /* MODEREG4/6 */
-	ast_moutdwm(ast, 0x1E6E0024, ddr_table[REGIDX_024]);	     /* MODEREG5 */
-	ast_moutdwm(ast, 0x1E6E002C, ddr_table[REGIDX_02C] | 0x100); /* MODEREG0/2 */
-	ast_moutdwm(ast, 0x1E6E0030, ddr_table[REGIDX_030]);	     /* MODEREG1/3 */
+	ast_moutdwm(ast, AST_REG_MCR04, 0x00000303);
+	ast_moutdwm(ast, AST_REG_MCR10, ddr_table[REGIDX_010]);
+	ast_moutdwm(ast, AST_REG_MCR14, ddr_table[REGIDX_014]);
+	ast_moutdwm(ast, AST_REG_MCR18, ddr_table[REGIDX_018]);
+	ast_moutdwm(ast, AST_REG_MCR20, ddr_table[REGIDX_020]);	     /* MODEREG4/6 */
+	ast_moutdwm(ast, AST_REG_MCR24, ddr_table[REGIDX_024]);	     /* MODEREG5 */
+	ast_moutdwm(ast, AST_REG_MCR2C, ddr_table[REGIDX_02C] | 0x100); /* MODEREG0/2 */
+	ast_moutdwm(ast, AST_REG_MCR30, ddr_table[REGIDX_030]);	     /* MODEREG1/3 */
 
 	/* DDR PHY Setting */
-	ast_moutdwm(ast, 0x1E6E0200, 0x02492AAE);
-	ast_moutdwm(ast, 0x1E6E0204, 0x00001001);
-	ast_moutdwm(ast, 0x1E6E020C, 0x55E00B0B);
-	ast_moutdwm(ast, 0x1E6E0210, 0x20000000);
-	ast_moutdwm(ast, 0x1E6E0214, ddr_table[REGIDX_214]);
-	ast_moutdwm(ast, 0x1E6E02E0, ddr_table[REGIDX_2E0]);
-	ast_moutdwm(ast, 0x1E6E02E4, ddr_table[REGIDX_2E4]);
-	ast_moutdwm(ast, 0x1E6E02E8, ddr_table[REGIDX_2E8]);
-	ast_moutdwm(ast, 0x1E6E02EC, ddr_table[REGIDX_2EC]);
-	ast_moutdwm(ast, 0x1E6E02F0, ddr_table[REGIDX_2F0]);
-	ast_moutdwm(ast, 0x1E6E02F4, ddr_table[REGIDX_2F4]);
-	ast_moutdwm(ast, 0x1E6E02F8, ddr_table[REGIDX_2F8]);
-	ast_moutdwm(ast, 0x1E6E0290, 0x00100008);
-	ast_moutdwm(ast, 0x1E6E02C0, 0x00000006);
+	ast_moutdwm(ast, AST_REG_MCR200, 0x02492AAE);
+	ast_moutdwm(ast, AST_REG_MCR204, 0x00001001);
+	ast_moutdwm(ast, AST_REG_MCR20C, 0x55E00B0B);
+	ast_moutdwm(ast, AST_REG_MCR210, 0x20000000);
+	ast_moutdwm(ast, AST_REG_MCR214, ddr_table[REGIDX_214]);
+	ast_moutdwm(ast, AST_REG_MCR2E0, ddr_table[REGIDX_2E0]);
+	ast_moutdwm(ast, AST_REG_MCR2E4, ddr_table[REGIDX_2E4]);
+	ast_moutdwm(ast, AST_REG_MCR2E8, ddr_table[REGIDX_2E8]);
+	ast_moutdwm(ast, AST_REG_MCR2EC, ddr_table[REGIDX_2EC]);
+	ast_moutdwm(ast, AST_REG_MCR2F0, ddr_table[REGIDX_2F0]);
+	ast_moutdwm(ast, AST_REG_MCR2F4, ddr_table[REGIDX_2F4]);
+	ast_moutdwm(ast, AST_REG_MCR2F8, ddr_table[REGIDX_2F8]);
+	ast_moutdwm(ast, AST_REG_MCR290, 0x00100008);
+	ast_moutdwm(ast, AST_REG_MCR2C0, 0x00000006);
 
 	/* Controller Setting */
-	ast_moutdwm(ast, 0x1E6E0034, 0x00020091);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x00020091);
 
 	/* Wait DDR PHY init done */
 	ddr_phy_init_2500(ast);
 
-	ast_moutdwm(ast, 0x1E6E0120, ddr_table[REGIDX_PLL]);
-	ast_moutdwm(ast, 0x1E6E000C, 0x42AA5C81);
-	ast_moutdwm(ast, 0x1E6E0034, 0x0001AF93);
+	ast_moutdwm(ast, AST_REG_MCR120, ddr_table[REGIDX_PLL]);
+	ast_moutdwm(ast, AST_REG_MCR0C, 0x42AA5C81);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x0001AF93);
 
 	check_dram_size_2500(ast, ddr_table[REGIDX_RFC]);
 	enable_cache_2500(ast);
-	ast_moutdwm(ast, 0x1E6E001C, 0x00000008);
-	ast_moutdwm(ast, 0x1E6E0038, 0xFFFFFF00);
+	ast_moutdwm(ast, AST_REG_MCR1C, 0x00000008);
+	ast_moutdwm(ast, AST_REG_MCR38, 0xFFFFFF00);
 }
 
 static void ddr4_init_2500(struct ast_device *ast, const u32 *ddr_table)
@@ -363,34 +364,34 @@ static void ddr4_init_2500(struct ast_device *ast, const u32 *ddr_table)
 	u32 min_ddr_vref = 0, min_phy_vref = 0;
 	u32 max_ddr_vref = 0, max_phy_vref = 0;
 
-	ast_moutdwm(ast, 0x1E6E0004, 0x00000313);
-	ast_moutdwm(ast, 0x1E6E0010, ddr_table[REGIDX_010]);
-	ast_moutdwm(ast, 0x1E6E0014, ddr_table[REGIDX_014]);
-	ast_moutdwm(ast, 0x1E6E0018, ddr_table[REGIDX_018]);
-	ast_moutdwm(ast, 0x1E6E0020, ddr_table[REGIDX_020]);	     /* MODEREG4/6 */
-	ast_moutdwm(ast, 0x1E6E0024, ddr_table[REGIDX_024]);	     /* MODEREG5 */
-	ast_moutdwm(ast, 0x1E6E002C, ddr_table[REGIDX_02C] | 0x100); /* MODEREG0/2 */
-	ast_moutdwm(ast, 0x1E6E0030, ddr_table[REGIDX_030]);	     /* MODEREG1/3 */
+	ast_moutdwm(ast, AST_REG_MCR04, 0x00000313);
+	ast_moutdwm(ast, AST_REG_MCR10, ddr_table[REGIDX_010]);
+	ast_moutdwm(ast, AST_REG_MCR14, ddr_table[REGIDX_014]);
+	ast_moutdwm(ast, AST_REG_MCR18, ddr_table[REGIDX_018]);
+	ast_moutdwm(ast, AST_REG_MCR20, ddr_table[REGIDX_020]);	     /* MODEREG4/6 */
+	ast_moutdwm(ast, AST_REG_MCR24, ddr_table[REGIDX_024]);	     /* MODEREG5 */
+	ast_moutdwm(ast, AST_REG_MCR2C, ddr_table[REGIDX_02C] | 0x100); /* MODEREG0/2 */
+	ast_moutdwm(ast, AST_REG_MCR30, ddr_table[REGIDX_030]);	     /* MODEREG1/3 */
 
 	/* DDR PHY Setting */
-	ast_moutdwm(ast, 0x1E6E0200, 0x42492AAE);
-	ast_moutdwm(ast, 0x1E6E0204, 0x09002000);
-	ast_moutdwm(ast, 0x1E6E020C, 0x55E00B0B);
-	ast_moutdwm(ast, 0x1E6E0210, 0x20000000);
-	ast_moutdwm(ast, 0x1E6E0214, ddr_table[REGIDX_214]);
-	ast_moutdwm(ast, 0x1E6E02E0, ddr_table[REGIDX_2E0]);
-	ast_moutdwm(ast, 0x1E6E02E4, ddr_table[REGIDX_2E4]);
-	ast_moutdwm(ast, 0x1E6E02E8, ddr_table[REGIDX_2E8]);
-	ast_moutdwm(ast, 0x1E6E02EC, ddr_table[REGIDX_2EC]);
-	ast_moutdwm(ast, 0x1E6E02F0, ddr_table[REGIDX_2F0]);
-	ast_moutdwm(ast, 0x1E6E02F4, ddr_table[REGIDX_2F4]);
-	ast_moutdwm(ast, 0x1E6E02F8, ddr_table[REGIDX_2F8]);
-	ast_moutdwm(ast, 0x1E6E0290, 0x00100008);
-	ast_moutdwm(ast, 0x1E6E02C4, 0x3C183C3C);
-	ast_moutdwm(ast, 0x1E6E02C8, 0x00631E0E);
+	ast_moutdwm(ast, AST_REG_MCR200, 0x42492AAE);
+	ast_moutdwm(ast, AST_REG_MCR204, 0x09002000);
+	ast_moutdwm(ast, AST_REG_MCR20C, 0x55E00B0B);
+	ast_moutdwm(ast, AST_REG_MCR210, 0x20000000);
+	ast_moutdwm(ast, AST_REG_MCR214, ddr_table[REGIDX_214]);
+	ast_moutdwm(ast, AST_REG_MCR2E0, ddr_table[REGIDX_2E0]);
+	ast_moutdwm(ast, AST_REG_MCR2E4, ddr_table[REGIDX_2E4]);
+	ast_moutdwm(ast, AST_REG_MCR2E8, ddr_table[REGIDX_2E8]);
+	ast_moutdwm(ast, AST_REG_MCR2EC, ddr_table[REGIDX_2EC]);
+	ast_moutdwm(ast, AST_REG_MCR2F0, ddr_table[REGIDX_2F0]);
+	ast_moutdwm(ast, AST_REG_MCR2F4, ddr_table[REGIDX_2F4]);
+	ast_moutdwm(ast, AST_REG_MCR2F8, ddr_table[REGIDX_2F8]);
+	ast_moutdwm(ast, AST_REG_MCR290, 0x00100008);
+	ast_moutdwm(ast, AST_REG_MCR2C4, 0x3C183C3C);
+	ast_moutdwm(ast, AST_REG_MCR2C8, 0x00631E0E);
 
 	/* Controller Setting */
-	ast_moutdwm(ast, 0x1E6E0034, 0x0001A991);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x0001A991);
 
 	/* Train PHY Vref first */
 	pass = 0;
@@ -398,17 +399,17 @@ static void ddr4_init_2500(struct ast_device *ast, const u32 *ddr_table)
 	for (retrycnt = 0; retrycnt < 4 && pass == 0; retrycnt++) {
 		max_phy_vref = 0x0;
 		pass = 0;
-		ast_moutdwm(ast, 0x1E6E02C0, 0x00001C06);
+		ast_moutdwm(ast, AST_REG_MCR2C0, 0x00001C06);
 		for (phy_vref = 0x40; phy_vref < 0x80; phy_vref++) {
-			ast_moutdwm(ast, 0x1E6E000C, 0x00000000);
-			ast_moutdwm(ast, 0x1E6E0060, 0x00000000);
-			ast_moutdwm(ast, 0x1E6E02CC, phy_vref | (phy_vref << 8));
+			ast_moutdwm(ast, AST_REG_MCR0C, 0x00000000);
+			ast_moutdwm(ast, AST_REG_MCR60, 0x00000000);
+			ast_moutdwm(ast, AST_REG_MCR2CC, phy_vref | (phy_vref << 8));
 			/* Fire DFI Init */
 			ddr_phy_init_2500(ast);
-			ast_moutdwm(ast, 0x1E6E000C, 0x00005C01);
+			ast_moutdwm(ast, AST_REG_MCR0C, 0x00005C01);
 			if (cbr_test_2500(ast)) {
 				pass++;
-				data = ast_mindwm(ast, 0x1E6E03D0);
+				data = ast_mindwm(ast, AST_REG_MCR3D0);
 				data2 = data >> 8;
 				data  = data & 0xff;
 				if (data > data2)
@@ -422,7 +423,7 @@ static void ddr4_init_2500(struct ast_device *ast, const u32 *ddr_table)
 			}
 		}
 	}
-	ast_moutdwm(ast, 0x1E6E02CC, min_phy_vref | (min_phy_vref << 8));
+	ast_moutdwm(ast, AST_REG_MCR2CC, min_phy_vref | (min_phy_vref << 8));
 
 	/* Train DDR Vref next */
 	pass = 0;
@@ -432,12 +433,12 @@ static void ddr4_init_2500(struct ast_device *ast, const u32 *ddr_table)
 		max_ddr_vref = 0x0;
 		pass = 0;
 		for (ddr_vref = 0x00; ddr_vref < 0x40; ddr_vref++) {
-			ast_moutdwm(ast, 0x1E6E000C, 0x00000000);
-			ast_moutdwm(ast, 0x1E6E0060, 0x00000000);
-			ast_moutdwm(ast, 0x1E6E02C0, 0x00000006 | (ddr_vref << 8));
+			ast_moutdwm(ast, AST_REG_MCR0C, 0x00000000);
+			ast_moutdwm(ast, AST_REG_MCR60, 0x00000000);
+			ast_moutdwm(ast, AST_REG_MCR2C0, 0x00000006 | (ddr_vref << 8));
 			/* Fire DFI Init */
 			ddr_phy_init_2500(ast);
-			ast_moutdwm(ast, 0x1E6E000C, 0x00005C01);
+			ast_moutdwm(ast, AST_REG_MCR0C, 0x00005C01);
 			if (cbr_test_2500(ast)) {
 				pass++;
 				if (min_ddr_vref > ddr_vref)
@@ -450,22 +451,22 @@ static void ddr4_init_2500(struct ast_device *ast, const u32 *ddr_table)
 		}
 	}
 
-	ast_moutdwm(ast, 0x1E6E000C, 0x00000000);
-	ast_moutdwm(ast, 0x1E6E0060, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR0C, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR60, 0x00000000);
 	ddr_vref = (min_ddr_vref + max_ddr_vref + 1) >> 1;
-	ast_moutdwm(ast, 0x1E6E02C0, 0x00000006 | (ddr_vref << 8));
+	ast_moutdwm(ast, AST_REG_MCR2C0, 0x00000006 | (ddr_vref << 8));
 
 	/* Wait DDR PHY init done */
 	ddr_phy_init_2500(ast);
 
-	ast_moutdwm(ast, 0x1E6E0120, ddr_table[REGIDX_PLL]);
-	ast_moutdwm(ast, 0x1E6E000C, 0x42AA5C81);
-	ast_moutdwm(ast, 0x1E6E0034, 0x0001AF93);
+	ast_moutdwm(ast, AST_REG_MCR120, ddr_table[REGIDX_PLL]);
+	ast_moutdwm(ast, AST_REG_MCR0C, 0x42AA5C81);
+	ast_moutdwm(ast, AST_REG_MCR34, 0x0001AF93);
 
 	check_dram_size_2500(ast, ddr_table[REGIDX_RFC]);
 	enable_cache_2500(ast);
-	ast_moutdwm(ast, 0x1E6E001C, 0x00000008);
-	ast_moutdwm(ast, 0x1E6E0038, 0xFFFFFF00);
+	ast_moutdwm(ast, AST_REG_MCR1C, 0x00000008);
+	ast_moutdwm(ast, AST_REG_MCR38, 0xFFFFFF00);
 }
 
 static bool ast_dram_init_2500(struct ast_device *ast)
@@ -480,18 +481,18 @@ static bool ast_dram_init_2500(struct ast_device *ast)
 		reset_mmc_2500(ast);
 		ddr_init_common_2500(ast);
 
-		data = ast_mindwm(ast, 0x1E6E2070);
+		data = ast_mindwm(ast, AST_REG_SCU070);
 		if (data & 0x01000000)
 			ddr4_init_2500(ast, ast2500_ddr4_1600_timing_table);
 		else
 			ddr3_init_2500(ast, ast2500_ddr3_1600_timing_table);
 	} while (!ddr_test_2500(ast));
 
-	ast_moutdwm(ast, 0x1E6E2040, ast_mindwm(ast, 0x1E6E2040) | 0x41);
+	ast_moutdwm(ast, AST_REG_SCU040, ast_mindwm(ast, AST_REG_SCU040) | 0x41);
 
 	/* Patch code */
-	data = ast_mindwm(ast, 0x1E6E200C) & 0xF9FFFFFF;
-	ast_moutdwm(ast, 0x1E6E200C, data | 0x10000000);
+	data = ast_mindwm(ast, AST_REG_SCU00C) & 0xF9FFFFFF;
+	ast_moutdwm(ast, AST_REG_SCU00C, data | 0x10000000);
 
 	return true;
 }
@@ -504,12 +505,14 @@ static void ast_post_chip_2500(struct ast_device *ast)
 
 	reg = ast_get_index_reg_mask(ast, AST_IO_VGACRI, 0xd0, 0xff);
 	if ((reg & AST_IO_VGACRD0_VRAM_INIT_STATUS_MASK) == 0) {/* vga only */
+		u32 scu008;
+
 		/* Clear bus lock condition */
 		ast_2500_patch_ahb(ast->regs);
 
 		/* Disable watchdog */
-		ast_moutdwm(ast, 0x1E78502C, 0x00000000);
-		ast_moutdwm(ast, 0x1E78504C, 0x00000000);
+		ast_moutdwm(ast, AST_REG_WDT2C(0), 0x00000000);
+		ast_moutdwm(ast, AST_REG_WDT0C(1), 0x00000000);
 
 		/*
 		 * Reset USB port to patch USB unknown device issue
@@ -524,28 +527,28 @@ static void ast_post_chip_2500(struct ast_device *ast)
 		 * SCU7C is Write clear reg to SCU70
 		 *	[23]:= write 1 and then SCU70[23] will be clear as 0b.
 		 */
-		ast_moutdwm(ast, 0x1E6E2090, 0x20000000);
-		ast_moutdwm(ast, 0x1E6E2094, 0x00004000);
-		if (ast_mindwm(ast, 0x1E6E2070) & 0x00800000) {
-			ast_moutdwm(ast, 0x1E6E207C, 0x00800000);
+		ast_moutdwm(ast, AST_REG_SCU090, 0x20000000);
+		ast_moutdwm(ast, AST_REG_SCU094, 0x00004000);
+		if (ast_mindwm(ast, AST_REG_SCU070) & 0x00800000) {
+			ast_moutdwm(ast, AST_REG_SCU07C, 0x00800000);
 			mdelay(100);
-			ast_moutdwm(ast, 0x1E6E2070, 0x00800000);
+			ast_moutdwm(ast, AST_REG_SCU070, 0x00800000);
 		}
 		/* Modify eSPI reset pin */
-		temp = ast_mindwm(ast, 0x1E6E2070);
+		temp = ast_mindwm(ast, AST_REG_SCU070);
 		if (temp & 0x02000000)
-			ast_moutdwm(ast, 0x1E6E207C, 0x00004000);
+			ast_moutdwm(ast, AST_REG_SCU07C, 0x00004000);
 
 		/* Slow down CPU/AHB CLK in VGA only mode */
-		temp = ast_read32(ast, 0x12008);
-		temp |= 0x73;
-		ast_write32(ast, 0x12008, temp);
+		scu008 = ast_mindwm(ast, AST_REG_SCU008);
+		scu008 |= 0x00000073;
+		ast_moutdwm(ast, AST_REG_SCU008, scu008);
 
 		if (!ast_dram_init_2500(ast))
 			drm_err(dev, "DRAM init failed !\n");
 
-		temp = ast_mindwm(ast, 0x1e6e2040);
-		ast_moutdwm(ast, 0x1e6e2040, temp | 0x40);
+		temp = ast_mindwm(ast, AST_REG_SCU040);
+		ast_moutdwm(ast, AST_REG_SCU040, temp | 0x40);
 	}
 
 	/* wait ready */
diff --git a/drivers/gpu/drm/ast/ast_cursor.c b/drivers/gpu/drm/ast/ast_cursor.c
index 3fa8e75db9c05..fd19c45f2abe2 100644
--- a/drivers/gpu/drm/ast/ast_cursor.c
+++ b/drivers/gpu/drm/ast/ast_cursor.c
@@ -164,7 +164,7 @@ static const uint32_t ast_cursor_plane_formats[] = {
 };
 
 static int ast_cursor_plane_helper_atomic_check(struct drm_plane *plane,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_framebuffer *new_fb = new_plane_state->fb;
@@ -244,7 +244,7 @@ static const u8 *ast_cursor_plane_get_argb4444(struct ast_cursor_plane *ast_curs
 }
 
 static void ast_cursor_plane_helper_atomic_update(struct drm_plane *plane,
-						  struct drm_atomic_state *state)
+						  struct drm_atomic_commit *state)
 {
 	struct ast_cursor_plane *ast_cursor_plane = to_ast_cursor_plane(plane);
 	struct ast_plane *ast_plane = to_ast_plane(plane);
@@ -309,7 +309,7 @@ static void ast_cursor_plane_helper_atomic_update(struct drm_plane *plane,
 }
 
 static void ast_cursor_plane_helper_atomic_disable(struct drm_plane *plane,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct ast_device *ast = to_ast_device(plane->dev);
 
diff --git a/drivers/gpu/drm/ast/ast_dp.c b/drivers/gpu/drm/ast/ast_dp.c
index 9d07dad358cfd..5f266a05419ba 100644
--- a/drivers/gpu/drm/ast/ast_dp.c
+++ b/drivers/gpu/drm/ast/ast_dp.c
@@ -359,7 +359,7 @@ static void ast_astdp_encoder_helper_atomic_mode_set(struct drm_encoder *encoder
 }
 
 static void ast_astdp_encoder_helper_atomic_enable(struct drm_encoder *encoder,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct ast_device *ast = to_ast_device(encoder->dev);
 	struct ast_connector *ast_connector = &ast->output.astdp.connector;
@@ -374,7 +374,7 @@ static void ast_astdp_encoder_helper_atomic_enable(struct drm_encoder *encoder,
 }
 
 static void ast_astdp_encoder_helper_atomic_disable(struct drm_encoder *encoder,
-						    struct drm_atomic_state *state)
+						    struct drm_atomic_commit *state)
 {
 	struct ast_device *ast = to_ast_device(encoder->dev);
 
diff --git a/drivers/gpu/drm/ast/ast_dp501.c b/drivers/gpu/drm/ast/ast_dp501.c
index 677c52c0d99a3..6d6ccfad14154 100644
--- a/drivers/gpu/drm/ast/ast_dp501.c
+++ b/drivers/gpu/drm/ast/ast_dp501.c
@@ -183,7 +183,7 @@ static void ast_set_dp501_video_output(struct ast_device *ast, u8 mode)
 
 static u32 get_fw_base(struct ast_device *ast)
 {
-	return ast_mindwm(ast, 0x1e6e2104) & 0x7fffffff;
+	return ast_mindwm(ast, AST_REG_SCU104) & 0x7fffffff;
 }
 
 bool ast_backup_fw(struct ast_device *ast, u8 *addr, u32 size)
@@ -194,7 +194,7 @@ bool ast_backup_fw(struct ast_device *ast, u8 *addr, u32 size)
 	if (ast->config_mode != ast_use_p2a)
 		return false;
 
-	data = ast_mindwm(ast, 0x1e6e2100) & 0x01;
+	data = ast_mindwm(ast, AST_REG_SCU100) & 0x01;
 	if (data) {
 		boot_address = get_fw_base(ast);
 		for (i = 0; i < size; i += 4)
@@ -214,7 +214,7 @@ static bool ast_launch_m68k(struct ast_device *ast)
 	if (ast->config_mode != ast_use_p2a)
 		return false;
 
-	data = ast_mindwm(ast, 0x1e6e2100) & 0x01;
+	data = ast_mindwm(ast, AST_REG_SCU100) & 0x01;
 	if (!data) {
 
 		if (ast->dp501_fw_addr) {
@@ -229,8 +229,8 @@ static bool ast_launch_m68k(struct ast_device *ast)
 			len = ast->dp501_fw->size;
 		}
 		/* Get BootAddress */
-		ast_moutdwm(ast, 0x1e6e2000, 0x1688a8a8);
-		data = ast_mindwm(ast, 0x1e6e0004);
+		ast_moutdwm(ast, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY);
+		data = ast_mindwm(ast, AST_REG_MCR04);
 		switch (data & 0x03) {
 		case 0:
 			boot_address = 0x44000000;
@@ -255,16 +255,16 @@ static bool ast_launch_m68k(struct ast_device *ast)
 		}
 
 		/* Init SCU */
-		ast_moutdwm(ast, 0x1e6e2000, 0x1688a8a8);
+		ast_moutdwm(ast, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY);
 
 		/* Launch FW */
-		ast_moutdwm(ast, 0x1e6e2104, 0x80000000 + boot_address);
-		ast_moutdwm(ast, 0x1e6e2100, 1);
+		ast_moutdwm(ast, AST_REG_SCU104, 0x80000000 + boot_address);
+		ast_moutdwm(ast, AST_REG_SCU100, 1);
 
 		/* Update Scratch */
-		data = ast_mindwm(ast, 0x1e6e2040) & 0xfffff1ff;		/* D[11:9] = 100b: UEFI handling */
-		data |= 0x800;
-		ast_moutdwm(ast, 0x1e6e2040, data);
+		data = ast_mindwm(ast, AST_REG_SCU040) & 0xfffff1ff;
+		data |= 0x800; /* D[11:9] = 100b: UEFI handling */
+		ast_moutdwm(ast, AST_REG_SCU040, data);
 
 		jreg = ast_get_index_reg_mask(ast, AST_IO_VGACRI, 0x99, 0xfc); /* D[1:0]: Reserved Video Buffer */
 		jreg |= 0x02;
@@ -347,68 +347,65 @@ static int ast_dp512_read_edid_block(void *data, u8 *buf, unsigned int block, si
 static bool ast_init_dvo(struct ast_device *ast)
 {
 	u8 jreg;
-	u32 data;
-	ast_write32(ast, 0xf004, 0x1e6e0000);
-	ast_write32(ast, 0xf000, 0x1);
-	ast_write32(ast, 0x12000, 0x1688a8a8);
+	u32 scu02c;
+
+	ast_moutdwm(ast, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY);
 
 	jreg = ast_get_index_reg_mask(ast, AST_IO_VGACRI, 0xd0, 0xff);
 	if (!(jreg & 0x80)) {
+		u32 scu008;
+
 		/* Init SCU DVO Settings */
-		data = ast_read32(ast, 0x12008);
-		/* delay phase */
-		data &= 0xfffff8ff;
-		data |= 0x00000500;
-		ast_write32(ast, 0x12008, data);
+
+		scu008 = ast_mindwm(ast, AST_REG_SCU008);
+		scu008 &= 0xfffff8ff;
+		scu008 |= 0x00000500; /* delay phase */
+		ast_moutdwm(ast, AST_REG_SCU008, scu008);
 
 		if (IS_AST_GEN4(ast)) {
-			data = ast_read32(ast, 0x12084);
-			/* multi-pins for DVO single-edge */
-			data |= 0xfffe0000;
-			ast_write32(ast, 0x12084, data);
-
-			data = ast_read32(ast, 0x12088);
-			/* multi-pins for DVO single-edge */
-			data |= 0x000fffff;
-			ast_write32(ast, 0x12088, data);
-
-			data = ast_read32(ast, 0x12090);
-			/* multi-pins for DVO single-edge */
-			data &= 0xffffffcf;
-			data |= 0x00000020;
-			ast_write32(ast, 0x12090, data);
+			u32 scu084, scu088, scu090;
+
+			scu084 = ast_mindwm(ast, AST_REG_SCU084);
+			scu084 |= 0xfffe0000; /* multi-pins for DVO single-edge */
+			ast_moutdwm(ast, AST_REG_SCU084, scu084);
+
+			scu088 = ast_mindwm(ast, AST_REG_SCU088);
+			scu088 |= 0x000fffff; /* multi-pins for DVO single-edge */
+			ast_moutdwm(ast, AST_REG_SCU088, scu088);
+
+			scu090 = ast_mindwm(ast, AST_REG_SCU090);
+			scu090 &= 0xffffffcf;
+			scu090 |= 0x00000020; /* multi-pins for DVO single-edge */
+			ast_moutdwm(ast, AST_REG_SCU090, scu090);
 		} else { /* AST GEN5+ */
-			data = ast_read32(ast, 0x12088);
-			/* multi-pins for DVO single-edge */
-			data |= 0x30000000;
-			ast_write32(ast, 0x12088, data);
-
-			data = ast_read32(ast, 0x1208c);
-			/* multi-pins for DVO single-edge */
-			data |= 0x000000cf;
-			ast_write32(ast, 0x1208c, data);
-
-			data = ast_read32(ast, 0x120a4);
-			/* multi-pins for DVO single-edge */
-			data |= 0xffff0000;
-			ast_write32(ast, 0x120a4, data);
-
-			data = ast_read32(ast, 0x120a8);
-			/* multi-pins for DVO single-edge */
-			data |= 0x0000000f;
-			ast_write32(ast, 0x120a8, data);
-
-			data = ast_read32(ast, 0x12094);
-			/* multi-pins for DVO single-edge */
-			data |= 0x00000002;
-			ast_write32(ast, 0x12094, data);
+			u32 scu088, scu08c, scu0a4, scu0a8, scu094;
+
+			scu088 = ast_mindwm(ast, AST_REG_SCU088);
+			scu088 |= 0x30000000; /* multi-pins for DVO single-edge */
+			ast_moutdwm(ast, AST_REG_SCU088, scu088);
+
+			scu08c = ast_mindwm(ast, AST_REG_SCU08C);
+			scu08c |= 0x000000cf; /* multi-pins for DVO single-edge */
+			ast_moutdwm(ast, AST_REG_SCU08C, scu08c);
+
+			scu0a4 = ast_mindwm(ast, AST_REG_SCU0A4);
+			scu0a4 |= 0xffff0000; /* multi-pins for DVO single-edge */
+			ast_moutdwm(ast, AST_REG_SCU0A4, scu0a4);
+
+			scu0a8 = ast_mindwm(ast, AST_REG_SCU0A8);
+			scu0a8 |= 0x0000000f; /* multi-pins for DVO single-edge */
+			ast_moutdwm(ast, AST_REG_SCU0A8, scu0a8);
+
+			scu094 = ast_mindwm(ast, AST_REG_SCU094);
+			scu094 |= 0x00000002; /* multi-pins for DVO single-edge */
+			ast_moutdwm(ast, AST_REG_SCU094, scu094);
 		}
 	}
 
 	/* Force to DVO */
-	data = ast_read32(ast, 0x1202c);
-	data &= 0xfffbffff;
-	ast_write32(ast, 0x1202c, data);
+	scu02c = ast_mindwm(ast, AST_REG_SCU02C);
+	scu02c &= 0xfffbffff;
+	ast_moutdwm(ast, AST_REG_SCU02C, scu02c);
 
 	/* Init VGA DVO Settings */
 	ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa3, 0xcf, 0x80);
@@ -418,25 +415,20 @@ static bool ast_init_dvo(struct ast_device *ast)
 
 static void ast_init_analog(struct ast_device *ast)
 {
-	u32 data;
+	u32 scu02c;
 
 	/*
 	 * Set DAC source to VGA mode in SCU2C via the P2A
-	 * bridge. First configure the P2U to target the SCU
-	 * in case it isn't at this stage.
+	 * bridge.
 	 */
-	ast_write32(ast, 0xf004, 0x1e6e0000);
-	ast_write32(ast, 0xf000, 0x1);
 
-	/* Then unlock the SCU with the magic password */
-	ast_write32(ast, 0x12000, 0x1688a8a8);
-	ast_write32(ast, 0x12000, 0x1688a8a8);
-	ast_write32(ast, 0x12000, 0x1688a8a8);
+	/* Unlock the SCU with the magic password */
+	ast_moutdwm_poll(ast, AST_REG_SCU000, AST_REG_SCU000_PROTECTION_KEY, 0x01);
 
-	/* Finally, clear bits [17:16] of SCU2c */
-	data = ast_read32(ast, 0x1202c);
-	data &= 0xfffcffff;
-	ast_write32(ast, 0x1202c, data);
+	/* Clear bits [17:16] of SCU2C */
+	scu02c = ast_mindwm(ast, AST_REG_SCU02C);
+	scu02c &= 0xfffcffff;
+	ast_moutdwm(ast, AST_REG_SCU02C, scu02c);
 
 	/* Disable DVO */
 	ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa3, 0xcf, 0x00);
@@ -477,7 +469,7 @@ static const struct drm_encoder_funcs ast_dp501_encoder_funcs = {
 };
 
 static void ast_dp501_encoder_helper_atomic_enable(struct drm_encoder *encoder,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct ast_device *ast = to_ast_device(encoder->dev);
 
@@ -485,7 +477,7 @@ static void ast_dp501_encoder_helper_atomic_enable(struct drm_encoder *encoder,
 }
 
 static void ast_dp501_encoder_helper_atomic_disable(struct drm_encoder *encoder,
-						    struct drm_atomic_state *state)
+						    struct drm_atomic_commit *state)
 {
 	struct ast_device *ast = to_ast_device(encoder->dev);
 
diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c
index b9a9b050b5461..ba6cf5fa901c9 100644
--- a/drivers/gpu/drm/ast/ast_drv.c
+++ b/drivers/gpu/drm/ast/ast_drv.c
@@ -47,6 +47,77 @@ static int ast_modeset = -1;
 MODULE_PARM_DESC(modeset, "Disable/Enable modesetting");
 module_param_named(modeset, ast_modeset, int, 0400);
 
+/*
+ * Register access
+ */
+
+/* Select R/W segment */
+static void __ast_selseg(void __iomem *regs, u32 r)
+{
+	u32 p2a04, p2a04_base;
+
+	p2a04 = r & AST_REG_P2A04_BASE_MASK;
+	__ast_write32(regs, AST_REG_P2A04, p2a04);
+	__ast_write32(regs, AST_REG_P2A00, AST_REG_P2A00_PROTECTION_KEY);
+
+	do {
+		cpu_relax();
+		p2a04_base = __ast_read32(regs, AST_REG_P2A04);
+		p2a04_base &= AST_REG_P2A04_BASE_MASK;
+	} while (p2a04_base != p2a04);
+}
+
+/* Read within segment */
+static u32 __ast_rdseg32(void __iomem *regs, u32 r)
+{
+	return __ast_read32(regs, AST_REG_P2A_ADDR(r));
+}
+
+/* Write within segment */
+static void __ast_wrseg32(void __iomem *regs, u32 r, u32 v)
+{
+	__ast_write32(regs, AST_REG_P2A_ADDR(r), v);
+}
+
+u32 __ast_mindwm(void __iomem *regs, u32 r)
+{
+	__ast_selseg(regs, r);
+
+	return __ast_rdseg32(regs, r);
+}
+
+void __ast_moutdwm(void __iomem *regs, u32 r, u32 v)
+{
+	__ast_selseg(regs, r);
+	__ast_wrseg32(regs, r, v);
+}
+
+u32 ast_mindwm(struct ast_device *ast, u32 r)
+{
+	return __ast_mindwm(ast->regs, r);
+}
+
+void ast_moutdwm(struct ast_device *ast, u32 r, u32 v)
+{
+	__ast_moutdwm(ast->regs, r, v);
+}
+
+void ast_moutdwm_poll(struct ast_device *ast, u32 r, u32 v, u32 res)
+{
+	void __iomem *regs = ast->regs;
+
+	__ast_selseg(regs, r);
+	__ast_wrseg32(regs, r, v);
+
+	do {
+		cpu_relax();
+	} while (__ast_rdseg32(regs, r) != res);
+}
+
+/*
+ * AST device
+ */
+
 void ast_device_init(struct ast_device *ast,
 		     enum ast_chip chip,
 		     enum ast_config_mode config_mode,
@@ -171,7 +242,7 @@ static int ast_detect_chip(struct pci_dev *pdev,
 	enum ast_config_mode config_mode = ast_use_defaults;
 	uint32_t scu_rev = 0xffffffff;
 	enum ast_chip chip;
-	u32 data;
+	u32 data, p2a04, scu07c;
 	u8 vgacrd0, vgacrd1;
 
 	/*
@@ -204,14 +275,13 @@ static int ast_detect_chip(struct pci_dev *pdev,
 			}
 
 			/* Double check that it's actually working */
-			data = __ast_read32(regs, 0xf004);
-			if ((data != 0xffffffff) && (data != 0x00)) {
+			p2a04 = __ast_read32(regs, AST_REG_P2A04);
+			if (p2a04 != 0xffffffff && p2a04 != 0x00000000) {
 				config_mode = ast_use_p2a;
 
-				/* Read SCU7c (silicon revision register) */
-				__ast_write32(regs, 0xf004, 0x1e6e0000);
-				__ast_write32(regs, 0xf000, 0x1);
-				scu_rev = __ast_read32(regs, 0x1207c);
+				/* Read SCU7C (silicon revision register) */
+				scu07c = __ast_mindwm(regs, AST_REG_SCU07C);
+				scu_rev = scu07c & AST_REG_SCU07C_CHIP_BONDING_MASK;
 			}
 		}
 	}
diff --git a/drivers/gpu/drm/ast/ast_drv.h b/drivers/gpu/drm/ast/ast_drv.h
index 787e38c6c17dd..4f221b848d68b 100644
--- a/drivers/gpu/drm/ast/ast_drv.h
+++ b/drivers/gpu/drm/ast/ast_drv.h
@@ -259,26 +259,20 @@ static inline bool __ast_gen_is_eq(struct ast_device *ast, unsigned long gen)
 #define IS_AST_GEN6(__ast)	__ast_gen_is_eq(__ast, 6)
 #define IS_AST_GEN7(__ast)	__ast_gen_is_eq(__ast, 7)
 
+/*
+ * MMIO access
+ */
+
 static inline u8 __ast_read8(const void __iomem *addr, u32 reg)
 {
 	return ioread8(addr + reg);
 }
 
-static inline u32 __ast_read32(const void __iomem *addr, u32 reg)
-{
-	return ioread32(addr + reg);
-}
-
 static inline void __ast_write8(void __iomem *addr, u32 reg, u8 val)
 {
 	iowrite8(val, addr + reg);
 }
 
-static inline void __ast_write32(void __iomem *addr, u32 reg, u32 val)
-{
-	iowrite32(val, addr + reg);
-}
-
 static inline u8 __ast_read8_i(void __iomem *addr, u32 reg, u8 index)
 {
 	__ast_write8(addr, reg, index);
@@ -307,16 +301,6 @@ static inline void __ast_write8_i_masked(void __iomem *addr, u32 reg, u8 index,
 	__ast_write8_i(addr, reg, index, tmp | val);
 }
 
-static inline u32 ast_read32(struct ast_device *ast, u32 reg)
-{
-	return __ast_read32(ast->regs, reg);
-}
-
-static inline void ast_write32(struct ast_device *ast, u32 reg, u32 val)
-{
-	__ast_write32(ast->regs, reg, val);
-}
-
 static inline u8 ast_io_read8(struct ast_device *ast, u32 reg)
 {
 	return __ast_read8(ast->ioregs, reg);
@@ -349,6 +333,40 @@ static inline void ast_set_index_reg_mask(struct ast_device *ast, u32 base, u8 i
 	__ast_write8_i_masked(ast->ioregs, base, index, preserve_mask, val);
 }
 
+/*
+ * Register access
+ */
+
+static inline u32 __ast_read32(const void __iomem *addr, u32 reg)
+{
+	return ioread32(addr + reg);
+}
+
+static inline void __ast_write32(void __iomem *addr, u32 reg, u32 val)
+{
+	iowrite32(val, addr + reg);
+}
+
+static inline u32 ast_read32(struct ast_device *ast, u32 reg)
+{
+	return __ast_read32(ast->regs, reg);
+}
+
+static inline void ast_write32(struct ast_device *ast, u32 reg, u32 val)
+{
+	__ast_write32(ast->regs, reg, val);
+}
+
+u32 __ast_mindwm(void __iomem *regs, u32 r);
+void __ast_moutdwm(void __iomem *regs, u32 r, u32 v);
+u32 ast_mindwm(struct ast_device *ast, u32 r);
+void ast_moutdwm(struct ast_device *ast, u32 r, u32 v);
+void ast_moutdwm_poll(struct ast_device *ast, u32 r, u32 v, u32 res);
+
+/*
+ * VBIOS
+ */
+
 struct ast_vbios_stdtable {
 	u8 misc;
 	u8 seq[4];
@@ -517,8 +535,6 @@ struct drm_device *ast_2600_device_create(struct pci_dev *pdev,
 
 /* ast post */
 int ast_post_gpu(struct ast_device *ast);
-u32 ast_mindwm(struct ast_device *ast, u32 r);
-void ast_moutdwm(struct ast_device *ast, u32 r, u32 v);
 
 int ast_vga_output_init(struct ast_device *ast);
 int ast_sil164_output_init(struct ast_device *ast);
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index 21abb6a6d8bcd..d5ed8c5c79253 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -105,8 +105,9 @@ static void ast_crtc_fill_gamma(struct ast_device *ast,
 		/* gamma table is used as color palette */
 		drm_crtc_fill_palette_8(crtc, ast_set_gamma_lut);
 		break;
+	case DRM_FORMAT_XRGB1555:
 	case DRM_FORMAT_RGB565:
-		/* also uses 8-bit gamma ramp on low-color modes */
+		/* also uses 24-bit gamma correction on high-color modes */
 		fallthrough;
 	case DRM_FORMAT_XRGB8888:
 		drm_crtc_fill_gamma_888(crtc, ast_set_gamma_lut);
@@ -129,8 +130,9 @@ static void ast_crtc_load_gamma(struct ast_device *ast,
 		/* gamma table is used as color palette */
 		drm_crtc_load_palette_8(crtc, lut, ast_set_gamma_lut);
 		break;
+	case DRM_FORMAT_XRGB1555:
 	case DRM_FORMAT_RGB565:
-		/* also uses 8-bit gamma ramp on low-color modes */
+		/* also uses 24-bit gamma correction on high-color modes */
 		fallthrough;
 	case DRM_FORMAT_XRGB8888:
 		drm_crtc_load_gamma_888(crtc, lut, ast_set_gamma_lut);
@@ -146,30 +148,35 @@ static void ast_set_vbios_color_reg(struct ast_device *ast,
 				    const struct drm_format_info *format,
 				    const struct ast_vbios_enhtable *vmode)
 {
-	u32 color_index;
+	u8 vgacr8c = 0x00;
+	u8 vgacr92 = 0x00;
 
-	switch (format->cpp[0]) {
-	case 1:
-		color_index = VGAModeIndex - 1;
+	switch (format->format) {
+	case DRM_FORMAT_C8:
+		vgacr8c |= AST_IO_VGACR8C_CUR_MODE_VGA;
+		vgacr92 = 8;
+		break;
+	case DRM_FORMAT_XRGB1555:
+		vgacr8c |= AST_IO_VGACR8C_CUR_MODE_15_BPP;
+		vgacr92 = 15;
 		break;
-	case 2:
-		color_index = HiCModeIndex;
+	case DRM_FORMAT_RGB565:
+		vgacr8c |= AST_IO_VGACR8C_CUR_MODE_16_BPP;
+		vgacr92 = 16;
 		break;
-	case 3:
-	case 4:
-		color_index = TrueCModeIndex;
+	case DRM_FORMAT_XRGB8888:
+		vgacr8c |= AST_IO_VGACR8C_CUR_MODE_32_BPP;
+		vgacr92 = 32;
 		break;
-	default:
-		return;
 	}
 
-	ast_set_index_reg(ast, AST_IO_VGACRI, 0x8c, (u8)((color_index & 0x0f) << 4));
+	ast_set_index_reg(ast, AST_IO_VGACRI, 0x8c, vgacr8c);
 
 	ast_set_index_reg(ast, AST_IO_VGACRI, 0x91, 0x00);
 
 	if (vmode->flags & NewModeInfo) {
-		ast_set_index_reg(ast, AST_IO_VGACRI, 0x91, 0xa8);
-		ast_set_index_reg(ast, AST_IO_VGACRI, 0x92, format->cpp[0] * 8);
+		ast_set_index_reg(ast, AST_IO_VGACRI, 0x91, AST_IO_VGACR91_PASSWORD);
+		ast_set_index_reg(ast, AST_IO_VGACRI, 0x92, vgacr92);
 	}
 }
 
@@ -188,7 +195,7 @@ static void ast_set_vbios_mode_reg(struct ast_device *ast,
 	ast_set_index_reg(ast, AST_IO_VGACRI, 0x91, 0x00);
 
 	if (vmode->flags & NewModeInfo) {
-		ast_set_index_reg(ast, AST_IO_VGACRI, 0x91, 0xa8);
+		ast_set_index_reg(ast, AST_IO_VGACRI, 0x91, AST_IO_VGACR91_PASSWORD);
 		ast_set_index_reg(ast, AST_IO_VGACRI, 0x93, adjusted_mode->clock / 1000);
 		ast_set_index_reg(ast, AST_IO_VGACRI, 0x94, adjusted_mode->crtc_hdisplay);
 		ast_set_index_reg(ast, AST_IO_VGACRI, 0x95, adjusted_mode->crtc_hdisplay >> 8);
@@ -381,30 +388,36 @@ static void ast_set_dclk_reg(struct ast_device *ast,
 static void ast_set_color_reg(struct ast_device *ast,
 			      const struct drm_format_info *format)
 {
-	u8 jregA0 = 0, jregA3 = 0, jregA8 = 0;
+	u8 vgacra0 = 0x00;
+	u8 vgacra3 = 0x00;
+	u8 vgacra8 = 0x00;
 
-	switch (format->cpp[0] * 8) {
-	case 8:
-		jregA0 = 0x70;
-		jregA3 = 0x01;
-		jregA8 = 0x00;
+	vgacra0 |= AST_IO_VGACRA0_MEMORY_CHAIN4_MODE |
+		   AST_IO_VGACRA0_LINEAR_EXT_ACCESS |
+		   AST_IO_VGACRA0_SEGMENTED_EXT_ACCESS;
+
+	switch (format->format) {
+	case DRM_FORMAT_C8:
+		vgacra3 |= AST_IO_VGACRA3_256_COLORS;
+		vgacra8 &= ~AST_IO_VGACRA8_GAMMA_CORRECTION_ENABLED;
 		break;
-	case 15:
-	case 16:
-		jregA0 = 0x70;
-		jregA3 = 0x04;
-		jregA8 = 0x02;
+	case DRM_FORMAT_XRGB1555:
+		vgacra3 |= AST_IO_VGACRA3_15_BPP;
+		vgacra8 |= AST_IO_VGACRA8_GAMMA_CORRECTION_ENABLED;
 		break;
-	case 32:
-		jregA0 = 0x70;
-		jregA3 = 0x08;
-		jregA8 = 0x02;
+	case DRM_FORMAT_RGB565:
+		vgacra3 |= AST_IO_VGACRA3_16_BPP;
+		vgacra8 |= AST_IO_VGACRA8_GAMMA_CORRECTION_ENABLED;
+		break;
+	case DRM_FORMAT_XRGB8888:
+		vgacra3 |= AST_IO_VGACRA3_32_BPP;
+		vgacra8 |= AST_IO_VGACRA8_GAMMA_CORRECTION_ENABLED;
 		break;
 	}
 
-	ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa0, 0x8f, jregA0);
-	ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa3, 0xf0, jregA3);
-	ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa8, 0xfd, jregA8);
+	ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa0, 0x8f, vgacra0);
+	ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa3, 0xf0, vgacra3);
+	ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xa8, 0xfd, vgacra8);
 }
 
 static void ast_set_crtthd_reg(struct ast_device *ast)
@@ -489,11 +502,12 @@ void __iomem *ast_plane_vaddr(struct ast_plane *ast_plane)
 static const uint32_t ast_primary_plane_formats[] = {
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB1555,
 	DRM_FORMAT_C8,
 };
 
 static int ast_primary_plane_helper_atomic_check(struct drm_plane *plane,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = plane->dev;
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
@@ -541,7 +555,7 @@ static void ast_handle_damage(struct ast_plane *ast_plane, struct iosys_map *src
 }
 
 static void ast_primary_plane_helper_atomic_update(struct drm_plane *plane,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = plane->dev;
 	struct ast_device *ast = to_ast_device(dev);
@@ -585,7 +599,7 @@ static void ast_primary_plane_helper_atomic_update(struct drm_plane *plane,
 }
 
 static void ast_primary_plane_helper_atomic_enable(struct drm_plane *plane,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct ast_device *ast = to_ast_device(plane->dev);
 	struct ast_plane *ast_plane = to_ast_plane(plane);
@@ -600,7 +614,7 @@ static void ast_primary_plane_helper_atomic_enable(struct drm_plane *plane,
 }
 
 static void ast_primary_plane_helper_atomic_disable(struct drm_plane *plane,
-						    struct drm_atomic_state *state)
+						    struct drm_atomic_commit *state)
 {
 	/*
 	 * Keep this empty function to avoid calling
@@ -708,7 +722,7 @@ static void ast_crtc_helper_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static int ast_crtc_helper_atomic_check(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
@@ -762,10 +776,10 @@ static int ast_crtc_helper_atomic_check(struct drm_crtc *crtc,
 	case DRM_FORMAT_C8:
 		ast_state->std_table = &vbios_stdtable[VGAModeIndex];
 		break;
+	case DRM_FORMAT_XRGB1555:
 	case DRM_FORMAT_RGB565:
 		ast_state->std_table = &vbios_stdtable[HiCModeIndex];
 		break;
-	case DRM_FORMAT_RGB888:
 	case DRM_FORMAT_XRGB8888:
 		ast_state->std_table = &vbios_stdtable[TrueCModeIndex];
 		break;
@@ -808,7 +822,7 @@ static int ast_crtc_helper_atomic_check(struct drm_crtc *crtc,
 
 static void
 ast_crtc_helper_atomic_flush(struct drm_crtc *crtc,
-			     struct drm_atomic_state *state)
+			     struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -830,7 +844,7 @@ ast_crtc_helper_atomic_flush(struct drm_crtc *crtc,
 	}
 }
 
-static void ast_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state)
+static void ast_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct ast_device *ast = to_ast_device(crtc->dev);
 	u8 vgacr17 = 0x00;
@@ -843,7 +857,7 @@ static void ast_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atom
 	ast_set_index_reg_mask(ast, AST_IO_VGACRI, 0xb6, 0xfc, vgacrb6);
 }
 
-static void ast_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state)
+static void ast_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state, crtc);
 	struct ast_device *ast = to_ast_device(crtc->dev);
@@ -952,7 +966,7 @@ static int ast_crtc_init(struct ast_device *ast)
  * Mode config
  */
 
-static void ast_mode_config_helper_atomic_commit_tail(struct drm_atomic_state *state)
+static void ast_mode_config_helper_atomic_commit_tail(struct drm_atomic_commit *state)
 {
 	struct ast_device *ast = to_ast_device(state->dev);
 
diff --git a/drivers/gpu/drm/ast/ast_post.c b/drivers/gpu/drm/ast/ast_post.c
index b72914dbed38d..297230f1efafe 100644
--- a/drivers/gpu/drm/ast/ast_post.c
+++ b/drivers/gpu/drm/ast/ast_post.c
@@ -34,44 +34,6 @@
 #include "ast_drv.h"
 #include "ast_post.h"
 
-u32 __ast_mindwm(void __iomem *regs, u32 r)
-{
-	u32 data;
-
-	__ast_write32(regs, 0xf004, r & 0xffff0000);
-	__ast_write32(regs, 0xf000, 0x1);
-
-	do {
-		data = __ast_read32(regs, 0xf004) & 0xffff0000;
-	} while (data != (r & 0xffff0000));
-
-	return __ast_read32(regs, 0x10000 + (r & 0x0000ffff));
-}
-
-void __ast_moutdwm(void __iomem *regs, u32 r, u32 v)
-{
-	u32 data;
-
-	__ast_write32(regs, 0xf004, r & 0xffff0000);
-	__ast_write32(regs, 0xf000, 0x1);
-
-	do {
-		data = __ast_read32(regs, 0xf004) & 0xffff0000;
-	} while (data != (r & 0xffff0000));
-
-	__ast_write32(regs, 0x10000 + (r & 0x0000ffff), v);
-}
-
-u32 ast_mindwm(struct ast_device *ast, u32 r)
-{
-	return __ast_mindwm(ast->regs, r);
-}
-
-void ast_moutdwm(struct ast_device *ast, u32 r, u32 v)
-{
-	__ast_moutdwm(ast->regs, r, v);
-}
-
 int ast_post_gpu(struct ast_device *ast)
 {
 	int ret;
@@ -107,19 +69,19 @@ bool mmc_test(struct ast_device *ast, u32 datagen, u8 test_ctl)
 {
 	u32 data, timeout;
 
-	ast_moutdwm(ast, 0x1e6e0070, 0x00000000);
-	ast_moutdwm(ast, 0x1e6e0070, (datagen << 3) | test_ctl);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
+	ast_moutdwm(ast, AST_REG_MCR70, (datagen << 3) | test_ctl);
 	timeout = 0;
 	do {
-		data = ast_mindwm(ast, 0x1e6e0070) & 0x3000;
+		data = ast_mindwm(ast, AST_REG_MCR70) & 0x3000;
 		if (data & 0x2000)
 			return false;
 		if (++timeout > TIMEOUT) {
-			ast_moutdwm(ast, 0x1e6e0070, 0x00000000);
+			ast_moutdwm(ast, AST_REG_MCR70, 0x00000000);
 			return false;
 		}
 	} while (!data);
-	ast_moutdwm(ast, 0x1e6e0070, 0x0);
+	ast_moutdwm(ast, AST_REG_MCR70, 0x0);
 	return true;
 }
 
diff --git a/drivers/gpu/drm/ast/ast_post.h b/drivers/gpu/drm/ast/ast_post.h
index aa5d247bebe8d..41b2db08a8709 100644
--- a/drivers/gpu/drm/ast/ast_post.h
+++ b/drivers/gpu/drm/ast/ast_post.h
@@ -10,13 +10,10 @@ struct ast_device;
 
 /* DRAM timing tables */
 struct ast_dramstruct {
-	u16 index;
+	u32 index;
 	u32 data;
 };
 
-/* hardware fields */
-#define __AST_DRAMSTRUCT_DRAM_TYPE      0x0004
-
 /* control commands */
 #define __AST_DRAMSTRUCT_UDELAY         0xff00
 #define __AST_DRAMSTRUCT_INVALID        0xffff
@@ -24,20 +21,21 @@ struct ast_dramstruct {
 #define __AST_DRAMSTRUCT_INDEX(_name) \
 	(__AST_DRAMSTRUCT_ ## _name)
 
-#define AST_DRAMSTRUCT_INIT(_name, _value) \
-	{ __AST_DRAMSTRUCT_INDEX(_name), (_value) }
+#define __AST_DRAMSTRUCT_INIT(_index, _value) \
+	{ (_index), (_value) }
 
+#define AST_DRAMSTRUCT_REG(_reg, _value) \
+	__AST_DRAMSTRUCT_INIT(_reg, _value)
 #define AST_DRAMSTRUCT_UDELAY(_usecs) \
-	AST_DRAMSTRUCT_INIT(UDELAY, _usecs)
+	__AST_DRAMSTRUCT_INIT(__AST_DRAMSTRUCT_UDELAY, _usecs)
 #define AST_DRAMSTRUCT_INVALID \
-	AST_DRAMSTRUCT_INIT(INVALID, U32_MAX)
+	__AST_DRAMSTRUCT_INIT(__AST_DRAMSTRUCT_INVALID, U32_MAX)
 
+#define AST_DRAMSTRUCT_IS_REG(_entry, _reg) \
+	((_entry)->index == (_reg))
 #define AST_DRAMSTRUCT_IS(_entry, _name) \
 	((_entry)->index == __AST_DRAMSTRUCT_INDEX(_name))
 
-u32 __ast_mindwm(void __iomem *regs, u32 r);
-void __ast_moutdwm(void __iomem *regs, u32 r, u32 v);
-
 bool mmc_test(struct ast_device *ast, u32 datagen, u8 test_ctl);
 bool mmc_test_burst(struct ast_device *ast, u32 datagen);
 
diff --git a/drivers/gpu/drm/ast/ast_reg.h b/drivers/gpu/drm/ast/ast_reg.h
index 30578e3b07e43..b808b0a02ba81 100644
--- a/drivers/gpu/drm/ast/ast_reg.h
+++ b/drivers/gpu/drm/ast/ast_reg.h
@@ -28,18 +28,49 @@
 #define AST_IO_VGAPDR		        (0x49)
 #define AST_IO_VGAGRI			(0x4E)
 
-#define AST_IO_VGACRI			(0x54)
-#define AST_IO_VGACR17_SYNC_ENABLE	BIT(7) /* called "Hardware reset" in docs */
-#define AST_IO_VGACR80_PASSWORD		(0xa8)
-#define AST_IO_VGACR99_VGAMEM_RSRV_MASK	GENMASK(1, 0)
-#define AST_IO_VGACRA1_VGAIO_DISABLED	BIT(1)
-#define AST_IO_VGACRA1_MMIO_ENABLED	BIT(2)
-#define AST_IO_VGACRA3_DVO_ENABLED	BIT(7)
-#define AST_IO_VGACRAA_VGAMEM_SIZE_MASK	GENMASK(1, 0)
-#define AST_IO_VGACRB6_HSYNC_OFF	BIT(0)
-#define AST_IO_VGACRB6_VSYNC_OFF	BIT(1)
-#define AST_IO_VGACRCB_HWC_16BPP	BIT(0) /* set: ARGB4444, cleared: 2bpp palette */
-#define AST_IO_VGACRCB_HWC_ENABLED	BIT(1)
+#define AST_IO_VGACRI				(0x54)
+#define AST_IO_VGACR17_SYNC_ENABLE		BIT(7) /* called "Hardware reset" in docs */
+#define AST_IO_VGACR80_PASSWORD			(0xa8)
+
+#define AST_IO_VGACR8C_NEW_MODE_MASK		GENMASK(3, 0)
+#define AST_IO_VGACR8C_NEW_MODE_EGA		(0x00)
+#define AST_IO_VGACR8C_NEW_MODE_VGA		(0x01)
+#define AST_IO_VGACR8C_NEW_MODE_15_BPP		(0x02)
+#define AST_IO_VGACR8C_NEW_MODE_16_BPP		(0x03)
+#define AST_IO_VGACR8C_NEW_MODE_32_BPP		(0x04)
+#define AST_IO_VGACR8C_NEW_MODE_CGA		(0x0f)
+#define AST_IO_VGACR8C_NEW_MODE_TEXT		(0x0e)
+#define AST_IO_VGACR8C_CUR_MODE_MASK		GENMASK(7, 4)
+#define AST_IO_VGACR8C_CUR_MODE_EGA		(0x00)
+#define AST_IO_VGACR8C_CUR_MODE_VGA		(0x10)
+#define AST_IO_VGACR8C_CUR_MODE_15_BPP		(0x20)
+#define AST_IO_VGACR8C_CUR_MODE_16_BPP		(0x30)
+#define AST_IO_VGACR8C_CUR_MODE_32_BPP		(0x40)
+#define AST_IO_VGACR8C_CUR_MODE_CGA		(0xf0)
+#define AST_IO_VGACR8C_CUR_MODE_TEXT		(0xe0)
+
+#define AST_IO_VGACR91_PASSWORD			(0xa8)
+#define AST_IO_VGACR99_VGAMEM_RSRV_MASK		GENMASK(1, 0)
+
+#define AST_IO_VGACRA0_MEMORY_CHAIN4_MODE	BIT(6)
+#define AST_IO_VGACRA0_LINEAR_EXT_ACCESS	BIT(5)
+#define AST_IO_VGACRA0_SEGMENTED_EXT_ACCESS	BIT(4)
+
+#define AST_IO_VGACRA1_VGAIO_DISABLED		BIT(1)
+#define AST_IO_VGACRA1_MMIO_ENABLED		BIT(2)
+
+#define AST_IO_VGACRA3_DVO_ENABLED		BIT(7)
+#define AST_IO_VGACRA3_32_BPP			BIT(3)
+#define AST_IO_VGACRA3_16_BPP			BIT(2)
+#define AST_IO_VGACRA3_15_BPP			BIT(1)
+#define AST_IO_VGACRA3_256_COLORS		BIT(0)
+
+#define AST_IO_VGACRA8_GAMMA_CORRECTION_ENABLED	BIT(1)
+#define AST_IO_VGACRAA_VGAMEM_SIZE_MASK		GENMASK(1, 0)
+#define AST_IO_VGACRB6_HSYNC_OFF		BIT(0)
+#define AST_IO_VGACRB6_VSYNC_OFF		BIT(1)
+#define AST_IO_VGACRCB_HWC_16BPP		BIT(0) /* set: ARGB4444, cleared: 2bpp palette */
+#define AST_IO_VGACRCB_HWC_ENABLED		BIT(1)
 
 /* mirrors SCU100[7:0] */
 #define AST_IO_VGACRD0_VRAM_INIT_STATUS_MASK	GENMASK(7, 6)
@@ -75,4 +106,160 @@
 #define AST_IO_VGAIR1_R			(0x5A)
 #define AST_IO_VGAIR1_VREFRESH		BIT(3)
 
+/*
+ * P-Bus to AHB Bridge (0x00000000 - 0x0001ffff)
+ */
+
+#define AST_REG_P2A_BASE			(0x00000000)
+#define AST_REG_P2A(__offset)			(AST_REG_P2A_BASE + (__offset))
+#define AST_REG_P2A_ADDR(__addr)		AST_REG_P2A(0x10000 + ((__addr) & GENMASK(15, 0)))
+#define AST_REG_P2A00				AST_REG_P2A(0xf000)
+#define AST_REG_P2A00_PROTECTION_KEY		(0x01)
+#define AST_REG_P2A04				AST_REG_P2A(0xf004)
+#define AST_REG_P2A04_BASE_MASK			GENMASK(31, 16)
+
+/*
+ * AHB Controller (0x1e600000 - 0x1e61ffff)
+ */
+
+#define AST_REG_AHBC_BASE			(0x1e600000)
+#define AST_REG_AHBC(__offset)			(AST_REG_AHBC_BASE + (__offset))
+#define AST_REG_AHBC00				AST_REG_AHBC(0x00)
+#define AST_REG_AHBC00_PROTECT_KEY		(0xaeed1a03)
+#define AST_REG_AHBC84				AST_REG_AHBC(0x84)
+#define AST_REG_AHBC88				AST_REG_AHBC(0x88)
+
+/*
+ * SDRAM Memory Controller (0x1e6e0000 - 0x1e6e0fff)
+ */
+
+#define AST_REG_MCR_BASE			(0x1e6e0000)
+#define AST_REG_MCR(__offset)			(AST_REG_MCR_BASE + (__offset))
+#define AST_REG_MCR00				AST_REG_MCR(0x00)
+#define AST_REG_MCR00_PROTECTION_KEY		(0xfc600309)
+#define AST_REG_MCR04				AST_REG_MCR(0x04)
+#define AST_REG_MCR08				AST_REG_MCR(0x08)
+#define AST_REG_MCR0C				AST_REG_MCR(0x0c)
+#define AST_REG_MCR10				AST_REG_MCR(0x10)
+#define AST_REG_MCR14				AST_REG_MCR(0x14)
+#define AST_REG_MCR18				AST_REG_MCR(0x18)
+#define AST_REG_MCR1C				AST_REG_MCR(0x1c)
+#define AST_REG_MCR20				AST_REG_MCR(0x20)
+#define AST_REG_MCR24				AST_REG_MCR(0x24)
+#define AST_REG_MCR28				AST_REG_MCR(0x28)
+#define AST_REG_MCR2C				AST_REG_MCR(0x2C)
+#define AST_REG_MCR30				AST_REG_MCR(0x30)
+#define AST_REG_MCR34				AST_REG_MCR(0x34)
+#define AST_REG_MCR38				AST_REG_MCR(0x38)
+#define AST_REG_MCR3C				AST_REG_MCR(0x3c)
+#define AST_REG_MCR40				AST_REG_MCR(0x40)
+#define AST_REG_MCR44				AST_REG_MCR(0x44)
+#define AST_REG_MCR48				AST_REG_MCR(0x48)
+#define AST_REG_MCR4C				AST_REG_MCR(0x4C)
+#define AST_REG_MCR50				AST_REG_MCR(0x50)
+#define AST_REG_MCR54				AST_REG_MCR(0x54)
+#define AST_REG_MCR58				AST_REG_MCR(0x58)
+#define AST_REG_MCR5C				AST_REG_MCR(0x5c)
+#define AST_REG_MCR60				AST_REG_MCR(0x60)
+#define AST_REG_MCR64				AST_REG_MCR(0x64)
+#define AST_REG_MCR68				AST_REG_MCR(0x68)
+#define AST_REG_MCR6C				AST_REG_MCR(0x6c)
+#define AST_REG_MCR70				AST_REG_MCR(0x70)
+#define AST_REG_MCR74				AST_REG_MCR(0x74)
+#define AST_REG_MCR78				AST_REG_MCR(0x78)
+#define AST_REG_MCR7C				AST_REG_MCR(0x7c)
+#define AST_REG_MCR80				AST_REG_MCR(0x80)
+#define AST_REG_MCR84				AST_REG_MCR(0x84)
+#define AST_REG_MCR88				AST_REG_MCR(0x88)
+#define AST_REG_MCR8C				AST_REG_MCR(0x8c)
+#define AST_REG_MCR100				AST_REG_MCR(0x100)
+#define AST_REG_MCR108				AST_REG_MCR(0x108)
+#define AST_REG_MCR120				AST_REG_MCR(0x120)
+#define AST_REG_MCR140				AST_REG_MCR(0x140)
+#define AST_REG_MCR200				AST_REG_MCR(0x200)
+#define AST_REG_MCR204				AST_REG_MCR(0x204)
+#define AST_REG_MCR208				AST_REG_MCR(0x208)
+#define AST_REG_MCR20C				AST_REG_MCR(0x20C)
+#define AST_REG_MCR210				AST_REG_MCR(0x210)
+#define AST_REG_MCR214				AST_REG_MCR(0x214)
+#define AST_REG_MCR218				AST_REG_MCR(0x218)
+#define AST_REG_MCR220				AST_REG_MCR(0x220)
+#define AST_REG_MCR228				AST_REG_MCR(0x228)
+#define AST_REG_MCR230				AST_REG_MCR(0x230)
+#define AST_REG_MCR2A8				AST_REG_MCR(0x2a8)
+#define AST_REG_MCR2B0				AST_REG_MCR(0x2b0)
+#define AST_REG_MCR240				AST_REG_MCR(0x240)
+#define AST_REG_MCR244				AST_REG_MCR(0x244)
+#define AST_REG_MCR248				AST_REG_MCR(0x248)
+#define AST_REG_MCR24C				AST_REG_MCR(0x24c)
+#define AST_REG_MCR290				AST_REG_MCR(0x290)
+#define AST_REG_MCR2C0				AST_REG_MCR(0x2c0)
+#define AST_REG_MCR2C4				AST_REG_MCR(0x2c4)
+#define AST_REG_MCR2C8				AST_REG_MCR(0x2c8)
+#define AST_REG_MCR2CC				AST_REG_MCR(0x2cc)
+#define AST_REG_MCR2E0				AST_REG_MCR(0x2e0)
+#define AST_REG_MCR2E4				AST_REG_MCR(0x2e4)
+#define AST_REG_MCR2E8				AST_REG_MCR(0x2e8)
+#define AST_REG_MCR2EC				AST_REG_MCR(0x2ec)
+#define AST_REG_MCR2F0				AST_REG_MCR(0x2f0)
+#define AST_REG_MCR2F4				AST_REG_MCR(0x2f4)
+#define AST_REG_MCR2F8				AST_REG_MCR(0x2f8)
+#define AST_REG_MCR300				AST_REG_MCR(0x300)
+#define AST_REG_MCR3D0				AST_REG_MCR(0x3d0)
+
+/*
+ * System Control Unit (0x1e6e2000 - 0x1e6e2fff)
+ */
+
+#define AST_REG_SCU_BASE			(0x1e6e2000)
+#define AST_REG_SCU(__offset)			(AST_REG_SCU_BASE + (__offset))
+#define AST_REG_SCU000				AST_REG_SCU(0x000)
+#define AST_REG_SCU000_PROTECTION_KEY		(0x1688a8a8)
+#define AST_REG_SCU008				AST_REG_SCU(0x008)
+#define AST_REG_SCU00C				AST_REG_SCU(0x00c)
+#define AST_REG_SCU020				AST_REG_SCU(0x020)
+#define AST_REG_SCU02C				AST_REG_SCU(0x02c)
+#define AST_REG_SCU040				AST_REG_SCU(0x040)
+#define AST_REG_SCU070				AST_REG_SCU(0x070)
+#define AST_REG_SCU07C				AST_REG_SCU(0x07c)
+#define AST_REG_SCU07C_CHIP_BONDING_MASK	GENMASK(15, 8)
+#define AST_REG_SCU084				AST_REG_SCU(0x084)
+#define AST_REG_SCU088				AST_REG_SCU(0x088)
+#define AST_REG_SCU08C				AST_REG_SCU(0x08c)
+#define AST_REG_SCU090				AST_REG_SCU(0x090)
+#define AST_REG_SCU094				AST_REG_SCU(0x094)
+#define AST_REG_SCU0A4				AST_REG_SCU(0x0a4)
+#define AST_REG_SCU0A8				AST_REG_SCU(0x0a8)
+#define AST_REG_SCU100				AST_REG_SCU(0x100)
+#define AST_REG_SCU104				AST_REG_SCU(0x104)
+#define AST_REG_SCU160				AST_REG_SCU(0x160)
+
+/*
+ * AHB-to-P Bus Bridge (0x1e720000 - 0x1e73ffff)
+ */
+
+#define AST_REG_A2P_BASE			(0x1e720000)
+#define AST_REG_A2P(__offset)			(AST_REG_A2P_BASE + (__offset))
+#define AST_REG_A2P58				AST_REG_A2P(0x58)
+
+/*
+ * Watchdog timer (0x1e785000 - 0x1e785fff)
+ */
+
+#define AST_REG_WDT_BASE(__n)			(0x1e785000 + (__n) * 0x40)
+#define AST_REG_WDT(__n, __offset)		(AST_REG_WDT_BASE((__n)) + (__offset))
+#define AST_REG_WDT04(__n)			AST_REG_WDT((__n), 0x04)
+#define AST_REG_WDT08(__n)			AST_REG_WDT((__n), 0x08)
+#define AST_REG_WDT0C(__n)			AST_REG_WDT((__n), 0x0c)
+#define AST_REG_WDT14(__n)			AST_REG_WDT((__n), 0x14)
+#define AST_REG_WDT1C(__n)			AST_REG_WDT((__n), 0x1c)
+#define AST_REG_WDT2C(__n)			AST_REG_WDT((__n), 0x2c)
+
+/*
+ * SDRAM (0x80000000 - 0xffffffff)
+ */
+
+#define AST_SDRAM_BASE				(0x80000000)
+#define AST_SDRAM(__offset)			(AST_SDRAM_BASE + (__offset))
+
 #endif
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
index d8cf3d231920c..9dbac2def333e 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
@@ -200,7 +200,7 @@ atmel_hlcdc_crtc_mode_valid(struct drm_crtc *c,
 }
 
 static void atmel_hlcdc_crtc_atomic_disable(struct drm_crtc *c,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = c->dev;
 	struct atmel_hlcdc_crtc *crtc = drm_crtc_to_atmel_hlcdc_crtc(c);
@@ -252,7 +252,7 @@ static void atmel_hlcdc_crtc_atomic_disable(struct drm_crtc *c,
 }
 
 static void atmel_hlcdc_crtc_atomic_enable(struct drm_crtc *c,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = c->dev;
 	struct atmel_hlcdc_crtc *crtc = drm_crtc_to_atmel_hlcdc_crtc(c);
@@ -458,7 +458,7 @@ static int atmel_hlcdc_crtc_select_output_mode(struct drm_crtc_state *state)
 }
 
 static int atmel_hlcdc_crtc_atomic_check(struct drm_crtc *c,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *s = drm_atomic_get_new_crtc_state(state, c);
 	int ret;
@@ -475,13 +475,13 @@ static int atmel_hlcdc_crtc_atomic_check(struct drm_crtc *c,
 }
 
 static void atmel_hlcdc_crtc_atomic_begin(struct drm_crtc *c,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_on(c);
 }
 
 static void atmel_hlcdc_crtc_atomic_flush(struct drm_crtc *c,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct atmel_hlcdc_crtc *crtc = drm_crtc_to_atmel_hlcdc_crtc(c);
 	unsigned long flags;
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index 97482fc82ec2f..2f41bce574dc5 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -983,7 +983,7 @@ static int atmel_hlcdc_dc_drm_suspend(struct device *dev)
 	struct drm_device *drm_dev = dev_get_drvdata(dev);
 	struct atmel_hlcdc_dc *dc = drm_dev->dev_private;
 	struct regmap *regmap = dc->hlcdc->regmap;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 
 	state = drm_atomic_helper_suspend(drm_dev);
 	if (IS_ERR(state))
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
index 26b26185cf34a..390dfbf3c77dd 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.h
@@ -354,7 +354,7 @@ struct atmel_hlcdc_dc {
 	struct atmel_hlcdc_layer *layers[ATMEL_HLCDC_MAX_LAYERS];
 	struct {
 		u32 imr;
-		struct drm_atomic_state *state;
+		struct drm_atomic_commit *state;
 	} suspend;
 };
 
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
index bccd1564216e6..f17a832351e9d 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_plane.c
@@ -718,7 +718,7 @@ atmel_hlcdc_plane_update_disc_area(struct atmel_hlcdc_plane *plane,
 }
 
 static int atmel_hlcdc_plane_atomic_check(struct drm_plane *p,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *s = drm_atomic_get_new_plane_state(state, p);
 	struct atmel_hlcdc_plane *plane = drm_plane_to_atmel_hlcdc_plane(p);
@@ -869,7 +869,7 @@ static void atmel_xlcdc_atomic_disable(struct atmel_hlcdc_plane *plane,
 }
 
 static void atmel_hlcdc_plane_atomic_disable(struct drm_plane *p,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct atmel_hlcdc_plane *plane = drm_plane_to_atmel_hlcdc_plane(p);
 	struct atmel_hlcdc_dc *dc = plane->base.dev->dev_private;
@@ -919,7 +919,7 @@ static void atmel_xlcdc_atomic_update(struct atmel_hlcdc_plane *plane,
 }
 
 static void atmel_hlcdc_plane_atomic_update(struct drm_plane *p,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_s = drm_atomic_get_new_plane_state(state,
 								       p);
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index c3209b0f46786..f81b566c82a18 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -262,6 +262,16 @@ config DRM_NXP_PTN3460
 	help
 	  NXP PTN3460 eDP-LVDS bridge chip driver.
 
+config DRM_OF_DISPLAY_MODE_BRIDGE
+	tristate
+	depends on DRM_BRIDGE && OF
+	help
+	  This is a DRM bridge implementation that uses of_get_drm_display_mode
+	  to acquire display mode.
+
+	  It exists for compatibility with legacy display mode parsing, in order
+	  to conform to the panel-bridge framework.
+
 config DRM_PARADE_PS8622
 	tristate "Parade eDP/LVDS bridge"
 	depends on OF
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
index beab5b695a6e1..15cc821d85b7e 100644
--- a/drivers/gpu/drm/bridge/Makefile
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -22,6 +22,7 @@ obj-$(CONFIG_DRM_LVDS_CODEC) += lvds-codec.o
 obj-$(CONFIG_DRM_MEGACHIPS_STDPXXXX_GE_B850V3_FW) += megachips-stdpxxxx-ge-b850v3-fw.o
 obj-$(CONFIG_DRM_MICROCHIP_LVDS_SERIALIZER) += microchip-lvds.o
 obj-$(CONFIG_DRM_NXP_PTN3460) += nxp-ptn3460.o
+obj-$(CONFIG_DRM_OF_DISPLAY_MODE_BRIDGE) += of-display-mode-bridge.o
 obj-$(CONFIG_DRM_PARADE_PS8622) += parade-ps8622.o
 obj-$(CONFIG_DRM_PARADE_PS8640) += parade-ps8640.o
 obj-$(CONFIG_DRM_SAMSUNG_DSIM) += samsung-dsim.o
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511.h b/drivers/gpu/drm/bridge/adv7511/adv7511.h
index 8be7266fd4f49..12c95198d9a41 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511.h
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511.h
@@ -354,7 +354,6 @@ struct adv7511 {
 	enum drm_connector_status status;
 	bool powered;
 
-	struct drm_bridge *next_bridge;
 	struct drm_display_mode curr_mode;
 
 	unsigned int f_tmds;
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index 6bd76c1fb0071..02f8f7e78a163 100644
--- a/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -768,8 +768,6 @@ static int adv7511_connector_init(struct adv7511 *adv)
 		return PTR_ERR(connector);
 	}
 
-	drm_connector_attach_encoder(connector, bridge->encoder);
-
 	return 0;
 }
 
@@ -783,7 +781,7 @@ static const struct adv7511 *bridge_to_adv7511_const(const struct drm_bridge *br
 }
 
 static void adv7511_bridge_atomic_enable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct adv7511 *adv = bridge_to_adv7511(bridge);
 	struct drm_connector *connector;
@@ -812,7 +810,7 @@ static void adv7511_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void adv7511_bridge_atomic_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct adv7511 *adv = bridge_to_adv7511(bridge);
 
@@ -851,8 +849,8 @@ static int adv7511_bridge_attach(struct drm_bridge *bridge,
 	struct adv7511 *adv = bridge_to_adv7511(bridge);
 	int ret = 0;
 
-	if (adv->next_bridge) {
-		ret = drm_bridge_attach(encoder, adv->next_bridge, bridge,
+	if (adv->bridge.next_bridge) {
+		ret = drm_bridge_attach(encoder, adv->bridge.next_bridge, bridge,
 					flags | DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 		if (ret)
 			return ret;
@@ -1251,10 +1249,13 @@ static int adv7511_probe(struct i2c_client *i2c)
 
 	memset(&link_config, 0, sizeof(link_config));
 
-	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, -1, NULL,
-					  &adv7511->next_bridge);
-	if (ret && ret != -ENODEV)
-		return ret;
+	adv7511->bridge.next_bridge = of_drm_get_bridge_by_endpoint(dev->of_node, 1, -1);
+	if (IS_ERR(adv7511->bridge.next_bridge)) {
+		if (PTR_ERR(adv7511->bridge.next_bridge) == -ENODEV)
+			adv7511->bridge.next_bridge = NULL;
+		else
+			return PTR_ERR(adv7511->bridge.next_bridge);
+	}
 
 	if (adv7511->info->link_config)
 		ret = adv7511_parse_dt(dev->of_node, &link_config);
diff --git a/drivers/gpu/drm/bridge/analogix/Kconfig b/drivers/gpu/drm/bridge/analogix/Kconfig
index 03dc7ffe824a9..57bb2daa5aaf4 100644
--- a/drivers/gpu/drm/bridge/analogix/Kconfig
+++ b/drivers/gpu/drm/bridge/analogix/Kconfig
@@ -15,6 +15,7 @@ config DRM_ANALOGIX_ANX6345
 
 config DRM_ANALOGIX_ANX78XX
 	tristate "Analogix ANX78XX bridge"
+	depends on OF
 	select DRM_ANALOGIX_DP
 	select DRM_DISPLAY_DP_HELPER
 	select DRM_DISPLAY_HELPER
@@ -29,6 +30,8 @@ config DRM_ANALOGIX_ANX78XX
 config DRM_ANALOGIX_DP
 	tristate
 	depends on DRM
+	depends on OF
+	select DRM_DISPLAY_DP_AUX_BUS
 
 config DRM_ANALOGIX_ANX7625
 	tristate "Analogix Anx7625 MIPI to DP interface support"
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index 8dee5f2fbde51..8cf6b73bceac0 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -6,6 +6,7 @@
 * Author: Jingoo Han <jg1.han@samsung.com>
 */
 
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/component.h>
 #include <linux/err.h>
@@ -20,12 +21,14 @@
 #include <linux/platform_device.h>
 
 #include <drm/bridge/analogix_dp.h>
+#include <drm/display/drm_dp_aux_bus.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_device.h>
 #include <drm/drm_edid.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -747,9 +750,6 @@ static int analogix_dp_commit(struct analogix_dp_device *dp)
 {
 	int ret;
 
-	/* Keep the panel disabled while we configure video */
-	drm_panel_disable(dp->plat_data->panel);
-
 	ret = analogix_dp_train_link(dp);
 	if (ret) {
 		dev_err(dp->dev, "unable to do link train, ret=%d\n", ret);
@@ -769,9 +769,6 @@ static int analogix_dp_commit(struct analogix_dp_device *dp)
 		return ret;
 	}
 
-	/* Safe to enable the panel now */
-	drm_panel_enable(dp->plat_data->panel);
-
 	/* Check whether panel supports fast training */
 	ret = analogix_dp_fast_link_train_detection(dp);
 	if (ret)
@@ -856,79 +853,47 @@ static int analogix_dp_disable_psr(struct analogix_dp_device *dp)
 	return analogix_dp_send_psr_spd(dp, &psr_vsc, true);
 }
 
-static int analogix_dp_get_modes(struct drm_connector *connector)
-{
-	struct analogix_dp_device *dp = to_dp(connector);
-	const struct drm_edid *drm_edid;
-	int num_modes = 0;
-
-	if (dp->plat_data->panel) {
-		num_modes += drm_panel_get_modes(dp->plat_data->panel, connector);
-	} else {
-		drm_edid = drm_edid_read_ddc(connector, &dp->aux.ddc);
-
-		drm_edid_connector_update(&dp->connector, drm_edid);
-
-		if (drm_edid) {
-			num_modes += drm_edid_connector_add_modes(&dp->connector);
-			drm_edid_free(drm_edid);
-		}
-	}
-
-	if (dp->plat_data->get_modes)
-		num_modes += dp->plat_data->get_modes(dp->plat_data, connector);
-
-	return num_modes;
-}
-
-static struct drm_encoder *
-analogix_dp_best_encoder(struct drm_connector *connector)
+static const struct drm_edid *analogix_dp_bridge_edid_read(struct drm_bridge *bridge,
+							   struct drm_connector *connector)
 {
-	struct analogix_dp_device *dp = to_dp(connector);
+	struct analogix_dp_device *dp = to_dp(bridge);
 
-	return dp->encoder;
+	return drm_edid_read_ddc(connector, &dp->aux.ddc);
 }
 
-
-static int analogix_dp_atomic_check(struct drm_connector *connector,
-				    struct drm_atomic_state *state)
+static int analogix_dp_bridge_atomic_check(struct drm_bridge *bridge,
+					   struct drm_bridge_state *bridge_state,
+					   struct drm_crtc_state *crtc_state,
+					   struct drm_connector_state *conn_state)
 {
-	struct analogix_dp_device *dp = to_dp(connector);
-	struct drm_connector_state *conn_state;
-	struct drm_crtc_state *crtc_state;
-
-	conn_state = drm_atomic_get_new_connector_state(state, connector);
-	if (WARN_ON(!conn_state))
-		return -ENODEV;
+	struct analogix_dp_device *dp = to_dp(bridge);
+	struct drm_display_info *di = &conn_state->connector->display_info;
+	u32 mask = BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) | BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422);
+
+	if (is_rockchip(dp->plat_data->dev_type)) {
+		if ((di->color_formats & mask)) {
+			DRM_DEBUG_KMS("Swapping display color format from YUV to RGB\n");
+			di->color_formats &= ~mask;
+			di->color_formats |= BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444);
+			di->bpc = 8;
+		}
+	}
 
 	conn_state->self_refresh_aware = true;
 
-	if (!conn_state->crtc)
-		return 0;
-
-	crtc_state = drm_atomic_get_new_crtc_state(state, conn_state->crtc);
-	if (!crtc_state)
-		return 0;
-
 	if (crtc_state->self_refresh_active && !dp->psr_supported)
 		return -EINVAL;
 
 	return 0;
 }
 
-static const struct drm_connector_helper_funcs analogix_dp_connector_helper_funcs = {
-	.get_modes = analogix_dp_get_modes,
-	.best_encoder = analogix_dp_best_encoder,
-	.atomic_check = analogix_dp_atomic_check,
-};
-
 static enum drm_connector_status
-analogix_dp_detect(struct drm_connector *connector, bool force)
+analogix_dp_bridge_detect(struct drm_bridge *bridge, struct drm_connector *connector)
 {
-	struct analogix_dp_device *dp = to_dp(connector);
+	struct analogix_dp_device *dp = to_dp(bridge);
 	enum drm_connector_status status = connector_status_disconnected;
 
-	if (dp->plat_data->panel)
+	if (dp->plat_data->next_bridge)
 		return connector_status_connected;
 
 	if (!analogix_dp_detect_hpd(dp))
@@ -937,55 +902,23 @@ analogix_dp_detect(struct drm_connector *connector, bool force)
 	return status;
 }
 
-static const struct drm_connector_funcs analogix_dp_connector_funcs = {
-	.fill_modes = drm_helper_probe_single_connector_modes,
-	.detect = analogix_dp_detect,
-	.destroy = drm_connector_cleanup,
-	.reset = drm_atomic_helper_connector_reset,
-	.atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-};
-
 static int analogix_dp_bridge_attach(struct drm_bridge *bridge,
 				     struct drm_encoder *encoder,
 				     enum drm_bridge_attach_flags flags)
 {
 	struct analogix_dp_device *dp = to_dp(bridge);
-	struct drm_connector *connector = NULL;
 	int ret = 0;
 
-	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) {
-		DRM_ERROR("Fix bridge driver to make connector optional!");
+	if (!(flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)) {
+		DRM_ERROR("Unsupported connector creation\n");
 		return -EINVAL;
 	}
 
-	if (!dp->plat_data->skip_connector) {
-		connector = &dp->connector;
-		connector->polled = DRM_CONNECTOR_POLL_HPD;
-
-		ret = drm_connector_init(dp->drm_dev, connector,
-					 &analogix_dp_connector_funcs,
-					 DRM_MODE_CONNECTOR_eDP);
+	if (dp->plat_data->next_bridge) {
+		ret = drm_bridge_attach(dp->encoder, dp->plat_data->next_bridge, bridge,
+					DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 		if (ret) {
-			DRM_ERROR("Failed to initialize connector with drm\n");
-			return ret;
-		}
-
-		drm_connector_helper_add(connector,
-					 &analogix_dp_connector_helper_funcs);
-		drm_connector_attach_encoder(connector, encoder);
-	}
-
-	/*
-	 * NOTE: the connector registration is implemented in analogix
-	 * platform driver, that to say connector would be exist after
-	 * plat_data->attch return, that's why we record the connector
-	 * point after plat attached.
-	 */
-	if (dp->plat_data->attach) {
-		ret = dp->plat_data->attach(dp->plat_data, bridge, connector);
-		if (ret) {
-			DRM_ERROR("Failed at platform attach func\n");
+			dev_err(dp->dev, "failed to attach following panel or bridge (%d)\n", ret);
 			return ret;
 		}
 	}
@@ -995,7 +928,7 @@ static int analogix_dp_bridge_attach(struct drm_bridge *bridge,
 
 static
 struct drm_crtc *analogix_dp_get_old_crtc(struct analogix_dp_device *dp,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_encoder *encoder = dp->encoder;
 	struct drm_connector *connector;
@@ -1014,7 +947,7 @@ struct drm_crtc *analogix_dp_get_old_crtc(struct analogix_dp_device *dp,
 
 static
 struct drm_crtc *analogix_dp_get_new_crtc(struct analogix_dp_device *dp,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_encoder *encoder = dp->encoder;
 	struct drm_connector *connector;
@@ -1032,7 +965,7 @@ struct drm_crtc *analogix_dp_get_new_crtc(struct analogix_dp_device *dp,
 }
 
 static void analogix_dp_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-						 struct drm_atomic_state *old_state)
+						 struct drm_atomic_commit *old_state)
 {
 	struct analogix_dp_device *dp = to_dp(bridge);
 	struct drm_crtc *crtc;
@@ -1046,8 +979,6 @@ static void analogix_dp_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 	/* Don't touch the panel if we're coming back from PSR */
 	if (old_crtc_state && old_crtc_state->self_refresh_active)
 		return;
-
-	drm_panel_prepare(dp->plat_data->panel);
 }
 
 static int analogix_dp_set_bridge(struct analogix_dp_device *dp)
@@ -1087,14 +1018,21 @@ static int analogix_dp_set_bridge(struct analogix_dp_device *dp)
 }
 
 static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge,
+					struct drm_atomic_commit *state,
 					const struct drm_display_mode *mode)
 {
 	struct analogix_dp_device *dp = to_dp(bridge);
-	struct drm_display_info *display_info = &dp->connector.display_info;
 	struct video_info *video = &dp->video_info;
 	struct device_node *dp_node = dp->dev->of_node;
+	struct drm_connector *connector;
+	struct drm_display_info *display_info;
 	int vic;
 
+	connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder);
+	if (!connector)
+		return;
+	display_info = &connector->display_info;
+
 	/* Input video interlaces & hsync pol & vsync pol */
 	video->interlaced = !!(mode->flags & DRM_MODE_FLAG_INTERLACE);
 	video->v_sync_polarity = !!(mode->flags & DRM_MODE_FLAG_NVSYNC);
@@ -1163,7 +1101,7 @@ static void analogix_dp_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void analogix_dp_bridge_atomic_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *old_state)
+					     struct drm_atomic_commit *old_state)
 {
 	struct analogix_dp_device *dp = to_dp(bridge);
 	struct drm_crtc *crtc;
@@ -1178,7 +1116,7 @@ static void analogix_dp_bridge_atomic_enable(struct drm_bridge *bridge,
 	new_crtc_state = drm_atomic_get_new_crtc_state(old_state, crtc);
 	if (!new_crtc_state)
 		return;
-	analogix_dp_bridge_mode_set(bridge, &new_crtc_state->adjusted_mode);
+	analogix_dp_bridge_mode_set(bridge, old_state, &new_crtc_state->adjusted_mode);
 
 	old_crtc_state = drm_atomic_get_old_crtc_state(old_state, crtc);
 	/* Not a full enable, just disable PSR and continue */
@@ -1212,23 +1150,19 @@ static void analogix_dp_bridge_disable(struct drm_bridge *bridge)
 	if (dp->dpms_mode != DRM_MODE_DPMS_ON)
 		return;
 
-	drm_panel_disable(dp->plat_data->panel);
-
 	disable_irq(dp->irq);
 
 	analogix_dp_set_analog_power_down(dp, POWER_ALL, 1);
 
 	pm_runtime_put_sync(dp->dev);
 
-	drm_panel_unprepare(dp->plat_data->panel);
-
 	dp->fast_train_enable = false;
 	dp->psr_supported = false;
 	dp->dpms_mode = DRM_MODE_DPMS_OFF;
 }
 
 static void analogix_dp_bridge_atomic_disable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *old_state)
+					      struct drm_atomic_commit *old_state)
 {
 	struct analogix_dp_device *dp = to_dp(bridge);
 	struct drm_crtc *old_crtc, *new_crtc;
@@ -1266,7 +1200,7 @@ static void analogix_dp_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void analogix_dp_bridge_atomic_post_disable(struct drm_bridge *bridge,
-						   struct drm_atomic_state *old_state)
+						   struct drm_atomic_commit *old_state)
 {
 	struct analogix_dp_device *dp = to_dp(bridge);
 	struct drm_crtc *crtc;
@@ -1294,7 +1228,10 @@ static const struct drm_bridge_funcs analogix_dp_bridge_funcs = {
 	.atomic_enable = analogix_dp_bridge_atomic_enable,
 	.atomic_disable = analogix_dp_bridge_atomic_disable,
 	.atomic_post_disable = analogix_dp_bridge_atomic_post_disable,
+	.atomic_check = analogix_dp_bridge_atomic_check,
 	.attach = analogix_dp_bridge_attach,
+	.edid_read = analogix_dp_bridge_edid_read,
+	.detect = analogix_dp_bridge_detect,
 };
 
 static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp)
@@ -1524,6 +1461,7 @@ EXPORT_SYMBOL_GPL(analogix_dp_resume);
 
 int analogix_dp_bind(struct analogix_dp_device *dp, struct drm_device *drm_dev)
 {
+	struct drm_bridge *bridge = &dp->bridge;
 	int ret;
 
 	dp->drm_dev = drm_dev;
@@ -1537,7 +1475,23 @@ int analogix_dp_bind(struct analogix_dp_device *dp, struct drm_device *drm_dev)
 		return ret;
 	}
 
-	ret = drm_bridge_attach(dp->encoder, &dp->bridge, NULL, 0);
+	bridge->ops = DRM_BRIDGE_OP_EDID | DRM_BRIDGE_OP_DETECT;
+	bridge->of_node = dp->dev->of_node;
+	bridge->type = DRM_MODE_CONNECTOR_eDP;
+	ret = devm_drm_bridge_add(dp->dev, &dp->bridge);
+	if (ret)
+		goto err_unregister_aux;
+
+	if (dp->plat_data->panel) {
+		dp->plat_data->next_bridge = devm_drm_panel_bridge_add(dp->dev,
+								       dp->plat_data->panel);
+		if (IS_ERR(dp->plat_data->next_bridge)) {
+			ret = PTR_ERR(dp->plat_data->next_bridge);
+			goto err_unregister_aux;
+		}
+	}
+
+	ret = drm_bridge_attach(dp->encoder, bridge, NULL, DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 	if (ret) {
 		DRM_ERROR("failed to create bridge (%d)\n", ret);
 		goto err_unregister_aux;
@@ -1554,18 +1508,15 @@ EXPORT_SYMBOL_GPL(analogix_dp_bind);
 
 void analogix_dp_unbind(struct analogix_dp_device *dp)
 {
-	analogix_dp_bridge_disable(&dp->bridge);
-	dp->connector.funcs->destroy(&dp->connector);
-
-	drm_panel_unprepare(dp->plat_data->panel);
-
 	drm_dp_aux_unregister(&dp->aux);
 }
 EXPORT_SYMBOL_GPL(analogix_dp_unbind);
 
 int analogix_dp_start_crc(struct drm_connector *connector)
 {
-	struct analogix_dp_device *dp = to_dp(connector);
+	struct analogix_dp_device *dp;
+	struct drm_bridge *bridge __free(drm_bridge_put) =
+		drm_bridge_chain_get_first_bridge(connector->encoder);
 
 	if (!connector->state->crtc) {
 		DRM_ERROR("Connector %s doesn't currently have a CRTC.\n",
@@ -1573,13 +1524,25 @@ int analogix_dp_start_crc(struct drm_connector *connector)
 		return -EINVAL;
 	}
 
+	if (!bridge || bridge->type != DRM_MODE_CONNECTOR_eDP)
+		return -EINVAL;
+
+	dp = to_dp(bridge);
+
 	return drm_dp_start_crc(&dp->aux, connector->state->crtc);
 }
 EXPORT_SYMBOL_GPL(analogix_dp_start_crc);
 
 int analogix_dp_stop_crc(struct drm_connector *connector)
 {
-	struct analogix_dp_device *dp = to_dp(connector);
+	struct analogix_dp_device *dp;
+	struct drm_bridge *bridge __free(drm_bridge_put) =
+		drm_bridge_chain_get_first_bridge(connector->encoder);
+
+	if (!bridge || bridge->type != DRM_MODE_CONNECTOR_eDP)
+		return -EINVAL;
+
+	dp = to_dp(bridge);
 
 	return drm_dp_stop_crc(&dp->aux);
 }
@@ -1599,6 +1562,50 @@ struct drm_dp_aux *analogix_dp_get_aux(struct analogix_dp_device *dp)
 }
 EXPORT_SYMBOL_GPL(analogix_dp_get_aux);
 
+static int analogix_dp_aux_done_probing(struct drm_dp_aux *aux)
+{
+	struct analogix_dp_device *dp = to_dp(aux);
+	struct analogix_dp_plat_data *plat_data = dp->plat_data;
+	int port = plat_data->dev_type == EXYNOS_DP ? 0 : 1;
+	int ret;
+
+	/*
+	 * If drm_of_find_panel_or_bridge() returns -ENODEV, there may be no valid panel
+	 * or bridge nodes. The driver should go on for the driver-free bridge or the DP
+	 * mode applications.
+	 */
+	ret = drm_of_find_panel_or_bridge(dp->dev->of_node, port, 0,
+					  &plat_data->panel, &plat_data->next_bridge);
+	if (ret && ret != -ENODEV)
+		return ret;
+
+	return component_add(dp->dev, plat_data->ops);
+}
+
+int analogix_dp_finish_probe(struct analogix_dp_device *dp)
+{
+	int ret;
+
+	ret = devm_of_dp_aux_populate_bus(&dp->aux, analogix_dp_aux_done_probing);
+	if (ret) {
+		/*
+		 * If devm_of_dp_aux_populate_bus() returns -ENODEV, the done_probing() will
+		 * not be called because there are no EP devices. Then the callback function
+		 * analogix_dp_aux_done_probing() will be called directly in order to support
+		 * the other valid DT configurations.
+		 *
+		 * NOTE: The devm_of_dp_aux_populate_bus() is allowed to return -EPROBE_DEFER.
+		 */
+		if (ret != -ENODEV)
+			return dev_err_probe(dp->dev, ret, "failed to populate aux bus\n");
+
+		return analogix_dp_aux_done_probing(&dp->aux);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(analogix_dp_finish_probe);
+
 MODULE_AUTHOR("Jingoo Han <jg1.han@samsung.com>");
 MODULE_DESCRIPTION("Analogix DP Core Driver");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
index 91b215c6a0cfb..17347448c6b0b 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
@@ -154,7 +154,6 @@ struct analogix_dp_device {
 	struct drm_encoder	*encoder;
 	struct device		*dev;
 	struct drm_device	*drm_dev;
-	struct drm_connector	connector;
 	struct drm_bridge	bridge;
 	struct drm_dp_aux	aux;
 	struct clk		*clock;
diff --git a/drivers/gpu/drm/bridge/analogix/anx7625.c b/drivers/gpu/drm/bridge/analogix/anx7625.c
index 54b02242d6db4..513c11cdbc74a 100644
--- a/drivers/gpu/drm/bridge/analogix/anx7625.c
+++ b/drivers/gpu/drm/bridge/analogix/anx7625.c
@@ -2559,7 +2559,7 @@ static int anx7625_bridge_atomic_check(struct drm_bridge *bridge,
 }
 
 static void anx7625_bridge_atomic_enable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct anx7625_data *ctx = bridge_to_anx7625(bridge);
 	struct device *dev = ctx->dev;
@@ -2598,7 +2598,7 @@ static void anx7625_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void anx7625_bridge_atomic_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct anx7625_data *ctx = bridge_to_anx7625(bridge);
 	struct device *dev = ctx->dev;
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
index 0dd85e26248cc..cf90d4468b5c1 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-dsi-core.c
@@ -591,7 +591,7 @@ cdns_dsi_bridge_mode_valid(struct drm_bridge *bridge,
 }
 
 static void cdns_dsi_bridge_atomic_post_disable(struct drm_bridge *bridge,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct cdns_dsi_input *input = bridge_to_cdns_dsi_input(bridge);
 	struct cdns_dsi *dsi = input_to_dsi(input);
@@ -701,7 +701,7 @@ static void cdns_dsi_init_link(struct cdns_dsi *dsi)
 }
 
 static void cdns_dsi_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct cdns_dsi_input *input = bridge_to_cdns_dsi_input(bridge);
 	struct cdns_dsi *dsi = input_to_dsi(input);
@@ -1230,7 +1230,7 @@ static const struct mipi_dsi_host_ops cdns_dsi_ops = {
 	.transfer = cdns_dsi_transfer,
 };
 
-static int __maybe_unused cdns_dsi_resume(struct device *dev)
+static int cdns_dsi_resume(struct device *dev)
 {
 	struct cdns_dsi *dsi = dev_get_drvdata(dev);
 
@@ -1241,7 +1241,7 @@ static int __maybe_unused cdns_dsi_resume(struct device *dev)
 	return 0;
 }
 
-static int __maybe_unused cdns_dsi_suspend(struct device *dev)
+static int cdns_dsi_suspend(struct device *dev)
 {
 	struct cdns_dsi *dsi = dev_get_drvdata(dev);
 
@@ -1251,8 +1251,9 @@ static int __maybe_unused cdns_dsi_suspend(struct device *dev)
 	return 0;
 }
 
-static UNIVERSAL_DEV_PM_OPS(cdns_dsi_pm_ops, cdns_dsi_suspend, cdns_dsi_resume,
-			    NULL);
+static const struct dev_pm_ops cdns_dsi_pm_ops = {
+	RUNTIME_PM_OPS(cdns_dsi_suspend, cdns_dsi_resume, NULL)
+};
 
 static int cdns_dsi_drm_probe(struct platform_device *pdev)
 {
@@ -1399,7 +1400,7 @@ static struct platform_driver cdns_dsi_platform_driver = {
 	.driver = {
 		.name   = "cdns-dsi",
 		.of_match_table = cdns_dsi_of_match,
-		.pm = &cdns_dsi_pm_ops,
+		.pm = pm_ptr(&cdns_dsi_pm_ops),
 	},
 };
 module_platform_driver(cdns_dsi_platform_driver);
diff --git a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
index 064c6915c8960..36c07b71fe04b 100644
--- a/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
+++ b/drivers/gpu/drm/bridge/cadence/cdns-mhdp8546-core.c
@@ -1750,7 +1750,7 @@ static void cdns_mhdp_sst_enable(struct cdns_mhdp_device *mhdp,
 }
 
 static void cdns_mhdp_atomic_enable(struct drm_bridge *bridge,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
 	struct cdns_mhdp_bridge_state *mhdp_state;
@@ -1841,7 +1841,7 @@ static void cdns_mhdp_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void cdns_mhdp_atomic_disable(struct drm_bridge *bridge,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
 	u32 resp;
@@ -1963,7 +1963,7 @@ static int cdns_mhdp_atomic_check(struct drm_bridge *bridge,
 	struct cdns_mhdp_device *mhdp = bridge_to_mhdp(bridge);
 	const struct drm_display_mode *mode = &crtc_state->adjusted_mode;
 	struct drm_connector_state *old_state, *new_state;
-	struct drm_atomic_state *state = crtc_state->state;
+	struct drm_atomic_commit *state = crtc_state->state;
 	struct drm_connector *conn = mhdp->connector;
 	u64 old_cp, new_cp;
 
diff --git a/drivers/gpu/drm/bridge/chipone-icn6211.c b/drivers/gpu/drm/bridge/chipone-icn6211.c
index 814713c5bea97..d8344e011818c 100644
--- a/drivers/gpu/drm/bridge/chipone-icn6211.c
+++ b/drivers/gpu/drm/bridge/chipone-icn6211.c
@@ -341,7 +341,7 @@ static void chipone_configure_pll(struct chipone *icn,
 }
 
 static void chipone_atomic_enable(struct drm_bridge *bridge,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct chipone *icn = bridge_to_chipone(bridge);
 	struct drm_display_mode *mode = &icn->mode;
@@ -444,7 +444,7 @@ static void chipone_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void chipone_atomic_pre_enable(struct drm_bridge *bridge,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct chipone *icn = bridge_to_chipone(bridge);
 	int ret;
@@ -481,7 +481,7 @@ static void chipone_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void chipone_atomic_post_disable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct chipone *icn = bridge_to_chipone(bridge);
 
@@ -730,13 +730,11 @@ static int chipone_dsi_probe(struct mipi_dsi_device *dsi)
 
 	mipi_dsi_set_drvdata(dsi, icn);
 
-	drm_bridge_add(&icn->bridge);
-
-	ret = chipone_dsi_attach(icn);
+	ret = devm_drm_bridge_add(dev, &icn->bridge);
 	if (ret)
-		drm_bridge_remove(&icn->bridge);
+		return ret;
 
-	return ret;
+	return chipone_dsi_attach(icn);
 }
 
 static int chipone_i2c_probe(struct i2c_client *client)
@@ -758,17 +756,16 @@ static int chipone_i2c_probe(struct i2c_client *client)
 	dev_set_drvdata(dev, icn);
 	i2c_set_clientdata(client, icn);
 
-	drm_bridge_add(&icn->bridge);
+	ret = devm_drm_bridge_add(dev, &icn->bridge);
+	if (ret)
+		return ret;
 
 	return chipone_dsi_host_attach(icn);
 }
 
 static void chipone_dsi_remove(struct mipi_dsi_device *dsi)
 {
-	struct chipone *icn = mipi_dsi_get_drvdata(dsi);
-
 	mipi_dsi_detach(dsi);
-	drm_bridge_remove(&icn->bridge);
 }
 
 static const struct of_device_id chipone_of_match[] = {
diff --git a/drivers/gpu/drm/bridge/chrontel-ch7033.c b/drivers/gpu/drm/bridge/chrontel-ch7033.c
index 54d49d4882c85..a237c65ebd690 100644
--- a/drivers/gpu/drm/bridge/chrontel-ch7033.c
+++ b/drivers/gpu/drm/bridge/chrontel-ch7033.c
@@ -199,7 +199,6 @@ enum {
 
 struct ch7033_priv {
 	struct regmap *regmap;
-	struct drm_bridge *next_bridge;
 	struct drm_bridge bridge;
 	struct drm_connector connector;
 };
@@ -215,7 +214,7 @@ static enum drm_connector_status ch7033_connector_detect(
 {
 	struct ch7033_priv *priv = conn_to_ch7033_priv(connector);
 
-	return drm_bridge_detect(priv->next_bridge, connector);
+	return drm_bridge_detect(priv->bridge.next_bridge, connector);
 }
 
 static const struct drm_connector_funcs ch7033_connector_funcs = {
@@ -233,7 +232,7 @@ static int ch7033_connector_get_modes(struct drm_connector *connector)
 	const struct drm_edid *drm_edid;
 	int ret;
 
-	drm_edid = drm_bridge_edid_read(priv->next_bridge, connector);
+	drm_edid = drm_bridge_edid_read(priv->bridge.next_bridge, connector);
 	drm_edid_connector_update(connector, drm_edid);
 	if (drm_edid) {
 		ret = drm_edid_connector_add_modes(connector);
@@ -275,7 +274,7 @@ static int ch7033_bridge_attach(struct drm_bridge *bridge,
 	struct drm_connector *connector = &priv->connector;
 	int ret;
 
-	ret = drm_bridge_attach(encoder, priv->next_bridge, bridge,
+	ret = drm_bridge_attach(encoder, priv->bridge.next_bridge, bridge,
 				DRM_BRIDGE_ATTACH_NO_CONNECTOR);
 	if (ret)
 		return ret;
@@ -283,15 +282,15 @@ static int ch7033_bridge_attach(struct drm_bridge *bridge,
 	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)
 		return 0;
 
-	if (priv->next_bridge->ops & DRM_BRIDGE_OP_DETECT) {
+	if (priv->bridge.next_bridge->ops & DRM_BRIDGE_OP_DETECT) {
 		connector->polled = DRM_CONNECTOR_POLL_HPD;
 	} else {
 		connector->polled = DRM_CONNECTOR_POLL_CONNECT |
 				    DRM_CONNECTOR_POLL_DISCONNECT;
 	}
 
-	if (priv->next_bridge->ops & DRM_BRIDGE_OP_HPD) {
-		drm_bridge_hpd_enable(priv->next_bridge, ch7033_hpd_event,
+	if (priv->bridge.next_bridge->ops & DRM_BRIDGE_OP_HPD) {
+		drm_bridge_hpd_enable(priv->bridge.next_bridge, ch7033_hpd_event,
 				      priv);
 	}
 
@@ -299,8 +298,8 @@ static int ch7033_bridge_attach(struct drm_bridge *bridge,
 				 &ch7033_connector_helper_funcs);
 	ret = drm_connector_init_with_ddc(bridge->dev, &priv->connector,
 					  &ch7033_connector_funcs,
-					  priv->next_bridge->type,
-					  priv->next_bridge->ddc);
+					  priv->bridge.next_bridge->type,
+					  priv->bridge.next_bridge->ddc);
 	if (ret) {
 		DRM_ERROR("Failed to initialize connector\n");
 		return ret;
@@ -313,8 +312,8 @@ static void ch7033_bridge_detach(struct drm_bridge *bridge)
 {
 	struct ch7033_priv *priv = bridge_to_ch7033_priv(bridge);
 
-	if (priv->next_bridge->ops & DRM_BRIDGE_OP_HPD)
-		drm_bridge_hpd_disable(priv->next_bridge);
+	if (priv->bridge.next_bridge->ops & DRM_BRIDGE_OP_HPD)
+		drm_bridge_hpd_disable(priv->bridge.next_bridge);
 	drm_connector_cleanup(&priv->connector);
 }
 
@@ -543,10 +542,9 @@ static int ch7033_probe(struct i2c_client *client)
 
 	dev_set_drvdata(dev, priv);
 
-	ret = drm_of_find_panel_or_bridge(dev->of_node, 1, -1, NULL,
-					  &priv->next_bridge);
-	if (ret)
-		return ret;
+	priv->bridge.next_bridge = of_drm_get_bridge_by_endpoint(dev->of_node, 1, -1);
+	if (IS_ERR(priv->bridge.next_bridge))
+		return PTR_ERR(priv->bridge.next_bridge);
 
 	priv->regmap = devm_regmap_init_i2c(client, &ch7033_regmap_config);
 	if (IS_ERR(priv->regmap)) {
diff --git a/drivers/gpu/drm/bridge/fsl-ldb.c b/drivers/gpu/drm/bridge/fsl-ldb.c
index 7b71cde173e0c..9bfaa3f933709 100644
--- a/drivers/gpu/drm/bridge/fsl-ldb.c
+++ b/drivers/gpu/drm/bridge/fsl-ldb.c
@@ -124,7 +124,7 @@ static int fsl_ldb_attach(struct drm_bridge *bridge,
 }
 
 static void fsl_ldb_atomic_enable(struct drm_bridge *bridge,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct fsl_ldb *fsl_ldb = to_fsl_ldb(bridge);
 	const struct drm_bridge_state *bridge_state;
@@ -228,7 +228,7 @@ static void fsl_ldb_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void fsl_ldb_atomic_disable(struct drm_bridge *bridge,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct fsl_ldb *fsl_ldb = to_fsl_ldb(bridge);
 
diff --git a/drivers/gpu/drm/bridge/imx/Kconfig b/drivers/gpu/drm/bridge/imx/Kconfig
index b9028a5e5a065..005a745e5c473 100644
--- a/drivers/gpu/drm/bridge/imx/Kconfig
+++ b/drivers/gpu/drm/bridge/imx/Kconfig
@@ -3,21 +3,13 @@ if ARCH_MXC || COMPILE_TEST
 config DRM_IMX_LDB_HELPER
 	tristate
 
-config DRM_IMX_LEGACY_BRIDGE
-	tristate
-	depends on DRM_IMX
-	help
-	  This is a DRM bridge implementation for the DRM i.MX IPUv3 driver,
-	  that uses of_get_drm_display_mode to acquire display mode.
-
-	  Newer designs should not use this bridge and should use proper panel
-	  driver instead.
-
 config DRM_IMX8MP_DW_HDMI_BRIDGE
 	tristate "Freescale i.MX8MP HDMI-TX bridge support"
 	depends on OF
 	depends on COMMON_CLK
 	select DRM_DW_HDMI
+	select OF_OVERLAY
+	select DRM_DISPLAY_CONNECTOR
 	imply DRM_IMX8MP_HDMI_PAI
 	imply DRM_IMX8MP_HDMI_PVI
 	imply PHY_FSL_SAMSUNG_HDMI_PHY
@@ -25,6 +17,22 @@ config DRM_IMX8MP_DW_HDMI_BRIDGE
 	  Choose this to enable support for the internal HDMI encoder found
 	  on the i.MX8MP SoC.
 
+config DRM_IMX8MP_DW_HDMI_BRIDGE_CONNECTOR_FIXUP
+	bool
+	default y
+	depends on DRM_IMX_LCDIF
+	depends on DRM_IMX8MP_DW_HDMI_BRIDGE
+	depends on OF
+	help
+	  Modifies at early boot the live device tree of boards using the
+	  i.MX8MP fsl,imx8mp-hdmi-tx adding a hdmi-connector node linked to
+	  the hdmi-tx. This is needed to support bridge-connector usage in
+	  the i.MX8MP LCDIF driver.
+
+	  You need this if you use the i.MX8MP HDMI output and your board
+	  device tree file does not have an hdmi-connector node connected
+	  to it.
+
 config DRM_IMX8MP_HDMI_PAI
 	tristate "Freescale i.MX8MP HDMI PAI bridge support"
 	depends on OF
diff --git a/drivers/gpu/drm/bridge/imx/Makefile b/drivers/gpu/drm/bridge/imx/Makefile
index 8d01fda25451a..94ac8c40ebe96 100644
--- a/drivers/gpu/drm/bridge/imx/Makefile
+++ b/drivers/gpu/drm/bridge/imx/Makefile
@@ -1,6 +1,7 @@
 obj-$(CONFIG_DRM_IMX_LDB_HELPER) += imx-ldb-helper.o
-obj-$(CONFIG_DRM_IMX_LEGACY_BRIDGE) += imx-legacy-bridge.o
 obj-$(CONFIG_DRM_IMX8MP_DW_HDMI_BRIDGE) += imx8mp-hdmi-tx.o
+obj-$(CONFIG_DRM_IMX8MP_DW_HDMI_BRIDGE_CONNECTOR_FIXUP) += imx8mp-hdmi-tx-connector-fixup.o \
+							   imx8mp-hdmi-tx-connector-fixup.dtbo.o
 obj-$(CONFIG_DRM_IMX8MP_HDMI_PAI) += imx8mp-hdmi-pai.o
 obj-$(CONFIG_DRM_IMX8MP_HDMI_PVI) += imx8mp-hdmi-pvi.o
 obj-$(CONFIG_DRM_IMX8QM_LDB) += imx8qm-ldb.o
diff --git a/drivers/gpu/drm/bridge/imx/imx-legacy-bridge.c b/drivers/gpu/drm/bridge/imx/imx-legacy-bridge.c
deleted file mode 100644
index 0e31d5000e7c6..0000000000000
--- a/drivers/gpu/drm/bridge/imx/imx-legacy-bridge.c
+++ /dev/null
@@ -1,91 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Freescale i.MX drm driver
- *
- * bridge driver for legacy DT bindings, utilizing display-timings node
- */
-
-#include <linux/export.h>
-
-#include <drm/drm_bridge.h>
-#include <drm/drm_modes.h>
-#include <drm/drm_probe_helper.h>
-#include <drm/bridge/imx.h>
-
-#include <video/of_display_timing.h>
-#include <video/of_videomode.h>
-
-struct imx_legacy_bridge {
-	struct drm_bridge base;
-
-	struct drm_display_mode mode;
-	u32 bus_flags;
-};
-
-#define to_imx_legacy_bridge(bridge)	container_of(bridge, struct imx_legacy_bridge, base)
-
-static int imx_legacy_bridge_attach(struct drm_bridge *bridge,
-				    struct drm_encoder *encoder,
-				    enum drm_bridge_attach_flags flags)
-{
-	if (!(flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR))
-		return -EINVAL;
-
-	return 0;
-}
-
-static int imx_legacy_bridge_get_modes(struct drm_bridge *bridge,
-				       struct drm_connector *connector)
-{
-	struct imx_legacy_bridge *imx_bridge = to_imx_legacy_bridge(bridge);
-	int ret;
-
-	ret = drm_connector_helper_get_modes_fixed(connector, &imx_bridge->mode);
-	if (ret)
-		return ret;
-
-	connector->display_info.bus_flags = imx_bridge->bus_flags;
-
-	return 0;
-}
-
-struct drm_bridge_funcs imx_legacy_bridge_funcs = {
-	.attach = imx_legacy_bridge_attach,
-	.get_modes = imx_legacy_bridge_get_modes,
-};
-
-struct drm_bridge *devm_imx_drm_legacy_bridge(struct device *dev,
-					      struct device_node *np,
-					      int type)
-{
-	struct imx_legacy_bridge *imx_bridge;
-	int ret;
-
-	imx_bridge = devm_drm_bridge_alloc(dev, struct imx_legacy_bridge,
-					   base, &imx_legacy_bridge_funcs);
-	if (IS_ERR(imx_bridge))
-		return ERR_CAST(imx_bridge);
-
-	ret = of_get_drm_display_mode(np,
-				      &imx_bridge->mode,
-				      &imx_bridge->bus_flags,
-				      OF_USE_NATIVE_MODE);
-	if (ret)
-		return ERR_PTR(ret);
-
-	imx_bridge->mode.type |= DRM_MODE_TYPE_DRIVER;
-
-	imx_bridge->base.of_node = np;
-	imx_bridge->base.ops = DRM_BRIDGE_OP_MODES;
-	imx_bridge->base.type = type;
-
-	ret = devm_drm_bridge_add(dev, &imx_bridge->base);
-	if (ret)
-		return ERR_PTR(ret);
-
-	return &imx_bridge->base;
-}
-EXPORT_SYMBOL_GPL(devm_imx_drm_legacy_bridge);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Freescale i.MX DRM bridge driver for legacy DT bindings");
diff --git a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c
index 15fbb1be07cdf..7d5fda7173e5f 100644
--- a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c
+++ b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-pvi.c
@@ -49,7 +49,7 @@ static int imx8mp_hdmi_pvi_bridge_attach(struct drm_bridge *bridge,
 }
 
 static void imx8mp_hdmi_pvi_bridge_enable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct imx8mp_hdmi_pvi *pvi = to_imx8mp_hdmi_pvi(bridge);
 	struct drm_connector_state *conn_state;
@@ -89,7 +89,7 @@ static void imx8mp_hdmi_pvi_bridge_enable(struct drm_bridge *bridge,
 }
 
 static void imx8mp_hdmi_pvi_bridge_disable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct imx8mp_hdmi_pvi *pvi = to_imx8mp_hdmi_pvi(bridge);
 
diff --git a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx-connector-fixup.c b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx-connector-fixup.c
new file mode 100644
index 0000000000000..aaac27d00590c
--- /dev/null
+++ b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx-connector-fixup.c
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Add an hdmi-connector node to boards using the imx8mp hdmi_tx which
+ * don't have one. This is needed for the i.MX LCDIF to work with
+ * DRM_BRIDGE_ATTACH_NO_CONNECTOR.
+ *
+ * Copyright (C) 2026 GE HealthCare
+ * Author: Luca Ceresoli <luca.ceresoli@bootlin.com>
+ */
+
+#include <linux/cleanup.h>
+#include <linux/init.h>
+#include <linux/of.h>
+#include <linux/of_graph.h>
+
+/* Embedded dtbo symbols created by cmd_wrap_S_dtb in scripts/Makefile.dtbs */
+extern char __dtbo_imx8mp_hdmi_tx_connector_fixup_begin[];
+extern char __dtbo_imx8mp_hdmi_tx_connector_fixup_end[];
+
+static int __init imx8mp_hdmi_tx_connector_fixup_init(void)
+{
+	struct device_node *soc      __free(device_node) = NULL;
+	struct device_node *hdmi_tx  __free(device_node) = NULL;
+	struct device_node *endpoint __free(device_node) = NULL;
+	void *dtbo_start;
+	u32 dtbo_size;
+	int ovcs_id;
+	int err;
+
+	soc = of_find_node_by_path("/soc@0");
+	if (!soc)
+		return 0;
+
+	/* This applies to i.MX8MP only, do nothing on other systems */
+	if (!of_device_is_compatible(soc, "fsl,imx8mp-soc"))
+		return 0;
+
+	hdmi_tx = of_find_node_by_path("/soc@0/bus@32c00000/hdmi@32fd8000");
+	if (!of_device_is_available(hdmi_tx))
+		return 0;
+
+	/* If endpoint exists, assume an hdmi-connector exists already */
+	endpoint = of_graph_get_endpoint_by_regs(hdmi_tx, 1, -1);
+	if (endpoint)
+		return 0;
+
+	/*
+	 * Boards with an HDMI connector should describe it in a device
+	 * tree node with compatible = "hdmi-connector".
+	 *
+	 * If you see this warning, it means such a node was not found and
+	 * a fallback one is added using a device tree overlay. Please add
+	 * one in your device tree, also describing the exact connector
+	 * type (the added overlay assumes Type A as a fallback, but it
+	 * might be wrong).
+	 *
+	 * This node is necessary for modern DRM, where bridge drivers do
+	 * not create a connector (see the DRM_BRIDGE_ATTACH_NO_CONNECTOR
+	 * flag). See https://docs.kernel.org/gpu/drm-kms-helpers.html for
+	 * more info.
+	 */
+	pr_warn("Please add a hdmi-connector DT node for imx8mp-hdmi-tx.\n");
+
+	dtbo_start = __dtbo_imx8mp_hdmi_tx_connector_fixup_begin;
+	dtbo_size = __dtbo_imx8mp_hdmi_tx_connector_fixup_end -
+		    __dtbo_imx8mp_hdmi_tx_connector_fixup_begin;
+
+	err = of_overlay_fdt_apply(dtbo_start, dtbo_size, &ovcs_id, NULL);
+	if (err)
+		err = of_overlay_remove(&ovcs_id);
+
+	return err;
+}
+
+subsys_initcall(imx8mp_hdmi_tx_connector_fixup_init);
diff --git a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx-connector-fixup.dtso b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx-connector-fixup.dtso
new file mode 100644
index 0000000000000..6ba1c1854aeef
--- /dev/null
+++ b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx-connector-fixup.dtso
@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * DTS overlay adding an hdmi-connector node to boards using the imx8mp hdmi_tx
+ *
+ * Copyright (C) 2026 GE HealthCare
+ * Author: Luca Ceresoli <luca.ceresoli@bootlin.com>
+ */
+
+/dts-v1/;
+/plugin/;
+
+&{/} {
+	fixup-hdmi-connector {
+		compatible = "hdmi-connector";
+		label = "HDMI";
+		type = "a";
+
+		port {
+			fixup_hdmi_connector_in: endpoint {
+				remote-endpoint = <&hdmi_tx_out>;
+			};
+		};
+	};
+};
+
+&{/soc@0/bus@32c00000/hdmi@32fd8000/ports/port@1} {
+	hdmi_tx_out: endpoint {
+		remote-endpoint = <&fixup_hdmi_connector_in>;
+	};
+};
diff --git a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx.c b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx.c
index 32fd3554e2675..8e8cfd66f23b3 100644
--- a/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx.c
+++ b/drivers/gpu/drm/bridge/imx/imx8mp-hdmi-tx.c
@@ -138,6 +138,7 @@ static int imx8mp_dw_hdmi_probe(struct platform_device *pdev)
 	plat_data->phy_name = "SAMSUNG HDMI TX PHY";
 	plat_data->priv_data = hdmi;
 	plat_data->phy_force_vendor = true;
+	plat_data->output_port = 1;
 
 	platform_set_drvdata(pdev, hdmi);
 
diff --git a/drivers/gpu/drm/bridge/imx/imx8qm-ldb.c b/drivers/gpu/drm/bridge/imx/imx8qm-ldb.c
index bdecee56fa493..a6ca4f5c6cc60 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qm-ldb.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qm-ldb.c
@@ -201,7 +201,7 @@ imx8qm_ldb_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void imx8qm_ldb_bridge_atomic_enable(struct drm_bridge *bridge,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct ldb_channel *ldb_ch = bridge->driver_private;
 	struct ldb *ldb = ldb_ch->ldb;
@@ -247,7 +247,7 @@ static void imx8qm_ldb_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void imx8qm_ldb_bridge_atomic_disable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct ldb_channel *ldb_ch = bridge->driver_private;
 	struct ldb *ldb = ldb_ch->ldb;
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c b/drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c
index fbd7a37c08129..a7906314ade14 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-ldb.c
@@ -216,7 +216,7 @@ imx8qxp_ldb_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void imx8qxp_ldb_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct ldb_channel *ldb_ch = bridge->driver_private;
 	struct ldb *ldb = ldb_ch->ldb;
@@ -232,7 +232,7 @@ static void imx8qxp_ldb_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void imx8qxp_ldb_bridge_atomic_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct ldb_channel *ldb_ch = bridge->driver_private;
 	struct ldb *ldb = ldb_ch->ldb;
@@ -266,7 +266,7 @@ static void imx8qxp_ldb_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void imx8qxp_ldb_bridge_atomic_disable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct ldb_channel *ldb_ch = bridge->driver_private;
 	struct ldb *ldb = ldb_ch->ldb;
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c
index 5582456a44909..e0ee51a9ca7f8 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-combiner.c
@@ -176,7 +176,7 @@ imx8qxp_pc_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void imx8qxp_pc_bridge_atomic_disable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct imx8qxp_pc_channel *ch = bridge->driver_private;
 	struct imx8qxp_pc *pc = ch->pc;
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
index 63a8d8b1f76b1..ee6b6dbbe9527 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pixel-link.c
@@ -154,7 +154,7 @@ imx8qxp_pixel_link_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void imx8qxp_pixel_link_bridge_atomic_enable(struct drm_bridge *bridge,
-						    struct drm_atomic_state *state)
+						    struct drm_atomic_commit *state)
 {
 	struct imx8qxp_pixel_link *pl = bridge->driver_private;
 
@@ -164,7 +164,7 @@ static void imx8qxp_pixel_link_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void imx8qxp_pixel_link_bridge_atomic_disable(struct drm_bridge *bridge,
-						     struct drm_atomic_state *state)
+						     struct drm_atomic_commit *state)
 {
 	struct imx8qxp_pixel_link *pl = bridge->driver_private;
 
diff --git a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
index d64e328bf542f..87305d3e0c398 100644
--- a/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
+++ b/drivers/gpu/drm/bridge/imx/imx8qxp-pxl2dpi.c
@@ -133,7 +133,7 @@ imx8qxp_pxl2dpi_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void imx8qxp_pxl2dpi_bridge_atomic_disable(struct drm_bridge *bridge,
-						  struct drm_atomic_state *state)
+						  struct drm_atomic_commit *state)
 {
 	struct imx8qxp_pxl2dpi *p2d = bridge->driver_private;
 
diff --git a/drivers/gpu/drm/bridge/inno-hdmi.c b/drivers/gpu/drm/bridge/inno-hdmi.c
index 87422d15d9a21..1091af29ad8f1 100644
--- a/drivers/gpu/drm/bridge/inno-hdmi.c
+++ b/drivers/gpu/drm/bridge/inno-hdmi.c
@@ -756,7 +756,7 @@ static int inno_hdmi_config_video_timing(struct inno_hdmi *hdmi,
 	return 0;
 }
 
-static int inno_hdmi_setup(struct inno_hdmi *hdmi, struct drm_atomic_state *state)
+static int inno_hdmi_setup(struct inno_hdmi *hdmi, struct drm_atomic_commit *state)
 {
 	struct drm_bridge *bridge = &hdmi->bridge;
 	struct drm_connector *connector;
@@ -863,7 +863,7 @@ inno_hdmi_bridge_edid_read(struct drm_bridge *bridge, struct drm_connector *conn
 }
 
 static void inno_hdmi_bridge_atomic_enable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct inno_hdmi *hdmi = bridge_to_inno_hdmi(bridge);
 
@@ -871,7 +871,7 @@ static void inno_hdmi_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void inno_hdmi_bridge_atomic_disable(struct drm_bridge *bridge,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct inno_hdmi *hdmi = bridge_to_inno_hdmi(bridge);
 
diff --git a/drivers/gpu/drm/bridge/ite-it6263.c b/drivers/gpu/drm/bridge/ite-it6263.c
index 4f3ebb7af4d4d..c7d588be12cbf 100644
--- a/drivers/gpu/drm/bridge/ite-it6263.c
+++ b/drivers/gpu/drm/bridge/ite-it6263.c
@@ -200,6 +200,7 @@ struct it6263 {
 	struct regmap *lvds_regmap;
 	struct drm_bridge bridge;
 	struct drm_bridge *next_bridge;
+	struct gpio_desc *reset_gpio;
 	int lvds_data_mapping;
 	bool lvds_dual_link;
 	bool lvds_link12_swap;
@@ -579,7 +580,7 @@ static int it6263_read_edid(void *data, u8 *buf, unsigned int block, size_t len)
 }
 
 static void it6263_bridge_atomic_disable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct it6263 *it = bridge_to_it6263(bridge);
 
@@ -590,7 +591,7 @@ static void it6263_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void it6263_bridge_atomic_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct it6263 *it = bridge_to_it6263(bridge);
 	const struct drm_crtc_state *crtc_state;
@@ -603,6 +604,15 @@ static void it6263_bridge_atomic_enable(struct drm_bridge *bridge,
 	bool pclk_high;
 	int i, ret;
 
+	it6263_hw_reset(it->reset_gpio);
+
+	ret = it6263_lvds_set_i2c_addr(it);
+	if (ret)
+		dev_err(it->dev, "failed to set I2C addr\n");
+
+	it6263_lvds_config(it);
+	it6263_hdmi_config(it);
+
 	connector = drm_atomic_get_new_connector_for_encoder(state,
 							     bridge->encoder);
 	crtc = drm_atomic_get_new_connector_state(state, connector)->crtc;
@@ -697,8 +707,6 @@ static int it6263_bridge_attach(struct drm_bridge *bridge,
 		return ret;
 	}
 
-	drm_connector_attach_encoder(connector, encoder);
-
 	return 0;
 }
 
@@ -815,7 +823,6 @@ static int it6263_hdmi_write_hdmi_infoframe(struct drm_bridge *bridge,
 	regmap_write(regmap, HDMI_REG_PKT_NULL_CTRL,
 		     ENABLE_PKT | REPEAT_PKT);
 
-
 	return 0;
 }
 
@@ -840,7 +847,6 @@ static const struct drm_bridge_funcs it6263_bridge_funcs = {
 static int it6263_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
-	struct gpio_desc *reset_gpio;
 	struct it6263 *it;
 	int ret;
 
@@ -858,9 +864,9 @@ static int it6263_probe(struct i2c_client *client)
 		return dev_err_probe(dev, PTR_ERR(it->hdmi_regmap),
 				     "failed to init I2C regmap for HDMI\n");
 
-	reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
-	if (IS_ERR(reset_gpio))
-		return dev_err_probe(dev, PTR_ERR(reset_gpio),
+	it->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(it->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(it->reset_gpio),
 				     "failed to get reset gpio\n");
 
 	ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(it6263_supplies),
@@ -872,12 +878,6 @@ static int it6263_probe(struct i2c_client *client)
 	if (ret)
 		return ret;
 
-	it6263_hw_reset(reset_gpio);
-
-	ret = it6263_lvds_set_i2c_addr(it);
-	if (ret)
-		return dev_err_probe(dev, ret, "failed to set I2C addr\n");
-
 	it->lvds_i2c = devm_i2c_new_dummy_device(dev, client->adapter,
 						 LVDS_INPUT_CTRL_I2C_ADDR);
 	if (IS_ERR(it->lvds_i2c))
@@ -890,9 +890,6 @@ static int it6263_probe(struct i2c_client *client)
 		return dev_err_probe(dev, PTR_ERR(it->lvds_regmap),
 				     "failed to init I2C regmap for LVDS\n");
 
-	it6263_lvds_config(it);
-	it6263_hdmi_config(it);
-
 	i2c_set_clientdata(client, it);
 
 	it->bridge.of_node = dev->of_node;
diff --git a/drivers/gpu/drm/bridge/ite-it6505.c b/drivers/gpu/drm/bridge/ite-it6505.c
index a094803ba7aa4..6b7cb132a4371 100644
--- a/drivers/gpu/drm/bridge/ite-it6505.c
+++ b/drivers/gpu/drm/bridge/ite-it6505.c
@@ -2048,7 +2048,7 @@ static void it6505_start_hdcp(struct it6505 *it6505)
 
 	DRM_DEV_DEBUG_DRIVER(dev, "start");
 	it6505_reset_hdcp(it6505);
-	queue_delayed_work(system_wq, &it6505->hdcp_work,
+	queue_delayed_work(system_percpu_wq, &it6505->hdcp_work,
 			   msecs_to_jiffies(2400));
 }
 
@@ -3119,7 +3119,7 @@ it6505_bridge_mode_valid(struct drm_bridge *bridge,
 }
 
 static void it6505_bridge_atomic_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct it6505 *it6505 = bridge_to_it6505(bridge);
 	struct device *dev = it6505->dev;
@@ -3173,7 +3173,7 @@ static void it6505_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void it6505_bridge_atomic_disable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct it6505 *it6505 = bridge_to_it6505(bridge);
 	struct device *dev = it6505->dev;
@@ -3187,7 +3187,7 @@ static void it6505_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void it6505_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct it6505 *it6505 = bridge_to_it6505(bridge);
 	struct device *dev = it6505->dev;
@@ -3198,7 +3198,7 @@ static void it6505_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void it6505_bridge_atomic_post_disable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct it6505 *it6505 = bridge_to_it6505(bridge);
 	struct device *dev = it6505->dev;
diff --git a/drivers/gpu/drm/bridge/ite-it66121.c b/drivers/gpu/drm/bridge/ite-it66121.c
index 9246e9c15a6ef..d3d895ee88de6 100644
--- a/drivers/gpu/drm/bridge/ite-it66121.c
+++ b/drivers/gpu/drm/bridge/ite-it66121.c
@@ -722,7 +722,7 @@ static u32 *it66121_bridge_atomic_get_input_bus_fmts(struct drm_bridge *bridge,
 }
 
 static void it66121_bridge_enable(struct drm_bridge *bridge,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct it66121_ctx *ctx = container_of(bridge, struct it66121_ctx, bridge);
 
@@ -732,7 +732,7 @@ static void it66121_bridge_enable(struct drm_bridge *bridge,
 }
 
 static void it66121_bridge_disable(struct drm_bridge *bridge,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct it66121_ctx *ctx = container_of(bridge, struct it66121_ctx, bridge);
 
@@ -1559,6 +1559,11 @@ static int it66121_probe(struct i2c_client *client)
 		return ret;
 	}
 
+	ctx->gpio_reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_LOW);
+	if (IS_ERR(ctx->gpio_reset))
+		return dev_err_probe(dev, PTR_ERR(ctx->gpio_reset),
+				     "Failed to get reset GPIO\n");
+
 	it66121_hw_reset(ctx);
 
 	ctx->regmap = devm_regmap_init_i2c(client, &it66121_regmap_config);
diff --git a/drivers/gpu/drm/bridge/lontium-lt8713sx.c b/drivers/gpu/drm/bridge/lontium-lt8713sx.c
index 18fac6a46db46..cee485adf5e56 100644
--- a/drivers/gpu/drm/bridge/lontium-lt8713sx.c
+++ b/drivers/gpu/drm/bridge/lontium-lt8713sx.c
@@ -32,7 +32,6 @@ DECLARE_CRC8_TABLE(lt8713sx_crc_table);
 struct lt8713sx {
 	struct device *dev;
 	struct drm_bridge bridge;
-	struct drm_bridge *next_bridge;
 
 	struct regmap *regmap;
 	/* Protects all accesses to registers by stopping the on-chip MCU */
@@ -458,7 +457,7 @@ static int lt8713sx_bridge_attach(struct drm_bridge *bridge,
 	struct lt8713sx *lt8713sx = container_of(bridge, struct lt8713sx, bridge);
 
 	return drm_bridge_attach(encoder,
-				lt8713sx->next_bridge,
+				lt8713sx->bridge.next_bridge,
 				bridge, flags);
 }
 
@@ -537,10 +536,9 @@ static int lt8713sx_probe(struct i2c_client *client)
 	if (IS_ERR(lt8713sx->regmap))
 		return dev_err_probe(dev, PTR_ERR(lt8713sx->regmap), "regmap i2c init failed\n");
 
-	ret = drm_of_find_panel_or_bridge(lt8713sx->dev->of_node, 1, -1, NULL,
-					  &lt8713sx->next_bridge);
-	if (ret < 0)
-		return ret;
+	lt8713sx->bridge.next_bridge = of_drm_get_bridge_by_endpoint(lt8713sx->dev->of_node, 1, -1);
+	if (IS_ERR(lt8713sx->bridge.next_bridge))
+		return PTR_ERR(lt8713sx->bridge.next_bridge);
 
 	ret = lt8713sx_gpio_init(lt8713sx);
 	if (ret < 0)
diff --git a/drivers/gpu/drm/bridge/lontium-lt8912b.c b/drivers/gpu/drm/bridge/lontium-lt8912b.c
index 8a0b48efca580..729b12b674706 100644
--- a/drivers/gpu/drm/bridge/lontium-lt8912b.c
+++ b/drivers/gpu/drm/bridge/lontium-lt8912b.c
@@ -493,6 +493,7 @@ static int lt8912_attach_dsi(struct lt8912 *lt)
 
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
 			  MIPI_DSI_MODE_LPM |
+			  MIPI_DSI_MODE_VIDEO_NO_HFP |
 			  MIPI_DSI_MODE_NO_EOT_PACKET;
 
 	ret = devm_mipi_dsi_attach(dev, dsi);
diff --git a/drivers/gpu/drm/bridge/lontium-lt9211.c b/drivers/gpu/drm/bridge/lontium-lt9211.c
index 03fc8fd10f20a..870175a848390 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9211.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9211.c
@@ -455,7 +455,7 @@ static int lt9211_configure_tx(struct lt9211 *ctx, bool jeida,
 }
 
 static void lt9211_atomic_enable(struct drm_bridge *bridge,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct lt9211 *ctx = bridge_to_lt9211(bridge);
 	const struct drm_bridge_state *bridge_state;
@@ -552,7 +552,7 @@ static void lt9211_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void lt9211_atomic_disable(struct drm_bridge *bridge,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct lt9211 *ctx = bridge_to_lt9211(bridge);
 	int ret;
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611.c b/drivers/gpu/drm/bridge/lontium-lt9611.c
index 4517aee83332a..21305296e1117 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611.c
@@ -37,7 +37,6 @@
 struct lt9611 {
 	struct device *dev;
 	struct drm_bridge bridge;
-	struct drm_bridge *next_bridge;
 
 	struct regmap *regmap;
 
@@ -657,7 +656,7 @@ lt9611_get_edid_block(void *data, u8 *buf, unsigned int block, size_t len)
 
 /* bridge funcs */
 static void lt9611_bridge_atomic_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct lt9611 *lt9611 = bridge_to_lt9611(bridge);
 	struct drm_connector *connector;
@@ -704,7 +703,7 @@ static void lt9611_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void lt9611_bridge_atomic_disable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct lt9611 *lt9611 = bridge_to_lt9611(bridge);
 	int ret;
@@ -761,7 +760,7 @@ static int lt9611_bridge_attach(struct drm_bridge *bridge,
 {
 	struct lt9611 *lt9611 = bridge_to_lt9611(bridge);
 
-	return drm_bridge_attach(encoder, lt9611->next_bridge,
+	return drm_bridge_attach(encoder, lt9611->bridge.next_bridge,
 				 bridge, flags);
 }
 
@@ -782,7 +781,7 @@ static enum drm_mode_status lt9611_bridge_mode_valid(struct drm_bridge *bridge,
 }
 
 static void lt9611_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct lt9611 *lt9611 = bridge_to_lt9611(bridge);
 	static const struct reg_sequence reg_cfg[] = {
@@ -802,7 +801,7 @@ static void lt9611_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void lt9611_bridge_atomic_post_disable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct lt9611 *lt9611 = bridge_to_lt9611(bridge);
 
@@ -1058,7 +1057,11 @@ static int lt9611_parse_dt(struct device *dev,
 
 	lt9611->ac_mode = of_property_read_bool(dev->of_node, "lt,ac-mode");
 
-	return drm_of_find_panel_or_bridge(dev->of_node, 2, -1, NULL, &lt9611->next_bridge);
+	lt9611->bridge.next_bridge = of_drm_get_bridge_by_endpoint(dev->of_node, 2, -1);
+	if (IS_ERR(lt9611->bridge.next_bridge))
+		return PTR_ERR(lt9611->bridge.next_bridge);
+
+	return 0;
 }
 
 static int lt9611_gpio_init(struct lt9611 *lt9611)
diff --git a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
index 11aab07d88df6..9427cc2358ae1 100644
--- a/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
+++ b/drivers/gpu/drm/bridge/lontium-lt9611uxc.c
@@ -35,7 +35,6 @@
 struct lt9611uxc {
 	struct device *dev;
 	struct drm_bridge bridge;
-	struct drm_bridge *next_bridge;
 
 	struct regmap *regmap;
 	/* Protects all accesses to registers by stopping the on-chip MCU */
@@ -284,7 +283,7 @@ static int lt9611uxc_bridge_attach(struct drm_bridge *bridge,
 {
 	struct lt9611uxc *lt9611uxc = bridge_to_lt9611uxc(bridge);
 
-	return drm_bridge_attach(encoder, lt9611uxc->next_bridge,
+	return drm_bridge_attach(encoder, lt9611uxc->bridge.next_bridge,
 				 bridge, flags);
 }
 
@@ -487,7 +486,11 @@ static int lt9611uxc_parse_dt(struct device *dev,
 
 	lt9611uxc->dsi1_node = of_graph_get_remote_node(dev->of_node, 1, -1);
 
-	return drm_of_find_panel_or_bridge(dev->of_node, 2, -1, NULL, &lt9611uxc->next_bridge);
+	lt9611uxc->bridge.next_bridge = of_drm_get_bridge_by_endpoint(dev->of_node, 2, -1);
+	if (IS_ERR(lt9611uxc->bridge.next_bridge))
+		return PTR_ERR(lt9611uxc->bridge.next_bridge);
+
+	return 0;
 }
 
 static int lt9611uxc_gpio_init(struct lt9611uxc *lt9611uxc)
diff --git a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
index c9e6505cbd882..2d02cc69f2374 100644
--- a/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
+++ b/drivers/gpu/drm/bridge/megachips-stdpxxxx-ge-b850v3-fw.c
@@ -251,7 +251,6 @@ static void ge_b850v3_lvds_remove(void)
 		goto out;
 
 	drm_bridge_remove(&ge_b850v3_lvds_ptr->bridge);
-
 	ge_b850v3_lvds_ptr = NULL;
 out:
 	mutex_unlock(&ge_b850v3_lvds_dev_mutex);
@@ -261,6 +260,7 @@ static int ge_b850v3_register(void)
 {
 	struct i2c_client *stdp4028_i2c = ge_b850v3_lvds_ptr->stdp4028_i2c;
 	struct device *dev = &stdp4028_i2c->dev;
+	int ret;
 
 	/* drm bridge initialization */
 	ge_b850v3_lvds_ptr->bridge.ops = DRM_BRIDGE_OP_DETECT |
@@ -277,11 +277,15 @@ static int ge_b850v3_register(void)
 	if (!stdp4028_i2c->irq)
 		return 0;
 
-	return devm_request_threaded_irq(&stdp4028_i2c->dev,
-			stdp4028_i2c->irq, NULL,
-			ge_b850v3_lvds_irq_handler,
-			IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
-			"ge-b850v3-lvds-dp", ge_b850v3_lvds_ptr);
+	ret = devm_request_threaded_irq(&stdp4028_i2c->dev,
+					stdp4028_i2c->irq, NULL,
+					ge_b850v3_lvds_irq_handler,
+					IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
+					"ge-b850v3-lvds-dp", ge_b850v3_lvds_ptr);
+	if (ret)
+		drm_bridge_remove(&ge_b850v3_lvds_ptr->bridge);
+
+	return ret;
 }
 
 static int stdp4028_ge_b850v3_fw_probe(struct i2c_client *stdp4028_i2c)
diff --git a/drivers/gpu/drm/bridge/microchip-lvds.c b/drivers/gpu/drm/bridge/microchip-lvds.c
index 9f4ff82bc6b49..5fb8633f43c5c 100644
--- a/drivers/gpu/drm/bridge/microchip-lvds.c
+++ b/drivers/gpu/drm/bridge/microchip-lvds.c
@@ -11,6 +11,7 @@
 #include <linux/component.h>
 #include <linux/delay.h>
 #include <linux/jiffies.h>
+#include <linux/media-bus-format.h>
 #include <linux/mfd/syscon.h>
 #include <linux/of_graph.h>
 #include <linux/pinctrl/devinfo.h>
@@ -23,7 +24,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
@@ -41,9 +41,11 @@
 
 /* Bitfields in LVDSC_CFGR (Configuration Register) */
 #define LVDSC_CFGR_PIXSIZE_24BITS	0
+#define LVDSC_CFGR_PIXSIZE_18BITS	BIT(0)
 #define LVDSC_CFGR_DEN_POL_HIGH		0
 #define LVDSC_CFGR_DC_UNBALANCED	0
 #define LVDSC_CFGR_MAPPING_JEIDA	BIT(6)
+#define LVDSC_CFGR_MAPPING_VESA		0
 
 /*Bitfields in LVDSC_SR */
 #define LVDSC_SR_CS	BIT(0)
@@ -56,7 +58,6 @@ struct mchp_lvds {
 	struct device *dev;
 	void __iomem *regs;
 	struct clk *pclk;
-	struct drm_panel *panel;
 	struct drm_bridge bridge;
 	struct drm_bridge *panel_bridge;
 };
@@ -76,9 +77,10 @@ static inline void lvds_writel(struct mchp_lvds *lvds, u32 offset, u32 val)
 	writel_relaxed(val, lvds->regs + offset);
 }
 
-static void lvds_serialiser_on(struct mchp_lvds *lvds)
+static void lvds_serialiser_on(struct mchp_lvds *lvds, u32 bus_format)
 {
 	unsigned long timeout = jiffies + msecs_to_jiffies(LVDS_POLL_TIMEOUT_MS);
+	u8 map, pix_size;
 
 	/* The LVDSC registers can only be written if WPEN is cleared */
 	lvds_writel(lvds, LVDSC_WPMR, (LVDSC_WPMR_WPKEY_PSSWD &
@@ -93,11 +95,24 @@ static void lvds_serialiser_on(struct mchp_lvds *lvds)
 		usleep_range(1000, 2000);
 	}
 
+	switch (bus_format) {
+	case MEDIA_BUS_FMT_RGB666_1X7X3_SPWG:
+		map = LVDSC_CFGR_MAPPING_JEIDA;
+		pix_size = LVDSC_CFGR_PIXSIZE_18BITS;
+		break;
+	case MEDIA_BUS_FMT_RGB888_1X7X4_SPWG:
+		map = LVDSC_CFGR_MAPPING_VESA;
+		pix_size = LVDSC_CFGR_PIXSIZE_24BITS;
+		break;
+	default:
+		map = LVDSC_CFGR_MAPPING_JEIDA;
+		pix_size = LVDSC_CFGR_PIXSIZE_24BITS;
+		break;
+	}
+
 	/* Configure the LVDSC */
-	lvds_writel(lvds, LVDSC_CFGR, (LVDSC_CFGR_MAPPING_JEIDA |
-				LVDSC_CFGR_DC_UNBALANCED |
-				LVDSC_CFGR_DEN_POL_HIGH |
-				LVDSC_CFGR_PIXSIZE_24BITS));
+	lvds_writel(lvds, LVDSC_CFGR, map | LVDSC_CFGR_DC_UNBALANCED |
+		    LVDSC_CFGR_DEN_POL_HIGH | pix_size);
 
 	/* Enable the LVDS serializer */
 	lvds_writel(lvds, LVDSC_CR, LVDSC_CR_SER_EN);
@@ -113,9 +128,11 @@ static int mchp_lvds_attach(struct drm_bridge *bridge,
 				 bridge, flags);
 }
 
-static void mchp_lvds_enable(struct drm_bridge *bridge)
+static void mchp_lvds_atomic_enable(struct drm_bridge *bridge,
+				    struct drm_atomic_commit *state)
 {
 	struct mchp_lvds *lvds = bridge_to_lvds(bridge);
+	struct drm_connector *connector;
 	int ret;
 
 	ret = clk_prepare_enable(lvds->pclk);
@@ -130,10 +147,18 @@ static void mchp_lvds_enable(struct drm_bridge *bridge)
 		return;
 	}
 
-	lvds_serialiser_on(lvds);
+	/* default to jeida-24 */
+	u32 bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA;
+
+	connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder);
+	if (connector && connector->display_info.num_bus_formats)
+		bus_format = connector->display_info.bus_formats[0];
+
+	lvds_serialiser_on(lvds, bus_format);
 }
 
-static void mchp_lvds_disable(struct drm_bridge *bridge)
+static void mchp_lvds_atomic_disable(struct drm_bridge *bridge,
+				     struct drm_atomic_commit *state)
 {
 	struct mchp_lvds *lvds = bridge_to_lvds(bridge);
 
@@ -143,15 +168,14 @@ static void mchp_lvds_disable(struct drm_bridge *bridge)
 
 static const struct drm_bridge_funcs mchp_lvds_bridge_funcs = {
 	.attach = mchp_lvds_attach,
-	.enable = mchp_lvds_enable,
-	.disable = mchp_lvds_disable,
+	.atomic_enable = mchp_lvds_atomic_enable,
+	.atomic_disable = mchp_lvds_atomic_disable,
 };
 
 static int mchp_lvds_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct mchp_lvds *lvds;
-	struct device_node *port;
 	int ret;
 
 	if (!dev->of_node)
@@ -173,19 +197,6 @@ static int mchp_lvds_probe(struct platform_device *pdev)
 		return dev_err_probe(lvds->dev, PTR_ERR(lvds->pclk),
 				"could not get pclk_lvds\n");
 
-	port = of_graph_get_remote_node(dev->of_node, 1, 0);
-	if (!port) {
-		dev_err(dev,
-			"can't find port point, please init lvds panel port!\n");
-		return -ENODEV;
-	}
-
-	lvds->panel = of_drm_find_panel(port);
-	of_node_put(port);
-
-	if (IS_ERR(lvds->panel))
-		return -EPROBE_DEFER;
-
 	lvds->panel_bridge = devm_drm_of_get_bridge(dev, dev->of_node, 1, 0);
 
 	if (IS_ERR(lvds->panel_bridge))
diff --git a/drivers/gpu/drm/bridge/nwl-dsi.c b/drivers/gpu/drm/bridge/nwl-dsi.c
index 2f7429b24fc20..3c9ae93c4f671 100644
--- a/drivers/gpu/drm/bridge/nwl-dsi.c
+++ b/drivers/gpu/drm/bridge/nwl-dsi.c
@@ -737,7 +737,7 @@ static int nwl_dsi_disable(struct nwl_dsi *dsi)
 }
 
 static void nwl_dsi_bridge_atomic_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct nwl_dsi *dsi = bridge_to_dsi(bridge);
 	int ret;
@@ -898,7 +898,7 @@ nwl_dsi_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void nwl_dsi_bridge_atomic_enable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct nwl_dsi *dsi = bridge_to_dsi(bridge);
 	int ret;
diff --git a/drivers/gpu/drm/bridge/of-display-mode-bridge.c b/drivers/gpu/drm/bridge/of-display-mode-bridge.c
new file mode 100644
index 0000000000000..cb15713f3a796
--- /dev/null
+++ b/drivers/gpu/drm/bridge/of-display-mode-bridge.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2012 Sascha Hauer, Pengutronix
+ *
+ * bridge driver for legacy DT bindings, utilizing display-timings node
+ *
+ * Author: Dmitry Baryshkov <dmitry.baryshkov@oss.qualcomm.com>
+ */
+
+#include <linux/export.h>
+
+#include <drm/drm_bridge.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_probe_helper.h>
+#include <drm/bridge/of-display-mode-bridge.h>
+
+#include <video/of_display_timing.h>
+#include <video/of_videomode.h>
+
+struct of_display_mode_bridge {
+	struct drm_bridge base;
+
+	struct drm_display_mode mode;
+	u32 bus_flags;
+};
+
+#define to_of_display_mode_bridge(bridge) container_of(bridge, struct of_display_mode_bridge, base)
+
+static int of_display_mode_bridge_attach(struct drm_bridge *bridge,
+					 struct drm_encoder *encoder,
+					 enum drm_bridge_attach_flags flags)
+{
+	if (!(flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR))
+		return -EINVAL;
+
+	return 0;
+}
+
+static int of_display_mode_bridge_get_modes(struct drm_bridge *bridge,
+					    struct drm_connector *connector)
+{
+	struct of_display_mode_bridge *of_bridge = to_of_display_mode_bridge(bridge);
+	int ret;
+
+	ret = drm_connector_helper_get_modes_fixed(connector, &of_bridge->mode);
+	if (ret)
+		return ret;
+
+	connector->display_info.bus_flags = of_bridge->bus_flags;
+
+	return 0;
+}
+
+struct drm_bridge_funcs of_display_mode_bridge_funcs = {
+	.attach = of_display_mode_bridge_attach,
+	.get_modes = of_display_mode_bridge_get_modes,
+};
+
+struct drm_bridge *devm_drm_of_display_mode_bridge(struct device *dev,
+						   struct device_node *np,
+						   int type)
+{
+	struct of_display_mode_bridge *of_bridge;
+	int ret;
+
+	of_bridge = devm_drm_bridge_alloc(dev, struct of_display_mode_bridge,
+					  base, &of_display_mode_bridge_funcs);
+	if (IS_ERR(of_bridge))
+		return ERR_CAST(of_bridge);
+
+	ret = of_get_drm_display_mode(np,
+				      &of_bridge->mode,
+				      &of_bridge->bus_flags,
+				      OF_USE_NATIVE_MODE);
+	if (ret)
+		return ERR_PTR(ret);
+
+	of_bridge->mode.type |= DRM_MODE_TYPE_DRIVER;
+
+	of_bridge->base.of_node = np;
+	of_bridge->base.ops = DRM_BRIDGE_OP_MODES;
+	of_bridge->base.type = type;
+
+	ret = devm_drm_bridge_add(dev, &of_bridge->base);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return &of_bridge->base;
+}
+EXPORT_SYMBOL_GPL(devm_drm_of_display_mode_bridge);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("DRM bridge driver for legacy DT bindings");
diff --git a/drivers/gpu/drm/bridge/panel.c b/drivers/gpu/drm/bridge/panel.c
index 184a8b7049a7e..4978ec98a0828 100644
--- a/drivers/gpu/drm/bridge/panel.c
+++ b/drivers/gpu/drm/bridge/panel.c
@@ -111,7 +111,7 @@ static void panel_bridge_detach(struct drm_bridge *bridge)
 }
 
 static void panel_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *atomic_state)
+					   struct drm_atomic_commit *atomic_state)
 {
 	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
 	struct drm_encoder *encoder = bridge->encoder;
@@ -130,7 +130,7 @@ static void panel_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void panel_bridge_atomic_enable(struct drm_bridge *bridge,
-				       struct drm_atomic_state *atomic_state)
+				       struct drm_atomic_commit *atomic_state)
 {
 	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
 	struct drm_encoder *encoder = bridge->encoder;
@@ -149,7 +149,7 @@ static void panel_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void panel_bridge_atomic_disable(struct drm_bridge *bridge,
-					struct drm_atomic_state *atomic_state)
+					struct drm_atomic_commit *atomic_state)
 {
 	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
 	struct drm_encoder *encoder = bridge->encoder;
@@ -168,7 +168,7 @@ static void panel_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void panel_bridge_atomic_post_disable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *atomic_state)
+					     struct drm_atomic_commit *atomic_state)
 {
 	struct panel_bridge *panel_bridge = drm_bridge_to_panel_bridge(bridge);
 	struct drm_encoder *encoder = bridge->encoder;
diff --git a/drivers/gpu/drm/bridge/parade-ps8640.c b/drivers/gpu/drm/bridge/parade-ps8640.c
index 825777a5758f6..b93514023baa8 100644
--- a/drivers/gpu/drm/bridge/parade-ps8640.c
+++ b/drivers/gpu/drm/bridge/parade-ps8640.c
@@ -437,7 +437,7 @@ static const struct dev_pm_ops ps8640_pm_ops = {
 };
 
 static void ps8640_atomic_pre_enable(struct drm_bridge *bridge,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct ps8640 *ps_bridge = bridge_to_ps8640(bridge);
 	struct regmap *map = ps_bridge->regmap[PAGE2_TOP_CNTL];
@@ -472,7 +472,7 @@ static void ps8640_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void ps8640_atomic_post_disable(struct drm_bridge *bridge,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct ps8640 *ps_bridge = bridge_to_ps8640(bridge);
 
diff --git a/drivers/gpu/drm/bridge/samsung-dsim.c b/drivers/gpu/drm/bridge/samsung-dsim.c
index c3eb437ef1b0b..9ee0515074c78 100644
--- a/drivers/gpu/drm/bridge/samsung-dsim.c
+++ b/drivers/gpu/drm/bridge/samsung-dsim.c
@@ -1640,7 +1640,7 @@ static int samsung_dsim_init(struct samsung_dsim *dsi)
 }
 
 static void samsung_dsim_atomic_pre_enable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
 	int ret;
@@ -1668,7 +1668,7 @@ static void samsung_dsim_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void samsung_dsim_atomic_enable(struct drm_bridge *bridge,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
 
@@ -1679,7 +1679,7 @@ static void samsung_dsim_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void samsung_dsim_atomic_disable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
 
@@ -1691,7 +1691,7 @@ static void samsung_dsim_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void samsung_dsim_atomic_post_disable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct samsung_dsim *dsi = bridge_to_dsi(bridge);
 
diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c
index 12497f5ce4ff5..ea543be4ae3e2 100644
--- a/drivers/gpu/drm/bridge/sii902x.c
+++ b/drivers/gpu/drm/bridge/sii902x.c
@@ -321,7 +321,7 @@ static const struct drm_connector_helper_funcs sii902x_connector_helper_funcs =
 };
 
 static void sii902x_bridge_atomic_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct sii902x *sii902x = bridge_to_sii902x(bridge);
 
@@ -335,7 +335,7 @@ static void sii902x_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void sii902x_bridge_atomic_enable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct sii902x *sii902x = bridge_to_sii902x(bridge);
 	struct drm_connector *connector;
diff --git a/drivers/gpu/drm/bridge/simple-bridge.c b/drivers/gpu/drm/bridge/simple-bridge.c
index 8aa31ca3c72d4..cc13c98f9be6b 100644
--- a/drivers/gpu/drm/bridge/simple-bridge.c
+++ b/drivers/gpu/drm/bridge/simple-bridge.c
@@ -270,6 +270,11 @@ static const struct of_device_id simple_bridge_match[] = {
 		.data = &(const struct simple_bridge_info) {
 			.connector_type = DRM_MODE_CONNECTOR_HDMIA,
 		},
+	}, {
+		.compatible = "mstar,tsumu88adt3-lf-1",
+		.data = &(const struct simple_bridge_info) {
+			.connector_type = DRM_MODE_CONNECTOR_HDMIA,
+		},
 	}, {
 		.compatible = "parade,ps185hdm",
 		.data = &(const struct simple_bridge_info) {
diff --git a/drivers/gpu/drm/bridge/ssd2825.c b/drivers/gpu/drm/bridge/ssd2825.c
index f2fdbf7c117d3..91f1510fc2d40 100644
--- a/drivers/gpu/drm/bridge/ssd2825.c
+++ b/drivers/gpu/drm/bridge/ssd2825.c
@@ -482,7 +482,7 @@ static int ssd2825_setup_pll(struct ssd2825_priv *priv,
 }
 
 static void ssd2825_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct ssd2825_priv *priv = bridge_to_ssd2825(bridge);
 	struct mipi_dsi_device *dsi_dev = priv->output.dev;
@@ -585,7 +585,7 @@ static void ssd2825_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void ssd2825_bridge_atomic_enable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct ssd2825_priv *priv = bridge_to_ssd2825(bridge);
 	struct mipi_dsi_device *dsi_dev = priv->output.dev;
@@ -607,7 +607,7 @@ static void ssd2825_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void ssd2825_bridge_atomic_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct ssd2825_priv *priv = bridge_to_ssd2825(bridge);
 	int ret;
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-dp.c b/drivers/gpu/drm/bridge/synopsys/dw-dp.c
index 45b37885d719d..21541be094c47 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-dp.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-dp.c
@@ -1644,7 +1644,7 @@ static int dw_dp_link_enable(struct dw_dp *dp)
 }
 
 static void dw_dp_bridge_atomic_enable(struct drm_bridge *bridge,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct dw_dp *dp = bridge_to_dp(bridge);
 	struct drm_connector *connector;
@@ -1697,7 +1697,7 @@ static void dw_dp_reset(struct dw_dp *dp)
 }
 
 static void dw_dp_bridge_atomic_disable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct dw_dp *dp = bridge_to_dp(bridge);
 
@@ -1970,10 +1970,6 @@ struct dw_dp *dw_dp_bind(struct device *dev, struct drm_encoder *encoder,
 	void __iomem *res;
 	int ret;
 
-	dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL);
-	if (!dp)
-		return ERR_PTR(-ENOMEM);
-
 	dp = devm_drm_bridge_alloc(dev, struct dw_dp, bridge, &dw_dp_bridge_funcs);
 	if (IS_ERR(dp))
 		return ERR_CAST(dp);
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
index d649a1cf07f5c..0dbb127436094 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi-qp.c
@@ -750,7 +750,7 @@ static struct i2c_adapter *dw_hdmi_qp_i2c_adapter(struct dw_hdmi_qp *hdmi)
 }
 
 static void dw_hdmi_qp_bridge_atomic_enable(struct drm_bridge *bridge,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct dw_hdmi_qp *hdmi = bridge->driver_private;
 	struct drm_connector_state *conn_state;
@@ -785,7 +785,7 @@ static void dw_hdmi_qp_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void dw_hdmi_qp_bridge_atomic_disable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct dw_hdmi_qp *hdmi = bridge->driver_private;
 
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
index 0296e110ce65f..41b3a9cfa2f54 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-hdmi.c
@@ -6,6 +6,8 @@
  * Copyright (C) 2011-2013 Freescale Semiconductor, Inc.
  * Copyright (C) 2010, Guennadi Liakhovetski <g.liakhovetski@gmx.de>
  */
+
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/err.h>
@@ -2529,7 +2531,7 @@ static int dw_hdmi_connector_get_modes(struct drm_connector *connector)
 }
 
 static int dw_hdmi_connector_atomic_check(struct drm_connector *connector,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *old_state =
 		drm_atomic_get_old_connector_state(state, connector);
@@ -2910,9 +2912,24 @@ static int dw_hdmi_bridge_attach(struct drm_bridge *bridge,
 {
 	struct dw_hdmi *hdmi = bridge->driver_private;
 
-	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR)
-		return drm_bridge_attach(encoder, hdmi->bridge.next_bridge,
-					 bridge, flags);
+	/* DRM_BRIDGE_ATTACH_NO_CONNECTOR requires a remote-endpoint to the next bridge */
+	if (WARN_ON((flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) && !hdmi->plat_data->output_port))
+		return -EINVAL;
+
+	if (flags & DRM_BRIDGE_ATTACH_NO_CONNECTOR) {
+		struct device_node *remote __free(device_node) =
+			of_graph_get_remote_node(hdmi->dev->of_node,
+						 hdmi->plat_data->output_port, -1);
+		if (!remote)
+			return -ENODEV;
+
+		struct drm_bridge *next_bridge __free(drm_bridge_put) =
+			of_drm_find_and_get_bridge(remote);
+		if (!next_bridge)
+			return -EPROBE_DEFER;
+
+		return drm_bridge_attach(encoder, next_bridge, bridge, flags);
+	}
 
 	return dw_hdmi_connector_create(hdmi);
 }
@@ -2962,7 +2979,7 @@ static void dw_hdmi_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void dw_hdmi_bridge_atomic_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct dw_hdmi *hdmi = bridge->driver_private;
 
@@ -2976,7 +2993,7 @@ static void dw_hdmi_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void dw_hdmi_bridge_atomic_enable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct dw_hdmi *hdmi = bridge->driver_private;
 	struct drm_connector *connector;
@@ -3303,28 +3320,6 @@ static void dw_hdmi_init_hw(struct dw_hdmi *hdmi)
  * Probe/remove API, used from platforms based on the DRM bridge API.
  */
 
-static int dw_hdmi_parse_dt(struct dw_hdmi *hdmi)
-{
-	struct device_node *remote;
-
-	if (!hdmi->plat_data->output_port)
-		return 0;
-
-
-	remote = of_graph_get_remote_node(hdmi->dev->of_node,
-					  hdmi->plat_data->output_port,
-					  -1);
-	if (!remote)
-		return -ENODEV;
-
-	hdmi->bridge.next_bridge = of_drm_find_and_get_bridge(remote);
-	of_node_put(remote);
-	if (!hdmi->bridge.next_bridge)
-		return -EPROBE_DEFER;
-
-	return 0;
-}
-
 bool dw_hdmi_bus_fmt_is_420(struct dw_hdmi *hdmi)
 {
 	return hdmi_bus_fmt_is_yuv420(hdmi->hdmi_data.enc_out_bus_format);
@@ -3369,10 +3364,6 @@ struct dw_hdmi *dw_hdmi_probe(struct platform_device *pdev,
 	mutex_init(&hdmi->cec_notifier_mutex);
 	spin_lock_init(&hdmi->audio_lock);
 
-	ret = dw_hdmi_parse_dt(hdmi);
-	if (ret < 0)
-		return ERR_PTR(ret);
-
 	ddc_node = of_parse_phandle(np, "ddc-i2c-bus", 0);
 	if (ddc_node) {
 		hdmi->ddc = of_get_i2c_adapter_by_node(ddc_node);
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
index ef7be20a59cde..af176e136f85e 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi.c
@@ -939,7 +939,7 @@ static void dw_mipi_dsi_clear_err(struct dw_mipi_dsi *dsi)
 }
 
 static void dw_mipi_dsi_bridge_post_atomic_disable(struct drm_bridge *bridge,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge);
 	const struct dw_mipi_dsi_phy_ops *phy_ops = dsi->plat_data->phy_ops;
@@ -1027,7 +1027,7 @@ static void dw_mipi_dsi_mode_set(struct dw_mipi_dsi *dsi,
 }
 
 static void dw_mipi_dsi_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge);
 
@@ -1048,7 +1048,7 @@ static void dw_mipi_dsi_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void dw_mipi_dsi_bridge_atomic_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct dw_mipi_dsi *dsi = bridge_to_dsi(bridge);
 
diff --git a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c
index a4bfd3ad166d1..b2a022d266d90 100644
--- a/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c
+++ b/drivers/gpu/drm/bridge/synopsys/dw-mipi-dsi2.c
@@ -750,7 +750,7 @@ static int dw_mipi_dsi2_bridge_atomic_check(struct drm_bridge *bridge,
 }
 
 static void dw_mipi_dsi2_bridge_post_atomic_disable(struct drm_bridge *bridge,
-						    struct drm_atomic_state *state)
+						    struct drm_atomic_commit *state)
 {
 	struct dw_mipi_dsi2 *dsi2 = bridge_to_dsi2(bridge);
 	const struct dw_mipi_dsi2_phy_ops *phy_ops = dsi2->plat_data->phy_ops;
@@ -826,7 +826,7 @@ static void dw_mipi_dsi2_mode_set(struct dw_mipi_dsi2 *dsi2,
 }
 
 static void dw_mipi_dsi2_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-						  struct drm_atomic_state *state)
+						  struct drm_atomic_commit *state)
 {
 	struct dw_mipi_dsi2 *dsi2 = bridge_to_dsi2(bridge);
 
@@ -845,7 +845,7 @@ static void dw_mipi_dsi2_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void dw_mipi_dsi2_bridge_atomic_enable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct dw_mipi_dsi2 *dsi2 = bridge_to_dsi2(bridge);
 
diff --git a/drivers/gpu/drm/bridge/tc358762.c b/drivers/gpu/drm/bridge/tc358762.c
index 98df3e667d4aa..3d75d9cfa45e0 100644
--- a/drivers/gpu/drm/bridge/tc358762.c
+++ b/drivers/gpu/drm/bridge/tc358762.c
@@ -150,7 +150,7 @@ static int tc358762_init(struct tc358762 *ctx)
 }
 
 static void tc358762_post_disable(struct drm_bridge *bridge,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct tc358762 *ctx = bridge_to_tc358762(bridge);
 	int ret;
@@ -173,7 +173,7 @@ static void tc358762_post_disable(struct drm_bridge *bridge,
 }
 
 static void tc358762_pre_enable(struct drm_bridge *bridge,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct tc358762 *ctx = bridge_to_tc358762(bridge);
 	int ret;
@@ -191,7 +191,7 @@ static void tc358762_pre_enable(struct drm_bridge *bridge,
 }
 
 static void tc358762_enable(struct drm_bridge *bridge,
-			    struct drm_atomic_state *state)
+			    struct drm_atomic_commit *state)
 {
 	struct tc358762 *ctx = bridge_to_tc358762(bridge);
 	int ret;
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
index 4097fef4b86b5..7188935fdb826 100644
--- a/drivers/gpu/drm/bridge/tc358767.c
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -1557,7 +1557,7 @@ static int tc_edp_stream_disable(struct tc_data *tc)
 }
 
 static void tc_dpi_bridge_atomic_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
@@ -1572,7 +1572,7 @@ static void tc_dpi_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void tc_dpi_bridge_atomic_disable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
 	int ret;
@@ -1583,7 +1583,7 @@ static void tc_dpi_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void tc_edp_bridge_atomic_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
 	int ret;
@@ -1609,7 +1609,7 @@ static void tc_edp_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void tc_edp_bridge_atomic_disable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
 	int ret;
diff --git a/drivers/gpu/drm/bridge/tc358768.c b/drivers/gpu/drm/bridge/tc358768.c
index fbdc44e162293..0f2820b373029 100644
--- a/drivers/gpu/drm/bridge/tc358768.c
+++ b/drivers/gpu/drm/bridge/tc358768.c
@@ -45,6 +45,9 @@
 
 /* Debug (16-bit addressable) */
 #define TC358768_VBUFCTRL		0x00E0
+#define TC358768_VBUFCTRL_VBUF_EN	BIT(15)
+#define TC358768_VBUFCTRL_TX_EN		BIT(14)
+#define TC358768_VBUFCTRL_MASK		BIT(13)
 #define TC358768_DBG_WIDTH		0x00E2
 #define TC358768_DBG_VBLANK		0x00E4
 #define TC358768_DBG_DATA		0x00E8
@@ -115,7 +118,7 @@
 #define TC358768_DSI_HACT		0x062C
 
 /* TC358768_DSI_CONTROL (0x040C) register */
-#define TC358768_DSI_CONTROL_DIS_MODE	BIT(15)
+#define TC358768_DSI_CONTROL_DSI_MODE	BIT(15)
 #define TC358768_DSI_CONTROL_TXMD	BIT(7)
 #define TC358768_DSI_CONTROL_HSCKMD	BIT(5)
 #define TC358768_DSI_CONTROL_EOTDIS	BIT(0)
@@ -123,7 +126,7 @@
 /* TC358768_DSI_CONFW (0x0500) register */
 #define TC358768_DSI_CONFW_MODE_SET	(5 << 29)
 #define TC358768_DSI_CONFW_MODE_CLR	(6 << 29)
-#define TC358768_DSI_CONFW_ADDR_DSI_CONTROL	(0x3 << 24)
+#define TC358768_DSI_CONFW_ADDR(x)	((x) << 24)
 
 /* TC358768_DSICMD_TX (0x0600) register */
 #define TC358768_DSI_CMDTX_DC_START	BIT(0)
@@ -232,6 +235,36 @@ static void tc358768_update_bits(struct tc358768_priv *priv, u32 reg, u32 mask,
 		tc358768_write(priv, reg, tmp);
 }
 
+static void tc358768_confw_update_bits(struct tc358768_priv *priv, u16 reg,
+				       u16 mask, u16 val)
+{
+	u8 confw_addr;
+	u32 confw_val;
+
+	switch (reg) {
+	case TC358768_DSI_CONTROL:
+		confw_addr = 0x3;
+		break;
+	default:
+		priv->error = -EINVAL;
+		return;
+	}
+
+	if (mask != val) {
+		confw_val = TC358768_DSI_CONFW_MODE_CLR |
+			TC358768_DSI_CONFW_ADDR(confw_addr) |
+			mask;
+		tc358768_write(priv, TC358768_DSI_CONFW, confw_val);
+	}
+
+	if (val & mask) {
+		confw_val = TC358768_DSI_CONFW_MODE_SET |
+			TC358768_DSI_CONFW_ADDR(confw_addr) |
+			(val & mask);
+		tc358768_write(priv, TC358768_DSI_CONFW, confw_val);
+	}
+}
+
 static void tc358768_dsicmd_tx(struct tc358768_priv *priv)
 {
 	u32 val;
@@ -448,6 +481,8 @@ static int tc358768_dsi_host_attach(struct mipi_dsi_host *host,
 						    DRM_MODE_CONNECTOR_DSI);
 		if (IS_ERR(bridge))
 			return PTR_ERR(bridge);
+
+		bridge->pre_enable_prev_first = true;
 	}
 
 	priv->output.dev = dev;
@@ -505,9 +540,21 @@ static ssize_t tc358768_dsi_host_transfer(struct mipi_dsi_host *host,
 		return -ENOTSUPP;
 	}
 
+	if (msg->tx_len > 1024) {
+		dev_warn(priv->dev, "Maximum 1024 byte MIPI tx is supported\n");
+		return -EINVAL;
+	}
+
 	if (msg->tx_len > 8) {
-		dev_warn(priv->dev, "Maximum 8 byte MIPI tx is supported\n");
-		return -ENOTSUPP;
+		u32 confctl;
+
+		tc358768_read(priv, TC358768_CONFCTL, &confctl);
+
+		if (confctl & BIT(6)) {
+			dev_warn(priv->dev,
+				 "Video is currently active. Unable to transmit long command\n");
+			return -EBUSY;
+		}
 	}
 
 	ret = mipi_dsi_create_packet(&packet, msg);
@@ -520,23 +567,66 @@ static ssize_t tc358768_dsi_host_transfer(struct mipi_dsi_host *host,
 		tc358768_write(priv, TC358768_DSICMD_WC, 0);
 		tc358768_write(priv, TC358768_DSICMD_WD0,
 			       (packet.header[2] << 8) | packet.header[1]);
-	} else {
-		int i;
-
+		tc358768_dsicmd_tx(priv);
+	} else if (packet.payload_length <= 8) {
 		tc358768_write(priv, TC358768_DSICMD_TYPE,
 			       (0x40 << 8) | (packet.header[0] & 0x3f));
 		tc358768_write(priv, TC358768_DSICMD_WC, packet.payload_length);
-		for (i = 0; i < packet.payload_length; i += 2) {
+
+		for (int i = 0; i < packet.payload_length; i += 2) {
 			u16 val = packet.payload[i];
 
 			if (i + 1 < packet.payload_length)
 				val |= packet.payload[i + 1] << 8;
-
 			tc358768_write(priv, TC358768_DSICMD_WD0 + i, val);
 		}
-	}
 
-	tc358768_dsicmd_tx(priv);
+		tc358768_dsicmd_tx(priv);
+	} else {
+		unsigned long tx_sleep_us;
+		size_t len;
+
+		/* For packets over 8 bytes we need to use the video buffer */
+		tc358768_write(priv, TC358768_DATAFMT, BIT(0));	/* txdt_en */
+		tc358768_write(priv, TC358768_DSITX_DT, packet.header[0] & 0x3f);
+		tc358768_write(priv, TC358768_CMDBYTE, packet.payload_length);
+		tc358768_write(priv, TC358768_VBUFCTRL, TC358768_VBUFCTRL_VBUF_EN);
+
+		/*
+		 * Write the payload in 2-byte chunks, and pad with zeroes to
+		 * align to 4 bytes.
+		 */
+		len = ALIGN(packet.payload_length, 4);
+
+		for (int i = 0; i < len; i += 2) {
+			u16 val = 0;
+
+			if (i < packet.payload_length)
+				val |= packet.payload[i];
+			if (i + 1 < packet.payload_length)
+				val |= packet.payload[i + 1] << 8;
+
+			tc358768_write(priv, TC358768_DBG_DATA, val);
+		}
+
+		/* Start transmission */
+		tc358768_write(priv, TC358768_VBUFCTRL,
+			       TC358768_VBUFCTRL_VBUF_EN |
+			       TC358768_VBUFCTRL_TX_EN |
+			       TC358768_VBUFCTRL_MASK);
+
+		/*
+		 * The TC358768 spec says to wait until the transmission has
+		 * been finished, estimating the sleep time based on the payload
+		 * and clock rates. We use a simple safe estimate of 2us per
+		 * byte (LP mode transmission).
+		 */
+		tx_sleep_us = packet.payload_length * 2;
+		usleep_range(tx_sleep_us, tx_sleep_us * 2);
+
+		tc358768_write(priv, TC358768_VBUFCTRL, TC358768_VBUFCTRL_MASK);
+		tc358768_write(priv, TC358768_VBUFCTRL, 0); /* Stop transmission */
+	}
 
 	ret = tc358768_clear_error(priv);
 	if (ret)
@@ -582,7 +672,7 @@ tc358768_bridge_mode_valid(struct drm_bridge *bridge,
 }
 
 static void tc358768_bridge_atomic_disable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct tc358768_priv *priv = bridge_to_tc358768(bridge);
 	int ret;
@@ -605,7 +695,7 @@ static void tc358768_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void tc358768_bridge_atomic_post_disable(struct drm_bridge *bridge,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct tc358768_priv *priv = bridge_to_tc358768(bridge);
 
@@ -686,12 +776,11 @@ static u32 tc358768_dsi_bytes_to_ns(struct tc358768_priv *priv, u32 val)
 }
 
 static void tc358768_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct tc358768_priv *priv = bridge_to_tc358768(bridge);
 	struct mipi_dsi_device *dsi_dev = priv->output.dev;
-	unsigned long mode_flags = dsi_dev->mode_flags;
-	u32 val, val2, lptxcnt, hact, data_type;
+	u32 val, mask, val2, lptxcnt, hact;
 	s32 raw_val;
 	struct drm_crtc_state *crtc_state;
 	struct drm_connector_state *conn_state;
@@ -712,11 +801,6 @@ static void tc358768_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 	u32 dsi_vsdly;
 	const u32 internal_dly = 40;
 
-	if (mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS) {
-		dev_warn_once(dev, "Non-continuous mode unimplemented, falling back to continuous\n");
-		mode_flags &= ~MIPI_DSI_CLOCK_NON_CONTINUOUS;
-	}
-
 	tc358768_hw_enable(priv);
 
 	ret = tc358768_sw_reset(priv);
@@ -726,6 +810,9 @@ static void tc358768_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 		return;
 	}
 
+	/* Release RstPtr so that the video buffer can be used for DSI commands */
+	tc358768_update_bits(priv, TC358768_PP_MISC, BIT(14), 0);
+
 	connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder);
 	conn_state = drm_atomic_get_new_connector_state(state, connector);
 	crtc_state = drm_atomic_get_new_crtc_state(state, conn_state->crtc);
@@ -742,30 +829,20 @@ static void tc358768_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 	dsiclk = priv->dsiclk;
 	hsbyteclk = dsiclk / 4;
 
-	/* Data Format Control Register */
-	val = BIT(2) | BIT(1) | BIT(0); /* rdswap_en | dsitx_en | txdt_en */
 	switch (dsi_dev->format) {
 	case MIPI_DSI_FMT_RGB888:
-		val |= (0x3 << 4);
 		hact = vm.hactive * 3;
-		data_type = MIPI_DSI_PACKED_PIXEL_STREAM_24;
 		break;
 	case MIPI_DSI_FMT_RGB666:
-		val |= (0x4 << 4);
 		hact = vm.hactive * 3;
-		data_type = MIPI_DSI_PACKED_PIXEL_STREAM_18;
 		break;
 
 	case MIPI_DSI_FMT_RGB666_PACKED:
-		val |= (0x4 << 4) | BIT(3);
 		hact = vm.hactive * 18 / 8;
-		data_type = MIPI_DSI_PIXEL_STREAM_3BYTE_18;
 		break;
 
 	case MIPI_DSI_FMT_RGB565:
-		val |= (0x5 << 4);
 		hact = vm.hactive * 2;
-		data_type = MIPI_DSI_PACKED_PIXEL_STREAM_16;
 		break;
 	default:
 		dev_err(dev, "Invalid data format (%u)\n",
@@ -921,9 +998,6 @@ static void tc358768_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 	/* VSDly[9:0] */
 	tc358768_write(priv, TC358768_VSDLY, dsi_vsdly - internal_dly);
 
-	tc358768_write(priv, TC358768_DATAFMT, val);
-	tc358768_write(priv, TC358768_DSITX_DT, data_type);
-
 	/* Enable D-PHY (HiZ->LP11) */
 	tc358768_write(priv, TC358768_CLW_CNTRL, 0x0000);
 	/* Enable lanes */
@@ -1000,7 +1074,7 @@ static void tc358768_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 	tc358768_write(priv, TC358768_HSTXVREGEN, val);
 
 	tc358768_write(priv, TC358768_TXOPTIONCNTRL,
-		       (mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS) ? 0 : BIT(0));
+		       (dsi_dev->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS) ? 0 : BIT(0));
 
 	/* TXTAGOCNT[26:16] RXTASURECNT[10:0] */
 	val = tc358768_ps_to_ns((lptxcnt + 1) * hsbyteclk_ps * 4);
@@ -1063,35 +1137,65 @@ static void tc358768_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 	tc358768_write(priv, TC358768_DSI_START, 0x1);
 
 	/* Configure DSI_Control register */
-	val = TC358768_DSI_CONFW_MODE_CLR | TC358768_DSI_CONFW_ADDR_DSI_CONTROL;
-	val |= TC358768_DSI_CONTROL_TXMD | TC358768_DSI_CONTROL_HSCKMD |
-	       0x3 << 1 | TC358768_DSI_CONTROL_EOTDIS;
-	tc358768_write(priv, TC358768_DSI_CONFW, val);
+	val = (dsi_dev->lanes - 1) << 1;
 
-	val = TC358768_DSI_CONFW_MODE_SET | TC358768_DSI_CONFW_ADDR_DSI_CONTROL;
-	val |= (dsi_dev->lanes - 1) << 1;
+	if (!(dsi_dev->mode_flags & MIPI_DSI_MODE_LPM))
+		val |= TC358768_DSI_CONTROL_TXMD;
 
-	val |= TC358768_DSI_CONTROL_TXMD;
-
-	if (!(mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS))
+	if (!(dsi_dev->mode_flags & MIPI_DSI_CLOCK_NON_CONTINUOUS))
 		val |= TC358768_DSI_CONTROL_HSCKMD;
 
 	if (dsi_dev->mode_flags & MIPI_DSI_MODE_NO_EOT_PACKET)
 		val |= TC358768_DSI_CONTROL_EOTDIS;
 
-	tc358768_write(priv, TC358768_DSI_CONFW, val);
+	mask = TC358768_DSI_CONTROL_TXMD | TC358768_DSI_CONTROL_HSCKMD |
+	       0x3 << 1 | TC358768_DSI_CONTROL_EOTDIS;
+
+	tc358768_confw_update_bits(priv, TC358768_DSI_CONTROL, mask, val);
 
-	val = TC358768_DSI_CONFW_MODE_CLR | TC358768_DSI_CONFW_ADDR_DSI_CONTROL;
-	val |= TC358768_DSI_CONTROL_DIS_MODE; /* DSI mode */
-	tc358768_write(priv, TC358768_DSI_CONFW, val);
+	tc358768_confw_update_bits(priv, TC358768_DSI_CONTROL,
+				   TC358768_DSI_CONTROL_DSI_MODE, 0);
 
 	ret = tc358768_clear_error(priv);
 	if (ret)
 		dev_err(dev, "Bridge pre_enable failed: %d\n", ret);
 }
 
+static void tc358768_config_video_format(struct tc358768_priv *priv)
+{
+	struct mipi_dsi_device *dsi_dev = priv->output.dev;
+	u32 val, data_type;
+
+	/* Data Format Control Register */
+	val = BIT(2) | BIT(1) | BIT(0); /* rdswap_en | dsitx_en | txdt_en */
+	switch (dsi_dev->format) {
+	case MIPI_DSI_FMT_RGB888:
+		val |= (0x3 << 4);
+		data_type = MIPI_DSI_PACKED_PIXEL_STREAM_24;
+		break;
+	case MIPI_DSI_FMT_RGB666:
+		val |= (0x4 << 4);
+		data_type = MIPI_DSI_PACKED_PIXEL_STREAM_18;
+		break;
+	case MIPI_DSI_FMT_RGB666_PACKED:
+		val |= (0x4 << 4) | BIT(3);
+		data_type = MIPI_DSI_PIXEL_STREAM_3BYTE_18;
+		break;
+	case MIPI_DSI_FMT_RGB565:
+		val |= (0x5 << 4);
+		data_type = MIPI_DSI_PACKED_PIXEL_STREAM_16;
+		break;
+	default:
+		dev_err(priv->dev, "Invalid data format (%u)\n", dsi_dev->format);
+		return;
+	}
+
+	tc358768_write(priv, TC358768_DATAFMT, val);
+	tc358768_write(priv, TC358768_DSITX_DT, data_type);
+}
+
 static void tc358768_bridge_atomic_enable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct tc358768_priv *priv = bridge_to_tc358768(bridge);
 	int ret;
@@ -1101,6 +1205,14 @@ static void tc358768_bridge_atomic_enable(struct drm_bridge *bridge,
 		return;
 	}
 
+	/* Configure video format registers */
+	tc358768_config_video_format(priv);
+
+	/* Enable HS mode for video TX */
+	tc358768_confw_update_bits(priv, TC358768_DSI_CONTROL,
+				   TC358768_DSI_CONTROL_TXMD,
+				   TC358768_DSI_CONTROL_TXMD);
+
 	/* clear FrmStop and RstPtr */
 	tc358768_update_bits(priv, TC358768_PP_MISC, 0x3 << 14, 0);
 
diff --git a/drivers/gpu/drm/bridge/tc358775.c b/drivers/gpu/drm/bridge/tc358775.c
index 366b12db0e7cb..e067b671cecee 100644
--- a/drivers/gpu/drm/bridge/tc358775.c
+++ b/drivers/gpu/drm/bridge/tc358775.c
@@ -287,7 +287,7 @@ static inline struct tc_data *bridge_to_tc(struct drm_bridge *b)
 }
 
 static void tc_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
 	struct device *dev = &tc->dsi->dev;
@@ -311,7 +311,7 @@ static void tc_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void tc_bridge_atomic_post_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
 	struct device *dev = &tc->dsi->dev;
@@ -371,7 +371,7 @@ static void d2l_write(struct i2c_client *i2c, u16 addr, u32 val)
 }
 
 static void tc_bridge_atomic_enable(struct drm_bridge *bridge,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct tc_data *tc = bridge_to_tc(bridge);
 	u32 hback_porch, hsync_len, hfront_porch, hactive, htime1, htime2;
diff --git a/drivers/gpu/drm/bridge/ti-dlpc3433.c b/drivers/gpu/drm/bridge/ti-dlpc3433.c
index b07f7c9d58909..9c35eade0a78e 100644
--- a/drivers/gpu/drm/bridge/ti-dlpc3433.c
+++ b/drivers/gpu/drm/bridge/ti-dlpc3433.c
@@ -105,7 +105,7 @@ static const struct regmap_config dlpc_regmap_config = {
 };
 
 static void dlpc_atomic_enable(struct drm_bridge *bridge,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct dlpc *dlpc = bridge_to_dlpc(bridge);
 	struct device *dev = dlpc->dev;
@@ -170,7 +170,7 @@ static void dlpc_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void dlpc_atomic_pre_enable(struct drm_bridge *bridge,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct dlpc *dlpc = bridge_to_dlpc(bridge);
 	int ret;
@@ -193,7 +193,7 @@ static void dlpc_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void dlpc_atomic_post_disable(struct drm_bridge *bridge,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct dlpc *dlpc = bridge_to_dlpc(bridge);
 
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi83.c b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
index 17a885244e1e1..42b451432bbb6 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi83.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi83.c
@@ -114,6 +114,7 @@
 #define REG_VID_CHA_HORIZONTAL_FRONT_PORCH	0x38
 #define REG_VID_CHA_VERTICAL_FRONT_PORCH	0x3a
 #define REG_VID_CHA_TEST_PATTERN		0x3c
+#define  REG_VID_CHA_TEST_PATTERN_EN		BIT(4)
 /* IRQ registers */
 #define REG_IRQ_GLOBAL				0xe0
 #define  REG_IRQ_GLOBAL_IRQ_EN			BIT(0)
@@ -134,6 +135,9 @@
 #define  REG_IRQ_STAT_CHA_SOT_BIT_ERR		BIT(2)
 #define  REG_IRQ_STAT_CHA_PLL_UNLOCK		BIT(0)
 
+static bool sn65dsi83_test_pattern;
+module_param_named(test_pattern, sn65dsi83_test_pattern, bool, 0644);
+
 enum sn65dsi83_channel {
 	CHANNEL_A,
 	CHANNEL_B
@@ -514,7 +518,7 @@ static void sn65dsi83_release_resources(void *data)
 }
 
 static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct sn65dsi83 *ctx = bridge_to_sn65dsi83(bridge);
 	const unsigned int dual_factor = ctx->lvds_dual_link ? 2 : 1;
@@ -523,6 +527,7 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 	const struct drm_display_mode *mode;
 	struct drm_connector *connector;
 	struct drm_crtc *crtc;
+	bool test_pattern = sn65dsi83_test_pattern;
 	bool lvds_format_24bpp;
 	bool lvds_format_jeida;
 	unsigned int pval;
@@ -645,7 +650,11 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 			  REG_LVDS_LANE_CHB_LVDS_TERM : 0));
 	regmap_write(ctx->regmap, REG_LVDS_CM, 0x00);
 
-	le16val = cpu_to_le16(mode->hdisplay);
+	/*
+	 * Active line length needs to be halved for test pattern
+	 * generation in dual LVDS output.
+	 */
+	le16val = cpu_to_le16(mode->hdisplay / (test_pattern ? dual_factor : 1));
 	regmap_bulk_write(ctx->regmap, REG_VID_CHA_ACTIVE_LINE_LENGTH_LOW,
 			  &le16val, 2);
 	le16val = cpu_to_le16(mode->vdisplay);
@@ -668,7 +677,8 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 		     (mode->hsync_start - mode->hdisplay) / dual_factor);
 	regmap_write(ctx->regmap, REG_VID_CHA_VERTICAL_FRONT_PORCH,
 		     mode->vsync_start - mode->vdisplay);
-	regmap_write(ctx->regmap, REG_VID_CHA_TEST_PATTERN, 0x00);
+	regmap_write(ctx->regmap, REG_VID_CHA_TEST_PATTERN,
+		     test_pattern ? REG_VID_CHA_TEST_PATTERN_EN : 0);
 
 	/* Enable PLL */
 	regmap_write(ctx->regmap, REG_RC_PLL_EN, REG_RC_PLL_EN_PLL_EN);
@@ -696,7 +706,7 @@ static void sn65dsi83_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void sn65dsi83_atomic_enable(struct drm_bridge *bridge,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct sn65dsi83 *ctx = bridge_to_sn65dsi83(bridge);
 	unsigned int pval;
@@ -728,7 +738,7 @@ static void sn65dsi83_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void sn65dsi83_atomic_disable(struct drm_bridge *bridge,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct sn65dsi83 *ctx = bridge_to_sn65dsi83(bridge);
 	int idx;
diff --git a/drivers/gpu/drm/bridge/ti-sn65dsi86.c b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
index 98d64ad791d04..0884ec459cc54 100644
--- a/drivers/gpu/drm/bridge/ti-sn65dsi86.c
+++ b/drivers/gpu/drm/bridge/ti-sn65dsi86.c
@@ -278,7 +278,7 @@ static void ti_sn65dsi86_write_u16(struct ti_sn65dsi86 *pdata,
 
 static struct drm_display_mode *
 get_new_adjusted_display_mode(struct drm_bridge *bridge,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector =
 		drm_atomic_get_new_connector_for_encoder(state, bridge->encoder);
@@ -291,7 +291,7 @@ get_new_adjusted_display_mode(struct drm_bridge *bridge,
 }
 
 static u32 ti_sn_bridge_get_dsi_freq(struct ti_sn65dsi86 *pdata,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	u32 bit_rate_khz, clk_freq_khz;
 	struct drm_display_mode *mode =
@@ -323,7 +323,7 @@ static const u32 ti_sn_bridge_dsiclk_lut[] = {
 };
 
 static void ti_sn_bridge_set_refclk_freq(struct ti_sn65dsi86 *pdata,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	int i;
 	u32 refclk_rate;
@@ -361,7 +361,7 @@ static void ti_sn_bridge_set_refclk_freq(struct ti_sn65dsi86 *pdata,
 }
 
 static void ti_sn65dsi86_enable_comms(struct ti_sn65dsi86 *pdata,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	mutex_lock(&pdata->comms_mutex);
 
@@ -776,8 +776,6 @@ static int ti_sn_bridge_attach(struct drm_bridge *bridge,
 		goto err_initted_aux;
 	}
 
-	drm_connector_attach_encoder(pdata->connector, pdata->bridge.encoder);
-
 	return 0;
 
 err_initted_aux:
@@ -826,7 +824,7 @@ ti_sn_bridge_mode_valid(struct drm_bridge *bridge,
 }
 
 static void ti_sn_bridge_atomic_disable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge);
 
@@ -835,7 +833,7 @@ static void ti_sn_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void ti_sn_bridge_set_dsi_rate(struct ti_sn65dsi86 *pdata,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	unsigned int bit_rate_mhz, clk_freq_mhz;
 	unsigned int val;
@@ -871,7 +869,7 @@ static const unsigned int ti_sn_bridge_dp_rate_lut[] = {
 };
 
 static int ti_sn_bridge_calc_min_dp_rate_idx(struct ti_sn65dsi86 *pdata,
-					     struct drm_atomic_state *state,
+					     struct drm_atomic_commit *state,
 					     unsigned int bpp)
 {
 	unsigned int bit_rate_khz, dp_rate_mhz;
@@ -977,7 +975,7 @@ static unsigned int ti_sn_bridge_read_valid_rates(struct ti_sn65dsi86 *pdata)
 }
 
 static void ti_sn_bridge_set_video_timings(struct ti_sn65dsi86 *pdata,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_display_mode *mode =
 		get_new_adjusted_display_mode(&pdata->bridge, state);
@@ -1090,7 +1088,7 @@ static int ti_sn_link_training(struct ti_sn65dsi86 *pdata, int dp_rate_idx,
 }
 
 static void ti_sn_bridge_atomic_enable(struct drm_bridge *bridge,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge);
 	struct drm_connector *connector;
@@ -1181,7 +1179,7 @@ static void ti_sn_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void ti_sn_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge);
 
@@ -1195,7 +1193,7 @@ static void ti_sn_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void ti_sn_bridge_atomic_post_disable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct ti_sn65dsi86 *pdata = bridge_to_ti_sn65dsi86(bridge);
 
diff --git a/drivers/gpu/drm/bridge/ti-tdp158.c b/drivers/gpu/drm/bridge/ti-tdp158.c
index 27053d020df7c..3a19967f213fb 100644
--- a/drivers/gpu/drm/bridge/ti-tdp158.c
+++ b/drivers/gpu/drm/bridge/ti-tdp158.c
@@ -19,7 +19,7 @@ struct tdp158 {
 };
 
 static void tdp158_enable(struct drm_bridge *bridge,
-			  struct drm_atomic_state *state)
+			  struct drm_atomic_commit *state)
 {
 	int err;
 	struct tdp158 *tdp158 = bridge->driver_private;
@@ -36,7 +36,7 @@ static void tdp158_enable(struct drm_bridge *bridge,
 }
 
 static void tdp158_disable(struct drm_bridge *bridge,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct tdp158 *tdp158 = bridge->driver_private;
 
diff --git a/drivers/gpu/drm/bridge/ti-tfp410.c b/drivers/gpu/drm/bridge/ti-tfp410.c
index 3b6b0e92cf89f..bf4ab4eaf2697 100644
--- a/drivers/gpu/drm/bridge/ti-tfp410.c
+++ b/drivers/gpu/drm/bridge/ti-tfp410.c
@@ -114,7 +114,7 @@ static void tfp410_hpd_callback(void *arg, enum drm_connector_status status)
 {
 	struct tfp410 *dvi = arg;
 
-	mod_delayed_work(system_wq, &dvi->hpd_work,
+	mod_delayed_work(system_percpu_wq, &dvi->hpd_work,
 			 msecs_to_jiffies(HOTPLUG_DEBOUNCE_MS));
 }
 
diff --git a/drivers/gpu/drm/bridge/waveshare-dsi.c b/drivers/gpu/drm/bridge/waveshare-dsi.c
index 32d40414adb9f..ded57f298d640 100644
--- a/drivers/gpu/drm/bridge/waveshare-dsi.c
+++ b/drivers/gpu/drm/bridge/waveshare-dsi.c
@@ -177,7 +177,7 @@ static int ws_bridge_probe(struct i2c_client *i2c)
 	regmap_write(ws->reg_map, 0xc2, 0x01);
 	regmap_write(ws->reg_map, 0xac, 0x01);
 
-	ws->bridge.type = DRM_MODE_CONNECTOR_DPI;
+	ws->bridge.type = (uintptr_t)i2c_get_match_data(i2c);
 	ws->bridge.of_node = dev->of_node;
 	devm_drm_bridge_add(dev, &ws->bridge);
 
@@ -185,7 +185,8 @@ static int ws_bridge_probe(struct i2c_client *i2c)
 }
 
 static const struct of_device_id ws_bridge_of_ids[] = {
-	{.compatible = "waveshare,dsi2dpi",},
+	{.compatible = "waveshare,dsi2dpi", .data = (void *)DRM_MODE_CONNECTOR_DPI, },
+	{.compatible = "waveshare,dsi2lvds", .data = (void *)DRM_MODE_CONNECTOR_LVDS, },
 	{ }
 };
 
diff --git a/drivers/gpu/drm/ci/xfails/vkms-none-skips.txt b/drivers/gpu/drm/ci/xfails/vkms-none-skips.txt
index 716d2d4e452d8..afcc09d1683d8 100644
--- a/drivers/gpu/drm/ci/xfails/vkms-none-skips.txt
+++ b/drivers/gpu/drm/ci/xfails/vkms-none-skips.txt
@@ -288,8 +288,8 @@ kms_cursor_edge_walk@128x128-left-edge
 #  ? __pfx_drm_property_free_blob+0x10/0x10
 #  ? vkms_atomic_crtc_destroy_state+0x31/0x40 [vkms]
 #  ? vkms_atomic_crtc_destroy_state+0x10/0x40 [vkms]
-#  drm_atomic_state_default_clear+0x137/0x2f0
-#  __drm_atomic_state_free+0x6c/0xb0
+#  drm_atomic_commit_default_clear+0x137/0x2f0
+#  __drm_atomic_commit_free+0x6c/0xb0
 #  drm_atomic_helper_update_plane+0x100/0x150
 #  drm_mode_cursor_universal+0x10e/0x270
 #  drm_mode_cursor_common+0x115/0x240
@@ -394,8 +394,8 @@ kms_cursor_edge_walk@128x128-left-edge
 #  ? __pfx_drm_property_free_blob+0x10/0x10
 #  ? vkms_atomic_crtc_destroy_state+0x31/0x40 [vkms]
 #  ? vkms_atomic_crtc_destroy_state+0x10/0x40 [vkms]
-#  drm_atomic_state_default_clear+0x137/0x2f0
-#  __drm_atomic_state_free+0x6c/0xb0
+#  drm_atomic_commit_default_clear+0x137/0x2f0
+#  __drm_atomic_commit_free+0x6c/0xb0
 #  drm_atomic_helper_update_plane+0x100/0x150
 #  drm_mode_cursor_universal+0x10e/0x270
 #  drm_mode_cursor_common+0x115/0x240
@@ -500,8 +500,8 @@ kms_cursor_edge_walk@128x128-left-edge
 #  ? __pfx_drm_property_free_blob+0x10/0x10
 #  ? vkms_atomic_crtc_destroy_state+0x31/0x40 [vkms]
 #  ? vkms_atomic_crtc_destroy_state+0x10/0x40 [vkms]
-#  drm_atomic_state_default_clear+0x137/0x2f0
-#  __drm_atomic_state_free+0x6c/0xb0
+#  drm_atomic_commit_default_clear+0x137/0x2f0
+#  __drm_atomic_commit_free+0x6c/0xb0
 #  drm_atomic_helper_update_plane+0x100/0x150
 #  drm_mode_cursor_universal+0x10e/0x270
 #  drm_mode_cursor_common+0x115/0x240
@@ -606,8 +606,8 @@ kms_cursor_edge_walk@128x128-left-edge
 #  ? __pfx_drm_property_free_blob+0x10/0x10
 #  ? vkms_atomic_crtc_destroy_state+0x31/0x40 [vkms]
 #  ? vkms_atomic_crtc_destroy_state+0x10/0x40 [vkms]
-#  drm_atomic_state_default_clear+0x137/0x2f0
-#  __drm_atomic_state_free+0x6c/0xb0
+#  drm_atomic_commit_default_clear+0x137/0x2f0
+#  __drm_atomic_commit_free+0x6c/0xb0
 #  drm_atomic_helper_update_plane+0x100/0x150
 #  drm_mode_cursor_universal+0x10e/0x270
 #  drm_mode_cursor_common+0x115/0x240
@@ -712,8 +712,8 @@ kms_cursor_edge_walk@128x128-left-edge
 #  ? __pfx_drm_property_free_blob+0x10/0x10
 #  ? vkms_atomic_crtc_destroy_state+0x31/0x40 [vkms]
 #  ? vkms_atomic_crtc_destroy_state+0x10/0x40 [vkms]
-#  drm_atomic_state_default_clear+0x137/0x2f0
-#  __drm_atomic_state_free+0x6c/0xb0
+#  drm_atomic_commit_default_clear+0x137/0x2f0
+#  __drm_atomic_commit_free+0x6c/0xb0
 #  drm_atomic_helper_update_plane+0x100/0x150
 #  drm_mode_cursor_universal+0x10e/0x270
 #  drm_mode_cursor_common+0x115/0x240
diff --git a/drivers/gpu/drm/display/drm_bridge_connector.c b/drivers/gpu/drm/display/drm_bridge_connector.c
index 39cc18f78eda1..649969fca1413 100644
--- a/drivers/gpu/drm/display/drm_bridge_connector.c
+++ b/drivers/gpu/drm/display/drm_bridge_connector.c
@@ -371,7 +371,7 @@ drm_bridge_connector_mode_valid(struct drm_connector *connector,
 }
 
 static int drm_bridge_connector_atomic_check(struct drm_connector *connector,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_bridge_connector *bridge_connector =
 		to_drm_bridge_connector(connector);
@@ -773,8 +773,11 @@ static void drm_bridge_connector_put_bridges(struct drm_device *dev, void *data)
  * @drm: the DRM device
  * @encoder: the encoder where the bridge chain starts
  *
- * Allocate, initialise and register a &drm_bridge_connector with the @drm
- * device. The connector is associated with a chain of bridges that starts at
+ * Create a new &drm_bridge_connector for the @drm device. The connector is
+ * allocated, initialised, registered with the @drm device and attached to
+ * @encoder.
+ *
+ * The connector is associated with a chain of bridges that starts at
  * the @encoder. All bridges in the chain shall report bridge operation flags
  * (&drm_bridge->ops) and bridge output type (&drm_bridge->type), and none of
  * them may create a DRM connector directly.
@@ -1055,6 +1058,10 @@ struct drm_connector *drm_bridge_connector_init(struct drm_device *drm,
 	    IS_ENABLED(CONFIG_DRM_DISPLAY_HDCP_HELPER))
 		drm_connector_attach_content_protection_property(connector, true);
 
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret)
+		return ERR_PTR(ret);
+
 	return connector;
 }
 EXPORT_SYMBOL_GPL(drm_bridge_connector_init);
diff --git a/drivers/gpu/drm/display/drm_dp_helper.c b/drivers/gpu/drm/display/drm_dp_helper.c
index a697cc227e289..a6067471dd563 100644
--- a/drivers/gpu/drm/display/drm_dp_helper.c
+++ b/drivers/gpu/drm/display/drm_dp_helper.c
@@ -3487,31 +3487,64 @@ static const char *dp_content_type_get_name(enum dp_content_type content_type)
 	}
 }
 
+static const char *dp_sdp_type_get_name(unsigned char type)
+{
+	switch (type) {
+	case DP_SDP_AUDIO_TIMESTAMP:
+		return "Audio_TimeStamp";
+	case DP_SDP_AUDIO_STREAM:
+		return "Audio_Stream";
+	case DP_SDP_EXTENSION:
+		return "Extension";
+	case DP_SDP_AUDIO_COPYMANAGEMENT:
+		return "Audio_CopyManagement";
+	case DP_SDP_ISRC:
+		return "ISRC";
+	case DP_SDP_VSC:
+		return "VSC";
+	case DP_SDP_PPS:
+		return "PPS";
+	case DP_SDP_VSC_EXT_VESA:
+		return "VSC_EXT_VESA";
+	case DP_SDP_VSC_EXT_CEA:
+		return "VSC_EXT_CEA";
+	case DP_SDP_ADAPTIVE_SYNC:
+		return "Adaptive-Sync";
+	default:
+		return "Unknown";
+	}
+}
+
 void drm_dp_vsc_sdp_log(struct drm_printer *p, const struct drm_dp_vsc_sdp *vsc)
 {
-	drm_printf(p, "DP SDP: VSC, revision %u, length %u\n",
-		   vsc->revision, vsc->length);
-	drm_printf(p, "    pixelformat: %s\n",
-		   dp_pixelformat_get_name(vsc->pixelformat));
-	drm_printf(p, "    colorimetry: %s\n",
-		   dp_colorimetry_get_name(vsc->pixelformat, vsc->colorimetry));
-	drm_printf(p, "    bpc: %u\n", vsc->bpc);
-	drm_printf(p, "    dynamic range: %s\n",
-		   dp_dynamic_range_get_name(vsc->dynamic_range));
-	drm_printf(p, "    content type: %s\n",
-		   dp_content_type_get_name(vsc->content_type));
+	drm_printf(p, "DP SDP: %s, revision %u, length %u\n",
+		   dp_sdp_type_get_name(vsc->sdp_type), vsc->revision, vsc->length);
+
+	drm_printf_indent(p, 1, "pixelformat: %s\n",
+			  dp_pixelformat_get_name(vsc->pixelformat));
+	drm_printf_indent(p, 1, "colorimetry: %s\n",
+			  dp_colorimetry_get_name(vsc->pixelformat, vsc->colorimetry));
+	drm_printf_indent(p, 1, "bpc: %u\n", vsc->bpc);
+	drm_printf_indent(p, 1, "dynamic range: %s\n",
+			  dp_dynamic_range_get_name(vsc->dynamic_range));
+	drm_printf_indent(p, 1, "content type: %s\n",
+			  dp_content_type_get_name(vsc->content_type));
 }
 EXPORT_SYMBOL(drm_dp_vsc_sdp_log);
 
 void drm_dp_as_sdp_log(struct drm_printer *p, const struct drm_dp_as_sdp *as_sdp)
 {
-	drm_printf(p, "DP SDP: AS_SDP, revision %u, length %u\n",
-		   as_sdp->revision, as_sdp->length);
-	drm_printf(p, "    vtotal: %d\n", as_sdp->vtotal);
-	drm_printf(p, "    target_rr: %d\n", as_sdp->target_rr);
-	drm_printf(p, "    duration_incr_ms: %d\n", as_sdp->duration_incr_ms);
-	drm_printf(p, "    duration_decr_ms: %d\n", as_sdp->duration_decr_ms);
-	drm_printf(p, "    operation_mode: %d\n", as_sdp->mode);
+	drm_printf(p, "DP SDP: %s, revision %u, length %u\n",
+		   dp_sdp_type_get_name(as_sdp->sdp_type), as_sdp->revision, as_sdp->length);
+
+	drm_printf_indent(p, 1, "vtotal: %d\n", as_sdp->vtotal);
+	drm_printf_indent(p, 1, "target rr: %d\n", as_sdp->target_rr);
+	drm_printf_indent(p, 1, "duration increase ms: %d\n", as_sdp->duration_incr_ms);
+	drm_printf_indent(p, 1, "duration decrease ms: %d\n", as_sdp->duration_decr_ms);
+	drm_printf_indent(p, 1, "operation mode: %d\n", as_sdp->mode);
+	drm_printf_indent(p, 1, "target rr divider: %s\n",
+			  as_sdp->target_rr_divider ? "1.001" : "1.000");
+	drm_printf_indent(p, 1, "coasting vtotal: %d\n", as_sdp->coasting_vtotal);
 }
 EXPORT_SYMBOL(drm_dp_as_sdp_log);
 
@@ -4626,8 +4659,6 @@ static const struct backlight_ops dp_aux_bl_ops = {
  * Backlight will then be handled transparently without requiring
  * any intervention from the driver.
  *
- * drm_panel_dp_aux_backlight() must be called after the call to drm_panel_init().
- *
  * Return: 0 on success or a negative error code on failure.
  */
 int drm_panel_dp_aux_backlight(struct drm_panel *panel, struct drm_dp_aux *aux)
diff --git a/drivers/gpu/drm/display/drm_dp_mst_topology.c b/drivers/gpu/drm/display/drm_dp_mst_topology.c
index 8757972e8e242..4de36fda05440 100644
--- a/drivers/gpu/drm/display/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/display/drm_dp_mst_topology.c
@@ -789,6 +789,12 @@ static bool drm_dp_sideband_append_payload(struct drm_dp_sideband_msg_rx *msg,
 {
 	u8 crc4;
 
+	/* curchunk_len must be >= 1 (min 1 CRC byte) and fit in chunk[] */
+	if (!msg->curchunk_len ||
+	    msg->curchunk_len > ARRAY_SIZE(msg->chunk) ||
+	    msg->curchunk_idx + replybuflen > ARRAY_SIZE(msg->chunk))
+		return false;
+
 	memcpy(&msg->chunk[msg->curchunk_idx], replybuf, replybuflen);
 	msg->curchunk_idx += replybuflen;
 
@@ -799,6 +805,9 @@ static bool drm_dp_sideband_append_payload(struct drm_dp_sideband_msg_rx *msg,
 			print_hex_dump(KERN_DEBUG, "wrong crc",
 				       DUMP_PREFIX_NONE, 16, 1,
 				       msg->chunk,  msg->curchunk_len, false);
+		/* Guard against accumulated msg[] overflow */
+		if (msg->curlen + msg->curchunk_len - 1 > ARRAY_SIZE(msg->msg))
+			return false;
 		/* copy chunk into bigger msg */
 		memcpy(&msg->msg[msg->curlen], msg->chunk, msg->curchunk_len - 1);
 		msg->curlen += msg->curchunk_len - 1;
@@ -871,7 +880,7 @@ static bool drm_dp_sideband_parse_remote_dpcd_read(struct drm_dp_sideband_msg_rx
 		goto fail_len;
 	repmsg->u.remote_dpcd_read_ack.num_bytes = raw->msg[idx];
 	idx++;
-	if (idx > raw->curlen)
+	if (idx + repmsg->u.remote_dpcd_read_ack.num_bytes > raw->curlen)
 		goto fail_len;
 
 	memcpy(repmsg->u.remote_dpcd_read_ack.bytes, &raw->msg[idx], repmsg->u.remote_dpcd_read_ack.num_bytes);
@@ -907,7 +916,9 @@ static bool drm_dp_sideband_parse_remote_i2c_read_ack(struct drm_dp_sideband_msg
 		goto fail_len;
 	repmsg->u.remote_i2c_read_ack.num_bytes = raw->msg[idx];
 	idx++;
-	/* TODO check */
+	if (idx + repmsg->u.remote_i2c_read_ack.num_bytes > raw->curlen)
+		goto fail_len;
+
 	memcpy(repmsg->u.remote_i2c_read_ack.bytes, &raw->msg[idx], repmsg->u.remote_i2c_read_ack.num_bytes);
 	return true;
 fail_len:
@@ -923,16 +934,13 @@ static bool drm_dp_sideband_parse_enum_path_resources_ack(struct drm_dp_sideband
 	repmsg->u.path_resources.port_number = (raw->msg[idx] >> 4) & 0xf;
 	repmsg->u.path_resources.fec_capable = raw->msg[idx] & 0x1;
 	idx++;
-	if (idx > raw->curlen)
+	if (idx + 2 > raw->curlen)
 		goto fail_len;
 	repmsg->u.path_resources.full_payload_bw_number = (raw->msg[idx] << 8) | (raw->msg[idx+1]);
 	idx += 2;
-	if (idx > raw->curlen)
+	if (idx + 2 > raw->curlen)
 		goto fail_len;
 	repmsg->u.path_resources.avail_payload_bw_number = (raw->msg[idx] << 8) | (raw->msg[idx+1]);
-	idx += 2;
-	if (idx > raw->curlen)
-		goto fail_len;
 	return true;
 fail_len:
 	DRM_DEBUG_KMS("enum resource parse length fail %d %d\n", idx, raw->curlen);
@@ -950,12 +958,9 @@ static bool drm_dp_sideband_parse_allocate_payload_ack(struct drm_dp_sideband_ms
 		goto fail_len;
 	repmsg->u.allocate_payload.vcpi = raw->msg[idx];
 	idx++;
-	if (idx > raw->curlen)
+	if (idx + 2 > raw->curlen)
 		goto fail_len;
 	repmsg->u.allocate_payload.allocated_pbn = (raw->msg[idx] << 8) | (raw->msg[idx+1]);
-	idx += 2;
-	if (idx > raw->curlen)
-		goto fail_len;
 	return true;
 fail_len:
 	DRM_DEBUG_KMS("allocate payload parse length fail %d %d\n", idx, raw->curlen);
@@ -969,12 +974,9 @@ static bool drm_dp_sideband_parse_query_payload_ack(struct drm_dp_sideband_msg_r
 
 	repmsg->u.query_payload.port_number = (raw->msg[idx] >> 4) & 0xf;
 	idx++;
-	if (idx > raw->curlen)
+	if (idx + 2 > raw->curlen)
 		goto fail_len;
 	repmsg->u.query_payload.allocated_pbn = (raw->msg[idx] << 8) | (raw->msg[idx + 1]);
-	idx += 2;
-	if (idx > raw->curlen)
-		goto fail_len;
 	return true;
 fail_len:
 	DRM_DEBUG_KMS("query payload parse length fail %d %d\n", idx, raw->curlen);
@@ -4433,7 +4435,7 @@ EXPORT_SYMBOL(drm_dp_mst_get_edid);
  * Total slots in the atomic state assigned for this port, or a negative error
  * code if the port no longer exists
  */
-int drm_dp_atomic_find_time_slots(struct drm_atomic_state *state,
+int drm_dp_atomic_find_time_slots(struct drm_atomic_commit *state,
 				  struct drm_dp_mst_topology_mgr *mgr,
 				  struct drm_dp_mst_port *port, int pbn)
 {
@@ -4523,7 +4525,7 @@ EXPORT_SYMBOL(drm_dp_atomic_find_time_slots);
  * Returns:
  * 0 on success, negative error code otherwise
  */
-int drm_dp_atomic_release_time_slots(struct drm_atomic_state *state,
+int drm_dp_atomic_release_time_slots(struct drm_atomic_commit *state,
 				     struct drm_dp_mst_topology_mgr *mgr,
 				     struct drm_dp_mst_port *port)
 {
@@ -4591,7 +4593,7 @@ EXPORT_SYMBOL(drm_dp_atomic_release_time_slots);
  * Returns:
  * 0 if all CRTC commits were retrieved successfully, negative error code otherwise
  */
-int drm_dp_mst_atomic_setup_commit(struct drm_atomic_state *state)
+int drm_dp_mst_atomic_setup_commit(struct drm_atomic_commit *state)
 {
 	struct drm_dp_mst_topology_mgr *mgr;
 	struct drm_dp_mst_topology_state *mst_state;
@@ -4641,7 +4643,7 @@ EXPORT_SYMBOL(drm_dp_mst_atomic_setup_commit);
  * All MST drivers must call this function after calling drm_atomic_helper_wait_for_dependencies(),
  * or whatever their equivalent of that is.
  */
-void drm_dp_mst_atomic_wait_for_dependencies(struct drm_atomic_state *state)
+void drm_dp_mst_atomic_wait_for_dependencies(struct drm_atomic_commit *state)
 {
 	struct drm_dp_mst_topology_state *old_mst_state, *new_mst_state;
 	struct drm_dp_mst_topology_mgr *mgr;
@@ -4698,7 +4700,7 @@ EXPORT_SYMBOL(drm_dp_mst_atomic_wait_for_dependencies);
 int drm_dp_mst_root_conn_atomic_check(struct drm_connector_state *new_conn_state,
 				      struct drm_dp_mst_topology_mgr *mgr)
 {
-	struct drm_atomic_state *state = new_conn_state->state;
+	struct drm_atomic_commit *state = new_conn_state->state;
 	struct drm_connector_state *old_conn_state =
 		drm_atomic_get_old_connector_state(state, new_conn_state->connector);
 	struct drm_crtc_state *crtc_state;
@@ -5436,7 +5438,7 @@ drm_dp_mst_atomic_check_payload_alloc_limits(struct drm_dp_mst_topology_mgr *mgr
  * See also:
  * drm_dp_mst_atomic_enable_dsc()
  */
-int drm_dp_mst_add_affected_dsc_crtcs(struct drm_atomic_state *state, struct drm_dp_mst_topology_mgr *mgr)
+int drm_dp_mst_add_affected_dsc_crtcs(struct drm_atomic_commit *state, struct drm_dp_mst_topology_mgr *mgr)
 {
 	struct drm_dp_mst_topology_state *mst_state;
 	struct drm_dp_mst_atomic_payload *pos;
@@ -5486,7 +5488,7 @@ EXPORT_SYMBOL(drm_dp_mst_add_affected_dsc_crtcs);
 
 /**
  * drm_dp_mst_atomic_enable_dsc - Set DSC Enable Flag to On/Off
- * @state: Pointer to the new drm_atomic_state
+ * @state: Pointer to the new drm_atomic_commit
  * @port: Pointer to the affected MST Port
  * @pbn: Newly recalculated bw required for link with DSC enabled
  * @enable: Boolean flag to enable or disable DSC on the port
@@ -5497,7 +5499,7 @@ EXPORT_SYMBOL(drm_dp_mst_add_affected_dsc_crtcs);
  * ports have DSC enabled
  *
  */
-int drm_dp_mst_atomic_enable_dsc(struct drm_atomic_state *state,
+int drm_dp_mst_atomic_enable_dsc(struct drm_atomic_commit *state,
 				 struct drm_dp_mst_port *port,
 				 int pbn, bool enable)
 {
@@ -5576,7 +5578,7 @@ EXPORT_SYMBOL(drm_dp_mst_atomic_enable_dsc);
  *   - %-EINVAL, if the new state is invalid, because the root port has
  *     too many payloads.
  */
-int drm_dp_mst_atomic_check_mgr(struct drm_atomic_state *state,
+int drm_dp_mst_atomic_check_mgr(struct drm_atomic_commit *state,
 				struct drm_dp_mst_topology_mgr *mgr,
 				struct drm_dp_mst_topology_state *mst_state,
 				struct drm_dp_mst_port **failing_port)
@@ -5623,7 +5625,7 @@ EXPORT_SYMBOL(drm_dp_mst_atomic_check_mgr);
  * Returns:
  * 0 if the new state is valid, negative error code otherwise.
  */
-int drm_dp_mst_atomic_check(struct drm_atomic_state *state)
+int drm_dp_mst_atomic_check(struct drm_atomic_commit *state)
 {
 	struct drm_dp_mst_topology_mgr *mgr;
 	struct drm_dp_mst_topology_state *mst_state;
@@ -5660,7 +5662,7 @@ EXPORT_SYMBOL(drm_dp_mst_topology_state_funcs);
  * RETURNS:
  * The MST topology state or error pointer.
  */
-struct drm_dp_mst_topology_state *drm_atomic_get_mst_topology_state(struct drm_atomic_state *state,
+struct drm_dp_mst_topology_state *drm_atomic_get_mst_topology_state(struct drm_atomic_commit *state,
 								    struct drm_dp_mst_topology_mgr *mgr)
 {
 	return to_dp_mst_topology_state(drm_atomic_get_private_obj_state(state, &mgr->base));
@@ -5681,7 +5683,7 @@ EXPORT_SYMBOL(drm_atomic_get_mst_topology_state);
  * in the global atomic state
  */
 struct drm_dp_mst_topology_state *
-drm_atomic_get_old_mst_topology_state(struct drm_atomic_state *state,
+drm_atomic_get_old_mst_topology_state(struct drm_atomic_commit *state,
 				      struct drm_dp_mst_topology_mgr *mgr)
 {
 	struct drm_private_state *old_priv_state =
@@ -5705,7 +5707,7 @@ EXPORT_SYMBOL(drm_atomic_get_old_mst_topology_state);
  * in the global atomic state
  */
 struct drm_dp_mst_topology_state *
-drm_atomic_get_new_mst_topology_state(struct drm_atomic_state *state,
+drm_atomic_get_new_mst_topology_state(struct drm_atomic_commit *state,
 				      struct drm_dp_mst_topology_mgr *mgr)
 {
 	struct drm_private_state *new_priv_state =
diff --git a/drivers/gpu/drm/display/drm_dp_tunnel.c b/drivers/gpu/drm/display/drm_dp_tunnel.c
index 6519b42447285..76c6bc84a806f 100644
--- a/drivers/gpu/drm/display/drm_dp_tunnel.c
+++ b/drivers/gpu/drm/display/drm_dp_tunnel.c
@@ -149,6 +149,7 @@ struct drm_dp_tunnel {
 	bool bw_alloc_enabled:1;
 	bool has_io_error:1;
 	bool destroyed:1;
+	bool pr_optimization_support:1;
 };
 
 struct drm_dp_tunnel_group_state;
@@ -508,6 +509,8 @@ create_tunnel(struct drm_dp_tunnel_mgr *mgr,
 
 	tunnel->bw_alloc_supported = tunnel_reg_bw_alloc_supported(regs);
 	tunnel->bw_alloc_enabled = tunnel_reg_bw_alloc_enabled(regs);
+	tunnel->pr_optimization_support = tunnel_reg(regs, DP_TUNNELING_CAPABILITIES) &
+					  DP_PANEL_REPLAY_OPTIMIZATION_SUPPORT;
 
 	if (!add_tunnel_to_group(mgr, drv_group_id, tunnel)) {
 		kfree(tunnel);
@@ -1036,6 +1039,20 @@ bool drm_dp_tunnel_bw_alloc_is_enabled(const struct drm_dp_tunnel *tunnel)
 }
 EXPORT_SYMBOL(drm_dp_tunnel_bw_alloc_is_enabled);
 
+/**
+ * drm_dp_tunnel_pr_optimization_supported - Query the PR BW optimization support
+ * @tunnel: Tunnel object
+ *
+ * Query if the PR BW optimization is supported for @tunnel.
+ *
+ * Returns %true if the PR BW optimiation is supported for @tunnel.
+ */
+bool drm_dp_tunnel_pr_optimization_supported(const struct drm_dp_tunnel *tunnel)
+{
+	return tunnel && tunnel->pr_optimization_support;
+}
+EXPORT_SYMBOL(drm_dp_tunnel_pr_optimization_supported);
+
 static int clear_bw_req_state(struct drm_dp_aux *aux)
 {
 	u8 bw_req_mask = DP_BW_REQUEST_SUCCEEDED | DP_BW_REQUEST_FAILED;
@@ -1369,7 +1386,7 @@ int drm_dp_tunnel_available_bw(const struct drm_dp_tunnel *tunnel)
 EXPORT_SYMBOL(drm_dp_tunnel_available_bw);
 
 static struct drm_dp_tunnel_group_state *
-drm_dp_tunnel_atomic_get_group_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_group_state(struct drm_atomic_commit *state,
 				     const struct drm_dp_tunnel *tunnel)
 {
 	return (struct drm_dp_tunnel_group_state *)
@@ -1528,7 +1545,7 @@ static const struct drm_private_state_funcs tunnel_group_funcs = {
  * Return the state or an ERR_PTR() error on failure.
  */
 struct drm_dp_tunnel_state *
-drm_dp_tunnel_atomic_get_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_state(struct drm_atomic_commit *state,
 			       struct drm_dp_tunnel *tunnel)
 {
 	struct drm_dp_tunnel_group_state *group_state;
@@ -1556,7 +1573,7 @@ EXPORT_SYMBOL(drm_dp_tunnel_atomic_get_state);
  * Return the old state or NULL if the tunnel's atomic state is not in @state.
  */
 struct drm_dp_tunnel_state *
-drm_dp_tunnel_atomic_get_old_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_old_state(struct drm_atomic_commit *state,
 				   const struct drm_dp_tunnel *tunnel)
 {
 	struct drm_dp_tunnel_group_state *old_group_state;
@@ -1580,7 +1597,7 @@ EXPORT_SYMBOL(drm_dp_tunnel_atomic_get_old_state);
  * Return the new state or NULL if the tunnel's atomic state is not in @state.
  */
 struct drm_dp_tunnel_state *
-drm_dp_tunnel_atomic_get_new_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_new_state(struct drm_atomic_commit *state,
 				   const struct drm_dp_tunnel *tunnel)
 {
 	struct drm_dp_tunnel_group_state *new_group_state;
@@ -1706,7 +1723,7 @@ static int clear_stream_bw(struct drm_dp_tunnel_state *tunnel_state,
  *
  * Returns 0 in case of success, a negative error code otherwise.
  */
-int drm_dp_tunnel_atomic_set_stream_bw(struct drm_atomic_state *state,
+int drm_dp_tunnel_atomic_set_stream_bw(struct drm_atomic_commit *state,
 				       struct drm_dp_tunnel *tunnel,
 				       u8 stream_id, int bw)
 {
@@ -1783,7 +1800,7 @@ EXPORT_SYMBOL(drm_dp_tunnel_atomic_get_required_bw);
  * Return 0 in case of success - with the stream IDs in @stream_mask - or a
  * negative error code in case of failure.
  */
-int drm_dp_tunnel_atomic_get_group_streams_in_state(struct drm_atomic_state *state,
+int drm_dp_tunnel_atomic_get_group_streams_in_state(struct drm_atomic_commit *state,
 						    const struct drm_dp_tunnel *tunnel,
 						    u32 *stream_mask)
 {
@@ -1861,7 +1878,7 @@ drm_dp_tunnel_atomic_check_group_bw(struct drm_dp_tunnel_group_state *new_group_
  * check failed - with @failed_stream_mask containing the streams failing the
  * check - or a negative error code otherwise.
  */
-int drm_dp_tunnel_atomic_check_stream_bws(struct drm_atomic_state *state,
+int drm_dp_tunnel_atomic_check_stream_bws(struct drm_atomic_commit *state,
 					  u32 *failed_stream_mask)
 {
 	struct drm_dp_tunnel_group_state *new_group_state;
diff --git a/drivers/gpu/drm/display/drm_hdmi_state_helper.c b/drivers/gpu/drm/display/drm_hdmi_state_helper.c
index 9f3b696aceeb4..4867edbf2622b 100644
--- a/drivers/gpu/drm/display/drm_hdmi_state_helper.c
+++ b/drivers/gpu/drm/display/drm_hdmi_state_helper.c
@@ -348,7 +348,7 @@ output_color_format_to_hdmi_colorspace(const struct drm_connector *connector,
 static const struct drm_display_mode *
 connector_state_get_mode(const struct drm_connector_state *conn_state)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_crtc_state *crtc_state;
 	struct drm_crtc *crtc;
 
@@ -858,7 +858,7 @@ hdmi_generate_infoframes(const struct drm_connector *connector,
  * Zero on success, or an errno code otherwise.
  */
 int drm_atomic_helper_connector_hdmi_check(struct drm_connector *connector,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *old_conn_state =
 		drm_atomic_get_old_connector_state(state, connector);
@@ -1000,7 +1000,7 @@ static int write_or_clear_infoframe(struct drm_connector *connector,
  * Zero on success, error code on failure.
  */
 int drm_atomic_helper_connector_hdmi_update_infoframes(struct drm_connector *connector,
-						       struct drm_atomic_state *state)
+						       struct drm_atomic_commit *state)
 {
 	const struct drm_connector_hdmi_funcs *funcs = connector->hdmi.funcs;
 	struct drm_connector_state *old_conn_state =
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index 41c57063f3b4d..170de30c28ae4 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -95,15 +95,15 @@ int drm_crtc_commit_wait(struct drm_crtc_commit *commit)
 EXPORT_SYMBOL(drm_crtc_commit_wait);
 
 /**
- * drm_atomic_state_default_release -
- * release memory initialized by drm_atomic_state_init
+ * drm_atomic_commit_default_release -
+ * release memory initialized by drm_atomic_commit_init
  * @state: atomic state
  *
- * Free all the memory allocated by drm_atomic_state_init.
+ * Free all the memory allocated by drm_atomic_commit_init.
  * This should only be used by drivers which are still subclassing
- * &drm_atomic_state and haven't switched to &drm_private_state yet.
+ * &drm_atomic_commit and haven't switched to &drm_private_state yet.
  */
-void drm_atomic_state_default_release(struct drm_atomic_state *state)
+void drm_atomic_commit_default_release(struct drm_atomic_commit *state)
 {
 	kfree(state->connectors);
 	kfree(state->crtcs);
@@ -111,19 +111,19 @@ void drm_atomic_state_default_release(struct drm_atomic_state *state)
 	kfree(state->colorops);
 	kfree(state->private_objs);
 }
-EXPORT_SYMBOL(drm_atomic_state_default_release);
+EXPORT_SYMBOL(drm_atomic_commit_default_release);
 
 /**
- * drm_atomic_state_init - init new atomic state
+ * drm_atomic_commit_init - init new atomic state
  * @dev: DRM device
  * @state: atomic state
  *
  * Default implementation for filling in a new atomic state.
  * This should only be used by drivers which are still subclassing
- * &drm_atomic_state and haven't switched to &drm_private_state yet.
+ * &drm_atomic_commit and haven't switched to &drm_private_state yet.
  */
 int
-drm_atomic_state_init(struct drm_device *dev, struct drm_atomic_state *state)
+drm_atomic_commit_init(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	kref_init(&state->ref);
 
@@ -145,7 +145,7 @@ drm_atomic_state_init(struct drm_device *dev, struct drm_atomic_state *state)
 		goto fail;
 
 	/*
-	 * Because drm_atomic_state can be committed asynchronously we need our
+	 * Because drm_atomic_commit can be committed asynchronously we need our
 	 * own reference and cannot rely on the on implied by drm_file in the
 	 * ioctl call.
 	 */
@@ -156,29 +156,29 @@ drm_atomic_state_init(struct drm_device *dev, struct drm_atomic_state *state)
 
 	return 0;
 fail:
-	drm_atomic_state_default_release(state);
+	drm_atomic_commit_default_release(state);
 	return -ENOMEM;
 }
-EXPORT_SYMBOL(drm_atomic_state_init);
+EXPORT_SYMBOL(drm_atomic_commit_init);
 
 /**
- * drm_atomic_state_alloc - allocate atomic state
+ * drm_atomic_commit_alloc - allocate atomic state
  * @dev: DRM device
  *
  * This allocates an empty atomic state to track updates.
  */
-struct drm_atomic_state *
-drm_atomic_state_alloc(struct drm_device *dev)
+struct drm_atomic_commit *
+drm_atomic_commit_alloc(struct drm_device *dev)
 {
 	struct drm_mode_config *config = &dev->mode_config;
 
 	if (!config->funcs->atomic_state_alloc) {
-		struct drm_atomic_state *state;
+		struct drm_atomic_commit *state;
 
 		state = kzalloc_obj(*state);
 		if (!state)
 			return NULL;
-		if (drm_atomic_state_init(dev, state) < 0) {
+		if (drm_atomic_commit_init(dev, state) < 0) {
 			kfree(state);
 			return NULL;
 		}
@@ -187,17 +187,17 @@ drm_atomic_state_alloc(struct drm_device *dev)
 
 	return config->funcs->atomic_state_alloc(dev);
 }
-EXPORT_SYMBOL(drm_atomic_state_alloc);
+EXPORT_SYMBOL(drm_atomic_commit_alloc);
 
 /**
- * drm_atomic_state_default_clear - clear base atomic state
+ * drm_atomic_commit_default_clear - clear base atomic state
  * @state: atomic state
  *
  * Default implementation for clearing atomic state.
  * This should only be used by drivers which are still subclassing
- * &drm_atomic_state and haven't switched to &drm_private_state yet.
+ * &drm_atomic_commit and haven't switched to &drm_private_state yet.
  */
-void drm_atomic_state_default_clear(struct drm_atomic_state *state)
+void drm_atomic_commit_default_clear(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct drm_mode_config *config = &dev->mode_config;
@@ -287,10 +287,10 @@ void drm_atomic_state_default_clear(struct drm_atomic_state *state)
 		state->fake_commit = NULL;
 	}
 }
-EXPORT_SYMBOL(drm_atomic_state_default_clear);
+EXPORT_SYMBOL(drm_atomic_commit_default_clear);
 
 /**
- * drm_atomic_state_clear - clear state object
+ * drm_atomic_commit_clear - clear state object
  * @state: atomic state
  *
  * When the w/w mutex algorithm detects a deadlock we need to back off and drop
@@ -303,7 +303,7 @@ EXPORT_SYMBOL(drm_atomic_state_default_clear);
  * Hence we must clear all cached state and completely start over, using this
  * function.
  */
-void drm_atomic_state_clear(struct drm_atomic_state *state)
+void drm_atomic_commit_clear(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct drm_mode_config *config = &dev->mode_config;
@@ -311,37 +311,37 @@ void drm_atomic_state_clear(struct drm_atomic_state *state)
 	if (config->funcs->atomic_state_clear)
 		config->funcs->atomic_state_clear(state);
 	else
-		drm_atomic_state_default_clear(state);
+		drm_atomic_commit_default_clear(state);
 }
-EXPORT_SYMBOL(drm_atomic_state_clear);
+EXPORT_SYMBOL(drm_atomic_commit_clear);
 
 /**
- * __drm_atomic_state_free - free all memory for an atomic state
+ * __drm_atomic_commit_free - free all memory for an atomic state
  * @ref: This atomic state to deallocate
  *
  * This frees all memory associated with an atomic state, including all the
  * per-object state for planes, CRTCs and connectors.
  */
-void __drm_atomic_state_free(struct kref *ref)
+void __drm_atomic_commit_free(struct kref *ref)
 {
-	struct drm_atomic_state *state = container_of(ref, typeof(*state), ref);
+	struct drm_atomic_commit *state = container_of(ref, typeof(*state), ref);
 	struct drm_device *dev = state->dev;
 	struct drm_mode_config *config = &dev->mode_config;
 
-	drm_atomic_state_clear(state);
+	drm_atomic_commit_clear(state);
 
 	drm_dbg_atomic(state->dev, "Freeing atomic state %p\n", state);
 
 	if (config->funcs->atomic_state_free) {
 		config->funcs->atomic_state_free(state);
 	} else {
-		drm_atomic_state_default_release(state);
+		drm_atomic_commit_default_release(state);
 		kfree(state);
 	}
 
 	drm_dev_put(dev);
 }
-EXPORT_SYMBOL(__drm_atomic_state_free);
+EXPORT_SYMBOL(__drm_atomic_commit_free);
 
 /**
  * drm_atomic_get_crtc_state - get CRTC state
@@ -353,7 +353,7 @@ EXPORT_SYMBOL(__drm_atomic_state_free);
  * is consistent.
  *
  * WARNING: Drivers may only add new CRTC states to a @state if
- * drm_atomic_state.allow_modeset is set, or if it's a driver-internal commit
+ * drm_atomic_commit.allow_modeset is set, or if it's a driver-internal commit
  * not created by userspace through an IOCTL call.
  *
  * Returns:
@@ -362,7 +362,7 @@ EXPORT_SYMBOL(__drm_atomic_state_free);
  * entire atomic sequence must be restarted. All other errors are fatal.
  */
 struct drm_crtc_state *
-drm_atomic_get_crtc_state(struct drm_atomic_state *state,
+drm_atomic_get_crtc_state(struct drm_atomic_commit *state,
 			  struct drm_crtc *crtc)
 {
 	int ret, index = drm_crtc_index(crtc);
@@ -463,19 +463,19 @@ static void drm_atomic_crtc_print_state(struct drm_printer *p,
 	struct drm_crtc *crtc = state->crtc;
 
 	drm_printf(p, "crtc[%u]: %s\n", crtc->base.id, crtc->name);
-	drm_printf(p, "\tenable=%d\n", state->enable);
-	drm_printf(p, "\tactive=%d\n", state->active);
-	drm_printf(p, "\tself_refresh_active=%d\n", state->self_refresh_active);
-	drm_printf(p, "\tplanes_changed=%d\n", state->planes_changed);
-	drm_printf(p, "\tmode_changed=%d\n", state->mode_changed);
-	drm_printf(p, "\tactive_changed=%d\n", state->active_changed);
-	drm_printf(p, "\tconnectors_changed=%d\n", state->connectors_changed);
-	drm_printf(p, "\tcolor_mgmt_changed=%d\n", state->color_mgmt_changed);
-	drm_printf(p, "\tplane_mask=%x\n", state->plane_mask);
-	drm_printf(p, "\tconnector_mask=%x\n", state->connector_mask);
-	drm_printf(p, "\tencoder_mask=%x\n", state->encoder_mask);
-	drm_printf(p, "\tmode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(&state->mode));
-	drm_printf(p, "\tbackground_color=%llx\n", state->background_color);
+	drm_printf_indent(p, 1, "enable=%d\n", state->enable);
+	drm_printf_indent(p, 1, "active=%d\n", state->active);
+	drm_printf_indent(p, 1, "self_refresh_active=%d\n", state->self_refresh_active);
+	drm_printf_indent(p, 1, "planes_changed=%d\n", state->planes_changed);
+	drm_printf_indent(p, 1, "mode_changed=%d\n", state->mode_changed);
+	drm_printf_indent(p, 1, "active_changed=%d\n", state->active_changed);
+	drm_printf_indent(p, 1, "connectors_changed=%d\n", state->connectors_changed);
+	drm_printf_indent(p, 1, "color_mgmt_changed=%d\n", state->color_mgmt_changed);
+	drm_printf_indent(p, 1, "plane_mask=%x\n", state->plane_mask);
+	drm_printf_indent(p, 1, "connector_mask=%x\n", state->connector_mask);
+	drm_printf_indent(p, 1, "encoder_mask=%x\n", state->encoder_mask);
+	drm_printf_indent(p, 1, "mode: " DRM_MODE_FMT "\n", DRM_MODE_ARG(&state->mode));
+	drm_printf_indent(p, 1, "background_color=%llx\n", state->background_color);
 
 	if (crtc->funcs->atomic_print_state)
 		crtc->funcs->atomic_print_state(p, state);
@@ -544,7 +544,7 @@ static int drm_atomic_connector_check(struct drm_connector *connector,
  * entire atomic sequence must be restarted. All other errors are fatal.
  */
 struct drm_plane_state *
-drm_atomic_get_plane_state(struct drm_atomic_state *state,
+drm_atomic_get_plane_state(struct drm_atomic_commit *state,
 			  struct drm_plane *plane)
 {
 	int ret, index = drm_plane_index(plane);
@@ -608,7 +608,7 @@ EXPORT_SYMBOL(drm_atomic_get_plane_state);
  * entire atomic sequence must be restarted. All other errors are fatal.
  */
 struct drm_colorop_state *
-drm_atomic_get_colorop_state(struct drm_atomic_state *state,
+drm_atomic_get_colorop_state(struct drm_atomic_commit *state,
 			     struct drm_colorop *colorop)
 {
 	int ret, index = drm_colorop_index(colorop);
@@ -650,7 +650,7 @@ EXPORT_SYMBOL(drm_atomic_get_colorop_state);
  * NULL if the colorop is not part of the global atomic state.
  */
 struct drm_colorop_state *
-drm_atomic_get_old_colorop_state(struct drm_atomic_state *state,
+drm_atomic_get_old_colorop_state(struct drm_atomic_commit *state,
 				 struct drm_colorop *colorop)
 {
 	return state->colorops[drm_colorop_index(colorop)].old_state;
@@ -666,7 +666,7 @@ EXPORT_SYMBOL(drm_atomic_get_old_colorop_state);
  * NULL if the colorop is not part of the global atomic state.
  */
 struct drm_colorop_state *
-drm_atomic_get_new_colorop_state(struct drm_atomic_state *state,
+drm_atomic_get_new_colorop_state(struct drm_atomic_commit *state,
 				 struct drm_colorop *colorop)
 {
 	return state->colorops[drm_colorop_index(colorop)].new_state;
@@ -818,38 +818,38 @@ static void drm_atomic_colorop_print_state(struct drm_printer *p,
 	struct drm_colorop *colorop = state->colorop;
 
 	drm_printf(p, "colorop[%u]:\n", colorop->base.id);
-	drm_printf(p, "\ttype=%s\n", drm_get_colorop_type_name(colorop->type));
+	drm_printf_indent(p, 1, "type=%s\n", drm_get_colorop_type_name(colorop->type));
 	if (colorop->bypass_property)
-		drm_printf(p, "\tbypass=%u\n", state->bypass);
+		drm_printf_indent(p, 1, "bypass=%u\n", state->bypass);
 
 	switch (colorop->type) {
 	case DRM_COLOROP_1D_CURVE:
-		drm_printf(p, "\tcurve_1d_type=%s\n",
-			   drm_get_colorop_curve_1d_type_name(state->curve_1d_type));
+		drm_printf_indent(p, 1, "curve_1d_type=%s\n",
+				  drm_get_colorop_curve_1d_type_name(state->curve_1d_type));
 		break;
 	case DRM_COLOROP_1D_LUT:
-		drm_printf(p, "\tsize=%d\n", colorop->size);
-		drm_printf(p, "\tinterpolation=%s\n",
-			   drm_get_colorop_lut1d_interpolation_name(colorop->lut1d_interpolation));
-		drm_printf(p, "\tdata blob id=%d\n", state->data ? state->data->base.id : 0);
+		drm_printf_indent(p, 1, "size=%d\n", colorop->size);
+		drm_printf_indent(p, 1, "interpolation=%s\n",
+				  drm_get_colorop_lut1d_interpolation_name(colorop->lut1d_interpolation));
+		drm_printf_indent(p, 1, "data blob id=%d\n", state->data ? state->data->base.id : 0);
 		break;
 	case DRM_COLOROP_CTM_3X4:
-		drm_printf(p, "\tdata blob id=%d\n", state->data ? state->data->base.id : 0);
+		drm_printf_indent(p, 1, "data blob id=%d\n", state->data ? state->data->base.id : 0);
 		break;
 	case DRM_COLOROP_MULTIPLIER:
-		drm_printf(p, "\tmultiplier=%llu\n", state->multiplier);
+		drm_printf_indent(p, 1, "multiplier=%llu\n", state->multiplier);
 		break;
 	case DRM_COLOROP_3D_LUT:
-		drm_printf(p, "\tsize=%d\n", colorop->size);
-		drm_printf(p, "\tinterpolation=%s\n",
-			   drm_get_colorop_lut3d_interpolation_name(colorop->lut3d_interpolation));
-		drm_printf(p, "\tdata blob id=%d\n", state->data ? state->data->base.id : 0);
+		drm_printf_indent(p, 1, "size=%d\n", colorop->size);
+		drm_printf_indent(p, 1, "interpolation=%s\n",
+				  drm_get_colorop_lut3d_interpolation_name(colorop->lut3d_interpolation));
+		drm_printf_indent(p, 1, "data blob id=%d\n", state->data ? state->data->base.id : 0);
 		break;
 	default:
 		break;
 	}
 
-	drm_printf(p, "\tnext=%d\n", colorop->next ? colorop->next->base.id : 0);
+	drm_printf_indent(p, 1, "next=%d\n", colorop->next ? colorop->next->base.id : 0);
 }
 
 static void drm_atomic_plane_print_state(struct drm_printer *p,
@@ -860,21 +860,21 @@ static void drm_atomic_plane_print_state(struct drm_printer *p,
 	struct drm_rect dest = drm_plane_state_dest(state);
 
 	drm_printf(p, "plane[%u]: %s\n", plane->base.id, plane->name);
-	drm_printf(p, "\tcrtc=%s\n", state->crtc ? state->crtc->name : "(null)");
-	drm_printf(p, "\tfb=%u\n", state->fb ? state->fb->base.id : 0);
+	drm_printf_indent(p, 1, "crtc=%s\n", state->crtc ? state->crtc->name : "(null)");
+	drm_printf_indent(p, 1, "fb=%u\n", state->fb ? state->fb->base.id : 0);
 	if (state->fb)
 		drm_framebuffer_print_info(p, 2, state->fb);
-	drm_printf(p, "\tcrtc-pos=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&dest));
-	drm_printf(p, "\tsrc-pos=" DRM_RECT_FP_FMT "\n", DRM_RECT_FP_ARG(&src));
-	drm_printf(p, "\trotation=%x\n", state->rotation);
-	drm_printf(p, "\tnormalized-zpos=%x\n", state->normalized_zpos);
-	drm_printf(p, "\tcolor-encoding=%s\n",
-		   drm_get_color_encoding_name(state->color_encoding));
-	drm_printf(p, "\tcolor-range=%s\n",
-		   drm_get_color_range_name(state->color_range));
-	drm_printf(p, "\tcolor_mgmt_changed=%d\n", state->color_mgmt_changed);
-	drm_printf(p, "\tcolor-pipeline=%d\n",
-		   state->color_pipeline ? state->color_pipeline->base.id : 0);
+	drm_printf_indent(p, 1, "crtc-pos=" DRM_RECT_FMT "\n", DRM_RECT_ARG(&dest));
+	drm_printf_indent(p, 1, "src-pos=" DRM_RECT_FP_FMT "\n", DRM_RECT_FP_ARG(&src));
+	drm_printf_indent(p, 1, "rotation=%x\n", state->rotation);
+	drm_printf_indent(p, 1, "normalized-zpos=%x\n", state->normalized_zpos);
+	drm_printf_indent(p, 1, "color-encoding=%s\n",
+			  drm_get_color_encoding_name(state->color_encoding));
+	drm_printf_indent(p, 1, "color-range=%s\n",
+			  drm_get_color_range_name(state->color_range));
+	drm_printf_indent(p, 1, "color_mgmt_changed=%d\n", state->color_mgmt_changed);
+	drm_printf_indent(p, 1, "color-pipeline=%d\n",
+			  state->color_pipeline ? state->color_pipeline->base.id : 0);
 	if (plane->funcs->atomic_print_state)
 		plane->funcs->atomic_print_state(p, state);
 }
@@ -901,9 +901,9 @@ static void drm_atomic_plane_print_state(struct drm_printer *p,
  * directly. Sequence of the actual hardware state commit is not handled,
  * drivers might need to keep track of struct drm_crtc_commit within subclassed
  * structure of &drm_private_state as necessary, e.g. similar to
- * &drm_plane_state.commit. See also &drm_atomic_state.fake_commit.
+ * &drm_plane_state.commit. See also &drm_atomic_commit.fake_commit.
  *
- * All private state structures contained in a &drm_atomic_state update can be
+ * All private state structures contained in a &drm_atomic_commit update can be
  * iterated using for_each_oldnew_private_obj_in_state(),
  * for_each_new_private_obj_in_state() and for_each_old_private_obj_in_state().
  * Drivers are recommended to wrap these for each type of driver private state
@@ -911,7 +911,7 @@ static void drm_atomic_plane_print_state(struct drm_printer *p,
  * least if they want to iterate over all objects of a given type.
  *
  * An earlier way to handle driver private state was by subclassing struct
- * &drm_atomic_state. But since that encourages non-standard ways to implement
+ * &drm_atomic_commit. But since that encourages non-standard ways to implement
  * the check/commit split atomic requires (by using e.g. "check and rollback or
  * commit instead" of "duplicate state, check, then either commit or release
  * duplicated state) it is deprecated in favour of using &drm_private_state.
@@ -981,7 +981,7 @@ EXPORT_SYMBOL(drm_atomic_private_obj_fini);
  * Either the allocated state or the error code encoded into a pointer.
  */
 struct drm_private_state *
-drm_atomic_get_private_obj_state(struct drm_atomic_state *state,
+drm_atomic_get_private_obj_state(struct drm_atomic_commit *state,
 				 struct drm_private_obj *obj)
 {
 	int index, num_objs, ret;
@@ -1039,7 +1039,7 @@ EXPORT_SYMBOL(drm_atomic_get_private_obj_state);
  * or NULL if the private_obj is not part of the global atomic state.
  */
 struct drm_private_state *
-drm_atomic_get_old_private_obj_state(const struct drm_atomic_state *state,
+drm_atomic_get_old_private_obj_state(const struct drm_atomic_commit *state,
 				     struct drm_private_obj *obj)
 {
 	int i;
@@ -1061,7 +1061,7 @@ EXPORT_SYMBOL(drm_atomic_get_old_private_obj_state);
  * or NULL if the private_obj is not part of the global atomic state.
  */
 struct drm_private_state *
-drm_atomic_get_new_private_obj_state(const struct drm_atomic_state *state,
+drm_atomic_get_new_private_obj_state(const struct drm_atomic_commit *state,
 				     struct drm_private_obj *obj)
 {
 	int i;
@@ -1096,7 +1096,7 @@ EXPORT_SYMBOL(drm_atomic_get_new_private_obj_state);
  * not connected.
  */
 struct drm_connector *
-drm_atomic_get_old_connector_for_encoder(const struct drm_atomic_state *state,
+drm_atomic_get_old_connector_for_encoder(const struct drm_atomic_commit *state,
 					 struct drm_encoder *encoder)
 {
 	struct drm_connector_state *conn_state;
@@ -1133,7 +1133,7 @@ EXPORT_SYMBOL(drm_atomic_get_old_connector_for_encoder);
  * not connected.
  */
 struct drm_connector *
-drm_atomic_get_new_connector_for_encoder(const struct drm_atomic_state *state,
+drm_atomic_get_new_connector_for_encoder(const struct drm_atomic_commit *state,
 					 struct drm_encoder *encoder)
 {
 	struct drm_connector_state *conn_state;
@@ -1214,7 +1214,7 @@ EXPORT_SYMBOL(drm_atomic_get_connector_for_encoder);
  * not connected.
  */
 struct drm_crtc *
-drm_atomic_get_old_crtc_for_encoder(struct drm_atomic_state *state,
+drm_atomic_get_old_crtc_for_encoder(struct drm_atomic_commit *state,
 				    struct drm_encoder *encoder)
 {
 	struct drm_connector *connector;
@@ -1244,7 +1244,7 @@ EXPORT_SYMBOL(drm_atomic_get_old_crtc_for_encoder);
  * not connected.
  */
 struct drm_crtc *
-drm_atomic_get_new_crtc_for_encoder(struct drm_atomic_state *state,
+drm_atomic_get_new_crtc_for_encoder(struct drm_atomic_commit *state,
 				    struct drm_encoder *encoder)
 {
 	struct drm_connector *connector;
@@ -1277,7 +1277,7 @@ EXPORT_SYMBOL(drm_atomic_get_new_crtc_for_encoder);
  * entire atomic sequence must be restarted. All other errors are fatal.
  */
 struct drm_connector_state *
-drm_atomic_get_connector_state(struct drm_atomic_state *state,
+drm_atomic_get_connector_state(struct drm_atomic_commit *state,
 			  struct drm_connector *connector)
 {
 	int ret, index;
@@ -1347,27 +1347,27 @@ static void drm_atomic_connector_print_state(struct drm_printer *p,
 	struct drm_connector *connector = state->connector;
 
 	drm_printf(p, "connector[%u]: %s\n", connector->base.id, connector->name);
-	drm_printf(p, "\tcrtc=%s\n", state->crtc ? state->crtc->name : "(null)");
-	drm_printf(p, "\tself_refresh_aware=%d\n", state->self_refresh_aware);
-	drm_printf(p, "\tinterlace_allowed=%d\n", connector->interlace_allowed);
-	drm_printf(p, "\tycbcr_420_allowed=%d\n", connector->ycbcr_420_allowed);
-	drm_printf(p, "\tmax_requested_bpc=%d\n", state->max_requested_bpc);
-	drm_printf(p, "\tcolorspace=%s\n", drm_get_colorspace_name(state->colorspace));
+	drm_printf_indent(p, 1, "crtc=%s\n", state->crtc ? state->crtc->name : "(null)");
+	drm_printf_indent(p, 1, "self_refresh_aware=%d\n", state->self_refresh_aware);
+	drm_printf_indent(p, 1, "interlace_allowed=%d\n", connector->interlace_allowed);
+	drm_printf_indent(p, 1, "ycbcr_420_allowed=%d\n", connector->ycbcr_420_allowed);
+	drm_printf_indent(p, 1, "max_requested_bpc=%d\n", state->max_requested_bpc);
+	drm_printf_indent(p, 1, "colorspace=%s\n", drm_get_colorspace_name(state->colorspace));
 
 	if (connector->connector_type == DRM_MODE_CONNECTOR_HDMIA ||
 	    connector->connector_type == DRM_MODE_CONNECTOR_HDMIB) {
-		drm_printf(p, "\tbroadcast_rgb=%s\n",
-			   drm_hdmi_connector_get_broadcast_rgb_name(state->hdmi.broadcast_rgb));
-		drm_printf(p, "\tis_limited_range=%c\n", state->hdmi.is_limited_range ? 'y' : 'n');
-		drm_printf(p, "\toutput_bpc=%u\n", state->hdmi.output_bpc);
-		drm_printf(p, "\toutput_format=%s\n",
-			   drm_hdmi_connector_get_output_format_name(state->hdmi.output_format));
-		drm_printf(p, "\ttmds_char_rate=%llu\n", state->hdmi.tmds_char_rate);
+		drm_printf_indent(p, 1, "broadcast_rgb=%s\n",
+				  drm_hdmi_connector_get_broadcast_rgb_name(state->hdmi.broadcast_rgb));
+		drm_printf_indent(p, 1, "is_limited_range=%c\n", state->hdmi.is_limited_range ? 'y' : 'n');
+		drm_printf_indent(p, 1, "output_bpc=%u\n", state->hdmi.output_bpc);
+		drm_printf_indent(p, 1, "output_format=%s\n",
+				  drm_hdmi_connector_get_output_format_name(state->hdmi.output_format));
+		drm_printf_indent(p, 1, "tmds_char_rate=%llu\n", state->hdmi.tmds_char_rate);
 	}
 
 	if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK)
 		if (state->writeback_job && state->writeback_job->fb)
-			drm_printf(p, "\tfb=%d\n", state->writeback_job->fb->base.id);
+			drm_printf_indent(p, 1, "fb=%d\n", state->writeback_job->fb->base.id);
 
 	if (connector->funcs->atomic_print_state)
 		connector->funcs->atomic_print_state(p, state);
@@ -1388,7 +1388,7 @@ static void drm_atomic_connector_print_state(struct drm_printer *p,
  * entire atomic sequence must be restarted.
  */
 struct drm_bridge_state *
-drm_atomic_get_bridge_state(struct drm_atomic_state *state,
+drm_atomic_get_bridge_state(struct drm_atomic_commit *state,
 			    struct drm_bridge *bridge)
 {
 	struct drm_private_state *obj_state;
@@ -1410,7 +1410,7 @@ EXPORT_SYMBOL(drm_atomic_get_bridge_state);
  * the bridge is not part of the global atomic state.
  */
 struct drm_bridge_state *
-drm_atomic_get_old_bridge_state(const struct drm_atomic_state *state,
+drm_atomic_get_old_bridge_state(const struct drm_atomic_commit *state,
 				struct drm_bridge *bridge)
 {
 	struct drm_private_state *obj_state;
@@ -1432,7 +1432,7 @@ EXPORT_SYMBOL(drm_atomic_get_old_bridge_state);
  * the bridge is not part of the global atomic state.
  */
 struct drm_bridge_state *
-drm_atomic_get_new_bridge_state(const struct drm_atomic_state *state,
+drm_atomic_get_new_bridge_state(const struct drm_atomic_commit *state,
 				struct drm_bridge *bridge)
 {
 	struct drm_private_state *obj_state;
@@ -1462,7 +1462,7 @@ EXPORT_SYMBOL(drm_atomic_get_new_bridge_state);
  * sequence must be restarted. All other errors are fatal.
  */
 int
-drm_atomic_add_encoder_bridges(struct drm_atomic_state *state,
+drm_atomic_add_encoder_bridges(struct drm_atomic_commit *state,
 			       struct drm_encoder *encoder)
 {
 	struct drm_bridge_state *bridge_state;
@@ -1506,7 +1506,7 @@ EXPORT_SYMBOL(drm_atomic_add_encoder_bridges);
  * sequence must be restarted. All other errors are fatal.
  */
 int
-drm_atomic_add_affected_connectors(struct drm_atomic_state *state,
+drm_atomic_add_affected_connectors(struct drm_atomic_commit *state,
 				   struct drm_crtc *crtc)
 {
 	struct drm_mode_config *config = &state->dev->mode_config;
@@ -1570,7 +1570,7 @@ EXPORT_SYMBOL(drm_atomic_add_affected_connectors);
  * sequence must be restarted. All other errors are fatal.
  */
 int
-drm_atomic_add_affected_planes(struct drm_atomic_state *state,
+drm_atomic_add_affected_planes(struct drm_atomic_commit *state,
 			       struct drm_crtc *crtc)
 {
 	const struct drm_crtc_state *old_crtc_state =
@@ -1622,7 +1622,7 @@ EXPORT_SYMBOL(drm_atomic_add_affected_planes);
  * sequence must be restarted. All other errors are fatal.
  */
 int
-drm_atomic_add_affected_colorops(struct drm_atomic_state *state,
+drm_atomic_add_affected_colorops(struct drm_atomic_commit *state,
 				 struct drm_plane *plane)
 {
 	struct drm_colorop *colorop;
@@ -1658,7 +1658,7 @@ EXPORT_SYMBOL(drm_atomic_add_affected_colorops);
  * Returns:
  * 0 on success, negative error code on failure.
  */
-int drm_atomic_check_only(struct drm_atomic_state *state)
+int drm_atomic_check_only(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct drm_mode_config *config = &dev->mode_config;
@@ -1766,12 +1766,12 @@ EXPORT_SYMBOL(drm_atomic_check_only);
  * backoff dance and restart. All other errors are fatal.
  *
  * This function will take its own reference on @state.
- * Callers should always release their reference with drm_atomic_state_put().
+ * Callers should always release their reference with drm_atomic_commit_put().
  *
  * Returns:
  * 0 on success, negative error code on failure.
  */
-int drm_atomic_commit(struct drm_atomic_state *state)
+int drm_atomic_commit(struct drm_atomic_commit *state)
 {
 	struct drm_mode_config *config = &state->dev->mode_config;
 	struct drm_printer p = drm_info_printer(state->dev->dev);
@@ -1799,12 +1799,12 @@ EXPORT_SYMBOL(drm_atomic_commit);
  * backoff dance and restart. All other errors are fatal.
  *
  * This function will take its own reference on @state.
- * Callers should always release their reference with drm_atomic_state_put().
+ * Callers should always release their reference with drm_atomic_commit_put().
  *
  * Returns:
  * 0 on success, negative error code on failure.
  */
-int drm_atomic_nonblocking_commit(struct drm_atomic_state *state)
+int drm_atomic_nonblocking_commit(struct drm_atomic_commit *state)
 {
 	struct drm_mode_config *config = &state->dev->mode_config;
 	int ret;
@@ -1843,7 +1843,7 @@ int __drm_atomic_helper_disable_plane(struct drm_plane *plane,
 }
 EXPORT_SYMBOL(__drm_atomic_helper_disable_plane);
 
-static int update_output_state(struct drm_atomic_state *state,
+static int update_output_state(struct drm_atomic_commit *state,
 			       struct drm_mode_set *set)
 {
 	struct drm_device *dev = set->crtc->dev;
@@ -1913,7 +1913,7 @@ static int update_output_state(struct drm_atomic_state *state,
 
 /* just used from drm-client and atomic-helper: */
 int __drm_atomic_helper_set_config(struct drm_mode_set *set,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state;
 	struct drm_plane_state *primary_state;
@@ -2007,7 +2007,7 @@ static void drm_atomic_private_obj_print_state(struct drm_printer *p,
  * Note that this function looks into the new state objects and hence its not
  * safe to be used after the call to drm_atomic_helper_commit_hw_done().
  */
-void drm_atomic_print_new_state(const struct drm_atomic_state *state,
+void drm_atomic_print_new_state(const struct drm_atomic_commit *state,
 		struct drm_printer *p)
 {
 	struct drm_plane *plane;
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index a768398a1884c..51f39edc31edc 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -77,7 +77,7 @@
  * helpers.
  */
 static void
-drm_atomic_helper_plane_changed(struct drm_atomic_state *state,
+drm_atomic_helper_plane_changed(struct drm_atomic_commit *state,
 				struct drm_plane_state *old_plane_state,
 				struct drm_plane_state *plane_state,
 				struct drm_plane *plane)
@@ -104,7 +104,7 @@ drm_atomic_helper_plane_changed(struct drm_atomic_state *state,
 	}
 }
 
-static int handle_conflicting_encoders(struct drm_atomic_state *state,
+static int handle_conflicting_encoders(struct drm_atomic_commit *state,
 				       bool disable_conflicting_encoders)
 {
 	struct drm_connector_state *new_conn_state;
@@ -218,7 +218,7 @@ static int handle_conflicting_encoders(struct drm_atomic_state *state,
 }
 
 static void
-set_best_encoder(struct drm_atomic_state *state,
+set_best_encoder(struct drm_atomic_commit *state,
 		 struct drm_connector_state *conn_state,
 		 struct drm_encoder *encoder)
 {
@@ -259,7 +259,7 @@ set_best_encoder(struct drm_atomic_state *state,
 }
 
 static void
-steal_encoder(struct drm_atomic_state *state,
+steal_encoder(struct drm_atomic_commit *state,
 	      struct drm_encoder *encoder)
 {
 	struct drm_crtc_state *crtc_state;
@@ -290,7 +290,7 @@ steal_encoder(struct drm_atomic_state *state,
 }
 
 static int
-update_connector_routing(struct drm_atomic_state *state,
+update_connector_routing(struct drm_atomic_commit *state,
 			 struct drm_connector *connector,
 			 struct drm_connector_state *old_connector_state,
 			 struct drm_connector_state *new_connector_state,
@@ -416,7 +416,7 @@ update_connector_routing(struct drm_atomic_state *state,
 }
 
 static int
-mode_fixup(struct drm_atomic_state *state)
+mode_fixup(struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state;
@@ -546,7 +546,7 @@ static enum drm_mode_status mode_valid_path(struct drm_connector *connector,
 }
 
 static int
-mode_valid(struct drm_atomic_state *state)
+mode_valid(struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *conn_state;
 	struct drm_connector *connector;
@@ -578,7 +578,7 @@ mode_valid(struct drm_atomic_state *state)
 	return 0;
 }
 
-static int drm_atomic_check_valid_clones(struct drm_atomic_state *state,
+static int drm_atomic_check_valid_clones(struct drm_atomic_commit *state,
 					 struct drm_crtc *crtc)
 {
 	struct drm_encoder *drm_enc;
@@ -647,7 +647,7 @@ static int drm_atomic_check_valid_clones(struct drm_atomic_state *state,
  */
 int
 drm_atomic_helper_check_modeset(struct drm_device *dev,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
@@ -841,7 +841,7 @@ EXPORT_SYMBOL(drm_atomic_helper_check_modeset);
  */
 int
 drm_atomic_helper_check_wb_connector_state(struct drm_connector *connector,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *conn_state =
 		drm_atomic_get_new_connector_state(state, connector);
@@ -1017,7 +1017,7 @@ EXPORT_SYMBOL(drm_atomic_helper_check_crtc_primary_plane);
  */
 int
 drm_atomic_helper_check_planes(struct drm_device *dev,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state;
@@ -1105,7 +1105,7 @@ EXPORT_SYMBOL(drm_atomic_helper_check_planes);
  * Zero for success or -errno
  */
 int drm_atomic_helper_check(struct drm_device *dev,
-			    struct drm_atomic_state *state)
+			    struct drm_atomic_commit *state)
 {
 	int ret;
 
@@ -1174,7 +1174,7 @@ crtc_needs_disable(struct drm_crtc_state *old_state,
  */
 void
 drm_atomic_helper_commit_encoder_bridge_disable(struct drm_device *dev,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *old_conn_state, *new_conn_state;
@@ -1251,7 +1251,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_encoder_bridge_disable);
  * it, disables it.
  */
 void
-drm_atomic_helper_commit_crtc_disable(struct drm_device *dev, struct drm_atomic_state *state)
+drm_atomic_helper_commit_crtc_disable(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
@@ -1313,7 +1313,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_crtc_disable);
  * it, post-disables all encoder bridges.
  */
 void
-drm_atomic_helper_commit_encoder_bridge_post_disable(struct drm_device *dev, struct drm_atomic_state *state)
+drm_atomic_helper_commit_encoder_bridge_post_disable(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *old_conn_state, *new_conn_state;
@@ -1367,7 +1367,7 @@ drm_atomic_helper_commit_encoder_bridge_post_disable(struct drm_device *dev, str
 EXPORT_SYMBOL(drm_atomic_helper_commit_encoder_bridge_post_disable);
 
 static void
-disable_outputs(struct drm_device *dev, struct drm_atomic_state *state)
+disable_outputs(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	drm_atomic_helper_commit_encoder_bridge_disable(dev, state);
 
@@ -1395,7 +1395,7 @@ disable_outputs(struct drm_device *dev, struct drm_atomic_state *state)
  */
 void
 drm_atomic_helper_update_legacy_modeset_state(struct drm_device *dev,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *old_conn_state, *new_conn_state;
@@ -1462,7 +1462,7 @@ EXPORT_SYMBOL(drm_atomic_helper_update_legacy_modeset_state);
  * Updates the timestamping constants used for precise vblank timestamps
  * by calling drm_calc_timestamping_constants() for all enabled crtcs in @state.
  */
-void drm_atomic_helper_calc_timestamping_constants(struct drm_atomic_state *state)
+void drm_atomic_helper_calc_timestamping_constants(struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_crtc *crtc;
@@ -1486,7 +1486,7 @@ EXPORT_SYMBOL(drm_atomic_helper_calc_timestamping_constants);
  * chain and change the mode in all bridges as well.
  */
 void
-drm_atomic_helper_commit_crtc_set_mode(struct drm_device *dev, struct drm_atomic_state *state)
+drm_atomic_helper_commit_crtc_set_mode(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state;
@@ -1564,7 +1564,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_crtc_set_mode);
  * PM since planes updates then only happen when the CRTC is actually enabled.
  */
 void drm_atomic_helper_commit_modeset_disables(struct drm_device *dev,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	disable_outputs(dev, state);
 
@@ -1585,7 +1585,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_modeset_disables);
  * commit on each connector.
  */
 void drm_atomic_helper_commit_writebacks(struct drm_device *dev,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *new_conn_state;
@@ -1615,7 +1615,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_writebacks);
  * the entire bridge chain.
  */
 void
-drm_atomic_helper_commit_encoder_bridge_pre_enable(struct drm_device *dev, struct drm_atomic_state *state)
+drm_atomic_helper_commit_encoder_bridge_pre_enable(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *new_conn_state;
@@ -1657,7 +1657,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_encoder_bridge_pre_enable);
  * it, enables it.
  */
 void
-drm_atomic_helper_commit_crtc_enable(struct drm_device *dev, struct drm_atomic_state *state)
+drm_atomic_helper_commit_crtc_enable(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state;
@@ -1697,7 +1697,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_crtc_enable);
  * it, enables the entire bridge chain.
  */
 void
-drm_atomic_helper_commit_encoder_bridge_enable(struct drm_device *dev, struct drm_atomic_state *state)
+drm_atomic_helper_commit_encoder_bridge_enable(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *new_conn_state;
@@ -1757,7 +1757,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_encoder_bridge_enable);
  * PM since planes updates then only happen when the CRTC is actually enabled.
  */
 void drm_atomic_helper_commit_modeset_enables(struct drm_device *dev,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	drm_atomic_helper_commit_crtc_enable(dev, state);
 
@@ -1774,7 +1774,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_modeset_enables);
  * next vblank, and inform all the fences of the deadline.
  */
 static void set_fence_deadline(struct drm_device *dev,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state;
@@ -1832,7 +1832,7 @@ static void set_fence_deadline(struct drm_device *dev,
  * Returns zero if success or < 0 if dma_fence_wait() fails.
  */
 int drm_atomic_helper_wait_for_fences(struct drm_device *dev,
-				      struct drm_atomic_state *state,
+				      struct drm_atomic_commit *state,
 				      bool pre_swap)
 {
 	struct drm_plane *plane;
@@ -1881,7 +1881,7 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_fences);
  */
 void
 drm_atomic_helper_wait_for_vblanks(struct drm_device *dev,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
@@ -1942,7 +1942,7 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_vblanks);
  * initialized using drm_atomic_helper_setup_commit().
  */
 void drm_atomic_helper_wait_for_flip_done(struct drm_device *dev,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	int i;
@@ -1980,7 +1980,7 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_flip_done);
  * Note that the default ordering of how the various stages are called is to
  * match the legacy modeset helper library closest.
  */
-void drm_atomic_helper_commit_tail(struct drm_atomic_state *state)
+void drm_atomic_helper_commit_tail(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 
@@ -2010,7 +2010,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_tail);
  * commit. Otherwise, one should use the default implementation
  * drm_atomic_helper_commit_tail().
  */
-void drm_atomic_helper_commit_tail_rpm(struct drm_atomic_state *state)
+void drm_atomic_helper_commit_tail_rpm(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 
@@ -2031,7 +2031,7 @@ void drm_atomic_helper_commit_tail_rpm(struct drm_atomic_state *state)
 }
 EXPORT_SYMBOL(drm_atomic_helper_commit_tail_rpm);
 
-static void commit_tail(struct drm_atomic_state *state)
+static void commit_tail(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	const struct drm_mode_config_helper_funcs *funcs;
@@ -2081,13 +2081,13 @@ static void commit_tail(struct drm_atomic_state *state)
 
 	drm_atomic_helper_commit_cleanup_done(state);
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 }
 
 static void commit_work(struct work_struct *work)
 {
-	struct drm_atomic_state *state = container_of(work,
-						      struct drm_atomic_state,
+	struct drm_atomic_commit *state = container_of(work,
+						      struct drm_atomic_commit,
 						      commit_work);
 	commit_tail(state);
 }
@@ -2106,7 +2106,7 @@ static void commit_work(struct work_struct *work)
  * fails the commit should be treated like a normal synchronous commit.
  */
 int drm_atomic_helper_async_check(struct drm_device *dev,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
@@ -2190,7 +2190,7 @@ EXPORT_SYMBOL(drm_atomic_helper_async_check);
  * TODO: Implement full swap instead of doing in-place changes.
  */
 void drm_atomic_helper_async_commit(struct drm_device *dev,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane *plane;
 	struct drm_plane_state *plane_state;
@@ -2243,7 +2243,7 @@ EXPORT_SYMBOL(drm_atomic_helper_async_commit);
  * Zero for success or -errno.
  */
 int drm_atomic_helper_commit(struct drm_device *dev,
-			     struct drm_atomic_state *state,
+			     struct drm_atomic_commit *state,
 			     bool nonblock)
 {
 	int ret;
@@ -2305,7 +2305,7 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 	 * make sure work items don't artificially stall on each another.
 	 */
 
-	drm_atomic_state_get(state);
+	drm_atomic_commit_get(state);
 	if (nonblock)
 		queue_work(system_dfl_wq, &state->commit_work);
 	else
@@ -2347,7 +2347,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit);
  * must be done as one global operation, and enabling or disabling a CRTC can
  * take a long time. But even that is not required.
  *
- * IMPORTANT: A &drm_atomic_state update for multiple CRTCs is sequenced
+ * IMPORTANT: A &drm_atomic_commit update for multiple CRTCs is sequenced
  * against all CRTCs therein. Therefore for atomic state updates which only flip
  * planes the driver must not get the struct &drm_crtc_state of unrelated CRTCs
  * in its atomic check code: This would prevent committing of atomic updates to
@@ -2442,7 +2442,7 @@ static void init_commit(struct drm_crtc_commit *commit, struct drm_crtc *crtc)
 }
 
 static struct drm_crtc_commit *
-crtc_or_fake_commit(struct drm_atomic_state *state, struct drm_crtc *crtc)
+crtc_or_fake_commit(struct drm_atomic_commit *state, struct drm_crtc *crtc)
 {
 	if (crtc) {
 		struct drm_crtc_state *new_crtc_state;
@@ -2501,14 +2501,14 @@ crtc_or_fake_commit(struct drm_atomic_state *state, struct drm_crtc *crtc)
  * using the core structure &drm_crtc_commit.
  *
  * By default there's no need to clean up resources allocated by this function
- * explicitly: drm_atomic_state_default_clear() will take care of that
+ * explicitly: drm_atomic_commit_default_clear() will take care of that
  * automatically.
  *
  * Returns:
  * 0 on success. -EBUSY when userspace schedules nonblocking commits too fast,
  * -ENOMEM on allocation failures and -EINTR when a signal is pending.
  */
-int drm_atomic_helper_setup_commit(struct drm_atomic_state *state,
+int drm_atomic_helper_setup_commit(struct drm_atomic_commit *state,
 				   bool nonblock)
 {
 	struct drm_crtc *crtc;
@@ -2633,7 +2633,7 @@ EXPORT_SYMBOL(drm_atomic_helper_setup_commit);
  * This is part of the atomic helper support for nonblocking commits, see
  * drm_atomic_helper_setup_commit() for an overview.
  */
-void drm_atomic_helper_wait_for_dependencies(struct drm_atomic_state *state)
+void drm_atomic_helper_wait_for_dependencies(struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state;
@@ -2688,7 +2688,7 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_dependencies);
  * This is part of the atomic helper support for nonblocking commits, see
  * drm_atomic_helper_setup_commit() for an overview.
  */
-void drm_atomic_helper_fake_vblank(struct drm_atomic_state *state)
+void drm_atomic_helper_fake_vblank(struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_crtc *crtc;
@@ -2726,7 +2726,7 @@ EXPORT_SYMBOL(drm_atomic_helper_fake_vblank);
  * This is part of the atomic helper support for nonblocking commits, see
  * drm_atomic_helper_setup_commit() for an overview.
  */
-void drm_atomic_helper_commit_hw_done(struct drm_atomic_state *state)
+void drm_atomic_helper_commit_hw_done(struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
@@ -2766,12 +2766,12 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_hw_done);
  *
  * This signals completion of the atomic update @state, including any
  * cleanup work. If used, it must be called right before calling
- * drm_atomic_state_put().
+ * drm_atomic_commit_put().
  *
  * This is part of the atomic helper support for nonblocking commits, see
  * drm_atomic_helper_setup_commit() for an overview.
  */
-void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *state)
+void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state;
@@ -2812,7 +2812,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_cleanup_done);
  * 0 on success, negative error code on failure.
  */
 int drm_atomic_helper_prepare_planes(struct drm_device *dev,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *new_conn_state;
@@ -2901,7 +2901,7 @@ EXPORT_SYMBOL(drm_atomic_helper_prepare_planes);
  * use drm_atomic_helper_cleanup_planes().
  */
 void drm_atomic_helper_unprepare_planes(struct drm_device *dev,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane *plane;
 	struct drm_plane_state *new_plane_state;
@@ -2970,7 +2970,7 @@ static bool plane_crtc_active(const struct drm_plane_state *state)
  * This should not be copied blindly by drivers.
  */
 void drm_atomic_helper_commit_planes(struct drm_device *dev,
-				     struct drm_atomic_state *state,
+				     struct drm_atomic_commit *state,
 				     uint32_t flags)
 {
 	struct drm_crtc *crtc;
@@ -3093,7 +3093,7 @@ drm_atomic_helper_commit_planes_on_crtc(struct drm_crtc_state *old_crtc_state)
 {
 	const struct drm_crtc_helper_funcs *crtc_funcs;
 	struct drm_crtc *crtc = old_crtc_state->crtc;
-	struct drm_atomic_state *old_state = old_crtc_state->state;
+	struct drm_atomic_commit *old_state = old_crtc_state->state;
 	struct drm_crtc_state *new_crtc_state =
 		drm_atomic_get_new_crtc_state(old_state, crtc);
 	struct drm_plane *plane;
@@ -3200,7 +3200,7 @@ EXPORT_SYMBOL(drm_atomic_helper_disable_planes_on_crtc);
  * drm_atomic_helper_unprepare_planes() in this case.
  */
 void drm_atomic_helper_cleanup_planes(struct drm_device *dev,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane *plane;
 	struct drm_plane_state *old_plane_state;
@@ -3249,7 +3249,7 @@ EXPORT_SYMBOL(drm_atomic_helper_cleanup_planes);
  * Returns 0 on success. Can return -ERESTARTSYS when @stall is true and the
  * waiting for the previous commits has been interrupted.
  */
-int drm_atomic_helper_swap_state(struct drm_atomic_state *state,
+int drm_atomic_helper_swap_state(struct drm_atomic_commit *state,
 				  bool stall)
 {
 	int i, ret;
@@ -3404,11 +3404,11 @@ int drm_atomic_helper_update_plane(struct drm_plane *plane,
 				   uint32_t src_w, uint32_t src_h,
 				   struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_plane_state *plane_state;
 	int ret = 0;
 
-	state = drm_atomic_state_alloc(plane->dev);
+	state = drm_atomic_commit_alloc(plane->dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -3437,7 +3437,7 @@ int drm_atomic_helper_update_plane(struct drm_plane *plane,
 
 	ret = drm_atomic_commit(state);
 fail:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_update_plane);
@@ -3455,11 +3455,11 @@ EXPORT_SYMBOL(drm_atomic_helper_update_plane);
 int drm_atomic_helper_disable_plane(struct drm_plane *plane,
 				    struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_plane_state *plane_state;
 	int ret = 0;
 
-	state = drm_atomic_state_alloc(plane->dev);
+	state = drm_atomic_commit_alloc(plane->dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -3479,7 +3479,7 @@ int drm_atomic_helper_disable_plane(struct drm_plane *plane,
 
 	ret = drm_atomic_commit(state);
 fail:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_disable_plane);
@@ -3503,11 +3503,11 @@ EXPORT_SYMBOL(drm_atomic_helper_disable_plane);
 int drm_atomic_helper_set_config(struct drm_mode_set *set,
 				 struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_crtc *crtc = set->crtc;
 	int ret = 0;
 
-	state = drm_atomic_state_alloc(crtc->dev);
+	state = drm_atomic_commit_alloc(crtc->dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -3523,7 +3523,7 @@ int drm_atomic_helper_set_config(struct drm_mode_set *set,
 	ret = drm_atomic_commit(state);
 
 fail:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_set_config);
@@ -3554,7 +3554,7 @@ EXPORT_SYMBOL(drm_atomic_helper_set_config);
 int drm_atomic_helper_disable_all(struct drm_device *dev,
 				  struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_connector_state *conn_state;
 	struct drm_connector *conn;
 	struct drm_plane_state *plane_state;
@@ -3563,7 +3563,7 @@ int drm_atomic_helper_disable_all(struct drm_device *dev,
 	struct drm_crtc *crtc;
 	int ret, i;
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_commit_alloc(dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -3607,7 +3607,7 @@ int drm_atomic_helper_disable_all(struct drm_device *dev,
 
 	ret = drm_atomic_commit(state);
 free:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_disable_all);
@@ -3634,11 +3634,11 @@ EXPORT_SYMBOL(drm_atomic_helper_disable_all);
 int drm_atomic_helper_reset_crtc(struct drm_crtc *crtc,
 				 struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_crtc_state *crtc_state;
 	int ret;
 
-	state = drm_atomic_state_alloc(crtc->dev);
+	state = drm_atomic_commit_alloc(crtc->dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -3654,7 +3654,7 @@ int drm_atomic_helper_reset_crtc(struct drm_crtc *crtc,
 
 	ret = drm_atomic_commit(state);
 out:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	return ret;
 }
@@ -3715,18 +3715,18 @@ EXPORT_SYMBOL(drm_atomic_helper_shutdown);
  * See also:
  * drm_atomic_helper_suspend(), drm_atomic_helper_resume()
  */
-struct drm_atomic_state *
+struct drm_atomic_commit *
 drm_atomic_helper_duplicate_state(struct drm_device *dev,
 				  struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_connector *conn;
 	struct drm_connector_list_iter conn_iter;
 	struct drm_plane *plane;
 	struct drm_crtc *crtc;
 	int err = 0;
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_commit_alloc(dev);
 	if (!state)
 		return ERR_PTR(-ENOMEM);
 
@@ -3778,7 +3778,7 @@ drm_atomic_helper_duplicate_state(struct drm_device *dev,
 
 free:
 	if (err < 0) {
-		drm_atomic_state_put(state);
+		drm_atomic_commit_put(state);
 		state = ERR_PTR(err);
 	}
 
@@ -3811,10 +3811,10 @@ EXPORT_SYMBOL(drm_atomic_helper_duplicate_state);
  * drm_atomic_helper_duplicate_state(), drm_atomic_helper_disable_all(),
  * drm_atomic_helper_resume(), drm_atomic_helper_commit_duplicated_state()
  */
-struct drm_atomic_state *drm_atomic_helper_suspend(struct drm_device *dev)
+struct drm_atomic_commit *drm_atomic_helper_suspend(struct drm_device *dev)
 {
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	int err;
 
 	/* This can never be returned, but it makes the compiler happy */
@@ -3828,7 +3828,7 @@ struct drm_atomic_state *drm_atomic_helper_suspend(struct drm_device *dev)
 
 	err = drm_atomic_helper_disable_all(dev, &ctx);
 	if (err < 0) {
-		drm_atomic_state_put(state);
+		drm_atomic_commit_put(state);
 		state = ERR_PTR(err);
 		goto unlock;
 	}
@@ -3857,7 +3857,7 @@ EXPORT_SYMBOL(drm_atomic_helper_suspend);
  * See also:
  * drm_atomic_helper_suspend()
  */
-int drm_atomic_helper_commit_duplicated_state(struct drm_atomic_state *state,
+int drm_atomic_helper_commit_duplicated_state(struct drm_atomic_commit *state,
 					      struct drm_modeset_acquire_ctx *ctx)
 {
 	int i, ret;
@@ -3909,7 +3909,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit_duplicated_state);
  * drm_atomic_helper_suspend()
  */
 int drm_atomic_helper_resume(struct drm_device *dev,
-			     struct drm_atomic_state *state)
+			     struct drm_atomic_commit *state)
 {
 	struct drm_modeset_acquire_ctx ctx;
 	int err;
@@ -3921,13 +3921,13 @@ int drm_atomic_helper_resume(struct drm_device *dev,
 	err = drm_atomic_helper_commit_duplicated_state(state, &ctx);
 
 	DRM_MODESET_LOCK_ALL_END(dev, ctx, err);
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	return err;
 }
 EXPORT_SYMBOL(drm_atomic_helper_resume);
 
-static int page_flip_common(struct drm_atomic_state *state,
+static int page_flip_common(struct drm_atomic_commit *state,
 			    struct drm_crtc *crtc,
 			    struct drm_framebuffer *fb,
 			    struct drm_pending_vblank_event *event,
@@ -3990,10 +3990,10 @@ int drm_atomic_helper_page_flip(struct drm_crtc *crtc,
 				struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_plane *plane = crtc->primary;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	int ret = 0;
 
-	state = drm_atomic_state_alloc(plane->dev);
+	state = drm_atomic_commit_alloc(plane->dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -4005,7 +4005,7 @@ int drm_atomic_helper_page_flip(struct drm_crtc *crtc,
 
 	ret = drm_atomic_nonblocking_commit(state);
 fail:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_page_flip);
@@ -4034,11 +4034,11 @@ int drm_atomic_helper_page_flip_target(struct drm_crtc *crtc,
 				       struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_plane *plane = crtc->primary;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_crtc_state *crtc_state;
 	int ret = 0;
 
-	state = drm_atomic_state_alloc(plane->dev);
+	state = drm_atomic_commit_alloc(plane->dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -4057,7 +4057,7 @@ int drm_atomic_helper_page_flip_target(struct drm_crtc *crtc,
 
 	ret = drm_atomic_nonblocking_commit(state);
 fail:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 	return ret;
 }
 EXPORT_SYMBOL(drm_atomic_helper_page_flip_target);
diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c
index 76746ad4a1bbb..cc70508d4fdba 100644
--- a/drivers/gpu/drm/drm_atomic_state_helper.c
+++ b/drivers/gpu/drm/drm_atomic_state_helper.c
@@ -589,7 +589,7 @@ EXPORT_SYMBOL(drm_atomic_helper_connector_tv_reset);
  * %0 for success, a negative error code on error.
  */
 int drm_atomic_helper_connector_tv_check(struct drm_connector *connector,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *old_conn_state =
 		drm_atomic_get_old_connector_state(state, connector);
diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c
index 5bd5bf6661df7..6441b55cc2740 100644
--- a/drivers/gpu/drm/drm_atomic_uapi.c
+++ b/drivers/gpu/drm/drm_atomic_uapi.c
@@ -347,13 +347,13 @@ drm_atomic_set_crtc_for_connector(struct drm_connector_state *conn_state,
 }
 EXPORT_SYMBOL(drm_atomic_set_crtc_for_connector);
 
-static void set_out_fence_for_crtc(struct drm_atomic_state *state,
+static void set_out_fence_for_crtc(struct drm_atomic_commit *state,
 				   struct drm_crtc *crtc, s32 __user *fence_ptr)
 {
 	state->crtcs[drm_crtc_index(crtc)].out_fence_ptr = fence_ptr;
 }
 
-static s32 __user *get_out_fence_for_crtc(struct drm_atomic_state *state,
+static s32 __user *get_out_fence_for_crtc(struct drm_atomic_commit *state,
 					  struct drm_crtc *crtc)
 {
 	s32 __user *fence_ptr;
@@ -364,7 +364,7 @@ static s32 __user *get_out_fence_for_crtc(struct drm_atomic_state *state,
 	return fence_ptr;
 }
 
-static int set_out_fence_for_connector(struct drm_atomic_state *state,
+static int set_out_fence_for_connector(struct drm_atomic_commit *state,
 					struct drm_connector *connector,
 					s32 __user *fence_ptr)
 {
@@ -381,7 +381,7 @@ static int set_out_fence_for_connector(struct drm_atomic_state *state,
 	return 0;
 }
 
-static s32 __user *get_out_fence_for_connector(struct drm_atomic_state *state,
+static s32 __user *get_out_fence_for_connector(struct drm_atomic_commit *state,
 					       struct drm_connector *connector)
 {
 	unsigned int index = drm_connector_index(connector);
@@ -1104,7 +1104,7 @@ static struct drm_pending_vblank_event *create_vblank_event(
 	return e;
 }
 
-int drm_atomic_connector_commit_dpms(struct drm_atomic_state *state,
+int drm_atomic_connector_commit_dpms(struct drm_atomic_commit *state,
 				     struct drm_connector *connector,
 				     int mode)
 {
@@ -1171,7 +1171,7 @@ static int drm_atomic_check_prop_changes(int ret, uint64_t old_val, uint64_t pro
 	return 0;
 }
 
-int drm_atomic_set_property(struct drm_atomic_state *state,
+int drm_atomic_set_property(struct drm_atomic_commit *state,
 			    struct drm_file *file_priv,
 			    struct drm_mode_object *obj,
 			    struct drm_property *prop,
@@ -1374,7 +1374,7 @@ static int setup_out_fence(struct drm_out_fence_state *fence_state,
 }
 
 static int prepare_signaling(struct drm_device *dev,
-				  struct drm_atomic_state *state,
+				  struct drm_atomic_commit *state,
 				  struct drm_mode_atomic *arg,
 				  struct drm_file *file_priv,
 				  struct drm_out_fence_state **fence_state,
@@ -1499,7 +1499,7 @@ static int prepare_signaling(struct drm_device *dev,
 }
 
 static void complete_signaling(struct drm_device *dev,
-			       struct drm_atomic_state *state,
+			       struct drm_atomic_commit *state,
 			       struct drm_out_fence_state *fence_state,
 			       unsigned int num_fences,
 			       bool install_fds)
@@ -1522,7 +1522,7 @@ static void complete_signaling(struct drm_device *dev,
 		/*
 		 * Free the allocated event. drm_atomic_helper_setup_commit
 		 * can allocate an event too, so only free it if it's ours
-		 * to prevent a double free in drm_atomic_state_clear.
+		 * to prevent a double free in drm_atomic_commit_clear.
 		 */
 		if (event && (event->base.fence || event->base.file_priv)) {
 			drm_event_cancel_free(dev, &event->base);
@@ -1549,7 +1549,7 @@ static void complete_signaling(struct drm_device *dev,
 }
 
 static void
-set_async_flip(struct drm_atomic_state *state)
+set_async_flip(struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
@@ -1569,7 +1569,7 @@ int drm_mode_atomic_ioctl(struct drm_device *dev,
 	uint32_t __user *props_ptr = (uint32_t __user *)(unsigned long)(arg->props_ptr);
 	uint64_t __user *prop_values_ptr = (uint64_t __user *)(unsigned long)(arg->prop_values_ptr);
 	unsigned int copied_objs, copied_props;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_out_fence_state *fence_state;
 	int ret = 0;
@@ -1618,7 +1618,7 @@ int drm_mode_atomic_ioctl(struct drm_device *dev,
 		return -EINVAL;
 	}
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_commit_alloc(dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -1726,13 +1726,13 @@ int drm_mode_atomic_ioctl(struct drm_device *dev,
 	complete_signaling(dev, state, fence_state, num_fences, !ret);
 
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry;
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c
index e17bb0f1f9e0b..e5013b870ba06 100644
--- a/drivers/gpu/drm/drm_auth.c
+++ b/drivers/gpu/drm/drm_auth.c
@@ -347,7 +347,7 @@ void drm_master_release(struct drm_file *file_priv)
 	if (!drm_is_current_master_locked(file_priv))
 		goto out;
 
-	if (dev->master == file_priv->master)
+	if (dev->master && dev->master == file_priv->master)
 		drm_drop_master(dev, file_priv);
 out:
 	if (drm_core_check_feature(dev, DRIVER_MODESET) && file_priv->is_master) {
diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c
index 1f3af27d2418e..2f0d1ba285be8 100644
--- a/drivers/gpu/drm/drm_blend.c
+++ b/drivers/gpu/drm/drm_blend.c
@@ -449,7 +449,7 @@ int drm_plane_create_zpos_immutable_property(struct drm_plane *plane,
 }
 EXPORT_SYMBOL(drm_plane_create_zpos_immutable_property);
 
-static int drm_atomic_state_zpos_cmp(const void *a, const void *b)
+static int drm_atomic_commit_zpos_cmp(const void *a, const void *b)
 {
 	const struct drm_plane_state *sa = *(struct drm_plane_state **)a;
 	const struct drm_plane_state *sb = *(struct drm_plane_state **)b;
@@ -463,7 +463,7 @@ static int drm_atomic_state_zpos_cmp(const void *a, const void *b)
 static int drm_atomic_helper_crtc_normalize_zpos(struct drm_crtc *crtc,
 					  struct drm_crtc_state *crtc_state)
 {
-	struct drm_atomic_state *state = crtc_state->state;
+	struct drm_atomic_commit *state = crtc_state->state;
 	struct drm_device *dev = crtc->dev;
 	int total_planes = dev->mode_config.num_total_plane;
 	struct drm_plane_state **states;
@@ -494,7 +494,7 @@ static int drm_atomic_helper_crtc_normalize_zpos(struct drm_crtc *crtc,
 			       plane->base.id, plane->name, plane_state->zpos);
 	}
 
-	sort(states, n, sizeof(*states), drm_atomic_state_zpos_cmp, NULL);
+	sort(states, n, sizeof(*states), drm_atomic_commit_zpos_cmp, NULL);
 
 	for (i = 0; i < n; i++) {
 		plane = states[i]->plane;
@@ -529,7 +529,7 @@ static int drm_atomic_helper_crtc_normalize_zpos(struct drm_crtc *crtc,
  * Zero for success or -errno
  */
 int drm_atomic_normalize_zpos(struct drm_device *dev,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index d6f512b733896..687b36eea0c70 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -282,7 +282,7 @@ static void __drm_bridge_free(struct kref *kref)
 
 /**
  * drm_bridge_get - Acquire a bridge reference
- * @bridge: DRM bridge
+ * @bridge: DRM bridge; if NULL this function does nothing
  *
  * This function increments the bridge's refcount.
  *
@@ -300,7 +300,7 @@ EXPORT_SYMBOL(drm_bridge_get);
 
 /**
  * drm_bridge_put - Release a bridge reference
- * @bridge: DRM bridge
+ * @bridge: DRM bridge; if NULL or an ERR_PTR this function does nothing
  *
  * This function decrements the bridge's reference count and frees the
  * object if the reference count drops to zero.
@@ -310,7 +310,7 @@ EXPORT_SYMBOL(drm_bridge_get);
  */
 void drm_bridge_put(struct drm_bridge *bridge)
 {
-	if (bridge)
+	if (!IS_ERR_OR_NULL(bridge))
 		kref_put(&bridge->refcount, __drm_bridge_free);
 }
 EXPORT_SYMBOL(drm_bridge_put);
@@ -574,10 +574,12 @@ int drm_bridge_attach(struct drm_encoder *encoder, struct drm_bridge *bridge,
 	bridge->dev = encoder->dev;
 	bridge->encoder = encoder;
 
+	mutex_lock(&encoder->bridge_chain_mutex);
 	if (previous)
 		list_add(&bridge->chain_node, &previous->chain_node);
 	else
 		list_add(&bridge->chain_node, &encoder->bridge_chain);
+	mutex_unlock(&encoder->bridge_chain_mutex);
 
 	if (bridge->funcs->attach) {
 		ret = bridge->funcs->attach(bridge, encoder, flags);
@@ -594,7 +596,9 @@ int drm_bridge_attach(struct drm_encoder *encoder, struct drm_bridge *bridge,
 err_reset_bridge:
 	bridge->dev = NULL;
 	bridge->encoder = NULL;
+	mutex_lock(&encoder->bridge_chain_mutex);
 	list_del(&bridge->chain_node);
+	mutex_unlock(&encoder->bridge_chain_mutex);
 
 	if (ret != -EPROBE_DEFER)
 		DRM_ERROR("failed to attach bridge %pOF to encoder %s: %d\n",
@@ -721,7 +725,7 @@ void drm_bridge_detach(struct drm_bridge *bridge)
 /**
  * drm_bridge_chain_mode_valid - validate the mode against all bridges in the
  *				 encoder chain.
- * @bridge: bridge control structure
+ * @first_bridge: bridge control structure
  * @info: display info against which the mode shall be validated
  * @mode: desired mode to be validated
  *
@@ -735,17 +739,14 @@ void drm_bridge_detach(struct drm_bridge *bridge)
  * MODE_OK on success, drm_mode_status Enum error code on failure
  */
 enum drm_mode_status
-drm_bridge_chain_mode_valid(struct drm_bridge *bridge,
+drm_bridge_chain_mode_valid(struct drm_bridge *first_bridge,
 			    const struct drm_display_info *info,
 			    const struct drm_display_mode *mode)
 {
-	struct drm_encoder *encoder;
-
-	if (!bridge)
+	if (!first_bridge)
 		return MODE_OK;
 
-	encoder = bridge->encoder;
-	list_for_each_entry_from(bridge, &encoder->bridge_chain, chain_node) {
+	drm_for_each_bridge_in_chain_from(first_bridge, bridge) {
 		enum drm_mode_status ret;
 
 		if (!bridge->funcs->mode_valid)
@@ -763,7 +764,7 @@ EXPORT_SYMBOL(drm_bridge_chain_mode_valid);
 /**
  * drm_bridge_chain_mode_set - set proposed mode for all bridges in the
  *			       encoder chain
- * @bridge: bridge control structure
+ * @first_bridge: bridge control structure
  * @mode: desired mode to be set for the encoder chain
  * @adjusted_mode: updated mode that works for this encoder chain
  *
@@ -772,20 +773,16 @@ EXPORT_SYMBOL(drm_bridge_chain_mode_valid);
  *
  * Note: the bridge passed should be the one closest to the encoder
  */
-void drm_bridge_chain_mode_set(struct drm_bridge *bridge,
+void drm_bridge_chain_mode_set(struct drm_bridge *first_bridge,
 			       const struct drm_display_mode *mode,
 			       const struct drm_display_mode *adjusted_mode)
 {
-	struct drm_encoder *encoder;
-
-	if (!bridge)
+	if (!first_bridge)
 		return;
 
-	encoder = bridge->encoder;
-	list_for_each_entry_from(bridge, &encoder->bridge_chain, chain_node) {
+	drm_for_each_bridge_in_chain_from(first_bridge, bridge)
 		if (bridge->funcs->mode_set)
 			bridge->funcs->mode_set(bridge, mode, adjusted_mode);
-	}
 }
 EXPORT_SYMBOL(drm_bridge_chain_mode_set);
 
@@ -802,7 +799,7 @@ EXPORT_SYMBOL(drm_bridge_chain_mode_set);
  * Note: the bridge passed should be the one closest to the encoder
  */
 void drm_atomic_bridge_chain_disable(struct drm_bridge *bridge,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_encoder *encoder;
 	struct drm_bridge *iter;
@@ -811,6 +808,7 @@ void drm_atomic_bridge_chain_disable(struct drm_bridge *bridge,
 		return;
 
 	encoder = bridge->encoder;
+	mutex_lock(&encoder->bridge_chain_mutex);
 	list_for_each_entry_reverse(iter, &encoder->bridge_chain, chain_node) {
 		if (iter->funcs->atomic_disable) {
 			iter->funcs->atomic_disable(iter, state);
@@ -821,11 +819,12 @@ void drm_atomic_bridge_chain_disable(struct drm_bridge *bridge,
 		if (iter == bridge)
 			break;
 	}
+	mutex_unlock(&encoder->bridge_chain_mutex);
 }
 EXPORT_SYMBOL(drm_atomic_bridge_chain_disable);
 
 static void drm_atomic_bridge_call_post_disable(struct drm_bridge *bridge,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	if (state && bridge->funcs->atomic_post_disable)
 		bridge->funcs->atomic_post_disable(bridge, state);
@@ -858,7 +857,7 @@ static void drm_atomic_bridge_call_post_disable(struct drm_bridge *bridge,
  * Note: the bridge passed should be the one closest to the encoder
  */
 void drm_atomic_bridge_chain_post_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_encoder *encoder;
 	struct drm_bridge *next, *limit;
@@ -868,6 +867,7 @@ void drm_atomic_bridge_chain_post_disable(struct drm_bridge *bridge,
 
 	encoder = bridge->encoder;
 
+	mutex_lock(&encoder->bridge_chain_mutex);
 	list_for_each_entry_from(bridge, &encoder->bridge_chain, chain_node) {
 		limit = NULL;
 
@@ -916,11 +916,12 @@ void drm_atomic_bridge_chain_post_disable(struct drm_bridge *bridge,
 			/* Jump all bridges that we have already post_disabled */
 			bridge = limit;
 	}
+	mutex_unlock(&encoder->bridge_chain_mutex);
 }
 EXPORT_SYMBOL(drm_atomic_bridge_chain_post_disable);
 
 static void drm_atomic_bridge_call_pre_enable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	if (state && bridge->funcs->atomic_pre_enable)
 		bridge->funcs->atomic_pre_enable(bridge, state);
@@ -952,7 +953,7 @@ static void drm_atomic_bridge_call_pre_enable(struct drm_bridge *bridge,
  * Note: the bridge passed should be the one closest to the encoder
  */
 void drm_atomic_bridge_chain_pre_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_encoder *encoder;
 	struct drm_bridge *iter, *next, *limit;
@@ -962,6 +963,7 @@ void drm_atomic_bridge_chain_pre_enable(struct drm_bridge *bridge,
 
 	encoder = bridge->encoder;
 
+	mutex_lock(&encoder->bridge_chain_mutex);
 	list_for_each_entry_reverse(iter, &encoder->bridge_chain, chain_node) {
 		if (iter->pre_enable_prev_first) {
 			next = iter;
@@ -1004,12 +1006,13 @@ void drm_atomic_bridge_chain_pre_enable(struct drm_bridge *bridge,
 		if (iter == bridge)
 			break;
 	}
+	mutex_unlock(&encoder->bridge_chain_mutex);
 }
 EXPORT_SYMBOL(drm_atomic_bridge_chain_pre_enable);
 
 /**
  * drm_atomic_bridge_chain_enable - enables all bridges in the encoder chain
- * @bridge: bridge control structure
+ * @first_bridge: bridge control structure
  * @state: atomic state being committed
  *
  * Calls &drm_bridge_funcs.atomic_enable (falls back on
@@ -1019,22 +1022,18 @@ EXPORT_SYMBOL(drm_atomic_bridge_chain_pre_enable);
  *
  * Note: the bridge passed should be the one closest to the encoder
  */
-void drm_atomic_bridge_chain_enable(struct drm_bridge *bridge,
-				    struct drm_atomic_state *state)
+void drm_atomic_bridge_chain_enable(struct drm_bridge *first_bridge,
+				    struct drm_atomic_commit *state)
 {
-	struct drm_encoder *encoder;
-
-	if (!bridge)
+	if (!first_bridge)
 		return;
 
-	encoder = bridge->encoder;
-	list_for_each_entry_from(bridge, &encoder->bridge_chain, chain_node) {
+	drm_for_each_bridge_in_chain_from(first_bridge, bridge)
 		if (bridge->funcs->atomic_enable) {
 			bridge->funcs->atomic_enable(bridge, state);
 		} else if (bridge->funcs->enable) {
 			bridge->funcs->enable(bridge);
 		}
-	}
 }
 EXPORT_SYMBOL(drm_atomic_bridge_chain_enable);
 
@@ -1249,7 +1248,7 @@ drm_atomic_bridge_chain_select_bus_fmts(struct drm_bridge *bridge,
 static void
 drm_atomic_bridge_propagate_bus_flags(struct drm_bridge *bridge,
 				      struct drm_connector *conn,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_bridge_state *bridge_state, *next_bridge_state;
 	u32 output_flags = 0;
@@ -1329,25 +1328,27 @@ int drm_atomic_bridge_chain_check(struct drm_bridge *bridge,
 		return ret;
 
 	encoder = bridge->encoder;
-	list_for_each_entry_reverse(iter, &encoder->bridge_chain, chain_node) {
-		int ret;
-
-		/*
-		 * Bus flags are propagated by default. If a bridge needs to
-		 * tweak the input bus flags for any reason, it should happen
-		 * in its &drm_bridge_funcs.atomic_check() implementation such
-		 * that preceding bridges in the chain can propagate the new
-		 * bus flags.
-		 */
-		drm_atomic_bridge_propagate_bus_flags(iter, conn,
-						      crtc_state->state);
-
-		ret = drm_atomic_bridge_check(iter, crtc_state, conn_state);
-		if (ret)
-			return ret;
+	scoped_guard(mutex, &encoder->bridge_chain_mutex) {
+		list_for_each_entry_reverse(iter, &encoder->bridge_chain, chain_node) {
+			int ret;
+
+			/*
+			 * Bus flags are propagated by default. If a bridge needs to
+			 * tweak the input bus flags for any reason, it should happen
+			 * in its &drm_bridge_funcs.atomic_check() implementation such
+			 * that preceding bridges in the chain can propagate the new
+			 * bus flags.
+			 */
+			drm_atomic_bridge_propagate_bus_flags(iter, conn,
+							      crtc_state->state);
+
+			ret = drm_atomic_bridge_check(iter, crtc_state, conn_state);
+			if (ret)
+				return ret;
 
-		if (iter == bridge)
-			break;
+			if (iter == bridge)
+				break;
+		}
 	}
 
 	return 0;
@@ -1581,6 +1582,47 @@ struct drm_bridge *of_drm_find_bridge(struct device_node *np)
 	return bridge;
 }
 EXPORT_SYMBOL(of_drm_find_bridge);
+
+/**
+ * of_drm_get_bridge_by_endpoint - return DRM bridge connected to a port/endpoint
+ * @np: device tree node containing output ports
+ * @port: port in the device tree node, or -1 for the first port found
+ * @endpoint: endpoint in the device tree node, or -1 for the first endpoint found
+ *
+ * Given a DT node's port and endpoint number, find the connected node and
+ * return the associated drm_bridge device.
+ *
+ * The refcount of the returned bridge is incremented. Use drm_bridge_put()
+ * when done with it.
+ *
+ * Returns a pointer to the connected drm_bridge, or a negative error on failure
+ */
+struct drm_bridge *of_drm_get_bridge_by_endpoint(const struct device_node *np,
+						 int port, int endpoint)
+{
+	struct drm_bridge *bridge;
+
+	/*
+	 * of_graph_get_remote_node() produces a noisy error message if port
+	 * node isn't found and the absence of the port is a legit case here,
+	 * so at first we silently check whether graph is present in the
+	 * device-tree node.
+	 */
+	if (!of_graph_is_present(np))
+		return ERR_PTR(-ENODEV);
+
+	struct device_node *remote __free(device_node) =
+		of_graph_get_remote_node(np, port, endpoint);
+	if (!remote)
+		return ERR_PTR(-ENODEV);
+
+	bridge = of_drm_find_and_get_bridge(remote);
+	if (!bridge)
+		return ERR_PTR(-EPROBE_DEFER);
+
+	return bridge;
+}
+EXPORT_SYMBOL_GPL(of_drm_get_bridge_by_endpoint);
 #endif
 
 /**
@@ -1613,18 +1655,17 @@ static void drm_bridge_debugfs_show_bridge(struct drm_printer *p,
 
 	drm_printf(p, "bridge[%u]: %ps\n", idx, bridge->funcs);
 
-	drm_printf(p, "\trefcount: %u%s\n", refcount,
-		   lingering ? " [lingering]" : "");
+	drm_printf_indent(p, 1, "refcount: %u%s\n", refcount,
+			  lingering ? " [lingering]" : "");
 
-	drm_printf(p, "\ttype: [%d] %s\n",
-		   bridge->type,
-		   drm_get_connector_type_name(bridge->type));
+	drm_printf_indent(p, 1, "type: [%d] %s\n", bridge->type,
+			  drm_get_connector_type_name(bridge->type));
 
 	/* The OF node could be freed after drm_bridge_remove() */
 	if (bridge->of_node && !lingering)
-		drm_printf(p, "\tOF: %pOFfc\n", bridge->of_node);
+		drm_printf_indent(p, 1, "OF: %pOFfc\n", bridge->of_node);
 
-	drm_printf(p, "\tops: [0x%x]", bridge->ops);
+	drm_printf_indent(p, 1, "ops: [0x%x]", bridge->ops);
 	if (bridge->ops & DRM_BRIDGE_OP_DETECT)
 		drm_puts(p, " detect");
 	if (bridge->ops & DRM_BRIDGE_OP_EDID)
diff --git a/drivers/gpu/drm/drm_buddy.c b/drivers/gpu/drm/drm_buddy.c
index 841f3de5f307a..faa025498de45 100644
--- a/drivers/gpu/drm/drm_buddy.c
+++ b/drivers/gpu/drm/drm_buddy.c
@@ -42,6 +42,7 @@ void drm_buddy_print(struct gpu_buddy *mm, struct drm_printer *p)
 {
 	int order;
 
+	gpu_buddy_driver_lock_held(mm);
 	drm_printf(p, "chunk_size: %lluKiB, total: %lluMiB, free: %lluMiB, clear_free: %lluMiB\n",
 		   mm->chunk_size >> 10, mm->size >> 20, mm->avail >> 20, mm->clear_avail >> 20);
 
diff --git a/drivers/gpu/drm/drm_client_modeset.c b/drivers/gpu/drm/drm_client_modeset.c
index bb49b8361271a..0080a8e9556a5 100644
--- a/drivers/gpu/drm/drm_client_modeset.c
+++ b/drivers/gpu/drm/drm_client_modeset.c
@@ -1035,14 +1035,14 @@ static int drm_client_modeset_commit_atomic(struct drm_client_dev *client, bool
 {
 	struct drm_device *dev = client->dev;
 	struct drm_plane *plane;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_mode_set *mode_set;
 	int ret;
 
 	drm_modeset_acquire_init(&ctx, 0);
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_commit_alloc(dev);
 	if (!state) {
 		ret = -ENOMEM;
 		goto out_ctx;
@@ -1107,7 +1107,7 @@ static int drm_client_modeset_commit_atomic(struct drm_client_dev *client, bool
 	if (ret == -EDEADLK)
 		goto backoff;
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 out_ctx:
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
@@ -1115,7 +1115,7 @@ static int drm_client_modeset_commit_atomic(struct drm_client_dev *client, bool
 	return ret;
 
 backoff:
-	drm_atomic_state_clear(state);
+	drm_atomic_commit_clear(state);
 	drm_modeset_backoff(&ctx);
 
 	goto retry;
diff --git a/drivers/gpu/drm/drm_color_mgmt.c b/drivers/gpu/drm/drm_color_mgmt.c
index e7db4e4ea700f..3bcf82cf61a7c 100644
--- a/drivers/gpu/drm/drm_color_mgmt.c
+++ b/drivers/gpu/drm/drm_color_mgmt.c
@@ -280,7 +280,7 @@ static int drm_crtc_legacy_gamma_set(struct drm_crtc *crtc,
 				     struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_device *dev = crtc->dev;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_crtc_state *crtc_state;
 	struct drm_property_blob *blob;
 	struct drm_color_lut *blob_data;
@@ -300,7 +300,7 @@ static int drm_crtc_legacy_gamma_set(struct drm_crtc *crtc,
 	else
 		return -ENODEV;
 
-	state = drm_atomic_state_alloc(crtc->dev);
+	state = drm_atomic_commit_alloc(crtc->dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -339,7 +339,7 @@ static int drm_crtc_legacy_gamma_set(struct drm_crtc *crtc,
 	ret = drm_atomic_commit(state);
 
 fail:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 	drm_property_blob_put(blob);
 	return ret;
 }
diff --git a/drivers/gpu/drm/drm_colorop.c b/drivers/gpu/drm/drm_colorop.c
index 566816e3c6f0c..764d120606665 100644
--- a/drivers/gpu/drm/drm_colorop.c
+++ b/drivers/gpu/drm/drm_colorop.c
@@ -516,10 +516,10 @@ static void __drm_colorop_state_reset(struct drm_colorop_state *colorop_state,
 	colorop_state->bypass = true;
 
 	if (colorop->curve_1d_type_property) {
-		drm_object_property_get_default_value(&colorop->base,
-						      colorop->curve_1d_type_property,
-						      &val);
-		colorop_state->curve_1d_type = val;
+		if (!drm_object_property_get_default_value(&colorop->base,
+							   colorop->curve_1d_type_property,
+							   &val))
+			colorop_state->curve_1d_type = val;
 	}
 }
 
diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c
index 47dc53c4a738f..3fa4d2082cd72 100644
--- a/drivers/gpu/drm/drm_connector.c
+++ b/drivers/gpu/drm/drm_connector.c
@@ -2867,23 +2867,18 @@ int drm_connector_attach_max_bpc_property(struct drm_connector *connector,
 EXPORT_SYMBOL(drm_connector_attach_max_bpc_property);
 
 /**
- * drm_connector_attach_hdr_output_metadata_property - attach "HDR_OUTPUT_METADA" property
+ * drm_connector_attach_hdr_output_metadata_property - attach "HDR_OUTPUT_METADATA" property
  * @connector: connector to attach the property on.
  *
  * This is used to allow the userspace to send HDR Metadata to the
  * driver.
- *
- * Returns:
- * Zero on success, negative errno on failure.
  */
-int drm_connector_attach_hdr_output_metadata_property(struct drm_connector *connector)
+void drm_connector_attach_hdr_output_metadata_property(struct drm_connector *connector)
 {
 	struct drm_device *dev = connector->dev;
 	struct drm_property *prop = dev->mode_config.hdr_output_metadata_property;
 
 	drm_object_attach_property(&connector->base, prop, 0);
-
-	return 0;
 }
 EXPORT_SYMBOL(drm_connector_attach_hdr_output_metadata_property);
 
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 5de9aaa5ba043..4a8cfe046e4ab 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -446,7 +446,7 @@ EXPORT_SYMBOL(drm_crtc_helper_set_mode);
  * RETURNS:
  * Zero on success, or an errno code otherwise.
  */
-int drm_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
+int drm_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index c094092296448..83146ffef00cd 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -44,7 +44,7 @@ enum drm_connector_force;
 enum drm_mode_status;
 
 struct cea_sad;
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_bridge;
 struct drm_connector;
 struct drm_crtc;
@@ -248,16 +248,16 @@ void drm_atomic_debugfs_init(struct drm_device *dev);
 int __drm_atomic_helper_disable_plane(struct drm_plane *plane,
 				      struct drm_plane_state *plane_state);
 int __drm_atomic_helper_set_config(struct drm_mode_set *set,
-				   struct drm_atomic_state *state);
+				   struct drm_atomic_commit *state);
 
-void drm_atomic_print_new_state(const struct drm_atomic_state *state,
+void drm_atomic_print_new_state(const struct drm_atomic_commit *state,
 		struct drm_printer *p);
 
 /* drm_atomic_uapi.c */
-int drm_atomic_connector_commit_dpms(struct drm_atomic_state *state,
+int drm_atomic_connector_commit_dpms(struct drm_atomic_commit *state,
 				     struct drm_connector *connector,
 				     int mode);
-int drm_atomic_set_property(struct drm_atomic_state *state,
+int drm_atomic_set_property(struct drm_atomic_commit *state,
 			    struct drm_file *file_priv,
 			    struct drm_mode_object *obj,
 			    struct drm_property *prop,
diff --git a/drivers/gpu/drm/drm_damage_helper.c b/drivers/gpu/drm/drm_damage_helper.c
index 1b6850aa1688a..74a7f4252ecff 100644
--- a/drivers/gpu/drm/drm_damage_helper.c
+++ b/drivers/gpu/drm/drm_damage_helper.c
@@ -66,7 +66,7 @@ static void convert_clip_rect_to_rect(const struct drm_clip_rect *src,
  * full plane update should happen. It also ensure helper iterator will return
  * &drm_plane_state.src as damage.
  */
-void drm_atomic_helper_check_plane_damage(struct drm_atomic_state *state,
+void drm_atomic_helper_check_plane_damage(struct drm_atomic_commit *state,
 					  struct drm_plane_state *plane_state)
 {
 	struct drm_crtc_state *crtc_state;
@@ -114,7 +114,7 @@ int drm_atomic_helper_dirtyfb(struct drm_framebuffer *fb,
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_property_blob *damage = NULL;
 	struct drm_mode_rect *rects = NULL;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_plane *plane;
 	int ret = 0;
 
@@ -125,7 +125,7 @@ int drm_atomic_helper_dirtyfb(struct drm_framebuffer *fb,
 	drm_modeset_acquire_init(&ctx,
 		file_priv ? DRM_MODESET_ACQUIRE_INTERRUPTIBLE : 0);
 
-	state = drm_atomic_state_alloc(fb->dev);
+	state = drm_atomic_commit_alloc(fb->dev);
 	if (!state) {
 		ret = -ENOMEM;
 		goto out_drop_locks;
@@ -184,7 +184,7 @@ int drm_atomic_helper_dirtyfb(struct drm_framebuffer *fb,
 
 out:
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry;
@@ -192,7 +192,7 @@ int drm_atomic_helper_dirtyfb(struct drm_framebuffer *fb,
 
 	drm_property_blob_put(damage);
 	kfree(rects);
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 out_drop_locks:
 	drm_modeset_drop_locks(&ctx);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 985c283cf59fa..675675480da49 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -697,6 +697,7 @@ static void drm_dev_init_release(struct drm_device *dev, void *res)
 	mutex_destroy(&dev->master_mutex);
 	mutex_destroy(&dev->clientlist_mutex);
 	mutex_destroy(&dev->filelist_mutex);
+	mutex_destroy(&dev->gem_lru_mutex);
 }
 
 static int drm_dev_init(struct drm_device *dev,
@@ -738,6 +739,7 @@ static int drm_dev_init(struct drm_device *dev,
 	INIT_LIST_HEAD(&dev->vblank_event_list);
 
 	spin_lock_init(&dev->event_lock);
+	mutex_init(&dev->gem_lru_mutex);
 	mutex_init(&dev->filelist_mutex);
 	mutex_init(&dev->clientlist_mutex);
 	mutex_init(&dev->master_mutex);
diff --git a/drivers/gpu/drm/drm_encoder.c b/drivers/gpu/drm/drm_encoder.c
index 8f2bc6a284822..0d5dbed06db44 100644
--- a/drivers/gpu/drm/drm_encoder.c
+++ b/drivers/gpu/drm/drm_encoder.c
@@ -129,6 +129,7 @@ static int __drm_encoder_init(struct drm_device *dev,
 	}
 
 	INIT_LIST_HEAD(&encoder->bridge_chain);
+	mutex_init(&encoder->bridge_chain_mutex);
 	list_add_tail(&encoder->head, &dev->mode_config.encoder_list);
 	encoder->index = dev->mode_config.num_encoder++;
 
@@ -188,20 +189,33 @@ void drm_encoder_cleanup(struct drm_encoder *encoder)
 {
 	struct drm_device *dev = encoder->dev;
 	struct drm_bridge *bridge, *next;
+	LIST_HEAD(tmplist);
 
 	/* Note that the encoder_list is considered to be static; should we
 	 * remove the drm_encoder at runtime we would have to decrement all
 	 * the indices on the drm_encoder after us in the encoder_list.
 	 */
 
-	list_for_each_entry_safe(bridge, next, &encoder->bridge_chain,
-				 chain_node)
+	/*
+	 * We need the bridge_chain_mutex to modify the chain, but
+	 * drm_bridge_detach() will call DRM_MODESET_LOCK_ALL_BEGIN() (in
+	 * drm_modeset_lock_fini()), resulting in a possible ABBA circular
+	 * deadlock. Avoid it by first moving all the bridges to a
+	 * temporary list holding the lock, and then calling
+	 * drm_bridge_detach() without the lock.
+	 */
+	mutex_lock(&encoder->bridge_chain_mutex);
+	list_cut_before(&tmplist, &encoder->bridge_chain, &encoder->bridge_chain);
+	mutex_unlock(&encoder->bridge_chain_mutex);
+
+	list_for_each_entry_safe(bridge, next, &tmplist, chain_node)
 		drm_bridge_detach(bridge);
 
 	drm_mode_object_unregister(dev, &encoder->base);
 	kfree(encoder->name);
 	list_del(&encoder->head);
 	dev->mode_config.num_encoder--;
+	mutex_destroy(&encoder->bridge_chain_mutex);
 
 	memset(encoder, 0, sizeof(*encoder));
 }
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 1541fc8a9ac2d..7b11a582f8ec4 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -763,7 +763,7 @@ static int setcmap_atomic(struct fb_cmap *cmap, struct fb_info *info)
 	struct drm_property_blob *gamma_lut = NULL;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_crtc_state *crtc_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_mode_set *modeset;
 	struct drm_crtc *crtc;
 	u16 *r, *g, *b;
@@ -772,7 +772,7 @@ static int setcmap_atomic(struct fb_cmap *cmap, struct fb_info *info)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_commit_alloc(dev);
 	if (!state) {
 		ret = -ENOMEM;
 		goto out_ctx;
@@ -831,7 +831,7 @@ static int setcmap_atomic(struct fb_cmap *cmap, struct fb_info *info)
 		goto backoff;
 
 	drm_property_blob_put(gamma_lut);
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 out_ctx:
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
@@ -839,7 +839,7 @@ static int setcmap_atomic(struct fb_cmap *cmap, struct fb_info *info)
 	return ret;
 
 backoff:
-	drm_atomic_state_clear(state);
+	drm_atomic_commit_clear(state);
 	drm_modeset_backoff(&ctx);
 	goto retry;
 }
diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c
index e0d5336110404..60cd02b7ea64c 100644
--- a/drivers/gpu/drm/drm_fourcc.c
+++ b/drivers/gpu/drm/drm_fourcc.c
@@ -275,6 +275,7 @@ const struct drm_format_info *__drm_format_info(u32 format)
 		{ .format = DRM_FORMAT_YVU422,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 2, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_YUV444,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 1, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_YVU444,		.depth = 0,  .num_planes = 3, .cpp = { 1, 1, 1 }, .hsub = 1, .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_Y8,		.depth = 8,  .num_planes = 1, .cpp = { 1, 0, 0 }, .hsub = 1, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_NV12,		.depth = 0,  .num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 2, .vsub = 2, .is_yuv = true },
 		{ .format = DRM_FORMAT_NV21,		.depth = 0,  .num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 2, .vsub = 2, .is_yuv = true },
 		{ .format = DRM_FORMAT_NV16,		.depth = 0,  .num_planes = 2, .cpp = { 1, 2, 0 }, .hsub = 2, .vsub = 1, .is_yuv = true },
@@ -287,6 +288,7 @@ const struct drm_format_info *__drm_format_info(u32 format)
 		{ .format = DRM_FORMAT_VYUY,		.depth = 0,  .num_planes = 1, .cpp = { 2, 0, 0 }, .hsub = 2, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_XYUV8888,	.depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_VUY888,          .depth = 0,  .num_planes = 1, .cpp = { 3, 0, 0 }, .hsub = 1, .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_XVUY2101010,     .depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_AYUV,		.depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 1, .vsub = 1, .has_alpha = true, .is_yuv = true },
 		{ .format = DRM_FORMAT_Y210,            .depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 2, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_Y212,            .depth = 0,  .num_planes = 1, .cpp = { 4, 0, 0 }, .hsub = 2, .vsub = 1, .is_yuv = true },
@@ -354,6 +356,9 @@ const struct drm_format_info *__drm_format_info(u32 format)
 		{ .format = DRM_FORMAT_P030,            .depth = 0,  .num_planes = 2,
 		  .char_per_block = { 4, 8, 0 }, .block_w = { 3, 3, 0 }, .block_h = { 1, 1, 0 },
 		  .hsub = 2, .vsub = 2, .is_yuv = true},
+		{ .format = DRM_FORMAT_P230,		.depth = 0,  .num_planes = 2,
+		  .char_per_block = { 4, 8, 0 }, .block_w = { 3, 3, 0 }, .block_h = { 1, 1, 0 },
+		  .hsub = 2, .vsub = 1, .is_yuv = true },
 		{ .format = DRM_FORMAT_S010,            .depth = 0,  .num_planes = 3,
 		  .char_per_block = { 2, 2, 2 }, .block_w = { 1, 1, 1 }, .block_h = { 1, 1, 1 },
 		  .hsub = 2, .vsub = 2, .is_yuv = true},
@@ -381,6 +386,12 @@ const struct drm_format_info *__drm_format_info(u32 format)
 		{ .format = DRM_FORMAT_S416,            .depth = 0,  .num_planes = 3,
 		  .char_per_block = { 2, 2, 2 }, .block_w = { 1, 1, 1 }, .block_h = { 1, 1, 1 },
 		  .hsub = 1, .vsub = 1, .is_yuv = true},
+		{ .format = DRM_FORMAT_XYYY2101010,	.depth = 0,  .num_planes = 1,
+		  .char_per_block = { 4, 0, 0 }, .block_w = { 3, 0, 0 }, .block_h = { 1, 0, 0 },
+		  .hsub = 1, .vsub = 1, .is_yuv = true },
+		{ .format = DRM_FORMAT_T430,		.depth = 0,  .num_planes = 3,
+		  .char_per_block = { 4, 4, 4 }, .block_w = { 3, 3, 3 }, .block_h = { 1, 1, 1 },
+		  .hsub = 1, .vsub = 1, .is_yuv = true },
 	};
 
 	unsigned int i;
@@ -491,12 +502,20 @@ EXPORT_SYMBOL(drm_format_info_block_height);
  */
 unsigned int drm_format_info_bpp(const struct drm_format_info *info, int plane)
 {
+	unsigned int block_size;
+
 	if (!info || plane < 0 || plane >= info->num_planes)
 		return 0;
 
-	return info->char_per_block[plane] * 8 /
-	       (drm_format_info_block_width(info, plane) *
-		drm_format_info_block_height(info, plane));
+	block_size = drm_format_info_block_width(info, plane) *
+		     drm_format_info_block_height(info, plane);
+
+	if (info->char_per_block[plane] * 8 % block_size) {
+		pr_warn("unable to return an integer bpp\n");
+		return 0;
+	}
+
+	return info->char_per_block[plane] * 8 / block_size;
 }
 EXPORT_SYMBOL(drm_format_info_bpp);
 
diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c
index 147ac19f851b2..d32aceb6ca9b8 100644
--- a/drivers/gpu/drm/drm_framebuffer.c
+++ b/drivers/gpu/drm/drm_framebuffer.c
@@ -1000,7 +1000,7 @@ static int atomic_remove_fb(struct drm_framebuffer *fb)
 {
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_device *dev = fb->dev;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_plane *plane;
 	struct drm_connector *conn __maybe_unused;
 	struct drm_connector_state *conn_state;
@@ -1011,7 +1011,7 @@ static int atomic_remove_fb(struct drm_framebuffer *fb)
 retry_disable:
 	drm_modeset_acquire_init(&ctx, 0);
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_commit_alloc(dev);
 	if (!state) {
 		ret = -ENOMEM;
 		goto out;
@@ -1081,12 +1081,12 @@ static int atomic_remove_fb(struct drm_framebuffer *fb)
 
 unlock:
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 		goto retry;
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 out:
 	drm_modeset_drop_locks(&ctx);
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 8afab57fc0554..4c781c4316425 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -1561,12 +1561,10 @@ EXPORT_SYMBOL(drm_gem_unlock_reservations);
  * drm_gem_lru_init - initialize a LRU
  *
  * @lru: The LRU to initialize
- * @lock: The lock protecting the LRU
  */
 void
-drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock)
+drm_gem_lru_init(struct drm_gem_lru *lru)
 {
-	lru->lock = lock;
 	lru->count = 0;
 	INIT_LIST_HEAD(&lru->list);
 }
@@ -1591,14 +1589,10 @@ drm_gem_lru_remove_locked(struct drm_gem_object *obj)
 void
 drm_gem_lru_remove(struct drm_gem_object *obj)
 {
-	struct drm_gem_lru *lru = obj->lru;
-
-	if (!lru)
-		return;
-
-	mutex_lock(lru->lock);
-	drm_gem_lru_remove_locked(obj);
-	mutex_unlock(lru->lock);
+	mutex_lock(&obj->dev->gem_lru_mutex);
+	if (obj->lru)
+		drm_gem_lru_remove_locked(obj);
+	mutex_unlock(&obj->dev->gem_lru_mutex);
 }
 EXPORT_SYMBOL(drm_gem_lru_remove);
 
@@ -1613,7 +1607,7 @@ EXPORT_SYMBOL(drm_gem_lru_remove);
 void
 drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj)
 {
-	lockdep_assert_held_once(lru->lock);
+	lockdep_assert_held_once(&obj->dev->gem_lru_mutex);
 
 	if (obj->lru)
 		drm_gem_lru_remove_locked(obj);
@@ -1637,9 +1631,9 @@ EXPORT_SYMBOL(drm_gem_lru_move_tail_locked);
 void
 drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj)
 {
-	mutex_lock(lru->lock);
+	mutex_lock(&obj->dev->gem_lru_mutex);
 	drm_gem_lru_move_tail_locked(lru, obj);
-	mutex_unlock(lru->lock);
+	mutex_unlock(&obj->dev->gem_lru_mutex);
 }
 EXPORT_SYMBOL(drm_gem_lru_move_tail);
 
@@ -1653,6 +1647,7 @@ EXPORT_SYMBOL(drm_gem_lru_move_tail);
  * of the shrink callback to check for this (ie. dma_resv_test_signaled())
  * or if necessary block until the buffer becomes idle.
  *
+ * @dev: DRM device the LRU belongs to
  * @lru: The LRU to scan
  * @nr_to_scan: The number of pages to try to reclaim
  * @remaining: The number of pages left to reclaim, should be initialized by caller
@@ -1660,7 +1655,8 @@ EXPORT_SYMBOL(drm_gem_lru_move_tail);
  * @ticket: Optional ww_acquire_ctx context to use for locking
  */
 unsigned long
-drm_gem_lru_scan(struct drm_gem_lru *lru,
+drm_gem_lru_scan(struct drm_device *dev,
+		 struct drm_gem_lru *lru,
 		 unsigned int nr_to_scan,
 		 unsigned long *remaining,
 		 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),
@@ -1670,9 +1666,9 @@ drm_gem_lru_scan(struct drm_gem_lru *lru,
 	struct drm_gem_object *obj;
 	unsigned freed = 0;
 
-	drm_gem_lru_init(&still_in_lru, lru->lock);
+	drm_gem_lru_init(&still_in_lru);
 
-	mutex_lock(lru->lock);
+	mutex_lock(&dev->gem_lru_mutex);
 
 	while (freed < nr_to_scan) {
 		obj = list_first_entry_or_null(&lru->list, typeof(*obj), lru_node);
@@ -1695,7 +1691,7 @@ drm_gem_lru_scan(struct drm_gem_lru *lru,
 		 * rest of the loop body, to reduce contention with other
 		 * code paths that need the LRU lock
 		 */
-		mutex_unlock(lru->lock);
+		mutex_unlock(&dev->gem_lru_mutex);
 
 		if (ticket)
 			ww_acquire_init(ticket, &reservation_ww_class);
@@ -1720,6 +1716,16 @@ drm_gem_lru_scan(struct drm_gem_lru *lru,
 			 */
 			WARN_ON(obj->lru == &still_in_lru);
 			WARN_ON(obj->lru == lru);
+		} else if (obj->lru == &still_in_lru) {
+			/*
+			 * If the object wasn't moved and wasn't shrunk either,
+			 * it's still remaining as reclaimable. Note that
+			 * obj->lru is supposed to be checked with the LRU lock
+			 * held for an accurate result, but we don't care about
+			 * accuracy here. Worst thing that could happen is an
+			 * extra scan.
+			 */
+			*remaining += obj->size >> PAGE_SHIFT;
 		}
 
 		dma_resv_unlock(obj->resv);
@@ -1729,7 +1735,7 @@ drm_gem_lru_scan(struct drm_gem_lru *lru,
 
 tail:
 		drm_gem_object_put(obj);
-		mutex_lock(lru->lock);
+		mutex_lock(&dev->gem_lru_mutex);
 	}
 
 	/*
@@ -1741,7 +1747,7 @@ drm_gem_lru_scan(struct drm_gem_lru *lru,
 	list_splice_tail(&still_in_lru.list, &lru->list);
 	lru->count += still_in_lru.count;
 
-	mutex_unlock(lru->lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 
 	return freed;
 }
diff --git a/drivers/gpu/drm/drm_gpusvm.c b/drivers/gpu/drm/drm_gpusvm.c
index 4b928fda5b127..958cb605aedd5 100644
--- a/drivers/gpu/drm/drm_gpusvm.c
+++ b/drivers/gpu/drm/drm_gpusvm.c
@@ -1065,6 +1065,11 @@ drm_gpusvm_range_find_or_insert(struct drm_gpusvm *gpusvm,
 		goto err_notifier_remove;
 	}
 
+	if (vas->vm_flags & (VM_IO | VM_PFNMAP)) {
+		err = -EIO;
+		goto err_notifier_remove;
+	}
+
 	range = drm_gpusvm_range_find(notifier, fault_addr, fault_addr + 1);
 	if (range)
 		goto out_mmunlock;
@@ -1139,11 +1144,17 @@ static void __drm_gpusvm_unmap_pages(struct drm_gpusvm *gpusvm,
 		struct drm_gpusvm_pages_flags flags = {
 			.__flags = svm_pages->flags.__flags,
 		};
+		bool use_iova = dma_use_iova(&svm_pages->state);
+
+		if (use_iova)
+			dma_iova_destroy(dev, &svm_pages->state,
+					 svm_pages->state_offset,
+					 svm_pages->dma_addr[0].dir, 0);
 
 		for (i = 0, j = 0; i < npages; j++) {
 			struct drm_pagemap_addr *addr = &svm_pages->dma_addr[j];
 
-			if (addr->proto == DRM_INTERCONNECT_SYSTEM)
+			if (!use_iova && addr->proto == DRM_INTERCONNECT_SYSTEM)
 				dma_unmap_page(dev,
 					       addr->addr,
 					       PAGE_SIZE << addr->order,
@@ -1408,6 +1419,7 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 	struct drm_gpusvm_pages_flags flags;
 	enum dma_data_direction dma_dir = ctx->read_only ? DMA_TO_DEVICE :
 							   DMA_BIDIRECTIONAL;
+	struct dma_iova_state *state = &svm_pages->state;
 
 retry:
 	if (time_after(jiffies, timeout))
@@ -1446,6 +1458,9 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 	if (err)
 		goto err_free;
 
+	*state = (struct dma_iova_state){};
+	svm_pages->state_offset = 0;
+
 map_pages:
 	/*
 	 * Perform all dma mappings under the notifier lock to not
@@ -1539,13 +1554,30 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 				goto err_unmap;
 			}
 
-			addr = dma_map_page(gpusvm->drm->dev,
-					    page, 0,
-					    PAGE_SIZE << order,
-					    dma_dir);
-			if (dma_mapping_error(gpusvm->drm->dev, addr)) {
-				err = -EFAULT;
-				goto err_unmap;
+			if (!i)
+				dma_iova_try_alloc(gpusvm->drm->dev, state,
+						   0, npages * PAGE_SIZE);
+
+			if (dma_use_iova(state)) {
+				err = dma_iova_link(gpusvm->drm->dev, state,
+						    hmm_pfn_to_phys(pfns[i]),
+						    svm_pages->state_offset,
+						    PAGE_SIZE << order,
+						    dma_dir, 0);
+				if (err)
+					goto err_unmap;
+
+				addr = state->addr + svm_pages->state_offset;
+				svm_pages->state_offset += PAGE_SIZE << order;
+			} else {
+				addr = dma_map_page(gpusvm->drm->dev,
+						    page, 0,
+						    PAGE_SIZE << order,
+						    dma_dir);
+				if (dma_mapping_error(gpusvm->drm->dev, addr)) {
+					err = -EFAULT;
+					goto err_unmap;
+				}
 			}
 
 			svm_pages->dma_addr[j] = drm_pagemap_addr_encode
@@ -1557,6 +1589,13 @@ int drm_gpusvm_get_pages(struct drm_gpusvm *gpusvm,
 		flags.has_dma_mapping = true;
 	}
 
+	if (dma_use_iova(state)) {
+		err = dma_iova_sync(gpusvm->drm->dev, state, 0,
+				    svm_pages->state_offset);
+		if (err)
+			goto err_unmap;
+	}
+
 	if (pagemap) {
 		flags.has_devmem_pages = true;
 		drm_pagemap_get(dpagemap);
diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c
index 44acfe4120d24..83020b6cf9ba1 100644
--- a/drivers/gpu/drm/drm_gpuvm.c
+++ b/drivers/gpu/drm/drm_gpuvm.c
@@ -25,6 +25,7 @@
  *
  */
 
+#include <drm/drm_drv.h>
 #include <drm/drm_gpuvm.h>
 #include <drm/drm_print.h>
 
@@ -1117,6 +1118,7 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name,
 	gpuvm->drm = drm;
 	gpuvm->r_obj = r_obj;
 
+	drm_dev_get(drm);
 	drm_gem_object_get(r_obj);
 
 	drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range);
@@ -1160,13 +1162,15 @@ static void
 drm_gpuvm_free(struct kref *kref)
 {
 	struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref);
+	struct drm_device *drm = gpuvm->drm;
 
 	drm_gpuvm_fini(gpuvm);
 
-	if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free))
+	if (drm_WARN_ON(drm, !gpuvm->ops->vm_free))
 		return;
 
 	gpuvm->ops->vm_free(gpuvm);
+	drm_dev_put(drm);
 }
 
 /**
@@ -1322,6 +1326,9 @@ drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec,
 	drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) {
 		struct drm_gem_object *obj = va->gem.obj;
 
+		if (unlikely(!obj))
+			continue;
+
 		ret = exec_prepare_obj(exec, obj, num_fences);
 		if (ret)
 			return ret;
diff --git a/drivers/gpu/drm/drm_managed.c b/drivers/gpu/drm/drm_managed.c
index 247f468731de0..a9da94319b052 100644
--- a/drivers/gpu/drm/drm_managed.c
+++ b/drivers/gpu/drm/drm_managed.c
@@ -232,8 +232,8 @@ void *drmm_kmalloc(struct drm_device *dev, size_t size, gfp_t gfp)
 
 	dr = alloc_dr(NULL, size, gfp, dev_to_node(dev->dev));
 	if (!dr) {
-		drm_dbg_drmres(dev, "failed to allocate %zu bytes, %u flags\n",
-			       size, gfp);
+		drm_dbg_drmres(dev, "failed to allocate %zu bytes, %pGg\n",
+			       size, &gfp);
 		return NULL;
 	}
 	dr->node.name = kstrdup_const("kmalloc", gfp);
diff --git a/drivers/gpu/drm/drm_mipi_dbi.c b/drivers/gpu/drm/drm_mipi_dbi.c
index 9ab6113d74a21..25cf04d029f77 100644
--- a/drivers/gpu/drm/drm_mipi_dbi.c
+++ b/drivers/gpu/drm/drm_mipi_dbi.c
@@ -344,7 +344,7 @@ EXPORT_SYMBOL(drm_mipi_dbi_crtc_helper_mode_valid);
  * 0 on success, or a negative errno code otherwise.
  */
 int drm_mipi_dbi_plane_helper_atomic_check(struct drm_plane *plane,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_crtc_state *new_crtc_state = NULL;
@@ -375,7 +375,7 @@ EXPORT_SYMBOL(drm_mipi_dbi_plane_helper_atomic_check);
  * this as their struct &drm_plane_helper_funcs.atomic_update callback.
  */
 void drm_mipi_dbi_plane_helper_atomic_update(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = plane->state;
 	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
@@ -434,7 +434,7 @@ static void mipi_dbi_blank(struct mipi_dbi_dev *dbidev)
  * Returns:
  * 0 on success, or a negative errno code otherwise.
  */
-int drm_mipi_dbi_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
+int drm_mipi_dbi_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	int ret;
@@ -461,7 +461,7 @@ EXPORT_SYMBOL(drm_mipi_dbi_crtc_helper_atomic_check);
  * struct &drm_crtc_helper_funcs.atomic_disable callback.
  */
 void drm_mipi_dbi_crtc_helper_atomic_disable(struct drm_crtc *crtc,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct mipi_dbi_dev *dbidev = drm_to_mipi_dbi_dev(crtc->dev);
 
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index 0390e14d31572..3ac1dd5ad640a 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -587,6 +587,9 @@ EXPORT_SYMBOL(mipi_dsi_create_packet);
  * mipi_dsi_shutdown_peripheral() - sends a Shutdown Peripheral command
  * @dsi: DSI peripheral device
  *
+ * This function is deprecated. Use mipi_dsi_shutdown_peripheral_multi()
+ * instead.
+ *
  * Return: 0 on success or a negative error code on failure.
  */
 int mipi_dsi_shutdown_peripheral(struct mipi_dsi_device *dsi)
@@ -1980,6 +1983,31 @@ void mipi_dsi_dcs_set_tear_scanline_multi(struct mipi_dsi_multi_context *ctx,
 }
 EXPORT_SYMBOL(mipi_dsi_dcs_set_tear_scanline_multi);
 
+/**
+ * mipi_dsi_shutdown_peripheral_multi() - sends a Shutdown Peripheral command
+ * @ctx: Context for multiple DSI transactions
+ *
+ * Like mipi_dsi_shutdown_peripheral() but deals with errors in a way that
+ * makes it convenient to make several calls in a row.
+ */
+void mipi_dsi_shutdown_peripheral_multi(struct mipi_dsi_multi_context *ctx)
+{
+	struct mipi_dsi_device *dsi = ctx->dsi;
+	struct device *dev = &dsi->dev;
+	int ret;
+
+	if (ctx->accum_err)
+		return;
+
+	ret = mipi_dsi_shutdown_peripheral(dsi);
+	if (ret < 0) {
+		ctx->accum_err = ret;
+		dev_err(dev, "Failed to shutdown peripheral: %d\n",
+			ctx->accum_err);
+	}
+}
+EXPORT_SYMBOL(mipi_dsi_shutdown_peripheral_multi);
+
 static int mipi_dsi_drv_probe(struct device *dev)
 {
 	struct mipi_dsi_driver *drv = to_mipi_dsi_driver(dev->driver);
diff --git a/drivers/gpu/drm/drm_mode_object.c b/drivers/gpu/drm/drm_mode_object.c
index 2d943a610b88b..21fc9deda4371 100644
--- a/drivers/gpu/drm/drm_mode_object.c
+++ b/drivers/gpu/drm/drm_mode_object.c
@@ -561,11 +561,11 @@ static int set_property_atomic(struct drm_mode_object *obj,
 			       uint64_t prop_value)
 {
 	struct drm_device *dev = prop->dev;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_modeset_acquire_ctx ctx;
 	int ret;
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_commit_alloc(dev);
 	if (!state)
 		return -ENOMEM;
 
@@ -590,12 +590,12 @@ static int set_property_atomic(struct drm_mode_object *obj,
 	}
 out:
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 		goto retry;
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
diff --git a/drivers/gpu/drm/drm_modeset_helper.c b/drivers/gpu/drm/drm_modeset_helper.c
index d7721df744e72..e54584da4c3d1 100644
--- a/drivers/gpu/drm/drm_modeset_helper.c
+++ b/drivers/gpu/drm/drm_modeset_helper.c
@@ -190,7 +190,7 @@ EXPORT_SYMBOL(drm_crtc_init);
  */
 int drm_mode_config_helper_suspend(struct drm_device *dev)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 
 	if (!dev)
 		return 0;
diff --git a/drivers/gpu/drm/drm_of.c b/drivers/gpu/drm/drm_of.c
index 4f65ce729a473..d03ada82eac96 100644
--- a/drivers/gpu/drm/drm_of.c
+++ b/drivers/gpu/drm/drm_of.c
@@ -225,15 +225,15 @@ EXPORT_SYMBOL_GPL(drm_of_encoder_active_endpoint);
  * @np: device tree node containing encoder output ports
  * @port: port in the device tree node
  * @endpoint: endpoint in the device tree node
- * @panel: pointer to hold returned drm_panel
+ * @panel: pointer to hold returned drm_panel, must not be NULL
  * @bridge: pointer to hold returned drm_bridge
  *
  * Given a DT node's port and endpoint number, find the connected node and
- * return either the associated struct drm_panel or drm_bridge device. Either
- * @panel or @bridge must not be NULL.
+ * return either the associated struct drm_panel or drm_bridge device.
  *
  * This function is deprecated and should not be used in new drivers. Use
- * devm_drm_of_get_bridge() instead.
+ * of_drm_get_bridge_by_endpoint() instead when not looking for a panel, or
+ * devm_drm_of_get_bridge() otherwise.
  *
  * Returns zero if successful, or one of the standard error codes if it fails.
  */
@@ -245,10 +245,10 @@ int drm_of_find_panel_or_bridge(const struct device_node *np,
 	int ret = -EPROBE_DEFER;
 	struct device_node *remote;
 
-	if (!panel && !bridge)
+	if (WARN_ON(!panel))
 		return -EINVAL;
-	if (panel)
-		*panel = NULL;
+
+	*panel = NULL;
 
 	/*
 	 * of_graph_get_remote_node() produces a noisy error message if port
@@ -263,13 +263,11 @@ int drm_of_find_panel_or_bridge(const struct device_node *np,
 	if (!remote)
 		return -ENODEV;
 
-	if (panel) {
-		*panel = of_drm_find_panel(remote);
-		if (!IS_ERR(*panel))
-			ret = 0;
-		else
-			*panel = NULL;
-	}
+	*panel = of_drm_find_panel(remote);
+	if (!IS_ERR(*panel))
+		ret = 0;
+	else
+		*panel = NULL;
 
 	if (bridge) {
 		if (ret) {
@@ -558,6 +556,40 @@ int drm_of_get_data_lanes_count_ep(const struct device_node *port,
 }
 EXPORT_SYMBOL_GPL(drm_of_get_data_lanes_count_ep);
 
+/**
+ * drm_of_get_data_lanes_count_remote - Get DSI/(e)DP data lane count by endpoint
+ * @port: DT port node of the DSI/(e)DP source or sink
+ * @port_reg: identifier (value of reg property) of the parent port node
+ * @reg: identifier (value of reg property) of the endpoint node
+ * @min: minimum supported number of data lanes
+ * @max: maximum supported number of data lanes
+ *
+ * Count DT "data-lanes" property elements in the remote endpoint and check for
+ * validity.  This variant uses endpoint specifier.
+ *
+ * Return:
+ * * min..max - positive integer count of "data-lanes" elements
+ * * -EINVAL - the "data-lanes" property is unsupported
+ * * -ENODEV - the "data-lanes" property is missing
+ */
+int drm_of_get_data_lanes_count_remote(const struct device_node *port,
+				       int port_reg, int reg,
+				       const unsigned int min,
+				       const unsigned int max)
+{
+	struct device_node *endpoint, *remote;
+	int ret;
+
+	endpoint = of_graph_get_endpoint_by_regs(port, port_reg, reg);
+	remote = of_graph_get_remote_endpoint(endpoint);
+	of_node_put(endpoint);
+	ret = drm_of_get_data_lanes_count(remote, min, max);
+	of_node_put(remote);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(drm_of_get_data_lanes_count_remote);
+
 #if IS_ENABLED(CONFIG_DRM_MIPI_DSI)
 
 /**
diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c
index 5002049e01981..15c78eca180b7 100644
--- a/drivers/gpu/drm/drm_pagemap.c
+++ b/drivers/gpu/drm/drm_pagemap.c
@@ -216,7 +216,8 @@ static void drm_pagemap_get_devmem_page(struct page *page,
 }
 
 /**
- * drm_pagemap_migrate_map_pages() - Map migration pages for GPU SVM migration
+ * drm_pagemap_migrate_map_device_private_pages() - Map device private migration
+ * pages for GPU SVM migration
  * @dev: The device performing the migration.
  * @local_dpagemap: The drm_pagemap local to the migrating device.
  * @pagemap_addr: Array to store DMA information corresponding to mapped pages.
@@ -232,58 +233,50 @@ static void drm_pagemap_get_devmem_page(struct page *page,
  *
  * Returns: 0 on success, -EFAULT if an error occurs during mapping.
  */
-static int drm_pagemap_migrate_map_pages(struct device *dev,
-					 struct drm_pagemap *local_dpagemap,
-					 struct drm_pagemap_addr *pagemap_addr,
-					 unsigned long *migrate_pfn,
-					 unsigned long npages,
-					 enum dma_data_direction dir,
-					 const struct drm_pagemap_migrate_details *mdetails)
+static int
+drm_pagemap_migrate_map_device_private_pages(struct device *dev,
+					     struct drm_pagemap *local_dpagemap,
+					     struct drm_pagemap_addr *pagemap_addr,
+					     unsigned long *migrate_pfn,
+					     unsigned long npages,
+					     enum dma_data_direction dir,
+					     const struct drm_pagemap_migrate_details *mdetails)
 {
 	unsigned long num_peer_pages = 0, num_local_pages = 0, i;
 
 	for (i = 0; i < npages;) {
 		struct page *page = migrate_pfn_to_page(migrate_pfn[i]);
-		dma_addr_t dma_addr;
+		struct drm_pagemap_zdd *zdd;
+		struct drm_pagemap *dpagemap;
+		struct drm_pagemap_addr addr;
 		struct folio *folio;
 		unsigned int order = 0;
 
 		if (!page)
 			goto next;
 
+		WARN_ON_ONCE(!is_device_private_page(page));
 		folio = page_folio(page);
 		order = folio_order(folio);
 
-		if (is_device_private_page(page)) {
-			struct drm_pagemap_zdd *zdd = drm_pagemap_page_zone_device_data(page);
-			struct drm_pagemap *dpagemap = zdd->dpagemap;
-			struct drm_pagemap_addr addr;
-
-			if (dpagemap == local_dpagemap) {
-				if (!mdetails->can_migrate_same_pagemap)
-					goto next;
+		zdd = drm_pagemap_page_zone_device_data(page);
+		dpagemap = zdd->dpagemap;
 
-				num_local_pages += NR_PAGES(order);
-			} else {
-				num_peer_pages += NR_PAGES(order);
-			}
-
-			addr = dpagemap->ops->device_map(dpagemap, dev, page, order, dir);
-			if (dma_mapping_error(dev, addr.addr))
-				return -EFAULT;
+		if (dpagemap == local_dpagemap) {
+			if (!mdetails->can_migrate_same_pagemap)
+				goto next;
 
-			pagemap_addr[i] = addr;
+			num_local_pages += NR_PAGES(order);
 		} else {
-			dma_addr = dma_map_page(dev, page, 0, page_size(page), dir);
-			if (dma_mapping_error(dev, dma_addr))
-				return -EFAULT;
-
-			pagemap_addr[i] =
-				drm_pagemap_addr_encode(dma_addr,
-							DRM_INTERCONNECT_SYSTEM,
-							order, dir);
+			num_peer_pages += NR_PAGES(order);
 		}
 
+		addr = dpagemap->ops->device_map(dpagemap, dev, page, order, dir);
+		if (dma_mapping_error(dev, addr.addr))
+			return -EFAULT;
+
+		pagemap_addr[i] = addr;
+
 next:
 		i += NR_PAGES(order);
 	}
@@ -298,6 +291,98 @@ static int drm_pagemap_migrate_map_pages(struct device *dev,
 	return 0;
 }
 
+/**
+ * struct drm_pagemap_iova_state - DRM pagemap IOVA state
+ * @dma_state: DMA IOVA state.
+ * @offset: Current offset in IOVA.
+ *
+ * This structure acts as an iterator for packing all IOVA addresses within a
+ * contiguous range.
+ */
+struct drm_pagemap_iova_state {
+	struct dma_iova_state dma_state;
+	unsigned long offset;
+};
+
+/**
+ * drm_pagemap_migrate_map_system_pages() - Map system or device coherent
+ * migration pages for GPU SVM migration
+ * @dev: The device performing the migration.
+ * @pagemap_addr: Array to store DMA information corresponding to mapped pages.
+ * @migrate_pfn: Array of page frame numbers of system pages or peer pages to map.
+ * @npages: Number of system or device coherent pages to map.
+ * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL)
+ * @state: DMA IOVA state for mapping.
+ *
+ * This function maps pages of memory for migration usage in GPU SVM. It
+ * iterates over each page frame number provided in @migrate_pfn, maps the
+ * corresponding page, and stores the DMA address in the provided @dma_addr
+ * array.
+ *
+ * Returns: 0 on success, negative error code on failure.
+ */
+static int
+drm_pagemap_migrate_map_system_pages(struct device *dev,
+				     struct drm_pagemap_addr *pagemap_addr,
+				     unsigned long *migrate_pfn,
+				     unsigned long npages,
+				     enum dma_data_direction dir,
+				     struct drm_pagemap_iova_state *state)
+{
+	unsigned long i;
+	bool try_alloc = false;
+
+	for (i = 0; i < npages;) {
+		struct page *page = migrate_pfn_to_page(migrate_pfn[i]);
+		dma_addr_t dma_addr;
+		struct folio *folio;
+		unsigned int order = 0;
+
+		if (!page)
+			goto next;
+
+		WARN_ON_ONCE(is_device_private_page(page));
+		folio = page_folio(page);
+		order = folio_order(folio);
+
+		if (!try_alloc) {
+			dma_iova_try_alloc(dev, &state->dma_state,
+					   0, npages * PAGE_SIZE);
+			try_alloc = true;
+		}
+
+		if (dma_use_iova(&state->dma_state)) {
+			int err = dma_iova_link(dev, &state->dma_state,
+						page_to_phys(page),
+						state->offset, page_size(page),
+						dir, 0);
+			if (err)
+				return err;
+
+			dma_addr = state->dma_state.addr + state->offset;
+			state->offset += page_size(page);
+		} else {
+			dma_addr = dma_map_page(dev, page, 0, page_size(page),
+						dir);
+			if (dma_mapping_error(dev, dma_addr))
+				return -EFAULT;
+		}
+
+		pagemap_addr[i] =
+			drm_pagemap_addr_encode(dma_addr,
+						DRM_INTERCONNECT_SYSTEM,
+						order, dir);
+
+next:
+		i += NR_PAGES(order);
+	}
+
+	if (dma_use_iova(&state->dma_state))
+		return dma_iova_sync(dev, &state->dma_state, 0, state->offset);
+
+	return 0;
+}
+
 /**
  * drm_pagemap_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration
  * @dev: The device for which the pages were mapped
@@ -306,19 +391,27 @@ static int drm_pagemap_migrate_map_pages(struct device *dev,
  * @pagemap_addr: Array of DMA information corresponding to mapped pages
  * @npages: Number of pages to unmap
  * @dir: Direction of data transfer (e.g., DMA_BIDIRECTIONAL)
+ * @state: DMA IOVA state for mapping.
  *
  * This function unmaps previously mapped pages of memory for GPU Shared Virtual
- * Memory (SVM). It iterates over each DMA address provided in @dma_addr, checks
- * if it's valid and not already unmapped, and unmaps the corresponding page.
+ * Memory (SVM). It iterates over each DMA address provided in @pagemap_addr,
+ * checks if it's valid and not already unmapped, and unmaps the corresponding
+ * page.
  */
 static void drm_pagemap_migrate_unmap_pages(struct device *dev,
 					    struct drm_pagemap_addr *pagemap_addr,
 					    unsigned long *migrate_pfn,
 					    unsigned long npages,
-					    enum dma_data_direction dir)
+					    enum dma_data_direction dir,
+					    struct drm_pagemap_iova_state *state)
 {
 	unsigned long i;
 
+	if (state && dma_use_iova(&state->dma_state)) {
+		dma_iova_destroy(dev, &state->dma_state, state->offset, dir, 0);
+		return;
+	}
+
 	for (i = 0; i < npages;) {
 		struct page *page = migrate_pfn_to_page(migrate_pfn[i]);
 
@@ -358,9 +451,13 @@ drm_pagemap_migrate_remote_to_local(struct drm_pagemap_devmem *devmem,
 				    const struct drm_pagemap_migrate_details *mdetails)
 
 {
-	int err = drm_pagemap_migrate_map_pages(remote_device, remote_dpagemap,
-						pagemap_addr, local_pfns,
-						npages, DMA_FROM_DEVICE, mdetails);
+	int err = drm_pagemap_migrate_map_device_private_pages(remote_device,
+							       remote_dpagemap,
+							       pagemap_addr,
+							       local_pfns,
+							       npages,
+							       DMA_FROM_DEVICE,
+							       mdetails);
 
 	if (err)
 		goto out;
@@ -369,7 +466,7 @@ drm_pagemap_migrate_remote_to_local(struct drm_pagemap_devmem *devmem,
 			       devmem->pre_migrate_fence);
 out:
 	drm_pagemap_migrate_unmap_pages(remote_device, pagemap_addr, local_pfns,
-					npages, DMA_FROM_DEVICE);
+					npages, DMA_FROM_DEVICE, NULL);
 	return err;
 }
 
@@ -380,11 +477,12 @@ drm_pagemap_migrate_sys_to_dev(struct drm_pagemap_devmem *devmem,
 			       struct drm_pagemap_addr pagemap_addr[],
 			       unsigned long npages,
 			       const struct drm_pagemap_devmem_ops *ops,
-			       const struct drm_pagemap_migrate_details *mdetails)
+			       struct drm_pagemap_iova_state *state)
 {
-	int err = drm_pagemap_migrate_map_pages(devmem->dev, devmem->dpagemap,
-						pagemap_addr, sys_pfns, npages,
-						DMA_TO_DEVICE, mdetails);
+	int err = drm_pagemap_migrate_map_system_pages(devmem->dev,
+						       pagemap_addr, sys_pfns,
+						       npages, DMA_TO_DEVICE,
+						       state);
 
 	if (err)
 		goto out;
@@ -393,7 +491,7 @@ drm_pagemap_migrate_sys_to_dev(struct drm_pagemap_devmem *devmem,
 				  devmem->pre_migrate_fence);
 out:
 	drm_pagemap_migrate_unmap_pages(devmem->dev, pagemap_addr, sys_pfns, npages,
-					DMA_TO_DEVICE);
+					DMA_TO_DEVICE, state);
 	return err;
 }
 
@@ -421,6 +519,7 @@ static int drm_pagemap_migrate_range(struct drm_pagemap_devmem *devmem,
 				     const struct migrate_range_loc *cur,
 				     const struct drm_pagemap_migrate_details *mdetails)
 {
+	struct drm_pagemap_iova_state state = {};
 	int ret = 0;
 
 	if (cur->start == 0)
@@ -448,7 +547,7 @@ static int drm_pagemap_migrate_range(struct drm_pagemap_devmem *devmem,
 						     &pages[last->start],
 						     &pagemap_addr[last->start],
 						     cur->start - last->start,
-						     last->ops, mdetails);
+						     last->ops, &state);
 
 out:
 	*last = *cur;
@@ -651,12 +750,10 @@ int drm_pagemap_migrate_to_devmem(struct drm_pagemap_devmem *devmem_allocation,
 				own_pages++;
 				goto next;
 			}
-			if (mdetails->source_peer_migrates) {
-				cur.dpagemap = src_zdd->dpagemap;
-				cur.ops = src_zdd->devmem_allocation->ops;
-				cur.device = cur.dpagemap->drm->dev;
-				pages[i] = src_page;
-			}
+			cur.dpagemap = src_zdd->dpagemap;
+			cur.ops = src_zdd->devmem_allocation->ops;
+			cur.device = cur.dpagemap->drm->dev;
+			pages[i] = src_page;
 		}
 		if (!pages[i]) {
 			cur.dpagemap = NULL;
@@ -1012,7 +1109,7 @@ EXPORT_SYMBOL(drm_pagemap_put);
 int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation)
 {
 	const struct drm_pagemap_devmem_ops *ops = devmem_allocation->ops;
-	struct drm_pagemap_migrate_details mdetails = {};
+	struct drm_pagemap_iova_state state = {};
 	unsigned long npages, mpages = 0;
 	struct page **pages;
 	unsigned long *src, *dst;
@@ -1051,10 +1148,10 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation)
 	if (err || !mpages)
 		goto err_finalize;
 
-	err = drm_pagemap_migrate_map_pages(devmem_allocation->dev,
-					    devmem_allocation->dpagemap, pagemap_addr,
-					    dst, npages, DMA_FROM_DEVICE,
-					    &mdetails);
+	err = drm_pagemap_migrate_map_system_pages(devmem_allocation->dev,
+						   pagemap_addr,
+						   dst, npages,
+						   DMA_FROM_DEVICE, &state);
 	if (err)
 		goto err_finalize;
 
@@ -1078,7 +1175,7 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation)
 	migrate_device_pages(src, dst, npages);
 	migrate_device_finalize(src, dst, npages);
 	drm_pagemap_migrate_unmap_pages(devmem_allocation->dev, pagemap_addr, dst, npages,
-					DMA_FROM_DEVICE);
+					DMA_FROM_DEVICE, &state);
 
 err_free:
 	kvfree(buf);
@@ -1090,6 +1187,7 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation)
 
 	if (retry_count--) {
 		cond_resched();
+		state = (struct drm_pagemap_iova_state){};
 		goto retry;
 	}
 
@@ -1123,7 +1221,7 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
 				  MIGRATE_VMA_SELECT_COMPOUND,
 		.fault_page	= page,
 	};
-	struct drm_pagemap_migrate_details mdetails = {};
+	struct drm_pagemap_iova_state state = {};
 	struct drm_pagemap_zdd *zdd;
 	const struct drm_pagemap_devmem_ops *ops;
 	struct device *dev = NULL;
@@ -1181,8 +1279,9 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
 	if (err)
 		goto err_finalize;
 
-	err = drm_pagemap_migrate_map_pages(dev, zdd->dpagemap, pagemap_addr, migrate.dst, npages,
-					    DMA_FROM_DEVICE, &mdetails);
+	err = drm_pagemap_migrate_map_system_pages(dev, pagemap_addr,
+						   migrate.dst, npages,
+						   DMA_FROM_DEVICE, &state);
 	if (err)
 		goto err_finalize;
 
@@ -1207,7 +1306,8 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
 	migrate_vma_finalize(&migrate);
 	if (dev)
 		drm_pagemap_migrate_unmap_pages(dev, pagemap_addr, migrate.dst,
-						npages, DMA_FROM_DEVICE);
+						npages, DMA_FROM_DEVICE,
+						&state);
 err_free:
 	kvfree(buf);
 err_out:
diff --git a/drivers/gpu/drm/drm_panel.c b/drivers/gpu/drm/drm_panel.c
index d1e6598ea3bc0..04f4a31ed27af 100644
--- a/drivers/gpu/drm/drm_panel.c
+++ b/drivers/gpu/drm/drm_panel.c
@@ -56,8 +56,9 @@ static LIST_HEAD(panel_list);
  * Initialize the panel structure for subsequent registration with
  * drm_panel_add().
  */
-void drm_panel_init(struct drm_panel *panel, struct device *dev,
-		    const struct drm_panel_funcs *funcs, int connector_type)
+static void drm_panel_init(struct drm_panel *panel, struct device *dev,
+			   const struct drm_panel_funcs *funcs,
+			   int connector_type)
 {
 	if (connector_type == DRM_MODE_CONNECTOR_Unknown)
 		DRM_WARN("%s: %s: a valid connector type is required!\n", __func__, dev_name(dev));
@@ -69,7 +70,6 @@ void drm_panel_init(struct drm_panel *panel, struct device *dev,
 	panel->funcs = funcs;
 	panel->connector_type = connector_type;
 }
-EXPORT_SYMBOL(drm_panel_init);
 
 /**
  * drm_panel_add - add a panel to the global registry
@@ -101,6 +101,29 @@ void drm_panel_remove(struct drm_panel *panel)
 }
 EXPORT_SYMBOL(drm_panel_remove);
 
+static void drm_panel_add_release(void *data)
+{
+	drm_panel_remove(data);
+}
+
+/**
+ * devm_drm_panel_add - add a panel to the global registry using devres
+ * @panel: panel to add
+ *
+ * Add a panel to the global registry so that it can be looked
+ * up by display drivers. The panel to be added must have been
+ * allocated by devm_drm_panel_alloc(). Unlike drm_panel_add() with this
+ * function there is no need to call drm_panel_remove(), it will be called
+ * automatically.
+ */
+int devm_drm_panel_add(struct device *dev, struct drm_panel *panel)
+{
+	drm_panel_add(panel);
+
+	return devm_add_action_or_reset(dev, drm_panel_add_release, panel);
+}
+EXPORT_SYMBOL(devm_drm_panel_add);
+
 /**
  * drm_panel_prepare - power on a panel
  * @panel: DRM panel
@@ -708,7 +731,6 @@ EXPORT_SYMBOL(devm_drm_panel_add_follower);
  * A typical implementation for a panel driver supporting device tree
  * will call this function at probe time. Backlight will then be handled
  * transparently without requiring any intervention from the driver.
- * drm_panel_of_backlight() must be called after the call to drm_panel_init().
  *
  * Return: 0 on success or a negative error code on failure.
  */
diff --git a/drivers/gpu/drm/drm_ras.c b/drivers/gpu/drm/drm_ras.c
index b2fa5ab86d870..d6eab29a1394c 100644
--- a/drivers/gpu/drm/drm_ras.c
+++ b/drivers/gpu/drm/drm_ras.c
@@ -26,7 +26,7 @@
  * efficient lookup by ID. Nodes can be registered or unregistered
  * dynamically at runtime.
  *
- * A Generic Netlink family `drm_ras` exposes two main operations to
+ * A Generic Netlink family `drm_ras` exposes the below operations to
  * userspace:
  *
  * 1. LIST_NODES: Dump all currently registered RAS nodes.
@@ -37,6 +37,10 @@
  *    Returns all counters of a node if only Node ID is provided or specific
  *    error counters.
  *
+ * 3. CLEAR_ERROR_COUNTER: Clear error counter of a given node.
+ *    Userspace must provide Node ID, Error ID.
+ *    Clears specific error counter of a node if supported.
+ *
  * Node registration:
  *
  * - drm_ras_node_register(): Registers a new node and assigns
@@ -66,6 +70,8 @@
  *   operation, fetching all counters from a specific node.
  * - drm_ras_nl_get_error_counter_doit(): Implements the GET_ERROR_COUNTER doit
  *   operation, fetching a counter value from a specific node.
+ * - drm_ras_nl_clear_error_counter_doit(): Implements the CLEAR_ERROR_COUNTER doit
+ *   operation, clearing a counter value from a specific node.
  */
 
 static DEFINE_XARRAY_ALLOC(drm_ras_xa);
@@ -314,6 +320,41 @@ int drm_ras_nl_get_error_counter_doit(struct sk_buff *skb,
 	return doit_reply_value(info, node_id, error_id);
 }
 
+/**
+ * drm_ras_nl_clear_error_counter_doit() - Clear an error counter of a node
+ * @skb: Netlink message buffer
+ * @info: Generic Netlink info containing attributes of the request
+ *
+ * Extracts the node ID and error ID from the netlink attributes and
+ * clears the current value.
+ *
+ * Return: 0 on success, or negative errno on failure.
+ */
+int drm_ras_nl_clear_error_counter_doit(struct sk_buff *skb,
+					struct genl_info *info)
+{
+	struct drm_ras_node *node;
+	u32 node_id, error_id;
+
+	if (!info->attrs ||
+	    GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID) ||
+	    GENL_REQ_ATTR_CHECK(info, DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID))
+		return -EINVAL;
+
+	node_id = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID]);
+	error_id = nla_get_u32(info->attrs[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID]);
+
+	node = xa_load(&drm_ras_xa, node_id);
+	if (!node || !node->clear_error_counter)
+		return -ENOENT;
+
+	if (error_id < node->error_counter_range.first ||
+	    error_id > node->error_counter_range.last)
+		return -EINVAL;
+
+	return node->clear_error_counter(node, error_id);
+}
+
 /**
  * drm_ras_node_register() - Register a new RAS node
  * @node: Node structure to register
diff --git a/drivers/gpu/drm/drm_ras_nl.c b/drivers/gpu/drm/drm_ras_nl.c
index 16803d0c4a44c..dea1c1b2494e8 100644
--- a/drivers/gpu/drm/drm_ras_nl.c
+++ b/drivers/gpu/drm/drm_ras_nl.c
@@ -22,6 +22,12 @@ static const struct nla_policy drm_ras_get_error_counter_dump_nl_policy[DRM_RAS_
 	[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID] = { .type = NLA_U32, },
 };
 
+/* DRM_RAS_CMD_CLEAR_ERROR_COUNTER - do */
+static const struct nla_policy drm_ras_clear_error_counter_nl_policy[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID + 1] = {
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID] = { .type = NLA_U32, },
+	[DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID] = { .type = NLA_U32, },
+};
+
 /* Ops table for drm_ras */
 static const struct genl_split_ops drm_ras_nl_ops[] = {
 	{
@@ -43,6 +49,13 @@ static const struct genl_split_ops drm_ras_nl_ops[] = {
 		.maxattr	= DRM_RAS_A_ERROR_COUNTER_ATTRS_NODE_ID,
 		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP,
 	},
+	{
+		.cmd		= DRM_RAS_CMD_CLEAR_ERROR_COUNTER,
+		.doit		= drm_ras_nl_clear_error_counter_doit,
+		.policy		= drm_ras_clear_error_counter_nl_policy,
+		.maxattr	= DRM_RAS_A_ERROR_COUNTER_ATTRS_ERROR_ID,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
 };
 
 struct genl_family drm_ras_nl_family __ro_after_init = {
diff --git a/drivers/gpu/drm/drm_ras_nl.h b/drivers/gpu/drm/drm_ras_nl.h
index 06ccd93427739..a398643572a5a 100644
--- a/drivers/gpu/drm/drm_ras_nl.h
+++ b/drivers/gpu/drm/drm_ras_nl.h
@@ -18,6 +18,8 @@ int drm_ras_nl_get_error_counter_doit(struct sk_buff *skb,
 				      struct genl_info *info);
 int drm_ras_nl_get_error_counter_dumpit(struct sk_buff *skb,
 					struct netlink_callback *cb);
+int drm_ras_nl_clear_error_counter_doit(struct sk_buff *skb,
+					struct genl_info *info);
 
 extern struct genl_family drm_ras_nl_family;
 
diff --git a/drivers/gpu/drm/drm_self_refresh_helper.c b/drivers/gpu/drm/drm_self_refresh_helper.c
index c08e7f590ca55..37ec7fb00f034 100644
--- a/drivers/gpu/drm/drm_self_refresh_helper.c
+++ b/drivers/gpu/drm/drm_self_refresh_helper.c
@@ -73,7 +73,7 @@ static void drm_self_refresh_helper_entry_work(struct work_struct *work)
 	struct drm_crtc *crtc = sr_data->crtc;
 	struct drm_device *dev = crtc->dev;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_connector *conn;
 	struct drm_connector_state *conn_state;
 	struct drm_crtc_state *crtc_state;
@@ -81,7 +81,7 @@ static void drm_self_refresh_helper_entry_work(struct work_struct *work)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_commit_alloc(dev);
 	if (!state) {
 		ret = -ENOMEM;
 		goto out_drop_locks;
@@ -117,13 +117,13 @@ static void drm_self_refresh_helper_entry_work(struct work_struct *work)
 
 out:
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry;
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 out_drop_locks:
 	drm_modeset_drop_locks(&ctx);
@@ -143,7 +143,7 @@ static void drm_self_refresh_helper_entry_work(struct work_struct *work)
  * entering self refresh mode after activity.
  */
 void
-drm_self_refresh_helper_update_avg_times(struct drm_atomic_state *state,
+drm_self_refresh_helper_update_avg_times(struct drm_atomic_commit *state,
 					 unsigned int commit_time_ms,
 					 unsigned int new_self_refresh_mask)
 {
@@ -185,7 +185,7 @@ EXPORT_SYMBOL(drm_self_refresh_helper_update_avg_times);
  * At the end, we queue up the self refresh entry work so we can enter PSR after
  * the desired delay.
  */
-void drm_self_refresh_helper_alter_state(struct drm_atomic_state *state)
+void drm_self_refresh_helper_alter_state(struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c
index 4d91513a1e34c..8e1d07b9f1e30 100644
--- a/drivers/gpu/drm/drm_simple_kms_helper.c
+++ b/drivers/gpu/drm/drm_simple_kms_helper.c
@@ -53,7 +53,7 @@ drm_simple_kms_crtc_mode_valid(struct drm_crtc *crtc,
 }
 
 static int drm_simple_kms_crtc_check(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	int ret;
@@ -70,7 +70,7 @@ static int drm_simple_kms_crtc_check(struct drm_crtc *crtc,
 }
 
 static void drm_simple_kms_crtc_enable(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane *plane;
 	struct drm_simple_display_pipe *pipe;
@@ -84,7 +84,7 @@ static void drm_simple_kms_crtc_enable(struct drm_crtc *crtc,
 }
 
 static void drm_simple_kms_crtc_disable(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_simple_display_pipe *pipe;
 
@@ -169,7 +169,7 @@ static const struct drm_crtc_funcs drm_simple_kms_crtc_funcs = {
 };
 
 static int drm_simple_kms_plane_atomic_check(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state,
 									     plane);
@@ -198,7 +198,7 @@ static int drm_simple_kms_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void drm_simple_kms_plane_atomic_update(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_pstate = drm_atomic_get_old_plane_state(state,
 									    plane);
diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c
index 8d9fd1917c6e6..c9dbf64c0c9f2 100644
--- a/drivers/gpu/drm/drm_syncobj.c
+++ b/drivers/gpu/drm/drm_syncobj.c
@@ -442,13 +442,15 @@ int drm_syncobj_find_fence(struct drm_file *file_private,
 	u64 timeout = nsecs_to_jiffies64(DRM_SYNCOBJ_WAIT_FOR_SUBMIT_TIMEOUT);
 	int ret;
 
-	if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)
-		return -EINVAL;
-
 	if (!syncobj)
 		return -ENOENT;
 
-	/* Waiting for userspace with locks help is illegal cause that can
+	if (flags & ~DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* Waiting for userspace with locks held is illegal cause that can
 	 * trivial deadlock with page faults for example. Make lockdep complain
 	 * about it early on.
 	 */
diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c
index f78bf37f1e0a7..f90fb2d13e423 100644
--- a/drivers/gpu/drm/drm_vblank.c
+++ b/drivers/gpu/drm/drm_vblank.c
@@ -236,6 +236,21 @@ static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 	return drm_vblank_no_hw_counter(dev, pipe);
 }
 
+static bool get_vblank_counter_and_timestamp(struct drm_device *dev, unsigned int pipe,
+					     u32 *cur_vblank, ktime_t *t_vblank,
+					     bool in_vblank_irq)
+{
+	int count = DRM_TIMESTAMP_MAXRETRIES;
+	bool rc;
+
+	do {
+		*cur_vblank = __get_vblank_counter(dev, pipe);
+		rc = drm_get_last_vbltimestamp(dev, pipe, t_vblank, in_vblank_irq);
+	} while (*cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
+
+	return rc;
+}
+
 /*
  * Reset the stored timestamp for the current vblank count to correspond
  * to the last vblank occurred.
@@ -250,7 +265,6 @@ static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe
 	u32 cur_vblank;
 	bool rc;
 	ktime_t t_vblank;
-	int count = DRM_TIMESTAMP_MAXRETRIES;
 
 	spin_lock(&dev->vblank_time_lock);
 
@@ -258,10 +272,8 @@ static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe
 	 * sample the current counter to avoid random jumps
 	 * when drm_vblank_enable() applies the diff
 	 */
-	do {
-		cur_vblank = __get_vblank_counter(dev, pipe);
-		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
-	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
+	rc = get_vblank_counter_and_timestamp(dev, pipe, &cur_vblank,
+					      &t_vblank, false);
 
 	/*
 	 * Only reinitialize corresponding vblank timestamp if high-precision query
@@ -299,7 +311,6 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 	u32 cur_vblank, diff;
 	bool rc;
 	ktime_t t_vblank;
-	int count = DRM_TIMESTAMP_MAXRETRIES;
 	int framedur_ns = vblank->framedur_ns;
 	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
 
@@ -315,10 +326,8 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 	 * updating its hardware counter while we are retrieving the
 	 * corresponding vblank timestamp.
 	 */
-	do {
-		cur_vblank = __get_vblank_counter(dev, pipe);
-		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
-	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
+	rc = get_vblank_counter_and_timestamp(dev, pipe, &cur_vblank,
+					      &t_vblank, in_vblank_irq);
 
 	if (max_vblank_count) {
 		/* trust the hw counter when it's around */
@@ -1543,7 +1552,6 @@ static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
 	int framedur_ns;
 	u64 diff_ns;
 	u32 cur_vblank, diff = 1;
-	int count = DRM_TIMESTAMP_MAXRETRIES;
 	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
 
 	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
@@ -1558,10 +1566,8 @@ static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
 		      "Cannot compute missed vblanks without frame duration\n");
 	framedur_ns = vblank->framedur_ns;
 
-	do {
-		cur_vblank = __get_vblank_counter(dev, pipe);
-		drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
-	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
+	get_vblank_counter_and_timestamp(dev, pipe, &cur_vblank,
+					 &t_vblank, false);
 
 	diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
 	if (framedur_ns)
diff --git a/drivers/gpu/drm/drm_vblank_helper.c b/drivers/gpu/drm/drm_vblank_helper.c
index a04a6ba1b0ca0..d3f8147ecdc12 100644
--- a/drivers/gpu/drm/drm_vblank_helper.c
+++ b/drivers/gpu/drm/drm_vblank_helper.c
@@ -49,7 +49,7 @@
  * See also struct &drm_crtc_helper_funcs.atomic_flush.
  */
 void drm_crtc_vblank_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
@@ -82,7 +82,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_atomic_flush);
  * See also struct &drm_crtc_helper_funcs.atomic_enable.
  */
 void drm_crtc_vblank_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_on(crtc);
 }
@@ -99,7 +99,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_atomic_enable);
  * See also struct &drm_crtc_funcs.atomic_disable.
  */
 void drm_crtc_vblank_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_off(crtc);
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
index 20dad16fd5542..55a9e745604db 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h
@@ -93,18 +93,6 @@ void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv,
 #define DBG(fmt, ...) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 #define VERB(fmt, ...) if (0) DRM_DEBUG(fmt"\n", ##__VA_ARGS__)
 
-/*
- * Return the storage size of a structure with a variable length array.
- * The array is nelem elements of elem_size, where the base structure
- * is defined by base.  If the size overflows size_t, return zero.
- */
-static inline size_t size_vstruct(size_t nelem, size_t elem_size, size_t base)
-{
-	if (elem_size && nelem > (SIZE_MAX - base) / elem_size)
-		return 0;
-	return base + nelem * elem_size;
-}
-
 /*
  * Etnaviv timeouts are specified wrt CLOCK_MONOTONIC, not jiffies.
  * We need to calculate the timeout in terms of number of jiffies
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
index fbf693addd749..1a77a09b3377e 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c
@@ -32,9 +32,8 @@ static struct etnaviv_gem_submit *submit_create(struct drm_device *dev,
 		struct etnaviv_gpu *gpu, size_t nr_bos, size_t nr_pmrs)
 {
 	struct etnaviv_gem_submit *submit;
-	size_t sz = size_vstruct(nr_bos, sizeof(submit->bos[0]), sizeof(*submit));
 
-	submit = kzalloc(sz, GFP_KERNEL);
+	submit = kzalloc_flex(*submit, bos, nr_bos);
 	if (!submit)
 		return NULL;
 
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
index 3cc50d697c891..139e6e38784b3 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c
@@ -144,7 +144,6 @@ int etnaviv_sched_init(struct etnaviv_gpu *gpu)
 {
 	const struct drm_sched_init_args args = {
 		.ops = &etnaviv_sched_ops,
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit = etnaviv_hw_jobs_limit,
 		.hang_limit = etnaviv_job_hang_limit,
 		.timeout = msecs_to_jiffies(500),
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index 0d13828e7d9e6..a59131ae122c9 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -69,9 +69,12 @@ config DRM_EXYNOS_DSI
 config DRM_EXYNOS_DP
 	bool "Exynos specific extensions for Analogix DP driver"
 	depends on DRM_EXYNOS_FIMD || DRM_EXYNOS7_DECON
+	depends on OF
 	select DRM_ANALOGIX_DP
+	select DRM_BRIDGE_CONNECTOR
 	select DRM_DISPLAY_DP_HELPER
 	default DRM_EXYNOS
+	select DRM_OF_DISPLAY_MODE_BRIDGE
 	select DRM_PANEL
 	help
 	  This enables support for DP device.
diff --git a/drivers/gpu/drm/exynos/exynos_dp.c b/drivers/gpu/drm/exynos/exynos_dp.c
index 5bcf41e0bd047..b805403281504 100644
--- a/drivers/gpu/drm/exynos/exynos_dp.c
+++ b/drivers/gpu/drm/exynos/exynos_dp.c
@@ -19,8 +19,10 @@
 #include <video/videomode.h>
 
 #include <drm/bridge/analogix_dp.h>
+#include <drm/bridge/of-display-mode-bridge.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_bridge_connector.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_of.h>
 #include <drm/drm_panel.h>
@@ -35,12 +37,9 @@
 
 struct exynos_dp_device {
 	struct drm_encoder         encoder;
-	struct drm_connector       *connector;
-	struct drm_bridge          *ptn_bridge;
 	struct drm_device          *drm_dev;
 	struct device              *dev;
 
-	struct videomode           vm;
 	struct analogix_dp_device *adp;
 	struct analogix_dp_plat_data plat_data;
 };
@@ -69,53 +68,6 @@ static int exynos_dp_poweroff(struct analogix_dp_plat_data *plat_data)
 	return exynos_dp_crtc_clock_enable(plat_data, false);
 }
 
-static int exynos_dp_get_modes(struct analogix_dp_plat_data *plat_data,
-			       struct drm_connector *connector)
-{
-	struct exynos_dp_device *dp = to_dp(plat_data);
-	struct drm_display_mode *mode;
-
-	if (dp->plat_data.panel)
-		return 0;
-
-	mode = drm_mode_create(connector->dev);
-	if (!mode) {
-		DRM_DEV_ERROR(dp->dev,
-			      "failed to create a new display mode.\n");
-		return 0;
-	}
-
-	drm_display_mode_from_videomode(&dp->vm, mode);
-	connector->display_info.width_mm = mode->width_mm;
-	connector->display_info.height_mm = mode->height_mm;
-
-	mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
-	drm_mode_set_name(mode);
-	drm_mode_probed_add(connector, mode);
-
-	return 1;
-}
-
-static int exynos_dp_bridge_attach(struct analogix_dp_plat_data *plat_data,
-				   struct drm_bridge *bridge,
-				   struct drm_connector *connector)
-{
-	struct exynos_dp_device *dp = to_dp(plat_data);
-	int ret;
-
-	dp->connector = connector;
-
-	/* Pre-empt DP connector creation if there's a bridge */
-	if (dp->ptn_bridge) {
-		ret = drm_bridge_attach(&dp->encoder, dp->ptn_bridge, bridge,
-					0);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
 static void exynos_dp_mode_set(struct drm_encoder *encoder,
 			       struct drm_display_mode *mode,
 			       struct drm_display_mode *adjusted_mode)
@@ -133,34 +85,16 @@ static const struct drm_encoder_helper_funcs exynos_dp_encoder_helper_funcs = {
 	.disable = exynos_dp_nop,
 };
 
-static int exynos_dp_dt_parse_panel(struct exynos_dp_device *dp)
-{
-	int ret;
-
-	ret = of_get_videomode(dp->dev->of_node, &dp->vm, OF_USE_NATIVE_MODE);
-	if (ret) {
-		DRM_DEV_ERROR(dp->dev,
-			      "failed: of_get_videomode() : %d\n", ret);
-		return ret;
-	}
-	return 0;
-}
-
 static int exynos_dp_bind(struct device *dev, struct device *master, void *data)
 {
 	struct exynos_dp_device *dp = dev_get_drvdata(dev);
 	struct drm_encoder *encoder = &dp->encoder;
 	struct drm_device *drm_dev = data;
+	struct drm_connector *connector;
 	int ret;
 
 	dp->drm_dev = drm_dev;
 
-	if (!dp->plat_data.panel && !dp->ptn_bridge) {
-		ret = exynos_dp_dt_parse_panel(dp);
-		if (ret)
-			return ret;
-	}
-
 	drm_simple_encoder_init(drm_dev, encoder, DRM_MODE_ENCODER_TMDS);
 
 	drm_encoder_helper_add(encoder, &exynos_dp_encoder_helper_funcs);
@@ -172,10 +106,19 @@ static int exynos_dp_bind(struct device *dev, struct device *master, void *data)
 	dp->plat_data.encoder = encoder;
 
 	ret = analogix_dp_bind(dp->adp, dp->drm_dev);
-	if (ret)
+	if (ret) {
 		dp->encoder.funcs->destroy(&dp->encoder);
+		return ret;
+	}
+
+	connector = drm_bridge_connector_init(dp->drm_dev, dp->plat_data.encoder);
+	if (IS_ERR(connector)) {
+		ret = PTR_ERR(connector);
+		dev_err(dp->dev, "Failed to initialize bridge_connector\n");
+		return ret;
+	}
 
-	return ret;
+	return 0;
 }
 
 static void exynos_dp_unbind(struct device *dev, struct device *master,
@@ -197,9 +140,6 @@ static int exynos_dp_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct device_node *np;
 	struct exynos_dp_device *dp;
-	struct drm_panel *panel;
-	struct drm_bridge *bridge;
-	int ret;
 
 	dp = devm_kzalloc(&pdev->dev, sizeof(struct exynos_dp_device),
 			  GFP_KERNEL);
@@ -226,27 +166,29 @@ static int exynos_dp_probe(struct platform_device *pdev)
 		goto out;
 	}
 
-	ret = drm_of_find_panel_or_bridge(dev->of_node, 0, 0, &panel, &bridge);
-	if (ret)
-		return ret;
+	if (of_get_display_timings(dev->of_node)) {
+		dp->plat_data.next_bridge = devm_drm_of_display_mode_bridge(dp->dev,
+									dp->dev->of_node,
+									DRM_MODE_CONNECTOR_eDP);
+		if (IS_ERR(dp->plat_data.next_bridge))
+			return PTR_ERR(dp->plat_data.next_bridge);
+	}
 
 	/* The remote port can be either a panel or a bridge */
-	dp->plat_data.panel = panel;
 	dp->plat_data.dev_type = EXYNOS_DP;
 	dp->plat_data.power_on = exynos_dp_poweron;
 	dp->plat_data.power_off = exynos_dp_poweroff;
-	dp->plat_data.attach = exynos_dp_bridge_attach;
-	dp->plat_data.get_modes = exynos_dp_get_modes;
-	dp->plat_data.skip_connector = !!bridge;
-
-	dp->ptn_bridge = bridge;
+	dp->plat_data.ops = &exynos_dp_ops;
 
 out:
 	dp->adp = analogix_dp_probe(dev, &dp->plat_data);
 	if (IS_ERR(dp->adp))
 		return PTR_ERR(dp->adp);
 
-	return component_add(&pdev->dev, &exynos_dp_ops);
+	if (dp->plat_data.panel || dp->plat_data.next_bridge)
+		return component_add(&pdev->dev, &exynos_dp_ops);
+	else
+		return analogix_dp_finish_probe(dp->adp);
 }
 
 static void exynos_dp_remove(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
index 41252caccd674..b071c019670de 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
@@ -19,7 +19,7 @@
 #include "exynos_drm_plane.h"
 
 static void exynos_drm_crtc_atomic_enable(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
@@ -30,7 +30,7 @@ static void exynos_drm_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void exynos_drm_crtc_atomic_disable(struct drm_crtc *crtc,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
@@ -48,7 +48,7 @@ static void exynos_drm_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static int exynos_crtc_atomic_check(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -64,7 +64,7 @@ static int exynos_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void exynos_crtc_atomic_begin(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
@@ -73,7 +73,7 @@ static void exynos_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void exynos_crtc_atomic_flush(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index da043bd357a4f..5c04ef87e4007 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -230,7 +230,7 @@ exynos_drm_plane_check_size(const struct exynos_drm_plane_config *config,
 }
 
 static int exynos_plane_atomic_check(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -254,7 +254,7 @@ static int exynos_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void exynos_plane_atomic_update(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 								           plane);
@@ -269,7 +269,7 @@ static void exynos_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void exynos_plane_atomic_disable(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct exynos_drm_plane *exynos_plane = to_exynos_plane(plane);
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
index 2af60d98f48f9..9ef16f5eb613e 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
@@ -21,7 +21,7 @@
 #include "fsl_dcu_drm_plane.h"
 
 static void fsl_dcu_drm_crtc_atomic_flush(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
@@ -43,7 +43,7 @@ static void fsl_dcu_drm_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static void fsl_dcu_drm_crtc_atomic_disable(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
 									      crtc);
@@ -64,7 +64,7 @@ static void fsl_dcu_drm_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void fsl_dcu_drm_crtc_atomic_enable(struct drm_crtc *crtc,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
index a9b3467263d62..b34f880c1e098 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
@@ -36,7 +36,7 @@ static int fsl_dcu_drm_plane_index(struct drm_plane *plane)
 }
 
 static int fsl_dcu_drm_plane_atomic_check(struct drm_plane *plane,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -62,7 +62,7 @@ static int fsl_dcu_drm_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void fsl_dcu_drm_plane_atomic_disable(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct fsl_dcu_drm_device *fsl_dev = plane->dev->dev_private;
 	unsigned int value;
@@ -78,7 +78,7 @@ static void fsl_dcu_drm_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static void fsl_dcu_drm_plane_atomic_update(struct drm_plane *plane,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 
 {
 	struct fsl_dcu_drm_device *fsl_dev = plane->dev->dev_private;
diff --git a/drivers/gpu/drm/gma500/mmu.c b/drivers/gpu/drm/gma500/mmu.c
index 6b6b44e426cf5..4fbc22a59ac7a 100644
--- a/drivers/gpu/drm/gma500/mmu.c
+++ b/drivers/gpu/drm/gma500/mmu.c
@@ -7,6 +7,8 @@
 #include <linux/highmem.h>
 #include <linux/vmalloc.h>
 
+#include <asm/cpuid/api.h>
+
 #include "mmu.h"
 #include "psb_drv.h"
 #include "psb_reg.h"
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index 005ab7f5355f3..947f7c4ff4e9a 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -24,7 +24,6 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_file.h>
 #include <drm/drm_ioctl.h>
-#include <drm/drm_pciids.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
 
diff --git a/drivers/gpu/drm/gud/gud_connector.c b/drivers/gpu/drm/gud/gud_connector.c
index a91a7e96cb886..ea0cca58b7c89 100644
--- a/drivers/gpu/drm/gud/gud_connector.c
+++ b/drivers/gpu/drm/gud/gud_connector.c
@@ -62,13 +62,13 @@ static void gud_connector_backlight_update_status_work(struct work_struct *work)
 	struct drm_connector_state *connector_state;
 	struct drm_device *drm = connector->dev;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	int idx, ret;
 
 	if (!drm_dev_enter(drm, &idx))
 		return;
 
-	state = drm_atomic_state_alloc(drm);
+	state = drm_atomic_commit_alloc(drm);
 	if (!state) {
 		ret = -ENOMEM;
 		goto exit;
@@ -89,12 +89,12 @@ static void gud_connector_backlight_update_status_work(struct work_struct *work)
 	ret = drm_atomic_commit(state);
 out:
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 		goto retry;
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
@@ -285,7 +285,7 @@ static int gud_connector_get_modes(struct drm_connector *connector)
 }
 
 static int gud_connector_atomic_check(struct drm_connector *connector,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *new_state;
 	struct drm_crtc_state *new_crtc_state;
diff --git a/drivers/gpu/drm/gud/gud_drv.c b/drivers/gpu/drm/gud/gud_drv.c
index 17c2dead2c13b..89bd6ca36003f 100644
--- a/drivers/gpu/drm/gud/gud_drv.c
+++ b/drivers/gpu/drm/gud/gud_drv.c
@@ -684,6 +684,7 @@ static int gud_resume(struct usb_interface *intf)
 static const struct usb_device_id gud_id_table[] = {
 	{ USB_DEVICE_INTERFACE_CLASS(0x1d50, 0x614d, USB_CLASS_VENDOR_SPEC) },
 	{ USB_DEVICE_INTERFACE_CLASS(0x16d0, 0x10a9, USB_CLASS_VENDOR_SPEC) },
+	{ USB_DEVICE_INTERFACE_CLASS(0x1209, 0x4fb3, USB_CLASS_VENDOR_SPEC) },
 	{ }
 };
 
diff --git a/drivers/gpu/drm/gud/gud_internal.h b/drivers/gpu/drm/gud/gud_internal.h
index 8eec8335f5f90..95744b0b6c727 100644
--- a/drivers/gpu/drm/gud/gud_internal.h
+++ b/drivers/gpu/drm/gud/gud_internal.h
@@ -63,13 +63,13 @@ int gud_usb_set_u8(struct gud_device *gdrm, u8 request, u8 val);
 void gud_clear_damage(struct gud_device *gdrm);
 void gud_flush_work(struct work_struct *work);
 void gud_crtc_atomic_enable(struct drm_crtc *crtc,
-			    struct drm_atomic_state *state);
+			    struct drm_atomic_commit *state);
 void gud_crtc_atomic_disable(struct drm_crtc *crtc,
-			     struct drm_atomic_state *state);
+			     struct drm_atomic_commit *state);
 int gud_plane_atomic_check(struct drm_plane *plane,
-			   struct drm_atomic_state *state);
+			   struct drm_atomic_commit *state);
 void gud_plane_atomic_update(struct drm_plane *plane,
-			     struct drm_atomic_state *atomic_state);
+			     struct drm_atomic_commit *atomic_state);
 int gud_connector_fill_properties(struct drm_connector_state *connector_state,
 				  struct gud_property_req *properties);
 int gud_get_connectors(struct gud_device *gdrm);
diff --git a/drivers/gpu/drm/gud/gud_pipe.c b/drivers/gpu/drm/gud/gud_pipe.c
index 495804e1d2005..5ef887d8485a3 100644
--- a/drivers/gpu/drm/gud/gud_pipe.c
+++ b/drivers/gpu/drm/gud/gud_pipe.c
@@ -451,7 +451,7 @@ static void gud_fb_handle_damage(struct gud_device *gdrm, struct drm_framebuffer
 }
 
 int gud_plane_atomic_check(struct drm_plane *plane,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct gud_device *gdrm = to_gud_device(plane->dev);
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
@@ -581,7 +581,7 @@ int gud_plane_atomic_check(struct drm_plane *plane,
 }
 
 void gud_crtc_atomic_enable(struct drm_crtc *crtc,
-			    struct drm_atomic_state *state)
+			    struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct gud_device *gdrm = to_gud_device(drm);
@@ -598,7 +598,7 @@ void gud_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 void gud_crtc_atomic_disable(struct drm_crtc *crtc,
-			     struct drm_atomic_state *state)
+			     struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct gud_device *gdrm = to_gud_device(drm);
@@ -614,7 +614,7 @@ void gud_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 void gud_plane_atomic_update(struct drm_plane *plane,
-			     struct drm_atomic_state *atomic_state)
+			     struct drm_atomic_commit *atomic_state)
 {
 	struct drm_device *drm = plane->dev;
 	struct gud_device *gdrm = to_gud_device(drm);
diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h
index f9ee7ebfec55c..f53dac256ee0b 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h
+++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_comm.h
@@ -69,5 +69,6 @@ int hibmc_dp_link_training(struct hibmc_dp_dev *dp);
 int hibmc_dp_serdes_init(struct hibmc_dp_dev *dp);
 int hibmc_dp_serdes_rate_switch(u8 rate, struct hibmc_dp_dev *dp);
 int hibmc_dp_serdes_set_tx_cfg(struct hibmc_dp_dev *dp, u8 train_set[HIBMC_DP_LANE_NUM_MAX]);
+void hibmc_dp_update_caps(struct hibmc_dp_dev *dp);
 
 #endif
diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h
index 31316fe1ea8db..0f3662d8737e9 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h
+++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_hw.h
@@ -55,6 +55,7 @@ struct hibmc_dp {
 	struct drm_dp_aux aux;
 	struct hibmc_dp_cbar_cfg cfg;
 	u32 irq_status;
+	int phys_status;
 };
 
 int hibmc_dp_hw_init(struct hibmc_dp *dp);
diff --git a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_link.c b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_link.c
index 0726cb5b736e6..8c53f16db5160 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/dp/dp_link.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/dp/dp_link.c
@@ -325,7 +325,7 @@ static int hibmc_dp_link_downgrade_training_eq(struct hibmc_dp_dev *dp)
 	return hibmc_dp_link_reduce_rate(dp);
 }
 
-static void hibmc_dp_update_caps(struct hibmc_dp_dev *dp)
+void hibmc_dp_update_caps(struct hibmc_dp_dev *dp)
 {
 	dp->link.cap.link_rate = dp->dpcd[DP_MAX_LINK_RATE];
 	if (dp->link.cap.link_rate > DP_LINK_BW_8_1 || !dp->link.cap.link_rate)
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
index 89bed78f14666..3066dc9ebc647 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_de.c
@@ -32,28 +32,45 @@ struct hibmc_display_panel_pll {
 struct hibmc_dislay_pll_config {
 	u64 hdisplay;
 	u64 vdisplay;
+	int clock;
 	u32 pll1_config_value;
 	u32 pll2_config_value;
 };
 
 static const struct hibmc_dislay_pll_config hibmc_pll_table[] = {
-	{640, 480, CRT_PLL1_HS_25MHZ, CRT_PLL2_HS_25MHZ},
-	{800, 600, CRT_PLL1_HS_40MHZ, CRT_PLL2_HS_40MHZ},
-	{1024, 768, CRT_PLL1_HS_65MHZ, CRT_PLL2_HS_65MHZ},
-	{1152, 864, CRT_PLL1_HS_80MHZ_1152, CRT_PLL2_HS_80MHZ},
-	{1280, 768, CRT_PLL1_HS_80MHZ, CRT_PLL2_HS_80MHZ},
-	{1280, 720, CRT_PLL1_HS_74MHZ, CRT_PLL2_HS_74MHZ},
-	{1280, 960, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ},
-	{1280, 1024, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ},
-	{1440, 900, CRT_PLL1_HS_106MHZ, CRT_PLL2_HS_106MHZ},
-	{1600, 900, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ},
-	{1600, 1200, CRT_PLL1_HS_162MHZ, CRT_PLL2_HS_162MHZ},
-	{1920, 1080, CRT_PLL1_HS_148MHZ, CRT_PLL2_HS_148MHZ},
-	{1920, 1200, CRT_PLL1_HS_193MHZ, CRT_PLL2_HS_193MHZ},
+	{640, 480, 25000, CRT_PLL1_HS_25MHZ, CRT_PLL2_HS_25MHZ},
+	{800, 600, 40000, CRT_PLL1_HS_40MHZ, CRT_PLL2_HS_40MHZ},
+	{1024, 768, 65000, CRT_PLL1_HS_65MHZ, CRT_PLL2_HS_65MHZ},
+	{1152, 864, 78750, CRT_PLL1_HS_80MHZ_1152, CRT_PLL2_HS_80MHZ},
+	{1280, 768, 80000, CRT_PLL1_HS_80MHZ, CRT_PLL2_HS_80MHZ},
+	{1280, 720, 74375, CRT_PLL1_HS_74MHZ, CRT_PLL2_HS_74MHZ},
+	{1280, 960, 108000, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ},
+	{1280, 1024, 108000, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ},
+	{1440, 900, 105952, CRT_PLL1_HS_106MHZ, CRT_PLL2_HS_106MHZ},
+	{1600, 900, 108000, CRT_PLL1_HS_108MHZ, CRT_PLL2_HS_108MHZ},
+	{1600, 1200, 162500, CRT_PLL1_HS_162MHZ, CRT_PLL2_HS_162MHZ},
+	{1920, 1080, 148750, CRT_PLL1_HS_148MHZ, CRT_PLL2_HS_148MHZ},
+	{1920, 1200, 193750, CRT_PLL1_HS_193MHZ, CRT_PLL2_HS_193MHZ},
 };
 
+static int hibmc_get_best_clock_idx(const struct drm_display_mode *mode)
+{
+	int i, diff;
+
+	for (i = 0; i < ARRAY_SIZE(hibmc_pll_table); i++) {
+		if (hibmc_pll_table[i].hdisplay == mode->hdisplay &&
+		    hibmc_pll_table[i].vdisplay == mode->vdisplay) {
+			diff = abs(mode->clock - hibmc_pll_table[i].clock);
+			if (diff < mode->clock / 100) /* tolerance 1/100 */
+				return i;
+		}
+	}
+
+	return -MODE_CLOCK_RANGE;
+}
+
 static int hibmc_plane_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -99,7 +116,7 @@ static int hibmc_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void hibmc_plane_atomic_update(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -172,7 +189,7 @@ static void hibmc_crtc_dpms(struct drm_crtc *crtc, u32 dpms)
 }
 
 static void hibmc_crtc_atomic_enable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	u32 reg;
 	struct hibmc_drm_private *priv = to_hibmc_drm_private(crtc->dev);
@@ -191,7 +208,7 @@ static void hibmc_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void hibmc_crtc_atomic_disable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	u32 reg;
 	struct hibmc_drm_private *priv = to_hibmc_drm_private(crtc->dev);
@@ -214,19 +231,15 @@ static enum drm_mode_status
 hibmc_crtc_mode_valid(struct drm_crtc *crtc,
 		      const struct drm_display_mode *mode)
 {
-	size_t i = 0;
 	int vrefresh = drm_mode_vrefresh(mode);
 
 	if (vrefresh < 59 || vrefresh > 61)
 		return MODE_NOCLOCK;
 
-	for (i = 0; i < ARRAY_SIZE(hibmc_pll_table); i++) {
-		if (hibmc_pll_table[i].hdisplay == mode->hdisplay &&
-		    hibmc_pll_table[i].vdisplay == mode->vdisplay)
-			return MODE_OK;
-	}
+	if (hibmc_get_best_clock_idx(mode) >= 0)
+		return MODE_OK;
 
-	return MODE_BAD;
+	return MODE_CLOCK_RANGE;
 }
 
 static u32 format_pll_reg(void)
@@ -281,23 +294,20 @@ static void set_vclock_hisilicon(struct drm_device *dev, u64 pll)
 	writel(val, priv->mmio + CRT_PLL1_HS);
 }
 
-static void get_pll_config(u64 x, u64 y, u32 *pll1, u32 *pll2)
+static void get_pll_config(struct drm_display_mode *mode, u32 *pll1, u32 *pll2)
 {
-	size_t i;
-	size_t count = ARRAY_SIZE(hibmc_pll_table);
-
-	for (i = 0; i < count; i++) {
-		if (hibmc_pll_table[i].hdisplay == x &&
-		    hibmc_pll_table[i].vdisplay == y) {
-			*pll1 = hibmc_pll_table[i].pll1_config_value;
-			*pll2 = hibmc_pll_table[i].pll2_config_value;
-			return;
-		}
+	int idx;
+
+	idx = hibmc_get_best_clock_idx(mode);
+	if (idx < 0) {
+		/* if found none, we use default value */
+		*pll1 = CRT_PLL1_HS_25MHZ;
+		*pll2 = CRT_PLL2_HS_25MHZ;
+		return;
 	}
 
-	/* if found none, we use default value */
-	*pll1 = CRT_PLL1_HS_25MHZ;
-	*pll2 = CRT_PLL2_HS_25MHZ;
+	*pll1 = hibmc_pll_table[idx].pll1_config_value;
+	*pll2 = hibmc_pll_table[idx].pll2_config_value;
 }
 
 /*
@@ -319,7 +329,7 @@ static u32 display_ctrl_adjust(struct drm_device *dev,
 	x = mode->hdisplay;
 	y = mode->vdisplay;
 
-	get_pll_config(x, y, &pll1, &pll2);
+	get_pll_config(mode, &pll1, &pll2);
 	writel(pll2, priv->mmio + CRT_PLL2_HS);
 	set_vclock_hisilicon(dev, pll1);
 
@@ -392,7 +402,7 @@ static void hibmc_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static void hibmc_crtc_atomic_begin(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	u32 reg;
 	struct drm_device *dev = crtc->dev;
@@ -412,7 +422,7 @@ static void hibmc_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void hibmc_crtc_atomic_flush(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 
 {
 	unsigned long flags;
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c
index 616821e3c933b..2e9403b8bf3c8 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_dp.c
@@ -41,6 +41,8 @@ static bool hibmc_dp_get_dpcd(struct hibmc_dp_dev *dp_dev)
 	if (ret)
 		return false;
 
+	hibmc_dp_update_caps(dp_dev);
+
 	dp_dev->is_branch = drm_dp_is_branch(dp_dev->dpcd);
 
 	ret = drm_dp_read_desc(dp_dev->aux, &dp_dev->desc, dp_dev->is_branch);
@@ -59,27 +61,38 @@ static int hibmc_dp_detect(struct drm_connector *connector,
 {
 	struct hibmc_dp *dp = to_hibmc_dp(connector);
 	struct hibmc_dp_dev *dp_dev = dp->dp_dev;
-	int ret;
+	int ret = connector_status_disconnected;
 
 	if (dp->irq_status) {
-		if (dp_dev->hpd_status != HIBMC_HPD_IN)
-			return connector_status_disconnected;
+		if (dp_dev->hpd_status != HIBMC_HPD_IN) {
+			ret = connector_status_disconnected;
+			goto exit;
+		}
 	}
 
-	if (!hibmc_dp_get_dpcd(dp_dev))
-		return connector_status_disconnected;
+	if (!hibmc_dp_get_dpcd(dp_dev)) {
+		ret = connector_status_disconnected;
+		goto exit;
+	}
 
-	if (!dp_dev->is_branch)
-		return connector_status_connected;
+	if (!dp_dev->is_branch) {
+		ret = connector_status_connected;
+		goto exit;
+	}
 
 	if (drm_dp_read_sink_count_cap(connector, dp_dev->dpcd, &dp_dev->desc) &&
 	    dp_dev->downstream_ports[0] & DP_DS_PORT_HPD) {
 		ret = drm_dp_read_sink_count(dp_dev->aux);
-		if (ret > 0)
-			return connector_status_connected;
+		if (ret > 0) {
+			ret = connector_status_connected;
+			goto exit;
+		}
 	}
 
-	return connector_status_disconnected;
+exit:
+	dp->phys_status = ret;
+
+	return ret;
 }
 
 static int hibmc_dp_mode_valid(struct drm_connector *connector,
@@ -148,7 +161,7 @@ static inline int hibmc_dp_prepare(struct hibmc_dp *dp, struct drm_display_mode
 }
 
 static void hibmc_dp_encoder_enable(struct drm_encoder *drm_encoder,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct hibmc_dp *dp = container_of(drm_encoder, struct hibmc_dp, encoder);
 	struct drm_display_mode *mode = &drm_encoder->crtc->state->mode;
@@ -160,7 +173,7 @@ static void hibmc_dp_encoder_enable(struct drm_encoder *drm_encoder,
 }
 
 static void hibmc_dp_encoder_disable(struct drm_encoder *drm_encoder,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct hibmc_dp *dp = container_of(drm_encoder, struct hibmc_dp, encoder);
 
@@ -241,5 +254,7 @@ int hibmc_dp_init(struct hibmc_drm_private *priv)
 
 	connector->polled = DRM_CONNECTOR_POLL_HPD;
 
+	dp->phys_status = connector_status_disconnected;
+
 	return 0;
 }
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
index 289304500ab09..99b36de1fe137 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.c
@@ -24,6 +24,7 @@
 #include <drm/drm_managed.h>
 #include <drm/drm_module.h>
 #include <drm/drm_vblank.h>
+#include <drm/drm_probe_helper.h>
 
 #include "hibmc_drm_drv.h"
 #include "hibmc_drm_regs.h"
@@ -214,6 +215,15 @@ void hibmc_set_current_gate(struct hibmc_drm_private *priv, unsigned int gate)
 	writel(gate, mmio + gate_reg);
 }
 
+static void hibmc_display_ctrl(struct hibmc_drm_private *priv)
+{
+	u32 reg;
+
+	reg = readl(priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE);
+	reg |= HIBMC_DISPLAY_CONTROL_PANELDATE(1);
+	writel(reg, priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE);
+}
+
 static void hibmc_hw_config(struct hibmc_drm_private *priv)
 {
 	u32 reg;
@@ -245,6 +255,8 @@ static void hibmc_hw_config(struct hibmc_drm_private *priv)
 	reg |= HIBMC_MSCCTL_LOCALMEM_RESET(1);
 
 	writel(reg, priv->mmio + HIBMC_MISC_CTRL);
+
+	hibmc_display_ctrl(priv);
 }
 
 static int hibmc_hw_map(struct hibmc_drm_private *priv)
@@ -355,6 +367,8 @@ static int hibmc_load(struct drm_device *dev)
 	/* reset all the states of crtc/plane/encoder/connector */
 	drm_mode_config_reset(dev);
 
+	drmm_kms_helper_poll_init(dev);
+
 	return 0;
 
 err:
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
index ca8502e2760c1..cd3a3fca1fe60 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_drv.h
@@ -31,6 +31,7 @@ struct hibmc_vdac {
 	struct drm_connector connector;
 	struct i2c_adapter adapter;
 	struct i2c_algo_bit_data bit_data;
+	int phys_status;
 };
 
 struct hibmc_drm_private {
diff --git a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
index 841e81f47b686..b9bd6d33fb0f7 100644
--- a/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
+++ b/drivers/gpu/drm/hisilicon/hibmc/hibmc_drm_vdac.c
@@ -24,28 +24,21 @@
 
 static int hibmc_connector_get_modes(struct drm_connector *connector)
 {
+	struct drm_mode_config *mode_config = &connector->dev->mode_config;
 	struct hibmc_vdac *vdac = to_hibmc_vdac(connector);
-	const struct drm_edid *drm_edid;
 	int count;
 
-	drm_edid = drm_edid_read_ddc(connector, &vdac->adapter);
-
-	drm_edid_connector_update(connector, drm_edid);
-
-	if (drm_edid) {
-		count = drm_edid_connector_add_modes(connector);
+	if (vdac->phys_status == connector_status_connected) {
+		count = drm_connector_helper_get_modes(connector);
+	} else {
+		drm_edid_connector_update(connector, NULL);
+		count = drm_add_modes_noedid(connector,
+					     mode_config->max_width,
+					     mode_config->max_height);
 		if (count)
-			goto out;
+			drm_set_preferred_mode(connector, 1024, 768);
 	}
 
-	count = drm_add_modes_noedid(connector,
-				     connector->dev->mode_config.max_width,
-				     connector->dev->mode_config.max_height);
-	drm_set_preferred_mode(connector, 1024, 768);
-
-out:
-	drm_edid_free(drm_edid);
-
 	return count;
 }
 
@@ -57,10 +50,34 @@ static void hibmc_connector_destroy(struct drm_connector *connector)
 	drm_connector_cleanup(connector);
 }
 
+static int hibmc_vdac_detect(struct drm_connector *connector,
+			     struct drm_modeset_acquire_ctx *ctx,
+			     bool force)
+{
+	struct hibmc_drm_private *priv = to_hibmc_drm_private(connector->dev);
+	int status = drm_connector_helper_detect_from_ddc(connector, ctx,
+							 force);
+	struct hibmc_vdac *vdac = to_hibmc_vdac(connector);
+
+	if (priv->dp.phys_status == connector_status_connected) {
+		vdac->phys_status = status;
+		return status;
+	}
+
+	if (status != vdac->phys_status)
+		++connector->epoch_counter;
+	vdac->phys_status = status;
+
+	/* When both the DP and VDAC physical status are disconnected,
+	 * the "connected" status is returned to support KVM display.
+	 */
+	return connector_status_connected;
+}
+
 static const struct drm_connector_helper_funcs
 	hibmc_connector_helper_funcs = {
 	.get_modes = hibmc_connector_get_modes,
-	.detect_ctx = drm_connector_helper_detect_from_ddc,
+	.detect_ctx = hibmc_vdac_detect,
 };
 
 static const struct drm_connector_funcs hibmc_connector_funcs = {
@@ -71,26 +88,6 @@ static const struct drm_connector_funcs hibmc_connector_funcs = {
 	.atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
-static void hibmc_encoder_mode_set(struct drm_encoder *encoder,
-				   struct drm_display_mode *mode,
-				   struct drm_display_mode *adj_mode)
-{
-	u32 reg;
-	struct drm_device *dev = encoder->dev;
-	struct hibmc_drm_private *priv = to_hibmc_drm_private(dev);
-
-	reg = readl(priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE);
-	reg |= HIBMC_DISPLAY_CONTROL_FPVDDEN(1);
-	reg |= HIBMC_DISPLAY_CONTROL_PANELDATE(1);
-	reg |= HIBMC_DISPLAY_CONTROL_FPEN(1);
-	reg |= HIBMC_DISPLAY_CONTROL_VBIASEN(1);
-	writel(reg, priv->mmio + HIBMC_DISPLAY_CONTROL_HISILE);
-}
-
-static const struct drm_encoder_helper_funcs hibmc_encoder_helper_funcs = {
-	.mode_set = hibmc_encoder_mode_set,
-};
-
 int hibmc_vdac_init(struct hibmc_drm_private *priv)
 {
 	struct drm_device *dev = &priv->dev;
@@ -113,8 +110,6 @@ int hibmc_vdac_init(struct hibmc_drm_private *priv)
 		goto err;
 	}
 
-	drm_encoder_helper_add(encoder, &hibmc_encoder_helper_funcs);
-
 	ret = drm_connector_init_with_ddc(dev, connector,
 					  &hibmc_connector_funcs,
 					  DRM_MODE_CONNECTOR_VGA,
@@ -130,6 +125,8 @@ int hibmc_vdac_init(struct hibmc_drm_private *priv)
 
 	connector->polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
 
+	vdac->phys_status = connector_status_disconnected;
+
 	return 0;
 
 err:
diff --git a/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c b/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c
index e80debdc41763..ab3cd309505aa 100644
--- a/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c
+++ b/drivers/gpu/drm/hisilicon/kirin/dw_drm_dsi.c
@@ -778,17 +778,16 @@ static int dsi_host_init(struct device *dev, struct dw_dsi *dsi)
 static int dsi_bridge_init(struct drm_device *dev, struct dw_dsi *dsi)
 {
 	struct drm_encoder *encoder = &dsi->encoder;
-	struct drm_bridge *bridge;
+	struct drm_bridge *bridge __free(drm_bridge_put) = NULL;
 	struct device_node *np = dsi->dev->of_node;
-	int ret;
 
 	/*
 	 * Get the endpoint node. In our case, dsi has one output port1
 	 * to which the external HDMI bridge is connected.
 	 */
-	ret = drm_of_find_panel_or_bridge(np, 1, 0, NULL, &bridge);
-	if (ret)
-		return ret;
+	bridge = of_drm_get_bridge_by_endpoint(np, 1, 0);
+	if (IS_ERR(bridge))
+		return PTR_ERR(bridge);
 
 	/* associate the bridge to dsi encoder */
 	return drm_bridge_attach(encoder, bridge, NULL, 0);
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
index 76384b4581bf3..902c5dace4ff5 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
@@ -438,7 +438,7 @@ static void ade_dump_regs(void __iomem *base) { }
 #endif
 
 static void ade_crtc_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct kirin_crtc *kcrtc = to_kirin_crtc(crtc);
 	struct ade_hw_ctx *ctx = kcrtc->hw_ctx;
@@ -461,7 +461,7 @@ static void ade_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void ade_crtc_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct kirin_crtc *kcrtc = to_kirin_crtc(crtc);
 	struct ade_hw_ctx *ctx = kcrtc->hw_ctx;
@@ -487,7 +487,7 @@ static void ade_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static void ade_crtc_atomic_begin(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct kirin_crtc *kcrtc = to_kirin_crtc(crtc);
 	struct ade_hw_ctx *ctx = kcrtc->hw_ctx;
@@ -500,7 +500,7 @@ static void ade_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void ade_crtc_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 
 {
 	struct kirin_crtc *kcrtc = to_kirin_crtc(crtc);
@@ -759,7 +759,7 @@ static void ade_disable_channel(struct kirin_plane *kplane)
 }
 
 static int ade_plane_atomic_check(struct drm_plane *plane,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -806,7 +806,7 @@ static int ade_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void ade_plane_atomic_update(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -820,7 +820,7 @@ static void ade_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void ade_plane_atomic_disable(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct kirin_plane *kplane = to_kirin_plane(plane);
 
diff --git a/drivers/gpu/drm/hyperv/Kconfig b/drivers/gpu/drm/hyperv/Kconfig
index 86234f6a73f2a..e48e35fb7f8b2 100644
--- a/drivers/gpu/drm/hyperv/Kconfig
+++ b/drivers/gpu/drm/hyperv/Kconfig
@@ -8,7 +8,6 @@ config DRM_HYPERV
 	help
 	 This is a KMS driver for Hyper-V synthetic video device. Choose this
 	 option if you would like to enable drm driver for Hyper-V virtual
-	 machine. Unselect Hyper-V framebuffer driver (CONFIG_FB_HYPERV) so
-	 that DRM driver is used by default.
+	 machine.
 
 	 If M is selected the module will be called hyperv_drm.
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
index 06b5d96e6eaf6..b6bf6412ae34a 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_drv.c
@@ -150,6 +150,10 @@ static int hyperv_vmbus_probe(struct hv_device *hdev,
 		goto err_free_mmio;
 	}
 
+	/* If DRM panic path is stubbed out VMBus code must do the unload */
+	if (IS_ENABLED(CONFIG_DRM_PANIC))
+		vmbus_set_skip_unload(true);
+
 	drm_client_setup(dev, NULL);
 
 	return 0;
@@ -169,6 +173,7 @@ static void hyperv_vmbus_remove(struct hv_device *hdev)
 	struct drm_device *dev = hv_get_drvdata(hdev);
 	struct hyperv_drm_device *hv = to_hv(dev);
 
+	vmbus_set_skip_unload(false);
 	drm_dev_unplug(dev);
 	drm_atomic_helper_shutdown(dev);
 	vmbus_close(hdev->channel);
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c b/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
index 7978f8c8108c2..793dbbf618931 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_modeset.c
@@ -102,7 +102,7 @@ static const uint64_t hyperv_modifiers[] = {
 };
 
 static void hyperv_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct hyperv_drm_device *hv = to_hv(crtc->dev);
 	struct drm_plane *plane = &hv->plane;
@@ -136,7 +136,7 @@ static const struct drm_crtc_funcs hyperv_crtc_funcs = {
 };
 
 static int hyperv_plane_atomic_check(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct hyperv_drm_device *hv = to_hv(plane->dev);
@@ -168,7 +168,7 @@ static int hyperv_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void hyperv_plane_atomic_update(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct hyperv_drm_device *hv = to_hv(plane->dev);
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
@@ -212,15 +212,16 @@ static void hyperv_plane_panic_flush(struct drm_plane *plane)
 	struct hyperv_drm_device *hv = to_hv(plane->dev);
 	struct drm_rect rect;
 
-	if (!plane->state || !plane->state->fb)
-		return;
+	if (plane->state && plane->state->fb) {
+		rect.x1 = 0;
+		rect.y1 = 0;
+		rect.x2 = plane->state->fb->width;
+		rect.y2 = plane->state->fb->height;
 
-	rect.x1 = 0;
-	rect.y1 = 0;
-	rect.x2 = plane->state->fb->width;
-	rect.y2 = plane->state->fb->height;
+		hyperv_update_dirt(hv->hdev, &rect);
+	}
 
-	hyperv_update_dirt(hv->hdev, &rect);
+	vmbus_initiate_unload(true);
 }
 
 static const struct drm_plane_helper_funcs hyperv_plane_helper_funcs = {
diff --git a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
index 051ecc5268321..753d97bff76ff 100644
--- a/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
+++ b/drivers/gpu/drm/hyperv/hyperv_drm_proto.c
@@ -10,7 +10,7 @@
 
 #include "hyperv_drm.h"
 
-#define VMBUS_RING_BUFSIZE (256 * 1024)
+#define VMBUS_RING_BUFSIZE VMBUS_RING_SIZE(256 * 1024)
 #define VMBUS_VSP_TIMEOUT (10 * HZ)
 
 #define SYNTHVID_VERSION(major, minor) ((minor) << 16 | (major))
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index 3562a02ef7adc..52a3a59b4ba2c 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -51,7 +51,7 @@ config DRM_I915_DEBUG
 	select DRM_DEBUG_MM if DRM=y
 	select DRM_EXPORT_FOR_TESTS if m
 	select DRM_KUNIT_TEST if KUNIT
-	select DMABUF_SELFTESTS
+	select DMABUF_KUNIT_TEST if KUNIT
 	select SW_SYNC # signaling validation framework (igt/syncobj*)
 	select DRM_I915_WERROR
 	select DRM_I915_DEBUG_GEM
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index b677720a1c2db..07802a7f4ce50 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -80,6 +80,7 @@ i915-y += \
 	i915_display_pc8.o \
 	i915_dpt.o \
 	i915_dsb_buffer.o \
+	i915_fb_pin.o \
 	i915_hdcp_gsc.o \
 	i915_initial_plane.o \
 	i915_overlay.o \
@@ -255,6 +256,7 @@ i915-y += \
 	display/intel_cursor.o \
 	display/intel_dbuf_bw.o \
 	display/intel_de.o \
+	display/intel_display_clock_gating.o \
 	display/intel_display.o \
 	display/intel_display_conversion.o \
 	display/intel_display_driver.o \
@@ -279,7 +281,6 @@ i915-y += \
 	display/intel_drrs.o \
 	display/intel_dsb.o \
 	display/intel_fb.o \
-	display/intel_fb_pin.o \
 	display/intel_fbc.o \
 	display/intel_fdi.o \
 	display/intel_fifo_underrun.o \
@@ -295,6 +296,7 @@ i915-y += \
 	display/intel_link_bw.o \
 	display/intel_load_detect.o \
 	display/intel_lpe_audio.o \
+	display/intel_mchbar.o \
 	display/intel_modeset_lock.o \
 	display/intel_modeset_setup.o \
 	display/intel_modeset_verify.o \
@@ -325,8 +327,7 @@ i915-$(CONFIG_ACPI) += \
 	display/intel_acpi.o \
 	display/intel_opregion.o
 i915-$(CONFIG_DRM_FBDEV_EMULATION) += \
-	display/intel_fbdev.o \
-	display/intel_fbdev_fb.o
+	display/intel_fbdev.o
 i915-$(CONFIG_DEBUG_FS) += \
 	display/intel_display_debugfs.o \
 	display/intel_display_debugfs_params.o \
diff --git a/drivers/gpu/drm/i915/display/g4x_dp.c b/drivers/gpu/drm/i915/display/g4x_dp.c
index 5e74d8a3ba5c8..5ff1cdf4581a5 100644
--- a/drivers/gpu/drm/i915/display/g4x_dp.c
+++ b/drivers/gpu/drm/i915/display/g4x_dp.c
@@ -273,7 +273,7 @@ static bool cpt_dp_port_selected(struct intel_display *display,
 }
 
 bool g4x_dp_port_enabled(struct intel_display *display,
-			 i915_reg_t dp_reg, enum port port,
+			 intel_reg_t dp_reg, enum port port,
 			 enum pipe *pipe)
 {
 	bool ret;
@@ -1280,7 +1280,7 @@ static const struct drm_encoder_funcs intel_dp_enc_funcs = {
 };
 
 bool g4x_dp_init(struct intel_display *display,
-		 i915_reg_t output_reg, enum port port)
+		 intel_reg_t output_reg, enum port port)
 {
 	const struct intel_bios_encoder_data *devdata;
 	struct intel_digital_port *dig_port;
diff --git a/drivers/gpu/drm/i915/display/g4x_dp.h b/drivers/gpu/drm/i915/display/g4x_dp.h
index 0b28951b8365d..99de690ebe7f8 100644
--- a/drivers/gpu/drm/i915/display/g4x_dp.h
+++ b/drivers/gpu/drm/i915/display/g4x_dp.h
@@ -8,7 +8,7 @@
 
 #include <linux/types.h>
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 
 enum pipe;
 enum port;
@@ -20,23 +20,23 @@ struct intel_encoder;
 #ifdef I915
 const struct dpll *vlv_get_dpll(struct intel_display *display);
 bool g4x_dp_port_enabled(struct intel_display *display,
-			 i915_reg_t dp_reg, enum port port,
+			 intel_reg_t dp_reg, enum port port,
 			 enum pipe *pipe);
 bool g4x_dp_init(struct intel_display *display,
-		 i915_reg_t output_reg, enum port port);
+		 intel_reg_t output_reg, enum port port);
 #else
 static inline const struct dpll *vlv_get_dpll(struct intel_display *display)
 {
 	return NULL;
 }
 static inline bool g4x_dp_port_enabled(struct intel_display *display,
-				       i915_reg_t dp_reg, int port,
+				       intel_reg_t dp_reg, int port,
 				       enum pipe *pipe)
 {
 	return false;
 }
 static inline bool g4x_dp_init(struct intel_display *display,
-			       i915_reg_t output_reg, int port)
+			       intel_reg_t output_reg, int port)
 {
 	return false;
 }
diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.c b/drivers/gpu/drm/i915/display/g4x_hdmi.c
index 5fe5067c4237c..acb36cab999c5 100644
--- a/drivers/gpu/drm/i915/display/g4x_hdmi.c
+++ b/drivers/gpu/drm/i915/display/g4x_hdmi.c
@@ -592,7 +592,7 @@ intel_hdmi_hotplug(struct intel_encoder *encoder,
 }
 
 int g4x_hdmi_connector_atomic_check(struct drm_connector *connector,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct intel_display *display = to_intel_display(connector->dev);
 	struct drm_connector_list_iter conn_iter;
@@ -666,7 +666,7 @@ static bool assert_hdmi_port_valid(struct intel_display *display, enum port port
 }
 
 bool g4x_hdmi_init(struct intel_display *display,
-		   i915_reg_t hdmi_reg, enum port port)
+		   intel_reg_t hdmi_reg, enum port port)
 {
 	const struct intel_bios_encoder_data *devdata;
 	struct intel_digital_port *dig_port;
diff --git a/drivers/gpu/drm/i915/display/g4x_hdmi.h b/drivers/gpu/drm/i915/display/g4x_hdmi.h
index 039d2bdba06c1..7586b1a6e0de3 100644
--- a/drivers/gpu/drm/i915/display/g4x_hdmi.h
+++ b/drivers/gpu/drm/i915/display/g4x_hdmi.h
@@ -8,26 +8,26 @@
 
 #include <linux/types.h>
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 
 enum port;
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_connector;
 struct intel_display;
 
 #ifdef I915
 bool g4x_hdmi_init(struct intel_display *display,
-		   i915_reg_t hdmi_reg, enum port port);
+		   intel_reg_t hdmi_reg, enum port port);
 int g4x_hdmi_connector_atomic_check(struct drm_connector *connector,
-				    struct drm_atomic_state *state);
+				    struct drm_atomic_commit *state);
 #else
 static inline bool g4x_hdmi_init(struct intel_display *display,
-				 i915_reg_t hdmi_reg, int port)
+				 intel_reg_t hdmi_reg, int port)
 {
 	return false;
 }
 static inline int g4x_hdmi_connector_atomic_check(struct drm_connector *connector,
-						  struct drm_atomic_state *state)
+						  struct drm_atomic_commit *state)
 {
 	return 0;
 }
diff --git a/drivers/gpu/drm/i915/display/i9xx_display_sr.c b/drivers/gpu/drm/i915/display/i9xx_display_sr.c
index 935419441709d..1eb2f636cc650 100644
--- a/drivers/gpu/drm/i915/display/i9xx_display_sr.c
+++ b/drivers/gpu/drm/i915/display/i9xx_display_sr.c
@@ -4,13 +4,13 @@
  */
 
 #include <drm/drm_device.h>
+#include <drm/intel/pci_config.h>
 
 #include "i9xx_display_sr.h"
 #include "i9xx_wm_regs.h"
 #include "intel_de.h"
 #include "intel_display_regs.h"
 #include "intel_gmbus.h"
-#include "intel_pci_config.h"
 
 static void i9xx_display_save_swf(struct intel_display *display)
 {
diff --git a/drivers/gpu/drm/i915/display/i9xx_plane.c b/drivers/gpu/drm/i915/display/i9xx_plane.c
index 9c16753a1f3ba..70734d32a4090 100644
--- a/drivers/gpu/drm/i915/display/i9xx_plane.c
+++ b/drivers/gpu/drm/i915/display/i9xx_plane.c
@@ -1240,7 +1240,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
 		    fb->width, fb->height, fb->format->cpp[0] * 8,
 		    base, fb->pitches[0], plane_config->size);
 
-	plane_config->fb = intel_fb;
+	plane_config->fb = &intel_fb->base;
 }
 
 bool i9xx_fixup_initial_plane_config(struct intel_crtc *crtc,
diff --git a/drivers/gpu/drm/i915/display/i9xx_wm.c b/drivers/gpu/drm/i915/display/i9xx_wm.c
index 9e170e79dcf67..33d8f6b6afeaf 100644
--- a/drivers/gpu/drm/i915/display/i9xx_wm.c
+++ b/drivers/gpu/drm/i915/display/i9xx_wm.c
@@ -19,7 +19,7 @@
 #include "intel_display_utils.h"
 #include "intel_dram.h"
 #include "intel_fb.h"
-#include "intel_mchbar_regs.h"
+#include "intel_mchbar.h"
 #include "intel_wm.h"
 #include "skl_watermark.h"
 #include "vlv_sideband.h"
@@ -118,41 +118,41 @@ static void chv_set_memory_dvfs(struct intel_display *display, bool enable)
 	u32 val;
 	int ret;
 
-	vlv_punit_get(display->drm);
+	vlv_punit_get(display);
 
-	val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2);
+	val = vlv_punit_read(display, PUNIT_REG_DDR_SETUP2);
 	if (enable)
 		val &= ~FORCE_DDR_HIGH_FREQ;
 	else
 		val |= FORCE_DDR_HIGH_FREQ;
 	val &= ~FORCE_DDR_LOW_FREQ;
 	val |= FORCE_DDR_FREQ_REQ_ACK;
-	vlv_punit_write(display->drm, PUNIT_REG_DDR_SETUP2, val);
+	vlv_punit_write(display, PUNIT_REG_DDR_SETUP2, val);
 
-	ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2),
+	ret = poll_timeout_us(val = vlv_punit_read(display, PUNIT_REG_DDR_SETUP2),
 			      (val & FORCE_DDR_FREQ_REQ_ACK) == 0,
 			      500, 3000, false);
 	if (ret)
 		drm_err(display->drm,
 			"timed out waiting for Punit DDR DVFS request\n");
 
-	vlv_punit_put(display->drm);
+	vlv_punit_put(display);
 }
 
 static void chv_set_memory_pm5(struct intel_display *display, bool enable)
 {
 	u32 val;
 
-	vlv_punit_get(display->drm);
+	vlv_punit_get(display);
 
-	val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM);
+	val = vlv_punit_read(display, PUNIT_REG_DSPSSPM);
 	if (enable)
 		val |= DSP_MAXFIFO_PM5_ENABLE;
 	else
 		val &= ~DSP_MAXFIFO_PM5_ENABLE;
-	vlv_punit_write(display->drm, PUNIT_REG_DSPSSPM, val);
+	vlv_punit_write(display, PUNIT_REG_DSPSSPM, val);
 
-	vlv_punit_put(display->drm);
+	vlv_punit_put(display);
 }
 
 #define FW_WM(value, plane) \
@@ -2742,12 +2742,11 @@ static void ilk_compute_wm_level(struct intel_display *display,
 
 static void hsw_read_wm_latency(struct intel_display *display, u16 wm[])
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u64 sskpd;
 
 	display->wm.num_levels = 5;
 
-	sskpd = intel_uncore_read64(uncore, MCH_SSKPD);
+	sskpd = intel_mchbar_read64_2x32(display, MCH_SSKPD);
 
 	wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
 	if (wm[0] == 0)
@@ -2760,12 +2759,11 @@ static void hsw_read_wm_latency(struct intel_display *display, u16 wm[])
 
 static void snb_read_wm_latency(struct intel_display *display, u16 wm[])
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u32 sskpd;
 
 	display->wm.num_levels = 4;
 
-	sskpd = intel_uncore_read(uncore, MCH_SSKPD);
+	sskpd = intel_mchbar_read(display, MCH_SSKPD);
 
 	wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
 	wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
@@ -2775,12 +2773,11 @@ static void snb_read_wm_latency(struct intel_display *display, u16 wm[])
 
 static void ilk_read_wm_latency(struct intel_display *display, u16 wm[])
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u32 mltr;
 
 	display->wm.num_levels = 3;
 
-	mltr = intel_uncore_read(uncore, MLTR_ILK);
+	mltr = intel_mchbar_read(display, MLTR_ILK);
 
 	/* ILK primary LP0 latency is 700 ns */
 	wm[0] = 7;
@@ -3534,7 +3531,7 @@ static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
 	crtc->wm.active.ilk = *active;
 }
 
-static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
+static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_commit *state)
 {
 	struct drm_plane *plane;
 	struct intel_crtc *crtc;
@@ -3578,7 +3575,7 @@ static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
  */
 void ilk_wm_sanitize(struct intel_display *display)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct intel_atomic_state *intel_state;
 	struct intel_crtc *crtc;
 	struct intel_crtc_state *crtc_state;
@@ -3587,13 +3584,13 @@ void ilk_wm_sanitize(struct intel_display *display)
 	int i;
 
 	/* Only supported on platforms that use atomic watermark design */
-	if (!display->funcs.wm->optimize_watermarks)
+	if (!display->wm.funcs->optimize_watermarks)
 		return;
 
 	if (drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 9))
 		return;
 
-	state = drm_atomic_state_alloc(display->drm);
+	state = drm_atomic_commit_alloc(display->drm);
 	if (drm_WARN_ON(display->drm, !state))
 		return;
 
@@ -3631,7 +3628,7 @@ void ilk_wm_sanitize(struct intel_display *display)
 
 fail:
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 		goto retry;
 	}
@@ -3650,7 +3647,7 @@ void ilk_wm_sanitize(struct intel_display *display)
 	drm_WARN(display->drm, ret,
 		 "Could not determine valid watermarks for inherited state\n");
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
@@ -3918,9 +3915,9 @@ static void vlv_wm_get_hw_state(struct intel_display *display)
 	wm->level = VLV_WM_LEVEL_PM2;
 
 	if (display->platform.cherryview) {
-		vlv_punit_get(display->drm);
+		vlv_punit_get(display);
 
-		val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM);
+		val = vlv_punit_read(display, PUNIT_REG_DSPSSPM);
 		if (val & DSP_MAXFIFO_PM5_ENABLE)
 			wm->level = VLV_WM_LEVEL_PM5;
 
@@ -3933,11 +3930,11 @@ static void vlv_wm_get_hw_state(struct intel_display *display)
 		 * HIGH/LOW bits so that we don't actually change
 		 * the current state.
 		 */
-		val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2);
+		val = vlv_punit_read(display, PUNIT_REG_DDR_SETUP2);
 		val |= FORCE_DDR_FREQ_REQ_ACK;
-		vlv_punit_write(display->drm, PUNIT_REG_DDR_SETUP2, val);
+		vlv_punit_write(display, PUNIT_REG_DDR_SETUP2, val);
 
-		ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2),
+		ret = poll_timeout_us(val = vlv_punit_read(display, PUNIT_REG_DDR_SETUP2),
 				      (val & FORCE_DDR_FREQ_REQ_ACK) == 0,
 				      500, 3000, false);
 		if (ret) {
@@ -3946,12 +3943,12 @@ static void vlv_wm_get_hw_state(struct intel_display *display)
 				    "assuming DDR DVFS is disabled\n");
 			display->wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
 		} else {
-			val = vlv_punit_read(display->drm, PUNIT_REG_DDR_SETUP2);
+			val = vlv_punit_read(display, PUNIT_REG_DDR_SETUP2);
 			if ((val & FORCE_DDR_HIGH_FREQ) == 0)
 				wm->level = VLV_WM_LEVEL_DDR_DVFS;
 		}
 
-		vlv_punit_put(display->drm);
+		vlv_punit_put(display);
 	}
 
 	for_each_intel_crtc(display->drm, crtc) {
@@ -4155,34 +4152,34 @@ void i9xx_wm_init(struct intel_display *display)
 	/* For FIFO watermark updates */
 	if (HAS_PCH_SPLIT(display)) {
 		ilk_setup_wm_latency(display);
-		display->funcs.wm = &ilk_wm_funcs;
+		display->wm.funcs = &ilk_wm_funcs;
 	} else if (display->platform.valleyview || display->platform.cherryview) {
 		vlv_setup_wm_latency(display);
-		display->funcs.wm = &vlv_wm_funcs;
+		display->wm.funcs = &vlv_wm_funcs;
 	} else if (display->platform.g4x) {
 		g4x_setup_wm_latency(display);
-		display->funcs.wm = &g4x_wm_funcs;
+		display->wm.funcs = &g4x_wm_funcs;
 	} else if (display->platform.pineview) {
 		if (!pnv_get_cxsr_latency(display)) {
 			drm_info(display->drm, "Unknown FSB/MEM, disabling CxSR\n");
 			/* Disable CxSR and never update its watermark again */
 			intel_set_memory_cxsr(display, false);
-			display->funcs.wm = &nop_funcs;
+			display->wm.funcs = &nop_funcs;
 		} else {
-			display->funcs.wm = &pnv_wm_funcs;
+			display->wm.funcs = &pnv_wm_funcs;
 		}
 	} else if (DISPLAY_VER(display) == 4) {
-		display->funcs.wm = &i965_wm_funcs;
+		display->wm.funcs = &i965_wm_funcs;
 	} else if (DISPLAY_VER(display) == 3) {
-		display->funcs.wm = &i9xx_wm_funcs;
+		display->wm.funcs = &i9xx_wm_funcs;
 	} else if (DISPLAY_VER(display) == 2) {
 		if (INTEL_NUM_PIPES(display) == 1)
-			display->funcs.wm = &i845_wm_funcs;
+			display->wm.funcs = &i845_wm_funcs;
 		else
-			display->funcs.wm = &i9xx_wm_funcs;
+			display->wm.funcs = &i9xx_wm_funcs;
 	} else {
 		drm_err(display->drm,
 			"unexpected fall-through in %s\n", __func__);
-		display->funcs.wm = &nop_funcs;
+		display->wm.funcs = &nop_funcs;
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/icl_dsi.c b/drivers/gpu/drm/i915/display/icl_dsi.c
index afbaa0465842a..a549f1fac8103 100644
--- a/drivers/gpu/drm/i915/display/icl_dsi.c
+++ b/drivers/gpu/drm/i915/display/icl_dsi.c
@@ -296,7 +296,7 @@ static void configure_dual_link_mode(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
-	i915_reg_t dss_ctl1_reg, dss_ctl2_reg;
+	intel_reg_t dss_ctl1_reg, dss_ctl2_reg;
 	u32 dss_ctl1;
 
 	/* FIXME: Move all DSS handling to intel_vdsc.c */
diff --git a/drivers/gpu/drm/i915/display/intel_atomic.c b/drivers/gpu/drm/i915/display/intel_atomic.c
index 71b7325917b6e..38bbd6964d8ee 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic.c
+++ b/drivers/gpu/drm/i915/display/intel_atomic.c
@@ -112,7 +112,7 @@ int intel_digital_connector_atomic_set_property(struct drm_connector *connector,
 }
 
 int intel_digital_connector_atomic_check(struct drm_connector *conn,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *new_state =
 		drm_atomic_get_new_connector_state(state, conn);
@@ -320,12 +320,12 @@ intel_crtc_destroy_state(struct drm_crtc *crtc,
 	kfree(crtc_state);
 }
 
-struct drm_atomic_state *
+struct drm_atomic_commit *
 intel_atomic_state_alloc(struct drm_device *dev)
 {
 	struct intel_atomic_state *state = kzalloc_obj(*state);
 
-	if (!state || drm_atomic_state_init(dev, &state->base) < 0) {
+	if (!state || drm_atomic_commit_init(dev, &state->base) < 0) {
 		kfree(state);
 		return NULL;
 	}
@@ -333,20 +333,20 @@ intel_atomic_state_alloc(struct drm_device *dev)
 	return &state->base;
 }
 
-void intel_atomic_state_free(struct drm_atomic_state *_state)
+void intel_atomic_state_free(struct drm_atomic_commit *_state)
 {
 	struct intel_atomic_state *state = to_intel_atomic_state(_state);
 
-	drm_atomic_state_default_release(&state->base);
+	drm_atomic_commit_default_release(&state->base);
 	kfree(state->global_objs);
 	kfree(state);
 }
 
-void intel_atomic_state_clear(struct drm_atomic_state *s)
+void intel_atomic_state_clear(struct drm_atomic_commit *s)
 {
 	struct intel_atomic_state *state = to_intel_atomic_state(s);
 
-	drm_atomic_state_default_clear(&state->base);
+	drm_atomic_commit_default_clear(&state->base);
 	intel_atomic_clear_global_state(state);
 
 	/* state->internal not reset on purpose */
@@ -357,7 +357,7 @@ void intel_atomic_state_clear(struct drm_atomic_state *s)
 }
 
 struct intel_crtc_state *
-intel_atomic_get_crtc_state(struct drm_atomic_state *state,
+intel_atomic_get_crtc_state(struct drm_atomic_commit *state,
 			    struct intel_crtc *crtc)
 {
 	struct drm_crtc_state *crtc_state;
diff --git a/drivers/gpu/drm/i915/display/intel_atomic.h b/drivers/gpu/drm/i915/display/intel_atomic.h
index a5a7e2906ba89..4dc5205a47331 100644
--- a/drivers/gpu/drm/i915/display/intel_atomic.h
+++ b/drivers/gpu/drm/i915/display/intel_atomic.h
@@ -8,7 +8,7 @@
 
 #include <linux/types.h>
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_connector;
 struct drm_connector_state;
 struct drm_crtc;
@@ -29,7 +29,7 @@ int intel_digital_connector_atomic_set_property(struct drm_connector *connector,
 						struct drm_property *property,
 						u64 val);
 int intel_digital_connector_atomic_check(struct drm_connector *conn,
-					 struct drm_atomic_state *state);
+					 struct drm_atomic_commit *state);
 struct drm_connector_state *
 intel_digital_connector_duplicate_state(struct drm_connector *connector);
 bool intel_connector_needs_modeset(struct intel_atomic_state *state,
@@ -43,12 +43,12 @@ struct drm_crtc_state *intel_crtc_duplicate_state(struct drm_crtc *crtc);
 void intel_crtc_destroy_state(struct drm_crtc *crtc,
 			       struct drm_crtc_state *state);
 void intel_crtc_free_hw_state(struct intel_crtc_state *crtc_state);
-struct drm_atomic_state *intel_atomic_state_alloc(struct drm_device *dev);
-void intel_atomic_state_free(struct drm_atomic_state *state);
-void intel_atomic_state_clear(struct drm_atomic_state *state);
+struct drm_atomic_commit *intel_atomic_state_alloc(struct drm_device *dev);
+void intel_atomic_state_free(struct drm_atomic_commit *state);
+void intel_atomic_state_clear(struct drm_atomic_commit *state);
 
 struct intel_crtc_state *
-intel_atomic_get_crtc_state(struct drm_atomic_state *state,
+intel_atomic_get_crtc_state(struct drm_atomic_commit *state,
 			    struct intel_crtc *crtc);
 
 #endif /* __INTEL_ATOMIC_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_audio.c b/drivers/gpu/drm/i915/display/intel_audio.c
index 081627e0d9171..9729f1837d2c1 100644
--- a/drivers/gpu/drm/i915/display/intel_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_audio.c
@@ -595,7 +595,7 @@ static void hsw_audio_codec_enable(struct intel_encoder *encoder,
 }
 
 struct ibx_audio_regs {
-	i915_reg_t hdmiw_hdmiedid, aud_config, aud_cntl_st, aud_cntrl_st2;
+	intel_reg_t hdmiw_hdmiedid, aud_config, aud_cntl_st, aud_cntrl_st2;
 };
 
 static void ibx_audio_regs_init(struct intel_display *display,
@@ -754,10 +754,8 @@ void intel_audio_codec_enable(struct intel_encoder *encoder,
 		    crtc->base.base.id, crtc->base.name,
 		    drm_eld_size(crtc_state->eld));
 
-	if (display->funcs.audio)
-		display->funcs.audio->audio_codec_enable(encoder,
-							      crtc_state,
-							      conn_state);
+	if (display->audio.funcs)
+		display->audio.funcs->audio_codec_enable(encoder, crtc_state, conn_state);
 
 	mutex_lock(&display->audio.mutex);
 
@@ -813,10 +811,8 @@ void intel_audio_codec_disable(struct intel_encoder *encoder,
 		    encoder->base.base.id, encoder->base.name,
 		    crtc->base.base.id, crtc->base.name);
 
-	if (display->funcs.audio)
-		display->funcs.audio->audio_codec_disable(encoder,
-							       old_crtc_state,
-							       old_conn_state);
+	if (display->audio.funcs)
+		display->audio.funcs->audio_codec_disable(encoder, old_crtc_state, old_conn_state);
 
 	mutex_lock(&display->audio.mutex);
 
@@ -864,8 +860,8 @@ void intel_audio_codec_get_config(struct intel_encoder *encoder,
 	if (!crtc_state->has_audio)
 		return;
 
-	if (display->funcs.audio)
-		display->funcs.audio->audio_codec_get_config(encoder, crtc_state);
+	if (display->audio.funcs)
+		display->audio.funcs->audio_codec_get_config(encoder, crtc_state);
 }
 
 static const struct intel_audio_funcs g4x_audio_funcs = {
@@ -893,12 +889,12 @@ static const struct intel_audio_funcs hsw_audio_funcs = {
 void intel_audio_hooks_init(struct intel_display *display)
 {
 	if (display->platform.g4x)
-		display->funcs.audio = &g4x_audio_funcs;
+		display->audio.funcs = &g4x_audio_funcs;
 	else if (display->platform.valleyview || display->platform.cherryview ||
 		 HAS_PCH_CPT(display) || HAS_PCH_IBX(display))
-		display->funcs.audio = &ibx_audio_funcs;
+		display->audio.funcs = &ibx_audio_funcs;
 	else if (display->platform.haswell || DISPLAY_VER(display) >= 8)
-		display->funcs.audio = &hsw_audio_funcs;
+		display->audio.funcs = &hsw_audio_funcs;
 }
 
 struct aud_ts_cdclk_m_n {
@@ -958,7 +954,7 @@ static void glk_force_audio_cdclk(struct intel_display *display,
 				  bool enable)
 {
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct intel_crtc *crtc;
 	int ret;
 
@@ -967,7 +963,7 @@ static void glk_force_audio_cdclk(struct intel_display *display,
 		return;
 
 	drm_modeset_acquire_init(&ctx, 0);
-	state = drm_atomic_state_alloc(display->drm);
+	state = drm_atomic_commit_alloc(display->drm);
 	if (drm_WARN_ON(display->drm, !state))
 		return;
 
@@ -978,14 +974,14 @@ static void glk_force_audio_cdclk(struct intel_display *display,
 	ret = glk_force_audio_cdclk_commit(to_intel_atomic_state(state), crtc,
 					   enable);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 		goto retry;
 	}
 
 	drm_WARN_ON(display->drm, ret);
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
diff --git a/drivers/gpu/drm/i915/display/intel_backlight.c b/drivers/gpu/drm/i915/display/intel_backlight.c
index 34e95f05936e5..b128896cb1c2d 100644
--- a/drivers/gpu/drm/i915/display/intel_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_backlight.c
@@ -11,6 +11,7 @@
 
 #include <drm/drm_file.h>
 #include <drm/drm_print.h>
+#include <drm/intel/pci_config.h>
 
 #include "intel_backlight.h"
 #include "intel_backlight_regs.h"
@@ -23,7 +24,6 @@
 #include "intel_dp_aux_backlight.h"
 #include "intel_dsi_dcs_backlight.h"
 #include "intel_panel.h"
-#include "intel_pci_config.h"
 #include "intel_pps.h"
 #include "intel_quirks.h"
 
diff --git a/drivers/gpu/drm/i915/display/intel_bo.c b/drivers/gpu/drm/i915/display/intel_bo.c
index 3b82d38a0504f..8ecdbb7e39f3d 100644
--- a/drivers/gpu/drm/i915/display/intel_bo.c
+++ b/drivers/gpu/drm/i915/display/intel_bo.c
@@ -85,3 +85,30 @@ struct drm_gem_object *intel_bo_framebuffer_lookup(struct intel_display *display
 {
 	return display->parent->bo->framebuffer_lookup(display->drm, filp, user_mode_cmd);
 }
+
+#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
+u32 intel_bo_fbdev_pitch_align(struct intel_display *display, u32 stride)
+{
+	return display->parent->bo->fbdev_pitch_align(stride);
+}
+
+struct drm_gem_object *intel_bo_fbdev_create(struct intel_display *display, int size)
+{
+	return display->parent->bo->fbdev_create(display->drm, size);
+}
+
+void intel_bo_fbdev_destroy(struct drm_gem_object *obj)
+{
+	struct intel_display *display = to_intel_display(obj->dev);
+
+	display->parent->bo->fbdev_destroy(obj);
+}
+
+int intel_bo_fbdev_fill_info(struct drm_gem_object *obj, struct fb_info *info,
+			     struct i915_vma *vma)
+{
+	struct intel_display *display = to_intel_display(obj->dev);
+
+	return display->parent->bo->fbdev_fill_info(obj, info, vma);
+}
+#endif
diff --git a/drivers/gpu/drm/i915/display/intel_bo.h b/drivers/gpu/drm/i915/display/intel_bo.h
index aec188c706c27..348f7fa669608 100644
--- a/drivers/gpu/drm/i915/display/intel_bo.h
+++ b/drivers/gpu/drm/i915/display/intel_bo.h
@@ -10,6 +10,8 @@ struct drm_file;
 struct drm_gem_object;
 struct drm_mode_fb_cmd2;
 struct drm_scanout_buffer;
+struct fb_info;
+struct i915_vma;
 struct intel_display;
 struct intel_framebuffer;
 struct seq_file;
@@ -31,4 +33,10 @@ struct drm_gem_object *intel_bo_framebuffer_lookup(struct intel_display *display
 						   struct drm_file *filp,
 						   const struct drm_mode_fb_cmd2 *user_mode_cmd);
 
+u32 intel_bo_fbdev_pitch_align(struct intel_display *display, u32 stride);
+struct drm_gem_object *intel_bo_fbdev_create(struct intel_display *display, int size);
+void intel_bo_fbdev_destroy(struct drm_gem_object *obj);
+int intel_bo_fbdev_fill_info(struct drm_gem_object *obj, struct fb_info *info,
+			     struct i915_vma *vma);
+
 #endif /* __INTEL_BO__ */
diff --git a/drivers/gpu/drm/i915/display/intel_bw.c b/drivers/gpu/drm/i915/display/intel_bw.c
index 07b4531a43760..d7b2bc80f8e35 100644
--- a/drivers/gpu/drm/i915/display/intel_bw.c
+++ b/drivers/gpu/drm/i915/display/intel_bw.c
@@ -9,14 +9,14 @@
 
 #include "intel_bw.h"
 #include "intel_crtc.h"
+#include "intel_de.h"
 #include "intel_display_core.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_display_utils.h"
 #include "intel_dram.h"
-#include "intel_mchbar_regs.h"
+#include "intel_mchbar.h"
 #include "intel_parent.h"
-#include "intel_uncore.h"
 #include "skl_watermark.h"
 
 struct intel_bw_state {
@@ -28,9 +28,6 @@ struct intel_bw_state {
 	 */
 	u8 pipe_sagv_reject;
 
-	/* bitmask of active pipes */
-	u8 active_pipes;
-
 	/*
 	 * From MTL onwards, to lock a QGV point, punit expects the peak BW of
 	 * the selected QGV point as the parameter in multiples of 100MB/s
@@ -74,11 +71,10 @@ static int dg1_mchbar_read_qgv_point_info(struct intel_display *display,
 					  struct intel_qgv_point *sp,
 					  int point)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u32 dclk_ratio, dclk_reference;
 	u32 val;
 
-	val = intel_uncore_read(uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
+	val = intel_mchbar_read(display, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
 	dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
 	if (val & DG1_QCLK_REFERENCE)
 		dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
@@ -86,18 +82,18 @@ static int dg1_mchbar_read_qgv_point_info(struct intel_display *display,
 		dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
 	sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
 
-	val = intel_uncore_read(uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
+	val = intel_mchbar_read(display, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
 	if (val & DG1_GEAR_TYPE)
 		sp->dclk *= 2;
 
 	if (sp->dclk == 0)
 		return -EINVAL;
 
-	val = intel_uncore_read(uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
+	val = intel_mchbar_read(display, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
 	sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
 	sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
 
-	val = intel_uncore_read(uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
+	val = intel_mchbar_read(display, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
 	sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
 	sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
 
@@ -211,12 +207,11 @@ static int icl_pcode_restrict_qgv_points(struct intel_display *display,
 static int mtl_read_qgv_point_info(struct intel_display *display,
 				   struct intel_qgv_point *sp, int point)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u32 val, val2;
 	u16 dclk;
 
-	val = intel_uncore_read(uncore, MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
-	val2 = intel_uncore_read(uncore, MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
+	val = intel_de_read(display, MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
+	val2 = intel_de_read(display, MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
 	dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
 	sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000);
 	sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
@@ -377,82 +372,142 @@ static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
 	return dclk;
 }
 
-struct intel_sa_info {
-	u16 displayrtids;
-	u8 deburst, deprogbwlimit, derating;
+/*
+ * Bandwidth parameters that are tied to the SoC (as opposed to struct
+ * intel_display_bw_params).
+ */
+struct intel_soc_bw_params {
+	u8 deprogbwlimit;
+	u8 derating;
 };
 
-static const struct intel_sa_info icl_sa_info = {
-	.deburst = 8,
-	.deprogbwlimit = 25, /* GB/s */
-	.displayrtids = 128,
+static const struct intel_soc_bw_params icl_bw_params = {
+	.deprogbwlimit = 25,
 	.derating = 10,
 };
 
-static const struct intel_sa_info tgl_sa_info = {
-	.deburst = 16,
-	.deprogbwlimit = 34, /* GB/s */
-	.displayrtids = 256,
+static const struct intel_soc_bw_params tgl_bw_params = {
+	.deprogbwlimit = 34,
 	.derating = 10,
 };
 
-static const struct intel_sa_info rkl_sa_info = {
-	.deburst = 8,
-	.deprogbwlimit = 20, /* GB/s */
-	.displayrtids = 128,
+static const struct intel_soc_bw_params rkl_bw_params = {
+	.deprogbwlimit = 20,
 	.derating = 10,
 };
 
-static const struct intel_sa_info adls_sa_info = {
-	.deburst = 16,
-	.deprogbwlimit = 38, /* GB/s */
-	.displayrtids = 256,
+static const struct intel_soc_bw_params adl_s_bw_params = {
+	.deprogbwlimit = 38,
 	.derating = 10,
 };
 
-static const struct intel_sa_info adlp_sa_info = {
-	.deburst = 16,
-	.deprogbwlimit = 38, /* GB/s */
-	.displayrtids = 256,
+static const struct intel_soc_bw_params adl_p_bw_params = {
+	.deprogbwlimit = 38,
 	.derating = 20,
 };
 
-static const struct intel_sa_info mtl_sa_info = {
-	.deburst = 32,
-	.deprogbwlimit = 38, /* GB/s */
-	.displayrtids = 256,
-	.derating = 10,
+static const struct intel_soc_bw_params bmg_bw_params = {
+	.deprogbwlimit = 53,
+	.derating = 30,
 };
 
-static const struct intel_sa_info xe2_hpd_sa_info = {
-	.derating = 30,
+static const struct intel_soc_bw_params bmg_ecc_bw_params = {
 	.deprogbwlimit = 53,
-	/* Other values not used by simplified algorithm */
+	.derating = 45,
 };
 
-static const struct intel_sa_info xe2_hpd_ecc_sa_info = {
-	.derating = 45,
-	.deprogbwlimit = 53,
-	/* Other values not used by simplified algorithm */
+static const struct intel_soc_bw_params ptl_bw_params = {
+	.deprogbwlimit = 65,
+	.derating = 10,
 };
 
-static const struct intel_sa_info xe3lpd_sa_info = {
-	.deburst = 32,
-	.deprogbwlimit = 65, /* GB/s */
-	.displayrtids = 256,
+static const struct intel_soc_bw_params wcl_bw_params = {
+	.deprogbwlimit = 22,
 	.derating = 10,
 };
 
-static const struct intel_sa_info xe3lpd_3002_sa_info = {
+static const struct intel_soc_bw_params *get_soc_bw_params(struct intel_display *display,
+							   const struct dram_info *dram_info)
+{
+	if (display->platform.icelake ||
+	    display->platform.jasperlake ||
+	    display->platform.elkhartlake)
+		return &icl_bw_params;
+	else if (display->platform.tigerlake ||
+		 display->platform.dg1)
+		return &tgl_bw_params;
+	else if (display->platform.rocketlake)
+		return &rkl_bw_params;
+	else if (display->platform.alderlake_s ||
+		 display->platform.meteorlake ||
+		 display->platform.lunarlake)
+		return &adl_s_bw_params;
+	else if (display->platform.alderlake_p)
+		return &adl_p_bw_params;
+	else if (display->platform.battlemage &&
+		 dram_info->type == INTEL_DRAM_GDDR_ECC)
+		return &bmg_ecc_bw_params;
+	else if (display->platform.battlemage)
+		return &bmg_bw_params;
+	else if (display->platform.pantherlake_wildcatlake)
+		return &wcl_bw_params;
+	else if (display->platform.pantherlake ||
+		 display->platform.novalake)
+		return &ptl_bw_params;
+
+	return NULL;
+}
+
+/*
+ * Bandwidth parameters that are tied to the display IP (as opposed to struct
+ * intel_soc_bw_params).
+ */
+struct intel_display_bw_params {
+	u16 displayrtids;
+	u8 deburst;
+};
+
+static const struct intel_display_bw_params gen11_bw_params = {
+	.deburst = 8,
+	.displayrtids = 128,
+};
+
+static const struct intel_display_bw_params gen12_bw_params = {
+	.deburst = 16,
+	.displayrtids = 256,
+};
+
+static const struct intel_display_bw_params xelpdp_bw_params = {
 	.deburst = 32,
-	.deprogbwlimit = 22, /* GB/s */
 	.displayrtids = 256,
-	.derating = 10,
 };
 
+static const struct intel_display_bw_params *get_display_bw_params(struct intel_display *display)
+{
+	if (DISPLAY_VER(display) >= 14) {
+		return &xelpdp_bw_params;
+	} else if (DISPLAY_VER(display) >= 12) {
+		/*
+		 * RKL's SoC was based on ICL and the display, even though being
+		 * gen12, had changes to the memory interface to match gen11's,
+		 * consequently inheriting gen11's display-specific bandwidth
+		 * parameters.
+		 */
+		if (display->platform.rocketlake)
+			return &gen11_bw_params;
+		else
+			return &gen12_bw_params;
+	} else if (DISPLAY_VER(display) == 11) {
+		return &gen11_bw_params;
+	}
+
+	return NULL;
+}
+
 static int icl_get_bw_info(struct intel_display *display,
 			   const struct dram_info *dram_info,
-			   const struct intel_sa_info *sa)
+			   const struct intel_soc_bw_params *soc_bw_params,
+			   const struct intel_display_bw_params *display_bw_params)
 {
 	struct intel_qgv_info qi = {};
 	bool is_y_tile = true; /* assume y tile may be used */
@@ -471,8 +526,8 @@ static int icl_get_bw_info(struct intel_display *display,
 	}
 
 	dclk_max = icl_sagv_max_dclk(&qi);
-	maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
-	ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
+	maxdebw = min(soc_bw_params->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
+	ipqdepth = min(ipqdepthpch, display_bw_params->displayrtids / num_channels);
 	qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
 
 	for (i = 0; i < num_groups; i++) {
@@ -480,7 +535,7 @@ static int icl_get_bw_info(struct intel_display *display,
 		int clpchgroup;
 		int j;
 
-		clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
+		clpchgroup = (display_bw_params->deburst * qi.deinterleave / num_channels) << i;
 		bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
 
 		bi->num_qgv_points = qi.num_points;
@@ -501,7 +556,7 @@ static int icl_get_bw_info(struct intel_display *display,
 			bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 
 			bi->deratedbw[j] = min(maxdebw,
-					       bw * (100 - sa->derating) / 100);
+					       bw * (100 - soc_bw_params->derating) / 100);
 
 			drm_dbg_kms(display->drm,
 				    "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
@@ -523,7 +578,8 @@ static int icl_get_bw_info(struct intel_display *display,
 
 static int tgl_get_bw_info(struct intel_display *display,
 			   const struct dram_info *dram_info,
-			   const struct intel_sa_info *sa)
+			   const struct intel_soc_bw_params *soc_bw_params,
+			   const struct intel_display_bw_params *display_bw_params)
 {
 	struct intel_qgv_info qi = {};
 	bool is_y_tile = true; /* assume y tile may be used */
@@ -559,9 +615,9 @@ static int tgl_get_bw_info(struct intel_display *display,
 	dclk_max = icl_sagv_max_dclk(&qi);
 
 	peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
-	maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100);
+	maxdebw = min(soc_bw_params->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100);
 
-	ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
+	ipqdepth = min(ipqdepthpch, display_bw_params->displayrtids / num_channels);
 	/*
 	 * clperchgroup = 4kpagespermempage * clperchperblock,
 	 * clperchperblock = 8 / num_channels * interleave
@@ -574,7 +630,7 @@ static int tgl_get_bw_info(struct intel_display *display,
 		int clpchgroup;
 		int j;
 
-		clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
+		clpchgroup = (display_bw_params->deburst * qi.deinterleave / num_channels) << i;
 
 		if (i < num_groups - 1) {
 			bi_next = &display->bw.max[i + 1];
@@ -604,7 +660,7 @@ static int tgl_get_bw_info(struct intel_display *display,
 			bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
 
 			bi->deratedbw[j] = min(maxdebw,
-					       bw * (100 - sa->derating) / 100);
+					       bw * (100 - soc_bw_params->derating) / 100);
 			bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk *
 							  num_channels *
 							  qi.channel_width, 8);
@@ -666,7 +722,7 @@ static void dg2_get_bw_info(struct intel_display *display)
 
 static int xe2_hpd_get_bw_info(struct intel_display *display,
 			       const struct dram_info *dram_info,
-			       const struct intel_sa_info *sa)
+			       const struct intel_soc_bw_params *soc_bw_params)
 {
 	struct intel_qgv_info qi = {};
 	int num_channels = dram_info->num_channels;
@@ -681,14 +737,14 @@ static int xe2_hpd_get_bw_info(struct intel_display *display,
 	}
 
 	peakbw = num_channels * qi.channel_width / 8 * icl_sagv_max_dclk(&qi);
-	maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 10);
+	maxdebw = min(soc_bw_params->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 10);
 
 	for (i = 0; i < qi.num_points; i++) {
 		const struct intel_qgv_point *point = &qi.points[i];
 		int bw = num_channels * (qi.channel_width / 8) * point->dclk;
 
 		display->bw.max[0].deratedbw[i] =
-			min(maxdebw, (100 - sa->derating) * bw / 100);
+			min(maxdebw, (100 - soc_bw_params->derating) * bw / 100);
 		display->bw.max[0].peakbw[i] = bw;
 
 		drm_dbg_kms(display->drm, "QGV %d: deratedbw=%u peakbw: %u\n",
@@ -796,11 +852,17 @@ static unsigned int icl_qgv_bw(struct intel_display *display,
 
 void intel_bw_init_hw(struct intel_display *display)
 {
-	const struct dram_info *dram_info = intel_dram_info(display);
+	const struct dram_info *dram_info;
+	const struct intel_soc_bw_params *soc_bw_params;
+	const struct intel_display_bw_params *display_bw_params;
 
 	if (!HAS_DISPLAY(display))
 		return;
 
+	dram_info = intel_dram_info(display);
+	soc_bw_params = get_soc_bw_params(display, dram_info);
+	display_bw_params = get_display_bw_params(display);
+
 	/*
 	 * Starting with Xe3p_LPD, the hardware tells us whether memory has ECC
 	 * enabled that would impact display bandwidth.  However, so far there
@@ -810,30 +872,14 @@ void intel_bw_init_hw(struct intel_display *display)
 	if (DISPLAY_VER(display) >= 35)
 		drm_WARN_ON(display->drm, dram_info->ecc_impacting_de_bw);
 
-	if (DISPLAY_VER(display) >= 30) {
-		if (DISPLAY_VERx100(display) == 3002)
-			tgl_get_bw_info(display, dram_info, &xe3lpd_3002_sa_info);
-		else
-			tgl_get_bw_info(display, dram_info, &xe3lpd_sa_info);
-	} else if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx) {
-		if (dram_info->type == INTEL_DRAM_GDDR_ECC)
-			xe2_hpd_get_bw_info(display, dram_info, &xe2_hpd_ecc_sa_info);
-		else
-			xe2_hpd_get_bw_info(display, dram_info, &xe2_hpd_sa_info);
-	} else if (DISPLAY_VER(display) >= 14) {
-		tgl_get_bw_info(display, dram_info, &mtl_sa_info);
+	if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx) {
+		xe2_hpd_get_bw_info(display, dram_info, soc_bw_params);
 	} else if (display->platform.dg2) {
 		dg2_get_bw_info(display);
-	} else if (display->platform.alderlake_p) {
-		tgl_get_bw_info(display, dram_info, &adlp_sa_info);
-	} else if (display->platform.alderlake_s) {
-		tgl_get_bw_info(display, dram_info, &adls_sa_info);
-	} else if (display->platform.rocketlake) {
-		tgl_get_bw_info(display, dram_info, &rkl_sa_info);
-	} else if (DISPLAY_VER(display) == 12) {
-		tgl_get_bw_info(display, dram_info, &tgl_sa_info);
+	} else if (DISPLAY_VER(display) >= 12) {
+		tgl_get_bw_info(display, dram_info, soc_bw_params, display_bw_params);
 	} else if (DISPLAY_VER(display) == 11) {
-		icl_get_bw_info(display, dram_info, &icl_sa_info);
+		icl_get_bw_info(display, dram_info, soc_bw_params, display_bw_params);
 	}
 }
 
@@ -1265,31 +1311,6 @@ static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *chan
 	return 0;
 }
 
-static int intel_bw_modeset_checks(struct intel_atomic_state *state)
-{
-	const struct intel_bw_state *old_bw_state;
-	struct intel_bw_state *new_bw_state;
-	int ret;
-
-	if (!intel_any_crtc_active_changed(state))
-		return 0;
-
-	new_bw_state = intel_atomic_get_bw_state(state);
-	if (IS_ERR(new_bw_state))
-		return PTR_ERR(new_bw_state);
-
-	old_bw_state = intel_atomic_get_old_bw_state(state);
-
-	new_bw_state->active_pipes =
-		intel_calc_active_pipes(state, old_bw_state->active_pipes);
-
-	ret = intel_atomic_lock_global_state(&new_bw_state->base);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-
 static int intel_bw_check_sagv_mask(struct intel_atomic_state *state)
 {
 	struct intel_display *display = to_intel_display(state);
@@ -1346,10 +1367,6 @@ int intel_bw_atomic_check(struct intel_atomic_state *state)
 	if (DISPLAY_VER(display) < 9)
 		return 0;
 
-	ret = intel_bw_modeset_checks(state);
-	if (ret)
-		return ret;
-
 	ret = intel_bw_check_sagv_mask(state);
 	if (ret)
 		return ret;
@@ -1410,7 +1427,6 @@ void intel_bw_update_hw_state(struct intel_display *display)
 	if (DISPLAY_VER(display) < 9)
 		return;
 
-	bw_state->active_pipes = 0;
 	bw_state->pipe_sagv_reject = 0;
 
 	for_each_intel_crtc(display->drm, crtc) {
@@ -1418,9 +1434,6 @@ void intel_bw_update_hw_state(struct intel_display *display)
 			to_intel_crtc_state(crtc->base.state);
 		enum pipe pipe = crtc->pipe;
 
-		if (crtc_state->hw.active)
-			bw_state->active_pipes |= BIT(pipe);
-
 		if (DISPLAY_VER(display) >= 11)
 			intel_bw_crtc_update(bw_state, crtc_state);
 
@@ -1504,10 +1517,6 @@ bool intel_bw_pmdemand_needs_update(struct intel_atomic_state *state)
 bool intel_bw_can_enable_sagv(struct intel_display *display,
 			      const struct intel_bw_state *bw_state)
 {
-	if (DISPLAY_VER(display) < 11 &&
-	    bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
-		return false;
-
 	return bw_state->pipe_sagv_reject == 0;
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_casf.c b/drivers/gpu/drm/i915/display/intel_casf.c
index b167af31de5b0..cafd21a036320 100644
--- a/drivers/gpu/drm/i915/display/intel_casf.c
+++ b/drivers/gpu/drm/i915/display/intel_casf.c
@@ -61,10 +61,10 @@ const u16 filtercoeff_3[] = {
 	FILTER_COEFF_0_125,
 };
 
-static void intel_casf_filter_lut_load(struct intel_crtc *crtc,
-				       const struct intel_crtc_state *crtc_state)
+static void intel_casf_filter_lut_load(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	int i;
 
 	intel_de_write(display, SHRPLUT_INDEX(crtc->pipe),
@@ -75,45 +75,27 @@ static void intel_casf_filter_lut_load(struct intel_crtc *crtc,
 			       sharpness_lut[i]);
 }
 
-void intel_casf_update_strength(struct intel_crtc_state *crtc_state)
-{
-	struct intel_display *display = to_intel_display(crtc_state);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	int win_size;
-
-	intel_de_rmw(display, SHARPNESS_CTL(crtc->pipe), FILTER_STRENGTH_MASK,
-		     FILTER_STRENGTH(crtc_state->hw.casf_params.strength));
-
-	win_size = intel_de_read(display, SKL_PS_WIN_SZ(crtc->pipe, 1));
-
-	intel_de_write_fw(display, SKL_PS_WIN_SZ(crtc->pipe, 1), win_size);
-}
-
 static void intel_casf_compute_win_size(struct intel_crtc_state *crtc_state)
 {
 	const struct drm_display_mode *mode = &crtc_state->hw.adjusted_mode;
 	u32 total_pixels = mode->hdisplay * mode->vdisplay;
 
 	if (total_pixels <= MAX_PIXELS_FOR_3_TAP_FILTER)
-		crtc_state->hw.casf_params.win_size = SHARPNESS_FILTER_SIZE_3X3;
+		crtc_state->pch_pfit.casf.win_size = SHARPNESS_FILTER_SIZE_3X3;
 	else if (total_pixels <= MAX_PIXELS_FOR_5_TAP_FILTER)
-		crtc_state->hw.casf_params.win_size = SHARPNESS_FILTER_SIZE_5X5;
+		crtc_state->pch_pfit.casf.win_size = SHARPNESS_FILTER_SIZE_5X5;
 	else
-		crtc_state->hw.casf_params.win_size = SHARPNESS_FILTER_SIZE_7X7;
+		crtc_state->pch_pfit.casf.win_size = SHARPNESS_FILTER_SIZE_7X7;
 }
 
+static void intel_casf_scaler_compute_coeff(struct intel_crtc_state *crtc_state);
+
 int intel_casf_compute_config(struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 
-	if (!HAS_CASF(display))
-		return 0;
-
-	if (crtc_state->uapi.sharpness_strength == 0) {
-		crtc_state->hw.casf_params.casf_enable = false;
-		crtc_state->hw.casf_params.strength = 0;
+	if (crtc_state->hw.sharpness_strength == 0)
 		return 0;
-	}
 
 	/* CASF with joiner not supported in hardware */
 	if (crtc_state->joiner_pipes) {
@@ -121,7 +103,7 @@ int intel_casf_compute_config(struct intel_crtc_state *crtc_state)
 		return -EINVAL;
 	}
 
-	crtc_state->hw.casf_params.casf_enable = true;
+	crtc_state->pch_pfit.casf.enable = true;
 
 	/*
 	 * HW takes a value in form (1.0 + strength) in 4.4 fixed format.
@@ -131,12 +113,12 @@ int intel_casf_compute_config(struct intel_crtc_state *crtc_state)
 	 * 6.3125 in 4.4 format is b01100101 which is equal to 101.
 	 * Also 85 + 16 = 101.
 	 */
-	crtc_state->hw.casf_params.strength =
-		min(crtc_state->uapi.sharpness_strength, 0xEF) + 0x10;
+	crtc_state->pch_pfit.casf.strength =
+		min(crtc_state->hw.sharpness_strength, 0xEF) + 0x10;
 
 	intel_casf_compute_win_size(crtc_state);
 
-	intel_casf_scaler_compute_config(crtc_state);
+	intel_casf_scaler_compute_coeff(crtc_state);
 
 	return 0;
 }
@@ -151,35 +133,27 @@ void intel_casf_sharpness_get_config(struct intel_crtc_state *crtc_state)
 	if (sharp & FILTER_EN) {
 		if (drm_WARN_ON(display->drm,
 				REG_FIELD_GET(FILTER_STRENGTH_MASK, sharp) < 16))
-			crtc_state->hw.casf_params.strength = 0;
+			crtc_state->pch_pfit.casf.strength = 0;
 		else
-			crtc_state->hw.casf_params.strength =
+			crtc_state->pch_pfit.casf.strength =
 				REG_FIELD_GET(FILTER_STRENGTH_MASK, sharp);
-		crtc_state->hw.casf_params.casf_enable = true;
-		crtc_state->hw.casf_params.win_size =
+		crtc_state->pch_pfit.casf.enable = true;
+		crtc_state->pch_pfit.casf.win_size =
 			REG_FIELD_GET(FILTER_SIZE_MASK, sharp);
 	}
 }
 
-bool intel_casf_needs_scaler(const struct intel_crtc_state *crtc_state)
-{
-	if (crtc_state->hw.casf_params.casf_enable)
-		return true;
-
-	return false;
-}
-
 static int casf_coeff_tap(int i)
 {
 	return i % SCALER_FILTER_NUM_TAPS;
 }
 
-static u32 casf_coeff(struct intel_crtc_state *crtc_state, int t)
+static u32 casf_coeff(const struct intel_crtc_state *crtc_state, int tap)
 {
 	struct scaler_filter_coeff value;
 	u32 coeff;
 
-	value = crtc_state->hw.casf_params.coeff[t];
+	value = crtc_state->pch_pfit.casf.coeff[tap];
 	value.sign = 0;
 
 	coeff = value.sign << 15 | value.exp << 12 | value.mantissa << 3;
@@ -188,11 +162,11 @@ static u32 casf_coeff(struct intel_crtc_state *crtc_state, int t)
 
 /*
  * 17 phase of 7 taps requires 119 coefficients in 60 dwords per set.
- * To enable casf:  program scaler coefficients with the coeffients
- * that are calculated and stored in hw.casf_params.coeff as per
+ * To enable casf: program scaler coefficients with the coefficients
+ * that are calculated and stored in pch_pfit.casf.coeff as per
  * SCALER_COEFFICIENT_FORMAT
  */
-static void intel_casf_write_coeff(struct intel_crtc_state *crtc_state)
+static void intel_casf_write_coeff(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
@@ -209,20 +183,20 @@ static void intel_casf_write_coeff(struct intel_crtc_state *crtc_state)
 
 	for (i = 0; i < 17 * SCALER_FILTER_NUM_TAPS; i += 2) {
 		u32 tmp;
-		int t;
+		int tap;
 
-		t = casf_coeff_tap(i);
-		tmp = casf_coeff(crtc_state, t);
+		tap = casf_coeff_tap(i);
+		tmp = casf_coeff(crtc_state, tap);
 
-		t = casf_coeff_tap(i + 1);
-		tmp |= casf_coeff(crtc_state, t) << 16;
+		tap = casf_coeff_tap(i + 1);
+		tmp |= casf_coeff(crtc_state, tap) << 16;
 
 		intel_de_write_fw(display, GLK_PS_COEF_DATA_SET(crtc->pipe, id, 0),
 				  tmp);
 	}
 }
 
-static void convert_sharpness_coef_binary(struct scaler_filter_coeff *coeff,
+static void convert_sharpness_coeff_binary(struct scaler_filter_coeff *coeff,
 					  u16 coefficient)
 {
 	if (coefficient < 25) {
@@ -240,56 +214,32 @@ static void convert_sharpness_coef_binary(struct scaler_filter_coeff *coeff,
 	}
 }
 
-void intel_casf_scaler_compute_config(struct intel_crtc_state *crtc_state)
+static void intel_casf_scaler_compute_coeff(struct intel_crtc_state *crtc_state)
 {
 	const u16 *filtercoeff;
 	u16 filter_coeff[SCALER_FILTER_NUM_TAPS];
-	u16 sumcoeff = 0;
+	u16 sum_coeff = 0;
 	int i;
 
-	if (crtc_state->hw.casf_params.win_size == 0)
+	if (crtc_state->pch_pfit.casf.win_size == 0)
 		filtercoeff = filtercoeff_1;
-	else if (crtc_state->hw.casf_params.win_size == 1)
+	else if (crtc_state->pch_pfit.casf.win_size == 1)
 		filtercoeff = filtercoeff_2;
 	else
 		filtercoeff = filtercoeff_3;
 
 	for (i = 0; i < SCALER_FILTER_NUM_TAPS; i++)
-		sumcoeff += *(filtercoeff + i);
+		sum_coeff += *(filtercoeff + i);
 
 	for (i = 0; i < SCALER_FILTER_NUM_TAPS; i++) {
-		filter_coeff[i] = (*(filtercoeff + i) * 100 / sumcoeff);
-		convert_sharpness_coef_binary(&crtc_state->hw.casf_params.coeff[i],
+		filter_coeff[i] = (*(filtercoeff + i) * 100 / sum_coeff);
+		convert_sharpness_coeff_binary(&crtc_state->pch_pfit.casf.coeff[i],
 					      filter_coeff[i]);
 	}
 }
 
-void intel_casf_enable(struct intel_crtc_state *crtc_state)
+void intel_casf_setup(const struct intel_crtc_state *crtc_state)
 {
-	struct intel_display *display = to_intel_display(crtc_state);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	u32 sharpness_ctl;
-
-	intel_casf_filter_lut_load(crtc, crtc_state);
-
+	intel_casf_filter_lut_load(crtc_state);
 	intel_casf_write_coeff(crtc_state);
-
-	sharpness_ctl = FILTER_EN | FILTER_STRENGTH(crtc_state->hw.casf_params.strength);
-
-	sharpness_ctl |= crtc_state->hw.casf_params.win_size;
-
-	intel_de_write(display, SHARPNESS_CTL(crtc->pipe), sharpness_ctl);
-
-	skl_scaler_setup_casf(crtc_state);
-}
-
-void intel_casf_disable(const struct intel_crtc_state *crtc_state)
-{
-	struct intel_display *display = to_intel_display(crtc_state);
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-
-	intel_de_write(display, SKL_PS_CTRL(crtc->pipe, 1), 0);
-	intel_de_write(display, SKL_PS_WIN_POS(crtc->pipe, 1), 0);
-	intel_de_write(display, SHARPNESS_CTL(crtc->pipe), 0);
-	intel_de_write(display, SKL_PS_WIN_SZ(crtc->pipe, 1), 0);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_casf.h b/drivers/gpu/drm/i915/display/intel_casf.h
index b3fb0bcb3f5b7..3ebb7522af0a4 100644
--- a/drivers/gpu/drm/i915/display/intel_casf.h
+++ b/drivers/gpu/drm/i915/display/intel_casf.h
@@ -11,11 +11,7 @@
 struct intel_crtc_state;
 
 int intel_casf_compute_config(struct intel_crtc_state *crtc_state);
-void intel_casf_update_strength(struct intel_crtc_state *new_crtc_state);
 void intel_casf_sharpness_get_config(struct intel_crtc_state *crtc_state);
-void intel_casf_enable(struct intel_crtc_state *crtc_state);
-void intel_casf_disable(const struct intel_crtc_state *crtc_state);
-void intel_casf_scaler_compute_config(struct intel_crtc_state *crtc_state);
-bool intel_casf_needs_scaler(const struct intel_crtc_state *crtc_state);
+void intel_casf_setup(const struct intel_crtc_state *crtc_state);
 
 #endif /* __INTEL_CASF_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_cdclk.c b/drivers/gpu/drm/i915/display/intel_cdclk.c
index a47736613f6e8..a1bf01021d65d 100644
--- a/drivers/gpu/drm/i915/display/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/display/intel_cdclk.c
@@ -28,6 +28,8 @@
 #include <drm/drm_fixed.h>
 #include <drm/drm_print.h>
 #include <drm/intel/intel_pcode_regs.h>
+#include <drm/intel/pci_config.h>
+#include <drm/intel/step.h>
 
 #include "hsw_ips.h"
 #include "intel_atomic.h"
@@ -41,12 +43,10 @@
 #include "intel_display_utils.h"
 #include "intel_display_wa.h"
 #include "intel_dram.h"
-#include "intel_mchbar_regs.h"
+#include "intel_mchbar.h"
 #include "intel_parent.h"
-#include "intel_pci_config.h"
 #include "intel_plane.h"
 #include "intel_psr.h"
-#include "intel_step.h"
 #include "intel_vdsc.h"
 #include "skl_watermark.h"
 #include "skl_watermark_regs.h"
@@ -170,27 +170,27 @@ struct intel_cdclk_funcs {
 void intel_cdclk_get_cdclk(struct intel_display *display,
 			   struct intel_cdclk_config *cdclk_config)
 {
-	display->funcs.cdclk->get_cdclk(display, cdclk_config);
+	display->cdclk.funcs->get_cdclk(display, cdclk_config);
 }
 
 static void intel_cdclk_set_cdclk(struct intel_display *display,
 				  const struct intel_cdclk_config *cdclk_config,
 				  enum pipe pipe)
 {
-	display->funcs.cdclk->set_cdclk(display, cdclk_config, pipe);
+	display->cdclk.funcs->set_cdclk(display, cdclk_config, pipe);
 }
 
 static int intel_cdclk_modeset_calc_cdclk(struct intel_atomic_state *state)
 {
 	struct intel_display *display = to_intel_display(state);
 
-	return display->funcs.cdclk->modeset_calc_cdclk(state);
+	return display->cdclk.funcs->modeset_calc_cdclk(state);
 }
 
 static u8 intel_cdclk_calc_voltage_level(struct intel_display *display,
 					 int cdclk)
 {
-	return display->funcs.cdclk->calc_voltage_level(cdclk);
+	return display->cdclk.funcs->calc_voltage_level(cdclk);
 }
 
 static void fixed_133mhz_get_cdclk(struct intel_display *display,
@@ -376,8 +376,8 @@ static unsigned int intel_hpll_vco(struct intel_display *display)
 	else
 		return 0;
 
-	tmp = intel_de_read(display, display->platform.pineview ||
-			    display->platform.mobile ? HPLLVCO_MOBILE : HPLLVCO);
+	tmp = intel_mchbar_read(display, display->platform.pineview ||
+				display->platform.mobile ? HPLLVCO_MOBILE : HPLLVCO);
 
 	vco = vco_table[tmp & 0x7];
 	if (vco == 0)
@@ -613,9 +613,9 @@ static void vlv_get_cdclk(struct intel_display *display,
 	cdclk_config->vco = vlv_clock_get_hpll_vco(display->drm);
 	cdclk_config->cdclk = vlv_clock_get_cdclk(display->drm);
 
-	vlv_punit_get(display->drm);
-	val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM);
-	vlv_punit_put(display->drm);
+	vlv_punit_get(display);
+	val = vlv_punit_read(display, PUNIT_REG_DSPSSPM);
+	vlv_punit_put(display);
 
 	if (display->platform.valleyview)
 		cdclk_config->voltage_level = (val & DSPFREQGUAR_MASK) >>
@@ -691,17 +691,17 @@ static void vlv_set_cdclk(struct intel_display *display,
 	 */
 	wakeref = intel_display_power_get(display, POWER_DOMAIN_DISPLAY_CORE);
 
-	vlv_iosf_sb_get(display->drm,
-			BIT(VLV_IOSF_SB_CCK) |
-			BIT(VLV_IOSF_SB_BUNIT) |
-			BIT(VLV_IOSF_SB_PUNIT));
+	intel_parent_vlv_iosf_get(display,
+				  BIT(VLV_IOSF_SB_CCK) |
+				  BIT(VLV_IOSF_SB_BUNIT) |
+				  BIT(VLV_IOSF_SB_PUNIT));
 
-	val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM);
+	val = vlv_punit_read(display, PUNIT_REG_DSPSSPM);
 	val &= ~DSPFREQGUAR_MASK;
 	val |= (cmd << DSPFREQGUAR_SHIFT);
-	vlv_punit_write(display->drm, PUNIT_REG_DSPSSPM, val);
+	vlv_punit_write(display, PUNIT_REG_DSPSSPM, val);
 
-	ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM),
+	ret = poll_timeout_us(val = vlv_punit_read(display, PUNIT_REG_DSPSSPM),
 			      (val & DSPFREQSTAT_MASK) == (cmd << DSPFREQSTAT_SHIFT),
 			      500, 50 * 1000, false);
 	if (ret)
@@ -714,12 +714,12 @@ static void vlv_set_cdclk(struct intel_display *display,
 					    cdclk) - 1;
 
 		/* adjust cdclk divider */
-		val = vlv_cck_read(display->drm, CCK_DISPLAY_CLOCK_CONTROL);
+		val = vlv_cck_read(display, CCK_DISPLAY_CLOCK_CONTROL);
 		val &= ~CCK_FREQUENCY_VALUES;
 		val |= divider;
-		vlv_cck_write(display->drm, CCK_DISPLAY_CLOCK_CONTROL, val);
+		vlv_cck_write(display, CCK_DISPLAY_CLOCK_CONTROL, val);
 
-		ret = poll_timeout_us(val = vlv_cck_read(display->drm, CCK_DISPLAY_CLOCK_CONTROL),
+		ret = poll_timeout_us(val = vlv_cck_read(display, CCK_DISPLAY_CLOCK_CONTROL),
 				      (val & CCK_FREQUENCY_STATUS) == (divider << CCK_FREQUENCY_STATUS_SHIFT),
 				      500, 50 * 1000, false);
 		if (ret)
@@ -727,7 +727,7 @@ static void vlv_set_cdclk(struct intel_display *display,
 	}
 
 	/* adjust self-refresh exit latency value */
-	val = vlv_bunit_read(display->drm, BUNIT_REG_BISOC);
+	val = vlv_bunit_read(display, BUNIT_REG_BISOC);
 	val &= ~0x7f;
 
 	/*
@@ -738,12 +738,12 @@ static void vlv_set_cdclk(struct intel_display *display,
 		val |= 4500 / 250; /* 4.5 usec */
 	else
 		val |= 3000 / 250; /* 3.0 usec */
-	vlv_bunit_write(display->drm, BUNIT_REG_BISOC, val);
+	vlv_bunit_write(display, BUNIT_REG_BISOC, val);
 
-	vlv_iosf_sb_put(display->drm,
-			BIT(VLV_IOSF_SB_CCK) |
-			BIT(VLV_IOSF_SB_BUNIT) |
-			BIT(VLV_IOSF_SB_PUNIT));
+	intel_parent_vlv_iosf_put(display,
+				  BIT(VLV_IOSF_SB_CCK) |
+				  BIT(VLV_IOSF_SB_BUNIT) |
+				  BIT(VLV_IOSF_SB_PUNIT));
 
 	intel_update_cdclk(display);
 
@@ -780,19 +780,19 @@ static void chv_set_cdclk(struct intel_display *display,
 	 */
 	wakeref = intel_display_power_get(display, POWER_DOMAIN_DISPLAY_CORE);
 
-	vlv_punit_get(display->drm);
-	val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM);
+	vlv_punit_get(display);
+	val = vlv_punit_read(display, PUNIT_REG_DSPSSPM);
 	val &= ~DSPFREQGUAR_MASK_CHV;
 	val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
-	vlv_punit_write(display->drm, PUNIT_REG_DSPSSPM, val);
+	vlv_punit_write(display, PUNIT_REG_DSPSSPM, val);
 
-	ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM),
+	ret = poll_timeout_us(val = vlv_punit_read(display, PUNIT_REG_DSPSSPM),
 			      (val & DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
 			      500, 50 * 1000, false);
 	if (ret)
 		drm_err(display->drm, "timed out waiting for CDCLK change\n");
 
-	vlv_punit_put(display->drm);
+	vlv_punit_put(display);
 
 	intel_update_cdclk(display);
 
@@ -2619,7 +2619,7 @@ static void intel_set_cdclk(struct intel_display *display,
 	if (!intel_cdclk_changed(&display->cdclk.hw, cdclk_config))
 		return;
 
-	if (drm_WARN_ON_ONCE(display->drm, !display->funcs.cdclk->set_cdclk))
+	if (drm_WARN_ON_ONCE(display->drm, !display->cdclk.funcs->set_cdclk))
 		return;
 
 	intel_cdclk_dump_config(display, cdclk_config, context);
@@ -4042,100 +4042,100 @@ static const struct intel_cdclk_funcs i830_cdclk_funcs = {
 void intel_init_cdclk_hooks(struct intel_display *display)
 {
 	if (DISPLAY_VER(display) >= 35) {
-		display->funcs.cdclk = &xe3lpd_cdclk_funcs;
+		display->cdclk.funcs = &xe3lpd_cdclk_funcs;
 		display->cdclk.table = xe3p_lpd_cdclk_table;
 	} else if (DISPLAY_VER(display) >= 30) {
-		display->funcs.cdclk = &xe3lpd_cdclk_funcs;
+		display->cdclk.funcs = &xe3lpd_cdclk_funcs;
 		display->cdclk.table = xe3lpd_cdclk_table;
 	} else if (DISPLAY_VER(display) >= 20) {
-		display->funcs.cdclk = &rplu_cdclk_funcs;
+		display->cdclk.funcs = &rplu_cdclk_funcs;
 		display->cdclk.table = xe2lpd_cdclk_table;
 	} else if (DISPLAY_VERx100(display) >= 1401) {
-		display->funcs.cdclk = &rplu_cdclk_funcs;
+		display->cdclk.funcs = &rplu_cdclk_funcs;
 		display->cdclk.table = xe2hpd_cdclk_table;
 	} else if (DISPLAY_VER(display) >= 14) {
-		display->funcs.cdclk = &rplu_cdclk_funcs;
+		display->cdclk.funcs = &rplu_cdclk_funcs;
 		display->cdclk.table = mtl_cdclk_table;
 	} else if (display->platform.dg2) {
-		display->funcs.cdclk = &tgl_cdclk_funcs;
+		display->cdclk.funcs = &tgl_cdclk_funcs;
 		display->cdclk.table = dg2_cdclk_table;
 	} else if (display->platform.alderlake_p) {
 		/* Wa_22011320316:adl-p[a0] */
 		if (intel_display_wa(display, INTEL_DISPLAY_WA_22011320316)) {
 			display->cdclk.table = adlp_a_step_cdclk_table;
-			display->funcs.cdclk = &tgl_cdclk_funcs;
+			display->cdclk.funcs = &tgl_cdclk_funcs;
 		} else if (display->platform.alderlake_p_raptorlake_u) {
 			display->cdclk.table = rplu_cdclk_table;
-			display->funcs.cdclk = &rplu_cdclk_funcs;
+			display->cdclk.funcs = &rplu_cdclk_funcs;
 		} else {
 			display->cdclk.table = adlp_cdclk_table;
-			display->funcs.cdclk = &tgl_cdclk_funcs;
+			display->cdclk.funcs = &tgl_cdclk_funcs;
 		}
 	} else if (display->platform.rocketlake) {
-		display->funcs.cdclk = &tgl_cdclk_funcs;
+		display->cdclk.funcs = &tgl_cdclk_funcs;
 		display->cdclk.table = rkl_cdclk_table;
 	} else if (DISPLAY_VER(display) >= 12) {
-		display->funcs.cdclk = &tgl_cdclk_funcs;
+		display->cdclk.funcs = &tgl_cdclk_funcs;
 		display->cdclk.table = icl_cdclk_table;
 	} else if (display->platform.jasperlake || display->platform.elkhartlake) {
-		display->funcs.cdclk = &ehl_cdclk_funcs;
+		display->cdclk.funcs = &ehl_cdclk_funcs;
 		display->cdclk.table = icl_cdclk_table;
 	} else if (DISPLAY_VER(display) >= 11) {
-		display->funcs.cdclk = &icl_cdclk_funcs;
+		display->cdclk.funcs = &icl_cdclk_funcs;
 		display->cdclk.table = icl_cdclk_table;
 	} else if (display->platform.geminilake || display->platform.broxton) {
-		display->funcs.cdclk = &bxt_cdclk_funcs;
+		display->cdclk.funcs = &bxt_cdclk_funcs;
 		if (display->platform.geminilake)
 			display->cdclk.table = glk_cdclk_table;
 		else
 			display->cdclk.table = bxt_cdclk_table;
 	} else if (DISPLAY_VER(display) == 9) {
-		display->funcs.cdclk = &skl_cdclk_funcs;
+		display->cdclk.funcs = &skl_cdclk_funcs;
 	} else if (display->platform.broadwell) {
-		display->funcs.cdclk = &bdw_cdclk_funcs;
+		display->cdclk.funcs = &bdw_cdclk_funcs;
 	} else if (display->platform.haswell) {
-		display->funcs.cdclk = &hsw_cdclk_funcs;
+		display->cdclk.funcs = &hsw_cdclk_funcs;
 	} else if (display->platform.cherryview) {
-		display->funcs.cdclk = &chv_cdclk_funcs;
+		display->cdclk.funcs = &chv_cdclk_funcs;
 	} else if (display->platform.valleyview) {
-		display->funcs.cdclk = &vlv_cdclk_funcs;
+		display->cdclk.funcs = &vlv_cdclk_funcs;
 	} else if (display->platform.sandybridge || display->platform.ivybridge) {
-		display->funcs.cdclk = &fixed_400mhz_cdclk_funcs;
+		display->cdclk.funcs = &fixed_400mhz_cdclk_funcs;
 	} else if (display->platform.ironlake) {
-		display->funcs.cdclk = &ilk_cdclk_funcs;
+		display->cdclk.funcs = &ilk_cdclk_funcs;
 	} else if (display->platform.gm45) {
-		display->funcs.cdclk = &gm45_cdclk_funcs;
+		display->cdclk.funcs = &gm45_cdclk_funcs;
 	} else if (display->platform.g45) {
-		display->funcs.cdclk = &g33_cdclk_funcs;
+		display->cdclk.funcs = &g33_cdclk_funcs;
 	} else if (display->platform.i965gm) {
-		display->funcs.cdclk = &i965gm_cdclk_funcs;
+		display->cdclk.funcs = &i965gm_cdclk_funcs;
 	} else if (display->platform.i965g) {
-		display->funcs.cdclk = &fixed_400mhz_cdclk_funcs;
+		display->cdclk.funcs = &fixed_400mhz_cdclk_funcs;
 	} else if (display->platform.pineview) {
-		display->funcs.cdclk = &pnv_cdclk_funcs;
+		display->cdclk.funcs = &pnv_cdclk_funcs;
 	} else if (display->platform.g33) {
-		display->funcs.cdclk = &g33_cdclk_funcs;
+		display->cdclk.funcs = &g33_cdclk_funcs;
 	} else if (display->platform.i945gm) {
-		display->funcs.cdclk = &i945gm_cdclk_funcs;
+		display->cdclk.funcs = &i945gm_cdclk_funcs;
 	} else if (display->platform.i945g) {
-		display->funcs.cdclk = &fixed_400mhz_cdclk_funcs;
+		display->cdclk.funcs = &fixed_400mhz_cdclk_funcs;
 	} else if (display->platform.i915gm) {
-		display->funcs.cdclk = &i915gm_cdclk_funcs;
+		display->cdclk.funcs = &i915gm_cdclk_funcs;
 	} else if (display->platform.i915g) {
-		display->funcs.cdclk = &i915g_cdclk_funcs;
+		display->cdclk.funcs = &i915g_cdclk_funcs;
 	} else if (display->platform.i865g) {
-		display->funcs.cdclk = &i865g_cdclk_funcs;
+		display->cdclk.funcs = &i865g_cdclk_funcs;
 	} else if (display->platform.i85x) {
-		display->funcs.cdclk = &i85x_cdclk_funcs;
+		display->cdclk.funcs = &i85x_cdclk_funcs;
 	} else if (display->platform.i845g) {
-		display->funcs.cdclk = &i845g_cdclk_funcs;
+		display->cdclk.funcs = &i845g_cdclk_funcs;
 	} else if (display->platform.i830) {
-		display->funcs.cdclk = &i830_cdclk_funcs;
+		display->cdclk.funcs = &i830_cdclk_funcs;
 	}
 
-	if (drm_WARN(display->drm, !display->funcs.cdclk,
+	if (drm_WARN(display->drm, !display->cdclk.funcs,
 		     "Unknown platform. Assuming i830\n"))
-		display->funcs.cdclk = &i830_cdclk_funcs;
+		display->cdclk.funcs = &i830_cdclk_funcs;
 }
 
 int intel_cdclk_logical(const struct intel_cdclk_state *cdclk_state)
diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c
index e7950655434b8..2d318e922671b 100644
--- a/drivers/gpu/drm/i915/display/intel_color.c
+++ b/drivers/gpu/drm/i915/display/intel_color.c
@@ -1348,7 +1348,7 @@ static void i965_load_luts(const struct intel_crtc_state *crtc_state)
 }
 
 static void ilk_lut_write(const struct intel_crtc_state *crtc_state,
-			  i915_reg_t reg, u32 val)
+			  intel_reg_t reg, u32 val)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 
@@ -1359,7 +1359,7 @@ static void ilk_lut_write(const struct intel_crtc_state *crtc_state,
 }
 
 static void ilk_lut_write_indexed(const struct intel_crtc_state *crtc_state,
-				  i915_reg_t reg, u32 val)
+				  intel_reg_t reg, u32 val)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 
@@ -1933,7 +1933,7 @@ void intel_color_load_luts(const struct intel_crtc_state *crtc_state)
 	if (crtc_state->dsb_color)
 		return;
 
-	display->funcs.color->load_luts(crtc_state);
+	display->color.funcs->load_luts(crtc_state);
 }
 
 void intel_color_commit_noarm(struct intel_dsb *dsb,
@@ -1941,8 +1941,8 @@ void intel_color_commit_noarm(struct intel_dsb *dsb,
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 
-	if (display->funcs.color->color_commit_noarm)
-		display->funcs.color->color_commit_noarm(dsb, crtc_state);
+	if (display->color.funcs->color_commit_noarm)
+		display->color.funcs->color_commit_noarm(dsb, crtc_state);
 }
 
 void intel_color_commit_arm(struct intel_dsb *dsb,
@@ -1950,15 +1950,15 @@ void intel_color_commit_arm(struct intel_dsb *dsb,
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 
-	display->funcs.color->color_commit_arm(dsb, crtc_state);
+	display->color.funcs->color_commit_arm(dsb, crtc_state);
 }
 
 void intel_color_post_update(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 
-	if (display->funcs.color->color_post_update)
-		display->funcs.color->color_post_update(crtc_state);
+	if (display->color.funcs->color_post_update)
+		display->color.funcs->color_post_update(crtc_state);
 }
 
 void intel_color_modeset(const struct intel_crtc_state *crtc_state)
@@ -2022,7 +2022,7 @@ void intel_color_prepare_commit(struct intel_atomic_state *state,
 	if (!intel_color_uses_dsb(crtc_state))
 		return;
 
-	display->funcs.color->load_luts(crtc_state);
+	display->color.funcs->load_luts(crtc_state);
 
 	if (crtc_state->use_dsb && intel_color_uses_chained_dsb(crtc_state)) {
 		intel_vrr_send_push(crtc_state->dsb_color, crtc_state);
@@ -2113,19 +2113,19 @@ int intel_color_check(struct intel_atomic_state *state,
 	if (!intel_crtc_needs_color_update(new_crtc_state))
 		return 0;
 
-	return display->funcs.color->color_check(state, crtc);
+	return display->color.funcs->color_check(state, crtc);
 }
 
 void intel_color_get_config(struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 
-	display->funcs.color->get_config(crtc_state);
+	display->color.funcs->get_config(crtc_state);
 
-	display->funcs.color->read_luts(crtc_state);
+	display->color.funcs->read_luts(crtc_state);
 
-	if (display->funcs.color->read_csc)
-		display->funcs.color->read_csc(crtc_state);
+	if (display->color.funcs->read_csc)
+		display->color.funcs->read_csc(crtc_state);
 }
 
 bool intel_color_lut_equal(const struct intel_crtc_state *crtc_state,
@@ -2142,7 +2142,7 @@ bool intel_color_lut_equal(const struct intel_crtc_state *crtc_state,
 	if (!is_pre_csc_lut && crtc_state->c8_planes)
 		return true;
 
-	return display->funcs.color->lut_equal(crtc_state, blob1, blob2,
+	return display->color.funcs->lut_equal(crtc_state, blob1, blob2,
 					       is_pre_csc_lut);
 }
 
@@ -4253,8 +4253,8 @@ intel_color_load_plane_csc_matrix(struct intel_dsb *dsb,
 {
 	struct intel_display *display = to_intel_display(plane_state);
 
-	if (display->funcs.color->load_plane_csc_matrix)
-		display->funcs.color->load_plane_csc_matrix(dsb, plane_state);
+	if (display->color.funcs->load_plane_csc_matrix)
+		display->color.funcs->load_plane_csc_matrix(dsb, plane_state);
 }
 
 static void
@@ -4263,8 +4263,8 @@ intel_color_load_plane_luts(struct intel_dsb *dsb,
 {
 	struct intel_display *display = to_intel_display(plane_state);
 
-	if (display->funcs.color->load_plane_luts)
-		display->funcs.color->load_plane_luts(dsb, plane_state);
+	if (display->color.funcs->load_plane_luts)
+		display->color.funcs->load_plane_luts(dsb, plane_state);
 }
 
 bool
@@ -4346,29 +4346,29 @@ void intel_color_init_hooks(struct intel_display *display)
 {
 	if (HAS_GMCH(display)) {
 		if (display->platform.cherryview)
-			display->funcs.color = &chv_color_funcs;
+			display->color.funcs = &chv_color_funcs;
 		else if (display->platform.valleyview)
-			display->funcs.color = &vlv_color_funcs;
+			display->color.funcs = &vlv_color_funcs;
 		else if (DISPLAY_VER(display) >= 4)
-			display->funcs.color = &i965_color_funcs;
+			display->color.funcs = &i965_color_funcs;
 		else
-			display->funcs.color = &i9xx_color_funcs;
+			display->color.funcs = &i9xx_color_funcs;
 	} else {
 		if (DISPLAY_VER(display) >= 12)
-			display->funcs.color = &tgl_color_funcs;
+			display->color.funcs = &tgl_color_funcs;
 		else if (DISPLAY_VER(display) == 11)
-			display->funcs.color = &icl_color_funcs;
+			display->color.funcs = &icl_color_funcs;
 		else if (DISPLAY_VER(display) == 10)
-			display->funcs.color = &glk_color_funcs;
+			display->color.funcs = &glk_color_funcs;
 		else if (DISPLAY_VER(display) == 9)
-			display->funcs.color = &skl_color_funcs;
+			display->color.funcs = &skl_color_funcs;
 		else if (DISPLAY_VER(display) == 8)
-			display->funcs.color = &bdw_color_funcs;
+			display->color.funcs = &bdw_color_funcs;
 		else if (display->platform.haswell)
-			display->funcs.color = &hsw_color_funcs;
+			display->color.funcs = &hsw_color_funcs;
 		else if (DISPLAY_VER(display) == 7)
-			display->funcs.color = &ivb_color_funcs;
+			display->color.funcs = &ivb_color_funcs;
 		else
-			display->funcs.color = &ilk_color_funcs;
+			display->color.funcs = &ilk_color_funcs;
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_combo_phy.c b/drivers/gpu/drm/i915/display/intel_combo_phy.c
index f401558ac14e7..9b4ecfbbed4f9 100644
--- a/drivers/gpu/drm/i915/display/intel_combo_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_combo_phy.c
@@ -92,7 +92,7 @@ static void icl_set_procmon_ref_values(struct intel_display *display,
 }
 
 static bool check_phy_reg(struct intel_display *display,
-			  enum phy phy, i915_reg_t reg, u32 mask,
+			  enum phy phy, intel_reg_t reg, u32 mask,
 			  u32 expected_val)
 {
 	u32 val = intel_de_read(display, reg);
diff --git a/drivers/gpu/drm/i915/display/intel_crt.c b/drivers/gpu/drm/i915/display/intel_crt.c
index 6aa6a1dd6e1b0..243e332bef57a 100644
--- a/drivers/gpu/drm/i915/display/intel_crt.c
+++ b/drivers/gpu/drm/i915/display/intel_crt.c
@@ -77,7 +77,7 @@
 struct intel_crt {
 	struct intel_encoder base;
 	bool force_hotplug_required;
-	i915_reg_t adpa_reg;
+	intel_reg_t adpa_reg;
 };
 
 static struct intel_crt *intel_encoder_to_crt(struct intel_encoder *encoder)
@@ -91,7 +91,7 @@ static struct intel_crt *intel_attached_crt(struct intel_connector *connector)
 }
 
 bool intel_crt_port_enabled(struct intel_display *display,
-			    i915_reg_t adpa_reg, enum pipe *pipe)
+			    intel_reg_t adpa_reg, enum pipe *pipe)
 {
 	u32 val;
 
@@ -850,7 +850,7 @@ intel_crt_detect(struct drm_connector *connector,
 	struct intel_display *display = to_intel_display(connector->dev);
 	struct intel_crt *crt = intel_attached_crt(to_intel_connector(connector));
 	struct intel_encoder *encoder = &crt->base;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct ref_tracker *wakeref;
 	int status;
 
@@ -1011,7 +1011,7 @@ void intel_crt_init(struct intel_display *display)
 {
 	struct intel_connector *connector;
 	struct intel_crt *crt;
-	i915_reg_t adpa_reg;
+	intel_reg_t adpa_reg;
 	u8 ddc_pin;
 	u32 adpa;
 
diff --git a/drivers/gpu/drm/i915/display/intel_crt.h b/drivers/gpu/drm/i915/display/intel_crt.h
index e0abfe96a3d29..7d0ae8d8c28d7 100644
--- a/drivers/gpu/drm/i915/display/intel_crt.h
+++ b/drivers/gpu/drm/i915/display/intel_crt.h
@@ -6,7 +6,7 @@
 #ifndef __INTEL_CRT_H__
 #define __INTEL_CRT_H__
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 
 enum pipe;
 struct drm_encoder;
@@ -14,12 +14,12 @@ struct intel_display;
 
 #ifdef I915
 bool intel_crt_port_enabled(struct intel_display *display,
-			    i915_reg_t adpa_reg, enum pipe *pipe);
+			    intel_reg_t adpa_reg, enum pipe *pipe);
 void intel_crt_init(struct intel_display *display);
 void intel_crt_reset(struct drm_encoder *encoder);
 #else
 static inline bool intel_crt_port_enabled(struct intel_display *display,
-					  i915_reg_t adpa_reg, enum pipe *pipe)
+					  intel_reg_t adpa_reg, enum pipe *pipe)
 {
 	return false;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
index 234843b8f83a4..4493483f10a9e 100644
--- a/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
+++ b/drivers/gpu/drm/i915/display/intel_crtc_state_dump.c
@@ -250,7 +250,7 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 			   str_enabled_disabled(pipe_config->has_sel_update),
 			   str_enabled_disabled(pipe_config->has_panel_replay),
 			   str_enabled_disabled(pipe_config->enable_psr2_sel_fetch));
-		drm_printf(&p, "minimum HBlank: %d\n", pipe_config->min_hblank);
+		drm_printf(&p, "minimum hblank: %d\n", pipe_config->min_hblank);
 	}
 
 	drm_printf(&p, "audio: %i, infoframes: %i, infoframes enabled: 0x%x\n",
@@ -320,19 +320,23 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 	drm_printf(&p, "pipe mode: " DRM_MODE_FMT "\n",
 		   DRM_MODE_ARG(&pipe_config->hw.pipe_mode));
 	intel_dump_crtc_timings(&p, &pipe_config->hw.pipe_mode);
-	drm_printf(&p, "port clock: %d, pipe src: " DRM_RECT_FMT ", pixel rate %d, min cdclk %d\n",
-		   pipe_config->port_clock, DRM_RECT_ARG(&pipe_config->pipe_src),
-		   pipe_config->pixel_rate, pipe_config->min_cdclk);
+	drm_printf(&p, "port clock: %d, pixel rate %d, min cdclk %d, min voltage level %d\n",
+		   pipe_config->port_clock, pipe_config->pixel_rate,
+		   pipe_config->min_cdclk, pipe_config->min_voltage_level);
 
 	drm_printf(&p, "linetime: %d, ips linetime: %d\n",
 		   pipe_config->linetime, pipe_config->ips_linetime);
 
 	if (DISPLAY_VER(display) >= 9)
-		drm_printf(&p, "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d, scaling_filter: %d\n",
+		drm_printf(&p, "num_scalers: %d, scaler_users: 0x%x, scaler_id: %d, scaling_filter: %d, sharpness_strength: %d\n",
 			   crtc->num_scalers,
 			   pipe_config->scaler_state.scaler_users,
 			   pipe_config->scaler_state.scaler_id,
-			   pipe_config->hw.scaling_filter);
+			   pipe_config->hw.scaling_filter,
+			   pipe_config->hw.sharpness_strength);
+
+	drm_printf(&p, "pipe src: " DRM_RECT_FMT "\n",
+		   DRM_RECT_ARG(&pipe_config->pipe_src));
 
 	if (HAS_GMCH(display))
 		drm_printf(&p, "gmch pfit: control: 0x%08x, ratios: 0x%08x, lvds border: 0x%08x\n",
@@ -345,6 +349,11 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 			   str_enabled_disabled(pipe_config->pch_pfit.enabled),
 			   str_yes_no(pipe_config->pch_pfit.force_thru));
 
+	drm_printf(&p, "sharpness strength: %d, sharpness tap size: %d, sharpness enable: %d\n",
+		   pipe_config->pch_pfit.casf.strength,
+		   pipe_config->pch_pfit.casf.win_size,
+		   pipe_config->pch_pfit.casf.enable);
+
 	drm_printf(&p, "ips: %i, double wide: %i, drrs: %i\n",
 		   pipe_config->ips_enabled, pipe_config->double_wide,
 		   pipe_config->has_drrs);
@@ -380,11 +389,6 @@ void intel_crtc_state_dump(const struct intel_crtc_state *pipe_config,
 
 	intel_vdsc_state_dump(&p, 0, pipe_config);
 
-	drm_printf(&p, "sharpness strength: %d, sharpness tap size: %d, sharpness enable: %d\n",
-		   pipe_config->hw.casf_params.strength,
-		   pipe_config->hw.casf_params.win_size,
-		   pipe_config->hw.casf_params.casf_enable);
-
 dump_planes:
 	if (!state)
 		return;
diff --git a/drivers/gpu/drm/i915/display/intel_cursor.c b/drivers/gpu/drm/i915/display/intel_cursor.c
index 18d1014de3613..52347668f27d6 100644
--- a/drivers/gpu/drm/i915/display/intel_cursor.c
+++ b/drivers/gpu/drm/i915/display/intel_cursor.c
@@ -21,7 +21,6 @@
 #include "intel_display_utils.h"
 #include "intel_display_wa.h"
 #include "intel_fb.h"
-#include "intel_fb_pin.h"
 #include "intel_frontbuffer.h"
 #include "intel_plane.h"
 #include "intel_psr.h"
diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.c b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
index 7e59409bbf013..24a51ab21b557 100644
--- a/drivers/gpu/drm/i915/display/intel_cx0_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.c
@@ -492,7 +492,7 @@ void intel_cx0_phy_set_signal_levels(struct intel_encoder *encoder,
 
 	wakeref = intel_cx0_phy_transaction_begin(encoder);
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans)) {
 		intel_cx0_phy_transaction_end(encoder, wakeref);
 		return;
@@ -2980,7 +2980,7 @@ void intel_cx0_powerdown_change_sequence(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	enum port port = encoder->port;
 	enum phy phy = intel_encoder_to_phy(encoder);
-	i915_reg_t buf_ctl2_reg = XELPDP_PORT_BUF_CTL2(display, port);
+	intel_reg_t buf_ctl2_reg = XELPDP_PORT_BUF_CTL2(display, port);
 	int lane;
 
 	intel_de_rmw(display, buf_ctl2_reg,
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.c b/drivers/gpu/drm/i915/display/intel_ddi.c
index ebefa889bc8c5..86520848892e0 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi.c
@@ -33,6 +33,7 @@
 #include <drm/display/drm_scdc_helper.h>
 #include <drm/drm_print.h>
 #include <drm/drm_privacy_screen_consumer.h>
+#include <drm/intel/step.h>
 
 #include "icl_dsi.h"
 #include "intel_alpm.h"
@@ -81,7 +82,6 @@
 #include "intel_psr.h"
 #include "intel_quirks.h"
 #include "intel_snps_phy.h"
-#include "intel_step.h"
 #include "intel_tc.h"
 #include "intel_vdsc.h"
 #include "intel_vdsc_regs.h"
@@ -140,7 +140,7 @@ void hsw_prepare_dp_ddi_buffers(struct intel_encoder *encoder,
 	enum port port = encoder->port;
 	const struct intel_ddi_buf_trans *trans;
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans))
 		return;
 
@@ -172,7 +172,7 @@ static void hsw_prepare_hdmi_ddi_buffers(struct intel_encoder *encoder,
 	enum port port = encoder->port;
 	const struct intel_ddi_buf_trans *trans;
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans))
 		return;
 
@@ -188,7 +188,7 @@ static void hsw_prepare_hdmi_ddi_buffers(struct intel_encoder *encoder,
 		       trans->entries[level].hsw.trans2);
 }
 
-static i915_reg_t intel_ddi_buf_status_reg(struct intel_display *display, enum port port)
+static intel_reg_t intel_ddi_buf_status_reg(struct intel_display *display, enum port port)
 {
 	if (DISPLAY_VER(display) >= 14)
 		return XELPDP_PORT_BUF_CTL1(display, port);
@@ -1106,7 +1106,7 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder,
 		const struct intel_ddi_buf_trans *trans;
 		int n_entries;
 
-		trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+		trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 		if (drm_WARN_ON_ONCE(display->drm, !trans))
 			return;
 
@@ -1132,7 +1132,7 @@ static u8 intel_ddi_dp_voltage_max(struct intel_dp *intel_dp,
 	struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
 	int n_entries;
 
-	encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 
 	if (drm_WARN_ON(display->drm, n_entries < 1))
 		n_entries = 1;
@@ -1175,7 +1175,7 @@ static void icl_ddi_combo_vswing_program(struct intel_encoder *encoder,
 	int n_entries, ln;
 	u32 val;
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans))
 		return;
 
@@ -1293,7 +1293,7 @@ static void icl_mg_phy_set_signal_levels(struct intel_encoder *encoder,
 	if (intel_tc_port_in_tbt_alt_mode(enc_to_dig_port(encoder)))
 		return;
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans))
 		return;
 
@@ -1394,7 +1394,7 @@ static void tgl_dkl_phy_set_signal_levels(struct intel_encoder *encoder,
 	if (intel_tc_port_in_tbt_alt_mode(enc_to_dig_port(encoder)))
 		return;
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans))
 		return;
 
@@ -1511,7 +1511,7 @@ int intel_ddi_level(struct intel_encoder *encoder,
 	const struct intel_ddi_buf_trans *trans;
 	int level, n_entries;
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans))
 		return 0;
 
@@ -1556,7 +1556,7 @@ hsw_set_signal_levels(struct intel_encoder *encoder,
 	intel_de_posting_read(display, DDI_BUF_CTL(port));
 }
 
-static void _icl_ddi_enable_clock(struct intel_display *display, i915_reg_t reg,
+static void _icl_ddi_enable_clock(struct intel_display *display, intel_reg_t reg,
 				  u32 clk_sel_mask, u32 clk_sel, u32 clk_off)
 {
 	mutex_lock(&display->dpll.lock);
@@ -1572,7 +1572,7 @@ static void _icl_ddi_enable_clock(struct intel_display *display, i915_reg_t reg,
 	mutex_unlock(&display->dpll.lock);
 }
 
-static void _icl_ddi_disable_clock(struct intel_display *display, i915_reg_t reg,
+static void _icl_ddi_disable_clock(struct intel_display *display, intel_reg_t reg,
 				   u32 clk_off)
 {
 	mutex_lock(&display->dpll.lock);
@@ -1582,14 +1582,14 @@ static void _icl_ddi_disable_clock(struct intel_display *display, i915_reg_t reg
 	mutex_unlock(&display->dpll.lock);
 }
 
-static bool _icl_ddi_is_clock_enabled(struct intel_display *display, i915_reg_t reg,
+static bool _icl_ddi_is_clock_enabled(struct intel_display *display, intel_reg_t reg,
 				      u32 clk_off)
 {
 	return !(intel_de_read(display, reg) & clk_off);
 }
 
 static struct intel_dpll *
-_icl_ddi_get_pll(struct intel_display *display, i915_reg_t reg,
+_icl_ddi_get_pll(struct intel_display *display, intel_reg_t reg,
 		 u32 clk_sel_mask, u32 clk_sel_shift)
 {
 	enum intel_dpll_id id;
@@ -2270,8 +2270,8 @@ tgl_dp_tp_transcoder(const struct intel_crtc_state *crtc_state)
 		return crtc_state->cpu_transcoder;
 }
 
-i915_reg_t dp_tp_ctl_reg(struct intel_encoder *encoder,
-			 const struct intel_crtc_state *crtc_state)
+intel_reg_t dp_tp_ctl_reg(struct intel_encoder *encoder,
+			  const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
 
@@ -2282,8 +2282,8 @@ i915_reg_t dp_tp_ctl_reg(struct intel_encoder *encoder,
 		return DP_TP_CTL(encoder->port);
 }
 
-static i915_reg_t dp_tp_status_reg(struct intel_encoder *encoder,
-				   const struct intel_crtc_state *crtc_state)
+static intel_reg_t dp_tp_status_reg(struct intel_encoder *encoder,
+				    const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
 
@@ -2559,7 +2559,7 @@ mtl_ddi_enable_d2d(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	enum port port = encoder->port;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 set_bits, wait_bits;
 	int ret;
 
@@ -3059,7 +3059,7 @@ mtl_ddi_disable_d2d(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	enum port port = encoder->port;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 clr_bits, wait_bits;
 	int ret;
 
@@ -3386,7 +3386,7 @@ static void intel_ddi_enable_dp(struct intel_atomic_state *state,
 	trans_port_sync_stop_link_train(state, encoder, crtc_state);
 }
 
-static i915_reg_t
+static intel_reg_t
 gen9_chicken_trans_reg_by_port(struct intel_display *display, enum port port)
 {
 	static const enum transcoder trans[] = {
@@ -3439,7 +3439,7 @@ static void intel_ddi_enable_hdmi(struct intel_atomic_state *state,
 		 * the bits affect a specific DDI port rather than
 		 * a specific transcoder.
 		 */
-		i915_reg_t reg = gen9_chicken_trans_reg_by_port(display, port);
+		intel_reg_t reg = gen9_chicken_trans_reg_by_port(display, port);
 		u32 val;
 
 		val = intel_de_read(display, reg);
@@ -4729,6 +4729,15 @@ static int intel_ddi_init_dp_connector(struct intel_digital_port *dig_port)
 	return 0;
 }
 
+static void intel_ddi_cleanup_dp_connector(struct intel_digital_port *dig_port)
+{
+	struct intel_dp *intel_dp = &dig_port->dp;
+	struct intel_connector *connector = intel_dp->attached_connector;
+
+	intel_dp_cleanup_connector(dig_port, connector);
+	kfree(connector);
+}
+
 static int intel_hdmi_reset_link(struct intel_encoder *encoder,
 				 struct drm_modeset_acquire_ctx *ctx)
 {
@@ -5411,7 +5420,7 @@ void intel_ddi_init(struct intel_display *display,
 	if (need_aux_ch(encoder, init_dp)) {
 		dig_port->aux_ch = intel_dp_aux_ch(encoder);
 		if (dig_port->aux_ch == AUX_CH_NONE)
-			goto err;
+			goto err_aux_ch_init;
 	}
 
 	/*
@@ -5447,7 +5456,7 @@ void intel_ddi_init(struct intel_display *display,
 		dig_port->unlock = intel_tc_port_unlock;
 
 		if (intel_tc_port_init(dig_port, is_legacy) < 0)
-			goto err;
+			goto err_aux_ch_init;
 	}
 
 	drm_WARN_ON(display->drm, port > PORT_I);
@@ -5478,7 +5487,7 @@ void intel_ddi_init(struct intel_display *display,
 
 	if (init_dp) {
 		if (intel_ddi_init_dp_connector(dig_port))
-			goto err;
+			goto err_dp_connector_init;
 
 		dig_port->hpd_pulse = intel_dp_hpd_pulse;
 
@@ -5492,12 +5501,18 @@ void intel_ddi_init(struct intel_display *display,
 	 */
 	if (encoder->type != INTEL_OUTPUT_EDP && init_hdmi) {
 		if (intel_ddi_init_hdmi_connector(dig_port))
-			goto err;
+			goto err_hdmi_connector_init;
 	}
 
 	return;
 
-err:
+err_hdmi_connector_init:
+	if (init_dp)
+		intel_ddi_cleanup_dp_connector(dig_port);
+err_dp_connector_init:
+	if (intel_encoder_is_tc(encoder))
+		intel_tc_port_cleanup(dig_port);
+err_aux_ch_init:
 	drm_encoder_cleanup(&encoder->base);
 	kfree(dig_port);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_ddi.h b/drivers/gpu/drm/i915/display/intel_ddi.h
index f6f511bb04314..580ecb09b8b60 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi.h
+++ b/drivers/gpu/drm/i915/display/intel_ddi.h
@@ -6,7 +6,7 @@
 #ifndef __INTEL_DDI_H__
 #define __INTEL_DDI_H__
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 
 struct drm_connector_state;
 struct intel_atomic_state;
@@ -23,8 +23,8 @@ enum pipe;
 enum port;
 enum transcoder;
 
-i915_reg_t dp_tp_ctl_reg(struct intel_encoder *encoder,
-			 const struct intel_crtc_state *crtc_state);
+intel_reg_t dp_tp_ctl_reg(struct intel_encoder *encoder,
+			  const struct intel_crtc_state *crtc_state);
 
 void intel_ddi_clear_act_sent(struct intel_encoder *encoder,
 			      const struct intel_crtc_state *crtc_state);
diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
index 395dba8c9e4d3..4cd1e4d76c7af 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
+++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.c
@@ -1123,8 +1123,8 @@ static const union intel_ddi_buf_trans_entry _xe3plpd_lt_trans_dp14[] = {
 	{ .lt = { 1, 2, 0, 28, 7  } },
 	{ .lt = { 0, 3, 0, 35, 13 } },
 	{ .lt = { 1, 1, 0, 27, 0  } },
-	{ .lt = { 1, 2, 0, 31, 4  } },
-	{ .lt = { 0, 3, 0, 39, 9  } },
+	{ .lt = { 1, 2, 0, 31, 5  } },
+	{ .lt = { 0, 3, 0, 37, 11 } },
 	{ .lt = { 1, 2, 0, 35, 0  } },
 	{ .lt = { 0, 3, 0, 41, 7  } },
 	{ .lt = { 0, 3, 0, 48, 0  } },
@@ -1852,3 +1852,10 @@ void intel_ddi_buf_trans_init(struct intel_encoder *encoder)
 		MISSING_CASE(pdev->device);
 	}
 }
+
+const struct intel_ddi_buf_trans *intel_ddi_buf_trans_get(struct intel_encoder *encoder,
+							  const struct intel_crtc_state *crtc_state,
+							  int *n_entries)
+{
+	return encoder->get_buf_trans(encoder, crtc_state, n_entries);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h
index cec332090a20f..9698697f39177 100644
--- a/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h
+++ b/drivers/gpu/drm/i915/display/intel_ddi_buf_trans.h
@@ -77,5 +77,8 @@ struct intel_ddi_buf_trans {
 bool is_hobl_buf_trans(const struct intel_ddi_buf_trans *table);
 
 void intel_ddi_buf_trans_init(struct intel_encoder *encoder);
+const struct intel_ddi_buf_trans *
+intel_ddi_buf_trans_get(struct intel_encoder *encoder,
+			const struct intel_crtc_state *crtc_state, int *n_entries);
 
 #endif
diff --git a/drivers/gpu/drm/i915/display/intel_de.c b/drivers/gpu/drm/i915/display/intel_de.c
index d2a418da2d54b..6daee9e82503b 100644
--- a/drivers/gpu/drm/i915/display/intel_de.c
+++ b/drivers/gpu/drm/i915/display/intel_de.c
@@ -10,9 +10,9 @@
 #include "intel_de.h"
 
 static int __intel_de_wait_for_register(struct intel_display *display,
-					i915_reg_t reg, u32 mask, u32 value,
+					intel_reg_t reg, u32 mask, u32 value,
 					unsigned int timeout_us,
-					u32 (*read)(struct intel_display *display, i915_reg_t reg),
+					u32 (*read)(struct intel_display *display, intel_reg_t reg),
 					u32 *out_val, bool is_atomic)
 {
 	const ktime_t end = ktime_add_us(ktime_get_raw(), timeout_us);
@@ -61,10 +61,10 @@ static int __intel_de_wait_for_register(struct intel_display *display,
 }
 
 static int intel_de_wait_for_register(struct intel_display *display,
-				      i915_reg_t reg, u32 mask, u32 value,
+				      intel_reg_t reg, u32 mask, u32 value,
 				      unsigned int fast_timeout_us,
 				      unsigned int slow_timeout_us,
-				      u32 (*read)(struct intel_display *display, i915_reg_t reg),
+				      u32 (*read)(struct intel_display *display, intel_reg_t reg),
 				      u32 *out_value, bool is_atomic)
 {
 	int ret = -EINVAL;
@@ -82,7 +82,7 @@ static int intel_de_wait_for_register(struct intel_display *display,
 	return ret;
 }
 
-int intel_de_wait_us(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_us(struct intel_display *display, intel_reg_t reg,
 		     u32 mask, u32 value, unsigned int timeout_us,
 		     u32 *out_value)
 {
@@ -100,7 +100,7 @@ int intel_de_wait_us(struct intel_display *display, i915_reg_t reg,
 	return ret;
 }
 
-int intel_de_wait_ms(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_ms(struct intel_display *display, intel_reg_t reg,
 		     u32 mask, u32 value, unsigned int timeout_ms,
 		     u32 *out_value)
 {
@@ -118,7 +118,7 @@ int intel_de_wait_ms(struct intel_display *display, i915_reg_t reg,
 	return ret;
 }
 
-int intel_de_wait_fw_ms(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_fw_ms(struct intel_display *display, intel_reg_t reg,
 			u32 mask, u32 value, unsigned int timeout_ms,
 			u32 *out_value)
 {
@@ -128,7 +128,7 @@ int intel_de_wait_fw_ms(struct intel_display *display, i915_reg_t reg,
 					  out_value, false);
 }
 
-int intel_de_wait_fw_us_atomic(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_fw_us_atomic(struct intel_display *display, intel_reg_t reg,
 			       u32 mask, u32 value, unsigned int timeout_us,
 			       u32 *out_value)
 {
@@ -138,31 +138,31 @@ int intel_de_wait_fw_us_atomic(struct intel_display *display, i915_reg_t reg,
 					  out_value, true);
 }
 
-int intel_de_wait_for_set_us(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_for_set_us(struct intel_display *display, intel_reg_t reg,
 			     u32 mask, unsigned int timeout_us)
 {
 	return intel_de_wait_us(display, reg, mask, mask, timeout_us, NULL);
 }
 
-int intel_de_wait_for_clear_us(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_for_clear_us(struct intel_display *display, intel_reg_t reg,
 			       u32 mask, unsigned int timeout_us)
 {
 	return intel_de_wait_us(display, reg, mask, 0, timeout_us, NULL);
 }
 
-int intel_de_wait_for_set_ms(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_for_set_ms(struct intel_display *display, intel_reg_t reg,
 			     u32 mask, unsigned int timeout_ms)
 {
 	return intel_de_wait_ms(display, reg, mask, mask, timeout_ms, NULL);
 }
 
-int intel_de_wait_for_clear_ms(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_for_clear_ms(struct intel_display *display, intel_reg_t reg,
 			       u32 mask, unsigned int timeout_ms)
 {
 	return intel_de_wait_ms(display, reg, mask, 0, timeout_ms, NULL);
 }
 
-u8 intel_de_read8(struct intel_display *display, i915_reg_t reg)
+u8 intel_de_read8(struct intel_display *display, intel_reg_t reg)
 {
 	/* this is only used on VGA registers (possible on pre-g4x) */
 	drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 5 || display->platform.g4x);
@@ -170,9 +170,17 @@ u8 intel_de_read8(struct intel_display *display, i915_reg_t reg)
 	return intel_uncore_read8(__to_uncore(display), reg);
 }
 
-void intel_de_write8(struct intel_display *display, i915_reg_t reg, u8 val)
+void intel_de_write8(struct intel_display *display, intel_reg_t reg, u8 val)
 {
 	drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 5 || display->platform.g4x);
 
 	intel_uncore_write8(__to_uncore(display), reg, val);
 }
+
+u16 intel_de_read16(struct intel_display *display, intel_reg_t reg)
+{
+	/* this is only used on MCHBAR registers on pre-SNB */
+	drm_WARN_ON(display->drm, DISPLAY_VER(display) >= 6);
+
+	return intel_uncore_read16(__to_uncore(display), reg);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_de.h b/drivers/gpu/drm/i915/display/intel_de.h
index f87b84ab9d6d6..1029790194296 100644
--- a/drivers/gpu/drm/i915/display/intel_de.h
+++ b/drivers/gpu/drm/i915/display/intel_de.h
@@ -17,11 +17,12 @@ static inline struct intel_uncore *__to_uncore(struct intel_display *display)
 	return to_intel_uncore(display->drm);
 }
 
-u8 intel_de_read8(struct intel_display *display, i915_reg_t reg);
-void intel_de_write8(struct intel_display *display, i915_reg_t reg, u8 val);
+u8 intel_de_read8(struct intel_display *display, intel_reg_t reg);
+void intel_de_write8(struct intel_display *display, intel_reg_t reg, u8 val);
+u16 intel_de_read16(struct intel_display *display, intel_reg_t reg);
 
 static inline u32
-intel_de_read(struct intel_display *display, i915_reg_t reg)
+intel_de_read(struct intel_display *display, intel_reg_t reg)
 {
 	u32 val;
 
@@ -35,8 +36,8 @@ intel_de_read(struct intel_display *display, i915_reg_t reg)
 }
 
 static inline u64
-intel_de_read64_2x32(struct intel_display *display,
-		     i915_reg_t lower_reg, i915_reg_t upper_reg)
+intel_de_read64_2x32_volatile(struct intel_display *display,
+			      intel_reg_t lower_reg, intel_reg_t upper_reg)
 {
 	u64 val;
 
@@ -52,8 +53,20 @@ intel_de_read64_2x32(struct intel_display *display,
 	return val;
 }
 
+static inline u64
+intel_de_read64_2x32(struct intel_display *display, intel_reg_t reg)
+{
+	intel_reg_t upper_reg = _MMIO(intel_reg_offset(reg) + 4);
+	u32 lower, upper;
+
+	lower = intel_de_read(display, reg);
+	upper = intel_de_read(display, upper_reg);
+
+	return (u64)upper << 32 | lower;
+}
+
 static inline void
-intel_de_posting_read(struct intel_display *display, i915_reg_t reg)
+intel_de_posting_read(struct intel_display *display, intel_reg_t reg)
 {
 	intel_dmc_wl_get(display, reg);
 
@@ -63,7 +76,7 @@ intel_de_posting_read(struct intel_display *display, i915_reg_t reg)
 }
 
 static inline void
-intel_de_write(struct intel_display *display, i915_reg_t reg, u32 val)
+intel_de_write(struct intel_display *display, intel_reg_t reg, u32 val)
 {
 	intel_dmc_wl_get(display, reg);
 
@@ -73,7 +86,7 @@ intel_de_write(struct intel_display *display, i915_reg_t reg, u32 val)
 }
 
 static inline u32
-intel_de_rmw(struct intel_display *display, i915_reg_t reg, u32 clear, u32 set)
+intel_de_rmw(struct intel_display *display, intel_reg_t reg, u32 clear, u32 set)
 {
 	u32 val;
 
@@ -86,25 +99,25 @@ intel_de_rmw(struct intel_display *display, i915_reg_t reg, u32 clear, u32 set)
 	return val;
 }
 
-int intel_de_wait_us(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_us(struct intel_display *display, intel_reg_t reg,
 		     u32 mask, u32 value, unsigned int timeout_us,
 		     u32 *out_value);
-int intel_de_wait_ms(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_ms(struct intel_display *display, intel_reg_t reg,
 		     u32 mask, u32 value, unsigned int timeout_ms,
 		     u32 *out_value);
-int intel_de_wait_fw_ms(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_fw_ms(struct intel_display *display, intel_reg_t reg,
 			u32 mask, u32 value, unsigned int timeout_ms,
 			u32 *out_value);
-int intel_de_wait_fw_us_atomic(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_fw_us_atomic(struct intel_display *display, intel_reg_t reg,
 			       u32 mask, u32 value, unsigned int timeout_us,
 			       u32 *out_value);
-int intel_de_wait_for_set_us(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_for_set_us(struct intel_display *display, intel_reg_t reg,
 			     u32 mask, unsigned int timeout_us);
-int intel_de_wait_for_clear_us(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_for_clear_us(struct intel_display *display, intel_reg_t reg,
 			       u32 mask, unsigned int timeout_us);
-int intel_de_wait_for_set_ms(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_for_set_ms(struct intel_display *display, intel_reg_t reg,
 			     u32 mask, unsigned int timeout_ms);
-int intel_de_wait_for_clear_ms(struct intel_display *display, i915_reg_t reg,
+int intel_de_wait_for_clear_ms(struct intel_display *display, intel_reg_t reg,
 			       u32 mask, unsigned int timeout_ms);
 
 /*
@@ -116,7 +129,7 @@ int intel_de_wait_for_clear_ms(struct intel_display *display, i915_reg_t reg,
  * a more localised lock guarding all access to that bank of registers.
  */
 static inline u32
-intel_de_read_fw(struct intel_display *display, i915_reg_t reg)
+intel_de_read_fw(struct intel_display *display, intel_reg_t reg)
 {
 	u32 val;
 
@@ -127,14 +140,14 @@ intel_de_read_fw(struct intel_display *display, i915_reg_t reg)
 }
 
 static inline void
-intel_de_write_fw(struct intel_display *display, i915_reg_t reg, u32 val)
+intel_de_write_fw(struct intel_display *display, intel_reg_t reg, u32 val)
 {
 	trace_i915_reg_rw(true, reg, val, sizeof(val), true);
 	intel_uncore_write_fw(__to_uncore(display), reg, val);
 }
 
 static inline u32
-intel_de_rmw_fw(struct intel_display *display, i915_reg_t reg, u32 clear, u32 set)
+intel_de_rmw_fw(struct intel_display *display, intel_reg_t reg, u32 clear, u32 set)
 {
 	u32 old, val;
 
@@ -146,20 +159,20 @@ intel_de_rmw_fw(struct intel_display *display, i915_reg_t reg, u32 clear, u32 se
 }
 
 static inline u32
-intel_de_read_notrace(struct intel_display *display, i915_reg_t reg)
+intel_de_read_notrace(struct intel_display *display, intel_reg_t reg)
 {
 	return intel_uncore_read_notrace(__to_uncore(display), reg);
 }
 
 static inline void
-intel_de_write_notrace(struct intel_display *display, i915_reg_t reg, u32 val)
+intel_de_write_notrace(struct intel_display *display, intel_reg_t reg, u32 val)
 {
 	intel_uncore_write_notrace(__to_uncore(display), reg, val);
 }
 
 static __always_inline void
 intel_de_write_dsb(struct intel_display *display, struct intel_dsb *dsb,
-		   i915_reg_t reg, u32 val)
+		   intel_reg_t reg, u32 val)
 {
 	if (dsb)
 		intel_dsb_reg_write(dsb, reg, val);
diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c
index ad2fe10b6b1f2..757a78c75bbff 100644
--- a/drivers/gpu/drm/i915/display/intel_display.c
+++ b/drivers/gpu/drm/i915/display/intel_display.c
@@ -58,7 +58,6 @@
 #include "intel_audio.h"
 #include "intel_bo.h"
 #include "intel_bw.h"
-#include "intel_casf.h"
 #include "intel_cdclk.h"
 #include "intel_clock_gating.h"
 #include "intel_color.h"
@@ -988,24 +987,6 @@ static bool audio_disabling(const struct intel_crtc_state *old_crtc_state,
 		 memcmp(old_crtc_state->eld, new_crtc_state->eld, MAX_ELD_BYTES) != 0);
 }
 
-static bool intel_casf_enabling(const struct intel_crtc_state *new_crtc_state,
-				const struct intel_crtc_state *old_crtc_state)
-{
-	if (!new_crtc_state->hw.active)
-		return false;
-
-	return is_enabling(hw.casf_params.casf_enable, old_crtc_state, new_crtc_state);
-}
-
-static bool intel_casf_disabling(const struct intel_crtc_state *old_crtc_state,
-				 const struct intel_crtc_state *new_crtc_state)
-{
-	if (!new_crtc_state->hw.active)
-		return false;
-
-	return is_disabling(hw.casf_params.casf_enable, old_crtc_state, new_crtc_state);
-}
-
 static bool intel_crtc_lobf_enabling(const struct intel_crtc_state *old_crtc_state,
 				     const struct intel_crtc_state *new_crtc_state)
 {
@@ -1187,9 +1168,6 @@ static void intel_pre_plane_update(struct intel_atomic_state *state,
 	if (audio_disabling(old_crtc_state, new_crtc_state))
 		intel_encoders_audio_disable(state, crtc);
 
-	if (intel_casf_disabling(old_crtc_state, new_crtc_state))
-		intel_casf_disable(new_crtc_state);
-
 	intel_drrs_deactivate(old_crtc_state);
 
 	if (hsw_ips_pre_update(state, crtc))
@@ -2184,7 +2162,7 @@ static void i9xx_crtc_disable(struct intel_atomic_state *state,
 	if (DISPLAY_VER(display) != 2)
 		intel_set_cpu_fifo_underrun_reporting(display, pipe, false);
 
-	if (!display->funcs.wm->initial_watermarks)
+	if (!display->wm.funcs->initial_watermarks)
 		intel_update_watermarks(display);
 
 	/* clock the pipe down to 640x480@60 to potentially save power */
@@ -2601,8 +2579,8 @@ void intel_zero_m_n(struct intel_link_m_n *m_n)
 
 void intel_set_m_n(struct intel_display *display,
 		   const struct intel_link_m_n *m_n,
-		   i915_reg_t data_m_reg, i915_reg_t data_n_reg,
-		   i915_reg_t link_m_reg, i915_reg_t link_n_reg)
+		   intel_reg_t data_m_reg, intel_reg_t data_n_reg,
+		   intel_reg_t link_m_reg, intel_reg_t link_n_reg)
 {
 	intel_de_write(display, data_m_reg, TU_SIZE(m_n->tu) | m_n->data_m);
 	intel_de_write(display, data_n_reg, m_n->data_n);
@@ -3364,8 +3342,8 @@ int ilk_get_lanes_required(int target_clock, int link_bw, int bpp)
 
 void intel_get_m_n(struct intel_display *display,
 		   struct intel_link_m_n *m_n,
-		   i915_reg_t data_m_reg, i915_reg_t data_n_reg,
-		   i915_reg_t link_m_reg, i915_reg_t link_n_reg)
+		   intel_reg_t data_m_reg, intel_reg_t data_n_reg,
+		   intel_reg_t link_m_reg, intel_reg_t link_n_reg)
 {
 	m_n->link_m = intel_de_read(display, link_m_reg) & DATA_LINK_M_N_MASK;
 	m_n->link_n = intel_de_read(display, link_n_reg) & DATA_LINK_M_N_MASK;
@@ -4073,7 +4051,7 @@ bool intel_crtc_get_pipe_config(struct intel_crtc_state *crtc_state)
 	struct intel_display *display = to_intel_display(crtc_state);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 
-	if (!display->funcs.display->get_pipe_config(crtc, crtc_state))
+	if (!display->modeset.funcs->get_pipe_config(crtc, crtc_state))
 		return false;
 
 	crtc_state->hw.active = true;
@@ -4308,14 +4286,9 @@ static int intel_crtc_atomic_check(struct intel_atomic_state *state,
 		return ret;
 	}
 
-	ret = intel_casf_compute_config(crtc_state);
-	if (ret)
-		return ret;
-
 	if (DISPLAY_VER(display) >= 9) {
 		if (intel_crtc_needs_modeset(crtc_state) ||
-		    intel_crtc_needs_fastset(crtc_state) ||
-		    intel_casf_needs_scaler(crtc_state)) {
+		    intel_crtc_needs_fastset(crtc_state)) {
 			ret = skl_update_scaler_crtc(crtc_state);
 			if (ret)
 				return ret;
@@ -4550,6 +4523,7 @@ intel_crtc_copy_uapi_to_hw_state_modeset(struct intel_atomic_state *state,
 	drm_mode_copy(&crtc_state->hw.adjusted_mode,
 		      &crtc_state->uapi.adjusted_mode);
 	crtc_state->hw.scaling_filter = crtc_state->uapi.scaling_filter;
+	crtc_state->hw.sharpness_strength = crtc_state->uapi.sharpness_strength;
 
 	intel_crtc_copy_uapi_to_hw_state_nomodeset(state, crtc);
 }
@@ -4615,6 +4589,7 @@ copy_joiner_crtc_state_modeset(struct intel_atomic_state *state,
 	drm_mode_copy(&secondary_crtc_state->hw.adjusted_mode,
 		      &primary_crtc_state->hw.adjusted_mode);
 	secondary_crtc_state->hw.scaling_filter = primary_crtc_state->hw.scaling_filter;
+	secondary_crtc_state->hw.sharpness_strength = primary_crtc_state->hw.sharpness_strength;
 
 	if (primary_crtc_state->dp_tunnel_ref.tunnel)
 		drm_dp_tunnel_ref_get(primary_crtc_state->dp_tunnel_ref.tunnel,
@@ -4906,10 +4881,14 @@ static bool
 intel_compare_dp_as_sdp(const struct drm_dp_as_sdp *a,
 			const struct drm_dp_as_sdp *b)
 {
-	return a->vtotal == b->vtotal &&
+	return a->sdp_type == b->sdp_type &&
+		a->revision == b->revision &&
+		a->length == b->length &&
+		a->vtotal == b->vtotal &&
 		a->target_rr == b->target_rr &&
 		a->duration_incr_ms == b->duration_incr_ms &&
 		a->duration_decr_ms == b->duration_decr_ms &&
+		a->target_rr_divider == b->target_rr_divider &&
 		a->mode == b->mode;
 }
 
@@ -5368,12 +5347,12 @@ intel_pipe_config_compare(const struct intel_crtc_state *current_config,
 
 		PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
 		PIPE_CONF_CHECK_RECT(pch_pfit.dst);
+		PIPE_CONF_CHECK_BOOL(pch_pfit.casf.enable);
+		PIPE_CONF_CHECK_I(pch_pfit.casf.win_size);
+		PIPE_CONF_CHECK_I(pch_pfit.casf.strength);
 
 		PIPE_CONF_CHECK_I(scaler_state.scaler_id);
 		PIPE_CONF_CHECK_I(pixel_rate);
-		PIPE_CONF_CHECK_BOOL(hw.casf_params.casf_enable);
-		PIPE_CONF_CHECK_I(hw.casf_params.win_size);
-		PIPE_CONF_CHECK_I(hw.casf_params.strength);
 
 		PIPE_CONF_CHECK_X(gamma_mode);
 		if (display->platform.cherryview)
@@ -5658,11 +5637,11 @@ int intel_modeset_commit_pipes(struct intel_display *display,
 			       u8 pipe_mask,
 			       struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct intel_crtc *crtc;
 	int ret;
 
-	state = drm_atomic_state_alloc(display->drm);
+	state = drm_atomic_commit_alloc(display->drm);
 	if (!state)
 		return -ENOMEM;
 
@@ -5683,7 +5662,7 @@ int intel_modeset_commit_pipes(struct intel_display *display,
 
 	ret = drm_atomic_commit(state);
 out:
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	return ret;
 }
@@ -6402,7 +6381,7 @@ static int intel_atomic_check_config_and_link(struct intel_atomic_state *state)
  * @_state: state to validate
  */
 int intel_atomic_check(struct drm_device *dev,
-		       struct drm_atomic_state *_state)
+		       struct drm_atomic_commit *_state)
 {
 	struct intel_display *display = to_intel_display(dev);
 	struct intel_atomic_state *state = to_intel_atomic_state(_state);
@@ -6428,6 +6407,10 @@ int intel_atomic_check(struct drm_device *dev,
 		if (new_crtc_state->uapi.scaling_filter !=
 		    old_crtc_state->uapi.scaling_filter)
 			new_crtc_state->uapi.mode_changed = true;
+
+		if (new_crtc_state->uapi.sharpness_strength !=
+		    old_crtc_state->uapi.sharpness_strength)
+			new_crtc_state->uapi.mode_changed = true;
 	}
 
 	intel_vrr_check_modeset(state);
@@ -6760,7 +6743,9 @@ static void intel_enable_crtc(struct intel_atomic_state *state,
 
 	intel_psr_notify_pipe_change(state, crtc, true);
 
-	display->funcs.display->crtc_enable(state, crtc);
+	display->modeset.funcs->crtc_enable(state, crtc);
+
+	intel_crtc_wait_for_next_vblank(crtc);
 
 	/* vblanks work again, re-enable pipe CRC. */
 	intel_crtc_enable_pipe_crc(crtc);
@@ -6801,11 +6786,6 @@ static void intel_pre_update_crtc(struct intel_atomic_state *state,
 			intel_vrr_set_transcoder_timings(new_crtc_state);
 	}
 
-	if (intel_casf_enabling(new_crtc_state, old_crtc_state))
-		intel_casf_enable(new_crtc_state);
-	else if (new_crtc_state->hw.casf_params.strength != old_crtc_state->hw.casf_params.strength)
-		intel_casf_update_strength(new_crtc_state);
-
 	intel_fbc_update(state, crtc);
 
 	drm_WARN_ON(display->drm, !intel_display_power_is_enabled(display, POWER_DOMAIN_DC_OFF));
@@ -6896,7 +6876,7 @@ static void intel_old_crtc_state_disables(struct intel_atomic_state *state,
 
 	intel_psr_notify_pipe_change(state, crtc, false);
 
-	display->funcs.display->crtc_disable(state, crtc);
+	display->modeset.funcs->crtc_disable(state, crtc);
 
 	for_each_intel_crtc_in_pipe_mask(display->drm, pipe_crtc,
 					 intel_crtc_joined_pipe_mask(old_crtc_state)) {
@@ -7215,7 +7195,7 @@ static void intel_atomic_cleanup_work(struct work_struct *work)
 
 	drm_atomic_helper_cleanup_planes(display->drm, &state->base);
 	drm_atomic_helper_commit_cleanup_done(&state->base);
-	drm_atomic_state_put(&state->base);
+	drm_atomic_commit_put(&state->base);
 }
 
 static void intel_atomic_prepare_plane_clear_colors(struct intel_atomic_state *state)
@@ -7548,7 +7528,7 @@ static void intel_atomic_commit_tail(struct intel_atomic_state *state)
 	}
 
 	/* Now enable the clocks, plane, pipe, and connectors that we set up. */
-	display->funcs.display->commit_modeset_enables(state);
+	display->modeset.funcs->commit_modeset_enables(state);
 
 	/* FIXME probably need to sequence this properly */
 	intel_program_dpkgc_latency(state);
@@ -7719,7 +7699,7 @@ static int intel_atomic_swap_state(struct intel_atomic_state *state)
 	return 0;
 }
 
-int intel_atomic_commit(struct drm_device *dev, struct drm_atomic_state *_state,
+int intel_atomic_commit(struct drm_device *dev, struct drm_atomic_commit *_state,
 			bool nonblock)
 {
 	struct intel_display *display = to_intel_display(dev);
@@ -7774,7 +7754,7 @@ int intel_atomic_commit(struct drm_device *dev, struct drm_atomic_state *_state,
 		return ret;
 	}
 
-	drm_atomic_state_get(&state->base);
+	drm_atomic_commit_get(&state->base);
 	INIT_WORK(&state->base.commit_work, intel_atomic_commit_work);
 
 	if (nonblock && state->modeset) {
@@ -8221,7 +8201,7 @@ intel_mode_valid_max_plane_size(struct intel_display *display,
 	return MODE_OK;
 }
 
-static const struct intel_display_funcs skl_display_funcs = {
+static const struct intel_modeset_funcs skl_display_funcs = {
 	.get_pipe_config = hsw_get_pipe_config,
 	.crtc_enable = hsw_crtc_enable,
 	.crtc_disable = hsw_crtc_disable,
@@ -8230,7 +8210,7 @@ static const struct intel_display_funcs skl_display_funcs = {
 	.fixup_initial_plane_config = skl_fixup_initial_plane_config,
 };
 
-static const struct intel_display_funcs ddi_display_funcs = {
+static const struct intel_modeset_funcs ddi_display_funcs = {
 	.get_pipe_config = hsw_get_pipe_config,
 	.crtc_enable = hsw_crtc_enable,
 	.crtc_disable = hsw_crtc_disable,
@@ -8239,7 +8219,7 @@ static const struct intel_display_funcs ddi_display_funcs = {
 	.fixup_initial_plane_config = i9xx_fixup_initial_plane_config,
 };
 
-static const struct intel_display_funcs pch_split_display_funcs = {
+static const struct intel_modeset_funcs pch_split_display_funcs = {
 	.get_pipe_config = ilk_get_pipe_config,
 	.crtc_enable = ilk_crtc_enable,
 	.crtc_disable = ilk_crtc_disable,
@@ -8248,7 +8228,7 @@ static const struct intel_display_funcs pch_split_display_funcs = {
 	.fixup_initial_plane_config = i9xx_fixup_initial_plane_config,
 };
 
-static const struct intel_display_funcs vlv_display_funcs = {
+static const struct intel_modeset_funcs vlv_display_funcs = {
 	.get_pipe_config = i9xx_get_pipe_config,
 	.crtc_enable = valleyview_crtc_enable,
 	.crtc_disable = i9xx_crtc_disable,
@@ -8257,7 +8237,7 @@ static const struct intel_display_funcs vlv_display_funcs = {
 	.fixup_initial_plane_config = i9xx_fixup_initial_plane_config,
 };
 
-static const struct intel_display_funcs i9xx_display_funcs = {
+static const struct intel_modeset_funcs i9xx_display_funcs = {
 	.get_pipe_config = i9xx_get_pipe_config,
 	.crtc_enable = i9xx_crtc_enable,
 	.crtc_disable = i9xx_crtc_disable,
@@ -8273,27 +8253,27 @@ static const struct intel_display_funcs i9xx_display_funcs = {
 void intel_init_display_hooks(struct intel_display *display)
 {
 	if (DISPLAY_VER(display) >= 9) {
-		display->funcs.display = &skl_display_funcs;
+		display->modeset.funcs = &skl_display_funcs;
 	} else if (HAS_DDI(display)) {
-		display->funcs.display = &ddi_display_funcs;
+		display->modeset.funcs = &ddi_display_funcs;
 	} else if (HAS_PCH_SPLIT(display)) {
-		display->funcs.display = &pch_split_display_funcs;
+		display->modeset.funcs = &pch_split_display_funcs;
 	} else if (display->platform.cherryview ||
 		   display->platform.valleyview) {
-		display->funcs.display = &vlv_display_funcs;
+		display->modeset.funcs = &vlv_display_funcs;
 	} else {
-		display->funcs.display = &i9xx_display_funcs;
+		display->modeset.funcs = &i9xx_display_funcs;
 	}
 }
 
 int intel_initial_commit(struct intel_display *display)
 {
-	struct drm_atomic_state *state = NULL;
+	struct drm_atomic_commit *state = NULL;
 	struct drm_modeset_acquire_ctx ctx;
 	struct intel_crtc *crtc;
 	int ret = 0;
 
-	state = drm_atomic_state_alloc(display->drm);
+	state = drm_atomic_commit_alloc(display->drm);
 	if (!state)
 		return -ENOMEM;
 
@@ -8347,12 +8327,12 @@ int intel_initial_commit(struct intel_display *display)
 
 out:
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 		goto retry;
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
diff --git a/drivers/gpu/drm/i915/display/intel_display.h b/drivers/gpu/drm/i915/display/intel_display.h
index 1e76a455d7c43..45a90d2fe6ec5 100644
--- a/drivers/gpu/drm/i915/display/intel_display.h
+++ b/drivers/gpu/drm/i915/display/intel_display.h
@@ -27,10 +27,10 @@
 
 #include <drm/drm_util.h>
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 #include "intel_display_limits.h"
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_device;
 struct drm_display_mode;
 struct drm_encoder;
@@ -354,7 +354,7 @@ enum phy_fia {
 				       _intel_modeset_secondary_pipes(crtc_state), \
 				       i)
 
-int intel_atomic_check(struct drm_device *dev, struct drm_atomic_state *state);
+int intel_atomic_check(struct drm_device *dev, struct drm_atomic_commit *state);
 u8 intel_calc_enabled_pipes(struct intel_atomic_state *state,
 			    u8 enabled_pipes);
 u8 intel_calc_active_pipes(struct intel_atomic_state *state,
@@ -426,12 +426,12 @@ bool intel_fuzzy_clock_check(int clock1, int clock2);
 void intel_zero_m_n(struct intel_link_m_n *m_n);
 void intel_set_m_n(struct intel_display *display,
 		   const struct intel_link_m_n *m_n,
-		   i915_reg_t data_m_reg, i915_reg_t data_n_reg,
-		   i915_reg_t link_m_reg, i915_reg_t link_n_reg);
+		   intel_reg_t data_m_reg, intel_reg_t data_n_reg,
+		   intel_reg_t link_m_reg, intel_reg_t link_n_reg);
 void intel_get_m_n(struct intel_display *display,
 		   struct intel_link_m_n *m_n,
-		   i915_reg_t data_m_reg, i915_reg_t data_n_reg,
-		   i915_reg_t link_m_reg, i915_reg_t link_n_reg);
+		   intel_reg_t data_m_reg, intel_reg_t data_n_reg,
+		   intel_reg_t link_m_reg, intel_reg_t link_n_reg);
 bool intel_cpu_transcoder_has_m2_n2(struct intel_display *display,
 				    enum transcoder transcoder);
 void intel_cpu_transcoder_set_m1_n1(struct intel_crtc *crtc,
@@ -493,7 +493,7 @@ int intel_initial_commit(struct intel_display *display);
 void intel_panel_sanitize_ssc(struct intel_display *display);
 enum drm_mode_status intel_mode_valid(struct drm_device *dev,
 				      const struct drm_display_mode *mode);
-int intel_atomic_commit(struct drm_device *dev, struct drm_atomic_state *_state,
+int intel_atomic_commit(struct drm_device *dev, struct drm_atomic_commit *_state,
 			bool nonblock);
 
 /* modesetting asserts */
diff --git a/drivers/gpu/drm/i915/display/intel_display_clock_gating.c b/drivers/gpu/drm/i915/display/intel_display_clock_gating.c
new file mode 100644
index 0000000000000..d036c7e5863d4
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_clock_gating.c
@@ -0,0 +1,270 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright 2026 Intel Corporation
+ */
+
+#include <drm/intel/intel_gmd_misc_regs.h>
+
+#include "intel_de.h"
+#include "i9xx_plane_regs.h"
+#include "intel_display.h"
+#include "intel_display_clock_gating.h"
+#include "intel_display_core.h"
+#include "intel_display_regs.h"
+
+static void intel_display_gen9_init_clock_gating(struct intel_display *display)
+{
+	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
+	intel_de_rmw(display, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
+
+	/* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
+	intel_de_rmw(display, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
+
+	/*
+	 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
+	 * Display WA #0859: skl,bxt,kbl,glk,cfl
+	 */
+	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
+}
+
+void intel_display_skl_init_clock_gating(struct intel_display *display)
+{
+	/*
+	 * WaCompressedResourceDisplayNewHashMode:skl,kbl
+	 * Display WA #0390: skl,kbl
+	 *
+	 * Must match Sampler, Pixel Back End, and Media. See
+	 * WaCompressedResourceSamplerPbeMediaNewHashMode.
+	 */
+	intel_de_rmw(display, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
+
+	intel_display_gen9_init_clock_gating(display);
+
+	/*
+	 * WaFbcTurnOffFbcWatermark:skl
+	 * Display WA #0562: skl
+	 */
+	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+}
+
+void intel_display_kbl_init_clock_gating(struct intel_display *display)
+{
+	/*
+	 * WaCompressedResourceDisplayNewHashMode:skl,kbl
+	 * Display WA #0390: skl,kbl
+	 *
+	 * Must match Sampler, Pixel Back End, and Media. See
+	 * WaCompressedResourceSamplerPbeMediaNewHashMode.
+	 */
+	intel_de_rmw(display, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
+
+	intel_display_gen9_init_clock_gating(display);
+
+	/*
+	 * WaFbcTurnOffFbcWatermark:kbl
+	 * Display WA #0562: kbl
+	 */
+	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+}
+
+void intel_display_cfl_init_clock_gating(struct intel_display *display)
+{
+	/*
+	 * WaCompressedResourceDisplayNewHashMode:skl,kbl (and cfl, cml)
+	 * Display WA #0390: skl,kbl (and cfl, cml)
+	 *
+	 * Must match Sampler, Pixel Back End, and Media. See
+	 * WaCompressedResourceSamplerPbeMediaNewHashMode.
+	 *
+	 * NOTE: this is the same workaround used for skl and kbl,
+	 * because the original implementation was checking HAS_LLC(),
+	 * which cfl/cml have, even though the comment for the
+	 * workaround doesn't mention it.
+	 *
+	 */
+	intel_de_rmw(display, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
+
+	intel_display_gen9_init_clock_gating(display);
+
+	/*
+	 * WaFbcTurnOffFbcWatermark:cfl
+	 * Display WA #0562: cfl
+	 */
+	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+}
+
+void intel_display_bxt_init_clock_gating(struct intel_display *display)
+{
+	intel_display_gen9_init_clock_gating(display);
+
+	/*
+	 * Wa: Backlight PWM may stop in the asserted state, causing backlight
+	 * to stay fully on.
+	 */
+	intel_de_write(display, GEN9_CLKGATE_DIS_0,
+		       intel_de_read(display, GEN9_CLKGATE_DIS_0) |
+		       PWM1_GATING_DIS | PWM2_GATING_DIS);
+
+	/*
+	 * Lower the display internal timeout.
+	 * This is needed to avoid any hard hangs when DSI port PLL
+	 * is off and a MMIO access is attempted by any privilege
+	 * application, using batch buffers or any other means.
+	 */
+	intel_de_write(display, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
+
+	/*
+	 * WaFbcTurnOffFbcWatermark:bxt
+	 * Display WA #0562: bxt
+	 */
+	intel_de_rmw(display, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+}
+
+void intel_display_glk_init_clock_gating(struct intel_display *display)
+{
+	intel_display_gen9_init_clock_gating(display);
+
+	/*
+	 * WaDisablePWMClockGating:glk
+	 * Backlight PWM may stop in the asserted state, causing backlight
+	 * to stay fully on.
+	 */
+	intel_de_write(display, GEN9_CLKGATE_DIS_0,
+		       intel_de_read(display, GEN9_CLKGATE_DIS_0) |
+		       PWM1_GATING_DIS | PWM2_GATING_DIS);
+}
+
+void intel_display_bdw_clock_gating_disable_fbcq(struct intel_display *display)
+{
+	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
+	intel_de_rmw(display, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
+}
+
+void intel_display_bdw_clock_gating_vblank_in_srd(struct intel_display *display)
+{
+	enum pipe pipe;
+
+	/* WaPsrDPAMaskVBlankInSRD:hsw */
+	intel_de_rmw(display, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
+
+	for_each_pipe(display, pipe) {
+		/* WaPsrDPRSUnmaskVBlankInSRD:hsw,bdw */
+		intel_de_rmw(display, CHICKEN_PIPESL_1(pipe), 0,
+			     BDW_UNMASK_VBL_TO_REGS_IN_SRD);
+	}
+}
+
+void intel_display_bdw_clock_gating_kvm_notif(struct intel_display *display)
+{
+	/* WaKVMNotificationOnConfigChange:bdw */
+	intel_de_rmw(display, CHICKEN_PAR2_1, 0,
+		     KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
+}
+
+void intel_display_hsw_init_clock_gating(struct intel_display *display)
+{
+	enum pipe pipe;
+
+	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
+	intel_de_rmw(display, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
+
+	/* WaPsrDPAMaskVBlankInSRD:hsw */
+	intel_de_rmw(display, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
+
+	for_each_pipe(display, pipe) {
+		/* WaPsrDPRSUnmaskVBlankInSRD:hsw,bdw */
+		intel_de_rmw(display, CHICKEN_PIPESL_1(pipe), 0,
+			     HSW_UNMASK_VBL_TO_REGS_IN_SRD);
+	}
+}
+
+void intel_display_disable_trickle_feed(struct intel_display *display)
+{
+	enum pipe pipe;
+
+	for_each_pipe(display, pipe) {
+		intel_de_rmw(display, DSPCNTR(display, pipe), 0,
+			     DISP_TRICKLE_FEED_DISABLE);
+
+		intel_de_rmw(display, DSPSURF(display, pipe), 0, 0);
+		intel_de_posting_read(display, DSPSURF(display, pipe));
+	}
+}
+
+void intel_display_ilk_init_clock_gating(struct intel_display *display)
+{
+	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
+
+	/*
+	 * Required for FBC
+	 * WaFbcDisableDpfcClockGating:ilk
+	 */
+	dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
+		       ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
+		       ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
+
+	intel_de_write(display, ILK_DISPLAY_CHICKEN2,
+		       intel_de_read(display, ILK_DISPLAY_CHICKEN2) |
+		       ILK_DPARB_GATE | ILK_VSDPFD_FULL);
+	dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
+	intel_de_write(display, DISP_ARB_CTL,
+		       intel_de_read(display, DISP_ARB_CTL) |
+		       DISP_FBC_WM_DIS);
+
+	if (display->platform.ironlake && display->platform.mobile) {
+		/* WaFbcAsynchFlipDisableFbcQueue:ilk */
+		intel_de_rmw(display, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
+		intel_de_rmw(display, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
+	}
+
+	intel_de_write(display, ILK_DSPCLK_GATE_D, dspclk_gate);
+	intel_de_rmw(display, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
+
+	intel_display_disable_trickle_feed(display);
+}
+
+void intel_display_gen6_init_clock_gating(struct intel_display *display)
+{
+	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
+
+	intel_de_write(display, ILK_DSPCLK_GATE_D, dspclk_gate);
+	intel_de_rmw(display, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
+
+	intel_de_write(display, ILK_DISPLAY_CHICKEN1,
+		       intel_de_read(display, ILK_DISPLAY_CHICKEN1) |
+		       ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
+	intel_de_write(display, ILK_DISPLAY_CHICKEN2,
+		       intel_de_read(display, ILK_DISPLAY_CHICKEN2) |
+		       ILK_DPARB_GATE | ILK_VSDPFD_FULL);
+	intel_de_write(display, ILK_DSPCLK_GATE_D,
+		       intel_de_read(display, ILK_DSPCLK_GATE_D) |
+		       ILK_DPARBUNIT_CLOCK_GATE_ENABLE |
+		       ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
+
+	intel_display_disable_trickle_feed(display);
+}
+
+void intel_display_ivb_init_clock_gating(struct intel_display *display)
+{
+	intel_de_write(display, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
+	intel_de_rmw(display, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
+}
+
+void intel_display_g4x_init_clock_gating(struct intel_display *display)
+{
+	u32 dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
+			  OVRUNIT_CLOCK_GATE_DISABLE |
+			  OVCUNIT_CLOCK_GATE_DISABLE;
+
+	if (display->platform.gm45)
+		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
+
+	intel_de_write(display, DSPCLK_GATE_D, dspclk_gate);
+
+	intel_display_disable_trickle_feed(display);
+}
+
+void intel_display_i965gm_init_clock_gating(struct intel_display *display)
+{
+	intel_de_write(display, DSPCLK_GATE_D, 0);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_display_clock_gating.h b/drivers/gpu/drm/i915/display/intel_display_clock_gating.h
new file mode 100644
index 0000000000000..b6dd34ca92ddb
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_display_clock_gating.h
@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright 2026 Intel Corporation
+ */
+
+#ifndef __INTEL_DISPLAY_CLOCK_GATING_H__
+#define __INTEL_DISPLAY_CLOCK_GATING_H__
+
+struct intel_display;
+
+void intel_display_skl_init_clock_gating(struct intel_display *display);
+void intel_display_kbl_init_clock_gating(struct intel_display *display);
+void intel_display_cfl_init_clock_gating(struct intel_display *display);
+void intel_display_bxt_init_clock_gating(struct intel_display *display);
+void intel_display_glk_init_clock_gating(struct intel_display *display);
+void intel_display_bdw_clock_gating_disable_fbcq(struct intel_display *display);
+void intel_display_bdw_clock_gating_vblank_in_srd(struct intel_display *display);
+void intel_display_bdw_clock_gating_kvm_notif(struct intel_display *display);
+void intel_display_hsw_init_clock_gating(struct intel_display *display);
+void intel_display_disable_trickle_feed(struct intel_display *display);
+void intel_display_ilk_init_clock_gating(struct intel_display *display);
+void intel_display_gen6_init_clock_gating(struct intel_display *display);
+void intel_display_ivb_init_clock_gating(struct intel_display *display);
+void intel_display_g4x_init_clock_gating(struct intel_display *display);
+void intel_display_i965gm_init_clock_gating(struct intel_display *display);
+
+#endif /* __INTEL_DISPLAY_CLOCK_GATING_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_core.h b/drivers/gpu/drm/i915/display/intel_display_core.h
index d9baca2d5aaf8..3dc5ac75a98be 100644
--- a/drivers/gpu/drm/i915/display/intel_display_core.h
+++ b/drivers/gpu/drm/i915/display/intel_display_core.h
@@ -47,7 +47,7 @@ struct intel_dpll_global_funcs;
 struct intel_dpll_mgr;
 struct intel_fbdev;
 struct intel_fdi_funcs;
-struct intel_hotplug_funcs;
+struct intel_hotplug_irq_funcs;
 struct intel_initial_plane_config;
 struct intel_opregion;
 struct intel_overlay;
@@ -59,7 +59,7 @@ struct task_struct;
 /* Amount of PSF GV points, BSpec precisely defines this */
 #define I915_NUM_PSF_GV_POINTS 3
 
-struct intel_display_funcs {
+struct intel_modeset_funcs {
 	/*
 	 * Returns the active state of the crtc, and if the crtc is active,
 	 * fills out the pipe-config with the hw state.
@@ -100,6 +100,9 @@ struct intel_audio_state {
 };
 
 struct intel_audio {
+	/* internal display audio functions */
+	const struct intel_audio_funcs *funcs;
+
 	/* hda/i915 audio component */
 	struct i915_audio_component *component;
 	bool component_registered;
@@ -124,6 +127,9 @@ struct intel_audio {
  * dpll, because on some platforms plls share registers.
  */
 struct intel_dpll_global {
+	/* internal dpll functions */
+	const struct intel_dpll_global_funcs *funcs;
+
 	struct mutex lock;
 
 	int num_dpll;
@@ -152,6 +158,9 @@ struct intel_frontbuffer_tracking {
 };
 
 struct intel_hotplug {
+	/* internal hotplug irq functions */
+	const struct intel_hotplug_irq_funcs *funcs;
+
 	struct delayed_work hotplug_work;
 
 	const u32 *hpd, *pch_hpd;
@@ -244,6 +253,9 @@ struct intel_vbt_data {
 };
 
 struct intel_wm {
+	/* internal watermark functions */
+	const struct intel_wm_funcs *funcs;
+
 	/*
 	 * Raw watermark latency values:
 	 * in 0.1us units for WM0,
@@ -297,33 +309,6 @@ struct intel_display {
 	/* list of all intel_crtcs sorted by pipe */
 	struct list_head pipe_list;
 
-	/* Display functions */
-	struct {
-		/* Top level crtc-ish functions */
-		const struct intel_display_funcs *display;
-
-		/* Display CDCLK functions */
-		const struct intel_cdclk_funcs *cdclk;
-
-		/* Display pll funcs */
-		const struct intel_dpll_global_funcs *dpll;
-
-		/* irq display functions */
-		const struct intel_hotplug_funcs *hotplug;
-
-		/* pm display functions */
-		const struct intel_wm_funcs *wm;
-
-		/* fdi display functions */
-		const struct intel_fdi_funcs *fdi;
-
-		/* Display internal color functions */
-		const struct intel_color_funcs *color;
-
-		/* Display internal audio functions */
-		const struct intel_audio_funcs *audio;
-	} funcs;
-
 	struct {
 		bool any_task_allowed;
 		struct task_struct *allowed_task;
@@ -351,6 +336,9 @@ struct intel_display {
 	} bw;
 
 	struct {
+		/* Internal CDCLK functions */
+		const struct intel_cdclk_funcs *funcs;
+
 		/* The current hardware cdclk configuration */
 		struct intel_cdclk_config hw;
 
@@ -365,6 +353,9 @@ struct intel_display {
 	} cdclk;
 
 	struct {
+		/* internal color functions */
+		const struct intel_color_funcs *funcs;
+
 		struct drm_property_blob *glk_linear_degamma_lut;
 	} color;
 
@@ -418,6 +409,9 @@ struct intel_display {
 	} fbdev;
 
 	struct {
+		/* internal fdi functions */
+		const struct intel_fdi_funcs *funcs;
+
 		unsigned int pll_freq;
 		u32 rx_config;
 	} fdi;
@@ -481,6 +475,9 @@ struct intel_display {
 	} ips;
 
 	struct {
+		/* internal display irq functions */
+		const struct intel_display_irq_funcs *funcs;
+
 		/* protects the irq masks */
 		spinlock_t lock;
 
@@ -518,6 +515,11 @@ struct intel_display {
 		u32 pipestat_irq_mask[I915_MAX_PIPES];
 	} irq;
 
+	struct {
+		/* Top level crtc-ish functions */
+		const struct intel_modeset_funcs *funcs;
+	} modeset;
+
 	struct {
 		/* protected by wm.wm_mutex */
 		u16 linetime[I915_MAX_PIPES];
@@ -559,12 +561,15 @@ struct intel_display {
 		unsigned long mask;
 	} quirks;
 
+	struct {
+		u32 count;
+	} reset;
+
 	struct {
 		/* restore state for suspend/resume and display reset */
-		struct drm_atomic_state *modeset_state;
+		struct drm_atomic_commit *modeset_state;
 		struct drm_modeset_acquire_ctx reset_ctx;
 		/* modeset stuck tracking for reset */
-		atomic_t pending_fb_pin;
 		u32 saveDSPARB;
 		u32 saveSWF0[16];
 		u32 saveSWF1[16];
diff --git a/drivers/gpu/drm/i915/display/intel_display_debugfs.c b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
index 2614c4863c877..81bef000a4e3e 100644
--- a/drivers/gpu/drm/i915/display/intel_display_debugfs.c
+++ b/drivers/gpu/drm/i915/display/intel_display_debugfs.c
@@ -27,6 +27,7 @@
 #include "intel_display_power.h"
 #include "intel_display_power_well.h"
 #include "intel_display_regs.h"
+#include "intel_display_reset.h"
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
 #include "intel_dmc.h"
@@ -416,11 +417,12 @@ static void intel_scaler_info(struct seq_file *m, struct intel_crtc *crtc)
 
 	/* Not all platforms have a scaler */
 	if (num_scalers) {
-		seq_printf(m, "\tnum_scalers=%d, scaler_users=%x scaler_id=%d scaling_filter=%d",
+		seq_printf(m, "\tnum_scalers=%d, scaler_users=%x scaler_id=%d scaling_filter=%d sharpness_strength=%d",
 			   num_scalers,
 			   crtc_state->scaler_state.scaler_users,
 			   crtc_state->scaler_state.scaler_id,
-			   crtc_state->hw.scaling_filter);
+			   crtc_state->hw.scaling_filter,
+			   crtc_state->hw.sharpness_strength);
 
 		for (i = 0; i < num_scalers; i++) {
 			const struct intel_scaler *sc =
@@ -837,6 +839,7 @@ void intel_display_debugfs_register(struct intel_display *display)
 
 	intel_bios_debugfs_register(display);
 	intel_cdclk_debugfs_register(display);
+	intel_display_reset_debugfs_register(display);
 	intel_dmc_debugfs_register(display);
 	intel_dp_test_debugfs_register(display);
 	intel_fbc_debugfs_register(display);
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.c b/drivers/gpu/drm/i915/display/intel_display_device.c
index be55ef8ea6178..69a9f782935cf 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.c
+++ b/drivers/gpu/drm/i915/display/intel_display_device.c
@@ -9,6 +9,7 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_print.h>
 #include <drm/intel/pciids.h>
+#include <drm/intel/step.h>
 
 #include "intel_cx0_phy_regs.h"
 #include "intel_de.h"
@@ -21,7 +22,6 @@
 #include "intel_display_types.h"
 #include "intel_display_wa.h"
 #include "intel_fbc.h"
-#include "intel_step.h"
 
 __diag_push();
 __diag_ignore_all("-Woverride-init", "Allow field initialization overrides for display info");
@@ -1525,7 +1525,7 @@ probe_gmdid_display(struct intel_display *display, struct intel_display_ip_ver *
 	u32 val;
 	int i;
 
-	addr = pci_iomap_range(pdev, 0, i915_mmio_reg_offset(GMD_ID_DISPLAY), sizeof(u32));
+	addr = pci_iomap_range(pdev, 0, intel_reg_offset(GMD_ID_DISPLAY), sizeof(u32));
 	if (!addr) {
 		drm_err(display->drm,
 			"Cannot map MMIO BAR to read display GMD_ID\n");
diff --git a/drivers/gpu/drm/i915/display/intel_display_device.h b/drivers/gpu/drm/i915/display/intel_display_device.h
index 1170ac3466156..12e5a522a2994 100644
--- a/drivers/gpu/drm/i915/display/intel_display_device.h
+++ b/drivers/gpu/drm/i915/display/intel_display_device.h
@@ -162,7 +162,6 @@ struct intel_display_platforms {
 #define HAS_DDI(__display)		(DISPLAY_INFO(__display)->has_ddi)
 #define HAS_DISPLAY(__display)		(DISPLAY_RUNTIME_INFO(__display)->pipe_mask != 0)
 #define HAS_DMC(__display)		(DISPLAY_RUNTIME_INFO(__display)->has_dmc)
-#define HAS_DMC_WAKELOCK(__display)	(DISPLAY_VER(__display) >= 20)
 #define HAS_DOUBLE_BUFFERED_M_N(__display)	(IS_DISPLAY_VER((__display), 9, 14) || (__display)->platform.broadwell)
 #define HAS_DOUBLE_BUFFERED_LUT(__display)	(DISPLAY_VER(__display) >= 30)
 #define HAS_DOUBLE_WIDE(__display)	(DISPLAY_VER(__display) < 4)
@@ -200,6 +199,7 @@ struct intel_display_platforms {
 #define HAS_PSR_TRANS_PUSH_FRAME_CHANGE(__display)	(DISPLAY_VER(__display) >= 20)
 #define HAS_SAGV(__display)		(DISPLAY_VER(__display) >= 9 && \
 					 !(__display)->platform.broxton && !(__display)->platform.geminilake)
+#define HAS_SAGV_WM(__display)		(DISPLAY_VER(__display) >= 12 && !(__display)->platform.rocketlake)
 #define HAS_TRANSCODER(__display, trans)	((DISPLAY_RUNTIME_INFO(__display)->cpu_transcoder_mask & \
 						  BIT(trans)) != 0)
 #define HAS_UNCOMPRESSED_JOINER(__display)	(DISPLAY_VER(__display) >= 13)
diff --git a/drivers/gpu/drm/i915/display/intel_display_driver.c b/drivers/gpu/drm/i915/display/intel_display_driver.c
index 9c2f7ad6c7b79..8e6c3dfab5c0c 100644
--- a/drivers/gpu/drm/i915/display/intel_display_driver.c
+++ b/drivers/gpu/drm/i915/display/intel_display_driver.c
@@ -680,7 +680,7 @@ void intel_display_driver_unregister(struct intel_display *display)
  */
 int intel_display_driver_suspend(struct intel_display *display)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	int ret;
 
 	if (!HAS_DISPLAY(display))
@@ -704,7 +704,7 @@ int intel_display_driver_suspend(struct intel_display *display)
 
 int
 __intel_display_driver_resume(struct intel_display *display,
-			      struct drm_atomic_state *state,
+			      struct drm_atomic_commit *state,
 			      struct drm_modeset_acquire_ctx *ctx)
 {
 	struct drm_crtc_state *crtc_state;
@@ -743,7 +743,7 @@ __intel_display_driver_resume(struct intel_display *display,
 
 void intel_display_driver_resume(struct intel_display *display)
 {
-	struct drm_atomic_state *state = display->restore.modeset_state;
+	struct drm_atomic_commit *state = display->restore.modeset_state;
 	struct drm_modeset_acquire_ctx ctx;
 	int ret;
 
@@ -778,5 +778,5 @@ void intel_display_driver_resume(struct intel_display *display)
 		drm_err(display->drm,
 			"Restoring old state failed with %i\n", ret);
 	if (state)
-		drm_atomic_state_put(state);
+		drm_atomic_commit_put(state);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_driver.h b/drivers/gpu/drm/i915/display/intel_display_driver.h
index 2966ff91b219c..5270c26a32e0a 100644
--- a/drivers/gpu/drm/i915/display/intel_display_driver.h
+++ b/drivers/gpu/drm/i915/display/intel_display_driver.h
@@ -8,7 +8,7 @@
 
 #include <linux/types.h>
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_modeset_acquire_ctx;
 struct intel_display;
 struct pci_dev;
@@ -29,7 +29,7 @@ void intel_display_driver_resume(struct intel_display *display);
 
 /* interface for intel_display_reset.c */
 int __intel_display_driver_resume(struct intel_display *display,
-				  struct drm_atomic_state *state,
+				  struct drm_atomic_commit *state,
 				  struct drm_modeset_acquire_ctx *ctx);
 
 void intel_display_driver_enable_user_access(struct intel_display *display);
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.c b/drivers/gpu/drm/i915/display/intel_display_irq.c
index 70c1bba7c0a8f..899a38c0a7b75 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.c
@@ -23,6 +23,7 @@
 #include "intel_fifo_underrun.h"
 #include "intel_gmbus.h"
 #include "intel_hotplug_irq.h"
+#include "intel_lpe_audio.h"
 #include "intel_parent.h"
 #include "intel_pipe_crc_regs.h"
 #include "intel_plane.h"
@@ -30,7 +31,7 @@
 #include "intel_psr.h"
 #include "intel_psr_regs.h"
 
-static void irq_reset(struct intel_display *display, struct i915_irq_regs regs)
+static void irq_reset(struct intel_display *display, struct intel_irq_regs regs)
 {
 	intel_de_write(display, regs.imr, 0xffffffff);
 	intel_de_posting_read(display, regs.imr);
@@ -47,7 +48,7 @@ static void irq_reset(struct intel_display *display, struct i915_irq_regs regs)
 /*
  * We should clear IMR at preinstall/uninstall, and just check at postinstall.
  */
-static void assert_iir_is_zero(struct intel_display *display, i915_reg_t reg)
+static void assert_iir_is_zero(struct intel_display *display, intel_reg_t reg)
 {
 	u32 val = intel_de_read(display, reg);
 
@@ -56,14 +57,14 @@ static void assert_iir_is_zero(struct intel_display *display, i915_reg_t reg)
 
 	drm_WARN(display->drm, 1,
 		 "Interrupt register 0x%x is not zero: 0x%08x\n",
-		 i915_mmio_reg_offset(reg), val);
+		 intel_reg_offset(reg), val);
 	intel_de_write(display, reg, 0xffffffff);
 	intel_de_posting_read(display, reg);
 	intel_de_write(display, reg, 0xffffffff);
 	intel_de_posting_read(display, reg);
 }
 
-static void irq_init(struct intel_display *display, struct i915_irq_regs regs,
+static void irq_init(struct intel_display *display, struct intel_irq_regs regs,
 		     u32 imr_val, u32 ier_val)
 {
 	assert_iir_is_zero(display, regs.iir);
@@ -73,7 +74,7 @@ static void irq_init(struct intel_display *display, struct i915_irq_regs regs,
 	intel_de_posting_read(display, regs.imr);
 }
 
-static void error_reset(struct intel_display *display, struct i915_error_regs regs)
+static void error_reset(struct intel_display *display, struct intel_error_regs regs)
 {
 	intel_de_write(display, regs.emr, 0xffffffff);
 	intel_de_posting_read(display, regs.emr);
@@ -84,7 +85,7 @@ static void error_reset(struct intel_display *display, struct i915_error_regs re
 	intel_de_posting_read(display, regs.eir);
 }
 
-static void error_init(struct intel_display *display, struct i915_error_regs regs,
+static void error_init(struct intel_display *display, struct intel_error_regs regs,
 		       u32 emr_val)
 {
 	intel_de_write(display, regs.eir, 0xffffffff);
@@ -343,7 +344,7 @@ u32 i915_pipestat_enable_mask(struct intel_display *display,
 void i915_enable_pipestat(struct intel_display *display,
 			  enum pipe pipe, u32 status_mask)
 {
-	i915_reg_t reg = PIPESTAT(display, pipe);
+	intel_reg_t reg = PIPESTAT(display, pipe);
 	u32 enable_mask;
 
 	drm_WARN_ONCE(display->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
@@ -366,7 +367,7 @@ void i915_enable_pipestat(struct intel_display *display,
 void i915_disable_pipestat(struct intel_display *display,
 			   enum pipe pipe, u32 status_mask)
 {
-	i915_reg_t reg = PIPESTAT(display, pipe);
+	intel_reg_t reg = PIPESTAT(display, pipe);
 	u32 enable_mask;
 
 	drm_WARN_ONCE(display->drm, status_mask & ~PIPESTAT_INT_STATUS_MASK,
@@ -529,8 +530,8 @@ static void i9xx_pipestat_irq_reset(struct intel_display *display)
 	}
 }
 
-void i9xx_pipestat_irq_ack(struct intel_display *display,
-			   u32 iir, u32 pipe_stats[I915_MAX_PIPES])
+static void i9xx_pipestat_irq_ack(struct intel_display *display,
+				  u32 iir, u32 pipe_stats[I915_MAX_PIPES])
 {
 	enum pipe pipe;
 
@@ -543,7 +544,7 @@ void i9xx_pipestat_irq_ack(struct intel_display *display,
 	}
 
 	for_each_pipe(display, pipe) {
-		i915_reg_t reg;
+		intel_reg_t reg;
 		u32 status_mask, enable_mask, iir_bit = 0;
 
 		/*
@@ -596,8 +597,8 @@ void i9xx_pipestat_irq_ack(struct intel_display *display,
 	spin_unlock(&display->irq.lock);
 }
 
-void i915_pipestat_irq_handler(struct intel_display *display,
-			       u32 iir, u32 pipe_stats[I915_MAX_PIPES])
+static void i915_pipestat_irq_handler(struct intel_display *display,
+				      u32 iir, const u32 pipe_stats[I915_MAX_PIPES])
 {
 	bool blc_event = false;
 	enum pipe pipe;
@@ -620,8 +621,8 @@ void i915_pipestat_irq_handler(struct intel_display *display,
 		intel_opregion_asle_intr(display);
 }
 
-void i965_pipestat_irq_handler(struct intel_display *display,
-			       u32 iir, u32 pipe_stats[I915_MAX_PIPES])
+static void i965_pipestat_irq_handler(struct intel_display *display,
+				      u32 iir, const u32 pipe_stats[I915_MAX_PIPES])
 {
 	bool blc_event = false;
 	enum pipe pipe;
@@ -647,8 +648,8 @@ void i965_pipestat_irq_handler(struct intel_display *display,
 		intel_gmbus_irq_handler(display);
 }
 
-void valleyview_pipestat_irq_handler(struct intel_display *display,
-				     u32 pipe_stats[I915_MAX_PIPES])
+static void valleyview_pipestat_irq_handler(struct intel_display *display,
+					    const u32 pipe_stats[I915_MAX_PIPES])
 {
 	enum pipe pipe;
 
@@ -1020,7 +1021,8 @@ void ilk_display_irq_master_enable(struct intel_display *display, u32 de_ier, u3
 		intel_de_write_fw(display, SDEIER, sde_ier);
 }
 
-bool ilk_display_irq_handler(struct intel_display *display)
+static bool ilk_display_irq_handler(struct intel_display *display,
+				    const struct intel_display_irq_state *state)
 {
 	u32 de_iir;
 	bool handled = false;
@@ -1284,7 +1286,7 @@ gen8_de_misc_irq_handler(struct intel_display *display, u32 iir)
 	if (iir & GEN8_DE_EDP_PSR) {
 		struct intel_encoder *encoder;
 		u32 psr_iir;
-		i915_reg_t iir_reg;
+		intel_reg_t iir_reg;
 
 		for_each_intel_encoder_with_psr(display->drm, encoder) {
 			struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
@@ -1404,7 +1406,7 @@ static void gen8_read_and_ack_pch_irqs(struct intel_display *display, u32 *pch_i
 		intel_de_write(display, PICAINTERRUPT_IER, pica_ier);
 }
 
-void gen8_de_irq_handler(struct intel_display *display, u32 master_ctl)
+static void gen8_de_irq_handler(struct intel_display *display, u32 master_ctl)
 {
 	u32 iir;
 	enum pipe pipe;
@@ -1565,6 +1567,14 @@ void gen8_de_irq_handler(struct intel_display *display, u32 master_ctl)
 	}
 }
 
+static bool gen8_display_irq_handler(struct intel_display *display,
+				     const struct intel_display_irq_state *state)
+{
+	gen8_de_irq_handler(display, state->master_ctl);
+
+	return true;
+}
+
 u32 gen11_gu_misc_irq_ack(struct intel_display *display, const u32 master_ctl)
 {
 	u32 iir;
@@ -1589,7 +1599,8 @@ void gen11_gu_misc_irq_handler(struct intel_display *display, const u32 iir)
 		intel_opregion_asle_intr(display);
 }
 
-void gen11_display_irq_handler(struct intel_display *display)
+static bool gen11_display_irq_handler(struct intel_display *display,
+				      const struct intel_display_irq_state *state)
 {
 	u32 disp_ctl;
 
@@ -1605,6 +1616,8 @@ void gen11_display_irq_handler(struct intel_display *display)
 	intel_de_write(display, GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
 
 	intel_display_rpm_assert_unblock(display);
+
+	return true;
 }
 
 static void i915gm_irq_cstate_wa_enable(struct intel_display *display)
@@ -1898,8 +1911,8 @@ static void vlv_page_table_error_irq_handler(struct intel_display *display, u32
 	}
 }
 
-void vlv_display_error_irq_ack(struct intel_display *display,
-			       u32 *eir, u32 *dpinvgtt)
+static void vlv_display_error_irq_ack(struct intel_display *display,
+				      u32 *eir, u32 *dpinvgtt)
 {
 	u32 emr;
 
@@ -1920,8 +1933,8 @@ void vlv_display_error_irq_ack(struct intel_display *display,
 	intel_de_write(display, VLV_EMR, emr);
 }
 
-void vlv_display_error_irq_handler(struct intel_display *display,
-				   u32 eir, u32 dpinvgtt)
+static void vlv_display_error_irq_handler(struct intel_display *display,
+					  u32 eir, u32 dpinvgtt)
 {
 	drm_dbg(display->drm, "Master Error, EIR 0x%08x\n", eir);
 
@@ -1947,7 +1960,7 @@ static void _vlv_display_irq_reset(struct intel_display *display)
 	display->irq.vlv_imr_mask = ~0u;
 }
 
-void vlv_display_irq_reset(struct intel_display *display)
+static void vlv_display_irq_reset(struct intel_display *display)
 {
 	spin_lock_irq(&display->irq.lock);
 	if (display->irq.vlv_display_irqs_enabled)
@@ -1955,7 +1968,7 @@ void vlv_display_irq_reset(struct intel_display *display)
 	spin_unlock_irq(&display->irq.lock);
 }
 
-void i9xx_display_irq_reset(struct intel_display *display)
+static void i9xx_display_irq_reset(struct intel_display *display)
 {
 	if (HAS_HOTPLUG(display)) {
 		i915_hotplug_interrupt_update(display, 0xffffffff, 0);
@@ -1981,7 +1994,7 @@ u32 i9xx_display_irq_enable_mask(struct intel_display *display)
 	return enable_mask;
 }
 
-void i915_display_irq_postinstall(struct intel_display *display)
+static void i915_display_irq_postinstall(struct intel_display *display)
 {
 	/*
 	 * Interrupt setup is already guaranteed to be single-threaded, this is
@@ -1995,7 +2008,7 @@ void i915_display_irq_postinstall(struct intel_display *display)
 	i915_enable_asle_pipestat(display);
 }
 
-void i965_display_irq_postinstall(struct intel_display *display)
+static void i965_display_irq_postinstall(struct intel_display *display)
 {
 	/*
 	 * Interrupt setup is already guaranteed to be single-threaded, this is
@@ -2010,6 +2023,38 @@ void i965_display_irq_postinstall(struct intel_display *display)
 	i915_enable_asle_pipestat(display);
 }
 
+static void i9xx_display_irq_ack(struct intel_display *display,
+				 struct intel_display_irq_state *state)
+{
+	if (state->iir & I915_DISPLAY_PORT_INTERRUPT)
+		state->hotplug_status = i9xx_hpd_irq_ack(display);
+
+	/* Call regardless, as some status bits might not be signalled in IIR */
+	i9xx_pipestat_irq_ack(display, state->iir, state->pipe_stats);
+}
+
+static bool i965_display_irq_handler(struct intel_display *display,
+				     const struct intel_display_irq_state *state)
+{
+	if (state->hotplug_status)
+		i9xx_hpd_irq_handler(display, state->hotplug_status);
+
+	i965_pipestat_irq_handler(display, state->iir, state->pipe_stats);
+
+	return true;
+}
+
+static bool i915_display_irq_handler(struct intel_display *display,
+				     const struct intel_display_irq_state *state)
+{
+	if (state->hotplug_status)
+		i9xx_hpd_irq_handler(display, state->hotplug_status);
+
+	i915_pipestat_irq_handler(display, state->iir, state->pipe_stats);
+
+	return true;
+}
+
 static u32 vlv_error_mask(void)
 {
 	/* TODO enable other errors too? */
@@ -2057,7 +2102,7 @@ static void _vlv_display_irq_postinstall(struct intel_display *display)
 	irq_init(display, VLV_IRQ_REGS, display->irq.vlv_imr_mask, enable_mask);
 }
 
-void vlv_display_irq_postinstall(struct intel_display *display)
+static void vlv_display_irq_postinstall(struct intel_display *display)
 {
 	spin_lock_irq(&display->irq.lock);
 	if (display->irq.vlv_display_irqs_enabled)
@@ -2065,6 +2110,46 @@ void vlv_display_irq_postinstall(struct intel_display *display)
 	spin_unlock_irq(&display->irq.lock);
 }
 
+static u32 vlv_lpe_irq_mask(struct intel_display *display)
+{
+	if (display->platform.cherryview)
+		return I915_LPE_PIPE_A_INTERRUPT | I915_LPE_PIPE_B_INTERRUPT |
+			I915_LPE_PIPE_C_INTERRUPT;
+	else
+		return I915_LPE_PIPE_A_INTERRUPT | I915_LPE_PIPE_B_INTERRUPT;
+}
+
+static void vlv_display_irq_ack(struct intel_display *display,
+				struct intel_display_irq_state *state)
+{
+	if (state->iir & I915_DISPLAY_PORT_INTERRUPT)
+		state->hotplug_status = i9xx_hpd_irq_ack(display);
+
+	if (state->iir & I915_MASTER_ERROR_INTERRUPT)
+		vlv_display_error_irq_ack(display, &state->eir, &state->dpinvgtt);
+
+	/* Call regardless, as some status bits might not be signalled in IIR */
+	i9xx_pipestat_irq_ack(display, state->iir, state->pipe_stats);
+
+	/* The handler acks the irq, so need to call the handler here */
+	if (state->iir & vlv_lpe_irq_mask(display))
+		intel_lpe_audio_irq_handler(display);
+}
+
+static bool vlv_display_irq_handler(struct intel_display *display,
+				    const struct intel_display_irq_state *state)
+{
+	if (state->hotplug_status)
+		i9xx_hpd_irq_handler(display, state->hotplug_status);
+
+	if (state->iir & I915_MASTER_ERROR_INTERRUPT)
+		vlv_display_error_irq_handler(display, state->eir, state->dpinvgtt);
+
+	valleyview_pipestat_irq_handler(display, state->pipe_stats);
+
+	return true;
+}
+
 static void ibx_display_irq_reset(struct intel_display *display)
 {
 	if (HAS_PCH_NOP(display))
@@ -2076,7 +2161,7 @@ static void ibx_display_irq_reset(struct intel_display *display)
 		intel_de_write(display, SERR_INT, 0xffffffff);
 }
 
-void ilk_display_irq_reset(struct intel_display *display)
+static void ilk_display_irq_reset(struct intel_display *display)
 {
 	irq_reset(display, DE_IRQ_REGS);
 	display->irq.ilk_de_imr_mask = ~0u;
@@ -2092,13 +2177,10 @@ void ilk_display_irq_reset(struct intel_display *display)
 	ibx_display_irq_reset(display);
 }
 
-void gen8_display_irq_reset(struct intel_display *display)
+static void gen8_display_irq_reset(struct intel_display *display)
 {
 	enum pipe pipe;
 
-	if (!HAS_DISPLAY(display))
-		return;
-
 	intel_de_write(display, EDP_PSR_IMR, 0xffffffff);
 	intel_de_write(display, EDP_PSR_IIR, 0xffffffff);
 
@@ -2114,15 +2196,12 @@ void gen8_display_irq_reset(struct intel_display *display)
 		ibx_display_irq_reset(display);
 }
 
-void gen11_display_irq_reset(struct intel_display *display)
+static void gen11_display_irq_reset(struct intel_display *display)
 {
 	enum pipe pipe;
 	u32 trans_mask = BIT(TRANSCODER_A) | BIT(TRANSCODER_B) |
 		BIT(TRANSCODER_C) | BIT(TRANSCODER_D);
 
-	if (!HAS_DISPLAY(display))
-		return;
-
 	intel_de_write(display, GEN11_DISPLAY_INT_CTL, 0);
 
 	if (DISPLAY_VER(display) >= 12) {
@@ -2268,7 +2347,7 @@ void valleyview_disable_display_irqs(struct intel_display *display)
 	spin_unlock_irq(&display->irq.lock);
 }
 
-void ilk_de_irq_postinstall(struct intel_display *display)
+static void ilk_de_irq_postinstall(struct intel_display *display)
 {
 	u32 display_mask, extra_mask;
 
@@ -2312,7 +2391,7 @@ void ilk_de_irq_postinstall(struct intel_display *display)
 static void mtp_irq_postinstall(struct intel_display *display);
 static void icp_irq_postinstall(struct intel_display *display);
 
-void gen8_de_irq_postinstall(struct intel_display *display)
+static void gen8_de_irq_postinstall(struct intel_display *display)
 {
 	u32 de_pipe_masked = gen8_de_pipe_fault_mask(display) |
 		GEN8_PIPE_CDCLK_CRC_DONE;
@@ -2439,23 +2518,91 @@ static void icp_irq_postinstall(struct intel_display *display)
 	irq_init(display, SDE_IRQ_REGS, ~mask, 0xffffffff);
 }
 
-void gen11_de_irq_postinstall(struct intel_display *display)
+static void gen11_de_irq_postinstall(struct intel_display *display)
 {
-	if (!HAS_DISPLAY(display))
-		return;
-
 	gen8_de_irq_postinstall(display);
 
 	intel_de_write(display, GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
 }
 
-void dg1_de_irq_postinstall(struct intel_display *display)
+struct intel_display_irq_funcs {
+	void (*reset)(struct intel_display *display);
+	void (*postinstall)(struct intel_display *display);
+	void (*ack)(struct intel_display *display, struct intel_display_irq_state *state);
+	bool (*handler)(struct intel_display *display, const struct intel_display_irq_state *state);
+};
+
+static const struct intel_display_irq_funcs gen11_display_irq_funcs = {
+	.reset = gen11_display_irq_reset,
+	.postinstall = gen11_de_irq_postinstall,
+	.handler = gen11_display_irq_handler,
+};
+
+static const struct intel_display_irq_funcs gen8_display_irq_funcs = {
+	.reset = gen8_display_irq_reset,
+	.postinstall = gen8_de_irq_postinstall,
+	.handler = gen8_display_irq_handler,
+};
+
+static const struct intel_display_irq_funcs vlv_display_irq_funcs = {
+	.reset = vlv_display_irq_reset,
+	.postinstall = vlv_display_irq_postinstall,
+	.ack = vlv_display_irq_ack,
+	.handler = vlv_display_irq_handler,
+};
+
+static const struct intel_display_irq_funcs ilk_display_irq_funcs = {
+	.reset = ilk_display_irq_reset,
+	.postinstall = ilk_de_irq_postinstall,
+	.handler = ilk_display_irq_handler,
+};
+
+static const struct intel_display_irq_funcs i965_display_irq_funcs = {
+	.reset = i9xx_display_irq_reset,
+	.postinstall = i965_display_irq_postinstall,
+	.ack = i9xx_display_irq_ack,
+	.handler = i965_display_irq_handler,
+};
+
+static const struct intel_display_irq_funcs i915_display_irq_funcs = {
+	.reset = i9xx_display_irq_reset,
+	.postinstall = i915_display_irq_postinstall,
+	.ack = i9xx_display_irq_ack,
+	.handler = i915_display_irq_handler,
+};
+
+void intel_display_irq_reset(struct intel_display *display)
+{
+	if (!HAS_DISPLAY(display))
+		return;
+
+	display->irq.funcs->reset(display);
+}
+
+void intel_display_irq_postinstall(struct intel_display *display)
 {
 	if (!HAS_DISPLAY(display))
 		return;
 
-	gen8_de_irq_postinstall(display);
-	intel_de_write(display, GEN11_DISPLAY_INT_CTL, GEN11_DISPLAY_IRQ_ENABLE);
+	display->irq.funcs->postinstall(display);
+}
+
+void intel_display_irq_ack(struct intel_display *display,
+			   struct intel_display_irq_state *state)
+{
+	if (!HAS_DISPLAY(display) || !display->irq.funcs->ack)
+		return;
+
+	display->irq.funcs->ack(display, state);
+}
+
+bool intel_display_irq_handler(struct intel_display *display,
+			       const struct intel_display_irq_state *state)
+{
+	if (!HAS_DISPLAY(display) || !display->irq.funcs->handler)
+		return true;
+
+	return display->irq.funcs->handler(display, state);
 }
 
 void intel_display_irq_init(struct intel_display *display)
@@ -2468,6 +2615,19 @@ void intel_display_irq_init(struct intel_display *display)
 
 	INIT_WORK(&display->irq.vblank_notify_work,
 		  intel_display_vblank_notify_work);
+
+	if (DISPLAY_VER(display) >= 11)
+		display->irq.funcs = &gen11_display_irq_funcs;
+	else if (display->platform.cherryview || display->platform.valleyview)
+		display->irq.funcs = &vlv_display_irq_funcs;
+	else if (DISPLAY_VER(display) >= 8)
+		display->irq.funcs = &gen8_display_irq_funcs;
+	else if (DISPLAY_VER(display) >= 5)
+		display->irq.funcs = &ilk_display_irq_funcs;
+	else if (DISPLAY_VER(display) == 4)
+		display->irq.funcs = &i965_display_irq_funcs;
+	else
+		display->irq.funcs = &i915_display_irq_funcs;
 }
 
 struct intel_display_irq_snapshot {
diff --git a/drivers/gpu/drm/i915/display/intel_display_irq.h b/drivers/gpu/drm/i915/display/intel_display_irq.h
index b25d180254d75..a1227cee885ae 100644
--- a/drivers/gpu/drm/i915/display/intel_display_irq.h
+++ b/drivers/gpu/drm/i915/display/intel_display_irq.h
@@ -51,41 +51,30 @@ void bdw_disable_vblank(struct drm_crtc *crtc);
 
 void ilk_display_irq_master_disable(struct intel_display *display, u32 *de_ier, u32 *sde_ier);
 void ilk_display_irq_master_enable(struct intel_display *display, u32 de_ier, u32 sde_ier);
-bool ilk_display_irq_handler(struct intel_display *display);
-void gen8_de_irq_handler(struct intel_display *display, u32 master_ctl);
-void gen11_display_irq_handler(struct intel_display *display);
 
 u32 gen11_gu_misc_irq_ack(struct intel_display *display, const u32 master_ctl);
 void gen11_gu_misc_irq_handler(struct intel_display *display, const u32 iir);
 
-void i9xx_display_irq_reset(struct intel_display *display);
-void ilk_display_irq_reset(struct intel_display *display);
-void vlv_display_irq_reset(struct intel_display *display);
-void gen8_display_irq_reset(struct intel_display *display);
-void gen11_display_irq_reset(struct intel_display *display);
+struct intel_display_irq_state {
+	u32 master_ctl;
+	u32 iir;
+	u32 eir;
+	u32 hotplug_status;
+	u32 dpinvgtt;
+	u32 pipe_stats[I915_MAX_PIPES];
+};
+
+void intel_display_irq_reset(struct intel_display *display);
+void intel_display_irq_postinstall(struct intel_display *display);
+void intel_display_irq_ack(struct intel_display *display, struct intel_display_irq_state *state);
+bool intel_display_irq_handler(struct intel_display *display, const struct intel_display_irq_state *state);
 
 u32 i9xx_display_irq_enable_mask(struct intel_display *display);
-void i915_display_irq_postinstall(struct intel_display *display);
-void i965_display_irq_postinstall(struct intel_display *display);
-void vlv_display_irq_postinstall(struct intel_display *display);
-void ilk_de_irq_postinstall(struct intel_display *display);
-void gen8_de_irq_postinstall(struct intel_display *display);
-void gen11_de_irq_postinstall(struct intel_display *display);
-void dg1_de_irq_postinstall(struct intel_display *display);
 
 u32 i915_pipestat_enable_mask(struct intel_display *display, enum pipe pipe);
 void i915_enable_pipestat(struct intel_display *display, enum pipe pipe, u32 status_mask);
 void i915_disable_pipestat(struct intel_display *display, enum pipe pipe, u32 status_mask);
 
-void i9xx_pipestat_irq_ack(struct intel_display *display, u32 iir, u32 pipe_stats[I915_MAX_PIPES]);
-
-void i915_pipestat_irq_handler(struct intel_display *display, u32 iir, u32 pipe_stats[I915_MAX_PIPES]);
-void i965_pipestat_irq_handler(struct intel_display *display, u32 iir, u32 pipe_stats[I915_MAX_PIPES]);
-void valleyview_pipestat_irq_handler(struct intel_display *display, u32 pipe_stats[I915_MAX_PIPES]);
-
-void vlv_display_error_irq_ack(struct intel_display *display, u32 *eir, u32 *dpinvgtt);
-void vlv_display_error_irq_handler(struct intel_display *display, u32 eir, u32 dpinvgtt);
-
 void intel_display_irq_init(struct intel_display *display);
 
 void i915gm_irq_cstate_wa(struct intel_display *display, bool enable);
diff --git a/drivers/gpu/drm/i915/display/intel_display_power.c b/drivers/gpu/drm/i915/display/intel_display_power.c
index ec96b141c74c1..751e6b7d4a292 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power.c
@@ -8,6 +8,7 @@
 
 #include <drm/drm_print.h>
 #include <drm/intel/intel_pcode_regs.h>
+#include <drm/intel/step.h>
 
 #include "intel_backlight_regs.h"
 #include "intel_cdclk.h"
@@ -24,13 +25,12 @@
 #include "intel_display_wa.h"
 #include "intel_dmc.h"
 #include "intel_dram.h"
-#include "intel_mchbar_regs.h"
+#include "intel_mchbar.h"
 #include "intel_parent.h"
 #include "intel_pch_refclk.h"
 #include "intel_pmdemand.h"
 #include "intel_pps_regs.h"
 #include "intel_snps_phy.h"
-#include "intel_step.h"
 #include "skl_watermark.h"
 #include "skl_watermark_regs.h"
 #include "vlv_sideband.h"
@@ -1069,7 +1069,7 @@ static void intel_power_domains_sync_hw(struct intel_display *display)
 static void gen9_dbuf_slice_set(struct intel_display *display,
 				enum dbuf_slice slice, bool enable)
 {
-	i915_reg_t reg = DBUF_CTL_S(slice);
+	intel_reg_t reg = DBUF_CTL_S(slice);
 	bool state;
 
 	intel_de_rmw(display, reg, DBUF_POWER_REQUEST,
@@ -1252,7 +1252,7 @@ static void assert_can_disable_lcpll(struct intel_display *display)
 static u32 hsw_read_dcomp(struct intel_display *display)
 {
 	if (display->platform.haswell)
-		return intel_de_read(display, D_COMP_HSW);
+		return intel_mchbar_read(display, D_COMP_HSW);
 	else
 		return intel_de_read(display, D_COMP_BDW);
 }
@@ -1420,15 +1420,13 @@ static void hsw_disable_pc8(struct intel_display *display)
 	intel_init_pch_refclk(display);
 
 	/* Many display registers don't survive PC8+ */
-#ifdef I915 /* FIXME */
 	intel_clock_gating_init(display->drm);
-#endif
 }
 
 static void intel_pch_reset_handshake(struct intel_display *display,
 				      bool enable)
 {
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 reset_bits;
 
 	if (DISPLAY_VER(display) >= 35)
@@ -1890,9 +1888,9 @@ static bool vlv_punit_is_power_gated(struct intel_display *display, u32 reg0)
 {
 	bool ret;
 
-	vlv_punit_get(display->drm);
-	ret = (vlv_punit_read(display->drm, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
-	vlv_punit_put(display->drm);
+	vlv_punit_get(display);
+	ret = (vlv_punit_read(display, reg0) & SSPM0_SSC_MASK) == SSPM0_SSC_PWR_GATE;
+	vlv_punit_put(display);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_map.c b/drivers/gpu/drm/i915/display/intel_display_power_map.c
index 65204d68a7598..3400080d78d2a 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_map.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_map.c
@@ -4,13 +4,13 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/vlv_iosf_sb_regs.h>
 
 #include "intel_display_core.h"
 #include "intel_display_power_map.h"
 #include "intel_display_power_well.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
-#include "vlv_iosf_sb_reg.h"
 
 #define __LIST_INLINE_ELEMS(__elem_type, ...) \
 	((__elem_type[]) { __VA_ARGS__ })
diff --git a/drivers/gpu/drm/i915/display/intel_display_power_well.c b/drivers/gpu/drm/i915/display/intel_display_power_well.c
index f855f0f886946..04bd0dde5beda 100644
--- a/drivers/gpu/drm/i915/display/intel_display_power_well.c
+++ b/drivers/gpu/drm/i915/display/intel_display_power_well.c
@@ -34,7 +34,6 @@
 #include "intel_vga.h"
 #include "skl_watermark.h"
 #include "vlv_dpio_phy_regs.h"
-#include "vlv_iosf_sb_reg.h"
 #include "vlv_sideband.h"
 
 /*
@@ -50,10 +49,10 @@ static enum skl_power_gate pw_idx_to_pg(struct intel_display *display, int pw_id
 }
 
 struct i915_power_well_regs {
-	i915_reg_t bios;
-	i915_reg_t driver;
-	i915_reg_t kvmr;
-	i915_reg_t debug;
+	intel_reg_t bios;
+	intel_reg_t driver;
+	intel_reg_t kvmr;
+	intel_reg_t debug;
 };
 
 struct i915_power_well_ops {
@@ -1189,28 +1188,28 @@ static void vlv_set_power_well(struct intel_display *display,
 	state = enable ? PUNIT_PWRGT_PWR_ON(pw_idx) :
 			 PUNIT_PWRGT_PWR_GATE(pw_idx);
 
-	vlv_punit_get(display->drm);
+	vlv_punit_get(display);
 
-	val = vlv_punit_read(display->drm, PUNIT_REG_PWRGT_STATUS);
+	val = vlv_punit_read(display, PUNIT_REG_PWRGT_STATUS);
 	if ((val & mask) == state)
 		goto out;
 
-	ctrl = vlv_punit_read(display->drm, PUNIT_REG_PWRGT_CTRL);
+	ctrl = vlv_punit_read(display, PUNIT_REG_PWRGT_CTRL);
 	ctrl &= ~mask;
 	ctrl |= state;
-	vlv_punit_write(display->drm, PUNIT_REG_PWRGT_CTRL, ctrl);
+	vlv_punit_write(display, PUNIT_REG_PWRGT_CTRL, ctrl);
 
-	ret = poll_timeout_us(val = vlv_punit_read(display->drm, PUNIT_REG_PWRGT_STATUS),
+	ret = poll_timeout_us(val = vlv_punit_read(display, PUNIT_REG_PWRGT_STATUS),
 			      (val & mask) == state,
 			      500, 100 * 1000, false);
 	if (ret)
 		drm_err(display->drm,
 			"timeout setting power well state %08x (%08x)\n",
 			state,
-			vlv_punit_read(display->drm, PUNIT_REG_PWRGT_CTRL));
+			vlv_punit_read(display, PUNIT_REG_PWRGT_CTRL));
 
 out:
-	vlv_punit_put(display->drm);
+	vlv_punit_put(display);
 }
 
 static void vlv_power_well_enable(struct intel_display *display,
@@ -1237,9 +1236,9 @@ static bool vlv_power_well_enabled(struct intel_display *display,
 	mask = PUNIT_PWRGT_MASK(pw_idx);
 	ctrl = PUNIT_PWRGT_PWR_ON(pw_idx);
 
-	vlv_punit_get(display->drm);
+	vlv_punit_get(display);
 
-	state = vlv_punit_read(display->drm, PUNIT_REG_PWRGT_STATUS) & mask;
+	state = vlv_punit_read(display, PUNIT_REG_PWRGT_STATUS) & mask;
 	/*
 	 * We only ever set the power-on and power-gate states, anything
 	 * else is unexpected.
@@ -1253,10 +1252,10 @@ static bool vlv_power_well_enabled(struct intel_display *display,
 	 * A transient state at this point would mean some unexpected party
 	 * is poking at the power controls too.
 	 */
-	ctrl = vlv_punit_read(display->drm, PUNIT_REG_PWRGT_CTRL) & mask;
+	ctrl = vlv_punit_read(display, PUNIT_REG_PWRGT_CTRL) & mask;
 	drm_WARN_ON(display->drm, ctrl != state);
 
-	vlv_punit_put(display->drm);
+	vlv_punit_put(display);
 
 	return enabled;
 }
@@ -1533,30 +1532,30 @@ static void chv_dpio_cmn_power_well_enable(struct intel_display *display,
 		drm_err(display->drm, "Display PHY %d is not power up\n",
 			phy);
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* Enable dynamic power down */
-	tmp = vlv_dpio_read(display->drm, phy, CHV_CMN_DW28);
+	tmp = vlv_dpio_read(display, phy, CHV_CMN_DW28);
 	tmp |= DPIO_DYNPWRDOWNEN_CH0 | DPIO_CL1POWERDOWNEN |
 		DPIO_SUS_CLK_CONFIG_GATE_CLKREQ;
-	vlv_dpio_write(display->drm, phy, CHV_CMN_DW28, tmp);
+	vlv_dpio_write(display, phy, CHV_CMN_DW28, tmp);
 
 	if (id == VLV_DISP_PW_DPIO_CMN_BC) {
-		tmp = vlv_dpio_read(display->drm, phy, CHV_CMN_DW6_CH1);
+		tmp = vlv_dpio_read(display, phy, CHV_CMN_DW6_CH1);
 		tmp |= DPIO_DYNPWRDOWNEN_CH1;
-		vlv_dpio_write(display->drm, phy, CHV_CMN_DW6_CH1, tmp);
+		vlv_dpio_write(display, phy, CHV_CMN_DW6_CH1, tmp);
 	} else {
 		/*
 		 * Force the non-existing CL2 off. BXT does this
 		 * too, so maybe it saves some power even though
 		 * CL2 doesn't exist?
 		 */
-		tmp = vlv_dpio_read(display->drm, phy, CHV_CMN_DW30);
+		tmp = vlv_dpio_read(display, phy, CHV_CMN_DW30);
 		tmp |= DPIO_CL2_LDOFUSE_PWRENB;
-		vlv_dpio_write(display->drm, phy, CHV_CMN_DW30, tmp);
+		vlv_dpio_write(display, phy, CHV_CMN_DW30, tmp);
 	}
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 
 	display->power.chv_phy_control |= PHY_COM_LANE_RESET_DEASSERT(phy);
 	intel_de_write(display, DISPLAY_PHY_CONTROL,
@@ -1624,9 +1623,9 @@ static void assert_chv_phy_powergate(struct intel_display *display, enum dpio_ph
 	else
 		reg = CHV_CMN_DW6_CH1;
 
-	vlv_dpio_get(display->drm);
-	val = vlv_dpio_read(display->drm, phy, reg);
-	vlv_dpio_put(display->drm);
+	vlv_dpio_get(display);
+	val = vlv_dpio_read(display, phy, reg);
+	vlv_dpio_put(display);
 
 	/*
 	 * This assumes !override is only used when the port is disabled.
@@ -1740,9 +1739,9 @@ static bool chv_pipe_power_well_enabled(struct intel_display *display,
 	bool enabled;
 	u32 state, ctrl;
 
-	vlv_punit_get(display->drm);
+	vlv_punit_get(display);
 
-	state = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe);
+	state = vlv_punit_read(display, PUNIT_REG_DSPSSPM) & DP_SSS_MASK(pipe);
 	/*
 	 * We only ever set the power-on and power-gate states, anything
 	 * else is unexpected.
@@ -1755,10 +1754,10 @@ static bool chv_pipe_power_well_enabled(struct intel_display *display,
 	 * A transient state at this point would mean some unexpected party
 	 * is poking at the power controls too.
 	 */
-	ctrl = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe);
+	ctrl = vlv_punit_read(display, PUNIT_REG_DSPSSPM) & DP_SSC_MASK(pipe);
 	drm_WARN_ON(display->drm, ctrl << 16 != state);
 
-	vlv_punit_put(display->drm);
+	vlv_punit_put(display);
 
 	return enabled;
 }
@@ -1774,29 +1773,29 @@ static void chv_set_pipe_power_well(struct intel_display *display,
 
 	state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
 
-	vlv_punit_get(display->drm);
+	vlv_punit_get(display);
 
-	ctrl = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM);
+	ctrl = vlv_punit_read(display, PUNIT_REG_DSPSSPM);
 	if ((ctrl & DP_SSS_MASK(pipe)) == state)
 		goto out;
 
 	ctrl &= ~DP_SSC_MASK(pipe);
 	ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
-	vlv_punit_write(display->drm, PUNIT_REG_DSPSSPM, ctrl);
+	vlv_punit_write(display, PUNIT_REG_DSPSSPM, ctrl);
 
-	ret = poll_timeout_us(ctrl = vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM),
+	ret = poll_timeout_us(ctrl = vlv_punit_read(display, PUNIT_REG_DSPSSPM),
 			      (ctrl & DP_SSS_MASK(pipe)) == state,
 			      500, 100 * 1000, false);
 	if (ret)
 		drm_err(display->drm,
 			"timeout setting power well state %08x (%08x)\n",
 			state,
-			vlv_punit_read(display->drm, PUNIT_REG_DSPSSPM));
+			vlv_punit_read(display, PUNIT_REG_DSPSSPM));
 
 #undef COND
 
 out:
-	vlv_punit_put(display->drm);
+	vlv_punit_put(display);
 }
 
 static void chv_pipe_power_well_sync_hw(struct intel_display *display,
diff --git a/drivers/gpu/drm/i915/display/intel_display_reg_defs.h b/drivers/gpu/drm/i915/display/intel_display_reg_defs.h
index 175334b41bba2..9220fcbfcb244 100644
--- a/drivers/gpu/drm/i915/display/intel_display_reg_defs.h
+++ b/drivers/gpu/drm/i915/display/intel_display_reg_defs.h
@@ -8,6 +8,41 @@
 
 #include "i915_reg_defs.h"
 
+typedef i915_reg_t intel_reg_t;
+
+static inline u32 intel_reg_offset(intel_reg_t r)
+{
+	return r.reg;
+}
+
+static inline bool intel_reg_equal(intel_reg_t a, intel_reg_t b)
+{
+	return intel_reg_offset(a) == intel_reg_offset(b);
+}
+
+static inline bool intel_reg_valid(intel_reg_t r)
+{
+	return !intel_reg_equal(r, INVALID_MMIO_REG);
+}
+
+/* A triplet for IMR/IER/IIR registers. */
+struct intel_irq_regs {
+	intel_reg_t imr;
+	intel_reg_t ier;
+	intel_reg_t iir;
+};
+
+#define INTEL_IRQ_REGS(_imr, _ier, _iir) \
+	((const struct intel_irq_regs){ .imr = (_imr), .ier = (_ier), .iir = (_iir) })
+
+struct intel_error_regs {
+	intel_reg_t emr;
+	intel_reg_t eir;
+};
+
+#define INTEL_ERROR_REGS(_emr, _eir) \
+	((const struct intel_error_regs){ .emr = (_emr), .eir = (_eir) })
+
 #define VLV_DISPLAY_BASE		0x180000
 
 /*
diff --git a/drivers/gpu/drm/i915/display/intel_display_regs.h b/drivers/gpu/drm/i915/display/intel_display_regs.h
index 4746e9ebd9200..4321f8b529dae 100644
--- a/drivers/gpu/drm/i915/display/intel_display_regs.h
+++ b/drivers/gpu/drm/i915/display/intel_display_regs.h
@@ -105,9 +105,9 @@
 #define   ERR_INT_FIFO_UNDERRUN_A	(1 << 0)
 #define   ERR_INT_FIFO_UNDERRUN(pipe)	(1 << ((pipe) * 3))
 
-#define VLV_IRQ_REGS		I915_IRQ_REGS(VLV_IMR, \
-					      VLV_IER, \
-					      VLV_IIR)
+#define VLV_IRQ_REGS		INTEL_IRQ_REGS(VLV_IMR, \
+					       VLV_IER, \
+					       VLV_IIR)
 
 #define VLV_EIR		_MMIO(VLV_DISPLAY_BASE + 0x20b0)
 #define VLV_EMR		_MMIO(VLV_DISPLAY_BASE + 0x20b4)
@@ -117,7 +117,7 @@
 #define   VLV_ERROR_PAGE_TABLE				(1 << 4)
 #define   VLV_ERROR_CLAIM				(1 << 0)
 
-#define VLV_ERROR_REGS		I915_ERROR_REGS(VLV_EMR, VLV_EIR)
+#define VLV_ERROR_REGS		INTEL_ERROR_REGS(VLV_EMR, VLV_EIR)
 
 #define _MBUS_ABOX0_CTL			0x45038
 #define _MBUS_ABOX1_CTL			0x45048
@@ -217,6 +217,34 @@
 # define DPIOUNIT_CLOCK_GATE_DISABLE		(1 << 6) /* 915-945 */
 # define OVFUNIT_CLOCK_GATE_DISABLE		(1 << 5)
 # define OVBUNIT_CLOCK_GATE_DISABLE		(1 << 4)
+
+#define ILK_DISPLAY_CHICKEN1		_MMIO(0x42000)
+#define   ILK_FBCQ_DIS			REG_BIT(22)
+#define   ILK_PABSTRETCH_DIS		REG_BIT(21)
+#define   ILK_SABSTRETCH_DIS		REG_BIT(20)
+#define   IVB_PRI_STRETCH_MAX_MASK	REG_GENMASK(21, 20)
+#define   IVB_PRI_STRETCH_MAX_X8	REG_FIELD_PREP(IVB_PRI_STRETCH_MAX_MASK, 0)
+#define   IVB_PRI_STRETCH_MAX_X4	REG_FIELD_PREP(IVB_PRI_STRETCH_MAX_MASK, 1)
+#define   IVB_PRI_STRETCH_MAX_X2	REG_FIELD_PREP(IVB_PRI_STRETCH_MAX_MASK, 2)
+#define   IVB_PRI_STRETCH_MAX_X1	REG_FIELD_PREP(IVB_PRI_STRETCH_MAX_MASK, 3)
+#define   IVB_SPR_STRETCH_MAX_MASK	REG_GENMASK(19, 18)
+#define   IVB_SPR_STRETCH_MAX_X8	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 0)
+#define   IVB_SPR_STRETCH_MAX_X4	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 1)
+#define   IVB_SPR_STRETCH_MAX_X2	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 2)
+#define   IVB_SPR_STRETCH_MAX_X1	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 3)
+
+#define ILK_DISPLAY_CHICKEN2		_MMIO(0x42004)
+/* Required on all Ironlake and Sandybridge according to the B-Spec. */
+#define   ILK_ELPIN_409_SELECT		REG_BIT(25)
+#define   ILK_DPARB_GATE		REG_BIT(22)
+#define   ILK_VSDPFD_FULL		REG_BIT(21)
+
+#define ILK_DSPCLK_GATE_D		_MMIO(0x42020)
+#define   ILK_VRHUNIT_CLOCK_GATE_DISABLE	REG_BIT(28)
+#define   ILK_DPFCUNIT_CLOCK_GATE_DISABLE	REG_BIT(9)
+#define   ILK_DPFCRUNIT_CLOCK_GATE_DISABLE	REG_BIT(8)
+#define   ILK_DPFDUNIT_CLOCK_GATE_ENABLE	REG_BIT(7)
+#define   ILK_DPARBUNIT_CLOCK_GATE_ENABLE	REG_BIT(5)
 /*
  * This bit must be set on the 830 to prevent hangs when turning off the
  * overlay scaler.
@@ -405,6 +433,9 @@
 #define   SKL_EDP_PSR_FIX_RDWRAP	REG_BIT(3)
 #define   IGNORE_PSR2_HW_TRACKING	REG_BIT(1)
 
+#define CHICKEN_PAR2_1		_MMIO(0x42090)
+#define   KVM_CONFIG_CHANGE_NOTIFICATION_SELECT	REG_BIT(14)
+
 /*
  * GEN9 clock gating regs
  */
@@ -1116,9 +1147,9 @@
 #define DEIIR   _MMIO(0x44008)
 #define DEIER   _MMIO(0x4400c)
 
-#define DE_IRQ_REGS		I915_IRQ_REGS(DEIMR, \
-					      DEIER, \
-					      DEIIR)
+#define DE_IRQ_REGS		INTEL_IRQ_REGS(DEIMR, \
+					       DEIER, \
+					       DEIIR)
 
 #define DIGITAL_PORT_HOTPLUG_CNTRL	_MMIO(0x44030)
 #define  DIGITAL_PORTA_HOTPLUG_ENABLE		(1 << 4)
@@ -1407,9 +1438,9 @@
 #define  GEN8_PIPE_VSYNC		REG_BIT(1)
 #define  GEN8_PIPE_VBLANK		REG_BIT(0)
 
-#define GEN8_DE_PIPE_IRQ_REGS(pipe)	I915_IRQ_REGS(GEN8_DE_PIPE_IMR(pipe), \
-						      GEN8_DE_PIPE_IER(pipe), \
-						      GEN8_DE_PIPE_IIR(pipe))
+#define GEN8_DE_PIPE_IRQ_REGS(pipe)	INTEL_IRQ_REGS(GEN8_DE_PIPE_IMR(pipe), \
+						       GEN8_DE_PIPE_IER(pipe), \
+						       GEN8_DE_PIPE_IIR(pipe))
 
 #define _HPD_PIN_DDI(hpd_pin)	((hpd_pin) - HPD_PORT_A)
 #define _HPD_PIN_TC(hpd_pin)	((hpd_pin) - HPD_PORT_TC1)
@@ -1446,9 +1477,9 @@
 #define  TGL_DE_PORT_AUX_DDIB		REG_BIT(1)
 #define  TGL_DE_PORT_AUX_DDIA		REG_BIT(0)
 
-#define GEN8_DE_PORT_IRQ_REGS		I915_IRQ_REGS(GEN8_DE_PORT_IMR, \
-						      GEN8_DE_PORT_IER, \
-						      GEN8_DE_PORT_IIR)
+#define GEN8_DE_PORT_IRQ_REGS		INTEL_IRQ_REGS(GEN8_DE_PORT_IMR, \
+						       GEN8_DE_PORT_IER, \
+						       GEN8_DE_PORT_IIR)
 
 /* interrupts */
 #define DE_MASTER_IRQ_CONTROL   (1 << 31)
@@ -1499,9 +1530,9 @@
 #define  XELPDP_PMDEMAND_RSP		REG_BIT(3)
 #define  XE2LPD_DBUF_OVERLAP_DETECTED	REG_BIT(1)
 
-#define GEN8_DE_MISC_IRQ_REGS		I915_IRQ_REGS(GEN8_DE_MISC_IMR, \
-						      GEN8_DE_MISC_IER, \
-						      GEN8_DE_MISC_IIR)
+#define GEN8_DE_MISC_IRQ_REGS		INTEL_IRQ_REGS(GEN8_DE_MISC_IMR, \
+						       GEN8_DE_MISC_IER, \
+						       GEN8_DE_MISC_IIR)
 
 #define GEN11_DISPLAY_INT_CTL		_MMIO(0x44200)
 #define  GEN11_DISPLAY_IRQ_ENABLE	(1 << 31)
@@ -1533,9 +1564,9 @@
 						 GEN11_TBT_HOTPLUG(HPD_PORT_TC2) | \
 						 GEN11_TBT_HOTPLUG(HPD_PORT_TC1))
 
-#define GEN11_DE_HPD_IRQ_REGS		I915_IRQ_REGS(GEN11_DE_HPD_IMR, \
-						      GEN11_DE_HPD_IER, \
-						      GEN11_DE_HPD_IIR)
+#define GEN11_DE_HPD_IRQ_REGS		INTEL_IRQ_REGS(GEN11_DE_HPD_IMR, \
+						       GEN11_DE_HPD_IER, \
+						       GEN11_DE_HPD_IIR)
 
 #define GEN11_TBT_HOTPLUG_CTL				_MMIO(0x44030)
 #define GEN11_TC_HOTPLUG_CTL				_MMIO(0x44038)
@@ -1557,9 +1588,9 @@
 #define  XELPDP_TBT_HOTPLUG(hpd_pin)		REG_BIT(_HPD_PIN_TC(hpd_pin))
 #define  XELPDP_TBT_HOTPLUG_MASK		REG_GENMASK(3, 0)
 
-#define PICAINTERRUPT_IRQ_REGS			I915_IRQ_REGS(PICAINTERRUPT_IMR, \
-							      PICAINTERRUPT_IER, \
-							      PICAINTERRUPT_IIR)
+#define PICAINTERRUPT_IRQ_REGS			INTEL_IRQ_REGS(PICAINTERRUPT_IMR, \
+							       PICAINTERRUPT_IER, \
+							       PICAINTERRUPT_IIR)
 
 #define XELPDP_PORT_HOTPLUG_CTL(hpd_pin)	_MMIO(0x16F270 + (_HPD_PIN_TC(hpd_pin) * 0x200))
 #define  XELPDP_TBT_HOTPLUG_ENABLE		REG_BIT(6)
@@ -1875,9 +1906,9 @@
 #define SDEIIR  _MMIO(0xc4008)
 #define SDEIER  _MMIO(0xc400c)
 
-#define SDE_IRQ_REGS			I915_IRQ_REGS(SDEIMR, \
-						      SDEIER, \
-						      SDEIIR)
+#define SDE_IRQ_REGS			INTEL_IRQ_REGS(SDEIMR, \
+						       SDEIER, \
+						       SDEIIR)
 
 #define SERR_INT			_MMIO(0xc4040)
 #define  SERR_INT_POISON		(1 << 31)
@@ -2263,6 +2294,7 @@
 #define TRANS_DP2_CTL(trans)			_MMIO_TRANS(trans, _TRANS_DP2_CTL_A, _TRANS_DP2_CTL_B)
 #define  TRANS_DP2_128B132B_CHANNEL_CODING	REG_BIT(31)
 #define  TRANS_DP2_PANEL_REPLAY_ENABLE		REG_BIT(30)
+#define  TRANS_DP2_PR_TUNNELING_ENABLE		REG_BIT(26)
 #define  TRANS_DP2_DEBUG_ENABLE			REG_BIT(23)
 
 #define _TRANS_DP2_VFREQHIGH_A			0x600a4
diff --git a/drivers/gpu/drm/i915/display/intel_display_reset.c b/drivers/gpu/drm/i915/display/intel_display_reset.c
index d00ef5bdcbda9..504ec7043cd91 100644
--- a/drivers/gpu/drm/i915/display/intel_display_reset.c
+++ b/drivers/gpu/drm/i915/display/intel_display_reset.c
@@ -3,6 +3,8 @@
  * Copyright © 2023 Intel Corporation
  */
 
+#include <linux/debugfs.h>
+
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_print.h>
 
@@ -16,28 +18,23 @@
 #include "intel_hotplug.h"
 #include "intel_pps.h"
 
+bool intel_display_reset_supported(struct intel_display *display)
+{
+	return display && HAS_DISPLAY(display);
+}
+
 bool intel_display_reset_test(struct intel_display *display)
 {
-	return display->params.force_reset_modeset_test;
+	return display && HAS_DISPLAY(display) &&
+		display->params.force_reset_modeset_test;
 }
 
-/* returns true if intel_display_reset_finish() needs to be called */
-bool intel_display_reset_prepare(struct intel_display *display,
-				 modeset_stuck_fn modeset_stuck, void *context)
+void intel_display_reset_prepare(struct intel_display *display)
 {
 	struct drm_modeset_acquire_ctx *ctx = &display->restore.reset_ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	int ret;
 
-	if (!HAS_DISPLAY(display))
-		return false;
-
-	if (atomic_read(&display->restore.pending_fb_pin)) {
-		drm_dbg_kms(display->drm,
-			    "Modeset potentially stuck, unbreaking through wedging\n");
-		modeset_stuck(context);
-	}
-
 	/*
 	 * Need mode_config.mutex so that we don't
 	 * trample ongoing ->detect() and whatnot.
@@ -60,32 +57,28 @@ bool intel_display_reset_prepare(struct intel_display *display,
 		ret = PTR_ERR(state);
 		drm_err(display->drm, "Duplicating state failed with %i\n",
 			ret);
-		return true;
+		return;
 	}
 
 	ret = drm_atomic_helper_disable_all(display->drm, ctx);
 	if (ret) {
 		drm_err(display->drm, "Suspending crtc's failed with %i\n",
 			ret);
-		drm_atomic_state_put(state);
-		return true;
+		drm_atomic_commit_put(state);
+		return;
 	}
 
+	display->reset.count++;
 	display->restore.modeset_state = state;
 	state->acquire_ctx = ctx;
-
-	return true;
 }
 
 void intel_display_reset_finish(struct intel_display *display, bool test_only)
 {
 	struct drm_modeset_acquire_ctx *ctx = &display->restore.reset_ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	int ret;
 
-	if (!HAS_DISPLAY(display))
-		return;
-
 	state = fetch_and_zero(&display->restore.modeset_state);
 	if (!state)
 		goto unlock;
@@ -118,9 +111,16 @@ void intel_display_reset_finish(struct intel_display *display, bool test_only)
 		intel_hpd_poll_disable(display);
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 unlock:
 	drm_modeset_drop_locks(ctx);
 	drm_modeset_acquire_fini(ctx);
 	mutex_unlock(&display->drm->mode_config.mutex);
 }
+
+void intel_display_reset_debugfs_register(struct intel_display *display)
+{
+	debugfs_create_u32("intel_display_reset_count", 0400,
+			   display->drm->debugfs_root,
+			   &display->reset.count);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_display_reset.h b/drivers/gpu/drm/i915/display/intel_display_reset.h
index 8b3bda134454e..b88c330a3441b 100644
--- a/drivers/gpu/drm/i915/display/intel_display_reset.h
+++ b/drivers/gpu/drm/i915/display/intel_display_reset.h
@@ -10,11 +10,11 @@
 
 struct intel_display;
 
-typedef void modeset_stuck_fn(void *context);
-
+bool intel_display_reset_supported(struct intel_display *display);
 bool intel_display_reset_test(struct intel_display *display);
-bool intel_display_reset_prepare(struct intel_display *display,
-				 modeset_stuck_fn modeset_stuck, void *context);
+void intel_display_reset_prepare(struct intel_display *display);
 void intel_display_reset_finish(struct intel_display *display, bool test_only);
 
+void intel_display_reset_debugfs_register(struct intel_display *display);
+
 #endif /* __INTEL_RESET_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h
index f6cd0a0620908..f44be5c689aef 100644
--- a/drivers/gpu/drm/i915/display/intel_display_types.h
+++ b/drivers/gpu/drm/i915/display/intel_display_types.h
@@ -584,6 +584,7 @@ struct intel_connector {
 
 		struct {
 			u8 dpcd[EDP_PSR_RECEIVER_CAP_SIZE];
+			u8 intel_wa_dpcd;
 
 			bool support;
 			bool su_support;
@@ -633,7 +634,7 @@ struct dpll {
 };
 
 struct intel_atomic_state {
-	struct drm_atomic_state base;
+	struct drm_atomic_commit base;
 
 	struct ref_tracker *wakeref;
 
@@ -683,14 +684,15 @@ struct intel_plane_state {
 
 	struct i915_vma *ggtt_vma;
 	struct i915_vma *dpt_vma;
-	unsigned long flags;
-#define PLANE_HAS_FENCE BIT(0)
 
 	struct intel_fb_view view;
 
 	/* for legacy cursor fb unpin */
 	struct drm_vblank_work unpin_work;
 
+	/* fenced region ID (-1 if none) */
+	s8 fence_id;
+
 	/* Plane pxp decryption state */
 	bool decrypt;
 
@@ -755,9 +757,7 @@ struct intel_plane_state {
 };
 
 struct intel_initial_plane_config {
-	struct intel_framebuffer *fb;
-	struct intel_memory_region *mem;
-	resource_size_t phys_base;
+	struct drm_framebuffer *fb;
 	struct i915_vma *vma;
 	int size;
 	u32 base;
@@ -997,7 +997,7 @@ struct intel_casf {
 	struct scaler_filter_coeff coeff[SCALER_FILTER_NUM_TAPS];
 	u8 strength;
 	u8 win_size;
-	bool casf_enable;
+	bool enable;
 };
 
 struct intel_crtc_state {
@@ -1036,7 +1036,7 @@ struct intel_crtc_state {
 		struct drm_property_blob *degamma_lut, *gamma_lut, *ctm;
 		struct drm_display_mode mode, pipe_mode, adjusted_mode;
 		enum drm_scaling_filter scaling_filter;
-		struct intel_casf casf_params;
+		u8 sharpness_strength;
 	} hw;
 
 	/* actual state of LUTs */
@@ -1223,6 +1223,7 @@ struct intel_crtc_state {
 
 	/* Panel fitter placement and size for Ironlake+ */
 	struct {
+		struct intel_casf casf;
 		struct drm_rect dst;
 		bool enabled;
 		bool force_thru;
@@ -1662,7 +1663,7 @@ struct intel_plane {
 	container_of_const((fb), struct intel_framebuffer, base)
 
 struct intel_hdmi {
-	i915_reg_t hdmi_reg;
+	intel_reg_t hdmi_reg;
 	struct {
 		enum drm_dp_dual_mode_type type;
 		int max_tmds_clock;
@@ -1792,7 +1793,7 @@ struct intel_psr {
 };
 
 struct intel_dp {
-	i915_reg_t output_reg;
+	intel_reg_t output_reg;
 	u32 DP;
 	int link_rate;
 	u8 lane_count;
@@ -1889,8 +1890,8 @@ struct intel_dp {
 	u32 (*get_aux_send_ctl)(struct intel_dp *dp, int send_bytes,
 				u32 aux_clock_divider);
 
-	i915_reg_t (*aux_ch_ctl_reg)(struct intel_dp *dp);
-	i915_reg_t (*aux_ch_data_reg)(struct intel_dp *dp, int index);
+	intel_reg_t (*aux_ch_ctl_reg)(struct intel_dp *dp);
+	intel_reg_t (*aux_ch_data_reg)(struct intel_dp *dp, int index);
 
 	/* This is called before a link training is starterd */
 	void (*prepare_link_retrain)(struct intel_dp *intel_dp,
@@ -2117,7 +2118,7 @@ static inline bool intel_encoder_is_dp(struct intel_encoder *encoder)
 		return true;
 	case INTEL_OUTPUT_DDI:
 		/* Skip pure HDMI/DVI DDI encoders */
-		return i915_mmio_reg_valid(enc_to_intel_dp(encoder)->output_reg);
+		return intel_reg_valid(enc_to_intel_dp(encoder)->output_reg);
 	default:
 		return false;
 	}
@@ -2130,7 +2131,7 @@ static inline bool intel_encoder_is_hdmi(struct intel_encoder *encoder)
 		return true;
 	case INTEL_OUTPUT_DDI:
 		/* See if the HDMI encoder is valid. */
-		return i915_mmio_reg_valid(enc_to_intel_hdmi(encoder)->hdmi_reg);
+		return intel_reg_valid(enc_to_intel_hdmi(encoder)->hdmi_reg);
 	default:
 		return false;
 	}
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.c b/drivers/gpu/drm/i915/display/intel_display_wa.c
index 081a4092cd13e..2094eda09c912 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.c
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.c
@@ -4,12 +4,12 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/step.h>
 
 #include "intel_de.h"
 #include "intel_display_core.h"
 #include "intel_display_regs.h"
 #include "intel_display_wa.h"
-#include "intel_step.h"
 
 static void gen11_display_wa_apply(struct intel_display *display)
 {
@@ -136,6 +136,8 @@ bool __intel_display_wa(struct intel_display *display, enum intel_display_wa wa,
 		return DISPLAY_VER(display) == 20 &&
 			IS_DISPLAY_VERx100_STEP(display, 3000,
 						STEP_A0, STEP_B0);
+	case INTEL_DISPLAY_WA_16029024088:
+		return DISPLAY_VER(display) >= 35;
 	case INTEL_DISPLAY_WA_18034343758:
 		return DISPLAY_VER(display) == 20 ||
 			(display->platform.pantherlake &&
diff --git a/drivers/gpu/drm/i915/display/intel_display_wa.h b/drivers/gpu/drm/i915/display/intel_display_wa.h
index 15fec843f15e1..9cdd148ea4fad 100644
--- a/drivers/gpu/drm/i915/display/intel_display_wa.h
+++ b/drivers/gpu/drm/i915/display/intel_display_wa.h
@@ -52,6 +52,7 @@ enum intel_display_wa {
 	INTEL_DISPLAY_WA_16023588340,
 	INTEL_DISPLAY_WA_16025573575,
 	INTEL_DISPLAY_WA_16025596647,
+	INTEL_DISPLAY_WA_16029024088,
 	INTEL_DISPLAY_WA_18034343758,
 	INTEL_DISPLAY_WA_22010178259,
 	INTEL_DISPLAY_WA_22010947358,
diff --git a/drivers/gpu/drm/i915/display/intel_dmc.c b/drivers/gpu/drm/i915/display/intel_dmc.c
index 1667a829e708a..481fb65b71102 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc.c
+++ b/drivers/gpu/drm/i915/display/intel_dmc.c
@@ -71,7 +71,7 @@ struct intel_dmc {
 	} dc6_allowed;
 	struct dmc_fw_info {
 		u32 mmio_count;
-		i915_reg_t mmioaddr[20];
+		intel_reg_t mmioaddr[20];
 		u32 mmiodata[20];
 		u32 dmc_offset;
 		u32 start_mmioaddr;
@@ -434,7 +434,7 @@ static void gen9_set_dc_state_debugmask(struct intel_display *display)
 }
 
 static void disable_event_handler(struct intel_display *display,
-				  i915_reg_t ctl_reg, i915_reg_t htp_reg)
+				  intel_reg_t ctl_reg, intel_reg_t htp_reg)
 {
 	intel_de_write(display, ctl_reg,
 		       REG_FIELD_PREP(DMC_EVT_CTL_TYPE_MASK,
@@ -505,6 +505,14 @@ static void pipedmc_clock_gating_wa(struct intel_display *display, bool enable)
 
 static u32 pipedmc_interrupt_mask(struct intel_display *display)
 {
+	/*
+	 * TODO: Check if PIPEDMC_ERROR bit enabling causes errors
+	 * on PTL, enable it if validation passes
+	 */
+	if (DISPLAY_VER(display) >= 35)
+		return PIPEDMC_FLIPQ_PROG_DONE |
+			PIPEDMC_ERROR;
+
 	/*
 	 * FIXME PIPEDMC_ERROR not enabled for now due to LNL pipe B
 	 * triggering it during the first DC state transition. Figure
@@ -530,21 +538,21 @@ static u32 dmc_evt_ctl_disable(u32 dmc_evt_ctl)
 }
 
 static bool is_dmc_evt_ctl_reg(struct intel_display *display,
-			       enum intel_dmc_id dmc_id, i915_reg_t reg)
+			       enum intel_dmc_id dmc_id, intel_reg_t reg)
 {
-	u32 offset = i915_mmio_reg_offset(reg);
-	u32 start = i915_mmio_reg_offset(DMC_EVT_CTL(display, dmc_id, 0));
-	u32 end = i915_mmio_reg_offset(DMC_EVT_CTL(display, dmc_id, DMC_EVENT_HANDLER_COUNT_GEN12));
+	u32 offset = intel_reg_offset(reg);
+	u32 start = intel_reg_offset(DMC_EVT_CTL(display, dmc_id, 0));
+	u32 end = intel_reg_offset(DMC_EVT_CTL(display, dmc_id, DMC_EVENT_HANDLER_COUNT_GEN12));
 
 	return offset >= start && offset < end;
 }
 
 static bool is_dmc_evt_htp_reg(struct intel_display *display,
-			       enum intel_dmc_id dmc_id, i915_reg_t reg)
+			       enum intel_dmc_id dmc_id, intel_reg_t reg)
 {
-	u32 offset = i915_mmio_reg_offset(reg);
-	u32 start = i915_mmio_reg_offset(DMC_EVT_HTP(display, dmc_id, 0));
-	u32 end = i915_mmio_reg_offset(DMC_EVT_HTP(display, dmc_id, DMC_EVENT_HANDLER_COUNT_GEN12));
+	u32 offset = intel_reg_offset(reg);
+	u32 start = intel_reg_offset(DMC_EVT_HTP(display, dmc_id, 0));
+	u32 end = intel_reg_offset(DMC_EVT_HTP(display, dmc_id, DMC_EVENT_HANDLER_COUNT_GEN12));
 
 	return offset >= start && offset < end;
 }
@@ -552,7 +560,7 @@ static bool is_dmc_evt_htp_reg(struct intel_display *display,
 static bool is_event_handler(struct intel_display *display,
 			     enum intel_dmc_id dmc_id,
 			     unsigned int event_id,
-			     i915_reg_t reg, u32 data)
+			     intel_reg_t reg, u32 data)
 {
 	return is_dmc_evt_ctl_reg(display, dmc_id, reg) &&
 		REG_FIELD_GET(DMC_EVT_CTL_EVENT_ID_MASK, data) == event_id;
@@ -560,8 +568,8 @@ static bool is_event_handler(struct intel_display *display,
 
 static bool fixup_dmc_evt(struct intel_display *display,
 			  enum intel_dmc_id dmc_id,
-			  i915_reg_t reg_ctl, u32 *data_ctl,
-			  i915_reg_t reg_htp, u32 *data_htp)
+			  intel_reg_t reg_ctl, u32 *data_ctl,
+			  intel_reg_t reg_htp, u32 *data_htp)
 {
 	if (!is_dmc_evt_ctl_reg(display, dmc_id, reg_ctl))
 		return false;
@@ -570,8 +578,8 @@ static bool fixup_dmc_evt(struct intel_display *display,
 		return false;
 
 	/* make sure reg_ctl and reg_htp are for the same event */
-	if (i915_mmio_reg_offset(reg_ctl) - i915_mmio_reg_offset(DMC_EVT_CTL(display, dmc_id, 0)) !=
-	    i915_mmio_reg_offset(reg_htp) - i915_mmio_reg_offset(DMC_EVT_HTP(display, dmc_id, 0)))
+	if (intel_reg_offset(reg_ctl) - intel_reg_offset(DMC_EVT_CTL(display, dmc_id, 0)) !=
+	    intel_reg_offset(reg_htp) - intel_reg_offset(DMC_EVT_HTP(display, dmc_id, 0)))
 		return false;
 
 	/*
@@ -605,7 +613,7 @@ static bool fixup_dmc_evt(struct intel_display *display,
 
 static bool disable_dmc_evt(struct intel_display *display,
 			    enum intel_dmc_id dmc_id,
-			    i915_reg_t reg, u32 data)
+			    intel_reg_t reg, u32 data)
 {
 	if (!is_dmc_evt_ctl_reg(display, dmc_id, reg))
 		return false;
@@ -688,14 +696,14 @@ static void assert_dmc_loaded(struct intel_display *display,
 		 dmc_id, expected, found);
 
 	for (i = 0; i < dmc->dmc_info[dmc_id].mmio_count; i++) {
-		i915_reg_t reg = dmc->dmc_info[dmc_id].mmioaddr[i];
+		intel_reg_t reg = dmc->dmc_info[dmc_id].mmioaddr[i];
 
 		found = intel_de_read(display, reg);
 		expected = dmc_mmiodata(display, dmc, dmc_id, i);
 
 		drm_WARN(display->drm, found != expected,
 			 "DMC %d mmio[%d]/0x%x incorrect (expected 0x%x, current 0x%x)\n",
-			 dmc_id, i, i915_mmio_reg_offset(reg), expected, found);
+			 dmc_id, i, intel_reg_offset(reg), expected, found);
 	}
 }
 
@@ -839,7 +847,7 @@ static void dmc_configure_event(struct intel_display *display,
 	int i;
 
 	for (i = 0; i < dmc->dmc_info[dmc_id].mmio_count; i++) {
-		i915_reg_t reg = dmc->dmc_info[dmc_id].mmioaddr[i];
+		intel_reg_t reg = dmc->dmc_info[dmc_id].mmioaddr[i];
 		u32 data = dmc->dmc_info[dmc_id].mmiodata[i];
 
 		if (!is_event_handler(display, dmc_id, event_id, reg, data))
@@ -1138,17 +1146,17 @@ static u32 parse_dmc_fw_header(struct intel_dmc *dmc,
 
 		drm_dbg_kms(display->drm,
 			    " mmio[%d]: 0x%x = 0x%x->0x%x (EVT_CTL)\n",
-			    i, i915_mmio_reg_offset(dmc_info->mmioaddr[i]),
+			    i, intel_reg_offset(dmc_info->mmioaddr[i]),
 			    orig_mmiodata[0], dmc_info->mmiodata[i]);
 		drm_dbg_kms(display->drm,
 			    " mmio[%d]: 0x%x = 0x%x->0x%x (EVT_HTP)\n",
-			    i+1, i915_mmio_reg_offset(dmc_info->mmioaddr[i+1]),
+			    i+1, intel_reg_offset(dmc_info->mmioaddr[i+1]),
 			    orig_mmiodata[1], dmc_info->mmiodata[i+1]);
 	}
 
 	for (i = 0; i < mmio_count; i++) {
 		drm_dbg_kms(display->drm, " mmio[%d]: 0x%x = 0x%x%s%s\n",
-			    i, i915_mmio_reg_offset(dmc_info->mmioaddr[i]), dmc_info->mmiodata[i],
+			    i, intel_reg_offset(dmc_info->mmioaddr[i]), dmc_info->mmiodata[i],
 			    is_dmc_evt_ctl_reg(display, dmc_id, dmc_info->mmioaddr[i]) ? " (EVT_CTL)" :
 			    is_dmc_evt_htp_reg(display, dmc_id, dmc_info->mmioaddr[i]) ? " (EVT_HTP)" : "",
 			    disable_dmc_evt(display, dmc_id, dmc_info->mmioaddr[i],
@@ -1574,10 +1582,10 @@ void intel_dmc_update_dc6_allowed_count(struct intel_display *display,
 	struct intel_dmc *dmc = display_to_dmc(display);
 	u32 dc5_cur_count;
 
-	if (DISPLAY_VER(dmc->display) < 14)
+	if (DISPLAY_VER(display) < 14)
 		return;
 
-	dc5_cur_count = intel_de_read(dmc->display, DG1_DMC_DEBUG_DC5_COUNT);
+	dc5_cur_count = intel_de_read(display, DG1_DMC_DEBUG_DC5_COUNT);
 
 	if (!start_tracking)
 		dmc->dc6_allowed.count += dc5_cur_count - dmc->dc6_allowed.dc5_start;
@@ -1610,7 +1618,7 @@ static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused)
 	struct intel_display *display = m->private;
 	struct intel_dmc *dmc = display_to_dmc(display);
 	struct ref_tracker *wakeref;
-	i915_reg_t dc5_reg, dc6_reg = INVALID_MMIO_REG;
+	intel_reg_t dc5_reg, dc6_reg = INVALID_MMIO_REG;
 	u32 dc6_allowed_count;
 
 	if (!HAS_DMC(display))
@@ -1639,7 +1647,7 @@ static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused)
 		   DMC_VERSION_MINOR(dmc->version));
 
 	if (DISPLAY_VER(display) >= 12) {
-		i915_reg_t dc3co_reg;
+		intel_reg_t dc3co_reg;
 
 		if (display->platform.dgfx || DISPLAY_VER(display) >= 14) {
 			dc3co_reg = DG1_DMC_DEBUG3;
@@ -1664,7 +1672,7 @@ static int intel_dmc_debugfs_status_show(struct seq_file *m, void *unused)
 	if (intel_dmc_get_dc6_allowed_count(display, &dc6_allowed_count))
 		seq_printf(m, "DC5 -> DC6 allowed count: %d\n",
 			   dc6_allowed_count);
-	else if (i915_mmio_reg_valid(dc6_reg))
+	else if (intel_reg_valid(dc6_reg))
 		seq_printf(m, "DC5 -> DC6 count: %d\n",
 			   intel_de_read(display, dc6_reg));
 
diff --git a/drivers/gpu/drm/i915/display/intel_dmc_wl.c b/drivers/gpu/drm/i915/display/intel_dmc_wl.c
index 73a3101514f3b..b007343721e12 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc_wl.c
+++ b/drivers/gpu/drm/i915/display/intel_dmc_wl.c
@@ -39,6 +39,13 @@
  * current implementation, we only need one wakelock, so only
  * DMC_WAKELOCK1_CTL is used.  The other definitions are here for
  * potential future use.
+ *
+ * This is available starting with Xe2_LPD (display version 20) as an
+ * experimental feature and on Xe3_LPD (display version 30) as the
+ * first display release with official support.  That means that we
+ * only enable the feature by default on the latter and using it on
+ * the former requires explicitly using the enable_dmc_wl module
+ * parameter.
  */
 
 /*
@@ -46,7 +53,7 @@
  * atomic variant of waiting MMIO.
  */
 #define DMC_WAKELOCK_CTL_TIMEOUT_US 5000
-#define DMC_WAKELOCK_HOLD_TIME 50
+#define DMC_WAKELOCK_HOLD_TIME 5
 
 /*
  * Possible non-negative values for the enable_dmc_wl param.
@@ -224,10 +231,10 @@ static void __intel_dmc_wl_take(struct intel_display *display)
 	wl->taken = true;
 }
 
-static bool intel_dmc_wl_reg_in_range(i915_reg_t reg,
+static bool intel_dmc_wl_reg_in_range(intel_reg_t reg,
 				      const struct intel_dmc_wl_range ranges[])
 {
-	u32 offset = i915_mmio_reg_offset(reg);
+	u32 offset = intel_reg_offset(reg);
 
 	for (int i = 0; ranges[i].start; i++) {
 		u32 end = ranges[i].end ?: ranges[i].start;
@@ -240,7 +247,7 @@ static bool intel_dmc_wl_reg_in_range(i915_reg_t reg,
 }
 
 static bool intel_dmc_wl_check_range(struct intel_display *display,
-				     i915_reg_t reg,
+				     intel_reg_t reg,
 				     u32 dc_state)
 {
 	const struct intel_dmc_wl_range *ranges;
@@ -286,7 +293,7 @@ static void intel_dmc_wl_sanitize_param(struct intel_display *display)
 {
 	const char *desc;
 
-	if (!HAS_DMC_WAKELOCK(display)) {
+	if (DISPLAY_VER(display) < 20) {
 		display->params.enable_dmc_wl = ENABLE_DMC_WL_DISABLED;
 	} else if (display->params.enable_dmc_wl < 0) {
 		if (DISPLAY_VER(display) >= 30)
@@ -431,7 +438,7 @@ void intel_dmc_wl_flush_release_work(struct intel_display *display)
 	flush_delayed_work(&wl->work);
 }
 
-void intel_dmc_wl_get(struct intel_display *display, i915_reg_t reg)
+void intel_dmc_wl_get(struct intel_display *display, intel_reg_t reg)
 {
 	struct intel_dmc_wl *wl = &display->wl;
 	unsigned long flags;
@@ -441,7 +448,7 @@ void intel_dmc_wl_get(struct intel_display *display, i915_reg_t reg)
 
 	spin_lock_irqsave(&wl->lock, flags);
 
-	if (i915_mmio_reg_valid(reg) &&
+	if (intel_reg_valid(reg) &&
 	    !intel_dmc_wl_check_range(display, reg, wl->dc_state))
 		goto out_unlock;
 
@@ -464,7 +471,7 @@ void intel_dmc_wl_get(struct intel_display *display, i915_reg_t reg)
 	spin_unlock_irqrestore(&wl->lock, flags);
 }
 
-void intel_dmc_wl_put(struct intel_display *display, i915_reg_t reg)
+void intel_dmc_wl_put(struct intel_display *display, intel_reg_t reg)
 {
 	struct intel_dmc_wl *wl = &display->wl;
 	unsigned long flags;
@@ -474,7 +481,7 @@ void intel_dmc_wl_put(struct intel_display *display, i915_reg_t reg)
 
 	spin_lock_irqsave(&wl->lock, flags);
 
-	if (i915_mmio_reg_valid(reg) &&
+	if (intel_reg_valid(reg) &&
 	    !intel_dmc_wl_check_range(display, reg, wl->dc_state))
 		goto out_unlock;
 
diff --git a/drivers/gpu/drm/i915/display/intel_dmc_wl.h b/drivers/gpu/drm/i915/display/intel_dmc_wl.h
index 5488fbdf29b89..215107f0a4f95 100644
--- a/drivers/gpu/drm/i915/display/intel_dmc_wl.h
+++ b/drivers/gpu/drm/i915/display/intel_dmc_wl.h
@@ -10,7 +10,7 @@
 #include <linux/workqueue.h>
 #include <linux/refcount.h>
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 
 struct intel_display;
 
@@ -33,8 +33,8 @@ void intel_dmc_wl_init(struct intel_display *display);
 void intel_dmc_wl_enable(struct intel_display *display, u32 dc_state);
 void intel_dmc_wl_disable(struct intel_display *display);
 void intel_dmc_wl_flush_release_work(struct intel_display *display);
-void intel_dmc_wl_get(struct intel_display *display, i915_reg_t reg);
-void intel_dmc_wl_put(struct intel_display *display, i915_reg_t reg);
+void intel_dmc_wl_get(struct intel_display *display, intel_reg_t reg);
+void intel_dmc_wl_put(struct intel_display *display, intel_reg_t reg);
 void intel_dmc_wl_get_noreg(struct intel_display *display);
 void intel_dmc_wl_put_noreg(struct intel_display *display);
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c
index 50d34b4fdf82a..1920d2f026665 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.c
+++ b/drivers/gpu/drm/i915/display/intel_dp.c
@@ -1234,28 +1234,11 @@ int intel_dp_output_format_link_bpp_x16(enum intel_output_format output_format,
 	return fxp_q4_from_int(pipe_bpp);
 }
 
-static enum intel_output_format
-intel_dp_sink_format(struct intel_connector *connector,
-		     const struct drm_display_mode *mode)
-{
-	const struct drm_display_info *info = &connector->base.display_info;
-
-	if (drm_mode_is_420_only(info, mode))
-		return INTEL_OUTPUT_FORMAT_YCBCR420;
-
-	return INTEL_OUTPUT_FORMAT_RGB;
-}
-
 static int
 intel_dp_mode_min_link_bpp_x16(struct intel_connector *connector,
-			       const struct drm_display_mode *mode)
+			       const struct drm_display_mode *mode,
+			       enum intel_output_format output_format)
 {
-	enum intel_output_format output_format, sink_format;
-
-	sink_format = intel_dp_sink_format(connector, mode);
-
-	output_format = intel_dp_output_format(connector, sink_format);
-
 	return intel_dp_output_format_link_bpp_x16(output_format,
 						   intel_dp_min_bpp(output_format));
 }
@@ -1317,23 +1300,29 @@ intel_dp_tmds_clock_valid(struct intel_dp *intel_dp,
 	return MODE_OK;
 }
 
+static int frl_required_bw(int clock, int bpc,
+			   enum intel_output_format sink_format)
+{
+	if (sink_format == INTEL_OUTPUT_FORMAT_YCBCR420)
+		clock /= 2;
+
+	return clock * bpc * 3;
+}
+
 static enum drm_mode_status
 intel_dp_mode_valid_downstream(struct intel_connector *connector,
 			       const struct drm_display_mode *mode,
-			       int target_clock)
+			       int target_clock,
+			       enum intel_output_format sink_format)
 {
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	const struct drm_display_info *info = &connector->base.display_info;
-	enum drm_mode_status status;
-	enum intel_output_format sink_format;
 
 	/* If PCON supports FRL MODE, check FRL bandwidth constraints */
 	if (intel_dp->dfp.pcon_max_frl_bw) {
-		int link_bpp_x16 = intel_dp_mode_min_link_bpp_x16(connector, mode);
-		int target_bw;
-		int max_frl_bw;
+		int target_bw, max_frl_bw;
 
-		target_bw = fxp_q4_to_int_roundup(link_bpp_x16) * target_clock;
+		/* Assume 8bpc for the FRL bandwidth check */
+		target_bw = frl_required_bw(target_clock, 8, sink_format);
 
 		max_frl_bw = intel_dp->dfp.pcon_max_frl_bw;
 
@@ -1350,25 +1339,36 @@ intel_dp_mode_valid_downstream(struct intel_connector *connector,
 	    target_clock > intel_dp->dfp.max_dotclock)
 		return MODE_CLOCK_HIGH;
 
-	sink_format = intel_dp_sink_format(connector, mode);
-
 	/* Assume 8bpc for the DP++/HDMI/DVI TMDS clock check */
-	status = intel_dp_tmds_clock_valid(intel_dp, target_clock,
-					   8, sink_format, true);
+	return intel_dp_tmds_clock_valid(intel_dp, target_clock,
+					 8, sink_format, true);
+}
 
-	if (status != MODE_OK) {
-		if (sink_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
-		    !connector->base.ycbcr_420_allowed ||
-		    !drm_mode_is_420_also(info, mode))
-			return status;
-		sink_format = INTEL_OUTPUT_FORMAT_YCBCR420;
-		status = intel_dp_tmds_clock_valid(intel_dp, target_clock,
-						   8, sink_format, true);
-		if (status != MODE_OK)
-			return status;
-	}
+static enum drm_mode_status
+intel_dp_sink_format_valid(struct intel_connector *connector,
+			   const struct drm_display_mode *mode,
+			   enum intel_output_format sink_format)
+{
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	const struct drm_display_info *info = &connector->base.display_info;
 
-	return MODE_OK;
+	switch (sink_format) {
+	case INTEL_OUTPUT_FORMAT_YCBCR420:
+		if (intel_dp->dfp.min_tmds_clock &&
+		    !intel_dp_has_hdmi_sink(intel_dp))
+			return MODE_NO_420;
+
+		if (!connector->base.ycbcr_420_allowed ||
+		    !drm_mode_is_420(info, mode))
+			return MODE_NO_420;
+
+		return MODE_OK;
+	case INTEL_OUTPUT_FORMAT_RGB:
+		return MODE_OK;
+	default:
+		MISSING_CASE(sink_format);
+		return MODE_BAD;
+	}
 }
 
 int intel_dp_max_hdisplay_per_pipe(struct intel_display *display)
@@ -1442,15 +1442,14 @@ bool intel_dp_dotclk_valid(struct intel_display *display,
 }
 
 static enum drm_mode_status
-intel_dp_mode_valid(struct drm_connector *_connector,
-		    const struct drm_display_mode *mode)
+intel_dp_mode_valid_format(struct intel_connector *connector,
+			   const struct drm_display_mode *mode,
+			   int target_clock,
+			   enum intel_output_format sink_format)
 {
-	struct intel_display *display = to_intel_display(_connector->dev);
-	struct intel_connector *connector = to_intel_connector(_connector);
+	struct intel_display *display = to_intel_display(connector);
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
-	enum intel_output_format sink_format, output_format;
-	const struct drm_display_mode *fixed_mode;
-	int target_clock = mode->clock;
+	enum intel_output_format output_format;
 	int max_rate, mode_rate, max_lanes, max_link_clock;
 	u16 dsc_max_compressed_bpp = 0;
 	enum drm_mode_status status;
@@ -1458,29 +1457,10 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 	int num_joined_pipes;
 	int link_bpp_x16;
 
-	status = intel_cpu_transcoder_mode_valid(display, mode);
+	status = intel_dp_sink_format_valid(connector, mode, sink_format);
 	if (status != MODE_OK)
 		return status;
 
-	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
-		return MODE_H_ILLEGAL;
-
-	if (mode->clock < 10000)
-		return MODE_CLOCK_LOW;
-
-	if (intel_dp_hdisplay_bad(display, mode->hdisplay))
-		return MODE_H_ILLEGAL;
-
-	fixed_mode = intel_panel_fixed_mode(connector, mode);
-	if (intel_dp_is_edp(intel_dp) && fixed_mode) {
-		status = intel_panel_mode_valid(connector, mode);
-		if (status != MODE_OK)
-			return status;
-
-		target_clock = fixed_mode->clock;
-	}
-
-	sink_format = intel_dp_sink_format(connector, mode);
 	output_format = intel_dp_output_format(connector, sink_format);
 
 	max_link_clock = intel_dp_max_link_rate(intel_dp);
@@ -1488,7 +1468,8 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 
 	max_rate = intel_dp_max_link_data_rate(intel_dp, max_link_clock, max_lanes);
 
-	link_bpp_x16 = intel_dp_mode_min_link_bpp_x16(connector, mode);
+	link_bpp_x16 = intel_dp_mode_min_link_bpp_x16(connector, mode,
+						      output_format);
 	mode_rate = intel_dp_link_required(max_link_clock, max_lanes,
 					   target_clock, mode->hdisplay,
 					   link_bpp_x16, 0);
@@ -1578,7 +1559,66 @@ intel_dp_mode_valid(struct drm_connector *_connector,
 	if (status != MODE_OK)
 		return status;
 
-	return intel_dp_mode_valid_downstream(connector, mode, target_clock);
+	return intel_dp_mode_valid_downstream(connector, mode,
+					      target_clock, sink_format);
+}
+
+static enum drm_mode_status
+intel_dp_mode_valid(struct drm_connector *_connector,
+		    const struct drm_display_mode *mode)
+{
+	struct intel_display *display = to_intel_display(_connector->dev);
+	struct intel_connector *connector = to_intel_connector(_connector);
+	const struct drm_display_info *info = &connector->base.display_info;
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	int target_clock = mode->clock;
+	enum drm_mode_status status;
+
+	status = intel_cpu_transcoder_mode_valid(display, mode);
+	if (status != MODE_OK)
+		return status;
+
+	if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+		return MODE_H_ILLEGAL;
+
+	if (mode->clock < 10000)
+		return MODE_CLOCK_LOW;
+
+	if (intel_dp_hdisplay_bad(display, mode->hdisplay))
+		return MODE_H_ILLEGAL;
+
+	if (intel_dp_is_edp(intel_dp)) {
+		status = intel_panel_mode_valid(connector, mode, &target_clock);
+		if (status != MODE_OK)
+			return status;
+	}
+
+	/*
+	 * TODO: Even when using a 4:2:0 sink_format intel_dp_output_format()
+	 * will always choose a 4:4:4 output_format if the DFP can do the
+	 * 4:4:4->4:2:0 conversion for us. Thus a mode may still be rejected
+	 * if we only have enough DP link bandwidth for 4:2:0 but not for
+	 * 4:4:4. Another attempt with an explicit 4:2:0 output_format might
+	 * be needed here. intel_dp_compute_config() would need the same
+	 * logic, or else the actual modeset would still fail.
+	 *
+	 * Also a lot of the checks only depend on output_format but not
+	 * sink_format, so we are potentially doing redundant work by
+	 * testing the same output_format for two different sink_formats.
+	 */
+	if (drm_mode_is_420_only(info, mode)) {
+		status = intel_dp_mode_valid_format(connector, mode, target_clock,
+						    INTEL_OUTPUT_FORMAT_YCBCR420);
+	} else {
+		status = intel_dp_mode_valid_format(connector, mode, target_clock,
+						    INTEL_OUTPUT_FORMAT_RGB);
+
+		if (status != MODE_OK && drm_mode_is_420_also(info, mode))
+			status = intel_dp_mode_valid_format(connector, mode, target_clock,
+							    INTEL_OUTPUT_FORMAT_YCBCR420);
+	}
+
+	return status;
 }
 
 bool intel_dp_source_supports_tps3(struct intel_display *display)
@@ -3129,6 +3169,21 @@ static void intel_dp_compute_vsc_colorimetry(const struct intel_crtc_state *crtc
 	vsc->content_type = DP_CONTENT_TYPE_NOT_DEFINED;
 }
 
+static bool intel_dp_needs_as_sdp(struct intel_dp *intel_dp,
+				  struct intel_crtc_state *crtc_state)
+{
+	if (!intel_dp->as_sdp_supported)
+		return false;
+
+	/*
+	 * #TODO Implement AS SDP for DP branch device.
+	 */
+	if (drm_dp_is_branch(intel_dp->dpcd))
+		return false;
+
+	return crtc_state->vrr.enable;
+}
+
 static void intel_dp_compute_as_sdp(struct intel_dp *intel_dp,
 				    struct intel_crtc_state *crtc_state)
 {
@@ -3136,7 +3191,12 @@ static void intel_dp_compute_as_sdp(struct intel_dp *intel_dp,
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->hw.adjusted_mode;
 
-	if (!crtc_state->vrr.enable || !intel_dp->as_sdp_supported)
+	/*
+	 * #FIXME: SDP/infoframe updates aren’t truly atomic, and with the new
+	 * cdclk->tc clock crossing we may transiently send a corrupted packet
+	 * if the update lands mid‑transmission.
+	 */
+	if (!intel_dp_needs_as_sdp(intel_dp, crtc_state))
 		return;
 
 	crtc_state->infoframes.enable |= intel_hdmi_infoframe_enable(DP_SDP_ADAPTIVE_SYNC);
@@ -3144,6 +3204,7 @@ static void intel_dp_compute_as_sdp(struct intel_dp *intel_dp,
 	as_sdp->sdp_type = DP_SDP_ADAPTIVE_SYNC;
 	as_sdp->length = 0x9;
 	as_sdp->duration_incr_ms = 0;
+	as_sdp->revision = 0x2;
 	as_sdp->vtotal = intel_vrr_vmin_vtotal(crtc_state);
 
 	if (crtc_state->cmrr.enable) {
@@ -3335,41 +3396,59 @@ static int
 intel_dp_compute_output_format(struct intel_encoder *encoder,
 			       struct intel_crtc_state *crtc_state,
 			       struct drm_connector_state *conn_state,
-			       bool respect_downstream_limits)
+			       bool respect_downstream_limits,
+			       enum intel_output_format sink_format)
+{
+	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
+	struct intel_connector *connector = intel_dp->attached_connector;
+	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+
+	if (intel_dp_sink_format_valid(connector, adjusted_mode,
+				       sink_format) != MODE_OK)
+		return -EINVAL;
+
+	crtc_state->sink_format = sink_format;
+	crtc_state->output_format = intel_dp_output_format(connector, crtc_state->sink_format);
+
+	return intel_dp_compute_link_config(encoder, crtc_state, conn_state,
+					    respect_downstream_limits);
+}
+
+static int
+intel_dp_compute_formats(struct intel_encoder *encoder,
+			 struct intel_crtc_state *crtc_state,
+			 struct drm_connector_state *conn_state,
+			 bool respect_downstream_limits)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 	struct intel_connector *connector = intel_dp->attached_connector;
 	const struct drm_display_info *info = &connector->base.display_info;
 	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
-	bool ycbcr_420_only;
 	int ret;
 
-	ycbcr_420_only = drm_mode_is_420_only(info, adjusted_mode);
-
-	if (ycbcr_420_only && !connector->base.ycbcr_420_allowed) {
-		drm_dbg_kms(display->drm,
-			    "YCbCr 4:2:0 mode but YCbCr 4:2:0 output not possible. Falling back to RGB.\n");
-		crtc_state->sink_format = INTEL_OUTPUT_FORMAT_RGB;
-	} else {
-		crtc_state->sink_format = intel_dp_sink_format(connector, adjusted_mode);
-	}
+	if (drm_mode_is_420_only(info, adjusted_mode)) {
+		ret = intel_dp_compute_output_format(encoder, crtc_state, conn_state,
+						     respect_downstream_limits,
+						     INTEL_OUTPUT_FORMAT_YCBCR420);
 
-	crtc_state->output_format = intel_dp_output_format(connector, crtc_state->sink_format);
+		if (ret && !respect_downstream_limits) {
+			drm_dbg_kms(display->drm,
+				    "YCbCr 4:2:0 mode but YCbCr 4:2:0 output not possible. Falling back to RGB.\n");
 
-	ret = intel_dp_compute_link_config(encoder, crtc_state, conn_state,
-					   respect_downstream_limits);
-	if (ret) {
-		if (crtc_state->sink_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
-		    !connector->base.ycbcr_420_allowed ||
-		    !drm_mode_is_420_also(info, adjusted_mode))
-			return ret;
+			ret = intel_dp_compute_output_format(encoder, crtc_state, conn_state,
+							     respect_downstream_limits,
+							     INTEL_OUTPUT_FORMAT_RGB);
+		}
+	} else {
+		ret = intel_dp_compute_output_format(encoder, crtc_state, conn_state,
+						     respect_downstream_limits,
+						     INTEL_OUTPUT_FORMAT_RGB);
 
-		crtc_state->sink_format = INTEL_OUTPUT_FORMAT_YCBCR420;
-		crtc_state->output_format = intel_dp_output_format(connector,
-								   crtc_state->sink_format);
-		ret = intel_dp_compute_link_config(encoder, crtc_state, conn_state,
-						   respect_downstream_limits);
+		if (ret && drm_mode_is_420_also(info, adjusted_mode))
+			ret = intel_dp_compute_output_format(encoder, crtc_state, conn_state,
+							     respect_downstream_limits,
+							     INTEL_OUTPUT_FORMAT_YCBCR420);
 	}
 
 	return ret;
@@ -3516,12 +3595,10 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	struct intel_atomic_state *state = to_intel_atomic_state(conn_state->state);
 	struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
 	struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-	const struct drm_display_mode *fixed_mode;
 	struct intel_connector *connector = intel_dp->attached_connector;
 	int ret = 0, link_bpp_x16;
 
-	fixed_mode = intel_panel_fixed_mode(connector, adjusted_mode);
-	if (intel_dp_is_edp(intel_dp) && fixed_mode) {
+	if (intel_dp_is_edp(intel_dp)) {
 		ret = intel_panel_compute_config(connector, adjusted_mode);
 		if (ret)
 			return ret;
@@ -3544,18 +3621,15 @@ intel_dp_compute_config(struct intel_encoder *encoder,
 	 * Try to respect downstream TMDS clock limits first, if
 	 * that fails assume the user might know something we don't.
 	 */
-	ret = intel_dp_compute_output_format(encoder, pipe_config, conn_state, true);
+	ret = intel_dp_compute_formats(encoder, pipe_config, conn_state, true);
 	if (ret)
-		ret = intel_dp_compute_output_format(encoder, pipe_config, conn_state, false);
+		ret = intel_dp_compute_formats(encoder, pipe_config, conn_state, false);
 	if (ret)
 		return ret;
 
-	if ((intel_dp_is_edp(intel_dp) && fixed_mode) ||
-	    pipe_config->output_format == INTEL_OUTPUT_FORMAT_YCBCR420) {
-		ret = intel_pfit_compute_config(pipe_config, conn_state);
-		if (ret)
-			return ret;
-	}
+	ret = intel_pfit_compute_config(pipe_config, conn_state);
+	if (ret)
+		return ret;
 
 	pipe_config->limited_color_range =
 		intel_dp_limited_color_range(pipe_config, conn_state);
@@ -4510,7 +4584,7 @@ void intel_dp_get_dsc_sink_cap(u8 dpcd_rev,
 
 	if (drm_dp_dpcd_readb(connector->dp.dsc_decompression_aux, DP_FEC_CAPABILITY,
 			      &connector->dp.fec_capability) < 0) {
-		drm_err(display->drm, "Failed to read FEC DPCD register\n");
+		drm_dbg_kms(display->drm, "Could not read FEC DPCD register\n");
 		return;
 	}
 
@@ -5107,7 +5181,7 @@ static ssize_t intel_dp_as_sdp_pack(const struct drm_dp_as_sdp *as_sdp,
 	/* Prepare AS (Adaptive Sync) SDP Header */
 	sdp->sdp_header.HB0 = 0;
 	sdp->sdp_header.HB1 = as_sdp->sdp_type;
-	sdp->sdp_header.HB2 = 0x02;
+	sdp->sdp_header.HB2 = as_sdp->revision;
 	sdp->sdp_header.HB3 = as_sdp->length;
 
 	/* Fill AS (Adaptive Sync) SDP Payload */
@@ -5244,7 +5318,7 @@ void intel_dp_set_infoframes(struct intel_encoder *encoder,
 			     const struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	i915_reg_t reg = HSW_TVIDEO_DIP_CTL(display, crtc_state->cpu_transcoder);
+	intel_reg_t reg = HSW_TVIDEO_DIP_CTL(display, crtc_state->cpu_transcoder);
 	u32 dip_enable = VIDEO_DIP_ENABLE_AVI_HSW | VIDEO_DIP_ENABLE_GCP_HSW |
 			 VIDEO_DIP_ENABLE_VS_HSW | VIDEO_DIP_ENABLE_GMP_HSW |
 			 VIDEO_DIP_ENABLE_SPD_HSW | VIDEO_DIP_ENABLE_DRM_GLK;
@@ -5294,16 +5368,15 @@ int intel_dp_as_sdp_unpack(struct drm_dp_as_sdp *as_sdp,
 	if (sdp->sdp_header.HB1 != DP_SDP_ADAPTIVE_SYNC)
 		return -EINVAL;
 
-	if (sdp->sdp_header.HB2 != 0x02)
-		return -EINVAL;
-
 	if ((sdp->sdp_header.HB3 & 0x3F) != 9)
 		return -EINVAL;
 
-	as_sdp->length = sdp->sdp_header.HB3 & DP_ADAPTIVE_SYNC_SDP_LENGTH;
-	as_sdp->mode = sdp->db[0] & DP_ADAPTIVE_SYNC_SDP_OPERATION_MODE;
+	as_sdp->sdp_type = sdp->sdp_header.HB1;
+	as_sdp->length = sdp->sdp_header.HB3 & DP_AS_SDP_LENGTH_MASK;
+	as_sdp->revision = sdp->sdp_header.HB2;
+	as_sdp->mode = sdp->db[0] & DP_AS_SDP_OPERATION_MODE_MASK;
 	as_sdp->vtotal = (sdp->db[2] << 8) | sdp->db[1];
-	as_sdp->target_rr = (u64)sdp->db[3] | ((u64)sdp->db[4] & 0x3);
+	as_sdp->target_rr = ((sdp->db[4] & 0x3) << 8) | sdp->db[3];
 	as_sdp->target_rr_divider = sdp->db[4] & 0x20 ? true : false;
 
 	return 0;
@@ -6738,7 +6811,7 @@ static int intel_modeset_synced_crtcs(struct intel_atomic_state *state,
 }
 
 static int intel_dp_connector_atomic_check(struct drm_connector *_connector,
-					   struct drm_atomic_state *_state)
+					   struct drm_atomic_commit *_state)
 {
 	struct intel_connector *connector = to_intel_connector(_connector);
 	struct intel_display *display = to_intel_display(connector);
@@ -7272,6 +7345,19 @@ intel_dp_init_connector(struct intel_digital_port *dig_port,
 	return false;
 }
 
+void intel_dp_cleanup_connector(struct intel_digital_port *dig_port,
+				struct intel_connector *connector)
+{
+	struct intel_display *display = to_intel_display(connector);
+	struct intel_dp *intel_dp = &dig_port->dp;
+
+	intel_display_power_flush_work(display);
+
+	intel_dp_mst_encoder_cleanup(dig_port);
+	intel_dp_aux_fini(intel_dp);
+	drm_connector_cleanup(&connector->base);
+}
+
 void intel_dp_mst_suspend(struct intel_display *display)
 {
 	struct intel_encoder *encoder;
diff --git a/drivers/gpu/drm/i915/display/intel_dp.h b/drivers/gpu/drm/i915/display/intel_dp.h
index 2849b9ecdc71a..f41480d247142 100644
--- a/drivers/gpu/drm/i915/display/intel_dp.h
+++ b/drivers/gpu/drm/i915/display/intel_dp.h
@@ -48,6 +48,9 @@ intel_dp_queue_modeset_retry_for_link(struct intel_atomic_state *state,
 				      const struct intel_crtc_state *crtc_state);
 bool intel_dp_init_connector(struct intel_digital_port *dig_port,
 			     struct intel_connector *intel_connector);
+void intel_dp_cleanup_connector(struct intel_digital_port *dig_port,
+				struct intel_connector *connector);
+
 void intel_dp_connector_sync_state(struct intel_connector *connector,
 				   const struct intel_crtc_state *crtc_state);
 void intel_dp_set_link_params(struct intel_dp *intel_dp,
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux.c b/drivers/gpu/drm/i915/display/intel_dp_aux.c
index b20ec3e589fad..3cb07b0a901cf 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux.c
@@ -12,6 +12,7 @@
 #include "intel_dp.h"
 #include "intel_dp_aux.h"
 #include "intel_dp_aux_regs.h"
+#include "intel_parent.h"
 #include "intel_pps.h"
 #include "intel_quirks.h"
 #include "intel_tc.h"
@@ -58,20 +59,31 @@ static u32
 intel_dp_aux_wait_done(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
+	intel_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
 	const unsigned int timeout_ms = 10;
+	bool done = true;
 	u32 status;
-	bool done;
+	int ret;
 
+	if (intel_parent_irq_enabled(display)) {
 #define C (((status = intel_de_read_notrace(display, ch_ctl)) & DP_AUX_CH_CTL_SEND_BUSY) == 0)
-	done = wait_event_timeout(display->gmbus.wait_queue, C,
-				  msecs_to_jiffies_timeout(timeout_ms));
+		done = wait_event_timeout(display->gmbus.wait_queue, C,
+					  msecs_to_jiffies_timeout(timeout_ms));
+
+#undef C
+	} else {
+		ret = intel_de_wait_ms(display, ch_ctl,
+				       DP_AUX_CH_CTL_SEND_BUSY, 0,
+				       timeout_ms, &status);
+
+		if (ret == -ETIMEDOUT)
+			done = false;
+	}
 
 	if (!done)
 		drm_err(display->drm,
 			"%s: did not complete or timeout within %ums (status 0x%08x)\n",
 			intel_dp->aux.name, timeout_ms, status);
-#undef C
 
 	return status;
 }
@@ -244,7 +256,7 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	struct intel_encoder *encoder = &dig_port->base;
-	i915_reg_t ch_ctl, ch_data[5];
+	intel_reg_t ch_ctl, ch_data[5];
 	u32 aux_clock_divider;
 	enum intel_display_power_domain aux_domain;
 	struct ref_tracker *aux_wakeref;
@@ -554,7 +566,7 @@ intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
 	return ret;
 }
 
-static i915_reg_t vlv_aux_ctl_reg(struct intel_dp *intel_dp)
+static intel_reg_t vlv_aux_ctl_reg(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -570,7 +582,7 @@ static i915_reg_t vlv_aux_ctl_reg(struct intel_dp *intel_dp)
 	}
 }
 
-static i915_reg_t vlv_aux_data_reg(struct intel_dp *intel_dp, int index)
+static intel_reg_t vlv_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -586,7 +598,7 @@ static i915_reg_t vlv_aux_data_reg(struct intel_dp *intel_dp, int index)
 	}
 }
 
-static i915_reg_t g4x_aux_ctl_reg(struct intel_dp *intel_dp)
+static intel_reg_t g4x_aux_ctl_reg(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -602,7 +614,7 @@ static i915_reg_t g4x_aux_ctl_reg(struct intel_dp *intel_dp)
 	}
 }
 
-static i915_reg_t g4x_aux_data_reg(struct intel_dp *intel_dp, int index)
+static intel_reg_t g4x_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -618,7 +630,7 @@ static i915_reg_t g4x_aux_data_reg(struct intel_dp *intel_dp, int index)
 	}
 }
 
-static i915_reg_t ilk_aux_ctl_reg(struct intel_dp *intel_dp)
+static intel_reg_t ilk_aux_ctl_reg(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -636,7 +648,7 @@ static i915_reg_t ilk_aux_ctl_reg(struct intel_dp *intel_dp)
 	}
 }
 
-static i915_reg_t ilk_aux_data_reg(struct intel_dp *intel_dp, int index)
+static intel_reg_t ilk_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -654,7 +666,7 @@ static i915_reg_t ilk_aux_data_reg(struct intel_dp *intel_dp, int index)
 	}
 }
 
-static i915_reg_t skl_aux_ctl_reg(struct intel_dp *intel_dp)
+static intel_reg_t skl_aux_ctl_reg(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -673,7 +685,7 @@ static i915_reg_t skl_aux_ctl_reg(struct intel_dp *intel_dp)
 	}
 }
 
-static i915_reg_t skl_aux_data_reg(struct intel_dp *intel_dp, int index)
+static intel_reg_t skl_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -692,7 +704,7 @@ static i915_reg_t skl_aux_data_reg(struct intel_dp *intel_dp, int index)
 	}
 }
 
-static i915_reg_t tgl_aux_ctl_reg(struct intel_dp *intel_dp)
+static intel_reg_t tgl_aux_ctl_reg(struct intel_dp *intel_dp)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -714,7 +726,7 @@ static i915_reg_t tgl_aux_ctl_reg(struct intel_dp *intel_dp)
 	}
 }
 
-static i915_reg_t tgl_aux_data_reg(struct intel_dp *intel_dp, int index)
+static intel_reg_t tgl_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum aux_ch aux_ch = dig_port->aux_ch;
@@ -736,7 +748,7 @@ static i915_reg_t tgl_aux_data_reg(struct intel_dp *intel_dp, int index)
 	}
 }
 
-static i915_reg_t xelpdp_aux_ctl_reg(struct intel_dp *intel_dp)
+static intel_reg_t xelpdp_aux_ctl_reg(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
@@ -756,7 +768,7 @@ static i915_reg_t xelpdp_aux_ctl_reg(struct intel_dp *intel_dp)
 	}
 }
 
-static i915_reg_t xelpdp_aux_data_reg(struct intel_dp *intel_dp, int index)
+static intel_reg_t xelpdp_aux_data_reg(struct intel_dp *intel_dp, int index)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
diff --git a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
index a8d56ebf06a2f..7a6c07f6aaeb4 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_aux_backlight.c
@@ -691,10 +691,9 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	struct drm_device *dev = connector->base.dev;
 	struct intel_panel *panel = &connector->panel;
-	bool try_intel_interface = false;
+	bool try_intel_interface = false, try_vesa_interface = false;
 
-	/*
-	 * Check the VBT and user's module parameters to figure out which
+	/* Check the VBT and user's module parameters to figure out which
 	 * interfaces to probe
 	 */
 	switch (display->params.enable_dpcd_backlight) {
@@ -703,6 +702,7 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 	case INTEL_DP_AUX_BACKLIGHT_AUTO:
 		switch (panel->vbt.backlight.type) {
 		case INTEL_BACKLIGHT_VESA_EDP_AUX_INTERFACE:
+			try_vesa_interface = true;
 			break;
 		case INTEL_BACKLIGHT_DISPLAY_DDI:
 			try_intel_interface = true;
@@ -715,12 +715,20 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 		if (panel->vbt.backlight.type != INTEL_BACKLIGHT_VESA_EDP_AUX_INTERFACE)
 			try_intel_interface = true;
 
+		try_vesa_interface = true;
+		break;
+	case INTEL_DP_AUX_BACKLIGHT_FORCE_VESA:
+		try_vesa_interface = true;
 		break;
 	case INTEL_DP_AUX_BACKLIGHT_FORCE_INTEL:
 		try_intel_interface = true;
 		break;
 	}
 
+	/* For eDP 1.5 and above we are supposed to use VESA interface for brightness control */
+	if (intel_dp->edp_dpcd[0] >= DP_EDP_15)
+		try_vesa_interface = true;
+
 	/*
 	 * Since Intel has their own backlight control interface, the majority of machines out there
 	 * using DPCD backlight controls with Intel GPUs will be using this interface as opposed to
@@ -733,9 +741,6 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 	 * panel with Intel's OUI - which is also required for us to be able to detect Intel's
 	 * backlight interface at all. This means that the only sensible way for us to detect both
 	 * interfaces is to probe for Intel's first, and VESA's second.
-	 *
-	 * Also there is a chance some VBTs may advertise false Intel backlight support even if the
-	 * TCON DPCD says otherwise. This means we keep VESA interface as fallback in that case.
 	 */
 	if (try_intel_interface && intel_dp->edp_dpcd[0] <= DP_EDP_14b &&
 	    intel_dp_aux_supports_hdr_backlight(connector)) {
@@ -745,7 +750,7 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *connector)
 		return 0;
 	}
 
-	if (intel_dp_aux_supports_vesa_backlight(connector)) {
+	if (try_vesa_interface && intel_dp_aux_supports_vesa_backlight(connector)) {
 		drm_dbg_kms(dev, "[CONNECTOR:%d:%s] Using VESA eDP backlight controls\n",
 			    connector->base.base.id, connector->base.name);
 		panel->backlight.funcs = &intel_dp_vesa_bl_funcs;
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
index 54c585c59b900..a26094223f780 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c
@@ -742,11 +742,14 @@ static void intel_dp_update_downspread_ctrl(struct intel_dp *intel_dp,
 
 void intel_dp_link_training_set_bw(struct intel_dp *intel_dp,
 				   int link_bw, int rate_select, int lane_count,
-				   bool enhanced_framing)
+				   bool enhanced_framing, bool post_lt_adj_req)
 {
 	if (enhanced_framing)
 		lane_count |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
 
+	if (post_lt_adj_req)
+		lane_count |= DP_POST_LT_ADJ_REQ_GRANTED;
+
 	if (link_bw) {
 		/* DP and eDP v1.3 and earlier link bw set method. */
 		u8 link_config[] = { link_bw, lane_count };
@@ -825,12 +828,21 @@ static u32 intel_dp_training_pattern(struct intel_dp *intel_dp,
 	return DP_TRAINING_PATTERN_2;
 }
 
+static bool intel_dp_use_post_lt_adj_req(struct intel_dp *intel_dp,
+					 const struct intel_crtc_state *crtc_state)
+{
+	return intel_dp->set_idle_link_train &&
+		drm_dp_post_lt_adj_req_supported(intel_dp->dpcd) &&
+		intel_dp_training_pattern(intel_dp, crtc_state, DP_PHY_DPRX) != DP_TRAINING_PATTERN_4;
+}
+
 static void intel_dp_update_link_bw_set(struct intel_dp *intel_dp,
 					const struct intel_crtc_state *crtc_state,
 					u8 link_bw, u8 rate_select)
 {
 	intel_dp_link_training_set_bw(intel_dp, link_bw, rate_select, crtc_state->lane_count,
-				      crtc_state->enhanced_framing);
+				      crtc_state->enhanced_framing,
+				      intel_dp_use_post_lt_adj_req(intel_dp, crtc_state));
 }
 
 /*
@@ -1091,6 +1103,109 @@ intel_dp_link_training_channel_equalization(struct intel_dp *intel_dp,
 	return channel_eq;
 }
 
+static bool
+intel_dp_post_lt_adj_req(struct intel_dp *intel_dp,
+			 const struct intel_crtc_state *crtc_state)
+{
+	u8 link_status[DP_LINK_STATUS_SIZE];
+	unsigned long deadline;
+	bool timeout = false;
+	bool success = false;
+	int changes = 0;
+
+	if (!intel_dp_use_post_lt_adj_req(intel_dp, crtc_state))
+		return true;
+
+	if (drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, DP_PHY_DPRX,
+					     link_status) < 0) {
+		lt_err(intel_dp, DP_PHY_DPRX, "Failed to get link status\n");
+		return false;
+	}
+
+	deadline = jiffies + msecs_to_jiffies_timeout(200);
+
+	for (;;) {
+		/* Make sure clock is still ok */
+		if (!drm_dp_clock_recovery_ok(link_status,
+					      crtc_state->lane_count)) {
+			intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status);
+			lt_dbg(intel_dp, DP_PHY_DPRX,
+			       "Clock recovery check failed, cannot continue POST_LT_ADJ_REQ\n");
+			break;
+		}
+
+		if (!drm_dp_channel_eq_ok(link_status,
+					  crtc_state->lane_count)) {
+			intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status);
+			lt_dbg(intel_dp, DP_PHY_DPRX, "Channel EQ check failed. cannot continue POST_LT_ADJ_REQ\n");
+			break;
+		}
+
+		if (!drm_dp_post_lt_adj_req_in_progress(link_status)) {
+			success = true;
+			intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status);
+			lt_dbg(intel_dp, DP_PHY_DPRX,
+			       "POST_LT_ADJ_REQ done (%d changes). DP Training successful\n", changes);
+			break;
+		}
+
+		if (changes == 6) {
+			success = true;
+			intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status);
+			lt_dbg(intel_dp, DP_PHY_DPRX,
+			       "POST_LT_ADJ_REQ limit reached (%d changes). DP Training successful\n", changes);
+			break;
+		}
+
+		if (timeout) {
+			success = true;
+			intel_dp_dump_link_status(intel_dp, DP_PHY_DPRX, link_status);
+			lt_dbg(intel_dp, DP_PHY_DPRX,
+			       "POST_LT_ADJ_REQ timeout reached (%d changes). DP Training successful\n", changes);
+			break;
+		}
+
+		fsleep(5000);
+
+		if (drm_dp_dpcd_read_phy_link_status(&intel_dp->aux, DP_PHY_DPRX,
+						     link_status) < 0) {
+			lt_err(intel_dp, DP_PHY_DPRX, "Failed to get link status\n");
+			break;
+		}
+
+		/* Update training set as requested by target */
+		if (intel_dp_get_adjust_train(intel_dp, crtc_state, DP_PHY_DPRX, link_status)) {
+			deadline = jiffies + msecs_to_jiffies_timeout(200);
+			changes++;
+
+			if (!intel_dp_update_link_train(intel_dp, crtc_state, DP_PHY_DPRX)) {
+				lt_err(intel_dp, DP_PHY_DPRX, "Failed to update link training\n");
+				break;
+			}
+		} else if (time_after(jiffies, deadline)) {
+			timeout = true;
+		}
+	}
+
+	return success;
+}
+
+static void intel_dp_stop_post_lt_adj_req(struct intel_dp *intel_dp,
+					  const struct intel_crtc_state *crtc_state)
+{
+	u8 lane_count;
+
+	if (!intel_dp_use_post_lt_adj_req(intel_dp, crtc_state))
+		return;
+
+	/* clear DP_POST_LT_ADJ_REQ_GRANTED */
+	lane_count = crtc_state->lane_count;
+	if (crtc_state->enhanced_framing)
+		lane_count |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
+
+	drm_dp_dpcd_writeb(&intel_dp->aux, DP_LANE_COUNT_SET, lane_count);
+}
+
 static bool intel_dp_disable_dpcd_training_pattern(struct intel_dp *intel_dp,
 						   enum drm_dp_phy dp_phy)
 {
@@ -1389,6 +1504,11 @@ intel_dp_link_train_all_phys(struct intel_dp *intel_dp,
 	intel_dp_disable_dpcd_training_pattern(intel_dp, DP_PHY_DPRX);
 	intel_dp->set_idle_link_train(intel_dp, crtc_state);
 
+	if (ret)
+		ret = intel_dp_post_lt_adj_req(intel_dp, crtc_state);
+
+	intel_dp_stop_post_lt_adj_req(intel_dp, crtc_state);
+
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.h b/drivers/gpu/drm/i915/display/intel_dp_link_training.h
index 1ba22ed6db087..33dcbde6a4089 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_link_training.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.h
@@ -21,7 +21,7 @@ void intel_dp_link_training_set_mode(struct intel_dp *intel_dp,
 				     int link_rate, bool is_vrr);
 void intel_dp_link_training_set_bw(struct intel_dp *intel_dp,
 				   int link_bw, int rate_select, int lane_count,
-				   bool enhanced_framing);
+				   bool enhanced_framing, bool post_lt_adj_req);
 
 bool intel_dp_get_adjust_train(struct intel_dp *intel_dp,
 			       const struct intel_crtc_state *crtc_state,
diff --git a/drivers/gpu/drm/i915/display/intel_dp_mst.c b/drivers/gpu/drm/i915/display/intel_dp_mst.c
index 887b6de14e467..8f73e01db17c9 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_mst.c
@@ -32,6 +32,7 @@
 #include <drm/drm_fixed.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
+#include <drm/intel/step.h>
 
 #include "intel_atomic.h"
 #include "intel_audio.h"
@@ -56,7 +57,6 @@
 #include "intel_link_bw.h"
 #include "intel_pfit.h"
 #include "intel_psr.h"
-#include "intel_step.h"
 #include "intel_vdsc.h"
 #include "intel_vrr.h"
 #include "skl_scaler.h"
@@ -251,7 +251,7 @@ int intel_dp_mtp_tu_compute_config(struct intel_dp *intel_dp,
 				   int min_bpp_x16, int max_bpp_x16, int bpp_step_x16, bool dsc)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
-	struct drm_atomic_state *state = crtc_state->uapi.state;
+	struct drm_atomic_commit *state = crtc_state->uapi.state;
 	struct drm_dp_mst_topology_state *mst_state = NULL;
 	struct intel_connector *connector =
 		to_intel_connector(conn_state->connector);
@@ -721,7 +721,10 @@ static int mst_stream_compute_config(struct intel_encoder *encoder,
 
 	pipe_config->sink_format = INTEL_OUTPUT_FORMAT_RGB;
 	pipe_config->output_format = INTEL_OUTPUT_FORMAT_RGB;
-	pipe_config->has_pch_encoder = false;
+
+	ret = intel_pfit_compute_config(pipe_config, conn_state);
+	if (ret)
+		return ret;
 
 	for_each_joiner_candidate(connector, adjusted_mode, num_joined_pipes) {
 		if (num_joined_pipes > 1)
@@ -996,7 +999,7 @@ mst_connector_atomic_topology_check(struct intel_connector *connector,
 
 static int
 mst_connector_atomic_check(struct drm_connector *_connector,
-			   struct drm_atomic_state *_state)
+			   struct drm_atomic_commit *_state)
 {
 	struct intel_atomic_state *state = to_intel_atomic_state(_state);
 	struct intel_connector *connector = to_intel_connector(_connector);
@@ -1603,7 +1606,7 @@ mst_connector_mode_valid_ctx(struct drm_connector *_connector,
 
 static struct drm_encoder *
 mst_connector_atomic_best_encoder(struct drm_connector *_connector,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct intel_connector *connector = to_intel_connector(_connector);
 	struct drm_connector_state *connector_state =
@@ -2144,7 +2147,7 @@ void intel_dp_mst_prepare_probe(struct intel_dp *intel_dp)
 
 	intel_dp_link_training_set_mode(intel_dp, link_rate, false);
 	intel_dp_link_training_set_bw(intel_dp, link_bw, rate_select, lane_count,
-				      drm_dp_enhanced_frame_cap(intel_dp->dpcd));
+				      drm_dp_enhanced_frame_cap(intel_dp->dpcd), false);
 
 	intel_mst_set_probed_link_params(intel_dp, link_rate, lane_count);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
index 7363c98172971..11712a151f729 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
+++ b/drivers/gpu/drm/i915/display/intel_dp_tunnel.c
@@ -296,6 +296,24 @@ bool intel_dp_tunnel_bw_alloc_is_enabled(struct intel_dp *intel_dp)
 	return drm_dp_tunnel_bw_alloc_is_enabled(intel_dp->tunnel);
 }
 
+/**
+ * intel_dp_tunnel_pr_optimization_supported - Query the PR BW optimization support
+ * @intel_dp: DP port object
+ *
+ * Query whether a DP tunnel supports the PR BW optimization.
+ *
+ * Returns %true if the BW allocation mode is supported on @intel_dp.
+ */
+bool intel_dp_tunnel_pr_optimization_supported(struct intel_dp *intel_dp)
+{
+	struct intel_display *display = to_intel_display(intel_dp);
+
+	if (DISPLAY_VER(display) < 35)
+		return false;
+
+	return drm_dp_tunnel_pr_optimization_supported(intel_dp->tunnel);
+}
+
 /**
  * intel_dp_tunnel_suspend - Suspend a DP tunnel connected on a port
  * @intel_dp: DP port object
diff --git a/drivers/gpu/drm/i915/display/intel_dp_tunnel.h b/drivers/gpu/drm/i915/display/intel_dp_tunnel.h
index 10ab9eebcef69..8273e681a5128 100644
--- a/drivers/gpu/drm/i915/display/intel_dp_tunnel.h
+++ b/drivers/gpu/drm/i915/display/intel_dp_tunnel.h
@@ -32,6 +32,7 @@ void intel_dp_tunnel_resume(struct intel_dp *intel_dp,
 void intel_dp_tunnel_suspend(struct intel_dp *intel_dp);
 
 bool intel_dp_tunnel_bw_alloc_is_enabled(struct intel_dp *intel_dp);
+bool intel_dp_tunnel_pr_optimization_supported(struct intel_dp *intel_dp);
 
 void
 intel_dp_tunnel_atomic_cleanup_inherited_state(struct intel_atomic_state *state);
@@ -76,6 +77,11 @@ static inline bool intel_dp_tunnel_bw_alloc_is_enabled(struct intel_dp *intel_dp
 	return false;
 }
 
+static inline bool intel_dp_tunnel_pr_optimization_supported(struct intel_dp *intel_dp)
+{
+	return false;
+}
+
 static inline void
 intel_dp_tunnel_atomic_cleanup_inherited_state(struct intel_atomic_state *state) {}
 
diff --git a/drivers/gpu/drm/i915/display/intel_dpcd.h b/drivers/gpu/drm/i915/display/intel_dpcd.h
new file mode 100644
index 0000000000000..4aea5326f2ed4
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_dpcd.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef __INTEL_DPCD_H__
+#define __INTEL_DPCD_H__
+
+#define INTEL_DPCD_INTEL_WA_REGISTER_CAPS					0x3f0
+# define INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_EARLYSCANLINE_SDP_SUPPORT_MASK	REG_GENMASK(1, 0)
+# define INTEL_DPCD_INTEL_WA_REGISTER_CAPS_FALL_BACK_TO_PSR1			0
+# define INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_WITH_EARLY_SCANLINE		1
+# define INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_WITHOUT_EARLY_SCANLINE		2
+
+#endif /* __INTEL_DPCD_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_dpio_phy.c b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
index 8027bab2951bd..8055f6ccc5c86 100644
--- a/drivers/gpu/drm/i915/display/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_dpio_phy.c
@@ -279,8 +279,8 @@ void bxt_port_to_phy_channel(struct intel_display *display, enum port port,
  * writes to the group register to write the same value to all the lanes.
  */
 static u32 bxt_dpio_phy_rmw_grp(struct intel_display *display,
-				i915_reg_t reg_single,
-				i915_reg_t reg_group,
+				intel_reg_t reg_single,
+				intel_reg_t reg_group,
 				u32 clear, u32 set)
 {
 	u32 old, val;
@@ -301,7 +301,7 @@ void bxt_dpio_phy_set_signal_levels(struct intel_encoder *encoder,
 	enum dpio_phy phy;
 	int lane, n_entries;
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans))
 		return;
 
@@ -512,7 +512,7 @@ void bxt_dpio_phy_init(struct intel_display *display, enum dpio_phy phy)
 
 static bool __printf(6, 7)
 __phy_reg_verify_state(struct intel_display *display, enum dpio_phy phy,
-		       i915_reg_t reg, u32 mask, u32 expected,
+		       intel_reg_t reg, u32 mask, u32 expected,
 		       const char *reg_fmt, ...)
 {
 	struct va_format vaf;
@@ -724,46 +724,46 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 	u32 val;
 	int i;
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* Clear calc init */
-	val = vlv_dpio_read(display->drm, phy, VLV_PCS01_DW10(ch));
+	val = vlv_dpio_read(display, phy, VLV_PCS01_DW10(ch));
 	val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
 	val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
 	val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
-	vlv_dpio_write(display->drm, phy, VLV_PCS01_DW10(ch), val);
+	vlv_dpio_write(display, phy, VLV_PCS01_DW10(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(display->drm, phy, VLV_PCS23_DW10(ch));
+		val = vlv_dpio_read(display, phy, VLV_PCS23_DW10(ch));
 		val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
 		val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
 		val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
-		vlv_dpio_write(display->drm, phy, VLV_PCS23_DW10(ch), val);
+		vlv_dpio_write(display, phy, VLV_PCS23_DW10(ch), val);
 	}
 
-	val = vlv_dpio_read(display->drm, phy, VLV_PCS01_DW9(ch));
+	val = vlv_dpio_read(display, phy, VLV_PCS01_DW9(ch));
 	val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
 	val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
-	vlv_dpio_write(display->drm, phy, VLV_PCS01_DW9(ch), val);
+	vlv_dpio_write(display, phy, VLV_PCS01_DW9(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(display->drm, phy, VLV_PCS23_DW9(ch));
+		val = vlv_dpio_read(display, phy, VLV_PCS23_DW9(ch));
 		val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
 		val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
-		vlv_dpio_write(display->drm, phy, VLV_PCS23_DW9(ch), val);
+		vlv_dpio_write(display, phy, VLV_PCS23_DW9(ch), val);
 	}
 
 	/* Program swing deemph */
 	for (i = 0; i < crtc_state->lane_count; i++) {
-		val = vlv_dpio_read(display->drm, phy, CHV_TX_DW4(ch, i));
+		val = vlv_dpio_read(display, phy, CHV_TX_DW4(ch, i));
 		val &= ~DPIO_SWING_DEEMPH9P5_MASK;
 		val |= DPIO_SWING_DEEMPH9P5(deemph_reg_value);
-		vlv_dpio_write(display->drm, phy, CHV_TX_DW4(ch, i), val);
+		vlv_dpio_write(display, phy, CHV_TX_DW4(ch, i), val);
 	}
 
 	/* Program swing margin */
 	for (i = 0; i < crtc_state->lane_count; i++) {
-		val = vlv_dpio_read(display->drm, phy, CHV_TX_DW2(ch, i));
+		val = vlv_dpio_read(display, phy, CHV_TX_DW2(ch, i));
 
 		val &= ~DPIO_SWING_MARGIN000_MASK;
 		val |= DPIO_SWING_MARGIN000(margin_reg_value);
@@ -776,7 +776,7 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 		val &= ~DPIO_UNIQ_TRANS_SCALE_MASK;
 		val |= DPIO_UNIQ_TRANS_SCALE(0x9a);
 
-		vlv_dpio_write(display->drm, phy, CHV_TX_DW2(ch, i), val);
+		vlv_dpio_write(display, phy, CHV_TX_DW2(ch, i), val);
 	}
 
 	/*
@@ -786,26 +786,26 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 	 * 27 for ch0 and ch1.
 	 */
 	for (i = 0; i < crtc_state->lane_count; i++) {
-		val = vlv_dpio_read(display->drm, phy, CHV_TX_DW3(ch, i));
+		val = vlv_dpio_read(display, phy, CHV_TX_DW3(ch, i));
 		if (uniq_trans_scale)
 			val |= DPIO_TX_UNIQ_TRANS_SCALE_EN;
 		else
 			val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN;
-		vlv_dpio_write(display->drm, phy, CHV_TX_DW3(ch, i), val);
+		vlv_dpio_write(display, phy, CHV_TX_DW3(ch, i), val);
 	}
 
 	/* Start swing calculation */
-	val = vlv_dpio_read(display->drm, phy, VLV_PCS01_DW10(ch));
+	val = vlv_dpio_read(display, phy, VLV_PCS01_DW10(ch));
 	val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
-	vlv_dpio_write(display->drm, phy, VLV_PCS01_DW10(ch), val);
+	vlv_dpio_write(display, phy, VLV_PCS01_DW10(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(display->drm, phy, VLV_PCS23_DW10(ch));
+		val = vlv_dpio_read(display, phy, VLV_PCS23_DW10(ch));
 		val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
-		vlv_dpio_write(display->drm, phy, VLV_PCS23_DW10(ch), val);
+		vlv_dpio_write(display, phy, VLV_PCS23_DW10(ch), val);
 	}
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 static void __chv_data_lane_soft_reset(struct intel_encoder *encoder,
@@ -818,38 +818,38 @@ static void __chv_data_lane_soft_reset(struct intel_encoder *encoder,
 	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 	u32 val;
 
-	val = vlv_dpio_read(display->drm, phy, VLV_PCS01_DW0(ch));
+	val = vlv_dpio_read(display, phy, VLV_PCS01_DW0(ch));
 	if (reset)
 		val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
 	else
 		val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
-	vlv_dpio_write(display->drm, phy, VLV_PCS01_DW0(ch), val);
+	vlv_dpio_write(display, phy, VLV_PCS01_DW0(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(display->drm, phy, VLV_PCS23_DW0(ch));
+		val = vlv_dpio_read(display, phy, VLV_PCS23_DW0(ch));
 		if (reset)
 			val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
 		else
 			val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
-		vlv_dpio_write(display->drm, phy, VLV_PCS23_DW0(ch), val);
+		vlv_dpio_write(display, phy, VLV_PCS23_DW0(ch), val);
 	}
 
-	val = vlv_dpio_read(display->drm, phy, VLV_PCS01_DW1(ch));
+	val = vlv_dpio_read(display, phy, VLV_PCS01_DW1(ch));
 	val |= CHV_PCS_REQ_SOFTRESET_EN;
 	if (reset)
 		val &= ~DPIO_PCS_CLK_SOFT_RESET;
 	else
 		val |= DPIO_PCS_CLK_SOFT_RESET;
-	vlv_dpio_write(display->drm, phy, VLV_PCS01_DW1(ch), val);
+	vlv_dpio_write(display, phy, VLV_PCS01_DW1(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(display->drm, phy, VLV_PCS23_DW1(ch));
+		val = vlv_dpio_read(display, phy, VLV_PCS23_DW1(ch));
 		val |= CHV_PCS_REQ_SOFTRESET_EN;
 		if (reset)
 			val &= ~DPIO_PCS_CLK_SOFT_RESET;
 		else
 			val |= DPIO_PCS_CLK_SOFT_RESET;
-		vlv_dpio_write(display->drm, phy, VLV_PCS23_DW1(ch), val);
+		vlv_dpio_write(display, phy, VLV_PCS23_DW1(ch), val);
 	}
 }
 
@@ -859,9 +859,9 @@ void chv_data_lane_soft_reset(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 	__chv_data_lane_soft_reset(encoder, crtc_state, reset);
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
@@ -887,47 +887,47 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
 
 	chv_phy_powergate_lanes(encoder, true, lane_mask);
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* Assert data lane reset */
 	__chv_data_lane_soft_reset(encoder, crtc_state, true);
 
 	/* program left/right clock distribution */
 	if (pipe != PIPE_B) {
-		val = vlv_dpio_read(display->drm, phy, CHV_CMN_DW5_CH0);
+		val = vlv_dpio_read(display, phy, CHV_CMN_DW5_CH0);
 		val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
 		if (ch == DPIO_CH0)
 			val |= CHV_BUFLEFTENA1_FORCE;
 		if (ch == DPIO_CH1)
 			val |= CHV_BUFRIGHTENA1_FORCE;
-		vlv_dpio_write(display->drm, phy, CHV_CMN_DW5_CH0, val);
+		vlv_dpio_write(display, phy, CHV_CMN_DW5_CH0, val);
 	} else {
-		val = vlv_dpio_read(display->drm, phy, CHV_CMN_DW1_CH1);
+		val = vlv_dpio_read(display, phy, CHV_CMN_DW1_CH1);
 		val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
 		if (ch == DPIO_CH0)
 			val |= CHV_BUFLEFTENA2_FORCE;
 		if (ch == DPIO_CH1)
 			val |= CHV_BUFRIGHTENA2_FORCE;
-		vlv_dpio_write(display->drm, phy, CHV_CMN_DW1_CH1, val);
+		vlv_dpio_write(display, phy, CHV_CMN_DW1_CH1, val);
 	}
 
 	/* program clock channel usage */
-	val = vlv_dpio_read(display->drm, phy, VLV_PCS01_DW8(ch));
+	val = vlv_dpio_read(display, phy, VLV_PCS01_DW8(ch));
 	val |= DPIO_PCS_USEDCLKCHANNEL_OVRRIDE;
 	if (pipe == PIPE_B)
 		val |= DPIO_PCS_USEDCLKCHANNEL;
 	else
 		val &= ~DPIO_PCS_USEDCLKCHANNEL;
-	vlv_dpio_write(display->drm, phy, VLV_PCS01_DW8(ch), val);
+	vlv_dpio_write(display, phy, VLV_PCS01_DW8(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(display->drm, phy, VLV_PCS23_DW8(ch));
+		val = vlv_dpio_read(display, phy, VLV_PCS23_DW8(ch));
 		val |= DPIO_PCS_USEDCLKCHANNEL_OVRRIDE;
 		if (pipe == PIPE_B)
 			val |= DPIO_PCS_USEDCLKCHANNEL;
 		else
 			val &= ~DPIO_PCS_USEDCLKCHANNEL;
-		vlv_dpio_write(display->drm, phy, VLV_PCS23_DW8(ch), val);
+		vlv_dpio_write(display, phy, VLV_PCS23_DW8(ch), val);
 	}
 
 	/*
@@ -935,14 +935,14 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
 	 * matches the pipe, but here we need to
 	 * pick the CL based on the port.
 	 */
-	val = vlv_dpio_read(display->drm, phy, CHV_CMN_DW19(ch));
+	val = vlv_dpio_read(display, phy, CHV_CMN_DW19(ch));
 	if (pipe == PIPE_B)
 		val |= CHV_CMN_USEDCLKCHANNEL;
 	else
 		val &= ~CHV_CMN_USEDCLKCHANNEL;
-	vlv_dpio_write(display->drm, phy, CHV_CMN_DW19(ch), val);
+	vlv_dpio_write(display, phy, CHV_CMN_DW19(ch), val);
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
@@ -956,17 +956,17 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	int data, i, stagger;
 	u32 val;
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* allow hardware to manage TX FIFO reset source */
-	val = vlv_dpio_read(display->drm, phy, VLV_PCS01_DW11(ch));
+	val = vlv_dpio_read(display, phy, VLV_PCS01_DW11(ch));
 	val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
-	vlv_dpio_write(display->drm, phy, VLV_PCS01_DW11(ch), val);
+	vlv_dpio_write(display, phy, VLV_PCS01_DW11(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(display->drm, phy, VLV_PCS23_DW11(ch));
+		val = vlv_dpio_read(display, phy, VLV_PCS23_DW11(ch));
 		val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
-		vlv_dpio_write(display->drm, phy, VLV_PCS23_DW11(ch), val);
+		vlv_dpio_write(display, phy, VLV_PCS23_DW11(ch), val);
 	}
 
 	/* Program Tx lane latency optimal setting*/
@@ -976,7 +976,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 			data = 0;
 		else
 			data = (i == 1) ? 0 : DPIO_UPAR;
-		vlv_dpio_write(display->drm, phy, CHV_TX_DW14(ch, i), data);
+		vlv_dpio_write(display, phy, CHV_TX_DW14(ch, i), data);
 	}
 
 	/* Data lane stagger programming */
@@ -991,17 +991,17 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	else
 		stagger = 0x2;
 
-	val = vlv_dpio_read(display->drm, phy, VLV_PCS01_DW11(ch));
+	val = vlv_dpio_read(display, phy, VLV_PCS01_DW11(ch));
 	val |= DPIO_TX2_STAGGER_MASK(0x1f);
-	vlv_dpio_write(display->drm, phy, VLV_PCS01_DW11(ch), val);
+	vlv_dpio_write(display, phy, VLV_PCS01_DW11(ch), val);
 
 	if (crtc_state->lane_count > 2) {
-		val = vlv_dpio_read(display->drm, phy, VLV_PCS23_DW11(ch));
+		val = vlv_dpio_read(display, phy, VLV_PCS23_DW11(ch));
 		val |= DPIO_TX2_STAGGER_MASK(0x1f);
-		vlv_dpio_write(display->drm, phy, VLV_PCS23_DW11(ch), val);
+		vlv_dpio_write(display, phy, VLV_PCS23_DW11(ch), val);
 	}
 
-	vlv_dpio_write(display->drm, phy, VLV_PCS01_DW12(ch),
+	vlv_dpio_write(display, phy, VLV_PCS01_DW12(ch),
 		       DPIO_LANESTAGGER_STRAP(stagger) |
 		       DPIO_LANESTAGGER_STRAP_OVRD |
 		       DPIO_TX1_STAGGER_MASK(0x1f) |
@@ -1009,7 +1009,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 		       DPIO_TX2_STAGGER_MULT(0));
 
 	if (crtc_state->lane_count > 2) {
-		vlv_dpio_write(display->drm, phy, VLV_PCS23_DW12(ch),
+		vlv_dpio_write(display, phy, VLV_PCS23_DW12(ch),
 			       DPIO_LANESTAGGER_STRAP(stagger) |
 			       DPIO_LANESTAGGER_STRAP_OVRD |
 			       DPIO_TX1_STAGGER_MASK(0x1f) |
@@ -1020,7 +1020,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	/* Deassert data lane reset */
 	__chv_data_lane_soft_reset(encoder, crtc_state, false);
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 void chv_phy_release_cl2_override(struct intel_encoder *encoder)
@@ -1042,20 +1042,20 @@ void chv_phy_post_pll_disable(struct intel_encoder *encoder,
 	enum pipe pipe = to_intel_crtc(old_crtc_state->uapi.crtc)->pipe;
 	u32 val;
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* disable left/right clock distribution */
 	if (pipe != PIPE_B) {
-		val = vlv_dpio_read(display->drm, phy, CHV_CMN_DW5_CH0);
+		val = vlv_dpio_read(display, phy, CHV_CMN_DW5_CH0);
 		val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
-		vlv_dpio_write(display->drm, phy, CHV_CMN_DW5_CH0, val);
+		vlv_dpio_write(display, phy, CHV_CMN_DW5_CH0, val);
 	} else {
-		val = vlv_dpio_read(display->drm, phy, CHV_CMN_DW1_CH1);
+		val = vlv_dpio_read(display, phy, CHV_CMN_DW1_CH1);
 		val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
-		vlv_dpio_write(display->drm, phy, CHV_CMN_DW1_CH1, val);
+		vlv_dpio_write(display, phy, CHV_CMN_DW1_CH1, val);
 	}
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 
 	/*
 	 * Leave the power down bit cleared for at least one
@@ -1079,22 +1079,22 @@ void vlv_set_phy_signal_level(struct intel_encoder *encoder,
 	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
 	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
-	vlv_dpio_write(display->drm, phy, VLV_TX_DW5_GRP(ch), 0x00000000);
-	vlv_dpio_write(display->drm, phy, VLV_TX_DW4_GRP(ch), demph_reg_value);
-	vlv_dpio_write(display->drm, phy, VLV_TX_DW2_GRP(ch),
+	vlv_dpio_write(display, phy, VLV_TX_DW5_GRP(ch), 0x00000000);
+	vlv_dpio_write(display, phy, VLV_TX_DW4_GRP(ch), demph_reg_value);
+	vlv_dpio_write(display, phy, VLV_TX_DW2_GRP(ch),
 		       uniqtranscale_reg_value);
-	vlv_dpio_write(display->drm, phy, VLV_TX_DW3_GRP(ch), 0x0C782040);
+	vlv_dpio_write(display, phy, VLV_TX_DW3_GRP(ch), 0x0C782040);
 
 	if (tx3_demph)
-		vlv_dpio_write(display->drm, phy, VLV_TX_DW4(ch, 3), tx3_demph);
+		vlv_dpio_write(display, phy, VLV_TX_DW4(ch, 3), tx3_demph);
 
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW11_GRP(ch), 0x00030000);
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW9_GRP(ch), preemph_reg_value);
-	vlv_dpio_write(display->drm, phy, VLV_TX_DW5_GRP(ch), DPIO_TX_OCALINIT_EN);
+	vlv_dpio_write(display, phy, VLV_PCS_DW11_GRP(ch), 0x00030000);
+	vlv_dpio_write(display, phy, VLV_PCS_DW9_GRP(ch), preemph_reg_value);
+	vlv_dpio_write(display, phy, VLV_TX_DW5_GRP(ch), DPIO_TX_OCALINIT_EN);
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 void vlv_phy_pre_pll_enable(struct intel_encoder *encoder,
@@ -1106,23 +1106,23 @@ void vlv_phy_pre_pll_enable(struct intel_encoder *encoder,
 	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 
 	/* Program Tx lane resets to default */
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW0_GRP(ch),
+	vlv_dpio_write(display, phy, VLV_PCS_DW0_GRP(ch),
 		       DPIO_PCS_TX_LANE2_RESET |
 		       DPIO_PCS_TX_LANE1_RESET);
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW1_GRP(ch),
+	vlv_dpio_write(display, phy, VLV_PCS_DW1_GRP(ch),
 		       DPIO_PCS_CLK_CRI_RXEB_EIOS_EN |
 		       DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN |
 		       DPIO_PCS_CLK_DATAWIDTH_8_10 |
 		       DPIO_PCS_CLK_SOFT_RESET);
 
 	/* Fix up inter-pair skew failure */
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW12_GRP(ch), 0x00750f00);
-	vlv_dpio_write(display->drm, phy, VLV_TX_DW11_GRP(ch), 0x00001500);
-	vlv_dpio_write(display->drm, phy, VLV_TX_DW14_GRP(ch), 0x40400000);
+	vlv_dpio_write(display, phy, VLV_PCS_DW12_GRP(ch), 0x00750f00);
+	vlv_dpio_write(display, phy, VLV_TX_DW11_GRP(ch), 0x00001500);
+	vlv_dpio_write(display, phy, VLV_TX_DW14_GRP(ch), 0x40400000);
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder,
@@ -1137,20 +1137,20 @@ void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder,
 	enum pipe pipe = crtc->pipe;
 	u32 val;
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* Enable clock channels for this port */
 	val = DPIO_PCS_USEDCLKCHANNEL_OVRRIDE;
 	if (pipe == PIPE_B)
 		val |= DPIO_PCS_USEDCLKCHANNEL;
 	val |= 0xc4;
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW8_GRP(ch), val);
+	vlv_dpio_write(display, phy, VLV_PCS_DW8_GRP(ch), val);
 
 	/* Program lane clock */
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW14_GRP(ch), 0x00760018);
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW23_GRP(ch), 0x00400888);
+	vlv_dpio_write(display, phy, VLV_PCS_DW14_GRP(ch), 0x00760018);
+	vlv_dpio_write(display, phy, VLV_PCS_DW23_GRP(ch), 0x00400888);
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 void vlv_phy_reset_lanes(struct intel_encoder *encoder,
@@ -1161,10 +1161,10 @@ void vlv_phy_reset_lanes(struct intel_encoder *encoder,
 	enum dpio_channel ch = vlv_dig_port_to_channel(dig_port);
 	enum dpio_phy phy = vlv_dig_port_to_phy(dig_port);
 
-	vlv_dpio_get(display->drm);
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW0_GRP(ch), 0x00000000);
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW1_GRP(ch), 0x00e00060);
-	vlv_dpio_put(display->drm);
+	vlv_dpio_get(display);
+	vlv_dpio_write(display, phy, VLV_PCS_DW0_GRP(ch), 0x00000000);
+	vlv_dpio_write(display, phy, VLV_PCS_DW1_GRP(ch), 0x00e00060);
+	vlv_dpio_put(display);
 }
 
 void vlv_wait_port_ready(struct intel_encoder *encoder,
@@ -1172,7 +1172,7 @@ void vlv_wait_port_ready(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	u32 port_mask;
-	i915_reg_t dpll_reg;
+	intel_reg_t dpll_reg;
 	u32 val;
 
 	switch (encoder->port) {
diff --git a/drivers/gpu/drm/i915/display/intel_dpll.c b/drivers/gpu/drm/i915/display/intel_dpll.c
index c7d37e74fbe99..f40807a5566b6 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll.c
@@ -528,9 +528,9 @@ void vlv_crtc_clock_get(struct intel_crtc_state *crtc_state)
 	if ((hw_state->dpll & DPLL_VCO_ENABLE) == 0)
 		return;
 
-	vlv_dpio_get(display->drm);
-	tmp = vlv_dpio_read(display->drm, phy, VLV_PLL_DW3(ch));
-	vlv_dpio_put(display->drm);
+	vlv_dpio_get(display);
+	tmp = vlv_dpio_read(display, phy, VLV_PLL_DW3(ch));
+	vlv_dpio_put(display);
 
 	clock.m1 = REG_FIELD_GET(DPIO_M1_DIV_MASK, tmp);
 	clock.m2 = REG_FIELD_GET(DPIO_M2_DIV_MASK, tmp);
@@ -556,13 +556,13 @@ void chv_crtc_clock_get(struct intel_crtc_state *crtc_state)
 	if ((hw_state->dpll & DPLL_VCO_ENABLE) == 0)
 		return;
 
-	vlv_dpio_get(display->drm);
-	cmn_dw13 = vlv_dpio_read(display->drm, phy, CHV_CMN_DW13(ch));
-	pll_dw0 = vlv_dpio_read(display->drm, phy, CHV_PLL_DW0(ch));
-	pll_dw1 = vlv_dpio_read(display->drm, phy, CHV_PLL_DW1(ch));
-	pll_dw2 = vlv_dpio_read(display->drm, phy, CHV_PLL_DW2(ch));
-	pll_dw3 = vlv_dpio_read(display->drm, phy, CHV_PLL_DW3(ch));
-	vlv_dpio_put(display->drm);
+	vlv_dpio_get(display);
+	cmn_dw13 = vlv_dpio_read(display, phy, CHV_CMN_DW13(ch));
+	pll_dw0 = vlv_dpio_read(display, phy, CHV_PLL_DW0(ch));
+	pll_dw1 = vlv_dpio_read(display, phy, CHV_PLL_DW1(ch));
+	pll_dw2 = vlv_dpio_read(display, phy, CHV_PLL_DW2(ch));
+	pll_dw3 = vlv_dpio_read(display, phy, CHV_PLL_DW3(ch));
+	vlv_dpio_put(display);
 
 	clock.m1 = REG_FIELD_GET(DPIO_CHV_M1_DIV_MASK, pll_dw1) == DPIO_CHV_M1_DIV_BY_2 ? 2 : 0;
 	clock.m2 = REG_FIELD_GET(DPIO_CHV_M2_DIV_MASK, pll_dw0) << 22;
@@ -1735,7 +1735,7 @@ int intel_dpll_crtc_compute_clock(struct intel_atomic_state *state,
 	if (!crtc_state->hw.enable)
 		return 0;
 
-	ret = display->funcs.dpll->crtc_compute_clock(state, crtc);
+	ret = display->dpll.funcs->crtc_compute_clock(state, crtc);
 	if (ret) {
 		drm_dbg_kms(display->drm, "[CRTC:%d:%s] Couldn't calculate DPLL settings\n",
 			    crtc->base.base.id, crtc->base.name);
@@ -1759,10 +1759,10 @@ int intel_dpll_crtc_get_dpll(struct intel_atomic_state *state,
 	if (!crtc_state->hw.enable || crtc_state->intel_dpll)
 		return 0;
 
-	if (!display->funcs.dpll->crtc_get_dpll)
+	if (!display->dpll.funcs->crtc_get_dpll)
 		return 0;
 
-	ret = display->funcs.dpll->crtc_get_dpll(state, crtc);
+	ret = display->dpll.funcs->crtc_get_dpll(state, crtc);
 	if (ret) {
 		drm_dbg_kms(display->drm, "[CRTC:%d:%s] Couldn't get a shared DPLL\n",
 			    crtc->base.base.id, crtc->base.name);
@@ -1776,27 +1776,27 @@ void
 intel_dpll_init_clock_hook(struct intel_display *display)
 {
 	if (HAS_LT_PHY(display))
-		display->funcs.dpll = &xe3plpd_dpll_funcs;
+		display->dpll.funcs = &xe3plpd_dpll_funcs;
 	else if (DISPLAY_VER(display) >= 14)
-		display->funcs.dpll = &mtl_dpll_funcs;
+		display->dpll.funcs = &mtl_dpll_funcs;
 	else if (display->platform.dg2)
-		display->funcs.dpll = &dg2_dpll_funcs;
+		display->dpll.funcs = &dg2_dpll_funcs;
 	else if (DISPLAY_VER(display) >= 9 || HAS_DDI(display))
-		display->funcs.dpll = &hsw_dpll_funcs;
+		display->dpll.funcs = &hsw_dpll_funcs;
 	else if (HAS_PCH_SPLIT(display))
-		display->funcs.dpll = &ilk_dpll_funcs;
+		display->dpll.funcs = &ilk_dpll_funcs;
 	else if (display->platform.cherryview)
-		display->funcs.dpll = &chv_dpll_funcs;
+		display->dpll.funcs = &chv_dpll_funcs;
 	else if (display->platform.valleyview)
-		display->funcs.dpll = &vlv_dpll_funcs;
+		display->dpll.funcs = &vlv_dpll_funcs;
 	else if (display->platform.g4x)
-		display->funcs.dpll = &g4x_dpll_funcs;
+		display->dpll.funcs = &g4x_dpll_funcs;
 	else if (display->platform.pineview)
-		display->funcs.dpll = &pnv_dpll_funcs;
+		display->dpll.funcs = &pnv_dpll_funcs;
 	else if (DISPLAY_VER(display) != 2)
-		display->funcs.dpll = &i9xx_dpll_funcs;
+		display->dpll.funcs = &i9xx_dpll_funcs;
 	else
-		display->funcs.dpll = &i8xx_dpll_funcs;
+		display->dpll.funcs = &i8xx_dpll_funcs;
 }
 
 static bool i9xx_has_pps(struct intel_display *display)
@@ -1866,24 +1866,24 @@ static void vlv_pllb_recal_opamp(struct intel_display *display,
 	 * PLLB opamp always calibrates to max value of 0x3f, force enable it
 	 * and set it to a reasonable value instead.
 	 */
-	tmp = vlv_dpio_read(display->drm, phy, VLV_PLL_DW17(ch));
+	tmp = vlv_dpio_read(display, phy, VLV_PLL_DW17(ch));
 	tmp &= 0xffffff00;
 	tmp |= 0x00000030;
-	vlv_dpio_write(display->drm, phy, VLV_PLL_DW17(ch), tmp);
+	vlv_dpio_write(display, phy, VLV_PLL_DW17(ch), tmp);
 
-	tmp = vlv_dpio_read(display->drm, phy, VLV_REF_DW11);
+	tmp = vlv_dpio_read(display, phy, VLV_REF_DW11);
 	tmp &= 0x00ffffff;
 	tmp |= 0x8c000000;
-	vlv_dpio_write(display->drm, phy, VLV_REF_DW11, tmp);
+	vlv_dpio_write(display, phy, VLV_REF_DW11, tmp);
 
-	tmp = vlv_dpio_read(display->drm, phy, VLV_PLL_DW17(ch));
+	tmp = vlv_dpio_read(display, phy, VLV_PLL_DW17(ch));
 	tmp &= 0xffffff00;
-	vlv_dpio_write(display->drm, phy, VLV_PLL_DW17(ch), tmp);
+	vlv_dpio_write(display, phy, VLV_PLL_DW17(ch), tmp);
 
-	tmp = vlv_dpio_read(display->drm, phy, VLV_REF_DW11);
+	tmp = vlv_dpio_read(display, phy, VLV_REF_DW11);
 	tmp &= 0x00ffffff;
 	tmp |= 0xb0000000;
-	vlv_dpio_write(display->drm, phy, VLV_REF_DW11, tmp);
+	vlv_dpio_write(display, phy, VLV_REF_DW11, tmp);
 }
 
 static void vlv_prepare_pll(const struct intel_crtc_state *crtc_state)
@@ -1896,7 +1896,7 @@ static void vlv_prepare_pll(const struct intel_crtc_state *crtc_state)
 	enum pipe pipe = crtc->pipe;
 	u32 tmp, coreclk;
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* See eDP HDMI DPIO driver vbios notes doc */
 
@@ -1905,15 +1905,15 @@ static void vlv_prepare_pll(const struct intel_crtc_state *crtc_state)
 		vlv_pllb_recal_opamp(display, phy, ch);
 
 	/* Set up Tx target for periodic Rcomp update */
-	vlv_dpio_write(display->drm, phy, VLV_PCS_DW17_BCAST, 0x0100000f);
+	vlv_dpio_write(display, phy, VLV_PCS_DW17_BCAST, 0x0100000f);
 
 	/* Disable target IRef on PLL */
-	tmp = vlv_dpio_read(display->drm, phy, VLV_PLL_DW16(ch));
+	tmp = vlv_dpio_read(display, phy, VLV_PLL_DW16(ch));
 	tmp &= 0x00ffffff;
-	vlv_dpio_write(display->drm, phy, VLV_PLL_DW16(ch), tmp);
+	vlv_dpio_write(display, phy, VLV_PLL_DW16(ch), tmp);
 
 	/* Disable fast lock */
-	vlv_dpio_write(display->drm, phy, VLV_CMN_DW0, 0x610);
+	vlv_dpio_write(display, phy, VLV_CMN_DW0, 0x610);
 
 	/* Set idtafcrecal before PLL is enabled */
 	tmp = DPIO_M1_DIV(clock->m1) |
@@ -1929,42 +1929,42 @@ static void vlv_prepare_pll(const struct intel_crtc_state *crtc_state)
 	 * Note: don't use the DAC post divider as it seems unstable.
 	 */
 	tmp |= DPIO_S1_DIV(DPIO_S1_DIV_HDMIDP);
-	vlv_dpio_write(display->drm, phy, VLV_PLL_DW3(ch), tmp);
+	vlv_dpio_write(display, phy, VLV_PLL_DW3(ch), tmp);
 
 	tmp |= DPIO_ENABLE_CALIBRATION;
-	vlv_dpio_write(display->drm, phy, VLV_PLL_DW3(ch), tmp);
+	vlv_dpio_write(display, phy, VLV_PLL_DW3(ch), tmp);
 
 	/* Set HBR and RBR LPF coefficients */
 	if (crtc_state->port_clock == 162000 ||
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG) ||
 	    intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
-		vlv_dpio_write(display->drm, phy, VLV_PLL_DW18(ch), 0x009f0003);
+		vlv_dpio_write(display, phy, VLV_PLL_DW18(ch), 0x009f0003);
 	else
-		vlv_dpio_write(display->drm, phy, VLV_PLL_DW18(ch), 0x00d0000f);
+		vlv_dpio_write(display, phy, VLV_PLL_DW18(ch), 0x00d0000f);
 
 	if (intel_crtc_has_dp_encoder(crtc_state)) {
 		/* Use SSC source */
 		if (pipe == PIPE_A)
-			vlv_dpio_write(display->drm, phy, VLV_PLL_DW5(ch), 0x0df40000);
+			vlv_dpio_write(display, phy, VLV_PLL_DW5(ch), 0x0df40000);
 		else
-			vlv_dpio_write(display->drm, phy, VLV_PLL_DW5(ch), 0x0df70000);
+			vlv_dpio_write(display, phy, VLV_PLL_DW5(ch), 0x0df70000);
 	} else { /* HDMI or VGA */
 		/* Use bend source */
 		if (pipe == PIPE_A)
-			vlv_dpio_write(display->drm, phy, VLV_PLL_DW5(ch), 0x0df70000);
+			vlv_dpio_write(display, phy, VLV_PLL_DW5(ch), 0x0df70000);
 		else
-			vlv_dpio_write(display->drm, phy, VLV_PLL_DW5(ch), 0x0df40000);
+			vlv_dpio_write(display, phy, VLV_PLL_DW5(ch), 0x0df40000);
 	}
 
-	coreclk = vlv_dpio_read(display->drm, phy, VLV_PLL_DW7(ch));
+	coreclk = vlv_dpio_read(display, phy, VLV_PLL_DW7(ch));
 	coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
 	if (intel_crtc_has_dp_encoder(crtc_state))
 		coreclk |= 0x01000000;
-	vlv_dpio_write(display->drm, phy, VLV_PLL_DW7(ch), coreclk);
+	vlv_dpio_write(display, phy, VLV_PLL_DW7(ch), coreclk);
 
-	vlv_dpio_write(display->drm, phy, VLV_PLL_DW19(ch), 0x87871000);
+	vlv_dpio_write(display, phy, VLV_PLL_DW19(ch), 0x87871000);
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 static void _vlv_enable_pll(const struct intel_crtc_state *crtc_state)
@@ -2019,44 +2019,44 @@ static void chv_prepare_pll(const struct intel_crtc_state *crtc_state)
 
 	m2_frac = clock->m2 & 0x3fffff;
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* p1 and p2 divider */
-	vlv_dpio_write(display->drm, phy, CHV_CMN_DW13(ch),
+	vlv_dpio_write(display, phy, CHV_CMN_DW13(ch),
 		       DPIO_CHV_S1_DIV(5) |
 		       DPIO_CHV_P1_DIV(clock->p1) |
 		       DPIO_CHV_P2_DIV(clock->p2) |
 		       DPIO_CHV_K_DIV(1));
 
 	/* Feedback post-divider - m2 */
-	vlv_dpio_write(display->drm, phy, CHV_PLL_DW0(ch),
+	vlv_dpio_write(display, phy, CHV_PLL_DW0(ch),
 		       DPIO_CHV_M2_DIV(clock->m2 >> 22));
 
 	/* Feedback refclk divider - n and m1 */
-	vlv_dpio_write(display->drm, phy, CHV_PLL_DW1(ch),
+	vlv_dpio_write(display, phy, CHV_PLL_DW1(ch),
 		       DPIO_CHV_M1_DIV(DPIO_CHV_M1_DIV_BY_2) |
 		       DPIO_CHV_N_DIV(1));
 
 	/* M2 fraction division */
-	vlv_dpio_write(display->drm, phy, CHV_PLL_DW2(ch),
+	vlv_dpio_write(display, phy, CHV_PLL_DW2(ch),
 		       DPIO_CHV_M2_FRAC_DIV(m2_frac));
 
 	/* M2 fraction division enable */
-	tmp = vlv_dpio_read(display->drm, phy, CHV_PLL_DW3(ch));
+	tmp = vlv_dpio_read(display, phy, CHV_PLL_DW3(ch));
 	tmp &= ~(DPIO_CHV_FEEDFWD_GAIN_MASK | DPIO_CHV_FRAC_DIV_EN);
 	tmp |= DPIO_CHV_FEEDFWD_GAIN(2);
 	if (m2_frac)
 		tmp |= DPIO_CHV_FRAC_DIV_EN;
-	vlv_dpio_write(display->drm, phy, CHV_PLL_DW3(ch), tmp);
+	vlv_dpio_write(display, phy, CHV_PLL_DW3(ch), tmp);
 
 	/* Program digital lock detect threshold */
-	tmp = vlv_dpio_read(display->drm, phy, CHV_PLL_DW9(ch));
+	tmp = vlv_dpio_read(display, phy, CHV_PLL_DW9(ch));
 	tmp &= ~(DPIO_CHV_INT_LOCK_THRESHOLD_MASK |
 		      DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE);
 	tmp |= DPIO_CHV_INT_LOCK_THRESHOLD(0x5);
 	if (!m2_frac)
 		tmp |= DPIO_CHV_INT_LOCK_THRESHOLD_SEL_COARSE;
-	vlv_dpio_write(display->drm, phy, CHV_PLL_DW9(ch), tmp);
+	vlv_dpio_write(display, phy, CHV_PLL_DW9(ch), tmp);
 
 	/* Loop filter */
 	if (clock->vco == 5400000) {
@@ -2081,19 +2081,19 @@ static void chv_prepare_pll(const struct intel_crtc_state *crtc_state)
 			DPIO_CHV_GAIN_CTRL(0x3);
 		tribuf_calcntr = 0;
 	}
-	vlv_dpio_write(display->drm, phy, CHV_PLL_DW6(ch), loopfilter);
+	vlv_dpio_write(display, phy, CHV_PLL_DW6(ch), loopfilter);
 
-	tmp = vlv_dpio_read(display->drm, phy, CHV_PLL_DW8(ch));
+	tmp = vlv_dpio_read(display, phy, CHV_PLL_DW8(ch));
 	tmp &= ~DPIO_CHV_TDC_TARGET_CNT_MASK;
 	tmp |= DPIO_CHV_TDC_TARGET_CNT(tribuf_calcntr);
-	vlv_dpio_write(display->drm, phy, CHV_PLL_DW8(ch), tmp);
+	vlv_dpio_write(display, phy, CHV_PLL_DW8(ch), tmp);
 
 	/* AFC Recal */
-	vlv_dpio_write(display->drm, phy, CHV_CMN_DW14(ch),
-		       vlv_dpio_read(display->drm, phy, CHV_CMN_DW14(ch)) |
+	vlv_dpio_write(display, phy, CHV_CMN_DW14(ch),
+		       vlv_dpio_read(display, phy, CHV_CMN_DW14(ch)) |
 		       DPIO_AFC_RECAL);
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 static void _chv_enable_pll(const struct intel_crtc_state *crtc_state)
@@ -2106,14 +2106,14 @@ static void _chv_enable_pll(const struct intel_crtc_state *crtc_state)
 	enum pipe pipe = crtc->pipe;
 	u32 tmp;
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* Enable back the 10bit clock to display controller */
-	tmp = vlv_dpio_read(display->drm, phy, CHV_CMN_DW14(ch));
+	tmp = vlv_dpio_read(display, phy, CHV_CMN_DW14(ch));
 	tmp |= DPIO_DCLKP_EN;
-	vlv_dpio_write(display->drm, phy, CHV_CMN_DW14(ch), tmp);
+	vlv_dpio_write(display, phy, CHV_CMN_DW14(ch), tmp);
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 
 	/*
 	 * Need to wait > 100ns between dclkp clock enable bit and PLL enable.
@@ -2247,14 +2247,14 @@ void chv_disable_pll(struct intel_display *display, enum pipe pipe)
 	intel_de_write(display, DPLL(display, pipe), val);
 	intel_de_posting_read(display, DPLL(display, pipe));
 
-	vlv_dpio_get(display->drm);
+	vlv_dpio_get(display);
 
 	/* Disable 10bit clock to display controller */
-	val = vlv_dpio_read(display->drm, phy, CHV_CMN_DW14(ch));
+	val = vlv_dpio_read(display, phy, CHV_CMN_DW14(ch));
 	val &= ~DPIO_DCLKP_EN;
-	vlv_dpio_write(display->drm, phy, CHV_CMN_DW14(ch), val);
+	vlv_dpio_write(display, phy, CHV_CMN_DW14(ch), val);
 
-	vlv_dpio_put(display->drm);
+	vlv_dpio_put(display);
 }
 
 void i9xx_disable_pll(const struct intel_crtc_state *crtc_state)
diff --git a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
index f5d4f7146fbc6..bb487e647f767 100644
--- a/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/display/intel_dpll_mgr.c
@@ -25,6 +25,7 @@
 #include <linux/string_helpers.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/step.h>
 
 #include "bxt_dpio_phy_regs.h"
 #include "intel_cx0_phy.h"
@@ -41,7 +42,6 @@
 #include "intel_lt_phy.h"
 #include "intel_mg_phy_regs.h"
 #include "intel_pch_refclk.h"
-#include "intel_step.h"
 #include "intel_tc.h"
 
 /**
@@ -134,7 +134,7 @@ intel_atomic_duplicate_dpll_state(struct intel_display *display,
 }
 
 static struct intel_dpll_state *
-intel_atomic_get_dpll_state(struct drm_atomic_state *s)
+intel_atomic_get_dpll_state(struct drm_atomic_commit *s)
 {
 	struct intel_atomic_state *state = to_intel_atomic_state(s);
 	struct intel_display *display = to_intel_display(state);
@@ -220,7 +220,7 @@ enum intel_dpll_id mtl_port_to_pll_id(struct intel_display *display, enum port p
 	}
 }
 
-static i915_reg_t
+static intel_reg_t
 intel_combo_pll_enable_reg(struct intel_display *display,
 			   struct intel_dpll *pll)
 {
@@ -233,7 +233,7 @@ intel_combo_pll_enable_reg(struct intel_display *display,
 	return ICL_DPLL_ENABLE(pll->info->id);
 }
 
-static i915_reg_t
+static intel_reg_t
 intel_tc_pll_enable_reg(struct intel_display *display,
 			struct intel_dpll *pll)
 {
@@ -1350,7 +1350,7 @@ static const struct intel_dpll_mgr hsw_pll_mgr = {
 };
 
 struct skl_dpll_regs {
-	i915_reg_t ctl, cfgcr1, cfgcr2;
+	intel_reg_t ctl, cfgcr1, cfgcr2;
 };
 
 /* this array is indexed by the *shared* pll id */
@@ -3603,7 +3603,7 @@ static bool mg_pll_get_hw_state(struct intel_display *display,
 	bool ret = false;
 	u32 val;
 
-	i915_reg_t enable_reg = intel_tc_pll_enable_reg(display, pll);
+	intel_reg_t enable_reg = intel_tc_pll_enable_reg(display, pll);
 
 	wakeref = intel_display_power_get_if_enabled(display,
 						     POWER_DOMAIN_DISPLAY_CORE);
@@ -3734,7 +3734,7 @@ static bool dkl_pll_get_hw_state(struct intel_display *display,
 static bool icl_pll_get_hw_state(struct intel_display *display,
 				 struct intel_dpll *pll,
 				 struct intel_dpll_hw_state *dpll_hw_state,
-				 i915_reg_t enable_reg)
+				 intel_reg_t enable_reg)
 {
 	struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
 	const enum intel_dpll_id id = pll->info->id;
@@ -3796,7 +3796,7 @@ static bool combo_pll_get_hw_state(struct intel_display *display,
 				   struct intel_dpll *pll,
 				   struct intel_dpll_hw_state *dpll_hw_state)
 {
-	i915_reg_t enable_reg = intel_combo_pll_enable_reg(display, pll);
+	intel_reg_t enable_reg = intel_combo_pll_enable_reg(display, pll);
 
 	return icl_pll_get_hw_state(display, pll, dpll_hw_state, enable_reg);
 }
@@ -3813,7 +3813,7 @@ static void icl_dpll_write(struct intel_display *display,
 			   const struct icl_dpll_hw_state *hw_state)
 {
 	const enum intel_dpll_id id = pll->info->id;
-	i915_reg_t cfgcr0_reg, cfgcr1_reg, div0_reg = INVALID_MMIO_REG;
+	intel_reg_t cfgcr0_reg, cfgcr1_reg, div0_reg = INVALID_MMIO_REG;
 
 	if (display->platform.alderlake_s) {
 		cfgcr0_reg = ADLS_DPLL_CFGCR0(id);
@@ -3842,9 +3842,9 @@ static void icl_dpll_write(struct intel_display *display,
 	intel_de_write(display, cfgcr0_reg, hw_state->cfgcr0);
 	intel_de_write(display, cfgcr1_reg, hw_state->cfgcr1);
 	drm_WARN_ON_ONCE(display->drm, display->vbt.override_afc_startup &&
-			 !i915_mmio_reg_valid(div0_reg));
+			 !intel_reg_valid(div0_reg));
 	if (display->vbt.override_afc_startup &&
-	    i915_mmio_reg_valid(div0_reg))
+	    intel_reg_valid(div0_reg))
 		intel_de_rmw(display, div0_reg,
 			     TGL_DPLL0_DIV0_AFC_STARTUP_MASK, hw_state->div0);
 	intel_de_posting_read(display, cfgcr1_reg);
@@ -3960,7 +3960,7 @@ static void dkl_pll_write(struct intel_display *display,
 
 static void icl_pll_power_enable(struct intel_display *display,
 				 struct intel_dpll *pll,
-				 i915_reg_t enable_reg)
+				 intel_reg_t enable_reg)
 {
 	intel_de_rmw(display, enable_reg, 0, PLL_POWER_ENABLE);
 
@@ -3975,7 +3975,7 @@ static void icl_pll_power_enable(struct intel_display *display,
 
 static void icl_pll_enable(struct intel_display *display,
 			   struct intel_dpll *pll,
-			   i915_reg_t enable_reg)
+			   intel_reg_t enable_reg)
 {
 	intel_de_rmw(display, enable_reg, 0, PLL_ENABLE);
 
@@ -4013,7 +4013,7 @@ static void combo_pll_enable(struct intel_display *display,
 			     const struct intel_dpll_hw_state *dpll_hw_state)
 {
 	const struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
-	i915_reg_t enable_reg = intel_combo_pll_enable_reg(display, pll);
+	intel_reg_t enable_reg = intel_combo_pll_enable_reg(display, pll);
 
 	icl_pll_power_enable(display, pll, enable_reg);
 
@@ -4058,7 +4058,7 @@ static void mg_pll_enable(struct intel_display *display,
 			  const struct intel_dpll_hw_state *dpll_hw_state)
 {
 	const struct icl_dpll_hw_state *hw_state = &dpll_hw_state->icl;
-	i915_reg_t enable_reg = intel_tc_pll_enable_reg(display, pll);
+	intel_reg_t enable_reg = intel_tc_pll_enable_reg(display, pll);
 
 	icl_pll_power_enable(display, pll, enable_reg);
 
@@ -4080,7 +4080,7 @@ static void mg_pll_enable(struct intel_display *display,
 
 static void icl_pll_disable(struct intel_display *display,
 			    struct intel_dpll *pll,
-			    i915_reg_t enable_reg)
+			    intel_reg_t enable_reg)
 {
 	/* The first steps are done by intel_ddi_post_disable(). */
 
@@ -4112,7 +4112,7 @@ static void icl_pll_disable(struct intel_display *display,
 static void combo_pll_disable(struct intel_display *display,
 			      struct intel_dpll *pll)
 {
-	i915_reg_t enable_reg = intel_combo_pll_enable_reg(display, pll);
+	intel_reg_t enable_reg = intel_combo_pll_enable_reg(display, pll);
 
 	icl_pll_disable(display, pll, enable_reg);
 }
@@ -4126,7 +4126,7 @@ static void icl_tbt_pll_disable(struct intel_display *display,
 static void mg_pll_disable(struct intel_display *display,
 			   struct intel_dpll *pll)
 {
-	i915_reg_t enable_reg = intel_tc_pll_enable_reg(display, pll);
+	intel_reg_t enable_reg = intel_tc_pll_enable_reg(display, pll);
 
 	icl_pll_disable(display, pll, enable_reg);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_dram.c b/drivers/gpu/drm/i915/display/intel_dram.c
index bd281d4b4c05c..f103f7cba0183 100644
--- a/drivers/gpu/drm/i915/display/intel_dram.c
+++ b/drivers/gpu/drm/i915/display/intel_dram.c
@@ -9,14 +9,14 @@
 #include <drm/drm_print.h>
 #include <drm/intel/intel_pcode_regs.h>
 
+#include "intel_de.h"
 #include "intel_display_core.h"
 #include "intel_display_utils.h"
 #include "intel_display_regs.h"
 #include "intel_dram.h"
-#include "intel_mchbar_regs.h"
+#include "intel_mchbar.h"
 #include "intel_parent.h"
-#include "intel_uncore.h"
-#include "vlv_iosf_sb.h"
+#include "vlv_sideband.h"
 
 struct dram_dimm_info {
 	u16 size;
@@ -58,18 +58,15 @@ const char *intel_dram_type_str(enum intel_dram_type type)
 
 static enum intel_dram_type pnv_dram_type(struct intel_display *display)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
-
-	return intel_uncore_read(uncore, CSHRDDR3CTL) & CSHRDDR3CTL_DDR3 ?
+	return intel_mchbar_read(display, CSHRDDR3CTL) & CSHRDDR3CTL_DDR3 ?
 		INTEL_DRAM_DDR3 : INTEL_DRAM_DDR2;
 }
 
 static unsigned int pnv_mem_freq(struct intel_display *display)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u32 tmp;
 
-	tmp = intel_uncore_read(uncore, CLKCFG);
+	tmp = intel_mchbar_read(display, CLKCFG);
 
 	switch (tmp & CLKCFG_MEM_MASK) {
 	case CLKCFG_MEM_533:
@@ -85,10 +82,9 @@ static unsigned int pnv_mem_freq(struct intel_display *display)
 
 static unsigned int ilk_mem_freq(struct intel_display *display)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u16 ddrpll;
 
-	ddrpll = intel_uncore_read16(uncore, DDRMPLL1);
+	ddrpll = intel_mchbar_read16(display, DDRMPLL1);
 	switch (ddrpll & 0xff) {
 	case 0xc:
 		return 800000;
@@ -109,9 +105,9 @@ static unsigned int chv_mem_freq(struct intel_display *display)
 {
 	u32 val;
 
-	vlv_iosf_sb_get(display->drm, BIT(VLV_IOSF_SB_CCK));
-	val = vlv_iosf_sb_read(display->drm, VLV_IOSF_SB_CCK, CCK_FUSE_REG);
-	vlv_iosf_sb_put(display->drm, BIT(VLV_IOSF_SB_CCK));
+	vlv_cck_get(display);
+	val = vlv_cck_read(display, CCK_FUSE_REG);
+	vlv_cck_put(display);
 
 	switch ((val >> 2) & 0x7) {
 	case 3:
@@ -125,9 +121,9 @@ static unsigned int vlv_mem_freq(struct intel_display *display)
 {
 	u32 val;
 
-	vlv_iosf_sb_get(display->drm, BIT(VLV_IOSF_SB_PUNIT));
-	val = vlv_iosf_sb_read(display->drm, VLV_IOSF_SB_PUNIT, PUNIT_REG_GPU_FREQ_STS);
-	vlv_iosf_sb_put(display->drm, BIT(VLV_IOSF_SB_PUNIT));
+	vlv_punit_get(display);
+	val = vlv_punit_read(display, PUNIT_REG_GPU_FREQ_STS);
+	vlv_punit_put(display);
 
 	switch ((val >> 6) & 3) {
 	case 0:
@@ -158,7 +154,6 @@ unsigned int intel_mem_freq(struct intel_display *display)
 
 static unsigned int i9xx_fsb_freq(struct intel_display *display)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u32 fsb;
 
 	/*
@@ -169,7 +164,7 @@ static unsigned int i9xx_fsb_freq(struct intel_display *display)
 	 * don't know which registers have that information,
 	 * and all the relevant docs have gone to bit heaven :(
 	 */
-	fsb = intel_uncore_read(uncore, CLKCFG) & CLKCFG_FSB_MASK;
+	fsb = intel_mchbar_read(display, CLKCFG) & CLKCFG_FSB_MASK;
 
 	if (display->platform.pineview || display->platform.mobile) {
 		switch (fsb) {
@@ -214,10 +209,9 @@ static unsigned int i9xx_fsb_freq(struct intel_display *display)
 
 static unsigned int ilk_fsb_freq(struct intel_display *display)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u16 fsb;
 
-	fsb = intel_uncore_read16(uncore, CSIPLL0) & 0x3ff;
+	fsb = intel_mchbar_read16(display, CSIPLL0) & 0x3ff;
 
 	switch (fsb) {
 	case 0x00c:
@@ -484,7 +478,6 @@ intel_is_dram_symmetric(const struct dram_channel_info *ch0,
 static int
 skl_dram_get_channels_info(struct intel_display *display, struct dram_info *dram_info)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	struct dram_channel_info ch0 = {}, ch1 = {};
 	u32 val;
 	int ret;
@@ -492,12 +485,12 @@ skl_dram_get_channels_info(struct intel_display *display, struct dram_info *dram
 	/* Assume 16Gb+ DIMMs are present until proven otherwise */
 	dram_info->has_16gb_dimms = true;
 
-	val = intel_uncore_read(uncore, SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
+	val = intel_mchbar_read(display, SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN);
 	ret = skl_dram_get_channel_info(display, &ch0, 0, val);
 	if (ret == 0)
 		dram_info->num_channels++;
 
-	val = intel_uncore_read(uncore, SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
+	val = intel_mchbar_read(display, SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN);
 	ret = skl_dram_get_channel_info(display, &ch1, 1, val);
 	if (ret == 0)
 		dram_info->num_channels++;
@@ -528,10 +521,9 @@ skl_dram_get_channels_info(struct intel_display *display, struct dram_info *dram
 static enum intel_dram_type
 skl_get_dram_type(struct intel_display *display)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u32 val;
 
-	val = intel_uncore_read(uncore, SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN);
+	val = intel_mchbar_read(display, SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN);
 
 	switch (val & SKL_DRAM_DDR_TYPE_MASK) {
 	case SKL_DRAM_DDR_TYPE_DDR3:
@@ -642,7 +634,6 @@ static void bxt_get_dimm_info(struct dram_dimm_info *dimm, u32 val)
 
 static int bxt_get_dram_info(struct intel_display *display, struct dram_info *dram_info)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
 	u32 val;
 	u8 valid_ranks = 0;
 	int i;
@@ -654,7 +645,7 @@ static int bxt_get_dram_info(struct intel_display *display, struct dram_info *dr
 		struct dram_dimm_info dimm;
 		enum intel_dram_type type;
 
-		val = intel_uncore_read(uncore, BXT_D_CR_DRP0_DUNIT(i));
+		val = intel_mchbar_read(display, BXT_D_CR_DRP0_DUNIT(i));
 		if (val == 0xFFFFFFFF)
 			continue;
 
@@ -767,8 +758,7 @@ static int gen12_get_dram_info(struct intel_display *display, struct dram_info *
 
 static int xelpdp_get_dram_info(struct intel_display *display, struct dram_info *dram_info)
 {
-	struct intel_uncore *uncore = to_intel_uncore(display->drm);
-	u32 val = intel_uncore_read(uncore, MTL_MEM_SS_INFO_GLOBAL);
+	u32 val = intel_de_read(display, MTL_MEM_SS_INFO_GLOBAL);
 
 	switch (REG_FIELD_GET(MTL_DDR_TYPE_MASK, val)) {
 	case 0:
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.c b/drivers/gpu/drm/i915/display/intel_dsb.c
index c8d3968f659f0..fec8a56e21eaf 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.c
+++ b/drivers/gpu/drm/i915/display/intel_dsb.c
@@ -326,7 +326,7 @@ static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
 }
 
 static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
-					u32 opcode, i915_reg_t reg)
+					u32 opcode, intel_reg_t reg)
 {
 	u32 prev_opcode, prev_reg;
 
@@ -341,10 +341,10 @@ static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
 	prev_opcode = dsb->ins[1] & ~DSB_REG_VALUE_MASK;
 	prev_reg =  dsb->ins[1] & DSB_REG_VALUE_MASK;
 
-	return prev_opcode == opcode && prev_reg == i915_mmio_reg_offset(reg);
+	return prev_opcode == opcode && prev_reg == intel_reg_offset(reg);
 }
 
-static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_t reg)
+static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, intel_reg_t reg)
 {
 	return intel_dsb_prev_ins_is_write(dsb,
 					   DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT,
@@ -365,7 +365,7 @@ static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_
  * register.
  */
 void intel_dsb_reg_write_indexed(struct intel_dsb *dsb,
-				 i915_reg_t reg, u32 val)
+				 intel_reg_t reg, u32 val)
 {
 	/*
 	 * For example the buffer will look like below for 3 dwords for auto
@@ -386,7 +386,7 @@ void intel_dsb_reg_write_indexed(struct intel_dsb *dsb,
 	if (!intel_dsb_prev_ins_is_indexed_write(dsb, reg))
 		intel_dsb_emit(dsb, 0, /* count */
 			       (DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
-			       i915_mmio_reg_offset(reg));
+			       intel_reg_offset(reg));
 
 	if (!assert_dsb_has_room(dsb))
 		return;
@@ -402,12 +402,12 @@ void intel_dsb_reg_write_indexed(struct intel_dsb *dsb,
 }
 
 void intel_dsb_reg_write(struct intel_dsb *dsb,
-			 i915_reg_t reg, u32 val)
+			 intel_reg_t reg, u32 val)
 {
 	intel_dsb_emit(dsb, val,
 		       (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
 		       (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) |
-		       i915_mmio_reg_offset(reg));
+		       intel_reg_offset(reg));
 }
 
 static u32 intel_dsb_mask_to_byte_en(u32 mask)
@@ -420,12 +420,12 @@ static u32 intel_dsb_mask_to_byte_en(u32 mask)
 
 /* Note: mask implemented via byte enables! */
 void intel_dsb_reg_write_masked(struct intel_dsb *dsb,
-				i915_reg_t reg, u32 mask, u32 val)
+				intel_reg_t reg, u32 mask, u32 val)
 {
 	intel_dsb_emit(dsb, val,
 		       (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
 		       (intel_dsb_mask_to_byte_en(mask) << DSB_BYTE_EN_SHIFT) |
-		       i915_mmio_reg_offset(reg));
+		       intel_reg_offset(reg));
 }
 
 void intel_dsb_noop(struct intel_dsb *dsb, int count)
@@ -550,7 +550,7 @@ void intel_dsb_wait_scanline_out(struct intel_atomic_state *state,
 }
 
 void intel_dsb_poll(struct intel_dsb *dsb,
-		    i915_reg_t reg, u32 mask, u32 val,
+		    intel_reg_t reg, u32 mask, u32 val,
 		    int wait_us, int count)
 {
 	struct intel_crtc *crtc = dsb->crtc;
@@ -565,7 +565,7 @@ void intel_dsb_poll(struct intel_dsb *dsb,
 
 	intel_dsb_emit(dsb, val,
 		       (DSB_OPCODE_POLL << DSB_OPCODE_SHIFT) |
-		       i915_mmio_reg_offset(reg));
+		       intel_reg_offset(reg));
 }
 
 static void intel_dsb_align_tail(struct intel_dsb *dsb)
diff --git a/drivers/gpu/drm/i915/display/intel_dsb.h b/drivers/gpu/drm/i915/display/intel_dsb.h
index 386a5a942572e..3dcca9ed53713 100644
--- a/drivers/gpu/drm/i915/display/intel_dsb.h
+++ b/drivers/gpu/drm/i915/display/intel_dsb.h
@@ -8,7 +8,7 @@
 
 #include <linux/types.h>
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 
 struct intel_atomic_state;
 struct intel_crtc;
@@ -37,11 +37,11 @@ void intel_dsb_gosub_finish(struct intel_dsb *dsb);
 void intel_dsb_cleanup(struct intel_dsb *dsb);
 int intel_dsb_exec_time_us(void);
 void intel_dsb_reg_write(struct intel_dsb *dsb,
-			 i915_reg_t reg, u32 val);
+			 intel_reg_t reg, u32 val);
 void intel_dsb_reg_write_indexed(struct intel_dsb *dsb,
-				 i915_reg_t reg, u32 val);
+				 intel_reg_t reg, u32 val);
 void intel_dsb_reg_write_masked(struct intel_dsb *dsb,
-				i915_reg_t reg, u32 mask, u32 val);
+				intel_reg_t reg, u32 mask, u32 val);
 void intel_dsb_noop(struct intel_dsb *dsb, int count);
 void intel_dsb_nonpost_start(struct intel_dsb *dsb);
 void intel_dsb_nonpost_end(struct intel_dsb *dsb);
@@ -59,7 +59,7 @@ void intel_dsb_wait_scanline_out(struct intel_atomic_state *state,
 void intel_dsb_vblank_evade(struct intel_atomic_state *state,
 			    struct intel_dsb *dsb);
 void intel_dsb_poll(struct intel_dsb *dsb,
-		    i915_reg_t reg, u32 mask, u32 val,
+		    intel_reg_t reg, u32 mask, u32 val,
 		    int wait_us, int count);
 void intel_dsb_gosub(struct intel_dsb *dsb,
 		     struct intel_dsb *sub_dsb);
diff --git a/drivers/gpu/drm/i915/display/intel_dsi.c b/drivers/gpu/drm/i915/display/intel_dsi.c
index 9005c1f5d857e..3a111f041071a 100644
--- a/drivers/gpu/drm/i915/display/intel_dsi.c
+++ b/drivers/gpu/drm/i915/display/intel_dsi.c
@@ -63,18 +63,17 @@ enum drm_mode_status intel_dsi_mode_valid(struct drm_connector *connector,
 {
 	struct intel_display *display = to_intel_display(connector->dev);
 	struct intel_connector *intel_connector = to_intel_connector(connector);
-	const struct drm_display_mode *fixed_mode =
-		intel_panel_fixed_mode(intel_connector, mode);
 	int max_dotclk = display->cdclk.max_dotclk_freq;
 	enum drm_mode_status status;
+	int target_clock;
 
 	drm_dbg_kms(display->drm, "\n");
 
-	status = intel_panel_mode_valid(intel_connector, mode);
+	status = intel_panel_mode_valid(intel_connector, mode, &target_clock);
 	if (status != MODE_OK)
 		return status;
 
-	if (fixed_mode->clock > max_dotclk)
+	if (target_clock > max_dotclk)
 		return MODE_CLOCK_HIGH;
 
 	return intel_mode_valid_max_plane_size(display, mode, 1);
diff --git a/drivers/gpu/drm/i915/display/intel_dvo.c b/drivers/gpu/drm/i915/display/intel_dvo.c
index 405b33aca9dde..dd1a995c29799 100644
--- a/drivers/gpu/drm/i915/display/intel_dvo.c
+++ b/drivers/gpu/drm/i915/display/intel_dvo.c
@@ -223,8 +223,6 @@ intel_dvo_mode_valid(struct drm_connector *_connector,
 	struct intel_display *display = to_intel_display(_connector->dev);
 	struct intel_connector *connector = to_intel_connector(_connector);
 	struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
-	const struct drm_display_mode *fixed_mode =
-		intel_panel_fixed_mode(connector, mode);
 	int max_dotclk = display->cdclk.max_dotclk_freq;
 	int target_clock = mode->clock;
 	enum drm_mode_status status;
@@ -234,16 +232,9 @@ intel_dvo_mode_valid(struct drm_connector *_connector,
 		return status;
 
 	/* XXX: Validate clock range */
-
-	if (fixed_mode) {
-		enum drm_mode_status status;
-
-		status = intel_panel_mode_valid(connector, mode);
-		if (status != MODE_OK)
-			return status;
-
-		target_clock = fixed_mode->clock;
-	}
+	status = intel_panel_mode_valid(connector, mode, &target_clock);
+	if (status != MODE_OK)
+		return status;
 
 	if (target_clock > max_dotclk)
 		return MODE_CLOCK_HIGH;
@@ -255,11 +246,9 @@ static int intel_dvo_compute_config(struct intel_encoder *encoder,
 				    struct intel_crtc_state *pipe_config,
 				    struct drm_connector_state *conn_state)
 {
-	struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
 	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	struct drm_display_mode *adjusted_mode = &pipe_config->hw.adjusted_mode;
-	const struct drm_display_mode *fixed_mode =
-		intel_panel_fixed_mode(intel_dvo->attached_connector, adjusted_mode);
+	int ret;
 
 	/*
 	 * If we have timings from the BIOS for the panel, put them in
@@ -267,13 +256,9 @@ static int intel_dvo_compute_config(struct intel_encoder *encoder,
 	 * with the panel scaling set up to source from the H/VDisplay
 	 * of the original mode.
 	 */
-	if (fixed_mode) {
-		int ret;
-
-		ret = intel_panel_compute_config(connector, adjusted_mode);
-		if (ret)
-			return ret;
-	}
+	ret = intel_panel_compute_config(connector, adjusted_mode);
+	if (ret)
+		return ret;
 
 	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLSCAN)
 		return -EINVAL;
diff --git a/drivers/gpu/drm/i915/display/intel_dvo_dev.h b/drivers/gpu/drm/i915/display/intel_dvo_dev.h
index f1e939aaf7fac..1f049f79a5348 100644
--- a/drivers/gpu/drm/i915/display/intel_dvo_dev.h
+++ b/drivers/gpu/drm/i915/display/intel_dvo_dev.h
@@ -23,7 +23,7 @@
 #ifndef __INTEL_DVO_DEV_H__
 #define __INTEL_DVO_DEV_H__
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 
 #include "intel_display_limits.h"
 
diff --git a/drivers/gpu/drm/i915/display/intel_fb.c b/drivers/gpu/drm/i915/display/intel_fb.c
index 5768619f840f4..1c0859d5f8298 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.c
+++ b/drivers/gpu/drm/i915/display/intel_fb.c
@@ -521,6 +521,11 @@ bool intel_fb_needs_64k_phys(u64 modifier)
 				      INTEL_PLANE_CAP_NEED64K_PHYS);
 }
 
+bool intel_fb_needs_cpu_access(const struct drm_framebuffer *fb)
+{
+	return intel_fb_rc_ccs_cc_plane(fb) >= 0;
+}
+
 /**
  * intel_fb_is_tile4_modifier: Check if a modifier is a tile4 modifier type
  * @modifier: Modifier to check
@@ -876,15 +881,6 @@ static void intel_tile_block_dims(const struct drm_framebuffer *fb, int color_pl
 		*tile_height = 1;
 }
 
-unsigned int intel_tile_row_size(const struct drm_framebuffer *fb, int color_plane)
-{
-	unsigned int tile_width, tile_height;
-
-	intel_tile_dims(fb, color_plane, &tile_width, &tile_height);
-
-	return fb->pitches[color_plane] * tile_height;
-}
-
 unsigned int
 intel_fb_align_height(const struct drm_framebuffer *fb,
 		      int color_plane, unsigned int height)
@@ -1261,6 +1257,10 @@ static bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
 	if (intel_fb_is_ccs_modifier(fb->modifier))
 		return false;
 
+	/* TODO implement remapping with DPT */
+	if (intel_fb_uses_dpt(fb))
+		return false;
+
 	/* Linear needs a page aligned stride for remapping */
 	if (fb->modifier == DRM_FORMAT_MOD_LINEAR) {
 		unsigned int alignment = intel_tile_size(display) - 1;
@@ -1274,7 +1274,7 @@ static bool intel_plane_can_remap(const struct intel_plane_state *plane_state)
 	return true;
 }
 
-bool intel_fb_needs_pot_stride_remap(const struct intel_framebuffer *fb)
+static bool intel_fb_needs_pot_stride_remap(const struct intel_framebuffer *fb)
 {
 	struct intel_display *display = to_intel_display(fb->base.dev);
 
@@ -1287,9 +1287,9 @@ bool intel_plane_uses_fence(const struct intel_plane_state *plane_state)
 	struct intel_display *display = to_intel_display(plane_state);
 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
 
-	return DISPLAY_VER(display) < 4 ||
+	return intel_plane_needs_fence(display) ||
 		(plane->fbc && !plane_state->no_fbc_reason &&
-		 plane_state->view.gtt.type == I915_GTT_VIEW_NORMAL);
+		 i915_gtt_view_is_normal(&plane_state->view.gtt));
 }
 
 static int intel_fb_pitch(const struct intel_framebuffer *fb, int color_plane, unsigned int rotation)
@@ -1511,7 +1511,7 @@ static u32 calc_plane_remap_info(const struct intel_framebuffer *fb, int color_p
 			       plane_view_height_tiles(fb, color_plane, dims, y));
 	}
 
-	if (view->gtt.type == I915_GTT_VIEW_ROTATED) {
+	if (i915_gtt_view_is_rotated(&view->gtt)) {
 		drm_WARN_ON(display->drm, remap_info->linear);
 		check_array_bounds(display, view->gtt.rotated.plane, color_plane);
 
@@ -1536,7 +1536,7 @@ static u32 calc_plane_remap_info(const struct intel_framebuffer *fb, int color_p
 		/* rotate the tile dimensions to match the GTT view */
 		swap(tile_width, tile_height);
 	} else {
-		drm_WARN_ON(display->drm, view->gtt.type != I915_GTT_VIEW_REMAPPED);
+		drm_WARN_ON(display->drm, !i915_gtt_view_is_remapped(&view->gtt));
 
 		check_array_bounds(display, view->gtt.remapped.plane, color_plane);
 
@@ -1632,13 +1632,14 @@ calc_plane_normal_size(const struct intel_framebuffer *fb, int color_plane,
 
 static void intel_fb_view_init(struct intel_display *display,
 			       struct intel_fb_view *view,
-			       enum i915_gtt_view_type view_type)
+			       enum i915_gtt_view_type view_type,
+			       const struct intel_framebuffer *fb)
 {
 	memset(view, 0, sizeof(*view));
 	view->gtt.type = view_type;
 
-	if (view_type == I915_GTT_VIEW_REMAPPED &&
-	    (display->platform.alderlake_p || DISPLAY_VER(display) >= 14))
+	if (i915_gtt_view_is_remapped(&view->gtt) &&
+	    intel_fb_needs_pot_stride_remap(fb))
 		view->gtt.remapped.plane_alignment = SZ_2M / PAGE_SIZE;
 }
 
@@ -1704,16 +1705,19 @@ int intel_fill_fb_info(struct intel_display *display, struct intel_framebuffer *
 	int i, num_planes = fb->base.format->num_planes;
 	unsigned int tile_size = intel_tile_size(display);
 
-	intel_fb_view_init(display, &fb->normal_view, I915_GTT_VIEW_NORMAL);
+	intel_fb_view_init(display, &fb->normal_view,
+			   I915_GTT_VIEW_NORMAL, fb);
 
 	drm_WARN_ON(display->drm,
 		    intel_fb_supports_90_270_rotation(fb) &&
 		    intel_fb_needs_pot_stride_remap(fb));
 
 	if (intel_fb_supports_90_270_rotation(fb))
-		intel_fb_view_init(display, &fb->rotated_view, I915_GTT_VIEW_ROTATED);
+		intel_fb_view_init(display, &fb->rotated_view,
+				   I915_GTT_VIEW_ROTATED, fb);
 	if (intel_fb_needs_pot_stride_remap(fb))
-		intel_fb_view_init(display, &fb->remapped_view, I915_GTT_VIEW_REMAPPED);
+		intel_fb_view_init(display, &fb->remapped_view,
+				   I915_GTT_VIEW_REMAPPED, fb);
 
 	for (i = 0; i < num_planes; i++) {
 		struct fb_plane_view_dims view_dims;
@@ -1840,8 +1844,9 @@ static void intel_plane_remap_gtt(struct intel_plane_state *plane_state)
 	u32 gtt_offset = 0;
 
 	intel_fb_view_init(display, &plane_state->view,
-			   drm_rotation_90_or_270(rotation) ? I915_GTT_VIEW_ROTATED :
-							      I915_GTT_VIEW_REMAPPED);
+			   drm_rotation_90_or_270(rotation) ?
+			   I915_GTT_VIEW_ROTATED : I915_GTT_VIEW_REMAPPED,
+			   intel_fb);
 
 	src_x = plane_state->uapi.src.x1 >> 16;
 	src_y = plane_state->uapi.src.y1 >> 16;
diff --git a/drivers/gpu/drm/i915/display/intel_fb.h b/drivers/gpu/drm/i915/display/intel_fb.h
index 22514d5f2bb66..0a027e2595b2d 100644
--- a/drivers/gpu/drm/i915/display/intel_fb.h
+++ b/drivers/gpu/drm/i915/display/intel_fb.h
@@ -38,6 +38,7 @@ bool intel_fb_is_rc_ccs_cc_modifier(u64 modifier);
 bool intel_fb_is_mc_ccs_modifier(u64 modifier);
 bool intel_fb_needs_64k_phys(u64 modifier);
 bool intel_fb_is_tile4_modifier(u64 modifier);
+bool intel_fb_needs_cpu_access(const struct drm_framebuffer *fb);
 
 bool intel_fb_is_ccs_aux_plane(const struct drm_framebuffer *fb, int color_plane);
 int intel_fb_rc_ccs_cc_plane(const struct drm_framebuffer *fb);
@@ -62,7 +63,6 @@ int skl_main_to_aux_plane(const struct drm_framebuffer *fb, int main_plane);
 unsigned int intel_tile_size(struct intel_display *display);
 unsigned int intel_tile_width_bytes(const struct drm_framebuffer *fb, int color_plane);
 unsigned int intel_tile_height(const struct drm_framebuffer *fb, int color_plane);
-unsigned int intel_tile_row_size(const struct drm_framebuffer *fb, int color_plane);
 unsigned int intel_fb_align_height(const struct drm_framebuffer *fb,
 				   int color_plane, unsigned int height);
 
@@ -78,7 +78,6 @@ u32 intel_plane_compute_aligned_offset(int *x, int *y,
 				       const struct intel_plane_state *plane_state,
 				       int color_plane);
 
-bool intel_fb_needs_pot_stride_remap(const struct intel_framebuffer *fb);
 bool intel_plane_uses_fence(const struct intel_plane_state *plane_state);
 bool intel_fb_supports_90_270_rotation(const struct intel_framebuffer *fb);
 
diff --git a/drivers/gpu/drm/i915/display/intel_fb_pin.c b/drivers/gpu/drm/i915/display/intel_fb_pin.c
deleted file mode 100644
index 738d77a1468a5..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_fb_pin.c
+++ /dev/null
@@ -1,357 +0,0 @@
-// SPDX-License-Identifier: MIT
-/*
- * Copyright © 2021 Intel Corporation
- */
-
-/**
- * DOC: display pinning helpers
- */
-
-#include <drm/drm_print.h>
-
-#include "gem/i915_gem_domain.h"
-#include "gem/i915_gem_object.h"
-
-#include "i915_drv.h"
-#include "i915_vma.h"
-#include "intel_display_core.h"
-#include "intel_display_rpm.h"
-#include "intel_display_types.h"
-#include "i915_dpt.h"
-#include "intel_fb.h"
-#include "intel_fb_pin.h"
-#include "intel_plane.h"
-
-static struct i915_vma *
-intel_fb_pin_to_dpt(const struct drm_framebuffer *fb,
-		    const struct i915_gtt_view *view,
-		    unsigned int alignment,
-		    unsigned long *out_flags,
-		    struct intel_dpt *dpt)
-{
-	struct drm_device *dev = fb->dev;
-	struct intel_display *display = to_intel_display(dev);
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_gem_object *_obj = intel_fb_bo(fb);
-	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
-	struct i915_address_space *vm = i915_dpt_to_vm(dpt);
-	struct i915_gem_ww_ctx ww;
-	struct i915_vma *vma;
-	int ret;
-
-	/*
-	 * We are not syncing against the binding (and potential migrations)
-	 * below, so this vm must never be async.
-	 */
-	if (drm_WARN_ON(&dev_priv->drm, vm->bind_async_flags))
-		return ERR_PTR(-EINVAL);
-
-	if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
-		return ERR_PTR(-EINVAL);
-
-	atomic_inc(&display->restore.pending_fb_pin);
-
-	for_i915_gem_ww(&ww, ret, true) {
-		ret = i915_gem_object_lock(obj, &ww);
-		if (ret)
-			continue;
-
-		if (HAS_LMEM(dev_priv)) {
-			unsigned int flags = obj->flags;
-
-			/*
-			 * For this type of buffer we need to able to read from the CPU
-			 * the clear color value found in the buffer, hence we need to
-			 * ensure it is always in the mappable part of lmem, if this is
-			 * a small-bar device.
-			 */
-			if (intel_fb_rc_ccs_cc_plane(fb) >= 0)
-				flags &= ~I915_BO_ALLOC_GPU_ONLY;
-			ret = __i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM_0,
-							flags);
-			if (ret)
-				continue;
-		}
-
-		ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
-		if (ret)
-			continue;
-
-		vma = i915_vma_instance(obj, vm, view);
-		if (IS_ERR(vma)) {
-			ret = PTR_ERR(vma);
-			continue;
-		}
-
-		if (i915_vma_misplaced(vma, 0, alignment, 0)) {
-			ret = i915_vma_unbind(vma);
-			if (ret)
-				continue;
-		}
-
-		ret = i915_vma_pin_ww(vma, &ww, 0, alignment, PIN_GLOBAL);
-		if (ret)
-			continue;
-	}
-	if (ret) {
-		vma = ERR_PTR(ret);
-		goto err;
-	}
-
-	vma->display_alignment = max(vma->display_alignment, alignment);
-
-	i915_gem_object_flush_if_display(obj);
-
-	i915_vma_get(vma);
-err:
-	atomic_dec(&display->restore.pending_fb_pin);
-
-	return vma;
-}
-
-struct i915_vma *
-intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,
-		     const struct i915_gtt_view *view,
-		     unsigned int alignment,
-		     unsigned int phys_alignment,
-		     unsigned int vtd_guard,
-		     bool uses_fence,
-		     unsigned long *out_flags)
-{
-	struct drm_device *dev = fb->dev;
-	struct intel_display *display = to_intel_display(dev);
-	struct drm_i915_private *dev_priv = to_i915(dev);
-	struct drm_gem_object *_obj = intel_fb_bo(fb);
-	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
-	struct ref_tracker *wakeref;
-	struct i915_gem_ww_ctx ww;
-	struct i915_vma *vma;
-	unsigned int pinctl;
-	int ret;
-
-	if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj)))
-		return ERR_PTR(-EINVAL);
-
-	if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment)))
-		return ERR_PTR(-EINVAL);
-
-	/*
-	 * Global gtt pte registers are special registers which actually forward
-	 * writes to a chunk of system memory. Which means that there is no risk
-	 * that the register values disappear as soon as we call
-	 * intel_runtime_pm_put(), so it is correct to wrap only the
-	 * pin/unpin/fence and not more.
-	 */
-	wakeref = intel_display_rpm_get(display);
-
-	atomic_inc(&display->restore.pending_fb_pin);
-
-	/*
-	 * Valleyview is definitely limited to scanning out the first
-	 * 512MiB. Lets presume this behaviour was inherited from the
-	 * g4x display engine and that all earlier gen are similarly
-	 * limited. Testing suggests that it is a little more
-	 * complicated than this. For example, Cherryview appears quite
-	 * happy to scanout from anywhere within its global aperture.
-	 */
-	pinctl = 0;
-	if (HAS_GMCH(display))
-		pinctl |= PIN_MAPPABLE;
-
-	i915_gem_ww_ctx_init(&ww, true);
-retry:
-	ret = i915_gem_object_lock(obj, &ww);
-	if (!ret && phys_alignment)
-		ret = i915_gem_object_attach_phys(obj, phys_alignment);
-	else if (!ret && HAS_LMEM(dev_priv))
-		ret = i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM_0);
-	if (!ret)
-		ret = i915_gem_object_pin_pages(obj);
-	if (ret)
-		goto err;
-
-	vma = i915_gem_object_pin_to_display_plane(obj, &ww, alignment,
-						   vtd_guard, view, pinctl);
-	if (IS_ERR(vma)) {
-		ret = PTR_ERR(vma);
-		goto err_unpin;
-	}
-
-	if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
-		/*
-		 * Install a fence for tiled scan-out. Pre-i965 always needs a
-		 * fence, whereas 965+ only requires a fence if using
-		 * framebuffer compression.  For simplicity, we always, when
-		 * possible, install a fence as the cost is not that onerous.
-		 *
-		 * If we fail to fence the tiled scanout, then either the
-		 * modeset will reject the change (which is highly unlikely as
-		 * the affected systems, all but one, do not have unmappable
-		 * space) or we will not be able to enable full powersaving
-		 * techniques (also likely not to apply due to various limits
-		 * FBC and the like impose on the size of the buffer, which
-		 * presumably we violated anyway with this unmappable buffer).
-		 * Anyway, it is presumably better to stumble onwards with
-		 * something and try to run the system in a "less than optimal"
-		 * mode that matches the user configuration.
-		 */
-		ret = i915_vma_pin_fence(vma);
-		if (ret != 0 && DISPLAY_VER(display) < 4) {
-			i915_vma_unpin(vma);
-			goto err_unpin;
-		}
-		ret = 0;
-
-		if (vma->fence)
-			*out_flags |= PLANE_HAS_FENCE;
-	}
-
-	i915_vma_get(vma);
-
-err_unpin:
-	i915_gem_object_unpin_pages(obj);
-err:
-	if (ret == -EDEADLK) {
-		ret = i915_gem_ww_ctx_backoff(&ww);
-		if (!ret)
-			goto retry;
-	}
-	i915_gem_ww_ctx_fini(&ww);
-	if (ret)
-		vma = ERR_PTR(ret);
-
-	atomic_dec(&display->restore.pending_fb_pin);
-	intel_display_rpm_put(display, wakeref);
-	return vma;
-}
-
-void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags)
-{
-	if (flags & PLANE_HAS_FENCE)
-		i915_vma_unpin_fence(vma);
-	i915_vma_unpin(vma);
-	i915_vma_put(vma);
-}
-
-static unsigned int
-intel_plane_fb_min_alignment(const struct intel_plane_state *plane_state)
-{
-	const struct intel_framebuffer *fb = to_intel_framebuffer(plane_state->hw.fb);
-
-	return fb->min_alignment;
-}
-
-static unsigned int
-intel_plane_fb_min_phys_alignment(const struct intel_plane_state *plane_state)
-{
-	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
-	const struct drm_framebuffer *fb = plane_state->hw.fb;
-
-	if (!intel_plane_needs_physical(plane))
-		return 0;
-
-	return plane->min_alignment(plane, fb, 0);
-}
-
-static unsigned int
-intel_plane_fb_vtd_guard(const struct intel_plane_state *plane_state)
-{
-	return intel_fb_view_vtd_guard(plane_state->hw.fb,
-				       &plane_state->view,
-				       plane_state->hw.rotation);
-}
-
-int intel_plane_pin_fb(struct intel_plane_state *plane_state,
-		       const struct intel_plane_state *old_plane_state)
-{
-	struct intel_display *display = to_intel_display(plane_state);
-	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
-	const struct intel_framebuffer *fb =
-		to_intel_framebuffer(plane_state->hw.fb);
-	struct i915_vma *vma;
-
-	if (!intel_fb_uses_dpt(&fb->base)) {
-		vma = intel_fb_pin_to_ggtt(&fb->base, &plane_state->view.gtt,
-					   intel_plane_fb_min_alignment(plane_state),
-					   intel_plane_fb_min_phys_alignment(plane_state),
-					   intel_plane_fb_vtd_guard(plane_state),
-					   intel_plane_uses_fence(plane_state),
-					   &plane_state->flags);
-		if (IS_ERR(vma))
-			return PTR_ERR(vma);
-
-		plane_state->ggtt_vma = vma;
-
-	} else {
-		unsigned int alignment = intel_plane_fb_min_alignment(plane_state);
-
-		vma = i915_dpt_pin_to_ggtt(fb->dpt, alignment / 512);
-		if (IS_ERR(vma))
-			return PTR_ERR(vma);
-
-		plane_state->ggtt_vma = vma;
-
-		vma = intel_fb_pin_to_dpt(&fb->base, &plane_state->view.gtt,
-					  alignment, &plane_state->flags,
-					  fb->dpt);
-		if (IS_ERR(vma)) {
-			i915_dpt_unpin_from_ggtt(fb->dpt);
-			plane_state->ggtt_vma = NULL;
-			return PTR_ERR(vma);
-		}
-
-		plane_state->dpt_vma = vma;
-
-		WARN_ON(plane_state->ggtt_vma == plane_state->dpt_vma);
-
-		/*
-		 * The DPT object contains only one vma, and there is no VT-d
-		 * guard, so the VMA's offset within the DPT is always 0.
-		 */
-		drm_WARN_ON(display->drm, i915_dpt_offset(plane_state->dpt_vma));
-	}
-
-	/*
-	 * Pre-populate the dma address before we enter the vblank
-	 * evade critical section as i915_gem_object_get_dma_address()
-	 * will trigger might_sleep() even if it won't actually sleep,
-	 * which is the case when the fb has already been pinned.
-	 */
-	if (intel_plane_needs_physical(plane)) {
-		struct drm_i915_gem_object *obj = to_intel_bo(intel_fb_bo(&fb->base));
-
-		plane_state->surf = i915_gem_object_get_dma_address(obj, 0) +
-			plane->surf_offset(plane_state);
-	} else {
-		plane_state->surf = i915_ggtt_offset(plane_state->ggtt_vma) +
-			plane->surf_offset(plane_state);
-	}
-
-	return 0;
-}
-
-void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
-{
-	const struct intel_framebuffer *fb =
-		to_intel_framebuffer(old_plane_state->hw.fb);
-	struct i915_vma *vma;
-
-	if (!intel_fb_uses_dpt(&fb->base)) {
-		vma = fetch_and_zero(&old_plane_state->ggtt_vma);
-		if (vma)
-			intel_fb_unpin_vma(vma, old_plane_state->flags);
-	} else {
-		vma = fetch_and_zero(&old_plane_state->dpt_vma);
-		if (vma)
-			intel_fb_unpin_vma(vma, old_plane_state->flags);
-
-		vma = fetch_and_zero(&old_plane_state->ggtt_vma);
-		if (vma)
-			i915_dpt_unpin_from_ggtt(fb->dpt);
-	}
-}
-
-void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map)
-{
-	iosys_map_set_vaddr_iomem(map, i915_vma_get_iomap(vma));
-}
diff --git a/drivers/gpu/drm/i915/display/intel_fb_pin.h b/drivers/gpu/drm/i915/display/intel_fb_pin.h
deleted file mode 100644
index 81ab79da1af74..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_fb_pin.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2021 Intel Corporation
- */
-
-#ifndef __INTEL_FB_PIN_H__
-#define __INTEL_FB_PIN_H__
-
-#include <linux/types.h>
-
-struct drm_framebuffer;
-struct i915_vma;
-struct intel_plane_state;
-struct i915_gtt_view;
-struct iosys_map;
-
-struct i915_vma *
-intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,
-		     const struct i915_gtt_view *view,
-		     unsigned int alignment,
-		     unsigned int phys_alignment,
-		     unsigned int vtd_guard,
-		     bool uses_fence,
-		     unsigned long *out_flags);
-
-void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags);
-
-int intel_plane_pin_fb(struct intel_plane_state *new_plane_state,
-		       const struct intel_plane_state *old_plane_state);
-void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state);
-void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map);
-
-#endif
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c
index ea0ce00c84743..a7d02dd6ca968 100644
--- a/drivers/gpu/drm/i915/display/intel_fbc.c
+++ b/drivers/gpu/drm/i915/display/intel_fbc.c
@@ -44,6 +44,7 @@
 #include <drm/drm_blend.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_print.h>
+#include <drm/intel/step.h>
 
 #include "i9xx_plane_regs.h"
 #include "intel_de.h"
@@ -58,7 +59,6 @@
 #include "intel_fbc_regs.h"
 #include "intel_frontbuffer.h"
 #include "intel_parent.h"
-#include "intel_step.h"
 
 #define for_each_fbc_id(__display, __fbc_id) \
 	for ((__fbc_id) = INTEL_FBC_A; (__fbc_id) < I915_MAX_FBCS; (__fbc_id)++) \
@@ -1458,13 +1458,10 @@ static void intel_fbc_update_state(struct intel_atomic_state *state,
 
 	fbc_state->fence_y_offset = intel_plane_fence_y_offset(plane_state);
 
-	drm_WARN_ON(display->drm, plane_state->flags & PLANE_HAS_FENCE &&
+	drm_WARN_ON(display->drm, plane_state->fence_id >= 0 &&
 		    !intel_fbc_has_fences(display));
 
-	if (plane_state->flags & PLANE_HAS_FENCE)
-		fbc_state->fence_id = intel_parent_vma_fence_id(display, plane_state->ggtt_vma);
-	else
-		fbc_state->fence_id = -1;
+	fbc_state->fence_id = plane_state->fence_id;
 
 	fbc_state->cfb_stride = intel_fbc_cfb_stride(plane_state);
 	fbc_state->cfb_size = intel_fbc_cfb_size(plane_state);
@@ -1487,9 +1484,7 @@ static bool intel_fbc_is_fence_ok(const struct intel_plane_state *plane_state)
 	 * so have no fence associated with it) due to aperture constraints
 	 * at the time of pinning.
 	 */
-	return DISPLAY_VER(display) >= 9 ||
-		(plane_state->flags & PLANE_HAS_FENCE &&
-		 intel_parent_vma_fence_id(display, plane_state->ggtt_vma) != -1);
+	return DISPLAY_VER(display) >= 9 || plane_state->fence_id >= 0;
 }
 
 static bool intel_fbc_is_cfb_ok(const struct intel_plane_state *plane_state)
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c
index bdaaf3edba0cb..df1d3d9dc3e5b 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.c
@@ -38,6 +38,7 @@
 #include <linux/vga_switcheroo.h>
 
 #include <drm/clients/drm_client_setup.h>
+#include <drm/intel/display_parent_interface.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
@@ -52,15 +53,14 @@
 #include "intel_display_rpm.h"
 #include "intel_display_types.h"
 #include "intel_fb.h"
-#include "intel_fb_pin.h"
 #include "intel_fbdev.h"
-#include "intel_fbdev_fb.h"
 #include "intel_frontbuffer.h"
+#include "intel_parent.h"
+#include "intel_plane.h"
 
 struct intel_fbdev {
 	struct intel_framebuffer *fb;
 	struct i915_vma *vma;
-	unsigned long vma_flags;
 };
 
 static struct intel_fbdev *to_intel_fbdev(struct drm_fb_helper *fb_helper)
@@ -132,6 +132,7 @@ static int intel_fbdev_mmap(struct fb_info *info, struct vm_area_struct *vma)
 static void intel_fbdev_fb_destroy(struct fb_info *info)
 {
 	struct drm_fb_helper *fb_helper = info->par;
+	struct intel_display *display = to_intel_display(fb_helper->client.dev);
 	struct intel_fbdev *ifbdev = to_intel_fbdev(fb_helper);
 
 	drm_fb_helper_fini(fb_helper);
@@ -141,7 +142,7 @@ static void intel_fbdev_fb_destroy(struct fb_info *info)
 	 * the info->screen_base mmaping. Leaking the VMA is simpler than
 	 * trying to rectify all the possible error paths leading here.
 	 */
-	intel_fb_unpin_vma(ifbdev->vma, ifbdev->vma_flags);
+	intel_parent_fb_pin_ggtt_unpin(display, ifbdev->vma, -1);
 	drm_framebuffer_remove(fb_helper->fb);
 
 	drm_client_release(&fb_helper->client);
@@ -204,7 +205,8 @@ static const struct drm_fb_helper_funcs intel_fb_helper_funcs = {
 	.fb_set_suspend = intelfb_set_suspend,
 };
 
-static void intel_fbdev_fill_mode_cmd(struct drm_fb_helper_surface_size *sizes,
+static void intel_fbdev_fill_mode_cmd(struct intel_display *display,
+				      struct drm_fb_helper_surface_size *sizes,
 				      struct drm_mode_fb_cmd2 *mode_cmd)
 {
 	/* we don't do packed 24bpp */
@@ -215,7 +217,7 @@ static void intel_fbdev_fill_mode_cmd(struct drm_fb_helper_surface_size *sizes,
 	mode_cmd->width = sizes->surface_width;
 	mode_cmd->height = sizes->surface_height;
 
-	mode_cmd->pitches[0] = intel_fbdev_fb_pitch_align(mode_cmd->width * DIV_ROUND_UP(sizes->surface_bpp, 8));
+	mode_cmd->pitches[0] = intel_bo_fbdev_pitch_align(display, mode_cmd->width * DIV_ROUND_UP(sizes->surface_bpp, 8));
 	mode_cmd->pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
 							   sizes->surface_depth);
 	mode_cmd->modifier[0] = DRM_FORMAT_MOD_LINEAR;
@@ -230,12 +232,12 @@ __intel_fbdev_fb_alloc(struct intel_display *display,
 	struct drm_gem_object *obj;
 	int size;
 
-	intel_fbdev_fill_mode_cmd(sizes, &mode_cmd);
+	intel_fbdev_fill_mode_cmd(display, sizes, &mode_cmd);
 
 	size = mode_cmd.pitches[0] * mode_cmd.height;
 	size = PAGE_ALIGN(size);
 
-	obj = intel_fbdev_fb_bo_create(display->drm, size);
+	obj = intel_bo_fbdev_create(display, size);
 	if (IS_ERR(obj)) {
 		fb = ERR_CAST(obj);
 		goto err;
@@ -247,7 +249,7 @@ __intel_fbdev_fb_alloc(struct intel_display *display,
 							  mode_cmd.modifier[0]),
 				      &mode_cmd);
 	if (IS_ERR(fb)) {
-		intel_fbdev_fb_bo_destroy(obj);
+		intel_bo_fbdev_destroy(obj);
 		goto err;
 	}
 
@@ -265,13 +267,14 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper,
 {
 	struct intel_display *display = to_intel_display(helper->dev);
 	struct intel_fbdev *ifbdev = to_intel_fbdev(helper);
+	struct intel_fb_pin_params pin_params = {};
 	struct intel_framebuffer *fb = ifbdev->fb;
 	struct fb_info *info = helper->info;
 	struct ref_tracker *wakeref;
 	struct i915_vma *vma;
-	unsigned long flags = 0;
 	bool prealloc = false;
 	struct drm_gem_object *obj;
+	u32 offset;
 	int ret;
 
 	ifbdev->fb = NULL;
@@ -306,28 +309,30 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper,
 		sizes->fb_height = fb->base.height;
 	}
 
+	obj = intel_fb_bo(&fb->base);
+
 	/* Pin the GGTT vma for our access via info->screen_base.
 	 * This also validates that any existing fb inherited from the
 	 * BIOS is suitable for own access.
 	 */
-	vma = intel_fb_pin_to_ggtt(&fb->base, &fb->normal_view.gtt,
-				   fb->min_alignment, 0,
-				   intel_fb_view_vtd_guard(&fb->base, &fb->normal_view,
-							   DRM_MODE_ROTATE_0),
-				   false, &flags);
-	if (IS_ERR(vma)) {
-		ret = PTR_ERR(vma);
+	pin_params.view = &fb->normal_view.gtt;
+	pin_params.alignment = fb->min_alignment;
+	pin_params.vtd_guard = intel_fb_view_vtd_guard(&fb->base,
+						       &fb->normal_view,
+						       DRM_MODE_ROTATE_0);
+	pin_params.needs_low_address = intel_plane_needs_low_address(display);
+
+	ret = intel_parent_fb_pin_ggtt_pin(display, obj, &pin_params,
+					   &vma, &offset, NULL);
+	if (ret)
 		goto out_unlock;
-	}
 
 	helper->funcs = &intel_fb_helper_funcs;
 	helper->fb = &fb->base;
 
 	info->fbops = &intelfb_ops;
 
-	obj = intel_fb_bo(&fb->base);
-
-	ret = intel_fbdev_fb_fill_info(display->drm, info, obj, vma);
+	ret = intel_bo_fbdev_fill_info(obj, info, vma);
 	if (ret)
 		goto out_unpin;
 
@@ -345,14 +350,13 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper,
 	drm_dbg_kms(display->drm, "allocated %dx%d fb\n", fb->base.width, fb->base.height);
 	ifbdev->fb = fb;
 	ifbdev->vma = vma;
-	ifbdev->vma_flags = flags;
 
 	intel_display_rpm_put(display, wakeref);
 
 	return 0;
 
 out_unpin:
-	intel_fb_unpin_vma(vma, flags);
+	intel_parent_fb_pin_ggtt_unpin(display, vma, -1);
 out_unlock:
 	intel_display_rpm_put(display, wakeref);
 
@@ -556,12 +560,9 @@ struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev)
 	return fbdev->fb;
 }
 
-struct i915_vma *intel_fbdev_vma_pointer(struct intel_fbdev *fbdev)
+void intel_fbdev_get_map(struct intel_display *display, struct iosys_map *map)
 {
-	return fbdev ? fbdev->vma : NULL;
-}
+	struct intel_fbdev *fbdev = display->fbdev.fbdev;
 
-void intel_fbdev_get_map(struct intel_fbdev *fbdev, struct iosys_map *map)
-{
-	intel_fb_get_map(fbdev->vma, map);
+	intel_parent_fb_pin_get_map(display, fbdev->vma, map);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.h b/drivers/gpu/drm/i915/display/intel_fbdev.h
index 150cc5f45bb31..0ae36d321c773 100644
--- a/drivers/gpu/drm/i915/display/intel_fbdev.h
+++ b/drivers/gpu/drm/i915/display/intel_fbdev.h
@@ -22,8 +22,7 @@ int intel_fbdev_driver_fbdev_probe(struct drm_fb_helper *helper,
 	.fbdev_probe = intel_fbdev_driver_fbdev_probe
 void intel_fbdev_setup(struct intel_display *display);
 struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbdev *fbdev);
-struct i915_vma *intel_fbdev_vma_pointer(struct intel_fbdev *fbdev);
-void intel_fbdev_get_map(struct intel_fbdev *fbdev, struct iosys_map *map);
+void intel_fbdev_get_map(struct intel_display *display, struct iosys_map *map);
 #else
 #define INTEL_FBDEV_DRIVER_OPS \
 	.fbdev_probe = NULL
@@ -34,13 +33,7 @@ static inline struct intel_framebuffer *intel_fbdev_framebuffer(struct intel_fbd
 {
 	return NULL;
 }
-
-static inline struct i915_vma *intel_fbdev_vma_pointer(struct intel_fbdev *fbdev)
-{
-	return NULL;
-}
-
-static inline void intel_fbdev_get_map(struct intel_fbdev *fbdev, struct iosys_map *map)
+static inline void intel_fbdev_get_map(struct intel_display *display, struct iosys_map *map)
 {
 }
 #endif
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev_fb.c b/drivers/gpu/drm/i915/display/intel_fbdev_fb.c
deleted file mode 100644
index 6f84eb6355ded..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_fbdev_fb.c
+++ /dev/null
@@ -1,118 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include <linux/fb.h>
-
-#include <drm/drm_print.h>
-
-#include "gem/i915_gem_lmem.h"
-
-#include "i915_drv.h"
-#include "intel_fbdev_fb.h"
-
-u32 intel_fbdev_fb_pitch_align(u32 stride)
-{
-	return ALIGN(stride, 64);
-}
-
-bool intel_fbdev_fb_prefer_stolen(struct drm_device *drm, unsigned int size)
-{
-	struct drm_i915_private *i915 = to_i915(drm);
-
-	/* Skip stolen on MTL as Wa_22018444074 mitigation. */
-	if (IS_METEORLAKE(i915))
-		return false;
-
-	/*
-	 * If the FB is too big, just don't use it since fbdev is not very
-	 * important and we should probably use that space with FBC or other
-	 * features.
-	 */
-	return i915->dsm.usable_size >= size * 2;
-}
-
-struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size)
-{
-	struct drm_i915_private *i915 = to_i915(drm);
-	struct drm_i915_gem_object *obj;
-
-	obj = ERR_PTR(-ENODEV);
-	if (HAS_LMEM(i915)) {
-		obj = i915_gem_object_create_lmem(i915, size,
-						  I915_BO_ALLOC_CONTIGUOUS |
-						  I915_BO_ALLOC_USER);
-	} else {
-		if (intel_fbdev_fb_prefer_stolen(drm, size))
-			obj = i915_gem_object_create_stolen(i915, size);
-		else
-			drm_info(drm, "Allocating fbdev: Stolen memory not preferred.\n");
-
-		if (IS_ERR(obj))
-			obj = i915_gem_object_create_shmem(i915, size);
-	}
-
-	if (IS_ERR(obj)) {
-		drm_err(drm, "failed to allocate framebuffer (%pe)\n", obj);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	return &obj->base;
-}
-
-void intel_fbdev_fb_bo_destroy(struct drm_gem_object *obj)
-{
-	drm_gem_object_put(obj);
-}
-
-int intel_fbdev_fb_fill_info(struct drm_device *drm, struct fb_info *info,
-			     struct drm_gem_object *_obj, struct i915_vma *vma)
-{
-	struct drm_i915_private *i915 = to_i915(drm);
-	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
-	struct i915_gem_ww_ctx ww;
-	void __iomem *vaddr;
-	int ret;
-
-	if (i915_gem_object_is_lmem(obj)) {
-		struct intel_memory_region *mem = obj->mm.region;
-
-		/* Use fbdev's framebuffer from lmem for discrete */
-		info->fix.smem_start =
-			(unsigned long)(mem->io.start +
-					i915_gem_object_get_dma_address(obj, 0) -
-					mem->region.start);
-		info->fix.smem_len = obj->base.size;
-	} else {
-		struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
-
-		/* Our framebuffer is the entirety of fbdev's system memory */
-		info->fix.smem_start =
-			(unsigned long)(ggtt->gmadr.start + i915_ggtt_offset(vma));
-		info->fix.smem_len = vma->size;
-	}
-
-	for_i915_gem_ww(&ww, ret, false) {
-		ret = i915_gem_object_lock(vma->obj, &ww);
-
-		if (ret)
-			continue;
-
-		vaddr = i915_vma_pin_iomap(vma);
-		if (IS_ERR(vaddr)) {
-			drm_err(drm,
-				"Failed to remap framebuffer into virtual memory (%pe)\n", vaddr);
-			ret = PTR_ERR(vaddr);
-			continue;
-		}
-	}
-
-	if (ret)
-		return ret;
-
-	info->screen_base = vaddr;
-	info->screen_size = intel_bo_to_drm_bo(obj)->size;
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/i915/display/intel_fbdev_fb.h b/drivers/gpu/drm/i915/display/intel_fbdev_fb.h
deleted file mode 100644
index 34ed2b9c2b4fc..0000000000000
--- a/drivers/gpu/drm/i915/display/intel_fbdev_fb.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __INTEL_FBDEV_FB_H__
-#define __INTEL_FBDEV_FB_H__
-
-#include <linux/types.h>
-
-struct drm_device;
-struct drm_gem_object;
-struct drm_mode_fb_cmd2;
-struct fb_info;
-struct i915_vma;
-
-u32 intel_fbdev_fb_pitch_align(u32 stride);
-struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size);
-void intel_fbdev_fb_bo_destroy(struct drm_gem_object *obj);
-int intel_fbdev_fb_fill_info(struct drm_device *drm, struct fb_info *info,
-			     struct drm_gem_object *obj, struct i915_vma *vma);
-bool intel_fbdev_fb_prefer_stolen(struct drm_device *drm, unsigned int size);
-
-#endif
diff --git a/drivers/gpu/drm/i915/display/intel_fdi.c b/drivers/gpu/drm/i915/display/intel_fdi.c
index 24ce8a7842c7d..f8876aa23c9ff 100644
--- a/drivers/gpu/drm/i915/display/intel_fdi.c
+++ b/drivers/gpu/drm/i915/display/intel_fdi.c
@@ -11,6 +11,7 @@
 #include "intel_atomic.h"
 #include "intel_crtc.h"
 #include "intel_ddi.h"
+#include "intel_ddi_buf_trans.h"
 #include "intel_de.h"
 #include "intel_display_regs.h"
 #include "intel_display_types.h"
@@ -122,7 +123,7 @@ void intel_fdi_link_train(struct intel_crtc *crtc,
 {
 	struct intel_display *display = to_intel_display(crtc);
 
-	display->funcs.fdi->fdi_link_train(crtc, crtc_state);
+	display->fdi.funcs->fdi_link_train(crtc, crtc_state);
 }
 
 /**
@@ -185,7 +186,7 @@ static int ilk_check_fdi_lanes(struct intel_display *display, enum pipe pipe,
 			       struct intel_crtc_state *pipe_config,
 			       enum pipe *pipe_to_reduce)
 {
-	struct drm_atomic_state *state = pipe_config->uapi.state;
+	struct drm_atomic_commit *state = pipe_config->uapi.state;
 	struct intel_crtc *other_crtc;
 	struct intel_crtc_state *other_crtc_state;
 
@@ -438,7 +439,7 @@ void intel_fdi_normal_train(struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(crtc);
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 temp;
 
 	/* enable normal train */
@@ -479,7 +480,7 @@ static void ilk_fdi_link_train(struct intel_crtc *crtc,
 {
 	struct intel_display *display = to_intel_display(crtc);
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 temp, tries;
 
 	/*
@@ -580,7 +581,7 @@ static void gen6_fdi_link_train(struct intel_crtc *crtc,
 {
 	struct intel_display *display = to_intel_display(crtc);
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 temp, i, retry;
 
 	/*
@@ -715,7 +716,7 @@ static void ivb_manual_fdi_link_train(struct intel_crtc *crtc,
 {
 	struct intel_display *display = to_intel_display(crtc);
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 temp, i, j;
 
 	ivb_update_fdi_bc_bifurcation(crtc_state);
@@ -852,7 +853,7 @@ void hsw_fdi_link_train(struct intel_encoder *encoder,
 	u32 temp, i, rx_ctl_val;
 	int n_entries;
 
-	encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 
 	hsw_prepare_dp_ddi_buffers(encoder, crtc_state);
 
@@ -996,7 +997,7 @@ void ilk_fdi_pll_enable(const struct intel_crtc_state *crtc_state)
 	struct intel_display *display = to_intel_display(crtc_state);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 temp;
 
 	/* enable PCH FDI RX PLL, wait warmup plus DMI latency */
@@ -1049,7 +1050,7 @@ void ilk_fdi_disable(struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(crtc);
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 temp;
 
 	/* disable CPU FDI tx and PCH FDI rx */
@@ -1108,11 +1109,11 @@ void
 intel_fdi_init_hook(struct intel_display *display)
 {
 	if (display->platform.ironlake) {
-		display->funcs.fdi = &ilk_funcs;
+		display->fdi.funcs = &ilk_funcs;
 	} else if (display->platform.sandybridge) {
-		display->funcs.fdi = &gen6_funcs;
+		display->fdi.funcs = &gen6_funcs;
 	} else if (display->platform.ivybridge) {
 		/* FIXME: detect B0+ stepping and use auto training */
-		display->funcs.fdi = &ivb_funcs;
+		display->fdi.funcs = &ivb_funcs;
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
index bf047180def9f..02013e89d8f2e 100644
--- a/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/display/intel_fifo_underrun.c
@@ -189,7 +189,7 @@ static bool cpt_can_enable_serr_int(struct intel_display *display)
 static void i9xx_check_fifo_underruns(struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(crtc);
-	i915_reg_t reg = PIPESTAT(display, crtc->pipe);
+	intel_reg_t reg = PIPESTAT(display, crtc->pipe);
 	u32 enable_mask;
 
 	lockdep_assert_held(&display->irq.lock);
@@ -209,7 +209,7 @@ static void i9xx_set_fifo_underrun_reporting(struct intel_display *display,
 					     enum pipe pipe,
 					     bool enable, bool old)
 {
-	i915_reg_t reg = PIPESTAT(display, pipe);
+	intel_reg_t reg = PIPESTAT(display, pipe);
 
 	lockdep_assert_held(&display->irq.lock);
 
diff --git a/drivers/gpu/drm/i915/display/intel_flipq.c b/drivers/gpu/drm/i915/display/intel_flipq.c
index 253dc2e96d2d6..333d28faf4caa 100644
--- a/drivers/gpu/drm/i915/display/intel_flipq.c
+++ b/drivers/gpu/drm/i915/display/intel_flipq.c
@@ -6,6 +6,7 @@
 #include <linux/pci.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/step.h>
 
 #include "intel_crtc.h"
 #include "intel_de.h"
@@ -17,7 +18,6 @@
 #include "intel_dmc_regs.h"
 #include "intel_dsb.h"
 #include "intel_flipq.h"
-#include "intel_step.h"
 #include "intel_vblank.h"
 #include "intel_vrr.h"
 
diff --git a/drivers/gpu/drm/i915/display/intel_gmbus.c b/drivers/gpu/drm/i915/display/intel_gmbus.c
index ea5cf8f51b316..049157c41fe25 100644
--- a/drivers/gpu/drm/i915/display/intel_gmbus.c
+++ b/drivers/gpu/drm/i915/display/intel_gmbus.c
@@ -48,7 +48,7 @@ struct intel_gmbus {
 #define GMBUS_FORCE_BIT_RETRY (1U << 31)
 	u32 force_bit;
 	u32 reg0;
-	i915_reg_t gpio_reg;
+	intel_reg_t gpio_reg;
 	struct i2c_algo_bit_data bit_algo;
 	struct intel_display *display;
 };
@@ -368,7 +368,7 @@ intel_gpio_post_xfer(struct i2c_adapter *adapter)
 }
 
 static void
-intel_gpio_setup(struct intel_gmbus *bus, i915_reg_t gpio_reg)
+intel_gpio_setup(struct intel_gmbus *bus, intel_reg_t gpio_reg)
 {
 	struct i2c_algo_bit_data *algo;
 
diff --git a/drivers/gpu/drm/i915/display/intel_hdcp.c b/drivers/gpu/drm/i915/display/intel_hdcp.c
index 892eab4b6f925..e88fec24af49f 100644
--- a/drivers/gpu/drm/i915/display/intel_hdcp.c
+++ b/drivers/gpu/drm/i915/display/intel_hdcp.c
@@ -18,6 +18,7 @@
 #include <drm/drm_print.h>
 #include <drm/intel/i915_component.h>
 #include <drm/intel/intel_pcode_regs.h>
+#include <drm/intel/step.h>
 
 #include "intel_connector.h"
 #include "intel_de.h"
@@ -33,7 +34,6 @@
 #include "intel_hdcp_regs.h"
 #include "intel_hdcp_shim.h"
 #include "intel_parent.h"
-#include "intel_step.h"
 
 #define USE_HDCP_GSC(__display)		(DISPLAY_VER(__display) >= 14)
 
@@ -46,7 +46,7 @@ intel_hdcp_adjust_hdcp_line_rekeying(struct intel_encoder *encoder,
 				     bool enable)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	i915_reg_t rekey_reg;
+	intel_reg_t rekey_reg;
 	u32 rekey_bit = 0;
 
 	/* Here we assume HDMI is in TMDS mode of operation */
@@ -72,6 +72,7 @@ intel_hdcp_adjust_hdcp_line_rekeying(struct intel_encoder *encoder,
 static int intel_conn_to_vcpi(struct intel_atomic_state *state,
 			      struct intel_connector *connector)
 {
+	struct intel_display *display = to_intel_display(state);
 	struct drm_dp_mst_topology_mgr *mgr;
 	struct drm_dp_mst_atomic_payload *payload;
 	struct drm_dp_mst_topology_state *mst_state;
@@ -79,13 +80,19 @@ static int intel_conn_to_vcpi(struct intel_atomic_state *state,
 	/* For HDMI this is forced to be 0x0. For DP SST also this is 0x0. */
 	if (!connector->mst.port)
 		return 0;
+
 	mgr = connector->mst.port->mgr;
+	mst_state = drm_atomic_get_new_mst_topology_state(&state->base, mgr);
+	if (!mst_state) {
+		drm_dbg_kms(display->drm, "MST topology still not created\n");
+		return 0;
+	}
 
-	drm_modeset_lock(&mgr->base.lock, state->base.acquire_ctx);
-	mst_state = to_drm_dp_mst_topology_state(mgr->base.state);
 	payload = drm_atomic_get_mst_payload_state(mst_state, connector->mst.port);
-	if (drm_WARN_ON(mgr->dev, !payload))
+	if (!payload) {
+		drm_dbg_kms(display->drm, "MST Payload not present\n");
 		return 0;
+	}
 
 	return payload->vcpi;
 }
@@ -106,6 +113,7 @@ intel_hdcp_required_content_stream(struct intel_atomic_state *state,
 {
 	struct intel_display *display = to_intel_display(state);
 	struct drm_connector_list_iter conn_iter;
+	struct drm_connector_state *new_conn_state;
 	struct intel_digital_port *conn_dig_port;
 	struct intel_connector *connector;
 	struct hdcp_port_data *data = &dig_port->hdcp.port_data;
@@ -132,6 +140,11 @@ intel_hdcp_required_content_stream(struct intel_atomic_state *state,
 		if (conn_dig_port != dig_port)
 			continue;
 
+		new_conn_state = drm_atomic_get_new_connector_state(&state->base,
+								    &connector->base);
+		if (!new_conn_state || !new_conn_state->crtc)
+			continue;
+
 		data->streams[data->k].stream_id =
 			intel_conn_to_vcpi(state, connector);
 		data->k++;
diff --git a/drivers/gpu/drm/i915/display/intel_hdmi.c b/drivers/gpu/drm/i915/display/intel_hdmi.c
index 05e898d10a2be..9076c2b176ec7 100644
--- a/drivers/gpu/drm/i915/display/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/display/intel_hdmi.c
@@ -173,7 +173,7 @@ static u32 hsw_infoframe_enable(unsigned int type)
 	}
 }
 
-static i915_reg_t
+static intel_reg_t
 hsw_dip_data_reg(struct intel_display *display,
 		 enum transcoder cpu_transcoder,
 		 unsigned int type,
@@ -298,7 +298,7 @@ static void ibx_write_infoframe(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	const u32 *data = frame;
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	i915_reg_t reg = TVIDEO_DIP_CTL(crtc->pipe);
+	intel_reg_t reg = TVIDEO_DIP_CTL(crtc->pipe);
 	u32 val = intel_de_read(display, reg);
 	int i;
 
@@ -351,7 +351,7 @@ static u32 ibx_infoframes_enabled(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	enum pipe pipe = to_intel_crtc(pipe_config->uapi.crtc)->pipe;
-	i915_reg_t reg = TVIDEO_DIP_CTL(pipe);
+	intel_reg_t reg = TVIDEO_DIP_CTL(pipe);
 	u32 val = intel_de_read(display, reg);
 
 	if ((val & VIDEO_DIP_ENABLE) == 0)
@@ -373,7 +373,7 @@ static void cpt_write_infoframe(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	const u32 *data = frame;
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	i915_reg_t reg = TVIDEO_DIP_CTL(crtc->pipe);
+	intel_reg_t reg = TVIDEO_DIP_CTL(crtc->pipe);
 	u32 val = intel_de_read(display, reg);
 	int i;
 
@@ -447,7 +447,7 @@ static void vlv_write_infoframe(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	const u32 *data = frame;
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	i915_reg_t reg = VLV_TVIDEO_DIP_CTL(crtc->pipe);
+	intel_reg_t reg = VLV_TVIDEO_DIP_CTL(crtc->pipe);
 	u32 val = intel_de_read(display, reg);
 	int i;
 
@@ -523,7 +523,7 @@ void hsw_write_infoframe(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	const u32 *data = frame;
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
-	i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(display, cpu_transcoder);
+	intel_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(display, cpu_transcoder);
 	int data_size;
 	int i;
 	u32 val = intel_de_read(display, ctl_reg);
@@ -884,7 +884,7 @@ static void g4x_set_infoframes(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct intel_hdmi *intel_hdmi = &dig_port->hdmi;
-	i915_reg_t reg = VIDEO_DIP_CTL;
+	intel_reg_t reg = VIDEO_DIP_CTL;
 	u32 val = intel_de_read(display, reg);
 	u32 port = VIDEO_DIP_PORT(encoder->port);
 
@@ -995,7 +995,7 @@ static bool intel_hdmi_set_gcp_infoframe(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	i915_reg_t reg;
+	intel_reg_t reg;
 
 	if ((crtc_state->infoframes.enable &
 	     intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL)) == 0)
@@ -1020,7 +1020,7 @@ void intel_hdmi_read_gcp_infoframe(struct intel_encoder *encoder,
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	i915_reg_t reg;
+	intel_reg_t reg;
 
 	if ((crtc_state->infoframes.enable &
 	     intel_hdmi_infoframe_enable(HDMI_PACKET_TYPE_GENERAL_CONTROL)) == 0)
@@ -1069,7 +1069,7 @@ static void ibx_set_infoframes(struct intel_encoder *encoder,
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct intel_digital_port *dig_port = enc_to_dig_port(encoder);
 	struct intel_hdmi *intel_hdmi = &dig_port->hdmi;
-	i915_reg_t reg = TVIDEO_DIP_CTL(crtc->pipe);
+	intel_reg_t reg = TVIDEO_DIP_CTL(crtc->pipe);
 	u32 val = intel_de_read(display, reg);
 	u32 port = VIDEO_DIP_PORT(encoder->port);
 
@@ -1127,7 +1127,7 @@ static void cpt_set_infoframes(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-	i915_reg_t reg = TVIDEO_DIP_CTL(crtc->pipe);
+	intel_reg_t reg = TVIDEO_DIP_CTL(crtc->pipe);
 	u32 val = intel_de_read(display, reg);
 
 	assert_hdmi_port_disabled(intel_hdmi);
@@ -1176,7 +1176,7 @@ static void vlv_set_infoframes(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-	i915_reg_t reg = VLV_TVIDEO_DIP_CTL(crtc->pipe);
+	intel_reg_t reg = VLV_TVIDEO_DIP_CTL(crtc->pipe);
 	u32 val = intel_de_read(display, reg);
 	u32 port = VIDEO_DIP_PORT(encoder->port);
 
@@ -1231,7 +1231,7 @@ void intel_hdmi_fastset_infoframes(struct intel_encoder *encoder,
 				   const struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	i915_reg_t reg = HSW_TVIDEO_DIP_CTL(display,
+	intel_reg_t reg = HSW_TVIDEO_DIP_CTL(display,
 					    crtc_state->cpu_transcoder);
 	u32 val = intel_de_read(display, reg);
 
@@ -1256,7 +1256,7 @@ static void hsw_set_infoframes(struct intel_encoder *encoder,
 			       const struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(encoder);
-	i915_reg_t reg = HSW_TVIDEO_DIP_CTL(display,
+	intel_reg_t reg = HSW_TVIDEO_DIP_CTL(display,
 					    crtc_state->cpu_transcoder);
 	u32 val = intel_de_read(display, reg);
 
@@ -2021,6 +2021,51 @@ intel_hdmi_mode_clock_valid(struct drm_connector *_connector, int clock,
 	return status;
 }
 
+static enum drm_mode_status
+intel_hdmi_sink_format_valid(struct intel_connector *connector,
+			     const struct drm_display_mode *mode,
+			     bool has_hdmi_sink,
+			     enum intel_output_format sink_format)
+{
+	const struct drm_display_info *info = &connector->base.display_info;
+
+	switch (sink_format) {
+	case INTEL_OUTPUT_FORMAT_YCBCR420:
+		if (!has_hdmi_sink ||
+		    !connector->base.ycbcr_420_allowed ||
+		    !drm_mode_is_420(info, mode))
+			return MODE_NO_420;
+
+		return MODE_OK;
+	case INTEL_OUTPUT_FORMAT_RGB:
+		return MODE_OK;
+	default:
+		MISSING_CASE(sink_format);
+		return MODE_BAD;
+	}
+}
+
+static enum drm_mode_status
+intel_hdmi_mode_valid_format(struct intel_connector *connector,
+			     const struct drm_display_mode *mode,
+			     int clock, bool has_hdmi_sink,
+			     enum intel_output_format sink_format)
+{
+	struct intel_display *display = to_intel_display(connector);
+	enum drm_mode_status status;
+
+	status = intel_hdmi_sink_format_valid(connector, mode,
+					      has_hdmi_sink, sink_format);
+	if (status != MODE_OK)
+		return status;
+
+	status = intel_pfit_mode_valid(display, mode, sink_format, 0);
+	if (status != MODE_OK)
+		return status;
+
+	return intel_hdmi_mode_clock_valid(&connector->base, clock, has_hdmi_sink, sink_format);
+}
+
 static enum drm_mode_status
 intel_hdmi_mode_valid(struct drm_connector *_connector,
 		      const struct drm_display_mode *mode)
@@ -2028,12 +2073,11 @@ intel_hdmi_mode_valid(struct drm_connector *_connector,
 	struct intel_connector *connector = to_intel_connector(_connector);
 	struct intel_display *display = to_intel_display(connector);
 	struct intel_hdmi *hdmi = intel_attached_hdmi(connector);
+	const struct drm_display_info *info = &connector->base.display_info;
 	enum drm_mode_status status;
 	int clock = mode->clock;
 	int max_dotclk = display->cdclk.max_dotclk_freq;
 	bool has_hdmi_sink = intel_has_hdmi_sink(hdmi, connector->base.state);
-	bool ycbcr_420_only;
-	enum intel_output_format sink_format;
 
 	status = intel_cpu_transcoder_mode_valid(display, mode);
 	if (status != MODE_OK)
@@ -2060,31 +2104,20 @@ intel_hdmi_mode_valid(struct drm_connector *_connector,
 	if (clock > 600000)
 		return MODE_CLOCK_HIGH;
 
-	ycbcr_420_only = drm_mode_is_420_only(&connector->base.display_info, mode);
-
-	if (ycbcr_420_only)
-		sink_format = INTEL_OUTPUT_FORMAT_YCBCR420;
-	else
-		sink_format = INTEL_OUTPUT_FORMAT_RGB;
+	if (drm_mode_is_420_only(info, mode)) {
+		status = intel_hdmi_mode_valid_format(connector, mode, clock, has_hdmi_sink,
+						      INTEL_OUTPUT_FORMAT_YCBCR420);
+	} else {
+		status = intel_hdmi_mode_valid_format(connector, mode, clock, has_hdmi_sink,
+						      INTEL_OUTPUT_FORMAT_RGB);
 
-	status = intel_pfit_mode_valid(display, mode, sink_format, 0);
+		if (status != MODE_OK && drm_mode_is_420_also(info, mode))
+			status = intel_hdmi_mode_valid_format(connector, mode, clock, has_hdmi_sink,
+							      INTEL_OUTPUT_FORMAT_YCBCR420);
+	}
 	if (status != MODE_OK)
 		return status;
 
-	status = intel_hdmi_mode_clock_valid(&connector->base, clock, has_hdmi_sink, sink_format);
-	if (status != MODE_OK) {
-		if (ycbcr_420_only ||
-		    !connector->base.ycbcr_420_allowed ||
-		    !drm_mode_is_420_also(&connector->base.display_info, mode))
-			return status;
-
-		sink_format = INTEL_OUTPUT_FORMAT_YCBCR420;
-		status = intel_hdmi_mode_clock_valid(&connector->base, clock, has_hdmi_sink,
-						     sink_format);
-		if (status != MODE_OK)
-			return status;
-	}
-
 	return intel_mode_valid_max_plane_size(display, mode, 1);
 }
 
@@ -2241,20 +2274,6 @@ static bool intel_hdmi_has_audio(struct intel_encoder *encoder,
 		return intel_conn_state->force_audio == HDMI_AUDIO_ON;
 }
 
-static enum intel_output_format
-intel_hdmi_sink_format(const struct intel_crtc_state *crtc_state,
-		       struct intel_connector *connector,
-		       bool ycbcr_420_output)
-{
-	if (!crtc_state->has_hdmi_sink)
-		return INTEL_OUTPUT_FORMAT_RGB;
-
-	if (connector->base.ycbcr_420_allowed && ycbcr_420_output)
-		return INTEL_OUTPUT_FORMAT_YCBCR420;
-	else
-		return INTEL_OUTPUT_FORMAT_RGB;
-}
-
 static enum intel_output_format
 intel_hdmi_output_format(const struct intel_crtc_state *crtc_state)
 {
@@ -2263,37 +2282,55 @@ intel_hdmi_output_format(const struct intel_crtc_state *crtc_state)
 
 static int intel_hdmi_compute_output_format(struct intel_encoder *encoder,
 					    struct intel_crtc_state *crtc_state,
-					    const struct drm_connector_state *conn_state,
-					    bool respect_downstream_limits)
+					    struct intel_connector *connector,
+					    bool respect_downstream_limits,
+					    enum intel_output_format sink_format)
+{
+	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
+
+	if (intel_hdmi_sink_format_valid(connector, adjusted_mode,
+					 crtc_state->has_hdmi_sink, sink_format) != MODE_OK)
+		return -EINVAL;
+
+	crtc_state->sink_format = sink_format;
+	crtc_state->output_format = intel_hdmi_output_format(crtc_state);
+
+	return intel_hdmi_compute_clock(encoder, crtc_state, respect_downstream_limits);
+}
+
+static int intel_hdmi_compute_formats(struct intel_encoder *encoder,
+				      struct intel_crtc_state *crtc_state,
+				      const struct drm_connector_state *conn_state,
+				      bool respect_downstream_limits)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	struct intel_connector *connector = to_intel_connector(conn_state->connector);
 	const struct drm_display_mode *adjusted_mode = &crtc_state->hw.adjusted_mode;
 	const struct drm_display_info *info = &connector->base.display_info;
-	bool ycbcr_420_only = drm_mode_is_420_only(info, adjusted_mode);
 	int ret;
 
-	crtc_state->sink_format =
-		intel_hdmi_sink_format(crtc_state, connector, ycbcr_420_only);
+	if (drm_mode_is_420_only(info, adjusted_mode)) {
+		ret = intel_hdmi_compute_output_format(encoder, crtc_state, connector,
+						       respect_downstream_limits,
+						       INTEL_OUTPUT_FORMAT_YCBCR420);
 
-	if (ycbcr_420_only && crtc_state->sink_format != INTEL_OUTPUT_FORMAT_YCBCR420) {
-		drm_dbg_kms(display->drm,
-			    "YCbCr 4:2:0 mode but YCbCr 4:2:0 output not possible. Falling back to RGB.\n");
-		crtc_state->sink_format = INTEL_OUTPUT_FORMAT_RGB;
-	}
+		if (ret && !respect_downstream_limits) {
+			drm_dbg_kms(display->drm,
+				    "YCbCr 4:2:0 mode but YCbCr 4:2:0 output not possible. Falling back to RGB.\n");
 
-	crtc_state->output_format = intel_hdmi_output_format(crtc_state);
-	ret = intel_hdmi_compute_clock(encoder, crtc_state, respect_downstream_limits);
-	if (ret) {
-		if (crtc_state->sink_format == INTEL_OUTPUT_FORMAT_YCBCR420 ||
-		    !crtc_state->has_hdmi_sink ||
-		    !connector->base.ycbcr_420_allowed ||
-		    !drm_mode_is_420_also(info, adjusted_mode))
-			return ret;
+			ret = intel_hdmi_compute_output_format(encoder, crtc_state, connector,
+							       respect_downstream_limits,
+							       INTEL_OUTPUT_FORMAT_RGB);
+		}
+	} else {
+		ret = intel_hdmi_compute_output_format(encoder, crtc_state, connector,
+						       respect_downstream_limits,
+						       INTEL_OUTPUT_FORMAT_RGB);
 
-		crtc_state->sink_format = INTEL_OUTPUT_FORMAT_YCBCR420;
-		crtc_state->output_format = intel_hdmi_output_format(crtc_state);
-		ret = intel_hdmi_compute_clock(encoder, crtc_state, respect_downstream_limits);
+		if (ret && drm_mode_is_420_also(info, adjusted_mode))
+			ret = intel_hdmi_compute_output_format(encoder, crtc_state, connector,
+							       respect_downstream_limits,
+							       INTEL_OUTPUT_FORMAT_YCBCR420);
 	}
 
 	return ret;
@@ -2370,9 +2407,9 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder,
 	 * Try to respect downstream TMDS clock limits first, if
 	 * that fails assume the user might know something we don't.
 	 */
-	ret = intel_hdmi_compute_output_format(encoder, pipe_config, conn_state, true);
+	ret = intel_hdmi_compute_formats(encoder, pipe_config, conn_state, true);
 	if (ret)
-		ret = intel_hdmi_compute_output_format(encoder, pipe_config, conn_state, false);
+		ret = intel_hdmi_compute_formats(encoder, pipe_config, conn_state, false);
 	if (ret) {
 		drm_dbg_kms(display->drm,
 			    "unsupported HDMI clock (%d kHz), rejecting mode\n",
@@ -2380,11 +2417,9 @@ int intel_hdmi_compute_config(struct intel_encoder *encoder,
 		return ret;
 	}
 
-	if (intel_hdmi_is_ycbcr420(pipe_config)) {
-		ret = intel_pfit_compute_config(pipe_config, conn_state);
-		if (ret)
-			return ret;
-	}
+	ret = intel_pfit_compute_config(pipe_config, conn_state);
+	if (ret)
+		return ret;
 
 	pipe_config->limited_color_range =
 		intel_hdmi_limited_color_range(pipe_config, conn_state);
@@ -2656,7 +2691,7 @@ static const struct drm_connector_funcs intel_hdmi_connector_funcs = {
 };
 
 static int intel_hdmi_connector_atomic_check(struct drm_connector *_connector,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct intel_connector *connector = to_intel_connector(_connector);
 	struct intel_display *display = to_intel_display(connector);
diff --git a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
index 8865cb2ac5697..539fd555edcea 100644
--- a/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
+++ b/drivers/gpu/drm/i915/display/intel_hotplug_irq.c
@@ -1420,7 +1420,7 @@ static void i915_hpd_irq_setup(struct intel_display *display)
 					     hotplug_en);
 }
 
-struct intel_hotplug_funcs {
+struct intel_hotplug_irq_funcs {
 	/* Enable HPD sense and interrupts for all present encoders */
 	void (*hpd_irq_setup)(struct intel_display *display);
 	/* Enable HPD sense for a single encoder */
@@ -1428,7 +1428,7 @@ struct intel_hotplug_funcs {
 };
 
 #define HPD_FUNCS(platform)					 \
-static const struct intel_hotplug_funcs platform##_hpd_funcs = { \
+static const struct intel_hotplug_irq_funcs platform##_hpd_funcs = { \
 	.hpd_irq_setup = platform##_hpd_irq_setup,		 \
 	.hpd_enable_detection = platform##_hpd_enable_detection, \
 }
@@ -1447,8 +1447,8 @@ void intel_hpd_enable_detection(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
 
-	if (display->funcs.hotplug)
-		display->funcs.hotplug->hpd_enable_detection(encoder);
+	if (display->hotplug.funcs)
+		display->hotplug.funcs->hpd_enable_detection(encoder);
 }
 
 void intel_hpd_irq_setup(struct intel_display *display)
@@ -1457,8 +1457,8 @@ void intel_hpd_irq_setup(struct intel_display *display)
 	    !display->irq.vlv_display_irqs_enabled)
 		return;
 
-	if (display->funcs.hotplug)
-		display->funcs.hotplug->hpd_irq_setup(display);
+	if (display->hotplug.funcs)
+		display->hotplug.funcs->hpd_irq_setup(display);
 }
 
 void intel_hotplug_irq_init(struct intel_display *display)
@@ -1469,23 +1469,23 @@ void intel_hotplug_irq_init(struct intel_display *display)
 
 	if (HAS_GMCH(display)) {
 		if (HAS_HOTPLUG(display))
-			display->funcs.hotplug = &i915_hpd_funcs;
+			display->hotplug.funcs = &i915_hpd_funcs;
 	} else {
 		if (HAS_PCH_DG2(display))
-			display->funcs.hotplug = &icp_hpd_funcs;
+			display->hotplug.funcs = &icp_hpd_funcs;
 		else if (HAS_PCH_DG1(display))
-			display->funcs.hotplug = &dg1_hpd_funcs;
+			display->hotplug.funcs = &dg1_hpd_funcs;
 		else if (DISPLAY_VER(display) >= 14)
-			display->funcs.hotplug = &xelpdp_hpd_funcs;
+			display->hotplug.funcs = &xelpdp_hpd_funcs;
 		else if (DISPLAY_VER(display) >= 11)
-			display->funcs.hotplug = &gen11_hpd_funcs;
+			display->hotplug.funcs = &gen11_hpd_funcs;
 		else if (display->platform.geminilake || display->platform.broxton)
-			display->funcs.hotplug = &bxt_hpd_funcs;
+			display->hotplug.funcs = &bxt_hpd_funcs;
 		else if (INTEL_PCH_TYPE(display) >= PCH_ICP)
-			display->funcs.hotplug = &icp_hpd_funcs;
+			display->hotplug.funcs = &icp_hpd_funcs;
 		else if (INTEL_PCH_TYPE(display) >= PCH_SPT)
-			display->funcs.hotplug = &spt_hpd_funcs;
+			display->hotplug.funcs = &spt_hpd_funcs;
 		else
-			display->funcs.hotplug = &ilk_hpd_funcs;
+			display->hotplug.funcs = &ilk_hpd_funcs;
 	}
 }
diff --git a/drivers/gpu/drm/i915/display/intel_initial_plane.c b/drivers/gpu/drm/i915/display/intel_initial_plane.c
index ee545c033da68..034fe199c2a15 100644
--- a/drivers/gpu/drm/i915/display/intel_initial_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_initial_plane.c
@@ -1,26 +1,51 @@
 // SPDX-License-Identifier: MIT
 /* Copyright © 2025 Intel Corporation */
 
+#include <linux/iopoll.h>
+
+#include <drm/drm_blend.h>
 #include <drm/drm_print.h>
 #include <drm/intel/display_parent_interface.h>
 
+#include "intel_crtc.h"
+#include "intel_de.h"
 #include "intel_display_core.h"
+#include "intel_display_regs.h"
 #include "intel_display_types.h"
 #include "intel_fb.h"
 #include "intel_frontbuffer.h"
 #include "intel_initial_plane.h"
+#include "intel_parent.h"
 #include "intel_plane.h"
 
+struct intel_initial_plane_configs {
+	struct intel_initial_plane_config config[I915_MAX_PIPES];
+};
+
 void intel_initial_plane_vblank_wait(struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(crtc);
+	u32 start_ts, end_ts;
+	int ret;
+
+	/* xe doesn't have interrupts enabled this early */
+	if (intel_parent_irq_enabled(display)) {
+		intel_crtc_wait_for_next_vblank(crtc);
+		return;
+	}
+
+	start_ts = intel_de_read(display, PIPE_FRMTMSTMP(crtc->pipe));
 
-	display->parent->initial_plane->vblank_wait(&crtc->base);
+	ret = poll_timeout_us(end_ts = intel_de_read(display, PIPE_FRMTMSTMP(crtc->pipe)),
+			      end_ts != start_ts, 1000, 1000 * 1000, false);
+	if (ret)
+		drm_warn(display->drm, "[CRTC:%d:%s] early vblank wait timed out\n",
+			 crtc->base.base.id, crtc->base.name);
 }
 
 static const struct intel_plane_state *
 intel_reuse_initial_plane_obj(struct intel_crtc *this,
-			      const struct intel_initial_plane_config plane_configs[])
+			      const struct intel_initial_plane_configs *all_plane_configs)
 {
 	struct intel_display *display = to_intel_display(this);
 	struct intel_crtc *crtc;
@@ -39,7 +64,8 @@ intel_reuse_initial_plane_obj(struct intel_crtc *this,
 		if (!plane_state->ggtt_vma)
 			continue;
 
-		if (plane_configs[this->pipe].base == plane_configs[crtc->pipe].base)
+		if (all_plane_configs->config[this->pipe].base ==
+		    all_plane_configs->config[crtc->pipe].base)
 			return plane_state;
 	}
 
@@ -50,17 +76,38 @@ static struct drm_gem_object *
 intel_alloc_initial_plane_obj(struct intel_display *display,
 			      struct intel_initial_plane_config *plane_config)
 {
-	struct intel_framebuffer *fb = plane_config->fb;
+	struct drm_framebuffer *fb = plane_config->fb;
 
-	switch (fb->base.modifier) {
+	switch (fb->modifier) {
 	case DRM_FORMAT_MOD_LINEAR:
+		break;
 	case I915_FORMAT_MOD_X_TILED:
 	case I915_FORMAT_MOD_Y_TILED:
-	case I915_FORMAT_MOD_4_TILED:
-		break;
+		/* fenced region needed for linear CPU access to tiled FB */
+		if (intel_parent_has_fenced_regions(display))
+			break;
+		fallthrough;
 	default:
 		drm_dbg_kms(display->drm, "Unsupported modifier for initial FB: 0x%llx\n",
-			    fb->base.modifier);
+			    fb->modifier);
+		return NULL;
+	}
+
+	/*
+	 * Would need to preserve the DPT, its GGTT
+	 * mapping, and the actual FB memory.
+	 */
+	if (intel_fb_modifier_uses_dpt(display, fb->modifier)) {
+		drm_dbg_kms(display->drm, "DPT not supported for initial FB\n");
+		return NULL;
+	}
+
+	/*
+	 * Would need to preserve the 270 degree rotated
+	 * GGTT mapping used by the display hardware.
+	 */
+	if (drm_rotation_90_or_270(plane_config->rotation)) {
+		drm_dbg_kms(display->drm, "90/270 degree rotation not supported for initial FB\n");
 		return NULL;
 	}
 
@@ -69,10 +116,10 @@ intel_alloc_initial_plane_obj(struct intel_display *display,
 
 static void
 intel_find_initial_plane_obj(struct intel_crtc *crtc,
-			     struct intel_initial_plane_config plane_configs[])
+			     struct intel_initial_plane_configs *all_plane_configs)
 {
 	struct intel_display *display = to_intel_display(crtc);
-	struct intel_initial_plane_config *plane_config = &plane_configs[crtc->pipe];
+	struct intel_initial_plane_config *plane_config = &all_plane_configs->config[crtc->pipe];
 	struct intel_plane *plane = to_intel_plane(crtc->base.primary);
 	struct intel_plane_state *plane_state = to_intel_plane_state(plane->base.state);
 	struct drm_framebuffer *fb;
@@ -88,12 +135,12 @@ intel_find_initial_plane_obj(struct intel_crtc *crtc,
 		return;
 
 	if (intel_alloc_initial_plane_obj(display, plane_config)) {
-		fb = &plane_config->fb->base;
+		fb = plane_config->fb;
 		vma = plane_config->vma;
 	} else {
 		const struct intel_plane_state *other_plane_state;
 
-		other_plane_state = intel_reuse_initial_plane_obj(crtc, plane_configs);
+		other_plane_state = intel_reuse_initial_plane_obj(crtc, all_plane_configs);
 		if (!other_plane_state)
 			goto nofb;
 
@@ -144,7 +191,7 @@ static void plane_config_fini(struct intel_display *display,
 			      struct intel_initial_plane_config *plane_config)
 {
 	if (plane_config->fb) {
-		struct drm_framebuffer *fb = &plane_config->fb->base;
+		struct drm_framebuffer *fb = plane_config->fb;
 
 		/* We may only have the stub and not a full framebuffer */
 		if (drm_framebuffer_read_refcount(fb))
@@ -158,14 +205,14 @@ static void plane_config_fini(struct intel_display *display,
 
 void intel_initial_plane_config(struct intel_display *display)
 {
-	struct intel_initial_plane_config plane_configs[I915_MAX_PIPES] = {};
+	struct intel_initial_plane_configs all_plane_configs = {};
 	struct intel_crtc *crtc;
 
 	for_each_intel_crtc(display->drm, crtc) {
 		const struct intel_crtc_state *crtc_state =
 			to_intel_crtc_state(crtc->base.state);
 		struct intel_initial_plane_config *plane_config =
-			&plane_configs[crtc->pipe];
+			&all_plane_configs.config[crtc->pipe];
 
 		if (!crtc_state->hw.active)
 			continue;
@@ -177,15 +224,15 @@ void intel_initial_plane_config(struct intel_display *display)
 		 * can even allow for smooth boot transitions if the BIOS
 		 * fb is large enough for the active pipe configuration.
 		 */
-		display->funcs.display->get_initial_plane_config(crtc, plane_config);
+		display->modeset.funcs->get_initial_plane_config(crtc, plane_config);
 
 		/*
 		 * If the fb is shared between multiple heads, we'll
 		 * just get the first one.
 		 */
-		intel_find_initial_plane_obj(crtc, plane_configs);
+		intel_find_initial_plane_obj(crtc, &all_plane_configs);
 
-		if (display->funcs.display->fixup_initial_plane_config(crtc, plane_config))
+		if (display->modeset.funcs->fixup_initial_plane_config(crtc, plane_config))
 			intel_initial_plane_vblank_wait(crtc);
 
 		plane_config_fini(display, plane_config);
diff --git a/drivers/gpu/drm/i915/display/intel_load_detect.c b/drivers/gpu/drm/i915/display/intel_load_detect.c
index aad52d0d83e1e..2f767b15a7f9b 100644
--- a/drivers/gpu/drm/i915/display/intel_load_detect.c
+++ b/drivers/gpu/drm/i915/display/intel_load_detect.c
@@ -20,7 +20,7 @@ static const struct drm_display_mode load_detect_mode = {
 		 704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
 };
 
-static int intel_modeset_disable_planes(struct drm_atomic_state *state,
+static int intel_modeset_disable_planes(struct drm_atomic_commit *state,
 					struct drm_crtc *crtc)
 {
 	struct drm_plane *plane;
@@ -45,7 +45,7 @@ static int intel_modeset_disable_planes(struct drm_atomic_state *state,
 	return 0;
 }
 
-struct drm_atomic_state *
+struct drm_atomic_commit *
 intel_load_detect_get_pipe(struct drm_connector *connector,
 			   struct drm_modeset_acquire_ctx *ctx)
 {
@@ -55,7 +55,7 @@ intel_load_detect_get_pipe(struct drm_connector *connector,
 	struct intel_crtc *possible_crtc;
 	struct intel_crtc *crtc = NULL;
 	struct drm_mode_config *config = &display->drm->mode_config;
-	struct drm_atomic_state *state = NULL, *restore_state = NULL;
+	struct drm_atomic_commit *state = NULL, *restore_state = NULL;
 	struct drm_connector_state *connector_state;
 	struct intel_crtc_state *crtc_state;
 	int ret;
@@ -118,8 +118,8 @@ intel_load_detect_get_pipe(struct drm_connector *connector,
 	}
 
 found:
-	state = drm_atomic_state_alloc(display->drm);
-	restore_state = drm_atomic_state_alloc(display->drm);
+	state = drm_atomic_commit_alloc(display->drm);
+	restore_state = drm_atomic_commit_alloc(display->drm);
 	if (!state || !restore_state) {
 		ret = -ENOMEM;
 		goto fail;
@@ -177,7 +177,7 @@ intel_load_detect_get_pipe(struct drm_connector *connector,
 		goto fail;
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	/* let the connector get through one full cycle before testing */
 	intel_crtc_wait_for_next_vblank(crtc);
@@ -186,11 +186,11 @@ intel_load_detect_get_pipe(struct drm_connector *connector,
 
 fail:
 	if (state) {
-		drm_atomic_state_put(state);
+		drm_atomic_commit_put(state);
 		state = NULL;
 	}
 	if (restore_state) {
-		drm_atomic_state_put(restore_state);
+		drm_atomic_commit_put(restore_state);
 		restore_state = NULL;
 	}
 
@@ -201,7 +201,7 @@ intel_load_detect_get_pipe(struct drm_connector *connector,
 }
 
 void intel_load_detect_release_pipe(struct drm_connector *connector,
-				    struct drm_atomic_state *state,
+				    struct drm_atomic_commit *state,
 				    struct drm_modeset_acquire_ctx *ctx)
 {
 	struct intel_display *display = to_intel_display(connector->dev);
@@ -221,5 +221,5 @@ void intel_load_detect_release_pipe(struct drm_connector *connector,
 	if (ret)
 		drm_dbg_kms(display->drm,
 			    "Couldn't release load detect pipe: %i\n", ret);
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 }
diff --git a/drivers/gpu/drm/i915/display/intel_load_detect.h b/drivers/gpu/drm/i915/display/intel_load_detect.h
index aed51901b9ba8..7525988e45b18 100644
--- a/drivers/gpu/drm/i915/display/intel_load_detect.h
+++ b/drivers/gpu/drm/i915/display/intel_load_detect.h
@@ -6,15 +6,15 @@
 #ifndef __INTEL_LOAD_DETECT_H__
 #define __INTEL_LOAD_DETECT_H__
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_connector;
 struct drm_modeset_acquire_ctx;
 
-struct drm_atomic_state *
+struct drm_atomic_commit *
 intel_load_detect_get_pipe(struct drm_connector *connector,
 			   struct drm_modeset_acquire_ctx *ctx);
 void intel_load_detect_release_pipe(struct drm_connector *connector,
-				    struct drm_atomic_state *old,
+				    struct drm_atomic_commit *old,
 				    struct drm_modeset_acquire_ctx *ctx);
 
 #endif /* __INTEL_LOAD_DETECT_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_lpe_audio.c b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
index 117b60656ca19..775493306a833 100644
--- a/drivers/gpu/drm/i915/display/intel_lpe_audio.c
+++ b/drivers/gpu/drm/i915/display/intel_lpe_audio.c
@@ -70,11 +70,11 @@
 
 #include <drm/drm_print.h>
 #include <drm/intel/intel_lpe_audio.h>
+#include <drm/intel/pci_config.h>
 
 #include "intel_audio_regs.h"
 #include "intel_de.h"
 #include "intel_lpe_audio.h"
-#include "intel_pci_config.h"
 
 #define HAS_LPE_AUDIO(display) ((display)->audio.lpe.platdev)
 
diff --git a/drivers/gpu/drm/i915/display/intel_lt_phy.c b/drivers/gpu/drm/i915/display/intel_lt_phy.c
index 657ad5cb0eff9..615ee980470ee 100644
--- a/drivers/gpu/drm/i915/display/intel_lt_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_lt_phy.c
@@ -1007,7 +1007,7 @@ assert_dc_off(struct intel_display *display)
 
 static int __intel_lt_phy_p2p_write_once(struct intel_encoder *encoder,
 					 int lane, u16 addr, u8 data,
-					 i915_reg_t mac_reg_addr,
+					 intel_reg_t mac_reg_addr,
 					 u8 expected_mac_val)
 {
 	struct intel_display *display = to_intel_display(encoder);
@@ -1062,7 +1062,7 @@ static int __intel_lt_phy_p2p_write_once(struct intel_encoder *encoder,
 
 static void __intel_lt_phy_p2p_write(struct intel_encoder *encoder,
 				     int lane, u16 addr, u8 data,
-				     i915_reg_t mac_reg_addr,
+				     intel_reg_t mac_reg_addr,
 				     u8 expected_mac_val)
 {
 	struct intel_display *display = to_intel_display(encoder);
@@ -1086,7 +1086,7 @@ static void __intel_lt_phy_p2p_write(struct intel_encoder *encoder,
 
 static void intel_lt_phy_p2p_write(struct intel_encoder *encoder,
 				   u8 lane_mask, u16 addr, u8 data,
-				   i915_reg_t mac_reg_addr,
+				   intel_reg_t mac_reg_addr,
 				   u8 expected_mac_val)
 {
 	int lane;
@@ -2135,7 +2135,7 @@ void intel_lt_phy_set_signal_levels(struct intel_encoder *encoder,
 
 	wakeref = intel_lt_phy_transaction_begin(encoder);
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans)) {
 		intel_lt_phy_transaction_end(encoder, wakeref);
 		return;
diff --git a/drivers/gpu/drm/i915/display/intel_lvds.c b/drivers/gpu/drm/i915/display/intel_lvds.c
index cc6d4bfcff102..c8098104d853a 100644
--- a/drivers/gpu/drm/i915/display/intel_lvds.c
+++ b/drivers/gpu/drm/i915/display/intel_lvds.c
@@ -70,7 +70,7 @@ struct intel_lvds_encoder {
 	struct intel_encoder base;
 
 	bool is_dual_link;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 a3_power;
 
 	struct intel_lvds_pps init_pps;
@@ -85,7 +85,7 @@ static struct intel_lvds_encoder *to_lvds_encoder(struct intel_encoder *encoder)
 }
 
 bool intel_lvds_port_enabled(struct intel_display *display,
-			     i915_reg_t lvds_reg, enum pipe *pipe)
+			     intel_reg_t lvds_reg, enum pipe *pipe)
 {
 	u32 val;
 
@@ -395,20 +395,19 @@ intel_lvds_mode_valid(struct drm_connector *_connector,
 {
 	struct intel_display *display = to_intel_display(_connector->dev);
 	struct intel_connector *connector = to_intel_connector(_connector);
-	const struct drm_display_mode *fixed_mode =
-		intel_panel_fixed_mode(connector, mode);
 	int max_pixclk = display->cdclk.max_dotclk_freq;
 	enum drm_mode_status status;
+	int target_clock;
 
 	status = intel_cpu_transcoder_mode_valid(display, mode);
 	if (status != MODE_OK)
 		return status;
 
-	status = intel_panel_mode_valid(connector, mode);
+	status = intel_panel_mode_valid(connector, mode, &target_clock);
 	if (status != MODE_OK)
 		return status;
 
-	if (fixed_mode->clock > max_pixclk)
+	if (target_clock > max_pixclk)
 		return MODE_CLOCK_HIGH;
 
 	return MODE_OK;
@@ -846,7 +845,7 @@ void intel_lvds_init(struct intel_display *display)
 	struct intel_connector *connector;
 	const struct drm_edid *drm_edid;
 	struct intel_encoder *encoder;
-	i915_reg_t lvds_reg;
+	intel_reg_t lvds_reg;
 	u32 lvds;
 	u8 ddc_pin;
 
diff --git a/drivers/gpu/drm/i915/display/intel_lvds.h b/drivers/gpu/drm/i915/display/intel_lvds.h
index a6db1706a97c8..6ac70d5e8ce62 100644
--- a/drivers/gpu/drm/i915/display/intel_lvds.h
+++ b/drivers/gpu/drm/i915/display/intel_lvds.h
@@ -8,20 +8,20 @@
 
 #include <linux/types.h>
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 
 enum pipe;
 struct intel_display;
 
 #ifdef I915
 bool intel_lvds_port_enabled(struct intel_display *display,
-			     i915_reg_t lvds_reg, enum pipe *pipe);
+			     intel_reg_t lvds_reg, enum pipe *pipe);
 void intel_lvds_init(struct intel_display *display);
 struct intel_encoder *intel_get_lvds_encoder(struct intel_display *display);
 bool intel_is_dual_link_lvds(struct intel_display *display);
 #else
 static inline bool intel_lvds_port_enabled(struct intel_display *display,
-					   i915_reg_t lvds_reg, enum pipe *pipe)
+					   intel_reg_t lvds_reg, enum pipe *pipe)
 {
 	return false;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_mchbar.c b/drivers/gpu/drm/i915/display/intel_mchbar.c
new file mode 100644
index 0000000000000..a3a69e11c390e
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_mchbar.c
@@ -0,0 +1,78 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <linux/minmax.h>
+
+#include <drm/drm_print.h>
+
+#include "intel_de.h"
+#include "intel_display_core.h"
+#include "intel_mchbar.h"
+
+static bool has_mchbar_mirror(struct intel_display *display)
+{
+	return DISPLAY_VER(display) < 14;
+}
+
+static u32 mchbar_mirror_base(struct intel_display *display)
+{
+	if (DISPLAY_VER(display) >= 6)
+		return MCHBAR_MIRROR_BASE_SNB;
+	else
+		return MCHBAR_MIRROR_BASE;
+}
+
+static u32 mchbar_mirror_end(struct intel_display *display)
+{
+	if (DISPLAY_VER(display) >= 12 && !display->platform.rocketlake)
+		return MCHBAR_MIRROR_END_TGL;
+	else if (DISPLAY_VER(display) >= 11)
+		return MCHBAR_MIRROR_END_ICL_RKL;
+	else if (DISPLAY_VER(display) >= 6)
+		return MCHBAR_MIRROR_END_SNB;
+	else
+		return MCHBAR_MIRROR_END;
+}
+
+static u32 mchbar_mirror_len(struct intel_display *display)
+{
+	return mchbar_mirror_end(display) - mchbar_mirror_base(display) + 1;
+}
+
+static bool is_mchbar_reg(struct intel_display *display, intel_reg_t reg)
+{
+	return has_mchbar_mirror(display) &&
+		in_range32(intel_reg_offset(reg),
+			   mchbar_mirror_base(display),
+			   mchbar_mirror_len(display));
+}
+
+static void assert_is_mchbar_reg(struct intel_display *display, intel_reg_t reg)
+{
+	drm_WARN(display->drm, !is_mchbar_reg(display, reg),
+		 "Reading non-MCHBAR register 0x%x\n",
+		 intel_reg_offset(reg));
+}
+
+u16 intel_mchbar_read16(struct intel_display *display, intel_reg_t reg)
+{
+	assert_is_mchbar_reg(display, reg);
+
+	return intel_de_read16(display, reg);
+}
+
+u32 intel_mchbar_read(struct intel_display *display, intel_reg_t reg)
+{
+	assert_is_mchbar_reg(display, reg);
+
+	return intel_de_read(display, reg);
+}
+
+u64 intel_mchbar_read64_2x32(struct intel_display *display, intel_reg_t reg)
+{
+	assert_is_mchbar_reg(display, reg);
+
+	return intel_de_read64_2x32(display, reg);
+}
diff --git a/drivers/gpu/drm/i915/display/intel_mchbar.h b/drivers/gpu/drm/i915/display/intel_mchbar.h
new file mode 100644
index 0000000000000..1ced5dc8a8fdb
--- /dev/null
+++ b/drivers/gpu/drm/i915/display/intel_mchbar.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef __INTEL_MCHBAR_H__
+#define __INTEL_MCHBAR_H__
+
+#include <linux/types.h>
+
+#include <drm/intel/mchbar_regs.h>
+
+#include "intel_display_reg_defs.h"
+
+struct intel_display;
+
+u16 intel_mchbar_read16(struct intel_display *display, intel_reg_t reg);
+u32 intel_mchbar_read(struct intel_display *display, intel_reg_t reg);
+u64 intel_mchbar_read64_2x32(struct intel_display *display, intel_reg_t reg);
+
+#endif /* __INTEL_MCHBAR_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_lock.c b/drivers/gpu/drm/i915/display/intel_modeset_lock.c
index 8fb6fd849a75d..eb6a5a99328b6 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_lock.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_lock.c
@@ -36,7 +36,7 @@ void _intel_modeset_lock_end(struct drm_modeset_acquire_ctx *ctx,
 {
 	if (*ret == -EDEADLK) {
 		if (state)
-			drm_atomic_state_clear(&state->base);
+			drm_atomic_commit_clear(&state->base);
 
 		*ret = drm_modeset_backoff(ctx);
 		if (*ret == 0) {
diff --git a/drivers/gpu/drm/i915/display/intel_modeset_setup.c b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
index 4086f16a12bf4..e88082c8caace 100644
--- a/drivers/gpu/drm/i915/display/intel_modeset_setup.c
+++ b/drivers/gpu/drm/i915/display/intel_modeset_setup.c
@@ -44,7 +44,7 @@ static void intel_crtc_disable_noatomic_begin(struct intel_crtc *crtc,
 	struct intel_crtc_state *crtc_state =
 		to_intel_crtc_state(crtc->base.state);
 	struct intel_plane *plane;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct intel_crtc *temp_crtc;
 	enum pipe pipe = crtc->pipe;
 
@@ -59,7 +59,7 @@ static void intel_crtc_disable_noatomic_begin(struct intel_crtc *crtc,
 			intel_plane_disable_noatomic(crtc, plane);
 	}
 
-	state = drm_atomic_state_alloc(display->drm);
+	state = drm_atomic_commit_alloc(display->drm);
 	if (!state) {
 		drm_dbg_kms(display->drm,
 			    "failed to disable [CRTC:%d:%s], out of memory",
@@ -83,9 +83,9 @@ static void intel_crtc_disable_noatomic_begin(struct intel_crtc *crtc,
 		drm_WARN_ON(display->drm, IS_ERR(temp_crtc_state) || ret);
 	}
 
-	display->funcs.display->crtc_disable(to_intel_atomic_state(state), crtc);
+	display->modeset.funcs->crtc_disable(to_intel_atomic_state(state), crtc);
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	drm_dbg_kms(display->drm,
 		    "[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
@@ -333,6 +333,7 @@ static void intel_crtc_copy_hw_to_uapi_state(struct intel_crtc_state *crtc_state
 
 	crtc_state->uapi.adjusted_mode = crtc_state->hw.adjusted_mode;
 	crtc_state->uapi.scaling_filter = crtc_state->hw.scaling_filter;
+	crtc_state->uapi.sharpness_strength = crtc_state->hw.sharpness_strength;
 
 	if (DISPLAY_INFO(display)->color.degamma_lut_size) {
 		/* assume 1:1 mapping */
diff --git a/drivers/gpu/drm/i915/display/intel_opregion.c b/drivers/gpu/drm/i915/display/intel_opregion.c
index e25be56e678bc..9f88b7cac9f74 100644
--- a/drivers/gpu/drm/i915/display/intel_opregion.c
+++ b/drivers/gpu/drm/i915/display/intel_opregion.c
@@ -34,13 +34,13 @@
 #include <drm/drm_edid.h>
 #include <drm/drm_file.h>
 #include <drm/drm_print.h>
+#include <drm/intel/pci_config.h>
 
 #include "intel_acpi.h"
 #include "intel_backlight.h"
 #include "intel_display_core.h"
 #include "intel_display_types.h"
 #include "intel_opregion.h"
-#include "intel_pci_config.h"
 
 #define OPREGION_HEADER_OFFSET 0
 #define OPREGION_ACPI_OFFSET   0x100
diff --git a/drivers/gpu/drm/i915/display/intel_overlay.c b/drivers/gpu/drm/i915/display/intel_overlay.c
index 12a325ceae6fe..a809aa2950ac3 100644
--- a/drivers/gpu/drm/i915/display/intel_overlay.c
+++ b/drivers/gpu/drm/i915/display/intel_overlay.c
@@ -481,13 +481,9 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	if (ret != 0)
 		return ret;
 
-	atomic_inc(&display->restore.pending_fb_pin);
-
 	vma = intel_parent_overlay_pin_fb(display, obj, &offset);
-	if (IS_ERR(vma)) {
-		ret = PTR_ERR(vma);
-		goto out_pin_section;
-	}
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
 
 	if (!intel_parent_overlay_is_active(display)) {
 		const struct intel_crtc_state *crtc_state =
@@ -571,8 +567,6 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 
 out_unpin:
 	intel_parent_overlay_unpin_fb(display, vma);
-out_pin_section:
-	atomic_dec(&display->restore.pending_fb_pin);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_panel.c b/drivers/gpu/drm/i915/display/intel_panel.c
index 2a20aaaaac39b..20c548eea6da3 100644
--- a/drivers/gpu/drm/i915/display/intel_panel.c
+++ b/drivers/gpu/drm/i915/display/intel_panel.c
@@ -396,11 +396,15 @@ intel_panel_detect(struct drm_connector *connector, bool force)
 
 enum drm_mode_status
 intel_panel_mode_valid(struct intel_connector *connector,
-		       const struct drm_display_mode *mode)
+		       const struct drm_display_mode *mode,
+		       int *target_clock)
 {
 	const struct drm_display_mode *fixed_mode =
 		intel_panel_fixed_mode(connector, mode);
 
+	if (target_clock)
+		*target_clock = mode->clock;
+
 	if (!fixed_mode)
 		return MODE_OK;
 
@@ -413,6 +417,9 @@ intel_panel_mode_valid(struct intel_connector *connector,
 	if (drm_mode_vrefresh(mode) != drm_mode_vrefresh(fixed_mode))
 		return MODE_PANEL;
 
+	if (target_clock)
+		*target_clock = fixed_mode->clock;
+
 	return MODE_OK;
 }
 
diff --git a/drivers/gpu/drm/i915/display/intel_panel.h b/drivers/gpu/drm/i915/display/intel_panel.h
index 56a6412cf0fb1..23bd227826c9e 100644
--- a/drivers/gpu/drm/i915/display/intel_panel.h
+++ b/drivers/gpu/drm/i915/display/intel_panel.h
@@ -43,7 +43,8 @@ int intel_panel_get_modes(struct intel_connector *connector);
 enum drrs_type intel_panel_drrs_type(struct intel_connector *connector);
 enum drm_mode_status
 intel_panel_mode_valid(struct intel_connector *connector,
-		       const struct drm_display_mode *mode);
+		       const struct drm_display_mode *mode,
+		       int *target_clock);
 int intel_panel_compute_config(struct intel_connector *connector,
 			       struct drm_display_mode *adjusted_mode);
 void intel_panel_add_edid_fixed_modes(struct intel_connector *connector,
diff --git a/drivers/gpu/drm/i915/display/intel_parent.c b/drivers/gpu/drm/i915/display/intel_parent.c
index 2e3bad2b3e6b7..a5816561be408 100644
--- a/drivers/gpu/drm/i915/display/intel_parent.c
+++ b/drivers/gpu/drm/i915/display/intel_parent.c
@@ -19,6 +19,7 @@
 
 #include <drm/drm_print.h>
 #include <drm/intel/display_parent_interface.h>
+#include <drm/intel/vlv_iosf_sb_regs.h>
 
 #include "intel_display_core.h"
 #include "intel_parent.h"
@@ -51,6 +52,66 @@ void intel_parent_dpt_resume(struct intel_display *display, struct intel_dpt *dp
 		display->parent->dpt->resume(dpt);
 }
 
+/* fb_pin */
+int intel_parent_fb_pin_ggtt_pin(struct intel_display *display,
+				 struct drm_gem_object *obj,
+				 const struct intel_fb_pin_params *pin_params,
+				 struct i915_vma **out_ggtt_vma,
+				 u32 *out_offset,
+				 int *out_fence_id)
+{
+	return display->parent->fb_pin->ggtt_pin(obj, pin_params,
+						 out_ggtt_vma, out_offset, out_fence_id);
+}
+
+void intel_parent_fb_pin_ggtt_unpin(struct intel_display *display,
+				    struct i915_vma *ggtt_vma,
+				    int fence_id)
+{
+	return display->parent->fb_pin->ggtt_unpin(ggtt_vma, fence_id);
+}
+
+int intel_parent_fb_pin_dpt_pin(struct intel_display *display,
+				struct drm_gem_object *obj,
+				struct intel_dpt *dpt,
+				const struct intel_fb_pin_params *pin_params,
+				struct i915_vma **out_dpt_vma,
+				struct i915_vma **out_ggtt_vma,
+				u32 *out_offset)
+{
+	return display->parent->fb_pin->dpt_pin(obj, dpt, pin_params,
+						out_dpt_vma, out_ggtt_vma, out_offset);
+}
+
+void intel_parent_fb_pin_dpt_unpin(struct intel_display *display,
+				   struct intel_dpt *dpt,
+				   struct i915_vma *dpt_vma,
+				   struct i915_vma *ggtt_vma)
+{
+	return display->parent->fb_pin->dpt_unpin(dpt, dpt_vma, ggtt_vma);
+}
+
+struct i915_vma *intel_parent_fb_pin_reuse_vma(struct intel_display *display,
+					       struct i915_vma *old_ggtt_vma,
+					       struct drm_gem_object *old_obj,
+					       const struct i915_gtt_view *old_view,
+					       struct drm_gem_object *new_obj,
+					       const struct i915_gtt_view *new_view,
+					       u32 *out_offset)
+{
+	if (!display->parent->fb_pin->reuse_vma)
+		return NULL;
+
+	return display->parent->fb_pin->reuse_vma(old_ggtt_vma, old_obj, old_view,
+						  new_obj, new_view, out_offset);
+}
+
+void intel_parent_fb_pin_get_map(struct intel_display *display,
+				 struct i915_vma *vma, struct iosys_map *map)
+{
+	return display->parent->fb_pin->get_map(vma, map);
+}
+
 /* frontbuffer */
 struct intel_frontbuffer *intel_parent_frontbuffer_get(struct intel_display *display, struct drm_gem_object *obj)
 {
@@ -338,13 +399,37 @@ void intel_parent_stolen_node_free(struct intel_display *display, const struct i
 	display->parent->stolen->node_free(node);
 }
 
-/* vma */
-int intel_parent_vma_fence_id(struct intel_display *display, const struct i915_vma *vma)
+/* vlv iosf */
+void intel_parent_vlv_iosf_get(struct intel_display *display, unsigned long unit_mask)
+{
+	if (drm_WARN_ON_ONCE(display->drm, !display->parent->vlv_iosf))
+		return;
+
+	display->parent->vlv_iosf->get(display->drm, unit_mask);
+}
+
+void intel_parent_vlv_iosf_put(struct intel_display *display, unsigned long unit_mask)
+{
+	if (drm_WARN_ON_ONCE(display->drm, !display->parent->vlv_iosf))
+		return;
+
+	display->parent->vlv_iosf->put(display->drm, unit_mask);
+}
+
+u32 intel_parent_vlv_iosf_read(struct intel_display *display, enum vlv_iosf_sb_unit unit, u32 addr)
+{
+	if (drm_WARN_ON_ONCE(display->drm, !display->parent->vlv_iosf))
+		return 0;
+
+	return display->parent->vlv_iosf->read(display->drm, unit, addr);
+}
+
+int intel_parent_vlv_iosf_write(struct intel_display *display, enum vlv_iosf_sb_unit unit, u32 addr, u32 val)
 {
-	if (!display->parent->vma)
-		return -1;
+	if (drm_WARN_ON_ONCE(display->drm, !display->parent->vlv_iosf))
+		return -EINVAL;
 
-	return display->parent->vma->fence_id(vma);
+	return display->parent->vlv_iosf->write(display->drm, unit, addr, val);
 }
 
 /* generic */
diff --git a/drivers/gpu/drm/i915/display/intel_parent.h b/drivers/gpu/drm/i915/display/intel_parent.h
index 2013e5ed5aa9b..27e35f891a6bb 100644
--- a/drivers/gpu/drm/i915/display/intel_parent.h
+++ b/drivers/gpu/drm/i915/display/intel_parent.h
@@ -6,17 +6,21 @@
 
 #include <linux/types.h>
 
+enum vlv_iosf_sb_unit;
 struct dma_fence;
 struct drm_file;
 struct drm_gem_object;
 struct drm_scanout_buffer;
+struct i915_gtt_view;
 struct i915_vma;
 struct intel_display;
 struct intel_dpt;
+struct intel_fb_pin_params;
 struct intel_frontbuffer;
 struct intel_hdcp_gsc_context;
 struct intel_panic;
 struct intel_stolen_node;
+struct iosys_map;
 
 /* dpt */
 struct intel_dpt *intel_parent_dpt_create(struct intel_display *display,
@@ -25,6 +29,37 @@ void intel_parent_dpt_destroy(struct intel_display *display, struct intel_dpt *d
 void intel_parent_dpt_suspend(struct intel_display *display, struct intel_dpt *dpt);
 void intel_parent_dpt_resume(struct intel_display *display, struct intel_dpt *dpt);
 
+/* fb_pin */
+int intel_parent_fb_pin_ggtt_pin(struct intel_display *display,
+				 struct drm_gem_object *obj,
+				 const struct intel_fb_pin_params *pin_params,
+				 struct i915_vma **out_ggtt_vma,
+				 u32 *out_offset,
+				 int *out_fence_id);
+void intel_parent_fb_pin_ggtt_unpin(struct intel_display *display,
+				    struct i915_vma *ggtt_vma,
+				    int fence_id);
+int intel_parent_fb_pin_dpt_pin(struct intel_display *display,
+				struct drm_gem_object *obj,
+				struct intel_dpt *dpt,
+				const struct intel_fb_pin_params *pin_params,
+				struct i915_vma **out_dpt_vma,
+				struct i915_vma **out_ggtt_vma,
+				u32 *out_offset);
+void intel_parent_fb_pin_dpt_unpin(struct intel_display *display,
+				   struct intel_dpt *dpt,
+				   struct i915_vma *dpt_vma,
+				   struct i915_vma *ggtt_vma);
+struct i915_vma *intel_parent_fb_pin_reuse_vma(struct intel_display *display,
+					       struct i915_vma *old_ggtt_vma,
+					       struct drm_gem_object *old_obj,
+					       const struct i915_gtt_view *old_view,
+					       struct drm_gem_object *new_obj,
+					       const struct i915_gtt_view *new_view,
+					       u32 *out_offset);
+void intel_parent_fb_pin_get_map(struct intel_display *display,
+				 struct i915_vma *vma, struct iosys_map *map);
+
 /* frontbuffer */
 struct intel_frontbuffer *intel_parent_frontbuffer_get(struct intel_display *display, struct drm_gem_object *obj);
 void intel_parent_frontbuffer_ref(struct intel_display *display, struct intel_frontbuffer *front);
@@ -109,8 +144,11 @@ u64 intel_parent_stolen_node_size(struct intel_display *display, const struct in
 struct intel_stolen_node *intel_parent_stolen_node_alloc(struct intel_display *display);
 void intel_parent_stolen_node_free(struct intel_display *display, const struct intel_stolen_node *node);
 
-/* vma */
-int intel_parent_vma_fence_id(struct intel_display *display, const struct i915_vma *vma);
+/* vlv iosf */
+void intel_parent_vlv_iosf_get(struct intel_display *display, unsigned long unit_mask);
+void intel_parent_vlv_iosf_put(struct intel_display *display, unsigned long unit_mask);
+u32 intel_parent_vlv_iosf_read(struct intel_display *display, enum vlv_iosf_sb_unit unit, u32 addr);
+int intel_parent_vlv_iosf_write(struct intel_display *display, enum vlv_iosf_sb_unit unit, u32 addr, u32 val);
 
 /* generic */
 bool intel_parent_has_auxccs(struct intel_display *display);
diff --git a/drivers/gpu/drm/i915/display/intel_pch_display.c b/drivers/gpu/drm/i915/display/intel_pch_display.c
index 69c7952a14136..2fcf6c9c84cc8 100644
--- a/drivers/gpu/drm/i915/display/intel_pch_display.c
+++ b/drivers/gpu/drm/i915/display/intel_pch_display.c
@@ -40,7 +40,7 @@ enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
 
 static void assert_pch_dp_disabled(struct intel_display *display,
 				   enum pipe pipe, enum port port,
-				   i915_reg_t dp_reg)
+				   intel_reg_t dp_reg)
 {
 	enum pipe port_pipe;
 	bool state;
@@ -59,7 +59,7 @@ static void assert_pch_dp_disabled(struct intel_display *display,
 
 static void assert_pch_hdmi_disabled(struct intel_display *display,
 				     enum pipe pipe, enum port port,
-				     i915_reg_t hdmi_reg)
+				     intel_reg_t hdmi_reg)
 {
 	enum pipe port_pipe;
 	bool state;
@@ -115,7 +115,7 @@ static void assert_pch_transcoder_disabled(struct intel_display *display,
 }
 
 static void ibx_sanitize_pch_hdmi_port(struct intel_display *display,
-				       enum port port, i915_reg_t hdmi_reg)
+				       enum port port, intel_reg_t hdmi_reg)
 {
 	u32 val = intel_de_read(display, hdmi_reg);
 
@@ -134,7 +134,7 @@ static void ibx_sanitize_pch_hdmi_port(struct intel_display *display,
 }
 
 static void ibx_sanitize_pch_dp_port(struct intel_display *display,
-				     enum port port, i915_reg_t dp_reg)
+				     enum port port, intel_reg_t dp_reg)
 {
 	u32 val = intel_de_read(display, dp_reg);
 
@@ -247,7 +247,7 @@ static void ilk_enable_pch_transcoder(const struct intel_crtc_state *crtc_state)
 	struct intel_display *display = to_intel_display(crtc_state);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 val, pipeconf_val;
 
 	/* Make sure PCH DPLL is enabled */
@@ -313,7 +313,7 @@ static void ilk_disable_pch_transcoder(struct intel_crtc *crtc)
 {
 	struct intel_display *display = to_intel_display(crtc);
 	enum pipe pipe = crtc->pipe;
-	i915_reg_t reg;
+	intel_reg_t reg;
 
 	/* FDI relies on the transcoder */
 	assert_fdi_tx_disabled(display, pipe);
@@ -417,7 +417,7 @@ void ilk_pch_enable(struct intel_atomic_state *state,
 			&crtc_state->hw.adjusted_mode;
 		u32 bpc = (intel_de_read(display, TRANSCONF(display, pipe))
 			   & TRANSCONF_BPC_MASK) >> 5;
-		i915_reg_t reg = TRANS_DP_CTL(pipe);
+		intel_reg_t reg = TRANS_DP_CTL(pipe);
 		enum port port;
 
 		temp = intel_de_read(display, reg);
diff --git a/drivers/gpu/drm/i915/display/intel_pfit.c b/drivers/gpu/drm/i915/display/intel_pfit.c
index 6dda496190e07..5b170c74a5100 100644
--- a/drivers/gpu/drm/i915/display/intel_pfit.c
+++ b/drivers/gpu/drm/i915/display/intel_pfit.c
@@ -186,6 +186,7 @@ static int pch_panel_fitting(struct intel_crtc_state *crtc_state,
 			     const struct drm_connector_state *conn_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	const struct drm_display_mode *adjusted_mode =
 		&crtc_state->hw.adjusted_mode;
 	int pipe_src_w = drm_rect_width(&crtc_state->pipe_src);
@@ -195,11 +196,22 @@ static int pch_panel_fitting(struct intel_crtc_state *crtc_state,
 	/* Native modes don't need fitting */
 	if (adjusted_mode->crtc_hdisplay == pipe_src_w &&
 	    adjusted_mode->crtc_vdisplay == pipe_src_h &&
-	    crtc_state->output_format != INTEL_OUTPUT_FORMAT_YCBCR420)
+	    crtc_state->output_format != INTEL_OUTPUT_FORMAT_YCBCR420 &&
+	    crtc_state->hw.sharpness_strength == 0)
 		return 0;
 
 	switch (conn_state->scaling_mode) {
 	case DRM_MODE_SCALE_CENTER:
+		if (adjusted_mode->crtc_hdisplay < pipe_src_w ||
+		    adjusted_mode->crtc_vdisplay < pipe_src_h) {
+			drm_dbg_kms(display->drm,
+				    "[CRTC:%d:%s] pfit center mode source (%dx%d) exceeds display (%dx%d)\n",
+				    crtc->base.base.id, crtc->base.name,
+				    pipe_src_w, pipe_src_h,
+				    adjusted_mode->crtc_hdisplay,
+				    adjusted_mode->crtc_vdisplay);
+			return -EINVAL;
+		}
 		width = pipe_src_w;
 		height = pipe_src_h;
 		x = (adjusted_mode->crtc_hdisplay - width + 1)/2;
@@ -249,6 +261,16 @@ static int pch_panel_fitting(struct intel_crtc_state *crtc_state,
 		return -EINVAL;
 	}
 
+	if (crtc_state->hw.sharpness_strength &&
+	    (width != pipe_src_w || height != pipe_src_h ||
+	     crtc_state->hw.scaling_filter != DRM_SCALING_FILTER_DEFAULT ||
+	     crtc_state->output_format != INTEL_OUTPUT_FORMAT_RGB)) {
+		drm_dbg_kms(display->drm,
+			    "[CRTC:%d:%s] no scaling/YCbCr output with sharpness filter\n",
+			    crtc->base.base.id, crtc->base.name);
+		return -EINVAL;
+	}
+
 	drm_rect_init(&crtc_state->pch_pfit.dst,
 		      x, y, width, height);
 	crtc_state->pch_pfit.enabled = true;
diff --git a/drivers/gpu/drm/i915/display/intel_pipe_crc.c b/drivers/gpu/drm/i915/display/intel_pipe_crc.c
index 57586c78582da..5434d7f823d4d 100644
--- a/drivers/gpu/drm/i915/display/intel_pipe_crc.c
+++ b/drivers/gpu/drm/i915/display/intel_pipe_crc.c
@@ -284,7 +284,7 @@ intel_crtc_crc_setup_workarounds(struct intel_crtc *crtc, bool enable)
 {
 	struct intel_display *display = to_intel_display(crtc);
 	struct intel_crtc_state *pipe_config;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_modeset_acquire_ctx ctx;
 	int ret;
 
@@ -293,7 +293,7 @@ intel_crtc_crc_setup_workarounds(struct intel_crtc *crtc, bool enable)
 
 	drm_modeset_acquire_init(&ctx, 0);
 
-	state = drm_atomic_state_alloc(display->drm);
+	state = drm_atomic_commit_alloc(display->drm);
 	if (!state) {
 		ret = -ENOMEM;
 		goto unlock;
@@ -321,12 +321,12 @@ intel_crtc_crc_setup_workarounds(struct intel_crtc *crtc, bool enable)
 
 put_state:
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 		goto retry;
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 unlock:
 	drm_WARN(display->drm, ret,
 		 "Toggling workaround to %i returns %i\n", enable, ret);
diff --git a/drivers/gpu/drm/i915/display/intel_plane.c b/drivers/gpu/drm/i915/display/intel_plane.c
index 5390ceb21ca42..a1f9558d53af2 100644
--- a/drivers/gpu/drm/i915/display/intel_plane.c
+++ b/drivers/gpu/drm/i915/display/intel_plane.c
@@ -44,6 +44,7 @@
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_panic.h>
 #include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
 
 #include "i9xx_plane_regs.h"
 #include "intel_cdclk.h"
@@ -53,7 +54,6 @@
 #include "intel_display_trace.h"
 #include "intel_display_types.h"
 #include "intel_fb.h"
-#include "intel_fb_pin.h"
 #include "intel_fbdev.h"
 #include "intel_parent.h"
 #include "intel_plane.h"
@@ -70,6 +70,7 @@ static void intel_plane_state_reset(struct intel_plane_state *plane_state,
 	__drm_atomic_helper_plane_state_reset(&plane_state->uapi, &plane->base);
 
 	plane_state->scaler_id = -1;
+	plane_state->fence_id = -1;
 }
 
 struct intel_plane *intel_plane_alloc(void)
@@ -137,7 +138,7 @@ intel_plane_duplicate_state(struct drm_plane *plane)
 
 	intel_state->ggtt_vma = NULL;
 	intel_state->dpt_vma = NULL;
-	intel_state->flags = 0;
+	intel_state->fence_id = -1;
 	intel_state->damage = DRM_RECT_INIT(0, 0, 0, 0);
 
 	/* add reference to fb */
@@ -170,6 +171,19 @@ intel_plane_destroy_state(struct drm_plane *plane,
 	kfree(plane_state);
 }
 
+bool intel_plane_needs_low_address(struct intel_display *display)
+{
+	/*
+	 * Valleyview is definitely limited to scanning out the first
+	 * 512MiB. Lets presume this behaviour was inherited from the
+	 * g4x display engine and that all earlier gen are similarly
+	 * limited. Testing suggests that it is a little more
+	 * complicated than this. For example, Cherryview appears quite
+	 * happy to scanout from anywhere within its global aperture.
+	 */
+	return HAS_GMCH(display);
+}
+
 bool intel_plane_needs_physical(struct intel_plane *plane)
 {
 	struct intel_display *display = to_intel_display(plane);
@@ -178,6 +192,15 @@ bool intel_plane_needs_physical(struct intel_plane *plane)
 		DISPLAY_INFO(display)->cursor_needs_physical;
 }
 
+bool intel_plane_needs_fence(struct intel_display *display)
+{
+	/*
+	 * pre-i965 planes use the fence for tiled scanout.
+	 * i965+ planes have their own tiled scanout control bit.
+	 */
+	return DISPLAY_VER(display) < 4;
+}
+
 bool intel_plane_can_async_flip(struct intel_plane *plane,
 				const struct drm_format_info *info,
 				u64 modifier)
@@ -364,7 +387,7 @@ intel_plane_color_copy_uapi_to_hw_state(struct intel_plane_state *plane_state,
 {
 	struct drm_colorop *iter_colorop, *colorop;
 	struct drm_colorop_state *new_colorop_state;
-	struct drm_atomic_state *state = plane_state->uapi.state;
+	struct drm_atomic_commit *state = plane_state->uapi.state;
 	struct intel_colorop *intel_colorop;
 	struct drm_property_blob *blob;
 	struct intel_atomic_state *intel_atomic_state = to_intel_atomic_state(state);
@@ -373,7 +396,7 @@ intel_plane_color_copy_uapi_to_hw_state(struct intel_plane_state *plane_state,
 	bool changed = false;
 	int i = 0;
 
-	iter_colorop = plane_state->uapi.color_pipeline;
+	iter_colorop = from_plane_state->uapi.color_pipeline;
 
 	while (iter_colorop) {
 		for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) {
@@ -1168,6 +1191,122 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state)
 	return 0;
 }
 
+static unsigned int
+intel_plane_fb_min_alignment(const struct intel_plane_state *plane_state)
+{
+	const struct intel_framebuffer *fb = to_intel_framebuffer(plane_state->hw.fb);
+
+	return fb->min_alignment;
+}
+
+static unsigned int
+intel_plane_fb_min_phys_alignment(const struct intel_plane_state *plane_state)
+{
+	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
+	const struct drm_framebuffer *fb = plane_state->hw.fb;
+
+	if (!intel_plane_needs_physical(plane))
+		return 0;
+
+	return plane->min_alignment(plane, fb, 0);
+}
+
+static unsigned int
+intel_plane_fb_vtd_guard(const struct intel_plane_state *plane_state)
+{
+	return intel_fb_view_vtd_guard(plane_state->hw.fb,
+				       &plane_state->view,
+				       plane_state->hw.rotation);
+}
+
+int intel_plane_pin_fb(struct intel_plane_state *plane_state,
+		       const struct intel_plane_state *old_plane_state)
+{
+	struct intel_display *display = to_intel_display(plane_state);
+	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
+	const struct intel_framebuffer *fb =
+		to_intel_framebuffer(plane_state->hw.fb);
+	const struct intel_framebuffer *old_fb =
+		to_intel_framebuffer(old_plane_state->hw.fb);
+	struct i915_vma *ggtt_vma = NULL;
+	struct i915_vma *dpt_vma = NULL;
+	int fence_id = -1;
+	u32 offset = 0;
+	int ret;
+
+	/* hack for xe since it can't keep track of vmas properly */
+	ggtt_vma = intel_parent_fb_pin_reuse_vma(display,
+						 old_plane_state->ggtt_vma,
+						 intel_fb_bo(&old_fb->base),
+						 &old_plane_state->view.gtt,
+						 intel_fb_bo(&fb->base),
+						 &plane_state->view.gtt,
+						 &offset);
+	if (ggtt_vma)
+		goto got_vma;
+
+	if (!intel_fb_uses_dpt(&fb->base)) {
+		struct intel_fb_pin_params pin_params = {
+			.view = &plane_state->view.gtt,
+			.alignment = intel_plane_fb_min_alignment(plane_state),
+			.phys_alignment = intel_plane_fb_min_phys_alignment(plane_state),
+			.vtd_guard = intel_plane_fb_vtd_guard(plane_state),
+			.needs_cpu_lmem_access = intel_fb_needs_cpu_access(&fb->base),
+			.needs_low_address = intel_plane_needs_low_address(display),
+			.needs_physical = intel_plane_needs_physical(plane),
+			.needs_fence = intel_plane_needs_fence(display),
+		};
+
+		ret = intel_parent_fb_pin_ggtt_pin(display, intel_fb_bo(&fb->base),
+						   &pin_params, &ggtt_vma, &offset,
+						   intel_plane_uses_fence(plane_state) ? &fence_id : NULL);
+	} else {
+		struct intel_fb_pin_params pin_params = {
+			.view = &plane_state->view.gtt,
+			.alignment = intel_plane_fb_min_alignment(plane_state),
+			.needs_cpu_lmem_access = intel_fb_needs_cpu_access(&fb->base),
+		};
+
+		ret = intel_parent_fb_pin_dpt_pin(display, intel_fb_bo(&fb->base),
+						  fb->dpt, &pin_params,
+						  &dpt_vma, &ggtt_vma, &offset);
+	}
+	if (ret)
+		return ret;
+
+got_vma:
+	plane_state->dpt_vma = dpt_vma;
+	plane_state->ggtt_vma = ggtt_vma;
+	plane_state->fence_id = fence_id;
+
+	plane_state->surf = offset + plane->surf_offset(plane_state);
+
+	return 0;
+}
+
+void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
+{
+	struct intel_display *display = to_intel_display(old_plane_state);
+	const struct intel_framebuffer *fb =
+		to_intel_framebuffer(old_plane_state->hw.fb);
+
+	if (!intel_fb_uses_dpt(&fb->base)) {
+		intel_parent_fb_pin_ggtt_unpin(display,
+					       old_plane_state->ggtt_vma,
+					       old_plane_state->fence_id);
+
+		old_plane_state->ggtt_vma = NULL;
+		old_plane_state->fence_id = -1;
+	} else {
+		intel_parent_fb_pin_dpt_unpin(display, fb->dpt,
+					      old_plane_state->dpt_vma,
+					      old_plane_state->ggtt_vma);
+
+		old_plane_state->dpt_vma = NULL;
+		old_plane_state->ggtt_vma = NULL;
+	}
+}
+
 static int add_dma_resv_fences(struct dma_resv *resv,
 			       struct drm_plane_state *new_plane_state)
 {
@@ -1381,7 +1520,7 @@ static void intel_panic_flush(struct drm_plane *_plane)
 	if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
 		struct iosys_map map;
 
-		intel_fbdev_get_map(display->fbdev.fbdev, &map);
+		intel_fbdev_get_map(display, &map);
 		drm_clflush_virt_range(map.vaddr, fb->base.pitches[0] * fb->base.height);
 		return;
 	}
@@ -1439,7 +1578,7 @@ static int intel_get_scanout_buffer(struct drm_plane *plane,
 		return -ENODEV;
 
 	if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
-		intel_fbdev_get_map(display->fbdev.fbdev, &sb->map[0]);
+		intel_fbdev_get_map(display, &sb->map[0]);
 	} else {
 		int ret;
 		/* Can't disable tiling if DPT is in use */
diff --git a/drivers/gpu/drm/i915/display/intel_plane.h b/drivers/gpu/drm/i915/display/intel_plane.h
index 5a8f2f3baab5f..a6338bba72d97 100644
--- a/drivers/gpu/drm/i915/display/intel_plane.h
+++ b/drivers/gpu/drm/i915/display/intel_plane.h
@@ -15,6 +15,7 @@ struct drm_rect;
 struct intel_atomic_state;
 struct intel_crtc;
 struct intel_crtc_state;
+struct intel_display;
 struct intel_dsb;
 struct intel_plane;
 struct intel_plane_state;
@@ -79,7 +80,9 @@ int intel_plane_check_src_coordinates(struct intel_plane_state *plane_state);
 void intel_plane_set_invisible(struct intel_crtc_state *crtc_state,
 			       struct intel_plane_state *plane_state);
 void intel_plane_helper_add(struct intel_plane *plane);
+bool intel_plane_needs_low_address(struct intel_display *display);
 bool intel_plane_needs_physical(struct intel_plane *plane);
+bool intel_plane_needs_fence(struct intel_display *display);
 void intel_plane_init_cursor_vblank_work(struct intel_plane_state *old_plane_state,
 					 struct intel_plane_state *new_plane_state);
 int intel_plane_add_affected(struct intel_atomic_state *state,
@@ -89,5 +92,8 @@ int intel_plane_atomic_check(struct intel_atomic_state *state);
 bool intel_plane_format_mod_supported_async(struct drm_plane *plane,
 					    u32 format,
 					    u64 modifier);
+int intel_plane_pin_fb(struct intel_plane_state *new_plane_state,
+		       const struct intel_plane_state *old_plane_state);
+void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state);
 
 #endif /* __INTEL_PLANE_H__ */
diff --git a/drivers/gpu/drm/i915/display/intel_pmdemand.c b/drivers/gpu/drm/i915/display/intel_pmdemand.c
index 244806a26da39..7819b724795bb 100644
--- a/drivers/gpu/drm/i915/display/intel_pmdemand.c
+++ b/drivers/gpu/drm/i915/display/intel_pmdemand.c
@@ -6,6 +6,7 @@
 #include <linux/bitops.h>
 
 #include <drm/drm_print.h>
+#include <drm/intel/step.h>
 
 #include "intel_atomic.h"
 #include "intel_bw.h"
@@ -17,7 +18,6 @@
 #include "intel_display_utils.h"
 #include "intel_display_wa.h"
 #include "intel_pmdemand.h"
-#include "intel_step.h"
 #include "skl_watermark.h"
 
 struct pmdemand_params {
diff --git a/drivers/gpu/drm/i915/display/intel_pps.c b/drivers/gpu/drm/i915/display/intel_pps.c
index 2d799af73bb7f..d4c98b150fa2b 100644
--- a/drivers/gpu/drm/i915/display/intel_pps.c
+++ b/drivers/gpu/drm/i915/display/intel_pps.c
@@ -488,11 +488,11 @@ void bxt_pps_reset_all(struct intel_display *display)
 }
 
 struct pps_registers {
-	i915_reg_t pp_ctrl;
-	i915_reg_t pp_stat;
-	i915_reg_t pp_on;
-	i915_reg_t pp_off;
-	i915_reg_t pp_div;
+	intel_reg_t pp_ctrl;
+	intel_reg_t pp_stat;
+	intel_reg_t pp_on;
+	intel_reg_t pp_off;
+	intel_reg_t pp_div;
 };
 
 static void intel_pps_get_registers(struct intel_dp *intel_dp,
@@ -523,7 +523,7 @@ static void intel_pps_get_registers(struct intel_dp *intel_dp,
 		regs->pp_div = PP_DIVISOR(display, pps_idx);
 }
 
-static i915_reg_t
+static intel_reg_t
 _pp_ctrl_reg(struct intel_dp *intel_dp)
 {
 	struct pps_registers regs;
@@ -533,7 +533,7 @@ _pp_ctrl_reg(struct intel_dp *intel_dp)
 	return regs.pp_ctrl;
 }
 
-static i915_reg_t
+static intel_reg_t
 _pp_stat_reg(struct intel_dp *intel_dp)
 {
 	struct pps_registers regs;
@@ -607,7 +607,7 @@ static void wait_panel_status(struct intel_dp *intel_dp,
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-	i915_reg_t pp_stat_reg, pp_ctrl_reg;
+	intel_reg_t pp_stat_reg, pp_ctrl_reg;
 	int ret;
 	u32 val;
 
@@ -744,7 +744,7 @@ bool intel_pps_vdd_on_unlocked(struct intel_dp *intel_dp)
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	u32 pp;
-	i915_reg_t pp_stat_reg, pp_ctrl_reg;
+	intel_reg_t pp_stat_reg, pp_ctrl_reg;
 	bool need_to_disable = !intel_dp->pps.want_panel_vdd;
 
 	if (!intel_dp_is_edp(intel_dp))
@@ -826,7 +826,7 @@ static void intel_pps_vdd_off_sync_unlocked(struct intel_dp *intel_dp)
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	u32 pp;
-	i915_reg_t pp_stat_reg, pp_ctrl_reg;
+	intel_reg_t pp_stat_reg, pp_ctrl_reg;
 
 	lockdep_assert_held(&display->pps.mutex);
 
@@ -955,7 +955,7 @@ void intel_pps_on_unlocked(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	u32 pp;
-	i915_reg_t pp_ctrl_reg;
+	intel_reg_t pp_ctrl_reg;
 
 	lockdep_assert_held(&display->pps.mutex);
 
@@ -1028,7 +1028,7 @@ void intel_pps_off_unlocked(struct intel_dp *intel_dp)
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	u32 pp;
-	i915_reg_t pp_ctrl_reg;
+	intel_reg_t pp_ctrl_reg;
 
 	lockdep_assert_held(&display->pps.mutex);
 
@@ -1091,7 +1091,7 @@ void intel_pps_backlight_on(struct intel_dp *intel_dp)
 	wait_backlight_on(intel_dp);
 
 	with_intel_pps_lock(intel_dp) {
-		i915_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
+		intel_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
 		u32 pp;
 
 		pp = ilk_get_pp_control(intel_dp);
@@ -1111,7 +1111,7 @@ void intel_pps_backlight_off(struct intel_dp *intel_dp)
 		return;
 
 	with_intel_pps_lock(intel_dp) {
-		i915_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
+		intel_reg_t pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
 		u32 pp;
 
 		pp = ilk_get_pp_control(intel_dp);
@@ -1155,7 +1155,7 @@ static void vlv_detach_power_sequencer(struct intel_dp *intel_dp)
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
 	enum pipe pipe = intel_dp->pps.vlv_pps_pipe;
-	i915_reg_t pp_on_reg = PP_ON_DELAYS(display, pipe);
+	intel_reg_t pp_on_reg = PP_ON_DELAYS(display, pipe);
 
 	drm_WARN_ON(display->drm, intel_dp->pps.vlv_active_pipe != INVALID_PIPE);
 
@@ -1388,7 +1388,7 @@ intel_pps_readout_hw_state(struct intel_dp *intel_dp, struct intel_pps_delays *s
 	seq->backlight_off = REG_FIELD_GET(PANEL_LIGHT_OFF_DELAY_MASK, pp_off);
 	seq->power_down = REG_FIELD_GET(PANEL_POWER_DOWN_DELAY_MASK, pp_off);
 
-	if (i915_mmio_reg_valid(regs.pp_div)) {
+	if (intel_reg_valid(regs.pp_div)) {
 		u32 pp_div;
 
 		pp_div = intel_de_read(display, regs.pp_div);
@@ -1647,7 +1647,7 @@ static void pps_init_registers(struct intel_dp *intel_dp, bool force_disable_vdd
 	/*
 	 * Compute the divisor for the pp clock, simply match the Bspec formula.
 	 */
-	if (i915_mmio_reg_valid(regs.pp_div))
+	if (intel_reg_valid(regs.pp_div))
 		intel_de_write(display, regs.pp_div,
 			       REG_FIELD_PREP(PP_REFERENCE_DIVIDER_MASK,
 					      (100 * div) / 2 - 1) |
@@ -1662,7 +1662,7 @@ static void pps_init_registers(struct intel_dp *intel_dp, bool force_disable_vdd
 		    "panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n",
 		    intel_de_read(display, regs.pp_on),
 		    intel_de_read(display, regs.pp_off),
-		    i915_mmio_reg_valid(regs.pp_div) ?
+		    intel_reg_valid(regs.pp_div) ?
 		    intel_de_read(display, regs.pp_div) :
 		    (intel_de_read(display, regs.pp_ctrl) & BXT_POWER_CYCLE_DELAY_MASK));
 }
@@ -1814,7 +1814,7 @@ void intel_pps_connector_debugfs_add(struct intel_connector *connector)
 
 void assert_pps_unlocked(struct intel_display *display, enum pipe pipe)
 {
-	i915_reg_t pp_reg;
+	intel_reg_t pp_reg;
 	u32 val;
 	enum pipe panel_pipe = INVALID_PIPE;
 	bool locked = true;
diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c
index 53c10ae76ab52..b0414bd1dc6b1 100644
--- a/drivers/gpu/drm/i915/display/intel_psr.c
+++ b/drivers/gpu/drm/i915/display/intel_psr.c
@@ -28,6 +28,7 @@
 #include <drm/drm_debugfs.h>
 #include <drm/drm_print.h>
 #include <drm/drm_vblank.h>
+#include <drm/intel/step.h>
 
 #include "intel_alpm.h"
 #include "intel_atomic.h"
@@ -43,7 +44,9 @@
 #include "intel_display_wa.h"
 #include "intel_dmc.h"
 #include "intel_dp.h"
+#include "intel_dpcd.h"
 #include "intel_dp_aux.h"
+#include "intel_dp_tunnel.h"
 #include "intel_dsb.h"
 #include "intel_frontbuffer.h"
 #include "intel_hdmi.h"
@@ -51,7 +54,6 @@
 #include "intel_psr_regs.h"
 #include "intel_quirks.h"
 #include "intel_snps_phy.h"
-#include "intel_step.h"
 #include "intel_vblank.h"
 #include "intel_vdsc.h"
 #include "intel_vrr.h"
@@ -302,8 +304,8 @@ static u32 psr_irq_mask_get(struct intel_dp *intel_dp)
 		EDP_PSR_MASK(intel_dp->psr.transcoder);
 }
 
-static i915_reg_t psr_ctl_reg(struct intel_display *display,
-			      enum transcoder cpu_transcoder)
+static intel_reg_t psr_ctl_reg(struct intel_display *display,
+			       enum transcoder cpu_transcoder)
 {
 	if (DISPLAY_VER(display) >= 8)
 		return EDP_PSR_CTL(display, cpu_transcoder);
@@ -311,8 +313,8 @@ static i915_reg_t psr_ctl_reg(struct intel_display *display,
 		return HSW_SRD_CTL;
 }
 
-static i915_reg_t psr_debug_reg(struct intel_display *display,
-				enum transcoder cpu_transcoder)
+static intel_reg_t psr_debug_reg(struct intel_display *display,
+				 enum transcoder cpu_transcoder)
 {
 	if (DISPLAY_VER(display) >= 8)
 		return EDP_PSR_DEBUG(display, cpu_transcoder);
@@ -320,8 +322,8 @@ static i915_reg_t psr_debug_reg(struct intel_display *display,
 		return HSW_SRD_DEBUG;
 }
 
-static i915_reg_t psr_perf_cnt_reg(struct intel_display *display,
-				   enum transcoder cpu_transcoder)
+static intel_reg_t psr_perf_cnt_reg(struct intel_display *display,
+				    enum transcoder cpu_transcoder)
 {
 	if (DISPLAY_VER(display) >= 8)
 		return EDP_PSR_PERF_CNT(display, cpu_transcoder);
@@ -329,8 +331,8 @@ static i915_reg_t psr_perf_cnt_reg(struct intel_display *display,
 		return HSW_SRD_PERF_CNT;
 }
 
-static i915_reg_t psr_status_reg(struct intel_display *display,
-				 enum transcoder cpu_transcoder)
+static intel_reg_t psr_status_reg(struct intel_display *display,
+				  enum transcoder cpu_transcoder)
 {
 	if (DISPLAY_VER(display) >= 8)
 		return EDP_PSR_STATUS(display, cpu_transcoder);
@@ -338,8 +340,8 @@ static i915_reg_t psr_status_reg(struct intel_display *display,
 		return HSW_SRD_STATUS;
 }
 
-static i915_reg_t psr_imr_reg(struct intel_display *display,
-			      enum transcoder cpu_transcoder)
+static intel_reg_t psr_imr_reg(struct intel_display *display,
+			       enum transcoder cpu_transcoder)
 {
 	if (DISPLAY_VER(display) >= 12)
 		return TRANS_PSR_IMR(display, cpu_transcoder);
@@ -347,8 +349,8 @@ static i915_reg_t psr_imr_reg(struct intel_display *display,
 		return EDP_PSR_IMR;
 }
 
-static i915_reg_t psr_iir_reg(struct intel_display *display,
-			      enum transcoder cpu_transcoder)
+static intel_reg_t psr_iir_reg(struct intel_display *display,
+			       enum transcoder cpu_transcoder)
 {
 	if (DISPLAY_VER(display) >= 12)
 		return TRANS_PSR_IIR(display, cpu_transcoder);
@@ -356,8 +358,8 @@ static i915_reg_t psr_iir_reg(struct intel_display *display,
 		return EDP_PSR_IIR;
 }
 
-static i915_reg_t psr_aux_ctl_reg(struct intel_display *display,
-				  enum transcoder cpu_transcoder)
+static intel_reg_t psr_aux_ctl_reg(struct intel_display *display,
+				   enum transcoder cpu_transcoder)
 {
 	if (DISPLAY_VER(display) >= 8)
 		return EDP_PSR_AUX_CTL(display, cpu_transcoder);
@@ -365,8 +367,8 @@ static i915_reg_t psr_aux_ctl_reg(struct intel_display *display,
 		return HSW_SRD_AUX_CTL;
 }
 
-static i915_reg_t psr_aux_data_reg(struct intel_display *display,
-				   enum transcoder cpu_transcoder, int i)
+static intel_reg_t psr_aux_data_reg(struct intel_display *display,
+				    enum transcoder cpu_transcoder, int i)
 {
 	if (DISPLAY_VER(display) >= 8)
 		return EDP_PSR_AUX_DATA(display, cpu_transcoder, i);
@@ -694,6 +696,9 @@ static void _psr_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *co
 
 	connector->dp.psr_caps.sync_latency = intel_dp_get_sink_sync_latency(intel_dp);
 
+	if (intel_has_quirk(display, QUIRK_DISABLE_PSR2))
+		return;
+
 	if (DISPLAY_VER(display) >= 9 &&
 	    connector->dp.psr_caps.dpcd[0] >= DP_PSR2_WITH_Y_COORD_IS_SUPPORTED) {
 		bool y_req = connector->dp.psr_caps.dpcd[1] &
@@ -716,8 +721,14 @@ static void _psr_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *co
 			    connector->dp.psr_caps.su_support ? "" : "not ");
 	}
 
-	if (connector->dp.psr_caps.su_support)
+	if (connector->dp.psr_caps.su_support) {
+		ret = drm_dp_dpcd_read_byte(&intel_dp->aux,
+					    INTEL_DPCD_INTEL_WA_REGISTER_CAPS,
+					    &connector->dp.psr_caps.intel_wa_dpcd);
+		if (ret < 0)
+			return;
 		_psr_compute_su_granularity(intel_dp, connector);
+	}
 }
 
 void intel_psr_init_dpcd(struct intel_dp *intel_dp, struct intel_connector *connector)
@@ -1031,11 +1042,27 @@ static u8 frames_before_su_entry(struct intel_dp *intel_dp)
 	return frames_before_su_entry;
 }
 
+static bool intel_psr_allow_pr_bw_optimization(struct intel_dp *intel_dp)
+{
+	if (intel_dp_is_edp(intel_dp))
+		return false;
+
+	if (!intel_dp_tunnel_bw_alloc_is_enabled(intel_dp))
+		return false;
+
+	if (!intel_dp_tunnel_pr_optimization_supported(intel_dp))
+		return false;
+
+	return true;
+}
+
 static void dg2_activate_panel_replay(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	struct intel_psr *psr = &intel_dp->psr;
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
+	u32 dp2_ctl_set = TRANS_DP2_PANEL_REPLAY_ENABLE;
+	u32 dp2_ctl_clear = 0;
 
 	if (intel_dp_is_edp(intel_dp) && psr->sel_update_enabled) {
 		u32 val = psr->su_region_et_enabled ?
@@ -1048,12 +1075,16 @@ static void dg2_activate_panel_replay(struct intel_dp *intel_dp)
 			       val);
 	}
 
+	if (intel_psr_allow_pr_bw_optimization(intel_dp))
+		dp2_ctl_set |= TRANS_DP2_PR_TUNNELING_ENABLE;
+	else
+		dp2_ctl_clear = TRANS_DP2_PR_TUNNELING_ENABLE;
+
 	intel_de_rmw(display,
 		     PSR2_MAN_TRK_CTL(display, intel_dp->psr.transcoder),
 		     0, ADLP_PSR2_MAN_TRK_CTL_SF_CONTINUOS_FULL_FRAME);
 
-	intel_de_rmw(display, TRANS_DP2_CTL(intel_dp->psr.transcoder), 0,
-		     TRANS_DP2_PANEL_REPLAY_ENABLE);
+	intel_de_rmw(display, TRANS_DP2_CTL(intel_dp->psr.transcoder), dp2_ctl_clear, dp2_ctl_set);
 }
 
 static void hsw_activate_psr2(struct intel_dp *intel_dp)
@@ -1358,9 +1389,35 @@ static bool psr2_granularity_check(struct intel_crtc_state *crtc_state,
 	return true;
 }
 
-static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_dp,
-							struct intel_crtc_state *crtc_state)
+static bool apply_scanline_indication_wa(struct intel_crtc_state *crtc_state,
+					 struct intel_connector *connector)
+{
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
+	u8 early_scanline_support = connector->dp.psr_caps.intel_wa_dpcd &
+		INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_EARLYSCANLINE_SDP_SUPPORT_MASK;
+
+	if (intel_dp->edp_dpcd[0] >= DP_EDP_15)
+		return true;
+
+	switch (early_scanline_support)	{
+	case INTEL_DPCD_INTEL_WA_REGISTER_CAPS_FALL_BACK_TO_PSR1:
+		crtc_state->req_psr2_sdp_prior_scanline = false;
+		return false;
+	case INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_WITH_EARLY_SCANLINE:
+		return true;
+	case INTEL_DPCD_INTEL_WA_REGISTER_CAPS_PSR2_WITHOUT_EARLY_SCANLINE:
+		crtc_state->req_psr2_sdp_prior_scanline = false;
+		return true;
+	default:
+		MISSING_CASE(early_scanline_support);
+		return false;
+	}
+}
+
+static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_crtc_state *crtc_state,
+							struct intel_connector *connector)
 {
+	struct intel_dp *intel_dp = intel_attached_dp(connector);
 	struct intel_display *display = to_intel_display(intel_dp);
 	const struct drm_display_mode *adjusted_mode = &crtc_state->uapi.adjusted_mode;
 	u32 hblank_total, hblank_ns, req_ns;
@@ -1379,7 +1436,8 @@ static bool _compute_psr2_sdp_prior_scanline_indication(struct intel_dp *intel_d
 		return false;
 
 	crtc_state->req_psr2_sdp_prior_scanline = true;
-	return true;
+
+	return apply_scanline_indication_wa(crtc_state, connector);
 }
 
 static int intel_psr_entry_setup_frames(struct intel_dp *intel_dp,
@@ -1660,7 +1718,7 @@ static bool intel_sel_update_config_valid(struct intel_crtc_state *crtc_state,
 								      conn_state))
 		goto unsupported;
 
-	if (!_compute_psr2_sdp_prior_scanline_indication(intel_dp, crtc_state)) {
+	if (!_compute_psr2_sdp_prior_scanline_indication(crtc_state, connector)) {
 		drm_dbg_kms(display->drm,
 			    "Selective update not enabled, SDP indication do not fit in hblank\n");
 		goto unsupported;
@@ -2295,7 +2353,7 @@ static void intel_psr_wait_exit_locked(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
-	i915_reg_t psr_status;
+	intel_reg_t psr_status;
 	u32 psr_status_mask;
 
 	if (intel_dp_is_edp(intel_dp) && (intel_dp->psr.sel_update_enabled ||
@@ -2855,6 +2913,11 @@ intel_psr_apply_su_area_workarounds(struct intel_crtc_state *crtc_state)
 	    crtc_state->splitter.enable)
 		crtc_state->psr2_su_area.y1 = 0;
 
+	if (intel_display_wa(display, INTEL_DISPLAY_WA_16029024088) &&
+	    crtc_state->req_psr2_sdp_prior_scanline &&
+	    !crtc_state->enable_psr2_su_region_et)
+		crtc_state->psr2_su_area.y1 = 0;
+
 	/* Wa 14019834836 */
 	if (DISPLAY_VER(display) == 30)
 		intel_psr_apply_pr_link_on_su_wa(crtc_state);
@@ -3326,7 +3389,7 @@ static bool __psr_wait_for_idle_locked(struct intel_dp *intel_dp)
 {
 	struct intel_display *display = to_intel_display(intel_dp);
 	enum transcoder cpu_transcoder = intel_dp->psr.transcoder;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 mask;
 	int err;
 
@@ -3358,11 +3421,11 @@ static int intel_psr_fastset_force(struct intel_display *display)
 {
 	struct drm_connector_list_iter conn_iter;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_connector *conn;
 	int err = 0;
 
-	state = drm_atomic_state_alloc(display->drm);
+	state = drm_atomic_commit_alloc(display->drm);
 	if (!state)
 		return -ENOMEM;
 
@@ -3404,7 +3467,7 @@ static int intel_psr_fastset_force(struct intel_display *display)
 		err = drm_atomic_commit(state);
 
 	if (err == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		err = drm_modeset_backoff(&ctx);
 		if (!err)
 			goto retry;
@@ -3412,7 +3475,7 @@ static int intel_psr_fastset_force(struct intel_display *display)
 
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	return err;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_quirks.c b/drivers/gpu/drm/i915/display/intel_quirks.c
index 883f297d4b837..33245f44c0d50 100644
--- a/drivers/gpu/drm/i915/display/intel_quirks.c
+++ b/drivers/gpu/drm/i915/display/intel_quirks.c
@@ -94,6 +94,12 @@ static void quirk_disable_edp_panel_replay(struct intel_dp *intel_dp)
 	drm_info(display->drm, "Applying disable Panel Replay quirk\n");
 }
 
+static void quirk_disable_psr2(struct intel_display *display)
+{
+	intel_set_quirk(display, QUIRK_DISABLE_PSR2);
+	drm_info(display->drm, "PSR2 support not currently available for this setup, applying disable PSR2 quirk\n");
+}
+
 struct intel_quirk {
 	int device;
 	int subsystem_vendor;
@@ -250,6 +256,9 @@ static struct intel_quirk intel_quirks[] = {
 
 	/* Dell XPS 13 7390 2-in-1 */
 	{ 0x8a52, 0x1028, 0x08b0, quirk_edp_limit_rate_hbr2 },
+
+	/* Xiaomi Book Pro 14 2026 */
+	{ 0xb081, 0x1d72, 0x2424, quirk_disable_psr2 },
 };
 
 static const struct intel_dpcd_quirk intel_dpcd_quirks[] = {
@@ -269,6 +278,14 @@ static const struct intel_dpcd_quirk intel_dpcd_quirks[] = {
 		.sink_oui = SINK_OUI(0x00, 0x22, 0xb9),
 		.hook = quirk_disable_edp_panel_replay,
 	},
+	/* Dell XPS 16 DA16260 */
+	{
+		.device = DEVICE_ID_ANY,
+		.subsystem_vendor = 0x1028,
+		.subsystem_device = 0x0dba,
+		.sink_oui = SINK_OUI(0x00, 0x22, 0xb9),
+		.hook = quirk_disable_edp_panel_replay,
+	},
 };
 
 void intel_init_quirks(struct intel_display *display)
diff --git a/drivers/gpu/drm/i915/display/intel_quirks.h b/drivers/gpu/drm/i915/display/intel_quirks.h
index 83214eb94b0c4..970a4fe52fafc 100644
--- a/drivers/gpu/drm/i915/display/intel_quirks.h
+++ b/drivers/gpu/drm/i915/display/intel_quirks.h
@@ -22,6 +22,7 @@ enum intel_quirk_id {
 	QUIRK_FW_SYNC_LEN,
 	QUIRK_EDP_LIMIT_RATE_HBR2,
 	QUIRK_DISABLE_EDP_PANEL_REPLAY,
+	QUIRK_DISABLE_PSR2,
 };
 
 void intel_init_quirks(struct intel_display *display);
diff --git a/drivers/gpu/drm/i915/display/intel_rom.c b/drivers/gpu/drm/i915/display/intel_rom.c
index d573059fb0d99..54f842c09fb06 100644
--- a/drivers/gpu/drm/i915/display/intel_rom.c
+++ b/drivers/gpu/drm/i915/display/intel_rom.c
@@ -7,8 +7,9 @@
 
 #include <drm/drm_device.h>
 
+#include "intel_de.h"
+#include "intel_display_types.h"
 #include "intel_rom.h"
-#include "intel_uncore.h"
 #include "intel_oprom_regs.h"
 
 struct intel_rom {
@@ -17,7 +18,7 @@ struct intel_rom {
 	void __iomem *oprom;
 
 	/* for SPI */
-	struct intel_uncore *uncore;
+	struct intel_display *display;
 	loff_t offset;
 
 	size_t size;
@@ -30,10 +31,10 @@ struct intel_rom {
 
 static u32 spi_read32(struct intel_rom *rom, loff_t offset)
 {
-	intel_uncore_write(rom->uncore, PRIMARY_SPI_ADDRESS,
-			   rom->offset + offset);
+	intel_de_write(rom->display, PRIMARY_SPI_ADDRESS,
+		       rom->offset + offset);
 
-	return intel_uncore_read(rom->uncore, PRIMARY_SPI_TRIGGER);
+	return intel_de_read(rom->display, PRIMARY_SPI_TRIGGER);
 }
 
 static u16 spi_read16(struct intel_rom *rom, loff_t offset)
@@ -50,13 +51,13 @@ struct intel_rom *intel_rom_spi(struct drm_device *drm)
 	if (!rom)
 		return NULL;
 
-	rom->uncore = to_intel_uncore(drm);
+	rom->display = to_intel_display(drm);
 
-	static_region = intel_uncore_read(rom->uncore, SPI_STATIC_REGIONS);
+	static_region = intel_de_read(rom->display, SPI_STATIC_REGIONS);
 	static_region &= OPTIONROM_SPI_REGIONID_MASK;
-	intel_uncore_write(rom->uncore, PRIMARY_SPI_REGIONID, static_region);
+	intel_de_write(rom->display, PRIMARY_SPI_REGIONID, static_region);
 
-	rom->offset = intel_uncore_read(rom->uncore, OROM_OFFSET) & OROM_OFFSET_MASK;
+	rom->offset = intel_de_read(rom->display, OROM_OFFSET) & OROM_OFFSET_MASK;
 
 	rom->size = 0x200000;
 
diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.c b/drivers/gpu/drm/i915/display/intel_sdvo.c
index 2e1af9e869ded..d83d350959d88 100644
--- a/drivers/gpu/drm/i915/display/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/display/intel_sdvo.c
@@ -101,7 +101,7 @@ struct intel_sdvo {
 	struct intel_sdvo_ddc ddc[3];
 
 	/* Register for the SDVO device: SDVOB or SDVOC */
-	i915_reg_t sdvo_reg;
+	intel_reg_t sdvo_reg;
 
 	/*
 	 * Capabilities of the SDVO device returned by
@@ -1395,8 +1395,7 @@ static int intel_sdvo_compute_config(struct intel_encoder *encoder,
 							   adjusted_mode);
 		pipe_config->sdvo_tv_clock = true;
 	} else if (IS_LVDS(intel_sdvo_connector)) {
-		const struct drm_display_mode *fixed_mode =
-			intel_panel_fixed_mode(&intel_sdvo_connector->base, mode);
+		const struct drm_display_mode *fixed_mode;
 		int ret;
 
 		ret = intel_panel_compute_config(&intel_sdvo_connector->base,
@@ -1404,6 +1403,8 @@ static int intel_sdvo_compute_config(struct intel_encoder *encoder,
 		if (ret)
 			return ret;
 
+		fixed_mode = &pipe_config->hw.adjusted_mode;
+
 		if (!intel_sdvo_set_output_timings_from_mode(intel_sdvo,
 							     intel_sdvo_connector,
 							     fixed_mode))
@@ -1665,7 +1666,7 @@ static bool intel_sdvo_connector_get_hw_state(struct intel_connector *connector)
 }
 
 bool intel_sdvo_port_enabled(struct intel_display *display,
-			     i915_reg_t sdvo_reg, enum pipe *pipe)
+			     intel_reg_t sdvo_reg, enum pipe *pipe)
 {
 	u32 val;
 
@@ -1967,10 +1968,14 @@ intel_sdvo_mode_valid(struct drm_connector *connector,
 
 	if (IS_LVDS(intel_sdvo_connector)) {
 		enum drm_mode_status status;
+		int target_clock;
 
-		status = intel_panel_mode_valid(&intel_sdvo_connector->base, mode);
+		status = intel_panel_mode_valid(&intel_sdvo_connector->base, mode, &target_clock);
 		if (status != MODE_OK)
 			return status;
+
+		if (target_clock > max_dotclk)
+			return MODE_CLOCK_HIGH;
 	}
 
 	return MODE_OK;
@@ -2512,7 +2517,7 @@ static const struct drm_connector_funcs intel_sdvo_connector_funcs = {
 };
 
 static int intel_sdvo_atomic_check(struct drm_connector *conn,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *new_conn_state =
 		drm_atomic_get_new_connector_state(state, conn);
@@ -3314,7 +3319,7 @@ static void proxy_lock_bus(struct i2c_adapter *adapter,
 	struct intel_sdvo_ddc *ddc = adapter->algo_data;
 	struct intel_sdvo *sdvo = ddc->sdvo;
 
-	sdvo->i2c->lock_ops->lock_bus(sdvo->i2c, flags);
+	i2c_lock_bus(sdvo->i2c, flags);
 }
 
 static int proxy_trylock_bus(struct i2c_adapter *adapter,
@@ -3323,7 +3328,7 @@ static int proxy_trylock_bus(struct i2c_adapter *adapter,
 	struct intel_sdvo_ddc *ddc = adapter->algo_data;
 	struct intel_sdvo *sdvo = ddc->sdvo;
 
-	return sdvo->i2c->lock_ops->trylock_bus(sdvo->i2c, flags);
+	return i2c_trylock_bus(sdvo->i2c, flags);
 }
 
 static void proxy_unlock_bus(struct i2c_adapter *adapter,
@@ -3332,7 +3337,7 @@ static void proxy_unlock_bus(struct i2c_adapter *adapter,
 	struct intel_sdvo_ddc *ddc = adapter->algo_data;
 	struct intel_sdvo *sdvo = ddc->sdvo;
 
-	sdvo->i2c->lock_ops->unlock_bus(sdvo->i2c, flags);
+	i2c_unlock_bus(sdvo->i2c, flags);
 }
 
 static const struct i2c_lock_operations proxy_lock_ops = {
@@ -3377,7 +3382,7 @@ static bool assert_sdvo_port_valid(struct intel_display *display, enum port port
 }
 
 bool intel_sdvo_init(struct intel_display *display,
-		     i915_reg_t sdvo_reg, enum port port)
+		     intel_reg_t sdvo_reg, enum port port)
 {
 	struct intel_encoder *intel_encoder;
 	struct intel_sdvo *intel_sdvo;
diff --git a/drivers/gpu/drm/i915/display/intel_sdvo.h b/drivers/gpu/drm/i915/display/intel_sdvo.h
index 1a9e40fdd8a8f..d1e1083de0373 100644
--- a/drivers/gpu/drm/i915/display/intel_sdvo.h
+++ b/drivers/gpu/drm/i915/display/intel_sdvo.h
@@ -8,7 +8,7 @@
 
 #include <linux/types.h>
 
-#include "i915_reg_defs.h"
+#include "intel_display_reg_defs.h"
 
 enum pipe;
 enum port;
@@ -16,17 +16,17 @@ struct intel_display;
 
 #ifdef I915
 bool intel_sdvo_port_enabled(struct intel_display *display,
-			     i915_reg_t sdvo_reg, enum pipe *pipe);
+			     intel_reg_t sdvo_reg, enum pipe *pipe);
 bool intel_sdvo_init(struct intel_display *display,
-		     i915_reg_t reg, enum port port);
+		     intel_reg_t reg, enum port port);
 #else
 static inline bool intel_sdvo_port_enabled(struct intel_display *display,
-					   i915_reg_t sdvo_reg, enum pipe *pipe)
+					   intel_reg_t sdvo_reg, enum pipe *pipe)
 {
 	return false;
 }
 static inline bool intel_sdvo_init(struct intel_display *display,
-				   i915_reg_t reg, enum port port)
+				   intel_reg_t reg, enum port port)
 {
 	return false;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_snps_phy.c b/drivers/gpu/drm/i915/display/intel_snps_phy.c
index 295030742294b..bf9df566630f0 100644
--- a/drivers/gpu/drm/i915/display/intel_snps_phy.c
+++ b/drivers/gpu/drm/i915/display/intel_snps_phy.c
@@ -72,7 +72,7 @@ void intel_snps_phy_set_signal_levels(struct intel_encoder *encoder,
 	enum phy phy = intel_encoder_to_phy(encoder);
 	int n_entries, ln;
 
-	trans = encoder->get_buf_trans(encoder, crtc_state, &n_entries);
+	trans = intel_ddi_buf_trans_get(encoder, crtc_state, &n_entries);
 	if (drm_WARN_ON_ONCE(display->drm, !trans))
 		return;
 
@@ -1822,7 +1822,7 @@ void intel_mpllb_enable(struct intel_encoder *encoder,
 	struct intel_display *display = to_intel_display(encoder);
 	const struct intel_mpllb_state *pll_state = &crtc_state->dpll_hw_state.mpllb;
 	enum phy phy = intel_encoder_to_phy(encoder);
-	i915_reg_t enable_reg = (phy <= PHY_D ?
+	intel_reg_t enable_reg = (phy <= PHY_D ?
 				 DG2_PLL_ENABLE(phy) : MG_PLL_ENABLE(0));
 
 	/*
@@ -1879,7 +1879,7 @@ void intel_mpllb_disable(struct intel_encoder *encoder)
 {
 	struct intel_display *display = to_intel_display(encoder);
 	enum phy phy = intel_encoder_to_phy(encoder);
-	i915_reg_t enable_reg = (phy <= PHY_D ?
+	intel_reg_t enable_reg = (phy <= PHY_D ?
 				 DG2_PLL_ENABLE(phy) : MG_PLL_ENABLE(0));
 
 	/*
diff --git a/drivers/gpu/drm/i915/display/intel_sprite_uapi.c b/drivers/gpu/drm/i915/display/intel_sprite_uapi.c
index 4981cc34da05f..a62146133f990 100644
--- a/drivers/gpu/drm/i915/display/intel_sprite_uapi.c
+++ b/drivers/gpu/drm/i915/display/intel_sprite_uapi.c
@@ -46,7 +46,7 @@ int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data,
 	struct drm_intel_sprite_colorkey *set = data;
 	struct drm_plane *plane;
 	struct drm_plane_state *plane_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_modeset_acquire_ctx ctx;
 	int ret = 0;
 
@@ -80,7 +80,7 @@ int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data,
 
 	drm_modeset_acquire_init(&ctx, 0);
 
-	state = drm_atomic_state_alloc(plane->dev);
+	state = drm_atomic_commit_alloc(plane->dev);
 	if (!state) {
 		ret = -ENOMEM;
 		goto out;
@@ -116,11 +116,11 @@ int intel_sprite_set_colorkey_ioctl(struct drm_device *dev, void *data,
 		if (ret != -EDEADLK)
 			break;
 
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 out:
 	drm_modeset_drop_locks(&ctx);
 	drm_modeset_acquire_fini(&ctx);
diff --git a/drivers/gpu/drm/i915/display/intel_tc.c b/drivers/gpu/drm/i915/display/intel_tc.c
index a21dd4e3fe4ca..2880f2aa42437 100644
--- a/drivers/gpu/drm/i915/display/intel_tc.c
+++ b/drivers/gpu/drm/i915/display/intel_tc.c
@@ -294,7 +294,7 @@ get_pin_assignment(struct intel_tc_port *tc)
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	enum tc_port tc_port = intel_encoder_to_tc(&tc->dig_port->base);
 	enum intel_tc_pin_assignment pin_assignment;
-	i915_reg_t reg;
+	intel_reg_t reg;
 	u32 mask;
 	u32 val;
 
@@ -1034,7 +1034,7 @@ xelpdp_tc_phy_tcss_power_is_enabled(struct intel_tc_port *tc)
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	enum port port = tc->dig_port->base.port;
-	i915_reg_t reg = XELPDP_PORT_BUF_CTL1(display, port);
+	intel_reg_t reg = XELPDP_PORT_BUF_CTL1(display, port);
 
 	assert_tc_cold_blocked(tc);
 
@@ -1094,7 +1094,7 @@ static void __xelpdp_tc_phy_enable_tcss_power(struct intel_tc_port *tc, bool ena
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	enum port port = tc->dig_port->base.port;
-	i915_reg_t reg = XELPDP_PORT_BUF_CTL1(display, port);
+	intel_reg_t reg = XELPDP_PORT_BUF_CTL1(display, port);
 	u32 val;
 
 	assert_tc_cold_blocked(tc);
@@ -1141,7 +1141,7 @@ static void xelpdp_tc_phy_take_ownership(struct intel_tc_port *tc, bool take)
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	enum port port = tc->dig_port->base.port;
-	i915_reg_t reg = XELPDP_PORT_BUF_CTL1(display, port);
+	intel_reg_t reg = XELPDP_PORT_BUF_CTL1(display, port);
 	u32 val;
 
 	assert_tc_cold_blocked(tc);
@@ -1158,7 +1158,7 @@ static bool xelpdp_tc_phy_is_owned(struct intel_tc_port *tc)
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	enum port port = tc->dig_port->base.port;
-	i915_reg_t reg = XELPDP_PORT_BUF_CTL1(display, port);
+	intel_reg_t reg = XELPDP_PORT_BUF_CTL1(display, port);
 
 	assert_tc_cold_blocked(tc);
 
@@ -1799,11 +1799,11 @@ static int reset_link(struct intel_tc_port *tc)
 {
 	struct intel_display *display = to_intel_display(tc->dig_port);
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *_state;
+	struct drm_atomic_commit *_state;
 	struct intel_atomic_state *state;
 	int ret;
 
-	_state = drm_atomic_state_alloc(display->drm);
+	_state = drm_atomic_commit_alloc(display->drm);
 	if (!_state)
 		return -ENOMEM;
 
@@ -1813,7 +1813,7 @@ static int reset_link(struct intel_tc_port *tc)
 	intel_modeset_lock_ctx_retry(&ctx, state, 0, ret)
 		ret = reset_link_commit(tc, state, &ctx);
 
-	drm_atomic_state_put(&state->base);
+	drm_atomic_commit_put(&state->base);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/i915/display/intel_tv.c b/drivers/gpu/drm/i915/display/intel_tv.c
index 8fbf0adb56992..0a926c6f25f46 100644
--- a/drivers/gpu/drm/i915/display/intel_tv.c
+++ b/drivers/gpu/drm/i915/display/intel_tv.c
@@ -1724,7 +1724,7 @@ intel_tv_detect(struct drm_connector *connector,
 		return connector->status;
 
 	if (force) {
-		struct drm_atomic_state *state;
+		struct drm_atomic_commit *state;
 
 		state = intel_load_detect_get_pipe(connector, ctx);
 		if (IS_ERR(state))
@@ -1847,7 +1847,7 @@ static const struct drm_connector_funcs intel_tv_connector_funcs = {
 };
 
 static int intel_tv_atomic_check(struct drm_connector *connector,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *new_state;
 	struct drm_crtc_state *new_crtc_state;
diff --git a/drivers/gpu/drm/i915/display/intel_vblank.c b/drivers/gpu/drm/i915/display/intel_vblank.c
index 1b7cfe226ff8f..28d81199792ef 100644
--- a/drivers/gpu/drm/i915/display/intel_vblank.c
+++ b/drivers/gpu/drm/i915/display/intel_vblank.c
@@ -109,8 +109,8 @@ u32 i915_get_vblank_counter(struct drm_crtc *crtc)
 	 * we get a low value that's stable across two reads of the high
 	 * register.
 	 */
-	frame = intel_de_read64_2x32(display, PIPEFRAMEPIXEL(display, pipe),
-				     PIPEFRAME(display, pipe));
+	frame = intel_de_read64_2x32_volatile(display, PIPEFRAMEPIXEL(display, pipe),
+					      PIPEFRAME(display, pipe));
 
 	pixel = frame & PIPE_PIXEL_MASK;
 	frame = (frame >> PIPE_FRAME_LOW_SHIFT) & 0xffffff;
@@ -482,7 +482,7 @@ int intel_get_crtc_scanline(struct intel_crtc *crtc)
 static bool pipe_scanline_is_moving(struct intel_display *display,
 				    enum pipe pipe)
 {
-	i915_reg_t reg = PIPEDSL(display, pipe);
+	intel_reg_t reg = PIPEDSL(display, pipe);
 	u32 line1, line2;
 
 	line1 = intel_de_read(display, reg) & PIPEDSL_LINE_MASK;
diff --git a/drivers/gpu/drm/i915/display/intel_vdsc.c b/drivers/gpu/drm/i915/display/intel_vdsc.c
index 6c09c6d99ffe1..8f06c3a4d56df 100644
--- a/drivers/gpu/drm/i915/display/intel_vdsc.c
+++ b/drivers/gpu/drm/i915/display/intel_vdsc.c
@@ -483,7 +483,7 @@ int intel_dsc_get_num_vdsc_instances(const struct intel_crtc_state *crtc_state)
 }
 
 static void intel_dsc_get_pps_reg(const struct intel_crtc_state *crtc_state, int pps,
-				  i915_reg_t *dsc_reg, int dsc_reg_num)
+				  intel_reg_t *dsc_reg, int dsc_reg_num)
 {
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
@@ -506,7 +506,7 @@ static void intel_dsc_pps_write(const struct intel_crtc_state *crtc_state,
 				int pps, u32 pps_val)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
-	i915_reg_t dsc_reg[3];
+	intel_reg_t dsc_reg[3];
 	int i, vdsc_per_pipe, dsc_reg_num;
 
 	vdsc_per_pipe = intel_dsc_get_vdsc_per_pipe(crtc_state);
@@ -843,13 +843,13 @@ void intel_dsc_su_et_parameters_configure(struct intel_dsb *dsb, struct intel_en
 		intel_de_write_dsb(display, dsb, LNL_DSC1_SU_PARAMETER_SET_0(pipe), val);
 }
 
-static i915_reg_t dss_ctl1_reg(struct intel_crtc *crtc, enum transcoder cpu_transcoder)
+static intel_reg_t dss_ctl1_reg(struct intel_crtc *crtc, enum transcoder cpu_transcoder)
 {
 	return is_pipe_dsc(crtc, cpu_transcoder) ?
 		ICL_PIPE_DSS_CTL1(crtc->pipe) : DSS_CTL1;
 }
 
-static i915_reg_t dss_ctl2_reg(struct intel_crtc *crtc, enum transcoder cpu_transcoder)
+static intel_reg_t dss_ctl2_reg(struct intel_crtc *crtc, enum transcoder cpu_transcoder)
 {
 	return is_pipe_dsc(crtc, cpu_transcoder) ?
 		ICL_PIPE_DSS_CTL2(crtc->pipe) : DSS_CTL2;
@@ -929,7 +929,7 @@ static u32 intel_dsc_pps_read(struct intel_crtc_state *crtc_state, int pps,
 			      bool *all_equal)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
-	i915_reg_t dsc_reg[3];
+	intel_reg_t dsc_reg[3];
 	int i, vdsc_per_pipe, dsc_reg_num;
 	u32 val;
 
diff --git a/drivers/gpu/drm/i915/display/intel_vga.c b/drivers/gpu/drm/i915/display/intel_vga.c
index 9832a4ade318c..4966fd7eea92b 100644
--- a/drivers/gpu/drm/i915/display/intel_vga.c
+++ b/drivers/gpu/drm/i915/display/intel_vga.c
@@ -35,7 +35,7 @@ static bool intel_vga_decode_is_enabled(struct intel_display *display)
 	return !(gmch_ctrl & INTEL_GMCH_VGA_DISABLE);
 }
 
-static i915_reg_t intel_vga_cntrl_reg(struct intel_display *display)
+static intel_reg_t intel_vga_cntrl_reg(struct intel_display *display)
 {
 	if (display->platform.valleyview || display->platform.cherryview)
 		return VLV_VGACNTRL;
@@ -179,7 +179,7 @@ static void intel_vga_write(struct intel_display *display, u16 reg, u8 val, bool
 void intel_vga_disable(struct intel_display *display)
 {
 	struct pci_dev *pdev = to_pci_dev(display->drm->dev);
-	i915_reg_t vga_reg = intel_vga_cntrl_reg(display);
+	intel_reg_t vga_reg = intel_vga_cntrl_reg(display);
 	bool mmio = has_vga_mmio_access(display);
 	bool io_decode;
 	u8 msr, sr1;
diff --git a/drivers/gpu/drm/i915/display/intel_vrr.c b/drivers/gpu/drm/i915/display/intel_vrr.c
index 8a957804cb970..1b09992ce9fdd 100644
--- a/drivers/gpu/drm/i915/display/intel_vrr.c
+++ b/drivers/gpu/drm/i915/display/intel_vrr.c
@@ -55,6 +55,16 @@ bool intel_vrr_is_capable(struct intel_connector *connector)
 		if (connector->mst.dp)
 			return false;
 		intel_dp = intel_attached_dp(connector);
+		/*
+		 * Among non-MST DP branch devices, only an HDMI 2.1 sink connected
+		 * via a PCON could support VRR. However, supporting VRR through a
+		 * PCON requires non-trivial changes that are not implemented yet.
+		 * Until that support exists, avoid VRR on all DP branch devices.
+		 *
+		 * TODO: Add support for VRR for DP->HDMI 2.1 PCON.
+		 */
+		if (drm_dp_is_branch(intel_dp->dpcd))
+			return false;
 
 		if (!drm_dp_sink_can_do_video_without_timing_msa(intel_dp->dpcd))
 			return false;
@@ -1053,11 +1063,9 @@ void intel_vrr_get_config(struct intel_crtc_state *crtc_state)
 
 	if (crtc_state->cmrr.enable) {
 		crtc_state->cmrr.cmrr_n =
-			intel_de_read64_2x32(display, TRANS_CMRR_N_LO(display, cpu_transcoder),
-					     TRANS_CMRR_N_HI(display, cpu_transcoder));
+			intel_de_read64_2x32(display, TRANS_CMRR_N_LO(display, cpu_transcoder));
 		crtc_state->cmrr.cmrr_m =
-			intel_de_read64_2x32(display, TRANS_CMRR_M_LO(display, cpu_transcoder),
-					     TRANS_CMRR_M_HI(display, cpu_transcoder));
+			intel_de_read64_2x32(display, TRANS_CMRR_M_LO(display, cpu_transcoder));
 	}
 
 	if (DISPLAY_VER(display) >= 13) {
diff --git a/drivers/gpu/drm/i915/display/intel_wm.c b/drivers/gpu/drm/i915/display/intel_wm.c
index f887a664fe22c..b4aded774ce66 100644
--- a/drivers/gpu/drm/i915/display/intel_wm.c
+++ b/drivers/gpu/drm/i915/display/intel_wm.c
@@ -48,8 +48,8 @@
  */
 void intel_update_watermarks(struct intel_display *display)
 {
-	if (display->funcs.wm->update_wm)
-		display->funcs.wm->update_wm(display);
+	if (display->wm.funcs->update_wm)
+		display->wm.funcs->update_wm(display);
 }
 
 int intel_wm_compute(struct intel_atomic_state *state,
@@ -57,10 +57,10 @@ int intel_wm_compute(struct intel_atomic_state *state,
 {
 	struct intel_display *display = to_intel_display(state);
 
-	if (!display->funcs.wm->compute_watermarks)
+	if (!display->wm.funcs->compute_watermarks)
 		return 0;
 
-	return display->funcs.wm->compute_watermarks(state, crtc);
+	return display->wm.funcs->compute_watermarks(state, crtc);
 }
 
 bool intel_initial_watermarks(struct intel_atomic_state *state,
@@ -68,8 +68,8 @@ bool intel_initial_watermarks(struct intel_atomic_state *state,
 {
 	struct intel_display *display = to_intel_display(state);
 
-	if (display->funcs.wm->initial_watermarks) {
-		display->funcs.wm->initial_watermarks(state, crtc);
+	if (display->wm.funcs->initial_watermarks) {
+		display->wm.funcs->initial_watermarks(state, crtc);
 		return true;
 	}
 
@@ -81,8 +81,8 @@ void intel_atomic_update_watermarks(struct intel_atomic_state *state,
 {
 	struct intel_display *display = to_intel_display(state);
 
-	if (display->funcs.wm->atomic_update_watermarks)
-		display->funcs.wm->atomic_update_watermarks(state, crtc);
+	if (display->wm.funcs->atomic_update_watermarks)
+		display->wm.funcs->atomic_update_watermarks(state, crtc);
 }
 
 void intel_optimize_watermarks(struct intel_atomic_state *state,
@@ -90,30 +90,30 @@ void intel_optimize_watermarks(struct intel_atomic_state *state,
 {
 	struct intel_display *display = to_intel_display(state);
 
-	if (display->funcs.wm->optimize_watermarks)
-		display->funcs.wm->optimize_watermarks(state, crtc);
+	if (display->wm.funcs->optimize_watermarks)
+		display->wm.funcs->optimize_watermarks(state, crtc);
 }
 
 int intel_compute_global_watermarks(struct intel_atomic_state *state)
 {
 	struct intel_display *display = to_intel_display(state);
 
-	if (display->funcs.wm->compute_global_watermarks)
-		return display->funcs.wm->compute_global_watermarks(state);
+	if (display->wm.funcs->compute_global_watermarks)
+		return display->wm.funcs->compute_global_watermarks(state);
 
 	return 0;
 }
 
 void intel_wm_get_hw_state(struct intel_display *display)
 {
-	if (display->funcs.wm->get_hw_state)
-		return display->funcs.wm->get_hw_state(display);
+	if (display->wm.funcs->get_hw_state)
+		return display->wm.funcs->get_hw_state(display);
 }
 
 void intel_wm_sanitize(struct intel_display *display)
 {
-	if (display->funcs.wm->sanitize)
-		return display->funcs.wm->sanitize(display);
+	if (display->wm.funcs->sanitize)
+		return display->wm.funcs->sanitize(display);
 }
 
 bool intel_wm_plane_visible(const struct intel_crtc_state *crtc_state,
diff --git a/drivers/gpu/drm/i915/display/skl_scaler.c b/drivers/gpu/drm/i915/display/skl_scaler.c
index 7c5cb188ebf03..7994b983d509d 100644
--- a/drivers/gpu/drm/i915/display/skl_scaler.c
+++ b/drivers/gpu/drm/i915/display/skl_scaler.c
@@ -270,6 +270,11 @@ int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state)
 {
 	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
 	int width, height;
+	int ret;
+
+	ret = intel_casf_compute_config(crtc_state);
+	if (ret)
+		return ret;
 
 	if (crtc_state->pch_pfit.enabled) {
 		width = drm_rect_width(&crtc_state->pch_pfit.dst);
@@ -284,8 +289,7 @@ int skl_update_scaler_crtc(struct intel_crtc_state *crtc_state)
 				 drm_rect_width(&crtc_state->pipe_src),
 				 drm_rect_height(&crtc_state->pipe_src),
 				 width, height, NULL, 0,
-				 crtc_state->pch_pfit.enabled ||
-				 intel_casf_needs_scaler(crtc_state));
+				 crtc_state->pch_pfit.enabled);
 }
 
 /**
@@ -323,19 +327,24 @@ int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 				 need_scaler);
 }
 
+static bool scaler_has_casf(struct intel_display *display, int scaler_id)
+{
+	return HAS_CASF(display) && scaler_id == 1;
+}
+
 static int intel_allocate_scaler(struct intel_crtc_scaler_state *scaler_state,
 				 struct intel_crtc *crtc,
 				 struct intel_plane_state *plane_state,
 				 bool casf_scaler)
 {
+	struct intel_display *display = to_intel_display(crtc);
 	int i;
 
 	for (i = 0; i < crtc->num_scalers; i++) {
 		if (scaler_state->scalers[i].in_use)
 			continue;
 
-		/* CASF needs second scaler */
-		if (!plane_state && casf_scaler && i != 1)
+		if (casf_scaler && !scaler_has_casf(display, i))
 			continue;
 
 		scaler_state->scalers[i].in_use = true;
@@ -529,14 +538,11 @@ static int setup_crtc_scaler(struct intel_atomic_state *state,
 	struct intel_crtc_scaler_state *scaler_state =
 		&crtc_state->scaler_state;
 
-	if (intel_casf_needs_scaler(crtc_state) && crtc_state->pch_pfit.enabled)
-		return -EINVAL;
-
 	return intel_atomic_setup_scaler(crtc_state,
 					 hweight32(scaler_state->scaler_users),
 					 crtc, "CRTC", crtc->base.base.id,
 					 NULL, &scaler_state->scaler_id,
-					 intel_casf_needs_scaler(crtc_state));
+					 crtc_state->pch_pfit.casf.enable);
 }
 
 static int setup_plane_scaler(struct intel_atomic_state *state,
@@ -655,9 +661,9 @@ static int glk_coef_tap(int i)
 	return i % 7;
 }
 
-static u16 glk_nearest_filter_coef(int t)
+static u16 glk_nearest_filter_coef(int tap)
 {
-	return t == 3 ? 0x0800 : 0x3000;
+	return tap == 3 ? 0x0800 : 0x3000;
 }
 
 /*
@@ -709,13 +715,13 @@ static void glk_program_nearest_filter_coefs(struct intel_display *display,
 
 	for (i = 0; i < 17 * 7; i += 2) {
 		u32 tmp;
-		int t;
+		int tap;
 
-		t = glk_coef_tap(i);
-		tmp = glk_nearest_filter_coef(t);
+		tap = glk_coef_tap(i);
+		tmp = glk_nearest_filter_coef(tap);
 
-		t = glk_coef_tap(i + 1);
-		tmp |= glk_nearest_filter_coef(t) << 16;
+		tap = glk_coef_tap(i + 1);
+		tmp |= glk_nearest_filter_coef(tap) << 16;
 
 		intel_de_write_dsb(display, dsb,
 				   GLK_PS_COEF_DATA_SET(pipe, id, set), tmp);
@@ -725,9 +731,9 @@ static void glk_program_nearest_filter_coefs(struct intel_display *display,
 			   GLK_PS_COEF_INDEX_SET(pipe, id, set), 0);
 }
 
-static u32 skl_scaler_get_filter_select(enum drm_scaling_filter filter)
+static u32 skl_scaler_get_filter_select(enum drm_scaling_filter filter, bool casf)
 {
-	if (filter == DRM_SCALING_FILTER_NEAREST_NEIGHBOR)
+	if (filter == DRM_SCALING_FILTER_NEAREST_NEIGHBOR || casf)
 		return (PS_FILTER_PROGRAMMED |
 			PS_Y_VERT_FILTER_SELECT(0) |
 			PS_Y_HORZ_FILTER_SELECT(0) |
@@ -752,50 +758,14 @@ static void skl_scaler_setup_filter(struct intel_display *display,
 	}
 }
 
-#define CASF_SCALER_FILTER_SELECT \
-	(PS_FILTER_PROGRAMMED | \
-	PS_Y_VERT_FILTER_SELECT(0) | \
-	PS_Y_HORZ_FILTER_SELECT(0) | \
-	PS_UV_VERT_FILTER_SELECT(0) | \
-	PS_UV_HORZ_FILTER_SELECT(0))
-
-void skl_scaler_setup_casf(struct intel_crtc_state *crtc_state)
+static u32 casf_sharpness_ctl(const struct intel_crtc_state *crtc_state)
 {
-	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
-	struct intel_display *display = to_intel_display(crtc);
-	struct drm_display_mode *adjusted_mode =
-	&crtc_state->hw.adjusted_mode;
-	struct intel_crtc_scaler_state *scaler_state =
-		&crtc_state->scaler_state;
-	struct drm_rect src, dest;
-	int id, width, height;
-	int x = 0, y = 0;
-	enum pipe pipe = crtc->pipe;
-	u32 ps_ctrl;
-
-	width = adjusted_mode->crtc_hdisplay;
-	height = adjusted_mode->crtc_vdisplay;
-
-	drm_rect_init(&dest, x, y, width, height);
-
-	width = drm_rect_width(&dest);
-	height = drm_rect_height(&dest);
-	id = scaler_state->scaler_id;
-
-	drm_rect_init(&src, 0, 0,
-		      drm_rect_width(&crtc_state->pipe_src) << 16,
-		      drm_rect_height(&crtc_state->pipe_src) << 16);
-
-	trace_intel_pipe_scaler_update_arm(crtc, id, x, y, width, height);
-
-	ps_ctrl = PS_SCALER_EN | PS_BINDING_PIPE | scaler_state->scalers[id].mode |
-		  CASF_SCALER_FILTER_SELECT;
+	if (!crtc_state->pch_pfit.casf.enable)
+		return 0;
 
-	intel_de_write_fw(display, SKL_PS_CTRL(pipe, id), ps_ctrl);
-	intel_de_write_fw(display, SKL_PS_WIN_POS(pipe, id),
-			  PS_WIN_XPOS(x) | PS_WIN_YPOS(y));
-	intel_de_write_fw(display, SKL_PS_WIN_SZ(pipe, id),
-			  PS_WIN_XSIZE(width) | PS_WIN_YSIZE(height));
+	return FILTER_EN |
+		FILTER_STRENGTH(crtc_state->pch_pfit.casf.strength) |
+		crtc_state->pch_pfit.casf.win_size;
 }
 
 void skl_pfit_enable(const struct intel_crtc_state *crtc_state)
@@ -839,12 +809,20 @@ void skl_pfit_enable(const struct intel_crtc_state *crtc_state)
 	id = scaler_state->scaler_id;
 
 	ps_ctrl = PS_SCALER_EN | PS_BINDING_PIPE | scaler_state->scalers[id].mode |
-		skl_scaler_get_filter_select(crtc_state->hw.scaling_filter);
+		skl_scaler_get_filter_select(crtc_state->hw.scaling_filter,
+					     crtc_state->pch_pfit.casf.enable);
 
 	trace_intel_pipe_scaler_update_arm(crtc, id, x, y, width, height);
 
-	skl_scaler_setup_filter(display, NULL, pipe, id, 0,
-				crtc_state->hw.scaling_filter);
+	if (crtc_state->pch_pfit.casf.enable)
+		intel_casf_setup(crtc_state);
+	else
+		skl_scaler_setup_filter(display, NULL, pipe, id, 0,
+					crtc_state->hw.scaling_filter);
+
+	if (scaler_has_casf(display, id))
+		intel_de_write_fw(display, SHARPNESS_CTL(crtc->pipe),
+				  casf_sharpness_ctl(crtc_state));
 
 	intel_de_write_fw(display, SKL_PS_CTRL(pipe, id), ps_ctrl);
 
@@ -858,6 +836,22 @@ void skl_pfit_enable(const struct intel_crtc_state *crtc_state)
 			  PS_WIN_XSIZE(width) | PS_WIN_YSIZE(height));
 }
 
+static int skl_pipe_scaler_get_hw_state(struct intel_crtc_state *crtc_state)
+{
+	struct intel_display *display = to_intel_display(crtc_state);
+	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
+
+	for (int scaler_id = 0; scaler_id < crtc->num_scalers; scaler_id++) {
+		u32 ctl;
+
+		ctl = intel_de_read(display, SKL_PS_CTRL(crtc->pipe, scaler_id));
+		if ((ctl & (PS_SCALER_EN | PS_BINDING_MASK)) == (PS_SCALER_EN | PS_BINDING_PIPE))
+			return scaler_id;
+	}
+
+	return -1;
+}
+
 void
 skl_program_plane_scaler(struct intel_dsb *dsb,
 			 struct intel_plane *plane,
@@ -905,7 +899,7 @@ skl_program_plane_scaler(struct intel_dsb *dsb,
 	}
 
 	ps_ctrl = PS_SCALER_EN | PS_BINDING_PLANE(plane->id) | scaler->mode |
-		skl_scaler_get_filter_select(plane_state->hw.scaling_filter);
+		skl_scaler_get_filter_select(plane_state->hw.scaling_filter, false);
 
 	trace_intel_plane_scaler_update_arm(plane, scaler_id,
 					    crtc_x, crtc_y, crtc_w, crtc_h);
@@ -932,6 +926,9 @@ static void skl_detach_scaler(struct intel_dsb *dsb,
 
 	trace_intel_scaler_disable_arm(crtc, id);
 
+	if (scaler_has_casf(display, id))
+		intel_de_write_dsb(display, dsb, SHARPNESS_CTL(crtc->pipe), 0);
+
 	intel_de_write_dsb(display, dsb, SKL_PS_CTRL(crtc->pipe, id), 0);
 	intel_de_write_dsb(display, dsb, SKL_PS_WIN_POS(crtc->pipe, id), 0);
 	intel_de_write_dsb(display, dsb, SKL_PS_WIN_SZ(crtc->pipe, id), 0);
@@ -969,45 +966,32 @@ void skl_scaler_get_config(struct intel_crtc_state *crtc_state)
 	struct intel_display *display = to_intel_display(crtc_state);
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	struct intel_crtc_scaler_state *scaler_state = &crtc_state->scaler_state;
-	int id = -1;
-	int i;
+	int scaler_id;
+	u32 pos, size;
 
 	/* find scaler attached to this pipe */
-	for (i = 0; i < crtc->num_scalers; i++) {
-		u32 ctl, pos, size;
-
-		ctl = intel_de_read(display, SKL_PS_CTRL(crtc->pipe, i));
-		if ((ctl & (PS_SCALER_EN | PS_BINDING_MASK)) != (PS_SCALER_EN | PS_BINDING_PIPE))
-			continue;
-
-		id = i;
+	scaler_id = skl_pipe_scaler_get_hw_state(crtc_state);
+	if (scaler_id < 0)
+		return;
 
-		/* Read CASF regs for second scaler */
-		if (HAS_CASF(display) && id == 1)
-			intel_casf_sharpness_get_config(crtc_state);
+	if (scaler_has_casf(display, scaler_id))
+		intel_casf_sharpness_get_config(crtc_state);
 
-		if (!crtc_state->hw.casf_params.casf_enable)
-			crtc_state->pch_pfit.enabled = true;
+	crtc_state->pch_pfit.enabled = true;
 
-		pos = intel_de_read(display, SKL_PS_WIN_POS(crtc->pipe, i));
-		size = intel_de_read(display, SKL_PS_WIN_SZ(crtc->pipe, i));
+	pos = intel_de_read(display, SKL_PS_WIN_POS(crtc->pipe, scaler_id));
+	size = intel_de_read(display, SKL_PS_WIN_SZ(crtc->pipe, scaler_id));
 
-		if (!crtc_state->hw.casf_params.casf_enable)
-			drm_rect_init(&crtc_state->pch_pfit.dst,
-				      REG_FIELD_GET(PS_WIN_XPOS_MASK, pos),
-				      REG_FIELD_GET(PS_WIN_YPOS_MASK, pos),
-				      REG_FIELD_GET(PS_WIN_XSIZE_MASK, size),
-				      REG_FIELD_GET(PS_WIN_YSIZE_MASK, size));
+	drm_rect_init(&crtc_state->pch_pfit.dst,
+		      REG_FIELD_GET(PS_WIN_XPOS_MASK, pos),
+		      REG_FIELD_GET(PS_WIN_YPOS_MASK, pos),
+		      REG_FIELD_GET(PS_WIN_XSIZE_MASK, size),
+		      REG_FIELD_GET(PS_WIN_YSIZE_MASK, size));
 
-		scaler_state->scalers[i].in_use = true;
-		break;
-	}
+	scaler_state->scalers[scaler_id].in_use = true;
 
-	scaler_state->scaler_id = id;
-	if (id >= 0)
-		scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
-	else
-		scaler_state->scaler_users &= ~(1 << SKL_CRTC_INDEX);
+	scaler_state->scaler_id = scaler_id;
+	scaler_state->scaler_users |= (1 << SKL_CRTC_INDEX);
 }
 
 void adl_scaler_ecc_mask(const struct intel_crtc_state *crtc_state)
diff --git a/drivers/gpu/drm/i915/display/skl_scaler.h b/drivers/gpu/drm/i915/display/skl_scaler.h
index 7e8d819c019de..5deabca909e65 100644
--- a/drivers/gpu/drm/i915/display/skl_scaler.h
+++ b/drivers/gpu/drm/i915/display/skl_scaler.h
@@ -36,8 +36,6 @@ void skl_scaler_disable(const struct intel_crtc_state *old_crtc_state);
 
 void skl_scaler_get_config(struct intel_crtc_state *crtc_state);
 
-void skl_scaler_setup_casf(struct intel_crtc_state *crtc_state);
-
 enum drm_mode_status
 skl_scaler_mode_valid(struct intel_display *display,
 		      const struct drm_display_mode *mode,
diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c
index 11ba42c67e3ed..ef431dd32e74a 100644
--- a/drivers/gpu/drm/i915/display/skl_universal_plane.c
+++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c
@@ -8,6 +8,7 @@
 #include <drm/drm_damage_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_print.h>
+#include <drm/intel/step.h>
 
 #include "intel_bo.h"
 #include "intel_color.h"
@@ -25,7 +26,6 @@
 #include "intel_plane.h"
 #include "intel_psr.h"
 #include "intel_psr_regs.h"
-#include "intel_step.h"
 #include "skl_scaler.h"
 #include "skl_universal_plane.h"
 #include "skl_universal_plane_regs.h"
@@ -2126,6 +2126,19 @@ static int skl_check_main_surface(struct intel_plane_state *plane_state)
 	return 0;
 }
 
+
+/* Divide a U16.16 fixed-point value by 2, staying in fixed-point domain */
+static inline u32 fp_16_16_div2(u32 fp)
+{
+	return fp >> 1;
+}
+
+/* Convert a U16.16 fixed-point value to integer, rounding up */
+static inline int fp_16_16_to_int_ceil(u32 fp)
+{
+	return DIV_ROUND_UP(fp, 1 << 16);
+}
+
 static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
 {
 	struct intel_display *display = to_intel_display(plane_state);
@@ -2139,10 +2152,16 @@ static int skl_check_nv12_aux_surface(struct intel_plane_state *plane_state)
 	int min_height = intel_plane_min_height(plane, fb, uv_plane, rotation);
 	int max_width = intel_plane_max_width(plane, fb, uv_plane, rotation);
 	int max_height = intel_plane_max_height(plane, fb, uv_plane, rotation);
-	int x = plane_state->uapi.src.x1 >> 17;
-	int y = plane_state->uapi.src.y1 >> 17;
-	int w = drm_rect_width(&plane_state->uapi.src) >> 17;
-	int h = drm_rect_height(&plane_state->uapi.src) >> 17;
+
+	/*
+	 * LNL+ UV surface start/size =
+	 * ceiling(half of Y plane start/size). Use ceiling division
+	 * unconditionally; it is a no-op for even values.
+	 */
+	int x = fp_16_16_to_int_ceil(fp_16_16_div2(plane_state->uapi.src.x1));
+	int y = fp_16_16_to_int_ceil(fp_16_16_div2(plane_state->uapi.src.y1));
+	int w = fp_16_16_to_int_ceil(fp_16_16_div2(drm_rect_width(&plane_state->uapi.src)));
+	int h = fp_16_16_to_int_ceil(fp_16_16_div2(drm_rect_height(&plane_state->uapi.src)));
 	u32 offset;
 
 	/* FIXME not quite sure how/if these apply to the chroma plane */
@@ -3148,12 +3167,6 @@ skl_get_initial_plane_config(struct intel_crtc *crtc,
 
 	fb->format = drm_get_format_info(display->drm, fourcc, fb->modifier);
 
-	if (!display->params.enable_dpt &&
-	    intel_fb_modifier_uses_dpt(display, fb->modifier)) {
-		drm_dbg_kms(display->drm, "DPT disabled, skipping initial FB\n");
-		goto error;
-	}
-
 	/*
 	 * DRM_MODE_ROTATE_ is counter clockwise to stay compatible with Xrandr
 	 * while i915 HW rotation is clockwise, that's why this swapping.
@@ -3206,7 +3219,7 @@ skl_get_initial_plane_config(struct intel_crtc *crtc,
 		    fb->width, fb->height, fb->format->cpp[0] * 8,
 		    base, fb->pitches[0], plane_config->size);
 
-	plane_config->fb = intel_fb;
+	plane_config->fb = &intel_fb->base;
 	return;
 
 error:
diff --git a/drivers/gpu/drm/i915/display/skl_watermark.c b/drivers/gpu/drm/i915/display/skl_watermark.c
index e0ac4e2ce4dcb..96d2dcbe7bbcf 100644
--- a/drivers/gpu/drm/i915/display/skl_watermark.c
+++ b/drivers/gpu/drm/i915/display/skl_watermark.c
@@ -308,15 +308,6 @@ static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 	enum plane_id plane_id;
 	int max_level = INT_MAX;
 
-	if (!intel_has_sagv(display))
-		return false;
-
-	if (!crtc_state->hw.active)
-		return true;
-
-	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
-		return false;
-
 	for_each_plane_id_on_crtc(crtc, plane_id) {
 		const struct skl_plane_wm *wm =
 			&crtc_state->wm.skl.optimal.planes[plane_id];
@@ -359,9 +350,6 @@ static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
 	enum plane_id plane_id;
 
-	if (!crtc_state->hw.active)
-		return true;
-
 	for_each_plane_id_on_crtc(crtc, plane_id) {
 		const struct skl_plane_wm *wm =
 			&crtc_state->wm.skl.optimal.planes[plane_id];
@@ -377,6 +365,9 @@ bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 {
 	struct intel_display *display = to_intel_display(crtc_state);
 
+	if (!display->sagv.block_time_us)
+		return false;
+
 	if (!display->params.enable_sagv)
 		return false;
 
@@ -388,7 +379,13 @@ bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
 	if (crtc_state->inherited)
 		return false;
 
-	if (DISPLAY_VER(display) >= 12)
+	if (!crtc_state->hw.active)
+		return true;
+
+	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
+		return false;
+
+	if (HAS_SAGV_WM(display))
 		return tgl_crtc_can_enable_sagv(crtc_state);
 	else
 		return skl_crtc_can_enable_sagv(crtc_state);
@@ -1939,7 +1936,7 @@ static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
 	result->enable = true;
 	result->auto_min_alloc_wm_enable = xe3_auto_min_alloc_capable(plane, level);
 
-	if (DISPLAY_VER(display) < 12 && display->sagv.block_time_us)
+	if (!HAS_SAGV_WM(display) && display->sagv.block_time_us)
 		result->can_sagv = latency >= display->sagv.block_time_us;
 }
 
@@ -2065,7 +2062,7 @@ static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
 	skl_compute_transition_wm(display, &wm->trans_wm,
 				  &wm->wm[0], &wm_params);
 
-	if (DISPLAY_VER(display) >= 12) {
+	if (HAS_SAGV_WM(display)) {
 		tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
 
 		skl_compute_transition_wm(display, &wm->sagv.trans_wm,
@@ -2324,7 +2321,7 @@ static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
 		}
 	}
 
-	if (DISPLAY_VER(display) >= 12 &&
+	if (HAS_SAGV_WM(display) &&
 	    display->sagv.block_time_us &&
 	    skl_prefill_vblank_too_short(&ctx, crtc_state,
 					 display->sagv.block_time_us)) {
@@ -2616,6 +2613,26 @@ skl_print_plane_ddb_changes(struct intel_plane *plane,
 		    skl_ddb_entry_size(old), skl_ddb_entry_size(new));
 }
 
+#define PLANE_WM_EN_FMT "%cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
+#define PLANE_WM_EN_ARGS(__wm) \
+	enast((__wm)->wm[0].enable), enast((__wm)->wm[1].enable), \
+	enast((__wm)->wm[2].enable), enast((__wm)->wm[3].enable), \
+	enast((__wm)->wm[4].enable), enast((__wm)->wm[5].enable), \
+	enast((__wm)->wm[6].enable), enast((__wm)->wm[7].enable), \
+	enast((__wm)->trans_wm.enable), \
+	enast((__wm)->sagv.wm0.enable), \
+	enast((__wm)->sagv.trans_wm.enable)
+
+#define PLANE_WM_FMT "%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
+#define PLANE_WM_ARGS(__wm, __field) \
+	(__wm)->wm[0].__field, (__wm)->wm[1].__field, \
+	(__wm)->wm[2].__field, (__wm)->wm[3].__field, \
+	(__wm)->wm[4].__field, (__wm)->wm[5].__field, \
+	(__wm)->wm[6].__field, (__wm)->wm[7].__field, \
+	(__wm)->trans_wm.__field, \
+	(__wm)->sagv.wm0.__field, \
+	(__wm)->sagv.trans_wm.__field
+
 static noinline_for_stack void
 skl_print_plane_wm_changes(struct intel_plane *plane,
 			   const struct skl_plane_wm *old_wm,
@@ -2624,112 +2641,44 @@ skl_print_plane_wm_changes(struct intel_plane *plane,
 	struct intel_display *display = to_intel_display(plane);
 
 	drm_dbg_kms(display->drm,
-		    "[PLANE:%d:%s]      level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
-		    " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
+		    "[PLANE:%d:%s]      level " PLANE_WM_EN_FMT " -> " PLANE_WM_EN_FMT "\n",
 		    plane->base.base.id, plane->base.name,
-		    enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
-		    enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
-		    enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
-		    enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
-		    enast(old_wm->trans_wm.enable),
-		    enast(old_wm->sagv.wm0.enable),
-		    enast(old_wm->sagv.trans_wm.enable),
-		    enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
-		    enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
-		    enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
-		    enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
-		    enast(new_wm->trans_wm.enable),
-		    enast(new_wm->sagv.wm0.enable),
-		    enast(new_wm->sagv.trans_wm.enable));
+		    PLANE_WM_EN_ARGS(old_wm),
+		    PLANE_WM_EN_ARGS(new_wm));
 
 	drm_dbg_kms(display->drm,
-		    "[PLANE:%d:%s]      lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
-		      " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
+		    "[PLANE:%d:%s]      lines " PLANE_WM_FMT " -> " PLANE_WM_FMT "\n",
 		    plane->base.base.id, plane->base.name,
-		    enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
-		    enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
-		    enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
-		    enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
-		    enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
-		    enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
-		    enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
-		    enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
-		    enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
-		    enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
-		    enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
-		    enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
-		    enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
-		    enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
-		    enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
-		    enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
-		    enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
-		    enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
-		    enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
-		    enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
-		    enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
-		    enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
+		    PLANE_WM_ARGS(old_wm, lines),
+		    PLANE_WM_ARGS(new_wm, lines));
 
 	drm_dbg_kms(display->drm,
-		    "[PLANE:%d:%s]     blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
-		    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
+		    "[PLANE:%d:%s]     blocks " PLANE_WM_FMT " -> " PLANE_WM_FMT "\n",
 		    plane->base.base.id, plane->base.name,
-		    old_wm->wm[0].blocks, old_wm->wm[1].blocks,
-		    old_wm->wm[2].blocks, old_wm->wm[3].blocks,
-		    old_wm->wm[4].blocks, old_wm->wm[5].blocks,
-		    old_wm->wm[6].blocks, old_wm->wm[7].blocks,
-		    old_wm->trans_wm.blocks,
-		    old_wm->sagv.wm0.blocks,
-		    old_wm->sagv.trans_wm.blocks,
-		    new_wm->wm[0].blocks, new_wm->wm[1].blocks,
-		    new_wm->wm[2].blocks, new_wm->wm[3].blocks,
-		    new_wm->wm[4].blocks, new_wm->wm[5].blocks,
-		    new_wm->wm[6].blocks, new_wm->wm[7].blocks,
-		    new_wm->trans_wm.blocks,
-		    new_wm->sagv.wm0.blocks,
-		    new_wm->sagv.trans_wm.blocks);
+		    PLANE_WM_ARGS(old_wm, blocks),
+		    PLANE_WM_ARGS(new_wm, blocks));
 
 	drm_dbg_kms(display->drm,
-		    "[PLANE:%d:%s]    min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
-		    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
+		    "[PLANE:%d:%s]    min_ddb " PLANE_WM_FMT " -> " PLANE_WM_FMT "\n",
 		    plane->base.base.id, plane->base.name,
-		    old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
-		    old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
-		    old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
-		    old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
-		    old_wm->trans_wm.min_ddb_alloc,
-		    old_wm->sagv.wm0.min_ddb_alloc,
-		    old_wm->sagv.trans_wm.min_ddb_alloc,
-		    new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
-		    new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
-		    new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
-		    new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
-		    new_wm->trans_wm.min_ddb_alloc,
-		    new_wm->sagv.wm0.min_ddb_alloc,
-		    new_wm->sagv.trans_wm.min_ddb_alloc);
+		    PLANE_WM_ARGS(old_wm, min_ddb_alloc),
+		    PLANE_WM_ARGS(new_wm, min_ddb_alloc));
 
 	if (DISPLAY_VER(display) >= 11)
 		return;
 
 	drm_dbg_kms(display->drm,
-		    "[PLANE:%d:%s] min_ddb_uv %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
-		    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
+		    "[PLANE:%d:%s] min_ddb_uv " PLANE_WM_FMT " -> " PLANE_WM_FMT "\n",
 		    plane->base.base.id, plane->base.name,
-		    old_wm->wm[0].min_ddb_alloc_uv, old_wm->wm[1].min_ddb_alloc_uv,
-		    old_wm->wm[2].min_ddb_alloc_uv, old_wm->wm[3].min_ddb_alloc_uv,
-		    old_wm->wm[4].min_ddb_alloc_uv, old_wm->wm[5].min_ddb_alloc_uv,
-		    old_wm->wm[6].min_ddb_alloc_uv, old_wm->wm[7].min_ddb_alloc_uv,
-		    old_wm->trans_wm.min_ddb_alloc_uv,
-		    old_wm->sagv.wm0.min_ddb_alloc_uv,
-		    old_wm->sagv.trans_wm.min_ddb_alloc_uv,
-		    new_wm->wm[0].min_ddb_alloc_uv, new_wm->wm[1].min_ddb_alloc_uv,
-		    new_wm->wm[2].min_ddb_alloc_uv, new_wm->wm[3].min_ddb_alloc_uv,
-		    new_wm->wm[4].min_ddb_alloc_uv, new_wm->wm[5].min_ddb_alloc_uv,
-		    new_wm->wm[6].min_ddb_alloc_uv, new_wm->wm[7].min_ddb_alloc_uv,
-		    new_wm->trans_wm.min_ddb_alloc_uv,
-		    new_wm->sagv.wm0.min_ddb_alloc_uv,
-		    new_wm->sagv.trans_wm.min_ddb_alloc_uv);
+		    PLANE_WM_ARGS(old_wm, min_ddb_alloc_uv),
+		    PLANE_WM_ARGS(new_wm, min_ddb_alloc_uv));
 }
 
+#undef PLANE_WM_EN_FMT
+#undef PLANE_WM_EN_ARGS
+#undef PLANE_WM_FMT
+#undef PLANE_WM_ARGS
+
 static void
 skl_print_wm_changes(struct intel_atomic_state *state)
 {
@@ -2997,8 +2946,9 @@ skl_compute_wm(struct intel_atomic_state *state)
 		 * other crtcs can't be allowed to use the more optimal
 		 * normal (ie. non-SAGV) watermarks.
 		 */
-		pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(display) &&
-			DISPLAY_VER(display) >= 12 &&
+		pipe_wm->use_sagv_wm =
+			HAS_SAGV_WM(display) &&
+			!HAS_HW_SAGV_WM(display) &&
 			intel_crtc_can_enable_sagv(new_crtc_state);
 
 		ret = skl_wm_add_affected_planes(state, crtc);
@@ -3064,7 +3014,7 @@ static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
 				val = intel_de_read(display, CUR_WM_SAGV_TRANS(pipe));
 
 			skl_wm_level_from_reg_val(display, val, &wm->sagv.trans_wm);
-		} else if (DISPLAY_VER(display) >= 12) {
+		} else if (HAS_SAGV_WM(display)) {
 			wm->sagv.wm0 = wm->wm[0];
 			wm->sagv.trans_wm = wm->trans_wm;
 		}
@@ -3919,6 +3869,40 @@ void skl_wm_plane_disable_noatomic(struct intel_crtc *crtc,
 	       sizeof(crtc_state->wm.skl.optimal.planes[plane->id]));
 }
 
+static void skl_wm_level_verify(struct intel_plane *plane,
+				const char *wm_name,
+				const struct skl_wm_level *hw_wm_level,
+				const struct skl_wm_level *sw_wm_level)
+{
+	struct intel_display *display = to_intel_display(plane);
+
+	if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
+		return;
+
+	drm_err(display->drm,
+		"[PLANE:%d:%s] mismatch in %s (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
+		plane->base.base.id, plane->base.name, wm_name,
+		sw_wm_level->enable, sw_wm_level->blocks, sw_wm_level->lines,
+		hw_wm_level->enable, hw_wm_level->blocks, hw_wm_level->lines);
+}
+
+static void skl_ddb_entry_verify(struct intel_plane *plane,
+				 const char *ddb_name,
+				 const struct skl_ddb_entry *hw_ddb_entry,
+				 const struct skl_ddb_entry *sw_ddb_entry)
+{
+	struct intel_display *display = to_intel_display(plane);
+
+	if (skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry))
+		return;
+
+	drm_err(display->drm,
+		"[PLANE:%d:%s] mismatch in %s (expected (%u,%u), found (%u,%u))\n",
+		plane->base.base.id, plane->base.name, ddb_name,
+		sw_ddb_entry->start, sw_ddb_entry->end,
+		hw_ddb_entry->start, hw_ddb_entry->end);
+}
+
 void intel_wm_state_verify(struct intel_atomic_state *state,
 			   struct intel_crtc *crtc)
 {
@@ -3958,86 +3942,42 @@ void intel_wm_state_verify(struct intel_atomic_state *state,
 			hw_enabled_slices);
 
 	for_each_intel_plane_on_crtc(display->drm, crtc, plane) {
-		const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
-		const struct skl_wm_level *hw_wm_level, *sw_wm_level;
+		const struct skl_plane_wm *hw_plane_wm =
+			&hw->wm.planes[plane->id];
+		const struct skl_plane_wm *sw_plane_wm =
+			&sw_wm->planes[plane->id];
 
-		/* Watermarks */
 		for (level = 0; level < display->wm.num_levels; level++) {
-			hw_wm_level = &hw->wm.planes[plane->id].wm[level];
-			sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
+			char wm_name[16];
 
-			if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
-				continue;
+			snprintf(wm_name, sizeof(wm_name), "WM%d", level);
 
-			drm_err(display->drm,
-				"[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
-				plane->base.base.id, plane->base.name, level,
-				sw_wm_level->enable,
-				sw_wm_level->blocks,
-				sw_wm_level->lines,
-				hw_wm_level->enable,
-				hw_wm_level->blocks,
-				hw_wm_level->lines);
+			skl_wm_level_verify(plane, wm_name,
+					    &hw_plane_wm->wm[level],
+					    skl_plane_wm_level(sw_wm, plane->id, level));
 		}
 
-		hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
-		sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
-
-		if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
-			drm_err(display->drm,
-				"[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
-				plane->base.base.id, plane->base.name,
-				sw_wm_level->enable,
-				sw_wm_level->blocks,
-				sw_wm_level->lines,
-				hw_wm_level->enable,
-				hw_wm_level->blocks,
-				hw_wm_level->lines);
-		}
+		skl_wm_level_verify(plane, "trans WM",
+				    &hw_plane_wm->trans_wm,
+				    skl_plane_trans_wm(sw_wm, plane->id));
 
-		hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
-		sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
-
-		if (HAS_HW_SAGV_WM(display) &&
-		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
-			drm_err(display->drm,
-				"[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
-				plane->base.base.id, plane->base.name,
-				sw_wm_level->enable,
-				sw_wm_level->blocks,
-				sw_wm_level->lines,
-				hw_wm_level->enable,
-				hw_wm_level->blocks,
-				hw_wm_level->lines);
-		}
+		if (HAS_HW_SAGV_WM(display)) {
+			skl_wm_level_verify(plane, "SAGV WM",
+					    &hw_plane_wm->sagv.wm0,
+					    &sw_plane_wm->sagv.wm0);
 
-		hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
-		sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
-
-		if (HAS_HW_SAGV_WM(display) &&
-		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
-			drm_err(display->drm,
-				"[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
-				plane->base.base.id, plane->base.name,
-				sw_wm_level->enable,
-				sw_wm_level->blocks,
-				sw_wm_level->lines,
-				hw_wm_level->enable,
-				hw_wm_level->blocks,
-				hw_wm_level->lines);
+			skl_wm_level_verify(plane, "SAGV trans WM",
+					    &hw_plane_wm->sagv.trans_wm,
+					    &sw_plane_wm->sagv.trans_wm);
 		}
 
-		/* DDB */
-		hw_ddb_entry = &hw->ddb[plane->id];
-		sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[plane->id];
+		skl_ddb_entry_verify(plane, "DDB",
+				     &hw->ddb[plane->id],
+				     &new_crtc_state->wm.skl.plane_ddb[plane->id]);
 
-		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
-			drm_err(display->drm,
-				"[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
-				plane->base.base.id, plane->base.name,
-				sw_ddb_entry->start, sw_ddb_entry->end,
-				hw_ddb_entry->start, hw_ddb_entry->end);
-		}
+		skl_ddb_entry_verify(plane, "DDB Y",
+				     &hw->ddb_y[plane->id],
+				     &new_crtc_state->wm.skl.plane_ddb_y[plane->id]);
 	}
 
 	kfree(hw);
@@ -4055,7 +3995,7 @@ void skl_wm_init(struct intel_display *display)
 
 	skl_setup_wm_latency(display);
 
-	display->funcs.wm = &skl_wm_funcs;
+	display->wm.funcs = &skl_wm_funcs;
 }
 
 static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
diff --git a/drivers/gpu/drm/i915/display/vlv_clock.c b/drivers/gpu/drm/i915/display/vlv_clock.c
index 1abdae453514e..a589cdb749033 100644
--- a/drivers/gpu/drm/i915/display/vlv_clock.c
+++ b/drivers/gpu/drm/i915/display/vlv_clock.c
@@ -22,11 +22,11 @@ int vlv_clock_get_hpll_vco(struct drm_device *drm)
 	int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
 
 	if (!display->vlv_clock.hpll_freq) {
-		vlv_cck_get(drm);
+		vlv_cck_get(display);
 		/* Obtain SKU information */
-		hpll_freq = vlv_cck_read(drm, CCK_FUSE_REG) &
+		hpll_freq = vlv_cck_read(display, CCK_FUSE_REG) &
 			CCK_FUSE_HPLL_FREQ_MASK;
-		vlv_cck_put(drm);
+		vlv_cck_put(display);
 
 		display->vlv_clock.hpll_freq = vco_freq[hpll_freq] * 1000;
 
@@ -39,12 +39,13 @@ int vlv_clock_get_hpll_vco(struct drm_device *drm)
 static int vlv_clock_get_cck(struct drm_device *drm,
 			     const char *name, u32 reg, int ref_freq)
 {
+	struct intel_display *display = to_intel_display(drm);
 	u32 val;
 	int divider;
 
-	vlv_cck_get(drm);
-	val = vlv_cck_read(drm, reg);
-	vlv_cck_put(drm);
+	vlv_cck_get(display);
+	val = vlv_cck_read(display, reg);
+	vlv_cck_put(display);
 
 	divider = val & CCK_FREQUENCY_VALUES;
 
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi.c b/drivers/gpu/drm/i915/display/vlv_dsi.c
index d4db73c184e5c..877eab75f19af 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi.c
@@ -99,7 +99,7 @@ void vlv_dsi_wait_for_fifo_empty(struct intel_dsi *intel_dsi, enum port port)
 }
 
 static void write_data(struct intel_display *display,
-		       i915_reg_t reg,
+		       intel_reg_t reg,
 		       const u8 *data, u32 len)
 {
 	u32 i, j;
@@ -115,7 +115,7 @@ static void write_data(struct intel_display *display,
 }
 
 static void read_data(struct intel_display *display,
-		      i915_reg_t reg,
+		      intel_reg_t reg,
 		      u8 *data, u32 len)
 {
 	u32 i, j;
@@ -138,7 +138,7 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
 	struct mipi_dsi_packet packet;
 	ssize_t ret;
 	const u8 *header;
-	i915_reg_t data_reg, ctrl_reg;
+	intel_reg_t data_reg, ctrl_reg;
 	u32 data_mask, ctrl_mask;
 
 	ret = mipi_dsi_create_packet(&packet, msg);
@@ -254,16 +254,16 @@ static int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs,
 
 static void band_gap_reset(struct intel_display *display)
 {
-	vlv_flisdsi_get(display->drm);
+	vlv_flisdsi_get(display);
 
-	vlv_flisdsi_write(display->drm, 0x08, 0x0001);
-	vlv_flisdsi_write(display->drm, 0x0F, 0x0005);
-	vlv_flisdsi_write(display->drm, 0x0F, 0x0025);
+	vlv_flisdsi_write(display, 0x08, 0x0001);
+	vlv_flisdsi_write(display, 0x0F, 0x0005);
+	vlv_flisdsi_write(display, 0x0F, 0x0025);
 	udelay(150);
-	vlv_flisdsi_write(display->drm, 0x0F, 0x0000);
-	vlv_flisdsi_write(display->drm, 0x08, 0x0000);
+	vlv_flisdsi_write(display, 0x0F, 0x0000);
+	vlv_flisdsi_write(display, 0x08, 0x0000);
 
-	vlv_flisdsi_put(display->drm);
+	vlv_flisdsi_put(display);
 }
 
 static int intel_dsi_compute_config(struct intel_encoder *encoder,
@@ -461,11 +461,11 @@ static void vlv_dsi_device_ready(struct intel_encoder *encoder)
 
 	drm_dbg_kms(display->drm, "\n");
 
-	vlv_flisdsi_get(display->drm);
+	vlv_flisdsi_get(display);
 	/* program rcomp for compliance, reduce from 50 ohms to 45 ohms
 	 * needed everytime after power gate */
-	vlv_flisdsi_write(display->drm, 0x04, 0x0004);
-	vlv_flisdsi_put(display->drm);
+	vlv_flisdsi_write(display, 0x04, 0x0004);
+	vlv_flisdsi_put(display);
 
 	/* bandgap reset is needed after everytime we do power gate */
 	band_gap_reset(display);
@@ -559,7 +559,7 @@ static void glk_dsi_clear_device_ready(struct intel_encoder *encoder)
 	glk_dsi_disable_mipi_io(encoder);
 }
 
-static i915_reg_t port_ctrl_reg(struct intel_display *display, enum port port)
+static intel_reg_t port_ctrl_reg(struct intel_display *display, enum port port)
 {
 	return display->platform.geminilake || display->platform.broxton ?
 		BXT_MIPI_PORT_CTRL(port) : VLV_MIPI_PORT_CTRL(port);
@@ -574,7 +574,7 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
 	drm_dbg_kms(display->drm, "\n");
 	for_each_dsi_port(port, intel_dsi->ports) {
 		/* Common bit for both MIPI Port A & MIPI Port C on VLV/CHV */
-		i915_reg_t port_ctrl = display->platform.broxton ?
+		intel_reg_t port_ctrl = display->platform.broxton ?
 			BXT_MIPI_PORT_CTRL(port) : VLV_MIPI_PORT_CTRL(PORT_A);
 
 		intel_de_write(display, MIPI_DEVICE_READY(display, port),
@@ -631,7 +631,7 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder,
 	}
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		i915_reg_t port_ctrl = port_ctrl_reg(display, port);
+		intel_reg_t port_ctrl = port_ctrl_reg(display, port);
 		u32 temp;
 
 		temp = intel_de_read(display, port_ctrl);
@@ -666,7 +666,7 @@ static void intel_dsi_port_disable(struct intel_encoder *encoder)
 	enum port port;
 
 	for_each_dsi_port(port, intel_dsi->ports) {
-		i915_reg_t port_ctrl = port_ctrl_reg(display, port);
+		intel_reg_t port_ctrl = port_ctrl_reg(display, port);
 
 		/* de-assert ip_tg_enable signal */
 		intel_de_rmw(display, port_ctrl, DPI_ENABLE, 0);
@@ -957,7 +957,7 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
 
 	/* XXX: this only works for one DSI output */
 	for_each_dsi_port(port, intel_dsi->ports) {
-		i915_reg_t port_ctrl = port_ctrl_reg(display, port);
+		intel_reg_t port_ctrl = port_ctrl_reg(display, port);
 		bool enabled = intel_de_read(display, port_ctrl) & DPI_ENABLE;
 
 		/*
diff --git a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
index a2da6285890b9..68b73cca2aece 100644
--- a/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
+++ b/drivers/gpu/drm/i915/display/vlv_dsi_pll.c
@@ -219,11 +219,11 @@ void vlv_dsi_pll_enable(struct intel_encoder *encoder,
 
 	drm_dbg_kms(display->drm, "\n");
 
-	vlv_cck_get(display->drm);
+	vlv_cck_get(display);
 
-	vlv_cck_write(display->drm, CCK_REG_DSI_PLL_CONTROL, 0);
-	vlv_cck_write(display->drm, CCK_REG_DSI_PLL_DIVIDER, config->dsi_pll.div);
-	vlv_cck_write(display->drm, CCK_REG_DSI_PLL_CONTROL,
+	vlv_cck_write(display, CCK_REG_DSI_PLL_CONTROL, 0);
+	vlv_cck_write(display, CCK_REG_DSI_PLL_DIVIDER, config->dsi_pll.div);
+	vlv_cck_write(display, CCK_REG_DSI_PLL_CONTROL,
 		      config->dsi_pll.ctrl & ~DSI_PLL_VCO_EN);
 
 	/* wait at least 0.5 us after ungating before enabling VCO,
@@ -231,17 +231,17 @@ void vlv_dsi_pll_enable(struct intel_encoder *encoder,
 	 */
 	usleep_range(10, 50);
 
-	vlv_cck_write(display->drm, CCK_REG_DSI_PLL_CONTROL, config->dsi_pll.ctrl);
+	vlv_cck_write(display, CCK_REG_DSI_PLL_CONTROL, config->dsi_pll.ctrl);
 
-	ret = poll_timeout_us(val = vlv_cck_read(display->drm, CCK_REG_DSI_PLL_CONTROL),
+	ret = poll_timeout_us(val = vlv_cck_read(display, CCK_REG_DSI_PLL_CONTROL),
 			      val & DSI_PLL_LOCK,
 			      500, 20 * 1000, false);
 	if (ret) {
-		vlv_cck_put(display->drm);
+		vlv_cck_put(display);
 		drm_err(display->drm, "DSI PLL lock failed\n");
 		return;
 	}
-	vlv_cck_put(display->drm);
+	vlv_cck_put(display);
 
 	drm_dbg_kms(display->drm, "DSI PLL locked\n");
 }
@@ -253,14 +253,14 @@ void vlv_dsi_pll_disable(struct intel_encoder *encoder)
 
 	drm_dbg_kms(display->drm, "\n");
 
-	vlv_cck_get(display->drm);
+	vlv_cck_get(display);
 
-	tmp = vlv_cck_read(display->drm, CCK_REG_DSI_PLL_CONTROL);
+	tmp = vlv_cck_read(display, CCK_REG_DSI_PLL_CONTROL);
 	tmp &= ~DSI_PLL_VCO_EN;
 	tmp |= DSI_PLL_LDO_GATE;
-	vlv_cck_write(display->drm, CCK_REG_DSI_PLL_CONTROL, tmp);
+	vlv_cck_write(display, CCK_REG_DSI_PLL_CONTROL, tmp);
 
-	vlv_cck_put(display->drm);
+	vlv_cck_put(display);
 }
 
 static bool has_dsic_clock(struct intel_display *display)
@@ -333,10 +333,10 @@ u32 vlv_dsi_get_pclk(struct intel_encoder *encoder,
 
 	drm_dbg_kms(display->drm, "\n");
 
-	vlv_cck_get(display->drm);
-	pll_ctl = vlv_cck_read(display->drm, CCK_REG_DSI_PLL_CONTROL);
-	pll_div = vlv_cck_read(display->drm, CCK_REG_DSI_PLL_DIVIDER);
-	vlv_cck_put(display->drm);
+	vlv_cck_get(display);
+	pll_ctl = vlv_cck_read(display, CCK_REG_DSI_PLL_CONTROL);
+	pll_div = vlv_cck_read(display, CCK_REG_DSI_PLL_DIVIDER);
+	vlv_cck_put(display);
 
 	config->dsi_pll.ctrl = pll_ctl & ~DSI_PLL_LOCK;
 	config->dsi_pll.div = pll_div;
@@ -603,9 +603,9 @@ static void assert_dsi_pll(struct intel_display *display, bool state)
 {
 	bool cur_state;
 
-	vlv_cck_get(display->drm);
-	cur_state = vlv_cck_read(display->drm, CCK_REG_DSI_PLL_CONTROL) & DSI_PLL_VCO_EN;
-	vlv_cck_put(display->drm);
+	vlv_cck_get(display);
+	cur_state = vlv_cck_read(display, CCK_REG_DSI_PLL_CONTROL) & DSI_PLL_VCO_EN;
+	vlv_cck_put(display);
 
 	INTEL_DISPLAY_STATE_WARN(display, cur_state != state,
 				 "DSI PLL state assertion failure (expected %s, current %s)\n",
diff --git a/drivers/gpu/drm/i915/display/vlv_sideband.c b/drivers/gpu/drm/i915/display/vlv_sideband.c
index e18045f2b89d2..068f58bd9a2b2 100644
--- a/drivers/gpu/drm/i915/display/vlv_sideband.c
+++ b/drivers/gpu/drm/i915/display/vlv_sideband.c
@@ -6,8 +6,74 @@
 #include "intel_display_core.h"
 #include "intel_display_types.h"
 #include "intel_dpio_phy.h"
+#include "intel_parent.h"
 #include "vlv_sideband.h"
 
+void vlv_bunit_get(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_get(display, BIT(VLV_IOSF_SB_BUNIT));
+}
+
+u32 vlv_bunit_read(struct intel_display *display, u32 reg)
+{
+	return intel_parent_vlv_iosf_read(display, VLV_IOSF_SB_BUNIT, reg);
+}
+
+void vlv_bunit_write(struct intel_display *display, u32 reg, u32 val)
+{
+	intel_parent_vlv_iosf_write(display, VLV_IOSF_SB_BUNIT, reg, val);
+}
+
+void vlv_bunit_put(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_put(display, BIT(VLV_IOSF_SB_BUNIT));
+}
+
+void vlv_cck_get(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_get(display, BIT(VLV_IOSF_SB_CCK));
+}
+
+u32 vlv_cck_read(struct intel_display *display, u32 reg)
+{
+	return intel_parent_vlv_iosf_read(display, VLV_IOSF_SB_CCK, reg);
+}
+
+void vlv_cck_write(struct intel_display *display, u32 reg, u32 val)
+{
+	intel_parent_vlv_iosf_write(display, VLV_IOSF_SB_CCK, reg, val);
+}
+
+void vlv_cck_put(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_put(display, BIT(VLV_IOSF_SB_CCK));
+}
+
+void vlv_ccu_get(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_get(display, BIT(VLV_IOSF_SB_CCU));
+}
+
+u32 vlv_ccu_read(struct intel_display *display, u32 reg)
+{
+	return intel_parent_vlv_iosf_read(display, VLV_IOSF_SB_CCU, reg);
+}
+
+void vlv_ccu_write(struct intel_display *display, u32 reg, u32 val)
+{
+	intel_parent_vlv_iosf_write(display, VLV_IOSF_SB_CCU, reg, val);
+}
+
+void vlv_ccu_put(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_put(display, BIT(VLV_IOSF_SB_CCU));
+}
+
+void vlv_dpio_get(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_get(display, BIT(VLV_IOSF_SB_DPIO) | BIT(VLV_IOSF_SB_DPIO_2));
+}
+
 static enum vlv_iosf_sb_unit vlv_dpio_phy_to_unit(struct intel_display *display,
 						  enum dpio_phy phy)
 {
@@ -21,13 +87,12 @@ static enum vlv_iosf_sb_unit vlv_dpio_phy_to_unit(struct intel_display *display,
 		return VLV_IOSF_SB_DPIO;
 }
 
-u32 vlv_dpio_read(struct drm_device *drm, enum dpio_phy phy, int reg)
+u32 vlv_dpio_read(struct intel_display *display, enum dpio_phy phy, int reg)
 {
-	struct intel_display *display = to_intel_display(drm);
 	enum vlv_iosf_sb_unit unit = vlv_dpio_phy_to_unit(display, phy);
 	u32 val;
 
-	val = vlv_iosf_sb_read(drm, unit, reg);
+	val = intel_parent_vlv_iosf_read(display, unit, reg);
 
 	/*
 	 * FIXME: There might be some registers where all 1's is a valid value,
@@ -40,11 +105,70 @@ u32 vlv_dpio_read(struct drm_device *drm, enum dpio_phy phy, int reg)
 	return val;
 }
 
-void vlv_dpio_write(struct drm_device *drm,
+void vlv_dpio_write(struct intel_display *display,
 		    enum dpio_phy phy, int reg, u32 val)
 {
-	struct intel_display *display = to_intel_display(drm);
 	enum vlv_iosf_sb_unit unit = vlv_dpio_phy_to_unit(display, phy);
 
-	vlv_iosf_sb_write(drm, unit, reg, val);
+	intel_parent_vlv_iosf_write(display, unit, reg, val);
+}
+
+void vlv_dpio_put(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_put(display, BIT(VLV_IOSF_SB_DPIO) | BIT(VLV_IOSF_SB_DPIO_2));
+}
+
+void vlv_flisdsi_get(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_get(display, BIT(VLV_IOSF_SB_FLISDSI));
+}
+
+u32 vlv_flisdsi_read(struct intel_display *display, u32 reg)
+{
+	return intel_parent_vlv_iosf_read(display, VLV_IOSF_SB_FLISDSI, reg);
+}
+
+void vlv_flisdsi_write(struct intel_display *display, u32 reg, u32 val)
+{
+	intel_parent_vlv_iosf_write(display, VLV_IOSF_SB_FLISDSI, reg, val);
+}
+
+void vlv_flisdsi_put(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_put(display, BIT(VLV_IOSF_SB_FLISDSI));
+}
+
+void vlv_nc_get(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_get(display, BIT(VLV_IOSF_SB_NC));
+}
+
+u32 vlv_nc_read(struct intel_display *display, u8 addr)
+{
+	return intel_parent_vlv_iosf_read(display, VLV_IOSF_SB_NC, addr);
+}
+
+void vlv_nc_put(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_put(display, BIT(VLV_IOSF_SB_NC));
+}
+
+void vlv_punit_get(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_get(display, BIT(VLV_IOSF_SB_PUNIT));
+}
+
+u32 vlv_punit_read(struct intel_display *display, u32 addr)
+{
+	return intel_parent_vlv_iosf_read(display, VLV_IOSF_SB_PUNIT, addr);
+}
+
+int vlv_punit_write(struct intel_display *display, u32 addr, u32 val)
+{
+	return intel_parent_vlv_iosf_write(display, VLV_IOSF_SB_PUNIT, addr, val);
+}
+
+void vlv_punit_put(struct intel_display *display)
+{
+	intel_parent_vlv_iosf_put(display, BIT(VLV_IOSF_SB_PUNIT));
 }
diff --git a/drivers/gpu/drm/i915/display/vlv_sideband.h b/drivers/gpu/drm/i915/display/vlv_sideband.h
index 2c240d81fead6..60a66abc1649c 100644
--- a/drivers/gpu/drm/i915/display/vlv_sideband.h
+++ b/drivers/gpu/drm/i915/display/vlv_sideband.h
@@ -4,153 +4,45 @@
 #ifndef _VLV_SIDEBAND_H_
 #define _VLV_SIDEBAND_H_
 
-#include <linux/bitops.h>
 #include <linux/types.h>
 
-#include "vlv_iosf_sb.h"
-#include "vlv_iosf_sb_reg.h"
+#include <drm/intel/vlv_iosf_sb_regs.h>
 
 enum dpio_phy;
-struct drm_device;
-
-static inline void vlv_bunit_get(struct drm_device *drm)
-{
-	vlv_iosf_sb_get(drm, BIT(VLV_IOSF_SB_BUNIT));
-}
-
-static inline u32 vlv_bunit_read(struct drm_device *drm, u32 reg)
-{
-	return vlv_iosf_sb_read(drm, VLV_IOSF_SB_BUNIT, reg);
-}
-
-static inline void vlv_bunit_write(struct drm_device *drm, u32 reg, u32 val)
-{
-	vlv_iosf_sb_write(drm, VLV_IOSF_SB_BUNIT, reg, val);
-}
-
-static inline void vlv_bunit_put(struct drm_device *drm)
-{
-	vlv_iosf_sb_put(drm, BIT(VLV_IOSF_SB_BUNIT));
-}
-
-static inline void vlv_cck_get(struct drm_device *drm)
-{
-	vlv_iosf_sb_get(drm, BIT(VLV_IOSF_SB_CCK));
-}
-
-static inline u32 vlv_cck_read(struct drm_device *drm, u32 reg)
-{
-	return vlv_iosf_sb_read(drm, VLV_IOSF_SB_CCK, reg);
-}
-
-static inline void vlv_cck_write(struct drm_device *drm, u32 reg, u32 val)
-{
-	vlv_iosf_sb_write(drm, VLV_IOSF_SB_CCK, reg, val);
-}
-
-static inline void vlv_cck_put(struct drm_device *drm)
-{
-	vlv_iosf_sb_put(drm, BIT(VLV_IOSF_SB_CCK));
-}
-
-static inline void vlv_ccu_get(struct drm_device *drm)
-{
-	vlv_iosf_sb_get(drm, BIT(VLV_IOSF_SB_CCU));
-}
-
-static inline u32 vlv_ccu_read(struct drm_device *drm, u32 reg)
-{
-	return vlv_iosf_sb_read(drm, VLV_IOSF_SB_CCU, reg);
-}
-
-static inline void vlv_ccu_write(struct drm_device *drm, u32 reg, u32 val)
-{
-	vlv_iosf_sb_write(drm, VLV_IOSF_SB_CCU, reg, val);
-}
-
-static inline void vlv_ccu_put(struct drm_device *drm)
-{
-	vlv_iosf_sb_put(drm, BIT(VLV_IOSF_SB_CCU));
-}
-
-static inline void vlv_dpio_get(struct drm_device *drm)
-{
-	vlv_iosf_sb_get(drm, BIT(VLV_IOSF_SB_DPIO) | BIT(VLV_IOSF_SB_DPIO_2));
-}
-
-#ifdef I915
-u32 vlv_dpio_read(struct drm_device *drm, enum dpio_phy phy, int reg);
-void vlv_dpio_write(struct drm_device *drm,
-		    enum dpio_phy phy, int reg, u32 val);
-#else
-static inline u32 vlv_dpio_read(struct drm_device *drm, int phy, int reg)
-{
-	return 0;
-}
-static inline void vlv_dpio_write(struct drm_device *drm,
-				  int phy, int reg, u32 val)
-{
-}
-#endif
-
-static inline void vlv_dpio_put(struct drm_device *drm)
-{
-	vlv_iosf_sb_put(drm, BIT(VLV_IOSF_SB_DPIO) | BIT(VLV_IOSF_SB_DPIO_2));
-}
-
-static inline void vlv_flisdsi_get(struct drm_device *drm)
-{
-	vlv_iosf_sb_get(drm, BIT(VLV_IOSF_SB_FLISDSI));
-}
-
-static inline u32 vlv_flisdsi_read(struct drm_device *drm, u32 reg)
-{
-	return vlv_iosf_sb_read(drm, VLV_IOSF_SB_FLISDSI, reg);
-}
-
-static inline void vlv_flisdsi_write(struct drm_device *drm, u32 reg, u32 val)
-{
-	vlv_iosf_sb_write(drm, VLV_IOSF_SB_FLISDSI, reg, val);
-}
-
-static inline void vlv_flisdsi_put(struct drm_device *drm)
-{
-	vlv_iosf_sb_put(drm, BIT(VLV_IOSF_SB_FLISDSI));
-}
-
-static inline void vlv_nc_get(struct drm_device *drm)
-{
-	vlv_iosf_sb_get(drm, BIT(VLV_IOSF_SB_NC));
-}
-
-static inline u32 vlv_nc_read(struct drm_device *drm, u8 addr)
-{
-	return vlv_iosf_sb_read(drm, VLV_IOSF_SB_NC, addr);
-}
-
-static inline void vlv_nc_put(struct drm_device *drm)
-{
-	vlv_iosf_sb_put(drm, BIT(VLV_IOSF_SB_NC));
-}
-
-static inline void vlv_punit_get(struct drm_device *drm)
-{
-	vlv_iosf_sb_get(drm, BIT(VLV_IOSF_SB_PUNIT));
-}
-
-static inline u32 vlv_punit_read(struct drm_device *drm, u32 addr)
-{
-	return vlv_iosf_sb_read(drm, VLV_IOSF_SB_PUNIT, addr);
-}
-
-static inline int vlv_punit_write(struct drm_device *drm, u32 addr, u32 val)
-{
-	return vlv_iosf_sb_write(drm, VLV_IOSF_SB_PUNIT, addr, val);
-}
-
-static inline void vlv_punit_put(struct drm_device *drm)
-{
-	vlv_iosf_sb_put(drm, BIT(VLV_IOSF_SB_PUNIT));
-}
+struct intel_display;
+
+void vlv_bunit_get(struct intel_display *display);
+u32 vlv_bunit_read(struct intel_display *display, u32 reg);
+void vlv_bunit_write(struct intel_display *display, u32 reg, u32 val);
+void vlv_bunit_put(struct intel_display *display);
+
+void vlv_cck_get(struct intel_display *display);
+u32 vlv_cck_read(struct intel_display *display, u32 reg);
+void vlv_cck_write(struct intel_display *display, u32 reg, u32 val);
+void vlv_cck_put(struct intel_display *display);
+
+void vlv_ccu_get(struct intel_display *display);
+u32 vlv_ccu_read(struct intel_display *display, u32 reg);
+void vlv_ccu_write(struct intel_display *display, u32 reg, u32 val);
+void vlv_ccu_put(struct intel_display *display);
+
+void vlv_dpio_get(struct intel_display *display);
+u32 vlv_dpio_read(struct intel_display *display, enum dpio_phy phy, int reg);
+void vlv_dpio_write(struct intel_display *display, enum dpio_phy phy, int reg, u32 val);
+void vlv_dpio_put(struct intel_display *display);
+
+void vlv_flisdsi_get(struct intel_display *display);
+u32 vlv_flisdsi_read(struct intel_display *display, u32 reg);
+void vlv_flisdsi_write(struct intel_display *display, u32 reg, u32 val);
+void vlv_flisdsi_put(struct intel_display *display);
+
+void vlv_nc_get(struct intel_display *display);
+u32 vlv_nc_read(struct intel_display *display, u8 addr);
+void vlv_nc_put(struct intel_display *display);
+
+void vlv_punit_get(struct intel_display *display);
+u32 vlv_punit_read(struct intel_display *display, u32 addr);
+int vlv_punit_write(struct intel_display *display, u32 addr, u32 val);
+void vlv_punit_put(struct intel_display *display);
 
 #endif /* _VLV_SIDEBAND_H_ */
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
index 65ce54b20ec29..05997e8bbb29a 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c
@@ -3205,8 +3205,7 @@ eb_composite_fence_create(struct i915_execbuffer *eb, int out_fence_fd)
 	fence_array = dma_fence_array_create(eb->num_batches,
 					     fences,
 					     eb->context->parallel.fence_context,
-					     eb->context->parallel.seqno++,
-					     false);
+					     eb->context->parallel.seqno++);
 	if (!fence_array) {
 		kfree(fences);
 		return ERR_PTR(-ENOMEM);
diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
index 77f85359f279f..1cfdcf5c11186 100644
--- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c
@@ -10,6 +10,8 @@
 #include <drm/drm_print.h>
 #include <drm/intel/display_parent_interface.h>
 #include <drm/intel/i915_drm.h>
+#include <drm/intel/mchbar_regs.h>
+#include <drm/intel/pci_config.h>
 
 #include "gem/i915_gem_lmem.h"
 #include "gem/i915_gem_region.h"
@@ -23,8 +25,6 @@
 #include "i915_reg.h"
 #include "i915_utils.h"
 #include "i915_vgpu.h"
-#include "intel_mchbar_regs.h"
-#include "intel_pci_config.h"
 
 struct intel_stolen_node {
 	struct drm_i915_private *i915;
diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c
index 08c4e735481b0..64ca5bbc53c68 100644
--- a/drivers/gpu/drm/i915/gt/intel_ggtt.c
+++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c
@@ -12,6 +12,7 @@
 #include <drm/drm_print.h>
 #include <drm/intel/i915_drm.h>
 #include <drm/intel/intel-gtt.h>
+#include <drm/intel/pci_config.h>
 
 #include "gem/i915_gem_lmem.h"
 
@@ -20,7 +21,6 @@
 #include "intel_gpu_commands.h"
 #include "intel_gt.h"
 #include "intel_gt_regs.h"
-#include "intel_pci_config.h"
 #include "intel_ring.h"
 #include "i915_drv.h"
 #include "i915_pci.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c
index ac9aede823203..6efc1ed3831b4 100644
--- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c
+++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c
@@ -7,6 +7,7 @@
 
 #include <drm/drm_print.h>
 #include <drm/intel/intel_gmd_misc_regs.h>
+#include <drm/intel/mchbar_regs.h>
 
 #include "display/intel_display.h"
 #include "i915_drv.h"
@@ -15,7 +16,6 @@
 #include "i915_pvinfo.h"
 #include "i915_vgpu.h"
 #include "intel_gt_regs.h"
-#include "intel_mchbar_regs.h"
 
 /**
  * DOC: fence register handling
diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c
index d76121e117e17..5c7f862f7100a 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt.c
@@ -6,6 +6,7 @@
 #include <drm/drm_managed.h>
 #include <drm/intel/intel-gtt.h>
 #include <drm/intel/intel_gmd_interrupt_regs.h>
+#include <drm/intel/pci_config.h>
 
 #include "gem/i915_gem_internal.h"
 #include "gem/i915_gem_lmem.h"
@@ -28,7 +29,6 @@
 #include "intel_gt_requests.h"
 #include "intel_migrate.h"
 #include "intel_mocs.h"
-#include "intel_pci_config.h"
 #include "intel_rc6.h"
 #include "intel_renderstate.h"
 #include "intel_rps.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
index babaf16e72f27..adf2512e2ed52 100644
--- a/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
+++ b/drivers/gpu/drm/i915/gt/intel_gt_pm_debugfs.c
@@ -19,7 +19,6 @@
 #include "intel_gt_pm_debugfs.h"
 #include "intel_gt_regs.h"
 #include "intel_llc.h"
-#include "intel_mchbar_regs.h"
 #include "intel_pcode.h"
 #include "intel_rc6.h"
 #include "intel_rps.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_llc.c b/drivers/gpu/drm/i915/gt/intel_llc.c
index bcd707e3d4364..82d9a8e50867a 100644
--- a/drivers/gpu/drm/i915/gt/intel_llc.c
+++ b/drivers/gpu/drm/i915/gt/intel_llc.c
@@ -7,12 +7,12 @@
 #include <linux/cpufreq.h>
 
 #include <drm/intel/intel_pcode_regs.h>
+#include <drm/intel/mchbar_regs.h>
 
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "intel_gt.h"
 #include "intel_llc.h"
-#include "intel_mchbar_regs.h"
 #include "intel_pcode.h"
 #include "intel_rps.h"
 
diff --git a/drivers/gpu/drm/i915/gt/intel_region_lmem.c b/drivers/gpu/drm/i915/gt/intel_region_lmem.c
index a30060fd44292..b8a39567a334b 100644
--- a/drivers/gpu/drm/i915/gt/intel_region_lmem.c
+++ b/drivers/gpu/drm/i915/gt/intel_region_lmem.c
@@ -4,12 +4,12 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/pci_config.h>
 
 #include "i915_drv.h"
 #include "i915_pci.h"
 #include "i915_reg.h"
 #include "intel_memory_region.h"
-#include "intel_pci_config.h"
 #include "intel_region_lmem.h"
 #include "intel_region_ttm.h"
 #include "gem/i915_gem_lmem.h"
diff --git a/drivers/gpu/drm/i915/gt/intel_reset.c b/drivers/gpu/drm/i915/gt/intel_reset.c
index adff482a6c9cd..b2cf672564dd9 100644
--- a/drivers/gpu/drm/i915/gt/intel_reset.c
+++ b/drivers/gpu/drm/i915/gt/intel_reset.c
@@ -7,6 +7,9 @@
 #include <linux/stop_machine.h>
 #include <linux/string_helpers.h>
 
+#include <drm/intel/mchbar_regs.h>
+#include <drm/intel/pci_config.h>
+
 #include "display/intel_display_reset.h"
 #include "display/intel_overlay.h"
 #include "gem/i915_gem_context.h"
@@ -27,8 +30,6 @@
 #include "intel_gt_pm.h"
 #include "intel_gt_print.h"
 #include "intel_gt_requests.h"
-#include "intel_mchbar_regs.h"
-#include "intel_pci_config.h"
 #include "intel_reset.h"
 
 #define RESET_MAX_RETRIES 3
@@ -967,6 +968,7 @@ static void nop_submit_request(struct i915_request *request)
 
 static void __intel_gt_set_wedged(struct intel_gt *gt)
 {
+	struct intel_display *display = gt->i915->display;
 	struct intel_engine_cs *engine;
 	intel_engine_mask_t awake;
 	enum intel_engine_id id;
@@ -984,7 +986,8 @@ static void __intel_gt_set_wedged(struct intel_gt *gt)
 	awake = reset_prepare(gt);
 
 	/* Even if the GPU reset fails, it should still stop the engines */
-	if (!intel_gt_gpu_reset_clobbers_display(gt))
+	if (!intel_gt_gpu_reset_clobbers_display(gt) &&
+	    !intel_display_reset_test(display))
 		intel_gt_reset_all_engines(gt);
 
 	for_each_engine(engine, gt, id)
@@ -1398,11 +1401,6 @@ int intel_engine_reset(struct intel_engine_cs *engine, const char *msg)
 	return err;
 }
 
-static void display_reset_modeset_stuck(void *gt)
-{
-	intel_gt_set_wedged(gt);
-}
-
 static void intel_gt_reset_global(struct intel_gt *gt,
 				  u32 engine_mask,
 				  const char *reason)
@@ -1425,16 +1423,25 @@ static void intel_gt_reset_global(struct intel_gt *gt,
 		bool need_display_reset;
 		bool reset_display;
 
-		need_display_reset = intel_gt_gpu_reset_clobbers_display(gt) &&
+		need_display_reset =
+			intel_display_reset_supported(display) &&
+			intel_gt_gpu_reset_clobbers_display(gt) &&
 			intel_has_gpu_reset(gt);
 
-		reset_display = intel_display_reset_test(display) ||
+		reset_display =
+			intel_display_reset_test(display) ||
 			need_display_reset;
 
-		if (reset_display)
-			reset_display = intel_display_reset_prepare(display,
-								    display_reset_modeset_stuck,
-								    gt);
+		if (reset_display) {
+			if (atomic_read(&i915->pending_fb_pin)) {
+				drm_dbg_kms(&i915->drm,
+					    "Modeset potentially stuck, unbreaking through wedging\n");
+
+				intel_gt_set_wedged(gt);
+			}
+
+			intel_display_reset_prepare(display);
+		}
 
 		intel_gt_reset(gt, engine_mask, reason);
 
@@ -1502,9 +1509,10 @@ void intel_gt_handle_error(struct intel_gt *gt,
 
 	/*
 	 * Try engine reset when available. We fall back to full reset if
-	 * single reset fails.
+	 * single reset fails. Display reset test needs a full reset.
 	 */
-	if (!intel_uc_uses_guc_submission(&gt->uc) &&
+	if (!intel_display_reset_test(gt->i915->display) &&
+	    !intel_uc_uses_guc_submission(&gt->uc) &&
 	    intel_has_reset_engine(gt) && !intel_gt_is_wedged(gt)) {
 		local_bh_disable();
 		for_each_engine_masked(engine, gt, engine_mask, tmp) {
diff --git a/drivers/gpu/drm/i915/gt/intel_rps.c b/drivers/gpu/drm/i915/gt/intel_rps.c
index 844f2716a386b..a33b19c04737a 100644
--- a/drivers/gpu/drm/i915/gt/intel_rps.c
+++ b/drivers/gpu/drm/i915/gt/intel_rps.c
@@ -5,9 +5,10 @@
 
 #include <linux/string_helpers.h>
 
-#include <drm/intel/i915_drm.h>
 #include <drm/intel/display_parent_interface.h>
+#include <drm/intel/i915_drm.h>
 #include <drm/intel/intel_pcode_regs.h>
+#include <drm/intel/mchbar_regs.h>
 
 #include "display/intel_display_rps.h"
 #include "display/vlv_clock.h"
@@ -25,7 +26,6 @@
 #include "intel_gt_pm_irq.h"
 #include "intel_gt_print.h"
 #include "intel_gt_regs.h"
-#include "intel_mchbar_regs.h"
 #include "intel_pcode.h"
 #include "intel_rps.h"
 #include "vlv_iosf_sb.h"
diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
index fa9af08f97081..a75f05ff1110e 100644
--- a/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
+++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_slpc.c
@@ -16,7 +16,6 @@
 #include "i915_wait_util.h"
 #include "intel_guc_print.h"
 #include "intel_guc_slpc.h"
-#include "intel_mchbar_regs.h"
 
 /**
  * DOC: SLPC - Dynamic Frequency management
diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c
index 1937e04d37915..e00c1478a24eb 100644
--- a/drivers/gpu/drm/i915/gvt/cfg_space.c
+++ b/drivers/gpu/drm/i915/gvt/cfg_space.c
@@ -32,10 +32,10 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/pci_config.h>
 
 #include "gvt.h"
 #include "i915_drv.h"
-#include "intel_pci_config.h"
 
 enum {
 	INTEL_GVT_PCI_BAR_GTTMMIO = 0,
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index a34f56630af98..9f61867e24781 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -71,7 +71,6 @@
 #include "i915_drv.h"
 #include "i915_pvinfo.h"
 #include "i915_reg.h"
-#include "intel_mchbar_regs.h"
 #include "sched_policy.h"
 
 /* XXX FIXME i915 has changed PP_XXX definition */
diff --git a/drivers/gpu/drm/i915/i915_bo.c b/drivers/gpu/drm/i915/i915_bo.c
index 1789f7cab05c4..559341103ca7d 100644
--- a/drivers/gpu/drm/i915/i915_bo.c
+++ b/drivers/gpu/drm/i915/i915_bo.c
@@ -1,11 +1,14 @@
 // SPDX-License-Identifier: MIT
 /* Copyright © 2024 Intel Corporation */
 
+#include <linux/fb.h>
+
 #include <drm/drm_panic.h>
 #include <drm/drm_print.h>
 #include <drm/intel/display_parent_interface.h>
 
 #include "display/intel_fb.h"
+#include "gem/i915_gem_lmem.h"
 #include "gem/i915_gem_mman.h"
 #include "gem/i915_gem_object.h"
 #include "gem/i915_gem_object_frontbuffer.h"
@@ -141,6 +144,111 @@ i915_bo_framebuffer_lookup(struct drm_device *drm,
 	return intel_bo_to_drm_bo(obj);
 }
 
+#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
+static u32 i915_bo_fbdev_pitch_align(u32 stride)
+{
+	return ALIGN(stride, 64);
+}
+
+bool i915_bo_fbdev_prefer_stolen(struct drm_i915_private *i915, unsigned int size)
+{
+	/* Skip stolen on MTL as Wa_22018444074 mitigation. */
+	if (IS_METEORLAKE(i915))
+		return false;
+
+	/*
+	 * If the FB is too big, just don't use it since fbdev is not very
+	 * important and we should probably use that space with FBC or other
+	 * features.
+	 */
+	return i915->dsm.usable_size >= size * 2;
+}
+
+static struct drm_gem_object *i915_bo_fbdev_create(struct drm_device *drm, int size)
+{
+	struct drm_i915_private *i915 = to_i915(drm);
+	struct drm_i915_gem_object *obj;
+
+	obj = ERR_PTR(-ENODEV);
+	if (HAS_LMEM(i915)) {
+		obj = i915_gem_object_create_lmem(i915, size,
+						  I915_BO_ALLOC_CONTIGUOUS |
+						  I915_BO_ALLOC_USER);
+	} else {
+		if (i915_bo_fbdev_prefer_stolen(i915, size))
+			obj = i915_gem_object_create_stolen(i915, size);
+		else
+			drm_info(drm, "Allocating fbdev: Stolen memory not preferred.\n");
+
+		if (IS_ERR(obj))
+			obj = i915_gem_object_create_shmem(i915, size);
+	}
+
+	if (IS_ERR(obj)) {
+		drm_err(drm, "failed to allocate framebuffer (%pe)\n", obj);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return &obj->base;
+}
+
+static void i915_bo_fbdev_destroy(struct drm_gem_object *obj)
+{
+	drm_gem_object_put(obj);
+}
+
+static int i915_bo_fbdev_fill_info(struct drm_gem_object *_obj, struct fb_info *info,
+				   struct i915_vma *vma)
+{
+	struct drm_i915_private *i915 = to_i915(_obj->dev);
+	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
+	struct i915_gem_ww_ctx ww;
+	void __iomem *vaddr;
+	int ret;
+
+	if (i915_gem_object_is_lmem(obj)) {
+		struct intel_memory_region *mem = obj->mm.region;
+
+		/* Use fbdev's framebuffer from lmem for discrete */
+		info->fix.smem_start =
+			(unsigned long)(mem->io.start +
+					i915_gem_object_get_dma_address(obj, 0) -
+					mem->region.start);
+		info->fix.smem_len = obj->base.size;
+	} else {
+		struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
+
+		/* Our framebuffer is the entirety of fbdev's system memory */
+		info->fix.smem_start =
+			(unsigned long)(ggtt->gmadr.start + i915_ggtt_offset(vma));
+		info->fix.smem_len = vma->size;
+	}
+
+	for_i915_gem_ww(&ww, ret, false) {
+		ret = i915_gem_object_lock(vma->obj, &ww);
+
+		if (ret)
+			continue;
+
+		vaddr = i915_vma_pin_iomap(vma);
+		if (IS_ERR(vaddr)) {
+			drm_err(&i915->drm,
+				"Failed to remap framebuffer into virtual memory (%pe)\n", vaddr);
+			ret = PTR_ERR(vaddr);
+			continue;
+		}
+	}
+
+	if (ret)
+		return ret;
+
+	info->screen_base = vaddr;
+	info->screen_size = intel_bo_to_drm_bo(obj)->size;
+
+	return 0;
+}
+#endif
+
 const struct intel_display_bo_interface i915_display_bo_interface = {
 	.is_tiled = i915_bo_is_tiled,
 	.is_userptr = i915_bo_is_userptr,
@@ -153,4 +261,10 @@ const struct intel_display_bo_interface i915_display_bo_interface = {
 	.framebuffer_init = i915_bo_framebuffer_init,
 	.framebuffer_fini = i915_bo_framebuffer_fini,
 	.framebuffer_lookup = i915_bo_framebuffer_lookup,
+#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
+	.fbdev_create = i915_bo_fbdev_create,
+	.fbdev_destroy = i915_bo_fbdev_destroy,
+	.fbdev_fill_info = i915_bo_fbdev_fill_info,
+	.fbdev_pitch_align = i915_bo_fbdev_pitch_align,
+#endif
 };
diff --git a/drivers/gpu/drm/i915/i915_bo.h b/drivers/gpu/drm/i915/i915_bo.h
index 57255d052dd9a..39ba62696550e 100644
--- a/drivers/gpu/drm/i915/i915_bo.h
+++ b/drivers/gpu/drm/i915/i915_bo.h
@@ -4,6 +4,12 @@
 #ifndef __I915_BO_H__
 #define __I915_BO_H__
 
+#include <linux/types.h>
+
+struct drm_i915_private;
+
+bool i915_bo_fbdev_prefer_stolen(struct drm_i915_private *i915, unsigned int size);
+
 extern const struct intel_display_bo_interface i915_display_bo_interface;
 
 #endif /* __I915_BO_H__ */
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 4778ba664ec75..a3e27f9e4f476 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -34,6 +34,7 @@
 #include <drm/drm_debugfs.h>
 #include <drm/drm_print.h>
 #include <drm/intel/intel_gmd_misc_regs.h>
+#include <drm/intel/mchbar_regs.h>
 
 #include "gem/i915_gem_context.h"
 #include "gt/intel_gt.h"
@@ -57,7 +58,6 @@
 #include "i915_reg.h"
 #include "i915_scheduler.h"
 #include "i915_wait_util.h"
-#include "intel_mchbar_regs.h"
 
 static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
 {
diff --git a/drivers/gpu/drm/i915/i915_dpt.c b/drivers/gpu/drm/i915/i915_dpt.c
index 9f47bb563c858..fcd7cced771d5 100644
--- a/drivers/gpu/drm/i915/i915_dpt.c
+++ b/drivers/gpu/drm/i915/i915_dpt.c
@@ -129,7 +129,6 @@ static void dpt_cleanup(struct i915_address_space *vm)
 struct i915_vma *i915_dpt_pin_to_ggtt(struct intel_dpt *dpt, unsigned int alignment)
 {
 	struct drm_i915_private *i915 = dpt->vm.i915;
-	struct intel_display *display = i915->display;
 	struct ref_tracker *wakeref;
 	struct i915_vma *vma;
 	void __iomem *iomem;
@@ -141,7 +140,7 @@ struct i915_vma *i915_dpt_pin_to_ggtt(struct intel_dpt *dpt, unsigned int alignm
 		pin_flags |= PIN_MAPPABLE;
 
 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
-	atomic_inc(&display->restore.pending_fb_pin);
+	atomic_inc(&i915->pending_fb_pin);
 
 	for_i915_gem_ww(&ww, err, true) {
 		err = i915_gem_object_lock(dpt->obj, &ww);
@@ -171,7 +170,7 @@ struct i915_vma *i915_dpt_pin_to_ggtt(struct intel_dpt *dpt, unsigned int alignm
 
 	dpt->obj->mm.dirty = true;
 
-	atomic_dec(&display->restore.pending_fb_pin);
+	atomic_dec(&i915->pending_fb_pin);
 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 
 	return err ? ERR_PTR(err) : vma;
diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c
index d9be7a5a239c1..f06b2e8cf7d4b 100644
--- a/drivers/gpu/drm/i915/i915_driver.c
+++ b/drivers/gpu/drm/i915/i915_driver.c
@@ -99,6 +99,7 @@
 #include "i915_drv.h"
 #include "i915_dsb_buffer.h"
 #include "i915_edram.h"
+#include "i915_fb_pin.h"
 #include "i915_file_private.h"
 #include "i915_getparam.h"
 #include "i915_gmch.h"
@@ -122,7 +123,6 @@
 #include "intel_cpu_info.h"
 #include "intel_gvt.h"
 #include "intel_memory_region.h"
-#include "intel_pci_config.h"
 #include "intel_pcode.h"
 #include "intel_region_ttm.h"
 #include "vlv_iosf_sb.h"
@@ -311,6 +311,7 @@ static void i915_driver_late_release(struct drm_i915_private *dev_priv)
 	i915_params_free(&dev_priv->params);
 
 	intel_display_device_remove(display);
+	dev_priv->display = NULL;
 }
 
 /**
@@ -768,6 +769,7 @@ static const struct intel_display_parent_interface parent = {
 	.bo = &i915_display_bo_interface,
 	.dpt = &i915_display_dpt_interface,
 	.dsb = &i915_display_dsb_interface,
+	.fb_pin = &i915_display_fb_pin_interface,
 	.frontbuffer = &i915_display_frontbuffer_interface,
 	.hdcp = &i915_display_hdcp_interface,
 	.initial_plane = &i915_display_initial_plane_interface,
@@ -779,7 +781,7 @@ static const struct intel_display_parent_interface parent = {
 	.rpm = &i915_display_rpm_interface,
 	.rps = &i915_display_rps_interface,
 	.stolen = &i915_display_stolen_interface,
-	.vma = &i915_display_vma_interface,
+	.vlv_iosf = &i915_display_vlv_iosf_interface,
 
 	.fence_priority_display = fence_priority_display,
 	.has_auxccs = has_auxccs,
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index dafee3dcd1c52..844ed79e72114 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -315,6 +315,8 @@ struct drm_i915_private {
 	/* The TTM device structure. */
 	struct ttm_device bdev;
 
+	atomic_t pending_fb_pin;
+
 	I915_SELFTEST_DECLARE(struct i915_selftest_stash selftest;)
 
 	/*
diff --git a/drivers/gpu/drm/i915/i915_fb_pin.c b/drivers/gpu/drm/i915/i915_fb_pin.c
new file mode 100644
index 0000000000000..1034cb767e9f9
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_fb_pin.c
@@ -0,0 +1,311 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2021 Intel Corporation
+ */
+
+#include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
+
+#include "gem/i915_gem_domain.h"
+#include "gem/i915_gem_object.h"
+
+#include "i915_fb_pin.h"
+#include "i915_dpt.h"
+#include "i915_drv.h"
+#include "i915_vma.h"
+
+static struct i915_vma *
+intel_fb_pin_to_dpt(struct drm_gem_object *_obj, struct intel_dpt *dpt,
+		    const struct intel_fb_pin_params *pin_params)
+{
+	struct drm_i915_private *i915 = to_i915(_obj->dev);
+	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
+	struct i915_address_space *vm = i915_dpt_to_vm(dpt);
+	struct i915_gem_ww_ctx ww;
+	struct i915_vma *vma;
+	int ret;
+
+	/*
+	 * We are not syncing against the binding (and potential migrations)
+	 * below, so this vm must never be async.
+	 */
+	if (drm_WARN_ON(&i915->drm, vm->bind_async_flags))
+		return ERR_PTR(-EINVAL);
+
+	if (WARN_ON(!i915_gem_object_is_framebuffer(obj)))
+		return ERR_PTR(-EINVAL);
+
+	atomic_inc(&i915->pending_fb_pin);
+
+	for_i915_gem_ww(&ww, ret, true) {
+		ret = i915_gem_object_lock(obj, &ww);
+		if (ret)
+			continue;
+
+		if (HAS_LMEM(i915)) {
+			unsigned int flags = obj->flags;
+
+			/*
+			 * For this type of buffer we need to able to read from the CPU
+			 * the clear color value found in the buffer, hence we need to
+			 * ensure it is always in the mappable part of lmem, if this is
+			 * a small-bar device.
+			 */
+			if (pin_params->needs_cpu_lmem_access)
+				flags &= ~I915_BO_ALLOC_GPU_ONLY;
+			ret = __i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM_0,
+							flags);
+			if (ret)
+				continue;
+		}
+
+		ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
+		if (ret)
+			continue;
+
+		vma = i915_vma_instance(obj, vm, pin_params->view);
+		if (IS_ERR(vma)) {
+			ret = PTR_ERR(vma);
+			continue;
+		}
+
+		if (i915_vma_misplaced(vma, 0, pin_params->alignment, 0)) {
+			ret = i915_vma_unbind(vma);
+			if (ret)
+				continue;
+		}
+
+		ret = i915_vma_pin_ww(vma, &ww, 0, pin_params->alignment,
+				      PIN_GLOBAL);
+		if (ret)
+			continue;
+	}
+	if (ret) {
+		vma = ERR_PTR(ret);
+		goto err;
+	}
+
+	vma->display_alignment = max(vma->display_alignment,
+				     pin_params->alignment);
+
+	i915_gem_object_flush_if_display(obj);
+
+	i915_vma_get(vma);
+
+	/*
+	 * The DPT object contains only one vma, and there is no VT-d
+	 * guard, so the VMA's offset within the DPT is always 0.
+	 */
+	drm_WARN_ON(&i915->drm, i915_dpt_offset(vma));
+err:
+	atomic_dec(&i915->pending_fb_pin);
+
+	return vma;
+}
+
+static struct i915_vma *
+intel_fb_pin_to_ggtt(struct drm_gem_object *_obj,
+		     const struct intel_fb_pin_params *pin_params,
+		     int *out_fence_id)
+{
+	struct drm_i915_private *i915 = to_i915(_obj->dev);
+	struct drm_i915_gem_object *obj = to_intel_bo(_obj);
+	intel_wakeref_t wakeref;
+	struct i915_gem_ww_ctx ww;
+	struct i915_vma *vma;
+	unsigned int pinctl;
+	int ret;
+
+	if (drm_WARN_ON(&i915->drm, !i915_gem_object_is_framebuffer(obj)))
+		return ERR_PTR(-EINVAL);
+
+	if (drm_WARN_ON(&i915->drm, pin_params->alignment &&
+			!is_power_of_2(pin_params->alignment)))
+		return ERR_PTR(-EINVAL);
+
+	/*
+	 * Global gtt pte registers are special registers which actually forward
+	 * writes to a chunk of system memory. Which means that there is no risk
+	 * that the register values disappear as soon as we call
+	 * intel_runtime_pm_put(), so it is correct to wrap only the
+	 * pin/unpin/fence and not more.
+	 */
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
+
+	atomic_inc(&i915->pending_fb_pin);
+
+	pinctl = 0;
+	/* PIN_MAPPABLE limits the address to GMADR size */
+	if (pin_params->needs_low_address)
+		pinctl |= PIN_MAPPABLE;
+
+	i915_gem_ww_ctx_init(&ww, true);
+retry:
+	ret = i915_gem_object_lock(obj, &ww);
+	if (!ret && pin_params->needs_physical)
+		ret = i915_gem_object_attach_phys(obj, pin_params->phys_alignment);
+	else if (!ret && HAS_LMEM(i915))
+		ret = i915_gem_object_migrate(obj, &ww, INTEL_REGION_LMEM_0);
+	if (!ret)
+		ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		goto err;
+
+	vma = i915_gem_object_pin_to_display_plane(obj, &ww,
+						   pin_params->alignment,
+						   pin_params->vtd_guard,
+						   pin_params->view, pinctl);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
+		goto err_unpin;
+	}
+
+	if (out_fence_id)
+		*out_fence_id = -1;
+
+	if (out_fence_id && i915_vma_is_map_and_fenceable(vma)) {
+		/*
+		 * Install a fence for tiled scan-out. Pre-i965 always needs a
+		 * fence, whereas 965+ only requires a fence if using
+		 * framebuffer compression.  For simplicity, we always, when
+		 * possible, install a fence as the cost is not that onerous.
+		 *
+		 * If we fail to fence the tiled scanout, then either the
+		 * modeset will reject the change (which is highly unlikely as
+		 * the affected systems, all but one, do not have unmappable
+		 * space) or we will not be able to enable full powersaving
+		 * techniques (also likely not to apply due to various limits
+		 * FBC and the like impose on the size of the buffer, which
+		 * presumably we violated anyway with this unmappable buffer).
+		 * Anyway, it is presumably better to stumble onwards with
+		 * something and try to run the system in a "less than optimal"
+		 * mode that matches the user configuration.
+		 */
+		ret = i915_vma_pin_fence(vma);
+		if (ret != 0 && pin_params->needs_fence) {
+			i915_vma_unpin(vma);
+			goto err_unpin;
+		}
+		ret = 0;
+
+		if (vma->fence)
+			*out_fence_id = vma->fence->id;
+	}
+
+	i915_vma_get(vma);
+
+err_unpin:
+	i915_gem_object_unpin_pages(obj);
+err:
+	if (ret == -EDEADLK) {
+		ret = i915_gem_ww_ctx_backoff(&ww);
+		if (!ret)
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
+	if (ret)
+		vma = ERR_PTR(ret);
+
+	atomic_dec(&i915->pending_fb_pin);
+	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
+	return vma;
+}
+
+static void intel_fb_unpin_vma(struct i915_vma *vma, int fence_id)
+{
+	if (fence_id >= 0)
+		i915_vma_unpin_fence(vma);
+	i915_vma_unpin(vma);
+	i915_vma_put(vma);
+}
+
+static int i915_fb_pin_ggtt_pin(struct drm_gem_object *obj,
+				const struct intel_fb_pin_params *pin_params,
+				struct i915_vma **out_ggtt_vma,
+				u32 *out_offset,
+				int *out_fence_id)
+{
+	struct i915_vma *ggtt_vma;
+
+	ggtt_vma = intel_fb_pin_to_ggtt(obj, pin_params, out_fence_id);
+	if (IS_ERR(ggtt_vma))
+		return PTR_ERR(ggtt_vma);
+
+	*out_ggtt_vma = ggtt_vma;
+
+	/*
+	 * Pre-populate the dma address before we enter the vblank
+	 * evade critical section as i915_gem_object_get_dma_address()
+	 * will trigger might_sleep() even if it won't actually sleep,
+	 * which is the case when the fb has already been pinned.
+	 */
+	if (pin_params->needs_physical)
+		*out_offset = i915_gem_object_get_dma_address(to_intel_bo(obj), 0);
+	else
+		*out_offset = i915_ggtt_offset(ggtt_vma);
+
+	return 0;
+}
+
+static void i915_fb_pin_ggtt_unpin(struct i915_vma *ggtt_vma,
+				   int fence_id)
+{
+	if (ggtt_vma)
+		intel_fb_unpin_vma(ggtt_vma, fence_id);
+}
+
+static int i915_fb_pin_dpt_pin(struct drm_gem_object *obj, struct intel_dpt *dpt,
+			       const struct intel_fb_pin_params *pin_params,
+			       struct i915_vma **out_dpt_vma,
+			       struct i915_vma **out_ggtt_vma,
+			       u32 *out_offset)
+{
+	struct i915_vma *ggtt_vma, *dpt_vma;
+
+	WARN_ON(!dpt);
+
+	ggtt_vma = i915_dpt_pin_to_ggtt(dpt, pin_params->alignment / 512);
+	if (IS_ERR(ggtt_vma))
+		return PTR_ERR(ggtt_vma);
+
+	dpt_vma = intel_fb_pin_to_dpt(obj, dpt, pin_params);
+	if (IS_ERR(dpt_vma)) {
+		i915_dpt_unpin_from_ggtt(dpt);
+		return PTR_ERR(dpt_vma);
+	}
+
+	drm_WARN_ON(obj->dev, ggtt_vma == dpt_vma);
+
+	*out_ggtt_vma = ggtt_vma;
+	*out_dpt_vma = dpt_vma;
+
+	*out_offset = i915_ggtt_offset(ggtt_vma);
+
+	return 0;
+}
+
+static void i915_fb_pin_dpt_unpin(struct intel_dpt *dpt,
+				  struct i915_vma *dpt_vma,
+				  struct i915_vma *ggtt_vma)
+{
+	WARN_ON(!dpt);
+	WARN_ON(!!dpt_vma != !!ggtt_vma);
+
+	if (dpt_vma)
+		intel_fb_unpin_vma(dpt_vma, -1);
+	if (ggtt_vma)
+		i915_dpt_unpin_from_ggtt(dpt);
+}
+
+static void i915_fb_pin_get_map(struct i915_vma *vma, struct iosys_map *map)
+{
+	iosys_map_set_vaddr_iomem(map, i915_vma_get_iomap(vma));
+}
+
+const struct intel_display_fb_pin_interface i915_display_fb_pin_interface = {
+	.ggtt_pin = i915_fb_pin_ggtt_pin,
+	.ggtt_unpin = i915_fb_pin_ggtt_unpin,
+	.dpt_pin = i915_fb_pin_dpt_pin,
+	.dpt_unpin = i915_fb_pin_dpt_unpin,
+	.get_map = i915_fb_pin_get_map,
+};
diff --git a/drivers/gpu/drm/i915/i915_fb_pin.h b/drivers/gpu/drm/i915/i915_fb_pin.h
new file mode 100644
index 0000000000000..18c6c0b61ab8f
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_fb_pin.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __I915_FB_PIN_H__
+#define __I915_FB_PIN_H__
+
+extern const struct intel_display_fb_pin_interface i915_display_fb_pin_interface;
+
+#endif /* __I915_FB_PIN_H__ */
diff --git a/drivers/gpu/drm/i915/i915_freq.c b/drivers/gpu/drm/i915/i915_freq.c
index 9bdaea34aef91..9547d087555f4 100644
--- a/drivers/gpu/drm/i915/i915_freq.c
+++ b/drivers/gpu/drm/i915/i915_freq.c
@@ -2,10 +2,10 @@
 /* Copyright © 2025 Intel Corporation */
 
 #include <drm/drm_print.h>
+#include <drm/intel/mchbar_regs.h>
 
 #include "i915_drv.h"
 #include "i915_freq.h"
-#include "intel_mchbar_regs.h"
 
 unsigned int i9xx_fsb_freq(struct drm_i915_private *i915)
 {
diff --git a/drivers/gpu/drm/i915/i915_gmch.c b/drivers/gpu/drm/i915/i915_gmch.c
index 2d55831b3c58a..b0ef6ef577a3e 100644
--- a/drivers/gpu/drm/i915/i915_gmch.c
+++ b/drivers/gpu/drm/i915/i915_gmch.c
@@ -5,10 +5,10 @@
 
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
+#include <drm/intel/pci_config.h>
 
 #include "i915_drv.h"
 #include "i915_gmch.h"
-#include "intel_pci_config.h"
 
 static void i915_gmch_bridge_release(struct drm_device *dev, void *bridge)
 {
diff --git a/drivers/gpu/drm/i915/i915_gtt_view_types.h b/drivers/gpu/drm/i915/i915_gtt_view_types.h
index c084f67bc8807..9c4f38db32ffa 100644
--- a/drivers/gpu/drm/i915/i915_gtt_view_types.h
+++ b/drivers/gpu/drm/i915/i915_gtt_view_types.h
@@ -56,4 +56,19 @@ struct i915_gtt_view {
 	};
 };
 
+static inline bool i915_gtt_view_is_normal(const struct i915_gtt_view *view)
+{
+	return view->type == I915_GTT_VIEW_NORMAL;
+}
+
+static inline bool i915_gtt_view_is_remapped(const struct i915_gtt_view *view)
+{
+	return view->type == I915_GTT_VIEW_REMAPPED;
+}
+
+static inline bool i915_gtt_view_is_rotated(const struct i915_gtt_view *view)
+{
+	return view->type == I915_GTT_VIEW_ROTATED;
+}
+
 #endif /* __I915_GTT_VIEW_TYPES_H__ */
diff --git a/drivers/gpu/drm/i915/i915_hwmon.c b/drivers/gpu/drm/i915/i915_hwmon.c
index c4a799f5fe925..da643b38064c8 100644
--- a/drivers/gpu/drm/i915/i915_hwmon.c
+++ b/drivers/gpu/drm/i915/i915_hwmon.c
@@ -10,11 +10,11 @@
 #include <linux/units.h>
 
 #include <drm/intel/intel_pcode_regs.h>
+#include <drm/intel/mchbar_regs.h>
 
 #include "i915_drv.h"
 #include "i915_hwmon.h"
 #include "i915_reg.h"
-#include "intel_mchbar_regs.h"
 #include "intel_pcode.h"
 #include "gt/intel_gt.h"
 #include "gt/intel_gt_regs.h"
diff --git a/drivers/gpu/drm/i915/i915_initial_plane.c b/drivers/gpu/drm/i915/i915_initial_plane.c
index 390a9248d6314..4902c07486640 100644
--- a/drivers/gpu/drm/i915/i915_initial_plane.c
+++ b/drivers/gpu/drm/i915/i915_initial_plane.c
@@ -9,18 +9,13 @@
 #include "display/intel_crtc.h"
 #include "display/intel_display_types.h"
 #include "display/intel_fb.h"
-#include "display/intel_fbdev_fb.h"
 #include "gem/i915_gem_lmem.h"
 #include "gem/i915_gem_region.h"
 
+#include "i915_bo.h"
 #include "i915_drv.h"
 #include "i915_initial_plane.h"
 
-static void i915_initial_plane_vblank_wait(struct drm_crtc *crtc)
-{
-	intel_crtc_wait_for_next_vblank(to_intel_crtc(crtc));
-}
-
 static enum intel_memory_type
 initial_plane_memory_type(struct drm_i915_private *i915)
 {
@@ -34,14 +29,14 @@ initial_plane_memory_type(struct drm_i915_private *i915)
 
 static bool
 initial_plane_phys(struct drm_i915_private *i915,
-		   struct intel_initial_plane_config *plane_config)
+		   u32 base, resource_size_t *out_phys_base,
+		   struct intel_memory_region **out_mem)
 {
 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 	struct intel_memory_region *mem;
 	enum intel_memory_type mem_type;
 	bool is_present, is_local;
 	dma_addr_t dma_addr;
-	u32 base;
 
 	mem_type = initial_plane_memory_type(i915);
 	mem = intel_memory_region_by_type(i915, mem_type);
@@ -52,7 +47,7 @@ initial_plane_phys(struct drm_i915_private *i915,
 		return false;
 	}
 
-	base = round_down(plane_config->base, I915_GTT_MIN_ALIGNMENT);
+	base = round_down(base, I915_GTT_MIN_ALIGNMENT);
 
 	dma_addr = intel_ggtt_read_entry(&ggtt->vm, base, &is_present, &is_local);
 
@@ -77,8 +72,8 @@ initial_plane_phys(struct drm_i915_private *i915,
 	drm_dbg(&i915->drm, "Using dma_addr=%pa, based on initial plane programming\n",
 		&dma_addr);
 
-	plane_config->phys_base = dma_addr - mem->region.start;
-	plane_config->mem = mem;
+	*out_phys_base = dma_addr - mem->region.start;
+	*out_mem = mem;
 
 	return true;
 }
@@ -99,12 +94,9 @@ initial_plane_vma(struct drm_i915_private *i915,
 	if (plane_config->size == 0)
 		return NULL;
 
-	if (!initial_plane_phys(i915, plane_config))
+	if (!initial_plane_phys(i915, plane_config->base, &phys_base, &mem))
 		return NULL;
 
-	phys_base = plane_config->phys_base;
-	mem = plane_config->mem;
-
 	base = round_down(plane_config->base, I915_GTT_MIN_ALIGNMENT);
 	size = round_up(plane_config->base + plane_config->size,
 			mem->min_page_size);
@@ -118,7 +110,7 @@ initial_plane_vma(struct drm_i915_private *i915,
 	if (IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) &&
 	    IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) &&
 	    mem == i915->mm.stolen_region &&
-	    !intel_fbdev_fb_prefer_stolen(&i915->drm, size)) {
+	    !i915_bo_fbdev_prefer_stolen(i915, size)) {
 		drm_dbg_kms(&i915->drm, "Initial FB size exceeds half of stolen, discarding\n");
 		return NULL;
 	}
@@ -140,7 +132,7 @@ initial_plane_vma(struct drm_i915_private *i915,
 	i915_gem_object_set_cache_coherency(obj, HAS_WT(i915) ?
 					    I915_CACHE_WT : I915_CACHE_NONE);
 
-	tiling = intel_fb_modifier_to_tiling(plane_config->fb->base.modifier);
+	tiling = intel_fb_modifier_to_tiling(plane_config->fb->modifier);
 
 	switch (tiling) {
 	case I915_TILING_NONE:
@@ -148,7 +140,7 @@ initial_plane_vma(struct drm_i915_private *i915,
 	case I915_TILING_X:
 	case I915_TILING_Y:
 		obj->tiling_and_stride =
-			plane_config->fb->base.pitches[0] |
+			plane_config->fb->pitches[0] |
 			tiling;
 		break;
 	default:
@@ -226,7 +218,7 @@ i915_alloc_initial_plane_obj(struct drm_device *drm,
 {
 	struct drm_i915_private *i915 = to_i915(drm);
 	struct drm_mode_fb_cmd2 mode_cmd = {};
-	struct drm_framebuffer *fb = &plane_config->fb->base;
+	struct drm_framebuffer *fb = plane_config->fb;
 	struct i915_vma *vma;
 
 	vma = initial_plane_vma(i915, plane_config);
@@ -268,7 +260,7 @@ i915_initial_plane_setup(struct drm_plane_state *_plane_state,
 	plane_state->ggtt_vma = i915_vma_get(vma);
 	if (intel_plane_uses_fence(plane_state) &&
 	    i915_vma_pin_fence(vma) == 0 && vma->fence)
-		plane_state->flags |= PLANE_HAS_FENCE;
+		plane_state->fence_id = vma->fence->id;
 
 	plane_state->surf = i915_ggtt_offset(plane_state->ggtt_vma);
 
@@ -285,7 +277,6 @@ static void i915_plane_config_fini(struct intel_initial_plane_config *plane_conf
 }
 
 const struct intel_display_initial_plane_interface i915_display_initial_plane_interface = {
-	.vblank_wait = i915_initial_plane_vblank_wait,
 	.alloc_obj = i915_alloc_initial_plane_obj,
 	.setup = i915_initial_plane_setup,
 	.config_fini = i915_plane_config_fini,
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index d4d8dd0a41749..30ce462e92ab8 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -38,8 +38,6 @@
 
 #include "display/intel_display_irq.h"
 #include "display/intel_hotplug.h"
-#include "display/intel_hotplug_irq.h"
-#include "display/intel_lpe_audio.h"
 
 #include "gt/intel_breadcrumbs.h"
 #include "gt/intel_gt.h"
@@ -236,17 +234,15 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
 
 	do {
-		u32 iir, gt_iir, pm_iir;
-		u32 eir = 0, dpinvgtt = 0;
-		u32 pipe_stats[I915_MAX_PIPES] = {};
-		u32 hotplug_status = 0;
+		struct intel_display_irq_state state = {};
+		u32 gt_iir, pm_iir;
 		u32 ier = 0;
 
 		gt_iir = intel_uncore_read(&dev_priv->uncore, GTIIR);
 		pm_iir = intel_uncore_read(&dev_priv->uncore, GEN6_PMIIR);
-		iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
+		state.iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
 
-		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
+		if (gt_iir == 0 && pm_iir == 0 && state.iir == 0)
 			break;
 
 		ret = IRQ_HANDLED;
@@ -272,26 +268,14 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 		if (pm_iir)
 			intel_uncore_write(&dev_priv->uncore, GEN6_PMIIR, pm_iir);
 
-		if (iir & I915_DISPLAY_PORT_INTERRUPT)
-			hotplug_status = i9xx_hpd_irq_ack(display);
-
-		if (iir & I915_MASTER_ERROR_INTERRUPT)
-			vlv_display_error_irq_ack(display, &eir, &dpinvgtt);
-
-		/* Call regardless, as some status bits might not be
-		 * signalled in IIR */
-		i9xx_pipestat_irq_ack(display, iir, pipe_stats);
-
-		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
-			   I915_LPE_PIPE_B_INTERRUPT))
-			intel_lpe_audio_irq_handler(display);
+		intel_display_irq_ack(display, &state);
 
 		/*
 		 * VLV_IIR is single buffered, and reflects the level
 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
 		 */
-		if (iir)
-			intel_uncore_write(&dev_priv->uncore, VLV_IIR, iir);
+		if (state.iir)
+			intel_uncore_write(&dev_priv->uncore, VLV_IIR, state.iir);
 
 		intel_uncore_write(&dev_priv->uncore, VLV_IER, ier);
 		intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
@@ -301,13 +285,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 		if (pm_iir)
 			gen6_rps_irq_handler(&to_gt(dev_priv)->rps, pm_iir);
 
-		if (hotplug_status)
-			i9xx_hpd_irq_handler(display, hotplug_status);
-
-		if (iir & I915_MASTER_ERROR_INTERRUPT)
-			vlv_display_error_irq_handler(display, eir, dpinvgtt);
-
-		valleyview_pipestat_irq_handler(display, pipe_stats);
+		intel_display_irq_handler(display, &state);
 	} while (0);
 
 	pmu_irq_stats(dev_priv, ret);
@@ -330,16 +308,14 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg)
 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
 
 	do {
-		u32 master_ctl, iir;
-		u32 eir = 0, dpinvgtt = 0;
-		u32 pipe_stats[I915_MAX_PIPES] = {};
-		u32 hotplug_status = 0;
+		struct intel_display_irq_state state = {};
+		u32 master_ctl;
 		u32 ier = 0;
 
 		master_ctl = intel_uncore_read(&dev_priv->uncore, GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
-		iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
+		state.iir = intel_uncore_read(&dev_priv->uncore, VLV_IIR);
 
-		if (master_ctl == 0 && iir == 0)
+		if (master_ctl == 0 && state.iir == 0)
 			break;
 
 		ret = IRQ_HANDLED;
@@ -362,38 +338,19 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg)
 
 		gen8_gt_irq_handler(to_gt(dev_priv), master_ctl);
 
-		if (iir & I915_DISPLAY_PORT_INTERRUPT)
-			hotplug_status = i9xx_hpd_irq_ack(display);
-
-		if (iir & I915_MASTER_ERROR_INTERRUPT)
-			vlv_display_error_irq_ack(display, &eir, &dpinvgtt);
-
-		/* Call regardless, as some status bits might not be
-		 * signalled in IIR */
-		i9xx_pipestat_irq_ack(display, iir, pipe_stats);
-
-		if (iir & (I915_LPE_PIPE_A_INTERRUPT |
-			   I915_LPE_PIPE_B_INTERRUPT |
-			   I915_LPE_PIPE_C_INTERRUPT))
-			intel_lpe_audio_irq_handler(display);
+		intel_display_irq_ack(display, &state);
 
 		/*
 		 * VLV_IIR is single buffered, and reflects the level
 		 * from PIPESTAT/PORT_HOTPLUG_STAT, hence clear it last.
 		 */
-		if (iir)
-			intel_uncore_write(&dev_priv->uncore, VLV_IIR, iir);
+		if (state.iir)
+			intel_uncore_write(&dev_priv->uncore, VLV_IIR, state.iir);
 
 		intel_uncore_write(&dev_priv->uncore, VLV_IER, ier);
 		intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
 
-		if (hotplug_status)
-			i9xx_hpd_irq_handler(display, hotplug_status);
-
-		if (iir & I915_MASTER_ERROR_INTERRUPT)
-			vlv_display_error_irq_handler(display, eir, dpinvgtt);
-
-		valleyview_pipestat_irq_handler(display, pipe_stats);
+		intel_display_irq_handler(display, &state);
 	} while (0);
 
 	pmu_irq_stats(dev_priv, ret);
@@ -440,7 +397,7 @@ static irqreturn_t ilk_irq_handler(int irq, void *arg)
 		ret = IRQ_HANDLED;
 	}
 
-	if (ilk_display_irq_handler(display))
+	if (intel_display_irq_handler(display, NULL))
 		ret = IRQ_HANDLED;
 
 	if (GRAPHICS_VER(i915) >= 6) {
@@ -502,8 +459,11 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
 
 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
 	if (master_ctl & ~GEN8_GT_IRQS) {
+		const struct intel_display_irq_state state = {
+			.master_ctl = master_ctl,
+		};
 		disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
-		gen8_de_irq_handler(display, master_ctl);
+		intel_display_irq_handler(display, &state);
 		enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
 	}
 
@@ -555,7 +515,7 @@ static irqreturn_t gen11_irq_handler(int irq, void *arg)
 
 	/* IRQs are synced during runtime_suspend, we don't require a wakeref */
 	if (master_ctl & GEN11_DISPLAY_IRQ)
-		gen11_display_irq_handler(display);
+		intel_display_irq_handler(display, NULL);
 
 	gu_misc_iir = gen11_gu_misc_irq_ack(display, master_ctl);
 
@@ -622,7 +582,7 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg)
 	gen11_gt_irq_handler(gt, master_ctl);
 
 	if (master_ctl & GEN11_DISPLAY_IRQ)
-		gen11_display_irq_handler(display);
+		intel_display_irq_handler(display, NULL);
 
 	gu_misc_iir = gen11_gu_misc_irq_ack(display, master_ctl);
 
@@ -640,7 +600,7 @@ static void ilk_irq_reset(struct drm_i915_private *dev_priv)
 	struct intel_display *display = dev_priv->display;
 
 	/* The master interrupt enable is in DEIER, reset display irq first */
-	ilk_display_irq_reset(display);
+	intel_display_irq_reset(display);
 	gen5_gt_irq_reset(to_gt(dev_priv));
 }
 
@@ -653,7 +613,7 @@ static void valleyview_irq_reset(struct drm_i915_private *dev_priv)
 
 	gen5_gt_irq_reset(to_gt(dev_priv));
 
-	vlv_display_irq_reset(display);
+	intel_display_irq_reset(display);
 }
 
 static void gen8_irq_reset(struct drm_i915_private *dev_priv)
@@ -664,7 +624,7 @@ static void gen8_irq_reset(struct drm_i915_private *dev_priv)
 	gen8_master_intr_disable(intel_uncore_regs(uncore));
 
 	gen8_gt_irq_reset(to_gt(dev_priv));
-	gen8_display_irq_reset(display);
+	intel_display_irq_reset(display);
 	gen2_irq_reset(uncore, GEN8_PCU_IRQ_REGS);
 }
 
@@ -677,7 +637,7 @@ static void gen11_irq_reset(struct drm_i915_private *dev_priv)
 	gen11_master_intr_disable(intel_uncore_regs(&dev_priv->uncore));
 
 	gen11_gt_irq_reset(gt);
-	gen11_display_irq_reset(display);
+	intel_display_irq_reset(display);
 
 	gen2_irq_reset(uncore, GEN11_GU_MISC_IRQ_REGS);
 	gen2_irq_reset(uncore, GEN8_PCU_IRQ_REGS);
@@ -695,7 +655,7 @@ static void dg1_irq_reset(struct drm_i915_private *dev_priv)
 	for_each_gt(gt, dev_priv, i)
 		gen11_gt_irq_reset(gt);
 
-	gen11_display_irq_reset(display);
+	intel_display_irq_reset(display);
 
 	gen2_irq_reset(uncore, GEN11_GU_MISC_IRQ_REGS);
 	gen2_irq_reset(uncore, GEN8_PCU_IRQ_REGS);
@@ -715,7 +675,7 @@ static void cherryview_irq_reset(struct drm_i915_private *dev_priv)
 
 	gen2_irq_reset(uncore, GEN8_PCU_IRQ_REGS);
 
-	vlv_display_irq_reset(display);
+	intel_display_irq_reset(display);
 }
 
 static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
@@ -724,7 +684,7 @@ static void ilk_irq_postinstall(struct drm_i915_private *dev_priv)
 
 	gen5_gt_irq_postinstall(to_gt(dev_priv));
 
-	ilk_de_irq_postinstall(display);
+	intel_display_irq_postinstall(display);
 }
 
 static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
@@ -733,7 +693,7 @@ static void valleyview_irq_postinstall(struct drm_i915_private *dev_priv)
 
 	gen5_gt_irq_postinstall(to_gt(dev_priv));
 
-	vlv_display_irq_postinstall(display);
+	intel_display_irq_postinstall(display);
 
 	intel_uncore_write(&dev_priv->uncore, VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
 	intel_uncore_posting_read(&dev_priv->uncore, VLV_MASTER_IER);
@@ -744,7 +704,7 @@ static void gen8_irq_postinstall(struct drm_i915_private *dev_priv)
 	struct intel_display *display = dev_priv->display;
 
 	gen8_gt_irq_postinstall(to_gt(dev_priv));
-	gen8_de_irq_postinstall(display);
+	intel_display_irq_postinstall(display);
 
 	gen8_master_intr_enable(intel_uncore_regs(&dev_priv->uncore));
 }
@@ -757,7 +717,7 @@ static void gen11_irq_postinstall(struct drm_i915_private *dev_priv)
 	u32 gu_misc_masked = GEN11_GU_MISC_GSE;
 
 	gen11_gt_irq_postinstall(gt);
-	gen11_de_irq_postinstall(display);
+	intel_display_irq_postinstall(display);
 
 	gen2_irq_init(uncore, GEN11_GU_MISC_IRQ_REGS, ~gu_misc_masked, gu_misc_masked);
 
@@ -778,7 +738,7 @@ static void dg1_irq_postinstall(struct drm_i915_private *dev_priv)
 
 	gen2_irq_init(uncore, GEN11_GU_MISC_IRQ_REGS, ~gu_misc_masked, gu_misc_masked);
 
-	dg1_de_irq_postinstall(display);
+	intel_display_irq_postinstall(display);
 
 	dg1_master_intr_enable(intel_uncore_regs(uncore));
 	intel_uncore_posting_read(uncore, DG1_MSTR_TILE_INTR);
@@ -790,7 +750,7 @@ static void cherryview_irq_postinstall(struct drm_i915_private *dev_priv)
 
 	gen8_gt_irq_postinstall(to_gt(dev_priv));
 
-	vlv_display_irq_postinstall(display);
+	intel_display_irq_postinstall(display);
 
 	intel_uncore_write(&dev_priv->uncore, GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
 	intel_uncore_posting_read(&dev_priv->uncore, GEN8_MASTER_IRQ);
@@ -864,7 +824,7 @@ static void i915_irq_reset(struct drm_i915_private *dev_priv)
 	struct intel_display *display = dev_priv->display;
 	struct intel_uncore *uncore = &dev_priv->uncore;
 
-	i9xx_display_irq_reset(display);
+	intel_display_irq_reset(display);
 
 	gen2_error_reset(uncore, GEN2_ERROR_REGS);
 	gen2_irq_reset(uncore, GEN2_IRQ_REGS);
@@ -888,7 +848,7 @@ static void i915_irq_postinstall(struct drm_i915_private *dev_priv)
 
 	gen2_irq_init(uncore, GEN2_IRQ_REGS, dev_priv->gen2_imr_mask, enable_mask);
 
-	i915_display_irq_postinstall(display);
+	intel_display_irq_postinstall(display);
 }
 
 static irqreturn_t i915_irq_handler(int irq, void *arg)
@@ -904,39 +864,29 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
 
 	do {
-		u32 pipe_stats[I915_MAX_PIPES] = {};
+		struct intel_display_irq_state state = {};
 		u32 eir = 0, eir_stuck = 0;
-		u32 hotplug_status = 0;
-		u32 iir;
 
-		iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
-		if (iir == 0)
+		state.iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
+		if (state.iir == 0)
 			break;
 
 		ret = IRQ_HANDLED;
 
-		if (iir & I915_DISPLAY_PORT_INTERRUPT)
-			hotplug_status = i9xx_hpd_irq_ack(display);
-
-		/* Call regardless, as some status bits might not be
-		 * signalled in IIR */
-		i9xx_pipestat_irq_ack(display, iir, pipe_stats);
+		intel_display_irq_ack(display, &state);
 
-		if (iir & I915_MASTER_ERROR_INTERRUPT)
+		if (state.iir & I915_MASTER_ERROR_INTERRUPT)
 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
 
-		intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir);
+		intel_uncore_write(&dev_priv->uncore, GEN2_IIR, state.iir);
 
-		if (iir & I915_USER_INTERRUPT)
-			intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0], iir);
+		if (state.iir & I915_USER_INTERRUPT)
+			intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0], state.iir);
 
-		if (iir & I915_MASTER_ERROR_INTERRUPT)
+		if (state.iir & I915_MASTER_ERROR_INTERRUPT)
 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
 
-		if (hotplug_status)
-			i9xx_hpd_irq_handler(display, hotplug_status);
-
-		i915_pipestat_irq_handler(display, iir, pipe_stats);
+		intel_display_irq_handler(display, &state);
 	} while (0);
 
 	pmu_irq_stats(dev_priv, ret);
@@ -951,7 +901,7 @@ static void i965_irq_reset(struct drm_i915_private *dev_priv)
 	struct intel_display *display = dev_priv->display;
 	struct intel_uncore *uncore = &dev_priv->uncore;
 
-	i9xx_display_irq_reset(display);
+	intel_display_irq_reset(display);
 
 	gen2_error_reset(uncore, GEN2_ERROR_REGS);
 	gen2_irq_reset(uncore, GEN2_IRQ_REGS);
@@ -997,7 +947,7 @@ static void i965_irq_postinstall(struct drm_i915_private *dev_priv)
 
 	gen2_irq_init(uncore, GEN2_IRQ_REGS, dev_priv->gen2_imr_mask, enable_mask);
 
-	i965_display_irq_postinstall(display);
+	intel_display_irq_postinstall(display);
 }
 
 static irqreturn_t i965_irq_handler(int irq, void *arg)
@@ -1013,44 +963,34 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
 
 	do {
-		u32 pipe_stats[I915_MAX_PIPES] = {};
+		struct intel_display_irq_state state = {};
 		u32 eir = 0, eir_stuck = 0;
-		u32 hotplug_status = 0;
-		u32 iir;
 
-		iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
-		if (iir == 0)
+		state.iir = intel_uncore_read(&dev_priv->uncore, GEN2_IIR);
+		if (state.iir == 0)
 			break;
 
 		ret = IRQ_HANDLED;
 
-		if (iir & I915_DISPLAY_PORT_INTERRUPT)
-			hotplug_status = i9xx_hpd_irq_ack(display);
-
-		/* Call regardless, as some status bits might not be
-		 * signalled in IIR */
-		i9xx_pipestat_irq_ack(display, iir, pipe_stats);
+		intel_display_irq_ack(display, &state);
 
-		if (iir & I915_MASTER_ERROR_INTERRUPT)
+		if (state.iir & I915_MASTER_ERROR_INTERRUPT)
 			i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
 
-		intel_uncore_write(&dev_priv->uncore, GEN2_IIR, iir);
+		intel_uncore_write(&dev_priv->uncore, GEN2_IIR, state.iir);
 
-		if (iir & I915_USER_INTERRUPT)
+		if (state.iir & I915_USER_INTERRUPT)
 			intel_engine_cs_irq(to_gt(dev_priv)->engine[RCS0],
-					    iir);
+					    state.iir);
 
-		if (iir & I915_BSD_USER_INTERRUPT)
+		if (state.iir & I915_BSD_USER_INTERRUPT)
 			intel_engine_cs_irq(to_gt(dev_priv)->engine[VCS0],
-					    iir >> 25);
+					    state.iir >> 25);
 
-		if (iir & I915_MASTER_ERROR_INTERRUPT)
+		if (state.iir & I915_MASTER_ERROR_INTERRUPT)
 			i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
 
-		if (hotplug_status)
-			i9xx_hpd_irq_handler(display, hotplug_status);
-
-		i965_pipestat_irq_handler(display, iir, pipe_stats);
+		intel_display_irq_handler(display, &state);
 	} while (0);
 
 	pmu_irq_stats(dev_priv, IRQ_HANDLED);
diff --git a/drivers/gpu/drm/i915/i915_irq.h b/drivers/gpu/drm/i915/i915_irq.h
index 5c87d6d41c743..108a187618de2 100644
--- a/drivers/gpu/drm/i915/i915_irq.h
+++ b/drivers/gpu/drm/i915/i915_irq.h
@@ -6,18 +6,11 @@
 #ifndef __I915_IRQ_H__
 #define __I915_IRQ_H__
 
-#include <linux/ktime.h>
 #include <linux/types.h>
 
 #include "i915_reg_defs.h"
 
-enum pipe;
-struct drm_crtc;
-struct drm_device;
-struct drm_display_mode;
 struct drm_i915_private;
-struct intel_crtc;
-struct intel_encoder;
 struct intel_uncore;
 
 void intel_irq_init(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_overlay.c b/drivers/gpu/drm/i915/i915_overlay.c
index c2d712bd2b0da..6de550a177567 100644
--- a/drivers/gpu/drm/i915/i915_overlay.c
+++ b/drivers/gpu/drm/i915/i915_overlay.c
@@ -7,6 +7,7 @@
 
 #include <drm/intel/display_parent_interface.h>
 #include <drm/intel/intel_gmd_interrupt_regs.h>
+#include <drm/intel/pci_config.h>
 
 #include "gem/i915_gem_internal.h"
 #include "gem/i915_gem_object_frontbuffer.h"
@@ -18,7 +19,6 @@
 #include "i915_drv.h"
 #include "i915_overlay.h"
 #include "i915_reg.h"
-#include "intel_pci_config.h"
 
 #include "display/intel_frontbuffer.h"
 
@@ -354,11 +354,14 @@ static struct i915_vma *i915_overlay_pin_fb(struct drm_device *drm,
 					    struct drm_gem_object *obj,
 					    u32 *offset)
 {
+	struct drm_i915_private *i915 = to_i915(drm);
 	struct drm_i915_gem_object *new_bo = to_intel_bo(obj);
 	struct i915_gem_ww_ctx ww;
 	struct i915_vma *vma;
 	int ret;
 
+	atomic_inc(&i915->pending_fb_pin);
+
 	i915_gem_ww_ctx_init(&ww, true);
 retry:
 	ret = i915_gem_object_lock(new_bo, &ww);
@@ -373,6 +376,9 @@ static struct i915_vma *i915_overlay_pin_fb(struct drm_device *drm,
 			goto retry;
 	}
 	i915_gem_ww_ctx_fini(&ww);
+
+	atomic_dec(&i915->pending_fb_pin);
+
 	if (ret)
 		return ERR_PTR(ret);
 
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
index d966a00520f15..82415af47d540 100644
--- a/drivers/gpu/drm/i915/i915_pci.c
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -24,6 +24,7 @@
 
 #include <drm/drm_color_mgmt.h>
 #include <drm/drm_drv.h>
+#include <drm/intel/pci_config.h>
 #include <drm/intel/pciids.h>
 
 #include "display/intel_display_driver.h"
@@ -35,7 +36,6 @@
 #include "i915_drv.h"
 #include "i915_pci.h"
 #include "i915_reg.h"
-#include "intel_pci_config.h"
 
 __diag_push();
 __diag_ignore_all("-Woverride-init", "Allow field initialization overrides for device info");
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 5d99b99b0c573..64e906380131a 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -494,21 +494,6 @@
 #define   GEN7_FF_DS_SCHED_LOAD_BALANCE	(0x1 << 4)  /* Default */
 #define   GEN7_FF_DS_SCHED_HW		(0x0 << 4)
 
-#define ILK_DISPLAY_CHICKEN1	_MMIO(0x42000)
-#define   ILK_FBCQ_DIS			REG_BIT(22)
-#define   ILK_PABSTRETCH_DIS		REG_BIT(21)
-#define   ILK_SABSTRETCH_DIS		REG_BIT(20)
-#define   IVB_PRI_STRETCH_MAX_MASK	REG_GENMASK(21, 20)
-#define   IVB_PRI_STRETCH_MAX_X8	REG_FIELD_PREP(IVB_PRI_STRETCH_MAX_MASK, 0)
-#define   IVB_PRI_STRETCH_MAX_X4	REG_FIELD_PREP(IVB_PRI_STRETCH_MAX_MASK, 1)
-#define   IVB_PRI_STRETCH_MAX_X2	REG_FIELD_PREP(IVB_PRI_STRETCH_MAX_MASK, 2)
-#define   IVB_PRI_STRETCH_MAX_X1	REG_FIELD_PREP(IVB_PRI_STRETCH_MAX_MASK, 3)
-#define   IVB_SPR_STRETCH_MAX_MASK	REG_GENMASK(19, 18)
-#define   IVB_SPR_STRETCH_MAX_X8	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 0)
-#define   IVB_SPR_STRETCH_MAX_X4	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 1)
-#define   IVB_SPR_STRETCH_MAX_X2	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 2)
-#define   IVB_SPR_STRETCH_MAX_X1	REG_FIELD_PREP(IVB_SPR_STRETCH_MAX_MASK, 3)
-
 #define DPLL_TEST	_MMIO(0x606c)
 #define   DPLLB_TEST_SDVO_DIV_1		(0 << 22)
 #define   DPLLB_TEST_SDVO_DIV_2		(1 << 22)
@@ -700,26 +685,10 @@
 #define   DG1_MSTR_IRQ			REG_BIT(31)
 #define   DG1_MSTR_TILE(t)		REG_BIT(t)
 
-#define ILK_DISPLAY_CHICKEN2	_MMIO(0x42004)
-/* Required on all Ironlake and Sandybridge according to the B-Spec. */
-#define   ILK_ELPIN_409_SELECT	REG_BIT(25)
-#define   ILK_DPARB_GATE	REG_BIT(22)
-#define   ILK_VSDPFD_FULL	REG_BIT(21)
-
-#define ILK_DSPCLK_GATE_D	_MMIO(0x42020)
-#define   ILK_VRHUNIT_CLOCK_GATE_DISABLE	REG_BIT(28)
-#define   ILK_DPFCUNIT_CLOCK_GATE_DISABLE	REG_BIT(9)
-#define   ILK_DPFCRUNIT_CLOCK_GATE_DISABLE	REG_BIT(8)
-#define   ILK_DPFDUNIT_CLOCK_GATE_ENABLE	REG_BIT(7)
-#define   ILK_DPARBUNIT_CLOCK_GATE_ENABLE	REG_BIT(5)
-
 #define IVB_CHICKEN3		_MMIO(0x4200c)
 #define   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE	REG_BIT(5)
 #define   CHICKEN3_DGMG_DONE_FIX_DISABLE	REG_BIT(2)
 
-#define CHICKEN_PAR2_1		_MMIO(0x42090)
-#define   KVM_CONFIG_CHANGE_NOTIFICATION_SELECT	REG_BIT(14)
-
 #define  VLV_PMWGICZ				_MMIO(0x1300a4)
 
 #define  HSW_EDRAM_CAP				_MMIO(0x120010)
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index 6a3a4d4244dc3..afc192d9931b8 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -27,7 +27,6 @@
 
 #include <drm/drm_gem.h>
 #include <drm/drm_print.h>
-#include <drm/intel/display_parent_interface.h>
 
 #include "display/intel_fb.h"
 #include "display/intel_frontbuffer.h"
@@ -2333,12 +2332,3 @@ int __init i915_vma_module_init(void)
 
 	return 0;
 }
-
-static int i915_vma_fence_id(const struct i915_vma *vma)
-{
-	return vma->fence ? vma->fence->id : -1;
-}
-
-const struct intel_display_vma_interface i915_display_vma_interface = {
-	.fence_id = i915_vma_fence_id,
-};
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index fa2d9b429db68..892306ab935dc 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -476,6 +476,4 @@ int i915_vma_module_init(void);
 I915_SELFTEST_DECLARE(int i915_vma_get_pages(struct i915_vma *vma));
 I915_SELFTEST_DECLARE(void i915_vma_put_pages(struct i915_vma *vma));
 
-extern const struct intel_display_vma_interface i915_display_vma_interface;
-
 #endif
diff --git a/drivers/gpu/drm/i915/intel_clock_gating.c b/drivers/gpu/drm/i915/intel_clock_gating.c
index ee2489a2fbe7f..86bdeb20f427d 100644
--- a/drivers/gpu/drm/i915/intel_clock_gating.c
+++ b/drivers/gpu/drm/i915/intel_clock_gating.c
@@ -26,13 +26,11 @@
  */
 
 #include <drm/drm_print.h>
-#include <drm/intel/intel_gmd_misc_regs.h>
 #include <drm/intel/intel_gmd_interrupt_regs.h>
+#include <drm/intel/intel_gmd_misc_regs.h>
+#include <drm/intel/mchbar_regs.h>
 
-#include "display/i9xx_plane_regs.h"
-#include "display/intel_display.h"
-#include "display/intel_display_core.h"
-#include "display/intel_display_regs.h"
+#include "display/intel_display_clock_gating.h"
 #include "display/intel_pch.h"
 #include "gt/intel_engine_regs.h"
 #include "gt/intel_gt.h"
@@ -42,43 +40,14 @@
 #include "i915_drv.h"
 #include "i915_reg.h"
 #include "intel_clock_gating.h"
-#include "intel_mchbar_regs.h"
 #include "vlv_iosf_sb.h"
 
 struct drm_i915_clock_gating_funcs {
 	void (*init_clock_gating)(struct drm_i915_private *i915);
 };
 
-static void gen9_init_clock_gating(struct drm_i915_private *i915)
-{
-	if (HAS_LLC(i915)) {
-		/*
-		 * WaCompressedResourceDisplayNewHashMode:skl,kbl
-		 * Display WA #0390: skl,kbl
-		 *
-		 * Must match Sampler, Pixel Back End, and Media. See
-		 * WaCompressedResourceSamplerPbeMediaNewHashMode.
-		 */
-		intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_DE_COMPRESSED_HASH_MODE);
-	}
-
-	/* See Bspec note for PSR2_CTL bit 31, Wa#828:skl,bxt,kbl,cfl */
-	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, SKL_EDP_PSR_FIX_RDWRAP);
-
-	/* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
-	intel_uncore_rmw(&i915->uncore, GEN8_CHICKEN_DCPR_1, 0, MASK_WAKEMEM);
-
-	/*
-	 * WaFbcWakeMemOn:skl,bxt,kbl,glk,cfl
-	 * Display WA #0859: skl,bxt,kbl,glk,cfl
-	 */
-	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_MEMORY_WAKE);
-}
-
 static void bxt_init_clock_gating(struct drm_i915_private *i915)
 {
-	gen9_init_clock_gating(i915);
-
 	/* WaDisableSDEUnitClockGating:bxt */
 	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 
@@ -88,111 +57,23 @@ static void bxt_init_clock_gating(struct drm_i915_private *i915)
 	 */
 	intel_uncore_rmw(&i915->uncore, GEN8_UCGCTL6, 0, GEN8_HDCUNIT_CLOCK_GATE_DISABLE_HDCREQ);
 
-	/*
-	 * Wa: Backlight PWM may stop in the asserted state, causing backlight
-	 * to stay fully on.
-	 */
-	intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
-			   intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
-			   PWM1_GATING_DIS | PWM2_GATING_DIS);
-
-	/*
-	 * Lower the display internal timeout.
-	 * This is needed to avoid any hard hangs when DSI port PLL
-	 * is off and a MMIO access is attempted by any privilege
-	 * application, using batch buffers or any other means.
-	 */
-	intel_uncore_write(&i915->uncore, RM_TIMEOUT, MMIO_TIMEOUT_US(950));
-
-	/*
-	 * WaFbcTurnOffFbcWatermark:bxt
-	 * Display WA #0562: bxt
-	 */
-	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+	intel_display_bxt_init_clock_gating(i915->display);
 }
 
 static void glk_init_clock_gating(struct drm_i915_private *i915)
 {
-	gen9_init_clock_gating(i915);
-
-	/*
-	 * WaDisablePWMClockGating:glk
-	 * Backlight PWM may stop in the asserted state, causing backlight
-	 * to stay fully on.
-	 */
-	intel_uncore_write(&i915->uncore, GEN9_CLKGATE_DIS_0,
-			   intel_uncore_read(&i915->uncore, GEN9_CLKGATE_DIS_0) |
-			   PWM1_GATING_DIS | PWM2_GATING_DIS);
-}
-
-static void g4x_disable_trickle_feed(struct drm_i915_private *dev_priv)
-{
-	struct intel_display *display = dev_priv->display;
-	enum pipe pipe;
-
-	for_each_pipe(display, pipe) {
-		intel_uncore_rmw(&dev_priv->uncore, DSPCNTR(display, pipe),
-				 0, DISP_TRICKLE_FEED_DISABLE);
-
-		intel_uncore_rmw(&dev_priv->uncore, DSPSURF(display, pipe),
-				 0, 0);
-		intel_uncore_posting_read(&dev_priv->uncore,
-					  DSPSURF(display, pipe));
-	}
+	intel_display_glk_init_clock_gating(i915->display);
 }
 
 static void ilk_init_clock_gating(struct drm_i915_private *i915)
 {
-	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
-
-	/*
-	 * Required for FBC
-	 * WaFbcDisableDpfcClockGating:ilk
-	 */
-	dspclk_gate |= ILK_DPFCRUNIT_CLOCK_GATE_DISABLE |
-		   ILK_DPFCUNIT_CLOCK_GATE_DISABLE |
-		   ILK_DPFDUNIT_CLOCK_GATE_ENABLE;
-
 	intel_uncore_write(&i915->uncore, PCH_3DCGDIS0,
 			   MARIUNIT_CLOCK_GATE_DISABLE |
 			   SVSMUNIT_CLOCK_GATE_DISABLE);
 	intel_uncore_write(&i915->uncore, PCH_3DCGDIS1,
 			   VFMUNIT_CLOCK_GATE_DISABLE);
 
-	/*
-	 * According to the spec the following bits should be set in
-	 * order to enable memory self-refresh
-	 * The bit 22/21 of 0x42004
-	 * The bit 5 of 0x42020
-	 * The bit 15 of 0x45000
-	 */
-	intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
-			   (intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
-			    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
-	dspclk_gate |= ILK_DPARBUNIT_CLOCK_GATE_ENABLE;
-	intel_uncore_write(&i915->uncore, DISP_ARB_CTL,
-			   (intel_uncore_read(&i915->uncore, DISP_ARB_CTL) |
-			    DISP_FBC_WM_DIS));
-
-	/*
-	 * Based on the document from hardware guys the following bits
-	 * should be set unconditionally in order to enable FBC.
-	 * The bit 22 of 0x42000
-	 * The bit 22 of 0x42004
-	 * The bit 7,8,9 of 0x42020.
-	 */
-	if (IS_IRONLAKE_M(i915)) {
-		/* WaFbcAsynchFlipDisableFbcQueue:ilk */
-		intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
-		intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_DPARB_GATE);
-	}
-
-	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
-
-	intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
-
-	g4x_disable_trickle_feed(i915);
-
+	intel_display_ilk_init_clock_gating(i915->display);
 	intel_pch_init_clock_gating(i915->display);
 }
 
@@ -209,11 +90,7 @@ static void gen6_check_mch_setup(struct drm_i915_private *i915)
 
 static void gen6_init_clock_gating(struct drm_i915_private *i915)
 {
-	u32 dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
-
-	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, dspclk_gate);
-
-	intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN2, 0, ILK_ELPIN_409_SELECT);
+	intel_display_gen6_init_clock_gating(i915->display);
 
 	intel_uncore_write(&i915->uncore, GEN6_UCGCTL1,
 			   intel_uncore_read(&i915->uncore, GEN6_UCGCTL1) |
@@ -248,19 +125,6 @@ static void gen6_init_clock_gating(struct drm_i915_private *i915)
 	 *
 	 * WaFbcAsynchFlipDisableFbcQueue:snb
 	 */
-	intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN1,
-			   intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN1) |
-			   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
-	intel_uncore_write(&i915->uncore, ILK_DISPLAY_CHICKEN2,
-			   intel_uncore_read(&i915->uncore, ILK_DISPLAY_CHICKEN2) |
-			   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
-	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D,
-			   intel_uncore_read(&i915->uncore, ILK_DSPCLK_GATE_D) |
-			   ILK_DPARBUNIT_CLOCK_GATE_ENABLE  |
-			   ILK_DPFDUNIT_CLOCK_GATE_ENABLE);
-
-	g4x_disable_trickle_feed(i915);
-
 	intel_pch_init_clock_gating(i915->display);
 
 	gen6_check_mch_setup(i915);
@@ -302,22 +166,15 @@ static void dg2_init_clock_gating(struct drm_i915_private *i915)
 static void cfl_init_clock_gating(struct drm_i915_private *i915)
 {
 	intel_pch_init_clock_gating(i915->display);
-	gen9_init_clock_gating(i915);
 
 	/* WAC6entrylatency:cfl */
 	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 
-	/*
-	 * WaFbcTurnOffFbcWatermark:cfl
-	 * Display WA #0562: cfl
-	 */
-	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+	intel_display_cfl_init_clock_gating(i915->display);
 }
 
 static void kbl_init_clock_gating(struct drm_i915_private *i915)
 {
-	gen9_init_clock_gating(i915);
-
 	/* WAC6entrylatency:kbl */
 	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 
@@ -331,17 +188,11 @@ static void kbl_init_clock_gating(struct drm_i915_private *i915)
 		intel_uncore_rmw(&i915->uncore, GEN6_UCGCTL1,
 				 0, GEN6_GAMUNIT_CLOCK_GATE_DISABLE);
 
-	/*
-	 * WaFbcTurnOffFbcWatermark:kbl
-	 * Display WA #0562: kbl
-	 */
-	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+	intel_display_kbl_init_clock_gating(i915->display);
 }
 
 static void skl_init_clock_gating(struct drm_i915_private *i915)
 {
-	gen9_init_clock_gating(i915);
-
 	/* WaDisableDopClockGating:skl */
 	intel_uncore_rmw(&i915->uncore, GEN7_MISCCPCTL,
 			 GEN7_DOP_CLOCK_GATE_ENABLE, 0);
@@ -349,32 +200,17 @@ static void skl_init_clock_gating(struct drm_i915_private *i915)
 	/* WAC6entrylatency:skl */
 	intel_uncore_rmw(&i915->uncore, FBC_LLC_READ_CTRL, 0, FBC_LLC_FULLY_OPEN);
 
-	/*
-	 * WaFbcTurnOffFbcWatermark:skl
-	 * Display WA #0562: skl
-	 */
-	intel_uncore_rmw(&i915->uncore, DISP_ARB_CTL, 0, DISP_FBC_WM_DIS);
+	intel_display_skl_init_clock_gating(i915->display);
 }
 
 static void bdw_init_clock_gating(struct drm_i915_private *i915)
 {
-	struct intel_display *display = i915->display;
-	enum pipe pipe;
-
-	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
-	intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
+	intel_display_bdw_clock_gating_disable_fbcq(i915->display);
 
 	/* WaSwitchSolVfFArbitrationPriority:bdw */
 	intel_uncore_rmw(&i915->uncore, GAM_ECOCHK, 0, HSW_ECOCHK_ARB_PRIO_SOL);
 
-	/* WaPsrDPAMaskVBlankInSRD:bdw */
-	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
-
-	for_each_pipe(display, pipe) {
-		/* WaPsrDPRSUnmaskVBlankInSRD:bdw */
-		intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
-				 0, BDW_UNMASK_VBL_TO_REGS_IN_SRD);
-	}
+	intel_display_bdw_clock_gating_vblank_in_srd(i915->display);
 
 	/* WaVSRefCountFullforceMissDisable:bdw */
 	/* WaDSRefCountFullforceMissDisable:bdw */
@@ -390,9 +226,7 @@ static void bdw_init_clock_gating(struct drm_i915_private *i915)
 	/* WaProgramL3SqcReg1Default:bdw */
 	gen8_set_l3sqc_credits(i915, 30, 2);
 
-	/* WaKVMNotificationOnConfigChange:bdw */
-	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR2_1,
-			 0, KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
+	intel_display_bdw_clock_gating_kvm_notif(i915->display);
 
 	intel_pch_init_clock_gating(i915->display);
 
@@ -406,20 +240,7 @@ static void bdw_init_clock_gating(struct drm_i915_private *i915)
 
 static void hsw_init_clock_gating(struct drm_i915_private *i915)
 {
-	struct intel_display *display = i915->display;
-	enum pipe pipe;
-
-	/* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
-	intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(PIPE_A), 0, HSW_FBCQ_DIS);
-
-	/* WaPsrDPAMaskVBlankInSRD:hsw */
-	intel_uncore_rmw(&i915->uncore, CHICKEN_PAR1_1, 0, HSW_MASK_VBL_TO_PIPE_IN_SRD);
-
-	for_each_pipe(display, pipe) {
-		/* WaPsrDPRSUnmaskVBlankInSRD:hsw */
-		intel_uncore_rmw(&i915->uncore, CHICKEN_PIPESL_1(pipe),
-				 0, HSW_UNMASK_VBL_TO_REGS_IN_SRD);
-	}
+	intel_display_hsw_init_clock_gating(i915->display);
 
 	/* This is required by WaCatErrorRejectionIssue:hsw */
 	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
@@ -435,10 +256,7 @@ static void ivb_init_clock_gating(struct drm_i915_private *i915)
 {
 	struct intel_display *display = i915->display;
 
-	intel_uncore_write(&i915->uncore, ILK_DSPCLK_GATE_D, ILK_VRHUNIT_CLOCK_GATE_DISABLE);
-
-	/* WaFbcAsynchFlipDisableFbcQueue:ivb */
-	intel_uncore_rmw(&i915->uncore, ILK_DISPLAY_CHICKEN1, 0, ILK_FBCQ_DIS);
+	intel_display_ivb_init_clock_gating(display);
 
 	/* WaDisableBackToBackFlipFix:ivb */
 	intel_uncore_write(&i915->uncore, IVB_CHICKEN3,
@@ -467,13 +285,12 @@ static void ivb_init_clock_gating(struct drm_i915_private *i915)
 	intel_uncore_rmw(&i915->uncore, GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
 			 0, GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
-	g4x_disable_trickle_feed(i915);
+	intel_display_disable_trickle_feed(display);
 
 	intel_uncore_rmw(&i915->uncore, GEN6_MBCUNIT_SNPCR, GEN6_MBC_SNPCR_MASK,
 			 GEN6_MBC_SNPCR_MED);
 
-	if (!HAS_PCH_NOP(display))
-		intel_pch_init_clock_gating(display);
+	intel_pch_init_clock_gating(display);
 
 	gen6_check_mch_setup(i915);
 }
@@ -540,21 +357,12 @@ static void chv_init_clock_gating(struct drm_i915_private *i915)
 
 static void g4x_init_clock_gating(struct drm_i915_private *i915)
 {
-	u32 dspclk_gate;
-
 	intel_uncore_write(&i915->uncore, RENCLK_GATE_D1, 0);
 	intel_uncore_write(&i915->uncore, RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
 			   GS_UNIT_CLOCK_GATE_DISABLE |
 			   CL_UNIT_CLOCK_GATE_DISABLE);
 	intel_uncore_write(&i915->uncore, RAMCLK_GATE_D, 0);
-	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
-		OVRUNIT_CLOCK_GATE_DISABLE |
-		OVCUNIT_CLOCK_GATE_DISABLE;
-	if (IS_GM45(i915))
-		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
-	intel_uncore_write(&i915->uncore, DSPCLK_GATE_D, dspclk_gate);
-
-	g4x_disable_trickle_feed(i915);
+	intel_display_g4x_init_clock_gating(i915->display);
 }
 
 static void i965gm_init_clock_gating(struct drm_i915_private *i915)
@@ -563,7 +371,7 @@ static void i965gm_init_clock_gating(struct drm_i915_private *i915)
 
 	intel_uncore_write(uncore, RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
 	intel_uncore_write(uncore, RENCLK_GATE_D2, 0);
-	intel_uncore_write(uncore, DSPCLK_GATE_D, 0);
+	intel_display_i965gm_init_clock_gating(i915->display);
 	intel_uncore_write(uncore, RAMCLK_GATE_D, 0);
 	intel_uncore_write16(uncore, DEUC, 0);
 	intel_uncore_write(uncore,
diff --git a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
index ae42818ab6e07..de118fae0a497 100644
--- a/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
+++ b/drivers/gpu/drm/i915/intel_gvt_mmio_table.c
@@ -7,6 +7,7 @@
 #include <drm/intel/intel_gmd_misc_regs.h>
 
 #include <drm/intel/intel_gmd_interrupt_regs.h>
+#include <drm/intel/mchbar_regs.h>
 
 #include "display/bxt_dpio_phy_regs.h"
 #include "display/i9xx_plane_regs.h"
@@ -44,7 +45,6 @@
 #include "i915_pvinfo.h"
 #include "i915_reg.h"
 #include "intel_gvt.h"
-#include "intel_mchbar_regs.h"
 
 #define MMIO_F(reg, s) do { \
 	int ret; \
diff --git a/drivers/gpu/drm/i915/intel_mchbar_regs.h b/drivers/gpu/drm/i915/intel_mchbar_regs.h
deleted file mode 100644
index 614d4017b57b3..0000000000000
--- a/drivers/gpu/drm/i915/intel_mchbar_regs.h
+++ /dev/null
@@ -1,268 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2022 Intel Corporation
- */
-
-#ifndef __INTEL_MCHBAR_REGS__
-#define __INTEL_MCHBAR_REGS__
-
-#include "i915_reg_defs.h"
-
-/*
- * MCHBAR mirror.
- *
- * This mirrors the MCHBAR MMIO space whose location is determined by
- * device 0 function 0's pci config register 0x44 or 0x48 and matches it in
- * every way.  It is not accessible from the CP register read instructions.
- *
- * Starting from Haswell, you can't write registers using the MCHBAR mirror,
- * just read.
- */
-
-#define MCHBAR_MIRROR_BASE			0x10000
-#define MCHBAR_MIRROR_BASE_SNB			0x140000
-
-#define CTG_STOLEN_RESERVED			_MMIO(MCHBAR_MIRROR_BASE + 0x34)
-#define ELK_STOLEN_RESERVED			_MMIO(MCHBAR_MIRROR_BASE + 0x48)
-#define   G4X_STOLEN_RESERVED_ADDR1_MASK	(0xFFFF << 16)
-#define   G4X_STOLEN_RESERVED_ADDR2_MASK	(0xFFF << 4)
-#define   G4X_STOLEN_RESERVED_ENABLE		(1 << 0)
-
-/* Pineview MCH register contains DDR3 setting */
-#define CSHRDDR3CTL				_MMIO(MCHBAR_MIRROR_BASE + 0x1a8)
-#define   CSHRDDR3CTL_DDR3			(1 << 2)
-
-/* 915-945 and GM965 MCH register controlling DRAM channel access */
-#define DCC					_MMIO(MCHBAR_MIRROR_BASE + 0x200)
-#define   DCC_ADDRESSING_MODE_SINGLE_CHANNEL	(0 << 0)
-#define   DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC	(1 << 0)
-#define   DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED	(2 << 0)
-#define   DCC_ADDRESSING_MODE_MASK		(3 << 0)
-#define   DCC_CHANNEL_XOR_DISABLE		(1 << 10)
-#define   DCC_CHANNEL_XOR_BIT_17		(1 << 9)
-#define DCC2					_MMIO(MCHBAR_MIRROR_BASE + 0x204)
-#define   DCC2_MODIFIED_ENHANCED_DISABLE	(1 << 20)
-
-/* 965 MCH register controlling DRAM channel configuration */
-#define C0DRB3_BW				_MMIO(MCHBAR_MIRROR_BASE + 0x206)
-#define C1DRB3_BW				_MMIO(MCHBAR_MIRROR_BASE + 0x606)
-
-/* Clocking configuration register */
-#define CLKCFG					_MMIO(MCHBAR_MIRROR_BASE + 0xc00)
-#define CLKCFG_FSB_400				(0 << 0)	/* hrawclk 100 */
-#define CLKCFG_FSB_400_ALT			(5 << 0)	/* hrawclk 100 */
-#define CLKCFG_FSB_533				(1 << 0)	/* hrawclk 133 */
-#define CLKCFG_FSB_667				(3 << 0)	/* hrawclk 166 */
-#define CLKCFG_FSB_800				(2 << 0)	/* hrawclk 200 */
-#define CLKCFG_FSB_1067				(6 << 0)	/* hrawclk 266 */
-#define CLKCFG_FSB_1067_ALT			(0 << 0)	/* hrawclk 266 */
-#define CLKCFG_FSB_1333				(7 << 0)	/* hrawclk 333 */
-#define CLKCFG_FSB_1333_ALT			(4 << 0)	/* hrawclk 333 */
-#define CLKCFG_FSB_1600_ALT			(6 << 0)	/* hrawclk 400 */
-#define CLKCFG_FSB_MASK				(7 << 0)
-#define CLKCFG_MEM_533				(1 << 4)
-#define CLKCFG_MEM_667				(2 << 4)
-#define CLKCFG_MEM_800				(3 << 4)
-#define CLKCFG_MEM_MASK				(7 << 4)
-
-#define HPLLVCO_MOBILE				_MMIO(MCHBAR_MIRROR_BASE + 0xc0f)
-#define HPLLVCO					_MMIO(MCHBAR_MIRROR_BASE + 0xc38)
-
-#define TSC1					_MMIO(MCHBAR_MIRROR_BASE + 0x1001)
-#define   TSE					(1 << 0)
-#define TR1					_MMIO(MCHBAR_MIRROR_BASE + 0x1006)
-#define TSFS					_MMIO(MCHBAR_MIRROR_BASE + 0x1020)
-#define   TSFS_SLOPE_MASK			0x0000ff00
-#define   TSFS_SLOPE_SHIFT			8
-#define   TSFS_INTR_MASK			0x000000ff
-
-/* Memory latency timer register */
-#define MLTR_ILK				_MMIO(MCHBAR_MIRROR_BASE + 0x1222)
-/* the unit of memory self-refresh latency time is 0.5us */
-#define   MLTR_WM2_MASK				REG_GENMASK(13, 8)
-#define   MLTR_WM1_MASK				REG_GENMASK(5, 0)
-
-#define CSIPLL0					_MMIO(MCHBAR_MIRROR_BASE + 0x2c10)
-#define DDRMPLL1				_MMIO(MCHBAR_MIRROR_BASE + 0x2c20)
-
-#define ILK_GDSR				_MMIO(MCHBAR_MIRROR_BASE + 0x2ca4)
-#define  ILK_GRDOM_FULL				(0 << 1)
-#define  ILK_GRDOM_RENDER			(1 << 1)
-#define  ILK_GRDOM_MEDIA			(3 << 1)
-#define  ILK_GRDOM_MASK				(3 << 1)
-#define  ILK_GRDOM_RESET_ENABLE			(1 << 0)
-
-#define BXT_D_CR_DRP0_DUNIT8			0x1000
-#define BXT_D_CR_DRP0_DUNIT9			0x1200
-#define   BXT_D_CR_DRP0_DUNIT_START		8
-#define   BXT_D_CR_DRP0_DUNIT_END		11
-#define BXT_D_CR_DRP0_DUNIT(x)			_MMIO(MCHBAR_MIRROR_BASE_SNB + \
-						      _PICK_EVEN((x) - 8, BXT_D_CR_DRP0_DUNIT8,\
-								 BXT_D_CR_DRP0_DUNIT9))
-#define   BXT_DRAM_RANK_MASK			0x3
-#define   BXT_DRAM_RANK_SINGLE			0x1
-#define   BXT_DRAM_RANK_DUAL			0x3
-#define   BXT_DRAM_WIDTH_MASK			(0x3 << 4)
-#define   BXT_DRAM_WIDTH_SHIFT			4
-#define   BXT_DRAM_WIDTH_X8			(0x0 << 4)
-#define   BXT_DRAM_WIDTH_X16			(0x1 << 4)
-#define   BXT_DRAM_WIDTH_X32			(0x2 << 4)
-#define   BXT_DRAM_WIDTH_X64			(0x3 << 4)
-#define   BXT_DRAM_SIZE_MASK			(0x7 << 6)
-#define   BXT_DRAM_SIZE_SHIFT			6
-#define   BXT_DRAM_SIZE_4GBIT			(0x0 << 6)
-#define   BXT_DRAM_SIZE_6GBIT			(0x1 << 6)
-#define   BXT_DRAM_SIZE_8GBIT			(0x2 << 6)
-#define   BXT_DRAM_SIZE_12GBIT			(0x3 << 6)
-#define   BXT_DRAM_SIZE_16GBIT			(0x4 << 6)
-#define   BXT_DRAM_TYPE_MASK			(0x7 << 22)
-#define   BXT_DRAM_TYPE_SHIFT			22
-#define   BXT_DRAM_TYPE_DDR3			(0x0 << 22)
-#define   BXT_DRAM_TYPE_LPDDR3			(0x1 << 22)
-#define   BXT_DRAM_TYPE_LPDDR4			(0x2 << 22)
-#define   BXT_DRAM_TYPE_DDR4			(0x4 << 22)
-
-#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4000)
-#define   DG1_DRAM_T_RDPRE_MASK			REG_GENMASK(16, 11)
-#define   DG1_DRAM_T_RP_MASK			REG_GENMASK(6, 0)
-#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4004)
-#define   DG1_DRAM_T_RCD_MASK			REG_GENMASK(15, 9)
-#define   DG1_DRAM_T_RAS_MASK			REG_GENMASK(8, 1)
-
-#define SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5000)
-#define   SKL_DRAM_DDR_TYPE_MASK		REG_GENMASK(1, 0)
-#define   SKL_DRAM_DDR_TYPE_DDR4		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 0)
-#define   SKL_DRAM_DDR_TYPE_DDR3		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 1)
-#define   SKL_DRAM_DDR_TYPE_LPDDR3		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 2)
-#define   SKL_DRAM_DDR_TYPE_LPDDR4		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 3)
-
-/* snb MCH registers for reading the DRAM channel configuration */
-#define MAD_DIMM_C0				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004)
-#define MAD_DIMM_C1				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5008)
-#define MAD_DIMM_C2				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C)
-#define   MAD_DIMM_ECC_MASK			(0x3 << 24)
-#define   MAD_DIMM_ECC_OFF			(0x0 << 24)
-#define   MAD_DIMM_ECC_IO_ON_LOGIC_OFF		(0x1 << 24)
-#define   MAD_DIMM_ECC_IO_OFF_LOGIC_ON		(0x2 << 24)
-#define   MAD_DIMM_ECC_ON			(0x3 << 24)
-#define   MAD_DIMM_ENH_INTERLEAVE		(0x1 << 22)
-#define   MAD_DIMM_RANK_INTERLEAVE		(0x1 << 21)
-#define   MAD_DIMM_B_WIDTH_X16			(0x1 << 20) /* X8 chips if unset */
-#define   MAD_DIMM_A_WIDTH_X16			(0x1 << 19) /* X8 chips if unset */
-#define   MAD_DIMM_B_DUAL_RANK			(0x1 << 18)
-#define   MAD_DIMM_A_DUAL_RANK			(0x1 << 17)
-#define   MAD_DIMM_A_SELECT			(0x1 << 16)
-/* DIMM sizes are in multiples of 256mb. */
-#define   MAD_DIMM_B_SIZE_SHIFT			8
-#define   MAD_DIMM_B_SIZE_MASK			(0xff << MAD_DIMM_B_SIZE_SHIFT)
-#define   MAD_DIMM_A_SIZE_SHIFT			0
-#define   MAD_DIMM_A_SIZE_MASK			(0xff << MAD_DIMM_A_SIZE_SHIFT)
-
-#define SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C)
-#define SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5010)
-#define   SKL_DIMM_S_RANK_MASK			REG_GENMASK(26, 26)
-#define   SKL_DIMM_S_RANK_1			REG_FIELD_PREP(SKL_DIMM_S_RANK_MASK, 0)
-#define   SKL_DIMM_S_RANK_2			REG_FIELD_PREP(SKL_DIMM_S_RANK_MASK, 1)
-#define   SKL_DIMM_S_WIDTH_MASK			REG_GENMASK(25, 24)
-#define   SKL_DIMM_S_WIDTH_X8			REG_FIELD_PREP(SKL_DIMM_S_WIDTH_MASK, 0)
-#define   SKL_DIMM_S_WIDTH_X16			REG_FIELD_PREP(SKL_DIMM_S_WIDTH_MASK, 1)
-#define   SKL_DIMM_S_WIDTH_X32			REG_FIELD_PREP(SKL_DIMM_S_WIDTH_MASK, 2)
-#define   SKL_DIMM_S_SIZE_MASK			REG_GENMASK(21, 16)
-#define   SKL_DIMM_L_RANK_MASK			REG_GENMASK(10, 10)
-#define   SKL_DIMM_L_RANK_1			REG_FIELD_PREP(SKL_DIMM_L_RANK_MASK, 0)
-#define   SKL_DIMM_L_RANK_2			REG_FIELD_PREP(SKL_DIMM_L_RANK_MASK, 1)
-#define   SKL_DIMM_L_WIDTH_MASK			REG_GENMASK(9, 8)
-#define   SKL_DIMM_L_WIDTH_X8			REG_FIELD_PREP(SKL_DIMM_L_WIDTH_MASK, 0)
-#define   SKL_DIMM_L_WIDTH_X16			REG_FIELD_PREP(SKL_DIMM_L_WIDTH_MASK, 1)
-#define   SKL_DIMM_L_WIDTH_X32			REG_FIELD_PREP(SKL_DIMM_L_WIDTH_MASK, 2)
-#define   SKL_DIMM_L_SIZE_MASK			REG_GENMASK(5, 0)
-#define   ICL_DIMM_S_RANK_MASK			REG_GENMASK(27, 26)
-#define   ICL_DIMM_S_RANK_1			REG_FIELD_PREP(ICL_DIMM_S_RANK_MASK, 0)
-#define   ICL_DIMM_S_RANK_2			REG_FIELD_PREP(ICL_DIMM_S_RANK_MASK, 1)
-#define   ICL_DIMM_S_WIDTH_MASK			REG_GENMASK(25, 24)
-#define   ICL_DIMM_S_WIDTH_X8			REG_FIELD_PREP(ICL_DIMM_S_WIDTH_MASK, 0)
-#define   ICL_DIMM_S_WIDTH_X16			REG_FIELD_PREP(ICL_DIMM_S_WIDTH_MASK, 1)
-#define   ICL_DIMM_S_WIDTH_X32			REG_FIELD_PREP(ICL_DIMM_S_WIDTH_MASK, 2)
-#define   ICL_DIMM_S_SIZE_MASK			REG_GENMASK(22, 16)
-#define   ICL_DIMM_L_RANK_MASK			REG_GENMASK(10, 9)
-#define   ICL_DIMM_L_RANK_1			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 0)
-#define   ICL_DIMM_L_RANK_2			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 1)
-#define   ICL_DIMM_L_RANK_3			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 2)
-#define   ICL_DIMM_L_RANK_4			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 3)
-#define   ICL_DIMM_L_WIDTH_MASK			REG_GENMASK(8, 7)
-#define   ICL_DIMM_L_WIDTH_X8			REG_FIELD_PREP(ICL_DIMM_L_WIDTH_MASK, 0)
-#define   ICL_DIMM_L_WIDTH_X16			REG_FIELD_PREP(ICL_DIMM_L_WIDTH_MASK, 1)
-#define   ICL_DIMM_L_WIDTH_X32			REG_FIELD_PREP(ICL_DIMM_L_WIDTH_MASK, 2)
-#define   ICL_DIMM_L_SIZE_MASK			REG_GENMASK(6, 0)
-
-#define SA_PERF_STATUS_0_0_0_MCHBAR_PC		_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5918)
-#define  DG1_QCLK_RATIO_MASK			REG_GENMASK(9, 2)
-#define  DG1_QCLK_REFERENCE			REG_BIT(10)
-
-/*
- * *_PACKAGE_POWER_SKU - SKU power and timing parameters.
- */
-#define PCU_PACKAGE_POWER_SKU			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5930)
-#define   PKG_PKG_TDP				GENMASK_ULL(14, 0)
-#define   PKG_MIN_PWR				GENMASK_ULL(30, 16)
-#define   PKG_MAX_PWR				GENMASK_ULL(46, 32)
-#define   PKG_MAX_WIN				GENMASK_ULL(54, 48)
-#define     PKG_MAX_WIN_X			GENMASK_ULL(54, 53)
-#define     PKG_MAX_WIN_Y			GENMASK_ULL(52, 48)
-
-#define PCU_PACKAGE_POWER_SKU_UNIT		_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5938)
-#define   PKG_PWR_UNIT				REG_GENMASK(3, 0)
-#define   PKG_ENERGY_UNIT			REG_GENMASK(12, 8)
-#define   PKG_TIME_UNIT				REG_GENMASK(19, 16)
-#define PCU_PACKAGE_ENERGY_STATUS              _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x593c)
-
-#define GEN6_GT_PERF_STATUS			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5948)
-
-#define PCU_PACKAGE_TEMPERATURE			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5978)
-#define   TEMP_MASK				REG_GENMASK(7, 0)
-
-#define GEN6_RP_STATE_LIMITS			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5994)
-#define GEN6_RP_STATE_CAP			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5998)
-#define   RP0_CAP_MASK				REG_GENMASK(7, 0)
-#define   RP1_CAP_MASK				REG_GENMASK(15, 8)
-#define   RPN_CAP_MASK				REG_GENMASK(23, 16)
-
-#define GEN10_FREQ_INFO_REC			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5ef0)
-#define   RPE_MASK				REG_GENMASK(15, 8)
-#define PCU_PACKAGE_RAPL_LIMIT			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x59a0)
-#define   PKG_PWR_LIM_1				REG_GENMASK(14, 0)
-#define   PKG_PWR_LIM_1_EN			REG_BIT(15)
-#define   PKG_PWR_LIM_1_TIME			REG_GENMASK(23, 17)
-#define   PKG_PWR_LIM_1_TIME_X			REG_GENMASK(23, 22)
-#define   PKG_PWR_LIM_1_TIME_Y			REG_GENMASK(21, 17)
-
-/* snb MCH registers for priority tuning */
-#define MCH_SSKPD				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5d10)
-#define   SSKPD_NEW_WM0_MASK_HSW		REG_GENMASK64(63, 56)
-#define   SSKPD_WM4_MASK_HSW			REG_GENMASK64(40, 32)
-#define   SSKPD_WM3_MASK_HSW			REG_GENMASK64(28, 20)
-#define   SSKPD_WM2_MASK_HSW			REG_GENMASK64(19, 12)
-#define   SSKPD_WM1_MASK_HSW			REG_GENMASK64(11, 4)
-#define   SSKPD_OLD_WM0_MASK_HSW		REG_GENMASK64(3, 0)
-#define   SSKPD_WM3_MASK_SNB			REG_GENMASK(29, 24)
-#define   SSKPD_WM2_MASK_SNB			REG_GENMASK(21, 16)
-#define   SSKPD_WM1_MASK_SNB			REG_GENMASK(13, 8)
-#define   SSKPD_WM0_MASK_SNB			REG_GENMASK(5, 0)
-
-/* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */
-#define DCLK					_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04)
-#define SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04)
-#define   DG1_GEAR_TYPE				REG_BIT(16)
-
-/*
- * Please see hsw_read_dcomp() and hsw_write_dcomp() before using this register,
- * since on HSW we can't write to it using intel_uncore_write.
- */
-#define D_COMP_HSW				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5f0c)
-#define  D_COMP_RCOMP_IN_PROGRESS		(1 << 9)
-#define  D_COMP_COMP_FORCE			(1 << 8)
-#define  D_COMP_COMP_DISABLE			(1 << 0)
-
-#define BXT_GT_PERF_STATUS			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x7070)
-
-#endif /* __INTEL_MCHBAR_REGS */
diff --git a/drivers/gpu/drm/i915/intel_pci_config.h b/drivers/gpu/drm/i915/intel_pci_config.h
deleted file mode 100644
index ebe040828e204..0000000000000
--- a/drivers/gpu/drm/i915/intel_pci_config.h
+++ /dev/null
@@ -1,110 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2022 Intel Corporation
- */
-
-#ifndef __INTEL_PCI_CONFIG_H__
-#define __INTEL_PCI_CONFIG_H__
-
-/* PCI BARs */
-#define GEN2_GMADR_BAR				0
-#define GEN2_MMADR_BAR				1 /* MMIO+GTT, despite the name */
-#define GEN2_IO_BAR				2 /* 85x/865 */
-
-#define GEN3_MMADR_BAR				0 /* MMIO only */
-#define GEN3_IO_BAR				1
-#define GEN3_GMADR_BAR				2
-#define GEN3_GTTADR_BAR				3 /* GTT only */
-
-#define GEN4_GTTMMADR_BAR			0 /* MMIO+GTT */
-#define GEN4_GMADR_BAR				2
-#define GEN4_IO_BAR				4
-
-#define GEN12_LMEM_BAR				2
-
-static inline int intel_mmio_bar(int graphics_ver)
-{
-	switch (graphics_ver) {
-	case 2: return GEN2_MMADR_BAR;
-	case 3: return GEN3_MMADR_BAR;
-	default: return GEN4_GTTMMADR_BAR;
-	}
-}
-
-/* BSM in include/drm/intel/i915_drm.h */
-
-#define MCHBAR_I915				0x44
-#define MCHBAR_I965				0x48
-#define   MCHBAR_SIZE				(4 * 4096)
-
-#define DEVEN					0x54
-#define   DEVEN_MCHBAR_EN			(1 << 28)
-
-#define HPLLCC					0xc0 /* 85x only */
-#define   GC_CLOCK_CONTROL_MASK			(0x7 << 0)
-#define   GC_CLOCK_133_200			(0 << 0)
-#define   GC_CLOCK_100_200			(1 << 0)
-#define   GC_CLOCK_100_133			(2 << 0)
-#define   GC_CLOCK_133_266			(3 << 0)
-#define   GC_CLOCK_133_200_2			(4 << 0)
-#define   GC_CLOCK_133_266_2			(5 << 0)
-#define   GC_CLOCK_166_266			(6 << 0)
-#define   GC_CLOCK_166_250			(7 << 0)
-
-#define I915_GDRST				0xc0
-#define   GRDOM_FULL				(0 << 2)
-#define   GRDOM_RENDER				(1 << 2)
-#define   GRDOM_MEDIA				(3 << 2)
-#define   GRDOM_MASK				(3 << 2)
-#define   GRDOM_RESET_STATUS			(1 << 1)
-#define   GRDOM_RESET_ENABLE			(1 << 0)
-
-/* BSpec only has register offset, PCI device and bit found empirically */
-#define I830_CLOCK_GATE				0xc8 /* device 0 */
-#define   I830_L2_CACHE_CLOCK_GATE_DISABLE	(1 << 2)
-
-#define GCDGMBUS				0xcc
-
-#define GCFGC2					0xda
-#define GCFGC					0xf0 /* 915+ only */
-#define   GC_LOW_FREQUENCY_ENABLE		(1 << 7)
-#define   GC_DISPLAY_CLOCK_190_200_MHZ		(0 << 4)
-#define   GC_DISPLAY_CLOCK_333_320_MHZ		(4 << 4)
-#define   GC_DISPLAY_CLOCK_267_MHZ_PNV		(0 << 4)
-#define   GC_DISPLAY_CLOCK_333_MHZ_PNV		(1 << 4)
-#define   GC_DISPLAY_CLOCK_444_MHZ_PNV		(2 << 4)
-#define   GC_DISPLAY_CLOCK_200_MHZ_PNV		(5 << 4)
-#define   GC_DISPLAY_CLOCK_133_MHZ_PNV		(6 << 4)
-#define   GC_DISPLAY_CLOCK_167_MHZ_PNV		(7 << 4)
-#define   GC_DISPLAY_CLOCK_MASK			(7 << 4)
-#define   GM45_GC_RENDER_CLOCK_MASK		(0xf << 0)
-#define   GM45_GC_RENDER_CLOCK_266_MHZ		(8 << 0)
-#define   GM45_GC_RENDER_CLOCK_320_MHZ		(9 << 0)
-#define   GM45_GC_RENDER_CLOCK_400_MHZ		(0xb << 0)
-#define   GM45_GC_RENDER_CLOCK_533_MHZ		(0xc << 0)
-#define   I965_GC_RENDER_CLOCK_MASK		(0xf << 0)
-#define   I965_GC_RENDER_CLOCK_267_MHZ		(2 << 0)
-#define   I965_GC_RENDER_CLOCK_333_MHZ		(3 << 0)
-#define   I965_GC_RENDER_CLOCK_444_MHZ		(4 << 0)
-#define   I965_GC_RENDER_CLOCK_533_MHZ		(5 << 0)
-#define   I945_GC_RENDER_CLOCK_MASK		(7 << 0)
-#define   I945_GC_RENDER_CLOCK_166_MHZ		(0 << 0)
-#define   I945_GC_RENDER_CLOCK_200_MHZ		(1 << 0)
-#define   I945_GC_RENDER_CLOCK_250_MHZ		(3 << 0)
-#define   I945_GC_RENDER_CLOCK_400_MHZ		(5 << 0)
-#define   I915_GC_RENDER_CLOCK_MASK		(7 << 0)
-#define   I915_GC_RENDER_CLOCK_166_MHZ		(0 << 0)
-#define   I915_GC_RENDER_CLOCK_200_MHZ		(1 << 0)
-#define   I915_GC_RENDER_CLOCK_333_MHZ		(4 << 0)
-
-#define ASLE					0xe4
-#define ASLS					0xfc
-
-#define SWSCI					0xe8
-#define   SWSCI_SCISEL				(1 << 15)
-#define   SWSCI_GSSCIE				(1 << 0)
-
-/* legacy/combination backlight modes, also called LBB */
-#define LBPC					0xf4
-
-#endif /* __INTEL_PCI_CONFIG_H__ */
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 27dc0e40a8d75..796c9b070e41f 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -277,6 +277,7 @@ void mock_destroy_device(struct drm_i915_private *i915)
 	struct device *dev = i915->drm.dev;
 
 	intel_display_device_remove(i915->display);
+	i915->display = NULL;
 
 	devres_release_group(dev, NULL);
 	put_device(dev);
diff --git a/drivers/gpu/drm/i915/vlv_iosf_sb.c b/drivers/gpu/drm/i915/vlv_iosf_sb.c
index 38a75651b0dc3..1f0332b4ad0df 100644
--- a/drivers/gpu/drm/i915/vlv_iosf_sb.c
+++ b/drivers/gpu/drm/i915/vlv_iosf_sb.c
@@ -4,6 +4,7 @@
  */
 
 #include <drm/drm_print.h>
+#include <drm/intel/display_parent_interface.h>
 
 #include "i915_drv.h"
 #include "i915_iosf_mbi.h"
@@ -229,3 +230,10 @@ void vlv_iosf_sb_fini(struct drm_i915_private *i915)
 	if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
 		mutex_destroy(&i915->vlv_iosf_sb.lock);
 }
+
+const struct intel_display_vlv_iosf_interface i915_display_vlv_iosf_interface = {
+	.get = vlv_iosf_sb_get,
+	.put = vlv_iosf_sb_put,
+	.read = vlv_iosf_sb_read,
+	.write = vlv_iosf_sb_write,
+};
diff --git a/drivers/gpu/drm/i915/vlv_iosf_sb.h b/drivers/gpu/drm/i915/vlv_iosf_sb.h
index e2fea29a30ea2..8129ba11c7509 100644
--- a/drivers/gpu/drm/i915/vlv_iosf_sb.h
+++ b/drivers/gpu/drm/i915/vlv_iosf_sb.h
@@ -8,23 +8,11 @@
 
 #include <linux/types.h>
 
-#include "vlv_iosf_sb_reg.h"
+#include <drm/intel/vlv_iosf_sb_regs.h>
 
 struct drm_device;
 struct drm_i915_private;
 
-enum vlv_iosf_sb_unit {
-	VLV_IOSF_SB_BUNIT,
-	VLV_IOSF_SB_CCK,
-	VLV_IOSF_SB_CCU,
-	VLV_IOSF_SB_DPIO,
-	VLV_IOSF_SB_DPIO_2,
-	VLV_IOSF_SB_FLISDSI,
-	VLV_IOSF_SB_GPIO,
-	VLV_IOSF_SB_NC,
-	VLV_IOSF_SB_PUNIT,
-};
-
 void vlv_iosf_sb_init(struct drm_i915_private *i915);
 void vlv_iosf_sb_fini(struct drm_i915_private *i915);
 
@@ -34,4 +22,6 @@ void vlv_iosf_sb_put(struct drm_device *drm, unsigned long unit_mask);
 u32 vlv_iosf_sb_read(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr);
 int vlv_iosf_sb_write(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr, u32 val);
 
+extern const struct intel_display_vlv_iosf_interface i915_display_vlv_iosf_interface;
+
 #endif /* _VLV_IOSF_SB_H_ */
diff --git a/drivers/gpu/drm/i915/vlv_iosf_sb_reg.h b/drivers/gpu/drm/i915/vlv_iosf_sb_reg.h
deleted file mode 100644
index f977fb3b6e173..0000000000000
--- a/drivers/gpu/drm/i915/vlv_iosf_sb_reg.h
+++ /dev/null
@@ -1,180 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2022 Intel Corporation
- */
-
-#ifndef _VLV_IOSF_SB_REG_H_
-#define _VLV_IOSF_SB_REG_H_
-
-/* See configdb bunit SB addr map */
-#define BUNIT_REG_BISOC				0x11
-
-/* PUNIT_REG_*SSPM0 */
-#define   _SSPM0_SSC(val)			((val) << 0)
-#define   SSPM0_SSC_MASK			_SSPM0_SSC(0x3)
-#define   SSPM0_SSC_PWR_ON			_SSPM0_SSC(0x0)
-#define   SSPM0_SSC_CLK_GATE			_SSPM0_SSC(0x1)
-#define   SSPM0_SSC_RESET			_SSPM0_SSC(0x2)
-#define   SSPM0_SSC_PWR_GATE			_SSPM0_SSC(0x3)
-#define   _SSPM0_SSS(val)			((val) << 24)
-#define   SSPM0_SSS_MASK			_SSPM0_SSS(0x3)
-#define   SSPM0_SSS_PWR_ON			_SSPM0_SSS(0x0)
-#define   SSPM0_SSS_CLK_GATE			_SSPM0_SSS(0x1)
-#define   SSPM0_SSS_RESET			_SSPM0_SSS(0x2)
-#define   SSPM0_SSS_PWR_GATE			_SSPM0_SSS(0x3)
-
-/* PUNIT_REG_*SSPM1 */
-#define   SSPM1_FREQSTAT_SHIFT			24
-#define   SSPM1_FREQSTAT_MASK			(0x1f << SSPM1_FREQSTAT_SHIFT)
-#define   SSPM1_FREQGUAR_SHIFT			8
-#define   SSPM1_FREQGUAR_MASK			(0x1f << SSPM1_FREQGUAR_SHIFT)
-#define   SSPM1_FREQ_SHIFT			0
-#define   SSPM1_FREQ_MASK			(0x1f << SSPM1_FREQ_SHIFT)
-
-#define PUNIT_REG_VEDSSPM0			0x32
-#define PUNIT_REG_VEDSSPM1			0x33
-
-#define PUNIT_REG_DSPSSPM			0x36
-#define   DSPFREQSTAT_SHIFT_CHV			24
-#define   DSPFREQSTAT_MASK_CHV			(0x1f << DSPFREQSTAT_SHIFT_CHV)
-#define   DSPFREQGUAR_SHIFT_CHV			8
-#define   DSPFREQGUAR_MASK_CHV			(0x1f << DSPFREQGUAR_SHIFT_CHV)
-#define   DSPFREQSTAT_SHIFT			30
-#define   DSPFREQSTAT_MASK			(0x3 << DSPFREQSTAT_SHIFT)
-#define   DSPFREQGUAR_SHIFT			14
-#define   DSPFREQGUAR_MASK			(0x3 << DSPFREQGUAR_SHIFT)
-#define   DSP_MAXFIFO_PM5_STATUS		(1 << 22) /* chv */
-#define   DSP_AUTO_CDCLK_GATE_DISABLE		(1 << 7) /* chv */
-#define   DSP_MAXFIFO_PM5_ENABLE		(1 << 6) /* chv */
-#define   _DP_SSC(val, pipe)			((val) << (2 * (pipe)))
-#define   DP_SSC_MASK(pipe)			_DP_SSC(0x3, (pipe))
-#define   DP_SSC_PWR_ON(pipe)			_DP_SSC(0x0, (pipe))
-#define   DP_SSC_CLK_GATE(pipe)			_DP_SSC(0x1, (pipe))
-#define   DP_SSC_RESET(pipe)			_DP_SSC(0x2, (pipe))
-#define   DP_SSC_PWR_GATE(pipe)			_DP_SSC(0x3, (pipe))
-#define   _DP_SSS(val, pipe)			((val) << (2 * (pipe) + 16))
-#define   DP_SSS_MASK(pipe)			_DP_SSS(0x3, (pipe))
-#define   DP_SSS_PWR_ON(pipe)			_DP_SSS(0x0, (pipe))
-#define   DP_SSS_CLK_GATE(pipe)			_DP_SSS(0x1, (pipe))
-#define   DP_SSS_RESET(pipe)			_DP_SSS(0x2, (pipe))
-#define   DP_SSS_PWR_GATE(pipe)			_DP_SSS(0x3, (pipe))
-
-#define PUNIT_REG_ISPSSPM0			0x39
-#define PUNIT_REG_ISPSSPM1			0x3a
-
-#define PUNIT_REG_PWRGT_CTRL			0x60
-#define PUNIT_REG_PWRGT_STATUS			0x61
-#define   PUNIT_PWRGT_MASK(pw_idx)		(3 << ((pw_idx) * 2))
-#define   PUNIT_PWRGT_PWR_ON(pw_idx)		(0 << ((pw_idx) * 2))
-#define   PUNIT_PWRGT_CLK_GATE(pw_idx)		(1 << ((pw_idx) * 2))
-#define   PUNIT_PWRGT_RESET(pw_idx)		(2 << ((pw_idx) * 2))
-#define   PUNIT_PWRGT_PWR_GATE(pw_idx)		(3 << ((pw_idx) * 2))
-
-#define PUNIT_PWGT_IDX_RENDER			0
-#define PUNIT_PWGT_IDX_MEDIA			1
-#define PUNIT_PWGT_IDX_DISP2D			3
-#define PUNIT_PWGT_IDX_DPIO_CMN_BC		5
-#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_01	6
-#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_23	7
-#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_01	8
-#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_23	9
-#define PUNIT_PWGT_IDX_DPIO_RX0			10
-#define PUNIT_PWGT_IDX_DPIO_RX1			11
-#define PUNIT_PWGT_IDX_DPIO_CMN_D		12
-
-#define PUNIT_REG_GPU_LFM			0xd3
-#define PUNIT_REG_GPU_FREQ_REQ			0xd4
-#define PUNIT_REG_GPU_FREQ_STS			0xd8
-#define   GPLLENABLE				(1 << 4)
-#define   GENFREQSTATUS				(1 << 0)
-#define PUNIT_REG_MEDIA_TURBO_FREQ_REQ		0xdc
-#define PUNIT_REG_CZ_TIMESTAMP			0xce
-
-#define PUNIT_FUSE_BUS2				0xf6 /* bits 47:40 */
-#define PUNIT_FUSE_BUS1				0xf5 /* bits 55:48 */
-
-#define FB_GFX_FMAX_AT_VMAX_FUSE		0x136
-#define FB_GFX_FREQ_FUSE_MASK			0xff
-#define FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT	24
-#define FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT	16
-#define FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT	8
-
-#define FB_GFX_FMIN_AT_VMIN_FUSE		0x137
-#define FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT		8
-
-#define PUNIT_REG_DDR_SETUP2			0x139
-#define   FORCE_DDR_FREQ_REQ_ACK		(1 << 8)
-#define   FORCE_DDR_LOW_FREQ			(1 << 1)
-#define   FORCE_DDR_HIGH_FREQ			(1 << 0)
-
-#define PUNIT_GPU_STATUS_REG			0xdb
-#define PUNIT_GPU_STATUS_MAX_FREQ_SHIFT	16
-#define PUNIT_GPU_STATUS_MAX_FREQ_MASK		0xff
-#define PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT	8
-#define PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK	0xff
-
-#define PUNIT_GPU_DUTYCYCLE_REG		0xdf
-#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT	8
-#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK	0xff
-
-#define IOSF_NC_FB_GFX_FREQ_FUSE		0x1c
-#define   FB_GFX_MAX_FREQ_FUSE_SHIFT		3
-#define   FB_GFX_MAX_FREQ_FUSE_MASK		0x000007f8
-#define   FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT	11
-#define   FB_GFX_FGUARANTEED_FREQ_FUSE_MASK	0x0007f800
-#define IOSF_NC_FB_GFX_FMAX_FUSE_HI		0x34
-#define   FB_FMAX_VMIN_FREQ_HI_MASK		0x00000007
-#define IOSF_NC_FB_GFX_FMAX_FUSE_LO		0x30
-#define   FB_FMAX_VMIN_FREQ_LO_SHIFT		27
-#define   FB_FMAX_VMIN_FREQ_LO_MASK		0xf8000000
-
-#define VLV_TURBO_SOC_OVERRIDE		0x04
-#define   VLV_OVERRIDE_EN		1
-#define   VLV_SOC_TDP_EN		(1 << 1)
-#define   VLV_BIAS_CPU_125_SOC_875	(6 << 2)
-#define   CHV_BIAS_CPU_50_SOC_50	(3 << 2)
-
-/* vlv2 north clock has */
-#define CCK_FUSE_REG				0x8
-#define  CCK_FUSE_HPLL_FREQ_MASK		0x3
-#define CCK_REG_DSI_PLL_FUSE			0x44
-#define CCK_REG_DSI_PLL_CONTROL			0x48
-#define  DSI_PLL_VCO_EN				(1 << 31)
-#define  DSI_PLL_LDO_GATE			(1 << 30)
-#define  DSI_PLL_P1_POST_DIV_SHIFT		17
-#define  DSI_PLL_P1_POST_DIV_MASK		(0x1ff << 17)
-#define  DSI_PLL_P2_MUX_DSI0_DIV2		(1 << 13)
-#define  DSI_PLL_P3_MUX_DSI1_DIV2		(1 << 12)
-#define  DSI_PLL_MUX_MASK			(3 << 9)
-#define  DSI_PLL_MUX_DSI0_DSIPLL		(0 << 10)
-#define  DSI_PLL_MUX_DSI0_CCK			(1 << 10)
-#define  DSI_PLL_MUX_DSI1_DSIPLL		(0 << 9)
-#define  DSI_PLL_MUX_DSI1_CCK			(1 << 9)
-#define  DSI_PLL_CLK_GATE_MASK			(0xf << 5)
-#define  DSI_PLL_CLK_GATE_DSI0_DSIPLL		(1 << 8)
-#define  DSI_PLL_CLK_GATE_DSI1_DSIPLL		(1 << 7)
-#define  DSI_PLL_CLK_GATE_DSI0_CCK		(1 << 6)
-#define  DSI_PLL_CLK_GATE_DSI1_CCK		(1 << 5)
-#define  DSI_PLL_LOCK				(1 << 0)
-#define CCK_REG_DSI_PLL_DIVIDER			0x4c
-#define  DSI_PLL_LFSR				(1 << 31)
-#define  DSI_PLL_FRACTION_EN			(1 << 30)
-#define  DSI_PLL_FRAC_COUNTER_SHIFT		27
-#define  DSI_PLL_FRAC_COUNTER_MASK		(7 << 27)
-#define  DSI_PLL_USYNC_CNT_SHIFT		18
-#define  DSI_PLL_USYNC_CNT_MASK			(0x1ff << 18)
-#define  DSI_PLL_N1_DIV_SHIFT			16
-#define  DSI_PLL_N1_DIV_MASK			(3 << 16)
-#define  DSI_PLL_M1_DIV_SHIFT			0
-#define  DSI_PLL_M1_DIV_MASK			(0x1ff << 0)
-#define CCK_CZ_CLOCK_CONTROL			0x62
-#define CCK_GPLL_CLOCK_CONTROL			0x67
-#define CCK_DISPLAY_CLOCK_CONTROL		0x6b
-#define CCK_DISPLAY_REF_CLOCK_CONTROL		0x6c
-#define  CCK_TRUNK_FORCE_ON			(1 << 17)
-#define  CCK_TRUNK_FORCE_OFF			(1 << 16)
-#define  CCK_FREQUENCY_STATUS			(0x1f << 8)
-#define  CCK_FREQUENCY_STATUS_SHIFT		8
-#define  CCK_FREQUENCY_VALUES			(0x1f << 0)
-
-#endif /* _VLV_IOSF_SB_REG_H_ */
diff --git a/drivers/gpu/drm/imagination/Makefile b/drivers/gpu/drm/imagination/Makefile
index 1222a14262e4a..a5c5e386f16a7 100644
--- a/drivers/gpu/drm/imagination/Makefile
+++ b/drivers/gpu/drm/imagination/Makefile
@@ -32,6 +32,9 @@ powervr-y := \
 powervr-$(CONFIG_DEBUG_FS) += \
 	pvr_debugfs.o
 
+powervr-$(CONFIG_TRACEPOINTS) += \
+	pvr_trace_points.o
+
 obj-$(CONFIG_DRM_POWERVR) += powervr.o
 
 obj-$(CONFIG_DRM_POWERVR_KUNIT_TEST) += pvr_test.o
diff --git a/drivers/gpu/drm/imagination/pvr_context.c b/drivers/gpu/drm/imagination/pvr_context.c
index 8de70c30b9de0..eba4694400b5a 100644
--- a/drivers/gpu/drm/imagination/pvr_context.c
+++ b/drivers/gpu/drm/imagination/pvr_context.c
@@ -318,10 +318,14 @@ int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_co
 		goto err_put_vm;
 	}
 
-	err = pvr_context_create_queues(ctx, args, ctx->data);
+	err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL);
 	if (err)
 		goto err_free_ctx_data;
 
+	err = pvr_context_create_queues(ctx, args, ctx->data);
+	if (err)
+		goto err_free_ctx_id;
+
 	err = init_fw_objs(ctx, args, ctx->data);
 	if (err)
 		goto err_destroy_queues;
@@ -329,23 +333,12 @@ int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_co
 	err = pvr_fw_object_create(pvr_dev, ctx_size, PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 				   ctx_fw_data_init, ctx, &ctx->fw_obj);
 	if (err)
-		goto err_free_ctx_data;
+		goto err_destroy_queues;
 
-	err = xa_alloc(&pvr_dev->ctx_ids, &ctx->ctx_id, ctx, xa_limit_32b, GFP_KERNEL);
+	err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL);
 	if (err)
 		goto err_destroy_fw_obj;
 
-	err = xa_alloc(&pvr_file->ctx_handles, &args->handle, ctx, xa_limit_32b, GFP_KERNEL);
-	if (err) {
-		/*
-		 * It's possible that another thread could have taken a reference on the context at
-		 * this point as it is in the ctx_ids xarray. Therefore instead of directly
-		 * destroying the context, drop a reference instead.
-		 */
-		pvr_context_put(ctx);
-		return err;
-	}
-
 	spin_lock(&pvr_dev->ctx_list_lock);
 	list_add_tail(&ctx->file_link, &pvr_file->contexts);
 	spin_unlock(&pvr_dev->ctx_list_lock);
@@ -358,6 +351,15 @@ int pvr_context_create(struct pvr_file *pvr_file, struct drm_pvr_ioctl_create_co
 err_destroy_queues:
 	pvr_context_destroy_queues(ctx);
 
+err_free_ctx_id:
+	/*
+	 * Ctx_id is not exposed to userspace and not visible yet within
+	 * the kernel/FW, plus a matching context handle (exposed to userspace)
+	 * hasn't been allocated yet, so it is safe to remove ctx_id
+	 * from the ctx_ids xarray.
+	 */
+	xa_erase(&pvr_dev->ctx_ids, ctx->ctx_id);
+
 err_free_ctx_data:
 	kfree(ctx->data);
 
diff --git a/drivers/gpu/drm/imagination/pvr_device.c b/drivers/gpu/drm/imagination/pvr_device.c
index dbb6f5a8ded12..2691ef9af0ca8 100644
--- a/drivers/gpu/drm/imagination/pvr_device.c
+++ b/drivers/gpu/drm/imagination/pvr_device.c
@@ -213,7 +213,7 @@ static irqreturn_t pvr_device_irq_thread_handler(int irq, void *data)
 	while (pvr_fw_irq_pending(pvr_dev)) {
 		pvr_fw_irq_clear(pvr_dev);
 
-		if (pvr_dev->fw_dev.booted) {
+		if (READ_ONCE(pvr_dev->fw_dev.initialised)) {
 			pvr_fwccb_process(pvr_dev);
 			pvr_kccb_wake_up_waiters(pvr_dev);
 			pvr_device_process_active_queues(pvr_dev);
diff --git a/drivers/gpu/drm/imagination/pvr_drv.c b/drivers/gpu/drm/imagination/pvr_drv.c
index 268900464ab69..b20c462bcba0b 100644
--- a/drivers/gpu/drm/imagination/pvr_drv.c
+++ b/drivers/gpu/drm/imagination/pvr_drv.c
@@ -14,6 +14,7 @@
 #include "pvr_rogue_defs.h"
 #include "pvr_rogue_fwif_client.h"
 #include "pvr_rogue_fwif_shared.h"
+#include "pvr_trace.h"
 #include "pvr_vm.h"
 
 #include <uapi/drm/pvr_drm.h>
@@ -1150,6 +1151,8 @@ pvr_ioctl_submit_jobs(struct drm_device *drm_dev, void *raw_args,
 	int idx;
 	int err;
 
+	trace_pvr_job_submit_ioctl(pvr_dev, args->jobs.count);
+
 	if (!drm_dev_enter(drm_dev, &idx))
 		return -EIO;
 
diff --git a/drivers/gpu/drm/imagination/pvr_fw.c b/drivers/gpu/drm/imagination/pvr_fw.c
index 288516dc2560f..850a3ec8e7750 100644
--- a/drivers/gpu/drm/imagination/pvr_fw.c
+++ b/drivers/gpu/drm/imagination/pvr_fw.c
@@ -1004,7 +1004,7 @@ pvr_fw_init(struct pvr_device *pvr_dev)
 		goto err_fw_stop;
 	}
 
-	fw_dev->booted = true;
+	WRITE_ONCE(fw_dev->initialised, true);
 
 	return 0;
 
@@ -1044,7 +1044,7 @@ pvr_fw_fini(struct pvr_device *pvr_dev)
 {
 	struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 
-	fw_dev->booted = false;
+	WRITE_ONCE(fw_dev->initialised, false);
 
 	pvr_fw_destroy_structures(pvr_dev);
 	pvr_fw_object_unmap_and_destroy(pvr_dev->kccb.rtn_obj);
diff --git a/drivers/gpu/drm/imagination/pvr_fw.h b/drivers/gpu/drm/imagination/pvr_fw.h
index 1404dd492d7cd..3390c84e4fd31 100644
--- a/drivers/gpu/drm/imagination/pvr_fw.h
+++ b/drivers/gpu/drm/imagination/pvr_fw.h
@@ -295,8 +295,11 @@ struct pvr_fw_device {
 	/** @mem: Structure containing objects representing firmware memory allocations. */
 	struct pvr_fw_mem mem;
 
-	/** @booted: %true if the firmware has been booted, %false otherwise. */
-	bool booted;
+	/**
+	 * @initialised: %true if the firmware has been successfully initialised,
+	 *               %false otherwise.
+	 */
+	bool initialised;
 
 	/**
 	 * @processor_type: FW processor type for this device. Must be one of
diff --git a/drivers/gpu/drm/imagination/pvr_fw_trace.c b/drivers/gpu/drm/imagination/pvr_fw_trace.c
index 6193811ef7beb..6bb5baa6c41b7 100644
--- a/drivers/gpu/drm/imagination/pvr_fw_trace.c
+++ b/drivers/gpu/drm/imagination/pvr_fw_trace.c
@@ -77,7 +77,7 @@ const struct kernel_param_ops pvr_fw_trace_init_mask_ops = {
 };
 
 param_check_hexint(init_fw_trace_mask, &pvr_fw_trace_init_mask);
-module_param_cb(init_fw_trace_mask, &pvr_fw_trace_init_mask_ops, &pvr_fw_trace_init_mask, 0600);
+module_param_cb(init_fw_trace_mask, &pvr_fw_trace_init_mask_ops, &pvr_fw_trace_init_mask, 0400);
 __MODULE_PARM_TYPE(init_fw_trace_mask, "hexint");
 MODULE_PARM_DESC(init_fw_trace_mask,
 		 "Enable FW trace for the specified groups at device init time");
diff --git a/drivers/gpu/drm/imagination/pvr_job.c b/drivers/gpu/drm/imagination/pvr_job.c
index 0c2f511a61785..b8a58d817009b 100644
--- a/drivers/gpu/drm/imagination/pvr_job.c
+++ b/drivers/gpu/drm/imagination/pvr_job.c
@@ -14,6 +14,7 @@
 #include "pvr_stream.h"
 #include "pvr_stream_defs.h"
 #include "pvr_sync.h"
+#include "pvr_trace.h"
 
 #include <drm/drm_exec.h>
 #include <drm/drm_gem.h>
@@ -326,7 +327,7 @@ prepare_job_syncs(struct pvr_file *pvr_file,
 		  struct pvr_job_data *job_data,
 		  struct xarray *signal_array)
 {
-	struct dma_fence *done_fence;
+	struct dma_fence *finished_fence;
 	int err = pvr_sync_signal_array_collect_ops(signal_array,
 						    from_pvr_file(pvr_file),
 						    job_data->sync_op_count,
@@ -359,13 +360,13 @@ prepare_job_syncs(struct pvr_file *pvr_file,
 			return err;
 	}
 
-	/* We need to arm the job to get the job done fence. */
-	done_fence = pvr_queue_job_arm(job_data->job);
+	/* We need to arm the job to get the job finished fence. */
+	finished_fence = pvr_queue_job_arm(job_data->job);
 
 	err = pvr_sync_signal_array_update_fences(signal_array,
 						  job_data->sync_op_count,
 						  job_data->sync_ops,
-						  done_fence);
+						  finished_fence);
 	return err;
 }
 
@@ -510,6 +511,8 @@ static int pvr_job_data_init(struct pvr_device *pvr_dev,
 		}
 
 		job_data_out[i].sync_op_count = job_args[i].sync_ops.count;
+
+		trace_pvr_job_create(pvr_dev, job_data_out[i].job, job_data_out[i].sync_op_count);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/imagination/pvr_mmu.c b/drivers/gpu/drm/imagination/pvr_mmu.c
index 2e4da5b2c4996..3cac482e10347 100644
--- a/drivers/gpu/drm/imagination/pvr_mmu.c
+++ b/drivers/gpu/drm/imagination/pvr_mmu.c
@@ -133,8 +133,8 @@ int pvr_mmu_flush_exec(struct pvr_device *pvr_dev, bool wait)
 	if (!drm_dev_enter(from_pvr_device(pvr_dev), &idx))
 		return -EIO;
 
-	/* Can't flush MMU if the firmware hasn't booted yet. */
-	if (!pvr_dev->fw_dev.booted)
+	/* Can't flush MMU if the firmware hasn't been initialised yet. */
+	if (!READ_ONCE(pvr_dev->fw_dev.initialised))
 		goto err_drm_dev_exit;
 
 	cmd_mmu_cache_data->cache_flags =
diff --git a/drivers/gpu/drm/imagination/pvr_power.c b/drivers/gpu/drm/imagination/pvr_power.c
index 88fcce433d123..a71d5b35601e7 100644
--- a/drivers/gpu/drm/imagination/pvr_power.c
+++ b/drivers/gpu/drm/imagination/pvr_power.c
@@ -216,7 +216,7 @@ pvr_watchdog_worker(struct work_struct *work)
 	if (pm_runtime_get_if_in_use(from_pvr_device(pvr_dev)->dev) <= 0)
 		goto out_requeue;
 
-	if (!pvr_dev->fw_dev.booted)
+	if (!READ_ONCE(pvr_dev->fw_dev.initialised))
 		goto out_pm_runtime_put;
 
 	stalled = pvr_watchdog_kccb_stalled(pvr_dev);
@@ -378,7 +378,7 @@ pvr_power_device_suspend(struct device *dev)
 	if (!drm_dev_enter(drm_dev, &idx))
 		return -EIO;
 
-	if (pvr_dev->fw_dev.booted) {
+	if (READ_ONCE(pvr_dev->fw_dev.initialised)) {
 		err = pvr_power_fw_disable(pvr_dev, false, true);
 		if (err)
 			goto err_drm_dev_exit;
@@ -408,7 +408,7 @@ pvr_power_device_resume(struct device *dev)
 	if (err)
 		goto err_drm_dev_exit;
 
-	if (pvr_dev->fw_dev.booted) {
+	if (READ_ONCE(pvr_dev->fw_dev.initialised)) {
 		err = pvr_power_fw_enable(pvr_dev, true);
 		if (err)
 			goto err_power_off;
@@ -548,7 +548,7 @@ pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset)
 		err = pvr_power_fw_disable(pvr_dev, hard_reset, false);
 		if (!err) {
 			if (hard_reset) {
-				pvr_dev->fw_dev.booted = false;
+				WRITE_ONCE(pvr_dev->fw_dev.initialised, false);
 				WARN_ON(pvr_power_device_suspend(from_pvr_device(pvr_dev)->dev));
 
 				err = pvr_fw_hard_reset(pvr_dev);
@@ -556,7 +556,7 @@ pvr_power_reset(struct pvr_device *pvr_dev, bool hard_reset)
 					goto err_device_lost;
 
 				err = pvr_power_device_resume(from_pvr_device(pvr_dev)->dev);
-				pvr_dev->fw_dev.booted = true;
+				WRITE_ONCE(pvr_dev->fw_dev.initialised, true);
 				if (err)
 					goto err_device_lost;
 			} else {
diff --git a/drivers/gpu/drm/imagination/pvr_queue.c b/drivers/gpu/drm/imagination/pvr_queue.c
index dd88949f61944..7ed60e1c1a866 100644
--- a/drivers/gpu/drm/imagination/pvr_queue.c
+++ b/drivers/gpu/drm/imagination/pvr_queue.c
@@ -10,6 +10,7 @@
 #include "pvr_drv.h"
 #include "pvr_job.h"
 #include "pvr_queue.h"
+#include "pvr_trace.h"
 #include "pvr_vm.h"
 
 #include "pvr_rogue_fwif_client.h"
@@ -177,11 +178,36 @@ static const struct dma_fence_ops pvr_queue_job_fence_ops = {
 	.release = pvr_queue_fence_release,
 };
 
+/**
+ * pvr_queue_fence_is_ufo_backed() - Check if a dma_fence is backed by a UFO.
+ * @f: The dma_fence to check.
+ *
+ * Return:
+ * * true if the dma_fence is backed by a UFO, or
+ * * false otherwise.
+ */
+static inline bool
+pvr_queue_fence_is_ufo_backed(struct dma_fence *f)
+{
+	/*
+	 * Currently the only dma_fence backed by a UFO object is the job fence,
+	 * e.g. pvr_job::done_fence, wrapped by a pvr_queue_fence object.
+	 */
+	return f && f->ops == &pvr_queue_job_fence_ops;
+}
+
 /**
  * to_pvr_queue_job_fence() - Return a pvr_queue_fence object if the fence is
- * backed by a UFO.
+ * already backed by a UFO.
  * @f: The dma_fence to turn into a pvr_queue_fence.
  *
+ * This could be called on:
+ * - a job fence directly, in which case it simply returns the containing pvr_queue_fence;
+ * - a drm_sched_fence's scheduled or finished fence, in which case it will first try to follow
+ *   the parent pointer to find the job fence (note that the parent pointer is initialized
+ *   only after the run_job() callback is called on the drm_sched_fence's owning job);
+ * - any other dma_fence, in which case it will return NULL.
+ *
  * Return:
  *  * A non-NULL pvr_queue_fence object if the dma_fence is backed by a UFO, or
  *  * NULL otherwise.
@@ -194,7 +220,7 @@ to_pvr_queue_job_fence(struct dma_fence *f)
 	if (sched_fence)
 		f = sched_fence->parent;
 
-	if (f && f->ops == &pvr_queue_job_fence_ops)
+	if (pvr_queue_fence_is_ufo_backed(f))
 		return container_of(f, struct pvr_queue_fence, base);
 
 	return NULL;
@@ -349,11 +375,23 @@ static u32 ufo_cmds_size(u32 elem_count)
 
 static u32 job_cmds_size(struct pvr_job *job, u32 ufo_wait_count)
 {
-	/* One UFO cmd for the fence signaling, one UFO cmd per native fence native,
-	 * and a command for the job itself.
+	/*
+	 * One UFO command per native fence this job will be waiting on (unless any are
+	 * signaled by the time the job is submitted), plus a command for the job itself,
+	 * plus one UFO command for the fence signaling.
 	 */
-	return ufo_cmds_size(1) + ufo_cmds_size(ufo_wait_count) +
-	       pvr_cccb_get_size_of_cmd_with_hdr(job->cmd_len);
+	return ufo_cmds_size(ufo_wait_count) +
+	       pvr_cccb_get_size_of_cmd_with_hdr(job->cmd_len) +
+	       ufo_cmds_size(1);
+}
+
+static bool
+is_paired_job_fence(struct dma_fence *fence, struct pvr_job *job)
+{
+	/* This assumes "fence" is one of "job"'s drm_sched_job::dependencies */
+	return job->type == DRM_PVR_JOB_TYPE_FRAGMENT &&
+	       job->paired_job &&
+	       &job->paired_job->base.s_fence->scheduled == fence;
 }
 
 /**
@@ -371,6 +409,17 @@ static unsigned long job_count_remaining_native_deps(struct pvr_job *job)
 	xa_for_each(&job->base.dependencies, index, fence) {
 		struct pvr_queue_fence *jfence;
 
+		if (is_paired_job_fence(fence, job)) {
+			/*
+			 * A fence between paired jobs won't resolve to a pvr_queue_fence (i.e.
+			 * be backed by a UFO) until the jobs have been submitted, together.
+			 * The submitting code will insert a partial render fence command for this.
+			 */
+			WARN_ON(dma_fence_is_signaled(fence));
+			remaining_count++;
+			continue;
+		}
+
 		jfence = to_pvr_queue_job_fence(fence);
 		if (!jfence)
 			continue;
@@ -468,29 +517,37 @@ pvr_queue_get_job_kccb_fence(struct pvr_queue *queue, struct pvr_job *job)
 }
 
 static struct dma_fence *
-pvr_queue_get_paired_frag_job_dep(struct pvr_queue *queue, struct pvr_job *job)
+pvr_queue_get_paired_frag_job_dep(struct pvr_job *job)
 {
 	struct pvr_job *frag_job = job->type == DRM_PVR_JOB_TYPE_GEOMETRY ?
 				   job->paired_job : NULL;
+	struct pvr_queue *frag_queue = frag_job ? frag_job->ctx->queues.fragment : NULL;
 	struct dma_fence *f;
 	unsigned long index;
 
 	if (!frag_job)
 		return NULL;
 
+	/* Have the geometry job wait on the paired fragment job's dependencies as well. */
 	xa_for_each(&frag_job->base.dependencies, index, f) {
 		/* Skip already signaled fences. */
 		if (dma_fence_is_signaled(f))
 			continue;
 
-		/* Skip our own fence. */
+		/*
+		 * The paired job fence won't be signaled until both jobs have
+		 * been submitted, so we can't wait on it to schedule them.
+		 */
 		if (f == &job->base.s_fence->scheduled)
 			continue;
 
 		return dma_fence_get(f);
 	}
 
-	return frag_job->base.sched->ops->prepare_job(&frag_job->base, &queue->entity);
+	/* Initialize the paired fragment job's done_fence, so we can signal it. */
+	pvr_queue_job_fence_init(frag_job->done_fence, frag_queue);
+
+	return pvr_queue_get_job_cccb_fence(frag_queue, frag_job);
 }
 
 /**
@@ -509,32 +566,25 @@ pvr_queue_prepare_job(struct drm_sched_job *sched_job,
 	struct pvr_queue *queue = container_of(s_entity, struct pvr_queue, entity);
 	struct dma_fence *internal_dep = NULL;
 
-	/*
-	 * Initialize the done_fence, so we can signal it. This must be done
-	 * here because otherwise by the time of run_job() the job will end up
-	 * in the pending list without a valid fence.
-	 */
 	if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) {
 		/*
-		 * This will be called on a paired fragment job after being
-		 * submitted to firmware. We can tell if this is the case and
-		 * bail early from whether run_job() has been called on the
-		 * geometry job, which would issue a pm ref.
+		 * This will be called on a paired fragment job after being submitted
+		 * to the firmware as part of the paired geometry job's submission.
+		 * We can tell if this is the case and bail early from whether run_job()
+		 * has been called on the geometry job, which would issue a pm ref on
+		 * this job as well.
 		 */
-		if (job->paired_job->has_pm_ref)
+		if (job->has_pm_ref)
 			return NULL;
-
-		/*
-		 * In this case we need to use the job's own ctx to initialise
-		 * the done_fence.  The other steps are done in the ctx of the
-		 * paired geometry job.
-		 */
-		pvr_queue_job_fence_init(job->done_fence,
-					 job->ctx->queues.fragment);
-	} else {
-		pvr_queue_job_fence_init(job->done_fence, queue);
 	}
 
+	/*
+	 * Initialize the done_fence, so we can signal it. This must be done
+	 * here because otherwise by the time of run_job() the job will end up
+	 * in the pending list without a valid fence.
+	 */
+	pvr_queue_job_fence_init(job->done_fence, queue);
+
 	/* CCCB fence is used to make sure we have enough space in the CCCB to
 	 * submit our commands.
 	 */
@@ -555,7 +605,7 @@ pvr_queue_prepare_job(struct drm_sched_job *sched_job,
 
 	/* The paired job fence should come last, when everything else is ready. */
 	if (!internal_dep)
-		internal_dep = pvr_queue_get_paired_frag_job_dep(queue, job);
+		internal_dep = pvr_queue_get_paired_frag_job_dep(job);
 
 	return internal_dep;
 }
@@ -630,11 +680,7 @@ static void pvr_queue_submit_job_to_cccb(struct pvr_job *job)
 		if (!jfence)
 			continue;
 
-		/* Skip the partial render fence, we will place it at the end. */
-		if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job &&
-		    &job->paired_job->base.s_fence->scheduled == fence)
-			continue;
-
+		/* Some dependencies might have been signaled since prepare_job() */
 		if (dma_fence_is_signaled(&jfence->base))
 			continue;
 
@@ -649,8 +695,13 @@ static void pvr_queue_submit_job_to_cccb(struct pvr_job *job)
 		}
 	}
 
-	/* Partial render fence goes last. */
 	if (job->type == DRM_PVR_JOB_TYPE_FRAGMENT && job->paired_job) {
+		/*
+		 * The loop above will only process dependencies backed by a UFO i.e. with
+		 * a valid parent fence assigned, but the paired job dependency won't have
+		 * one until both jobs have been submitted. Access the parent fence directly
+		 * here instead, submitting it last as partial render fence.
+		 */
 		jfence = to_pvr_queue_job_fence(job->paired_job->done_fence);
 		if (!WARN_ON(!jfence)) {
 			pvr_fw_object_get_fw_addr(jfence->queue->timeline_ufo.fw_obj,
@@ -675,11 +726,13 @@ static void pvr_queue_submit_job_to_cccb(struct pvr_job *job)
 		cmd->partial_render_geom_frag_fence.value = job->done_fence->seqno - 1;
 	}
 
+	trace_pvr_job_submit_fw(job);
+
 	/* Submit job to FW */
 	pvr_cccb_write_command_with_header(cccb, job->fw_ccb_cmd_type, job->cmd_len, job->cmd,
 					   job->id, job->id);
 
-	/* Signal the job fence. */
+	/* Update command to signal the job fence. */
 	pvr_fw_object_get_fw_addr(queue->timeline_ufo.fw_obj, &ufos[0].addr);
 	ufos[0].value = job->done_fence->seqno;
 	pvr_cccb_write_command_with_header(cccb, ROGUE_FWIF_CCB_CMD_TYPE_UPDATE,
@@ -709,10 +762,8 @@ static struct dma_fence *pvr_queue_run_job(struct drm_sched_job *sched_job)
 	}
 
 	/* The only kind of jobs that can be paired are geometry and fragment, and
-	 * we bail out early if we see a fragment job that's paired with a geomtry
-	 * job.
-	 * Paired jobs must also target the same context and point to the same
-	 * HWRT.
+	 * we bail out early if we see a fragment job that's paired with a geometry job.
+	 * Paired jobs must also target the same context and point to the same HWRT.
 	 */
 	if (WARN_ON(job->paired_job &&
 		    (job->type != DRM_PVR_JOB_TYPE_GEOMETRY ||
@@ -803,7 +854,9 @@ static void pvr_queue_start(struct pvr_queue *queue)
  * the scheduler, and re-assign parent fences in the middle.
  *
  * Return:
- *  * DRM_GPU_SCHED_STAT_RESET.
+ *  *%DRM_GPU_SCHED_STAT_NO_HANG if the job fence has already been
+ *   signaled, or
+ *  *%DRM_GPU_SCHED_STAT_RESET otherwise.
  */
 static enum drm_gpu_sched_stat
 pvr_queue_timedout_job(struct drm_sched_job *s_job)
@@ -814,6 +867,9 @@ pvr_queue_timedout_job(struct drm_sched_job *s_job)
 	struct pvr_job *job;
 	u32 job_count = 0;
 
+	if (dma_fence_is_signaled(s_job->s_fence->parent))
+		return DRM_GPU_SCHED_STAT_NO_HANG;
+
 	dev_err(sched->dev, "Job timeout\n");
 
 	/* Before we stop the scheduler, make sure the queue is out of any list, so
@@ -882,16 +938,16 @@ static const struct drm_sched_backend_ops pvr_queue_sched_ops = {
 };
 
 /**
- * pvr_queue_fence_is_ufo_backed() - Check if a dma_fence is backed by a UFO object
+ * pvr_queue_fence_is_native() - Check if a dma_fence is native to this driver.
  * @f: Fence to test.
  *
- * A UFO-backed fence is a fence that can be signaled or waited upon FW-side.
- * pvr_job::done_fence objects are backed by the timeline UFO attached to the queue
- * they are pushed to, but those fences are not directly exposed to the outside
- * world, so we also need to check if the fence we're being passed is a
- * drm_sched_fence that was coming from our driver.
+ * Check if the fence we're being passed is a drm_sched_fence that is coming from this driver.
+ *
+ * It may be a UFO-backed fence i.e. a fence that can be signaled or waited upon FW-side,
+ * such as pvr_job::done_fence objects that are backed by the timeline UFO attached to the queue
+ * they are pushed to.
  */
-bool pvr_queue_fence_is_ufo_backed(struct dma_fence *f)
+bool pvr_queue_fence_is_native(struct dma_fence *f)
 {
 	struct drm_sched_fence *sched_fence = f ? to_drm_sched_fence(f) : NULL;
 
@@ -899,10 +955,7 @@ bool pvr_queue_fence_is_ufo_backed(struct dma_fence *f)
 	    sched_fence->sched->ops == &pvr_queue_sched_ops)
 		return true;
 
-	if (f && f->ops == &pvr_queue_job_fence_ops)
-		return true;
-
-	return false;
+	return pvr_queue_fence_is_ufo_backed(f);
 }
 
 /**
@@ -934,9 +987,8 @@ pvr_queue_signal_done_fences(struct pvr_queue *queue)
 }
 
 /**
- * pvr_queue_check_job_waiting_for_cccb_space() - Check if the job waiting for CCCB space
- * can be unblocked
- * pushed to the CCCB
+ * pvr_queue_check_job_waiting_for_cccb_space() - Check if a job waiting for CCCB space
+ * can be unblocked and pushed to the CCCB.
  * @queue: Queue to check
  *
  * If we have a job waiting for CCCB, and this job now fits in the CCCB, we signal
@@ -1149,6 +1201,8 @@ void pvr_queue_job_cleanup(struct pvr_job *job)
 
 	if (job->base.s_fence)
 		drm_sched_job_cleanup(&job->base);
+
+	trace_pvr_job_done(job);
 }
 
 /**
@@ -1228,7 +1282,6 @@ struct pvr_queue *pvr_queue_create(struct pvr_context *ctx,
 	const struct drm_sched_init_args sched_args = {
 		.ops = &pvr_queue_sched_ops,
 		.submit_wq = pvr_dev->sched_wq,
-		.num_rqs = 1,
 		.credit_limit = 64 * 1024,
 		.hang_limit = 1,
 		.timeout = msecs_to_jiffies(500),
diff --git a/drivers/gpu/drm/imagination/pvr_queue.h b/drivers/gpu/drm/imagination/pvr_queue.h
index fc1986d73fc88..4aa72665ce256 100644
--- a/drivers/gpu/drm/imagination/pvr_queue.h
+++ b/drivers/gpu/drm/imagination/pvr_queue.h
@@ -141,7 +141,7 @@ struct pvr_queue {
 	u64 callstack_addr;
 };
 
-bool pvr_queue_fence_is_ufo_backed(struct dma_fence *f);
+bool pvr_queue_fence_is_native(struct dma_fence *f);
 
 int pvr_queue_job_init(struct pvr_job *job, u64 drm_client_id);
 
diff --git a/drivers/gpu/drm/imagination/pvr_rogue_fwif_shared.h b/drivers/gpu/drm/imagination/pvr_rogue_fwif_shared.h
index 869d904e3649f..fe54c1cad7a9c 100644
--- a/drivers/gpu/drm/imagination/pvr_rogue_fwif_shared.h
+++ b/drivers/gpu/drm/imagination/pvr_rogue_fwif_shared.h
@@ -14,15 +14,7 @@
 
 #define ROGUE_NUM_GEOM_CORES_SIZE 2U
 
-/*
- * Maximum number of UFOs in a CCB command.
- * The number is based on having 32 sync prims (as originally), plus 32 sync
- * checkpoints.
- * Once the use of sync prims is no longer supported, we will retain
- * the same total (64) as the number of sync checkpoints which may be
- * supporting a fence is not visible to the client driver and has to
- * allow for the number of different timelines involved in fence merges.
- */
+/* Maximum number of UFOs in a CCB command. */
 #define ROGUE_FWIF_CCB_CMD_MAX_UFOS (32U + 32U)
 
 /*
diff --git a/drivers/gpu/drm/imagination/pvr_sync.c b/drivers/gpu/drm/imagination/pvr_sync.c
index 3582616ff7220..936f840a52218 100644
--- a/drivers/gpu/drm/imagination/pvr_sync.c
+++ b/drivers/gpu/drm/imagination/pvr_sync.c
@@ -160,7 +160,7 @@ int
 pvr_sync_signal_array_update_fences(struct xarray *array,
 				    u32 sync_op_count,
 				    const struct drm_pvr_sync_op *sync_ops,
-				    struct dma_fence *done_fence)
+				    struct dma_fence *finished_fence)
 {
 	for (u32 i = 0; i < sync_op_count; i++) {
 		struct dma_fence *old_fence;
@@ -175,7 +175,7 @@ pvr_sync_signal_array_update_fences(struct xarray *array,
 			return -EINVAL;
 
 		old_fence = sig_sync->fence;
-		sig_sync->fence = dma_fence_get(done_fence);
+		sig_sync->fence = dma_fence_get(finished_fence);
 		dma_fence_put(old_fence);
 
 		if (WARN_ON(!sig_sync->fence))
@@ -211,7 +211,7 @@ pvr_sync_add_dep_to_job(struct drm_sched_job *job, struct dma_fence *f)
 	int err = 0;
 
 	dma_fence_unwrap_for_each(uf, &iter, f) {
-		if (pvr_queue_fence_is_ufo_backed(uf))
+		if (pvr_queue_fence_is_native(uf))
 			native_fence_count++;
 	}
 
@@ -227,7 +227,7 @@ pvr_sync_add_dep_to_job(struct drm_sched_job *job, struct dma_fence *f)
 		if (err)
 			continue;
 
-		if (pvr_queue_fence_is_ufo_backed(uf)) {
+		if (pvr_queue_fence_is_native(uf)) {
 			struct drm_sched_fence *s_fence = to_drm_sched_fence(uf);
 
 			/* If this is a native dependency, we wait for the scheduled fence,
diff --git a/drivers/gpu/drm/imagination/pvr_sync.h b/drivers/gpu/drm/imagination/pvr_sync.h
index db6ccfda104a3..48501ad277942 100644
--- a/drivers/gpu/drm/imagination/pvr_sync.h
+++ b/drivers/gpu/drm/imagination/pvr_sync.h
@@ -70,7 +70,7 @@ int
 pvr_sync_signal_array_update_fences(struct xarray *array,
 				    u32 sync_op_count,
 				    const struct drm_pvr_sync_op *sync_ops,
-				    struct dma_fence *done_fence);
+				    struct dma_fence *finished_fence);
 
 void
 pvr_sync_signal_array_push_fences(struct xarray *array);
diff --git a/drivers/gpu/drm/imagination/pvr_trace.h b/drivers/gpu/drm/imagination/pvr_trace.h
new file mode 100644
index 0000000000000..ffbbde3b2bcce
--- /dev/null
+++ b/drivers/gpu/drm/imagination/pvr_trace.h
@@ -0,0 +1,113 @@
+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
+/* Copyright (c) 2026 Imagination Technologies Ltd. */
+
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM pvr
+
+#if !defined(PVR_TRACE_H) || defined(TRACE_HEADER_MULTI_READ)
+#define PVR_TRACE_H
+
+#include "pvr_context.h"
+#include "pvr_device.h"
+#include "pvr_fw.h"
+#include "pvr_hwrt.h"
+#include "pvr_job.h"
+#include "pvr_sync.h"
+
+#include <linux/stringify.h>
+#include <linux/types.h>
+#include <linux/tracepoint.h>
+
+/*
+ * NOTE:
+ * When adding trace points, or extra data to existing ones - you must capture
+ * all the data in the TP_fast_assign section that you wish to use in the
+ * TP_printk section. This is because the printk is performed on demand from the
+ * captured data when you `cat /sys/kernel/tracing/trace` and by the time this
+ * happens any pointers you captured will likely no longer point to valid data.
+ */
+
+/* Job submit */
+
+TRACE_EVENT(pvr_job_submit_ioctl,
+	    TP_PROTO(struct pvr_device *pvr_dev, u32 count),
+	    TP_ARGS(pvr_dev, count),
+	    TP_STRUCT__entry(__field(struct pvr_device *, pvr_dev)
+			     __field(u32, count)),
+	    TP_fast_assign(__entry->pvr_dev = pvr_dev;
+			   __entry->count = count;),
+	    TP_printk("pvr_dev=%p count=%u",
+		      __entry->pvr_dev,
+		      __entry->count)
+);
+
+#define PVR_JOB_TYPE_TO_STR(val)                                         \
+	__print_symbolic(val,                                            \
+			 { DRM_PVR_JOB_TYPE_GEOMETRY, "geometry" },      \
+			 { DRM_PVR_JOB_TYPE_FRAGMENT, "fragment" },      \
+			 { DRM_PVR_JOB_TYPE_COMPUTE, "compute" },        \
+			 { DRM_PVR_JOB_TYPE_TRANSFER_FRAG, "transfer" })
+
+TRACE_EVENT(pvr_job_create,
+	    TP_PROTO(struct pvr_device *pvr_dev, struct pvr_job *job,
+		     u32 sync_op_count),
+	    TP_ARGS(pvr_dev, job, sync_op_count),
+	    TP_STRUCT__entry(__field(struct pvr_device *, pvr_dev)
+			     __field(struct pvr_context *, ctx)
+			     __field(struct pvr_fw_object *, fw_obj)
+			     __field(u32, fw_addr)
+			     __field(u32, hwrt_addr)
+			     __field(struct pvr_job *, job)
+			     __field(enum drm_pvr_job_type, job_type)
+			     __field(u32, sync_op_count)),
+	    TP_fast_assign(__entry->pvr_dev = pvr_dev;
+			   __entry->ctx = job->ctx;
+			   __entry->fw_obj = job->ctx->fw_obj;
+			   pvr_fw_object_get_fw_addr(job->ctx->fw_obj, &__entry->fw_addr);
+			   __entry->hwrt_addr = job->hwrt ?
+						job->hwrt->fw_obj->fw_addr_offset :
+						0;
+			   __entry->job = job;
+			   __entry->job_type = job->type;
+			   __entry->sync_op_count = sync_op_count;),
+	    TP_printk("pvr_dev=%p ctx=%p fw_obj=%p fw_addr=0x%x hwrt_addr=0x%x job=%p job_type=%s sync_op_count=%u",
+		      __entry->pvr_dev,
+		      __entry->ctx,
+		      __entry->fw_obj,
+		      __entry->fw_addr,
+		      __entry->hwrt_addr,
+		      __entry->job,
+		      PVR_JOB_TYPE_TO_STR(__entry->job_type),
+		      __entry->sync_op_count)
+);
+
+#undef PVR_JOB_TYPE_TO_STR
+
+TRACE_EVENT(pvr_job_submit_fw,
+	    TP_PROTO(struct pvr_job *job),
+	    TP_ARGS(job),
+	    TP_STRUCT__entry(__field(struct pvr_job *, job)
+			     __field(u32, done_seqno)),
+	    TP_fast_assign(__entry->job = job;
+			   __entry->done_seqno = job->done_fence->seqno;),
+	    TP_printk("job=%p done_seqno=%u",
+		      __entry->job,
+		      __entry->done_seqno)
+);
+
+TRACE_EVENT(pvr_job_done,
+	    TP_PROTO(struct pvr_job *job),
+	    TP_ARGS(job),
+	    TP_STRUCT__entry(__field(struct pvr_job *, job)),
+	    TP_fast_assign(__entry->job = job;),
+	    TP_printk("job=%p", __entry->job)
+);
+
+#endif /* PVR_TRACE_H */
+
+/* This part must be outside protection */
+#undef TRACE_INCLUDE_PATH
+#undef TRACE_INCLUDE_FILE
+#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/imagination
+#define TRACE_INCLUDE_FILE pvr_trace
+#include <trace/define_trace.h>
diff --git a/drivers/gpu/drm/imagination/pvr_trace_points.c b/drivers/gpu/drm/imagination/pvr_trace_points.c
new file mode 100644
index 0000000000000..71b53f267af02
--- /dev/null
+++ b/drivers/gpu/drm/imagination/pvr_trace_points.c
@@ -0,0 +1,7 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+/* Copyright (c) 2026 Imagination Technologies Ltd. */
+
+#ifndef __CHECKER__
+#define CREATE_TRACE_POINTS
+#include "pvr_trace.h"
+#endif
diff --git a/drivers/gpu/drm/imx/dc/dc-crtc.c b/drivers/gpu/drm/imx/dc/dc-crtc.c
index 608c610662dce..764510ff90a4a 100644
--- a/drivers/gpu/drm/imx/dc/dc-crtc.c
+++ b/drivers/gpu/drm/imx/dc/dc-crtc.c
@@ -148,7 +148,7 @@ dc_crtc_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode *mode)
 }
 
 static int
-dc_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
+dc_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_crtc_state =
 				drm_atomic_get_new_crtc_state(state, crtc);
@@ -164,7 +164,7 @@ dc_crtc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
 }
 
 static void
-dc_crtc_atomic_begin(struct drm_crtc *crtc, struct drm_atomic_state *state)
+dc_crtc_atomic_begin(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_crtc_state =
 				drm_atomic_get_new_crtc_state(state, crtc);
@@ -188,7 +188,7 @@ dc_crtc_atomic_begin(struct drm_crtc *crtc, struct drm_atomic_state *state)
 }
 
 static void
-dc_crtc_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *state)
+dc_crtc_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state =
 				drm_atomic_get_old_crtc_state(state, crtc);
@@ -222,7 +222,7 @@ dc_crtc_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *state)
 }
 
 static void
-dc_crtc_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state)
+dc_crtc_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_crtc_state =
 				drm_atomic_get_new_crtc_state(state, crtc);
@@ -294,7 +294,7 @@ dc_crtc_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state)
 }
 
 static void
-dc_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state)
+dc_crtc_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_crtc_state =
 				drm_atomic_get_new_crtc_state(state, crtc);
diff --git a/drivers/gpu/drm/imx/dc/dc-kms.c b/drivers/gpu/drm/imx/dc/dc-kms.c
index 2b18aa37a4a8b..0f8cfaf4c4d15 100644
--- a/drivers/gpu/drm/imx/dc/dc-kms.c
+++ b/drivers/gpu/drm/imx/dc/dc-kms.c
@@ -81,13 +81,7 @@ static int dc_kms_init_encoder_per_crtc(struct dc_drm_device *dc_drm,
 		return ret;
 	}
 
-	ret = drm_connector_attach_encoder(connector, encoder);
-	if (ret)
-		dev_err(dev,
-			"failed to attach encoder to connector for CRTC%u: %d\n",
-			crtc->index, ret);
-
-	return ret;
+	return 0;
 }
 
 int dc_kms_init(struct dc_drm_device *dc_drm)
diff --git a/drivers/gpu/drm/imx/dc/dc-plane.c b/drivers/gpu/drm/imx/dc/dc-plane.c
index e40d5d66c5c1f..dea7404fe6590 100644
--- a/drivers/gpu/drm/imx/dc/dc-plane.c
+++ b/drivers/gpu/drm/imx/dc/dc-plane.c
@@ -89,7 +89,7 @@ static int dc_plane_check_fb(struct drm_plane_state *state)
 }
 
 static int
-dc_plane_atomic_check(struct drm_plane *plane, struct drm_atomic_state *state)
+dc_plane_atomic_check(struct drm_plane *plane, struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state =
 				drm_atomic_get_new_plane_state(state, plane);
@@ -127,7 +127,7 @@ dc_plane_atomic_check(struct drm_plane *plane, struct drm_atomic_state *state)
 }
 
 static void
-dc_plane_atomic_update(struct drm_plane *plane, struct drm_atomic_state *state)
+dc_plane_atomic_update(struct drm_plane *plane, struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state =
 				drm_atomic_get_new_plane_state(state, plane);
@@ -176,7 +176,7 @@ dc_plane_atomic_update(struct drm_plane *plane, struct drm_atomic_state *state)
 }
 
 static void dc_plane_atomic_disable(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct dc_plane *dplane = to_dc_plane(plane);
 	const struct dc_fu_ops *fu_ops;
diff --git a/drivers/gpu/drm/imx/dcss/dcss-crtc.c b/drivers/gpu/drm/imx/dcss/dcss-crtc.c
index af91e45b5d13b..ab41759a9f52d 100644
--- a/drivers/gpu/drm/imx/dcss/dcss-crtc.c
+++ b/drivers/gpu/drm/imx/dcss/dcss-crtc.c
@@ -53,13 +53,13 @@ static const struct drm_crtc_funcs dcss_crtc_funcs = {
 };
 
 static void dcss_crtc_atomic_begin(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_on(crtc);
 }
 
 static void dcss_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct dcss_crtc *dcss_crtc = container_of(crtc, struct dcss_crtc,
 						   base);
@@ -78,7 +78,7 @@ static void dcss_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static void dcss_crtc_atomic_enable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
 									      crtc);
@@ -114,7 +114,7 @@ static void dcss_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void dcss_crtc_atomic_disable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
 									      crtc);
diff --git a/drivers/gpu/drm/imx/dcss/dcss-kms.c b/drivers/gpu/drm/imx/dcss/dcss-kms.c
index 3633e8f3aff60..50bd7f36d36dd 100644
--- a/drivers/gpu/drm/imx/dcss/dcss-kms.c
+++ b/drivers/gpu/drm/imx/dcss/dcss-kms.c
@@ -103,8 +103,6 @@ static int dcss_kms_bridge_connector_init(struct dcss_kms_dev *kms)
 		return PTR_ERR(kms->connector);
 	}
 
-	drm_connector_attach_encoder(kms->connector, encoder);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/imx/dcss/dcss-plane.c b/drivers/gpu/drm/imx/dcss/dcss-plane.c
index c014ef70dfd16..1746dc5d4b642 100644
--- a/drivers/gpu/drm/imx/dcss/dcss-plane.c
+++ b/drivers/gpu/drm/imx/dcss/dcss-plane.c
@@ -140,7 +140,7 @@ static bool dcss_plane_is_source_size_allowed(u16 src_w, u16 src_h, u32 pix_fmt)
 }
 
 static int dcss_plane_atomic_check(struct drm_plane *plane,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -267,7 +267,7 @@ static bool dcss_plane_needs_setup(struct drm_plane_state *state,
 }
 
 static void dcss_plane_atomic_update(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -352,7 +352,7 @@ static void dcss_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void dcss_plane_atomic_disable(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct dcss_plane *dcss_plane = to_dcss_plane(plane);
 	struct dcss_dev *dcss = plane->dev->dev_private;
diff --git a/drivers/gpu/drm/imx/ipuv3/Kconfig b/drivers/gpu/drm/imx/ipuv3/Kconfig
index b2240998df4f1..3ca237515e463 100644
--- a/drivers/gpu/drm/imx/ipuv3/Kconfig
+++ b/drivers/gpu/drm/imx/ipuv3/Kconfig
@@ -13,10 +13,11 @@ config DRM_IMX
 config DRM_IMX_PARALLEL_DISPLAY
 	tristate "Support for parallel displays"
 	depends on DRM_IMX
+	depends on OF
 	select DRM_BRIDGE
 	select DRM_BRIDGE_CONNECTOR
 	select DRM_DISPLAY_HELPER
-	select DRM_IMX_LEGACY_BRIDGE
+	select DRM_OF_DISPLAY_MODE_BRIDGE
 	select DRM_PANEL_BRIDGE
 	select VIDEOMODE_HELPERS
 
@@ -33,11 +34,12 @@ config DRM_IMX_LDB
 	tristate "Support for LVDS displays"
 	depends on DRM_IMX
 	depends on COMMON_CLK
+	depends on OF
 	select MFD_SYSCON
 	select DRM_BRIDGE
 	select DRM_BRIDGE_CONNECTOR
 	select DRM_PANEL_BRIDGE
-	select DRM_IMX_LEGACY_BRIDGE
+	select DRM_OF_DISPLAY_MODE_BRIDGE
 	help
 	  Choose this to enable the internal LVDS Display Bridge (LDB)
 	  found on i.MX53 and i.MX6 processors.
diff --git a/drivers/gpu/drm/imx/ipuv3/imx-drm-core.c b/drivers/gpu/drm/imx/ipuv3/imx-drm-core.c
index eddb471119c66..51a0ec0432484 100644
--- a/drivers/gpu/drm/imx/ipuv3/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/ipuv3/imx-drm-core.c
@@ -38,7 +38,7 @@ module_param(legacyfb_depth, int, 0444);
 DEFINE_DRM_GEM_DMA_FOPS(imx_drm_driver_fops);
 
 static int imx_drm_atomic_check(struct drm_device *dev,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	int ret;
 
@@ -68,7 +68,7 @@ static const struct drm_mode_config_funcs imx_drm_mode_config_funcs = {
 	.atomic_commit = drm_atomic_helper_commit,
 };
 
-static void imx_drm_atomic_commit_tail(struct drm_atomic_state *state)
+static void imx_drm_atomic_commit_tail(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct drm_plane *plane;
diff --git a/drivers/gpu/drm/imx/ipuv3/imx-drm.h b/drivers/gpu/drm/imx/ipuv3/imx-drm.h
index 0c85bf83ffbff..2c0f4bd2569b1 100644
--- a/drivers/gpu/drm/imx/ipuv3/imx-drm.h
+++ b/drivers/gpu/drm/imx/ipuv3/imx-drm.h
@@ -26,6 +26,6 @@ int imx_drm_encoder_parse_of(struct drm_device *drm,
 	struct drm_encoder *encoder, struct device_node *np);
 
 int ipu_planes_assign_pre(struct drm_device *dev,
-			  struct drm_atomic_state *state);
+			  struct drm_atomic_commit *state);
 
 #endif /* _IMX_DRM_H_ */
diff --git a/drivers/gpu/drm/imx/ipuv3/imx-ldb.c b/drivers/gpu/drm/imx/ipuv3/imx-ldb.c
index 626d410d91507..63d6a1e7c1bda 100644
--- a/drivers/gpu/drm/imx/ipuv3/imx-ldb.c
+++ b/drivers/gpu/drm/imx/ipuv3/imx-ldb.c
@@ -28,7 +28,7 @@
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
-#include <drm/bridge/imx.h>
+#include <drm/bridge/of-display-mode-bridge.h>
 
 #include "imx-drm.h"
 
@@ -407,8 +407,6 @@ static int imx_ldb_register(struct drm_device *drm,
 	if (IS_ERR(connector))
 		return PTR_ERR(connector);
 
-	drm_connector_attach_encoder(connector, encoder);
-
 	return 0;
 }
 
@@ -605,8 +603,8 @@ static int imx_ldb_probe(struct platform_device *pdev)
 		 * checking the bus_format property.
 		 */
 		if (!channel->bridge) {
-			channel->bridge = devm_imx_drm_legacy_bridge(dev, child,
-								     DRM_MODE_CONNECTOR_LVDS);
+			channel->bridge = devm_drm_of_display_mode_bridge(dev, child,
+									  DRM_MODE_CONNECTOR_LVDS);
 			if (IS_ERR(channel->bridge)) {
 				ret = PTR_ERR(channel->bridge);
 				goto free_child;
diff --git a/drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c
index a18b9d1a68b6c..1405e50ee56b5 100644
--- a/drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3/ipuv3-crtc.c
@@ -47,7 +47,7 @@ static inline struct ipu_crtc *to_ipu_crtc(struct drm_crtc *crtc)
 }
 
 static void ipu_crtc_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
 	struct ipu_soc *ipu = dev_get_drvdata(ipu_crtc->dev->parent);
@@ -79,7 +79,7 @@ static void ipu_crtc_disable_planes(struct ipu_crtc *ipu_crtc,
 }
 
 static void ipu_crtc_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
 									      crtc);
@@ -226,7 +226,7 @@ static bool ipu_crtc_mode_fixup(struct drm_crtc *crtc,
 }
 
 static int ipu_crtc_atomic_check(struct drm_crtc *crtc,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -239,13 +239,13 @@ static int ipu_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void ipu_crtc_atomic_begin(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_on(crtc);
 }
 
 static void ipu_crtc_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	spin_lock_irq(&crtc->dev->event_lock);
 	if (crtc->state->event) {
diff --git a/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
index c1c7be4e26c64..c7ec09e557c19 100644
--- a/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3/ipuv3-plane.c
@@ -365,7 +365,7 @@ static const struct drm_plane_funcs ipu_plane_funcs = {
 };
 
 static int ipu_plane_atomic_check(struct drm_plane *plane,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -527,7 +527,7 @@ static int ipu_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void ipu_plane_atomic_disable(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct ipu_plane *ipu_plane = to_ipu_plane(plane);
 
@@ -572,7 +572,7 @@ static void ipu_calculate_bursts(u32 width, u32 cpp, u32 stride,
 }
 
 static void ipu_plane_atomic_update(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -804,7 +804,7 @@ bool ipu_plane_atomic_update_pending(struct drm_plane *plane)
 	return false;
 }
 int ipu_planes_assign_pre(struct drm_device *dev,
-			  struct drm_atomic_state *state)
+			  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state, *crtc_state;
 	struct drm_plane_state *plane_state;
diff --git a/drivers/gpu/drm/imx/ipuv3/parallel-display.c b/drivers/gpu/drm/imx/ipuv3/parallel-display.c
index a356f0b764cb7..0f06db95f00f1 100644
--- a/drivers/gpu/drm/imx/ipuv3/parallel-display.c
+++ b/drivers/gpu/drm/imx/ipuv3/parallel-display.c
@@ -19,7 +19,7 @@
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
-#include <drm/bridge/imx.h>
+#include <drm/bridge/of-display-mode-bridge.h>
 
 #include "imx-drm.h"
 
@@ -216,8 +216,6 @@ static int imx_pd_bind(struct device *dev, struct device *master, void *data)
 	if (IS_ERR(connector))
 		return PTR_ERR(connector);
 
-	drm_connector_attach_encoder(connector, encoder);
-
 	return 0;
 }
 
@@ -242,7 +240,8 @@ static int imx_pd_probe(struct platform_device *pdev)
 	/* port@1 is the output port */
 	imxpd->next_bridge = devm_drm_of_get_bridge(dev, np, 1, 0);
 	if (imxpd->next_bridge == ERR_PTR(-ENODEV))
-		imxpd->next_bridge = devm_imx_drm_legacy_bridge(dev, np, DRM_MODE_CONNECTOR_DPI);
+		imxpd->next_bridge = devm_drm_of_display_mode_bridge(dev, np,
+								     DRM_MODE_CONNECTOR_DPI);
 	if (IS_ERR(imxpd->next_bridge)) {
 		ret = PTR_ERR(imxpd->next_bridge);
 		return ret;
diff --git a/drivers/gpu/drm/imx/lcdc/imx-lcdc.c b/drivers/gpu/drm/imx/lcdc/imx-lcdc.c
index e200b40f30fe4..f52832b43acad 100644
--- a/drivers/gpu/drm/imx/lcdc/imx-lcdc.c
+++ b/drivers/gpu/drm/imx/lcdc/imx-lcdc.c
@@ -448,8 +448,6 @@ static int imx_lcdc_probe(struct platform_device *pdev)
 	if (IS_ERR(lcdc->connector))
 		return dev_err_probe(drm->dev, PTR_ERR(lcdc->connector), "Cannot init bridge connector\n");
 
-	drm_connector_attach_encoder(lcdc->connector, &lcdc->pipe.encoder);
-
 	/*
 	 * The LCDC controller does not have an enable bit. The
 	 * controller starts directly when the clocks are enabled.
diff --git a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
index 4068114adf8c2..42c86f195c66b 100644
--- a/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
+++ b/drivers/gpu/drm/ingenic/ingenic-drm-drv.c
@@ -140,7 +140,7 @@ to_ingenic_drm_priv_state(struct drm_private_state *state)
 }
 
 static struct ingenic_drm_private_state *
-ingenic_drm_get_priv_state(struct ingenic_drm *priv, struct drm_atomic_state *state)
+ingenic_drm_get_priv_state(struct ingenic_drm *priv, struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv_state;
 
@@ -152,7 +152,7 @@ ingenic_drm_get_priv_state(struct ingenic_drm *priv, struct drm_atomic_state *st
 }
 
 static struct ingenic_drm_private_state *
-ingenic_drm_get_new_priv_state(struct ingenic_drm *priv, struct drm_atomic_state *state)
+ingenic_drm_get_new_priv_state(struct ingenic_drm *priv, struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv_state;
 
@@ -229,7 +229,7 @@ static int ingenic_drm_update_pixclk(struct notifier_block *nb,
 }
 
 static void ingenic_drm_bridge_atomic_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct ingenic_drm *priv = drm_device_get_priv(bridge->dev);
 
@@ -241,7 +241,7 @@ static void ingenic_drm_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void ingenic_drm_crtc_atomic_enable(struct drm_crtc *crtc,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct ingenic_drm *priv = drm_crtc_get_priv(crtc);
 	struct ingenic_drm_private_state *priv_state;
@@ -260,7 +260,7 @@ static void ingenic_drm_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void ingenic_drm_bridge_atomic_disable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct ingenic_drm *priv = drm_device_get_priv(bridge->dev);
 	unsigned int var;
@@ -274,7 +274,7 @@ static void ingenic_drm_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void ingenic_drm_crtc_atomic_disable(struct drm_crtc *crtc,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_off(crtc);
 }
@@ -334,7 +334,7 @@ static void ingenic_drm_crtc_update_timings(struct ingenic_drm *priv,
 }
 
 static int ingenic_drm_crtc_atomic_check(struct drm_crtc *crtc,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -404,7 +404,7 @@ ingenic_drm_crtc_mode_valid(struct drm_crtc *crtc, const struct drm_display_mode
 }
 
 static void ingenic_drm_crtc_atomic_begin(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -426,7 +426,7 @@ static void ingenic_drm_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void ingenic_drm_crtc_atomic_flush(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct ingenic_drm *priv = drm_crtc_get_priv(crtc);
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
@@ -459,7 +459,7 @@ static void ingenic_drm_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static int ingenic_drm_plane_atomic_check(struct drm_plane *plane,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state,
 										 plane);
@@ -556,7 +556,7 @@ void ingenic_drm_plane_disable(struct device *dev, struct drm_plane *plane)
 }
 
 static void ingenic_drm_plane_atomic_disable(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct ingenic_drm *priv = drm_device_get_priv(plane->dev);
 
@@ -660,7 +660,7 @@ static void ingenic_drm_update_palette(struct ingenic_drm *priv,
 }
 
 static void ingenic_drm_plane_atomic_update(struct drm_plane *plane,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct ingenic_drm *priv = drm_device_get_priv(plane->dev);
 	struct drm_plane_state *newstate = drm_atomic_get_new_plane_state(state, plane);
@@ -1330,8 +1330,6 @@ static int ingenic_drm_bind(struct device *dev, bool has_components)
 			ret = PTR_ERR(connector);
 			goto err_drvdata;
 		}
-
-		drm_connector_attach_encoder(connector, encoder);
 	}
 
 	drm_for_each_encoder(encoder, drm) {
diff --git a/drivers/gpu/drm/ingenic/ingenic-ipu.c b/drivers/gpu/drm/ingenic/ingenic-ipu.c
index 34545b9c8c338..56143a191f361 100644
--- a/drivers/gpu/drm/ingenic/ingenic-ipu.c
+++ b/drivers/gpu/drm/ingenic/ingenic-ipu.c
@@ -86,7 +86,7 @@ to_ingenic_ipu_priv_state(struct drm_private_state *state)
 }
 
 static struct ingenic_ipu_private_state *
-ingenic_ipu_get_priv_state(struct ingenic_ipu *priv, struct drm_atomic_state *state)
+ingenic_ipu_get_priv_state(struct ingenic_ipu *priv, struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv_state;
 
@@ -98,7 +98,7 @@ ingenic_ipu_get_priv_state(struct ingenic_ipu *priv, struct drm_atomic_state *st
 }
 
 static struct ingenic_ipu_private_state *
-ingenic_ipu_get_new_priv_state(struct ingenic_ipu *priv, struct drm_atomic_state *state)
+ingenic_ipu_get_new_priv_state(struct ingenic_ipu *priv, struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv_state;
 
@@ -321,7 +321,7 @@ static inline bool osd_changed(struct drm_plane_state *state,
 }
 
 static void ingenic_ipu_plane_atomic_update(struct drm_plane *plane,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct ingenic_ipu *ipu = plane_to_ingenic_ipu(plane);
 	struct drm_plane_state *newstate = drm_atomic_get_new_plane_state(state, plane);
@@ -565,7 +565,7 @@ static void ingenic_ipu_plane_atomic_update(struct drm_plane *plane,
 }
 
 static int ingenic_ipu_plane_atomic_check(struct drm_plane *plane,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state,
 										 plane);
@@ -653,7 +653,7 @@ static int ingenic_ipu_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void ingenic_ipu_plane_atomic_disable(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct ingenic_ipu *ipu = plane_to_ingenic_ipu(plane);
 
diff --git a/drivers/gpu/drm/kmb/kmb_crtc.c b/drivers/gpu/drm/kmb/kmb_crtc.c
index 647872f65bff9..1dcdc55b00696 100644
--- a/drivers/gpu/drm/kmb/kmb_crtc.c
+++ b/drivers/gpu/drm/kmb/kmb_crtc.c
@@ -66,7 +66,7 @@ static const struct drm_crtc_funcs kmb_crtc_funcs = {
 };
 
 static void kmb_crtc_set_mode(struct drm_crtc *crtc,
-			      struct drm_atomic_state *old_state)
+			      struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_display_mode *m = &crtc->state->adjusted_mode;
@@ -133,7 +133,7 @@ static void kmb_crtc_set_mode(struct drm_crtc *crtc,
 }
 
 static void kmb_crtc_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct kmb_drm_private *kmb = crtc_to_kmb_priv(crtc);
 
@@ -143,7 +143,7 @@ static void kmb_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void kmb_crtc_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct kmb_drm_private *kmb = crtc_to_kmb_priv(crtc);
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state, crtc);
@@ -156,7 +156,7 @@ static void kmb_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void kmb_crtc_atomic_begin(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct kmb_drm_private *kmb = to_kmb(dev);
@@ -166,7 +166,7 @@ static void kmb_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void kmb_crtc_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct kmb_drm_private *kmb = to_kmb(dev);
diff --git a/drivers/gpu/drm/kmb/kmb_drv.h b/drivers/gpu/drm/kmb/kmb_drv.h
index 1f0c10d317feb..52ec4c2369825 100644
--- a/drivers/gpu/drm/kmb/kmb_drv.h
+++ b/drivers/gpu/drm/kmb/kmb_drv.h
@@ -52,7 +52,7 @@ struct kmb_drm_private {
 	struct kmb_clock		kmb_clk;
 	struct drm_crtc			crtc;
 	struct kmb_plane		*plane;
-	struct drm_atomic_state		*state;
+	struct drm_atomic_commit		*state;
 	spinlock_t			irq_lock;
 	int				irq_lcd;
 	int				sys_clk_mhz;
diff --git a/drivers/gpu/drm/kmb/kmb_dsi.c b/drivers/gpu/drm/kmb/kmb_dsi.c
index aeb2f9f98f231..59d0e856392fc 100644
--- a/drivers/gpu/drm/kmb/kmb_dsi.c
+++ b/drivers/gpu/drm/kmb/kmb_dsi.c
@@ -251,7 +251,7 @@ int kmb_dsi_host_bridge_init(struct device *dev)
 		return -EINVAL;
 	}
 	/* Locate drm bridge from the hdmi encoder DT node */
-	adv_bridge = of_drm_find_bridge(encoder_node);
+	adv_bridge = of_drm_find_and_get_bridge(encoder_node);
 	of_node_put(dsi_out);
 	of_node_put(encoder_node);
 	if (!adv_bridge) {
@@ -1333,7 +1333,7 @@ static u32 mipi_tx_init_dphy(struct kmb_dsi *kmb_dsi,
 }
 
 static void connect_lcd_to_mipi(struct kmb_dsi *kmb_dsi,
-				struct drm_atomic_state *old_state)
+				struct drm_atomic_commit *old_state)
 {
 	struct regmap *msscam;
 
@@ -1353,7 +1353,7 @@ static void connect_lcd_to_mipi(struct kmb_dsi *kmb_dsi,
 }
 
 int kmb_dsi_mode_set(struct kmb_dsi *kmb_dsi, struct drm_display_mode *mode,
-		     int sys_clk_mhz, struct drm_atomic_state *old_state)
+		     int sys_clk_mhz, struct drm_atomic_commit *old_state)
 {
 	u64 data_rate;
 
@@ -1457,7 +1457,7 @@ int kmb_dsi_encoder_init(struct drm_device *dev, struct kmb_dsi *kmb_dsi)
 		drm_encoder_cleanup(encoder);
 		return PTR_ERR(connector);
 	}
-	drm_connector_attach_encoder(connector, encoder);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/kmb/kmb_dsi.h b/drivers/gpu/drm/kmb/kmb_dsi.h
index 09dc88743d779..0faadeb9f1ddc 100644
--- a/drivers/gpu/drm/kmb/kmb_dsi.h
+++ b/drivers/gpu/drm/kmb/kmb_dsi.h
@@ -380,7 +380,7 @@ int kmb_dsi_host_bridge_init(struct device *dev);
 struct kmb_dsi *kmb_dsi_init(struct platform_device *pdev);
 void kmb_dsi_host_unregister(struct kmb_dsi *kmb_dsi);
 int kmb_dsi_mode_set(struct kmb_dsi *kmb_dsi, struct drm_display_mode *mode,
-		     int sys_clk_mhz, struct drm_atomic_state *old_state);
+		     int sys_clk_mhz, struct drm_atomic_commit *old_state);
 int kmb_dsi_map_mmio(struct kmb_dsi *kmb_dsi);
 int kmb_dsi_clk_init(struct kmb_dsi *kmb_dsi);
 int kmb_dsi_encoder_init(struct drm_device *dev, struct kmb_dsi *kmb_dsi);
diff --git a/drivers/gpu/drm/kmb/kmb_plane.c b/drivers/gpu/drm/kmb/kmb_plane.c
index a935ff1503cdf..a0001edb1e21a 100644
--- a/drivers/gpu/drm/kmb/kmb_plane.c
+++ b/drivers/gpu/drm/kmb/kmb_plane.c
@@ -90,7 +90,7 @@ static unsigned int check_pixel_format(struct drm_plane *plane, u32 format)
 }
 
 static int kmb_plane_atomic_check(struct drm_plane *plane,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -139,7 +139,7 @@ static int kmb_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void kmb_plane_atomic_disable(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct kmb_plane *kmb_plane = to_kmb_plane(plane);
 	int plane_id = kmb_plane->id;
@@ -339,7 +339,7 @@ static void kmb_plane_set_alpha(struct kmb_drm_private *kmb,
 }
 
 static void kmb_plane_atomic_update(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state,
 										 plane);
diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c
index 9a1e6b9ecbe5c..0a01213c4878f 100644
--- a/drivers/gpu/drm/lima/lima_sched.c
+++ b/drivers/gpu/drm/lima/lima_sched.c
@@ -521,7 +521,6 @@ int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name)
 			       lima_sched_timeout_ms : 10000;
 	const struct drm_sched_init_args args = {
 		.ops = &lima_sched_ops,
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit = 1,
 		.hang_limit = lima_job_hang_limit,
 		.timeout = msecs_to_jiffies(timeout),
diff --git a/drivers/gpu/drm/logicvc/logicvc_crtc.c b/drivers/gpu/drm/logicvc/logicvc_crtc.c
index 43a675d03808f..81e5b80736fa7 100644
--- a/drivers/gpu/drm/logicvc/logicvc_crtc.c
+++ b/drivers/gpu/drm/logicvc/logicvc_crtc.c
@@ -36,7 +36,7 @@ logicvc_crtc_mode_valid(struct drm_crtc *drm_crtc,
 }
 
 static void logicvc_crtc_atomic_begin(struct drm_crtc *drm_crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct logicvc_crtc *crtc = logicvc_crtc(drm_crtc);
 	struct drm_crtc_state *old_state =
@@ -60,7 +60,7 @@ static void logicvc_crtc_atomic_begin(struct drm_crtc *drm_crtc,
 }
 
 static void logicvc_crtc_atomic_enable(struct drm_crtc *drm_crtc,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct logicvc_crtc *crtc = logicvc_crtc(drm_crtc);
 	struct logicvc_drm *logicvc = logicvc_drm(drm_crtc->dev);
@@ -150,7 +150,7 @@ static void logicvc_crtc_atomic_enable(struct drm_crtc *drm_crtc,
 }
 
 static void logicvc_crtc_atomic_disable(struct drm_crtc *drm_crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct logicvc_drm *logicvc = logicvc_drm(drm_crtc->dev);
 	struct drm_device *drm_dev = drm_crtc->dev;
diff --git a/drivers/gpu/drm/logicvc/logicvc_layer.c b/drivers/gpu/drm/logicvc/logicvc_layer.c
index eab4d773f92b6..7f5ba000188ab 100644
--- a/drivers/gpu/drm/logicvc/logicvc_layer.c
+++ b/drivers/gpu/drm/logicvc/logicvc_layer.c
@@ -81,7 +81,7 @@ static bool logicvc_layer_format_inverted(uint32_t format)
 }
 
 static int logicvc_plane_atomic_check(struct drm_plane *drm_plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm_dev = drm_plane->dev;
 	struct logicvc_layer *layer = logicvc_layer(drm_plane);
@@ -135,7 +135,7 @@ static int logicvc_plane_atomic_check(struct drm_plane *drm_plane,
 }
 
 static void logicvc_plane_atomic_update(struct drm_plane *drm_plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct logicvc_layer *layer = logicvc_layer(drm_plane);
 	struct logicvc_drm *logicvc = logicvc_drm(drm_plane->dev);
@@ -232,7 +232,7 @@ static void logicvc_plane_atomic_update(struct drm_plane *drm_plane,
 }
 
 static void logicvc_plane_atomic_disable(struct drm_plane *drm_plane,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct logicvc_layer *layer = logicvc_layer(drm_plane);
 	struct logicvc_drm *logicvc = logicvc_drm(drm_plane->dev);
diff --git a/drivers/gpu/drm/loongson/lsdc_crtc.c b/drivers/gpu/drm/loongson/lsdc_crtc.c
index 587fbe285e9ef..16b8f36a40710 100644
--- a/drivers/gpu/drm/loongson/lsdc_crtc.c
+++ b/drivers/gpu/drm/loongson/lsdc_crtc.c
@@ -793,7 +793,7 @@ static int lsdc_pixpll_atomic_check(struct drm_crtc *crtc,
 }
 
 static int lsdc_crtc_helper_atomic_check(struct drm_crtc *crtc,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 
@@ -855,7 +855,7 @@ static void lsdc_crtc_send_vblank(struct drm_crtc *crtc)
 }
 
 static void lsdc_crtc_atomic_enable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct lsdc_crtc *lcrtc = to_lsdc_crtc(crtc);
 
@@ -866,7 +866,7 @@ static void lsdc_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void lsdc_crtc_atomic_disable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct lsdc_crtc *lcrtc = to_lsdc_crtc(crtc);
 
@@ -883,7 +883,7 @@ static void lsdc_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void lsdc_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	spin_lock_irq(&crtc->dev->event_lock);
 	if (crtc->state->event) {
diff --git a/drivers/gpu/drm/loongson/lsdc_output_7a1000.c b/drivers/gpu/drm/loongson/lsdc_output_7a1000.c
index ccca67e01fd95..5c483baa3762e 100644
--- a/drivers/gpu/drm/loongson/lsdc_output_7a1000.c
+++ b/drivers/gpu/drm/loongson/lsdc_output_7a1000.c
@@ -63,7 +63,7 @@ static int ls7a1000_dpi_connector_get_modes(struct drm_connector *conn)
 
 static struct drm_encoder *
 ls7a1000_dpi_connector_get_best_encoder(struct drm_connector *connector,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct lsdc_output *output = connector_to_lsdc_output(connector);
 
diff --git a/drivers/gpu/drm/loongson/lsdc_output_7a2000.c b/drivers/gpu/drm/loongson/lsdc_output_7a2000.c
index aa7daee4c0656..a9798b3dadee0 100644
--- a/drivers/gpu/drm/loongson/lsdc_output_7a2000.c
+++ b/drivers/gpu/drm/loongson/lsdc_output_7a2000.c
@@ -67,7 +67,7 @@ static int ls7a2000_connector_get_modes(struct drm_connector *connector)
 
 static struct drm_encoder *
 ls7a2000_connector_get_best_encoder(struct drm_connector *connector,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct lsdc_output *output = connector_to_lsdc_output(connector);
 
@@ -332,7 +332,7 @@ static int ls7a2000_hdmi_set_avi_infoframe(struct drm_encoder *encoder,
 }
 
 static void ls7a2000_hdmi_atomic_disable(struct drm_encoder *encoder,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct lsdc_output *output = encoder_to_lsdc_output(encoder);
 	struct lsdc_display_pipe *dispipe = output_to_display_pipe(output);
@@ -355,7 +355,7 @@ static void ls7a2000_hdmi_atomic_disable(struct drm_encoder *encoder,
 }
 
 static void ls7a2000_hdmi_atomic_enable(struct drm_encoder *encoder,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_device *ddev = encoder->dev;
 	struct lsdc_device *ldev = to_lsdc(ddev);
diff --git a/drivers/gpu/drm/loongson/lsdc_plane.c b/drivers/gpu/drm/loongson/lsdc_plane.c
index 9675344128d02..bea42215796d4 100644
--- a/drivers/gpu/drm/loongson/lsdc_plane.c
+++ b/drivers/gpu/drm/loongson/lsdc_plane.c
@@ -48,7 +48,7 @@ static u64 lsdc_fb_base_addr(struct drm_framebuffer *fb)
 }
 
 static int lsdc_primary_atomic_check(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_crtc *crtc = new_plane_state->crtc;
@@ -67,7 +67,7 @@ static int lsdc_primary_atomic_check(struct drm_plane *plane,
 }
 
 static void lsdc_primary_atomic_update(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct lsdc_primary *primary = to_lsdc_primary(plane);
 	const struct lsdc_primary_plane_ops *ops = primary->ops;
@@ -87,7 +87,7 @@ static void lsdc_primary_atomic_update(struct drm_plane *plane,
 }
 
 static void lsdc_primary_atomic_disable(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	/*
 	 * Do nothing, just prevent call into atomic_update().
@@ -172,7 +172,7 @@ static const struct drm_plane_helper_funcs lsdc_primary_helper_funcs = {
 };
 
 static int lsdc_cursor_plane_atomic_async_check(struct drm_plane *plane,
-						struct drm_atomic_state *state,
+						struct drm_atomic_commit *state,
 						bool flip)
 {
 	struct drm_plane_state *new_state;
@@ -219,7 +219,7 @@ static int lsdc_cursor_plane_atomic_async_check(struct drm_plane *plane,
 }
 
 static void lsdc_cursor_plane_atomic_async_update(struct drm_plane *plane,
-						  struct drm_atomic_state *state)
+						  struct drm_atomic_commit *state)
 {
 	struct lsdc_cursor *cursor = to_lsdc_cursor(plane);
 	const struct lsdc_cursor_plane_ops *ops = cursor->ops;
@@ -268,7 +268,7 @@ static void lsdc_cursor_plane_atomic_async_update(struct drm_plane *plane,
 /* ls7a1000 cursor plane helpers */
 
 static int ls7a1000_cursor_plane_atomic_check(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state;
 	struct drm_crtc_state *new_crtc_state;
@@ -298,7 +298,7 @@ static int ls7a1000_cursor_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void ls7a1000_cursor_plane_atomic_update(struct drm_plane *plane,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct lsdc_cursor *cursor = to_lsdc_cursor(plane);
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
@@ -320,7 +320,7 @@ static void ls7a1000_cursor_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void ls7a1000_cursor_plane_atomic_disable(struct drm_plane *plane,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct lsdc_cursor *cursor = to_lsdc_cursor(plane);
 	const struct lsdc_cursor_plane_ops *ops = cursor->ops;
@@ -341,7 +341,7 @@ static const struct drm_plane_helper_funcs ls7a1000_cursor_plane_helper_funcs =
 /* ls7a2000 cursor plane helpers */
 
 static int ls7a2000_cursor_plane_atomic_check(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state;
 	struct drm_crtc_state *new_crtc_state;
@@ -379,7 +379,7 @@ static int ls7a2000_cursor_plane_atomic_check(struct drm_plane *plane,
 /* Update the format, size and location of the cursor */
 
 static void ls7a2000_cursor_plane_atomic_update(struct drm_plane *plane,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct lsdc_cursor *cursor = to_lsdc_cursor(plane);
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
@@ -416,7 +416,7 @@ static void ls7a2000_cursor_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void ls7a2000_cursor_plane_atomic_disable(struct drm_plane *plane,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct lsdc_cursor *cursor = to_lsdc_cursor(plane);
 	const struct lsdc_cursor_plane_ops *hw_ops = cursor->ops;
diff --git a/drivers/gpu/drm/mediatek/mtk_crtc.c b/drivers/gpu/drm/mediatek/mtk_crtc.c
index fcb16f3f7b23b..caa02a708d0a5 100644
--- a/drivers/gpu/drm/mediatek/mtk_crtc.c
+++ b/drivers/gpu/drm/mediatek/mtk_crtc.c
@@ -681,7 +681,7 @@ static void mtk_crtc_disable_vblank(struct drm_crtc *crtc)
 }
 
 static void mtk_crtc_update_output(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	int crtc_index = drm_crtc_index(crtc);
 	int i;
@@ -763,7 +763,7 @@ void mtk_crtc_plane_disable(struct drm_crtc *crtc, struct drm_plane *plane)
 }
 
 void mtk_crtc_async_update(struct drm_crtc *crtc, struct drm_plane *plane,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 
@@ -774,7 +774,7 @@ void mtk_crtc_async_update(struct drm_crtc *crtc, struct drm_plane *plane,
 }
 
 static void mtk_crtc_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_ddp_comp *comp = mtk_crtc->ddp_comp[0];
@@ -802,7 +802,7 @@ static void mtk_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void mtk_crtc_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	struct mtk_ddp_comp *comp = mtk_crtc->ddp_comp[0];
@@ -843,7 +843,7 @@ static void mtk_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void mtk_crtc_atomic_begin(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -868,7 +868,7 @@ static void mtk_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void mtk_crtc_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct mtk_crtc *mtk_crtc = to_mtk_crtc(crtc);
 	int i;
diff --git a/drivers/gpu/drm/mediatek/mtk_crtc.h b/drivers/gpu/drm/mediatek/mtk_crtc.h
index 828f109b83e78..a6b8697cca977 100644
--- a/drivers/gpu/drm/mediatek/mtk_crtc.h
+++ b/drivers/gpu/drm/mediatek/mtk_crtc.h
@@ -23,7 +23,7 @@ int mtk_crtc_plane_check(struct drm_crtc *crtc, struct drm_plane *plane,
 			 struct mtk_plane_state *state);
 void mtk_crtc_plane_disable(struct drm_crtc *crtc, struct drm_plane *plane);
 void mtk_crtc_async_update(struct drm_crtc *crtc, struct drm_plane *plane,
-			   struct drm_atomic_state *plane_state);
+			   struct drm_atomic_commit *plane_state);
 struct device *mtk_crtc_dma_dev_get(struct drm_crtc *crtc);
 
 #endif /* MTK_CRTC_H */
diff --git a/drivers/gpu/drm/mediatek/mtk_dp.c b/drivers/gpu/drm/mediatek/mtk_dp.c
index c52cc7c2e2006..eefbc7e0f9c8d 100644
--- a/drivers/gpu/drm/mediatek/mtk_dp.c
+++ b/drivers/gpu/drm/mediatek/mtk_dp.c
@@ -2374,7 +2374,7 @@ static void mtk_dp_bridge_detach(struct drm_bridge *bridge)
 }
 
 static void mtk_dp_bridge_atomic_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
 	int ret;
@@ -2428,7 +2428,7 @@ static void mtk_dp_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void mtk_dp_bridge_atomic_disable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct mtk_dp *mtk_dp = mtk_dp_from_bridge(bridge);
 
diff --git a/drivers/gpu/drm/mediatek/mtk_dpi.c b/drivers/gpu/drm/mediatek/mtk_dpi.c
index 53360b5d12ba5..cc6d74ce8b5e7 100644
--- a/drivers/gpu/drm/mediatek/mtk_dpi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dpi.c
@@ -1049,7 +1049,6 @@ static int mtk_dpi_bind(struct device *dev, struct device *master, void *data)
 		ret = PTR_ERR(dpi->connector);
 		goto err_cleanup;
 	}
-	drm_connector_attach_encoder(dpi->connector, &dpi->encoder);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.h b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
index 1fc3df4b54858..d48da019a4a3e 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.h
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.h
@@ -63,7 +63,7 @@ struct mtk_drm_private {
 	struct device_node *comp_node[DDP_COMPONENT_DRM_ID_MAX];
 	struct mtk_ddp_comp ddp_comp[DDP_COMPONENT_DRM_ID_MAX];
 	struct mtk_mmsys_driver_data *data;
-	struct drm_atomic_state *suspend_state;
+	struct drm_atomic_commit *suspend_state;
 	unsigned int mbox_index;
 	struct mtk_drm_private **all_drm_private;
 };
diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
index 5aa71fcdcfab7..46cf4b043f123 100644
--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
@@ -827,7 +827,7 @@ static void mtk_dsi_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void mtk_dsi_bridge_atomic_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct mtk_dsi *dsi = bridge_to_dsi(bridge);
 
@@ -835,7 +835,7 @@ static void mtk_dsi_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void mtk_dsi_bridge_atomic_enable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct mtk_dsi *dsi = bridge_to_dsi(bridge);
 
@@ -846,7 +846,7 @@ static void mtk_dsi_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void mtk_dsi_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct mtk_dsi *dsi = bridge_to_dsi(bridge);
 	int ret;
@@ -857,7 +857,7 @@ static void mtk_dsi_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void mtk_dsi_bridge_atomic_post_disable(struct drm_bridge *bridge,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct mtk_dsi *dsi = bridge_to_dsi(bridge);
 
@@ -936,7 +936,6 @@ static int mtk_dsi_encoder_init(struct drm_device *drm, struct mtk_dsi *dsi)
 		ret = PTR_ERR(dsi->connector);
 		goto err_cleanup_encoder;
 	}
-	drm_connector_attach_encoder(dsi->connector, &dsi->encoder);
 
 	return 0;
 
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
index 1ea2598547800..17b10de5ff60d 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
@@ -999,7 +999,7 @@ static int mtk_hdmi_bridge_attach(struct drm_bridge *bridge,
 }
 
 static void mtk_hdmi_bridge_atomic_disable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
 
@@ -1016,7 +1016,7 @@ static void mtk_hdmi_bridge_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void mtk_hdmi_bridge_atomic_post_disable(struct drm_bridge *bridge,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
 
@@ -1030,7 +1030,7 @@ static void mtk_hdmi_bridge_atomic_post_disable(struct drm_bridge *bridge,
 }
 
 static void mtk_hdmi_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
 
@@ -1051,7 +1051,7 @@ static void mtk_hdmi_send_infoframe(struct mtk_hdmi *hdmi,
 }
 
 static void mtk_hdmi_bridge_atomic_enable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
 
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c b/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c
index b5c738380dc25..81787cc95d2aa 100644
--- a/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi_v2.c
@@ -1010,7 +1010,7 @@ static void mtk_hdmi_v2_handle_plugged_change(struct mtk_hdmi *hdmi, bool plugge
 }
 
 static void mtk_hdmi_v2_bridge_pre_enable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
 	struct drm_connector_state *conn_state;
@@ -1048,7 +1048,7 @@ static void mtk_hdmi_v2_bridge_pre_enable(struct drm_bridge *bridge,
 }
 
 static void mtk_hdmi_v2_bridge_enable(struct drm_bridge *bridge,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
 	int ret;
@@ -1069,7 +1069,7 @@ static void mtk_hdmi_v2_bridge_enable(struct drm_bridge *bridge,
 }
 
 static void mtk_hdmi_v2_bridge_disable(struct drm_bridge *bridge,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
 
@@ -1086,7 +1086,7 @@ static void mtk_hdmi_v2_bridge_disable(struct drm_bridge *bridge,
 }
 
 static void mtk_hdmi_v2_bridge_post_disable(struct drm_bridge *bridge,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
 
diff --git a/drivers/gpu/drm/mediatek/mtk_plane.c b/drivers/gpu/drm/mediatek/mtk_plane.c
index 86c6f60eadb32..2598c8edb443d 100644
--- a/drivers/gpu/drm/mediatek/mtk_plane.c
+++ b/drivers/gpu/drm/mediatek/mtk_plane.c
@@ -80,7 +80,7 @@ static void mtk_plane_destroy_state(struct drm_plane *plane,
 }
 
 static int mtk_plane_atomic_async_check(struct drm_plane *plane,
-					struct drm_atomic_state *state, bool flip)
+					struct drm_atomic_commit *state, bool flip)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -193,7 +193,7 @@ static void mtk_plane_update_new_state(struct drm_plane_state *new_state,
 }
 
 static void mtk_plane_atomic_async_update(struct drm_plane *plane,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -228,7 +228,7 @@ static const struct drm_plane_funcs mtk_plane_funcs = {
 };
 
 static int mtk_plane_atomic_check(struct drm_plane *plane,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -260,7 +260,7 @@ static int mtk_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void mtk_plane_atomic_disable(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -277,7 +277,7 @@ static void mtk_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static void mtk_plane_atomic_update(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/meson/meson_crtc.c b/drivers/gpu/drm/meson/meson_crtc.c
index d70616da8ce2f..b77d2d24ead1c 100644
--- a/drivers/gpu/drm/meson/meson_crtc.c
+++ b/drivers/gpu/drm/meson/meson_crtc.c
@@ -82,7 +82,7 @@ static const struct drm_crtc_funcs meson_crtc_funcs = {
 };
 
 static void meson_g12a_crtc_atomic_enable(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
 	struct drm_crtc_state *crtc_state = crtc->state;
@@ -118,7 +118,7 @@ static void meson_g12a_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void meson_crtc_atomic_enable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
 	struct drm_crtc_state *crtc_state = crtc->state;
@@ -146,7 +146,7 @@ static void meson_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void meson_g12a_crtc_atomic_disable(struct drm_crtc *crtc,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
 	struct meson_drm *priv = meson_crtc->priv;
@@ -171,7 +171,7 @@ static void meson_g12a_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void meson_crtc_atomic_disable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
 	struct meson_drm *priv = meson_crtc->priv;
@@ -201,7 +201,7 @@ static void meson_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void meson_crtc_atomic_begin(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
 	unsigned long flags;
@@ -217,7 +217,7 @@ static void meson_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void meson_crtc_atomic_flush(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct meson_crtc *meson_crtc = to_meson_crtc(crtc);
 	struct meson_drm *priv = meson_crtc->priv;
diff --git a/drivers/gpu/drm/meson/meson_encoder_cvbs.c b/drivers/gpu/drm/meson/meson_encoder_cvbs.c
index 41071d6e05e5f..8b26a0031cde0 100644
--- a/drivers/gpu/drm/meson/meson_encoder_cvbs.c
+++ b/drivers/gpu/drm/meson/meson_encoder_cvbs.c
@@ -139,7 +139,7 @@ static int meson_encoder_cvbs_atomic_check(struct drm_bridge *bridge,
 }
 
 static void meson_encoder_cvbs_atomic_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct meson_encoder_cvbs *encoder_cvbs = bridge_to_meson_encoder_cvbs(bridge);
 	struct meson_drm *priv = encoder_cvbs->priv;
@@ -190,7 +190,7 @@ static void meson_encoder_cvbs_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void meson_encoder_cvbs_atomic_disable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct meson_encoder_cvbs *meson_encoder_cvbs =
 					bridge_to_meson_encoder_cvbs(bridge);
@@ -279,8 +279,6 @@ int meson_encoder_cvbs_probe(struct meson_drm *priv)
 		return dev_err_probe(priv->dev, PTR_ERR(connector),
 				     "Unable to create CVBS bridge connector\n");
 
-	drm_connector_attach_encoder(connector, &meson_encoder_cvbs->encoder);
-
 	priv->encoders[MESON_ENC_CVBS] = meson_encoder_cvbs;
 
 	return 0;
diff --git a/drivers/gpu/drm/meson/meson_encoder_dsi.c b/drivers/gpu/drm/meson/meson_encoder_dsi.c
index eba246791c686..c1f4685073bbc 100644
--- a/drivers/gpu/drm/meson/meson_encoder_dsi.c
+++ b/drivers/gpu/drm/meson/meson_encoder_dsi.c
@@ -42,7 +42,7 @@ static int meson_encoder_dsi_attach(struct drm_bridge *bridge,
 }
 
 static void meson_encoder_dsi_atomic_enable(struct drm_bridge *bridge,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct meson_encoder_dsi *encoder_dsi = bridge_to_meson_encoder_dsi(bridge);
 	struct meson_drm *priv = encoder_dsi->priv;
@@ -79,7 +79,7 @@ static void meson_encoder_dsi_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void meson_encoder_dsi_atomic_disable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct meson_encoder_dsi *meson_encoder_dsi =
 					bridge_to_meson_encoder_dsi(bridge);
diff --git a/drivers/gpu/drm/meson/meson_encoder_hdmi.c b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
index 1abb0572bb5fd..55c0601df3c62 100644
--- a/drivers/gpu/drm/meson/meson_encoder_hdmi.c
+++ b/drivers/gpu/drm/meson/meson_encoder_hdmi.c
@@ -189,7 +189,7 @@ static enum drm_mode_status meson_encoder_hdmi_mode_valid(struct drm_bridge *bri
 }
 
 static void meson_encoder_hdmi_atomic_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct meson_encoder_hdmi *encoder_hdmi = bridge_to_meson_encoder_hdmi(bridge);
 	unsigned int ycrcb_map = VPU_HDMI_OUTPUT_CBYCR;
@@ -252,7 +252,7 @@ static void meson_encoder_hdmi_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void meson_encoder_hdmi_atomic_disable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct meson_encoder_hdmi *encoder_hdmi = bridge_to_meson_encoder_hdmi(bridge);
 	struct meson_drm *priv = encoder_hdmi->priv;
@@ -433,8 +433,6 @@ int meson_encoder_hdmi_probe(struct meson_drm *priv)
 				    "Unable to create HDMI bridge connector\n");
 		goto err_put_node;
 	}
-	drm_connector_attach_encoder(meson_encoder_hdmi->connector,
-				     &meson_encoder_hdmi->encoder);
 
 	/*
 	 * We should have now in place:
diff --git a/drivers/gpu/drm/meson/meson_overlay.c b/drivers/gpu/drm/meson/meson_overlay.c
index 783572b16963e..7047807f1a3e2 100644
--- a/drivers/gpu/drm/meson/meson_overlay.c
+++ b/drivers/gpu/drm/meson/meson_overlay.c
@@ -167,7 +167,7 @@ struct meson_overlay {
 #define FRAC_16_16(mult, div)    (((mult) << 16) / (div))
 
 static int meson_overlay_atomic_check(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -470,7 +470,7 @@ static void meson_overlay_setup_scaler_params(struct meson_drm *priv,
 }
 
 static void meson_overlay_atomic_update(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct meson_overlay *meson_overlay = to_meson_overlay(plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
@@ -724,7 +724,7 @@ static void meson_overlay_atomic_update(struct drm_plane *plane,
 }
 
 static void meson_overlay_atomic_disable(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct meson_overlay *meson_overlay = to_meson_overlay(plane);
 	struct meson_drm *priv = meson_overlay->priv;
diff --git a/drivers/gpu/drm/meson/meson_plane.c b/drivers/gpu/drm/meson/meson_plane.c
index f8d0e0874a5d6..2d88fe87a11ac 100644
--- a/drivers/gpu/drm/meson/meson_plane.c
+++ b/drivers/gpu/drm/meson/meson_plane.c
@@ -73,7 +73,7 @@ struct meson_plane {
 #define FRAC_16_16(mult, div)    (((mult) << 16) / (div))
 
 static int meson_plane_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -132,7 +132,7 @@ static u32 meson_g12a_afbcd_line_stride(struct meson_drm *priv)
 }
 
 static void meson_plane_atomic_update(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct meson_plane *meson_plane = to_meson_plane(plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
@@ -398,7 +398,7 @@ static void meson_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void meson_plane_atomic_disable(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct meson_plane *meson_plane = to_meson_plane(plane);
 	struct meson_drm *priv = meson_plane->priv;
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c
index a32be27c39e8f..d7b5dfdb28aa0 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.c
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.c
@@ -19,7 +19,6 @@
 #include <drm/drm_ioctl.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_module.h>
-#include <drm/drm_pciids.h>
 #include <drm/drm_print.h>
 
 #include "mgag200_drv.h"
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.h b/drivers/gpu/drm/mgag200/mgag200_drv.h
index a875c4bf8cbe4..3732c62351bbb 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.h
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.h
@@ -253,14 +253,14 @@ struct mgag200_device_funcs {
 	 * success, the calculated parameters should be stored in the CRTC's
 	 * state in struct @mgag200_crtc_state.pixpllc.
 	 */
-	int (*pixpllc_atomic_check)(struct drm_crtc *crtc, struct drm_atomic_state *new_state);
+	int (*pixpllc_atomic_check)(struct drm_crtc *crtc, struct drm_atomic_commit *new_state);
 
 	/*
 	 * Program PIXPLLC from the CRTC state. The parameters should have been
 	 * stored in struct @mgag200_crtc_state.pixpllc by the corresponding
 	 * implementation of @pixpllc_atomic_check.
 	 */
-	void (*pixpllc_atomic_update)(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+	void (*pixpllc_atomic_update)(struct drm_crtc *crtc, struct drm_atomic_commit *old_state);
 };
 
 struct mga_device {
@@ -332,11 +332,11 @@ struct mga_device *mgag200_g200_device_create(struct pci_dev *pdev, const struct
 struct mga_device *mgag200_g200se_device_create(struct pci_dev *pdev, const struct drm_driver *drv,
 						enum mga_type type);
 void mgag200_g200wb_init_registers(struct mga_device *mdev);
-void mgag200_g200wb_pixpllc_atomic_update(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+void mgag200_g200wb_pixpllc_atomic_update(struct drm_crtc *crtc, struct drm_atomic_commit *old_state);
 struct mga_device *mgag200_g200wb_device_create(struct pci_dev *pdev, const struct drm_driver *drv);
 struct mga_device *mgag200_g200ev_device_create(struct pci_dev *pdev, const struct drm_driver *drv);
 void mgag200_g200eh_init_registers(struct mga_device *mdev);
-void mgag200_g200eh_pixpllc_atomic_update(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+void mgag200_g200eh_pixpllc_atomic_update(struct drm_crtc *crtc, struct drm_atomic_commit *old_state);
 struct mga_device *mgag200_g200eh_device_create(struct pci_dev *pdev,
 						const struct drm_driver *drv);
 struct mga_device *mgag200_g200eh3_device_create(struct pci_dev *pdev,
@@ -356,7 +356,7 @@ struct drm_crtc;
 struct drm_crtc_state;
 struct drm_display_mode;
 struct drm_plane;
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_scanout_buffer;
 
 extern const uint32_t mgag200_primary_plane_formats[];
@@ -364,13 +364,13 @@ extern const size_t   mgag200_primary_plane_formats_size;
 extern const uint64_t mgag200_primary_plane_fmtmods[];
 
 int mgag200_primary_plane_helper_atomic_check(struct drm_plane *plane,
-					      struct drm_atomic_state *new_state);
+					      struct drm_atomic_commit *new_state);
 void mgag200_primary_plane_helper_atomic_update(struct drm_plane *plane,
-						struct drm_atomic_state *old_state);
+						struct drm_atomic_commit *old_state);
 void mgag200_primary_plane_helper_atomic_enable(struct drm_plane *plane,
-						struct drm_atomic_state *state);
+						struct drm_atomic_commit *state);
 void mgag200_primary_plane_helper_atomic_disable(struct drm_plane *plane,
-						 struct drm_atomic_state *old_state);
+						 struct drm_atomic_commit *old_state);
 int mgag200_primary_plane_helper_get_scanout_buffer(struct drm_plane *plane,
 						    struct drm_scanout_buffer *sb);
 
@@ -395,10 +395,10 @@ void mgag200_crtc_load_gamma(struct mga_device *mdev,
 
 enum drm_mode_status mgag200_crtc_helper_mode_valid(struct drm_crtc *crtc,
 						    const struct drm_display_mode *mode);
-int mgag200_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *new_state);
-void mgag200_crtc_helper_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
-void mgag200_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
-void mgag200_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *old_state);
+int mgag200_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *new_state);
+void mgag200_crtc_helper_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_commit *old_state);
+void mgag200_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_commit *old_state);
+void mgag200_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_commit *old_state);
 
 #define MGAG200_CRTC_HELPER_FUNCS \
 	.mode_valid = mgag200_crtc_helper_mode_valid, \
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200.c b/drivers/gpu/drm/mgag200/mgag200_g200.c
index a5e291b344dbd..9e6b4618fadda 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200.c
@@ -62,7 +62,7 @@ static void mgag200_g200_init_registers(struct mgag200_g200_device *g200)
  * PIXPLLC
  */
 
-static int mgag200_g200_pixpllc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *new_state)
+static int mgag200_g200_pixpllc_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *new_state)
 {
 	static const int post_div_max = 7;
 	static const int in_div_min = 1;
@@ -138,7 +138,7 @@ static int mgag200_g200_pixpllc_atomic_check(struct drm_crtc *crtc, struct drm_a
 }
 
 static void mgag200_g200_pixpllc_atomic_update(struct drm_crtc *crtc,
-					       struct drm_atomic_state *old_state)
+					       struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200eh.c b/drivers/gpu/drm/mgag200/mgag200_g200eh.c
index d2aa931f579d9..e64bfeafbefc4 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200eh.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200eh.c
@@ -41,7 +41,7 @@ void mgag200_g200eh_init_registers(struct mga_device *mdev)
  */
 
 static int mgag200_g200eh_pixpllc_atomic_check(struct drm_crtc *crtc,
-					       struct drm_atomic_state *new_state)
+					       struct drm_atomic_commit *new_state)
 {
 	static const unsigned int vcomax = 800000;
 	static const unsigned int vcomin = 400000;
@@ -91,7 +91,7 @@ static int mgag200_g200eh_pixpllc_atomic_check(struct drm_crtc *crtc,
 }
 
 void mgag200_g200eh_pixpllc_atomic_update(struct drm_crtc *crtc,
-					  struct drm_atomic_state *old_state)
+					  struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200eh3.c b/drivers/gpu/drm/mgag200/mgag200_g200eh3.c
index 7bea7a728f562..183b7676ea6a1 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200eh3.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200eh3.c
@@ -16,7 +16,7 @@
  */
 
 static int mgag200_g200eh3_pixpllc_atomic_check(struct drm_crtc *crtc,
-						struct drm_atomic_state *new_state)
+						struct drm_atomic_commit *new_state)
 {
 	static const unsigned int vcomax = 3000000;
 	static const unsigned int vcomin = 1500000;
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200eh5.c b/drivers/gpu/drm/mgag200/mgag200_g200eh5.c
index 36da6529d74fc..dc375c055b0e6 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200eh5.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200eh5.c
@@ -18,7 +18,7 @@
  */
 
 static int mgag200_g200eh5_pixpllc_atomic_check(struct drm_crtc *crtc,
-						struct drm_atomic_state *new_state)
+						struct drm_atomic_commit *new_state)
 {
 	const unsigned long long VCO_MAX = 10 * GIGA; // Hz
 	const unsigned long long VCO_MIN = 2500 * MEGA; // Hz
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200er.c b/drivers/gpu/drm/mgag200/mgag200_g200er.c
index 8fa8fe943abfd..205678a57bfe9 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200er.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200er.c
@@ -58,7 +58,7 @@ static void mgag200_g200er_reset_tagfifo(struct mga_device *mdev)
  */
 
 static int mgag200_g200er_pixpllc_atomic_check(struct drm_crtc *crtc,
-					       struct drm_atomic_state *new_state)
+					       struct drm_atomic_commit *new_state)
 {
 	static const unsigned int vcomax = 1488000;
 	static const unsigned int vcomin = 1056000;
@@ -119,7 +119,7 @@ static int mgag200_g200er_pixpllc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void mgag200_g200er_pixpllc_atomic_update(struct drm_crtc *crtc,
-						 struct drm_atomic_state *old_state)
+						 struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
@@ -182,7 +182,7 @@ static const struct drm_plane_funcs mgag200_g200er_primary_plane_funcs = {
 };
 
 static void mgag200_g200er_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-						     struct drm_atomic_state *old_state)
+						     struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200ev.c b/drivers/gpu/drm/mgag200/mgag200_g200ev.c
index 3fadbeb10af96..d60693ac0749d 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200ev.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200ev.c
@@ -47,7 +47,7 @@ static void mgag200_g200ev_set_hiprilvl(struct mga_device *mdev)
  */
 
 static int mgag200_g200ev_pixpllc_atomic_check(struct drm_crtc *crtc,
-					       struct drm_atomic_state *new_state)
+					       struct drm_atomic_commit *new_state)
 {
 	static const unsigned int vcomax = 550000;
 	static const unsigned int vcomin = 150000;
@@ -98,7 +98,7 @@ static int mgag200_g200ev_pixpllc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void mgag200_g200ev_pixpllc_atomic_update(struct drm_crtc *crtc,
-						 struct drm_atomic_state *old_state)
+						 struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
@@ -183,7 +183,7 @@ static const struct drm_plane_funcs mgag200_g200ev_primary_plane_funcs = {
 };
 
 static void mgag200_g200ev_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-						     struct drm_atomic_state *old_state)
+						     struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200ew3.c b/drivers/gpu/drm/mgag200/mgag200_g200ew3.c
index e387a455eae5c..9c14d0eb66bca 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200ew3.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200ew3.c
@@ -23,7 +23,7 @@ static void mgag200_g200ew3_init_registers(struct mga_device *mdev)
  */
 
 static int mgag200_g200ew3_pixpllc_atomic_check(struct drm_crtc *crtc,
-						struct drm_atomic_state *new_state)
+						struct drm_atomic_commit *new_state)
 {
 	static const unsigned int vcomax = 800000;
 	static const unsigned int vcomin = 400000;
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200se.c b/drivers/gpu/drm/mgag200/mgag200_g200se.c
index a0ac19ee03531..349ec36f6d491 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200se.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200se.c
@@ -76,12 +76,19 @@ static void mgag200_g200se_set_hiprilvl(struct mga_device *mdev,
 		unsigned int bpp;
 		unsigned long mb;
 
-		if (format->cpp[0] * 8 > 16)
+		switch (format->format) {
+		case DRM_FORMAT_XRGB8888:
+		case DRM_FORMAT_RGB888:
 			bpp = 32;
-		else if (format->cpp[0] * 8 > 8)
+			break;
+		case DRM_FORMAT_RGB565:
+		case DRM_FORMAT_XRGB1555:
 			bpp = 16;
-		else
+			break;
+		case DRM_FORMAT_C8:
 			bpp = 8;
+			break;
+		}
 
 		mb = (mode->clock * bpp) / 1000;
 		if (mb > 3100)
@@ -113,7 +120,7 @@ static void mgag200_g200se_set_hiprilvl(struct mga_device *mdev,
  */
 
 static int mgag200_g200se_00_pixpllc_atomic_check(struct drm_crtc *crtc,
-						  struct drm_atomic_state *new_state)
+						  struct drm_atomic_commit *new_state)
 {
 	static const unsigned int vcomax = 320000;
 	static const unsigned int vcomin = 160000;
@@ -169,7 +176,7 @@ static int mgag200_g200se_00_pixpllc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void mgag200_g200se_00_pixpllc_atomic_update(struct drm_crtc *crtc,
-						    struct drm_atomic_state *old_state)
+						    struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
@@ -196,7 +203,7 @@ static void mgag200_g200se_00_pixpllc_atomic_update(struct drm_crtc *crtc,
 }
 
 static int mgag200_g200se_04_pixpllc_atomic_check(struct drm_crtc *crtc,
-						  struct drm_atomic_state *new_state)
+						  struct drm_atomic_commit *new_state)
 {
 	static const unsigned int vcomax = 1600000;
 	static const unsigned int vcomin = 800000;
@@ -270,7 +277,7 @@ static int mgag200_g200se_04_pixpllc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void mgag200_g200se_04_pixpllc_atomic_update(struct drm_crtc *crtc,
-						    struct drm_atomic_state *old_state)
+						    struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
@@ -314,7 +321,7 @@ static const struct drm_plane_funcs mgag200_g200se_primary_plane_funcs = {
 };
 
 static void mgag200_g200se_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-						     struct drm_atomic_state *old_state)
+						     struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_g200wb.c b/drivers/gpu/drm/mgag200/mgag200_g200wb.c
index d847fa8ded8c8..f65d4238422f4 100644
--- a/drivers/gpu/drm/mgag200/mgag200_g200wb.c
+++ b/drivers/gpu/drm/mgag200/mgag200_g200wb.c
@@ -39,7 +39,7 @@ void mgag200_g200wb_init_registers(struct mga_device *mdev)
  */
 
 static int mgag200_g200wb_pixpllc_atomic_check(struct drm_crtc *crtc,
-					       struct drm_atomic_state *new_state)
+					       struct drm_atomic_commit *new_state)
 {
 	static const unsigned int vcomax = 550000;
 	static const unsigned int vcomin = 150000;
@@ -90,7 +90,7 @@ static int mgag200_g200wb_pixpllc_atomic_check(struct drm_crtc *crtc,
 }
 
 void mgag200_g200wb_pixpllc_atomic_update(struct drm_crtc *crtc,
-					  struct drm_atomic_state *old_state)
+					  struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index 8894a063b1a15..7e07fc3f1a603 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -56,6 +56,12 @@ void mgag200_crtc_fill_gamma(struct mga_device *mdev,
 	struct drm_crtc *crtc = &mdev->crtc;
 
 	switch (format->format) {
+	case DRM_FORMAT_C8:
+		drm_crtc_fill_palette_8(crtc, mgag200_set_gamma_lut);
+		break;
+	case DRM_FORMAT_XRGB1555:
+		drm_crtc_fill_gamma_555(crtc, mgag200_set_gamma_lut);
+		break;
 	case DRM_FORMAT_RGB565:
 		drm_crtc_fill_gamma_565(crtc, mgag200_set_gamma_lut);
 		break;
@@ -77,6 +83,12 @@ void mgag200_crtc_load_gamma(struct mga_device *mdev,
 	struct drm_crtc *crtc = &mdev->crtc;
 
 	switch (format->format) {
+	case DRM_FORMAT_C8:
+		drm_crtc_load_palette_8(crtc, lut, mgag200_set_gamma_lut);
+		break;
+	case DRM_FORMAT_XRGB1555:
+		drm_crtc_load_gamma_555_from_888(crtc, lut, mgag200_set_gamma_lut);
+		break;
 	case DRM_FORMAT_RGB565:
 		drm_crtc_load_gamma_565_from_888(crtc, lut, mgag200_set_gamma_lut);
 		break;
@@ -174,6 +186,7 @@ static void mgag200_set_datasiz(struct mga_device *mdev, u32 format)
 
 	/* Big-endian byte-swapping */
 	switch (format) {
+	case DRM_FORMAT_XRGB1555:
 	case DRM_FORMAT_RGB565:
 		opmode |= 0x10100;
 		break;
@@ -298,37 +311,11 @@ void mgag200_set_mode_regs(struct mga_device *mdev, const struct drm_display_mod
 	WREG8(MGA_MISC_OUT, misc);
 }
 
-static u8 mgag200_get_bpp_shift(const struct drm_format_info *format)
-{
-	static const u8 bpp_shift[] = {0, 1, 0, 2};
-
-	return bpp_shift[format->cpp[0] - 1];
-}
-
-/*
- * Calculates the HW offset value from the framebuffer's pitch. The
- * offset is a multiple of the pixel size and depends on the display
- * format.
- */
-static u32 mgag200_calculate_offset(struct mga_device *mdev,
-				    const struct drm_framebuffer *fb)
-{
-	u32 offset = fb->pitches[0] / fb->format->cpp[0];
-	u8 bppshift = mgag200_get_bpp_shift(fb->format);
-
-	if (fb->format->cpp[0] * 8 == 24)
-		offset = (offset * 3) >> (4 - bppshift);
-	else
-		offset = offset >> (4 - bppshift);
-
-	return offset;
-}
-
 static void mgag200_set_offset(struct mga_device *mdev,
 			       const struct drm_framebuffer *fb)
 {
 	u8 crtc13, crtcext0;
-	u32 offset = mgag200_calculate_offset(mdev, fb);
+	u32 offset = fb->pitches[0] / 16;
 
 	RREG_ECRT(0, crtcext0);
 
@@ -343,49 +330,49 @@ static void mgag200_set_offset(struct mga_device *mdev,
 
 void mgag200_set_format_regs(struct mga_device *mdev, const struct drm_format_info *format)
 {
-	struct drm_device *dev = &mdev->base;
-	unsigned int bpp, bppshift, scale;
-	u8 crtcext3, xmulctrl;
+	u8 xmulctrl = 0;
+	u8 crtcext3;
 
-	bpp = format->cpp[0] * 8;
+	RREG_ECRT(3, crtcext3);
 
-	bppshift = mgag200_get_bpp_shift(format);
-	switch (bpp) {
-	case 24:
-		scale = ((1 << bppshift) * 3) - 1;
+	switch (format->format) {
+	case DRM_FORMAT_C8:
+		crtcext3 &= ~MGAREG_CRTCEXT3_SCALE_MASK;
+		crtcext3 |= 0x0;
 		break;
-	default:
-		scale = (1 << bppshift) - 1;
+	case DRM_FORMAT_XRGB1555:
+	case DRM_FORMAT_RGB565:
+		crtcext3 &= ~MGAREG_CRTCEXT3_SCALE_MASK;
+		crtcext3 |= 0x01;
+		break;
+	case DRM_FORMAT_RGB888:
+		crtcext3 &= ~MGAREG_CRTCEXT3_SCALE_MASK;
+		crtcext3 |= 0x02;
+		break;
+	case DRM_FORMAT_XRGB8888:
+		crtcext3 &= ~MGAREG_CRTCEXT3_SCALE_MASK;
+		crtcext3 |= 0x03;
 		break;
 	}
 
-	RREG_ECRT(3, crtcext3);
-
-	switch (bpp) {
-	case 8:
+	switch (format->format) {
+	case DRM_FORMAT_C8:
 		xmulctrl = MGA1064_MUL_CTL_8bits;
 		break;
-	case 16:
-		if (format->depth == 15)
-			xmulctrl = MGA1064_MUL_CTL_15bits;
-		else
-			xmulctrl = MGA1064_MUL_CTL_16bits;
+	case DRM_FORMAT_XRGB1555:
+		xmulctrl = MGA1064_MUL_CTL_15bits;
+		break;
+	case DRM_FORMAT_RGB565:
+		xmulctrl = MGA1064_MUL_CTL_16bits;
 		break;
-	case 24:
+	case DRM_FORMAT_RGB888:
 		xmulctrl = MGA1064_MUL_CTL_24bits;
 		break;
-	case 32:
+	case DRM_FORMAT_XRGB8888:
 		xmulctrl = MGA1064_MUL_CTL_32_24bits;
 		break;
-	default:
-		/* BUG: We should have caught this problem already. */
-		drm_WARN_ON(dev, "invalid format depth\n");
-		return;
 	}
 
-	crtcext3 &= ~GENMASK(2, 0);
-	crtcext3 |= scale;
-
 	WREG_DAC(MGA1064_MUL_CTL, xmulctrl);
 
 	WREG_GFX(0, 0x00);
@@ -463,7 +450,9 @@ static void mgag200_handle_damage(struct mga_device *mdev, const struct iosys_ma
 const uint32_t mgag200_primary_plane_formats[] = {
 	DRM_FORMAT_XRGB8888,
 	DRM_FORMAT_RGB565,
+	DRM_FORMAT_XRGB1555,
 	DRM_FORMAT_RGB888,
+	DRM_FORMAT_C8,
 };
 
 const size_t mgag200_primary_plane_formats_size = ARRAY_SIZE(mgag200_primary_plane_formats);
@@ -474,7 +463,7 @@ const uint64_t mgag200_primary_plane_fmtmods[] = {
 };
 
 int mgag200_primary_plane_helper_atomic_check(struct drm_plane *plane,
-					      struct drm_atomic_state *new_state)
+					      struct drm_atomic_commit *new_state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(new_state, plane);
 	struct drm_framebuffer *new_fb = new_plane_state->fb;
@@ -509,7 +498,7 @@ int mgag200_primary_plane_helper_atomic_check(struct drm_plane *plane,
 }
 
 void mgag200_primary_plane_helper_atomic_update(struct drm_plane *plane,
-						struct drm_atomic_state *old_state)
+						struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = plane->dev;
 	struct mga_device *mdev = to_mga_device(dev);
@@ -532,7 +521,7 @@ void mgag200_primary_plane_helper_atomic_update(struct drm_plane *plane,
 }
 
 void mgag200_primary_plane_helper_atomic_enable(struct drm_plane *plane,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = plane->dev;
 	struct mga_device *mdev = to_mga_device(dev);
@@ -545,7 +534,7 @@ void mgag200_primary_plane_helper_atomic_enable(struct drm_plane *plane,
 }
 
 void mgag200_primary_plane_helper_atomic_disable(struct drm_plane *plane,
-						 struct drm_atomic_state *old_state)
+						 struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = plane->dev;
 	struct mga_device *mdev = to_mga_device(dev);
@@ -608,7 +597,7 @@ enum drm_mode_status mgag200_crtc_helper_mode_valid(struct drm_crtc *crtc,
 	return MODE_OK;
 }
 
-int mgag200_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *new_state)
+int mgag200_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *new_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
@@ -642,7 +631,7 @@ int mgag200_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_st
 	return 0;
 }
 
-void mgag200_crtc_helper_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *old_state)
+void mgag200_crtc_helper_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_commit *old_state)
 {
 	struct drm_crtc_state *crtc_state = crtc->state;
 	struct mgag200_crtc_state *mgag200_crtc_state = to_mgag200_crtc_state(crtc_state);
@@ -659,7 +648,7 @@ void mgag200_crtc_helper_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_s
 	}
 }
 
-void mgag200_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *old_state)
+void mgag200_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct mga_device *mdev = to_mga_device(dev);
@@ -683,7 +672,7 @@ void mgag200_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_
 	mgag200_enable_display(mdev);
 }
 
-void mgag200_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *old_state)
+void mgag200_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_commit *old_state)
 {
 	struct mga_device *mdev = to_mga_device(crtc->dev);
 
@@ -738,7 +727,7 @@ void mgag200_crtc_atomic_destroy_state(struct drm_crtc *crtc, struct drm_crtc_st
  * Mode config
  */
 
-static void mgag200_mode_config_helper_atomic_commit_tail(struct drm_atomic_state *state)
+static void mgag200_mode_config_helper_atomic_commit_tail(struct drm_atomic_commit *state)
 {
 	struct mga_device *mdev = to_mga_device(state->dev);
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_reg.h b/drivers/gpu/drm/mgag200/mgag200_reg.h
index d4fef8f25871f..088b6c16f7f62 100644
--- a/drivers/gpu/drm/mgag200/mgag200_reg.h
+++ b/drivers/gpu/drm/mgag200/mgag200_reg.h
@@ -265,6 +265,7 @@
 #define MGAREG_CRTCEXT1_HRSTEN		BIT(3)
 
 #define MGAREG_CRTCEXT3_MGAMODE		BIT(7)
+#define MGAREG_CRTCEXT3_SCALE_MASK	GENMASK(2, 0)
 
 /* Cursor X and Y position */
 #define MGA_CURPOSXL 0x3c0c
diff --git a/drivers/gpu/drm/mgag200/mgag200_vga_bmc.c b/drivers/gpu/drm/mgag200/mgag200_vga_bmc.c
index a855f17343161..805636990b55f 100644
--- a/drivers/gpu/drm/mgag200/mgag200_vga_bmc.c
+++ b/drivers/gpu/drm/mgag200/mgag200_vga_bmc.c
@@ -10,7 +10,7 @@
 #include "mgag200_drv.h"
 
 static void mgag200_vga_bmc_encoder_atomic_disable(struct drm_encoder *encoder,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct mga_device *mdev = to_mga_device(encoder->dev);
 
@@ -19,7 +19,7 @@ static void mgag200_vga_bmc_encoder_atomic_disable(struct drm_encoder *encoder,
 }
 
 static void mgag200_vga_bmc_encoder_atomic_enable(struct drm_encoder *encoder,
-						  struct drm_atomic_state *state)
+						  struct drm_atomic_commit *state)
 {
 	struct mga_device *mdev = to_mga_device(encoder->dev);
 
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index d5aba072f44cc..7a3e3c2f5cf36 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -2621,7 +2621,6 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
 	struct platform_device *pdev = priv->gpu_pdev;
 	struct adreno_platform_config *config = pdev->dev.platform_data;
 	const struct adreno_info *info = config->info;
-	struct device_node *node;
 	struct a6xx_gpu *a6xx_gpu;
 	struct adreno_gpu *adreno_gpu;
 	struct msm_gpu *gpu;
@@ -2643,7 +2642,8 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
 	adreno_gpu->registers = NULL;
 
 	/* Check if there is a GMU phandle and set it up */
-	node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0);
+	struct device_node *node __free(device_node) =
+		of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0);
 	/* FIXME: How do we gracefully handle this? */
 	BUG_ON(!node);
 
@@ -2690,7 +2690,6 @@ static struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
 		ret = a6xx_gmu_wrapper_init(a6xx_gpu, node);
 	else
 		ret = a6xx_gmu_init(a6xx_gpu, node);
-	of_node_put(node);
 	if (ret) {
 		a6xx_destroy(&(a6xx_gpu->base.base));
 		return ERR_PTR(ret);
@@ -2740,6 +2739,7 @@ const struct adreno_gpu_funcs a6xx_gpu_funcs = {
 		.create_private_vm = a6xx_create_private_vm,
 		.get_rptr = a6xx_get_rptr,
 		.progress = a6xx_progress,
+		.sysprof_setup = a6xx_gmu_sysprof_setup,
 	},
 	.init = a6xx_gpu_init,
 	.get_timestamp = a6xx_gmu_get_timestamp,
@@ -2808,6 +2808,7 @@ const struct adreno_gpu_funcs a7xx_gpu_funcs = {
 		.create_private_vm = a6xx_create_private_vm,
 		.get_rptr = a6xx_get_rptr,
 		.progress = a6xx_progress,
+		.sysprof_setup = a6xx_gmu_sysprof_setup,
 	},
 	.init = a6xx_gpu_init,
 	.get_timestamp = a6xx_gmu_get_timestamp,
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
index 487c2736f2b37..186a73c0b99c0 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c
@@ -289,6 +289,8 @@ static int a8xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
 		(gmu->nr_gpu_freqs * num_gx_votes * sizeof(gmu->gx_arc_votes[0])) +
 		(gmu->nr_gmu_freqs * num_cx_votes * sizeof(gmu->cx_arc_votes[0]));
 	tbl = kzalloc(size, GFP_KERNEL);
+	if (!tbl)
+		return -ENOMEM;
 	tbl->type = HFI_TABLE_GPU_PERF;
 
 	/* First fill GX votes */
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 4edfe80c5be7c..fc38331ce6408 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -17,7 +17,7 @@ MODULE_PARM_DESC(snapshot_debugbus, "Include debugbus sections in GPU devcoredum
 module_param_named(snapshot_debugbus, snapshot_debugbus, bool, 0600);
 
 int enable_preemption = -1;
-MODULE_PARM_DESC(enable_preemption, "Enable preemption (A7xx only) (1=on , 0=disable, -1=auto (default))");
+MODULE_PARM_DESC(enable_preemption, "Enable preemption (A7xx+ only) (1=on , 0=disable, -1=auto (default))");
 module_param(enable_preemption, int, 0600);
 
 bool disable_acd;
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index 66f80f2d12f95..03f96a1154e1c 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -376,7 +376,7 @@ int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx,
 		*value = adreno_gpu->info->gmem;
 		return 0;
 	case MSM_PARAM_GMEM_BASE:
-		if (adreno_gpu->info->family >= ADRENO_6XX_GEN4)
+		if (adreno_gpu->info->family >= ADRENO_6XX_GEN3)
 			*value = 0;
 		else
 			*value = 0x100000;
@@ -424,15 +424,21 @@ int adreno_get_param(struct msm_gpu *gpu, struct msm_context *ctx,
 		*value = vm->mm_range;
 		return 0;
 	case MSM_PARAM_HIGHEST_BANK_BIT:
+		if (!adreno_gpu->ubwc_config)
+			return UERR(ENOENT, drm, "no UBWC on this platform");
 		*value = adreno_gpu->ubwc_config->highest_bank_bit;
 		return 0;
 	case MSM_PARAM_RAYTRACING:
 		*value = adreno_gpu->has_ray_tracing;
 		return 0;
 	case MSM_PARAM_UBWC_SWIZZLE:
+		if (!adreno_gpu->ubwc_config)
+			return UERR(ENOENT, drm, "no UBWC on this platform");
 		*value = adreno_gpu->ubwc_config->ubwc_swizzle;
 		return 0;
 	case MSM_PARAM_MACROTILE_MODE:
+		if (!adreno_gpu->ubwc_config)
+			return UERR(ENOENT, drm, "no UBWC on this platform");
 		*value = adreno_gpu->ubwc_config->macrotile_mode;
 		return 0;
 	case MSM_PARAM_UCHE_TRAP_BASE:
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_10_2_milos.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_10_2_milos.h
new file mode 100644
index 0000000000000..1aa8aea4e3525
--- /dev/null
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_10_2_milos.h
@@ -0,0 +1,279 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2022. Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2015-2018, 2020 The Linux Foundation. All rights reserved.
+ * Copyright (c) 2026, Luca Weiss <luca.weiss@fairphone.com>
+ */
+
+#ifndef _DPU_10_2_MILOS_H
+#define _DPU_10_2_MILOS_H
+
+static const struct dpu_caps milos_dpu_caps = {
+	.max_mixer_width = DEFAULT_DPU_OUTPUT_LINE_WIDTH,
+	.max_mixer_blendstages = 0x7,
+	.has_src_split = true,
+	.has_dim_layer = true,
+	.has_idle_pc = true,
+	.has_3d_merge = true,
+	.max_linewidth = 8192,
+	.pixel_ram_size = DEFAULT_PIXEL_RAM_SIZE,
+};
+
+static const struct dpu_mdp_cfg milos_mdp = {
+	.name = "top_0",
+	.base = 0, .len = 0x494,
+	.clk_ctrls = {
+		[DPU_CLK_CTRL_REG_DMA] = { .reg_off = 0x2bc, .bit_off = 20 },
+	},
+};
+
+static const struct dpu_ctl_cfg milos_ctl[] = {
+	{
+		.name = "ctl_0", .id = CTL_0,
+		.base = 0x15000, .len = 0x1000,
+		.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 9),
+	}, {
+		.name = "ctl_1", .id = CTL_1,
+		.base = 0x16000, .len = 0x1000,
+		.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 10),
+	}, {
+		.name = "ctl_2", .id = CTL_2,
+		.base = 0x17000, .len = 0x1000,
+		.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 11),
+	}, {
+		.name = "ctl_3", .id = CTL_3,
+		.base = 0x18000, .len = 0x1000,
+		.intr_start = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR2, 12),
+	},
+};
+
+static const struct dpu_sspp_cfg milos_sspp[] = {
+	{
+		.name = "sspp_0", .id = SSPP_VIG0,
+		.base = 0x4000, .len = 0x344,
+		.features = VIG_SDM845_MASK_SDMA,
+		.sblk = &dpu_vig_sblk_qseed3_3_3,
+		.xin_id = 0,
+		.type = SSPP_TYPE_VIG,
+	}, {
+		.name = "sspp_8", .id = SSPP_DMA0,
+		.base = 0x24000, .len = 0x344,
+		.features = DMA_SDM845_MASK_SDMA,
+		.sblk = &dpu_dma_sblk,
+		.xin_id = 1,
+		.type = SSPP_TYPE_DMA,
+	}, {
+		.name = "sspp_9", .id = SSPP_DMA1,
+		.base = 0x26000, .len = 0x344,
+		.features = DMA_SDM845_MASK_SDMA,
+		.sblk = &dpu_dma_sblk,
+		.xin_id = 5,
+		.type = SSPP_TYPE_DMA,
+	}, {
+		.name = "sspp_10", .id = SSPP_DMA2,
+		.base = 0x28000, .len = 0x344,
+		.features = DMA_SDM845_MASK_SDMA,
+		.sblk = &dpu_dma_sblk,
+		.xin_id = 9,
+		.type = SSPP_TYPE_DMA,
+	},
+};
+
+static const struct dpu_lm_cfg milos_lm[] = {
+	{
+		.name = "lm_0", .id = LM_0,
+		.base = 0x44000, .len = 0x400,
+		.features = MIXER_MSM8998_MASK,
+		.sblk = &sdm845_lm_sblk,
+		.pingpong = PINGPONG_0,
+		.dspp = DSPP_0,
+	}, {
+		.name = "lm_2", .id = LM_2,
+		.base = 0x46000, .len = 0x400,
+		.features = MIXER_MSM8998_MASK,
+		.sblk = &sdm845_lm_sblk,
+		.lm_pair = LM_3,
+		.pingpong = PINGPONG_2,
+	}, {
+		.name = "lm_3", .id = LM_3,
+		.base = 0x47000, .len = 0x400,
+		.features = MIXER_MSM8998_MASK,
+		.sblk = &sdm845_lm_sblk,
+		.lm_pair = LM_2,
+		.pingpong = PINGPONG_3,
+	},
+};
+
+static const struct dpu_dspp_cfg milos_dspp[] = {
+	{
+		.name = "dspp_0", .id = DSPP_0,
+		.base = 0x54000, .len = 0x1800,
+		.sblk = &sdm845_dspp_sblk,
+	},
+};
+
+static const struct dpu_pingpong_cfg milos_pp[] = {
+	{
+		.name = "pingpong_0", .id = PINGPONG_0,
+		.base = 0x69000, .len = 0,
+		.sblk = &sc7280_pp_sblk,
+		.intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 8),
+	}, {
+		.name = "pingpong_2", .id = PINGPONG_2,
+		.base = 0x6b000, .len = 0,
+		.sblk = &sc7280_pp_sblk,
+		.merge_3d = MERGE_3D_1,
+		.intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 10),
+	}, {
+		.name = "pingpong_3", .id = PINGPONG_3,
+		.base = 0x6c000, .len = 0,
+		.sblk = &sc7280_pp_sblk,
+		.merge_3d = MERGE_3D_1,
+		.intr_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 11),
+	}, {
+		.name = "pingpong_cwb_0", .id = PINGPONG_CWB_0,
+		.base = 0x66000, .len = 0,
+		.sblk = &sc7280_pp_sblk,
+	},
+};
+
+static const struct dpu_merge_3d_cfg milos_merge_3d[] = {
+	{
+		.name = "merge_3d_1", .id = MERGE_3D_1,
+		.base = 0x4f000, .len = 0x8,
+	},
+};
+
+/*
+ * NOTE: Each display compression engine (DCE) contains dual hard
+ * slice DSC encoders so both share same base address but with
+ * its own different sub block address.
+ */
+static const struct dpu_dsc_cfg milos_dsc[] = {
+	{
+		.name = "dce_0_0", .id = DSC_0,
+		.base = 0x80000, .len = 0x6,
+		.features = BIT(DPU_DSC_NATIVE_42x_EN),
+		.sblk = &milos_dsc_sblk_0,
+	}, {
+		.name = "dce_0_1", .id = DSC_1,
+		.base = 0x80000, .len = 0x6,
+		.features = BIT(DPU_DSC_NATIVE_42x_EN),
+		.sblk = &milos_dsc_sblk_1,
+	},
+};
+
+static const struct dpu_wb_cfg milos_wb[] = {
+	{
+		.name = "wb_2", .id = WB_2,
+		.base = 0x65000, .len = 0x2c8,
+		.features = WB_SDM845_MASK,
+		.format_list = wb2_formats_rgb_yuv,
+		.num_formats = ARRAY_SIZE(wb2_formats_rgb_yuv),
+		.xin_id = 6,
+		.maxlinewidth = 4096,
+		.intr_wb_done = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 4),
+	},
+};
+
+static const struct dpu_cwb_cfg milos_cwb[] = {
+	{
+		.name = "cwb_0", .id = CWB_0,
+		.base = 0x66200, .len = 0x8,
+	},
+};
+
+static const struct dpu_intf_cfg milos_intf[] = {
+	{
+		.name = "intf_0", .id = INTF_0,
+		.base = 0x34000, .len = 0x300,
+		.type = INTF_DP,
+		.controller_id = MSM_DP_CONTROLLER_0,
+		.prog_fetch_lines_worst_case = 24,
+		.intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 24),
+		.intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 25),
+	}, {
+		.name = "intf_1", .id = INTF_1,
+		.base = 0x35000, .len = 0x300,
+		.type = INTF_DSI,
+		.controller_id = MSM_DSI_CONTROLLER_0,
+		.prog_fetch_lines_worst_case = 24,
+		.intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 26),
+		.intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 27),
+		.intr_tear_rd_ptr = DPU_IRQ_IDX(MDP_INTF1_TEAR_INTR, 2),
+	}, {
+		.name = "intf_3", .id = INTF_3,
+		.base = 0x37000, .len = 0x300,
+		.type = INTF_DP,
+		.controller_id = MSM_DP_CONTROLLER_0,	/* pair with intf_0 for DP MST */
+		.prog_fetch_lines_worst_case = 24,
+		.intr_underrun = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 30),
+		.intr_vsync = DPU_IRQ_IDX(MDP_SSPP_TOP0_INTR, 31),
+	},
+};
+
+static const struct dpu_perf_cfg milos_perf_data = {
+	.max_bw_low = 7100000,
+	.max_bw_high = 9800000,
+	.min_core_ib = 2500000,
+	.min_llcc_ib = 0,
+	.min_dram_ib = 1600000,
+	.min_prefill_lines = 40,
+	/* FIXME: lut tables */
+	.danger_lut_tbl = {0x3ffff, 0x3ffff, 0x0},
+	.safe_lut_tbl = {0xff00, 0xfff0, 0x0fff},
+	.qos_lut_tbl = {
+		{.nentry = ARRAY_SIZE(sc7180_qos_linear),
+		.entries = sc7180_qos_linear
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_macrotile),
+		.entries = sc7180_qos_macrotile
+		},
+		{.nentry = ARRAY_SIZE(sc7180_qos_nrt),
+		.entries = sc7180_qos_nrt
+		},
+		/* TODO: macrotile-qseed is different from macrotile */
+	},
+	.cdp_cfg = {
+		{.rd_enable = 1, .wr_enable = 1},
+		{.rd_enable = 1, .wr_enable = 0}
+	},
+	.clk_inefficiency_factor = 105,
+	.bw_inefficiency_factor = 120,
+};
+
+static const struct dpu_mdss_version milos_mdss_ver = {
+	.core_major_ver = 10,
+	.core_minor_ver = 2,
+};
+
+const struct dpu_mdss_cfg dpu_milos_cfg = {
+	.mdss_ver = &milos_mdss_ver,
+	.caps = &milos_dpu_caps,
+	.mdp = &milos_mdp,
+	.cdm = &dpu_cdm_5_x,
+	.ctl_count = ARRAY_SIZE(milos_ctl),
+	.ctl = milos_ctl,
+	.sspp_count = ARRAY_SIZE(milos_sspp),
+	.sspp = milos_sspp,
+	.mixer_count = ARRAY_SIZE(milos_lm),
+	.mixer = milos_lm,
+	.dspp_count = ARRAY_SIZE(milos_dspp),
+	.dspp = milos_dspp,
+	.pingpong_count = ARRAY_SIZE(milos_pp),
+	.pingpong = milos_pp,
+	.dsc_count = ARRAY_SIZE(milos_dsc),
+	.dsc = milos_dsc,
+	.merge_3d_count = ARRAY_SIZE(milos_merge_3d),
+	.merge_3d = milos_merge_3d,
+	.wb_count = ARRAY_SIZE(milos_wb),
+	.wb = milos_wb,
+	.cwb_count = ARRAY_SIZE(milos_cwb),
+	.cwb = milos_cwb,
+	.intf_count = ARRAY_SIZE(milos_intf),
+	.intf = milos_intf,
+	.vbif = &milos_vbif,
+	.perf = &milos_perf_data,
+};
+
+#endif
diff --git a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h
index b7b06e45b529b..06da1583fb1eb 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/catalog/dpu_13_0_kaanapali.h
@@ -480,7 +480,7 @@ const struct dpu_mdss_cfg dpu_kaanapali_cfg = {
 	.wb_count = ARRAY_SIZE(kaanapali_wb),
 	.wb = kaanapali_wb,
 	.cwb_count = ARRAY_SIZE(kaanapali_cwb),
-	.cwb = sm8650_cwb,
+	.cwb = kaanapali_cwb,
 	.intf_count = ARRAY_SIZE(kaanapali_intf),
 	.intf = kaanapali_intf,
 	.vbif = &sm8650_vbif,
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
index 411a6fa832b53..42d0a529b4d51 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_crtc.c
@@ -899,7 +899,7 @@ static void _dpu_crtc_setup_cp_blocks(struct drm_crtc *crtc)
 }
 
 static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
-		struct drm_atomic_state *state)
+		struct drm_atomic_commit *state)
 {
 	struct dpu_crtc_state *cstate = to_dpu_crtc_state(crtc->state);
 	struct drm_encoder *encoder;
@@ -940,7 +940,7 @@ static void dpu_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void dpu_crtc_atomic_flush(struct drm_crtc *crtc,
-		struct drm_atomic_state *state)
+		struct drm_atomic_commit *state)
 {
 	struct dpu_crtc *dpu_crtc;
 	struct drm_device *dev;
@@ -1188,7 +1188,7 @@ static void dpu_crtc_atomic_print_state(struct drm_printer *p,
 }
 
 static void dpu_crtc_disable(struct drm_crtc *crtc,
-			     struct drm_atomic_state *state)
+			     struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
 									      crtc);
@@ -1267,7 +1267,7 @@ static void dpu_crtc_disable(struct drm_crtc *crtc,
 }
 
 static void dpu_crtc_enable(struct drm_crtc *crtc,
-		struct drm_atomic_state *state)
+		struct drm_atomic_commit *state)
 {
 	struct dpu_crtc *dpu_crtc = to_dpu_crtc(crtc);
 	struct drm_encoder *encoder;
@@ -1324,7 +1324,7 @@ static bool dpu_crtc_needs_dirtyfb(struct drm_crtc_state *cstate)
 static int dpu_crtc_assign_planes(struct drm_crtc *crtc, struct drm_crtc_state *crtc_state)
 {
 	int total_planes = crtc->dev->mode_config.num_total_plane;
-	struct drm_atomic_state *state = crtc_state->state;
+	struct drm_atomic_commit *state = crtc_state->state;
 	struct dpu_global_state *global_state;
 	struct drm_plane_state **states;
 	struct drm_plane *plane;
@@ -1520,7 +1520,7 @@ int dpu_crtc_check_mode_changed(struct drm_crtc_state *old_crtc_state,
 }
 
 static int dpu_crtc_atomic_check(struct drm_crtc *crtc,
-		struct drm_atomic_state *state)
+		struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
index eba1d52211f68..778e231d49677 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c
@@ -655,7 +655,7 @@ struct drm_dsc_config *dpu_encoder_get_dsc_config(struct drm_encoder *drm_enc)
 
 void dpu_encoder_update_topology(struct drm_encoder *drm_enc,
 				 struct msm_display_topology *topology,
-				 struct drm_atomic_state *state,
+				 struct drm_atomic_commit *state,
 				 const struct drm_display_mode *adj_mode)
 {
 	struct dpu_encoder_virt *dpu_enc = to_dpu_encoder_virt(drm_enc);
@@ -716,7 +716,7 @@ void dpu_encoder_update_topology(struct drm_encoder *drm_enc,
 	}
 }
 
-bool dpu_encoder_needs_modeset(struct drm_encoder *drm_enc, struct drm_atomic_state *state)
+bool dpu_encoder_needs_modeset(struct drm_encoder *drm_enc, struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *conn_state;
@@ -1335,7 +1335,7 @@ void dpu_encoder_virt_runtime_resume(struct drm_encoder *drm_enc)
 }
 
 static void dpu_encoder_virt_atomic_enable(struct drm_encoder *drm_enc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct dpu_encoder_virt *dpu_enc = NULL;
 	int ret = 0;
@@ -1382,7 +1382,7 @@ static void dpu_encoder_virt_atomic_enable(struct drm_encoder *drm_enc,
 }
 
 static void dpu_encoder_virt_atomic_disable(struct drm_encoder *drm_enc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct dpu_encoder_virt *dpu_enc = NULL;
 	struct drm_crtc *crtc;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
index ca1ca2e51d7ea..4942097e76137 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.h
@@ -84,10 +84,10 @@ bool dpu_encoder_use_dsc_merge(struct drm_encoder *drm_enc);
 
 void dpu_encoder_update_topology(struct drm_encoder *drm_enc,
 				 struct msm_display_topology *topology,
-				 struct drm_atomic_state *state,
+				 struct drm_atomic_commit *state,
 				 const struct drm_display_mode *adj_mode);
 
-bool dpu_encoder_needs_modeset(struct drm_encoder *drm_enc, struct drm_atomic_state *state);
+bool dpu_encoder_needs_modeset(struct drm_encoder *drm_enc, struct drm_atomic_commit *state);
 
 void dpu_encoder_prepare_wb_job(struct drm_encoder *drm_enc,
 		struct drm_writeback_job *job);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
index 6e8883dbfad43..590922c4f69bf 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_formats.c
@@ -61,7 +61,7 @@ static int _dpu_format_populate_plane_sizes_ubwc(
 	bool meta = MSM_FORMAT_IS_UBWC(fmt);
 
 	if (MSM_FORMAT_IS_YUV(fmt)) {
-		unsigned int stride, sclines;
+		unsigned int stride, y_sclines, uv_sclines;
 		unsigned int y_tile_width, y_tile_height;
 		unsigned int y_meta_stride, y_meta_scanlines;
 		unsigned int uv_meta_stride, uv_meta_scanlines;
@@ -77,23 +77,25 @@ static int _dpu_format_populate_plane_sizes_ubwc(
 				y_tile_width = 32;
 			}
 
-			sclines = round_up(fb->height, 16);
+			y_sclines = round_up(fb->height, 16);
+			uv_sclines = round_up((fb->height+1)>>1, 16);
 			y_tile_height = 4;
 		} else {
 			stride = round_up(fb->width, 128);
 			y_tile_width = 32;
 
-			sclines = round_up(fb->height, 32);
+			y_sclines = round_up(fb->height, 32);
+			uv_sclines = round_up((fb->height+1)>>1, 32);
 			y_tile_height = 8;
 		}
 
 		layout->plane_pitch[0] = stride;
 		layout->plane_size[0] = round_up(layout->plane_pitch[0] *
-			sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+			y_sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
 
 		layout->plane_pitch[1] = stride;
 		layout->plane_size[1] = round_up(layout->plane_pitch[1] *
-			sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
+			uv_sclines, DPU_UBWC_PLANE_SIZE_ALIGNMENT);
 
 		if (!meta)
 			return 0;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
index bb4fd5fa4b22e..2e10add84fd71 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.c
@@ -454,6 +454,16 @@ static const struct dpu_dsc_sub_blks dsc_sblk_1 = {
 	.ctl = {.name = "ctl", .base = 0xF80, .len = 0x10},
 };
 
+static const struct dpu_dsc_sub_blks milos_dsc_sblk_0 = {
+	.enc = {.name = "enc", .base = 0x100, .len = 0x100},
+	.ctl = {.name = "ctl", .base = 0xF00, .len = 0x80},
+};
+
+static const struct dpu_dsc_sub_blks milos_dsc_sblk_1 = {
+	.enc = {.name = "enc", .base = 0x200, .len = 0x100},
+	.ctl = {.name = "ctl", .base = 0xF80, .len = 0x80},
+};
+
 static const struct dpu_dsc_sub_blks sm8750_dsc_sblk_0 = {
 	.enc = {.name = "enc", .base = 0x100, .len = 0x100},
 	.ctl = {.name = "ctl", .base = 0xF00, .len = 0x24},
@@ -513,6 +523,23 @@ static const struct dpu_vbif_dynamic_ot_cfg msm8998_ot_rdwr_cfg[] = {
 	},
 };
 
+static const struct dpu_vbif_cfg milos_vbif = {
+	.len = 0x1074,
+	.features = BIT(DPU_VBIF_QOS_REMAP),
+	.xin_halt_timeout = 0x4000,
+	.qos_rp_remap_size = 0x40,
+	.qos_rt_tbl = {
+		.npriority_lvl = ARRAY_SIZE(sdm845_rt_pri_lvl),
+		.priority_lvl = sdm845_rt_pri_lvl,
+		},
+	.qos_nrt_tbl = {
+		.npriority_lvl = ARRAY_SIZE(sdm845_nrt_pri_lvl),
+		.priority_lvl = sdm845_nrt_pri_lvl,
+		},
+	.memtype_count = 16,
+	.memtype = {3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3},
+};
+
 static const struct dpu_vbif_cfg msm8996_vbif = {
 	.len = 0x1040,
 	.default_ot_rd_limit = 32,
@@ -754,6 +781,8 @@ static const struct dpu_qos_lut_entry sc7180_qos_nrt[] = {
 #include "catalog/dpu_9_2_x1e80100.h"
 
 #include "catalog/dpu_10_0_sm8650.h"
+#include "catalog/dpu_10_2_milos.h"
+
 #include "catalog/dpu_12_0_sm8750.h"
 #include "catalog/dpu_12_2_glymur.h"
 #include "catalog/dpu_12_4_eliza.h"
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
index ba04ac24d5a9e..f45faf87333e8 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_hw_catalog.h
@@ -766,6 +766,7 @@ struct dpu_mdss_cfg {
 extern const struct dpu_mdss_cfg dpu_eliza_cfg;
 extern const struct dpu_mdss_cfg dpu_glymur_cfg;
 extern const struct dpu_mdss_cfg dpu_kaanapali_cfg;
+extern const struct dpu_mdss_cfg dpu_milos_cfg;
 extern const struct dpu_mdss_cfg dpu_msm8917_cfg;
 extern const struct dpu_mdss_cfg dpu_msm8937_cfg;
 extern const struct dpu_mdss_cfg dpu_msm8953_cfg;
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
index 35f7af4743d7e..da3556eb6ecc2 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.c
@@ -331,7 +331,7 @@ dpu_kms_get_existing_global_state(struct dpu_kms *dpu_kms)
  * This acquires the modeset lock set aside for global state, creates
  * a new duplicated private object state.
  */
-struct dpu_global_state *dpu_kms_get_global_state(struct drm_atomic_state *s)
+struct dpu_global_state *dpu_kms_get_global_state(struct drm_atomic_commit *s)
 {
 	struct msm_drm_private *priv = s->dev->dev_private;
 	struct dpu_kms *dpu_kms = to_dpu_kms(priv->kms);
@@ -449,7 +449,7 @@ static void dpu_kms_disable_commit(struct msm_kms *kms)
 	pm_runtime_put_sync(&dpu_kms->pdev->dev);
 }
 
-static int dpu_kms_check_mode_changed(struct msm_kms *kms, struct drm_atomic_state *state)
+static int dpu_kms_check_mode_changed(struct msm_kms *kms, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_crtc_state *old_crtc_state;
@@ -1483,6 +1483,7 @@ static const struct of_device_id dpu_dt_match[] = {
 	{ .compatible = "qcom,eliza-dpu", .data = &dpu_eliza_cfg, },
 	{ .compatible = "qcom,glymur-dpu", .data = &dpu_glymur_cfg, },
 	{ .compatible = "qcom,kaanapali-dpu", .data = &dpu_kaanapali_cfg, },
+	{ .compatible = "qcom,milos-dpu", .data = &dpu_milos_cfg, },
 	{ .compatible = "qcom,msm8917-mdp5", .data = &dpu_msm8917_cfg, },
 	{ .compatible = "qcom,msm8937-mdp5", .data = &dpu_msm8937_cfg, },
 	{ .compatible = "qcom,msm8953-mdp5", .data = &dpu_msm8953_cfg, },
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
index bb3393bd102eb..e39831a397b16 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_kms.h
@@ -138,7 +138,7 @@ struct dpu_global_state {
 struct dpu_global_state
 	*dpu_kms_get_existing_global_state(struct dpu_kms *dpu_kms);
 struct dpu_global_state
-	*__must_check dpu_kms_get_global_state(struct drm_atomic_state *s);
+	*__must_check dpu_kms_get_global_state(struct drm_atomic_commit *s);
 
 /**
  * Debugfs functions - extra helper functions for debugfs support
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
index 3c315d5805b81..5642f3bc35688 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.c
@@ -1049,7 +1049,7 @@ static bool dpu_plane_get_single_pipe_in_stage(struct dpu_plane_state *pstate,
 }
 
 static int dpu_plane_atomic_check_sspp(struct drm_plane *plane,
-				       struct drm_atomic_state *state,
+				       struct drm_atomic_commit *state,
 				       const struct drm_crtc_state *crtc_state)
 {
 	struct drm_plane_state *new_plane_state =
@@ -1170,7 +1170,7 @@ static int dpu_plane_try_multirect_shared(struct dpu_plane_state *pstate,
 }
 
 static int dpu_plane_atomic_check(struct drm_plane *plane,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state =
 		drm_atomic_get_plane_state(state, plane);
@@ -1250,7 +1250,7 @@ static int dpu_plane_assign_resource_in_stage(struct dpu_sw_pipe *pipe,
 
 static int dpu_plane_virtual_assign_resources(struct drm_crtc *crtc,
 					      struct dpu_global_state *global_state,
-					      struct drm_atomic_state *state,
+					      struct drm_atomic_commit *state,
 					      struct drm_plane_state *plane_state,
 					      const struct drm_crtc_state *crtc_state,
 					      struct drm_plane_state **prev_adjacent_plane_state)
@@ -1311,7 +1311,7 @@ static int dpu_plane_virtual_assign_resources(struct drm_crtc *crtc,
 
 static int dpu_plane_assign_resources(struct drm_crtc *crtc,
 				      struct dpu_global_state *global_state,
-				      struct drm_atomic_state *state,
+				      struct drm_atomic_commit *state,
 				      struct drm_plane_state *plane_state,
 				      const struct drm_crtc_state *crtc_state)
 {
@@ -1349,7 +1349,7 @@ static int dpu_plane_assign_resources(struct drm_crtc *crtc,
 }
 
 int dpu_assign_plane_resources(struct dpu_global_state *global_state,
-			       struct drm_atomic_state *state,
+			       struct drm_atomic_commit *state,
 			       struct drm_crtc *crtc,
 			       struct drm_plane_state **states,
 			       unsigned int num_planes)
@@ -1617,7 +1617,7 @@ static void _dpu_plane_atomic_disable(struct drm_plane *plane)
 }
 
 static void dpu_plane_atomic_update(struct drm_plane *plane,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct dpu_plane *pdpu = to_dpu_plane(plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
index 1ef5a041b8aca..d2e7c8c251151 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_plane.h
@@ -70,7 +70,7 @@ static inline void dpu_plane_danger_signal_ctrl(struct drm_plane *plane, bool en
 #endif
 
 int dpu_assign_plane_resources(struct dpu_global_state *global_state,
-			       struct drm_atomic_state *state,
+			       struct drm_atomic_commit *state,
 			       struct drm_crtc *crtc,
 			       struct drm_plane_state **states,
 			       unsigned int num_planes);
diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c
index 7545c0293efbd..e7b09013ae4c8 100644
--- a/drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c
+++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_writeback.c
@@ -5,6 +5,7 @@
 
 #include <drm/drm_edid.h>
 #include <drm/drm_framebuffer.h>
+#include <drm/drm_managed.h>
 
 #include "dpu_writeback.h"
 
@@ -26,7 +27,7 @@ static int dpu_wb_conn_get_modes(struct drm_connector *connector)
 }
 
 static int dpu_wb_conn_atomic_check(struct drm_connector *connector,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_writeback_connector *wb_conn = drm_connector_to_writeback(connector);
 	struct dpu_wb_connector *dpu_wb_conn = to_dpu_wb_conn(wb_conn);
@@ -125,7 +126,7 @@ int dpu_writeback_init(struct drm_device *dev, struct drm_encoder *enc,
 	struct dpu_wb_connector *dpu_wb_conn;
 	int rc = 0;
 
-	dpu_wb_conn = devm_kzalloc(dev->dev, sizeof(*dpu_wb_conn), GFP_KERNEL);
+	dpu_wb_conn = drmm_kzalloc(dev, sizeof(*dpu_wb_conn), GFP_KERNEL);
 	if (!dpu_wb_conn)
 		return -ENOMEM;
 
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c
index e8066f9fd5343..57dfce58450b3 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_crtc.c
@@ -255,7 +255,7 @@ static void mdp4_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static void mdp4_crtc_atomic_disable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
 	struct mdp4_kms *mdp4_kms = get_kms(crtc);
@@ -284,7 +284,7 @@ static void mdp4_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void mdp4_crtc_atomic_enable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
 	struct mdp4_kms *mdp4_kms = get_kms(crtc);
@@ -307,7 +307,7 @@ static void mdp4_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static int mdp4_crtc_atomic_check(struct drm_crtc *crtc,
-		struct drm_atomic_state *state)
+		struct drm_atomic_commit *state)
 {
 	struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
 	DBG("%s: check", mdp4_crtc->name);
@@ -316,14 +316,14 @@ static int mdp4_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void mdp4_crtc_atomic_begin(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
 	DBG("%s: begin", mdp4_crtc->name);
 }
 
 static void mdp4_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
 	struct drm_device *dev = crtc->dev;
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
index 809ca191e9de7..7726edb0d4ede 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_kms.c
@@ -232,13 +232,6 @@ static int mdp4_modeset_init_intf(struct mdp4_kms *mdp4_kms,
 			return PTR_ERR(connector);
 		}
 
-		ret = drm_connector_attach_encoder(connector, encoder);
-		if (ret) {
-			DRM_DEV_ERROR(dev->dev, "failed to attach LVDS connector: %d\n", ret);
-
-			return ret;
-		}
-
 		break;
 	case DRM_MODE_ENCODER_TMDS:
 		encoder = mdp4_dtv_encoder_init(dev);
diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
index 098c3b5ff2b28..9459f70ce0bad 100644
--- a/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4_plane.c
@@ -102,13 +102,13 @@ static void mdp4_plane_cleanup_fb(struct drm_plane *plane,
 
 
 static int mdp4_plane_atomic_check(struct drm_plane *plane,
-		struct drm_atomic_state *state)
+		struct drm_atomic_commit *state)
 {
 	return 0;
 }
 
 static void mdp4_plane_atomic_update(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
index b29541bc48438..4c4a897fc1eee 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_crtc.c
@@ -485,7 +485,7 @@ static u32 mdp5_crtc_get_vblank_counter(struct drm_crtc *crtc)
 }
 
 static void mdp5_crtc_atomic_disable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 	struct mdp5_crtc_state *mdp5_cstate = to_mdp5_crtc_state(crtc->state);
@@ -531,7 +531,7 @@ static void mdp5_crtc_vblank_on(struct drm_crtc *crtc)
 }
 
 static void mdp5_crtc_atomic_enable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 	struct mdp5_crtc_state *mdp5_cstate = to_mdp5_crtc_state(crtc->state);
@@ -690,7 +690,7 @@ static enum mdp_mixer_stage_id get_start_stage(struct drm_crtc *crtc,
 }
 
 static int mdp5_crtc_atomic_check(struct drm_crtc *crtc,
-		struct drm_atomic_state *state)
+		struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -787,13 +787,13 @@ static int mdp5_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void mdp5_crtc_atomic_begin(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	DBG("%s: begin", crtc->name);
 }
 
 static void mdp5_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct mdp5_crtc *mdp5_crtc = to_mdp5_crtc(crtc);
 	struct mdp5_crtc_state *mdp5_cstate = to_mdp5_crtc_state(crtc->state);
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
index 2d26b07b06f54..0a004ab9fc856 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.c
@@ -79,7 +79,7 @@ mdp5_get_existing_global_state(struct mdp5_kms *mdp5_kms)
  * This acquires the modeset lock set aside for global state, creates
  * a new duplicated private object state.
  */
-struct mdp5_global_state *mdp5_get_global_state(struct drm_atomic_state *s)
+struct mdp5_global_state *mdp5_get_global_state(struct drm_atomic_commit *s)
 {
 	struct msm_drm_private *priv = s->dev->dev_private;
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
@@ -160,7 +160,7 @@ static void mdp5_disable_commit(struct msm_kms *kms)
 	pm_runtime_put_sync(&mdp5_kms->pdev->dev);
 }
 
-static void mdp5_prepare_commit(struct msm_kms *kms, struct drm_atomic_state *state)
+static void mdp5_prepare_commit(struct msm_kms *kms, struct drm_atomic_commit *state)
 {
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
 	struct mdp5_global_state *global_state;
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h
index 36b6842dfc9c6..7bf2172fce0bd 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_kms.h
@@ -77,7 +77,7 @@ struct mdp5_kms {
 struct mdp5_global_state {
 	struct drm_private_state base;
 
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct mdp5_kms *mdp5_kms;
 
 	struct mdp5_hw_pipe_state hwpipe;
@@ -86,7 +86,7 @@ struct mdp5_global_state {
 };
 
 struct mdp5_global_state * mdp5_get_existing_global_state(struct mdp5_kms *mdp5_kms);
-struct mdp5_global_state *__must_check mdp5_get_global_state(struct drm_atomic_state *s);
+struct mdp5_global_state *__must_check mdp5_get_global_state(struct drm_atomic_commit *s);
 
 /* Atomic plane state.  Subclasses the base drm_plane_state in order to
  * track assigned hwpipe and hw specific state.
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c
index 2822b533f8077..253b29629ed70 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.c
@@ -35,7 +35,7 @@ static int get_right_pair_idx(struct mdp5_kms *mdp5_kms, int lm)
 	return -1;
 }
 
-int mdp5_mixer_assign(struct drm_atomic_state *s, struct drm_crtc *crtc,
+int mdp5_mixer_assign(struct drm_atomic_commit *s, struct drm_crtc *crtc,
 		      uint32_t caps, struct mdp5_hw_mixer **mixer,
 		      struct mdp5_hw_mixer **r_mixer)
 {
@@ -116,7 +116,7 @@ int mdp5_mixer_assign(struct drm_atomic_state *s, struct drm_crtc *crtc,
 	return 0;
 }
 
-int mdp5_mixer_release(struct drm_atomic_state *s, struct mdp5_hw_mixer *mixer)
+int mdp5_mixer_release(struct drm_atomic_commit *s, struct mdp5_hw_mixer *mixer)
 {
 	struct mdp5_global_state *global_state = mdp5_get_global_state(s);
 	struct mdp5_hw_mixer_state *new_state;
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.h
index 2bedd75835bcd..46bca461b2d05 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.h
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_mixer.h
@@ -27,10 +27,10 @@ struct mdp5_hw_mixer_state {
 
 struct mdp5_hw_mixer *mdp5_mixer_init(struct drm_device *dev,
 				      const struct mdp5_lm_instance *lm);
-int mdp5_mixer_assign(struct drm_atomic_state *s, struct drm_crtc *crtc,
+int mdp5_mixer_assign(struct drm_atomic_commit *s, struct drm_crtc *crtc,
 		      uint32_t caps, struct mdp5_hw_mixer **mixer,
 		      struct mdp5_hw_mixer **r_mixer);
-int mdp5_mixer_release(struct drm_atomic_state *s,
+int mdp5_mixer_release(struct drm_atomic_commit *s,
 		       struct mdp5_hw_mixer *mixer);
 
 #endif /* __MDP5_LM_H__ */
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c
index 99b2c30b1d486..606e18973ea4b 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.c
@@ -6,7 +6,7 @@
 
 #include "mdp5_kms.h"
 
-int mdp5_pipe_assign(struct drm_atomic_state *s, struct drm_plane *plane,
+int mdp5_pipe_assign(struct drm_atomic_commit *s, struct drm_plane *plane,
 		     uint32_t caps, uint32_t blkcfg,
 		     struct mdp5_hw_pipe **hwpipe,
 		     struct mdp5_hw_pipe **r_hwpipe)
@@ -119,7 +119,7 @@ int mdp5_pipe_assign(struct drm_atomic_state *s, struct drm_plane *plane,
 	return 0;
 }
 
-int mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe)
+int mdp5_pipe_release(struct drm_atomic_commit *s, struct mdp5_hw_pipe *hwpipe)
 {
 	struct msm_drm_private *priv = s->dev->dev_private;
 	struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(priv->kms));
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.h
index 452138821f600..37485d2fdebaf 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.h
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_pipe.h
@@ -33,11 +33,11 @@ struct mdp5_hw_pipe_state {
 	struct drm_plane *hwpipe_to_plane[SSPP_MAX];
 };
 
-int mdp5_pipe_assign(struct drm_atomic_state *s, struct drm_plane *plane,
+int mdp5_pipe_assign(struct drm_atomic_commit *s, struct drm_plane *plane,
 		     uint32_t caps, uint32_t blkcfg,
 		     struct mdp5_hw_pipe **hwpipe,
 		     struct mdp5_hw_pipe **r_hwpipe);
-int mdp5_pipe_release(struct drm_atomic_state *s, struct mdp5_hw_pipe *hwpipe);
+int mdp5_pipe_release(struct drm_atomic_commit *s, struct mdp5_hw_pipe *hwpipe);
 
 struct mdp5_hw_pipe *mdp5_pipe_init(struct drm_device *dev,
 		enum mdp5_pipe pipe,
diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
index c4624a49b32f3..841f444a8d68d 100644
--- a/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5_plane.c
@@ -323,7 +323,7 @@ static int mdp5_plane_atomic_check_with_state(struct drm_crtc_state *crtc_state,
 }
 
 static int mdp5_plane_atomic_check(struct drm_plane *plane,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state,
 										 plane);
@@ -344,7 +344,7 @@ static int mdp5_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void mdp5_plane_atomic_update(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -363,7 +363,7 @@ static void mdp5_plane_atomic_update(struct drm_plane *plane,
 }
 
 static int mdp5_plane_atomic_async_check(struct drm_plane *plane,
-					 struct drm_atomic_state *state, bool flip)
+					 struct drm_atomic_commit *state, bool flip)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -417,7 +417,7 @@ static int mdp5_plane_atomic_async_check(struct drm_plane *plane,
 }
 
 static void mdp5_plane_atomic_async_update(struct drm_plane *plane,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h
index 53bd1dcde15fb..38bde9e60e11f 100644
--- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h
+++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot.h
@@ -48,7 +48,7 @@ struct msm_disp_state {
 
 	struct list_head blocks;
 
-	struct drm_atomic_state *atomic_state;
+	struct drm_atomic_commit *atomic_state;
 
 	struct timespec64 time;
 };
diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
index 427d3ee2b8337..850246fc89e19 100644
--- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
+++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c
@@ -5,11 +5,11 @@
 
 #define pr_fmt(fmt)	"[drm:%s:%d] " fmt, __func__, __LINE__
 
-#include <generated/utsrelease.h>
+#include <linux/utsname.h>
 
 #include "msm_disp_snapshot.h"
 
-static void msm_disp_state_dump_regs(u32 **reg, u32 aligned_len, void __iomem *base_addr)
+static void msm_disp_state_dump_regs(u32 **reg, u32 len, void __iomem *base_addr)
 {
 	u32 len_padded;
 	u32 num_rows;
@@ -19,11 +19,11 @@ static void msm_disp_state_dump_regs(u32 **reg, u32 aligned_len, void __iomem *b
 	void __iomem *end_addr;
 	int i;
 
-	len_padded = aligned_len * REG_DUMP_ALIGN;
-	num_rows = aligned_len / REG_DUMP_ALIGN;
+	len_padded = round_up(len, REG_DUMP_ALIGN);
+	num_rows = DIV_ROUND_UP(len, REG_DUMP_ALIGN);
 
 	addr = base_addr;
-	end_addr = base_addr + aligned_len;
+	end_addr = base_addr + len;
 
 	*reg = kvzalloc(len_padded, GFP_KERNEL);
 	if (!*reg)
@@ -48,8 +48,8 @@ static void msm_disp_state_dump_regs(u32 **reg, u32 aligned_len, void __iomem *b
 static void msm_disp_state_print_regs(const u32 *dump_addr, u32 len,
 		void __iomem *base_addr, struct drm_printer *p)
 {
+	void __iomem *addr, *end_addr;
 	int i;
-	void __iomem *addr;
 	u32 num_rows;
 
 	if (!dump_addr) {
@@ -58,6 +58,7 @@ static void msm_disp_state_print_regs(const u32 *dump_addr, u32 len,
 	}
 
 	addr = base_addr;
+	end_addr = base_addr + len;
 	num_rows = len / REG_DUMP_ALIGN;
 
 	for (i = 0; i < num_rows; i++) {
@@ -67,6 +68,17 @@ static void msm_disp_state_print_regs(const u32 *dump_addr, u32 len,
 				dump_addr[i * 4 + 2], dump_addr[i * 4 + 3]);
 		addr += REG_DUMP_ALIGN;
 	}
+
+	if (addr != end_addr) {
+		drm_printf(p, "0x%lx : %08x",
+			   (unsigned long)(addr - base_addr),
+			   dump_addr[i * 4]);
+		if (addr + 0x4 < end_addr)
+			drm_printf(p, " %08x", dump_addr[i * 4 + 1]);
+		if (addr + 0x8 < end_addr)
+			drm_printf(p, " %08x", dump_addr[i * 4 + 2]);
+		drm_printf(p, "\n");
+	}
 }
 
 void msm_disp_state_print(struct msm_disp_state *state, struct drm_printer *p)
@@ -79,7 +91,7 @@ void msm_disp_state_print(struct msm_disp_state *state, struct drm_printer *p)
 	}
 
 	drm_printf(p, "---\n");
-	drm_printf(p, "kernel: " UTS_RELEASE "\n");
+	drm_printf(p, "kernel: %s\n", init_utsname()->release);
 	drm_printf(p, "module: " KBUILD_MODNAME "\n");
 	drm_printf(p, "dpu devcoredump\n");
 	drm_printf(p, "time: %ptSp\n", &state->time);
@@ -152,7 +164,7 @@ void msm_disp_state_free(void *data)
 	struct msm_disp_state_block *block, *tmp;
 
 	if (disp_state->atomic_state) {
-		drm_atomic_state_put(disp_state->atomic_state);
+		drm_atomic_commit_put(disp_state->atomic_state);
 		disp_state->atomic_state = NULL;
 	}
 
@@ -185,7 +197,7 @@ void msm_disp_snapshot_add_block(struct msm_disp_state *disp_state, u32 len,
 	va_end(va);
 
 	INIT_LIST_HEAD(&new_blk->node);
-	new_blk->size = ALIGN(len, REG_DUMP_ALIGN);
+	new_blk->size = len;
 	new_blk->base_addr = base_addr;
 
 	msm_disp_state_dump_regs(&new_blk->state, new_blk->size, base_addr);
diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.c b/drivers/gpu/drm/msm/dp/dp_ctrl.c
index cba8a71a2561c..86ef8c89ad443 100644
--- a/drivers/gpu/drm/msm/dp/dp_ctrl.c
+++ b/drivers/gpu/drm/msm/dp/dp_ctrl.c
@@ -2583,22 +2583,6 @@ void msm_dp_ctrl_off_link_stream(struct msm_dp_ctrl *msm_dp_ctrl)
 	phy_init(phy);
 }
 
-void msm_dp_ctrl_off_link(struct msm_dp_ctrl *msm_dp_ctrl)
-{
-	struct msm_dp_ctrl_private *ctrl;
-	struct phy *phy;
-
-	ctrl = container_of(msm_dp_ctrl, struct msm_dp_ctrl_private, msm_dp_ctrl);
-	phy = ctrl->phy;
-
-	msm_dp_ctrl_mainlink_disable(ctrl);
-
-	dev_pm_opp_set_rate(ctrl->dev, 0);
-	msm_dp_ctrl_link_clk_disable(&ctrl->msm_dp_ctrl);
-
-	phy_power_off(phy);
-}
-
 void msm_dp_ctrl_off(struct msm_dp_ctrl *msm_dp_ctrl)
 {
 	struct msm_dp_ctrl_private *ctrl;
diff --git a/drivers/gpu/drm/msm/dp/dp_ctrl.h b/drivers/gpu/drm/msm/dp/dp_ctrl.h
index 124b9b21bb7f2..f68bee62713f1 100644
--- a/drivers/gpu/drm/msm/dp/dp_ctrl.h
+++ b/drivers/gpu/drm/msm/dp/dp_ctrl.h
@@ -19,7 +19,6 @@ struct phy;
 int msm_dp_ctrl_on_link(struct msm_dp_ctrl *msm_dp_ctrl);
 int msm_dp_ctrl_on_stream(struct msm_dp_ctrl *msm_dp_ctrl, bool force_link_train);
 void msm_dp_ctrl_off_link_stream(struct msm_dp_ctrl *msm_dp_ctrl);
-void msm_dp_ctrl_off_link(struct msm_dp_ctrl *msm_dp_ctrl);
 void msm_dp_ctrl_off(struct msm_dp_ctrl *msm_dp_ctrl);
 void msm_dp_ctrl_push_idle(struct msm_dp_ctrl *msm_dp_ctrl);
 irqreturn_t msm_dp_ctrl_isr(struct msm_dp_ctrl *msm_dp_ctrl);
diff --git a/drivers/gpu/drm/msm/dp/dp_display.c b/drivers/gpu/drm/msm/dp/dp_display.c
index d2124d6254855..fee1cc73d5af1 100644
--- a/drivers/gpu/drm/msm/dp/dp_display.c
+++ b/drivers/gpu/drm/msm/dp/dp_display.c
@@ -38,40 +38,9 @@ enum {
 	ISR_DISCONNECTED,
 	ISR_CONNECT_PENDING,
 	ISR_CONNECTED,
-	ISR_HPD_REPLUG_COUNT,
+	ISR_HPD_IO_GLITCH_COUNT,
 	ISR_IRQ_HPD_PULSE_COUNT,
-	ISR_HPD_LO_GLITH_COUNT,
-};
-
-/* event thread connection state */
-enum {
-	ST_DISCONNECTED,
-	ST_MAINLINK_READY,
-	ST_CONNECTED,
-	ST_DISCONNECT_PENDING,
-	ST_DISPLAY_OFF,
-};
-
-enum {
-	EV_NO_EVENT,
-	/* hpd events */
-	EV_HPD_PLUG_INT,
-	EV_IRQ_HPD_INT,
-	EV_HPD_UNPLUG_INT,
-	EV_USER_NOTIFICATION,
-};
-
-#define EVENT_TIMEOUT	(HZ/10)	/* 100ms */
-#define DP_EVENT_Q_MAX	8
-
-#define DP_TIMEOUT_NONE		0
-
-#define WAIT_FOR_RESUME_TIMEOUT_JIFFIES (HZ / 2)
-
-struct msm_dp_event {
-	u32 event_id;
-	u32 data;
-	u32 delay;
+	ISR_HPD_REPLUG_COUNT,
 };
 
 struct msm_dp_display_private {
@@ -84,6 +53,9 @@ struct msm_dp_display_private {
 	bool phy_initialized;
 	bool audio_supported;
 
+	struct mutex plugged_lock;
+	bool plugged;
+
 	struct drm_device *drm_dev;
 
 	struct drm_dp_aux *aux;
@@ -97,15 +69,9 @@ struct msm_dp_display_private {
 	/* wait for audio signaling */
 	struct completion audio_comp;
 
-	/* event related only access by event thread */
-	struct mutex event_mutex;
-	wait_queue_head_t event_q;
-	u32 hpd_state;
-	u32 event_pndx;
-	u32 event_gndx;
-	struct task_struct *ev_tsk;
-	struct msm_dp_event event_list[DP_EVENT_Q_MAX];
-	spinlock_t event_lock;
+	/* HPD IRQ handling */
+	spinlock_t irq_thread_lock;
+	u32 hpd_isr_status;
 
 	bool wide_bus_supported;
 
@@ -219,60 +185,6 @@ static struct msm_dp_display_private *dev_get_dp_display_private(struct device *
 	return container_of(dp, struct msm_dp_display_private, msm_dp_display);
 }
 
-static int msm_dp_add_event(struct msm_dp_display_private *msm_dp_priv, u32 event,
-						u32 data, u32 delay)
-{
-	unsigned long flag;
-	struct msm_dp_event *todo;
-	int pndx;
-
-	spin_lock_irqsave(&msm_dp_priv->event_lock, flag);
-	pndx = msm_dp_priv->event_pndx + 1;
-	pndx %= DP_EVENT_Q_MAX;
-	if (pndx == msm_dp_priv->event_gndx) {
-		pr_err("event_q is full: pndx=%d gndx=%d\n",
-			msm_dp_priv->event_pndx, msm_dp_priv->event_gndx);
-		spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag);
-		return -EPERM;
-	}
-	todo = &msm_dp_priv->event_list[msm_dp_priv->event_pndx++];
-	msm_dp_priv->event_pndx %= DP_EVENT_Q_MAX;
-	todo->event_id = event;
-	todo->data = data;
-	todo->delay = delay;
-	wake_up(&msm_dp_priv->event_q);
-	spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag);
-
-	return 0;
-}
-
-static int msm_dp_del_event(struct msm_dp_display_private *msm_dp_priv, u32 event)
-{
-	unsigned long flag;
-	struct msm_dp_event *todo;
-	u32	gndx;
-
-	spin_lock_irqsave(&msm_dp_priv->event_lock, flag);
-	if (msm_dp_priv->event_pndx == msm_dp_priv->event_gndx) {
-		spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag);
-		return -ENOENT;
-	}
-
-	gndx = msm_dp_priv->event_gndx;
-	while (msm_dp_priv->event_pndx != gndx) {
-		todo = &msm_dp_priv->event_list[gndx];
-		if (todo->event_id == event) {
-			todo->event_id = EV_NO_EVENT;	/* deleted */
-			todo->delay = 0;
-		}
-		gndx++;
-		gndx %= DP_EVENT_Q_MAX;
-	}
-	spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag);
-
-	return 0;
-}
-
 void msm_dp_display_signal_audio_start(struct msm_dp *msm_dp_display)
 {
 	struct msm_dp_display_private *dp;
@@ -291,8 +203,6 @@ void msm_dp_display_signal_audio_complete(struct msm_dp *msm_dp_display)
 	complete_all(&dp->audio_comp);
 }
 
-static int msm_dp_hpd_event_thread_start(struct msm_dp_display_private *msm_dp_priv);
-
 static int msm_dp_display_bind(struct device *dev, struct device *master,
 			   void *data)
 {
@@ -312,12 +222,6 @@ static int msm_dp_display_bind(struct device *dev, struct device *master,
 		goto end;
 	}
 
-	rc = msm_dp_hpd_event_thread_start(dp);
-	if (rc) {
-		DRM_ERROR("Event thread create failed\n");
-		goto end;
-	}
-
 	return 0;
 end:
 	return rc;
@@ -329,8 +233,6 @@ static void msm_dp_display_unbind(struct device *dev, struct device *master,
 	struct msm_dp_display_private *dp = dev_get_dp_display_private(dev);
 	struct msm_drm_private *priv = dev_get_drvdata(master);
 
-	kthread_stop(dp->ev_tsk);
-
 	of_dp_aux_depopulate_bus(dp->aux);
 
 	msm_dp_aux_unregister(dp->aux);
@@ -344,45 +246,6 @@ static const struct component_ops msm_dp_display_comp_ops = {
 	.unbind = msm_dp_display_unbind,
 };
 
-static void msm_dp_display_send_hpd_event(struct msm_dp *msm_dp_display)
-{
-	struct msm_dp_display_private *dp;
-	struct drm_connector *connector;
-
-	dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display);
-
-	connector = dp->msm_dp_display.connector;
-	drm_helper_hpd_irq_event(connector->dev);
-}
-
-static int msm_dp_display_send_hpd_notification(struct msm_dp_display_private *dp,
-					    bool hpd)
-{
-	if ((hpd && dp->msm_dp_display.link_ready) ||
-			(!hpd && !dp->msm_dp_display.link_ready)) {
-		drm_dbg_dp(dp->drm_dev, "HPD already %s\n", str_on_off(hpd));
-		return 0;
-	}
-
-	/* reset video pattern flag on disconnect */
-	if (!hpd) {
-		dp->panel->video_test = false;
-		if (!dp->msm_dp_display.is_edp)
-			drm_dp_set_subconnector_property(dp->msm_dp_display.connector,
-							 connector_status_disconnected,
-							 dp->panel->dpcd,
-							 dp->panel->downstream_ports);
-	}
-
-	dp->msm_dp_display.link_ready = hpd;
-
-	drm_dbg_dp(dp->drm_dev, "type=%d hpd=%d\n",
-			dp->msm_dp_display.connector_type, hpd);
-	msm_dp_display_send_hpd_event(&dp->msm_dp_display);
-
-	return 0;
-}
-
 static int msm_dp_display_lttpr_init(struct msm_dp_display_private *dp, u8 *dpcd)
 {
 	int rc, lttpr_count;
@@ -437,19 +300,12 @@ static int msm_dp_display_process_hpd_high(struct msm_dp_display_private *dp)
 	msm_dp_link_psm_config(dp->link, &dp->panel->link_info, false);
 
 	msm_dp_link_reset_phy_params_vx_px(dp->link);
-	rc = msm_dp_ctrl_on_link(dp->ctrl);
-	if (rc) {
-		DRM_ERROR("failed to complete DP link training\n");
-		goto end;
-	}
-
-	msm_dp_add_event(dp, EV_USER_NOTIFICATION, true, 0);
 
 end:
 	return rc;
 }
 
-static void msm_dp_display_host_phy_init(struct msm_dp_display_private *dp)
+static bool msm_dp_display_host_phy_init(struct msm_dp_display_private *dp)
 {
 	drm_dbg_dp(dp->drm_dev, "type=%d core_init=%d phy_init=%d\n",
 		dp->msm_dp_display.connector_type, dp->core_initialized,
@@ -458,7 +314,10 @@ static void msm_dp_display_host_phy_init(struct msm_dp_display_private *dp)
 	if (!dp->phy_initialized) {
 		msm_dp_ctrl_phy_init(dp->ctrl);
 		dp->phy_initialized = true;
+		return true;
 	}
+
+	return false;
 }
 
 static void msm_dp_display_host_phy_exit(struct msm_dp_display_private *dp)
@@ -499,24 +358,6 @@ static void msm_dp_display_host_deinit(struct msm_dp_display_private *dp)
 	dp->core_initialized = false;
 }
 
-static int msm_dp_display_usbpd_configure_cb(struct device *dev)
-{
-	struct msm_dp_display_private *dp = dev_get_dp_display_private(dev);
-
-	msm_dp_display_host_phy_init(dp);
-
-	return msm_dp_display_process_hpd_high(dp);
-}
-
-static int msm_dp_display_notify_disconnect(struct device *dev)
-{
-	struct msm_dp_display_private *dp = dev_get_dp_display_private(dev);
-
-	msm_dp_add_event(dp, EV_USER_NOTIFICATION, false, 0);
-
-	return 0;
-}
-
 static void msm_dp_display_handle_video_request(struct msm_dp_display_private *dp)
 {
 	if (dp->link->sink_request & DP_TEST_LINK_VIDEO_PATTERN) {
@@ -525,41 +366,11 @@ static void msm_dp_display_handle_video_request(struct msm_dp_display_private *d
 	}
 }
 
-static int msm_dp_display_handle_port_status_changed(struct msm_dp_display_private *dp)
-{
-	int rc = 0;
-
-	if (drm_dp_is_branch(dp->panel->dpcd) && dp->link->sink_count == 0) {
-		drm_dbg_dp(dp->drm_dev, "sink count is zero, nothing to do\n");
-		if (dp->hpd_state != ST_DISCONNECTED) {
-			dp->hpd_state = ST_DISCONNECT_PENDING;
-			msm_dp_add_event(dp, EV_USER_NOTIFICATION, false, 0);
-		}
-	} else {
-		if (dp->hpd_state == ST_DISCONNECTED) {
-			dp->hpd_state = ST_MAINLINK_READY;
-			rc = msm_dp_display_process_hpd_high(dp);
-			if (rc)
-				dp->hpd_state = ST_DISCONNECTED;
-		}
-	}
-
-	return rc;
-}
-
 static int msm_dp_display_handle_irq_hpd(struct msm_dp_display_private *dp)
 {
 	u32 sink_request = dp->link->sink_request;
 
 	drm_dbg_dp(dp->drm_dev, "%d\n", sink_request);
-	if (dp->hpd_state == ST_DISCONNECTED) {
-		if (sink_request & DP_LINK_STATUS_UPDATED) {
-			drm_dbg_dp(dp->drm_dev, "Disconnected sink_request: %d\n",
-							sink_request);
-			DRM_ERROR("Disconnected, no DP_LINK_STATUS_UPDATED\n");
-			return -EINVAL;
-		}
-	}
 
 	msm_dp_ctrl_handle_sink_request(dp->ctrl);
 
@@ -569,79 +380,39 @@ static int msm_dp_display_handle_irq_hpd(struct msm_dp_display_private *dp)
 	return 0;
 }
 
-static int msm_dp_display_usbpd_attention_cb(struct device *dev)
+static int msm_dp_hpd_plug_handle(struct msm_dp_display_private *dp)
 {
-	int rc = 0;
-	u32 sink_request;
-	struct msm_dp_display_private *dp = dev_get_dp_display_private(dev);
-
-	/* check for any test request issued by sink */
-	rc = msm_dp_link_process_request(dp->link);
-	if (!rc) {
-		sink_request = dp->link->sink_request;
-		drm_dbg_dp(dp->drm_dev, "hpd_state=%d sink_request=%d\n",
-					dp->hpd_state, sink_request);
-		if (sink_request & DS_PORT_STATUS_CHANGED)
-			rc = msm_dp_display_handle_port_status_changed(dp);
-		else
-			rc = msm_dp_display_handle_irq_hpd(dp);
-	}
-
-	return rc;
-}
-
-static int msm_dp_hpd_plug_handle(struct msm_dp_display_private *dp, u32 data)
-{
-	u32 state;
 	int ret;
 	struct platform_device *pdev = dp->msm_dp_display.pdev;
 
-	msm_dp_aux_enable_xfers(dp->aux, true);
-
-	mutex_lock(&dp->event_mutex);
-
-	state =  dp->hpd_state;
-	drm_dbg_dp(dp->drm_dev, "Before, type=%d hpd_state=%d\n",
-			dp->msm_dp_display.connector_type, state);
+	drm_dbg_dp(dp->drm_dev, "Before, type=%d sink_count=%d\n",
+			dp->msm_dp_display.connector_type,
+			dp->link->sink_count);
 
-	if (state == ST_DISPLAY_OFF) {
-		mutex_unlock(&dp->event_mutex);
-		return 0;
-	}
-
-	if (state == ST_MAINLINK_READY || state == ST_CONNECTED) {
-		mutex_unlock(&dp->event_mutex);
-		return 0;
-	}
-
-	if (state == ST_DISCONNECT_PENDING) {
-		/* wait until ST_DISCONNECTED */
-		msm_dp_add_event(dp, EV_HPD_PLUG_INT, 0, 1); /* delay = 1 */
-		mutex_unlock(&dp->event_mutex);
-		return 0;
-	}
+	mutex_lock(&dp->plugged_lock);
 
 	ret = pm_runtime_resume_and_get(&pdev->dev);
 	if (ret) {
+		mutex_unlock(&dp->plugged_lock);
 		DRM_ERROR("failed to pm_runtime_resume\n");
-		mutex_unlock(&dp->event_mutex);
 		return ret;
 	}
 
-	ret = msm_dp_display_usbpd_configure_cb(&pdev->dev);
-	if (ret) {	/* link train failed */
-		dp->hpd_state = ST_DISCONNECTED;
-		pm_runtime_put_sync(&pdev->dev);
-	} else {
-		dp->hpd_state = ST_MAINLINK_READY;
-	}
+	msm_dp_aux_enable_xfers(dp->aux, true);
 
-	drm_dbg_dp(dp->drm_dev, "After, type=%d hpd_state=%d\n",
-			dp->msm_dp_display.connector_type, state);
-	mutex_unlock(&dp->event_mutex);
+	msm_dp_display_host_phy_init(dp);
 
-	/* uevent will complete connection part */
-	return 0;
+	ret = msm_dp_display_process_hpd_high(dp);
+
+	drm_dbg_dp(dp->drm_dev, "After, type=%d sink_count=%d\n",
+			dp->msm_dp_display.connector_type,
+			dp->link->sink_count);
+
+	dp->plugged = true;
+
+	mutex_unlock(&dp->plugged_lock);
+
+	return ret;
 };
 
 static void msm_dp_display_handle_plugged_change(struct msm_dp *msm_dp_display,
@@ -658,99 +429,85 @@ static void msm_dp_display_handle_plugged_change(struct msm_dp *msm_dp_display,
 							plugged);
 }
 
-static int msm_dp_hpd_unplug_handle(struct msm_dp_display_private *dp, u32 data)
+static int msm_dp_hpd_unplug_handle(struct msm_dp_display_private *dp)
 {
-	u32 state;
 	struct platform_device *pdev = dp->msm_dp_display.pdev;
 
+	dp->panel->video_test = false;
+
 	msm_dp_aux_enable_xfers(dp->aux, false);
 
-	mutex_lock(&dp->event_mutex);
+	drm_dbg_dp(dp->drm_dev, "Before, type=%d sink_count=%d\n",
+			dp->msm_dp_display.connector_type,
+			dp->link->sink_count);
 
-	state = dp->hpd_state;
+	mutex_lock(&dp->plugged_lock);
+	if (!dp->plugged) {
+		mutex_unlock(&dp->plugged_lock);
 
-	drm_dbg_dp(dp->drm_dev, "Before, type=%d hpd_state=%d\n",
-			dp->msm_dp_display.connector_type, state);
+		return 0;
+	}
 
-	/* unplugged, no more irq_hpd handle */
-	msm_dp_del_event(dp, EV_IRQ_HPD_INT);
+	/* Don't forget modes for eDP */
+	if (!dp->msm_dp_display.is_edp)
+		msm_dp_panel_unplugged(dp->panel, dp->msm_dp_display.connector);
 
-	if (state == ST_DISCONNECTED) {
-		/* triggered by irq_hdp with sink_count = 0 */
-		if (dp->link->sink_count == 0) {
-			msm_dp_display_host_phy_exit(dp);
-		}
-		msm_dp_display_notify_disconnect(&dp->msm_dp_display.pdev->dev);
-		mutex_unlock(&dp->event_mutex);
-		return 0;
-	} else if (state == ST_DISCONNECT_PENDING) {
-		mutex_unlock(&dp->event_mutex);
-		return 0;
-	} else if (state == ST_MAINLINK_READY) {
-		msm_dp_ctrl_off_link(dp->ctrl);
+	/* triggered by irq_hdp with sink_count = 0 */
+	if (dp->link->sink_count == 0)
 		msm_dp_display_host_phy_exit(dp);
-		dp->hpd_state = ST_DISCONNECTED;
-		msm_dp_display_notify_disconnect(&dp->msm_dp_display.pdev->dev);
-		pm_runtime_put_sync(&pdev->dev);
-		mutex_unlock(&dp->event_mutex);
-		return 0;
-	}
 
 	/*
 	 * We don't need separate work for disconnect as
 	 * connect/attention interrupts are disabled
 	 */
-	msm_dp_display_notify_disconnect(&dp->msm_dp_display.pdev->dev);
-
-	if (state == ST_DISPLAY_OFF) {
-		dp->hpd_state = ST_DISCONNECTED;
-	} else {
-		dp->hpd_state = ST_DISCONNECT_PENDING;
-	}
+	if (!dp->msm_dp_display.is_edp)
+		drm_dp_set_subconnector_property(dp->msm_dp_display.connector,
+						 connector_status_disconnected,
+						 dp->panel->dpcd,
+						 dp->panel->downstream_ports);
 
 	/* signal the disconnect event early to ensure proper teardown */
 	msm_dp_display_handle_plugged_change(&dp->msm_dp_display, false);
 
-	drm_dbg_dp(dp->drm_dev, "After, type=%d hpd_state=%d\n",
-			dp->msm_dp_display.connector_type, state);
+	drm_dbg_dp(dp->drm_dev, "After, type=%d, sink_count=%d\n",
+			dp->msm_dp_display.connector_type,
+			dp->link->sink_count);
+
+	if (dp->plugged) {
+		pm_runtime_put_sync(&pdev->dev);
+		dp->plugged = false;
+	}
+	mutex_unlock(&dp->plugged_lock);
 
-	/* uevent will complete disconnection part */
-	pm_runtime_put_sync(&pdev->dev);
-	mutex_unlock(&dp->event_mutex);
 	return 0;
 }
 
-static int msm_dp_irq_hpd_handle(struct msm_dp_display_private *dp, u32 data)
+static int msm_dp_irq_hpd_handle(struct msm_dp_display_private *dp)
 {
-	u32 state;
-
-	mutex_lock(&dp->event_mutex);
+	u32 sink_request;
+	int rc = 0;
 
 	/* irq_hpd can happen at either connected or disconnected state */
-	state =  dp->hpd_state;
-	drm_dbg_dp(dp->drm_dev, "Before, type=%d hpd_state=%d\n",
-			dp->msm_dp_display.connector_type, state);
-
-	if (state == ST_DISPLAY_OFF) {
-		mutex_unlock(&dp->event_mutex);
-		return 0;
-	}
+	drm_dbg_dp(dp->drm_dev, "Before, type=%d, sink_count=%d\n",
+			dp->msm_dp_display.connector_type,
+			dp->link->sink_count);
 
-	if (state == ST_MAINLINK_READY || state == ST_DISCONNECT_PENDING) {
-		/* wait until ST_CONNECTED */
-		msm_dp_add_event(dp, EV_IRQ_HPD_INT, 0, 1); /* delay = 1 */
-		mutex_unlock(&dp->event_mutex);
-		return 0;
+	/* check for any test request issued by sink */
+	rc = msm_dp_link_process_request(dp->link);
+	if (!rc) {
+		sink_request = dp->link->sink_request;
+		drm_dbg_dp(dp->drm_dev, "sink_request=%d\n", sink_request);
+		if (sink_request & DS_PORT_STATUS_CHANGED)
+			rc = msm_dp_display_process_hpd_high(dp);
+		else
+			rc = msm_dp_display_handle_irq_hpd(dp);
 	}
 
-	msm_dp_display_usbpd_attention_cb(&dp->msm_dp_display.pdev->dev);
-
-	drm_dbg_dp(dp->drm_dev, "After, type=%d hpd_state=%d\n",
-			dp->msm_dp_display.connector_type, state);
-
-	mutex_unlock(&dp->event_mutex);
+	drm_dbg_dp(dp->drm_dev, "After, type=%d, sink_count=%d\n",
+			dp->msm_dp_display.connector_type,
+			dp->link->sink_count);
 
-	return 0;
+	return rc;
 }
 
 static void msm_dp_display_deinit_sub_modules(struct msm_dp_display_private *dp)
@@ -1026,12 +783,8 @@ void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm_dp *dp)
 	 * power_on status before dumping DP registers to avoid crash due
 	 * to unclocked access
 	 */
-	mutex_lock(&msm_dp_display->event_mutex);
-
-	if (!dp->power_on) {
-		mutex_unlock(&msm_dp_display->event_mutex);
+	if (!dp->power_on)
 		return;
-	}
 
 	msm_disp_snapshot_add_block(disp_state, msm_dp_display->ahb_len,
 				    msm_dp_display->ahb_base, "dp_ahb");
@@ -1041,8 +794,6 @@ void msm_dp_snapshot(struct msm_disp_state *disp_state, struct msm_dp *dp)
 				    msm_dp_display->link_base, "dp_link");
 	msm_disp_snapshot_add_block(disp_state, msm_dp_display->p0_len,
 				    msm_dp_display->p0_base, "dp_p0");
-
-	mutex_unlock(&msm_dp_display->event_mutex);
 }
 
 void msm_dp_display_set_psr(struct msm_dp *msm_dp_display, bool enter)
@@ -1058,133 +809,110 @@ void msm_dp_display_set_psr(struct msm_dp *msm_dp_display, bool enter)
 	msm_dp_ctrl_set_psr(dp->ctrl, enter);
 }
 
-static int hpd_event_thread(void *data)
+/**
+ * msm_dp_bridge_detect - callback to determine if connector is connected
+ * @bridge: Pointer to drm bridge structure
+ * @connector: Pointer to drm connector structure
+ * Returns: Bridge's 'is connected' status
+ */
+enum drm_connector_status msm_dp_bridge_detect(struct drm_bridge *bridge,
+					       struct drm_connector *connector)
 {
-	struct msm_dp_display_private *msm_dp_priv;
-	unsigned long flag;
-	struct msm_dp_event *todo;
-	int timeout_mode = 0;
+	struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(bridge);
+	struct msm_dp *dp = msm_dp_bridge->msm_dp_display;
+	struct msm_dp_display_private *priv;
+	int ret = 0;
+	int status = connector_status_disconnected;
+	u8 dpcd[DP_RECEIVER_CAP_SIZE];
+	struct drm_dp_desc desc;
+	bool phy_deinit;
 
-	msm_dp_priv = (struct msm_dp_display_private *)data;
+	dp = to_dp_bridge(bridge)->msm_dp_display;
 
-	while (1) {
-		if (timeout_mode) {
-			wait_event_timeout(msm_dp_priv->event_q,
-				(msm_dp_priv->event_pndx == msm_dp_priv->event_gndx) ||
-					kthread_should_stop(), EVENT_TIMEOUT);
-		} else {
-			wait_event_interruptible(msm_dp_priv->event_q,
-				(msm_dp_priv->event_pndx != msm_dp_priv->event_gndx) ||
-					kthread_should_stop());
-		}
+	priv = container_of(dp, struct msm_dp_display_private, msm_dp_display);
 
-		if (kthread_should_stop())
-			break;
+	mutex_lock(&priv->plugged_lock);
+	ret = pm_runtime_resume_and_get(&dp->pdev->dev);
+	if (ret) {
+		DRM_ERROR("failed to pm_runtime_resume\n");
+		mutex_unlock(&priv->plugged_lock);
+		return status;
+	}
 
-		spin_lock_irqsave(&msm_dp_priv->event_lock, flag);
-		todo = &msm_dp_priv->event_list[msm_dp_priv->event_gndx];
-		if (todo->delay) {
-			struct msm_dp_event *todo_next;
+	phy_deinit = msm_dp_display_host_phy_init(priv);
 
-			msm_dp_priv->event_gndx++;
-			msm_dp_priv->event_gndx %= DP_EVENT_Q_MAX;
+	msm_dp_aux_enable_xfers(priv->aux, true);
 
-			/* re enter delay event into q */
-			todo_next = &msm_dp_priv->event_list[msm_dp_priv->event_pndx++];
-			msm_dp_priv->event_pndx %= DP_EVENT_Q_MAX;
-			todo_next->event_id = todo->event_id;
-			todo_next->data = todo->data;
-			todo_next->delay = todo->delay - 1;
+	ret = msm_dp_aux_is_link_connected(priv->aux);
+	DRM_DEBUG_DP("aux link status: %x\n", ret);
+	if (!priv->plugged && !ret) {
+		DRM_DEBUG_DP("aux not connected\n");
+		priv->plugged = false;
+		goto end;
+	}
 
-			/* clean up older event */
-			todo->event_id = EV_NO_EVENT;
-			todo->delay = 0;
+	ret = drm_dp_read_dpcd_caps(priv->aux, dpcd);
+	if (ret) {
+		DRM_DEBUG_DP("failed to read caps\n");
+		priv->plugged = false;
+		goto end;
+	}
 
-			/* switch to timeout mode */
-			timeout_mode = 1;
-			spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag);
-			continue;
-		}
+	ret = drm_dp_read_desc(priv->aux, &desc, drm_dp_is_branch(dpcd));
+	if (ret) {
+		DRM_DEBUG_DP("failed to read desc\n");
+		priv->plugged = false;
+		goto end;
+	}
 
-		/* timeout with no events in q */
-		if (msm_dp_priv->event_pndx == msm_dp_priv->event_gndx) {
-			spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag);
-			continue;
-		}
+	status = connector_status_connected;
+	priv->plugged = true;
 
-		msm_dp_priv->event_gndx++;
-		msm_dp_priv->event_gndx %= DP_EVENT_Q_MAX;
-		timeout_mode = 0;
-		spin_unlock_irqrestore(&msm_dp_priv->event_lock, flag);
-
-		switch (todo->event_id) {
-		case EV_HPD_PLUG_INT:
-			msm_dp_hpd_plug_handle(msm_dp_priv, todo->data);
-			break;
-		case EV_HPD_UNPLUG_INT:
-			msm_dp_hpd_unplug_handle(msm_dp_priv, todo->data);
-			break;
-		case EV_IRQ_HPD_INT:
-			msm_dp_irq_hpd_handle(msm_dp_priv, todo->data);
-			break;
-		case EV_USER_NOTIFICATION:
-			msm_dp_display_send_hpd_notification(msm_dp_priv,
-						todo->data);
-			break;
-		default:
-			break;
-		}
+	if (drm_dp_read_sink_count_cap(connector, dpcd, &desc)) {
+		int sink_count = drm_dp_read_sink_count(priv->aux);
+
+		drm_dbg_dp(dp->drm_dev, "sink_count = %d\n", sink_count);
+
+		if (sink_count <= 0)
+			status = connector_status_disconnected;
 	}
 
-	return 0;
-}
+end:
+	/*
+	 * If we detected the DPRX, leave the controller on so that it doesn't
+	 * loose the state.
+	 */
+	if (!priv->plugged) {
+		if (phy_deinit) {
+			msm_dp_aux_enable_xfers(priv->aux, false);
+			msm_dp_display_host_phy_exit(priv);
+		}
 
-static int msm_dp_hpd_event_thread_start(struct msm_dp_display_private *msm_dp_priv)
-{
-	/* set event q to empty */
-	msm_dp_priv->event_gndx = 0;
-	msm_dp_priv->event_pndx = 0;
+		pm_runtime_put_sync(&dp->pdev->dev);
+	}
 
-	msm_dp_priv->ev_tsk = kthread_run(hpd_event_thread, msm_dp_priv, "dp_hpd_handler");
-	if (IS_ERR(msm_dp_priv->ev_tsk))
-		return PTR_ERR(msm_dp_priv->ev_tsk);
+	mutex_unlock(&priv->plugged_lock);
 
-	return 0;
+	return status;
 }
 
 static irqreturn_t msm_dp_display_irq_handler(int irq, void *dev_id)
 {
 	struct msm_dp_display_private *dp = dev_id;
-	irqreturn_t ret = IRQ_NONE;
 	u32 hpd_isr_status;
-
-	if (!dp) {
-		DRM_ERROR("invalid data\n");
-		return IRQ_NONE;
-	}
+	unsigned long flags;
+	irqreturn_t ret = IRQ_HANDLED;
 
 	hpd_isr_status = msm_dp_aux_get_hpd_intr_status(dp->aux);
 
 	if (hpd_isr_status & 0x0F) {
 		drm_dbg_dp(dp->drm_dev, "type=%d isr=0x%x\n",
 			dp->msm_dp_display.connector_type, hpd_isr_status);
-		/* hpd related interrupts */
-		if (hpd_isr_status & DP_DP_HPD_PLUG_INT_MASK)
-			msm_dp_add_event(dp, EV_HPD_PLUG_INT, 0, 0);
-
-		if (hpd_isr_status & DP_DP_IRQ_HPD_INT_MASK) {
-			msm_dp_add_event(dp, EV_IRQ_HPD_INT, 0, 0);
-		}
 
-		if (hpd_isr_status & DP_DP_HPD_REPLUG_INT_MASK) {
-			msm_dp_add_event(dp, EV_HPD_UNPLUG_INT, 0, 0);
-			msm_dp_add_event(dp, EV_HPD_PLUG_INT, 0, 3);
-		}
-
-		if (hpd_isr_status & DP_DP_HPD_UNPLUG_INT_MASK)
-			msm_dp_add_event(dp, EV_HPD_UNPLUG_INT, 0, 0);
-
-		ret = IRQ_HANDLED;
+		spin_lock_irqsave(&dp->irq_thread_lock, flags);
+		dp->hpd_isr_status |= hpd_isr_status;
+		ret = IRQ_WAKE_THREAD;
+		spin_unlock_irqrestore(&dp->irq_thread_lock, flags);
 	}
 
 	/* DP controller isr */
@@ -1193,6 +921,36 @@ static irqreturn_t msm_dp_display_irq_handler(int irq, void *dev_id)
 	return ret;
 }
 
+static irqreturn_t msm_dp_display_irq_thread(int irq, void *dev_id)
+{
+	struct msm_dp_display_private *dp = dev_id;
+	irqreturn_t ret = IRQ_NONE;
+	unsigned long flags;
+	u32 hpd_isr_status;
+
+	spin_lock_irqsave(&dp->irq_thread_lock, flags);
+	hpd_isr_status = dp->hpd_isr_status;
+	dp->hpd_isr_status = 0;
+	spin_unlock_irqrestore(&dp->irq_thread_lock, flags);
+
+	if (hpd_isr_status & DP_DP_HPD_UNPLUG_INT_MASK)
+		drm_bridge_hpd_notify(dp->msm_dp_display.bridge,
+				      connector_status_disconnected);
+
+	if (hpd_isr_status & DP_DP_HPD_PLUG_INT_MASK)
+		drm_bridge_hpd_notify(dp->msm_dp_display.bridge,
+				      connector_status_connected);
+
+	/* Send HPD as connected and distinguish it in the notifier */
+	if (hpd_isr_status & DP_DP_IRQ_HPD_INT_MASK)
+		drm_bridge_hpd_notify(dp->msm_dp_display.bridge,
+				      connector_status_connected);
+
+	ret = IRQ_HANDLED;
+
+	return ret;
+}
+
 static int msm_dp_display_request_irq(struct msm_dp_display_private *dp)
 {
 	int rc = 0;
@@ -1204,9 +962,13 @@ static int msm_dp_display_request_irq(struct msm_dp_display_private *dp)
 		return dp->irq;
 	}
 
-	rc = devm_request_irq(&pdev->dev, dp->irq, msm_dp_display_irq_handler,
-			      IRQF_TRIGGER_HIGH|IRQF_NO_AUTOEN,
-			      "dp_display_isr", dp);
+	spin_lock_init(&dp->irq_thread_lock);
+	irq_set_status_flags(dp->irq, IRQ_NOAUTOEN);
+	rc = devm_request_threaded_irq(&pdev->dev, dp->irq,
+				       msm_dp_display_irq_handler,
+				       msm_dp_display_irq_thread,
+				       IRQ_TYPE_LEVEL_HIGH,
+				       "dp_display_isr", dp);
 
 	if (rc < 0) {
 		DRM_ERROR("failed to request IRQ%u: %d\n",
@@ -1386,6 +1148,9 @@ static int msm_dp_display_probe(struct platform_device *pdev)
 	dp->wide_bus_supported = desc->wide_bus_supported;
 	dp->msm_dp_display.is_edp =
 		(dp->msm_dp_display.connector_type == DRM_MODE_CONNECTOR_eDP);
+	dp->hpd_isr_status = 0;
+
+	mutex_init(&dp->plugged_lock);
 
 	rc = msm_dp_display_get_io(dp);
 	if (rc)
@@ -1397,11 +1162,6 @@ static int msm_dp_display_probe(struct platform_device *pdev)
 		return -EPROBE_DEFER;
 	}
 
-	/* setup event q */
-	mutex_init(&dp->event_mutex);
-	init_waitqueue_head(&dp->event_q);
-	spin_lock_init(&dp->event_lock);
-
 	/* Store DP audio handle inside DP display */
 	dp->msm_dp_display.msm_dp_audio = dp->audio;
 
@@ -1591,13 +1351,12 @@ int msm_dp_modeset_init(struct msm_dp *msm_dp_display, struct drm_device *dev,
 }
 
 void msm_dp_bridge_atomic_enable(struct drm_bridge *drm_bridge,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(drm_bridge);
 	struct msm_dp *dp = msm_dp_bridge->msm_dp_display;
 	int rc = 0;
 	struct msm_dp_display_private *msm_dp_display;
-	u32 hpd_state;
 	bool force_link_train = false;
 
 	msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display);
@@ -1607,35 +1366,32 @@ void msm_dp_bridge_atomic_enable(struct drm_bridge *drm_bridge,
 	}
 
 	if (dp->is_edp)
-		msm_dp_hpd_plug_handle(msm_dp_display, 0);
+		msm_dp_hpd_plug_handle(msm_dp_display);
 
-	mutex_lock(&msm_dp_display->event_mutex);
 	if (pm_runtime_resume_and_get(&dp->pdev->dev)) {
 		DRM_ERROR("failed to pm_runtime_resume\n");
-		mutex_unlock(&msm_dp_display->event_mutex);
 		return;
 	}
 
-	hpd_state = msm_dp_display->hpd_state;
-	if (hpd_state != ST_DISPLAY_OFF && hpd_state != ST_MAINLINK_READY) {
-		mutex_unlock(&msm_dp_display->event_mutex);
+	if (msm_dp_display->link->sink_count == 0)
 		return;
-	}
 
 	rc = msm_dp_display_set_mode(dp, &msm_dp_display->msm_dp_mode);
 	if (rc) {
 		DRM_ERROR("Failed to perform a mode set, rc=%d\n", rc);
-		mutex_unlock(&msm_dp_display->event_mutex);
 		return;
 	}
 
-	hpd_state =  msm_dp_display->hpd_state;
-
-	if (hpd_state == ST_DISPLAY_OFF) {
+	if (!dp->power_on) {
 		msm_dp_display_host_phy_init(msm_dp_display);
 		force_link_train = true;
 	}
 
+	rc = msm_dp_ctrl_on_link(msm_dp_display->ctrl);
+	if (rc)
+		DRM_ERROR("Failed link training (rc=%d)\n", rc);
+	// TODO: schedule drm_connector_set_link_status_property()
+
 	msm_dp_display_enable(msm_dp_display, force_link_train);
 
 	rc = msm_dp_display_post_enable(dp);
@@ -1644,15 +1400,11 @@ void msm_dp_bridge_atomic_enable(struct drm_bridge *drm_bridge,
 		msm_dp_display_disable(msm_dp_display);
 	}
 
-	/* completed connection */
-	msm_dp_display->hpd_state = ST_CONNECTED;
-
 	drm_dbg_dp(dp->drm_dev, "type=%d Done\n", dp->connector_type);
-	mutex_unlock(&msm_dp_display->event_mutex);
 }
 
 void msm_dp_bridge_atomic_disable(struct drm_bridge *drm_bridge,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(drm_bridge);
 	struct msm_dp *dp = msm_dp_bridge->msm_dp_display;
@@ -1664,39 +1416,22 @@ void msm_dp_bridge_atomic_disable(struct drm_bridge *drm_bridge,
 }
 
 void msm_dp_bridge_atomic_post_disable(struct drm_bridge *drm_bridge,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(drm_bridge);
 	struct msm_dp *dp = msm_dp_bridge->msm_dp_display;
-	u32 hpd_state;
 	struct msm_dp_display_private *msm_dp_display;
 
 	msm_dp_display = container_of(dp, struct msm_dp_display_private, msm_dp_display);
 
 	if (dp->is_edp)
-		msm_dp_hpd_unplug_handle(msm_dp_display, 0);
-
-	mutex_lock(&msm_dp_display->event_mutex);
-
-	hpd_state = msm_dp_display->hpd_state;
-	if (hpd_state != ST_DISCONNECT_PENDING && hpd_state != ST_CONNECTED)
-		drm_dbg_dp(dp->drm_dev, "type=%d wrong hpd_state=%d\n",
-			   dp->connector_type, hpd_state);
+		msm_dp_hpd_unplug_handle(msm_dp_display);
 
 	msm_dp_display_disable(msm_dp_display);
 
-	hpd_state =  msm_dp_display->hpd_state;
-	if (hpd_state == ST_DISCONNECT_PENDING) {
-		/* completed disconnection */
-		msm_dp_display->hpd_state = ST_DISCONNECTED;
-	} else {
-		msm_dp_display->hpd_state = ST_DISPLAY_OFF;
-	}
-
 	drm_dbg_dp(dp->drm_dev, "type=%d Done\n", dp->connector_type);
 
 	pm_runtime_put_sync(&dp->pdev->dev);
-	mutex_unlock(&msm_dp_display->event_mutex);
 }
 
 void msm_dp_bridge_mode_set(struct drm_bridge *drm_bridge,
@@ -1752,18 +1487,13 @@ void msm_dp_bridge_hpd_enable(struct drm_bridge *bridge)
 	 * step-4: DP PHY is initialized at plugin handler before link training
 	 *
 	 */
-	mutex_lock(&dp->event_mutex);
 	if (pm_runtime_resume_and_get(&msm_dp_display->pdev->dev)) {
 		DRM_ERROR("failed to resume power\n");
-		mutex_unlock(&dp->event_mutex);
 		return;
 	}
 
 	msm_dp_aux_hpd_enable(dp->aux);
 	msm_dp_aux_hpd_intr_enable(dp->aux);
-
-	msm_dp_display->internal_hpd = true;
-	mutex_unlock(&dp->event_mutex);
 }
 
 void msm_dp_bridge_hpd_disable(struct drm_bridge *bridge)
@@ -1772,15 +1502,10 @@ void msm_dp_bridge_hpd_disable(struct drm_bridge *bridge)
 	struct msm_dp *msm_dp_display = msm_dp_bridge->msm_dp_display;
 	struct msm_dp_display_private *dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display);
 
-	mutex_lock(&dp->event_mutex);
-
 	msm_dp_aux_hpd_intr_disable(dp->aux);
 	msm_dp_aux_hpd_disable(dp->aux);
 
-	msm_dp_display->internal_hpd = false;
-
 	pm_runtime_put_sync(&msm_dp_display->pdev->dev);
-	mutex_unlock(&dp->event_mutex);
 }
 
 void msm_dp_bridge_hpd_notify(struct drm_bridge *bridge,
@@ -1790,13 +1515,30 @@ void msm_dp_bridge_hpd_notify(struct drm_bridge *bridge,
 	struct msm_dp_bridge *msm_dp_bridge = to_dp_bridge(bridge);
 	struct msm_dp *msm_dp_display = msm_dp_bridge->msm_dp_display;
 	struct msm_dp_display_private *dp = container_of(msm_dp_display, struct msm_dp_display_private, msm_dp_display);
+	u32 hpd_link_status = 0;
 
-	/* Without next_bridge interrupts are handled by the DP core directly */
-	if (msm_dp_display->internal_hpd)
+	if (pm_runtime_resume_and_get(&msm_dp_display->pdev->dev)) {
+		DRM_ERROR("failed to pm_runtime_resume\n");
 		return;
+	}
+
+	hpd_link_status = msm_dp_aux_is_link_connected(dp->aux);
 
-	if (!msm_dp_display->link_ready && status == connector_status_connected)
-		msm_dp_add_event(dp, EV_HPD_PLUG_INT, 0, 0);
-	else if (msm_dp_display->link_ready && status == connector_status_disconnected)
-		msm_dp_add_event(dp, EV_HPD_UNPLUG_INT, 0, 0);
+	drm_dbg_dp(dp->drm_dev, "type=%d link hpd_link_status=0x%x, status=%d\n",
+		   msm_dp_display->connector_type, hpd_link_status, status);
+
+	if (status == connector_status_connected) {
+		if (hpd_link_status == ISR_HPD_REPLUG_COUNT) {
+			msm_dp_hpd_unplug_handle(dp);
+			msm_dp_hpd_plug_handle(dp);
+		} else if (hpd_link_status == ISR_IRQ_HPD_PULSE_COUNT) {
+			msm_dp_irq_hpd_handle(dp);
+		} else {
+			msm_dp_hpd_plug_handle(dp);
+		}
+	} else {
+		msm_dp_hpd_unplug_handle(dp);
+	}
+
+	pm_runtime_put_sync(&msm_dp_display->pdev->dev);
 }
diff --git a/drivers/gpu/drm/msm/dp/dp_display.h b/drivers/gpu/drm/msm/dp/dp_display.h
index cc6e2cab36e9c..0b65e16c790da 100644
--- a/drivers/gpu/drm/msm/dp/dp_display.h
+++ b/drivers/gpu/drm/msm/dp/dp_display.h
@@ -16,12 +16,11 @@ struct msm_dp {
 	struct platform_device *pdev;
 	struct drm_connector *connector;
 	struct drm_bridge *next_bridge;
-	bool link_ready;
+	struct drm_bridge *bridge;
 	bool audio_enabled;
 	bool power_on;
 	unsigned int connector_type;
 	bool is_edp;
-	bool internal_hpd;
 
 	struct msm_dp_audio *msm_dp_audio;
 	bool psr_supported;
diff --git a/drivers/gpu/drm/msm/dp/dp_drm.c b/drivers/gpu/drm/msm/dp/dp_drm.c
index fd6443d2b6cea..b659d22f5f284 100644
--- a/drivers/gpu/drm/msm/dp/dp_drm.c
+++ b/drivers/gpu/drm/msm/dp/dp_drm.c
@@ -15,56 +15,6 @@
 #include "dp_audio.h"
 #include "dp_drm.h"
 
-/**
- * msm_dp_bridge_detect - callback to determine if connector is connected
- * @bridge: Pointer to drm bridge structure
- * @connector: Pointer to drm connector structure
- * Returns: Bridge's 'is connected' status
- */
-static enum drm_connector_status
-msm_dp_bridge_detect(struct drm_bridge *bridge, struct drm_connector *connector)
-{
-	struct msm_dp *dp;
-
-	dp = to_dp_bridge(bridge)->msm_dp_display;
-
-	drm_dbg_dp(dp->drm_dev, "link_ready = %s\n",
-		str_true_false(dp->link_ready));
-
-	return (dp->link_ready) ? connector_status_connected :
-					connector_status_disconnected;
-}
-
-static int msm_dp_bridge_atomic_check(struct drm_bridge *bridge,
-			    struct drm_bridge_state *bridge_state,
-			    struct drm_crtc_state *crtc_state,
-			    struct drm_connector_state *conn_state)
-{
-	struct msm_dp *dp;
-
-	dp = to_dp_bridge(bridge)->msm_dp_display;
-
-	drm_dbg_dp(dp->drm_dev, "link_ready = %s\n",
-		str_true_false(dp->link_ready));
-
-	/*
-	 * There is no protection in the DRM framework to check if the display
-	 * pipeline has been already disabled before trying to disable it again.
-	 * Hence if the sink is unplugged, the pipeline gets disabled, but the
-	 * crtc->active is still true. Any attempt to set the mode or manually
-	 * disable this encoder will result in the crash.
-	 *
-	 * TODO: add support for telling the DRM subsystem that the pipeline is
-	 * disabled by the hardware and thus all access to it should be forbidden.
-	 * After that this piece of code can be removed.
-	 */
-	if (bridge->ops & DRM_BRIDGE_OP_HPD)
-		return (dp->link_ready) ? 0 : -ENOTCONN;
-
-	return 0;
-}
-
-
 /**
  * msm_dp_bridge_get_modes - callback to add drm modes via drm_mode_probed_add()
  * @bridge: Poiner to drm bridge
@@ -82,12 +32,10 @@ static int msm_dp_bridge_get_modes(struct drm_bridge *bridge, struct drm_connect
 	dp = to_dp_bridge(bridge)->msm_dp_display;
 
 	/* pluggable case assumes EDID is read when HPD */
-	if (dp->link_ready) {
-		rc = msm_dp_display_get_modes(dp);
-		if (rc <= 0) {
-			DRM_ERROR("failed to get DP sink modes, rc=%d\n", rc);
-			return rc;
-		}
+	rc = msm_dp_display_get_modes(dp);
+	if (rc <= 0) {
+		DRM_ERROR("failed to get DP sink modes, rc=%d\n", rc);
+		return rc;
 	} else {
 		drm_dbg_dp(connector->dev, "No sink connected\n");
 	}
@@ -112,7 +60,6 @@ static const struct drm_bridge_funcs msm_dp_bridge_ops = {
 	.mode_valid   = msm_dp_bridge_mode_valid,
 	.get_modes    = msm_dp_bridge_get_modes,
 	.detect       = msm_dp_bridge_detect,
-	.atomic_check = msm_dp_bridge_atomic_check,
 	.hpd_enable   = msm_dp_bridge_hpd_enable,
 	.hpd_disable  = msm_dp_bridge_hpd_disable,
 	.hpd_notify   = msm_dp_bridge_hpd_notify,
@@ -144,7 +91,7 @@ static int msm_edp_bridge_atomic_check(struct drm_bridge *drm_bridge,
 }
 
 static void msm_edp_bridge_atomic_enable(struct drm_bridge *drm_bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_crtc_state;
@@ -173,7 +120,7 @@ static void msm_edp_bridge_atomic_enable(struct drm_bridge *drm_bridge,
 }
 
 static void msm_edp_bridge_atomic_disable(struct drm_bridge *drm_bridge,
-					  struct drm_atomic_state *atomic_state)
+					  struct drm_atomic_commit *atomic_state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state = NULL, *old_crtc_state = NULL;
@@ -217,7 +164,7 @@ static void msm_edp_bridge_atomic_disable(struct drm_bridge *drm_bridge,
 }
 
 static void msm_edp_bridge_atomic_post_disable(struct drm_bridge *drm_bridge,
-					       struct drm_atomic_state *atomic_state)
+					       struct drm_atomic_commit *atomic_state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state = NULL;
@@ -360,6 +307,8 @@ int msm_dp_bridge_init(struct msm_dp *msm_dp_display, struct drm_device *dev,
 		}
 	}
 
+	msm_dp_display->bridge = bridge;
+
 	return 0;
 }
 
@@ -376,7 +325,5 @@ struct drm_connector *msm_dp_drm_connector_init(struct msm_dp *msm_dp_display,
 	if (!msm_dp_display->is_edp)
 		drm_connector_attach_dp_subconnector_property(connector);
 
-	drm_connector_attach_encoder(connector, encoder);
-
 	return connector;
 }
diff --git a/drivers/gpu/drm/msm/dp/dp_drm.h b/drivers/gpu/drm/msm/dp/dp_drm.h
index 9eb3431dd93ad..041aa026ae2ee 100644
--- a/drivers/gpu/drm/msm/dp/dp_drm.h
+++ b/drivers/gpu/drm/msm/dp/dp_drm.h
@@ -25,12 +25,14 @@ int msm_dp_bridge_init(struct msm_dp *msm_dp_display, struct drm_device *dev,
 		   struct drm_encoder *encoder,
 		   bool yuv_supported);
 
+enum drm_connector_status msm_dp_bridge_detect(struct drm_bridge *bridge,
+					       struct drm_connector *connector);
 void msm_dp_bridge_atomic_enable(struct drm_bridge *drm_bridge,
-				 struct drm_atomic_state *state);
+				 struct drm_atomic_commit *state);
 void msm_dp_bridge_atomic_disable(struct drm_bridge *drm_bridge,
-				  struct drm_atomic_state *state);
+				  struct drm_atomic_commit *state);
 void msm_dp_bridge_atomic_post_disable(struct drm_bridge *drm_bridge,
-				       struct drm_atomic_state *state);
+				       struct drm_atomic_commit *state);
 enum drm_mode_status msm_dp_bridge_mode_valid(struct drm_bridge *bridge,
 					  const struct drm_display_info *info,
 					  const struct drm_display_mode *mode);
diff --git a/drivers/gpu/drm/msm/dp/dp_panel.c b/drivers/gpu/drm/msm/dp/dp_panel.c
index 891211b232023..6bb021820d7c5 100644
--- a/drivers/gpu/drm/msm/dp/dp_panel.c
+++ b/drivers/gpu/drm/msm/dp/dp_panel.c
@@ -293,6 +293,14 @@ int msm_dp_panel_read_sink_caps(struct msm_dp_panel *msm_dp_panel,
 	return rc;
 }
 
+void msm_dp_panel_unplugged(struct msm_dp_panel *msm_dp_panel,
+			    struct drm_connector *connector)
+{
+	drm_edid_connector_update(connector, NULL);
+	drm_edid_free(msm_dp_panel->drm_edid);
+	msm_dp_panel->drm_edid = NULL;
+}
+
 u32 msm_dp_panel_get_mode_bpp(struct msm_dp_panel *msm_dp_panel,
 		u32 mode_edid_bpp, u32 mode_pclk_khz)
 {
diff --git a/drivers/gpu/drm/msm/dp/dp_panel.h b/drivers/gpu/drm/msm/dp/dp_panel.h
index 177c1328fd997..9173e90a50530 100644
--- a/drivers/gpu/drm/msm/dp/dp_panel.h
+++ b/drivers/gpu/drm/msm/dp/dp_panel.h
@@ -49,6 +49,8 @@ int msm_dp_panel_deinit(struct msm_dp_panel *msm_dp_panel);
 int msm_dp_panel_timing_cfg(struct msm_dp_panel *msm_dp_panel, bool wide_bus_en);
 int msm_dp_panel_read_sink_caps(struct msm_dp_panel *msm_dp_panel,
 		struct drm_connector *connector);
+void msm_dp_panel_unplugged(struct msm_dp_panel *msm_dp_panel,
+			    struct drm_connector *connector);
 u32 msm_dp_panel_get_mode_bpp(struct msm_dp_panel *msm_dp_panel, u32 mode_max_bpp,
 			u32 mode_pclk_khz);
 int msm_dp_panel_get_modes(struct msm_dp_panel *msm_dp_panel,
diff --git a/drivers/gpu/drm/msm/dp/dp_reg.h b/drivers/gpu/drm/msm/dp/dp_reg.h
index 7c44d4e2cf139..3689642b7fc06 100644
--- a/drivers/gpu/drm/msm/dp/dp_reg.h
+++ b/drivers/gpu/drm/msm/dp/dp_reg.h
@@ -68,8 +68,8 @@
 #define DP_DP_IRQ_HPD_INT_ACK			(0x00000002)
 #define DP_DP_HPD_REPLUG_INT_ACK		(0x00000004)
 #define DP_DP_HPD_UNPLUG_INT_ACK		(0x00000008)
-#define DP_DP_HPD_STATE_STATUS_BITS_MASK	(0x0000000F)
-#define DP_DP_HPD_STATE_STATUS_BITS_SHIFT	(0x1C)
+#define DP_DP_HPD_STATE_STATUS_BITS_MASK	(0x00000007)
+#define DP_DP_HPD_STATE_STATUS_BITS_SHIFT	(0x1D)
 
 #define REG_DP_DP_HPD_INT_MASK			(0x0000000C)
 #define DP_DP_HPD_PLUG_INT_MASK			(0x00000001)
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index 565d425f88b86..982abaaac00d5 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -2033,6 +2033,7 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 
 	/* fixup base address by io offset */
 	msm_host->ctrl_base += cfg->io_offset;
+	msm_host->ctrl_size -= cfg->io_offset;
 
 	ret = devm_regulator_bulk_get_const(&pdev->dev, cfg->num_regulators,
 					    cfg->regulator_data,
diff --git a/drivers/gpu/drm/msm/dsi/dsi_manager.c b/drivers/gpu/drm/msm/dsi/dsi_manager.c
index ca400924d4eea..46faeaae9d94a 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_manager.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_manager.c
@@ -485,10 +485,6 @@ int msm_dsi_manager_connector_init(struct msm_dsi *msm_dsi,
 		return PTR_ERR(connector);
 	}
 
-	ret = drm_connector_attach_encoder(connector, encoder);
-	if (ret < 0)
-		return ret;
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
index c59375aaae197..1fb3899b88bf2 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
@@ -571,6 +571,8 @@ static const struct of_device_id dsi_phy_dt_match[] = {
 	  .data = &dsi_phy_5nm_8350_cfgs },
 	{ .compatible = "qcom,sm8450-dsi-phy-5nm",
 	  .data = &dsi_phy_5nm_8450_cfgs },
+	{ .compatible = "qcom,milos-dsi-phy-4nm",
+	  .data = &dsi_phy_4nm_milos_cfgs },
 	{ .compatible = "qcom,sm8550-dsi-phy-4nm",
 	  .data = &dsi_phy_4nm_8550_cfgs },
 	{ .compatible = "qcom,sm8650-dsi-phy-4nm",
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
index c01784ca38edc..21a59d66e8dc5 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
@@ -61,6 +61,7 @@ extern const struct msm_dsi_phy_cfg dsi_phy_5nm_8350_cfgs;
 extern const struct msm_dsi_phy_cfg dsi_phy_5nm_8450_cfgs;
 extern const struct msm_dsi_phy_cfg dsi_phy_5nm_8775p_cfgs;
 extern const struct msm_dsi_phy_cfg dsi_phy_5nm_sar2130p_cfgs;
+extern const struct msm_dsi_phy_cfg dsi_phy_4nm_milos_cfgs;
 extern const struct msm_dsi_phy_cfg dsi_phy_4nm_8550_cfgs;
 extern const struct msm_dsi_phy_cfg dsi_phy_4nm_8650_cfgs;
 extern const struct msm_dsi_phy_cfg dsi_phy_3nm_8750_cfgs;
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
index 8f4b03713f256..984a66085dfbf 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_7nm.c
@@ -1436,6 +1436,29 @@ const struct msm_dsi_phy_cfg dsi_phy_5nm_sar2130p_cfgs = {
 	.quirks = DSI_PHY_7NM_QUIRK_V5_2,
 };
 
+const struct msm_dsi_phy_cfg dsi_phy_4nm_milos_cfgs = {
+	.has_phy_lane = true,
+	.regulator_data = dsi_phy_7nm_98000uA_regulators,
+	.num_regulators = ARRAY_SIZE(dsi_phy_7nm_98000uA_regulators),
+	.ops = {
+		.enable = dsi_7nm_phy_enable,
+		.disable = dsi_7nm_phy_disable,
+		.pll_init = dsi_pll_7nm_init,
+		.save_pll_state = dsi_7nm_pll_save_state,
+		.restore_pll_state = dsi_7nm_pll_restore_state,
+		.set_continuous_clock = dsi_7nm_set_continuous_clock,
+	},
+	.min_pll_rate = 600000000UL,
+#ifdef CONFIG_64BIT
+	.max_pll_rate = 5000000000UL,
+#else
+	.max_pll_rate = ULONG_MAX,
+#endif
+	.io_start = { 0xae95000 },
+	.num_dsi_phy = 1,
+	.quirks = DSI_PHY_7NM_QUIRK_V5_2,
+};
+
 const struct msm_dsi_phy_cfg dsi_phy_4nm_8550_cfgs = {
 	.has_phy_lane = true,
 	.regulator_data = dsi_phy_7nm_98400uA_regulators,
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c
index 852abb2466f0c..4740060846335 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.c
@@ -190,8 +190,6 @@ int msm_hdmi_modeset_init(struct hdmi *hdmi,
 		goto fail;
 	}
 
-	drm_connector_attach_encoder(hdmi->connector, hdmi->encoder);
-
 	ret = devm_request_irq(dev->dev, hdmi->irq,
 			msm_hdmi_irq, IRQF_TRIGGER_HIGH,
 			"hdmi_isr", hdmi);
@@ -287,19 +285,27 @@ static int msm_hdmi_dev_probe(struct platform_device *pdev)
 	spin_lock_init(&hdmi->reg_lock);
 	mutex_init(&hdmi->state_mutex);
 
-	ret = drm_of_find_panel_or_bridge(dev_of_node(dev), 1, 0, NULL, &hdmi->next_bridge);
-	if (ret && ret != -ENODEV)
-		return ret;
+	hdmi->next_bridge = of_drm_get_bridge_by_endpoint(dev_of_node(dev), 1, 0);
+	if (IS_ERR(hdmi->next_bridge)) {
+		if (PTR_ERR(hdmi->next_bridge) != -ENODEV)
+			return PTR_ERR(hdmi->next_bridge);
+
+		hdmi->next_bridge = NULL;
+	}
 
 	hdmi->mmio = msm_ioremap(pdev, "core_physical");
-	if (IS_ERR(hdmi->mmio))
-		return PTR_ERR(hdmi->mmio);
+	if (IS_ERR(hdmi->mmio)) {
+		ret = PTR_ERR(hdmi->mmio);
+		goto err_put_bridge;
+	}
 
 	/* HDCP needs physical address of hdmi register */
 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
 		"core_physical");
-	if (!res)
-		return -EINVAL;
+	if (!res) {
+		ret = -EINVAL;
+		goto err_put_bridge;
+	}
 	hdmi->mmio_phy_addr = res->start;
 
 	hdmi->qfprom_mmio = msm_ioremap(pdev, "qfprom_physical");
@@ -309,45 +315,58 @@ static int msm_hdmi_dev_probe(struct platform_device *pdev)
 	}
 
 	hdmi->irq = platform_get_irq(pdev, 0);
-	if (hdmi->irq < 0)
-		return hdmi->irq;
+	if (hdmi->irq < 0) {
+		ret = hdmi->irq;
+		goto err_put_bridge;
+	}
 
 	hdmi->pwr_regs = devm_kcalloc(dev, config->pwr_reg_cnt,
 				      sizeof(hdmi->pwr_regs[0]),
 				      GFP_KERNEL);
-	if (!hdmi->pwr_regs)
-		return -ENOMEM;
+	if (!hdmi->pwr_regs) {
+		ret = -ENOMEM;
+		goto err_put_bridge;
+	}
 
 	for (i = 0; i < config->pwr_reg_cnt; i++)
 		hdmi->pwr_regs[i].supply = config->pwr_reg_names[i];
 
 	ret = devm_regulator_bulk_get(dev, config->pwr_reg_cnt, hdmi->pwr_regs);
-	if (ret)
-		return dev_err_probe(dev, ret, "failed to get pwr regulators\n");
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to get pwr regulators\n");
+		goto err_put_bridge;
+	}
 
 	hdmi->pwr_clks = devm_kcalloc(dev, config->pwr_clk_cnt,
 				      sizeof(hdmi->pwr_clks[0]),
 				      GFP_KERNEL);
-	if (!hdmi->pwr_clks)
-		return -ENOMEM;
+	if (!hdmi->pwr_clks) {
+		ret = -ENOMEM;
+		goto err_put_bridge;
+	}
 
 	for (i = 0; i < config->pwr_clk_cnt; i++)
 		hdmi->pwr_clks[i].id = config->pwr_clk_names[i];
 
 	ret = devm_clk_bulk_get(dev, config->pwr_clk_cnt, hdmi->pwr_clks);
 	if (ret)
-		return ret;
+		goto err_put_bridge;
+
 
 	hdmi->extp_clk = devm_clk_get_optional(dev, "extp");
-	if (IS_ERR(hdmi->extp_clk))
-		return dev_err_probe(dev, PTR_ERR(hdmi->extp_clk),
-				     "failed to get extp clock\n");
+	if (IS_ERR(hdmi->extp_clk)) {
+		ret = dev_err_probe(dev, PTR_ERR(hdmi->extp_clk),
+				    "failed to get extp clock\n");
+		goto err_put_bridge;
+	}
 
 	hdmi->hpd_gpiod = devm_gpiod_get_optional(dev, "hpd", GPIOD_IN);
 	/* This will catch e.g. -EPROBE_DEFER */
-	if (IS_ERR(hdmi->hpd_gpiod))
-		return dev_err_probe(dev, PTR_ERR(hdmi->hpd_gpiod),
-				     "failed to get hpd gpio\n");
+	if (IS_ERR(hdmi->hpd_gpiod)) {
+		ret = dev_err_probe(dev, PTR_ERR(hdmi->hpd_gpiod),
+				    "failed to get hpd gpio\n");
+		goto err_put_bridge;
+	}
 
 	if (!hdmi->hpd_gpiod)
 		DBG("failed to get HPD gpio");
@@ -357,7 +376,7 @@ static int msm_hdmi_dev_probe(struct platform_device *pdev)
 
 	ret = msm_hdmi_get_phy(hdmi);
 	if (ret)
-		return ret;
+		goto err_put_bridge;
 
 	ret = devm_pm_runtime_enable(dev);
 	if (ret)
@@ -373,6 +392,8 @@ static int msm_hdmi_dev_probe(struct platform_device *pdev)
 
 err_put_phy:
 	msm_hdmi_put_phy(hdmi);
+err_put_bridge:
+	drm_bridge_put(hdmi->next_bridge);
 	return ret;
 }
 
@@ -383,6 +404,7 @@ static void msm_hdmi_dev_remove(struct platform_device *pdev)
 	component_del(&pdev->dev, &msm_hdmi_ops);
 
 	msm_hdmi_put_phy(hdmi);
+	drm_bridge_put(hdmi->next_bridge);
 }
 
 static int msm_hdmi_runtime_suspend(struct device *dev)
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
index a9eb6489c5203..c4d5e0faf3b32 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
@@ -283,7 +283,7 @@ static void msm_hdmi_set_timings(struct hdmi *hdmi,
 				 const struct drm_display_mode *mode);
 
 static void msm_hdmi_bridge_atomic_pre_enable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct hdmi_bridge *hdmi_bridge = to_hdmi_bridge(bridge);
 	struct hdmi *hdmi = hdmi_bridge->hdmi;
@@ -325,7 +325,7 @@ static void msm_hdmi_bridge_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void msm_hdmi_bridge_atomic_post_disable(struct drm_bridge *bridge,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct hdmi_bridge *hdmi_bridge = to_hdmi_bridge(bridge);
 	struct hdmi *hdmi = hdmi_bridge->hdmi;
diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c
index 87a91148a731d..a8babf1dbe0d3 100644
--- a/drivers/gpu/drm/msm/msm_atomic.c
+++ b/drivers/gpu/drm/msm/msm_atomic.c
@@ -136,7 +136,7 @@ void msm_atomic_destroy_pending_timer(struct msm_pending_timer *timer)
 		kthread_destroy_worker(timer->worker);
 }
 
-static bool can_do_async(struct drm_atomic_state *state,
+static bool can_do_async(struct drm_atomic_commit *state,
 		struct drm_crtc **async_crtc)
 {
 	struct drm_connector_state *connector_state;
@@ -169,7 +169,7 @@ static bool can_do_async(struct drm_atomic_state *state,
  * can be used with for_each_crtc_mask() iterator, to iterate
  * effected crtcs without needing to preserve the atomic state.
  */
-static unsigned get_crtc_mask(struct drm_atomic_state *state)
+static unsigned get_crtc_mask(struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state;
 	struct drm_crtc *crtc;
@@ -181,7 +181,7 @@ static unsigned get_crtc_mask(struct drm_atomic_state *state)
 	return mask;
 }
 
-int msm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+int msm_atomic_check(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct msm_drm_private *priv = dev->dev_private;
 	struct msm_kms *kms = priv->kms;
@@ -210,7 +210,7 @@ int msm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 	return drm_atomic_helper_check(dev, state);
 }
 
-void msm_atomic_commit_tail(struct drm_atomic_state *state)
+void msm_atomic_commit_tail(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct msm_drm_private *priv = dev->dev_private;
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 195f40e331e5a..cc2bcd14b1c26 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -128,11 +128,10 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv,
 	/*
 	 * Initialize the LRUs:
 	 */
-	mutex_init(&priv->lru.lock);
-	drm_gem_lru_init(&priv->lru.unbacked, &priv->lru.lock);
-	drm_gem_lru_init(&priv->lru.pinned,   &priv->lru.lock);
-	drm_gem_lru_init(&priv->lru.willneed, &priv->lru.lock);
-	drm_gem_lru_init(&priv->lru.dontneed, &priv->lru.lock);
+	drm_gem_lru_init(&priv->lru.unbacked);
+	drm_gem_lru_init(&priv->lru.pinned);
+	drm_gem_lru_init(&priv->lru.willneed);
+	drm_gem_lru_init(&priv->lru.dontneed);
 
 	/* Initialize stall-on-fault */
 	spin_lock_init(&priv->fault_stall_lock);
@@ -140,7 +139,7 @@ static int msm_drm_init(struct device *dev, const struct drm_driver *drv,
 
 	/* Teach lockdep about lock ordering wrt. shrinker: */
 	fs_reclaim_acquire(GFP_KERNEL);
-	might_lock(&priv->lru.lock);
+	might_lock(&ddev->gem_lru_mutex);
 	fs_reclaim_release(GFP_KERNEL);
 
 	if (priv->kms_init) {
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 6d847d593f1ae..c3fb3205f6831 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -150,13 +150,6 @@ struct msm_drm_private {
 		 * DONTNEED state (ie. can be purged)
 		 */
 		struct drm_gem_lru dontneed;
-
-		/**
-		 * lock:
-		 *
-		 * Protects manipulation of all of the LRUs.
-		 */
-		struct mutex lock;
 	} lru;
 
 	struct notifier_block vmap_notifier;
@@ -219,9 +212,9 @@ struct msm_pending_timer;
 int msm_atomic_init_pending_timer(struct msm_pending_timer *timer,
 		struct msm_kms *kms, int crtc_idx);
 void msm_atomic_destroy_pending_timer(struct msm_pending_timer *timer);
-void msm_atomic_commit_tail(struct drm_atomic_state *state);
-int msm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state);
-struct drm_atomic_state *msm_atomic_state_alloc(struct drm_device *dev);
+void msm_atomic_commit_tail(struct drm_atomic_commit *state);
+int msm_atomic_check(struct drm_device *dev, struct drm_atomic_commit *state);
+struct drm_atomic_commit *msm_atomic_state_alloc(struct drm_device *dev);
 
 int msm_crtc_enable_vblank(struct drm_crtc *crtc);
 void msm_crtc_disable_vblank(struct drm_crtc *crtc);
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 2cb3ab04f1250..efd3d3c9a4490 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -177,11 +177,11 @@ static void update_lru_locked(struct drm_gem_object *obj)
 
 static void update_lru(struct drm_gem_object *obj)
 {
-	struct msm_drm_private *priv = obj->dev->dev_private;
+	struct drm_device *dev = obj->dev;
 
-	mutex_lock(&priv->lru.lock);
+	mutex_lock(&dev->gem_lru_mutex);
 	update_lru_locked(obj);
-	mutex_unlock(&priv->lru.lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 }
 
 static struct page **get_pages(struct drm_gem_object *obj)
@@ -292,11 +292,11 @@ void msm_gem_pin_obj_locked(struct drm_gem_object *obj)
 
 static void pin_obj_locked(struct drm_gem_object *obj)
 {
-	struct msm_drm_private *priv = obj->dev->dev_private;
+	struct drm_device *dev = obj->dev;
 
-	mutex_lock(&priv->lru.lock);
+	mutex_lock(&dev->gem_lru_mutex);
 	msm_gem_pin_obj_locked(obj);
-	mutex_unlock(&priv->lru.lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 }
 
 struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj)
@@ -487,16 +487,16 @@ int msm_gem_pin_vma_locked(struct drm_gem_object *obj, struct drm_gpuva *vma)
 
 void msm_gem_unpin_locked(struct drm_gem_object *obj)
 {
-	struct msm_drm_private *priv = obj->dev->dev_private;
+	struct drm_device *dev = obj->dev;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
 	msm_gem_assert_locked(obj);
 
-	mutex_lock(&priv->lru.lock);
+	mutex_lock(&dev->gem_lru_mutex);
 	msm_obj->pin_count--;
 	GEM_WARN_ON(msm_obj->pin_count < 0);
 	update_lru_locked(obj);
-	mutex_unlock(&priv->lru.lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 }
 
 /* Special unpin path for use in fence-signaling path, avoiding the need
@@ -507,10 +507,10 @@ void msm_gem_unpin_locked(struct drm_gem_object *obj)
  */
 void msm_gem_unpin_active(struct drm_gem_object *obj)
 {
-	struct msm_drm_private *priv = obj->dev->dev_private;
+	struct drm_device *dev = obj->dev;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-	GEM_WARN_ON(!mutex_is_locked(&priv->lru.lock));
+	GEM_WARN_ON(!mutex_is_locked(&dev->gem_lru_mutex));
 
 	msm_obj->pin_count--;
 	GEM_WARN_ON(msm_obj->pin_count < 0);
@@ -797,12 +797,12 @@ void msm_gem_put_vaddr(struct drm_gem_object *obj)
  */
 int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
 {
-	struct msm_drm_private *priv = obj->dev->dev_private;
+	struct drm_device *dev = obj->dev;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
 	msm_gem_lock(obj);
 
-	mutex_lock(&priv->lru.lock);
+	mutex_lock(&dev->gem_lru_mutex);
 
 	if (msm_obj->madv != __MSM_MADV_PURGED)
 		msm_obj->madv = madv;
@@ -814,7 +814,7 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
 	 */
 	update_lru_locked(obj);
 
-	mutex_unlock(&priv->lru.lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 
 	msm_gem_unlock(obj);
 
@@ -824,7 +824,6 @@ int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
 void msm_gem_purge(struct drm_gem_object *obj)
 {
 	struct drm_device *dev = obj->dev;
-	struct msm_drm_private *priv = obj->dev->dev_private;
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
 	msm_gem_assert_locked(obj);
@@ -839,10 +838,10 @@ void msm_gem_purge(struct drm_gem_object *obj)
 
 	put_pages(obj);
 
-	mutex_lock(&priv->lru.lock);
+	mutex_lock(&dev->gem_lru_mutex);
 	/* A one-way transition: */
 	msm_obj->madv = __MSM_MADV_PURGED;
-	mutex_unlock(&priv->lru.lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 
 	drm_gem_free_mmap_offset(obj);
 
diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c
index 31fa51a44f86e..c8dda2b68cff2 100644
--- a/drivers/gpu/drm/msm/msm_gem_shrinker.c
+++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c
@@ -43,8 +43,7 @@ msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 }
 
 static bool
-with_vm_locks(struct ww_acquire_ctx *ticket,
-	      void (*fn)(struct drm_gem_object *obj),
+with_vm_locks(void (*fn)(struct drm_gem_object *obj),
 	      struct drm_gem_object *obj)
 {
 	/*
@@ -52,7 +51,7 @@ with_vm_locks(struct ww_acquire_ctx *ticket,
 	 * success paths
 	 */
 	struct drm_gpuvm_bo *vm_bo, *last_locked = NULL;
-	int ret = 0;
+	bool locked = true;
 
 	drm_gem_for_each_gpuvm_bo (vm_bo, obj) {
 		struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
@@ -60,23 +59,14 @@ with_vm_locks(struct ww_acquire_ctx *ticket,
 		if (resv == obj->resv)
 			continue;
 
-		ret = dma_resv_lock(resv, ticket);
-
-		/*
-		 * Since we already skip the case when the VM and obj
-		 * share a resv (ie. _NO_SHARE objs), we don't expect
-		 * to hit a double-locking scenario... which the lock
-		 * unwinding cannot really cope with.
-		 */
-		WARN_ON(ret == -EALREADY);
-
 		/*
-		 * Don't bother with slow-lock / backoff / retry sequence,
-		 * if we can't get the lock just give up and move on to
-		 * the next object.
+		 * dma_resv_lock can't be used due to acquiring 'ticket' before the
+		 * fs_reclaim lock, which is held in shrinker context
 		 */
-		if (ret)
+		if (!dma_resv_trylock(resv)) {
+			locked = false;
 			goto out_unlock;
+		}
 
 		/*
 		 * Hold a ref to prevent the vm_bo from being freed
@@ -108,11 +98,11 @@ with_vm_locks(struct ww_acquire_ctx *ticket,
 		}
 	}
 
-	return ret == 0;
+	return locked;
 }
 
 static bool
-purge(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
+purge(struct drm_gem_object *obj, struct ww_acquire_ctx *)
 {
 	if (!is_purgeable(to_msm_bo(obj)))
 		return false;
@@ -120,11 +110,11 @@ purge(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
 	if (msm_gem_active(obj))
 		return false;
 
-	return with_vm_locks(ticket, msm_gem_purge, obj);
+	return with_vm_locks(msm_gem_purge, obj);
 }
 
 static bool
-evict(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
+evict(struct drm_gem_object *obj, struct ww_acquire_ctx *)
 {
 	if (is_unevictable(to_msm_bo(obj)))
 		return false;
@@ -132,7 +122,7 @@ evict(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket)
 	if (msm_gem_active(obj))
 		return false;
 
-	return with_vm_locks(ticket, msm_gem_evict, obj);
+	return with_vm_locks(msm_gem_evict, obj);
 }
 
 static bool
@@ -164,7 +154,6 @@ static unsigned long
 msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 {
 	struct msm_drm_private *priv = shrinker->private_data;
-	struct ww_acquire_ctx ticket;
 	struct {
 		struct drm_gem_lru *lru;
 		bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket);
@@ -185,11 +174,14 @@ msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 	for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) {
 		if (!stages[i].cond)
 			continue;
+		/*
+		 * 'ticket' not needed on trylock paths
+		 */
 		stages[i].freed =
-			drm_gem_lru_scan(stages[i].lru, nr,
+			drm_gem_lru_scan(priv->dev, stages[i].lru, nr,
 					 &stages[i].remaining,
 					 stages[i].shrink,
-					 &ticket);
+					 NULL);
 		nr -= stages[i].freed;
 		freed += stages[i].freed;
 		remaining += stages[i].remaining;
@@ -255,7 +247,7 @@ msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
 	unsigned long remaining = 0;
 
 	for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) {
-		unmapped += drm_gem_lru_scan(lrus[idx],
+		unmapped += drm_gem_lru_scan(priv->dev, lrus[idx],
 					     vmap_shrink_limit - unmapped,
 					     &remaining,
 					     vmap_shrink,
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index 26ea8a28be474..3c6bc90c3d486 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -352,7 +352,7 @@ static int submit_fence_sync(struct msm_gem_submit *submit)
 
 static int submit_pin_objects(struct msm_gem_submit *submit)
 {
-	struct msm_drm_private *priv = submit->dev->dev_private;
+	struct drm_device *dev = submit->dev;
 	int i, ret = 0;
 
 	for (i = 0; i < submit->nr_bos; i++) {
@@ -381,11 +381,11 @@ static int submit_pin_objects(struct msm_gem_submit *submit)
 	 * get_pages() which could trigger reclaim.. and if we held the LRU lock
 	 * could trigger deadlock with the shrinker).
 	 */
-	mutex_lock(&priv->lru.lock);
+	mutex_lock(&dev->gem_lru_mutex);
 	for (i = 0; i < submit->nr_bos; i++) {
 		msm_gem_pin_obj_locked(submit->bos[i].obj);
 	}
-	mutex_unlock(&priv->lru.lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 
 	submit->bos_pinned = true;
 
diff --git a/drivers/gpu/drm/msm/msm_gem_vma.c b/drivers/gpu/drm/msm/msm_gem_vma.c
index 1a952b171ed7f..3ed05ab0eeefd 100644
--- a/drivers/gpu/drm/msm/msm_gem_vma.c
+++ b/drivers/gpu/drm/msm/msm_gem_vma.c
@@ -702,7 +702,7 @@ static struct dma_fence *
 msm_vma_job_run(struct drm_sched_job *_job)
 {
 	struct msm_vm_bind_job *job = to_msm_vm_bind_job(_job);
-	struct msm_drm_private *priv = job->vm->drm->dev_private;
+	struct drm_device *dev = job->vm->drm;
 	struct msm_gem_vm *vm = to_msm_vm(job->vm);
 	struct drm_gem_object *obj;
 	int ret = vm->unusable ? -EINVAL : 0;
@@ -745,13 +745,13 @@ msm_vma_job_run(struct drm_sched_job *_job)
 	if (ret)
 		msm_gem_vm_unusable(job->vm);
 
-	mutex_lock(&priv->lru.lock);
+	mutex_lock(&dev->gem_lru_mutex);
 
 	job_foreach_bo (obj, job) {
 		msm_gem_unpin_active(obj);
 	}
 
-	mutex_unlock(&priv->lru.lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 
 	/* VM_BIND ops are synchronous, so no fence to wait on: */
 	return NULL;
@@ -841,7 +841,6 @@ msm_gem_vm_create(struct drm_device *drm, struct msm_mmu *mmu, const char *name,
 	if (!managed) {
 		struct drm_sched_init_args args = {
 			.ops = &msm_vm_bind_ops,
-			.num_rqs = 1,
 			.credit_limit = 1,
 			.timeout = MAX_SCHEDULE_TIMEOUT,
 			.name = "msm-vm-bind",
@@ -1305,7 +1304,7 @@ vm_bind_job_pin_objects(struct msm_vm_bind_job *job)
 			return PTR_ERR(pages);
 	}
 
-	struct msm_drm_private *priv = job->vm->drm->dev_private;
+	struct drm_device *dev = job->vm->drm;
 
 	/*
 	 * A second loop while holding the LRU lock (a) avoids acquiring/dropping
@@ -1314,10 +1313,10 @@ vm_bind_job_pin_objects(struct msm_vm_bind_job *job)
 	 * get_pages() which could trigger reclaim.. and if we held the LRU lock
 	 * could trigger deadlock with the shrinker).
 	 */
-	mutex_lock(&priv->lru.lock);
+	mutex_lock(&dev->gem_lru_mutex);
 	job_foreach_bo (obj, job)
 		msm_gem_pin_obj_locked(obj);
-	mutex_unlock(&priv->lru.lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 
 	job->bos_pinned = true;
 
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 930e54d1b0a7d..3f3925b11eead 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -13,11 +13,11 @@
 #include "msm_gpu_trace.h"
 //#include "adreno/adreno_gpu.h"
 
-#include <generated/utsrelease.h>
 #include <linux/string_helpers.h>
 #include <linux/devcoredump.h>
 #include <linux/sched/task.h>
 #include <linux/sched/mm.h>
+#include <linux/utsname.h>
 
 /*
  * Power Management:
@@ -196,7 +196,7 @@ static ssize_t msm_gpu_devcoredump_read(char *buffer, loff_t offset,
 	p = drm_coredump_printer(&iter);
 
 	drm_printf(&p, "---\n");
-	drm_printf(&p, "kernel: " UTS_RELEASE "\n");
+	drm_printf(&p, "kernel: %s\n", init_utsname()->release);
 	drm_printf(&p, "module: " KBUILD_MODNAME "\n");
 	drm_printf(&p, "time: %ptSp\n", &state->time);
 	if (state->comm)
diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c
index 7d449e5202c5d..058c71c82cf54 100644
--- a/drivers/gpu/drm/msm/msm_iommu.c
+++ b/drivers/gpu/drm/msm/msm_iommu.c
@@ -677,7 +677,7 @@ static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
 			 int prot)
 {
 	struct msm_iommu *iommu = to_msm_iommu(mmu);
-	size_t ret;
+	ssize_t ret;
 
 	WARN_ON(off != 0);
 
@@ -686,7 +686,8 @@ static int msm_iommu_map(struct msm_mmu *mmu, uint64_t iova,
 		iova |= GENMASK_ULL(63, 49);
 
 	ret = iommu_map_sgtable(iommu->domain, iova, sgt, prot);
-	WARN_ON(!ret);
+	if (ret < 0)
+		return ret;
 
 	return (ret == len) ? 0 : -EINVAL;
 }
diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h
index 8a7be7b854dee..f25b31e502d21 100644
--- a/drivers/gpu/drm/msm/msm_kms.h
+++ b/drivers/gpu/drm/msm/msm_kms.h
@@ -66,13 +66,13 @@ struct msm_kms_funcs {
 	 *
 	 * Verify if the commit requires a full modeset on one of CRTCs.
 	 */
-	int (*check_mode_changed)(struct msm_kms *kms, struct drm_atomic_state *state);
+	int (*check_mode_changed)(struct msm_kms *kms, struct drm_atomic_commit *state);
 
 	/**
 	 * Prepare for atomic commit.  This is called after any previous
 	 * (async or otherwise) commit has completed.
 	 */
-	void (*prepare_commit)(struct msm_kms *kms, struct drm_atomic_state *state);
+	void (*prepare_commit)(struct msm_kms *kms, struct drm_atomic_commit *state);
 
 	/**
 	 * Flush an atomic commit.  This is called after the hardware
diff --git a/drivers/gpu/drm/msm/msm_mdss.c b/drivers/gpu/drm/msm/msm_mdss.c
index 90c3fa0681a06..19ab0ff800493 100644
--- a/drivers/gpu/drm/msm/msm_mdss.c
+++ b/drivers/gpu/drm/msm/msm_mdss.c
@@ -166,22 +166,19 @@ static int _msm_mdss_irq_domain_add(struct msm_mdss *msm_mdss)
 	return 0;
 }
 
-static void msm_mdss_setup_ubwc_dec_20(struct msm_mdss *msm_mdss)
+static void msm_mdss_4x_setup_ubwc(struct msm_mdss *msm_mdss)
 {
 	const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data;
-	u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) |
+	u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle & 0x1) |
 		    MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13);
 
-	if (data->ubwc_bank_spread)
-		value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD;
-
 	if (data->ubwc_enc_version == UBWC_1_0)
 		value |= MDSS_UBWC_STATIC_UBWC_MIN_ACC_LEN(1);
 
 	writel_relaxed(value, msm_mdss->mmio + REG_MDSS_UBWC_STATIC);
 }
 
-static void msm_mdss_setup_ubwc_dec_30(struct msm_mdss *msm_mdss)
+static void msm_mdss_5x_setup_ubwc(struct msm_mdss *msm_mdss)
 {
 	const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data;
 	u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle & 0x1) |
@@ -199,11 +196,12 @@ static void msm_mdss_setup_ubwc_dec_30(struct msm_mdss *msm_mdss)
 	writel_relaxed(value, msm_mdss->mmio + REG_MDSS_UBWC_STATIC);
 }
 
-static void msm_mdss_setup_ubwc_dec_40(struct msm_mdss *msm_mdss)
+static void msm_mdss_6x_setup_ubwc(struct msm_mdss *msm_mdss)
 {
 	const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data;
 	u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) |
 		    MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13);
+	u32 ver, prediction_mode;
 
 	if (data->ubwc_bank_spread)
 		value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD;
@@ -211,45 +209,42 @@ static void msm_mdss_setup_ubwc_dec_40(struct msm_mdss *msm_mdss)
 	if (data->macrotile_mode)
 		value |= MDSS_UBWC_STATIC_MACROTILE_MODE;
 
-	writel_relaxed(value, msm_mdss->mmio + REG_MDSS_UBWC_STATIC);
-
-	if (data->ubwc_enc_version == UBWC_3_0) {
-		writel_relaxed(1, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2);
-		writel_relaxed(0, msm_mdss->mmio + REG_MDSS_UBWC_PREDICTION_MODE);
-	} else {
-		if (data->ubwc_dec_version == UBWC_4_3)
-			writel_relaxed(3, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2);
-		else
-			writel_relaxed(2, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2);
-		writel_relaxed(1, msm_mdss->mmio + REG_MDSS_UBWC_PREDICTION_MODE);
-	}
-}
-
-static void msm_mdss_setup_ubwc_dec_50(struct msm_mdss *msm_mdss)
-{
-	const struct qcom_ubwc_cfg_data *data = msm_mdss->mdss_data;
-	u32 value = MDSS_UBWC_STATIC_UBWC_SWIZZLE(data->ubwc_swizzle) |
-		    MDSS_UBWC_STATIC_HIGHEST_BANK_BIT(data->highest_bank_bit - 13);
-
-	if (data->ubwc_bank_spread)
-		value |= MDSS_UBWC_STATIC_UBWC_BANK_SPREAD;
-
-	if (data->macrotile_mode)
-		value |= MDSS_UBWC_STATIC_MACROTILE_MODE;
+	if (data->ubwc_enc_version == UBWC_1_0)
+		value |= MDSS_UBWC_STATIC_UBWC_MIN_ACC_LEN(1);
 
 	writel_relaxed(value, msm_mdss->mmio + REG_MDSS_UBWC_STATIC);
 
-	if (data->ubwc_dec_version == UBWC_6_0)
-		writel_relaxed(5, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2);
+	if (data->ubwc_enc_version < UBWC_4_0)
+		prediction_mode = 0;
 	else
-		writel_relaxed(4, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2);
-
-	writel_relaxed(1, msm_mdss->mmio + REG_MDSS_UBWC_PREDICTION_MODE);
+		prediction_mode = 1;
+
+	if (data->ubwc_enc_version >= UBWC_6_0)
+		ver = 5;
+	else if (data->ubwc_enc_version >= UBWC_5_0)
+		ver = 4;
+	else if (data->ubwc_enc_version >= UBWC_4_3)
+		ver = 3;
+	else if (data->ubwc_enc_version >= UBWC_4_0)
+		ver = 2;
+	else if (data->ubwc_enc_version >= UBWC_3_0)
+		ver = 1;
+	else /* UBWC 1.0 and 2.0 */
+		ver = 0;
+
+	writel_relaxed(ver, msm_mdss->mmio + REG_MDSS_UBWC_CTRL_2);
+	writel_relaxed(prediction_mode, msm_mdss->mmio + REG_MDSS_UBWC_PREDICTION_MODE);
 }
 
+#define MDSS_HW_VER(major, minor, step)	\
+	((((major) & 0xf) << 28) |	\
+	 (((minor) & 0xfff) << 16) |	\
+	 ((step) & 0xffff))
+
 static int msm_mdss_enable(struct msm_mdss *msm_mdss)
 {
 	int ret, i;
+	u32 hw_rev;
 
 	/*
 	 * Several components have AXI clocks that can only be turned on if
@@ -283,43 +278,15 @@ static int msm_mdss_enable(struct msm_mdss *msm_mdss)
 	if (msm_mdss->is_mdp5 || !msm_mdss->mdss_data)
 		return 0;
 
-	/*
-	 * ubwc config is part of the "mdss" region which is not accessible
-	 * from the rest of the driver. hardcode known configurations here
-	 *
-	 * Decoder version can be read from the UBWC_DEC_HW_VERSION reg,
-	 * UBWC_n and the rest of params comes from hw data.
-	 */
-	switch (msm_mdss->mdss_data->ubwc_dec_version) {
-	case 0: /* no UBWC */
-	case UBWC_1_0:
-		/* do nothing */
-		break;
-	case UBWC_2_0:
-		msm_mdss_setup_ubwc_dec_20(msm_mdss);
-		break;
-	case UBWC_3_0:
-		msm_mdss_setup_ubwc_dec_30(msm_mdss);
-		break;
-	case UBWC_4_0:
-	case UBWC_4_3:
-		msm_mdss_setup_ubwc_dec_40(msm_mdss);
-		break;
-	case UBWC_5_0:
-		msm_mdss_setup_ubwc_dec_50(msm_mdss);
-		break;
-	case UBWC_6_0:
-		msm_mdss_setup_ubwc_dec_50(msm_mdss);
-		break;
-	default:
-		dev_err(msm_mdss->dev, "Unsupported UBWC decoder version %x\n",
-			msm_mdss->mdss_data->ubwc_dec_version);
-		dev_err(msm_mdss->dev, "HW_REV: 0x%x\n",
-			readl_relaxed(msm_mdss->mmio + REG_MDSS_HW_VERSION));
-		dev_err(msm_mdss->dev, "UBWC_DEC_HW_VERSION: 0x%x\n",
-			readl_relaxed(msm_mdss->mmio + REG_MDSS_UBWC_DEC_HW_VERSION));
-		break;
-	}
+	hw_rev = readl_relaxed(msm_mdss->mmio + REG_MDSS_HW_VERSION);
+
+	if (hw_rev >= MDSS_HW_VER(6, 0, 0))
+		msm_mdss_6x_setup_ubwc(msm_mdss);
+	else if (hw_rev >= MDSS_HW_VER(5, 0, 0))
+		msm_mdss_5x_setup_ubwc(msm_mdss);
+	else if (hw_rev >= MDSS_HW_VER(4, 0, 0))
+		msm_mdss_4x_setup_ubwc(msm_mdss);
+	/* else UBWC 1.0 or none, no params to program */
 
 	return ret;
 }
@@ -550,6 +517,10 @@ static void mdss_remove(struct platform_device *pdev)
 	msm_mdss_destroy(mdss);
 }
 
+static const struct msm_mdss_data data_14k = {
+	.reg_bus_bw = 14000,
+};
+
 static const struct msm_mdss_data data_57k = {
 	.reg_bus_bw = 57000,
 };
@@ -571,6 +542,7 @@ static const struct of_device_id mdss_dt_match[] = {
 	{ .compatible = "qcom,eliza-mdss", .data = &data_57k },
 	{ .compatible = "qcom,glymur-mdss", .data = &data_57k },
 	{ .compatible = "qcom,kaanapali-mdss", .data = &data_57k },
+	{ .compatible = "qcom,milos-mdss", .data = &data_14k },
 	{ .compatible = "qcom,msm8998-mdss", .data = &data_76k8 },
 	{ .compatible = "qcom,qcm2290-mdss", .data = &data_76k8 },
 	{ .compatible = "qcom,qcs8300-mdss", .data = &data_74k },
diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c
index 30ddb5351e983..0d14c31bd4e47 100644
--- a/drivers/gpu/drm/msm/msm_ringbuffer.c
+++ b/drivers/gpu/drm/msm/msm_ringbuffer.c
@@ -16,13 +16,13 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)
 	struct msm_gem_submit *submit = to_msm_submit(job);
 	struct msm_fence_context *fctx = submit->ring->fctx;
 	struct msm_gpu *gpu = submit->gpu;
-	struct msm_drm_private *priv = gpu->dev->dev_private;
+	struct drm_device *dev = gpu->dev;
 	unsigned nr_cmds = submit->nr_cmds;
 	int i;
 
 	msm_fence_init(submit->hw_fence, fctx);
 
-	mutex_lock(&priv->lru.lock);
+	mutex_lock(&dev->gem_lru_mutex);
 
 	for (i = 0; i < submit->nr_bos; i++) {
 		struct drm_gem_object *obj = submit->bos[i].obj;
@@ -32,7 +32,7 @@ static struct dma_fence *msm_job_run(struct drm_sched_job *job)
 
 	submit->bos_pinned = false;
 
-	mutex_unlock(&priv->lru.lock);
+	mutex_unlock(&dev->gem_lru_mutex);
 
 	/* TODO move submit path over to using a per-ring lock.. */
 	mutex_lock(&gpu->lock);
@@ -67,7 +67,6 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id,
 {
 	struct drm_sched_init_args args = {
 		.ops = &msm_sched_ops,
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit = num_hw_submissions,
 		.timeout = MAX_SCHEDULE_TIMEOUT,
 		.dev = gpu->dev->dev,
diff --git a/drivers/gpu/drm/mxsfb/Kconfig b/drivers/gpu/drm/mxsfb/Kconfig
index 264e74f455547..31db7a824a93c 100644
--- a/drivers/gpu/drm/mxsfb/Kconfig
+++ b/drivers/gpu/drm/mxsfb/Kconfig
@@ -33,6 +33,8 @@ config DRM_IMX_LCDIF
 	select DRM_GEM_DMA_HELPER
 	select DRM_PANEL
 	select DRM_PANEL_BRIDGE
+	select DRM_DISPLAY_HELPER
+	select DRM_BRIDGE_CONNECTOR
 	help
 	  Choose this option if you have an LCDIFv3 LCD controller.
 	  Those devices are found in various i.MX SoC (i.MX8MP,
diff --git a/drivers/gpu/drm/mxsfb/lcdif_drv.c b/drivers/gpu/drm/mxsfb/lcdif_drv.c
index 47da1d9336b90..f5bb59cd50285 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_drv.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_drv.c
@@ -5,6 +5,7 @@
  * This code is based on drivers/gpu/drm/mxsfb/mxsfb*
  */
 
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
 #include <linux/io.h>
@@ -17,6 +18,7 @@
 #include <drm/clients/drm_client_setup.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
+#include <drm/drm_bridge_connector.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_encoder.h>
 #include <drm/drm_fbdev_dma.h>
@@ -48,62 +50,55 @@ static const struct drm_encoder_funcs lcdif_encoder_funcs = {
 static int lcdif_attach_bridge(struct lcdif_drm_private *lcdif)
 {
 	struct device *dev = lcdif->drm->dev;
-	struct device_node *ep;
-	struct drm_bridge *bridge;
-	int ret;
+	struct device_node *ep __free(device_node) = NULL;
 
 	for_each_endpoint_of_node(dev->of_node, ep) {
-		struct device_node *remote;
+		struct device_node *remote __free(device_node) =
+			of_graph_get_remote_port_parent(ep);
 		struct of_endpoint of_ep;
+		struct drm_bridge *bridge;
 		struct drm_encoder *encoder;
+		struct drm_connector *connector;
+		int ret;
 
-		remote = of_graph_get_remote_port_parent(ep);
-		if (!of_device_is_available(remote)) {
-			of_node_put(remote);
+		if (!of_device_is_available(remote))
 			continue;
-		}
-		of_node_put(remote);
 
 		ret = of_graph_parse_endpoint(ep, &of_ep);
-		if (ret < 0) {
-			dev_err(dev, "Failed to parse endpoint %pOF\n", ep);
-			of_node_put(ep);
-			return ret;
-		}
+		if (ret < 0)
+			return dev_err_probe(dev, ret, "Failed to parse endpoint %pOF\n", ep);
 
 		bridge = devm_drm_of_get_bridge(dev, dev->of_node, 0, of_ep.id);
-		if (IS_ERR(bridge)) {
-			of_node_put(ep);
+		if (IS_ERR(bridge))
 			return dev_err_probe(dev, PTR_ERR(bridge),
 					     "Failed to get bridge for endpoint%u\n",
 					     of_ep.id);
-		}
 
 		encoder = devm_kzalloc(dev, sizeof(*encoder), GFP_KERNEL);
-		if (!encoder) {
-			dev_err(dev, "Failed to allocate encoder for endpoint%u\n",
-				of_ep.id);
-			of_node_put(ep);
-			return -ENOMEM;
-		}
+		if (!encoder)
+			return dev_err_probe(dev, -ENOMEM,
+					     "Failed to allocate encoder for endpoint%u\n",
+					     of_ep.id);
 
 		encoder->possible_crtcs = drm_crtc_mask(&lcdif->crtc);
 		ret = drm_encoder_init(lcdif->drm, encoder, &lcdif_encoder_funcs,
 				       DRM_MODE_ENCODER_NONE, NULL);
-		if (ret) {
-			dev_err(dev, "Failed to initialize encoder for endpoint%u: %d\n",
-				of_ep.id, ret);
-			of_node_put(ep);
-			return ret;
-		}
-
-		ret = drm_bridge_attach(encoder, bridge, NULL, 0);
-		if (ret) {
-			of_node_put(ep);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to initialize encoder for endpoint%u\n",
+					     of_ep.id);
+
+		ret = drm_bridge_attach(encoder, bridge, NULL, DRM_BRIDGE_ATTACH_NO_CONNECTOR);
+		if (ret)
 			return dev_err_probe(dev, ret,
 					     "Failed to attach bridge for endpoint%u\n",
 					     of_ep.id);
-		}
+
+		connector = drm_bridge_connector_init(lcdif->drm, encoder);
+		if (IS_ERR(connector))
+			return dev_err_probe(dev, PTR_ERR(connector),
+					     "Failed to init bridge_connector for endpoint%u\n",
+					     of_ep.id);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/mxsfb/lcdif_kms.c b/drivers/gpu/drm/mxsfb/lcdif_kms.c
index ef3250a5c54f0..ade76c3f4e4e9 100644
--- a/drivers/gpu/drm/mxsfb/lcdif_kms.c
+++ b/drivers/gpu/drm/mxsfb/lcdif_kms.c
@@ -422,7 +422,7 @@ static void lcdif_crtc_mode_set_nofb(struct drm_crtc_state *crtc_state,
 }
 
 static int lcdif_crtc_atomic_check(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
@@ -503,7 +503,7 @@ static int lcdif_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void lcdif_crtc_atomic_flush(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct lcdif_drm_private *lcdif = to_lcdif_drm_private(crtc->dev);
 	struct drm_pending_vblank_event *event;
@@ -528,7 +528,7 @@ static void lcdif_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static void lcdif_crtc_atomic_enable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct lcdif_drm_private *lcdif = to_lcdif_drm_private(crtc->dev);
 	struct drm_crtc_state *new_cstate = drm_atomic_get_new_crtc_state(state, crtc);
@@ -558,7 +558,7 @@ static void lcdif_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void lcdif_crtc_atomic_disable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct lcdif_drm_private *lcdif = to_lcdif_drm_private(crtc->dev);
 	struct drm_device *drm = lcdif->drm;
@@ -664,7 +664,7 @@ static const struct drm_crtc_funcs lcdif_crtc_funcs = {
  */
 
 static int lcdif_plane_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state,
 									     plane);
@@ -681,7 +681,7 @@ static int lcdif_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void lcdif_plane_primary_atomic_update(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct lcdif_drm_private *lcdif = to_lcdif_drm_private(plane->dev);
 	struct drm_plane_state *new_pstate = drm_atomic_get_new_plane_state(state,
diff --git a/drivers/gpu/drm/mxsfb/mxsfb_kms.c b/drivers/gpu/drm/mxsfb/mxsfb_kms.c
index 8cac0a275b7da..d8ebebc5314bb 100644
--- a/drivers/gpu/drm/mxsfb/mxsfb_kms.c
+++ b/drivers/gpu/drm/mxsfb/mxsfb_kms.c
@@ -310,7 +310,7 @@ static void mxsfb_crtc_mode_set_nofb(struct mxsfb_drm_private *mxsfb,
 }
 
 static int mxsfb_crtc_atomic_check(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -326,7 +326,7 @@ static int mxsfb_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void mxsfb_crtc_atomic_flush(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_pending_vblank_event *event;
 
@@ -345,7 +345,7 @@ static void mxsfb_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static void mxsfb_crtc_atomic_enable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct mxsfb_drm_private *mxsfb = to_mxsfb_drm_private(crtc->dev);
 	struct drm_plane_state *new_pstate = drm_atomic_get_new_plane_state(state,
@@ -399,7 +399,7 @@ static void mxsfb_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void mxsfb_crtc_atomic_disable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct mxsfb_drm_private *mxsfb = to_mxsfb_drm_private(crtc->dev);
 	struct drm_device *drm = mxsfb->drm;
@@ -520,7 +520,7 @@ static const struct drm_encoder_funcs mxsfb_encoder_funcs = {
  */
 
 static int mxsfb_plane_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state,
 									     plane);
@@ -537,7 +537,7 @@ static int mxsfb_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void mxsfb_plane_primary_atomic_update(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct mxsfb_drm_private *mxsfb = to_mxsfb_drm_private(plane->dev);
 	struct drm_plane_state *new_pstate = drm_atomic_get_new_plane_state(state,
@@ -550,7 +550,7 @@ static void mxsfb_plane_primary_atomic_update(struct drm_plane *plane,
 }
 
 static void mxsfb_plane_overlay_atomic_update(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_pstate = drm_atomic_get_old_plane_state(state,
 									    plane);
@@ -613,7 +613,7 @@ static void mxsfb_plane_overlay_atomic_update(struct drm_plane *plane,
 }
 
 static void mxsfb_plane_overlay_atomic_disable(struct drm_plane *plane,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct mxsfb_drm_private *mxsfb = to_mxsfb_drm_private(plane->dev);
 
diff --git a/drivers/gpu/drm/nouveau/dispnv50/atom.h b/drivers/gpu/drm/nouveau/dispnv50/atom.h
index b43c4f9bbcdf5..c34a42ea9ac47 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/atom.h
+++ b/drivers/gpu/drm/nouveau/dispnv50/atom.h
@@ -7,7 +7,7 @@
 struct nouveau_encoder;
 
 struct nv50_atom {
-	struct drm_atomic_state state;
+	struct drm_atomic_commit state;
 
 	struct list_head outp;
 	bool lock_core;
@@ -149,7 +149,7 @@ struct nv50_head_atom {
 };
 
 static inline struct nv50_head_atom *
-nv50_head_atom_get(struct drm_atomic_state *state, struct drm_crtc *crtc)
+nv50_head_atom_get(struct drm_atomic_commit *state, struct drm_crtc *crtc)
 {
 	struct drm_crtc_state *statec = drm_atomic_get_crtc_state(state, crtc);
 
@@ -160,7 +160,7 @@ nv50_head_atom_get(struct drm_atomic_state *state, struct drm_crtc *crtc)
 }
 
 static inline struct nv50_head_atom *
-nv50_head_atom_get_new(struct drm_atomic_state *state, struct drm_crtc *crtc)
+nv50_head_atom_get_new(struct drm_atomic_commit *state, struct drm_crtc *crtc)
 {
 	struct drm_crtc_state *statec = drm_atomic_get_new_crtc_state(state, crtc);
 
diff --git a/drivers/gpu/drm/nouveau/dispnv50/crc.c b/drivers/gpu/drm/nouveau/dispnv50/crc.c
index deb6af40ef328..cb74bbec5e9c6 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/crc.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/crc.c
@@ -16,7 +16,6 @@
 #include "nouveau_drv.h"
 #include "core.h"
 #include "head.h"
-#include "wndw.h"
 #include "handles.h"
 #include "crc.h"
 
@@ -237,7 +236,7 @@ static void nv50_crc_wait_ctx_finished(struct nv50_head *head,
 			  head->base.index, ret);
 }
 
-void nv50_crc_atomic_stop_reporting(struct drm_atomic_state *state)
+void nv50_crc_atomic_stop_reporting(struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state;
 	struct drm_crtc *crtc;
@@ -269,7 +268,7 @@ void nv50_crc_atomic_stop_reporting(struct drm_atomic_state *state)
 	}
 }
 
-void nv50_crc_atomic_init_notifier_contexts(struct drm_atomic_state *state)
+void nv50_crc_atomic_init_notifier_contexts(struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_crtc *crtc;
@@ -291,7 +290,7 @@ void nv50_crc_atomic_init_notifier_contexts(struct drm_atomic_state *state)
 	}
 }
 
-void nv50_crc_atomic_release_notifier_contexts(struct drm_atomic_state *state)
+void nv50_crc_atomic_release_notifier_contexts(struct drm_atomic_commit *state)
 {
 	const struct nv50_crc_func *func =
 		nv50_disp(state->dev)->core->func->crc;
@@ -316,7 +315,7 @@ void nv50_crc_atomic_release_notifier_contexts(struct drm_atomic_state *state)
 	}
 }
 
-void nv50_crc_atomic_start_reporting(struct drm_atomic_state *state)
+void nv50_crc_atomic_start_reporting(struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state;
 	struct drm_crtc *crtc;
@@ -544,7 +543,7 @@ nv50_crc_ctx_fini(struct nv50_crc_notifier_ctx *ctx)
 int nv50_crc_set_source(struct drm_crtc *crtc, const char *source_str)
 {
 	struct drm_device *dev = crtc->dev;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_modeset_acquire_ctx ctx;
 	struct nv50_head *head = nv50_head(crtc);
 	struct nv50_crc *crc = &head->crc;
@@ -567,7 +566,7 @@ int nv50_crc_set_source(struct drm_crtc *crtc, const char *source_str)
 		ctx_flags |= DRM_MODESET_ACQUIRE_INTERRUPTIBLE;
 	drm_modeset_acquire_init(&ctx, ctx_flags);
 
-	state = drm_atomic_state_alloc(dev);
+	state = drm_atomic_commit_alloc(dev);
 	if (!state) {
 		ret = -ENOMEM;
 		goto out_acquire_fini;
@@ -617,13 +616,13 @@ int nv50_crc_set_source(struct drm_crtc *crtc, const char *source_str)
 		for (i = 0; i < ARRAY_SIZE(crc->ctx); i++)
 			nv50_crc_ctx_fini(&crc->ctx[i]);
 	}
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 out_acquire_fini:
 	drm_modeset_acquire_fini(&ctx);
 	return ret;
 
 deadlock:
-	drm_atomic_state_clear(state);
+	drm_atomic_commit_clear(state);
 	drm_modeset_backoff(&ctx);
 	goto retry;
 }
diff --git a/drivers/gpu/drm/nouveau/dispnv50/crc.h b/drivers/gpu/drm/nouveau/dispnv50/crc.h
index 75a2009e81934..cefa2dd52f12d 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/crc.h
+++ b/drivers/gpu/drm/nouveau/dispnv50/crc.h
@@ -84,10 +84,10 @@ int nv50_crc_set_source(struct drm_crtc *, const char *);
 int nv50_crc_atomic_check_head(struct nv50_head *, struct nv50_head_atom *,
 			       struct nv50_head_atom *);
 void nv50_crc_atomic_check_outp(struct nv50_atom *atom);
-void nv50_crc_atomic_stop_reporting(struct drm_atomic_state *);
-void nv50_crc_atomic_init_notifier_contexts(struct drm_atomic_state *);
-void nv50_crc_atomic_release_notifier_contexts(struct drm_atomic_state *);
-void nv50_crc_atomic_start_reporting(struct drm_atomic_state *);
+void nv50_crc_atomic_stop_reporting(struct drm_atomic_commit *);
+void nv50_crc_atomic_init_notifier_contexts(struct drm_atomic_commit *);
+void nv50_crc_atomic_release_notifier_contexts(struct drm_atomic_commit *);
+void nv50_crc_atomic_start_reporting(struct drm_atomic_commit *);
 void nv50_crc_atomic_set(struct nv50_head *, struct nv50_head_atom *);
 void nv50_crc_atomic_clr(struct nv50_head *);
 
@@ -116,13 +116,13 @@ nv50_crc_atomic_check_head(struct nv50_head *head,
 			   struct nv50_head_atom *armh) { return 0; }
 static inline void nv50_crc_atomic_check_outp(struct nv50_atom *atom) {}
 static inline void
-nv50_crc_atomic_stop_reporting(struct drm_atomic_state *state) {}
+nv50_crc_atomic_stop_reporting(struct drm_atomic_commit *state) {}
 static inline void
-nv50_crc_atomic_init_notifier_contexts(struct drm_atomic_state *state) {}
+nv50_crc_atomic_init_notifier_contexts(struct drm_atomic_commit *state) {}
 static inline void
-nv50_crc_atomic_release_notifier_contexts(struct drm_atomic_state *state) {}
+nv50_crc_atomic_release_notifier_contexts(struct drm_atomic_commit *state) {}
 static inline void
-nv50_crc_atomic_start_reporting(struct drm_atomic_state *state) {}
+nv50_crc_atomic_start_reporting(struct drm_atomic_commit *state) {}
 static inline void
 nv50_crc_atomic_set(struct nv50_head *head, struct nv50_head_atom *state) {}
 static inline void
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
index 6c3a8712d38ab..364227f5456f1 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -428,7 +428,7 @@ nv50_outp_atomic_check(struct drm_encoder *encoder,
 }
 
 struct nouveau_connector *
-nv50_outp_get_new_connector(struct drm_atomic_state *state, struct nouveau_encoder *outp)
+nv50_outp_get_new_connector(struct drm_atomic_commit *state, struct nouveau_encoder *outp)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *connector_state;
@@ -444,7 +444,7 @@ nv50_outp_get_new_connector(struct drm_atomic_state *state, struct nouveau_encod
 }
 
 struct nouveau_connector *
-nv50_outp_get_old_connector(struct drm_atomic_state *state, struct nouveau_encoder *outp)
+nv50_outp_get_old_connector(struct drm_atomic_commit *state, struct nouveau_encoder *outp)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *connector_state;
@@ -460,7 +460,7 @@ nv50_outp_get_old_connector(struct drm_atomic_state *state, struct nouveau_encod
 }
 
 static struct nouveau_crtc *
-nv50_outp_get_new_crtc(const struct drm_atomic_state *state, const struct nouveau_encoder *outp)
+nv50_outp_get_new_crtc(const struct drm_atomic_commit *state, const struct nouveau_encoder *outp)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
@@ -479,7 +479,7 @@ nv50_outp_get_new_crtc(const struct drm_atomic_state *state, const struct nouvea
  * DAC
  *****************************************************************************/
 static void
-nv50_dac_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
+nv50_dac_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_commit *state)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nv50_core *core = nv50_disp(encoder->dev)->core;
@@ -490,7 +490,7 @@ nv50_dac_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *st
 }
 
 static void
-nv50_dac_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
+nv50_dac_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_commit *state)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nv50_outp_get_new_crtc(state, nv_encoder);
@@ -750,7 +750,7 @@ nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
 
 static void
 nv50_audio_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc,
-		  struct nouveau_connector *nv_connector, struct drm_atomic_state *state,
+		  struct nouveau_connector *nv_connector, struct drm_atomic_commit *state,
 		  struct drm_display_mode *mode)
 {
 	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
@@ -776,7 +776,7 @@ nv50_audio_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc,
  *****************************************************************************/
 static void
 nv50_hdmi_enable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc,
-		 struct nouveau_connector *nv_connector, struct drm_atomic_state *state,
+		 struct nouveau_connector *nv_connector, struct drm_atomic_commit *state,
 		 struct drm_display_mode *mode, bool hda)
 {
 	struct nouveau_drm *drm = nouveau_drm(encoder->dev);
@@ -892,7 +892,7 @@ struct nouveau_encoder *nv50_real_outp(struct drm_encoder *encoder)
 }
 
 static void
-nv50_msto_cleanup(struct drm_atomic_state *state,
+nv50_msto_cleanup(struct drm_atomic_commit *state,
 		  struct drm_dp_mst_topology_state *new_mst_state,
 		  struct drm_dp_mst_topology_mgr *mgr,
 		  struct nv50_msto *msto)
@@ -925,7 +925,7 @@ nv50_msto_cleanup(struct drm_atomic_state *state,
 }
 
 static void
-nv50_msto_prepare(struct drm_atomic_state *state,
+nv50_msto_prepare(struct drm_atomic_commit *state,
 		  struct drm_dp_mst_topology_state *mst_state,
 		  struct drm_dp_mst_topology_mgr *mgr,
 		  struct nv50_msto *msto)
@@ -964,7 +964,7 @@ nv50_msto_atomic_check(struct drm_encoder *encoder,
 		       struct drm_crtc_state *crtc_state,
 		       struct drm_connector_state *conn_state)
 {
-	struct drm_atomic_state *state = crtc_state->state;
+	struct drm_atomic_commit *state = crtc_state->state;
 	struct drm_connector *connector = conn_state->connector;
 	struct drm_dp_mst_topology_state *mst_state;
 	struct nv50_mstc *mstc = nv50_mstc(connector);
@@ -1024,7 +1024,7 @@ nv50_dp_bpc_to_depth(unsigned int bpc)
 }
 
 static void
-nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
+nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_commit *state)
 {
 	struct nv50_msto *msto = nv50_msto(encoder);
 	struct nv50_head *head = msto->head;
@@ -1073,7 +1073,7 @@ nv50_msto_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *st
 }
 
 static void
-nv50_msto_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
+nv50_msto_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_commit *state)
 {
 	struct nv50_msto *msto = nv50_msto(encoder);
 	struct nv50_mstc *mstc = msto->mstc;
@@ -1134,7 +1134,7 @@ nv50_msto_new(struct drm_device *dev, struct nv50_head *head, int id)
 
 static struct drm_encoder *
 nv50_mstc_atomic_best_encoder(struct drm_connector *connector,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *connector_state = drm_atomic_get_new_connector_state(state,
 											 connector);
@@ -1192,7 +1192,7 @@ nv50_mstc_get_modes(struct drm_connector *connector)
 
 static int
 nv50_mstc_atomic_check(struct drm_connector *connector,
-		       struct drm_atomic_state *state)
+		       struct drm_atomic_commit *state)
 {
 	struct nv50_mstc *mstc = nv50_mstc(connector);
 	struct drm_dp_mst_topology_mgr *mgr = &mstc->mstm->mgr;
@@ -1301,7 +1301,7 @@ nv50_mstc_new(struct nv50_mstm *mstm, struct drm_dp_mst_port *port,
 }
 
 static void
-nv50_mstm_cleanup(struct drm_atomic_state *state,
+nv50_mstm_cleanup(struct drm_atomic_commit *state,
 		  struct drm_dp_mst_topology_state *mst_state,
 		  struct nv50_mstm *mstm)
 {
@@ -1330,7 +1330,7 @@ nv50_mstm_cleanup(struct drm_atomic_state *state,
 }
 
 static void
-nv50_mstm_prepare(struct drm_atomic_state *state,
+nv50_mstm_prepare(struct drm_atomic_commit *state,
 		  struct drm_dp_mst_topology_state *mst_state,
 		  struct nv50_mstm *mstm)
 {
@@ -1562,7 +1562,7 @@ nv50_sor_update(struct nouveau_encoder *nv_encoder, u8 head,
  * fixed time delay from the vbios…
  */
 static void
-nv50_sor_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
+nv50_sor_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_commit *state)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nv50_head *head = nv50_head(nv_encoder->crtc);
@@ -1741,7 +1741,7 @@ nv50_sor_dp_watermark_sst(struct nouveau_encoder *outp,
 }
 
 static void
-nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
+nv50_sor_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_commit *state)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nv50_outp_get_new_crtc(state, nv_encoder);
@@ -1967,7 +1967,7 @@ nv50_pior_atomic_check(struct drm_encoder *encoder,
 }
 
 static void
-nv50_pior_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *state)
+nv50_pior_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_commit *state)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nv50_core *core = nv50_disp(encoder->dev)->core;
@@ -1978,7 +1978,7 @@ nv50_pior_atomic_disable(struct drm_encoder *encoder, struct drm_atomic_state *s
 }
 
 static void
-nv50_pior_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_state *state)
+nv50_pior_atomic_enable(struct drm_encoder *encoder, struct drm_atomic_commit *state)
 {
 	struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
 	struct nouveau_crtc *nv_crtc = nv50_outp_get_new_crtc(state, nv_encoder);
@@ -2099,7 +2099,7 @@ nv50_pior_create(struct nouveau_encoder *nv_encoder)
  *****************************************************************************/
 
 static void
-nv50_disp_atomic_commit_core(struct drm_atomic_state *state, u32 *interlock)
+nv50_disp_atomic_commit_core(struct drm_atomic_commit *state, u32 *interlock)
 {
 	struct drm_dp_mst_topology_mgr *mgr;
 	struct drm_dp_mst_topology_state *mst_state;
@@ -2150,7 +2150,7 @@ nv50_disp_atomic_commit_core(struct drm_atomic_state *state, u32 *interlock)
 }
 
 static void
-nv50_disp_atomic_commit_wndw(struct drm_atomic_state *state, u32 *interlock)
+nv50_disp_atomic_commit_wndw(struct drm_atomic_commit *state, u32 *interlock)
 {
 	struct drm_plane_state *new_plane_state;
 	struct drm_plane *plane;
@@ -2166,7 +2166,7 @@ nv50_disp_atomic_commit_wndw(struct drm_atomic_state *state, u32 *interlock)
 }
 
 static void
-nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
+nv50_disp_atomic_commit_tail(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct drm_crtc_state *new_crtc_state, *old_crtc_state;
@@ -2408,7 +2408,7 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 	drm_atomic_helper_commit_hw_done(state);
 	drm_atomic_helper_cleanup_planes(dev, state);
 	drm_atomic_helper_commit_cleanup_done(state);
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 	/* Drop the RPM ref we got from nv50_disp_atomic_commit() */
 	pm_runtime_mark_last_busy(dev->dev);
@@ -2418,14 +2418,14 @@ nv50_disp_atomic_commit_tail(struct drm_atomic_state *state)
 static void
 nv50_disp_atomic_commit_work(struct work_struct *work)
 {
-	struct drm_atomic_state *state =
+	struct drm_atomic_commit *state =
 		container_of(work, typeof(*state), commit_work);
 	nv50_disp_atomic_commit_tail(state);
 }
 
 static int
 nv50_disp_atomic_commit(struct drm_device *dev,
-			struct drm_atomic_state *state, bool nonblock)
+			struct drm_atomic_commit *state, bool nonblock)
 {
 	struct drm_plane_state *new_plane_state;
 	struct drm_plane *plane;
@@ -2465,7 +2465,7 @@ nv50_disp_atomic_commit(struct drm_device *dev,
 			nv50_wndw_ntfy_enable(wndw, asyw);
 	}
 
-	drm_atomic_state_get(state);
+	drm_atomic_commit_get(state);
 
 	/*
 	 * Grab another RPM ref for the commit tail, which will release the
@@ -2474,7 +2474,7 @@ nv50_disp_atomic_commit(struct drm_device *dev,
 	pm_runtime_get_noresume(dev->dev);
 
 	if (nonblock)
-		queue_work(system_unbound_wq, &state->commit_work);
+		queue_work(system_dfl_wq, &state->commit_work);
 	else
 		nv50_disp_atomic_commit_tail(state);
 
@@ -2560,7 +2560,7 @@ nv50_disp_outp_atomic_check_set(struct nv50_atom *atom,
 }
 
 static int
-nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct nv50_atom *atom = nv50_atom(state);
 	struct nv50_core *core = nv50_disp(dev)->core;
@@ -2618,7 +2618,7 @@ nv50_disp_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 }
 
 static void
-nv50_disp_atomic_state_clear(struct drm_atomic_state *state)
+nv50_disp_atomic_state_clear(struct drm_atomic_commit *state)
 {
 	struct nv50_atom *atom = nv50_atom(state);
 	struct nv50_outp_atom *outp, *outt;
@@ -2628,23 +2628,23 @@ nv50_disp_atomic_state_clear(struct drm_atomic_state *state)
 		kfree(outp);
 	}
 
-	drm_atomic_state_default_clear(state);
+	drm_atomic_commit_default_clear(state);
 }
 
 static void
-nv50_disp_atomic_state_free(struct drm_atomic_state *state)
+nv50_disp_atomic_state_free(struct drm_atomic_commit *state)
 {
 	struct nv50_atom *atom = nv50_atom(state);
-	drm_atomic_state_default_release(&atom->state);
+	drm_atomic_commit_default_release(&atom->state);
 	kfree(atom);
 }
 
-static struct drm_atomic_state *
+static struct drm_atomic_commit *
 nv50_disp_atomic_state_alloc(struct drm_device *dev)
 {
 	struct nv50_atom *atom;
 	if (!(atom = kzalloc_obj(*atom)) ||
-	    drm_atomic_state_init(dev, &atom->state) < 0) {
+	    drm_atomic_commit_init(dev, &atom->state) < 0) {
 		kfree(atom);
 		return NULL;
 	}
diff --git a/drivers/gpu/drm/nouveau/dispnv50/head.c b/drivers/gpu/drm/nouveau/dispnv50/head.c
index 4ca7700fb784d..1affa0378bd02 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/head.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/head.c
@@ -330,7 +330,7 @@ nv50_head_atomic_check_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
 }
 
 static int
-nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
+nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
 									      crtc);
diff --git a/drivers/gpu/drm/nouveau/dispnv50/wndw.c b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
index 9ad4973716f7f..2635458d52acc 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/wndw.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/wndw.c
@@ -441,7 +441,7 @@ nv50_wndw_atomic_check_lut(struct nv50_wndw *wndw,
 
 static int
 nv50_wndw_atomic_check(struct drm_plane *plane,
-		       struct drm_atomic_state *state)
+		       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index 7860877d909b0..291203121f0c2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -176,7 +176,7 @@ nouveau_abi16_chan_fini(struct nouveau_abi16 *abi16,
 
 	/* Cancel all jobs from the entity's queue. */
 	if (chan->sched)
-		drm_sched_entity_fini(&chan->sched->entity);
+		drm_sched_entity_kill(&chan->sched->entity);
 
 	if (chan->chan)
 		nouveau_channel_idle(chan->chan);
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index c8335f5b49dbf..38032bb958262 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -2085,10 +2085,27 @@ nouveau_bios_init(struct drm_device *dev)
 	int ret;
 
 	/* only relevant for PCI devices */
-	if (!dev_is_pci(dev->dev) ||
-	    nvkm_gsp_rm(nvxx_device(drm)->gsp))
+	if (!dev_is_pci(dev->dev))
 		return 0;
 
+	if (nvkm_gsp_rm(nvxx_device(drm)->gsp)) {
+		struct nvkm_bios *nvkm_bios = nvxx_bios(drm);
+
+		/*
+		 * If this GPU has an nvkm_device_chip.bios entry, then the VBIOS
+		 * data was already read by the nvkm layer during nvkm_bios_new().
+		 * Point the legacy DRM-level VBIOS structure at the same buffer
+		 * so that any remaining legacy code can access it. This exposes
+		 * the VBIOS via the DRM debugfs entries.
+		 */
+		if (nvkm_bios) {
+			bios->data = nvkm_bios->data;
+			bios->length = nvkm_bios->size;
+		}
+
+		return 0;
+	}
+
 	if (!NVInitVBIOS(dev))
 		return -ENODEV;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index cc239492c7f0b..b0b0ad9a0c248 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -1132,7 +1132,7 @@ nouveau_connector_best_encoder(struct drm_connector *connector)
 }
 
 static int
-nouveau_connector_atomic_check(struct drm_connector *connector, struct drm_atomic_state *state)
+nouveau_connector_atomic_check(struct drm_connector *connector, struct drm_atomic_commit *state)
 {
 	struct nouveau_connector *nv_conn = nouveau_connector(connector);
 	struct drm_connector_state *conn_state =
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h
index 470e0910d4845..e711c89723e80 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.h
+++ b/drivers/gpu/drm/nouveau/nouveau_display.h
@@ -34,7 +34,7 @@ struct nouveau_display {
 	struct drm_property *vibrant_hue_property;
 	struct drm_property *color_vibrance_property;
 
-	struct drm_atomic_state *suspend;
+	struct drm_atomic_commit *suspend;
 
 	const u64 *format_modifiers;
 };
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 517ff2c31dce9..e16f59b00f6fa 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -632,7 +632,7 @@ nouveau_drm_device_init(struct nouveau_drm *drm)
 	struct drm_device *dev = drm->dev;
 	int ret;
 
-	drm->sched_wq = alloc_workqueue("nouveau_sched_wq_shared", 0,
+	drm->sched_wq = alloc_workqueue("nouveau_sched_wq_shared", WQ_PERCPU,
 					WQ_MAX_ACTIVE);
 	if (!drm->sched_wq)
 		return -ENOMEM;
diff --git a/drivers/gpu/drm/nouveau/nouveau_encoder.h b/drivers/gpu/drm/nouveau/nouveau_encoder.h
index dce8e5d9d4964..4422c6185d498 100644
--- a/drivers/gpu/drm/nouveau/nouveau_encoder.h
+++ b/drivers/gpu/drm/nouveau/nouveau_encoder.h
@@ -165,9 +165,9 @@ enum drm_mode_status nv50_dp_mode_valid(struct nouveau_encoder *,
 					unsigned *clock);
 
 struct nouveau_connector *
-nv50_outp_get_new_connector(struct drm_atomic_state *state, struct nouveau_encoder *outp);
+nv50_outp_get_new_connector(struct drm_atomic_commit *state, struct nouveau_encoder *outp);
 struct nouveau_connector *
-nv50_outp_get_old_connector(struct drm_atomic_state *state, struct nouveau_encoder *outp);
+nv50_outp_get_old_connector(struct drm_atomic_commit *state, struct nouveau_encoder *outp);
 
 int nv50_mstm_detect(struct nouveau_encoder *encoder);
 void nv50_mstm_remove(struct nv50_mstm *mstm);
diff --git a/drivers/gpu/drm/nouveau/nouveau_sched.c b/drivers/gpu/drm/nouveau/nouveau_sched.c
index 5546be4887954..2cbae003d6de5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sched.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sched.c
@@ -405,7 +405,6 @@ nouveau_sched_init(struct nouveau_sched *sched, struct nouveau_drm *drm,
 	struct drm_sched_entity *entity = &sched->entity;
 	struct drm_sched_init_args args = {
 		.ops = &nouveau_sched_ops,
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit = credit_limit,
 		.timeout = msecs_to_jiffies(NOUVEAU_SCHED_JOB_TIMEOUT_MS),
 		.name = "nouveau_sched",
@@ -414,7 +413,8 @@ nouveau_sched_init(struct nouveau_sched *sched, struct nouveau_drm *drm,
 	int ret;
 
 	if (!wq) {
-		wq = alloc_workqueue("nouveau_sched_wq_%d", 0, WQ_MAX_ACTIVE,
+		wq = alloc_workqueue("nouveau_sched_wq_%d", WQ_PERCPU,
+				     WQ_MAX_ACTIVE,
 				     current->pid);
 		if (!wq)
 			return -ENOMEM;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c
index 6e63df816d855..1155f079b0c3d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/gsp/rm/r535/disp.c
@@ -1734,7 +1734,8 @@ r535_disp_new(const struct nvkm_disp_func *hw, struct nvkm_device *device,
 	struct nvkm_disp_func *rm;
 	int ret;
 
-	if (!(rm = kzalloc(sizeof(*rm) + 6 * sizeof(rm->user[0]), GFP_KERNEL)))
+	rm = kzalloc_flex(*rm, user, 6);
+	if (!rm)
 		return -ENOMEM;
 
 	rm->dtor = r535_disp_dtor;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
index 958fd78080bdb..107bdb642f229 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmm.c
@@ -53,7 +53,8 @@ nvkm_vmm_pt_new(const struct nvkm_vmm_desc *desc, bool sparse,
 		}
 	}
 
-	if (!(pgt = kzalloc(sizeof(*pgt) + (sizeof(pgt->pte[0]) * lpte), GFP_KERNEL)))
+	pgt = kzalloc_flex(*pgt, pte, lpte);
+	if (!pgt)
 		return NULL;
 	pgt->page = page ? page->shift : 0;
 	pgt->sparse = sparse;
diff --git a/drivers/gpu/drm/nova/Kconfig b/drivers/gpu/drm/nova/Kconfig
index 3e637ad7b5bad..ba16c74401f8f 100644
--- a/drivers/gpu/drm/nova/Kconfig
+++ b/drivers/gpu/drm/nova/Kconfig
@@ -4,6 +4,7 @@ config DRM_NOVA
 	depends on DRM=y
 	depends on PCI
 	depends on RUST
+	depends on !CPU_BIG_ENDIAN
 	select AUXILIARY_BUS
 	select NOVA_CORE
 	default n
@@ -13,4 +14,4 @@ config DRM_NOVA
 
 	  This driver is work in progress and may not be functional.
 
-	  If M is selected, the module will be called nova.
+	  If M is selected, the module will be called nova-drm.
diff --git a/drivers/gpu/drm/nova/Makefile b/drivers/gpu/drm/nova/Makefile
index 42019bff31731..f8527b2b7b4ab 100644
--- a/drivers/gpu/drm/nova/Makefile
+++ b/drivers/gpu/drm/nova/Makefile
@@ -1,3 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-$(CONFIG_DRM_NOVA) += nova.o
+obj-$(CONFIG_DRM_NOVA) += nova-drm.o
+nova-drm-y := nova.o
diff --git a/drivers/gpu/drm/nova/driver.rs b/drivers/gpu/drm/nova/driver.rs
index b1af0a099551d..e3de04f358f06 100644
--- a/drivers/gpu/drm/nova/driver.rs
+++ b/drivers/gpu/drm/nova/driver.rs
@@ -32,11 +32,11 @@ pub(crate) struct NovaData {
     major: 0,
     minor: 0,
     patchlevel: 0,
-    name: c"nova",
-    desc: c"Nvidia Graphics",
+    name: c"nova-drm",
+    desc: c"NVIDIA Graphics and Compute",
 };
 
-const NOVA_CORE_MODULE_NAME: &CStr = c"NovaCore";
+const NOVA_CORE_MODULE_NAME: &CStr = c"nova-core";
 const AUXILIARY_NAME: &CStr = c"nova-drm";
 
 kernel::auxiliary_device_table!(
diff --git a/drivers/gpu/drm/nova/nova.rs b/drivers/gpu/drm/nova/nova.rs
index 8893e58ee0dbe..1fd454c7e0dfb 100644
--- a/drivers/gpu/drm/nova/nova.rs
+++ b/drivers/gpu/drm/nova/nova.rs
@@ -10,7 +10,7 @@
 
 kernel::module_auxiliary_driver! {
     type: NovaDriver,
-    name: "Nova",
+    name: "nova-drm",
     authors: ["Danilo Krummrich"],
     description: "Nova GPU driver",
     license: "GPL v2",
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
index 29b2dfb90b5fa..e306247ed8a05 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
@@ -342,7 +342,7 @@ static void hdmi4_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void hdmi4_bridge_enable(struct drm_bridge *bridge,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct omap_hdmi *hdmi = drm_bridge_to_hdmi(bridge);
 	struct drm_connector_state *conn_state;
@@ -410,7 +410,7 @@ static void hdmi4_bridge_enable(struct drm_bridge *bridge,
 }
 
 static void hdmi4_bridge_disable(struct drm_bridge *bridge,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct omap_hdmi *hdmi = drm_bridge_to_hdmi(bridge);
 	unsigned long flags;
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
index 5636b3dfec1c9..ab2d4eb6787f2 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
@@ -340,7 +340,7 @@ static void hdmi5_bridge_mode_set(struct drm_bridge *bridge,
 }
 
 static void hdmi5_bridge_enable(struct drm_bridge *bridge,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct omap_hdmi *hdmi = drm_bridge_to_hdmi(bridge);
 	struct drm_connector_state *conn_state;
@@ -408,7 +408,7 @@ static void hdmi5_bridge_enable(struct drm_bridge *bridge,
 }
 
 static void hdmi5_bridge_disable(struct drm_bridge *bridge,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct omap_hdmi *hdmi = drm_bridge_to_hdmi(bridge);
 	unsigned long flags;
diff --git a/drivers/gpu/drm/omapdrm/dss/output.c b/drivers/gpu/drm/omapdrm/dss/output.c
index 7378e855c278c..ca891aba38209 100644
--- a/drivers/gpu/drm/omapdrm/dss/output.c
+++ b/drivers/gpu/drm/omapdrm/dss/output.c
@@ -30,11 +30,13 @@ int omapdss_device_init_output(struct omap_dss_device *out,
 		return 0;
 	}
 
-	out->bridge = of_drm_find_bridge(remote_node);
 	out->panel = of_drm_find_panel(remote_node);
 	if (IS_ERR(out->panel))
 		out->panel = NULL;
 
+	if (!out->panel)
+		out->bridge = of_drm_find_and_get_bridge(remote_node);
+
 	of_node_put(remote_node);
 
 	if (out->panel) {
@@ -49,7 +51,7 @@ int omapdss_device_init_output(struct omap_dss_device *out,
 			goto error;
 		}
 
-		out->bridge = bridge;
+		out->bridge = drm_bridge_get(bridge);
 	}
 
 	if (local_bridge) {
@@ -59,7 +61,7 @@ int omapdss_device_init_output(struct omap_dss_device *out,
 		}
 
 		out->next_bridge = out->bridge;
-		out->bridge = local_bridge;
+		out->bridge = drm_bridge_get(local_bridge);
 	}
 
 	if (!out->bridge) {
@@ -79,6 +81,9 @@ void omapdss_device_cleanup_output(struct omap_dss_device *out)
 	if (out->bridge && out->panel)
 		drm_panel_bridge_remove(out->next_bridge ?
 					out->next_bridge : out->bridge);
+
+	drm_bridge_put(out->next_bridge);
+	drm_bridge_put(out->bridge);
 }
 
 void dss_mgr_set_timings(struct omap_dss_device *dssdev,
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index ee4c0119d4089..c1b41e010a004 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -446,7 +446,7 @@ static void omap_crtc_arm_event(struct drm_crtc *crtc)
 }
 
 static void omap_crtc_atomic_enable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct omap_drm_private *priv = crtc->dev->dev_private;
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
@@ -472,7 +472,7 @@ static void omap_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void omap_crtc_atomic_disable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct omap_drm_private *priv = crtc->dev->dev_private;
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
@@ -577,7 +577,7 @@ static bool omap_crtc_is_manually_updated(struct drm_crtc *crtc)
 }
 
 static int omap_crtc_atomic_check(struct drm_crtc *crtc,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -609,12 +609,12 @@ static int omap_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void omap_crtc_atomic_begin(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 }
 
 static void omap_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct omap_drm_private *priv = crtc->dev->dev_private;
 	struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index ae678696fbacb..92d6a1f9c0a1a 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -46,7 +46,7 @@
  */
 
 static void omap_atomic_wait_for_completion(struct drm_device *dev,
-					    struct drm_atomic_state *old_state)
+					    struct drm_atomic_commit *old_state)
 {
 	struct drm_crtc_state *new_crtc_state;
 	struct drm_crtc *crtc;
@@ -65,7 +65,7 @@ static void omap_atomic_wait_for_completion(struct drm_device *dev,
 	}
 }
 
-static void omap_atomic_commit_tail(struct drm_atomic_state *old_state)
+static void omap_atomic_commit_tail(struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = old_state->dev;
 	struct omap_drm_private *priv = dev->dev_private;
@@ -117,7 +117,7 @@ static void omap_atomic_commit_tail(struct drm_atomic_state *old_state)
 	dispc_runtime_put(priv->dispc);
 }
 
-static int drm_atomic_state_normalized_zpos_cmp(const void *a, const void *b)
+static int drm_atomic_commit_normalized_zpos_cmp(const void *a, const void *b)
 {
 	const struct drm_plane_state *sa = *(struct drm_plane_state **)a;
 	const struct drm_plane_state *sb = *(struct drm_plane_state **)b;
@@ -136,7 +136,7 @@ static int drm_atomic_state_normalized_zpos_cmp(const void *a, const void *b)
  * planes zpos is consistent.
  */
 static int omap_atomic_update_normalize_zpos(struct drm_device *dev,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *old_state, *new_state;
@@ -174,7 +174,7 @@ static int omap_atomic_update_normalize_zpos(struct drm_device *dev,
 		}
 
 		sort(states, n, sizeof(*states),
-		     drm_atomic_state_normalized_zpos_cmp, NULL);
+		     drm_atomic_commit_normalized_zpos_cmp, NULL);
 
 		for (i = 0, inc = 0; i < n; i++) {
 			plane = states[i]->plane;
@@ -196,7 +196,7 @@ static int omap_atomic_update_normalize_zpos(struct drm_device *dev,
 }
 
 static int omap_atomic_check(struct drm_device *dev,
-			     struct drm_atomic_state *state)
+			     struct drm_atomic_commit *state)
 {
 	int ret;
 
@@ -241,7 +241,7 @@ omap_get_existing_global_state(struct omap_drm_private *priv)
  * a new duplicated private object state.
  */
 struct omap_global_state *__must_check
-omap_get_global_state(struct drm_atomic_state *s)
+omap_get_global_state(struct drm_atomic_commit *s)
 {
 	struct omap_drm_private *priv = s->dev->dev_private;
 	struct drm_private_state *priv_state;
@@ -507,8 +507,6 @@ static int omap_modeset_init(struct drm_device *dev)
 			return PTR_ERR(pipe->connector);
 		}
 
-		drm_connector_attach_encoder(pipe->connector, encoder);
-
 		crtc = omap_crtc_init(dev, pipe, priv->planes[i]);
 		if (IS_ERR(crtc))
 			return PTR_ERR(crtc);
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.h b/drivers/gpu/drm/omapdrm/omap_drv.h
index d903568fd8cca..c1f24ce929d96 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.h
+++ b/drivers/gpu/drm/omapdrm/omap_drv.h
@@ -105,7 +105,7 @@ struct omap_drm_private {
 
 void omap_debugfs_init(struct drm_minor *minor);
 
-struct omap_global_state * __must_check omap_get_global_state(struct drm_atomic_state *s);
+struct omap_global_state * __must_check omap_get_global_state(struct drm_atomic_commit *s);
 
 struct omap_global_state *omap_get_existing_global_state(struct omap_drm_private *priv);
 
diff --git a/drivers/gpu/drm/omapdrm/omap_overlay.c b/drivers/gpu/drm/omapdrm/omap_overlay.c
index 77c1b78d0163a..d722e3c7db760 100644
--- a/drivers/gpu/drm/omapdrm/omap_overlay.c
+++ b/drivers/gpu/drm/omapdrm/omap_overlay.c
@@ -66,7 +66,7 @@ omap_plane_find_free_overlay(struct drm_device *dev, struct drm_plane *hwoverlay
  * This should be called from the plane atomic_check() in order to prepare the
  * next global overlay_map to be enabled when atomic transaction is valid.
  */
-int omap_overlay_assign(struct drm_atomic_state *s, struct drm_plane *plane,
+int omap_overlay_assign(struct drm_atomic_commit *s, struct drm_plane *plane,
 			u32 caps, u32 fourcc, struct omap_hw_overlay **overlay,
 			struct omap_hw_overlay **r_overlay)
 {
@@ -111,7 +111,7 @@ int omap_overlay_assign(struct drm_atomic_state *s, struct drm_plane *plane,
  * This should be called from the plane atomic_check() in order to prepare the
  * next global overlay_map to be enabled when atomic transaction is valid.
  */
-void omap_overlay_release(struct drm_atomic_state *s, struct omap_hw_overlay *overlay)
+void omap_overlay_release(struct drm_atomic_commit *s, struct omap_hw_overlay *overlay)
 {
 	/* Get the global state of the current atomic transaction */
 	struct omap_global_state *state = omap_get_global_state(s);
diff --git a/drivers/gpu/drm/omapdrm/omap_overlay.h b/drivers/gpu/drm/omapdrm/omap_overlay.h
index e36a43f355638..e44332455210c 100644
--- a/drivers/gpu/drm/omapdrm/omap_overlay.h
+++ b/drivers/gpu/drm/omapdrm/omap_overlay.h
@@ -27,9 +27,9 @@ struct omap_hw_overlay {
 
 int omap_hwoverlays_init(struct omap_drm_private *priv);
 void omap_hwoverlays_destroy(struct omap_drm_private *priv);
-int omap_overlay_assign(struct drm_atomic_state *s, struct drm_plane *plane,
+int omap_overlay_assign(struct drm_atomic_commit *s, struct drm_plane *plane,
 			u32 caps, u32 fourcc, struct omap_hw_overlay **overlay,
 			struct omap_hw_overlay **r_overlay);
-void omap_overlay_release(struct drm_atomic_state *s, struct omap_hw_overlay *overlay);
+void omap_overlay_release(struct drm_atomic_commit *s, struct omap_hw_overlay *overlay);
 void omap_overlay_update_state(struct omap_drm_private *priv, struct omap_hw_overlay *overlay);
 #endif /* __OMAPDRM_OVERLAY_H__ */
diff --git a/drivers/gpu/drm/omapdrm/omap_plane.c b/drivers/gpu/drm/omapdrm/omap_plane.c
index 0e09d8f1fe5e3..c024cdbbd87af 100644
--- a/drivers/gpu/drm/omapdrm/omap_plane.c
+++ b/drivers/gpu/drm/omapdrm/omap_plane.c
@@ -62,7 +62,7 @@ static void omap_plane_cleanup_fb(struct drm_plane *plane,
 }
 
 static void omap_plane_atomic_update(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct omap_drm_private *priv = plane->dev->dev_private;
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
@@ -167,7 +167,7 @@ static void omap_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void omap_plane_atomic_disable(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct omap_drm_private *priv = plane->dev->dev_private;
 	struct omap_plane *omap_plane = to_omap_plane(plane);
@@ -199,7 +199,7 @@ static void omap_plane_atomic_disable(struct drm_plane *plane,
 #define FRAC_16_16(mult, div)    (((mult) << 16) / (div))
 
 static int omap_plane_atomic_check(struct drm_plane *plane,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index d592f4f4b939a..7450b27622a23 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -105,8 +105,20 @@ config DRM_PANEL_BOE_TV101WUM_LL2
 	  Say Y here if you want to support for BOE TV101WUM-LL2
 	  WUXGA PANEL DSI Video Mode panel
 
+config DRM_PANEL_CHIPWEALTH_CH13726A
+	tristate "CHIPWEALTH CH13726A-based DSI panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_DISPLAY_DP_HELPER
+	select DRM_DISPLAY_HELPER
+	help
+	  Say Y here if you want to enable support for ChipWealth
+	  CH13726A-based display panels.
+
 config DRM_PANEL_EBBG_FT8719
 	tristate "EBBG FT8719 panel driver"
+	depends on GPIOLIB
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
@@ -144,6 +156,18 @@ config DRM_PANEL_FEIYANG_FY07024DI26A30D
 	  Say Y if you want to enable support for panels based on the
 	  Feiyang FY07024DI26A30-D MIPI-DSI interface.
 
+config DRM_PANEL_FOCALTECH_OTA7290B
+	tristate "Focaltech OTA7290B"
+	depends on DRM_MIPI_DSI
+	depends on I2C
+	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_KMS_HELPER
+	help
+	  Enable support for panels using OTA7290B as a controller (for
+	  example, Waveshare 12.3" DSI TOUCH-A panel). Say Y here if you want
+	  to enable support for this panel. To compile this driver as a module,
+	  choose M here.
+
 config DRM_PANEL_DSI_CM
 	tristate "Generic DSI command mode panels"
 	depends on OF
@@ -338,6 +362,7 @@ config DRM_PANEL_JADARD_JD9365DA_H3
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_KMS_HELPER
 	help
 	  Say Y here if you want to enable support for Jadard JD9365DA-H3
 	  WXGA MIPI DSI panel. The panel support TFT dot matrix LCD with
@@ -345,7 +370,8 @@ config DRM_PANEL_JADARD_JD9365DA_H3
 
 config DRM_PANEL_JDI_LPM102A188A
 	tristate "JDI LPM102A188A DSI panel"
-	depends on OF && GPIOLIB
+	depends on GPIOLIB
+	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
 	help
@@ -460,6 +486,7 @@ config DRM_PANEL_LG_LG4573
 
 config DRM_PANEL_LG_SW43408
 	tristate "LG SW43408 panel"
+	depends on GPIOLIB
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
@@ -550,6 +577,16 @@ config DRM_PANEL_NOVATEK_NT35510
 	  around the Novatek NT35510 display controller, such as some
 	  Hydis panels.
 
+config DRM_PANEL_NOVATEK_NT35532
+	tristate "Novatek NT35532-based DSI video mode panel"
+	depends on OF
+	depends on DRM_MIPI_DSI
+	depends on BACKLIGHT_CLASS_DEVICE
+	select DRM_KMS_HELPER
+	help
+	  Say Y or M here if you want to enable support for Novatek
+	  NT35532-based 1080p video mode DSI panels.
+
 config DRM_PANEL_NOVATEK_NT35560
 	tristate "Novatek NT35560 DSI command mode panel"
 	depends on OF
@@ -585,6 +622,7 @@ config DRM_PANEL_NOVATEK_NT36523
 
 config DRM_PANEL_NOVATEK_NT36672A
 	tristate "Novatek NT36672A DSI panel"
+	depends on GPIOLIB
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
@@ -595,6 +633,7 @@ config DRM_PANEL_NOVATEK_NT36672A
 
 config DRM_PANEL_NOVATEK_NT36672E
 	tristate "Novatek NT36672E DSI panel"
+	depends on GPIOLIB
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
@@ -732,7 +771,8 @@ config DRM_PANEL_RAYDIUM_RM692E5
 
 config DRM_PANEL_RAYDIUM_RM69380
 	tristate "Raydium RM69380-based DSI panel"
-	depends on OF && GPIOLIB
+	depends on GPIOLIB
+	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
 	help
@@ -890,10 +930,10 @@ config DRM_PANEL_SAMSUNG_S6D7AA0
 
 config DRM_PANEL_SAMSUNG_S6E3FC2X01
 	tristate "Samsung S6E3FC2X01 DSI panel controller"
+	depends on GPIOLIB
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
-	select VIDEOMODE_HELPERS
 	help
 	  Say Y or M here if you want to enable support for the
 	  Samsung S6E3FC2 DDIC and connected MIPI DSI panel.
@@ -910,11 +950,18 @@ config DRM_PANEL_SAMSUNG_S6E3HA2
 
 config DRM_PANEL_SAMSUNG_S6E3HA8
 	tristate "Samsung S6E3HA8 DSI video mode panel"
+	depends on GPIOLIB
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
 	select DRM_DISPLAY_DSC_HELPER
-	select VIDEOMODE_HELPERS
+	help
+	  Say Y or M here if you want to enable support for the
+	  Samsung S6E3HA8 DDIC and connected MIPI DSI panel.
+	  Currently supported panels:
+
+	    Samsung AMB577PX01 (found in the Samsung S9 smartphone)
+
 
 config DRM_PANEL_SAMSUNG_S6E63J0X03
 	tristate "Samsung S6E63J0X03 DSI command mode panel"
@@ -982,10 +1029,10 @@ config DRM_PANEL_SAMSUNG_S6E8FC0
 
 config DRM_PANEL_SAMSUNG_SOFEF00
 	tristate "Samsung SOFEF00 DSI panel controller"
+	depends on GPIOLIB
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
-	select VIDEOMODE_HELPERS
 	help
 	  Say Y or M here if you want to enable support for the Samsung AMOLED
 	  panel SOFEF00 DDIC and connected panel.
@@ -1250,6 +1297,7 @@ config DRM_PANEL_VISIONOX_R66451
 
 config DRM_PANEL_VISIONOX_RM69299
 	tristate "Visionox RM69299"
+	depends on GPIOLIB
 	depends on OF
 	depends on DRM_MIPI_DSI
 	depends on BACKLIGHT_CLASS_DEVICE
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index a4291dc3905be..c2c5cf8171163 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -9,6 +9,7 @@ obj-$(CONFIG_DRM_PANEL_BOE_TD4320) += panel-boe-td4320.o
 obj-$(CONFIG_DRM_PANEL_BOE_TH101MB31UIG002_28A) += panel-boe-th101mb31ig002-28a.o
 obj-$(CONFIG_DRM_PANEL_BOE_TV101WUM_LL2) += panel-boe-tv101wum-ll2.o
 obj-$(CONFIG_DRM_PANEL_BOE_TV101WUM_NL6) += panel-boe-tv101wum-nl6.o
+obj-$(CONFIG_DRM_PANEL_CHIPWEALTH_CH13726A) += panel-chipwealth-ch13726a.o
 obj-$(CONFIG_DRM_PANEL_DSI_CM) += panel-dsi-cm.o
 obj-$(CONFIG_DRM_PANEL_LVDS) += panel-lvds.o
 obj-$(CONFIG_DRM_PANEL_SIMPLE) += panel-simple.o
@@ -17,6 +18,7 @@ obj-$(CONFIG_DRM_PANEL_EBBG_FT8719) += panel-ebbg-ft8719.o
 obj-$(CONFIG_DRM_PANEL_ELIDA_KD35T133) += panel-elida-kd35t133.o
 obj-$(CONFIG_DRM_PANEL_FEIXIN_K101_IM2BA02) += panel-feixin-k101-im2ba02.o
 obj-$(CONFIG_DRM_PANEL_FEIYANG_FY07024DI26A30D) += panel-feiyang-fy07024di26a30d.o
+obj-$(CONFIG_DRM_PANEL_FOCALTECH_OTA7290B) += panel-focaltech-ota7290b.o
 obj-$(CONFIG_DRM_PANEL_HIMAX_HX8279) += panel-himax-hx8279.o
 obj-$(CONFIG_DRM_PANEL_HIMAX_HX83102) += panel-himax-hx83102.o
 obj-$(CONFIG_DRM_PANEL_HIMAX_HX83112A) += panel-himax-hx83112a.o
@@ -54,6 +56,7 @@ obj-$(CONFIG_DRM_PANEL_NEC_NL8048HL11) += panel-nec-nl8048hl11.o
 obj-$(CONFIG_DRM_PANEL_NEWVISION_NV3051D) += panel-newvision-nv3051d.o
 obj-$(CONFIG_DRM_PANEL_NEWVISION_NV3052C) += panel-newvision-nv3052c.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35510) += panel-novatek-nt35510.o
+obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35532) += panel-novatek-nt35532.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35560) += panel-novatek-nt35560.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT35950) += panel-novatek-nt35950.o
 obj-$(CONFIG_DRM_PANEL_NOVATEK_NT36523) += panel-novatek-nt36523.o
diff --git a/drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c b/drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c
new file mode 100644
index 0000000000000..be76bc825c3fd
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-chipwealth-ch13726a.c
@@ -0,0 +1,333 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * ChipWealth CH13726A MIPI-DSI panel driver
+ * Copyright (c) 2024, Teguh Sobirin <teguh@sobir.in>.
+ */
+
+#include <linux/backlight.h>
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+
+#include <video/mipi_display.h>
+
+static const struct regulator_bulk_data ch13726a_supplies[] = {
+	{ .supply = "vdd1v2", },
+	{ .supply = "vddio", },
+	{ .supply = "vdd", },
+	{ .supply = "avdd", },
+};
+
+struct ch13726a_panel {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+	struct regulator_bulk_data *supplies;
+	struct gpio_desc *reset_gpio;
+	struct ch13726a_desc *desc;
+	enum drm_panel_orientation orientation;
+};
+
+struct ch13726a_desc {
+	unsigned int width_mm;
+	unsigned int height_mm;
+	unsigned int bpc;
+
+	const struct drm_display_mode *modes;
+	unsigned int num_modes;
+};
+
+static inline struct ch13726a_panel *to_ch13726a_panel(struct drm_panel *panel)
+{
+	return container_of(panel, struct ch13726a_panel, panel);
+}
+
+static void ch13726a_reset(struct ch13726a_panel *ctx)
+{
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(10000, 11000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	usleep_range(10000, 11000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(10000, 11000);
+}
+
+static int ch13726a_on(struct ch13726a_panel *ctx)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	ctx->dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf0, 0x50);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb9, 0x00);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+
+	return dsi_ctx.accum_err;
+}
+
+static int ch13726a_disable(struct drm_panel *panel)
+{
+	struct ch13726a_panel *ctx = to_ch13726a_panel(panel);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	ctx->dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 50);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+
+	return dsi_ctx.accum_err;
+}
+
+static int ch13726a_prepare(struct drm_panel *panel)
+{
+	struct ch13726a_panel *ctx = to_ch13726a_panel(panel);
+	struct device *dev = &ctx->dsi->dev;
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(ch13726a_supplies), ctx->supplies);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	ch13726a_reset(ctx);
+
+	ret = ch13726a_on(ctx);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize panel: %d\n", ret);
+		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+		regulator_bulk_disable(ARRAY_SIZE(ch13726a_supplies), ctx->supplies);
+		return ret;
+	}
+
+	msleep(28);
+
+	return 0;
+}
+
+static int ch13726a_unprepare(struct drm_panel *panel)
+{
+	struct ch13726a_panel *ctx = to_ch13726a_panel(panel);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(ch13726a_supplies), ctx->supplies);
+
+	return 0;
+}
+
+static const struct drm_display_mode thor_bottom_modes[] = {
+	{
+		/* 120Hz */
+		.clock = (1080 + 28 + 4 + 36) * (1240 + 16 + 4 + 8) * 120 / 1000,
+		.hdisplay = 1080,
+		.hsync_start = 1080 + 28,
+		.hsync_end = 1080 + 28 + 4,
+		.htotal = 1080 + 28 + 4 + 36,
+		.vdisplay = 1240,
+		.vsync_start = 1240 + 16,
+		.vsync_end = 1240 + 16 + 4,
+		.vtotal = 1240 + 16 + 4 + 8,
+	},
+	{
+		/* 60Hz */
+		.clock = (1080 + 28 + 4 + 36) * (1240 + 16 + 4 + 8) * 60 / 1000,
+		.hdisplay = 1080,
+		.hsync_start = 1080 + 28,
+		.hsync_end = 1080 + 28 + 4,
+		.htotal = 1080 + 28 + 4 + 36,
+		.vdisplay = 1240,
+		.vsync_start = 1240 + 16,
+		.vsync_end = 1240 + 16 + 4,
+		.vtotal = 1240 + 16 + 4 + 8,
+	}
+};
+
+static struct ch13726a_desc thor_bottom_desc = {
+	.modes = thor_bottom_modes,
+	.num_modes = ARRAY_SIZE(thor_bottom_modes),
+	.width_mm = 65,
+	.height_mm = 75,
+	.bpc = 8,
+};
+
+static int ch13726a_get_modes(struct drm_panel *panel,
+					struct drm_connector *connector)
+{
+	struct ch13726a_panel *ctx = to_ch13726a_panel(panel);
+
+	for (uint8_t i = 0; i < ctx->desc->num_modes; i++) {
+		const struct drm_display_mode *m = &ctx->desc->modes[i];
+		struct drm_display_mode *mode;
+
+		mode = drm_mode_duplicate(connector->dev, m);
+		if (!mode) {
+			dev_err(&ctx->dsi->dev, "failed to add mode %ux%u@%u\n",
+				m->hdisplay, m->vdisplay, drm_mode_vrefresh(m));
+			return -ENOMEM;
+		}
+
+		mode->type = DRM_MODE_TYPE_DRIVER;
+		if (i == 0)
+			mode->type |= DRM_MODE_TYPE_PREFERRED;
+
+		drm_mode_set_name(mode);
+		drm_mode_probed_add(connector, mode);
+	}
+
+	connector->display_info.width_mm = ctx->desc->width_mm;
+	connector->display_info.height_mm = ctx->desc->height_mm;
+	connector->display_info.bpc = ctx->desc->bpc;
+
+	return ctx->desc->num_modes;
+}
+
+static enum drm_panel_orientation ch13726a_get_orientation(struct drm_panel *panel)
+{
+	struct ch13726a_panel *ctx = to_ch13726a_panel(panel);
+
+	return ctx->orientation;
+}
+
+static const struct drm_panel_funcs ch13726a_panel_funcs = {
+	.prepare = ch13726a_prepare,
+	.unprepare = ch13726a_unprepare,
+	.disable = ch13726a_disable,
+	.get_modes = ch13726a_get_modes,
+	.get_orientation = ch13726a_get_orientation,
+};
+
+static int ch13726a_bl_update_status(struct backlight_device *bl)
+{
+	struct mipi_dsi_device *dsi = bl_get_data(bl);
+	u16 brightness = backlight_get_brightness(bl);
+	int ret;
+
+	dsi->mode_flags &= ~MIPI_DSI_MODE_LPM;
+
+	ret = mipi_dsi_dcs_set_display_brightness(dsi, brightness);
+	if (ret < 0)
+		return ret;
+
+	dsi->mode_flags |= MIPI_DSI_MODE_LPM;
+
+	return 0;
+}
+
+static const struct backlight_ops ch13726a_bl_ops = {
+	.update_status = ch13726a_bl_update_status,
+};
+
+static struct backlight_device *
+ch13726a_create_backlight(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	const struct backlight_properties props = {
+		.type = BACKLIGHT_RAW,
+		.brightness = 255,
+		.max_brightness = 255,
+	};
+
+	return devm_backlight_device_register(dev, dev_name(dev), dev, dsi,
+					      &ch13726a_bl_ops, &props);
+}
+
+static int ch13726a_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct ch13726a_panel *ctx;
+	int ret;
+
+	ctx = devm_drm_panel_alloc(dev, __typeof(*ctx), panel,
+				   &ch13726a_panel_funcs,
+				   DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	ctx->desc = (struct ch13726a_desc *)of_device_get_match_data(dev);
+	if (!ctx->desc)
+		return -ENODEV;
+
+	ret = devm_regulator_bulk_get_const(dev,
+					    ARRAY_SIZE(ch13726a_supplies),
+					    ch13726a_supplies,
+					    &ctx->supplies);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to get regulators\n");
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset-gpios\n");
+
+	ret = of_drm_get_panel_orientation(dev->of_node, &ctx->orientation);
+	if (ret < 0) {
+		dev_err(dev, "%pOF: failed to get orientation %d\n", dev->of_node, ret);
+		return ret;
+	}
+
+	ctx->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
+			  MIPI_DSI_CLOCK_NON_CONTINUOUS;
+
+	ctx->panel.prepare_prev_first = true;
+
+	ctx->panel.backlight = ch13726a_create_backlight(dsi);
+	if (IS_ERR(ctx->panel.backlight))
+		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
+				     "Failed to create backlight\n");
+
+	drm_panel_add(&ctx->panel);
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0) {
+		dev_err(dev, "Failed to attach to DSI host: %d\n", ret);
+		drm_panel_remove(&ctx->panel);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void ch13726a_remove(struct mipi_dsi_device *dsi)
+{
+	struct ch13726a_panel *ctx = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_remove(&ctx->panel);
+}
+
+static const struct of_device_id ch13726a_of_match[] = {
+	{ .compatible = "ayntec,thor-panel-bottom", .data = &thor_bottom_desc },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, ch13726a_of_match);
+
+static struct mipi_dsi_driver ch13726a_driver = {
+	.probe = ch13726a_probe,
+	.remove = ch13726a_remove,
+	.driver = {
+		.name = "panel-ch13726a-amoled",
+		.of_match_table = ch13726a_of_match,
+	},
+};
+module_mipi_dsi_driver(ch13726a_driver);
+
+MODULE_DESCRIPTION("DRM driver for CH13726A DSI panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-edp.c b/drivers/gpu/drm/panel/panel-edp.c
index 497dcd48f57b9..cb7d6c43466eb 100644
--- a/drivers/gpu/drm/panel/panel-edp.c
+++ b/drivers/gpu/drm/panel/panel-edp.c
@@ -1918,6 +1918,7 @@ static const struct panel_delay delay_80_500_e50_d50 = {
  * Sort first by vendor, then by product ID.
  */
 static const struct edp_panel_entry edp_panels[] = {
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x0290, &delay_200_500_e50, "B140XTN07.5"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x04a4, &delay_200_500_e50, "B122UAN01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x0ba4, &delay_200_500_e50, "B140QAX01.H"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x105c, &delay_200_500_e50, "B116XTN01.0"),
@@ -1934,17 +1935,20 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x239b, &delay_200_500_e50, "B116XAN06.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x255c, &delay_200_500_e50, "B116XTN02.5"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x30ed, &delay_200_500_e50, "G156HAN03.0"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x3c9f, &delay_200_500_e50, "B140HAK03.5"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x403d, &delay_200_500_e50, "B140HAN04.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAN04.0"),
 	EDP_PANEL_ENTRY2('A', 'U', 'O', 0x405c, &auo_b116xak01.delay, "B116XAK01.0",
 			 &auo_b116xa3_mode),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x435c, &delay_200_500_e50, "Unknown"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x49ba, &delay_200_500_e50, "B116XTN02.3"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x52b0, &delay_200_500_e50, "B116XAK02.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x582d, &delay_200_500_e50, "B133UAN01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x615c, &delay_200_500_e50, "B116XAN06.1"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x635c, &delay_200_500_e50, "B116XAN06.3"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x639c, &delay_200_500_e50, "B140HAK02.7"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x643d, &delay_200_500_e50, "B140HAN06.4"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0x67a8, &delay_200_500_e50, "B140XTK02.4"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x723c, &delay_200_500_e50, "B140XTN07.2"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x73aa, &delay_200_500_e50, "B116XTN02.3"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0x8594, &delay_200_500_e50, "B133UAN01.0"),
@@ -1954,7 +1958,9 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xa7b3, &delay_200_500_e50, "B140UAN04.4"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xb7a9, &delay_200_500_e50, "B140HAK03.3"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xc4b4, &delay_200_500_e50, "B116XAT04.1"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0xc7ad, &delay_200_500_e50, "B140HAN07.7"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xc9a8, &delay_200_500_e50, "B140QAN08.H"),
+	EDP_PANEL_ENTRY('A', 'U', 'O', 0xcb8f, &delay_200_500_e50, "B133HAN06.6"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xcdba, &delay_200_500_e50, "B140UAX01.2"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xd497, &delay_200_500_e50, "B120XAN01.0"),
 	EDP_PANEL_ENTRY('A', 'U', 'O', 0xf390, &delay_200_500_e50, "B140XTN07.7"),
@@ -1990,6 +1996,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x08b2, &delay_200_500_e200, "NT140WHM-N49"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0848, &delay_200_500_e200, "Unknown"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0849, &delay_200_500_e200, "Unknown"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x090d, &delay_200_500_e50, "NT140WHM-N4T"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x09c3, &delay_200_500_e50, "NT116WHM-N21,836X2"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x094b, &delay_200_500_e50, "NT116WHM-N21"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0951, &delay_200_500_e80, "NV116WHM-N47"),
@@ -2003,6 +2010,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x09ae, &delay_200_500_e200, "NT140FHM-N45"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x09dd, &delay_200_500_e50, "NT116WHM-N21"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a1b, &delay_200_500_e50, "NV133WUM-N63"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a25, &delay_200_500_e50_po2e200, "NV133FHM-N4F V8.0"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a36, &delay_200_500_e200, "Unknown"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a3e, &delay_200_500_e80_d50, "NV116WHM-N49"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0a5d, &delay_200_500_e50, "NV116WHM-N45"),
@@ -2016,10 +2024,13 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b43, &delay_200_500_e200, "NV140FHM-T09"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b56, &delay_200_500_e80, "NT140FHM-N47"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b66, &delay_200_500_e80, "NE140WUM-N6G"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0b85, &delay_200_500_e50, "NT140WHM-T05"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c20, &delay_200_500_e80, "NT140FHM-N47"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c26, &delay_200_500_p2e200, "NV140WUM-T08"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c6f, &delay_200_500_e50, "NV140FHM-N40"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0c93, &delay_200_500_e200, "Unknown"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cb6, &delay_200_500_e200, "NT116WHM-N44"),
+	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cc9, &delay_200_500_e50, "NE120DRM-N28"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cf2, &delay_200_500_e200, "NV156FHM-N4S"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cf6, &delay_200_500_e200_d100, "NV140WUM-N64"),
 	EDP_PANEL_ENTRY('B', 'O', 'E', 0x0cfa, &delay_200_500_e50, "NV116WHM-A4D"),
@@ -2048,6 +2059,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1160, &delay_200_500_e80_d50, "N116BCJ-EAK"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1161, &delay_200_500_e80, "N116BCP-EA2"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1163, &delay_200_500_e80_d50, "N116BCJ-EAK"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1169, &delay_200_500_e80_d50, "N116BCN-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x117a, &delay_200_500_e80_d50, "N116BCL-EAK"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1247, &delay_200_500_e80_d50, "N120ACA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x124c, &delay_200_500_e80_d50, "N122JCA-ENK"),
@@ -2058,8 +2070,9 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1468, &delay_200_500_e80, "N140HGA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x148f, &delay_200_500_e80, "N140HCA-EAC"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14a8, &delay_200_500_e80, "N140JCA-ELP"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14c7, &delay_200_500_e80, "N140HCA-EEK"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14d4, &delay_200_500_e80_d50, "N140HCA-EAC"),
-	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14d6, &delay_200_500_e80_d50, "N140BGA-EA4"),
+	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14d6, &delay_200_500_e80, "N140BGA-E54"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x14e5, &delay_200_500_e80_d50, "N140HGA-EA1"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x1565, &delay_200_500_e80, "N156HCA-EAB"),
 	EDP_PANEL_ENTRY('C', 'M', 'N', 0x156b, &delay_200_500_e80_d50, "N153JCA-ELK"),
@@ -2072,6 +2085,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1100, &delay_200_500_e80_d50, "MNB601LS1-1"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1103, &delay_200_500_e80_d50, "MNB601LS1-3"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1104, &delay_200_500_e50_d100, "MNB601LS1-4"),
+	EDP_PANEL_ENTRY('C', 'S', 'W', 0x110a, &delay_200_500_e50, "PNB601LS1-2"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x143f, &delay_200_500_e50, "MNE007QS3-6"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x1448, &delay_200_500_e50, "MNE007QS3-7"),
 	EDP_PANEL_ENTRY('C', 'S', 'W', 0x144b, &delay_200_500_e80, "MNE001BS1-4"),
@@ -2094,6 +2108,8 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('I', 'V', 'O', 0x854a, &delay_200_500_p2e100, "M133NW4J"),
 	EDP_PANEL_ENTRY('I', 'V', 'O', 0x854b, &delay_200_500_p2e100, "R133NW4K-R0"),
 	EDP_PANEL_ENTRY('I', 'V', 'O', 0x8c4d, &delay_200_150_e200, "R140NWFM R1"),
+	EDP_PANEL_ENTRY('I', 'V', 'O', 0x8cd5, &delay_200_500_e200_d10, "M140NWFQ R5"),
+	EDP_PANEL_ENTRY('I', 'V', 'O', 0x8ce6, &delay_200_500_e200, "R140NWFW R0"),
 
 	EDP_PANEL_ENTRY('K', 'D', 'B', 0x044f, &delay_200_500_e80_d50, "Unknown"),
 	EDP_PANEL_ENTRY('K', 'D', 'B', 0x0624, &kingdisplay_kd116n21_30nv_a010.delay, "116N21-30NV-A010"),
@@ -2119,6 +2135,7 @@ static const struct edp_panel_entry edp_panels[] = {
 	EDP_PANEL_ENTRY('L', 'G', 'D', 0x05af, &delay_200_500_e200_d200, "Unknown"),
 	EDP_PANEL_ENTRY('L', 'G', 'D', 0x05f1, &delay_200_500_e200_d200, "Unknown"),
 	EDP_PANEL_ENTRY('L', 'G', 'D', 0x0778, &delay_200_500_e200_d200, "134WT1"),
+	EDP_PANEL_ENTRY('L', 'G', 'D', 0x07fe, &delay_200_500_e200_d200, "LP116WHA-SPB1"),
 
 	EDP_PANEL_ENTRY('S', 'H', 'P', 0x1511, &delay_200_500_e50, "LQ140M1JW48"),
 	EDP_PANEL_ENTRY('S', 'H', 'P', 0x1523, &delay_80_500_e50, "LQ140M1JW46"),
diff --git a/drivers/gpu/drm/panel/panel-focaltech-ota7290b.c b/drivers/gpu/drm/panel/panel-focaltech-ota7290b.c
new file mode 100644
index 0000000000000..ed02a8daf96f9
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-focaltech-ota7290b.c
@@ -0,0 +1,225 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2024 Waveshare International Limited
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of.h>
+
+#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+struct ota7290b {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+
+	struct regulator *power;
+	struct gpio_desc *reset;
+	struct regulator *avdd;
+	struct regulator *vdd;
+	struct regulator *vcc;
+
+	enum drm_panel_orientation orientation;
+};
+
+static inline struct ota7290b *panel_to_ota(struct drm_panel *panel)
+{
+	return container_of(panel, struct ota7290b, panel);
+}
+
+static int ota7290b_prepare(struct drm_panel *panel)
+{
+	struct ota7290b *ctx = panel_to_ota(panel);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+	int ret;
+
+	if (ctx->vcc) {
+		ret = regulator_enable(ctx->vcc);
+		if (ret)
+			dev_err(panel->dev, "failed to enable VCC regulator: %d\n", ret);
+	}
+
+	if (ctx->reset) {
+		gpiod_set_value_cansleep(ctx->reset, 0);
+		msleep(60);
+		gpiod_set_value_cansleep(ctx->reset, 1);
+		msleep(60);
+	}
+
+	if (ctx->vdd) {
+		ret = regulator_enable(ctx->vdd);
+		if (ret)
+			dev_err(panel->dev, "failed to enable VDD regulator: %d\n", ret);
+	}
+
+	if (ctx->reset) {
+		gpiod_set_value_cansleep(ctx->reset, 0);
+		msleep(60);
+	}
+
+	if (ctx->avdd) {
+		ret = regulator_enable(ctx->avdd);
+		if (ret)
+			dev_err(panel->dev, "failed to enable AVDD regulator: %d\n", ret);
+	}
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 50);
+
+	if (dsi_ctx.accum_err < 0)
+		dev_err(panel->dev, "failed to init panel: %d\n", dsi_ctx.accum_err);
+
+	return dsi_ctx.accum_err;
+}
+
+static int ota7290b_unprepare(struct drm_panel *panel)
+{
+	struct ota7290b *ctx = panel_to_ota(panel);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+
+	if (ctx->avdd)
+		regulator_disable(ctx->avdd);
+
+	if (ctx->reset) {
+		gpiod_set_value_cansleep(ctx->reset, 1);
+		msleep(5);
+	}
+
+	if (ctx->vdd)
+		regulator_disable(ctx->vdd);
+
+	if (ctx->vcc)
+		regulator_disable(ctx->vcc);
+
+	return 0;
+}
+
+static const struct drm_display_mode waveshare_dsi_touch_8_8_a_mode = {
+	.clock = 75000,
+
+	.hdisplay = 480,
+	.hsync_start = 480 + 50,
+	.hsync_end = 480 + 50 + 50,
+	.htotal = 480 + 50 + 50 + 50,
+
+	.vdisplay = 1920,
+	.vsync_start = 1920 + 20,
+	.vsync_end = 1920 + 20 + 20,
+	.vtotal = 1920 + 20 + 20 + 20,
+
+	.width_mm = 68,
+	.height_mm = 219,
+	.type = DRM_MODE_TYPE_DRIVER,
+};
+
+static int ota7290b_get_modes(struct drm_panel *panel,
+			      struct drm_connector *connector)
+{
+	return drm_connector_helper_get_modes_fixed(connector, &waveshare_dsi_touch_8_8_a_mode);
+}
+
+static enum drm_panel_orientation ota7290b_get_orientation(struct drm_panel *panel)
+{
+	struct ota7290b *ctx = panel_to_ota(panel);
+
+	return ctx->orientation;
+}
+
+static const struct drm_panel_funcs ota7290b_funcs = {
+	.prepare = ota7290b_prepare,
+	.unprepare = ota7290b_unprepare,
+	.get_modes = ota7290b_get_modes,
+	.get_orientation = ota7290b_get_orientation,
+};
+
+static int ota7290b_probe(struct mipi_dsi_device *dsi)
+{
+	struct ota7290b *ctx;
+	int ret;
+
+	ctx = devm_drm_panel_alloc(&dsi->dev, struct ota7290b, panel,
+				   &ota7290b_funcs,
+				   DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+	mipi_dsi_set_drvdata(dsi, ctx);
+	ctx->dsi = dsi;
+
+	ctx->reset = devm_gpiod_get_optional(&dsi->dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->reset))
+		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->reset),
+				     "Couldn't get our reset GPIO\n");
+
+	ctx->vcc = devm_regulator_get_optional(&dsi->dev, "vcc");
+	if (IS_ERR(ctx->vcc))
+		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->vcc),
+					"Couldn't get our VCC supply\n");
+
+	ctx->avdd = devm_regulator_get_optional(&dsi->dev, "avdd");
+	if (IS_ERR(ctx->avdd))
+		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->avdd),
+					"Couldn't get our AVDD supply\n");
+
+	ctx->vdd = devm_regulator_get_optional(&dsi->dev, "vdd");
+	if (IS_ERR(ctx->vdd))
+		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->vdd),
+					"Couldn't get our VDD supply\n");
+
+	ret = of_drm_get_panel_orientation(dsi->dev.of_node, &ctx->orientation);
+	if (ret) {
+		dev_err(&dsi->dev, "%pOF: failed to get orientation: %d\n",
+			dsi->dev.of_node, ret);
+		return ret;
+	}
+
+	ret = drm_panel_of_backlight(&ctx->panel);
+	if (ret)
+		return ret;
+
+	ctx->panel.prepare_prev_first = true;
+
+	ret = devm_drm_panel_add(&dsi->dev, &ctx->panel);
+	if (ret)
+		return ret;
+
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_HSE |
+		MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+	dsi->format = MIPI_DSI_FMT_RGB888,
+	dsi->lanes = 2;
+
+	return devm_mipi_dsi_attach(&dsi->dev, dsi);
+}
+
+static const struct of_device_id ota7290b_of_match[] = {
+	{ .compatible = "waveshare,8.8-dsi-touch-a", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ota7290b_of_match);
+
+static struct mipi_dsi_driver ota7290b_driver = {
+	.probe		= ota7290b_probe,
+	.driver = {
+		.name		= "focaltech-ota7290b",
+		.of_match_table	= ota7290b_of_match,
+	},
+};
+module_mipi_dsi_driver(ota7290b_driver);
+
+MODULE_DESCRIPTION("Panel driver for Focaltech OTA7290B panels");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-himax-hx83102.c b/drivers/gpu/drm/panel/panel-himax-hx83102.c
index a5e5c9ea7a73f..d7e5664a5838a 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx83102.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx83102.c
@@ -29,11 +29,14 @@
 #define HX83102_UNKNOWN_B8	0xb8
 #define HX83102_SETEXTC		0xb9
 #define HX83102_SETMIPI		0xba
+#define HX83102_UNKNOWN_BB	0xbb
 #define HX83102_SETVDC		0xbc
 #define HX83102_SETBANK		0xbd
 #define HX83102_UNKNOWN_BE	0xbe
 #define HX83102_SETPTBA		0xbf
 #define HX83102_SETSTBA		0xc0
+#define HX83102_UNKNOWN_C2	0xc2
+#define HX83102_UNKNOWN_C6	0xc6
 #define HX83102_SETTCON		0xc7
 #define HX83102_SETRAMDMY	0xc8
 #define HX83102_SETPWM		0xc9
@@ -78,6 +81,7 @@ struct hx83102_panel_desc {
 	} size;
 
 	bool has_backlight;
+	unsigned long mode_flags;
 
 	int (*init)(struct hx83102 *ctx);
 };
@@ -765,6 +769,111 @@ static int holitech_htf065h045_init(struct hx83102 *ctx)
 	return dsi_ctx.accum_err;
 }
 
+/* This is HX83102-E, assuming commands are the same as the normal HX83102 */
+static int waveshare_12_3_a_init(struct hx83102 *ctx)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETEXTC, 0x83, 0x10, 0x2e);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETSPCCMD, 0xcd);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_UNKNOWN_BB, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETSPCCMD, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETPCTRL, 0x67, 0x2c, 0xff, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_UNKNOWN_BE, 0x11, 0x96, 0x89);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_UNKNOWN_D9, 0x04, 0x03, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETPOWER,
+				     0x10, 0xfa, 0xaf, 0xaf, 0x33, 0x33, 0xb1, 0x4d, 0x2f, 0x36,
+				     0x36, 0x36, 0x36, 0x22, 0x21, 0x15, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETDISP,
+				     0x00, 0xd0, 0x27, 0x80, 0x00, 0x14, 0x40, 0x2c, 0x32, 0x02,
+				     0x00, 0x00, 0x15, 0x20, 0xd7, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETCYC,
+				     0x98, 0xa0, 0x01, 0x01, 0x98, 0xa0, 0x68, 0x50, 0x01, 0xc7,
+				     0x01, 0x58, 0x00, 0xff, 0x00, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_UNKNOWN_B6, 0x4d, 0x4d, 0xe3);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETPTBA, 0xfc, 0x85, 0x80);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_UNKNOWN_D2, 0x33, 0x33);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETGIP0,
+				     0x00, 0x00, 0x00, 0x00, 0x64, 0x04, 0x00, 0x08, 0x08, 0x27,
+				     0x27, 0x22, 0x2f, 0x15, 0x15, 0x04, 0x04, 0x32, 0x10, 0x13,
+				     0x00, 0x13, 0x32, 0x10, 0x1f, 0x00,
+				     0x02, 0x32, 0x17, 0xfd, 0x00, 0x10, 0x00, 0x00, 0x20,
+				     0x30, 0x01, 0x55, 0x21, 0x38, 0x01, 0x55, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETGMA,
+				     0x00, 0x0c, 0x1a, 0x23, 0x2b, 0x4f, 0x64, 0x69, 0x6c, 0x64,
+				     0x77, 0x77, 0x76, 0x80, 0x79, 0x7e, 0x85, 0x9a, 0x97, 0x4d,
+				     0x56, 0x64, 0x70, 0x00, 0x0c, 0x1a, 0x23, 0x2b, 0x4f, 0x64,
+				     0x69, 0x6c, 0x64, 0x77, 0x77, 0x76, 0x80, 0x79, 0x7e, 0x85,
+				     0x9a, 0x97, 0x4d, 0x56, 0x64, 0x76);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETPOWER, 0x01, 0x9b, 0x01, 0x31);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETCLOCK,
+				     0x80, 0x36, 0x12, 0x16, 0xc0, 0x28, 0x40, 0x84, 0x22);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETGIP0,
+				     0x01, 0x00, 0xfc, 0x00, 0x00, 0x11, 0x10, 0x00, 0x0e, 0x00,
+				     0x01);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETCYC, 0x4e, 0x00, 0x33, 0x11, 0x33, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETPTBA, 0xf2, 0x00, 0x02);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETSTBA,
+				     0x23, 0x23, 0x22, 0x11, 0xa2, 0x17, 0x00, 0x80, 0x00, 0x00,
+				     0x08, 0x00, 0x63, 0x63);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_UNKNOWN_C6, 0xf9);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETTCON, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETRAMDMY,
+				     0x00, 0x04, 0x04, 0x00, 0x00, 0x82, 0x13, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETCASCADE, 0x07, 0x04, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETGIP1,
+				     0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x21, 0x20, 0x21, 0x20,
+				     0x01, 0x00, 0x03, 0x02, 0x05, 0x04, 0x07, 0x06, 0x1a, 0x1a,
+				     0x1a, 0x1a, 0x9a, 0x9a, 0x9a, 0x9a, 0x18, 0x18, 0x18, 0x18,
+				     0x21, 0x20, 0x21, 0x20, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18,
+				     0x18, 0x18, 0x18, 0x18);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETGIP2,
+				     0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x20, 0x21, 0x20, 0x21,
+				     0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x1a, 0x1a,
+				     0x1a, 0x1a, 0x1a, 0x1a, 0x1a, 0x1a, 0x18, 0x18, 0x18, 0x18,
+				     0x20, 0x21, 0x20, 0x21, 0x98, 0x98, 0x98, 0x98, 0x98, 0x98,
+				     0x98, 0x98, 0x98, 0x98);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETTP1,
+				     0x00, 0x34, 0x01, 0x88, 0x0e, 0xbe, 0x0f);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_UNKNOWN_C2, 0x43, 0xff, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETPANEL, 0x02);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETDISP, 0x80);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETGIP3,
+				     0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+				     0xaa, 0xaa, 0xaa, 0x80, 0x2a, 0xaa, 0xaa, 0xaa, 0xaa, 0x80,
+				     0x2a, 0xaa, 0xaa, 0xaa);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETGIP3,
+				     0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa, 0xaa,
+				     0xaa, 0xaa);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETGIP3,
+				     0xff, 0xff, 0xff, 0xff,
+				     0xff, 0xf0, 0xff, 0xff,
+				     0xff, 0xff, 0xff, 0xf0);
+
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, HX83102_SETBANK, 0x00);
+
+	return dsi_ctx.accum_err;
+};
+
 static const struct drm_display_mode starry_mode = {
 	.clock = 162680,
 	.hdisplay = 1200,
@@ -920,6 +1029,30 @@ static const struct hx83102_panel_desc holitech_htf065h045_desc = {
 	.init = holitech_htf065h045_init,
 };
 
+static const struct drm_display_mode waveshare_12_3_a_mode = {
+	.clock = 95000,
+	.hdisplay = 720,
+	.hsync_start = 720 + 10,
+	.hsync_end = 720 + 10 + 10,
+	.htotal = 720 + 10 + 10 + 12,
+	.vdisplay = 1920,
+	.vsync_start = 1920 + 64,
+	.vsync_end = 1920 + 64 + 18,
+	.vtotal = 1920 + 64 + 18 + 4,
+	.type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static const struct hx83102_panel_desc waveshare_12_3_inch_a_desc = {
+	.modes = &waveshare_12_3_a_mode,
+	.size = {
+		.width_mm = 109,
+		.height_mm = 292,
+	},
+	.mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+	.init = waveshare_12_3_a_init,
+};
+
 static int hx83102_enable(struct drm_panel *panel)
 {
 	msleep(130);
@@ -1170,8 +1303,12 @@ static int hx83102_probe(struct mipi_dsi_device *dsi)
 	desc = of_device_get_match_data(&dsi->dev);
 	dsi->lanes = 4;
 	dsi->format = MIPI_DSI_FMT_RGB888;
-	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
-					  MIPI_DSI_MODE_LPM;
+	if (desc->mode_flags)
+		dsi->mode_flags = desc->mode_flags;
+	else
+		dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
+			MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
+			MIPI_DSI_MODE_LPM;
 	ctx->desc = desc;
 	ctx->dsi = dsi;
 	ret = hx83102_panel_add(ctx);
@@ -1222,6 +1359,9 @@ static const struct of_device_id hx83102_of_match[] = {
 	{ .compatible = "holitech,htf065h045",
 	  .data = &holitech_htf065h045_desc
 	},
+	{ .compatible = "waveshare,12.3-dsi-touch-a",
+	  .data = &waveshare_12_3_inch_a_desc
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, hx83102_of_match);
diff --git a/drivers/gpu/drm/panel/panel-himax-hx8394.c b/drivers/gpu/drm/panel/panel-himax-hx8394.c
index c4d3e09a228da..bf80354567df3 100644
--- a/drivers/gpu/drm/panel/panel-himax-hx8394.c
+++ b/drivers/gpu/drm/panel/panel-himax-hx8394.c
@@ -44,6 +44,7 @@
 #define HX8394_CMD_SETID	  0xc3
 #define HX8394_CMD_SETDDB	  0xc4
 #define HX8394_CMD_UNKNOWN2	  0xc6
+#define HX8394_CMD_UNKNOWN6	  0xc7
 #define HX8394_CMD_SETCABC	  0xc9
 #define HX8394_CMD_SETCABCGAIN	  0xca
 #define HX8394_CMD_SETPANEL	  0xcc
@@ -618,38 +619,246 @@ static const struct hx8394_panel_desc hl055fhav028c_desc = {
 	.init_sequence = hl055fhav028c_init_sequence,
 };
 
-static int hx8394_enable(struct drm_panel *panel)
+static void waveshare_5_0_inch_a_init_sequence(struct mipi_dsi_multi_context *dsi_ctx)
 {
-	struct hx8394 *ctx = panel_to_hx8394(panel);
-	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
-	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
-	int ret;
+	/* 5.19.8 SETEXTC: Set extension command (B9h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETEXTC,
+				     0xff, 0x83, 0x94);
 
-	ctx->desc->init_sequence(&dsi_ctx);
+	/* 5.19.2 SETPOWER: Set power (B1h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETPOWER,
+				     0x48, 0x0a, 0x6a, 0x09, 0x33, 0x54, 0x71, 0x71, 0x2e, 0x45);
 
-	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	/* 5.19.9 SETMIPI: Set MIPI control (BAh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETMIPI,
+				     0x61, 0x03, 0x68, 0x6b, 0xb2, 0xc0);
 
-	if (dsi_ctx.accum_err)
-		return dsi_ctx.accum_err;
-	/* Panel is operational 120 msec after reset */
-	msleep(120);
+	/* 5.19.3 SETDISP: Set display related register (B2h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETDISP,
+				     0x00, 0x80, 0x64, 0x0c, 0x06, 0x2f);
 
-	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
-	if (dsi_ctx.accum_err)
-		goto sleep_in;
+	/* 5.19.4 SETCYC: Set display waveform cycles (B4h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETCYC,
+				     0x1c, 0x78, 0x1c, 0x78, 0x1c, 0x78, 0x01, 0x0c, 0x86, 0x75,
+				     0x00, 0x3f, 0x1c, 0x78, 0x1c, 0x78, 0x1c, 0x78, 0x01, 0x0c,
+				     0x86);
 
-	return 0;
+	/* 5.19.19 SETGIP0: Set GIP Option0 (D3h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETGIP0,
+				     0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x08, 0x32, 0x10,
+				     0x05, 0x00, 0x05, 0x32, 0x13, 0xc1, 0x00, 0x01, 0x32, 0x10,
+				     0x08, 0x00, 0x00, 0x37, 0x03, 0x07, 0x07, 0x37, 0x05, 0x05,
+				     0x37, 0x0c, 0x40);
 
-sleep_in:
-	ret = dsi_ctx.accum_err;
-	dsi_ctx.accum_err = 0;
+	/* 5.19.20 Set GIP Option1 (D5h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETGIP1,
+				     0x18, 0x18, 0x18, 0x18, 0x22, 0x23, 0x20, 0x21, 0x04, 0x05,
+				     0x06, 0x07, 0x00, 0x01, 0x02, 0x03, 0x18, 0x18, 0x18, 0x18,
+				     0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18,
+				     0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18,
+				     0x19, 0x19, 0x19, 0x19);
 
-	/* This will probably fail, but let's try orderly power off anyway. */
-	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
-	mipi_dsi_msleep(&dsi_ctx, 50);
+	/* 5.19.21 Set GIP Option2 (D6h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETGIP2,
+				     0x18, 0x18, 0x19, 0x19, 0x21, 0x20, 0x23, 0x22, 0x03, 0x02,
+				     0x01, 0x00, 0x07, 0x06, 0x05, 0x04, 0x18, 0x18, 0x18, 0x18,
+				     0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18,
+				     0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18,
+				     0x19, 0x19, 0x18, 0x18);
 
-	return ret;
-}
+	/* 5.19.25 SETGAMMA: Set gamma curve related setting (E0h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETGAMMA,
+				     0x07, 0x08, 0x09, 0x0d, 0x10, 0x14, 0x16, 0x13, 0x24, 0x36,
+				     0x48, 0x4a, 0x58, 0x6f, 0x76, 0x80, 0x97, 0xa5, 0xa8, 0xb5,
+				     0xc6, 0x62, 0x63, 0x68, 0x6f, 0x72, 0x78, 0x7f, 0x7f, 0x00,
+				     0x02, 0x08, 0x0d, 0x0c, 0x0e, 0x0f, 0x10, 0x24, 0x36, 0x48,
+				     0x4a, 0x58, 0x6f, 0x78, 0x82, 0x99, 0xa4, 0xa0, 0xb1, 0xc0,
+				     0x5e, 0x5e, 0x64, 0x6b, 0x6c, 0x73, 0x7f, 0x7f);
+
+	/* 5.19.17 SETPANEL (CCh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETPANEL,
+				     0x0b);
+
+	/* Unknown command, not listed in the HX8394-F datasheet */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_UNKNOWN1,
+				     0x1f, 0x73);
+
+	/* 5.19.5 SETVCOM: Set VCOM voltage (B6h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETVCOM,
+				     0x6b, 0x6b);
+
+	/* Unknown command, not listed in the HX8394-F datasheet */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_UNKNOWN3,
+				     0x02);
+
+	/* 5.19.11 Set register bank (BDh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETREGBANK,
+				     0x01);
+
+	/* 5.19.2 SETPOWER: Set power (B1h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETPOWER,
+				     0x00);
+
+	/* 5.19.11 Set register bank (BDh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETREGBANK,
+				     0x00);
+
+	/* Unknown command, not listed in the HX8394-F datasheet */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_UNKNOWN5,
+				     0x40, 0x81, 0x50, 0x00, 0x1a, 0xfc, 0x01);
+};
+
+static const struct drm_display_mode waveshare_5_0_inch_a_mode = {
+	.clock = 70000,
+	.hdisplay = 720,
+	.hsync_start = 720 + 40,
+	.hsync_end = 720 + 40 + 20,
+	.htotal = 720 + 40 + 20 + 20,
+	.vdisplay = 1280,
+	.vsync_start = 1280 + 30,
+	.vsync_end = 1280 + 30 + 10,
+	.vtotal = 1280 + 30 + 10 + 4,
+	.width_mm = 62,
+	.height_mm = 110,
+};
+
+static const struct hx8394_panel_desc waveshare_5_0_inch_a_desc = {
+	.mode = &waveshare_5_0_inch_a_mode,
+	.lanes = 2,
+	.mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+	.format = MIPI_DSI_FMT_RGB888,
+	.init_sequence = waveshare_5_0_inch_a_init_sequence,
+};
+
+static const struct drm_display_mode waveshare_5_5_inch_a_mode = {
+	.clock = 65000,
+	.hdisplay = 720,
+	.hsync_start = 720 + 50,
+	.hsync_end = 720 + 50 + 50,
+	.htotal = 720 + 50 + 50 + 10,
+	.vdisplay = 1280,
+	.vsync_start = 1280 + 15,
+	.vsync_end = 1280 + 15 + 12,
+	.vtotal = 1280 + 15 + 12 + 4,
+	.width_mm = 62,
+	.height_mm = 110,
+};
+
+static void waveshare_5_5_inch_a_init_sequence(struct mipi_dsi_multi_context *dsi_ctx)
+{
+	/* 5.19.8 SETEXTC: Set extension command (B9h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETEXTC,
+				     0xff, 0x83, 0x94);
+
+	/* 5.19.9 SETMIPI: Set MIPI control (BAh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETMIPI,
+				     0x61, 0x03, 0x68, 0x6b, 0xb2, 0xc0);
+
+	/* 5.19.2 SETPOWER: Set power (B1h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETPOWER,
+				     0x48, 0x12, 0x72, 0x09, 0x32, 0x54, 0x71, 0x71, 0x57, 0x47);
+
+	/* 5.19.3 SETDISP: Set display related register (B2h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETDISP,
+				     0x00, 0x80, 0x64, 0x0c, 0x0d, 0x2f);
+
+	/* 5.19.4 SETCYC: Set display waveform cycles (B4h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETCYC,
+				     0x73, 0x74, 0x73, 0x74, 0x73, 0x74, 0x01, 0x0c, 0x86, 0x75,
+				     0x00, 0x3f, 0x73, 0x74, 0x73, 0x74, 0x73, 0x74, 0x01, 0x0c,
+				     0x86);
+
+	/* 5.19.5 SETVCOM: Set VCOM voltage (B6h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETVCOM,
+				     0x86, 0x86);
+
+	/* 5.19.19 SETGIP0: Set GIP Option0 (D3h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETGIP0,
+				     0x00, 0x00, 0x07, 0x07, 0x40, 0x07, 0x0c, 0x00, 0x08, 0x10,
+				     0x08, 0x00, 0x08, 0x54, 0x15, 0x0a, 0x05, 0x0a, 0x02, 0x15,
+				     0x06, 0x05, 0x06, 0x47, 0x44, 0x0a, 0x0a, 0x4b, 0x10, 0x07,
+				     0x07, 0x0c, 0x40);
+
+	/* 5.19.20 Set GIP Option1 (D5h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETGIP1,
+				     0x1c, 0x1c, 0x1d, 0x1d, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05,
+				     0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x24, 0x25, 0x18, 0x18,
+				     0x26, 0x27, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18,
+				     0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x20, 0x21,
+				     0x18, 0x18, 0x18, 0x18);
+
+	/* 5.19.21 Set GIP Option2 (D6h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETGIP2,
+				     0x1c, 0x1c, 0x1d, 0x1d, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02,
+				     0x01, 0x00, 0x0b, 0x0a, 0x09, 0x08, 0x21, 0x20, 0x18, 0x18,
+				     0x27, 0x26, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18,
+				     0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x18, 0x25, 0x24,
+				     0x18, 0x18, 0x18, 0x18);
+
+	/* 5.19.25 SETGAMMA: Set gamma curve related setting (E0h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETGAMMA,
+				     0x00, 0x13, 0x21, 0x28, 0x2b, 0x2e, 0x32, 0x2f, 0x61, 0x6e,
+				     0x7e, 0x7b, 0x80, 0x8f, 0x91, 0x93, 0x9d, 0x9d, 0x97, 0xa4,
+				     0xb1, 0x57, 0x55, 0x58, 0x5d, 0x60, 0x67, 0x74, 0x7f, 0x00,
+				     0x13, 0x21, 0x28, 0x2b, 0x2e, 0x32, 0x2f, 0x61, 0x6e, 0x7d,
+				     0x7b, 0x7f, 0x8e, 0x90, 0x93, 0x9c, 0x9d, 0x98, 0xa4, 0xb1,
+				     0x58, 0x55, 0x59, 0x5e, 0x61, 0x68, 0x76, 0x7f);
+
+	/* Unknown command, not listed in the HX8394-F datasheet */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_UNKNOWN1,
+				     0x1f, 0x31);
+
+	/* 5.19.17 SETPANEL (CCh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETPANEL,
+				     0x07);
+
+	/* Unknown command, not listed in the HX8394-F datasheet */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_UNKNOWN3,
+				     0x02);
+
+	/* 5.19.11 Set register bank (BDh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETREGBANK,
+				     0x02);
+
+	/* Unknown command, not listed in the HX8394-F datasheet */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_UNKNOWN4,
+				     0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+				     0xff, 0xff);
+
+	/* 5.19.11 Set register bank (BDh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETREGBANK,
+				     0x00);
+
+	/* 5.19.11 Set register bank (BDh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETREGBANK,
+				     0x01);
+
+	/* 5.19.2 SETPOWER: Set power (B1h) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETPOWER,
+				     0x00);
+
+	/* 5.19.11 Set register bank (BDh) */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_SETREGBANK,
+				     0x00);
+
+	/* Unknown command, not listed in the HX8394-F datasheet */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_UNKNOWN2,
+				     0xed);
+
+	/* Unknown command, not listed in the HX8394-F datasheet */
+	mipi_dsi_dcs_write_seq_multi(dsi_ctx, HX8394_CMD_UNKNOWN6,
+				     0x00, 0xc0);
+};
+
+static const struct hx8394_panel_desc waveshare_5_5_inch_a_desc = {
+	.mode = &waveshare_5_5_inch_a_mode,
+	.lanes = 2,
+	.mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+	.format = MIPI_DSI_FMT_RGB888,
+	.init_sequence = waveshare_5_5_inch_a_init_sequence,
+};
 
 static int hx8394_disable(struct drm_panel *panel)
 {
@@ -663,6 +872,26 @@ static int hx8394_disable(struct drm_panel *panel)
 	return dsi_ctx.accum_err;
 }
 
+static int hx8394_enable(struct drm_panel *panel)
+{
+	struct hx8394 *ctx = panel_to_hx8394(panel);
+	struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = dsi };
+
+	ctx->desc->init_sequence(&dsi_ctx);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+
+	/* Panel is operational 120 msec after reset */
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	if (dsi_ctx.accum_err)
+		hx8394_disable(panel);
+
+	return dsi_ctx.accum_err;
+}
+
 static int hx8394_unprepare(struct drm_panel *panel)
 {
 	struct hx8394 *ctx = panel_to_hx8394(panel);
@@ -792,6 +1021,8 @@ static int hx8394_probe(struct mipi_dsi_device *dsi)
 	if (ret)
 		return ret;
 
+	ctx->panel.prepare_prev_first = true;
+
 	drm_panel_add(&ctx->panel);
 
 	ret = mipi_dsi_attach(dsi);
@@ -826,6 +1057,8 @@ static const struct of_device_id hx8394_of_match[] = {
 	{ .compatible = "huiling,hl055fhav028c", .data = &hl055fhav028c_desc },
 	{ .compatible = "powkiddy,x55-panel", .data = &powkiddy_x55_desc },
 	{ .compatible = "microchip,ac40t08a-mipi-panel", .data = &mchp_ac40t08a_desc },
+	{ .compatible = "waveshare,5.0-dsi-touch-a", .data = &waveshare_5_0_inch_a_desc },
+	{ .compatible = "waveshare,5.5-dsi-touch-a", .data = &waveshare_5_5_inch_a_desc },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, hx8394_of_match);
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c
index be2cf14401553..53e25d1086db8 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9806e-core.c
@@ -78,9 +78,11 @@ int ili9806e_probe(struct device *dev, void *transport,
 	bool set_prepare_prev_first = false;
 	int ret;
 
-	ctx = devm_kzalloc(dev, sizeof(struct ili9806e), GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
+	ctx = devm_drm_panel_alloc(dev, __typeof(*ctx), panel,
+				  funcs, connector_type);
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
 
 	dev_set_drvdata(dev, ctx);
 	ctx->transport = transport;
@@ -103,8 +105,6 @@ int ili9806e_probe(struct device *dev, void *transport,
 		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
 				     "Failed to get reset-gpios\n");
 
-	drm_panel_init(&ctx->panel, dev, funcs, connector_type);
-
 	ret = drm_panel_of_backlight(&ctx->panel);
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to get backlight\n");
diff --git a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
index 947b47841b010..0652cdb57d112 100644
--- a/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
+++ b/drivers/gpu/drm/panel/panel-ilitek-ili9881c.c
@@ -52,6 +52,7 @@ struct ili9881c {
 	const struct ili9881c_desc	*desc;
 
 	struct regulator	*power;
+	struct regulator	*iovcc;
 	struct gpio_desc	*reset;
 
 	enum drm_panel_orientation	orientation;
@@ -1997,6 +1998,205 @@ static const struct ili9881c_instr bsd1218_a101kl68_init[] = {
 	ILI9881C_COMMAND_INSTR(0xd3, 0x3f),
 };
 
+static const struct ili9881c_instr waveshare_7inch_a_init[] = {
+	ILI9881C_SWITCH_PAGE_INSTR(3),
+	ILI9881C_COMMAND_INSTR(0x01, 0x00),
+	ILI9881C_COMMAND_INSTR(0x02, 0x00),
+	ILI9881C_COMMAND_INSTR(0x03, 0x73),
+	ILI9881C_COMMAND_INSTR(0x04, 0x00),
+	ILI9881C_COMMAND_INSTR(0x05, 0x00),
+	ILI9881C_COMMAND_INSTR(0x06, 0x0a),
+	ILI9881C_COMMAND_INSTR(0x07, 0x00),
+	ILI9881C_COMMAND_INSTR(0x08, 0x00),
+	ILI9881C_COMMAND_INSTR(0x09, 0x61),
+	ILI9881C_COMMAND_INSTR(0x0a, 0x00),
+	ILI9881C_COMMAND_INSTR(0x0b, 0x00),
+	ILI9881C_COMMAND_INSTR(0x0c, 0x01),
+	ILI9881C_COMMAND_INSTR(0x0d, 0x00),
+	ILI9881C_COMMAND_INSTR(0x0e, 0x00),
+	ILI9881C_COMMAND_INSTR(0x0f, 0x61),
+	ILI9881C_COMMAND_INSTR(0x10, 0x61),
+	ILI9881C_COMMAND_INSTR(0x11, 0x00),
+	ILI9881C_COMMAND_INSTR(0x12, 0x00),
+	ILI9881C_COMMAND_INSTR(0x13, 0x00),
+	ILI9881C_COMMAND_INSTR(0x14, 0x00),
+	ILI9881C_COMMAND_INSTR(0x15, 0x00),
+	ILI9881C_COMMAND_INSTR(0x16, 0x00),
+	ILI9881C_COMMAND_INSTR(0x17, 0x00),
+	ILI9881C_COMMAND_INSTR(0x18, 0x00),
+	ILI9881C_COMMAND_INSTR(0x19, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1a, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1b, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1c, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1d, 0x00),
+	ILI9881C_COMMAND_INSTR(0x1e, 0x40),
+	ILI9881C_COMMAND_INSTR(0x1f, 0x80),
+	ILI9881C_COMMAND_INSTR(0x20, 0x06),
+	ILI9881C_COMMAND_INSTR(0x21, 0x01),
+	ILI9881C_COMMAND_INSTR(0x22, 0x00),
+	ILI9881C_COMMAND_INSTR(0x23, 0x00),
+	ILI9881C_COMMAND_INSTR(0x24, 0x00),
+	ILI9881C_COMMAND_INSTR(0x25, 0x00),
+	ILI9881C_COMMAND_INSTR(0x26, 0x00),
+	ILI9881C_COMMAND_INSTR(0x27, 0x00),
+	ILI9881C_COMMAND_INSTR(0x28, 0x33),
+	ILI9881C_COMMAND_INSTR(0x29, 0x03),
+	ILI9881C_COMMAND_INSTR(0x2a, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2b, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2c, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2d, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2e, 0x00),
+	ILI9881C_COMMAND_INSTR(0x2f, 0x00),
+	ILI9881C_COMMAND_INSTR(0x30, 0x00),
+	ILI9881C_COMMAND_INSTR(0x31, 0x00),
+	ILI9881C_COMMAND_INSTR(0x32, 0x00),
+	ILI9881C_COMMAND_INSTR(0x33, 0x00),
+	ILI9881C_COMMAND_INSTR(0x34, 0x04),
+	ILI9881C_COMMAND_INSTR(0x35, 0x00),
+	ILI9881C_COMMAND_INSTR(0x36, 0x00),
+	ILI9881C_COMMAND_INSTR(0x37, 0x00),
+	ILI9881C_COMMAND_INSTR(0x38, 0x3c),
+	ILI9881C_COMMAND_INSTR(0x39, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3a, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3b, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3c, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3d, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3e, 0x00),
+	ILI9881C_COMMAND_INSTR(0x3f, 0x00),
+	ILI9881C_COMMAND_INSTR(0x40, 0x00),
+	ILI9881C_COMMAND_INSTR(0x41, 0x00),
+	ILI9881C_COMMAND_INSTR(0x42, 0x00),
+	ILI9881C_COMMAND_INSTR(0x43, 0x00),
+	ILI9881C_COMMAND_INSTR(0x44, 0x00),
+	ILI9881C_COMMAND_INSTR(0x50, 0x10),
+	ILI9881C_COMMAND_INSTR(0x51, 0x32),
+	ILI9881C_COMMAND_INSTR(0x52, 0x54),
+	ILI9881C_COMMAND_INSTR(0x53, 0x76),
+	ILI9881C_COMMAND_INSTR(0x54, 0x98),
+	ILI9881C_COMMAND_INSTR(0x55, 0xba),
+	ILI9881C_COMMAND_INSTR(0x56, 0x10),
+	ILI9881C_COMMAND_INSTR(0x57, 0x32),
+	ILI9881C_COMMAND_INSTR(0x58, 0x54),
+	ILI9881C_COMMAND_INSTR(0x59, 0x76),
+	ILI9881C_COMMAND_INSTR(0x5a, 0x98),
+	ILI9881C_COMMAND_INSTR(0x5b, 0xba),
+	ILI9881C_COMMAND_INSTR(0x5c, 0xdc),
+	ILI9881C_COMMAND_INSTR(0x5d, 0xfe),
+	ILI9881C_COMMAND_INSTR(0x5e, 0x00),
+	ILI9881C_COMMAND_INSTR(0x5f, 0x0e),
+	ILI9881C_COMMAND_INSTR(0x60, 0x0f),
+	ILI9881C_COMMAND_INSTR(0x61, 0x0c),
+	ILI9881C_COMMAND_INSTR(0x62, 0x0d),
+	ILI9881C_COMMAND_INSTR(0x63, 0x06),
+	ILI9881C_COMMAND_INSTR(0x64, 0x07),
+	ILI9881C_COMMAND_INSTR(0x65, 0x02),
+	ILI9881C_COMMAND_INSTR(0x66, 0x02),
+	ILI9881C_COMMAND_INSTR(0x67, 0x02),
+	ILI9881C_COMMAND_INSTR(0x68, 0x02),
+	ILI9881C_COMMAND_INSTR(0x69, 0x01),
+	ILI9881C_COMMAND_INSTR(0x6a, 0x00),
+	ILI9881C_COMMAND_INSTR(0x6b, 0x02),
+	ILI9881C_COMMAND_INSTR(0x6c, 0x15),
+	ILI9881C_COMMAND_INSTR(0x6d, 0x14),
+	ILI9881C_COMMAND_INSTR(0x6e, 0x02),
+	ILI9881C_COMMAND_INSTR(0x6f, 0x02),
+	ILI9881C_COMMAND_INSTR(0x70, 0x02),
+	ILI9881C_COMMAND_INSTR(0x71, 0x02),
+	ILI9881C_COMMAND_INSTR(0x72, 0x02),
+	ILI9881C_COMMAND_INSTR(0x73, 0x02),
+	ILI9881C_COMMAND_INSTR(0x74, 0x02),
+	ILI9881C_COMMAND_INSTR(0x75, 0x0e),
+	ILI9881C_COMMAND_INSTR(0x76, 0x0f),
+	ILI9881C_COMMAND_INSTR(0x77, 0x0c),
+	ILI9881C_COMMAND_INSTR(0x78, 0x0d),
+	ILI9881C_COMMAND_INSTR(0x79, 0x06),
+	ILI9881C_COMMAND_INSTR(0x7a, 0x07),
+	ILI9881C_COMMAND_INSTR(0x7b, 0x02),
+	ILI9881C_COMMAND_INSTR(0x7c, 0x02),
+	ILI9881C_COMMAND_INSTR(0x7d, 0x02),
+	ILI9881C_COMMAND_INSTR(0x7e, 0x02),
+	ILI9881C_COMMAND_INSTR(0x7f, 0x01),
+	ILI9881C_COMMAND_INSTR(0x80, 0x00),
+	ILI9881C_COMMAND_INSTR(0x81, 0x02),
+	ILI9881C_COMMAND_INSTR(0x82, 0x14),
+	ILI9881C_COMMAND_INSTR(0x83, 0x15),
+	ILI9881C_COMMAND_INSTR(0x84, 0x02),
+	ILI9881C_COMMAND_INSTR(0x85, 0x02),
+	ILI9881C_COMMAND_INSTR(0x86, 0x02),
+	ILI9881C_COMMAND_INSTR(0x87, 0x02),
+	ILI9881C_COMMAND_INSTR(0x88, 0x02),
+	ILI9881C_COMMAND_INSTR(0x89, 0x02),
+	ILI9881C_COMMAND_INSTR(0x8a, 0x02),
+
+	ILI9881C_SWITCH_PAGE_INSTR(4),
+	ILI9881C_COMMAND_INSTR(0x38, 0x01),
+	ILI9881C_COMMAND_INSTR(0x39, 0x00),
+	ILI9881C_COMMAND_INSTR(0x6c, 0x15),
+	ILI9881C_COMMAND_INSTR(0x6e, 0x2a),
+	ILI9881C_COMMAND_INSTR(0x6f, 0x33),
+	ILI9881C_COMMAND_INSTR(0x3a, 0x94),
+	ILI9881C_COMMAND_INSTR(0x8d, 0x14),
+	ILI9881C_COMMAND_INSTR(0x87, 0xba),
+	ILI9881C_COMMAND_INSTR(0x26, 0x76),
+	ILI9881C_COMMAND_INSTR(0xb2, 0xd1),
+	ILI9881C_COMMAND_INSTR(0xb5, 0x06),
+	ILI9881C_COMMAND_INSTR(0x3b, 0x98),
+
+	ILI9881C_SWITCH_PAGE_INSTR(1),
+	ILI9881C_COMMAND_INSTR(0x22, 0x0a),
+	ILI9881C_COMMAND_INSTR(0x31, 0x00),
+	ILI9881C_COMMAND_INSTR(0x53, 0x71),
+	ILI9881C_COMMAND_INSTR(0x55, 0x8f),
+	ILI9881C_COMMAND_INSTR(0x40, 0x33),
+	ILI9881C_COMMAND_INSTR(0x50, 0x96),
+	ILI9881C_COMMAND_INSTR(0x51, 0x96),
+	ILI9881C_COMMAND_INSTR(0x60, 0x23),
+	ILI9881C_COMMAND_INSTR(0xa0, 0x08),
+	ILI9881C_COMMAND_INSTR(0xa1, 0x1d),
+	ILI9881C_COMMAND_INSTR(0xa2, 0x2a),
+	ILI9881C_COMMAND_INSTR(0xa3, 0x10),
+	ILI9881C_COMMAND_INSTR(0xa4, 0x15),
+	ILI9881C_COMMAND_INSTR(0xa5, 0x28),
+	ILI9881C_COMMAND_INSTR(0xa6, 0x1c),
+	ILI9881C_COMMAND_INSTR(0xa7, 0x1d),
+	ILI9881C_COMMAND_INSTR(0xa8, 0x7e),
+	ILI9881C_COMMAND_INSTR(0xa9, 0x1d),
+	ILI9881C_COMMAND_INSTR(0xaa, 0x29),
+	ILI9881C_COMMAND_INSTR(0xab, 0x6b),
+	ILI9881C_COMMAND_INSTR(0xac, 0x1a),
+	ILI9881C_COMMAND_INSTR(0xad, 0x18),
+	ILI9881C_COMMAND_INSTR(0xae, 0x4b),
+	ILI9881C_COMMAND_INSTR(0xaf, 0x20),
+	ILI9881C_COMMAND_INSTR(0xb0, 0x27),
+	ILI9881C_COMMAND_INSTR(0xb1, 0x50),
+	ILI9881C_COMMAND_INSTR(0xb2, 0x64),
+	ILI9881C_COMMAND_INSTR(0xb3, 0x39),
+	ILI9881C_COMMAND_INSTR(0xc0, 0x08),
+	ILI9881C_COMMAND_INSTR(0xc1, 0x1d),
+	ILI9881C_COMMAND_INSTR(0xc2, 0x2a),
+	ILI9881C_COMMAND_INSTR(0xc3, 0x10),
+	ILI9881C_COMMAND_INSTR(0xc4, 0x15),
+	ILI9881C_COMMAND_INSTR(0xc5, 0x28),
+	ILI9881C_COMMAND_INSTR(0xc6, 0x1c),
+	ILI9881C_COMMAND_INSTR(0xc7, 0x1d),
+	ILI9881C_COMMAND_INSTR(0xc8, 0x7e),
+	ILI9881C_COMMAND_INSTR(0xc9, 0x1d),
+	ILI9881C_COMMAND_INSTR(0xca, 0x29),
+	ILI9881C_COMMAND_INSTR(0xcb, 0x6b),
+	ILI9881C_COMMAND_INSTR(0xcc, 0x1a),
+	ILI9881C_COMMAND_INSTR(0xcd, 0x18),
+	ILI9881C_COMMAND_INSTR(0xce, 0x4b),
+	ILI9881C_COMMAND_INSTR(0xcf, 0x20),
+	ILI9881C_COMMAND_INSTR(0xd0, 0x27),
+	ILI9881C_COMMAND_INSTR(0xd1, 0x50),
+	ILI9881C_COMMAND_INSTR(0xd2, 0x64),
+	ILI9881C_COMMAND_INSTR(0xd3, 0x39),
+
+	ILI9881C_SWITCH_PAGE_INSTR(0),
+	ILI9881C_COMMAND_INSTR(0x3a, 0x77),
+	ILI9881C_COMMAND_INSTR(0x36, 0x00),
+};
+
 static inline struct ili9881c *panel_to_ili9881c(struct drm_panel *panel)
 {
 	return container_of(panel, struct ili9881c, panel);
@@ -2035,9 +2235,19 @@ static int ili9881c_prepare(struct drm_panel *panel)
 	int ret;
 
 	/* Power the panel */
+	if (ctx->iovcc) {
+		ret = regulator_enable(ctx->iovcc);
+		if (ret)
+			return ret;
+	}
+
+	msleep(5);
 	ret = regulator_enable(ctx->power);
-	if (ret)
-		return ret;
+	if (ret) {
+		mctx.accum_err = ret;
+		goto disable_iovcc;
+	}
+
 	msleep(5);
 
 	/* And reset it */
@@ -2074,6 +2284,9 @@ static int ili9881c_prepare(struct drm_panel *panel)
 
 disable_power:
 	regulator_disable(ctx->power);
+disable_iovcc:
+	if (ctx->iovcc)
+		regulator_disable(ctx->iovcc);
 	return mctx.accum_err;
 }
 
@@ -2085,6 +2298,8 @@ static int ili9881c_unprepare(struct drm_panel *panel)
 	mipi_dsi_dcs_set_display_off_multi(&mctx);
 	mipi_dsi_dcs_enter_sleep_mode_multi(&mctx);
 	regulator_disable(ctx->power);
+	if (ctx->iovcc)
+		regulator_disable(ctx->iovcc);
 	gpiod_set_value_cansleep(ctx->reset, 1);
 
 	return 0;
@@ -2260,6 +2475,23 @@ static const struct drm_display_mode bsd1218_a101kl68_default_mode = {
 	.height_mm	= 170,
 };
 
+static const struct drm_display_mode waveshare_7inch_a_mode = {
+	.clock		= 83333,
+
+	.hdisplay	= 720,
+	.hsync_start	= 720 + 120,
+	.hsync_end	= 720 + 120 + 100,
+	.htotal		= 720 + 120 + 100 + 100,
+
+	.vdisplay	= 1280,
+	.vsync_start	= 1280 + 10,
+	.vsync_end	= 1280 + 10 + 10,
+	.vtotal		= 1280 + 10 + 10 + 10,
+
+	.width_mm	= 85,
+	.height_mm	= 154,
+};
+
 static int ili9881c_get_modes(struct drm_panel *panel,
 			      struct drm_connector *connector)
 {
@@ -2329,6 +2561,11 @@ static int ili9881c_dsi_probe(struct mipi_dsi_device *dsi)
 		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->power),
 				     "Couldn't get our power regulator\n");
 
+	ctx->iovcc = devm_regulator_get_optional(&dsi->dev, "iovcc");
+	if (IS_ERR(ctx->iovcc))
+		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->iovcc),
+				     "Couldn't get our iovcc regulator\n");
+
 	ctx->reset = devm_gpiod_get_optional(&dsi->dev, "reset", GPIOD_OUT_LOW);
 	if (IS_ERR(ctx->reset))
 		return dev_err_probe(&dsi->dev, PTR_ERR(ctx->reset),
@@ -2454,6 +2691,15 @@ static const struct ili9881c_desc bsd1218_a101kl68_desc = {
 	.lanes = 4,
 };
 
+static const struct ili9881c_desc waveshare_7inch_a_desc = {
+	.init = waveshare_7inch_a_init,
+	.init_length = ARRAY_SIZE(waveshare_7inch_a_init),
+	.mode = &waveshare_7inch_a_mode,
+	.mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_HSE |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+	.lanes = 2,
+};
+
 static const struct of_device_id ili9881c_of_match[] = {
 	{ .compatible = "bananapi,lhr050h41", .data = &lhr050h41_desc },
 	{ .compatible = "bestar,bsd1218-a101kl68", .data = &bsd1218_a101kl68_desc },
@@ -2462,6 +2708,7 @@ static const struct of_device_id ili9881c_of_match[] = {
 	{ .compatible = "tdo,tl050hdv35", .data = &tl050hdv35_desc },
 	{ .compatible = "wanchanglong,w552946aaa", .data = &w552946aaa_desc },
 	{ .compatible = "wanchanglong,w552946aba", .data = &w552946aba_desc },
+	{ .compatible = "waveshare,7.0-dsi-touch-a", .data = &waveshare_7inch_a_desc },
 	{ .compatible = "ampire,am8001280g", .data = &am8001280g_desc },
 	{ .compatible = "raspberrypi,dsi-5inch", &rpi_5inch_desc },
 	{ .compatible = "raspberrypi,dsi-7inch", &rpi_7inch_desc },
diff --git a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
index c33c611e03c00..f6b04de1182e3 100644
--- a/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
+++ b/drivers/gpu/drm/panel/panel-jadard-jd9365da-h3.c
@@ -10,8 +10,10 @@
 
 #include <drm/drm_mipi_dsi.h>
 #include <drm/drm_modes.h>
+#include <drm/drm_of.h>
 #include <drm/drm_panel.h>
 #include <drm/drm_print.h>
+#include <drm/drm_probe_helper.h>
 
 #include <linux/gpio/consumer.h>
 #include <linux/delay.h>
@@ -19,10 +21,13 @@
 #include <linux/of.h>
 #include <linux/regulator/consumer.h>
 
+#include <video/mipi_display.h>
+
 struct jadard;
 
 struct jadard_panel_desc {
-	const struct drm_display_mode mode;
+	const struct drm_display_mode *mode_4ln;
+	const struct drm_display_mode *mode_2ln;
 	unsigned int lanes;
 	enum mipi_dsi_pixel_format format;
 	int (*init)(struct jadard *jadard);
@@ -56,7 +61,10 @@ static void jadard_enable_standard_cmds(struct mipi_dsi_multi_context *dsi_ctx)
 	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xe1, 0x93);
 	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xe2, 0x65);
 	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0xe3, 0xf8);
-	mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x80, 0x03);
+	if (dsi_ctx->dsi->lanes == 2)
+		mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x80, 0x01);
+	else
+		mipi_dsi_dcs_write_seq_multi(dsi_ctx, 0x80, 0x03);
 }
 
 static inline struct jadard *panel_to_jadard(struct drm_panel *panel)
@@ -149,24 +157,11 @@ static int jadard_get_modes(struct drm_panel *panel,
 			    struct drm_connector *connector)
 {
 	struct jadard *jadard = panel_to_jadard(panel);
-	const struct drm_display_mode *desc_mode = &jadard->desc->mode;
-	struct drm_display_mode *mode;
-
-	mode = drm_mode_duplicate(connector->dev, desc_mode);
-	if (!mode) {
-		DRM_DEV_ERROR(&jadard->dsi->dev, "failed to add mode %ux%ux@%u\n",
-			      desc_mode->hdisplay, desc_mode->vdisplay,
-			      drm_mode_vrefresh(desc_mode));
-		return -ENOMEM;
-	}
-
-	drm_mode_set_name(mode);
-	drm_mode_probed_add(connector, mode);
 
-	connector->display_info.width_mm = mode->width_mm;
-	connector->display_info.height_mm = mode->height_mm;
-
-	return 1;
+	if (jadard->dsi->lanes == 2)
+		return drm_connector_helper_get_modes_fixed(connector, jadard->desc->mode_2ln);
+	else
+		return drm_connector_helper_get_modes_fixed(connector, jadard->desc->mode_4ln);
 }
 
 static enum drm_panel_orientation jadard_panel_get_orientation(struct drm_panel *panel)
@@ -369,7 +364,7 @@ static int radxa_display_8hd_ad002_init_cmds(struct jadard *jadard)
 };
 
 static const struct jadard_panel_desc radxa_display_8hd_ad002_desc = {
-	.mode = {
+	.mode_4ln = &(const struct drm_display_mode) {
 		.clock		= 70000,
 
 		.hdisplay	= 800,
@@ -601,7 +596,7 @@ static int cz101b4001_init_cmds(struct jadard *jadard)
 };
 
 static const struct jadard_panel_desc cz101b4001_desc = {
-	.mode = {
+	.mode_4ln = &(const struct drm_display_mode) {
 		.clock		= 70000,
 
 		.hdisplay	= 800,
@@ -834,7 +829,7 @@ static int kingdisplay_kd101ne3_init_cmds(struct jadard *jadard)
 };
 
 static const struct jadard_panel_desc kingdisplay_kd101ne3_40ti_desc = {
-	.mode = {
+	.mode_4ln = &(const struct drm_display_mode) {
 		.clock		= (800 + 24 + 24 + 24) * (1280 + 30 + 4 + 8) * 60 / 1000,
 
 		.hdisplay	= 800,
@@ -1085,7 +1080,7 @@ static int melfas_lmfbx101117480_init_cmds(struct jadard *jadard)
 };
 
 static const struct jadard_panel_desc melfas_lmfbx101117480_desc = {
-	.mode = {
+	.mode_4ln = &(const struct drm_display_mode) {
 		.clock		= (800 + 24 + 24 + 24) * (1280 + 30 + 4 + 8) * 60 / 1000,
 
 		.hdisplay	= 800,
@@ -1341,7 +1336,7 @@ static int anbernic_rgds_init_cmds(struct jadard *jadard)
 };
 
 static const struct jadard_panel_desc anbernic_rgds_display_desc = {
-	.mode = {
+	.mode_4ln = &(const struct drm_display_mode) {
 		.clock		= (640 + 260 + 220 + 260) * (480 + 10 + 2 + 16) * 60 / 1000,
 
 		.hdisplay	= 640,
@@ -1577,7 +1572,7 @@ static int taiguan_xti05101_01a_init_cmds(struct jadard *jadard)
 };
 
 static const struct jadard_panel_desc taiguan_xti05101_01a_desc = {
-	.mode = {
+	.mode_4ln = &(const struct drm_display_mode) {
 		.clock		= (800 + 24 + 24 + 24) * (1280 + 30 + 4 + 8) * 60 / 1000,
 
 		.hdisplay	= 800,
@@ -1606,107 +1601,1485 @@ static const struct jadard_panel_desc taiguan_xti05101_01a_desc = {
 	.enter_sleep_to_reset_down_delay_ms = 100,
 };
 
-static int jadard_dsi_probe(struct mipi_dsi_device *dsi)
+static int waveshare_3_4_c_init(struct jadard *jadard)
 {
-	struct device *dev = &dsi->dev;
-	const struct jadard_panel_desc *desc;
-	struct jadard *jadard;
-	int ret;
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = jadard->dsi };
 
-	jadard = devm_drm_panel_alloc(dev, struct jadard, panel, &jadard_funcs,
-				      DRM_MODE_CONNECTOR_DSI);
-	if (IS_ERR(jadard))
-		return PTR_ERR(jadard);
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+	jadard_enable_standard_cmds(&dsi_ctx);
 
-	desc = of_device_get_match_data(dev);
+	jd9365da_switch_page(&dsi_ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0xd0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0xd0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0xfe);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0x26);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x78);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0x64);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x42, 0xc7);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x18);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x1b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x19);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x56);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x33);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x25);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x16);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x2f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x32);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x53);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x4c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x3d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x31);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x20);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x70, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x71, 0x56);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x72, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x73, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x74, 0x33);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0x25);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x16);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x79, 0x2f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x32);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7b, 0x53);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7c, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x4c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7f, 0x3d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x80, 0x31);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x81, 0x20);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x82, 0x0f);
 
-	if (desc->mode_flags)
-		dsi->mode_flags = desc->mode_flags;
-	else
-		dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
-				  MIPI_DSI_MODE_VIDEO_BURST |
-				  MIPI_DSI_MODE_NO_EOT_PACKET;
+	jd9365da_switch_page(&dsi_ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0x5e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x5e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x50);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x05, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x06, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x07, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x08, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0a, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0b, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0d, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0f, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x10, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x11, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x12, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x13, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x14, 0x42);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x15, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x16, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0x5e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x5e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x50);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1d, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1e, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1f, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x20, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x21, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x22, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x23, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x25, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x26, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x27, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x28, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x29, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2a, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2c, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2d, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2e, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2f, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x30, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x31, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x32, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x33, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x34, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x36, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3b, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x42, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x46, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x47, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x48, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x49, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4a, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4b, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4c, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4d, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4e, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4f, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x50, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x51, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x52, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x53, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x54, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x56, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0x1f);
 
-	dsi->format = desc->format;
-	dsi->lanes = desc->lanes;
+	jd9365da_switch_page(&dsi_ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x58, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5c, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0xa6);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x73);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0xd9);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x33);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x43);
 
-	jadard->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
-	if (IS_ERR(jadard->reset))
-		return dev_err_probe(&dsi->dev, PTR_ERR(jadard->reset),
-				"failed to get our reset GPIO\n");
+	jd9365da_switch_page(&dsi_ctx, 0x00);
 
-	jadard->vdd = devm_regulator_get(dev, "vdd");
-	if (IS_ERR(jadard->vdd))
-		return dev_err_probe(&dsi->dev, PTR_ERR(jadard->vdd),
-				"failed to get vdd regulator\n");
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	msleep(120);
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	msleep(5);
+	mipi_dsi_dcs_set_tear_on_multi(&dsi_ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
 
-	jadard->vccio = devm_regulator_get(dev, "vccio");
-	if (IS_ERR(jadard->vccio))
-		return dev_err_probe(&dsi->dev, PTR_ERR(jadard->vccio),
-				"failed to get vccio regulator\n");
+	return dsi_ctx.accum_err;
+}
 
-	ret = of_drm_get_panel_orientation(dev->of_node, &jadard->orientation);
-	if (ret < 0)
-		return dev_err_probe(dev, ret, "failed to get orientation\n");
+static const struct jadard_panel_desc waveshare_3_4_inch_c_desc = {
+	.mode_2ln = &(const struct drm_display_mode) {
+		.clock		= (800 + 40 + 20 + 20) * (800 + 24 + 4 + 12) * 60 / 1000,
 
-	ret = drm_panel_of_backlight(&jadard->panel);
-	if (ret)
-		return ret;
+		.hdisplay	= 800,
+		.hsync_start	= 800 + 40,
+		.hsync_end	= 800 + 40 + 20,
+		.htotal		= 800 + 40 + 20 + 20,
 
-	drm_panel_add(&jadard->panel);
+		.vdisplay	= 800,
+		.vsync_start	= 800 + 24,
+		.vsync_end	= 800 + 24 + 4,
+		.vtotal		= 800 + 24 + 4 + 12,
 
-	mipi_dsi_set_drvdata(dsi, jadard);
-	jadard->dsi = dsi;
-	jadard->desc = desc;
+		.width_mm	= 88,
+		.height_mm	= 88,
+		.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+	},
+	.lanes = 2,
+	.format = MIPI_DSI_FMT_RGB888,
+	.init = waveshare_3_4_c_init,
+	.mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+};
 
-	ret = mipi_dsi_attach(dsi);
-	if (ret < 0)
-		drm_panel_remove(&jadard->panel);
+static int waveshare_4_0_c_init(struct jadard *jadard)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = jadard->dsi };
 
-	return ret;
-}
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+	jadard_enable_standard_cmds(&dsi_ctx);
 
-static void jadard_dsi_remove(struct mipi_dsi_device *dsi)
-{
-	struct jadard *jadard = mipi_dsi_get_drvdata(dsi);
+	jd9365da_switch_page(&dsi_ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0xd0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0xd0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0xfe);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0x26);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x78);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0x64);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x42, 0xc7);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x18);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x1b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x19);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x56);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x33);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x25);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x16);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x2f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x32);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x53);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x4c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x3d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x31);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x20);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x70, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x71, 0x56);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x72, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x73, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x74, 0x33);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0x25);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x16);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x79, 0x2f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x32);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7b, 0x53);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7c, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x4c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7f, 0x3d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x80, 0x31);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x81, 0x20);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x82, 0x0f);
 
-	mipi_dsi_detach(dsi);
-	drm_panel_remove(&jadard->panel);
+	jd9365da_switch_page(&dsi_ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0x5e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x5e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x50);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x05, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x06, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x07, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x08, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0a, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0b, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0d, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0f, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x10, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x11, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x12, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x13, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x14, 0x42);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x15, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x16, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0x5e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x5e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x50);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1d, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1e, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1f, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x20, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x21, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x22, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x23, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x25, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x26, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x27, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x28, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x29, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2a, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2c, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2d, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2e, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2f, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x30, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x31, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x32, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x33, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x34, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x36, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3b, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x42, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x46, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x47, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x48, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x49, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4a, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4b, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4c, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4d, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4e, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4f, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x50, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x51, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x52, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x53, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x54, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x56, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0x1f);
+
+	jd9365da_switch_page(&dsi_ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x58, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5c, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0xa6);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x73);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0xd9);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x33);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x43);
+
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	msleep(120);
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	msleep(5);
+	mipi_dsi_dcs_set_tear_on_multi(&dsi_ctx, MIPI_DSI_DCS_TEAR_MODE_VBLANK);
+
+	return dsi_ctx.accum_err;
 }
 
-static const struct of_device_id jadard_of_match[] = {
-	{
-		.compatible = "anbernic,rg-ds-display-bottom",
-		.data = &anbernic_rgds_display_desc
-	},
-	{
-		.compatible = "anbernic,rg-ds-display-top",
-		.data = &anbernic_rgds_display_desc
-	},
-	{
-		.compatible = "chongzhou,cz101b4001",
-		.data = &cz101b4001_desc
-	},
-	{
-		.compatible = "kingdisplay,kd101ne3-40ti",
-		.data = &kingdisplay_kd101ne3_40ti_desc
-	},
-	{
-		.compatible = "melfas,lmfbx101117480",
-		.data = &melfas_lmfbx101117480_desc
-	},
-	{
-		.compatible = "radxa,display-10hd-ad001",
-		.data = &cz101b4001_desc
-	},
-	{
-		.compatible = "radxa,display-8hd-ad002",
-		.data = &radxa_display_8hd_ad002_desc
+static const struct jadard_panel_desc waveshare_4_0_inch_c_desc = {
+	.mode_2ln = &(const struct drm_display_mode) {
+		.clock		= (720 + 40 + 20 + 20) * (720 + 24 + 4 + 12) * 60 / 1000,
+
+		.hdisplay	= 720,
+		.hsync_start	= 720 + 40,
+		.hsync_end	= 720 + 40 + 20,
+		.htotal		= 720 + 40 + 20 + 20,
+
+		.vdisplay	= 720,
+		.vsync_start	= 720 + 24,
+		.vsync_end	= 720 + 24 + 4,
+		.vtotal		= 720 + 24 + 4 + 12,
+
+		.width_mm	= 88,
+		.height_mm	= 88,
+		.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+	},
+	.lanes = 2,
+	.format = MIPI_DSI_FMT_RGB888,
+	.init = waveshare_4_0_c_init,
+	.mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+};
+
+static int waveshare_8_0_a_init(struct jadard *jadard)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = jadard->dsi };
+
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+	jadard_enable_standard_cmds(&dsi_ctx);
+
+	jd9365da_switch_page(&dsi_ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x00);
+	if (jadard->dsi->lanes == 4)
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x7e);
+	else
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x4e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x65);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x74);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0xb7);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0xb7);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0xfe);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x19);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3b, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x70);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0xa0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4b, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x56, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0xa9);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x58, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x19);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x78);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x63);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x54);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x38);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x3d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x62);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x50);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x23);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x70, 0x78);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x71, 0x63);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x72, 0x54);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x73, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x74, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0x38);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x3d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x79, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x43);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7b, 0x62);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7c, 0x50);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7f, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x80, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x81, 0x23);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x82, 0x10);
+
+	jd9365da_switch_page(&dsi_ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x05, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x06, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x07, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x08, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0a, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0b, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0d, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0f, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x10, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x11, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x12, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x13, 0x35);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x14, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x15, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x16, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1d, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1e, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1f, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x20, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x21, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x22, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x23, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x25, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x26, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x27, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x28, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x29, 0x35);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2a, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x58, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5c, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x30);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x6b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x0c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x73);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x56);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x70, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x71, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x72, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x73, 0x7b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x74, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0xf8);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0xd5);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x2e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x79, 0x12);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7b, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x03);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x7b);
+
+	jd9365da_switch_page(&dsi_ctx, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x0e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0xb3);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x60);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x36, 0x59);
+	if (jadard->dsi->lanes != 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x58);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x0f);
+	}
+
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	msleep(120);
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	msleep(60);
+
+	return 0;
+}
+
+static const struct drm_display_mode waveshare_8_0_a_mode = {
+	.clock		= (800 + 40 + 20 + 20) * (1280 + 30 + 12 + 4) * 60 / 1000,
+
+	.hdisplay	= 800,
+	.hsync_start	= 800 + 40,
+	.hsync_end	= 800 + 40 + 20,
+	.htotal		= 800 + 40 + 20 + 20,
+
+	.vdisplay	= 1280,
+	.vsync_start	= 1280 + 30,
+	.vsync_end	= 1280 + 30 + 12,
+	.vtotal		= 1280 + 30 + 12 + 4,
+
+	.width_mm	= 107,
+	.height_mm	= 172,
+	.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static const struct jadard_panel_desc waveshare_8_0_inch_a_desc = {
+	.mode_4ln = &waveshare_8_0_a_mode,
+	.mode_2ln = &waveshare_8_0_a_mode,
+	.format = MIPI_DSI_FMT_RGB888,
+	.init = waveshare_8_0_a_init,
+	.mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+		      MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+};
+
+static int waveshare_10_1_b_init(struct jadard *jadard);
+
+static const struct jadard_panel_desc waveshare_9_0_inch_b_desc = {
+	.mode_4ln = &(const struct drm_display_mode) {
+		.clock		= (720 + 60 + 60 + 4) * (1280 + 16 + 12 + 4) * 60 / 1000,
+
+		.hdisplay	= 720,
+		.hsync_start	= 720 + 60,
+		.hsync_end	= 720 + 60 + 60,
+		.htotal		= 720 + 60 + 60 + 4,
+
+		.vdisplay	= 1280,
+		.vsync_start	= 1280 + 16,
+		.vsync_end	= 1280 + 16 + 12,
+		.vtotal		= 1280 + 16 + 12 + 4,
+
+		.width_mm	= 114,
+		.height_mm	= 196,
+		.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+	},
+	.mode_2ln = &(const struct drm_display_mode) {
+		.clock		= (720 + 50 + 50 + 50) * (1280 + 26 + 12 + 4) * 60 / 1000,
+
+		.hdisplay	= 720,
+		.hsync_start	= 720 + 50,
+		.hsync_end	= 720 + 50 + 50,
+		.htotal		= 720 + 50 + 50 + 50,
+
+		.vdisplay	= 1280,
+		.vsync_start	= 1280 + 26,
+		.vsync_end	= 1280 + 26 + 12,
+		.vtotal		= 1280 + 26 + 12 + 4,
+
+		.width_mm	= 114,
+		.height_mm	= 196,
+		.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+	},
+	.format = MIPI_DSI_FMT_RGB888,
+	.init = waveshare_10_1_b_init,
+	.mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+		MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+};
+
+static const struct drm_display_mode waveshare_10_1_a_mode = {
+	.clock		= (800 + 40 + 20 + 20) * (1280 + 20 + 20 + 4) * 60 / 1000,
+
+	.hdisplay	= 800,
+	.hsync_start	= 800 + 40,
+	.hsync_end	= 800 + 40 + 20,
+	.htotal		= 800 + 40 + 20 + 20,
+
+	.vdisplay	= 1280,
+	.vsync_start	= 1280 + 20,
+	.vsync_end	= 1280 + 20 + 20,
+	.vtotal		= 1280 + 20 + 20 + 4,
+
+	.width_mm	= 135,
+	.height_mm	= 216,
+	.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+};
+
+static int waveshare_10_1_a_init(struct jadard *jadard)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = jadard->dsi };
+
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+	jadard_enable_standard_cmds(&dsi_ctx);
+
+	jd9365da_switch_page(&dsi_ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x00);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x3b);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x38);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x10);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x38);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x74);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0xaf);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0xaf);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0x26);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x78);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0xa0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x42, 0x81);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x14);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x23);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x1e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x0d);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0x69);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x7f);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x6b);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x5c);
+	} else  {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x6a);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x5b);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x4f);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x4d);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x3f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x42);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x2b);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x4a);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x3d);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x41);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x2a);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x43);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x43);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x63);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x44);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x62);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x52);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x5a);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x4f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x4e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x20);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x0f);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x59);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x4c);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x48);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x3a);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x26);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x70, 0x7f);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x71, 0x6b);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x72, 0x5c);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x71, 0x6a);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x72, 0x5b);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x73, 0x4f);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x74, 0x4d);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0x3f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x42);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x2b);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x74, 0x4a);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0x3d);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x41);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x2a);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x79, 0x43);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x43);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7b, 0x63);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x44);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7b, 0x62);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7c, 0x52);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x5a);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x4f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7f, 0x4e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x80, 0x20);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x81, 0x0f);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x59);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x4c);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7f, 0x48);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x80, 0x3a);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x81, 0x26);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x82, 0x00);
+
+	jd9365da_switch_page(&dsi_ctx, 0x02);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x02);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x02);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0x00);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x00);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x1e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x05, 0x1e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x06, 0x1f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x07, 0x1f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x08, 0x1f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x17);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0a, 0x17);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0b, 0x37);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x37);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x42);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x42);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0x40);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x40);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x5e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x05, 0x5e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x06, 0x5f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x07, 0x5f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x08, 0x5f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x57);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0a, 0x57);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0b, 0x77);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x77);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0d, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0f, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x10, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x11, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x12, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x13, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x14, 0x49);
+	if (jadard->dsi->lanes == 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x15, 0x1f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x16, 0x01);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x01);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0x00);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x00);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x1e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0x1e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x1f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1d, 0x1f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1e, 0x1f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1f, 0x17);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x20, 0x17);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x21, 0x37);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x22, 0x37);
+	} else {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x15, 0x5f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x16, 0x41);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x41);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0x40);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x40);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x5e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0x5e);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x5f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1d, 0x5f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1e, 0x5f);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1f, 0x57);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x20, 0x57);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x21, 0x77);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x22, 0x77);
+	}
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x23, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x25, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x26, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x27, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x28, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x29, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2a, 0x48);
+	if (jadard->dsi->lanes == 4)
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x1f);
+	else
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2c, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2d, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2e, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2f, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x30, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x31, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x32, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x33, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x34, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x36, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3b, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x42, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x46, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x47, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x48, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x49, 0x1e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4a, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4b, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4c, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4d, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4e, 0x37);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4f, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x50, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x51, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x52, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x53, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x54, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x56, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x58, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x30);
+	if (jadard->dsi->lanes == 4)
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5c, 0x16);
+	else
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x34);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x6a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x73);
+	if (jadard->dsi->lanes == 4)
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x1d);
+	else
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x6a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0xdd);
+	if (jadard->dsi->lanes == 4)
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x3f);
+	else
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x2c);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x79, 0x15);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x82);
+
+	jd9365da_switch_page(&dsi_ctx, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x0e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0xb3);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x61);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x48);
+	if (jadard->dsi->lanes != 4) {
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x58);
+		mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x0f);
+	}
+
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xe6, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0xe7, 0x0c);
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	msleep(120);
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	msleep(60);
+
+	return dsi_ctx.accum_err;
+}
+
+static const struct jadard_panel_desc waveshare_10_1_inch_a_desc = {
+	.mode_4ln = &waveshare_10_1_a_mode,
+	.mode_2ln = &waveshare_10_1_a_mode,
+	.format = MIPI_DSI_FMT_RGB888,
+	.init = waveshare_10_1_a_init,
+	.mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+		MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+};
+
+static int waveshare_10_1_b_init(struct jadard *jadard)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = jadard->dsi };
+
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+	jadard_enable_standard_cmds(&dsi_ctx);
+
+	jd9365da_switch_page(&dsi_ctx, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x3d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x3f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0xbf);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0xbf);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0xfe);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x19);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x74);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0xff);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0xa0);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x42, 0x7e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x1a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x0f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x24);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0xa9);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x59, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5a, 0x38);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x1a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x65);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x52);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x3d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x2d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x2d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x25);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x23);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x3f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x2d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x34);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x27);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x24);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x18);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x70, 0x7f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x71, 0x65);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x72, 0x52);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x73, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x74, 0x3d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0x2d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x2d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0x14);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x28);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x79, 0x25);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x23);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7b, 0x3f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7c, 0x2d);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x34);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x27);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7f, 0x24);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x80, 0x18);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x81, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x82, 0x00);
+
+	jd9365da_switch_page(&dsi_ctx, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x51);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x01, 0x55);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0x50);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x03, 0x51);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x04, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x05, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x06, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x07, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x08, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0a, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0b, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0c, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0d, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0f, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x10, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x11, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x12, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x13, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x14, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x15, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x16, 0x51);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x17, 0x55);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x18, 0x50);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x19, 0x51);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1a, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1b, 0x77);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1c, 0x57);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1d, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1e, 0x47);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x1f, 0x46);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x20, 0x45);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x21, 0x44);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x22, 0x4b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x23, 0x4a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x24, 0x49);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x25, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x26, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x27, 0x41);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x28, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x29, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2a, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x5f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2c, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2d, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2e, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2f, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x30, 0x15);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x31, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x32, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x33, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x34, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x35, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x36, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x38, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x39, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3a, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3b, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3c, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3d, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3e, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x3f, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x40, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x41, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x42, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x43, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x44, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x45, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x46, 0x15);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x47, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x48, 0x17);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x49, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4a, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4b, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4c, 0x0a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4d, 0x0b);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4e, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x4f, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x50, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x51, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x52, 0x11);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x53, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x54, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x55, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x56, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x57, 0x1f);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x58, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5b, 0x10);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5c, 0x07);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5d, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5e, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x5f, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x60, 0x40);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x61, 0x01);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x62, 0x02);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x63, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x64, 0x66);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x65, 0x55);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x66, 0x13);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x67, 0x73);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x68, 0x09);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x69, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6a, 0x66);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6b, 0x08);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6c, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6d, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6e, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x6f, 0x88);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x75, 0xe3);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x76, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x77, 0xd5);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x78, 0x2a);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x79, 0x21);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7a, 0x00);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7d, 0x06);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x7e, 0x66);
+
+	jd9365da_switch_page(&dsi_ctx, 0x04);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x00, 0x0e);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x02, 0xb3);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x09, 0x60);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x0e, 0x48);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x37, 0x58);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x2b, 0x0f);
+
+	jd9365da_switch_page(&dsi_ctx, 0x05);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, 0x15, 0x1d);
+
+	jd9365da_switch_page(&dsi_ctx, 0x00);
+	mipi_dsi_dcs_exit_sleep_mode_multi(&dsi_ctx);
+	msleep(120);
+	mipi_dsi_dcs_set_display_on_multi(&dsi_ctx);
+	msleep(5);
+	mipi_dsi_dcs_write_seq_multi(&dsi_ctx, MIPI_DCS_SET_TEAR_ON);
+
+	return 0;
+}
+
+static const struct jadard_panel_desc waveshare_10_1_inch_b_desc = {
+	.mode_4ln = &(const struct drm_display_mode) {
+		.clock		= (720 + 60 + 60 + 4) * (1280 + 16 + 12 + 4) * 60 / 1000,
+
+		.hdisplay	= 720,
+		.hsync_start	= 720 + 60,
+		.hsync_end	= 720 + 60 + 60,
+		.htotal		= 720 + 60 + 60 + 4,
+
+		.vdisplay	= 1280,
+		.vsync_start	= 1280 + 16,
+		.vsync_end	= 1280 + 16 + 12,
+		.vtotal		= 1280 + 16 + 12 + 4,
+
+		.width_mm	= 125,
+		.height_mm	= 222,
+		.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+	},
+	.mode_2ln = &(const struct drm_display_mode) {
+		.clock		= (720 + 50 + 50 + 50) * (1280 + 26 + 12 + 4) * 60 / 1000,
+
+		.hdisplay	= 720,
+		.hsync_start	= 720 + 50,
+		.hsync_end	= 720 + 50 + 50,
+		.htotal		= 720 + 50 + 50 + 50,
+
+		.vdisplay	= 1280,
+		.vsync_start	= 1280 + 26,
+		.vsync_end	= 1280 + 26 + 12,
+		.vtotal		= 1280 + 26 + 12 + 4,
+
+		.width_mm	= 125,
+		.height_mm	= 222,
+		.type		= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
+	},
+	.format = MIPI_DSI_FMT_RGB888,
+	.init = waveshare_10_1_b_init,
+	.mode_flags = MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_VIDEO |
+		MIPI_DSI_MODE_LPM | MIPI_DSI_CLOCK_NON_CONTINUOUS,
+};
+
+static int jadard_dsi_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	const struct jadard_panel_desc *desc;
+	struct jadard *jadard;
+	int ret;
+
+	jadard = devm_drm_panel_alloc(dev, struct jadard, panel, &jadard_funcs,
+				      DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(jadard))
+		return PTR_ERR(jadard);
+
+	desc = of_device_get_match_data(dev);
+
+	if (desc->mode_flags)
+		dsi->mode_flags = desc->mode_flags;
+	else
+		dsi->mode_flags = MIPI_DSI_MODE_VIDEO |
+				  MIPI_DSI_MODE_VIDEO_BURST |
+				  MIPI_DSI_MODE_NO_EOT_PACKET;
+
+	dsi->format = desc->format;
+	dsi->lanes = desc->lanes;
+	if (!dsi->lanes) {
+		ret = drm_of_get_data_lanes_count_remote(dsi->dev.of_node, 0, -1, 2, 4);
+		if (ret < 0)
+			return ret;
+		dsi->lanes = ret;
+
+		if (dsi->lanes == 4) {
+			if (!desc->mode_4ln) {
+				dev_err(&dsi->dev, "4-lane config is not supported\n");
+				return -EINVAL;
+			}
+		} else if (dsi->lanes == 2) {
+			if (!desc->mode_2ln) {
+				dev_err(&dsi->dev, "2-lane config is not supported\n");
+				return -EINVAL;
+			}
+		} else {
+			dev_err(&dsi->dev, "Unsupported number of lanes, %d\n", dsi->lanes);
+			return -ENODEV;
+		}
+	}
+	dev_dbg(&dsi->dev, "lanes: %d\n", dsi->lanes);
+
+	jadard->reset = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(jadard->reset))
+		return dev_err_probe(&dsi->dev, PTR_ERR(jadard->reset),
+				"failed to get our reset GPIO\n");
+
+	jadard->vdd = devm_regulator_get(dev, "vdd");
+	if (IS_ERR(jadard->vdd))
+		return dev_err_probe(&dsi->dev, PTR_ERR(jadard->vdd),
+				"failed to get vdd regulator\n");
+
+	jadard->vccio = devm_regulator_get(dev, "vccio");
+	if (IS_ERR(jadard->vccio))
+		return dev_err_probe(&dsi->dev, PTR_ERR(jadard->vccio),
+				"failed to get vccio regulator\n");
+
+	ret = of_drm_get_panel_orientation(dev->of_node, &jadard->orientation);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to get orientation\n");
+
+	ret = drm_panel_of_backlight(&jadard->panel);
+	if (ret)
+		return ret;
+
+	jadard->panel.prepare_prev_first = true;
+
+	drm_panel_add(&jadard->panel);
+
+	mipi_dsi_set_drvdata(dsi, jadard);
+	jadard->dsi = dsi;
+	jadard->desc = desc;
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0)
+		drm_panel_remove(&jadard->panel);
+
+	return ret;
+}
+
+static void jadard_dsi_remove(struct mipi_dsi_device *dsi)
+{
+	struct jadard *jadard = mipi_dsi_get_drvdata(dsi);
+
+	mipi_dsi_detach(dsi);
+	drm_panel_remove(&jadard->panel);
+}
+
+static const struct of_device_id jadard_of_match[] = {
+	{
+		.compatible = "anbernic,rg-ds-display-bottom",
+		.data = &anbernic_rgds_display_desc
+	},
+	{
+		.compatible = "anbernic,rg-ds-display-top",
+		.data = &anbernic_rgds_display_desc
+	},
+	{
+		.compatible = "chongzhou,cz101b4001",
+		.data = &cz101b4001_desc
+	},
+	{
+		.compatible = "kingdisplay,kd101ne3-40ti",
+		.data = &kingdisplay_kd101ne3_40ti_desc
+	},
+	{
+		.compatible = "melfas,lmfbx101117480",
+		.data = &melfas_lmfbx101117480_desc
+	},
+	{
+		.compatible = "radxa,display-10hd-ad001",
+		.data = &cz101b4001_desc
+	},
+	{
+		.compatible = "radxa,display-8hd-ad002",
+		.data = &radxa_display_8hd_ad002_desc
 	},
 	{
 		.compatible = "taiguanck,xti05101-01a",
 		.data = &taiguan_xti05101_01a_desc
 	},
+	{
+		.compatible = "waveshare,3.4-dsi-touch-c",
+		.data = &waveshare_3_4_inch_c_desc
+	},
+	{
+		.compatible = "waveshare,4.0-dsi-touch-c",
+		.data = &waveshare_4_0_inch_c_desc
+	},
+	{
+		.compatible = "waveshare,8.0-dsi-touch-a",
+		.data = &waveshare_8_0_inch_a_desc
+	},
+	{
+		.compatible = "waveshare,9.0-dsi-touch-b",
+		.data = &waveshare_9_0_inch_b_desc
+	},
+	{
+		.compatible = "waveshare,10.1-dsi-touch-a",
+		.data = &waveshare_10_1_inch_a_desc
+	},
+	{
+		.compatible = "waveshare,10.1-dsi-touch-b",
+		.data = &waveshare_10_1_inch_b_desc
+	},
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, jadard_of_match);
diff --git a/drivers/gpu/drm/panel/panel-lxd-m9189a.c b/drivers/gpu/drm/panel/panel-lxd-m9189a.c
index 68019e1e43a96..baaf170779e03 100644
--- a/drivers/gpu/drm/panel/panel-lxd-m9189a.c
+++ b/drivers/gpu/drm/panel/panel-lxd-m9189a.c
@@ -165,9 +165,12 @@ static int lxd_m9189_probe(struct mipi_dsi_device *dsi)
 	struct m9189_panel *m9189;
 	int ret;
 
-	m9189 = devm_kzalloc(dev, sizeof(*m9189), GFP_KERNEL);
-	if (!m9189)
-		return -ENOMEM;
+	m9189 = devm_drm_panel_alloc(dev, __typeof(*m9189), panel,
+				    &m9189_panel_funcs,
+				    DRM_MODE_CONNECTOR_DSI);
+
+	if (IS_ERR(m9189))
+		return PTR_ERR(m9189);
 
 	m9189->supply = devm_regulator_get(dev, "power");
 	if (IS_ERR(m9189->supply))
@@ -191,8 +194,6 @@ static int lxd_m9189_probe(struct mipi_dsi_device *dsi)
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST;
 
-	drm_panel_init(&m9189->panel, dev, &m9189_panel_funcs,
-		       DRM_MODE_CONNECTOR_DSI);
 	m9189->panel.prepare_prev_first = true;
 
 	ret = drm_panel_of_backlight(&m9189->panel);
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt35532.c b/drivers/gpu/drm/panel/panel-novatek-nt35532.c
new file mode 100644
index 0000000000000..184f61bca7ca4
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-novatek-nt35532.c
@@ -0,0 +1,796 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Generated with linux-mdss-dsi-panel-driver-generator from vendor device tree.
+ * Copyright (c) 2026 Cristian Cozzolino <cristian_ci@protonmail.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/gpio/consumer.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/property.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_modes.h>
+#include <drm/drm_panel.h>
+#include <drm/drm_probe_helper.h>
+
+struct novatek_nt35532 {
+	struct drm_panel panel;
+	struct mipi_dsi_device *dsi;
+	struct regulator_bulk_data *supplies;
+	struct gpio_desc *reset_gpio;
+	const struct nt35532_panel_desc *desc;
+};
+
+struct nt35532_panel_desc {
+	const struct drm_display_mode *mode;
+	int (*on)(struct novatek_nt35532 *ctx);
+};
+
+static const struct regulator_bulk_data nt35532_supplies[] = {
+	{ .supply = "vci" },
+	{ .supply = "vddi" },
+	{ .supply = "avee" },
+	{ .supply = "avdd" },
+};
+
+static inline struct novatek_nt35532 *to_novatek_nt35532(struct drm_panel *panel)
+{
+	return container_of_const(panel, struct novatek_nt35532, panel);
+}
+
+static void nt35532_reset(struct novatek_nt35532 *ctx)
+{
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(10000, 11000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	usleep_range(5000, 6000);
+	gpiod_set_value_cansleep(ctx->reset_gpio, 0);
+	usleep_range(10000, 11000);
+}
+
+static int rimob_panel_on(struct novatek_nt35532 *ctx)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xff, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6e, 0x80);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x68, 0x13);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xff, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xff, 0x05);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd7, 0x31);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd8, 0x7e);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xff, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x01, 0x55);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x04, 0x0c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x05, 0x3a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x06, 0x50);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x07, 0xd0);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0a, 0x0f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0c, 0x06);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0d, 0x6b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0e, 0x6b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0f, 0x70);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x10, 0x63);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x11, 0x3c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x12, 0x5c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x15, 0x60);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x16, 0x15);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x17, 0x15);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5b, 0xca);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5c, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5d, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5f, 0x1b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x60, 0xd5);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x61, 0xf0);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6c, 0xab);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6d, 0x44);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6e, 0x80);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xff, 0x05);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x00, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x01, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x02, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x03, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x04, 0x38);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x05, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x06, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x07, 0x19);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x08, 0x1b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x09, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0a, 0x1d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0b, 0x17);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0c, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0d, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0e, 0x08);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0f, 0x0c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x10, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x11, 0x10);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x12, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x13, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x14, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x15, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x16, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x17, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x18, 0x38);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x19, 0x18);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1a, 0x1a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1b, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1c, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1d, 0x1c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1e, 0x16);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1f, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x20, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x21, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x22, 0x06);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x23, 0x0a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x24, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x25, 0x0e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x26, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x27, 0x3f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x54, 0x08);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x55, 0x07);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x56, 0x1a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x58, 0x19);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x59, 0x36);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5a, 0x1b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5b, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5c, 0x32);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5e, 0x27);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5f, 0x28);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x60, 0x2b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x61, 0x2c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x62, 0x18);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x63, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x64, 0x32);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x65, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x66, 0x44);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x67, 0x11);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x68, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x69, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6a, 0x06);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6b, 0x22);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6c, 0x08);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6d, 0x08);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x78, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x79, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7e, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7f, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x80, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x81, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8d, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8e, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8f, 0xc0);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x90, 0x73);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x91, 0x10);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x92, 0x09);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x96, 0x11);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x97, 0x14);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x98, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x99, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9a, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9b, 0x61);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9c, 0x15);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9d, 0x30);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9f, 0x0f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa2, 0xb0);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa7, 0x0a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa9, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xaa, 0x70);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xab, 0xda);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xac, 0xff);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xae, 0xf4);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xaf, 0x40);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb0, 0x7f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb1, 0x16);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb2, 0x53);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb3, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb4, 0x2a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb5, 0x3a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb6, 0xf0);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbc, 0x85);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbd, 0xf8);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbe, 0x3b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbf, 0x13);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc0, 0x77);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc1, 0x77);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc2, 0x77);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc3, 0x77);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc4, 0x77);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc5, 0x77);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc6, 0x77);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc7, 0x77);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc8, 0xaa);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc9, 0x2a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xca, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcb, 0xaa);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcc, 0x92);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcd, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xce, 0x18);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcf, 0x88);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd0, 0xaa);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd1, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd2, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd3, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd6, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xed, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xee, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xef, 0x70);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfa, 0x03);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xff, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x75, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x76, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x77, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x78, 0x20);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x79, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7a, 0x49);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7b, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7c, 0x66);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7d, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7e, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7f, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x80, 0x91);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x81, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x82, 0xa3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x83, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x84, 0xb3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x85, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x86, 0xc1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x87, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x88, 0xf1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x89, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8a, 0x18);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8b, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8c, 0x54);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8d, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8e, 0x85);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8f, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x90, 0xd1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x91, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x92, 0x0c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x93, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x94, 0x0e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x95, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x96, 0x43);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x97, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x98, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x99, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9a, 0xa2);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9b, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9c, 0xd7);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9d, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9e, 0xf8);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9f, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa0, 0x2b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa2, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa3, 0x3a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa4, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa5, 0x4b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa6, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa7, 0x5d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa9, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xaa, 0x73);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xab, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xac, 0x8e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xad, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xae, 0xae);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xaf, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb0, 0xc9);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb1, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb2, 0xcd);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb3, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb4, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb5, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb6, 0x20);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb7, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb8, 0x49);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb9, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xba, 0x66);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbb, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbc, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbd, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbe, 0x91);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbf, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc0, 0xa3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc1, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc2, 0xb3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc3, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc4, 0xc1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc5, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc6, 0xf1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc7, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc8, 0x18);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc9, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xca, 0x54);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcb, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcc, 0x85);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcd, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xce, 0xd1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcf, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd0, 0x0c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd1, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd2, 0x0e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd3, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd4, 0x43);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd5, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd6, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd7, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd8, 0xa2);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd9, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xda, 0xd7);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xdb, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xdc, 0xf8);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xdd, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xde, 0x2b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xdf, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe0, 0x3a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe1, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe2, 0x4b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe3, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe4, 0x5d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe5, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe6, 0x73);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe7, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe8, 0x8e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe9, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xea, 0xae);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xeb, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xec, 0xc9);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xed, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xee, 0xcd);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xef, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf0, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf1, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf2, 0x20);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf3, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf4, 0x49);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf5, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf6, 0x66);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf7, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf8, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xf9, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfa, 0x91);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xff, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x00, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x01, 0xa3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x02, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x03, 0xb3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x04, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x05, 0xc1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x06, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x07, 0xf1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x08, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x09, 0x18);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0a, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0b, 0x54);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0c, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0d, 0x85);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0e, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x0f, 0xd1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x10, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x11, 0x0c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x12, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x13, 0x0e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x14, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x15, 0x43);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x16, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x17, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x18, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x19, 0xa2);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1a, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1b, 0xd7);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1c, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1d, 0xf8);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1e, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x1f, 0x2b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x20, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x21, 0x3a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x22, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x23, 0x4b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x24, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x25, 0x5d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x26, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x27, 0x73);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x28, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x29, 0x8e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x2a, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x2b, 0xae);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x2d, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x2f, 0xc9);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x30, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x31, 0xcd);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x32, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x33, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x34, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x35, 0x20);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x36, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x37, 0x49);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x38, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x39, 0x66);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x3a, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x3b, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x3d, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x3f, 0x91);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x40, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x41, 0xa3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x42, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x43, 0xb3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x44, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x45, 0xc1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x46, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x47, 0xf1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x48, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x49, 0x18);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x4a, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x4b, 0x54);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x4c, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x4d, 0x85);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x4e, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x4f, 0xd1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x50, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x51, 0x0c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x52, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x53, 0x0e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x54, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x55, 0x43);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x56, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x58, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x59, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5a, 0xa2);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5b, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5c, 0xd7);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5d, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5e, 0xf8);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x5f, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x60, 0x2b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x61, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x62, 0x3a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x63, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x64, 0x4b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x65, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x66, 0x5d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x67, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x68, 0x73);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x69, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6a, 0x8e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6b, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6c, 0xae);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6d, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6e, 0xc9);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x6f, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x70, 0xcd);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x71, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x72, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x73, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x74, 0x20);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x75, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x76, 0x49);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x77, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x78, 0x66);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x79, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7a, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7b, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7c, 0x91);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7d, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7e, 0xa3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x7f, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x80, 0xb3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x81, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x82, 0xc1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x83, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x84, 0xf1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x85, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x86, 0x18);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x87, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x88, 0x54);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x89, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8a, 0x85);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8b, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8c, 0xd1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8d, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8e, 0x0c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x8f, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x90, 0x0e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x91, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x92, 0x43);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x93, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x94, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x95, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x96, 0xa2);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x97, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x98, 0xd7);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x99, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9a, 0xf8);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9b, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9c, 0x2b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9d, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9e, 0x3a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x9f, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa0, 0x4b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa2, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa3, 0x5d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa4, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa5, 0x73);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa6, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa7, 0x8e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xa9, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xaa, 0xae);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xab, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xac, 0xc9);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xad, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xae, 0xcd);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xaf, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb0, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb1, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb2, 0x20);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb3, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb4, 0x49);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb5, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb6, 0x66);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb7, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb8, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb9, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xba, 0x91);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbb, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbc, 0xa3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbd, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbe, 0xb3);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xbf, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc0, 0xc1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc1, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc2, 0xf1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc3, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc4, 0x18);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc5, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc6, 0x54);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc7, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc8, 0x85);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xc9, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xca, 0xd1);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcb, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcc, 0x0c);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcd, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xce, 0x0e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xcf, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd0, 0x43);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd1, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd2, 0x7d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd3, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd4, 0xa2);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd5, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd6, 0xd7);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd7, 0x02);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd8, 0xf8);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd9, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xda, 0x2b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xdb, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xdc, 0x3a);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xdd, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xde, 0x4b);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xdf, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe0, 0x5d);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe1, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe2, 0x73);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe3, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe4, 0x8e);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe5, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe6, 0xae);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe7, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe8, 0xc9);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xe9, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xea, 0xcd);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xff, 0xee);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x02, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x40, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x02, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x41, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x02, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x42, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xff, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xfb, 0x01);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xba, 0x03);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x35, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0x36, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xb0, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd3, 0x10);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd4, 0x0f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd5, 0x0f);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd6, 0x48);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd7, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, 0xd9, 0x00);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, MIPI_DCS_EXIT_SLEEP_MODE, 0x00);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+	mipi_dsi_generic_write_seq_multi(&dsi_ctx, MIPI_DCS_SET_DISPLAY_ON, 0x00);
+	mipi_dsi_msleep(&dsi_ctx, 50);
+
+	return dsi_ctx.accum_err;
+}
+
+static int nt35532_off(struct novatek_nt35532 *ctx)
+{
+	struct mipi_dsi_multi_context dsi_ctx = { .dsi = ctx->dsi };
+
+	mipi_dsi_dcs_set_display_off_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 50);
+	mipi_dsi_dcs_enter_sleep_mode_multi(&dsi_ctx);
+	mipi_dsi_msleep(&dsi_ctx, 120);
+
+	return dsi_ctx.accum_err;
+}
+
+static int nt35532_prepare(struct drm_panel *panel)
+{
+	struct novatek_nt35532 *ctx = to_novatek_nt35532(panel);
+	struct device *dev = &ctx->dsi->dev;
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(nt35532_supplies), ctx->supplies);
+	if (ret < 0) {
+		dev_err(dev, "Failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	nt35532_reset(ctx);
+
+	ret = ctx->desc->on(ctx);
+	if (ret < 0) {
+		dev_err(dev, "Failed to initialize panel: %d\n", ret);
+		gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+		regulator_bulk_disable(ARRAY_SIZE(nt35532_supplies), ctx->supplies);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int nt35532_unprepare(struct drm_panel *panel)
+{
+	struct novatek_nt35532 *ctx = to_novatek_nt35532(panel);
+	struct device *dev = &ctx->dsi->dev;
+	int ret;
+
+	ret = nt35532_off(ctx);
+	if (ret < 0)
+		dev_err(dev, "Failed to un-initialize panel: %d\n", ret);
+
+	gpiod_set_value_cansleep(ctx->reset_gpio, 1);
+	regulator_bulk_disable(ARRAY_SIZE(nt35532_supplies), ctx->supplies);
+
+	return 0;
+}
+
+static const struct drm_display_mode rimob_panel_mode = {
+	.clock = (1080 + 100 + 6 + 94) * (1920 + 15 + 6 + 10) * 60 / 1000,
+	.hdisplay = 1080,
+	.hsync_start = 1080 + 100,
+	.hsync_end = 1080 + 100 + 6,
+	.htotal = 1080 + 100 + 6 + 94,
+	.vdisplay = 1920,
+	.vsync_start = 1920 + 15,
+	.vsync_end = 1920 + 15 + 6,
+	.vtotal = 1920 + 15 + 6 + 10,
+	.width_mm = 68,
+	.height_mm = 121,
+	.type = DRM_MODE_TYPE_DRIVER,
+};
+
+static int nt35532_get_modes(struct drm_panel *panel,
+				struct drm_connector *connector)
+{
+	struct novatek_nt35532 *ctx = to_novatek_nt35532(panel);
+	const struct drm_display_mode *mode = ctx->desc->mode;
+
+	return drm_connector_helper_get_modes_fixed(connector, mode);
+}
+
+static const struct nt35532_panel_desc rimob_panel_desc = {
+	.mode = &rimob_panel_mode,
+	.on = rimob_panel_on,
+};
+
+static const struct drm_panel_funcs novatek_nt35532_panel_funcs = {
+	.prepare = nt35532_prepare,
+	.unprepare = nt35532_unprepare,
+	.get_modes = nt35532_get_modes,
+};
+
+static int nt35532_probe(struct mipi_dsi_device *dsi)
+{
+	struct device *dev = &dsi->dev;
+	struct novatek_nt35532 *ctx;
+	int ret;
+
+	ctx = devm_drm_panel_alloc(dev, struct novatek_nt35532, panel,
+				   &novatek_nt35532_panel_funcs,
+				   DRM_MODE_CONNECTOR_DSI);
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
+
+	ctx->desc = device_get_match_data(dev);
+
+	ret = devm_regulator_bulk_get_const(dev,
+					    ARRAY_SIZE(nt35532_supplies),
+					    nt35532_supplies,
+					    &ctx->supplies);
+	if (ret < 0)
+		return ret;
+
+	ctx->reset_gpio = devm_gpiod_get(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(ctx->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(ctx->reset_gpio),
+				     "Failed to get reset-gpios\n");
+
+	ctx->dsi = dsi;
+	mipi_dsi_set_drvdata(dsi, ctx);
+
+	dsi->lanes = 4;
+	dsi->format = MIPI_DSI_FMT_RGB888;
+	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST |
+			  MIPI_DSI_MODE_VIDEO_HSE | MIPI_DSI_MODE_LPM;
+
+	ctx->panel.prepare_prev_first = true;
+
+	ret = drm_panel_of_backlight(&ctx->panel);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get backlight\n");
+
+	drm_panel_add(&ctx->panel);
+
+	ret = mipi_dsi_attach(dsi);
+	if (ret < 0) {
+		drm_panel_remove(&ctx->panel);
+		return dev_err_probe(dev, ret, "Failed to attach to DSI host\n");
+	}
+
+	return 0;
+}
+
+static void nt35532_remove(struct mipi_dsi_device *dsi)
+{
+	struct novatek_nt35532 *ctx = mipi_dsi_get_drvdata(dsi);
+	int ret;
+
+	ret = mipi_dsi_detach(dsi);
+	if (ret < 0)
+		dev_err(&dsi->dev, "Failed to detach from DSI host: %d\n", ret);
+
+	drm_panel_remove(&ctx->panel);
+}
+
+static const struct of_device_id nt35532_of_match[] = {
+	{ .compatible = "flipkart,rimob-panel-nt35532-cs", .data = &rimob_panel_desc },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, nt35532_of_match);
+
+static struct mipi_dsi_driver nt35532_driver = {
+	.probe = nt35532_probe,
+	.remove = nt35532_remove,
+	.driver = {
+		.name = "panel-novatek-nt35532",
+		.of_match_table = nt35532_of_match,
+	},
+};
+module_mipi_dsi_driver(nt35532_driver);
+
+MODULE_DESCRIPTION("DRM driver for Novatek NT35532-based 1080p video mode DSI panel");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/panel/panel-novatek-nt37700f.c b/drivers/gpu/drm/panel/panel-novatek-nt37700f.c
index 74f46a268c0f6..c221dd498fed4 100644
--- a/drivers/gpu/drm/panel/panel-novatek-nt37700f.c
+++ b/drivers/gpu/drm/panel/panel-novatek-nt37700f.c
@@ -225,9 +225,12 @@ static int nt37700f_tianma_probe(struct mipi_dsi_device *dsi)
 	struct nt37700f_tianma *ctx;
 	int ret;
 
-	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
+	ctx = devm_drm_panel_alloc(dev, __typeof(*ctx), panel,
+				  &nt37700f_tianma_panel_funcs,
+				  DRM_MODE_CONNECTOR_DSI);
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
 
 	ctx->supply = devm_regulator_get(dev, "power");
 	if (IS_ERR(ctx->supply))
@@ -247,8 +250,6 @@ static int nt37700f_tianma_probe(struct mipi_dsi_device *dsi)
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO_BURST |
 			  MIPI_DSI_CLOCK_NON_CONTINUOUS | MIPI_DSI_MODE_LPM;
 
-	drm_panel_init(&ctx->panel, dev, &nt37700f_tianma_panel_funcs,
-		       DRM_MODE_CONNECTOR_DSI);
 	ctx->panel.prepare_prev_first = true;
 
 	ctx->panel.backlight = nt37700f_tianma_create_backlight(dsi);
diff --git a/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c b/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
index 3c3308fc55df0..d21d93a0700ee 100644
--- a/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
+++ b/drivers/gpu/drm/panel/panel-panasonic-vvx10f034n00.c
@@ -44,14 +44,23 @@ static inline struct wuxga_nt_panel *to_wuxga_nt_panel(struct drm_panel *panel)
 
 static int wuxga_nt_panel_on(struct wuxga_nt_panel *wuxga_nt)
 {
-	return mipi_dsi_turn_on_peripheral(wuxga_nt->dsi);
+	struct mipi_dsi_multi_context dsi_ctx = {
+		.dsi = wuxga_nt->dsi
+	};
+
+	mipi_dsi_turn_on_peripheral_multi(&dsi_ctx);
+	return dsi_ctx.accum_err;
 }
 
 static int wuxga_nt_panel_disable(struct drm_panel *panel)
 {
 	struct wuxga_nt_panel *wuxga_nt = to_wuxga_nt_panel(panel);
+	struct mipi_dsi_multi_context dsi_ctx = {
+		.dsi = wuxga_nt->dsi
+	};
 
-	return mipi_dsi_shutdown_peripheral(wuxga_nt->dsi);
+	mipi_dsi_shutdown_peripheral_multi(&dsi_ctx);
+	return dsi_ctx.accum_err;
 }
 
 static int wuxga_nt_panel_unprepare(struct drm_panel *panel)
@@ -94,15 +103,8 @@ static int wuxga_nt_panel_prepare(struct drm_panel *panel)
 	msleep(250);
 
 	ret = wuxga_nt_panel_on(wuxga_nt);
-	if (ret < 0) {
-		dev_err(panel->dev, "failed to set panel on: %d\n", ret);
-		goto poweroff;
-	}
-
-	return 0;
-
-poweroff:
-	regulator_disable(wuxga_nt->supply);
+	if (ret < 0)
+		regulator_disable(wuxga_nt->supply);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/panel/panel-samsung-s6e63m0.c b/drivers/gpu/drm/panel/panel-samsung-s6e63m0.c
index ea241c89593b6..997115b1e541d 100644
--- a/drivers/gpu/drm/panel/panel-samsung-s6e63m0.c
+++ b/drivers/gpu/drm/panel/panel-samsung-s6e63m0.c
@@ -677,9 +677,13 @@ int s6e63m0_probe(struct device *dev, void *trsp,
 	u32 max_brightness;
 	int ret;
 
-	ctx = devm_kzalloc(dev, sizeof(struct s6e63m0), GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
+	ctx = devm_drm_panel_alloc(dev, __typeof(*ctx), panel,
+				  &s6e63m0_drm_funcs,
+				  dsi_mode ? DRM_MODE_CONNECTOR_DSI :
+				  DRM_MODE_CONNECTOR_DPI);
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
 
 	ctx->transport_data = trsp;
 	ctx->dsi_mode = dsi_mode;
@@ -712,10 +716,6 @@ int s6e63m0_probe(struct device *dev, void *trsp,
 		return PTR_ERR(ctx->reset_gpio);
 	}
 
-	drm_panel_init(&ctx->panel, dev, &s6e63m0_drm_funcs,
-		       dsi_mode ? DRM_MODE_CONNECTOR_DSI :
-		       DRM_MODE_CONNECTOR_DPI);
-
 	ret = s6e63m0_backlight_register(ctx, max_brightness);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
index dd1eaba23ad3c..989b030ea22de 100644
--- a/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
+++ b/drivers/gpu/drm/panel/panel-sharp-ls043t1le01.c
@@ -199,8 +199,6 @@ static int sharp_nt_panel_add(struct sharp_nt_panel *sharp_nt)
 		gpiod_set_value(sharp_nt->reset_gpio, 0);
 	}
 
-	drm_panel_init(&sharp_nt->base, &sharp_nt->dsi->dev,
-		       &sharp_nt_panel_funcs, DRM_MODE_CONNECTOR_DSI);
 	sharp_nt->base.prepare_prev_first = true;
 
 	ret = drm_panel_of_backlight(&sharp_nt->base);
@@ -231,9 +229,12 @@ static int sharp_nt_panel_probe(struct mipi_dsi_device *dsi)
 			MIPI_DSI_CLOCK_NON_CONTINUOUS |
 			MIPI_DSI_MODE_NO_EOT_PACKET;
 
-	sharp_nt = devm_kzalloc(&dsi->dev, sizeof(*sharp_nt), GFP_KERNEL);
-	if (!sharp_nt)
-		return -ENOMEM;
+	sharp_nt = devm_drm_panel_alloc(&dsi->dev, __typeof(*sharp_nt), base,
+					&sharp_nt_panel_funcs,
+					DRM_MODE_CONNECTOR_DSI);
+
+	if (IS_ERR(sharp_nt))
+		return PTR_ERR(sharp_nt);
 
 	mipi_dsi_set_drvdata(dsi, sharp_nt);
 
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 236bd56208ccf..2fbb9dff0a34d 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -1902,6 +1902,39 @@ static const struct panel_desc dataimage_scf0700c48ggu18 = {
 	.connector_type = DRM_MODE_CONNECTOR_DPI,
 };
 
+static const struct display_timing displaytech_dt050btft_pts_timing = {
+	/* The TYP pixel clock are recalculated from tV * tH * 60 Hz */
+	.pixelclock = { 30000000, 33264000, 50000000 },
+	.hactive = { 800, 800, 800 },
+	.hfront_porch = { 16, 210, 354 },
+	/* Datasheet Figure 3 indicates, that tHPW is part of tHBP */
+	.hback_porch = { 41, 26, 6 },
+	.hsync_len = { 1, 20, 40 },
+	.vactive = { 480, 480, 480 },
+	.vfront_porch = { 7, 22, 147 },
+	/* Datasheet Figure 2 indicates, that tVPW is part of tVBP */
+	.vback_porch = { 22, 13, 3 },
+	.vsync_len = { 1, 10, 20 },
+	.flags = DISPLAY_FLAGS_HSYNC_LOW | DISPLAY_FLAGS_VSYNC_LOW |
+		 DISPLAY_FLAGS_DE_HIGH | DISPLAY_FLAGS_PIXDATA_POSEDGE |
+		 DISPLAY_FLAGS_SYNC_POSEDGE,
+};
+
+static const struct panel_desc displaytech_dt050btft_pts = {
+	.timings = &displaytech_dt050btft_pts_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 108,
+		.height = 65,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH |
+		     DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE |
+		     DRM_BUS_FLAG_SYNC_SAMPLE_NEGEDGE,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+};
+
 static const struct display_timing dlc_dlc0700yzg_1_timing = {
 	.pixelclock = { 45000000, 51200000, 57000000 },
 	.hactive = { 1024, 1024, 1024 },
@@ -4603,6 +4636,32 @@ static const struct panel_desc starry_kr070pe2t = {
 	.connector_type = DRM_MODE_CONNECTOR_DPI,
 };
 
+static const struct display_timing startek_kd070hdfld092_timing = {
+	.pixelclock = { 40800000, 51200000, 67200000 },
+	.hactive = { 1024, 1024, 1024 },
+	.hfront_porch = { 40, 160, 216 },
+	.hback_porch = { 30, 140, 140 },
+	.hsync_len = { 20, 20, 20 },
+	.vactive = { 600, 600, 600 },
+	.vfront_porch = { 2, 12, 177 },
+	.vback_porch = { 5, 20, 20 },
+	.vsync_len = { 3, 3, 3 },
+	.flags = DISPLAY_FLAGS_DE_HIGH,
+};
+
+static const struct panel_desc startek_kd070hdfld092 = {
+	.timings = &startek_kd070hdfld092_timing,
+	.num_timings = 1,
+	.bpc = 8,
+	.size = {
+		.width = 154,
+		.height = 86,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.bus_flags = DRM_BUS_FLAG_DE_HIGH,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
 static const struct display_timing startek_kd070wvfpa_mode = {
 	.pixelclock = { 25200000, 27200000, 30500000 },
 	.hactive = { 800, 800, 800 },
@@ -5061,6 +5120,342 @@ static const struct panel_desc vl050_8048nt_c01 = {
 	.bus_flags = DRM_BUS_FLAG_DE_HIGH | DRM_BUS_FLAG_PIXDATA_SAMPLE_NEGEDGE,
 };
 
+static const struct drm_display_mode waveshare_28_lcd_mode = {
+	.clock = 50000,
+	.hdisplay = 480,
+	.hsync_start = 480 + 150,
+	.hsync_end = 480 + 150 + 50,
+	.htotal = 480 + 150 + 50 + 150,
+	.vdisplay = 640,
+	.vsync_start = 640 + 150,
+	.vsync_end = 640 + 150 + 50,
+	.vtotal = 640 + 150 + 50 + 150,
+	.flags = DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_PHSYNC,
+};
+
+static const struct panel_desc waveshare_28_lcd_panel = {
+	.modes = &waveshare_28_lcd_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 44,
+		.height = 58,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE |
+		     DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE,
+};
+
+static const struct drm_display_mode waveshare_34_lcd_c_mode = {
+	.clock = 50000,
+	.hdisplay = 800,
+	.hsync_start = 800 + 32,
+	.hsync_end = 800 + 32 + 6,
+	.htotal = 800 + 32 + 6 + 120,
+	.vdisplay = 800,
+	.vsync_start = 800 + 8,
+	.vsync_end = 800 + 8 + 4,
+	.vtotal = 800 + 8 + 4 + 16,
+};
+
+static const struct panel_desc waveshare_34_lcd_c_panel = {
+	.modes = &waveshare_34_lcd_c_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 88,
+		.height = 88,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct drm_display_mode waveshare_40_lcd_mode = {
+	.clock = 50000,
+	.hdisplay = 480,
+	.hsync_start = 480 + 150,
+	.hsync_end = 480 + 150 + 100,
+	.htotal = 480 + 150 + 100 + 150,
+	.vdisplay = 800,
+	.vsync_start = 800 + 20,
+	.vsync_end = 800 + 20 + 100,
+	.vtotal = 800 + 20 + 100 + 20,
+	.flags = DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_PHSYNC,
+};
+
+static const struct panel_desc waveshare_40_lcd_panel = {
+	.modes = &waveshare_40_lcd_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 52,
+		.height = 87,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE |
+		     DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE,
+};
+
+static const struct drm_display_mode waveshare_40_lcd_c_mode = {
+	.clock = 50000,
+	.hdisplay = 720,
+	.hsync_start = 720 + 32,
+	.hsync_end = 720 + 32 + 200,
+	.htotal = 720 + 32 + 200 + 120,
+	.vdisplay = 720,
+	.vsync_start = 720 + 8,
+	.vsync_end = 720 + 8 + 4,
+	.vtotal = 720 + 8 + 4 + 16,
+};
+
+static const struct panel_desc waveshare_40_lcd_c_panel = {
+	.modes = &waveshare_40_lcd_c_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 102,
+		.height = 102,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct drm_display_mode waveshare_50_lcd_c_mode = {
+	.clock = 50000,
+	.hdisplay = 1024,
+	.hsync_start = 1024 + 100,
+	.hsync_end = 1024 + 100 + 100,
+	.htotal = 1024 + 100 + 100 + 100,
+	.vdisplay = 600,
+	.vsync_start = 600 + 10,
+	.vsync_end = 600 + 10 + 10,
+	.vtotal = 600 + 10 + 10 + 10,
+	.flags = DRM_MODE_FLAG_PVSYNC | DRM_MODE_FLAG_PHSYNC,
+};
+
+static const struct panel_desc waveshare_50_lcd_c_panel = {
+	.modes = &waveshare_50_lcd_c_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 109,
+		.height = 66,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE |
+		     DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE,
+};
+
+static const struct drm_display_mode waveshare_50_lcd_d_mode = {
+	.clock = 83333,
+	.hdisplay = 720,
+	.hsync_start = 720 + 100,
+	.hsync_end = 720 + 100 + 80,
+	.htotal = 720 + 100 + 80 + 100,
+	.vdisplay = 1280,
+	.vsync_start = 1280 + 20,
+	.vsync_end = 1280 + 20 + 20,
+	.vtotal = 1280 + 20 + 20 + 20,
+};
+
+static const struct panel_desc waveshare_50_lcd_d_panel = {
+	.modes = &waveshare_50_lcd_d_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 62,
+		.height = 110,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct drm_display_mode waveshare_625_lcd_mode = {
+	.clock = 83333,
+	.hdisplay = 720,
+	.hsync_start = 720 + 50,
+	.hsync_end = 720 + 50 + 50,
+	.htotal = 720 + 50 + 50 + 50,
+	.vdisplay = 1560,
+	.vsync_start = 1560 + 20,
+	.vsync_end = 1560 + 20 + 20,
+	.vtotal = 1560 + 20 + 20 + 20,
+};
+
+static const struct panel_desc waveshare_625_lcd_panel = {
+	.modes = &waveshare_625_lcd_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 66,
+		.height = 144,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct panel_desc waveshare_70_lcd_c_panel = {
+	.modes = &waveshare_50_lcd_c_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 155,
+		.height = 87,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X24,
+	.connector_type = DRM_MODE_CONNECTOR_DPI,
+	.bus_flags = DRM_BUS_FLAG_PIXDATA_SAMPLE_POSEDGE |
+		     DRM_BUS_FLAG_SYNC_SAMPLE_POSEDGE,
+};
+
+static const struct drm_display_mode waveshare_80_lcd_c_mode;
+static const struct panel_desc waveshare_70_lcd_e_panel = {
+	.modes = &waveshare_80_lcd_c_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 152,
+		.height = 95,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct drm_display_mode waveshare_70_lcd_h_mode = {
+	.clock = 83333,
+	.hdisplay = 1280,
+	.hsync_start = 1280 + 64,
+	.hsync_end = 1280 + 64 + 64,
+	.htotal = 1280 + 64 + 64 + 64,
+	.vdisplay = 720,
+	.vsync_start = 720 + 64,
+	.vsync_end = 720 + 64 + 64,
+	.vtotal = 720 + 64 + 64 + 64,
+};
+
+static const struct panel_desc waveshare_70_lcd_h_panel = {
+	.modes = &waveshare_70_lcd_h_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 155,
+		.height = 88,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct drm_display_mode waveshare_79_lcd_mode = {
+	.clock = 50000,
+	.hdisplay = 400,
+	.hsync_start = 400 + 40,
+	.hsync_end = 400 + 40 + 30,
+	.htotal = 400 + 40 + 30 + 40,
+	.vdisplay = 1280,
+	.vsync_start = 1280 + 20,
+	.vsync_end = 1280 + 20 + 10,
+	.vtotal = 1280 + 20 + 10 + 20,
+};
+
+static const struct panel_desc waveshare_79_lcd_panel = {
+	.modes = &waveshare_79_lcd_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 60,
+		.height = 191,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct drm_display_mode waveshare_80_lcd_c_mode = {
+	.clock = 83333,
+	.hdisplay = 1280,
+	.hsync_start = 1280 + 156,
+	.hsync_end = 1280 + 156 + 20,
+	.htotal = 1280 + 156 + 20 + 40,
+	.vdisplay = 800,
+	.vsync_start = 800 + 40,
+	.vsync_end = 800 + 40 + 48,
+	.vtotal = 800 + 40 + 48 + 40,
+};
+
+static const struct panel_desc waveshare_80_lcd_c_panel = {
+	.modes = &waveshare_80_lcd_c_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 173,
+		.height = 108,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct drm_display_mode waveshare_88_lcd_mode = {
+	.clock = 83333,
+	.hdisplay = 480,
+	.hsync_start = 480 + 50,
+	.hsync_end = 480 + 50 + 50,
+	.htotal = 480 + 50 + 50 + 50,
+	.vdisplay = 1920,
+	.vsync_start = 1920 + 20,
+	.vsync_end = 1920 + 20 + 20,
+	.vtotal = 1920 + 20 + 20 + 20,
+};
+
+static const struct panel_desc waveshare_88_lcd_panel = {
+	.modes = &waveshare_88_lcd_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 56,
+		.height = 220,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct panel_desc waveshare_101_lcd_c_panel = {
+	.modes = &waveshare_80_lcd_c_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 217,
+		.height = 136,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
+static const struct drm_display_mode waveshare_119_lcd_mode = {
+	.clock = 50000,
+	.hdisplay = 320,
+	.hsync_start = 320 + 60,
+	.hsync_end = 320 + 60 + 60,
+	.htotal = 320 + 60 + 60 + 60,
+	.vdisplay = 1480,
+	.vsync_start = 1480 + 60,
+	.vsync_end = 1480 + 60 + 60,
+	.vtotal = 1480 + 60 + 60 + 60,
+};
+
+static const struct panel_desc waveshare_119_lcd_panel = {
+	.modes = &waveshare_119_lcd_mode,
+	.num_modes = 1,
+	.bpc = 8,
+	.size = {
+		.width = 58,
+		.height = 268,
+	},
+	.bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
+	.connector_type = DRM_MODE_CONNECTOR_LVDS,
+};
+
 static const struct drm_display_mode waveshare_133inch_mode = {
 	.clock = 148500,
 	.hdisplay = 1920,
@@ -5319,6 +5714,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "dataimage,scf0700c48ggu18",
 		.data = &dataimage_scf0700c48ggu18,
+	}, {
+		.compatible = "displaytech,dt050btft-pts",
+		.data = &displaytech_dt050btft_pts,
 	}, {
 		.compatible = "dlc,dlc0700yzg-1",
 		.data = &dlc_dlc0700yzg_1,
@@ -5628,6 +6026,9 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "starry,kr070pe2t",
 		.data = &starry_kr070pe2t,
+	}, {
+		.compatible = "startek,kd070hdfld092",
+		.data = &startek_kd070hdfld092,
 	}, {
 		.compatible = "startek,kd070wvfpa",
 		.data = &startek_kd070wvfpa,
@@ -5697,6 +6098,51 @@ static const struct of_device_id platform_of_match[] = {
 	}, {
 		.compatible = "vxt,vl050-8048nt-c01",
 		.data = &vl050_8048nt_c01,
+	}, {
+		.compatible = "waveshare,2.8inch-panel",
+		.data = &waveshare_28_lcd_panel
+	}, {
+		.compatible = "waveshare,3.4inch-c-panel",
+		.data = &waveshare_34_lcd_c_panel
+	}, {
+		.compatible = "waveshare,4.0inch-panel",
+		.data = &waveshare_40_lcd_panel
+	}, {
+		.compatible = "waveshare,4.0inch-c-panel",
+		.data = &waveshare_40_lcd_c_panel
+	}, {
+		.compatible = "waveshare,5.0inch-c-panel",
+		.data = &waveshare_50_lcd_c_panel
+	}, {
+		.compatible = "waveshare,5.0inch-d-panel",
+		.data = &waveshare_50_lcd_d_panel
+	}, {
+		.compatible = "waveshare,6.25inch-panel",
+		.data = &waveshare_625_lcd_panel
+	}, {
+		.compatible = "waveshare,7.0inch-c-panel",
+		.data = &waveshare_70_lcd_c_panel
+	}, {
+		.compatible = "waveshare,7.0inch-e-panel",
+		.data = &waveshare_70_lcd_e_panel
+	}, {
+		.compatible = "waveshare,7.0inch-h-panel",
+		.data = &waveshare_70_lcd_h_panel
+	}, {
+		.compatible = "waveshare,7.9inch-panel",
+		.data = &waveshare_79_lcd_panel
+	}, {
+		.compatible = "waveshare,8.0inch-c-panel",
+		.data = &waveshare_80_lcd_c_panel
+	}, {
+		.compatible = "waveshare,8.8inch-panel",
+		.data = &waveshare_88_lcd_panel
+	}, {
+		.compatible = "waveshare,10.1inch-c-panel",
+		.data = &waveshare_101_lcd_c_panel
+	}, {
+		.compatible = "waveshare,11.9inch-panel",
+		.data = &waveshare_119_lcd_panel
 	}, {
 		.compatible = "waveshare,13.3inch-panel",
 		.data = &waveshare_133inch,
@@ -5936,6 +6382,39 @@ static const struct panel_desc_dsi osd101t2045_53ts = {
 	.lanes = 4,
 };
 
+static const struct drm_display_mode tsd_tst070wsbe_196c_mode = {
+	.clock = 52477,
+	.hdisplay = 1024,
+	.hsync_start = 1024 + 160,
+	.hsync_end = 1024 + 160 + 12,
+	.htotal = 1024 + 160 + 160 + 12,
+	.vdisplay = 600,
+	.vsync_start = 600 + 12,
+	.vsync_end = 600 + 12 + 10,
+	.vtotal = 600 + 12 + 10 + 23,
+};
+
+static const struct panel_desc_dsi tsd_tst070wsbe_196c = {
+	.desc = {
+		.modes = &tsd_tst070wsbe_196c_mode,
+		.num_modes = 1,
+		.bpc = 8,
+		.size = {
+			.width = 190,
+			.height = 121,
+		},
+		.delay = {
+			.prepare = 20,
+		},
+		.connector_type = DRM_MODE_CONNECTOR_DSI,
+	},
+	.flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_LPM |
+		 MIPI_DSI_MODE_VIDEO_BURST |
+		 MIPI_DSI_MODE_VIDEO_SYNC_PULSE,
+	.format = MIPI_DSI_FMT_RGB888,
+	.lanes = 4,
+};
+
 static const struct of_device_id dsi_of_match[] = {
 	{
 		.compatible = "auo,b080uan01",
@@ -5955,6 +6434,9 @@ static const struct of_device_id dsi_of_match[] = {
 	}, {
 		.compatible = "osddisplays,osd101t2045-53ts",
 		.data = &osd101t2045_53ts
+	}, {
+		.compatible = "team-source-display,tst070wsbe-196c",
+		.data = &tsd_tst070wsbe_196c
 	}, {
 		/* sentinel */
 	}
diff --git a/drivers/gpu/drm/panel/panel-startek-kd070fhfid015.c b/drivers/gpu/drm/panel/panel-startek-kd070fhfid015.c
index c0c95355b7435..7f04b7b404e7d 100644
--- a/drivers/gpu/drm/panel/panel-startek-kd070fhfid015.c
+++ b/drivers/gpu/drm/panel/panel-startek-kd070fhfid015.c
@@ -286,9 +286,6 @@ static int stk_panel_add(struct stk_panel *stk)
 		return ret;
 	}
 
-	drm_panel_init(&stk->base, &stk->dsi->dev, &stk_panel_funcs,
-		       DRM_MODE_CONNECTOR_DSI);
-
 	drm_panel_add(&stk->base);
 
 	return 0;
@@ -303,9 +300,12 @@ static int stk_panel_probe(struct mipi_dsi_device *dsi)
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->mode_flags = (MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_LPM);
 
-	stk = devm_kzalloc(&dsi->dev, sizeof(*stk), GFP_KERNEL);
-	if (!stk)
-		return -ENOMEM;
+	stk = devm_drm_panel_alloc(&dsi->dev, __typeof(*stk), base,
+				  &stk_panel_funcs,
+				  DRM_MODE_CONNECTOR_DSI);
+
+	if (IS_ERR(stk))
+		return PTR_ERR(stk);
 
 	mipi_dsi_set_drvdata(dsi, stk);
 
diff --git a/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c b/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c
index 227f97f9b136f..13cfe252a838d 100644
--- a/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c
+++ b/drivers/gpu/drm/panel/panel-tdo-tl070wsh30.c
@@ -162,9 +162,6 @@ static int tdo_tl070wsh30_panel_add(struct tdo_tl070wsh30_panel *tdo_tl070wsh30)
 		return err;
 	}
 
-	drm_panel_init(&tdo_tl070wsh30->base, &tdo_tl070wsh30->link->dev,
-		       &tdo_tl070wsh30_panel_funcs, DRM_MODE_CONNECTOR_DSI);
-
 	err = drm_panel_of_backlight(&tdo_tl070wsh30->base);
 	if (err)
 		return err;
@@ -183,10 +180,13 @@ static int tdo_tl070wsh30_panel_probe(struct mipi_dsi_device *dsi)
 	dsi->format = MIPI_DSI_FMT_RGB888;
 	dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST | MIPI_DSI_MODE_LPM;
 
-	tdo_tl070wsh30 = devm_kzalloc(&dsi->dev, sizeof(*tdo_tl070wsh30),
-				    GFP_KERNEL);
-	if (!tdo_tl070wsh30)
-		return -ENOMEM;
+	tdo_tl070wsh30 = devm_drm_panel_alloc(&dsi->dev,
+					      __typeof(*tdo_tl070wsh30), base,
+					      &tdo_tl070wsh30_panel_funcs,
+					      DRM_MODE_CONNECTOR_DSI);
+
+	if (IS_ERR(tdo_tl070wsh30))
+		return PTR_ERR(tdo_tl070wsh30);
 
 	mipi_dsi_set_drvdata(dsi, tdo_tl070wsh30);
 	tdo_tl070wsh30->link = dsi;
diff --git a/drivers/gpu/drm/panel/panel-truly-nt35597.c b/drivers/gpu/drm/panel/panel-truly-nt35597.c
index d447db912a61e..f740b9257c09c 100644
--- a/drivers/gpu/drm/panel/panel-truly-nt35597.c
+++ b/drivers/gpu/drm/panel/panel-truly-nt35597.c
@@ -476,8 +476,6 @@ static int truly_nt35597_panel_add(struct truly_nt35597 *ctx)
 	/* dual port */
 	gpiod_set_value(ctx->mode_gpio, 0);
 
-	drm_panel_init(&ctx->panel, dev, &truly_nt35597_drm_funcs,
-		       DRM_MODE_CONNECTOR_DSI);
 	drm_panel_add(&ctx->panel);
 
 	return 0;
@@ -523,10 +521,12 @@ static int truly_nt35597_probe(struct mipi_dsi_device *dsi)
 		.node = NULL,
 	};
 
-	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+	ctx = devm_drm_panel_alloc(dev, __typeof(*ctx), panel,
+				  &truly_nt35597_drm_funcs,
+				  DRM_MODE_CONNECTOR_DSI);
 
-	if (!ctx)
-		return -ENOMEM;
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
 
 	/*
 	 * This device represents itself as one with two input ports which are
diff --git a/drivers/gpu/drm/panel/panel-visionox-g2647fb105.c b/drivers/gpu/drm/panel/panel-visionox-g2647fb105.c
index 413849f7b4dec..d5555a1731575 100644
--- a/drivers/gpu/drm/panel/panel-visionox-g2647fb105.c
+++ b/drivers/gpu/drm/panel/panel-visionox-g2647fb105.c
@@ -207,9 +207,12 @@ static int visionox_g2647fb105_probe(struct mipi_dsi_device *dsi)
 	struct visionox_g2647fb105 *ctx;
 	int ret;
 
-	ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
-	if (!ctx)
-		return -ENOMEM;
+	ctx = devm_drm_panel_alloc(dev, __typeof(*ctx), panel,
+				  &visionox_g2647fb105_panel_funcs,
+				  DRM_MODE_CONNECTOR_DSI);
+
+	if (IS_ERR(ctx))
+		return PTR_ERR(ctx);
 
 	ret = devm_regulator_bulk_get_const(dev,
 					    ARRAY_SIZE(visionox_g2647fb105_supplies),
@@ -233,10 +236,6 @@ static int visionox_g2647fb105_probe(struct mipi_dsi_device *dsi)
 
 	ctx->panel.prepare_prev_first = true;
 
-	drm_panel_init(&ctx->panel, dev, &visionox_g2647fb105_panel_funcs,
-		       DRM_MODE_CONNECTOR_DSI);
-	ctx->panel.prepare_prev_first = true;
-
 	ctx->panel.backlight = visionox_g2647fb105_create_backlight(dsi);
 	if (IS_ERR(ctx->panel.backlight))
 		return dev_err_probe(dev, PTR_ERR(ctx->panel.backlight),
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c
index dedc13e56631e..7fed22d555a57 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.c
+++ b/drivers/gpu/drm/panfrost/panfrost_device.c
@@ -70,8 +70,21 @@ static int panfrost_clk_init(struct panfrost_device *pfdev)
 			goto disable_clock;
 	}
 
+	pfdev->bus_ace_clock = devm_clk_get_optional(pfdev->base.dev, "bus_ace");
+	if (IS_ERR(pfdev->bus_ace_clock)) {
+		err = PTR_ERR(pfdev->bus_ace_clock);
+		dev_err(pfdev->base.dev, "get bus_ace_clock failed %d\n", err);
+		goto disable_bus_clock;
+	}
+
+	err = clk_prepare_enable(pfdev->bus_ace_clock);
+	if (err)
+		goto disable_bus_clock;
+
 	return 0;
 
+disable_bus_clock:
+	clk_disable_unprepare(pfdev->bus_clock);
 disable_clock:
 	clk_disable_unprepare(pfdev->clock);
 
@@ -80,6 +93,7 @@ static int panfrost_clk_init(struct panfrost_device *pfdev)
 
 static void panfrost_clk_fini(struct panfrost_device *pfdev)
 {
+	clk_disable_unprepare(pfdev->bus_ace_clock);
 	clk_disable_unprepare(pfdev->bus_clock);
 	clk_disable_unprepare(pfdev->clock);
 }
@@ -429,11 +443,13 @@ static int panfrost_device_runtime_resume(struct device *dev)
 		if (ret)
 			goto err_clk;
 
-		if (pfdev->bus_clock) {
-			ret = clk_enable(pfdev->bus_clock);
-			if (ret)
-				goto err_bus_clk;
-		}
+		ret = clk_enable(pfdev->bus_clock);
+		if (ret)
+			goto err_bus_clk;
+
+		ret = clk_enable(pfdev->bus_ace_clock);
+		if (ret)
+			goto err_bus_ace_clk;
 	}
 
 	panfrost_device_reset(pfdev, true);
@@ -441,6 +457,9 @@ static int panfrost_device_runtime_resume(struct device *dev)
 
 	return 0;
 
+err_bus_ace_clk:
+	if (pfdev->comp->pm_features & BIT(GPU_PM_RT))
+		clk_disable(pfdev->bus_clock);
 err_bus_clk:
 	if (pfdev->comp->pm_features & BIT(GPU_PM_RT))
 		clk_disable(pfdev->clock);
@@ -464,9 +483,8 @@ static int panfrost_device_runtime_suspend(struct device *dev)
 	panfrost_gpu_power_off(pfdev);
 
 	if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) {
-		if (pfdev->bus_clock)
-			clk_disable(pfdev->bus_clock);
-
+		clk_disable(pfdev->bus_ace_clock);
+		clk_disable(pfdev->bus_clock);
 		clk_disable(pfdev->clock);
 		reset_control_assert(pfdev->rstc);
 	}
diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h
index 0f39924122052..ec55c136b1b66 100644
--- a/drivers/gpu/drm/panfrost/panfrost_device.h
+++ b/drivers/gpu/drm/panfrost/panfrost_device.h
@@ -136,6 +136,7 @@ struct panfrost_device {
 	void __iomem *iomem;
 	struct clk *clock;
 	struct clk *bus_clock;
+	struct clk *bus_ace_clock;
 	struct regulator_bulk_data *regulators;
 	struct reset_control *rstc;
 	/* pm_domains for devices with more than one. */
diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c
index 074c0995ddc26..784e36d72c2ba 100644
--- a/drivers/gpu/drm/panfrost/panfrost_drv.c
+++ b/drivers/gpu/drm/panfrost/panfrost_drv.c
@@ -1158,6 +1158,7 @@ static const struct of_device_id dt_match[] = {
 	  .data = &amlogic_data, },
 	{ .compatible = "amlogic,meson-g12a-mali",
 	  .data = &amlogic_data, },
+	{ .compatible = "renesas,r9a08g046-mali", .data = &default_pm_rt_data },
 	{ .compatible = "renesas,r9a09g047-mali", .data = &default_pm_rt_data },
 	{ .compatible = "arm,mali-t604", .data = &default_data, },
 	{ .compatible = "arm,mali-t624", .data = &default_data, },
diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c
index d59b4863b8ade..2d12b83e900ab 100644
--- a/drivers/gpu/drm/panfrost/panfrost_job.c
+++ b/drivers/gpu/drm/panfrost/panfrost_job.c
@@ -850,7 +850,6 @@ int panfrost_jm_init(struct panfrost_device *pfdev)
 {
 	struct drm_sched_init_args args = {
 		.ops = &panfrost_sched_ops,
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit = 2,
 		.timeout = msecs_to_jiffies(JOB_TIMEOUT_MS),
 		.dev = pfdev->base.dev,
diff --git a/drivers/gpu/drm/panthor/Kconfig b/drivers/gpu/drm/panthor/Kconfig
index 55b40ad07f3b0..911e7f4810c39 100644
--- a/drivers/gpu/drm/panthor/Kconfig
+++ b/drivers/gpu/drm/panthor/Kconfig
@@ -8,7 +8,6 @@ config DRM_PANTHOR
 	depends on MMU
 	select DEVFREQ_GOV_SIMPLE_ONDEMAND
 	select DRM_EXEC
-	select DRM_GEM_SHMEM_HELPER
 	select DRM_GPUVM
 	select DRM_SCHED
 	select IOMMU_IO_PGTABLE_LPAE
diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c
index 54fbb1aa07c52..bd417d6ae8c00 100644
--- a/drivers/gpu/drm/panthor/panthor_device.c
+++ b/drivers/gpu/drm/panthor/panthor_device.c
@@ -2,6 +2,7 @@
 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
 /* Copyright 2023 Collabora ltd. */
+/* Copyright 2025 ARM Limited. All rights reserved. */
 
 #include <linux/clk.h>
 #include <linux/mm.h>
@@ -18,40 +19,14 @@
 #include "panthor_devfreq.h"
 #include "panthor_device.h"
 #include "panthor_fw.h"
+#include "panthor_fw_regs.h"
 #include "panthor_gem.h"
 #include "panthor_gpu.h"
 #include "panthor_hw.h"
 #include "panthor_mmu.h"
 #include "panthor_pwr.h"
-#include "panthor_regs.h"
 #include "panthor_sched.h"
 
-static int panthor_gpu_coherency_init(struct panthor_device *ptdev)
-{
-	BUILD_BUG_ON(GPU_COHERENCY_NONE != DRM_PANTHOR_GPU_COHERENCY_NONE);
-	BUILD_BUG_ON(GPU_COHERENCY_ACE_LITE != DRM_PANTHOR_GPU_COHERENCY_ACE_LITE);
-	BUILD_BUG_ON(GPU_COHERENCY_ACE != DRM_PANTHOR_GPU_COHERENCY_ACE);
-
-	/* Start with no coherency, and update it if the device is flagged coherent. */
-	ptdev->gpu_info.selected_coherency = GPU_COHERENCY_NONE;
-	ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT;
-
-	if (!ptdev->coherent)
-		return 0;
-
-	/* Check if the ACE-Lite coherency protocol is actually supported by the GPU.
-	 * ACE protocol has never been supported for command stream frontend GPUs.
-	 */
-	if ((gpu_read(ptdev, GPU_COHERENCY_FEATURES) &
-		      GPU_COHERENCY_PROT_BIT(ACE_LITE))) {
-		ptdev->gpu_info.selected_coherency = GPU_COHERENCY_ACE_LITE;
-		return 0;
-	}
-
-	drm_err(&ptdev->base, "Coherency not supported by the device");
-	return -ENOTSUPP;
-}
-
 static int panthor_clk_init(struct panthor_device *ptdev)
 {
 	ptdev->clks.core = devm_clk_get(ptdev->base.dev, NULL);
@@ -122,6 +97,7 @@ void panthor_device_unplug(struct panthor_device *ptdev)
 	panthor_sched_unplug(ptdev);
 	panthor_fw_unplug(ptdev);
 	panthor_mmu_unplug(ptdev);
+	panthor_gem_shrinker_unplug(ptdev);
 	panthor_gpu_unplug(ptdev);
 	panthor_pwr_unplug(ptdev);
 
@@ -291,10 +267,14 @@ int panthor_device_init(struct panthor_device *ptdev)
 	if (ret)
 		goto err_unplug_gpu;
 
-	ret = panthor_mmu_init(ptdev);
+	ret = panthor_gem_shrinker_init(ptdev);
 	if (ret)
 		goto err_unplug_gpu;
 
+	ret = panthor_mmu_init(ptdev);
+	if (ret)
+		goto err_unplug_shrinker;
+
 	ret = panthor_fw_init(ptdev);
 	if (ret)
 		goto err_unplug_mmu;
@@ -326,6 +306,9 @@ int panthor_device_init(struct panthor_device *ptdev)
 err_unplug_mmu:
 	panthor_mmu_unplug(ptdev);
 
+err_unplug_shrinker:
+	panthor_gem_shrinker_unplug(ptdev);
+
 err_unplug_gpu:
 	panthor_gpu_unplug(ptdev);
 
diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h
index b6696f73a5367..a412a50eec76c 100644
--- a/drivers/gpu/drm/panthor/panthor_device.h
+++ b/drivers/gpu/drm/panthor/panthor_device.h
@@ -14,6 +14,7 @@
 #include <linux/spinlock.h>
 
 #include <drm/drm_device.h>
+#include <drm/drm_gem.h>
 #include <drm/drm_mm.h>
 #include <drm/gpu_scheduler.h>
 #include <drm/panthor_drm.h>
@@ -81,6 +82,9 @@ struct panthor_irq {
 	/** @ptdev: Panthor device */
 	struct panthor_device *ptdev;
 
+	/** @iomem: CPU mapping of IRQ base address */
+	void __iomem *iomem;
+
 	/** @irq: IRQ number. */
 	int irq;
 
@@ -178,6 +182,75 @@ struct panthor_device {
 	/** @devfreq: Device frequency scaling management data. */
 	struct panthor_devfreq *devfreq;
 
+	/** @reclaim: Reclaim related stuff */
+	struct {
+		/** @reclaim.shrinker: Shrinker instance */
+		struct shrinker *shrinker;
+
+		/**
+		 * @reclaim.unused: BOs with unused pages
+		 *
+		 * Basically all buffers that got mmapped, vmapped or GPU mapped and
+		 * then unmapped. There should be no contention on these buffers,
+		 * making them ideal to reclaim.
+		 */
+		struct drm_gem_lru unused;
+
+		/**
+		 * @reclaim.mmapped: mmap()-ed buffers
+		 *
+		 * Those are relatively easy to reclaim since we don't need user
+		 * agreement, we can simply teardown the mapping and let it fault on
+		 * the next access.
+		 */
+		struct drm_gem_lru mmapped;
+
+		/**
+		 * @reclaim.gpu_mapped_shared: shared BO LRU list
+		 *
+		 * That's the most tricky BO type to reclaim, because it involves
+		 * tearing down all mappings in all VMs where this BO is mapped,
+		 * which increases the risk of contention and thus decreases the
+		 * likeliness of success.
+		 */
+		struct drm_gem_lru gpu_mapped_shared;
+
+		/**
+		 * @reclaim.vms: VM LRU list
+		 *
+		 * VMs that have reclaimable BOs only mapped to a single VM are placed
+		 * in this LRU. Reclaiming such BOs implies waiting for VM idleness
+		 * (no in-flight GPU jobs targeting this VM), meaning we can't reclaim
+		 * those if we're in a context where we can't block/sleep.
+		 */
+		struct list_head vms;
+
+		/**
+		 * @reclaim.gpu_mapped_count: Global counter of pages that are GPU mapped
+		 *
+		 * Allows us to get the number of reclaimable pages without walking
+		 * the vms and gpu_mapped_shared LRUs.
+		 */
+		long gpu_mapped_count;
+
+		/**
+		 * @reclaim.retry_count: Number of times we ran the shrinker without being
+		 * able to reclaim stuff
+		 *
+		 * Used to stop scanning GEMs when too many attempts were made
+		 * without progress.
+		 */
+		atomic_t retry_count;
+
+#ifdef CONFIG_DEBUG_FS
+		/**
+		 * @reclaim.nr_pages_reclaimed_on_last_scan: Number of pages reclaimed on the last
+		 * shrinker scan
+		 */
+		unsigned long nr_pages_reclaimed_on_last_scan;
+#endif
+	} reclaim;
+
 	/** @unplug: Device unplug related fields. */
 	struct {
 		/** @lock: Lock used to serialize unplug operations. */
@@ -415,6 +488,11 @@ panthor_exception_is_fault(u32 exception_code)
 const char *panthor_exception_name(struct panthor_device *ptdev,
 				   u32 exception_code);
 
+#define INT_RAWSTAT 0x0
+#define INT_CLEAR   0x4
+#define INT_MASK    0x8
+#define INT_STAT    0xc
+
 /**
  * PANTHOR_IRQ_HANDLER() - Define interrupt handlers and the interrupt
  * registration function.
@@ -425,14 +503,13 @@ const char *panthor_exception_name(struct panthor_device *ptdev,
  *
  * void (*handler)(struct panthor_device *, u32 status);
  */
-#define PANTHOR_IRQ_HANDLER(__name, __reg_prefix, __handler)					\
+#define PANTHOR_IRQ_HANDLER(__name, __handler)							\
 static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data)			\
 {												\
 	struct panthor_irq *pirq = data;							\
-	struct panthor_device *ptdev = pirq->ptdev;						\
 	enum panthor_irq_state old_state;							\
 												\
-	if (!gpu_read(ptdev, __reg_prefix ## _INT_STAT))					\
+	if (!gpu_read(pirq->iomem, INT_STAT))							\
 		return IRQ_NONE;								\
 												\
 	guard(spinlock_irqsave)(&pirq->mask_lock);						\
@@ -442,7 +519,7 @@ static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data)
 	if (old_state != PANTHOR_IRQ_STATE_ACTIVE)						\
 		return IRQ_NONE;								\
 												\
-	gpu_write(ptdev, __reg_prefix ## _INT_MASK, 0);						\
+	gpu_write(pirq->iomem, INT_MASK, 0);							\
 	return IRQ_WAKE_THREAD;									\
 }												\
 												\
@@ -461,7 +538,7 @@ static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *da
 		 * right before the HW event kicks in. TLDR; it's all expected races we're	\
 		 * covered for.									\
 		 */										\
-		u32 status = gpu_read(ptdev, __reg_prefix ## _INT_RAWSTAT) & pirq->mask;	\
+		u32 status = gpu_read(pirq->iomem, INT_RAWSTAT) & pirq->mask;			\
 												\
 		if (!status)									\
 			break;									\
@@ -477,7 +554,7 @@ static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *da
 					   PANTHOR_IRQ_STATE_PROCESSING,			\
 					   PANTHOR_IRQ_STATE_ACTIVE);				\
 		if (old_state == PANTHOR_IRQ_STATE_PROCESSING)					\
-			gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask);		\
+			gpu_write(pirq->iomem, INT_MASK, pirq->mask);				\
 	}											\
 												\
 	return ret;										\
@@ -487,7 +564,7 @@ static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq)
 {												\
 	scoped_guard(spinlock_irqsave, &pirq->mask_lock) {					\
 		atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDING);				\
-		gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0);				\
+		gpu_write(pirq->iomem, INT_MASK, 0);						\
 	}											\
 	synchronize_irq(pirq->irq);								\
 	atomic_set(&pirq->state, PANTHOR_IRQ_STATE_SUSPENDED);					\
@@ -498,17 +575,18 @@ static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq)
 	guard(spinlock_irqsave)(&pirq->mask_lock);						\
 												\
 	atomic_set(&pirq->state, PANTHOR_IRQ_STATE_ACTIVE);					\
-	gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, pirq->mask);				\
-	gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask);				\
+	gpu_write(pirq->iomem, INT_CLEAR, pirq->mask);						\
+	gpu_write(pirq->iomem, INT_MASK, pirq->mask);						\
 }												\
 												\
 static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev,			\
 					      struct panthor_irq *pirq,				\
-					      int irq, u32 mask)				\
+					      int irq, u32 mask, void __iomem *iomem)		\
 {												\
 	pirq->ptdev = ptdev;									\
 	pirq->irq = irq;									\
 	pirq->mask = mask;									\
+	pirq->iomem = iomem;									\
 	spin_lock_init(&pirq->mask_lock);							\
 	panthor_ ## __name ## _irq_resume(pirq);						\
 												\
@@ -530,7 +608,7 @@ static inline void panthor_ ## __name ## _irq_enable_events(struct panthor_irq *
 	 * If the IRQ is suspended/suspending, the mask is restored at resume time.		\
 	 */											\
 	if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE)				\
-		gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask);			\
+		gpu_write(pirq->iomem, INT_MASK, pirq->mask);					\
 }												\
 												\
 static inline void panthor_ ## __name ## _irq_disable_events(struct panthor_irq *pirq, u32 mask)\
@@ -544,80 +622,80 @@ static inline void panthor_ ## __name ## _irq_disable_events(struct panthor_irq
 	 * If the IRQ is suspended/suspending, the mask is restored at resume time.		\
 	 */											\
 	if (atomic_read(&pirq->state) == PANTHOR_IRQ_STATE_ACTIVE)				\
-		gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, pirq->mask);			\
+		gpu_write(pirq->iomem, INT_MASK, pirq->mask);					\
 }
 
 extern struct workqueue_struct *panthor_cleanup_wq;
 
-static inline void gpu_write(struct panthor_device *ptdev, u32 reg, u32 data)
+static inline void gpu_write(void __iomem *iomem, u32 reg, u32 data)
 {
-	writel(data, ptdev->iomem + reg);
+	writel(data, iomem + reg);
 }
 
-static inline u32 gpu_read(struct panthor_device *ptdev, u32 reg)
+static inline u32 gpu_read(void __iomem *iomem, u32 reg)
 {
-	return readl(ptdev->iomem + reg);
+	return readl(iomem + reg);
 }
 
-static inline u32 gpu_read_relaxed(struct panthor_device *ptdev, u32 reg)
+static inline u32 gpu_read_relaxed(void __iomem *iomem, u32 reg)
 {
-	return readl_relaxed(ptdev->iomem + reg);
+	return readl_relaxed(iomem + reg);
 }
 
-static inline void gpu_write64(struct panthor_device *ptdev, u32 reg, u64 data)
+static inline void gpu_write64(void __iomem *iomem, u32 reg, u64 data)
 {
-	gpu_write(ptdev, reg, lower_32_bits(data));
-	gpu_write(ptdev, reg + 4, upper_32_bits(data));
+	gpu_write(iomem, reg, lower_32_bits(data));
+	gpu_write(iomem, reg + 4, upper_32_bits(data));
 }
 
-static inline u64 gpu_read64(struct panthor_device *ptdev, u32 reg)
+static inline u64 gpu_read64(void __iomem *iomem, u32 reg)
 {
-	return (gpu_read(ptdev, reg) | ((u64)gpu_read(ptdev, reg + 4) << 32));
+	return (gpu_read(iomem, reg) | ((u64)gpu_read(iomem, reg + 4) << 32));
 }
 
-static inline u64 gpu_read64_relaxed(struct panthor_device *ptdev, u32 reg)
+static inline u64 gpu_read64_relaxed(void __iomem *iomem, u32 reg)
 {
-	return (gpu_read_relaxed(ptdev, reg) |
-		((u64)gpu_read_relaxed(ptdev, reg + 4) << 32));
+	return (gpu_read_relaxed(iomem, reg) |
+		((u64)gpu_read_relaxed(iomem, reg + 4) << 32));
 }
 
-static inline u64 gpu_read64_counter(struct panthor_device *ptdev, u32 reg)
+static inline u64 gpu_read64_counter(void __iomem *iomem, u32 reg)
 {
 	u32 lo, hi1, hi2;
 	do {
-		hi1 = gpu_read(ptdev, reg + 4);
-		lo = gpu_read(ptdev, reg);
-		hi2 = gpu_read(ptdev, reg + 4);
+		hi1 = gpu_read(iomem, reg + 4);
+		lo = gpu_read(iomem, reg);
+		hi2 = gpu_read(iomem, reg + 4);
 	} while (hi1 != hi2);
 	return lo | ((u64)hi2 << 32);
 }
 
-#define gpu_read_poll_timeout(dev, reg, val, cond, delay_us, timeout_us)	\
+#define gpu_read_poll_timeout(iomem, reg, val, cond, delay_us, timeout_us)	\
 	read_poll_timeout(gpu_read, val, cond, delay_us, timeout_us, false,	\
-			  dev, reg)
+			  iomem, reg)
 
-#define gpu_read_poll_timeout_atomic(dev, reg, val, cond, delay_us,		\
+#define gpu_read_poll_timeout_atomic(iomem, reg, val, cond, delay_us,		\
 				     timeout_us)				\
 	read_poll_timeout_atomic(gpu_read, val, cond, delay_us, timeout_us,	\
-				 false, dev, reg)
+				 false, iomem, reg)
 
-#define gpu_read64_poll_timeout(dev, reg, val, cond, delay_us, timeout_us)	\
+#define gpu_read64_poll_timeout(iomem, reg, val, cond, delay_us, timeout_us)	\
 	read_poll_timeout(gpu_read64, val, cond, delay_us, timeout_us, false,	\
-			  dev, reg)
+			  iomem, reg)
 
-#define gpu_read64_poll_timeout_atomic(dev, reg, val, cond, delay_us,		\
+#define gpu_read64_poll_timeout_atomic(iomem, reg, val, cond, delay_us,		\
 				       timeout_us)				\
 	read_poll_timeout_atomic(gpu_read64, val, cond, delay_us, timeout_us,	\
-				 false, dev, reg)
+				 false, iomem, reg)
 
-#define gpu_read_relaxed_poll_timeout_atomic(dev, reg, val, cond, delay_us,	\
+#define gpu_read_relaxed_poll_timeout_atomic(iomem, reg, val, cond, delay_us,	\
 					     timeout_us)			\
 	read_poll_timeout_atomic(gpu_read_relaxed, val, cond, delay_us,		\
-				 timeout_us, false, dev, reg)
+				 timeout_us, false, iomem, reg)
 
-#define gpu_read64_relaxed_poll_timeout(dev, reg, val, cond, delay_us,		\
+#define gpu_read64_relaxed_poll_timeout(iomem, reg, val, cond, delay_us,	\
 					timeout_us)				\
 	read_poll_timeout(gpu_read64_relaxed, val, cond, delay_us, timeout_us,	\
-			  false, dev, reg)
+			  false, iomem, reg)
 
 #endif
diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c
index 87d27c3c14569..e8dc4096c1d25 100644
--- a/drivers/gpu/drm/panthor/panthor_drv.c
+++ b/drivers/gpu/drm/panthor/panthor_drv.c
@@ -21,6 +21,7 @@
 #include <drm/drm_debugfs.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_exec.h>
+#include <drm/drm_file.h>
 #include <drm/drm_ioctl.h>
 #include <drm/drm_print.h>
 #include <drm/drm_syncobj.h>
@@ -30,12 +31,12 @@
 
 #include "panthor_devfreq.h"
 #include "panthor_device.h"
+#include "panthor_drv.h"
 #include "panthor_fw.h"
 #include "panthor_gem.h"
 #include "panthor_gpu.h"
 #include "panthor_heap.h"
 #include "panthor_mmu.h"
-#include "panthor_regs.h"
 #include "panthor_sched.h"
 
 /**
@@ -838,7 +839,7 @@ static int panthor_query_timestamp_info(struct panthor_device *ptdev,
 	}
 
 	if (flags & DRM_PANTHOR_TIMESTAMP_GPU_OFFSET)
-		arg->timestamp_offset = gpu_read64(ptdev, GPU_TIMESTAMP_OFFSET);
+		arg->timestamp_offset = panthor_gpu_get_timestamp_offset(ptdev);
 	else
 		arg->timestamp_offset = 0;
 
@@ -853,7 +854,7 @@ static int panthor_query_timestamp_info(struct panthor_device *ptdev,
 		query_start_time = 0;
 
 	if (flags & DRM_PANTHOR_TIMESTAMP_GPU)
-		arg->current_timestamp = gpu_read64_counter(ptdev, GPU_TIMESTAMP);
+		arg->current_timestamp = panthor_gpu_get_timestamp(ptdev);
 	else
 		arg->current_timestamp = 0;
 
@@ -869,7 +870,7 @@ static int panthor_query_timestamp_info(struct panthor_device *ptdev,
 	}
 
 	if (flags & DRM_PANTHOR_TIMESTAMP_GPU_CYCLE_COUNT)
-		arg->cycle_count = gpu_read64_counter(ptdev, GPU_CYCLE_COUNT);
+		arg->cycle_count = panthor_gpu_get_cycle_count(ptdev);
 	else
 		arg->cycle_count = 0;
 
@@ -1578,7 +1579,7 @@ static int panthor_ioctl_bo_query_info(struct drm_device *ddev, void *data,
 	args->create_flags = bo->flags;
 
 	args->extra_flags = 0;
-	if (drm_gem_is_imported(&bo->base.base))
+	if (drm_gem_is_imported(&bo->base))
 		args->extra_flags |= DRM_PANTHOR_BO_IS_IMPORTED;
 
 	drm_gem_object_put(obj);
@@ -1756,34 +1757,10 @@ static const struct file_operations panthor_drm_driver_fops = {
 };
 
 #ifdef CONFIG_DEBUG_FS
-static int panthor_gems_show(struct seq_file *m, void *data)
-{
-	struct drm_info_node *node = m->private;
-	struct drm_device *dev = node->minor->dev;
-	struct panthor_device *ptdev = container_of(dev, struct panthor_device, base);
-
-	panthor_gem_debugfs_print_bos(ptdev, m);
-
-	return 0;
-}
-
-static struct drm_info_list panthor_debugfs_list[] = {
-	{"gems", panthor_gems_show, 0, NULL},
-};
-
-static int panthor_gems_debugfs_init(struct drm_minor *minor)
-{
-	drm_debugfs_create_files(panthor_debugfs_list,
-				 ARRAY_SIZE(panthor_debugfs_list),
-				 minor->debugfs_root, minor);
-
-	return 0;
-}
-
 static void panthor_debugfs_init(struct drm_minor *minor)
 {
 	panthor_mmu_debugfs_init(minor);
-	panthor_gems_debugfs_init(minor);
+	panthor_gem_debugfs_init(minor);
 }
 #endif
 
@@ -1817,8 +1794,7 @@ static const struct drm_driver panthor_drm_driver = {
 	.major = 1,
 	.minor = 8,
 
-	.gem_create_object = panthor_gem_create_object,
-	.gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table,
+	.gem_prime_import_sg_table = panthor_gem_prime_import_sg_table,
 	.gem_prime_import = panthor_gem_prime_import,
 #ifdef CONFIG_DEBUG_FS
 	.debugfs_init = panthor_debugfs_init,
@@ -1968,3 +1944,4 @@ module_exit(panthor_exit);
 MODULE_AUTHOR("Panthor Project Developers");
 MODULE_DESCRIPTION("Panthor DRM Driver");
 MODULE_LICENSE("Dual MIT/GPL");
+MODULE_IMPORT_NS("DMA_BUF");
diff --git a/drivers/gpu/drm/panthor/panthor_drv.h b/drivers/gpu/drm/panthor/panthor_drv.h
index 1bc7ddbad23ed..b81f21ed53981 100644
--- a/drivers/gpu/drm/panthor/panthor_drv.h
+++ b/drivers/gpu/drm/panthor/panthor_drv.h
@@ -4,6 +4,8 @@
 #ifndef __PANTHOR_DRV_H__
 #define __PANTHOR_DRV_H__
 
+#include <linux/types.h>
+
 extern bool panthor_transparent_hugepage;
 
 #endif
diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c
index 8886002e1d316..986151681b246 100644
--- a/drivers/gpu/drm/panthor/panthor_fw.c
+++ b/drivers/gpu/drm/panthor/panthor_fw.c
@@ -20,11 +20,11 @@
 
 #include "panthor_device.h"
 #include "panthor_fw.h"
+#include "panthor_fw_regs.h"
 #include "panthor_gem.h"
 #include "panthor_gpu.h"
 #include "panthor_hw.h"
 #include "panthor_mmu.h"
-#include "panthor_regs.h"
 #include "panthor_sched.h"
 #include "panthor_trace.h"
 
@@ -234,6 +234,9 @@ struct panthor_fw_iface {
  * struct panthor_fw - Firmware management
  */
 struct panthor_fw {
+	/** @iomem: CPU mapping of MCU_CONTROL iomem region */
+	void __iomem *iomem;
+
 	/** @vm: MCU VM. */
 	struct panthor_vm *vm;
 
@@ -628,7 +631,6 @@ static int panthor_fw_load_section_entry(struct panthor_device *ptdev,
 		u32 cache_mode = hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_CACHE_MODE_MASK;
 		struct panthor_gem_object *bo;
 		u32 vm_map_flags = 0;
-		struct sg_table *sgt;
 		u64 va = hdr.va.start;
 
 		if (!(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_WR))
@@ -666,11 +668,12 @@ static int panthor_fw_load_section_entry(struct panthor_device *ptdev,
 		panthor_fw_init_section_mem(ptdev, section);
 
 		bo = to_panthor_bo(section->mem->obj);
-		sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
-		if (IS_ERR(sgt))
-			return PTR_ERR(sgt);
 
-		dma_sync_sgtable_for_device(ptdev->base.dev, sgt, DMA_TO_DEVICE);
+		/* An sgt should have been requested when the kernel BO was GPU-mapped. */
+		if (drm_WARN_ON_ONCE(&ptdev->base, !bo->dmap.sgt))
+			return -EINVAL;
+
+		dma_sync_sgtable_for_device(ptdev->base.dev, bo->dmap.sgt, DMA_TO_DEVICE);
 	}
 
 	if (hdr.va.start == CSF_MCU_SHARED_REGION_START)
@@ -730,8 +733,10 @@ panthor_reload_fw_sections(struct panthor_device *ptdev, bool full_reload)
 			continue;
 
 		panthor_fw_init_section_mem(ptdev, section);
-		sgt = drm_gem_shmem_get_pages_sgt(&to_panthor_bo(section->mem->obj)->base);
-		if (!drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(sgt)))
+
+		/* An sgt should have been requested when the kernel BO was GPU-mapped. */
+		sgt = to_panthor_bo(section->mem->obj)->dmap.sgt;
+		if (!drm_WARN_ON_ONCE(&ptdev->base, !sgt))
 			dma_sync_sgtable_for_device(ptdev->base.dev, sgt, DMA_TO_DEVICE);
 	}
 }
@@ -1052,7 +1057,7 @@ static void panthor_fw_init_global_iface(struct panthor_device *ptdev)
 			       GLB_CFG_POWEROFF_TIMER |
 			       GLB_CFG_PROGRESS_TIMER);
 
-	gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1);
+	panthor_fw_ring_doorbell(ptdev, CSF_GLB_DOORBELL_ID);
 
 	/* Kick the watchdog. */
 	mod_delayed_work(ptdev->reset.wq, &ptdev->fw->watchdog.ping_work,
@@ -1067,7 +1072,7 @@ static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status)
 	if (tracepoint_enabled(gpu_job_irq))
 		start = ktime_get_ns();
 
-	gpu_write(ptdev, JOB_INT_CLEAR, status);
+	gpu_write(ptdev->fw->irq.iomem, INT_CLEAR, status);
 
 	if (!ptdev->fw->booted && (status & JOB_INT_GLOBAL_IF))
 		ptdev->fw->booted = true;
@@ -1086,22 +1091,23 @@ static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status)
 		trace_gpu_job_irq(ptdev->base.dev, status, duration);
 	}
 }
-PANTHOR_IRQ_HANDLER(job, JOB, panthor_job_irq_handler);
+PANTHOR_IRQ_HANDLER(job, panthor_job_irq_handler);
 
 static int panthor_fw_start(struct panthor_device *ptdev)
 {
+	struct panthor_fw *fw = ptdev->fw;
 	bool timedout = false;
 
 	ptdev->fw->booted = false;
 	panthor_job_irq_enable_events(&ptdev->fw->irq, ~0);
 	panthor_job_irq_resume(&ptdev->fw->irq);
-	gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_AUTO);
+	gpu_write(fw->iomem, MCU_CONTROL, MCU_CONTROL_AUTO);
 
 	if (!wait_event_timeout(ptdev->fw->req_waitqueue,
 				ptdev->fw->booted,
 				msecs_to_jiffies(1000))) {
 		if (!ptdev->fw->booted &&
-		    !(gpu_read(ptdev, JOB_INT_STAT) & JOB_INT_GLOBAL_IF))
+		    !(gpu_read(fw->irq.iomem, INT_STAT) & JOB_INT_GLOBAL_IF))
 			timedout = true;
 	}
 
@@ -1112,7 +1118,7 @@ static int panthor_fw_start(struct panthor_device *ptdev)
 			[MCU_STATUS_HALT] = "halt",
 			[MCU_STATUS_FATAL] = "fatal",
 		};
-		u32 status = gpu_read(ptdev, MCU_STATUS);
+		u32 status = gpu_read(fw->iomem, MCU_STATUS);
 
 		drm_err(&ptdev->base, "Failed to boot MCU (status=%s)",
 			status < ARRAY_SIZE(status_str) ? status_str[status] : "unknown");
@@ -1124,10 +1130,11 @@ static int panthor_fw_start(struct panthor_device *ptdev)
 
 static void panthor_fw_stop(struct panthor_device *ptdev)
 {
+	struct panthor_fw *fw = ptdev->fw;
 	u32 status;
 
-	gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_DISABLE);
-	if (gpu_read_poll_timeout(ptdev, MCU_STATUS, status,
+	gpu_write(fw->iomem, MCU_CONTROL, MCU_CONTROL_DISABLE);
+	if (gpu_read_poll_timeout(fw->iomem, MCU_STATUS, status,
 				  status == MCU_STATUS_DISABLED, 10, 100000))
 		drm_err(&ptdev->base, "Failed to stop MCU");
 }
@@ -1137,7 +1144,7 @@ static bool panthor_fw_mcu_halted(struct panthor_device *ptdev)
 	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
 	bool halted;
 
-	halted = gpu_read(ptdev, MCU_STATUS) == MCU_STATUS_HALT;
+	halted = gpu_read(ptdev->fw->iomem, MCU_STATUS) == MCU_STATUS_HALT;
 
 	if (panthor_fw_has_glb_state(ptdev))
 		halted &= (GLB_STATE_GET(glb_iface->output->ack) == GLB_STATE_HALT);
@@ -1154,7 +1161,7 @@ static void panthor_fw_halt_mcu(struct panthor_device *ptdev)
 	else
 		panthor_fw_update_reqs(glb_iface, req, GLB_HALT, GLB_HALT);
 
-	gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1);
+	panthor_fw_ring_doorbell(ptdev, CSF_GLB_DOORBELL_ID);
 }
 
 static bool panthor_fw_wait_mcu_halted(struct panthor_device *ptdev)
@@ -1398,6 +1405,11 @@ int panthor_fw_csg_wait_acks(struct panthor_device *ptdev, u32 csg_slot,
 	return ret;
 }
 
+void panthor_fw_ring_doorbell(struct panthor_device *ptdev, u32 doorbell_id)
+{
+	gpu_write(ptdev->iomem, CSF_DOORBELL(doorbell_id), 1);
+}
+
 /**
  * panthor_fw_ring_csg_doorbells() - Ring command stream group doorbells.
  * @ptdev: Device.
@@ -1412,7 +1424,7 @@ void panthor_fw_ring_csg_doorbells(struct panthor_device *ptdev, u32 csg_mask)
 	struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
 
 	panthor_fw_toggle_reqs(glb_iface, doorbell_req, doorbell_ack, csg_mask);
-	gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1);
+	panthor_fw_ring_doorbell(ptdev, CSF_GLB_DOORBELL_ID);
 }
 
 static void panthor_fw_ping_work(struct work_struct *work)
@@ -1427,7 +1439,7 @@ static void panthor_fw_ping_work(struct work_struct *work)
 		return;
 
 	panthor_fw_toggle_reqs(glb_iface, req, ack, GLB_PING);
-	gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1);
+	panthor_fw_ring_doorbell(ptdev, CSF_GLB_DOORBELL_ID);
 
 	ret = panthor_fw_glb_wait_acks(ptdev, GLB_PING, &acked, 100);
 	if (ret) {
@@ -1454,6 +1466,7 @@ int panthor_fw_init(struct panthor_device *ptdev)
 	if (!fw)
 		return -ENOMEM;
 
+	fw->iomem = ptdev->iomem + MCU_CONTROL_BASE;
 	ptdev->fw = fw;
 	init_waitqueue_head(&fw->req_waitqueue);
 	INIT_LIST_HEAD(&fw->sections);
@@ -1463,7 +1476,8 @@ int panthor_fw_init(struct panthor_device *ptdev)
 	if (irq <= 0)
 		return -ENODEV;
 
-	ret = panthor_request_job_irq(ptdev, &fw->irq, irq, 0);
+	ret = panthor_request_job_irq(ptdev, &fw->irq, irq, 0,
+				      ptdev->iomem + JOB_INT_BASE);
 	if (ret) {
 		drm_err(&ptdev->base, "failed to request job irq");
 		return ret;
diff --git a/drivers/gpu/drm/panthor/panthor_fw.h b/drivers/gpu/drm/panthor/panthor_fw.h
index fbdc21469ba32..a99a9b6f4825c 100644
--- a/drivers/gpu/drm/panthor/panthor_fw.h
+++ b/drivers/gpu/drm/panthor/panthor_fw.h
@@ -500,6 +500,7 @@ int panthor_fw_csg_wait_acks(struct panthor_device *ptdev, u32 csg_id, u32 req_m
 int panthor_fw_glb_wait_acks(struct panthor_device *ptdev, u32 req_mask, u32 *acked,
 			     u32 timeout_ms);
 
+void panthor_fw_ring_doorbell(struct panthor_device *ptdev, u32 doorbell_id);
 void panthor_fw_ring_csg_doorbells(struct panthor_device *ptdev, u32 csg_slot);
 
 struct panthor_kernel_bo *
diff --git a/drivers/gpu/drm/panthor/panthor_fw_regs.h b/drivers/gpu/drm/panthor/panthor_fw_regs.h
new file mode 100644
index 0000000000000..b2e59cfc22b00
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_fw_regs.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2026 ARM Limited. All rights reserved. */
+
+#ifndef __PANTHOR_FW_REGS_H__
+#define __PANTHOR_FW_REGS_H__
+
+#define MCU_CONTROL_BASE				0x700
+
+#define MCU_CONTROL					0x0
+#define   MCU_CONTROL_ENABLE				1
+#define   MCU_CONTROL_AUTO				2
+#define   MCU_CONTROL_DISABLE				0
+
+#define MCU_STATUS					0x4
+#define   MCU_STATUS_DISABLED				0
+#define   MCU_STATUS_ENABLED				1
+#define   MCU_STATUS_HALT				2
+#define   MCU_STATUS_FATAL				3
+
+#define JOB_INT_BASE					0x1000
+#define   JOB_INT_GLOBAL_IF				BIT(31)
+#define   JOB_INT_CSG_IF(x)				BIT(x)
+
+#define CSF_GPU_LATEST_FLUSH_ID				0x10000
+
+#define CSF_DOORBELL(i)					(0x80000 + ((i) * 0x10000))
+#define CSF_GLB_DOORBELL_ID				0
+
+#endif /* __PANTHOR_FW_REGS_H__ */
diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c
index cd49859da89b1..abe0c5bb1bca3 100644
--- a/drivers/gpu/drm/panthor/panthor_gem.c
+++ b/drivers/gpu/drm/panthor/panthor_gem.c
@@ -2,13 +2,21 @@
 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
 /* Copyright 2023 Collabora ltd. */
 /* Copyright 2025 Amazon.com, Inc. or its affiliates */
+/* Copyright 2025 ARM Limited. All rights reserved. */
 
 #include <linux/cleanup.h>
+#include <linux/debugfs.h>
 #include <linux/dma-buf.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
 #include <linux/slab.h>
+#include <linux/vmalloc.h>
 
+#include <drm/drm_debugfs.h>
+#include <drm/drm_file.h>
+#include <drm/drm_gpuvm.h>
+#include <drm/drm_managed.h>
+#include <drm/drm_prime.h>
 #include <drm/drm_print.h>
 #include <drm/panthor_drm.h>
 
@@ -42,7 +50,7 @@ static void panthor_gem_debugfs_bo_init(struct panthor_gem_object *bo)
 
 static void panthor_gem_debugfs_bo_add(struct panthor_gem_object *bo)
 {
-	struct panthor_device *ptdev = container_of(bo->base.base.dev,
+	struct panthor_device *ptdev = container_of(bo->base.dev,
 						    struct panthor_device, base);
 
 	bo->debugfs.creator.tgid = current->tgid;
@@ -55,7 +63,7 @@ static void panthor_gem_debugfs_bo_add(struct panthor_gem_object *bo)
 
 static void panthor_gem_debugfs_bo_rm(struct panthor_gem_object *bo)
 {
-	struct panthor_device *ptdev = container_of(bo->base.base.dev,
+	struct panthor_device *ptdev = container_of(bo->base.dev,
 						    struct panthor_device, base);
 
 	if (list_empty(&bo->debugfs.node))
@@ -78,9 +86,9 @@ static void panthor_gem_debugfs_bo_init(struct panthor_gem_object *bo) {}
 #endif
 
 static bool
-should_map_wc(struct panthor_gem_object *bo, struct panthor_vm *exclusive_vm)
+should_map_wc(struct panthor_gem_object *bo)
 {
-	struct panthor_device *ptdev = container_of(bo->base.base.dev, struct panthor_device, base);
+	struct panthor_device *ptdev = container_of(bo->base.dev, struct panthor_device, base);
 
 	/* We can't do uncached mappings if the device is coherent,
 	 * because the zeroing done by the shmem layer at page allocation
@@ -110,132 +118,359 @@ should_map_wc(struct panthor_gem_object *bo, struct panthor_vm *exclusive_vm)
 	return true;
 }
 
-static void panthor_gem_free_object(struct drm_gem_object *obj)
+static bool is_gpu_mapped(struct panthor_gem_object *bo,
+			  enum panthor_gem_reclaim_state *state)
 {
-	struct panthor_gem_object *bo = to_panthor_bo(obj);
-	struct drm_gem_object *vm_root_gem = bo->exclusive_vm_root_gem;
+	struct drm_gpuvm_bo *vm_bo;
+	u32 vm_count = 0;
 
-	panthor_gem_debugfs_bo_rm(bo);
+	drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) {
+		/* Skip evicted GPU mappings. */
+		if (vm_bo->evicted)
+			continue;
 
-	/*
-	 * Label might have been allocated with kstrdup_const(),
-	 * we need to take that into account when freeing the memory
-	 */
-	kfree_const(bo->label.str);
+		if (vm_count++) {
+			*state = PANTHOR_GEM_GPU_MAPPED_MULTI_VM;
+			break;
+		}
 
-	mutex_destroy(&bo->label.lock);
+		*state = PANTHOR_GEM_GPU_MAPPED_SINGLE_VM;
+	}
 
-	drm_gem_free_mmap_offset(&bo->base.base);
-	drm_gem_shmem_free(&bo->base);
-	drm_gem_object_put(vm_root_gem);
+	return vm_count > 0;
 }
 
-/**
- * panthor_kernel_bo_destroy() - Destroy a kernel buffer object
- * @bo: Kernel buffer object to destroy. If NULL or an ERR_PTR(), the destruction
- * is skipped.
- */
-void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo)
+static enum panthor_gem_reclaim_state
+panthor_gem_evaluate_reclaim_state_locked(struct panthor_gem_object *bo)
 {
-	struct panthor_vm *vm;
+	enum panthor_gem_reclaim_state gpu_mapped_state;
 
-	if (IS_ERR_OR_NULL(bo))
+	dma_resv_assert_held(bo->base.resv);
+	lockdep_assert_held(&bo->base.gpuva.lock);
+
+	/* If pages have not been allocated, there's nothing to reclaim. */
+	if (!bo->backing.pages)
+		return PANTHOR_GEM_UNRECLAIMABLE;
+
+	/* If memory is pinned, we prevent reclaim. */
+	if (refcount_read(&bo->backing.pin_count))
+		return PANTHOR_GEM_UNRECLAIMABLE;
+
+	if (is_gpu_mapped(bo, &gpu_mapped_state))
+		return gpu_mapped_state;
+
+	if (refcount_read(&bo->cmap.mmap_count))
+		return PANTHOR_GEM_MMAPPED;
+
+	return PANTHOR_GEM_UNUSED;
+}
+
+void panthor_gem_update_reclaim_state_locked(struct panthor_gem_object *bo,
+					     enum panthor_gem_reclaim_state *old_statep)
+{
+	struct panthor_device *ptdev = container_of(bo->base.dev, struct panthor_device, base);
+	enum panthor_gem_reclaim_state old_state = bo->reclaim_state;
+	enum panthor_gem_reclaim_state new_state;
+	bool was_gpu_mapped, is_gpu_mapped;
+
+	if (old_statep)
+		*old_statep = old_state;
+
+	new_state = panthor_gem_evaluate_reclaim_state_locked(bo);
+	if (new_state == old_state)
 		return;
 
-	vm = bo->vm;
-	panthor_kernel_bo_vunmap(bo);
+	was_gpu_mapped = old_state == PANTHOR_GEM_GPU_MAPPED_MULTI_VM ||
+			 old_state == PANTHOR_GEM_GPU_MAPPED_SINGLE_VM;
+	is_gpu_mapped = new_state == PANTHOR_GEM_GPU_MAPPED_MULTI_VM ||
+			new_state == PANTHOR_GEM_GPU_MAPPED_SINGLE_VM;
 
-	drm_WARN_ON(bo->obj->dev,
-		    to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm));
-	panthor_vm_unmap_range(vm, bo->va_node.start, bo->va_node.size);
-	panthor_vm_free_va(vm, &bo->va_node);
-	drm_gem_object_put(bo->obj);
-	panthor_vm_put(vm);
-	kfree(bo);
+	if (is_gpu_mapped && !was_gpu_mapped)
+		ptdev->reclaim.gpu_mapped_count += bo->base.size >> PAGE_SHIFT;
+	else if (!is_gpu_mapped && was_gpu_mapped)
+		ptdev->reclaim.gpu_mapped_count -= bo->base.size >> PAGE_SHIFT;
+
+	switch (new_state) {
+	case PANTHOR_GEM_UNUSED:
+		drm_gem_lru_move_tail(&ptdev->reclaim.unused, &bo->base);
+		break;
+	case PANTHOR_GEM_MMAPPED:
+		drm_gem_lru_move_tail(&ptdev->reclaim.mmapped, &bo->base);
+		break;
+	case PANTHOR_GEM_GPU_MAPPED_SINGLE_VM:
+		panthor_vm_update_bo_reclaim_lru_locked(bo);
+		break;
+	case PANTHOR_GEM_GPU_MAPPED_MULTI_VM:
+		drm_gem_lru_move_tail(&ptdev->reclaim.gpu_mapped_shared, &bo->base);
+		break;
+	case PANTHOR_GEM_UNRECLAIMABLE:
+		drm_gem_lru_remove(&bo->base);
+		break;
+	default:
+		drm_WARN(&ptdev->base, true, "invalid GEM reclaim state (%d)\n", new_state);
+		break;
+	}
+
+	bo->reclaim_state = new_state;
 }
 
-/**
- * panthor_kernel_bo_create() - Create and map a GEM object to a VM
- * @ptdev: Device.
- * @vm: VM to map the GEM to.
- * @size: Size of the buffer object.
- * @bo_flags: Combination of drm_panthor_bo_flags flags.
- * @vm_map_flags: Combination of drm_panthor_vm_bind_op_flags (only those
- * that are related to map operations).
- * @gpu_va: GPU address assigned when mapping to the VM.
- * If gpu_va == PANTHOR_VM_KERNEL_AUTO_VA, the virtual address will be
- * automatically allocated.
- * @name: Descriptive label of the BO's contents
- *
- * Return: A valid pointer in case of success, an ERR_PTR() otherwise.
- */
-struct panthor_kernel_bo *
-panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
-			 size_t size, u32 bo_flags, u32 vm_map_flags,
-			 u64 gpu_va, const char *name)
+static void
+bo_assert_locked_or_gone(struct panthor_gem_object *bo)
+{
+	/* If the refcount is zero, the BO is being freed, and we
+	 * allow the lock to not be held in that particular case.
+	 */
+	if (kref_read(&bo->base.refcount))
+		dma_resv_assert_held(bo->base.resv);
+}
+
+static void
+panthor_gem_backing_cleanup_locked(struct panthor_gem_object *bo)
+{
+	bo_assert_locked_or_gone(bo);
+
+	if (!bo->backing.pages)
+		return;
+
+	drm_gem_put_pages(&bo->base, bo->backing.pages, true, false);
+	bo->backing.pages = NULL;
+}
+
+static int
+panthor_gem_backing_get_pages_locked(struct panthor_gem_object *bo)
+{
+	struct page **pages;
+
+	dma_resv_assert_held(bo->base.resv);
+
+	if (bo->backing.pages)
+		return 0;
+
+	pages = drm_gem_get_pages(&bo->base);
+	if (IS_ERR(pages)) {
+		drm_dbg_driver(bo->base.dev, "Failed to get pages (%pe)\n", pages);
+		return PTR_ERR(pages);
+	}
+
+	bo->backing.pages = pages;
+	return 0;
+}
+
+static int panthor_gem_backing_pin_locked(struct panthor_gem_object *bo)
 {
-	struct drm_gem_shmem_object *obj;
-	struct panthor_kernel_bo *kbo;
-	struct panthor_gem_object *bo;
-	u32 debug_flags = PANTHOR_DEBUGFS_GEM_USAGE_FLAG_KERNEL;
 	int ret;
 
-	if (drm_WARN_ON(&ptdev->base, !vm))
-		return ERR_PTR(-EINVAL);
+	dma_resv_assert_held(bo->base.resv);
+	drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base));
 
-	kbo = kzalloc_obj(*kbo);
-	if (!kbo)
-		return ERR_PTR(-ENOMEM);
+	if (refcount_inc_not_zero(&bo->backing.pin_count))
+		return 0;
 
-	obj = drm_gem_shmem_create(&ptdev->base, size);
-	if (IS_ERR(obj)) {
-		ret = PTR_ERR(obj);
-		goto err_free_bo;
+	ret = panthor_gem_backing_get_pages_locked(bo);
+	if (!ret) {
+		refcount_set(&bo->backing.pin_count, 1);
+		mutex_lock(&bo->base.gpuva.lock);
+		panthor_gem_update_reclaim_state_locked(bo, NULL);
+		mutex_unlock(&bo->base.gpuva.lock);
 	}
 
-	bo = to_panthor_bo(&obj->base);
-	kbo->obj = &obj->base;
-	bo->flags = bo_flags;
-	bo->base.map_wc = should_map_wc(bo, vm);
-	bo->exclusive_vm_root_gem = panthor_vm_root_gem(vm);
-	drm_gem_object_get(bo->exclusive_vm_root_gem);
-	bo->base.base.resv = bo->exclusive_vm_root_gem->resv;
+	return ret;
+}
 
-	if (vm == panthor_fw_vm(ptdev))
-		debug_flags |= PANTHOR_DEBUGFS_GEM_USAGE_FLAG_FW_MAPPED;
+static void panthor_gem_backing_unpin_locked(struct panthor_gem_object *bo)
+{
+	bo_assert_locked_or_gone(bo);
+	drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base));
 
-	panthor_gem_kernel_bo_set_label(kbo, name);
-	panthor_gem_debugfs_set_usage_flags(to_panthor_bo(kbo->obj), debug_flags);
+	if (refcount_dec_and_test(&bo->backing.pin_count)) {
+		/* We don't release anything when pin_count drops to zero.
+		 * Pages stay there until an explicit cleanup is requested.
+		 */
+		mutex_lock(&bo->base.gpuva.lock);
+		panthor_gem_update_reclaim_state_locked(bo, NULL);
+		mutex_unlock(&bo->base.gpuva.lock);
+	}
+}
 
-	/* The system and GPU MMU page size might differ, which becomes a
-	 * problem for FW sections that need to be mapped at explicit address
-	 * since our PAGE_SIZE alignment might cover a VA range that's
-	 * expected to be used for another section.
-	 * Make sure we never map more than we need.
-	 */
-	size = ALIGN(size, panthor_vm_page_size(vm));
-	ret = panthor_vm_alloc_va(vm, gpu_va, size, &kbo->va_node);
+static void
+panthor_gem_dev_map_cleanup_locked(struct panthor_gem_object *bo)
+{
+	bo_assert_locked_or_gone(bo);
+
+	if (!bo->dmap.sgt)
+		return;
+
+	dma_unmap_sgtable(drm_dev_dma_dev(bo->base.dev), bo->dmap.sgt, DMA_BIDIRECTIONAL, 0);
+	sg_free_table(bo->dmap.sgt);
+	kfree(bo->dmap.sgt);
+	bo->dmap.sgt = NULL;
+}
+
+static struct sg_table *
+panthor_gem_dev_map_get_sgt_locked(struct panthor_gem_object *bo)
+{
+	struct sg_table *sgt;
+	int ret;
+
+	dma_resv_assert_held(bo->base.resv);
+
+	if (bo->dmap.sgt)
+		return bo->dmap.sgt;
+
+	ret = panthor_gem_backing_get_pages_locked(bo);
 	if (ret)
-		goto err_put_obj;
+		return ERR_PTR(ret);
 
-	ret = panthor_vm_map_bo_range(vm, bo, 0, size, kbo->va_node.start, vm_map_flags);
+	sgt = drm_prime_pages_to_sg(bo->base.dev, bo->backing.pages,
+				    bo->base.size >> PAGE_SHIFT);
+	if (IS_ERR(sgt))
+		return sgt;
+
+	/* Map the pages for use by the h/w. */
+	ret = dma_map_sgtable(drm_dev_dma_dev(bo->base.dev), sgt, DMA_BIDIRECTIONAL, 0);
 	if (ret)
-		goto err_free_va;
+		goto err_free_sgt;
 
-	kbo->vm = panthor_vm_get(vm);
-	return kbo;
+	bo->dmap.sgt = sgt;
+	return sgt;
 
-err_free_va:
-	panthor_vm_free_va(vm, &kbo->va_node);
+err_free_sgt:
+	sg_free_table(sgt);
+	kfree(sgt);
+	return ERR_PTR(ret);
+}
 
-err_put_obj:
-	drm_gem_object_put(&obj->base);
+struct sg_table *
+panthor_gem_get_dev_sgt(struct panthor_gem_object *bo)
+{
+	struct sg_table *sgt;
 
-err_free_bo:
-	kfree(kbo);
+	dma_resv_lock(bo->base.resv, NULL);
+	sgt = panthor_gem_dev_map_get_sgt_locked(bo);
+	dma_resv_unlock(bo->base.resv);
+
+	return sgt;
+}
+
+static void
+panthor_gem_vmap_cleanup_locked(struct panthor_gem_object *bo)
+{
+	if (!bo->cmap.vaddr)
+		return;
+
+	vunmap(bo->cmap.vaddr);
+	bo->cmap.vaddr = NULL;
+	panthor_gem_backing_unpin_locked(bo);
+}
+
+static int
+panthor_gem_prep_for_cpu_map_locked(struct panthor_gem_object *bo)
+{
+	if (should_map_wc(bo)) {
+		struct sg_table *sgt;
+
+		sgt = panthor_gem_dev_map_get_sgt_locked(bo);
+		if (IS_ERR(sgt))
+			return PTR_ERR(sgt);
+	}
+
+	return 0;
+}
+
+static void *
+panthor_gem_vmap_get_locked(struct panthor_gem_object *bo)
+{
+	pgprot_t prot = PAGE_KERNEL;
+	void *vaddr;
+	int ret;
+
+	dma_resv_assert_held(bo->base.resv);
+
+	if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base)))
+		return ERR_PTR(-EINVAL);
+
+	if (refcount_inc_not_zero(&bo->cmap.vaddr_use_count)) {
+		drm_WARN_ON_ONCE(bo->base.dev, !bo->cmap.vaddr);
+		return bo->cmap.vaddr;
+	}
+
+	ret = panthor_gem_backing_pin_locked(bo);
+	if (ret)
+		return ERR_PTR(ret);
+
+	ret = panthor_gem_prep_for_cpu_map_locked(bo);
+	if (ret)
+		goto err_unpin;
+
+	if (should_map_wc(bo))
+		prot = pgprot_writecombine(prot);
+
+	vaddr = vmap(bo->backing.pages, bo->base.size >> PAGE_SHIFT, VM_MAP, prot);
+	if (!vaddr) {
+		ret = -ENOMEM;
+		goto err_unpin;
+	}
+
+	bo->cmap.vaddr = vaddr;
+	refcount_set(&bo->cmap.vaddr_use_count, 1);
+	return vaddr;
+
+err_unpin:
+	panthor_gem_backing_unpin_locked(bo);
 	return ERR_PTR(ret);
 }
 
+static void
+panthor_gem_vmap_put_locked(struct panthor_gem_object *bo)
+{
+	dma_resv_assert_held(bo->base.resv);
+
+	if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base)))
+		return;
+
+	if (refcount_dec_and_test(&bo->cmap.vaddr_use_count))
+		panthor_gem_vmap_cleanup_locked(bo);
+}
+
+static void panthor_gem_free_object(struct drm_gem_object *obj)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+	struct drm_gem_object *vm_root_gem = bo->exclusive_vm_root_gem;
+
+	panthor_gem_debugfs_bo_rm(bo);
+
+	/*
+	 * Label might have been allocated with kstrdup_const(),
+	 * we need to take that into account when freeing the memory
+	 */
+	kfree_const(bo->label.str);
+
+	mutex_destroy(&bo->label.lock);
+
+	if (drm_gem_is_imported(obj)) {
+		drm_prime_gem_destroy(obj, bo->dmap.sgt);
+	} else {
+		/* The last ref on the GEM object can be released
+		 * by the shrinker, which can't block on the resv
+		 * lock acquisition. In practice, even if we were
+		 * taking the lock, it wouldn't block because we're
+		 * the last piece of code having a visibility on
+		 * this GEM, but lockdep can't see that, so we've
+		 * just tought the _cleanup_locked() helpers about
+		 * this "being freed" exception, and we call those
+		 * without the lock held here.
+		 */
+		panthor_gem_vmap_cleanup_locked(bo);
+		panthor_gem_dev_map_cleanup_locked(bo);
+		panthor_gem_backing_cleanup_locked(bo);
+	}
+
+	drm_gem_object_release(obj);
+
+	kfree(bo);
+	drm_gem_object_put(vm_root_gem);
+}
+
 static struct sg_table *
 panthor_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
 			      enum dma_data_direction dir)
@@ -263,15 +498,15 @@ panthor_gem_prime_begin_cpu_access(struct dma_buf *dma_buf,
 {
 	struct drm_gem_object *obj = dma_buf->priv;
 	struct drm_device *dev = obj->dev;
-	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
 	struct dma_buf_attachment *attach;
 
 	dma_resv_lock(obj->resv, NULL);
-	if (shmem->sgt)
-		dma_sync_sgtable_for_cpu(dev->dev, shmem->sgt, dir);
+	if (bo->dmap.sgt)
+		dma_sync_sgtable_for_cpu(drm_dev_dma_dev(dev), bo->dmap.sgt, dir);
 
-	if (shmem->vaddr)
-		invalidate_kernel_vmap_range(shmem->vaddr, shmem->base.size);
+	if (bo->cmap.vaddr)
+		invalidate_kernel_vmap_range(bo->cmap.vaddr, bo->base.size);
 
 	list_for_each_entry(attach, &dma_buf->attachments, node) {
 		struct sg_table *sgt = attach->priv;
@@ -290,7 +525,7 @@ panthor_gem_prime_end_cpu_access(struct dma_buf *dma_buf,
 {
 	struct drm_gem_object *obj = dma_buf->priv;
 	struct drm_device *dev = obj->dev;
-	struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj);
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
 	struct dma_buf_attachment *attach;
 
 	dma_resv_lock(obj->resv, NULL);
@@ -301,11 +536,11 @@ panthor_gem_prime_end_cpu_access(struct dma_buf *dma_buf,
 			dma_sync_sgtable_for_device(attach->dev, sgt, dir);
 	}
 
-	if (shmem->vaddr)
-		flush_kernel_vmap_range(shmem->vaddr, shmem->base.size);
+	if (bo->cmap.vaddr)
+		flush_kernel_vmap_range(bo->cmap.vaddr, bo->base.size);
 
-	if (shmem->sgt)
-		dma_sync_sgtable_for_device(dev->dev, shmem->sgt, dir);
+	if (bo->dmap.sgt)
+		dma_sync_sgtable_for_device(drm_dev_dma_dev(dev), bo->dmap.sgt, dir);
 
 	dma_resv_unlock(obj->resv);
 	return 0;
@@ -359,56 +594,469 @@ panthor_gem_prime_import(struct drm_device *dev,
 		return obj;
 	}
 
-	return drm_gem_prime_import(dev, dma_buf);
-}
+	return drm_gem_prime_import(dev, dma_buf);
+}
+
+static void panthor_gem_print_info(struct drm_printer *p, unsigned int indent,
+				   const struct drm_gem_object *obj)
+{
+	const struct panthor_gem_object *bo = to_panthor_bo(obj);
+
+	if (drm_gem_is_imported(&bo->base))
+		return;
+
+	drm_printf_indent(p, indent, "resident=%s\n", str_true_false(bo->backing.pages));
+	drm_printf_indent(p, indent, "pages_pin_count=%u\n", refcount_read(&bo->backing.pin_count));
+	drm_printf_indent(p, indent, "vmap_use_count=%u\n",
+			  refcount_read(&bo->cmap.vaddr_use_count));
+	drm_printf_indent(p, indent, "vaddr=%p\n", bo->cmap.vaddr);
+	drm_printf_indent(p, indent, "mmap_count=%u\n", refcount_read(&bo->cmap.mmap_count));
+}
+
+static int panthor_gem_pin_locked(struct drm_gem_object *obj)
+{
+	if (!drm_gem_is_imported(obj))
+		return panthor_gem_backing_pin_locked(to_panthor_bo(obj));
+
+	return 0;
+}
+
+static void panthor_gem_unpin_locked(struct drm_gem_object *obj)
+{
+	if (!drm_gem_is_imported(obj))
+		panthor_gem_backing_unpin_locked(to_panthor_bo(obj));
+}
+
+int panthor_gem_pin(struct panthor_gem_object *bo)
+{
+	int ret = 0;
+
+	if (drm_gem_is_imported(&bo->base))
+		return 0;
+
+	if (refcount_inc_not_zero(&bo->backing.pin_count))
+		return 0;
+
+	dma_resv_lock(bo->base.resv, NULL);
+	ret = panthor_gem_backing_pin_locked(bo);
+	dma_resv_unlock(bo->base.resv);
+
+	return ret;
+}
+
+void panthor_gem_unpin(struct panthor_gem_object *bo)
+{
+	if (drm_gem_is_imported(&bo->base))
+		return;
+
+	if (refcount_dec_not_one(&bo->backing.pin_count))
+		return;
+
+	dma_resv_lock(bo->base.resv, NULL);
+	panthor_gem_backing_unpin_locked(bo);
+	dma_resv_unlock(bo->base.resv);
+}
+
+int panthor_gem_swapin_locked(struct panthor_gem_object *bo)
+{
+	struct sg_table *sgt;
+
+	dma_resv_assert_held(bo->base.resv);
+
+	if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base)))
+		return -EINVAL;
+
+	sgt = panthor_gem_dev_map_get_sgt_locked(bo);
+	if (IS_ERR(sgt))
+		return PTR_ERR(sgt);
+
+	return 0;
+}
+
+static void panthor_gem_evict_locked(struct panthor_gem_object *bo)
+{
+	dma_resv_assert_held(bo->base.resv);
+	lockdep_assert_held(&bo->base.gpuva.lock);
+
+	if (drm_WARN_ON_ONCE(bo->base.dev, drm_gem_is_imported(&bo->base)))
+		return;
+
+	if (drm_WARN_ON_ONCE(bo->base.dev, refcount_read(&bo->backing.pin_count)))
+		return;
+
+	if (drm_WARN_ON_ONCE(bo->base.dev, !bo->backing.pages))
+		return;
+
+	panthor_gem_dev_map_cleanup_locked(bo);
+	panthor_gem_backing_cleanup_locked(bo);
+	panthor_gem_update_reclaim_state_locked(bo, NULL);
+}
+
+static struct sg_table *panthor_gem_get_sg_table(struct drm_gem_object *obj)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+
+	drm_WARN_ON_ONCE(obj->dev, drm_gem_is_imported(obj));
+	drm_WARN_ON_ONCE(obj->dev, !bo->backing.pages);
+	drm_WARN_ON_ONCE(obj->dev, !refcount_read(&bo->backing.pin_count));
+
+	return drm_prime_pages_to_sg(obj->dev, bo->backing.pages, obj->size >> PAGE_SHIFT);
+}
+
+static int panthor_gem_vmap_locked(struct drm_gem_object *obj,
+				   struct iosys_map *map)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+	void *vaddr;
+
+	dma_resv_assert_held(obj->resv);
+
+	if (drm_gem_is_imported(obj))
+		return dma_buf_vmap(obj->import_attach->dmabuf, map);
+
+	vaddr = panthor_gem_vmap_get_locked(bo);
+	if (IS_ERR(vaddr))
+		return PTR_ERR(vaddr);
+
+	iosys_map_set_vaddr(map, vaddr);
+	return 0;
+}
+
+static void panthor_gem_vunmap_locked(struct drm_gem_object *obj,
+				      struct iosys_map *map)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+
+	dma_resv_assert_held(obj->resv);
+
+	if (drm_gem_is_imported(obj)) {
+		dma_buf_vunmap(obj->import_attach->dmabuf, map);
+	} else {
+		drm_WARN_ON_ONCE(obj->dev, bo->cmap.vaddr != map->vaddr);
+		panthor_gem_vmap_put_locked(bo);
+	}
+}
+
+static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+	int ret;
+
+	if (drm_gem_is_imported(obj)) {
+		/* Reset both vm_ops and vm_private_data, so we don't end up with
+		 * vm_ops pointing to our implementation if the dma-buf backend
+		 * doesn't set those fields.
+		 */
+		vma->vm_private_data = NULL;
+		vma->vm_ops = NULL;
+
+		ret = dma_buf_mmap(obj->dma_buf, vma, 0);
+
+		/* Drop the reference drm_gem_mmap_obj() acquired.*/
+		if (!ret)
+			drm_gem_object_put(obj);
+
+		return ret;
+	}
+
+	if (is_cow_mapping(vma->vm_flags))
+		return -EINVAL;
+
+	if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) {
+		dma_resv_lock(obj->resv, NULL);
+		if (!refcount_inc_not_zero(&bo->cmap.mmap_count)) {
+			refcount_set(&bo->cmap.mmap_count, 1);
+			mutex_lock(&bo->base.gpuva.lock);
+			panthor_gem_update_reclaim_state_locked(bo, NULL);
+			mutex_unlock(&bo->base.gpuva.lock);
+		}
+		dma_resv_unlock(obj->resv);
+	}
+
+	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP);
+	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
+	if (should_map_wc(bo))
+		vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
+
+	return 0;
+}
+
+static enum drm_gem_object_status panthor_gem_status(struct drm_gem_object *obj)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+	enum drm_gem_object_status res = 0;
+
+	if (drm_gem_is_imported(&bo->base) || bo->backing.pages)
+		res |= DRM_GEM_OBJECT_RESIDENT;
+
+	return res;
+}
+
+static vm_fault_t insert_page(struct vm_fault *vmf, unsigned int order, struct page *page)
+{
+	if (!order) {
+		return vmf_insert_pfn(vmf->vma, vmf->address, page_to_pfn(page));
+#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
+	} else if (order == PMD_ORDER) {
+		unsigned long pfn = page_to_pfn(page);
+		unsigned long paddr = pfn << PAGE_SHIFT;
+		bool aligned = (vmf->address & ~PMD_MASK) == (paddr & ~PMD_MASK);
+
+		if (aligned &&
+		    folio_test_pmd_mappable(page_folio(page))) {
+			pfn &= PMD_MASK >> PAGE_SHIFT;
+			return vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE);
+		}
+#endif
+	}
+
+	return VM_FAULT_FALLBACK;
+}
+
+static vm_fault_t nonblocking_page_setup(struct vm_fault *vmf,
+					 unsigned int order,
+					 pgoff_t page_offset)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct panthor_gem_object *bo = to_panthor_bo(vma->vm_private_data);
+	vm_fault_t ret;
+
+	if (!dma_resv_trylock(bo->base.resv))
+		return VM_FAULT_RETRY;
+
+	if (bo->backing.pages)
+		ret = insert_page(vmf, order, bo->backing.pages[page_offset]);
+	else
+		ret = VM_FAULT_RETRY;
+
+	dma_resv_unlock(bo->base.resv);
+	return ret;
+}
+
+static vm_fault_t blocking_page_setup(struct vm_fault *vmf, unsigned int order,
+				      struct panthor_gem_object *bo,
+				      pgoff_t page_offset, bool mmap_lock_held)
+{
+	vm_fault_t ret;
+	int err;
+
+	err = dma_resv_lock_interruptible(bo->base.resv, NULL);
+	if (err)
+		return mmap_lock_held ? VM_FAULT_NOPAGE : VM_FAULT_RETRY;
+
+	err = panthor_gem_backing_get_pages_locked(bo);
+	if (!err)
+		err = panthor_gem_prep_for_cpu_map_locked(bo);
+
+	if (err) {
+		ret = mmap_lock_held ? VM_FAULT_SIGBUS : VM_FAULT_RETRY;
+	} else {
+		struct page *page = bo->backing.pages[page_offset];
+
+		mutex_lock(&bo->base.gpuva.lock);
+		panthor_gem_update_reclaim_state_locked(bo, NULL);
+		mutex_unlock(&bo->base.gpuva.lock);
+
+		if (mmap_lock_held)
+			ret = insert_page(vmf, order, page);
+		else
+			ret = VM_FAULT_RETRY;
+	}
+
+	dma_resv_unlock(bo->base.resv);
+
+	return ret;
+}
+
+static vm_fault_t panthor_gem_any_fault(struct vm_fault *vmf, unsigned int order)
+{
+	struct vm_area_struct *vma = vmf->vma;
+	struct panthor_gem_object *bo = to_panthor_bo(vma->vm_private_data);
+	loff_t num_pages = bo->base.size >> PAGE_SHIFT;
+	pgoff_t page_offset;
+	vm_fault_t ret;
+
+	if (order && order != PMD_ORDER)
+		return VM_FAULT_FALLBACK;
+
+	/* Offset to faulty address in the VMA. */
+	page_offset = vmf->pgoff - vma->vm_pgoff;
+	if (page_offset >= num_pages)
+		return VM_FAULT_SIGBUS;
+
+	ret = nonblocking_page_setup(vmf, order, page_offset);
+	if (ret != VM_FAULT_RETRY)
+		return ret;
+
+	/* Check if we're allowed to retry. */
+	if (fault_flag_allow_retry_first(vmf->flags)) {
+		/* If we're allowed to retry but not wait here, return
+		 * immediately, the wait will be done when the fault
+		 * handler is called again, with the mmap_lock held.
+		 */
+		if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
+			return VM_FAULT_RETRY;
+
+		/* Wait with the mmap lock released, if we're allowed to. */
+		drm_gem_object_get(&bo->base);
+
+		if (vmf->flags & FAULT_FLAG_VMA_LOCK)
+			vma_end_read(vmf->vma);
+		else
+			mmap_read_unlock(vmf->vma->vm_mm);
+
+		ret = blocking_page_setup(vmf, order, bo, page_offset, false);
+		drm_gem_object_put(&bo->base);
+		return ret;
+	}
+
+	return blocking_page_setup(vmf, order, bo, page_offset, true);
+}
+
+static vm_fault_t panthor_gem_fault(struct vm_fault *vmf)
+{
+	return panthor_gem_any_fault(vmf, 0);
+}
+
+static void panthor_gem_vm_open(struct vm_area_struct *vma)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(vma->vm_private_data);
+
+	drm_WARN_ON(bo->base.dev, drm_gem_is_imported(&bo->base));
+	drm_WARN_ON(bo->base.dev, !refcount_inc_not_zero(&bo->cmap.mmap_count));
+
+	drm_gem_vm_open(vma);
+}
+
+static void panthor_gem_vm_close(struct vm_area_struct *vma)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(vma->vm_private_data);
+
+	if (drm_gem_is_imported(&bo->base))
+		goto out;
+
+	if (refcount_dec_not_one(&bo->cmap.mmap_count))
+		goto out;
+
+	dma_resv_lock(bo->base.resv, NULL);
+	if (refcount_dec_and_test(&bo->cmap.mmap_count)) {
+		mutex_lock(&bo->base.gpuva.lock);
+		panthor_gem_update_reclaim_state_locked(bo, NULL);
+		mutex_unlock(&bo->base.gpuva.lock);
+	}
+	dma_resv_unlock(bo->base.resv);
+
+out:
+	drm_gem_object_put(&bo->base);
+}
+
+static const struct vm_operations_struct panthor_gem_vm_ops = {
+	.fault = panthor_gem_fault,
+#ifdef CONFIG_ARCH_SUPPORTS_PMD_PFNMAP
+	.huge_fault = panthor_gem_any_fault,
+#endif
+	.open = panthor_gem_vm_open,
+	.close = panthor_gem_vm_close,
+};
+
+static const struct drm_gem_object_funcs panthor_gem_funcs = {
+	.free = panthor_gem_free_object,
+	.print_info = panthor_gem_print_info,
+	.pin = panthor_gem_pin_locked,
+	.unpin = panthor_gem_unpin_locked,
+	.get_sg_table = panthor_gem_get_sg_table,
+	.vmap = panthor_gem_vmap_locked,
+	.vunmap = panthor_gem_vunmap_locked,
+	.mmap = panthor_gem_mmap,
+	.status = panthor_gem_status,
+	.export = panthor_gem_prime_export,
+	.vm_ops = &panthor_gem_vm_ops,
+};
+
+static struct panthor_gem_object *
+panthor_gem_alloc_object(u32 flags)
+{
+	struct panthor_gem_object *bo;
+
+	bo = kzalloc_obj(*bo);
+	if (!bo)
+		return ERR_PTR(-ENOMEM);
+
+	bo->reclaim_state = PANTHOR_GEM_UNRECLAIMABLE;
+	bo->base.funcs = &panthor_gem_funcs;
+	bo->flags = flags;
+	mutex_init(&bo->label.lock);
+	panthor_gem_debugfs_bo_init(bo);
+	return bo;
+}
+
+static struct panthor_gem_object *
+panthor_gem_create(struct drm_device *dev, size_t size, uint32_t flags,
+		   struct panthor_vm *exclusive_vm, u32 usage_flags)
+{
+	struct panthor_gem_object *bo;
+	int ret;
+
+	bo = panthor_gem_alloc_object(flags);
+	if (IS_ERR(bo))
+		return bo;
+
+	size = PAGE_ALIGN(size);
+	ret = drm_gem_object_init(dev, &bo->base, size);
+	if (ret)
+		goto err_put;
 
-static enum drm_gem_object_status panthor_gem_status(struct drm_gem_object *obj)
-{
-	struct panthor_gem_object *bo = to_panthor_bo(obj);
-	enum drm_gem_object_status res = 0;
+	/* Our buffers are kept pinned, so allocating them
+	 * from the MOVABLE zone is a really bad idea, and
+	 * conflicts with CMA. See comments above new_inode()
+	 * why this is required _and_ expected if you're
+	 * going to pin these pages.
+	 */
+	mapping_set_gfp_mask(bo->base.filp->f_mapping,
+			     GFP_HIGHUSER | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
 
-	if (drm_gem_is_imported(&bo->base.base) || bo->base.pages)
-		res |= DRM_GEM_OBJECT_RESIDENT;
+	ret = drm_gem_create_mmap_offset(&bo->base);
+	if (ret)
+		goto err_put;
 
-	return res;
-}
+	if (exclusive_vm) {
+		bo->exclusive_vm_root_gem = panthor_vm_root_gem(exclusive_vm);
+		drm_gem_object_get(bo->exclusive_vm_root_gem);
+		bo->base.resv = bo->exclusive_vm_root_gem->resv;
+	}
 
-static const struct drm_gem_object_funcs panthor_gem_funcs = {
-	.free = panthor_gem_free_object,
-	.print_info = drm_gem_shmem_object_print_info,
-	.pin = drm_gem_shmem_object_pin,
-	.unpin = drm_gem_shmem_object_unpin,
-	.get_sg_table = drm_gem_shmem_object_get_sg_table,
-	.vmap = drm_gem_shmem_object_vmap,
-	.vunmap = drm_gem_shmem_object_vunmap,
-	.mmap = drm_gem_shmem_object_mmap,
-	.status = panthor_gem_status,
-	.export = panthor_gem_prime_export,
-	.vm_ops = &drm_gem_shmem_vm_ops,
-};
+	panthor_gem_debugfs_set_usage_flags(bo, usage_flags);
+	return bo;
 
-/**
- * panthor_gem_create_object - Implementation of driver->gem_create_object.
- * @ddev: DRM device
- * @size: Size in bytes of the memory the object will reference
- *
- * This lets the GEM helpers allocate object structs for us, and keep
- * our BO stats correct.
- */
-struct drm_gem_object *panthor_gem_create_object(struct drm_device *ddev, size_t size)
+err_put:
+	drm_gem_object_put(&bo->base);
+	return ERR_PTR(ret);
+}
+
+struct drm_gem_object *
+panthor_gem_prime_import_sg_table(struct drm_device *dev,
+				  struct dma_buf_attachment *attach,
+				  struct sg_table *sgt)
 {
-	struct panthor_gem_object *obj;
+	struct panthor_gem_object *bo;
+	int ret;
 
-	obj = kzalloc_obj(*obj);
-	if (!obj)
-		return ERR_PTR(-ENOMEM);
+	bo = panthor_gem_alloc_object(0);
+	if (IS_ERR(bo))
+		return ERR_CAST(bo);
+
+	drm_gem_private_object_init(dev, &bo->base, attach->dmabuf->size);
 
-	obj->base.base.funcs = &panthor_gem_funcs;
-	mutex_init(&obj->label.lock);
+	ret = drm_gem_create_mmap_offset(&bo->base);
+	if (ret)
+		goto err_put;
 
-	panthor_gem_debugfs_bo_init(obj);
+	bo->dmap.sgt = sgt;
+	return &bo->base;
 
-	return &obj->base.base;
+err_put:
+	drm_gem_object_put(&bo->base);
+	return ERR_PTR(ret);
 }
 
 /**
@@ -429,54 +1077,22 @@ panthor_gem_create_with_handle(struct drm_file *file,
 			       u64 *size, u32 flags, u32 *handle)
 {
 	int ret;
-	struct drm_gem_shmem_object *shmem;
 	struct panthor_gem_object *bo;
 
-	shmem = drm_gem_shmem_create(ddev, *size);
-	if (IS_ERR(shmem))
-		return PTR_ERR(shmem);
-
-	bo = to_panthor_bo(&shmem->base);
-	bo->flags = flags;
-	bo->base.map_wc = should_map_wc(bo, exclusive_vm);
-
-	if (exclusive_vm) {
-		bo->exclusive_vm_root_gem = panthor_vm_root_gem(exclusive_vm);
-		drm_gem_object_get(bo->exclusive_vm_root_gem);
-		bo->base.base.resv = bo->exclusive_vm_root_gem->resv;
-	}
-
-	panthor_gem_debugfs_set_usage_flags(bo, 0);
-
-	/* If this is a write-combine mapping, we query the sgt to force a CPU
-	 * cache flush (dma_map_sgtable() is called when the sgt is created).
-	 * This ensures the zero-ing is visible to any uncached mapping created
-	 * by vmap/mmap.
-	 * FIXME: Ideally this should be done when pages are allocated, not at
-	 * BO creation time.
-	 */
-	if (shmem->map_wc) {
-		struct sg_table *sgt;
-
-		sgt = drm_gem_shmem_get_pages_sgt(shmem);
-		if (IS_ERR(sgt)) {
-			ret = PTR_ERR(sgt);
-			goto out_put_gem;
-		}
-	}
+	bo = panthor_gem_create(ddev, *size, flags, exclusive_vm, 0);
+	if (IS_ERR(bo))
+		return PTR_ERR(bo);
 
 	/*
 	 * Allocate an id of idr table where the obj is registered
 	 * and handle has the id what user can see.
 	 */
-	ret = drm_gem_handle_create(file, &shmem->base, handle);
+	ret = drm_gem_handle_create(file, &bo->base, handle);
 	if (!ret)
-		*size = bo->base.base.size;
+		*size = bo->base.size;
 
-out_put_gem:
 	/* drop reference from allocate - handle holds it now. */
-	drm_gem_object_put(&shmem->base);
-
+	drm_gem_object_put(&bo->base);
 	return ret;
 }
 
@@ -521,18 +1137,18 @@ panthor_gem_sync(struct drm_gem_object *obj, u32 type,
 		 u64 offset, u64 size)
 {
 	struct panthor_gem_object *bo = to_panthor_bo(obj);
-	struct drm_gem_shmem_object *shmem = &bo->base;
-	const struct drm_device *dev = shmem->base.dev;
+	struct device *dma_dev = drm_dev_dma_dev(bo->base.dev);
 	struct sg_table *sgt;
 	struct scatterlist *sgl;
 	unsigned int count;
+	int ret;
 
 	/* Make sure the range is in bounds. */
-	if (offset + size < offset || offset + size > shmem->base.size)
+	if (offset + size < offset || offset + size > bo->base.size)
 		return -EINVAL;
 
 	/* Disallow CPU-cache maintenance on imported buffers. */
-	if (drm_gem_is_imported(&shmem->base))
+	if (drm_gem_is_imported(&bo->base))
 		return -EINVAL;
 
 	switch (type) {
@@ -545,16 +1161,28 @@ panthor_gem_sync(struct drm_gem_object *obj, u32 type,
 	}
 
 	/* Don't bother if it's WC-mapped */
-	if (shmem->map_wc)
+	if (should_map_wc(bo))
 		return 0;
 
 	/* Nothing to do if the size is zero. */
 	if (size == 0)
 		return 0;
 
-	sgt = drm_gem_shmem_get_pages_sgt(shmem);
-	if (IS_ERR(sgt))
-		return PTR_ERR(sgt);
+	ret = dma_resv_lock_interruptible(bo->base.resv, NULL);
+	if (ret)
+		return ret;
+
+	/* If there's no pages, there's no point pulling those back, bail out early. */
+	if (!bo->backing.pages) {
+		ret = 0;
+		goto out_unlock;
+	}
+
+	sgt = panthor_gem_dev_map_get_sgt_locked(bo);
+	if (IS_ERR(sgt)) {
+		ret = PTR_ERR(sgt);
+		goto out_unlock;
+	}
 
 	for_each_sgtable_dma_sg(sgt, sgl, count) {
 		if (size == 0)
@@ -593,14 +1221,360 @@ panthor_gem_sync(struct drm_gem_object *obj, u32 type,
 		 *
 		 * for the flush+invalidate case.
 		 */
-		dma_sync_single_for_device(dev->dev, paddr, len, DMA_TO_DEVICE);
+		dma_sync_single_for_device(dma_dev, paddr, len, DMA_TO_DEVICE);
 		if (type == DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE)
-			dma_sync_single_for_cpu(dev->dev, paddr, len, DMA_FROM_DEVICE);
+			dma_sync_single_for_cpu(dma_dev, paddr, len, DMA_FROM_DEVICE);
+	}
+
+	ret = 0;
+
+out_unlock:
+	dma_resv_unlock(bo->base.resv);
+	return ret;
+}
+
+/**
+ * panthor_kernel_bo_destroy() - Destroy a kernel buffer object
+ * @bo: Kernel buffer object to destroy. If NULL or an ERR_PTR(), the destruction
+ * is skipped.
+ */
+void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo)
+{
+	struct panthor_device *ptdev;
+	struct panthor_vm *vm;
+
+	if (IS_ERR_OR_NULL(bo))
+		return;
+
+	ptdev = container_of(bo->obj->dev, struct panthor_device, base);
+	vm = bo->vm;
+	panthor_kernel_bo_vunmap(bo);
+
+	drm_WARN_ON(bo->obj->dev,
+		    to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm));
+	panthor_vm_unmap_range(vm, bo->va_node.start, bo->va_node.size);
+	panthor_vm_free_va(vm, &bo->va_node);
+	if (vm == panthor_fw_vm(ptdev))
+		panthor_gem_unpin(to_panthor_bo(bo->obj));
+	drm_gem_object_put(bo->obj);
+	panthor_vm_put(vm);
+	kfree(bo);
+}
+
+/**
+ * panthor_kernel_bo_create() - Create and map a GEM object to a VM
+ * @ptdev: Device.
+ * @vm: VM to map the GEM to.
+ * @size: Size of the buffer object.
+ * @bo_flags: Combination of drm_panthor_bo_flags flags.
+ * @vm_map_flags: Combination of drm_panthor_vm_bind_op_flags (only those
+ * that are related to map operations).
+ * @gpu_va: GPU address assigned when mapping to the VM.
+ * If gpu_va == PANTHOR_VM_KERNEL_AUTO_VA, the virtual address will be
+ * automatically allocated.
+ * @name: Descriptive label of the BO's contents
+ *
+ * Return: A valid pointer in case of success, an ERR_PTR() otherwise.
+ */
+struct panthor_kernel_bo *
+panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
+			 size_t size, u32 bo_flags, u32 vm_map_flags,
+			 u64 gpu_va, const char *name)
+{
+	struct panthor_kernel_bo *kbo;
+	struct panthor_gem_object *bo;
+	u32 debug_flags = PANTHOR_DEBUGFS_GEM_USAGE_FLAG_KERNEL;
+	int ret;
+
+	if (drm_WARN_ON(&ptdev->base, !vm))
+		return ERR_PTR(-EINVAL);
+
+	kbo = kzalloc_obj(*kbo);
+	if (!kbo)
+		return ERR_PTR(-ENOMEM);
+
+	if (vm == panthor_fw_vm(ptdev))
+		debug_flags |= PANTHOR_DEBUGFS_GEM_USAGE_FLAG_FW_MAPPED;
+
+	bo = panthor_gem_create(&ptdev->base, size, bo_flags, vm, debug_flags);
+	if (IS_ERR(bo)) {
+		ret = PTR_ERR(bo);
+		goto err_free_kbo;
+	}
+
+	kbo->obj = &bo->base;
+
+	if (vm == panthor_fw_vm(ptdev)) {
+		ret = panthor_gem_pin(bo);
+		if (ret)
+			goto err_put_obj;
+	}
+
+	panthor_gem_kernel_bo_set_label(kbo, name);
+
+	/* The system and GPU MMU page size might differ, which becomes a
+	 * problem for FW sections that need to be mapped at explicit address
+	 * since our PAGE_SIZE alignment might cover a VA range that's
+	 * expected to be used for another section.
+	 * Make sure we never map more than we need.
+	 */
+	size = ALIGN(size, panthor_vm_page_size(vm));
+	ret = panthor_vm_alloc_va(vm, gpu_va, size, &kbo->va_node);
+	if (ret)
+		goto err_unpin;
+
+	ret = panthor_vm_map_bo_range(vm, bo, 0, size, kbo->va_node.start, vm_map_flags);
+	if (ret)
+		goto err_free_va;
+
+	kbo->vm = panthor_vm_get(vm);
+	return kbo;
+
+err_free_va:
+	panthor_vm_free_va(vm, &kbo->va_node);
+
+err_unpin:
+	if (vm == panthor_fw_vm(ptdev))
+		panthor_gem_unpin(bo);
+
+err_put_obj:
+	drm_gem_object_put(&bo->base);
+
+err_free_kbo:
+	kfree(kbo);
+	return ERR_PTR(ret);
+}
+
+static bool can_swap(void)
+{
+	return get_nr_swap_pages() > 0;
+}
+
+static bool can_block(struct shrink_control *sc)
+{
+	/* If direct reclaim is allowed, we can always block.
+	 * If kswapd reclaim is allowed, we can block, but only if we're called
+	 * by the kswapd thread.
+	 */
+	return (sc->gfp_mask & __GFP_DIRECT_RECLAIM) ||
+	       ((sc->gfp_mask & __GFP_KSWAPD_RECLAIM) && current_is_kswapd());
+}
+
+static unsigned long
+panthor_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
+{
+	struct panthor_device *ptdev = shrinker->private_data;
+	unsigned long count;
+
+	/* We currently don't have a flag to tell when the content of a
+	 * BO can be discarded.
+	 */
+	if (!can_swap())
+		return 0;
+
+	/* This is racy, but that's okay because the returned count is just a
+	 * hint. That's also what MSM is doing (no atomic var, it's relying on
+	 * the fact unsigned long access is usually atomic), so if it's good
+	 * enough for them, it's good enough for us too.
+	 */
+	count = ptdev->reclaim.unused.count;
+	count += ptdev->reclaim.mmapped.count;
+
+	if (can_block(sc))
+		count += ptdev->reclaim.gpu_mapped_count;
+
+	return count ? count : SHRINK_EMPTY;
+}
+
+static bool panthor_gem_try_evict_no_resv_wait(struct drm_gem_object *obj,
+					       struct ww_acquire_ctx *ticket)
+{
+	/*
+	 * Track last locked entry for unwinding locks in error and
+	 * success paths
+	 */
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+	struct drm_gpuvm_bo *vm_bo, *last_locked = NULL;
+	enum panthor_gem_reclaim_state old_state;
+	int ret = 0;
+
+	/* To avoid potential lock ordering issue between bo_gpuva and
+	 * mapping->i_mmap_rwsem, unmap the pages from CPU side before
+	 * acquring the bo_gpuva lock. As the bo_resv lock is held, CPU
+	 * page fault handler won't be able to map in the pages whilst
+	 * eviction is in progress.
+	 */
+	drm_vma_node_unmap(&bo->base.vma_node, bo->base.dev->anon_inode->i_mapping);
+
+	/* We take this lock when walking the list to prevent
+	 * insertion/deletion.
+	 */
+	/* We can only trylock in that path, because
+	 * - allocation might happen while some of these locks are held
+	 * - lock ordering is different in other paths
+	 *     vm_resv -> bo_resv -> bo_gpuva
+	 *     vs
+	 *     bo_resv -> bo_gpuva -> vm_resv
+	 *
+	 * If we fail to lock that's fine, we back off and will get
+	 * back to it later.
+	 */
+	if (!mutex_trylock(&bo->base.gpuva.lock))
+		return false;
+
+	drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+		struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
+
+		if (resv == obj->resv)
+			continue;
+
+		if (!dma_resv_trylock(resv)) {
+			ret = -EDEADLK;
+			goto out_unlock;
+		}
+
+		last_locked = vm_bo;
+	}
+
+	/* Update the state before trying to evict the buffer, if the state was
+	 * updated to something that's harder to reclaim (higher value in the
+	 * enum), skip it (will be processed when the relevant LRU is).
+	 */
+	panthor_gem_update_reclaim_state_locked(bo, &old_state);
+	if (old_state < bo->reclaim_state) {
+		ret = -EAGAIN;
+		goto out_unlock;
+	}
+
+	/* Couldn't teardown the GPU mappings? Skip. */
+	ret = panthor_vm_evict_bo_mappings_locked(bo);
+	if (ret)
+		goto out_unlock;
+
+	/* If everything went fine, evict the object. */
+	panthor_gem_evict_locked(bo);
+
+out_unlock:
+	if (last_locked) {
+		drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
+			struct dma_resv *resv = drm_gpuvm_resv(vm_bo->vm);
+
+			if (resv == obj->resv)
+				continue;
+
+			dma_resv_unlock(resv);
+
+			if (last_locked == vm_bo)
+				break;
+		}
 	}
+	mutex_unlock(&bo->base.gpuva.lock);
+
+	return ret == 0;
+}
+
+static bool panthor_gem_try_evict(struct drm_gem_object *obj,
+				  struct ww_acquire_ctx *ticket)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(obj);
+
+	/* Wait was too long, skip. */
+	if (dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_BOOKKEEP, false, 10) <= 0)
+		return false;
+
+	return panthor_gem_try_evict_no_resv_wait(&bo->base, ticket);
+}
+
+static unsigned long
+panthor_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
+{
+	struct panthor_device *ptdev = shrinker->private_data;
+	unsigned long remaining = 0;
+	unsigned long freed = 0;
+
+	if (!can_swap())
+		goto out;
+
+	freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.unused,
+				  sc->nr_to_scan - freed, &remaining,
+				  panthor_gem_try_evict_no_resv_wait, NULL);
+	if (freed >= sc->nr_to_scan)
+		goto out;
+
+	freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.mmapped,
+				  sc->nr_to_scan - freed, &remaining,
+				  panthor_gem_try_evict_no_resv_wait, NULL);
+	if (freed >= sc->nr_to_scan)
+		goto out;
+
+	if (!can_block(sc))
+		goto out;
+
+	freed += panthor_mmu_reclaim_priv_bos(ptdev, sc->nr_to_scan - freed,
+					      &remaining, panthor_gem_try_evict);
+	if (freed >= sc->nr_to_scan)
+		goto out;
+
+	freed += drm_gem_lru_scan(&ptdev->base, &ptdev->reclaim.gpu_mapped_shared,
+				  sc->nr_to_scan - freed, &remaining,
+				  panthor_gem_try_evict, NULL);
+
+out:
+#ifdef CONFIG_DEBUG_FS
+	/* This is racy, but that's okay, because this is just debugfs
+	 * reporting and doesn't need to be accurate.
+	 */
+	ptdev->reclaim.nr_pages_reclaimed_on_last_scan = freed;
+#endif
+
+	/* If there are things to reclaim, try a couple times before giving up. */
+	if (!freed && remaining > 0 &&
+	    atomic_inc_return(&ptdev->reclaim.retry_count) < 2)
+		return 0;
+
+	atomic_set(&ptdev->reclaim.retry_count, 0);
+
+	if (freed)
+		return freed;
+
+	/* There's nothing left to reclaim, or the resources are contended. Give up now. */
+	return SHRINK_STOP;
+}
+
+int panthor_gem_shrinker_init(struct panthor_device *ptdev)
+{
+	struct shrinker *shrinker;
+
+	INIT_LIST_HEAD(&ptdev->reclaim.vms);
+	drm_gem_lru_init(&ptdev->reclaim.unused);
+	drm_gem_lru_init(&ptdev->reclaim.mmapped);
+	drm_gem_lru_init(&ptdev->reclaim.gpu_mapped_shared);
+	ptdev->reclaim.gpu_mapped_count = 0;
+
+	/* Teach lockdep about lock ordering wrt. shrinker: */
+	fs_reclaim_acquire(GFP_KERNEL);
+	might_lock(&ptdev->base.gem_lru_mutex);
+	fs_reclaim_release(GFP_KERNEL);
+
+	shrinker = shrinker_alloc(0, "drm-panthor-gem");
+	if (!shrinker)
+		return -ENOMEM;
 
+	shrinker->count_objects = panthor_gem_shrinker_count;
+	shrinker->scan_objects = panthor_gem_shrinker_scan;
+	shrinker->private_data = ptdev;
+	ptdev->reclaim.shrinker = shrinker;
+
+	shrinker_register(shrinker);
 	return 0;
 }
 
+void panthor_gem_shrinker_unplug(struct panthor_device *ptdev)
+{
+	if (ptdev->reclaim.shrinker)
+		shrinker_free(ptdev->reclaim.shrinker);
+}
+
 #ifdef CONFIG_DEBUG_FS
 struct gem_size_totals {
 	size_t size;
@@ -644,7 +1618,8 @@ static void panthor_gem_debugfs_bo_print(struct panthor_gem_object *bo,
 					 struct seq_file *m,
 					 struct gem_size_totals *totals)
 {
-	unsigned int refcount = kref_read(&bo->base.base.refcount);
+	enum panthor_gem_reclaim_state reclaim_state = bo->reclaim_state;
+	unsigned int refcount = kref_read(&bo->base.refcount);
 	char creator_info[32] = {};
 	size_t resident_size;
 	u32 gem_usage_flags = bo->debugfs.flags;
@@ -654,21 +1629,21 @@ static void panthor_gem_debugfs_bo_print(struct panthor_gem_object *bo,
 	if (!refcount)
 		return;
 
-	resident_size = bo->base.pages ? bo->base.base.size : 0;
+	resident_size = bo->backing.pages ? bo->base.size : 0;
 
 	snprintf(creator_info, sizeof(creator_info),
 		 "%s/%d", bo->debugfs.creator.process_name, bo->debugfs.creator.tgid);
 	seq_printf(m, "%-32s%-16d%-16d%-16zd%-16zd0x%-16lx",
 		   creator_info,
-		   bo->base.base.name,
+		   bo->base.name,
 		   refcount,
-		   bo->base.base.size,
+		   bo->base.size,
 		   resident_size,
-		   drm_vma_node_start(&bo->base.base.vma_node));
+		   drm_vma_node_start(&bo->base.vma_node));
 
-	if (drm_gem_is_imported(&bo->base.base))
+	if (drm_gem_is_imported(&bo->base))
 		gem_state_flags |= PANTHOR_DEBUGFS_GEM_STATE_FLAG_IMPORTED;
-	if (bo->base.base.dma_buf)
+	if (bo->base.dma_buf)
 		gem_state_flags |= PANTHOR_DEBUGFS_GEM_STATE_FLAG_EXPORTED;
 
 	seq_printf(m, "0x%-8x 0x%-10x", gem_state_flags, gem_usage_flags);
@@ -677,14 +1652,14 @@ static void panthor_gem_debugfs_bo_print(struct panthor_gem_object *bo,
 		seq_printf(m, "%s\n", bo->label.str ? : "");
 	}
 
-	totals->size += bo->base.base.size;
+	totals->size += bo->base.size;
 	totals->resident += resident_size;
-	if (bo->base.madv > 0)
+	if (reclaim_state != PANTHOR_GEM_UNRECLAIMABLE)
 		totals->reclaimable += resident_size;
 }
 
-void panthor_gem_debugfs_print_bos(struct panthor_device *ptdev,
-				   struct seq_file *m)
+static void panthor_gem_debugfs_print_bos(struct panthor_device *ptdev,
+					  struct seq_file *m)
 {
 	struct gem_size_totals totals = {0};
 	struct panthor_gem_object *bo;
@@ -704,4 +1679,59 @@ void panthor_gem_debugfs_print_bos(struct panthor_device *ptdev,
 	seq_printf(m, "Total size: %zd, Total resident: %zd, Total reclaimable: %zd\n",
 		   totals.size, totals.resident, totals.reclaimable);
 }
+
+static int panthor_gem_show_bos(struct seq_file *m, void *data)
+{
+	struct drm_info_node *node = m->private;
+	struct drm_device *dev = node->minor->dev;
+	struct panthor_device *ptdev =
+		container_of(dev, struct panthor_device, base);
+
+	panthor_gem_debugfs_print_bos(ptdev, m);
+
+	return 0;
+}
+
+static struct drm_info_list panthor_gem_debugfs_list[] = {
+	{ "gems", panthor_gem_show_bos, 0, NULL },
+};
+
+static int shrink_get(void *data, u64 *val)
+{
+	struct panthor_device *ptdev =
+		container_of(data, struct panthor_device, base);
+
+	*val = ptdev->reclaim.nr_pages_reclaimed_on_last_scan;
+	return 0;
+}
+
+static int shrink_set(void *data, u64 val)
+{
+	struct panthor_device *ptdev =
+		container_of(data, struct panthor_device, base);
+	struct shrink_control sc = {
+		.gfp_mask = GFP_KERNEL,
+		.nr_to_scan = val,
+	};
+
+	fs_reclaim_acquire(GFP_KERNEL);
+	if (ptdev->reclaim.shrinker)
+		panthor_gem_shrinker_scan(ptdev->reclaim.shrinker, &sc);
+	fs_reclaim_release(GFP_KERNEL);
+
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(panthor_gem_debugfs_shrink_fops,
+			 shrink_get, shrink_set,
+			 "0x%08llx\n");
+
+void panthor_gem_debugfs_init(struct drm_minor *minor)
+{
+	drm_debugfs_create_files(panthor_gem_debugfs_list,
+				 ARRAY_SIZE(panthor_gem_debugfs_list),
+				 minor->debugfs_root, minor);
+	debugfs_create_file("shrink", 0600, minor->debugfs_root,
+			    minor->dev, &panthor_gem_debugfs_shrink_fops);
+}
 #endif
diff --git a/drivers/gpu/drm/panthor/panthor_gem.h b/drivers/gpu/drm/panthor/panthor_gem.h
index 22519c570b5a9..ae0491d0b1216 100644
--- a/drivers/gpu/drm/panthor/panthor_gem.h
+++ b/drivers/gpu/drm/panthor/panthor_gem.h
@@ -1,11 +1,12 @@
 /* SPDX-License-Identifier: GPL-2.0 or MIT */
 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
 /* Copyright 2023 Collabora ltd. */
+/* Copyright 2025 ARM Limited. All rights reserved. */
 
 #ifndef __PANTHOR_GEM_H__
 #define __PANTHOR_GEM_H__
 
-#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_gem.h>
 #include <drm/drm_mm.h>
 
 #include <linux/iosys-map.h>
@@ -60,12 +61,116 @@ struct panthor_gem_debugfs {
 	u32 flags;
 };
 
+/**
+ * struct panthor_gem_backing - GEM memory backing related data
+ */
+struct panthor_gem_backing {
+	/** @pages: Pages requested with drm_gem_get_pages() */
+	struct page **pages;
+
+	/** @pin_count: Number of active pin requests on this GEM */
+	refcount_t pin_count;
+};
+
+/**
+ * struct panthor_gem_cpu_map - GEM CPU mapping related data
+ */
+struct panthor_gem_cpu_map {
+	/** @vaddr: Address returned by vmap() */
+	void *vaddr;
+
+	/** @vaddr_use_count: Number of active vmap() requests on this GEM */
+	refcount_t vaddr_use_count;
+
+	/** @mmap_count: Number of active mmap() requests on this GEM */
+	refcount_t mmap_count;
+};
+
+/**
+ * struct panthor_gem_dev_map - GEM device mapping related data
+ */
+struct panthor_gem_dev_map {
+	/** @sgt: Device mapped sg_table for this GEM */
+	struct sg_table *sgt;
+};
+
+/**
+ * enum panthor_gem_reclaim_state - Reclaim state of a GEM object
+ *
+ * This is defined in descending reclaimability order and some part
+ * of the code depends on that.
+ */
+enum panthor_gem_reclaim_state {
+	/**
+	 * @PANTHOR_GEM_UNUSED: GEM is currently unused
+	 *
+	 * This can happen when the GEM was previously vmap-ed, mmap-ed,
+	 * and/or GPU mapped and got unmapped. Because pages are lazily
+	 * returned to the shmem layer, we want to keep a list of such
+	 * BOs, because they should be fairly easy to reclaim (no need
+	 * to wait for GPU to be done, and no need to tear down user
+	 * mappings either).
+	 */
+	PANTHOR_GEM_UNUSED,
+
+	/**
+	 * @PANTHOR_GEM_MMAPPED: GEM is currently mmap-ed
+	 *
+	 * When a GEM has pages allocated and the mmap_count is > 0, the
+	 * GEM is placed in the mmapped list. This comes right after
+	 * unused because we can relatively easily tear down user mappings.
+	 */
+	PANTHOR_GEM_MMAPPED,
+
+	/**
+	 * @PANTHOR_GEM_GPU_MAPPED_SINGLE_VM: GEM is GPU mapped to only one VM
+	 *
+	 * When a GEM is mapped to a single VM, reclaim requests have more
+	 * chances to succeed, because we only need to synchronize against
+	 * a single GPU context. This is more annoying than reclaiming
+	 * mmap-ed pages still, because we have to wait for in-flight jobs
+	 * to land, and we might not be able to acquire all necessary locks
+	 * at reclaim time either.
+	 */
+	PANTHOR_GEM_GPU_MAPPED_SINGLE_VM,
+
+	/**
+	 * @PANTHOR_GEM_GPU_MAPPED_MULTI_VM: GEM is GPU mapped to multiple VMs
+	 *
+	 * Like PANTHOR_GEM_GPU_MAPPED_SINGLE_VM, but the synchronization across
+	 * VMs makes such BOs harder to reclaim.
+	 */
+	PANTHOR_GEM_GPU_MAPPED_MULTI_VM,
+
+	/**
+	 * @PANTHOR_GEM_UNRECLAIMABLE: GEM can't be reclaimed
+	 *
+	 * Happens when the GEM memory is pinned. It's also the state all GEM
+	 * objects start in, because no memory is allocated until explicitly
+	 * requested by a CPU or GPU map, meaning there's nothing to reclaim
+	 * until such an allocation happens.
+	 */
+	PANTHOR_GEM_UNRECLAIMABLE,
+};
+
 /**
  * struct panthor_gem_object - Driver specific GEM object.
  */
 struct panthor_gem_object {
-	/** @base: Inherit from drm_gem_shmem_object. */
-	struct drm_gem_shmem_object base;
+	/** @base: Inherit from drm_gem_object. */
+	struct drm_gem_object base;
+
+	/** @backing: Memory backing state */
+	struct panthor_gem_backing backing;
+
+	/** @cmap: CPU mapping state */
+	struct panthor_gem_cpu_map cmap;
+
+	/** @dmap: Device mapping state */
+	struct panthor_gem_dev_map dmap;
+
+	/** @reclaim_state: Cached reclaim state */
+	enum panthor_gem_reclaim_state reclaim_state;
 
 	/**
 	 * @exclusive_vm_root_gem: Root GEM of the exclusive VM this GEM object
@@ -130,22 +235,30 @@ struct panthor_kernel_bo {
 	void *kmap;
 };
 
-static inline
-struct panthor_gem_object *to_panthor_bo(struct drm_gem_object *obj)
-{
-	return container_of(to_drm_gem_shmem_obj(obj), struct panthor_gem_object, base);
-}
+#define to_panthor_bo(obj) container_of_const(obj, struct panthor_gem_object, base)
 
 void panthor_gem_init(struct panthor_device *ptdev);
 
-struct drm_gem_object *panthor_gem_create_object(struct drm_device *ddev, size_t size);
-
+struct drm_gem_object *
+panthor_gem_prime_import_sg_table(struct drm_device *dev,
+				  struct dma_buf_attachment *attach,
+				  struct sg_table *sgt);
 int
 panthor_gem_create_with_handle(struct drm_file *file,
 			       struct drm_device *ddev,
 			       struct panthor_vm *exclusive_vm,
 			       u64 *size, u32 flags, uint32_t *handle);
 
+struct sg_table *
+panthor_gem_get_dev_sgt(struct panthor_gem_object *bo);
+int panthor_gem_pin(struct panthor_gem_object *bo);
+void panthor_gem_unpin(struct panthor_gem_object *bo);
+int panthor_gem_swapin_locked(struct panthor_gem_object *bo);
+void panthor_gem_update_reclaim_state_locked(struct panthor_gem_object *bo,
+					     enum panthor_gem_reclaim_state *old_state);
+int panthor_gem_shrinker_init(struct panthor_device *ptdev);
+void panthor_gem_shrinker_unplug(struct panthor_device *ptdev);
+
 void panthor_gem_bo_set_label(struct drm_gem_object *obj, const char *label);
 void panthor_gem_kernel_bo_set_label(struct panthor_kernel_bo *bo, const char *label);
 int panthor_gem_sync(struct drm_gem_object *obj,
@@ -203,8 +316,7 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm,
 void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo);
 
 #ifdef CONFIG_DEBUG_FS
-void panthor_gem_debugfs_print_bos(struct panthor_device *pfdev,
-				   struct seq_file *m);
+void panthor_gem_debugfs_init(struct drm_minor *minor);
 #endif
 
 #endif /* __PANTHOR_GEM_H__ */
diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c
index 2ab444ee8c710..e52c5675981f5 100644
--- a/drivers/gpu/drm/panthor/panthor_gpu.c
+++ b/drivers/gpu/drm/panthor/panthor_gpu.c
@@ -19,8 +19,8 @@
 
 #include "panthor_device.h"
 #include "panthor_gpu.h"
+#include "panthor_gpu_regs.h"
 #include "panthor_hw.h"
-#include "panthor_regs.h"
 
 #define CREATE_TRACE_POINTS
 #include "panthor_trace.h"
@@ -29,6 +29,9 @@
  * struct panthor_gpu - GPU block management data.
  */
 struct panthor_gpu {
+	/** @iomem: CPU mapping of GPU_CONTROL iomem region */
+	void __iomem *iomem;
+
 	/** @irq: GPU irq. */
 	struct panthor_irq irq;
 
@@ -56,12 +59,13 @@ struct panthor_gpu {
 
 static void panthor_gpu_coherency_set(struct panthor_device *ptdev)
 {
-	gpu_write(ptdev, GPU_COHERENCY_PROTOCOL,
+	gpu_write(ptdev->gpu->iomem, GPU_COHERENCY_PROTOCOL,
 		  ptdev->gpu_info.selected_coherency);
 }
 
 static void panthor_gpu_l2_config_set(struct panthor_device *ptdev)
 {
+	struct panthor_gpu *gpu = ptdev->gpu;
 	const struct panthor_soc_data *data = ptdev->soc_data;
 	u32 l2_config;
 	u32 i;
@@ -75,26 +79,28 @@ static void panthor_gpu_l2_config_set(struct panthor_device *ptdev)
 	}
 
 	for (i = 0; i < ARRAY_SIZE(data->asn_hash); i++)
-		gpu_write(ptdev, GPU_ASN_HASH(i), data->asn_hash[i]);
+		gpu_write(gpu->iomem, GPU_ASN_HASH(i), data->asn_hash[i]);
 
-	l2_config = gpu_read(ptdev, GPU_L2_CONFIG);
+	l2_config = gpu_read(gpu->iomem, GPU_L2_CONFIG);
 	l2_config |= GPU_L2_CONFIG_ASN_HASH_ENABLE;
-	gpu_write(ptdev, GPU_L2_CONFIG, l2_config);
+	gpu_write(gpu->iomem, GPU_L2_CONFIG, l2_config);
 }
 
 static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status)
 {
-	gpu_write(ptdev, GPU_INT_CLEAR, status);
+	struct panthor_gpu *gpu = ptdev->gpu;
+
+	gpu_write(gpu->irq.iomem, INT_CLEAR, status);
 
 	if (tracepoint_enabled(gpu_power_status) && (status & GPU_POWER_INTERRUPTS_MASK))
 		trace_gpu_power_status(ptdev->base.dev,
-				       gpu_read64(ptdev, SHADER_READY),
-				       gpu_read64(ptdev, TILER_READY),
-				       gpu_read64(ptdev, L2_READY));
+				       gpu_read64(gpu->iomem, SHADER_READY),
+				       gpu_read64(gpu->iomem, TILER_READY),
+				       gpu_read64(gpu->iomem, L2_READY));
 
 	if (status & GPU_IRQ_FAULT) {
-		u32 fault_status = gpu_read(ptdev, GPU_FAULT_STATUS);
-		u64 address = gpu_read64(ptdev, GPU_FAULT_ADDR);
+		u32 fault_status = gpu_read(gpu->iomem, GPU_FAULT_STATUS);
+		u64 address = gpu_read64(gpu->iomem, GPU_FAULT_ADDR);
 
 		drm_warn(&ptdev->base, "GPU Fault 0x%08x (%s) at 0x%016llx\n",
 			 fault_status, panthor_exception_name(ptdev, fault_status & 0xFF),
@@ -110,7 +116,7 @@ static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status)
 	}
 	spin_unlock(&ptdev->gpu->reqs_lock);
 }
-PANTHOR_IRQ_HANDLER(gpu, GPU, panthor_gpu_irq_handler);
+PANTHOR_IRQ_HANDLER(gpu, panthor_gpu_irq_handler);
 
 /**
  * panthor_gpu_unplug() - Called when the GPU is unplugged.
@@ -147,6 +153,7 @@ int panthor_gpu_init(struct panthor_device *ptdev)
 	if (!gpu)
 		return -ENOMEM;
 
+	gpu->iomem = ptdev->iomem + GPU_CONTROL_BASE;
 	spin_lock_init(&gpu->reqs_lock);
 	init_waitqueue_head(&gpu->reqs_acked);
 	mutex_init(&gpu->cache_flush_lock);
@@ -162,7 +169,9 @@ int panthor_gpu_init(struct panthor_device *ptdev)
 	if (irq < 0)
 		return irq;
 
-	ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq, GPU_INTERRUPTS_MASK);
+	ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq,
+				      GPU_INTERRUPTS_MASK,
+				      ptdev->iomem + GPU_INT_BASE);
 	if (ret)
 		return ret;
 
@@ -201,10 +210,11 @@ int panthor_gpu_block_power_off(struct panthor_device *ptdev,
 				u32 pwroff_reg, u32 pwrtrans_reg,
 				u64 mask, u32 timeout_us)
 {
+	struct panthor_gpu *gpu = ptdev->gpu;
 	u32 val;
 	int ret;
 
-	ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val,
+	ret = gpu_read64_relaxed_poll_timeout(gpu->iomem, pwrtrans_reg, val,
 					      !(mask & val), 100, timeout_us);
 	if (ret) {
 		drm_err(&ptdev->base,
@@ -213,9 +223,9 @@ int panthor_gpu_block_power_off(struct panthor_device *ptdev,
 		return ret;
 	}
 
-	gpu_write64(ptdev, pwroff_reg, mask);
+	gpu_write64(gpu->iomem, pwroff_reg, mask);
 
-	ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val,
+	ret = gpu_read64_relaxed_poll_timeout(gpu->iomem, pwrtrans_reg, val,
 					      !(mask & val), 100, timeout_us);
 	if (ret) {
 		drm_err(&ptdev->base,
@@ -244,10 +254,11 @@ int panthor_gpu_block_power_on(struct panthor_device *ptdev,
 			       u32 pwron_reg, u32 pwrtrans_reg,
 			       u32 rdy_reg, u64 mask, u32 timeout_us)
 {
+	struct panthor_gpu *gpu = ptdev->gpu;
 	u32 val;
 	int ret;
 
-	ret = gpu_read64_relaxed_poll_timeout(ptdev, pwrtrans_reg, val,
+	ret = gpu_read64_relaxed_poll_timeout(gpu->iomem, pwrtrans_reg, val,
 					      !(mask & val), 100, timeout_us);
 	if (ret) {
 		drm_err(&ptdev->base,
@@ -256,9 +267,9 @@ int panthor_gpu_block_power_on(struct panthor_device *ptdev,
 		return ret;
 	}
 
-	gpu_write64(ptdev, pwron_reg, mask);
+	gpu_write64(gpu->iomem, pwron_reg, mask);
 
-	ret = gpu_read64_relaxed_poll_timeout(ptdev, rdy_reg, val,
+	ret = gpu_read64_relaxed_poll_timeout(gpu->iomem, rdy_reg, val,
 					      (mask & val) == val,
 					      100, timeout_us);
 	if (ret) {
@@ -317,6 +328,7 @@ int panthor_gpu_l2_power_on(struct panthor_device *ptdev)
 int panthor_gpu_flush_caches(struct panthor_device *ptdev,
 			     u32 l2, u32 lsc, u32 other)
 {
+	struct panthor_gpu *gpu = ptdev->gpu;
 	unsigned long flags;
 	int ret = 0;
 
@@ -326,7 +338,7 @@ int panthor_gpu_flush_caches(struct panthor_device *ptdev,
 	spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
 	if (!(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) {
 		ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED;
-		gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other));
+		gpu_write(gpu->iomem, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other));
 	} else {
 		ret = -EIO;
 	}
@@ -340,7 +352,7 @@ int panthor_gpu_flush_caches(struct panthor_device *ptdev,
 				msecs_to_jiffies(100))) {
 		spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
 		if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 &&
-		    !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED))
+		    !(gpu_read(gpu->irq.iomem, INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED))
 			ret = -ETIMEDOUT;
 		else
 			ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED;
@@ -363,6 +375,7 @@ int panthor_gpu_flush_caches(struct panthor_device *ptdev,
  */
 int panthor_gpu_soft_reset(struct panthor_device *ptdev)
 {
+	struct panthor_gpu *gpu = ptdev->gpu;
 	bool timedout = false;
 	unsigned long flags;
 
@@ -370,8 +383,8 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev)
 	if (!drm_WARN_ON(&ptdev->base,
 			 ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) {
 		ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED;
-		gpu_write(ptdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
-		gpu_write(ptdev, GPU_CMD, GPU_SOFT_RESET);
+		gpu_write(gpu->irq.iomem, INT_CLEAR, GPU_IRQ_RESET_COMPLETED);
+		gpu_write(gpu->iomem, GPU_CMD, GPU_SOFT_RESET);
 	}
 	spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags);
 
@@ -380,7 +393,7 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev)
 				msecs_to_jiffies(100))) {
 		spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags);
 		if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 &&
-		    !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED))
+		    !(gpu_read(gpu->irq.iomem, INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED))
 			timedout = true;
 		else
 			ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED;
@@ -427,3 +440,43 @@ void panthor_gpu_resume(struct panthor_device *ptdev)
 	panthor_hw_l2_power_on(ptdev);
 }
 
+u64 panthor_gpu_get_timestamp(struct panthor_device *ptdev)
+{
+	return gpu_read64_counter(ptdev->gpu->iomem, GPU_TIMESTAMP);
+}
+
+u64 panthor_gpu_get_timestamp_offset(struct panthor_device *ptdev)
+{
+	return gpu_read64(ptdev->gpu->iomem, GPU_TIMESTAMP_OFFSET);
+}
+
+u64 panthor_gpu_get_cycle_count(struct panthor_device *ptdev)
+{
+	return gpu_read64_counter(ptdev->gpu->iomem, GPU_CYCLE_COUNT);
+}
+
+int panthor_gpu_coherency_init(struct panthor_device *ptdev)
+{
+	BUILD_BUG_ON(GPU_COHERENCY_NONE != DRM_PANTHOR_GPU_COHERENCY_NONE);
+	BUILD_BUG_ON(GPU_COHERENCY_ACE_LITE != DRM_PANTHOR_GPU_COHERENCY_ACE_LITE);
+	BUILD_BUG_ON(GPU_COHERENCY_ACE != DRM_PANTHOR_GPU_COHERENCY_ACE);
+
+	/* Start with no coherency, and update it if the device is flagged coherent. */
+	ptdev->gpu_info.selected_coherency = GPU_COHERENCY_NONE;
+	ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT;
+
+	if (!ptdev->coherent)
+		return 0;
+
+	/* Check if the ACE-Lite coherency protocol is actually supported by the GPU.
+	 * ACE protocol has never been supported for command stream frontend GPUs.
+	 */
+	if ((gpu_read(ptdev->gpu->iomem, GPU_COHERENCY_FEATURES) &
+		      GPU_COHERENCY_PROT_BIT(ACE_LITE))) {
+		ptdev->gpu_info.selected_coherency = GPU_COHERENCY_ACE_LITE;
+		return 0;
+	}
+
+	drm_err(&ptdev->base, "Coherency not supported by the device");
+	return -ENOTSUPP;
+}
diff --git a/drivers/gpu/drm/panthor/panthor_gpu.h b/drivers/gpu/drm/panthor/panthor_gpu.h
index 12c263a399281..f615feb056094 100644
--- a/drivers/gpu/drm/panthor/panthor_gpu.h
+++ b/drivers/gpu/drm/panthor/panthor_gpu.h
@@ -54,4 +54,10 @@ int panthor_gpu_soft_reset(struct panthor_device *ptdev);
 void panthor_gpu_power_changed_off(struct panthor_device *ptdev);
 int panthor_gpu_power_changed_on(struct panthor_device *ptdev);
 
+u64 panthor_gpu_get_timestamp(struct panthor_device *ptdev);
+u64 panthor_gpu_get_timestamp_offset(struct panthor_device *ptdev);
+u64 panthor_gpu_get_cycle_count(struct panthor_device *ptdev);
+
+int panthor_gpu_coherency_init(struct panthor_device *ptdev);
+
 #endif
diff --git a/drivers/gpu/drm/panthor/panthor_gpu_regs.h b/drivers/gpu/drm/panthor/panthor_gpu_regs.h
new file mode 100644
index 0000000000000..4c5b953796e4f
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_gpu_regs.h
@@ -0,0 +1,119 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2026 ARM Limited. All rights reserved. */
+
+#ifndef __PANTHOR_GPU_REGS_H__
+#define __PANTHOR_GPU_REGS_H__
+
+#define GPU_CONTROL_BASE				0x0
+
+#define GPU_ID						0x0
+#define   GPU_ARCH_MAJOR(x)				((x) >> 28)
+#define   GPU_ARCH_MINOR(x)				(((x) & GENMASK(27, 24)) >> 24)
+#define   GPU_ARCH_REV(x)				(((x) & GENMASK(23, 20)) >> 20)
+#define   GPU_PROD_MAJOR(x)				(((x) & GENMASK(19, 16)) >> 16)
+#define   GPU_VER_MAJOR(x)				(((x) & GENMASK(15, 12)) >> 12)
+#define   GPU_VER_MINOR(x)				(((x) & GENMASK(11, 4)) >> 4)
+#define   GPU_VER_STATUS(x)				((x) & GENMASK(3, 0))
+
+#define GPU_L2_FEATURES					0x4
+#define  GPU_L2_FEATURES_LINE_SIZE(x)			(1 << ((x) & GENMASK(7, 0)))
+
+#define GPU_CORE_FEATURES				0x8
+
+#define GPU_TILER_FEATURES				0xC
+#define GPU_MEM_FEATURES				0x10
+#define   GROUPS_L2_COHERENT				BIT(0)
+
+#define GPU_MMU_FEATURES				0x14
+#define  GPU_MMU_FEATURES_VA_BITS(x)			((x) & GENMASK(7, 0))
+#define  GPU_MMU_FEATURES_PA_BITS(x)			(((x) >> 8) & GENMASK(7, 0))
+#define GPU_AS_PRESENT					0x18
+#define GPU_CSF_ID					0x1C
+
+#define GPU_INT_BASE					0x20
+#define   GPU_IRQ_FAULT					BIT(0)
+#define   GPU_IRQ_PROTM_FAULT				BIT(1)
+#define   GPU_IRQ_RESET_COMPLETED			BIT(8)
+#define   GPU_IRQ_POWER_CHANGED				BIT(9)
+#define   GPU_IRQ_POWER_CHANGED_ALL			BIT(10)
+#define   GPU_IRQ_CLEAN_CACHES_COMPLETED		BIT(17)
+#define   GPU_IRQ_DOORBELL_MIRROR			BIT(18)
+#define   GPU_IRQ_MCU_STATUS_CHANGED			BIT(19)
+#define GPU_CMD						0x30
+#define   GPU_CMD_DEF(type, payload)			((type) | ((payload) << 8))
+#define   GPU_SOFT_RESET				GPU_CMD_DEF(1, 1)
+#define   GPU_HARD_RESET				GPU_CMD_DEF(1, 2)
+#define   CACHE_CLEAN					BIT(0)
+#define   CACHE_INV					BIT(1)
+#define   GPU_FLUSH_CACHES(l2, lsc, oth)		\
+	  GPU_CMD_DEF(4, ((l2) << 0) | ((lsc) << 4) | ((oth) << 8))
+
+#define GPU_STATUS					0x34
+#define   GPU_STATUS_ACTIVE				BIT(0)
+#define   GPU_STATUS_PWR_ACTIVE				BIT(1)
+#define   GPU_STATUS_PAGE_FAULT				BIT(4)
+#define   GPU_STATUS_PROTM_ACTIVE			BIT(7)
+#define   GPU_STATUS_DBG_ENABLED			BIT(8)
+
+#define GPU_FAULT_STATUS				0x3C
+#define GPU_FAULT_ADDR					0x40
+#define GPU_L2_CONFIG					0x48
+#define   GPU_L2_CONFIG_ASN_HASH_ENABLE			BIT(24)
+
+#define GPU_PWR_KEY					0x50
+#define  GPU_PWR_KEY_UNLOCK				0x2968A819
+#define GPU_PWR_OVERRIDE0				0x54
+#define GPU_PWR_OVERRIDE1				0x58
+
+#define GPU_FEATURES					0x60
+#define   GPU_FEATURES_RAY_INTERSECTION			BIT(2)
+#define   GPU_FEATURES_RAY_TRAVERSAL			BIT(5)
+
+#define GPU_TIMESTAMP_OFFSET				0x88
+#define GPU_CYCLE_COUNT					0x90
+#define GPU_TIMESTAMP					0x98
+
+#define GPU_THREAD_MAX_THREADS				0xA0
+#define GPU_THREAD_MAX_WORKGROUP_SIZE			0xA4
+#define GPU_THREAD_MAX_BARRIER_SIZE			0xA8
+#define GPU_THREAD_FEATURES				0xAC
+
+#define GPU_TEXTURE_FEATURES(n)				(0xB0 + ((n) * 4))
+
+#define GPU_SHADER_PRESENT				0x100
+#define GPU_TILER_PRESENT				0x110
+#define GPU_L2_PRESENT					0x120
+
+#define SHADER_READY					0x140
+#define TILER_READY					0x150
+#define L2_READY					0x160
+
+#define SHADER_PWRON					0x180
+#define TILER_PWRON					0x190
+#define L2_PWRON					0x1A0
+
+#define SHADER_PWROFF					0x1C0
+#define TILER_PWROFF					0x1D0
+#define L2_PWROFF					0x1E0
+
+#define SHADER_PWRTRANS					0x200
+#define TILER_PWRTRANS					0x210
+#define L2_PWRTRANS					0x220
+
+#define SHADER_PWRACTIVE				0x240
+#define TILER_PWRACTIVE					0x250
+#define L2_PWRACTIVE					0x260
+
+#define GPU_REVID					0x280
+
+#define GPU_ASN_HASH(n)					(0x2C0 + ((n) * 4))
+
+#define GPU_COHERENCY_FEATURES				0x300
+#define GPU_COHERENCY_PROT_BIT(name)			BIT(GPU_COHERENCY_  ## name)
+
+#define GPU_COHERENCY_PROTOCOL				0x304
+#define   GPU_COHERENCY_ACE_LITE			0
+#define   GPU_COHERENCY_ACE				1
+#define   GPU_COHERENCY_NONE				31
+
+#endif /* __PANTHOR_GPU_REGS_H__ */
diff --git a/drivers/gpu/drm/panthor/panthor_heap.c b/drivers/gpu/drm/panthor/panthor_heap.c
index 1ee30dc7066f7..99311abdf1e91 100644
--- a/drivers/gpu/drm/panthor/panthor_heap.c
+++ b/drivers/gpu/drm/panthor/panthor_heap.c
@@ -9,9 +9,9 @@
 
 #include "panthor_device.h"
 #include "panthor_gem.h"
+#include "panthor_gpu_regs.h"
 #include "panthor_heap.h"
 #include "panthor_mmu.h"
-#include "panthor_regs.h"
 
 /*
  * The GPU heap context is an opaque structure used by the GPU to track the
diff --git a/drivers/gpu/drm/panthor/panthor_hw.c b/drivers/gpu/drm/panthor/panthor_hw.c
index d135aa6724fa5..7e315708ca7ca 100644
--- a/drivers/gpu/drm/panthor/panthor_hw.c
+++ b/drivers/gpu/drm/panthor/panthor_hw.c
@@ -8,9 +8,10 @@
 
 #include "panthor_device.h"
 #include "panthor_gpu.h"
+#include "panthor_gpu_regs.h"
 #include "panthor_hw.h"
 #include "panthor_pwr.h"
-#include "panthor_regs.h"
+#include "panthor_pwr_regs.h"
 
 #define GPU_PROD_ID_MAKE(arch_major, prod_major) \
 	(((arch_major) << 24) | (prod_major))
@@ -194,35 +195,42 @@ static int panthor_gpu_info_init(struct panthor_device *ptdev)
 {
 	unsigned int i;
 
-	ptdev->gpu_info.csf_id = gpu_read(ptdev, GPU_CSF_ID);
-	ptdev->gpu_info.gpu_rev = gpu_read(ptdev, GPU_REVID);
-	ptdev->gpu_info.core_features = gpu_read(ptdev, GPU_CORE_FEATURES);
-	ptdev->gpu_info.l2_features = gpu_read(ptdev, GPU_L2_FEATURES);
-	ptdev->gpu_info.tiler_features = gpu_read(ptdev, GPU_TILER_FEATURES);
-	ptdev->gpu_info.mem_features = gpu_read(ptdev, GPU_MEM_FEATURES);
-	ptdev->gpu_info.mmu_features = gpu_read(ptdev, GPU_MMU_FEATURES);
-	ptdev->gpu_info.thread_features = gpu_read(ptdev, GPU_THREAD_FEATURES);
-	ptdev->gpu_info.max_threads = gpu_read(ptdev, GPU_THREAD_MAX_THREADS);
-	ptdev->gpu_info.thread_max_workgroup_size = gpu_read(ptdev, GPU_THREAD_MAX_WORKGROUP_SIZE);
-	ptdev->gpu_info.thread_max_barrier_size = gpu_read(ptdev, GPU_THREAD_MAX_BARRIER_SIZE);
-	ptdev->gpu_info.coherency_features = gpu_read(ptdev, GPU_COHERENCY_FEATURES);
+	void __iomem *gpu_iomem = ptdev->iomem + GPU_CONTROL_BASE;
+
+	ptdev->gpu_info.csf_id = gpu_read(gpu_iomem, GPU_CSF_ID);
+	ptdev->gpu_info.gpu_rev = gpu_read(gpu_iomem, GPU_REVID);
+	ptdev->gpu_info.core_features = gpu_read(gpu_iomem, GPU_CORE_FEATURES);
+	ptdev->gpu_info.l2_features = gpu_read(gpu_iomem, GPU_L2_FEATURES);
+	ptdev->gpu_info.tiler_features = gpu_read(gpu_iomem, GPU_TILER_FEATURES);
+	ptdev->gpu_info.mem_features = gpu_read(gpu_iomem, GPU_MEM_FEATURES);
+	ptdev->gpu_info.mmu_features = gpu_read(gpu_iomem, GPU_MMU_FEATURES);
+	ptdev->gpu_info.thread_features = gpu_read(gpu_iomem, GPU_THREAD_FEATURES);
+	ptdev->gpu_info.max_threads = gpu_read(gpu_iomem, GPU_THREAD_MAX_THREADS);
+	ptdev->gpu_info.thread_max_workgroup_size =
+		gpu_read(gpu_iomem, GPU_THREAD_MAX_WORKGROUP_SIZE);
+	ptdev->gpu_info.thread_max_barrier_size =
+		gpu_read(gpu_iomem, GPU_THREAD_MAX_BARRIER_SIZE);
+	ptdev->gpu_info.coherency_features = gpu_read(gpu_iomem, GPU_COHERENCY_FEATURES);
 	for (i = 0; i < 4; i++)
-		ptdev->gpu_info.texture_features[i] = gpu_read(ptdev, GPU_TEXTURE_FEATURES(i));
+		ptdev->gpu_info.texture_features[i] =
+			gpu_read(gpu_iomem, GPU_TEXTURE_FEATURES(i));
 
-	ptdev->gpu_info.as_present = gpu_read(ptdev, GPU_AS_PRESENT);
+	ptdev->gpu_info.as_present = gpu_read(gpu_iomem, GPU_AS_PRESENT);
 
 	/* Introduced in arch 11.x */
-	ptdev->gpu_info.gpu_features = gpu_read64(ptdev, GPU_FEATURES);
+	ptdev->gpu_info.gpu_features = gpu_read64(gpu_iomem, GPU_FEATURES);
 
 	if (panthor_hw_has_pwr_ctrl(ptdev)) {
+		void __iomem *pwr_iomem = gpu_iomem + PWR_CONTROL_BASE;
+
 		/* Introduced in arch 14.x */
-		ptdev->gpu_info.l2_present = gpu_read64(ptdev, PWR_L2_PRESENT);
-		ptdev->gpu_info.tiler_present = gpu_read64(ptdev, PWR_TILER_PRESENT);
-		ptdev->gpu_info.shader_present = gpu_read64(ptdev, PWR_SHADER_PRESENT);
+		ptdev->gpu_info.l2_present = gpu_read64(pwr_iomem, PWR_L2_PRESENT);
+		ptdev->gpu_info.tiler_present = gpu_read64(pwr_iomem, PWR_TILER_PRESENT);
+		ptdev->gpu_info.shader_present = gpu_read64(pwr_iomem, PWR_SHADER_PRESENT);
 	} else {
-		ptdev->gpu_info.shader_present = gpu_read64(ptdev, GPU_SHADER_PRESENT);
-		ptdev->gpu_info.tiler_present = gpu_read64(ptdev, GPU_TILER_PRESENT);
-		ptdev->gpu_info.l2_present = gpu_read64(ptdev, GPU_L2_PRESENT);
+		ptdev->gpu_info.shader_present = gpu_read64(gpu_iomem, GPU_SHADER_PRESENT);
+		ptdev->gpu_info.tiler_present = gpu_read64(gpu_iomem, GPU_TILER_PRESENT);
+		ptdev->gpu_info.l2_present = gpu_read64(gpu_iomem, GPU_L2_PRESENT);
 	}
 
 	return overload_shader_present(ptdev);
@@ -287,7 +295,7 @@ static int panthor_hw_bind_device(struct panthor_device *ptdev)
 
 static int panthor_hw_gpu_id_init(struct panthor_device *ptdev)
 {
-	ptdev->gpu_info.gpu_id = gpu_read(ptdev, GPU_ID);
+	ptdev->gpu_info.gpu_id = gpu_read(ptdev->iomem, GPU_ID);
 	if (!ptdev->gpu_info.gpu_id)
 		return -ENXIO;
 
diff --git a/drivers/gpu/drm/panthor/panthor_hw.h b/drivers/gpu/drm/panthor/panthor_hw.h
index 2c28aea828418..f797663893b2d 100644
--- a/drivers/gpu/drm/panthor/panthor_hw.h
+++ b/drivers/gpu/drm/panthor/panthor_hw.h
@@ -5,7 +5,7 @@
 #define __PANTHOR_HW_H__
 
 #include "panthor_device.h"
-#include "panthor_regs.h"
+#include "panthor_gpu_regs.h"
 
 /**
  * struct panthor_hw_ops - HW operations that are specific to a GPU
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
index 75d98dad7b1dd..9d45008505619 100644
--- a/drivers/gpu/drm/panthor/panthor_mmu.c
+++ b/drivers/gpu/drm/panthor/panthor_mmu.c
@@ -1,10 +1,12 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
 /* Copyright 2023 Collabora ltd. */
+/* Copyright 2025 ARM Limited. All rights reserved. */
 
 #include <drm/drm_debugfs.h>
 #include <drm/drm_drv.h>
 #include <drm/drm_exec.h>
+#include <drm/drm_file.h>
 #include <drm/drm_gpuvm.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
@@ -31,9 +33,10 @@
 #include "panthor_device.h"
 #include "panthor_gem.h"
 #include "panthor_gpu.h"
+#include "panthor_gpu_regs.h"
 #include "panthor_heap.h"
 #include "panthor_mmu.h"
-#include "panthor_regs.h"
+#include "panthor_mmu_regs.h"
 #include "panthor_sched.h"
 
 #define MAX_AS_SLOTS			32
@@ -52,6 +55,9 @@ struct panthor_as_slot {
  * struct panthor_mmu - MMU related data
  */
 struct panthor_mmu {
+	/** @iomem: CPU mapping of MMU_AS_CONTROL iomem region */
+	void __iomem *iomem;
+
 	/** @irq: The MMU irq. */
 	struct panthor_irq irq;
 
@@ -130,6 +136,9 @@ struct panthor_vma {
 	 * Only map related flags are accepted.
 	 */
 	u32 flags;
+
+	/** @evicted: True if the VMA has been evicted. */
+	bool evicted;
 };
 
 /**
@@ -198,10 +207,8 @@ struct panthor_vm_op_ctx {
 		 */
 		struct sg_table *sgt;
 
-		/**
-		 * @map.new_vma: The new VMA object that will be inserted to the VA tree.
-		 */
-		struct panthor_vma *new_vma;
+		/** @map.bo: the BO being mapped. */
+		struct panthor_gem_object *bo;
 	} map;
 };
 
@@ -384,6 +391,18 @@ struct panthor_vm {
 		/** @locked_region.size: Size of the locked region. */
 		u64 size;
 	} locked_region;
+
+	/** @reclaim: Fields related to BO reclaim. */
+	struct {
+		/** @reclaim.lru: LRU of BOs that are only mapped to this VM. */
+		struct drm_gem_lru lru;
+
+		/**
+		 * @reclaim.lru_node: Node used to insert the VM in
+		 * panthor_device::reclaim::vms.
+		 */
+		struct list_head lru_node;
+	} reclaim;
 };
 
 /**
@@ -501,15 +520,15 @@ static void free_pt(void *cookie, void *data, size_t size)
 
 static int wait_ready(struct panthor_device *ptdev, u32 as_nr)
 {
+	struct panthor_mmu *mmu = ptdev->mmu;
 	int ret;
 	u32 val;
 
 	/* Wait for the MMU status to indicate there is no active command, in
 	 * case one is pending.
 	 */
-	ret = gpu_read_relaxed_poll_timeout_atomic(ptdev, AS_STATUS(as_nr), val,
-						   !(val & AS_STATUS_AS_ACTIVE),
-						   10, 100000);
+	ret = gpu_read_relaxed_poll_timeout_atomic(mmu->iomem, AS_STATUS(as_nr), val,
+						   !(val & AS_STATUS_AS_ACTIVE), 10, 100000);
 
 	if (ret) {
 		panthor_device_schedule_reset(ptdev);
@@ -526,7 +545,7 @@ static int as_send_cmd_and_wait(struct panthor_device *ptdev, u32 as_nr, u32 cmd
 	/* write AS_COMMAND when MMU is ready to accept another command */
 	status = wait_ready(ptdev, as_nr);
 	if (!status) {
-		gpu_write(ptdev, AS_COMMAND(as_nr), cmd);
+		gpu_write(ptdev->mmu->iomem, AS_COMMAND(as_nr), cmd);
 		status = wait_ready(ptdev, as_nr);
 	}
 
@@ -569,17 +588,19 @@ static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
 
 /* Forward declaration to call helpers within as_enable/disable */
 static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status);
-PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
+PANTHOR_IRQ_HANDLER(mmu, panthor_mmu_irq_handler);
 
 static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
 				 u64 transtab, u64 transcfg, u64 memattr)
 {
+	struct panthor_mmu *mmu = ptdev->mmu;
+
 	panthor_mmu_irq_enable_events(&ptdev->mmu->irq,
 				      panthor_mmu_as_fault_mask(ptdev, as_nr));
 
-	gpu_write64(ptdev, AS_TRANSTAB(as_nr), transtab);
-	gpu_write64(ptdev, AS_MEMATTR(as_nr), memattr);
-	gpu_write64(ptdev, AS_TRANSCFG(as_nr), transcfg);
+	gpu_write64(mmu->iomem, AS_TRANSTAB(as_nr), transtab);
+	gpu_write64(mmu->iomem, AS_MEMATTR(as_nr), memattr);
+	gpu_write64(mmu->iomem, AS_TRANSCFG(as_nr), transcfg);
 
 	return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE);
 }
@@ -587,6 +608,7 @@ static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
 static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr,
 				  bool recycle_slot)
 {
+	struct panthor_mmu *mmu = ptdev->mmu;
 	struct panthor_vm *vm = ptdev->mmu->as.slots[as_nr].vm;
 	int ret;
 
@@ -614,9 +636,9 @@ static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr,
 	if (recycle_slot)
 		return 0;
 
-	gpu_write64(ptdev, AS_TRANSTAB(as_nr), 0);
-	gpu_write64(ptdev, AS_MEMATTR(as_nr), 0);
-	gpu_write64(ptdev, AS_TRANSCFG(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);
+	gpu_write64(mmu->iomem, AS_TRANSTAB(as_nr), 0);
+	gpu_write64(mmu->iomem, AS_MEMATTR(as_nr), 0);
+	gpu_write64(mmu->iomem, AS_TRANSCFG(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);
 
 	return as_send_cmd_and_wait(ptdev, as_nr, AS_COMMAND_UPDATE);
 }
@@ -688,6 +710,16 @@ int panthor_vm_active(struct panthor_vm *vm)
 	if (refcount_inc_not_zero(&vm->as.active_cnt))
 		goto out_dev_exit;
 
+	/* As soon as active is called, we place the VM at the end of the VM LRU.
+	 * If something fails after that, the only downside is that this VM that
+	 * never became active in the first place will be reclaimed last, but
+	 * that's an acceptable trade-off.
+	 */
+	mutex_lock(&ptdev->base.gem_lru_mutex);
+	if (vm->reclaim.lru.count)
+		list_move_tail(&vm->reclaim.lru_node, &ptdev->reclaim.vms);
+	mutex_unlock(&ptdev->base.gem_lru_mutex);
+
 	/* Make sure we don't race with lock/unlock_region() calls
 	 * happening around VM bind operations.
 	 */
@@ -759,7 +791,7 @@ int panthor_vm_active(struct panthor_vm *vm)
 	 */
 	fault_mask = panthor_mmu_as_fault_mask(ptdev, as);
 	if (ptdev->mmu->as.faulty_mask & fault_mask) {
-		gpu_write(ptdev, MMU_INT_CLEAR, fault_mask);
+		gpu_write(ptdev->mmu->irq.iomem, INT_CLEAR, fault_mask);
 		ptdev->mmu->as.faulty_mask &= ~fault_mask;
 	}
 
@@ -1083,8 +1115,15 @@ static void panthor_vm_bo_free(struct drm_gpuvm_bo *vm_bo)
 {
 	struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
 
-	if (!drm_gem_is_imported(&bo->base.base))
-		drm_gem_shmem_unpin(&bo->base);
+	/* We couldn't call this when we unlinked, because the resv lock can't
+	 * be taken in the dma signalling path, so call it now.
+	 */
+	dma_resv_lock(bo->base.resv, NULL);
+	mutex_lock(&bo->base.gpuva.lock);
+	panthor_gem_update_reclaim_state_locked(bo, NULL);
+	mutex_unlock(&bo->base.gpuva.lock);
+	dma_resv_unlock(bo->base.resv);
+
 	kfree(vm_bo);
 }
 
@@ -1093,6 +1132,16 @@ static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
 {
 	u32 remaining_pt_count = op_ctx->rsvd_page_tables.count -
 				 op_ctx->rsvd_page_tables.ptr;
+	u32 op_type = op_ctx->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK;
+
+	/* If this is a map operation and no BO is attached, we're being called
+	 * from vm_bo_validate() and we can't acquire the VM lock because it's
+	 * already held. In that case, we just skip the deferred vm_bo cleanup,
+	 * which is fine, because the vm_bo validation is not calling
+	 * drm_gpuvm_bo_put_deferred().
+	 */
+	bool skip_deferred_cleanup = op_type == DRM_PANTHOR_VM_BIND_OP_TYPE_MAP &&
+				     !op_ctx->map.bo;
 
 	if (remaining_pt_count) {
 		kmem_cache_free_bulk(pt_cache, remaining_pt_count,
@@ -1105,10 +1154,16 @@ static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
 	if (op_ctx->map.vm_bo)
 		drm_gpuvm_bo_put_deferred(op_ctx->map.vm_bo);
 
+	if (op_ctx->map.bo) {
+		panthor_gem_unpin(op_ctx->map.bo);
+		drm_gem_object_put(&op_ctx->map.bo->base);
+	}
+
 	for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++)
 		kfree(op_ctx->preallocated_vmas[i]);
 
-	drm_gpuvm_bo_deferred_cleanup(&vm->base);
+	if (!skip_deferred_cleanup)
+		drm_gpuvm_bo_deferred_cleanup(&vm->base);
 }
 
 static void
@@ -1180,6 +1235,44 @@ panthor_vm_op_ctx_prealloc_vmas(struct panthor_vm_op_ctx *op_ctx)
 	return 0;
 }
 
+static void panthor_vm_init_op_ctx(struct panthor_vm_op_ctx *op_ctx,
+				   u64 size, u64 va, u32 flags)
+{
+	memset(op_ctx, 0, sizeof(*op_ctx));
+	op_ctx->flags = flags;
+	op_ctx->va.range = size;
+	op_ctx->va.addr = va;
+}
+
+static int panthor_vm_op_ctx_prealloc_pts(struct panthor_vm_op_ctx *op_ctx)
+{
+	u64 size = op_ctx->va.range;
+	u64 va = op_ctx->va.addr;
+	int ret;
+
+	/* L1, L2 and L3 page tables.
+	 * We could optimize L3 allocation by iterating over the sgt and merging
+	 * 2M contiguous blocks, but it's simpler to over-provision and return
+	 * the pages if they're not used.
+	 */
+	u64 pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) +
+		       ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) +
+		       ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21);
+
+	op_ctx->rsvd_page_tables.pages = kzalloc_objs(*op_ctx->rsvd_page_tables.pages,
+						      pt_count);
+	if (!op_ctx->rsvd_page_tables.pages)
+		return -ENOMEM;
+
+	ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
+				    op_ctx->rsvd_page_tables.pages);
+	op_ctx->rsvd_page_tables.count = ret;
+	if (ret != pt_count)
+		return -ENOMEM;
+
+	return 0;
+}
+
 #define PANTHOR_VM_BIND_OP_MAP_FLAGS \
 	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
 	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
@@ -1195,7 +1288,6 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
 {
 	struct drm_gpuvm_bo *preallocated_vm_bo;
 	struct sg_table *sgt = NULL;
-	u64 pt_count;
 	int ret;
 
 	if (!bo)
@@ -1206,7 +1298,7 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
 		return -EINVAL;
 
 	/* Make sure the VA and size are in-bounds. */
-	if (size > bo->base.base.size || offset > bo->base.base.size - size)
+	if (size > bo->base.size || offset > bo->base.size - size)
 		return -EINVAL;
 
 	/* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */
@@ -1214,78 +1306,54 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
 	    bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm))
 		return -EINVAL;
 
-	memset(op_ctx, 0, sizeof(*op_ctx));
-	op_ctx->flags = flags;
-	op_ctx->va.range = size;
-	op_ctx->va.addr = va;
+	panthor_vm_init_op_ctx(op_ctx, size, va, flags);
 
 	ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);
 	if (ret)
 		goto err_cleanup;
 
-	if (!drm_gem_is_imported(&bo->base.base)) {
-		/* Pre-reserve the BO pages, so the map operation doesn't have to
-		 * allocate. This pin is dropped in panthor_vm_bo_free(), so
-		 * once we have successfully called drm_gpuvm_bo_create(),
-		 * GPUVM will take care of dropping the pin for us.
-		 */
-		ret = drm_gem_shmem_pin(&bo->base);
-		if (ret)
-			goto err_cleanup;
-	}
+	/* Pre-reserve the BO pages, so the map operation doesn't have to
+	 * allocate.
+	 */
+	ret = panthor_gem_pin(bo);
+	if (ret)
+		goto err_cleanup;
 
-	sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
-	if (IS_ERR(sgt)) {
-		if (!drm_gem_is_imported(&bo->base.base))
-			drm_gem_shmem_unpin(&bo->base);
+	drm_gem_object_get(&bo->base);
+	op_ctx->map.bo = bo;
 
+	sgt = panthor_gem_get_dev_sgt(bo);
+	if (IS_ERR(sgt)) {
 		ret = PTR_ERR(sgt);
 		goto err_cleanup;
 	}
 
-	op_ctx->map.sgt = sgt;
-
-	preallocated_vm_bo = drm_gpuvm_bo_create(&vm->base, &bo->base.base);
+	preallocated_vm_bo = drm_gpuvm_bo_create(&vm->base, &bo->base);
 	if (!preallocated_vm_bo) {
-		if (!drm_gem_is_imported(&bo->base.base))
-			drm_gem_shmem_unpin(&bo->base);
-
 		ret = -ENOMEM;
 		goto err_cleanup;
 	}
 
 	op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo);
-
 	op_ctx->map.bo_offset = offset;
 
-	/* L1, L2 and L3 page tables.
-	 * We could optimize L3 allocation by iterating over the sgt and merging
-	 * 2M contiguous blocks, but it's simpler to over-provision and return
-	 * the pages if they're not used.
-	 */
-	pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) +
-		   ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) +
-		   ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21);
-
-	op_ctx->rsvd_page_tables.pages = kzalloc_objs(*op_ctx->rsvd_page_tables.pages,
-						      pt_count);
-	if (!op_ctx->rsvd_page_tables.pages) {
-		ret = -ENOMEM;
+	ret = panthor_vm_op_ctx_prealloc_pts(op_ctx);
+	if (ret)
 		goto err_cleanup;
-	}
 
-	ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
-				    op_ctx->rsvd_page_tables.pages);
-	op_ctx->rsvd_page_tables.count = ret;
-	if (ret != pt_count) {
-		ret = -ENOMEM;
-		goto err_cleanup;
+	/* Insert BO into the extobj list last, when we know nothing can fail. */
+	if (bo->base.resv != panthor_vm_resv(vm)) {
+		dma_resv_lock(panthor_vm_resv(vm), NULL);
+		drm_gpuvm_bo_extobj_add(op_ctx->map.vm_bo);
+		dma_resv_unlock(panthor_vm_resv(vm));
 	}
 
-	/* Insert BO into the extobj list last, when we know nothing can fail. */
-	dma_resv_lock(panthor_vm_resv(vm), NULL);
-	drm_gpuvm_bo_extobj_add(op_ctx->map.vm_bo);
-	dma_resv_unlock(panthor_vm_resv(vm));
+	/* And finally update the BO state. */
+	dma_resv_lock(bo->base.resv, NULL);
+	mutex_lock(&bo->base.gpuva.lock);
+	panthor_gem_update_reclaim_state_locked(bo, NULL);
+	mutex_unlock(&bo->base.gpuva.lock);
+	dma_resv_unlock(bo->base.resv);
 
 	return 0;
 
@@ -1670,7 +1738,7 @@ static int panthor_vm_lock_region(struct panthor_vm *vm, u64 start, u64 size)
 	mutex_lock(&ptdev->mmu->as.slots_lock);
 	if (vm->as.id >= 0 && size) {
 		/* Lock the region that needs to be updated */
-		gpu_write64(ptdev, AS_LOCKADDR(vm->as.id),
+		gpu_write64(ptdev->mmu->iomem, AS_LOCKADDR(vm->as.id),
 			    pack_region_range(ptdev, &start, &size));
 
 		/* If the lock succeeded, update the locked_region info. */
@@ -1719,6 +1787,7 @@ static void panthor_vm_unlock_region(struct panthor_vm *vm)
 
 static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 {
+	struct panthor_mmu *mmu = ptdev->mmu;
 	bool has_unhandled_faults = false;
 
 	status = panthor_mmu_fault_mask(ptdev, status);
@@ -1731,8 +1800,8 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 		u32 access_type;
 		u32 source_id;
 
-		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
-		addr = gpu_read64(ptdev, AS_FAULTADDRESS(as));
+		fault_status = gpu_read(mmu->iomem, AS_FAULTSTATUS(as));
+		addr = gpu_read64(mmu->iomem, AS_FAULTADDRESS(as));
 
 		/* decode the fault status */
 		exception_type = fault_status & 0xFF;
@@ -1763,7 +1832,7 @@ static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
 		 * Note that COMPLETED irqs are never cleared, but this is fine
 		 * because they are always masked.
 		 */
-		gpu_write(ptdev, MMU_INT_CLEAR, mask);
+		gpu_write(mmu->irq.iomem, INT_CLEAR, mask);
 
 		if (ptdev->mmu->as.slots[as].vm)
 			ptdev->mmu->as.slots[as].vm->unhandled_fault = true;
@@ -1893,6 +1962,10 @@ static void panthor_vm_free(struct drm_gpuvm *gpuvm)
 	struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base);
 	struct panthor_device *ptdev = vm->ptdev;
 
+	mutex_lock(&ptdev->base.gem_lru_mutex);
+	list_del_init(&vm->reclaim.lru_node);
+	mutex_unlock(&ptdev->base.gem_lru_mutex);
+
 	mutex_lock(&vm->heaps.lock);
 	if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
 		panthor_heap_pool_destroy(vm->heaps.pool);
@@ -2071,9 +2144,9 @@ static void panthor_vma_link(struct panthor_vm *vm,
 {
 	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
 
-	mutex_lock(&bo->base.base.gpuva.lock);
+	mutex_lock(&bo->base.gpuva.lock);
 	drm_gpuva_link(&vma->base, vm_bo);
-	mutex_unlock(&bo->base.base.gpuva.lock);
+	mutex_unlock(&bo->base.gpuva.lock);
 }
 
 static void panthor_vma_unlink(struct panthor_vma *vma)
@@ -2106,7 +2179,7 @@ static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
 	panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);
 
 	ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags),
-				   op_ctx->map.sgt, op->map.gem.offset,
+				   op_ctx->map.bo->dmap.sgt, op->map.gem.offset,
 				   op->map.va.range);
 	if (ret) {
 		panthor_vm_op_ctx_return_vma(op_ctx, vma);
@@ -2125,11 +2198,12 @@ static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
 static bool
 iova_mapped_as_huge_page(struct drm_gpuva_op_map *op, u64 addr)
 {
+	struct panthor_gem_object *bo = to_panthor_bo(op->gem.obj);
 	const struct page *pg;
 	pgoff_t bo_offset;
 
 	bo_offset = addr - op->va.addr + op->gem.offset;
-	pg = to_panthor_bo(op->gem.obj)->base.pages[bo_offset >> PAGE_SHIFT];
+	pg = bo->backing.pages[bo_offset >> PAGE_SHIFT];
 
 	return folio_size(page_folio(pg)) >= SZ_2M;
 }
@@ -2189,21 +2263,27 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
 	 * atomicity. panthor_vm_lock_region() bails out early if the new region
 	 * is already part of the locked region, so no need to do this check here.
 	 */
-	panthor_vm_lock_region(vm, unmap_start, unmap_range);
-	panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
+	if (!unmap_vma->evicted) {
+		panthor_vm_lock_region(vm, unmap_start, unmap_range);
+		panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
+	}
 
 	if (op->remap.prev) {
 		struct panthor_gem_object *bo = to_panthor_bo(op->remap.prev->gem.obj);
 		u64 offset = op->remap.prev->gem.offset + unmap_start - op->remap.prev->va.addr;
 		u64 size = op->remap.prev->va.addr + op->remap.prev->va.range - unmap_start;
 
-		ret = panthor_vm_map_pages(vm, unmap_start, flags_to_prot(unmap_vma->flags),
-					   bo->base.sgt, offset, size);
-		if (ret)
-			return ret;
+		if (!unmap_vma->evicted) {
+			ret = panthor_vm_map_pages(vm, unmap_start,
+						   flags_to_prot(unmap_vma->flags),
+						   bo->dmap.sgt, offset, size);
+			if (ret)
+				return ret;
+		}
 
 		prev_vma = panthor_vm_op_ctx_get_vma(op_ctx);
 		panthor_vma_init(prev_vma, unmap_vma->flags);
+		prev_vma->evicted = unmap_vma->evicted;
 	}
 
 	if (op->remap.next) {
@@ -2211,13 +2291,17 @@ static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
 		u64 addr = op->remap.next->va.addr;
 		u64 size = unmap_start + unmap_range - op->remap.next->va.addr;
 
-		ret = panthor_vm_map_pages(vm, addr, flags_to_prot(unmap_vma->flags),
-					   bo->base.sgt, op->remap.next->gem.offset, size);
-		if (ret)
-			return ret;
+		if (!unmap_vma->evicted) {
+			ret = panthor_vm_map_pages(vm, addr, flags_to_prot(unmap_vma->flags),
+						   bo->dmap.sgt, op->remap.next->gem.offset,
+						   size);
+			if (ret)
+				return ret;
+		}
 
 		next_vma = panthor_vm_op_ctx_get_vma(op_ctx);
 		panthor_vma_init(next_vma, unmap_vma->flags);
+		next_vma->evicted = unmap_vma->evicted;
 	}
 
 	drm_gpuva_remap(prev_vma ? &prev_vma->base : NULL,
@@ -2247,19 +2331,236 @@ static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,
 	struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);
 	struct panthor_vm *vm = priv;
 
-	panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
-			       unmap_vma->base.va.range);
+	if (!unmap_vma->evicted) {
+		panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
+				       unmap_vma->base.va.range);
+	}
+
 	drm_gpuva_unmap(&op->unmap);
 	panthor_vma_unlink(unmap_vma);
 	return 0;
 }
 
+void panthor_vm_update_bo_reclaim_lru_locked(struct panthor_gem_object *bo)
+{
+	struct panthor_device *ptdev = container_of(bo->base.dev, struct panthor_device, base);
+	struct panthor_vm *vm = NULL;
+	struct drm_gpuvm_bo *vm_bo;
+
+	dma_resv_assert_held(bo->base.resv);
+	lockdep_assert_held(&bo->base.gpuva.lock);
+
+	drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) {
+		if (vm_bo->evicted)
+			continue;
+
+		/* We're only supposed to have one non-evicted vm_bo in the list if we get
+		 * there.
+		 */
+		drm_WARN_ON(&ptdev->base, vm);
+		vm = container_of(vm_bo->vm, struct panthor_vm, base);
+
+		mutex_lock(&ptdev->base.gem_lru_mutex);
+		drm_gem_lru_move_tail_locked(&vm->reclaim.lru, &bo->base);
+		if (list_empty(&vm->reclaim.lru_node))
+			list_move(&vm->reclaim.lru_node, &ptdev->reclaim.vms);
+		mutex_unlock(&ptdev->base.gem_lru_mutex);
+	}
+}
+
+int panthor_vm_evict_bo_mappings_locked(struct panthor_gem_object *bo)
+{
+	struct drm_gpuvm_bo *vm_bo;
+	int ret = 0;
+
+	drm_gem_for_each_gpuvm_bo(vm_bo, &bo->base) {
+		struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
+		struct drm_gpuva *va;
+
+		if (!mutex_trylock(&vm->op_lock))
+			return -EDEADLK;
+
+		/* It can be that the vm_bo was already evicted but a new
+		 * mapping pointing to this BO got created in the meantime,
+		 * thus turning the vm_bo in partially evicted state. In that case
+		 * we don't call drm_gpuvm_bo_evict() again because this would
+		 * mess up with the internal gpuvm lists, but we do walk the
+		 * VAs on this vm_bo to make sure the non-evicted ones are
+		 * torn down.
+		 */
+		if (!vm_bo->evicted)
+			drm_gpuvm_bo_evict(vm_bo, true);
+
+		drm_gpuvm_bo_for_each_va(va, vm_bo) {
+			struct panthor_vma *vma = container_of(va, struct panthor_vma, base);
+
+			if (vma->evicted)
+				continue;
+
+			/* If something fail in the middle of a VM_BO eviction, the VM_BO
+			 * is considered fully evicted, but some of its VMAs might still be
+			 * active. That's okay because the pages won't be released if this
+			 * function returns an error.
+			 *
+			 * On the next job targeting this VM, the partially evicted VM_BO
+			 * will be validated, causing all its evicted VMAs to be repopulated
+			 * before the job runs. So no GPU fault expected.
+			 */
+			ret = panthor_vm_lock_region(vm, va->va.addr, va->va.range);
+			if (ret)
+				break;
+
+			panthor_vm_unmap_pages(vm, va->va.addr, va->va.range);
+			panthor_vm_unlock_region(vm);
+			vma->evicted = true;
+		}
+
+		mutex_unlock(&vm->op_lock);
+
+		if (ret)
+			break;
+	}
+
+	return ret;
+}
+
+static struct panthor_vma *select_evicted_vma(struct drm_gpuvm_bo *vm_bo,
+					      struct panthor_vm_op_ctx *op_ctx)
+{
+	struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
+	struct panthor_vma *first_evicted_vma = NULL;
+	struct drm_gpuva *va;
+
+	/* Take op_lock to protect against va insertion/removal. */
+	mutex_lock(&vm->op_lock);
+	drm_gpuvm_bo_for_each_va(va, vm_bo) {
+		struct panthor_vma *vma = container_of(va, struct panthor_vma, base);
+
+		if (vma->evicted) {
+			first_evicted_vma = vma;
+			panthor_vm_init_op_ctx(op_ctx, va->va.range, va->va.addr, vma->flags);
+			op_ctx->map.bo_offset = va->gem.offset;
+			break;
+		}
+	}
+	mutex_unlock(&vm->op_lock);
+
+	return first_evicted_vma;
+}
+
+static int remap_evicted_vma(struct drm_gpuvm_bo *vm_bo,
+			     struct panthor_vma *evicted_vma,
+			     struct panthor_vm_op_ctx *op_ctx)
+{
+	struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
+	struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
+	struct drm_gpuva *va;
+	bool found = false;
+	int ret;
+
+	ret = panthor_vm_op_ctx_prealloc_pts(op_ctx);
+	if (ret)
+		goto out_cleanup;
+
+	/* Take op_lock to protect against va insertion/removal. Note that the
+	 * evicted_vma selection was done with the same lock held, but we had
+	 * to release it so we can allocate PTs, because this very same lock
+	 * is taken in a DMA-signalling path.
+	 */
+	mutex_lock(&vm->op_lock);
+	drm_gpuvm_bo_for_each_va(va, vm_bo) {
+		struct panthor_vma *vma = container_of(va, struct panthor_vma, base);
+
+		if (vma != evicted_vma)
+			continue;
+
+		/* Because we had to release the lock between the evicted_vma selection
+		 * and its repopulation, we can't rely solely on pointer equality (the
+		 * VMA might have been freed and a new one allocated at the same address).
+		 * If the evicted bit is still set, we're sure it's our VMA, because
+		 * population/eviction is serialized with the BO resv lock.
+		 */
+		if (vma->evicted)
+			found = true;
+
+		break;
+	}
+
+	if (found) {
+		vm->op_ctx = op_ctx;
+		ret = panthor_vm_lock_region(vm, evicted_vma->base.va.addr,
+					     evicted_vma->base.va.range);
+		if (!ret) {
+			ret = panthor_vm_map_pages(vm, evicted_vma->base.va.addr,
+						   flags_to_prot(evicted_vma->flags),
+						   bo->dmap.sgt,
+						   evicted_vma->base.gem.offset,
+						   evicted_vma->base.va.range);
+			if (!ret)
+				evicted_vma->evicted = false;
+
+			panthor_vm_unlock_region(vm);
+		}
+
+		vm->op_ctx = NULL;
+	}
+
+	mutex_unlock(&vm->op_lock);
+
+out_cleanup:
+	panthor_vm_cleanup_op_ctx(op_ctx, vm);
+	return ret;
+}
+
+static int panthor_vm_restore_vmas(struct drm_gpuvm_bo *vm_bo)
+{
+	struct panthor_vm *vm = container_of(vm_bo->vm, struct panthor_vm, base);
+	struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
+	struct panthor_vm_op_ctx op_ctx;
+
+	if (drm_WARN_ON_ONCE(&vm->ptdev->base, !bo->dmap.sgt))
+		return -EINVAL;
+
+	for (struct panthor_vma *vma = select_evicted_vma(vm_bo, &op_ctx);
+	     vma; vma = select_evicted_vma(vm_bo, &op_ctx)) {
+		int ret;
+
+		ret = remap_evicted_vma(vm_bo, vma, &op_ctx);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int panthor_vm_bo_validate(struct drm_gpuvm_bo *vm_bo,
+				  struct drm_exec *exec)
+{
+	struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
+	int ret;
+
+	ret = panthor_gem_swapin_locked(bo);
+	if (ret)
+		return ret;
+
+	ret = panthor_vm_restore_vmas(vm_bo);
+	if (ret)
+		return ret;
+
+	drm_gpuvm_bo_evict(vm_bo, false);
+	mutex_lock(&bo->base.gpuva.lock);
+	panthor_gem_update_reclaim_state_locked(bo, NULL);
+	mutex_unlock(&bo->base.gpuva.lock);
+	return 0;
+}
+
 static const struct drm_gpuvm_ops panthor_gpuvm_ops = {
 	.vm_free = panthor_vm_free,
 	.vm_bo_free = panthor_vm_bo_free,
 	.sm_step_map = panthor_gpuva_sm_step_map,
 	.sm_step_remap = panthor_gpuva_sm_step_remap,
 	.sm_step_unmap = panthor_gpuva_sm_step_unmap,
+	.vm_bo_validate = panthor_vm_bo_validate,
 };
 
 /**
@@ -2432,7 +2733,6 @@ panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
 	const struct drm_sched_init_args sched_args = {
 		.ops = &panthor_vm_bind_ops,
 		.submit_wq = ptdev->mmu->vm.wq,
-		.num_rqs = 1,
 		.credit_limit = 1,
 		/* Bind operations are synchronous for now, no timeout needed. */
 		.timeout = MAX_SCHEDULE_TIMEOUT,
@@ -2474,6 +2774,8 @@ panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
 	vm->kernel_auto_va.start = auto_kernel_va_start;
 	vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1;
 
+	drm_gem_lru_init(&vm->reclaim.lru);
+	INIT_LIST_HEAD(&vm->reclaim.lru_node);
 	INIT_LIST_HEAD(&vm->node);
 	INIT_LIST_HEAD(&vm->as.lru_node);
 	vm->as.id = -1;
@@ -2821,7 +3123,79 @@ int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm
 	if (ret)
 		return ret;
 
-	return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
+	ret = drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
+	if (ret)
+		return ret;
+
+	return drm_gpuvm_validate(&vm->base, exec);
+}
+
+unsigned long
+panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev,
+			     unsigned int nr_to_scan, unsigned long *remaining,
+			     bool (*shrink)(struct drm_gem_object *,
+					    struct ww_acquire_ctx *))
+{
+	unsigned long freed = 0;
+	LIST_HEAD(remaining_vms);
+	LIST_HEAD(vms);
+
+	mutex_lock(&ptdev->base.gem_lru_mutex);
+	list_splice_init(&ptdev->reclaim.vms, &vms);
+
+	while (freed < nr_to_scan) {
+		struct panthor_vm *vm;
+
+		vm = list_first_entry_or_null(&vms, typeof(*vm),
+					      reclaim.lru_node);
+		if (!vm)
+			break;
+
+		if (!kref_get_unless_zero(&vm->base.kref)) {
+			list_del_init(&vm->reclaim.lru_node);
+			continue;
+		}
+
+		mutex_unlock(&ptdev->base.gem_lru_mutex);
+
+		freed += drm_gem_lru_scan(&ptdev->base, &vm->reclaim.lru,
+					  nr_to_scan - freed,
+					  remaining, shrink, NULL);
+
+		mutex_lock(&ptdev->base.gem_lru_mutex);
+
+		/* If the VM is still in the temporary list, remove it so we
+		 * can proceed with the next VM.
+		 */
+		if (vm == list_first_entry_or_null(&vms, typeof(*vm), reclaim.lru_node)) {
+			list_del_init(&vm->reclaim.lru_node);
+
+			/* Keep the VM around if there are still things to
+			 * reclaim, so we can preserve the LRU order when
+			 * re-inserting in ptdev->reclaim.vms at the end.
+			 */
+			if (vm->reclaim.lru.count > 0)
+				list_add_tail(&vm->reclaim.lru_node, &remaining_vms);
+		}
+
+		mutex_unlock(&ptdev->base.gem_lru_mutex);
+
+		panthor_vm_put(vm);
+
+		mutex_lock(&ptdev->base.gem_lru_mutex);
+	}
+
+	/* Re-insert VMs with remaining data to reclaim at the beginning of
+	 * the LRU. Note that any activeness change on the VM that happened
+	 * while we were reclaiming would have moved the VM out of our
+	 * temporary [remaining_]vms list, meaning anything we re-insert here
+	 * preserves the LRU order.
+	 */
+	list_splice_tail(&vms, &remaining_vms);
+	list_splice(&remaining_vms, &ptdev->reclaim.vms);
+	mutex_unlock(&ptdev->base.gem_lru_mutex);
+
+	return freed;
 }
 
 /**
@@ -2881,6 +3255,7 @@ int panthor_mmu_init(struct panthor_device *ptdev)
 	if (ret)
 		return ret;
 
+	mmu->iomem = ptdev->iomem + MMU_AS_BASE;
 	ptdev->mmu = mmu;
 
 	irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu");
@@ -2888,7 +3263,8 @@ int panthor_mmu_init(struct panthor_device *ptdev)
 		return -ENODEV;
 
 	ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,
-				      panthor_mmu_fault_mask(ptdev, ~0));
+				      panthor_mmu_fault_mask(ptdev, ~0),
+				      ptdev->iomem + MMU_INT_BASE);
 	if (ret)
 		return ret;
 
diff --git a/drivers/gpu/drm/panthor/panthor_mmu.h b/drivers/gpu/drm/panthor/panthor_mmu.h
index 0e268fdfdb2f0..3522fbbce369c 100644
--- a/drivers/gpu/drm/panthor/panthor_mmu.h
+++ b/drivers/gpu/drm/panthor/panthor_mmu.h
@@ -1,6 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 or MIT */
 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
 /* Copyright 2023 Collabora ltd. */
+/* Copyright 2025 ARM Limited. All rights reserved. */
 
 #ifndef __PANTHOR_MMU_H__
 #define __PANTHOR_MMU_H__
@@ -46,6 +47,13 @@ struct panthor_vm *panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
 				     u64 kernel_auto_va_start,
 				     u64 kernel_auto_va_size);
 
+void panthor_vm_update_bo_reclaim_lru_locked(struct panthor_gem_object *bo);
+int panthor_vm_evict_bo_mappings_locked(struct panthor_gem_object *bo);
+unsigned long
+panthor_mmu_reclaim_priv_bos(struct panthor_device *ptdev,
+			     unsigned int nr_to_scan, unsigned long *remaining,
+			     bool (*shrink)(struct drm_gem_object *,
+					    struct ww_acquire_ctx *));
 int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec,
 					struct panthor_vm *vm,
 					u32 slot_count);
diff --git a/drivers/gpu/drm/panthor/panthor_mmu_regs.h b/drivers/gpu/drm/panthor/panthor_mmu_regs.h
new file mode 100644
index 0000000000000..4e32ab931949c
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_mmu_regs.h
@@ -0,0 +1,69 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2026 ARM Limited. All rights reserved. */
+
+#ifndef __PANTHOR_MMU_REGS_H__
+#define __PANTHOR_MMU_REGS_H__
+
+/* MMU regs */
+
+#define MMU_INT_BASE					0x2000
+
+/* AS_COMMAND register commands */
+
+#define MMU_AS_BASE					0x2400
+
+#define MMU_AS_SHIFT					6
+#define MMU_AS(as)					((as) << MMU_AS_SHIFT)
+
+#define AS_TRANSTAB(as)					(MMU_AS(as) + 0x0)
+#define AS_MEMATTR(as)					(MMU_AS(as) + 0x8)
+#define   AS_MEMATTR_AARCH64_INNER_ALLOC_IMPL		(2 << 2)
+#define   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(w, r)	((3 << 2) | \
+							 ((w) ? BIT(0) : 0) | \
+							 ((r) ? BIT(1) : 0))
+#define   AS_MEMATTR_AARCH64_SH_MIDGARD_INNER		(0 << 4)
+#define   AS_MEMATTR_AARCH64_SH_CPU_INNER		(1 << 4)
+#define   AS_MEMATTR_AARCH64_SH_CPU_INNER_SHADER_COH	(2 << 4)
+#define   AS_MEMATTR_AARCH64_SHARED			(0 << 6)
+#define   AS_MEMATTR_AARCH64_INNER_OUTER_NC		(1 << 6)
+#define   AS_MEMATTR_AARCH64_INNER_OUTER_WB		(2 << 6)
+#define   AS_MEMATTR_AARCH64_FAULT			(3 << 6)
+#define AS_LOCKADDR(as)					(MMU_AS(as) + 0x10)
+#define AS_COMMAND(as)					(MMU_AS(as) + 0x18)
+#define   AS_COMMAND_NOP				0
+#define   AS_COMMAND_UPDATE				1
+#define   AS_COMMAND_LOCK				2
+#define   AS_COMMAND_UNLOCK				3
+#define   AS_COMMAND_FLUSH_PT				4
+#define   AS_COMMAND_FLUSH_MEM				5
+#define   AS_LOCK_REGION_MIN_SIZE			(1ULL << 15)
+#define AS_FAULTSTATUS(as)				(MMU_AS(as) + 0x1C)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_MASK		(0x3 << 8)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC		(0x0 << 8)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_EX			(0x1 << 8)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_READ		(0x2 << 8)
+#define  AS_FAULTSTATUS_ACCESS_TYPE_WRITE		(0x3 << 8)
+#define AS_FAULTADDRESS(as)				(MMU_AS(as) + 0x20)
+#define AS_STATUS(as)					(MMU_AS(as) + 0x28)
+#define   AS_STATUS_AS_ACTIVE				BIT(0)
+#define AS_TRANSCFG(as)					(MMU_AS(as) + 0x30)
+#define   AS_TRANSCFG_ADRMODE_UNMAPPED			(1 << 0)
+#define   AS_TRANSCFG_ADRMODE_IDENTITY			(2 << 0)
+#define   AS_TRANSCFG_ADRMODE_AARCH64_4K		(6 << 0)
+#define   AS_TRANSCFG_ADRMODE_AARCH64_64K		(8 << 0)
+#define   AS_TRANSCFG_INA_BITS(x)			((x) << 6)
+#define   AS_TRANSCFG_OUTA_BITS(x)			((x) << 14)
+#define   AS_TRANSCFG_SL_CONCAT				BIT(22)
+#define   AS_TRANSCFG_PTW_MEMATTR_NC			(1 << 24)
+#define   AS_TRANSCFG_PTW_MEMATTR_WB			(2 << 24)
+#define   AS_TRANSCFG_PTW_SH_NS				(0 << 28)
+#define   AS_TRANSCFG_PTW_SH_OS				(2 << 28)
+#define   AS_TRANSCFG_PTW_SH_IS				(3 << 28)
+#define   AS_TRANSCFG_PTW_RA				BIT(30)
+#define   AS_TRANSCFG_DISABLE_HIER_AP			BIT(33)
+#define   AS_TRANSCFG_DISABLE_AF_FAULT			BIT(34)
+#define   AS_TRANSCFG_WXN				BIT(35)
+#define   AS_TRANSCFG_XREADABLE				BIT(36)
+#define AS_FAULTEXTRA(as)				(MMU_AS(as) + 0x38)
+
+#endif /* __PANTHOR_MMU_REGS_H__ */
diff --git a/drivers/gpu/drm/panthor/panthor_pwr.c b/drivers/gpu/drm/panthor/panthor_pwr.c
index ed3b2b4479ca3..7c7f424a1436d 100644
--- a/drivers/gpu/drm/panthor/panthor_pwr.c
+++ b/drivers/gpu/drm/panthor/panthor_pwr.c
@@ -11,9 +11,10 @@
 #include <drm/drm_print.h>
 
 #include "panthor_device.h"
+#include "panthor_gpu_regs.h"
 #include "panthor_hw.h"
 #include "panthor_pwr.h"
-#include "panthor_regs.h"
+#include "panthor_pwr_regs.h"
 
 #define PWR_INTERRUPTS_MASK \
 	(PWR_IRQ_POWER_CHANGED_SINGLE | \
@@ -39,6 +40,9 @@
  * struct panthor_pwr - PWR_CONTROL block management data.
  */
 struct panthor_pwr {
+	/** @iomem: CPU mapping of PWR_CONTROL iomem region */
+	void __iomem *iomem;
+
 	/** @irq: PWR irq. */
 	struct panthor_irq irq;
 
@@ -54,8 +58,10 @@ struct panthor_pwr {
 
 static void panthor_pwr_irq_handler(struct panthor_device *ptdev, u32 status)
 {
+	struct panthor_pwr *pwr = ptdev->pwr;
+
 	spin_lock(&ptdev->pwr->reqs_lock);
-	gpu_write(ptdev, PWR_INT_CLEAR, status);
+	gpu_write(pwr->irq.iomem, INT_CLEAR, status);
 
 	if (unlikely(status & PWR_IRQ_COMMAND_NOT_ALLOWED))
 		drm_err(&ptdev->base, "PWR_IRQ: COMMAND_NOT_ALLOWED");
@@ -69,19 +75,23 @@ static void panthor_pwr_irq_handler(struct panthor_device *ptdev, u32 status)
 	}
 	spin_unlock(&ptdev->pwr->reqs_lock);
 }
-PANTHOR_IRQ_HANDLER(pwr, PWR, panthor_pwr_irq_handler);
+PANTHOR_IRQ_HANDLER(pwr, panthor_pwr_irq_handler);
 
 static void panthor_pwr_write_command(struct panthor_device *ptdev, u32 command, u64 args)
 {
+	struct panthor_pwr *pwr = ptdev->pwr;
+
 	if (args)
-		gpu_write64(ptdev, PWR_CMDARG, args);
+		gpu_write64(pwr->iomem, PWR_CMDARG, args);
 
-	gpu_write(ptdev, PWR_COMMAND, command);
+	gpu_write(pwr->iomem, PWR_COMMAND, command);
 }
 
 static bool reset_irq_raised(struct panthor_device *ptdev)
 {
-	return gpu_read(ptdev, PWR_INT_RAWSTAT) & PWR_IRQ_RESET_COMPLETED;
+	struct panthor_pwr *pwr = ptdev->pwr;
+
+	return gpu_read(pwr->irq.iomem, INT_RAWSTAT) & PWR_IRQ_RESET_COMPLETED;
 }
 
 static bool reset_pending(struct panthor_device *ptdev)
@@ -91,12 +101,14 @@ static bool reset_pending(struct panthor_device *ptdev)
 
 static int panthor_pwr_reset(struct panthor_device *ptdev, u32 reset_cmd)
 {
+	struct panthor_pwr *pwr = ptdev->pwr;
+
 	scoped_guard(spinlock_irqsave, &ptdev->pwr->reqs_lock) {
 		if (reset_pending(ptdev)) {
 			drm_WARN(&ptdev->base, 1, "Reset already pending");
 		} else {
 			ptdev->pwr->pending_reqs |= PWR_IRQ_RESET_COMPLETED;
-			gpu_write(ptdev, PWR_INT_CLEAR, PWR_IRQ_RESET_COMPLETED);
+			gpu_write(pwr->irq.iomem, INT_CLEAR, PWR_IRQ_RESET_COMPLETED);
 			panthor_pwr_write_command(ptdev, reset_cmd, 0);
 		}
 	}
@@ -181,11 +193,12 @@ static u8 get_domain_subdomain(struct panthor_device *ptdev, u32 domain)
 static int panthor_pwr_domain_wait_transition(struct panthor_device *ptdev, u32 domain,
 					      u32 timeout_us)
 {
+	struct panthor_pwr *pwr = ptdev->pwr;
 	u32 pwrtrans_reg = get_domain_pwrtrans_reg(domain);
 	u64 val;
 	int ret = 0;
 
-	ret = gpu_read64_poll_timeout(ptdev, pwrtrans_reg, val, !(PWR_ALL_CORES_MASK & val), 100,
+	ret = gpu_read64_poll_timeout(pwr->iomem, pwrtrans_reg, val, !(PWR_ALL_CORES_MASK & val), 100,
 				      timeout_us);
 	if (ret) {
 		drm_err(&ptdev->base, "%s domain power in transition, pwrtrans(0x%llx)",
@@ -198,22 +211,25 @@ static int panthor_pwr_domain_wait_transition(struct panthor_device *ptdev, u32
 
 static void panthor_pwr_debug_info_show(struct panthor_device *ptdev)
 {
-	drm_info(&ptdev->base, "GPU_FEATURES:    0x%016llx", gpu_read64(ptdev, GPU_FEATURES));
-	drm_info(&ptdev->base, "PWR_STATUS:      0x%016llx", gpu_read64(ptdev, PWR_STATUS));
-	drm_info(&ptdev->base, "L2_PRESENT:      0x%016llx", gpu_read64(ptdev, PWR_L2_PRESENT));
-	drm_info(&ptdev->base, "L2_PWRTRANS:     0x%016llx", gpu_read64(ptdev, PWR_L2_PWRTRANS));
-	drm_info(&ptdev->base, "L2_READY:        0x%016llx", gpu_read64(ptdev, PWR_L2_READY));
-	drm_info(&ptdev->base, "TILER_PRESENT:   0x%016llx", gpu_read64(ptdev, PWR_TILER_PRESENT));
-	drm_info(&ptdev->base, "TILER_PWRTRANS:  0x%016llx", gpu_read64(ptdev, PWR_TILER_PWRTRANS));
-	drm_info(&ptdev->base, "TILER_READY:     0x%016llx", gpu_read64(ptdev, PWR_TILER_READY));
-	drm_info(&ptdev->base, "SHADER_PRESENT:  0x%016llx", gpu_read64(ptdev, PWR_SHADER_PRESENT));
-	drm_info(&ptdev->base, "SHADER_PWRTRANS: 0x%016llx", gpu_read64(ptdev, PWR_SHADER_PWRTRANS));
-	drm_info(&ptdev->base, "SHADER_READY:    0x%016llx", gpu_read64(ptdev, PWR_SHADER_READY));
+	struct panthor_pwr *pwr = ptdev->pwr;
+
+	drm_info(&ptdev->base, "GPU_FEATURES:    0x%016llx", ptdev->gpu_info.gpu_features);
+	drm_info(&ptdev->base, "PWR_STATUS:      0x%016llx", gpu_read64(pwr->iomem, PWR_STATUS));
+	drm_info(&ptdev->base, "L2_PRESENT:      0x%016llx", gpu_read64(pwr->iomem, PWR_L2_PRESENT));
+	drm_info(&ptdev->base, "L2_PWRTRANS:     0x%016llx", gpu_read64(pwr->iomem, PWR_L2_PWRTRANS));
+	drm_info(&ptdev->base, "L2_READY:        0x%016llx", gpu_read64(pwr->iomem, PWR_L2_READY));
+	drm_info(&ptdev->base, "TILER_PRESENT:   0x%016llx", gpu_read64(pwr->iomem, PWR_TILER_PRESENT));
+	drm_info(&ptdev->base, "TILER_PWRTRANS:  0x%016llx", gpu_read64(pwr->iomem, PWR_TILER_PWRTRANS));
+	drm_info(&ptdev->base, "TILER_READY:     0x%016llx", gpu_read64(pwr->iomem, PWR_TILER_READY));
+	drm_info(&ptdev->base, "SHADER_PRESENT:  0x%016llx", gpu_read64(pwr->iomem, PWR_SHADER_PRESENT));
+	drm_info(&ptdev->base, "SHADER_PWRTRANS: 0x%016llx", gpu_read64(pwr->iomem, PWR_SHADER_PWRTRANS));
+	drm_info(&ptdev->base, "SHADER_READY:    0x%016llx", gpu_read64(pwr->iomem, PWR_SHADER_READY));
 }
 
 static int panthor_pwr_domain_transition(struct panthor_device *ptdev, u32 cmd, u32 domain,
 					 u64 mask, u32 timeout_us)
 {
+	struct panthor_pwr *pwr = ptdev->pwr;
 	u32 ready_reg = get_domain_ready_reg(domain);
 	u32 pwr_cmd = PWR_COMMAND_DEF(cmd, domain, get_domain_subdomain(ptdev, domain));
 	u64 expected_val = 0;
@@ -240,13 +256,13 @@ static int panthor_pwr_domain_transition(struct panthor_device *ptdev, u32 cmd,
 		return ret;
 
 	/* domain already in target state, return early */
-	if ((gpu_read64(ptdev, ready_reg) & mask) == expected_val)
+	if ((gpu_read64(pwr->iomem, ready_reg) & mask) == expected_val)
 		return 0;
 
 	panthor_pwr_write_command(ptdev, pwr_cmd, mask);
 
-	ret = gpu_read64_poll_timeout(ptdev, ready_reg, val, (mask & val) == expected_val, 100,
-				      timeout_us);
+	ret = gpu_read64_poll_timeout(pwr->iomem, ready_reg, val, (mask & val) == expected_val,
+				      100, timeout_us);
 	if (ret) {
 		drm_err(&ptdev->base,
 			"timeout waiting on %s power domain transition, cmd(0x%x), arg(0x%llx)",
@@ -278,8 +294,9 @@ static int panthor_pwr_domain_transition(struct panthor_device *ptdev, u32 cmd,
  */
 static int retract_domain(struct panthor_device *ptdev, u32 domain)
 {
+	struct panthor_pwr *pwr = ptdev->pwr;
 	const u32 pwr_cmd = PWR_COMMAND_DEF(PWR_COMMAND_RETRACT, domain, 0);
-	const u64 pwr_status = gpu_read64(ptdev, PWR_STATUS);
+	const u64 pwr_status = gpu_read64(pwr->iomem, PWR_STATUS);
 	const u64 delegated_mask = PWR_STATUS_DOMAIN_DELEGATED(domain);
 	const u64 allow_mask = PWR_STATUS_DOMAIN_ALLOWED(domain);
 	u64 val;
@@ -288,8 +305,9 @@ static int retract_domain(struct panthor_device *ptdev, u32 domain)
 	if (drm_WARN_ON(&ptdev->base, domain == PWR_COMMAND_DOMAIN_L2))
 		return -EPERM;
 
-	ret = gpu_read64_poll_timeout(ptdev, PWR_STATUS, val, !(PWR_STATUS_RETRACT_PENDING & val),
-				      0, PWR_RETRACT_TIMEOUT_US);
+	ret = gpu_read64_poll_timeout(pwr->iomem, PWR_STATUS, val,
+				      !(PWR_STATUS_RETRACT_PENDING & val), 0,
+				      PWR_RETRACT_TIMEOUT_US);
 	if (ret) {
 		drm_err(&ptdev->base, "%s domain retract pending", get_domain_name(domain));
 		return ret;
@@ -306,7 +324,7 @@ static int retract_domain(struct panthor_device *ptdev, u32 domain)
 	 * On successful retraction
 	 * allow-flag will be set with delegated-flag being cleared.
 	 */
-	ret = gpu_read64_poll_timeout(ptdev, PWR_STATUS, val,
+	ret = gpu_read64_poll_timeout(pwr->iomem, PWR_STATUS, val,
 				      ((delegated_mask | allow_mask) & val) == allow_mask, 10,
 				      PWR_TRANSITION_TIMEOUT_US);
 	if (ret) {
@@ -332,8 +350,9 @@ static int retract_domain(struct panthor_device *ptdev, u32 domain)
  */
 static int delegate_domain(struct panthor_device *ptdev, u32 domain)
 {
+	struct panthor_pwr *pwr = ptdev->pwr;
 	const u32 pwr_cmd = PWR_COMMAND_DEF(PWR_COMMAND_DELEGATE, domain, 0);
-	const u64 pwr_status = gpu_read64(ptdev, PWR_STATUS);
+	const u64 pwr_status = gpu_read64(pwr->iomem, PWR_STATUS);
 	const u64 allow_mask = PWR_STATUS_DOMAIN_ALLOWED(domain);
 	const u64 delegated_mask = PWR_STATUS_DOMAIN_DELEGATED(domain);
 	u64 val;
@@ -362,7 +381,7 @@ static int delegate_domain(struct panthor_device *ptdev, u32 domain)
 	 * On successful delegation
 	 * allow-flag will be cleared with delegated-flag being set.
 	 */
-	ret = gpu_read64_poll_timeout(ptdev, PWR_STATUS, val,
+	ret = gpu_read64_poll_timeout(pwr->iomem, PWR_STATUS, val,
 				      ((delegated_mask | allow_mask) & val) == delegated_mask,
 				      10, PWR_TRANSITION_TIMEOUT_US);
 	if (ret) {
@@ -410,7 +429,8 @@ static int panthor_pwr_delegate_domains(struct panthor_device *ptdev)
  */
 static int panthor_pwr_domain_force_off(struct panthor_device *ptdev, u32 domain)
 {
-	const u64 domain_ready = gpu_read64(ptdev, get_domain_ready_reg(domain));
+	struct panthor_pwr *pwr = ptdev->pwr;
+	const u64 domain_ready = gpu_read64(pwr->iomem, get_domain_ready_reg(domain));
 	int ret;
 
 	/* Domain already powered down, early exit. */
@@ -454,6 +474,7 @@ int panthor_pwr_init(struct panthor_device *ptdev)
 	if (!pwr)
 		return -ENOMEM;
 
+	pwr->iomem = ptdev->iomem + PWR_CONTROL_BASE;
 	spin_lock_init(&pwr->reqs_lock);
 	init_waitqueue_head(&pwr->reqs_acked);
 	ptdev->pwr = pwr;
@@ -462,7 +483,9 @@ int panthor_pwr_init(struct panthor_device *ptdev)
 	if (irq < 0)
 		return irq;
 
-	err = panthor_request_pwr_irq(ptdev, &pwr->irq, irq, PWR_INTERRUPTS_MASK);
+	err = panthor_request_pwr_irq(
+		ptdev, &pwr->irq, irq, PWR_INTERRUPTS_MASK,
+		pwr->iomem + PWR_INT_BASE);
 	if (err)
 		return err;
 
@@ -471,7 +494,9 @@ int panthor_pwr_init(struct panthor_device *ptdev)
 
 int panthor_pwr_reset_soft(struct panthor_device *ptdev)
 {
-	if (!(gpu_read64(ptdev, PWR_STATUS) & PWR_STATUS_ALLOW_SOFT_RESET)) {
+	struct panthor_pwr *pwr = ptdev->pwr;
+
+	if (!(gpu_read64(pwr->iomem, PWR_STATUS) & PWR_STATUS_ALLOW_SOFT_RESET)) {
 		drm_err(&ptdev->base, "RESET_SOFT not allowed");
 		return -EOPNOTSUPP;
 	}
@@ -481,8 +506,9 @@ int panthor_pwr_reset_soft(struct panthor_device *ptdev)
 
 void panthor_pwr_l2_power_off(struct panthor_device *ptdev)
 {
+	struct panthor_pwr *pwr = ptdev->pwr;
 	const u64 l2_allow_mask = PWR_STATUS_DOMAIN_ALLOWED(PWR_COMMAND_DOMAIN_L2);
-	const u64 pwr_status = gpu_read64(ptdev, PWR_STATUS);
+	const u64 pwr_status = gpu_read64(pwr->iomem, PWR_STATUS);
 
 	/* Abort if L2 power off constraints are not satisfied */
 	if (!(pwr_status & l2_allow_mask)) {
@@ -508,7 +534,8 @@ void panthor_pwr_l2_power_off(struct panthor_device *ptdev)
 
 int panthor_pwr_l2_power_on(struct panthor_device *ptdev)
 {
-	const u32 pwr_status = gpu_read64(ptdev, PWR_STATUS);
+	struct panthor_pwr *pwr = ptdev->pwr;
+	const u32 pwr_status = gpu_read64(pwr->iomem, PWR_STATUS);
 	const u32 l2_allow_mask = PWR_STATUS_DOMAIN_ALLOWED(PWR_COMMAND_DOMAIN_L2);
 	int ret;
 
diff --git a/drivers/gpu/drm/panthor/panthor_pwr_regs.h b/drivers/gpu/drm/panthor/panthor_pwr_regs.h
new file mode 100644
index 0000000000000..9cf7a715066f8
--- /dev/null
+++ b/drivers/gpu/drm/panthor/panthor_pwr_regs.h
@@ -0,0 +1,79 @@
+/* SPDX-License-Identifier: GPL-2.0 or MIT */
+/* Copyright 2026 ARM Limited. All rights reserved. */
+
+#ifndef __PANTHOR_PWR_REGS_H__
+#define __PANTHOR_PWR_REGS_H__
+
+#define PWR_CONTROL_BASE				0x800
+
+#define PWR_INT_BASE					0x0
+#define   PWR_IRQ_POWER_CHANGED_SINGLE			BIT(0)
+#define   PWR_IRQ_POWER_CHANGED_ALL			BIT(1)
+#define   PWR_IRQ_DELEGATION_CHANGED			BIT(2)
+#define   PWR_IRQ_RESET_COMPLETED			BIT(3)
+#define   PWR_IRQ_RETRACT_COMPLETED			BIT(4)
+#define   PWR_IRQ_INSPECT_COMPLETED			BIT(5)
+#define   PWR_IRQ_COMMAND_NOT_ALLOWED			BIT(30)
+#define   PWR_IRQ_COMMAND_INVALID			BIT(31)
+
+#define PWR_STATUS					0x20
+#define   PWR_STATUS_ALLOW_L2				BIT_U64(0)
+#define   PWR_STATUS_ALLOW_TILER			BIT_U64(1)
+#define   PWR_STATUS_ALLOW_SHADER			BIT_U64(8)
+#define   PWR_STATUS_ALLOW_BASE				BIT_U64(14)
+#define   PWR_STATUS_ALLOW_STACK			BIT_U64(15)
+#define   PWR_STATUS_DOMAIN_ALLOWED(x)			BIT_U64(x)
+#define   PWR_STATUS_DELEGATED_L2			BIT_U64(16)
+#define   PWR_STATUS_DELEGATED_TILER			BIT_U64(17)
+#define   PWR_STATUS_DELEGATED_SHADER			BIT_U64(24)
+#define   PWR_STATUS_DELEGATED_BASE			BIT_U64(30)
+#define   PWR_STATUS_DELEGATED_STACK			BIT_U64(31)
+#define   PWR_STATUS_DELEGATED_SHIFT			16
+#define   PWR_STATUS_DOMAIN_DELEGATED(x)		BIT_U64((x) + PWR_STATUS_DELEGATED_SHIFT)
+#define   PWR_STATUS_ALLOW_SOFT_RESET			BIT_U64(33)
+#define   PWR_STATUS_ALLOW_FAST_RESET			BIT_U64(34)
+#define   PWR_STATUS_POWER_PENDING			BIT_U64(41)
+#define   PWR_STATUS_RESET_PENDING			BIT_U64(42)
+#define   PWR_STATUS_RETRACT_PENDING			BIT_U64(43)
+#define   PWR_STATUS_INSPECT_PENDING			BIT_U64(44)
+
+#define PWR_COMMAND					0x28
+#define   PWR_COMMAND_POWER_UP				0x10
+#define   PWR_COMMAND_POWER_DOWN			0x11
+#define   PWR_COMMAND_DELEGATE				0x20
+#define   PWR_COMMAND_RETRACT				0x21
+#define   PWR_COMMAND_RESET_SOFT			0x31
+#define   PWR_COMMAND_RESET_FAST			0x32
+#define   PWR_COMMAND_INSPECT				0xF0
+#define   PWR_COMMAND_DOMAIN_L2				0
+#define   PWR_COMMAND_DOMAIN_TILER			1
+#define   PWR_COMMAND_DOMAIN_SHADER			8
+#define   PWR_COMMAND_DOMAIN_BASE			14
+#define   PWR_COMMAND_DOMAIN_STACK			15
+#define   PWR_COMMAND_SUBDOMAIN_RTU			BIT(0)
+#define   PWR_COMMAND_DEF(cmd, domain, subdomain)	\
+	(((subdomain) << 16) | ((domain) << 8) | (cmd))
+
+#define PWR_CMDARG					0x30
+
+#define PWR_L2_PRESENT					0x100
+#define PWR_L2_READY					0x108
+#define PWR_L2_PWRTRANS					0x110
+#define PWR_L2_PWRACTIVE				0x118
+#define PWR_TILER_PRESENT				0x140
+#define PWR_TILER_READY					0x148
+#define PWR_TILER_PWRTRANS				0x150
+#define PWR_TILER_PWRACTIVE				0x158
+#define PWR_SHADER_PRESENT				0x200
+#define PWR_SHADER_READY				0x208
+#define PWR_SHADER_PWRTRANS				0x210
+#define PWR_SHADER_PWRACTIVE				0x218
+#define PWR_BASE_PRESENT				0x380
+#define PWR_BASE_READY					0x388
+#define PWR_BASE_PWRTRANS				0x390
+#define PWR_BASE_PWRACTIVE				0x398
+#define PWR_STACK_PRESENT				0x3c0
+#define PWR_STACK_READY					0x3c8
+#define PWR_STACK_PWRTRANS				0x3d0
+
+#endif /* __PANTHOR_PWR_REGS_H__ */
diff --git a/drivers/gpu/drm/panthor/panthor_regs.h b/drivers/gpu/drm/panthor/panthor_regs.h
deleted file mode 100644
index 08bf06c452d69..0000000000000
--- a/drivers/gpu/drm/panthor/panthor_regs.h
+++ /dev/null
@@ -1,291 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 or MIT */
-/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
-/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
-/* Copyright 2023 Collabora ltd. */
-/*
- * Register definitions based on mali_kbase_gpu_regmap.h and
- * mali_kbase_gpu_regmap_csf.h
- * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved.
- */
-#ifndef __PANTHOR_REGS_H__
-#define __PANTHOR_REGS_H__
-
-#define GPU_ID						0x0
-#define   GPU_ARCH_MAJOR(x)				((x) >> 28)
-#define   GPU_ARCH_MINOR(x)				(((x) & GENMASK(27, 24)) >> 24)
-#define   GPU_ARCH_REV(x)				(((x) & GENMASK(23, 20)) >> 20)
-#define   GPU_PROD_MAJOR(x)				(((x) & GENMASK(19, 16)) >> 16)
-#define   GPU_VER_MAJOR(x)				(((x) & GENMASK(15, 12)) >> 12)
-#define   GPU_VER_MINOR(x)				(((x) & GENMASK(11, 4)) >> 4)
-#define   GPU_VER_STATUS(x)				((x) & GENMASK(3, 0))
-
-#define GPU_L2_FEATURES					0x4
-#define  GPU_L2_FEATURES_LINE_SIZE(x)			(1 << ((x) & GENMASK(7, 0)))
-
-#define GPU_CORE_FEATURES				0x8
-
-#define GPU_TILER_FEATURES				0xC
-#define GPU_MEM_FEATURES				0x10
-#define   GROUPS_L2_COHERENT				BIT(0)
-
-#define GPU_MMU_FEATURES				0x14
-#define  GPU_MMU_FEATURES_VA_BITS(x)			((x) & GENMASK(7, 0))
-#define  GPU_MMU_FEATURES_PA_BITS(x)			(((x) >> 8) & GENMASK(7, 0))
-#define GPU_AS_PRESENT					0x18
-#define GPU_CSF_ID					0x1C
-
-#define GPU_INT_RAWSTAT					0x20
-#define GPU_INT_CLEAR					0x24
-#define GPU_INT_MASK					0x28
-#define GPU_INT_STAT					0x2c
-#define   GPU_IRQ_FAULT					BIT(0)
-#define   GPU_IRQ_PROTM_FAULT				BIT(1)
-#define   GPU_IRQ_RESET_COMPLETED			BIT(8)
-#define   GPU_IRQ_POWER_CHANGED				BIT(9)
-#define   GPU_IRQ_POWER_CHANGED_ALL			BIT(10)
-#define   GPU_IRQ_CLEAN_CACHES_COMPLETED		BIT(17)
-#define   GPU_IRQ_DOORBELL_MIRROR			BIT(18)
-#define   GPU_IRQ_MCU_STATUS_CHANGED			BIT(19)
-#define GPU_CMD						0x30
-#define   GPU_CMD_DEF(type, payload)			((type) | ((payload) << 8))
-#define   GPU_SOFT_RESET				GPU_CMD_DEF(1, 1)
-#define   GPU_HARD_RESET				GPU_CMD_DEF(1, 2)
-#define   CACHE_CLEAN					BIT(0)
-#define   CACHE_INV					BIT(1)
-#define   GPU_FLUSH_CACHES(l2, lsc, oth)		\
-	  GPU_CMD_DEF(4, ((l2) << 0) | ((lsc) << 4) | ((oth) << 8))
-
-#define GPU_STATUS					0x34
-#define   GPU_STATUS_ACTIVE				BIT(0)
-#define   GPU_STATUS_PWR_ACTIVE				BIT(1)
-#define   GPU_STATUS_PAGE_FAULT				BIT(4)
-#define   GPU_STATUS_PROTM_ACTIVE			BIT(7)
-#define   GPU_STATUS_DBG_ENABLED			BIT(8)
-
-#define GPU_FAULT_STATUS				0x3C
-#define GPU_FAULT_ADDR					0x40
-#define GPU_L2_CONFIG					0x48
-#define   GPU_L2_CONFIG_ASN_HASH_ENABLE			BIT(24)
-
-#define GPU_PWR_KEY					0x50
-#define  GPU_PWR_KEY_UNLOCK				0x2968A819
-#define GPU_PWR_OVERRIDE0				0x54
-#define GPU_PWR_OVERRIDE1				0x58
-
-#define GPU_FEATURES					0x60
-#define   GPU_FEATURES_RAY_INTERSECTION			BIT(2)
-#define   GPU_FEATURES_RAY_TRAVERSAL			BIT(5)
-
-#define GPU_TIMESTAMP_OFFSET				0x88
-#define GPU_CYCLE_COUNT					0x90
-#define GPU_TIMESTAMP					0x98
-
-#define GPU_THREAD_MAX_THREADS				0xA0
-#define GPU_THREAD_MAX_WORKGROUP_SIZE			0xA4
-#define GPU_THREAD_MAX_BARRIER_SIZE			0xA8
-#define GPU_THREAD_FEATURES				0xAC
-
-#define GPU_TEXTURE_FEATURES(n)				(0xB0 + ((n) * 4))
-
-#define GPU_SHADER_PRESENT				0x100
-#define GPU_TILER_PRESENT				0x110
-#define GPU_L2_PRESENT					0x120
-
-#define SHADER_READY					0x140
-#define TILER_READY					0x150
-#define L2_READY					0x160
-
-#define SHADER_PWRON					0x180
-#define TILER_PWRON					0x190
-#define L2_PWRON					0x1A0
-
-#define SHADER_PWROFF					0x1C0
-#define TILER_PWROFF					0x1D0
-#define L2_PWROFF					0x1E0
-
-#define SHADER_PWRTRANS					0x200
-#define TILER_PWRTRANS					0x210
-#define L2_PWRTRANS					0x220
-
-#define SHADER_PWRACTIVE				0x240
-#define TILER_PWRACTIVE					0x250
-#define L2_PWRACTIVE					0x260
-
-#define GPU_REVID					0x280
-
-#define GPU_ASN_HASH(n)					(0x2C0 + ((n) * 4))
-
-#define GPU_COHERENCY_FEATURES				0x300
-#define GPU_COHERENCY_PROT_BIT(name)			BIT(GPU_COHERENCY_  ## name)
-
-#define GPU_COHERENCY_PROTOCOL				0x304
-#define   GPU_COHERENCY_ACE_LITE			0
-#define   GPU_COHERENCY_ACE				1
-#define   GPU_COHERENCY_NONE				31
-
-#define MCU_CONTROL					0x700
-#define MCU_CONTROL_ENABLE				1
-#define MCU_CONTROL_AUTO				2
-#define MCU_CONTROL_DISABLE				0
-
-#define MCU_STATUS					0x704
-#define MCU_STATUS_DISABLED				0
-#define MCU_STATUS_ENABLED				1
-#define MCU_STATUS_HALT					2
-#define MCU_STATUS_FATAL				3
-
-/* Job Control regs */
-#define JOB_INT_RAWSTAT					0x1000
-#define JOB_INT_CLEAR					0x1004
-#define JOB_INT_MASK					0x1008
-#define JOB_INT_STAT					0x100c
-#define   JOB_INT_GLOBAL_IF				BIT(31)
-#define   JOB_INT_CSG_IF(x)				BIT(x)
-
-/* MMU regs */
-#define MMU_INT_RAWSTAT					0x2000
-#define MMU_INT_CLEAR					0x2004
-#define MMU_INT_MASK					0x2008
-#define MMU_INT_STAT					0x200c
-
-/* AS_COMMAND register commands */
-
-#define MMU_BASE					0x2400
-#define MMU_AS_SHIFT					6
-#define MMU_AS(as)					(MMU_BASE + ((as) << MMU_AS_SHIFT))
-
-#define AS_TRANSTAB(as)					(MMU_AS(as) + 0x0)
-#define AS_MEMATTR(as)					(MMU_AS(as) + 0x8)
-#define   AS_MEMATTR_AARCH64_INNER_ALLOC_IMPL		(2 << 2)
-#define   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(w, r)	((3 << 2) | \
-							 ((w) ? BIT(0) : 0) | \
-							 ((r) ? BIT(1) : 0))
-#define   AS_MEMATTR_AARCH64_SH_MIDGARD_INNER		(0 << 4)
-#define   AS_MEMATTR_AARCH64_SH_CPU_INNER		(1 << 4)
-#define   AS_MEMATTR_AARCH64_SH_CPU_INNER_SHADER_COH	(2 << 4)
-#define   AS_MEMATTR_AARCH64_SHARED			(0 << 6)
-#define   AS_MEMATTR_AARCH64_INNER_OUTER_NC		(1 << 6)
-#define   AS_MEMATTR_AARCH64_INNER_OUTER_WB		(2 << 6)
-#define   AS_MEMATTR_AARCH64_FAULT			(3 << 6)
-#define AS_LOCKADDR(as)					(MMU_AS(as) + 0x10)
-#define AS_COMMAND(as)					(MMU_AS(as) + 0x18)
-#define   AS_COMMAND_NOP				0
-#define   AS_COMMAND_UPDATE				1
-#define   AS_COMMAND_LOCK				2
-#define   AS_COMMAND_UNLOCK				3
-#define   AS_COMMAND_FLUSH_PT				4
-#define   AS_COMMAND_FLUSH_MEM				5
-#define   AS_LOCK_REGION_MIN_SIZE			(1ULL << 15)
-#define AS_FAULTSTATUS(as)				(MMU_AS(as) + 0x1C)
-#define  AS_FAULTSTATUS_ACCESS_TYPE_MASK		(0x3 << 8)
-#define  AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC		(0x0 << 8)
-#define  AS_FAULTSTATUS_ACCESS_TYPE_EX			(0x1 << 8)
-#define  AS_FAULTSTATUS_ACCESS_TYPE_READ		(0x2 << 8)
-#define  AS_FAULTSTATUS_ACCESS_TYPE_WRITE		(0x3 << 8)
-#define AS_FAULTADDRESS(as)				(MMU_AS(as) + 0x20)
-#define AS_STATUS(as)					(MMU_AS(as) + 0x28)
-#define   AS_STATUS_AS_ACTIVE				BIT(0)
-#define AS_TRANSCFG(as)					(MMU_AS(as) + 0x30)
-#define   AS_TRANSCFG_ADRMODE_UNMAPPED			(1 << 0)
-#define   AS_TRANSCFG_ADRMODE_IDENTITY			(2 << 0)
-#define   AS_TRANSCFG_ADRMODE_AARCH64_4K		(6 << 0)
-#define   AS_TRANSCFG_ADRMODE_AARCH64_64K		(8 << 0)
-#define   AS_TRANSCFG_INA_BITS(x)			((x) << 6)
-#define   AS_TRANSCFG_OUTA_BITS(x)			((x) << 14)
-#define   AS_TRANSCFG_SL_CONCAT				BIT(22)
-#define   AS_TRANSCFG_PTW_MEMATTR_NC			(1 << 24)
-#define   AS_TRANSCFG_PTW_MEMATTR_WB			(2 << 24)
-#define   AS_TRANSCFG_PTW_SH_NS				(0 << 28)
-#define   AS_TRANSCFG_PTW_SH_OS				(2 << 28)
-#define   AS_TRANSCFG_PTW_SH_IS				(3 << 28)
-#define   AS_TRANSCFG_PTW_RA				BIT(30)
-#define   AS_TRANSCFG_DISABLE_HIER_AP			BIT(33)
-#define   AS_TRANSCFG_DISABLE_AF_FAULT			BIT(34)
-#define   AS_TRANSCFG_WXN				BIT(35)
-#define   AS_TRANSCFG_XREADABLE				BIT(36)
-#define AS_FAULTEXTRA(as)				(MMU_AS(as) + 0x38)
-
-#define CSF_GPU_LATEST_FLUSH_ID				0x10000
-
-#define CSF_DOORBELL(i)					(0x80000 + ((i) * 0x10000))
-#define CSF_GLB_DOORBELL_ID				0
-
-/* PWR Control registers */
-
-#define PWR_CONTROL_BASE				0x800
-#define PWR_CTRL_REG(x)					(PWR_CONTROL_BASE + (x))
-
-#define PWR_INT_RAWSTAT					PWR_CTRL_REG(0x0)
-#define PWR_INT_CLEAR					PWR_CTRL_REG(0x4)
-#define PWR_INT_MASK					PWR_CTRL_REG(0x8)
-#define PWR_INT_STAT					PWR_CTRL_REG(0xc)
-#define   PWR_IRQ_POWER_CHANGED_SINGLE			BIT(0)
-#define   PWR_IRQ_POWER_CHANGED_ALL			BIT(1)
-#define   PWR_IRQ_DELEGATION_CHANGED			BIT(2)
-#define   PWR_IRQ_RESET_COMPLETED			BIT(3)
-#define   PWR_IRQ_RETRACT_COMPLETED			BIT(4)
-#define   PWR_IRQ_INSPECT_COMPLETED			BIT(5)
-#define   PWR_IRQ_COMMAND_NOT_ALLOWED			BIT(30)
-#define   PWR_IRQ_COMMAND_INVALID			BIT(31)
-
-#define PWR_STATUS					PWR_CTRL_REG(0x20)
-#define   PWR_STATUS_ALLOW_L2				BIT_U64(0)
-#define   PWR_STATUS_ALLOW_TILER			BIT_U64(1)
-#define   PWR_STATUS_ALLOW_SHADER			BIT_U64(8)
-#define   PWR_STATUS_ALLOW_BASE				BIT_U64(14)
-#define   PWR_STATUS_ALLOW_STACK			BIT_U64(15)
-#define   PWR_STATUS_DOMAIN_ALLOWED(x)			BIT_U64(x)
-#define   PWR_STATUS_DELEGATED_L2			BIT_U64(16)
-#define   PWR_STATUS_DELEGATED_TILER			BIT_U64(17)
-#define   PWR_STATUS_DELEGATED_SHADER			BIT_U64(24)
-#define   PWR_STATUS_DELEGATED_BASE			BIT_U64(30)
-#define   PWR_STATUS_DELEGATED_STACK			BIT_U64(31)
-#define   PWR_STATUS_DELEGATED_SHIFT			16
-#define   PWR_STATUS_DOMAIN_DELEGATED(x)		BIT_U64((x) + PWR_STATUS_DELEGATED_SHIFT)
-#define   PWR_STATUS_ALLOW_SOFT_RESET			BIT_U64(33)
-#define   PWR_STATUS_ALLOW_FAST_RESET			BIT_U64(34)
-#define   PWR_STATUS_POWER_PENDING			BIT_U64(41)
-#define   PWR_STATUS_RESET_PENDING			BIT_U64(42)
-#define   PWR_STATUS_RETRACT_PENDING			BIT_U64(43)
-#define   PWR_STATUS_INSPECT_PENDING			BIT_U64(44)
-
-#define PWR_COMMAND					PWR_CTRL_REG(0x28)
-#define   PWR_COMMAND_POWER_UP				0x10
-#define   PWR_COMMAND_POWER_DOWN			0x11
-#define   PWR_COMMAND_DELEGATE				0x20
-#define   PWR_COMMAND_RETRACT				0x21
-#define   PWR_COMMAND_RESET_SOFT			0x31
-#define   PWR_COMMAND_RESET_FAST			0x32
-#define   PWR_COMMAND_INSPECT				0xF0
-#define   PWR_COMMAND_DOMAIN_L2				0
-#define   PWR_COMMAND_DOMAIN_TILER			1
-#define   PWR_COMMAND_DOMAIN_SHADER			8
-#define   PWR_COMMAND_DOMAIN_BASE			14
-#define   PWR_COMMAND_DOMAIN_STACK			15
-#define   PWR_COMMAND_SUBDOMAIN_RTU			BIT(0)
-#define   PWR_COMMAND_DEF(cmd, domain, subdomain)	\
-	(((subdomain) << 16) | ((domain) << 8) | (cmd))
-
-#define PWR_CMDARG					PWR_CTRL_REG(0x30)
-
-#define PWR_L2_PRESENT					PWR_CTRL_REG(0x100)
-#define PWR_L2_READY					PWR_CTRL_REG(0x108)
-#define PWR_L2_PWRTRANS					PWR_CTRL_REG(0x110)
-#define PWR_L2_PWRACTIVE				PWR_CTRL_REG(0x118)
-#define PWR_TILER_PRESENT				PWR_CTRL_REG(0x140)
-#define PWR_TILER_READY					PWR_CTRL_REG(0x148)
-#define PWR_TILER_PWRTRANS				PWR_CTRL_REG(0x150)
-#define PWR_TILER_PWRACTIVE				PWR_CTRL_REG(0x158)
-#define PWR_SHADER_PRESENT				PWR_CTRL_REG(0x200)
-#define PWR_SHADER_READY				PWR_CTRL_REG(0x208)
-#define PWR_SHADER_PWRTRANS				PWR_CTRL_REG(0x210)
-#define PWR_SHADER_PWRACTIVE				PWR_CTRL_REG(0x218)
-#define PWR_BASE_PRESENT				PWR_CTRL_REG(0x380)
-#define PWR_BASE_READY					PWR_CTRL_REG(0x388)
-#define PWR_BASE_PWRTRANS				PWR_CTRL_REG(0x390)
-#define PWR_BASE_PWRACTIVE				PWR_CTRL_REG(0x398)
-#define PWR_STACK_PRESENT				PWR_CTRL_REG(0x3c0)
-#define PWR_STACK_READY					PWR_CTRL_REG(0x3c8)
-#define PWR_STACK_PWRTRANS				PWR_CTRL_REG(0x3d0)
-
-#endif
diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c
index 2fe04d0f0e3a8..5b34032deff81 100644
--- a/drivers/gpu/drm/panthor/panthor_sched.c
+++ b/drivers/gpu/drm/panthor/panthor_sched.c
@@ -3,7 +3,7 @@
 
 #include <drm/drm_drv.h>
 #include <drm/drm_exec.h>
-#include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_file.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_print.h>
 #include <drm/gpu_scheduler.h>
@@ -28,11 +28,12 @@
 #include "panthor_devfreq.h"
 #include "panthor_device.h"
 #include "panthor_fw.h"
+#include "panthor_fw_regs.h"
 #include "panthor_gem.h"
 #include "panthor_gpu.h"
+#include "panthor_gpu_regs.h"
 #include "panthor_heap.h"
 #include "panthor_mmu.h"
-#include "panthor_regs.h"
 #include "panthor_sched.h"
 
 /**
@@ -221,7 +222,7 @@ struct panthor_scheduler {
 	/** @groups: Various lists used to classify groups. */
 	struct {
 		/**
-		 * @runnable: Runnable group lists.
+		 * @groups.runnable: Runnable group lists.
 		 *
 		 * When a group has queues that want to execute something,
 		 * its panthor_group::run_node should be inserted here.
@@ -231,7 +232,7 @@ struct panthor_scheduler {
 		struct list_head runnable[PANTHOR_CSG_PRIORITY_COUNT];
 
 		/**
-		 * @idle: Idle group lists.
+		 * @groups.idle: Idle group lists.
 		 *
 		 * When all queues of a group are idle (either because they
 		 * have nothing to execute, or because they are blocked), the
@@ -242,7 +243,7 @@ struct panthor_scheduler {
 		struct list_head idle[PANTHOR_CSG_PRIORITY_COUNT];
 
 		/**
-		 * @waiting: List of groups whose queues are blocked on a
+		 * @groups.waiting: List of groups whose queues are blocked on a
 		 * synchronization object.
 		 *
 		 * Insert panthor_group::wait_node here when a group is waiting
@@ -283,17 +284,17 @@ struct panthor_scheduler {
 
 	/** @pm: Power management related fields. */
 	struct {
-		/** @has_ref: True if the scheduler owns a runtime PM reference. */
+		/** @pm.has_ref: True if the scheduler owns a runtime PM reference. */
 		bool has_ref;
 	} pm;
 
 	/** @reset: Reset related fields. */
 	struct {
-		/** @lock: Lock protecting the other reset fields. */
+		/** @reset.lock: Lock protecting the other reset fields. */
 		struct mutex lock;
 
 		/**
-		 * @in_progress: True if a reset is in progress.
+		 * @reset.in_progress: True if a reset is in progress.
 		 *
 		 * Set to true in panthor_sched_pre_reset() and back to false in
 		 * panthor_sched_post_reset().
@@ -301,7 +302,7 @@ struct panthor_scheduler {
 		atomic_t in_progress;
 
 		/**
-		 * @stopped_groups: List containing all groups that were stopped
+		 * @reset.stopped_groups: List containing all groups that were stopped
 		 * before a reset.
 		 *
 		 * Insert panthor_group::run_node in the pre_reset path.
@@ -395,19 +396,19 @@ struct panthor_queue {
 
 	/** @iface: Firmware interface. */
 	struct {
-		/** @mem: FW memory allocated for this interface. */
+		/** @iface.mem: FW memory allocated for this interface. */
 		struct panthor_kernel_bo *mem;
 
-		/** @input: Input interface. */
+		/** @iface.input: Input interface. */
 		struct panthor_fw_ringbuf_input_iface *input;
 
-		/** @output: Output interface. */
+		/** @iface.output: Output interface. */
 		const struct panthor_fw_ringbuf_output_iface *output;
 
-		/** @input_fw_va: FW virtual address of the input interface buffer. */
+		/** @iface.input_fw_va: FW virtual address of the input interface buffer. */
 		u32 input_fw_va;
 
-		/** @output_fw_va: FW virtual address of the output interface buffer. */
+		/** @iface.output_fw_va: FW virtual address of the output interface buffer. */
 		u32 output_fw_va;
 	} iface;
 
@@ -416,26 +417,26 @@ struct panthor_queue {
 	 * queue is waiting on.
 	 */
 	struct {
-		/** @gpu_va: GPU address of the synchronization object. */
+		/** @syncwait.gpu_va: GPU address of the synchronization object. */
 		u64 gpu_va;
 
-		/** @ref: Reference value to compare against. */
+		/** @syncwait.ref: Reference value to compare against. */
 		u64 ref;
 
-		/** @gt: True if this is a greater-than test. */
+		/** @syncwait.gt: True if this is a greater-than test. */
 		bool gt;
 
-		/** @sync64: True if this is a 64-bit sync object. */
+		/** @syncwait.sync64: True if this is a 64-bit sync object. */
 		bool sync64;
 
-		/** @bo: Buffer object holding the synchronization object. */
+		/** @syncwait.obj: Buffer object holding the synchronization object. */
 		struct drm_gem_object *obj;
 
-		/** @offset: Offset of the synchronization object inside @bo. */
+		/** @syncwait.offset: Offset of the synchronization object inside @bo. */
 		u64 offset;
 
 		/**
-		 * @kmap: Kernel mapping of the buffer object holding the
+		 * @syncwait.kmap: Kernel mapping of the buffer object holding the
 		 * synchronization object.
 		 */
 		void *kmap;
@@ -443,21 +444,21 @@ struct panthor_queue {
 
 	/** @fence_ctx: Fence context fields. */
 	struct {
-		/** @lock: Used to protect access to all fences allocated by this context. */
+		/** @fence_ctx.lock: Used to protect access to all fences allocated by this context. */
 		spinlock_t lock;
 
 		/**
-		 * @id: Fence context ID.
+		 * @fence_ctx.id: Fence context ID.
 		 *
 		 * Allocated with dma_fence_context_alloc().
 		 */
 		u64 id;
 
-		/** @seqno: Sequence number of the last initialized fence. */
+		/** @fence_ctx.seqno: Sequence number of the last initialized fence. */
 		atomic64_t seqno;
 
 		/**
-		 * @last_fence: Fence of the last submitted job.
+		 * @fence_ctx.last_fence: Fence of the last submitted job.
 		 *
 		 * We return this fence when we get an empty command stream.
 		 * This way, we are guaranteed that all earlier jobs have completed
@@ -467,7 +468,7 @@ struct panthor_queue {
 		struct dma_fence *last_fence;
 
 		/**
-		 * @in_flight_jobs: List containing all in-flight jobs.
+		 * @fence_ctx.in_flight_jobs: List containing all in-flight jobs.
 		 *
 		 * Used to keep track and signal panthor_job::done_fence when the
 		 * synchronization object attached to the queue is signaled.
@@ -477,13 +478,13 @@ struct panthor_queue {
 
 	/** @profiling: Job profiling data slots and access information. */
 	struct {
-		/** @slots: Kernel BO holding the slots. */
+		/** @profiling.slots: Kernel BO holding the slots. */
 		struct panthor_kernel_bo *slots;
 
-		/** @slot_count: Number of jobs ringbuffer can hold at once. */
+		/** @profiling.slot_count: Number of jobs ringbuffer can hold at once. */
 		u32 slot_count;
 
-		/** @seqno: Index of the next available profiling information slot. */
+		/** @profiling.seqno: Index of the next available profiling information slot. */
 		u32 seqno;
 	} profiling;
 };
@@ -627,7 +628,7 @@ struct panthor_group {
 
 	/** @fdinfo: Per-file info exposed through /proc/<process>/fdinfo */
 	struct {
-		/** @data: Total sampled values for jobs in queues from this group. */
+		/** @fdinfo.data: Total sampled values for jobs in queues from this group. */
 		struct panthor_gpu_usage data;
 
 		/**
@@ -805,15 +806,15 @@ struct panthor_job {
 
 	/** @call_info: Information about the userspace command stream call. */
 	struct {
-		/** @start: GPU address of the userspace command stream. */
+		/** @call_info.start: GPU address of the userspace command stream. */
 		u64 start;
 
-		/** @size: Size of the userspace command stream. */
+		/** @call_info.size: Size of the userspace command stream. */
 		u32 size;
 
 		/**
-		 * @latest_flush: Flush ID at the time the userspace command
-		 * stream was built.
+		 * @call_info.latest_flush: Flush ID at the time the userspace
+		 * command stream was built.
 		 *
 		 * Needed for the flush reduction mechanism.
 		 */
@@ -822,10 +823,10 @@ struct panthor_job {
 
 	/** @ringbuf: Position of this job is in the ring buffer. */
 	struct {
-		/** @start: Start offset. */
+		/** @ringbuf.start: Start offset. */
 		u64 start;
 
-		/** @end: End offset. */
+		/** @ringbuf.end: End offset. */
 		u64 end;
 	} ringbuf;
 
@@ -840,10 +841,10 @@ struct panthor_job {
 
 	/** @profiling: Job profiling information. */
 	struct {
-		/** @mask: Current device job profiling enablement bitmask. */
+		/** @profiling.mask: Current device job profiling enablement bitmask. */
 		u32 mask;
 
-		/** @slot: Job index in the profiling slots BO. */
+		/** @profiling.slot: Job index in the profiling slots BO. */
 		u32 slot;
 	} profiling;
 };
@@ -871,8 +872,7 @@ panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue
 	int ret;
 
 	if (queue->syncwait.kmap) {
-		bo = container_of(queue->syncwait.obj,
-				  struct panthor_gem_object, base.base);
+		bo = to_panthor_bo(queue->syncwait.obj);
 		goto out_sync;
 	}
 
@@ -882,7 +882,7 @@ panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue
 	if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo)))
 		goto err_put_syncwait_obj;
 
-	queue->syncwait.obj = &bo->base.base;
+	queue->syncwait.obj = &bo->base;
 	ret = drm_gem_vmap(queue->syncwait.obj, &map);
 	if (drm_WARN_ON(&ptdev->base, ret))
 		goto err_put_syncwait_obj;
@@ -896,7 +896,7 @@ panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue
 	 * panthor_gem_sync() is a NOP if map_wc=true, so no need to check
 	 * it here.
 	 */
-	panthor_gem_sync(&bo->base.base,
+	panthor_gem_sync(&bo->base,
 			 DRM_PANTHOR_BO_SYNC_CPU_CACHE_FLUSH_AND_INVALIDATE,
 			 queue->syncwait.offset,
 			 queue->syncwait.sync64 ?
@@ -1894,6 +1894,8 @@ static void process_fw_events_work(struct work_struct *work)
 
 /**
  * panthor_sched_report_fw_events() - Report FW events to the scheduler.
+ * @ptdev: Device.
+ * @events: Bitmask of pending FW events to report.
  */
 void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events)
 {
@@ -2779,6 +2781,7 @@ static void panthor_group_start(struct panthor_group *group)
 
 /**
  * panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler.
+ * @ptdev: Device.
  */
 void panthor_sched_report_mmu_fault(struct panthor_device *ptdev)
 {
@@ -3370,7 +3373,7 @@ queue_run_job(struct drm_sched_job *sched_job)
 		if (resume_tick)
 			sched_resume_tick(ptdev);
 
-		gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1);
+		panthor_fw_ring_doorbell(ptdev, queue->doorbell_id);
 		if (!sched->pm.has_ref &&
 		    !(group->blocked_queues & BIT(job->queue_idx))) {
 			pm_runtime_get(ptdev->base.dev);
@@ -3487,7 +3490,6 @@ group_create_queue(struct panthor_group *group,
 	struct drm_sched_init_args sched_args = {
 		.ops = &panthor_queue_sched_ops,
 		.submit_wq = group->ptdev->scheduler->wq,
-		.num_rqs = 1,
 		/*
 		 * The credit limit argument tells us the total number of
 		 * instructions across all CS slots in the ringbuffer, with
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 75d47e6b91422..a026bd35ef485 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -382,7 +382,7 @@ static void qxl_crtc_update_monitors_config(struct drm_crtc *crtc,
 }
 
 static void qxl_crtc_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
@@ -473,7 +473,7 @@ static const struct drm_framebuffer_funcs qxl_fb_funcs = {
 };
 
 static void qxl_crtc_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	qxl_crtc_update_monitors_config(crtc, "enable");
 
@@ -481,7 +481,7 @@ static void qxl_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void qxl_crtc_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_off(crtc);
 
@@ -495,7 +495,7 @@ static const struct drm_crtc_helper_funcs qxl_crtc_helper_funcs = {
 };
 
 static int qxl_primary_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -662,7 +662,7 @@ static void qxl_free_cursor(struct qxl_bo *cursor_bo)
 }
 
 static void qxl_primary_atomic_update(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -695,7 +695,7 @@ static void qxl_primary_atomic_update(struct drm_plane *plane,
 }
 
 static void qxl_primary_atomic_disable(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -712,7 +712,7 @@ static void qxl_primary_atomic_disable(struct drm_plane *plane,
 }
 
 static void qxl_cursor_atomic_update(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -729,7 +729,7 @@ static void qxl_cursor_atomic_update(struct drm_plane *plane,
 }
 
 static void qxl_cursor_atomic_disable(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index 51a3e0fc2f56b..a13635443833e 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -9063,7 +9063,7 @@ static u32 dce8_available_bandwidth(struct dce8_wm_params *wm)
 	u32 data_return_bandwidth = dce8_data_return_bandwidth(wm);
 	u32 dmif_req_bandwidth = dce8_dmif_request_bandwidth(wm);
 
-	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
+	return min3(dram_bandwidth, data_return_bandwidth, dmif_req_bandwidth);
 }
 
 /**
diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c
index 3142ef4da7f4a..9196f85db9ce0 100644
--- a/drivers/gpu/drm/radeon/evergreen_cs.c
+++ b/drivers/gpu/drm/radeon/evergreen_cs.c
@@ -312,8 +312,10 @@ static int evergreen_surface_check(struct radeon_cs_parser *p,
 	case ARRAY_2D_TILED_THIN1:
 		return evergreen_surface_check_2d(p, surf, prefix);
 	default:
-		dev_warn(p->dev, "%s:%d %s invalid array mode %d\n",
-				__func__, __LINE__, prefix, surf->mode);
+		if (prefix) {
+			dev_warn(p->dev, "%s:%d %s invalid array mode %d\n",
+					__func__, __LINE__, prefix, surf->mode);
+		}
 		return -EINVAL;
 	}
 	return -EINVAL;
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index 20fc87409f2e4..8ce180e22d1d1 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -28,6 +28,7 @@
 
 #include <linux/debugfs.h>
 #include <linux/iosys-map.h>
+#include <linux/overflow.h>
 #include <linux/pci.h>
 
 #include <drm/drm_device.h>
@@ -812,6 +813,7 @@ int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tile
 	int aligned = width;
 	int align_large = (ASIC_IS_AVIVO(rdev)) || tiled;
 	int pitch_mask = 0;
+	int pitch;
 
 	switch (cpp) {
 	case 1:
@@ -826,9 +828,12 @@ int radeon_align_pitch(struct radeon_device *rdev, int width, int cpp, bool tile
 		break;
 	}
 
-	aligned += pitch_mask;
+	if (check_add_overflow(aligned, pitch_mask, &aligned))
+		return 0;
 	aligned &= ~pitch_mask;
-	return aligned * cpp;
+	if (check_mul_overflow(aligned, cpp, &pitch))
+		return 0;
+	return pitch;
 }
 
 int radeon_mode_dumb_create(struct drm_file *file_priv,
@@ -842,8 +847,12 @@ int radeon_mode_dumb_create(struct drm_file *file_priv,
 
 	args->pitch = radeon_align_pitch(rdev, args->width,
 					 DIV_ROUND_UP(args->bpp, 8), 0);
+	if (!args->pitch)
+		return -EINVAL;
 	args->size = (u64)args->pitch * args->height;
 	args->size = ALIGN(args->size, PAGE_SIZE);
+	if (!args->size)
+		return -EINVAL;
 
 	r = radeon_gem_object_create(rdev, args->size, 0,
 				     RADEON_GEM_DOMAIN_VRAM, 0,
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index 581ae20c46e4b..a5dff072c1ac0 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -356,8 +356,10 @@ int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,
 
 	/* restore the saved ring content */
 	r = radeon_ring_lock(rdev, ring, size);
-	if (r)
+	if (r) {
+		kvfree(data);
 		return r;
+	}
 
 	for (i = 0; i < size; ++i) {
 		radeon_ring_write(ring, data[i]);
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index 26197aceb001c..4681ff28630a7 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -2150,7 +2150,7 @@ static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
 	u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
 	u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
 
-	return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
+	return min3(dram_bandwidth, data_return_bandwidth, dmif_req_bandwidth);
 }
 
 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c
index 1a246ebbfc613..6c07c406f7258 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_crtc.c
@@ -689,7 +689,7 @@ static void rcar_du_crtc_stop(struct rcar_du_crtc *rcrtc)
  */
 
 static int rcar_du_crtc_atomic_check(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -720,7 +720,7 @@ static int rcar_du_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void rcar_du_crtc_atomic_enable(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 	struct rcar_du_crtc_state *rstate = to_rcar_crtc_state(crtc->state);
@@ -768,7 +768,7 @@ static void rcar_du_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void rcar_du_crtc_atomic_disable(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 									 crtc);
@@ -812,7 +812,7 @@ static void rcar_du_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void rcar_du_crtc_atomic_begin(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 
@@ -841,7 +841,7 @@ static void rcar_du_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void rcar_du_crtc_atomic_flush(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 	struct drm_device *dev = rcrtc->crtc.dev;
@@ -1103,7 +1103,7 @@ static int rcar_du_crtc_set_crc_source(struct drm_crtc *crtc,
 	struct rcar_du_crtc *rcrtc = to_rcar_crtc(crtc);
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_crtc_state *crtc_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	enum vsp1_du_crc_source source;
 	unsigned int index;
 	int ret;
@@ -1117,7 +1117,7 @@ static int rcar_du_crtc_set_crc_source(struct drm_crtc *crtc,
 	/* Perform an atomic commit to set the CRC source. */
 	drm_modeset_acquire_init(&ctx, 0);
 
-	state = drm_atomic_state_alloc(crtc->dev);
+	state = drm_atomic_commit_alloc(crtc->dev);
 	if (!state) {
 		ret = -ENOMEM;
 		goto unlock;
@@ -1140,12 +1140,12 @@ static int rcar_du_crtc_set_crc_source(struct drm_crtc *crtc,
 	}
 
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 		goto retry;
 	}
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 
 unlock:
 	drm_modeset_drop_locks(&ctx);
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c
index 7ecec7b04a8d0..db2088529b480 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.c
@@ -51,7 +51,7 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 {
 	struct rcar_du_encoder *renc;
 	struct drm_connector *connector;
-	struct drm_bridge *bridge;
+	struct drm_bridge *bridge __free(drm_bridge_put) = NULL;
 	int ret;
 
 	/*
@@ -70,19 +70,28 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 		bridge = devm_drm_panel_bridge_add_typed(rcdu->dev, panel,
 							 DRM_MODE_CONNECTOR_DPI);
 		if (IS_ERR(bridge))
-			return PTR_ERR(bridge);
+			return PTR_ERR(no_free_ptr(bridge));
+
+		/*
+		 * The reference taken by devm_drm_panel_bridge_add_typed() is
+		 * released automatically. Take a second one for the __free()
+		 * when this function will return.
+		 */
+		drm_bridge_get(bridge);
 	} else {
-		bridge = of_drm_find_bridge(enc_node);
+		bridge = of_drm_find_and_get_bridge(enc_node);
 		if (!bridge)
 			return -EPROBE_DEFER;
 
 		if (output == RCAR_DU_OUTPUT_LVDS0 ||
 		    output == RCAR_DU_OUTPUT_LVDS1)
-			rcdu->lvds[output - RCAR_DU_OUTPUT_LVDS0] = bridge;
+			rcdu->lvds[output - RCAR_DU_OUTPUT_LVDS0] =
+				drm_bridge_get(bridge);
 
 		if (output == RCAR_DU_OUTPUT_DSI0 ||
 		    output == RCAR_DU_OUTPUT_DSI1)
-			rcdu->dsi[output - RCAR_DU_OUTPUT_DSI0] = bridge;
+			rcdu->dsi[output - RCAR_DU_OUTPUT_DSI0] =
+				drm_bridge_get(bridge);
 	}
 
 	/*
@@ -133,5 +142,15 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 		return PTR_ERR(connector);
 	}
 
-	return drm_connector_attach_encoder(connector, &renc->base);
+	return 0;
+}
+
+void rcar_du_encoder_cleanup(struct rcar_du_device *rcdu)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(rcdu->lvds); i++)
+		drm_bridge_put(rcdu->lvds[i]);
+	for (i = 0; i < ARRAY_SIZE(rcdu->dsi); i++)
+		drm_bridge_put(rcdu->dsi[i]);
 }
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.h b/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.h
index e5ec8fbb3979e..b2b5e93f30f8f 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.h
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_encoder.h
@@ -25,5 +25,6 @@ struct rcar_du_encoder {
 int rcar_du_encoder_init(struct rcar_du_device *rcdu,
 			 enum rcar_du_output output,
 			 struct device_node *enc_node);
+void rcar_du_encoder_cleanup(struct rcar_du_device *rcdu);
 
 #endif /* __RCAR_DU_ENCODER_H__ */
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_kms.c
index b2d0e4651e352..9228a7ad0dfa4 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_kms.c
@@ -503,7 +503,7 @@ rcar_du_fb_create(struct drm_device *dev, struct drm_file *file_priv,
  */
 
 static int rcar_du_atomic_check(struct drm_device *dev,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct rcar_du_device *rcdu = to_rcar_du_device(dev);
 	int ret;
@@ -518,7 +518,7 @@ static int rcar_du_atomic_check(struct drm_device *dev,
 	return rcar_du_atomic_check_planes(dev, state);
 }
 
-static void rcar_du_atomic_commit_tail(struct drm_atomic_state *old_state)
+static void rcar_du_atomic_commit_tail(struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = old_state->dev;
 	struct rcar_du_device *rcdu = to_rcar_du_device(dev);
@@ -832,6 +832,8 @@ static void rcar_du_modeset_cleanup(struct drm_device *dev, void *res)
 
 		put_device(cmm->dev);
 	}
+
+	rcar_du_encoder_cleanup(rcdu);
 }
 
 int rcar_du_modeset_init(struct rcar_du_device *rcdu)
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c
index 01840feabdbed..8870766b9e541 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.c
@@ -127,7 +127,7 @@ static int rcar_du_plane_hwalloc(struct rcar_du_plane *plane,
 }
 
 int rcar_du_atomic_check_planes(struct drm_device *dev,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct rcar_du_device *rcdu = to_rcar_du_device(dev);
 	unsigned int group_freed_planes[RCAR_DU_MAX_GROUPS] = { 0, };
@@ -635,7 +635,7 @@ int __rcar_du_plane_atomic_check(struct drm_plane *plane,
 }
 
 static int rcar_du_plane_atomic_check(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -646,7 +646,7 @@ static int rcar_du_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void rcar_du_plane_atomic_update(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.h b/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.h
index e9e59c5e70d54..49970f098b7be 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.h
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_plane.h
@@ -64,7 +64,7 @@ to_rcar_plane_state(struct drm_plane_state *state)
 }
 
 int rcar_du_atomic_check_planes(struct drm_device *dev,
-				struct drm_atomic_state *state);
+				struct drm_atomic_commit *state);
 
 int __rcar_du_plane_atomic_check(struct drm_plane *plane,
 				 struct drm_plane_state *state,
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c b/drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c
index a4a49dcd82330..ae9f381b03c86 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_du_vsp.c
@@ -87,12 +87,12 @@ void rcar_du_vsp_enable(struct rcar_du_crtc *crtc)
 
 	__rcar_du_plane_setup(crtc->group, &state);
 
-	vsp1_du_setup_lif(crtc->vsp->vsp, crtc->vsp_pipe, &cfg);
+	vsp1_du_enable(crtc->vsp->vsp, crtc->vsp_pipe, &cfg);
 }
 
 void rcar_du_vsp_disable(struct rcar_du_crtc *crtc)
 {
-	vsp1_du_setup_lif(crtc->vsp->vsp, crtc->vsp_pipe, NULL);
+	vsp1_du_disable(crtc->vsp->vsp, crtc->vsp_pipe);
 }
 
 void rcar_du_vsp_atomic_begin(struct rcar_du_crtc *crtc)
@@ -365,7 +365,7 @@ static void rcar_du_vsp_plane_cleanup_fb(struct drm_plane *plane,
 }
 
 static int rcar_du_vsp_plane_atomic_check(struct drm_plane *plane,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -376,7 +376,7 @@ static int rcar_du_vsp_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void rcar_du_vsp_plane_atomic_update(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c b/drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c
index 227818e37390f..154410745a74b 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_lvds.c
@@ -351,7 +351,7 @@ static enum rcar_lvds_mode rcar_lvds_get_lvds_mode(struct rcar_lvds *lvds,
 }
 
 static void rcar_lvds_enable(struct drm_bridge *bridge,
-			     struct drm_atomic_state *state,
+			     struct drm_atomic_commit *state,
 			     struct drm_crtc *crtc,
 			     struct drm_connector *connector)
 {
@@ -582,7 +582,7 @@ EXPORT_SYMBOL_GPL(rcar_lvds_pclk_disable);
  */
 
 static void rcar_lvds_atomic_enable(struct drm_bridge *bridge,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_crtc *crtc;
@@ -595,7 +595,7 @@ static void rcar_lvds_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void rcar_lvds_atomic_disable(struct drm_bridge *bridge,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct rcar_lvds *lvds = bridge_to_rcar_lvds(bridge);
 
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
index 508977b9e8926..aaafee1c060bc 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.c
@@ -846,7 +846,7 @@ static int rcar_mipi_dsi_attach(struct drm_bridge *bridge,
 }
 
 static void rcar_mipi_dsi_atomic_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge);
 
@@ -854,7 +854,7 @@ static void rcar_mipi_dsi_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void rcar_mipi_dsi_atomic_disable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge);
 
@@ -862,7 +862,7 @@ static void rcar_mipi_dsi_atomic_disable(struct drm_bridge *bridge,
 }
 
 void rcar_mipi_dsi_pclk_enable(struct drm_bridge *bridge,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct rcar_mipi_dsi *dsi = bridge_to_rcar_mipi_dsi(bridge);
 	const struct drm_display_mode *mode;
diff --git a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.h b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.h
index 528a196e6eddb..cfb5d70a6b1f7 100644
--- a/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.h
+++ b/drivers/gpu/drm/renesas/rcar-du/rcar_mipi_dsi.h
@@ -10,16 +10,16 @@
 #ifndef __RCAR_MIPI_DSI_H__
 #define __RCAR_MIPI_DSI_H__
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_bridge;
 
 #if IS_ENABLED(CONFIG_DRM_RCAR_MIPI_DSI)
 void rcar_mipi_dsi_pclk_enable(struct drm_bridge *bridge,
-			       struct drm_atomic_state *state);
+			       struct drm_atomic_commit *state);
 void rcar_mipi_dsi_pclk_disable(struct drm_bridge *bridge);
 #else
 static inline void rcar_mipi_dsi_pclk_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 }
 
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c
index 18e2b981b691f..26b95153ce881 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_crtc.c
@@ -249,7 +249,7 @@ static void rzg2l_du_crtc_stop(struct rzg2l_du_crtc *rcrtc)
  */
 
 static void rzg2l_du_crtc_atomic_enable(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct rzg2l_du_crtc *rcrtc = to_rzg2l_crtc(crtc);
 
@@ -259,7 +259,7 @@ static void rzg2l_du_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void rzg2l_du_crtc_atomic_disable(struct drm_crtc *crtc,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct rzg2l_du_crtc *rcrtc = to_rzg2l_crtc(crtc);
 
@@ -275,7 +275,7 @@ static void rzg2l_du_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void rzg2l_du_crtc_atomic_flush(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct rzg2l_du_crtc *rcrtc = to_rzg2l_crtc(crtc);
 	struct drm_device *dev = rcrtc->crtc.dev;
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_encoder.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_encoder.c
index 5e6dd16705e60..0e567b57a4087 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_encoder.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_encoder.c
@@ -67,7 +67,7 @@ int rzg2l_du_encoder_init(struct rzg2l_du_device  *rcdu,
 {
 	struct rzg2l_du_encoder *renc;
 	struct drm_connector *connector;
-	struct drm_bridge *bridge;
+	struct drm_bridge *bridge __free(drm_bridge_put) = NULL;
 	int ret;
 
 	/*
@@ -84,9 +84,16 @@ int rzg2l_du_encoder_init(struct rzg2l_du_device  *rcdu,
 		bridge = devm_drm_panel_bridge_add_typed(rcdu->dev, panel,
 							 DRM_MODE_CONNECTOR_DPI);
 		if (IS_ERR(bridge))
-			return PTR_ERR(bridge);
+			return PTR_ERR(no_free_ptr(bridge));
+
+		/*
+		 * The reference taken by devm_drm_panel_bridge_add_typed() is
+		 * released automatically. Take a second one for the __free()
+		 * when this function will return.
+		 */
+		drm_bridge_get(bridge);
 	} else {
-		bridge = of_drm_find_bridge(enc_node);
+		bridge = of_drm_find_and_get_bridge(enc_node);
 		if (!bridge)
 			return -EPROBE_DEFER;
 	}
@@ -122,5 +129,5 @@ int rzg2l_du_encoder_init(struct rzg2l_du_device  *rcdu,
 		return PTR_ERR(connector);
 	}
 
-	return drm_connector_attach_encoder(connector, &renc->base);
+	return 0;
 }
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
index bd486377f037e..1efa0f0451fe9 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.c
@@ -20,6 +20,7 @@
 #include <drm/drm_vblank.h>
 
 #include <linux/bitops.h>
+#include <linux/device.h>
 #include <linux/dma-mapping.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
@@ -55,12 +56,12 @@ void rzg2l_du_vsp_enable(struct rzg2l_du_crtc *crtc)
 		.callback_data = crtc,
 	};
 
-	vsp1_du_setup_lif(crtc->vsp->vsp, crtc->vsp_pipe, &cfg);
+	vsp1_du_enable(crtc->vsp->vsp, crtc->vsp_pipe, &cfg);
 }
 
 void rzg2l_du_vsp_disable(struct rzg2l_du_crtc *crtc)
 {
-	vsp1_du_setup_lif(crtc->vsp->vsp, crtc->vsp_pipe, NULL);
+	vsp1_du_disable(crtc->vsp->vsp, crtc->vsp_pipe);
 }
 
 void rzg2l_du_vsp_atomic_flush(struct rzg2l_du_crtc *crtc)
@@ -212,7 +213,7 @@ static int __rzg2l_du_vsp_plane_atomic_check(struct drm_plane *plane,
 }
 
 static int rzg2l_du_vsp_plane_atomic_check(struct drm_plane *plane,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -222,7 +223,7 @@ static int rzg2l_du_vsp_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void rzg2l_du_vsp_plane_atomic_update(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
@@ -293,6 +294,9 @@ static void rzg2l_du_vsp_cleanup(struct drm_device *dev, void *res)
 {
 	struct rzg2l_du_vsp *vsp = res;
 
+	if (vsp->link)
+		device_link_del(vsp->link);
+
 	put_device(vsp->vsp);
 }
 
@@ -317,6 +321,18 @@ int rzg2l_du_vsp_init(struct rzg2l_du_vsp *vsp, struct device_node *np,
 	if (ret < 0)
 		return ret;
 
+	/*
+	 * Enforce suspend/resume ordering between the DU (consumer) and the
+	 * VSP (supplier). The DU will be suspended before and resume after the
+	 * VSP.
+	 */
+	vsp->link = device_link_add(rcdu->dev, vsp->vsp, DL_FLAG_STATELESS);
+	if (!vsp->link) {
+		dev_err(rcdu->dev, "Failed to create device link to VSP %s\n",
+			dev_name(vsp->vsp));
+		return -EINVAL;
+	}
+
 	ret = vsp1_du_init(vsp->vsp);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.h b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.h
index 322eb80dcbaff..a22aaf0843ed7 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.h
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_du_vsp.h
@@ -15,6 +15,7 @@
 #include <linux/scatterlist.h>
 
 struct device;
+struct device_link;
 struct drm_framebuffer;
 struct rzg2l_du_device;
 struct rzg2l_du_format_info;
@@ -29,6 +30,7 @@ struct rzg2l_du_vsp_plane {
 struct rzg2l_du_vsp {
 	unsigned int index;
 	struct device *vsp;
+	struct device_link *link;
 	struct rzg2l_du_device *dev;
 };
 
diff --git a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
index a87a301326c7a..a2c8648600756 100644
--- a/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
+++ b/drivers/gpu/drm/renesas/rz-du/rzg2l_mipi_dsi.c
@@ -484,7 +484,6 @@ static int rzg2l_mipi_dsi_dphy_init(struct rzg2l_mipi_dsi *dsi,
 	u32 dphytim1;
 	u32 dphytim2;
 	u32 dphytim3;
-	int ret;
 
 	/* All DSI global operation timings are set with recommended setting */
 	for (i = 0; i < ARRAY_SIZE(rzg2l_mipi_dsi_global_timings); ++i) {
@@ -524,12 +523,6 @@ static int rzg2l_mipi_dsi_dphy_init(struct rzg2l_mipi_dsi *dsi,
 	rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYTIM2, dphytim2);
 	rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYTIM3, dphytim3);
 
-	ret = reset_control_deassert(dsi->rstc);
-	if (ret < 0)
-		return ret;
-
-	udelay(1);
-
 	return 0;
 }
 
@@ -541,8 +534,6 @@ static void rzg2l_mipi_dsi_dphy_exit(struct rzg2l_mipi_dsi *dsi)
 
 	dphyctrl0 &= ~(DSIDPHYCTRL0_EN_LDO1200 | DSIDPHYCTRL0_EN_BGR);
 	rzg2l_mipi_dsi_phy_write(dsi, DSIDPHYCTRL0, dphyctrl0);
-
-	reset_control_assert(dsi->rstc);
 }
 
 static int rzg2l_dphy_conf_clks(struct rzg2l_mipi_dsi *dsi, unsigned long mode_freq,
@@ -811,6 +802,14 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi,
 	FIELD_MODIFY(DSISETR_MRPSZ, &dsisetr, RZG2L_DCS_BUF_SIZE);
 	rzg2l_mipi_dsi_link_write(dsi, DSISETR, dsisetr);
 
+	if (dsi->rstc) {
+		ret = reset_control_deassert(dsi->rstc);
+		if (ret < 0)
+			goto err_phy;
+
+		fsleep(1000);
+	}
+
 	return 0;
 
 err_phy:
@@ -822,6 +821,7 @@ static int rzg2l_mipi_dsi_startup(struct rzg2l_mipi_dsi *dsi,
 
 static void rzg2l_mipi_dsi_stop(struct rzg2l_mipi_dsi *dsi)
 {
+	reset_control_assert(dsi->rstc);
 	dsi->info->dphy_exit(dsi);
 	pm_runtime_put(dsi->dev);
 }
@@ -1019,35 +1019,39 @@ static int rzg2l_mipi_dsi_attach(struct drm_bridge *bridge,
 }
 
 static void rzg2l_mipi_dsi_atomic_pre_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge);
 	const struct drm_display_mode *mode;
 	struct drm_connector *connector;
 	struct drm_crtc *crtc;
-	int ret;
 
 	connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder);
 	crtc = drm_atomic_get_new_connector_state(state, connector)->crtc;
 	mode = &drm_atomic_get_new_crtc_state(state, crtc)->adjusted_mode;
 
-	ret = rzg2l_mipi_dsi_startup(dsi, mode);
-	if (ret < 0)
-		return;
-
-	rzg2l_mipi_dsi_set_display_timing(dsi, mode);
+	rzg2l_mipi_dsi_startup(dsi, mode);
 }
 
 static void rzg2l_mipi_dsi_atomic_enable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge);
+	const struct drm_display_mode *mode;
+	struct drm_connector *connector;
+	struct drm_crtc *crtc;
 	int ret;
 
 	ret = rzg2l_mipi_dsi_start_hs_clock(dsi);
 	if (ret < 0)
 		goto err_stop;
 
+	connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder);
+	crtc = drm_atomic_get_new_connector_state(state, connector)->crtc;
+	mode = &drm_atomic_get_new_crtc_state(state, crtc)->adjusted_mode;
+
+	rzg2l_mipi_dsi_set_display_timing(dsi, mode);
+
 	ret = rzg2l_mipi_dsi_start_video(dsi);
 	if (ret < 0)
 		goto err_stop_clock;
@@ -1061,7 +1065,7 @@ static void rzg2l_mipi_dsi_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void rzg2l_mipi_dsi_atomic_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge);
 
@@ -1070,7 +1074,7 @@ static void rzg2l_mipi_dsi_atomic_disable(struct drm_bridge *bridge,
 }
 
 static void rzg2l_mipi_dsi_atomic_post_disable(struct drm_bridge *bridge,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct rzg2l_mipi_dsi *dsi = bridge_to_rzg2l_mipi_dsi(bridge);
 
@@ -1441,6 +1445,11 @@ static int rzg2l_mipi_dsi_probe(struct platform_device *pdev)
 		return dev_err_probe(dsi->dev, PTR_ERR(dsi->prstc),
 				     "failed to get prst\n");
 
+	dsi->dcs_buf_virt = dmam_alloc_coherent(dsi->dev, RZG2L_DCS_BUF_SIZE,
+						&dsi->dcs_buf_phys, GFP_KERNEL);
+	if (!dsi->dcs_buf_virt)
+		return -ENOMEM;
+
 	platform_set_drvdata(pdev, dsi);
 
 	pm_runtime_enable(dsi->dev);
@@ -1473,11 +1482,6 @@ static int rzg2l_mipi_dsi_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto err_pm_disable;
 
-	dsi->dcs_buf_virt = dma_alloc_coherent(dsi->host.dev, RZG2L_DCS_BUF_SIZE,
-					       &dsi->dcs_buf_phys, GFP_KERNEL);
-	if (!dsi->dcs_buf_virt)
-		return -ENOMEM;
-
 	return 0;
 
 err_phy:
@@ -1492,8 +1496,6 @@ static void rzg2l_mipi_dsi_remove(struct platform_device *pdev)
 {
 	struct rzg2l_mipi_dsi *dsi = platform_get_drvdata(pdev);
 
-	dma_free_coherent(dsi->host.dev, RZG2L_DCS_BUF_SIZE, dsi->dcs_buf_virt,
-			  dsi->dcs_buf_phys);
 	mipi_dsi_host_unregister(&dsi->host);
 	pm_runtime_disable(&pdev->dev);
 }
diff --git a/drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c
index 5f460b38596ce..1a2b9b68af6f1 100644
--- a/drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c
+++ b/drivers/gpu/drm/renesas/shmobile/shmob_drm_crtc.c
@@ -198,7 +198,7 @@ static inline struct shmob_drm_crtc *to_shmob_crtc(struct drm_crtc *crtc)
 }
 
 static void shmob_drm_crtc_atomic_enable(struct drm_crtc *crtc,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct shmob_drm_crtc *scrtc = to_shmob_crtc(crtc);
 	struct shmob_drm_device *sdev = to_shmob_device(crtc->dev);
@@ -256,7 +256,7 @@ static void shmob_drm_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void shmob_drm_crtc_atomic_disable(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct shmob_drm_crtc *scrtc = to_shmob_crtc(crtc);
 	struct shmob_drm_device *sdev = to_shmob_device(crtc->dev);
@@ -279,7 +279,7 @@ static void shmob_drm_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void shmob_drm_crtc_atomic_flush(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_pending_vblank_event *event;
 	struct drm_device *dev = crtc->dev;
@@ -583,6 +583,13 @@ shmob_drm_connector_init(struct shmob_drm_device *sdev,
 
 	drm_connector_helper_add(connector, &connector_helper_funcs);
 
+	ret = drm_connector_attach_encoder(connector, encoder);
+	if (ret < 0) {
+		drm_connector_cleanup(connector);
+		kfree(scon);
+		return ERR_PTR(ret);
+	}
+
 	return connector;
 }
 
@@ -594,7 +601,6 @@ int shmob_drm_connector_create(struct shmob_drm_device *sdev,
 			       struct drm_encoder *encoder)
 {
 	struct drm_connector *connector;
-	int ret;
 
 	if (sdev->pdata)
 		connector = shmob_drm_connector_init(sdev, encoder);
@@ -606,17 +612,9 @@ int shmob_drm_connector_create(struct shmob_drm_device *sdev,
 		return PTR_ERR(connector);
 	}
 
-	ret = drm_connector_attach_encoder(connector, encoder);
-	if (ret < 0)
-		goto error;
-
 	connector->dpms = DRM_MODE_DPMS_OFF;
 
 	sdev->connector = connector;
 
 	return 0;
-
-error:
-	drm_connector_cleanup(connector);
-	return ret;
 }
diff --git a/drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c b/drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c
index b61fda52d17a4..148de34af7852 100644
--- a/drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c
+++ b/drivers/gpu/drm/renesas/shmobile/shmob_drm_plane.c
@@ -143,7 +143,7 @@ static void shmob_drm_overlay_plane_setup(struct shmob_drm_plane *splane,
 }
 
 static int shmob_drm_plane_atomic_check(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct shmob_drm_plane_state *sstate = to_shmob_plane_state(new_plane_state);
@@ -192,7 +192,7 @@ static int shmob_drm_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void shmob_drm_plane_atomic_update(struct drm_plane *plane,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct shmob_drm_plane *splane = to_shmob_plane(plane);
@@ -207,7 +207,7 @@ static void shmob_drm_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void shmob_drm_plane_atomic_disable(struct drm_plane *plane,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct shmob_drm_device *sdev = to_shmob_device(plane->dev);
diff --git a/drivers/gpu/drm/rockchip/Kconfig b/drivers/gpu/drm/rockchip/Kconfig
index 1479b8c4ed40a..e7f49fe845eae 100644
--- a/drivers/gpu/drm/rockchip/Kconfig
+++ b/drivers/gpu/drm/rockchip/Kconfig
@@ -46,6 +46,7 @@ config ROCKCHIP_VOP2
 config ROCKCHIP_ANALOGIX_DP
 	bool "Rockchip specific extensions for Analogix DP driver"
 	depends on ROCKCHIP_VOP
+	select DRM_BRIDGE_CONNECTOR
 	select DRM_DISPLAY_HELPER
 	select DRM_DISPLAY_DP_HELPER
 	help
diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
index 96bd3dd239d25..06072efd7fca3 100644
--- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
@@ -21,13 +21,12 @@
 #include <video/of_videomode.h>
 #include <video/videomode.h>
 
-#include <drm/display/drm_dp_aux_bus.h>
 #include <drm/display/drm_dp_helper.h>
 #include <drm/drm_atomic.h>
 #include <drm/drm_atomic_helper.h>
+#include <drm/drm_bridge_connector.h>
 #include <drm/bridge/analogix_dp.h>
 #include <drm/drm_of.h>
-#include <drm/drm_panel.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
@@ -166,23 +165,6 @@ static int rockchip_dp_powerdown(struct analogix_dp_plat_data *plat_data)
 	return 0;
 }
 
-static int rockchip_dp_get_modes(struct analogix_dp_plat_data *plat_data,
-				 struct drm_connector *connector)
-{
-	struct drm_display_info *di = &connector->display_info;
-	/* VOP couldn't output YUV video format for eDP rightly */
-	u32 mask = BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR444) | BIT(DRM_OUTPUT_COLOR_FORMAT_YCBCR422);
-
-	if ((di->color_formats & mask)) {
-		DRM_DEBUG_KMS("Swapping display color format from YUV to RGB\n");
-		di->color_formats &= ~mask;
-		di->color_formats |= BIT(DRM_OUTPUT_COLOR_FORMAT_RGB444);
-		di->bpc = 8;
-	}
-
-	return 0;
-}
-
 static bool
 rockchip_dp_drm_encoder_mode_fixup(struct drm_encoder *encoder,
 				   const struct drm_display_mode *mode,
@@ -201,7 +183,7 @@ static void rockchip_dp_drm_encoder_mode_set(struct drm_encoder *encoder,
 
 static
 struct drm_crtc *rockchip_dp_drm_get_new_crtc(struct drm_encoder *encoder,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_connector *connector;
 	struct drm_connector_state *conn_state;
@@ -218,7 +200,7 @@ struct drm_crtc *rockchip_dp_drm_get_new_crtc(struct drm_encoder *encoder,
 }
 
 static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct rockchip_dp_device *dp = encoder_to_dp(encoder);
 	struct drm_crtc *crtc;
@@ -272,7 +254,7 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder,
 }
 
 static void rockchip_dp_drm_encoder_disable(struct drm_encoder *encoder,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct rockchip_dp_device *dp = encoder_to_dp(encoder);
 	struct drm_crtc *crtc;
@@ -386,6 +368,7 @@ static int rockchip_dp_bind(struct device *dev, struct device *master,
 {
 	struct rockchip_dp_device *dp = dev_get_drvdata(dev);
 	struct drm_device *drm_dev = data;
+	struct drm_connector *connector;
 	int ret;
 
 	dp->drm_dev = drm_dev;
@@ -405,6 +388,13 @@ static int rockchip_dp_bind(struct device *dev, struct device *master,
 	if (ret)
 		goto err_cleanup_encoder;
 
+	connector = drm_bridge_connector_init(dp->drm_dev, dp->plat_data.encoder);
+	if (IS_ERR(connector)) {
+		ret = PTR_ERR(connector);
+		dev_err(dp->dev, "Failed to initialize bridge_connector\n");
+		goto err_cleanup_encoder;
+	}
+
 	return 0;
 err_cleanup_encoder:
 	dp->encoder.encoder.funcs->destroy(&dp->encoder.encoder);
@@ -425,24 +415,6 @@ static const struct component_ops rockchip_dp_component_ops = {
 	.unbind = rockchip_dp_unbind,
 };
 
-static int rockchip_dp_link_panel(struct drm_dp_aux *aux)
-{
-	struct analogix_dp_plat_data *plat_data = analogix_dp_aux_to_plat_data(aux);
-	struct rockchip_dp_device *dp = pdata_encoder_to_dp(plat_data);
-	int ret;
-
-	/*
-	 * If drm_of_find_panel_or_bridge() returns -ENODEV, there may be no valid panel
-	 * or bridge nodes. The driver should go on for the driver-free bridge or the DP
-	 * mode applications.
-	 */
-	ret = drm_of_find_panel_or_bridge(dp->dev->of_node, 1, 0, &plat_data->panel, NULL);
-	if (ret && ret != -ENODEV)
-		return ret;
-
-	return component_add(dp->dev, &rockchip_dp_component_ops);
-}
-
 static int rockchip_dp_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -461,6 +433,8 @@ static int rockchip_dp_probe(struct platform_device *pdev)
 		return -ENOMEM;
 
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+	if (!res)
+		return -EINVAL;
 
 	i = 0;
 	while (dp_data[i].reg) {
@@ -481,7 +455,7 @@ static int rockchip_dp_probe(struct platform_device *pdev)
 	dp->plat_data.dev_type = dp->data->chip_type;
 	dp->plat_data.power_on = rockchip_dp_poweron;
 	dp->plat_data.power_off = rockchip_dp_powerdown;
-	dp->plat_data.get_modes = rockchip_dp_get_modes;
+	dp->plat_data.ops = &rockchip_dp_component_ops;
 
 	ret = rockchip_dp_of_probe(dp);
 	if (ret < 0)
@@ -493,22 +467,7 @@ static int rockchip_dp_probe(struct platform_device *pdev)
 	if (IS_ERR(dp->adp))
 		return PTR_ERR(dp->adp);
 
-	ret = devm_of_dp_aux_populate_bus(analogix_dp_get_aux(dp->adp), rockchip_dp_link_panel);
-	if (ret) {
-		/*
-		 * If devm_of_dp_aux_populate_bus() returns -ENODEV, the done_probing() will not
-		 * be called because there are no EP devices. Then the rockchip_dp_link_panel()
-		 * will be called directly in order to support the other valid DT configurations.
-		 *
-		 * NOTE: The devm_of_dp_aux_populate_bus() is allowed to return -EPROBE_DEFER.
-		 */
-		if (ret != -ENODEV)
-			return dev_err_probe(dp->dev, ret, "failed to populate aux bus\n");
-
-		return rockchip_dp_link_panel(analogix_dp_get_aux(dp->adp));
-	}
-
-	return 0;
+	return analogix_dp_finish_probe(dp->adp);
 }
 
 static void rockchip_dp_remove(struct platform_device *pdev)
diff --git a/drivers/gpu/drm/rockchip/cdn-dp-core.c b/drivers/gpu/drm/rockchip/cdn-dp-core.c
index 177e30445ee83..190cce9530c82 100644
--- a/drivers/gpu/drm/rockchip/cdn-dp-core.c
+++ b/drivers/gpu/drm/rockchip/cdn-dp-core.c
@@ -563,7 +563,7 @@ static void cdn_dp_display_info_update(struct cdn_dp_device *dp,
 	}
 }
 
-static void cdn_dp_bridge_atomic_enable(struct drm_bridge *bridge, struct drm_atomic_state *state)
+static void cdn_dp_bridge_atomic_enable(struct drm_bridge *bridge, struct drm_atomic_commit *state)
 {
 	struct cdn_dp_device *dp = bridge_to_dp(bridge);
 	struct drm_connector *connector;
@@ -630,7 +630,7 @@ static void cdn_dp_bridge_atomic_enable(struct drm_bridge *bridge, struct drm_at
 	mutex_unlock(&dp->lock);
 }
 
-static void cdn_dp_bridge_atomic_disable(struct drm_bridge *bridge, struct drm_atomic_state *state)
+static void cdn_dp_bridge_atomic_disable(struct drm_bridge *bridge, struct drm_atomic_commit *state)
 {
 	struct cdn_dp_device *dp = bridge_to_dp(bridge);
 	int ret;
@@ -1024,8 +1024,6 @@ static int cdn_dp_bind(struct device *dev, struct device *master, void *data)
 		return ret;
 	}
 
-	drm_connector_attach_encoder(connector, encoder);
-
 	for (i = 0; i < dp->ports; i++) {
 		port = dp->port[i];
 
diff --git a/drivers/gpu/drm/rockchip/cdn-dp-reg.c b/drivers/gpu/drm/rockchip/cdn-dp-reg.c
index 0dc3804051a94..9b82b27770e5c 100644
--- a/drivers/gpu/drm/rockchip/cdn-dp-reg.c
+++ b/drivers/gpu/drm/rockchip/cdn-dp-reg.c
@@ -685,6 +685,8 @@ int cdn_dp_config_video(struct cdn_dp_device *dp)
 	val = div_u64(8 * (symbol + 1), bit_per_pix) - val;
 	val += 2;
 	ret = cdn_dp_reg_write(dp, DP_VC_TABLE(15), val);
+	if (ret)
+		goto err_config_video;
 
 	switch (video->color_depth) {
 	case 6:
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c
index 0aea764e29b2c..d2e76d36d7247 100644
--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi2-rockchip.c
@@ -212,7 +212,7 @@ static const struct dw_mipi_dsi2_phy_ops dw_mipi_dsi2_rockchip_phy_ops = {
 };
 
 static void dw_mipi_dsi2_encoder_atomic_enable(struct drm_encoder *encoder,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct dw_mipi_dsi2_rockchip *dsi2 = to_dsi2(encoder);
 	u32 color_depth;
diff --git a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c
index dac3d202971ed..32bc73a1d5e45 100644
--- a/drivers/gpu/drm/rockchip/dw_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_dp-rockchip.c
@@ -7,22 +7,22 @@
  */
 
 #include <linux/component.h>
+#include <linux/media-bus-format.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
+#include <linux/videodev2.h>
+
 #include <drm/bridge/dw_dp.h>
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_bridge.h>
 #include <drm/drm_bridge_connector.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_of.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
 
-#include <linux/media-bus-format.h>
-#include <linux/videodev2.h>
-
 #include "rockchip_drm_drv.h"
-#include "rockchip_drm_vop.h"
 
 struct rockchip_dw_dp {
 	struct dw_dp *base;
@@ -35,7 +35,7 @@ static int dw_dp_encoder_atomic_check(struct drm_encoder *encoder,
 				      struct drm_connector_state *conn_state)
 {
 	struct rockchip_crtc_state *s = to_rockchip_crtc_state(crtc_state);
-	struct drm_atomic_state *state = conn_state->state;
+	struct drm_atomic_commit *state = conn_state->state;
 	struct drm_display_info *di = &conn_state->connector->display_info;
 	struct drm_bridge *bridge  = drm_bridge_chain_get_first_bridge(encoder);
 	struct drm_bridge_state *bridge_state = drm_atomic_get_new_bridge_state(state, bridge);
@@ -82,7 +82,7 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void *
 	struct drm_connector *connector;
 	int ret;
 
-	dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL);
+	dp = drmm_kzalloc(drm_dev, sizeof(*dp), GFP_KERNEL);
 	if (!dp)
 		return -ENOMEM;
 
@@ -103,18 +103,13 @@ static int dw_dp_rockchip_bind(struct device *dev, struct device *master, void *
 	drm_encoder_helper_add(encoder, &dw_dp_encoder_helper_funcs);
 
 	dp->base = dw_dp_bind(dev, encoder, plat_data);
-	if (IS_ERR(dp->base)) {
-		ret = PTR_ERR(dp->base);
-		return ret;
-	}
+	if (IS_ERR(dp->base))
+		return PTR_ERR(dp->base);
 
 	connector = drm_bridge_connector_init(drm_dev, encoder);
-	if (IS_ERR(connector)) {
-		ret = PTR_ERR(connector);
-		return dev_err_probe(dev, ret, "Failed to init bridge connector");
-	}
-
-	drm_connector_attach_encoder(connector, encoder);
+	if (IS_ERR(connector))
+		return dev_err_probe(dev, PTR_ERR(connector),
+				     "Failed to init bridge connector");
 
 	return 0;
 }
@@ -125,16 +120,12 @@ static const struct component_ops dw_dp_rockchip_component_ops = {
 
 static int dw_dp_probe(struct platform_device *pdev)
 {
-	struct device *dev = &pdev->dev;
-
-	return component_add(dev, &dw_dp_rockchip_component_ops);
+	return component_add(&pdev->dev, &dw_dp_rockchip_component_ops);
 }
 
 static void dw_dp_remove(struct platform_device *pdev)
 {
-	struct rockchip_dw_dp *dp = platform_get_drvdata(pdev);
-
-	component_del(dp->dev, &dw_dp_rockchip_component_ops);
+	component_del(&pdev->dev, &dw_dp_rockchip_component_ops);
 }
 
 static const struct dw_dp_plat_data rk3588_dp_plat_data = {
diff --git a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c
index 1a09bcc96c3ec..f35484715c2d1 100644
--- a/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_hdmi_qp-rockchip.c
@@ -21,6 +21,7 @@
 #include <drm/bridge/dw_hdmi_qp.h>
 #include <drm/display/drm_hdmi_helper.h>
 #include <drm/drm_bridge_connector.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_of.h>
 #include <drm/drm_probe_helper.h>
 #include <drm/drm_simple_kms_helper.h>
@@ -477,7 +478,7 @@ static int dw_hdmi_qp_rockchip_bind(struct device *dev, struct device *master,
 	if (!pdev->dev.of_node)
 		return -ENODEV;
 
-	hdmi = devm_kzalloc(&pdev->dev, sizeof(*hdmi), GFP_KERNEL);
+	hdmi = drmm_kzalloc(drm, sizeof(*hdmi), GFP_KERNEL);
 	if (!hdmi)
 		return -ENOMEM;
 
@@ -586,23 +587,23 @@ static int dw_hdmi_qp_rockchip_bind(struct device *dev, struct device *master,
 		return ret;
 
 	drm_encoder_helper_add(encoder, &dw_hdmi_qp_rockchip_encoder_helper_funcs);
-	drm_simple_encoder_init(drm, encoder, DRM_MODE_ENCODER_TMDS);
+	ret = drmm_encoder_init(drm, encoder, NULL, DRM_MODE_ENCODER_TMDS, NULL);
+	if (ret)
+		return dev_err_probe(hdmi->dev, ret, "Failed to init encoder");
 
 	platform_set_drvdata(pdev, hdmi);
 
 	hdmi->hdmi = dw_hdmi_qp_bind(pdev, encoder, &plat_data);
-	if (IS_ERR(hdmi->hdmi)) {
-		drm_encoder_cleanup(encoder);
+	if (IS_ERR(hdmi->hdmi))
 		return dev_err_probe(hdmi->dev, PTR_ERR(hdmi->hdmi),
 				     "Failed to bind dw-hdmi-qp");
-	}
 
 	connector = drm_bridge_connector_init(drm, encoder);
 	if (IS_ERR(connector))
 		return dev_err_probe(hdmi->dev, PTR_ERR(connector),
 				     "Failed to init bridge connector\n");
 
-	return drm_connector_attach_encoder(connector, encoder);
+	return 0;
 }
 
 static void dw_hdmi_qp_rockchip_unbind(struct device *dev,
@@ -612,8 +613,6 @@ static void dw_hdmi_qp_rockchip_unbind(struct device *dev,
 	struct rockchip_hdmi_qp *hdmi = dev_get_drvdata(dev);
 
 	cancel_delayed_work_sync(&hdmi->hpd_work);
-
-	drm_encoder_cleanup(&hdmi->encoder.encoder);
 }
 
 static const struct component_ops dw_hdmi_qp_rockchip_ops = {
diff --git a/drivers/gpu/drm/rockchip/inno_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/inno_hdmi-rockchip.c
index 97c20500f7901..45a6dae4de31d 100644
--- a/drivers/gpu/drm/rockchip/inno_hdmi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/inno_hdmi-rockchip.c
@@ -14,6 +14,7 @@
 
 #include <drm/bridge/inno_hdmi.h>
 #include <drm/drm_bridge_connector.h>
+#include <drm/drm_managed.h>
 #include <drm/drm_of.h>
 
 #include "rockchip_drm_drv.h"
@@ -90,7 +91,7 @@ static int inno_hdmi_rockchip_bind(struct device *dev, struct device *master, vo
 	const struct inno_hdmi_plat_data *plat_data;
 	int ret;
 
-	hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL);
+	hdmi = drmm_kzalloc(drm, sizeof(*hdmi), GFP_KERNEL);
 	if (!hdmi)
 		return -ENOMEM;
 
@@ -136,7 +137,7 @@ static int inno_hdmi_rockchip_bind(struct device *dev, struct device *master, vo
 		return ret;
 	}
 
-	return drm_connector_attach_encoder(connector, encoder);
+	return 0;
 }
 
 static const struct component_ops inno_hdmi_rockchip_ops = {
diff --git a/drivers/gpu/drm/rockchip/rk3066_hdmi.c b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
index 9066ee2d1dff4..1996f8656612d 100644
--- a/drivers/gpu/drm/rockchip/rk3066_hdmi.c
+++ b/drivers/gpu/drm/rockchip/rk3066_hdmi.c
@@ -313,7 +313,7 @@ static void rk3066_hdmi_config_phy(struct rk3066_hdmi *hdmi)
 }
 
 static int rk3066_hdmi_setup(struct rk3066_hdmi *hdmi,
-			     struct drm_atomic_state *state)
+			     struct drm_atomic_commit *state)
 {
 	struct drm_bridge *bridge = &hdmi->bridge;
 	struct drm_connector *connector;
@@ -394,7 +394,7 @@ static int rk3066_hdmi_setup(struct rk3066_hdmi *hdmi,
 }
 
 static void rk3066_hdmi_bridge_atomic_enable(struct drm_bridge *bridge,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct rk3066_hdmi *hdmi = bridge_to_rk3066_hdmi(bridge);
 	struct drm_connector_state *conn_state;
@@ -424,7 +424,7 @@ static void rk3066_hdmi_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void rk3066_hdmi_bridge_atomic_disable(struct drm_bridge *bridge,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct rk3066_hdmi *hdmi = bridge_to_rk3066_hdmi(bridge);
 
@@ -731,8 +731,6 @@ rk3066_hdmi_register(struct drm_device *drm, struct rk3066_hdmi *hdmi)
 		return ret;
 	}
 
-	drm_connector_attach_encoder(hdmi->connector, encoder);
-
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
index 09d14a072d274..b188539dca0b2 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
@@ -334,7 +334,7 @@ void rockchip_gem_free_object(struct drm_gem_object *obj)
 	struct rockchip_drm_private *private = drm->dev_private;
 	struct rockchip_gem_object *rk_obj = to_rockchip_obj(obj);
 
-	if (obj->import_attach) {
+	if (drm_gem_is_imported(obj)) {
 		if (private->domain) {
 			rockchip_gem_iommu_unmap(rk_obj);
 		} else {
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 572f4adfd111b..0090d8ff0c79e 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -728,7 +728,7 @@ static void rockchip_drm_set_win_enabled(struct drm_crtc *crtc, bool enabled)
 }
 
 static void vop_crtc_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct vop *vop = to_vop(crtc);
 
@@ -809,7 +809,7 @@ static bool rockchip_mod_supported(struct drm_plane *plane,
 }
 
 static int vop_plane_atomic_check(struct drm_plane *plane,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -889,7 +889,7 @@ static int vop_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void vop_plane_atomic_disable(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -907,7 +907,7 @@ static void vop_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static void vop_plane_atomic_update(struct drm_plane *plane,
-		struct drm_atomic_state *state)
+		struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -1071,7 +1071,7 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
 }
 
 static int vop_plane_atomic_async_check(struct drm_plane *plane,
-					struct drm_atomic_state *state, bool flip)
+					struct drm_atomic_commit *state, bool flip)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -1105,7 +1105,7 @@ static int vop_plane_atomic_async_check(struct drm_plane *plane,
 }
 
 static void vop_plane_atomic_async_update(struct drm_plane *plane,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -1342,7 +1342,7 @@ static void vop_crtc_gamma_set(struct vop *vop, struct drm_crtc *crtc,
 }
 
 static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -1360,7 +1360,7 @@ static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void vop_crtc_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 									 crtc);
@@ -1515,7 +1515,7 @@ static void vop_wait_for_irq_handler(struct vop *vop)
 }
 
 static int vop_crtc_atomic_check(struct drm_crtc *crtc,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -1562,11 +1562,11 @@ static int vop_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void vop_crtc_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
 									      crtc);
-	struct drm_atomic_state *old_state = old_crtc_state->state;
+	struct drm_atomic_commit *old_state = old_crtc_state->state;
 	struct drm_plane_state *old_plane_state, *new_plane_state;
 	struct vop *vop = to_vop(crtc);
 	struct drm_plane *plane;
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
index 843c7ef979b21..a160077a507f2 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop2.c
@@ -923,7 +923,7 @@ static bool vop2_gamma_lut_in_use(struct vop2 *vop2, struct vop2_video_port *vp)
 }
 
 static void vop2_crtc_atomic_disable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct vop2_video_port *vp = to_vop2_video_port(crtc);
 	struct vop2 *vop2 = vp->vop2;
@@ -979,7 +979,7 @@ static void vop2_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static int vop2_plane_atomic_check(struct drm_plane *plane,
-				   struct drm_atomic_state *astate)
+				   struct drm_atomic_commit *astate)
 {
 	struct drm_plane_state *pstate = drm_atomic_get_new_plane_state(astate, plane);
 	struct drm_framebuffer *fb = pstate->fb;
@@ -1091,7 +1091,7 @@ static int vop2_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void vop2_plane_atomic_disable(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_pstate = NULL;
 	struct vop2_win *win = to_vop2_win(plane);
@@ -1159,7 +1159,7 @@ static void vop2_plane_setup_color_key(struct drm_plane *plane, u32 color_key)
 }
 
 static void vop2_plane_atomic_update(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *pstate = plane->state;
 	struct drm_crtc *crtc = pstate->crtc;
@@ -1622,7 +1622,7 @@ static int us_to_vertical_line(struct drm_display_mode *mode, int us)
 }
 
 static void vop2_crtc_atomic_enable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct vop2_video_port *vp = to_vop2_video_port(crtc);
 	struct vop2 *vop2 = vp->vop2;
@@ -1814,7 +1814,7 @@ static void vop2_crtc_atomic_enable(struct drm_crtc *crtc,
 
 static int vop2_crtc_atomic_check_gamma(struct vop2_video_port *vp,
 					struct drm_crtc *crtc,
-					struct drm_atomic_state *state,
+					struct drm_atomic_commit *state,
 					struct drm_crtc_state *crtc_state)
 {
 	struct vop2 *vop2 = vp->vop2;
@@ -1840,7 +1840,7 @@ static int vop2_crtc_atomic_check_gamma(struct vop2_video_port *vp,
 }
 
 static int vop2_crtc_atomic_check(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct vop2_video_port *vp = to_vop2_video_port(crtc);
 	struct drm_plane *plane;
@@ -1862,7 +1862,7 @@ static int vop2_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void vop2_crtc_atomic_begin(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct vop2_video_port *vp = to_vop2_video_port(crtc);
 	struct vop2 *vop2 = vp->vop2;
@@ -1871,7 +1871,7 @@ static void vop2_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void vop2_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	struct vop2_video_port *vp = to_vop2_video_port(crtc);
diff --git a/drivers/gpu/drm/rockchip/rockchip_lvds.c b/drivers/gpu/drm/rockchip/rockchip_lvds.c
index 75f898a10cbc2..7a0c4fa29f2f0 100644
--- a/drivers/gpu/drm/rockchip/rockchip_lvds.c
+++ b/drivers/gpu/drm/rockchip/rockchip_lvds.c
@@ -628,12 +628,6 @@ static int rockchip_lvds_bind(struct device *dev, struct device *master,
 			ret = PTR_ERR(connector);
 			goto err_free_bridge;
 		}
-
-		ret = drm_connector_attach_encoder(connector, encoder);
-		if (ret < 0) {
-			drm_err(drm_dev, "failed to attach encoder: %d\n", ret);
-			goto err_free_bridge;
-		}
 	}
 
 	pm_runtime_enable(dev);
diff --git a/drivers/gpu/drm/rockchip/rockchip_rgb.c b/drivers/gpu/drm/rockchip/rockchip_rgb.c
index 5c0c6e2cc28dc..add3123e5ce70 100644
--- a/drivers/gpu/drm/rockchip/rockchip_rgb.c
+++ b/drivers/gpu/drm/rockchip/rockchip_rgb.c
@@ -162,17 +162,8 @@ struct rockchip_rgb *rockchip_rgb_init(struct device *dev,
 
 	rgb->encoder.crtc_endpoint_id = endpoint_id;
 
-	ret = drm_connector_attach_encoder(connector, encoder);
-	if (ret < 0) {
-		DRM_DEV_ERROR(drm_dev->dev,
-			      "failed to attach encoder: %d\n", ret);
-		goto err_free_connector;
-	}
-
 	return rgb;
 
-err_free_connector:
-	drm_connector_cleanup(connector);
 err_free_encoder:
 	drm_encoder_cleanup(encoder);
 	return ERR_PTR(ret);
diff --git a/drivers/gpu/drm/scheduler/Makefile b/drivers/gpu/drm/scheduler/Makefile
index 6e13e4c63e9d8..74e75eff6df50 100644
--- a/drivers/gpu/drm/scheduler/Makefile
+++ b/drivers/gpu/drm/scheduler/Makefile
@@ -20,7 +20,7 @@
 # OTHER DEALINGS IN THE SOFTWARE.
 #
 #
-gpu-sched-y := sched_main.o sched_fence.o sched_entity.o
+gpu-sched-y := sched_main.o sched_fence.o sched_entity.o sched_rq.o
 
 obj-$(CONFIG_DRM_SCHED) += gpu-sched.o
 
diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c
index fe174a4857be7..4ebb513255edd 100644
--- a/drivers/gpu/drm/scheduler/sched_entity.c
+++ b/drivers/gpu/drm/scheduler/sched_entity.c
@@ -32,6 +32,64 @@
 
 #include "gpu_scheduler_trace.h"
 
+/**
+ * drm_sched_entity_stats_release - Entity stats kref release function
+ * @kref: Entity stats embedded kref pointer
+ */
+void drm_sched_entity_stats_release(struct kref *kref)
+{
+	struct drm_sched_entity_stats *stats =
+		container_of(kref, typeof(*stats), kref);
+
+	kfree(stats);
+}
+
+/**
+ * drm_sched_entity_stats_new - Allocate a new struct drm_sched_entity_stats object
+ *
+ * Return: Pointer to newly allocated struct drm_sched_entity_stats object.
+ */
+static struct drm_sched_entity_stats *drm_sched_entity_stats_new(void)
+{
+	struct drm_sched_entity_stats *stats;
+
+	stats = kzalloc_obj(*stats);
+	if (!stats)
+		return NULL;
+
+	kref_init(&stats->kref);
+	spin_lock_init(&stats->lock);
+	ewma_drm_sched_avgtime_init(&stats->avg_job_us);
+
+	return stats;
+}
+
+/**
+ * drm_sched_entity_stats_job_add_gpu_time - Account job execution time to entity
+ * @job: Scheduler job to account.
+ *
+ * Accounts the execution time of @job to its respective entity stats object.
+ *
+ * Return: Job's real duration in micro seconds.
+ */
+ktime_t drm_sched_entity_stats_job_add_gpu_time(struct drm_sched_job *job)
+{
+	struct drm_sched_entity_stats *stats = job->entity_stats;
+	struct drm_sched_fence *s_fence = job->s_fence;
+	ktime_t start, end, duration;
+
+	start = dma_fence_timestamp(&s_fence->scheduled);
+	end = dma_fence_timestamp(&s_fence->finished);
+	duration = ktime_sub(end, start);
+
+	spin_lock(&stats->lock);
+	stats->runtime = ktime_add(stats->runtime, duration);
+	ewma_drm_sched_avgtime_add(&stats->avg_job_us, ktime_to_us(duration));
+	spin_unlock(&stats->lock);
+
+	return duration;
+}
+
 /**
  * drm_sched_entity_init - Init a context entity used by scheduler when
  * submit to HW ring.
@@ -61,45 +119,25 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 			  unsigned int num_sched_list,
 			  atomic_t *guilty)
 {
-	if (!(entity && sched_list && (num_sched_list == 0 || sched_list[0])))
+	if (!entity || !sched_list || !num_sched_list || !sched_list[0])
 		return -EINVAL;
 
 	memset(entity, 0, sizeof(struct drm_sched_entity));
+
+	entity->stats = drm_sched_entity_stats_new();
+	if (!entity->stats)
+		return -ENOMEM;
+
 	INIT_LIST_HEAD(&entity->list);
 	entity->rq = NULL;
 	entity->guilty = guilty;
-	entity->num_sched_list = num_sched_list;
 	entity->priority = priority;
 	entity->last_user = current->group_leader;
-	/*
-	 * It's perfectly valid to initialize an entity without having a valid
-	 * scheduler attached. It's just not valid to use the scheduler before it
-	 * is initialized itself.
-	 */
+	entity->num_sched_list = num_sched_list;
 	entity->sched_list = num_sched_list > 1 ? sched_list : NULL;
+	entity->rq = &sched_list[0]->rq;
 	RCU_INIT_POINTER(entity->last_scheduled, NULL);
 	RB_CLEAR_NODE(&entity->rb_tree_node);
-
-	if (num_sched_list && !sched_list[0]->sched_rq) {
-		/* Since every entry covered by num_sched_list
-		 * should be non-NULL and therefore we warn drivers
-		 * not to do this and to fix their DRM calling order.
-		 */
-		pr_warn("%s: called with uninitialized scheduler\n", __func__);
-	} else if (num_sched_list) {
-		/* The "priority" of an entity cannot exceed the number of run-queues of a
-		 * scheduler. Protect against num_rqs being 0, by converting to signed. Choose
-		 * the lowest priority available.
-		 */
-		if (entity->priority >= sched_list[0]->num_rqs) {
-			dev_err(sched_list[0]->dev, "entity has out-of-bounds priority: %u. num_rqs: %u\n",
-				entity->priority, sched_list[0]->num_rqs);
-			entity->priority = max_t(s32, (s32) sched_list[0]->num_rqs - 1,
-						 (s32) DRM_SCHED_PRIORITY_KERNEL);
-		}
-		entity->rq = sched_list[0]->sched_rq[entity->priority];
-	}
-
 	init_completion(&entity->entity_idle);
 
 	/* We start in an idle state. */
@@ -228,8 +266,16 @@ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f,
 	schedule_work(&job->work);
 }
 
-/* Remove the entity from the scheduler and kill all pending jobs */
-static void drm_sched_entity_kill(struct drm_sched_entity *entity)
+/**
+ * drm_sched_entity_kill - kill an entity's pending jobs and remove it
+ * @entity: the entity to remove
+ *
+ * Removes the entity from the scheduler's run queue and kills all pending jobs.
+ *
+ * This function should be used over drm_sched_entity_flush() if it is not
+ * desired to actually wait for all pending jobs to finish.
+ */
+void drm_sched_entity_kill(struct drm_sched_entity *entity)
 {
 	struct drm_sched_job *job;
 	struct dma_fence *prev;
@@ -267,6 +313,7 @@ static void drm_sched_entity_kill(struct drm_sched_entity *entity)
 	}
 	dma_fence_put(prev);
 }
+EXPORT_SYMBOL(drm_sched_entity_kill);
 
 /**
  * drm_sched_entity_flush - Flush a context entity
@@ -289,7 +336,7 @@ long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout)
 	if (!entity->rq)
 		return 0;
 
-	sched = entity->rq->sched;
+	sched = container_of(entity->rq, typeof(*sched), rq);
 	/*
 	 * The client will not queue more jobs during this fini - consume
 	 * existing queued ones, or discard them on SIGKILL.
@@ -343,6 +390,7 @@ void drm_sched_entity_fini(struct drm_sched_entity *entity)
 
 	dma_fence_put(rcu_dereference_check(entity->last_scheduled, true));
 	RCU_INIT_POINTER(entity->last_scheduled, NULL);
+	drm_sched_entity_stats_put(entity->stats);
 }
 EXPORT_SYMBOL(drm_sched_entity_fini);
 
@@ -369,10 +417,12 @@ static void drm_sched_entity_wakeup(struct dma_fence *f,
 {
 	struct drm_sched_entity *entity =
 		container_of(cb, struct drm_sched_entity, cb);
+	struct drm_gpu_scheduler *sched =
+		container_of(entity->rq, typeof(*sched), rq);
 
 	entity->dependency = NULL;
 	dma_fence_put(f);
-	drm_sched_wakeup(entity->rq->sched);
+	drm_sched_wakeup(sched);
 }
 
 /**
@@ -399,7 +449,8 @@ EXPORT_SYMBOL(drm_sched_entity_set_priority);
 static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity,
 					       struct drm_sched_job *sched_job)
 {
-	struct drm_gpu_scheduler *sched = entity->rq->sched;
+	struct drm_gpu_scheduler *sched =
+		container_of(entity->rq, typeof(*sched), rq);
 	struct dma_fence *fence = entity->dependency;
 	struct drm_sched_fence *s_fence;
 
@@ -492,26 +543,7 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity)
 
 	spsc_queue_pop(&entity->job_queue);
 
-	/*
-	 * Update the entity's location in the min heap according to
-	 * the timestamp of the next job, if any.
-	 */
-	if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) {
-		struct drm_sched_job *next;
-
-		next = drm_sched_entity_queue_peek(entity);
-		if (next) {
-			struct drm_sched_rq *rq;
-
-			spin_lock(&entity->lock);
-			rq = entity->rq;
-			spin_lock(&rq->lock);
-			drm_sched_rq_update_fifo_locked(entity, rq,
-							next->submit_ts);
-			spin_unlock(&rq->lock);
-			spin_unlock(&entity->lock);
-		}
-	}
+	drm_sched_rq_pop_entity(entity);
 
 	/* Jobs and entities might have different lifecycles. Since we're
 	 * removing the job from the entities queue, set the jobs entity pointer
@@ -552,7 +584,7 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
 
 	spin_lock(&entity->lock);
 	sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list);
-	rq = sched ? sched->sched_rq[entity->priority] : NULL;
+	rq = sched ? &sched->rq : NULL;
 	if (rq != entity->rq) {
 		drm_sched_rq_remove_entity(entity->rq, entity);
 		entity->rq = rq;
@@ -576,8 +608,9 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity)
 void drm_sched_entity_push_job(struct drm_sched_job *sched_job)
 {
 	struct drm_sched_entity *entity = sched_job->entity;
+	struct drm_gpu_scheduler *sched =
+		container_of(entity->rq, typeof(*sched), rq);
 	bool first;
-	ktime_t submit_ts;
 
 	trace_drm_sched_job_queue(sched_job, entity);
 
@@ -588,44 +621,20 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job)
 		xa_for_each(&sched_job->dependencies, index, entry)
 			trace_drm_sched_job_add_dep(sched_job, entry);
 	}
-	atomic_inc(entity->rq->sched->score);
+	atomic_inc(sched->score);
 	WRITE_ONCE(entity->last_user, current->group_leader);
 
 	/*
 	 * After the sched_job is pushed into the entity queue, it may be
 	 * completed and freed up at any time. We can no longer access it.
-	 * Make sure to set the submit_ts first, to avoid a race.
 	 */
-	sched_job->submit_ts = submit_ts = ktime_get();
 	first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node);
 
 	/* first job wakes up scheduler */
 	if (first) {
-		struct drm_gpu_scheduler *sched;
-		struct drm_sched_rq *rq;
-
-		/* Add the entity to the run queue */
-		spin_lock(&entity->lock);
-		if (entity->stopped) {
-			spin_unlock(&entity->lock);
-
-			DRM_ERROR("Trying to push to a killed entity\n");
-			return;
-		}
-
-		rq = entity->rq;
-		sched = rq->sched;
-
-		spin_lock(&rq->lock);
-		drm_sched_rq_add_entity(rq, entity);
-
-		if (drm_sched_policy == DRM_SCHED_POLICY_FIFO)
-			drm_sched_rq_update_fifo_locked(entity, rq, submit_ts);
-
-		spin_unlock(&rq->lock);
-		spin_unlock(&entity->lock);
-
-		drm_sched_wakeup(sched);
+		sched = drm_sched_rq_add_entity(entity);
+		if (sched)
+			drm_sched_wakeup(sched);
 	}
 }
 EXPORT_SYMBOL(drm_sched_entity_push_job);
diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c
index 096fe28aa9c9e..85ee3d694dc91 100644
--- a/drivers/gpu/drm/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/scheduler/sched_fence.c
@@ -227,7 +227,7 @@ void drm_sched_fence_init(struct drm_sched_fence *fence,
 {
 	unsigned seq;
 
-	fence->sched = entity->rq->sched;
+	fence->sched = container_of(entity->rq, typeof(*fence->sched), rq);
 	seq = atomic_inc_return(&entity->fence_seq);
 	dma_fence_init(&fence->scheduled, &drm_sched_fence_ops_scheduled,
 		       &fence->lock, entity->fence_context, seq);
diff --git a/drivers/gpu/drm/scheduler/sched_internal.h b/drivers/gpu/drm/scheduler/sched_internal.h
index 7ea5a6736f980..13ecb771d7a24 100644
--- a/drivers/gpu/drm/scheduler/sched_internal.h
+++ b/drivers/gpu/drm/scheduler/sched_internal.h
@@ -3,22 +3,48 @@
 #ifndef _DRM_GPU_SCHEDULER_INTERNAL_H_
 #define _DRM_GPU_SCHEDULER_INTERNAL_H_
 
+#include <linux/ktime.h>
+#include <linux/kref.h>
+#include <linux/spinlock.h>
 
-/* Used to choose between FIFO and RR job-scheduling */
-extern int drm_sched_policy;
-
-#define DRM_SCHED_POLICY_RR    0
-#define DRM_SCHED_POLICY_FIFO  1
-
+/**
+ * struct drm_sched_entity_stats - execution stats for an entity.
+ * @kref: reference count for the object.
+ * @lock: lock guarding the @runtime updates.
+ * @runtime: time entity spent on the GPU.
+ * @prev_runtime: previous @runtime used to get the runtime delta.
+ * @vruntime: virtual runtime as accumulated by the fair algorithm.
+ * @avg_job_us: average job duration.
+ *
+ * Because jobs and entities have decoupled lifetimes, ie. we cannot access the
+ * entity once the job has been de-queued, and we do need know how much GPU time
+ * each entity has spent, we need to track this in a separate object which is
+ * reference counted by both entities and jobs.
+ */
+struct drm_sched_entity_stats {
+	struct kref	kref;
+	spinlock_t	lock; /* Protects the below fields. */
+	ktime_t		runtime;
+	ktime_t		prev_runtime;
+	ktime_t		vruntime;
+
+	struct ewma_drm_sched_avgtime   avg_job_us;
+};
+
+bool drm_sched_can_queue(struct drm_gpu_scheduler *sched,
+			 struct drm_sched_entity *entity);
 void drm_sched_wakeup(struct drm_gpu_scheduler *sched);
 
-void drm_sched_rq_add_entity(struct drm_sched_rq *rq,
-			     struct drm_sched_entity *entity);
+void drm_sched_rq_init(struct drm_sched_rq *rq);
+
+struct drm_gpu_scheduler *
+drm_sched_rq_add_entity(struct drm_sched_entity *entity);
 void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
 				struct drm_sched_entity *entity);
+void drm_sched_rq_pop_entity(struct drm_sched_entity *entity);
 
-void drm_sched_rq_update_fifo_locked(struct drm_sched_entity *entity,
-				     struct drm_sched_rq *rq, ktime_t ts);
+struct drm_sched_entity *
+drm_sched_select_entity(struct drm_gpu_scheduler *sched);
 
 void drm_sched_entity_select_rq(struct drm_sched_entity *entity);
 struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity);
@@ -88,4 +114,32 @@ drm_sched_entity_is_ready(struct drm_sched_entity *entity)
 	return true;
 }
 
+void drm_sched_entity_stats_release(struct kref *kref);
+
+/**
+ * drm_sched_entity_stats_get - Obtain a reference count on &struct drm_sched_entity_stats object
+ * @stats: struct drm_sched_entity_stats pointer
+ *
+ * Return: struct drm_sched_entity_stats pointer
+ */
+static inline struct drm_sched_entity_stats *
+drm_sched_entity_stats_get(struct drm_sched_entity_stats *stats)
+{
+	kref_get(&stats->kref);
+
+	return stats;
+}
+
+/**
+ * drm_sched_entity_stats_put - Release a reference count on &struct drm_sched_entity_stats object
+ * @stats: struct drm_sched_entity_stats pointer
+ */
+static inline void
+drm_sched_entity_stats_put(struct drm_sched_entity_stats *stats)
+{
+	kref_put(&stats->kref, drm_sched_entity_stats_release);
+}
+
+ktime_t drm_sched_entity_stats_job_add_gpu_time(struct drm_sched_job *job);
+
 #endif
diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c
index 21dc82c75c9ef..818d3d4434b57 100644
--- a/drivers/gpu/drm/scheduler/sched_main.c
+++ b/drivers/gpu/drm/scheduler/sched_main.c
@@ -84,15 +84,6 @@
 #define CREATE_TRACE_POINTS
 #include "gpu_scheduler_trace.h"
 
-int drm_sched_policy = DRM_SCHED_POLICY_FIFO;
-
-/**
- * DOC: sched_policy (int)
- * Used to override default entities scheduling policy in a run queue.
- */
-MODULE_PARM_DESC(sched_policy, "Specify the scheduling policy for entities on a run-queue, " __stringify(DRM_SCHED_POLICY_RR) " = Round Robin, " __stringify(DRM_SCHED_POLICY_FIFO) " = FIFO (default).");
-module_param_named(sched_policy, drm_sched_policy, int, 0444);
-
 static u32 drm_sched_available_credits(struct drm_gpu_scheduler *sched)
 {
 	u32 credits;
@@ -112,8 +103,8 @@ static u32 drm_sched_available_credits(struct drm_gpu_scheduler *sched)
  * Return true if we can push at least one more job from @entity, false
  * otherwise.
  */
-static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched,
-				struct drm_sched_entity *entity)
+bool drm_sched_can_queue(struct drm_gpu_scheduler *sched,
+			 struct drm_sched_entity *entity)
 {
 	struct drm_sched_job *s_job;
 
@@ -133,211 +124,6 @@ static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched,
 	return drm_sched_available_credits(sched) >= s_job->credits;
 }
 
-static __always_inline bool drm_sched_entity_compare_before(struct rb_node *a,
-							    const struct rb_node *b)
-{
-	struct drm_sched_entity *ent_a =  rb_entry((a), struct drm_sched_entity, rb_tree_node);
-	struct drm_sched_entity *ent_b =  rb_entry((b), struct drm_sched_entity, rb_tree_node);
-
-	return ktime_before(ent_a->oldest_job_waiting, ent_b->oldest_job_waiting);
-}
-
-static void drm_sched_rq_remove_fifo_locked(struct drm_sched_entity *entity,
-					    struct drm_sched_rq *rq)
-{
-	if (!RB_EMPTY_NODE(&entity->rb_tree_node)) {
-		rb_erase_cached(&entity->rb_tree_node, &rq->rb_tree_root);
-		RB_CLEAR_NODE(&entity->rb_tree_node);
-	}
-}
-
-void drm_sched_rq_update_fifo_locked(struct drm_sched_entity *entity,
-				     struct drm_sched_rq *rq,
-				     ktime_t ts)
-{
-	/*
-	 * Both locks need to be grabbed, one to protect from entity->rq change
-	 * for entity from within concurrent drm_sched_entity_select_rq and the
-	 * other to update the rb tree structure.
-	 */
-	lockdep_assert_held(&entity->lock);
-	lockdep_assert_held(&rq->lock);
-
-	drm_sched_rq_remove_fifo_locked(entity, rq);
-
-	entity->oldest_job_waiting = ts;
-
-	rb_add_cached(&entity->rb_tree_node, &rq->rb_tree_root,
-		      drm_sched_entity_compare_before);
-}
-
-/**
- * drm_sched_rq_init - initialize a given run queue struct
- *
- * @sched: scheduler instance to associate with this run queue
- * @rq: scheduler run queue
- *
- * Initializes a scheduler runqueue.
- */
-static void drm_sched_rq_init(struct drm_gpu_scheduler *sched,
-			      struct drm_sched_rq *rq)
-{
-	spin_lock_init(&rq->lock);
-	INIT_LIST_HEAD(&rq->entities);
-	rq->rb_tree_root = RB_ROOT_CACHED;
-	rq->current_entity = NULL;
-	rq->sched = sched;
-}
-
-/**
- * drm_sched_rq_add_entity - add an entity
- *
- * @rq: scheduler run queue
- * @entity: scheduler entity
- *
- * Adds a scheduler entity to the run queue.
- */
-void drm_sched_rq_add_entity(struct drm_sched_rq *rq,
-			     struct drm_sched_entity *entity)
-{
-	lockdep_assert_held(&entity->lock);
-	lockdep_assert_held(&rq->lock);
-
-	if (!list_empty(&entity->list))
-		return;
-
-	atomic_inc(rq->sched->score);
-	list_add_tail(&entity->list, &rq->entities);
-}
-
-/**
- * drm_sched_rq_remove_entity - remove an entity
- *
- * @rq: scheduler run queue
- * @entity: scheduler entity
- *
- * Removes a scheduler entity from the run queue.
- */
-void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
-				struct drm_sched_entity *entity)
-{
-	lockdep_assert_held(&entity->lock);
-
-	if (list_empty(&entity->list))
-		return;
-
-	spin_lock(&rq->lock);
-
-	atomic_dec(rq->sched->score);
-	list_del_init(&entity->list);
-
-	if (rq->current_entity == entity)
-		rq->current_entity = NULL;
-
-	if (drm_sched_policy == DRM_SCHED_POLICY_FIFO)
-		drm_sched_rq_remove_fifo_locked(entity, rq);
-
-	spin_unlock(&rq->lock);
-}
-
-/**
- * drm_sched_rq_select_entity_rr - Select an entity which could provide a job to run
- *
- * @sched: the gpu scheduler
- * @rq: scheduler run queue to check.
- *
- * Try to find the next ready entity.
- *
- * Return an entity if one is found; return an error-pointer (!NULL) if an
- * entity was ready, but the scheduler had insufficient credits to accommodate
- * its job; return NULL, if no ready entity was found.
- */
-static struct drm_sched_entity *
-drm_sched_rq_select_entity_rr(struct drm_gpu_scheduler *sched,
-			      struct drm_sched_rq *rq)
-{
-	struct drm_sched_entity *entity;
-
-	spin_lock(&rq->lock);
-
-	entity = rq->current_entity;
-	if (entity) {
-		list_for_each_entry_continue(entity, &rq->entities, list) {
-			if (drm_sched_entity_is_ready(entity))
-				goto found;
-		}
-	}
-
-	list_for_each_entry(entity, &rq->entities, list) {
-		if (drm_sched_entity_is_ready(entity))
-			goto found;
-
-		if (entity == rq->current_entity)
-			break;
-	}
-
-	spin_unlock(&rq->lock);
-
-	return NULL;
-
-found:
-	if (!drm_sched_can_queue(sched, entity)) {
-		/*
-		 * If scheduler cannot take more jobs signal the caller to not
-		 * consider lower priority queues.
-		 */
-		entity = ERR_PTR(-ENOSPC);
-	} else {
-		rq->current_entity = entity;
-		reinit_completion(&entity->entity_idle);
-	}
-
-	spin_unlock(&rq->lock);
-
-	return entity;
-}
-
-/**
- * drm_sched_rq_select_entity_fifo - Select an entity which provides a job to run
- *
- * @sched: the gpu scheduler
- * @rq: scheduler run queue to check.
- *
- * Find oldest waiting ready entity.
- *
- * Return an entity if one is found; return an error-pointer (!NULL) if an
- * entity was ready, but the scheduler had insufficient credits to accommodate
- * its job; return NULL, if no ready entity was found.
- */
-static struct drm_sched_entity *
-drm_sched_rq_select_entity_fifo(struct drm_gpu_scheduler *sched,
-				struct drm_sched_rq *rq)
-{
-	struct rb_node *rb;
-
-	spin_lock(&rq->lock);
-	for (rb = rb_first_cached(&rq->rb_tree_root); rb; rb = rb_next(rb)) {
-		struct drm_sched_entity *entity;
-
-		entity = rb_entry(rb, struct drm_sched_entity, rb_tree_node);
-		if (drm_sched_entity_is_ready(entity)) {
-			/* If we can't queue yet, preserve the current entity in
-			 * terms of fairness.
-			 */
-			if (!drm_sched_can_queue(sched, entity)) {
-				spin_unlock(&rq->lock);
-				return ERR_PTR(-ENOSPC);
-			}
-
-			reinit_completion(&entity->entity_idle);
-			break;
-		}
-	}
-	spin_unlock(&rq->lock);
-
-	return rb ? rb_entry(rb, struct drm_sched_entity, rb_tree_node) : NULL;
-}
-
 /**
  * drm_sched_run_job_queue - enqueue run-job work
  * @sched: scheduler instance
@@ -862,10 +648,11 @@ void drm_sched_job_arm(struct drm_sched_job *job)
 
 	BUG_ON(!entity);
 	drm_sched_entity_select_rq(entity);
-	sched = entity->rq->sched;
+	sched = container_of(entity->rq, typeof(*sched), rq);
 
 	job->sched = sched;
 	job->s_priority = entity->priority;
+	job->entity_stats = drm_sched_entity_stats_get(entity->stats);
 
 	drm_sched_fence_init(job->s_fence, job->entity);
 }
@@ -1055,6 +842,7 @@ void drm_sched_job_cleanup(struct drm_sched_job *job)
 		 * been called.
 		 */
 		dma_fence_put(&job->s_fence->finished);
+		drm_sched_entity_stats_put(job->entity_stats);
 	} else {
 		/* The job was aborted before it has been committed to be run;
 		 * notably, drm_sched_job_arm() has not been called.
@@ -1083,41 +871,10 @@ void drm_sched_wakeup(struct drm_gpu_scheduler *sched)
 	drm_sched_run_job_queue(sched);
 }
 
-/**
- * drm_sched_select_entity - Select next entity to process
- *
- * @sched: scheduler instance
- *
- * Return an entity to process or NULL if none are found.
- *
- * Note, that we break out of the for-loop when "entity" is non-null, which can
- * also be an error-pointer--this assures we don't process lower priority
- * run-queues. See comments in the respectively called functions.
- */
-static struct drm_sched_entity *
-drm_sched_select_entity(struct drm_gpu_scheduler *sched)
-{
-	struct drm_sched_entity *entity;
-	int i;
-
-	/* Start with the highest priority.
-	 */
-	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) {
-		entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ?
-			drm_sched_rq_select_entity_fifo(sched, sched->sched_rq[i]) :
-			drm_sched_rq_select_entity_rr(sched, sched->sched_rq[i]);
-		if (entity)
-			break;
-	}
-
-	return IS_ERR(entity) ? NULL : entity;
-}
-
 /**
  * drm_sched_get_finished_job - fetch the next finished job to be destroyed
  *
  * @sched: scheduler instance
- * @have_more: are there more finished jobs on the list
  *
  * Informs the caller through @have_more whether there are more finished jobs
  * besides the returned one.
@@ -1126,7 +883,7 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched)
  * ready for it to be destroyed.
  */
 static struct drm_sched_job *
-drm_sched_get_finished_job(struct drm_gpu_scheduler *sched, bool *have_more)
+drm_sched_get_finished_job(struct drm_gpu_scheduler *sched)
 {
 	struct drm_sched_job *job, *next;
 
@@ -1141,7 +898,6 @@ drm_sched_get_finished_job(struct drm_gpu_scheduler *sched, bool *have_more)
 		/* cancel this job's TO timer */
 		cancel_delayed_work(&sched->work_tdr);
 
-		*have_more = false;
 		next = list_first_entry_or_null(&sched->pending_list,
 						typeof(*next), list);
 		if (next) {
@@ -1151,8 +907,6 @@ drm_sched_get_finished_job(struct drm_gpu_scheduler *sched, bool *have_more)
 				next->s_fence->scheduled.timestamp =
 					dma_fence_timestamp(&job->s_fence->finished);
 
-			*have_more = dma_fence_is_signaled(&next->s_fence->finished);
-
 			/* start TO timer for next job */
 			drm_sched_start_timeout(sched);
 		}
@@ -1211,13 +965,15 @@ static void drm_sched_free_job_work(struct work_struct *w)
 	struct drm_gpu_scheduler *sched =
 		container_of(w, struct drm_gpu_scheduler, work_free_job);
 	struct drm_sched_job *job;
-	bool have_more;
 
-	job = drm_sched_get_finished_job(sched, &have_more);
-	if (job) {
+	while ((job = drm_sched_get_finished_job(sched))) {
+		ktime_t duration = drm_sched_entity_stats_job_add_gpu_time(job);
+
+		/* Serialized by the worker. */
+		ewma_drm_sched_avgtime_add(&sched->avg_job_us,
+					   ktime_to_us(duration));
+
 		sched->ops->free_job(job);
-		if (have_more)
-			drm_sched_run_free_queue(sched);
 	}
 
 	drm_sched_run_job_queue(sched);
@@ -1240,7 +996,7 @@ static void drm_sched_run_job_work(struct work_struct *w)
 
 	/* Find entity with a ready job */
 	entity = drm_sched_select_entity(sched);
-	if (!entity) {
+	if (IS_ERR_OR_NULL(entity)) {
 		/*
 		 * Either no more work to do, or the next ready job needs more
 		 * credits than the scheduler has currently available.
@@ -1316,8 +1072,6 @@ static struct workqueue_struct *drm_sched_alloc_wq(const char *name)
  */
 int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_args *args)
 {
-	int i;
-
 	sched->ops = args->ops;
 	sched->credit_limit = args->credit_limit;
 	sched->name = args->name;
@@ -1327,21 +1081,6 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_
 	sched->score = args->score ? args->score : &sched->_score;
 	sched->dev = args->dev;
 
-	if (args->num_rqs > DRM_SCHED_PRIORITY_COUNT) {
-		/* This is a gross violation--tell drivers what the  problem is.
-		 */
-		dev_err(sched->dev, "%s: num_rqs cannot be greater than DRM_SCHED_PRIORITY_COUNT\n",
-			__func__);
-		return -EINVAL;
-	} else if (sched->sched_rq) {
-		/* Not an error, but warn anyway so drivers can
-		 * fine-tune their DRM calling order, and return all
-		 * is good.
-		 */
-		dev_warn(sched->dev, "%s: scheduler already initialized!\n", __func__);
-		return 0;
-	}
-
 	if (args->submit_wq) {
 		sched->submit_wq = args->submit_wq;
 		sched->own_submit_wq = false;
@@ -1353,17 +1092,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_
 		sched->own_submit_wq = true;
 	}
 
-	sched->sched_rq = kmalloc_objs(*sched->sched_rq, args->num_rqs,
-				       GFP_KERNEL | __GFP_ZERO);
-	if (!sched->sched_rq)
-		goto Out_check_own;
-	sched->num_rqs = args->num_rqs;
-	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) {
-		sched->sched_rq[i] = kzalloc_obj(*sched->sched_rq[i]);
-		if (!sched->sched_rq[i])
-			goto Out_unroll;
-		drm_sched_rq_init(sched, sched->sched_rq[i]);
-	}
+	drm_sched_rq_init(&sched->rq);
 
 	init_waitqueue_head(&sched->job_scheduled);
 	INIT_LIST_HEAD(&sched->pending_list);
@@ -1375,20 +1104,10 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, const struct drm_sched_init_
 	atomic_set(&sched->_score, 0);
 	atomic64_set(&sched->job_id_count, 0);
 	sched->pause_submit = false;
+	ewma_drm_sched_avgtime_init(&sched->avg_job_us);
 
 	sched->ready = true;
 	return 0;
-Out_unroll:
-	for (--i ; i >= DRM_SCHED_PRIORITY_KERNEL; i--)
-		kfree(sched->sched_rq[i]);
-
-	kfree(sched->sched_rq);
-	sched->sched_rq = NULL;
-Out_check_own:
-	if (sched->own_submit_wq)
-		destroy_workqueue(sched->submit_wq);
-	dev_err(sched->dev, "%s: Failed to setup GPU scheduler--out of memory\n", __func__);
-	return -ENOMEM;
 }
 EXPORT_SYMBOL(drm_sched_init);
 
@@ -1419,13 +1138,8 @@ static void drm_sched_cancel_remaining_jobs(struct drm_gpu_scheduler *sched)
  */
 void drm_sched_fini(struct drm_gpu_scheduler *sched)
 {
-	int i;
-
 	drm_sched_wqueue_stop(sched);
 
-	for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++)
-		kfree(sched->sched_rq[i]);
-
 	/* Wakeup everyone stuck in drm_sched_entity_flush for this scheduler */
 	wake_up_all(&sched->job_scheduled);
 
@@ -1439,8 +1153,6 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched)
 	if (sched->own_submit_wq)
 		destroy_workqueue(sched->submit_wq);
 	sched->ready = false;
-	kfree(sched->sched_rq);
-	sched->sched_rq = NULL;
 
 	if (!list_empty(&sched->pending_list))
 		dev_warn(sched->dev, "Tearing down scheduler while jobs are pending!\n");
@@ -1458,35 +1170,28 @@ EXPORT_SYMBOL(drm_sched_fini);
  */
 void drm_sched_increase_karma(struct drm_sched_job *bad)
 {
-	int i;
-	struct drm_sched_entity *tmp;
-	struct drm_sched_entity *entity;
 	struct drm_gpu_scheduler *sched = bad->sched;
+	struct drm_sched_entity *entity, *tmp;
+	struct drm_sched_rq *rq = &sched->rq;
 
 	/* don't change @bad's karma if it's from KERNEL RQ,
 	 * because sometimes GPU hang would cause kernel jobs (like VM updating jobs)
 	 * corrupt but keep in mind that kernel jobs always considered good.
 	 */
-	if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
-		atomic_inc(&bad->karma);
-
-		for (i = DRM_SCHED_PRIORITY_HIGH; i < sched->num_rqs; i++) {
-			struct drm_sched_rq *rq = sched->sched_rq[i];
-
-			spin_lock(&rq->lock);
-			list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
-				if (bad->s_fence->scheduled.context ==
-				    entity->fence_context) {
-					if (entity->guilty)
-						atomic_set(entity->guilty, 1);
-					break;
-				}
-			}
-			spin_unlock(&rq->lock);
-			if (&entity->list != &rq->entities)
-				break;
+	if (bad->s_priority == DRM_SCHED_PRIORITY_KERNEL)
+		return;
+
+	atomic_inc(&bad->karma);
+
+	spin_lock(&rq->lock);
+	list_for_each_entry_safe(entity, tmp, &rq->entities, list) {
+		if (bad->s_fence->scheduled.context == entity->fence_context) {
+			if (entity->guilty)
+				atomic_set(entity->guilty, 1);
+			break;
 		}
 	}
+	spin_unlock(&rq->lock);
 }
 EXPORT_SYMBOL(drm_sched_increase_karma);
 
diff --git a/drivers/gpu/drm/scheduler/sched_rq.c b/drivers/gpu/drm/scheduler/sched_rq.c
new file mode 100644
index 0000000000000..044546bcb5f82
--- /dev/null
+++ b/drivers/gpu/drm/scheduler/sched_rq.c
@@ -0,0 +1,383 @@
+// SPDX-License-Identifier: MIT
+/* Copyright 2015 Advanced Micro Devices, Inc. */
+/* Copyright (c) 2025 Valve Corporation */
+
+#include <linux/rbtree.h>
+
+#include <drm/drm_print.h>
+#include <drm/gpu_scheduler.h>
+
+#include "sched_internal.h"
+
+static __always_inline bool
+drm_sched_entity_compare_before(struct rb_node *a, const struct rb_node *b)
+{
+	struct drm_sched_entity *ea =
+		rb_entry((a), struct drm_sched_entity, rb_tree_node);
+	struct drm_sched_entity *eb =
+		rb_entry((b), struct drm_sched_entity, rb_tree_node);
+
+	return ktime_before(ea->oldest_job_waiting, eb->oldest_job_waiting);
+}
+
+static void drm_sched_rq_update_prio(struct drm_sched_rq *rq)
+{
+	enum drm_sched_priority prio = DRM_SCHED_PRIORITY_INVALID;
+	struct rb_node *rb;
+
+	lockdep_assert_held(&rq->lock);
+
+	rb = rb_first_cached(&rq->rb_tree_root);
+	if (rb) {
+		struct drm_sched_entity *entity =
+			rb_entry(rb, typeof(*entity), rb_tree_node);
+
+		/*
+		 * The normal locking order is entity then run-queue so taking
+		 * the entity lock here would be a locking inversion for the
+		 * case when the current head of the run-queue is different from
+		 * the one we already have locked. The unlocked read is fine
+		 * though, because if the priority had just changed it is no big
+		 * deal for our algorithm, but just a transient reachable only
+		 * by drivers with userspace dynamic priority changes API. Equal
+		 * in effect to the priority change becoming visible a few
+		 * instructions later.
+		 */
+		prio = READ_ONCE(entity->priority);
+	}
+
+	rq->head_prio = prio;
+}
+
+static void drm_sched_rq_remove_tree_locked(struct drm_sched_entity *entity,
+					    struct drm_sched_rq *rq)
+{
+	lockdep_assert_held(&entity->lock);
+	lockdep_assert_held(&rq->lock);
+
+	if (!RB_EMPTY_NODE(&entity->rb_tree_node)) {
+		rb_erase_cached(&entity->rb_tree_node, &rq->rb_tree_root);
+		RB_CLEAR_NODE(&entity->rb_tree_node);
+		drm_sched_rq_update_prio(rq);
+	}
+}
+
+static void drm_sched_rq_update_tree_locked(struct drm_sched_entity *entity,
+					    struct drm_sched_rq *rq,
+					    ktime_t ts)
+{
+	/*
+	 * Both locks need to be grabbed, one to protect from entity->rq change
+	 * for entity from within concurrent drm_sched_entity_select_rq and the
+	 * other to update the rb tree structure.
+	 */
+	lockdep_assert_held(&entity->lock);
+	lockdep_assert_held(&rq->lock);
+
+	drm_sched_rq_remove_tree_locked(entity, rq);
+
+	entity->oldest_job_waiting = ts;
+
+	rb_add_cached(&entity->rb_tree_node, &rq->rb_tree_root,
+		      drm_sched_entity_compare_before);
+	drm_sched_rq_update_prio(rq);
+}
+
+/**
+ * drm_sched_rq_init - initialize a given run queue struct
+ * @rq: scheduler run queue
+ *
+ * Initializes a scheduler runqueue.
+ */
+void drm_sched_rq_init(struct drm_sched_rq *rq)
+{
+	spin_lock_init(&rq->lock);
+	INIT_LIST_HEAD(&rq->entities);
+	rq->rb_tree_root = RB_ROOT_CACHED;
+	rq->head_prio = DRM_SCHED_PRIORITY_INVALID;
+}
+
+/*
+ * Core part of the CFS-like algorithm is that the virtual runtime of lower
+ * priority tasks should grow quicker than the higher priority ones, so that
+ * when we then schedule entities with the aim of keeping their accumulated
+ * virtual time balanced, we can approach fair distribution of GPU time.
+ *
+ * For converting the real GPU time into virtual we pick some multipliers with
+ * the idea to achieve the following GPU time distribution:
+ *
+ *  - Kernel priority gets roughly 2x GPU time compared to high.
+ *  - High gets ~4x relative to normal.
+ *  - Normal gets ~8x relative to low.
+ */
+static const unsigned int vruntime_shift[] = {
+	[DRM_SCHED_PRIORITY_KERNEL] = 1,
+	[DRM_SCHED_PRIORITY_HIGH]   = 2,
+	[DRM_SCHED_PRIORITY_NORMAL] = 4,
+	[DRM_SCHED_PRIORITY_LOW]    = 7,
+};
+
+static ktime_t
+drm_sched_rq_get_min_vruntime(struct drm_sched_rq *rq)
+{
+	ktime_t vruntime = 0;
+	struct rb_node *rb;
+
+	lockdep_assert_held(&rq->lock);
+
+	rb = rb_first_cached(&rq->rb_tree_root);
+	if (rb) {
+		struct drm_sched_entity *entity =
+			rb_entry(rb, typeof(*entity), rb_tree_node);
+		struct drm_sched_entity_stats *stats = entity->stats;
+
+		spin_lock(&stats->lock);
+		vruntime = stats->vruntime;
+		spin_unlock(&stats->lock);
+	}
+
+	return vruntime;
+}
+
+static void
+drm_sched_entity_save_vruntime(struct drm_sched_entity *entity,
+			       ktime_t min_vruntime)
+{
+	struct drm_sched_entity_stats *stats = entity->stats;
+	ktime_t vruntime;
+
+	spin_lock(&stats->lock);
+	vruntime = stats->vruntime;
+	if (min_vruntime && vruntime > min_vruntime)
+		vruntime = ktime_sub(vruntime, min_vruntime);
+	else
+		vruntime = 0;
+	stats->vruntime = vruntime;
+	spin_unlock(&stats->lock);
+}
+
+static ktime_t
+drm_sched_entity_restore_vruntime(struct drm_sched_entity *entity,
+				  ktime_t min_vruntime,
+				  enum drm_sched_priority rq_prio)
+{
+	struct drm_sched_entity_stats *stats = entity->stats;
+	struct drm_gpu_scheduler *sched =
+			container_of(entity->rq, typeof(*sched), rq);
+	enum drm_sched_priority prio = entity->priority;
+	unsigned long avg_us, sched_avg_us;
+	ktime_t vruntime;
+
+	BUILD_BUG_ON(DRM_SCHED_PRIORITY_NORMAL < DRM_SCHED_PRIORITY_HIGH);
+
+	spin_lock(&stats->lock);
+	vruntime = stats->vruntime;
+	avg_us = ewma_drm_sched_avgtime_read(&stats->avg_job_us);
+	/*
+	 * Unlocked read of the scheduler average is fine since it is just
+	 * heuristics and data type is a natural word size.
+	 */
+	sched_avg_us = ewma_drm_sched_avgtime_read(&sched->avg_job_us);
+
+	/*
+	 * Special handling for entities which were picked from the top of the
+	 * queue and are now re-joining the top with another one already there.
+	 */
+	if (!vruntime && rq_prio != DRM_SCHED_PRIORITY_INVALID) {
+		if (prio > rq_prio) {
+			/*
+			 * Lower priority should not overtake higher when re-
+			 * joining at the top of the queue so push it back
+			 * somewhere behind the "middle" of the run-queue,
+			 * proportional to the scheduler and entity average job
+			 * durations.
+			 */
+			vruntime = us_to_ktime((1 + avg_us + sched_avg_us) <<
+					       vruntime_shift[prio]);
+		} else if (prio < rq_prio) {
+			/*
+			 * Higher priority can go first.
+			 */
+			vruntime = -ns_to_ktime(rq_prio - prio);
+		} else {
+			/* Favour entity with shorter jobs (interactivity). */
+			if (avg_us <= sched_avg_us)
+				vruntime = -ns_to_ktime(1);
+			else
+				vruntime = ns_to_ktime(1);
+		}
+	}
+
+	/*
+	 * Restore saved relative position in the queue.
+	 */
+	vruntime = ktime_add(min_vruntime, vruntime);
+
+	stats->vruntime = vruntime;
+	spin_unlock(&stats->lock);
+
+	return vruntime;
+}
+
+static ktime_t drm_sched_entity_update_vruntime(struct drm_sched_entity *entity)
+{
+	struct drm_sched_entity_stats *stats = entity->stats;
+	ktime_t runtime, prev;
+
+	spin_lock(&stats->lock);
+	prev = stats->prev_runtime;
+	runtime = stats->runtime;
+	stats->prev_runtime = runtime;
+	runtime = ktime_add_ns(stats->vruntime,
+			       ktime_to_ns(ktime_sub(runtime, prev)) <<
+			       vruntime_shift[entity->priority]);
+	stats->vruntime = runtime;
+	spin_unlock(&stats->lock);
+
+	return runtime;
+}
+
+/**
+ * drm_sched_rq_add_entity - add an entity
+ * @entity: scheduler entity
+ *
+ * Adds a scheduler entity to the run queue.
+ *
+ * Return: DRM scheduler selected to handle this entity or NULL if entity has
+ * been stopped and cannot be submitted to.
+ */
+struct drm_gpu_scheduler *
+drm_sched_rq_add_entity(struct drm_sched_entity *entity)
+{
+	struct drm_gpu_scheduler *sched;
+	struct drm_sched_rq *rq;
+	ktime_t ts;
+
+	/* Add the entity to the run queue */
+	spin_lock(&entity->lock);
+	if (entity->stopped) {
+		spin_unlock(&entity->lock);
+
+		DRM_ERROR("Trying to push to a killed entity\n");
+		return NULL;
+	}
+
+	rq = entity->rq;
+	sched = container_of(rq, typeof(*sched), rq);
+	spin_lock(&rq->lock);
+
+	if (list_empty(&entity->list)) {
+		atomic_inc(sched->score);
+		list_add_tail(&entity->list, &rq->entities);
+	}
+
+	ts = drm_sched_rq_get_min_vruntime(rq);
+	ts = drm_sched_entity_restore_vruntime(entity, ts, rq->head_prio);
+	drm_sched_rq_update_tree_locked(entity, rq, ts);
+
+	spin_unlock(&rq->lock);
+	spin_unlock(&entity->lock);
+
+	return sched;
+}
+
+/**
+ * drm_sched_rq_remove_entity - remove an entity
+ * @rq: scheduler run queue
+ * @entity: scheduler entity
+ *
+ * Removes a scheduler entity from the run queue.
+ */
+void drm_sched_rq_remove_entity(struct drm_sched_rq *rq,
+				struct drm_sched_entity *entity)
+{
+	struct drm_gpu_scheduler *sched = container_of(rq, typeof(*sched), rq);
+
+	lockdep_assert_held(&entity->lock);
+
+	if (list_empty(&entity->list))
+		return;
+
+	spin_lock(&rq->lock);
+
+	atomic_dec(sched->score);
+	list_del_init(&entity->list);
+
+	drm_sched_rq_remove_tree_locked(entity, rq);
+
+	spin_unlock(&rq->lock);
+}
+
+/**
+ * drm_sched_rq_pop_entity - pops an entity
+ * @entity: scheduler entity
+ *
+ * To be called every time after a job is popped from the entity.
+ */
+void drm_sched_rq_pop_entity(struct drm_sched_entity *entity)
+{
+	struct drm_sched_job *next_job;
+	struct drm_sched_rq *rq;
+
+	/*
+	 * Update the entity's location in the min heap according to
+	 * the timestamp of the next job, if any.
+	 */
+	spin_lock(&entity->lock);
+	rq = entity->rq;
+	spin_lock(&rq->lock);
+	next_job = drm_sched_entity_queue_peek(entity);
+	if (next_job) {
+		ktime_t ts;
+
+		ts = drm_sched_entity_update_vruntime(entity);
+		drm_sched_rq_update_tree_locked(entity, rq, ts);
+	} else {
+		ktime_t min_vruntime;
+
+		drm_sched_rq_remove_tree_locked(entity, rq);
+		min_vruntime = drm_sched_rq_get_min_vruntime(rq);
+		drm_sched_entity_save_vruntime(entity, min_vruntime);
+	}
+	spin_unlock(&rq->lock);
+	spin_unlock(&entity->lock);
+}
+
+/**
+ * drm_sched_select_entity - Select an entity which provides a job to run
+ * @sched: the gpu scheduler
+ *
+ * Find oldest waiting ready entity.
+ *
+ * Return an entity if one is found; return an error-pointer (!NULL) if an
+ * entity was ready, but the scheduler had insufficient credits to accommodate
+ * its job; return NULL, if no ready entity was found.
+ */
+struct drm_sched_entity *
+drm_sched_select_entity(struct drm_gpu_scheduler *sched)
+{
+	struct drm_sched_rq *rq = &sched->rq;
+	struct rb_node *rb;
+
+	spin_lock(&rq->lock);
+	for (rb = rb_first_cached(&rq->rb_tree_root); rb; rb = rb_next(rb)) {
+		struct drm_sched_entity *entity;
+
+		entity = rb_entry(rb, struct drm_sched_entity, rb_tree_node);
+		if (drm_sched_entity_is_ready(entity)) {
+			/* If we can't queue yet, preserve the current entity in
+			 * terms of fairness.
+			 */
+			if (!drm_sched_can_queue(sched, entity)) {
+				spin_unlock(&rq->lock);
+				return ERR_PTR(-ENOSPC);
+			}
+
+			reinit_completion(&entity->entity_idle);
+			break;
+		}
+	}
+	spin_unlock(&rq->lock);
+
+	return rb ? rb_entry(rb, struct drm_sched_entity, rb_tree_node) : NULL;
+}
diff --git a/drivers/gpu/drm/scheduler/tests/Makefile b/drivers/gpu/drm/scheduler/tests/Makefile
index 5bf707bad3734..9ec185fbbc153 100644
--- a/drivers/gpu/drm/scheduler/tests/Makefile
+++ b/drivers/gpu/drm/scheduler/tests/Makefile
@@ -2,6 +2,7 @@
 
 drm-sched-tests-y := \
         mock_scheduler.o \
-        tests_basic.o
+        tests_basic.o \
+        tests_scheduler.o
 
 obj-$(CONFIG_DRM_SCHED_KUNIT_TEST) += drm-sched-tests.o
diff --git a/drivers/gpu/drm/scheduler/tests/mock_scheduler.c b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c
index 8e9ae7d980eb2..14403a7623352 100644
--- a/drivers/gpu/drm/scheduler/tests/mock_scheduler.c
+++ b/drivers/gpu/drm/scheduler/tests/mock_scheduler.c
@@ -290,7 +290,6 @@ struct drm_mock_scheduler *drm_mock_sched_new(struct kunit *test, long timeout)
 {
 	struct drm_sched_init_args args = {
 		.ops		= &drm_mock_scheduler_ops,
-		.num_rqs	= DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit	= U32_MAX,
 		.hang_limit	= 1,
 		.timeout	= timeout,
diff --git a/drivers/gpu/drm/scheduler/tests/tests_scheduler.c b/drivers/gpu/drm/scheduler/tests/tests_scheduler.c
new file mode 100644
index 0000000000000..8b2e4ef9915f0
--- /dev/null
+++ b/drivers/gpu/drm/scheduler/tests/tests_scheduler.c
@@ -0,0 +1,882 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2025 Valve Corporation */
+
+#include <linux/delay.h>
+#include <linux/kthread.h>
+#include <linux/ktime.h>
+#include <linux/math64.h>
+
+#include "sched_tests.h"
+
+/*
+ * DRM scheduler tests exercise load balancing decisions ie. entity selection
+ * logic.
+ */
+
+static int drm_sched_scheduler_init(struct kunit *test)
+{
+	struct drm_mock_scheduler *sched;
+
+	sched = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT);
+	sched->base.credit_limit = 1;
+
+	test->priv = sched;
+
+	return 0;
+}
+
+static int drm_sched_scheduler_init2(struct kunit *test)
+{
+	struct drm_mock_scheduler *sched;
+
+	sched = drm_mock_sched_new(test, MAX_SCHEDULE_TIMEOUT);
+	sched->base.credit_limit = 2;
+
+	test->priv = sched;
+
+	return 0;
+}
+
+static void drm_sched_scheduler_exit(struct kunit *test)
+{
+	struct drm_mock_scheduler *sched = test->priv;
+
+	drm_mock_sched_fini(sched);
+}
+
+static void drm_sched_scheduler_queue_overhead(struct kunit *test)
+{
+	struct drm_mock_scheduler *sched = test->priv;
+	struct drm_mock_sched_entity *entity;
+	const unsigned int job_us = 1000;
+	const unsigned int jobs = 1000;
+	const unsigned int total_us = jobs * job_us;
+	struct drm_mock_sched_job *job, *first;
+	ktime_t start, end;
+	bool done;
+	int i;
+
+	/*
+	 * Deep queue job at a time processing (single credit).
+	 *
+	 * This measures the overhead of picking and processing a job at a time
+	 * by comparing the ideal total "GPU" time of all submitted jobs versus
+	 * the time actually taken.
+	 */
+
+	KUNIT_ASSERT_EQ(test, sched->base.credit_limit, 1);
+
+	entity = drm_mock_sched_entity_new(test,
+					   DRM_SCHED_PRIORITY_NORMAL,
+					   sched);
+
+	for (i = 0; i <= jobs; i++) {
+		job = drm_mock_sched_job_new(test, entity);
+		if (i == 0)
+			first = job; /* Extra first job blocks the queue */
+		else
+			drm_mock_sched_job_set_duration_us(job, job_us);
+		drm_mock_sched_job_submit(job);
+	}
+
+	done = drm_mock_sched_job_wait_scheduled(first, HZ);
+	KUNIT_ASSERT_TRUE(test, done);
+
+	start = ktime_get();
+	i = drm_mock_sched_advance(sched, 1); /* Release the queue */
+	KUNIT_ASSERT_EQ(test, i, 1);
+
+	/* Wait with a safe margin to avoid every failing. */
+	done = drm_mock_sched_job_wait_finished(job,
+						usecs_to_jiffies(total_us) * 5);
+	end = ktime_get();
+	KUNIT_ASSERT_TRUE(test, done);
+
+	pr_info("Expected %uus, actual %lldus\n",
+		total_us,
+		ktime_to_us(ktime_sub(end, start)));
+
+	drm_mock_sched_entity_free(entity);
+}
+
+static void drm_sched_scheduler_ping_pong(struct kunit *test)
+{
+	struct drm_mock_sched_job *job, *first, *prev = NULL;
+	struct drm_mock_scheduler *sched = test->priv;
+	struct drm_mock_sched_entity *entity[2];
+	const unsigned int job_us = 1000;
+	const unsigned int jobs = 1000;
+	const unsigned int total_us = jobs * job_us;
+	ktime_t start, end;
+	bool done;
+	int i;
+
+	/*
+	 * Two entitites in inter-dependency chain.
+	 *
+	 * This measures the overhead of picking and processing a job at a time,
+	 * where each job depends on the previous one from the diffferent
+	 * entity, by comparing the ideal total "GPU" time of all submitted jobs
+	 * versus the time actually taken.
+	 */
+
+	KUNIT_ASSERT_EQ(test, sched->base.credit_limit, 1);
+
+	for (i = 0; i < ARRAY_SIZE(entity); i++)
+		entity[i] = drm_mock_sched_entity_new(test,
+						      DRM_SCHED_PRIORITY_NORMAL,
+						      sched);
+
+	for (i = 0; i <= jobs; i++) {
+		job = drm_mock_sched_job_new(test, entity[i & 1]);
+		if (i == 0)
+			first = job; /* Extra first job blocks the queue */
+		else
+			drm_mock_sched_job_set_duration_us(job, job_us);
+		if (prev)
+			drm_sched_job_add_dependency(&job->base,
+						     dma_fence_get(&prev->base.s_fence->finished));
+		drm_mock_sched_job_submit(job);
+		prev = job;
+	}
+
+	done = drm_mock_sched_job_wait_scheduled(first, HZ);
+	KUNIT_ASSERT_TRUE(test, done);
+
+	start = ktime_get();
+	i = drm_mock_sched_advance(sched, 1); /* Release the queue */
+	KUNIT_ASSERT_EQ(test, i, 1);
+
+	/* Wait with a safe margin to avoid every failing. */
+	done = drm_mock_sched_job_wait_finished(job,
+						usecs_to_jiffies(total_us) * 5);
+	end = ktime_get();
+	KUNIT_ASSERT_TRUE(test, done);
+
+	pr_info("Expected %uus, actual %lldus\n",
+		total_us,
+		ktime_to_us(ktime_sub(end, start)));
+
+	for (i = 0; i < ARRAY_SIZE(entity); i++)
+		drm_mock_sched_entity_free(entity[i]);
+}
+
+static struct kunit_case drm_sched_scheduler_overhead_tests[] = {
+	KUNIT_CASE_SLOW(drm_sched_scheduler_queue_overhead),
+	KUNIT_CASE_SLOW(drm_sched_scheduler_ping_pong),
+	{}
+};
+
+static struct kunit_suite drm_sched_scheduler_overhead = {
+	.name = "drm_sched_scheduler_overhead_tests",
+	.init = drm_sched_scheduler_init,
+	.exit = drm_sched_scheduler_exit,
+	.test_cases = drm_sched_scheduler_overhead_tests,
+};
+
+/*
+ * struct drm_sched_client_params - describe a workload emitted from a client
+ *
+ * A simulated client will create an entity with a scheduling @priority and emit
+ * jobs in a loop where each iteration will consist of:
+ *
+ * 1. Submit @job_cnt jobs, each with a set duration of @job_us.
+ * 2. If @sync is true wait for last submitted job to finish.
+ * 3. Sleep for @wait_us micro-seconds.
+ * 4. Repeat.
+ */
+struct drm_sched_client_params {
+	enum drm_sched_priority priority;
+	unsigned int job_cnt;
+	unsigned int job_us;
+	bool sync;
+	unsigned int wait_us;
+};
+
+struct drm_sched_test_params {
+	const char *description;
+	unsigned int num_clients;
+	struct drm_sched_client_params client[2];
+};
+
+static const struct drm_sched_test_params drm_sched_cases[] = {
+	{
+		.description = "Normal priority and normal priority",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 8000,
+			.wait_us = 0,
+			.sync = false,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 8000,
+			.wait_us = 0,
+			.sync = false,
+		},
+	},
+	{
+		.description = "Normal priority and low priority",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 8000,
+			.wait_us = 0,
+			.sync = false,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_LOW,
+			.job_cnt = 1,
+			.job_us = 8000,
+			.wait_us = 0,
+			.sync = false,
+		},
+	},
+	{
+		.description = "High priority and normal priority",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_HIGH,
+			.job_cnt = 1,
+			.job_us = 8000,
+			.wait_us = 0,
+			.sync = false,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 8000,
+			.wait_us = 0,
+			.sync = false,
+		},
+	},
+	{
+		.description = "High priority and low priority",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_HIGH,
+			.job_cnt = 1,
+			.job_us = 8000,
+			.wait_us = 0,
+			.sync = false,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_LOW,
+			.job_cnt = 1,
+			.job_us = 8000,
+			.wait_us = 0,
+			.sync = false,
+		},
+	},
+	{
+		.description = "50% and 50%",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 1500,
+			.wait_us = 1500,
+			.sync = true,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 2500,
+			.wait_us = 2500,
+			.sync = true,
+		},
+	},
+	{
+		.description = "50% and 50% low priority",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 1500,
+			.wait_us = 1500,
+			.sync = true,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_LOW,
+			.job_cnt = 1,
+			.job_us = 2500,
+			.wait_us = 2500,
+			.sync = true,
+		},
+	},
+	{
+		.description = "50% high priority and 50%",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_HIGH,
+			.job_cnt = 1,
+			.job_us = 1500,
+			.wait_us = 1500,
+			.sync = true,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 2500,
+			.wait_us = 2500,
+			.sync = true,
+		},
+	},
+	{
+		.description = "Low priority hog and interactive client",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_LOW,
+			.job_cnt = 3,
+			.job_us = 2500,
+			.wait_us = 500,
+			.sync = false,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 500,
+			.wait_us = 10000,
+			.sync = true,
+		},
+	},
+	{
+		.description = "Heavy rendering and interactive client",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 3,
+			.job_us = 2500,
+			.wait_us = 2500,
+			.sync = true,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 1000,
+			.wait_us = 9000,
+			.sync = true,
+		},
+	},
+	{
+		.description = "Very heavy rendering and interactive client",
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 4,
+			.job_us = 50000,
+			.wait_us = 1,
+			.sync = true,
+		},
+		.client[1] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 1,
+			.job_us = 1000,
+			.wait_us = 9000,
+			.sync = true,
+		},
+	},
+};
+
+static void
+drm_sched_desc(const struct drm_sched_test_params *params, char *desc)
+{
+	strscpy(desc, params->description, KUNIT_PARAM_DESC_SIZE);
+}
+
+KUNIT_ARRAY_PARAM(drm_sched_scheduler_two_clients,
+		  drm_sched_cases,
+		  drm_sched_desc);
+
+/*
+ * struct test_client_stats - track client stats
+ *
+ * For each client executing a simulated workload we track some timings for
+ * which we are interested in the minimum of all iterations (@min_us), maximum
+ * (@max_us) and the overall total for all iterations (@tot_us).
+ */
+struct test_client_stats {
+	unsigned int min_us;
+	unsigned int max_us;
+	unsigned long tot_us;
+};
+
+/*
+ * struct test_client - a simulated userspace client submitting scheduler work
+ *
+ * Each client executing a simulated workload is represented by one of these.
+ *
+ * Each of them instantiates a scheduling @entity and executes a workloads as
+ * defined in @params. Based on those @params the theoretical execution time of
+ * the client is calculated as @ideal_duration, while the actual wall time is
+ * tracked in @duration (calculated based on the @start and @end client time-
+ * stamps).
+ *
+ * Numerical @id is assigned to each for logging purposes.
+ *
+ * @worker and @work are used to provide an independent execution context from
+ * which scheduler jobs are submitted.
+ *
+ * During execution statistics on how long it took to submit and execute one
+ * iteration (whether or not synchronous) is kept in @cycle_time, while
+ * @latency_time tracks the @cycle_time minus the ideal duration of the one
+ * cycle.
+ *
+ * Once the client has completed the set number of iterations it will write the
+ * completion status into @done.
+ */
+struct test_client {
+	struct kunit	*test; /* Backpointer to the kunit test. */
+
+	struct drm_mock_sched_entity	*entity;
+	struct kthread_worker		*worker;
+	struct kthread_work		work;
+
+	struct drm_sched_client_params	params;
+
+	unsigned int	id;
+	ktime_t		duration;
+	ktime_t		ideal_duration;
+	unsigned int	cycles;
+	unsigned int	cycle;
+	ktime_t		start;
+	ktime_t		end;
+	bool		done;
+
+	struct test_client_stats	cycle_time;
+	struct test_client_stats	latency_time;
+};
+
+static void
+update_stats(struct test_client_stats *stats, unsigned int us)
+{
+	if (us > stats->max_us)
+		stats->max_us = us;
+	if (us < stats->min_us)
+		stats->min_us = us;
+	stats->tot_us += us;
+}
+
+static unsigned int
+get_stats_avg(struct test_client_stats *stats, unsigned int cycles)
+{
+	return div_u64(stats->tot_us, cycles);
+}
+
+static void drm_sched_client_work(struct kthread_work *work)
+{
+	struct test_client *client = container_of(work, typeof(*client), work);
+	const long sync_wait = MAX_SCHEDULE_TIMEOUT;
+	unsigned int cycle, work_us, period_us;
+	struct drm_mock_sched_job *job = NULL;
+
+	work_us = client->params.job_cnt * client->params.job_us;
+	period_us = work_us + client->params.wait_us;
+	client->cycles =
+		DIV_ROUND_UP((unsigned int)ktime_to_us(client->duration),
+			     period_us);
+	client->ideal_duration = us_to_ktime(client->cycles * period_us);
+
+	client->start = ktime_get();
+
+	for (cycle = 0; cycle < client->cycles; cycle++) {
+		ktime_t cycle_time;
+		unsigned int batch;
+		unsigned long us;
+
+		if (READ_ONCE(client->done))
+			break;
+
+		cycle_time = ktime_get();
+		for (batch = 0; batch < client->params.job_cnt; batch++) {
+			job = drm_mock_sched_job_new(client->test,
+						     client->entity);
+			drm_mock_sched_job_set_duration_us(job,
+							   client->params.job_us);
+			drm_mock_sched_job_submit(job);
+		}
+
+		if (client->params.sync)
+			drm_mock_sched_job_wait_finished(job, sync_wait);
+
+		cycle_time = ktime_sub(ktime_get(), cycle_time);
+		us = ktime_to_us(cycle_time);
+		update_stats(&client->cycle_time, us);
+		if (ktime_to_us(cycle_time) >= (long)work_us)
+			us = ktime_to_us(cycle_time) - work_us;
+		else if (WARN_ON_ONCE(client->params.sync)) /* GPU job took less than expected. */
+			us = 0;
+		update_stats(&client->latency_time, us);
+		WRITE_ONCE(client->cycle, cycle);
+
+		if (READ_ONCE(client->done))
+			break;
+
+		if (client->params.wait_us)
+			fsleep(client->params.wait_us);
+		else if (!client->params.sync)
+			cond_resched(); /* Do not hog the CPU if fully async. */
+	}
+
+	client->done = drm_mock_sched_job_wait_finished(job, sync_wait);
+	client->end = ktime_get();
+}
+
+static const char *prio_str(enum drm_sched_priority prio)
+{
+	switch (prio) {
+	case DRM_SCHED_PRIORITY_KERNEL:
+		return "kernel";
+	case DRM_SCHED_PRIORITY_LOW:
+		return "low";
+	case DRM_SCHED_PRIORITY_NORMAL:
+		return "normal";
+	case DRM_SCHED_PRIORITY_HIGH:
+		return "high";
+	default:
+		return "???";
+	}
+}
+
+static bool client_done(struct test_client *client)
+{
+	return READ_ONCE(client->done); /* READ_ONCE to document lockless read from a loop. */
+}
+
+static void drm_sched_scheduler_two_clients_test(struct kunit *test)
+{
+	const struct drm_sched_test_params *params = test->param_value;
+	struct drm_mock_scheduler *sched = test->priv;
+	struct test_client client[2] = { };
+	unsigned int prev_cycle[2] = { };
+	unsigned int i, j;
+	ktime_t start;
+
+	/*
+	 * Same job stream from two clients.
+	 */
+
+	for (i = 0; i < ARRAY_SIZE(client); i++)
+		client[i].entity =
+			drm_mock_sched_entity_new(test,
+						  params->client[i].priority,
+						  sched);
+
+	for (i = 0; i < ARRAY_SIZE(client); i++) {
+		client[i].test = test;
+		client[i].id = i;
+		client[i].duration = ms_to_ktime(1000);
+		client[i].params = params->client[i];
+		client[i].cycle_time.min_us = ~0U;
+		client[i].latency_time.min_us = ~0U;
+		client[i].worker =
+			kthread_create_worker(0, "%s-%u", __func__, i);
+		if (IS_ERR(client[i].worker)) {
+			for (j = 0; j < i; j++)
+				kthread_destroy_worker(client[j].worker);
+			KUNIT_FAIL(test, "Failed to create worker!\n");
+		}
+
+		kthread_init_work(&client[i].work, drm_sched_client_work);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(client); i++)
+		kthread_queue_work(client[i].worker, &client[i].work);
+
+	/*
+	 * The clients (workers) can be a mix of async (deep submission queue),
+	 * sync (one job at a time), or something in between. Therefore it is
+	 * difficult to display a single metric representing their progress.
+	 *
+	 * Each struct drm_sched_client_params describes the actual submission
+	 * pattern which happens in the following steps:
+	 *  1. Submit N jobs
+	 *  2. Wait for last submitted job to finish
+	 *  3. Sleep for U micro-seconds
+	 *  4. Goto 1. for C cycles
+	 *
+	 * Where number of cycles is calculated to match the target client
+	 * duration from the respective struct drm_sched_test_params.
+	 *
+	 * To asses scheduling behaviour what we output for both clients is:
+	 *  - pct: Percentage progress of the jobs submitted
+	 *  - cps: "Cycles" per second (where one cycle is one complete
+	 *         iteration from the above)
+	 *  -  qd: Number of outstanding jobs in the client/entity
+	 */
+
+	pr_info(" [pct] - Job sumission progress\n"
+" [cps] - Cycles per second\n"
+"  [qd] - Number of outstanding jobs in the client/entity\n");
+	pr_info("%s:\n\t            pct1 cps1 qd1;  pct2 cps2 qd2\n",
+		params->description);
+	start = ktime_get();
+	while (!client_done(&client[0]) || !client_done(&client[1])) {
+		const unsigned int period_ms = 100;
+		const unsigned int frequency = 1000 / period_ms;
+		unsigned int pct[2], qd[2], cycle[2], cps[2];
+
+		for (i = 0; i < ARRAY_SIZE(client); i++) {
+			qd[i] = spsc_queue_count(&client[i].entity->base.job_queue);
+			cycle[i] = READ_ONCE(client[i].cycle);
+			cps[i] = DIV_ROUND_UP(100 * frequency *
+					      (cycle[i] - prev_cycle[i]),
+					      100);
+			if (client[i].cycles)
+				pct[i] = DIV_ROUND_UP(100 * (1 + cycle[i]),
+						      client[i].cycles);
+			else
+				pct[i] = 0;
+			prev_cycle[i] = cycle[i];
+		}
+
+		if (client_done(&client[0]))
+			pr_info("\t+%6lldms:               ; %3u %5u %4u\n",
+				ktime_to_ms(ktime_sub(ktime_get(), start)),
+				pct[1], cps[1], qd[1]);
+		else if (client_done(&client[1]))
+			pr_info("\t+%6lldms: %3u %5u %4u;\n",
+				ktime_to_ms(ktime_sub(ktime_get(), start)),
+				pct[0], cps[0], qd[0]);
+		else
+			pr_info("\t+%6lldms: %3u %5u %4u; %3u %5u %4u\n",
+				ktime_to_ms(ktime_sub(ktime_get(), start)),
+				pct[0], cps[0], qd[0],
+				pct[1], cps[1], qd[1]);
+
+		msleep(period_ms);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(client); i++) {
+		kthread_flush_work(&client[i].work);
+		kthread_destroy_worker(client[i].worker);
+	}
+
+	for (i = 0; i < ARRAY_SIZE(client); i++)
+		KUNIT_ASSERT_TRUE(test, client[i].done);
+
+	for (i = 0; i < ARRAY_SIZE(client); i++) {
+		pr_info("    %u: prio=%s sync=%u elapsed_ms=%lldms (ideal_ms=%lldms) cycle_time(min,avg,max)=%u,%u,%u us latency_time(min,avg,max)=%u,%u,%u us",
+			i,
+			prio_str(params->client[i].priority),
+			params->client[i].sync,
+			ktime_to_ms(ktime_sub(client[i].end, client[i].start)),
+			ktime_to_ms(client[i].ideal_duration),
+			client[i].cycle_time.min_us,
+			get_stats_avg(&client[i].cycle_time, client[i].cycles),
+			client[i].cycle_time.max_us,
+			client[i].latency_time.min_us,
+			get_stats_avg(&client[i].latency_time, client[i].cycles),
+			client[i].latency_time.max_us);
+		drm_mock_sched_entity_free(client[i].entity);
+	}
+}
+
+static const struct kunit_attributes drm_sched_scheduler_two_clients_attr = {
+	.speed = KUNIT_SPEED_SLOW,
+};
+
+static struct kunit_case drm_sched_scheduler_two_clients_tests[] = {
+	KUNIT_CASE_PARAM_ATTR(drm_sched_scheduler_two_clients_test,
+			      drm_sched_scheduler_two_clients_gen_params,
+			      drm_sched_scheduler_two_clients_attr),
+	{}
+};
+
+static struct kunit_suite drm_sched_scheduler_two_clients1 = {
+	.name = "drm_sched_scheduler_two_clients_one_credit_tests",
+	.init = drm_sched_scheduler_init,
+	.exit = drm_sched_scheduler_exit,
+	.test_cases = drm_sched_scheduler_two_clients_tests,
+};
+
+static struct kunit_suite drm_sched_scheduler_two_clients2 = {
+	.name = "drm_sched_scheduler_two_clients_two_credits_tests",
+	.init = drm_sched_scheduler_init2,
+	.exit = drm_sched_scheduler_exit,
+	.test_cases = drm_sched_scheduler_two_clients_tests,
+};
+
+static const struct drm_sched_test_params drm_sched_many_cases[] = {
+	{
+		.description = "2 clients",
+		.num_clients = 2,
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 4,
+			.job_us = 1000,
+			.wait_us = 0,
+			.sync = true,
+		},
+	},
+	{
+		.description = "3 clients",
+		.num_clients = 3,
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 4,
+			.job_us = 1000,
+			.wait_us = 0,
+			.sync = true,
+		},
+	},
+	{
+		.description = "7 clients",
+		.num_clients = 7,
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 4,
+			.job_us = 1000,
+			.wait_us = 0,
+			.sync = true,
+		},
+	},
+	{
+		.description = "13 clients",
+		.num_clients = 13,
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 4,
+			.job_us = 1000,
+			.wait_us = 0,
+			.sync = true,
+		},
+	},
+	{
+		.description = "31 clients",
+		.num_clients = 31,
+		.client[0] = {
+			.priority = DRM_SCHED_PRIORITY_NORMAL,
+			.job_cnt = 2,
+			.job_us = 1000,
+			.wait_us = 0,
+			.sync = true,
+		},
+	},
+};
+
+KUNIT_ARRAY_PARAM(drm_sched_scheduler_many_clients,
+		  drm_sched_many_cases,
+		  drm_sched_desc);
+
+static void drm_sched_scheduler_many_clients_test(struct kunit *test)
+{
+	const struct drm_sched_test_params *params = test->param_value;
+	struct drm_mock_scheduler *sched = test->priv;
+	const unsigned int clients = params->num_clients;
+	unsigned int i, j, delta_total = 0, loops = 0;
+	struct test_client *client;
+	unsigned int *prev_cycle;
+	ktime_t start;
+	char *buf;
+
+	/*
+	 * Many clients with deep-ish async queues.
+	 */
+
+	buf = kunit_kmalloc(test, PAGE_SIZE, GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, buf);
+	client = kunit_kcalloc(test, clients, sizeof(*client), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, client);
+	prev_cycle = kunit_kcalloc(test, clients, sizeof(*prev_cycle),
+				   GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, prev_cycle);
+
+	for (i = 0; i < clients; i++)
+		client[i].entity =
+			drm_mock_sched_entity_new(test,
+						  DRM_SCHED_PRIORITY_NORMAL,
+						  sched);
+
+	for (i = 0; i < clients; i++) {
+		client[i].test = test;
+		client[i].id = i;
+		client[i].params = params->client[0];
+		client[i].duration = ms_to_ktime(1000 / clients);
+		client[i].cycle_time.min_us = ~0U;
+		client[i].latency_time.min_us = ~0U;
+		client[i].worker =
+			kthread_create_worker(0, "%s-%u", __func__, i);
+		if (IS_ERR(client[i].worker)) {
+			for (j = 0; j < i; j++)
+				kthread_destroy_worker(client[j].worker);
+			KUNIT_FAIL(test, "Failed to create worker!\n");
+		}
+
+		kthread_init_work(&client[i].work, drm_sched_client_work);
+	}
+
+	for (i = 0; i < clients; i++)
+		kthread_queue_work(client[i].worker, &client[i].work);
+
+	start = ktime_get();
+	pr_info("%u clients:\n\tt\t\tcycle:\t  min    avg    max : ...\n", clients);
+	for (;;) {
+		unsigned int min = ~0;
+		unsigned int max = 0;
+		unsigned int total = 0;
+		bool done = true;
+		char pbuf[16];
+
+		memset(buf, 0, PAGE_SIZE);
+		for (i = 0; i < clients; i++) {
+			unsigned int cycle, cycles;
+
+			/* Read current progress from the threaded worker. */
+			cycle = READ_ONCE(client[i].cycle);
+			cycles = READ_ONCE(client[i].cycles);
+
+			snprintf(pbuf, sizeof(pbuf), " %3d", cycle);
+			strncat(buf, pbuf, PAGE_SIZE);
+
+			total += cycle;
+			if (cycle < min)
+				min = cycle;
+			if (cycle > max)
+				max = cycle;
+
+			if (!min || (cycle + 1) < cycles)
+				done = false;
+		}
+
+		loops++;
+		delta_total += max - min;
+
+		pr_info("\t+%6lldms\t\t  %3u  %3u  %3u :%s\n",
+			ktime_to_ms(ktime_sub(ktime_get(), start)),
+			min, DIV_ROUND_UP(total, clients), max, buf);
+
+		if (done)
+			break;
+
+		msleep(100);
+	}
+
+	pr_info("    avg_max_min_delta(x100)=%u\n",
+		loops ? DIV_ROUND_UP(delta_total * 100, loops) : 0);
+
+	for (i = 0; i < clients; i++) {
+		kthread_flush_work(&client[i].work);
+		kthread_destroy_worker(client[i].worker);
+	}
+
+	for (i = 0; i < clients; i++)
+		drm_mock_sched_entity_free(client[i].entity);
+}
+
+static const struct kunit_attributes drm_sched_scheduler_many_clients_attr = {
+	.speed = KUNIT_SPEED_SLOW,
+};
+
+static struct kunit_case drm_sched_scheduler_many_clients_tests[] = {
+	KUNIT_CASE_PARAM_ATTR(drm_sched_scheduler_many_clients_test,
+			      drm_sched_scheduler_many_clients_gen_params,
+			      drm_sched_scheduler_many_clients_attr),
+	{}
+};
+
+static struct kunit_suite drm_sched_scheduler_many_clients = {
+	.name = "drm_sched_scheduler_many_clients_tests",
+	.init = drm_sched_scheduler_init2,
+	.exit = drm_sched_scheduler_exit,
+	.test_cases = drm_sched_scheduler_many_clients_tests,
+};
+
+kunit_test_suites(&drm_sched_scheduler_overhead,
+		  &drm_sched_scheduler_two_clients1,
+		  &drm_sched_scheduler_two_clients2,
+		  &drm_sched_scheduler_many_clients);
diff --git a/drivers/gpu/drm/sitronix/st7571.c b/drivers/gpu/drm/sitronix/st7571.c
index 5fd575d972a2e..20954c33eca93 100644
--- a/drivers/gpu/drm/sitronix/st7571.c
+++ b/drivers/gpu/drm/sitronix/st7571.c
@@ -324,7 +324,7 @@ static const u64 st7571_primary_plane_fmtmods[] = {
 };
 
 static int st7571_primary_plane_helper_atomic_check(struct drm_plane *plane,
-						    struct drm_atomic_state *state)
+						    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_crtc *new_crtc = new_plane_state->crtc;
@@ -340,7 +340,7 @@ static int st7571_primary_plane_helper_atomic_check(struct drm_plane *plane,
 }
 
 static void st7571_primary_plane_helper_atomic_update(struct drm_plane *plane,
-						      struct drm_atomic_state *state)
+						      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
@@ -379,7 +379,7 @@ static void st7571_primary_plane_helper_atomic_update(struct drm_plane *plane,
 }
 
 static void st7571_primary_plane_helper_atomic_disable(struct drm_plane *plane,
-						       struct drm_atomic_state *state)
+						       struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = plane->dev;
 	struct st7571_device *st7571 = drm_to_st7571(plane->dev);
@@ -437,7 +437,7 @@ static const struct drm_crtc_funcs st7571_crtc_funcs = {
  */
 
 static void st7571_encoder_atomic_enable(struct drm_encoder *encoder,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = encoder->dev;
 	struct st7571_device *st7571 = drm_to_st7571(drm);
@@ -452,7 +452,7 @@ static void st7571_encoder_atomic_enable(struct drm_encoder *encoder,
 }
 
 static void st7571_encoder_atomic_disable(struct drm_encoder *encoder,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = encoder->dev;
 	struct st7571_device *st7571 = drm_to_st7571(drm);
diff --git a/drivers/gpu/drm/sitronix/st7586.c b/drivers/gpu/drm/sitronix/st7586.c
index fa58b3d83564f..28b2245f6b79e 100644
--- a/drivers/gpu/drm/sitronix/st7586.c
+++ b/drivers/gpu/drm/sitronix/st7586.c
@@ -171,7 +171,7 @@ static const u64 st7586_plane_format_modifiers[] = {
 };
 
 static void st7586_plane_helper_atomic_update(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = plane->state;
 	struct drm_shadow_plane_state *shadow_plane_state = to_drm_shadow_plane_state(plane_state);
@@ -205,7 +205,7 @@ static const struct drm_plane_funcs st7586_plane_funcs = {
 };
 
 static void st7586_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct st7586_device *st7586 = to_st7586_device(drm);
@@ -276,7 +276,7 @@ static void st7586_crtc_helper_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void st7586_crtc_helper_atomic_disable(struct drm_crtc *crtc,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct st7586_device *st7586 = to_st7586_device(drm);
diff --git a/drivers/gpu/drm/sitronix/st7735r.c b/drivers/gpu/drm/sitronix/st7735r.c
index 5a75716ed5fff..bd763871dca0c 100644
--- a/drivers/gpu/drm/sitronix/st7735r.c
+++ b/drivers/gpu/drm/sitronix/st7735r.c
@@ -85,7 +85,7 @@ static const struct drm_plane_funcs st7735r_plane_funcs = {
 };
 
 static void st7735r_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct st7735r_device *st7735r = to_st7735r_device(drm);
diff --git a/drivers/gpu/drm/sitronix/st7920.c b/drivers/gpu/drm/sitronix/st7920.c
index 18fe7a28ed073..d320391801f3f 100644
--- a/drivers/gpu/drm/sitronix/st7920.c
+++ b/drivers/gpu/drm/sitronix/st7920.c
@@ -338,7 +338,7 @@ static int st7920_fb_blit_rect(struct drm_framebuffer *fb,
 }
 
 static int st7920_primary_plane_atomic_check(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = plane->dev;
 	struct st7920_device *st7920 = drm_to_st7920(drm);
@@ -387,7 +387,7 @@ static int st7920_primary_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void st7920_primary_plane_atomic_update(struct drm_plane *plane,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
@@ -429,7 +429,7 @@ static void st7920_primary_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void st7920_primary_plane_atomic_disable(struct drm_plane *plane,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = plane->dev;
 	struct st7920_device *st7920 = drm_to_st7920(drm);
@@ -522,7 +522,7 @@ static enum drm_mode_status st7920_crtc_mode_valid(struct drm_crtc *crtc,
 }
 
 static int st7920_crtc_atomic_check(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	struct st7920_crtc_state *st7920_state = to_st7920_crtc_state(crtc_state);
@@ -540,7 +540,7 @@ static int st7920_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void st7920_crtc_atomic_enable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct st7920_device *st7920 = drm_to_st7920(drm);
@@ -560,7 +560,7 @@ static void st7920_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void st7920_crtc_atomic_disable(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct spi7920_error err = {0};
 	struct drm_device *drm = crtc->dev;
diff --git a/drivers/gpu/drm/solomon/ssd130x.c b/drivers/gpu/drm/solomon/ssd130x.c
index c77455b1834dd..cae92a3ae8a41 100644
--- a/drivers/gpu/drm/solomon/ssd130x.c
+++ b/drivers/gpu/drm/solomon/ssd130x.c
@@ -1074,7 +1074,7 @@ static int ssd133x_fb_blit_rect(struct drm_framebuffer *fb,
 }
 
 static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = plane->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1123,7 +1123,7 @@ static int ssd130x_primary_plane_atomic_check(struct drm_plane *plane,
 }
 
 static int ssd132x_primary_plane_atomic_check(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = plane->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1172,7 +1172,7 @@ static int ssd132x_primary_plane_atomic_check(struct drm_plane *plane,
 }
 
 static int ssd133x_primary_plane_atomic_check(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_crtc *crtc = plane_state->crtc;
@@ -1195,7 +1195,7 @@ static int ssd133x_primary_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void ssd130x_primary_plane_atomic_update(struct drm_plane *plane,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
@@ -1236,7 +1236,7 @@ static void ssd130x_primary_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void ssd132x_primary_plane_atomic_update(struct drm_plane *plane,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
@@ -1277,7 +1277,7 @@ static void ssd132x_primary_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void ssd133x_primary_plane_atomic_update(struct drm_plane *plane,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
@@ -1316,7 +1316,7 @@ static void ssd133x_primary_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void ssd130x_primary_plane_atomic_disable(struct drm_plane *plane,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = plane->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1340,7 +1340,7 @@ static void ssd130x_primary_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static void ssd132x_primary_plane_atomic_disable(struct drm_plane *plane,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = plane->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1364,7 +1364,7 @@ static void ssd132x_primary_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static void ssd133x_primary_plane_atomic_disable(struct drm_plane *plane,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = plane->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1476,7 +1476,7 @@ static enum drm_mode_status ssd130x_crtc_mode_valid(struct drm_crtc *crtc,
 }
 
 static int ssd130x_crtc_atomic_check(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1497,7 +1497,7 @@ static int ssd130x_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static int ssd132x_crtc_atomic_check(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1518,7 +1518,7 @@ static int ssd132x_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static int ssd133x_crtc_atomic_check(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1621,7 +1621,7 @@ static const struct drm_crtc_funcs ssd130x_crtc_funcs = {
 };
 
 static void ssd130x_encoder_atomic_enable(struct drm_encoder *encoder,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = encoder->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1647,7 +1647,7 @@ static void ssd130x_encoder_atomic_enable(struct drm_encoder *encoder,
 }
 
 static void ssd132x_encoder_atomic_enable(struct drm_encoder *encoder,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = encoder->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1672,7 +1672,7 @@ static void ssd132x_encoder_atomic_enable(struct drm_encoder *encoder,
 }
 
 static void ssd133x_encoder_atomic_enable(struct drm_encoder *encoder,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = encoder->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
@@ -1697,7 +1697,7 @@ static void ssd133x_encoder_atomic_enable(struct drm_encoder *encoder,
 }
 
 static void ssd130x_encoder_atomic_disable(struct drm_encoder *encoder,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = encoder->dev;
 	struct ssd130x_device *ssd130x = drm_to_ssd130x(drm);
diff --git a/drivers/gpu/drm/sprd/sprd_dpu.c b/drivers/gpu/drm/sprd/sprd_dpu.c
index a3447622a33cd..66990398b9037 100644
--- a/drivers/gpu/drm/sprd/sprd_dpu.c
+++ b/drivers/gpu/drm/sprd/sprd_dpu.c
@@ -501,7 +501,7 @@ void sprd_dpu_stop(struct sprd_dpu *dpu)
 }
 
 static int sprd_plane_atomic_check(struct drm_plane *plane,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state,
 									     plane);
@@ -526,7 +526,7 @@ static int sprd_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void sprd_plane_atomic_update(struct drm_plane *drm_plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   drm_plane);
@@ -537,7 +537,7 @@ static void sprd_plane_atomic_update(struct drm_plane *drm_plane,
 }
 
 static void sprd_plane_atomic_disable(struct drm_plane *drm_plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   drm_plane);
@@ -636,7 +636,7 @@ static void sprd_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static void sprd_crtc_atomic_enable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct sprd_dpu *dpu = to_sprd_crtc(crtc);
 
@@ -646,7 +646,7 @@ static void sprd_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void sprd_crtc_atomic_disable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct sprd_dpu *dpu = to_sprd_crtc(crtc);
 	struct drm_device *drm = dpu->base.dev;
@@ -664,7 +664,7 @@ static void sprd_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void sprd_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 
 {
 	struct sprd_dpu *dpu = to_sprd_crtc(crtc);
diff --git a/drivers/gpu/drm/sti/sti_crtc.c b/drivers/gpu/drm/sti/sti_crtc.c
index 3c7154f2d5f37..f0e98653d35a2 100644
--- a/drivers/gpu/drm/sti/sti_crtc.c
+++ b/drivers/gpu/drm/sti/sti_crtc.c
@@ -22,7 +22,7 @@
 #include "sti_vtg.h"
 
 static void sti_crtc_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct sti_mixer *mixer = to_sti_mixer(crtc);
 
@@ -34,7 +34,7 @@ static void sti_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void sti_crtc_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct sti_mixer *mixer = to_sti_mixer(crtc);
 
@@ -132,7 +132,7 @@ sti_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static void sti_crtc_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_device *drm_dev = crtc->dev;
 	struct sti_mixer *mixer = to_sti_mixer(crtc);
diff --git a/drivers/gpu/drm/sti/sti_cursor.c b/drivers/gpu/drm/sti/sti_cursor.c
index 4e12a465be7fe..d0b89b28e50cc 100644
--- a/drivers/gpu/drm/sti/sti_cursor.c
+++ b/drivers/gpu/drm/sti/sti_cursor.c
@@ -183,7 +183,7 @@ static void sti_cursor_init(struct sti_cursor *cursor)
 }
 
 static int sti_cursor_atomic_check(struct drm_plane *drm_plane,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 drm_plane);
@@ -261,7 +261,7 @@ static int sti_cursor_atomic_check(struct drm_plane *drm_plane,
 }
 
 static void sti_cursor_atomic_update(struct drm_plane *drm_plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *newstate = drm_atomic_get_new_plane_state(state,
 									  drm_plane);
@@ -316,7 +316,7 @@ static void sti_cursor_atomic_update(struct drm_plane *drm_plane,
 }
 
 static void sti_cursor_atomic_disable(struct drm_plane *drm_plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *oldstate = drm_atomic_get_old_plane_state(state,
 									  drm_plane);
diff --git a/drivers/gpu/drm/sti/sti_gdp.c b/drivers/gpu/drm/sti/sti_gdp.c
index 1e5aa8c306457..9a17b590aa3d8 100644
--- a/drivers/gpu/drm/sti/sti_gdp.c
+++ b/drivers/gpu/drm/sti/sti_gdp.c
@@ -618,7 +618,7 @@ static int sti_gdp_get_dst(struct device *dev, int dst, int src)
 }
 
 static int sti_gdp_atomic_check(struct drm_plane *drm_plane,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 drm_plane);
@@ -705,7 +705,7 @@ static int sti_gdp_atomic_check(struct drm_plane *drm_plane,
 }
 
 static void sti_gdp_atomic_update(struct drm_plane *drm_plane,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *oldstate = drm_atomic_get_old_plane_state(state,
 									  drm_plane);
@@ -873,7 +873,7 @@ static void sti_gdp_atomic_update(struct drm_plane *drm_plane,
 }
 
 static void sti_gdp_atomic_disable(struct drm_plane *drm_plane,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *oldstate = drm_atomic_get_old_plane_state(state,
 									  drm_plane);
diff --git a/drivers/gpu/drm/sti/sti_hqvdp.c b/drivers/gpu/drm/sti/sti_hqvdp.c
index 57ef4ba3554eb..cf1ed8a33b8e1 100644
--- a/drivers/gpu/drm/sti/sti_hqvdp.c
+++ b/drivers/gpu/drm/sti/sti_hqvdp.c
@@ -1020,7 +1020,7 @@ static void sti_hqvdp_start_xp70(struct sti_hqvdp *hqvdp)
 }
 
 static int sti_hqvdp_atomic_check(struct drm_plane *drm_plane,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 drm_plane);
@@ -1115,7 +1115,7 @@ static int sti_hqvdp_atomic_check(struct drm_plane *drm_plane,
 }
 
 static void sti_hqvdp_atomic_update(struct drm_plane *drm_plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *oldstate = drm_atomic_get_old_plane_state(state,
 									  drm_plane);
@@ -1249,7 +1249,7 @@ static void sti_hqvdp_atomic_update(struct drm_plane *drm_plane,
 }
 
 static void sti_hqvdp_atomic_disable(struct drm_plane *drm_plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *oldstate = drm_atomic_get_old_plane_state(state,
 									  drm_plane);
diff --git a/drivers/gpu/drm/stm/drv.c b/drivers/gpu/drm/stm/drv.c
index 56d53ac3082d9..144b7cda989ac 100644
--- a/drivers/gpu/drm/stm/drv.c
+++ b/drivers/gpu/drm/stm/drv.c
@@ -126,7 +126,7 @@ static __maybe_unused int drv_suspend(struct device *dev)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct ltdc_device *ldev = ddev->dev_private;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 
 	WARN_ON(ldev->suspend_state);
 
diff --git a/drivers/gpu/drm/stm/ltdc.c b/drivers/gpu/drm/stm/ltdc.c
index f7e847cfa38f8..95fcfa48d8be3 100644
--- a/drivers/gpu/drm/stm/ltdc.c
+++ b/drivers/gpu/drm/stm/ltdc.c
@@ -773,7 +773,7 @@ static void ltdc_crtc_update_clut(struct drm_crtc *crtc)
 }
 
 static void ltdc_crtc_atomic_enable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct ltdc_device *ldev = crtc_to_ltdc(crtc);
 	struct drm_device *ddev = crtc->dev;
@@ -796,7 +796,7 @@ static void ltdc_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void ltdc_crtc_atomic_disable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct ltdc_device *ldev = crtc_to_ltdc(crtc);
 	struct drm_device *ddev = crtc->dev;
@@ -1037,7 +1037,7 @@ static void ltdc_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static void ltdc_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct ltdc_device *ldev = crtc_to_ltdc(crtc);
 	struct drm_device *ddev = crtc->dev;
@@ -1236,7 +1236,7 @@ static const struct drm_crtc_funcs ltdc_crtc_with_crc_support_funcs = {
  */
 
 static int ltdc_plane_atomic_check(struct drm_plane *plane,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -1263,7 +1263,7 @@ static int ltdc_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void ltdc_plane_atomic_update(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct ltdc_device *ldev = plane_to_ltdc(plane);
 	struct drm_plane_state *newstate = drm_atomic_get_new_plane_state(state,
@@ -1509,7 +1509,7 @@ static void ltdc_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void ltdc_plane_atomic_disable(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *oldstate = drm_atomic_get_old_plane_state(state,
 									  plane);
diff --git a/drivers/gpu/drm/stm/ltdc.h b/drivers/gpu/drm/stm/ltdc.h
index 17b51a7ce28ee..452352e645d71 100644
--- a/drivers/gpu/drm/stm/ltdc.h
+++ b/drivers/gpu/drm/stm/ltdc.h
@@ -58,7 +58,7 @@ struct ltdc_device {
 	u32 fifo_threshold;	/* fifo underrun threshold */
 	u32 transfer_err;	/* transfer error counter */
 	struct fps_info plane_fpsi[LTDC_MAX_LAYER];
-	struct drm_atomic_state *suspend_state;
+	struct drm_atomic_commit *suspend_state;
 	int crc_skip_count;
 	bool crc_active;
 };
diff --git a/drivers/gpu/drm/stm/lvds.c b/drivers/gpu/drm/stm/lvds.c
index 25e2ba98f36ae..50a878688e477 100644
--- a/drivers/gpu/drm/stm/lvds.c
+++ b/drivers/gpu/drm/stm/lvds.c
@@ -887,7 +887,7 @@ static int lvds_connector_get_modes(struct drm_connector *connector)
 }
 
 static int lvds_connector_atomic_check(struct drm_connector *connector,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	const struct drm_display_mode *panel_mode;
 	struct drm_connector_state *conn_state;
@@ -981,7 +981,7 @@ static int lvds_attach(struct drm_bridge *bridge, struct drm_encoder *encoder,
 }
 
 static void lvds_atomic_enable(struct drm_bridge *bridge,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct stm_lvds *lvds = bridge_to_stm_lvds(bridge);
 	struct drm_connector_state *conn_state;
@@ -1017,7 +1017,7 @@ static void lvds_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void lvds_atomic_disable(struct drm_bridge *bridge,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct stm_lvds *lvds = bridge_to_stm_lvds(bridge);
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_backend.c b/drivers/gpu/drm/sun4i/sun4i_backend.c
index bc35dad53b071..03e46a2af6463 100644
--- a/drivers/gpu/drm/sun4i/sun4i_backend.c
+++ b/drivers/gpu/drm/sun4i/sun4i_backend.c
@@ -72,7 +72,7 @@ static void sun4i_backend_disable_color_correction(struct sunxi_engine *engine)
 
 static void sun4i_backend_commit(struct sunxi_engine *engine,
 				 struct drm_crtc *crtc,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	DRM_DEBUG_DRIVER("Committing changes\n");
 
@@ -472,7 +472,7 @@ static int sun4i_backend_atomic_check(struct sunxi_engine *engine,
 {
 	struct drm_plane_state *plane_states[SUN4I_BACKEND_NUM_LAYERS] = { 0 };
 	struct sun4i_backend *backend = engine_to_sun4i_backend(engine);
-	struct drm_atomic_state *state = crtc_state->state;
+	struct drm_atomic_commit *state = crtc_state->state;
 	struct drm_device *drm = state->dev;
 	struct drm_plane *plane;
 	unsigned int num_planes = 0;
diff --git a/drivers/gpu/drm/sun4i/sun4i_crtc.c b/drivers/gpu/drm/sun4i/sun4i_crtc.c
index 18e74047b0f56..c2f7f69baf75a 100644
--- a/drivers/gpu/drm/sun4i/sun4i_crtc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_crtc.c
@@ -46,7 +46,7 @@ static struct drm_encoder *sun4i_crtc_get_encoder(struct drm_crtc *crtc)
 }
 
 static int sun4i_crtc_atomic_check(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -61,7 +61,7 @@ static int sun4i_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void sun4i_crtc_atomic_begin(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 									 crtc);
@@ -84,7 +84,7 @@ static void sun4i_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void sun4i_crtc_atomic_flush(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct sun4i_crtc *scrtc = drm_crtc_to_sun4i_crtc(crtc);
 	struct drm_pending_vblank_event *event = crtc->state->event;
@@ -106,7 +106,7 @@ static void sun4i_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static void sun4i_crtc_atomic_disable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_encoder *encoder = sun4i_crtc_get_encoder(crtc);
 	struct sun4i_crtc *scrtc = drm_crtc_to_sun4i_crtc(crtc);
@@ -127,7 +127,7 @@ static void sun4i_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void sun4i_crtc_atomic_enable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_encoder *encoder = sun4i_crtc_get_encoder(crtc);
 	struct sun4i_crtc *scrtc = drm_crtc_to_sun4i_crtc(crtc);
diff --git a/drivers/gpu/drm/sun4i/sun4i_framebuffer.c b/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
index 260136d60ceba..9c8ef927d9297 100644
--- a/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
@@ -15,7 +15,7 @@
 #include "sun4i_framebuffer.h"
 
 static int sun4i_de_atomic_check(struct drm_device *dev,
-				 struct drm_atomic_state *state)
+				 struct drm_atomic_commit *state)
 {
 	int ret;
 
diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
index dd2a78defdb4b..07e2afcb4f95c 100644
--- a/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_enc.c
@@ -76,7 +76,7 @@ static int sun4i_hdmi_write_hdmi_infoframe(struct drm_connector *connector,
 }
 
 static void sun4i_hdmi_disable(struct drm_encoder *encoder,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct sun4i_hdmi *hdmi = drm_encoder_to_sun4i_hdmi(encoder);
 	u32 val;
@@ -91,7 +91,7 @@ static void sun4i_hdmi_disable(struct drm_encoder *encoder,
 }
 
 static void sun4i_hdmi_enable(struct drm_encoder *encoder,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	struct drm_display_mode *mode = &encoder->crtc->state->adjusted_mode;
 	struct sun4i_hdmi *hdmi = drm_encoder_to_sun4i_hdmi(encoder);
diff --git a/drivers/gpu/drm/sun4i/sun4i_layer.c b/drivers/gpu/drm/sun4i/sun4i_layer.c
index 63cb8c7c36363..d65587d96d9ee 100644
--- a/drivers/gpu/drm/sun4i/sun4i_layer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_layer.c
@@ -61,7 +61,7 @@ static void sun4i_backend_layer_destroy_state(struct drm_plane *plane,
 }
 
 static void sun4i_backend_layer_atomic_disable(struct drm_plane *plane,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -81,7 +81,7 @@ static void sun4i_backend_layer_atomic_disable(struct drm_plane *plane,
 }
 
 static void sun4i_backend_layer_atomic_update(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/sun4i/sun4i_tv.c b/drivers/gpu/drm/sun4i/sun4i_tv.c
index cce4e38789b98..814b77f278f69 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tv.c
@@ -261,7 +261,7 @@ sun4i_tv_find_tv_by_mode(unsigned int mode)
 }
 
 static void sun4i_tv_disable(struct drm_encoder *encoder,
-			    struct drm_atomic_state *state)
+			    struct drm_atomic_commit *state)
 {
 	struct sun4i_tv *tv = drm_encoder_to_sun4i_tv(encoder);
 	struct sun4i_crtc *crtc = drm_crtc_to_sun4i_crtc(encoder->crtc);
@@ -276,7 +276,7 @@ static void sun4i_tv_disable(struct drm_encoder *encoder,
 }
 
 static void sun4i_tv_enable(struct drm_encoder *encoder,
-			    struct drm_atomic_state *state)
+			    struct drm_atomic_commit *state)
 {
 	struct sun4i_tv *tv = drm_encoder_to_sun4i_tv(encoder);
 	struct sun4i_crtc *crtc = drm_crtc_to_sun4i_crtc(encoder->crtc);
diff --git a/drivers/gpu/drm/sun4i/sun8i_mixer.c b/drivers/gpu/drm/sun4i/sun8i_mixer.c
index 4071ab38b4ae5..4e42262950104 100644
--- a/drivers/gpu/drm/sun4i/sun8i_mixer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_mixer.c
@@ -253,7 +253,7 @@ int sun8i_mixer_drm_format_to_hw(u32 format, u32 *hw_format)
 
 static void sun8i_mixer_commit(struct sunxi_engine *engine,
 			       struct drm_crtc *crtc,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct sun8i_mixer *mixer = engine_to_sun8i_mixer(engine);
 	u32 bld_base = sun8i_blender_base(mixer);
diff --git a/drivers/gpu/drm/sun4i/sun8i_ui_layer.c b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
index 72c92203ae633..00756ab78b4eb 100644
--- a/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_ui_layer.c
@@ -146,7 +146,7 @@ static void sun8i_ui_layer_update_buffer(struct sun8i_layer *layer,
 }
 
 static int sun8i_ui_layer_atomic_check(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -187,7 +187,7 @@ static int sun8i_ui_layer_atomic_check(struct drm_plane *plane,
 
 
 static void sun8i_ui_layer_atomic_update(struct drm_plane *plane,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
index cd8d6c2da0c72..09f668c8af247 100644
--- a/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
+++ b/drivers/gpu/drm/sun4i/sun8i_vi_layer.c
@@ -226,7 +226,7 @@ static void sun8i_vi_layer_update_buffer(struct sun8i_layer *layer,
 }
 
 static int sun8i_vi_layer_atomic_check(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -266,7 +266,7 @@ static int sun8i_vi_layer_atomic_check(struct drm_plane *plane,
 }
 
 static void sun8i_vi_layer_atomic_update(struct drm_plane *plane,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/sun4i/sunxi_engine.h b/drivers/gpu/drm/sun4i/sunxi_engine.h
index c9461de06cd04..1711c274a41b0 100644
--- a/drivers/gpu/drm/sun4i/sunxi_engine.h
+++ b/drivers/gpu/drm/sun4i/sunxi_engine.h
@@ -62,7 +62,7 @@ struct sunxi_engine_ops {
 	 */
 	void (*commit)(struct sunxi_engine *engine,
 		       struct drm_crtc *crtc,
-		       struct drm_atomic_state *state);
+		       struct drm_atomic_commit *state);
 
 	/**
 	 * @layers_init:
@@ -153,7 +153,7 @@ struct sunxi_engine {
 static inline void
 sunxi_engine_commit(struct sunxi_engine *engine,
 		    struct drm_crtc *crtc,
-		    struct drm_atomic_state *state)
+		    struct drm_atomic_commit *state)
 {
 	if (engine->ops && engine->ops->commit)
 		engine->ops->commit(engine, crtc, state);
diff --git a/drivers/gpu/drm/sysfb/corebootdrm.c b/drivers/gpu/drm/sysfb/corebootdrm.c
index 5dc6f3c76f7bd..853da9a609dfb 100644
--- a/drivers/gpu/drm/sysfb/corebootdrm.c
+++ b/drivers/gpu/drm/sysfb/corebootdrm.c
@@ -4,6 +4,7 @@
 #include <linux/coreboot.h>
 #include <linux/minmax.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <drm/clients/drm_client_setup.h>
 #include <drm/drm_atomic.h>
@@ -18,6 +19,7 @@
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
+#include <drm/drm_modeset_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -286,6 +288,24 @@ static struct drm_driver corebootdrm_drm_driver = {
  * Coreboot driver
  */
 
+static int corebootdrm_pm_suspend(struct device *dev)
+{
+	struct corebootdrm_device *cdev = dev_get_drvdata(dev);
+	struct drm_device *drm = &cdev->sysfb.dev;
+
+	return drm_mode_config_helper_suspend(drm);
+}
+
+static int corebootdrm_pm_resume(struct device *dev)
+{
+	struct corebootdrm_device *cdev = dev_get_drvdata(dev);
+	struct drm_device *drm = &cdev->sysfb.dev;
+
+	return drm_mode_config_helper_resume(drm);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(corebootdrm_pm_ops, corebootdrm_pm_suspend, corebootdrm_pm_resume);
+
 static int corebootdrm_probe(struct platform_device *pdev)
 {
 	const struct lb_framebuffer *fb = dev_get_platdata(&pdev->dev);
@@ -423,6 +443,7 @@ static void corebootdrm_remove(struct platform_device *pdev)
 static struct platform_driver corebootdrm_platform_driver = {
 	.driver = {
 		.name = "coreboot-framebuffer",
+		.pm = pm_sleep_ptr(&corebootdrm_pm_ops),
 	},
 	.probe = corebootdrm_probe,
 	.remove = corebootdrm_remove,
diff --git a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h
index b14df5b54bc9a..2a2b553366fb7 100644
--- a/drivers/gpu/drm/sysfb/drm_sysfb_helper.h
+++ b/drivers/gpu/drm/sysfb/drm_sysfb_helper.h
@@ -111,11 +111,11 @@ size_t drm_sysfb_build_fourcc_list(struct drm_device *dev,
 int drm_sysfb_plane_helper_begin_fb_access(struct drm_plane *plane,
 					   struct drm_plane_state *plane_state);
 int drm_sysfb_plane_helper_atomic_check(struct drm_plane *plane,
-					struct drm_atomic_state *new_state);
+					struct drm_atomic_commit *new_state);
 void drm_sysfb_plane_helper_atomic_update(struct drm_plane *plane,
-					  struct drm_atomic_state *state);
+					  struct drm_atomic_commit *state);
 void drm_sysfb_plane_helper_atomic_disable(struct drm_plane *plane,
-					   struct drm_atomic_state *state);
+					   struct drm_atomic_commit *state);
 int drm_sysfb_plane_helper_get_scanout_buffer(struct drm_plane *plane,
 					      struct drm_scanout_buffer *sb);
 
@@ -165,7 +165,7 @@ to_drm_sysfb_crtc_state(struct drm_crtc_state *base)
 
 enum drm_mode_status drm_sysfb_crtc_helper_mode_valid(struct drm_crtc *crtc,
 						      const struct drm_display_mode *mode);
-int drm_sysfb_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *new_state);
+int drm_sysfb_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *new_state);
 
 #define DRM_SYSFB_CRTC_HELPER_FUNCS \
 	.mode_valid = drm_sysfb_crtc_helper_mode_valid, \
diff --git a/drivers/gpu/drm/sysfb/drm_sysfb_modeset.c b/drivers/gpu/drm/sysfb/drm_sysfb_modeset.c
index 808b1eda871a1..d2de29caf89eb 100644
--- a/drivers/gpu/drm/sysfb/drm_sysfb_modeset.c
+++ b/drivers/gpu/drm/sysfb/drm_sysfb_modeset.c
@@ -283,7 +283,7 @@ int drm_sysfb_plane_helper_begin_fb_access(struct drm_plane *plane,
 EXPORT_SYMBOL(drm_sysfb_plane_helper_begin_fb_access);
 
 int drm_sysfb_plane_helper_atomic_check(struct drm_plane *plane,
-					struct drm_atomic_state *new_state)
+					struct drm_atomic_commit *new_state)
 {
 	struct drm_sysfb_device *sysfb = to_drm_sysfb_device(plane->dev);
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(new_state, plane);
@@ -326,7 +326,7 @@ int drm_sysfb_plane_helper_atomic_check(struct drm_plane *plane,
 }
 EXPORT_SYMBOL(drm_sysfb_plane_helper_atomic_check);
 
-void drm_sysfb_plane_helper_atomic_update(struct drm_plane *plane, struct drm_atomic_state *state)
+void drm_sysfb_plane_helper_atomic_update(struct drm_plane *plane, struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = plane->dev;
 	struct drm_sysfb_device *sysfb = to_drm_sysfb_device(dev);
@@ -371,7 +371,7 @@ void drm_sysfb_plane_helper_atomic_update(struct drm_plane *plane, struct drm_at
 EXPORT_SYMBOL(drm_sysfb_plane_helper_atomic_update);
 
 void drm_sysfb_plane_helper_atomic_disable(struct drm_plane *plane,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = plane->dev;
 	struct drm_sysfb_device *sysfb = to_drm_sysfb_device(dev);
@@ -486,7 +486,7 @@ enum drm_mode_status drm_sysfb_crtc_helper_mode_valid(struct drm_crtc *crtc,
 }
 EXPORT_SYMBOL(drm_sysfb_crtc_helper_mode_valid);
 
-int drm_sysfb_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *new_state)
+int drm_sysfb_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *new_state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_sysfb_device *sysfb = to_drm_sysfb_device(dev);
diff --git a/drivers/gpu/drm/sysfb/efidrm.c b/drivers/gpu/drm/sysfb/efidrm.c
index a335c94a7bd75..1a1e367009760 100644
--- a/drivers/gpu/drm/sysfb/efidrm.c
+++ b/drivers/gpu/drm/sysfb/efidrm.c
@@ -5,6 +5,7 @@
 #include <linux/limits.h>
 #include <linux/platform_device.h>
 #include <linux/sysfb.h>
+#include <linux/pm.h>
 
 #include <drm/clients/drm_client_setup.h>
 #include <drm/drm_atomic.h>
@@ -20,6 +21,7 @@
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
+#include <drm/drm_modeset_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -371,6 +373,22 @@ static struct drm_driver efidrm_driver = {
  * Platform driver
  */
 
+static int efidrm_pm_suspend(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_suspend(drm);
+}
+
+static int efidrm_pm_resume(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_resume(drm);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(efidrm_pm_ops, efidrm_pm_suspend, efidrm_pm_resume);
+
 static int efidrm_probe(struct platform_device *pdev)
 {
 	struct efidrm_device *efi;
@@ -403,6 +421,7 @@ static void efidrm_remove(struct platform_device *pdev)
 static struct platform_driver efidrm_platform_driver = {
 	.driver = {
 		.name = "efi-framebuffer",
+		.pm = pm_sleep_ptr(&efidrm_pm_ops),
 	},
 	.probe = efidrm_probe,
 	.remove = efidrm_remove,
diff --git a/drivers/gpu/drm/sysfb/ofdrm.c b/drivers/gpu/drm/sysfb/ofdrm.c
index 247cf13c80a05..819aed466727e 100644
--- a/drivers/gpu/drm/sysfb/ofdrm.c
+++ b/drivers/gpu/drm/sysfb/ofdrm.c
@@ -4,6 +4,7 @@
 #include <linux/of_address.h>
 #include <linux/pci.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 
 #include <drm/clients/drm_client_setup.h>
 #include <drm/drm_atomic.h>
@@ -20,6 +21,7 @@
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
+#include <drm/drm_modeset_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -725,7 +727,7 @@ static const struct drm_plane_funcs ofdrm_primary_plane_funcs = {
 	.destroy = drm_plane_cleanup,
 };
 
-static void ofdrm_crtc_helper_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *state)
+static void ofdrm_crtc_helper_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct ofdrm_device *odev = ofdrm_device_of_dev(crtc->dev);
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
@@ -1097,6 +1099,22 @@ static struct drm_driver ofdrm_driver = {
  * Platform driver
  */
 
+static int ofdrm_pm_suspend(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_suspend(drm);
+}
+
+static int ofdrm_pm_resume(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_resume(drm);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(ofdrm_pm_ops, ofdrm_pm_suspend, ofdrm_pm_resume);
+
 static int ofdrm_probe(struct platform_device *pdev)
 {
 	struct ofdrm_device *odev;
@@ -1136,6 +1154,7 @@ static struct platform_driver ofdrm_platform_driver = {
 	.driver = {
 		.name = "of-display",
 		.of_match_table = ofdrm_of_match_display,
+		.pm = pm_sleep_ptr(&ofdrm_pm_ops),
 	},
 	.probe = ofdrm_probe,
 	.remove = ofdrm_remove,
diff --git a/drivers/gpu/drm/sysfb/simpledrm.c b/drivers/gpu/drm/sysfb/simpledrm.c
index 7a95d2dacd9d2..fc168920f2c64 100644
--- a/drivers/gpu/drm/sysfb/simpledrm.c
+++ b/drivers/gpu/drm/sysfb/simpledrm.c
@@ -8,6 +8,7 @@
 #include <linux/of_reserved_mem.h>
 #include <linux/platform_data/simplefb.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 #include <linux/pm_domain.h>
 #include <linux/regulator/consumer.h>
 
@@ -24,6 +25,7 @@
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
+#include <drm/drm_modeset_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -834,6 +836,24 @@ static struct drm_driver simpledrm_driver = {
  * Platform driver
  */
 
+static int simpledrm_pm_suspend(struct device *dev)
+{
+	struct simpledrm_device *sdev = dev_get_drvdata(dev);
+	struct drm_device *drm = &sdev->sysfb.dev;
+
+	return drm_mode_config_helper_suspend(drm);
+}
+
+static int simpledrm_pm_resume(struct device *dev)
+{
+	struct simpledrm_device *sdev = dev_get_drvdata(dev);
+	struct drm_device *drm = &sdev->sysfb.dev;
+
+	return drm_mode_config_helper_resume(drm);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(simpledrm_pm_ops, simpledrm_pm_suspend, simpledrm_pm_resume);
+
 static int simpledrm_probe(struct platform_device *pdev)
 {
 	struct simpledrm_device *sdev;
@@ -874,6 +894,7 @@ static struct platform_driver simpledrm_platform_driver = {
 	.driver = {
 		.name = "simple-framebuffer", /* connect to sysfb */
 		.of_match_table = simpledrm_of_match_table,
+		.pm = pm_sleep_ptr(&simpledrm_pm_ops),
 	},
 	.probe = simpledrm_probe,
 	.remove = simpledrm_remove,
diff --git a/drivers/gpu/drm/sysfb/vesadrm.c b/drivers/gpu/drm/sysfb/vesadrm.c
index 4e00113e5c770..dbc317778d54e 100644
--- a/drivers/gpu/drm/sysfb/vesadrm.c
+++ b/drivers/gpu/drm/sysfb/vesadrm.c
@@ -5,6 +5,7 @@
 #include <linux/limits.h>
 #include <linux/platform_device.h>
 #include <linux/sysfb.h>
+#include <linux/pm.h>
 
 #include <drm/clients/drm_client_setup.h>
 #include <drm/drm_atomic.h>
@@ -21,6 +22,7 @@
 #include <drm/drm_gem_framebuffer_helper.h>
 #include <drm/drm_gem_shmem_helper.h>
 #include <drm/drm_managed.h>
+#include <drm/drm_modeset_helper.h>
 #include <drm/drm_modeset_helper_vtables.h>
 #include <drm/drm_print.h>
 #include <drm/drm_probe_helper.h>
@@ -246,7 +248,7 @@ static const u64 vesadrm_primary_plane_format_modifiers[] = {
 };
 
 static int vesadrm_primary_plane_helper_atomic_check(struct drm_plane *plane,
-						     struct drm_atomic_state *new_state)
+						     struct drm_atomic_commit *new_state)
 {
 	struct drm_sysfb_device *sysfb = to_drm_sysfb_device(plane->dev);
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(new_state, plane);
@@ -315,7 +317,7 @@ static const struct drm_plane_funcs vesadrm_primary_plane_funcs = {
 };
 
 static void vesadrm_crtc_helper_atomic_flush(struct drm_crtc *crtc,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_sysfb_device *sysfb = to_drm_sysfb_device(dev);
@@ -617,6 +619,22 @@ static struct drm_driver vesadrm_driver = {
  * Platform driver
  */
 
+static int vesadrm_pm_suspend(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_suspend(drm);
+}
+
+static int vesadrm_pm_resume(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+
+	return drm_mode_config_helper_resume(drm);
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(vesadrm_pm_ops, vesadrm_pm_suspend, vesadrm_pm_resume);
+
 static int vesadrm_probe(struct platform_device *pdev)
 {
 	struct vesadrm_device *vesa;
@@ -649,6 +667,7 @@ static void vesadrm_remove(struct platform_device *pdev)
 static struct platform_driver vesadrm_platform_driver = {
 	.driver = {
 		.name = "vesa-framebuffer",
+		.pm = pm_sleep_ptr(&vesadrm_pm_ops),
 	},
 	.probe = vesadrm_probe,
 	.remove = vesadrm_remove,
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index 06370b7e0e567..de683644d6587 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -616,7 +616,7 @@ static const u64 tegra124_modifiers[] = {
 };
 
 static int tegra_plane_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -708,7 +708,7 @@ static int tegra_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void tegra_plane_atomic_disable(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -725,7 +725,7 @@ static void tegra_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static void tegra_plane_atomic_update(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -864,7 +864,7 @@ static const u32 tegra_cursor_plane_formats[] = {
 };
 
 static int tegra_cursor_atomic_check(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -1000,7 +1000,7 @@ static void __tegra_cursor_atomic_update(struct drm_plane *plane,
 }
 
 static void tegra_cursor_atomic_update(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
 
@@ -1008,7 +1008,7 @@ static void tegra_cursor_atomic_update(struct drm_plane *plane,
 }
 
 static void tegra_cursor_atomic_disable(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -1026,7 +1026,7 @@ static void tegra_cursor_atomic_disable(struct drm_plane *plane,
 	tegra_dc_writel(dc, value, DC_DISP_DISP_WIN_OPTIONS);
 }
 
-static int tegra_cursor_atomic_async_check(struct drm_plane *plane, struct drm_atomic_state *state,
+static int tegra_cursor_atomic_async_check(struct drm_plane *plane, struct drm_atomic_commit *state,
 					   bool flip)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
@@ -1065,7 +1065,7 @@ static int tegra_cursor_atomic_async_check(struct drm_plane *plane, struct drm_a
 }
 
 static void tegra_cursor_atomic_async_update(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
 	struct tegra_dc *dc = to_tegra_dc(new_state->crtc);
@@ -1998,7 +1998,7 @@ static int tegra_dc_wait_idle(struct tegra_dc *dc, unsigned long timeout)
 
 static void
 tegra_crtc_update_memory_bandwidth(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state,
+				   struct drm_atomic_commit *state,
 				   bool prepare_bandwidth_transition)
 {
 	const struct tegra_plane_state *old_tegra_state, *new_tegra_state;
@@ -2094,7 +2094,7 @@ tegra_crtc_update_memory_bandwidth(struct drm_crtc *crtc,
 }
 
 static void tegra_crtc_atomic_disable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct tegra_dc *dc = to_tegra_dc(crtc);
 	u32 value;
@@ -2158,7 +2158,7 @@ static void tegra_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void tegra_crtc_atomic_enable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_display_mode *mode = &crtc->state->adjusted_mode;
 	struct tegra_dc_state *crtc_state = to_dc_state(crtc->state);
@@ -2289,7 +2289,7 @@ static void tegra_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void tegra_crtc_atomic_begin(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	unsigned long flags;
 
@@ -2310,7 +2310,7 @@ static void tegra_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void tegra_crtc_atomic_flush(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -2390,7 +2390,7 @@ tegra_plane_overlap_mask(struct drm_crtc_state *state,
 }
 
 static int tegra_crtc_calculate_memory_bandwidth(struct drm_crtc *crtc,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	ulong overlap_mask[TEGRA_DC_LEGACY_PLANES_NUM] = {}, mask;
 	u32 plane_peak_bw[TEGRA_DC_LEGACY_PLANES_NUM] = {};
@@ -2502,7 +2502,7 @@ static int tegra_crtc_calculate_memory_bandwidth(struct drm_crtc *crtc,
 }
 
 static int tegra_crtc_atomic_check(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	int err;
 
@@ -2514,7 +2514,7 @@ static int tegra_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 void tegra_crtc_atomic_post_commit(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	/*
 	 * Display bandwidth is allowed to go down only once hardware state
diff --git a/drivers/gpu/drm/tegra/dc.h b/drivers/gpu/drm/tegra/dc.h
index 0559fa6b1bf70..577c2e8e3ab3b 100644
--- a/drivers/gpu/drm/tegra/dc.h
+++ b/drivers/gpu/drm/tegra/dc.h
@@ -165,7 +165,7 @@ int tegra_dc_state_setup_clock(struct tegra_dc *dc,
 			       struct clk *clk, unsigned long pclk,
 			       unsigned int div);
 void tegra_crtc_atomic_post_commit(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state);
+				   struct drm_atomic_commit *state);
 
 /* from rgb.c */
 int tegra_dc_rgb_probe(struct tegra_dc *dc);
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 1dcef4e7d104c..dd6c564ff408b 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -44,7 +44,7 @@
 #define CDMA_GATHER_FETCHES_MAX_NB 16383
 
 static int tegra_atomic_check(struct drm_device *drm,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	int err;
 
@@ -62,7 +62,7 @@ static const struct drm_mode_config_funcs tegra_drm_mode_config_funcs = {
 };
 
 static void tegra_atomic_post_commit(struct drm_device *drm,
-				     struct drm_atomic_state *old_state)
+				     struct drm_atomic_commit *old_state)
 {
 	struct drm_crtc_state *old_crtc_state __maybe_unused;
 	struct drm_crtc *crtc;
@@ -72,7 +72,7 @@ static void tegra_atomic_post_commit(struct drm_device *drm,
 		tegra_crtc_atomic_post_commit(crtc, old_state);
 }
 
-static void tegra_atomic_commit_tail(struct drm_atomic_state *old_state)
+static void tegra_atomic_commit_tail(struct drm_atomic_commit *old_state)
 {
 	struct drm_device *drm = old_state->dev;
 	struct tegra_drm *tegra = drm->dev_private;
diff --git a/drivers/gpu/drm/tegra/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c
index 0adcd4244a429..10f02f35c8661 100644
--- a/drivers/gpu/drm/tegra/hdmi.c
+++ b/drivers/gpu/drm/tegra/hdmi.c
@@ -1576,8 +1576,6 @@ static int tegra_hdmi_init(struct host1x_client *client)
 				connector);
 			return PTR_ERR(connector);
 		}
-
-		drm_connector_attach_encoder(connector, &hdmi->output.encoder);
 	} else {
 		drm_connector_init_with_ddc(drm, &hdmi->output.connector,
 					    &tegra_hdmi_connector_funcs,
diff --git a/drivers/gpu/drm/tegra/hub.c b/drivers/gpu/drm/tegra/hub.c
index 10d993b8d043a..bd442bfd45407 100644
--- a/drivers/gpu/drm/tegra/hub.c
+++ b/drivers/gpu/drm/tegra/hub.c
@@ -424,7 +424,7 @@ static void tegra_dc_remove_shared_plane(struct tegra_dc *dc,
 }
 
 static int tegra_shared_plane_atomic_check(struct drm_plane *plane,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -482,7 +482,7 @@ static int tegra_shared_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void tegra_shared_plane_atomic_disable(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -532,7 +532,7 @@ static inline u32 compute_phase_incr(fixed20_12 in, unsigned int out)
 }
 
 static void tegra_shared_plane_atomic_update(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -847,7 +847,7 @@ static const struct drm_private_state_funcs tegra_display_hub_state_funcs = {
 
 static struct tegra_display_hub_state *
 tegra_display_hub_get_state(struct tegra_display_hub *hub,
-			    struct drm_atomic_state *state)
+			    struct drm_atomic_commit *state)
 {
 	struct drm_private_state *priv;
 
@@ -859,7 +859,7 @@ tegra_display_hub_get_state(struct tegra_display_hub *hub,
 }
 
 int tegra_display_hub_atomic_check(struct drm_device *drm,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct tegra_drm *tegra = drm->dev_private;
 	struct tegra_display_hub_state *hub_state;
@@ -925,7 +925,7 @@ static void tegra_display_hub_update(struct tegra_dc *dc)
 }
 
 void tegra_display_hub_atomic_commit(struct drm_device *drm,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct tegra_drm *tegra = drm->dev_private;
 	struct tegra_display_hub *hub = tegra->hub;
diff --git a/drivers/gpu/drm/tegra/hub.h b/drivers/gpu/drm/tegra/hub.h
index a66f18c4facc9..9f3a108f6b59b 100644
--- a/drivers/gpu/drm/tegra/hub.h
+++ b/drivers/gpu/drm/tegra/hub.h
@@ -84,9 +84,9 @@ struct drm_plane *tegra_shared_plane_create(struct drm_device *drm,
 					    enum drm_plane_type type);
 
 int tegra_display_hub_atomic_check(struct drm_device *drm,
-				   struct drm_atomic_state *state);
+				   struct drm_atomic_commit *state);
 void tegra_display_hub_atomic_commit(struct drm_device *drm,
-				     struct drm_atomic_state *state);
+				     struct drm_atomic_commit *state);
 
 #define DC_CMD_IHUB_COMMON_MISC_CTL 0x068
 #define  LATENCY_EVENT (1 << 3)
diff --git a/drivers/gpu/drm/tegra/rgb.c b/drivers/gpu/drm/tegra/rgb.c
index ff5a749710db3..e67fbb2362e6e 100644
--- a/drivers/gpu/drm/tegra/rgb.c
+++ b/drivers/gpu/drm/tegra/rgb.c
@@ -355,8 +355,6 @@ int tegra_dc_rgb_init(struct drm_device *drm, struct tegra_dc *dc)
 				connector);
 			return PTR_ERR(connector);
 		}
-
-		drm_connector_attach_encoder(connector, &output->encoder);
 	}
 
 	err = tegra_output_init(drm, output);
diff --git a/drivers/gpu/drm/tests/Makefile b/drivers/gpu/drm/tests/Makefile
index d2e2e3d8349ae..3b83e6616fb25 100644
--- a/drivers/gpu/drm/tests/Makefile
+++ b/drivers/gpu/drm/tests/Makefile
@@ -5,7 +5,7 @@ obj-$(CONFIG_DRM_KUNIT_TEST_HELPERS) += \
 
 obj-$(CONFIG_DRM_KUNIT_TEST) += \
 	drm_atomic_test.o \
-	drm_atomic_state_test.o \
+	drm_atomic_commit_test.o \
 	drm_bridge_test.o \
 	drm_cmdline_parser_test.o \
 	drm_connector_test.o \
diff --git a/drivers/gpu/drm/tests/drm_atomic_commit_test.c b/drivers/gpu/drm/tests/drm_atomic_commit_test.c
new file mode 100644
index 0000000000000..ff69bade19b89
--- /dev/null
+++ b/drivers/gpu/drm/tests/drm_atomic_commit_test.c
@@ -0,0 +1,409 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Test cases for the drm_atomic_commit helpers
+ *
+ * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_atomic_uapi.h>
+#include <drm/drm_kunit_helpers.h>
+#include <drm/drm_probe_helper.h>
+
+#define DRM_TEST_ENC_0 BIT(0)
+#define DRM_TEST_ENC_1 BIT(1)
+#define DRM_TEST_ENC_2 BIT(2)
+
+#define DRM_TEST_CONN_0 BIT(0)
+
+struct drm_clone_mode_test {
+	const char *name;
+	u32 encoder_mask;
+	int expected_result;
+};
+
+static const struct drm_display_mode drm_atomic_test_mode = {
+	DRM_MODE("1024x768", 0, 65000, 1024, 1048,
+		 1184, 1344, 0, 768, 771, 777, 806, 0,
+		 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC)
+};
+
+struct drm_atomic_test_priv {
+	struct drm_device drm;
+	struct drm_plane *plane;
+	struct drm_crtc *crtc;
+	struct drm_encoder encoders[3];
+	struct drm_connector connectors[2];
+};
+
+static int modeset_counter;
+
+static void drm_test_encoder_mode_set(struct drm_encoder *encoder,
+				      struct drm_crtc_state *crtc_state,
+				      struct drm_connector_state *conn_state)
+{
+	modeset_counter++;
+}
+
+static const struct drm_encoder_helper_funcs drm_atomic_test_encoder_funcs = {
+	.atomic_mode_set	= drm_test_encoder_mode_set,
+};
+
+static const struct drm_connector_funcs dummy_connector_funcs = {
+	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
+	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
+	.reset			= drm_atomic_helper_connector_reset,
+};
+
+static int drm_atomic_test_dummy_get_modes(struct drm_connector *connector)
+{
+	return drm_connector_helper_get_modes_fixed(connector,
+						    &drm_atomic_test_mode);
+}
+
+static const struct drm_connector_helper_funcs dummy_connector_helper_funcs = {
+	.get_modes	= drm_atomic_test_dummy_get_modes,
+};
+
+static struct drm_atomic_test_priv *
+drm_atomic_test_init_drm_components(struct kunit *test, bool has_connectors)
+{
+	struct drm_atomic_test_priv *priv;
+	struct drm_encoder *enc;
+	struct drm_connector *conn;
+	struct drm_device *drm;
+	struct device *dev;
+	int ret;
+
+	dev = drm_kunit_helper_alloc_device(test);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
+
+	priv = drm_kunit_helper_alloc_drm_device(test, dev,
+						 struct drm_atomic_test_priv,
+						 drm,
+						 DRIVER_MODESET | DRIVER_ATOMIC);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
+	test->priv = priv;
+
+	drm = &priv->drm;
+	priv->plane = drm_kunit_helper_create_primary_plane(test, drm,
+							    NULL,
+							    NULL,
+							    NULL, 0,
+							    NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->plane);
+
+	priv->crtc = drm_kunit_helper_create_crtc(test, drm,
+						  priv->plane, NULL,
+						  NULL,
+						  NULL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->crtc);
+
+	for (int i = 0; i < ARRAY_SIZE(priv->encoders); i++) {
+		enc = &priv->encoders[i];
+
+		ret = drmm_encoder_init(drm, enc, NULL,
+					DRM_MODE_ENCODER_DSI, NULL);
+		KUNIT_ASSERT_EQ(test, ret, 0);
+
+		enc->possible_crtcs = drm_crtc_mask(priv->crtc);
+	}
+
+	priv->encoders[0].possible_clones = DRM_TEST_ENC_0 | DRM_TEST_ENC_1;
+	priv->encoders[1].possible_clones = DRM_TEST_ENC_0 | DRM_TEST_ENC_1;
+	priv->encoders[2].possible_clones = DRM_TEST_ENC_2;
+
+	if (!has_connectors)
+		goto done;
+
+	BUILD_BUG_ON(ARRAY_SIZE(priv->connectors) > ARRAY_SIZE(priv->encoders));
+
+	for (int i = 0; i < ARRAY_SIZE(priv->connectors); i++) {
+		conn = &priv->connectors[i];
+
+		ret = drmm_connector_init(drm, conn, &dummy_connector_funcs,
+					  DRM_MODE_CONNECTOR_DSI, NULL);
+		KUNIT_ASSERT_EQ(test, ret, 0);
+
+		drm_connector_helper_add(conn, &dummy_connector_helper_funcs);
+		drm_encoder_helper_add(&priv->encoders[i],
+				       &drm_atomic_test_encoder_funcs);
+
+		drm_connector_attach_encoder(conn, &priv->encoders[i]);
+	}
+
+done:
+	drm_mode_config_reset(drm);
+
+	return priv;
+}
+
+static int set_up_atomic_state(struct kunit *test,
+			       struct drm_atomic_test_priv *priv,
+			       struct drm_connector *connector,
+			       struct drm_modeset_acquire_ctx *ctx)
+{
+	struct drm_device *drm = &priv->drm;
+	struct drm_crtc *crtc = priv->crtc;
+	struct drm_atomic_commit *state;
+	struct drm_connector_state *conn_state;
+	struct drm_crtc_state *crtc_state;
+	int ret;
+
+	state = drm_kunit_helper_atomic_state_alloc(test, drm, ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+	if (connector) {
+		conn_state = drm_atomic_get_connector_state(state, connector);
+		if (IS_ERR(conn_state))
+			return PTR_ERR(conn_state);
+
+		ret = drm_atomic_set_crtc_for_connector(conn_state, crtc);
+		if (ret)
+			return ret;
+	}
+
+	crtc_state = drm_atomic_get_crtc_state(state, crtc);
+	if (IS_ERR(crtc_state))
+		return PTR_ERR(crtc_state);
+
+	ret = drm_atomic_set_mode_for_crtc(crtc_state, &drm_atomic_test_mode);
+	if (ret)
+		return ret;
+
+	crtc_state->enable = true;
+	crtc_state->active = true;
+
+	if (connector) {
+		ret = drm_atomic_commit(state);
+		if (ret)
+			return ret;
+	} else {
+		// dummy connector mask
+		crtc_state->connector_mask = DRM_TEST_CONN_0;
+	}
+
+	return 0;
+}
+
+/*
+ * Test that the DRM encoder mode_set() is called when the atomic state
+ * connectors are changed but the CRTC mode is not.
+ */
+static void drm_test_check_connector_changed_modeset(struct kunit *test)
+{
+	struct drm_atomic_test_priv *priv;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_connector *old_conn, *new_conn;
+	struct drm_atomic_commit *state;
+	struct drm_device *drm;
+	struct drm_connector_state *new_conn_state, *old_conn_state;
+	int ret, initial_modeset_count;
+
+	priv = drm_atomic_test_init_drm_components(test, true);
+	KUNIT_ASSERT_NOT_NULL(test, priv);
+
+	drm = &priv->drm;
+	old_conn = &priv->connectors[0];
+	new_conn = &priv->connectors[1];
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+	// first modeset to enable
+retry_set_up:
+	ret = set_up_atomic_state(test, priv, old_conn, &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_set_up;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+	new_conn_state = drm_atomic_get_connector_state(state, new_conn);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, new_conn_state);
+
+	old_conn_state = drm_atomic_get_connector_state(state, old_conn);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, old_conn_state);
+
+	ret = drm_atomic_set_crtc_for_connector(old_conn_state, NULL);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	ret = drm_atomic_set_crtc_for_connector(new_conn_state, priv->crtc);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	initial_modeset_count = modeset_counter;
+
+	// modeset_disables is called as part of the atomic commit tail
+	ret = drm_atomic_commit(state);
+	KUNIT_ASSERT_EQ(test, ret, 0);
+	KUNIT_ASSERT_EQ(test, modeset_counter, initial_modeset_count + 1);
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+}
+
+/*
+ * Test that the drm_crtc_in_clone_mode() helper can detect if a given CRTC
+ * state is in clone mode
+ */
+static void drm_test_check_in_clone_mode(struct kunit *test)
+{
+	bool ret;
+	const struct drm_clone_mode_test *param = test->param_value;
+	struct drm_crtc_state *crtc_state;
+
+	crtc_state = kunit_kzalloc(test, sizeof(*crtc_state), GFP_KERNEL);
+	KUNIT_ASSERT_NOT_NULL(test, crtc_state);
+
+	crtc_state->encoder_mask = param->encoder_mask;
+
+	ret = drm_crtc_in_clone_mode(crtc_state);
+
+	KUNIT_ASSERT_EQ(test, ret, param->expected_result);
+}
+
+/*
+ * Test that the atomic commit path will succeed for valid clones (or non-cloned
+ * states) and fail for states where the cloned encoders are not possible_clones
+ * of each other.
+ */
+static void drm_test_check_valid_clones(struct kunit *test)
+{
+	int ret;
+	const struct drm_clone_mode_test *param = test->param_value;
+	struct drm_atomic_test_priv *priv;
+	struct drm_modeset_acquire_ctx ctx;
+	struct drm_device *drm;
+	struct drm_atomic_commit *state;
+	struct drm_crtc_state *crtc_state;
+
+	priv = drm_atomic_test_init_drm_components(test, false);
+	KUNIT_ASSERT_NOT_NULL(test, priv);
+
+	drm = &priv->drm;
+
+	drm_modeset_acquire_init(&ctx, 0);
+
+retry_set_up:
+	ret = set_up_atomic_state(test, priv, NULL, &ctx);
+	if (ret == -EDEADLK) {
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry_set_up;
+	}
+	KUNIT_ASSERT_EQ(test, ret, 0);
+
+	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
+
+retry:
+	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
+	if (PTR_ERR(crtc_state) == -EDEADLK) {
+		drm_atomic_commit_clear(state);
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry;
+	}
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
+
+	crtc_state->encoder_mask = param->encoder_mask;
+
+	// force modeset
+	crtc_state->mode_changed = true;
+
+	ret = drm_atomic_helper_check_modeset(drm, state);
+	if (ret == -EDEADLK) {
+		drm_atomic_commit_clear(state);
+		ret = drm_modeset_backoff(&ctx);
+		if (!ret)
+			goto retry;
+	}
+	KUNIT_ASSERT_EQ(test, ret, param->expected_result);
+
+	drm_modeset_drop_locks(&ctx);
+	drm_modeset_acquire_fini(&ctx);
+}
+
+static void drm_check_in_clone_mode_desc(const struct drm_clone_mode_test *t,
+				      char *desc)
+{
+	sprintf(desc, "%s", t->name);
+}
+
+static void drm_check_valid_clones_desc(const struct drm_clone_mode_test *t,
+				      char *desc)
+{
+	sprintf(desc, "%s", t->name);
+}
+
+static const struct drm_clone_mode_test drm_clone_mode_tests[] = {
+	{
+		.name = "in_clone_mode",
+		.encoder_mask = DRM_TEST_ENC_0 | DRM_TEST_ENC_1,
+		.expected_result = true,
+	},
+	{
+		.name = "not_in_clone_mode",
+		.encoder_mask = DRM_TEST_ENC_0,
+		.expected_result = false,
+	},
+};
+
+static const struct drm_clone_mode_test drm_valid_clone_mode_tests[] = {
+	{
+		.name = "not_in_clone_mode",
+		.encoder_mask = DRM_TEST_ENC_0,
+		.expected_result = 0,
+	},
+
+	{
+		.name = "valid_clone",
+		.encoder_mask = DRM_TEST_ENC_0 | DRM_TEST_ENC_1,
+		.expected_result = 0,
+	},
+	{
+		.name = "invalid_clone",
+		.encoder_mask = DRM_TEST_ENC_0 | DRM_TEST_ENC_2,
+		.expected_result = -EINVAL,
+	},
+};
+
+KUNIT_ARRAY_PARAM(drm_check_in_clone_mode, drm_clone_mode_tests,
+		  drm_check_in_clone_mode_desc);
+
+KUNIT_ARRAY_PARAM(drm_check_valid_clones, drm_valid_clone_mode_tests,
+		  drm_check_valid_clones_desc);
+
+static struct kunit_case drm_test_check_modeset_test[] = {
+	KUNIT_CASE(drm_test_check_connector_changed_modeset),
+	{}
+};
+
+static struct kunit_case drm_in_clone_mode_check_test[] = {
+	KUNIT_CASE_PARAM(drm_test_check_in_clone_mode,
+			 drm_check_in_clone_mode_gen_params),
+	KUNIT_CASE_PARAM(drm_test_check_valid_clones,
+			 drm_check_valid_clones_gen_params),
+	{}
+};
+
+static struct kunit_suite drm_test_check_modeset_test_suite = {
+	.name = "drm_validate_modeset",
+	.test_cases = drm_test_check_modeset_test,
+};
+
+static struct kunit_suite drm_in_clone_mode_check_test_suite = {
+	.name = "drm_validate_clone_mode",
+	.test_cases = drm_in_clone_mode_check_test,
+};
+
+kunit_test_suites(&drm_in_clone_mode_check_test_suite,
+		  &drm_test_check_modeset_test_suite);
+
+MODULE_AUTHOR("Jessica Zhang <quic_jesszhan@quicinc.com");
+MODULE_DESCRIPTION("Test cases for the drm_atomic_helper functions");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_atomic_state_test.c b/drivers/gpu/drm/tests/drm_atomic_state_test.c
deleted file mode 100644
index bc27f65b28233..0000000000000
--- a/drivers/gpu/drm/tests/drm_atomic_state_test.c
+++ /dev/null
@@ -1,409 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Test cases for the drm_atomic_state helpers
- *
- * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
- */
-
-#include <drm/drm_atomic.h>
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_atomic_uapi.h>
-#include <drm/drm_kunit_helpers.h>
-#include <drm/drm_probe_helper.h>
-
-#define DRM_TEST_ENC_0 BIT(0)
-#define DRM_TEST_ENC_1 BIT(1)
-#define DRM_TEST_ENC_2 BIT(2)
-
-#define DRM_TEST_CONN_0 BIT(0)
-
-struct drm_clone_mode_test {
-	const char *name;
-	u32 encoder_mask;
-	int expected_result;
-};
-
-static const struct drm_display_mode drm_atomic_test_mode = {
-	DRM_MODE("1024x768", 0, 65000, 1024, 1048,
-		 1184, 1344, 0, 768, 771, 777, 806, 0,
-		 DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC)
-};
-
-struct drm_atomic_test_priv {
-	struct drm_device drm;
-	struct drm_plane *plane;
-	struct drm_crtc *crtc;
-	struct drm_encoder encoders[3];
-	struct drm_connector connectors[2];
-};
-
-static int modeset_counter;
-
-static void drm_test_encoder_mode_set(struct drm_encoder *encoder,
-				      struct drm_crtc_state *crtc_state,
-				      struct drm_connector_state *conn_state)
-{
-	modeset_counter++;
-}
-
-static const struct drm_encoder_helper_funcs drm_atomic_test_encoder_funcs = {
-	.atomic_mode_set	= drm_test_encoder_mode_set,
-};
-
-static const struct drm_connector_funcs dummy_connector_funcs = {
-	.atomic_destroy_state	= drm_atomic_helper_connector_destroy_state,
-	.atomic_duplicate_state	= drm_atomic_helper_connector_duplicate_state,
-	.reset			= drm_atomic_helper_connector_reset,
-};
-
-static int drm_atomic_test_dummy_get_modes(struct drm_connector *connector)
-{
-	return drm_connector_helper_get_modes_fixed(connector,
-						    &drm_atomic_test_mode);
-}
-
-static const struct drm_connector_helper_funcs dummy_connector_helper_funcs = {
-	.get_modes	= drm_atomic_test_dummy_get_modes,
-};
-
-static struct drm_atomic_test_priv *
-drm_atomic_test_init_drm_components(struct kunit *test, bool has_connectors)
-{
-	struct drm_atomic_test_priv *priv;
-	struct drm_encoder *enc;
-	struct drm_connector *conn;
-	struct drm_device *drm;
-	struct device *dev;
-	int ret;
-
-	dev = drm_kunit_helper_alloc_device(test);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, dev);
-
-	priv = drm_kunit_helper_alloc_drm_device(test, dev,
-						 struct drm_atomic_test_priv,
-						 drm,
-						 DRIVER_MODESET | DRIVER_ATOMIC);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv);
-	test->priv = priv;
-
-	drm = &priv->drm;
-	priv->plane = drm_kunit_helper_create_primary_plane(test, drm,
-							    NULL,
-							    NULL,
-							    NULL, 0,
-							    NULL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->plane);
-
-	priv->crtc = drm_kunit_helper_create_crtc(test, drm,
-						  priv->plane, NULL,
-						  NULL,
-						  NULL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, priv->crtc);
-
-	for (int i = 0; i < ARRAY_SIZE(priv->encoders); i++) {
-		enc = &priv->encoders[i];
-
-		ret = drmm_encoder_init(drm, enc, NULL,
-					DRM_MODE_ENCODER_DSI, NULL);
-		KUNIT_ASSERT_EQ(test, ret, 0);
-
-		enc->possible_crtcs = drm_crtc_mask(priv->crtc);
-	}
-
-	priv->encoders[0].possible_clones = DRM_TEST_ENC_0 | DRM_TEST_ENC_1;
-	priv->encoders[1].possible_clones = DRM_TEST_ENC_0 | DRM_TEST_ENC_1;
-	priv->encoders[2].possible_clones = DRM_TEST_ENC_2;
-
-	if (!has_connectors)
-		goto done;
-
-	BUILD_BUG_ON(ARRAY_SIZE(priv->connectors) > ARRAY_SIZE(priv->encoders));
-
-	for (int i = 0; i < ARRAY_SIZE(priv->connectors); i++) {
-		conn = &priv->connectors[i];
-
-		ret = drmm_connector_init(drm, conn, &dummy_connector_funcs,
-					  DRM_MODE_CONNECTOR_DSI, NULL);
-		KUNIT_ASSERT_EQ(test, ret, 0);
-
-		drm_connector_helper_add(conn, &dummy_connector_helper_funcs);
-		drm_encoder_helper_add(&priv->encoders[i],
-				       &drm_atomic_test_encoder_funcs);
-
-		drm_connector_attach_encoder(conn, &priv->encoders[i]);
-	}
-
-done:
-	drm_mode_config_reset(drm);
-
-	return priv;
-}
-
-static int set_up_atomic_state(struct kunit *test,
-			       struct drm_atomic_test_priv *priv,
-			       struct drm_connector *connector,
-			       struct drm_modeset_acquire_ctx *ctx)
-{
-	struct drm_device *drm = &priv->drm;
-	struct drm_crtc *crtc = priv->crtc;
-	struct drm_atomic_state *state;
-	struct drm_connector_state *conn_state;
-	struct drm_crtc_state *crtc_state;
-	int ret;
-
-	state = drm_kunit_helper_atomic_state_alloc(test, drm, ctx);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
-
-	if (connector) {
-		conn_state = drm_atomic_get_connector_state(state, connector);
-		if (IS_ERR(conn_state))
-			return PTR_ERR(conn_state);
-
-		ret = drm_atomic_set_crtc_for_connector(conn_state, crtc);
-		if (ret)
-			return ret;
-	}
-
-	crtc_state = drm_atomic_get_crtc_state(state, crtc);
-	if (IS_ERR(crtc_state))
-		return PTR_ERR(crtc_state);
-
-	ret = drm_atomic_set_mode_for_crtc(crtc_state, &drm_atomic_test_mode);
-	if (ret)
-		return ret;
-
-	crtc_state->enable = true;
-	crtc_state->active = true;
-
-	if (connector) {
-		ret = drm_atomic_commit(state);
-		if (ret)
-			return ret;
-	} else {
-		// dummy connector mask
-		crtc_state->connector_mask = DRM_TEST_CONN_0;
-	}
-
-	return 0;
-}
-
-/*
- * Test that the DRM encoder mode_set() is called when the atomic state
- * connectors are changed but the CRTC mode is not.
- */
-static void drm_test_check_connector_changed_modeset(struct kunit *test)
-{
-	struct drm_atomic_test_priv *priv;
-	struct drm_modeset_acquire_ctx ctx;
-	struct drm_connector *old_conn, *new_conn;
-	struct drm_atomic_state *state;
-	struct drm_device *drm;
-	struct drm_connector_state *new_conn_state, *old_conn_state;
-	int ret, initial_modeset_count;
-
-	priv = drm_atomic_test_init_drm_components(test, true);
-	KUNIT_ASSERT_NOT_NULL(test, priv);
-
-	drm = &priv->drm;
-	old_conn = &priv->connectors[0];
-	new_conn = &priv->connectors[1];
-
-	drm_modeset_acquire_init(&ctx, 0);
-
-	// first modeset to enable
-retry_set_up:
-	ret = set_up_atomic_state(test, priv, old_conn, &ctx);
-	if (ret == -EDEADLK) {
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry_set_up;
-	}
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
-	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
-
-	new_conn_state = drm_atomic_get_connector_state(state, new_conn);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, new_conn_state);
-
-	old_conn_state = drm_atomic_get_connector_state(state, old_conn);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, old_conn_state);
-
-	ret = drm_atomic_set_crtc_for_connector(old_conn_state, NULL);
-	KUNIT_EXPECT_EQ(test, ret, 0);
-
-	ret = drm_atomic_set_crtc_for_connector(new_conn_state, priv->crtc);
-	KUNIT_EXPECT_EQ(test, ret, 0);
-
-	initial_modeset_count = modeset_counter;
-
-	// modeset_disables is called as part of the atomic commit tail
-	ret = drm_atomic_commit(state);
-	KUNIT_ASSERT_EQ(test, ret, 0);
-	KUNIT_ASSERT_EQ(test, modeset_counter, initial_modeset_count + 1);
-
-	drm_modeset_drop_locks(&ctx);
-	drm_modeset_acquire_fini(&ctx);
-}
-
-/*
- * Test that the drm_crtc_in_clone_mode() helper can detect if a given CRTC
- * state is in clone mode
- */
-static void drm_test_check_in_clone_mode(struct kunit *test)
-{
-	bool ret;
-	const struct drm_clone_mode_test *param = test->param_value;
-	struct drm_crtc_state *crtc_state;
-
-	crtc_state = kunit_kzalloc(test, sizeof(*crtc_state), GFP_KERNEL);
-	KUNIT_ASSERT_NOT_NULL(test, crtc_state);
-
-	crtc_state->encoder_mask = param->encoder_mask;
-
-	ret = drm_crtc_in_clone_mode(crtc_state);
-
-	KUNIT_ASSERT_EQ(test, ret, param->expected_result);
-}
-
-/*
- * Test that the atomic commit path will succeed for valid clones (or non-cloned
- * states) and fail for states where the cloned encoders are not possible_clones
- * of each other.
- */
-static void drm_test_check_valid_clones(struct kunit *test)
-{
-	int ret;
-	const struct drm_clone_mode_test *param = test->param_value;
-	struct drm_atomic_test_priv *priv;
-	struct drm_modeset_acquire_ctx ctx;
-	struct drm_device *drm;
-	struct drm_atomic_state *state;
-	struct drm_crtc_state *crtc_state;
-
-	priv = drm_atomic_test_init_drm_components(test, false);
-	KUNIT_ASSERT_NOT_NULL(test, priv);
-
-	drm = &priv->drm;
-
-	drm_modeset_acquire_init(&ctx, 0);
-
-retry_set_up:
-	ret = set_up_atomic_state(test, priv, NULL, &ctx);
-	if (ret == -EDEADLK) {
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry_set_up;
-	}
-	KUNIT_ASSERT_EQ(test, ret, 0);
-
-	state = drm_kunit_helper_atomic_state_alloc(test, drm, &ctx);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, state);
-
-retry:
-	crtc_state = drm_atomic_get_crtc_state(state, priv->crtc);
-	if (PTR_ERR(crtc_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry;
-	}
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, crtc_state);
-
-	crtc_state->encoder_mask = param->encoder_mask;
-
-	// force modeset
-	crtc_state->mode_changed = true;
-
-	ret = drm_atomic_helper_check_modeset(drm, state);
-	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
-		ret = drm_modeset_backoff(&ctx);
-		if (!ret)
-			goto retry;
-	}
-	KUNIT_ASSERT_EQ(test, ret, param->expected_result);
-
-	drm_modeset_drop_locks(&ctx);
-	drm_modeset_acquire_fini(&ctx);
-}
-
-static void drm_check_in_clone_mode_desc(const struct drm_clone_mode_test *t,
-				      char *desc)
-{
-	sprintf(desc, "%s", t->name);
-}
-
-static void drm_check_valid_clones_desc(const struct drm_clone_mode_test *t,
-				      char *desc)
-{
-	sprintf(desc, "%s", t->name);
-}
-
-static const struct drm_clone_mode_test drm_clone_mode_tests[] = {
-	{
-		.name = "in_clone_mode",
-		.encoder_mask = DRM_TEST_ENC_0 | DRM_TEST_ENC_1,
-		.expected_result = true,
-	},
-	{
-		.name = "not_in_clone_mode",
-		.encoder_mask = DRM_TEST_ENC_0,
-		.expected_result = false,
-	},
-};
-
-static const struct drm_clone_mode_test drm_valid_clone_mode_tests[] = {
-	{
-		.name = "not_in_clone_mode",
-		.encoder_mask = DRM_TEST_ENC_0,
-		.expected_result = 0,
-	},
-
-	{
-		.name = "valid_clone",
-		.encoder_mask = DRM_TEST_ENC_0 | DRM_TEST_ENC_1,
-		.expected_result = 0,
-	},
-	{
-		.name = "invalid_clone",
-		.encoder_mask = DRM_TEST_ENC_0 | DRM_TEST_ENC_2,
-		.expected_result = -EINVAL,
-	},
-};
-
-KUNIT_ARRAY_PARAM(drm_check_in_clone_mode, drm_clone_mode_tests,
-		  drm_check_in_clone_mode_desc);
-
-KUNIT_ARRAY_PARAM(drm_check_valid_clones, drm_valid_clone_mode_tests,
-		  drm_check_valid_clones_desc);
-
-static struct kunit_case drm_test_check_modeset_test[] = {
-	KUNIT_CASE(drm_test_check_connector_changed_modeset),
-	{}
-};
-
-static struct kunit_case drm_in_clone_mode_check_test[] = {
-	KUNIT_CASE_PARAM(drm_test_check_in_clone_mode,
-			 drm_check_in_clone_mode_gen_params),
-	KUNIT_CASE_PARAM(drm_test_check_valid_clones,
-			 drm_check_valid_clones_gen_params),
-	{}
-};
-
-static struct kunit_suite drm_test_check_modeset_test_suite = {
-	.name = "drm_validate_modeset",
-	.test_cases = drm_test_check_modeset_test,
-};
-
-static struct kunit_suite drm_in_clone_mode_check_test_suite = {
-	.name = "drm_validate_clone_mode",
-	.test_cases = drm_in_clone_mode_check_test,
-};
-
-kunit_test_suites(&drm_in_clone_mode_check_test_suite,
-		  &drm_test_check_modeset_test_suite);
-
-MODULE_AUTHOR("Jessica Zhang <quic_jesszhan@quicinc.com");
-MODULE_DESCRIPTION("Test cases for the drm_atomic_helper functions");
-MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/tests/drm_bridge_test.c b/drivers/gpu/drm/tests/drm_bridge_test.c
index 887020141c7f4..64b665580a885 100644
--- a/drivers/gpu/drm/tests/drm_bridge_test.c
+++ b/drivers/gpu/drm/tests/drm_bridge_test.c
@@ -71,7 +71,7 @@ static const struct drm_bridge_funcs drm_test_bridge_legacy_funcs = {
 };
 
 static void drm_test_bridge_atomic_enable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_bridge_priv *priv = bridge_to_priv(bridge);
 
@@ -79,7 +79,7 @@ static void drm_test_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void drm_test_bridge_atomic_disable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_bridge_priv *priv = bridge_to_priv(bridge);
 
@@ -173,8 +173,6 @@ drm_test_bridge_init(struct kunit *test, const struct drm_bridge_funcs *funcs)
 	if (IS_ERR(priv->connector))
 		return ERR_CAST(priv->connector);
 
-	drm_connector_attach_encoder(priv->connector, enc);
-
 	drm_mode_config_reset(drm);
 
 	return priv;
@@ -190,7 +188,7 @@ static void drm_test_drm_bridge_get_current_state_atomic(struct kunit *test)
 	struct drm_bridge_init_priv *priv;
 	struct drm_bridge_state *curr_bridge_state;
 	struct drm_bridge_state *bridge_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_bridge *bridge;
 	struct drm_device *drm;
 	int ret;
@@ -211,7 +209,7 @@ static void drm_test_drm_bridge_get_current_state_atomic(struct kunit *test)
 
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		drm_modeset_backoff(&ctx);
 		goto retry_commit;
 	}
diff --git a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
index a4357efaa983d..c9819c3fc635a 100644
--- a/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
+++ b/drivers/gpu/drm/tests/drm_hdmi_state_helper_test.c
@@ -179,7 +179,7 @@ static const struct drm_connector_funcs dummy_connector_funcs = {
 };
 
 static void test_encoder_atomic_enable(struct drm_encoder *encoder,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	struct drm_atomic_helper_connector_hdmi_priv *priv =
 		encoder_to_priv(encoder);
@@ -292,7 +292,7 @@ static void drm_test_check_broadcast_rgb_crtc_mode_changed(struct kunit *test)
 	struct drm_connector_state *old_conn_state;
 	struct drm_connector_state *new_conn_state;
 	struct drm_crtc_state *crtc_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -367,7 +367,7 @@ static void drm_test_check_broadcast_rgb_crtc_mode_not_changed(struct kunit *tes
 	struct drm_connector_state *old_conn_state;
 	struct drm_connector_state *new_conn_state;
 	struct drm_crtc_state *crtc_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -442,7 +442,7 @@ static void drm_test_check_broadcast_rgb_auto_cea_mode(struct kunit *test)
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_connector_state *conn_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -509,7 +509,7 @@ static void drm_test_check_broadcast_rgb_auto_cea_mode_vic_1(struct kunit *test)
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_connector_state *conn_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *mode;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -576,7 +576,7 @@ static void drm_test_check_broadcast_rgb_full_cea_mode(struct kunit *test)
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_connector_state *conn_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -645,7 +645,7 @@ static void drm_test_check_broadcast_rgb_full_cea_mode_vic_1(struct kunit *test)
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_connector_state *conn_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *mode;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -714,7 +714,7 @@ static void drm_test_check_broadcast_rgb_limited_cea_mode(struct kunit *test)
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_connector_state *conn_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -783,7 +783,7 @@ static void drm_test_check_broadcast_rgb_limited_cea_mode_vic_1(struct kunit *te
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_connector_state *conn_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *mode;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -853,7 +853,7 @@ static void drm_test_check_broadcast_rgb_cea_mode_yuv420(struct kunit *test)
 	enum drm_hdmi_broadcast_rgb broadcast_rgb;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_connector_state *conn_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *mode;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -896,7 +896,7 @@ static void drm_test_check_broadcast_rgb_cea_mode_yuv420(struct kunit *test)
 retry_conn_state:
 	conn_state = drm_atomic_get_connector_state(state, conn);
 	if (PTR_ERR(conn_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -907,7 +907,7 @@ static void drm_test_check_broadcast_rgb_cea_mode_yuv420(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -955,7 +955,7 @@ static void drm_test_check_output_bpc_crtc_mode_changed(struct kunit *test)
 	struct drm_connector_state *old_conn_state;
 	struct drm_connector_state *new_conn_state;
 	struct drm_crtc_state *crtc_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -1037,7 +1037,7 @@ static void drm_test_check_output_bpc_crtc_mode_not_changed(struct kunit *test)
 	struct drm_connector_state *old_conn_state;
 	struct drm_connector_state *new_conn_state;
 	struct drm_crtc_state *crtc_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -1339,7 +1339,7 @@ static void drm_test_check_hdmi_funcs_reject_rate(struct kunit *test)
 {
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_crtc_state *crtc_state;
 	struct drm_connector *conn;
@@ -1706,7 +1706,7 @@ static void drm_test_check_driver_unsupported_fallback_yuv420(struct kunit *test
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_connector_state *conn_state;
 	struct drm_crtc_state *crtc_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_info *info;
 	struct drm_display_mode *preferred, *yuv420_only_mode;
 	struct drm_connector *conn;
@@ -1758,7 +1758,7 @@ static void drm_test_check_driver_unsupported_fallback_yuv420(struct kunit *test
 retry_crtc_state:
 	crtc_state = drm_atomic_get_crtc_state(state, crtc);
 	if (PTR_ERR(crtc_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -1770,7 +1770,7 @@ static void drm_test_check_driver_unsupported_fallback_yuv420(struct kunit *test
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -2142,7 +2142,7 @@ static void drm_test_check_disable_connector(struct kunit *test)
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_connector_state *conn_state;
 	struct drm_crtc_state *crtc_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -2531,7 +2531,7 @@ static void drm_test_check_infoframes(struct kunit *test)
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
 	struct drm_crtc_state *crtc_state;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
 	struct drm_device *drm;
@@ -2573,7 +2573,7 @@ static void drm_test_check_infoframes(struct kunit *test)
 retry_crtc_state:
 	crtc_state = drm_atomic_get_crtc_state(state, crtc);
 	if (PTR_ERR(crtc_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -2586,7 +2586,7 @@ static void drm_test_check_infoframes(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -2595,7 +2595,7 @@ static void drm_test_check_infoframes(struct kunit *test)
 
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -2633,7 +2633,7 @@ static void drm_test_check_reject_avi_infoframe(struct kunit *test)
 {
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_crtc_state *crtc_state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
@@ -2678,7 +2678,7 @@ static void drm_test_check_reject_avi_infoframe(struct kunit *test)
 retry_crtc_state:
 	crtc_state = drm_atomic_get_crtc_state(state, crtc);
 	if (PTR_ERR(crtc_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -2691,7 +2691,7 @@ static void drm_test_check_reject_avi_infoframe(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -2700,7 +2700,7 @@ static void drm_test_check_reject_avi_infoframe(struct kunit *test)
 
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -2736,7 +2736,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_8(struct kunit *test)
 {
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_connector_state *new_conn_state;
 	struct drm_crtc_state *crtc_state;
 	struct drm_display_mode *preferred;
@@ -2782,7 +2782,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_8(struct kunit *test)
 retry_conn_state:
 	new_conn_state = drm_atomic_get_connector_state(state, conn);
 	if (PTR_ERR(new_conn_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -2791,7 +2791,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_8(struct kunit *test)
 
 	crtc_state = drm_atomic_get_crtc_state(state, crtc);
 	if (PTR_ERR(crtc_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -2810,7 +2810,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_8(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -2819,7 +2819,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_8(struct kunit *test)
 
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -2847,7 +2847,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_10(struct kunit *test)
 {
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_connector_state *new_conn_state;
 	struct drm_crtc_state *crtc_state;
 	struct drm_display_mode *preferred;
@@ -2896,7 +2896,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_10(struct kunit *test)
 retry_conn_state:
 	new_conn_state = drm_atomic_get_connector_state(state, conn);
 	if (PTR_ERR(new_conn_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -2905,7 +2905,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_10(struct kunit *test)
 
 	crtc_state = drm_atomic_get_crtc_state(state, crtc);
 	if (PTR_ERR(crtc_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -2933,7 +2933,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_10(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -2942,7 +2942,7 @@ static void drm_test_check_reject_hdr_infoframe_bpc_10(struct kunit *test)
 
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_conn_state;
@@ -2985,7 +2985,7 @@ static void drm_test_check_reject_audio_infoframe(struct kunit *test)
 {
 	struct drm_atomic_helper_connector_hdmi_priv *priv;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_crtc_state *crtc_state;
 	struct drm_display_mode *preferred;
 	struct drm_connector *conn;
@@ -3031,7 +3031,7 @@ static void drm_test_check_reject_audio_infoframe(struct kunit *test)
 retry_crtc_state:
 	crtc_state = drm_atomic_get_crtc_state(state, crtc);
 	if (PTR_ERR(crtc_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -3044,7 +3044,7 @@ static void drm_test_check_reject_audio_infoframe(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -3053,7 +3053,7 @@ static void drm_test_check_reject_audio_infoframe(struct kunit *test)
 
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state;
@@ -3082,7 +3082,7 @@ static void drm_test_check_reject_audio_infoframe(struct kunit *test)
 retry_crtc_state_2:
 	crtc_state = drm_atomic_get_crtc_state(state, crtc);
 	if (PTR_ERR(crtc_state) == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state_2;
@@ -3095,7 +3095,7 @@ static void drm_test_check_reject_audio_infoframe(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state_2;
@@ -3104,7 +3104,7 @@ static void drm_test_check_reject_audio_infoframe(struct kunit *test)
 
 	ret = drm_atomic_commit(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_crtc_state_2;
diff --git a/drivers/gpu/drm/tests/drm_kunit_helpers.c b/drivers/gpu/drm/tests/drm_kunit_helpers.c
index 04edb6079c0d4..727766444d638 100644
--- a/drivers/gpu/drm/tests/drm_kunit_helpers.c
+++ b/drivers/gpu/drm/tests/drm_kunit_helpers.c
@@ -82,11 +82,11 @@ __drm_kunit_helper_alloc_drm_device_with_driver(struct kunit *test,
 }
 EXPORT_SYMBOL_GPL(__drm_kunit_helper_alloc_drm_device_with_driver);
 
-static void kunit_action_drm_atomic_state_put(void *ptr)
+static void kunit_action_drm_atomic_commit_put(void *ptr)
 {
-	struct drm_atomic_state *state = ptr;
+	struct drm_atomic_commit *state = ptr;
 
-	drm_atomic_state_put(state);
+	drm_atomic_commit_put(state);
 }
 
 /**
@@ -98,25 +98,25 @@ static void kunit_action_drm_atomic_state_put(void *ptr)
  * Allocates a empty atomic state.
  *
  * The state is tied to the kunit test context, so we must not call
- * drm_atomic_state_put() on it, it will be done so automatically.
+ * drm_atomic_commit_put() on it, it will be done so automatically.
  *
  * Returns:
  * An ERR_PTR on error, a pointer to the newly allocated state otherwise
  */
-struct drm_atomic_state *
+struct drm_atomic_commit *
 drm_kunit_helper_atomic_state_alloc(struct kunit *test,
 				    struct drm_device *drm,
 				    struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	int ret;
 
-	state = drm_atomic_state_alloc(drm);
+	state = drm_atomic_commit_alloc(drm);
 	if (!state)
 		return ERR_PTR(-ENOMEM);
 
 	ret = kunit_add_action_or_reset(test,
-					kunit_action_drm_atomic_state_put,
+					kunit_action_drm_atomic_commit_put,
 					state);
 	if (ret)
 		return ERR_PTR(ret);
@@ -297,7 +297,7 @@ int drm_kunit_helper_enable_crtc_connector(struct kunit *test,
 					   const struct drm_display_mode *mode,
 					   struct drm_modeset_acquire_ctx *ctx)
 {
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_connector_state *conn_state;
 	struct drm_crtc_state *crtc_state;
 	int ret;
diff --git a/drivers/gpu/drm/tests/drm_rect_test.c b/drivers/gpu/drm/tests/drm_rect_test.c
index 17e1f34b76101..3402f993d7d33 100644
--- a/drivers/gpu/drm/tests/drm_rect_test.c
+++ b/drivers/gpu/drm/tests/drm_rect_test.c
@@ -10,6 +10,7 @@
 #include <drm/drm_rect.h>
 #include <drm/drm_mode.h>
 
+#include <linux/limits.h>
 #include <linux/string_helpers.h>
 #include <linux/errno.h>
 
@@ -407,10 +408,27 @@ KUNIT_ARRAY_PARAM(drm_rect_scale, drm_rect_scale_cases, drm_rect_scale_case_desc
 static void drm_test_rect_calc_hscale(struct kunit *test)
 {
 	const struct drm_rect_scale_case *params = test->param_value;
-	int scaling_factor;
+	int expected_warnings = params->expected_scaling_factor == -EINVAL;
+	int scaling_factor = INT_MIN;
 
-	scaling_factor = drm_rect_calc_hscale(&params->src, &params->dst,
-					      params->min_range, params->max_range);
+	/*
+	 * Without CONFIG_BUG, WARN_ON() is a no-op and the suppressed warning
+	 * count stays zero, failing the assertion.
+	 */
+	if (expected_warnings && !IS_ENABLED(CONFIG_BUG))
+		kunit_skip(test, "requires CONFIG_BUG");
+
+	/*
+	 * drm_rect_calc_hscale() generates a warning backtrace whenever bad
+	 * parameters are passed to it. This affects unit tests with -EINVAL
+	 * error code in expected_scaling_factor.
+	 */
+	kunit_warning_suppress(test) {
+		scaling_factor = drm_rect_calc_hscale(&params->src, &params->dst,
+						      params->min_range,
+						      params->max_range);
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, expected_warnings);
+	}
 
 	KUNIT_EXPECT_EQ(test, scaling_factor, params->expected_scaling_factor);
 }
@@ -418,10 +436,26 @@ static void drm_test_rect_calc_hscale(struct kunit *test)
 static void drm_test_rect_calc_vscale(struct kunit *test)
 {
 	const struct drm_rect_scale_case *params = test->param_value;
-	int scaling_factor;
+	int expected_warnings = params->expected_scaling_factor == -EINVAL;
+	int scaling_factor = INT_MIN;
 
-	scaling_factor = drm_rect_calc_vscale(&params->src, &params->dst,
-					      params->min_range, params->max_range);
+	/*
+	 * Without CONFIG_BUG, WARN_ON() is a no-op and the suppressed warning
+	 * count stays zero, failing the assertion.
+	 */
+	if (expected_warnings && !IS_ENABLED(CONFIG_BUG))
+		kunit_skip(test, "requires CONFIG_BUG");
+
+	/*
+	 * drm_rect_calc_vscale() generates a warning backtrace whenever bad
+	 * parameters are passed to it. This affects unit tests with -EINVAL
+	 * error code in expected_scaling_factor.
+	 */
+	kunit_warning_suppress(test) {
+		scaling_factor = drm_rect_calc_vscale(&params->src, &params->dst,
+						      params->min_range, params->max_range);
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, expected_warnings);
+	}
 
 	KUNIT_EXPECT_EQ(test, scaling_factor, params->expected_scaling_factor);
 }
diff --git a/drivers/gpu/drm/tidss/tidss_crtc.c b/drivers/gpu/drm/tidss/tidss_crtc.c
index a31c21c5f855a..acab9307bdf31 100644
--- a/drivers/gpu/drm/tidss/tidss_crtc.c
+++ b/drivers/gpu/drm/tidss/tidss_crtc.c
@@ -83,7 +83,7 @@ void tidss_crtc_error_irq(struct drm_crtc *crtc, u64 irqstatus)
 /* drm_crtc_helper_funcs */
 
 static int tidss_crtc_atomic_check(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -123,7 +123,7 @@ static void tidss_crtc_position_planes(struct tidss_device *tidss,
 				       struct drm_crtc_state *old_state,
 				       bool newmodeset)
 {
-	struct drm_atomic_state *ostate = old_state->state;
+	struct drm_atomic_commit *ostate = old_state->state;
 	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
 	struct drm_crtc_state *cstate = crtc->state;
 	int layer;
@@ -162,7 +162,7 @@ static void tidss_crtc_position_planes(struct tidss_device *tidss,
 }
 
 static void tidss_crtc_atomic_flush(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state = drm_atomic_get_old_crtc_state(state,
 									      crtc);
@@ -211,7 +211,7 @@ static void tidss_crtc_atomic_flush(struct drm_crtc *crtc,
 }
 
 static void tidss_crtc_atomic_enable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 									 crtc);
@@ -260,7 +260,7 @@ static void tidss_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void tidss_crtc_atomic_disable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct tidss_crtc *tcrtc = to_tidss_crtc(crtc);
 	struct drm_device *ddev = crtc->dev;
diff --git a/drivers/gpu/drm/tidss/tidss_encoder.c b/drivers/gpu/drm/tidss/tidss_encoder.c
index 81a04f7677701..34db8d2a3792e 100644
--- a/drivers/gpu/drm/tidss/tidss_encoder.c
+++ b/drivers/gpu/drm/tidss/tidss_encoder.c
@@ -106,6 +106,8 @@ int tidss_encoder_create(struct tidss_device *tidss,
 	enc = &t_enc->encoder;
 	enc->possible_crtcs = possible_crtcs;
 
+	devm_drm_bridge_add(tidss->dev, &t_enc->bridge);
+
 	/* Attaching first bridge to the encoder */
 	ret = drm_bridge_attach(enc, &t_enc->bridge, NULL,
 				DRM_BRIDGE_ATTACH_NO_CONNECTOR);
@@ -121,12 +123,6 @@ int tidss_encoder_create(struct tidss_device *tidss,
 		return PTR_ERR(connector);
 	}
 
-	ret = drm_connector_attach_encoder(connector, enc);
-	if (ret) {
-		dev_err(tidss->dev, "attaching encoder to connector failed\n");
-		return ret;
-	}
-
 	t_enc->connector = connector;
 
 	dev_dbg(tidss->dev, "Encoder create done\n");
diff --git a/drivers/gpu/drm/tidss/tidss_kms.c b/drivers/gpu/drm/tidss/tidss_kms.c
index 8bb93194e5ac6..1512ee2574b66 100644
--- a/drivers/gpu/drm/tidss/tidss_kms.c
+++ b/drivers/gpu/drm/tidss/tidss_kms.c
@@ -19,7 +19,7 @@
 #include "tidss_kms.h"
 #include "tidss_plane.h"
 
-static void tidss_atomic_commit_tail(struct drm_atomic_state *old_state)
+static void tidss_atomic_commit_tail(struct drm_atomic_commit *old_state)
 {
 	struct drm_device *ddev = old_state->dev;
 	struct tidss_device *tidss = to_tidss(ddev);
@@ -67,7 +67,7 @@ static const struct drm_mode_config_helper_funcs mode_config_helper_funcs = {
 };
 
 static int tidss_atomic_check(struct drm_device *ddev,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *opstate;
 	struct drm_plane_state *npstate;
@@ -287,8 +287,6 @@ int tidss_modeset_init(struct tidss_device *tidss)
 	if (ret)
 		return ret;
 
-	drm_mode_config_reset(ddev);
-
 	dev_dbg(tidss->dev, "%s done\n", __func__);
 
 	return 0;
diff --git a/drivers/gpu/drm/tidss/tidss_oldi.c b/drivers/gpu/drm/tidss/tidss_oldi.c
index 17c535bfa0576..9c9725aac8290 100644
--- a/drivers/gpu/drm/tidss/tidss_oldi.c
+++ b/drivers/gpu/drm/tidss/tidss_oldi.c
@@ -221,7 +221,7 @@ static int tidss_oldi_config(struct tidss_oldi *oldi)
 }
 
 static void tidss_oldi_atomic_pre_enable(struct drm_bridge *bridge,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct tidss_oldi *oldi = drm_bridge_to_tidss_oldi(bridge);
 	struct drm_connector *connector;
@@ -258,7 +258,7 @@ static void tidss_oldi_atomic_pre_enable(struct drm_bridge *bridge,
 }
 
 static void tidss_oldi_atomic_post_disable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct tidss_oldi *oldi = drm_bridge_to_tidss_oldi(bridge);
 
diff --git a/drivers/gpu/drm/tidss/tidss_plane.c b/drivers/gpu/drm/tidss/tidss_plane.c
index aaa02c851c595..1a8b44fb45f8f 100644
--- a/drivers/gpu/drm/tidss/tidss_plane.c
+++ b/drivers/gpu/drm/tidss/tidss_plane.c
@@ -29,7 +29,7 @@ void tidss_plane_error_irq(struct drm_plane *plane, u64 irqstatus)
 /* drm_plane_helper_funcs */
 
 static int tidss_plane_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -113,7 +113,7 @@ static int tidss_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void tidss_plane_atomic_update(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_device *ddev = plane->dev;
 	struct tidss_device *tidss = to_tidss(ddev);
@@ -133,7 +133,7 @@ static void tidss_plane_atomic_update(struct drm_plane *plane,
 }
 
 static void tidss_plane_atomic_enable(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_device *ddev = plane->dev;
 	struct tidss_device *tidss = to_tidss(ddev);
@@ -143,7 +143,7 @@ static void tidss_plane_atomic_enable(struct drm_plane *plane,
 }
 
 static void tidss_plane_atomic_disable(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_device *ddev = plane->dev;
 	struct tidss_device *tidss = to_tidss(ddev);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index 4d3b7059cd5b2..e07ecb36a27a8 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -453,7 +453,7 @@ static void tilcdc_crtc_enable(struct drm_crtc *crtc)
 }
 
 static void tilcdc_crtc_atomic_enable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	tilcdc_crtc_enable(crtc);
 }
@@ -510,13 +510,13 @@ static void tilcdc_crtc_disable(struct drm_crtc *crtc)
 }
 
 static void tilcdc_crtc_atomic_disable(struct drm_crtc *crtc,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	tilcdc_crtc_disable(crtc);
 }
 
 static void tilcdc_crtc_atomic_flush(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	if (!crtc->state->event)
 		return;
@@ -632,7 +632,7 @@ static bool tilcdc_crtc_mode_fixup(struct drm_crtc *crtc,
 }
 
 static int tilcdc_crtc_atomic_check(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -926,7 +926,7 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 				drm_err(dev,
 					"%s(0x%08x): Sync lost flood detected, recovering",
 					__func__, stat);
-				queue_work(system_wq,
+				queue_work(system_percpu_wq,
 					   &tilcdc_crtc->recover_work);
 				tilcdc_write(dev, LCDC_INT_ENABLE_CLR_REG,
 					     LCDC_SYNC_LOST);
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index c877b2be9c2ec..e7f675c15c20c 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -49,7 +49,7 @@ static const u32 tilcdc_legacy_formats[] = { DRM_FORMAT_RGB565,
 					     DRM_FORMAT_XRGB8888 };
 
 static int tilcdc_atomic_check(struct drm_device *dev,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	int ret;
 
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_encoder.c b/drivers/gpu/drm/tilcdc/tilcdc_encoder.c
index 680a2ac6ab594..ac4b5beb2dd58 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_encoder.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_encoder.c
@@ -35,12 +35,6 @@ int tilcdc_attach_bridge(struct drm_device *ddev, struct drm_bridge *bridge)
 		return PTR_ERR(connector);
 	}
 
-	ret = drm_connector_attach_encoder(connector, &priv->encoder->base);
-	if (ret) {
-		drm_err(ddev, "attaching encoder to connector failed\n");
-		return ret;
-	}
-
 	priv->connector = connector;
 	return 0;
 }
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_plane.c b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
index a9982a9956903..d2678af5dc22c 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_plane.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_plane.c
@@ -20,7 +20,7 @@ static const struct drm_plane_funcs tilcdc_plane_funcs = {
 };
 
 static int tilcdc_plane_atomic_check(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -74,7 +74,7 @@ static int tilcdc_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void tilcdc_plane_atomic_update(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/tiny/appletbdrm.c b/drivers/gpu/drm/tiny/appletbdrm.c
index 278bb23fe4c8d..cdd35af498920 100644
--- a/drivers/gpu/drm/tiny/appletbdrm.c
+++ b/drivers/gpu/drm/tiny/appletbdrm.c
@@ -316,7 +316,7 @@ static const u32 appletbdrm_primary_plane_formats[] = {
 };
 
 static int appletbdrm_primary_plane_helper_atomic_check(struct drm_plane *plane,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_plane_state *old_plane_state = drm_atomic_get_old_plane_state(state, plane);
@@ -468,7 +468,7 @@ static int appletbdrm_flush_damage(struct appletbdrm_device *adev,
 }
 
 static void appletbdrm_primary_plane_helper_atomic_update(struct drm_plane *plane,
-						     struct drm_atomic_state *old_state)
+						     struct drm_atomic_commit *old_state)
 {
 	struct appletbdrm_device *adev = drm_to_adev(plane->dev);
 	struct drm_device *drm = plane->dev;
@@ -485,7 +485,7 @@ static void appletbdrm_primary_plane_helper_atomic_update(struct drm_plane *plan
 }
 
 static void appletbdrm_primary_plane_helper_atomic_disable(struct drm_plane *plane,
-							   struct drm_atomic_state *state)
+							   struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = plane->dev;
 	struct appletbdrm_device *adev = drm_to_adev(dev);
diff --git a/drivers/gpu/drm/tiny/bochs.c b/drivers/gpu/drm/tiny/bochs.c
index 5d8dc5efec776..e2d957e51505c 100644
--- a/drivers/gpu/drm/tiny/bochs.c
+++ b/drivers/gpu/drm/tiny/bochs.c
@@ -419,7 +419,7 @@ static const uint32_t bochs_primary_plane_formats[] = {
 };
 
 static int bochs_primary_plane_helper_atomic_check(struct drm_plane *plane,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_crtc *new_crtc = new_plane_state->crtc;
@@ -442,7 +442,7 @@ static int bochs_primary_plane_helper_atomic_check(struct drm_plane *plane,
 }
 
 static void bochs_primary_plane_helper_atomic_update(struct drm_plane *plane,
-						     struct drm_atomic_state *state)
+						     struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = plane->dev;
 	struct bochs_device *bochs = to_bochs_device(dev);
@@ -513,7 +513,7 @@ static void bochs_crtc_helper_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static int bochs_crtc_helper_atomic_check(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 
@@ -524,7 +524,7 @@ static int bochs_crtc_helper_atomic_check(struct drm_crtc *crtc,
 }
 
 static void bochs_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct bochs_device *bochs = to_bochs_device(crtc->dev);
 
@@ -533,7 +533,7 @@ static void bochs_crtc_helper_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void bochs_crtc_helper_atomic_disable(struct drm_crtc *crtc,
-					     struct drm_atomic_state *crtc_state)
+					     struct drm_atomic_commit *crtc_state)
 {
 	struct bochs_device *bochs = to_bochs_device(crtc->dev);
 
diff --git a/drivers/gpu/drm/tiny/cirrus-qemu.c b/drivers/gpu/drm/tiny/cirrus-qemu.c
index 9ba0eab489bb3..075221b431d37 100644
--- a/drivers/gpu/drm/tiny/cirrus-qemu.c
+++ b/drivers/gpu/drm/tiny/cirrus-qemu.c
@@ -296,7 +296,7 @@ static const uint64_t cirrus_primary_plane_format_modifiers[] = {
 };
 
 static int cirrus_primary_plane_helper_atomic_check(struct drm_plane *plane,
-						    struct drm_atomic_state *state)
+						    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_framebuffer *fb = new_plane_state->fb;
@@ -326,7 +326,7 @@ static int cirrus_primary_plane_helper_atomic_check(struct drm_plane *plane,
 }
 
 static void cirrus_primary_plane_helper_atomic_update(struct drm_plane *plane,
-						      struct drm_atomic_state *state)
+						      struct drm_atomic_commit *state)
 {
 	struct cirrus_device *cirrus = to_cirrus(plane->dev);
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
@@ -374,7 +374,7 @@ static const struct drm_plane_funcs cirrus_primary_plane_funcs = {
 	DRM_GEM_SHADOW_PLANE_FUNCS,
 };
 
-static int cirrus_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
+static int cirrus_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	int ret;
@@ -390,7 +390,7 @@ static int cirrus_crtc_helper_atomic_check(struct drm_crtc *crtc, struct drm_ato
 }
 
 static void cirrus_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct cirrus_device *cirrus = to_cirrus(crtc->dev);
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
diff --git a/drivers/gpu/drm/tiny/hx8357d.c b/drivers/gpu/drm/tiny/hx8357d.c
index 53b152fc6b048..f942a8d09ff22 100644
--- a/drivers/gpu/drm/tiny/hx8357d.c
+++ b/drivers/gpu/drm/tiny/hx8357d.c
@@ -78,7 +78,7 @@ static const struct drm_plane_funcs hx8357d_plane_funcs = {
 };
 
 static void hx8357d_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct hx8357d_device *hx8357d = to_hx8357d_device(drm);
diff --git a/drivers/gpu/drm/tiny/ili9163.c b/drivers/gpu/drm/tiny/ili9163.c
index 1e581576fc7e0..8842424504e3f 100644
--- a/drivers/gpu/drm/tiny/ili9163.c
+++ b/drivers/gpu/drm/tiny/ili9163.c
@@ -67,7 +67,7 @@ static const struct drm_plane_funcs ili9163_plane_funcs = {
 };
 
 static void ili9163_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct ili9163_device *ili9163 = to_ili9163_device(drm);
diff --git a/drivers/gpu/drm/tiny/ili9225.c b/drivers/gpu/drm/tiny/ili9225.c
index 0371527002f92..5bf52a8fd75b2 100644
--- a/drivers/gpu/drm/tiny/ili9225.c
+++ b/drivers/gpu/drm/tiny/ili9225.c
@@ -179,7 +179,7 @@ static const u64 ili9225_plane_format_modifiers[] = {
 };
 
 static void ili9225_plane_helper_atomic_update(struct drm_plane *plane,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = plane->dev;
 	struct drm_plane_state *plane_state = plane->state;
@@ -214,7 +214,7 @@ static const struct drm_plane_funcs ili9225_plane_funcs = {
 };
 
 static void ili9225_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct ili9225_device *ili9225 = to_ili9225_device(drm);
@@ -316,7 +316,7 @@ static void ili9225_crtc_helper_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void ili9225_crtc_helper_atomic_disable(struct drm_crtc *crtc,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct ili9225_device *ili9225 = to_ili9225_device(drm);
diff --git a/drivers/gpu/drm/tiny/ili9341.c b/drivers/gpu/drm/tiny/ili9341.c
index e623bd8a1827e..003381aa22ea4 100644
--- a/drivers/gpu/drm/tiny/ili9341.c
+++ b/drivers/gpu/drm/tiny/ili9341.c
@@ -84,7 +84,7 @@ static const struct drm_plane_funcs ili9341_plane_funcs = {
 };
 
 static void ili9341_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct ili9341_device *ili9341 = to_ili9341_device(drm);
diff --git a/drivers/gpu/drm/tiny/ili9486.c b/drivers/gpu/drm/tiny/ili9486.c
index 030e87a8a3f0e..c945a91443158 100644
--- a/drivers/gpu/drm/tiny/ili9486.c
+++ b/drivers/gpu/drm/tiny/ili9486.c
@@ -126,7 +126,7 @@ static const struct drm_plane_funcs ili9486_plane_funcs = {
 };
 
 static void ili9486_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct ili9486_device *ili9486 = to_ili9486_device(drm);
diff --git a/drivers/gpu/drm/tiny/mi0283qt.c b/drivers/gpu/drm/tiny/mi0283qt.c
index ee597d0233905..b0121e89e2dc0 100644
--- a/drivers/gpu/drm/tiny/mi0283qt.c
+++ b/drivers/gpu/drm/tiny/mi0283qt.c
@@ -82,7 +82,7 @@ static const struct drm_plane_funcs mi0283qt_plane_funcs = {
 };
 
 static void mi0283qt_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct mi0283qt_device *mi0283qt = to_mi0283qt_device(drm);
diff --git a/drivers/gpu/drm/tiny/panel-mipi-dbi.c b/drivers/gpu/drm/tiny/panel-mipi-dbi.c
index 581e14528608b..60cd65cae41d8 100644
--- a/drivers/gpu/drm/tiny/panel-mipi-dbi.c
+++ b/drivers/gpu/drm/tiny/panel-mipi-dbi.c
@@ -261,7 +261,7 @@ static const struct drm_plane_funcs panel_mipi_dbi_plane_funcs = {
 };
 
 static void panel_mipi_dbi_crtc_helper_atomic_enable(struct drm_crtc *crtc,
-						     struct drm_atomic_state *state)
+						     struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = crtc->dev;
 	struct panel_mipi_dbi_device *panel_mipi_dbi = to_panel_mipi_dbi_device(drm);
diff --git a/drivers/gpu/drm/tiny/pixpaper.c b/drivers/gpu/drm/tiny/pixpaper.c
index df3ec42edd570..d02ac26d007c1 100644
--- a/drivers/gpu/drm/tiny/pixpaper.c
+++ b/drivers/gpu/drm/tiny/pixpaper.c
@@ -774,7 +774,7 @@ static u8 pack_pixels_to_byte(__le32 *src_pixels, int i, int j,
 }
 
 static int pixpaper_plane_helper_atomic_check(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state =
 		drm_atomic_get_new_plane_state(state, plane);
@@ -797,7 +797,7 @@ static int pixpaper_plane_helper_atomic_check(struct drm_plane *plane,
 }
 
 static int pixpaper_crtc_helper_atomic_check(struct drm_crtc *crtc,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state =
 		drm_atomic_get_new_crtc_state(state, crtc);
@@ -809,7 +809,7 @@ static int pixpaper_crtc_helper_atomic_check(struct drm_crtc *crtc,
 }
 
 static void pixpaper_crtc_atomic_enable(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct pixpaper_panel *panel = to_pixpaper_panel(crtc->dev);
 	struct drm_device *drm = &panel->drm;
@@ -834,7 +834,7 @@ static void pixpaper_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void pixpaper_crtc_atomic_disable(struct drm_crtc *crtc,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct pixpaper_panel *panel = to_pixpaper_panel(crtc->dev);
 	struct drm_device *drm = &panel->drm;
@@ -858,7 +858,7 @@ static void pixpaper_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void pixpaper_plane_atomic_update(struct drm_plane *plane,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state =
 		drm_atomic_get_new_plane_state(state, plane);
diff --git a/drivers/gpu/drm/tiny/sharp-memory.c b/drivers/gpu/drm/tiny/sharp-memory.c
index cbf69460ebf32..506e6432e70df 100644
--- a/drivers/gpu/drm/tiny/sharp-memory.c
+++ b/drivers/gpu/drm/tiny/sharp-memory.c
@@ -223,7 +223,7 @@ static void sharp_memory_fb_dirty(struct drm_framebuffer *fb, const struct iosys
 }
 
 static int sharp_memory_plane_atomic_check(struct drm_plane *plane,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct sharp_memory_device *smd;
@@ -239,7 +239,7 @@ static int sharp_memory_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void sharp_memory_plane_atomic_update(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_plane_state *plane_state = plane->state;
@@ -287,7 +287,7 @@ static enum drm_mode_status sharp_memory_crtc_mode_valid(struct drm_crtc *crtc,
 }
 
 static int sharp_memory_crtc_check(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	int ret;
@@ -317,7 +317,7 @@ static int sharp_memory_sw_vcom_signal_thread(void *data)
 }
 
 static void sharp_memory_crtc_enable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct sharp_memory_device *smd = drm_to_sharp_memory_device(crtc->dev);
 
@@ -328,7 +328,7 @@ static void sharp_memory_crtc_enable(struct drm_crtc *crtc,
 }
 
 static void sharp_memory_crtc_disable(struct drm_crtc *crtc,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct sharp_memory_device *smd = drm_to_sharp_memory_device(crtc->dev);
 
diff --git a/drivers/gpu/drm/tve200/tve200_drv.c b/drivers/gpu/drm/tve200/tve200_drv.c
index a048e37f1c2c1..562f3f11812a3 100644
--- a/drivers/gpu/drm/tve200/tve200_drv.c
+++ b/drivers/gpu/drm/tve200/tve200_drv.c
@@ -221,12 +221,16 @@ static int tve200_probe(struct platform_device *pdev)
 
 	ret = drm_dev_register(drm, 0);
 	if (ret < 0)
-		goto clk_disable;
+		goto mode_config_cleanup;
 
 	drm_client_setup_with_fourcc(drm, DRM_FORMAT_RGB565);
 
 	return 0;
 
+mode_config_cleanup:
+	if (priv->panel)
+		drm_panel_bridge_remove(priv->bridge);
+	drm_mode_config_cleanup(drm);
 clk_disable:
 	clk_disable_unprepare(priv->pclk);
 dev_unref:
diff --git a/drivers/gpu/drm/tyr/Kconfig b/drivers/gpu/drm/tyr/Kconfig
index e933e64780272..51a68ef8212ce 100644
--- a/drivers/gpu/drm/tyr/Kconfig
+++ b/drivers/gpu/drm/tyr/Kconfig
@@ -8,6 +8,7 @@ config DRM_TYR
 	depends on !GENERIC_ATOMIC64  # for IOMMU_IO_PGTABLE_LPAE
 	depends on COMMON_CLK
 	default n
+	select RUST_DRM_GEM_SHMEM_HELPER
 	help
 	  Rust DRM driver for ARM Mali CSF-based GPUs.
 
diff --git a/drivers/gpu/drm/tyr/driver.rs b/drivers/gpu/drm/tyr/driver.rs
index 279710b36a104..e20a5978eed6d 100644
--- a/drivers/gpu/drm/tyr/driver.rs
+++ b/drivers/gpu/drm/tyr/driver.rs
@@ -11,9 +11,16 @@
         Device, //
     },
     devres::Devres,
+    dma::{
+        Device as DmaDevice,
+        DmaMask, //
+    },
     drm,
     drm::ioctl,
-    io::poll,
+    io::{
+        poll,
+        Io, //
+    },
     new_mutex,
     of,
     platform,
@@ -31,10 +38,10 @@
 
 use crate::{
     file::TyrDrmFileData,
-    gem::TyrObject,
+    gem::BoData,
     gpu,
     gpu::GpuInfo,
-    regs, //
+    regs::gpu_control::*, //
 };
 
 pub(crate) type IoMem = kernel::io::mem::IoMem<SZ_2M>;
@@ -49,7 +56,7 @@ pub(crate) struct TyrPlatformDriverData {
     _device: ARef<TyrDrmDevice>,
 }
 
-#[pin_data(PinnedDrop)]
+#[pin_data]
 pub(crate) struct TyrDrmDeviceData {
     pub(crate) pdev: ARef<platform::Device>,
 
@@ -66,11 +73,15 @@ pub(crate) struct TyrDrmDeviceData {
 }
 
 fn issue_soft_reset(dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result {
-    regs::GPU_CMD.write(dev, iomem, regs::GPU_CMD_SOFT_RESET)?;
+    let io = (*iomem).access(dev)?;
+    io.write_reg(GPU_COMMAND::reset(ResetMode::SoftReset));
 
     poll::read_poll_timeout(
-        || regs::GPU_IRQ_RAWSTAT.read(dev, iomem),
-        |status| *status & regs::GPU_IRQ_RAWSTAT_RESET_COMPLETED != 0,
+        || {
+            let io = (*iomem).access(dev)?;
+            Ok(io.read(GPU_IRQ_RAWSTAT))
+        },
+        |status| status.reset_completed(),
         time::Delta::from_millis(1),
         time::Delta::from_millis(100),
     )
@@ -115,7 +126,15 @@ fn probe(
         gpu::l2_power_on(pdev.as_ref(), &iomem)?;
 
         let gpu_info = GpuInfo::new(pdev.as_ref(), &iomem)?;
-        gpu_info.log(pdev);
+        gpu_info.log(pdev.as_ref());
+
+        let pa_bits = MMU_FEATURES::from_raw(gpu_info.mmu_features)
+            .pa_bits()
+            .get();
+        // SAFETY: No concurrent DMA allocations or mappings can be made because
+        // the device is still being probed and therefore isn't being used by
+        // other threads of execution.
+        unsafe { pdev.dma_set_mask_and_coherent(DmaMask::try_new(pa_bits)?)? };
 
         let platform: ARef<platform::Device> = pdev.into();
 
@@ -150,17 +169,6 @@ impl PinnedDrop for TyrPlatformDriverData {
     fn drop(self: Pin<&mut Self>) {}
 }
 
-#[pinned_drop]
-impl PinnedDrop for TyrDrmDeviceData {
-    fn drop(self: Pin<&mut Self>) {
-        // TODO: the type-state pattern for Clks will fix this.
-        let clks = self.clks.lock();
-        clks.core.disable_unprepare();
-        clks.stacks.disable_unprepare();
-        clks.coregroup.disable_unprepare();
-    }
-}
-
 // We need to retain the name "panthor" to achieve drop-in compatibility with
 // the C driver in the userspace stack.
 const INFO: drm::DriverInfo = drm::DriverInfo {
@@ -175,7 +183,7 @@ fn drop(self: Pin<&mut Self>) {
 impl drm::Driver for TyrDrmDriver {
     type Data = TyrDrmDeviceData;
     type File = TyrDrmFileData;
-    type Object = drm::gem::Object<TyrObject>;
+    type Object = drm::gem::shmem::Object<BoData>;
 
     const INFO: drm::DriverInfo = INFO;
 
@@ -184,14 +192,20 @@ impl drm::Driver for TyrDrmDriver {
     }
 }
 
-#[pin_data]
 struct Clocks {
     core: Clk,
     stacks: OptionalClk,
     coregroup: OptionalClk,
 }
 
-#[pin_data]
+impl Drop for Clocks {
+    fn drop(&mut self) {
+        self.core.disable_unprepare();
+        self.stacks.disable_unprepare();
+        self.coregroup.disable_unprepare();
+    }
+}
+
 struct Regulators {
     _mali: Regulator<regulator::Enabled>,
     _sram: Regulator<regulator::Enabled>,
diff --git a/drivers/gpu/drm/tyr/gem.rs b/drivers/gpu/drm/tyr/gem.rs
index 5cc6eb0b5d3fa..1640a161754b2 100644
--- a/drivers/gpu/drm/tyr/gem.rs
+++ b/drivers/gpu/drm/tyr/gem.rs
@@ -1,4 +1,8 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
+//! GEM buffer object management for the Tyr driver.
+//!
+//! This module provides buffer object (BO) management functionality using
+//! DRM's GEM subsystem with shmem backing.
 
 use kernel::{
     drm::gem,
@@ -10,15 +14,23 @@
     TyrDrmDriver, //
 };
 
-/// GEM Object inner driver data
+/// Tyr's DriverObject type for GEM objects.
 #[pin_data]
-pub(crate) struct TyrObject {}
+pub(crate) struct BoData {
+    flags: u32,
+}
+
+/// Provides a way to pass arguments when creating BoData
+/// as required by the gem::DriverObject trait.
+pub(crate) struct BoCreateArgs {
+    flags: u32,
+}
 
-impl gem::DriverObject for TyrObject {
+impl gem::DriverObject for BoData {
     type Driver = TyrDrmDriver;
-    type Args = ();
+    type Args = BoCreateArgs;
 
-    fn new(_dev: &TyrDrmDevice, _size: usize, _args: ()) -> impl PinInit<Self, Error> {
-        try_pin_init!(TyrObject {})
+    fn new(_dev: &TyrDrmDevice, _size: usize, args: BoCreateArgs) -> impl PinInit<Self, Error> {
+        try_pin_init!(Self { flags: args.flags })
     }
 }
diff --git a/drivers/gpu/drm/tyr/gpu.rs b/drivers/gpu/drm/tyr/gpu.rs
index a88775160f981..652556026f50d 100644
--- a/drivers/gpu/drm/tyr/gpu.rs
+++ b/drivers/gpu/drm/tyr/gpu.rs
@@ -5,14 +5,16 @@
     DerefMut, //
 };
 use kernel::{
-    bits::genmask_u32,
     device::{
         Bound,
         Device, //
     },
     devres::Devres,
-    io::poll,
-    platform,
+    io::{
+        poll,
+        register::Array,
+        Io, //
+    },
     prelude::*,
     time::Delta,
     transmute::AsBytes,
@@ -21,7 +23,10 @@
 
 use crate::{
     driver::IoMem,
-    regs, //
+    regs::{
+        gpu_control::*,
+        join_u64, //
+    }, //
 };
 
 /// Struct containing information that can be queried by userspace. This is read from
@@ -29,120 +34,95 @@
 ///
 /// # Invariants
 ///
-/// - The layout of this struct identical to the C `struct drm_panthor_gpu_info`.
+/// - The layout of this struct is identical to the C `struct drm_panthor_gpu_info`.
 #[repr(transparent)]
 #[derive(Clone, Copy)]
 pub(crate) struct GpuInfo(pub(crate) uapi::drm_panthor_gpu_info);
 
 impl GpuInfo {
     pub(crate) fn new(dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result<Self> {
-        let gpu_id = regs::GPU_ID.read(dev, iomem)?;
-        let csf_id = regs::GPU_CSF_ID.read(dev, iomem)?;
-        let gpu_rev = regs::GPU_REVID.read(dev, iomem)?;
-        let core_features = regs::GPU_CORE_FEATURES.read(dev, iomem)?;
-        let l2_features = regs::GPU_L2_FEATURES.read(dev, iomem)?;
-        let tiler_features = regs::GPU_TILER_FEATURES.read(dev, iomem)?;
-        let mem_features = regs::GPU_MEM_FEATURES.read(dev, iomem)?;
-        let mmu_features = regs::GPU_MMU_FEATURES.read(dev, iomem)?;
-        let thread_features = regs::GPU_THREAD_FEATURES.read(dev, iomem)?;
-        let max_threads = regs::GPU_THREAD_MAX_THREADS.read(dev, iomem)?;
-        let thread_max_workgroup_size = regs::GPU_THREAD_MAX_WORKGROUP_SIZE.read(dev, iomem)?;
-        let thread_max_barrier_size = regs::GPU_THREAD_MAX_BARRIER_SIZE.read(dev, iomem)?;
-        let coherency_features = regs::GPU_COHERENCY_FEATURES.read(dev, iomem)?;
-
-        let texture_features = regs::GPU_TEXTURE_FEATURES0.read(dev, iomem)?;
-
-        let as_present = regs::GPU_AS_PRESENT.read(dev, iomem)?;
-
-        let shader_present = u64::from(regs::GPU_SHADER_PRESENT_LO.read(dev, iomem)?);
-        let shader_present =
-            shader_present | u64::from(regs::GPU_SHADER_PRESENT_HI.read(dev, iomem)?) << 32;
-
-        let tiler_present = u64::from(regs::GPU_TILER_PRESENT_LO.read(dev, iomem)?);
-        let tiler_present =
-            tiler_present | u64::from(regs::GPU_TILER_PRESENT_HI.read(dev, iomem)?) << 32;
-
-        let l2_present = u64::from(regs::GPU_L2_PRESENT_LO.read(dev, iomem)?);
-        let l2_present = l2_present | u64::from(regs::GPU_L2_PRESENT_HI.read(dev, iomem)?) << 32;
+        let io = (*iomem).access(dev)?;
 
         Ok(Self(uapi::drm_panthor_gpu_info {
-            gpu_id,
-            gpu_rev,
-            csf_id,
-            l2_features,
-            tiler_features,
-            mem_features,
-            mmu_features,
-            thread_features,
-            max_threads,
-            thread_max_workgroup_size,
-            thread_max_barrier_size,
-            coherency_features,
-            // TODO: Add texture_features_{1,2,3}.
-            texture_features: [texture_features, 0, 0, 0],
-            as_present,
+            gpu_id: io.read(GPU_ID).into_raw(),
+            gpu_rev: io.read(REVIDR).into_raw(),
+            csf_id: io.read(CSF_ID).into_raw(),
+            l2_features: io.read(L2_FEATURES).into_raw(),
+            tiler_features: io.read(TILER_FEATURES).into_raw(),
+            mem_features: io.read(MEM_FEATURES).into_raw(),
+            mmu_features: io.read(MMU_FEATURES).into_raw(),
+            thread_features: io.read(THREAD_FEATURES).into_raw(),
+            max_threads: io.read(THREAD_MAX_THREADS).into_raw(),
+            thread_max_workgroup_size: io.read(THREAD_MAX_WORKGROUP_SIZE).into_raw(),
+            thread_max_barrier_size: io.read(THREAD_MAX_BARRIER_SIZE).into_raw(),
+            coherency_features: io.read(COHERENCY_FEATURES).into_raw(),
+            texture_features: [
+                io.read(TEXTURE_FEATURES::at(0)).supported_formats().get(),
+                io.read(TEXTURE_FEATURES::at(1)).supported_formats().get(),
+                io.read(TEXTURE_FEATURES::at(2)).supported_formats().get(),
+                io.read(TEXTURE_FEATURES::at(3)).supported_formats().get(),
+            ],
+            as_present: io.read(AS_PRESENT).into_raw(),
             selected_coherency: uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE,
-            shader_present,
-            l2_present,
-            tiler_present,
-            core_features,
+            shader_present: join_u64(
+                io.read(SHADER_PRESENT_LO).into_raw(),
+                io.read(SHADER_PRESENT_HI).into_raw(),
+            ),
+            l2_present: join_u64(
+                io.read(L2_PRESENT_LO).into_raw(),
+                io.read(L2_PRESENT_HI).into_raw(),
+            ),
+            tiler_present: join_u64(
+                io.read(TILER_PRESENT_LO).into_raw(),
+                io.read(TILER_PRESENT_HI).into_raw(),
+            ),
+            core_features: io.read(CORE_FEATURES).into_raw(),
+            // Padding must be zero.
             pad: 0,
+            //GPU_FEATURES register is not available; it was introduced in arch 11.x.
             gpu_features: 0,
         }))
     }
 
-    pub(crate) fn log(&self, pdev: &platform::Device) {
-        let gpu_id = GpuId::from(self.gpu_id);
+    pub(crate) fn log(&self, dev: &Device<Bound>) {
+        let gpu_id = GPU_ID::from_raw(self.gpu_id);
 
-        let model_name = if let Some(model) = GPU_MODELS
-            .iter()
-            .find(|&f| f.arch_major == gpu_id.arch_major && f.prod_major == gpu_id.prod_major)
-        {
+        let model_name = if let Some(model) = GPU_MODELS.iter().find(|&f| {
+            f.arch_major == gpu_id.arch_major().get() && f.prod_major == gpu_id.prod_major().get()
+        }) {
             model.name
         } else {
             "unknown"
         };
 
         dev_info!(
-            pdev,
-            "mali-{} id 0x{:x} major 0x{:x} minor 0x{:x} status 0x{:x}",
+            dev,
+            "mali-{} GPU_ID 0x{:x} major 0x{:x} minor 0x{:x} status 0x{:x}",
             model_name,
-            self.gpu_id >> 16,
-            gpu_id.ver_major,
-            gpu_id.ver_minor,
-            gpu_id.ver_status
+            gpu_id.into_raw(),
+            gpu_id.ver_major().get(),
+            gpu_id.ver_minor().get(),
+            gpu_id.ver_status().get()
         );
 
         dev_info!(
-            pdev,
+            dev,
             "Features: L2:{:#x} Tiler:{:#x} Mem:{:#x} MMU:{:#x} AS:{:#x}",
             self.l2_features,
             self.tiler_features,
             self.mem_features,
             self.mmu_features,
-            self.as_present
+            self.as_present,
         );
 
         dev_info!(
-            pdev,
+            dev,
             "shader_present=0x{:016x} l2_present=0x{:016x} tiler_present=0x{:016x}",
             self.shader_present,
             self.l2_present,
-            self.tiler_present
+            self.tiler_present,
         );
     }
-
-    /// Returns the number of virtual address bits supported by the GPU.
-    #[expect(dead_code)]
-    pub(crate) fn va_bits(&self) -> u32 {
-        self.mmu_features & genmask_u32(0..=7)
-    }
-
-    /// Returns the number of physical address bits supported by the GPU.
-    #[expect(dead_code)]
-    pub(crate) fn pa_bits(&self) -> u32 {
-        (self.mmu_features >> 8) & genmask_u32(0..=7)
-    }
 }
 
 impl Deref for GpuInfo {
@@ -182,38 +162,17 @@ struct GpuModels {
     prod_major: 7,
 }];
 
-#[allow(dead_code)]
-pub(crate) struct GpuId {
-    pub(crate) arch_major: u32,
-    pub(crate) arch_minor: u32,
-    pub(crate) arch_rev: u32,
-    pub(crate) prod_major: u32,
-    pub(crate) ver_major: u32,
-    pub(crate) ver_minor: u32,
-    pub(crate) ver_status: u32,
-}
-
-impl From<u32> for GpuId {
-    fn from(value: u32) -> Self {
-        GpuId {
-            arch_major: (value & genmask_u32(28..=31)) >> 28,
-            arch_minor: (value & genmask_u32(24..=27)) >> 24,
-            arch_rev: (value & genmask_u32(20..=23)) >> 20,
-            prod_major: (value & genmask_u32(16..=19)) >> 16,
-            ver_major: (value & genmask_u32(12..=15)) >> 12,
-            ver_minor: (value & genmask_u32(4..=11)) >> 4,
-            ver_status: value & genmask_u32(0..=3),
-        }
-    }
-}
-
 /// Powers on the l2 block.
 pub(crate) fn l2_power_on(dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result {
-    regs::L2_PWRON_LO.write(dev, iomem, 1)?;
+    let io = (*iomem).access(dev)?;
+    io.write_reg(L2_PWRON_LO::zeroed().with_const_request::<1>());
 
     poll::read_poll_timeout(
-        || regs::L2_READY_LO.read(dev, iomem),
-        |status| *status == 1,
+        || {
+            let io = (*iomem).access(dev)?;
+            Ok(io.read(L2_READY_LO))
+        },
+        |status| status.ready() == 1,
         Delta::from_millis(1),
         Delta::from_millis(100),
     )
diff --git a/drivers/gpu/drm/tyr/regs.rs b/drivers/gpu/drm/tyr/regs.rs
index 611870c2e6af5..562023e5df2f3 100644
--- a/drivers/gpu/drm/tyr/regs.rs
+++ b/drivers/gpu/drm/tyr/regs.rs
@@ -1,5 +1,25 @@
 // SPDX-License-Identifier: GPL-2.0 or MIT
 
+//! # Definitions
+//!
+//! - **CEU**: Command Execution Unit - A hardware component that executes commands (instructions)
+//!   from the command stream.
+//! - **CS**: Command Stream - A sequence of instructions (commands) used to control a particular
+//!   job or sequence of jobs. The instructions exist in one or more command buffers.
+//! - **CSF**: Command Stream Frontend - The interface and implementation for job submission
+//!   exposed to the host CPU driver. This includes the global interface, as well as CSG and CS
+//!   interfaces.
+//! - **CSG**: Command Stream Group - A group of related command streams. The CSF manages multiple
+//!   CSGs, and each CSG contains multiple CSs.
+//! - **CSHW**: Command Stream Hardware - The hardware interpreting command streams, including the
+//!   iterator control aspects. Implements the CSF in conjunction with the MCU.
+//! - **GLB**: Global - Prefix for global interface registers that control operations common to
+//!   all CSs.
+//! - **JASID**: Job Address Space ID - Identifies the address space for a job.
+//! - **MCU**: Microcontroller Unit - Implements the CSF in conjunction with the command stream
+//!   hardware.
+//! - **MMU**: Memory Management Unit - Handles address translation and memory access protection.
+
 // We don't expect that all the registers and fields will be used, even in the
 // future.
 //
@@ -7,107 +27,1630 @@
 // does.
 #![allow(dead_code)]
 
-use kernel::{
-    bits::bit_u32,
-    device::{
-        Bound,
-        Device, //
-    },
-    devres::Devres,
-    io::Io,
-    prelude::*, //
-};
-
-use crate::driver::IoMem;
-
-/// Represents a register in the Register Set
-///
-/// TODO: Replace this with the Nova `register!()` macro when it is available.
-/// In particular, this will automatically give us 64bit register reads and
-/// writes.
-pub(crate) struct Register<const OFFSET: usize>;
-
-impl<const OFFSET: usize> Register<OFFSET> {
-    #[inline]
-    pub(crate) fn read(&self, dev: &Device<Bound>, iomem: &Devres<IoMem>) -> Result<u32> {
-        let value = (*iomem).access(dev)?.read32(OFFSET);
-        Ok(value)
-    }
-
-    #[inline]
-    pub(crate) fn write(&self, dev: &Device<Bound>, iomem: &Devres<IoMem>, value: u32) -> Result {
-        (*iomem).access(dev)?.write32(value, OFFSET);
-        Ok(())
+/// Combine two 32-bit values into a single 64-bit value.
+pub(crate) fn join_u64(lo: u32, hi: u32) -> u64 {
+    (u64::from(lo)) | ((u64::from(hi)) << 32)
+}
+
+/// Read a logical 64-bit value from split 32-bit registers without tearing.
+pub(crate) fn read_u64_no_tearing(lo_read: impl Fn() -> u32, hi_read: impl Fn() -> u32) -> u64 {
+    loop {
+        let hi1 = hi_read();
+        let lo = lo_read();
+        let hi2 = hi_read();
+
+        if hi1 == hi2 {
+            return join_u64(lo, hi1);
+        }
     }
 }
 
-pub(crate) const GPU_ID: Register<0x0> = Register;
-pub(crate) const GPU_L2_FEATURES: Register<0x4> = Register;
-pub(crate) const GPU_CORE_FEATURES: Register<0x8> = Register;
-pub(crate) const GPU_CSF_ID: Register<0x1c> = Register;
-pub(crate) const GPU_REVID: Register<0x280> = Register;
-pub(crate) const GPU_TILER_FEATURES: Register<0xc> = Register;
-pub(crate) const GPU_MEM_FEATURES: Register<0x10> = Register;
-pub(crate) const GPU_MMU_FEATURES: Register<0x14> = Register;
-pub(crate) const GPU_AS_PRESENT: Register<0x18> = Register;
-pub(crate) const GPU_IRQ_RAWSTAT: Register<0x20> = Register;
-
-pub(crate) const GPU_IRQ_RAWSTAT_FAULT: u32 = bit_u32(0);
-pub(crate) const GPU_IRQ_RAWSTAT_PROTECTED_FAULT: u32 = bit_u32(1);
-pub(crate) const GPU_IRQ_RAWSTAT_RESET_COMPLETED: u32 = bit_u32(8);
-pub(crate) const GPU_IRQ_RAWSTAT_POWER_CHANGED_SINGLE: u32 = bit_u32(9);
-pub(crate) const GPU_IRQ_RAWSTAT_POWER_CHANGED_ALL: u32 = bit_u32(10);
-pub(crate) const GPU_IRQ_RAWSTAT_CLEAN_CACHES_COMPLETED: u32 = bit_u32(17);
-pub(crate) const GPU_IRQ_RAWSTAT_DOORBELL_STATUS: u32 = bit_u32(18);
-pub(crate) const GPU_IRQ_RAWSTAT_MCU_STATUS: u32 = bit_u32(19);
-
-pub(crate) const GPU_IRQ_CLEAR: Register<0x24> = Register;
-pub(crate) const GPU_IRQ_MASK: Register<0x28> = Register;
-pub(crate) const GPU_IRQ_STAT: Register<0x2c> = Register;
-pub(crate) const GPU_CMD: Register<0x30> = Register;
-pub(crate) const GPU_CMD_SOFT_RESET: u32 = 1 | (1 << 8);
-pub(crate) const GPU_CMD_HARD_RESET: u32 = 1 | (2 << 8);
-pub(crate) const GPU_THREAD_FEATURES: Register<0xac> = Register;
-pub(crate) const GPU_THREAD_MAX_THREADS: Register<0xa0> = Register;
-pub(crate) const GPU_THREAD_MAX_WORKGROUP_SIZE: Register<0xa4> = Register;
-pub(crate) const GPU_THREAD_MAX_BARRIER_SIZE: Register<0xa8> = Register;
-pub(crate) const GPU_TEXTURE_FEATURES0: Register<0xb0> = Register;
-pub(crate) const GPU_SHADER_PRESENT_LO: Register<0x100> = Register;
-pub(crate) const GPU_SHADER_PRESENT_HI: Register<0x104> = Register;
-pub(crate) const GPU_TILER_PRESENT_LO: Register<0x110> = Register;
-pub(crate) const GPU_TILER_PRESENT_HI: Register<0x114> = Register;
-pub(crate) const GPU_L2_PRESENT_LO: Register<0x120> = Register;
-pub(crate) const GPU_L2_PRESENT_HI: Register<0x124> = Register;
-pub(crate) const L2_READY_LO: Register<0x160> = Register;
-pub(crate) const L2_READY_HI: Register<0x164> = Register;
-pub(crate) const L2_PWRON_LO: Register<0x1a0> = Register;
-pub(crate) const L2_PWRON_HI: Register<0x1a4> = Register;
-pub(crate) const L2_PWRTRANS_LO: Register<0x220> = Register;
-pub(crate) const L2_PWRTRANS_HI: Register<0x204> = Register;
-pub(crate) const L2_PWRACTIVE_LO: Register<0x260> = Register;
-pub(crate) const L2_PWRACTIVE_HI: Register<0x264> = Register;
-
-pub(crate) const MCU_CONTROL: Register<0x700> = Register;
-pub(crate) const MCU_CONTROL_ENABLE: u32 = 1;
-pub(crate) const MCU_CONTROL_AUTO: u32 = 2;
-pub(crate) const MCU_CONTROL_DISABLE: u32 = 0;
-
-pub(crate) const MCU_STATUS: Register<0x704> = Register;
-pub(crate) const MCU_STATUS_DISABLED: u32 = 0;
-pub(crate) const MCU_STATUS_ENABLED: u32 = 1;
-pub(crate) const MCU_STATUS_HALT: u32 = 2;
-pub(crate) const MCU_STATUS_FATAL: u32 = 3;
-
-pub(crate) const GPU_COHERENCY_FEATURES: Register<0x300> = Register;
-
-pub(crate) const JOB_IRQ_RAWSTAT: Register<0x1000> = Register;
-pub(crate) const JOB_IRQ_CLEAR: Register<0x1004> = Register;
-pub(crate) const JOB_IRQ_MASK: Register<0x1008> = Register;
-pub(crate) const JOB_IRQ_STAT: Register<0x100c> = Register;
-
-pub(crate) const JOB_IRQ_GLOBAL_IF: u32 = bit_u32(31);
-
-pub(crate) const MMU_IRQ_RAWSTAT: Register<0x2000> = Register;
-pub(crate) const MMU_IRQ_CLEAR: Register<0x2004> = Register;
-pub(crate) const MMU_IRQ_MASK: Register<0x2008> = Register;
-pub(crate) const MMU_IRQ_STAT: Register<0x200c> = Register;
+/// These registers correspond to the GPU_CONTROL register page.
+/// They are involved in GPU configuration and control.
+pub(crate) mod gpu_control {
+    use core::convert::TryFrom;
+    use kernel::{
+        error::{
+            code::EINVAL,
+            Error, //
+        },
+        num::Bounded,
+        register,
+        uapi, //
+    };
+    use pin_init::Zeroable;
+
+    register! {
+        /// GPU identification register.
+        pub(crate) GPU_ID(u32) @ 0x0 {
+            /// Status of the GPU release.
+            3:0     ver_status;
+            /// Minor release version number.
+            11:4    ver_minor;
+            /// Major release version number.
+            15:12   ver_major;
+            /// Product identifier.
+            19:16   prod_major;
+            /// Architecture patch revision.
+            23:20   arch_rev;
+            /// Architecture minor revision.
+            27:24   arch_minor;
+            /// Architecture major revision.
+            31:28   arch_major;
+        }
+
+        /// Level 2 cache features register.
+        pub(crate) L2_FEATURES(u32) @ 0x4 {
+            /// Cache line size.
+            7:0     line_size;
+            /// Cache associativity.
+            15:8    associativity;
+            /// Cache slice size.
+            23:16   cache_size;
+            /// External bus width.
+            31:24   bus_width;
+        }
+
+        /// Shader core features.
+        pub(crate) CORE_FEATURES(u32) @ 0x8 {
+            /// Shader core variant.
+            7:0     core_variant;
+        }
+
+        /// Tiler features.
+        pub(crate) TILER_FEATURES(u32) @ 0xc {
+            /// Log of the tiler's bin size.
+            5:0     bin_size;
+            /// Maximum number of active levels.
+            11:8    max_levels;
+        }
+
+        /// Memory system features.
+        pub(crate) MEM_FEATURES(u32) @ 0x10 {
+            0:0     coherent_core_group => bool;
+            1:1     coherent_super_group => bool;
+            11:8    l2_slices;
+        }
+
+        /// Memory management unit features.
+        pub(crate) MMU_FEATURES(u32) @ 0x14 {
+            /// Number of bits supported in virtual addresses.
+            7:0     va_bits;
+            /// Number of bits supported in physical addresses.
+            15:8    pa_bits;
+        }
+
+        /// Address spaces present.
+        pub(crate) AS_PRESENT(u32) @ 0x18 {
+            31:0    present;
+        }
+
+        /// CSF version information.
+        pub(crate) CSF_ID(u32) @ 0x1c {
+            /// MCU revision ID.
+            3:0     mcu_rev;
+            /// MCU minor revision number.
+            9:4     mcu_minor;
+            /// MCU major revision number.
+            15:10   mcu_major;
+            /// CSHW revision ID.
+            19:16   cshw_rev;
+            /// CSHW minor revision number.
+            25:20   cshw_minor;
+            /// CSHW major revision number.
+            31:26   cshw_major;
+        }
+
+        /// IRQ sources raw status.
+        /// Writing to this register forces bits on, but does not clear them.
+        pub(crate) GPU_IRQ_RAWSTAT(u32) @ 0x20 {
+            /// A GPU fault has occurred, a 1-bit boolean flag.
+            0:0     gpu_fault => bool;
+            /// A GPU fault has occurred, a 1-bit boolean flag.
+            1:1     gpu_protected_fault => bool;
+            /// Reset has completed, a 1-bit boolean flag.
+            8:8     reset_completed => bool;
+            /// Set when a single power domain has powered up or down, a 1-bit boolean flag.
+            9:9     power_changed_single => bool;
+            /// Set when the all pending power domain changes are completed, a 1-bit boolean flag.
+            10:10   power_changed_all => bool;
+            /// Set when cache cleaning has completed, a 1-bit boolean flag.
+            17:17   clean_caches_completed => bool;
+            /// Mirrors the doorbell interrupt line to the CPU, a 1-bit boolean flag.
+            18:18   doorbell_mirror => bool;
+            /// MCU requires attention, a 1-bit boolean flag.
+            19:19   mcu_status => bool;
+        }
+
+        /// IRQ sources to clear. Write only.
+        pub(crate) GPU_IRQ_CLEAR(u32) @ 0x24 {
+            /// Clear the GPU_FAULT interrupt, a 1-bit boolean flag.
+            0:0     gpu_fault => bool;
+            /// Clear the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag.
+            1:1     gpu_protected_fault => bool;
+            /// Clear the RESET_COMPLETED interrupt, a 1-bit boolean flag.
+            8:8     reset_completed => bool;
+            /// Clear the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag.
+            9:9     power_changed_single => bool;
+            /// Clear the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag.
+            10:10   power_changed_all => bool;
+            /// Clear the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag.
+            17:17   clean_caches_completed => bool;
+            /// Clear the MCU_STATUS interrupt, a 1-bit boolean flag.
+            19:19   mcu_status => bool;
+        }
+
+        /// IRQ sources enabled.
+        pub(crate) GPU_IRQ_MASK(u32) @ 0x28 {
+            /// Enable the GPU_FAULT interrupt, a 1-bit boolean flag.
+            0:0     gpu_fault => bool;
+            /// Enable the GPU_PROTECTED_FAULT interrupt, a 1-bit boolean flag.
+            1:1     gpu_protected_fault => bool;
+            /// Enable the RESET_COMPLETED interrupt, a 1-bit boolean flag.
+            8:8     reset_completed => bool;
+            /// Enable the POWER_CHANGED_SINGLE interrupt, a 1-bit boolean flag.
+            9:9     power_changed_single => bool;
+            /// Enable the POWER_CHANGED_ALL interrupt, a 1-bit boolean flag.
+            10:10   power_changed_all => bool;
+            /// Enable the CLEAN_CACHES_COMPLETED interrupt, a 1-bit boolean flag.
+            17:17   clean_caches_completed => bool;
+            /// Enable the DOORBELL_MIRROR interrupt, a 1-bit boolean flag.
+            18:18   doorbell_mirror => bool;
+            /// Enable the MCU_STATUS interrupt, a 1-bit boolean flag.
+            19:19   mcu_status => bool;
+        }
+
+        /// IRQ status for enabled sources. Read only.
+        pub(crate) GPU_IRQ_STATUS(u32) @ 0x2c {
+            /// GPU_FAULT interrupt status, a 1-bit boolean flag.
+            0:0     gpu_fault => bool;
+            /// GPU_PROTECTED_FAULT interrupt status, a 1-bit boolean flag.
+            1:1     gpu_protected_fault => bool;
+            /// RESET_COMPLETED interrupt status, a 1-bit boolean flag.
+            8:8     reset_completed => bool;
+            /// POWER_CHANGED_SINGLE interrupt status, a 1-bit boolean flag.
+            9:9     power_changed_single => bool;
+            /// POWER_CHANGED_ALL interrupt status, a 1-bit boolean flag.
+            10:10   power_changed_all => bool;
+            /// CLEAN_CACHES_COMPLETED interrupt status, a 1-bit boolean flag.
+            17:17   clean_caches_completed => bool;
+            /// DOORBELL_MIRROR interrupt status, a 1-bit boolean flag.
+            18:18   doorbell_mirror => bool;
+            /// MCU_STATUS interrupt status, a 1-bit boolean flag.
+            19:19   mcu_status => bool;
+        }
+    }
+
+    /// Helpers for GPU_COMMAND Register
+    #[derive(Copy, Clone, Debug, PartialEq)]
+    #[repr(u8)]
+    pub(crate) enum GpuCommand {
+        /// No operation. This is the default value.
+        Nop = 0,
+        /// Reset the GPU.
+        Reset = 1,
+        /// Flush caches.
+        FlushCaches = 4,
+        /// Clear GPU faults.
+        ClearFault = 7,
+    }
+
+    impl TryFrom<Bounded<u32, 8>> for GpuCommand {
+        type Error = Error;
+
+        fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> {
+            match val.get() {
+                0 => Ok(GpuCommand::Nop),
+                1 => Ok(GpuCommand::Reset),
+                4 => Ok(GpuCommand::FlushCaches),
+                7 => Ok(GpuCommand::ClearFault),
+                _ => Err(EINVAL),
+            }
+        }
+    }
+
+    impl From<GpuCommand> for Bounded<u32, 8> {
+        fn from(cmd: GpuCommand) -> Self {
+            (cmd as u8).into()
+        }
+    }
+
+    /// Reset mode for [`GPU_COMMAND::reset()`].
+    #[derive(Copy, Clone, Debug, PartialEq)]
+    #[repr(u8)]
+    pub(crate) enum ResetMode {
+        /// Stop all external bus interfaces, then reset the entire GPU.
+        SoftReset = 1,
+        /// Force a full GPU reset.
+        HardReset = 2,
+    }
+
+    impl TryFrom<Bounded<u32, 4>> for ResetMode {
+        type Error = Error;
+
+        fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> {
+            match val.get() {
+                1 => Ok(ResetMode::SoftReset),
+                2 => Ok(ResetMode::HardReset),
+                _ => Err(EINVAL),
+            }
+        }
+    }
+
+    impl From<ResetMode> for Bounded<u32, 4> {
+        fn from(mode: ResetMode) -> Self {
+            Bounded::try_new(mode as u32).unwrap()
+        }
+    }
+
+    /// Cache flush mode for [`GPU_COMMAND::flush_caches()`].
+    #[derive(Copy, Clone, Debug, PartialEq)]
+    #[repr(u8)]
+    pub(crate) enum FlushMode {
+        /// No flush.
+        None = 0,
+        /// Clean the caches.
+        Clean = 1,
+        /// Invalidate the caches.
+        Invalidate = 2,
+        /// Clean and invalidate the caches.
+        CleanInvalidate = 3,
+    }
+
+    impl TryFrom<Bounded<u32, 4>> for FlushMode {
+        type Error = Error;
+
+        fn try_from(val: Bounded<u32, 4>) -> Result<Self, Self::Error> {
+            match val.get() {
+                0 => Ok(FlushMode::None),
+                1 => Ok(FlushMode::Clean),
+                2 => Ok(FlushMode::Invalidate),
+                3 => Ok(FlushMode::CleanInvalidate),
+                _ => Err(EINVAL),
+            }
+        }
+    }
+
+    impl From<FlushMode> for Bounded<u32, 4> {
+        fn from(mode: FlushMode) -> Self {
+            Bounded::try_new(mode as u32).unwrap()
+        }
+    }
+
+    register! {
+        /// GPU command register.
+        ///
+        /// Use the constructor methods to create commands:
+        /// - [`GPU_COMMAND::nop()`]
+        /// - [`GPU_COMMAND::reset()`]
+        /// - [`GPU_COMMAND::flush_caches()`]
+        /// - [`GPU_COMMAND::clear_fault()`]
+        pub(crate) GPU_COMMAND (u32) @ 0x30 {
+            7:0     command ?=> GpuCommand;
+        }
+        /// Internal alias for GPU_COMMAND in reset mode.
+        /// Use [`GPU_COMMAND::reset()`] instead.
+        GPU_COMMAND_RESET (u32) => GPU_COMMAND {
+            7:0     command ?=> GpuCommand;
+            11:8    reset_mode ?=> ResetMode;
+        }
+
+        /// Internal alias for GPU_COMMAND in cache flush mode.
+        /// Use [`GPU_COMMAND::flush_caches()`] instead.
+        GPU_COMMAND_FLUSH (u32) => GPU_COMMAND {
+            7:0     command ?=> GpuCommand;
+            /// L2 cache flush mode.
+            11:8    l2_flush ?=> FlushMode;
+            /// Shader core load/store cache flush mode.
+            15:12   lsc_flush ?=> FlushMode;
+            /// Shader core other caches flush mode.
+            19:16   other_flush ?=> FlushMode;
+        }
+    }
+
+    impl GPU_COMMAND {
+        /// Create a NOP command.
+        pub(crate) fn nop() -> Self {
+            Self::zeroed()
+        }
+
+        /// Create a reset command with the specified reset mode.
+        pub(crate) fn reset(mode: ResetMode) -> Self {
+            Self::from_raw(
+                GPU_COMMAND_RESET::zeroed()
+                    .with_command(GpuCommand::Reset)
+                    .with_reset_mode(mode)
+                    .into_raw(),
+            )
+        }
+
+        /// Create a cache flush command with the specified flush modes.
+        pub(crate) fn flush_caches(l2: FlushMode, lsc: FlushMode, other: FlushMode) -> Self {
+            Self::from_raw(
+                GPU_COMMAND_FLUSH::zeroed()
+                    .with_command(GpuCommand::FlushCaches)
+                    .with_l2_flush(l2)
+                    .with_lsc_flush(lsc)
+                    .with_other_flush(other)
+                    .into_raw(),
+            )
+        }
+
+        /// Create a clear fault command.
+        pub(crate) fn clear_fault() -> Self {
+            Self::zeroed().with_command(GpuCommand::ClearFault)
+        }
+    }
+
+    register! {
+        /// GPU status register. Read only.
+        pub(crate) GPU_STATUS(u32) @ 0x34 {
+            /// GPU active, a 1-bit boolean flag.
+            0:0     gpu_active => bool;
+            /// Power manager active, a 1-bit boolean flag
+            1:1     pwr_active => bool;
+            /// Page fault active, a 1-bit boolean flag.
+            4:4     page_fault => bool;
+            /// Protected mode active, a 1-bit boolean flag.
+            7:7     protected_mode_active => bool;
+            /// Debug mode active, a 1-bit boolean flag.
+            8:8     gpu_dbg_enabled => bool;
+        }
+    }
+
+    #[derive(Copy, Clone, Debug, PartialEq)]
+    #[repr(u8)]
+    pub(crate) enum ExceptionType {
+        /// Exception type: No error.
+        Ok = 0x00,
+        /// Exception type: GPU external bus error.
+        GpuBusFault = 0x80,
+        /// Exception type: GPU shareability error.
+        GpuShareabilityFault = 0x88,
+        /// Exception type: System shareability error.
+        SystemShareabilityFault = 0x89,
+        /// Exception type: GPU cacheability error.
+        GpuCacheabilityFault = 0x8A,
+    }
+
+    impl TryFrom<Bounded<u32, 8>> for ExceptionType {
+        type Error = Error;
+
+        fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> {
+            match val.get() {
+                0x00 => Ok(ExceptionType::Ok),
+                0x80 => Ok(ExceptionType::GpuBusFault),
+                0x88 => Ok(ExceptionType::GpuShareabilityFault),
+                0x89 => Ok(ExceptionType::SystemShareabilityFault),
+                0x8A => Ok(ExceptionType::GpuCacheabilityFault),
+                _ => Err(EINVAL),
+            }
+        }
+    }
+
+    impl From<ExceptionType> for Bounded<u32, 8> {
+        fn from(exc: ExceptionType) -> Self {
+            (exc as u8).into()
+        }
+    }
+
+    #[derive(Copy, Clone, Debug, PartialEq)]
+    #[repr(u8)]
+    pub(crate) enum AccessType {
+        /// Access type: An atomic (read/write) transaction.
+        Atomic = 0,
+        /// Access type: An execute transaction.
+        Execute = 1,
+        /// Access type: A read transaction.
+        Read = 2,
+        /// Access type: A write transaction.
+        Write = 3,
+    }
+
+    impl From<Bounded<u32, 2>> for AccessType {
+        fn from(val: Bounded<u32, 2>) -> Self {
+            match val.get() {
+                0 => AccessType::Atomic,
+                1 => AccessType::Execute,
+                2 => AccessType::Read,
+                3 => AccessType::Write,
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    impl From<AccessType> for Bounded<u32, 2> {
+        fn from(access: AccessType) -> Self {
+            Bounded::try_new(access as u32).unwrap()
+        }
+    }
+
+    register! {
+        /// GPU fault status register. Read only.
+        pub(crate) GPU_FAULTSTATUS(u32) @ 0x3c {
+            /// Exception type.
+            7:0     exception_type ?=> ExceptionType;
+            /// Access type.
+            9:8     access_type => AccessType;
+            /// The GPU_FAULTADDRESS is valid, a 1-bit boolean flag.
+            10:10   address_valid => bool;
+            /// The JASID field is valid, a 1-bit boolean flag.
+            11:11   jasid_valid => bool;
+            /// JASID of the fault, if known.
+            15:12   jasid;
+            /// ID of the source that triggered the fault.
+            31:16   source_id;
+        }
+
+        /// GPU fault address. Read only.
+        /// Once a fault is reported, it must be manually cleared by issuing a
+        /// [`GPU_COMMAND::clear_fault()`] command to the [`GPU_COMMAND`] register. No further GPU
+        /// faults will be reported until the previous fault has been cleared.
+        pub(crate) GPU_FAULTADDRESS_LO(u32) @ 0x40 {
+            31:0    pointer;
+        }
+
+        pub(crate) GPU_FAULTADDRESS_HI(u32) @ 0x44 {
+            31:0    pointer;
+        }
+
+        /// Level 2 cache configuration.
+        pub(crate) L2_CONFIG(u32) @ 0x48 {
+            /// Requested cache size.
+            23:16   cache_size;
+            /// Requested hash function index.
+            31:24   hash_function;
+        }
+
+        /// Global time stamp offset.
+        pub(crate) TIMESTAMP_OFFSET_LO(u32) @ 0x88 {
+            31:0    offset;
+        }
+
+        pub(crate) TIMESTAMP_OFFSET_HI(u32) @ 0x8c {
+            31:0    offset;
+        }
+
+        /// GPU cycle counter. Read only.
+        pub(crate) CYCLE_COUNT_LO(u32) @ 0x90 {
+            31:0    count;
+        }
+
+        pub(crate) CYCLE_COUNT_HI(u32) @ 0x94 {
+            31:0    count;
+        }
+
+        /// Global time stamp. Read only.
+        pub(crate) TIMESTAMP_LO(u32) @ 0x98 {
+            31:0    timestamp;
+        }
+
+        pub(crate) TIMESTAMP_HI(u32) @ 0x9c {
+            31:0    timestamp;
+        }
+
+        /// Maximum number of threads per core. Read only constant.
+        pub(crate) THREAD_MAX_THREADS(u32) @ 0xa0 {
+            31:0    threads;
+        }
+
+        /// Maximum number of threads per workgroup. Read only constant.
+        pub(crate) THREAD_MAX_WORKGROUP_SIZE(u32) @ 0xa4 {
+            31:0    threads;
+        }
+
+        /// Maximum number of threads per barrier. Read only constant.
+        pub(crate) THREAD_MAX_BARRIER_SIZE(u32) @ 0xa8 {
+            31:0    threads;
+        }
+
+        /// Thread features. Read only constant.
+        pub(crate) THREAD_FEATURES(u32) @ 0xac {
+            /// Total number of registers per core.
+            21:0    max_registers;
+            /// Implementation technology type.
+            23:22   implementation_technology;
+            /// Maximum number of compute tasks waiting.
+            31:24   max_task_queue;
+        }
+
+        /// Support flags for compressed texture formats. Read only constant.
+        ///
+        /// A bitmap where each bit indicates support for a specific compressed texture format.
+        /// The bit position maps to an opaque format ID (`texture_features_key_t` in spec).
+        pub(crate) TEXTURE_FEATURES(u32)[4] @ 0xb0 {
+            31:0    supported_formats;
+        }
+
+        /// Shader core present bitmap. Read only constant.
+        pub(crate) SHADER_PRESENT_LO(u32) @ 0x100 {
+            31:0    value;
+        }
+
+        pub(crate) SHADER_PRESENT_HI(u32) @ 0x104 {
+            31:0    value;
+        }
+
+        /// Tiler present bitmap. Read only constant.
+        pub(crate) TILER_PRESENT_LO(u32) @ 0x110 {
+            31:0    present;
+        }
+
+        pub(crate) TILER_PRESENT_HI(u32) @ 0x114 {
+            31:0    present;
+        }
+
+        /// L2 cache present bitmap. Read only constant.
+        pub(crate) L2_PRESENT_LO(u32) @ 0x120 {
+            31:0    present;
+        }
+
+        pub(crate) L2_PRESENT_HI(u32) @ 0x124 {
+            31:0    present;
+        }
+
+        /// Shader core ready bitmap. Read only.
+        pub(crate) SHADER_READY_LO(u32) @ 0x140 {
+            31:0    ready;
+        }
+
+        pub(crate) SHADER_READY_HI(u32) @ 0x144 {
+            31:0    ready;
+        }
+
+        /// Tiler ready bitmap. Read only.
+        pub(crate) TILER_READY_LO(u32) @ 0x150 {
+            31:0    ready;
+        }
+
+        pub(crate) TILER_READY_HI(u32) @ 0x154 {
+            31:0    ready;
+        }
+
+        /// L2 ready bitmap. Read only.
+        pub(crate) L2_READY_LO(u32) @ 0x160 {
+            31:0    ready;
+        }
+
+        pub(crate) L2_READY_HI(u32) @ 0x164 {
+            31:0    ready;
+        }
+
+        /// Shader core power up bitmap.
+        pub(crate) SHADER_PWRON_LO(u32) @ 0x180 {
+            31:0    request;
+        }
+
+        pub(crate) SHADER_PWRON_HI(u32) @ 0x184 {
+            31:0    request;
+        }
+
+        /// Tiler power up bitmap.
+        pub(crate) TILER_PWRON_LO(u32) @ 0x190 {
+            31:0    request;
+        }
+
+        pub(crate) TILER_PWRON_HI(u32) @ 0x194 {
+            31:0    request;
+        }
+
+        /// L2 power up bitmap.
+        pub(crate) L2_PWRON_LO(u32) @ 0x1a0 {
+            31:0    request;
+        }
+
+        pub(crate) L2_PWRON_HI(u32) @ 0x1a4 {
+            31:0    request;
+        }
+
+        /// Shader core power down bitmap.
+        pub(crate) SHADER_PWROFF_LO(u32) @ 0x1c0 {
+            31:0    request;
+        }
+
+        pub(crate) SHADER_PWROFF_HI(u32) @ 0x1c4 {
+            31:0    request;
+        }
+
+        /// Tiler power down bitmap.
+        pub(crate) TILER_PWROFF_LO(u32) @ 0x1d0 {
+            31:0    request;
+        }
+
+        pub(crate) TILER_PWROFF_HI(u32) @ 0x1d4 {
+            31:0    request;
+        }
+
+        /// L2 power down bitmap.
+        pub(crate) L2_PWROFF_LO(u32) @ 0x1e0 {
+            31:0    request;
+        }
+
+        pub(crate) L2_PWROFF_HI(u32) @ 0x1e4 {
+            31:0    request;
+        }
+
+        /// Shader core power transition bitmap. Read-only.
+        pub(crate) SHADER_PWRTRANS_LO(u32) @ 0x200 {
+            31:0    changing;
+        }
+
+        pub(crate) SHADER_PWRTRANS_HI(u32) @ 0x204 {
+            31:0    changing;
+        }
+
+        /// Tiler power transition bitmap. Read-only.
+        pub(crate) TILER_PWRTRANS_LO(u32) @ 0x210 {
+            31:0    changing;
+        }
+
+        pub(crate) TILER_PWRTRANS_HI(u32) @ 0x214 {
+            31:0    changing;
+        }
+
+        /// L2 power transition bitmap. Read-only.
+        pub(crate) L2_PWRTRANS_LO(u32) @ 0x220 {
+            31:0    changing;
+        }
+
+        pub(crate) L2_PWRTRANS_HI(u32) @ 0x224 {
+            31:0    changing;
+        }
+
+        /// Shader core active bitmap. Read-only.
+        pub(crate) SHADER_PWRACTIVE_LO(u32) @ 0x240 {
+            31:0    active;
+        }
+
+        pub(crate) SHADER_PWRACTIVE_HI(u32) @ 0x244 {
+            31:0    active;
+        }
+
+        /// Tiler active bitmap. Read-only.
+        pub(crate) TILER_PWRACTIVE_LO(u32) @ 0x250 {
+            31:0    active;
+        }
+
+        pub(crate) TILER_PWRACTIVE_HI(u32) @ 0x254 {
+            31:0    active;
+        }
+
+        /// L2 active bitmap.  Read-only.
+        pub(crate) L2_PWRACTIVE_LO(u32) @ 0x260 {
+            31:0    active;
+        }
+
+        pub(crate) L2_PWRACTIVE_HI(u32) @ 0x264 {
+            31:0    active;
+        }
+
+        /// Revision ID. Read only constant.
+        pub(crate) REVIDR(u32) @ 0x280 {
+            31:0    revision;
+        }
+
+        /// Coherency features present. Read only constant.
+        /// Supported protocols on the interconnect between the GPU and the
+        /// system into which it is integrated.
+        pub(crate) COHERENCY_FEATURES(u32) @ 0x300 {
+            /// ACE-Lite protocol supported, a 1-bit boolean flag.
+            0:0     ace_lite => bool;
+            /// ACE protocol supported, a 1-bit boolean flag.
+            1:1     ace => bool;
+        }
+    }
+
+    #[derive(Copy, Clone, Debug, PartialEq)]
+    #[repr(u8)]
+    pub(crate) enum CoherencyMode {
+        /// ACE-Lite coherency protocol.
+        AceLite = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE_LITE as u8,
+        /// ACE coherency protocol.
+        Ace = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_ACE as u8,
+        /// No coherency protocol.
+        None = uapi::drm_panthor_gpu_coherency_DRM_PANTHOR_GPU_COHERENCY_NONE as u8,
+    }
+
+    impl TryFrom<Bounded<u32, 32>> for CoherencyMode {
+        type Error = Error;
+
+        fn try_from(val: Bounded<u32, 32>) -> Result<Self, Self::Error> {
+            match val.get() {
+                0 => Ok(CoherencyMode::AceLite),
+                1 => Ok(CoherencyMode::Ace),
+                31 => Ok(CoherencyMode::None),
+                _ => Err(EINVAL),
+            }
+        }
+    }
+
+    impl From<CoherencyMode> for Bounded<u32, 32> {
+        fn from(mode: CoherencyMode) -> Self {
+            (mode as u8).into()
+        }
+    }
+
+    register! {
+        /// Coherency enable. An index of which coherency protocols should be used.
+        /// This register only selects the protocol for coherency messages on the
+        /// interconnect. This is not to enable or disable coherency controlled by MMU.
+        pub(crate) COHERENCY_ENABLE(u32) @ 0x304 {
+            31:0    l2_cache_protocol_select ?=> CoherencyMode;
+        }
+    }
+
+    /// Helpers for MCU_CONTROL register
+    #[derive(Copy, Clone, Debug, PartialEq)]
+    #[repr(u8)]
+    pub(crate) enum McuControlMode {
+        /// Disable the MCU.
+        Disable = 0,
+        /// Enable the MCU.
+        Enable = 1,
+        /// Enable the MCU to execute and automatically reboot after a fast reset.
+        Auto = 2,
+    }
+
+    impl TryFrom<Bounded<u32, 2>> for McuControlMode {
+        type Error = Error;
+
+        fn try_from(val: Bounded<u32, 2>) -> Result<Self, Self::Error> {
+            match val.get() {
+                0 => Ok(McuControlMode::Disable),
+                1 => Ok(McuControlMode::Enable),
+                2 => Ok(McuControlMode::Auto),
+                _ => Err(EINVAL),
+            }
+        }
+    }
+
+    impl From<McuControlMode> for Bounded<u32, 2> {
+        fn from(mode: McuControlMode) -> Self {
+            Bounded::try_new(mode as u32).unwrap()
+        }
+    }
+
+    register! {
+        /// MCU control.
+        pub(crate) MCU_CONTROL(u32) @ 0x700 {
+            /// Request MCU state change.
+            1:0 req ?=> McuControlMode;
+        }
+    }
+
+    /// Helpers for MCU_STATUS register
+    #[derive(Copy, Clone, Debug, PartialEq)]
+    #[repr(u8)]
+    pub(crate) enum McuStatus {
+        /// MCU is disabled.
+        Disabled = 0,
+        /// MCU is enabled.
+        Enabled = 1,
+        /// The MCU has halted by itself in an orderly manner to enable the core group to be
+        /// powered down.
+        Halt = 2,
+        /// The MCU has encountered an error that prevents it from continuing.
+        Fatal = 3,
+    }
+
+    impl From<Bounded<u32, 2>> for McuStatus {
+        fn from(val: Bounded<u32, 2>) -> Self {
+            match val.get() {
+                0 => McuStatus::Disabled,
+                1 => McuStatus::Enabled,
+                2 => McuStatus::Halt,
+                3 => McuStatus::Fatal,
+                _ => unreachable!(),
+            }
+        }
+    }
+
+    impl From<McuStatus> for Bounded<u32, 2> {
+        fn from(status: McuStatus) -> Self {
+            Bounded::try_new(status as u32).unwrap()
+        }
+    }
+
+    register! {
+        /// MCU status. Read only.
+        pub(crate) MCU_STATUS(u32) @ 0x704 {
+            /// Read current state of MCU.
+            1:0 value => McuStatus;
+        }
+    }
+}
+
+/// These registers correspond to the JOB_CONTROL register page.
+/// They are involved in communication between the firmware running on the MCU and the host.
+pub(crate) mod job_control {
+    use kernel::register;
+
+    register! {
+        /// Raw status of job interrupts.
+        ///
+        /// Write to this register to trigger these interrupts.
+        /// Writing a 1 to a bit forces that bit on.
+        pub(crate) JOB_IRQ_RAWSTAT(u32) @ 0x1000 {
+            /// CSG request. These bits indicate that CSGn requires attention from the host.
+            30:0    csg;
+            /// GLB request. Indicates that the GLB interface requires attention from the host.
+            31:31   glb => bool;
+        }
+
+        /// Clear job interrupts. Write only.
+        ///
+        /// Write a 1 to a bit to clear the corresponding bit in [`JOB_IRQ_RAWSTAT`].
+        pub(crate) JOB_IRQ_CLEAR(u32) @ 0x1004 {
+            /// Clear CSG request interrupts.
+            30:0    csg;
+            /// Clear GLB request interrupt.
+            31:31   glb => bool;
+        }
+
+        /// Mask for job interrupts.
+        ///
+        /// Set each bit to 1 to enable the corresponding interrupt source or to 0 to disable it.
+        pub(crate) JOB_IRQ_MASK(u32) @ 0x1008 {
+            /// Enable CSG request interrupts.
+            30:0    csg;
+            /// Enable GLB request interrupt.
+            31:31   glb => bool;
+        }
+
+        /// Active job interrupts. Read only.
+        ///
+        /// This register contains the result of ANDing together [`JOB_IRQ_RAWSTAT`] and
+        /// [`JOB_IRQ_MASK`].
+        pub(crate) JOB_IRQ_STATUS(u32) @ 0x100c {
+            /// CSG request interrupt status.
+            30:0    csg;
+            /// GLB request interrupt status.
+            31:31   glb => bool;
+        }
+    }
+}
+
+/// These registers correspond to the MMU_CONTROL register page.
+/// They are involved in MMU configuration and control.
+pub(crate) mod mmu_control {
+    use kernel::register;
+
+    register! {
+        /// IRQ sources raw status.
+        ///
+        /// This register contains the raw unmasked interrupt sources for MMU status and exception
+        /// handling.
+        ///
+        /// Writing to this register forces bits on.
+        /// Use [`IRQ_CLEAR`] to clear interrupts.
+        pub(crate) IRQ_RAWSTAT(u32) @ 0x2000 {
+            /// Page fault for address spaces.
+            15:0    page_fault;
+            /// Command completed in address spaces.
+            31:16   command_completed;
+        }
+
+        /// IRQ sources to clear.
+        /// Write a 1 to a bit to clear the corresponding bit in [`IRQ_RAWSTAT`].
+        pub(crate) IRQ_CLEAR(u32) @ 0x2004 {
+            /// Clear the PAGE_FAULT interrupt.
+            15:0    page_fault;
+            /// Clear the COMMAND_COMPLETED interrupt.
+            31:16   command_completed;
+        }
+
+        /// IRQ sources enabled.
+        ///
+        /// Set each bit to 1 to enable the corresponding interrupt source, and to 0 to disable it.
+        pub(crate) IRQ_MASK(u32) @ 0x2008 {
+            /// Enable the PAGE_FAULT interrupt.
+            15:0    page_fault;
+            /// Enable the COMMAND_COMPLETED interrupt.
+            31:16   command_completed;
+        }
+
+        /// IRQ status for enabled sources. Read only.
+        ///
+        /// This register contains the result of ANDing together [`IRQ_RAWSTAT`] and [`IRQ_MASK`].
+        pub(crate) IRQ_STATUS(u32) @ 0x200c {
+            /// PAGE_FAULT interrupt status.
+            15:0    page_fault;
+            /// COMMAND_COMPLETED interrupt status.
+            31:16   command_completed;
+        }
+    }
+
+    /// Per-address space registers ASn [0..15] within the MMU_CONTROL page.
+    ///
+    /// This array contains 16 instances of the MMU_AS_CONTROL register page.
+    pub(crate) mod mmu_as_control {
+        use core::convert::TryFrom;
+
+        use kernel::{
+            error::{
+                code::EINVAL,
+                Error, //
+            },
+            num::Bounded,
+            register, //
+        };
+
+        /// Maximum number of hardware address space slots.
+        /// The actual number of slots available is usually lower.
+        pub(crate) const MAX_AS: usize = 16;
+
+        /// Address space register stride. The elements in the array are spaced 64B apart.
+        const STRIDE: usize = 0x40;
+
+        register! {
+            /// Translation table base address. A 64-bit pointer.
+            ///
+            /// This field contains the address of the top level of a translation table structure.
+            /// This must be 16-byte-aligned, so address bits [3:0] are assumed to be zero.
+            pub(crate) TRANSTAB(u64)[MAX_AS, stride = STRIDE] @ 0x2400 {
+                /// Base address of the translation table.
+                63:0    base;
+            }
+
+            // TRANSTAB is a logical 64-bit register, but it is laid out in hardware as two
+            // 32-bit halves. Define it as separate low/high u32 registers so accesses match
+            // the MMIO register layout and do not rely on native 64-bit MMIO transactions.
+            pub(crate) TRANSTAB_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2400 {
+                   31:0 value;
+            }
+
+            pub(crate) TRANSTAB_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2404 {
+                31:0 value;
+            }
+        }
+
+        /// Helpers for MEMATTR Register.
+
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        pub(crate) enum AllocPolicySelect {
+            /// Ignore ALLOC_R/ALLOC_W fields.
+            Impl = 2,
+            /// Use ALLOC_R/ALLOC_W fields for allocation policy.
+            Alloc = 3,
+        }
+
+        impl TryFrom<Bounded<u8, 2>> for AllocPolicySelect {
+            type Error = Error;
+
+            fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> {
+                match val.get() {
+                    2 => Ok(Self::Impl),
+                    3 => Ok(Self::Alloc),
+                    _ => Err(EINVAL),
+                }
+            }
+        }
+
+        impl From<AllocPolicySelect> for Bounded<u8, 2> {
+            fn from(val: AllocPolicySelect) -> Self {
+                Bounded::try_new(val as u8).unwrap()
+            }
+        }
+
+        /// Coherency policy for memory attributes. Indicates the shareability of cached accesses.
+        ///
+        /// The hardware spec defines different interpretations of these values depending on
+        /// whether TRANSCFG.MODE is set to IDENTITY or not. IDENTITY mode does not use translation
+        /// tables (all input addresses map to the same output address); it is deprecated and not
+        /// used by the driver. This enum assumes that TRANSCFG.MODE is not set to IDENTITY.
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        pub(crate) enum Coherency {
+            /// Midgard inner domain coherency.
+            ///
+            /// Most flexible mode - can map non-coherent, internally coherent, and system/IO
+            /// coherent memory. Used for non-cacheable memory in MAIR conversion.
+            MidgardInnerDomain = 0,
+            /// CPU inner domain coherency.
+            ///
+            /// Can map non-coherent and system/IO coherent memory. Used for write-back
+            /// cacheable memory in MAIR conversion to maintain CPU-GPU cache coherency.
+            CpuInnerDomain = 1,
+            /// CPU inner domain with shader coherency.
+            ///
+            /// Can map internally coherent and system/IO coherent memory. Used for
+            /// GPU-internal shared buffers requiring shader coherency.
+            CpuInnerDomainShaderCoh = 2,
+        }
+
+        impl TryFrom<Bounded<u8, 2>> for Coherency {
+            type Error = Error;
+
+            fn try_from(val: Bounded<u8, 2>) -> Result<Self, Self::Error> {
+                match val.get() {
+                    0 => Ok(Self::MidgardInnerDomain),
+                    1 => Ok(Self::CpuInnerDomain),
+                    2 => Ok(Self::CpuInnerDomainShaderCoh),
+                    _ => Err(EINVAL),
+                }
+            }
+        }
+
+        impl From<Coherency> for Bounded<u8, 2> {
+            fn from(val: Coherency) -> Self {
+                Bounded::try_new(val as u8).unwrap()
+            }
+        }
+
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        pub(crate) enum MemoryType {
+            /// Normal memory (shared).
+            Shared = 0,
+            /// Normal memory, inner/outer non-cacheable.
+            NonCacheable = 1,
+            /// Normal memory, inner/outer write-back cacheable.
+            WriteBack = 2,
+            /// Triggers MEMORY_ATTRIBUTE_FAULT.
+            Fault = 3,
+        }
+
+        impl From<Bounded<u8, 2>> for MemoryType {
+            fn from(val: Bounded<u8, 2>) -> Self {
+                match val.get() {
+                    0 => Self::Shared,
+                    1 => Self::NonCacheable,
+                    2 => Self::WriteBack,
+                    3 => Self::Fault,
+                    _ => unreachable!(),
+                }
+            }
+        }
+
+        impl From<MemoryType> for Bounded<u8, 2> {
+            fn from(val: MemoryType) -> Self {
+                Bounded::try_new(val as u8).unwrap()
+            }
+        }
+
+        register! {
+            /// Stage 1 memory attributes (8-bit bitfield).
+            ///
+            /// This is not an actual register, but a bitfield definition used by the MEMATTR
+            /// register. Each of the 8 bytes in MEMATTR follows this layout.
+            MMU_MEMATTR_STAGE1(u8) @ 0x0 {
+                /// Inner cache write allocation policy.
+                0:0     alloc_w => bool;
+                /// Inner cache read allocation policy.
+                1:1     alloc_r => bool;
+                /// Inner allocation policy select.
+                3:2     alloc_sel ?=> AllocPolicySelect;
+                /// Coherency policy.
+                5:4     coherency ?=> Coherency;
+                /// Memory type.
+                7:6     memory_type => MemoryType;
+            }
+        }
+
+        impl TryFrom<Bounded<u64, 8>> for MMU_MEMATTR_STAGE1 {
+            type Error = Error;
+
+            fn try_from(val: Bounded<u64, 8>) -> Result<Self, Self::Error> {
+                Ok(Self::from_raw(val.get() as u8))
+            }
+        }
+
+        impl From<MMU_MEMATTR_STAGE1> for Bounded<u64, 8> {
+            fn from(val: MMU_MEMATTR_STAGE1) -> Self {
+                Bounded::try_new(u64::from(val.into_raw())).unwrap()
+            }
+        }
+
+        register! {
+            /// Memory attributes.
+            ///
+            /// Each address space can configure up to 8 different memory attribute profiles.
+            /// Each attribute profile follows the MMU_MEMATTR_STAGE1 layout.
+            pub(crate) MEMATTR(u64)[MAX_AS, stride = STRIDE] @ 0x2408 {
+                7:0     attribute0 ?=> MMU_MEMATTR_STAGE1;
+                15:8    attribute1 ?=> MMU_MEMATTR_STAGE1;
+                23:16   attribute2 ?=> MMU_MEMATTR_STAGE1;
+                31:24   attribute3 ?=> MMU_MEMATTR_STAGE1;
+                39:32   attribute4 ?=> MMU_MEMATTR_STAGE1;
+                47:40   attribute5 ?=> MMU_MEMATTR_STAGE1;
+                55:48   attribute6 ?=> MMU_MEMATTR_STAGE1;
+                63:56   attribute7 ?=> MMU_MEMATTR_STAGE1;
+            }
+
+            // MEMATTR is a logical 64-bit register, but it is laid out in hardware as two
+            // 32-bit halves. Define it as separate low/high u32 registers so accesses match
+            // the MMIO register layout and do not rely on native 64-bit MMIO transactions.
+            pub(crate) MEMATTR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2408 {
+                31:0 value;
+            }
+
+            pub(crate) MEMATTR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x240c {
+                31:0 value;
+            }
+
+            /// Lock region address for each address space.
+            pub(crate) LOCKADDR(u64)[MAX_AS, stride = STRIDE] @ 0x2410 {
+                /// Lock region size.
+                5:0     size;
+                /// Lock region base address.
+                63:12   base;
+            }
+
+            // LOCKADDR is a logical 64-bit register, but it is laid out in hardware as two
+            // 32-bit halves. Define it as separate low/high u32 registers so accesses match
+            // the MMIO register layout and do not rely on native 64-bit MMIO transactions.
+            pub(crate) LOCKADDR_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2410 {
+               31:0 value;
+            }
+
+            pub(crate) LOCKADDR_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2414 {
+                31:0 value;
+            }
+        }
+
+        /// Helpers for MMU COMMAND register.
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        pub(crate) enum MmuCommand {
+            /// No operation, nothing happens.
+            Nop = 0,
+            /// Propagate settings to the MMU.
+            Update = 1,
+            /// Lock an address region.
+            Lock = 2,
+            /// Unlock an address region.
+            Unlock = 3,
+            /// Clean and invalidate the L2 cache, then unlock.
+            FlushPt = 4,
+            /// Clean and invalidate all caches, then unlock.
+            FlushMem = 5,
+        }
+
+        impl TryFrom<Bounded<u32, 8>> for MmuCommand {
+            type Error = Error;
+
+            fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> {
+                match val.get() {
+                    0 => Ok(MmuCommand::Nop),
+                    1 => Ok(MmuCommand::Update),
+                    2 => Ok(MmuCommand::Lock),
+                    3 => Ok(MmuCommand::Unlock),
+                    4 => Ok(MmuCommand::FlushPt),
+                    5 => Ok(MmuCommand::FlushMem),
+                    _ => Err(EINVAL),
+                }
+            }
+        }
+
+        impl From<MmuCommand> for Bounded<u32, 8> {
+            fn from(cmd: MmuCommand) -> Self {
+                (cmd as u8).into()
+            }
+        }
+
+        register! {
+            /// MMU command register for each address space. Write only.
+            pub(crate) COMMAND(u32)[MAX_AS, stride = STRIDE] @ 0x2418 {
+                7:0     command ?=> MmuCommand;
+            }
+        }
+
+        /// MMU exception types for FAULTSTATUS register.
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        pub(crate) enum MmuExceptionType {
+            /// No error.
+            Ok = 0x00,
+            /// Invalid translation table entry, level 0.
+            TranslationFault0 = 0xC0,
+            /// Invalid translation table entry, level 1.
+            TranslationFault1 = 0xC1,
+            /// Invalid translation table entry, level 2.
+            TranslationFault2 = 0xC2,
+            /// Invalid translation table entry, level 3.
+            TranslationFault3 = 0xC3,
+            /// Invalid block descriptor.
+            TranslationFault4 = 0xC4,
+            /// Page permission error, level 0.
+            PermissionFault0 = 0xC8,
+            /// Page permission error, level 1.
+            PermissionFault1 = 0xC9,
+            /// Page permission error, level 2.
+            PermissionFault2 = 0xCA,
+            /// Page permission error, level 3.
+            PermissionFault3 = 0xCB,
+            /// Access flag not set, level 1.
+            AccessFlag1 = 0xD9,
+            /// Access flag not set, level 2.
+            AccessFlag2 = 0xDA,
+            /// Access flag not set, level 3.
+            AccessFlag3 = 0xDB,
+            /// Virtual address out of range.
+            AddressSizeFaultIn = 0xE0,
+            /// Physical address out of range, level 0.
+            AddressSizeFaultOut0 = 0xE4,
+            /// Physical address out of range, level 1.
+            AddressSizeFaultOut1 = 0xE5,
+            /// Physical address out of range, level 2.
+            AddressSizeFaultOut2 = 0xE6,
+            /// Physical address out of range, level 3.
+            AddressSizeFaultOut3 = 0xE7,
+            /// Page attribute error, level 0.
+            MemoryAttributeFault0 = 0xE8,
+            /// Page attribute error, level 1.
+            MemoryAttributeFault1 = 0xE9,
+            /// Page attribute error, level 2.
+            MemoryAttributeFault2 = 0xEA,
+            /// Page attribute error, level 3.
+            MemoryAttributeFault3 = 0xEB,
+        }
+
+        impl TryFrom<Bounded<u32, 8>> for MmuExceptionType {
+            type Error = Error;
+
+            fn try_from(val: Bounded<u32, 8>) -> Result<Self, Self::Error> {
+                match val.get() {
+                    0x00 => Ok(MmuExceptionType::Ok),
+                    0xC0 => Ok(MmuExceptionType::TranslationFault0),
+                    0xC1 => Ok(MmuExceptionType::TranslationFault1),
+                    0xC2 => Ok(MmuExceptionType::TranslationFault2),
+                    0xC3 => Ok(MmuExceptionType::TranslationFault3),
+                    0xC4 => Ok(MmuExceptionType::TranslationFault4),
+                    0xC8 => Ok(MmuExceptionType::PermissionFault0),
+                    0xC9 => Ok(MmuExceptionType::PermissionFault1),
+                    0xCA => Ok(MmuExceptionType::PermissionFault2),
+                    0xCB => Ok(MmuExceptionType::PermissionFault3),
+                    0xD9 => Ok(MmuExceptionType::AccessFlag1),
+                    0xDA => Ok(MmuExceptionType::AccessFlag2),
+                    0xDB => Ok(MmuExceptionType::AccessFlag3),
+                    0xE0 => Ok(MmuExceptionType::AddressSizeFaultIn),
+                    0xE4 => Ok(MmuExceptionType::AddressSizeFaultOut0),
+                    0xE5 => Ok(MmuExceptionType::AddressSizeFaultOut1),
+                    0xE6 => Ok(MmuExceptionType::AddressSizeFaultOut2),
+                    0xE7 => Ok(MmuExceptionType::AddressSizeFaultOut3),
+                    0xE8 => Ok(MmuExceptionType::MemoryAttributeFault0),
+                    0xE9 => Ok(MmuExceptionType::MemoryAttributeFault1),
+                    0xEA => Ok(MmuExceptionType::MemoryAttributeFault2),
+                    0xEB => Ok(MmuExceptionType::MemoryAttributeFault3),
+                    _ => Err(EINVAL),
+                }
+            }
+        }
+
+        impl From<MmuExceptionType> for Bounded<u32, 8> {
+            fn from(exc: MmuExceptionType) -> Self {
+                (exc as u8).into()
+            }
+        }
+
+        /// Access type for MMU faults.
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        pub(crate) enum MmuAccessType {
+            /// An atomic (read/write) transaction.
+            Atomic = 0,
+            /// An execute transaction.
+            Execute = 1,
+            /// A read transaction.
+            Read = 2,
+            /// A write transaction.
+            Write = 3,
+        }
+
+        impl From<Bounded<u32, 2>> for MmuAccessType {
+            fn from(val: Bounded<u32, 2>) -> Self {
+                match val.get() {
+                    0 => MmuAccessType::Atomic,
+                    1 => MmuAccessType::Execute,
+                    2 => MmuAccessType::Read,
+                    3 => MmuAccessType::Write,
+                    _ => unreachable!(),
+                }
+            }
+        }
+
+        impl From<MmuAccessType> for Bounded<u32, 2> {
+            fn from(access: MmuAccessType) -> Self {
+                Bounded::try_new(access as u32).unwrap()
+            }
+        }
+
+        register! {
+            /// Fault status register for each address space. Read only.
+            pub(crate) FAULTSTATUS(u32)[MAX_AS, stride = STRIDE] @ 0x241c {
+                /// Exception type.
+                7:0     exception_type ?=> MmuExceptionType;
+                /// Access type.
+                9:8     access_type => MmuAccessType;
+                /// ID of the source that triggered the fault.
+                31:16   source_id;
+            }
+
+            /// Fault address for each address space. Read only.
+            pub(crate) FAULTADDRESS_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2420 {
+                31:0    pointer;
+            }
+
+            pub(crate) FAULTADDRESS_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2424 {
+                31:0    pointer;
+            }
+
+            /// MMU status register for each address space. Read only.
+            pub(crate) STATUS(u32)[MAX_AS, stride = STRIDE] @ 0x2428 {
+                /// External address space command is active, a 1-bit boolean flag.
+                0:0     active_ext => bool;
+                /// Internal address space command is active, a 1-bit boolean flag.
+                1:1     active_int => bool;
+            }
+        }
+
+        /// Helpers for TRANSCFG register.
+        ///
+        /// Address space mode for TRANSCFG register.
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        pub(crate) enum AddressSpaceMode {
+            /// The MMU forces all memory access to fail with a decode fault.
+            Unmapped = 1,
+            /// All input addresses map to the same output address (deprecated).
+            Identity = 2,
+            /// Translation tables interpreted according to AArch64 4kB granule specification.
+            Aarch64_4K = 6,
+            /// Translation tables interpreted according to AArch64 64kB granule specification.
+            Aarch64_64K = 8,
+        }
+
+        impl TryFrom<Bounded<u64, 4>> for AddressSpaceMode {
+            type Error = Error;
+
+            fn try_from(val: Bounded<u64, 4>) -> Result<Self, Self::Error> {
+                match val.get() {
+                    1 => Ok(AddressSpaceMode::Unmapped),
+                    2 => Ok(AddressSpaceMode::Identity),
+                    6 => Ok(AddressSpaceMode::Aarch64_4K),
+                    8 => Ok(AddressSpaceMode::Aarch64_64K),
+                    _ => Err(EINVAL),
+                }
+            }
+        }
+
+        impl From<AddressSpaceMode> for Bounded<u64, 4> {
+            fn from(mode: AddressSpaceMode) -> Self {
+                Bounded::try_new(mode as u64).unwrap()
+            }
+        }
+
+        /// Input address range restriction for TRANSCFG register.
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        pub(crate) enum InaBits {
+            /// Invalid VA range (reset value).
+            Reset = 0,
+            /// 48-bit VA range.
+            Bits48 = 7,
+            /// 47-bit VA range.
+            Bits47 = 8,
+            /// 46-bit VA range.
+            Bits46 = 9,
+            /// 45-bit VA range.
+            Bits45 = 10,
+            /// 44-bit VA range.
+            Bits44 = 11,
+            /// 43-bit VA range.
+            Bits43 = 12,
+            /// 42-bit VA range.
+            Bits42 = 13,
+            /// 41-bit VA range.
+            Bits41 = 14,
+            /// 40-bit VA range.
+            Bits40 = 15,
+            /// 39-bit VA range.
+            Bits39 = 16,
+            /// 38-bit VA range.
+            Bits38 = 17,
+            /// 37-bit VA range.
+            Bits37 = 18,
+            /// 36-bit VA range.
+            Bits36 = 19,
+            /// 35-bit VA range.
+            Bits35 = 20,
+            /// 34-bit VA range.
+            Bits34 = 21,
+            /// 33-bit VA range.
+            Bits33 = 22,
+            /// 32-bit VA range.
+            Bits32 = 23,
+            /// 31-bit VA range.
+            Bits31 = 24,
+            /// 30-bit VA range.
+            Bits30 = 25,
+            /// 29-bit VA range.
+            Bits29 = 26,
+            /// 28-bit VA range.
+            Bits28 = 27,
+            /// 27-bit VA range.
+            Bits27 = 28,
+            /// 26-bit VA range.
+            Bits26 = 29,
+            /// 25-bit VA range.
+            Bits25 = 30,
+        }
+
+        impl TryFrom<Bounded<u64, 5>> for InaBits {
+            type Error = Error;
+
+            fn try_from(val: Bounded<u64, 5>) -> Result<Self, Self::Error> {
+                match val.get() {
+                    0 => Ok(InaBits::Reset),
+                    7 => Ok(InaBits::Bits48),
+                    8 => Ok(InaBits::Bits47),
+                    9 => Ok(InaBits::Bits46),
+                    10 => Ok(InaBits::Bits45),
+                    11 => Ok(InaBits::Bits44),
+                    12 => Ok(InaBits::Bits43),
+                    13 => Ok(InaBits::Bits42),
+                    14 => Ok(InaBits::Bits41),
+                    15 => Ok(InaBits::Bits40),
+                    16 => Ok(InaBits::Bits39),
+                    17 => Ok(InaBits::Bits38),
+                    18 => Ok(InaBits::Bits37),
+                    19 => Ok(InaBits::Bits36),
+                    20 => Ok(InaBits::Bits35),
+                    21 => Ok(InaBits::Bits34),
+                    22 => Ok(InaBits::Bits33),
+                    23 => Ok(InaBits::Bits32),
+                    24 => Ok(InaBits::Bits31),
+                    25 => Ok(InaBits::Bits30),
+                    26 => Ok(InaBits::Bits29),
+                    27 => Ok(InaBits::Bits28),
+                    28 => Ok(InaBits::Bits27),
+                    29 => Ok(InaBits::Bits26),
+                    30 => Ok(InaBits::Bits25),
+                    _ => Err(EINVAL),
+                }
+            }
+        }
+
+        impl From<InaBits> for Bounded<u64, 5> {
+            fn from(bits: InaBits) -> Self {
+                Bounded::try_new(bits as u64).unwrap()
+            }
+        }
+
+        /// Translation table memory attributes for TRANSCFG register.
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        pub(crate) enum PtwMemattr {
+            /// Invalid (reset value, not valid for enabled address space).
+            Invalid = 0,
+            /// Normal memory, inner/outer non-cacheable.
+            NonCacheable = 1,
+            /// Normal memory, inner/outer write-back cacheable.
+            WriteBack = 2,
+        }
+
+        impl TryFrom<Bounded<u64, 2>> for PtwMemattr {
+            type Error = Error;
+
+            fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> {
+                match val.get() {
+                    0 => Ok(PtwMemattr::Invalid),
+                    1 => Ok(PtwMemattr::NonCacheable),
+                    2 => Ok(PtwMemattr::WriteBack),
+                    _ => Err(EINVAL),
+                }
+            }
+        }
+
+        impl From<PtwMemattr> for Bounded<u64, 2> {
+            fn from(attr: PtwMemattr) -> Self {
+                Bounded::try_new(attr as u64).unwrap()
+            }
+        }
+
+        /// Translation table memory shareability for TRANSCFG register.
+        #[derive(Copy, Clone, Debug, PartialEq)]
+        #[repr(u8)]
+        #[allow(clippy::enum_variant_names)]
+        pub(crate) enum PtwShareability {
+            /// Non-shareable.
+            NonShareable = 0,
+            /// Outer shareable.
+            OuterShareable = 2,
+            /// Inner shareable.
+            InnerShareable = 3,
+        }
+
+        impl TryFrom<Bounded<u64, 2>> for PtwShareability {
+            type Error = Error;
+
+            fn try_from(val: Bounded<u64, 2>) -> Result<Self, Self::Error> {
+                match val.get() {
+                    0 => Ok(PtwShareability::NonShareable),
+                    2 => Ok(PtwShareability::OuterShareable),
+                    3 => Ok(PtwShareability::InnerShareable),
+                    _ => Err(EINVAL),
+                }
+            }
+        }
+
+        impl From<PtwShareability> for Bounded<u64, 2> {
+            fn from(sh: PtwShareability) -> Self {
+                Bounded::try_new(sh as u64).unwrap()
+            }
+        }
+
+        register! {
+            /// Translation configuration and control.
+            pub(crate) TRANSCFG(u64)[MAX_AS, stride = STRIDE] @ 0x2430 {
+                /// Address space mode.
+                3:0     mode ?=> AddressSpaceMode;
+                /// Address input restriction.
+                10:6    ina_bits ?=> InaBits;
+                /// Address output restriction.
+                18:14   outa_bits;
+                /// Translation table concatenation enable, a 1-bit boolean flag.
+                22:22   sl_concat_en => bool;
+                /// Translation table memory attributes.
+                25:24   ptw_memattr ?=> PtwMemattr;
+                /// Translation table memory shareability.
+                29:28   ptw_sh ?=> PtwShareability;
+                /// Inner read allocation hint for translation table walks, a 1-bit boolean flag.
+                30:30   r_allocate => bool;
+                /// Disable hierarchical access permissions.
+                33:33   disable_hier_ap => bool;
+                /// Disable access fault checking.
+                34:34   disable_af_fault => bool;
+                /// Disable execution on all writable pages.
+                35:35   wxn => bool;
+                /// Enable execution on readable pages.
+                36:36   xreadable => bool;
+                /// Page-based hardware attributes for translation table walks.
+                63:60   ptw_pbha;
+            }
+
+            // TRANSCFG is a logical 64-bit register, but it is laid out in hardware as two
+            // 32-bit halves. Define it as separate low/high u32 registers so accesses match
+            // the MMIO register layout and do not rely on native 64-bit MMIO transactions.
+            pub(crate) TRANSCFG_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2430 {
+                31:0 value;
+            }
+
+            pub(crate) TRANSCFG_HI(u32)[MAX_AS, stride = STRIDE] @ 0x2434 {
+                31:0 value;
+            }
+
+            /// Extra fault information for each address space. Read only.
+            pub(crate) FAULTEXTRA_LO(u32)[MAX_AS, stride = STRIDE] @ 0x2438 {
+                31:0    value;
+            }
+
+            pub(crate) FAULTEXTRA_HI(u32)[MAX_AS, stride = STRIDE] @ 0x243c {
+                31:0    value;
+            }
+        }
+    }
+}
+
+/// This module corresponds to the DOORBELL_BLOCK_n[0-63] register pages.
+pub(crate) mod doorbell_block {
+    use kernel::register;
+
+    /// Number of doorbells available.
+    pub(crate) const NUM_DOORBELLS: usize = 64;
+
+    /// Doorbell block stride (64KiB).
+    ///
+    /// Each block occupies a full page, allowing it to be mapped
+    /// separately into a virtual address space.
+    const STRIDE: usize = 0x10000;
+
+    register! {
+        /// Doorbell request register. Write-only.
+        pub(crate) DOORBELL(u32)[NUM_DOORBELLS, stride = STRIDE] @ 0x80000 {
+            /// Doorbell set. Writing 1 triggers the doorbell.
+            0:0    ring => bool;
+        }
+    }
+}
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index 1ca073a4ecb25..289711035b67c 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -264,7 +264,7 @@ static const uint64_t udl_primary_plane_fmtmods[] = {
 };
 
 static int udl_primary_plane_helper_atomic_check(struct drm_plane *plane,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state, plane);
 	struct drm_crtc *new_crtc = new_plane_state->crtc;
@@ -280,7 +280,7 @@ static int udl_primary_plane_helper_atomic_check(struct drm_plane *plane,
 }
 
 static void udl_primary_plane_helper_atomic_update(struct drm_plane *plane,
-						   struct drm_atomic_state *state)
+						   struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = plane->dev;
 	struct drm_plane_state *plane_state = drm_atomic_get_new_plane_state(state, plane);
@@ -329,7 +329,7 @@ static const struct drm_plane_funcs udl_primary_plane_funcs = {
  * CRTC
  */
 
-static void udl_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state)
+static void udl_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct udl_device *udl = to_udl(dev);
@@ -366,7 +366,7 @@ static void udl_crtc_helper_atomic_enable(struct drm_crtc *crtc, struct drm_atom
 	drm_dev_exit(idx);
 }
 
-static void udl_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state)
+static void udl_crtc_helper_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct udl_device *udl = to_udl(dev);
diff --git a/drivers/gpu/drm/v3d/Makefile b/drivers/gpu/drm/v3d/Makefile
index b7d673f1153be..601b834e377e8 100644
--- a/drivers/gpu/drm/v3d/Makefile
+++ b/drivers/gpu/drm/v3d/Makefile
@@ -10,6 +10,7 @@ v3d-y := \
 	v3d_irq.o \
 	v3d_mmu.o \
 	v3d_perfmon.o \
+	v3d_power.o \
 	v3d_trace_points.o \
 	v3d_sched.o \
 	v3d_sysfs.o \
diff --git a/drivers/gpu/drm/v3d/v3d_debugfs.c b/drivers/gpu/drm/v3d/v3d_debugfs.c
index 89f24eec62a74..634cc796ba232 100644
--- a/drivers/gpu/drm/v3d/v3d_debugfs.c
+++ b/drivers/gpu/drm/v3d/v3d_debugfs.c
@@ -97,7 +97,11 @@ static int v3d_v3d_debugfs_regs(struct seq_file *m, void *unused)
 	struct drm_debugfs_entry *entry = m->private;
 	struct drm_device *dev = entry->dev;
 	struct v3d_dev *v3d = to_v3d_dev(dev);
-	int i, core;
+	int i, core, ret;
+
+	ret = v3d_pm_runtime_get(v3d);
+	if (ret)
+		return ret;
 
 	for (i = 0; i < ARRAY_SIZE(v3d_hub_reg_defs); i++) {
 		const struct v3d_reg_def *def = &v3d_hub_reg_defs[i];
@@ -139,6 +143,8 @@ static int v3d_v3d_debugfs_regs(struct seq_file *m, void *unused)
 		}
 	}
 
+	v3d_pm_runtime_put(v3d);
+
 	return 0;
 }
 
@@ -148,7 +154,11 @@ static int v3d_v3d_debugfs_ident(struct seq_file *m, void *unused)
 	struct drm_device *dev = entry->dev;
 	struct v3d_dev *v3d = to_v3d_dev(dev);
 	u32 ident0, ident1, ident2, ident3, cores;
-	int core;
+	int core, ret;
+
+	ret = v3d_pm_runtime_get(v3d);
+	if (ret)
+		return ret;
 
 	ident0 = V3D_READ(V3D_HUB_IDENT0);
 	ident1 = V3D_READ(V3D_HUB_IDENT1);
@@ -207,6 +217,8 @@ static int v3d_v3d_debugfs_ident(struct seq_file *m, void *unused)
 		}
 	}
 
+	v3d_pm_runtime_put(v3d);
+
 	return 0;
 }
 
@@ -234,6 +246,11 @@ static int v3d_measure_clock(struct seq_file *m, void *unused)
 	uint32_t cycles;
 	int core = 0;
 	int measure_ms = 1000;
+	int ret;
+
+	ret = v3d_pm_runtime_get(v3d);
+	if (ret)
+		return ret;
 
 	if (v3d->ver >= V3D_GEN_41) {
 		int cycle_count_reg = V3D_PCTR_CYCLE_COUNT(v3d->ver);
@@ -253,6 +270,8 @@ static int v3d_measure_clock(struct seq_file *m, void *unused)
 	msleep(measure_ms);
 	cycles = V3D_CORE_READ(core, V3D_PCTR_0_PCTR0);
 
+	v3d_pm_runtime_put(v3d);
+
 	seq_printf(m, "cycles: %d (%d.%d Mhz)\n",
 		   cycles,
 		   cycles / (measure_ms * 1000),
diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c
index 4b441afcb602d..fc81dd1247e33 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.c
+++ b/drivers/gpu/drm/v3d/v3d_drv.c
@@ -59,6 +59,7 @@ static int v3d_get_param_ioctl(struct drm_device *dev, void *data,
 		[DRM_V3D_PARAM_V3D_CORE0_IDENT1] = V3D_CTL_IDENT1,
 		[DRM_V3D_PARAM_V3D_CORE0_IDENT2] = V3D_CTL_IDENT2,
 	};
+	int ret;
 
 	if (args->pad != 0)
 		return -EINVAL;
@@ -75,12 +76,19 @@ static int v3d_get_param_ioctl(struct drm_device *dev, void *data,
 		if (args->value != 0)
 			return -EINVAL;
 
+		ret = v3d_pm_runtime_get(v3d);
+		if (ret)
+			return ret;
+
 		if (args->param >= DRM_V3D_PARAM_V3D_CORE0_IDENT0 &&
 		    args->param <= DRM_V3D_PARAM_V3D_CORE0_IDENT2) {
 			args->value = V3D_CORE_READ(0, offset);
 		} else {
 			args->value = V3D_READ(offset);
 		}
+
+		v3d_pm_runtime_put(v3d);
+
 		return 0;
 	}
 
@@ -290,36 +298,6 @@ static const struct of_device_id v3d_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, v3d_of_match);
 
-static void
-v3d_idle_sms(struct v3d_dev *v3d)
-{
-	if (v3d->ver < V3D_GEN_71)
-		return;
-
-	V3D_SMS_WRITE(V3D_SMS_TEE_CS, V3D_SMS_CLEAR_POWER_OFF);
-
-	if (wait_for((V3D_GET_FIELD(V3D_SMS_READ(V3D_SMS_TEE_CS),
-				    V3D_SMS_STATE) == V3D_SMS_IDLE), 100)) {
-		drm_err(&v3d->drm, "Failed to power up SMS\n");
-	}
-
-	v3d_reset_sms(v3d);
-}
-
-static void
-v3d_power_off_sms(struct v3d_dev *v3d)
-{
-	if (v3d->ver < V3D_GEN_71)
-		return;
-
-	V3D_SMS_WRITE(V3D_SMS_TEE_CS, V3D_SMS_POWER_OFF);
-
-	if (wait_for((V3D_GET_FIELD(V3D_SMS_READ(V3D_SMS_TEE_CS),
-				    V3D_SMS_STATE) == V3D_SMS_POWER_OFF_STATE), 100)) {
-		drm_err(&v3d->drm, "Failed to power off SMS\n");
-	}
-}
-
 static int
 map_regs(struct v3d_dev *v3d, void __iomem **regs, const char *name)
 {
@@ -363,23 +341,66 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
 			return ret;
 	}
 
+	if (v3d->ver < V3D_GEN_41) {
+		ret = map_regs(v3d, &v3d->gca_regs, "gca");
+		if (ret)
+			return ret;
+	}
+
+	v3d->reset = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(v3d->reset))
+		return dev_err_probe(dev, PTR_ERR(v3d->reset),
+				     "Failed to get reset control\n");
+
+	if (!v3d->reset) {
+		ret = map_regs(v3d, &v3d->bridge_regs, "bridge");
+		if (ret) {
+			dev_err(dev, "Failed to get bridge registers\n");
+			return ret;
+		}
+	}
+
 	v3d->clk = devm_clk_get_optional(dev, NULL);
 	if (IS_ERR(v3d->clk))
 		return dev_err_probe(dev, PTR_ERR(v3d->clk), "Failed to get V3D clock\n");
 
-	ret = clk_prepare_enable(v3d->clk);
-	if (ret) {
-		dev_err(&pdev->dev, "Couldn't enable the V3D clock\n");
+	ret = v3d_irq_init(v3d);
+	if (ret)
 		return ret;
+
+	v3d_perfmon_init(v3d);
+
+	v3d->mmu_scratch = dma_alloc_wc(dev, 4096, &v3d->mmu_scratch_paddr,
+					GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
+	if (!v3d->mmu_scratch) {
+		dev_err(dev, "Failed to allocate MMU scratch page\n");
+		return -ENOMEM;
 	}
 
-	v3d_idle_sms(v3d);
+	ret = v3d_gem_init(drm);
+	if (ret)
+		goto dma_free;
+
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		goto gem_destroy;
+
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		goto gem_destroy;
+
+	/* If PM is disabled, we need to call v3d_power_resume() manually. */
+	if (!IS_ENABLED(CONFIG_PM)) {
+		ret = v3d_power_resume(dev);
+		if (ret)
+			goto gem_destroy;
+	}
 
 	mmu_debug = V3D_READ(V3D_MMU_DEBUG_INFO);
 	mask = DMA_BIT_MASK(30 + V3D_GET_FIELD(mmu_debug, V3D_MMU_PA_WIDTH));
 	ret = dma_set_mask_and_coherent(dev, mask);
 	if (ret)
-		goto clk_disable;
+		goto runtime_pm_put;
 
 	dma_set_max_seg_size(&pdev->dev, UINT_MAX);
 
@@ -399,66 +420,30 @@ static int v3d_platform_drm_probe(struct platform_device *pdev)
 	ident3 = V3D_READ(V3D_HUB_IDENT3);
 	v3d->rev = V3D_GET_FIELD(ident3, V3D_HUB_IDENT3_IPREV);
 
-	v3d_perfmon_init(v3d);
-
-	v3d->reset = devm_reset_control_get_exclusive(dev, NULL);
-	if (IS_ERR(v3d->reset)) {
-		ret = PTR_ERR(v3d->reset);
-
-		if (ret == -EPROBE_DEFER)
-			goto clk_disable;
-
-		v3d->reset = NULL;
-		ret = map_regs(v3d, &v3d->bridge_regs, "bridge");
-		if (ret) {
-			dev_err(dev,
-				"Failed to get reset control or bridge regs\n");
-			goto clk_disable;
-		}
-	}
-
-	if (v3d->ver < V3D_GEN_41) {
-		ret = map_regs(v3d, &v3d->gca_regs, "gca");
-		if (ret)
-			goto clk_disable;
-	}
-
-	v3d->mmu_scratch = dma_alloc_wc(dev, 4096, &v3d->mmu_scratch_paddr,
-					GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
-	if (!v3d->mmu_scratch) {
-		dev_err(dev, "Failed to allocate MMU scratch page\n");
-		ret = -ENOMEM;
-		goto clk_disable;
-	}
-
-	ret = v3d_gem_init(drm);
-	if (ret)
-		goto dma_free;
-
-	ret = v3d_irq_init(v3d);
-	if (ret)
-		goto gem_destroy;
+	pm_runtime_set_autosuspend_delay(dev, 100);
+	pm_runtime_use_autosuspend(dev);
 
 	ret = drm_dev_register(drm, 0);
 	if (ret)
-		goto irq_disable;
+		goto runtime_pm_put;
 
 	ret = v3d_sysfs_init(dev);
 	if (ret)
 		goto drm_unregister;
 
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_put_autosuspend(dev);
+
 	return 0;
 
 drm_unregister:
 	drm_dev_unregister(drm);
-irq_disable:
-	v3d_irq_disable(v3d);
+runtime_pm_put:
+	pm_runtime_put_sync_suspend(dev);
 gem_destroy:
 	v3d_gem_destroy(drm);
 dma_free:
 	dma_free_wc(dev, 4096, v3d->mmu_scratch, v3d->mmu_scratch_paddr);
-clk_disable:
-	clk_disable_unprepare(v3d->clk);
 	return ret;
 }
 
@@ -472,22 +457,27 @@ static void v3d_platform_drm_remove(struct platform_device *pdev)
 
 	drm_dev_unregister(drm);
 
-	v3d_gem_destroy(drm);
+	pm_runtime_suspend(dev);
 
-	dma_free_wc(v3d->drm.dev, 4096, v3d->mmu_scratch,
-		    v3d->mmu_scratch_paddr);
+	/* If PM is disabled, we need to call v3d_power_suspend() manually. */
+	if (!IS_ENABLED(CONFIG_PM))
+		v3d_power_suspend(dev);
 
-	v3d_power_off_sms(v3d);
+	v3d_gem_destroy(drm);
 
-	clk_disable_unprepare(v3d->clk);
+	dma_free_wc(dev, 4096, v3d->mmu_scratch, v3d->mmu_scratch_paddr);
 }
 
+static DEFINE_RUNTIME_DEV_PM_OPS(v3d_pm_ops, v3d_power_suspend,
+				 v3d_power_resume, NULL);
+
 static struct platform_driver v3d_platform_driver = {
 	.probe		= v3d_platform_drm_probe,
 	.remove		= v3d_platform_drm_remove,
 	.driver		= {
 		.name	= "v3d",
 		.of_match_table = v3d_of_match,
+		.pm = pm_ptr(&v3d_pm_ops),
 	},
 };
 
diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h
index 6a3cad9334398..4ebe175a8c6b0 100644
--- a/drivers/gpu/drm/v3d/v3d_drv.h
+++ b/drivers/gpu/drm/v3d/v3d_drv.h
@@ -3,6 +3,7 @@
 
 #include <linux/delay.h>
 #include <linux/mutex.h>
+#include <linux/pm_runtime.h>
 #include <linux/spinlock_types.h>
 #include <linux/workqueue.h>
 
@@ -324,6 +325,8 @@ struct v3d_job {
 
 	/* Callback for the freeing of the job on refcount going to 0. */
 	void (*free)(struct kref *ref);
+
+	bool has_pm_ref;
 };
 
 struct v3d_bin_job {
@@ -565,6 +568,7 @@ struct dma_fence *v3d_fence_create(struct v3d_dev *v3d, enum v3d_queue q);
 
 /* v3d_gem.c */
 extern bool super_pages;
+void v3d_init_hw_state(struct v3d_dev *v3d);
 int v3d_gem_init(struct drm_device *dev);
 void v3d_gem_destroy(struct drm_device *dev);
 void v3d_reset_sms(struct v3d_dev *v3d);
@@ -596,6 +600,20 @@ int v3d_mmu_set_page_table(struct v3d_dev *v3d);
 void v3d_mmu_insert_ptes(struct v3d_bo *bo);
 void v3d_mmu_remove_ptes(struct v3d_bo *bo);
 
+/* v3d_power.c */
+int v3d_power_suspend(struct device *dev);
+int v3d_power_resume(struct device *dev);
+
+static __always_inline int v3d_pm_runtime_get(struct v3d_dev *v3d)
+{
+	return pm_runtime_resume_and_get(v3d->drm.dev);
+}
+
+static __always_inline int v3d_pm_runtime_put(struct v3d_dev *v3d)
+{
+	return pm_runtime_put_autosuspend(v3d->drm.dev);
+}
+
 /* v3d_sched.c */
 void v3d_timestamp_query_info_free(struct v3d_timestamp_query_info *query_info,
 				   unsigned int count);
diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c
index 75d9eccd79666..1ee3c038d5f6f 100644
--- a/drivers/gpu/drm/v3d/v3d_gem.c
+++ b/drivers/gpu/drm/v3d/v3d_gem.c
@@ -36,13 +36,6 @@ v3d_init_core(struct v3d_dev *v3d, int core)
 	V3D_CORE_WRITE(core, V3D_CTL_L2TFLEND, ~0);
 }
 
-/* Sets invariant state for the HW. */
-static void
-v3d_init_hw_state(struct v3d_dev *v3d)
-{
-	v3d_init_core(v3d, 0);
-}
-
 static void
 v3d_idle_axi(struct v3d_dev *v3d, int core)
 {
@@ -259,6 +252,13 @@ v3d_invalidate_caches(struct v3d_dev *v3d)
 	v3d_invalidate_slices(v3d, 0);
 }
 
+/* Sets invariant state for the HW. */
+void
+v3d_init_hw_state(struct v3d_dev *v3d)
+{
+	v3d_init_core(v3d, 0);
+}
+
 static void
 v3d_huge_mnt_init(struct v3d_dev *v3d)
 {
@@ -328,9 +328,6 @@ v3d_gem_init(struct drm_device *dev)
 		goto err_dma_alloc;
 	}
 
-	v3d_init_hw_state(v3d);
-	v3d_mmu_set_page_table(v3d);
-
 	v3d_huge_mnt_init(v3d);
 
 	ret = v3d_sched_init(v3d);
diff --git a/drivers/gpu/drm/v3d/v3d_irq.c b/drivers/gpu/drm/v3d/v3d_irq.c
index c28e74ab54428..86efaef2722c3 100644
--- a/drivers/gpu/drm/v3d/v3d_irq.c
+++ b/drivers/gpu/drm/v3d/v3d_irq.c
@@ -248,17 +248,10 @@ v3d_hub_irq(int irq, void *arg)
 int
 v3d_irq_init(struct v3d_dev *v3d)
 {
-	int irq, ret, core;
+	int irq, ret;
 
 	INIT_WORK(&v3d->overflow_mem_work, v3d_overflow_mem_work);
 
-	/* Clear any pending interrupts someone might have left around
-	 * for us.
-	 */
-	for (core = 0; core < v3d->cores; core++)
-		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
-	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
-
 	irq = platform_get_irq_optional(v3d_to_pdev(v3d), 1);
 	if (irq == -EPROBE_DEFER)
 		return irq;
@@ -296,7 +289,6 @@ v3d_irq_init(struct v3d_dev *v3d)
 			goto fail;
 	}
 
-	v3d_irq_enable(v3d);
 	return 0;
 
 fail:
@@ -310,6 +302,11 @@ v3d_irq_enable(struct v3d_dev *v3d)
 {
 	int core;
 
+	/* Clear any pending interrupts someone might have left around for us. */
+	for (core = 0; core < v3d->cores; core++)
+		V3D_CORE_WRITE(core, V3D_CTL_INT_CLR, V3D_CORE_IRQS(v3d->ver));
+	V3D_WRITE(V3D_HUB_INT_CLR, V3D_HUB_IRQS(v3d->ver));
+
 	/* Enable our set of interrupts, masking out any others. */
 	for (core = 0; core < v3d->cores; core++) {
 		V3D_CORE_WRITE(core, V3D_CTL_INT_MSK_SET, ~V3D_CORE_IRQS(v3d->ver));
diff --git a/drivers/gpu/drm/v3d/v3d_mmu.c b/drivers/gpu/drm/v3d/v3d_mmu.c
index c513a393c0313..630c64e51d2f2 100644
--- a/drivers/gpu/drm/v3d/v3d_mmu.c
+++ b/drivers/gpu/drm/v3d/v3d_mmu.c
@@ -39,7 +39,11 @@ static bool v3d_mmu_is_aligned(u32 page, u32 page_address, size_t alignment)
 
 int v3d_mmu_flush_all(struct v3d_dev *v3d)
 {
-	int ret;
+	int ret = 0;
+
+	/* Flush the PTs only if we're already awake */
+	if (!pm_runtime_get_if_active(v3d->drm.dev))
+		return 0;
 
 	V3D_WRITE(V3D_MMUC_CONTROL, V3D_MMUC_CONTROL_FLUSH |
 		  V3D_MMUC_CONTROL_ENABLE);
@@ -48,7 +52,7 @@ int v3d_mmu_flush_all(struct v3d_dev *v3d)
 			 V3D_MMUC_CONTROL_FLUSHING), 100);
 	if (ret) {
 		dev_err(v3d->drm.dev, "MMUC flush wait idle failed\n");
-		return ret;
+		goto pm_put;
 	}
 
 	V3D_WRITE(V3D_MMU_CTL, V3D_READ(V3D_MMU_CTL) |
@@ -59,6 +63,8 @@ int v3d_mmu_flush_all(struct v3d_dev *v3d)
 	if (ret)
 		dev_err(v3d->drm.dev, "MMU TLB clear wait idle failed\n");
 
+pm_put:
+	v3d_pm_runtime_put(v3d);
 	return ret;
 }
 
diff --git a/drivers/gpu/drm/v3d/v3d_perfmon.c b/drivers/gpu/drm/v3d/v3d_perfmon.c
index 8e0249580bbac..02451fc09dbbf 100644
--- a/drivers/gpu/drm/v3d/v3d_perfmon.c
+++ b/drivers/gpu/drm/v3d/v3d_perfmon.c
@@ -232,6 +232,9 @@ void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
 	if (WARN_ON_ONCE(!perfmon || v3d->active_perfmon))
 		return;
 
+	if (!pm_runtime_get_if_active(v3d->drm.dev))
+		return;
+
 	ncounters = perfmon->ncounters;
 	mask = GENMASK(ncounters - 1, 0);
 
@@ -257,6 +260,8 @@ void v3d_perfmon_start(struct v3d_dev *v3d, struct v3d_perfmon *perfmon)
 	V3D_CORE_WRITE(0, V3D_PCTR_0_OVERFLOW, mask);
 
 	v3d->active_perfmon = perfmon;
+
+	v3d_pm_runtime_put(v3d);
 }
 
 void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
@@ -268,10 +273,11 @@ void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
 		return;
 
 	mutex_lock(&perfmon->lock);
-	if (perfmon != v3d->active_perfmon) {
-		mutex_unlock(&perfmon->lock);
-		return;
-	}
+	if (perfmon != v3d->active_perfmon)
+		goto out;
+
+	if (!pm_runtime_get_if_active(v3d->drm.dev))
+		goto out_clear;
 
 	if (capture)
 		for (i = 0; i < perfmon->ncounters; i++)
@@ -279,7 +285,11 @@ void v3d_perfmon_stop(struct v3d_dev *v3d, struct v3d_perfmon *perfmon,
 
 	V3D_CORE_WRITE(0, V3D_V4_PCTR_0_EN, 0);
 
+	v3d_pm_runtime_put(v3d);
+
+out_clear:
 	v3d->active_perfmon = NULL;
+out:
 	mutex_unlock(&perfmon->lock);
 }
 
diff --git a/drivers/gpu/drm/v3d/v3d_power.c b/drivers/gpu/drm/v3d/v3d_power.c
new file mode 100644
index 0000000000000..769e90032b042
--- /dev/null
+++ b/drivers/gpu/drm/v3d/v3d_power.c
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: GPL-2.0+
+/* Copyright (C) 2026 Raspberry Pi */
+
+#include <linux/clk.h>
+
+#include <drm/drm_print.h>
+
+#include "v3d_drv.h"
+#include "v3d_regs.h"
+
+static int
+v3d_resume_sms(struct v3d_dev *v3d)
+{
+	if (v3d->ver < V3D_GEN_71)
+		return 0;
+
+	V3D_SMS_WRITE(V3D_SMS_TEE_CS, V3D_SMS_CLEAR_POWER_OFF);
+
+	if (wait_for((V3D_GET_FIELD(V3D_SMS_READ(V3D_SMS_TEE_CS),
+				    V3D_SMS_STATE) == V3D_SMS_IDLE), 100)) {
+		drm_err(&v3d->drm, "Failed to power up SMS\n");
+		return -ETIMEDOUT;
+	}
+
+	v3d_reset_sms(v3d);
+
+	return 0;
+}
+
+static int
+v3d_suspend_sms(struct v3d_dev *v3d)
+{
+	if (v3d->ver < V3D_GEN_71)
+		return 0;
+
+	V3D_SMS_WRITE(V3D_SMS_TEE_CS, V3D_SMS_POWER_OFF);
+
+	if (wait_for((V3D_GET_FIELD(V3D_SMS_READ(V3D_SMS_TEE_CS),
+				    V3D_SMS_STATE) == V3D_SMS_POWER_OFF_STATE), 100)) {
+		drm_err(&v3d->drm, "Failed to power off SMS\n");
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+int v3d_power_suspend(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+	int ret;
+
+	v3d_irq_disable(v3d);
+
+	ret = v3d_suspend_sms(v3d);
+	if (ret) {
+		v3d_irq_enable(v3d);
+		return ret;
+	}
+
+	clk_disable_unprepare(v3d->clk);
+
+	return 0;
+}
+
+int v3d_power_resume(struct device *dev)
+{
+	struct drm_device *drm = dev_get_drvdata(dev);
+	struct v3d_dev *v3d = to_v3d_dev(drm);
+	int ret;
+
+	ret = clk_prepare_enable(v3d->clk);
+	if (ret)
+		return ret;
+
+	ret = v3d_resume_sms(v3d);
+	if (ret) {
+		clk_disable_unprepare(v3d->clk);
+		return ret;
+	}
+
+	v3d_init_hw_state(v3d);
+	v3d_mmu_set_page_table(v3d);
+	v3d_irq_enable(v3d);
+
+	return 0;
+}
diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c
index 1855ef5b3b5fe..0e266b29317f3 100644
--- a/drivers/gpu/drm/v3d/v3d_sched.c
+++ b/drivers/gpu/drm/v3d/v3d_sched.c
@@ -125,20 +125,6 @@ v3d_performance_query_info_free(struct v3d_performance_query_info *query_info,
 	}
 }
 
-static void
-v3d_cpu_job_free(struct drm_sched_job *sched_job)
-{
-	struct v3d_cpu_job *job = to_cpu_job(sched_job);
-
-	v3d_timestamp_query_info_free(&job->timestamp_query,
-				      job->timestamp_query.count);
-
-	v3d_performance_query_info_free(&job->performance_query,
-					job->performance_query.count);
-
-	v3d_job_cleanup(&job->base);
-}
-
 static void
 v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job)
 {
@@ -830,7 +816,7 @@ static const struct drm_sched_backend_ops v3d_cache_clean_sched_ops = {
 
 static const struct drm_sched_backend_ops v3d_cpu_sched_ops = {
 	.run_job = v3d_cpu_job_run,
-	.free_job = v3d_cpu_job_free
+	.free_job = v3d_sched_job_free
 };
 
 static int
@@ -838,7 +824,6 @@ v3d_queue_sched_init(struct v3d_dev *v3d, const struct drm_sched_backend_ops *op
 		     enum v3d_queue queue, const char *name)
 {
 	struct drm_sched_init_args args = {
-		.num_rqs = DRM_SCHED_PRIORITY_COUNT,
 		.credit_limit = 1,
 		.timeout = msecs_to_jiffies(500),
 		.dev = v3d->drm.dev,
diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c
index ee4512db294b3..1db43c6a078d5 100644
--- a/drivers/gpu/drm/v3d/v3d_submit.c
+++ b/drivers/gpu/drm/v3d/v3d_submit.c
@@ -106,6 +106,9 @@ v3d_job_free(struct kref *ref)
 	v3d_stats_put(job->client_stats);
 	v3d_stats_put(job->global_stats);
 
+	if (job->has_pm_ref)
+		v3d_pm_runtime_put(job->v3d);
+
 	kfree(job);
 }
 
@@ -123,6 +126,24 @@ v3d_render_job_free(struct kref *ref)
 	v3d_job_free(ref);
 }
 
+static void
+v3d_cpu_job_free(struct kref *ref)
+{
+	struct v3d_cpu_job *job = container_of(ref, struct v3d_cpu_job,
+					       base.refcount);
+
+	v3d_timestamp_query_info_free(&job->timestamp_query,
+				      job->timestamp_query.count);
+
+	v3d_performance_query_info_free(&job->performance_query,
+					job->performance_query.count);
+
+	if (job->indirect_csd.indirect)
+		drm_gem_object_put(job->indirect_csd.indirect);
+
+	v3d_job_free(ref);
+}
+
 void v3d_job_cleanup(struct v3d_job *job)
 {
 	if (!job)
@@ -187,13 +208,13 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 				if (copy_from_user(&in, handle++, sizeof(in))) {
 					ret = -EFAULT;
 					drm_dbg(&v3d->drm, "Failed to copy wait dep handle.\n");
-					goto fail_deps;
+					goto fail_job_init;
 				}
 				ret = drm_sched_job_add_syncobj_dependency(&job->base, file_priv, in.handle, 0);
 
 				// TODO: Investigate why this was filtered out for the IOCTL.
 				if (ret && ret != -ENOENT)
-					goto fail_deps;
+					goto fail_job_init;
 			}
 		}
 	} else {
@@ -201,7 +222,15 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 
 		// TODO: Investigate why this was filtered out for the IOCTL.
 		if (ret && ret != -ENOENT)
-			goto fail_deps;
+			goto fail_job_init;
+	}
+
+	/* CPU jobs don't require hardware resources */
+	if (queue != V3D_CPU) {
+		ret = v3d_pm_runtime_get(v3d);
+		if (ret)
+			goto fail_job_init;
+		job->has_pm_ref = true;
 	}
 
 	kref_init(&job->refcount);
@@ -211,7 +240,7 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv,
 
 	return 0;
 
-fail_deps:
+fail_job_init:
 	drm_sched_job_cleanup(&job->base);
 	return ret;
 }
@@ -1302,7 +1331,7 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
 	trace_v3d_submit_cpu_ioctl(&v3d->drm, cpu_job->job_type);
 
 	ret = v3d_job_init(v3d, file_priv, &cpu_job->base,
-			   v3d_job_free, 0, &se, V3D_CPU);
+			   v3d_cpu_job_free, 0, &se, V3D_CPU);
 	if (ret) {
 		v3d_job_deallocate((void *)&cpu_job);
 		goto fail;
@@ -1385,8 +1414,6 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data,
 	v3d_job_cleanup((void *)csd_job);
 	v3d_job_cleanup(clean_job);
 	v3d_put_multisync_post_deps(&se);
-	kvfree(cpu_job->timestamp_query.queries);
-	kvfree(cpu_job->performance_query.queries);
 
 	return ret;
 }
diff --git a/drivers/gpu/drm/vboxvideo/vbox_mode.c b/drivers/gpu/drm/vboxvideo/vbox_mode.c
index 9c11a6b945ff4..8e4e5fc9d3c5a 100644
--- a/drivers/gpu/drm/vboxvideo/vbox_mode.c
+++ b/drivers/gpu/drm/vboxvideo/vbox_mode.c
@@ -219,17 +219,17 @@ static void vbox_crtc_set_base_and_mode(struct drm_crtc *crtc,
 }
 
 static void vbox_crtc_atomic_enable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 }
 
 static void vbox_crtc_atomic_disable(struct drm_crtc *crtc,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 }
 
 static void vbox_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 }
 
@@ -256,7 +256,7 @@ static const struct drm_crtc_funcs vbox_crtc_funcs = {
 };
 
 static int vbox_primary_atomic_check(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -276,7 +276,7 @@ static int vbox_primary_atomic_check(struct drm_plane *plane,
 }
 
 static void vbox_primary_atomic_update(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -322,7 +322,7 @@ static void vbox_primary_atomic_update(struct drm_plane *plane,
 }
 
 static void vbox_primary_atomic_disable(struct drm_plane *plane,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -335,7 +335,7 @@ static void vbox_primary_atomic_disable(struct drm_plane *plane,
 }
 
 static int vbox_cursor_atomic_check(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -387,7 +387,7 @@ static void copy_cursor_image(u8 *src, u8 *dst, u32 width, u32 height,
 }
 
 static void vbox_cursor_atomic_update(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -438,7 +438,7 @@ static void vbox_cursor_atomic_update(struct drm_plane *plane,
 }
 
 static void vbox_cursor_atomic_disable(struct drm_plane *plane,
-				       struct drm_atomic_state *state)
+				       struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
diff --git a/drivers/gpu/drm/vc4/tests/vc4_mock.h b/drivers/gpu/drm/vc4/tests/vc4_mock.h
index 002b6218960c4..3cc4b4a0feed2 100644
--- a/drivers/gpu/drm/vc4/tests/vc4_mock.h
+++ b/drivers/gpu/drm/vc4/tests/vc4_mock.h
@@ -52,10 +52,10 @@ struct vc4_dev *vc4_mock_device(struct kunit *test);
 struct vc4_dev *vc5_mock_device(struct kunit *test);
 
 int vc4_mock_atomic_add_output(struct kunit *test,
-			       struct drm_atomic_state *state,
+			       struct drm_atomic_commit *state,
 			       enum vc4_encoder_type type);
 int vc4_mock_atomic_del_output(struct kunit *test,
-			       struct drm_atomic_state *state,
+			       struct drm_atomic_commit *state,
 			       enum vc4_encoder_type type);
 
 #endif // VC4_MOCK_H_
diff --git a/drivers/gpu/drm/vc4/tests/vc4_mock_output.c b/drivers/gpu/drm/vc4/tests/vc4_mock_output.c
index 577d9a9563696..b53c3635ce673 100644
--- a/drivers/gpu/drm/vc4/tests/vc4_mock_output.c
+++ b/drivers/gpu/drm/vc4/tests/vc4_mock_output.c
@@ -75,7 +75,7 @@ static const struct drm_display_mode default_mode = {
  * errors are fatal.
  */
 int vc4_mock_atomic_add_output(struct kunit *test,
-			       struct drm_atomic_state *state,
+			       struct drm_atomic_commit *state,
 			       enum vc4_encoder_type type)
 {
 	struct drm_device *drm = state->dev;
@@ -132,7 +132,7 @@ int vc4_mock_atomic_add_output(struct kunit *test,
  * errors are fatal.
  */
 int vc4_mock_atomic_del_output(struct kunit *test,
-			       struct drm_atomic_state *state,
+			       struct drm_atomic_commit *state,
 			       enum vc4_encoder_type type)
 {
 	struct drm_device *drm = state->dev;
diff --git a/drivers/gpu/drm/vc4/tests/vc4_test_pv_muxing.c b/drivers/gpu/drm/vc4/tests/vc4_test_pv_muxing.c
index d1f694029169a..7a54f46acff91 100644
--- a/drivers/gpu/drm/vc4/tests/vc4_test_pv_muxing.c
+++ b/drivers/gpu/drm/vc4/tests/vc4_test_pv_muxing.c
@@ -23,7 +23,7 @@ struct pv_muxing_priv {
 };
 
 static bool check_fifo_conflict(struct kunit *test,
-				const struct drm_atomic_state *state)
+				const struct drm_atomic_commit *state)
 {
 	struct vc4_hvs_state *hvs_state;
 	unsigned int used_fifos = 0;
@@ -119,7 +119,7 @@ static bool check_vc5_encoder_constraints(enum vc4_encoder_type type, unsigned i
 
 static struct vc4_crtc_state *
 get_vc4_crtc_state_for_encoder(struct kunit *test,
-			       const struct drm_atomic_state *state,
+			       const struct drm_atomic_commit *state,
 			       enum vc4_encoder_type type)
 {
 	struct drm_device *drm = state->dev;
@@ -141,7 +141,7 @@ get_vc4_crtc_state_for_encoder(struct kunit *test,
 }
 
 static bool check_channel_for_encoder(struct kunit *test,
-				      const struct drm_atomic_state *state,
+				      const struct drm_atomic_commit *state,
 				      enum vc4_encoder_type type,
 				      bool (*check_fn)(enum vc4_encoder_type type, unsigned int channel))
 {
@@ -677,7 +677,7 @@ static void drm_vc4_test_pv_muxing(struct kunit *test)
 	const struct pv_muxing_param *params = test->param_value;
 	const struct pv_muxing_priv *priv = test->priv;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_device *drm;
 	struct vc4_dev *vc4;
 	unsigned int i;
@@ -696,7 +696,7 @@ static void drm_vc4_test_pv_muxing(struct kunit *test)
 
 		ret = vc4_mock_atomic_add_output(test, state, enc_type);
 		if (ret == -EDEADLK) {
-			drm_atomic_state_clear(state);
+			drm_atomic_commit_clear(state);
 			ret = drm_modeset_backoff(&ctx);
 			if (!ret)
 				goto retry;
@@ -706,7 +706,7 @@ static void drm_vc4_test_pv_muxing(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry;
@@ -732,7 +732,7 @@ static void drm_vc4_test_pv_muxing_invalid(struct kunit *test)
 	const struct pv_muxing_param *params = test->param_value;
 	const struct pv_muxing_priv *priv = test->priv;
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct drm_device *drm;
 	struct vc4_dev *vc4;
 	unsigned int i;
@@ -752,7 +752,7 @@ static void drm_vc4_test_pv_muxing_invalid(struct kunit *test)
 
 		ret = vc4_mock_atomic_add_output(test, state, enc_type);
 		if (ret == -EDEADLK) {
-			drm_atomic_state_clear(state);
+			drm_atomic_commit_clear(state);
 			ret = drm_modeset_backoff(&ctx);
 			if (!ret)
 				goto retry;
@@ -762,7 +762,7 @@ static void drm_vc4_test_pv_muxing_invalid(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry;
@@ -826,7 +826,7 @@ static struct kunit_suite vc5_pv_muxing_test_suite = {
 static void drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable(struct kunit *test)
 {
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct vc4_crtc_state *new_vc4_crtc_state;
 	struct vc4_hvs_state *new_hvs_state;
 	unsigned int hdmi0_channel;
@@ -847,7 +847,7 @@ static void drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable(struct kunit *tes
 
 	ret = vc4_mock_atomic_add_output(test, state, VC4_ENCODER_TYPE_HDMI0);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_first;
@@ -856,7 +856,7 @@ static void drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable(struct kunit *tes
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_first;
@@ -883,7 +883,7 @@ static void drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable(struct kunit *tes
 
 	ret = vc4_mock_atomic_add_output(test, state, VC4_ENCODER_TYPE_HDMI1);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_second;
@@ -892,7 +892,7 @@ static void drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable(struct kunit *tes
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_second;
@@ -926,7 +926,7 @@ static void drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable(struct kunit *tes
 static void drm_test_vc5_pv_muxing_bugs_stable_fifo(struct kunit *test)
 {
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct vc4_crtc_state *new_vc4_crtc_state;
 	struct vc4_hvs_state *new_hvs_state;
 	unsigned int old_hdmi0_channel;
@@ -947,7 +947,7 @@ static void drm_test_vc5_pv_muxing_bugs_stable_fifo(struct kunit *test)
 
 	ret = vc4_mock_atomic_add_output(test, state, VC4_ENCODER_TYPE_HDMI0);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_first;
@@ -956,7 +956,7 @@ static void drm_test_vc5_pv_muxing_bugs_stable_fifo(struct kunit *test)
 
 	ret = vc4_mock_atomic_add_output(test, state, VC4_ENCODER_TYPE_HDMI1);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_first;
@@ -965,7 +965,7 @@ static void drm_test_vc5_pv_muxing_bugs_stable_fifo(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_first;
@@ -1000,7 +1000,7 @@ static void drm_test_vc5_pv_muxing_bugs_stable_fifo(struct kunit *test)
 
 	ret = vc4_mock_atomic_del_output(test, state, VC4_ENCODER_TYPE_HDMI0);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_second;
@@ -1009,7 +1009,7 @@ static void drm_test_vc5_pv_muxing_bugs_stable_fifo(struct kunit *test)
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_second;
@@ -1055,7 +1055,7 @@ static void
 drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable_too_many_crtc_state(struct kunit *test)
 {
 	struct drm_modeset_acquire_ctx ctx;
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 	struct vc4_crtc_state *new_vc4_crtc_state;
 	struct drm_device *drm;
 	struct vc4_dev *vc4;
@@ -1073,7 +1073,7 @@ drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable_too_many_crtc_state(struct ku
 
 	ret = vc4_mock_atomic_add_output(test, state, VC4_ENCODER_TYPE_HDMI0);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_first;
@@ -1082,7 +1082,7 @@ drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable_too_many_crtc_state(struct ku
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_first;
@@ -1097,7 +1097,7 @@ drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable_too_many_crtc_state(struct ku
 
 	ret = vc4_mock_atomic_add_output(test, state, VC4_ENCODER_TYPE_HDMI1);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_second;
@@ -1106,7 +1106,7 @@ drm_test_vc5_pv_muxing_bugs_subsequent_crtc_enable_too_many_crtc_state(struct ku
 
 	ret = drm_atomic_check_only(state);
 	if (ret == -EDEADLK) {
-		drm_atomic_state_clear(state);
+		drm_atomic_commit_clear(state);
 		ret = drm_modeset_backoff(&ctx);
 		if (!ret)
 			goto retry_second;
diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
index 9377e58f9bc25..2161761b1f221 100644
--- a/drivers/gpu/drm/vc4/vc4_bo.c
+++ b/drivers/gpu/drm/vc4/vc4_bo.c
@@ -22,7 +22,6 @@
 #include <drm/drm_print.h>
 
 #include "vc4_drv.h"
-#include "uapi/drm/vc4_drm.h"
 
 static const struct drm_gem_object_funcs vc4_gem_object_funcs;
 
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index 4a606986afcc7..4572b0b9bae1c 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -343,7 +343,7 @@ static void vc4_crtc_pixelvalve_reset(struct drm_crtc *crtc)
 }
 
 static void vc4_crtc_config_pv(struct drm_crtc *crtc, struct drm_encoder *encoder,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
@@ -497,7 +497,7 @@ static void require_hvs_enabled(struct drm_device *dev)
 
 static int vc4_crtc_disable(struct drm_crtc *crtc,
 			    struct drm_encoder *encoder,
-			    struct drm_atomic_state *state,
+			    struct drm_atomic_commit *state,
 			    unsigned int channel)
 {
 	struct vc4_encoder *vc4_encoder = to_vc4_encoder(encoder);
@@ -622,7 +622,7 @@ void vc4_crtc_send_vblank(struct drm_crtc *crtc)
 }
 
 static void vc4_crtc_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 									 crtc);
@@ -648,7 +648,7 @@ static void vc4_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void vc4_crtc_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *new_state = drm_atomic_get_new_crtc_state(state,
 									 crtc);
@@ -740,7 +740,7 @@ void vc4_crtc_get_margins(struct drm_crtc_state *state,
 }
 
 int vc4_crtc_atomic_check(struct drm_crtc *crtc,
-			  struct drm_atomic_state *state)
+			  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index a14ecb7694613..616caf9d9915b 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -41,8 +41,6 @@
 
 #include <soc/bcm2835/raspberrypi-firmware.h>
 
-#include "uapi/drm/vc4_drm.h"
-
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index dbcc83b7df004..0f5958c1e6b60 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -375,9 +375,9 @@ struct vc4_hvs_state {
 #define to_vc4_hvs_state(_state)				\
 	container_of_const(_state, struct vc4_hvs_state, base)
 
-struct vc4_hvs_state *vc4_hvs_get_global_state(struct drm_atomic_state *state);
-struct vc4_hvs_state *vc4_hvs_get_old_global_state(const struct drm_atomic_state *state);
-struct vc4_hvs_state *vc4_hvs_get_new_global_state(const struct drm_atomic_state *state);
+struct vc4_hvs_state *vc4_hvs_get_global_state(struct drm_atomic_commit *state);
+struct vc4_hvs_state *vc4_hvs_get_old_global_state(const struct drm_atomic_commit *state);
+struct vc4_hvs_state *vc4_hvs_get_new_global_state(const struct drm_atomic_commit *state);
 
 struct vc4_plane {
 	struct drm_plane base;
@@ -478,12 +478,12 @@ struct vc4_encoder {
 	enum vc4_encoder_type type;
 	u32 clock_select;
 
-	void (*pre_crtc_configure)(struct drm_encoder *encoder, struct drm_atomic_state *state);
-	void (*pre_crtc_enable)(struct drm_encoder *encoder, struct drm_atomic_state *state);
-	void (*post_crtc_enable)(struct drm_encoder *encoder, struct drm_atomic_state *state);
+	void (*pre_crtc_configure)(struct drm_encoder *encoder, struct drm_atomic_commit *state);
+	void (*pre_crtc_enable)(struct drm_encoder *encoder, struct drm_atomic_commit *state);
+	void (*post_crtc_enable)(struct drm_encoder *encoder, struct drm_atomic_commit *state);
 
-	void (*post_crtc_disable)(struct drm_encoder *encoder, struct drm_atomic_state *state);
-	void (*post_crtc_powerdown)(struct drm_encoder *encoder, struct drm_atomic_state *state);
+	void (*post_crtc_disable)(struct drm_encoder *encoder, struct drm_atomic_commit *state);
+	void (*post_crtc_powerdown)(struct drm_encoder *encoder, struct drm_atomic_commit *state);
 };
 
 #define to_vc4_encoder(_encoder)				\
@@ -945,7 +945,7 @@ int vc4_page_flip(struct drm_crtc *crtc,
 		  uint32_t flags,
 		  struct drm_modeset_acquire_ctx *ctx);
 int vc4_crtc_atomic_check(struct drm_crtc *crtc,
-			  struct drm_atomic_state *state);
+			  struct drm_atomic_commit *state);
 struct drm_crtc_state *vc4_crtc_duplicate_state(struct drm_crtc *crtc);
 void vc4_crtc_destroy_state(struct drm_crtc *crtc,
 			    struct drm_crtc_state *state);
@@ -1025,11 +1025,11 @@ struct vc4_hvs *__vc4_hvs_alloc(struct vc4_dev *vc4,
 void vc4_hvs_stop_channel(struct vc4_hvs *hvs, unsigned int output);
 int vc4_hvs_get_fifo_from_output(struct vc4_hvs *hvs, unsigned int output);
 u8 vc4_hvs_get_fifo_frame_count(struct vc4_hvs *hvs, unsigned int fifo);
-int vc4_hvs_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state);
-void vc4_hvs_atomic_begin(struct drm_crtc *crtc, struct drm_atomic_state *state);
-void vc4_hvs_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_state *state);
-void vc4_hvs_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_state *state);
-void vc4_hvs_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *state);
+int vc4_hvs_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *state);
+void vc4_hvs_atomic_begin(struct drm_crtc *crtc, struct drm_atomic_commit *state);
+void vc4_hvs_atomic_enable(struct drm_crtc *crtc, struct drm_atomic_commit *state);
+void vc4_hvs_atomic_disable(struct drm_crtc *crtc, struct drm_atomic_commit *state);
+void vc4_hvs_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_commit *state);
 void vc4_hvs_dump_state(struct vc4_hvs *hvs);
 void vc4_hvs_unmask_underrun(struct vc4_hvs *hvs, int channel);
 void vc4_hvs_mask_underrun(struct vc4_hvs *hvs, int channel);
diff --git a/drivers/gpu/drm/vc4/vc4_dsi.c b/drivers/gpu/drm/vc4/vc4_dsi.c
index deeeaebc702f7..fc76018b044cd 100644
--- a/drivers/gpu/drm/vc4/vc4_dsi.c
+++ b/drivers/gpu/drm/vc4/vc4_dsi.c
@@ -799,7 +799,7 @@ dsi_esc_timing(u32 ns)
 }
 
 static void vc4_dsi_bridge_disable(struct drm_bridge *bridge,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
 	u32 disp0_ctrl;
@@ -810,7 +810,7 @@ static void vc4_dsi_bridge_disable(struct drm_bridge *bridge,
 }
 
 static void vc4_dsi_bridge_post_disable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
 	struct device *dev = &dsi->pdev->dev;
@@ -872,7 +872,7 @@ static bool vc4_dsi_bridge_mode_fixup(struct drm_bridge *bridge,
 }
 
 static void vc4_dsi_bridge_pre_enable(struct drm_bridge *bridge,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
 	const struct drm_crtc_state *crtc_state;
@@ -1141,7 +1141,7 @@ static void vc4_dsi_bridge_pre_enable(struct drm_bridge *bridge,
 }
 
 static void vc4_dsi_bridge_enable(struct drm_bridge *bridge,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct vc4_dsi *dsi = bridge_to_vc4_dsi(bridge);
 	bool debug_dump_regs = false;
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index ab3c6d5d4eb44..e231c906709c1 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -33,7 +33,6 @@
 #include <drm/drm_print.h>
 #include <drm/drm_syncobj.h>
 
-#include "uapi/drm/vc4_drm.h"
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 #include "vc4_trace.h"
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index a99f53dadb283..a161d3b00a255 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -455,7 +455,7 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
 }
 
 static int vc4_hdmi_connector_atomic_check(struct drm_connector *connector,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *old_state =
 		drm_atomic_get_old_connector_state(state, connector);
@@ -877,7 +877,7 @@ static void vc4_hdmi_scrambling_wq(struct work_struct *work)
 }
 
 static void vc4_hdmi_encoder_post_crtc_disable(struct drm_encoder *encoder,
-					       struct drm_atomic_state *state)
+					       struct drm_atomic_commit *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
 	struct drm_device *drm = vc4_hdmi->connector.dev;
@@ -927,7 +927,7 @@ static void vc4_hdmi_encoder_post_crtc_disable(struct drm_encoder *encoder,
 }
 
 static void vc4_hdmi_encoder_post_crtc_powerdown(struct drm_encoder *encoder,
-						 struct drm_atomic_state *state)
+						 struct drm_atomic_commit *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
 	struct drm_device *drm = vc4_hdmi->connector.dev;
@@ -1471,7 +1471,7 @@ static void vc4_hdmi_recenter_fifo(struct vc4_hdmi *vc4_hdmi)
 }
 
 static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
-						struct drm_atomic_state *state)
+						struct drm_atomic_commit *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
 	struct drm_device *drm = vc4_hdmi->connector.dev;
@@ -1587,7 +1587,7 @@ static void vc4_hdmi_encoder_pre_crtc_configure(struct drm_encoder *encoder,
 }
 
 static void vc4_hdmi_encoder_pre_crtc_enable(struct drm_encoder *encoder,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
 	struct drm_device *drm = vc4_hdmi->connector.dev;
@@ -1617,7 +1617,7 @@ static void vc4_hdmi_encoder_pre_crtc_enable(struct drm_encoder *encoder,
 }
 
 static void vc4_hdmi_encoder_post_crtc_enable(struct drm_encoder *encoder,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct vc4_hdmi *vc4_hdmi = encoder_to_vc4_hdmi(encoder);
 	struct drm_connector *connector = &vc4_hdmi->connector;
diff --git a/drivers/gpu/drm/vc4/vc4_hvs.c b/drivers/gpu/drm/vc4/vc4_hvs.c
index ee8d0738501b8..184d51ea3fa57 100644
--- a/drivers/gpu/drm/vc4/vc4_hvs.c
+++ b/drivers/gpu/drm/vc4/vc4_hvs.c
@@ -787,7 +787,7 @@ void vc4_hvs_stop_channel(struct vc4_hvs *hvs, unsigned int chan)
 		__vc4_hvs_stop_channel(hvs, chan);
 }
 
-int vc4_hvs_atomic_check(struct drm_crtc *crtc, struct drm_atomic_state *state)
+int vc4_hvs_atomic_check(struct drm_crtc *crtc, struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
 	struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc_state);
@@ -882,7 +882,7 @@ static void vc4_hvs_update_dlist(struct drm_crtc *crtc)
 }
 
 void vc4_hvs_atomic_begin(struct drm_crtc *crtc,
-			  struct drm_atomic_state *state)
+			  struct drm_atomic_commit *state)
 {
 	struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
 	struct vc4_crtc_state *vc4_state = to_vc4_crtc_state(crtc->state);
@@ -894,7 +894,7 @@ void vc4_hvs_atomic_begin(struct drm_crtc *crtc,
 }
 
 void vc4_hvs_atomic_enable(struct drm_crtc *crtc,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
@@ -912,7 +912,7 @@ void vc4_hvs_atomic_enable(struct drm_crtc *crtc,
 }
 
 void vc4_hvs_atomic_disable(struct drm_crtc *crtc,
-			    struct drm_atomic_state *state)
+			    struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
@@ -924,7 +924,7 @@ void vc4_hvs_atomic_disable(struct drm_crtc *crtc,
 }
 
 void vc4_hvs_atomic_flush(struct drm_crtc *crtc,
-			  struct drm_atomic_state *state)
+			  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_state = drm_atomic_get_old_crtc_state(state,
 									 crtc);
diff --git a/drivers/gpu/drm/vc4/vc4_irq.c b/drivers/gpu/drm/vc4/vc4_irq.c
index 63e88f90eef7c..8e5141bb50759 100644
--- a/drivers/gpu/drm/vc4/vc4_irq.c
+++ b/drivers/gpu/drm/vc4/vc4_irq.c
@@ -47,7 +47,6 @@
 
 #include <linux/platform_device.h>
 
-#include <drm/drm_drv.h>
 #include <drm/drm_print.h>
 
 #include "vc4_drv.h"
@@ -242,23 +241,6 @@ vc4_irq(int irq, void *arg)
 	return status;
 }
 
-static void
-vc4_irq_prepare(struct drm_device *dev)
-{
-	struct vc4_dev *vc4 = to_vc4_dev(dev);
-
-	if (!vc4->v3d)
-		return;
-
-	init_waitqueue_head(&vc4->job_wait_queue);
-	INIT_WORK(&vc4->overflow_mem_work, vc4_overflow_mem_work);
-
-	/* Clear any pending interrupts someone might have left around
-	 * for us.
-	 */
-	V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);
-}
-
 void
 vc4_irq_enable(struct drm_device *dev)
 {
@@ -307,12 +289,22 @@ int vc4_irq_install(struct drm_device *dev, int irq)
 	if (WARN_ON_ONCE(vc4->gen > VC4_GEN_4))
 		return -ENODEV;
 
+	if (!vc4->v3d)
+		return -ENODEV;
+
 	if (irq == IRQ_NOTCONNECTED)
 		return -ENOTCONN;
 
-	vc4_irq_prepare(dev);
+	init_waitqueue_head(&vc4->job_wait_queue);
+	INIT_WORK(&vc4->overflow_mem_work, vc4_overflow_mem_work);
+
+	/* Clear any pending interrupts someone might have left around
+	 * for us.
+	 */
+	V3D_WRITE(V3D_INTCTL, V3D_DRIVER_IRQS);
 
-	ret = request_irq(irq, vc4_irq, 0, dev->driver->name, dev);
+	ret = devm_request_irq(dev->dev, irq, vc4_irq, 0,
+			       dev_name(dev->dev), dev);
 	if (ret)
 		return ret;
 
@@ -329,7 +321,6 @@ void vc4_irq_uninstall(struct drm_device *dev)
 		return;
 
 	vc4_irq_disable(dev);
-	free_irq(vc4->irq, dev);
 }
 
 /** Reinitializes interrupt registers when a GPU reset is performed. */
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index 264b5e80c24d6..b17e73bce3841 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -44,7 +44,7 @@ struct vc4_load_tracker_state {
 #define to_vc4_load_tracker_state(_state)				\
 	container_of_const(_state, struct vc4_load_tracker_state, base)
 
-static struct vc4_ctm_state *vc4_get_ctm_state(struct drm_atomic_state *state,
+static struct vc4_ctm_state *vc4_get_ctm_state(struct drm_atomic_commit *state,
 					       struct drm_private_obj *manager)
 {
 	struct drm_device *dev = state->dev;
@@ -142,7 +142,7 @@ static u16 vc4_ctm_s31_32_to_s0_9(u64 in)
 }
 
 static void
-vc4_ctm_commit(struct vc4_dev *vc4, struct drm_atomic_state *state)
+vc4_ctm_commit(struct vc4_dev *vc4, struct drm_atomic_commit *state)
 {
 	struct vc4_hvs *hvs = vc4->hvs;
 	struct vc4_ctm_state *ctm_state = to_vc4_ctm_state(vc4->ctm_manager.state);
@@ -179,7 +179,7 @@ vc4_ctm_commit(struct vc4_dev *vc4, struct drm_atomic_state *state)
 }
 
 struct vc4_hvs_state *
-vc4_hvs_get_new_global_state(const struct drm_atomic_state *state)
+vc4_hvs_get_new_global_state(const struct drm_atomic_commit *state)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 	struct drm_private_state *priv_state;
@@ -192,7 +192,7 @@ vc4_hvs_get_new_global_state(const struct drm_atomic_state *state)
 }
 
 struct vc4_hvs_state *
-vc4_hvs_get_old_global_state(const struct drm_atomic_state *state)
+vc4_hvs_get_old_global_state(const struct drm_atomic_commit *state)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 	struct drm_private_state *priv_state;
@@ -205,7 +205,7 @@ vc4_hvs_get_old_global_state(const struct drm_atomic_state *state)
 }
 
 struct vc4_hvs_state *
-vc4_hvs_get_global_state(struct drm_atomic_state *state)
+vc4_hvs_get_global_state(struct drm_atomic_commit *state)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 	struct drm_private_state *priv_state;
@@ -218,7 +218,7 @@ vc4_hvs_get_global_state(struct drm_atomic_state *state)
 }
 
 static void vc4_hvs_pv_muxing_commit(struct vc4_dev *vc4,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct vc4_hvs *hvs = vc4->hvs;
 	struct drm_crtc_state *crtc_state;
@@ -261,7 +261,7 @@ static void vc4_hvs_pv_muxing_commit(struct vc4_dev *vc4,
 }
 
 static void vc5_hvs_pv_muxing_commit(struct vc4_dev *vc4,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct vc4_hvs *hvs = vc4->hvs;
 	struct drm_crtc_state *crtc_state;
@@ -337,7 +337,7 @@ static void vc5_hvs_pv_muxing_commit(struct vc4_dev *vc4,
 }
 
 static void vc6_hvs_pv_muxing_commit(struct vc4_dev *vc4,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct vc4_hvs *hvs = vc4->hvs;
 	struct drm_crtc_state *crtc_state;
@@ -384,7 +384,7 @@ static void vc6_hvs_pv_muxing_commit(struct vc4_dev *vc4,
 	}
 }
 
-static void vc4_atomic_commit_tail(struct drm_atomic_state *state)
+static void vc4_atomic_commit_tail(struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = state->dev;
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
@@ -508,7 +508,7 @@ static void vc4_atomic_commit_tail(struct drm_atomic_state *state)
 	}
 }
 
-static int vc4_atomic_commit_setup(struct drm_atomic_state *state)
+static int vc4_atomic_commit_setup(struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state;
 	struct vc4_hvs_state *hvs_state;
@@ -587,7 +587,7 @@ static struct drm_framebuffer *vc4_fb_create(struct drm_device *dev,
  * we don't allow userland to set a CTM that we have no hope of approximating.
  */
 static int
-vc4_ctm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+vc4_ctm_atomic_check(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(dev);
 	struct vc4_ctm_state *ctm_state = NULL;
@@ -653,7 +653,7 @@ vc4_ctm_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
 	return 0;
 }
 
-static int vc4_load_tracker_atomic_check(struct drm_atomic_state *state)
+static int vc4_load_tracker_atomic_check(struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_plane_state, *new_plane_state;
 	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
@@ -898,7 +898,7 @@ static int cmp_vc4_crtc_hvs_output(const void *a, const void *b)
  *   single display, and changing the resolution down and then back up.
  */
 static int vc4_pv_muxing_atomic_check(struct drm_device *dev,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct vc4_hvs_state *hvs_new_state;
 	struct drm_crtc **sorted_crtcs;
@@ -1019,7 +1019,7 @@ static int vc4_pv_muxing_atomic_check(struct drm_device *dev,
 }
 
 static int
-vc4_core_clock_atomic_check(struct drm_atomic_state *state)
+vc4_core_clock_atomic_check(struct drm_atomic_commit *state)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(state->dev);
 	struct drm_private_state *priv_state;
@@ -1091,7 +1091,7 @@ vc4_core_clock_atomic_check(struct drm_atomic_state *state)
 
 
 static int
-vc4_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+vc4_atomic_check(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	int ret;
 
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index 91d499fefba21..6c433270214d2 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -26,8 +26,6 @@
 #include <drm/drm_gem_atomic_helper.h>
 #include <drm/drm_print.h>
 
-#include "uapi/drm/vc4_drm.h"
-
 #include "vc4_drv.h"
 #include "vc4_regs.h"
 
@@ -2125,7 +2123,7 @@ static int vc6_plane_mode_set(struct drm_plane *plane,
  * in the CRTC's flush.
  */
 static int vc4_plane_atomic_check(struct drm_plane *plane,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(plane->dev);
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
@@ -2171,7 +2169,7 @@ static int vc4_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void vc4_plane_atomic_update(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	/* No contents here.  Since we don't know where in the CRTC's
 	 * dlist we should be stored, our dlist is uploaded to the
@@ -2263,7 +2261,7 @@ void vc4_plane_async_set_fb(struct drm_plane *plane, struct drm_framebuffer *fb)
 }
 
 static void vc4_plane_atomic_async_update(struct drm_plane *plane,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -2337,7 +2335,7 @@ static void vc4_plane_atomic_async_update(struct drm_plane *plane,
 }
 
 static int vc4_plane_atomic_async_check(struct drm_plane *plane,
-					struct drm_atomic_state *state, bool flip)
+					struct drm_atomic_commit *state, bool flip)
 {
 	struct vc4_dev *vc4 = to_vc4_dev(plane->dev);
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
diff --git a/drivers/gpu/drm/vc4/vc4_render_cl.c b/drivers/gpu/drm/vc4/vc4_render_cl.c
index edc471e71c0eb..c264d21bc3fe3 100644
--- a/drivers/gpu/drm/vc4/vc4_render_cl.c
+++ b/drivers/gpu/drm/vc4/vc4_render_cl.c
@@ -37,7 +37,6 @@
 
 #include <drm/drm_print.h>
 
-#include "uapi/drm/vc4_drm.h"
 #include "vc4_drv.h"
 #include "vc4_packet.h"
 
diff --git a/drivers/gpu/drm/vc4/vc4_txp.c b/drivers/gpu/drm/vc4/vc4_txp.c
index 9082902100e4b..66b6f2acf8626 100644
--- a/drivers/gpu/drm/vc4/vc4_txp.c
+++ b/drivers/gpu/drm/vc4/vc4_txp.c
@@ -246,7 +246,7 @@ static void vc4_txp_armed(struct drm_crtc_state *state)
 }
 
 static int vc4_txp_connector_atomic_check(struct drm_connector *conn,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *conn_state;
 	struct drm_crtc_state *crtc_state;
@@ -285,7 +285,7 @@ static int vc4_txp_connector_atomic_check(struct drm_connector *conn,
 }
 
 static void vc4_txp_connector_atomic_commit(struct drm_connector *conn,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = conn->dev;
 	struct drm_connector_state *conn_state = drm_atomic_get_new_connector_state(state,
@@ -435,7 +435,7 @@ static const struct drm_crtc_funcs vc4_txp_crtc_funcs = {
 };
 
 static int vc4_txp_atomic_check(struct drm_crtc *crtc,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -451,14 +451,14 @@ static int vc4_txp_atomic_check(struct drm_crtc *crtc,
 }
 
 static void vc4_txp_atomic_enable(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_on(crtc);
 	vc4_hvs_atomic_enable(crtc, state);
 }
 
 static void vc4_txp_atomic_disable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 
diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c
index 545c4c3608f56..7f2fadfde7a8e 100644
--- a/drivers/gpu/drm/vc4/vc4_validate.c
+++ b/drivers/gpu/drm/vc4/vc4_validate.c
@@ -45,7 +45,6 @@
 
 #include <drm/drm_print.h>
 
-#include "uapi/drm/vc4_drm.h"
 #include "vc4_drv.h"
 #include "vc4_packet.h"
 
diff --git a/drivers/gpu/drm/vc4/vc4_vec.c b/drivers/gpu/drm/vc4/vc4_vec.c
index b0b271d93b276..792b7b39be597 100644
--- a/drivers/gpu/drm/vc4/vc4_vec.c
+++ b/drivers/gpu/drm/vc4/vc4_vec.c
@@ -538,7 +538,7 @@ static int vc4_vec_connector_init(struct drm_device *dev, struct vc4_vec *vec)
 }
 
 static void vc4_vec_encoder_disable(struct drm_encoder *encoder,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = encoder->dev;
 	struct vc4_vec *vec = encoder_to_vc4_vec(encoder);
@@ -562,7 +562,7 @@ static void vc4_vec_encoder_disable(struct drm_encoder *encoder,
 }
 
 static void vc4_vec_encoder_enable(struct drm_encoder *encoder,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_device *drm = encoder->dev;
 	struct vc4_vec *vec = encoder_to_vc4_vec(encoder);
diff --git a/drivers/gpu/drm/verisilicon/vs_bridge.c b/drivers/gpu/drm/verisilicon/vs_bridge.c
index 2a0ad00a94d6d..7a93049368db2 100644
--- a/drivers/gpu/drm/verisilicon/vs_bridge.c
+++ b/drivers/gpu/drm/verisilicon/vs_bridge.c
@@ -174,7 +174,7 @@ static void vs_bridge_enable_common(struct vs_crtc *crtc,
 }
 
 static void vs_bridge_atomic_enable_dpi(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct vs_bridge *vbridge = drm_bridge_to_vs_bridge(bridge);
 	struct drm_bridge_state *br_state =
@@ -192,7 +192,7 @@ static void vs_bridge_atomic_enable_dpi(struct drm_bridge *bridge,
 }
 
 static void vs_bridge_atomic_enable_dp(struct drm_bridge *bridge,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct vs_bridge *vbridge = drm_bridge_to_vs_bridge(bridge);
 	struct drm_bridge_state *br_state =
@@ -221,7 +221,7 @@ static void vs_bridge_atomic_enable_dp(struct drm_bridge *bridge,
 }
 
 static void vs_bridge_atomic_disable(struct drm_bridge *bridge,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct vs_bridge *vbridge = drm_bridge_to_vs_bridge(bridge);
 	struct vs_crtc *crtc = vbridge->crtc;
@@ -365,7 +365,6 @@ struct vs_bridge *vs_bridge_init(struct drm_device *drm_dev,
 		ret = PTR_ERR(bridge->conn);
 		return ERR_PTR(ret);
 	}
-	drm_connector_attach_encoder(bridge->conn, bridge->enc);
 
 	return bridge;
 }
diff --git a/drivers/gpu/drm/verisilicon/vs_crtc.c b/drivers/gpu/drm/verisilicon/vs_crtc.c
index f494017130006..9080344398ca2 100644
--- a/drivers/gpu/drm/verisilicon/vs_crtc.c
+++ b/drivers/gpu/drm/verisilicon/vs_crtc.c
@@ -21,7 +21,7 @@
 #include "vs_plane.h"
 
 static void vs_crtc_atomic_disable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
 	struct vs_dc *dc = vcrtc->dc;
@@ -33,7 +33,7 @@ static void vs_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static void vs_crtc_atomic_enable(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct vs_crtc *vcrtc = drm_crtc_to_vs_crtc(crtc);
 	struct vs_dc *dc = vcrtc->dc;
diff --git a/drivers/gpu/drm/verisilicon/vs_plane.c b/drivers/gpu/drm/verisilicon/vs_plane.c
index 2f3953e588a34..d81f7b8f4c650 100644
--- a/drivers/gpu/drm/verisilicon/vs_plane.c
+++ b/drivers/gpu/drm/verisilicon/vs_plane.c
@@ -6,13 +6,15 @@
 #include <linux/errno.h>
 #include <linux/printk.h>
 
+#include <drm/drm_atomic_state_helper.h>
 #include <drm/drm_fb_dma_helper.h>
 #include <drm/drm_fourcc.h>
 #include <drm/drm_gem_dma_helper.h>
+#include <drm/drm_print.h>
 
 #include "vs_plane.h"
 
-void drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format)
+int drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format)
 {
 	switch (drm_format) {
 	case DRM_FORMAT_XRGB4444:
@@ -62,7 +64,7 @@ void drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format)
 		vs_format->color = VSDC_COLOR_FORMAT_A2R10G10B10;
 		break;
 	default:
-		pr_warn("Unexpected drm format!\n");
+		return -EINVAL;
 	}
 
 	switch (drm_format) {
@@ -101,6 +103,8 @@ void drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format)
 
 	/* N/A for non-YUV formats */
 	vs_format->uv_swizzle = false;
+
+	return 0;
 }
 
 dma_addr_t vs_fb_get_dma_addr(struct drm_framebuffer *fb,
@@ -122,3 +126,49 @@ dma_addr_t vs_fb_get_dma_addr(struct drm_framebuffer *fb,
 
 	return dma_addr;
 }
+
+struct drm_plane_state *vs_plane_duplicate_state(struct drm_plane *plane)
+{
+	struct vs_plane_state *vs_state, *vs_state_old;
+
+	if (drm_WARN_ON(plane->dev, !plane->state))
+		return NULL;
+
+	vs_state_old = to_vs_plane_state(plane->state);
+
+	vs_state = kzalloc_obj(*vs_state, GFP_KERNEL);
+	if (!vs_state)
+		return NULL;
+
+	__drm_atomic_helper_plane_duplicate_state(plane, &vs_state->base);
+
+	memcpy(&vs_state->format, &vs_state_old->format,
+	       sizeof(struct vs_format));
+
+	return &vs_state->base;
+}
+
+void vs_plane_destroy_state(struct drm_plane *plane,
+			    struct drm_plane_state *state)
+{
+	__drm_atomic_helper_plane_destroy_state(state);
+	kfree(state);
+}
+
+/* Called during init to allocate the plane's atomic state. */
+void vs_plane_reset(struct drm_plane *plane)
+{
+	struct vs_plane_state *vs_state;
+
+	if (plane->state) {
+		__drm_atomic_helper_plane_destroy_state(plane->state);
+		kfree(plane->state);
+		plane->state = NULL;
+	}
+
+	vs_state = kzalloc_obj(*vs_state, GFP_KERNEL);
+	if (!vs_state)
+		return;
+
+	__drm_atomic_helper_plane_reset(plane, &vs_state->base);
+}
diff --git a/drivers/gpu/drm/verisilicon/vs_plane.h b/drivers/gpu/drm/verisilicon/vs_plane.h
index 41875ea3d66a5..f18c36ef4cd90 100644
--- a/drivers/gpu/drm/verisilicon/vs_plane.h
+++ b/drivers/gpu/drm/verisilicon/vs_plane.h
@@ -63,10 +63,26 @@ struct vs_format {
 	bool uv_swizzle;
 };
 
-void drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format);
+struct vs_plane_state {
+	struct drm_plane_state base;
+
+	struct vs_format format;
+};
+
+static inline struct vs_plane_state *to_vs_plane_state(struct drm_plane_state *state)
+{
+	return container_of(state, struct vs_plane_state, base);
+}
+
+int drm_format_to_vs_format(u32 drm_format, struct vs_format *vs_format);
 dma_addr_t vs_fb_get_dma_addr(struct drm_framebuffer *fb,
 			      const struct drm_rect *src_rect);
 
+struct drm_plane_state *vs_plane_duplicate_state(struct drm_plane *plane);
+void vs_plane_destroy_state(struct drm_plane *plane,
+			    struct drm_plane_state *state);
+void vs_plane_reset(struct drm_plane *plane);
+
 struct drm_plane *vs_primary_plane_init(struct drm_device *dev, struct vs_dc *dc);
 
 #endif /* _VS_PLANE_H_ */
diff --git a/drivers/gpu/drm/verisilicon/vs_primary_plane.c b/drivers/gpu/drm/verisilicon/vs_primary_plane.c
index e8fcb5958615c..1f2be41ae496c 100644
--- a/drivers/gpu/drm/verisilicon/vs_primary_plane.c
+++ b/drivers/gpu/drm/verisilicon/vs_primary_plane.c
@@ -21,25 +21,36 @@
 #include "vs_primary_plane_regs.h"
 
 static int vs_primary_plane_atomic_check(struct drm_plane *plane,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
+	struct vs_plane_state *new_vs_plane_state = to_vs_plane_state(new_plane_state);
+	struct drm_framebuffer *fb = new_plane_state->fb;
 	struct drm_crtc *crtc = new_plane_state->crtc;
-	struct drm_crtc_state *crtc_state;
+	struct drm_crtc_state *crtc_state = NULL;
+	int ret;
 
-	if (!crtc)
+	if (crtc)
+		crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
+
+	ret = drm_atomic_helper_check_plane_state(new_plane_state,
+						  crtc_state,
+						  DRM_PLANE_NO_SCALING,
+						  DRM_PLANE_NO_SCALING,
+						  false, true);
+	if (ret)
+		return ret;
+
+	if (!new_plane_state->visible)
 		return 0;
 
-	crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
-	if (WARN_ON(!crtc_state))
-		return -EINVAL;
+	ret = drm_format_to_vs_format(fb->format->format,
+				      &new_vs_plane_state->format);
+	if (drm_WARN_ON_ONCE(plane->dev, ret))
+		return ret;
 
-	return drm_atomic_helper_check_plane_state(new_plane_state,
-						   crtc_state,
-						   DRM_PLANE_NO_SCALING,
-						   DRM_PLANE_NO_SCALING,
-						   false, true);
+	return 0;
 }
 
 static void vs_primary_plane_commit(struct vs_dc *dc, unsigned int output)
@@ -49,7 +60,7 @@ static void vs_primary_plane_commit(struct vs_dc *dc, unsigned int output)
 }
 
 static void vs_primary_plane_atomic_enable(struct drm_plane *plane,
-					   struct drm_atomic_state *atomic_state)
+					   struct drm_atomic_commit *atomic_state)
 {
 	struct drm_plane_state *state = drm_atomic_get_new_plane_state(atomic_state,
 								       plane);
@@ -68,7 +79,7 @@ static void vs_primary_plane_atomic_enable(struct drm_plane *plane,
 }
 
 static void vs_primary_plane_atomic_disable(struct drm_plane *plane,
-					    struct drm_atomic_state *atomic_state)
+					    struct drm_atomic_commit *atomic_state)
 {
 	struct drm_plane_state *state = drm_atomic_get_old_plane_state(atomic_state,
 								       plane);
@@ -84,15 +95,15 @@ static void vs_primary_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static void vs_primary_plane_atomic_update(struct drm_plane *plane,
-					   struct drm_atomic_state *atomic_state)
+					   struct drm_atomic_commit *atomic_state)
 {
 	struct drm_plane_state *state = drm_atomic_get_new_plane_state(atomic_state,
 								       plane);
+	struct vs_plane_state *vs_state = to_vs_plane_state(state);
 	struct drm_framebuffer *fb = state->fb;
 	struct drm_crtc *crtc = state->crtc;
 	struct vs_dc *dc;
 	struct vs_crtc *vcrtc;
-	struct vs_format fmt;
 	unsigned int output;
 	dma_addr_t dma_addr;
 
@@ -105,16 +116,15 @@ static void vs_primary_plane_atomic_update(struct drm_plane *plane,
 	output = vcrtc->id;
 	dc = vcrtc->dc;
 
-	drm_format_to_vs_format(state->fb->format->format, &fmt);
-
 	regmap_update_bits(dc->regs, VSDC_FB_CONFIG(output),
 			   VSDC_FB_CONFIG_FMT_MASK,
-			   VSDC_FB_CONFIG_FMT(fmt.color));
+			   VSDC_FB_CONFIG_FMT(vs_state->format.color));
 	regmap_update_bits(dc->regs, VSDC_FB_CONFIG(output),
 			   VSDC_FB_CONFIG_SWIZZLE_MASK,
-			   VSDC_FB_CONFIG_SWIZZLE(fmt.swizzle));
+			   VSDC_FB_CONFIG_SWIZZLE(vs_state->format.swizzle));
 	regmap_assign_bits(dc->regs, VSDC_FB_CONFIG(output),
-			   VSDC_FB_CONFIG_UV_SWIZZLE_EN, fmt.uv_swizzle);
+			   VSDC_FB_CONFIG_UV_SWIZZLE_EN,
+			   vs_state->format.uv_swizzle);
 
 	dma_addr = vs_fb_get_dma_addr(fb, &state->src);
 
@@ -145,10 +155,10 @@ static const struct drm_plane_helper_funcs vs_primary_plane_helper_funcs = {
 };
 
 static const struct drm_plane_funcs vs_primary_plane_funcs = {
-	.atomic_destroy_state	= drm_atomic_helper_plane_destroy_state,
-	.atomic_duplicate_state	= drm_atomic_helper_plane_duplicate_state,
+	.atomic_destroy_state	= vs_plane_destroy_state,
+	.atomic_duplicate_state	= vs_plane_duplicate_state,
 	.disable_plane		= drm_atomic_helper_disable_plane,
-	.reset			= drm_atomic_helper_plane_reset,
+	.reset			= vs_plane_reset,
 	.update_plane		= drm_atomic_helper_update_plane,
 };
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index f1dae95698053..44ffffec550fd 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -101,13 +101,13 @@ static void virtio_gpu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static void virtio_gpu_crtc_atomic_enable(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_on(crtc);
 }
 
 static void virtio_gpu_crtc_atomic_disable(struct drm_crtc *crtc,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct virtio_gpu_device *vgdev = dev->dev_private;
@@ -120,13 +120,13 @@ static void virtio_gpu_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static int virtio_gpu_crtc_atomic_check(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	return 0;
 }
 
 static void virtio_gpu_crtc_atomic_flush(struct drm_crtc *crtc,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state, crtc);
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index a5ce96fb8a1d2..812ee3f5e4aa9 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -163,6 +163,7 @@ static unsigned int features[] = {
 	VIRTIO_GPU_F_RESOURCE_UUID,
 	VIRTIO_GPU_F_RESOURCE_BLOB,
 	VIRTIO_GPU_F_CONTEXT_INIT,
+	VIRTIO_GPU_F_BLOB_ALIGNMENT,
 };
 static struct virtio_driver virtio_gpu_driver = {
 	.feature_table = features,
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index f17660a71a3e7..7449907754a43 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -84,6 +84,7 @@ struct virtio_gpu_object_params {
 	uint32_t blob_mem;
 	uint32_t blob_flags;
 	uint64_t blob_id;
+	uint32_t blob_hints;
 };
 
 struct virtio_gpu_object {
@@ -257,6 +258,7 @@ struct virtio_gpu_device {
 	bool has_resource_blob;
 	bool has_host_visible;
 	bool has_context_init;
+	bool has_blob_alignment;
 	struct virtio_shm_region host_visible_region;
 	struct drm_mm host_visible_mm;
 
@@ -270,6 +272,7 @@ struct virtio_gpu_device {
 	uint32_t num_capsets;
 	uint64_t capset_id_mask;
 	struct list_head cap_cache;
+	uint32_t blob_alignment;
 
 	/* protects uuid state when exporting */
 	spinlock_t resource_export_lock;
@@ -317,6 +320,7 @@ virtio_gpu_array_from_handles(struct drm_file *drm_file, u32 *handles, u32 nents
 void virtio_gpu_array_add_obj(struct virtio_gpu_object_array *objs,
 			      struct drm_gem_object *obj);
 int virtio_gpu_array_lock_resv(struct virtio_gpu_object_array *objs);
+int virtio_gpu_lock_one_resv_uninterruptible(struct virtio_gpu_object_array *objs);
 void virtio_gpu_array_unlock_resv(struct virtio_gpu_object_array *objs);
 void virtio_gpu_array_add_fence(struct virtio_gpu_object_array *objs,
 				struct dma_fence *fence);
@@ -507,6 +511,7 @@ struct sg_table *virtio_gpu_vram_map_dma_buf(struct virtio_gpu_object *bo,
 void virtio_gpu_vram_unmap_dma_buf(struct device *dev,
 				   struct sg_table *sgt,
 				   enum dma_data_direction dir);
+void virtio_gpu_vram_map_deferred(struct virtio_gpu_object_vram *vram);
 
 /* virtgpu_submit.c */
 int virtio_gpu_execbuffer_ioctl(struct drm_device *dev, void *data,
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c b/drivers/gpu/drm/virtio/virtgpu_gem.c
index f22dc5c21cd42..435d37d360340 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -238,6 +238,23 @@ int virtio_gpu_array_lock_resv(struct virtio_gpu_object_array *objs)
 	return ret;
 }
 
+int virtio_gpu_lock_one_resv_uninterruptible(struct virtio_gpu_object_array *objs)
+{
+	int ret;
+
+	if (objs->nents != 1)
+		return -EINVAL;
+
+	dma_resv_lock(objs->objs[0]->resv, NULL);
+
+	ret = dma_resv_reserve_fences(objs->objs[0]->resv, 1);
+	if (ret) {
+		virtio_gpu_array_unlock_resv(objs);
+		return ret;
+	}
+	return 0;
+}
+
 void virtio_gpu_array_unlock_resv(struct virtio_gpu_object_array *objs)
 {
 	if (objs->nents == 1) {
diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
index c33c057365f85..3d8e4ccdb7c1f 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c
@@ -117,6 +117,11 @@ static int virtio_gpu_getparam_ioctl(struct drm_device *dev, void *data,
 	case VIRTGPU_PARAM_EXPLICIT_DEBUG_NAME:
 		value = vgdev->has_context_init ? 1 : 0;
 		break;
+	case VIRTGPU_PARAM_BLOB_ALIGNMENT:
+		if (!vgdev->has_blob_alignment)
+			return -ENOENT;
+		value = vgdev->blob_alignment;
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -489,6 +494,12 @@ static int verify_blob(struct virtio_gpu_device *vgdev,
 	params->size = rc_blob->size;
 	params->blob = true;
 	params->blob_flags = rc_blob->blob_flags;
+	params->blob_hints = rc_blob->blob_hints;
+
+	if (vgdev->has_blob_alignment &&
+	    !IS_ALIGNED(params->size, vgdev->blob_alignment))
+		return -EINVAL;
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/virtio/virtgpu_kms.c b/drivers/gpu/drm/virtio/virtgpu_kms.c
index 80ba69b4860bc..cfde9f573df6e 100644
--- a/drivers/gpu/drm/virtio/virtgpu_kms.c
+++ b/drivers/gpu/drm/virtio/virtgpu_kms.c
@@ -124,7 +124,7 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
 	struct virtio_gpu_device *vgdev;
 	/* this will expand later */
 	struct virtqueue *vqs[2];
-	u32 num_scanouts, num_capsets;
+	u32 num_scanouts, num_capsets, blob_alignment;
 	int ret = 0;
 
 	if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
@@ -198,14 +198,22 @@ int virtio_gpu_init(struct virtio_device *vdev, struct drm_device *dev)
 	if (virtio_has_feature(vgdev->vdev, VIRTIO_GPU_F_CONTEXT_INIT))
 		vgdev->has_context_init = true;
 
+	if (virtio_has_feature(vgdev->vdev, VIRTIO_GPU_F_BLOB_ALIGNMENT)) {
+		vgdev->has_blob_alignment = true;
+		virtio_cread_le(vgdev->vdev, struct virtio_gpu_config,
+				blob_alignment, &blob_alignment);
+		vgdev->blob_alignment = blob_alignment;
+	}
+
 	DRM_INFO("features: %cvirgl %cedid %cresource_blob %chost_visible",
 		 vgdev->has_virgl_3d    ? '+' : '-',
 		 vgdev->has_edid        ? '+' : '-',
 		 vgdev->has_resource_blob ? '+' : '-',
 		 vgdev->has_host_visible ? '+' : '-');
 
-	DRM_INFO("features: %ccontext_init\n",
-		 vgdev->has_context_init ? '+' : '-');
+	DRM_INFO("features: %ccontext_init %cblob_alignment\n",
+		 vgdev->has_context_init ? '+' : '-',
+		 vgdev->has_blob_alignment ? '+' : '-');
 
 	ret = virtio_find_vqs(vgdev->vdev, 2, vqs, vqs_info, NULL);
 	if (ret) {
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index a126d1b25f464..1d1b27ece62a7 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -97,7 +97,7 @@ static const struct drm_plane_funcs virtio_gpu_plane_funcs = {
 };
 
 static int virtio_gpu_plane_atomic_check(struct drm_plane *plane,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -215,7 +215,10 @@ static void virtio_gpu_resource_flush(struct drm_plane *plane,
 		if (!objs)
 			return;
 		virtio_gpu_array_add_obj(objs, vgfb->base.obj[0]);
-		virtio_gpu_array_lock_resv(objs);
+		if (virtio_gpu_lock_one_resv_uninterruptible(objs)) {
+			virtio_gpu_array_put_free(objs);
+			return;
+		}
 		virtio_gpu_cmd_resource_flush(vgdev, bo->hw_res_handle, x, y,
 					      width, height, objs,
 					      vgplane_st->fence);
@@ -230,7 +233,7 @@ static void virtio_gpu_resource_flush(struct drm_plane *plane,
 }
 
 static void virtio_gpu_primary_plane_update(struct drm_plane *plane,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -423,7 +426,7 @@ static void virtio_gpu_plane_cleanup_fb(struct drm_plane *plane,
 }
 
 static void virtio_gpu_cursor_plane_update(struct drm_plane *plane,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -459,7 +462,10 @@ static void virtio_gpu_cursor_plane_update(struct drm_plane *plane,
 		if (!objs)
 			return;
 		virtio_gpu_array_add_obj(objs, vgfb->base.obj[0]);
-		virtio_gpu_array_lock_resv(objs);
+		if (virtio_gpu_lock_one_resv_uninterruptible(objs)) {
+			virtio_gpu_array_put_free(objs);
+			return;
+		}
 		virtio_gpu_cmd_transfer_to_host_2d
 			(vgdev, 0,
 			 plane->state->crtc_w,
diff --git a/drivers/gpu/drm/virtio/virtgpu_vram.c b/drivers/gpu/drm/virtio/virtgpu_vram.c
index 084e802274337..4ae3cbc35dd36 100644
--- a/drivers/gpu/drm/virtio/virtgpu_vram.c
+++ b/drivers/gpu/drm/virtio/virtgpu_vram.c
@@ -3,6 +3,8 @@
 
 #include <linux/dma-mapping.h>
 
+static DEFINE_MUTEX(map_lock);
+
 static void virtio_gpu_vram_free(struct drm_gem_object *obj)
 {
 	struct virtio_gpu_object *bo = gem_to_virtio_gpu_obj(obj);
@@ -42,6 +44,11 @@ static int virtio_gpu_vram_mmap(struct drm_gem_object *obj,
 	if (!(bo->blob_flags & VIRTGPU_BLOB_FLAG_USE_MAPPABLE))
 		return -EINVAL;
 
+	virtio_gpu_vram_map_deferred(vram);
+
+	if (vram->map_state == STATE_INITIALIZING)
+		virtio_gpu_notify(vgdev);
+
 	wait_event(vgdev->resp_wq, vram->map_state != STATE_INITIALIZING);
 	if (vram->map_state != STATE_OK)
 		return -EINVAL;
@@ -218,14 +225,27 @@ int virtio_gpu_vram_create(struct virtio_gpu_device *vgdev,
 
 	virtio_gpu_cmd_resource_create_blob(vgdev, &vram->base, params, NULL,
 					    0);
-	if (params->blob_flags & VIRTGPU_BLOB_FLAG_USE_MAPPABLE) {
-		ret = virtio_gpu_vram_map(&vram->base);
-		if (ret) {
-			virtio_gpu_vram_free(obj);
-			return ret;
+	if (!(params->blob_hints & DRM_VIRTGPU_BLOB_FLAG_HINT_DEFER_MAPPING)) {
+		if (params->blob_flags & VIRTGPU_BLOB_FLAG_USE_MAPPABLE) {
+			ret = virtio_gpu_vram_map(&vram->base);
+			if (ret) {
+				virtio_gpu_vram_free(obj);
+				return ret;
+			}
 		}
 	}
 
 	*bo_ptr = &vram->base;
 	return 0;
 }
+
+void virtio_gpu_vram_map_deferred(struct virtio_gpu_object_vram *vram)
+{
+	if (!(vram->base.blob_flags & VIRTGPU_BLOB_FLAG_USE_MAPPABLE))
+		return;
+
+	mutex_lock(&map_lock);
+	if (!drm_mm_node_allocated(&vram->vram_node))
+		virtio_gpu_vram_map(&vram->base);
+	mutex_unlock(&map_lock);
+}
diff --git a/drivers/gpu/drm/vkms/vkms_crtc.c b/drivers/gpu/drm/vkms/vkms_crtc.c
index 35ddc553a5e66..079abfba427d0 100644
--- a/drivers/gpu/drm/vkms/vkms_crtc.c
+++ b/drivers/gpu/drm/vkms/vkms_crtc.c
@@ -111,7 +111,7 @@ static const struct drm_crtc_funcs vkms_crtc_funcs = {
 };
 
 static int vkms_crtc_atomic_check(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *crtc_state = drm_atomic_get_new_crtc_state(state,
 									  crtc);
@@ -157,7 +157,7 @@ static int vkms_crtc_atomic_check(struct drm_crtc *crtc,
 }
 
 static void vkms_crtc_atomic_begin(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 	__acquires(&vkms_output->lock)
 {
 	struct vkms_output *vkms_output = drm_crtc_to_vkms_output(crtc);
@@ -169,7 +169,7 @@ static void vkms_crtc_atomic_begin(struct drm_crtc *crtc,
 }
 
 static void vkms_crtc_atomic_flush(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 	__releases(&vkms_output->lock)
 {
 	struct vkms_output *vkms_output = drm_crtc_to_vkms_output(crtc);
diff --git a/drivers/gpu/drm/vkms/vkms_drv.c b/drivers/gpu/drm/vkms/vkms_drv.c
index 95020765c4c29..5a640b531d88f 100644
--- a/drivers/gpu/drm/vkms/vkms_drv.c
+++ b/drivers/gpu/drm/vkms/vkms_drv.c
@@ -62,7 +62,7 @@ MODULE_PARM_DESC(create_default_dev, "Create or not the default VKMS device");
 
 DEFINE_DRM_GEM_FOPS(vkms_driver_fops);
 
-static void vkms_atomic_commit_tail(struct drm_atomic_state *old_state)
+static void vkms_atomic_commit_tail(struct drm_atomic_commit *old_state)
 {
 	struct drm_device *dev = old_state->dev;
 	struct drm_crtc *crtc;
@@ -102,7 +102,7 @@ static const struct drm_driver vkms_driver = {
 	.minor			= DRIVER_MINOR,
 };
 
-static int vkms_atomic_check(struct drm_device *dev, struct drm_atomic_state *state)
+static int vkms_atomic_check(struct drm_device *dev, struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *new_crtc_state;
diff --git a/drivers/gpu/drm/vkms/vkms_plane.c b/drivers/gpu/drm/vkms/vkms_plane.c
index ca7aee101a953..68cb2a3335e64 100644
--- a/drivers/gpu/drm/vkms/vkms_plane.c
+++ b/drivers/gpu/drm/vkms/vkms_plane.c
@@ -122,7 +122,7 @@ static const struct drm_plane_funcs vkms_plane_funcs = {
 };
 
 static void vkms_plane_atomic_update(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -153,7 +153,7 @@ static void vkms_plane_atomic_update(struct drm_plane *plane,
 }
 
 static int vkms_plane_atomic_check(struct drm_plane *plane,
-				   struct drm_atomic_state *state)
+				   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
diff --git a/drivers/gpu/drm/vkms/vkms_writeback.c b/drivers/gpu/drm/vkms/vkms_writeback.c
index 908b7e602ffbc..ecf29a2c0c8e0 100644
--- a/drivers/gpu/drm/vkms/vkms_writeback.c
+++ b/drivers/gpu/drm/vkms/vkms_writeback.c
@@ -32,7 +32,7 @@ static const struct drm_connector_funcs vkms_wb_connector_funcs = {
 };
 
 static int vkms_wb_atomic_check(struct drm_connector *connector,
-				struct drm_atomic_state *state)
+				struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *conn_state =
 		drm_atomic_get_new_connector_state(state, connector);
@@ -123,7 +123,7 @@ static void vkms_wb_cleanup_job(struct drm_writeback_connector *connector,
 }
 
 static void vkms_wb_atomic_commit(struct drm_connector *conn,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *connector_state = drm_atomic_get_new_connector_state(state,
 											 conn);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c
index c46f17ba7236d..b010fc7ca68e2 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.c
@@ -690,7 +690,7 @@ int vmw_cursor_plane_prepare_fb(struct drm_plane *plane,
  * Returns 0 on success
  */
 int vmw_cursor_plane_atomic_check(struct drm_plane *plane,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state =
 		drm_atomic_get_new_plane_state(state, plane);
@@ -739,7 +739,7 @@ int vmw_cursor_plane_atomic_check(struct drm_plane *plane,
 
 void
 vmw_cursor_plane_atomic_update(struct drm_plane *plane,
-			       struct drm_atomic_state *state)
+			       struct drm_atomic_commit *state)
 {
 	struct vmw_bo *bo;
 	struct drm_plane_state *new_state =
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.h b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.h
index 0c2cc0699b0d9..d32cf17d013b5 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cursor_plane.h
@@ -71,9 +71,9 @@ void vmw_cursor_cmd_dma_snoop(SVGA3dCmdHeader *header,
 void vmw_cursor_plane_destroy(struct drm_plane *plane);
 
 int vmw_cursor_plane_atomic_check(struct drm_plane *plane,
-				  struct drm_atomic_state *state);
+				  struct drm_atomic_commit *state);
 void vmw_cursor_plane_atomic_update(struct drm_plane *plane,
-				    struct drm_atomic_state *state);
+				    struct drm_atomic_commit *state);
 int vmw_cursor_plane_prepare_fb(struct drm_plane *plane,
 				struct drm_plane_state *new_state);
 void vmw_cursor_plane_cleanup_fb(struct drm_plane *plane,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 57465f69c6870..38bea8abab84c 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -507,7 +507,7 @@ struct vmw_private {
 	struct drm_property *hotplug_mode_update_property;
 	struct drm_property *implicit_placement_property;
 	spinlock_t cursor_lock;
-	struct drm_atomic_state *suspend_state;
+	struct drm_atomic_commit *suspend_state;
 
 	/*
 	 * Context and surface management.
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index e7bddf840a795..1b407b61f6838 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -101,7 +101,7 @@ vmw_du_plane_cleanup_fb(struct drm_plane *plane,
  * Returns 0 on success
  */
 int vmw_du_primary_plane_atomic_check(struct drm_plane *plane,
-				      struct drm_atomic_state *state)
+				      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state,
 									   plane);
@@ -132,7 +132,7 @@ int vmw_du_primary_plane_atomic_check(struct drm_plane *plane,
 }
 
 int vmw_du_crtc_atomic_check(struct drm_crtc *crtc,
-			     struct drm_atomic_state *state)
+			     struct drm_atomic_commit *state)
 {
 	struct vmw_private *vmw = vmw_priv(crtc->dev);
 	struct drm_crtc_state *new_state = drm_atomic_get_new_crtc_state(state,
@@ -169,7 +169,7 @@ int vmw_du_crtc_atomic_check(struct drm_crtc *crtc,
 
 
 void vmw_du_crtc_atomic_begin(struct drm_crtc *crtc,
-			      struct drm_atomic_state *state)
+			      struct drm_atomic_commit *state)
 {
 	vmw_vkms_crtc_atomic_begin(crtc, state);
 }
@@ -864,7 +864,7 @@ static int vmw_kms_check_display_memory(struct drm_device *dev,
  * pointer error, in particular -EDEADLK if locking needs to be rerun.
  */
 static struct drm_crtc_state *
-vmw_crtc_state_and_lock(struct drm_atomic_state *state, struct drm_crtc *crtc)
+vmw_crtc_state_and_lock(struct drm_atomic_commit *state, struct drm_crtc *crtc)
 {
 	struct drm_crtc_state *crtc_state;
 
@@ -895,7 +895,7 @@ vmw_crtc_state_and_lock(struct drm_atomic_state *state, struct drm_crtc *crtc)
  *   -EDEADLK if modeset locking needs to be rerun.
  */
 static int vmw_kms_check_implicit(struct drm_device *dev,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_framebuffer *implicit_fb = NULL;
 	struct drm_crtc *crtc;
@@ -933,7 +933,7 @@ static int vmw_kms_check_implicit(struct drm_device *dev,
 }
 
 /**
- * vmw_kms_check_topology - Validates topology in drm_atomic_state
+ * vmw_kms_check_topology - Validates topology in drm_atomic_commit
  * @dev: DRM device
  * @state: the driver state object
  *
@@ -941,7 +941,7 @@ static int vmw_kms_check_implicit(struct drm_device *dev,
  * 0 on success otherwise negative error code
  */
 static int vmw_kms_check_topology(struct drm_device *dev,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct drm_crtc_state *old_crtc_state, *new_crtc_state;
 	struct drm_rect *rects;
@@ -1035,7 +1035,7 @@ static int vmw_kms_check_topology(struct drm_device *dev,
  */
 static int
 vmw_kms_atomic_check_modeset(struct drm_device *dev,
-			     struct drm_atomic_state *state)
+			     struct drm_atomic_commit *state)
 {
 	struct drm_crtc *crtc;
 	struct drm_crtc_state *crtc_state;
@@ -1133,7 +1133,7 @@ vmw_kms_create_hotplug_mode_update_property(struct vmw_private *dev_priv)
 }
 
 static void
-vmw_atomic_commit_tail(struct drm_atomic_state *old_state)
+vmw_atomic_commit_tail(struct drm_atomic_commit *old_state)
 {
 	struct vmw_private *vmw = vmw_priv(old_state->dev);
 	struct drm_crtc *crtc;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index 445471fe9be6e..2224d7d91d1bb 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -322,8 +322,6 @@ struct vmw_display_unit {
 
 	struct {
 		struct work_struct crc_generator_work;
-		struct hrtimer timer;
-		ktime_t period_ns;
 
 		/* protects concurrent access to the vblank handler */
 		atomic_t atomic_lock;
@@ -410,7 +408,7 @@ void vmw_du_primary_plane_destroy(struct drm_plane *plane);
 
 /* Atomic Helpers */
 int vmw_du_primary_plane_atomic_check(struct drm_plane *plane,
-				      struct drm_atomic_state *state);
+				      struct drm_atomic_commit *state);
 void vmw_du_plane_cleanup_fb(struct drm_plane *plane,
 			     struct drm_plane_state *old_state);
 void vmw_du_plane_reset(struct drm_plane *plane);
@@ -420,9 +418,9 @@ void vmw_du_plane_destroy_state(struct drm_plane *plane,
 void vmw_du_plane_unpin_surf(struct vmw_plane_state *vps);
 
 int vmw_du_crtc_atomic_check(struct drm_crtc *crtc,
-			     struct drm_atomic_state *state);
+			     struct drm_atomic_commit *state);
 void vmw_du_crtc_atomic_begin(struct drm_crtc *crtc,
-			      struct drm_atomic_state *state);
+			      struct drm_atomic_commit *state);
 void vmw_du_crtc_reset(struct drm_crtc *crtc);
 struct drm_crtc_state *vmw_du_crtc_duplicate_state(struct drm_crtc *crtc);
 void vmw_du_crtc_destroy_state(struct drm_crtc *crtc,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index f07669b27fbaa..af3e321745638 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -310,7 +310,7 @@ static int vmw_kms_ldu_do_bo_dirty(struct vmw_private *dev_priv,
 
 static void
 vmw_ldu_primary_plane_atomic_update(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -402,6 +402,7 @@ static const struct drm_crtc_helper_funcs vmw_ldu_crtc_helper_funcs = {
 	.atomic_flush = vmw_vkms_crtc_atomic_flush,
 	.atomic_enable = vmw_vkms_crtc_atomic_enable,
 	.atomic_disable = vmw_vkms_crtc_atomic_disable,
+	.handle_vblank_timeout  = vmw_vkms_handle_vblank_timeout,
 };
 
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index 605d037d18c6b..c83061cf74557 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -277,7 +277,7 @@ static void vmw_sou_crtc_helper_prepare(struct drm_crtc *crtc)
  * @state: Unused
  */
 static void vmw_sou_crtc_atomic_disable(struct drm_crtc *crtc,
-					struct drm_atomic_state *state)
+					struct drm_atomic_commit *state)
 {
 	struct vmw_private *dev_priv;
 	struct vmw_screen_object_unit *sou;
@@ -719,7 +719,7 @@ static int vmw_sou_plane_update_surface(struct vmw_private *dev_priv,
 
 static void
 vmw_sou_primary_plane_atomic_update(struct drm_plane *plane,
-				    struct drm_atomic_state *state)
+				    struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
@@ -797,6 +797,7 @@ static const struct drm_crtc_helper_funcs vmw_sou_crtc_helper_funcs = {
 	.atomic_flush = vmw_vkms_crtc_atomic_flush,
 	.atomic_enable = vmw_vkms_crtc_atomic_enable,
 	.atomic_disable = vmw_sou_crtc_atomic_disable,
+	.handle_vblank_timeout  = vmw_vkms_handle_vblank_timeout,
 };
 
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
index dcbacee97f61c..4139837f4cafb 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_stdu.c
@@ -414,7 +414,7 @@ static void vmw_stdu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static void vmw_stdu_crtc_atomic_disable(struct drm_crtc *crtc,
-					 struct drm_atomic_state *state)
+					 struct drm_atomic_commit *state)
 {
 	struct vmw_private *dev_priv;
 	struct vmw_screen_target_display_unit *stdu;
@@ -879,7 +879,7 @@ vmw_stdu_connector_mode_valid(struct drm_connector *connector,
  * the same mode but its relative X,Y position in the topology will change.
  */
 static int vmw_stdu_connector_atomic_check(struct drm_connector *conn,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_connector_state *conn_state;
 	struct vmw_screen_target_display_unit *du;
@@ -1399,7 +1399,7 @@ static int vmw_stdu_plane_update_surface(struct vmw_private *dev_priv,
  */
 static void
 vmw_stdu_primary_plane_atomic_update(struct drm_plane *plane,
-				     struct drm_atomic_state *state)
+				     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
@@ -1460,7 +1460,7 @@ vmw_stdu_primary_plane_atomic_update(struct drm_plane *plane,
 
 static void
 vmw_stdu_crtc_atomic_flush(struct drm_crtc *crtc,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct vmw_private *vmw = vmw_priv(crtc->dev);
 	struct vmw_screen_target_display_unit *stdu = vmw_crtc_to_stdu(crtc);
@@ -1515,6 +1515,7 @@ static const struct drm_crtc_helper_funcs vmw_stdu_crtc_helper_funcs = {
 	.atomic_flush = vmw_stdu_crtc_atomic_flush,
 	.atomic_enable = vmw_vkms_crtc_atomic_enable,
 	.atomic_disable = vmw_stdu_crtc_atomic_disable,
+	.handle_vblank_timeout  = vmw_vkms_handle_vblank_timeout,
 };
 
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c
index 5abd7f5ad2db0..7b8163b5e5014 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.c
@@ -157,27 +157,19 @@ crc_generate_worker(struct work_struct *work)
 		drm_crtc_add_crc_entry(crtc, true, frame_start++, &crc32);
 }
 
-static enum hrtimer_restart
-vmw_vkms_vblank_simulate(struct hrtimer *timer)
+bool
+vmw_vkms_handle_vblank_timeout(struct drm_crtc *crtc)
 {
-	struct vmw_display_unit *du = container_of(timer, struct vmw_display_unit, vkms.timer);
-	struct drm_crtc *crtc = &du->crtc;
+	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
 	struct vmw_private *vmw = vmw_priv(crtc->dev);
 	bool has_surface = false;
-	u64 ret_overrun;
 	bool locked, ret;
 
-	ret_overrun = hrtimer_forward_now(&du->vkms.timer,
-					  du->vkms.period_ns);
-	if (ret_overrun != 1)
-		drm_dbg_driver(crtc->dev, "vblank timer missed %lld frames.\n",
-			       ret_overrun - 1);
-
 	locked = vmw_vkms_vblank_trylock(crtc);
 	ret = drm_crtc_handle_vblank(crtc);
 	WARN_ON(!ret);
 	if (!locked)
-		return HRTIMER_RESTART;
+		return true;
 	has_surface = du->vkms.surface != NULL;
 	vmw_vkms_unlock(crtc);
 
@@ -200,7 +192,7 @@ vmw_vkms_vblank_simulate(struct hrtimer *timer)
 			drm_dbg_driver(crtc->dev, "Composer worker already queued\n");
 	}
 
-	return HRTIMER_RESTART;
+	return true;
 }
 
 void
@@ -245,32 +237,12 @@ vmw_vkms_get_vblank_timestamp(struct drm_crtc *crtc,
 			      ktime_t *vblank_time,
 			      bool in_vblank_irq)
 {
-	struct drm_device *dev = crtc->dev;
-	struct vmw_private *vmw = vmw_priv(dev);
-	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
-	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
 
 	if (!vmw->vkms_enabled)
 		return false;
 
-	if (!READ_ONCE(vblank->enabled)) {
-		*vblank_time = ktime_get();
-		return true;
-	}
-
-	*vblank_time = READ_ONCE(du->vkms.timer.node.expires);
-
-	if (WARN_ON(*vblank_time == vblank->time))
-		return true;
-
-	/*
-	 * To prevent races we roll the hrtimer forward before we do any
-	 * interrupt processing - this is how real hw works (the interrupt is
-	 * only generated after all the vblank registers are updated) and what
-	 * the vblank core expects. Therefore we need to always correct the
-	 * timestampe by one frame.
-	 */
-	*vblank_time -= du->vkms.period_ns;
+	drm_crtc_vblank_get_vblank_timeout(crtc, vblank_time);
 
 	return true;
 }
@@ -280,20 +252,11 @@ vmw_vkms_enable_vblank(struct drm_crtc *crtc)
 {
 	struct drm_device *dev = crtc->dev;
 	struct vmw_private *vmw = vmw_priv(dev);
-	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
-	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
 
 	if (!vmw->vkms_enabled)
 		return -EINVAL;
 
-	drm_calc_timestamping_constants(crtc, &crtc->mode);
-
-	hrtimer_setup(&du->vkms.timer, &vmw_vkms_vblank_simulate, CLOCK_MONOTONIC,
-		      HRTIMER_MODE_REL);
-	du->vkms.period_ns = ktime_set(0, vblank->framedur_ns);
-	hrtimer_start(&du->vkms.timer, du->vkms.period_ns, HRTIMER_MODE_REL);
-
-	return 0;
+	return drm_crtc_vblank_start_timer(crtc);
 }
 
 void
@@ -305,9 +268,9 @@ vmw_vkms_disable_vblank(struct drm_crtc *crtc)
 	if (!vmw->vkms_enabled)
 		return;
 
-	hrtimer_cancel(&du->vkms.timer);
+	drm_crtc_vblank_cancel_timer(crtc);
+
 	du->vkms.surface = NULL;
-	du->vkms.period_ns = ktime_set(0, 0);
 }
 
 enum vmw_vkms_lock_state {
@@ -331,17 +294,20 @@ vmw_vkms_crtc_init(struct drm_crtc *crtc)
 void
 vmw_vkms_crtc_cleanup(struct drm_crtc *crtc)
 {
+	struct vmw_private *vmw = vmw_priv(crtc->dev);
 	struct vmw_display_unit *du = vmw_crtc_to_du(crtc);
 
+	if (vmw->vkms_enabled)
+		drm_crtc_vblank_cancel_timer(crtc);
+
 	if (du->vkms.surface)
 		vmw_surface_unreference(&du->vkms.surface);
 	WARN_ON(work_pending(&du->vkms.crc_generator_work));
-	hrtimer_cancel(&du->vkms.timer);
 }
 
 void
 vmw_vkms_crtc_atomic_begin(struct drm_crtc *crtc,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	struct vmw_private *vmw = vmw_priv(crtc->dev);
 
@@ -351,7 +317,7 @@ vmw_vkms_crtc_atomic_begin(struct drm_crtc *crtc,
 
 void
 vmw_vkms_crtc_atomic_flush(struct drm_crtc *crtc,
-			   struct drm_atomic_state *state)
+			   struct drm_atomic_commit *state)
 {
 	unsigned long flags;
 	struct vmw_private *vmw = vmw_priv(crtc->dev);
@@ -377,7 +343,7 @@ vmw_vkms_crtc_atomic_flush(struct drm_crtc *crtc,
 
 void
 vmw_vkms_crtc_atomic_enable(struct drm_crtc *crtc,
-			    struct drm_atomic_state *state)
+			    struct drm_atomic_commit *state)
 {
 	struct vmw_private *vmw = vmw_priv(crtc->dev);
 
@@ -387,7 +353,7 @@ vmw_vkms_crtc_atomic_enable(struct drm_crtc *crtc,
 
 void
 vmw_vkms_crtc_atomic_disable(struct drm_crtc *crtc,
-			     struct drm_atomic_state *state)
+			     struct drm_atomic_commit *state)
 {
 	struct vmw_private *vmw = vmw_priv(crtc->dev);
 
@@ -516,9 +482,13 @@ vmw_vkms_set_crc_surface(struct drm_crtc *crtc,
 static inline u64
 vmw_vkms_lock_max_wait_ns(struct vmw_display_unit *du)
 {
-	s64 nsecs = ktime_to_ns(du->vkms.period_ns);
+	struct drm_crtc *crtc = &du->crtc;
+	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
+
+	if (!vblank || !vblank->framedur_ns)
+		return NSEC_PER_SEC / 60; /* disabled; assume 60 Hz */
 
-	return  (nsecs > 0) ? nsecs : 16666666;
+	return vblank->framedur_ns;
 }
 
 /**
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.h
index 69ddd33a8444f..27bc846313caf 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_vkms.h
@@ -29,10 +29,9 @@
 #ifndef VMWGFX_VKMS_H_
 #define VMWGFX_VKMS_H_
 
-#include <linux/hrtimer_types.h>
 #include <linux/types.h>
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_crtc;
 struct vmw_private;
 struct vmw_surface;
@@ -45,6 +44,7 @@ bool vmw_vkms_modeset_lock_relaxed(struct drm_crtc *crtc);
 bool vmw_vkms_vblank_trylock(struct drm_crtc *crtc);
 void vmw_vkms_unlock(struct drm_crtc *crtc);
 
+bool vmw_vkms_handle_vblank_timeout(struct drm_crtc *crtc);
 bool vmw_vkms_get_vblank_timestamp(struct drm_crtc *crtc,
 				   int *max_error,
 				   ktime_t *vblank_time,
@@ -55,12 +55,12 @@ void vmw_vkms_disable_vblank(struct drm_crtc *crtc);
 void vmw_vkms_crtc_init(struct drm_crtc *crtc);
 void vmw_vkms_crtc_cleanup(struct drm_crtc *crtc);
 void  vmw_vkms_crtc_atomic_begin(struct drm_crtc *crtc,
-				 struct drm_atomic_state *state);
-void vmw_vkms_crtc_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_state *state);
+				 struct drm_atomic_commit *state);
+void vmw_vkms_crtc_atomic_flush(struct drm_crtc *crtc, struct drm_atomic_commit *state);
 void vmw_vkms_crtc_atomic_enable(struct drm_crtc *crtc,
-				 struct drm_atomic_state *state);
+				 struct drm_atomic_commit *state);
 void vmw_vkms_crtc_atomic_disable(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state);
+				  struct drm_atomic_commit *state);
 
 const char *const *vmw_vkms_get_crc_sources(struct drm_crtc *crtc,
 					    size_t *count);
diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
index 03242e8b3d876..09661f079d038 100644
--- a/drivers/gpu/drm/xe/Makefile
+++ b/drivers/gpu/drm/xe/Makefile
@@ -113,6 +113,7 @@ xe-y += xe_bb.o \
 	xe_pxp_submit.o \
 	xe_query.o \
 	xe_range_fence.o \
+	xe_ras.o \
 	xe_reg_sr.o \
 	xe_reg_whitelist.o \
 	xe_ring_ops.o \
@@ -124,6 +125,9 @@ xe-y += xe_bb.o \
 	xe_step.o \
 	xe_survivability_mode.o \
 	xe_sync.o \
+	xe_sysctrl.o \
+	xe_sysctrl_event.o \
+	xe_sysctrl_mailbox.o \
 	xe_tile.o \
 	xe_tile_sysfs.o \
 	xe_tlb_inval.o \
@@ -212,7 +216,6 @@ $(obj)/i915-display/%.o: $(srctree)/drivers/gpu/drm/i915/display/%.c FORCE
 
 # Display code specific to xe
 xe-$(CONFIG_DRM_XE_DISPLAY) += \
-	display/intel_fbdev_fb.o \
 	display/xe_display.o \
 	display/xe_display_bo.o \
 	display/xe_display_pcode.o \
@@ -262,6 +265,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_display_power.o \
 	i915-display/intel_display_power_map.o \
 	i915-display/intel_display_power_well.o \
+	i915-display/intel_display_reset.o \
 	i915-display/intel_display_rpm.o \
 	i915-display/intel_display_rps.o \
 	i915-display/intel_display_trace.o \
@@ -305,6 +309,7 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/intel_link_bw.o \
 	i915-display/intel_lspcon.o \
 	i915-display/intel_lt_phy.o \
+	i915-display/intel_mchbar.o \
 	i915-display/intel_modeset_lock.o \
 	i915-display/intel_modeset_setup.o \
 	i915-display/intel_modeset_verify.o \
@@ -331,7 +336,8 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \
 	i915-display/skl_prefill.o \
 	i915-display/skl_scaler.o \
 	i915-display/skl_universal_plane.o \
-	i915-display/skl_watermark.o
+	i915-display/skl_watermark.o \
+	i915-display/vlv_sideband.o
 
 ifeq ($(CONFIG_ACPI),y)
 	xe-$(CONFIG_DRM_XE_DISPLAY) += \
diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
index e33bd622ab448..644f5a4226d7e 100644
--- a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
+++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h
@@ -192,6 +192,13 @@ enum  {
  * `GuC KLV`_ keys available for use with PF2GUC_UPDATE_VGT_POLICY.
  *
  * _`GUC_KLV_VGT_POLICY_SCHED_IF_IDLE` : 0x8001
+ *      [From 70.12.0]
+ *      This config allows to update scheduling priority of PF and all VFs at once.
+ *      Setting this policy to 0 updates all VFs scheduling priorities to LOW, and
+ *      setting this policy to 1 updates all VFs scheduling priorities to NORMAL.
+ *      Those changes will take effect on the next VF-Switch event.
+ *
+ *      [Before 70.12.0]
  *      This config sets whether strict scheduling is enabled whereby any VF
  *      that doesn’t have work to submit is still allocated a fixed execution
  *      time-slice to ensure active VFs execution is always consistent even
@@ -496,6 +503,7 @@ enum xe_guc_klv_ids {
 	GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO					= 0x900a,
 	GUC_WA_KLV_RESET_BB_STACK_PTR_ON_VF_SWITCH					= 0x900b,
 	GUC_WA_KLV_RESTORE_UNSAVED_MEDIA_CONTROL_REG					= 0x900c,
+	GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG			= 0x900e,
 };
 
 #endif
diff --git a/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h b/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h
new file mode 100644
index 0000000000000..4cbde267ac442
--- /dev/null
+++ b/drivers/gpu/drm/xe/abi/xe_sysctrl_abi.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_SYSCTRL_ABI_H_
+#define _XE_SYSCTRL_ABI_H_
+
+#include <linux/types.h>
+
+/**
+ * DOC: System Controller ABI
+ *
+ * This header defines the Application Binary Interface (ABI) used by
+ * drm/xe to communicate with System Controller firmware on Intel Xe3p
+ * discrete GPU platforms.
+ *
+ * System Controller (sysctrl) is a firmware-managed entity on Intel
+ * dGPUs responsible for certain low-level platform management
+ * functions.
+ *
+ * Communication protocol:
+ *
+ * Communication uses a mailbox interface with messages composed of:
+ *
+ * - Application message header (struct xe_sysctrl_app_msg_hdr)
+ *   containing group_id, command, and version
+ * - Variable-length, command-specific payload
+ *
+ * Message header format:
+ *
+ * The 32-bit application message header is packed as:
+ *
+ * - Bits [7:0]   : Group ID identifying command group
+ * - Bits [15:8]  : Command identifier within group
+ * - Bits [23:16] : Command version for interface compatibility
+ * - Bits [31:24] : Reserved, must be zero
+ *
+ * This header defines firmware ABI message formats and constants shared
+ * between driver and System Controller firmware.
+ */
+
+/**
+ * struct xe_sysctrl_app_msg_hdr - Application layer message header
+ * @data: 32-bit header data
+ *
+ * Header structure for application-level messages.
+ */
+struct xe_sysctrl_app_msg_hdr {
+	u32 data;
+} __packed;
+
+#define SYSCTRL_HDR_GROUP_ID_MASK	GENMASK(7, 0)
+#define SYSCTRL_HDR_COMMAND_MASK	GENMASK(14, 8)
+#define SYSCTRL_HDR_COMMAND_MAX		0x7f
+#define SYSCTRL_HDR_IS_RESPONSE		BIT(15)
+#define SYSCTRL_HDR_RESERVED_MASK	GENMASK(23, 16)
+#define SYSCTRL_HDR_RESULT_MASK		GENMASK(31, 24)
+
+#define APP_HDR_GROUP_ID_MASK		GENMASK(7, 0)
+#define APP_HDR_COMMAND_MASK		GENMASK(15, 8)
+#define APP_HDR_VERSION_MASK		GENMASK(23, 16)
+#define APP_HDR_RESERVED_MASK		GENMASK(31, 24)
+
+#endif
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h
index ce986f0e8f38d..552975a30ba23 100644
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h
+++ b/drivers/gpu/drm/xe/compat-i915-headers/intel_clock_gating.h
@@ -3,4 +3,12 @@
  * Copyright © 2023 Intel Corporation
  */
 
-#include "../../i915/intel_clock_gating.h"
+#ifndef __INTEL_CLOCK_GATING_H__
+#define __INTEL_CLOCK_GATING_H__
+
+struct drm_device;
+
+static inline void intel_clock_gating_init(struct drm_device *drm) {}
+static inline void intel_clock_gating_hooks_init(struct drm_device *drm) {}
+
+#endif /* __INTEL_CLOCK_GATING_H__ */
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h
deleted file mode 100644
index 55b3169853401..0000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_mchbar_regs.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "../../i915/intel_mchbar_regs.h"
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h
deleted file mode 100644
index 8c15867fd613d..0000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_pci_config.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "../../i915/intel_pci_config.h"
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h b/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h
deleted file mode 100644
index 0eabe2866f5fd..0000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/intel_step.h
+++ /dev/null
@@ -1,13 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#ifndef __INTEL_STEP_H__
-#define __INTEL_STEP_H__
-
-#include "xe_step_types.h"
-
-#define intel_step xe_step
-
-#endif /* __INTEL_STEP_H__ */
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h b/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h
deleted file mode 100644
index 69e1935e9cdf5..0000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2013-2021 Intel Corporation
- */
-
-#ifndef _VLV_IOSF_SB_H_
-#define _VLV_IOSF_SB_H_
-
-#include <linux/types.h>
-
-#include "vlv_iosf_sb_reg.h"
-
-struct drm_device;
-
-enum vlv_iosf_sb_unit {
-	VLV_IOSF_SB_BUNIT,
-	VLV_IOSF_SB_CCK,
-	VLV_IOSF_SB_CCU,
-	VLV_IOSF_SB_DPIO,
-	VLV_IOSF_SB_DPIO_2,
-	VLV_IOSF_SB_FLISDSI,
-	VLV_IOSF_SB_GPIO,
-	VLV_IOSF_SB_NC,
-	VLV_IOSF_SB_PUNIT,
-};
-
-static inline void vlv_iosf_sb_get(struct drm_device *drm, unsigned long ports)
-{
-}
-static inline u32 vlv_iosf_sb_read(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr)
-{
-	return 0;
-}
-static inline int vlv_iosf_sb_write(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr, u32 val)
-{
-	return 0;
-}
-static inline void vlv_iosf_sb_put(struct drm_device *drm, unsigned long ports)
-{
-}
-
-#endif /* _VLV_IOSF_SB_H_ */
diff --git a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h b/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h
deleted file mode 100644
index cb7fa8e794a60..0000000000000
--- a/drivers/gpu/drm/xe/compat-i915-headers/vlv_iosf_sb_reg.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include "../../i915/vlv_iosf_sb_reg.h"
diff --git a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c b/drivers/gpu/drm/xe/display/intel_fbdev_fb.c
deleted file mode 100644
index d7030e4d814ca..0000000000000
--- a/drivers/gpu/drm/xe/display/intel_fbdev_fb.c
+++ /dev/null
@@ -1,120 +0,0 @@
-/* SPDX-License-Identifier: MIT */
-/*
- * Copyright © 2023 Intel Corporation
- */
-
-#include <linux/fb.h>
-
-#include "intel_fbdev_fb.h"
-#include "xe_bo.h"
-#include "xe_ttm_stolen_mgr.h"
-#include "xe_wa.h"
-
-#include <generated/xe_device_wa_oob.h>
-
-/*
- * FIXME: There shouldn't be any reason to have XE_PAGE_SIZE stride
- * alignment. The same 64 as i915 uses should be fine, and we shouldn't need to
- * have driver specific values. However, dropping the stride alignment to 64
- * leads to underflowing the bo pin count in the atomic cleanup work.
- */
-u32 intel_fbdev_fb_pitch_align(u32 stride)
-{
-	return ALIGN(stride, XE_PAGE_SIZE);
-}
-
-bool intel_fbdev_fb_prefer_stolen(struct drm_device *drm, unsigned int size)
-{
-	struct xe_device *xe = to_xe_device(drm);
-	struct ttm_resource_manager *stolen;
-
-	stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
-	if (!stolen)
-		return false;
-
-	if (IS_DGFX(xe))
-		return false;
-
-	if (XE_DEVICE_WA(xe, 22019338487_display))
-		return false;
-
-	/*
-	 * If the FB is too big, just don't use it since fbdev is not very
-	 * important and we should probably use that space with FBC or other
-	 * features.
-	 */
-	return stolen->size >= size * 2;
-}
-
-struct drm_gem_object *intel_fbdev_fb_bo_create(struct drm_device *drm, int size)
-{
-	struct xe_device *xe = to_xe_device(drm);
-	struct xe_bo *obj;
-
-	obj = ERR_PTR(-ENODEV);
-
-	if (intel_fbdev_fb_prefer_stolen(drm, size)) {
-		obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe),
-						size,
-						ttm_bo_type_kernel,
-						XE_BO_FLAG_FORCE_WC |
-						XE_BO_FLAG_STOLEN |
-						XE_BO_FLAG_GGTT,
-						false);
-		if (!IS_ERR(obj))
-			drm_info(&xe->drm, "Allocated fbdev into stolen\n");
-		else
-			drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj));
-	} else {
-		drm_info(&xe->drm, "Allocating fbdev: Stolen memory not preferred.\n");
-	}
-
-	if (IS_ERR(obj)) {
-		obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size,
-						ttm_bo_type_kernel,
-						XE_BO_FLAG_FORCE_WC |
-						XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
-						XE_BO_FLAG_GGTT,
-						false);
-	}
-
-	if (IS_ERR(obj)) {
-		drm_err(&xe->drm, "failed to allocate framebuffer (%pe)\n", obj);
-		return ERR_PTR(-ENOMEM);
-	}
-
-	return &obj->ttm.base;
-}
-
-void intel_fbdev_fb_bo_destroy(struct drm_gem_object *obj)
-{
-	xe_bo_unpin_map_no_vm(gem_to_xe_bo(obj));
-}
-
-int intel_fbdev_fb_fill_info(struct drm_device *drm, struct fb_info *info,
-			     struct drm_gem_object *_obj, struct i915_vma *vma)
-{
-	struct xe_bo *obj = gem_to_xe_bo(_obj);
-	struct pci_dev *pdev = to_pci_dev(drm->dev);
-
-	if (!(obj->flags & XE_BO_FLAG_SYSTEM)) {
-		if (obj->flags & XE_BO_FLAG_STOLEN)
-			info->fix.smem_start = xe_ttm_stolen_io_offset(obj, 0);
-		else
-			info->fix.smem_start =
-				pci_resource_start(pdev, 2) +
-				xe_bo_addr(obj, 0, XE_PAGE_SIZE);
-
-		info->fix.smem_len = obj->ttm.base.size;
-	} else {
-		/* XXX: Pure fiction, as the BO may not be physically accessible.. */
-		info->fix.smem_start = 0;
-		info->fix.smem_len = obj->ttm.base.size;
-	}
-	XE_WARN_ON(iosys_map_is_null(&obj->vmap));
-
-	info->screen_base = obj->vmap.vaddr_iomem;
-	info->screen_size = obj->ttm.base.size;
-
-	return 0;
-}
diff --git a/drivers/gpu/drm/xe/display/xe_display.c b/drivers/gpu/drm/xe/display/xe_display.c
index 00dfa68af29aa..8d08da60336d7 100644
--- a/drivers/gpu/drm/xe/display/xe_display.c
+++ b/drivers/gpu/drm/xe/display/xe_display.c
@@ -36,10 +36,12 @@
 #include "intel_hotplug.h"
 #include "intel_opregion.h"
 #include "skl_watermark.h"
+#include "xe_device.h"
 #include "xe_display_bo.h"
 #include "xe_display_pcode.h"
 #include "xe_display_rpm.h"
 #include "xe_dsb_buffer.h"
+#include "xe_fb_pin.h"
 #include "xe_frontbuffer.h"
 #include "xe_hdcp_gsc.h"
 #include "xe_initial_plane.h"
@@ -69,31 +71,9 @@ bool xe_display_driver_probe_defer(struct pci_dev *pdev)
 	return intel_display_driver_probe_defer(pdev);
 }
 
-/**
- * xe_display_driver_set_hooks - Add driver flags and hooks for display
- * @driver: DRM device driver
- *
- * Set features and function hooks in @driver that are needed for driving the
- * display IP. This sets the driver's capability of driving display, regardless
- * if the device has it enabled
- *
- * Note: This is called before xe or display device creation.
- */
-void xe_display_driver_set_hooks(struct drm_driver *driver)
-{
-	if (!xe_modparam.probe_display)
-		return;
-
-#ifdef CONFIG_DRM_FBDEV_EMULATION
-	driver->fbdev_probe = intel_fbdev_driver_fbdev_probe;
-#endif
-
-	driver->driver_features |= DRIVER_MODESET | DRIVER_ATOMIC;
-}
-
 static void unset_display_features(struct xe_device *xe)
 {
-	xe->drm.driver_features &= ~(DRIVER_MODESET | DRIVER_ATOMIC);
+	xe->drm.driver_features &= ~XE_DISPLAY_DRIVER_FEATURES;
 }
 
 static void xe_display_fini_early(void *arg)
@@ -124,6 +104,15 @@ int xe_display_init_early(struct xe_device *xe)
 
 	intel_display_driver_early_probe(display);
 
+	intel_display_device_info_runtime_init(display);
+
+	/* Display may have been disabled at runtime init */
+	if (!intel_display_device_present(display)) {
+		xe->info.probe_display = false;
+		unset_display_features(xe);
+		return 0;
+	}
+
 	/* Early display init.. */
 	intel_opregion_setup(display);
 
@@ -137,8 +126,6 @@ int xe_display_init_early(struct xe_device *xe)
 
 	intel_bw_init_hw(display);
 
-	intel_display_device_info_runtime_init(display);
-
 	err = intel_display_driver_probe_noirq(display);
 	if (err)
 		goto err_opregion;
@@ -214,7 +201,7 @@ void xe_display_irq_handler(struct xe_device *xe, u32 master_ctl)
 		return;
 
 	if (master_ctl & DISPLAY_IRQ)
-		gen11_display_irq_handler(display);
+		intel_display_irq_handler(display, NULL);
 }
 
 void xe_display_irq_enable(struct xe_device *xe, u32 gu_misc_iir)
@@ -235,7 +222,7 @@ void xe_display_irq_reset(struct xe_device *xe)
 	if (!xe->info.probe_display)
 		return;
 
-	gen11_display_irq_reset(display);
+	intel_display_irq_reset(display);
 }
 
 void xe_display_irq_postinstall(struct xe_device *xe)
@@ -245,7 +232,7 @@ void xe_display_irq_postinstall(struct xe_device *xe)
 	if (!xe->info.probe_display)
 		return;
 
-	gen11_de_irq_postinstall(display);
+	intel_display_irq_postinstall(display);
 }
 
 static bool suspend_to_idle(void)
@@ -520,9 +507,10 @@ void xe_display_pm_runtime_resume(struct xe_device *xe)
 
 static void display_device_remove(struct drm_device *dev, void *arg)
 {
-	struct intel_display *display = arg;
+	struct xe_device *xe = arg;
 
-	intel_display_device_remove(display);
+	intel_display_device_remove(xe->display);
+	xe->display = NULL;
 }
 
 static bool irq_enabled(struct drm_device *drm)
@@ -552,6 +540,7 @@ static bool has_auxccs(struct drm_device *drm)
 static const struct intel_display_parent_interface parent = {
 	.bo = &xe_display_bo_interface,
 	.dsb = &xe_display_dsb_interface,
+	.fb_pin = &xe_display_fb_pin_interface,
 	.frontbuffer = &xe_display_frontbuffer_interface,
 	.hdcp = &xe_display_hdcp_interface,
 	.initial_plane = &xe_display_initial_plane_interface,
@@ -587,12 +576,12 @@ int xe_display_probe(struct xe_device *xe)
 	if (IS_ERR(display))
 		return PTR_ERR(display);
 
-	err = drmm_add_action_or_reset(&xe->drm, display_device_remove, display);
+	xe->display = display;
+
+	err = drmm_add_action_or_reset(&xe->drm, display_device_remove, xe);
 	if (err)
 		return err;
 
-	xe->display = display;
-
 	if (intel_display_device_present(display))
 		return 0;
 
@@ -601,3 +590,11 @@ int xe_display_probe(struct xe_device *xe)
 	unset_display_features(xe);
 	return 0;
 }
+
+#ifdef CONFIG_DRM_FBDEV_EMULATION
+int xe_display_driver_fbdev_probe(struct drm_fb_helper *fbh,
+				  struct drm_fb_helper_surface_size *sizes)
+{
+	return intel_fbdev_driver_fbdev_probe(fbh, sizes);
+}
+#endif
diff --git a/drivers/gpu/drm/xe/display/xe_display.h b/drivers/gpu/drm/xe/display/xe_display.h
index 76db95c25f7e2..60291cb154df7 100644
--- a/drivers/gpu/drm/xe/display/xe_display.h
+++ b/drivers/gpu/drm/xe/display/xe_display.h
@@ -6,14 +6,20 @@
 #ifndef _XE_DISPLAY_H_
 #define _XE_DISPLAY_H_
 
-#include "xe_device.h"
+#include <linux/types.h>
 
 struct drm_driver;
+struct drm_fb_helper;
+struct drm_fb_helper_surface_size;
+struct pci_dev;
+struct xe_device;
 
 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY)
 
 bool xe_display_driver_probe_defer(struct pci_dev *pdev);
-void xe_display_driver_set_hooks(struct drm_driver *driver);
+
+int xe_display_driver_fbdev_probe(struct drm_fb_helper *fbh,
+				  struct drm_fb_helper_surface_size *sizes);
 
 int xe_display_probe(struct xe_device *xe);
 
@@ -38,11 +44,18 @@ void xe_display_pm_runtime_suspend(struct xe_device *xe);
 void xe_display_pm_runtime_suspend_late(struct xe_device *xe);
 void xe_display_pm_runtime_resume(struct xe_device *xe);
 
+#define XE_DISPLAY_DRIVER_FEATURES	(DRIVER_MODESET | DRIVER_ATOMIC)
+#define XE_DISPLAY_DRIVER_OPS						\
+	.fbdev_probe = PTR_IF(IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION),	\
+			      xe_display_driver_fbdev_probe)
+
 #else
 
+#define XE_DISPLAY_DRIVER_FEATURES	0
+#define XE_DISPLAY_DRIVER_OPS \
+	.fbdev_probe = NULL
+
 static inline int xe_display_driver_probe_defer(struct pci_dev *pdev) { return 0; }
-static inline void xe_display_driver_set_hooks(struct drm_driver *driver) { }
-static inline void xe_display_driver_remove(struct xe_device *xe) {}
 
 static inline int xe_display_probe(struct xe_device *xe) { return 0; }
 
diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.c b/drivers/gpu/drm/xe/display/xe_display_bo.c
index 1d81b9908265f..7fbac223b0977 100644
--- a/drivers/gpu/drm/xe/display/xe_display_bo.c
+++ b/drivers/gpu/drm/xe/display/xe_display_bo.c
@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: MIT
 /* Copyright © 2024 Intel Corporation */
 
+#include <linux/fb.h>
+
 #include <drm/drm_gem.h>
 #include <drm/intel/display_parent_interface.h>
 
@@ -8,6 +10,10 @@
 #include "xe_bo.h"
 #include "xe_display_bo.h"
 #include "xe_pxp.h"
+#include "xe_ttm_stolen_mgr.h"
+#include "xe_wa.h"
+
+#include <generated/xe_device_wa_oob.h>
 
 static bool xe_display_bo_is_protected(struct drm_gem_object *obj)
 {
@@ -101,6 +107,114 @@ xe_display_bo_framebuffer_lookup(struct drm_device *drm,
 	return gem;
 }
 
+#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
+/*
+ * FIXME: There shouldn't be any reason to have XE_PAGE_SIZE stride
+ * alignment. The same 64 as i915 uses should be fine, and we shouldn't need to
+ * have driver specific values. However, dropping the stride alignment to 64
+ * leads to underflowing the bo pin count in the atomic cleanup work.
+ */
+static u32 xe_display_bo_fbdev_pitch_align(u32 stride)
+{
+	return ALIGN(stride, XE_PAGE_SIZE);
+}
+
+bool xe_display_bo_fbdev_prefer_stolen(struct xe_device *xe, unsigned int size)
+{
+	struct ttm_resource_manager *stolen;
+
+	stolen = ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
+	if (!stolen)
+		return false;
+
+	if (IS_DGFX(xe))
+		return false;
+
+	if (XE_DEVICE_WA(xe, 22019338487_display))
+		return false;
+
+	/*
+	 * If the FB is too big, just don't use it since fbdev is not very
+	 * important and we should probably use that space with FBC or other
+	 * features.
+	 */
+	return stolen->size >= (size * 2) >> PAGE_SHIFT;
+}
+
+static struct drm_gem_object *xe_display_bo_fbdev_create(struct drm_device *drm, int size)
+{
+	struct xe_device *xe = to_xe_device(drm);
+	struct xe_bo *obj;
+
+	obj = ERR_PTR(-ENODEV);
+
+	if (xe_display_bo_fbdev_prefer_stolen(xe, size)) {
+		obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe),
+						size,
+						ttm_bo_type_kernel,
+						XE_BO_FLAG_FORCE_WC |
+						XE_BO_FLAG_STOLEN |
+						XE_BO_FLAG_GGTT,
+						false);
+		if (!IS_ERR(obj))
+			drm_info(&xe->drm, "Allocated fbdev into stolen\n");
+		else
+			drm_info(&xe->drm, "Allocated fbdev into stolen failed: %li\n", PTR_ERR(obj));
+	} else {
+		drm_info(&xe->drm, "Allocating fbdev: Stolen memory not preferred.\n");
+	}
+
+	if (IS_ERR(obj)) {
+		obj = xe_bo_create_pin_map_novm(xe, xe_device_get_root_tile(xe), size,
+						ttm_bo_type_kernel,
+						XE_BO_FLAG_FORCE_WC |
+						XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) |
+						XE_BO_FLAG_GGTT,
+						false);
+	}
+
+	if (IS_ERR(obj)) {
+		drm_err(&xe->drm, "failed to allocate framebuffer (%pe)\n", obj);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return &obj->ttm.base;
+}
+
+static void xe_display_bo_fbdev_destroy(struct drm_gem_object *obj)
+{
+	xe_bo_unpin_map_no_vm(gem_to_xe_bo(obj));
+}
+
+static int xe_display_bo_fbdev_fill_info(struct drm_gem_object *_obj, struct fb_info *info,
+					 struct i915_vma *vma)
+{
+	struct xe_bo *obj = gem_to_xe_bo(_obj);
+	struct pci_dev *pdev = to_pci_dev(_obj->dev->dev);
+
+	if (!(obj->flags & XE_BO_FLAG_SYSTEM)) {
+		if (obj->flags & XE_BO_FLAG_STOLEN)
+			info->fix.smem_start = xe_ttm_stolen_io_offset(obj, 0);
+		else
+			info->fix.smem_start =
+				pci_resource_start(pdev, 2) +
+				xe_bo_addr(obj, 0, XE_PAGE_SIZE);
+
+		info->fix.smem_len = obj->ttm.base.size;
+	} else {
+		/* XXX: Pure fiction, as the BO may not be physically accessible.. */
+		info->fix.smem_start = 0;
+		info->fix.smem_len = obj->ttm.base.size;
+	}
+	XE_WARN_ON(iosys_map_is_null(&obj->vmap));
+
+	info->screen_base = obj->vmap.vaddr_iomem;
+	info->screen_size = obj->ttm.base.size;
+
+	return 0;
+}
+#endif
+
 const struct intel_display_bo_interface xe_display_bo_interface = {
 	.is_protected = xe_display_bo_is_protected,
 	.key_check = xe_pxp_obj_key_check,
@@ -109,4 +223,10 @@ const struct intel_display_bo_interface xe_display_bo_interface = {
 	.framebuffer_init = xe_display_bo_framebuffer_init,
 	.framebuffer_fini = xe_display_bo_framebuffer_fini,
 	.framebuffer_lookup = xe_display_bo_framebuffer_lookup,
+#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
+	.fbdev_create = xe_display_bo_fbdev_create,
+	.fbdev_destroy = xe_display_bo_fbdev_destroy,
+	.fbdev_fill_info = xe_display_bo_fbdev_fill_info,
+	.fbdev_pitch_align = xe_display_bo_fbdev_pitch_align,
+#endif
 };
diff --git a/drivers/gpu/drm/xe/display/xe_display_bo.h b/drivers/gpu/drm/xe/display/xe_display_bo.h
index 6879c104b0b1f..c72056884ff41 100644
--- a/drivers/gpu/drm/xe/display/xe_display_bo.h
+++ b/drivers/gpu/drm/xe/display/xe_display_bo.h
@@ -4,6 +4,12 @@
 #ifndef __XE_DISPLAY_BO_H__
 #define __XE_DISPLAY_BO_H__
 
+#include <linux/types.h>
+
+struct xe_device;
+
+bool xe_display_bo_fbdev_prefer_stolen(struct xe_device *xe, unsigned int size);
+
 extern const struct intel_display_bo_interface xe_display_bo_interface;
 
 #endif
diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.c b/drivers/gpu/drm/xe/display/xe_fb_pin.c
index e45a1e7a4670e..f93c98bec5b5f 100644
--- a/drivers/gpu/drm/xe/display/xe_fb_pin.c
+++ b/drivers/gpu/drm/xe/display/xe_fb_pin.c
@@ -3,17 +3,21 @@
  * Copyright © 2021 Intel Corporation
  */
 
+#include <drm/intel/display_parent_interface.h>
 #include <drm/ttm/ttm_bo.h>
 
-#include "intel_display_core.h"
-#include "intel_display_types.h"
+/* FIXME move the types to parent interface? */
+#include "i915_gtt_view_types.h"
+
+/* FIXME move intel_remapped_info_size() & co. to parent interface? */
 #include "intel_fb.h"
-#include "intel_fb_pin.h"
-#include "intel_fbdev.h"
+
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_display_vma.h"
+#include "xe_fb_pin.h"
 #include "xe_ggtt.h"
+#include "xe_pat.h"
 #include "xe_pm.h"
 #include "xe_vram_types.h"
 
@@ -24,7 +28,7 @@ write_dpt_rotated(struct xe_bo *bo, struct iosys_map *map, u32 *dpt_ofs, u32 bo_
 	struct xe_device *xe = xe_bo_device(bo);
 	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
 	u32 column, row;
-	u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
+	u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE));
 
 	/* TODO: Maybe rewrite so we can traverse the bo addresses sequentially,
 	 * by writing dpt/ggtt in a different order?
@@ -64,7 +68,7 @@ write_dpt_remapped_linear(struct xe_bo *bo, struct iosys_map *map,
 	struct xe_device *xe = xe_bo_device(bo);
 	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
 	const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo,
-						 xe->pat.idx[XE_CACHE_NONE]);
+						 xe_cache_pat_idx(xe, XE_CACHE_NONE));
 	unsigned int offset = plane->offset * XE_PAGE_SIZE;
 	unsigned int size = plane->size;
 
@@ -87,7 +91,7 @@ write_dpt_remapped_tiled(struct xe_bo *bo, struct iosys_map *map,
 	struct xe_device *xe = xe_bo_device(bo);
 	struct xe_ggtt *ggtt = xe_device_get_root_tile(xe)->mem.ggtt;
 	const u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo,
-						 xe->pat.idx[XE_CACHE_NONE]);
+						 xe_cache_pat_idx(xe, XE_CACHE_NONE));
 	unsigned int offset, column, row;
 
 	for (row = 0; row < plane->height; row++) {
@@ -139,22 +143,21 @@ write_dpt_remapped(struct xe_bo *bo,
 	}
 }
 
-static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,
-			       const struct i915_gtt_view *view,
-			       struct i915_vma *vma,
-			       unsigned int alignment)
+static int __xe_pin_fb_vma_dpt(struct drm_gem_object *obj,
+			       const struct intel_fb_pin_params *pin_params,
+			       struct i915_vma *vma)
 {
-	struct xe_device *xe = to_xe_device(fb->base.dev);
+	struct xe_device *xe = to_xe_device(obj->dev);
 	struct xe_tile *tile0 = xe_device_get_root_tile(xe);
 	struct xe_ggtt *ggtt = tile0->mem.ggtt;
-	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
+	const struct i915_gtt_view *view = pin_params->view;
 	struct xe_bo *bo = gem_to_xe_bo(obj), *dpt;
 	u32 dpt_size, size = bo->ttm.base.size;
 
 	if (view->type == I915_GTT_VIEW_NORMAL)
 		dpt_size = ALIGN(size / XE_PAGE_SIZE * 8, XE_PAGE_SIZE);
 	else if (view->type == I915_GTT_VIEW_REMAPPED)
-		dpt_size = ALIGN(intel_remapped_info_size(&fb->remapped_view.gtt.remapped) * 8,
+		dpt_size = ALIGN(intel_remapped_info_size(&view->remapped) * 8,
 				 XE_PAGE_SIZE);
 	else
 		/* display uses 4K tiles instead of bytes here, convert to entries.. */
@@ -168,7 +171,7 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,
 						   XE_BO_FLAG_VRAM0 |
 						   XE_BO_FLAG_GGTT |
 						   XE_BO_FLAG_PAGETABLE,
-						   alignment, false);
+						   pin_params->alignment, false);
 	else
 		dpt = xe_bo_create_pin_map_at_novm(xe, tile0,
 						   dpt_size,  ~0ull,
@@ -176,7 +179,7 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,
 						   XE_BO_FLAG_STOLEN |
 						   XE_BO_FLAG_GGTT |
 						   XE_BO_FLAG_PAGETABLE,
-						   alignment, false);
+						   pin_params->alignment, false);
 	if (IS_ERR(dpt))
 		dpt = xe_bo_create_pin_map_at_novm(xe, tile0,
 						   dpt_size,  ~0ull,
@@ -185,12 +188,12 @@ static int __xe_pin_fb_vma_dpt(const struct intel_framebuffer *fb,
 						   XE_BO_FLAG_GGTT |
 						   XE_BO_FLAG_PAGETABLE |
 						   XE_BO_FLAG_FORCE_WC,
-						   alignment, false);
+						   pin_params->alignment, false);
 	if (IS_ERR(dpt))
 		return PTR_ERR(dpt);
 
 	if (view->type == I915_GTT_VIEW_NORMAL) {
-		u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
+		u64 pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE));
 		u32 x;
 
 		for (x = 0; x < size / XE_PAGE_SIZE; x++) {
@@ -268,14 +271,13 @@ static void write_ggtt_rotated_node(struct xe_ggtt *ggtt, struct xe_ggtt_node *n
 				   rot_info->plane[i].dst_stride);
 }
 
-static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
-				const struct i915_gtt_view *view,
-				struct i915_vma *vma,
-				unsigned int alignment)
+static int __xe_pin_fb_vma_ggtt(struct drm_gem_object *obj,
+				const struct intel_fb_pin_params *pin_params,
+				struct i915_vma *vma)
 {
-	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
+	const struct i915_gtt_view *view = pin_params->view;
 	struct xe_bo *bo = gem_to_xe_bo(obj);
-	struct xe_device *xe = to_xe_device(fb->base.dev);
+	struct xe_device *xe = to_xe_device(obj->dev);
 	struct xe_tile *tile0 = xe_device_get_root_tile(xe);
 	struct xe_ggtt *ggtt = tile0->mem.ggtt;
 	u64 pte, size;
@@ -287,7 +289,7 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
 	 */
 	guard(xe_pm_runtime_noresume)(xe);
 
-	align = XE_PAGE_SIZE;
+	align = max(XE_PAGE_SIZE, pin_params->alignment);
 	if (xe_bo_is_vram(bo) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K)
 		align = max(align, SZ_64K);
 
@@ -306,7 +308,7 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
 		/* display uses tiles instead of bytes here, so convert it back.. */
 		size = intel_rotation_info_size(&view->rotated) * XE_PAGE_SIZE;
 
-	pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe->pat.idx[XE_CACHE_NONE]);
+	pte = xe_ggtt_encode_pte_flags(ggtt, bo, xe_cache_pat_idx(xe, XE_CACHE_NONE));
 	vma->node = xe_ggtt_insert_node_transform(ggtt, bo, pte,
 						  ALIGN(size, align), align,
 						  view->type == I915_GTT_VIEW_NORMAL ?
@@ -318,25 +320,25 @@ static int __xe_pin_fb_vma_ggtt(const struct intel_framebuffer *fb,
 	return ret;
 }
 
-static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb,
-					const struct i915_gtt_view *view,
-					unsigned int alignment)
+static struct i915_vma *__xe_pin_fb_vma(struct drm_gem_object *obj, bool is_dpt,
+					const struct intel_fb_pin_params *pin_params)
 {
-	struct drm_device *dev = fb->base.dev;
-	struct xe_device *xe = to_xe_device(dev);
+	struct xe_device *xe = to_xe_device(obj->dev);
 	struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
-	struct drm_gem_object *obj = intel_fb_bo(&fb->base);
 	struct xe_bo *bo = gem_to_xe_bo(obj);
 	struct xe_validation_ctx ctx;
 	struct drm_exec exec;
 	int ret = 0;
 
+	/* We reject creating !SCANOUT fb's, so this is weird.. */
+	drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC));
+
 	if (!vma)
 		return ERR_PTR(-ENODEV);
 
 	refcount_set(&vma->ref, 1);
 	if (IS_DGFX(to_xe_device(bo->ttm.base.dev)) &&
-	    intel_fb_rc_ccs_cc_plane(&fb->base) >= 0 &&
+	    pin_params->needs_cpu_lmem_access &&
 	    !(bo->flags & XE_BO_FLAG_NEEDS_CPU_ACCESS)) {
 		struct xe_vram_region *vram = xe_device_get_root_tile(xe)->mem.vram;
 
@@ -376,10 +378,10 @@ static struct i915_vma *__xe_pin_fb_vma(const struct intel_framebuffer *fb,
 		goto err;
 
 	vma->bo = bo;
-	if (intel_fb_uses_dpt(&fb->base))
-		ret = __xe_pin_fb_vma_dpt(fb, view, vma, alignment);
+	if (is_dpt)
+		ret = __xe_pin_fb_vma_dpt(obj, pin_params, vma);
 	else
-		ret = __xe_pin_fb_vma_ggtt(fb, view, vma,  alignment);
+		ret = __xe_pin_fb_vma_ggtt(obj, pin_params, vma);
 	if (ret)
 		goto err_unpin;
 
@@ -412,97 +414,94 @@ static void __xe_unpin_fb_vma(struct i915_vma *vma)
 	kfree(vma);
 }
 
-struct i915_vma *
-intel_fb_pin_to_ggtt(const struct drm_framebuffer *fb,
-		     const struct i915_gtt_view *view,
-		     unsigned int alignment,
-		     unsigned int phys_alignment,
-		     unsigned int vtd_guard,
-		     bool uses_fence,
-		     unsigned long *out_flags)
+int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj,
+		       const struct intel_fb_pin_params *pin_params,
+		       struct i915_vma **out_ggtt_vma,
+		       u32 *out_offset,
+		       int *out_fence_id)
 {
-	*out_flags = 0;
+	struct i915_vma *ggtt_vma;
+
+	ggtt_vma = __xe_pin_fb_vma(obj, false, pin_params);
+	if (IS_ERR(ggtt_vma))
+		return PTR_ERR(ggtt_vma);
+
+	*out_ggtt_vma = ggtt_vma;
+	*out_offset = xe_ggtt_node_addr(ggtt_vma->node);
+	if (out_fence_id)
+		*out_fence_id = -1;
 
-	return __xe_pin_fb_vma(to_intel_framebuffer(fb), view, alignment);
+	return 0;
 }
 
-void intel_fb_unpin_vma(struct i915_vma *vma, unsigned long flags)
+static void xe_fb_pin_ggtt_unpin(struct i915_vma *ggtt_vma,
+				 int fence_id)
 {
-	__xe_unpin_fb_vma(vma);
+	WARN_ON(fence_id >= 0);
+
+	__xe_unpin_fb_vma(ggtt_vma);
 }
 
-static bool reuse_vma(struct intel_plane_state *new_plane_state,
-		      const struct intel_plane_state *old_plane_state)
+static int xe_fb_pin_dpt_pin(struct drm_gem_object *obj, struct intel_dpt *dpt,
+			     const struct intel_fb_pin_params *pin_params,
+			     struct i915_vma **out_dpt_vma,
+			     struct i915_vma **out_ggtt_vma,
+			     u32 *out_offset)
 {
-	struct intel_framebuffer *fb = to_intel_framebuffer(new_plane_state->hw.fb);
-	struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane);
-	struct xe_device *xe = to_xe_device(fb->base.dev);
-	struct intel_display *display = xe->display;
-	struct i915_vma *vma;
-
-	if (old_plane_state->hw.fb == new_plane_state->hw.fb &&
-	    !memcmp(&old_plane_state->view.gtt,
-		    &new_plane_state->view.gtt,
-		    sizeof(new_plane_state->view.gtt))) {
-		vma = old_plane_state->ggtt_vma;
-		goto found;
-	}
+	struct i915_vma *ggtt_vma;
 
-	if (fb == intel_fbdev_framebuffer(display->fbdev.fbdev)) {
-		vma = intel_fbdev_vma_pointer(display->fbdev.fbdev);
-		if (vma)
-			goto found;
-	}
+	WARN_ON(dpt);
 
-	return false;
+	ggtt_vma = __xe_pin_fb_vma(obj, true, pin_params);
+	if (IS_ERR(ggtt_vma))
+		return PTR_ERR(ggtt_vma);
 
-found:
-	refcount_inc(&vma->ref);
-	new_plane_state->ggtt_vma = vma;
+	*out_dpt_vma = NULL; /* not used on xe */
+	*out_ggtt_vma = ggtt_vma;
+	*out_offset = xe_ggtt_node_addr(ggtt_vma->node);
 
-	new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) +
-		plane->surf_offset(new_plane_state);
-
-	return true;
+	return 0;
 }
 
-int intel_plane_pin_fb(struct intel_plane_state *new_plane_state,
-		       const struct intel_plane_state *old_plane_state)
+static void xe_fb_pin_dpt_unpin(struct intel_dpt *dpt,
+				struct i915_vma *dpt_vma,
+				struct i915_vma *ggtt_vma)
 {
-	struct drm_framebuffer *fb = new_plane_state->hw.fb;
-	struct drm_gem_object *obj = intel_fb_bo(fb);
-	struct xe_bo *bo = gem_to_xe_bo(obj);
-	struct i915_vma *vma;
-	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-	struct intel_plane *plane = to_intel_plane(new_plane_state->uapi.plane);
-	unsigned int alignment = plane->min_alignment(plane, fb, 0);
+	WARN_ON(dpt || dpt_vma);
 
-	if (reuse_vma(new_plane_state, old_plane_state))
-		return 0;
-
-	/* We reject creating !SCANOUT fb's, so this is weird.. */
-	drm_WARN_ON(bo->ttm.base.dev, !(bo->flags & XE_BO_FLAG_FORCE_WC));
-
-	vma = __xe_pin_fb_vma(intel_fb, &new_plane_state->view.gtt, alignment);
-
-	if (IS_ERR(vma))
-		return PTR_ERR(vma);
+	__xe_unpin_fb_vma(ggtt_vma);
+}
 
-	new_plane_state->ggtt_vma = vma;
+static struct i915_vma *
+xe_fb_pin_reuse_vma(struct i915_vma *old_ggtt_vma,
+		    struct drm_gem_object *old_obj,
+		    const struct i915_gtt_view *old_view,
+		    struct drm_gem_object *new_obj,
+		    const struct i915_gtt_view *new_view,
+		    u32 *out_offset)
+{
+	if (old_ggtt_vma && old_obj == new_obj &&
+	    !memcmp(old_view, new_view, sizeof(*new_view))) {
+		refcount_inc(&old_ggtt_vma->ref);
 
-	new_plane_state->surf = xe_ggtt_node_addr(new_plane_state->ggtt_vma->node) +
-		plane->surf_offset(new_plane_state);
+		*out_offset = xe_ggtt_node_addr(old_ggtt_vma->node);
 
-	return 0;
-}
+		return old_ggtt_vma;
+	}
 
-void intel_plane_unpin_fb(struct intel_plane_state *old_plane_state)
-{
-	__xe_unpin_fb_vma(old_plane_state->ggtt_vma);
-	old_plane_state->ggtt_vma = NULL;
+	return NULL;
 }
 
-void intel_fb_get_map(struct i915_vma *vma, struct iosys_map *map)
+static void xe_fb_pin_get_map(struct i915_vma *vma, struct iosys_map *map)
 {
 	*map = vma->bo->vmap;
 }
+
+const struct intel_display_fb_pin_interface xe_display_fb_pin_interface = {
+	.ggtt_pin = xe_fb_pin_ggtt_pin,
+	.ggtt_unpin = xe_fb_pin_ggtt_unpin,
+	.dpt_pin = xe_fb_pin_dpt_pin,
+	.dpt_unpin = xe_fb_pin_dpt_unpin,
+	.reuse_vma = xe_fb_pin_reuse_vma,
+	.get_map = xe_fb_pin_get_map,
+};
diff --git a/drivers/gpu/drm/xe/display/xe_fb_pin.h b/drivers/gpu/drm/xe/display/xe_fb_pin.h
new file mode 100644
index 0000000000000..20dd8a99a25fc
--- /dev/null
+++ b/drivers/gpu/drm/xe/display/xe_fb_pin.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: MIT */
+/* Copyright © 2026 Intel Corporation */
+
+#ifndef __XE_FB_PIN_H__
+#define __XE_FB_PIN_H__
+
+#include <linux/types.h>
+
+struct drm_gem_object;
+struct i915_vma;
+struct intel_fb_pin_params;
+
+int xe_fb_pin_ggtt_pin(struct drm_gem_object *obj,
+		       const struct intel_fb_pin_params *pin_params,
+		       struct i915_vma **out_ggtt_vma,
+		       u32 *out_offset,
+		       int *out_fence_id);
+
+extern const struct intel_display_fb_pin_interface xe_display_fb_pin_interface;
+
+#endif /* __XE_FB_PIN_H__ */
diff --git a/drivers/gpu/drm/xe/display/xe_initial_plane.c b/drivers/gpu/drm/xe/display/xe_initial_plane.c
index 8bcae552dddcf..02b46cc3b6df9 100644
--- a/drivers/gpu/drm/xe/display/xe_initial_plane.c
+++ b/drivers/gpu/drm/xe/display/xe_initial_plane.c
@@ -7,34 +7,20 @@
 
 #include "regs/xe_gtt_defs.h"
 
-#include "intel_crtc.h"
-#include "intel_display_regs.h"
-#include "intel_display_types.h"
+/* FIXME move intel_remapped_info_size() & co. */
 #include "intel_fb.h"
-#include "intel_fb_pin.h"
-#include "intel_fbdev_fb.h"
+
+/* FIXME move intel_initial_plane_config */
+#include "intel_display_types.h"
+
 #include "xe_bo.h"
+#include "xe_display_bo.h"
 #include "xe_display_vma.h"
+#include "xe_fb_pin.h"
 #include "xe_ggtt.h"
 #include "xe_mmio.h"
 #include "xe_vram_types.h"
 
-/* Early xe has no irq */
-static void xe_initial_plane_vblank_wait(struct drm_crtc *_crtc)
-{
-	struct intel_crtc *crtc = to_intel_crtc(_crtc);
-	struct xe_device *xe = to_xe_device(crtc->base.dev);
-	struct xe_reg pipe_frmtmstmp = XE_REG(i915_mmio_reg_offset(PIPE_FRMTMSTMP(crtc->pipe)));
-	u32 timestamp;
-	int ret;
-
-	timestamp = xe_mmio_read32(xe_root_tile_mmio(xe), pipe_frmtmstmp);
-
-	ret = xe_mmio_wait32_not(xe_root_tile_mmio(xe), pipe_frmtmstmp, ~0U, timestamp, 40000U, &timestamp, false);
-	if (ret < 0)
-		drm_warn(&xe->drm, "waiting for early vblank failed with %i\n", ret);
-}
-
 static struct xe_bo *
 initial_plane_bo(struct xe_device *xe,
 		 struct intel_initial_plane_config *plane_config)
@@ -86,7 +72,8 @@ initial_plane_bo(struct xe_device *xe,
 		flags |= XE_BO_FLAG_STOLEN;
 
 		if (IS_ENABLED(CONFIG_FRAMEBUFFER_CONSOLE) &&
-		    !intel_fbdev_fb_prefer_stolen(&xe->drm, plane_config->size)) {
+		    IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) &&
+		    !xe_display_bo_fbdev_prefer_stolen(xe, plane_config->size)) {
 			drm_info(&xe->drm, "Initial FB size exceeds half of stolen, discarding\n");
 			return NULL;
 		}
@@ -114,7 +101,7 @@ xe_alloc_initial_plane_obj(struct drm_device *drm,
 {
 	struct xe_device *xe = to_xe_device(drm);
 	struct drm_mode_fb_cmd2 mode_cmd = { 0 };
-	struct drm_framebuffer *fb = &plane_config->fb->base;
+	struct drm_framebuffer *fb = plane_config->fb;
 	struct xe_bo *bo;
 
 	mode_cmd.pixel_format = fb->format->format;
@@ -151,15 +138,19 @@ xe_initial_plane_setup(struct drm_plane_state *_plane_state,
 {
 	struct intel_plane_state *plane_state = to_intel_plane_state(_plane_state);
 	struct i915_vma *vma;
+	struct intel_fb_pin_params pin_params = {
+		.view = &plane_state->view.gtt,
+	};
+	u32 offset;
+	int ret;
 
-	vma = intel_fb_pin_to_ggtt(fb, &plane_state->view.gtt,
-				   0, 0, 0, false, &plane_state->flags);
-	if (IS_ERR(vma))
-		return PTR_ERR(vma);
+	ret = xe_fb_pin_ggtt_pin(intel_fb_bo(fb), &pin_params, &vma, &offset, NULL);
+	if (ret)
+		return ret;
 
 	plane_state->ggtt_vma = vma;
 
-	plane_state->surf = xe_ggtt_node_addr(plane_state->ggtt_vma->node);
+	plane_state->surf = offset;
 
 	plane_config->vma = vma;
 
@@ -171,7 +162,6 @@ static void xe_plane_config_fini(struct intel_initial_plane_config *plane_config
 }
 
 const struct intel_display_initial_plane_interface xe_display_initial_plane_interface = {
-	.vblank_wait = xe_initial_plane_vblank_wait,
 	.alloc_obj = xe_alloc_initial_plane_obj,
 	.setup = xe_initial_plane_setup,
 	.config_fini = xe_plane_config_fini,
diff --git a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h
index 885fcf211e6d0..18d0fde8c98f9 100644
--- a/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h
+++ b/drivers/gpu/drm/xe/instructions/xe_gpu_commands.h
@@ -20,7 +20,6 @@
 
 #define XY_FAST_COLOR_BLT_CMD		(2 << 29 | 0x44 << 22)
 #define   XY_FAST_COLOR_BLT_DEPTH_32	(2 << 19)
-#define   XY_FAST_COLOR_BLT_DW		16
 #define   XY_FAST_COLOR_BLT_MOCS_MASK	GENMASK(27, 22)
 #define   XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK	GENMASK(27, 24)
 #define   XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT 31
@@ -31,14 +30,13 @@
 #define   XY_FAST_COPY_BLT_D1_DST_TILE4	REG_BIT(30)
 #define   XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK	GENMASK(23, 20)
 
-#define MEM_COPY_CMD (2 << 29 | 0x5a << 22 | 0x8)
+#define MEM_COPY_CMD (2 << 29 | 0x5a << 22)
 #define   MEM_COPY_PAGE_COPY_MODE REG_BIT(19)
 #define   MEM_COPY_MATRIX_COPY REG_BIT(17)
 #define   MEM_COPY_SRC_MOCS_INDEX_MASK	GENMASK(31, 28)
 #define   MEM_COPY_DST_MOCS_INDEX_MASK	GENMASK(6, 3)
 
 #define	PVC_MEM_SET_CMD		(2 << 29 | 0x5b << 22)
-#define   PVC_MEM_SET_CMD_LEN_DW	7
 #define   PVC_MEM_SET_MATRIX		REG_BIT(17)
 #define   PVC_MEM_SET_DATA_FIELD	GENMASK(31, 24)
 /* Bspec lists field as [6:0], but index alone is from [6:1] */
diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h
index 1b4a7e9a703df..94033982e6944 100644
--- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h
@@ -165,18 +165,21 @@
 #define	  CTX_CTRL_INHIBIT_SYN_CTX_SWITCH	REG_BIT(3)
 #define	  CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT	REG_BIT(0)
 
-#define RING_MODE(base)				XE_REG((base) + 0x29c)
-#define   GFX_DISABLE_LEGACY_MODE		REG_BIT(3)
+#define GFX_MODE(base)				XE_REG((base) + 0x29c, XE_REG_OPTION_MASKED)
 #define   GFX_MSIX_INTERRUPT_ENABLE		REG_BIT(13)
+#define   GFX_DISABLE_LEGACY_MODE		REG_BIT(3)
 
 #define RING_CSMQDEBUG(base)			XE_REG((base) + 0x2b0)
+#define   CURRENT_ACTIVE_QUEUE_ID_MASK		REG_GENMASK(7, 0)
+
+#define RING_QUEUE_TIMESTAMP(base)		XE_REG((base) + 0x4c0)
+#define RING_QUEUE_TIMESTAMP_UDW(base)		XE_REG((base) + 0x4c0 + 4)
 
 #define RING_TIMESTAMP(base)			XE_REG((base) + 0x358)
 
 #define RING_TIMESTAMP_UDW(base)		XE_REG((base) + 0x358 + 4)
 #define   RING_VALID_MASK			0x00000001
 #define   RING_VALID				0x00000001
-#define   STOP_RING				REG_BIT(8)
 
 #define RING_CTX_TIMESTAMP(base)		XE_REG((base) + 0x3a8)
 #define RING_CTX_TIMESTAMP_UDW(base)		XE_REG((base) + 0x3ac)
diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
index 9c88ca3ce768d..08251c7a1a4b6 100644
--- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h
@@ -152,10 +152,11 @@
 
 #define XEHPG_INSTDONE_GEOM_SVGUNIT		XE_REG_MCR(0x666c)
 
-#define CACHE_MODE_1				XE_REG(0x7004, XE_REG_OPTION_MASKED)
+#define CACHE_MODE_1				XE_REG_MCR(0x7004, XE_REG_OPTION_MASKED)
 #define   MSAA_OPTIMIZATION_REDUC_DISABLE	REG_BIT(11)
 
 #define COMMON_SLICE_CHICKEN1			XE_REG(0x7010, XE_REG_OPTION_MASKED)
+#define XEHP_COMMON_SLICE_CHICKEN1		XE_REG_MCR(0x7010, XE_REG_OPTION_MASKED)
 #define   DISABLE_BOTTOM_CLIP_RECTANGLE_TEST	REG_BIT(14)
 
 #define HIZ_CHICKEN					XE_REG(0x7018, XE_REG_OPTION_MASKED)
@@ -178,6 +179,7 @@
 #define XEHPG_SC_INSTDONE_EXTRA2		XE_REG_MCR(0x7108)
 
 #define COMMON_SLICE_CHICKEN4			XE_REG(0x7300, XE_REG_OPTION_MASKED)
+#define XEHP_COMMON_SLICE_CHICKEN4		XE_REG_MCR(0x7300, XE_REG_OPTION_MASKED)
 #define   SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE	REG_BIT(12)
 #define   DISABLE_TDC_LOAD_BALANCING_CALC	REG_BIT(6)
 #define   HW_FILTERING				REG_BIT(5)
@@ -528,6 +530,7 @@
 
 #define ROW_CHICKEN3				XE_REG_MCR(0xe49c, XE_REG_OPTION_MASKED)
 #define   XE2_EUPEND_CHK_FLUSH_DIS		REG_BIT(14)
+#define   DIS_EU_GRF_POISON_TO_LSC		REG_BIT(13)
 #define   DIS_FIX_EOT1_FLUSH			REG_BIT(9)
 
 #define TDL_TSL_CHICKEN				XE_REG_MCR(0xe4c4, XE_REG_OPTION_MASKED)
@@ -535,6 +538,9 @@
 #define   SLM_WMTP_RESTORE			REG_BIT(11)
 #define   RES_CHK_SPR_DIS			REG_BIT(6)
 
+#define TDL_TSL_CHICKEN2			XE_REG_MCR(0xe4cc, XE_REG_OPTION_MASKED)
+#define   TILEY_LOCALID				REG_BIT(2)
+
 #define ROW_CHICKEN				XE_REG_MCR(0xe4f0, XE_REG_OPTION_MASKED)
 #define   UGM_BACKUP_MODE			REG_BIT(13)
 #define   MDQ_ARBITRATION_MODE			REG_BIT(12)
@@ -560,6 +566,7 @@
 #define   DIS_ATOMIC_CHAINING_TYPED_WRITES	REG_BIT(3)
 
 #define TDL_CHICKEN				XE_REG_MCR(0xe5f4, XE_REG_OPTION_MASKED)
+#define   BIT_APQ_OPT_DIS			REG_BIT(14)
 #define   QID_WAIT_FOR_THREAD_NOT_RUN_DISABLE	REG_BIT(12)
 #define   EUSTALL_PERF_SAMPLING_DISABLE		REG_BIT(5)
 
diff --git a/drivers/gpu/drm/xe/regs/xe_irq_regs.h b/drivers/gpu/drm/xe/regs/xe_irq_regs.h
index 9d74f454d3ffd..1d6b976c4de06 100644
--- a/drivers/gpu/drm/xe/regs/xe_irq_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_irq_regs.h
@@ -22,6 +22,7 @@
 #define   DISPLAY_IRQ				REG_BIT(16)
 #define   SOC_H2DMEMINT_IRQ			REG_BIT(13)
 #define   I2C_IRQ				REG_BIT(12)
+#define   SYSCTRL_IRQ				REG_BIT(11)
 #define   GT_DW_IRQ(x)				REG_BIT(x)
 
 /*
diff --git a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h
index b5eff383902c5..4ab86fc369fdf 100644
--- a/drivers/gpu/drm/xe/regs/xe_lrc_layout.h
+++ b/drivers/gpu/drm/xe/regs/xe_lrc_layout.h
@@ -34,6 +34,9 @@
 #define CTX_CS_INT_VEC_REG		0x5a
 #define CTX_CS_INT_VEC_DATA		(CTX_CS_INT_VEC_REG + 1)
 
+#define CTX_QUEUE_TIMESTAMP		(0xd0 + 1)
+#define CTX_QUEUE_TIMESTAMP_UDW		(0xd2 + 1)
+
 #define INDIRECT_CTX_RING_HEAD		(0x02 + 1)
 #define INDIRECT_CTX_RING_TAIL		(0x04 + 1)
 #define INDIRECT_CTX_RING_START		(0x06 + 1)
diff --git a/drivers/gpu/drm/xe/regs/xe_oa_regs.h b/drivers/gpu/drm/xe/regs/xe_oa_regs.h
index 04a729e610aa9..aa66af7e99fee 100644
--- a/drivers/gpu/drm/xe/regs/xe_oa_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_oa_regs.h
@@ -6,6 +6,9 @@
 #ifndef __XE_OA_REGS__
 #define __XE_OA_REGS__
 
+#define SYS_MEM_LAT_MEASURE		XE_REG(0x145194)
+#define   SYS_MEM_LAT_MEASURE_EN	REG_BIT(31)
+
 #define RPM_CONFIG1			XE_REG(0xd04)
 #define   GT_NOA_ENABLE			REG_BIT(9)
 
diff --git a/drivers/gpu/drm/xe/regs/xe_pcode_regs.h b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h
index 4b3c46eb858f9..c63b409d7a821 100644
--- a/drivers/gpu/drm/xe/regs/xe_pcode_regs.h
+++ b/drivers/gpu/drm/xe/regs/xe_pcode_regs.h
@@ -27,4 +27,7 @@
 #define   TEMP_SIGN_MASK			REG_BIT(31)
 #define BMG_PACKAGE_TEMPERATURE			XE_REG(0x138434)
 
+#define CRI_PACKAGE_ENERGY_STATUS		XE_REG(0x138120)
+#define CRI_PLATFORM_ENERGY_STATUS		XE_REG(0x138458)
+
 #endif /* _XE_PCODE_REGS_H_ */
diff --git a/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h b/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h
new file mode 100644
index 0000000000000..59f3f3ec59a6e
--- /dev/null
+++ b/drivers/gpu/drm/xe/regs/xe_sysctrl_regs.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_SYSCTRL_REGS_H_
+#define _XE_SYSCTRL_REGS_H_
+
+#include "xe_regs.h"
+
+#define SYSCTRL_BASE_OFFSET			0xdb000
+#define SYSCTRL_BASE				(SOC_BASE + SYSCTRL_BASE_OFFSET)
+#define SYSCTRL_MAILBOX_INDEX			0x03
+#define SYSCTRL_BAR_LENGTH			0x1000
+
+#define SYSCTRL_MB_CTRL				XE_REG(0x10)
+#define   SYSCTRL_MB_CTRL_RUN_BUSY		REG_BIT(31)
+#define   SYSCTRL_MB_CTRL_IRQ			REG_BIT(30)
+#define   SYSCTRL_MB_CTRL_RUN_BUSY_OUT		REG_BIT(29)
+#define   SYSCTRL_MB_CTRL_PARAM3_MASK		REG_GENMASK(28, 24)
+#define   SYSCTRL_MB_CTRL_PARAM2_MASK		REG_GENMASK(23, 16)
+#define   SYSCTRL_MB_CTRL_PARAM1_MASK		REG_GENMASK(15, 8)
+#define   SYSCTRL_MB_CTRL_COMMAND_MASK		REG_GENMASK(7, 0)
+#define   SYSCTRL_MB_CTRL_CMD			REG_FIELD_PREP(SYSCTRL_MB_CTRL_COMMAND_MASK, 5)
+
+#define SYSCTRL_MB_DATA0			XE_REG(0x14)
+#define SYSCTRL_MB_DATA1			XE_REG(0x18)
+#define SYSCTRL_MB_DATA2			XE_REG(0x1c)
+#define SYSCTRL_MB_DATA3			XE_REG(0x20)
+
+#define SYSCTRL_FRAME_PHASE			REG_BIT(24)
+#define SYSCTRL_FRAME_CURRENT_MASK		REG_GENMASK(21, 16)
+#define SYSCTRL_FRAME_TOTAL_MASK		REG_GENMASK(13, 8)
+#define SYSCTRL_FRAME_COMMAND_MASK		REG_GENMASK(7, 0)
+
+#endif
diff --git a/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c
index efa8963ec2488..e6eaa94d4d30f 100644
--- a/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c
+++ b/drivers/gpu/drm/xe/tests/xe_gt_sriov_pf_config_kunit.c
@@ -13,11 +13,28 @@
 #define TEST_MAX_VFS	63
 #define TEST_VRAM	0x7a800000ull	/* random size that works on 32-bit */
 
+static bool xe_device_is_admin_only_stub_enable(const struct xe_device *xe)
+{
+	return true;
+}
+
+static bool xe_device_is_admin_only_stub_disable(const struct xe_device *xe)
+{
+	return false;
+}
+
 static void pf_set_admin_mode(struct xe_device *xe, bool enable)
 {
-	/* should match logic of xe_sriov_pf_admin_only() */
-	xe->sriov.pf.admin_only = enable;
+	typeof(xe_device_is_admin_only) *stub = enable ?
+		xe_device_is_admin_only_stub_enable :
+		xe_device_is_admin_only_stub_disable;
+
+	kunit_activate_static_stub(kunit_get_current_test(),
+				   xe_device_is_admin_only,
+				   *stub);
+
 	KUNIT_EXPECT_EQ(kunit_get_current_test(), enable, xe_sriov_pf_admin_only(xe));
+	KUNIT_EXPECT_EQ(kunit_get_current_test(), enable, xe_device_is_admin_only(xe));
 }
 
 static void pf_set_usable_vram(struct xe_device *xe, u64 usable)
diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c
index 34e2f0f4631f5..3c1be809be82d 100644
--- a/drivers/gpu/drm/xe/tests/xe_migrate.c
+++ b/drivers/gpu/drm/xe/tests/xe_migrate.c
@@ -9,6 +9,7 @@
 #include "tests/xe_kunit_helpers.h"
 #include "tests/xe_pci_test.h"
 
+#include "xe_pat.h"
 #include "xe_pci.h"
 #include "xe_pm.h"
 
@@ -246,7 +247,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test,
 	/* First part of the test, are we updating our pagetable bo with a new entry? */
 	xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64,
 		  0xdeaddeadbeefbeef);
-	expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe->pat.idx[XE_CACHE_WB], 0);
+	expected = m->q->vm->pt_ops->pte_encode_bo(pt, 0, xe_cache_pat_idx(xe, XE_CACHE_WB), 0);
 	if (m->q->vm->flags & XE_VM_FLAG_64K)
 		expected |= XE_PTE_PS64;
 	if (xe_bo_is_vram(pt))
@@ -420,7 +421,7 @@ static struct dma_fence *blt_copy(struct xe_tile *tile,
 					      avail_pts, avail_pts);
 
 		/* Add copy commands size here */
-		batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +
+		batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) +
 			((xe_device_has_flat_ccs(xe) && copy_only_ccs) ? EMIT_COPY_CCS_DW : 0);
 
 		bb = xe_bb_new(gt, batch_size, xe->info.has_usm);
diff --git a/drivers/gpu/drm/xe/tests/xe_pci.c b/drivers/gpu/drm/xe/tests/xe_pci.c
index f3179b31f13e3..860409c579f8c 100644
--- a/drivers/gpu/drm/xe/tests/xe_pci.c
+++ b/drivers/gpu/drm/xe/tests/xe_pci.c
@@ -131,12 +131,12 @@ static const char *subplatform_prefix(enum xe_subplatform s)
 	return s == XE_SUBPLATFORM_NONE ? "" : " ";
 }
 
-static const char *step_prefix(enum xe_step step)
+static const char *step_prefix(enum intel_step step)
 {
 	return step == STEP_NONE ? "" : " ";
 }
 
-static const char *step_name(enum xe_step step)
+static const char *step_name(enum intel_step step)
 {
 	return step == STEP_NONE ? "" : xe_step_name(step);
 }
diff --git a/drivers/gpu/drm/xe/tests/xe_rtp_test.c b/drivers/gpu/drm/xe/tests/xe_rtp_test.c
index e5a0f985a700b..5d78f2283df93 100644
--- a/drivers/gpu/drm/xe/tests/xe_rtp_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_rtp_test.c
@@ -9,24 +9,30 @@
 #include <drm/drm_drv.h>
 #include <drm/drm_kunit_helpers.h>
 
+#include <kunit/static_stub.h>
 #include <kunit/test.h>
 
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_reg_defs.h"
 #include "xe_device.h"
 #include "xe_device_types.h"
+#include "xe_gt_mcr.h"
 #include "xe_kunit_helpers.h"
 #include "xe_pci_test.h"
 #include "xe_reg_sr.h"
 #include "xe_rtp.h"
 
-#define REGULAR_REG1	XE_REG(1)
-#define REGULAR_REG2	XE_REG(2)
-#define REGULAR_REG3	XE_REG(3)
-#define MCR_REG1	XE_REG_MCR(1)
-#define MCR_REG2	XE_REG_MCR(2)
-#define MCR_REG3	XE_REG_MCR(3)
-#define MASKED_REG1	XE_REG(1, XE_REG_OPTION_MASKED)
+#define REGULAR_REG1		XE_REG(1)
+#define REGULAR_REG2		XE_REG(2)
+#define REGULAR_REG3		XE_REG(3)
+#define REGULAR_REG4		XE_REG(4)
+#define BAD_REGULAR_REG5	XE_REG(5)
+#define MCR_REG1		XE_REG_MCR(1)
+#define MCR_REG2		XE_REG_MCR(2)
+#define MCR_REG3		XE_REG_MCR(3)
+#define BAD_MCR_REG4		XE_REG_MCR(4)
+#define MCR_REG5		XE_REG_MCR(5)
+#define MASKED_REG1		XE_REG(1, XE_REG_OPTION_MASKED)
 
 #undef XE_REG_MCR
 #define XE_REG_MCR(...)     XE_REG(__VA_ARGS__, .mcr = 1)
@@ -48,6 +54,23 @@ struct rtp_test_case {
 	const struct xe_rtp_entry *entries;
 };
 
+static bool fake_xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg)
+{
+	/*
+	 * All supported platforms in this imaginary setup will always have REG4
+	 * as a non-MCR register and REG5 as MCR, meaning that BAD_MCR_REG4 and
+	 * BAD_REGULAR_REG5 represent programming errors to be captured by our
+	 * tests.
+	 */
+	if (reg.raw == BAD_REGULAR_REG5.raw)
+		return true;
+
+	if (reg.raw == BAD_MCR_REG4.raw)
+		return false;
+
+	return reg.mcr;
+}
+
 static bool match_yes(const struct xe_device *xe, const struct xe_gt *gt,
 		      const struct xe_hw_engine *hwe)
 {
@@ -304,6 +327,38 @@ static const struct rtp_to_sr_test_case rtp_to_sr_cases[] = {
 			{}
 		},
 	},
+	{
+		.name = "bad-mcr-reg-forced-to-regular",
+		.expected_reg = REGULAR_REG4,
+		.expected_set_bits = REG_BIT(0),
+		.expected_clr_bits = REG_BIT(0),
+		.expected_active = BIT(0),
+		.expected_count_sr_entries = 1,
+		.expected_sr_errors = 1,
+		.entries = (const struct xe_rtp_entry_sr[]) {
+			{ XE_RTP_NAME("bad-mcr-regular-reg"),
+			  XE_RTP_RULES(FUNC(match_yes)),
+			  XE_RTP_ACTIONS(SET(BAD_MCR_REG4, REG_BIT(0)))
+			},
+			{}
+		},
+	},
+	{
+		.name = "bad-regular-reg-forced-to-mcr",
+		.expected_reg = MCR_REG5,
+		.expected_set_bits = REG_BIT(0),
+		.expected_clr_bits = REG_BIT(0),
+		.expected_active = BIT(0),
+		.expected_count_sr_entries = 1,
+		.expected_sr_errors = 1,
+		.entries = (const struct xe_rtp_entry_sr[]) {
+			{ XE_RTP_NAME("bad-regular-reg"),
+			  XE_RTP_RULES(FUNC(match_yes)),
+			  XE_RTP_ACTIONS(SET(BAD_REGULAR_REG5, REG_BIT(0)))
+			},
+			{}
+		},
+	},
 };
 
 static void xe_rtp_process_to_sr_tests(struct kunit *test)
@@ -523,6 +578,8 @@ static int xe_rtp_test_init(struct kunit *test)
 	xe->drm.dev = dev;
 	test->priv = xe;
 
+	kunit_activate_static_stub(test, xe_gt_mcr_check_reg, fake_xe_gt_mcr_check_reg);
+
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/xe/tests/xe_wa_test.c b/drivers/gpu/drm/xe/tests/xe_wa_test.c
index 49d191043dfaa..ff0e2502b39f9 100644
--- a/drivers/gpu/drm/xe/tests/xe_wa_test.c
+++ b/drivers/gpu/drm/xe/tests/xe_wa_test.c
@@ -9,6 +9,8 @@
 #include <kunit/test.h>
 
 #include "xe_device.h"
+#include "xe_gt.h"
+#include "xe_gt_mcr.h"
 #include "xe_kunit_helpers.h"
 #include "xe_pci_test.h"
 #include "xe_reg_sr.h"
@@ -19,8 +21,10 @@ static int xe_wa_test_init(struct kunit *test)
 {
 	const struct xe_pci_fake_data *param = test->param_value;
 	struct xe_pci_fake_data data = *param;
-	struct xe_device *xe;
 	struct device *dev;
+	struct xe_device *xe;
+	struct xe_gt *gt;
+	int id;
 	int ret;
 
 	dev = drm_kunit_helper_alloc_device(test);
@@ -33,6 +37,12 @@ static int xe_wa_test_init(struct kunit *test)
 	ret = xe_pci_fake_device_init(xe);
 	KUNIT_ASSERT_EQ(test, ret, 0);
 
+	/* Needed for sanitize_mcr(). */
+	for_each_gt(gt, xe, id) {
+		xe_gt_mcr_init_early(gt);
+		xe_gt_mmio_init(gt);
+	}
+
 	if (!param->graphics_verx100)
 		xe->info.step = param->step;
 
@@ -55,7 +65,7 @@ static void xe_wa_gt(struct kunit *test)
 		xe_wa_process_gt(gt);
 		xe_tuning_process_gt(gt);
 
-		KUNIT_ASSERT_EQ(test, gt->reg_sr.errors, 0);
+		KUNIT_EXPECT_EQ(test, gt->reg_sr.errors, 0);
 	}
 }
 
diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
index 6b518858538f5..4c80bac676229 100644
--- a/drivers/gpu/drm/xe/xe_bo.c
+++ b/drivers/gpu/drm/xe/xe_bo.c
@@ -173,19 +173,6 @@ mem_type_to_migrate(struct xe_device *xe, u32 mem_type)
 	return tile->migrate;
 }
 
-static struct xe_vram_region *res_to_mem_region(struct ttm_resource *res)
-{
-	struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);
-	struct ttm_resource_manager *mgr;
-	struct xe_ttm_vram_mgr *vram_mgr;
-
-	xe_assert(xe, resource_is_vram(res));
-	mgr = ttm_manager_type(&xe->ttm, res->mem_type);
-	vram_mgr = to_xe_ttm_vram_mgr(mgr);
-
-	return container_of(vram_mgr, struct xe_vram_region, ttm);
-}
-
 static void try_add_system(struct xe_device *xe, struct xe_bo *bo,
 			   u32 bo_flags, u32 *c)
 {
@@ -599,11 +586,17 @@ static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt)
 	kfree(tt);
 }
 
-static bool xe_ttm_resource_visible(struct ttm_resource *mem)
+static bool xe_ttm_resource_visible(struct xe_device *xe, struct ttm_resource *mem)
 {
-	struct xe_ttm_vram_mgr_resource *vres =
-		to_xe_ttm_vram_mgr_resource(mem);
+	struct xe_ttm_vram_mgr_resource *vres;
+
+	if (mem->mem_type == XE_PL_STOLEN) {
+		struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr;
+
+		return mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe);
+	}
 
+	vres = to_xe_ttm_vram_mgr_resource(mem);
 	return vres->used_visible_size == mem->size;
 }
 
@@ -621,7 +614,7 @@ bool xe_bo_is_visible_vram(struct xe_bo *bo)
 	if (drm_WARN_ON(bo->ttm.base.dev, !xe_bo_is_vram(bo)))
 		return false;
 
-	return xe_ttm_resource_visible(bo->ttm.resource);
+	return xe_ttm_resource_visible(xe_bo_device(bo), bo->ttm.resource);
 }
 
 static int xe_ttm_io_mem_reserve(struct ttm_device *bdev,
@@ -635,9 +628,9 @@ static int xe_ttm_io_mem_reserve(struct ttm_device *bdev,
 		return 0;
 	case XE_PL_VRAM0:
 	case XE_PL_VRAM1: {
-		struct xe_vram_region *vram = res_to_mem_region(mem);
+		struct xe_vram_region *vram = xe_map_resource_to_region(mem);
 
-		if (!xe_ttm_resource_visible(mem))
+		if (!xe_ttm_resource_visible(xe, mem))
 			return -EINVAL;
 
 		mem->bus.offset = mem->start << PAGE_SHIFT;
@@ -1642,7 +1635,7 @@ static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *ttm_bo,
 	if (ttm_bo->resource->mem_type == XE_PL_STOLEN)
 		return xe_ttm_stolen_io_offset(bo, page_offset << PAGE_SHIFT) >> PAGE_SHIFT;
 
-	vram = res_to_mem_region(ttm_bo->resource);
+	vram = xe_map_resource_to_region(ttm_bo->resource);
 	xe_res_first(ttm_bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor);
 	return (vram->io_start + cursor.start) >> PAGE_SHIFT;
 }
@@ -1782,7 +1775,7 @@ static int xe_ttm_access_memory(struct ttm_buffer_object *ttm_bo,
 		goto out;
 	}
 
-	vram = res_to_mem_region(ttm_bo->resource);
+	vram = xe_map_resource_to_region(ttm_bo->resource);
 	xe_res_first(ttm_bo->resource, offset & PAGE_MASK,
 		     xe_bo_size(bo) - (offset & PAGE_MASK), &cursor);
 
@@ -2927,7 +2920,7 @@ uint64_t vram_region_gpu_offset(struct ttm_resource *res)
 	case XE_PL_SYSTEM:
 		return 0;
 	default:
-		return res_to_mem_region(res)->dpa_base;
+		return xe_map_resource_to_region(res)->dpa_base;
 	}
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h
index 077e35b4cdce7..fcc63ae3f4551 100644
--- a/drivers/gpu/drm/xe/xe_bo_types.h
+++ b/drivers/gpu/drm/xe/xe_bo_types.h
@@ -67,7 +67,7 @@ struct xe_bo {
 	/** @attr: User controlled attributes for bo */
 	struct {
 		/**
-		 * @atomic_access: type of atomic access bo needs
+		 * @attr.atomic_access: type of atomic access bo needs
 		 * protected by bo dma-resv lock
 		 */
 		u32 atomic_access;
diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c
index ad2d8f179eb68..22b471303984a 100644
--- a/drivers/gpu/drm/xe/xe_debugfs.c
+++ b/drivers/gpu/drm/xe/xe_debugfs.c
@@ -15,9 +15,11 @@
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_force_wake.h"
+#include "xe_gt.h"
 #include "xe_gt_debugfs.h"
 #include "xe_gt_printk.h"
 #include "xe_guc_ads.h"
+#include "xe_hw_engine.h"
 #include "xe_mmio.h"
 #include "xe_pm.h"
 #include "xe_psmi.h"
@@ -61,6 +63,37 @@ static struct xe_device *node_to_xe(struct drm_info_node *node)
 	return to_xe_device(node->minor->dev);
 }
 
+static void print_engine_class_mask(struct drm_printer *p, u16 mask)
+{
+	if (!mask) {
+		drm_printf(p, " none\n");
+		return;
+	}
+
+	for (enum xe_engine_class ec = 0; ec < XE_ENGINE_CLASS_MAX; ec++) {
+		if (mask & BIT(ec))
+			drm_printf(p, " %s", xe_hw_engine_class_to_str(ec));
+	}
+	drm_printf(p, "\n");
+}
+
+static void print_engine_mask(struct drm_printer *p, struct xe_gt *gt, u64 mask)
+{
+	struct xe_hw_engine *hwe;
+	enum xe_hw_engine_id id;
+
+	if (!mask) {
+		drm_printf(p, " none\n");
+		return;
+	}
+
+	for_each_hw_engine(hwe, gt, id) {
+		if (mask & BIT_ULL(id))
+			drm_printf(p, " %s", hwe->name);
+	}
+	drm_printf(p, "\n");
+}
+
 static int info(struct seq_file *m, void *data)
 {
 	struct xe_device *xe = node_to_xe(m->private);
@@ -88,13 +121,15 @@ static int info(struct seq_file *m, void *data)
 	drm_printf(&p, "has_flat_ccs %s\n", str_yes_no(xe->info.has_flat_ccs));
 	drm_printf(&p, "has_usm %s\n", str_yes_no(xe->info.has_usm));
 	drm_printf(&p, "skip_guc_pc %s\n", str_yes_no(xe->info.skip_guc_pc));
+	drm_printf(&p, "multi_lrc_engine_classes");
+	print_engine_class_mask(&p, xe->info.multi_lrc_mask);
 	for_each_gt(gt, xe, id) {
 		drm_printf(&p, "gt%d force wake %d\n", id,
 			   xe_force_wake_ref(gt_to_fw(gt), XE_FW_GT));
-		drm_printf(&p, "gt%d engine_mask 0x%llx\n", id,
-			   gt->info.engine_mask);
-		drm_printf(&p, "gt%d multi_queue_engine_class_mask 0x%x\n", id,
-			   gt->info.multi_queue_engine_class_mask);
+		drm_printf(&p, "gt%d engines", id);
+		print_engine_mask(&p, gt, gt->info.engine_mask);
+		drm_printf(&p, "gt%d multi_queue_engine_classes", id);
+		print_engine_class_mask(&p, gt->info.multi_queue_engine_class_mask);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.c b/drivers/gpu/drm/xe/xe_dep_scheduler.c
index 51d99fee9aa50..004aac8b89e6a 100644
--- a/drivers/gpu/drm/xe/xe_dep_scheduler.c
+++ b/drivers/gpu/drm/xe/xe_dep_scheduler.c
@@ -78,7 +78,6 @@ xe_dep_scheduler_create(struct xe_device *xe,
 	const struct drm_sched_init_args args = {
 		.ops = &sched_ops,
 		.submit_wq = submit_wq,
-		.num_rqs = 1,
 		.credit_limit = job_limit,
 		.timeout = MAX_SCHEDULE_TIMEOUT,
 		.name = name,
diff --git a/drivers/gpu/drm/xe/xe_dep_scheduler.h b/drivers/gpu/drm/xe/xe_dep_scheduler.h
index 853961eec64bf..f314fb5d80f5e 100644
--- a/drivers/gpu/drm/xe/xe_dep_scheduler.h
+++ b/drivers/gpu/drm/xe/xe_dep_scheduler.h
@@ -3,6 +3,9 @@
  * Copyright © 2025 Intel Corporation
  */
 
+#ifndef _XE_DEP_SCHEDULER_H_
+#define _XE_DEP_SCHEDULER_H_
+
 #include <linux/types.h>
 
 struct drm_sched_entity;
@@ -19,3 +22,5 @@ void xe_dep_scheduler_fini(struct xe_dep_scheduler *dep_scheduler);
 
 struct drm_sched_entity *
 xe_dep_scheduler_entity(struct xe_dep_scheduler *dep_scheduler);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c
index 558a1a9841a09..5f2b90b18f97b 100644
--- a/drivers/gpu/drm/xe/xe_devcoredump.c
+++ b/drivers/gpu/drm/xe/xe_devcoredump.c
@@ -8,7 +8,7 @@
 
 #include <linux/ascii85.h>
 #include <linux/devcoredump.h>
-#include <generated/utsrelease.h>
+#include <linux/utsname.h>
 
 #include <drm/drm_managed.h>
 
@@ -101,7 +101,7 @@ static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count,
 
 	drm_puts(&p, "**** Xe Device Coredump ****\n");
 	drm_printf(&p, "Reason: %s\n", ss->reason);
-	drm_puts(&p, "kernel: " UTS_RELEASE "\n");
+	drm_printf(&p, "kernel: %s\n", init_utsname()->release);
 	drm_puts(&p, "module: " KBUILD_MODNAME "\n");
 
 	ts = ktime_to_timespec64(ss->snapshot_time);
@@ -149,7 +149,8 @@ static void xe_devcoredump_snapshot_free(struct xe_devcoredump_snapshot *ss)
 	xe_guc_ct_snapshot_free(ss->guc.ct);
 	ss->guc.ct = NULL;
 
-	xe_guc_capture_put_matched_nodes(&ss->gt->uc.guc);
+	if (!IS_ERR_OR_NULL(ss->gt))
+		xe_guc_capture_put_matched_nodes(&ss->gt->uc.guc);
 	ss->matched_node = NULL;
 
 	xe_guc_exec_queue_snapshot_free(ss->ge);
@@ -254,7 +255,8 @@ static void xe_devcoredump_free(void *data)
 	if (!data || !coredump_to_xe(coredump))
 		return;
 
-	cancel_work_sync(&coredump->snapshot.work);
+	if (coredump->captured)
+		cancel_work_sync(&coredump->snapshot.work);
 
 	mutex_lock(&coredump->lock);
 
diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c
index ffea4a453c01c..576095cf09529 100644
--- a/drivers/gpu/drm/xe/xe_device.c
+++ b/drivers/gpu/drm/xe/xe_device.c
@@ -10,13 +10,13 @@
 #include <linux/fault-inject.h>
 #include <linux/units.h>
 
-#include <drm/drm_atomic_helper.h>
 #include <drm/drm_client.h>
 #include <drm/drm_gem_ttm_helper.h>
 #include <drm/drm_ioctl.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_pagemap_util.h>
 #include <drm/drm_print.h>
+#include <kunit/static_stub.h>
 #include <uapi/drm/xe_drm.h>
 
 #include "display/xe_display.h"
@@ -25,6 +25,7 @@
 #include "regs/xe_regs.h"
 #include "xe_bo.h"
 #include "xe_bo_evict.h"
+#include "xe_configfs.h"
 #include "xe_debugfs.h"
 #include "xe_defaults.h"
 #include "xe_devcoredump.h"
@@ -65,6 +66,7 @@
 #include "xe_survivability_mode.h"
 #include "xe_sriov.h"
 #include "xe_svm.h"
+#include "xe_sysctrl.h"
 #include "xe_tile.h"
 #include "xe_ttm_stolen_mgr.h"
 #include "xe_ttm_sys_mgr.h"
@@ -389,8 +391,9 @@ bool xe_is_xe_file(const struct file *file)
 	return file->f_op == &xe_driver_fops;
 }
 
-static struct drm_driver driver = {
+static const struct drm_driver regular_driver = {
 	.driver_features =
+	    XE_DISPLAY_DRIVER_FEATURES |
 	    DRIVER_GEM |
 	    DRIVER_RENDER | DRIVER_SYNCOBJ |
 	    DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
@@ -412,8 +415,45 @@ static struct drm_driver driver = {
 	.major = DRIVER_MAJOR,
 	.minor = DRIVER_MINOR,
 	.patchlevel = DRIVER_PATCHLEVEL,
+	XE_DISPLAY_DRIVER_OPS,
 };
 
+#ifdef CONFIG_PCI_IOV
+static const struct drm_ioctl_desc xe_ioctls_admin_only[] = {
+	DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
+	DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
+};
+
+static const struct drm_driver admin_only_driver = {
+	.driver_features =
+	    XE_DISPLAY_DRIVER_FEATURES |
+	    DRIVER_GEM | DRIVER_RENDER | DRIVER_GEM_GPUVA,
+	.open = xe_file_open,
+	.postclose = xe_file_close,
+	.ioctls = xe_ioctls_admin_only,
+	.num_ioctls = ARRAY_SIZE(xe_ioctls_admin_only),
+	.fops = &xe_driver_fops,
+	.name = DRIVER_NAME,
+	.desc = DRIVER_DESC,
+	.major = DRIVER_MAJOR,
+	.minor = DRIVER_MINOR,
+	.patchlevel = DRIVER_PATCHLEVEL,
+	XE_DISPLAY_DRIVER_OPS,
+};
+
+/**
+ * xe_device_is_admin_only() - Check whether device is admin only or not.
+ * @xe: the &xe_device to check
+ *
+ * Return: true if the device is admin only, false otherwise.
+ */
+bool xe_device_is_admin_only(const struct xe_device *xe)
+{
+	KUNIT_STATIC_STUB_REDIRECT(xe_device_is_admin_only, xe);
+	return xe->drm.driver == &admin_only_driver;
+}
+#endif
+
 static void xe_device_destroy(struct drm_device *dev, void *dummy)
 {
 	struct xe_device *xe = to_xe_device(dev);
@@ -438,16 +478,24 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy)
 struct xe_device *xe_device_create(struct pci_dev *pdev,
 				   const struct pci_device_id *ent)
 {
+	const struct drm_driver *driver = &regular_driver;
 	struct xe_device *xe;
 	int err;
 
-	xe_display_driver_set_hooks(&driver);
+#ifdef CONFIG_PCI_IOV
+	/*
+	 * Since XE device is not initialized yet, read from configfs
+	 * directly to decide whether we are in admin-only PF mode or not.
+	 */
+	if (xe_configfs_admin_only_pf(pdev))
+		driver = &admin_only_driver;
+#endif
 
-	err = aperture_remove_conflicting_pci_devices(pdev, driver.name);
+	err = aperture_remove_conflicting_pci_devices(pdev, driver->name);
 	if (err)
 		return ERR_PTR(err);
 
-	xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm);
+	xe = devm_drm_dev_alloc(&pdev->dev, driver, struct xe_device, drm);
 	if (IS_ERR(xe))
 		return xe;
 
@@ -707,6 +755,11 @@ int xe_device_probe_early(struct xe_device *xe)
 
 	xe_sriov_probe_early(xe);
 
+	if (xe_device_is_admin_only(xe) && !IS_SRIOV_PF(xe)) {
+		xe_err(xe, "Can't run Admin-only mode without SR-IOV PF mode!\n");
+		return -ENODEV;
+	}
+
 	if (IS_SRIOV_VF(xe))
 		vf_update_device_info(xe);
 
@@ -992,6 +1045,10 @@ int xe_device_probe(struct xe_device *xe)
 	if (err)
 		goto err_unregister_display;
 
+	err = xe_sysctrl_init(xe);
+	if (err)
+		goto err_unregister_display;
+
 	err = xe_device_sysfs_init(xe);
 	if (err)
 		goto err_unregister_display;
diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h
index e4b9de8d8e95c..355d69dc8f545 100644
--- a/drivers/gpu/drm/xe/xe_device.h
+++ b/drivers/gpu/drm/xe/xe_device.h
@@ -146,37 +146,37 @@ static inline struct xe_force_wake *gt_to_fw(struct xe_gt *gt)
 
 void xe_device_assert_mem_access(struct xe_device *xe);
 
-static inline bool xe_device_has_flat_ccs(struct xe_device *xe)
+static inline bool xe_device_has_flat_ccs(const struct xe_device *xe)
 {
 	return xe->info.has_flat_ccs;
 }
 
-static inline bool xe_device_has_sriov(struct xe_device *xe)
+static inline bool xe_device_has_sriov(const struct xe_device *xe)
 {
 	return xe->info.has_sriov;
 }
 
-static inline bool xe_device_has_msix(struct xe_device *xe)
+static inline bool xe_device_has_msix(const struct xe_device *xe)
 {
 	return xe->irq.msix.nvec > 0;
 }
 
-static inline bool xe_device_has_memirq(struct xe_device *xe)
+static inline bool xe_device_has_memirq(const struct xe_device *xe)
 {
 	return GRAPHICS_VERx100(xe) >= 1250;
 }
 
-static inline bool xe_device_uses_memirq(struct xe_device *xe)
+static inline bool xe_device_uses_memirq(const struct xe_device *xe)
 {
 	return xe_device_has_memirq(xe) && (IS_SRIOV_VF(xe) || xe_device_has_msix(xe));
 }
 
-static inline bool xe_device_has_lmtt(struct xe_device *xe)
+static inline bool xe_device_has_lmtt(const struct xe_device *xe)
 {
 	return IS_DGFX(xe);
 }
 
-static inline bool xe_device_has_mert(struct xe_device *xe)
+static inline bool xe_device_has_mert(const struct xe_device *xe)
 {
 	return xe->info.has_mert;
 }
@@ -211,6 +211,15 @@ bool xe_is_xe_file(const struct file *file);
 
 struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid);
 
+#ifdef CONFIG_PCI_IOV
+bool xe_device_is_admin_only(const struct xe_device *xe);
+#else
+static inline bool xe_device_is_admin_only(const struct xe_device *xe)
+{
+	return false;
+}
+#endif
+
 /*
  * Occasionally it is seen that the G2H worker starts running after a delay of more than
  * a second even after being queued and activated by the Linux workqueue subsystem. This
diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h
index 615218d775b14..32dd2ffbc7963 100644
--- a/drivers/gpu/drm/xe/xe_device_types.h
+++ b/drivers/gpu/drm/xe/xe_device_types.h
@@ -27,6 +27,7 @@
 #include "xe_sriov_vf_ccs_types.h"
 #include "xe_step_types.h"
 #include "xe_survivability_mode_types.h"
+#include "xe_sysctrl_types.h"
 #include "xe_tile_types.h"
 #include "xe_validation.h"
 
@@ -41,6 +42,7 @@ struct xe_ggtt;
 struct xe_i2c;
 struct xe_pat_ops;
 struct xe_pxp;
+struct xe_ttm_stolen_mgr;
 struct xe_vram_region;
 
 /**
@@ -196,6 +198,8 @@ struct xe_device {
 		u8 has_soc_remapper_telem:1;
 		/** @info.has_sriov: Supports SR-IOV */
 		u8 has_sriov:1;
+		/** @info.has_sysctrl: Supports System Controller */
+		u8 has_sysctrl:1;
 		/** @info.has_usm: Device has unified shared memory support */
 		u8 has_usm:1;
 		/** @info.has_64bit_timestamp: Device supports 64-bit timestamps */
@@ -215,8 +219,6 @@ struct xe_device {
 		u8 probe_display:1;
 		/** @info.skip_guc_pc: Skip GuC based PM feature init */
 		u8 skip_guc_pc:1;
-		/** @info.skip_mtcfg: skip Multi-Tile configuration from MTCFG register */
-		u8 skip_mtcfg:1;
 		/** @info.skip_pcode: skip access to PCODE uC */
 		u8 skip_pcode:1;
 		/** @info.needs_shared_vf_gt_wq: needs shared GT WQ on VF */
@@ -273,8 +275,10 @@ struct xe_device {
 		struct xe_vram_region *vram;
 		/** @mem.sys_mgr: system TTM manager */
 		struct ttm_resource_manager sys_mgr;
-		/** @mem.sys_mgr: system memory shrinker. */
+		/** @mem.shrinker: system memory shrinker. */
 		struct xe_shrinker *shrinker;
+		/** @mem.stolen_mgr: stolen memory manager. */
+		struct xe_ttm_stolen_mgr *stolen_mgr;
 	} mem;
 
 	/** @sriov: device level virtualization data */
@@ -295,7 +299,7 @@ struct xe_device {
 
 	/** @usm: unified memory state */
 	struct {
-		/** @usm.asid: convert a ASID to VM */
+		/** @usm.asid_to_vm: convert an ASID to VM */
 		struct xarray asid_to_vm;
 		/** @usm.next_asid: next ASID, used to cyclical alloc asids */
 		u32 next_asid;
@@ -312,7 +316,7 @@ struct xe_device {
 		/** @usm.pf_queue: Page fault queues */
 		struct xe_pagefault_queue pf_queue[XE_PAGEFAULT_QUEUE_COUNT];
 #if IS_ENABLED(CONFIG_DRM_XE_PAGEMAP)
-		/** @usm.pagemap_shrinker: Shrinker for unused pagemaps */
+		/** @usm.dpagemap_shrinker: Shrinker for unused pagemaps */
 		struct drm_pagemap_shrinker *dpagemap_shrinker;
 #endif
 	} usm;
@@ -334,7 +338,7 @@ struct xe_device {
 			struct list_head kernel_bo_present;
 			/** @pinned.late.evicted: pinned BO that have been evicted */
 			struct list_head evicted;
-			/** @pinned.external: pinned external and dma-buf. */
+			/** @pinned.late.external: pinned external and dma-buf. */
 			struct list_head external;
 		} late;
 	} pinned;
@@ -369,7 +373,7 @@ struct xe_device {
 		struct {
 			/**
 			 * @mem_access.vram_userfault.lock: Protects access to
-			 * @vram_usefault.list Using mutex instead of spinlock
+			 * @mem_access.vram_userfault.list Using mutex instead of spinlock
 			 * as lock is applied to entire list operation which
 			 * may sleep
 			 */
@@ -400,7 +404,7 @@ struct xe_device {
 		const struct xe_pat_table_entry *pat_primary_pta;
 		/** @pat.pat_media_pta: media GT PAT entry for page table accesses */
 		const struct xe_pat_table_entry *pat_media_pta;
-		u32 idx[__XE_CACHE_LEVEL_COUNT];
+		u16 idx[__XE_CACHE_LEVEL_COUNT];
 	} pat;
 
 	/** @d3cold: Encapsulate d3cold related stuff */
@@ -508,6 +512,9 @@ struct xe_device {
 	/** @i2c: I2C host controller */
 	struct xe_i2c *i2c;
 
+	/** @sc: System Controller */
+	struct xe_sysctrl sc;
+
 	/** @atomic_svm_timeslice_ms: Atomic SVM fault timeslice MS */
 	u32 atomic_svm_timeslice_ms;
 
@@ -578,7 +585,7 @@ struct xe_file {
 
 	/** @vm: VM state for file */
 	struct {
-		/** @vm.xe: xarray to store VMs */
+		/** @vm.xa: xarray to store VMs */
 		struct xarray xa;
 		/**
 		 * @vm.lock: Protects VM lookup + reference and removal from
diff --git a/drivers/gpu/drm/xe/xe_device_wa_oob.rules b/drivers/gpu/drm/xe/xe_device_wa_oob.rules
index d129cddb6ead4..d8dc41851425c 100644
--- a/drivers/gpu/drm/xe/xe_device_wa_oob.rules
+++ b/drivers/gpu/drm/xe/xe_device_wa_oob.rules
@@ -4,3 +4,6 @@
 22019338487_display	PLATFORM(LUNARLAKE)
 14022085890	SUBPLATFORM(BATTLEMAGE, G21)
 14026539277	PLATFORM(NOVALAKE_P), PLATFORM_STEP(A0, B0)
+14026633728	PLATFORM(CRESCENTISLAND)
+14026746987	PLATFORM(CRESCENTISLAND)
+14026779378	PLATFORM(CRESCENTISLAND)
diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c
index e07dc23a155ed..c21c8b428de6b 100644
--- a/drivers/gpu/drm/xe/xe_drm_ras.c
+++ b/drivers/gpu/drm/xe/xe_drm_ras.c
@@ -27,6 +27,16 @@ static int hw_query_error_counter(struct xe_drm_ras_counter *info,
 	return 0;
 }
 
+static int hw_clear_error_counter(struct xe_drm_ras_counter *info, u32 error_id)
+{
+	if (!info || !info[error_id].name)
+		return -ENOENT;
+
+	atomic_set(&info[error_id].counter, 0);
+
+	return 0;
+}
+
 static int query_uncorrectable_error_counter(struct drm_ras_node *ep, u32 error_id,
 					     const char **name, u32 *val)
 {
@@ -37,6 +47,15 @@ static int query_uncorrectable_error_counter(struct drm_ras_node *ep, u32 error_
 	return hw_query_error_counter(info, error_id, name, val);
 }
 
+static int clear_uncorrectable_error_counter(struct drm_ras_node *node, u32 error_id)
+{
+	struct xe_device *xe = node->priv;
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_UNCORRECTABLE];
+
+	return hw_clear_error_counter(info, error_id);
+}
+
 static int query_correctable_error_counter(struct drm_ras_node *ep, u32 error_id,
 					   const char **name, u32 *val)
 {
@@ -47,6 +66,15 @@ static int query_correctable_error_counter(struct drm_ras_node *ep, u32 error_id
 	return hw_query_error_counter(info, error_id, name, val);
 }
 
+static int clear_correctable_error_counter(struct drm_ras_node *node, u32 error_id)
+{
+	struct xe_device *xe = node->priv;
+	struct xe_drm_ras *ras = &xe->ras;
+	struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE];
+
+	return hw_clear_error_counter(info, error_id);
+}
+
 static struct xe_drm_ras_counter *allocate_and_copy_counters(struct xe_device *xe)
 {
 	struct xe_drm_ras_counter *counter;
@@ -92,10 +120,13 @@ static int assign_node_params(struct xe_device *xe, struct drm_ras_node *node,
 	if (IS_ERR(ras->info[severity]))
 		return PTR_ERR(ras->info[severity]);
 
-	if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE)
+	if (severity == DRM_XE_RAS_ERR_SEV_CORRECTABLE) {
 		node->query_error_counter = query_correctable_error_counter;
-	else
+		node->clear_error_counter = clear_correctable_error_counter;
+	} else {
 		node->query_error_counter = query_uncorrectable_error_counter;
+		node->clear_error_counter = clear_uncorrectable_error_counter;
+	}
 
 	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_drm_ras.h b/drivers/gpu/drm/xe/xe_drm_ras.h
index 5cc8f0124411a..365c70e93e82f 100644
--- a/drivers/gpu/drm/xe/xe_drm_ras.h
+++ b/drivers/gpu/drm/xe/xe_drm_ras.h
@@ -2,8 +2,8 @@
 /*
  * Copyright © 2026 Intel Corporation
  */
-#ifndef XE_DRM_RAS_H_
-#define XE_DRM_RAS_H_
+#ifndef _XE_DRM_RAS_H_
+#define _XE_DRM_RAS_H_
 
 struct xe_device;
 
diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c
index dddcdd0bb7a33..297be3c42b208 100644
--- a/drivers/gpu/drm/xe/xe_eu_stall.c
+++ b/drivers/gpu/drm/xe/xe_eu_stall.c
@@ -44,6 +44,7 @@ struct per_xecore_buf {
 struct xe_eu_stall_data_stream {
 	bool pollin;
 	bool enabled;
+	bool reset_detected;
 	int wait_num_reports;
 	int sampling_rate_mult;
 	wait_queue_head_t poll_wq;
@@ -428,9 +429,20 @@ static bool eu_stall_data_buf_poll(struct xe_eu_stall_data_stream *stream)
 			set_bit(xecore, stream->data_drop.mask);
 		xecore_buf->write = write_ptr;
 	}
+	/* If a GT or engine reset happens during EU stall sampling,
+	 * all EU stall registers get reset to 0 and the cached values of
+	 * the EU stall data buffers' read pointers are out of sync with
+	 * the register values. This causes invalid data to be returned
+	 * from read(). To prevent this, check the value of a EU stall base
+	 * register. If it is zero, there has been a reset.
+	 */
+	if (unlikely(!xe_gt_mcr_unicast_read_any(gt, XEHPC_EUSTALL_BASE)))
+		stream->reset_detected = true;
+
+	stream->pollin = min_data_present || stream->reset_detected;
 	mutex_unlock(&stream->xecore_buf_lock);
 
-	return min_data_present;
+	return stream->pollin;
 }
 
 static void clear_dropped_eviction_line_bit(struct xe_gt *gt, u16 group, u16 instance)
@@ -544,6 +556,15 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st
 	int ret = 0;
 
 	mutex_lock(&stream->xecore_buf_lock);
+	/* If EU stall registers got reset due to a GT/engine reset,
+	 * continuing with the read() will return invalid data to
+	 * the user space. Just return -ENODEV instead.
+	 */
+	if (unlikely(stream->reset_detected)) {
+		xe_gt_dbg(gt, "EU stall base register has been reset\n");
+		mutex_unlock(&stream->xecore_buf_lock);
+		return -ENODEV;
+	}
 	if (bitmap_weight(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS)) {
 		if (!stream->data_drop.reported_to_user) {
 			stream->data_drop.reported_to_user = true;
@@ -554,7 +575,6 @@ static ssize_t xe_eu_stall_stream_read_locked(struct xe_eu_stall_data_stream *st
 		}
 		stream->data_drop.reported_to_user = false;
 	}
-
 	for_each_dss_steering(xecore, gt, group, instance) {
 		ret = xe_eu_stall_data_buf_read(stream, buf, count, &total_size,
 						gt, group, instance, xecore);
@@ -609,7 +629,8 @@ static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf,
 	 * We don't want to block the next read() when there is data in the buffer
 	 * now, but couldn't be accommodated in the small user buffer.
 	 */
-	stream->pollin = false;
+	if (!stream->reset_detected)
+		stream->pollin = false;
 
 	return ret;
 }
@@ -692,6 +713,7 @@ static int xe_eu_stall_stream_enable(struct xe_eu_stall_data_stream *stream)
 		xecore_buf->write = write_ptr;
 		xecore_buf->read = write_ptr;
 	}
+	stream->reset_detected = false;
 	stream->data_drop.reported_to_user = false;
 	bitmap_zero(stream->data_drop.mask, XE_MAX_DSS_FUSE_BITS);
 
@@ -717,13 +739,13 @@ static void eu_stall_data_buf_poll_work_fn(struct work_struct *work)
 		container_of(work, typeof(*stream), buf_poll_work.work);
 	struct xe_gt *gt = stream->gt;
 
-	if (eu_stall_data_buf_poll(stream)) {
-		stream->pollin = true;
+	if (eu_stall_data_buf_poll(stream))
 		wake_up(&stream->poll_wq);
-	}
-	queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq,
-			   &stream->buf_poll_work,
-			   msecs_to_jiffies(POLL_PERIOD_MS));
+
+	if (!stream->reset_detected)
+		queue_delayed_work(gt->eu_stall->buf_ptr_poll_wq,
+				   &stream->buf_poll_work,
+				   msecs_to_jiffies(POLL_PERIOD_MS));
 }
 
 static int xe_eu_stall_stream_init(struct xe_eu_stall_data_stream *stream,
diff --git a/drivers/gpu/drm/xe/xe_eu_stall.h b/drivers/gpu/drm/xe/xe_eu_stall.h
index d1c76e5037992..842bef9f68722 100644
--- a/drivers/gpu/drm/xe/xe_eu_stall.h
+++ b/drivers/gpu/drm/xe/xe_eu_stall.h
@@ -3,8 +3,8 @@
  * Copyright © 2025 Intel Corporation
  */
 
-#ifndef __XE_EU_STALL_H__
-#define __XE_EU_STALL_H__
+#ifndef _XE_EU_STALL_H_
+#define _XE_EU_STALL_H_
 
 #include "xe_gt_types.h"
 #include "xe_sriov.h"
diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c
index 071b8c41df43e..1b5ca3ce578a4 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue.c
+++ b/drivers/gpu/drm/xe/xe_exec_queue.c
@@ -275,8 +275,12 @@ static void xe_exec_queue_set_lrc(struct xe_exec_queue *q, struct xe_lrc *lrc, u
 {
 	xe_assert(gt_to_xe(q->gt), idx < q->width);
 
-	scoped_guard(spinlock, &q->lrc_lookup_lock)
+	scoped_guard(spinlock, &q->lrc_lookup_lock) {
 		q->lrc[idx] = lrc;
+		if (xe_exec_queue_is_multi_queue(q))
+			q->lrc[idx]->multi_queue.primary_lrc =
+				q->multi_queue.group->primary->lrc[0];
+	}
 }
 
 /**
@@ -852,11 +856,6 @@ static int xe_exec_queue_group_init(struct xe_device *xe, struct xe_exec_queue *
 	return 0;
 }
 
-static inline bool xe_exec_queue_supports_multi_queue(struct xe_exec_queue *q)
-{
-	return q->gt->info.multi_queue_engine_class_mask & BIT(q->class);
-}
-
 static int xe_exec_queue_group_validate(struct xe_device *xe, struct xe_exec_queue *q,
 					u32 primary_id)
 {
@@ -912,6 +911,7 @@ static int xe_exec_queue_group_add(struct xe_device *xe, struct xe_exec_queue *q
 	}
 
 	q->multi_queue.pos = pos;
+	q->lrc[0]->multi_queue.pos = pos;
 
 	return 0;
 }
@@ -931,7 +931,7 @@ static void xe_exec_queue_group_delete(struct xe_device *xe, struct xe_exec_queu
 static int exec_queue_set_multi_group(struct xe_device *xe, struct xe_exec_queue *q,
 				      u64 value)
 {
-	if (XE_IOCTL_DBG(xe, !xe_exec_queue_supports_multi_queue(q)))
+	if (XE_IOCTL_DBG(xe, !xe_gt_supports_multi_queue(q->gt, q->class)))
 		return -ENODEV;
 
 	if (XE_IOCTL_DBG(xe, !xe_device_uc_enabled(xe)))
diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h
index 8ce78e0b1d502..2f5ccf294675a 100644
--- a/drivers/gpu/drm/xe/xe_exec_queue_types.h
+++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h
@@ -214,7 +214,7 @@ struct xe_exec_queue {
 		 */
 		struct xe_dep_scheduler *dep_scheduler;
 		/**
-		 * @last_fence: last fence for tlb invalidation, protected by
+		 * @tlb_inval.last_fence: last fence for tlb invalidation, protected by
 		 * vm->lock in write mode
 		 */
 		struct dma_fence *last_fence;
diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c
index 755a2bff5d7bc..9fb99c038ea88 100644
--- a/drivers/gpu/drm/xe/xe_execlist.c
+++ b/drivers/gpu/drm/xe/xe_execlist.c
@@ -47,7 +47,6 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc,
 	struct xe_mmio *mmio = &gt->mmio;
 	struct xe_device *xe = gt_to_xe(gt);
 	u64 lrc_desc;
-	u32 ring_mode = REG_MASKED_FIELD_ENABLE(GFX_DISABLE_LEGACY_MODE);
 
 	lrc_desc = xe_lrc_descriptor(lrc);
 
@@ -59,10 +58,6 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc,
 		lrc_desc |= FIELD_PREP(SW_CTX_ID, ctx_id);
 	}
 
-	if (hwe->class == XE_ENGINE_CLASS_COMPUTE)
-		xe_mmio_write32(mmio, RCU_MODE,
-				REG_MASKED_FIELD_ENABLE(RCU_MODE_CCS_ENABLE));
-
 	xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail);
 	lrc->ring.old_tail = lrc->ring.tail;
 
@@ -82,10 +77,6 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc,
 			xe_bo_ggtt_addr(hwe->hwsp));
 	xe_mmio_read32(mmio, RING_HWS_PGA(hwe->mmio_base));
 
-	if (xe_device_has_msix(gt_to_xe(hwe->gt)))
-		ring_mode |= REG_MASKED_FIELD_ENABLE(GFX_MSIX_INTERRUPT_ENABLE);
-	xe_mmio_write32(mmio, RING_MODE(hwe->mmio_base), ring_mode);
-
 	xe_mmio_write32(mmio, RING_EXECLIST_SQ_CONTENTS_LO(hwe->mmio_base),
 			lower_32_bits(lrc_desc));
 	xe_mmio_write32(mmio, RING_EXECLIST_SQ_CONTENTS_HI(hwe->mmio_base),
@@ -337,7 +328,6 @@ static int execlist_exec_queue_init(struct xe_exec_queue *q)
 	struct drm_gpu_scheduler *sched;
 	const struct drm_sched_init_args args = {
 		.ops = &drm_sched_ops,
-		.num_rqs = 1,
 		.credit_limit = xe_lrc_ring_size() / MAX_JOB_SIZE_BYTES,
 		.hang_limit = XE_SCHED_HANG_LIMIT,
 		.timeout = XE_SCHED_JOB_TIMEOUT,
diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c
index a848d1a41b9b9..a351c578b1705 100644
--- a/drivers/gpu/drm/xe/xe_ggtt.c
+++ b/drivers/gpu/drm/xe/xe_ggtt.c
@@ -24,6 +24,7 @@
 #include "xe_gt_types.h"
 #include "xe_map.h"
 #include "xe_mmio.h"
+#include "xe_pat.h"
 #include "xe_pm.h"
 #include "xe_res_cursor.h"
 #include "xe_sriov.h"
@@ -115,7 +116,6 @@ struct xe_ggtt {
 	/** @size: Total usable size of this GGTT */
 	u64 size;
 
-#define XE_GGTT_FLAGS_64K BIT(0)
 	/**
 	 * @flags: Flags for this GGTT
 	 * Acceptable flags:
@@ -259,7 +259,7 @@ static u64 xe_ggtt_get_pte(struct xe_ggtt *ggtt, u64 addr)
 
 static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size)
 {
-	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[XE_CACHE_WB];
+	u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), XE_CACHE_WB);
 	u64 end = start + size - 1;
 	u64 scratch_pte;
 
@@ -724,7 +724,7 @@ static void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_ggtt_node *node,
 void xe_ggtt_map_bo_unlocked(struct xe_ggtt *ggtt, struct xe_bo *bo)
 {
 	u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
-	u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
+	u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode);
 	u64 pte;
 
 	mutex_lock(&ggtt->lock);
@@ -841,7 +841,7 @@ static int __xe_ggtt_insert_bo_at(struct xe_ggtt *ggtt, struct xe_bo *bo,
 		bo->ggtt_node[tile_id] = NULL;
 	} else {
 		u16 cache_mode = bo->flags & XE_BO_FLAG_NEEDS_UC ? XE_CACHE_NONE : XE_CACHE_WB;
-		u16 pat_index = tile_to_xe(ggtt->tile)->pat.idx[cache_mode];
+		u16 pat_index = xe_cache_pat_idx(tile_to_xe(ggtt->tile), cache_mode);
 		u64 pte = ggtt->pt_ops->pte_encode_flags(bo, pat_index);
 
 		xe_ggtt_map_bo(ggtt, bo->ggtt_node[tile_id], bo, pte);
diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.c b/drivers/gpu/drm/xe/xe_gpu_scheduler.c
index 9c8004d5dd915..67d8ce3684869 100644
--- a/drivers/gpu/drm/xe/xe_gpu_scheduler.c
+++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.c
@@ -66,7 +66,6 @@ int xe_sched_init(struct xe_gpu_scheduler *sched,
 	const struct drm_sched_init_args args = {
 		.ops = ops,
 		.submit_wq = submit_wq,
-		.num_rqs = 1,
 		.credit_limit = hw_submission,
 		.hang_limit = hang_limit,
 		.timeout = timeout,
diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c
index 0d13e357fb43c..aab59dc647fbd 100644
--- a/drivers/gpu/drm/xe/xe_gsc.c
+++ b/drivers/gpu/drm/xe/xe_gsc.c
@@ -482,8 +482,7 @@ int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc)
 				 EXEC_QUEUE_FLAG_PERMANENT, 0);
 	if (IS_ERR(q)) {
 		xe_gt_err(gt, "Failed to create queue for GSC submission\n");
-		err = PTR_ERR(q);
-		goto out_bo;
+		return PTR_ERR(q);
 	}
 
 	wq = alloc_ordered_workqueue("gsc-ordered-wq", 0);
@@ -506,8 +505,6 @@ int xe_gsc_init_post_hwconfig(struct xe_gsc *gsc)
 
 out_q:
 	xe_exec_queue_put(q);
-out_bo:
-	xe_bo_unpin_map_no_vm(bo);
 	return err;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c
index 8a31c963c3721..783eb6d631b5c 100644
--- a/drivers/gpu/drm/xe/xe_gt.c
+++ b/drivers/gpu/drm/xe/xe_gt.c
@@ -7,6 +7,8 @@
 
 #include <linux/minmax.h>
 
+#include <kunit/visibility.h>
+
 #include <drm/drm_managed.h>
 #include <uapi/drm/xe_drm.h>
 
@@ -393,10 +395,7 @@ int xe_gt_record_default_lrcs(struct xe_gt *gt)
 		if (gt->default_lrc[hwe->class])
 			continue;
 
-		xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe);
-		xe_wa_process_lrc(hwe);
-		xe_hw_engine_setup_default_lrc_state(hwe);
-		xe_tuning_process_lrc(hwe);
+		xe_hw_engine_setup_reg_lrc(hwe);
 
 		default_lrc = drmm_kzalloc(&xe->drm,
 					   xe_gt_lrc_size(gt, hwe->class),
@@ -788,6 +787,7 @@ void xe_gt_mmio_init(struct xe_gt *gt)
 	if (IS_SRIOV_VF(xe))
 		gt->mmio.sriov_vf_gt = gt;
 }
+EXPORT_SYMBOL_IF_KUNIT(xe_gt_mmio_init);
 
 void xe_gt_record_user_engines(struct xe_gt *gt)
 {
@@ -993,7 +993,6 @@ void xe_gt_reset_async(struct xe_gt *gt)
 
 void xe_gt_suspend_prepare(struct xe_gt *gt)
 {
-	CLASS(xe_force_wake, fw_ref)(gt_to_fw(gt), XE_FORCEWAKE_ALL);
 	xe_uc_suspend_prepare(&gt->uc);
 }
 
diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h
index de7e47763411f..4150aa594f05f 100644
--- a/drivers/gpu/drm/xe/xe_gt.h
+++ b/drivers/gpu/drm/xe/xe_gt.h
@@ -155,4 +155,19 @@ static inline bool xe_gt_recovery_pending(struct xe_gt *gt)
 		xe_gt_sriov_vf_recovery_pending(gt);
 }
 
+/**
+ * xe_gt_supports_multi_queue() - Check if gt supports multi queue for the
+ * specified engine class.
+ *
+ * @gt: the GT object
+ * @class: hwe class type
+ *
+ * Return: true if the hw engine class supports multi queue, else false
+ */
+static inline bool xe_gt_supports_multi_queue(const struct xe_gt *gt,
+					      enum xe_engine_class class)
+{
+	return gt->info.multi_queue_engine_class_mask & BIT(class);
+}
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt_idle.c b/drivers/gpu/drm/xe/xe_gt_idle.c
index 4a2d9edb6a4c0..04b24e1c8b78e 100644
--- a/drivers/gpu/drm/xe/xe_gt_idle.c
+++ b/drivers/gpu/drm/xe/xe_gt_idle.c
@@ -3,6 +3,8 @@
  * Copyright © 2023 Intel Corporation
  */
 
+#include <linux/time64.h>
+
 #include <drm/drm_managed.h>
 
 #include <generated/xe_wa_oob.h>
@@ -93,7 +95,7 @@ static u64 get_residency_ms(struct xe_gt_idle *gtidle, u64 cur_residency)
 	gtidle->cur_residency = cur_residency;
 
 	/* residency multiplier in ns, convert to ms */
-	cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, 1e6);
+	cur_residency = mul_u64_u32_div(cur_residency, gtidle->residency_multiplier, NSEC_PER_MSEC);
 
 	return cur_residency;
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.c b/drivers/gpu/drm/xe/xe_gt_mcr.c
index 7c6f039c880df..04f0098070a4f 100644
--- a/drivers/gpu/drm/xe/xe_gt_mcr.c
+++ b/drivers/gpu/drm/xe/xe_gt_mcr.c
@@ -3,6 +3,9 @@
  * Copyright © 2022 Intel Corporation
  */
 
+#include <kunit/static_stub.h>
+#include <kunit/visibility.h>
+
 #include "xe_gt_mcr.h"
 
 #include "regs/xe_gt_regs.h"
@@ -216,9 +219,7 @@ static const struct xe_mmio_range xe2lpg_sqidi_psmi_steering_table[] = {
 static const struct xe_mmio_range xe2lpg_instance0_steering_table[] = {
 	{ 0x004000, 0x004AFF },         /* GAM, rsvd, GAMWKR */
 	{ 0x008700, 0x00887F },         /* SQIDI, MEMPIPE */
-	{ 0x00B000, 0x00B3FF },         /* NODE, L3BANK */
 	{ 0x00C800, 0x00CFFF },         /* GAM */
-	{ 0x00D880, 0x00D8FF },         /* NODE */
 	{ 0x00DD00, 0x00DDFF },         /* MEMPIPE */
 	{ 0x00E900, 0x00E97F },         /* MEMPIPE */
 	{ 0x00F000, 0x00FFFF },         /* GAM, GAMWKR */
@@ -267,7 +268,7 @@ static const struct xe_mmio_range xe3p_xpc_gam_grp1_steering_table[] = {
 	{},
 };
 
-static const struct xe_mmio_range xe3p_xpc_node_steering_table[] = {
+static const struct xe_mmio_range xe2_node_steering_table[] = {
 	{ 0x00B000, 0x00B0FF },
 	{ 0x00D880, 0x00D8FF },
 	{},
@@ -298,7 +299,7 @@ static void init_steering_l3bank(struct xe_gt *gt)
 	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_mmio *mmio = &gt->mmio;
 
-	if (GRAPHICS_VER(xe) >= 35) {
+	if (GRAPHICS_VER(xe) >= 20) {
 		unsigned int first_bank = xe_l3_bank_mask_ffs(gt->fuse_topo.l3_bank_mask);
 		const int banks_per_node = 4;
 		unsigned int node = first_bank / banks_per_node;
@@ -430,19 +431,6 @@ void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt, unsigned int dss, u16 *g
 	*instance = dss % gt->steering_dss_per_grp;
 }
 
-/**
- * xe_gt_mcr_steering_info_to_dss_id - Get DSS ID from group/instance steering
- * @gt: GT structure
- * @group: steering group ID
- * @instance: steering instance ID
- *
- * Return: the converted DSS id.
- */
-u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance)
-{
-	return group * dss_per_group(gt) + instance;
-}
-
 static void init_steering_dss(struct xe_gt *gt)
 {
 	gt->steering_dss_per_grp = dss_per_group(gt);
@@ -536,7 +524,7 @@ void xe_gt_mcr_init_early(struct xe_gt *gt)
 			gt->steering[GAM1].ranges = xe3p_xpc_gam_grp1_steering_table;
 			gt->steering[INSTANCE0].ranges = xe3p_xpc_instance0_steering_table;
 			gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
-			gt->steering[NODE].ranges = xe3p_xpc_node_steering_table;
+			gt->steering[NODE].ranges = xe2_node_steering_table;
 		} else if (GRAPHICS_VERx100(xe) >= 3510) {
 			gt->steering[DSS].ranges = xe2lpg_dss_steering_table;
 			gt->steering[INSTANCE0].ranges = xe3p_lpg_instance0_steering_table;
@@ -544,6 +532,8 @@ void xe_gt_mcr_init_early(struct xe_gt *gt)
 			gt->steering[DSS].ranges = xe2lpg_dss_steering_table;
 			gt->steering[SQIDI_PSMI].ranges = xe2lpg_sqidi_psmi_steering_table;
 			gt->steering[INSTANCE0].ranges = xe2lpg_instance0_steering_table;
+			gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
+			gt->steering[NODE].ranges = xe2_node_steering_table;
 		} else if (GRAPHICS_VERx100(xe) >= 1270) {
 			gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table;
 			gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table;
@@ -566,6 +556,7 @@ void xe_gt_mcr_init_early(struct xe_gt *gt)
 	/* Mark instance 0 as initialized, we need this early for VRAM and CCS probe. */
 	gt->steering[INSTANCE0].initialized = true;
 }
+EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_init_early);
 
 /**
  * xe_gt_mcr_init - Normal initialization of the MCR support
@@ -613,6 +604,40 @@ void xe_gt_mcr_set_implicit_defaults(struct xe_gt *gt)
 	}
 }
 
+static bool reg_in_steering_type_ranges(struct xe_gt *gt,
+					struct xe_reg reg,
+					int type)
+{
+	if (!gt->steering[type].ranges)
+		return false;
+
+	for (int i = 0; gt->steering[type].ranges[i].end > 0; i++)
+		if (xe_mmio_in_range(&gt->mmio, &gt->steering[type].ranges[i], reg))
+			return true;
+
+	return false;
+}
+
+/*
+ * xe_gt_mcr_check_reg - check if a register is recognized by this GT as MCR
+ * @gt: GT structure
+ * @reg: The register to check
+ *
+ * Returns true if the register offset falls within one of the MMIO ranges
+ * classified as MCR for the GT.
+ */
+bool xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg)
+{
+	KUNIT_STATIC_STUB_REDIRECT(xe_gt_mcr_check_reg, gt, reg);
+
+	for (int type = 0; type <= IMPLICIT_STEERING; type++)
+		if (reg_in_steering_type_ranges(gt, reg, type))
+			return true;
+
+	return false;
+}
+EXPORT_SYMBOL_IF_KUNIT(xe_gt_mcr_check_reg);
+
 /*
  * xe_gt_mcr_get_nonterminated_steering - find group/instance values that
  *    will steer a register to a non-terminated instance
@@ -634,30 +659,21 @@ bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
 					  u8 *group, u8 *instance)
 {
 	const struct xe_reg reg = to_xe_reg(reg_mcr);
-	const struct xe_mmio_range *implicit_ranges;
 
 	for (int type = 0; type < IMPLICIT_STEERING; type++) {
-		if (!gt->steering[type].ranges)
-			continue;
-
-		for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) {
-			if (xe_mmio_in_range(&gt->mmio, &gt->steering[type].ranges[i], reg)) {
-				drm_WARN(&gt_to_xe(gt)->drm, !gt->steering[type].initialized,
-					 "Uninitialized usage of MCR register %s/%#x\n",
-					 xe_steering_types[type].name, reg.addr);
-
-				*group = gt->steering[type].group_target;
-				*instance = gt->steering[type].instance_target;
-				return true;
-			}
+		if (reg_in_steering_type_ranges(gt, reg, type)) {
+			drm_WARN(&gt_to_xe(gt)->drm, !gt->steering[type].initialized,
+				 "Uninitialized usage of MCR register %s/%#x\n",
+				 xe_steering_types[type].name, reg.addr);
+
+			*group = gt->steering[type].group_target;
+			*instance = gt->steering[type].instance_target;
+			return true;
 		}
 	}
 
-	implicit_ranges = gt->steering[IMPLICIT_STEERING].ranges;
-	if (implicit_ranges)
-		for (int i = 0; implicit_ranges[i].end > 0; i++)
-			if (xe_mmio_in_range(&gt->mmio, &implicit_ranges[i], reg))
-				return false;
+	if (reg_in_steering_type_ranges(gt, reg, IMPLICIT_STEERING))
+		return false;
 
 	/*
 	 * Not found in a steering table and not a register with implicit
diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.h b/drivers/gpu/drm/xe/xe_gt_mcr.h
index 283a1c9770e2d..75374662f10d6 100644
--- a/drivers/gpu/drm/xe/xe_gt_mcr.h
+++ b/drivers/gpu/drm/xe/xe_gt_mcr.h
@@ -26,6 +26,7 @@ void xe_gt_mcr_unicast_write(struct xe_gt *gt, struct xe_reg_mcr mcr_reg,
 void xe_gt_mcr_multicast_write(struct xe_gt *gt, struct xe_reg_mcr mcr_reg,
 			       u32 value);
 
+bool xe_gt_mcr_check_reg(struct xe_gt *gt, struct xe_reg reg);
 bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
 					  struct xe_reg_mcr reg_mcr,
 					  u8 *group, u8 *instance);
@@ -33,7 +34,6 @@ bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt,
 void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p);
 void xe_gt_mcr_get_dss_steering(const struct xe_gt *gt,
 				unsigned int dss, u16 *group, u16 *instance);
-u32 xe_gt_mcr_steering_info_to_dss_id(struct xe_gt *gt, u16 group, u16 instance);
 
 /*
  * Loop over each DSS and determine the group and instance IDs that
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c
index fb5c9101e2756..7868056841b3c 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c
@@ -219,6 +219,7 @@ static void pf_restart(struct xe_gt *gt)
 
 	xe_gt_assert(gt, !xe_pm_runtime_suspended(xe));
 
+	xe_gt_sriov_pf_policy_restart(gt);
 	xe_gt_sriov_pf_config_restart(gt);
 	xe_gt_sriov_pf_control_restart(gt);
 
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
index 2f376b5fb0882..2c9b85b84b1be 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c
@@ -285,6 +285,13 @@ static u32 encode_config_ggtt(u32 *cfg, const struct xe_gt_sriov_config *config,
 	return encode_ggtt(cfg, xe_ggtt_node_addr(node), xe_ggtt_node_size(node), details);
 }
 
+static bool custom_sched_priority(struct xe_gt *gt, u32 priority)
+{
+	return xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt) ?
+		priority != GUC_SCHED_PRIORITY_NORMAL :
+		priority != GUC_SCHED_PRIORITY_LOW;
+}
+
 static u32 encode_config_sched(struct xe_gt *gt, u32 *cfg, u32 n,
 			       const struct xe_gt_sriov_config *config)
 {
@@ -313,6 +320,11 @@ static u32 encode_config_sched(struct xe_gt *gt, u32 *cfg, u32 n,
 		cfg[n++] = config->preempt_timeout[0];
 	}
 
+	if (custom_sched_priority(gt, config->sched_priority)) {
+		cfg[n++] = PREP_GUC_KLV_TAG(VF_CFG_SCHED_PRIORITY);
+		cfg[n++] = config->sched_priority;
+	}
+
 	return n;
 }
 
@@ -1922,29 +1934,45 @@ static u64 pf_profile_fair_lmem(struct xe_gt *gt, unsigned int num_vfs)
 	return ALIGN_DOWN(fair, alignment);
 }
 
-static void __pf_show_provisioning_lmem(struct xe_gt *gt, unsigned int first_vf,
-					unsigned int num_vfs, bool provisioned)
+static void __pf_show_provisioned(struct xe_gt *gt, unsigned int first_vf,
+				  unsigned int num_vfs, bool provisioned,
+				  u32 (*get32)(struct xe_gt *, unsigned int),
+				  u64 (*get64)(struct xe_gt *, unsigned int),
+				  const char *what)
 {
 	unsigned int allvfs = 1 + xe_gt_sriov_pf_get_totalvfs(gt); /* PF plus VFs */
 	unsigned long *bitmap __free(bitmap) = bitmap_zalloc(allvfs, GFP_KERNEL);
 	unsigned int weight;
 	unsigned int n;
+	bool pf;
+
+	xe_gt_assert(gt, get32 || get64);
 
 	if (!bitmap)
 		return;
 
 	for (n = first_vf; n < first_vf + num_vfs; n++) {
-		if (!!pf_get_vf_config_lmem(gt, VFID(n)) == provisioned)
+		if ((get32 && (!!get32(gt, VFID(n)) == provisioned)) ||
+		    (get64 && (!!get64(gt, VFID(n)) == provisioned)))
 			bitmap_set(bitmap, n, 1);
 	}
 
+	pf = test_and_clear_bit(0, bitmap);
 	weight = bitmap_weight(bitmap, allvfs);
-	if (!weight)
+	if (!pf && !weight)
 		return;
 
-	xe_gt_sriov_info(gt, "VF%s%*pbl %s provisioned with VRAM\n",
-			 weight > 1 ? "s " : "", allvfs, bitmap,
-			 provisioned ? "already" : "not");
+	xe_gt_sriov_info(gt, "%s%s%s%s%*pbl %s provisioned with %s\n",
+			 pf ? "PF" : "", pf && weight ? " and " : "",
+			 weight ? "VF" : "", weight > 1 ? "s " : "",
+			 allvfs, bitmap, provisioned ? "already" : "not", what);
+}
+
+static void __pf_show_provisioning_lmem(struct xe_gt *gt, unsigned int first_vf,
+					unsigned int num_vfs, bool provisioned)
+{
+	__pf_show_provisioned(gt, first_vf, num_vfs, provisioned,
+			      NULL, pf_get_vf_config_lmem, "VRAM");
 }
 
 static void pf_show_all_provisioned_lmem(struct xe_gt *gt)
@@ -2078,6 +2106,17 @@ static u32 pf_get_exec_quantum(struct xe_gt *gt, unsigned int vfid)
 	return config->exec_quantum[0];
 }
 
+static bool pf_non_default_exec_quantum(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(config->exec_quantum); i++)
+		if (config->exec_quantum[i])
+			return true;
+	return false;
+}
+
 /**
  * xe_gt_sriov_pf_config_set_exec_quantum_locked() - Configure PF/VF execution quantum.
  * @gt: the &xe_gt
@@ -2154,34 +2193,41 @@ u32 xe_gt_sriov_pf_config_get_exec_quantum(struct xe_gt *gt, unsigned int vfid)
 	return pf_get_exec_quantum(gt, vfid);
 }
 
-/**
- * xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked() - Configure EQ for PF and VFs.
- * @gt: the &xe_gt to configure
- * @exec_quantum: requested execution quantum in milliseconds (0 is infinity)
- *
- * This function can only be called on PF with the master mutex hold.
- *
- * Return: 0 on success or a negative error code on failure.
- */
-int xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(struct xe_gt *gt, u32 exec_quantum)
+static int pf_bulk_set_exec_quantum(struct xe_gt *gt, u32 exec_quantum,
+				    unsigned int first_vf, unsigned int num_vfs)
 {
-	unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt);
 	unsigned int n;
 	int err = 0;
 
 	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
-	for (n = 0; n <= totalvfs; n++) {
+	for (n = first_vf; n < first_vf + num_vfs; n++) {
 		err = pf_provision_exec_quantum(gt, VFID(n), exec_quantum);
 		if (err)
 			break;
 	}
 
-	return pf_config_bulk_set_u32_done(gt, 0, 1 + totalvfs, exec_quantum,
+	return pf_config_bulk_set_u32_done(gt, first_vf, num_vfs, exec_quantum,
 					   pf_get_exec_quantum, "execution quantum",
 					   exec_quantum_unit, n, err);
 }
 
+/**
+ * xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked() - Configure EQ for PF and VFs.
+ * @gt: the &xe_gt to configure
+ * @exec_quantum: requested execution quantum in milliseconds (0 is infinity)
+ *
+ * This function can only be called on PF with the master mutex hold.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_exec_quantum_locked(struct xe_gt *gt, u32 exec_quantum)
+{
+	unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt);
+
+	return pf_bulk_set_exec_quantum(gt, exec_quantum, PFID, 1 + totalvfs);
+}
+
 static int pf_provision_groups_exec_quantums(struct xe_gt *gt, unsigned int vfid,
 					     const u32 *exec_quantums, u32 count)
 {
@@ -2293,6 +2339,17 @@ static u32 pf_get_preempt_timeout(struct xe_gt *gt, unsigned int vfid)
 	return config->preempt_timeout[0];
 }
 
+static bool pf_non_default_preempt_timeout(struct xe_gt *gt, unsigned int vfid)
+{
+	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(config->preempt_timeout); i++)
+		if (config->preempt_timeout[i])
+			return true;
+	return false;
+}
+
 /**
  * xe_gt_sriov_pf_config_set_preempt_timeout_locked() - Configure PF/VF preemption timeout.
  * @gt: the &xe_gt
@@ -2368,34 +2425,41 @@ u32 xe_gt_sriov_pf_config_get_preempt_timeout(struct xe_gt *gt, unsigned int vfi
 	return pf_get_preempt_timeout(gt, vfid);
 }
 
-/**
- * xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked() - Configure PT for PF and VFs.
- * @gt: the &xe_gt to configure
- * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity)
- *
- * This function can only be called on PF with the master mutex hold.
- *
- * Return: 0 on success or a negative error code on failure.
- */
-int xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(struct xe_gt *gt, u32 preempt_timeout)
+static int pf_bulk_set_preempt_timeout(struct xe_gt *gt, u32 preempt_timeout,
+				       unsigned int first_vf, unsigned int num_vfs)
 {
-	unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt);
 	unsigned int n;
 	int err = 0;
 
 	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
-	for (n = 0; n <= totalvfs; n++) {
+	for (n = first_vf; n < first_vf + num_vfs; n++) {
 		err = pf_provision_preempt_timeout(gt, VFID(n), preempt_timeout);
 		if (err)
 			break;
 	}
 
-	return pf_config_bulk_set_u32_done(gt, 0, 1 + totalvfs, preempt_timeout,
+	return pf_config_bulk_set_u32_done(gt, first_vf, num_vfs, preempt_timeout,
 					   pf_get_preempt_timeout, "preemption timeout",
 					   preempt_timeout_unit, n, err);
 }
 
+/**
+ * xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked() - Configure PT for PF and VFs.
+ * @gt: the &xe_gt to configure
+ * @preempt_timeout: requested preemption timeout in microseconds (0 is infinity)
+ *
+ * This function can only be called on PF with the master mutex hold.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_bulk_set_preempt_timeout_locked(struct xe_gt *gt, u32 preempt_timeout)
+{
+	unsigned int totalvfs = xe_gt_sriov_pf_get_totalvfs(gt);
+
+	return pf_bulk_set_preempt_timeout(gt, preempt_timeout, PFID, 1 + totalvfs);
+}
+
 static int pf_provision_groups_preempt_timeouts(struct xe_gt *gt, unsigned int vfid,
 						const u32 *preempt_timeouts, u32 count)
 {
@@ -2500,7 +2564,7 @@ static int pf_provision_sched_priority(struct xe_gt *gt, unsigned int vfid, u32
 	return 0;
 }
 
-static int pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid)
+static u32 pf_get_sched_priority(struct xe_gt *gt, unsigned int vfid)
 {
 	struct xe_gt_sriov_config *config = pf_pick_vf_config(gt, vfid);
 
@@ -2550,6 +2614,35 @@ u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid
 	return priority;
 }
 
+/**
+ * xe_gt_sriov_pf_config_force_sched_priority_locked() - Force update scheduling priority.
+ * @gt: the &xe_gt
+ * @priority: new scheduling priority to set
+ *
+ * This function allows to update cached values of the scheduling priorities of all
+ * VFs (and PF) as result of applying the `GUC_KLV_VGT_POLICY_SCHED_IF_IDLE`_ policy.
+ *
+ * This function can only be called on PF.
+ */
+void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority)
+{
+	unsigned int total_vfs = xe_gt_sriov_pf_get_totalvfs(gt);
+	struct xe_gt_sriov_config *config;
+	unsigned int n;
+
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	for (n = 0; n <= total_vfs; n++) {
+		config = pf_pick_vf_config(gt, VFID(n));
+		config->sched_priority = priority;
+	}
+
+	pf_config_bulk_set_u32_done(gt, PFID, 1 + total_vfs, priority,
+				    pf_get_sched_priority, "scheduling priority",
+				    sched_priority_unit, n, 0);
+}
+
 static void pf_reset_config_sched(struct xe_gt *gt, struct xe_gt_sriov_config *config)
 {
 	int i;
@@ -2562,6 +2655,104 @@ static void pf_reset_config_sched(struct xe_gt *gt, struct xe_gt_sriov_config *c
 	}
 }
 
+static bool pf_non_default_sched(struct xe_gt *gt, unsigned int vfid)
+{
+	return pf_non_default_exec_quantum(gt, vfid) ||
+	       pf_non_default_preempt_timeout(gt, vfid) ||
+	       custom_sched_priority(gt, pf_get_sched_priority(gt, vfid));
+}
+
+static void __pf_show_provisioned_sched(struct xe_gt *gt, unsigned int first_vf,
+					unsigned int num_vfs, bool provisioned)
+{
+	__pf_show_provisioned(gt, first_vf, num_vfs, provisioned,
+			      pf_get_exec_quantum, NULL, "EQ");
+	__pf_show_provisioned(gt, first_vf, num_vfs, provisioned,
+			      pf_get_preempt_timeout, NULL, "PT");
+
+	/* we only care about non-default priorities */
+	if (provisioned)
+		__pf_show_provisioned(gt, first_vf, num_vfs, true,
+				      pf_get_sched_priority, NULL, "PRIORITY");
+}
+
+static void pf_show_all_provisioned_sched(struct xe_gt *gt)
+{
+	__pf_show_provisioned_sched(gt, PFID, 1 + xe_gt_sriov_pf_get_totalvfs(gt), true);
+}
+
+static void pf_show_unprovisioned_sched(struct xe_gt *gt, unsigned int num_vfs)
+{
+	__pf_show_provisioned_sched(gt, PFID, 1 + num_vfs, false);
+}
+
+static bool pf_needs_provision_sched(struct xe_gt *gt, unsigned int num_vfs)
+{
+	unsigned int vfid;
+
+	for (vfid = PFID; vfid <= PFID + num_vfs; vfid++) {
+		if (pf_non_default_sched(gt, vfid)) {
+			pf_show_all_provisioned_sched(gt);
+			pf_show_unprovisioned_sched(gt, num_vfs);
+			return false;
+		}
+	}
+
+	if (xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt)) {
+		pf_show_all_provisioned_sched(gt);
+		pf_show_unprovisioned_sched(gt, num_vfs);
+		return false;
+	}
+
+	pf_show_all_provisioned_sched(gt);
+	return true;
+}
+
+/* With 16ms EQ/PT GuC should be able to handle up to 63 VFs within 2s */
+#define XE_FAIR_EXEC_QUANTUM_MS		16
+#define XE_FAIR_PREEMPT_TIMEOUT_US	16000
+#define XE_FAIR_SCHED_PRIORITY		GUC_SCHED_PRIORITY_LOW
+#define XE_ADMIN_PF_SCHED_PRIORITY	GUC_SCHED_PRIORITY_HIGH
+
+/**
+ * xe_gt_sriov_pf_config_set_fair_sched() - Provision PF and VFs with fair scheduling.
+ * @gt: the &xe_gt
+ * @num_vfs: number of VFs to provision (can't be 0)
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs)
+{
+	int result = 0;
+	int err;
+
+	xe_gt_assert(gt, num_vfs);
+	xe_gt_assert(gt, XE_FAIR_EXEC_QUANTUM_MS);
+	xe_gt_assert(gt, XE_FAIR_PREEMPT_TIMEOUT_US);
+
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+
+	if (!pf_needs_provision_sched(gt, num_vfs))
+		return 0;
+
+	err = pf_bulk_set_exec_quantum(gt, XE_FAIR_EXEC_QUANTUM_MS, PFID, 1 + num_vfs);
+	result = result ?: err;
+	err = pf_bulk_set_preempt_timeout(gt, XE_FAIR_PREEMPT_TIMEOUT_US, PFID, 1 + num_vfs);
+	result = result ?: err;
+
+	xe_gt_assert(gt, XE_FAIR_SCHED_PRIORITY == GUC_SCHED_PRIORITY_LOW);
+	xe_gt_assert(gt, !xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt));
+
+	if (xe_sriov_pf_admin_only(gt_to_xe(gt))) {
+		err = pf_provision_sched_priority(gt, PFID, XE_ADMIN_PF_SCHED_PRIORITY);
+		result = result ?: err;
+	}
+
+	return result;
+}
+
 static int pf_provision_threshold(struct xe_gt *gt, unsigned int vfid,
 				  enum xe_guc_klv_threshold_index index, u32 value)
 {
@@ -2838,6 +3029,9 @@ static int pf_validate_vf_config(struct xe_gt *gt, unsigned int vfid)
 		valid_all = valid_all && valid_lmem;
 	}
 
+	/* also check optional EQ/PT/PRIO */
+	valid_any = valid_any || pf_non_default_sched(gt, vfid);
+
 	return valid_all ? 0 : valid_any ? -ENOKEY : -ENODATA;
 }
 
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
index 4a004ecd61406..2ec62c12ad5cf 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.h
@@ -71,12 +71,14 @@ int xe_gt_sriov_pf_config_set_groups_preempt_timeouts(struct xe_gt *gt, unsigned
 
 u32 xe_gt_sriov_pf_config_get_sched_priority(struct xe_gt *gt, unsigned int vfid);
 int xe_gt_sriov_pf_config_set_sched_priority(struct xe_gt *gt, unsigned int vfid, u32 priority);
+void xe_gt_sriov_pf_config_force_sched_priority_locked(struct xe_gt *gt, u32 priority);
 
 u32 xe_gt_sriov_pf_config_get_threshold(struct xe_gt *gt, unsigned int vfid,
 					enum xe_guc_klv_threshold_index index);
 int xe_gt_sriov_pf_config_set_threshold(struct xe_gt *gt, unsigned int vfid,
 					enum xe_guc_klv_threshold_index index, u32 value);
 
+int xe_gt_sriov_pf_config_set_fair_sched(struct xe_gt *gt, unsigned int num_vfs);
 int xe_gt_sriov_pf_config_set_fair(struct xe_gt *gt, unsigned int vfid, unsigned int num_vfs);
 int xe_gt_sriov_pf_config_sanitize(struct xe_gt *gt, unsigned int vfid, long timeout);
 int xe_gt_sriov_pf_config_release(struct xe_gt *gt, unsigned int vfid, bool force);
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c
index 7d532bded02a8..a85ba44353789 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.c
@@ -114,8 +114,10 @@ int xe_gt_sriov_pf_monitor_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32
  * VFs with no events are not printed.
  *
  * This function can only be called on PF.
+ *
+ * Return: always 0
  */
-void xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p)
+int xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p)
 {
 	unsigned int n, total_vfs = xe_gt_sriov_pf_get_totalvfs(gt);
 	const struct xe_gt_sriov_monitor *data;
@@ -144,4 +146,6 @@ void xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p
 #undef __format
 #undef __value
 	}
+
+	return 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h
index 7ca9351a271b7..0b8f088d3a16a 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_monitor.h
@@ -13,7 +13,7 @@ struct drm_printer;
 struct xe_gt;
 
 void xe_gt_sriov_pf_monitor_flr(struct xe_gt *gt, u32 vfid);
-void xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_pf_monitor_print_events(struct xe_gt *gt, struct drm_printer *p);
 
 #ifdef CONFIG_PCI_IOV
 int xe_gt_sriov_pf_monitor_process_guc2pf(struct xe_gt *gt, const u32 *msg, u32 len);
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c
index 848e24926ecd0..e8458d63742dd 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.c
@@ -8,6 +8,7 @@
 #include "abi/guc_actions_sriov_abi.h"
 
 #include "xe_gt.h"
+#include "xe_gt_sriov_pf_config.h"
 #include "xe_gt_sriov_pf_helpers.h"
 #include "xe_gt_sriov_pf_policy.h"
 #include "xe_gt_sriov_printk.h"
@@ -16,7 +17,6 @@
 #include "xe_guc_ct.h"
 #include "xe_guc_klv_helpers.h"
 #include "xe_guc_submit.h"
-#include "xe_pm.h"
 
 /*
  * Return: number of KLVs that were successfully parsed and saved,
@@ -67,6 +67,15 @@ static int pf_push_policy_buf_klvs(struct xe_gt *gt, u32 num_klvs,
 		return err;
 	}
 
+	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV)) {
+		struct drm_printer p = xe_gt_dbg_printer(gt);
+		void *klvs = xe_guc_buf_cpu_ptr(buf);
+
+		xe_gt_sriov_dbg(gt, "pushed policy update with %u KLV%s:\n",
+				num_klvs, str_plural(num_klvs));
+		xe_guc_klv_print(klvs, num_dwords, &p);
+	}
+
 	return 0;
 }
 
@@ -153,33 +162,14 @@ static int pf_update_policy_u32(struct xe_gt *gt, u16 key, u32 *policy, u32 valu
 	return 0;
 }
 
-static void pf_bulk_reset_sched_priority(struct xe_gt *gt, u32 priority)
-{
-	unsigned int total_vfs = 1 + xe_gt_sriov_pf_get_totalvfs(gt);
-	unsigned int n;
-
-	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
-	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
-
-	for (n = 0; n < total_vfs; n++)
-		gt->sriov.pf.vfs[n].config.sched_priority = priority;
-}
-
 static int pf_provision_sched_if_idle(struct xe_gt *gt, bool enable)
 {
-	int err;
-
 	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
 	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
-	err = pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY,
-				    &gt->sriov.pf.policy.guc.sched_if_idle,
-				    enable);
-
-	if (!err)
-		pf_bulk_reset_sched_priority(gt, enable ? GUC_SCHED_PRIORITY_NORMAL :
-					     GUC_SCHED_PRIORITY_LOW);
-	return err;
+	return pf_update_policy_bool(gt, GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY,
+				     &gt->sriov.pf.policy.guc.sched_if_idle,
+				     enable);
 }
 
 static int pf_reprovision_sched_if_idle(struct xe_gt *gt)
@@ -187,6 +177,9 @@ static int pf_reprovision_sched_if_idle(struct xe_gt *gt)
 	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
 	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
+	if (!gt->sriov.pf.policy.guc.sched_if_idle)
+		return 0;
+
 	return pf_provision_sched_if_idle(gt, gt->sriov.pf.policy.guc.sched_if_idle);
 }
 
@@ -199,7 +192,7 @@ static void pf_sanitize_sched_if_idle(struct xe_gt *gt)
 }
 
 /**
- * xe_gt_sriov_pf_policy_set_sched_if_idle - Control the 'sched_if_idle' policy.
+ * xe_gt_sriov_pf_policy_set_sched_if_idle_locked() - Control the 'sched_if_idle' policy.
  * @gt: the &xe_gt where to apply the policy
  * @enable: the value of the 'sched_if_idle' policy
  *
@@ -207,15 +200,60 @@ static void pf_sanitize_sched_if_idle(struct xe_gt *gt)
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable)
+int xe_gt_sriov_pf_policy_set_sched_if_idle_locked(struct xe_gt *gt, bool enable)
 {
+	u32 priority;
 	int err;
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
 	err = pf_provision_sched_if_idle(gt, enable);
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
+	if (err)
+		return err;
 
-	return err;
+	/*
+	 * As of GuC 70.12 a change of this policy impacts individual configs
+	 * of all VFs. See `GUC_KLV_VGT_POLICY_SCHED_IF_IDLE`_ for details.
+	 */
+	xe_gt_assert(gt, GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 12));
+
+	priority = enable ? GUC_SCHED_PRIORITY_NORMAL : GUC_SCHED_PRIORITY_LOW;
+	xe_gt_sriov_pf_config_force_sched_priority_locked(gt, priority);
+
+	return 0;
+}
+
+/**
+ * xe_gt_sriov_pf_policy_set_sched_if_idle() - Control the 'sched_if_idle' policy.
+ * @gt: the &xe_gt where to apply the policy
+ * @enable: the value of the 'sched_if_idle' policy
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
+
+	return xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, enable);
+}
+
+/**
+ * xe_gt_sriov_pf_policy_get_sched_if_idle_locked() - Retrieve value of 'sched_if_idle' policy.
+ * @gt: the &xe_gt where to read the policy from
+ *
+ * This function can only be called on PF.
+ *
+ * Return: last value of 'sched_if_idle' policy applied.
+ */
+bool xe_gt_sriov_pf_policy_get_sched_if_idle_locked(struct xe_gt *gt)
+{
+	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
+
+	return gt->sriov.pf.policy.guc.sched_if_idle;
 }
 
 /**
@@ -228,15 +266,10 @@ int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable)
  */
 bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt)
 {
-	bool enable;
-
 	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
-	enable = gt->sriov.pf.policy.guc.sched_if_idle;
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
-
-	return enable;
+	return xe_gt_sriov_pf_policy_get_sched_if_idle_locked(gt);
 }
 
 static int pf_provision_reset_engine(struct xe_gt *gt, bool enable)
@@ -253,6 +286,9 @@ static int pf_reprovision_reset_engine(struct xe_gt *gt)
 	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
 	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
+	if (!gt->sriov.pf.policy.guc.reset_engine)
+		return 0;
+
 	return pf_provision_reset_engine(gt, gt->sriov.pf.policy.guc.reset_engine);
 }
 
@@ -319,6 +355,9 @@ static int pf_reprovision_sample_period(struct xe_gt *gt)
 	xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt)));
 	lockdep_assert_held(xe_gt_sriov_pf_master_mutex(gt));
 
+	if (!gt->sriov.pf.policy.guc.sample_period)
+		return 0;
+
 	return pf_provision_sample_period(gt, gt->sriov.pf.policy.guc.sample_period);
 }
 
@@ -689,30 +728,23 @@ void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt)
 }
 
 /**
- * xe_gt_sriov_pf_policy_reprovision - Reprovision (and optionally reset) policy settings.
+ * xe_gt_sriov_pf_policy_restart() - Reprovision policy settings.
  * @gt: the &xe_gt
- * @reset: if true will reprovision using default values instead of latest
  *
  * This function can only be called on PF.
  *
  * Return: 0 on success or a negative error code on failure.
  */
-int xe_gt_sriov_pf_policy_reprovision(struct xe_gt *gt, bool reset)
+int xe_gt_sriov_pf_policy_restart(struct xe_gt *gt)
 {
 	int err = 0;
 
-	xe_pm_runtime_get_noresume(gt_to_xe(gt));
+	guard(mutex)(xe_gt_sriov_pf_master_mutex(gt));
 
-	mutex_lock(xe_gt_sriov_pf_master_mutex(gt));
-	if (reset)
-		pf_sanitize_guc_policies(gt);
 	err |= pf_reprovision_sched_if_idle(gt);
 	err |= pf_reprovision_reset_engine(gt);
 	err |= pf_reprovision_sample_period(gt);
 	err |= pf_reprovision_sched_groups(gt);
-	mutex_unlock(xe_gt_sriov_pf_master_mutex(gt));
-
-	xe_pm_runtime_put(gt_to_xe(gt));
 
 	return err ? -ENXIO : 0;
 }
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h
index bd73aa58f9ca7..d667d66897b10 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_policy.h
@@ -14,7 +14,9 @@ struct drm_printer;
 struct xe_gt;
 
 int xe_gt_sriov_pf_policy_set_sched_if_idle(struct xe_gt *gt, bool enable);
+int xe_gt_sriov_pf_policy_set_sched_if_idle_locked(struct xe_gt *gt, bool enable);
 bool xe_gt_sriov_pf_policy_get_sched_if_idle(struct xe_gt *gt);
+bool xe_gt_sriov_pf_policy_get_sched_if_idle_locked(struct xe_gt *gt);
 int xe_gt_sriov_pf_policy_set_reset_engine(struct xe_gt *gt, bool enable);
 bool xe_gt_sriov_pf_policy_get_reset_engine(struct xe_gt *gt);
 int xe_gt_sriov_pf_policy_set_sample_period(struct xe_gt *gt, u32 value);
@@ -29,7 +31,7 @@ bool xe_gt_sriov_pf_policy_sched_groups_enabled(struct xe_gt *gt);
 
 void xe_gt_sriov_pf_policy_init(struct xe_gt *gt);
 void xe_gt_sriov_pf_policy_sanitize(struct xe_gt *gt);
-int xe_gt_sriov_pf_policy_reprovision(struct xe_gt *gt, bool reset);
+int xe_gt_sriov_pf_policy_restart(struct xe_gt *gt);
 int xe_gt_sriov_pf_policy_print(struct xe_gt *gt, struct drm_printer *p);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h
index 667b8310478d4..6296483c44ec3 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_types.h
@@ -49,7 +49,6 @@ struct xe_gt_sriov_pf_workers {
  * @service: service data.
  * @control: control data.
  * @policy: policy data.
- * @migration: migration data.
  * @spare: PF-only provisioning configuration.
  * @vfs: metadata for all VFs.
  */
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
index 8989c8e1be95e..0cd9d77f33513 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c
@@ -1137,13 +1137,15 @@ void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val)
 }
 
 /**
- * xe_gt_sriov_vf_print_config - Print VF self config.
+ * xe_gt_sriov_vf_print_config() - Print VF self config.
  * @gt: the &xe_gt
  * @p: the &drm_printer
  *
  * This function is for VF use only.
+ *
+ * Return: always 0.
  */
-void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p)
+int xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_gt_sriov_vf_selfconfig *config = &gt->sriov.vf.self_config;
 	struct xe_device *xe = gt_to_xe(gt);
@@ -1170,16 +1172,20 @@ void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p)
 
 	drm_printf(p, "GuC contexts:\t%u\n", config->num_ctxs);
 	drm_printf(p, "GuC doorbells:\t%u\n", config->num_dbs);
+
+	return 0;
 }
 
 /**
- * xe_gt_sriov_vf_print_runtime - Print VF's runtime regs received from PF.
+ * xe_gt_sriov_vf_print_runtime() - Print VF's runtime regs received from PF.
  * @gt: the &xe_gt
  * @p: the &drm_printer
  *
  * This function is for VF use only.
+ *
+ * Return: always 0.
  */
-void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p)
+int xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct vf_runtime_reg *vf_regs = gt->sriov.vf.runtime.regs;
 	unsigned int size = gt->sriov.vf.runtime.num_regs;
@@ -1188,16 +1194,20 @@ void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p)
 
 	for (; size--; vf_regs++)
 		drm_printf(p, "%#x = %#x\n", vf_regs->offset, vf_regs->value);
+
+	return 0;
 }
 
 /**
- * xe_gt_sriov_vf_print_version - Print VF ABI versions.
+ * xe_gt_sriov_vf_print_version() - Print VF ABI versions.
  * @gt: the &xe_gt
  * @p: the &drm_printer
  *
  * This function is for VF use only.
+ *
+ * Return: always 0.
  */
-void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p)
+int xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 	struct xe_uc_fw_version *guc_version = &gt->sriov.vf.guc_version;
@@ -1227,6 +1237,8 @@ void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p)
 		   GUC_RELAY_VERSION_LATEST_MAJOR, GUC_RELAY_VERSION_LATEST_MINOR);
 	drm_printf(p, "\thandshake:\t%u.%u\n",
 		   pf_version->major, pf_version->minor);
+
+	return 0;
 }
 
 static bool vf_post_migration_shutdown(struct xe_gt *gt)
diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
index a6f7127521a5d..79878f21b1dae 100644
--- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
+++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.h
@@ -35,9 +35,9 @@ bool xe_gt_sriov_vf_sched_groups_enabled(struct xe_gt *gt);
 u32 xe_gt_sriov_vf_read32(struct xe_gt *gt, struct xe_reg reg);
 void xe_gt_sriov_vf_write32(struct xe_gt *gt, struct xe_reg reg, u32 val);
 
-void xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p);
-void xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p);
-void xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_vf_print_config(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_vf_print_runtime(struct xe_gt *gt, struct drm_printer *p);
+int xe_gt_sriov_vf_print_version(struct xe_gt *gt, struct drm_printer *p);
 
 int xe_gt_sriov_vf_wait_valid_ggtt(struct xe_gt *gt);
 int xe_vf_migration_fixups_complete_count(struct xe_gt *gt);
diff --git a/drivers/gpu/drm/xe/xe_gt_stats.c b/drivers/gpu/drm/xe/xe_gt_stats.c
index 59b3b23a54c89..789397514f3ef 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats.c
+++ b/drivers/gpu/drm/xe/xe_gt_stats.c
@@ -9,6 +9,47 @@
 #include "xe_device.h"
 #include "xe_gt_stats.h"
 
+/**
+ * DOC: Xe GT Statistics
+ *
+ * Overview
+ * ========
+ *
+ * The Xe driver exposes per-GT statistics through the debugfs filesystem at::
+ *
+ *   /sys/kernel/debug/dri/<device>/gt<id>/stats
+ *
+ * This interface requires the kernel to be built with ``CONFIG_DEBUG_FS=y``.
+ *
+ * Reading statistics
+ * ==================
+ *
+ * Reading the file prints all available statistics, one per line, in
+ * ``name: value`` format::
+ *
+ *   $ cat /sys/kernel/debug/dri/0/gt0/stats
+ *   svm_pagefault_count: 0
+ *   tlb_inval_count: 1234
+ *   ...
+ *
+ * All values are 64-bit unsigned integers aggregated across all CPUs.
+ * Counters accumulate since the driver was loaded or since the last explicit
+ * reset.  Timing counters use microseconds as their unit; data volume counters
+ * use KiB.
+ *
+ * Resetting statistics
+ * ====================
+ *
+ * Writing a boolean true value to the file resets all counters to zero::
+ *
+ *   echo 1 > /sys/kernel/debug/dri/0/gt0/stats
+ *
+ * Any value accepted by ``kstrtobool()`` (e.g. ``1``, ``y``, ``yes``,
+ * ``on``) triggers the reset.  Resetting while the GPU is active may yield
+ * unpredictable intermediate values; it is recommended to reset only when
+ * the GPU is idle.
+ */
+
 static void xe_gt_stats_fini(struct drm_device *drm, void *arg)
 {
 	struct xe_gt *gt = arg;
diff --git a/drivers/gpu/drm/xe/xe_gt_stats_types.h b/drivers/gpu/drm/xe/xe_gt_stats_types.h
index 081c787ddcb6e..425491bed6c4e 100644
--- a/drivers/gpu/drm/xe/xe_gt_stats_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_stats_types.h
@@ -8,6 +8,124 @@
 
 #include <linux/types.h>
 
+/**
+ * enum xe_gt_stats_id - GT statistics identifiers
+ * @XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT: Total SVM page faults handled.
+ * @XE_GT_STATS_ID_TLB_INVAL: Total GPU Translation Lookaside Buffer (TLB)
+ *   invalidations issued.
+ * @XE_GT_STATS_ID_SVM_TLB_INVAL_COUNT: TLB invalidations issued during SVM
+ *   page-fault handling.
+ * @XE_GT_STATS_ID_SVM_TLB_INVAL_US: Cumulative time (µs) waiting for TLB
+ *   invalidations during SVM page-fault handling.
+ *
+ * @XE_GT_STATS_ID_VMA_PAGEFAULT_COUNT: Buffer-object (non-SVM) page faults
+ *   handled.
+ * @XE_GT_STATS_ID_VMA_PAGEFAULT_KB: Size (KiB) of VMAs involved in
+ *   buffer-object page fault handling.
+ * @XE_GT_STATS_ID_INVALID_PREFETCH_PAGEFAULT_COUNT: GPU prefetch faults for
+ *   addresses with no valid backing.
+ *
+ * @XE_GT_STATS_ID_SVM_4K_PAGEFAULT_COUNT: SVM page faults resolved by
+ *   mapping 4K pages.
+ * @XE_GT_STATS_ID_SVM_64K_PAGEFAULT_COUNT: SVM page faults resolved by
+ *   mapping 64K pages.
+ * @XE_GT_STATS_ID_SVM_2M_PAGEFAULT_COUNT: SVM page faults resolved by
+ *   mapping 2M pages.
+ * @XE_GT_STATS_ID_SVM_4K_VALID_PAGEFAULT_COUNT: Valid SVM page faults
+ *   at 4K page size, where the GPU mapping was already valid — resolved without
+ *   creating new mappings.
+ * @XE_GT_STATS_ID_SVM_64K_VALID_PAGEFAULT_COUNT: Valid SVM page faults at 64K
+ *   page size.
+ * @XE_GT_STATS_ID_SVM_2M_VALID_PAGEFAULT_COUNT: Valid SVM page faults at 2M
+ *   page size.
+ * @XE_GT_STATS_ID_SVM_4K_PAGEFAULT_US: Cumulative time (µs) handling 4K SVM
+ *   page faults.
+ * @XE_GT_STATS_ID_SVM_64K_PAGEFAULT_US: Cumulative time (µs) handling 64K
+ *   SVM page faults.
+ * @XE_GT_STATS_ID_SVM_2M_PAGEFAULT_US: Cumulative time (µs) handling 2M SVM
+ *   page faults.
+ *
+ * @XE_GT_STATS_ID_SVM_4K_MIGRATE_COUNT: 4K pages moved from CPU to device
+ *   memory.
+ * @XE_GT_STATS_ID_SVM_64K_MIGRATE_COUNT: 64K pages moved from CPU to device
+ *   memory.
+ * @XE_GT_STATS_ID_SVM_2M_MIGRATE_COUNT: 2M pages moved from CPU to device
+ *   memory.
+ * @XE_GT_STATS_ID_SVM_4K_MIGRATE_US: Cumulative time (µs) moving 4K pages
+ *   from CPU to device memory.
+ * @XE_GT_STATS_ID_SVM_64K_MIGRATE_US: Cumulative time (µs) moving 64K pages
+ *   from CPU to device memory.
+ * @XE_GT_STATS_ID_SVM_2M_MIGRATE_US: Cumulative time (µs) moving 2M pages
+ *   from CPU to device memory.
+ *
+ * @XE_GT_STATS_ID_SVM_DEVICE_COPY_US: Cumulative time (µs) for memory copies to
+ *   device, across all page sizes.
+ * @XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_US: Cumulative time (µs) for memory copies
+ *   of 4K pages to device.
+ * @XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_US: Cumulative time (µs) for memory
+ *   copies of 64K pages to device.
+ * @XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_US: Cumulative time (µs) for memory copies
+ *   of 2M pages to device.
+ * @XE_GT_STATS_ID_SVM_CPU_COPY_US: Cumulative time (µs) for memory copies to
+ *   CPU, across all page sizes.
+ * @XE_GT_STATS_ID_SVM_4K_CPU_COPY_US: Cumulative time (µs) for memory copies of
+ *   4K pages to CPU.
+ * @XE_GT_STATS_ID_SVM_64K_CPU_COPY_US: Cumulative time (µs) for memory copies
+ *   of 64K pages to CPU.
+ * @XE_GT_STATS_ID_SVM_2M_CPU_COPY_US: Cumulative time (µs) for memory copies of
+ *   2M pages to CPU.
+ * @XE_GT_STATS_ID_SVM_DEVICE_COPY_KB: Data (KiB) copied to device across all
+ *   page sizes.
+ * @XE_GT_STATS_ID_SVM_4K_DEVICE_COPY_KB: Data (KiB) copied to device for 4K
+ *   pages.
+ * @XE_GT_STATS_ID_SVM_64K_DEVICE_COPY_KB: Data (KiB) copied to device for
+ *   64K pages.
+ * @XE_GT_STATS_ID_SVM_2M_DEVICE_COPY_KB: Data (KiB) copied to device for 2M
+ *   pages.
+ * @XE_GT_STATS_ID_SVM_CPU_COPY_KB: Data (KiB) copied to CPU across all page
+ *   sizes.
+ * @XE_GT_STATS_ID_SVM_4K_CPU_COPY_KB: Data (KiB) copied to CPU for 4K pages.
+ * @XE_GT_STATS_ID_SVM_64K_CPU_COPY_KB: Data (KiB) copied to CPU for 64K pages.
+ * @XE_GT_STATS_ID_SVM_2M_CPU_COPY_KB: Data (KiB) copied to CPU for 2M pages.
+ *
+ * @XE_GT_STATS_ID_SVM_4K_GET_PAGES_US: Cumulative time (µs) getting CPU
+ *   memory pages for GPU access at 4K page size.
+ * @XE_GT_STATS_ID_SVM_64K_GET_PAGES_US: Cumulative time (µs) getting CPU
+ *   memory pages for GPU access at 64K page size.
+ * @XE_GT_STATS_ID_SVM_2M_GET_PAGES_US: Cumulative time (µs) getting CPU
+ *   memory pages for GPU access at 2M page size.
+ * @XE_GT_STATS_ID_SVM_4K_BIND_US: Cumulative time (µs) binding 4K pages
+ *   into the GPU page table.
+ * @XE_GT_STATS_ID_SVM_64K_BIND_US: Cumulative time (µs) binding 64K pages
+ *   into the GPU page table.
+ * @XE_GT_STATS_ID_SVM_2M_BIND_US: Cumulative time (µs) binding 2M pages
+ *   into the GPU page table.
+ *
+ * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_COUNT: Times the
+ *   scheduler preempted a long-running (LR) GPU exec queue.
+ * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SKIP_LR_QUEUE_COUNT: Times the scheduler
+ *   skipped suspend because the system was idle.
+ * @XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_COUNT: Times the driver
+ *   stalled waiting for prior GPU work to complete before scheduling more.
+ * @XE_GT_STATS_ID_HW_ENGINE_GROUP_SUSPEND_LR_QUEUE_US: Cumulative time
+ *   (µs) spent preempting long-running (LR) GPU exec queues.
+ * @XE_GT_STATS_ID_HW_ENGINE_GROUP_WAIT_DMA_QUEUE_US: Cumulative time (µs)
+ *   stalled waiting for prior GPU work to complete.
+ *
+ * @XE_GT_STATS_ID_PRL_4K_ENTRY_COUNT: 4K-page entries from the page reclaim
+ *   list that were processed.
+ * @XE_GT_STATS_ID_PRL_64K_ENTRY_COUNT: 64K-page entries from the page reclaim
+ *   list that were processed.
+ * @XE_GT_STATS_ID_PRL_2M_ENTRY_COUNT: 2M-page entries from the page reclaim
+ *   list that were processed.
+ * @XE_GT_STATS_ID_PRL_ISSUED_COUNT: Times a page reclamation was issued.
+ * @XE_GT_STATS_ID_PRL_ABORTED_COUNT: Times the page reclaim process was
+ *   aborted.
+ *
+ * @__XE_GT_STATS_NUM_IDS: Number of valid IDs; not a real counter.
+ *
+ * See Documentation/gpu/xe/xe_gt_stats.rst.
+ */
 enum xe_gt_stats_id {
 	XE_GT_STATS_ID_SVM_PAGEFAULT_COUNT,
 	XE_GT_STATS_ID_TLB_INVAL,
diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h
index fffb5d631b69b..e5588c88800af 100644
--- a/drivers/gpu/drm/xe/xe_gt_types.h
+++ b/drivers/gpu/drm/xe/xe_gt_types.h
@@ -136,7 +136,7 @@ struct xe_gt {
 		/** @info.gmdid: raw GMD_ID value from hardware */
 		u32 gmdid;
 		/**
-		 * @multi_queue_engine_class_mask: Bitmask of engine classes with
+		 * @info.multi_queue_engine_class_mask: Bitmask of engine classes with
 		 * multi queue support enabled.
 		 */
 		u16 multi_queue_engine_class_mask;
@@ -362,7 +362,7 @@ struct xe_gt {
 	/** @user_engines: engines present in GT and available to userspace */
 	struct {
 		/**
-		 * @user_engines.mask: like @info->engine_mask, but take in
+		 * @user_engines.mask: like @info.engine_mask, but take in
 		 * consideration only engines available to userspace
 		 */
 		u64 mask;
diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c
index ccebb437e37fd..4023700ff2a96 100644
--- a/drivers/gpu/drm/xe/xe_guc.c
+++ b/drivers/gpu/drm/xe/xe_guc.c
@@ -98,7 +98,7 @@ static u32 guc_ctl_feature_flags(struct xe_guc *guc)
 	if (xe_guc_using_main_gamctrl_queues(guc))
 		flags |= GUC_CTL_MAIN_GAMCTRL_QUEUES;
 
-	if (GRAPHICS_VER(xe) >= 35 && !IS_DGFX(xe) && xe_gt_is_media_type(guc_to_gt(guc)))
+	if (xe_device_is_l2_flush_optimized(xe) && xe_gt_is_media_type(guc_to_gt(guc)))
 		flags |= GUC_CTL_ENABLE_L2FLUSH_OPT;
 
 	return flags;
@@ -780,6 +780,14 @@ int xe_guc_init(struct xe_guc *guc)
 	if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 14, 0))
 		xe->info.has_page_reclaim_hw_assist = false;
 
+	/* Disable indirect_ring_state if missing GuC 70.53+ WA 14025515070. */
+	if (gt->info.has_indirect_ring_state &&
+	    XE_GT_WA(gt, 14025515070) &&
+	    GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 26, 0)) {
+		gt->info.has_indirect_ring_state = 0;
+		xe_gt_notice(gt, "indirect ring state requires WA in GuC submit ver 1.26+\n");
+	}
+
 	if (IS_SRIOV_VF(xe)) {
 		ret = devm_add_action_or_reset(xe->drm.dev, vf_guc_fini_hw, guc);
 		if (ret)
@@ -1693,13 +1701,8 @@ void xe_guc_reset_wait(struct xe_guc *guc)
 
 void xe_guc_stop_prepare(struct xe_guc *guc)
 {
-	if (!IS_SRIOV_VF(guc_to_xe(guc))) {
-		int err;
-
-		err = xe_guc_pc_stop(&guc->pc);
-		xe_gt_WARN(guc_to_gt(guc), err, "Failed to stop GuC PC: %pe\n",
-			   ERR_PTR(err));
-	}
+	if (!IS_SRIOV_VF(guc_to_xe(guc)))
+		xe_guc_pc_stop(&guc->pc);
 }
 
 void xe_guc_stop(struct xe_guc *guc)
diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c
index 2b835d48b5652..b9bca6084a4fa 100644
--- a/drivers/gpu/drm/xe/xe_guc_ads.c
+++ b/drivers/gpu/drm/xe/xe_guc_ads.c
@@ -360,6 +360,9 @@ static void guc_waklv_init(struct xe_guc_ads *ads)
 	if (XE_GT_WA(gt, 14020001231))
 		guc_waklv_enable(ads, NULL, 0, &offset, &remain,
 				 GUC_WORKAROUND_KLV_DISABLE_PSMI_INTERRUPTS_AT_C6_ENTRY_RESTORE_AT_EXIT);
+	if (XE_GT_WA(gt, 14025515070) && GUC_FIRMWARE_VER_AT_LEAST(&gt->uc.guc, 70, 53))
+		guc_waklv_enable(ads, NULL, 0, &offset, &remain,
+				 GUC_WA_KLV_CLR_CS_INDIRECT_RING_STATE_IF_IDLE_AT_CTX_REG);
 
 	size = guc_ads_waklv_size(ads) - remain;
 	if (!size)
@@ -739,10 +742,8 @@ static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads,
 		struct xe_reg reg;
 		bool skip;
 	} *e, extra_regs[] = {
-		{ .reg = RING_MODE(hwe->mmio_base),			},
 		{ .reg = RING_HWS_PGA(hwe->mmio_base),			},
 		{ .reg = RING_IMR(hwe->mmio_base),			},
-		{ .reg = RCU_MODE, .skip = hwe != hwe_rcs_reset_domain	},
 		{ .reg = CCS_MODE,
 		  .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) },
 	};
diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c
index 2f5816c78fba2..21f7caf9ea084 100644
--- a/drivers/gpu/drm/xe/xe_guc_capture.c
+++ b/drivers/gpu/drm/xe/xe_guc_capture.c
@@ -111,7 +111,7 @@ struct __guc_capture_parsed_output {
 	{ RING_TAIL(0),			REG_32BIT,	0,	0,	0,	"RING_TAIL"}, \
 	{ RING_CTL(0),			REG_32BIT,	0,	0,	0,	"RING_CTL"}, \
 	{ RING_MI_MODE(0),		REG_32BIT,	0,	0,	0,	"RING_MI_MODE"}, \
-	{ RING_MODE(0),			REG_32BIT,	0,	0,	0,	"RING_MODE"}, \
+	{ GFX_MODE(0),			REG_32BIT,	0,	0,	0,	"GFX_MODE"}, \
 	{ RING_ESR(0),			REG_32BIT,	0,	0,	0,	"RING_ESR"}, \
 	{ RING_EMR(0),			REG_32BIT,	0,	0,	0,	"RING_EMR"}, \
 	{ RING_EIR(0),			REG_32BIT,	0,	0,	0,	"RING_EIR"}, \
@@ -1841,12 +1841,6 @@ void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm
 		   str_yes_no(snapshot->kernel_reserved));
 
 	for (type = GUC_STATE_CAPTURE_TYPE_GLOBAL; type < GUC_STATE_CAPTURE_TYPE_MAX; type++) {
-		/*
-		 * FIXME: During devcoredump print we should avoid accessing the
-		 * driver pointers for gt or engine. Printing should be done only
-		 * using the snapshot captured. Here we are accessing the gt
-		 * pointer. It should be fixed.
-		 */
 		list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, type,
 							capture_class, false);
 		snapshot_print_by_list_order(snapshot, p, type, list);
diff --git a/drivers/gpu/drm/xe/xe_guc_capture.h b/drivers/gpu/drm/xe/xe_guc_capture.h
index 34d6fdc64f567..dca97d52b192d 100644
--- a/drivers/gpu/drm/xe/xe_guc_capture.h
+++ b/drivers/gpu/drm/xe/xe_guc_capture.h
@@ -3,8 +3,8 @@
  * Copyright © 2021-2024 Intel Corporation
  */
 
-#ifndef _XE_GUC_CAPTURE_H
-#define _XE_GUC_CAPTURE_H
+#ifndef _XE_GUC_CAPTURE_H_
+#define _XE_GUC_CAPTURE_H_
 
 #include <linux/types.h>
 #include "abi/guc_capture_abi.h"
diff --git a/drivers/gpu/drm/xe/xe_guc_capture_types.h b/drivers/gpu/drm/xe/xe_guc_capture_types.h
index 6cb4391155979..058a3f2eadce9 100644
--- a/drivers/gpu/drm/xe/xe_guc_capture_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_capture_types.h
@@ -3,8 +3,8 @@
  * Copyright © 2021-2024 Intel Corporation
  */
 
-#ifndef _XE_GUC_CAPTURE_TYPES_H
-#define _XE_GUC_CAPTURE_TYPES_H
+#ifndef _XE_GUC_CAPTURE_TYPES_H_
+#define _XE_GUC_CAPTURE_TYPES_H_
 
 #include <linux/types.h>
 #include "regs/xe_reg_defs.h"
@@ -32,7 +32,7 @@ struct __guc_mmio_reg_descr {
 	/**
 	 * @data_type: data type of the register
 	 * Could be 32 bit, low or hi dword of a 64 bit, see enum
-	 * register_data_type
+	 * capture_register_data_type
 	 */
 	enum capture_register_data_type data_type;
 	/** @flags: Flags for the register */
diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c
index a11cff7a20be7..21e0dad9a4815 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct.c
+++ b/drivers/gpu/drm/xe/xe_guc_ct.c
@@ -186,13 +186,16 @@ static void fast_req_track(struct xe_guc_ct *ct, u16 fence, u16 action) { }
 struct g2h_fence {
 	u32 *response_buffer;
 	u32 seqno;
+	/* fields below this point are setup based on the response */
 	u32 response_data;
 	u16 response_len;
 	u16 error;
 	u16 hint;
 	u16 reason;
+	u32 counter;
 	bool cancel;
 	bool retry;
+	bool wait;
 	bool fail;
 	bool done;
 };
@@ -204,6 +207,11 @@ static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer)
 	g2h_fence->seqno = ~0x0;
 }
 
+static void g2h_fence_reinit(struct g2h_fence *g2h_fence)
+{
+	memset_after(g2h_fence, 0, seqno);
+}
+
 static void g2h_fence_cancel(struct g2h_fence *g2h_fence)
 {
 	g2h_fence->cancel = true;
@@ -1331,6 +1339,7 @@ static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
 	/* READ_ONCEs pairs with WRITE_ONCEs in parse_g2h_response
 	 * and g2h_fence_cancel.
 	 */
+wait_again:
 	ret = wait_event_timeout(ct->g2h_fence_wq, READ_ONCE(g2h_fence.done), HZ);
 	if (!ret) {
 		LNL_FLUSH_WORK(&ct->g2h_worker);
@@ -1356,6 +1365,14 @@ static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
 		return -ETIME;
 	}
 
+	if (g2h_fence.wait) {
+		xe_gt_dbg(gt, "H2G action %#x busy: counter %u\n",
+			  action[0], g2h_fence.counter);
+		/* we can't leave any response data if we want to wait again */
+		g2h_fence_reinit(&g2h_fence);
+		mutex_unlock(&ct->lock);
+		goto wait_again;
+	}
 	if (g2h_fence.retry) {
 		xe_gt_dbg(gt, "H2G action %#x retrying: reason %#x\n",
 			  action[0], g2h_fence.reason);
@@ -1508,7 +1525,12 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		return -EPROTO;
 	}
 
-	g2h_fence = xa_erase(&ct->fence_lookup, fence);
+	/* don't erase as we still expect a final response with the same fence */
+	if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY)
+		g2h_fence = xa_load(&ct->fence_lookup, fence);
+	else
+		g2h_fence = xa_erase(&ct->fence_lookup, fence);
+
 	if (unlikely(!g2h_fence)) {
 		/* Don't tear down channel, as send could've timed out */
 		/* CT_DEAD(ct, NULL, PARSE_G2H_UNKNOWN); */
@@ -1519,6 +1541,12 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
 
 	xe_gt_assert(gt, fence == g2h_fence->seqno);
 
+	/*
+	 * reinit as we might have already process this g2h_fence before
+	 * if we received a NO_RESPONSE_BUSY reply
+	 */
+	g2h_fence_reinit(g2h_fence);
+
 	if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) {
 		g2h_fence->fail = true;
 		g2h_fence->error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, hxg[0]);
@@ -1526,6 +1554,9 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
 	} else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
 		g2h_fence->retry = true;
 		g2h_fence->reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, hxg[0]);
+	} else if (type == GUC_HXG_TYPE_NO_RESPONSE_BUSY) {
+		g2h_fence->wait = true;
+		g2h_fence->counter = FIELD_GET(GUC_HXG_BUSY_MSG_0_COUNTER, hxg[0]);
 	} else if (g2h_fence->response_buffer) {
 		g2h_fence->response_len = hxg_len;
 		memcpy(g2h_fence->response_buffer, hxg, hxg_len * sizeof(u32));
@@ -1533,7 +1564,9 @@ static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
 		g2h_fence->response_data = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, hxg[0]);
 	}
 
-	g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
+	/* don't release any space if it was an intermediate message */
+	if (!g2h_fence->wait)
+		g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
 
 	/* WRITE_ONCE pairs with READ_ONCEs in guc_ct_send_recv. */
 	WRITE_ONCE(g2h_fence->done, true);
@@ -1570,6 +1603,7 @@ static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
 	case GUC_HXG_TYPE_RESPONSE_SUCCESS:
 	case GUC_HXG_TYPE_RESPONSE_FAILURE:
 	case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
+	case GUC_HXG_TYPE_NO_RESPONSE_BUSY:
 		ret = parse_g2h_response(ct, msg, len);
 		break;
 	default:
diff --git a/drivers/gpu/drm/xe/xe_guc_ct_types.h b/drivers/gpu/drm/xe/xe_guc_ct_types.h
index 5da1ce5dc3724..f88e588af0d3d 100644
--- a/drivers/gpu/drm/xe/xe_guc_ct_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_ct_types.h
@@ -102,9 +102,9 @@ struct xe_dead_ct {
 	bool reported;
 	/** @worker: worker thread to get out of interrupt context before dumping */
 	struct work_struct worker;
-	/** snapshot_ct: copy of CT state and CTB content at point of error */
+	/** @snapshot_ct: copy of CT state and CTB content at point of error */
 	struct xe_guc_ct_snapshot *snapshot_ct;
-	/** snapshot_log: copy of GuC log at point of error */
+	/** @snapshot_log: copy of GuC log at point of error */
 	struct xe_guc_log_snapshot *snapshot_log;
 };
 
@@ -134,9 +134,9 @@ struct xe_guc_ct {
 	spinlock_t fast_lock;
 	/** @ctbs: buffers for sending and receiving commands */
 	struct {
-		/** @ctbs.send: Host to GuC (H2G, send) channel */
+		/** @ctbs.h2g: Host to GuC (H2G, send) channel */
 		struct guc_ctb h2g;
-		/** @ctbs.recv: GuC to Host (G2H, receive) channel */
+		/** @ctbs.g2h: GuC to Host (G2H, receive) channel */
 		struct guc_ctb g2h;
 	} ctbs;
 	/** @g2h_outstanding: number of outstanding G2H */
diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h
index b73fae063facd..3fbda4798cff3 100644
--- a/drivers/gpu/drm/xe/xe_guc_fwif.h
+++ b/drivers/gpu/drm/xe/xe_guc_fwif.h
@@ -3,8 +3,8 @@
  * Copyright © 2022 Intel Corporation
  */
 
-#ifndef _XE_GUC_FWIF_H
-#define _XE_GUC_FWIF_H
+#ifndef _XE_GUC_FWIF_H_
+#define _XE_GUC_FWIF_H_
 
 #include <linux/bits.h>
 
diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c
index bb8c4e7934926..59f2fa79ad422 100644
--- a/drivers/gpu/drm/xe/xe_guc_pc.c
+++ b/drivers/gpu/drm/xe/xe_guc_pc.c
@@ -890,9 +890,26 @@ void xe_guc_pc_init_early(struct xe_guc_pc *pc)
 	pc_init_fused_rp_values(pc);
 }
 
+static bool pc_needs_min_freq_change(struct xe_guc_pc *pc)
+{
+	struct xe_device *xe = pc_to_xe(pc);
+	struct xe_gt *gt = pc_to_gt(pc);
+
+	if (XE_DEVICE_WA(xe, 14022085890))
+		return true;
+
+	if (xe_gt_is_media_type(gt))
+		return false;
+
+	if (xe->info.platform == XE_BATTLEMAGE ||
+	    xe->info.platform == XE_CRESCENTISLAND)
+		return true;
+
+	return false;
+}
+
 static int pc_adjust_freq_bounds(struct xe_guc_pc *pc)
 {
-	struct xe_tile *tile = gt_to_tile(pc_to_gt(pc));
 	int ret;
 
 	lockdep_assert_held(&pc->freq_lock);
@@ -919,7 +936,18 @@ static int pc_adjust_freq_bounds(struct xe_guc_pc *pc)
 	if (pc_get_min_freq(pc) > pc->rp0_freq)
 		ret = pc_set_min_freq(pc, pc->rp0_freq);
 
-	if (XE_DEVICE_WA(tile_to_xe(tile), 14022085890))
+	/*
+	 * Setting GT RP min frequency to 1.2GHz by default for
+	 * GT0(Graphics) Tile of BMG and CRI.
+	 *
+	 * While BMG G21 WA will apply min frequency for
+	 * both GT0(Graphics) and GT1(Media) Tile.
+	 *
+	 * This is an active frequency, so if the device is idle
+	 * we aren't expecting high power output across board
+	 *
+	 */
+	if (pc_needs_min_freq_change(pc))
 		ret = pc_set_min_freq(pc, max(BMG_MIN_FREQ, pc_get_min_freq(pc)));
 
 out:
@@ -1288,18 +1316,16 @@ int xe_guc_pc_start(struct xe_guc_pc *pc)
  * xe_guc_pc_stop - Stop GuC's Power Conservation component
  * @pc: Xe_GuC_PC instance
  */
-int xe_guc_pc_stop(struct xe_guc_pc *pc)
+void xe_guc_pc_stop(struct xe_guc_pc *pc)
 {
 	struct xe_device *xe = pc_to_xe(pc);
 
 	if (xe->info.skip_guc_pc)
-		return 0;
+		return;
 
 	mutex_lock(&pc->freq_lock);
 	pc->freq_ready = false;
 	mutex_unlock(&pc->freq_lock);
-
-	return 0;
 }
 
 /**
@@ -1314,10 +1340,10 @@ static void xe_guc_pc_fini_hw(void *arg)
 	if (xe_device_wedged(xe))
 		return;
 
-	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FW_GT);
-	XE_WARN_ON(xe_guc_pc_stop(pc));
+	xe_guc_pc_stop(pc);
 
 	/* Bind requested freq to mert_freq_cap before unload */
+	CLASS(xe_force_wake, fw_ref)(gt_to_fw(pc_to_gt(pc)), XE_FW_GT);
 	pc_set_cur_freq(pc, min(pc_max_freq_cap(pc), xe_guc_pc_get_rpe_freq(pc)));
 }
 
diff --git a/drivers/gpu/drm/xe/xe_guc_pc.h b/drivers/gpu/drm/xe/xe_guc_pc.h
index 0678a4e787b3f..1025a2b15f5f4 100644
--- a/drivers/gpu/drm/xe/xe_guc_pc.h
+++ b/drivers/gpu/drm/xe/xe_guc_pc.h
@@ -13,7 +13,7 @@ struct drm_printer;
 
 int xe_guc_pc_init(struct xe_guc_pc *pc);
 int xe_guc_pc_start(struct xe_guc_pc *pc);
-int xe_guc_pc_stop(struct xe_guc_pc *pc);
+void xe_guc_pc_stop(struct xe_guc_pc *pc);
 void xe_guc_pc_print(struct xe_guc_pc *pc, struct drm_printer *p);
 int xe_guc_pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value);
 int xe_guc_pc_action_unset_param(struct xe_guc_pc *pc, u8 id);
diff --git a/drivers/gpu/drm/xe/xe_guc_relay_types.h b/drivers/gpu/drm/xe/xe_guc_relay_types.h
index 20eee10856b25..b99a6686416ba 100644
--- a/drivers/gpu/drm/xe/xe_guc_relay_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_relay_types.h
@@ -15,7 +15,7 @@
  * struct xe_guc_relay - Data used by the VF-PF Relay Communication over GuC.
  */
 struct xe_guc_relay {
-	/**@lock: protects all internal data. */
+	/** @lock: protects all internal data. */
 	spinlock_t lock;
 
 	/** @worker: dispatches incoming action messages. */
diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c
index 10556156eaadb..afd8cc7bd231c 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit.c
+++ b/drivers/gpu/drm/xe/xe_guc_submit.c
@@ -852,10 +852,27 @@ static void xe_guc_exec_queue_group_cgp_sync(struct xe_guc *guc,
 	xe_guc_ct_send(&guc->ct, action, len, G2H_LEN_DW_MULTI_QUEUE_CONTEXT, 1);
 }
 
-static void __register_exec_queue_group(struct xe_guc *guc,
-					struct xe_exec_queue *q,
+static void guc_exec_queue_send_cgp_sync(struct xe_exec_queue *q)
+{
+#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE	(2)
+	struct xe_guc *guc = exec_queue_to_guc(q);
+	struct xe_exec_queue_group *group = q->multi_queue.group;
+	u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
+	int len = 0;
+
+	action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
+	action[len++] = group->primary->guc->id;
+
+	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
+#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
+
+	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
+}
+
+static void __register_exec_queue_group(struct xe_exec_queue *q,
 					struct guc_ctxt_registration_info *info)
 {
+	struct xe_guc *guc = exec_queue_to_guc(q);
 #define MAX_MULTI_QUEUE_REG_SIZE	(8)
 	u32 action[MAX_MULTI_QUEUE_REG_SIZE];
 	int len = 0;
@@ -880,29 +897,6 @@ static void __register_exec_queue_group(struct xe_guc *guc,
 	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
 }
 
-static void xe_guc_exec_queue_group_add(struct xe_guc *guc,
-					struct xe_exec_queue *q)
-{
-#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE  (2)
-	u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
-	int len = 0;
-
-	xe_gt_assert(guc_to_gt(guc), xe_exec_queue_is_multi_queue_secondary(q));
-
-	action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
-	action[len++] = q->multi_queue.group->primary->guc->id;
-
-	xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
-#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
-
-	/*
-	 * The above XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC do expect a
-	 * XE_GUC_ACTION_NOTIFY_MULTI_QUEUE_CONTEXT_CGP_SYNC_DONE response
-	 * from guc.
-	 */
-	xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
-}
-
 static void __register_mlrc_exec_queue(struct xe_guc *guc,
 				       struct xe_exec_queue *q,
 				       struct guc_ctxt_registration_info *info)
@@ -1028,7 +1022,7 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type)
 	set_exec_queue_registered(q);
 	trace_xe_exec_queue_register(q);
 	if (xe_exec_queue_is_multi_queue_primary(q))
-		__register_exec_queue_group(guc, q, &info);
+		__register_exec_queue_group(q, &info);
 	else if (xe_exec_queue_is_parallel(q))
 		__register_mlrc_exec_queue(guc, q, &info);
 	else if (!xe_exec_queue_is_multi_queue_secondary(q))
@@ -1038,7 +1032,7 @@ static void register_exec_queue(struct xe_exec_queue *q, int ctx_type)
 		init_policies(guc, q);
 
 	if (xe_exec_queue_is_multi_queue_secondary(q))
-		xe_guc_exec_queue_group_add(guc, q);
+		guc_exec_queue_send_cgp_sync(q);
 }
 
 static u32 wq_space_until_wrap(struct xe_exec_queue *q)
@@ -1216,10 +1210,8 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job)
 		if (xe_exec_queue_is_multi_queue_secondary(q)) {
 			struct xe_exec_queue *primary = xe_exec_queue_multi_queue_primary(q);
 
-			if (exec_queue_killed_or_banned_or_wedged(primary)) {
-				killed_or_banned_or_wedged = true;
+			if (exec_queue_killed_or_banned_or_wedged(primary))
 				goto run_job_out;
-			}
 
 			if (!exec_queue_registered(primary))
 				register_exec_queue(primary, GUC_CONTEXT_NORMAL);
@@ -1673,6 +1665,14 @@ static void guc_exec_queue_fini(struct xe_exec_queue *q)
 	struct xe_guc_exec_queue *ge = q->guc;
 	struct xe_guc *guc = exec_queue_to_guc(q);
 
+	if (xe_exec_queue_is_multi_queue_secondary(q)) {
+		struct xe_exec_queue_group *group = q->multi_queue.group;
+
+		mutex_lock(&group->list_lock);
+		list_del(&q->multi_queue.link);
+		mutex_unlock(&group->list_lock);
+	}
+
 	release_guc_id(guc, q);
 	xe_sched_entity_fini(&ge->entity);
 	xe_sched_fini(&ge->sched);
@@ -1694,14 +1694,6 @@ static void __guc_exec_queue_destroy_async(struct work_struct *w)
 	guard(xe_pm_runtime)(guc_to_xe(guc));
 	trace_xe_exec_queue_destroy(q);
 
-	if (xe_exec_queue_is_multi_queue_secondary(q)) {
-		struct xe_exec_queue_group *group = q->multi_queue.group;
-
-		mutex_lock(&group->list_lock);
-		list_del(&q->multi_queue.link);
-		mutex_unlock(&group->list_lock);
-	}
-
 	/* Confirm no work left behind accessing device structures */
 	cancel_delayed_work_sync(&ge->sched.base.work_tdr);
 
@@ -1889,21 +1881,8 @@ static void __guc_exec_queue_process_msg_set_multi_queue_priority(struct xe_sche
 {
 	struct xe_exec_queue *q = msg->private_data;
 
-	if (guc_exec_queue_allowed_to_change_state(q)) {
-#define MAX_MULTI_QUEUE_CGP_SYNC_SIZE        (2)
-		struct xe_guc *guc = exec_queue_to_guc(q);
-		struct xe_exec_queue_group *group = q->multi_queue.group;
-		u32 action[MAX_MULTI_QUEUE_CGP_SYNC_SIZE];
-		int len = 0;
-
-		action[len++] = XE_GUC_ACTION_MULTI_QUEUE_CONTEXT_CGP_SYNC;
-		action[len++] = group->primary->guc->id;
-
-		xe_gt_assert(guc_to_gt(guc), len <= MAX_MULTI_QUEUE_CGP_SYNC_SIZE);
-#undef MAX_MULTI_QUEUE_CGP_SYNC_SIZE
-
-		xe_guc_exec_queue_group_cgp_sync(guc, q, action, len);
-	}
+	if (guc_exec_queue_allowed_to_change_state(q))
+		guc_exec_queue_send_cgp_sync(q);
 
 	kfree(msg);
 }
@@ -2968,9 +2947,10 @@ int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len)
 	if (unlikely(!q))
 		return -EPROTO;
 
-	xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x",
-		   xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id,
-		   atomic_read(&q->guc->state));
+	if (!exec_queue_killed(q))
+		xe_gt_info(gt, "Engine reset: engine_class=%s, logical_mask: 0x%x, guc_id=%d, state=0x%0x",
+			   xe_hw_engine_class_to_str(q->class), q->logical_mask, guc_id,
+			   atomic_read(&q->guc->state));
 
 	trace_xe_exec_queue_reset(q);
 
diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h
index 5ccc5f959bb3a..7824f61b12906 100644
--- a/drivers/gpu/drm/xe/xe_guc_submit_types.h
+++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h
@@ -138,7 +138,7 @@ struct xe_guc_submit_exec_queue_snapshot {
 		u32 primary;
 		/** @multi_queue.pos: Position of the exec queue within the multi queue group */
 		u8 pos;
-		/** @valid: The exec queue is part of a multi queue group */
+		/** @multi_queue.valid: The exec queue is part of a multi queue group */
 		bool valid;
 	} multi_queue;
 };
diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c
index 6dd05fac65956..8c66ff6f3d3c6 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine.c
+++ b/drivers/gpu/drm/xe/xe_hw_engine.c
@@ -67,7 +67,7 @@ static const struct engine_info engine_infos[] = {
 		.class = XE_ENGINE_CLASS_COPY,
 		.instance = 0,
 		.irq_offset = ilog2(INTR_BCS(0)),
-		.domain = XE_FW_RENDER,
+		.domain = XE_FW_GT,
 		.mmio_base = BLT_RING_BASE,
 	},
 	[XE_HW_ENGINE_BCS1] = {
@@ -75,7 +75,7 @@ static const struct engine_info engine_infos[] = {
 		.class = XE_ENGINE_CLASS_COPY,
 		.instance = 1,
 		.irq_offset = ilog2(INTR_BCS(1)),
-		.domain = XE_FW_RENDER,
+		.domain = XE_FW_GT,
 		.mmio_base = XEHPC_BCS1_RING_BASE,
 	},
 	[XE_HW_ENGINE_BCS2] = {
@@ -83,7 +83,7 @@ static const struct engine_info engine_infos[] = {
 		.class = XE_ENGINE_CLASS_COPY,
 		.instance = 2,
 		.irq_offset = ilog2(INTR_BCS(2)),
-		.domain = XE_FW_RENDER,
+		.domain = XE_FW_GT,
 		.mmio_base = XEHPC_BCS2_RING_BASE,
 	},
 	[XE_HW_ENGINE_BCS3] = {
@@ -91,7 +91,7 @@ static const struct engine_info engine_infos[] = {
 		.class = XE_ENGINE_CLASS_COPY,
 		.instance = 3,
 		.irq_offset = ilog2(INTR_BCS(3)),
-		.domain = XE_FW_RENDER,
+		.domain = XE_FW_GT,
 		.mmio_base = XEHPC_BCS3_RING_BASE,
 	},
 	[XE_HW_ENGINE_BCS4] = {
@@ -99,7 +99,7 @@ static const struct engine_info engine_infos[] = {
 		.class = XE_ENGINE_CLASS_COPY,
 		.instance = 4,
 		.irq_offset = ilog2(INTR_BCS(4)),
-		.domain = XE_FW_RENDER,
+		.domain = XE_FW_GT,
 		.mmio_base = XEHPC_BCS4_RING_BASE,
 	},
 	[XE_HW_ENGINE_BCS5] = {
@@ -107,7 +107,7 @@ static const struct engine_info engine_infos[] = {
 		.class = XE_ENGINE_CLASS_COPY,
 		.instance = 5,
 		.irq_offset = ilog2(INTR_BCS(5)),
-		.domain = XE_FW_RENDER,
+		.domain = XE_FW_GT,
 		.mmio_base = XEHPC_BCS5_RING_BASE,
 	},
 	[XE_HW_ENGINE_BCS6] = {
@@ -115,7 +115,7 @@ static const struct engine_info engine_infos[] = {
 		.class = XE_ENGINE_CLASS_COPY,
 		.instance = 6,
 		.irq_offset = ilog2(INTR_BCS(6)),
-		.domain = XE_FW_RENDER,
+		.domain = XE_FW_GT,
 		.mmio_base = XEHPC_BCS6_RING_BASE,
 	},
 	[XE_HW_ENGINE_BCS7] = {
@@ -123,7 +123,7 @@ static const struct engine_info engine_infos[] = {
 		.class = XE_ENGINE_CLASS_COPY,
 		.irq_offset = ilog2(INTR_BCS(7)),
 		.instance = 7,
-		.domain = XE_FW_RENDER,
+		.domain = XE_FW_GT,
 		.mmio_base = XEHPC_BCS7_RING_BASE,
 	},
 	[XE_HW_ENGINE_BCS8] = {
@@ -131,7 +131,7 @@ static const struct engine_info engine_infos[] = {
 		.class = XE_ENGINE_CLASS_COPY,
 		.instance = 8,
 		.irq_offset = ilog2(INTR_BCS8),
-		.domain = XE_FW_RENDER,
+		.domain = XE_FW_GT,
 		.mmio_base = XEHPC_BCS8_RING_BASE,
 	},
 
@@ -282,27 +282,6 @@ static void hw_engine_fini(void *arg)
 	hwe->gt = NULL;
 }
 
-/**
- * xe_hw_engine_mmio_write32() - Write engine register
- * @hwe: engine
- * @reg: register to write into
- * @val: desired 32-bit value to write
- *
- * This function will write val into an engine specific register.
- * Forcewake must be held by the caller.
- *
- */
-void xe_hw_engine_mmio_write32(struct xe_hw_engine *hwe,
-			       struct xe_reg reg, u32 val)
-{
-	xe_gt_assert(hwe->gt, !(reg.addr & hwe->mmio_base));
-	xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain);
-
-	reg.addr += hwe->mmio_base;
-
-	xe_mmio_write32(&hwe->gt->mmio, reg, val);
-}
-
 /**
  * xe_hw_engine_mmio_read32() - Read engine register
  * @hwe: engine
@@ -325,24 +304,8 @@ u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg)
 
 void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe)
 {
-	u32 ccs_mask =
-		xe_hw_engine_mask_per_class(hwe->gt, XE_ENGINE_CLASS_COMPUTE);
-	u32 ring_mode = REG_MASKED_FIELD_ENABLE(GFX_DISABLE_LEGACY_MODE);
-
-	if (hwe->class == XE_ENGINE_CLASS_COMPUTE && ccs_mask)
-		xe_mmio_write32(&hwe->gt->mmio, RCU_MODE,
-				REG_MASKED_FIELD_ENABLE(RCU_MODE_CCS_ENABLE));
-
-	xe_hw_engine_mmio_write32(hwe, RING_HWSTAM(0), ~0x0);
-	xe_hw_engine_mmio_write32(hwe, RING_HWS_PGA(0),
-				  xe_bo_ggtt_addr(hwe->hwsp));
-
-	if (xe_device_has_msix(gt_to_xe(hwe->gt)))
-		ring_mode |= REG_MASKED_FIELD_ENABLE(GFX_MSIX_INTERRUPT_ENABLE);
-	xe_hw_engine_mmio_write32(hwe, RING_MODE(0), ring_mode);
-	xe_hw_engine_mmio_write32(hwe, RING_MI_MODE(0),
-				  REG_MASKED_FIELD_DISABLE(STOP_RING));
-	xe_hw_engine_mmio_read32(hwe, RING_MI_MODE(0));
+	xe_mmio_write32(&hwe->gt->mmio, RING_HWS_PGA(hwe->mmio_base),
+			xe_bo_ggtt_addr(hwe->hwsp));
 }
 
 static bool xe_hw_engine_match_fixed_cslice_mode(const struct xe_device *xe,
@@ -374,8 +337,8 @@ static bool xe_rtp_cfeg_wmtp_disabled(const struct xe_device *xe,
 	return xe_mmio_read32(&hwe->gt->mmio, XEHP_FUSE4) & CFEG_WMTP_DISABLE;
 }
 
-void
-xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe)
+static void
+hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe)
 {
 	struct xe_gt *gt = hwe->gt;
 	const u8 mocs_write_idx = gt->mocs.uc_index;
@@ -390,7 +353,7 @@ xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe)
 		 * BLIT_CCTL registers are needed to be programmed to un-cached.
 		 */
 		{ XE_RTP_NAME("BLIT_CCTL_default_MOCS"),
-		  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED),
+		  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1274),
 			       ENGINE_CLASS(COPY)),
 		  XE_RTP_ACTIONS(FIELD_SET(BLIT_CCTL(0),
 				 BLIT_CCTL_DST_MOCS_MASK |
@@ -412,6 +375,17 @@ xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe)
 			     &hwe->reg_lrc, true);
 }
 
+void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe)
+{
+	struct xe_gt *gt = hwe->gt;
+	struct xe_device *xe = gt_to_xe(gt);
+
+	xe_reg_sr_init(&hwe->reg_lrc, hwe->name, xe);
+	xe_wa_process_lrc(hwe);
+	hw_engine_setup_default_lrc_state(hwe);
+	xe_tuning_process_lrc(hwe);
+}
+
 static void
 hw_engine_setup_default_state(struct xe_hw_engine *hwe)
 {
@@ -436,13 +410,23 @@ hw_engine_setup_default_state(struct xe_hw_engine *hwe)
 	struct xe_rtp_process_ctx ctx = XE_RTP_PROCESS_CTX_INITIALIZER(hwe);
 	const struct xe_rtp_entry_sr engine_entries[] = {
 		{ XE_RTP_NAME("RING_CMD_CCTL_default_MOCS"),
-		  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, XE_RTP_END_VERSION_UNDEFINED)),
+		  XE_RTP_RULES(FUNC(xe_rtp_match_always)),
 		  XE_RTP_ACTIONS(FIELD_SET(RING_CMD_CCTL(0),
 					   CMD_CCTL_WRITE_OVERRIDE_MASK |
 					   CMD_CCTL_READ_OVERRIDE_MASK,
 					   ring_cmd_cctl_val,
 					   XE_RTP_ACTION_FLAG(ENGINE_BASE)))
 		},
+		{ XE_RTP_NAME("Disable HW status page updates for interrupts"),
+		  XE_RTP_RULES(FUNC(xe_rtp_match_always)),
+		  XE_RTP_ACTIONS(SET(RING_HWSTAM(0), ~0x0,
+				     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+		},
+		{ XE_RTP_NAME("Disable engine 'legacy' mode"),
+		  XE_RTP_RULES(FUNC(xe_rtp_match_always)),
+		  XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_DISABLE_LEGACY_MODE,
+				     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+		},
 		/*
 		 * To allow the GSC engine to go idle on MTL we need to enable
 		 * idle messaging and set the hysteresis value (we use 0xA=5us
@@ -465,12 +449,22 @@ hw_engine_setup_default_state(struct xe_hw_engine *hwe)
 		  XE_RTP_ACTIONS(SET(CSFE_CHICKEN1(0), CS_PRIORITY_MEM_READ,
 				     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
 		},
+		{ XE_RTP_NAME("Enable CCS Engine(s)"),
+		  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1255, XE_RTP_END_VERSION_UNDEFINED),
+			       FUNC(xe_rtp_match_first_render_or_compute)),
+		  XE_RTP_ACTIONS(SET(RCU_MODE, RCU_MODE_CCS_ENABLE))
+		},
 		/* Use Fixed slice CCS mode */
 		{ XE_RTP_NAME("RCU_MODE_FIXED_SLICE_CCS_MODE"),
 		  XE_RTP_RULES(FUNC(xe_hw_engine_match_fixed_cslice_mode)),
 		  XE_RTP_ACTIONS(FIELD_SET(RCU_MODE, RCU_MODE_FIXED_SLICE_CCS_MODE,
 					   RCU_MODE_FIXED_SLICE_CCS_MODE))
 		},
+		{ XE_RTP_NAME("Enable MSI-X interrupt support"),
+		  XE_RTP_RULES(FUNC(xe_rtp_match_has_msix)),
+		  XE_RTP_ACTIONS(SET(GFX_MODE(0), GFX_MSIX_INTERRUPT_ENABLE,
+				     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+		},
 	};
 
 	xe_rtp_process_to_sr(&ctx, engine_entries, ARRAY_SIZE(engine_entries),
@@ -520,9 +514,14 @@ static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe,
 	hwe->class = info->class;
 	hwe->instance = info->instance;
 	hwe->mmio_base = info->mmio_base;
-	hwe->irq_offset = xe_device_has_msix(gt_to_xe(gt)) ?
-		get_msix_irq_offset(gt, info->class) :
-		info->irq_offset;
+	if (xe_device_has_msix(gt_to_xe(gt))) {
+		hwe->irq_offset = get_msix_irq_offset(gt, info->class);
+		hwe->irq_page = info->instance;
+
+	} else {
+		hwe->irq_offset = info->irq_offset;
+		hwe->irq_page = 0;
+	}
 	hwe->domain = info->domain;
 	hwe->name = info->name;
 	hwe->fence_irq = &gt->fence_irq[info->class];
@@ -1026,6 +1025,9 @@ bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe)
 	struct xe_gt *gt = hwe->gt;
 	struct xe_device *xe = gt_to_xe(gt);
 
+	if (xe_device_is_admin_only(xe))
+		return true;
+
 	if (hwe->class == XE_ENGINE_CLASS_OTHER)
 		return true;
 
diff --git a/drivers/gpu/drm/xe/xe_hw_engine.h b/drivers/gpu/drm/xe/xe_hw_engine.h
index 6b5f9fa2a5948..c3ee37f8cfc0e 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine.h
+++ b/drivers/gpu/drm/xe/xe_hw_engine.h
@@ -59,7 +59,7 @@ struct xe_hw_engine_snapshot *
 xe_hw_engine_snapshot_capture(struct xe_hw_engine *hwe, struct xe_exec_queue *q);
 void xe_hw_engine_snapshot_free(struct xe_hw_engine_snapshot *snapshot);
 void xe_hw_engine_print(struct xe_hw_engine *hwe, struct drm_printer *p);
-void xe_hw_engine_setup_default_lrc_state(struct xe_hw_engine *hwe);
+void xe_hw_engine_setup_reg_lrc(struct xe_hw_engine *hwe);
 
 bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe);
 
@@ -76,7 +76,6 @@ const char *xe_hw_engine_class_to_str(enum xe_engine_class class);
 u64 xe_hw_engine_read_timestamp(struct xe_hw_engine *hwe);
 enum xe_force_wake_domains xe_hw_engine_to_fw_domain(struct xe_hw_engine *hwe);
 
-void xe_hw_engine_mmio_write32(struct xe_hw_engine *hwe, struct xe_reg reg, u32 val);
 u32 xe_hw_engine_mmio_read32(struct xe_hw_engine *hwe, struct xe_reg reg);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_hw_engine_types.h b/drivers/gpu/drm/xe/xe_hw_engine_types.h
index e4191a7a2c318..2cf898e682f55 100644
--- a/drivers/gpu/drm/xe/xe_hw_engine_types.h
+++ b/drivers/gpu/drm/xe/xe_hw_engine_types.h
@@ -79,7 +79,7 @@ struct xe_hw_engine_class_intf {
 	 * @defaults: default scheduling properties
 	 */
 	struct {
-		/** @sched_props.set_job_timeout: Set job timeout in ms for engine */
+		/** @sched_props.job_timeout_ms: Set job timeout in ms for engine */
 		u32 job_timeout_ms;
 		/** @sched_props.job_timeout_min: Min job timeout in ms for engine */
 		u32 job_timeout_min;
@@ -118,6 +118,8 @@ struct xe_hw_engine {
 	u16 logical_instance;
 	/** @irq_offset: IRQ offset of this hw engine */
 	u16 irq_offset;
+	/** @irq_page: MEMIRQ page used by this HW engine */
+	u16 irq_page;
 	/** @mmio_base: MMIO base address of this hw engine*/
 	u32 mmio_base;
 	/**
diff --git a/drivers/gpu/drm/xe/xe_hw_error.c b/drivers/gpu/drm/xe/xe_hw_error.c
index 2a31b430570e3..5135e8e4093f7 100644
--- a/drivers/gpu/drm/xe/xe_hw_error.c
+++ b/drivers/gpu/drm/xe/xe_hw_error.c
@@ -36,11 +36,6 @@ static const char * const hec_uncorrected_fw_errors[] = {
 	"Data Corruption"
 };
 
-static const unsigned long xe_hw_error_map[] = {
-	[XE_GT_ERROR]	= DRM_XE_RAS_ERR_COMP_CORE_COMPUTE,
-	[XE_SOC_ERROR]	= DRM_XE_RAS_ERR_COMP_SOC_INTERNAL,
-};
-
 enum gt_vector_regs {
 	ERR_STAT_GT_VECTOR0 = 0,
 	ERR_STAT_GT_VECTOR1,
@@ -65,6 +60,18 @@ static enum drm_xe_ras_error_severity hw_err_to_severity(const enum hardware_err
 	return DRM_XE_RAS_ERR_SEV_UNCORRECTABLE;
 }
 
+static inline u32 err_src_to_id(u32 err_bit)
+{
+	switch (err_bit) {
+	case XE_GT_ERROR:
+		return DRM_XE_RAS_ERR_COMP_CORE_COMPUTE;
+	case XE_SOC_ERROR:
+		return DRM_XE_RAS_ERR_COMP_SOC_INTERNAL;
+	default:
+		return 0;
+	}
+}
+
 static const char * const pvc_master_global_err_reg[] = {
 	[0 ... 1]	= "Undefined",
 	[2]		= "HBM SS0: Channel0",
@@ -169,11 +176,8 @@ static void csc_hw_error_work(struct work_struct *work)
 {
 	struct xe_tile *tile = container_of(work, typeof(*tile), csc_hw_error_work);
 	struct xe_device *xe = tile_to_xe(tile);
-	int ret;
 
-	ret = xe_survivability_mode_runtime_enable(xe);
-	if (ret)
-		drm_err(&xe->drm, "Failed to enable runtime survivability mode\n");
+	xe_survivability_mode_runtime_enable(xe);
 }
 
 static void csc_hw_error_handler(struct xe_tile *tile, const enum hardware_error hw_err)
@@ -459,14 +463,8 @@ static void hw_error_source_handler(struct xe_tile *tile, const enum hardware_er
 		const char *name;
 		u32 error_id;
 
-		/* Check error bit is within bounds */
-		if (err_bit >= ARRAY_SIZE(xe_hw_error_map))
-			break;
-
-		error_id = xe_hw_error_map[err_bit];
-
-		/* Check error component is within max */
-		if (!error_id || error_id >= DRM_XE_RAS_ERR_COMP_MAX)
+		error_id = err_src_to_id(err_bit);
+		if (!error_id)
 			continue;
 
 		name = info[error_id].name;
diff --git a/drivers/gpu/drm/xe/xe_hw_error.h b/drivers/gpu/drm/xe/xe_hw_error.h
index d86e28c5180c1..5e3a114241087 100644
--- a/drivers/gpu/drm/xe/xe_hw_error.h
+++ b/drivers/gpu/drm/xe/xe_hw_error.h
@@ -2,8 +2,8 @@
 /*
  * Copyright © 2025 Intel Corporation
  */
-#ifndef XE_HW_ERROR_H_
-#define XE_HW_ERROR_H_
+#ifndef _XE_HW_ERROR_H_
+#define _XE_HW_ERROR_H_
 
 #include <linux/types.h>
 
diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c
index 92e423a339f15..de3f2aeffc3fb 100644
--- a/drivers/gpu/drm/xe/xe_hwmon.c
+++ b/drivers/gpu/drm/xe/xe_hwmon.c
@@ -180,6 +180,18 @@ struct xe_hwmon {
 	struct xe_hwmon_thermal_info temp;
 };
 
+static inline int prepare_power_limit_param2(const struct xe_hwmon *hwmon)
+{
+	if (hwmon->boot_power_limit_read) {
+		if (hwmon->xe->info.platform >= XE_CRESCENTISLAND)
+			return READ_PL_ACCEPTED;
+		else
+			return READ_PL_FROM_PCODE;
+	} else {
+		return READ_PL_FROM_FW;
+	}
+}
+
 static int xe_hwmon_pcode_read_power_limit(const struct xe_hwmon *hwmon, u32 attr, int channel,
 					   u32 *uval)
 {
@@ -191,9 +203,7 @@ static int xe_hwmon_pcode_read_power_limit(const struct xe_hwmon *hwmon, u32 att
 						  (channel == CHANNEL_CARD) ?
 						  READ_PSYSGPU_POWER_LIMIT :
 						  READ_PACKAGE_POWER_LIMIT,
-						  hwmon->boot_power_limit_read ?
-						  READ_PL_FROM_PCODE : READ_PL_FROM_FW),
-						  &val0, &val1);
+						  prepare_power_limit_param2(hwmon)), &val0, &val1);
 
 	if (ret) {
 		drm_dbg(&hwmon->xe->drm, "read failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n",
@@ -226,10 +236,7 @@ static int xe_hwmon_pcode_rmw_power_limit(const struct xe_hwmon *hwmon, u32 attr
 						  (channel == CHANNEL_CARD) ?
 						  READ_PSYSGPU_POWER_LIMIT :
 						  READ_PACKAGE_POWER_LIMIT,
-						  hwmon->boot_power_limit_read ?
-						  READ_PL_FROM_PCODE : READ_PL_FROM_FW),
-						  &val0, &val1);
-
+						  prepare_power_limit_param2(hwmon)), &val0, &val1);
 	if (ret)
 		drm_dbg(&hwmon->xe->drm, "read failed ch %d val0 0x%08x, val1 0x%08x, ret %d\n",
 			channel, val0, val1, ret);
@@ -296,7 +303,12 @@ static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg
 			return GT_PERF_STATUS;
 		break;
 	case REG_PKG_ENERGY_STATUS:
-		if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG) {
+		if (xe->info.platform == XE_CRESCENTISLAND) {
+			if (channel == CHANNEL_CARD)
+				return CRI_PLATFORM_ENERGY_STATUS;
+			else if (channel == CHANNEL_PKG)
+				return CRI_PACKAGE_ENERGY_STATUS;
+		} else if (xe->info.platform == XE_PVC && channel == CHANNEL_PKG) {
 			return PVC_GT0_PLATFORM_ENERGY_STATUS;
 		} else if ((xe->info.platform == XE_DG2) && (channel == CHANNEL_PKG)) {
 			return PCU_CR_PACKAGE_ENERGY_STATUS;
diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c
index 7560a45f7f64e..9e49e2241da41 100644
--- a/drivers/gpu/drm/xe/xe_irq.c
+++ b/drivers/gpu/drm/xe/xe_irq.c
@@ -24,6 +24,7 @@
 #include "xe_mmio.h"
 #include "xe_pxp.h"
 #include "xe_sriov.h"
+#include "xe_sysctrl.h"
 #include "xe_tile.h"
 
 /*
@@ -525,6 +526,7 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg)
 				xe_heci_csc_irq_handler(xe, master_ctl);
 			xe_display_irq_handler(xe, master_ctl);
 			xe_i2c_irq_handler(xe, master_ctl);
+			xe_sysctrl_irq_handler(xe, master_ctl);
 			xe_mert_irq_handler(xe, master_ctl);
 			gu_misc_iir = gu_misc_irq_ack(xe, master_ctl);
 		}
diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c
index 4af9f0d7c6f3b..a4292a11391d3 100644
--- a/drivers/gpu/drm/xe/xe_lrc.c
+++ b/drivers/gpu/drm/xe/xe_lrc.c
@@ -21,8 +21,10 @@
 #include "xe_configfs.h"
 #include "xe_device.h"
 #include "xe_drm_client.h"
+#include "xe_exec_queue.h"
 #include "xe_exec_queue_types.h"
 #include "xe_gt.h"
+#include "xe_gt_clock.h"
 #include "xe_gt_printk.h"
 #include "xe_hw_fence.h"
 #include "xe_map.h"
@@ -682,25 +684,6 @@ static void set_memory_based_intr(u32 *regs, struct xe_hw_engine *hwe)
 	}
 }
 
-static int lrc_ring_mi_mode(struct xe_hw_engine *hwe)
-{
-	struct xe_device *xe = gt_to_xe(hwe->gt);
-
-	if (GRAPHICS_VERx100(xe) >= 1250)
-		return 0x70;
-	else
-		return 0x60;
-}
-
-static void reset_stop_ring(u32 *regs, struct xe_hw_engine *hwe)
-{
-	int x;
-
-	x = lrc_ring_mi_mode(hwe);
-	regs[x + 1] &= ~STOP_RING;
-	regs[x + 1] |= STOP_RING << 16;
-}
-
 static inline bool xe_lrc_has_indirect_ring_state(struct xe_lrc *lrc)
 {
 	return lrc->flags & XE_LRC_FLAG_INDIRECT_RING_STATE;
@@ -795,6 +778,16 @@ static u32 __xe_lrc_ctx_timestamp_udw_offset(struct xe_lrc *lrc)
 	return __xe_lrc_regs_offset(lrc) + CTX_TIMESTAMP_UDW * sizeof(u32);
 }
 
+static u32 __xe_lrc_queue_timestamp_offset(struct xe_lrc *lrc)
+{
+	return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP * sizeof(u32);
+}
+
+static u32 __xe_lrc_queue_timestamp_udw_offset(struct xe_lrc *lrc)
+{
+	return __xe_lrc_regs_offset(lrc) + CTX_QUEUE_TIMESTAMP_UDW * sizeof(u32);
+}
+
 static inline u32 __xe_lrc_indirect_ring_offset(struct xe_lrc *lrc)
 {
 	u32 offset = xe_bo_size(lrc->bo) - LRC_WA_BB_SIZE -
@@ -844,6 +837,8 @@ DECL_MAP_ADDR_HELPERS(ctx_timestamp_udw, lrc->bo)
 DECL_MAP_ADDR_HELPERS(parallel, lrc->bo)
 DECL_MAP_ADDR_HELPERS(indirect_ring, lrc->bo)
 DECL_MAP_ADDR_HELPERS(engine_id, lrc->bo)
+DECL_MAP_ADDR_HELPERS(queue_timestamp, lrc->bo)
+DECL_MAP_ADDR_HELPERS(queue_timestamp_udw, lrc->bo)
 
 #undef DECL_MAP_ADDR_HELPERS
 
@@ -892,6 +887,29 @@ static u64 xe_lrc_ctx_timestamp(struct xe_lrc *lrc)
 	return (u64)udw << 32 | ldw;
 }
 
+/**
+ * xe_lrc_queue_timestamp() - Read queue timestamp value
+ * @lrc: Pointer to the lrc.
+ *
+ * Returns: queue timestamp value
+ */
+static u64 xe_lrc_queue_timestamp(struct xe_lrc *lrc)
+{
+	struct xe_device *xe = lrc_to_xe(lrc);
+	struct iosys_map map;
+	u32 ldw, udw = 0;
+
+	xe_assert(xe, xe_lrc_is_multi_queue(lrc));
+
+	map = __xe_lrc_queue_timestamp_map(lrc);
+	ldw = xe_map_read32(xe, &map);
+
+	map = __xe_lrc_queue_timestamp_udw_map(lrc);
+	udw = xe_map_read32(xe, &map);
+
+	return (u64)udw << 32 | ldw;
+}
+
 /**
  * xe_lrc_ctx_job_timestamp_ggtt_addr() - Get ctx job timestamp GGTT address
  * @lrc: Pointer to the lrc.
@@ -987,7 +1005,6 @@ static void *empty_lrc_data(struct xe_hw_engine *hwe)
 	set_offsets(regs, reg_offsets(gt_to_xe(gt), hwe->class), hwe);
 	set_context_control(regs, hwe);
 	set_memory_based_intr(regs, hwe);
-	reset_stop_ring(regs, hwe);
 	if (xe_gt_has_indirect_ring_state(gt)) {
 		regs = data + xe_gt_lrc_size(gt, hwe->class) -
 		       LRC_INDIRECT_RING_STATE_SIZE;
@@ -1557,6 +1574,18 @@ static int xe_lrc_ctx_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct
 	if (lrc_to_xe(lrc)->info.has_64bit_timestamp)
 		xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP_UDW, 0);
 
+	/*
+	 * Note: It's possible that this LRC may belong to an exec_queue that is
+	 * not part of a multi-queue group. That said, it doesn't hurt to set
+	 * this field anyways since any class that supports multi-queue will
+	 * have these LRC fields defined.
+	 */
+	if (xe_gt_supports_multi_queue(gt, hwe->class)) {
+		lrc->queue_timestamp = 0;
+		xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP, 0);
+		xe_lrc_write_ctx_reg(lrc, CTX_QUEUE_TIMESTAMP_UDW, 0);
+	}
+
 	if (xe->info.has_asid && vm)
 		xe_lrc_write_ctx_reg(lrc, CTX_ASID, vm->usm.asid);
 
@@ -2482,7 +2511,17 @@ struct xe_lrc_snapshot *xe_lrc_snapshot_capture(struct xe_lrc *lrc)
 	snapshot->replay_offset = 0;
 	snapshot->replay_size = lrc->replay_size;
 	snapshot->lrc_snapshot = NULL;
-	snapshot->ctx_timestamp = lower_32_bits(xe_lrc_ctx_timestamp(lrc));
+	snapshot->ctx_timestamp = xe_lrc_ctx_timestamp(lrc);
+	snapshot->ctx_timestamp_ms =
+		xe_gt_clock_interval_to_ms(lrc->gt, xe_lrc_ctx_timestamp(lrc));
+	if (xe_lrc_is_multi_queue(lrc)) {
+		snapshot->queue_timestamp = xe_lrc_queue_timestamp(lrc);
+		snapshot->queue_timestamp_ms =
+			xe_gt_clock_interval_to_ms(lrc->gt, snapshot->queue_timestamp);
+	} else {
+		snapshot->queue_timestamp = 0;
+		snapshot->queue_timestamp_ms = 0;
+	}
 	snapshot->ctx_job_timestamp = xe_lrc_ctx_job_timestamp(lrc);
 	return snapshot;
 }
@@ -2535,7 +2574,10 @@ void xe_lrc_snapshot_print(struct xe_lrc_snapshot *snapshot, struct drm_printer
 	drm_printf(p, "\tRing start: (memory) 0x%08x\n", snapshot->start);
 	drm_printf(p, "\tStart seqno: (memory) %d\n", snapshot->start_seqno);
 	drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->seqno);
-	drm_printf(p, "\tTimestamp: 0x%08x\n", snapshot->ctx_timestamp);
+	drm_printf(p, "\tTimestamp: 0x%016llx\n", snapshot->ctx_timestamp);
+	drm_printf(p, "\tTimestamp ms: %llu\n", snapshot->ctx_timestamp_ms);
+	drm_printf(p, "\tQueue Timestamp: 0x%016llx\n", snapshot->queue_timestamp);
+	drm_printf(p, "\tQueue Timestamp ms: %llu\n", snapshot->queue_timestamp_ms);
 	drm_printf(p, "\tJob Timestamp: 0x%08x\n", snapshot->ctx_job_timestamp);
 
 	if (!snapshot->lrc_snapshot)
@@ -2576,17 +2618,27 @@ void xe_lrc_snapshot_free(struct xe_lrc_snapshot *snapshot)
 	kfree(snapshot);
 }
 
-static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts)
+static struct xe_hw_engine *engine_id_to_hwe(struct xe_gt *gt, u32 engine_id)
 {
 	u16 class = REG_FIELD_GET(ENGINE_CLASS_ID, engine_id);
 	u16 instance = REG_FIELD_GET(ENGINE_INSTANCE_ID, engine_id);
+	struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, class, instance, false);
+
+	if (xe_gt_WARN_ONCE(gt, !hwe || xe_hw_engine_is_reserved(hwe),
+			    "Unexpected engine class:instance %d:%d for utilization\n",
+			    class, instance))
+		return NULL;
+
+	return hwe;
+}
+
+static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts)
+{
 	struct xe_hw_engine *hwe;
 	u64 val;
 
-	hwe = xe_gt_hw_engine(lrc->gt, class, instance, false);
-	if (xe_gt_WARN_ONCE(lrc->gt, !hwe || xe_hw_engine_is_reserved(hwe),
-			    "Unexpected engine class:instance %d:%d for context utilization\n",
-			    class, instance))
+	hwe = engine_id_to_hwe(lrc->gt, engine_id);
+	if (!hwe)
 		return -1;
 
 	if (lrc_to_xe(lrc)->info.has_64bit_timestamp)
@@ -2601,66 +2653,136 @@ static int get_ctx_timestamp(struct xe_lrc *lrc, u32 engine_id, u64 *reg_ctx_ts)
 	return 0;
 }
 
-/**
- * xe_lrc_timestamp() - Current ctx timestamp
- * @lrc: Pointer to the lrc.
- *
- * Return latest ctx timestamp. With support for active contexts, the
- * calculation may be slightly racy, so follow a read-again logic to ensure that
- * the context is still active before returning the right timestamp.
- *
- * Returns: New ctx timestamp value
- */
-u64 xe_lrc_timestamp(struct xe_lrc *lrc)
+static u64 get_queue_timestamp(struct xe_hw_engine *hwe)
 {
-	u64 lrc_ts, reg_ts, new_ts = lrc->ctx_timestamp;
-	u32 engine_id;
+	return xe_mmio_read64_2x32(&hwe->gt->mmio,
+				   RING_QUEUE_TIMESTAMP(hwe->mmio_base));
+}
 
-	lrc_ts = xe_lrc_ctx_timestamp(lrc);
-	/* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */
-	if (IS_SRIOV_VF(lrc_to_xe(lrc))) {
-		new_ts = lrc_ts;
-		goto done;
-	}
+static u32 get_multi_queue_active_queue_id(struct xe_hw_engine *hwe)
+{
+	u32 val = xe_mmio_read32(&hwe->gt->mmio,
+				 RING_CSMQDEBUG(hwe->mmio_base));
 
-	if (lrc_ts == CONTEXT_ACTIVE) {
-		engine_id = xe_lrc_engine_id(lrc);
-		if (!get_ctx_timestamp(lrc, engine_id, &reg_ts))
-			new_ts = reg_ts;
+	return REG_FIELD_GET(CURRENT_ACTIVE_QUEUE_ID_MASK, val);
+}
 
-		/* read lrc again to ensure context is still active */
-		lrc_ts = xe_lrc_ctx_timestamp(lrc);
-	}
+static bool context_active(struct xe_lrc *lrc)
+{
+	return xe_lrc_ctx_timestamp(lrc) == CONTEXT_ACTIVE;
+}
+
+static u64 xe_lrc_multi_queue_timestamp(struct xe_lrc *lrc)
+{
+	struct xe_device *xe = lrc_to_xe(lrc);
+	struct xe_lrc *primary_lrc = lrc->multi_queue.primary_lrc;
+	struct xe_hw_engine *hwe;
+	u64 reg_queue_ts = lrc->queue_timestamp;
+
+	if (IS_SRIOV_VF(xe))
+		return xe_lrc_queue_timestamp(lrc);
+
+	xe_assert(xe, primary_lrc);
+
+	/* WA BB populates CONTEXT_ACTIVE cookie for primary context only */
+	if (!context_active(primary_lrc))
+		return xe_lrc_queue_timestamp(lrc);
+
+	/* WA BB populates engine id in PPHWSP of primary context only */
+	hwe = engine_id_to_hwe(primary_lrc->gt, xe_lrc_engine_id(primary_lrc));
+	if (!hwe)
+		return xe_lrc_queue_timestamp(lrc);
+
+	if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos)
+		return xe_lrc_queue_timestamp(lrc);
+
+	/* queue is active, so store the queue timestamp register */
+	reg_queue_ts = get_queue_timestamp(hwe);
+
+	/* double check queue and primary queue are both still active */
+	if (get_multi_queue_active_queue_id(hwe) != lrc->multi_queue.pos ||
+	    !context_active(primary_lrc))
+		return xe_lrc_queue_timestamp(lrc);
+
+	return reg_queue_ts;
+}
+
+static u64 xe_lrc_update_multi_queue_timestamp(struct xe_lrc *lrc, u64 *old_ts)
+{
+	*old_ts = lrc->queue_timestamp;
+	lrc->queue_timestamp = xe_lrc_multi_queue_timestamp(lrc);
+
+	trace_xe_lrc_update_queue_timestamp(lrc, *old_ts);
+
+	return lrc->queue_timestamp;
+}
+
+static u64 xe_lrc_context_timestamp(struct xe_lrc *lrc)
+{
+	u64 reg_ts, new_ts = lrc->ctx_timestamp;
+
+	/* CTX_TIMESTAMP mmio read is invalid on VF, so return the LRC value */
+	if (IS_SRIOV_VF(lrc_to_xe(lrc)))
+		return xe_lrc_ctx_timestamp(lrc);
+
+	if (context_active(lrc) &&
+	    !get_ctx_timestamp(lrc, xe_lrc_engine_id(lrc), &reg_ts))
+		new_ts = reg_ts;
 
 	/*
-	 * If context switched out, just use the lrc_ts. Note that this needs to
-	 * be a separate if condition.
+	 * If context swicthed out while we were here, just return the latest
+	 * LRC CTX TIMESTAMP value.
 	 */
-	if (lrc_ts != CONTEXT_ACTIVE)
-		new_ts = lrc_ts;
+	if (!context_active(lrc))
+		return xe_lrc_ctx_timestamp(lrc);
 
-done:
 	return new_ts;
 }
 
+static u64 xe_lrc_update_context_timestamp(struct xe_lrc *lrc, u64 *old_ts)
+{
+	*old_ts = lrc->ctx_timestamp;
+	lrc->ctx_timestamp = xe_lrc_context_timestamp(lrc);
+
+	trace_xe_lrc_update_timestamp(lrc, *old_ts);
+
+	return lrc->ctx_timestamp;
+}
+
+/**
+ * xe_lrc_timestamp() - Current lrc timestamp
+ * @lrc: Pointer to the lrc.
+ *
+ * Return latest lrc timestamp. With support for active contexts/queues, the
+ * calculation may be slightly racy, so follow a read-again logic to ensure that
+ * the context/queue is still active before returning the right timestamp.
+ *
+ * Returns: New lrc timestamp value
+ */
+u64 xe_lrc_timestamp(struct xe_lrc *lrc)
+{
+	if (xe_lrc_is_multi_queue(lrc))
+		return xe_lrc_multi_queue_timestamp(lrc);
+	else
+		return xe_lrc_context_timestamp(lrc);
+}
+
 /**
- * xe_lrc_update_timestamp() - Update ctx timestamp
+ * xe_lrc_update_timestamp() - Update lrc timestamp
  * @lrc: Pointer to the lrc.
  * @old_ts: Old timestamp value
  *
- * Populate @old_ts current saved ctx timestamp, read new ctx timestamp and
+ * Populate @old_ts with current saved lrc timestamp, read new lrc timestamp and
  * update saved value.
  *
- * Returns: New ctx timestamp value
+ * Returns: New lrc timestamp value
  */
 u64 xe_lrc_update_timestamp(struct xe_lrc *lrc, u64 *old_ts)
 {
-	*old_ts = lrc->ctx_timestamp;
-	lrc->ctx_timestamp = xe_lrc_timestamp(lrc);
-
-	trace_xe_lrc_update_timestamp(lrc, *old_ts);
-
-	return lrc->ctx_timestamp;
+	if (xe_lrc_is_multi_queue(lrc))
+		return xe_lrc_update_multi_queue_timestamp(lrc, old_ts);
+	else
+		return xe_lrc_update_context_timestamp(lrc, old_ts);
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h
index 5440663183f6d..0a3a611391ee0 100644
--- a/drivers/gpu/drm/xe/xe_lrc.h
+++ b/drivers/gpu/drm/xe/xe_lrc.h
@@ -37,7 +37,10 @@ struct xe_lrc_snapshot {
 	} tail;
 	u32 start_seqno;
 	u32 seqno;
-	u32 ctx_timestamp;
+	u64 ctx_timestamp;
+	u64 ctx_timestamp_ms;
+	u64 queue_timestamp;
+	u64 queue_timestamp_ms;
 	u32 ctx_job_timestamp;
 };
 
@@ -90,6 +93,11 @@ static inline size_t xe_lrc_ring_size(void)
 	return SZ_16K;
 }
 
+static inline bool xe_lrc_is_multi_queue(struct xe_lrc *lrc)
+{
+	return lrc->multi_queue.primary_lrc;
+}
+
 size_t xe_gt_lrc_hang_replay_size(struct xe_gt *gt, enum xe_engine_class class);
 size_t xe_gt_lrc_size(struct xe_gt *gt, enum xe_engine_class class);
 u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc);
diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h
index 5a718f759ed69..53ef48feebfc9 100644
--- a/drivers/gpu/drm/xe/xe_lrc_types.h
+++ b/drivers/gpu/drm/xe/xe_lrc_types.h
@@ -63,6 +63,17 @@ struct xe_lrc {
 
 	/** @ctx_timestamp: readout value of CTX_TIMESTAMP on last update */
 	u64 ctx_timestamp;
+
+	/** @queue_timestamp: value of QUEUE_TIMESTAMP on last update */
+	u64 queue_timestamp;
+
+	/** @multi_queue: Multi queue LRC related information */
+	struct {
+		/** @multi_queue.primary_lrc: Primary lrc of this multi-queue group*/
+		struct xe_lrc *primary_lrc;
+		/** @multi_queue.pos: Position of LRC within the multi-queue group */
+		u8 pos;
+	} multi_queue;
 };
 
 struct xe_lrc_snapshot;
diff --git a/drivers/gpu/drm/xe/xe_memirq.c b/drivers/gpu/drm/xe/xe_memirq.c
index 811e07136efbf..9dfe965cb46ec 100644
--- a/drivers/gpu/drm/xe/xe_memirq.c
+++ b/drivers/gpu/drm/xe/xe_memirq.c
@@ -152,6 +152,16 @@ static const char *guc_name(struct xe_guc *guc)
  *
  */
 
+/* ISR */
+#define XE_MEMIRQ_STATUS_OFFSET(inst)	((inst) * SZ_4K + 0x0)
+/* IIR */
+#define XE_MEMIRQ_SOURCE_OFFSET(inst)	((inst) * SZ_4K + 0x400)
+/* IMR */
+#define XE_MEMIRQ_ENABLE_OFFSET		0x440
+
+/* engine ISR vector offset */
+#define XE_MEMIRQ_VECTOR_OFFSET(page, source)	(XE_MEMIRQ_STATUS_OFFSET(page) + (source) * SZ_16)
+
 static inline bool hw_reports_to_instance_zero(struct xe_memirq *memirq)
 {
 	/*
@@ -162,18 +172,35 @@ static inline bool hw_reports_to_instance_zero(struct xe_memirq *memirq)
 	return xe_device_has_msix(memirq_to_xe(memirq));
 }
 
+static unsigned int hwe_max_count(struct xe_tile *tile)
+{
+	unsigned int max_instance = 0;
+	unsigned int gtid, hweid;
+	struct xe_hw_engine *hwe;
+	struct xe_gt *gt;
+
+	for_each_gt_on_tile(gt, tile, gtid)
+		for_each_hw_engine(hwe, gt, hweid)
+			max_instance = max(max_instance, hwe->instance);
+
+	return max_instance + 1;
+}
+
 static int memirq_alloc_pages(struct xe_memirq *memirq)
 {
 	struct xe_device *xe = memirq_to_xe(memirq);
 	struct xe_tile *tile = memirq_to_tile(memirq);
-	size_t bo_size = hw_reports_to_instance_zero(memirq) ?
-		XE_HW_ENGINE_MAX_INSTANCE * SZ_4K : SZ_4K;
+	unsigned int num_pages;
 	struct xe_bo *bo;
+	size_t bo_size;
 	int err;
 
 	BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_SOURCE_OFFSET(0), SZ_64));
 	BUILD_BUG_ON(!IS_ALIGNED(XE_MEMIRQ_STATUS_OFFSET(0), SZ_4K));
 
+	num_pages = hw_reports_to_instance_zero(memirq) ? hwe_max_count(tile) : 1;
+	bo_size = num_pages * SZ_4K;
+
 	bo = xe_managed_bo_create_pin_map(xe, tile, bo_size,
 					  XE_BO_FLAG_SYSTEM |
 					  XE_BO_FLAG_GGTT |
@@ -192,16 +219,15 @@ static int memirq_alloc_pages(struct xe_memirq *memirq)
 
 	memirq->bo = bo;
 	memirq->source = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_SOURCE_OFFSET(0));
-	memirq->status = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_STATUS_OFFSET(0));
-	memirq->mask = IOSYS_MAP_INIT_OFFSET(&bo->vmap, XE_MEMIRQ_ENABLE_OFFSET);
+	memirq->num_pages = num_pages;
 
 	memirq_assert(memirq, !memirq->source.is_iomem);
-	memirq_assert(memirq, !memirq->status.is_iomem);
-	memirq_assert(memirq, !memirq->mask.is_iomem);
 
-	memirq_debug(memirq, "page offsets: bo %#x bo_size %zu source %#x status %#x\n",
-		     xe_bo_ggtt_addr(bo), bo_size, XE_MEMIRQ_SOURCE_OFFSET(0),
-		     XE_MEMIRQ_STATUS_OFFSET(0));
+	memirq_debug(memirq, "pages: count %u size %zu\n", num_pages, bo_size);
+	memirq_debug(memirq, "page0: source %#x status %#x mask %#x\n",
+		     xe_bo_ggtt_addr(bo) + XE_MEMIRQ_SOURCE_OFFSET(0),
+		     xe_bo_ggtt_addr(bo) + XE_MEMIRQ_STATUS_OFFSET(0),
+		     xe_bo_ggtt_addr(bo) + XE_MEMIRQ_ENABLE_OFFSET);
 
 	return 0;
 
@@ -212,7 +238,12 @@ static int memirq_alloc_pages(struct xe_memirq *memirq)
 
 static void memirq_set_enable(struct xe_memirq *memirq, bool enable)
 {
-	iosys_map_wr(&memirq->mask, 0, u32, enable ? GENMASK(15, 0) : 0);
+	/*
+	 * We only care about the GT_MI_USER_INTERRUPT from the engines and
+	 * the GuC does not look at the ENABLE mask at all.
+	 */
+	iosys_map_wr(&memirq->bo->vmap, XE_MEMIRQ_ENABLE_OFFSET, u32,
+		     enable ? GT_MI_USER_INTERRUPT : 0);
 
 	memirq->enabled = enable;
 }
@@ -249,15 +280,6 @@ int xe_memirq_init(struct xe_memirq *memirq)
 	return 0;
 }
 
-static u32 __memirq_source_page(struct xe_memirq *memirq, u16 instance)
-{
-	memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE);
-	memirq_assert(memirq, memirq->bo);
-
-	instance = hw_reports_to_instance_zero(memirq) ? instance : 0;
-	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(instance);
-}
-
 /**
  * xe_memirq_source_ptr - Get GGTT's offset of the `Interrupt Source Report Page`_.
  * @memirq: the &xe_memirq to query
@@ -272,16 +294,7 @@ u32 xe_memirq_source_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
 {
 	memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
 
-	return __memirq_source_page(memirq, hwe->instance);
-}
-
-static u32 __memirq_status_page(struct xe_memirq *memirq, u16 instance)
-{
-	memirq_assert(memirq, instance <= XE_HW_ENGINE_MAX_INSTANCE);
-	memirq_assert(memirq, memirq->bo);
-
-	instance = hw_reports_to_instance_zero(memirq) ? instance : 0;
-	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(instance);
+	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(hwe->irq_page);
 }
 
 /**
@@ -298,7 +311,7 @@ u32 xe_memirq_status_ptr(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
 {
 	memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
 
-	return __memirq_status_page(memirq, hwe->instance);
+	return xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_STATUS_OFFSET(hwe->irq_page);
 }
 
 /**
@@ -335,13 +348,14 @@ int xe_memirq_init_guc(struct xe_memirq *memirq, struct xe_guc *guc)
 {
 	bool is_media = xe_gt_is_media_type(guc_to_gt(guc));
 	u32 offset = is_media ? ilog2(INTR_MGUC) : ilog2(INTR_GUC);
-	u32 source, status;
+	u64 source, status;
 	int err;
 
 	memirq_assert(memirq, xe_device_uses_memirq(memirq_to_xe(memirq)));
 
-	source = __memirq_source_page(memirq, 0) + offset;
-	status = __memirq_status_page(memirq, 0) + offset * SZ_16;
+	/* GuC expects exact locations, it doesn't add anything on its own */
+	source = xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_SOURCE_OFFSET(0) + offset;
+	status = xe_bo_ggtt_addr(memirq->bo) + XE_MEMIRQ_VECTOR_OFFSET(0, offset);
 
 	err = xe_guc_self_cfg64(guc, GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY,
 				source);
@@ -427,13 +441,25 @@ static bool memirq_received(struct xe_memirq *memirq, struct iosys_map *vector,
 	return __memirq_received(memirq, vector, offset, name, true);
 }
 
+static void memirq_assume_received(struct xe_memirq *memirq, const char *source,
+				   u16 offset, const char *status)
+{
+	memirq_debug(memirq, "ASSUME %s %s(%u)\n", source, status, offset);
+}
+
 static void memirq_dispatch_engine(struct xe_memirq *memirq, struct iosys_map *status,
 				   struct xe_hw_engine *hwe)
 {
 	memirq_debug(memirq, "STATUS %s %*ph\n", hwe->name, 16, status->vaddr);
 
-	if (memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name))
-		xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT);
+	/*
+	 * The programming note says to assume that GT_MI_USER_INTERRUPT is always
+	 * set. Check and clear related status byte just for a debug.
+	 */
+	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) &&
+	    !memirq_received(memirq, status, ilog2(GT_MI_USER_INTERRUPT), hwe->name))
+		memirq_assume_received(memirq, hwe->name, ilog2(GT_MI_USER_INTERRUPT), "USER");
+	xe_hw_engine_handle_irq(hwe, GT_MI_USER_INTERRUPT);
 }
 
 static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *status,
@@ -443,8 +469,14 @@ static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *stat
 
 	memirq_debug(memirq, "STATUS %s %*ph\n", name, 16, status->vaddr);
 
-	if (memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name))
-		xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST);
+	/*
+	 * The programming note says to assume that GUC_INTR_GUC2HOST is always
+	 * set. Check and clear related status byte just for a debug.
+	 */
+	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ) &&
+	    !memirq_received(memirq, status, ilog2(GUC_INTR_GUC2HOST), name))
+		memirq_assume_received(memirq, name, ilog2(GUC_INTR_GUC2HOST), "GUC2HOST");
+	xe_guc_irq_handler(guc, GUC_INTR_GUC2HOST);
 
 	/*
 	 * This is a software interrupt that must be cleared after it's consumed
@@ -467,16 +499,16 @@ static void memirq_dispatch_guc(struct xe_memirq *memirq, struct iosys_map *stat
  */
 void xe_memirq_hwe_handler(struct xe_memirq *memirq, struct xe_hw_engine *hwe)
 {
-	u16 offset = hwe->irq_offset;
-	u16 instance = hw_reports_to_instance_zero(memirq) ? hwe->instance : 0;
-	struct iosys_map src_offset = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
-							    XE_MEMIRQ_SOURCE_OFFSET(instance));
+	struct iosys_map source =
+		IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
+				      XE_MEMIRQ_SOURCE_OFFSET(hwe->irq_page));
 
-	if (memirq_received(memirq, &src_offset, offset, "SRC")) {
-		struct iosys_map status_offset =
+	if (memirq_received(memirq, &source, hwe->irq_offset, "SRC")) {
+		struct iosys_map status =
 			IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
-					      XE_MEMIRQ_STATUS_OFFSET(instance) + offset * SZ_16);
-		memirq_dispatch_engine(memirq, &status_offset, hwe);
+					      XE_MEMIRQ_VECTOR_OFFSET(hwe->irq_page,
+								      hwe->irq_offset));
+		memirq_dispatch_engine(memirq, &status, hwe);
 	}
 }
 
@@ -491,12 +523,25 @@ bool xe_memirq_guc_sw_int_0_irq_pending(struct xe_memirq *memirq, struct xe_guc
 {
 	struct xe_gt *gt = guc_to_gt(guc);
 	u32 offset = xe_gt_is_media_type(gt) ? ilog2(INTR_MGUC) : ilog2(INTR_GUC);
-	struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&memirq->status, offset * SZ_16);
+	struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
+						     XE_MEMIRQ_VECTOR_OFFSET(0, offset));
 
 	return memirq_received_noclear(memirq, &map, ilog2(GUC_INTR_SW_INT_0),
 				       guc_name(guc));
 }
 
+static void memirq_dump_source_pages(struct xe_memirq *memirq)
+{
+	memirq_assert(memirq, !memirq->bo->vmap.is_iomem);
+
+	for (int n = 0; n < memirq->num_pages; n++) {
+		memirq_debug(memirq, "SOURCE %*ph\n", 32,
+			     memirq->bo->vmap.vaddr + XE_MEMIRQ_SOURCE_OFFSET(n));
+		memirq_debug(memirq, "SOURCE %*ph\n", 32,
+			     memirq->bo->vmap.vaddr + XE_MEMIRQ_SOURCE_OFFSET(n) + 32);
+	}
+}
+
 /**
  * xe_memirq_handler - The `Memory Based Interrupts`_ Handler.
  * @memirq: the &xe_memirq
@@ -516,9 +561,8 @@ void xe_memirq_handler(struct xe_memirq *memirq)
 	if (!memirq->bo)
 		return;
 
-	memirq_assert(memirq, !memirq->source.is_iomem);
-	memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr);
-	memirq_debug(memirq, "SOURCE %*ph\n", 32, memirq->source.vaddr + 32);
+	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEMIRQ))
+		memirq_dump_source_pages(memirq);
 
 	for_each_gt(gt, xe, gtid) {
 		if (gt->tile != tile)
@@ -531,7 +575,8 @@ void xe_memirq_handler(struct xe_memirq *memirq)
 	/* GuC and media GuC (if present) must be checked separately */
 
 	if (memirq_received(memirq, &memirq->source, ilog2(INTR_GUC), "SRC")) {
-		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_GUC) * SZ_16);
+		map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
+					    XE_MEMIRQ_VECTOR_OFFSET(0, ilog2(INTR_GUC)));
 		memirq_dispatch_guc(memirq, &map, &tile->primary_gt->uc.guc);
 	}
 
@@ -539,7 +584,8 @@ void xe_memirq_handler(struct xe_memirq *memirq)
 		return;
 
 	if (memirq_received(memirq, &memirq->source, ilog2(INTR_MGUC), "SRC")) {
-		map = IOSYS_MAP_INIT_OFFSET(&memirq->status, ilog2(INTR_MGUC) * SZ_16);
+		map = IOSYS_MAP_INIT_OFFSET(&memirq->bo->vmap,
+					    XE_MEMIRQ_VECTOR_OFFSET(0, ilog2(INTR_MGUC)));
 		memirq_dispatch_guc(memirq, &map, &tile->media_gt->uc.guc);
 	}
 }
diff --git a/drivers/gpu/drm/xe/xe_memirq_types.h b/drivers/gpu/drm/xe/xe_memirq_types.h
index 9d0f6c1cdb9d5..c2b0b33d55cd3 100644
--- a/drivers/gpu/drm/xe/xe_memirq_types.h
+++ b/drivers/gpu/drm/xe/xe_memirq_types.h
@@ -10,27 +10,18 @@
 
 struct xe_bo;
 
-/* ISR */
-#define XE_MEMIRQ_STATUS_OFFSET(inst)	((inst) * SZ_4K + 0x0)
-/* IIR */
-#define XE_MEMIRQ_SOURCE_OFFSET(inst)	((inst) * SZ_4K + 0x400)
-/* IMR */
-#define XE_MEMIRQ_ENABLE_OFFSET		0x440
-
 /**
  * struct xe_memirq - Data used by the `Memory Based Interrupts`_.
  *
  * @bo: buffer object with `Memory Based Interrupts Page Layout`_.
+ * @num_pages: number of per-instance source/status pages.
  * @source: iosys pointer to `Interrupt Source Report Page`_.
- * @status: iosys pointer to `Interrupt Status Report Page`_.
- * @mask: iosys pointer to Interrupt Enable Mask.
  * @enabled: internal flag used to control processing of the interrupts.
  */
 struct xe_memirq {
 	struct xe_bo *bo;
+	unsigned int num_pages;
 	struct iosys_map source;
-	struct iosys_map status;
-	struct iosys_map mask;
 	bool enabled;
 };
 
diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
index a22413f892a09..9428dd5e7760c 100644
--- a/drivers/gpu/drm/xe/xe_migrate.c
+++ b/drivers/gpu/drm/xe/xe_migrate.c
@@ -31,6 +31,7 @@
 #include "xe_map.h"
 #include "xe_mem_pool.h"
 #include "xe_mocs.h"
+#include "xe_pat.h"
 #include "xe_printk.h"
 #include "xe_pt.h"
 #include "xe_res_cursor.h"
@@ -217,7 +218,7 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 				  struct xe_vm *vm, u32 *ofs)
 {
 	struct xe_device *xe = tile_to_xe(tile);
-	u16 pat_index = xe->pat.idx[XE_CACHE_WB];
+	u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
 	u8 id = tile->id;
 	u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level;
 #define VRAM_IDENTITY_MAP_COUNT	2
@@ -337,7 +338,7 @@ static void xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
 		 * if flat ccs is enabled.
 		 */
 		if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) {
-			u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION];
+			u16 comp_pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION);
 			u64 vram_offset = IDENTITY_OFFSET +
 				DIV_ROUND_UP_ULL(actual_phy_size, SZ_1G);
 			u64 pt31_ofs = xe_bo_size(bo) - XE_PAGE_SIZE;
@@ -637,10 +638,10 @@ static void emit_pte(struct xe_migrate *m,
 
 	/* Indirect access needs compression enabled uncached PAT index */
 	if (GRAPHICS_VERx100(xe) >= 2000)
-		pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] :
-					  xe->pat.idx[XE_CACHE_WB];
+		pat_index = is_comp_pte ? xe_cache_pat_idx(xe, XE_CACHE_NONE_COMPRESSION) :
+					  xe_cache_pat_idx(xe, XE_CACHE_WB);
 	else
-		pat_index = xe->pat.idx[XE_CACHE_WB];
+		pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
 
 	ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE);
 
@@ -727,7 +728,22 @@ static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb,
 	bb->len = cs - bb->cs;
 }
 
-#define EMIT_COPY_DW 10
+static u32 blt_fast_copy_cmd_len(struct xe_device *xe)
+{
+	return 10;
+}
+
+static u32 blt_mem_copy_cmd_len(struct xe_device *xe)
+{
+	return 10;
+}
+
+static u32 emit_copy_cmd_len(struct xe_device *xe)
+{
+	return (xe->info.has_mem_copy_instr) ? blt_mem_copy_cmd_len(xe) :
+		  blt_fast_copy_cmd_len(xe);
+}
+
 static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
 			      u64 dst_ofs, unsigned int size,
 			      unsigned int pitch)
@@ -735,6 +751,7 @@ static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
 	struct xe_device *xe = gt_to_xe(gt);
 	u32 mocs = 0;
 	u32 tile_y = 0;
+	u32 len;
 
 	xe_gt_assert(gt, !(pitch & 3));
 	xe_gt_assert(gt, size / pitch <= S16_MAX);
@@ -747,7 +764,8 @@ static void emit_xy_fast_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
 	if (GRAPHICS_VERx100(xe) >= 1250)
 		tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4;
 
-	bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2);
+	len = blt_fast_copy_cmd_len(xe);
+	bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (len - 2);
 	bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs;
 	bb->cs[bb->len++] = 0;
 	bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4;
@@ -764,6 +782,7 @@ static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
 			  u64 dst_ofs, unsigned int size, unsigned int pitch)
 {
 	u32 mode, copy_type, width;
+	u32 len;
 
 	xe_gt_assert(gt, IS_ALIGNED(size, pitch));
 	xe_gt_assert(gt, pitch <= U16_MAX);
@@ -789,7 +808,9 @@ static void emit_mem_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
 
 	xe_gt_assert(gt, width <= U16_MAX);
 
-	bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type;
+	len = blt_mem_copy_cmd_len(gt_to_xe(gt));
+
+	bb->cs[bb->len++] = MEM_COPY_CMD | mode | copy_type | (len - 2);
 	bb->cs[bb->len++] = width - 1;
 	bb->cs[bb->len++] = size / pitch - 1; /* ignored by hw for page-copy/linear above */
 	bb->cs[bb->len++] = pitch - 1;
@@ -966,7 +987,7 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
 		}
 
 		/* Add copy commands size here */
-		batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) +
+		batch_size += ((copy_only_ccs) ? 0 : emit_copy_cmd_len(xe)) +
 			((needs_ccs_emit ? EMIT_COPY_CCS_DW : 0));
 
 		bb = xe_bb_new(gt, batch_size, usm);
@@ -1405,7 +1426,7 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off
 
 		batch_size += pte_update_size(m, 0, sysmem, &sysmem_it, &vram_L0, &sysmem_L0_ofs,
 					      &sysmem_L0_pt, 0, avail_pts, avail_pts);
-		batch_size += EMIT_COPY_DW;
+		batch_size += emit_copy_cmd_len(xe);
 
 		bb = xe_bb_new(gt, batch_size, usm);
 		if (IS_ERR(bb)) {
@@ -1460,12 +1481,17 @@ struct dma_fence *xe_migrate_vram_copy_chunk(struct xe_bo *vram_bo, u64 vram_off
 	return fence;
 }
 
+static u32 blt_mem_set_cmd_len(struct xe_device *xe)
+{
+	return 7;
+}
+
 static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
 				 u32 size, u32 pitch)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 	u32 *cs = bb->cs + bb->len;
-	u32 len = PVC_MEM_SET_CMD_LEN_DW;
+	u32 len = blt_mem_set_cmd_len(xe);
 
 	*cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2);
 	*cs++ = pitch - 1;
@@ -1483,15 +1509,21 @@ static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs
 	bb->len += len;
 }
 
+static u32 blt_fast_color_cmd_len(struct xe_device *xe)
+{
+	if (GRAPHICS_VERx100(xe) >= 1250)
+		return 16;
+	else
+		return 11;
+}
+
 static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb,
 				 u64 src_ofs, u32 size, u32 pitch, bool is_vram)
 {
 	struct xe_device *xe = gt_to_xe(gt);
 	u32 *cs = bb->cs + bb->len;
-	u32 len = XY_FAST_COLOR_BLT_DW;
+	u32 len = blt_fast_color_cmd_len(xe);
 
-	if (GRAPHICS_VERx100(xe) < 1250)
-		len = 11;
 
 	*cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 |
 		(len - 2);
@@ -1526,10 +1558,12 @@ static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb,
 
 static u32 emit_clear_cmd_len(struct xe_gt *gt)
 {
+	struct xe_device *xe = gt_to_xe(gt);
+
 	if (gt->info.has_xe2_blt_instructions)
-		return PVC_MEM_SET_CMD_LEN_DW;
+		return blt_mem_set_cmd_len(xe);
 	else
-		return XY_FAST_COLOR_BLT_DW;
+		return blt_fast_color_cmd_len(xe);
 }
 
 static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs,
@@ -1862,7 +1896,7 @@ __xe_migrate_update_pgtables(struct xe_migrate *m,
 
 	/* For sysmem PTE's, need to map them in our hole.. */
 	if (!IS_DGFX(xe)) {
-		u16 pat_index = xe->pat.idx[XE_CACHE_WB];
+		u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
 		u32 ptes, ofs;
 
 		ppgtt_ofs = NUM_KERNEL_PDE - 1;
@@ -2084,7 +2118,7 @@ static void build_pt_update_batch_sram(struct xe_migrate *m,
 				       struct drm_pagemap_addr *sram_addr,
 				       u32 size, int level)
 {
-	u16 pat_index = tile_to_xe(m->tile)->pat.idx[XE_CACHE_WB];
+	u16 pat_index = xe_cache_pat_idx(tile_to_xe(m->tile), XE_CACHE_WB);
 	u64 gpu_page_size = 0x1ull << xe_pt_shift(level);
 	u32 ptes;
 	int i = 0;
@@ -2202,7 +2236,7 @@ static struct dma_fence *xe_migrate_vram(struct xe_migrate *m,
 	xe_assert(xe, npages * PAGE_SIZE <= MAX_PREEMPTDISABLE_TRANSFER);
 
 	batch_size += pte_update_cmd_size(npages << PAGE_SHIFT);
-	batch_size += EMIT_COPY_DW;
+	batch_size += emit_copy_cmd_len(xe);
 
 	bb = xe_bb_new(gt, batch_size, use_usm_batch);
 	if (IS_ERR(bb)) {
diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
index 169279d9d8c24..965c45889c726 100644
--- a/drivers/gpu/drm/xe/xe_migrate.h
+++ b/drivers/gpu/drm/xe/xe_migrate.h
@@ -3,8 +3,8 @@
  * Copyright © 2020 Intel Corporation
  */
 
-#ifndef _XE_MIGRATE_
-#define _XE_MIGRATE_
+#ifndef _XE_MIGRATE_H_
+#define _XE_MIGRATE_H_
 
 #include <linux/types.h>
 
diff --git a/drivers/gpu/drm/xe/xe_nvm.h b/drivers/gpu/drm/xe/xe_nvm.h
index fd3467ad35a4d..b14722103f818 100644
--- a/drivers/gpu/drm/xe/xe_nvm.h
+++ b/drivers/gpu/drm/xe/xe_nvm.h
@@ -3,8 +3,8 @@
  * Copyright(c) 2019-2025 Intel Corporation. All rights reserved.
  */
 
-#ifndef __XE_NVM_H__
-#define __XE_NVM_H__
+#ifndef _XE_NVM_H_
+#define _XE_NVM_H_
 
 struct xe_device;
 
diff --git a/drivers/gpu/drm/xe/xe_oa.c b/drivers/gpu/drm/xe/xe_oa.c
index 6337e671c97ae..4bf4b1f65929f 100644
--- a/drivers/gpu/drm/xe/xe_oa.c
+++ b/drivers/gpu/drm/xe/xe_oa.c
@@ -9,11 +9,13 @@
 #include <linux/poll.h>
 
 #include <drm/drm_drv.h>
+#include <drm/drm_gem.h>
 #include <drm/drm_managed.h>
 #include <drm/drm_syncobj.h>
 #include <uapi/drm/xe_drm.h>
 
 #include <generated/xe_wa_oob.h>
+#include <generated/xe_device_wa_oob.h>
 
 #include "abi/guc_actions_slpc_abi.h"
 #include "instructions/xe_mi_commands.h"
@@ -213,32 +215,45 @@ static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream)
 #define oa_report_header_64bit(__s) \
 	((__s)->oa_buffer.format->header == HDR_64_BIT)
 
-static u64 oa_report_id(struct xe_oa_stream *stream, void *report)
+static u64 oa_report_id(struct xe_oa_stream *stream, u32 report_offset)
 {
-	return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;
+	struct iosys_map *map = &stream->oa_buffer.bo->vmap;
+
+	return oa_report_header_64bit(stream) ?
+		xe_map_rd(stream->oa->xe, map, report_offset, u64) :
+		xe_map_rd(stream->oa->xe, map, report_offset, u32);
 }
 
-static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report)
+static void oa_report_id_clear(struct xe_oa_stream *stream, u32 report_offset)
 {
-	if (oa_report_header_64bit(stream))
-		*(u64 *)report = 0;
-	else
-		*report = 0;
+	struct iosys_map *map = &stream->oa_buffer.bo->vmap;
+
+	oa_report_header_64bit(stream) ?
+		xe_map_wr(stream->oa->xe, map, report_offset, u64, 0) :
+		xe_map_wr(stream->oa->xe, map, report_offset, u32, 0);
 }
 
-static u64 oa_timestamp(struct xe_oa_stream *stream, void *report)
+static u64 oa_timestamp(struct xe_oa_stream *stream, u32 report_offset)
 {
+	struct iosys_map *map = &stream->oa_buffer.bo->vmap;
+
 	return oa_report_header_64bit(stream) ?
-		*((u64 *)report + 1) :
-		*((u32 *)report + 1);
+		xe_map_rd(stream->oa->xe, map, report_offset + 8, u64) :
+		xe_map_rd(stream->oa->xe, map, report_offset + 4, u32);
 }
 
-static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report)
+static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 report_offset)
 {
-	if (oa_report_header_64bit(stream))
-		*(u64 *)&report[2] = 0;
-	else
-		report[1] = 0;
+	struct iosys_map *map = &stream->oa_buffer.bo->vmap;
+
+	oa_report_header_64bit(stream) ?
+		xe_map_wr(stream->oa->xe, map, report_offset + 8, u64, 0) :
+		xe_map_wr(stream->oa->xe, map, report_offset + 4, u32, 0);
+}
+
+static bool mert_wa_14026633728(struct xe_oa_stream *s)
+{
+	return s->oa_unit->type == DRM_XE_OA_UNIT_TYPE_MERT && XE_DEVICE_WA(s->oa->xe, 14026633728);
 }
 
 static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream)
@@ -275,9 +290,7 @@ static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream)
 	 * they were written.  If not : (╯°□°）╯︵ ┻━┻
 	 */
 	while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) {
-		void *report = stream->oa_buffer.vaddr + tail;
-
-		if (oa_report_id(stream, report) || oa_timestamp(stream, report))
+		if (oa_report_id(stream, tail) || oa_timestamp(stream, tail))
 			break;
 
 		tail = xe_oa_circ_diff(stream, tail, report_size);
@@ -311,30 +324,37 @@ static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer)
 	return HRTIMER_RESTART;
 }
 
+static unsigned long
+xe_oa_copy_to_user(struct xe_oa_stream *stream, void __user *dst, u32 report_offset, u32 len)
+{
+	xe_assert(stream->oa->xe, len <= stream->oa_buffer.format->size);
+
+	xe_map_memcpy_from(stream->oa->xe, stream->oa_buffer.bounce,
+			   &stream->oa_buffer.bo->vmap, report_offset, len);
+	return copy_to_user(dst, stream->oa_buffer.bounce, len);
+}
+
 static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf,
-			       size_t count, size_t *offset, const u8 *report)
+			       size_t count, size_t *offset, u32 report_offset)
 {
 	int report_size = stream->oa_buffer.format->size;
 	int report_size_partial;
-	u8 *oa_buf_end;
 
 	if ((count - *offset) < report_size)
 		return -ENOSPC;
 
 	buf += *offset;
 
-	oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;
-	report_size_partial = oa_buf_end - report;
+	report_size_partial = stream->oa_buffer.circ_size - report_offset;
 
 	if (report_size_partial < report_size) {
-		if (copy_to_user(buf, report, report_size_partial))
+		if (xe_oa_copy_to_user(stream, buf, report_offset, report_size_partial))
 			return -EFAULT;
 		buf += report_size_partial;
 
-		if (copy_to_user(buf, stream->oa_buffer.vaddr,
-				 report_size - report_size_partial))
+		if (xe_oa_copy_to_user(stream, buf, 0, report_size - report_size_partial))
 			return -EFAULT;
-	} else if (copy_to_user(buf, report, report_size)) {
+	} else if (xe_oa_copy_to_user(stream, buf, report_offset, report_size)) {
 		return -EFAULT;
 	}
 
@@ -347,7 +367,6 @@ static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf,
 				size_t count, size_t *offset)
 {
 	int report_size = stream->oa_buffer.format->size;
-	u8 *oa_buf_base = stream->oa_buffer.vaddr;
 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
 	size_t start_offset = *offset;
 	unsigned long flags;
@@ -364,26 +383,24 @@ static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf,
 
 	for (; xe_oa_circ_diff(stream, tail, head);
 	     head = xe_oa_circ_incr(stream, head, report_size)) {
-		u8 *report = oa_buf_base + head;
-
-		ret = xe_oa_append_report(stream, buf, count, offset, report);
+		ret = xe_oa_append_report(stream, buf, count, offset, head);
 		if (ret)
 			break;
 
 		if (!(stream->oa_buffer.circ_size % report_size)) {
 			/* Clear out report id and timestamp to detect unlanded reports */
-			oa_report_id_clear(stream, (void *)report);
-			oa_timestamp_clear(stream, (void *)report);
+			oa_report_id_clear(stream, head);
+			oa_timestamp_clear(stream, head);
 		} else {
-			u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;
-			u32 part = oa_buf_end - report;
+			struct iosys_map *map = &stream->oa_buffer.bo->vmap;
+			u32 part = stream->oa_buffer.circ_size - head;
 
 			/* Zero out the entire report */
 			if (report_size <= part) {
-				memset(report, 0, report_size);
+				xe_map_memset(stream->oa->xe, map, head, 0, report_size);
 			} else {
-				memset(report, 0, part);
-				memset(oa_buf_base, 0, report_size - part);
+				xe_map_memset(stream->oa->xe, map, head, 0, part);
+				xe_map_memset(stream->oa->xe, map, 0, 0, report_size - part);
 			}
 		}
 	}
@@ -436,7 +453,8 @@ static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream)
 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
 
 	/* Zero out the OA buffer since we rely on zero report id and timestamp fields */
-	memset(stream->oa_buffer.vaddr, 0, xe_bo_size(stream->oa_buffer.bo));
+	xe_map_memset(stream->oa->xe, &stream->oa_buffer.bo->vmap, 0, 0,
+		      xe_bo_size(stream->oa_buffer.bo));
 }
 
 static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask)
@@ -699,6 +717,7 @@ static int num_lri_dwords(int num_regs)
 static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream)
 {
 	xe_bo_unpin_map_no_vm(stream->oa_buffer.bo);
+	kfree(stream->oa_buffer.bounce);
 }
 
 static void xe_oa_free_configs(struct xe_oa_stream *stream)
@@ -880,18 +899,25 @@ static void xe_oa_stream_destroy(struct xe_oa_stream *stream)
 
 static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream, size_t size)
 {
+	u32 vram = mert_wa_14026633728(stream) ?
+		XE_BO_FLAG_VRAM_IF_DGFX(xe_device_get_root_tile(stream->oa->xe)) :
+		XE_BO_FLAG_SYSTEM;
 	struct xe_bo *bo;
 
 	bo = xe_bo_create_pin_map_novm(stream->oa->xe, stream->gt->tile,
 				       size, ttm_bo_type_kernel,
-				       XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT, false);
+				       vram | XE_BO_FLAG_GGTT, false);
 	if (IS_ERR(bo))
 		return PTR_ERR(bo);
 
 	stream->oa_buffer.bo = bo;
-	/* mmap implementation requires OA buffer to be in system memory */
-	xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0);
-	stream->oa_buffer.vaddr = bo->vmap.vaddr;
+
+	stream->oa_buffer.bounce = kmalloc(stream->oa_buffer.format->size, GFP_KERNEL);
+	if (!stream->oa_buffer.bounce) {
+		xe_bo_unpin_map_no_vm(stream->oa_buffer.bo);
+		return -ENOMEM;
+	}
+
 	return 0;
 }
 
@@ -1673,8 +1699,6 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma)
 {
 	struct xe_oa_stream *stream = file->private_data;
 	struct xe_bo *bo = stream->oa_buffer.bo;
-	unsigned long start = vma->vm_start;
-	int i, ret;
 
 	if (xe_observation_paranoid && !perfmon_capable()) {
 		drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n");
@@ -1682,7 +1706,7 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma)
 	}
 
 	/* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */
-	if (vma->vm_end - vma->vm_start != xe_bo_size(stream->oa_buffer.bo)) {
+	if (vma->vm_end - vma->vm_start != xe_bo_size(bo)) {
 		drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n");
 		return -EINVAL;
 	}
@@ -1698,17 +1722,7 @@ static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma)
 	vm_flags_mod(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY,
 		     VM_MAYWRITE | VM_MAYEXEC);
 
-	xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma));
-	for (i = 0; i < bo->ttm.ttm->num_pages; i++) {
-		ret = remap_pfn_range(vma, start, page_to_pfn(bo->ttm.ttm->pages[i]),
-				      PAGE_SIZE, vma->vm_page_prot);
-		if (ret)
-			break;
-
-		start += PAGE_SIZE;
-	}
-
-	return ret;
+	return drm_gem_mmap_obj(&bo->ttm.base, xe_bo_size(bo), vma);
 }
 
 static const struct file_operations xe_oa_fops = {
@@ -1920,16 +1934,21 @@ static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent)
 	return div_u64(nom + den - 1, den);
 }
 
-static bool oa_unit_supports_oa_format(struct xe_oa_open_param *param, int type)
+static bool oa_unit_supports_oa_format(struct xe_oa *oa, struct xe_oa_open_param *param)
 {
+	const struct xe_oa_format *f = &oa->oa_formats[param->oa_format];
+
 	switch (param->oa_unit->type) {
 	case DRM_XE_OA_UNIT_TYPE_OAG:
-		return type == DRM_XE_OA_FMT_TYPE_OAG || type == DRM_XE_OA_FMT_TYPE_OAR ||
-			type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC;
+		return f->type == DRM_XE_OA_FMT_TYPE_OAG || f->type == DRM_XE_OA_FMT_TYPE_OAR ||
+			f->type == DRM_XE_OA_FMT_TYPE_OAC || f->type == DRM_XE_OA_FMT_TYPE_PEC;
+	case DRM_XE_OA_UNIT_TYPE_MERT:
+		if (XE_DEVICE_WA(oa->xe, 14026746987))
+			return param->oa_format == XE_OAM_FORMAT_MPEC8u32_B8_C8;
+		fallthrough;
 	case DRM_XE_OA_UNIT_TYPE_OAM:
 	case DRM_XE_OA_UNIT_TYPE_OAM_SAG:
-	case DRM_XE_OA_UNIT_TYPE_MERT:
-		return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC;
+		return f->type == DRM_XE_OA_FMT_TYPE_OAM || f->type == DRM_XE_OA_FMT_TYPE_OAM_MPEC;
 	default:
 		return false;
 	}
@@ -2032,8 +2051,10 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f
 		if (XE_IOCTL_DBG(oa->xe, !param.exec_q))
 			return -ENOENT;
 
-		if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1))
-			return -EOPNOTSUPP;
+		if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1)) {
+			ret = -EOPNOTSUPP;
+			goto err_exec_q;
+		}
 	}
 
 	/*
@@ -2069,8 +2090,7 @@ int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *f
 		goto err_exec_q;
 
 	f = &oa->oa_formats[param.oa_format];
-	if (!param.oa_format || !f->size ||
-	    !oa_unit_supports_oa_format(&param, f->type)) {
+	if (!param.oa_format || !f->size || !oa_unit_supports_oa_format(oa, &param)) {
 		drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n",
 			param.oa_format, f->type, f->size, param.hwe->class);
 		ret = -EINVAL;
@@ -2231,15 +2251,19 @@ static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr)
 		return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs);
 }
 
-static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr)
+static bool xe_oa_is_valid_config_reg(struct xe_oa *oa, u32 addr, u32 val)
 {
+	if (XE_DEVICE_WA(oa->xe, 14026779378) &&
+	    addr == SYS_MEM_LAT_MEASURE.addr && val & SYS_MEM_LAT_MEASURE_EN)
+		return false;
+
 	return xe_oa_is_valid_flex_addr(oa, addr) ||
 		xe_oa_is_valid_b_counter_addr(oa, addr) ||
 		xe_oa_is_valid_mux_addr(oa, addr);
 }
 
 static struct xe_oa_reg *
-xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr),
+xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr, u32 val),
 		 u32 __user *regs, u32 n_regs)
 {
 	struct xe_oa_reg *oa_regs;
@@ -2257,16 +2281,16 @@ xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr),
 		if (err)
 			goto addr_err;
 
-		if (!is_valid(oa, addr)) {
-			drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr);
-			err = -EINVAL;
-			goto addr_err;
-		}
-
 		err = get_user(value, regs + 1);
 		if (err)
 			goto addr_err;
 
+		if (!is_valid(oa, addr, value)) {
+			drm_dbg(&oa->xe->drm, "Invalid oa_reg addr/value: %#x %#x\n", addr, value);
+			err = -EINVAL;
+			goto addr_err;
+		}
+
 		oa_regs[i].addr = XE_REG(addr);
 		oa_regs[i].value = value;
 
@@ -2365,7 +2389,7 @@ int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *fi
 	memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid));
 
 	oa_config->regs_len = arg->n_regs;
-	regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr,
+	regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg,
 				u64_to_user_ptr(arg->regs_ptr),
 				arg->n_regs);
 	if (IS_ERR(regs)) {
diff --git a/drivers/gpu/drm/xe/xe_oa_types.h b/drivers/gpu/drm/xe/xe_oa_types.h
index b03ffd5134834..3d9ec8490899c 100644
--- a/drivers/gpu/drm/xe/xe_oa_types.h
+++ b/drivers/gpu/drm/xe/xe_oa_types.h
@@ -67,7 +67,7 @@ struct xe_oa_format {
 	u32 counter_select;
 	/** @size: record size as written by HW (multiple of 64 byte cachelines) */
 	int size;
-	/** @type: of enum @drm_xe_oa_format_type */
+	/** @type: of enum drm_xe_oa_format_type */
 	int type;
 	/** @header: 32 or 64 bit report headers */
 	enum xe_oa_report_header header;
@@ -154,16 +154,18 @@ struct xe_oa {
 	u16 oa_unit_ids;
 };
 
-/** @xe_oa_buffer: State of the stream OA buffer */
+/**
+ * struct xe_oa_buffer - State of the stream OA buffer
+ */
 struct xe_oa_buffer {
 	/** @format: data format */
 	const struct xe_oa_format *format;
 
-	/** @format: xe_bo backing the OA buffer */
+	/** @bo: xe_bo backing the OA buffer */
 	struct xe_bo *bo;
 
-	/** @vaddr: mapped vaddr of the OA buffer */
-	u8 *vaddr;
+	/** @bounce: bounce buffer used with xe_map layer */
+	void *bounce;
 
 	/** @ptr_lock: Lock protecting reads/writes to head/tail pointers */
 	spinlock_t ptr_lock;
diff --git a/drivers/gpu/drm/xe/xe_pagefault_types.h b/drivers/gpu/drm/xe/xe_pagefault_types.h
index b3289219b1be5..c4ee625b93ddd 100644
--- a/drivers/gpu/drm/xe/xe_pagefault_types.h
+++ b/drivers/gpu/drm/xe/xe_pagefault_types.h
@@ -86,7 +86,7 @@ struct xe_pagefault {
 		u8 engine_class;
 		/** @consumer.engine_instance: engine instance */
 		u8 engine_instance;
-		/** consumer.reserved: reserved bits for future expansion */
+		/** @consumer.reserved: reserved bits for future expansion */
 		u64 reserved;
 	} consumer;
 	/**
@@ -112,7 +112,7 @@ struct xe_pagefault {
 };
 
 /**
- * struct xe_pagefault_queue: Xe pagefault queue (consumer)
+ * struct xe_pagefault_queue - Xe pagefault queue (consumer)
  *
  * Used to capture all device page faults for deferred processing. Size this
  * queue to absorb the device’s worst-case number of outstanding faults.
diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c
index 356f53bdb83c4..fad5b5a5ed4a0 100644
--- a/drivers/gpu/drm/xe/xe_pat.c
+++ b/drivers/gpu/drm/xe/xe_pat.c
@@ -531,6 +531,14 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p)
 	drm_printf(p, "Page Table Access:\n");
 	xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false);
 
+	if (xe_gt_is_media_type(gt))
+		pat = xe_mmio_read32(&gt->mmio, XE_REG(_PAT_ATS));
+	else
+		pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_ATS));
+
+	drm_printf(p, "PCIe ATS/PASID:\n");
+	xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false);
+
 	return 0;
 }
 
@@ -551,6 +559,7 @@ static const struct xe_pat_ops xe3p_xpc_pat_ops = {
 void xe_pat_init_early(struct xe_device *xe)
 {
 	xe->pat.idx[XE_CACHE_WB_COMPRESSION] = XE_PAT_INVALID_IDX;
+	xe->pat.idx[XE_CACHE_NONE_COMPRESSION] = XE_PAT_INVALID_IDX;
 	if (GRAPHICS_VERx100(xe) == 3511) {
 		xe->pat.ops = &xe3p_xpc_pat_ops;
 		xe->pat.table = xe3p_xpc_pat_table;
diff --git a/drivers/gpu/drm/xe/xe_pat.h b/drivers/gpu/drm/xe/xe_pat.h
index a1e287c08f570..7060f66e1d638 100644
--- a/drivers/gpu/drm/xe/xe_pat.h
+++ b/drivers/gpu/drm/xe/xe_pat.h
@@ -82,4 +82,12 @@ bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index);
  */
 u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index);
 
+#define xe_cache_pat_idx(xe, cache_mode) ({					\
+	const struct xe_device *__xedev = (xe);					\
+	enum xe_cache_level __mode = (cache_mode);				\
+	xe_assert(__xedev, __mode < __XE_CACHE_LEVEL_COUNT);			\
+	xe_assert(__xedev, __xedev->pat.idx[__mode] != XE_PAT_INVALID_IDX);	\
+	__xedev->pat.idx[__mode];						\
+})
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c
index c2ecd27ec770a..1243c7d8ed10a 100644
--- a/drivers/gpu/drm/xe/xe_pci.c
+++ b/drivers/gpu/drm/xe/xe_pci.c
@@ -444,6 +444,7 @@ static const struct xe_device_desc nvls_desc = {
 	.has_display = true,
 	.has_flat_ccs = 1,
 	.has_pre_prod_wa = 1,
+	.has_sriov = true,
 	.max_gt_per_tile = 2,
 	MULTI_LRC_MASK,
 	.require_force_probe = true,
@@ -466,6 +467,7 @@ static const struct xe_device_desc cri_desc = {
 	.has_soc_remapper_sysctrl = true,
 	.has_soc_remapper_telem = true,
 	.has_sriov = true,
+	.has_sysctrl = true,
 	.max_gt_per_tile = 2,
 	MULTI_LRC_MASK,
 	.require_force_probe = true,
@@ -481,6 +483,7 @@ static const struct xe_device_desc nvlp_desc = {
 	.has_flat_ccs = 1,
 	.has_page_reclaim_hw_assist = true,
 	.has_pre_prod_wa = true,
+	.has_sriov = true,
 	.max_gt_per_tile = 2,
 	MULTI_LRC_MASK,
 	.require_force_probe = true,
@@ -764,8 +767,8 @@ static int xe_info_init_early(struct xe_device *xe,
 	xe->info.has_soc_remapper_telem = desc->has_soc_remapper_telem;
 	xe->info.has_sriov = xe_configfs_primary_gt_allowed(to_pci_dev(xe->drm.dev)) &&
 		desc->has_sriov;
+	xe->info.has_sysctrl = desc->has_sysctrl;
 	xe->info.skip_guc_pc = desc->skip_guc_pc;
-	xe->info.skip_mtcfg = desc->skip_mtcfg;
 	xe->info.skip_pcode = desc->skip_pcode;
 	xe->info.needs_scratch = desc->needs_scratch;
 	xe->info.needs_shared_vf_gt_wq = desc->needs_shared_vf_gt_wq;
@@ -807,9 +810,6 @@ static void xe_info_probe_tile_count(struct xe_device *xe)
 	if (xe->info.tile_count == 1)
 		return;
 
-	if (xe->info.skip_mtcfg)
-		return;
-
 	mmio = xe_root_tile_mmio(xe);
 
 	/*
@@ -969,6 +969,12 @@ static int xe_info_init(struct xe_device *xe,
 	xe->info.has_64bit_timestamp = graphics_desc->has_64bit_timestamp;
 	xe->info.has_mem_copy_instr = GRAPHICS_VER(xe) >= 20;
 
+	if (IS_SRIOV_VF(xe)) {
+		xe->info.has_sysctrl = 0;
+		xe->info.has_soc_remapper_sysctrl = 0;
+		xe->info.has_soc_remapper_telem = 0;
+	}
+
 	xe_info_probe_tile_count(xe);
 
 	for_each_remote_tile(tile, xe, id) {
diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c
index 3fd22034f03e2..35e6b53e18ce2 100644
--- a/drivers/gpu/drm/xe/xe_pci_sriov.c
+++ b/drivers/gpu/drm/xe/xe_pci_sriov.c
@@ -191,6 +191,8 @@ static int pf_disable_vfs(struct xe_device *xe)
 
 	pci_disable_sriov(pdev);
 
+	xe_sriov_pf_reprovision_default(xe);
+
 	pf_reset_vfs(xe, num_vfs);
 
 	xe_sriov_pf_unprovision_vfs(xe, num_vfs);
diff --git a/drivers/gpu/drm/xe/xe_pci_types.h b/drivers/gpu/drm/xe/xe_pci_types.h
index 8eee4fb1c57cf..5b85e2c24b7b2 100644
--- a/drivers/gpu/drm/xe/xe_pci_types.h
+++ b/drivers/gpu/drm/xe/xe_pci_types.h
@@ -57,9 +57,9 @@ struct xe_device_desc {
 	u8 has_soc_remapper_sysctrl:1;
 	u8 has_soc_remapper_telem:1;
 	u8 has_sriov:1;
+	u8 has_sysctrl:1;
 	u8 needs_scratch:1;
 	u8 skip_guc_pc:1;
-	u8 skip_mtcfg:1;
 	u8 skip_pcode:1;
 	u8 needs_shared_vf_gt_wq:1;
 };
diff --git a/drivers/gpu/drm/xe/xe_pcode_api.h b/drivers/gpu/drm/xe/xe_pcode_api.h
index 85cc7478b787a..94575c476e3d0 100644
--- a/drivers/gpu/drm/xe/xe_pcode_api.h
+++ b/drivers/gpu/drm/xe/xe_pcode_api.h
@@ -3,6 +3,9 @@
  * Copyright © 2022 Intel Corporation
  */
 
+#ifndef _XE_PCODE_API_H_
+#define _XE_PCODE_API_H_
+
 /* Internal to xe_pcode */
 
 #include "regs/xe_reg_defs.h"
@@ -47,8 +50,9 @@
 #define	WRITE_PSYSGPU_POWER_LIMIT		0x7
 #define	READ_PACKAGE_POWER_LIMIT		0x8
 #define	WRITE_PACKAGE_POWER_LIMIT		0x9
-#define	READ_PL_FROM_FW				0x1
 #define	READ_PL_FROM_PCODE			0x0
+#define	READ_PL_FROM_FW				0x1
+#define	READ_PL_ACCEPTED			0x2
 
 #define   PCODE_THERMAL_INFO			0x25
 #define     READ_THERMAL_LIMITS			0x0
@@ -101,3 +105,5 @@
 #define BMG_PCIE_CAP			XE_REG(0x138340)
 #define   LINK_DOWNGRADE		REG_GENMASK(1, 0)
 #define     DOWNGRADE_CAPABLE		2
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c
index 01185f10a8835..d4672eb074762 100644
--- a/drivers/gpu/drm/xe/xe_pm.c
+++ b/drivers/gpu/drm/xe/xe_pm.c
@@ -26,6 +26,7 @@
 #include "xe_pcode.h"
 #include "xe_pxp.h"
 #include "xe_sriov_vf_ccs.h"
+#include "xe_sysctrl.h"
 #include "xe_trace.h"
 #include "xe_vm.h"
 #include "xe_wa.h"
@@ -259,6 +260,8 @@ int xe_pm_resume(struct xe_device *xe)
 
 	xe_i2c_pm_resume(xe, true);
 
+	xe_sysctrl_pm_resume(xe);
+
 	xe_irq_resume(xe);
 
 	for_each_gt(gt, xe, id) {
@@ -670,6 +673,9 @@ int xe_pm_runtime_resume(struct xe_device *xe)
 
 	xe_i2c_pm_resume(xe, xe->d3cold.allowed);
 
+	if (xe->d3cold.allowed)
+		xe_sysctrl_pm_resume(xe);
+
 	xe_irq_resume(xe);
 
 	for_each_gt(gt, xe, id) {
diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c
index 8e5f4f0dea3f9..2669ff5ee747d 100644
--- a/drivers/gpu/drm/xe/xe_pt.c
+++ b/drivers/gpu/drm/xe/xe_pt.c
@@ -14,6 +14,7 @@
 #include "xe_gt_stats.h"
 #include "xe_migrate.h"
 #include "xe_page_reclaim.h"
+#include "xe_pat.h"
 #include "xe_pt_types.h"
 #include "xe_pt_walk.h"
 #include "xe_res_cursor.h"
@@ -62,7 +63,7 @@ static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm,
 			     unsigned int level)
 {
 	struct xe_device *xe = tile_to_xe(tile);
-	u16 pat_index = xe->pat.idx[XE_CACHE_WB];
+	u16 pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
 	u8 id = tile->id;
 
 	if (!xe_vm_has_scratch(vm))
diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h
index 84b51d3762a4d..a7d1bb708b69e 100644
--- a/drivers/gpu/drm/xe/xe_pt_types.h
+++ b/drivers/gpu/drm/xe/xe_pt_types.h
@@ -34,7 +34,7 @@ struct xe_pt {
 	bool rebind;
 	bool is_compact;
 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)
-	/** addr: Virtual address start address of the PT. */
+	/** @addr: Virtual address start address of the PT. */
 	u64 addr;
 #endif
 };
diff --git a/drivers/gpu/drm/xe/xe_pt_walk.h b/drivers/gpu/drm/xe/xe_pt_walk.h
index 5c02c244f7de3..e91995fa703b0 100644
--- a/drivers/gpu/drm/xe/xe_pt_walk.h
+++ b/drivers/gpu/drm/xe/xe_pt_walk.h
@@ -2,8 +2,8 @@
 /*
  * Copyright © 2022 Intel Corporation
  */
-#ifndef __XE_PT_WALK__
-#define __XE_PT_WALK__
+#ifndef _XE_PT_WALK_H_
+#define _XE_PT_WALK_H_
 
 #include <linux/pagewalk.h>
 #include <linux/types.h>
@@ -34,7 +34,7 @@ struct xe_pt_walk {
 	 * changed during the walk.
 	 */
 	const u64 *shifts;
-	/** @max_level: Highest populated level in @sizes */
+	/** @max_level: Highest populated level in @shifts */
 	unsigned int max_level;
 	/**
 	 * @shared_pt_mode: Whether to skip all entries that are private
@@ -49,7 +49,7 @@ struct xe_pt_walk {
 
 /**
  * typedef xe_pt_entry_fn - gpu page-table-walk callback-function
- * @parent: The parent page.table.
+ * @parent: The parent page table.
  * @offset: The offset (number of entries) into the page table.
  * @level: The level of @parent.
  * @addr: The virtual address.
@@ -111,14 +111,14 @@ static inline bool xe_pt_covers(u64 addr, u64 end, unsigned int level,
 }
 
 /**
- * xe_pt_num_entries: Number of page-table entries of a given range at this
+ * xe_pt_num_entries - Number of page-table entries of a given range at this
  * level
  * @addr: Start address.
  * @end: End address.
  * @level: Page table level.
  * @walk: Walk info.
  *
- * Return: The number of page table entries at this level between @start and
+ * Return: The number of page table entries at this level between @addr and
  * @end.
  */
 static inline pgoff_t
@@ -132,7 +132,7 @@ xe_pt_num_entries(u64 addr, u64 end, unsigned int level,
 }
 
 /**
- * xe_pt_offset: Offset of the page-table entry for a given address.
+ * xe_pt_offset - Offset of the page-table entry for a given address.
  * @addr: The address.
  * @level: Page table level.
  * @walk: Walk info.
diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c
index 7244090b07825..968b7e70b3f97 100644
--- a/drivers/gpu/drm/xe/xe_pxp.c
+++ b/drivers/gpu/drm/xe/xe_pxp.c
@@ -13,10 +13,12 @@
 #include "xe_device_types.h"
 #include "xe_exec_queue.h"
 #include "xe_force_wake.h"
+#include "xe_guc_exec_queue_types.h"
 #include "xe_guc_submit.h"
 #include "xe_gsc_proxy.h"
 #include "xe_gt_types.h"
 #include "xe_huc.h"
+#include "xe_hw_engine.h"
 #include "xe_mmio.h"
 #include "xe_pm.h"
 #include "xe_pxp_submit.h"
@@ -740,6 +742,10 @@ static void pxp_invalidate_queues(struct xe_pxp *pxp)
 	spin_unlock_irq(&pxp->queues.lock);
 
 	list_for_each_entry_safe(q, tmp, &to_clean, pxp.link) {
+		drm_dbg(&pxp->xe->drm,
+			"Killing queue due to PXP termination: eclass=%s, guc_id=%d\n",
+			xe_hw_engine_class_to_str(q->class), q->guc->id);
+
 		xe_exec_queue_kill(q);
 
 		/*
diff --git a/drivers/gpu/drm/xe/xe_pxp.h b/drivers/gpu/drm/xe/xe_pxp.h
index 71a23280b9000..4fb6e0afffd25 100644
--- a/drivers/gpu/drm/xe/xe_pxp.h
+++ b/drivers/gpu/drm/xe/xe_pxp.h
@@ -3,8 +3,8 @@
  * Copyright(c) 2024, Intel Corporation. All rights reserved.
  */
 
-#ifndef __XE_PXP_H__
-#define __XE_PXP_H__
+#ifndef _XE_PXP_H_
+#define _XE_PXP_H_
 
 #include <linux/types.h>
 
@@ -32,4 +32,4 @@ int xe_pxp_key_assign(struct xe_pxp *pxp, struct xe_bo *bo);
 int xe_pxp_bo_key_check(struct xe_pxp *pxp, struct xe_bo *bo);
 int xe_pxp_obj_key_check(struct drm_gem_object *obj);
 
-#endif /* __XE_PXP_H__ */
+#endif /* _XE_PXP_H_ */
diff --git a/drivers/gpu/drm/xe/xe_pxp_debugfs.h b/drivers/gpu/drm/xe/xe_pxp_debugfs.h
index 988466aad50b3..2997de0c90b26 100644
--- a/drivers/gpu/drm/xe/xe_pxp_debugfs.h
+++ b/drivers/gpu/drm/xe/xe_pxp_debugfs.h
@@ -3,11 +3,11 @@
  * Copyright © 2024 Intel Corporation
  */
 
-#ifndef __XE_PXP_DEBUGFS_H__
-#define __XE_PXP_DEBUGFS_H__
+#ifndef _XE_PXP_DEBUGFS_H_
+#define _XE_PXP_DEBUGFS_H_
 
 struct xe_pxp;
 
 void xe_pxp_debugfs_register(struct xe_pxp *pxp);
 
-#endif /* __XE_PXP_DEBUGFS_H__ */
+#endif /* _XE_PXP_DEBUGFS_H_ */
diff --git a/drivers/gpu/drm/xe/xe_pxp_submit.h b/drivers/gpu/drm/xe/xe_pxp_submit.h
index c9efda02f4b0a..dbbbe6b92bb26 100644
--- a/drivers/gpu/drm/xe/xe_pxp_submit.h
+++ b/drivers/gpu/drm/xe/xe_pxp_submit.h
@@ -3,8 +3,8 @@
  * Copyright(c) 2024, Intel Corporation. All rights reserved.
  */
 
-#ifndef __XE_PXP_SUBMIT_H__
-#define __XE_PXP_SUBMIT_H__
+#ifndef _XE_PXP_SUBMIT_H_
+#define _XE_PXP_SUBMIT_H_
 
 #include <linux/types.h>
 
@@ -19,4 +19,4 @@ int xe_pxp_submit_session_termination(struct xe_pxp *pxp, u32 id);
 int xe_pxp_submit_session_invalidation(struct xe_pxp_gsc_client_resources *gsc_res,
 				       u32 id);
 
-#endif /* __XE_PXP_SUBMIT_H__ */
+#endif /* _XE_PXP_SUBMIT_H_ */
diff --git a/drivers/gpu/drm/xe/xe_pxp_types.h b/drivers/gpu/drm/xe/xe_pxp_types.h
index 53e9d48d10fb8..ec86306e16f49 100644
--- a/drivers/gpu/drm/xe/xe_pxp_types.h
+++ b/drivers/gpu/drm/xe/xe_pxp_types.h
@@ -3,8 +3,8 @@
  * Copyright(c) 2024, Intel Corporation. All rights reserved.
  */
 
-#ifndef __XE_PXP_TYPES_H__
-#define __XE_PXP_TYPES_H__
+#ifndef _XE_PXP_TYPES_H_
+#define _XE_PXP_TYPES_H_
 
 #include <linux/completion.h>
 #include <linux/iosys-map.h>
@@ -132,4 +132,4 @@ struct xe_pxp {
 	u32 last_suspend_key_instance;
 };
 
-#endif /* __XE_PXP_TYPES_H__ */
+#endif /* _XE_PXP_TYPES_H_ */
diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c
index d84d6a422c45d..8c7d54498f384 100644
--- a/drivers/gpu/drm/xe/xe_query.c
+++ b/drivers/gpu/drm/xe/xe_query.c
@@ -231,6 +231,9 @@ static size_t calc_mem_regions_size(struct xe_device *xe)
 	u32 num_managers = 1;
 	int i;
 
+	if (xe_device_is_admin_only(xe))
+		return sizeof(struct drm_xe_query_mem_regions);
+
 	for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i)
 		if (ttm_manager_type(&xe->ttm, i))
 			num_managers++;
@@ -259,6 +262,9 @@ static int query_mem_regions(struct xe_device *xe,
 	if (XE_IOCTL_DBG(xe, !mem_regions))
 		return -ENOMEM;
 
+	if (xe_device_is_admin_only(xe))
+		goto user_copy;
+
 	man = ttm_manager_type(&xe->ttm, XE_PL_TT);
 	mem_regions->mem_regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM;
 	/*
@@ -297,6 +303,7 @@ static int query_mem_regions(struct xe_device *xe,
 		}
 	}
 
+user_copy:
 	if (!copy_to_user(query_ptr, mem_regions, size))
 		ret = 0;
 	else
diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c
new file mode 100644
index 0000000000000..4cb16b419b0c6
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_ras.c
@@ -0,0 +1,93 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include "xe_device.h"
+#include "xe_printk.h"
+#include "xe_ras.h"
+#include "xe_ras_types.h"
+#include "xe_sysctrl.h"
+#include "xe_sysctrl_event_types.h"
+
+/* Severity of detected errors  */
+enum xe_ras_severity {
+	XE_RAS_SEV_NOT_SUPPORTED = 0,
+	XE_RAS_SEV_CORRECTABLE,
+	XE_RAS_SEV_UNCORRECTABLE,
+	XE_RAS_SEV_INFORMATIONAL,
+	XE_RAS_SEV_MAX
+};
+
+/* Major IP blocks/components where errors can originate */
+enum xe_ras_component {
+	XE_RAS_COMP_NOT_SUPPORTED = 0,
+	XE_RAS_COMP_DEVICE_MEMORY,
+	XE_RAS_COMP_CORE_COMPUTE,
+	XE_RAS_COMP_RESERVED,
+	XE_RAS_COMP_PCIE,
+	XE_RAS_COMP_FABRIC,
+	XE_RAS_COMP_SOC_INTERNAL,
+	XE_RAS_COMP_MAX
+};
+
+static const char *const xe_ras_severities[] = {
+	[XE_RAS_SEV_NOT_SUPPORTED]		= "Not Supported",
+	[XE_RAS_SEV_CORRECTABLE]		= "Correctable Error",
+	[XE_RAS_SEV_UNCORRECTABLE]		= "Uncorrectable Error",
+	[XE_RAS_SEV_INFORMATIONAL]		= "Informational Error",
+};
+static_assert(ARRAY_SIZE(xe_ras_severities) == XE_RAS_SEV_MAX);
+
+static const char *const xe_ras_components[] = {
+	[XE_RAS_COMP_NOT_SUPPORTED]		= "Not Supported",
+	[XE_RAS_COMP_DEVICE_MEMORY]		= "Device Memory",
+	[XE_RAS_COMP_CORE_COMPUTE]		= "Core Compute",
+	[XE_RAS_COMP_RESERVED]			= "Reserved",
+	[XE_RAS_COMP_PCIE]			= "PCIe",
+	[XE_RAS_COMP_FABRIC]			= "Fabric",
+	[XE_RAS_COMP_SOC_INTERNAL]		= "SoC Internal",
+};
+static_assert(ARRAY_SIZE(xe_ras_components) == XE_RAS_COMP_MAX);
+
+static inline const char *sev_to_str(u8 severity)
+{
+	if (severity >= XE_RAS_SEV_MAX)
+		severity = XE_RAS_SEV_NOT_SUPPORTED;
+
+	return xe_ras_severities[severity];
+}
+
+static inline const char *comp_to_str(u8 component)
+{
+	if (component >= XE_RAS_COMP_MAX)
+		component = XE_RAS_COMP_NOT_SUPPORTED;
+
+	return xe_ras_components[component];
+}
+
+void xe_ras_counter_threshold_crossed(struct xe_device *xe,
+				      struct xe_sysctrl_event_response *response)
+{
+	struct xe_ras_threshold_crossed *pending = (void *)&response->data;
+	struct xe_ras_error_class *errors = pending->counters;
+	u32 id, ncounters = pending->ncounters;
+
+	BUILD_BUG_ON(sizeof(response->data) < sizeof(*pending));
+	xe_device_assert_mem_access(xe);
+
+	if (!ncounters || ncounters > XE_RAS_NUM_COUNTERS)
+		xe_err(xe, "sysctrl: unexpected counter threshold crossed %u\n", ncounters);
+	else
+		xe_warn(xe, "[RAS]: counter threshold crossed, %u new errors\n", ncounters);
+
+	for (id = 0; id < ncounters && id < XE_RAS_NUM_COUNTERS; id++) {
+		u8 severity, component;
+
+		severity = errors[id].common.severity;
+		component = errors[id].common.component;
+
+		xe_warn(xe, "[RAS]: %s %s detected\n",
+			comp_to_str(component), sev_to_str(severity));
+	}
+}
diff --git a/drivers/gpu/drm/xe/xe_ras.h b/drivers/gpu/drm/xe/xe_ras.h
new file mode 100644
index 0000000000000..ea90593b62dc1
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_ras.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_RAS_H_
+#define _XE_RAS_H_
+
+struct xe_device;
+struct xe_sysctrl_event_response;
+
+void xe_ras_counter_threshold_crossed(struct xe_device *xe,
+				      struct xe_sysctrl_event_response *response);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_ras_types.h b/drivers/gpu/drm/xe/xe_ras_types.h
new file mode 100644
index 0000000000000..4e63c67f806a2
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_ras_types.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_RAS_TYPES_H_
+#define _XE_RAS_TYPES_H_
+
+#include <linux/types.h>
+
+#define XE_RAS_NUM_COUNTERS			16
+
+/**
+ * struct xe_ras_error_common - Error fields that are common across all products
+ */
+struct xe_ras_error_common {
+	/** @severity: Error severity */
+	u8 severity;
+	/** @component: IP block where error originated */
+	u8 component;
+} __packed;
+
+/**
+ * struct xe_ras_error_unit - Error unit information
+ */
+struct xe_ras_error_unit {
+	/** @tile: Tile identifier */
+	u8 tile;
+	/** @instance: Instance identifier specific to IP */
+	u32 instance;
+} __packed;
+
+/**
+ * struct xe_ras_error_cause - Error cause information
+ */
+struct xe_ras_error_cause {
+	/** @cause: Cause/checker */
+	u32 cause;
+	/** @reserved: For future use */
+	u8 reserved;
+} __packed;
+
+/**
+ * struct xe_ras_error_product - Error fields that are specific to the product
+ */
+struct xe_ras_error_product {
+	/** @unit: Unit within IP block */
+	struct xe_ras_error_unit unit;
+	/** @cause: Cause/checker */
+	struct xe_ras_error_cause cause;
+} __packed;
+
+/**
+ * struct xe_ras_error_class - Combines common and product-specific parts
+ */
+struct xe_ras_error_class {
+	/** @common: Common error type and component */
+	struct xe_ras_error_common common;
+	/** @product: Product-specific unit and cause */
+	struct xe_ras_error_product product;
+} __packed;
+
+/**
+ * struct xe_ras_threshold_crossed - Data for threshold crossed event
+ */
+struct xe_ras_threshold_crossed {
+	/** @ncounters: Number of error counters that crossed thresholds */
+	u32 ncounters;
+	/** @counters: Array of error counters that crossed threshold */
+	struct xe_ras_error_class counters[XE_RAS_NUM_COUNTERS];
+} __packed;
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c
index 2df0277efb2f5..e328f50725572 100644
--- a/drivers/gpu/drm/xe/xe_reg_sr.c
+++ b/drivers/gpu/drm/xe/xe_reg_sr.c
@@ -70,14 +70,49 @@ static void reg_sr_inc_error(struct xe_reg_sr *sr)
 #endif
 }
 
+static struct xe_reg sanitize_mcr(struct xe_reg_sr *sr,
+				  const struct xe_reg_sr_entry *e,
+				  struct xe_gt *gt)
+{
+	struct xe_reg reg = e->reg;
+	bool is_mcr;
+
+	/*
+	 * We need the gt structure to check MCR ranges.
+	 */
+	if (!gt)
+		return reg;
+
+	is_mcr = xe_gt_mcr_check_reg(gt, reg);
+
+	if (is_mcr && !reg.mcr) {
+		reg.mcr = 1;
+		xe_gt_notice(gt, "xe_reg_sr_entry using non-MCR register for address 0x%x, forcing MCR\n",
+			     reg.addr);
+		reg_sr_inc_error(sr);
+	}
+
+	if (!is_mcr && reg.mcr) {
+		reg.mcr = 0;
+		xe_gt_notice(gt, "xe_reg_sr_entry using MCR register for address 0x%x, forcing non-MCR\n",
+			     reg.addr);
+		reg_sr_inc_error(sr);
+	}
+
+	return reg;
+}
+
 int xe_reg_sr_add(struct xe_reg_sr *sr,
 		  const struct xe_reg_sr_entry *e,
 		  struct xe_gt *gt)
 {
 	unsigned long idx = e->reg.addr;
 	struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx);
+	struct xe_reg reg;
 	int ret;
 
+	reg = sanitize_mcr(sr, e, gt);
+
 	if (pentry) {
 		if (!compatible_entries(pentry, e)) {
 			ret = -EINVAL;
@@ -98,6 +133,7 @@ int xe_reg_sr_add(struct xe_reg_sr *sr,
 	}
 
 	*pentry = *e;
+	pentry->reg = reg;
 	ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL));
 	if (ret)
 		goto fail_free;
diff --git a/drivers/gpu/drm/xe/xe_reg_sr.h b/drivers/gpu/drm/xe/xe_reg_sr.h
index 1ec6e8ecf2784..d26cf47133831 100644
--- a/drivers/gpu/drm/xe/xe_reg_sr.h
+++ b/drivers/gpu/drm/xe/xe_reg_sr.h
@@ -3,8 +3,8 @@
  * Copyright © 2022 Intel Corporation
  */
 
-#ifndef _XE_REG_SR_
-#define _XE_REG_SR_
+#ifndef _XE_REG_SR_H_
+#define _XE_REG_SR_H_
 
 /*
  * Reg save/restore bookkeeping
diff --git a/drivers/gpu/drm/xe/xe_reg_sr_types.h b/drivers/gpu/drm/xe/xe_reg_sr_types.h
index ebe11f237fa26..0a6695db29670 100644
--- a/drivers/gpu/drm/xe/xe_reg_sr_types.h
+++ b/drivers/gpu/drm/xe/xe_reg_sr_types.h
@@ -3,8 +3,8 @@
  * Copyright © 2022 Intel Corporation
  */
 
-#ifndef _XE_REG_SR_TYPES_
-#define _XE_REG_SR_TYPES_
+#ifndef _XE_REG_SR_TYPES_H_
+#define _XE_REG_SR_TYPES_H_
 
 #include <linux/types.h>
 #include <linux/xarray.h>
diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c
index 8cc313182968d..fb65940848d7a 100644
--- a/drivers/gpu/drm/xe/xe_reg_whitelist.c
+++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c
@@ -9,6 +9,7 @@
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_oa_regs.h"
 #include "xe_device.h"
+#include "xe_gt.h"
 #include "xe_gt_types.h"
 #include "xe_gt_printk.h"
 #include "xe_platform_types.h"
@@ -33,6 +34,13 @@ static bool match_has_mert(const struct xe_device *xe,
 	return xe_device_has_mert((struct xe_device *)xe);
 }
 
+static bool match_multi_queue_class(const struct xe_device *xe,
+				    const struct xe_gt *gt,
+				    const struct xe_hw_engine *hwe)
+{
+	return xe_gt_supports_multi_queue(gt, hwe->class);
+}
+
 static const struct xe_rtp_entry_sr register_whitelist[] = {
 	{ XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)),
@@ -54,6 +62,12 @@ static const struct xe_rtp_entry_sr register_whitelist[] = {
 				RING_FORCE_TO_NONPRIV_ACCESS_RD,
 				XE_RTP_ACTION_FLAG(ENGINE_BASE)))
 	},
+	{ XE_RTP_NAME("allow_read_queue_timestamp"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3500, 3511), FUNC(match_multi_queue_class)),
+	  XE_RTP_ACTIONS(WHITELIST(RING_QUEUE_TIMESTAMP(0),
+				   RING_FORCE_TO_NONPRIV_ACCESS_RD,
+				   XE_RTP_ACTION_FLAG(ENGINE_BASE)))
+	},
 	{ XE_RTP_NAME("16014440446"),
 	  XE_RTP_RULES(PLATFORM(PVC)),
 	  XE_RTP_ACTIONS(WHITELIST(XE_REG(0x4400),
diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.h b/drivers/gpu/drm/xe/xe_reg_whitelist.h
index 69b121d377da0..3b64b42fe96e9 100644
--- a/drivers/gpu/drm/xe/xe_reg_whitelist.h
+++ b/drivers/gpu/drm/xe/xe_reg_whitelist.h
@@ -3,8 +3,8 @@
  * Copyright © 2023 Intel Corporation
  */
 
-#ifndef _XE_REG_WHITELIST_
-#define _XE_REG_WHITELIST_
+#ifndef _XE_REG_WHITELIST_H_
+#define _XE_REG_WHITELIST_H_
 
 #include <linux/types.h>
 
diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h
index 5f4ab08c06869..0522caafd89d7 100644
--- a/drivers/gpu/drm/xe/xe_res_cursor.h
+++ b/drivers/gpu/drm/xe/xe_res_cursor.h
@@ -101,7 +101,15 @@ static inline void xe_res_first(struct ttm_resource *res,
 	cur->mem_type = res->mem_type;
 
 	switch (cur->mem_type) {
-	case XE_PL_STOLEN:
+	case XE_PL_STOLEN: {
+		/* res->start is in pages (ttm_range_manager). */
+		cur->start = (res->start << PAGE_SHIFT) + start;
+		cur->size = size;
+		cur->remaining = size;
+		cur->node = NULL;
+		cur->mm = NULL;
+		break;
+	}
 	case XE_PL_VRAM0:
 	case XE_PL_VRAM1: {
 		struct gpu_buddy_block *block;
@@ -289,6 +297,10 @@ static inline void xe_res_next(struct xe_res_cursor *cur, u64 size)
 
 	switch (cur->mem_type) {
 	case XE_PL_STOLEN:
+		/* Just advance within the contiguous region. */
+		cur->start += size;
+		cur->size = cur->remaining;
+		break;
 	case XE_PL_VRAM0:
 	case XE_PL_VRAM1:
 		start = size - cur->size;
diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c
index cfeb4fc7d2177..39a670e91ba72 100644
--- a/drivers/gpu/drm/xe/xe_ring_ops.c
+++ b/drivers/gpu/drm/xe/xe_ring_ops.c
@@ -269,8 +269,12 @@ static u32 get_ppgtt_flag(struct xe_sched_job *job)
 static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc,
 			       u32 *dw, int i)
 {
+	const struct xe_reg reg = xe_lrc_is_multi_queue(lrc) ?
+				   RING_QUEUE_TIMESTAMP(0) :
+				   RING_CTX_TIMESTAMP(0);
+
 	dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT | MI_SRM_ADD_CS_OFFSET;
-	dw[i++] = RING_CTX_TIMESTAMP(0).addr;
+	dw[i++] = reg.addr;
 	dw[i++] = xe_lrc_ctx_job_timestamp_ggtt_addr(lrc);
 	dw[i++] = 0;
 
@@ -281,7 +285,7 @@ static int emit_copy_timestamp(struct xe_device *xe, struct xe_lrc *lrc,
 	if (IS_SRIOV_VF(xe)) {
 		dw[i++] = MI_STORE_REGISTER_MEM | MI_SRM_USE_GGTT |
 			MI_SRM_ADD_CS_OFFSET;
-		dw[i++] = RING_CTX_TIMESTAMP(0).addr;
+		dw[i++] = reg.addr;
 		dw[i++] = xe_lrc_ctx_timestamp_ggtt_addr(lrc);
 		dw[i++] = 0;
 	}
diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c
index 991f218f1cc30..1a4dcbbbc1764 100644
--- a/drivers/gpu/drm/xe/xe_rtp.c
+++ b/drivers/gpu/drm/xe/xe_rtp.c
@@ -10,6 +10,7 @@
 #include <uapi/drm/xe_drm.h>
 
 #include "xe_configfs.h"
+#include "xe_device.h"
 #include "xe_gt.h"
 #include "xe_gt_topology.h"
 #include "xe_reg_sr.h"
@@ -352,6 +353,13 @@ void xe_rtp_process(struct xe_rtp_process_ctx *ctx,
 }
 EXPORT_SYMBOL_IF_KUNIT(xe_rtp_process);
 
+bool xe_rtp_match_always(const struct xe_device *xe,
+			 const struct xe_gt *gt,
+			 const struct xe_hw_engine *hwe)
+{
+	return true;
+}
+
 bool xe_rtp_match_even_instance(const struct xe_device *xe,
 				const struct xe_gt *gt,
 				const struct xe_hw_engine *hwe)
@@ -397,3 +405,10 @@ bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe,
 {
 	return xe->info.has_flat_ccs;
 }
+
+bool xe_rtp_match_has_msix(const struct xe_device *xe,
+			   const struct xe_gt *gt,
+			   const struct xe_hw_engine *hwe)
+{
+	return xe_device_has_msix(xe);
+}
diff --git a/drivers/gpu/drm/xe/xe_rtp.h b/drivers/gpu/drm/xe/xe_rtp.h
index 7d6daa7eb1e4c..562082b18d7b8 100644
--- a/drivers/gpu/drm/xe/xe_rtp.h
+++ b/drivers/gpu/drm/xe/xe_rtp.h
@@ -3,8 +3,8 @@
  * Copyright © 2022 Intel Corporation
  */
 
-#ifndef _XE_RTP_
-#define _XE_RTP_
+#ifndef _XE_RTP_H_
+#define _XE_RTP_H_
 
 #include <linux/types.h>
 #include <linux/xarray.h>
@@ -459,6 +459,18 @@ void xe_rtp_process(struct xe_rtp_process_ctx *ctx,
 
 /* Match functions to be used with XE_RTP_MATCH_FUNC */
 
+/**
+ * xe_rtp_match_always - Match RTP entry unconditionally
+ * @xe: Device structure
+ * @gt: GT structure
+ * @hwe: Engine instance
+ *
+ * Returns: true, regardless of inputs
+ */
+bool xe_rtp_match_always(const struct xe_device *xe,
+			 const struct xe_gt *gt,
+			 const struct xe_hw_engine *hwe);
+
 /**
  * xe_rtp_match_even_instance - Match if engine instance is even
  * @xe: Device structure
@@ -524,4 +536,16 @@ bool xe_rtp_match_has_flat_ccs(const struct xe_device *xe,
 			       const struct xe_gt *gt,
 			       const struct xe_hw_engine *hwe);
 
+/**
+ * xe_rtp_match_has_msix - Match when platform has MSI-X
+ * @xe: Device structure
+ * @gt: GT structure
+ * @hwe: Engine instance
+ *
+ * Returns: true if platform has MSI-X interrupt support
+ */
+bool xe_rtp_match_has_msix(const struct xe_device *xe,
+			   const struct xe_gt *gt,
+			   const struct xe_hw_engine *hwe);
+
 #endif
diff --git a/drivers/gpu/drm/xe/xe_rtp_helpers.h b/drivers/gpu/drm/xe/xe_rtp_helpers.h
index a33b0ae98bbca..ffa8b2f8828d1 100644
--- a/drivers/gpu/drm/xe/xe_rtp_helpers.h
+++ b/drivers/gpu/drm/xe/xe_rtp_helpers.h
@@ -3,8 +3,8 @@
  * Copyright © 2023 Intel Corporation
  */
 
-#ifndef _XE_RTP_HELPERS_
-#define _XE_RTP_HELPERS_
+#ifndef _XE_RTP_HELPERS_H_
+#define _XE_RTP_HELPERS_H_
 
 #ifndef _XE_RTP_INCLUDE_PRIVATE_HELPERS
 #error "This header is supposed to be included by xe_rtp.h only"
@@ -66,6 +66,8 @@
 #define XE_RTP_PASTE_10(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_9(prefix_, sep_, _XE_TUPLE_TAIL args_)
 #define XE_RTP_PASTE_11(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_10(prefix_, sep_, _XE_TUPLE_TAIL args_)
 #define XE_RTP_PASTE_12(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_11(prefix_, sep_, _XE_TUPLE_TAIL args_)
+#define XE_RTP_PASTE_13(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_12(prefix_, sep_, _XE_TUPLE_TAIL args_)
+#define XE_RTP_PASTE_14(prefix_, sep_, args_) _XE_RTP_CONCAT(prefix_, FIRST_ARG args_) __XE_RTP_PASTE_SEP_ ## sep_ XE_RTP_PASTE_13(prefix_, sep_, _XE_TUPLE_TAIL args_)
 
 /*
  * XE_RTP_DROP_CAST - Drop cast to convert a compound statement to a initializer
diff --git a/drivers/gpu/drm/xe/xe_rtp_types.h b/drivers/gpu/drm/xe/xe_rtp_types.h
index 166251615be13..0265c16d27625 100644
--- a/drivers/gpu/drm/xe/xe_rtp_types.h
+++ b/drivers/gpu/drm/xe/xe_rtp_types.h
@@ -3,8 +3,8 @@
  * Copyright © 2022 Intel Corporation
  */
 
-#ifndef _XE_RTP_TYPES_
-#define _XE_RTP_TYPES_
+#ifndef _XE_RTP_TYPES_H_
+#define _XE_RTP_TYPES_H_
 
 #include <linux/types.h>
 
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c
index 47a6e0fd66e0f..33bd754d138fa 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf.c
@@ -20,11 +20,6 @@
 #include "xe_sriov_pf_sysfs.h"
 #include "xe_sriov_printk.h"
 
-static bool wanted_admin_only(struct xe_device *xe)
-{
-	return xe_configfs_admin_only_pf(to_pci_dev(xe->drm.dev));
-}
-
 static unsigned int wanted_max_vfs(struct xe_device *xe)
 {
 	return xe_configfs_get_max_vfs(to_pci_dev(xe->drm.dev));
@@ -79,7 +74,6 @@ bool xe_sriov_pf_readiness(struct xe_device *xe)
 
 	pf_reduce_totalvfs(xe, newlimit);
 
-	xe->sriov.pf.admin_only = wanted_admin_only(xe);
 	xe->sriov.pf.device_total_vfs = totalvfs;
 	xe->sriov.pf.driver_max_vfs = newlimit;
 
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h
index 0fcc6cec4afc4..19f6f8331c8df 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_helpers.h
@@ -7,6 +7,7 @@
 #define _XE_SRIOV_PF_HELPERS_H_
 
 #include "xe_assert.h"
+#include "xe_device.h"
 #include "xe_device_types.h"
 #include "xe_sriov.h"
 #include "xe_sriov_types.h"
@@ -57,7 +58,7 @@ static inline unsigned int xe_sriov_pf_num_vfs(const struct xe_device *xe)
 static inline bool xe_sriov_pf_admin_only(const struct xe_device *xe)
 {
 	xe_assert(xe, IS_SRIOV_PF(xe));
-	return xe->sriov.pf.admin_only;
+	return xe_device_is_admin_only(xe);
 }
 
 static inline struct mutex *xe_sriov_pf_master_mutex(struct xe_device *xe)
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c
index abe3677d33edf..0ec7ea83f12ac 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.c
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.c
@@ -41,6 +41,8 @@ static int pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs)
 	int err;
 
 	for_each_gt(gt, xe, id) {
+		err = xe_gt_sriov_pf_config_set_fair_sched(gt, num_vfs);
+		result = result ?: err;
 		err = xe_gt_sriov_pf_config_set_fair(gt, VFID(1), num_vfs);
 		result = result ?: err;
 	}
@@ -103,6 +105,45 @@ int xe_sriov_pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs)
 	return 0;
 }
 
+static int pf_reprovision_default(struct xe_device *xe)
+{
+	struct xe_gt *gt;
+	unsigned int id;
+	int result = 0;
+	int err;
+
+	guard(mutex)(xe_sriov_pf_master_mutex(xe));
+
+	for_each_gt(gt, xe, id) {
+		err = xe_gt_sriov_pf_policy_set_sched_if_idle_locked(gt, false);
+		result = result ?: err;
+		err = xe_gt_sriov_pf_config_set_exec_quantum_locked(gt, PFID, 0);
+		result = result ?: err;
+		err = xe_gt_sriov_pf_config_set_preempt_timeout_locked(gt, PFID, 0);
+		result = result ?: err;
+	}
+
+	return result;
+}
+
+/**
+ * xe_sriov_pf_reprovision_default() - Reprovision default PF in auto-mode.
+ * @xe: the PF &xe_device
+ *
+ * This function can only be called on PF.
+ *
+ * Return: 0 on success or a negative error code on failure.
+ */
+int xe_sriov_pf_reprovision_default(struct xe_device *xe)
+{
+	xe_assert(xe, IS_SRIOV_PF(xe));
+
+	if (!pf_auto_provisioning_mode(xe))
+		return 0;
+
+	return pf_reprovision_default(xe);
+}
+
 /**
  * xe_sriov_pf_provision_set_mode() - Change VFs provision mode.
  * @xe: the PF &xe_device
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_provision.h b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h
index f26f49539697b..b15e4d7ad9404 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_provision.h
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_provision.h
@@ -30,6 +30,7 @@ int xe_sriov_pf_provision_query_vf_vram(struct xe_device *xe, unsigned int vfid,
 
 int xe_sriov_pf_provision_vfs(struct xe_device *xe, unsigned int num_vfs);
 int xe_sriov_pf_unprovision_vfs(struct xe_device *xe, unsigned int num_vfs);
+int xe_sriov_pf_reprovision_default(struct xe_device *xe);
 
 int xe_sriov_pf_provision_set_mode(struct xe_device *xe, enum xe_sriov_provisioning_mode mode);
 
diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_types.h
index 080cf10512f4b..b0253e1ae5da4 100644
--- a/drivers/gpu/drm/xe/xe_sriov_pf_types.h
+++ b/drivers/gpu/drm/xe/xe_sriov_pf_types.h
@@ -36,9 +36,6 @@ struct xe_sriov_metadata {
  * @XE_SRIOV_MODE_PF mode.
  */
 struct xe_device_pf {
-	/** @admin_only: PF functionality focused on VFs management only. */
-	bool admin_only;
-
 	/** @device_total_vfs: Maximum number of VFs supported by the device. */
 	u16 device_total_vfs;
 
diff --git a/drivers/gpu/drm/xe/xe_step.c b/drivers/gpu/drm/xe/xe_step.c
index d0f888c31831f..fb9c31613ca7d 100644
--- a/drivers/gpu/drm/xe/xe_step.c
+++ b/drivers/gpu/drm/xe/xe_step.c
@@ -278,7 +278,7 @@ void xe_step_gmdid_get(struct xe_device *xe,
 	case STEP_##name:	\
 		return #name;
 
-const char *xe_step_name(enum xe_step step)
+const char *xe_step_name(enum intel_step step)
 {
 	switch (step) {
 	STEP_NAME_LIST(STEP_NAME_CASE);
diff --git a/drivers/gpu/drm/xe/xe_step.h b/drivers/gpu/drm/xe/xe_step.h
index 41f1c95c46e5c..ea36b22cc297c 100644
--- a/drivers/gpu/drm/xe/xe_step.h
+++ b/drivers/gpu/drm/xe/xe_step.h
@@ -18,8 +18,8 @@ void xe_step_pre_gmdid_get(struct xe_device *xe);
 void xe_step_gmdid_get(struct xe_device *xe,
 		       u32 graphics_gmdid_revid,
 		       u32 media_gmdid_revid);
-static inline u32 xe_step_to_gmdid(enum xe_step step) { return step - STEP_A0; }
+static inline u32 xe_step_to_gmdid(enum intel_step step) { return step - STEP_A0; }
 
-const char *xe_step_name(enum xe_step step);
+const char *xe_step_name(enum intel_step step);
 
 #endif
diff --git a/drivers/gpu/drm/xe/xe_step_types.h b/drivers/gpu/drm/xe/xe_step_types.h
index 43ca738507396..f60572b935233 100644
--- a/drivers/gpu/drm/xe/xe_step_types.h
+++ b/drivers/gpu/drm/xe/xe_step_types.h
@@ -8,6 +8,8 @@
 
 #include <linux/types.h>
 
+#include <drm/intel/step.h>
+
 struct xe_step_info {
 	u8 platform;
 	u8 graphics;
@@ -15,63 +17,4 @@ struct xe_step_info {
 	u8 basedie;
 };
 
-#define STEP_ENUM_VAL(name)  STEP_##name,
-
-/*
- * Always define four minor steppings 0-3 for each stepping to match GMD ID
- * spacing of values. See xe_step_gmdid_get().
- */
-#define STEP_NAME_LIST(func)		\
-	func(A0)			\
-	func(A1)			\
-	func(A2)			\
-	func(A3)			\
-	func(B0)			\
-	func(B1)			\
-	func(B2)			\
-	func(B3)			\
-	func(C0)			\
-	func(C1)			\
-	func(C2)			\
-	func(C3)			\
-	func(D0)			\
-	func(D1)			\
-	func(D2)			\
-	func(D3)			\
-	func(E0)			\
-	func(E1)			\
-	func(E2)			\
-	func(E3)			\
-	func(F0)			\
-	func(F1)			\
-	func(F2)			\
-	func(F3)			\
-	func(G0)			\
-	func(G1)			\
-	func(G2)			\
-	func(G3)			\
-	func(H0)			\
-	func(H1)			\
-	func(H2)			\
-	func(H3)			\
-	func(I0)			\
-	func(I1)			\
-	func(I2)			\
-	func(I3)			\
-	func(J0)			\
-	func(J1)			\
-	func(J2)			\
-	func(J3)
-
-/*
- * Symbolic steppings that do not match the hardware. These are valid both as gt
- * and display steppings as symbolic names.
- */
-enum xe_step {
-	STEP_NONE = 0,
-	STEP_NAME_LIST(STEP_ENUM_VAL)
-	STEP_FUTURE,
-	STEP_FOREVER,
-};
-
 #endif
diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c
index db64cac39c944..427afd144f3ac 100644
--- a/drivers/gpu/drm/xe/xe_survivability_mode.c
+++ b/drivers/gpu/drm/xe/xe_survivability_mode.c
@@ -396,25 +396,21 @@ bool xe_survivability_mode_is_requested(struct xe_device *xe)
  * Runtime survivability mode is enabled when certain errors cause the device to be
  * in non-recoverable state. The device is declared wedged with the appropriate
  * recovery method and survivability mode sysfs exposed to userspace
- *
- * Return: 0 if runtime survivability mode is enabled, negative error code otherwise.
  */
-int xe_survivability_mode_runtime_enable(struct xe_device *xe)
+void xe_survivability_mode_runtime_enable(struct xe_device *xe)
 {
 	struct xe_survivability *survivability = &xe->survivability;
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
-	int ret;
 
 	if (!IS_DGFX(xe) || IS_SRIOV_VF(xe) || xe->info.platform < XE_BATTLEMAGE) {
 		dev_err(&pdev->dev, "Runtime Survivability Mode not supported\n");
-		return -EINVAL;
+		return;
 	}
 
 	populate_survivability_info(xe);
 
-	ret = create_survivability_sysfs(pdev);
-	if (ret)
-		dev_err(&pdev->dev, "Failed to create survivability mode sysfs\n");
+	if (create_survivability_sysfs(pdev))
+		dev_err(&pdev->dev, "Failed to create survivability sysfs\n");
 
 	survivability->type = XE_SURVIVABILITY_TYPE_RUNTIME;
 	dev_err(&pdev->dev, "Runtime Survivability mode enabled\n");
@@ -422,8 +418,6 @@ int xe_survivability_mode_runtime_enable(struct xe_device *xe)
 	xe_device_set_wedged_method(xe, DRM_WEDGE_RECOVERY_VENDOR);
 	xe_device_declare_wedged(xe);
 	dev_err(&pdev->dev, "Firmware flash required, Please refer to the userspace documentation for more details!\n");
-
-	return 0;
 }
 
 /**
diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.h b/drivers/gpu/drm/xe/xe_survivability_mode.h
index 1cc94226aa82d..cd040e4d18bb8 100644
--- a/drivers/gpu/drm/xe/xe_survivability_mode.h
+++ b/drivers/gpu/drm/xe/xe_survivability_mode.h
@@ -11,7 +11,7 @@
 struct xe_device;
 
 int xe_survivability_mode_boot_enable(struct xe_device *xe);
-int xe_survivability_mode_runtime_enable(struct xe_device *xe);
+void xe_survivability_mode_runtime_enable(struct xe_device *xe);
 bool xe_survivability_mode_is_boot_enabled(struct xe_device *xe);
 bool xe_survivability_mode_is_requested(struct xe_device *xe);
 
diff --git a/drivers/gpu/drm/xe/xe_svm.c b/drivers/gpu/drm/xe/xe_svm.c
index 5933b2b6392b1..e1651e70c8f01 100644
--- a/drivers/gpu/drm/xe/xe_svm.c
+++ b/drivers/gpu/drm/xe/xe_svm.c
@@ -786,12 +786,12 @@ static int xe_svm_populate_devmem_pfn(struct drm_pagemap_devmem *devmem_allocati
 	struct xe_bo *bo = to_xe_bo(devmem_allocation);
 	struct ttm_resource *res = bo->ttm.resource;
 	struct list_head *blocks = &to_xe_ttm_vram_mgr_resource(res)->blocks;
+	struct xe_vram_region *vr = xe_map_resource_to_region(res);
+	struct gpu_buddy *buddy = vram_to_buddy(vr);
 	struct gpu_buddy_block *block;
 	int j = 0;
 
 	list_for_each_entry(block, blocks, link) {
-		struct xe_vram_region *vr = block->private;
-		struct gpu_buddy *buddy = vram_to_buddy(vr);
 		u64 block_pfn = block_offset_to_pfn(devmem_allocation->dpagemap,
 						    gpu_buddy_block_offset(block));
 		int i;
@@ -1055,15 +1055,12 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 	struct xe_pagemap *xpagemap = container_of(dpagemap, typeof(*xpagemap), dpagemap);
 	struct drm_pagemap_migrate_details mdetails = {
 		.timeslice_ms = timeslice_ms,
-		.source_peer_migrates = 1,
 	};
 	struct xe_vram_region *vr = xe_pagemap_to_vr(xpagemap);
 	struct dma_fence *pre_migrate_fence = NULL;
 	struct xe_device *xe = vr->xe;
 	struct device *dev = xe->drm.dev;
-	struct gpu_buddy_block *block;
 	struct xe_validation_ctx vctx;
-	struct list_head *blocks;
 	struct drm_exec exec;
 	struct xe_bo *bo;
 	int err = 0, idx;
@@ -1100,10 +1097,6 @@ static int xe_drm_pagemap_populate_mm(struct drm_pagemap *dpagemap,
 					&dpagemap_devmem_ops, dpagemap, end - start,
 					pre_migrate_fence);
 
-		blocks = &to_xe_ttm_vram_mgr_resource(bo->ttm.resource)->blocks;
-		list_for_each_entry(block, blocks, link)
-			block->private = vr;
-
 		xe_bo_get(bo);
 
 		/* Ensure the device has a pm ref while there are device pages active. */
diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c
index 24d6d9af20d6f..37866768d64c2 100644
--- a/drivers/gpu/drm/xe/xe_sync.c
+++ b/drivers/gpu/drm/xe/xe_sync.c
@@ -376,7 +376,7 @@ xe_sync_in_fence_get(struct xe_sync_entry *sync, int num_sync,
 		xe_assert(vm->xe, current_fence == num_fence);
 		cf = dma_fence_array_create(num_fence, fences,
 					    dma_fence_context_alloc(1),
-					    1, false);
+					    1);
 		if (!cf)
 			goto err_out;
 
diff --git a/drivers/gpu/drm/xe/xe_sysctrl.c b/drivers/gpu/drm/xe/xe_sysctrl.c
new file mode 100644
index 0000000000000..1db20be8158b5
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sysctrl.c
@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <linux/device.h>
+#include <linux/mutex.h>
+
+#include <drm/drm_managed.h>
+
+#include "regs/xe_irq_regs.h"
+#include "regs/xe_sysctrl_regs.h"
+#include "xe_device.h"
+#include "xe_mmio.h"
+#include "xe_pm.h"
+#include "xe_soc_remapper.h"
+#include "xe_sysctrl.h"
+#include "xe_sysctrl_mailbox.h"
+#include "xe_sysctrl_types.h"
+
+/**
+ * DOC: System Controller (sysctrl)
+ *
+ * System Controller (sysctrl) is a firmware-managed entity on Intel dGPUs
+ * responsible for selected low-level platform management functions.
+ * Communication between driver and System Controller is performed
+ * via a mailbox interface, enabling command and response exchange.
+ *
+ * This module provides initialization and support code for interacting
+ * with System Controller through the mailbox interface.
+ */
+static void sysctrl_fini(void *arg)
+{
+	struct xe_device *xe = arg;
+	struct xe_sysctrl *sc = &xe->sc;
+
+	disable_work_sync(&sc->work);
+	xe->soc_remapper.set_sysctrl_region(xe, 0);
+}
+
+static void xe_sysctrl_work(struct work_struct *work)
+{
+	struct xe_sysctrl *sc = container_of(work, struct xe_sysctrl, work);
+	struct xe_device *xe = sc_to_xe(sc);
+
+	guard(xe_pm_runtime)(xe);
+	xe_sysctrl_event(sc);
+}
+
+/**
+ * xe_sysctrl_init() - Initialize System Controller subsystem
+ * @xe: xe device instance
+ *
+ * Entry point for System Controller initialization, called from xe_device_probe.
+ * This function checks platform support and initializes the system controller.
+ *
+ * Return: 0 on success, error code on failure
+ */
+int xe_sysctrl_init(struct xe_device *xe)
+{
+	struct xe_tile *tile = xe_device_get_root_tile(xe);
+	struct xe_sysctrl *sc = &xe->sc;
+	int ret;
+
+	if (!xe->info.has_soc_remapper_sysctrl)
+		return 0;
+
+	if (!xe->info.has_sysctrl)
+		return 0;
+
+	sc->mmio = devm_kzalloc(xe->drm.dev, sizeof(*sc->mmio), GFP_KERNEL);
+	if (!sc->mmio)
+		return -ENOMEM;
+
+	xe_mmio_init(sc->mmio, tile, tile->mmio.regs, tile->mmio.regs_size);
+	sc->mmio->adj_offset = SYSCTRL_BASE;
+	sc->mmio->adj_limit = U32_MAX;
+
+	ret = devm_mutex_init(xe->drm.dev, &sc->cmd_lock);
+	if (ret)
+		return ret;
+
+	ret = devm_mutex_init(xe->drm.dev, &sc->event_lock);
+	if (ret)
+		return ret;
+
+	xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX);
+	xe_sysctrl_mailbox_init(sc);
+	INIT_WORK(&sc->work, xe_sysctrl_work);
+
+	return devm_add_action_or_reset(xe->drm.dev, sysctrl_fini, xe);
+}
+
+/**
+ * xe_sysctrl_irq_handler() - Handler for System Controller interrupts
+ * @xe: xe device instance
+ * @master_ctl: interrupt register
+ *
+ * Handle interrupts generated by System Controller.
+ */
+void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl)
+{
+	struct xe_sysctrl *sc = &xe->sc;
+
+	if (!xe->info.has_sysctrl || !sc->work.func)
+		return;
+
+	if (master_ctl & SYSCTRL_IRQ)
+		schedule_work(&sc->work);
+}
+
+/**
+ * xe_sysctrl_pm_resume() - System Controller resume handler
+ * @xe: xe device instance
+ *
+ * Invoked during system resume (S3/S4 to S0) and runtime resume from D3cold.
+ * Restores SoC remapper configuration and reinitializes mailbox interface.
+ */
+void xe_sysctrl_pm_resume(struct xe_device *xe)
+{
+	struct xe_sysctrl *sc = &xe->sc;
+
+	if (!xe->info.has_soc_remapper_sysctrl)
+		return;
+
+	if (!xe->info.has_sysctrl)
+		return;
+
+	xe->soc_remapper.set_sysctrl_region(xe, SYSCTRL_MAILBOX_INDEX);
+
+	xe_sysctrl_mailbox_init(sc);
+}
diff --git a/drivers/gpu/drm/xe/xe_sysctrl.h b/drivers/gpu/drm/xe/xe_sysctrl.h
new file mode 100644
index 0000000000000..090dffb6d55f6
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sysctrl.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_SYSCTRL_H_
+#define _XE_SYSCTRL_H_
+
+#include <linux/container_of.h>
+
+#include "xe_device_types.h"
+#include "xe_sysctrl_types.h"
+
+static inline struct xe_device *sc_to_xe(struct xe_sysctrl *sc)
+{
+	return container_of(sc, struct xe_device, sc);
+}
+
+void xe_sysctrl_event(struct xe_sysctrl *sc);
+int xe_sysctrl_init(struct xe_device *xe);
+void xe_sysctrl_irq_handler(struct xe_device *xe, u32 master_ctl);
+void xe_sysctrl_pm_resume(struct xe_device *xe);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event.c b/drivers/gpu/drm/xe/xe_sysctrl_event.c
new file mode 100644
index 0000000000000..b4d17329af6c8
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sysctrl_event.c
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include "xe_device.h"
+#include "xe_irq.h"
+#include "xe_printk.h"
+#include "xe_ras.h"
+#include "xe_sysctrl.h"
+#include "xe_sysctrl_event_types.h"
+#include "xe_sysctrl_mailbox.h"
+#include "xe_sysctrl_mailbox_types.h"
+
+static void get_pending_event(struct xe_sysctrl *sc, struct xe_sysctrl_mailbox_command *command)
+{
+	struct xe_sysctrl_event_response *response = command->data_out;
+	struct xe_device *xe = sc_to_xe(sc);
+	u32 count = XE_SYSCTRL_EVENT_FLOOD;
+	size_t len;
+	int ret;
+
+	do {
+		memset(response, 0, sizeof(*response));
+
+		ret = xe_sysctrl_send_command(sc, command, &len);
+		if (ret) {
+			xe_err(xe, "sysctrl: failed to get pending event %d\n", ret);
+			return;
+		}
+
+		if (len != sizeof(*response)) {
+			xe_err(xe, "sysctrl: unexpected event response length %zu (expected %zu)\n",
+			       len, sizeof(*response));
+			return;
+		}
+
+		if (response->event == XE_SYSCTRL_EVENT_THRESHOLD_CROSSED)
+			xe_ras_counter_threshold_crossed(xe, response);
+		else
+			xe_warn(xe, "sysctrl: unexpected event %#x\n", response->event);
+
+		if (!--count) {
+			xe_err(xe, "sysctrl: event flooding\n");
+			return;
+		}
+
+		xe_dbg(xe, "sysctrl: %u events pending\n", response->count);
+	} while (response->count);
+}
+
+static void event_request_prepare(struct xe_device *xe, struct xe_sysctrl_app_msg_hdr *header,
+				  struct xe_sysctrl_event_request *request)
+{
+	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
+
+	header->data = REG_FIELD_PREP(APP_HDR_GROUP_ID_MASK, XE_SYSCTRL_GROUP_GFSP) |
+		       REG_FIELD_PREP(APP_HDR_COMMAND_MASK, XE_SYSCTRL_CMD_GET_PENDING_EVENT);
+
+	request->vector = xe_device_has_msix(xe) ? XE_IRQ_DEFAULT_MSIX : 0;
+	request->fn = PCI_FUNC(pdev->devfn);
+}
+
+/**
+ * xe_sysctrl_event() - Handler for System Controller events
+ * @sc: System Controller instance
+ *
+ * Handle events generated by System Controller.
+ */
+void xe_sysctrl_event(struct xe_sysctrl *sc)
+{
+	struct xe_sysctrl_mailbox_command command = {};
+	struct xe_sysctrl_event_response response = {};
+	struct xe_sysctrl_event_request request = {};
+	struct xe_sysctrl_app_msg_hdr header = {};
+
+	xe_device_assert_mem_access(sc_to_xe(sc));
+	event_request_prepare(sc_to_xe(sc), &header, &request);
+
+	command.header = header;
+	command.data_in = &request;
+	command.data_in_len = sizeof(request);
+	command.data_out = &response;
+	command.data_out_len = sizeof(response);
+
+	guard(mutex)(&sc->event_lock);
+	get_pending_event(sc, &command);
+}
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_event_types.h b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h
new file mode 100644
index 0000000000000..c16c66b9fa7f7
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sysctrl_event_types.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_SYSCTRL_EVENT_TYPES_H_
+#define _XE_SYSCTRL_EVENT_TYPES_H_
+
+#include <linux/types.h>
+
+#define XE_SYSCTRL_EVENT_DATA_LEN		59
+
+/* Modify as needed */
+#define XE_SYSCTRL_EVENT_FLOOD			16
+
+/**
+ * enum xe_sysctrl_event - Events reported by System Controller
+ *
+ * @XE_SYSCTRL_EVENT_THRESHOLD_CROSSED: Error counter threshold crossed
+ */
+enum xe_sysctrl_event {
+	XE_SYSCTRL_EVENT_THRESHOLD_CROSSED	= 0x01,
+};
+
+/**
+ * struct xe_sysctrl_event_request - Request structure for pending event
+ */
+struct xe_sysctrl_event_request {
+	/** @vector: MSI-X vector that was triggered */
+	u32 vector;
+	/** @fn: Function index (0-7) of PCIe device */
+	u32 fn:8;
+	/** @reserved: Reserved for future use */
+	u32 reserved:24;
+	/** @reserved1: Reserved for future use */
+	u32 reserved1[2];
+} __packed;
+
+/**
+ * struct xe_sysctrl_event_response - Response structure for pending event
+ */
+struct xe_sysctrl_event_response {
+	/** @count: Pending event count after this response */
+	u32 count;
+	/** @event: Pending event type */
+	u32 event;
+	/** @timestamp: Timestamp of most recent event */
+	u64 timestamp;
+	/** @extended: Event has extended payload */
+	u32 extended:1;
+	/** @reserved: Reserved for future use */
+	u32 reserved:31;
+	/** @data: Generic event data */
+	u32 data[XE_SYSCTRL_EVENT_DATA_LEN];
+} __packed;
+
+#endif /* _XE_SYSCTRL_EVENT_TYPES_H_ */
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
new file mode 100644
index 0000000000000..3caa9f15875f5
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.c
@@ -0,0 +1,371 @@
+// SPDX-License-Identifier: MIT
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <linux/cleanup.h>
+#include <linux/minmax.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+
+#include "regs/xe_sysctrl_regs.h"
+#include "xe_device.h"
+#include "xe_mmio.h"
+#include "xe_pm.h"
+#include "xe_printk.h"
+#include "xe_sysctrl.h"
+#include "xe_sysctrl_mailbox.h"
+#include "xe_sysctrl_mailbox_types.h"
+
+struct xe_sysctrl_mailbox_msg_hdr {
+	__le32 data;
+} __packed;
+
+#define XE_SYSCTRL_HDR_GROUP_ID(hdr) \
+	FIELD_GET(SYSCTRL_HDR_GROUP_ID_MASK, le32_to_cpu((hdr)->data))
+
+#define XE_SYSCTRL_HDR_COMMAND(hdr) \
+	FIELD_GET(SYSCTRL_HDR_COMMAND_MASK, le32_to_cpu((hdr)->data))
+
+#define XE_SYSCTRL_HDR_IS_RESPONSE(hdr) \
+	FIELD_GET(SYSCTRL_HDR_IS_RESPONSE, le32_to_cpu((hdr)->data))
+
+#define XE_SYSCTRL_HDR_RESULT(hdr) \
+	FIELD_GET(SYSCTRL_HDR_RESULT_MASK, le32_to_cpu((hdr)->data))
+
+static bool sysctrl_wait_bit_clear(struct xe_sysctrl *sc, u32 bit_mask,
+				   unsigned int timeout_ms)
+{
+	int ret;
+
+	ret = xe_mmio_wait32_not(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask,
+				 timeout_ms * 1000, NULL, false);
+
+	return ret == 0;
+}
+
+static bool sysctrl_wait_bit_set(struct xe_sysctrl *sc, u32 bit_mask,
+				 unsigned int timeout_ms)
+{
+	int ret;
+
+	ret = xe_mmio_wait32(sc->mmio, SYSCTRL_MB_CTRL, bit_mask, bit_mask,
+			     timeout_ms * 1000, NULL, false);
+
+	return ret == 0;
+}
+
+static int sysctrl_write_frame(struct xe_sysctrl *sc, const void *frame,
+			       size_t len)
+{
+	static const struct xe_reg regs[] = {
+		SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3
+	};
+	struct xe_device *xe = sc_to_xe(sc);
+	u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0};
+	u32 dw = DIV_ROUND_UP(len, sizeof(u32));
+	u32 i;
+
+	xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE);
+
+	memcpy(val, frame, len);
+
+	for (i = 0; i < dw; i++)
+		xe_mmio_write32(sc->mmio, regs[i], val[i]);
+
+	return 0;
+}
+
+static int sysctrl_read_frame(struct xe_sysctrl *sc, void *frame,
+			      size_t len)
+{
+	static const struct xe_reg regs[] = {
+		SYSCTRL_MB_DATA0, SYSCTRL_MB_DATA1, SYSCTRL_MB_DATA2, SYSCTRL_MB_DATA3
+	};
+	struct xe_device *xe = sc_to_xe(sc);
+	u32 val[XE_SYSCTRL_MB_FRAME_SIZE / sizeof(u32)] = {0};
+	u32 dw = DIV_ROUND_UP(len, sizeof(u32));
+	u32 i;
+
+	xe_assert(xe, len > 0 && len <= XE_SYSCTRL_MB_FRAME_SIZE);
+
+	for (i = 0; i < dw; i++)
+		val[i] = xe_mmio_read32(sc->mmio, regs[i]);
+
+	memcpy(frame, val, len);
+
+	return 0;
+}
+
+static void sysctrl_clear_response(struct xe_sysctrl *sc)
+{
+	xe_mmio_rmw32(sc->mmio, SYSCTRL_MB_CTRL, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, 0);
+}
+
+static int sysctrl_prepare_command(struct xe_device *xe,
+				   u8 group_id, u8 command,
+				   const void *data_in, size_t data_in_len,
+				   u8 **mbox_cmd, size_t *cmd_size)
+{
+	struct xe_sysctrl_mailbox_msg_hdr *hdr;
+	size_t size;
+	u8 *buffer;
+
+	xe_assert(xe, command <= SYSCTRL_HDR_COMMAND_MAX);
+
+	if (data_in_len > XE_SYSCTRL_MB_MAX_MESSAGE_SIZE - sizeof(*hdr)) {
+		xe_err(xe, "sysctrl: Input data too large: %zu bytes\n", data_in_len);
+		return -EINVAL;
+	}
+
+	size = sizeof(*hdr) + data_in_len;
+
+	buffer = kmalloc(size, GFP_KERNEL);
+	if (!buffer)
+		return -ENOMEM;
+
+	hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer;
+	hdr->data = cpu_to_le32(FIELD_PREP(SYSCTRL_HDR_GROUP_ID_MASK, group_id) |
+				     FIELD_PREP(SYSCTRL_HDR_COMMAND_MASK, command));
+
+	if (data_in && data_in_len)
+		memcpy(buffer + sizeof(*hdr), data_in, data_in_len);
+
+	*mbox_cmd = buffer;
+	*cmd_size = size;
+
+	return 0;
+}
+
+static int sysctrl_send_frames(struct xe_sysctrl *sc,
+			       const u8 *mbox_cmd,
+			       size_t cmd_size, unsigned int timeout_ms)
+{
+	struct xe_device *xe = sc_to_xe(sc);
+	u32 ctrl_reg, total_frames, frame;
+	size_t bytes_sent, frame_size;
+
+	total_frames = DIV_ROUND_UP(cmd_size, XE_SYSCTRL_MB_FRAME_SIZE);
+
+	if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) {
+		xe_err(xe, "sysctrl: Mailbox busy\n");
+		return -EBUSY;
+	}
+
+	sc->phase_bit ^= 1;
+	bytes_sent = 0;
+
+	for (frame = 0; frame < total_frames; frame++) {
+		frame_size = min_t(size_t, cmd_size - bytes_sent, XE_SYSCTRL_MB_FRAME_SIZE);
+
+		if (sysctrl_write_frame(sc, mbox_cmd + bytes_sent, frame_size)) {
+			xe_err(xe, "sysctrl: Failed to write frame %u\n", frame);
+			sc->phase_bit = 0;
+			return -EIO;
+		}
+
+		ctrl_reg = SYSCTRL_MB_CTRL_RUN_BUSY |
+			   REG_FIELD_PREP(SYSCTRL_FRAME_CURRENT_MASK, frame) |
+			   REG_FIELD_PREP(SYSCTRL_FRAME_TOTAL_MASK, total_frames - 1) |
+			   SYSCTRL_MB_CTRL_CMD |
+			   (sc->phase_bit ? SYSCTRL_FRAME_PHASE : 0);
+
+		xe_mmio_write32(sc->mmio, SYSCTRL_MB_CTRL, ctrl_reg);
+
+		if (!sysctrl_wait_bit_clear(sc, SYSCTRL_MB_CTRL_RUN_BUSY, timeout_ms)) {
+			xe_err(xe, "sysctrl: Frame %u acknowledgment timeout\n", frame);
+			sc->phase_bit = 0;
+			return -ETIMEDOUT;
+		}
+
+		bytes_sent += frame_size;
+	}
+
+	return 0;
+}
+
+static int sysctrl_process_frame(struct xe_sysctrl *sc, void *out,
+				 size_t frame_size, unsigned int timeout_ms,
+				 bool *done)
+{
+	u32 curr_frame, total_frames, ctrl_reg;
+	struct xe_device *xe = sc_to_xe(sc);
+	int ret;
+
+	if (!sysctrl_wait_bit_set(sc, SYSCTRL_MB_CTRL_RUN_BUSY_OUT, timeout_ms)) {
+		xe_err(xe, "sysctrl: Response frame timeout\n");
+		return -ETIMEDOUT;
+	}
+
+	ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL);
+	total_frames = FIELD_GET(SYSCTRL_FRAME_TOTAL_MASK, ctrl_reg);
+	curr_frame = FIELD_GET(SYSCTRL_FRAME_CURRENT_MASK, ctrl_reg);
+
+	ret = sysctrl_read_frame(sc, out, frame_size);
+	if (ret)
+		return ret;
+
+	sysctrl_clear_response(sc);
+
+	if (curr_frame == total_frames)
+		*done = true;
+
+	return 0;
+}
+
+static int sysctrl_receive_frames(struct xe_sysctrl *sc,
+				  const struct xe_sysctrl_mailbox_msg_hdr *req,
+				  void *data_out, size_t data_out_len,
+				  size_t *rdata_len, unsigned int timeout_ms)
+{
+	struct xe_sysctrl_mailbox_msg_hdr *hdr;
+	struct xe_device *xe = sc_to_xe(sc);
+	size_t remain = sizeof(*hdr) + data_out_len;
+	u8 *buffer __free(kfree) = kzalloc(remain, GFP_KERNEL);
+	size_t frame_size;
+	bool done = false;
+	int ret = 0;
+	u8 *out;
+
+	if (!buffer)
+		return -ENOMEM;
+
+	out = buffer;
+	while (!done && remain) {
+		frame_size = min_t(size_t, remain, XE_SYSCTRL_MB_FRAME_SIZE);
+
+		ret = sysctrl_process_frame(sc, out, frame_size, timeout_ms,
+					    &done);
+		if (ret)
+			return ret;
+
+		remain -= frame_size;
+		out += frame_size;
+	}
+
+	hdr = (struct xe_sysctrl_mailbox_msg_hdr *)buffer;
+
+	if (!XE_SYSCTRL_HDR_IS_RESPONSE(hdr) ||
+	    XE_SYSCTRL_HDR_GROUP_ID(hdr) != XE_SYSCTRL_HDR_GROUP_ID(req) ||
+	    XE_SYSCTRL_HDR_COMMAND(hdr) != XE_SYSCTRL_HDR_COMMAND(req)) {
+		xe_err(xe, "sysctrl: Response header mismatch\n");
+		return -EPROTO;
+	}
+
+	if (XE_SYSCTRL_HDR_RESULT(hdr) != 0) {
+		xe_err(xe, "sysctrl: Firmware error: 0x%02lx\n",
+		       XE_SYSCTRL_HDR_RESULT(hdr));
+		return -EIO;
+	}
+
+	memcpy(data_out, hdr + 1, data_out_len);
+	*rdata_len = out - buffer - sizeof(*hdr);
+
+	return 0;
+}
+
+static int sysctrl_send_command(struct xe_sysctrl *sc,
+				const u8 *mbox_cmd, size_t cmd_size,
+				void *data_out, size_t data_out_len,
+				size_t *rdata_len, unsigned int timeout_ms)
+{
+	const struct xe_sysctrl_mailbox_msg_hdr *hdr;
+	size_t received;
+	int ret;
+
+	ret = sysctrl_send_frames(sc, mbox_cmd, cmd_size, timeout_ms);
+	if (ret)
+		return ret;
+
+	if (!data_out || !rdata_len)
+		return 0;
+
+	hdr = (const struct xe_sysctrl_mailbox_msg_hdr *)mbox_cmd;
+
+	ret = sysctrl_receive_frames(sc, hdr, data_out, data_out_len,
+				     &received, timeout_ms);
+	if (ret)
+		return ret;
+
+	*rdata_len = received;
+
+	return 0;
+}
+
+/**
+ * xe_sysctrl_mailbox_init - Initialize System Controller mailbox interface
+ * @sc: System controller structure
+ *
+ * Initialize system controller mailbox interface for communication.
+ */
+void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc)
+{
+	u32 ctrl_reg;
+
+	ctrl_reg = xe_mmio_read32(sc->mmio, SYSCTRL_MB_CTRL);
+	sc->phase_bit = (ctrl_reg & SYSCTRL_FRAME_PHASE) ? 1 : 0;
+}
+
+/**
+ * xe_sysctrl_send_command() - Send mailbox command to System Controller
+ * @sc: System Controller instance
+ * @cmd: Command descriptor containing request header and payload buffers
+ * @rdata_len: Pointer to store actual response data length
+ *
+ * Sends a mailbox command to System Controller firmware using
+ * System Controller mailbox and waits for a response.
+ *
+ * Request payload is provided via @cmd->data_in and @cmd->data_in_len.
+ * If a response is expected, @cmd->data_out must point to a buffer of
+ * size @cmd->data_out_len supplied by caller.
+ *
+ * On success, @rdata_len is updated with number of valid response bytes
+ * returned by firmware, bounded by @cmd->data_out_len.
+ *
+ * Return: 0 on success, or negative errno on failure.
+ */
+int xe_sysctrl_send_command(struct xe_sysctrl *sc,
+			    struct xe_sysctrl_mailbox_command *cmd,
+			    size_t *rdata_len)
+{
+	struct xe_device *xe = sc_to_xe(sc);
+	u8 group_id, command_code;
+	u8 *mbox_cmd = NULL;
+	size_t cmd_size = 0;
+	int ret;
+
+	guard(xe_pm_runtime_noresume)(xe);
+
+	if (!xe->info.has_sysctrl)
+		return -ENODEV;
+
+	xe_assert(xe, cmd->data_in || cmd->data_out);
+	xe_assert(xe, !cmd->data_in || cmd->data_in_len);
+	xe_assert(xe, !cmd->data_out || cmd->data_out_len);
+
+	group_id = XE_SYSCTRL_APP_HDR_GROUP_ID(&cmd->header);
+	command_code = XE_SYSCTRL_APP_HDR_COMMAND(&cmd->header);
+
+	might_sleep();
+
+	ret = sysctrl_prepare_command(xe, group_id, command_code,
+				      cmd->data_in, cmd->data_in_len,
+				      &mbox_cmd, &cmd_size);
+	if (ret) {
+		xe_err(xe, "sysctrl: Failed to prepare command: %pe\n", ERR_PTR(ret));
+		return ret;
+	}
+
+	guard(mutex)(&sc->cmd_lock);
+
+	ret = sysctrl_send_command(sc, mbox_cmd, cmd_size,
+				   cmd->data_out, cmd->data_out_len, rdata_len,
+				   XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS);
+	if (ret)
+		xe_err(xe, "sysctrl: Mailbox command failed: %pe\n", ERR_PTR(ret));
+
+	kfree(mbox_cmd);
+
+	return ret;
+}
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h
new file mode 100644
index 0000000000000..f67e9234de482
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_SYSCTRL_MAILBOX_H_
+#define _XE_SYSCTRL_MAILBOX_H_
+
+#include <linux/bitfield.h>
+#include <linux/types.h>
+
+#include "abi/xe_sysctrl_abi.h"
+
+struct xe_sysctrl;
+struct xe_sysctrl_mailbox_command;
+
+#define XE_SYSCTRL_APP_HDR_GROUP_ID(hdr) \
+	FIELD_GET(APP_HDR_GROUP_ID_MASK, (hdr)->data)
+
+#define XE_SYSCTRL_APP_HDR_COMMAND(hdr) \
+	FIELD_GET(APP_HDR_COMMAND_MASK, (hdr)->data)
+
+#define XE_SYSCTRL_APP_HDR_VERSION(hdr) \
+	FIELD_GET(APP_HDR_VERSION_MASK, (hdr)->data)
+
+void xe_sysctrl_mailbox_init(struct xe_sysctrl *sc);
+int xe_sysctrl_send_command(struct xe_sysctrl *sc,
+			    struct xe_sysctrl_mailbox_command *cmd,
+			    size_t *rdata_len);
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
new file mode 100644
index 0000000000000..84d7c647e7432
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sysctrl_mailbox_types.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_SYSCTRL_MAILBOX_TYPES_H_
+#define _XE_SYSCTRL_MAILBOX_TYPES_H_
+
+#include <linux/types.h>
+
+#include "abi/xe_sysctrl_abi.h"
+
+/**
+ * enum xe_sysctrl_group - System Controller command groups
+ *
+ * @XE_SYSCTRL_GROUP_GFSP: GFSP group
+ */
+enum xe_sysctrl_group {
+	XE_SYSCTRL_GROUP_GFSP			= 0x01,
+};
+
+/**
+ * enum xe_sysctrl_gfsp_cmd - Commands supported by GFSP group
+ *
+ * @XE_SYSCTRL_CMD_GET_PENDING_EVENT: Retrieve pending event
+ */
+enum xe_sysctrl_gfsp_cmd {
+	XE_SYSCTRL_CMD_GET_PENDING_EVENT	= 0x07,
+};
+
+/**
+ * struct xe_sysctrl_mailbox_command - System Controller mailbox command
+ */
+struct xe_sysctrl_mailbox_command {
+	/** @header: Application message header containing command information */
+	struct xe_sysctrl_app_msg_hdr header;
+
+	/** @data_in: Pointer to input payload data (can be NULL if no input data) */
+	void *data_in;
+
+	/** @data_in_len: Size of input payload in bytes (0 if no input data) */
+	size_t data_in_len;
+
+	/** @data_out: Pointer to output buffer for response data (can be NULL if no response) */
+	void *data_out;
+
+	/** @data_out_len: Size of output buffer in bytes (0 if no response expected) */
+	size_t data_out_len;
+};
+
+#define XE_SYSCTRL_MB_FRAME_SIZE	16
+#define XE_SYSCTRL_MB_MAX_FRAMES	64
+#define XE_SYSCTRL_MB_MAX_MESSAGE_SIZE	\
+	(XE_SYSCTRL_MB_FRAME_SIZE * XE_SYSCTRL_MB_MAX_FRAMES)
+
+#define XE_SYSCTRL_MB_DEFAULT_TIMEOUT_MS	500
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_sysctrl_types.h b/drivers/gpu/drm/xe/xe_sysctrl_types.h
new file mode 100644
index 0000000000000..66ba24f43017d
--- /dev/null
+++ b/drivers/gpu/drm/xe/xe_sysctrl_types.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2026 Intel Corporation
+ */
+
+#ifndef _XE_SYSCTRL_TYPES_H_
+#define _XE_SYSCTRL_TYPES_H_
+
+#include <linux/mutex.h>
+#include <linux/types.h>
+#include <linux/workqueue_types.h>
+
+struct xe_mmio;
+
+/**
+ * struct xe_sysctrl - System Controller driver context
+ *
+ * This structure maintains the runtime state for System Controller
+ * communication. All fields are initialized during xe_sysctrl_init()
+ * and protected appropriately for concurrent access.
+ */
+struct xe_sysctrl {
+	/** @mmio: MMIO region for system control registers */
+	struct xe_mmio *mmio;
+
+	/** @cmd_lock: Mutex protecting mailbox command operations */
+	struct mutex cmd_lock;
+
+	/** @phase_bit: Message boundary phase toggle bit (0 or 1) */
+	bool phase_bit;
+
+	/** @work: Pending events worker */
+	struct work_struct work;
+
+	/** @event_lock: Mutex protecting pending events */
+	struct mutex event_lock;
+};
+
+#endif
diff --git a/drivers/gpu/drm/xe/xe_tlb_inval.c b/drivers/gpu/drm/xe/xe_tlb_inval.c
index 10dcd4abb00f0..bbd21d393062f 100644
--- a/drivers/gpu/drm/xe/xe_tlb_inval.c
+++ b/drivers/gpu/drm/xe/xe_tlb_inval.c
@@ -529,7 +529,7 @@ int xe_tlb_inval_range_tilemask_submit(struct xe_device *xe, u32 asid,
 	struct xe_tile *tile;
 	u32 fence_id = 0;
 	u8 id;
-	int err;
+	int err = 0;
 
 	batch->num_fences = 0;
 	if (!tile_mask)
diff --git a/drivers/gpu/drm/xe/xe_trace_lrc.h b/drivers/gpu/drm/xe/xe_trace_lrc.h
index d525cbee1e341..5c4cfa0c1fe93 100644
--- a/drivers/gpu/drm/xe/xe_trace_lrc.h
+++ b/drivers/gpu/drm/xe/xe_trace_lrc.h
@@ -12,6 +12,7 @@
 #include <linux/tracepoint.h>
 #include <linux/types.h>
 
+#include "xe_exec_queue_types.h"
 #include "xe_gt_types.h"
 #include "xe_lrc.h"
 #include "xe_lrc_types.h"
@@ -42,6 +43,32 @@ TRACE_EVENT(xe_lrc_update_timestamp,
 		      __get_str(device_id))
 );
 
+TRACE_EVENT(xe_lrc_update_queue_timestamp,
+	    TP_PROTO(struct xe_lrc *lrc, uint64_t old),
+	    TP_ARGS(lrc, old),
+	    TP_STRUCT__entry(
+		     __field(struct xe_lrc *, lrc)
+		     __field(struct xe_lrc *, primary_lrc)
+		     __field(u64, old)
+		     __field(u64, new)
+		     __string(name, lrc->fence_ctx.name)
+		     __string(device_id, __dev_name_lrc(lrc))
+		     ),
+
+	    TP_fast_assign(
+		   __entry->lrc = lrc;
+		   __entry->primary_lrc = lrc->multi_queue.primary_lrc;
+		   __entry->old = old;
+		   __entry->new = lrc->queue_timestamp;
+		   __assign_str(name);
+		   __assign_str(device_id);
+		   ),
+	    TP_printk("lrc=%p primary_lrc=%p lrc->name=%s old=%llu new=%llu device_id:%s",
+		      __entry->lrc, __entry->primary_lrc, __get_str(name),
+		      __entry->old, __entry->new,
+		      __get_str(device_id))
+);
+
 #endif
 
 /* This part must be outside protection */
diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c
index 27c9d72222cf0..5e9070739e653 100644
--- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.c
@@ -19,30 +19,11 @@
 #include "xe_device.h"
 #include "xe_gt_printk.h"
 #include "xe_mmio.h"
-#include "xe_res_cursor.h"
 #include "xe_sriov.h"
 #include "xe_ttm_stolen_mgr.h"
-#include "xe_ttm_vram_mgr.h"
 #include "xe_vram.h"
 #include "xe_wa.h"
 
-struct xe_ttm_stolen_mgr {
-	struct xe_ttm_vram_mgr base;
-
-	/* PCI base offset */
-	resource_size_t io_base;
-	/* GPU base offset */
-	resource_size_t stolen_base;
-
-	void __iomem *mapping;
-};
-
-static inline struct xe_ttm_stolen_mgr *
-to_stolen_mgr(struct ttm_resource_manager *man)
-{
-	return container_of(man, struct xe_ttm_stolen_mgr, base.manager);
-}
-
 /**
  * xe_ttm_stolen_cpu_access_needs_ggtt() - If we can't directly CPU access
  * stolen, can we then fallback to mapping through the GGTT.
@@ -210,12 +191,19 @@ static u64 detect_stolen(struct xe_device *xe, struct xe_ttm_stolen_mgr *mgr)
 #endif
 }
 
+static void xe_ttm_stolen_mgr_fini(struct drm_device *dev, void *arg)
+{
+	struct xe_device *xe = to_xe_device(dev);
+
+	ttm_range_man_fini_nocheck(&xe->ttm, XE_PL_STOLEN);
+}
+
 int xe_ttm_stolen_mgr_init(struct xe_device *xe)
 {
 	struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
 	struct xe_ttm_stolen_mgr *mgr;
 	u64 stolen_size, io_size;
-	int err;
+	int ret;
 
 	mgr = drmm_kzalloc(&xe->drm, sizeof(*mgr), GFP_KERNEL);
 	if (!mgr)
@@ -244,12 +232,12 @@ int xe_ttm_stolen_mgr_init(struct xe_device *xe)
 	if (mgr->io_base && !xe_ttm_stolen_cpu_access_needs_ggtt(xe))
 		io_size = stolen_size;
 
-	err = __xe_ttm_vram_mgr_init(xe, &mgr->base, XE_PL_STOLEN, stolen_size,
-				     io_size, PAGE_SIZE);
-	if (err) {
-		drm_dbg_kms(&xe->drm, "Stolen mgr init failed: %i\n", err);
-		return err;
-	}
+	ret = ttm_range_man_init_nocheck(&xe->ttm, XE_PL_STOLEN, false,
+					 stolen_size >> PAGE_SHIFT);
+	if (ret)
+		return ret;
+
+	xe->mem.stolen_mgr = mgr;
 
 	drm_dbg_kms(&xe->drm, "Initialized stolen memory support with %llu bytes\n",
 		    stolen_size);
@@ -257,36 +245,32 @@ int xe_ttm_stolen_mgr_init(struct xe_device *xe)
 	if (io_size)
 		mgr->mapping = devm_ioremap_wc(&pdev->dev, mgr->io_base, io_size);
 
-	return 0;
+	return drmm_add_action_or_reset(&xe->drm, xe_ttm_stolen_mgr_fini, mgr);
 }
 
 u64 xe_ttm_stolen_io_offset(struct xe_bo *bo, u32 offset)
 {
 	struct xe_device *xe = xe_bo_device(bo);
-	struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
-	struct xe_ttm_stolen_mgr *mgr = to_stolen_mgr(ttm_mgr);
-	struct xe_res_cursor cur;
+	struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr;
 
 	XE_WARN_ON(!mgr->io_base);
 
 	if (xe_ttm_stolen_cpu_access_needs_ggtt(xe))
 		return mgr->io_base + xe_bo_ggtt_addr(bo) + offset;
 
-	xe_res_first(bo->ttm.resource, offset, 4096, &cur);
-	return mgr->io_base + cur.start;
+	/* Range allocator: res->start is in pages. */
+	return mgr->io_base + (bo->ttm.resource->start << PAGE_SHIFT) + offset;
 }
 
 static int __xe_ttm_stolen_io_mem_reserve_bar2(struct xe_device *xe,
 					       struct xe_ttm_stolen_mgr *mgr,
 					       struct ttm_resource *mem)
 {
-	struct xe_res_cursor cur;
-
 	if (!mgr->io_base)
 		return -EIO;
 
-	xe_res_first(mem, 0, 4096, &cur);
-	mem->bus.offset = cur.start;
+	/* Range allocator always produces contiguous allocations. */
+	mem->bus.offset = mem->start << PAGE_SHIFT;
 
 	drm_WARN_ON(&xe->drm, !(mem->placement & TTM_PL_FLAG_CONTIGUOUS));
 
@@ -329,8 +313,7 @@ static int __xe_ttm_stolen_io_mem_reserve_stolen(struct xe_device *xe,
 
 int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem)
 {
-	struct ttm_resource_manager *ttm_mgr = ttm_manager_type(&xe->ttm, XE_PL_STOLEN);
-	struct xe_ttm_stolen_mgr *mgr = ttm_mgr ? to_stolen_mgr(ttm_mgr) : NULL;
+	struct xe_ttm_stolen_mgr *mgr = xe->mem.stolen_mgr;
 
 	if (!mgr || !mgr->io_base)
 		return -EIO;
@@ -343,8 +326,5 @@ int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem)
 
 u64 xe_ttm_stolen_gpu_offset(struct xe_device *xe)
 {
-	struct xe_ttm_stolen_mgr *mgr =
-		to_stolen_mgr(ttm_manager_type(&xe->ttm, XE_PL_STOLEN));
-
-	return mgr->stolen_base;
+	return xe->mem.stolen_mgr->stolen_base;
 }
diff --git a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h
index 8e877d1e839bd..0675106d535b9 100644
--- a/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h
+++ b/drivers/gpu/drm/xe/xe_ttm_stolen_mgr.h
@@ -12,6 +12,18 @@ struct ttm_resource;
 struct xe_bo;
 struct xe_device;
 
+/**
+ * struct xe_ttm_stolen_mgr - Xe TTM stolen memory manager
+ */
+struct xe_ttm_stolen_mgr {
+	/** @io_base: PCI base offset for CPU I/O access */
+	resource_size_t io_base;
+	/** @stolen_base: GPU base offset */
+	resource_size_t stolen_base;
+	/** @mapping: I/O memory mapping for CPU access */
+	void __iomem *mapping;
+};
+
 int xe_ttm_stolen_mgr_init(struct xe_device *xe);
 int xe_ttm_stolen_io_mem_reserve(struct xe_device *xe, struct ttm_resource *mem);
 bool xe_ttm_stolen_cpu_access_needs_ggtt(struct xe_device *xe);
diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
index 5fd0d5506a7ea..b518f7dec680b 100644
--- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
+++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c
@@ -292,8 +292,6 @@ static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg)
 	ttm_resource_manager_cleanup(&mgr->manager);
 
 	ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL);
-
-	mutex_destroy(&mgr->lock);
 }
 
 int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
@@ -301,18 +299,19 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
 			   u64 default_page_size)
 {
 	struct ttm_resource_manager *man = &mgr->manager;
+	const char *name;
 	int err;
 
-	if (mem_type != XE_PL_STOLEN) {
-		const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1";
-		man->cg = drmm_cgroup_register_region(&xe->drm, name, size);
-		if (IS_ERR(man->cg))
-			return PTR_ERR(man->cg);
-	}
+	name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1";
+	man->cg = drmm_cgroup_register_region(&xe->drm, name, size);
+	if (IS_ERR(man->cg))
+		return PTR_ERR(man->cg);
 
 	man->func = &xe_ttm_vram_mgr_func;
 	mgr->mem_type = mem_type;
-	mutex_init(&mgr->lock);
+	err = drmm_mutex_init(&xe->drm, &mgr->lock);
+	if (err)
+		return err;
 	mgr->default_page_size = default_page_size;
 	mgr->visible_size = io_size;
 	mgr->visible_avail = io_size;
@@ -322,6 +321,7 @@ int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr,
 	if (err)
 		return err;
 
+	gpu_buddy_driver_set_lock(&mgr->mm, &mgr->lock);
 	ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager);
 	ttm_resource_manager_set_used(&mgr->manager, true);
 
diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c
index 0b78ec2bc6a4e..9a1b3862e1928 100644
--- a/drivers/gpu/drm/xe/xe_tuning.c
+++ b/drivers/gpu/drm/xe/xe_tuning.c
@@ -43,7 +43,7 @@ static const struct xe_rtp_entry_sr gt_tunings[] = {
 				   REG_FIELD_PREP(L3_PWM_TIMER_INIT_VAL_MASK, 0x7f)))
 	},
 	{ XE_RTP_NAME("Tuning: Compression Overfetch"),
-	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, XE_RTP_END_VERSION_UNDEFINED),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 3499),
 		       FUNC(xe_rtp_match_has_flat_ccs)),
 	  XE_RTP_ACTIONS(CLR(CCCHKNREG1, ENCOMPPERFFIX),
 			 SET(CCCHKNREG1, L3CMPCTRL))
@@ -124,12 +124,17 @@ static const struct xe_rtp_entry_sr engine_tunings[] = {
 			     GHWSP_CSB_REPORT_DIS,
 			     XE_RTP_ACTION_FLAG(ENGINE_BASE)))
 	},
+	{ XE_RTP_NAME("Tuning: TileY 2x2 Walk"),
+	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3510, XE_RTP_END_VERSION_UNDEFINED),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(TDL_TSL_CHICKEN2, TILEY_LOCALID))
+	},
 };
 
 static const struct xe_rtp_entry_sr lrc_tunings[] = {
 	{ XE_RTP_NAME("Tuning: Windower HW Filtering"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3599), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, HW_FILTERING))
+	  XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, HW_FILTERING))
 	},
 
 	/* DG2 */
diff --git a/drivers/gpu/drm/xe/xe_tuning.h b/drivers/gpu/drm/xe/xe_tuning.h
index c1cc5927fda77..d18e187debf63 100644
--- a/drivers/gpu/drm/xe/xe_tuning.h
+++ b/drivers/gpu/drm/xe/xe_tuning.h
@@ -3,8 +3,8 @@
  * Copyright © 2022 Intel Corporation
  */
 
-#ifndef _XE_TUNING_
-#define _XE_TUNING_
+#ifndef _XE_TUNING_H_
+#define _XE_TUNING_H_
 
 struct drm_printer;
 struct xe_gt;
diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c
index 9cebb24902457..df2aa196f6f9b 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw.c
+++ b/drivers/gpu/drm/xe/xe_uc_fw.c
@@ -214,6 +214,17 @@ static struct xe_device *uc_fw_to_xe(struct xe_uc_fw *uc_fw)
 	return gt_to_xe(uc_fw_to_gt(uc_fw));
 }
 
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
+void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status)
+{
+	xe_gt_dbg(uc_fw_to_gt(uc_fw), "%s %s->%s\n",
+		  xe_uc_fw_type_repr(uc_fw->type),
+		  xe_uc_fw_status_repr(uc_fw->status),
+		  xe_uc_fw_status_repr(status));
+	uc_fw->__status = status;
+}
+#endif
+
 static void
 uc_fw_auto_select(struct xe_device *xe, struct xe_uc_fw *uc_fw)
 {
diff --git a/drivers/gpu/drm/xe/xe_uc_fw.h b/drivers/gpu/drm/xe/xe_uc_fw.h
index 6195e353f269e..bb281b72a6772 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw.h
+++ b/drivers/gpu/drm/xe/xe_uc_fw.h
@@ -25,11 +25,15 @@ static inline u32 xe_uc_fw_rsa_offset(struct xe_uc_fw *uc_fw)
 	return sizeof(struct uc_css_header) + uc_fw->ucode_size + uc_fw->css_offset;
 }
 
+#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC)
+void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, enum xe_uc_fw_status status);
+#else
 static inline void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw,
 					  enum xe_uc_fw_status status)
 {
 	uc_fw->__status = status;
 }
+#endif
 
 static inline
 const char *xe_uc_fw_status_repr(enum xe_uc_fw_status status)
diff --git a/drivers/gpu/drm/xe/xe_uc_fw_abi.h b/drivers/gpu/drm/xe/xe_uc_fw_abi.h
index 3c9a63d13032a..74b888904fdc7 100644
--- a/drivers/gpu/drm/xe/xe_uc_fw_abi.h
+++ b/drivers/gpu/drm/xe/xe_uc_fw_abi.h
@@ -3,8 +3,8 @@
  * Copyright © 2022 Intel Corporation
  */
 
-#ifndef _XE_UC_FW_ABI_H
-#define _XE_UC_FW_ABI_H
+#ifndef _XE_UC_FW_ABI_H_
+#define _XE_UC_FW_ABI_H_
 
 #include <linux/build_bug.h>
 #include <linux/types.h>
diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c
index ab6cc1f0a7894..b01f31ed44170 100644
--- a/drivers/gpu/drm/xe/xe_vm.c
+++ b/drivers/gpu/drm/xe/xe_vm.c
@@ -634,9 +634,9 @@ void xe_vm_add_fault_entry_pf(struct xe_vm *vm, struct xe_pagefault *pf)
 	e->address_precision = SZ_4K;
 	e->access_type = pf->consumer.access_type;
 	e->fault_type = FIELD_GET(XE_PAGEFAULT_TYPE_MASK,
-				  pf->consumer.fault_type_level),
+				  pf->consumer.fault_type_level);
 	e->fault_level = FIELD_GET(XE_PAGEFAULT_LEVEL_MASK,
-				   pf->consumer.fault_type_level),
+				   pf->consumer.fault_type_level);
 
 	list_add_tail(&e->list, &vm->faults.list);
 	vm->faults.len++;
@@ -1425,9 +1425,9 @@ static u16 pde_pat_index(struct xe_bo *bo)
 	 * something which is always safe).
 	 */
 	if (!xe_bo_is_vram(bo) && bo->ttm.ttm->caching == ttm_cached)
-		pat_index = xe->pat.idx[XE_CACHE_WB];
+		pat_index = xe_cache_pat_idx(xe, XE_CACHE_WB);
 	else
-		pat_index = xe->pat.idx[XE_CACHE_NONE];
+		pat_index = xe_cache_pat_idx(xe, XE_CACHE_NONE);
 
 	xe_assert(xe, pat_index <= 3);
 
@@ -3447,7 +3447,7 @@ static struct dma_fence *ops_execute(struct xe_vm *vm,
 
 	xe_assert(vm->xe, current_fence == n_fence);
 	dma_fence_array_init(cf, n_fence, fences, dma_fence_context_alloc(1),
-			     1, false);
+			     1);
 	fence = &cf->base;
 
 	for_each_tile(tile, vm->xe, id) {
@@ -4239,7 +4239,7 @@ struct dma_fence *xe_vm_bind_kernel_bo(struct xe_vm *vm, struct xe_bo *bo,
 
 	ops = vm_bind_ioctl_ops_create(vm, &vops, bo, 0, addr, xe_bo_size(bo),
 				       DRM_XE_VM_BIND_OP_MAP, 0, 0,
-				       vm->xe->pat.idx[cache_lvl]);
+				       xe_cache_pat_idx(vm->xe, cache_lvl));
 	if (IS_ERR(ops)) {
 		err = PTR_ERR(ops);
 		goto release_vm_lock;
@@ -4442,7 +4442,7 @@ struct xe_vm_snapshot {
 #define XE_VM_SNAP_FLAG_IS_NULL		BIT(2)
 		unsigned long flags;
 		int uapi_mem_region;
-		int pat_index;
+		u16 pat_index;
 		int cpu_caching;
 		struct xe_bo *bo;
 		void *data;
diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h
index a94827d7fbeca..635ed29b9a69b 100644
--- a/drivers/gpu/drm/xe/xe_vm_types.h
+++ b/drivers/gpu/drm/xe/xe_vm_types.h
@@ -412,10 +412,11 @@ struct xe_vm {
 struct xe_vma_op_map {
 	/** @vma: VMA to map */
 	struct xe_vma *vma;
+	/** @vma_flags: VMA flags for this operation */
 	unsigned int vma_flags;
 	/** @immediate: Immediate bind */
 	bool immediate;
-	/** @read_only: Read only */
+	/** @invalidate_on_bind: Invalidate on bind */
 	bool invalidate_on_bind;
 	/** @request_decompress: schedule decompression for GPU map */
 	bool request_decompress;
diff --git a/drivers/gpu/drm/xe/xe_vram.c b/drivers/gpu/drm/xe/xe_vram.c
index 0538dcb8b18c8..23eb7edbdd572 100644
--- a/drivers/gpu/drm/xe/xe_vram.c
+++ b/drivers/gpu/drm/xe/xe_vram.c
@@ -13,6 +13,7 @@
 #include "regs/xe_gt_regs.h"
 #include "regs/xe_regs.h"
 #include "xe_assert.h"
+#include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_force_wake.h"
 #include "xe_gt_mcr.h"
@@ -249,6 +250,27 @@ static int vram_region_init(struct xe_device *xe, struct xe_vram_region *vram,
 	return 0;
 }
 
+/**
+ * xe_map_resource_to_region - Map ttm resource to vram memory region
+ * @res: The ttm resource
+ *
+ * Get vram memory region using vram memory manager managing this resource
+ *
+ * Returns: pointer to xe_vram_region
+ */
+struct xe_vram_region *xe_map_resource_to_region(struct ttm_resource *res)
+{
+	struct xe_device *xe = ttm_to_xe_device(res->bo->bdev);
+	struct ttm_resource_manager *mgr;
+	struct xe_ttm_vram_mgr *vram_mgr;
+
+	xe_assert(xe, mem_type_is_vram(res->mem_type));
+	mgr = ttm_manager_type(&xe->ttm, res->mem_type);
+	vram_mgr = to_xe_ttm_vram_mgr(mgr);
+
+	return container_of(vram_mgr, struct xe_vram_region, ttm);
+}
+
 /**
  * xe_vram_probe() - Probe VRAM configuration
  * @xe: the &xe_device
diff --git a/drivers/gpu/drm/xe/xe_vram.h b/drivers/gpu/drm/xe/xe_vram.h
index 72860f714fc66..dd1c8bf179223 100644
--- a/drivers/gpu/drm/xe/xe_vram.h
+++ b/drivers/gpu/drm/xe/xe_vram.h
@@ -10,7 +10,9 @@
 
 struct xe_device;
 struct xe_vram_region;
+struct ttm_resource;
 
+struct xe_vram_region *xe_map_resource_to_region(struct ttm_resource *res);
 int xe_vram_probe(struct xe_device *xe);
 
 struct xe_vram_region *xe_vram_region_alloc(struct xe_device *xe, u8 id, u32 placement);
diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c
index 4b1cbced06bec..cb811f8a77816 100644
--- a/drivers/gpu/drm/xe/xe_wa.c
+++ b/drivers/gpu/drm/xe/xe_wa.c
@@ -601,6 +601,19 @@ static const struct xe_rtp_entry_sr engine_was[] = {
 		       FUNC(xe_rtp_match_first_render_or_compute)),
 	  XE_RTP_ACTIONS(SET(ROW_CHICKEN5, CPSS_AWARE_DIS))
 	},
+
+	/* Xe3p_XPC */
+
+	{ XE_RTP_NAME("14026999295"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3511),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(ROW_CHICKEN3, DIS_EU_GRF_POISON_TO_LSC))
+	},
+	{ XE_RTP_NAME("18044193044"),
+	  XE_RTP_RULES(GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0),
+		       FUNC(xe_rtp_match_first_render_or_compute)),
+	  XE_RTP_ACTIONS(SET(TDL_CHICKEN, BIT_APQ_OPT_DIS))
+	},
 };
 
 static const struct xe_rtp_entry_sr lrc_was[] = {
@@ -651,7 +664,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 	},
 	{ XE_RTP_NAME("18033852989"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))
+	  XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN1, DISABLE_BOTTOM_CLIP_RECTANGLE_TEST))
 	},
 	{ XE_RTP_NAME("15016589081"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2004), ENGINE_CLASS(RENDER)),
@@ -754,7 +767,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 	},
 	{ XE_RTP_NAME("22021007897"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(2001, 2002), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE))
+	  XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE))
 	},
 
 	/* Xe3_LPG */
@@ -770,7 +783,7 @@ static const struct xe_rtp_entry_sr lrc_was[] = {
 	},
 	{ XE_RTP_NAME("22021007897"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)),
-	  XE_RTP_ACTIONS(SET(COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE))
+	  XE_RTP_ACTIONS(SET(XEHP_COMMON_SLICE_CHICKEN4, SBE_PUSH_CONSTANT_BEHIND_FIX_ENABLE))
 	},
 	{ XE_RTP_NAME("14024681466"),
 	  XE_RTP_RULES(GRAPHICS_VERSION_RANGE(3000, 3005), ENGINE_CLASS(RENDER)),
diff --git a/drivers/gpu/drm/xe/xe_wa.h b/drivers/gpu/drm/xe/xe_wa.h
index 8fd6a5af09101..a5f7d33c1b32b 100644
--- a/drivers/gpu/drm/xe/xe_wa.h
+++ b/drivers/gpu/drm/xe/xe_wa.h
@@ -3,8 +3,8 @@
  * Copyright © 2022 Intel Corporation
  */
 
-#ifndef _XE_WA_
-#define _XE_WA_
+#ifndef _XE_WA_H_
+#define _XE_WA_H_
 
 #include "xe_assert.h"
 
diff --git a/drivers/gpu/drm/xe/xe_wa_oob.rules b/drivers/gpu/drm/xe/xe_wa_oob.rules
index 80b54b195f20c..f8a185103b805 100644
--- a/drivers/gpu/drm/xe/xe_wa_oob.rules
+++ b/drivers/gpu/drm/xe/xe_wa_oob.rules
@@ -51,6 +51,13 @@
 		MEDIA_VERSION_RANGE(2000, 3002), FUNC(xe_rtp_match_psmi_enabled)
 16023683509	MEDIA_VERSION(2000), FUNC(xe_rtp_match_psmi_enabled)
 		MEDIA_VERSION(3000), MEDIA_STEP(A0, B0), FUNC(xe_rtp_match_psmi_enabled)
+14025515070	GRAPHICS_VERSION(2004)
+		MEDIA_VERSION_RANGE(1301, 3000)
+		MEDIA_VERSION(3002)
+		GRAPHICS_VERSION_RANGE(3000, 3001)
+		GRAPHICS_VERSION_RANGE(3003, 3005)
+		MEDIA_VERSION(3500)
+		GRAPHICS_VERSION(3510), GRAPHICS_STEP(A0, B0)
 
 15015404425_disable	PLATFORM(PANTHERLAKE), MEDIA_STEP(B0, FOREVER)
 16026007364    MEDIA_VERSION(3000)
diff --git a/drivers/gpu/drm/xlnx/Kconfig b/drivers/gpu/drm/xlnx/Kconfig
index cfabf5e2a0bb0..58c19e5a7e8fa 100644
--- a/drivers/gpu/drm/xlnx/Kconfig
+++ b/drivers/gpu/drm/xlnx/Kconfig
@@ -3,8 +3,8 @@ config DRM_ZYNQMP_DPSUB
 	depends on ARCH_ZYNQMP || COMPILE_TEST
 	depends on COMMON_CLK && DRM && OF
 	depends on DMADEVICES
-	depends on PHY_XILINX_ZYNQMP
-	depends on XILINX_ZYNQMP_DPDMA
+	depends on PHY_XILINX_ZYNQMP || COMPILE_TEST
+	depends on XILINX_ZYNQMP_DPDMA || COMPILE_TEST
 	select DMA_ENGINE
 	select DRM_CLIENT_SELECTION
 	select DRM_DISPLAY_DP_HELPER
diff --git a/drivers/gpu/drm/xlnx/zynqmp_disp.c b/drivers/gpu/drm/xlnx/zynqmp_disp.c
index 9a8f38230cb4e..52a0712e39c41 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_disp.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_disp.c
@@ -297,6 +297,36 @@ static const struct zynqmp_disp_format avbuf_vid_fmts[] = {
 		.buf_fmt	= ZYNQMP_DISP_AV_BUF_FMT_NL_VID_YV16CI_420,
 		.swap		= true,
 		.sf		= scaling_factors_888,
+	}, {
+		.drm_fmt	= DRM_FORMAT_P030,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_FMT_NL_VID_YV16CI_420_10,
+		.swap		= false,
+		.sf		= scaling_factors_101010,
+	}, {
+		.drm_fmt	= DRM_FORMAT_P230,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_FMT_NL_VID_YV16CI_10,
+		.swap		= false,
+		.sf		= scaling_factors_101010,
+	}, {
+		.drm_fmt	= DRM_FORMAT_Y8,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_FMT_NL_VID_MONO,
+		.swap		= false,
+		.sf		= scaling_factors_888,
+	}, {
+		.drm_fmt	= DRM_FORMAT_XYYY2101010,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_FMT_NL_VID_YONLY_10,
+		.swap		= false,
+		.sf		= scaling_factors_101010,
+	}, {
+		.drm_fmt	= DRM_FORMAT_T430,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_FMT_NL_VID_YV24_10,
+		.swap		= false,
+		.sf		= scaling_factors_101010,
+	}, {
+		.drm_fmt	= DRM_FORMAT_XVUY2101010,
+		.buf_fmt	= ZYNQMP_DISP_AV_BUF_FMT_NL_VID_YUV444_10,
+		.swap		= false,
+		.sf		= scaling_factors_101010,
 	},
 };
 
@@ -687,6 +717,17 @@ static const u32 csc_sdtv_to_rgb_offsets[] = {
 	0x0, 0x1800, 0x1800
 };
 
+/* In Y-only mode the single Y channel is on the third column  */
+static const u16 csc_sdtv_to_rgb_yonly_matrix[] = {
+	0x0, 0x0, 0x1000,
+	0x0, 0x0, 0x1000,
+	0x0, 0x0, 0x1000,
+};
+
+static const u32 csc_sdtv_to_rgb_yonly_offsets[] = {
+	0x0, 0x0, 0x0
+};
+
 /**
  * zynqmp_disp_blend_set_output_format - Set the output format of the blender
  * @disp: Display controller
@@ -836,7 +877,11 @@ static void zynqmp_disp_blend_layer_enable(struct zynqmp_disp *disp,
 				ZYNQMP_DISP_V_BLEND_LAYER_CONTROL(layer->id),
 				val);
 
-	if (layer->drm_fmt->is_yuv) {
+	if (layer->drm_fmt->format == DRM_FORMAT_Y8 ||
+	    layer->drm_fmt->format == DRM_FORMAT_XYYY2101010) {
+		coeffs = csc_sdtv_to_rgb_yonly_matrix;
+		offsets = csc_sdtv_to_rgb_yonly_offsets;
+	} else if (layer->drm_fmt->is_yuv) {
 		coeffs = csc_sdtv_to_rgb_matrix;
 		offsets = csc_sdtv_to_rgb_offsets;
 	} else {
@@ -1116,16 +1161,19 @@ int zynqmp_disp_layer_update(struct zynqmp_disp_layer *layer,
 		return 0;
 
 	for (i = 0; i < info->num_planes; i++) {
-		unsigned int width = state->crtc_w / (i ? info->hsub : 1);
-		unsigned int height = state->crtc_h / (i ? info->vsub : 1);
 		struct zynqmp_disp_layer_dma *dma = &layer->dmas[i];
 		struct dma_async_tx_descriptor *desc;
 		dma_addr_t dma_addr;
+		unsigned int width;
+		unsigned int height;
+
+		width = drm_format_info_plane_width(info, state->crtc_w, i);
+		height = drm_format_info_plane_height(info, state->crtc_h, i);
 
 		dma_addr = drm_fb_dma_get_gem_addr(state->fb, state, i);
 
 		dma->xt.numf = height;
-		dma->sgl.size = width * info->cpp[i];
+		dma->sgl.size = drm_format_info_min_pitch(info, i, width);
 		dma->sgl.icg = state->fb->pitches[i] - dma->sgl.size;
 		dma->xt.src_start = dma_addr;
 		dma->xt.frame_size = 1;
diff --git a/drivers/gpu/drm/xlnx/zynqmp_dp.c b/drivers/gpu/drm/xlnx/zynqmp_dp.c
index 379180fb30043..7fb11b0a44f0c 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_dp.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_dp.c
@@ -353,7 +353,6 @@ struct zynqmp_dp_train_set_priv {
  * @lock: Mutex protecting this struct and register access (but not AUX)
  * @irq: irq
  * @bridge: DRM bridge for the DP encoder
- * @next_bridge: The downstream bridge
  * @test: Configuration for test mode
  * @config: IP core configuration from DTS
  * @aux: aux channel
@@ -385,7 +384,6 @@ struct zynqmp_dp {
 	struct completion aux_done;
 	struct mutex lock;
 
-	struct drm_bridge *next_bridge;
 	struct device *dev;
 	struct zynqmp_dpsub *dpsub;
 	void __iomem *iomem;
@@ -1437,7 +1435,7 @@ zynqmp_dp_disp_connected_live_layer(struct zynqmp_dp *dp)
 }
 
 static void zynqmp_dp_disp_enable(struct zynqmp_dp *dp,
-				  struct drm_atomic_state *state)
+				  struct drm_atomic_commit *state)
 {
 	struct zynqmp_disp_layer *layer;
 	struct drm_bridge_state *bridge_state;
@@ -1494,8 +1492,8 @@ static int zynqmp_dp_bridge_attach(struct drm_bridge *bridge,
 		return ret;
 	}
 
-	if (dp->next_bridge) {
-		ret = drm_bridge_attach(encoder, dp->next_bridge,
+	if (dp->bridge.next_bridge) {
+		ret = drm_bridge_attach(encoder, dp->bridge.next_bridge,
 					bridge, flags);
 		if (ret < 0)
 			goto error;
@@ -1549,7 +1547,7 @@ zynqmp_dp_bridge_mode_valid(struct drm_bridge *bridge,
 }
 
 static void zynqmp_dp_bridge_atomic_enable(struct drm_bridge *bridge,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct zynqmp_dp *dp = bridge_to_dp(bridge);
 	const struct drm_crtc_state *crtc_state;
@@ -1626,7 +1624,7 @@ static void zynqmp_dp_bridge_atomic_enable(struct drm_bridge *bridge,
 }
 
 static void zynqmp_dp_bridge_atomic_disable(struct drm_bridge *bridge,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct drm_bridge_state *old_bridge_state = drm_atomic_get_old_bridge_state(state,
 										    bridge);
@@ -2461,10 +2459,15 @@ int zynqmp_dp_probe(struct zynqmp_dpsub *dpsub)
 	 * Acquire the next bridge in the chain. Ignore errors caused by port@5
 	 * not being connected for backward-compatibility with older DTs.
 	 */
-	ret = drm_of_find_panel_or_bridge(dp->dev->of_node, 5, 0, NULL,
-					  &dp->next_bridge);
-	if (ret < 0 && ret != -ENODEV)
-		goto err_reset;
+	dp->bridge.next_bridge = of_drm_get_bridge_by_endpoint(dp->dev->of_node, 5, 0);
+	if (IS_ERR(dp->bridge.next_bridge)) {
+		if (PTR_ERR(dp->bridge.next_bridge) != -ENODEV) {
+			ret = PTR_ERR(dp->bridge.next_bridge);
+			goto err_reset;
+		}
+
+		dp->bridge.next_bridge = NULL;
+	}
 
 	/* Initialize the hardware. */
 	dp->config.misc0 &= ~ZYNQMP_DP_MAIN_STREAM_MISC0_SYNC_LOCK;
diff --git a/drivers/gpu/drm/xlnx/zynqmp_kms.c b/drivers/gpu/drm/xlnx/zynqmp_kms.c
index 02f3a7d78cf8e..d5f9224505655 100644
--- a/drivers/gpu/drm/xlnx/zynqmp_kms.c
+++ b/drivers/gpu/drm/xlnx/zynqmp_kms.c
@@ -53,7 +53,7 @@ static inline struct zynqmp_dpsub *to_zynqmp_dpsub(struct drm_device *drm)
  */
 
 static int zynqmp_dpsub_plane_atomic_check(struct drm_plane *plane,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *new_plane_state = drm_atomic_get_new_plane_state(state,
 										 plane);
@@ -74,7 +74,7 @@ static int zynqmp_dpsub_plane_atomic_check(struct drm_plane *plane,
 }
 
 static void zynqmp_dpsub_plane_atomic_disable(struct drm_plane *plane,
-					      struct drm_atomic_state *state)
+					      struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state,
 									   plane);
@@ -92,7 +92,7 @@ static void zynqmp_dpsub_plane_atomic_disable(struct drm_plane *plane,
 }
 
 static void zynqmp_dpsub_plane_atomic_update(struct drm_plane *plane,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct drm_plane_state *old_state = drm_atomic_get_old_plane_state(state, plane);
 	struct drm_plane_state *new_state = drm_atomic_get_new_plane_state(state, plane);
@@ -193,7 +193,7 @@ static inline struct zynqmp_dpsub *crtc_to_dpsub(struct drm_crtc *crtc)
 }
 
 static void zynqmp_dpsub_crtc_atomic_enable(struct drm_crtc *crtc,
-					    struct drm_atomic_state *state)
+					    struct drm_atomic_commit *state)
 {
 	struct zynqmp_dpsub *dpsub = crtc_to_dpsub(crtc);
 	struct drm_display_mode *adjusted_mode = &crtc->state->adjusted_mode;
@@ -219,7 +219,7 @@ static void zynqmp_dpsub_crtc_atomic_enable(struct drm_crtc *crtc,
 }
 
 static void zynqmp_dpsub_crtc_atomic_disable(struct drm_crtc *crtc,
-					     struct drm_atomic_state *state)
+					     struct drm_atomic_commit *state)
 {
 	struct zynqmp_dpsub *dpsub = crtc_to_dpsub(crtc);
 	struct drm_plane_state *old_plane_state;
@@ -249,19 +249,19 @@ static void zynqmp_dpsub_crtc_atomic_disable(struct drm_crtc *crtc,
 }
 
 static int zynqmp_dpsub_crtc_atomic_check(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state)
+					  struct drm_atomic_commit *state)
 {
 	return drm_atomic_add_affected_planes(state, crtc);
 }
 
 static void zynqmp_dpsub_crtc_atomic_begin(struct drm_crtc *crtc,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	drm_crtc_vblank_on(crtc);
 }
 
 static void zynqmp_dpsub_crtc_atomic_flush(struct drm_crtc *crtc,
-					   struct drm_atomic_state *state)
+					   struct drm_atomic_commit *state)
 {
 	if (crtc->state->event) {
 		struct drm_pending_vblank_event *event;
@@ -453,12 +453,6 @@ static int zynqmp_dpsub_kms_init(struct zynqmp_dpsub *dpsub)
 		goto err_encoder;
 	}
 
-	ret = drm_connector_attach_encoder(connector, encoder);
-	if (ret < 0) {
-		dev_err(dpsub->dev, "failed to attach connector to encoder\n");
-		goto err_encoder;
-	}
-
 	return 0;
 
 err_encoder:
diff --git a/drivers/gpu/nova-core/Kconfig b/drivers/gpu/nova-core/Kconfig
index a4f2380654e2d..f918f69e05996 100644
--- a/drivers/gpu/nova-core/Kconfig
+++ b/drivers/gpu/nova-core/Kconfig
@@ -3,6 +3,7 @@ config NOVA_CORE
 	depends on 64BIT
 	depends on PCI
 	depends on RUST
+	depends on !CPU_BIG_ENDIAN
 	select AUXILIARY_BUS
 	select RUST_FW_LOADER_ABSTRACTIONS
 	default n
@@ -13,4 +14,4 @@ config NOVA_CORE
 
 	  This driver is work in progress and may not be functional.
 
-	  If M is selected, the module will be called nova_core.
+	  If M is selected, the module will be called nova-core.
diff --git a/drivers/gpu/nova-core/Makefile b/drivers/gpu/nova-core/Makefile
index 2d78c50126e18..4ae544f808f42 100644
--- a/drivers/gpu/nova-core/Makefile
+++ b/drivers/gpu/nova-core/Makefile
@@ -1,3 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-$(CONFIG_NOVA_CORE) += nova_core.o
+obj-$(CONFIG_NOVA_CORE) += nova-core.o
+nova-core-y := nova_core.o
diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs
index 84b0e1703150e..8fe484d357f6c 100644
--- a/drivers/gpu/nova-core/driver.rs
+++ b/drivers/gpu/nova-core/driver.rs
@@ -32,8 +32,7 @@
 pub(crate) struct NovaCore {
     #[pin]
     pub(crate) gpu: Gpu,
-    #[pin]
-    _reg: Devres<auxiliary::Registration>,
+    _reg: Devres<auxiliary::Registration<()>>,
 }
 
 const BAR0_SIZE: usize = SZ_16M;
@@ -96,14 +95,15 @@ fn probe(pdev: &pci::Device<Core>, _info: &Self::IdInfo) -> impl PinInit<Self, E
 
             Ok(try_pin_init!(Self {
                 gpu <- Gpu::new(pdev, bar.clone(), bar.access(pdev.as_ref())?),
-                _reg <- auxiliary::Registration::new(
+                _reg: auxiliary::Registration::new(
                     pdev.as_ref(),
                     c"nova-drm",
                     // TODO[XARR]: Use XArray or perhaps IDA for proper ID allocation/recycling. For
                     // now, use a simple atomic counter that never recycles IDs.
                     AUXILIARY_ID_COUNTER.fetch_add(1, Relaxed),
-                    crate::MODULE_NAME
-                ),
+                    crate::MODULE_NAME,
+                    (),
+                )?,
             }))
         })
     }
diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs
index 33927af4134c1..24cc2c26e28de 100644
--- a/drivers/gpu/nova-core/falcon.rs
+++ b/drivers/gpu/nova-core/falcon.rs
@@ -488,7 +488,7 @@ pub(crate) fn pio_load<F: FalconFirmware<Target = E> + FalconPioLoadable>(
         }
         self.pio_wr_dmem_slice(bar, fw.dmem_load_params())?;
 
-        self.hal.program_brom(self, bar, &fw.brom_params())?;
+        self.hal.program_brom(self, bar, &fw.brom_params());
 
         bar.write(
             WithBase::of::<E>(),
@@ -647,7 +647,7 @@ fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(
         )?;
         self.dma_wr(bar, &dma_obj, FalconMem::Dmem, fw.dmem_load_params())?;
 
-        self.hal.program_brom(self, bar, &fw.brom_params())?;
+        self.hal.program_brom(self, bar, &fw.brom_params());
 
         // Set `BootVec` to start of non-secure code.
         bar.write(
diff --git a/drivers/gpu/nova-core/falcon/hal.rs b/drivers/gpu/nova-core/falcon/hal.rs
index a7e5ea8d0272c..a524c8096b67e 100644
--- a/drivers/gpu/nova-core/falcon/hal.rs
+++ b/drivers/gpu/nova-core/falcon/hal.rs
@@ -9,7 +9,10 @@
         FalconBromParams,
         FalconEngine, //
     },
-    gpu::Chipset,
+    gpu::{
+        Architecture,
+        Chipset, //
+    },
 };
 
 mod ga102;
@@ -46,7 +49,7 @@ fn signature_reg_fuse_version(
     ) -> Result<u32>;
 
     /// Program the boot ROM registers prior to starting a secure firmware.
-    fn program_brom(&self, falcon: &Falcon<E>, bar: &Bar0, params: &FalconBromParams) -> Result;
+    fn program_brom(&self, falcon: &Falcon<E>, bar: &Bar0, params: &FalconBromParams);
 
     /// Check if the RISC-V core is active.
     /// Returns `true` if the RISC-V core is active, `false` otherwise.
@@ -74,16 +77,21 @@ fn signature_reg_fuse_version(
 pub(super) fn falcon_hal<E: FalconEngine + 'static>(
     chipset: Chipset,
 ) -> Result<KBox<dyn FalconHal<E>>> {
-    use Chipset::*;
-
-    let hal = match chipset {
-        TU102 | TU104 | TU106 | TU116 | TU117 => {
+    let hal = match chipset.arch() {
+        Architecture::Turing => {
+            KBox::new(tu102::Tu102::<E>::new(), GFP_KERNEL)? as KBox<dyn FalconHal<E>>
+        }
+        // GA100 boots like Turing so use Turing HAL
+        Architecture::Ampere if chipset == Chipset::GA100 => {
             KBox::new(tu102::Tu102::<E>::new(), GFP_KERNEL)? as KBox<dyn FalconHal<E>>
         }
-        GA102 | GA103 | GA104 | GA106 | GA107 | AD102 | AD103 | AD104 | AD106 | AD107 => {
+        Architecture::Ampere
+        | Architecture::Ada
+        | Architecture::Hopper
+        | Architecture::BlackwellGB10x
+        | Architecture::BlackwellGB20x => {
             KBox::new(ga102::Ga102::<E>::new(), GFP_KERNEL)? as KBox<dyn FalconHal<E>>
         }
-        _ => return Err(ENOTSUPP),
     };
 
     Ok(hal)
diff --git a/drivers/gpu/nova-core/falcon/hal/ga102.rs b/drivers/gpu/nova-core/falcon/hal/ga102.rs
index 8368a61ddeef5..3df1ffa159b8a 100644
--- a/drivers/gpu/nova-core/falcon/hal/ga102.rs
+++ b/drivers/gpu/nova-core/falcon/hal/ga102.rs
@@ -86,7 +86,7 @@ fn signature_reg_fuse_version_ga102(
     Ok(u16::BITS - reg_fuse_version.leading_zeros())
 }
 
-fn program_brom_ga102<E: FalconEngine>(bar: &Bar0, params: &FalconBromParams) -> Result {
+fn program_brom_ga102<E: FalconEngine>(bar: &Bar0, params: &FalconBromParams) {
     bar.write(
         WithBase::of::<E>().at(0),
         regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed().with_value(params.pkc_data_offset),
@@ -104,8 +104,6 @@ fn program_brom_ga102<E: FalconEngine>(bar: &Bar0, params: &FalconBromParams) ->
         WithBase::of::<E>(),
         regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k),
     );
-
-    Ok(())
 }
 
 pub(super) struct Ga102<E: FalconEngine>(PhantomData<E>);
@@ -131,8 +129,8 @@ fn signature_reg_fuse_version(
         signature_reg_fuse_version_ga102(&falcon.dev, bar, engine_id_mask, ucode_id)
     }
 
-    fn program_brom(&self, _falcon: &Falcon<E>, bar: &Bar0, params: &FalconBromParams) -> Result {
-        program_brom_ga102::<E>(bar, params)
+    fn program_brom(&self, _falcon: &Falcon<E>, bar: &Bar0, params: &FalconBromParams) {
+        program_brom_ga102::<E>(bar, params);
     }
 
     fn is_riscv_active(&self, bar: &Bar0) -> bool {
diff --git a/drivers/gpu/nova-core/falcon/hal/tu102.rs b/drivers/gpu/nova-core/falcon/hal/tu102.rs
index c7a90266cb445..d8f5d271811b4 100644
--- a/drivers/gpu/nova-core/falcon/hal/tu102.rs
+++ b/drivers/gpu/nova-core/falcon/hal/tu102.rs
@@ -48,9 +48,7 @@ fn signature_reg_fuse_version(
         Ok(0)
     }
 
-    fn program_brom(&self, _falcon: &Falcon<E>, _bar: &Bar0, _params: &FalconBromParams) -> Result {
-        Ok(())
-    }
+    fn program_brom(&self, _falcon: &Falcon<E>, _bar: &Bar0, _params: &FalconBromParams) {}
 
     fn is_riscv_active(&self, bar: &Bar0) -> bool {
         bar.read(regs::NV_PRISCV_RISCV_CORE_SWITCH_RISCV_STATUS::of::<E>())
diff --git a/drivers/gpu/nova-core/fb.rs b/drivers/gpu/nova-core/fb.rs
index bdd5eed760e17..6ee87050ce691 100644
--- a/drivers/gpu/nova-core/fb.rs
+++ b/drivers/gpu/nova-core/fb.rs
@@ -24,11 +24,8 @@
     firmware::gsp::GspFirmware,
     gpu::Chipset,
     gsp,
-    num::{
-        usize_as_u64,
-        FromSafeCast, //
-    },
-    regs,
+    num::FromSafeCast,
+    regs, //
 };
 
 mod hal;
@@ -127,8 +124,8 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         if f.alternate() {
             let size = self.len();
 
-            if size < usize_as_u64(SZ_1M) {
-                let size_kib = size / usize_as_u64(SZ_1K);
+            if size < u64::SZ_1M {
+                let size_kib = size / u64::SZ_1K;
                 f.write_fmt(fmt!(
                     "{:#x}..{:#x} ({} KiB)",
                     self.0.start,
@@ -136,7 +133,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
                     size_kib
                 ))
             } else {
-                let size_mib = size / usize_as_u64(SZ_1M);
+                let size_mib = size / u64::SZ_1M;
                 f.write_fmt(fmt!(
                     "{:#x}..{:#x} ({} MiB)",
                     self.0.start,
@@ -186,7 +183,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
 
         let vga_workspace = {
             let vga_base = {
-                const NV_PRAMIN_SIZE: u64 = usize_as_u64(SZ_1M);
+                const NV_PRAMIN_SIZE: u64 = u64::SZ_1M;
                 let base = fb.end - NV_PRAMIN_SIZE;
 
                 if hal.supports_display(bar) {
@@ -196,7 +193,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
                     {
                         Some(addr) => {
                             if addr < base {
-                                const VBIOS_WORKSPACE_SIZE: u64 = usize_as_u64(SZ_128K);
+                                const VBIOS_WORKSPACE_SIZE: u64 = u64::SZ_128K;
 
                                 // Point workspace address to end of framebuffer.
                                 fb.end - VBIOS_WORKSPACE_SIZE
@@ -216,10 +213,10 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
 
         let frts = {
             const FRTS_DOWN_ALIGN: Alignment = Alignment::new::<SZ_128K>();
-            const FRTS_SIZE: u64 = usize_as_u64(SZ_1M);
-            let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - FRTS_SIZE;
+            let frts_size: u64 = hal.frts_size();
+            let frts_base = vga_workspace.start.align_down(FRTS_DOWN_ALIGN) - frts_size;
 
-            FbRange(frts_base..frts_base + FRTS_SIZE)
+            FbRange(frts_base..frts_base + frts_size)
         };
 
         let boot = {
@@ -241,7 +238,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
         let wpr2_heap = {
             const WPR2_HEAP_DOWN_ALIGN: Alignment = Alignment::new::<SZ_1M>();
             let wpr2_heap_size =
-                gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end);
+                gsp::LibosParams::from_chipset(chipset).wpr_heap_size(chipset, fb.end)?;
             let wpr2_heap_addr = (elf.start - wpr2_heap_size).align_down(WPR2_HEAP_DOWN_ALIGN);
 
             FbRange(wpr2_heap_addr..(elf.start).align_down(WPR2_HEAP_DOWN_ALIGN))
@@ -256,7 +253,7 @@ pub(crate) fn new(chipset: Chipset, bar: &Bar0, gsp_fw: &GspFirmware) -> Result<
         };
 
         let heap = {
-            const HEAP_SIZE: u64 = usize_as_u64(SZ_1M);
+            const HEAP_SIZE: u64 = u64::SZ_1M;
 
             FbRange(wpr2.start - HEAP_SIZE..wpr2.start)
         };
diff --git a/drivers/gpu/nova-core/fb/hal.rs b/drivers/gpu/nova-core/fb/hal.rs
index aba0abd8ee005..8b192a503363a 100644
--- a/drivers/gpu/nova-core/fb/hal.rs
+++ b/drivers/gpu/nova-core/fb/hal.rs
@@ -4,7 +4,10 @@
 
 use crate::{
     driver::Bar0,
-    gpu::Chipset, //
+    gpu::{
+        Architecture,
+        Chipset, //
+    },
 };
 
 mod ga100;
@@ -25,17 +28,20 @@ pub(crate) trait FbHal {
 
     /// Returns the VRAM size, in bytes.
     fn vidmem_size(&self, bar: &Bar0) -> u64;
+
+    /// Returns the FRTS size, in bytes.
+    fn frts_size(&self) -> u64;
 }
 
 /// Returns the HAL corresponding to `chipset`.
 pub(super) fn fb_hal(chipset: Chipset) -> &'static dyn FbHal {
-    use Chipset::*;
-
-    match chipset {
-        TU102 | TU104 | TU106 | TU117 | TU116 => tu102::TU102_HAL,
-        GA100 => ga100::GA100_HAL,
-        GA102 | GA103 | GA104 | GA106 | GA107 | AD102 | AD103 | AD104 | AD106 | AD107 => {
-            ga102::GA102_HAL
-        }
+    match chipset.arch() {
+        Architecture::Turing => tu102::TU102_HAL,
+        Architecture::Ampere if chipset == Chipset::GA100 => ga100::GA100_HAL,
+        Architecture::Ampere => ga102::GA102_HAL,
+        Architecture::Ada
+        | Architecture::Hopper
+        | Architecture::BlackwellGB10x
+        | Architecture::BlackwellGB20x => ga102::GA102_HAL,
     }
 }
diff --git a/drivers/gpu/nova-core/fb/hal/ga100.rs b/drivers/gpu/nova-core/fb/hal/ga100.rs
index 1c03783cddefb..2f5871d915c37 100644
--- a/drivers/gpu/nova-core/fb/hal/ga100.rs
+++ b/drivers/gpu/nova-core/fb/hal/ga100.rs
@@ -66,6 +66,12 @@ fn supports_display(&self, bar: &Bar0) -> bool {
     fn vidmem_size(&self, bar: &Bar0) -> u64 {
         super::tu102::vidmem_size_gp102(bar)
     }
+
+    // GA100 is a special case where its FRTS region exists, but is empty.  We
+    // return a size of 0 because we still need to record where the region starts.
+    fn frts_size(&self) -> u64 {
+        0
+    }
 }
 
 const GA100: Ga100 = Ga100;
diff --git a/drivers/gpu/nova-core/fb/hal/ga102.rs b/drivers/gpu/nova-core/fb/hal/ga102.rs
index 4b9f0f74d0e76..3bb66f64bef7c 100644
--- a/drivers/gpu/nova-core/fb/hal/ga102.rs
+++ b/drivers/gpu/nova-core/fb/hal/ga102.rs
@@ -35,6 +35,10 @@ fn supports_display(&self, bar: &Bar0) -> bool {
     fn vidmem_size(&self, bar: &Bar0) -> u64 {
         vidmem_size_ga102(bar)
     }
+
+    fn frts_size(&self) -> u64 {
+        super::tu102::frts_size_tu102()
+    }
 }
 
 const GA102: Ga102 = Ga102;
diff --git a/drivers/gpu/nova-core/fb/hal/tu102.rs b/drivers/gpu/nova-core/fb/hal/tu102.rs
index 281bb796e198e..22c174bf1472d 100644
--- a/drivers/gpu/nova-core/fb/hal/tu102.rs
+++ b/drivers/gpu/nova-core/fb/hal/tu102.rs
@@ -2,7 +2,8 @@
 
 use kernel::{
     io::Io,
-    prelude::*, //
+    prelude::*,
+    sizes::*, //
 };
 
 use crate::{
@@ -38,6 +39,10 @@ pub(super) fn vidmem_size_gp102(bar: &Bar0) -> u64 {
         .usable_fb_size()
 }
 
+pub(super) const fn frts_size_tu102() -> u64 {
+    u64::SZ_1M
+}
+
 struct Tu102;
 
 impl FbHal for Tu102 {
@@ -56,6 +61,10 @@ fn supports_display(&self, bar: &Bar0) -> bool {
     fn vidmem_size(&self, bar: &Bar0) -> u64 {
         vidmem_size_gp102(bar)
     }
+
+    fn frts_size(&self) -> u64 {
+        frts_size_tu102()
+    }
 }
 
 const TU102: Tu102 = Tu102;
diff --git a/drivers/gpu/nova-core/firmware.rs b/drivers/gpu/nova-core/firmware.rs
index 6c2ab69cb605e..3aac073efee2b 100644
--- a/drivers/gpu/nova-core/firmware.rs
+++ b/drivers/gpu/nova-core/firmware.rs
@@ -388,13 +388,7 @@ fn new(fw: &'a firmware::Firmware) -> Result<Self> {
             // Extract header.
             .and_then(BinHdr::from_bytes_copy)
             // Validate header.
-            .and_then(|hdr| {
-                if hdr.bin_magic == BIN_MAGIC {
-                    Some(hdr)
-                } else {
-                    None
-                }
-            })
+            .filter(|hdr| hdr.bin_magic == BIN_MAGIC)
             .map(|hdr| Self { hdr, fw })
             .ok_or(EINVAL)
     }
diff --git a/drivers/gpu/nova-core/firmware/gsp.rs b/drivers/gpu/nova-core/firmware/gsp.rs
index 2fcc255c3bc81..e576bc8a9b1cd 100644
--- a/drivers/gpu/nova-core/firmware/gsp.rs
+++ b/drivers/gpu/nova-core/firmware/gsp.rs
@@ -63,6 +63,21 @@ pub(crate) struct GspFirmware {
 }
 
 impl GspFirmware {
+    fn find_gsp_sigs_section(chipset: Chipset) -> &'static str {
+        match chipset.arch() {
+            Architecture::Turing if matches!(chipset, Chipset::TU116 | Chipset::TU117) => {
+                ".fwsignature_tu11x"
+            }
+            Architecture::Turing => ".fwsignature_tu10x",
+            Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_ga100",
+            Architecture::Ampere => ".fwsignature_ga10x",
+            Architecture::Ada => ".fwsignature_ad10x",
+            Architecture::Hopper => ".fwsignature_gh10x",
+            Architecture::BlackwellGB10x => ".fwsignature_gb10x",
+            Architecture::BlackwellGB20x => ".fwsignature_gb20x",
+        }
+    }
+
     /// Loads the GSP firmware binaries, map them into `dev`'s address-space, and creates the page
     /// tables expected by the GSP bootloader to load it.
     pub(crate) fn new<'a>(
@@ -131,18 +146,7 @@ pub(crate) fn new<'a>(
                 },
                 size,
                 signatures: {
-                    let sigs_section = match chipset.arch() {
-                        Architecture::Turing
-                            if matches!(chipset, Chipset::TU116 | Chipset::TU117) =>
-                        {
-                            ".fwsignature_tu11x"
-                        }
-                        Architecture::Turing => ".fwsignature_tu10x",
-                        // GA100 uses the same firmware as Turing
-                        Architecture::Ampere if chipset == Chipset::GA100 => ".fwsignature_tu10x",
-                        Architecture::Ampere => ".fwsignature_ga10x",
-                        Architecture::Ada => ".fwsignature_ad10x",
-                    };
+                    let sigs_section = Self::find_gsp_sigs_section(chipset);
 
                     elf::elf64_section(firmware.data(), sigs_section)
                         .ok_or(EINVAL)
diff --git a/drivers/gpu/nova-core/gfw.rs b/drivers/gpu/nova-core/gfw.rs
deleted file mode 100644
index fb75dd10a172e..0000000000000
--- a/drivers/gpu/nova-core/gfw.rs
+++ /dev/null
@@ -1,76 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-
-//! GPU Firmware (`GFW`) support, a.k.a `devinit`.
-//!
-//! Upon reset, the GPU runs some firmware code from the BIOS to setup its core parameters. Most of
-//! the GPU is considered unusable until this step is completed, so we must wait on it before
-//! performing driver initialization.
-//!
-//! A clarification about devinit terminology: devinit is a sequence of register read/writes after
-//! reset that performs tasks such as:
-//! 1. Programming VRAM memory controller timings.
-//! 2. Power sequencing.
-//! 3. Clock and PLL configuration.
-//! 4. Thermal management.
-//!
-//! devinit itself is a 'script' which is interpreted by an interpreter program typically running
-//! on the PMU microcontroller.
-//!
-//! Note that the devinit sequence also needs to run during suspend/resume.
-
-use kernel::{
-    io::{
-        poll::read_poll_timeout,
-        Io, //
-    },
-    prelude::*,
-    time::Delta, //
-};
-
-use crate::{
-    driver::Bar0,
-    regs, //
-};
-
-/// Wait for the `GFW` (GPU firmware) boot completion signal (`GFW_BOOT`), or a 4 seconds timeout.
-///
-/// Upon GPU reset, several microcontrollers (such as PMU, SEC2, GSP etc) run some firmware code to
-/// setup its core parameters. Most of the GPU is considered unusable until this step is completed,
-/// so it must be waited on very early during driver initialization.
-///
-/// The `GFW` code includes several components that need to execute before the driver loads. These
-/// components are located in the VBIOS ROM and executed in a sequence on these different
-/// microcontrollers. The devinit sequence typically runs on the PMU, and the FWSEC runs on the
-/// GSP.
-///
-/// This function waits for a signal indicating that core initialization is complete. Before this
-/// signal is received, little can be done with the GPU. This signal is set by the FWSEC running on
-/// the GSP in Heavy-secured mode.
-pub(crate) fn wait_gfw_boot_completion(bar: &Bar0) -> Result {
-    // Before accessing the completion status in `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05`, we must
-    // first check `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK`. This is because
-    // `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05` becomes accessible only after the secure firmware
-    // (FWSEC) lowers the privilege level to allow CPU (LS/Light-secured) access. We can only
-    // safely read the status register from CPU (LS/Light-secured) once the mask indicates
-    // that the privilege level has been lowered.
-    //
-    // TIMEOUT: arbitrarily large value. GFW starts running immediately after the GPU is put out of
-    // reset, and should complete in less time than that.
-    read_poll_timeout(
-        || {
-            Ok(
-                // Check that FWSEC has lowered its protection level before reading the GFW_BOOT
-                // status.
-                bar.read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK)
-                    .read_protection_level0()
-                    && bar
-                        .read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT)
-                        .completed(),
-            )
-        },
-        |&gfw_booted| gfw_booted,
-        Delta::from_millis(1),
-        Delta::from_secs(4),
-    )
-    .map(|_| ())
-}
diff --git a/drivers/gpu/nova-core/gpu.rs b/drivers/gpu/nova-core/gpu.rs
index 0f6fe9a1b9555..659f6a24ee130 100644
--- a/drivers/gpu/nova-core/gpu.rs
+++ b/drivers/gpu/nova-core/gpu.rs
@@ -20,11 +20,12 @@
         Falcon, //
     },
     fb::SysmemFlush,
-    gfw,
     gsp::Gsp,
     regs,
 };
 
+mod hal;
+
 macro_rules! define_chipset {
     ({ $($variant:ident = $value:expr),* $(,)* }) =>
     {
@@ -86,12 +87,23 @@ fn try_from(value: u32) -> Result<Self, Self::Error> {
     GA104 = 0x174,
     GA106 = 0x176,
     GA107 = 0x177,
+    // Hopper
+    GH100 = 0x180,
     // Ada
     AD102 = 0x192,
     AD103 = 0x193,
     AD104 = 0x194,
     AD106 = 0x196,
     AD107 = 0x197,
+    // Blackwell GB10x
+    GB100 = 0x1a0,
+    GB102 = 0x1a2,
+    // Blackwell GB20x
+    GB202 = 0x1b2,
+    GB203 = 0x1b3,
+    GB205 = 0x1b5,
+    GB206 = 0x1b6,
+    GB207 = 0x1b7,
 });
 
 impl Chipset {
@@ -103,9 +115,14 @@ pub(crate) const fn arch(self) -> Architecture {
             Self::GA100 | Self::GA102 | Self::GA103 | Self::GA104 | Self::GA106 | Self::GA107 => {
                 Architecture::Ampere
             }
+            Self::GH100 => Architecture::Hopper,
             Self::AD102 | Self::AD103 | Self::AD104 | Self::AD106 | Self::AD107 => {
                 Architecture::Ada
             }
+            Self::GB100 | Self::GB102 => Architecture::BlackwellGB10x,
+            Self::GB202 | Self::GB203 | Self::GB205 | Self::GB206 | Self::GB207 => {
+                Architecture::BlackwellGB20x
+            }
         }
     }
 
@@ -137,10 +154,14 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
     pub(crate) enum Architecture with TryFrom<Bounded<u32, 6>> {
         Turing = 0x16,
         Ampere = 0x17,
+        Hopper = 0x18,
         Ada = 0x19,
+        BlackwellGB10x = 0x1a,
+        BlackwellGB20x = 0x1b,
     }
 }
 
+#[derive(Clone, Copy)]
 pub(crate) struct Revision {
     major: Bounded<u8, 4>,
     minor: Bounded<u8, 4>,
@@ -162,6 +183,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
 }
 
 /// Structure holding a basic description of the GPU: `Chipset` and `Revision`.
+#[derive(Clone, Copy)]
 pub(crate) struct Spec {
     chipset: Chipset,
     revision: Revision,
@@ -253,7 +275,7 @@ pub(crate) fn new<'a>(
 
             // We must wait for GFW_BOOT completion before doing any significant setup on the GPU.
             _: {
-                gfw::wait_gfw_boot_completion(bar)
+                hal::gpu_hal(spec.chipset).wait_gfw_boot_completion(bar)
                     .inspect_err(|_| dev_err!(pdev, "GFW boot did not complete\n"))?;
             },
 
diff --git a/drivers/gpu/nova-core/gpu/hal.rs b/drivers/gpu/nova-core/gpu/hal.rs
new file mode 100644
index 0000000000000..788de20ab5d3b
--- /dev/null
+++ b/drivers/gpu/nova-core/gpu/hal.rs
@@ -0,0 +1,28 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use kernel::prelude::*;
+
+use crate::{
+    driver::Bar0,
+    gpu::{
+        Architecture,
+        Chipset, //
+    },
+};
+
+mod gh100;
+mod tu102;
+
+pub(crate) trait GpuHal {
+    /// Waits for GFW_BOOT completion if required by this hardware family.
+    fn wait_gfw_boot_completion(&self, bar: &Bar0) -> Result;
+}
+
+pub(super) fn gpu_hal(chipset: Chipset) -> &'static dyn GpuHal {
+    match chipset.arch() {
+        Architecture::Turing | Architecture::Ampere | Architecture::Ada => tu102::TU102_HAL,
+        Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x => {
+            gh100::GH100_HAL
+        }
+    }
+}
diff --git a/drivers/gpu/nova-core/gpu/hal/gh100.rs b/drivers/gpu/nova-core/gpu/hal/gh100.rs
new file mode 100644
index 0000000000000..1ed5bccdda1d0
--- /dev/null
+++ b/drivers/gpu/nova-core/gpu/hal/gh100.rs
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0
+
+use kernel::prelude::*;
+
+use crate::driver::Bar0;
+
+use super::GpuHal;
+
+struct Gh100;
+
+impl GpuHal for Gh100 {
+    fn wait_gfw_boot_completion(&self, _bar: &Bar0) -> Result {
+        Ok(())
+    }
+}
+
+const GH100: Gh100 = Gh100;
+pub(super) const GH100_HAL: &dyn GpuHal = &GH100;
diff --git a/drivers/gpu/nova-core/gpu/hal/tu102.rs b/drivers/gpu/nova-core/gpu/hal/tu102.rs
new file mode 100644
index 0000000000000..08dd4434bd72a
--- /dev/null
+++ b/drivers/gpu/nova-core/gpu/hal/tu102.rs
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! GPU Firmware (`GFW`) support, a.k.a `devinit`.
+//!
+//! Upon reset, the GPU runs some firmware code from the BIOS to setup its core parameters. Most of
+//! the GPU is considered unusable until this step is completed, so we must wait on it before
+//! performing driver initialization.
+//!
+//! A clarification about devinit terminology: devinit is a sequence of register read/writes after
+//! reset that performs tasks such as:
+//! 1. Programming VRAM memory controller timings.
+//! 2. Power sequencing.
+//! 3. Clock and PLL configuration.
+//! 4. Thermal management.
+//!
+//! devinit itself is a 'script' which is interpreted by an interpreter program typically running
+//! on the PMU microcontroller.
+//!
+//! Note that the devinit sequence also needs to run during suspend/resume.
+
+use kernel::{
+    io::{
+        poll::read_poll_timeout,
+        Io, //
+    },
+    prelude::*,
+    time::Delta, //
+};
+
+use crate::{
+    driver::Bar0,
+    regs, //
+};
+
+use super::GpuHal;
+
+struct Tu102;
+
+impl GpuHal for Tu102 {
+    /// Wait for the `GFW` (GPU firmware) boot completion signal (`GFW_BOOT`), or a 4 seconds
+    /// timeout.
+    ///
+    /// Upon GPU reset, several microcontrollers (such as PMU, SEC2, GSP etc) run some firmware
+    /// code to setup its core parameters. Most of the GPU is considered unusable until this step
+    /// is completed, so it must be waited on very early during driver initialization.
+    ///
+    /// The `GFW` code includes several components that need to execute before the driver loads.
+    /// These components are located in the VBIOS ROM and executed in a sequence on these different
+    /// microcontrollers. The devinit sequence typically runs on the PMU, and the FWSEC runs on the
+    /// GSP.
+    ///
+    /// This function waits for a signal indicating that core initialization is complete. Before
+    /// this signal is received, little can be done with the GPU. This signal is set by the FWSEC
+    /// running on the GSP in Heavy-secured mode.
+    fn wait_gfw_boot_completion(&self, bar: &Bar0) -> Result {
+        // Before accessing the completion status in `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05`, we must
+        // first check `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK`. This is because
+        // `NV_PGC6_AON_SECURE_SCRATCH_GROUP_05` becomes accessible only after the secure firmware
+        // (FWSEC) lowers the privilege level to allow CPU (LS/Light-secured) access. We can only
+        // safely read the status register from CPU (LS/Light-secured) once the mask indicates
+        // that the privilege level has been lowered.
+        //
+        // TIMEOUT: arbitrarily large value. GFW starts running immediately after the GPU is put
+        // out of reset, and should complete in less time than that.
+        read_poll_timeout(
+            || {
+                Ok(
+                    // Check that FWSEC has lowered its protection level before reading the
+                    // GFW_BOOT status.
+                    bar.read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_PRIV_LEVEL_MASK)
+                        .read_protection_level0()
+                        && bar
+                            .read(regs::NV_PGC6_AON_SECURE_SCRATCH_GROUP_05_0_GFW_BOOT)
+                            .completed(),
+                )
+            },
+            |&gfw_booted| gfw_booted,
+            Delta::from_millis(1),
+            Delta::from_secs(4),
+        )
+        .map(|_| ())
+    }
+}
+
+const TU102: Tu102 = Tu102;
+pub(super) const TU102_HAL: &dyn GpuHal = &TU102;
diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs
index 18f356c9178eb..df105ef4b3718 100644
--- a/drivers/gpu/nova-core/gsp/boot.rs
+++ b/drivers/gpu/nova-core/gsp/boot.rs
@@ -31,7 +31,10 @@
         gsp::GspFirmware,
         FIRMWARE_VERSION, //
     },
-    gpu::Chipset,
+    gpu::{
+        Architecture,
+        Chipset, //
+    },
     gsp::{
         commands,
         sequencer::{
@@ -146,6 +149,14 @@ pub(crate) fn boot(
         gsp_falcon: &Falcon<Gsp>,
         sec2_falcon: &Falcon<Sec2>,
     ) -> Result {
+        // The FSP boot process of Hopper+ is not supported for now.
+        if matches!(
+            chipset.arch(),
+            Architecture::Hopper | Architecture::BlackwellGB10x | Architecture::BlackwellGB20x
+        ) {
+            return Err(ENOTSUPP);
+        }
+
         let dev = pdev.as_ref();
 
         let bios = Vbios::new(dev, bar)?;
@@ -155,7 +166,10 @@ pub(crate) fn boot(
         let fb_layout = FbLayout::new(chipset, bar, &gsp_fw)?;
         dev_dbg!(dev, "{:#x?}\n", fb_layout);
 
-        Self::run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?;
+        // FWSEC-FRTS is not executed on chips where the FRTS region size is 0 (e.g. GA100).
+        if !fb_layout.frts.is_empty() {
+            Self::run_fwsec_frts(dev, chipset, gsp_falcon, bar, &bios, &fb_layout)?;
+        }
 
         let booter_loader = BooterFirmware::new(
             dev,
diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs
index 0c8a74f0e8ac2..3245793bbe422 100644
--- a/drivers/gpu/nova-core/gsp/fw.rs
+++ b/drivers/gpu/nova-core/gsp/fw.rs
@@ -17,8 +17,8 @@
         KnownSize, //
     },
     sizes::{
-        SZ_128K,
-        SZ_1M, //
+        SizeConstants,
+        SZ_128K, //
     },
     transmute::{
         AsBytes,
@@ -122,13 +122,14 @@ fn client_alloc_size() -> u64 {
 
     /// Returns the amount of memory to reserve for management purposes for a framebuffer of size
     /// `fb_size`.
-    fn management_overhead(fb_size: u64) -> u64 {
-        let fb_size_gb = fb_size.div_ceil(u64::from_safe_cast(kernel::sizes::SZ_1G));
+    fn management_overhead(fb_size: u64) -> Result<u64> {
+        let fb_size_gb = fb_size.div_ceil(u64::SZ_1G);
 
         u64::from(bindings::GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB)
-            .saturating_mul(fb_size_gb)
+            .checked_mul(fb_size_gb)
+            .ok_or(EINVAL)?
             .align_up(GSP_HEAP_ALIGNMENT)
-            .unwrap_or(u64::MAX)
+            .ok_or(EINVAL)
     }
 }
 
@@ -145,9 +146,8 @@ impl LibosParams {
     const LIBOS2: LibosParams = LibosParams {
         carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS2),
         allowed_heap_size: num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MIN_MB)
-            * num::usize_as_u64(SZ_1M)
-            ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB)
-                * num::usize_as_u64(SZ_1M),
+            * u64::SZ_1M
+            ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS2_MAX_MB) * u64::SZ_1M,
     };
 
     /// Version 3 of the GSP LIBOS (GA102+)
@@ -155,9 +155,9 @@ impl LibosParams {
         carveout_size: num::u32_as_u64(bindings::GSP_FW_HEAP_PARAM_OS_SIZE_LIBOS3_BAREMETAL),
         allowed_heap_size: num::u32_as_u64(
             bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MIN_MB,
-        ) * num::usize_as_u64(SZ_1M)
+        ) * u64::SZ_1M
             ..num::u32_as_u64(bindings::GSP_FW_HEAP_SIZE_OVERRIDE_LIBOS3_BAREMETAL_MAX_MB)
-                * num::usize_as_u64(SZ_1M),
+                * u64::SZ_1M,
     };
 
     /// Returns the libos parameters corresponding to `chipset`.
@@ -171,18 +171,19 @@ pub(crate) fn from_chipset(chipset: Chipset) -> &'static LibosParams {
 
     /// Returns the amount of memory (in bytes) to allocate for the WPR heap for a framebuffer size
     /// of `fb_size` (in bytes) for `chipset`.
-    pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> u64 {
+    pub(crate) fn wpr_heap_size(&self, chipset: Chipset, fb_size: u64) -> Result<u64> {
         // The WPR heap will contain the following:
         // LIBOS carveout,
-        self.carveout_size
+        Ok(self
+            .carveout_size
             // RM boot working memory,
             .saturating_add(GspFwHeapParams::base_rm_size(chipset))
             // One RM client,
             .saturating_add(GspFwHeapParams::client_alloc_size())
             // Overhead for memory management.
-            .saturating_add(GspFwHeapParams::management_overhead(fb_size))
+            .saturating_add(GspFwHeapParams::management_overhead(fb_size)?)
             // Clamp to the supported heap sizes.
-            .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1)
+            .clamp(self.allowed_heap_size.start, self.allowed_heap_size.end - 1))
     }
 }
 
diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs
index 04a1fa6b25f8e..c488058880be0 100644
--- a/drivers/gpu/nova-core/nova_core.rs
+++ b/drivers/gpu/nova-core/nova_core.rs
@@ -17,7 +17,6 @@
 mod falcon;
 mod fb;
 mod firmware;
-mod gfw;
 mod gpu;
 mod gsp;
 #[macro_use]
@@ -53,7 +52,7 @@ struct NovaCoreModule {
 
 impl InPlaceModule for NovaCoreModule {
     fn init(module: &'static kernel::ThisModule) -> impl PinInit<Self, Error> {
-        let dir = debugfs::Dir::new(kernel::c_str!("nova_core"));
+        let dir = debugfs::Dir::new(kernel::c_str!("nova-core"));
 
         // SAFETY: We are the only driver code running during init, so there
         // cannot be any concurrent access to `DEBUGFS_ROOT`.
@@ -68,7 +67,7 @@ fn init(module: &'static kernel::ThisModule) -> impl PinInit<Self, Error> {
 
 module! {
     type: NovaCoreModule,
-    name: "NovaCore",
+    name: "nova-core",
     authors: ["Danilo Krummrich"],
     description: "Nova Core GPU driver",
     license: "GPL v2",
diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs
index 2f171a4ff9ba8..6faeed73901d4 100644
--- a/drivers/gpu/nova-core/regs.rs
+++ b/drivers/gpu/nova-core/regs.rs
@@ -7,6 +7,7 @@
         Io, //
     },
     prelude::*,
+    sizes::SizeConstants,
     time, //
 };
 
@@ -30,7 +31,6 @@
         Architecture,
         Chipset, //
     },
-    num::FromSafeCast,
 };
 
 // PMC
@@ -150,8 +150,7 @@ fn fmt(&self, f: &mut kernel::fmt::Formatter<'_>) -> kernel::fmt::Result {
 impl NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE {
     /// Returns the usable framebuffer size, in bytes.
     pub(crate) fn usable_fb_size(self) -> u64 {
-        let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale()))
-            * u64::from_safe_cast(kernel::sizes::SZ_1M);
+        let size = (u64::from(self.lower_mag()) << u64::from(self.lower_scale())) * u64::SZ_1M;
 
         if self.ecc_mode_enabled() {
             // Remove the amount of memory reserved for ECC (one per 16 units).
@@ -241,7 +240,7 @@ pub(crate) fn completed(self) -> bool {
 impl NV_USABLE_FB_SIZE_IN_MB {
     /// Returns the usable framebuffer size, in bytes.
     pub(crate) fn usable_fb_size(self) -> u64 {
-        u64::from(self.value()) * u64::from_safe_cast(kernel::sizes::SZ_1M)
+        u64::from(self.value()) * u64::SZ_1M
     }
 }
 
diff --git a/drivers/gpu/nova-core/vbios.rs b/drivers/gpu/nova-core/vbios.rs
index ebda28e596c5f..6bcfb6c5cf447 100644
--- a/drivers/gpu/nova-core/vbios.rs
+++ b/drivers/gpu/nova-core/vbios.rs
@@ -12,6 +12,8 @@
         Alignable,
         Alignment, //
     },
+    register,
+    sizes::SZ_4K,
     sync::aref::ARef,
     transmute::FromBytes,
 };
@@ -89,13 +91,94 @@ struct VbiosIterator<'a> {
     last_found: bool,
 }
 
+// IFR Header in VBIOS.
+
+register! {
+    pub(crate) NV_PBUS_IFR_FMT_FIXED0(u32) @ 0x300000 {
+        31:0    signature;
+    }
+}
+
+register! {
+    pub(crate) NV_PBUS_IFR_FMT_FIXED1(u32) @ 0x300004 {
+        30:16   fixed_data_size;
+        15:8    version => u8;
+    }
+}
+
+register! {
+    pub(crate) NV_PBUS_IFR_FMT_FIXED2(u32) @ 0x300008 {
+        19:0 total_data_size;
+    }
+}
+
+/// Return the byte offset where the PCI Expansion ROM images begin in the GPU's ROM.
+///
+/// The GPU's ROM may begin with an Init-from-ROM (IFR) header that precedes
+/// the PCI Expansion ROM images (VBIOS).  When present, the PROM shadow
+/// method must parse this header to determine the offset where the PCI ROM
+/// images actually begin, and adjust all subsequent reads accordingly.
+///
+/// On most GPUs this is not needed because the IFR microcode has already
+/// applied the ROM offset so that PROM reads transparently skip the header.
+/// On GA100, for some reason, the IFR offset is not applied to PROM
+/// reads.  Therefore, the search for the PCI expansion must skip the IFR
+/// header, if found.
+fn vbios_rom_offset(dev: &device::Device, bar0: &Bar0) -> Result<usize> {
+    /// IFR signature.
+    const NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE: u32 = u32::from_le_bytes(*b"NVGI");
+    /// ROM directory signature.
+    const NV_ROM_DIRECTORY_IDENTIFIER: u32 = u32::from_le_bytes(*b"RFRD");
+    /// Offset of the NV_PMGR_ROM_ADDR_OFFSET register in IFR Extended section.
+    const IFR_SW_EXT_ROM_ADDR_OFFSET: usize = 4;
+    /// Size of Redundant Firmware Flash Status section.
+    const RFW_FLASH_STATUS_SIZE: usize = SZ_4K;
+    /// Offset in the ROM Directory of the PCI Option ROM offset
+    const PCI_OPTION_ROM_OFFSET: usize = 8;
+
+    let signature = bar0.read(NV_PBUS_IFR_FMT_FIXED0).signature();
+
+    if signature == NV_PBUS_IFR_FMT_FIXED0_SIGNATURE_VALUE {
+        let fixed1 = bar0.read(NV_PBUS_IFR_FMT_FIXED1);
+
+        match fixed1.version() {
+            1 | 2 => {
+                let fixed_data_size = usize::from(fixed1.fixed_data_size());
+                let pmgr_rom_addr_offset = fixed_data_size + IFR_SW_EXT_ROM_ADDR_OFFSET;
+                bar0.try_read32(ROM_OFFSET + pmgr_rom_addr_offset)
+                    .map(usize::from_safe_cast)
+            }
+            3 => {
+                let fixed2 = bar0.read(NV_PBUS_IFR_FMT_FIXED2);
+                let total_data_size = usize::from(fixed2.total_data_size());
+                let flash_status_offset =
+                    usize::from_safe_cast(bar0.try_read32(ROM_OFFSET + total_data_size)?);
+                let dir_offset = flash_status_offset + RFW_FLASH_STATUS_SIZE;
+                let dir_sig = bar0.try_read32(ROM_OFFSET + dir_offset)?;
+                if dir_sig != NV_ROM_DIRECTORY_IDENTIFIER {
+                    dev_err!(dev, "could not find IFR ROM directory\n");
+                    return Err(EINVAL);
+                }
+                bar0.try_read32(ROM_OFFSET + dir_offset + PCI_OPTION_ROM_OFFSET)
+                    .map(usize::from_safe_cast)
+            }
+            _ => {
+                dev_err!(dev, "unsupported IFR header version {}\n", fixed1.version());
+                Err(EINVAL)
+            }
+        }
+    } else {
+        Ok(0)
+    }
+}
+
 impl<'a> VbiosIterator<'a> {
     fn new(dev: &'a device::Device, bar0: &'a Bar0) -> Result<Self> {
         Ok(Self {
             dev,
             bar0,
             data: KVec::new(),
-            current_offset: 0,
+            current_offset: vbios_rom_offset(dev, bar0)?,
             last_found: false,
         })
     }
@@ -491,7 +574,7 @@ fn new(dev: &device::Device, data: &[u8]) -> Result<Self> {
 
         // Check for valid ROM signatures.
         match signature {
-            0xAA55 | 0xBB77 | 0x4E56 => {}
+            0xAA55 | 0x4E56 => {}
             _ => {
                 dev_err!(dev, "ROM signature unknown {:#x}\n", signature);
                 return Err(EINVAL);
diff --git a/drivers/hid/Kconfig b/drivers/hid/Kconfig
index ff2f580b660ba..f9bcaeb66385b 100644
--- a/drivers/hid/Kconfig
+++ b/drivers/hid/Kconfig
@@ -766,6 +766,15 @@ config HID_MEGAWORLD_FF
 	Say Y here if you have a Mega World based game controller and want
 	to have force feedback support for it.
 
+config HID_RAKK
+	tristate "Rakk support"
+	help
+	  Support for Rakk gaming peripherals.
+
+	  Fixes the HID report descriptor of the Rakk Dasig X mouse,
+	  which declares USAGE_MAXIMUM=3 (buttons 1-3) while actually
+	  sending 5 button bits. This causes side buttons to be ignored.
+
 config HID_REDRAGON
 	tristate "Redragon keyboards"
 	help
@@ -896,6 +905,19 @@ config HID_ORTEK
 	   - Ortek WKB-2000
 	   - Skycable wireless presenter
 
+config HID_OXP
+	tristate "OneXPlayer handheld controller configuration support"
+	depends on USB_HID
+	depends on LEDS_CLASS
+	depends on LEDS_CLASS_MULTICOLOR
+	depends on DMI
+	help
+	  Say Y here if you would like to enable support for OneXPlayer handheld
+	  devices that come with RGB LED rings around the joysticks and macro buttons.
+
+	  To compile this driver as a module, choose M here: the module will
+	  be called hid-oxp.
+
 config HID_PANTHERLORD
 	tristate "Pantherlord/GreenAsia game controller"
 	help
diff --git a/drivers/hid/Makefile b/drivers/hid/Makefile
index 0597fd6a4ffd3..23e6e3dd0c56a 100644
--- a/drivers/hid/Makefile
+++ b/drivers/hid/Makefile
@@ -98,6 +98,7 @@ obj-$(CONFIG_HID_NTI)			+= hid-nti.o
 obj-$(CONFIG_HID_NTRIG)		+= hid-ntrig.o
 obj-$(CONFIG_HID_NVIDIA_SHIELD)	+= hid-nvidia-shield.o
 obj-$(CONFIG_HID_ORTEK)		+= hid-ortek.o
+obj-$(CONFIG_HID_OXP)		+= hid-oxp.o
 obj-$(CONFIG_HID_PRODIKEYS)	+= hid-prodikeys.o
 obj-$(CONFIG_HID_PANTHERLORD)	+= hid-pl.o
 obj-$(CONFIG_HID_PENMOUNT)	+= hid-penmount.o
@@ -115,6 +116,7 @@ obj-$(CONFIG_HID_PLANTRONICS)	+= hid-plantronics.o
 obj-$(CONFIG_HID_PLAYSTATION)	+= hid-playstation.o
 obj-$(CONFIG_HID_PRIMAX)	+= hid-primax.o
 obj-$(CONFIG_HID_PXRC)		+= hid-pxrc.o
+obj-$(CONFIG_HID_RAKK)		+= hid-rakk.o
 obj-$(CONFIG_HID_RAPOO) += hid-rapoo.o
 obj-$(CONFIG_HID_RAZER)	+= hid-razer.o
 obj-$(CONFIG_HID_REDRAGON)	+= hid-redragon.o
diff --git a/drivers/hid/hid-asus.c b/drivers/hid/hid-asus.c
index 3f5e96900b67a..d34d74df3dc0b 100644
--- a/drivers/hid/hid-asus.c
+++ b/drivers/hid/hid-asus.c
@@ -965,6 +965,12 @@ static int asus_input_configured(struct hid_device *hdev, struct hid_input *hi)
 		}
 	}
 
+	if (drvdata->quirks & QUIRK_HID_FN_LOCK) {
+		drvdata->fn_lock = true;
+		INIT_WORK(&drvdata->fn_lock_sync_work, asus_sync_fn_lock);
+		asus_kbd_set_fn_lock(hdev, true);
+	}
+
 	if (drvdata->tp) {
 		int ret;
 
diff --git a/drivers/hid/hid-core.c b/drivers/hid/hid-core.c
index b3596851c7191..41a79e43c82b3 100644
--- a/drivers/hid/hid-core.c
+++ b/drivers/hid/hid-core.c
@@ -2050,7 +2050,7 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *
 		return 0;
 
 	if (unlikely(bsize < csize)) {
-		hid_warn_ratelimited(hid, "Event data for report %d is incorrect (%d vs %ld)\n",
+		hid_warn_ratelimited(hid, "Event data for report %d is incorrect (%d vs %zu)\n",
 				     report->id, csize, bsize);
 		return -EINVAL;
 	}
@@ -2072,7 +2072,7 @@ int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *
 		rsize = max_buffer_size;
 
 	if (bsize < rsize) {
-		hid_warn_ratelimited(hid, "Event data for report %d was too short (%d vs %ld)\n",
+		hid_warn_ratelimited(hid, "Event data for report %d was too short (%d vs %zu)\n",
 				     report->id, rsize, bsize);
 		return -EINVAL;
 	}
diff --git a/drivers/hid/hid-ids.h b/drivers/hid/hid-ids.h
index 4657d96fb0836..20e7d2dbbf7cc 100644
--- a/drivers/hid/hid-ids.h
+++ b/drivers/hid/hid-ids.h
@@ -1131,6 +1131,12 @@
 #define USB_VENDOR_ID_NVIDIA				0x0955
 #define USB_DEVICE_ID_NVIDIA_THUNDERSTRIKE_CONTROLLER	0x7214
 
+#define USB_VENDOR_ID_CRSC			0x1a2c
+#define USB_DEVICE_ID_ONEXPLAYER_GEN1		0xb001
+
+#define USB_VENDOR_ID_WCH			0x1a86
+#define USB_DEVICE_ID_ONEXPLAYER_GEN2		0xfe00
+
 #define USB_VENDOR_ID_ONTRAK		0x0a07
 #define USB_DEVICE_ID_ONTRAK_ADU100	0x0064
 
@@ -1405,6 +1411,11 @@
 #define USB_DEVICE_ID_SYNAPTICS_ACER_SWITCH5_017	0x73f6
 #define USB_DEVICE_ID_SYNAPTICS_ACER_SWITCH5	0x81a7
 
+#define USB_VENDOR_ID_TELINK				0x248a
+#define USB_DEVICE_ID_TELINK_RAKK_DASIG_X		0xfb01
+#define USB_DEVICE_ID_TELINK_RAKK_DASIG_X_DONGLE	0xfa02
+#define USB_DEVICE_ID_TELINK_RAKK_DASIG_X_BT		0x8266
+
 #define USB_VENDOR_ID_TEXAS_INSTRUMENTS	0x2047
 #define USB_DEVICE_ID_TEXAS_INSTRUMENTS_LENOVO_YOGA	0x0855
 
diff --git a/drivers/hid/hid-multitouch.c b/drivers/hid/hid-multitouch.c
index eeab0b6e32ccc..648201ef5d9cb 100644
--- a/drivers/hid/hid-multitouch.c
+++ b/drivers/hid/hid-multitouch.c
@@ -443,11 +443,13 @@ static const struct mt_class mt_classes[] = {
 			MT_QUIRK_CONTACT_CNT_ACCURATE,
 	},
 		{ .name = MT_CLS_YOGABOOK9I,
-		.quirks = MT_QUIRK_ALWAYS_VALID |
+		.quirks = MT_QUIRK_NOT_SEEN_MEANS_UP |
+			MT_QUIRK_ALWAYS_VALID |
 			MT_QUIRK_FORCE_MULTI_INPUT |
 			MT_QUIRK_SEPARATE_APP_REPORT |
 			MT_QUIRK_HOVERING |
 			MT_QUIRK_YOGABOOK9I,
+		.maxcontacts = 10,
 		.export_all_inputs = true
 	},
 	{ .name = MT_CLS_EGALAX_P80H84,
@@ -1566,6 +1568,144 @@ static int mt_event(struct hid_device *hid, struct hid_field *field,
 	return 0;
 }
 
+/*
+ * Yoga Book 9 14IAH10 descriptor fixup.
+ *
+ * The device includes a HID_DG_TOUCHPAD application collection designed for
+ * the Windows inbox HID driver's Win8 PTP touchpad mode.  On Linux we want
+ * only the HID_DG_TOUCHSCREEN collections.  The touchpad collection (and the
+ * HID_DG_BUTTONTYPE and Win8 compliance blob features it contains) must be
+ * removed so hid-multitouch does not misclassify the touchscreen nodes as
+ * indirect buttonpads.
+ *
+ * The firmware also resets if any USB control request is received while the
+ * CDC-ACM interface is initialising (~1.18 s after enumeration).  Dropping
+ * the Win8 blob and Contact Count Max feature reports prevents the
+ * GET_REPORT calls that hid-multitouch issues at probe.
+ */
+static void mt_yogabook9_fixup(struct hid_device *hdev, __u8 *rdesc,
+			       unsigned int *size)
+{
+	/* Usage Page (Digitizer), Usage (Touch Pad), Collection (Application) */
+	static const __u8 tp_app_hdr[] = { 0x05, 0x0d, 0x09, 0x05, 0xa1, 0x01 };
+	/* Vendor Usage Page 0xff00 (Win8 compliance blob header) */
+	static const __u8 win8_page[] = { 0x06, 0x00, 0xff };
+	/* Usage (Contact Count Max = 0x55) */
+	static const __u8 ccmax_usage[] = { 0x09, 0x55 };
+	unsigned int i;
+
+	/*
+	 * Step 1: find and remove the Touch Pad application collection.
+	 * Walk HID short items from the collection header to its matching
+	 * End Collection, then close the gap with memmove.
+	 */
+	for (i = 0; i + sizeof(tp_app_hdr) <= *size; i++) {
+		if (memcmp(rdesc + i, tp_app_hdr, sizeof(tp_app_hdr)) == 0) {
+			__u8 *start = rdesc + i;
+			__u8 *coll_end = NULL;
+			__u8 *p = start;
+			unsigned int drop;
+			int depth = 0;
+
+			while (p < rdesc + *size) {
+				__u8 b = *p;
+				int ds = b & 3;
+				int item_len;
+
+				if (b == 0xfe) { /* long item */
+					if (p + 2 >= rdesc + *size)
+						break;
+					item_len = p[1] + 3;
+				} else {
+					item_len = (ds == 3) ? 5 : ds + 1;
+				}
+				if (p + item_len > rdesc + *size)
+					break;
+
+				if ((b & 0xfc) == 0xa0)
+					depth++; /* Collection */
+				else if (b == 0xc0) {
+					depth--; /* End Collection */
+					if (depth == 0) {
+						coll_end = p;
+						break;
+					}
+				}
+				p += item_len;
+			}
+
+			if (!coll_end) {
+				hid_err(hdev,
+					"Yoga Book 9: Touch Pad End Collection not found\n");
+				break;
+			}
+
+			drop = coll_end - start + 1;
+			memmove(start, coll_end + 1, rdesc + *size - coll_end - 1);
+			*size -= drop;
+			hid_dbg(hdev,
+				"Yoga Book 9: dropped Touch Pad collection (%u bytes)\n",
+				drop);
+			break;
+		}
+	}
+
+	/*
+	 * Step 2: neutralize Win8 compliance blob feature reports remaining
+	 * in the touchscreen collections.  Change Usage Page 0xff00 to 0x0f00
+	 * so the case 0xff0000c5 branch in mt_feature_mapping() is not reached
+	 * and no GET_REPORT is issued.
+	 */
+	for (i = 0; i + sizeof(win8_page) <= *size; i++) {
+		if (memcmp(rdesc + i, win8_page, sizeof(win8_page)) == 0) {
+			rdesc[i + 2] = 0x0f; /* 0xff00 -> 0x0f00 */
+			hid_dbg(hdev,
+				"Yoga Book 9: neutralized Win8 blob at offset %u\n",
+				i);
+		}
+	}
+
+	/*
+	 * Step 3: neutralize Contact Count Max feature reports.  Change usage
+	 * 0x55 (HID_DG_CONTACTMAX) to 0x00 so mt_feature_mapping() does not
+	 * issue GET_REPORT.  The class maxcontacts field provides the value.
+	 */
+	for (i = 0; i + sizeof(ccmax_usage) <= *size; i++) {
+		if (memcmp(rdesc + i, ccmax_usage, sizeof(ccmax_usage)) == 0) {
+			rdesc[i + 1] = 0x00;
+			hid_dbg(hdev,
+				"Yoga Book 9: neutralized ContactMax at offset %u\n",
+				i);
+		}
+	}
+
+	/*
+	 * Step 4: neutralize Surface Switch (0x57) and Button Switch (0x58)
+	 * feature report usages in the Device Configuration collection.
+	 * mt_set_modes() issues HID_REQ_SET_REPORT for these on every
+	 * input-device open/close; those repeated control requests hit the
+	 * firmware's CDC-ACM init window and trigger resets.
+	 *
+	 * Input Mode (0x52) is intentionally left intact.  mt_set_modes()
+	 * sends it once at probe to set the device into touchscreen mode,
+	 * which flushes the firmware's contact buffer and clears a persistent
+	 * ghost contact (cid 2, fixed coordinates) that otherwise appears on
+	 * every enumeration.  By probe time cdc_acm has already satisfied the
+	 * CDC-ACM init watchdog (~130 ms), so the single SET_REPORT for Input
+	 * Mode arrives safely after the reset window has closed.
+	 */
+	for (i = 0; i + 2 <= *size; i++) {
+		if (rdesc[i] == 0x09 &&
+		    (rdesc[i + 1] == 0x57 ||
+		     rdesc[i + 1] == 0x58)) {
+			hid_dbg(hdev,
+				"Yoga Book 9: neutralized set-modes usage 0x%02x at offset %u\n",
+				rdesc[i + 1], i);
+			rdesc[i + 1] = 0x00;
+		}
+	}
+}
+
 static const __u8 *mt_report_fixup(struct hid_device *hdev, __u8 *rdesc,
 			     unsigned int *size)
 {
@@ -1595,6 +1735,10 @@ got: %x\n",
 		}
 	}
 
+	if (hdev->vendor == USB_VENDOR_ID_LENOVO &&
+	    hdev->product == USB_DEVICE_ID_LENOVO_YOGABOOK9I)
+		mt_yogabook9_fixup(hdev, rdesc, size);
+
 	return rdesc;
 }
 
diff --git a/drivers/hid/hid-oxp.c b/drivers/hid/hid-oxp.c
new file mode 100644
index 0000000000000..20a54f337220d
--- /dev/null
+++ b/drivers/hid/hid-oxp.c
@@ -0,0 +1,1580 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ *  HID driver for OneXPlayer gamepad configuration devices.
+ *
+ *  Copyright (c) 2026 Valve Corporation
+ */
+
+#include <linux/array_size.h>
+#include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/device.h>
+#include <linux/dmi.h>
+#include <linux/hid.h>
+#include <linux/jiffies.h>
+#include <linux/kstrtox.h>
+#include <linux/led-class-multicolor.h>
+#include <linux/mutex.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+#include <linux/workqueue.h>
+
+#include "hid-ids.h"
+
+#define OXP_PACKET_SIZE 64
+
+#define GEN1_MESSAGE_ID	0xff
+#define GEN2_MESSAGE_ID	0x3f
+
+#define GEN1_USAGE_PAGE	0xff01
+#define GEN2_USAGE_PAGE	0xff00
+
+enum oxp_function_index {
+	OXP_FID_GEN1_RGB_SET =		0x07,
+	OXP_FID_GEN1_RGB_REPLY =	0x0f,
+	OXP_FID_GEN2_TOGGLE_MODE =	0xb2,
+	OXP_FID_GEN2_RUMBLE_SET =	0xb3,
+	OXP_FID_GEN2_KEY_STATE =	0xb4,
+	OXP_FID_GEN2_STATUS_EVENT =	0xb8,
+};
+
+#define OXP_MAPPING_GAMEPAD	0x01
+#define OXP_MAPPING_KEYBOARD	0x02
+
+struct oxp_button_data {
+	u8 mode;
+	u8 index;
+	u8 key_id;
+	u8 padding[2];
+} __packed;
+
+struct oxp_button_entry {
+	struct oxp_button_data data;
+	const char *name;
+};
+
+static const struct oxp_button_entry oxp_button_table[] = {
+	/* Gamepad Buttons */
+	{ { OXP_MAPPING_GAMEPAD, 0x01 }, "BTN_A" },
+	{ { OXP_MAPPING_GAMEPAD, 0x02 }, "BTN_B" },
+	{ { OXP_MAPPING_GAMEPAD, 0x03 }, "BTN_X" },
+	{ { OXP_MAPPING_GAMEPAD, 0x04 }, "BTN_Y" },
+	{ { OXP_MAPPING_GAMEPAD, 0x05 }, "BTN_LB" },
+	{ { OXP_MAPPING_GAMEPAD, 0x06 }, "BTN_RB" },
+	{ { OXP_MAPPING_GAMEPAD, 0x07 }, "BTN_LT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x08 }, "BTN_RT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x09 }, "BTN_START" },
+	{ { OXP_MAPPING_GAMEPAD, 0x0a }, "BTN_SELECT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x0b }, "BTN_L3" },
+	{ { OXP_MAPPING_GAMEPAD, 0x0c }, "BTN_R3" },
+	{ { OXP_MAPPING_GAMEPAD, 0x0d }, "DPAD_UP" },
+	{ { OXP_MAPPING_GAMEPAD, 0x0e }, "DPAD_DOWN" },
+	{ { OXP_MAPPING_GAMEPAD, 0x0f }, "DPAD_LEFT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x10 }, "DPAD_RIGHT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x11 }, "JOY_L_UP" },
+	{ { OXP_MAPPING_GAMEPAD, 0x12 }, "JOY_L_UP_RIGHT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x13 }, "JOY_L_RIGHT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x14 }, "JOY_L_DOWN_RIGHT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x15 }, "JOY_L_DOWN" },
+	{ { OXP_MAPPING_GAMEPAD, 0x16 }, "JOY_L_DOWN_LEFT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x17 }, "JOY_L_LEFT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x18 }, "JOY_L_UP_LEFT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x19 }, "JOY_R_UP" },
+	{ { OXP_MAPPING_GAMEPAD, 0x1a }, "JOY_R_UP_RIGHT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x1b }, "JOY_R_RIGHT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x1c }, "JOY_R_DOWN_RIGHT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x1d }, "JOY_R_DOWN" },
+	{ { OXP_MAPPING_GAMEPAD, 0x1e }, "JOY_R_DOWN_LEFT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x1f }, "JOY_R_LEFT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x20 }, "JOY_R_UP_LEFT" },
+	{ { OXP_MAPPING_GAMEPAD, 0x22 }, "BTN_GUIDE" },
+	/* Keyboard Keys */
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x5a }, "KEY_F1" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x5b }, "KEY_F2" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x5c }, "KEY_F3" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x5d }, "KEY_F4" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x5e }, "KEY_F5" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x5f }, "KEY_F6" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x60 }, "KEY_F7" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x61 }, "KEY_F8" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x62 }, "KEY_F9" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x63 }, "KEY_F10" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x64 }, "KEY_F11" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x65 }, "KEY_F12" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x66 }, "KEY_F13" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x67 }, "KEY_F14" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x68 }, "KEY_F15" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x69 }, "KEY_F16" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x6a }, "KEY_F17" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x6b }, "KEY_F18" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x6c }, "KEY_F19" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x6d }, "KEY_F20" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x6e }, "KEY_F21" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x6f }, "KEY_F22" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x70 }, "KEY_F23" },
+	{ { OXP_MAPPING_KEYBOARD, 0x01, 0x71 }, "KEY_F24" },
+};
+
+enum oxp_joybutton_index {
+	BUTTON_A =	0x01,
+	BUTTON_B,
+	BUTTON_X,
+	BUTTON_Y,
+	BUTTON_LB,
+	BUTTON_RB,
+	BUTTON_LT,
+	BUTTON_RT,
+	BUTTON_START,
+	BUTTON_SELECT,
+	BUTTON_L3,
+	BUTTON_R3,
+	BUTTON_DUP,
+	BUTTON_DDOWN,
+	BUTTON_DLEFT,
+	BUTTON_DRIGHT,
+	BUTTON_M1 =	0x22,
+	BUTTON_M2,
+	/* These are unused currently, reserved for future devices */
+	BUTTON_M3,
+	BUTTON_M4,
+	BUTTON_M5,
+	BUTTON_M6,
+};
+
+struct oxp_button_idx {
+	enum oxp_joybutton_index button_idx;
+	u8 mapping_idx;
+} __packed;
+
+struct oxp_bmap_page_1 {
+	struct oxp_button_idx btn_a;
+	struct oxp_button_idx btn_b;
+	struct oxp_button_idx btn_x;
+	struct oxp_button_idx btn_y;
+	struct oxp_button_idx btn_lb;
+	struct oxp_button_idx btn_rb;
+	struct oxp_button_idx btn_lt;
+	struct oxp_button_idx btn_rt;
+	struct oxp_button_idx btn_start;
+} __packed;
+
+struct oxp_bmap_page_2 {
+	struct oxp_button_idx btn_select;
+	struct oxp_button_idx btn_l3;
+	struct oxp_button_idx btn_r3;
+	struct oxp_button_idx btn_dup;
+	struct oxp_button_idx btn_ddown;
+	struct oxp_button_idx btn_dleft;
+	struct oxp_button_idx btn_dright;
+	struct oxp_button_idx btn_m1;
+	struct oxp_button_idx btn_m2;
+} __packed;
+
+static struct oxp_hid_cfg {
+	struct delayed_work oxp_rgb_queue;
+	struct delayed_work oxp_btn_queue;
+	struct oxp_bmap_page_1 *bmap_1;
+	struct oxp_bmap_page_2 *bmap_2;
+	struct delayed_work oxp_mcu_init;
+	struct led_classdev_mc *led_mc;
+	struct hid_device *hdev;
+	struct mutex cfg_mutex; /*ensure single synchronous output report*/
+	u8 rgb_brightness;
+	u8 gamepad_mode;
+	u8 rumble_intensity;
+	u8 rgb_effect;
+	u8 rgb_speed;
+	u8 rgb_en;
+} drvdata;
+
+#define OXP_FILL_PAGE_SLOT(page, btn)            \
+	{ .button_idx = (page)->btn.button_idx,  \
+	  .mapping_idx = (page)->btn.mapping_idx }
+
+enum oxp_gamepad_mode_index {
+	OXP_GP_MODE_XINPUT = 0x00,
+	OXP_GP_MODE_DEBUG = 0x03,
+};
+
+static const char *const oxp_gamepad_mode_text[] = {
+	[OXP_GP_MODE_XINPUT] = "xinput",
+	[OXP_GP_MODE_DEBUG] = "debug",
+};
+
+enum oxp_feature_en_index {
+	OXP_FEAT_DISABLED,
+	OXP_FEAT_ENABLED,
+};
+
+static const char *const oxp_feature_en_text[] = {
+	[OXP_FEAT_DISABLED] = "false",
+	[OXP_FEAT_ENABLED] = "true",
+};
+
+enum oxp_rgb_effect_index {
+	OXP_UNKNOWN,
+	OXP_EFFECT_AURORA,
+	OXP_EFFECT_BIRTHDAY,
+	OXP_EFFECT_FLOWING,
+	OXP_EFFECT_CHROMA_1,
+	OXP_EFFECT_NEON,
+	OXP_EFFECT_CHROMA_2,
+	OXP_EFFECT_DREAMY,
+	OXP_EFFECT_WARM,
+	OXP_EFFECT_CYBERPUNK,
+	OXP_EFFECT_SEA,
+	OXP_EFFECT_SUNSET,
+	OXP_EFFECT_COLORFUL,
+	OXP_EFFECT_MONSTER,
+	OXP_EFFECT_GREEN,
+	OXP_EFFECT_BLUE,
+	OXP_EFFECT_YELLOW,
+	OXP_EFFECT_TEAL,
+	OXP_EFFECT_PURPLE,
+	OXP_EFFECT_FOGGY,
+	OXP_EFFECT_MONO_LIST, /* placeholder for effect_index_show */
+};
+
+/* These belong to rgb_effect_index, but we want to hide them from
+ * rgb_effect_text
+ */
+
+#define OXP_GET_PROPERTY 0xfc
+#define OXP_SET_PROPERTY 0xfd
+#define OXP_EFFECT_MONO_TRUE 0xfe /* actual index for monocolor */
+
+static const char *const oxp_rgb_effect_text[] = {
+	[OXP_UNKNOWN] = "unknown",
+	[OXP_EFFECT_AURORA] = "aurora",
+	[OXP_EFFECT_BIRTHDAY] = "birthday_cake",
+	[OXP_EFFECT_FLOWING] = "flowing_light",
+	[OXP_EFFECT_CHROMA_1] = "chroma_popping",
+	[OXP_EFFECT_NEON] = "neon",
+	[OXP_EFFECT_CHROMA_2] = "chroma_breathing",
+	[OXP_EFFECT_DREAMY] = "dreamy",
+	[OXP_EFFECT_WARM] = "warm_sun",
+	[OXP_EFFECT_CYBERPUNK] = "cyberpunk",
+	[OXP_EFFECT_SEA] = "sea_foam",
+	[OXP_EFFECT_SUNSET] = "sunset_afterglow",
+	[OXP_EFFECT_COLORFUL] = "colorful",
+	[OXP_EFFECT_MONSTER] = "monster_woke",
+	[OXP_EFFECT_GREEN] = "green_breathing",
+	[OXP_EFFECT_BLUE] = "blue_breathing",
+	[OXP_EFFECT_YELLOW] = "yellow_breathing",
+	[OXP_EFFECT_TEAL] = "teal_breathing",
+	[OXP_EFFECT_PURPLE] = "purple_breathing",
+	[OXP_EFFECT_FOGGY] = "foggy_haze",
+	[OXP_EFFECT_MONO_LIST] = "monocolor",
+};
+
+enum oxp_rumble_side_index {
+	OXP_RUMBLE_LEFT = 0x00,
+	OXP_RUMBLE_RIGHT,
+};
+
+struct oxp_gen_1_rgb_report {
+	u8 report_id;
+	u8 message_id;
+	u8 padding_2[2];
+	u8 effect;
+	u8 enabled;
+	u8 speed;
+	u8 brightness;
+	u8 red;
+	u8 green;
+	u8 blue;
+} __packed;
+
+struct oxp_gen_2_rgb_report {
+	u8 report_id;
+	u8 header_id;
+	u8 padding_2;
+	u8 message_id;
+	u8 padding_4[2];
+	u8 enabled;
+	u8 speed;
+	u8 brightness;
+	u8 red;
+	u8 green;
+	u8 blue;
+	u8 padding_12[3];
+	u8 effect;
+} __packed;
+
+struct oxp_attr {
+	u8 index;
+};
+
+static u16 get_usage_page(struct hid_device *hdev)
+{
+	return hdev->collection[0].usage >> 16;
+}
+
+static int oxp_hid_raw_event_gen_1(struct hid_device *hdev,
+				   struct hid_report *report, u8 *data,
+				   int size)
+{
+	struct led_classdev_mc *led_mc = drvdata.led_mc;
+	struct oxp_gen_1_rgb_report *rgb_rep;
+
+	if (data[1] != OXP_FID_GEN1_RGB_REPLY)
+		return 0;
+
+	rgb_rep = (struct oxp_gen_1_rgb_report *)data;
+	/* Ensure we save monocolor as the list value */
+	drvdata.rgb_effect = rgb_rep->effect == OXP_EFFECT_MONO_TRUE ?
+			     OXP_EFFECT_MONO_LIST :
+			     rgb_rep->effect;
+	drvdata.rgb_speed = rgb_rep->speed;
+	drvdata.rgb_en = rgb_rep->enabled == 0 ? OXP_FEAT_DISABLED :
+						 OXP_FEAT_ENABLED;
+	drvdata.rgb_brightness = rgb_rep->brightness;
+	led_mc->led_cdev.brightness = rgb_rep->brightness / 4 *
+				      led_mc->led_cdev.max_brightness;
+	/* If monocolor had less than 100% brightness on the previous boot,
+	 * there will be no reliable way to determine the real intensity.
+	 * Since intensity scaling is used with a hardware brightness set at max,
+	 * our brightness will always look like 100%. Use the last set value to
+	 * prevent successive boots from lowering the brightness further.
+	 * Brightness will be "wrong" but the effect will remain the same visually.
+	 */
+	led_mc->subled_info[0].intensity = rgb_rep->red;
+	led_mc->subled_info[1].intensity = rgb_rep->green;
+	led_mc->subled_info[2].intensity = rgb_rep->blue;
+
+	return 0;
+}
+
+static int oxp_gen_2_property_out(enum oxp_function_index fid, u8 *data, u8 data_size);
+static int oxp_set_buttons(void);
+static int oxp_rumble_intensity_set(u8 intensity);
+
+static void oxp_mcu_init_fn(struct work_struct *work)
+{
+	u8 gp_mode_data[3] = { OXP_GP_MODE_DEBUG, 0x01, 0x02 };
+	int ret;
+
+	/* Re-apply the button mapping */
+	ret = oxp_set_buttons();
+	if (ret)
+		dev_err(&drvdata.hdev->dev,
+			"Error: Failed to set button mapping: %i\n", ret);
+
+	/* Cycle the gamepad mode */
+	ret = oxp_gen_2_property_out(OXP_FID_GEN2_TOGGLE_MODE, gp_mode_data, 3);
+	if (ret)
+		dev_err(&drvdata.hdev->dev,
+			"Error: Failed to set gamepad mode: %i\n", ret);
+
+	/* Remainder only applies for xinput mode */
+	if (drvdata.gamepad_mode == OXP_GP_MODE_DEBUG)
+		return;
+
+	gp_mode_data[0] = OXP_GP_MODE_XINPUT;
+	ret = oxp_gen_2_property_out(OXP_FID_GEN2_TOGGLE_MODE, gp_mode_data, 3);
+	if (ret)
+		dev_err(&drvdata.hdev->dev,
+			"Error: Failed to set gamepad mode: %i\n", ret);
+
+	/* Set vibration level */
+	ret = oxp_rumble_intensity_set(drvdata.rumble_intensity);
+	if (ret)
+		dev_err(&drvdata.hdev->dev,
+			"Error: Failed to set rumble intensity: %i\n", ret);
+}
+
+static int oxp_hid_raw_event_gen_2(struct hid_device *hdev,
+				   struct hid_report *report, u8 *data,
+				   int size)
+{
+	struct led_classdev_mc *led_mc = drvdata.led_mc;
+	struct oxp_gen_2_rgb_report *rgb_rep;
+
+	if (data[0] != OXP_FID_GEN2_STATUS_EVENT)
+		return 0;
+
+	/* Sent ~6s after resume event, indicating the MCU has fully reset.
+	 * Re-apply our settings after this has been received.
+	 */
+	if (data[3] == OXP_EFFECT_MONO_TRUE) {
+		mod_delayed_work(system_wq, &drvdata.oxp_mcu_init, msecs_to_jiffies(50));
+		return 0;
+	}
+
+	if (data[3] != OXP_GET_PROPERTY)
+		return 0;
+
+	rgb_rep = (struct oxp_gen_2_rgb_report *)data;
+	/* Ensure we save monocolor as the list value */
+	drvdata.rgb_effect = rgb_rep->effect == OXP_EFFECT_MONO_TRUE ?
+			     OXP_EFFECT_MONO_LIST :
+			     rgb_rep->effect;
+	drvdata.rgb_speed = rgb_rep->speed;
+	drvdata.rgb_en = rgb_rep->enabled == 0 ? OXP_FEAT_DISABLED :
+						 OXP_FEAT_ENABLED;
+	drvdata.rgb_brightness = rgb_rep->brightness;
+	led_mc->led_cdev.brightness = rgb_rep->brightness / 4 *
+				      led_mc->led_cdev.max_brightness;
+	/* If monocolor had less than 100% brightness on the previous boot,
+	 * there will be no reliable way to determine the real intensity.
+	 * Since intensity scaling is used with a hardware brightness set at max,
+	 * our brightness will always look like 100%. Use the last set value to
+	 * prevent successive boots from lowering the brightness further.
+	 * Brightness will be "wrong" but the effect will remain the same visually.
+	 */
+	led_mc->subled_info[0].intensity = rgb_rep->red;
+	led_mc->subled_info[1].intensity = rgb_rep->green;
+	led_mc->subled_info[2].intensity = rgb_rep->blue;
+
+	return 0;
+}
+
+static int oxp_hid_raw_event(struct hid_device *hdev, struct hid_report *report,
+			     u8 *data, int size)
+{
+	u16 up = get_usage_page(hdev);
+
+	dev_dbg(&hdev->dev, "raw event data: [%*ph]\n", OXP_PACKET_SIZE, data);
+
+	switch (up) {
+	case GEN1_USAGE_PAGE:
+		return oxp_hid_raw_event_gen_1(hdev, report, data, size);
+	case GEN2_USAGE_PAGE:
+		return oxp_hid_raw_event_gen_2(hdev, report, data, size);
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int mcu_property_out(u8 *header, size_t header_size, u8 *data,
+			    size_t data_size, u8 *footer, size_t footer_size)
+{
+	unsigned char *dmabuf __free(kfree) = kzalloc(OXP_PACKET_SIZE, GFP_KERNEL);
+	int ret;
+
+	if (!dmabuf)
+		return -ENOMEM;
+
+	if (header_size + data_size + footer_size > OXP_PACKET_SIZE)
+		return -EINVAL;
+
+	guard(mutex)(&drvdata.cfg_mutex);
+	memcpy(dmabuf, header, header_size);
+	memcpy(dmabuf + header_size, data, data_size);
+	if (footer_size)
+		memcpy(dmabuf + OXP_PACKET_SIZE - footer_size, footer, footer_size);
+
+	dev_dbg(&drvdata.hdev->dev, "raw data: [%*ph]\n", OXP_PACKET_SIZE, dmabuf);
+
+	ret = hid_hw_output_report(drvdata.hdev, dmabuf, OXP_PACKET_SIZE);
+	if (ret < 0)
+		return ret;
+
+	/* MCU takes 200ms to be ready for another command. */
+	msleep(200);
+	return ret == OXP_PACKET_SIZE ? 0 : -EIO;
+}
+
+static int oxp_gen_1_property_out(enum oxp_function_index fid, u8 *data,
+				  u8 data_size)
+{
+	u8 header[] = { fid, GEN1_MESSAGE_ID };
+	size_t header_size = ARRAY_SIZE(header);
+
+	return mcu_property_out(header, header_size, data, data_size, NULL, 0);
+}
+
+static int oxp_gen_2_property_out(enum oxp_function_index fid, u8 *data,
+				  u8 data_size)
+{
+	u8 header[] = { fid, GEN2_MESSAGE_ID, 0x01 };
+	u8 footer[] = { GEN2_MESSAGE_ID, fid };
+	size_t header_size = ARRAY_SIZE(header);
+	size_t footer_size = ARRAY_SIZE(footer);
+
+	return mcu_property_out(header, header_size, data, data_size, footer,
+				footer_size);
+}
+
+static ssize_t gamepad_mode_store(struct device *dev,
+				  struct device_attribute *attr, const char *buf,
+				  size_t count)
+{
+	u16 up = get_usage_page(drvdata.hdev);
+	u8 data[3] = { 0x00, 0x01, 0x02 };
+	int ret = -EINVAL;
+	int i;
+
+	if (up != GEN2_USAGE_PAGE)
+		return ret;
+
+	for (i = 0; i < ARRAY_SIZE(oxp_gamepad_mode_text); i++) {
+		if (oxp_gamepad_mode_text[i] && sysfs_streq(buf, oxp_gamepad_mode_text[i])) {
+			ret = i;
+			break;
+		}
+	}
+	if (ret < 0)
+		return ret;
+
+	data[0] = ret;
+
+	ret = oxp_gen_2_property_out(OXP_FID_GEN2_TOGGLE_MODE, data, 3);
+	if (ret)
+		return ret;
+
+	drvdata.gamepad_mode = data[0];
+
+	if (drvdata.gamepad_mode == OXP_GP_MODE_DEBUG)
+		return count;
+
+	/* Re-apply rumble settings as switching gamepad mode will override */
+	ret = oxp_rumble_intensity_set(drvdata.rumble_intensity);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static ssize_t gamepad_mode_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%s\n", oxp_gamepad_mode_text[drvdata.gamepad_mode]);
+}
+static DEVICE_ATTR_RW(gamepad_mode);
+
+static ssize_t gamepad_mode_index_show(struct device *dev,
+				       struct device_attribute *attr,
+				       char *buf)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(oxp_gamepad_mode_text); i++) {
+		if (!oxp_gamepad_mode_text[i] ||
+		    oxp_gamepad_mode_text[i][0] == '\0')
+			continue;
+
+		count += sysfs_emit_at(buf, count, "%s ", oxp_gamepad_mode_text[i]);
+	}
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+static DEVICE_ATTR_RO(gamepad_mode_index);
+
+static void oxp_set_defaults_bmap_1(struct oxp_bmap_page_1 *bmap)
+{
+	bmap->btn_a.button_idx = BUTTON_A;
+	bmap->btn_a.mapping_idx = 0;
+	bmap->btn_b.button_idx = BUTTON_B;
+	bmap->btn_b.mapping_idx = 1;
+	bmap->btn_x.button_idx = BUTTON_X;
+	bmap->btn_x.mapping_idx = 2;
+	bmap->btn_y.button_idx = BUTTON_Y;
+	bmap->btn_y.mapping_idx = 3;
+	bmap->btn_lb.button_idx = BUTTON_LB;
+	bmap->btn_lb.mapping_idx = 4;
+	bmap->btn_rb.button_idx = BUTTON_RB;
+	bmap->btn_rb.mapping_idx = 5;
+	bmap->btn_lt.button_idx = BUTTON_LT;
+	bmap->btn_lt.mapping_idx = 6;
+	bmap->btn_rt.button_idx = BUTTON_RT;
+	bmap->btn_rt.mapping_idx = 7;
+	bmap->btn_start.button_idx = BUTTON_START;
+	bmap->btn_start.mapping_idx = 8;
+}
+
+static void oxp_set_defaults_bmap_2(struct oxp_bmap_page_2 *bmap)
+{
+	bmap->btn_select.button_idx = BUTTON_SELECT;
+	bmap->btn_select.mapping_idx = 9;
+	bmap->btn_l3.button_idx = BUTTON_L3;
+	bmap->btn_l3.mapping_idx = 10;
+	bmap->btn_r3.button_idx = BUTTON_R3;
+	bmap->btn_r3.mapping_idx = 11;
+	bmap->btn_dup.button_idx = BUTTON_DUP;
+	bmap->btn_dup.mapping_idx = 12;
+	bmap->btn_ddown.button_idx = BUTTON_DDOWN;
+	bmap->btn_ddown.mapping_idx = 13;
+	bmap->btn_dleft.button_idx = BUTTON_DLEFT;
+	bmap->btn_dleft.mapping_idx = 14;
+	bmap->btn_dright.button_idx = BUTTON_DRIGHT;
+	bmap->btn_dright.mapping_idx = 15;
+	bmap->btn_m1.button_idx = BUTTON_M1;
+	bmap->btn_m1.mapping_idx = 48; /* KEY_F15 */
+	bmap->btn_m2.button_idx = BUTTON_M2;
+	bmap->btn_m2.mapping_idx = 49; /* KEY_F16 */
+}
+
+static void oxp_page_fill_data(char *buf, const struct oxp_button_idx *buttons,
+			       size_t len)
+{
+	size_t offset_increment = sizeof(u8) + sizeof(struct oxp_button_idx);
+	size_t offset = 5;
+	unsigned int i;
+
+	for (i = 0; i < len; i++, offset += offset_increment) {
+		buf[offset] = (u8)buttons[i].button_idx;
+		memcpy(buf + offset + 1,
+		       &oxp_button_table[buttons[i].mapping_idx].data,
+		       sizeof(struct oxp_button_data));
+	}
+}
+
+static int oxp_set_buttons(void)
+{
+	u8 page_1[59] = { 0x02, 0x38, 0x20, 0x01, 0x01 };
+	u8 page_2[59] = { 0x02, 0x38, 0x20, 0x02, 0x01 };
+	u16 up = get_usage_page(drvdata.hdev);
+	int ret;
+
+	if (up != GEN2_USAGE_PAGE)
+		return -EINVAL;
+
+	const struct oxp_button_idx p1[] = {
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_1, btn_a),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_1, btn_b),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_1, btn_x),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_1, btn_y),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_1, btn_lb),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_1, btn_rb),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_1, btn_lt),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_1, btn_rt),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_1, btn_start),
+	};
+
+	const struct oxp_button_idx p2[] = {
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_2, btn_select),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_2, btn_l3),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_2, btn_r3),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_2, btn_dup),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_2, btn_ddown),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_2, btn_dleft),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_2, btn_dright),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_2, btn_m1),
+		OXP_FILL_PAGE_SLOT(drvdata.bmap_2, btn_m2),
+	};
+
+	oxp_page_fill_data(page_1, p1, ARRAY_SIZE(p1));
+	oxp_page_fill_data(page_2, p2, ARRAY_SIZE(p2));
+
+	ret = oxp_gen_2_property_out(OXP_FID_GEN2_KEY_STATE, page_1, ARRAY_SIZE(page_1));
+	if (ret)
+		return ret;
+
+	return oxp_gen_2_property_out(OXP_FID_GEN2_KEY_STATE, page_2, ARRAY_SIZE(page_2));
+}
+
+static void oxp_reset_buttons(void)
+{
+	oxp_set_defaults_bmap_1(drvdata.bmap_1);
+	oxp_set_defaults_bmap_2(drvdata.bmap_2);
+}
+
+static ssize_t reset_buttons_store(struct device *dev,
+				   struct device_attribute *attr, const char *buf,
+				   size_t count)
+{
+	int val, ret;
+
+	ret = kstrtoint(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	if (val != 1)
+		return -EINVAL;
+
+	oxp_reset_buttons();
+	ret = oxp_set_buttons();
+	if (ret)
+		return ret;
+
+	return count;
+}
+static DEVICE_ATTR_WO(reset_buttons);
+
+static void oxp_btn_queue_fn(struct work_struct *work)
+{
+	int ret;
+
+	ret = oxp_set_buttons();
+	if (ret)
+		dev_err(&drvdata.hdev->dev,
+			"Error: Failed to write button mapping: %i\n", ret);
+}
+
+static int oxp_button_idx_from_str(const char *buf)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(oxp_button_table); i++)
+		if (sysfs_streq(buf, oxp_button_table[i].name))
+			return i;
+
+	return -EINVAL;
+}
+
+static ssize_t map_button_store(struct device *dev,
+				struct device_attribute *attr, const char *buf,
+				size_t count, u8 index)
+{
+	int idx;
+
+	idx = oxp_button_idx_from_str(buf);
+	if (idx < 0)
+		return idx;
+
+	switch (index) {
+	case BUTTON_A:
+		drvdata.bmap_1->btn_a.mapping_idx = idx;
+		break;
+	case BUTTON_B:
+		drvdata.bmap_1->btn_b.mapping_idx = idx;
+		break;
+	case BUTTON_X:
+		drvdata.bmap_1->btn_x.mapping_idx = idx;
+		break;
+	case BUTTON_Y:
+		drvdata.bmap_1->btn_y.mapping_idx = idx;
+		break;
+	case BUTTON_LB:
+		drvdata.bmap_1->btn_lb.mapping_idx = idx;
+		break;
+	case BUTTON_RB:
+		drvdata.bmap_1->btn_rb.mapping_idx = idx;
+		break;
+	case BUTTON_LT:
+		drvdata.bmap_1->btn_lt.mapping_idx = idx;
+		break;
+	case BUTTON_RT:
+		drvdata.bmap_1->btn_rt.mapping_idx = idx;
+		break;
+	case BUTTON_START:
+		drvdata.bmap_1->btn_start.mapping_idx = idx;
+		break;
+	case BUTTON_SELECT:
+		drvdata.bmap_2->btn_select.mapping_idx = idx;
+		break;
+	case BUTTON_L3:
+		drvdata.bmap_2->btn_l3.mapping_idx = idx;
+		break;
+	case BUTTON_R3:
+		drvdata.bmap_2->btn_r3.mapping_idx = idx;
+		break;
+	case BUTTON_DUP:
+		drvdata.bmap_2->btn_dup.mapping_idx = idx;
+		break;
+	case BUTTON_DDOWN:
+		drvdata.bmap_2->btn_ddown.mapping_idx = idx;
+		break;
+	case BUTTON_DLEFT:
+		drvdata.bmap_2->btn_dleft.mapping_idx = idx;
+		break;
+	case BUTTON_DRIGHT:
+		drvdata.bmap_2->btn_dright.mapping_idx = idx;
+		break;
+	case BUTTON_M1:
+		drvdata.bmap_2->btn_m1.mapping_idx = idx;
+		break;
+	case BUTTON_M2:
+		drvdata.bmap_2->btn_m2.mapping_idx = idx;
+		break;
+	default:
+		return -EINVAL;
+	}
+	mod_delayed_work(system_wq, &drvdata.oxp_btn_queue, msecs_to_jiffies(50));
+	return count;
+}
+
+static ssize_t map_button_show(struct device *dev,
+			       struct device_attribute *attr, char *buf,
+			       u8 index)
+{
+	u8 i;
+
+	switch (index) {
+	case BUTTON_A:
+		i = drvdata.bmap_1->btn_a.mapping_idx;
+		break;
+	case BUTTON_B:
+		i = drvdata.bmap_1->btn_b.mapping_idx;
+		break;
+	case BUTTON_X:
+		i = drvdata.bmap_1->btn_x.mapping_idx;
+		break;
+	case BUTTON_Y:
+		i = drvdata.bmap_1->btn_y.mapping_idx;
+		break;
+	case BUTTON_LB:
+		i = drvdata.bmap_1->btn_lb.mapping_idx;
+		break;
+	case BUTTON_RB:
+		i = drvdata.bmap_1->btn_rb.mapping_idx;
+		break;
+	case BUTTON_LT:
+		i = drvdata.bmap_1->btn_lt.mapping_idx;
+		break;
+	case BUTTON_RT:
+		i = drvdata.bmap_1->btn_rt.mapping_idx;
+		break;
+	case BUTTON_START:
+		i = drvdata.bmap_1->btn_start.mapping_idx;
+		break;
+	case BUTTON_SELECT:
+		i = drvdata.bmap_2->btn_select.mapping_idx;
+		break;
+	case BUTTON_L3:
+		i = drvdata.bmap_2->btn_l3.mapping_idx;
+		break;
+	case BUTTON_R3:
+		i = drvdata.bmap_2->btn_r3.mapping_idx;
+		break;
+	case BUTTON_DUP:
+		i = drvdata.bmap_2->btn_dup.mapping_idx;
+		break;
+	case BUTTON_DDOWN:
+		i = drvdata.bmap_2->btn_ddown.mapping_idx;
+		break;
+	case BUTTON_DLEFT:
+		i = drvdata.bmap_2->btn_dleft.mapping_idx;
+		break;
+	case BUTTON_DRIGHT:
+		i = drvdata.bmap_2->btn_dright.mapping_idx;
+		break;
+	case BUTTON_M1:
+		i = drvdata.bmap_2->btn_m1.mapping_idx;
+		break;
+	case BUTTON_M2:
+		i = drvdata.bmap_2->btn_m2.mapping_idx;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (i >= ARRAY_SIZE(oxp_button_table))
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%s\n", oxp_button_table[i].name);
+}
+
+static ssize_t button_mapping_options_show(struct device *dev,
+					   struct device_attribute *attr, char *buf)
+{
+	ssize_t count = 0;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(oxp_button_table); i++)
+		count += sysfs_emit_at(buf, count, "%s ", oxp_button_table[i].name);
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+static DEVICE_ATTR_RO(button_mapping_options);
+
+static int oxp_rumble_intensity_set(u8 intensity)
+{
+	u8 header[15] = { 0x02, 0x38, 0x02, 0xe3, 0x39, 0xe3, 0x39, 0xe3,
+			  0x39, 0x01, intensity, 0x05, 0xe3, 0x39, 0xe3 };
+	u8 footer[9] = { 0x39, 0xe3, 0x39, 0xe3, 0xe3, 0x02, 0x04, 0x39, 0x39 };
+	size_t footer_size = ARRAY_SIZE(footer);
+	size_t header_size = ARRAY_SIZE(header);
+	u8 data[59] = { 0x0 };
+	size_t data_size = ARRAY_SIZE(data);
+
+	memcpy(data, header, header_size);
+	memcpy(data + data_size - footer_size, footer, footer_size);
+
+	return oxp_gen_2_property_out(OXP_FID_GEN2_RUMBLE_SET, data, data_size);
+}
+
+static ssize_t rumble_intensity_store(struct device *dev,
+				      struct device_attribute *attr, const char *buf,
+				      size_t count)
+{
+	int ret;
+	u8 val;
+
+	ret = kstrtou8(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	if (val < 0 || val > 5)
+		return -EINVAL;
+
+	ret = oxp_rumble_intensity_set(val);
+	if (ret)
+		return ret;
+
+	drvdata.rumble_intensity = val;
+
+	return count;
+}
+
+static ssize_t rumble_intensity_show(struct device *dev,
+				     struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "%i\n", drvdata.rumble_intensity);
+}
+static DEVICE_ATTR_RW(rumble_intensity);
+
+static ssize_t rumble_intensity_range_show(struct device *dev,
+					   struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "0-5\n");
+}
+static DEVICE_ATTR_RO(rumble_intensity_range);
+
+#define OXP_DEVICE_ATTR_RW(_name, _group)                                     \
+	static ssize_t _name##_store(struct device *dev,                      \
+				     struct device_attribute *attr,           \
+				     const char *buf, size_t count)           \
+	{                                                                     \
+		return _group##_store(dev, attr, buf, count, _name.index);    \
+	}                                                                     \
+	static ssize_t _name##_show(struct device *dev,                       \
+				    struct device_attribute *attr, char *buf) \
+	{                                                                     \
+		return _group##_show(dev, attr, buf, _name.index);            \
+	}                                                                     \
+	static DEVICE_ATTR_RW(_name)
+
+static struct oxp_attr button_a = { BUTTON_A };
+OXP_DEVICE_ATTR_RW(button_a, map_button);
+
+static struct oxp_attr button_b = { BUTTON_B };
+OXP_DEVICE_ATTR_RW(button_b, map_button);
+
+static struct oxp_attr button_x = { BUTTON_X };
+OXP_DEVICE_ATTR_RW(button_x, map_button);
+
+static struct oxp_attr button_y = { BUTTON_Y };
+OXP_DEVICE_ATTR_RW(button_y, map_button);
+
+static struct oxp_attr button_lb = { BUTTON_LB };
+OXP_DEVICE_ATTR_RW(button_lb, map_button);
+
+static struct oxp_attr button_rb = { BUTTON_RB };
+OXP_DEVICE_ATTR_RW(button_rb, map_button);
+
+static struct oxp_attr button_lt = { BUTTON_LT };
+OXP_DEVICE_ATTR_RW(button_lt, map_button);
+
+static struct oxp_attr button_rt = { BUTTON_RT };
+OXP_DEVICE_ATTR_RW(button_rt, map_button);
+
+static struct oxp_attr button_start = { BUTTON_START };
+OXP_DEVICE_ATTR_RW(button_start, map_button);
+
+static struct oxp_attr button_select = { BUTTON_SELECT };
+OXP_DEVICE_ATTR_RW(button_select, map_button);
+
+static struct oxp_attr button_l3 = { BUTTON_L3 };
+OXP_DEVICE_ATTR_RW(button_l3, map_button);
+
+static struct oxp_attr button_r3 = { BUTTON_R3 };
+OXP_DEVICE_ATTR_RW(button_r3, map_button);
+
+static struct oxp_attr button_d_up = { BUTTON_DUP };
+OXP_DEVICE_ATTR_RW(button_d_up, map_button);
+
+static struct oxp_attr button_d_down = { BUTTON_DDOWN };
+OXP_DEVICE_ATTR_RW(button_d_down, map_button);
+
+static struct oxp_attr button_d_left = { BUTTON_DLEFT };
+OXP_DEVICE_ATTR_RW(button_d_left, map_button);
+
+static struct oxp_attr button_d_right = { BUTTON_DRIGHT };
+OXP_DEVICE_ATTR_RW(button_d_right, map_button);
+
+static struct oxp_attr button_m1 = { BUTTON_M1 };
+OXP_DEVICE_ATTR_RW(button_m1, map_button);
+
+static struct oxp_attr button_m2 = { BUTTON_M2 };
+OXP_DEVICE_ATTR_RW(button_m2, map_button);
+
+static struct attribute *oxp_cfg_attrs[] = {
+	&dev_attr_button_a.attr,
+	&dev_attr_button_b.attr,
+	&dev_attr_button_d_down.attr,
+	&dev_attr_button_d_left.attr,
+	&dev_attr_button_d_right.attr,
+	&dev_attr_button_d_up.attr,
+	&dev_attr_button_l3.attr,
+	&dev_attr_button_lb.attr,
+	&dev_attr_button_lt.attr,
+	&dev_attr_button_m1.attr,
+	&dev_attr_button_m2.attr,
+	&dev_attr_button_mapping_options.attr,
+	&dev_attr_button_r3.attr,
+	&dev_attr_button_rb.attr,
+	&dev_attr_button_rt.attr,
+	&dev_attr_button_select.attr,
+	&dev_attr_button_start.attr,
+	&dev_attr_button_x.attr,
+	&dev_attr_button_y.attr,
+	&dev_attr_gamepad_mode.attr,
+	&dev_attr_gamepad_mode_index.attr,
+	&dev_attr_reset_buttons.attr,
+	&dev_attr_rumble_intensity.attr,
+	&dev_attr_rumble_intensity_range.attr,
+	NULL,
+};
+
+static const struct attribute_group oxp_cfg_attrs_group = {
+	.attrs = oxp_cfg_attrs,
+};
+
+static int oxp_rgb_status_store(u8 enabled, u8 speed, u8 brightness)
+{
+	u16 up = get_usage_page(drvdata.hdev);
+	u8 *data;
+
+	/* Always default to max brightness and use intensity scaling when in
+	 * monocolor mode.
+	 */
+	switch (up) {
+	case GEN1_USAGE_PAGE:
+		data = (u8[4]) { OXP_SET_PROPERTY, enabled, speed, brightness };
+		if (drvdata.rgb_effect == OXP_EFFECT_MONO_LIST)
+			data[3] = 0x04;
+		return oxp_gen_1_property_out(OXP_FID_GEN1_RGB_SET, data, 4);
+	case GEN2_USAGE_PAGE:
+		data = (u8[6]) { OXP_SET_PROPERTY, 0x00, 0x02, enabled, speed, brightness };
+		if (drvdata.rgb_effect == OXP_EFFECT_MONO_LIST)
+			data[5] = 0x04;
+		return oxp_gen_2_property_out(OXP_FID_GEN2_STATUS_EVENT, data, 6);
+	default:
+		return -ENODEV;
+	}
+}
+
+static ssize_t oxp_rgb_status_show(void)
+{
+	u16 up = get_usage_page(drvdata.hdev);
+	u8 *data;
+
+	switch (up) {
+	case GEN1_USAGE_PAGE:
+		data = (u8[1]) { OXP_GET_PROPERTY };
+		return oxp_gen_1_property_out(OXP_FID_GEN1_RGB_SET, data, 1);
+	case GEN2_USAGE_PAGE:
+		data = (u8[3]) { OXP_GET_PROPERTY, 0x00, 0x02 };
+		return oxp_gen_2_property_out(OXP_FID_GEN2_STATUS_EVENT, data, 3);
+	default:
+		return -ENODEV;
+	}
+}
+
+static int oxp_rgb_color_set(void)
+{
+	u8 max_br = drvdata.led_mc->led_cdev.max_brightness;
+	u8 br = drvdata.led_mc->led_cdev.brightness;
+	u16 up = get_usage_page(drvdata.hdev);
+	u8 green, red, blue;
+	size_t size;
+	u8 *data;
+	int i;
+
+	red = br * drvdata.led_mc->subled_info[0].intensity / max_br;
+	green = br * drvdata.led_mc->subled_info[1].intensity / max_br;
+	blue = br * drvdata.led_mc->subled_info[2].intensity / max_br;
+
+	switch (up) {
+	case GEN1_USAGE_PAGE:
+		size = 55;
+		data = (u8[55]) { OXP_EFFECT_MONO_TRUE };
+
+		for (i = 0; i < (size - 1) / 3; i++) {
+			data[3 * i + 1] = red;
+			data[3 * i + 2] = green;
+			data[3 * i + 3] = blue;
+		}
+		return oxp_gen_1_property_out(OXP_FID_GEN1_RGB_SET, data, size);
+	case GEN2_USAGE_PAGE:
+		size = 57;
+		data = (u8[57]) { OXP_EFFECT_MONO_TRUE, 0x00, 0x02 };
+
+		for (i = 1; i < size / 3; i++) {
+			data[3 * i] = red;
+			data[3 * i + 1] = green;
+			data[3 * i + 2] = blue;
+		}
+		return oxp_gen_2_property_out(OXP_FID_GEN2_STATUS_EVENT, data, size);
+	default:
+		return -ENODEV;
+	}
+}
+
+static int oxp_rgb_effect_set(u8 effect)
+{
+	u16 up = get_usage_page(drvdata.hdev);
+	u8 *data;
+	int ret;
+
+	switch (effect) {
+	case OXP_EFFECT_AURORA:
+	case OXP_EFFECT_BIRTHDAY:
+	case OXP_EFFECT_FLOWING:
+	case OXP_EFFECT_CHROMA_1:
+	case OXP_EFFECT_NEON:
+	case OXP_EFFECT_CHROMA_2:
+	case OXP_EFFECT_DREAMY:
+	case OXP_EFFECT_WARM:
+	case OXP_EFFECT_CYBERPUNK:
+	case OXP_EFFECT_SEA:
+	case OXP_EFFECT_SUNSET:
+	case OXP_EFFECT_COLORFUL:
+	case OXP_EFFECT_MONSTER:
+	case OXP_EFFECT_GREEN:
+	case OXP_EFFECT_BLUE:
+	case OXP_EFFECT_YELLOW:
+	case OXP_EFFECT_TEAL:
+	case OXP_EFFECT_PURPLE:
+	case OXP_EFFECT_FOGGY:
+		switch (up) {
+		case GEN1_USAGE_PAGE:
+			data = (u8[1]) { effect };
+			ret = oxp_gen_1_property_out(OXP_FID_GEN1_RGB_SET, data, 1);
+			break;
+		case GEN2_USAGE_PAGE:
+			data = (u8[3]) { effect, 0x00, 0x02 };
+			ret = oxp_gen_2_property_out(OXP_FID_GEN2_STATUS_EVENT, data, 3);
+			break;
+		default:
+			ret = -ENODEV;
+		}
+		break;
+	case OXP_EFFECT_MONO_LIST:
+		ret = oxp_rgb_color_set();
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (ret)
+		return ret;
+
+	drvdata.rgb_effect = effect;
+
+	return 0;
+}
+
+static ssize_t enabled_store(struct device *dev, struct device_attribute *attr,
+			     const char *buf, size_t count)
+{
+	int ret;
+	u8 val;
+
+	ret = sysfs_match_string(oxp_feature_en_text, buf);
+	if (ret < 0)
+		return ret;
+	val = ret;
+
+	ret = oxp_rgb_status_store(val, drvdata.rgb_speed,
+				   drvdata.rgb_brightness);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_en = val;
+	return count;
+}
+
+static ssize_t enabled_show(struct device *dev, struct device_attribute *attr,
+			    char *buf)
+{
+	int ret;
+
+	ret = oxp_rgb_status_show();
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_en >= ARRAY_SIZE(oxp_feature_en_text))
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%s\n", oxp_feature_en_text[drvdata.rgb_en]);
+}
+static DEVICE_ATTR_RW(enabled);
+
+static ssize_t enabled_index_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	size_t count = 0;
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(oxp_feature_en_text); i++)
+		count += sysfs_emit_at(buf, count, "%s ", oxp_feature_en_text[i]);
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+static DEVICE_ATTR_RO(enabled_index);
+
+static ssize_t effect_store(struct device *dev, struct device_attribute *attr,
+			    const char *buf, size_t count)
+{
+	int ret;
+	u8 val;
+
+	ret = sysfs_match_string(oxp_rgb_effect_text, buf);
+	if (ret < 0)
+		return ret;
+
+	val = ret;
+
+	ret = oxp_rgb_status_store(drvdata.rgb_en, drvdata.rgb_speed,
+				   drvdata.rgb_brightness);
+	if (ret)
+		return ret;
+
+	ret = oxp_rgb_effect_set(val);
+	if (ret)
+		return ret;
+
+	return count;
+}
+
+static ssize_t effect_show(struct device *dev, struct device_attribute *attr,
+			   char *buf)
+{
+	int ret;
+
+	ret = oxp_rgb_status_show();
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_effect >= ARRAY_SIZE(oxp_rgb_effect_text))
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%s\n", oxp_rgb_effect_text[drvdata.rgb_effect]);
+}
+
+static DEVICE_ATTR_RW(effect);
+
+static ssize_t effect_index_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	size_t count = 0;
+	unsigned int i;
+
+	for (i = 1; i < ARRAY_SIZE(oxp_rgb_effect_text); i++)
+		count += sysfs_emit_at(buf, count, "%s ", oxp_rgb_effect_text[i]);
+
+	if (count)
+		buf[count - 1] = '\n';
+
+	return count;
+}
+static DEVICE_ATTR_RO(effect_index);
+
+static ssize_t speed_store(struct device *dev, struct device_attribute *attr,
+			   const char *buf, size_t count)
+{
+	int ret;
+	u8 val;
+
+	ret = kstrtou8(buf, 10, &val);
+	if (ret)
+		return ret;
+
+	if (val > 9)
+		return -EINVAL;
+
+	ret = oxp_rgb_status_store(drvdata.rgb_en, val, drvdata.rgb_brightness);
+	if (ret)
+		return ret;
+
+	drvdata.rgb_speed = val;
+	return count;
+}
+
+static ssize_t speed_show(struct device *dev, struct device_attribute *attr,
+			  char *buf)
+{
+	int ret;
+
+	ret = oxp_rgb_status_show();
+	if (ret)
+		return ret;
+
+	if (drvdata.rgb_speed > 9)
+		return -EINVAL;
+
+	return sysfs_emit(buf, "%hhu\n", drvdata.rgb_speed);
+}
+static DEVICE_ATTR_RW(speed);
+
+static ssize_t speed_range_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	return sysfs_emit(buf, "0-9\n");
+}
+static DEVICE_ATTR_RO(speed_range);
+
+static void oxp_rgb_queue_fn(struct work_struct *work)
+{
+	unsigned int max_brightness = drvdata.led_mc->led_cdev.max_brightness;
+	unsigned int brightness = drvdata.led_mc->led_cdev.brightness;
+	u8 val = 4 * brightness / max_brightness;
+	int ret;
+
+	if (drvdata.rgb_brightness != val) {
+		ret = oxp_rgb_status_store(drvdata.rgb_en, drvdata.rgb_speed, val);
+		if (ret)
+			dev_err(drvdata.led_mc->led_cdev.dev,
+				"Error: Failed to write RGB Status: %i\n", ret);
+
+		drvdata.rgb_brightness = val;
+	}
+
+	if (drvdata.rgb_effect != OXP_EFFECT_MONO_LIST)
+		return;
+
+	ret = oxp_rgb_effect_set(drvdata.rgb_effect);
+	if (ret)
+		dev_err(drvdata.led_mc->led_cdev.dev, "Error: Failed to write RGB color: %i\n",
+			ret);
+}
+
+static void oxp_rgb_brightness_set(struct led_classdev *led_cdev,
+				   enum led_brightness brightness)
+{
+	led_cdev->brightness = brightness;
+	mod_delayed_work(system_wq, &drvdata.oxp_rgb_queue, msecs_to_jiffies(50));
+}
+
+static struct attribute *oxp_rgb_attrs[] = {
+	&dev_attr_effect.attr,
+	&dev_attr_effect_index.attr,
+	&dev_attr_enabled.attr,
+	&dev_attr_enabled_index.attr,
+	&dev_attr_speed.attr,
+	&dev_attr_speed_range.attr,
+	NULL,
+};
+
+static const struct attribute_group oxp_rgb_attr_group = {
+	.attrs = oxp_rgb_attrs,
+};
+
+static struct mc_subled oxp_rgb_subled_info[] = {
+	{
+		.color_index = LED_COLOR_ID_RED,
+		.intensity = 0x24,
+		.channel = 0x1,
+	},
+	{
+		.color_index = LED_COLOR_ID_GREEN,
+		.intensity = 0x22,
+		.channel = 0x2,
+	},
+	{
+		.color_index = LED_COLOR_ID_BLUE,
+		.intensity = 0x99,
+		.channel = 0x3,
+	},
+};
+
+static struct led_classdev_mc oxp_cdev_rgb = {
+	.led_cdev = {
+		.name = "oxp:rgb:joystick_rings",
+		.color = LED_COLOR_ID_RGB,
+		.brightness = 0x64,
+		.max_brightness = 0x64,
+		.brightness_set = oxp_rgb_brightness_set,
+	},
+	.num_colors = ARRAY_SIZE(oxp_rgb_subled_info),
+	.subled_info = oxp_rgb_subled_info,
+};
+
+struct quirk_entry {
+	bool hybrid_mcu;
+};
+
+static struct quirk_entry quirk_hybrid_mcu = {
+	.hybrid_mcu = true,
+};
+
+static const struct dmi_system_id oxp_hybrid_mcu_list[] = {
+	{
+		.ident = "OneXPlayer Apex",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ONE-NETBOOK"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ONEXPLAYER APEX"),
+		},
+		.driver_data = &quirk_hybrid_mcu,
+	},
+	{
+		.ident = "OneXPlayer G1 AMD",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ONE-NETBOOK"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ONEXPLAYER G1 A"),
+		},
+		.driver_data = &quirk_hybrid_mcu,
+	},
+	{
+		.ident = "OneXPlayer G1 Intel",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "ONE-NETBOOK"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "ONEXPLAYER G1 i"),
+		},
+		.driver_data = &quirk_hybrid_mcu,
+	},
+	{},
+};
+
+static bool oxp_hybrid_mcu_device(void)
+{
+	const struct dmi_system_id *dmi_id;
+	struct quirk_entry *quirks;
+
+	dmi_id = dmi_first_match(oxp_hybrid_mcu_list);
+	if (!dmi_id)
+		return false;
+
+	quirks = dmi_id->driver_data;
+
+	return quirks->hybrid_mcu;
+}
+
+static int oxp_cfg_probe(struct hid_device *hdev, u16 up)
+{
+	struct oxp_bmap_page_1 *bmap_1;
+	struct oxp_bmap_page_2 *bmap_2;
+	int ret;
+
+	hid_set_drvdata(hdev, &drvdata);
+	mutex_init(&drvdata.cfg_mutex);
+	drvdata.hdev = hdev;
+
+	if (up == GEN2_USAGE_PAGE && oxp_hybrid_mcu_device())
+		goto skip_rgb;
+
+	drvdata.led_mc = &oxp_cdev_rgb;
+
+	INIT_DELAYED_WORK(&drvdata.oxp_rgb_queue, oxp_rgb_queue_fn);
+	ret = devm_led_classdev_multicolor_register(&hdev->dev, &oxp_cdev_rgb);
+	if (ret)
+		return dev_err_probe(&hdev->dev, ret,
+				     "Failed to create RGB device\n");
+
+	ret = devm_device_add_group(drvdata.led_mc->led_cdev.dev,
+				    &oxp_rgb_attr_group);
+	if (ret)
+		return dev_err_probe(drvdata.led_mc->led_cdev.dev, ret,
+				     "Failed to create RGB configuration attributes\n");
+
+	ret = oxp_rgb_status_show();
+	if (ret)
+		dev_warn(drvdata.led_mc->led_cdev.dev,
+			 "Failed to query RGB initial state: %i\n", ret);
+
+	/* Below features are only implemented in gen 2 */
+	if (up != GEN2_USAGE_PAGE)
+		return 0;
+
+skip_rgb:
+	bmap_1 = devm_kzalloc(&hdev->dev, sizeof(struct oxp_bmap_page_1), GFP_KERNEL);
+	if (!bmap_1)
+		return dev_err_probe(&hdev->dev, -ENOMEM,
+				     "Unable to allocate button map page 1\n");
+
+	bmap_2 = devm_kzalloc(&hdev->dev, sizeof(struct oxp_bmap_page_2), GFP_KERNEL);
+	if (!bmap_2)
+		return dev_err_probe(&hdev->dev, -ENOMEM,
+				     "Unable to allocate button map page 2\n");
+
+	drvdata.bmap_1 = bmap_1;
+	drvdata.bmap_2 = bmap_2;
+	oxp_reset_buttons();
+	INIT_DELAYED_WORK(&drvdata.oxp_btn_queue, oxp_btn_queue_fn);
+
+	drvdata.gamepad_mode = OXP_GP_MODE_XINPUT;
+	drvdata.rumble_intensity = 5;
+
+	INIT_DELAYED_WORK(&drvdata.oxp_mcu_init, oxp_mcu_init_fn);
+	mod_delayed_work(system_wq, &drvdata.oxp_mcu_init, msecs_to_jiffies(50));
+
+	ret = devm_device_add_group(&hdev->dev, &oxp_cfg_attrs_group);
+	if (ret)
+		return dev_err_probe(&hdev->dev, ret,
+				     "Failed to attach configuration attributes\n");
+
+	return 0;
+}
+
+static int oxp_hid_probe(struct hid_device *hdev,
+			 const struct hid_device_id *id)
+{
+	int ret;
+	u16 up;
+
+	ret = hid_parse(hdev);
+	if (ret)
+		return dev_err_probe(&hdev->dev, ret, "Failed to parse HID device\n");
+
+	ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT);
+	if (ret)
+		return dev_err_probe(&hdev->dev, ret, "Failed to start HID device\n");
+
+	ret = hid_hw_open(hdev);
+	if (ret) {
+		hid_hw_stop(hdev);
+		return dev_err_probe(&hdev->dev, ret, "Failed to open HID device\n");
+	}
+
+	up = get_usage_page(hdev);
+	dev_dbg(&hdev->dev, "Got usage page %04x\n", up);
+
+	switch (up) {
+	case GEN1_USAGE_PAGE:
+	case GEN2_USAGE_PAGE:
+		ret = oxp_cfg_probe(hdev, up);
+		if (ret) {
+			hid_hw_close(hdev);
+			hid_hw_stop(hdev);
+		}
+
+		return ret;
+	default:
+		return 0;
+	}
+}
+
+static void oxp_hid_remove(struct hid_device *hdev)
+{
+	cancel_delayed_work(&drvdata.oxp_rgb_queue);
+	cancel_delayed_work(&drvdata.oxp_btn_queue);
+	cancel_delayed_work(&drvdata.oxp_mcu_init);
+	hid_hw_close(hdev);
+	hid_hw_stop(hdev);
+}
+
+static const struct hid_device_id oxp_devices[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_CRSC, USB_DEVICE_ID_ONEXPLAYER_GEN1) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_WCH, USB_DEVICE_ID_ONEXPLAYER_GEN2) },
+	{}
+};
+
+MODULE_DEVICE_TABLE(hid, oxp_devices);
+static struct hid_driver hid_oxp = {
+	.name = "hid-oxp",
+	.id_table = oxp_devices,
+	.probe = oxp_hid_probe,
+	.remove = oxp_hid_remove,
+	.raw_event = oxp_hid_raw_event,
+};
+module_hid_driver(hid_oxp);
+
+MODULE_AUTHOR("Derek J. Clark <derekjohn.clark@gmail.com>");
+MODULE_DESCRIPTION("Driver for OneXPlayer HID Interfaces");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hid/hid-playstation.c b/drivers/hid/hid-playstation.c
index e485373316756..f9dc9378cf777 100644
--- a/drivers/hid/hid-playstation.c
+++ b/drivers/hid/hid-playstation.c
@@ -112,6 +112,12 @@ struct ps_led_info {
 #define DS_BUTTONS2_TOUCHPAD	BIT(1)
 #define DS_BUTTONS2_MIC_MUTE	BIT(2)
 
+/* DualSense Edge extra buttons in buttons[2], bits 4-7. */
+#define DS_EDGE_BUTTONS_FN1		BIT(4)
+#define DS_EDGE_BUTTONS_FN2		BIT(5)
+#define DS_EDGE_BUTTONS_LEFT_PADDLE	BIT(6)
+#define DS_EDGE_BUTTONS_RIGHT_PADDLE	BIT(7)
+
 /* Status fields of DualSense input report. */
 #define DS_STATUS0_BATTERY_CAPACITY		GENMASK(3, 0)
 #define DS_STATUS0_CHARGING			GENMASK(7, 4)
@@ -178,6 +184,9 @@ struct dualsense {
 	struct input_dev *touchpad;
 	struct input_dev *jack;
 
+	/* True if this is a DualSense Edge (product 0x0df2). */
+	bool is_edge;
+
 	/* Update version is used as a feature/capability version. */
 	u16 update_version;
 
@@ -1486,6 +1495,18 @@ static int dualsense_parse_report(struct ps_device *ps_dev, struct hid_report *r
 	input_report_key(ds->gamepad, BTN_THUMBL, ds_report->buttons[1] & DS_BUTTONS1_L3);
 	input_report_key(ds->gamepad, BTN_THUMBR, ds_report->buttons[1] & DS_BUTTONS1_R3);
 	input_report_key(ds->gamepad, BTN_MODE,   ds_report->buttons[2] & DS_BUTTONS2_PS_HOME);
+
+	if (ds->is_edge) {
+		input_report_key(ds->gamepad, BTN_TRIGGER_HAPPY1,
+				 ds_report->buttons[2] & DS_EDGE_BUTTONS_FN1);
+		input_report_key(ds->gamepad, BTN_TRIGGER_HAPPY2,
+				 ds_report->buttons[2] & DS_EDGE_BUTTONS_FN2);
+		input_report_key(ds->gamepad, BTN_TRIGGER_HAPPY3,
+				 ds_report->buttons[2] & DS_EDGE_BUTTONS_LEFT_PADDLE);
+		input_report_key(ds->gamepad, BTN_TRIGGER_HAPPY4,
+				 ds_report->buttons[2] & DS_EDGE_BUTTONS_RIGHT_PADDLE);
+	}
+
 	input_sync(ds->gamepad);
 
 	/*
@@ -1785,6 +1806,7 @@ static struct ps_device *dualsense_create(struct hid_device *hdev)
 		ds->use_vibration_v2 = ds->update_version >= DS_FEATURE_VERSION(2, 21);
 	} else if (hdev->product == USB_DEVICE_ID_SONY_PS5_CONTROLLER_2) {
 		ds->use_vibration_v2 = true;
+		ds->is_edge = true;
 	}
 
 	ret = ps_devices_list_add(ps_dev);
@@ -1802,6 +1824,15 @@ static struct ps_device *dualsense_create(struct hid_device *hdev)
 		ret = PTR_ERR(ds->gamepad);
 		goto err;
 	}
+
+	/* Register DualSense Edge back paddle and Fn buttons. */
+	if (ds->is_edge) {
+		input_set_capability(ds->gamepad, EV_KEY, BTN_TRIGGER_HAPPY1);
+		input_set_capability(ds->gamepad, EV_KEY, BTN_TRIGGER_HAPPY2);
+		input_set_capability(ds->gamepad, EV_KEY, BTN_TRIGGER_HAPPY3);
+		input_set_capability(ds->gamepad, EV_KEY, BTN_TRIGGER_HAPPY4);
+	}
+
 	/* Use gamepad input device name as primary device name for e.g. LEDs */
 	ps_dev->input_dev_name = dev_name(&ds->gamepad->dev);
 
diff --git a/drivers/hid/hid-rakk.c b/drivers/hid/hid-rakk.c
new file mode 100644
index 0000000000000..1140058a76ca4
--- /dev/null
+++ b/drivers/hid/hid-rakk.c
@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  HID driver for Rakk devices
+ *
+ *  Copyright (c) 2026 Karl Cayme
+ *
+ *  The Rakk Dasig X gaming mouse has a faulty HID report descriptor that
+ *  declares USAGE_MAXIMUM = 3 (buttons 1-3) while actually sending 5 button
+ *  bits (REPORT_COUNT = 5). This causes the kernel to ignore side buttons
+ *  (buttons 4 and 5). This driver fixes the descriptor so all 5 buttons
+ *  are properly recognized across 3 modes (wired, dongle, and Bluetooth).
+ */
+
+#include <linux/device.h>
+#include <linux/hid.h>
+#include <linux/module.h>
+#include "hid-ids.h"
+
+/*
+ * The faulty byte is at offset 17 in the report descriptor for all three
+ * connection modes (USB direct, wireless dongle, and Bluetooth).
+ *
+ * Bytes 16-17 are: 0x29 0x03 (USAGE_MAXIMUM = 3)
+ * The fix changes byte 17 to 0x05 (USAGE_MAXIMUM = 5).
+ *
+ * Original descriptor bytes 0-17:
+ *   05 01 09 02 a1 01 85 01 09 01 a1 00 05 09 19 01 29 03
+ *                                                       ^^
+ *   Should be 0x05 to declare 5 buttons instead of 3.
+ */
+#define RAKK_RDESC_USAGE_MAX_OFFSET	17
+#define RAKK_RDESC_USAGE_MAX_ORIG	0x03
+#define RAKK_RDESC_USAGE_MAX_FIXED	0x05
+#define RAKK_RDESC_USB_SIZE		193
+#define RAKK_RDESC_DONGLE_SIZE		150
+#define RAKK_RDESC_BT_SIZE		89
+
+static const __u8 *rakk_report_fixup(struct hid_device *hdev, __u8 *rdesc,
+				     unsigned int *rsize)
+{
+	if (((*rsize == RAKK_RDESC_USB_SIZE &&
+	      hdev->product == USB_DEVICE_ID_TELINK_RAKK_DASIG_X) ||
+	     (*rsize == RAKK_RDESC_DONGLE_SIZE &&
+	      hdev->product == USB_DEVICE_ID_TELINK_RAKK_DASIG_X_DONGLE) ||
+	     (*rsize == RAKK_RDESC_BT_SIZE &&
+	      hdev->product == USB_DEVICE_ID_TELINK_RAKK_DASIG_X_BT)) &&
+	    rdesc[RAKK_RDESC_USAGE_MAX_OFFSET] == RAKK_RDESC_USAGE_MAX_ORIG) {
+		hid_info(hdev, "fixing Rakk Dasig X button count (3 -> 5)\n");
+		rdesc[RAKK_RDESC_USAGE_MAX_OFFSET] = RAKK_RDESC_USAGE_MAX_FIXED;
+	}
+
+	return rdesc;
+}
+
+static const struct hid_device_id rakk_devices[] = {
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TELINK,
+			 USB_DEVICE_ID_TELINK_RAKK_DASIG_X) },
+	{ HID_USB_DEVICE(USB_VENDOR_ID_TELINK,
+			 USB_DEVICE_ID_TELINK_RAKK_DASIG_X_DONGLE) },
+	{ HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_TELINK,
+			       USB_DEVICE_ID_TELINK_RAKK_DASIG_X_BT) },
+	{ }
+};
+MODULE_DEVICE_TABLE(hid, rakk_devices);
+
+static struct hid_driver rakk_driver = {
+	.name = "rakk",
+	.id_table = rakk_devices,
+	.report_fixup = rakk_report_fixup,
+};
+module_hid_driver(rakk_driver);
+
+MODULE_DESCRIPTION("HID driver for Rakk Dasig X mouse - fix side button support");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Karl Cayme");
diff --git a/drivers/hid/hid-sony.c b/drivers/hid/hid-sony.c
index 315343415e8f1..e75246d29e16d 100644
--- a/drivers/hid/hid-sony.c
+++ b/drivers/hid/hid-sony.c
@@ -1852,18 +1852,8 @@ static int sony_play_effect(struct input_dev *dev, void *data,
 
 static int sony_init_ff(struct sony_sc *sc)
 {
-	struct hid_input *hidinput;
-	struct input_dev *input_dev;
-
-	if (list_empty(&sc->hdev->inputs)) {
-		hid_err(sc->hdev, "no inputs found\n");
-		return -ENODEV;
-	}
-	hidinput = list_entry(sc->hdev->inputs.next, struct hid_input, list);
-	input_dev = hidinput->input;
-
-	input_set_capability(input_dev, EV_FF, FF_RUMBLE);
-	return input_ff_create_memless(input_dev, NULL, sony_play_effect);
+	input_set_capability(sc->input_dev, EV_FF, FF_RUMBLE);
+	return input_ff_create_memless(sc->input_dev, NULL, sony_play_effect);
 }
 
 #else
@@ -2150,6 +2140,8 @@ static int sony_input_configured(struct hid_device *hdev,
 	int append_dev_id;
 	int ret;
 
+	sc->input_dev = hidinput->input;
+
 	ret = sony_set_device_id(sc);
 	if (ret < 0) {
 		hid_err(hdev, "failed to allocate the device id\n");
@@ -2310,7 +2302,6 @@ static int sony_input_configured(struct hid_device *hdev,
 			goto err_close;
 	}
 
-	sc->input_dev = hidinput->input;
 	return 0;
 err_close:
 	hid_hw_close(hdev);
diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c
index 5af93b9b1fb56..96b0181cf8194 100644
--- a/drivers/hid/usbhid/hid-core.c
+++ b/drivers/hid/usbhid/hid-core.c
@@ -27,6 +27,7 @@
 #include <linux/wait.h>
 #include <linux/workqueue.h>
 #include <linux/string.h>
+#include <linux/seq_buf.h>
 
 #include <linux/usb.h>
 
@@ -1367,6 +1368,7 @@ static int usbhid_probe(struct usb_interface *intf, const struct usb_device_id *
 	struct usb_endpoint_descriptor *ep;
 	struct usbhid_device *usbhid;
 	struct hid_device *hid;
+	struct seq_buf hid_name;
 	size_t len;
 	int ret;
 
@@ -1398,7 +1400,7 @@ static int usbhid_probe(struct usb_interface *intf, const struct usb_device_id *
 	hid->vendor = le16_to_cpu(dev->descriptor.idVendor);
 	hid->product = le16_to_cpu(dev->descriptor.idProduct);
 	hid->version = le16_to_cpu(dev->descriptor.bcdDevice);
-	hid->name[0] = 0;
+	seq_buf_init(&hid_name, hid->name, sizeof(hid->name));
 	if (intf->cur_altsetting->desc.bInterfaceProtocol ==
 			USB_INTERFACE_PROTOCOL_MOUSE)
 		hid->type = HID_TYPE_USBMOUSE;
@@ -1406,22 +1408,23 @@ static int usbhid_probe(struct usb_interface *intf, const struct usb_device_id *
 		hid->type = HID_TYPE_USBNONE;
 
 	if (dev->manufacturer)
-		strscpy(hid->name, dev->manufacturer, sizeof(hid->name));
+		seq_buf_puts(&hid_name, dev->manufacturer);
 
 	if (dev->product) {
 		if (dev->manufacturer)
-			strlcat(hid->name, " ", sizeof(hid->name));
-		strlcat(hid->name, dev->product, sizeof(hid->name));
+			seq_buf_puts(&hid_name, " ");
+		seq_buf_puts(&hid_name, dev->product);
 	}
 
-	if (!strlen(hid->name))
+	if (!seq_buf_used(&hid_name))
 		snprintf(hid->name, sizeof(hid->name), "HID %04x:%04x",
 			 le16_to_cpu(dev->descriptor.idVendor),
 			 le16_to_cpu(dev->descriptor.idProduct));
 
 	usb_make_path(dev, hid->phys, sizeof(hid->phys));
-	strlcat(hid->phys, "/input", sizeof(hid->phys));
-	len = strlen(hid->phys);
+	len = strnlen(hid->phys, sizeof(hid->phys));
+	strscpy(hid->phys + len, "/input", sizeof(hid->phys) - len);
+	len = strnlen(hid->phys, sizeof(hid->phys));
 	if (len < sizeof(hid->phys) - 1)
 		snprintf(hid->phys + len, sizeof(hid->phys) - len,
 			 "%d", intf->altsetting[0].desc.bInterfaceNumber);
diff --git a/drivers/hid/usbhid/usbkbd.c b/drivers/hid/usbhid/usbkbd.c
index 6b33e6ad0846e..83d4df0d7a457 100644
--- a/drivers/hid/usbhid/usbkbd.c
+++ b/drivers/hid/usbhid/usbkbd.c
@@ -20,6 +20,7 @@
 #include <linux/init.h>
 #include <linux/usb/input.h>
 #include <linux/hid.h>
+#include <linux/seq_buf.h>
 
 /*
  * Version Information
@@ -266,8 +267,10 @@ static int usb_kbd_probe(struct usb_interface *iface,
 	struct usb_endpoint_descriptor *endpoint;
 	struct usb_kbd *kbd;
 	struct input_dev *input_dev;
+	struct seq_buf kbd_name;
 	int i, pipe, maxp;
 	int error = -ENOMEM;
+	size_t len;
 
 	interface = iface->cur_altsetting;
 
@@ -292,24 +295,26 @@ static int usb_kbd_probe(struct usb_interface *iface,
 	kbd->usbdev = dev;
 	kbd->dev = input_dev;
 	spin_lock_init(&kbd->leds_lock);
+	seq_buf_init(&kbd_name, kbd->name, sizeof(kbd->name));
 
 	if (dev->manufacturer)
-		strscpy(kbd->name, dev->manufacturer, sizeof(kbd->name));
+		seq_buf_puts(&kbd_name, dev->manufacturer);
 
 	if (dev->product) {
 		if (dev->manufacturer)
-			strlcat(kbd->name, " ", sizeof(kbd->name));
-		strlcat(kbd->name, dev->product, sizeof(kbd->name));
+			seq_buf_puts(&kbd_name, " ");
+		seq_buf_puts(&kbd_name, dev->product);
 	}
 
-	if (!strlen(kbd->name))
+	if (!seq_buf_used(&kbd_name))
 		snprintf(kbd->name, sizeof(kbd->name),
 			 "USB HIDBP Keyboard %04x:%04x",
 			 le16_to_cpu(dev->descriptor.idVendor),
 			 le16_to_cpu(dev->descriptor.idProduct));
 
 	usb_make_path(dev, kbd->phys, sizeof(kbd->phys));
-	strlcat(kbd->phys, "/input0", sizeof(kbd->phys));
+	len = strnlen(kbd->phys, sizeof(kbd->phys));
+	strscpy(kbd->phys + len, "/input0", sizeof(kbd->phys) - len);
 
 	input_dev->name = kbd->name;
 	input_dev->phys = kbd->phys;
diff --git a/drivers/hid/usbhid/usbmouse.c b/drivers/hid/usbhid/usbmouse.c
index 7cc4f9558e5f2..b3b2abeee614a 100644
--- a/drivers/hid/usbhid/usbmouse.c
+++ b/drivers/hid/usbhid/usbmouse.c
@@ -18,6 +18,7 @@
 #include <linux/init.h>
 #include <linux/usb/input.h>
 #include <linux/hid.h>
+#include <linux/seq_buf.h>
 
 /* for apple IDs */
 #ifdef CONFIG_USB_HID_MODULE
@@ -110,8 +111,10 @@ static int usb_mouse_probe(struct usb_interface *intf, const struct usb_device_i
 	struct usb_endpoint_descriptor *endpoint;
 	struct usb_mouse *mouse;
 	struct input_dev *input_dev;
+	struct seq_buf mouse_name;
 	int pipe, maxp;
 	int error = -ENOMEM;
+	size_t len;
 
 	interface = intf->cur_altsetting;
 
@@ -140,24 +143,26 @@ static int usb_mouse_probe(struct usb_interface *intf, const struct usb_device_i
 
 	mouse->usbdev = dev;
 	mouse->dev = input_dev;
+	seq_buf_init(&mouse_name, mouse->name, sizeof(mouse->name));
 
 	if (dev->manufacturer)
-		strscpy(mouse->name, dev->manufacturer, sizeof(mouse->name));
+		seq_buf_puts(&mouse_name, dev->manufacturer);
 
 	if (dev->product) {
 		if (dev->manufacturer)
-			strlcat(mouse->name, " ", sizeof(mouse->name));
-		strlcat(mouse->name, dev->product, sizeof(mouse->name));
+			seq_buf_puts(&mouse_name, " ");
+		seq_buf_puts(&mouse_name, dev->product);
 	}
 
-	if (!strlen(mouse->name))
+	if (!seq_buf_used(&mouse_name))
 		snprintf(mouse->name, sizeof(mouse->name),
 			 "USB HIDBP Mouse %04x:%04x",
 			 le16_to_cpu(dev->descriptor.idVendor),
 			 le16_to_cpu(dev->descriptor.idProduct));
 
 	usb_make_path(dev, mouse->phys, sizeof(mouse->phys));
-	strlcat(mouse->phys, "/input0", sizeof(mouse->phys));
+	len = strnlen(mouse->phys, sizeof(mouse->phys));
+	strscpy(mouse->phys + len, "/input0", sizeof(mouse->phys) - len);
 
 	input_dev->name = mouse->name;
 	input_dev->phys = mouse->phys;
diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c
index 84eb0a6a0b546..89d214dda360a 100644
--- a/drivers/hv/channel_mgmt.c
+++ b/drivers/hv/channel_mgmt.c
@@ -952,6 +952,7 @@ void vmbus_initiate_unload(bool crash)
 	else
 		vmbus_wait_for_unload();
 }
+EXPORT_SYMBOL_GPL(vmbus_initiate_unload);
 
 static void vmbus_setup_channel_state(struct vmbus_channel *channel,
 				      struct vmbus_channel_offer_channel *offer)
diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c
index ae60fd5422929..ef4b1b03395d2 100644
--- a/drivers/hv/hv.c
+++ b/drivers/hv/hv.c
@@ -272,6 +272,9 @@ void hv_synic_free(void)
 /*
  * hv_hyp_synic_enable_regs - Initialize the Synthetic Interrupt Controller
  * with the hypervisor.
+ *
+ * Note: When MSHV is present, mshv_synic_cpu_init() intializes further
+ * registers later.
  */
 void hv_hyp_synic_enable_regs(unsigned int cpu)
 {
diff --git a/drivers/hv/hv_kvp.c b/drivers/hv/hv_kvp.c
index 0d73daf745a7b..336b278b2182a 100644
--- a/drivers/hv/hv_kvp.c
+++ b/drivers/hv/hv_kvp.c
@@ -27,6 +27,7 @@
 #include <linux/connector.h>
 #include <linux/workqueue.h>
 #include <linux/hyperv.h>
+#include <linux/string.h>
 #include <hyperv/hvhdk.h>
 
 #include "hyperv_vmbus.h"
@@ -93,7 +94,7 @@ static void kvp_send_key(struct work_struct *dummy);
 static void kvp_respond_to_host(struct hv_kvp_msg *msg, int error);
 static void kvp_timeout_func(struct work_struct *dummy);
 static void kvp_host_handshake_func(struct work_struct *dummy);
-static void kvp_register(int);
+static int kvp_register(int);
 
 static DECLARE_DELAYED_WORK(kvp_timeout_work, kvp_timeout_func);
 static DECLARE_DELAYED_WORK(kvp_host_handshake_work, kvp_host_handshake_func);
@@ -127,24 +128,23 @@ static void kvp_register_done(void)
 	hv_poll_channel(kvp_transaction.recv_channel, kvp_poll_wrapper);
 }
 
-static void
+static int
 kvp_register(int reg_value)
 {
-
 	struct hv_kvp_msg *kvp_msg;
-	char *version;
+	int ret;
 
 	kvp_msg = kzalloc_obj(*kvp_msg);
+	if (!kvp_msg)
+		return -ENOMEM;
 
-	if (kvp_msg) {
-		version = kvp_msg->body.kvp_register.version;
-		kvp_msg->kvp_hdr.operation = reg_value;
-		strcpy(version, HV_DRV_VERSION);
+	kvp_msg->kvp_hdr.operation = reg_value;
+	strscpy(kvp_msg->body.kvp_register.version, HV_DRV_VERSION);
 
-		hvutil_transport_send(hvt, kvp_msg, sizeof(*kvp_msg),
-				      kvp_register_done);
-		kfree(kvp_msg);
-	}
+	ret = hvutil_transport_send(hvt, kvp_msg, sizeof(*kvp_msg),
+				    kvp_register_done);
+	kfree(kvp_msg);
+	return ret;
 }
 
 static void kvp_timeout_func(struct work_struct *dummy)
@@ -186,9 +186,8 @@ static int kvp_handle_handshake(struct hv_kvp_msg *msg)
 	 */
 	pr_debug("KVP: userspace daemon ver. %d connected\n",
 		 msg->kvp_hdr.operation);
-	kvp_register(dm_reg_value);
 
-	return 0;
+	return kvp_register(dm_reg_value);
 }
 
 
diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h
index 05a36854389af..eb8bdd8bb1f58 100644
--- a/drivers/hv/hyperv_vmbus.h
+++ b/drivers/hv/hyperv_vmbus.h
@@ -441,7 +441,6 @@ void hv_vss_deinit(void);
 int hv_vss_pre_suspend(void);
 int hv_vss_pre_resume(void);
 void hv_vss_onchannelcallback(void *context);
-void vmbus_initiate_unload(bool crash);
 
 static inline void hv_poll_channel(struct vmbus_channel *channel,
 				   void (*cb)(void *))
diff --git a/drivers/hv/mshv_debugfs.c b/drivers/hv/mshv_debugfs.c
index 418b6dc8f3c2a..3c3e02237ae94 100644
--- a/drivers/hv/mshv_debugfs.c
+++ b/drivers/hv/mshv_debugfs.c
@@ -674,8 +674,10 @@ int __init mshv_debugfs_init(void)
 
 	mshv_debugfs = debugfs_create_dir("mshv", NULL);
 	if (IS_ERR(mshv_debugfs)) {
+		err = PTR_ERR(mshv_debugfs);
+		mshv_debugfs = NULL;
 		pr_err("%s: failed to create debugfs directory\n", __func__);
-		return PTR_ERR(mshv_debugfs);
+		return err;
 	}
 
 	if (hv_root_partition()) {
@@ -710,6 +712,9 @@ int __init mshv_debugfs_init(void)
 
 void mshv_debugfs_exit(void)
 {
+	if (!mshv_debugfs)
+		return;
+
 	mshv_debugfs_parent_partition_remove();
 
 	if (hv_root_partition()) {
diff --git a/drivers/hv/mshv_synic.c b/drivers/hv/mshv_synic.c
index e2288a726fec9..88170ce6b83f0 100644
--- a/drivers/hv/mshv_synic.c
+++ b/drivers/hv/mshv_synic.c
@@ -13,6 +13,7 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/cpuhotplug.h>
+#include <linux/hyperv.h>
 #include <linux/reboot.h>
 #include <asm/mshyperv.h>
 #include <linux/acpi.h>
@@ -456,46 +457,75 @@ static int mshv_synic_cpu_init(unsigned int cpu)
 	union hv_synic_siefp siefp;
 	union hv_synic_sirbp sirbp;
 	union hv_synic_sint sint;
-	union hv_synic_scontrol sctrl;
 	struct hv_synic_pages *spages = this_cpu_ptr(synic_pages);
 	struct hv_message_page **msg_page = &spages->hyp_synic_message_page;
 	struct hv_synic_event_flags_page **event_flags_page =
 			&spages->synic_event_flags_page;
 	struct hv_synic_event_ring_page **event_ring_page =
 			&spages->synic_event_ring_page;
+	/*
+	 * VMBus owns SIMP/SIEFP/SCONTROL when it is active.
+	 * See hv_hyp_synic_enable_regs() for that initialization.
+	 */
+	bool vmbus_active = hv_vmbus_exists();
 
-	/* Setup the Synic's message page */
+	/*
+	 * Map the SYNIC message page. When VMBus is not active the
+	 * hypervisor pre-provisions the SIMP GPA but may not set
+	 * simp_enabled — enable it here.
+	 */
 	simp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIMP);
-	simp.simp_enabled = true;
+	if (!vmbus_active) {
+		simp.simp_enabled = true;
+		hv_set_non_nested_msr(HV_MSR_SIMP, simp.as_uint64);
+	}
 	*msg_page = memremap(simp.base_simp_gpa << HV_HYP_PAGE_SHIFT,
 			     HV_HYP_PAGE_SIZE,
 			     MEMREMAP_WB);
 
 	if (!(*msg_page))
-		return -EFAULT;
-
-	hv_set_non_nested_msr(HV_MSR_SIMP, simp.as_uint64);
+		goto cleanup_simp;
 
-	/* Setup the Synic's event flags page */
+	/*
+	 * Map the event flags page. Same as SIMP: enable when
+	 * VMBus is not active, already enabled by VMBus otherwise.
+	 */
 	siefp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIEFP);
-	siefp.siefp_enabled = true;
-	*event_flags_page = memremap(siefp.base_siefp_gpa << PAGE_SHIFT,
-				     PAGE_SIZE, MEMREMAP_WB);
+	if (!vmbus_active) {
+		siefp.siefp_enabled = true;
+		hv_set_non_nested_msr(HV_MSR_SIEFP, siefp.as_uint64);
+	}
+	*event_flags_page = memremap(siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT,
+				     HV_HYP_PAGE_SIZE, MEMREMAP_WB);
 
 	if (!(*event_flags_page))
-		goto cleanup;
-
-	hv_set_non_nested_msr(HV_MSR_SIEFP, siefp.as_uint64);
+		goto cleanup_siefp;
 
 	/* Setup the Synic's event ring page */
 	sirbp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIRBP);
-	sirbp.sirbp_enabled = true;
-	*event_ring_page = memremap(sirbp.base_sirbp_gpa << PAGE_SHIFT,
-				    PAGE_SIZE, MEMREMAP_WB);
 
-	if (!(*event_ring_page))
-		goto cleanup;
+	if (hv_root_partition()) {
+		*event_ring_page = memremap(sirbp.base_sirbp_gpa << HV_HYP_PAGE_SHIFT,
+					    HV_HYP_PAGE_SIZE, MEMREMAP_WB);
 
+		if (!(*event_ring_page))
+			goto cleanup_siefp;
+	} else {
+		/*
+		 * On L1VH the hypervisor does not provide a SIRBP page.
+		 * Allocate one and program its GPA into the MSR.
+		 */
+		*event_ring_page = (struct hv_synic_event_ring_page *)
+			get_zeroed_page(GFP_KERNEL);
+
+		if (!(*event_ring_page))
+			goto cleanup_siefp;
+
+		sirbp.base_sirbp_gpa = virt_to_phys(*event_ring_page)
+				>> HV_HYP_PAGE_SHIFT;
+	}
+
+	sirbp.sirbp_enabled = true;
 	hv_set_non_nested_msr(HV_MSR_SIRBP, sirbp.as_uint64);
 
 	if (mshv_sint_irq != -1)
@@ -518,28 +548,30 @@ static int mshv_synic_cpu_init(unsigned int cpu)
 	hv_set_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_DOORBELL_SINT_INDEX,
 			      sint.as_uint64);
 
-	/* Enable global synic bit */
-	sctrl.as_uint64 = hv_get_non_nested_msr(HV_MSR_SCONTROL);
-	sctrl.enable = 1;
-	hv_set_non_nested_msr(HV_MSR_SCONTROL, sctrl.as_uint64);
+	/* When VMBus is active it already enabled SCONTROL. */
+	if (!vmbus_active) {
+		union hv_synic_scontrol sctrl;
+
+		sctrl.as_uint64 = hv_get_non_nested_msr(HV_MSR_SCONTROL);
+		sctrl.enable = 1;
+		hv_set_non_nested_msr(HV_MSR_SCONTROL, sctrl.as_uint64);
+	}
 
 	return 0;
 
-cleanup:
-	if (*event_ring_page) {
-		sirbp.sirbp_enabled = false;
-		hv_set_non_nested_msr(HV_MSR_SIRBP, sirbp.as_uint64);
-		memunmap(*event_ring_page);
-	}
-	if (*event_flags_page) {
+cleanup_siefp:
+	if (*event_flags_page)
+		memunmap(*event_flags_page);
+	if (!vmbus_active) {
 		siefp.siefp_enabled = false;
 		hv_set_non_nested_msr(HV_MSR_SIEFP, siefp.as_uint64);
-		memunmap(*event_flags_page);
 	}
-	if (*msg_page) {
+cleanup_simp:
+	if (*msg_page)
+		memunmap(*msg_page);
+	if (!vmbus_active) {
 		simp.simp_enabled = false;
 		hv_set_non_nested_msr(HV_MSR_SIMP, simp.as_uint64);
-		memunmap(*msg_page);
 	}
 
 	return -EFAULT;
@@ -548,16 +580,15 @@ static int mshv_synic_cpu_init(unsigned int cpu)
 static int mshv_synic_cpu_exit(unsigned int cpu)
 {
 	union hv_synic_sint sint;
-	union hv_synic_simp simp;
-	union hv_synic_siefp siefp;
 	union hv_synic_sirbp sirbp;
-	union hv_synic_scontrol sctrl;
 	struct hv_synic_pages *spages = this_cpu_ptr(synic_pages);
 	struct hv_message_page **msg_page = &spages->hyp_synic_message_page;
 	struct hv_synic_event_flags_page **event_flags_page =
 		&spages->synic_event_flags_page;
 	struct hv_synic_event_ring_page **event_ring_page =
 		&spages->synic_event_ring_page;
+	/* VMBus owns SIMP/SIEFP/SCONTROL when it is active */
+	bool vmbus_active = hv_vmbus_exists();
 
 	/* Disable the interrupt */
 	sint.as_uint64 = hv_get_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX);
@@ -574,28 +605,47 @@ static int mshv_synic_cpu_exit(unsigned int cpu)
 	if (mshv_sint_irq != -1)
 		disable_percpu_irq(mshv_sint_irq);
 
-	/* Disable Synic's event ring page */
+	/* Disable SYNIC event ring page owned by MSHV */
 	sirbp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIRBP);
 	sirbp.sirbp_enabled = false;
-	hv_set_non_nested_msr(HV_MSR_SIRBP, sirbp.as_uint64);
-	memunmap(*event_ring_page);
 
-	/* Disable Synic's event flags page */
-	siefp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIEFP);
-	siefp.siefp_enabled = false;
-	hv_set_non_nested_msr(HV_MSR_SIEFP, siefp.as_uint64);
+	if (hv_root_partition()) {
+		hv_set_non_nested_msr(HV_MSR_SIRBP, sirbp.as_uint64);
+		memunmap(*event_ring_page);
+	} else {
+		sirbp.base_sirbp_gpa = 0;
+		hv_set_non_nested_msr(HV_MSR_SIRBP, sirbp.as_uint64);
+		free_page((unsigned long)*event_ring_page);
+	}
+
+	/*
+	 * Release our mappings of the message and event flags pages.
+	 * When VMBus is not active, we enabled SIMP/SIEFP — disable
+	 * them. Otherwise VMBus owns the MSRs — leave them.
+	 */
 	memunmap(*event_flags_page);
+	if (!vmbus_active) {
+		union hv_synic_simp simp;
+		union hv_synic_siefp siefp;
 
-	/* Disable Synic's message page */
-	simp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIMP);
-	simp.simp_enabled = false;
-	hv_set_non_nested_msr(HV_MSR_SIMP, simp.as_uint64);
+		siefp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIEFP);
+		siefp.siefp_enabled = false;
+		hv_set_non_nested_msr(HV_MSR_SIEFP, siefp.as_uint64);
+
+		simp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIMP);
+		simp.simp_enabled = false;
+		hv_set_non_nested_msr(HV_MSR_SIMP, simp.as_uint64);
+	}
 	memunmap(*msg_page);
 
-	/* Disable global synic bit */
-	sctrl.as_uint64 = hv_get_non_nested_msr(HV_MSR_SCONTROL);
-	sctrl.enable = 0;
-	hv_set_non_nested_msr(HV_MSR_SCONTROL, sctrl.as_uint64);
+	/* When VMBus is active it owns SCONTROL — leave it. */
+	if (!vmbus_active) {
+		union hv_synic_scontrol sctrl;
+
+		sctrl.as_uint64 = hv_get_non_nested_msr(HV_MSR_SCONTROL);
+		sctrl.enable = 0;
+		hv_set_non_nested_msr(HV_MSR_SCONTROL, sctrl.as_uint64);
+	}
 
 	return 0;
 }
@@ -673,9 +723,7 @@ mshv_unregister_doorbell(u64 partition_id, int doorbell_portid)
 static int mshv_synic_reboot_notify(struct notifier_block *nb,
 			      unsigned long code, void *unused)
 {
-	if (!hv_root_partition())
-		return 0;
-
+	mshv_debugfs_exit();
 	cpuhp_remove_state(synic_cpuhp_online);
 	return 0;
 }
diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
index d28ff45d4cfdd..c9eeb2ec365dc 100644
--- a/drivers/hv/vmbus_drv.c
+++ b/drivers/hv/vmbus_drv.c
@@ -69,19 +69,29 @@ bool vmbus_is_confidential(void)
 }
 EXPORT_SYMBOL_GPL(vmbus_is_confidential);
 
+static bool skip_vmbus_unload;
+
+/*
+ * Allow a VMBus framebuffer driver to specify that in the case of a panic,
+ * it will do the VMbus unload operation once it has flushed any dirty
+ * portions of the framebuffer to the Hyper-V host.
+ */
+void vmbus_set_skip_unload(bool skip)
+{
+	skip_vmbus_unload = skip;
+}
+EXPORT_SYMBOL_GPL(vmbus_set_skip_unload);
+
 /*
  * The panic notifier below is responsible solely for unloading the
  * vmbus connection, which is necessary in a panic event.
- *
- * Notice an intrincate relation of this notifier with Hyper-V
- * framebuffer panic notifier exists - we need vmbus connection alive
- * there in order to succeed, so we need to order both with each other
- * [see hvfb_on_panic()] - this is done using notifiers' priorities.
  */
 static int hv_panic_vmbus_unload(struct notifier_block *nb, unsigned long val,
 			      void *args)
 {
-	vmbus_initiate_unload(true);
+	if (!skip_vmbus_unload)
+		vmbus_initiate_unload(true);
+
 	return NOTIFY_DONE;
 }
 static struct notifier_block hyperv_panic_vmbus_unload_block = {
@@ -2897,7 +2907,8 @@ static void hv_crash_handler(struct pt_regs *regs)
 {
 	int cpu;
 
-	vmbus_initiate_unload(true);
+	if (!skip_vmbus_unload)
+		vmbus_initiate_unload(true);
 	/*
 	 * In crash handler we can't schedule synic cleanup for all CPUs,
 	 * doing the cleanup for current CPU only. This should be sufficient
diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
index 14e4cea48acc4..997bba56fedfd 100644
--- a/drivers/hwmon/Kconfig
+++ b/drivers/hwmon/Kconfig
@@ -388,6 +388,18 @@ config SENSORS_APPLESMC
 	  Say Y here if you have an applicable laptop and want to experience
 	  the awesome power of applesmc.
 
+config SENSORS_ARCTIC_FAN_CONTROLLER
+	tristate "ARCTIC Fan Controller"
+	depends on USB_HID
+	help
+	  If you say yes here you get support for the ARCTIC Fan Controller,
+	  a USB HID device (VID 0x3904, PID 0xF001) with 10 fan channels.
+	  The driver exposes fan speed (RPM) and PWM control via the hwmon
+	  sysfs interface.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called arctic_fan_controller.
+
 config SENSORS_ARM_SCMI
 	tristate "ARM SCMI Sensors"
 	depends on ARM_SCMI_PROTOCOL
diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
index 982ee2c6f9deb..efbd1cb81816f 100644
--- a/drivers/hwmon/Makefile
+++ b/drivers/hwmon/Makefile
@@ -49,6 +49,7 @@ obj-$(CONFIG_SENSORS_ADT7475)	+= adt7475.o
 obj-$(CONFIG_SENSORS_AHT10)	+= aht10.o
 obj-$(CONFIG_SENSORS_APPLESMC)	+= applesmc.o
 obj-$(CONFIG_SENSORS_AQUACOMPUTER_D5NEXT) += aquacomputer_d5next.o
+obj-$(CONFIG_SENSORS_ARCTIC_FAN_CONTROLLER)	+= arctic_fan_controller.o
 obj-$(CONFIG_SENSORS_ARM_SCMI)	+= scmi-hwmon.o
 obj-$(CONFIG_SENSORS_ARM_SCPI)	+= scpi-hwmon.o
 obj-$(CONFIG_SENSORS_AS370)	+= as370-hwmon.o
diff --git a/drivers/hwmon/ad7414.c b/drivers/hwmon/ad7414.c
index f0b17e59827f9..c2df631ae93a3 100644
--- a/drivers/hwmon/ad7414.c
+++ b/drivers/hwmon/ad7414.c
@@ -205,8 +205,8 @@ static int ad7414_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ad7414_id[] = {
-	{ "ad7414" },
-	{}
+	{ .name = "ad7414" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ad7414_id);
 
diff --git a/drivers/hwmon/ad7418.c b/drivers/hwmon/ad7418.c
index 7a132accdf8a3..a0c9cf5e12cc4 100644
--- a/drivers/hwmon/ad7418.c
+++ b/drivers/hwmon/ad7418.c
@@ -281,9 +281,9 @@ static int ad7418_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ad7418_id[] = {
-	{ "ad7416", ad7416 },
-	{ "ad7417", ad7417 },
-	{ "ad7418", ad7418 },
+	{ .name = "ad7416", .driver_data = ad7416 },
+	{ .name = "ad7417", .driver_data = ad7417 },
+	{ .name = "ad7418", .driver_data = ad7418 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ad7418_id);
diff --git a/drivers/hwmon/adc128d818.c b/drivers/hwmon/adc128d818.c
index 5e805d4ee76ab..e45adc0b84d1c 100644
--- a/drivers/hwmon/adc128d818.c
+++ b/drivers/hwmon/adc128d818.c
@@ -480,7 +480,7 @@ static int adc128_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adc128_id[] = {
-	{ "adc128d818" },
+	{ .name = "adc128d818" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adc128_id);
diff --git a/drivers/hwmon/adm1025.c b/drivers/hwmon/adm1025.c
index 389382d54752e..ccac7ba601e99 100644
--- a/drivers/hwmon/adm1025.c
+++ b/drivers/hwmon/adm1025.c
@@ -548,8 +548,8 @@ static int adm1025_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adm1025_id[] = {
-	{ "adm1025", adm1025 },
-	{ "ne1619", ne1619 },
+	{ .name = "adm1025", .driver_data = adm1025 },
+	{ .name = "ne1619", .driver_data = ne1619 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adm1025_id);
diff --git a/drivers/hwmon/adm1026.c b/drivers/hwmon/adm1026.c
index c38c932e5d2af..3ad82c01f81d5 100644
--- a/drivers/hwmon/adm1026.c
+++ b/drivers/hwmon/adm1026.c
@@ -1857,7 +1857,7 @@ static int adm1026_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adm1026_id[] = {
-	{ "adm1026" },
+	{ .name = "adm1026" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adm1026_id);
diff --git a/drivers/hwmon/adm1029.c b/drivers/hwmon/adm1029.c
index 71eea8ae51b95..6cb0a238e059b 100644
--- a/drivers/hwmon/adm1029.c
+++ b/drivers/hwmon/adm1029.c
@@ -382,7 +382,7 @@ static int adm1029_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adm1029_id[] = {
-	{ "adm1029" },
+	{ .name = "adm1029" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adm1029_id);
diff --git a/drivers/hwmon/adm1031.c b/drivers/hwmon/adm1031.c
index 343118532cdb4..24d0d4146c877 100644
--- a/drivers/hwmon/adm1031.c
+++ b/drivers/hwmon/adm1031.c
@@ -1055,8 +1055,8 @@ static int adm1031_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adm1031_id[] = {
-	{ "adm1030", adm1030 },
-	{ "adm1031", adm1031 },
+	{ .name = "adm1030", .driver_data = adm1030 },
+	{ .name = "adm1031", .driver_data = adm1031 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adm1031_id);
diff --git a/drivers/hwmon/adm1177.c b/drivers/hwmon/adm1177.c
index 7888afe8dafd6..dc4d8a214d1b4 100644
--- a/drivers/hwmon/adm1177.c
+++ b/drivers/hwmon/adm1177.c
@@ -246,8 +246,8 @@ static int adm1177_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adm1177_id[] = {
-	{"adm1177"},
-	{}
+	{ .name = "adm1177" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adm1177_id);
 
diff --git a/drivers/hwmon/adm9240.c b/drivers/hwmon/adm9240.c
index 86f6044b5bd04..586650e8d043f 100644
--- a/drivers/hwmon/adm9240.c
+++ b/drivers/hwmon/adm9240.c
@@ -794,9 +794,9 @@ static int adm9240_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adm9240_id[] = {
-	{ "adm9240", adm9240 },
-	{ "ds1780", ds1780 },
-	{ "lm81", lm81 },
+	{ .name = "adm9240", .driver_data = adm9240 },
+	{ .name = "ds1780", .driver_data = ds1780 },
+	{ .name = "lm81", .driver_data = lm81 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adm9240_id);
diff --git a/drivers/hwmon/ads7828.c b/drivers/hwmon/ads7828.c
index 7f43565ca2841..149cfcec78dcc 100644
--- a/drivers/hwmon/ads7828.c
+++ b/drivers/hwmon/ads7828.c
@@ -176,8 +176,8 @@ static int ads7828_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ads7828_device_ids[] = {
-	{ "ads7828", ads7828 },
-	{ "ads7830", ads7830 },
+	{ .name = "ads7828", .driver_data = ads7828 },
+	{ .name = "ads7830", .driver_data = ads7830 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ads7828_device_ids);
diff --git a/drivers/hwmon/ads7871.c b/drivers/hwmon/ads7871.c
index 9ee3ce01f130b..d1cbff0e85720 100644
--- a/drivers/hwmon/ads7871.c
+++ b/drivers/hwmon/ads7871.c
@@ -56,7 +56,6 @@
 #include <linux/init.h>
 #include <linux/spi/spi.h>
 #include <linux/hwmon.h>
-#include <linux/hwmon-sysfs.h>
 #include <linux/err.h>
 #include <linux/delay.h>
 
@@ -64,8 +63,19 @@
 
 struct ads7871_data {
 	struct spi_device *spi;
+	u8 tx_buf[2] ____cacheline_aligned;
 };
 
+static umode_t ads7871_is_visible(const void *data,
+				  enum hwmon_sensor_types type,
+				  u32 attr, int channel)
+{
+	if (type == hwmon_in && attr == hwmon_in_input)
+		return 0444;
+
+	return 0;
+}
+
 static int ads7871_read_reg8(struct spi_device *spi, int reg)
 {
 	int ret;
@@ -86,29 +96,31 @@ static int ads7871_read_reg16(struct spi_device *spi, int reg)
 	return le16_to_cpu((__force __le16)ret);
 }
 
-static int ads7871_write_reg8(struct spi_device *spi, int reg, u8 val)
+static int ads7871_write_reg8(struct ads7871_data *pdata, int reg, u8 val)
 {
-	u8 tmp[2] = {reg, val};
-	return spi_write(spi, tmp, sizeof(tmp));
+	pdata->tx_buf[0] = reg;
+	pdata->tx_buf[1] = val;
+
+	return spi_write(pdata->spi, pdata->tx_buf, 2);
 }
 
-static ssize_t voltage_show(struct device *dev, struct device_attribute *da,
-			    char *buf)
+static int ads7871_read(struct device *dev, enum hwmon_sensor_types type,
+			u32 attr, int channel, long *val)
 {
 	struct ads7871_data *pdata = dev_get_drvdata(dev);
 	struct spi_device *spi = pdata->spi;
-	struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
-	int ret, val, i = 0;
-	uint8_t channel, mux_cnv;
+	int ret, raw_val, i = 0;
+	u8 mux_cnv;
 
-	channel = attr->index;
+	if (type != hwmon_in || attr != hwmon_in_input)
+		return -EOPNOTSUPP;
 	/*
 	 * TODO: add support for conversions
 	 * other than single ended with a gain of 1
 	 */
 	/*MUX_M3_BM forces single ended*/
 	/*This is also where the gain of the PGA would be set*/
-	ret = ads7871_write_reg8(spi, REG_GAIN_MUX,
+	ret = ads7871_write_reg8(pdata, REG_GAIN_MUX,
 				 (MUX_CNV_BM | MUX_M3_BM | channel));
 	if (ret < 0)
 		return ret;
@@ -116,6 +128,7 @@ static ssize_t voltage_show(struct device *dev, struct device_attribute *da,
 	ret = ads7871_read_reg8(spi, REG_GAIN_MUX);
 	if (ret < 0)
 		return ret;
+
 	mux_cnv = ((ret & MUX_CNV_BM) >> MUX_CNV_BV);
 	/*
 	 * on 400MHz arm9 platform the conversion
@@ -131,39 +144,37 @@ static ssize_t voltage_show(struct device *dev, struct device_attribute *da,
 	}
 
 	if (mux_cnv == 0) {
-		val = ads7871_read_reg16(spi, REG_LS_BYTE);
-		if (val < 0)
-			return val;
-		/*result in volts*10000 = (val/8192)*2.5*10000*/
-		val = ((val >> 2) * 25000) / 8192;
-		return sysfs_emit(buf, "%d\n", val);
+		raw_val = ads7871_read_reg16(spi, REG_LS_BYTE);
+		if (raw_val < 0)
+			return raw_val;
+
+		/*
+		 * Use (s16) to ensure the sign bit is preserved during the shift.
+		 * Report millivolts (2.5V = 2500mV).
+		 */
+		*val = ((s16)raw_val >> 2) * 2500 / 8192;
+		return 0;
 	}
 
 	return -ETIMEDOUT;
 }
 
-static SENSOR_DEVICE_ATTR_RO(in0_input, voltage, 0);
-static SENSOR_DEVICE_ATTR_RO(in1_input, voltage, 1);
-static SENSOR_DEVICE_ATTR_RO(in2_input, voltage, 2);
-static SENSOR_DEVICE_ATTR_RO(in3_input, voltage, 3);
-static SENSOR_DEVICE_ATTR_RO(in4_input, voltage, 4);
-static SENSOR_DEVICE_ATTR_RO(in5_input, voltage, 5);
-static SENSOR_DEVICE_ATTR_RO(in6_input, voltage, 6);
-static SENSOR_DEVICE_ATTR_RO(in7_input, voltage, 7);
-
-static struct attribute *ads7871_attrs[] = {
-	&sensor_dev_attr_in0_input.dev_attr.attr,
-	&sensor_dev_attr_in1_input.dev_attr.attr,
-	&sensor_dev_attr_in2_input.dev_attr.attr,
-	&sensor_dev_attr_in3_input.dev_attr.attr,
-	&sensor_dev_attr_in4_input.dev_attr.attr,
-	&sensor_dev_attr_in5_input.dev_attr.attr,
-	&sensor_dev_attr_in6_input.dev_attr.attr,
-	&sensor_dev_attr_in7_input.dev_attr.attr,
+static const struct hwmon_channel_info * const ads7871_info[] = {
+	HWMON_CHANNEL_INFO(in,
+			   HWMON_I_INPUT, HWMON_I_INPUT, HWMON_I_INPUT, HWMON_I_INPUT,
+			   HWMON_I_INPUT, HWMON_I_INPUT, HWMON_I_INPUT, HWMON_I_INPUT),
 	NULL
 };
 
-ATTRIBUTE_GROUPS(ads7871);
+static const struct hwmon_ops ads7871_hwmon_ops = {
+	.is_visible = ads7871_is_visible,
+	.read = ads7871_read,
+};
+
+static const struct hwmon_chip_info ads7871_chip_info = {
+	.ops = &ads7871_hwmon_ops,
+	.info = ads7871_info,
+};
 
 static int ads7871_probe(struct spi_device *spi)
 {
@@ -178,11 +189,17 @@ static int ads7871_probe(struct spi_device *spi)
 	spi->bits_per_word = 8;
 	spi_setup(spi);
 
-	ads7871_write_reg8(spi, REG_SER_CONTROL, 0);
-	ads7871_write_reg8(spi, REG_AD_CONTROL, 0);
+	pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
+	if (!pdata)
+		return -ENOMEM;
+
+	pdata->spi = spi;
+
+	ads7871_write_reg8(pdata, REG_SER_CONTROL, 0);
+	ads7871_write_reg8(pdata, REG_AD_CONTROL, 0);
 
 	val = (OSC_OSCR_BM | OSC_OSCE_BM | OSC_REFE_BM | OSC_BUFE_BM);
-	ads7871_write_reg8(spi, REG_OSC_CONTROL, val);
+	ads7871_write_reg8(pdata, REG_OSC_CONTROL, val);
 	ret = ads7871_read_reg8(spi, REG_OSC_CONTROL);
 
 	dev_dbg(dev, "REG_OSC_CONTROL write:%x, read:%x\n", val, ret);
@@ -193,15 +210,10 @@ static int ads7871_probe(struct spi_device *spi)
 	if (val != ret)
 		return -ENODEV;
 
-	pdata = devm_kzalloc(dev, sizeof(struct ads7871_data), GFP_KERNEL);
-	if (!pdata)
-		return -ENOMEM;
-
-	pdata->spi = spi;
-
-	hwmon_dev = devm_hwmon_device_register_with_groups(dev, spi->modalias,
-							   pdata,
-							   ads7871_groups);
+	hwmon_dev = devm_hwmon_device_register_with_info(dev, spi->modalias,
+							 pdata,
+							 &ads7871_chip_info,
+							 NULL);
 	return PTR_ERR_OR_ZERO(hwmon_dev);
 }
 
diff --git a/drivers/hwmon/adt7410.c b/drivers/hwmon/adt7410.c
index 73b196a78f3a0..0aa7ce0a04be0 100644
--- a/drivers/hwmon/adt7410.c
+++ b/drivers/hwmon/adt7410.c
@@ -89,10 +89,10 @@ static int adt7410_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adt7410_ids[] = {
-	{ "adt7410" },
-	{ "adt7420" },
-	{ "adt7422" },
-	{}
+	{ .name = "adt7410" },
+	{ .name = "adt7420" },
+	{ .name = "adt7422" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adt7410_ids);
 
diff --git a/drivers/hwmon/adt7411.c b/drivers/hwmon/adt7411.c
index b9991a69e6c60..598f316475446 100644
--- a/drivers/hwmon/adt7411.c
+++ b/drivers/hwmon/adt7411.c
@@ -169,11 +169,11 @@ static ssize_t adt7411_set_bit(struct device *dev,
 	if (ret || flag > 1)
 		return -EINVAL;
 
-	hwmon_lock(dev);
-	ret = adt7411_modify_bit(client, s_attr2->index, s_attr2->nr, flag);
-	/* force update */
-	data->next_update = jiffies;
-	hwmon_unlock(dev);
+	scoped_guard(hwmon_lock, dev) {
+		ret = adt7411_modify_bit(client, s_attr2->index, s_attr2->nr, flag);
+		/* force update */
+		data->next_update = jiffies;
+	}
 
 	return ret < 0 ? ret : count;
 }
@@ -670,7 +670,7 @@ static int adt7411_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adt7411_id[] = {
-	{ "adt7411" },
+	{ .name = "adt7411" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adt7411_id);
diff --git a/drivers/hwmon/adt7462.c b/drivers/hwmon/adt7462.c
index 174dfee47f7a7..5101d30ed9c68 100644
--- a/drivers/hwmon/adt7462.c
+++ b/drivers/hwmon/adt7462.c
@@ -1809,7 +1809,7 @@ static int adt7462_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adt7462_id[] = {
-	{ "adt7462" },
+	{ .name = "adt7462" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adt7462_id);
diff --git a/drivers/hwmon/adt7470.c b/drivers/hwmon/adt7470.c
index dbee6926fa055..664349756dc2b 100644
--- a/drivers/hwmon/adt7470.c
+++ b/drivers/hwmon/adt7470.c
@@ -1296,7 +1296,7 @@ static void adt7470_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adt7470_id[] = {
-	{ "adt7470" },
+	{ .name = "adt7470" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adt7470_id);
diff --git a/drivers/hwmon/adt7475.c b/drivers/hwmon/adt7475.c
index 8cefa14e1633b..7fb96f5395fa6 100644
--- a/drivers/hwmon/adt7475.c
+++ b/drivers/hwmon/adt7475.c
@@ -165,10 +165,10 @@ static const unsigned short normal_i2c[] = { 0x2c, 0x2d, 0x2e, I2C_CLIENT_END };
 enum chips { adt7473, adt7475, adt7476, adt7490 };
 
 static const struct i2c_device_id adt7475_id[] = {
-	{ "adt7473", adt7473 },
-	{ "adt7475", adt7475 },
-	{ "adt7476", adt7476 },
-	{ "adt7490", adt7490 },
+	{ .name = "adt7473", .driver_data = adt7473 },
+	{ .name = "adt7475", .driver_data = adt7475 },
+	{ .name = "adt7476", .driver_data = adt7476 },
+	{ .name = "adt7490", .driver_data = adt7490 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adt7475_id);
diff --git a/drivers/hwmon/aht10.c b/drivers/hwmon/aht10.c
index 66955395d0583..6c263cb577666 100644
--- a/drivers/hwmon/aht10.c
+++ b/drivers/hwmon/aht10.c
@@ -55,10 +55,10 @@
 enum aht10_variant { aht10, aht20, dht20};
 
 static const struct i2c_device_id aht10_id[] = {
-	{ "aht10", aht10 },
-	{ "aht20", aht20 },
-	{ "dht20", dht20 },
-	{ },
+	{ .name = "aht10", .driver_data = aht10 },
+	{ .name = "aht20", .driver_data = aht20 },
+	{ .name = "dht20", .driver_data = dht20 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, aht10_id);
 
diff --git a/drivers/hwmon/amc6821.c b/drivers/hwmon/amc6821.c
index d5f864b360b03..bbc0542a7d380 100644
--- a/drivers/hwmon/amc6821.c
+++ b/drivers/hwmon/amc6821.c
@@ -1083,7 +1083,7 @@ static int amc6821_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id amc6821_id[] = {
-	{ "amc6821" },
+	{ .name = "amc6821" },
 	{ }
 };
 
diff --git a/drivers/hwmon/applesmc.c b/drivers/hwmon/applesmc.c
index 24f3e86d0ebf2..90a14a7f2c4c0 100644
--- a/drivers/hwmon/applesmc.c
+++ b/drivers/hwmon/applesmc.c
@@ -1305,6 +1305,7 @@ static const struct dmi_system_id applesmc_whitelist[] __initconst = {
 	},
 	{ .ident = NULL }
 };
+MODULE_DEVICE_TABLE(dmi, applesmc_whitelist);
 
 static int __init applesmc_init(void)
 {
@@ -1416,4 +1417,3 @@ module_exit(applesmc_exit);
 MODULE_AUTHOR("Nicolas Boichat");
 MODULE_DESCRIPTION("Apple SMC");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(dmi, applesmc_whitelist);
diff --git a/drivers/hwmon/arctic_fan_controller.c b/drivers/hwmon/arctic_fan_controller.c
new file mode 100644
index 0000000000000..dbe84cd93c083
--- /dev/null
+++ b/drivers/hwmon/arctic_fan_controller.c
@@ -0,0 +1,374 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Linux hwmon driver for ARCTIC Fan Controller
+ *
+ * USB Custom HID device with 10 fan channels.
+ * Exposes fan RPM (input) and PWM (0-255) via hwmon. Device pushes IN reports
+ * at ~1 Hz; no GET_REPORT. OUT reports set PWM duty (bytes 1-10, 0-100%).
+ * PWM is manual-only: the device does not change duty autonomously, only
+ * when it receives an OUT report from the host.
+ */
+
+#include <linux/completion.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/hid.h>
+#include <linux/hwmon.h>
+#include <linux/jiffies.h>
+#include <linux/minmax.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <linux/string.h>
+#include <linux/unaligned.h>
+
+#define ARCTIC_VID			0x3904
+#define ARCTIC_PID			0xF001
+#define ARCTIC_NUM_FANS			10
+#define ARCTIC_OUTPUT_REPORT_ID		0x01
+#define ARCTIC_REPORT_LEN		32
+#define ARCTIC_RPM_OFFSET		11	/* bytes 11-30: 10 x uint16 LE */
+/* ACK report: device sends Report ID 0x02, 2 bytes (ID + status) after applying OUT report */
+#define ARCTIC_ACK_REPORT_ID		0x02
+#define ARCTIC_ACK_REPORT_LEN		2
+/*
+ * Time to wait for ACK report after send.
+ * Measured over 500 iterations: max ~563 ms. Keep 1 s as margin.
+ */
+#define ARCTIC_ACK_TIMEOUT_MS		1000
+
+struct arctic_fan_data {
+	struct hid_device *hdev;
+	struct device *hwmon_dev;	/* stored for explicit unregister in remove() */
+	spinlock_t in_report_lock;	/* protects fan_rpm, ack_status, write_pending, pwm_duty */
+	struct completion in_report_received; /* ACK (ID 0x02) received in raw_event */
+	int ack_status;			/* 0 = OK, negative errno on device error */
+	bool write_pending;		/* true while an OUT report ACK is in flight */
+	u32 fan_rpm[ARCTIC_NUM_FANS];
+	u8 pwm_duty[ARCTIC_NUM_FANS];	/* 0-255 matching sysfs range; converted to 0-100 on send */
+	/*
+	 * OUT report buffer passed to hid_hw_output_report(). Embedded in the
+	 * devm_kzalloc'd struct so it is heap-allocated and passes
+	 * usb_hcd_map_urb_for_dma(). Exclusively accessed by write(), which
+	 * the hwmon core serializes.
+	 */
+	__dma_from_device_group_begin();
+	u8 buf[ARCTIC_REPORT_LEN];
+	__dma_from_device_group_end();
+};
+
+/*
+ * Parse RPM values from the periodic status report (10 x uint16 LE at rpm_off).
+ * pwm_duty is not updated from the report: the device is manual-only, so the
+ * host cache is the authoritative source for PWM.
+ * Called from raw_event which may run in IRQ context; must not sleep.
+ */
+static void arctic_fan_parse_report(struct arctic_fan_data *priv, u8 *buf,
+				    int len, int rpm_off)
+{
+	unsigned long flags;
+	int i;
+
+	if (len < rpm_off + 20)
+		return;
+
+	spin_lock_irqsave(&priv->in_report_lock, flags);
+	for (i = 0; i < ARCTIC_NUM_FANS; i++)
+		priv->fan_rpm[i] = get_unaligned_le16(&buf[rpm_off + i * 2]);
+	spin_unlock_irqrestore(&priv->in_report_lock, flags);
+}
+
+/*
+ * raw_event: IN reports.
+ *
+ * Status report: Report ID 0x01, 32 bytes:
+ *   byte 0 = report ID, bytes 1-10 = PWM 0-100%, bytes 11-30 = 10 x RPM uint16 LE.
+ *   Device pushes these at ~1 Hz; no GET_REPORT.
+ *
+ * ACK report: Report ID 0x02, 2 bytes:
+ *   byte 0 = 0x02, byte 1 = status (0x00 = OK, 0x01 = ERROR).
+ *   Sent once after accepting and applying an OUT report (ID 0x01).
+ */
+static int arctic_fan_raw_event(struct hid_device *hdev,
+				struct hid_report *report, u8 *data, int size)
+{
+	struct arctic_fan_data *priv = hid_get_drvdata(hdev);
+	unsigned long flags;
+
+	hid_dbg(hdev, "arctic_fan: raw_event id=%u size=%d\n", report->id, size);
+
+	if (report->id == ARCTIC_ACK_REPORT_ID && size == ARCTIC_ACK_REPORT_LEN) {
+		spin_lock_irqsave(&priv->in_report_lock, flags);
+		/*
+		 * Only deliver if a write is in flight. This prevents a
+		 * late-arriving ACK from a timed-out write from erroneously
+		 * satisfying a subsequent write's completion wait.
+		 */
+		if (priv->write_pending) {
+			priv->ack_status = data[1] == 0x00 ? 0 : -EIO;
+			complete(&priv->in_report_received);
+		}
+		spin_unlock_irqrestore(&priv->in_report_lock, flags);
+		return 0;
+	}
+
+	if (report->id != ARCTIC_OUTPUT_REPORT_ID || size != ARCTIC_REPORT_LEN) {
+		hid_dbg(hdev, "arctic_fan: raw_event id=%u size=%d ignored\n",
+			report->id, size);
+		return 0;
+	}
+
+	arctic_fan_parse_report(priv, data, size, ARCTIC_RPM_OFFSET);
+	return 0;
+}
+
+static umode_t arctic_fan_is_visible(const void *data,
+				     enum hwmon_sensor_types type,
+				     u32 attr, int channel)
+{
+	if (type == hwmon_fan && attr == hwmon_fan_input)
+		return 0444;
+	if (type == hwmon_pwm && attr == hwmon_pwm_input)
+		return 0644;
+	return 0;
+}
+
+static int arctic_fan_read(struct device *dev, enum hwmon_sensor_types type,
+			   u32 attr, int channel, long *val)
+{
+	struct arctic_fan_data *priv = dev_get_drvdata(dev);
+	unsigned long flags;
+
+	if (type == hwmon_fan && attr == hwmon_fan_input) {
+		spin_lock_irqsave(&priv->in_report_lock, flags);
+		*val = priv->fan_rpm[channel];
+		spin_unlock_irqrestore(&priv->in_report_lock, flags);
+		return 0;
+	}
+	if (type == hwmon_pwm && attr == hwmon_pwm_input) {
+		spin_lock_irqsave(&priv->in_report_lock, flags);
+		*val = priv->pwm_duty[channel];
+		spin_unlock_irqrestore(&priv->in_report_lock, flags);
+		return 0;
+	}
+	return -EINVAL;
+}
+
+static int arctic_fan_write(struct device *dev, enum hwmon_sensor_types type,
+			    u32 attr, int channel, long val)
+{
+	struct arctic_fan_data *priv = dev_get_drvdata(dev);
+	u8 new_duty = (u8)clamp_val(val, 0, 255);
+	unsigned long flags;
+	unsigned long t;
+	int i, ret;
+
+	/*
+	 * Build the buffer and arm write_pending under in_report_lock so that
+	 * reset_resume() cannot clear pwm_duty[] between the pwm_duty[] read
+	 * and the buffer write, and raw_event() cannot deliver a stale ACK
+	 * from a previous write into this write's completion.
+	 *
+	 * priv->buf is heap-allocated (embedded in the devm_kzalloc'd struct),
+	 * satisfying usb_hcd_map_urb_for_dma(). Exclusively accessed by
+	 * write() which the hwmon core serializes.
+	 *
+	 * pwm_duty[channel] is committed only after a positive device ACK so a
+	 * failed or timed-out write does not corrupt the cached state.
+	 *
+	 * Residual theoretical race: if write A times out (write_pending
+	 * cleared), write B sets write_pending = true, and a late ACK from
+	 * write A—delayed beyond ARCTIC_ACK_TIMEOUT_MS—arrives during write
+	 * B's pending window, it would falsely satisfy write B's completion.
+	 * This cannot be prevented in driver code without protocol support
+	 * (for example, a correlation ID echoed in the device ACK report).
+	 * In testing, observed ACK latency stayed below the 1 s timeout
+	 * (maximum ~563 ms over 500 iterations).
+	 *
+	 * The wait is non-interruptible so that a signal cannot cause write()
+	 * to return early while the OUT report is already in flight; an
+	 * interruptible early return would create the same late-ACK window
+	 * without even the timeout guard.
+	 * Serialized by the hwmon core: only one arctic_fan_write() at a time.
+	 * Use irqsave to match the IRQ context in which raw_event may run.
+	 */
+	spin_lock_irqsave(&priv->in_report_lock, flags);
+	priv->buf[0] = ARCTIC_OUTPUT_REPORT_ID;
+	for (i = 0; i < ARCTIC_NUM_FANS; i++) {
+		u8 d = i == channel ? new_duty : priv->pwm_duty[i];
+
+		priv->buf[1 + i] = DIV_ROUND_CLOSEST((unsigned int)d * 100, 255);
+	}
+	priv->ack_status = -ETIMEDOUT;
+	priv->write_pending = true;
+	reinit_completion(&priv->in_report_received);
+	spin_unlock_irqrestore(&priv->in_report_lock, flags);
+
+	ret = hid_hw_output_report(priv->hdev, priv->buf, ARCTIC_REPORT_LEN);
+	if (ret < 0) {
+		spin_lock_irqsave(&priv->in_report_lock, flags);
+		priv->write_pending = false;
+		spin_unlock_irqrestore(&priv->in_report_lock, flags);
+		return ret;
+	}
+
+	t = wait_for_completion_timeout(&priv->in_report_received,
+					msecs_to_jiffies(ARCTIC_ACK_TIMEOUT_MS));
+	spin_lock_irqsave(&priv->in_report_lock, flags);
+	priv->write_pending = false;
+	/* Commit inside the lock so reset_resume() cannot race with this write */
+	if (t && priv->ack_status == 0)
+		priv->pwm_duty[channel] = new_duty;
+	spin_unlock_irqrestore(&priv->in_report_lock, flags);
+
+	if (!t)
+		return -ETIMEDOUT;
+	return priv->ack_status; /* 0=OK, -EIO=device error */
+}
+
+static const struct hwmon_ops arctic_fan_ops = {
+	.is_visible = arctic_fan_is_visible,
+	.read = arctic_fan_read,
+	.write = arctic_fan_write,
+};
+
+static const struct hwmon_channel_info *arctic_fan_info[] = {
+	HWMON_CHANNEL_INFO(fan,
+			   HWMON_F_INPUT, HWMON_F_INPUT, HWMON_F_INPUT,
+			   HWMON_F_INPUT, HWMON_F_INPUT, HWMON_F_INPUT,
+			   HWMON_F_INPUT, HWMON_F_INPUT, HWMON_F_INPUT,
+			   HWMON_F_INPUT),
+	HWMON_CHANNEL_INFO(pwm,
+			   HWMON_PWM_INPUT, HWMON_PWM_INPUT, HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT, HWMON_PWM_INPUT, HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT, HWMON_PWM_INPUT, HWMON_PWM_INPUT,
+			   HWMON_PWM_INPUT),
+	NULL
+};
+
+static const struct hwmon_chip_info arctic_fan_chip_info = {
+	.ops = &arctic_fan_ops,
+	.info = arctic_fan_info,
+};
+
+static int arctic_fan_reset_resume(struct hid_device *hdev)
+{
+	struct arctic_fan_data *priv = hid_get_drvdata(hdev);
+	unsigned long flags;
+
+	/*
+	 * The device resets its PWM channels to hardware defaults on power
+	 * loss during suspend. Clear the cached duty values so they reflect
+	 * the unknown hardware state, consistent with probe-time behaviour
+	 * (the device has no GET_REPORT support). Hold in_report_lock so
+	 * this does not race with a concurrent pwm read or write callback.
+	 */
+	spin_lock_irqsave(&priv->in_report_lock, flags);
+	memset(priv->pwm_duty, 0, sizeof(priv->pwm_duty));
+	spin_unlock_irqrestore(&priv->in_report_lock, flags);
+	return 0;
+}
+
+static int arctic_fan_probe(struct hid_device *hdev,
+			    const struct hid_device_id *id)
+{
+	struct arctic_fan_data *priv;
+	int ret;
+
+	if (!hid_is_usb(hdev))
+		return -ENODEV;
+
+	ret = hid_parse(hdev);
+	if (ret)
+		return ret;
+
+	priv = devm_kzalloc(&hdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->hdev = hdev;
+	spin_lock_init(&priv->in_report_lock);
+	init_completion(&priv->in_report_received);
+	hid_set_drvdata(hdev, priv);
+
+	ret = hid_hw_start(hdev, HID_CONNECT_DRIVER);
+	if (ret)
+		return ret;
+
+	ret = hid_hw_open(hdev);
+	if (ret)
+		goto out_stop;
+
+	/*
+	 * Start IO before registering with hwmon. If IO were started after
+	 * hwmon registration, a sysfs write arriving in that narrow window
+	 * would send an OUT report but the ACK could not be delivered (the HID
+	 * core discards events until io_started), causing a spurious timeout.
+	 */
+	hid_device_io_start(hdev);
+
+	/*
+	 * Use the non-devm variant and store the pointer so remove() can
+	 * call hwmon_device_unregister() before tearing down the HID
+	 * transport. devm_hwmon_device_register_with_info() would defer
+	 * unregistration until after remove() returns, leaving a window
+	 * where a concurrent sysfs write could call hid_hw_output_report()
+	 * on an already-stopped device (use-after-free).
+	 */
+	priv->hwmon_dev = hwmon_device_register_with_info(&hdev->dev, "arctic_fan",
+							  priv, &arctic_fan_chip_info,
+							  NULL);
+	if (IS_ERR(priv->hwmon_dev)) {
+		ret = PTR_ERR(priv->hwmon_dev);
+		goto out_close;
+	}
+
+	return 0;
+
+out_close:
+	hid_device_io_stop(hdev);
+	hid_hw_close(hdev);
+out_stop:
+	hid_hw_stop(hdev);
+	return ret;
+}
+
+static void arctic_fan_remove(struct hid_device *hdev)
+{
+	struct arctic_fan_data *priv = hid_get_drvdata(hdev);
+
+	/*
+	 * Unregister hwmon before stopping the HID transport. This removes
+	 * the sysfs files and waits for any in-progress write() callback to
+	 * return, so no hwmon op can call hid_hw_output_report() after
+	 * hid_hw_stop() frees the underlying USB resources.
+	 * Matches the pattern used by nzxt-smart2 and aquacomputer_d5next.
+	 *
+	 * The HID core clears hdev->io_started before invoking ->remove(),
+	 * so hid_device_io_stop() is not called here; doing so would emit
+	 * a spurious "io already stopped" warning.
+	 */
+	hwmon_device_unregister(priv->hwmon_dev);
+	hid_hw_close(hdev);
+	hid_hw_stop(hdev);
+}
+
+static const struct hid_device_id arctic_fan_id_table[] = {
+	{ HID_USB_DEVICE(ARCTIC_VID, ARCTIC_PID) },
+	{ }
+};
+MODULE_DEVICE_TABLE(hid, arctic_fan_id_table);
+
+static struct hid_driver arctic_fan_driver = {
+	.name = "arctic_fan",
+	.id_table = arctic_fan_id_table,
+	.probe = arctic_fan_probe,
+	.remove = arctic_fan_remove,
+	.raw_event = arctic_fan_raw_event,
+	.reset_resume = arctic_fan_reset_resume,
+};
+
+module_hid_driver(arctic_fan_driver);
+
+MODULE_AUTHOR("Aureo Serrano de Souza <aureo.serrano@arctic.de>");
+MODULE_DESCRIPTION("HID hwmon driver for ARCTIC Fan Controller");
+MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/asb100.c b/drivers/hwmon/asb100.c
index 14e7737866c25..5f5c945051665 100644
--- a/drivers/hwmon/asb100.c
+++ b/drivers/hwmon/asb100.c
@@ -213,7 +213,7 @@ static struct asb100_data *asb100_update_device(struct device *dev);
 static void asb100_init_client(struct i2c_client *client);
 
 static const struct i2c_device_id asb100_id[] = {
-	{ "asb100" },
+	{ .name = "asb100" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, asb100_id);
diff --git a/drivers/hwmon/asc7621.c b/drivers/hwmon/asc7621.c
index 87e1867008499..e8145bb13ab34 100644
--- a/drivers/hwmon/asc7621.c
+++ b/drivers/hwmon/asc7621.c
@@ -1179,9 +1179,9 @@ static void asc7621_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id asc7621_id[] = {
-	{"asc7621", asc7621},
-	{"asc7621a", asc7621a},
-	{},
+	{ .name = "asc7621", .driver_data = asc7621 },
+	{ .name = "asc7621a", .driver_data = asc7621a },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, asc7621_id);
diff --git a/drivers/hwmon/atxp1.c b/drivers/hwmon/atxp1.c
index 1c7e9a98b757c..f5de4894f0a2e 100644
--- a/drivers/hwmon/atxp1.c
+++ b/drivers/hwmon/atxp1.c
@@ -278,7 +278,7 @@ static int atxp1_probe(struct i2c_client *client)
 };
 
 static const struct i2c_device_id atxp1_id[] = {
-	{ "atxp1" },
+	{ .name = "atxp1" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, atxp1_id);
diff --git a/drivers/hwmon/chipcap2.c b/drivers/hwmon/chipcap2.c
index 645b8c2e704e6..4aecf463180fd 100644
--- a/drivers/hwmon/chipcap2.c
+++ b/drivers/hwmon/chipcap2.c
@@ -736,14 +736,14 @@ static void cc2_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id cc2_id[] = {
-	{ "cc2d23" },
-	{ "cc2d23s" },
-	{ "cc2d25" },
-	{ "cc2d25s" },
-	{ "cc2d33" },
-	{ "cc2d33s" },
-	{ "cc2d35" },
-	{ "cc2d35s" },
+	{ .name = "cc2d23" },
+	{ .name = "cc2d23s" },
+	{ .name = "cc2d25" },
+	{ .name = "cc2d25s" },
+	{ .name = "cc2d33" },
+	{ .name = "cc2d33s" },
+	{ .name = "cc2d35" },
+	{ .name = "cc2d35s" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, cc2_id);
diff --git a/drivers/hwmon/coretemp.c b/drivers/hwmon/coretemp.c
index 6a0d94711eadb..02b4e46d965b9 100644
--- a/drivers/hwmon/coretemp.c
+++ b/drivers/hwmon/coretemp.c
@@ -25,7 +25,7 @@
 #include <linux/moduleparam.h>
 #include <linux/pci.h>
 #include <asm/msr.h>
-#include <asm/processor.h>
+#include <linux/processor.h>
 #include <asm/cpu_device_id.h>
 #include <linux/sched/isolation.h>
 
@@ -39,7 +39,6 @@ static int force_tjmax;
 module_param_named(tjmax, force_tjmax, int, 0444);
 MODULE_PARM_DESC(tjmax, "TjMax value in degrees Celsius");
 
-#define NUM_REAL_CORES		512	/* Number of Real cores per cpu */
 #define CORETEMP_NAME_LENGTH	28	/* String Length of attrs */
 
 enum coretemp_attr_index {
@@ -136,14 +135,14 @@ static const struct tjmax_model tjmax_model_table[] = {
 							 * which are covered by tjmax_table
 							 */
 	{ INTEL_ATOM_BONNELL_MID,     ANY, 90000 },	/* Atom Tunnel Creek (Exx), Lincroft (Z6xx)
-							 * Note: TjMax for E6xxT is 110C, but CPU type
-							 * is undetectable by software
+							 * Note: TjMax for E6xxT is 110C, but CPU
+							 * type is undetectable by software
 							 */
 	{ INTEL_ATOM_SALTWELL_MID,    ANY, 90000 },	/* Atom Medfield (Z2460) */
 	{ INTEL_ATOM_SALTWELL_TABLET, ANY, 90000 },	/* Atom Clover Trail/Cloverview (Z27x0) */
 	{ INTEL_ATOM_SALTWELL,	      ANY, 100000 },	/* Atom Cedar Trail/Cedarview (N2xxx, D2xxx)
-							 * Also matches S12x0 (stepping 9), covered by
-							 * PCI table
+							 * Also matches S12x0 (stepping 9), covered
+							 * by PCI table
 							 */
 	{ INTEL_ATOM_SILVERMONT,      9, 110000 },	/* Atom Bay Trail E38xx (embedded) */
 	{ INTEL_ATOM_SILVERMONT,      ANY, 90000 },	/* Atom Bay Trail Z37xx (tablet) */
@@ -201,6 +200,7 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev)
 
 	for (i = 0; i < ARRAY_SIZE(tjmax_model_table); i++) {
 		const struct tjmax_model *tm = &tjmax_model_table[i];
+
 		if (c->x86_vfm == tm->vfm &&
 		    (tm->stepping_mask == ANY ||
 		     tm->stepping_mask == c->x86_stepping))
@@ -223,8 +223,7 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev)
 		err = rdmsr_safe_on_cpu(id, 0x17, &eax, &edx);
 		if (err) {
 			dev_warn(dev,
-				 "Unable to access MSR 0x17, assuming desktop"
-				 " CPU\n");
+				 "Unable to access MSR 0x17, assuming desktop CPU\n");
 			usemsr_ee = 0;
 		} else if (c->x86_vfm < INTEL_CORE2_PENRYN &&
 			   !(eax & 0x10000000)) {
@@ -258,8 +257,7 @@ static int adjust_tjmax(struct cpuinfo_x86 *c, u32 id, struct device *dev)
 		err = rdmsr_safe_on_cpu(id, 0xee, &eax, &edx);
 		if (err) {
 			dev_warn(dev,
-				 "Unable to access MSR 0xEE, for Tjmax, left"
-				 " at default\n");
+				 "Unable to access MSR 0xEE, for Tjmax, left at default\n");
 		} else if (eax & 0x40000000) {
 			tjmax = tjmax_ee;
 		}
@@ -342,7 +340,7 @@ static int max_zones __read_mostly;
 static struct platform_device **zone_devices;
 
 static ssize_t show_label(struct device *dev,
-				struct device_attribute *devattr, char *buf)
+			  struct device_attribute *devattr, char *buf)
 {
 	struct platform_data *pdata = dev_get_drvdata(dev);
 	struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_LABEL]);
@@ -354,7 +352,7 @@ static ssize_t show_label(struct device *dev,
 }
 
 static ssize_t show_crit_alarm(struct device *dev,
-				struct device_attribute *devattr, char *buf)
+			       struct device_attribute *devattr, char *buf)
 {
 	u32 eax, edx;
 	struct temp_data *tdata = container_of(devattr, struct temp_data,
@@ -368,7 +366,7 @@ static ssize_t show_crit_alarm(struct device *dev,
 }
 
 static ssize_t show_tjmax(struct device *dev,
-			struct device_attribute *devattr, char *buf)
+			  struct device_attribute *devattr, char *buf)
 {
 	struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TJMAX]);
 	int tjmax;
@@ -381,7 +379,7 @@ static ssize_t show_tjmax(struct device *dev,
 }
 
 static ssize_t show_ttarget(struct device *dev,
-				struct device_attribute *devattr, char *buf)
+			    struct device_attribute *devattr, char *buf)
 {
 	struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TTARGET]);
 	int ttarget;
@@ -396,7 +394,7 @@ static ssize_t show_ttarget(struct device *dev,
 }
 
 static ssize_t show_temp(struct device *dev,
-			struct device_attribute *devattr, char *buf)
+			 struct device_attribute *devattr, char *buf)
 {
 	u32 eax, edx;
 	struct temp_data *tdata = container_of(devattr, struct temp_data, sd_attrs[ATTR_TEMP]);
@@ -424,7 +422,6 @@ static ssize_t show_temp(struct device *dev,
 
 static int create_core_attrs(struct temp_data *tdata, struct device *dev)
 {
-	int i;
 	static ssize_t (*const rd_ptr[TOTAL_ATTRS]) (struct device *dev,
 			struct device_attribute *devattr, char *buf) = {
 			show_label, show_crit_alarm, show_temp, show_tjmax,
@@ -432,6 +429,7 @@ static int create_core_attrs(struct temp_data *tdata, struct device *dev)
 	static const char *const suffixes[TOTAL_ATTRS] = {
 		"label", "crit_alarm", "input", "crit", "max"
 	};
+	int i;
 
 	for (i = 0; i < tdata->attr_size; i++) {
 		/*
@@ -453,7 +451,6 @@ static int create_core_attrs(struct temp_data *tdata, struct device *dev)
 	return sysfs_create_group(&dev->kobj, &tdata->attr_group);
 }
 
-
 static int chk_ucode_version(unsigned int cpu)
 {
 	struct cpuinfo_x86 *c = &cpu_data(cpu);
@@ -485,13 +482,7 @@ init_temp_data(struct platform_data *pdata, unsigned int cpu, int pkg_flag)
 	struct temp_data *tdata;
 
 	if (!pdata->core_data) {
-		/*
-		 * TODO:
-		 * The information of actual possible cores in a package is broken for now.
-		 * Will replace hardcoded NUM_REAL_CORES with actual per package core count
-		 * when this information becomes available.
-		 */
-		pdata->nr_cores = NUM_REAL_CORES;
+		pdata->nr_cores = topology_num_cores_per_package();
 		pdata->core_data = kzalloc_objs(struct temp_data *,
 						pdata->nr_cores);
 		if (!pdata->core_data)
@@ -781,7 +772,8 @@ static int coretemp_cpu_offline(unsigned int cpu)
 	}
 	return 0;
 }
-static const struct x86_cpu_id __initconst coretemp_ids[] = {
+
+static const struct x86_cpu_id coretemp_ids[] __initconst = {
 	X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_DTHERM, NULL),
 	{}
 };
diff --git a/drivers/hwmon/cros_ec_hwmon.c b/drivers/hwmon/cros_ec_hwmon.c
index 6cf5ab0f4b738..7c308b0a4b9e2 100644
--- a/drivers/hwmon/cros_ec_hwmon.c
+++ b/drivers/hwmon/cros_ec_hwmon.c
@@ -657,6 +657,7 @@ static const struct platform_device_id cros_ec_hwmon_id[] = {
 	{ DRV_NAME, 0 },
 	{}
 };
+MODULE_DEVICE_TABLE(platform, cros_ec_hwmon_id);
 
 static struct platform_driver cros_ec_hwmon_driver = {
 	.driver.name	= DRV_NAME,
@@ -667,7 +668,6 @@ static struct platform_driver cros_ec_hwmon_driver = {
 };
 module_platform_driver(cros_ec_hwmon_driver);
 
-MODULE_DEVICE_TABLE(platform, cros_ec_hwmon_id);
 MODULE_DESCRIPTION("ChromeOS EC Hardware Monitoring Driver");
 MODULE_AUTHOR("Thomas Weißschuh <linux@weissschuh.net");
 MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/dme1737.c b/drivers/hwmon/dme1737.c
index 3d4057309950d..7e839308e58fa 100644
--- a/drivers/hwmon/dme1737.c
+++ b/drivers/hwmon/dme1737.c
@@ -2515,8 +2515,8 @@ static void dme1737_i2c_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id dme1737_id[] = {
-	{ "dme1737", dme1737 },
-	{ "sch5027", sch5027 },
+	{ .name = "dme1737", .driver_data = dme1737 },
+	{ .name = "sch5027", .driver_data = sch5027 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, dme1737_id);
diff --git a/drivers/hwmon/ds1621.c b/drivers/hwmon/ds1621.c
index 42ec34cb8a5f8..0618f6de9679c 100644
--- a/drivers/hwmon/ds1621.c
+++ b/drivers/hwmon/ds1621.c
@@ -367,11 +367,11 @@ static int ds1621_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ds1621_id[] = {
-	{ "ds1621", ds1621 },
-	{ "ds1625", ds1625 },
-	{ "ds1631", ds1631 },
-	{ "ds1721", ds1721 },
-	{ "ds1731", ds1731 },
+	{ .name = "ds1621", .driver_data = ds1621 },
+	{ .name = "ds1625", .driver_data = ds1625 },
+	{ .name = "ds1631", .driver_data = ds1631 },
+	{ .name = "ds1721", .driver_data = ds1721 },
+	{ .name = "ds1731", .driver_data = ds1731 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ds1621_id);
diff --git a/drivers/hwmon/ds620.c b/drivers/hwmon/ds620.c
index ce397042d90b8..25287f0fa4c2e 100644
--- a/drivers/hwmon/ds620.c
+++ b/drivers/hwmon/ds620.c
@@ -233,8 +233,8 @@ static int ds620_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ds620_id[] = {
-	{"ds620"},
-	{}
+	{ .name = "ds620" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, ds620_id);
diff --git a/drivers/hwmon/emc1403.c b/drivers/hwmon/emc1403.c
index 964a8cb278f16..cd753b38709f2 100644
--- a/drivers/hwmon/emc1403.c
+++ b/drivers/hwmon/emc1403.c
@@ -639,18 +639,18 @@ static const struct hwmon_chip_info emc1403_chip_info = {
 
 /* Last digit of chip name indicates number of channels */
 static const struct i2c_device_id emc1403_idtable[] = {
-	{ "emc1402", emc1402 },
-	{ "emc1403", emc1403 },
-	{ "emc1404", emc1404 },
-	{ "emc1412", emc1402 },
-	{ "emc1413", emc1403 },
-	{ "emc1414", emc1404 },
-	{ "emc1422", emc1402 },
-	{ "emc1423", emc1403 },
-	{ "emc1424", emc1404 },
-	{ "emc1428", emc1428 },
-	{ "emc1438", emc1428 },
-	{ "emc1442", emc1402 },
+	{ .name = "emc1402", .driver_data = emc1402 },
+	{ .name = "emc1403", .driver_data = emc1403 },
+	{ .name = "emc1404", .driver_data = emc1404 },
+	{ .name = "emc1412", .driver_data = emc1402 },
+	{ .name = "emc1413", .driver_data = emc1403 },
+	{ .name = "emc1414", .driver_data = emc1404 },
+	{ .name = "emc1422", .driver_data = emc1402 },
+	{ .name = "emc1423", .driver_data = emc1403 },
+	{ .name = "emc1424", .driver_data = emc1404 },
+	{ .name = "emc1428", .driver_data = emc1428 },
+	{ .name = "emc1438", .driver_data = emc1428 },
+	{ .name = "emc1442", .driver_data = emc1402 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, emc1403_idtable);
diff --git a/drivers/hwmon/emc2103.c b/drivers/hwmon/emc2103.c
index 9b8e925af0300..27dc149a3ed99 100644
--- a/drivers/hwmon/emc2103.c
+++ b/drivers/hwmon/emc2103.c
@@ -624,7 +624,7 @@ emc2103_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id emc2103_ids[] = {
-	{ "emc2103" },
+	{ .name = "emc2103" },
 	{ /* LIST END */ }
 };
 MODULE_DEVICE_TABLE(i2c, emc2103_ids);
diff --git a/drivers/hwmon/emc2305.c b/drivers/hwmon/emc2305.c
index 64b213e1451e7..cd7613e5e5475 100644
--- a/drivers/hwmon/emc2305.c
+++ b/drivers/hwmon/emc2305.c
@@ -32,6 +32,7 @@
 #define EMC2305_REG_DRIVE_PWM_OUT	0x2b
 #define EMC2305_OPEN_DRAIN		0x0
 #define EMC2305_PUSH_PULL		0x1
+#define EMC2305_PWM_SHUTDOWN_UNSET      -1
 
 #define EMC2305_PWM_DUTY2STATE(duty, max_state, pwm_max) \
 	DIV_ROUND_CLOSEST((duty) * (max_state), (pwm_max))
@@ -59,10 +60,10 @@ enum emc230x_product_id {
 };
 
 static const struct i2c_device_id emc2305_ids[] = {
-	{ "emc2305" },
-	{ "emc2303" },
-	{ "emc2302" },
-	{ "emc2301" },
+	{ .name = "emc2305" },
+	{ .name = "emc2303" },
+	{ .name = "emc2302" },
+	{ .name = "emc2301" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, emc2305_ids);
@@ -104,6 +105,7 @@ struct emc2305_cdev_data {
  * @pwm_output_mask: PWM output mask
  * @pwm_polarity_mask: PWM polarity mask
  * @pwm_separate: separate PWM settings for every channel
+ * @pwm_shutdown: Set shutdown PWM.
  * @pwm_min: array of minimum PWM per channel
  * @pwm_freq: array of PWM frequency per channel
  * @cdev_data: array of cooling devices data
@@ -116,6 +118,7 @@ struct emc2305_data {
 	u8 pwm_output_mask;
 	u8 pwm_polarity_mask;
 	bool pwm_separate;
+	s16 pwm_shutdown[EMC2305_PWM_MAX];
 	u8 pwm_min[EMC2305_PWM_MAX];
 	u16 pwm_freq[EMC2305_PWM_MAX];
 	struct emc2305_cdev_data cdev_data[EMC2305_PWM_MAX];
@@ -539,6 +542,7 @@ static int emc2305_of_parse_pwm_child(struct device *dev,
 				      struct device_node *child,
 				      struct emc2305_data *data)
 {	u32 ch;
+	u32 pwm_shutdown_percent;
 	int ret;
 	struct of_phandle_args args;
 
@@ -548,6 +552,12 @@ static int emc2305_of_parse_pwm_child(struct device *dev,
 		return ret;
 	}
 
+	if (ch >= data->pwm_num) {
+		dev_err(dev, "invalid reg %u for node %pOF (valid range 0-%u)\n", ch, child,
+			data->pwm_num - 1);
+		return -EINVAL;
+	}
+
 	ret = of_parse_phandle_with_args(child, "pwms", "#pwm-cells", 0, &args);
 
 	if (ret)
@@ -579,6 +589,16 @@ static int emc2305_of_parse_pwm_child(struct device *dev,
 	}
 
 	of_node_put(args.np);
+
+	ret = of_property_read_u32(child, "fan-shutdown-percent",
+				   &pwm_shutdown_percent);
+
+	if (!ret) {
+		pwm_shutdown_percent = clamp(pwm_shutdown_percent, 0, 100);
+		data->pwm_shutdown[ch] =
+			DIV_ROUND_CLOSEST(pwm_shutdown_percent * EMC2305_FAN_MAX, 100);
+	}
+
 	return 0;
 }
 
@@ -612,6 +632,7 @@ static int emc2305_probe(struct i2c_client *client)
 	int ret;
 	int i;
 	int pwm_childs;
+	u32 ch;
 
 	if (!i2c_check_functionality(adapter, I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA))
 		return -ENODEV;
@@ -631,6 +652,9 @@ static int emc2305_probe(struct i2c_client *client)
 	if (ret)
 		return ret;
 
+	for (i = 0; i < EMC2305_PWM_MAX; i++)
+		data->pwm_shutdown[i] = EMC2305_PWM_SHUTDOWN_UNSET;
+
 	pwm_childs = emc2305_probe_childs_from_dt(dev);
 
 	pdata = dev_get_platdata(&client->dev);
@@ -680,12 +704,18 @@ static int emc2305_probe(struct i2c_client *client)
 	if (IS_REACHABLE(CONFIG_THERMAL)) {
 		/* Parse and check for the available PWM child nodes */
 		if (pwm_childs > 0) {
-			i = 0;
 			for_each_child_of_node_scoped(dev->of_node, child) {
-				ret = emc2305_set_single_tz(dev, child, i);
+				ret = of_property_read_u32(child, "reg", &ch);
+				if (ret || ch >= data->pwm_num)
+					continue;
+
+				/*
+				 * emc2305_set_single_tz() uses 0 for the common cooling
+				 * device and 1..pwm_num for individual fan channels.
+				 */
+				ret = emc2305_set_single_tz(dev, child, ch + 1);
 				if (ret != 0)
 					return ret;
-				i++;
 			}
 		} else {
 			ret = emc2305_set_tz(dev);
@@ -714,6 +744,23 @@ static int emc2305_probe(struct i2c_client *client)
 	return 0;
 }
 
+static void emc2305_shutdown(struct i2c_client *client)
+{
+	int i;
+	int ret;
+	struct emc2305_data *data = i2c_get_clientdata(client);
+
+	for (i = 0; i < data->pwm_num; i++) {
+		if (data->pwm_shutdown[i] != EMC2305_PWM_SHUTDOWN_UNSET) {
+			ret = i2c_smbus_write_byte_data(client, EMC2305_REG_FAN_DRIVE(i),
+							data->pwm_shutdown[i]);
+			if (ret < 0)
+				dev_warn(&client->dev,
+					 "Failed to set shutdown PWM for ch %d\n", i);
+		}
+	}
+}
+
 static const struct of_device_id of_emc2305_match_table[] = {
 	{ .compatible = "microchip,emc2305", },
 	{},
@@ -726,6 +773,7 @@ static struct i2c_driver emc2305_driver = {
 		.of_match_table = of_emc2305_match_table,
 	},
 	.probe = emc2305_probe,
+	.shutdown = emc2305_shutdown,
 	.id_table = emc2305_ids,
 };
 
diff --git a/drivers/hwmon/emc6w201.c b/drivers/hwmon/emc6w201.c
index 1100c6e5daa77..c13ea87486835 100644
--- a/drivers/hwmon/emc6w201.c
+++ b/drivers/hwmon/emc6w201.c
@@ -464,7 +464,7 @@ static int emc6w201_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id emc6w201_id[] = {
-	{ "emc6w201" },
+	{ .name = "emc6w201" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, emc6w201_id);
diff --git a/drivers/hwmon/f75375s.c b/drivers/hwmon/f75375s.c
index 7e867f1324201..cc3a9612f2c0a 100644
--- a/drivers/hwmon/f75375s.c
+++ b/drivers/hwmon/f75375s.c
@@ -877,9 +877,9 @@ static int f75375_detect(struct i2c_client *client,
 }
 
 static const struct i2c_device_id f75375_id[] = {
-	{ "f75373", f75373 },
-	{ "f75375", f75375 },
-	{ "f75387", f75387 },
+	{ .name = "f75373", .driver_data = f75373 },
+	{ .name = "f75375", .driver_data = f75375 },
+	{ .name = "f75387", .driver_data = f75387 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, f75375_id);
diff --git a/drivers/hwmon/fam15h_power.c b/drivers/hwmon/fam15h_power.c
index efcbea2d070e4..ad4ed4162b574 100644
--- a/drivers/hwmon/fam15h_power.c
+++ b/drivers/hwmon/fam15h_power.c
@@ -19,6 +19,7 @@
 #include <linux/sched.h>
 #include <linux/topology.h>
 #include <asm/processor.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 
 MODULE_DESCRIPTION("AMD Family 15h CPU processor power monitor");
diff --git a/drivers/hwmon/fschmd.c b/drivers/hwmon/fschmd.c
index 1211fa2259e52..019fc32bf3181 100644
--- a/drivers/hwmon/fschmd.c
+++ b/drivers/hwmon/fschmd.c
@@ -225,13 +225,13 @@ static struct fschmd_data *fschmd_update_device(struct device *dev);
  */
 
 static const struct i2c_device_id fschmd_id[] = {
-	{ "fscpos", fscpos },
-	{ "fscher", fscher },
-	{ "fscscy", fscscy },
-	{ "fschrc", fschrc },
-	{ "fschmd", fschmd },
-	{ "fschds", fschds },
-	{ "fscsyl", fscsyl },
+	{ .name = "fscpos", .driver_data = fscpos },
+	{ .name = "fscher", .driver_data = fscher },
+	{ .name = "fscscy", .driver_data = fscscy },
+	{ .name = "fschrc", .driver_data = fschrc },
+	{ .name = "fschmd", .driver_data = fschmd },
+	{ .name = "fschds", .driver_data = fschds },
+	{ .name = "fscsyl", .driver_data = fscsyl },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, fschmd_id);
diff --git a/drivers/hwmon/ftsteutates.c b/drivers/hwmon/ftsteutates.c
index 08dcc6a7fb625..06be120a349d6 100644
--- a/drivers/hwmon/ftsteutates.c
+++ b/drivers/hwmon/ftsteutates.c
@@ -49,7 +49,7 @@
 static const unsigned short normal_i2c[] = { 0x73, I2C_CLIENT_END };
 
 static const struct i2c_device_id fts_id[] = {
-	{ "ftsteutates" },
+	{ .name = "ftsteutates" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, fts_id);
diff --git a/drivers/hwmon/g760a.c b/drivers/hwmon/g760a.c
index 39ae8f8264176..e2166ee762860 100644
--- a/drivers/hwmon/g760a.c
+++ b/drivers/hwmon/g760a.c
@@ -197,7 +197,7 @@ static int g760a_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id g760a_id[] = {
-	{ "g760a" },
+	{ .name = "g760a" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, g760a_id);
diff --git a/drivers/hwmon/g762.c b/drivers/hwmon/g762.c
index 4fa3aa1271daa..407cf7af4fdd9 100644
--- a/drivers/hwmon/g762.c
+++ b/drivers/hwmon/g762.c
@@ -44,9 +44,9 @@
 #define DRVNAME "g762"
 
 static const struct i2c_device_id g762_id[] = {
-	{ "g761" },
-	{ "g762" },
-	{ "g763" },
+	{ .name = "g761" },
+	{ .name = "g762" },
+	{ .name = "g763" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, g762_id);
diff --git a/drivers/hwmon/gl518sm.c b/drivers/hwmon/gl518sm.c
index 9c68bc0139503..742c130cec7f1 100644
--- a/drivers/hwmon/gl518sm.c
+++ b/drivers/hwmon/gl518sm.c
@@ -642,7 +642,7 @@ static int gl518_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id gl518_id[] = {
-	{ "gl518sm" },
+	{ .name = "gl518sm" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, gl518_id);
diff --git a/drivers/hwmon/gl520sm.c b/drivers/hwmon/gl520sm.c
index 972f4f8caa2b7..f4fe39912ee29 100644
--- a/drivers/hwmon/gl520sm.c
+++ b/drivers/hwmon/gl520sm.c
@@ -885,7 +885,7 @@ static int gl520_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id gl520_id[] = {
-	{ "gl520sm" },
+	{ .name = "gl520sm" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, gl520_id);
diff --git a/drivers/hwmon/hih6130.c b/drivers/hwmon/hih6130.c
index 85af8299150a5..7984be1e706d5 100644
--- a/drivers/hwmon/hih6130.c
+++ b/drivers/hwmon/hih6130.c
@@ -233,7 +233,7 @@ static int hih6130_probe(struct i2c_client *client)
 
 /* Device ID table */
 static const struct i2c_device_id hih6130_id[] = {
-	{ "hih6130" },
+	{ .name = "hih6130" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, hih6130_id);
diff --git a/drivers/hwmon/hs3001.c b/drivers/hwmon/hs3001.c
index 50c6c15f8b180..b263cbeca0747 100644
--- a/drivers/hwmon/hs3001.c
+++ b/drivers/hwmon/hs3001.c
@@ -169,8 +169,8 @@ static const struct hwmon_chip_info hs3001_chip_info = {
 
 /* device ID table */
 static const struct i2c_device_id hs3001_ids[] = {
-	{ "hs3001" },
-	{ },
+	{ .name = "hs3001" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, hs3001_ids);
diff --git a/drivers/hwmon/htu31.c b/drivers/hwmon/htu31.c
index 7521a371aa6cd..5d2cdbdb773b0 100644
--- a/drivers/hwmon/htu31.c
+++ b/drivers/hwmon/htu31.c
@@ -322,7 +322,7 @@ static int htu31_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id htu31_id[] = {
-	{ "htu31" },
+	{ .name = "htu31" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, htu31_id);
diff --git a/drivers/hwmon/hwmon.c b/drivers/hwmon/hwmon.c
index 6812d1fd7c283..c16e6da0690ce 100644
--- a/drivers/hwmon/hwmon.c
+++ b/drivers/hwmon/hwmon.c
@@ -1082,6 +1082,7 @@ EXPORT_SYMBOL_GPL(hwmon_device_register_with_info);
  * @dev: the parent device
  * @name: hwmon name attribute
  * @drvdata: driver data to attach to created device
+ * @extra_groups: pointer to list of additional non-standard attribute groups
  *
  * The use of this function is restricted. It is provided for legacy reasons
  * and must only be called from the thermal subsystem.
@@ -1093,12 +1094,13 @@ EXPORT_SYMBOL_GPL(hwmon_device_register_with_info);
  */
 struct device *
 hwmon_device_register_for_thermal(struct device *dev, const char *name,
-				  void *drvdata)
+				  void *drvdata,
+				  const struct attribute_group **extra_groups)
 {
 	if (!name || !dev)
 		return ERR_PTR(-EINVAL);
 
-	return __hwmon_device_register(dev, name, drvdata, NULL, NULL);
+	return __hwmon_device_register(dev, name, drvdata, NULL, extra_groups);
 }
 EXPORT_SYMBOL_NS_GPL(hwmon_device_register_for_thermal, "HWMON_THERMAL");
 
diff --git a/drivers/hwmon/ina209.c b/drivers/hwmon/ina209.c
index a116f1600e810..39aca2cdf27f7 100644
--- a/drivers/hwmon/ina209.c
+++ b/drivers/hwmon/ina209.c
@@ -569,7 +569,7 @@ static void ina209_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ina209_id[] = {
-	{ "ina209" },
+	{ .name = "ina209" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ina209_id);
diff --git a/drivers/hwmon/ina238.c b/drivers/hwmon/ina238.c
index ff67b03189f73..c20dd164a6a6c 100644
--- a/drivers/hwmon/ina238.c
+++ b/drivers/hwmon/ina238.c
@@ -837,12 +837,12 @@ static int ina238_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ina238_id[] = {
-	{ "ina228", ina228 },
-	{ "ina237", ina237 },
-	{ "ina238", ina238 },
-	{ "ina700", ina700 },
-	{ "ina780", ina780 },
-	{ "sq52206", sq52206 },
+	{ .name = "ina228", .driver_data = ina228 },
+	{ .name = "ina237", .driver_data = ina237 },
+	{ .name = "ina238", .driver_data = ina238 },
+	{ .name = "ina700", .driver_data = ina700 },
+	{ .name = "ina780", .driver_data = ina780 },
+	{ .name = "sq52206", .driver_data = sq52206 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ina238_id);
diff --git a/drivers/hwmon/ina2xx.c b/drivers/hwmon/ina2xx.c
index 613ffb622b7c4..c4742e84b999d 100644
--- a/drivers/hwmon/ina2xx.c
+++ b/drivers/hwmon/ina2xx.c
@@ -875,11 +875,11 @@ static ssize_t shunt_resistor_store(struct device *dev,
 	if (status < 0)
 		return status;
 
-	hwmon_lock(dev);
-	status = ina2xx_set_shunt(data, val);
-	hwmon_unlock(dev);
-	if (status < 0)
-		return status;
+	scoped_guard(hwmon_lock, dev) {
+		status = ina2xx_set_shunt(data, val);
+		if (status < 0)
+			return status;
+	}
 	return count;
 }
 
@@ -1000,14 +1000,14 @@ static int ina2xx_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ina2xx_id[] = {
-	{ "ina219", ina219 },
-	{ "ina220", ina219 },
-	{ "ina226", ina226 },
-	{ "ina230", ina226 },
-	{ "ina231", ina226 },
-	{ "ina234", ina234 },
-	{ "ina260", ina260 },
-	{ "sy24655", sy24655 },
+	{ .name = "ina219", .driver_data = ina219 },
+	{ .name = "ina220", .driver_data = ina219 },
+	{ .name = "ina226", .driver_data = ina226 },
+	{ .name = "ina230", .driver_data = ina226 },
+	{ .name = "ina231", .driver_data = ina226 },
+	{ .name = "ina234", .driver_data = ina234 },
+	{ .name = "ina260", .driver_data = ina260 },
+	{ .name = "sy24655", .driver_data = sy24655 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ina2xx_id);
diff --git a/drivers/hwmon/ina3221.c b/drivers/hwmon/ina3221.c
index 5ecc68dcf1698..3ab5de3a11102 100644
--- a/drivers/hwmon/ina3221.c
+++ b/drivers/hwmon/ina3221.c
@@ -1002,7 +1002,7 @@ static const struct of_device_id ina3221_of_match_table[] = {
 MODULE_DEVICE_TABLE(of, ina3221_of_match_table);
 
 static const struct i2c_device_id ina3221_ids[] = {
-	{ "ina3221" },
+	{ .name = "ina3221" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, ina3221_ids);
diff --git a/drivers/hwmon/intel-m10-bmc-hwmon.c b/drivers/hwmon/intel-m10-bmc-hwmon.c
index aa01a4bedc21b..e85d42a451135 100644
--- a/drivers/hwmon/intel-m10-bmc-hwmon.c
+++ b/drivers/hwmon/intel-m10-bmc-hwmon.c
@@ -773,6 +773,7 @@ static const struct platform_device_id intel_m10bmc_hwmon_ids[] = {
 	},
 	{ }
 };
+MODULE_DEVICE_TABLE(platform, intel_m10bmc_hwmon_ids);
 
 static struct platform_driver intel_m10bmc_hwmon_driver = {
 	.probe = m10bmc_hwmon_probe,
@@ -783,7 +784,6 @@ static struct platform_driver intel_m10bmc_hwmon_driver = {
 };
 module_platform_driver(intel_m10bmc_hwmon_driver);
 
-MODULE_DEVICE_TABLE(platform, intel_m10bmc_hwmon_ids);
 MODULE_AUTHOR("Intel Corporation");
 MODULE_DESCRIPTION("Intel MAX 10 BMC hardware monitor");
 MODULE_LICENSE("GPL");
diff --git a/drivers/hwmon/isl28022.c b/drivers/hwmon/isl28022.c
index 96fcfbfff48f4..0c60d2ef28f5d 100644
--- a/drivers/hwmon/isl28022.c
+++ b/drivers/hwmon/isl28022.c
@@ -475,7 +475,7 @@ static int isl28022_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id isl28022_ids[] = {
-	{ "isl28022" },
+	{ .name = "isl28022" },
 	{ /* LIST END */ }
 };
 MODULE_DEVICE_TABLE(i2c, isl28022_ids);
diff --git a/drivers/hwmon/jc42.c b/drivers/hwmon/jc42.c
index 6549dc5437812..77fece680358d 100644
--- a/drivers/hwmon/jc42.c
+++ b/drivers/hwmon/jc42.c
@@ -579,7 +579,7 @@ static const struct dev_pm_ops jc42_dev_pm_ops = {
 #endif /* CONFIG_PM */
 
 static const struct i2c_device_id jc42_id[] = {
-	{ "jc42" },
+	{ .name = "jc42" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, jc42_id);
diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c
index a5d8f45b78814..de0760dc597da 100644
--- a/drivers/hwmon/k10temp.c
+++ b/drivers/hwmon/k10temp.c
@@ -20,7 +20,9 @@
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/pci_ids.h>
+
 #include <asm/amd/node.h>
+#include <asm/cpuid/api.h>
 #include <asm/processor.h>
 
 MODULE_DESCRIPTION("AMD Family 10h+ CPU core temperature monitor");
diff --git a/drivers/hwmon/k8temp.c b/drivers/hwmon/k8temp.c
index 2b80ac410cd1b..53241164570eb 100644
--- a/drivers/hwmon/k8temp.c
+++ b/drivers/hwmon/k8temp.c
@@ -15,6 +15,7 @@
 #include <linux/err.h>
 #include <linux/mutex.h>
 #include <asm/processor.h>
+#include <asm/cpuid/api.h>
 
 #define TEMP_FROM_REG(val)	(((((val) >> 16) & 0xff) - 49) * 1000)
 #define REG_TEMP	0xe4
diff --git a/drivers/hwmon/lineage-pem.c b/drivers/hwmon/lineage-pem.c
index 64a335a64a2eb..2553e49f84015 100644
--- a/drivers/hwmon/lineage-pem.c
+++ b/drivers/hwmon/lineage-pem.c
@@ -502,8 +502,8 @@ static int pem_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id pem_id[] = {
-	{"lineage_pem"},
-	{}
+	{ .name = "lineage_pem" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pem_id);
 
diff --git a/drivers/hwmon/lm63.c b/drivers/hwmon/lm63.c
index 30500b4d22212..a0d77a7386a9d 100644
--- a/drivers/hwmon/lm63.c
+++ b/drivers/hwmon/lm63.c
@@ -1152,9 +1152,9 @@ static int lm63_probe(struct i2c_client *client)
  */
 
 static const struct i2c_device_id lm63_id[] = {
-	{ "lm63", lm63 },
-	{ "lm64", lm64 },
-	{ "lm96163", lm96163 },
+	{ .name = "lm63", .driver_data = lm63 },
+	{ .name = "lm64", .driver_data = lm64 },
+	{ .name = "lm96163", .driver_data = lm96163 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm63_id);
diff --git a/drivers/hwmon/lm73.c b/drivers/hwmon/lm73.c
index 581b01572e1bd..63ee67481a16c 100644
--- a/drivers/hwmon/lm73.c
+++ b/drivers/hwmon/lm73.c
@@ -220,7 +220,7 @@ lm73_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lm73_ids[] = {
-	{ "lm73" },
+	{ .name = "lm73" },
 	{ /* LIST END */ }
 };
 MODULE_DEVICE_TABLE(i2c, lm73_ids);
diff --git a/drivers/hwmon/lm75.c b/drivers/hwmon/lm75.c
index c283443e363b4..104149a03bad0 100644
--- a/drivers/hwmon/lm75.c
+++ b/drivers/hwmon/lm75.c
@@ -14,7 +14,8 @@
 #include <linux/i3c/device.h>
 #include <linux/hwmon.h>
 #include <linux/err.h>
-#include <linux/of.h>
+#include <linux/mod_devicetable.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/util_macros.h>
 #include <linux/regulator/consumer.h>
@@ -123,7 +124,9 @@ struct lm75_data {
 
 static const u8 lm75_sample_set_masks[] = { 0 << 5, 1 << 5, 2 << 5, 3 << 5 };
 
-#define LM75_SAMPLE_CLEAR_MASK	(3 << 5)
+#define LM75_ALERT_POLARITY_HIGH_8_BIT	(BIT(2))
+#define LM75_ALERT_POLARITY_HIGH_16_BIT	(BIT(2) << 8)
+#define LM75_SAMPLE_CLEAR_MASK		(3 << 5)
 
 /* The structure below stores the configuration values of the supported devices.
  * In case of being supported multiple configurations, the default one must
@@ -539,6 +542,8 @@ static umode_t lm75_is_visible(const void *data, enum hwmon_sensor_types type,
 			if (config_data->params->num_sample_times > 1)
 				return 0644;
 			return 0444;
+		default:
+			break;
 		}
 		break;
 	case hwmon_temp:
@@ -555,6 +560,8 @@ static umode_t lm75_is_visible(const void *data, enum hwmon_sensor_types type,
 			if (config_data->params->alarm)
 				return 0444;
 			break;
+		default:
+			break;
 		}
 		break;
 	default:
@@ -725,6 +732,7 @@ static void lm75_remove(void *data)
 static int lm75_generic_probe(struct device *dev, const char *name,
 			      enum lm75_type kind, int irq, struct regmap *regmap)
 {
+	u16 clr_mask, pol_mask, set_mask;
 	struct device *hwmon_dev;
 	struct lm75_data *data;
 	int status, err;
@@ -762,8 +770,18 @@ static int lm75_generic_probe(struct device *dev, const char *name,
 		return err;
 	data->orig_conf = status;
 
-	err = lm75_write_config(data, data->params->set_mask,
-				data->params->clr_mask);
+	/* Enforce polarity active-low (default) or active-high (devicetree) */
+	if (!data->params->config_reg_16bits)
+		pol_mask = LM75_ALERT_POLARITY_HIGH_8_BIT;
+	else
+		pol_mask = LM75_ALERT_POLARITY_HIGH_16_BIT;
+
+	clr_mask = data->params->clr_mask | pol_mask;
+	set_mask = data->params->set_mask & ~pol_mask;
+	if (device_property_read_bool(dev, "ti,alert-polarity-active-high"))
+		set_mask |= pol_mask;
+
+	err = lm75_write_config(data, set_mask, clr_mask);
 	if (err)
 		return err;
 
@@ -816,37 +834,37 @@ static int lm75_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lm75_i2c_ids[] = {
-	{ "adt75", adt75, },
-	{ "as6200", as6200, },
-	{ "at30ts74", at30ts74, },
-	{ "ds1775", ds1775, },
-	{ "ds75", ds75, },
-	{ "ds7505", ds7505, },
-	{ "g751", g751, },
-	{ "lm75", lm75, },
-	{ "lm75a", lm75a, },
-	{ "lm75b", lm75b, },
-	{ "max6625", max6625, },
-	{ "max6626", max6626, },
-	{ "max31725", max31725, },
-	{ "max31726", max31725, },
-	{ "mcp980x", mcp980x, },
-	{ "p3t1750", p3t1750, },
-	{ "p3t1755", p3t1755, },
-	{ "pct2075", pct2075, },
-	{ "stds75", stds75, },
-	{ "stlm75", stlm75, },
-	{ "tcn75", tcn75, },
-	{ "tmp100", tmp100, },
-	{ "tmp101", tmp101, },
-	{ "tmp105", tmp105, },
-	{ "tmp112", tmp112, },
-	{ "tmp175", tmp175, },
-	{ "tmp275", tmp275, },
-	{ "tmp75", tmp75, },
-	{ "tmp75b", tmp75b, },
-	{ "tmp75c", tmp75c, },
-	{ "tmp1075", tmp1075, },
+	{ .name = "adt75", .driver_data = adt75 },
+	{ .name = "as6200", .driver_data = as6200 },
+	{ .name = "at30ts74", .driver_data = at30ts74 },
+	{ .name = "ds1775", .driver_data = ds1775 },
+	{ .name = "ds75", .driver_data = ds75 },
+	{ .name = "ds7505", .driver_data = ds7505 },
+	{ .name = "g751", .driver_data = g751 },
+	{ .name = "lm75", .driver_data = lm75 },
+	{ .name = "lm75a", .driver_data = lm75a },
+	{ .name = "lm75b", .driver_data = lm75b },
+	{ .name = "max6625", .driver_data = max6625 },
+	{ .name = "max6626", .driver_data = max6626 },
+	{ .name = "max31725", .driver_data = max31725 },
+	{ .name = "max31726", .driver_data = max31725 },
+	{ .name = "mcp980x", .driver_data = mcp980x },
+	{ .name = "p3t1750", .driver_data = p3t1750 },
+	{ .name = "p3t1755", .driver_data = p3t1755 },
+	{ .name = "pct2075", .driver_data = pct2075 },
+	{ .name = "stds75", .driver_data = stds75 },
+	{ .name = "stlm75", .driver_data = stlm75 },
+	{ .name = "tcn75", .driver_data = tcn75 },
+	{ .name = "tmp100", .driver_data = tmp100 },
+	{ .name = "tmp101", .driver_data = tmp101 },
+	{ .name = "tmp105", .driver_data = tmp105 },
+	{ .name = "tmp112", .driver_data = tmp112 },
+	{ .name = "tmp175", .driver_data = tmp175 },
+	{ .name = "tmp275", .driver_data = tmp275 },
+	{ .name = "tmp75", .driver_data = tmp75 },
+	{ .name = "tmp75b", .driver_data = tmp75b },
+	{ .name = "tmp75c", .driver_data = tmp75c },
+	{ .name = "tmp1075", .driver_data = tmp1075 },
 	{ /* LIST END */ }
 };
 MODULE_DEVICE_TABLE(i2c, lm75_i2c_ids);
@@ -882,7 +900,7 @@ static int lm75_i3c_probe(struct i3c_device *i3cdev)
 	return lm75_generic_probe(dev, id_data->name, id_data->type, 0, regmap);
 }
 
-static const struct of_device_id __maybe_unused lm75_of_match[] = {
+static const struct of_device_id lm75_of_match[] = {
 	{
 		.compatible = "adi,adt75",
 		.data = (void *)adt75
@@ -1135,7 +1153,7 @@ static struct i2c_driver lm75_i2c_driver = {
 	.class		= I2C_CLASS_HWMON,
 	.driver = {
 		.name	= "lm75",
-		.of_match_table = of_match_ptr(lm75_of_match),
+		.of_match_table = lm75_of_match,
 		.pm	= LM75_DEV_PM_OPS,
 	},
 	.probe		= lm75_i2c_probe,
diff --git a/drivers/hwmon/lm77.c b/drivers/hwmon/lm77.c
index 80f7a6a3f9a29..96c5c2584d37e 100644
--- a/drivers/hwmon/lm77.c
+++ b/drivers/hwmon/lm77.c
@@ -337,7 +337,7 @@ static int lm77_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lm77_id[] = {
-	{ "lm77" },
+	{ .name = "lm77" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm77_id);
diff --git a/drivers/hwmon/lm78.c b/drivers/hwmon/lm78.c
index 9378a47bf5afa..e4834f9eca6a2 100644
--- a/drivers/hwmon/lm78.c
+++ b/drivers/hwmon/lm78.c
@@ -649,8 +649,8 @@ static int lm78_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lm78_i2c_id[] = {
-	{ "lm78", lm78 },
-	{ "lm79", lm79 },
+	{ .name = "lm78", .driver_data = lm78 },
+	{ .name = "lm79", .driver_data = lm79 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm78_i2c_id);
diff --git a/drivers/hwmon/lm80.c b/drivers/hwmon/lm80.c
index 63c7831bd3e11..94f3dabaaa6f6 100644
--- a/drivers/hwmon/lm80.c
+++ b/drivers/hwmon/lm80.c
@@ -622,8 +622,8 @@ static int lm80_probe(struct i2c_client *client)
  */
 
 static const struct i2c_device_id lm80_id[] = {
-	{ "lm80", 0 },
-	{ "lm96080", 1 },
+	{ .name = "lm80" },
+	{ .name = "lm96080" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm80_id);
diff --git a/drivers/hwmon/lm83.c b/drivers/hwmon/lm83.c
index f800fe2ef18b8..8d49df8c9314f 100644
--- a/drivers/hwmon/lm83.c
+++ b/drivers/hwmon/lm83.c
@@ -443,8 +443,8 @@ static int lm83_probe(struct i2c_client *client)
  */
 
 static const struct i2c_device_id lm83_id[] = {
-	{ "lm83", lm83 },
-	{ "lm82", lm82 },
+	{ .name = "lm83", .driver_data = lm83 },
+	{ .name = "lm82", .driver_data = lm82 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm83_id);
diff --git a/drivers/hwmon/lm85.c b/drivers/hwmon/lm85.c
index 1c244ed75122e..c56e164d61c18 100644
--- a/drivers/hwmon/lm85.c
+++ b/drivers/hwmon/lm85.c
@@ -1618,18 +1618,18 @@ static int lm85_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lm85_id[] = {
-	{ "adm1027", adm1027 },
-	{ "adt7463", adt7463 },
-	{ "adt7468", adt7468 },
-	{ "lm85", lm85 },
-	{ "lm85b", lm85 },
-	{ "lm85c", lm85 },
-	{ "lm96000", lm96000 },
-	{ "emc6d100", emc6d100 },
-	{ "emc6d101", emc6d100 },
-	{ "emc6d102", emc6d102 },
-	{ "emc6d103", emc6d103 },
-	{ "emc6d103s", emc6d103s },
+	{ .name = "adm1027", .driver_data = adm1027 },
+	{ .name = "adt7463", .driver_data = adt7463 },
+	{ .name = "adt7468", .driver_data = adt7468 },
+	{ .name = "lm85", .driver_data = lm85 },
+	{ .name = "lm85b", .driver_data = lm85 },
+	{ .name = "lm85c", .driver_data = lm85 },
+	{ .name = "lm96000", .driver_data = lm96000 },
+	{ .name = "emc6d100", .driver_data = emc6d100 },
+	{ .name = "emc6d101", .driver_data = emc6d100 },
+	{ .name = "emc6d102", .driver_data = emc6d102 },
+	{ .name = "emc6d103", .driver_data = emc6d103 },
+	{ .name = "emc6d103s", .driver_data = emc6d103s },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm85_id);
diff --git a/drivers/hwmon/lm87.c b/drivers/hwmon/lm87.c
index 37bf2d1d3d099..c09074f447083 100644
--- a/drivers/hwmon/lm87.c
+++ b/drivers/hwmon/lm87.c
@@ -981,8 +981,8 @@ static int lm87_probe(struct i2c_client *client)
  */
 
 static const struct i2c_device_id lm87_id[] = {
-	{ "lm87" },
-	{ "adm1024" },
+	{ .name = "lm87" },
+	{ .name = "adm1024" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm87_id);
diff --git a/drivers/hwmon/lm90.c b/drivers/hwmon/lm90.c
index 1eeb608e59039..4b9c0ccdf2609 100644
--- a/drivers/hwmon/lm90.c
+++ b/drivers/hwmon/lm90.c
@@ -243,54 +243,54 @@ enum chips { adm1023, adm1032, adt7461, adt7461a, adt7481,
  */
 
 static const struct i2c_device_id lm90_id[] = {
-	{ "adm1020", max1617 },
-	{ "adm1021", max1617 },
-	{ "adm1023", adm1023 },
-	{ "adm1032", adm1032 },
-	{ "adt7421", adt7461a },
-	{ "adt7461", adt7461 },
-	{ "adt7461a", adt7461a },
-	{ "adt7481", adt7481 },
-	{ "adt7482", adt7481 },
-	{ "adt7483a", adt7481 },
-	{ "g781", g781 },
-	{ "gl523sm", max1617 },
-	{ "lm84", lm84 },
-	{ "lm86", lm90 },
-	{ "lm89", lm90 },
-	{ "lm90", lm90 },
-	{ "lm99", lm99 },
-	{ "max1617", max1617 },
-	{ "max6642", max6642 },
-	{ "max6646", max6646 },
-	{ "max6647", max6646 },
-	{ "max6648", max6648 },
-	{ "max6649", max6646 },
-	{ "max6654", max6654 },
-	{ "max6657", max6657 },
-	{ "max6658", max6657 },
-	{ "max6659", max6659 },
-	{ "max6680", max6680 },
-	{ "max6681", max6680 },
-	{ "max6690", max6654 },
-	{ "max6692", max6648 },
-	{ "max6695", max6696 },
-	{ "max6696", max6696 },
-	{ "mc1066", max1617 },
-	{ "nct1008", adt7461a },
-	{ "nct210", nct210 },
-	{ "nct214", nct72 },
-	{ "nct218", nct72 },
-	{ "nct72", nct72 },
-	{ "nct7716", nct7716 },
-	{ "nct7717", nct7717 },
-	{ "nct7718", nct7718 },
-	{ "ne1618", ne1618 },
-	{ "w83l771", w83l771 },
-	{ "sa56004", sa56004 },
-	{ "thmc10", max1617 },
-	{ "tmp451", tmp451 },
-	{ "tmp461", tmp461 },
+	{ .name = "adm1020", .driver_data = max1617 },
+	{ .name = "adm1021", .driver_data = max1617 },
+	{ .name = "adm1023", .driver_data = adm1023 },
+	{ .name = "adm1032", .driver_data = adm1032 },
+	{ .name = "adt7421", .driver_data = adt7461a },
+	{ .name = "adt7461", .driver_data = adt7461 },
+	{ .name = "adt7461a", .driver_data = adt7461a },
+	{ .name = "adt7481", .driver_data = adt7481 },
+	{ .name = "adt7482", .driver_data = adt7481 },
+	{ .name = "adt7483a", .driver_data = adt7481 },
+	{ .name = "g781", .driver_data = g781 },
+	{ .name = "gl523sm", .driver_data = max1617 },
+	{ .name = "lm84", .driver_data = lm84 },
+	{ .name = "lm86", .driver_data = lm90 },
+	{ .name = "lm89", .driver_data = lm90 },
+	{ .name = "lm90", .driver_data = lm90 },
+	{ .name = "lm99", .driver_data = lm99 },
+	{ .name = "max1617", .driver_data = max1617 },
+	{ .name = "max6642", .driver_data = max6642 },
+	{ .name = "max6646", .driver_data = max6646 },
+	{ .name = "max6647", .driver_data = max6646 },
+	{ .name = "max6648", .driver_data = max6648 },
+	{ .name = "max6649", .driver_data = max6646 },
+	{ .name = "max6654", .driver_data = max6654 },
+	{ .name = "max6657", .driver_data = max6657 },
+	{ .name = "max6658", .driver_data = max6657 },
+	{ .name = "max6659", .driver_data = max6659 },
+	{ .name = "max6680", .driver_data = max6680 },
+	{ .name = "max6681", .driver_data = max6680 },
+	{ .name = "max6690", .driver_data = max6654 },
+	{ .name = "max6692", .driver_data = max6648 },
+	{ .name = "max6695", .driver_data = max6696 },
+	{ .name = "max6696", .driver_data = max6696 },
+	{ .name = "mc1066", .driver_data = max1617 },
+	{ .name = "nct1008", .driver_data = adt7461a },
+	{ .name = "nct210", .driver_data = nct210 },
+	{ .name = "nct214", .driver_data = nct72 },
+	{ .name = "nct218", .driver_data = nct72 },
+	{ .name = "nct72", .driver_data = nct72 },
+	{ .name = "nct7716", .driver_data = nct7716 },
+	{ .name = "nct7717", .driver_data = nct7717 },
+	{ .name = "nct7718", .driver_data = nct7718 },
+	{ .name = "ne1618", .driver_data = ne1618 },
+	{ .name = "w83l771", .driver_data = w83l771 },
+	{ .name = "sa56004", .driver_data = sa56004 },
+	{ .name = "thmc10", .driver_data = max1617 },
+	{ .name = "tmp451", .driver_data = tmp451 },
+	{ .name = "tmp461", .driver_data = tmp461 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm90_id);
@@ -1226,13 +1226,8 @@ static int lm90_update_alarms_locked(struct lm90_data *data, bool force)
 
 static int lm90_update_alarms(struct lm90_data *data, bool force)
 {
-	int err;
-
-	hwmon_lock(data->hwmon_dev);
-	err = lm90_update_alarms_locked(data, force);
-	hwmon_unlock(data->hwmon_dev);
-
-	return err;
+	guard(hwmon_lock)(data->hwmon_dev);
+	return lm90_update_alarms_locked(data, force);
 }
 
 static void lm90_alert_work(struct work_struct *__work)
@@ -2598,9 +2593,9 @@ static void lm90_stop_work(void *_data)
 {
 	struct lm90_data *data = _data;
 
-	hwmon_lock(data->hwmon_dev);
-	data->shutdown = true;
-	hwmon_unlock(data->hwmon_dev);
+	scoped_guard(hwmon_lock, data->hwmon_dev) {
+		data->shutdown = true;
+	}
 	cancel_delayed_work_sync(&data->alert_work);
 	cancel_work_sync(&data->report_work);
 }
@@ -2946,17 +2941,17 @@ static void lm90_alert(struct i2c_client *client, enum i2c_alert_protocol type,
 		 */
 		struct lm90_data *data = i2c_get_clientdata(client);
 
-		hwmon_lock(data->hwmon_dev);
-		if (!data->shutdown && (data->flags & LM90_HAVE_BROKEN_ALERT) &&
-		    (data->current_alarms & data->alert_alarms)) {
-			if (!(data->config & 0x80)) {
-				dev_dbg(&client->dev, "Disabling ALERT#\n");
-				lm90_update_confreg(data, data->config | 0x80);
+		scoped_guard(hwmon_lock, data->hwmon_dev) {
+			if (!data->shutdown && (data->flags & LM90_HAVE_BROKEN_ALERT) &&
+			    (data->current_alarms & data->alert_alarms)) {
+				if (!(data->config & 0x80)) {
+					dev_dbg(&client->dev, "Disabling ALERT#\n");
+					lm90_update_confreg(data, data->config | 0x80);
+				}
+				schedule_delayed_work(&data->alert_work,
+					max_t(int, HZ, msecs_to_jiffies(data->update_interval)));
 			}
-			schedule_delayed_work(&data->alert_work,
-				max_t(int, HZ, msecs_to_jiffies(data->update_interval)));
 		}
-		hwmon_unlock(data->hwmon_dev);
 	} else {
 		dev_dbg(&client->dev, "Everything OK\n");
 	}
diff --git a/drivers/hwmon/lm92.c b/drivers/hwmon/lm92.c
index 91a6b7525bb67..4aadbabc27bb9 100644
--- a/drivers/hwmon/lm92.c
+++ b/drivers/hwmon/lm92.c
@@ -405,10 +405,10 @@ static int lm92_probe(struct i2c_client *client)
  * Module and driver stuff
  */
 
-/* .driver_data is limit register resolution */ 
+/* .driver_data is limit register resolution */
 static const struct i2c_device_id lm92_id[] = {
-	{ "lm92", 13 },
-	{ "max6635", 9 },
+	{ .name = "lm92", .driver_data = 13 },
+	{ .name = "max6635", .driver_data = 9 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm92_id);
diff --git a/drivers/hwmon/lm93.c b/drivers/hwmon/lm93.c
index be4853fad80fd..d58a3c2a85930 100644
--- a/drivers/hwmon/lm93.c
+++ b/drivers/hwmon/lm93.c
@@ -2624,8 +2624,8 @@ static int lm93_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lm93_id[] = {
-	{ "lm93" },
-	{ "lm94" },
+	{ .name = "lm93" },
+	{ .name = "lm94" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm93_id);
diff --git a/drivers/hwmon/lm95234.c b/drivers/hwmon/lm95234.c
index 387b3ba81dbfc..74f280b90e3e9 100644
--- a/drivers/hwmon/lm95234.c
+++ b/drivers/hwmon/lm95234.c
@@ -532,8 +532,8 @@ static int lm95234_probe(struct i2c_client *client)
 
 /* Driver data (common to all clients) */
 static const struct i2c_device_id lm95234_id[] = {
-	{ "lm95233", lm95233 },
-	{ "lm95234", lm95234 },
+	{ .name = "lm95233", .driver_data = lm95233 },
+	{ .name = "lm95234", .driver_data = lm95234 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm95234_id);
diff --git a/drivers/hwmon/lm95241.c b/drivers/hwmon/lm95241.c
index 456381b0938e7..0cb0edb3845db 100644
--- a/drivers/hwmon/lm95241.c
+++ b/drivers/hwmon/lm95241.c
@@ -441,8 +441,8 @@ static int lm95241_probe(struct i2c_client *client)
 
 /* Driver data (common to all clients) */
 static const struct i2c_device_id lm95241_id[] = {
-	{ "lm95231" },
-	{ "lm95241" },
+	{ .name = "lm95231" },
+	{ .name = "lm95241" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm95241_id);
diff --git a/drivers/hwmon/lm95245.c b/drivers/hwmon/lm95245.c
index 9ed300c6b5f70..11553391f54b1 100644
--- a/drivers/hwmon/lm95245.c
+++ b/drivers/hwmon/lm95245.c
@@ -546,8 +546,8 @@ static int lm95245_probe(struct i2c_client *client)
 
 /* Driver data (common to all clients) */
 static const struct i2c_device_id lm95245_id[] = {
-	{ "lm95235" },
-	{ "lm95245" },
+	{ .name = "lm95235" },
+	{ .name = "lm95245" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm95245_id);
diff --git a/drivers/hwmon/ltc2945.c b/drivers/hwmon/ltc2945.c
index 3e0e0e0687bdd..b521100afe53a 100644
--- a/drivers/hwmon/ltc2945.c
+++ b/drivers/hwmon/ltc2945.c
@@ -508,7 +508,7 @@ static int ltc2945_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ltc2945_id[] = {
-	{"ltc2945"},
+	{ .name = "ltc2945" },
 	{ }
 };
 
diff --git a/drivers/hwmon/ltc2947-i2c.c b/drivers/hwmon/ltc2947-i2c.c
index 176d710706dd9..e07d33983d5ce 100644
--- a/drivers/hwmon/ltc2947-i2c.c
+++ b/drivers/hwmon/ltc2947-i2c.c
@@ -27,8 +27,8 @@ static int ltc2947_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id ltc2947_id[] = {
-	{"ltc2947"},
-	{}
+	{ .name = "ltc2947" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc2947_id);
 
diff --git a/drivers/hwmon/ltc2990.c b/drivers/hwmon/ltc2990.c
index f1c1933c52cf4..a2725e4b2f213 100644
--- a/drivers/hwmon/ltc2990.c
+++ b/drivers/hwmon/ltc2990.c
@@ -259,8 +259,8 @@ static int ltc2990_i2c_probe(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id ltc2990_i2c_id[] = {
-	{ "ltc2990" },
-	{}
+	{ .name = "ltc2990" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc2990_i2c_id);
 
diff --git a/drivers/hwmon/ltc2991.c b/drivers/hwmon/ltc2991.c
index 6d5d4cb846daf..bc8d803e8cc90 100644
--- a/drivers/hwmon/ltc2991.c
+++ b/drivers/hwmon/ltc2991.c
@@ -409,8 +409,8 @@ static const struct of_device_id ltc2991_of_match[] = {
 MODULE_DEVICE_TABLE(of, ltc2991_of_match);
 
 static const struct i2c_device_id ltc2991_i2c_id[] = {
-	{ "ltc2991" },
-	{}
+	{ .name = "ltc2991" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc2991_i2c_id);
 
diff --git a/drivers/hwmon/ltc2992.c b/drivers/hwmon/ltc2992.c
index 2617c4538af91..43b6029c08404 100644
--- a/drivers/hwmon/ltc2992.c
+++ b/drivers/hwmon/ltc2992.c
@@ -948,8 +948,8 @@ static const struct of_device_id ltc2992_of_match[] = {
 MODULE_DEVICE_TABLE(of, ltc2992_of_match);
 
 static const struct i2c_device_id ltc2992_i2c_id[] = {
-	{"ltc2992"},
-	{}
+	{ .name = "ltc2992" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc2992_i2c_id);
 
diff --git a/drivers/hwmon/ltc4151.c b/drivers/hwmon/ltc4151.c
index fa66eda78efe4..fa7c57aae5f5f 100644
--- a/drivers/hwmon/ltc4151.c
+++ b/drivers/hwmon/ltc4151.c
@@ -188,7 +188,7 @@ static int ltc4151_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ltc4151_id[] = {
-	{ "ltc4151" },
+	{ .name = "ltc4151" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc4151_id);
diff --git a/drivers/hwmon/ltc4215.c b/drivers/hwmon/ltc4215.c
index cce452711cec4..3d439fbbbef93 100644
--- a/drivers/hwmon/ltc4215.c
+++ b/drivers/hwmon/ltc4215.c
@@ -245,7 +245,7 @@ static int ltc4215_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ltc4215_id[] = {
-	{ "ltc4215" },
+	{ .name = "ltc4215" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc4215_id);
diff --git a/drivers/hwmon/ltc4222.c b/drivers/hwmon/ltc4222.c
index f7eb007fd766b..4fc69be5f2bc9 100644
--- a/drivers/hwmon/ltc4222.c
+++ b/drivers/hwmon/ltc4222.c
@@ -200,7 +200,7 @@ static int ltc4222_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ltc4222_id[] = {
-	{"ltc4222"},
+	{ .name = "ltc4222" },
 	{ }
 };
 
diff --git a/drivers/hwmon/ltc4245.c b/drivers/hwmon/ltc4245.c
index e8131a48bda7f..d87aa9c32c873 100644
--- a/drivers/hwmon/ltc4245.c
+++ b/drivers/hwmon/ltc4245.c
@@ -461,7 +461,7 @@ static int ltc4245_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ltc4245_id[] = {
-	{ "ltc4245" },
+	{ .name = "ltc4245" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc4245_id);
diff --git a/drivers/hwmon/ltc4260.c b/drivers/hwmon/ltc4260.c
index 9750dc9aa336d..37a85125d619c 100644
--- a/drivers/hwmon/ltc4260.c
+++ b/drivers/hwmon/ltc4260.c
@@ -163,7 +163,7 @@ static int ltc4260_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ltc4260_id[] = {
-	{"ltc4260"},
+	{ .name = "ltc4260" },
 	{ }
 };
 
diff --git a/drivers/hwmon/ltc4261.c b/drivers/hwmon/ltc4261.c
index 2cd218a6a3be7..a2e52ca0b06e1 100644
--- a/drivers/hwmon/ltc4261.c
+++ b/drivers/hwmon/ltc4261.c
@@ -222,8 +222,8 @@ static int ltc4261_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ltc4261_id[] = {
-	{"ltc4261"},
-	{}
+	{ .name = "ltc4261" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, ltc4261_id);
diff --git a/drivers/hwmon/max127.c b/drivers/hwmon/max127.c
index 5102d86d2619a..b294e86d52d16 100644
--- a/drivers/hwmon/max127.c
+++ b/drivers/hwmon/max127.c
@@ -312,7 +312,7 @@ static int max127_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max127_id[] = {
-	{ "max127" },
+	{ .name = "max127" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max127_id);
diff --git a/drivers/hwmon/max16065.c b/drivers/hwmon/max16065.c
index 43fbb9b26b102..f8b4217542203 100644
--- a/drivers/hwmon/max16065.c
+++ b/drivers/hwmon/max16065.c
@@ -592,12 +592,12 @@ static int max16065_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max16065_id[] = {
-	{ "max16065", max16065 },
-	{ "max16066", max16066 },
-	{ "max16067", max16067 },
-	{ "max16068", max16068 },
-	{ "max16070", max16070 },
-	{ "max16071", max16071 },
+	{ .name = "max16065", .driver_data = max16065 },
+	{ .name = "max16066", .driver_data = max16066 },
+	{ .name = "max16067", .driver_data = max16067 },
+	{ .name = "max16068", .driver_data = max16068 },
+	{ .name = "max16070", .driver_data = max16070 },
+	{ .name = "max16071", .driver_data = max16071 },
 	{ }
 };
 
diff --git a/drivers/hwmon/max1619.c b/drivers/hwmon/max1619.c
index 9b6d03cff4df5..77ef39e8ae7e3 100644
--- a/drivers/hwmon/max1619.c
+++ b/drivers/hwmon/max1619.c
@@ -366,7 +366,7 @@ static int max1619_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max1619_id[] = {
-	{ "max1619" },
+	{ .name = "max1619" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max1619_id);
diff --git a/drivers/hwmon/max1668.c b/drivers/hwmon/max1668.c
index a8197a86f5590..32548f56409ce 100644
--- a/drivers/hwmon/max1668.c
+++ b/drivers/hwmon/max1668.c
@@ -293,9 +293,9 @@ static int max1668_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max1668_id[] = {
-	{ "max1668", 5 },
-	{ "max1805", 3 },
-	{ "max1989", 5 },
+	{ .name = "max1668", .driver_data = 5 },
+	{ .name = "max1805", .driver_data = 3 },
+	{ .name = "max1989", .driver_data = 5 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max1668_id);
diff --git a/drivers/hwmon/max31730.c b/drivers/hwmon/max31730.c
index 2f4b419b6c9e3..d132be2d6487d 100644
--- a/drivers/hwmon/max31730.c
+++ b/drivers/hwmon/max31730.c
@@ -345,7 +345,7 @@ max31730_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max31730_ids[] = {
-	{ "max31730" },
+	{ .name = "max31730" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max31730_ids);
diff --git a/drivers/hwmon/max31760.c b/drivers/hwmon/max31760.c
index 127e31ca3c873..8d3be064b0251 100644
--- a/drivers/hwmon/max31760.c
+++ b/drivers/hwmon/max31760.c
@@ -555,7 +555,7 @@ static const struct of_device_id max31760_of_match[] = {
 MODULE_DEVICE_TABLE(of, max31760_of_match);
 
 static const struct i2c_device_id max31760_id[] = {
-	{"max31760"},
+	{ .name = "max31760" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max31760_id);
diff --git a/drivers/hwmon/max31790.c b/drivers/hwmon/max31790.c
index 4f6171a17d9f1..db8a0f6f18294 100644
--- a/drivers/hwmon/max31790.c
+++ b/drivers/hwmon/max31790.c
@@ -517,7 +517,7 @@ static int max31790_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max31790_id[] = {
-	{ "max31790" },
+	{ .name = "max31790" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max31790_id);
diff --git a/drivers/hwmon/max31827.c b/drivers/hwmon/max31827.c
index 9b2e56c040df1..6a7048fdd8ab7 100644
--- a/drivers/hwmon/max31827.c
+++ b/drivers/hwmon/max31827.c
@@ -463,9 +463,9 @@ static struct attribute *max31827_attrs[] = {
 ATTRIBUTE_GROUPS(max31827);
 
 static const struct i2c_device_id max31827_i2c_ids[] = {
-	{ "max31827", max31827 },
-	{ "max31828", max31828 },
-	{ "max31829", max31829 },
+	{ .name = "max31827", .driver_data = max31827 },
+	{ .name = "max31828", .driver_data = max31828 },
+	{ .name = "max31829", .driver_data = max31829 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max31827_i2c_ids);
diff --git a/drivers/hwmon/max6620.c b/drivers/hwmon/max6620.c
index 4316dcdd03fc2..100acf357b5f5 100644
--- a/drivers/hwmon/max6620.c
+++ b/drivers/hwmon/max6620.c
@@ -474,7 +474,7 @@ static int max6620_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max6620_id[] = {
-	{ "max6620" },
+	{ .name = "max6620" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max6620_id);
diff --git a/drivers/hwmon/max6621.c b/drivers/hwmon/max6621.c
index a7066f3a0bb4e..e86ec6d237ca6 100644
--- a/drivers/hwmon/max6621.c
+++ b/drivers/hwmon/max6621.c
@@ -537,7 +537,7 @@ static int max6621_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max6621_id[] = {
-	{ MAX6621_DRV_NAME },
+	{ .name = MAX6621_DRV_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max6621_id);
diff --git a/drivers/hwmon/max6639.c b/drivers/hwmon/max6639.c
index 163d31f17bd49..dd5f4b3b128da 100644
--- a/drivers/hwmon/max6639.c
+++ b/drivers/hwmon/max6639.c
@@ -778,7 +778,7 @@ static int max6639_resume(struct device *dev)
 }
 
 static const struct i2c_device_id max6639_id[] = {
-	{"max6639"},
+	{ .name = "max6639" },
 	{ }
 };
 
diff --git a/drivers/hwmon/max6650.c b/drivers/hwmon/max6650.c
index 56b8157885bbf..f0140ef9c40ee 100644
--- a/drivers/hwmon/max6650.c
+++ b/drivers/hwmon/max6650.c
@@ -807,8 +807,8 @@ static int max6650_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max6650_id[] = {
-	{ "max6650", 1 },
-	{ "max6651", 4 },
+	{ .name = "max6650", .driver_data = 1 },
+	{ .name = "max6651", .driver_data = 4 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max6650_id);
diff --git a/drivers/hwmon/max6697.c b/drivers/hwmon/max6697.c
index dd906cf491cae..c864093f015c0 100644
--- a/drivers/hwmon/max6697.c
+++ b/drivers/hwmon/max6697.c
@@ -564,16 +564,16 @@ static int max6697_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max6697_id[] = {
-	{ "max6581", max6581 },
-	{ "max6602", max6602 },
-	{ "max6622", max6622 },
-	{ "max6636", max6636 },
-	{ "max6689", max6689 },
-	{ "max6693", max6693 },
-	{ "max6694", max6694 },
-	{ "max6697", max6697 },
-	{ "max6698", max6698 },
-	{ "max6699", max6699 },
+	{ .name = "max6581", .driver_data = max6581 },
+	{ .name = "max6602", .driver_data = max6602 },
+	{ .name = "max6622", .driver_data = max6622 },
+	{ .name = "max6636", .driver_data = max6636 },
+	{ .name = "max6689", .driver_data = max6689 },
+	{ .name = "max6693", .driver_data = max6693 },
+	{ .name = "max6694", .driver_data = max6694 },
+	{ .name = "max6697", .driver_data = max6697 },
+	{ .name = "max6698", .driver_data = max6698 },
+	{ .name = "max6699", .driver_data = max6699 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max6697_id);
diff --git a/drivers/hwmon/mc34vr500.c b/drivers/hwmon/mc34vr500.c
index 84458e4533d86..8cb9d5c09033f 100644
--- a/drivers/hwmon/mc34vr500.c
+++ b/drivers/hwmon/mc34vr500.c
@@ -235,8 +235,8 @@ static int mc34vr500_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mc34vr500_id[] = {
-	{ "mc34vr500" },
-	{ },
+	{ .name = "mc34vr500" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mc34vr500_id);
 
diff --git a/drivers/hwmon/mcp3021.c b/drivers/hwmon/mcp3021.c
index bcddf6804d3ab..d6b0ebf27941f 100644
--- a/drivers/hwmon/mcp3021.c
+++ b/drivers/hwmon/mcp3021.c
@@ -177,8 +177,8 @@ static int mcp3021_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mcp3021_id[] = {
-	{ "mcp3021", mcp3021 },
-	{ "mcp3221", mcp3221 },
+	{ .name = "mcp3021", .driver_data = mcp3021 },
+	{ .name = "mcp3221", .driver_data = mcp3221 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mcp3021_id);
diff --git a/drivers/hwmon/mcp9982.c b/drivers/hwmon/mcp9982.c
index 26c69e3388ab8..9e19e2697e25f 100644
--- a/drivers/hwmon/mcp9982.c
+++ b/drivers/hwmon/mcp9982.c
@@ -92,19 +92,19 @@ static const struct hwmon_channel_info * const mcp9985_info[] = {
 			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_MIN |
 			   HWMON_T_MIN_ALARM | HWMON_T_MAX | HWMON_T_MAX_ALARM |
 			   HWMON_T_MAX_HYST | HWMON_T_CRIT | HWMON_T_CRIT_ALARM |
-			   HWMON_T_CRIT_HYST,
+			   HWMON_T_CRIT_HYST | HWMON_T_FAULT,
 			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_MIN |
 			   HWMON_T_MIN_ALARM | HWMON_T_MAX | HWMON_T_MAX_ALARM |
 			   HWMON_T_MAX_HYST | HWMON_T_CRIT | HWMON_T_CRIT_ALARM |
-			   HWMON_T_CRIT_HYST,
+			   HWMON_T_CRIT_HYST | HWMON_T_FAULT,
 			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_MIN |
 			   HWMON_T_MIN_ALARM | HWMON_T_MAX | HWMON_T_MAX_ALARM |
 			   HWMON_T_MAX_HYST | HWMON_T_CRIT | HWMON_T_CRIT_ALARM |
-			   HWMON_T_CRIT_HYST,
+			   HWMON_T_CRIT_HYST | HWMON_T_FAULT,
 			   HWMON_T_INPUT | HWMON_T_LABEL | HWMON_T_MIN |
 			   HWMON_T_MIN_ALARM | HWMON_T_MAX | HWMON_T_MAX_ALARM |
 			   HWMON_T_MAX_HYST | HWMON_T_CRIT | HWMON_T_CRIT_ALARM |
-			   HWMON_T_CRIT_HYST),
+			   HWMON_T_CRIT_HYST | HWMON_T_FAULT),
 	HWMON_CHANNEL_INFO(chip,
 			   HWMON_C_UPDATE_INTERVAL),
 	NULL
@@ -369,7 +369,8 @@ static int mcp9982_read(struct device *dev, enum hwmon_sensor_types type, u32 at
 
 	/*
 	 * In Standby State the conversion cycle must be initated manually in
-	 * order to read fresh temperature values and the status of the alarms.
+	 * order to read fresh temperature values, the status of the alarms and
+	 * fault information.
 	 */
 	if (!priv->run_state) {
 		switch (type) {
@@ -379,6 +380,7 @@ static int mcp9982_read(struct device *dev, enum hwmon_sensor_types type, u32 at
 			case hwmon_temp_max_alarm:
 			case hwmon_temp_min_alarm:
 			case hwmon_temp_crit_alarm:
+			case hwmon_temp_fault:
 				ret = regmap_write(priv->regmap, MCP9982_ONE_SHOT_ADDR, 1);
 				if (ret)
 					return ret;
@@ -402,6 +404,11 @@ static int mcp9982_read(struct device *dev, enum hwmon_sensor_types type, u32 at
 	}
 
 	switch (type) {
+	/*
+	 * Because the ALERT/THERM pin is set in Therm(Comparator) mode,
+	 * the external diode fault status, high limit status and low
+	 * limit status registers do not clear the bits after reading.
+	 */
 	case hwmon_temp:
 		switch (attr) {
 		case hwmon_temp_input:
@@ -510,6 +517,13 @@ static int mcp9982_read(struct device *dev, enum hwmon_sensor_types type, u32 at
 
 			*val -= hyst * 1000;
 
+			return 0;
+		case hwmon_temp_fault:
+			*val = regmap_test_bits(priv->regmap, MCP9982_EXT_FAULT_STATUS_ADDR,
+						BIT(channel));
+			if (*val < 0)
+				return *val;
+
 			return 0;
 		default:
 			return -EINVAL;
@@ -681,6 +695,7 @@ static umode_t mcp9982_is_visible(const void *_data, enum hwmon_sensor_types typ
 		case hwmon_temp_max_alarm:
 		case hwmon_temp_max_hyst:
 		case hwmon_temp_crit_alarm:
+		case hwmon_temp_fault:
 			return 0444;
 		case hwmon_temp_min:
 		case hwmon_temp_max:
diff --git a/drivers/hwmon/nct6775-i2c.c b/drivers/hwmon/nct6775-i2c.c
index ba71d776a2911..07783910c0588 100644
--- a/drivers/hwmon/nct6775-i2c.c
+++ b/drivers/hwmon/nct6775-i2c.c
@@ -93,19 +93,19 @@ static const struct of_device_id __maybe_unused nct6775_i2c_of_match[] = {
 MODULE_DEVICE_TABLE(of, nct6775_i2c_of_match);
 
 static const struct i2c_device_id nct6775_i2c_id[] = {
-	{ "nct6106", nct6106 },
-	{ "nct6116", nct6116 },
-	{ "nct6775", nct6775 },
-	{ "nct6776", nct6776 },
-	{ "nct6779", nct6779 },
-	{ "nct6791", nct6791 },
-	{ "nct6792", nct6792 },
-	{ "nct6793", nct6793 },
-	{ "nct6795", nct6795 },
-	{ "nct6796", nct6796 },
-	{ "nct6797", nct6797 },
-	{ "nct6798", nct6798 },
-	{ "nct6799", nct6799 },
+	{ .name = "nct6106", .driver_data = nct6106 },
+	{ .name = "nct6116", .driver_data = nct6116 },
+	{ .name = "nct6775", .driver_data = nct6775 },
+	{ .name = "nct6776", .driver_data = nct6776 },
+	{ .name = "nct6779", .driver_data = nct6779 },
+	{ .name = "nct6791", .driver_data = nct6791 },
+	{ .name = "nct6792", .driver_data = nct6792 },
+	{ .name = "nct6793", .driver_data = nct6793 },
+	{ .name = "nct6795", .driver_data = nct6795 },
+	{ .name = "nct6796", .driver_data = nct6796 },
+	{ .name = "nct6797", .driver_data = nct6797 },
+	{ .name = "nct6798", .driver_data = nct6798 },
+	{ .name = "nct6799", .driver_data = nct6799 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, nct6775_i2c_id);
diff --git a/drivers/hwmon/nct7802.c b/drivers/hwmon/nct7802.c
index 8c9351da12c6e..10d1a41fe2119 100644
--- a/drivers/hwmon/nct7802.c
+++ b/drivers/hwmon/nct7802.c
@@ -47,6 +47,8 @@ static const u8 REG_VOLTAGE_LIMIT_MSB_SHIFT[2][5] = {
 #define REG_PWM(x)		(0x60 + (x))
 #define REG_SMARTFAN_EN(x)      (0x64 + (x) / 2)
 #define SMARTFAN_EN_SHIFT(x)    ((x) % 2 * 4)
+#define REG_SMARTFAN_STEP_UP_TIME	0x6e
+#define REG_SMARTFAN_STEP_DOWN_TIME	0x6f
 #define REG_VENDOR_ID		0xfd
 #define REG_CHIP_ID		0xfe
 #define REG_VERSION_ID		0xff
@@ -560,6 +562,77 @@ beep_store(struct device *dev, struct device_attribute *attr, const char *buf,
 	return err ? : count;
 }
 
+static ssize_t step_time_show(struct device *dev, struct device_attribute *attr,
+			      char *buf, bool step_up)
+{
+	struct nct7802_data *data = dev_get_drvdata(dev);
+	unsigned int reg, val;
+	int ret;
+
+	if (step_up)
+		reg = REG_SMARTFAN_STEP_UP_TIME;
+	else
+		reg = REG_SMARTFAN_STEP_DOWN_TIME;
+
+	ret = regmap_read(data->regmap, reg, &val);
+	if (ret < 0)
+		return ret;
+
+	return sprintf(buf, "%u\n", val * 100); /* Convert from ds to ms */
+}
+
+static ssize_t step_up_time_show(struct device *dev,
+				 struct device_attribute *attr, char *buf)
+{
+	return step_time_show(dev, attr, buf, true);
+}
+
+static ssize_t step_down_time_show(struct device *dev,
+				   struct device_attribute *attr, char *buf)
+{
+	return step_time_show(dev, attr, buf, false);
+}
+
+static ssize_t step_time_store(struct device *dev,
+			       struct device_attribute *attr, const char *buf,
+			       size_t count, bool step_up)
+{
+	struct nct7802_data *data = dev_get_drvdata(dev);
+	unsigned long val;
+	unsigned int reg;
+	int ret;
+
+	ret = kstrtoul(buf, 10, &val);
+	if (ret < 0)
+		return ret;
+
+	/* Clamp range, and convert from ms to ds */
+	val = DIV_ROUND_CLOSEST(clamp_val(val, 100, 25500), 100);
+
+	if (step_up)
+		reg = REG_SMARTFAN_STEP_UP_TIME;
+	else
+		reg = REG_SMARTFAN_STEP_DOWN_TIME;
+
+	ret = regmap_write(data->regmap, reg, val);
+
+	return ret ? : count;
+}
+
+static ssize_t step_up_time_store(struct device *dev,
+				  struct device_attribute *attr,
+				  const char *buf, size_t count)
+{
+	return step_time_store(dev, attr, buf, count, true);
+}
+
+static ssize_t step_down_time_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	return step_time_store(dev, attr, buf, count, false);
+}
+
 static SENSOR_DEVICE_ATTR_RW(temp1_type, temp_type, 0);
 static SENSOR_DEVICE_ATTR_2_RO(temp1_input, temp, 0x01, REG_TEMP_LSB);
 static SENSOR_DEVICE_ATTR_2_RW(temp1_min, temp, 0x31, 0);
@@ -975,12 +1048,30 @@ static const struct attribute_group nct7802_auto_point_group = {
 	.attrs = nct7802_auto_point_attrs,
 };
 
+/* 7.2.102 0x6E FANCTL Step Up Time Register */
+static SENSOR_DEVICE_ATTR_RW(step_up_time, step_up_time, 0);
+
+/* 7.2.103 0x6F FANCTL Step Down Time Register */
+static SENSOR_DEVICE_ATTR_RW(step_down_time, step_down_time, 0);
+
+static struct attribute *nct7802_step_time_attrs[] = {
+	&sensor_dev_attr_step_up_time.dev_attr.attr,
+	&sensor_dev_attr_step_down_time.dev_attr.attr,
+
+	NULL
+};
+
+static const struct attribute_group nct7802_step_time_group = {
+	.attrs = nct7802_step_time_attrs,
+};
+
 static const struct attribute_group *nct7802_groups[] = {
 	&nct7802_temp_group,
 	&nct7802_in_group,
 	&nct7802_fan_group,
 	&nct7802_pwm_group,
 	&nct7802_auto_point_group,
+	&nct7802_step_time_group,
 	NULL
 };
 
@@ -1193,7 +1284,7 @@ static const unsigned short nct7802_address_list[] = {
 };
 
 static const struct i2c_device_id nct7802_idtable[] = {
-	{ "nct7802" },
+	{ .name = "nct7802" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, nct7802_idtable);
diff --git a/drivers/hwmon/nct7904.c b/drivers/hwmon/nct7904.c
index 2fa091720c79b..976b8a008e448 100644
--- a/drivers/hwmon/nct7904.c
+++ b/drivers/hwmon/nct7904.c
@@ -1146,8 +1146,8 @@ static int nct7904_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id nct7904_id[] = {
-	{"nct7904"},
-	{}
+	{ .name = "nct7904" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, nct7904_id);
 
diff --git a/drivers/hwmon/nsa320-hwmon.c b/drivers/hwmon/nsa320-hwmon.c
index 18076ba7fc147..5c99acc096776 100644
--- a/drivers/hwmon/nsa320-hwmon.c
+++ b/drivers/hwmon/nsa320-hwmon.c
@@ -153,6 +153,7 @@ static const struct of_device_id of_nsa320_hwmon_match[] = {
 	{ .compatible = "zyxel,nsa320-mcu", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, of_nsa320_hwmon_match);
 
 static int nsa320_hwmon_probe(struct platform_device *pdev)
 {
@@ -197,7 +198,6 @@ static struct platform_driver nsa320_hwmon_driver = {
 
 module_platform_driver(nsa320_hwmon_driver);
 
-MODULE_DEVICE_TABLE(of, of_nsa320_hwmon_match);
 MODULE_AUTHOR("Peter Schildmann <linux@schildmann.info>");
 MODULE_AUTHOR("Adam Baker <linux@baker-net.org.uk>");
 MODULE_DESCRIPTION("NSA320 Hardware Monitoring");
diff --git a/drivers/hwmon/pcf8591.c b/drivers/hwmon/pcf8591.c
index 167d2fe4d5432..c0220ebfd4b60 100644
--- a/drivers/hwmon/pcf8591.c
+++ b/drivers/hwmon/pcf8591.c
@@ -285,7 +285,7 @@ static int pcf8591_read_channel(struct device *dev, int channel)
 }
 
 static const struct i2c_device_id pcf8591_id[] = {
-	{ "pcf8591" },
+	{ .name = "pcf8591" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pcf8591_id);
diff --git a/drivers/hwmon/pmbus/Kconfig b/drivers/hwmon/pmbus/Kconfig
index 8f4bff375ecbc..64f38654f4e7c 100644
--- a/drivers/hwmon/pmbus/Kconfig
+++ b/drivers/hwmon/pmbus/Kconfig
@@ -113,6 +113,15 @@ config SENSORS_CRPS
 	  This driver can also be built as a module. If so, the module will
 	  be called crps.
 
+config SENSORS_D1U74T
+	tristate "Murata D1U74T Power Supply"
+	help
+	  If you say yes here you get hardware monitoring support for the Murata
+	  D1U74T Power Supply.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called d1u74t.
+
 config SENSORS_DELTA_AHE50DC_FAN
 	tristate "Delta AHE-50DC fan control module"
 	help
@@ -161,6 +170,15 @@ config SENSORS_DPS920AB
 	  This driver can also be built as a module. If so, the module will
 	  be called dps920ab.
 
+config SENSORS_E50SN12051
+	tristate "Delta E50SN12051 Power Modules"
+	help
+	  If you say yes here you get hardware monitoring support for Delta
+	  E50SN12051 Power Modules.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called e50sn12051.
+
 config SENSORS_INA233
 	tristate "Texas Instruments INA233 and compatibles"
 	help
@@ -311,6 +329,16 @@ config SENSORS_LTC4286
 	  If you say yes here you get hardware monitoring support for Analog
 	  Devices LTC4286.
 
+config SENSORS_LX1308
+	tristate "Luxshare LX1308 DC/DC Power Module"
+	help
+	  If you say yes here you get hardware monitoring support for
+	  Luxshare LX1308, a high-efficiency 12V 860W DC/DC power module
+	  with PMBus interface.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called lx1308.
+
 config SENSORS_MAX15301
 	tristate "Maxim MAX15301"
 	help
@@ -365,6 +393,15 @@ config SENSORS_MAX20751
 	  This driver can also be built as a module. If so, the module will
 	  be called max20751.
 
+config SENSORS_MAX20830
+	tristate "Analog Devices MAX20830"
+	help
+	  If you say yes here you get hardware monitoring support for Analog
+	  Devices MAX20830.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called max20830.
+
 config SENSORS_MAX31785
 	tristate "Maxim MAX31785 and compatibles"
 	help
@@ -456,6 +493,15 @@ config SENSORS_MP2975
 	  This driver can also be built as a module. If so, the module will
 	  be called mp2975.
 
+config SENSORS_MP2985
+	tristate "MPS MP2985"
+	help
+	  If you say yes here you get hardware monitoring support for MPS
+	  MP2985 Dual Loop Digital Multi-Phase Controller.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called mp2985.
+
 config SENSORS_MP2993
 	tristate "MPS MP2993"
 	help
diff --git a/drivers/hwmon/pmbus/Makefile b/drivers/hwmon/pmbus/Makefile
index 7129b62bc00f8..1f2c73b71953e 100644
--- a/drivers/hwmon/pmbus/Makefile
+++ b/drivers/hwmon/pmbus/Makefile
@@ -17,6 +17,7 @@ obj-$(CONFIG_SENSORS_FSP_3Y)	+= fsp-3y.o
 obj-$(CONFIG_SENSORS_HAC300S)	+= hac300s.o
 obj-$(CONFIG_SENSORS_IBM_CFFPS)	+= ibm-cffps.o
 obj-$(CONFIG_SENSORS_DPS920AB)	+= dps920ab.o
+obj-$(CONFIG_SENSORS_E50SN12051) += e50sn12051.o
 obj-$(CONFIG_SENSORS_INA233)	+= ina233.o
 obj-$(CONFIG_SENSORS_INSPUR_IPSPS) += inspur-ipsps.o
 obj-$(CONFIG_SENSORS_IR35221)	+= ir35221.o
@@ -30,12 +31,14 @@ obj-$(CONFIG_SENSORS_LT7182S)	+= lt7182s.o
 obj-$(CONFIG_SENSORS_LTC2978)	+= ltc2978.o
 obj-$(CONFIG_SENSORS_LTC3815)	+= ltc3815.o
 obj-$(CONFIG_SENSORS_LTC4286)	+= ltc4286.o
+obj-$(CONFIG_SENSORS_LX1308)	+= lx1308.o
 obj-$(CONFIG_SENSORS_MAX15301)	+= max15301.o
 obj-$(CONFIG_SENSORS_MAX16064)	+= max16064.o
 obj-$(CONFIG_SENSORS_MAX16601)	+= max16601.o
 obj-$(CONFIG_SENSORS_MAX17616)	+= max17616.o
 obj-$(CONFIG_SENSORS_MAX20730)	+= max20730.o
 obj-$(CONFIG_SENSORS_MAX20751)	+= max20751.o
+obj-$(CONFIG_SENSORS_MAX20830)	+= max20830.o
 obj-$(CONFIG_SENSORS_MAX31785)	+= max31785.o
 obj-$(CONFIG_SENSORS_MAX34440)	+= max34440.o
 obj-$(CONFIG_SENSORS_MAX8688)	+= max8688.o
@@ -46,6 +49,7 @@ obj-$(CONFIG_SENSORS_MP2891)	+= mp2891.o
 obj-$(CONFIG_SENSORS_MP2925)	+= mp2925.o
 obj-$(CONFIG_SENSORS_MP29502)	+= mp29502.o
 obj-$(CONFIG_SENSORS_MP2975)	+= mp2975.o
+obj-$(CONFIG_SENSORS_MP2985)	+= mp2985.o
 obj-$(CONFIG_SENSORS_MP2993)	+= mp2993.o
 obj-$(CONFIG_SENSORS_MP5023)	+= mp5023.o
 obj-$(CONFIG_SENSORS_MP5920)	+= mp5920.o
@@ -76,3 +80,4 @@ obj-$(CONFIG_SENSORS_XDPE1A2G7B)	+= xdpe1a2g7b.o
 obj-$(CONFIG_SENSORS_ZL6100)	+= zl6100.o
 obj-$(CONFIG_SENSORS_PIM4328)	+= pim4328.o
 obj-$(CONFIG_SENSORS_CRPS)	+= crps.o
+obj-$(CONFIG_SENSORS_D1U74T)	+= d1u74t.o
diff --git a/drivers/hwmon/pmbus/acbel-fsg032.c b/drivers/hwmon/pmbus/acbel-fsg032.c
index 9f07fb4abaffd..d283005d92ae7 100644
--- a/drivers/hwmon/pmbus/acbel-fsg032.c
+++ b/drivers/hwmon/pmbus/acbel-fsg032.c
@@ -49,8 +49,8 @@ static void acbel_fsg032_init_debugfs(struct i2c_client *client)
 }
 
 static const struct i2c_device_id acbel_fsg032_id[] = {
-	{ "acbel_fsg032" },
-	{}
+	{ .name = "acbel_fsg032" },
+	{ }
 };
 
 static struct pmbus_driver_info acbel_fsg032_info = {
diff --git a/drivers/hwmon/pmbus/adm1266.c b/drivers/hwmon/pmbus/adm1266.c
index d90f8f80be8e0..22ccb721dd514 100644
--- a/drivers/hwmon/pmbus/adm1266.c
+++ b/drivers/hwmon/pmbus/adm1266.c
@@ -21,12 +21,14 @@
 #include <linux/slab.h>
 #include <linux/timekeeping.h>
 
+#define ADM1266_IC_DEVICE_REV	0xAE
 #define ADM1266_BLACKBOX_CONFIG	0xD3
 #define ADM1266_PDIO_CONFIG	0xD4
 #define ADM1266_READ_STATE	0xD9
 #define ADM1266_READ_BLACKBOX	0xDE
 #define ADM1266_SET_RTC		0xDF
 #define ADM1266_GPIO_CONFIG	0xE1
+#define ADM1266_POWERUP_COUNTER	0xE4
 #define ADM1266_BLACKBOX_INFO	0xE6
 #define ADM1266_PDIO_STATUS	0xE9
 #define ADM1266_GPIO_STATUS	0xEA
@@ -46,6 +48,7 @@
 
 #define ADM1266_BLACKBOX_OFFSET		0
 #define ADM1266_BLACKBOX_SIZE		64
+#define ADM1266_BLACKBOX_MAX_RECORDS	32
 
 #define ADM1266_PMBUS_BLOCK_MAX		255
 
@@ -60,7 +63,7 @@ struct adm1266_data {
 	u8 *dev_mem;
 	struct mutex buf_mutex;
 	u8 write_buf[ADM1266_PMBUS_BLOCK_MAX + 1] ____cacheline_aligned;
-	u8 read_buf[ADM1266_PMBUS_BLOCK_MAX + 1] ____cacheline_aligned;
+	u8 read_buf[ADM1266_PMBUS_BLOCK_MAX + 2] ____cacheline_aligned;
 };
 
 static const struct nvmem_cell_info adm1266_nvmem_cells[] = {
@@ -172,9 +175,13 @@ static int adm1266_gpio_get(struct gpio_chip *chip, unsigned int offset)
 	else
 		pmbus_cmd = ADM1266_PDIO_STATUS;
 
+	guard(pmbus_lock)(data->client);
+
 	ret = i2c_smbus_read_block_data(data->client, pmbus_cmd, read_buf);
 	if (ret < 0)
 		return ret;
+	if (ret < 2)
+		return -EIO;
 
 	pins_status = read_buf[0] + (read_buf[1] << 8);
 	if (offset < ADM1266_GPIO_NR)
@@ -192,9 +199,13 @@ static int adm1266_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask
 	unsigned int gpio_nr;
 	int ret;
 
+	guard(pmbus_lock)(data->client);
+
 	ret = i2c_smbus_read_block_data(data->client, ADM1266_GPIO_STATUS, read_buf);
 	if (ret < 0)
 		return ret;
+	if (ret < 2)
+		return -EIO;
 
 	status = read_buf[0] + (read_buf[1] << 8);
 
@@ -207,11 +218,12 @@ static int adm1266_gpio_get_multiple(struct gpio_chip *chip, unsigned long *mask
 	ret = i2c_smbus_read_block_data(data->client, ADM1266_PDIO_STATUS, read_buf);
 	if (ret < 0)
 		return ret;
+	if (ret < 2)
+		return -EIO;
 
 	status = read_buf[0] + (read_buf[1] << 8);
 
-	*bits = 0;
-	for_each_set_bit_from(gpio_nr, mask, ADM1266_GPIO_NR + ADM1266_PDIO_STATUS) {
+	for_each_set_bit_from(gpio_nr, mask, ADM1266_GPIO_NR + ADM1266_PDIO_NR) {
 		if (test_bit(gpio_nr - ADM1266_GPIO_NR, &status))
 			set_bit(gpio_nr, bits);
 	}
@@ -230,6 +242,8 @@ static void adm1266_gpio_dbg_show(struct seq_file *s, struct gpio_chip *chip)
 	int ret;
 	int i;
 
+	guard(pmbus_lock)(data->client);
+
 	for (i = 0; i < ADM1266_GPIO_NR; i++) {
 		write_cmd = adm1266_gpio_mapping[i][1];
 		ret = adm1266_pmbus_block_xfer(data, ADM1266_GPIO_CONFIG, 1, &write_cmd, read_buf);
@@ -290,8 +304,9 @@ static int adm1266_config_gpio(struct adm1266_data *data)
 	int i;
 
 	for (i = 0; i < ARRAY_SIZE(data->gpio_names); i++) {
-		gpio_name = devm_kasprintf(&data->client->dev, GFP_KERNEL, "adm1266-%x-%s",
-					   data->client->addr, adm1266_names[i]);
+		gpio_name = devm_kasprintf(&data->client->dev, GFP_KERNEL, "adm1266-%d-%x-%s",
+					   data->client->adapter->nr, data->client->addr,
+					   adm1266_names[i]);
 		if (!gpio_name)
 			return -ENOMEM;
 
@@ -322,6 +337,7 @@ static int adm1266_state_read(struct seq_file *s, void *pdata)
 	struct i2c_client *client = to_i2c_client(dev);
 	int ret;
 
+	guard(pmbus_lock)(client);
 	ret = i2c_smbus_read_word_data(client, ADM1266_READ_STATE);
 	if (ret < 0)
 		return ret;
@@ -331,6 +347,154 @@ static int adm1266_state_read(struct seq_file *s, void *pdata)
 	return 0;
 }
 
+/*
+ * IC_DEVICE_REV (0xAE) returns an 8-byte block (datasheet Rev. D, Table 80):
+ *   [2:0] firmware revision  major.minor.patch
+ *   [5:3] bootloader revision major.minor.patch
+ *   [7:6] silicon revision    two ASCII characters
+ */
+static int adm1266_firmware_revision_read(struct seq_file *s, void *pdata)
+{
+	struct device *dev = s->private;
+	struct i2c_client *client = to_i2c_client(dev);
+	u8 buf[I2C_SMBUS_BLOCK_MAX];
+	int ret;
+
+	guard(pmbus_lock)(client);
+	ret = i2c_smbus_read_block_data(client, ADM1266_IC_DEVICE_REV, buf);
+	if (ret < 0)
+		return ret;
+	if (ret < 3)
+		return -EIO;
+
+	seq_printf(s, "%u.%u.%u\n", buf[0], buf[1], buf[2]);
+
+	return 0;
+}
+
+/*
+ * POWERUP_COUNTER (0xE4) is a 2-byte little-endian non-volatile counter
+ * that increments on every device power cycle (datasheet Rev. D, Table
+ * 93). It saturates at 65535 and cannot be reset by the host. Each
+ * blackbox record embeds the counter value at record time, so this live
+ * read is mainly useful for matching a record back to its boot.
+ */
+static int adm1266_powerup_counter_read(struct seq_file *s, void *pdata)
+{
+	struct device *dev = s->private;
+	struct i2c_client *client = to_i2c_client(dev);
+	u8 buf[I2C_SMBUS_BLOCK_MAX];
+	int ret;
+
+	guard(pmbus_lock)(client);
+	ret = i2c_smbus_read_block_data(client, ADM1266_POWERUP_COUNTER, buf);
+	if (ret < 0)
+		return ret;
+	if (ret != 2)
+		return -EIO;
+
+	seq_printf(s, "%u\n", buf[0] | (buf[1] << 8));
+
+	return 0;
+}
+
+/*
+ * Clearing the blackbox is required when the device is configured in
+ * single-recording mode (BLACKBOX_CONFIG[0] = 0): once the 32-record
+ * buffer is full the device stops recording until cleared.
+ *
+ * The clear is issued as a 2-byte block-write to READ_BLACKBOX with
+ * payload {0xFE, 0x00} per the datasheet. READ_BLACKBOX is also used
+ * by adm1266_nvmem_read_blackbox() to walk records one at a time;
+ * both paths run under pmbus_lock so the clear cannot interleave
+ * mid-iteration and corrupt the read sequence.
+ */
+static ssize_t adm1266_clear_blackbox_write(struct file *file, const char __user *ubuf,
+					    size_t count, loff_t *ppos)
+{
+	struct i2c_client *client = file->private_data;
+	u8 payload[2] = { 0xFE, 0x00 };
+	int ret;
+
+	guard(pmbus_lock)(client);
+	ret = i2c_smbus_write_block_data(client, ADM1266_READ_BLACKBOX,
+					 sizeof(payload), payload);
+	if (ret < 0)
+		return ret;
+
+	return count;
+}
+
+static const struct file_operations adm1266_clear_blackbox_fops = {
+	.owner = THIS_MODULE,
+	.open = simple_open,
+	.write = adm1266_clear_blackbox_write,
+	.llseek = noop_llseek,
+};
+
+/*
+ * SET_RTC (0xDF) is a 6-byte block (datasheet Rev. D, Table 84):
+ *   bytes [1:0] - fractional seconds (1/65536 s, written as zero)
+ *   bytes [5:2] - seconds since 1970-01-01 UTC, little-endian
+ *
+ * The driver seeds it once at probe via adm1266_rtc_set().  Over a
+ * long uptime the chip's counter drifts away from host wall-clock,
+ * so expose it via debugfs:
+ *
+ *   read  -- returns the chip's current seconds counter, which lets
+ *            userspace observe host-vs-chip drift.
+ *   write -- the kernel re-reads ktime_get_real_seconds() and writes
+ *            it to SET_RTC.  The write payload is ignored; userspace
+ *            does not get to supply its own timestamp value, so
+ *            there is no way to push a wrong time into the chip.
+ *
+ * A small userspace agent (chrony hook, systemd-timesyncd script,
+ * or a periodic cron job) can write to this file to keep the
+ * timestamp embedded in each blackbox record aligned with
+ * wall-clock across long uptimes.
+ */
+static int adm1266_rtc_get(void *data, u64 *val)
+{
+	struct i2c_client *client = data;
+	u8 buf[I2C_SMBUS_BLOCK_MAX];
+	u32 seconds = 0;
+	int ret, i;
+
+	guard(pmbus_lock)(client);
+	ret = i2c_smbus_read_block_data(client, ADM1266_SET_RTC, buf);
+	if (ret < 0)
+		return ret;
+	if (ret < 6)
+		return -EIO;
+
+	for (i = 0; i < 4; i++)
+		seconds |= (u32)buf[2 + i] << (i * 8);
+
+	*val = seconds;
+
+	return 0;
+}
+
+static int adm1266_rtc_set(void *data, u64 val)
+{
+	struct i2c_client *client = data;
+	time64_t kt = ktime_get_real_seconds();
+	u8 write_buf[6] = { 0 };
+	int i;
+
+	/* User-supplied @val is ignored on purpose; the kernel owns the
+	 * time source so userspace cannot push a wrong value into the chip.
+	 */
+	for (i = 0; i < 4; i++)
+		write_buf[2 + i] = (kt >> (i * 8)) & 0xFF;
+
+	guard(pmbus_lock)(client);
+	return i2c_smbus_write_block_data(client, ADM1266_SET_RTC,
+					  sizeof(write_buf), write_buf);
+}
+DEFINE_DEBUGFS_ATTRIBUTE(adm1266_rtc_fops,
+			 adm1266_rtc_get, adm1266_rtc_set, "%llu\n");
+
 static void adm1266_init_debugfs(struct adm1266_data *data)
 {
 	struct dentry *root;
@@ -343,13 +507,22 @@ static void adm1266_init_debugfs(struct adm1266_data *data)
 
 	debugfs_create_devm_seqfile(&data->client->dev, "sequencer_state", data->debugfs_dir,
 				    adm1266_state_read);
+	debugfs_create_devm_seqfile(&data->client->dev, "firmware_revision", data->debugfs_dir,
+				    adm1266_firmware_revision_read);
+	debugfs_create_devm_seqfile(&data->client->dev, "powerup_counter", data->debugfs_dir,
+				    adm1266_powerup_counter_read);
+	debugfs_create_file("clear_blackbox", 0200, data->debugfs_dir, data->client,
+			    &adm1266_clear_blackbox_fops);
+	debugfs_create_file("rtc", 0600, data->debugfs_dir, data->client,
+			    &adm1266_rtc_fops);
 }
 
 static int adm1266_nvmem_read_blackbox(struct adm1266_data *data, u8 *read_buff)
 {
+	u8 record[ADM1266_PMBUS_BLOCK_MAX];
 	int record_count;
 	char index;
-	u8 buf[5];
+	u8 buf[I2C_SMBUS_BLOCK_MAX];
 	int ret;
 
 	ret = i2c_smbus_read_block_data(data->client, ADM1266_BLACKBOX_INFO, buf);
@@ -360,15 +533,18 @@ static int adm1266_nvmem_read_blackbox(struct adm1266_data *data, u8 *read_buff)
 		return -EIO;
 
 	record_count = buf[3];
+	if (record_count > ADM1266_BLACKBOX_MAX_RECORDS)
+		return -EIO;
 
 	for (index = 0; index < record_count; index++) {
-		ret = adm1266_pmbus_block_xfer(data, ADM1266_READ_BLACKBOX, 1, &index, read_buff);
+		ret = adm1266_pmbus_block_xfer(data, ADM1266_READ_BLACKBOX, 1, &index, record);
 		if (ret < 0)
 			return ret;
 
 		if (ret != ADM1266_BLACKBOX_SIZE)
 			return -EIO;
 
+		memcpy(read_buff, record, ADM1266_BLACKBOX_SIZE);
 		read_buff += ADM1266_BLACKBOX_SIZE;
 	}
 
@@ -383,6 +559,8 @@ static int adm1266_nvmem_read(void *priv, unsigned int offset, void *val, size_t
 	if (offset + bytes > data->nvmem_config.size)
 		return -EINVAL;
 
+	guard(pmbus_lock)(data->client);
+
 	if (offset == 0) {
 		memset(data->dev_mem, 0, data->nvmem_config.size);
 
@@ -426,23 +604,6 @@ static int adm1266_config_nvmem(struct adm1266_data *data)
 	return 0;
 }
 
-static int adm1266_set_rtc(struct adm1266_data *data)
-{
-	time64_t kt;
-	char write_buf[6];
-	int i;
-
-	kt = ktime_get_seconds();
-
-	memset(write_buf, 0, sizeof(write_buf));
-
-	for (i = 0; i < 4; i++)
-		write_buf[2 + i] = (kt >> (i * 8)) & 0xFF;
-
-	return i2c_smbus_write_block_data(data->client, ADM1266_SET_RTC, sizeof(write_buf),
-					  write_buf);
-}
-
 static int adm1266_probe(struct i2c_client *client)
 {
 	struct adm1266_data *data;
@@ -462,11 +623,11 @@ static int adm1266_probe(struct i2c_client *client)
 	crc8_populate_msb(pmbus_crc_table, 0x7);
 	mutex_init(&data->buf_mutex);
 
-	ret = adm1266_config_gpio(data);
-	if (ret < 0)
+	ret = pmbus_do_probe(client, &data->info);
+	if (ret)
 		return ret;
 
-	ret = adm1266_set_rtc(data);
+	ret = adm1266_rtc_set(client, 0);
 	if (ret < 0)
 		return ret;
 
@@ -474,8 +635,8 @@ static int adm1266_probe(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
-	ret = pmbus_do_probe(client, &data->info);
-	if (ret)
+	ret = adm1266_config_gpio(data);
+	if (ret < 0)
 		return ret;
 
 	adm1266_init_debugfs(data);
@@ -490,7 +651,7 @@ static const struct of_device_id adm1266_of_match[] = {
 MODULE_DEVICE_TABLE(of, adm1266_of_match);
 
 static const struct i2c_device_id adm1266_id[] = {
-	{ "adm1266" },
+	{ .name = "adm1266" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adm1266_id);
diff --git a/drivers/hwmon/pmbus/adm1275.c b/drivers/hwmon/pmbus/adm1275.c
index bc2a6a07dc3e2..2e5963fb1e127 100644
--- a/drivers/hwmon/pmbus/adm1275.c
+++ b/drivers/hwmon/pmbus/adm1275.c
@@ -478,16 +478,16 @@ static int adm1275_read_byte_data(struct i2c_client *client, int page, int reg)
 }
 
 static const struct i2c_device_id adm1275_id[] = {
-	{ "adm1075", adm1075 },
-	{ "adm1272", adm1272 },
-	{ "adm1273", adm1273 },
-	{ "adm1275", adm1275 },
-	{ "adm1276", adm1276 },
-	{ "adm1278", adm1278 },
-	{ "adm1281", adm1281 },
-	{ "adm1293", adm1293 },
-	{ "adm1294", adm1294 },
-	{ "mc09c", sq24905c },
+	{ .name = "adm1075", .driver_data = adm1075 },
+	{ .name = "adm1272", .driver_data = adm1272 },
+	{ .name = "adm1273", .driver_data = adm1273 },
+	{ .name = "adm1275", .driver_data = adm1275 },
+	{ .name = "adm1276", .driver_data = adm1276 },
+	{ .name = "adm1278", .driver_data = adm1278 },
+	{ .name = "adm1281", .driver_data = adm1281 },
+	{ .name = "adm1293", .driver_data = adm1293 },
+	{ .name = "adm1294", .driver_data = adm1294 },
+	{ .name = "mc09c", .driver_data = sq24905c },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adm1275_id);
diff --git a/drivers/hwmon/pmbus/aps-379.c b/drivers/hwmon/pmbus/aps-379.c
index 7d46cd647e20a..3ec0940ae5644 100644
--- a/drivers/hwmon/pmbus/aps-379.c
+++ b/drivers/hwmon/pmbus/aps-379.c
@@ -100,8 +100,8 @@ static struct pmbus_driver_info aps_379_info = {
 };
 
 static const struct i2c_device_id aps_379_id[] = {
-	{ "aps-379", 0 },
-	{},
+	{ .name = "aps-379" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, aps_379_id);
 
diff --git a/drivers/hwmon/pmbus/bel-pfe.c b/drivers/hwmon/pmbus/bel-pfe.c
index 6499556f735b4..9e3dc9d29c562 100644
--- a/drivers/hwmon/pmbus/bel-pfe.c
+++ b/drivers/hwmon/pmbus/bel-pfe.c
@@ -106,9 +106,9 @@ static int pfe_pmbus_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id pfe_device_id[] = {
-	{"pfe1100", pfe1100},
-	{"pfe3000", pfe3000},
-	{}
+	{ .name = "pfe1100", .driver_data = pfe1100 },
+	{ .name = "pfe3000", .driver_data = pfe3000 },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, pfe_device_id);
diff --git a/drivers/hwmon/pmbus/bpa-rs600.c b/drivers/hwmon/pmbus/bpa-rs600.c
index 6c3875ba37a08..e364dcb59dd42 100644
--- a/drivers/hwmon/pmbus/bpa-rs600.c
+++ b/drivers/hwmon/pmbus/bpa-rs600.c
@@ -147,9 +147,9 @@ static struct pmbus_driver_info bpa_rs600_info = {
 };
 
 static const struct i2c_device_id bpa_rs600_id[] = {
-	{ "bpa-rs600", bpa_rs600 },
-	{ "bpd-rs600", bpd_rs600 },
-	{},
+	{ .name = "bpa-rs600", .driver_data = bpa_rs600 },
+	{ .name = "bpd-rs600", .driver_data = bpd_rs600 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, bpa_rs600_id);
 
diff --git a/drivers/hwmon/pmbus/crps.c b/drivers/hwmon/pmbus/crps.c
index 164b33fed312f..266ec89475190 100644
--- a/drivers/hwmon/pmbus/crps.c
+++ b/drivers/hwmon/pmbus/crps.c
@@ -10,8 +10,8 @@
 #include "pmbus.h"
 
 static const struct i2c_device_id crps_id[] = {
-	{ "intel_crps185" },
-	{}
+	{ .name = "intel_crps185" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, crps_id);
 
diff --git a/drivers/hwmon/pmbus/d1u74t.c b/drivers/hwmon/pmbus/d1u74t.c
new file mode 100644
index 0000000000000..752df3af9c1a1
--- /dev/null
+++ b/drivers/hwmon/pmbus/d1u74t.c
@@ -0,0 +1,88 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2026 Nexthop Systems.
+ */
+
+#include <linux/i2c.h>
+#include <linux/of.h>
+#include <linux/module.h>
+#include <linux/pmbus.h>
+#include <linux/string.h>
+
+#include "pmbus.h"
+
+static const struct i2c_device_id d1u74t_id[] = {
+	{ "d1u74t" },
+	{},
+};
+MODULE_DEVICE_TABLE(i2c, d1u74t_id);
+
+static struct pmbus_driver_info d1u74t_info = {
+	.pages = 1,
+	/* PSU uses default linear data format. */
+	.func[0] = PMBUS_HAVE_PIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
+		   PMBUS_HAVE_IIN | PMBUS_HAVE_VIN | PMBUS_HAVE_STATUS_INPUT |
+		   PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT | PMBUS_HAVE_TEMP |
+		   PMBUS_HAVE_TEMP2 | PMBUS_HAVE_TEMP3 |
+		   PMBUS_HAVE_STATUS_TEMP | PMBUS_HAVE_FAN12 |
+		   PMBUS_HAVE_STATUS_FAN12,
+};
+
+static int d1u74t_probe(struct i2c_client *client)
+{
+	char buf[I2C_SMBUS_BLOCK_MAX + 2] = { 0 };
+	struct device *dev = &client->dev;
+	int rc;
+
+	rc = i2c_smbus_read_block_data(client, PMBUS_MFR_ID, buf);
+	if (rc < 0)
+		return dev_err_probe(dev, rc, "Failed to read PMBUS_MFR_ID\n");
+
+	if (rc != 9 || strncmp(buf, "Murata-PS", 9)) {
+		buf[rc] = '\0';
+		return dev_err_probe(dev, -ENODEV,
+				     "Unsupported Manufacturer ID '%s'\n",
+				     buf);
+	}
+
+	rc = i2c_smbus_read_block_data(client, PMBUS_MFR_MODEL, buf);
+	if (rc < 0)
+		return dev_err_probe(dev, rc,
+				     "Failed to read PMBUS_MFR_MODEL\n");
+
+	if (rc < 8 || strncmp(buf, "D1U74T-W", 8)) {
+		buf[rc] = '\0';
+		return dev_err_probe(dev, -ENODEV, "Model '%s' not supported\n",
+				     buf);
+	}
+
+	rc = pmbus_do_probe(client, &d1u74t_info);
+	if (rc)
+		return dev_err_probe(dev, rc, "Failed to probe\n");
+
+	return 0;
+}
+
+static const struct of_device_id d1u74t_of_match[] = {
+	{
+		.compatible = "murata,d1u74t",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, d1u74t_of_match);
+
+static struct i2c_driver d1u74t_driver = {
+	.driver = {
+		.name = "d1u74t",
+		.of_match_table = d1u74t_of_match,
+	},
+	.probe = d1u74t_probe,
+	.id_table = d1u74t_id,
+};
+
+module_i2c_driver(d1u74t_driver);
+
+MODULE_AUTHOR("Abdurrahman Hussain");
+MODULE_DESCRIPTION("PMBus driver for Murata D1U74T-W power supplies");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/pmbus/delta-ahe50dc-fan.c b/drivers/hwmon/pmbus/delta-ahe50dc-fan.c
index 3850eaea75da2..2df655b20ea57 100644
--- a/drivers/hwmon/pmbus/delta-ahe50dc-fan.c
+++ b/drivers/hwmon/pmbus/delta-ahe50dc-fan.c
@@ -103,7 +103,7 @@ static int ahe50dc_fan_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ahe50dc_fan_id[] = {
-	{ "ahe50dc_fan" },
+	{ .name = "ahe50dc_fan" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ahe50dc_fan_id);
diff --git a/drivers/hwmon/pmbus/dps920ab.c b/drivers/hwmon/pmbus/dps920ab.c
index 325111a955e61..0d2901c314c2b 100644
--- a/drivers/hwmon/pmbus/dps920ab.c
+++ b/drivers/hwmon/pmbus/dps920ab.c
@@ -191,8 +191,8 @@ static const struct of_device_id __maybe_unused dps920ab_of_match[] = {
 MODULE_DEVICE_TABLE(of, dps920ab_of_match);
 
 static const struct i2c_device_id dps920ab_device_id[] = {
-	{ "dps920ab" },
-	{}
+	{ .name = "dps920ab" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, dps920ab_device_id);
 
diff --git a/drivers/hwmon/pmbus/e50sn12051.c b/drivers/hwmon/pmbus/e50sn12051.c
new file mode 100644
index 0000000000000..efb4d62b26038
--- /dev/null
+++ b/drivers/hwmon/pmbus/e50sn12051.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Hardware monitoring driver for E50SN12051
+ */
+
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include "pmbus.h"
+
+static struct pmbus_driver_info e50sn12051_info = {
+	.pages = 1,
+	.format[PSC_VOLTAGE_IN] = linear,
+	.format[PSC_VOLTAGE_OUT] = linear,
+	.format[PSC_CURRENT_OUT] = linear,
+	.format[PSC_TEMPERATURE] = linear,
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_STATUS_INPUT |
+		   PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT |
+		   PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT |
+		   PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP,
+};
+
+static const struct i2c_device_id e50sn12051_id[] = { { "e50sn12051", 0 }, {} };
+MODULE_DEVICE_TABLE(i2c, e50sn12051_id);
+
+static const struct of_device_id e50sn12051_of_match[] = {
+	{ .compatible = "delta,e50sn12051" },
+	{},
+};
+MODULE_DEVICE_TABLE(of, e50sn12051_of_match);
+
+static int e50sn12051_probe(struct i2c_client *client)
+{
+	return pmbus_do_probe(client, &e50sn12051_info);
+}
+
+static struct i2c_driver e50sn12051_driver = {
+	.driver = {
+		.name = "e50sn12051",
+		.of_match_table = e50sn12051_of_match,
+	},
+	.probe = e50sn12051_probe,
+
+	.id_table = e50sn12051_id,
+};
+
+module_i2c_driver(e50sn12051_driver);
+
+MODULE_AUTHOR("Kevin Chang <kevin.chang2@amd.com>");
+MODULE_DESCRIPTION("PMBus driver for E50SN12051");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/pmbus/fsp-3y.c b/drivers/hwmon/pmbus/fsp-3y.c
index cad4d23300036..44cf2db9364b9 100644
--- a/drivers/hwmon/pmbus/fsp-3y.c
+++ b/drivers/hwmon/pmbus/fsp-3y.c
@@ -271,8 +271,8 @@ static int fsp3y_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id fsp3y_id[] = {
-	{"ym2151e", ym2151e},
-	{"yh5151e", yh5151e},
+	{ .name = "ym2151e", .driver_data = ym2151e },
+	{ .name = "yh5151e", .driver_data = yh5151e },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, fsp3y_id);
diff --git a/drivers/hwmon/pmbus/hac300s.c b/drivers/hwmon/pmbus/hac300s.c
index a073db1cfe2e4..761e53890ebc4 100644
--- a/drivers/hwmon/pmbus/hac300s.c
+++ b/drivers/hwmon/pmbus/hac300s.c
@@ -112,8 +112,8 @@ static const struct of_device_id hac300s_of_match[] = {
 MODULE_DEVICE_TABLE(of, hac300s_of_match);
 
 static const struct i2c_device_id hac300s_id[] = {
-	{"hac300s", 0},
-	{}
+	{ .name = "hac300s" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, hac300s_id);
 
diff --git a/drivers/hwmon/pmbus/ibm-cffps.c b/drivers/hwmon/pmbus/ibm-cffps.c
index 6c7256d997f46..aad94bcb9ceb5 100644
--- a/drivers/hwmon/pmbus/ibm-cffps.c
+++ b/drivers/hwmon/pmbus/ibm-cffps.c
@@ -472,10 +472,10 @@ static struct pmbus_platform_data ibm_cffps_pdata = {
 };
 
 static const struct i2c_device_id ibm_cffps_id[] = {
-	{ "ibm_cffps1", cffps1 },
-	{ "ibm_cffps2", cffps2 },
-	{ "ibm_cffps", cffps_unknown },
-	{}
+	{ .name = "ibm_cffps1", .driver_data = cffps1 },
+	{ .name = "ibm_cffps2", .driver_data = cffps2 },
+	{ .name = "ibm_cffps", .driver_data = cffps_unknown },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ibm_cffps_id);
 
diff --git a/drivers/hwmon/pmbus/ina233.c b/drivers/hwmon/pmbus/ina233.c
index 652087589c55a..c72aa258b7c67 100644
--- a/drivers/hwmon/pmbus/ina233.c
+++ b/drivers/hwmon/pmbus/ina233.c
@@ -168,8 +168,8 @@ static int ina233_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ina233_id[] = {
-	{"ina233", 0},
-	{}
+	{ .name = "ina233" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ina233_id);
 
diff --git a/drivers/hwmon/pmbus/inspur-ipsps.c b/drivers/hwmon/pmbus/inspur-ipsps.c
index 074e0f164ee1c..57c0fc16909cf 100644
--- a/drivers/hwmon/pmbus/inspur-ipsps.c
+++ b/drivers/hwmon/pmbus/inspur-ipsps.c
@@ -197,8 +197,8 @@ static int ipsps_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ipsps_id[] = {
-	{ "ipsps1" },
-	{}
+	{ .name = "ipsps1" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ipsps_id);
 
diff --git a/drivers/hwmon/pmbus/ir35221.c b/drivers/hwmon/pmbus/ir35221.c
index 46d8f334d49a4..b2120fc76f409 100644
--- a/drivers/hwmon/pmbus/ir35221.c
+++ b/drivers/hwmon/pmbus/ir35221.c
@@ -126,8 +126,8 @@ static int ir35221_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ir35221_id[] = {
-	{"ir35221"},
-	{}
+	{ .name = "ir35221" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, ir35221_id);
diff --git a/drivers/hwmon/pmbus/ir36021.c b/drivers/hwmon/pmbus/ir36021.c
index 34ce15fc708bc..0dce4c3f666f5 100644
--- a/drivers/hwmon/pmbus/ir36021.c
+++ b/drivers/hwmon/pmbus/ir36021.c
@@ -51,8 +51,8 @@ static int ir36021_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ir36021_id[] = {
-	{ "ir36021" },
-	{},
+	{ .name = "ir36021" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ir36021_id);
 
diff --git a/drivers/hwmon/pmbus/ir38064.c b/drivers/hwmon/pmbus/ir38064.c
index 7b4188e8bf483..47ce88e9d13a3 100644
--- a/drivers/hwmon/pmbus/ir38064.c
+++ b/drivers/hwmon/pmbus/ir38064.c
@@ -53,11 +53,11 @@ static int ir38064_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ir38064_id[] = {
-	{"ir38060"},
-	{"ir38064"},
-	{"ir38164"},
-	{"ir38263"},
-	{}
+	{ .name = "ir38060" },
+	{ .name = "ir38064" },
+	{ .name = "ir38164" },
+	{ .name = "ir38263" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, ir38064_id);
diff --git a/drivers/hwmon/pmbus/irps5401.c b/drivers/hwmon/pmbus/irps5401.c
index 43674c64841d5..1694b96d7abf9 100644
--- a/drivers/hwmon/pmbus/irps5401.c
+++ b/drivers/hwmon/pmbus/irps5401.c
@@ -44,8 +44,8 @@ static int irps5401_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id irps5401_id[] = {
-	{"irps5401"},
-	{}
+	{ .name = "irps5401" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, irps5401_id);
diff --git a/drivers/hwmon/pmbus/isl68137.c b/drivers/hwmon/pmbus/isl68137.c
index 21d047b577a47..2f7f825bfb69e 100644
--- a/drivers/hwmon/pmbus/isl68137.c
+++ b/drivers/hwmon/pmbus/isl68137.c
@@ -409,54 +409,54 @@ static int isl68137_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id raa_dmpvr_id[] = {
-	{"isl68137", raa_dmpvr1_2rail},
-	{"isl68220", raa_dmpvr2_2rail},
-	{"isl68221", raa_dmpvr2_3rail},
-	{"isl68222", raa_dmpvr2_2rail},
-	{"isl68223", raa_dmpvr2_2rail},
-	{"isl68224", raa_dmpvr2_3rail},
-	{"isl68225", raa_dmpvr2_2rail},
-	{"isl68226", raa_dmpvr2_3rail},
-	{"isl68227", raa_dmpvr2_1rail},
-	{"isl68229", raa_dmpvr2_3rail},
-	{"isl68233", raa_dmpvr2_2rail},
-	{"isl68239", raa_dmpvr2_3rail},
-
-	{"isl69222", raa_dmpvr2_2rail},
-	{"isl69223", raa_dmpvr2_3rail},
-	{"isl69224", raa_dmpvr2_2rail},
-	{"isl69225", raa_dmpvr2_2rail},
-	{"isl69227", raa_dmpvr2_3rail},
-	{"isl69228", raa_dmpvr2_3rail},
-	{"isl69234", raa_dmpvr2_2rail},
-	{"isl69236", raa_dmpvr2_2rail},
-	{"isl69239", raa_dmpvr2_3rail},
-	{"isl69242", raa_dmpvr2_2rail},
-	{"isl69243", raa_dmpvr2_1rail},
-	{"isl69247", raa_dmpvr2_2rail},
-	{"isl69248", raa_dmpvr2_2rail},
-	{"isl69254", raa_dmpvr2_2rail},
-	{"isl69255", raa_dmpvr2_2rail},
-	{"isl69256", raa_dmpvr2_2rail},
-	{"isl69259", raa_dmpvr2_2rail},
-	{"isl69260", raa_dmpvr2_2rail},
-	{"isl69268", raa_dmpvr2_2rail},
-	{"isl69269", raa_dmpvr2_3rail},
-	{"isl69298", raa_dmpvr2_2rail},
-
-	{"raa228000", raa_dmpvr2_hv},
-	{"raa228004", raa_dmpvr2_hv},
-	{"raa228006", raa_dmpvr2_hv},
-	{"raa228228", raa_dmpvr2_2rail_nontc},
-	{"raa228244", raa_dmpvr2_2rail_nontc},
-	{"raa228246", raa_dmpvr2_2rail_nontc},
-	{"raa228942", raa_dmpvr2_2rail_nontc},
-	{"raa228943", raa_dmpvr2_2rail_nontc},
-	{"raa229001", raa_dmpvr2_2rail},
-	{"raa229004", raa_dmpvr2_2rail},
-	{"raa229141", raa_dmpvr2_2rail_pmbus},
-	{"raa229621", raa_dmpvr2_2rail},
-	{}
+	{ .name = "isl68137", .driver_data = raa_dmpvr1_2rail },
+	{ .name = "isl68220", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl68221", .driver_data = raa_dmpvr2_3rail },
+	{ .name = "isl68222", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl68223", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl68224", .driver_data = raa_dmpvr2_3rail },
+	{ .name = "isl68225", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl68226", .driver_data = raa_dmpvr2_3rail },
+	{ .name = "isl68227", .driver_data = raa_dmpvr2_1rail },
+	{ .name = "isl68229", .driver_data = raa_dmpvr2_3rail },
+	{ .name = "isl68233", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl68239", .driver_data = raa_dmpvr2_3rail },
+
+	{ .name = "isl69222", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69223", .driver_data = raa_dmpvr2_3rail },
+	{ .name = "isl69224", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69225", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69227", .driver_data = raa_dmpvr2_3rail },
+	{ .name = "isl69228", .driver_data = raa_dmpvr2_3rail },
+	{ .name = "isl69234", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69236", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69239", .driver_data = raa_dmpvr2_3rail },
+	{ .name = "isl69242", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69243", .driver_data = raa_dmpvr2_1rail },
+	{ .name = "isl69247", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69248", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69254", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69255", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69256", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69259", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69260", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69268", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "isl69269", .driver_data = raa_dmpvr2_3rail },
+	{ .name = "isl69298", .driver_data = raa_dmpvr2_2rail },
+
+	{ .name = "raa228000", .driver_data = raa_dmpvr2_hv },
+	{ .name = "raa228004", .driver_data = raa_dmpvr2_hv },
+	{ .name = "raa228006", .driver_data = raa_dmpvr2_hv },
+	{ .name = "raa228228", .driver_data = raa_dmpvr2_2rail_nontc },
+	{ .name = "raa228244", .driver_data = raa_dmpvr2_2rail_nontc },
+	{ .name = "raa228246", .driver_data = raa_dmpvr2_2rail_nontc },
+	{ .name = "raa228942", .driver_data = raa_dmpvr2_2rail_nontc },
+	{ .name = "raa228943", .driver_data = raa_dmpvr2_2rail_nontc },
+	{ .name = "raa229001", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "raa229004", .driver_data = raa_dmpvr2_2rail },
+	{ .name = "raa229141", .driver_data = raa_dmpvr2_2rail_pmbus },
+	{ .name = "raa229621", .driver_data = raa_dmpvr2_2rail },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, raa_dmpvr_id);
diff --git a/drivers/hwmon/pmbus/lm25066.c b/drivers/hwmon/pmbus/lm25066.c
index dd7275a67a0ab..0f7df7e2c9c5b 100644
--- a/drivers/hwmon/pmbus/lm25066.c
+++ b/drivers/hwmon/pmbus/lm25066.c
@@ -442,11 +442,11 @@ static const struct regulator_desc lm25066_reg_desc[] = {
 #endif
 
 static const struct i2c_device_id lm25066_id[] = {
-	{"lm25056", lm25056},
-	{"lm25066", lm25066},
-	{"lm5064", lm5064},
-	{"lm5066", lm5066},
-	{"lm5066i", lm5066i},
+	{ .name = "lm25056", .driver_data = lm25056 },
+	{ .name = "lm25066", .driver_data = lm25066 },
+	{ .name = "lm5064", .driver_data = lm5064 },
+	{ .name = "lm5066", .driver_data = lm5066 },
+	{ .name = "lm5066i", .driver_data = lm5066i },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lm25066_id);
diff --git a/drivers/hwmon/pmbus/lt3074.c b/drivers/hwmon/pmbus/lt3074.c
index 3704dbe7b54ab..ed932ddb4f77b 100644
--- a/drivers/hwmon/pmbus/lt3074.c
+++ b/drivers/hwmon/pmbus/lt3074.c
@@ -95,8 +95,8 @@ static int lt3074_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lt3074_id[] = {
-	{ "lt3074", 0 },
-	{}
+	{ .name = "lt3074" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lt3074_id);
 
diff --git a/drivers/hwmon/pmbus/lt7182s.c b/drivers/hwmon/pmbus/lt7182s.c
index 9d6d50f39bd65..f8971a786f25a 100644
--- a/drivers/hwmon/pmbus/lt7182s.c
+++ b/drivers/hwmon/pmbus/lt7182s.c
@@ -168,8 +168,8 @@ static int lt7182s_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lt7182s_id[] = {
-	{ "lt7182s" },
-	{}
+	{ .name = "lt7182s" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lt7182s_id);
 
diff --git a/drivers/hwmon/pmbus/ltc2978.c b/drivers/hwmon/pmbus/ltc2978.c
index d69a5e675e80e..10877b0867fd6 100644
--- a/drivers/hwmon/pmbus/ltc2978.c
+++ b/drivers/hwmon/pmbus/ltc2978.c
@@ -538,36 +538,36 @@ static int ltc2978_write_word_data(struct i2c_client *client, int page,
 }
 
 static const struct i2c_device_id ltc2978_id[] = {
-	{"lt7170", lt7170},
-	{"lt7171", lt7171},
-	{"ltc2972", ltc2972},
-	{"ltc2974", ltc2974},
-	{"ltc2975", ltc2975},
-	{"ltc2977", ltc2977},
-	{"ltc2978", ltc2978},
-	{"ltc2979", ltc2979},
-	{"ltc2980", ltc2980},
-	{"ltc3880", ltc3880},
-	{"ltc3882", ltc3882},
-	{"ltc3883", ltc3883},
-	{"ltc3884", ltc3884},
-	{"ltc3886", ltc3886},
-	{"ltc3887", ltc3887},
-	{"ltc3889", ltc3889},
-	{"ltc7132", ltc7132},
-	{"ltc7841", ltc7841},
-	{"ltc7880", ltc7880},
-	{"ltm2987", ltm2987},
-	{"ltm4664", ltm4664},
-	{"ltm4673", ltm4673},
-	{"ltm4675", ltm4675},
-	{"ltm4676", ltm4676},
-	{"ltm4677", ltm4677},
-	{"ltm4678", ltm4678},
-	{"ltm4680", ltm4680},
-	{"ltm4686", ltm4686},
-	{"ltm4700", ltm4700},
-	{}
+	{ .name = "lt7170", .driver_data = lt7170 },
+	{ .name = "lt7171", .driver_data = lt7171 },
+	{ .name = "ltc2972", .driver_data = ltc2972 },
+	{ .name = "ltc2974", .driver_data = ltc2974 },
+	{ .name = "ltc2975", .driver_data = ltc2975 },
+	{ .name = "ltc2977", .driver_data = ltc2977 },
+	{ .name = "ltc2978", .driver_data = ltc2978 },
+	{ .name = "ltc2979", .driver_data = ltc2979 },
+	{ .name = "ltc2980", .driver_data = ltc2980 },
+	{ .name = "ltc3880", .driver_data = ltc3880 },
+	{ .name = "ltc3882", .driver_data = ltc3882 },
+	{ .name = "ltc3883", .driver_data = ltc3883 },
+	{ .name = "ltc3884", .driver_data = ltc3884 },
+	{ .name = "ltc3886", .driver_data = ltc3886 },
+	{ .name = "ltc3887", .driver_data = ltc3887 },
+	{ .name = "ltc3889", .driver_data = ltc3889 },
+	{ .name = "ltc7132", .driver_data = ltc7132 },
+	{ .name = "ltc7841", .driver_data = ltc7841 },
+	{ .name = "ltc7880", .driver_data = ltc7880 },
+	{ .name = "ltm2987", .driver_data = ltm2987 },
+	{ .name = "ltm4664", .driver_data = ltm4664 },
+	{ .name = "ltm4673", .driver_data = ltm4673 },
+	{ .name = "ltm4675", .driver_data = ltm4675 },
+	{ .name = "ltm4676", .driver_data = ltm4676 },
+	{ .name = "ltm4677", .driver_data = ltm4677 },
+	{ .name = "ltm4678", .driver_data = ltm4678 },
+	{ .name = "ltm4680", .driver_data = ltm4680 },
+	{ .name = "ltm4686", .driver_data = ltm4686 },
+	{ .name = "ltm4700", .driver_data = ltm4700 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc2978_id);
 
diff --git a/drivers/hwmon/pmbus/ltc3815.c b/drivers/hwmon/pmbus/ltc3815.c
index 824c16a75e2c0..0219d03adb037 100644
--- a/drivers/hwmon/pmbus/ltc3815.c
+++ b/drivers/hwmon/pmbus/ltc3815.c
@@ -143,7 +143,7 @@ static int ltc3815_write_word_data(struct i2c_client *client, int page,
 }
 
 static const struct i2c_device_id ltc3815_id[] = {
-	{"ltc3815"},
+	{ .name = "ltc3815" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ltc3815_id);
diff --git a/drivers/hwmon/pmbus/lx1308.c b/drivers/hwmon/pmbus/lx1308.c
new file mode 100644
index 0000000000000..0c453393796bf
--- /dev/null
+++ b/drivers/hwmon/pmbus/lx1308.c
@@ -0,0 +1,204 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include "pmbus.h"
+
+#define LX1308_MFR_IOUT_OCP3_FAULT	0xBE
+#define LX1308_MFR_IOUT_OCP3_WARN	0xBF
+
+/*
+ * Decode a Linear11-encoded word to an integer value.
+ * Linear11 format: bits[15:11] = signed 5-bit exponent,
+ * bits[10:0] = signed 11-bit mantissa. Result = mant * 2^exp.
+ */
+static inline int linear11_to_int(u16 word)
+{
+	s16 exp = ((s16)word) >> 11;
+	s16 mant = ((s16)((word & 0x7ff) << 5)) >> 5;
+
+	return (exp >= 0) ? (int)((u32)mant << exp) : (mant >> -exp);
+}
+
+static int lx1308_read_word_data(struct i2c_client *client, int page,
+				 int phase, int reg)
+{
+	int ret;
+
+	if (page > 0)
+		return -ENXIO;
+
+	switch (reg) {
+	/*
+	 * The LX1308 OCP3 registers (slow OCP, 60ms delay) use a
+	 * manufacturer-specific U8.0 format. Read the byte value N and present
+	 * it as a Linear11 word with exponent 0.
+	 */
+	case PMBUS_IOUT_OC_FAULT_LIMIT:
+		ret = i2c_smbus_read_byte_data(client, LX1308_MFR_IOUT_OCP3_FAULT);
+		if (ret < 0)
+			break;
+		ret &= 0x7FF;
+		break;
+
+	case PMBUS_IOUT_OC_WARN_LIMIT:
+		ret = i2c_smbus_read_byte_data(client, LX1308_MFR_IOUT_OCP3_WARN);
+		if (ret < 0)
+			break;
+		ret &= 0x7FF;
+		break;
+
+	/*
+	 * The following registers are not implemented by the LX1308. Return
+	 * -ENXIO to suppress the corresponding sysfs attributes.
+	 */
+	case PMBUS_IIN_OC_WARN_LIMIT:
+	case PMBUS_IIN_OC_FAULT_LIMIT:
+	case PMBUS_IOUT_UC_FAULT_LIMIT:
+	case PMBUS_PIN_OP_WARN_LIMIT:
+	case PMBUS_POUT_OP_WARN_LIMIT:
+	case PMBUS_UT_WARN_LIMIT:
+	case PMBUS_UT_FAULT_LIMIT:
+	case PMBUS_MFR_IIN_MAX:
+	case PMBUS_MFR_IOUT_MAX:
+	case PMBUS_MFR_VIN_MIN:
+	case PMBUS_MFR_VIN_MAX:
+	case PMBUS_MFR_VOUT_MIN:
+	case PMBUS_MFR_VOUT_MAX:
+	case PMBUS_MFR_PIN_MAX:
+	case PMBUS_MFR_POUT_MAX:
+	case PMBUS_MFR_MAX_TEMP_1:
+		ret = -ENXIO;
+		break;
+
+	default:
+		ret = -ENODATA;
+		break;
+	}
+
+	return ret;
+}
+
+static int lx1308_write_word_data(struct i2c_client *client, int page,
+				  int reg, u16 word)
+{
+	int ret;
+
+	if (page > 0)
+		return -ENXIO;
+
+	switch (reg) {
+	case PMBUS_IOUT_OC_FAULT_LIMIT:
+		/*
+		 * Decode Linear11 word from pmbus_core back to a plain integer
+		 * and write as the U8.0 byte the device expects.
+		 */
+		ret = i2c_smbus_write_byte_data(client, LX1308_MFR_IOUT_OCP3_FAULT,
+						clamp_val(linear11_to_int(word), 0, 255));
+		break;
+
+	case PMBUS_IOUT_OC_WARN_LIMIT:
+		ret = i2c_smbus_write_byte_data(client, LX1308_MFR_IOUT_OCP3_WARN,
+						clamp_val(linear11_to_int(word), 0, 255));
+		break;
+
+	default:
+		ret = -ENODATA;
+		break;
+	}
+
+	return ret;
+}
+
+static struct pmbus_driver_info lx1308_info = {
+	.pages = 1,
+	.format[PSC_VOLTAGE_IN] = linear,
+	.format[PSC_VOLTAGE_OUT] = linear,
+	.format[PSC_CURRENT_IN] = linear,
+	.format[PSC_CURRENT_OUT] = linear,
+	.format[PSC_POWER] = linear,
+	.format[PSC_TEMPERATURE] = linear,
+
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_STATUS_VOUT
+		| PMBUS_HAVE_IIN | PMBUS_HAVE_IOUT | PMBUS_HAVE_STATUS_IOUT
+		| PMBUS_HAVE_PIN | PMBUS_HAVE_POUT
+		| PMBUS_HAVE_TEMP | PMBUS_HAVE_STATUS_TEMP
+		| PMBUS_HAVE_STATUS_INPUT,
+
+	.read_word_data  = lx1308_read_word_data,
+	.write_word_data = lx1308_write_word_data,
+};
+
+static const struct of_device_id lx1308_of_match[] = {
+	{ .compatible = "luxshare,lx1308" },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(of, lx1308_of_match);
+
+static const struct i2c_device_id lx1308_id[] = {
+	{ "lx1308" },
+	{ }
+};
+
+MODULE_DEVICE_TABLE(i2c, lx1308_id);
+
+static int lx1308_probe(struct i2c_client *client)
+{
+	u8 buf[I2C_SMBUS_BLOCK_MAX + 1];
+	const struct i2c_device_id *mid;
+	int ret;
+
+	ret = i2c_smbus_read_block_data(client, PMBUS_MFR_ID, buf);
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret,
+				     "Failed to read manufacturer id\n");
+	buf[ret] = '\0';
+
+	if (ret != 12 || strncmp(buf, "LUXSHARE", 8))
+		return dev_err_probe(&client->dev, -ENODEV,
+				     "Unsupported Manufacturer ID '%s'\n", buf);
+
+	ret = i2c_smbus_read_block_data(client, PMBUS_MFR_MODEL, buf);
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret,
+				     "Failed to read Manufacturer Model\n");
+	buf[ret] = '\0';
+
+	for (mid = lx1308_id; mid->name[0]; mid++) {
+		if (!strncasecmp(mid->name, buf, strlen(mid->name)))
+			break;
+	}
+	if (!mid->name[0])
+		return dev_err_probe(&client->dev, -ENODEV,
+				     "Unsupported Manufacturer Model '%s'\n", buf);
+
+	ret = i2c_smbus_read_block_data(client, PMBUS_MFR_REVISION, buf);
+	if (ret < 0)
+		return dev_err_probe(&client->dev, ret,
+				     "Failed to read Manufacturer Revision\n");
+	buf[ret] = '\0';
+
+	if (ret != 12 || buf[0] != 'V')
+		return dev_err_probe(&client->dev, -ENODEV,
+				     "Unsupported Manufacturer Revision '%s'\n", buf);
+	return pmbus_do_probe(client, &lx1308_info);
+}
+
+static struct i2c_driver lx1308_driver = {
+	.driver = {
+		.name = "lx1308",
+		.of_match_table = lx1308_of_match,
+	},
+	.probe = lx1308_probe,
+	.id_table = lx1308_id,
+};
+
+module_i2c_driver(lx1308_driver);
+
+MODULE_AUTHOR("Brian Chiang <chiang.brian@inventec.com>");
+MODULE_DESCRIPTION("PMBus driver for Luxshare LX1308");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/pmbus/max15301.c b/drivers/hwmon/pmbus/max15301.c
index d5810b88ea8d8..4c36f1ea27ee6 100644
--- a/drivers/hwmon/pmbus/max15301.c
+++ b/drivers/hwmon/pmbus/max15301.c
@@ -23,10 +23,10 @@
 #include "pmbus.h"
 
 static const struct i2c_device_id max15301_id[] = {
-	{ "bmr461" },
-	{ "max15301" },
-	{ "max15303" },
-	{}
+	{ .name = "bmr461" },
+	{ .name = "max15301" },
+	{ .name = "max15303" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max15301_id);
 
diff --git a/drivers/hwmon/pmbus/max16064.c b/drivers/hwmon/pmbus/max16064.c
index eb84915c2a83d..351a1f5359994 100644
--- a/drivers/hwmon/pmbus/max16064.c
+++ b/drivers/hwmon/pmbus/max16064.c
@@ -91,8 +91,8 @@ static int max16064_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max16064_id[] = {
-	{"max16064"},
-	{}
+	{ .name = "max16064" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, max16064_id);
diff --git a/drivers/hwmon/pmbus/max16601.c b/drivers/hwmon/pmbus/max16601.c
index 36dc13424d929..3dd1f6fd003b8 100644
--- a/drivers/hwmon/pmbus/max16601.c
+++ b/drivers/hwmon/pmbus/max16601.c
@@ -263,11 +263,11 @@ static void max16601_remove(void *_data)
 }
 
 static const struct i2c_device_id max16601_id[] = {
-	{"max16508", max16508},
-	{"max16600", max16600},
-	{"max16601", max16601},
-	{"max16602", max16602},
-	{}
+	{ .name = "max16508", .driver_data = max16508 },
+	{ .name = "max16600", .driver_data = max16600 },
+	{ .name = "max16601", .driver_data = max16601 },
+	{ .name = "max16602", .driver_data = max16602 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max16601_id);
 
diff --git a/drivers/hwmon/pmbus/max17616.c b/drivers/hwmon/pmbus/max17616.c
index 1d4a0ddb95bb6..744fa5aefe938 100644
--- a/drivers/hwmon/pmbus/max17616.c
+++ b/drivers/hwmon/pmbus/max17616.c
@@ -46,7 +46,7 @@ static int max17616_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max17616_id[] = {
-	{ "max17616" },
+	{ .name = "max17616" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max17616_id);
diff --git a/drivers/hwmon/pmbus/max20730.c b/drivers/hwmon/pmbus/max20730.c
index fe03164788dfa..4031f894e6ae7 100644
--- a/drivers/hwmon/pmbus/max20730.c
+++ b/drivers/hwmon/pmbus/max20730.c
@@ -751,11 +751,11 @@ static int max20730_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max20730_id[] = {
-	{ "max20710", max20710 },
-	{ "max20730", max20730 },
-	{ "max20734", max20734 },
-	{ "max20743", max20743 },
-	{ },
+	{ .name = "max20710", .driver_data = max20710 },
+	{ .name = "max20730", .driver_data = max20730 },
+	{ .name = "max20734", .driver_data = max20734 },
+	{ .name = "max20743", .driver_data = max20743 },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, max20730_id);
diff --git a/drivers/hwmon/pmbus/max20751.c b/drivers/hwmon/pmbus/max20751.c
index ac8c431221338..e85676433dca1 100644
--- a/drivers/hwmon/pmbus/max20751.c
+++ b/drivers/hwmon/pmbus/max20751.c
@@ -32,8 +32,8 @@ static int max20751_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max20751_id[] = {
-	{"max20751"},
-	{}
+	{ .name = "max20751" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, max20751_id);
diff --git a/drivers/hwmon/pmbus/max20830.c b/drivers/hwmon/pmbus/max20830.c
new file mode 100644
index 0000000000000..cb2c23672166d
--- /dev/null
+++ b/drivers/hwmon/pmbus/max20830.c
@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Hardware monitoring driver for Analog Devices MAX20830
+ *
+ * Copyright (C) 2026 Analog Devices, Inc.
+ */
+
+#include <linux/errno.h>
+#include <linux/i2c.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/string.h>
+#include "pmbus.h"
+
+#define MAX20830_IC_DEVICE_ID_LENGTH	9
+
+static struct pmbus_driver_info max20830_info = {
+	.pages = 1,
+	.format[PSC_VOLTAGE_IN] = linear,
+	.format[PSC_VOLTAGE_OUT] = linear,
+	.format[PSC_CURRENT_OUT] = linear,
+	.format[PSC_TEMPERATURE] = linear,
+	.func[0] = PMBUS_HAVE_VIN | PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT |
+		PMBUS_HAVE_TEMP |
+		PMBUS_HAVE_STATUS_VOUT | PMBUS_HAVE_STATUS_IOUT |
+		PMBUS_HAVE_STATUS_INPUT | PMBUS_HAVE_STATUS_TEMP,
+};
+
+static int max20830_probe(struct i2c_client *client)
+{
+	u8 buf[I2C_SMBUS_BLOCK_MAX + 1] = {};
+	int ret;
+
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_READ_BLOCK_DATA) &&
+	    !i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_READ_I2C_BLOCK))
+		return -ENODEV;
+
+	/*
+	 * Use i2c_smbus_read_block_data() if supported, otherwise fall back
+	 * to i2c_smbus_read_i2c_block_data() to support I2C controllers
+	 * which do not support SMBus block reads.
+	 */
+	if (i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_READ_BLOCK_DATA)) {
+		/* Reads 9 Data bytes from MAX20830 */
+		ret = i2c_smbus_read_block_data(client, PMBUS_IC_DEVICE_ID, buf);
+		if (ret < 0)
+			return dev_err_probe(&client->dev, ret,
+					     "Failed to read IC_DEVICE_ID\n");
+	} else {
+		/* Reads 1 length byte + 9 Data bytes from MAX20830 */
+		ret = i2c_smbus_read_i2c_block_data(client, PMBUS_IC_DEVICE_ID,
+						    MAX20830_IC_DEVICE_ID_LENGTH + 1,
+						    buf);
+		if (ret < 0)
+			return dev_err_probe(&client->dev, ret,
+					     "Failed to read IC_DEVICE_ID\n");
+		/*
+		 * Moves data forward, removing the length byte, this is to
+		 * match the format of i2c_smbus_read_block_data().
+		 * Also adjust return value to reflect length byte removal.
+		 */
+		memmove(buf, buf + 1, MAX20830_IC_DEVICE_ID_LENGTH);
+		ret = ret - 1;
+	}
+
+	/*
+	 * MAX20830 IC_DEVICE_ID sends string data "MAX20830\0".
+	 * Return value should at least be 9 bytes of data.
+	 */
+	if (ret < MAX20830_IC_DEVICE_ID_LENGTH)
+		return dev_err_probe(&client->dev, -ENODEV,
+				     "IC_DEVICE_ID too short: expected at least 9 bytes, got %d\n",
+				     ret);
+
+	/* 9 bytes of data, buf[0]-buf[7] = "MAX20830", buf[8] = '\0' */
+	buf[MAX20830_IC_DEVICE_ID_LENGTH - 1] = '\0';
+	if (strncmp(buf, "MAX20830", MAX20830_IC_DEVICE_ID_LENGTH - 1))
+		return dev_err_probe(&client->dev, -ENODEV,
+				     "Unsupported device: '%s'\n", buf);
+
+	return pmbus_do_probe(client, &max20830_info);
+}
+
+static const struct i2c_device_id max20830_id[] = {
+	{"max20830"},
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, max20830_id);
+
+static const struct of_device_id max20830_of_match[] = {
+	{ .compatible = "adi,max20830" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, max20830_of_match);
+
+static struct i2c_driver max20830_driver = {
+	.driver = {
+		.name = "max20830",
+		.of_match_table = max20830_of_match,
+	},
+	.probe = max20830_probe,
+	.id_table = max20830_id,
+};
+
+module_i2c_driver(max20830_driver);
+
+MODULE_AUTHOR("Alexis Czezar Torreno <alexisczezar.torreno@analog.com>");
+MODULE_DESCRIPTION("PMBus driver for Analog Devices MAX20830");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/pmbus/max31785.c b/drivers/hwmon/pmbus/max31785.c
index 3caa76bcbeb5e..bdcce810a4792 100644
--- a/drivers/hwmon/pmbus/max31785.c
+++ b/drivers/hwmon/pmbus/max31785.c
@@ -447,10 +447,10 @@ static int max31785_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max31785_id[] = {
-	{ "max31785" },
-	{ "max31785a" },
-	{ "max31785b" },
-	{ },
+	{ .name = "max31785" },
+	{ .name = "max31785a" },
+	{ .name = "max31785b" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, max31785_id);
diff --git a/drivers/hwmon/pmbus/max34440.c b/drivers/hwmon/pmbus/max34440.c
index cc96bb22f8f5a..4525b9fc56267 100644
--- a/drivers/hwmon/pmbus/max34440.c
+++ b/drivers/hwmon/pmbus/max34440.c
@@ -644,15 +644,15 @@ static int max34440_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max34440_id[] = {
-	{"adpm12160", adpm12160},
-	{"adpm12200", adpm12200},
-	{"max34440", max34440},
-	{"max34441", max34441},
-	{"max34446", max34446},
-	{"max34451", max34451},
-	{"max34460", max34460},
-	{"max34461", max34461},
-	{}
+	{ .name = "adpm12160", .driver_data = adpm12160 },
+	{ .name = "adpm12200", .driver_data = adpm12200 },
+	{ .name = "max34440", .driver_data = max34440 },
+	{ .name = "max34441", .driver_data = max34441 },
+	{ .name = "max34446", .driver_data = max34446 },
+	{ .name = "max34451", .driver_data = max34451 },
+	{ .name = "max34460", .driver_data = max34460 },
+	{ .name = "max34461", .driver_data = max34461 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max34440_id);
 
diff --git a/drivers/hwmon/pmbus/max8688.c b/drivers/hwmon/pmbus/max8688.c
index b3a2a7492bbfc..212b28fa4bf22 100644
--- a/drivers/hwmon/pmbus/max8688.c
+++ b/drivers/hwmon/pmbus/max8688.c
@@ -171,7 +171,7 @@ static int max8688_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max8688_id[] = {
-	{"max8688"},
+	{ .name = "max8688" },
 	{ }
 };
 
diff --git a/drivers/hwmon/pmbus/mp2856.c b/drivers/hwmon/pmbus/mp2856.c
index e83c70a3583f8..3d6621e36fd39 100644
--- a/drivers/hwmon/pmbus/mp2856.c
+++ b/drivers/hwmon/pmbus/mp2856.c
@@ -54,9 +54,9 @@ static const int mp2856_max_phases[][MP2856_PAGE_NUM] = {
 };
 
 static const struct i2c_device_id mp2856_id[] = {
-	{"mp2856", mp2856},
-	{"mp2857", mp2857},
-	{}
+	{ .name = "mp2856", .driver_data = mp2856 },
+	{ .name = "mp2857", .driver_data = mp2857 },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, mp2856_id);
diff --git a/drivers/hwmon/pmbus/mp2869.c b/drivers/hwmon/pmbus/mp2869.c
index 4647892e51121..76c6017507b3b 100644
--- a/drivers/hwmon/pmbus/mp2869.c
+++ b/drivers/hwmon/pmbus/mp2869.c
@@ -632,20 +632,20 @@ static int mp2869_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mp2869_id[] = {
-	{"mp2869", 0},
-	{"mp29608", 1},
-	{"mp29612", 2},
-	{"mp29816", 3},
-	{}
+	{ .name = "mp2869" },
+	{ .name = "mp29608" },
+	{ .name = "mp29612" },
+	{ .name = "mp29816" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp2869_id);
 
-static const struct of_device_id __maybe_unused mp2869_of_match[] = {
-	{.compatible = "mps,mp2869", .data = (void *)0},
-	{.compatible = "mps,mp29608", .data = (void *)1},
-	{.compatible = "mps,mp29612", .data = (void *)2},
-	{.compatible = "mps,mp29816", .data = (void *)3},
-	{}
+static const struct of_device_id mp2869_of_match[] = {
+	{ .compatible = "mps,mp2869" },
+	{ .compatible = "mps,mp29608" },
+	{ .compatible = "mps,mp29612" },
+	{ .compatible = "mps,mp29816" },
+	{ }
 };
 MODULE_DEVICE_TABLE(of, mp2869_of_match);
 
diff --git a/drivers/hwmon/pmbus/mp2888.c b/drivers/hwmon/pmbus/mp2888.c
index 772a623ca7d0a..c5f35daa3fe11 100644
--- a/drivers/hwmon/pmbus/mp2888.c
+++ b/drivers/hwmon/pmbus/mp2888.c
@@ -378,8 +378,8 @@ static int mp2888_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mp2888_id[] = {
-	{"mp2888"},
-	{}
+	{ .name = "mp2888" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, mp2888_id);
diff --git a/drivers/hwmon/pmbus/mp2891.c b/drivers/hwmon/pmbus/mp2891.c
index f8f4c91ec23cc..316c1fc0d6ccf 100644
--- a/drivers/hwmon/pmbus/mp2891.c
+++ b/drivers/hwmon/pmbus/mp2891.c
@@ -572,7 +572,7 @@ static int mp2891_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mp2891_id[] = {
-	{ "mp2891" },
+	{ .name = "mp2891" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp2891_id);
diff --git a/drivers/hwmon/pmbus/mp2925.c b/drivers/hwmon/pmbus/mp2925.c
index ad094842cf2d7..0a58b1ffd7910 100644
--- a/drivers/hwmon/pmbus/mp2925.c
+++ b/drivers/hwmon/pmbus/mp2925.c
@@ -305,9 +305,9 @@ static int mp2925_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mp2925_id[] = {
-	{"mp2925"},
-	{"mp2929"},
-	{}
+	{ .name = "mp2925" },
+	{ .name = "mp2929" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp2925_id);
 
diff --git a/drivers/hwmon/pmbus/mp29502.c b/drivers/hwmon/pmbus/mp29502.c
index 7241373f15577..afc5e8c07e259 100644
--- a/drivers/hwmon/pmbus/mp29502.c
+++ b/drivers/hwmon/pmbus/mp29502.c
@@ -642,8 +642,8 @@ static int mp29502_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mp29502_id[] = {
-	{"mp29502", 0},
-	{}
+	{ .name = "mp29502", .driver_data = 0 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp29502_id);
 
diff --git a/drivers/hwmon/pmbus/mp2975.c b/drivers/hwmon/pmbus/mp2975.c
index d0bc47b12cb07..dca7e2fbcb441 100644
--- a/drivers/hwmon/pmbus/mp2975.c
+++ b/drivers/hwmon/pmbus/mp2975.c
@@ -1082,10 +1082,10 @@ static const struct of_device_id mp2975_of_match[] = {
 MODULE_DEVICE_TABLE(of, mp2975_of_match);
 
 static const struct i2c_device_id mp2975_id[] = {
-	{"mp2971", (kernel_ulong_t)&mp2975_ddinfo[mp2971]},
-	{"mp2973", (kernel_ulong_t)&mp2975_ddinfo[mp2973]},
-	{"mp2975", (kernel_ulong_t)&mp2975_ddinfo[mp2975]},
-	{}
+	{ .name = "mp2971", .driver_data = (kernel_ulong_t)&mp2975_ddinfo[mp2971] },
+	{ .name = "mp2973", .driver_data = (kernel_ulong_t)&mp2975_ddinfo[mp2973] },
+	{ .name = "mp2975", .driver_data = (kernel_ulong_t)&mp2975_ddinfo[mp2975] },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp2975_id);
 
diff --git a/drivers/hwmon/pmbus/mp2985.c b/drivers/hwmon/pmbus/mp2985.c
new file mode 100644
index 0000000000000..b0a4796ca8ce0
--- /dev/null
+++ b/drivers/hwmon/pmbus/mp2985.c
@@ -0,0 +1,402 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Hardware monitoring driver for MPS Multi-phase Digital VR Controllers(MP2985)
+ *
+ * Copyright (C) 2026 MPS
+ */
+
+#include <linux/bitfield.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include "pmbus.h"
+
+/*
+ * Vender specific register READ_PIN_EST(0x93), READ_IIN_EST(0x8E),
+ * MFR_VR_MULTI_CONFIG_R1(0x0D) and MFR_VR_MULTI_CONFIG_R2(0x1D).
+ * The READ_PIN_EST is used to read pin telemetry, the READ_IIN_EST
+ * is used to read iin telemetry and the MFR_VR_MULTI_CONFIG_R1,
+ * MFR_VR_MULTI_CONFIG_R2 are used to obtain vid scale.
+ */
+#define READ_PIN_EST	0x93
+#define READ_IIN_EST	0x8E
+#define MFR_VR_MULTI_CONFIG_R1	0x0D
+#define MFR_VR_MULTI_CONFIG_R2	0x1D
+
+#define MP2985_VOUT_DIV	64
+#define MP2985_VOUT_OVUV_UINT	125
+#define MP2985_VOUT_OVUV_DIV	64
+
+#define MP2985_PAGE_NUM	2
+
+#define MP2985_RAIL1_FUNC	(PMBUS_HAVE_VIN | PMBUS_HAVE_PIN | \
+							 PMBUS_HAVE_VOUT | PMBUS_HAVE_IOUT | \
+							 PMBUS_HAVE_POUT | PMBUS_HAVE_TEMP | \
+							 PMBUS_HAVE_STATUS_VOUT | \
+							 PMBUS_HAVE_STATUS_IOUT | \
+							 PMBUS_HAVE_STATUS_TEMP | \
+							 PMBUS_HAVE_STATUS_INPUT)
+
+#define MP2985_RAIL2_FUNC	(PMBUS_HAVE_PIN | PMBUS_HAVE_VOUT | \
+							 PMBUS_HAVE_IOUT | PMBUS_HAVE_POUT | \
+							 PMBUS_HAVE_TEMP | PMBUS_HAVE_IIN | \
+							 PMBUS_HAVE_STATUS_VOUT | \
+							 PMBUS_HAVE_STATUS_IOUT | \
+							 PMBUS_HAVE_STATUS_TEMP | \
+							 PMBUS_HAVE_STATUS_INPUT)
+
+struct mp2985_data {
+	struct pmbus_driver_info info;
+	int vout_scale[MP2985_PAGE_NUM];
+	int vid_offset[MP2985_PAGE_NUM];
+};
+
+#define to_mp2985_data(x) container_of(x, struct mp2985_data, info)
+
+static u16 mp2985_linear_exp_transfer(u16 word, u16 expect_exponent)
+{
+	s16 exponent, mantissa, target_exponent;
+
+	exponent = ((s16)word) >> 11;
+	mantissa = ((s16)((word & 0x7ff) << 5)) >> 5;
+	target_exponent = (s16)((expect_exponent & 0x1f) << 11) >> 11;
+
+	/*
+	 * The MP2985 does not support negtive limit value, if a negtive
+	 * limit value is written, the limit value will become to 0. And
+	 * the maximum positive limit value is limitted to 0x3FF.
+	 */
+	if (mantissa < 0) {
+		mantissa = 0;
+	} else {
+		if (exponent > target_exponent) {
+			mantissa = (1023 >> (exponent - target_exponent)) >= mantissa ?
+						mantissa << (exponent - target_exponent) :
+						0x3FF;
+		} else {
+			mantissa = clamp_val(mantissa >> (target_exponent - exponent),
+					     0, 0x3FF);
+		}
+	}
+
+	return mantissa | ((expect_exponent << 11) & 0xf800);
+}
+
+static int mp2985_read_byte_data(struct i2c_client *client, int page, int reg)
+{
+	int ret;
+
+	switch (reg) {
+	case PMBUS_VOUT_MODE:
+		/*
+		 * The MP2985 does not follow standard PMBus protocol completely,
+		 * and the calculation of vout in this driver is based on direct
+		 * format. As a result, the format of vout is enforced to direct.
+		 */
+		ret = PB_VOUT_MODE_DIRECT;
+		break;
+	default:
+		ret = -ENODATA;
+		break;
+	}
+
+	return ret;
+}
+
+static int mp2985_read_word_data(struct i2c_client *client, int page, int phase,
+				 int reg)
+{
+	const struct pmbus_driver_info *info = pmbus_get_driver_info(client);
+	struct mp2985_data *data = to_mp2985_data(info);
+	int ret;
+
+	switch (reg) {
+	case PMBUS_READ_VOUT:
+		ret = pmbus_read_word_data(client, page, phase, reg);
+		if (ret < 0)
+			return ret;
+
+		/*
+		 * The MP2985 supports three vout mode, direct, linear11 and vid mode.
+		 * In vid mode, the MP2985 vout telemetry has 49 vid step offset, but
+		 * PMBUS_VOUT_OV_FAULT_LIMIT and PMBUS_VOUT_UV_FAULT_LIMIT do not take
+		 * this into consideration, their resolution are 1.953125mV/LSB, as a
+		 * result, format[PSC_VOLTAGE_OUT] can not be set to vid mode directly.
+		 * Adding extra vid_offset variable for vout telemetry.
+		 */
+		ret = clamp_val(DIV_ROUND_CLOSEST(((ret & GENMASK(11, 0)) +
+									data->vid_offset[page]) *
+							data->vout_scale[page], MP2985_VOUT_DIV),
+							0, 0x7FFF);
+		break;
+	case PMBUS_READ_IIN:
+		/*
+		 * The MP2985 has standard PMBUS_READ_IIN register(0x89), but this is
+		 * not used to read the input current of per rail. The input current
+		 * is read through the vender redefined register READ_IIN_EST(0x8E).
+		 */
+		ret = pmbus_read_word_data(client, page, phase, READ_IIN_EST);
+		break;
+	case PMBUS_READ_PIN:
+		/*
+		 * The MP2985 has standard PMBUS_READ_PIN register(0x97), but this
+		 * is not used to read the input power of per rail. The input power
+		 * of per rail is read through the vender redefined register
+		 * READ_PIN_EST(0x93).
+		 */
+		ret = pmbus_read_word_data(client, page, phase, READ_PIN_EST);
+		break;
+	case PMBUS_VOUT_OV_FAULT_LIMIT:
+	case PMBUS_VOUT_UV_FAULT_LIMIT:
+		ret = pmbus_read_word_data(client, page, phase, reg);
+		if (ret < 0)
+			return ret;
+
+		ret = DIV_ROUND_CLOSEST((ret & GENMASK(11, 0)) * MP2985_VOUT_OVUV_UINT,
+					MP2985_VOUT_OVUV_DIV);
+		break;
+	case PMBUS_STATUS_WORD:
+	case PMBUS_READ_VIN:
+	case PMBUS_READ_IOUT:
+	case PMBUS_READ_POUT:
+	case PMBUS_READ_TEMPERATURE_1:
+	case PMBUS_VIN_OV_FAULT_LIMIT:
+	case PMBUS_VIN_OV_WARN_LIMIT:
+	case PMBUS_VIN_UV_WARN_LIMIT:
+	case PMBUS_VIN_UV_FAULT_LIMIT:
+	case PMBUS_IOUT_OC_FAULT_LIMIT:
+	case PMBUS_IOUT_OC_WARN_LIMIT:
+	case PMBUS_OT_FAULT_LIMIT:
+	case PMBUS_OT_WARN_LIMIT:
+		/*
+		 * These register is not explicitly handled by the driver,
+		 * as a result, return -ENODATA directly.
+		 */
+		ret = -ENODATA;
+		break;
+	default:
+		/*
+		 * The MP2985 do not support other telemetry and limit value
+		 * reading, so, return -EINVAL directly.
+		 */
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int mp2985_write_word_data(struct i2c_client *client, int page, int reg,
+				  u16 word)
+{
+	int ret;
+
+	switch (reg) {
+	case PMBUS_VIN_OV_FAULT_LIMIT:
+	case PMBUS_VIN_OV_WARN_LIMIT:
+	case PMBUS_VIN_UV_WARN_LIMIT:
+	case PMBUS_VIN_UV_FAULT_LIMIT:
+		/*
+		 * The PMBUS_VIN_OV_FAULT_LIMIT, PMBUS_VIN_OV_WARN_LIMIT,
+		 * PMBUS_VIN_UV_WARN_LIMIT and PMBUS_VIN_UV_FAULT_LIMIT
+		 * of MP2985 is linear11 format, and the exponent is a
+		 * constant value(5'b11101), so the exponent of word
+		 * parameter should be converted to 5'b11101(0x1D).
+		 */
+		ret = pmbus_write_word_data(client, page, reg,
+					    mp2985_linear_exp_transfer(word, 0x1D));
+		break;
+	case PMBUS_VOUT_OV_FAULT_LIMIT:
+	case PMBUS_VOUT_UV_FAULT_LIMIT:
+		/*
+		 * The bit0-bit11 is the limit value, and bit12-bit15
+		 * should not be changed.
+		 */
+		ret = pmbus_read_word_data(client, page, 0xff, reg);
+		if (ret < 0)
+			return ret;
+
+		ret = pmbus_write_word_data(client, page, reg,
+					    (ret & ~GENMASK(11, 0)) |
+				clamp_val(DIV_ROUND_CLOSEST(word * MP2985_VOUT_OVUV_DIV,
+							    MP2985_VOUT_OVUV_UINT), 0, 0xFFF));
+		break;
+	case PMBUS_OT_FAULT_LIMIT:
+	case PMBUS_OT_WARN_LIMIT:
+		/*
+		 * The PMBUS_OT_FAULT_LIMIT and PMBUS_OT_WARN_LIMIT of
+		 * MP2985 is linear11 format, and the exponent is a
+		 * constant value(5'b00000), so the exponent of word
+		 * parameter should be converted to 5'b00000.
+		 */
+		ret = pmbus_write_word_data(client, page, reg,
+					    mp2985_linear_exp_transfer(word, 0x00));
+		break;
+	case PMBUS_IOUT_OC_FAULT_LIMIT:
+	case PMBUS_IOUT_OC_WARN_LIMIT:
+		/*
+		 * The PMBUS_IOUT_OC_FAULT_LIMIT and PMBUS_IOUT_OC_WARN_LIMIT
+		 * of MP2985 is linear11 format, and the exponent can not be
+		 * changed.
+		 */
+		ret = pmbus_read_word_data(client, page, 0xff, reg);
+		if (ret < 0)
+			return ret;
+
+		ret = pmbus_write_word_data(client, page, reg,
+					    mp2985_linear_exp_transfer(word,
+								       FIELD_GET(GENMASK(15, 11),
+										 ret)));
+		break;
+	default:
+		/*
+		 * The MP2985 do not support other limit value configuration,
+		 * so, return -EINVAL directly.
+		 */
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int
+mp2985_identify_vout_scale(struct i2c_client *client, struct pmbus_driver_info *info,
+			   int page)
+{
+	struct mp2985_data *data = to_mp2985_data(info);
+	int ret;
+
+	ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, page);
+	if (ret < 0)
+		return ret;
+
+	ret = i2c_smbus_read_byte_data(client, PMBUS_VOUT_MODE);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * The MP2985 supports three vout mode. If PMBUS_VOUT_MODE
+	 * bit5 is 1, it is vid mode. If PMBUS PMBUS_VOUT_MODE bit4
+	 * is 1, it is linear11 mode, the vout scale is 1.953125mv/LSB.
+	 * If PMBUS PMBUS_VOUT_MODE bit6 is 1, it is direct mode, the
+	 * vout scale is 1mv/LSB. In vid mode, the MP2985 vout telemetry
+	 * has 49 vid step offset.
+	 */
+	if (FIELD_GET(BIT(5), ret)) {
+		ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, 2);
+		if (ret < 0)
+			return ret;
+
+		ret = i2c_smbus_read_word_data(client, page == 0 ?
+						MFR_VR_MULTI_CONFIG_R1 :
+						MFR_VR_MULTI_CONFIG_R2);
+		if (ret < 0)
+			return ret;
+
+		if (page == 0) {
+			if (FIELD_GET(BIT(4), ret))
+				data->vout_scale[page] = 320;
+			else
+				data->vout_scale[page] = 640;
+		} else {
+			if (FIELD_GET(BIT(3), ret))
+				data->vout_scale[page] = 320;
+			else
+				data->vout_scale[page] = 640;
+		}
+
+		data->vid_offset[page] = 49;
+
+		/*
+		 * For vid mode, the MP2985 should be changed to page 2
+		 * to obtain vout scale value, this may confuse the PMBus
+		 * core. To avoid this, switch back to the previous page
+		 * again.
+		 */
+		ret = i2c_smbus_write_byte_data(client, PMBUS_PAGE, page);
+		if (ret < 0)
+			return ret;
+	} else if (FIELD_GET(BIT(4), ret)) {
+		data->vout_scale[page] = 125;
+		data->vid_offset[page] = 0;
+	} else {
+		data->vout_scale[page] = 64;
+		data->vid_offset[page] = 0;
+	}
+
+	return 0;
+}
+
+static int mp2985_identify(struct i2c_client *client, struct pmbus_driver_info *info)
+{
+	int ret;
+
+	ret = mp2985_identify_vout_scale(client, info, 0);
+	if (ret < 0)
+		return ret;
+
+	return mp2985_identify_vout_scale(client, info, 1);
+}
+
+static struct pmbus_driver_info mp2985_info = {
+	.pages = MP2985_PAGE_NUM,
+	.format[PSC_VOLTAGE_IN] = linear,
+	.format[PSC_CURRENT_IN] = linear,
+	.format[PSC_CURRENT_OUT] = linear,
+	.format[PSC_POWER] = linear,
+	.format[PSC_TEMPERATURE] = linear,
+	.format[PSC_VOLTAGE_OUT] = direct,
+
+	.m[PSC_VOLTAGE_OUT] = 1,
+	.R[PSC_VOLTAGE_OUT] = 3,
+	.b[PSC_VOLTAGE_OUT] = 0,
+
+	.func[0] = MP2985_RAIL1_FUNC,
+	.func[1] = MP2985_RAIL2_FUNC,
+	.read_word_data = mp2985_read_word_data,
+	.read_byte_data = mp2985_read_byte_data,
+	.write_word_data = mp2985_write_word_data,
+	.identify = mp2985_identify,
+};
+
+static int mp2985_probe(struct i2c_client *client)
+{
+	struct mp2985_data *data;
+
+	data = devm_kzalloc(&client->dev, sizeof(struct mp2985_data), GFP_KERNEL);
+	if (!data)
+		return -ENOMEM;
+
+	memcpy(&data->info, &mp2985_info, sizeof(mp2985_info));
+
+	return pmbus_do_probe(client, &data->info);
+}
+
+static const struct i2c_device_id mp2985_id[] = {
+	{"mp2985", 0},
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, mp2985_id);
+
+static const struct of_device_id __maybe_unused mp2985_of_match[] = {
+	{.compatible = "mps,mp2985"},
+	{}
+};
+MODULE_DEVICE_TABLE(of, mp2985_of_match);
+
+static struct i2c_driver mp2985_driver = {
+	.driver = {
+		.name = "mp2985",
+		.of_match_table = mp2985_of_match,
+	},
+	.probe = mp2985_probe,
+	.id_table = mp2985_id,
+};
+
+module_i2c_driver(mp2985_driver);
+
+MODULE_AUTHOR("Wensheng Wang <wenswang@yeah.net>");
+MODULE_DESCRIPTION("PMBus driver for MPS MP2985 device");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("PMBUS");
diff --git a/drivers/hwmon/pmbus/mp2993.c b/drivers/hwmon/pmbus/mp2993.c
index 81c84fc8ed47d..3a6a6c55a1f17 100644
--- a/drivers/hwmon/pmbus/mp2993.c
+++ b/drivers/hwmon/pmbus/mp2993.c
@@ -233,7 +233,7 @@ static int mp2993_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mp2993_id[] = {
-	{ "mp2993" },
+	{ .name = "mp2993" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp2993_id);
diff --git a/drivers/hwmon/pmbus/mp5920.c b/drivers/hwmon/pmbus/mp5920.c
index 319ae2721bcf4..b803f3ddf8ea1 100644
--- a/drivers/hwmon/pmbus/mp5920.c
+++ b/drivers/hwmon/pmbus/mp5920.c
@@ -66,7 +66,7 @@ static const struct of_device_id mp5920_of_match[] = {
 MODULE_DEVICE_TABLE(of, mp5920_of_match);
 
 static const struct i2c_device_id mp5920_id[] = {
-	{ "mp5920" },
+	{ .name = "mp5920" },
 	{ }
 };
 
diff --git a/drivers/hwmon/pmbus/mp5926.c b/drivers/hwmon/pmbus/mp5926.c
index f0d1b30c70134..a40647472b160 100644
--- a/drivers/hwmon/pmbus/mp5926.c
+++ b/drivers/hwmon/pmbus/mp5926.c
@@ -157,8 +157,8 @@ static int mp5926_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mp5926_id[] = {
-	{ "mp5926", 0 },
-	{}
+	{ .name = "mp5926", .driver_data = 0 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp5926_id);
 
diff --git a/drivers/hwmon/pmbus/mp5990.c b/drivers/hwmon/pmbus/mp5990.c
index 9a4ee79712cfc..f8a5d1b42ff7a 100644
--- a/drivers/hwmon/pmbus/mp5990.c
+++ b/drivers/hwmon/pmbus/mp5990.c
@@ -148,8 +148,8 @@ static struct pmbus_driver_info mp5998_info = {
 };
 
 static const struct i2c_device_id mp5990_id[] = {
-	{"mp5990", mp5990},
-	{"mp5998", mp5998},
+	{ .name = "mp5990", .driver_data = mp5990 },
+	{ .name = "mp5998", .driver_data = mp5998 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp5990_id);
diff --git a/drivers/hwmon/pmbus/mp9941.c b/drivers/hwmon/pmbus/mp9941.c
index 42ca6748777af..d9049d326c809 100644
--- a/drivers/hwmon/pmbus/mp9941.c
+++ b/drivers/hwmon/pmbus/mp9941.c
@@ -291,7 +291,7 @@ static int mp9941_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mp9941_id[] = {
-	{ "mp9941" },
+	{ .name = "mp9941" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp9941_id);
diff --git a/drivers/hwmon/pmbus/mp9945.c b/drivers/hwmon/pmbus/mp9945.c
index 34822e0de8128..199480d855157 100644
--- a/drivers/hwmon/pmbus/mp9945.c
+++ b/drivers/hwmon/pmbus/mp9945.c
@@ -215,8 +215,8 @@ static int mp9945_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mp9945_id[] = {
-	{"mp9945"},
-	{}
+	{ .name = "mp9945" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mp9945_id);
 
diff --git a/drivers/hwmon/pmbus/mpq7932.c b/drivers/hwmon/pmbus/mpq7932.c
index 8f10e37a7a767..f49610adff89c 100644
--- a/drivers/hwmon/pmbus/mpq7932.c
+++ b/drivers/hwmon/pmbus/mpq7932.c
@@ -145,9 +145,9 @@ static const struct of_device_id mpq7932_of_match[] = {
 MODULE_DEVICE_TABLE(of, mpq7932_of_match);
 
 static const struct i2c_device_id mpq7932_id[] = {
-	{ "mpq2286", },
-	{ "mpq7932", },
-	{ },
+	{ .name = "mpq2286" },
+	{ .name = "mpq7932" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mpq7932_id);
 
diff --git a/drivers/hwmon/pmbus/mpq8785.c b/drivers/hwmon/pmbus/mpq8785.c
index 87bd039c77b9b..bbde55028290a 100644
--- a/drivers/hwmon/pmbus/mpq8785.c
+++ b/drivers/hwmon/pmbus/mpq8785.c
@@ -110,11 +110,11 @@ static struct pmbus_driver_info mpq8785_info = {
 };
 
 static const struct i2c_device_id mpq8785_id[] = {
-	{ "mpm3695", mpm3695 },
-	{ "mpm3695-25", mpm3695_25 },
-	{ "mpm82504", mpm82504 },
-	{ "mpq8785", mpq8785 },
-	{ },
+	{ .name = "mpm3695", .driver_data = mpm3695 },
+	{ .name = "mpm3695-25", .driver_data = mpm3695_25 },
+	{ .name = "mpm82504", .driver_data = mpm82504 },
+	{ .name = "mpq8785", .driver_data = mpq8785 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mpq8785_id);
 
diff --git a/drivers/hwmon/pmbus/pim4328.c b/drivers/hwmon/pmbus/pim4328.c
index aa98284bbdd84..9056dc387e878 100644
--- a/drivers/hwmon/pmbus/pim4328.c
+++ b/drivers/hwmon/pmbus/pim4328.c
@@ -39,15 +39,15 @@ struct pim4328_data {
 #define PIM4328_MFR_READ_STATUS		0xd0
 
 static const struct i2c_device_id pim4328_id[] = {
-	{"bmr455", pim4328},
-	{"pim4006", pim4006},
-	{"pim4106", pim4006},
-	{"pim4206", pim4006},
-	{"pim4306", pim4006},
-	{"pim4328", pim4328},
-	{"pim4406", pim4006},
-	{"pim4820", pim4820},
-	{}
+	{ .name = "bmr455", .driver_data = pim4328 },
+	{ .name = "pim4006", .driver_data = pim4006 },
+	{ .name = "pim4106", .driver_data = pim4006 },
+	{ .name = "pim4206", .driver_data = pim4006 },
+	{ .name = "pim4306", .driver_data = pim4006 },
+	{ .name = "pim4328", .driver_data = pim4328 },
+	{ .name = "pim4406", .driver_data = pim4006 },
+	{ .name = "pim4820", .driver_data = pim4820 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pim4328_id);
 
diff --git a/drivers/hwmon/pmbus/pli1209bc.c b/drivers/hwmon/pmbus/pli1209bc.c
index 569b61dc1a32a..d5ab085a22730 100644
--- a/drivers/hwmon/pmbus/pli1209bc.c
+++ b/drivers/hwmon/pmbus/pli1209bc.c
@@ -117,8 +117,8 @@ static int pli1209bc_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id pli1209bc_id[] = {
-	{"pli1209bc"},
-	{}
+	{ .name = "pli1209bc" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, pli1209bc_id);
diff --git a/drivers/hwmon/pmbus/pm6764tr.c b/drivers/hwmon/pmbus/pm6764tr.c
index c96c0aecb920a..613654b15b4a1 100644
--- a/drivers/hwmon/pmbus/pm6764tr.c
+++ b/drivers/hwmon/pmbus/pm6764tr.c
@@ -48,8 +48,8 @@ static int pm6764tr_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id pm6764tr_id[] = {
-	{"pm6764tr"},
-	{}
+	{ .name = "pm6764tr" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pm6764tr_id);
 
diff --git a/drivers/hwmon/pmbus/pmbus.c b/drivers/hwmon/pmbus/pmbus.c
index d1844c7a51eef..34945ccd3afc6 100644
--- a/drivers/hwmon/pmbus/pmbus.c
+++ b/drivers/hwmon/pmbus/pmbus.c
@@ -213,36 +213,36 @@ static const struct pmbus_device_info pmbus_info_one_status = {
  * Use driver_data to set the number of pages supported by the chip.
  */
 static const struct i2c_device_id pmbus_id[] = {
-	{"adp4000", (kernel_ulong_t)&pmbus_info_one},
-	{"bmr310", (kernel_ulong_t)&pmbus_info_one_status},
-	{"bmr453", (kernel_ulong_t)&pmbus_info_one},
-	{"bmr454", (kernel_ulong_t)&pmbus_info_one},
-	{"bmr456", (kernel_ulong_t)&pmbus_info_one},
-	{"bmr457", (kernel_ulong_t)&pmbus_info_one},
-	{"bmr458", (kernel_ulong_t)&pmbus_info_one_status},
-	{"bmr480", (kernel_ulong_t)&pmbus_info_one_status},
-	{"bmr490", (kernel_ulong_t)&pmbus_info_one_status},
-	{"bmr491", (kernel_ulong_t)&pmbus_info_one_status},
-	{"bmr492", (kernel_ulong_t)&pmbus_info_one},
-	{"dps460", (kernel_ulong_t)&pmbus_info_one_skip},
-	{"dps650ab", (kernel_ulong_t)&pmbus_info_one_skip},
-	{"dps800", (kernel_ulong_t)&pmbus_info_one_skip},
-	{"max20796", (kernel_ulong_t)&pmbus_info_one},
-	{"mdt040", (kernel_ulong_t)&pmbus_info_one},
-	{"ncp4200", (kernel_ulong_t)&pmbus_info_one},
-	{"ncp4208", (kernel_ulong_t)&pmbus_info_one},
-	{"pdt003", (kernel_ulong_t)&pmbus_info_one},
-	{"pdt006", (kernel_ulong_t)&pmbus_info_one},
-	{"pdt012", (kernel_ulong_t)&pmbus_info_one},
-	{"pmbus", (kernel_ulong_t)&pmbus_info_zero},
-	{"sgd009", (kernel_ulong_t)&pmbus_info_one_skip},
-	{"tps40400", (kernel_ulong_t)&pmbus_info_one},
-	{"tps544b20", (kernel_ulong_t)&pmbus_info_one},
-	{"tps544b25", (kernel_ulong_t)&pmbus_info_one},
-	{"tps544c20", (kernel_ulong_t)&pmbus_info_one},
-	{"tps544c25", (kernel_ulong_t)&pmbus_info_one},
-	{"udt020", (kernel_ulong_t)&pmbus_info_one},
-	{}
+	{ .name = "adp4000", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "bmr310", .driver_data = (kernel_ulong_t)&pmbus_info_one_status },
+	{ .name = "bmr453", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "bmr454", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "bmr456", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "bmr457", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "bmr458", .driver_data = (kernel_ulong_t)&pmbus_info_one_status },
+	{ .name = "bmr480", .driver_data = (kernel_ulong_t)&pmbus_info_one_status },
+	{ .name = "bmr490", .driver_data = (kernel_ulong_t)&pmbus_info_one_status },
+	{ .name = "bmr491", .driver_data = (kernel_ulong_t)&pmbus_info_one_status },
+	{ .name = "bmr492", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "dps460", .driver_data = (kernel_ulong_t)&pmbus_info_one_skip },
+	{ .name = "dps650ab", .driver_data = (kernel_ulong_t)&pmbus_info_one_skip },
+	{ .name = "dps800", .driver_data = (kernel_ulong_t)&pmbus_info_one_skip },
+	{ .name = "max20796", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "mdt040", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "ncp4200", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "ncp4208", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "pdt003", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "pdt006", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "pdt012", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "pmbus", .driver_data = (kernel_ulong_t)&pmbus_info_zero },
+	{ .name = "sgd009", .driver_data = (kernel_ulong_t)&pmbus_info_one_skip },
+	{ .name = "tps40400", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "tps544b20", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "tps544b25", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "tps544c20", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "tps544c25", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ .name = "udt020", .driver_data = (kernel_ulong_t)&pmbus_info_one },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, pmbus_id);
diff --git a/drivers/hwmon/pmbus/pxe1610.c b/drivers/hwmon/pmbus/pxe1610.c
index 24c1f961c7668..6ba2b3e0e565b 100644
--- a/drivers/hwmon/pmbus/pxe1610.c
+++ b/drivers/hwmon/pmbus/pxe1610.c
@@ -130,10 +130,10 @@ static int pxe1610_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id pxe1610_id[] = {
-	{"pxe1610"},
-	{"pxe1110"},
-	{"pxm1310"},
-	{}
+	{ .name = "pxe1610" },
+	{ .name = "pxe1110" },
+	{ .name = "pxm1310" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, pxe1610_id);
diff --git a/drivers/hwmon/pmbus/q54sj108a2.c b/drivers/hwmon/pmbus/q54sj108a2.c
index a368cfa9d45ac..ecac405d75362 100644
--- a/drivers/hwmon/pmbus/q54sj108a2.c
+++ b/drivers/hwmon/pmbus/q54sj108a2.c
@@ -270,10 +270,10 @@ static const struct file_operations q54sj108a2_fops = {
 };
 
 static const struct i2c_device_id q54sj108a2_id[] = {
-	{ "q54sj108a2", q54sj108a2 },
-	{ "q54sn120a1", q54sj108a2 },
-	{ "q54sw120a7", q54sj108a2 },
-	{ },
+	{ .name = "q54sj108a2", .driver_data = q54sj108a2 },
+	{ .name = "q54sn120a1", .driver_data = q54sj108a2 },
+	{ .name = "q54sw120a7", .driver_data = q54sj108a2 },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, q54sj108a2_id);
diff --git a/drivers/hwmon/pmbus/stef48h28.c b/drivers/hwmon/pmbus/stef48h28.c
index 4bde2215697cb..8e48dd3ba74bf 100644
--- a/drivers/hwmon/pmbus/stef48h28.c
+++ b/drivers/hwmon/pmbus/stef48h28.c
@@ -48,8 +48,8 @@ static int stef48h28_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id stef48h28_id[] = {
-	{"stef48h28"},
-	{}
+	{ .name = "stef48h28" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, stef48h28_id);
 
diff --git a/drivers/hwmon/pmbus/stpddc60.c b/drivers/hwmon/pmbus/stpddc60.c
index 5cb905ed8ae5f..b5ce7f975eea4 100644
--- a/drivers/hwmon/pmbus/stpddc60.c
+++ b/drivers/hwmon/pmbus/stpddc60.c
@@ -18,9 +18,9 @@
 #define STPDDC60_MFR_UV_LIMIT_OFFSET	0xe6
 
 static const struct i2c_device_id stpddc60_id[] = {
-	{"stpddc60"},
-	{"bmr481"},
-	{}
+	{ .name = "stpddc60" },
+	{ .name = "bmr481" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, stpddc60_id);
 
diff --git a/drivers/hwmon/pmbus/tda38640.c b/drivers/hwmon/pmbus/tda38640.c
index d902d39f49f4d..8039266333700 100644
--- a/drivers/hwmon/pmbus/tda38640.c
+++ b/drivers/hwmon/pmbus/tda38640.c
@@ -195,8 +195,8 @@ static int tda38640_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tda38640_id[] = {
-	{"tda38640"},
-	{}
+	{ .name = "tda38640" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tda38640_id);
 
diff --git a/drivers/hwmon/pmbus/tps25990.c b/drivers/hwmon/pmbus/tps25990.c
index 05c6288ecafcc..9d318e6509abf 100644
--- a/drivers/hwmon/pmbus/tps25990.c
+++ b/drivers/hwmon/pmbus/tps25990.c
@@ -387,8 +387,8 @@ static const struct pmbus_driver_info tps25990_base_info = {
 };
 
 static const struct i2c_device_id tps25990_i2c_id[] = {
-	{ "tps25990" },
-	{}
+	{ .name = "tps25990" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tps25990_i2c_id);
 
diff --git a/drivers/hwmon/pmbus/tps40422.c b/drivers/hwmon/pmbus/tps40422.c
index 7c9fedaa068c0..f7be075dc7070 100644
--- a/drivers/hwmon/pmbus/tps40422.c
+++ b/drivers/hwmon/pmbus/tps40422.c
@@ -31,8 +31,8 @@ static int tps40422_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tps40422_id[] = {
-	{"tps40422"},
-	{}
+	{ .name = "tps40422" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, tps40422_id);
diff --git a/drivers/hwmon/pmbus/tps53679.c b/drivers/hwmon/pmbus/tps53679.c
index 94258e8cfd903..31e54608b3c9c 100644
--- a/drivers/hwmon/pmbus/tps53679.c
+++ b/drivers/hwmon/pmbus/tps53679.c
@@ -291,15 +291,15 @@ static int tps53679_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tps53679_id[] = {
-	{"bmr474", tps53676},
-	{"tps53647", tps53647},
-	{"tps53667", tps53667},
-	{"tps53676", tps53676},
-	{"tps53679", tps53679},
-	{"tps53681", tps53681},
-	{"tps53685", tps53685},
-	{"tps53688", tps53688},
-	{}
+	{ .name = "bmr474", .driver_data = tps53676 },
+	{ .name = "tps53647", .driver_data = tps53647 },
+	{ .name = "tps53667", .driver_data = tps53667 },
+	{ .name = "tps53676", .driver_data = tps53676 },
+	{ .name = "tps53679", .driver_data = tps53679 },
+	{ .name = "tps53681", .driver_data = tps53681 },
+	{ .name = "tps53685", .driver_data = tps53685 },
+	{ .name = "tps53688", .driver_data = tps53688 },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, tps53679_id);
diff --git a/drivers/hwmon/pmbus/tps546d24.c b/drivers/hwmon/pmbus/tps546d24.c
index 44d7a6df1dbdf..4222ff355e02b 100644
--- a/drivers/hwmon/pmbus/tps546d24.c
+++ b/drivers/hwmon/pmbus/tps546d24.c
@@ -42,8 +42,8 @@ static int tps546d24_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tps546d24_id[] = {
-	{"tps546d24"},
-	{}
+	{ .name = "tps546d24" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tps546d24_id);
 
diff --git a/drivers/hwmon/pmbus/ucd9000.c b/drivers/hwmon/pmbus/ucd9000.c
index 9b5d34a110ba5..f76c2913c9cae 100644
--- a/drivers/hwmon/pmbus/ucd9000.c
+++ b/drivers/hwmon/pmbus/ucd9000.c
@@ -143,14 +143,14 @@ static int ucd9000_read_byte_data(struct i2c_client *client, int page, int reg)
 }
 
 static const struct i2c_device_id ucd9000_id[] = {
-	{"ucd9000", ucd9000},
-	{"ucd90120", ucd90120},
-	{"ucd90124", ucd90124},
-	{"ucd90160", ucd90160},
-	{"ucd90320", ucd90320},
-	{"ucd9090", ucd9090},
-	{"ucd90910", ucd90910},
-	{}
+	{ .name = "ucd9000", .driver_data = ucd9000 },
+	{ .name = "ucd90120", .driver_data = ucd90120 },
+	{ .name = "ucd90124", .driver_data = ucd90124 },
+	{ .name = "ucd90160", .driver_data = ucd90160 },
+	{ .name = "ucd90320", .driver_data = ucd90320 },
+	{ .name = "ucd9090", .driver_data = ucd9090 },
+	{ .name = "ucd90910", .driver_data = ucd90910 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ucd9000_id);
 
diff --git a/drivers/hwmon/pmbus/ucd9200.c b/drivers/hwmon/pmbus/ucd9200.c
index f68adaf4a110e..5e07bba111a16 100644
--- a/drivers/hwmon/pmbus/ucd9200.c
+++ b/drivers/hwmon/pmbus/ucd9200.c
@@ -22,15 +22,15 @@ enum chips { ucd9200, ucd9220, ucd9222, ucd9224, ucd9240, ucd9244, ucd9246,
 	     ucd9248 };
 
 static const struct i2c_device_id ucd9200_id[] = {
-	{"ucd9200", ucd9200},
-	{"ucd9220", ucd9220},
-	{"ucd9222", ucd9222},
-	{"ucd9224", ucd9224},
-	{"ucd9240", ucd9240},
-	{"ucd9244", ucd9244},
-	{"ucd9246", ucd9246},
-	{"ucd9248", ucd9248},
-	{}
+	{ .name = "ucd9200", .driver_data = ucd9200 },
+	{ .name = "ucd9220", .driver_data = ucd9220 },
+	{ .name = "ucd9222", .driver_data = ucd9222 },
+	{ .name = "ucd9224", .driver_data = ucd9224 },
+	{ .name = "ucd9240", .driver_data = ucd9240 },
+	{ .name = "ucd9244", .driver_data = ucd9244 },
+	{ .name = "ucd9246", .driver_data = ucd9246 },
+	{ .name = "ucd9248", .driver_data = ucd9248 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ucd9200_id);
 
diff --git a/drivers/hwmon/pmbus/xdp710.c b/drivers/hwmon/pmbus/xdp710.c
index 660bbfe16e1e9..494dbe45ebc6b 100644
--- a/drivers/hwmon/pmbus/xdp710.c
+++ b/drivers/hwmon/pmbus/xdp710.c
@@ -110,7 +110,7 @@ static const struct of_device_id xdp710_of_match[] = {
 };
 
 static const struct i2c_device_id xdp710_id[] = {
-	{"xdp710"},
+	{ .name = "xdp710" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, xdp710_id);
diff --git a/drivers/hwmon/pmbus/xdp720.c b/drivers/hwmon/pmbus/xdp720.c
index 8729a771f2166..60491e9217bfb 100644
--- a/drivers/hwmon/pmbus/xdp720.c
+++ b/drivers/hwmon/pmbus/xdp720.c
@@ -106,8 +106,8 @@ static const struct of_device_id xdp720_of_match[] = {
 MODULE_DEVICE_TABLE(of, xdp720_of_match);
 
 static const struct i2c_device_id xdp720_id[] = {
-	{ "xdp720" },
-	{}
+	{ .name = "xdp720" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, xdp720_id);
 
diff --git a/drivers/hwmon/pmbus/xdpe12284.c b/drivers/hwmon/pmbus/xdpe12284.c
index f3aa6339d60d0..ed93e17ce8d51 100644
--- a/drivers/hwmon/pmbus/xdpe12284.c
+++ b/drivers/hwmon/pmbus/xdpe12284.c
@@ -164,10 +164,10 @@ static int xdpe122_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id xdpe122_id[] = {
-	{"xdpe11280"},
-	{"xdpe12254"},
-	{"xdpe12284"},
-	{}
+	{ .name = "xdpe11280" },
+	{ .name = "xdpe12254" },
+	{ .name = "xdpe12284" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, xdpe122_id);
diff --git a/drivers/hwmon/pmbus/xdpe152c4.c b/drivers/hwmon/pmbus/xdpe152c4.c
index 67a3d5fe1dafa..b557d2971d5f2 100644
--- a/drivers/hwmon/pmbus/xdpe152c4.c
+++ b/drivers/hwmon/pmbus/xdpe152c4.c
@@ -44,9 +44,9 @@ static int xdpe152_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id xdpe152_id[] = {
-	{"xdpe152c4"},
-	{"xdpe15284"},
-	{}
+	{ .name = "xdpe152c4" },
+	{ .name = "xdpe15284" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, xdpe152_id);
diff --git a/drivers/hwmon/pmbus/xdpe1a2g7b.c b/drivers/hwmon/pmbus/xdpe1a2g7b.c
index 1755e3522ede5..971e7b73752ea 100644
--- a/drivers/hwmon/pmbus/xdpe1a2g7b.c
+++ b/drivers/hwmon/pmbus/xdpe1a2g7b.c
@@ -87,9 +87,9 @@ static int xdpe1a2g7b_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id xdpe1a2g7b_id[] = {
-	{ "xdpe1a2g5b" },
-	{ "xdpe1a2g7b" },
-	{}
+	{ .name = "xdpe1a2g5b" },
+	{ .name = "xdpe1a2g7b" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, xdpe1a2g7b_id);
diff --git a/drivers/hwmon/pmbus/zl6100.c b/drivers/hwmon/pmbus/zl6100.c
index 97be69630cfbb..5db2a7818d165 100644
--- a/drivers/hwmon/pmbus/zl6100.c
+++ b/drivers/hwmon/pmbus/zl6100.c
@@ -250,28 +250,28 @@ static int zl6100_write_word_data(struct i2c_client *client, int page, int reg,
 }
 
 static const struct i2c_device_id zl6100_id[] = {
-	{"bmr450", zl2005},
-	{"bmr451", zl2005},
-	{"bmr462", zl2008},
-	{"bmr463", zl2008},
-	{"bmr464", zl2008},
-	{"bmr465", zls4009},
-	{"bmr466", zls1003},
-	{"bmr467", zls4009},
-	{"bmr469", zl8802},
-	{"zl2004", zl2004},
-	{"zl2005", zl2005},
-	{"zl2006", zl2006},
-	{"zl2008", zl2008},
-	{"zl2105", zl2105},
-	{"zl2106", zl2106},
-	{"zl6100", zl6100},
-	{"zl6105", zl6105},
-	{"zl8802", zl8802},
-	{"zl9101", zl9101},
-	{"zl9117", zl9117},
-	{"zls1003", zls1003},
-	{"zls4009", zls4009},
+	{ .name = "bmr450", .driver_data = zl2005 },
+	{ .name = "bmr451", .driver_data = zl2005 },
+	{ .name = "bmr462", .driver_data = zl2008 },
+	{ .name = "bmr463", .driver_data = zl2008 },
+	{ .name = "bmr464", .driver_data = zl2008 },
+	{ .name = "bmr465", .driver_data = zls4009 },
+	{ .name = "bmr466", .driver_data = zls1003 },
+	{ .name = "bmr467", .driver_data = zls4009 },
+	{ .name = "bmr469", .driver_data = zl8802 },
+	{ .name = "zl2004", .driver_data = zl2004 },
+	{ .name = "zl2005", .driver_data = zl2005 },
+	{ .name = "zl2006", .driver_data = zl2006 },
+	{ .name = "zl2008", .driver_data = zl2008 },
+	{ .name = "zl2105", .driver_data = zl2105 },
+	{ .name = "zl2106", .driver_data = zl2106 },
+	{ .name = "zl6100", .driver_data = zl6100 },
+	{ .name = "zl6105", .driver_data = zl6105 },
+	{ .name = "zl8802", .driver_data = zl8802 },
+	{ .name = "zl9101", .driver_data = zl9101 },
+	{ .name = "zl9117", .driver_data = zl9117 },
+	{ .name = "zls1003", .driver_data = zls1003 },
+	{ .name = "zls4009", .driver_data = zls4009 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, zl6100_id);
diff --git a/drivers/hwmon/powr1220.c b/drivers/hwmon/powr1220.c
index 06a2c56016d11..6e211601ea26b 100644
--- a/drivers/hwmon/powr1220.c
+++ b/drivers/hwmon/powr1220.c
@@ -306,8 +306,8 @@ static int powr1220_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id powr1220_ids[] = {
-	{ "powr1014", powr1014, },
-	{ "powr1220", powr1220, },
+	{ .name = "powr1014", .driver_data = powr1014 },
+	{ .name = "powr1220", .driver_data = powr1220 },
 	{ }
 };
 
diff --git a/drivers/hwmon/pt5161l.c b/drivers/hwmon/pt5161l.c
index 89d4da8aa4c09..2b408a69b085c 100644
--- a/drivers/hwmon/pt5161l.c
+++ b/drivers/hwmon/pt5161l.c
@@ -617,8 +617,8 @@ static const struct acpi_device_id __maybe_unused pt5161l_acpi_match[] = {
 MODULE_DEVICE_TABLE(acpi, pt5161l_acpi_match);
 
 static const struct i2c_device_id pt5161l_id[] = {
-	{ "pt5161l" },
-	{}
+	{ .name = "pt5161l" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pt5161l_id);
 
diff --git a/drivers/hwmon/raspberrypi-hwmon.c b/drivers/hwmon/raspberrypi-hwmon.c
index a2938881ccd2f..7b1d2829f012f 100644
--- a/drivers/hwmon/raspberrypi-hwmon.c
+++ b/drivers/hwmon/raspberrypi-hwmon.c
@@ -5,9 +5,9 @@
  * Based on firmware/raspberrypi.c by Noralf Trønnes
  *
  * Copyright (C) 2018 Stefan Wahren <stefan.wahren@i2se.com>
+ * Copyright (C) 2026 Shubham Chakraborty <chakrabortyshubham66@gmail.com>
  */
 #include <linux/device.h>
-#include <linux/devm-helpers.h>
 #include <linux/err.h>
 #include <linux/hwmon.h>
 #include <linux/module.h>
@@ -21,10 +21,18 @@
 struct rpi_hwmon_data {
 	struct device *hwmon_dev;
 	struct rpi_firmware *fw;
+	u32 valid_inputs;
 	u32 last_throttled;
 	struct delayed_work get_values_poll_work;
 };
 
+static const char * const rpi_hwmon_labels[] = {
+	"core",
+	"sdram_c",
+	"sdram_i",
+	"sdram_p",
+};
+
 static void rpi_firmware_get_throttled(struct rpi_hwmon_data *data)
 {
 	u32 new_uv, old_uv, value;
@@ -56,6 +64,22 @@ static void rpi_firmware_get_throttled(struct rpi_hwmon_data *data)
 	hwmon_notify_event(data->hwmon_dev, hwmon_in, hwmon_in_lcrit_alarm, 0);
 }
 
+static int rpi_firmware_get_voltage(struct rpi_hwmon_data *data, u32 id,
+				    long *val)
+{
+	struct rpi_firmware_get_voltage_request packet =
+		RPI_FIRMWARE_GET_VOLTAGE_REQUEST(id);
+	int ret;
+
+	ret = rpi_firmware_property(data->fw, RPI_FIRMWARE_GET_VOLTAGE,
+				    &packet, sizeof(packet));
+	if (ret)
+		return ret;
+
+	*val = le32_to_cpu(packet.value) / 1000;
+	return 0;
+}
+
 static void get_values_poll(struct work_struct *work)
 {
 	struct rpi_hwmon_data *data;
@@ -72,24 +96,106 @@ static void get_values_poll(struct work_struct *work)
 	schedule_delayed_work(&data->get_values_poll_work, 2 * HZ);
 }
 
+static void rpi_hwmon_cancel_poll_work(void *res)
+{
+	struct rpi_hwmon_data *data = res;
+
+	disable_delayed_work_sync(&data->get_values_poll_work);
+}
+
 static int rpi_read(struct device *dev, enum hwmon_sensor_types type,
 		    u32 attr, int channel, long *val)
 {
 	struct rpi_hwmon_data *data = dev_get_drvdata(dev);
 
-	*val = !!(data->last_throttled & UNDERVOLTAGE_STICKY_BIT);
+	if (type == hwmon_in) {
+		switch (attr) {
+		case hwmon_in_input:
+			switch (channel) {
+			case 0:
+				return rpi_firmware_get_voltage(data,
+						RPI_FIRMWARE_VOLT_ID_CORE,
+						val);
+			case 1:
+				return rpi_firmware_get_voltage(data,
+						RPI_FIRMWARE_VOLT_ID_SDRAM_C,
+						val);
+			case 2:
+				return rpi_firmware_get_voltage(data,
+						RPI_FIRMWARE_VOLT_ID_SDRAM_I,
+						val);
+			case 3:
+				return rpi_firmware_get_voltage(data,
+						RPI_FIRMWARE_VOLT_ID_SDRAM_P,
+						val);
+			default:
+				return -EOPNOTSUPP;
+			}
+		case hwmon_in_lcrit_alarm:
+			if (channel == 0) {
+				*val = !!(data->last_throttled & UNDERVOLTAGE_STICKY_BIT);
+				return 0;
+			}
+			return -EOPNOTSUPP;
+		default:
+			return -EOPNOTSUPP;
+		}
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static int rpi_read_string(struct device *dev, enum hwmon_sensor_types type,
+			   u32 attr, int channel, const char **str)
+{
+	if (type == hwmon_in && attr == hwmon_in_label) {
+		if (channel >= ARRAY_SIZE(rpi_hwmon_labels))
+			return -EOPNOTSUPP;
+
+		*str = rpi_hwmon_labels[channel];
+		return 0;
+	}
+
+	return -EOPNOTSUPP;
+}
+
+static umode_t rpi_is_visible(const void *_data, enum hwmon_sensor_types type,
+			      u32 attr, int channel)
+{
+	const struct rpi_hwmon_data *data = _data;
+
+	if (type == hwmon_in) {
+		switch (attr) {
+		case hwmon_in_input:
+		case hwmon_in_label:
+			if (!(data->valid_inputs & BIT(channel)))
+				return 0;
+			return 0444;
+		case hwmon_in_lcrit_alarm:
+			if (channel == 0)
+				return 0444;
+			return 0;
+		default:
+			return 0;
+		}
+	}
+
 	return 0;
 }
 
 static const struct hwmon_channel_info * const rpi_info[] = {
 	HWMON_CHANNEL_INFO(in,
-			   HWMON_I_LCRIT_ALARM),
+			   HWMON_I_INPUT | HWMON_I_LABEL | HWMON_I_LCRIT_ALARM,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL,
+			   HWMON_I_INPUT | HWMON_I_LABEL),
 	NULL
 };
 
 static const struct hwmon_ops rpi_hwmon_ops = {
-	.visible = 0444,
+	.is_visible = rpi_is_visible,
 	.read = rpi_read,
+	.read_string = rpi_read_string,
 };
 
 static const struct hwmon_chip_info rpi_chip_info = {
@@ -101,6 +207,7 @@ static int rpi_hwmon_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct rpi_hwmon_data *data;
+	long voltage;
 	int ret;
 
 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
@@ -110,6 +217,26 @@ static int rpi_hwmon_probe(struct platform_device *pdev)
 	/* Parent driver assure that firmware is correct */
 	data->fw = dev_get_drvdata(dev->parent);
 
+	ret = rpi_firmware_get_voltage(data, RPI_FIRMWARE_VOLT_ID_CORE,
+				       &voltage);
+	if (!ret)
+		data->valid_inputs |= BIT(0);
+
+	ret = rpi_firmware_get_voltage(data, RPI_FIRMWARE_VOLT_ID_SDRAM_C,
+				       &voltage);
+	if (!ret)
+		data->valid_inputs |= BIT(1);
+
+	ret = rpi_firmware_get_voltage(data, RPI_FIRMWARE_VOLT_ID_SDRAM_I,
+				       &voltage);
+	if (!ret)
+		data->valid_inputs |= BIT(2);
+
+	ret = rpi_firmware_get_voltage(data, RPI_FIRMWARE_VOLT_ID_SDRAM_P,
+				       &voltage);
+	if (!ret)
+		data->valid_inputs |= BIT(3);
+
 	data->hwmon_dev = devm_hwmon_device_register_with_info(dev, "rpi_volt",
 							       data,
 							       &rpi_chip_info,
@@ -117,8 +244,8 @@ static int rpi_hwmon_probe(struct platform_device *pdev)
 	if (IS_ERR(data->hwmon_dev))
 		return PTR_ERR(data->hwmon_dev);
 
-	ret = devm_delayed_work_autocancel(dev, &data->get_values_poll_work,
-					   get_values_poll);
+	INIT_DELAYED_WORK(&data->get_values_poll_work, get_values_poll);
+	ret = devm_add_action_or_reset(dev, rpi_hwmon_cancel_poll_work, data);
 	if (ret)
 		return ret;
 	platform_set_drvdata(pdev, data);
@@ -159,6 +286,7 @@ static struct platform_driver rpi_hwmon_driver = {
 module_platform_driver(rpi_hwmon_driver);
 
 MODULE_AUTHOR("Stefan Wahren <wahrenst@gmx.net>");
+MODULE_AUTHOR("Shubham Chakraborty <chakrabortyshubham66@gmail.com>");
 MODULE_DESCRIPTION("Raspberry Pi voltage sensor driver");
 MODULE_LICENSE("GPL v2");
 MODULE_ALIAS("platform:raspberrypi-hwmon");
diff --git a/drivers/hwmon/sbtsi_temp.c b/drivers/hwmon/sbtsi_temp.c
index c5b2488c4c7f7..c28f8625cd3a4 100644
--- a/drivers/hwmon/sbtsi_temp.c
+++ b/drivers/hwmon/sbtsi_temp.c
@@ -221,8 +221,8 @@ static int sbtsi_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id sbtsi_id[] = {
-	{"sbtsi"},
-	{}
+	{ .name = "sbtsi" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sbtsi_id);
 
diff --git a/drivers/hwmon/sg2042-mcu.c b/drivers/hwmon/sg2042-mcu.c
index 105131c4acf78..591f5f572fe4c 100644
--- a/drivers/hwmon/sg2042-mcu.c
+++ b/drivers/hwmon/sg2042-mcu.c
@@ -333,7 +333,7 @@ static int sg2042_mcu_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id sg2042_mcu_id[] = {
-	{ "sg2042-hwmon-mcu" },
+	{ .name = "sg2042-hwmon-mcu" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sg2042_mcu_id);
diff --git a/drivers/hwmon/sht21.c b/drivers/hwmon/sht21.c
index 627d35070a420..085492669eeb9 100644
--- a/drivers/hwmon/sht21.c
+++ b/drivers/hwmon/sht21.c
@@ -275,9 +275,9 @@ static int sht21_probe(struct i2c_client *client)
 
 /* Device ID table */
 static const struct i2c_device_id sht21_id[] = {
-	{ "sht20" },
-	{ "sht21" },
-	{ "sht25" },
+	{ .name = "sht20" },
+	{ .name = "sht21" },
+	{ .name = "sht25" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sht21_id);
diff --git a/drivers/hwmon/sht3x.c b/drivers/hwmon/sht3x.c
index 08306ccb6d0be..c2f6b73aa7f34 100644
--- a/drivers/hwmon/sht3x.c
+++ b/drivers/hwmon/sht3x.c
@@ -931,10 +931,10 @@ static int sht3x_probe(struct i2c_client *client)
 
 /* device ID table */
 static const struct i2c_device_id sht3x_ids[] = {
-	{"sht3x", sht3x},
-	{"sts3x", sts3x},
-	{"sht85", sht3x},
-	{}
+	{ .name = "sht3x", .driver_data = sht3x },
+	{ .name = "sts3x", .driver_data = sts3x },
+	{ .name = "sht85", .driver_data = sht3x },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, sht3x_ids);
diff --git a/drivers/hwmon/sht4x.c b/drivers/hwmon/sht4x.c
index 5abe1227e109e..9cace0e8acdab 100644
--- a/drivers/hwmon/sht4x.c
+++ b/drivers/hwmon/sht4x.c
@@ -424,8 +424,8 @@ static int sht4x_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id sht4x_id[] = {
-	{ "sht4x" },
-	{ },
+	{ .name = "sht4x" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sht4x_id);
 
diff --git a/drivers/hwmon/shtc1.c b/drivers/hwmon/shtc1.c
index 2ac906e8e1733..362a73cf661b2 100644
--- a/drivers/hwmon/shtc1.c
+++ b/drivers/hwmon/shtc1.c
@@ -257,9 +257,9 @@ static int shtc1_probe(struct i2c_client *client)
 
 /* device ID table */
 static const struct i2c_device_id shtc1_id[] = {
-	{ "shtc1", shtc1 },
-	{ "shtw1", shtc1 },
-	{ "shtc3", shtc3 },
+	{ .name = "shtc1", .driver_data = shtc1 },
+	{ .name = "shtw1", .driver_data = shtc1 },
+	{ .name = "shtc3", .driver_data = shtc3 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, shtc1_id);
diff --git a/drivers/hwmon/smsc47m192.c b/drivers/hwmon/smsc47m192.c
index 21103af4e1395..1429b66e09b07 100644
--- a/drivers/hwmon/smsc47m192.c
+++ b/drivers/hwmon/smsc47m192.c
@@ -618,7 +618,7 @@ static int smsc47m192_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id smsc47m192_id[] = {
-	{ "smsc47m192" },
+	{ .name = "smsc47m192" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, smsc47m192_id);
diff --git a/drivers/hwmon/spd5118.c b/drivers/hwmon/spd5118.c
index 5da44571b6a0c..cc40661cab211 100644
--- a/drivers/hwmon/spd5118.c
+++ b/drivers/hwmon/spd5118.c
@@ -746,7 +746,7 @@ static int spd5118_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id spd5118_i2c_id[] = {
-	{ "spd5118" },
+	{ .name = "spd5118" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, spd5118_i2c_id);
diff --git a/drivers/hwmon/stts751.c b/drivers/hwmon/stts751.c
index f9e8b2869164a..ce23681f79810 100644
--- a/drivers/hwmon/stts751.c
+++ b/drivers/hwmon/stts751.c
@@ -72,7 +72,7 @@ static const int stts751_intervals[] = {
 };
 
 static const struct i2c_device_id stts751_id[] = {
-	{ "stts751" },
+	{ .name = "stts751" },
 	{ }
 };
 
diff --git a/drivers/hwmon/tc654.c b/drivers/hwmon/tc654.c
index 39fe5836f237f..4b22e305b203e 100644
--- a/drivers/hwmon/tc654.c
+++ b/drivers/hwmon/tc654.c
@@ -550,9 +550,9 @@ static int tc654_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tc654_id[] = {
-	{"tc654"},
-	{"tc655"},
-	{}
+	{ .name = "tc654" },
+	{ .name = "tc655" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, tc654_id);
diff --git a/drivers/hwmon/tc74.c b/drivers/hwmon/tc74.c
index 7fb7b50ad1adb..e9113519be535 100644
--- a/drivers/hwmon/tc74.c
+++ b/drivers/hwmon/tc74.c
@@ -151,8 +151,8 @@ static int tc74_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tc74_id[] = {
-	{ "tc74" },
-	{}
+	{ .name = "tc74" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tc74_id);
 
diff --git a/drivers/hwmon/thmc50.c b/drivers/hwmon/thmc50.c
index 0cbdb91698b1f..b385ef64af754 100644
--- a/drivers/hwmon/thmc50.c
+++ b/drivers/hwmon/thmc50.c
@@ -407,8 +407,8 @@ static int thmc50_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id thmc50_id[] = {
-	{ "adm1022", adm1022 },
-	{ "thmc50", thmc50 },
+	{ .name = "adm1022", .driver_data = adm1022 },
+	{ .name = "thmc50", .driver_data = thmc50 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, thmc50_id);
diff --git a/drivers/hwmon/tmp102.c b/drivers/hwmon/tmp102.c
index 3aa1a3fbeaa92..6bd1bed3cdb89 100644
--- a/drivers/hwmon/tmp102.c
+++ b/drivers/hwmon/tmp102.c
@@ -15,7 +15,8 @@
 #include <linux/jiffies.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
-#include <linux/of.h>
+#include <linux/mod_devicetable.h>
+#include <linux/property.h>
 
 #define	DRIVER_NAME "tmp102"
 
@@ -313,7 +314,7 @@ static int tmp102_probe(struct i2c_client *client)
 	if (!tmp102)
 		return -ENOMEM;
 
-	of_property_read_string(dev->of_node, "label", &tmp102->label);
+	device_property_read_string(dev, "label", &tmp102->label);
 
 	i2c_set_clientdata(client, tmp102);
 
@@ -395,12 +396,12 @@ static int tmp102_resume(struct device *dev)
 static DEFINE_SIMPLE_DEV_PM_OPS(tmp102_dev_pm_ops, tmp102_suspend, tmp102_resume);
 
 static const struct i2c_device_id tmp102_id[] = {
-	{ "tmp102" },
+	{ .name = "tmp102" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tmp102_id);
 
-static const struct of_device_id __maybe_unused tmp102_of_match[] = {
+static const struct of_device_id tmp102_of_match[] = {
 	{ .compatible = "ti,tmp102" },
 	{ },
 };
@@ -408,7 +409,7 @@ MODULE_DEVICE_TABLE(of, tmp102_of_match);
 
 static struct i2c_driver tmp102_driver = {
 	.driver.name	= DRIVER_NAME,
-	.driver.of_match_table = of_match_ptr(tmp102_of_match),
+	.driver.of_match_table = tmp102_of_match,
 	.driver.pm	= pm_sleep_ptr(&tmp102_dev_pm_ops),
 	.probe		= tmp102_probe,
 	.id_table	= tmp102_id,
diff --git a/drivers/hwmon/tmp103.c b/drivers/hwmon/tmp103.c
index 221bba8a215d8..f13d247675751 100644
--- a/drivers/hwmon/tmp103.c
+++ b/drivers/hwmon/tmp103.c
@@ -194,7 +194,7 @@ static int tmp103_resume(struct device *dev)
 static DEFINE_SIMPLE_DEV_PM_OPS(tmp103_dev_pm_ops, tmp103_suspend, tmp103_resume);
 
 static const struct i2c_device_id tmp103_id[] = {
-	{ "tmp103" },
+	{ .name = "tmp103" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tmp103_id);
diff --git a/drivers/hwmon/tmp108.c b/drivers/hwmon/tmp108.c
index 3ea5f6485744d..1c4a58855e2d2 100644
--- a/drivers/hwmon/tmp108.c
+++ b/drivers/hwmon/tmp108.c
@@ -537,10 +537,10 @@ static int tmp108_resume(struct device *dev)
 static DEFINE_SIMPLE_DEV_PM_OPS(tmp108_dev_pm_ops, tmp108_suspend, tmp108_resume);
 
 static const struct i2c_device_id tmp108_i2c_ids[] = {
-	{ "p3t1035", (unsigned long)&p3t1035_data },
-	{ "p3t1085", (unsigned long)&tmp108_data },
-	{ "tmp108", (unsigned long)&tmp108_data },
-	{}
+	{ .name = "p3t1035", .driver_data = (unsigned long)&p3t1035_data },
+	{ .name = "p3t1085", .driver_data = (unsigned long)&tmp108_data },
+	{ .name = "tmp108", .driver_data = (unsigned long)&tmp108_data },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tmp108_i2c_ids);
 
diff --git a/drivers/hwmon/tmp401.c b/drivers/hwmon/tmp401.c
index 07f596581c6eb..a09225a87355d 100644
--- a/drivers/hwmon/tmp401.c
+++ b/drivers/hwmon/tmp401.c
@@ -90,11 +90,11 @@ static const u8 TMP432_STATUS_REG[] = {
  */
 
 static const struct i2c_device_id tmp401_id[] = {
-	{ "tmp401", tmp401 },
-	{ "tmp411", tmp411 },
-	{ "tmp431", tmp431 },
-	{ "tmp432", tmp432 },
-	{ "tmp435", tmp435 },
+	{ .name = "tmp401", .driver_data = tmp401 },
+	{ .name = "tmp411", .driver_data = tmp411 },
+	{ .name = "tmp431", .driver_data = tmp431 },
+	{ .name = "tmp432", .driver_data = tmp432 },
+	{ .name = "tmp435", .driver_data = tmp435 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tmp401_id);
diff --git a/drivers/hwmon/tmp421.c b/drivers/hwmon/tmp421.c
index 2ea9d3e9553db..ed00ccfdd7b3e 100644
--- a/drivers/hwmon/tmp421.c
+++ b/drivers/hwmon/tmp421.c
@@ -56,11 +56,11 @@ static const u8 TMP421_TEMP_LSB[MAX_CHANNELS]	= { 0x10, 0x11, 0x12, 0x13 };
 #define TMP442_DEVICE_ID			0x42
 
 static const struct i2c_device_id tmp421_id[] = {
-	{ "tmp421", 2 },
-	{ "tmp422", 3 },
-	{ "tmp423", 4 },
-	{ "tmp441", 2 },
-	{ "tmp442", 3 },
+	{ .name = "tmp421", .driver_data = 2 },
+	{ .name = "tmp422", .driver_data = 3 },
+	{ .name = "tmp423", .driver_data = 4 },
+	{ .name = "tmp441", .driver_data = 2 },
+	{ .name = "tmp442", .driver_data = 3 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tmp421_id);
diff --git a/drivers/hwmon/tmp464.c b/drivers/hwmon/tmp464.c
index 98f2576d94c67..c3e031044d1e7 100644
--- a/drivers/hwmon/tmp464.c
+++ b/drivers/hwmon/tmp464.c
@@ -65,8 +65,8 @@ static const u8 TMP464_THERM2_LIMIT[MAX_CHANNELS] = {
 #define TMP468_DEVICE_ID			0x0468
 
 static const struct i2c_device_id tmp464_id[] = {
-	{ "tmp464", TMP464_NUM_CHANNELS },
-	{ "tmp468", TMP468_NUM_CHANNELS },
+	{ .name = "tmp464", .driver_data = TMP464_NUM_CHANNELS },
+	{ .name = "tmp468", .driver_data = TMP468_NUM_CHANNELS },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tmp464_id);
diff --git a/drivers/hwmon/tmp513.c b/drivers/hwmon/tmp513.c
index 5acbfd7d088dd..b160aa045f89d 100644
--- a/drivers/hwmon/tmp513.c
+++ b/drivers/hwmon/tmp513.c
@@ -611,8 +611,8 @@ static int tmp51x_init(struct tmp51x_data *data)
 }
 
 static const struct i2c_device_id tmp51x_id[] = {
-	{ "tmp512", TMP512_MAX_CHANNELS },
-	{ "tmp513", TMP513_MAX_CHANNELS },
+	{ .name = "tmp512", .driver_data = TMP512_MAX_CHANNELS },
+	{ .name = "tmp513", .driver_data = TMP513_MAX_CHANNELS },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tmp51x_id);
diff --git a/drivers/hwmon/tsc1641.c b/drivers/hwmon/tsc1641.c
index 2b5d34bab146d..fc53cd5bb6e09 100644
--- a/drivers/hwmon/tsc1641.c
+++ b/drivers/hwmon/tsc1641.c
@@ -721,7 +721,7 @@ static int tsc1641_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tsc1641_id[] = {
-	{ "tsc1641", 0 },
+	{ .name = "tsc1641" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tsc1641_id);
diff --git a/drivers/hwmon/w83773g.c b/drivers/hwmon/w83773g.c
index 401a28f55f931..54224dac10aef 100644
--- a/drivers/hwmon/w83773g.c
+++ b/drivers/hwmon/w83773g.c
@@ -34,7 +34,7 @@ static const u8 W83773_OFFSET_MSB[2] = { 0x11, 0x15 };
 
 /* this is the number of sensors in the device */
 static const struct i2c_device_id w83773_id[] = {
-	{ "w83773g" },
+	{ .name = "w83773g" },
 	{ }
 };
 
diff --git a/drivers/hwmon/w83781d.c b/drivers/hwmon/w83781d.c
index f664c2152a6de..c40f84e534696 100644
--- a/drivers/hwmon/w83781d.c
+++ b/drivers/hwmon/w83781d.c
@@ -1559,10 +1559,10 @@ static struct w83781d_data *w83781d_update_device(struct device *dev)
 }
 
 static const struct i2c_device_id w83781d_ids[] = {
-	{ "w83781d", w83781d, },
-	{ "w83782d", w83782d, },
-	{ "w83783s", w83783s, },
-	{ "as99127f", as99127f },
+	{ .name = "w83781d", .driver_data = w83781d },
+	{ .name = "w83782d", .driver_data = w83782d },
+	{ .name = "w83783s", .driver_data = w83783s },
+	{ .name = "as99127f", .driver_data = as99127f },
 	{ /* LIST END */ }
 };
 MODULE_DEVICE_TABLE(i2c, w83781d_ids);
diff --git a/drivers/hwmon/w83791d.c b/drivers/hwmon/w83791d.c
index 996e36951f9da..4a777430af5cd 100644
--- a/drivers/hwmon/w83791d.c
+++ b/drivers/hwmon/w83791d.c
@@ -333,7 +333,7 @@ static void w83791d_print_debug(struct w83791d_data *data, struct device *dev);
 static void w83791d_init_client(struct i2c_client *client);
 
 static const struct i2c_device_id w83791d_id[] = {
-	{ "w83791d" },
+	{ .name = "w83791d" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, w83791d_id);
diff --git a/drivers/hwmon/w83792d.c b/drivers/hwmon/w83792d.c
index b0b5f60eea53b..f715c79389a56 100644
--- a/drivers/hwmon/w83792d.c
+++ b/drivers/hwmon/w83792d.c
@@ -296,7 +296,7 @@ static void w83792d_print_debug(struct w83792d_data *data, struct device *dev);
 static void w83792d_init_client(struct i2c_client *client);
 
 static const struct i2c_device_id w83792d_id[] = {
-	{ "w83792d" },
+	{ .name = "w83792d" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, w83792d_id);
diff --git a/drivers/hwmon/w83793.c b/drivers/hwmon/w83793.c
index 24772cfbecb32..b1f906f06ab43 100644
--- a/drivers/hwmon/w83793.c
+++ b/drivers/hwmon/w83793.c
@@ -291,7 +291,7 @@ static void w83793_update_nonvolatile(struct device *dev);
 static struct w83793_data *w83793_update_device(struct device *dev);
 
 static const struct i2c_device_id w83793_id[] = {
-	{ "w83793" },
+	{ .name = "w83793" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, w83793_id);
diff --git a/drivers/hwmon/w83795.c b/drivers/hwmon/w83795.c
index 5174db69db5e1..c5ce0bf1b08ec 100644
--- a/drivers/hwmon/w83795.c
+++ b/drivers/hwmon/w83795.c
@@ -2243,8 +2243,8 @@ static void w83795_remove(struct i2c_client *client)
 
 
 static const struct i2c_device_id w83795_id[] = {
-	{ "w83795g", w83795g },
-	{ "w83795adg", w83795adg },
+	{ .name = "w83795g", .driver_data = w83795g },
+	{ .name = "w83795adg", .driver_data = w83795adg },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, w83795_id);
diff --git a/drivers/hwmon/w83l785ts.c b/drivers/hwmon/w83l785ts.c
index df77b53a1b2f9..e42506a3bcbec 100644
--- a/drivers/hwmon/w83l785ts.c
+++ b/drivers/hwmon/w83l785ts.c
@@ -74,7 +74,7 @@ static struct w83l785ts_data *w83l785ts_update_device(struct device *dev);
  */
 
 static const struct i2c_device_id w83l785ts_id[] = {
-	{ "w83l785ts" },
+	{ .name = "w83l785ts" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, w83l785ts_id);
diff --git a/drivers/hwmon/w83l786ng.c b/drivers/hwmon/w83l786ng.c
index 1d9109ca1585e..a72397083cc4f 100644
--- a/drivers/hwmon/w83l786ng.c
+++ b/drivers/hwmon/w83l786ng.c
@@ -751,7 +751,7 @@ w83l786ng_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id w83l786ng_id[] = {
-	{ "w83l786ng" },
+	{ .name = "w83l786ng" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, w83l786ng_id);
diff --git a/drivers/hwtracing/coresight/coresight-catu.c b/drivers/hwtracing/coresight/coresight-catu.c
index ce71dcddfca25..43abe13995cf3 100644
--- a/drivers/hwtracing/coresight/coresight-catu.c
+++ b/drivers/hwtracing/coresight/coresight-catu.c
@@ -514,7 +514,7 @@ static int __catu_probe(struct device *dev, struct resource *res)
 	int ret = 0;
 	u32 dma_mask;
 	struct catu_drvdata *drvdata;
-	struct coresight_desc catu_desc;
+	struct coresight_desc catu_desc = { 0 };
 	struct coresight_platform_data *pdata = NULL;
 	void __iomem *base;
 
diff --git a/drivers/hwtracing/coresight/coresight-core.c b/drivers/hwtracing/coresight/coresight-core.c
index 46f247f73cf64..f7b1308a759c3 100644
--- a/drivers/hwtracing/coresight/coresight-core.c
+++ b/drivers/hwtracing/coresight/coresight-core.c
@@ -6,6 +6,7 @@
 #include <linux/acpi.h>
 #include <linux/bitfield.h>
 #include <linux/build_bug.h>
+#include <linux/cpu_pm.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
 #include <linux/types.h>
@@ -35,6 +36,10 @@
 DEFINE_MUTEX(coresight_mutex);
 static DEFINE_PER_CPU(struct coresight_device *, csdev_sink);
 
+static DEFINE_RAW_SPINLOCK(coresight_dev_lock);
+static DEFINE_PER_CPU(struct coresight_device *, csdev_source);
+static DEFINE_PER_CPU(bool, percpu_pm_failed);
+
 /**
  * struct coresight_node - elements of a path, from source to sink
  * @csdev:	Address of an element.
@@ -58,6 +63,24 @@ static LIST_HEAD(coresight_dev_idx_list);
 
 static const struct cti_assoc_op *cti_assoc_ops;
 
+static struct coresight_node *
+coresight_path_first_node(struct coresight_path *path)
+{
+	if (list_empty(&path->path_list))
+		return NULL;
+
+	return list_first_entry(&path->path_list, struct coresight_node, link);
+}
+
+static struct coresight_node *
+coresight_path_last_node(struct coresight_path *path)
+{
+	if (list_empty(&path->path_list))
+		return NULL;
+
+	return list_last_entry(&path->path_list, struct coresight_node, link);
+}
+
 void coresight_set_cti_ops(const struct cti_assoc_op *cti_op)
 {
 	cti_assoc_ops = cti_op;
@@ -82,14 +105,89 @@ struct coresight_device *coresight_get_percpu_sink(int cpu)
 }
 EXPORT_SYMBOL_GPL(coresight_get_percpu_sink);
 
-static struct coresight_device *coresight_get_source(struct coresight_path *path)
+static void coresight_set_percpu_source(struct coresight_device *csdev)
+{
+	if (!csdev || !coresight_is_percpu_source(csdev))
+		return;
+
+	guard(raw_spinlock_irqsave)(&coresight_dev_lock);
+
+	/* Expect no device to be set yet */
+	WARN_ON(per_cpu(csdev_source, csdev->cpu));
+	per_cpu(csdev_source, csdev->cpu) = csdev;
+}
+
+static void coresight_clear_percpu_source(struct coresight_device *csdev)
+{
+	if (!csdev || !coresight_is_percpu_source(csdev))
+		return;
+
+	/* Clear percpu_pm_failed */
+	per_cpu(percpu_pm_failed, csdev->cpu) = false;
+
+	guard(raw_spinlock_irqsave)(&coresight_dev_lock);
+
+	/* The per-CPU pointer should contain the same csdev */
+	WARN_ON(per_cpu(csdev_source, csdev->cpu) != csdev);
+	per_cpu(csdev_source, csdev->cpu) = NULL;
+}
+
+struct coresight_device *coresight_get_percpu_source_ref(int cpu)
+{
+	struct coresight_device *csdev;
+
+	if (WARN_ON(cpu < 0))
+		return NULL;
+
+	guard(raw_spinlock_irqsave)(&coresight_dev_lock);
+
+	csdev = per_cpu(csdev_source, cpu);
+	if (!csdev)
+		return NULL;
+
+	/*
+	 * Holding a reference to the csdev->dev ensures that the
+	 * coresight_device is live for the caller. The path building
+	 * logic can safely either build a path to the sink or fail
+	 * if the device is being unregistered (if there was a race).
+	 * The caller can skip the "source" device, if no path could
+	 * be built.
+	 */
+	get_device(&csdev->dev);
+
+	return csdev;
+}
+
+void coresight_put_percpu_source_ref(struct coresight_device *csdev)
+{
+	if (!csdev || !coresight_is_percpu_source(csdev))
+		return;
+
+	guard(raw_spinlock_irqsave)(&coresight_dev_lock);
+
+	/*
+	 * TODO: coresight_device_release() is invoked to release resources when
+	 * the device's refcount reaches zero. It then calls free_percpu(),
+	 * which acquires pcpu_lock — a sleepable lock when PREEMPT_RT is
+	 * enabled. Since the raw spinlock coresight_dev_lock is held, this can
+	 * lead to a potential "scheduling while atomic" issue.
+	 */
+	put_device(&csdev->dev);
+}
+
+struct coresight_device *coresight_get_source(struct coresight_path *path)
 {
 	struct coresight_device *csdev;
+	struct coresight_node *nd;
 
 	if (!path)
 		return NULL;
 
-	csdev = list_first_entry(&path->path_list, struct coresight_node, link)->csdev;
+	nd = coresight_path_first_node(path);
+	if (!nd)
+		return NULL;
+
+	csdev = nd->csdev;
 	if (!coresight_is_device_source(csdev))
 		return NULL;
 
@@ -385,19 +483,47 @@ static void coresight_disable_helpers(struct coresight_device *csdev,
 }
 
 /*
- * Helper function to call source_ops(csdev)->disable and also disable the
- * helpers.
- *
- * There is an imbalance between coresight_enable_path() and
- * coresight_disable_path(). Enabling also enables the source's helpers as part
- * of the path, but disabling always skips the first item in the path (which is
- * the source), so sources and their helpers don't get disabled as part of that
- * function and we need the extra step here.
+ * coresight_enable_source() and coresight_disable_source() only enable and
+ * disable the source, but do nothing for the associated helpers, which are
+ * controlled as part of the path.
  */
+int coresight_enable_source(struct coresight_device *csdev,
+			    struct perf_event *event, enum cs_mode mode,
+			    struct coresight_path *path)
+{
+	int ret;
+
+	if (!coresight_is_device_source(csdev))
+		return -EINVAL;
+
+	ret = source_ops(csdev)->enable(csdev, event, mode, path);
+	if (ret)
+		return ret;
+
+	/*
+	 * Update the path pointer until after the source is enabled to avoid
+	 * races where multiple paths attempt to enable the same source.
+	 *
+	 * Do not set the path pointer here for per-CPU sources; set it locally
+	 * on the CPU instead. Otherwise, there is a window where the path is
+	 * enabled but the pointer is not yet set, causing CPU PM notifiers to
+	 * miss PM operations due to reading a NULL pointer.
+	 */
+	if (!coresight_is_percpu_source(csdev))
+		csdev->path = path;
+
+	return 0;
+}
+
 void coresight_disable_source(struct coresight_device *csdev, void *data)
 {
+	if (!coresight_is_device_source(csdev))
+		return;
+
+	if (!coresight_is_percpu_source(csdev))
+		csdev->path = NULL;
+
 	source_ops(csdev)->disable(csdev, data);
-	coresight_disable_helpers(csdev, NULL);
 }
 EXPORT_SYMBOL_GPL(coresight_disable_source);
 
@@ -424,20 +550,42 @@ int coresight_resume_source(struct coresight_device *csdev)
 EXPORT_SYMBOL_GPL(coresight_resume_source);
 
 /*
- * coresight_disable_path_from : Disable components in the given path beyond
- * @nd in the list. If @nd is NULL, all the components, except the SOURCE are
- * disabled.
+ * Callers must fetch nodes from the path and pass @from and @to to the path
+ * enable/disable functions. Walk the path from @from to locate @to. If @to
+ * is found, it indicates @from and @to are in order. Otherwise, they are out
+ * of order.
  */
-static void coresight_disable_path_from(struct coresight_path *path,
-					struct coresight_node *nd)
+static bool coresight_path_nodes_in_order(struct coresight_path *path,
+					  struct coresight_node *from,
+					  struct coresight_node *to)
+{
+	struct coresight_node *nd;
+
+	if (WARN_ON_ONCE(!from || !to))
+		return false;
+
+	nd = from;
+	list_for_each_entry_from(nd, &path->path_list, link) {
+		if (nd == to)
+			return true;
+	}
+
+	return false;
+}
+
+static void coresight_disable_path_from_to(struct coresight_path *path,
+					   struct coresight_node *from,
+					   struct coresight_node *to)
 {
 	u32 type;
 	struct coresight_device *csdev, *parent, *child;
+	struct coresight_node *nd;
 
-	if (!nd)
-		nd = list_first_entry(&path->path_list, struct coresight_node, link);
+	if (!coresight_path_nodes_in_order(path, from, to))
+		return;
 
-	list_for_each_entry_continue(nd, &path->path_list, link) {
+	nd = from;
+	list_for_each_entry_from(nd, &path->path_list, link) {
 		csdev = nd->csdev;
 		type = csdev->type;
 
@@ -457,12 +605,6 @@ static void coresight_disable_path_from(struct coresight_path *path,
 			coresight_disable_sink(csdev);
 			break;
 		case CORESIGHT_DEV_TYPE_SOURCE:
-			/*
-			 * We skip the first node in the path assuming that it
-			 * is the source. So we don't expect a source device in
-			 * the middle of a path.
-			 */
-			WARN_ON(1);
 			break;
 		case CORESIGHT_DEV_TYPE_LINK:
 			parent = list_prev_entry(nd, link)->csdev;
@@ -476,12 +618,18 @@ static void coresight_disable_path_from(struct coresight_path *path,
 
 		/* Disable all helpers adjacent along the path last */
 		coresight_disable_helpers(csdev, path);
+
+		/* Iterate up to and including @to */
+		if (nd == to)
+			break;
 	}
 }
 
 void coresight_disable_path(struct coresight_path *path)
 {
-	coresight_disable_path_from(path, NULL);
+	coresight_disable_path_from_to(path,
+				       coresight_path_first_node(path),
+				       coresight_path_last_node(path));
 }
 EXPORT_SYMBOL_GPL(coresight_disable_path);
 
@@ -499,22 +647,36 @@ static int coresight_enable_helpers(struct coresight_device *csdev,
 
 		ret = coresight_enable_helper(helper, mode, path);
 		if (ret)
-			return ret;
+			goto err;
 	}
 
 	return 0;
+
+err:
+	while (i--) {
+		helper = csdev->pdata->out_conns[i]->dest_dev;
+		if (helper && coresight_is_helper(helper))
+			coresight_disable_helper(helper, path);
+	}
+
+	return ret;
 }
 
-int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)
+static int coresight_enable_path_from_to(struct coresight_path *path,
+					 enum cs_mode mode,
+					 struct coresight_node *from,
+					 struct coresight_node *to)
 {
 	int ret = 0;
 	u32 type;
 	struct coresight_node *nd;
 	struct coresight_device *csdev, *parent, *child;
-	struct coresight_device *source;
 
-	source = coresight_get_source(path);
-	list_for_each_entry_reverse(nd, &path->path_list, link) {
+	if (!coresight_path_nodes_in_order(path, from, to))
+		return -EINVAL;
+
+	nd = to;
+	list_for_each_entry_from_reverse(nd, &path->path_list, link) {
 		csdev = nd->csdev;
 		type = csdev->type;
 
@@ -553,7 +715,8 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)
 		case CORESIGHT_DEV_TYPE_LINK:
 			parent = list_prev_entry(nd, link)->csdev;
 			child = list_next_entry(nd, link)->csdev;
-			ret = coresight_enable_link(csdev, parent, child, source);
+			ret = coresight_enable_link(csdev, parent, child,
+						    coresight_get_source(path));
 			if (ret)
 				goto err_disable_helpers;
 			break;
@@ -561,6 +724,10 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)
 			ret = -EINVAL;
 			goto err_disable_helpers;
 		}
+
+		/* Iterate down to and including @from */
+		if (nd == from)
+			break;
 	}
 
 out:
@@ -568,18 +735,36 @@ int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)
 err_disable_helpers:
 	coresight_disable_helpers(csdev, path);
 err_disable_path:
-	coresight_disable_path_from(path, nd);
+	/* No device is actually enabled */
+	if (nd == to)
+		goto out;
+
+	/* Fetch the previous node, the last successfully enabled one */
+	nd = list_next_entry(nd, link);
+	coresight_disable_path_from_to(path, nd, to);
 	goto out;
 }
 
+int coresight_enable_path(struct coresight_path *path, enum cs_mode mode)
+{
+	return coresight_enable_path_from_to(path, mode,
+					     coresight_path_first_node(path),
+					     coresight_path_last_node(path));
+}
+
 struct coresight_device *coresight_get_sink(struct coresight_path *path)
 {
 	struct coresight_device *csdev;
+	struct coresight_node *nd;
 
 	if (!path)
 		return NULL;
 
-	csdev = list_last_entry(&path->path_list, struct coresight_node, link)->csdev;
+	nd = coresight_path_last_node(path);
+	if (!nd)
+		return NULL;
+
+	csdev = nd->csdev;
 	if (csdev->type != CORESIGHT_DEV_TYPE_SINK &&
 	    csdev->type != CORESIGHT_DEV_TYPE_LINKSINK)
 		return NULL;
@@ -642,15 +827,29 @@ struct coresight_device *coresight_get_sink_by_id(u32 id)
  */
 static bool coresight_get_ref(struct coresight_device *csdev)
 {
-	struct device *dev = csdev->dev.parent;
+	struct device *dev = &csdev->dev;
+	struct device *parent = csdev->dev.parent;
+	struct device_driver *drv;
 
-	/* Make sure the driver can't be removed */
-	if (!try_module_get(dev->driver->owner))
-		return false;
-	/* Make sure the device can't go away */
+	/* Make sure csdev can't go away */
 	get_device(dev);
-	pm_runtime_get_sync(dev);
+
+	/* Make sure parent device can't go away */
+	get_device(parent);
+
+	/* Make sure the driver can't be removed */
+	drv = parent->driver;
+	if (!drv || !try_module_get(drv->owner))
+		goto err_module;
+
+	/* Make sure the device is powered on */
+	pm_runtime_get_sync(parent);
 	return true;
+
+err_module:
+	put_device(parent);
+	put_device(dev);
+	return false;
 }
 
 /**
@@ -661,11 +860,15 @@ static bool coresight_get_ref(struct coresight_device *csdev)
  */
 static void coresight_put_ref(struct coresight_device *csdev)
 {
-	struct device *dev = csdev->dev.parent;
+	struct device *dev = &csdev->dev;
+	struct device *parent = csdev->dev.parent;
+	struct device_driver *drv = parent->driver;
 
-	pm_runtime_put(dev);
+	pm_runtime_put(parent);
+	if (drv)
+		module_put(drv->owner);
+	put_device(parent);
 	put_device(dev);
-	module_put(dev->driver->owner);
 }
 
 /*
@@ -739,8 +942,8 @@ static int coresight_get_trace_id(struct coresight_device *csdev,
  * Call this after creating the path and before enabling it. This leaves
  * the trace ID set on the path, or it remains 0 if it couldn't be assigned.
  */
-void coresight_path_assign_trace_id(struct coresight_path *path,
-				    enum cs_mode mode)
+int coresight_path_assign_trace_id(struct coresight_path *path,
+				   enum cs_mode mode)
 {
 	struct coresight_device *sink = coresight_get_sink(path);
 	struct coresight_node *nd;
@@ -750,15 +953,18 @@ void coresight_path_assign_trace_id(struct coresight_path *path,
 		/* Assign a trace ID to the path for the first device that wants to do it */
 		trace_id = coresight_get_trace_id(nd->csdev, mode, sink);
 
-		/*
-		 * 0 in this context is that it didn't want to assign so keep searching.
-		 * Non 0 is either success or fail.
-		 */
-		if (trace_id != 0) {
-			path->trace_id = trace_id;
-			return;
-		}
+		/* 0 means the device has no ID assignment, so keep searching */
+		if (trace_id == 0)
+			continue;
+
+		if (!IS_VALID_CS_TRACE_ID(trace_id))
+			return -EINVAL;
+
+		path->trace_id = trace_id;
+		return 0;
 	}
+
+	return -EINVAL;
 }
 
 /**
@@ -787,7 +993,7 @@ static int _coresight_build_path(struct coresight_device *csdev,
 		goto out;
 
 	if (coresight_is_percpu_source(csdev) && coresight_is_percpu_sink(sink) &&
-	    sink == per_cpu(csdev_sink, source_ops(csdev)->cpu_id(csdev))) {
+	    sink == per_cpu(csdev_sink, csdev->cpu)) {
 		if (_coresight_build_path(sink, source, sink, path) == 0) {
 			found = true;
 			goto out;
@@ -1014,7 +1220,7 @@ coresight_find_default_sink(struct coresight_device *csdev)
 	/* look for a default sink if we have not found for this device */
 	if (!csdev->def_sink) {
 		if (coresight_is_percpu_source(csdev))
-			csdev->def_sink = per_cpu(csdev_sink, source_ops(csdev)->cpu_id(csdev));
+			csdev->def_sink = per_cpu(csdev_sink, csdev->cpu);
 		if (!csdev->def_sink)
 			csdev->def_sink = coresight_find_sink(csdev, &depth);
 	}
@@ -1322,32 +1528,56 @@ void coresight_release_platform_data(struct device *dev,
 	devm_kfree(dev, pdata);
 }
 
-struct coresight_device *coresight_register(struct coresight_desc *desc)
+static struct coresight_device *
+coresight_init_device(struct coresight_desc *desc)
 {
-	int ret;
 	struct coresight_device *csdev;
-	bool registered = false;
 
 	csdev = kzalloc_obj(*csdev);
-	if (!csdev) {
-		ret = -ENOMEM;
-		goto err_out;
-	}
+	if (!csdev)
+		return ERR_PTR(-ENOMEM);
 
 	csdev->pdata = desc->pdata;
-
 	csdev->type = desc->type;
 	csdev->subtype = desc->subtype;
 	csdev->ops = desc->ops;
 	csdev->access = desc->access;
 	csdev->orphan = true;
 
+	if (desc->flags & CORESIGHT_DESC_CPU_BOUND) {
+		csdev->cpu = desc->cpu;
+	} else {
+		/* A per-CPU source or sink must set CPU_BOUND flag */
+		if (coresight_is_percpu_source(csdev) ||
+		    coresight_is_percpu_sink(csdev)) {
+			kfree(csdev);
+			return ERR_PTR(-EINVAL);
+		}
+
+		csdev->cpu = -1;
+	}
+
 	csdev->dev.type = &coresight_dev_type[desc->type];
 	csdev->dev.groups = desc->groups;
 	csdev->dev.parent = desc->dev;
 	csdev->dev.release = coresight_device_release;
 	csdev->dev.bus = &coresight_bustype;
 
+	return csdev;
+}
+
+struct coresight_device *coresight_register(struct coresight_desc *desc)
+{
+	int ret;
+	struct coresight_device *csdev;
+	bool registered = false;
+
+	csdev = coresight_init_device(desc);
+	if (IS_ERR(csdev)) {
+		ret = PTR_ERR(csdev);
+		goto err_out;
+	}
+
 	if (csdev->type == CORESIGHT_DEV_TYPE_SINK ||
 	    csdev->type == CORESIGHT_DEV_TYPE_LINKSINK) {
 		raw_spin_lock_init(&csdev->perf_sink_id_map.lock);
@@ -1398,6 +1628,7 @@ struct coresight_device *coresight_register(struct coresight_desc *desc)
 	if (ret)
 		goto out_unlock;
 
+	coresight_set_percpu_source(csdev);
 	mutex_unlock(&coresight_mutex);
 
 	if (cti_assoc_ops && cti_assoc_ops->add)
@@ -1427,6 +1658,7 @@ void coresight_unregister(struct coresight_device *csdev)
 		cti_assoc_ops->remove(csdev);
 
 	mutex_lock(&coresight_mutex);
+	coresight_clear_percpu_source(csdev);
 	etm_perf_del_symlink_sink(csdev);
 	coresight_remove_conns(csdev);
 	coresight_clear_default_sink(csdev);
@@ -1615,6 +1847,211 @@ static void coresight_release_device_list(void)
 	}
 }
 
+static struct coresight_path *coresight_cpu_get_active_path(enum cs_mode mode)
+{
+	struct coresight_device *source;
+	bool is_active = false;
+
+	source = coresight_get_percpu_source_ref(smp_processor_id());
+	if (!source)
+		return NULL;
+
+	if (coresight_get_mode(source) & mode)
+		is_active = true;
+
+	coresight_put_percpu_source_ref(source);
+
+	/*
+	 * It is expected to run in atomic context or with the CPU lock held for
+	 * sysfs mode, so it cannot be preempted to disable the path. Here
+	 * returns the active path pointer without concern that its state may
+	 * change. Since the build path has taken a reference on the component,
+	 * the path can be safely used by the caller.
+	 */
+	return is_active ? source->path : NULL;
+}
+
+/* Return: 1 if PM is required, 0 if skip, or a negative error */
+static int coresight_pm_is_needed(struct coresight_path *path)
+{
+	struct coresight_device *source, *sink;
+
+	if (this_cpu_read(percpu_pm_failed))
+		return -EIO;
+
+	if (!path)
+		return 0;
+
+	source = coresight_get_source(path);
+	sink = coresight_get_sink(path);
+	if (!source || !sink)
+		return 0;
+
+	/* pm_save_disable() and pm_restore_enable() must be paired */
+	if (coresight_ops(source)->pm_save_disable &&
+	    coresight_ops(source)->pm_restore_enable)
+		return 1;
+
+	/*
+	 * It is not permitted that the source has no callbacks while the sink
+	 * does, as the sink cannot be disabled without disabling the source,
+	 * which may lead to lockups. Fix this by enabling self-hosted PM
+	 * mode for ETM (see etm4_probe()).
+	 */
+	if (coresight_ops(sink)->pm_save_disable &&
+	    coresight_ops(sink)->pm_restore_enable) {
+		pr_warn_once("coresight PM failed: source has no PM callbacks; "
+			     "cannot safely control sink\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int coresight_pm_device_save(struct coresight_device *csdev)
+{
+	if (!csdev || !coresight_ops(csdev)->pm_save_disable)
+		return 0;
+
+	return coresight_ops(csdev)->pm_save_disable(csdev);
+}
+
+static void coresight_pm_device_restore(struct coresight_device *csdev)
+{
+	if (!csdev || !coresight_ops(csdev)->pm_restore_enable)
+		return;
+
+	coresight_ops(csdev)->pm_restore_enable(csdev);
+}
+
+static int coresight_pm_save(struct coresight_path *path)
+{
+	struct coresight_device *source = coresight_get_source(path);
+	struct coresight_node *from, *to;
+	int ret;
+
+	ret = coresight_pm_device_save(source);
+	if (ret)
+		return ret;
+
+	from = coresight_path_first_node(path);
+	/* Disable up to the node before sink */
+	to = list_prev_entry(coresight_path_last_node(path), link);
+	coresight_disable_path_from_to(path, from, to);
+
+	/*
+	 * Save the sink. Most sinks do not implement a save callback to avoid
+	 * latency from memory copying. We assume the sink's save and restore
+	 * always succeed.
+	 */
+	coresight_pm_device_save(coresight_get_sink(path));
+	return 0;
+}
+
+static void coresight_pm_restore(struct coresight_path *path)
+{
+	struct coresight_device *source = coresight_get_source(path);
+	struct coresight_device *sink = coresight_get_sink(path);
+	struct coresight_node *from, *to;
+	int ret;
+
+	coresight_pm_device_restore(sink);
+
+	from = coresight_path_first_node(path);
+	/* Enable up to the node before sink */
+	to = list_prev_entry(coresight_path_last_node(path), link);
+	ret = coresight_enable_path_from_to(path, coresight_get_mode(source),
+					    from, to);
+	if (ret)
+		goto path_failed;
+
+	coresight_pm_device_restore(source);
+	return;
+
+path_failed:
+	coresight_pm_device_save(sink);
+
+	pr_err("Failed in coresight PM restore on CPU%d: %d\n",
+	       smp_processor_id(), ret);
+
+	/*
+	 * Once PM fails on a CPU, set percpu_pm_failed and leave it set until
+	 * reboot. This prevents repeated partial transitions during idle
+	 * entry and exit.
+	 */
+	this_cpu_write(percpu_pm_failed, true);
+}
+
+static int coresight_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
+				   void *v)
+{
+	struct coresight_path *path =
+		coresight_cpu_get_active_path(CS_MODE_SYSFS | CS_MODE_PERF);
+	int ret;
+
+	ret = coresight_pm_is_needed(path);
+	if (ret <= 0)
+		return ret ? NOTIFY_BAD : NOTIFY_DONE;
+
+	switch (cmd) {
+	case CPU_PM_ENTER:
+		if (coresight_pm_save(path))
+			return NOTIFY_BAD;
+		break;
+	case CPU_PM_EXIT:
+	case CPU_PM_ENTER_FAILED:
+		coresight_pm_restore(path);
+		break;
+	default:
+		return NOTIFY_DONE;
+	}
+
+	return NOTIFY_OK;
+}
+
+static struct notifier_block coresight_cpu_pm_nb = {
+	.notifier_call = coresight_cpu_pm_notify,
+};
+
+static int coresight_dying_cpu(unsigned int cpu)
+{
+	struct coresight_path *path;
+
+	/*
+	 * The perf event layer will disable PMU events in the CPU
+	 * hotplug. Here only handles SYSFS case.
+	 */
+	path = coresight_cpu_get_active_path(CS_MODE_SYSFS);
+	if (!path)
+		return 0;
+
+	coresight_disable_sysfs(coresight_get_source(path));
+	return 0;
+}
+
+static int __init coresight_pm_setup(void)
+{
+	int ret;
+
+	ret = cpu_pm_register_notifier(&coresight_cpu_pm_nb);
+	if (ret)
+		return ret;
+
+	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_ONLINE,
+					"arm/coresight-core:dying",
+					NULL, coresight_dying_cpu);
+	if (ret)
+		cpu_pm_unregister_notifier(&coresight_cpu_pm_nb);
+
+	return ret;
+}
+
+static void coresight_pm_cleanup(void)
+{
+	cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_ONLINE);
+	cpu_pm_unregister_notifier(&coresight_cpu_pm_nb);
+}
+
 const struct bus_type coresight_bustype = {
 	.name	= "coresight",
 };
@@ -1669,9 +2106,15 @@ static int __init coresight_init(void)
 
 	/* initialise the coresight syscfg API */
 	ret = cscfg_init();
+	if (ret)
+		goto exit_notifier;
+
+	ret = coresight_pm_setup();
 	if (!ret)
 		return 0;
 
+	cscfg_exit();
+exit_notifier:
 	atomic_notifier_chain_unregister(&panic_notifier_list,
 					     &coresight_notifier);
 exit_perf:
@@ -1683,6 +2126,7 @@ static int __init coresight_init(void)
 
 static void __exit coresight_exit(void)
 {
+	coresight_pm_cleanup();
 	cscfg_exit();
 	atomic_notifier_chain_unregister(&panic_notifier_list,
 					     &coresight_notifier);
@@ -1728,10 +2172,10 @@ int coresight_etm_get_trace_id(struct coresight_device *csdev, enum cs_mode mode
 {
 	int cpu, trace_id;
 
-	if (csdev->type != CORESIGHT_DEV_TYPE_SOURCE || !source_ops(csdev)->cpu_id)
+	if (csdev->type != CORESIGHT_DEV_TYPE_SOURCE)
 		return -EINVAL;
 
-	cpu = source_ops(csdev)->cpu_id(csdev);
+	cpu = csdev->cpu;
 	switch (mode) {
 	case CS_MODE_SYSFS:
 		trace_id = coresight_trace_id_get_cpu_id(cpu);
diff --git a/drivers/hwtracing/coresight/coresight-cti-core.c b/drivers/hwtracing/coresight/coresight-cti-core.c
index 2f4c9362709a9..b2c9a4db13b4e 100644
--- a/drivers/hwtracing/coresight/coresight-cti-core.c
+++ b/drivers/hwtracing/coresight/coresight-cti-core.c
@@ -659,7 +659,7 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
 	void __iomem *base;
 	struct device *dev = &adev->dev;
 	struct cti_drvdata *drvdata = NULL;
-	struct coresight_desc cti_desc;
+	struct coresight_desc cti_desc = { 0 };
 	struct coresight_platform_data *pdata = NULL;
 	struct resource *res = &adev->res;
 
@@ -702,11 +702,14 @@ static int cti_probe(struct amba_device *adev, const struct amba_id *id)
 	 * eCPU ID. System CTIs will have the name cti_sys<I> where I is an
 	 * index allocated by order of discovery.
 	 */
-	if (drvdata->ctidev.cpu >= 0)
+	if (drvdata->ctidev.cpu >= 0) {
+		cti_desc.cpu = drvdata->ctidev.cpu;
+		cti_desc.flags = CORESIGHT_DESC_CPU_BOUND;
 		cti_desc.name = devm_kasprintf(dev, GFP_KERNEL, "cti_cpu%d",
 					       drvdata->ctidev.cpu);
-	else
+	} else {
 		cti_desc.name = coresight_alloc_device_name("cti_sys", dev);
+	}
 	if (!cti_desc.name)
 		return -ENOMEM;
 
diff --git a/drivers/hwtracing/coresight/coresight-cti-platform.c b/drivers/hwtracing/coresight/coresight-cti-platform.c
index 4eff96f48594e..d6d5388705c3e 100644
--- a/drivers/hwtracing/coresight/coresight-cti-platform.c
+++ b/drivers/hwtracing/coresight/coresight-cti-platform.c
@@ -329,6 +329,7 @@ static int cti_plat_process_filter_sigs(struct cti_drvdata *drvdata,
 	if (!tg)
 		return -ENOMEM;
 
+	tg->nr_sigs = nr_filter_sigs;
 	err = cti_plat_read_trig_group(tg, fwnode, CTI_DT_FILTER_OUT_SIGS);
 	if (!err)
 		drvdata->config.trig_out_filter |= tg->used_mask;
diff --git a/drivers/hwtracing/coresight/coresight-etm-perf.c b/drivers/hwtracing/coresight/coresight-etm-perf.c
index f85dedf89a3f9..09b21a711a876 100644
--- a/drivers/hwtracing/coresight/coresight-etm-perf.c
+++ b/drivers/hwtracing/coresight/coresight-etm-perf.c
@@ -49,7 +49,6 @@ struct etm_ctxt {
 };
 
 static DEFINE_PER_CPU(struct etm_ctxt, etm_ctxt);
-static DEFINE_PER_CPU(struct coresight_device *, csdev_src);
 
 GEN_PMU_FORMAT_ATTR(cycacc);
 GEN_PMU_FORMAT_ATTR(timestamp);
@@ -315,6 +314,115 @@ static bool sinks_compatible(struct coresight_device *a,
 	       (sink_ops(a) == sink_ops(b));
 }
 
+/*
+ * This helper is used for fetching the path pointer via the ctxt.
+ *
+ * Perf event callbacks run on the same CPU in atomic context, but AUX pause
+ * and resume may run in NMI context and preempt other callbacks. Since the
+ * event stop callback clears ctxt->event_data before the data is released,
+ * AUX pause/resume will either observe a NULL pointer and stop fetching the
+ * path pointer, or safely access event_data and the path, as the data has
+ * not yet been freed.
+ */
+static struct coresight_path *etm_event_get_ctxt_path(struct etm_ctxt *ctxt)
+{
+	struct etm_event_data *event_data;
+	struct coresight_path *path;
+
+	if (!ctxt)
+		return NULL;
+
+	event_data = READ_ONCE(ctxt->event_data);
+	if (!event_data)
+		return NULL;
+
+	path = etm_event_cpu_path(event_data, smp_processor_id());
+	if (!path)
+		return NULL;
+
+	return path;
+}
+
+static struct coresight_path *
+etm_event_build_path(struct perf_event *event, int cpu,
+		     struct coresight_device *user_sink,
+		     struct coresight_device *match_sink)
+{
+	struct coresight_path *path = NULL;
+	struct coresight_device *source, *sink;
+	int ret;
+
+	source = coresight_get_percpu_source_ref(cpu);
+
+	/*
+	 * If there is no ETM associated with this CPU or ever we try to trace
+	 * on this CPU, we handle it accordingly.
+	 */
+	if (!source)
+		return NULL;
+
+	/*
+	 * If AUX pause feature is enabled but the ETM driver does not
+	 * support the operations, skip for this source.
+	 */
+	if (event->attr.aux_start_paused &&
+	    (!source_ops(source)->pause_perf ||
+	     !source_ops(source)->resume_perf)) {
+		dev_err_once(&source->dev, "AUX pause is not supported.\n");
+		goto out;
+	}
+
+	/* If sink has been specified by user, directly use it */
+	if (user_sink) {
+		sink = user_sink;
+	} else {
+		/*
+		 * No sink provided - look for a default sink for all the ETMs,
+		 * where this event can be scheduled.
+		 *
+		 * We allocate the sink specific buffers only once for this
+		 * event. If the ETMs have different default sink devices, we
+		 * can only use a single "type" of sink as the event can carry
+		 * only one sink specific buffer. Thus we have to make sure
+		 * that the sinks are of the same type and driven by the same
+		 * driver, as the one we allocate the buffer for. We don't
+		 * trace on a CPU if the sink is not compatible.
+		 */
+
+		/* Find the default sink for this ETM */
+		sink = coresight_find_default_sink(source);
+		if (!sink)
+			goto out;
+
+		/* Check if this sink compatible with the last sink */
+		if (match_sink && !sinks_compatible(match_sink, sink))
+			goto out;
+	}
+
+	/*
+	 * Building a path doesn't enable it, it simply builds a
+	 * list of devices from source to sink that can be
+	 * referenced later when the path is actually needed.
+	 */
+	path = coresight_build_path(source, sink);
+	if (IS_ERR(path))
+		goto out;
+
+	/* ensure we can allocate a trace ID for this CPU */
+	ret = coresight_path_assign_trace_id(path, CS_MODE_PERF);
+	if (ret) {
+		coresight_release_path(path);
+		path = NULL;
+		goto out;
+	}
+
+	coresight_trace_id_perf_start(&sink->perf_sink_id_map);
+
+out:
+	coresight_put_percpu_source_ref(source);
+	return IS_ERR_OR_NULL(path) ? NULL : path;
+}
+
 static void *etm_setup_aux(struct perf_event *event, void **pages,
 			   int nr_pages, bool overwrite)
 {
@@ -322,7 +430,7 @@ static void *etm_setup_aux(struct perf_event *event, void **pages,
 	int cpu = event->cpu;
 	cpumask_t *mask;
 	struct coresight_device *sink = NULL;
-	struct coresight_device *user_sink = NULL, *last_sink = NULL;
+	struct coresight_device *user_sink = NULL;
 	struct etm_event_data *event_data = NULL;
 
 	event_data = alloc_event_data(cpu);
@@ -354,80 +462,25 @@ static void *etm_setup_aux(struct perf_event *event, void **pages,
 	 */
 	for_each_cpu(cpu, mask) {
 		struct coresight_path *path;
-		struct coresight_device *csdev;
 
-		csdev = per_cpu(csdev_src, cpu);
-		/*
-		 * If there is no ETM associated with this CPU clear it from
-		 * the mask and continue with the rest. If ever we try to trace
-		 * on this CPU, we handle it accordingly.
-		 */
-		if (!csdev) {
-			cpumask_clear_cpu(cpu, mask);
-			continue;
-		}
-
-		/*
-		 * If AUX pause feature is enabled but the ETM driver does not
-		 * support the operations, clear this CPU from the mask and
-		 * continue to next one.
-		 */
-		if (event->attr.aux_start_paused &&
-		    (!source_ops(csdev)->pause_perf || !source_ops(csdev)->resume_perf)) {
-			dev_err_once(&csdev->dev, "AUX pause is not supported.\n");
+		path = etm_event_build_path(event, cpu, user_sink, sink);
+		if (!path) {
+			/*
+			 * Failed to create a path for the CPU, clear it from
+			 * the mask and continue to next one.
+			 */
 			cpumask_clear_cpu(cpu, mask);
 			continue;
 		}
 
 		/*
-		 * No sink provided - look for a default sink for all the ETMs,
-		 * where this event can be scheduled.
-		 * We allocate the sink specific buffers only once for this
-		 * event. If the ETMs have different default sink devices, we
-		 * can only use a single "type" of sink as the event can carry
-		 * only one sink specific buffer. Thus we have to make sure
-		 * that the sinks are of the same type and driven by the same
-		 * driver, as the one we allocate the buffer for. As such
-		 * we choose the first sink and check if the remaining ETMs
-		 * have a compatible default sink. We don't trace on a CPU
-		 * if the sink is not compatible.
+		 * The first found sink is saved here and passed to
+		 * etm_event_build_path() to check whether the remaining ETMs
+		 * have a compatible default sink.
 		 */
-		if (!user_sink) {
-			/* Find the default sink for this ETM */
-			sink = coresight_find_default_sink(csdev);
-			if (!sink) {
-				cpumask_clear_cpu(cpu, mask);
-				continue;
-			}
-
-			/* Check if this sink compatible with the last sink */
-			if (last_sink && !sinks_compatible(last_sink, sink)) {
-				cpumask_clear_cpu(cpu, mask);
-				continue;
-			}
-			last_sink = sink;
-		}
+		if (!user_sink && !sink)
+			sink = coresight_get_sink(path);
 
-		/*
-		 * Building a path doesn't enable it, it simply builds a
-		 * list of devices from source to sink that can be
-		 * referenced later when the path is actually needed.
-		 */
-		path = coresight_build_path(csdev, sink);
-		if (IS_ERR(path)) {
-			cpumask_clear_cpu(cpu, mask);
-			continue;
-		}
-
-		/* ensure we can allocate a trace ID for this CPU */
-		coresight_path_assign_trace_id(path, CS_MODE_PERF);
-		if (!IS_VALID_CS_TRACE_ID(path->trace_id)) {
-			cpumask_clear_cpu(cpu, mask);
-			coresight_release_path(path);
-			continue;
-		}
-
-		coresight_trace_id_perf_start(&sink->perf_sink_id_map);
 		*etm_event_cpu_path_ptr(event_data, cpu) = path;
 	}
 
@@ -464,13 +517,23 @@ static void *etm_setup_aux(struct perf_event *event, void **pages,
 	goto out;
 }
 
-static int etm_event_resume(struct coresight_device *csdev,
-			     struct etm_ctxt *ctxt)
+static int etm_event_resume(struct coresight_path *path)
 {
-	if (!ctxt->event_data)
+	struct coresight_device *source;
+	int ret;
+
+	if (!path)
 		return 0;
 
-	return coresight_resume_source(csdev);
+	source = coresight_get_source(path);
+	if (!source)
+		return 0;
+
+	ret = coresight_resume_source(source);
+	if (ret < 0)
+		dev_err(&source->dev, "Failed to resume ETM event.\n");
+
+	return ret;
 }
 
 static void etm_event_start(struct perf_event *event, int flags)
@@ -479,23 +542,19 @@ static void etm_event_start(struct perf_event *event, int flags)
 	struct etm_event_data *event_data;
 	struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt);
 	struct perf_output_handle *handle = &ctxt->handle;
-	struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu);
+	struct coresight_device *source, *sink;
 	struct coresight_path *path;
 	u64 hw_id;
 
-	if (!csdev)
-		goto fail;
-
 	if (flags & PERF_EF_RESUME) {
-		if (etm_event_resume(csdev, ctxt) < 0) {
-			dev_err(&csdev->dev, "Failed to resume ETM event.\n");
+		path = etm_event_get_ctxt_path(ctxt);
+		if (etm_event_resume(path) < 0)
 			goto fail;
-		}
 		return;
 	}
 
 	/* Have we messed up our tracking ? */
-	if (WARN_ON(ctxt->event_data))
+	if (WARN_ON(READ_ONCE(ctxt->event_data)))
 		goto fail;
 
 	/*
@@ -523,9 +582,10 @@ static void etm_event_start(struct perf_event *event, int flags)
 
 	path = etm_event_cpu_path(event_data, cpu);
 	path->handle = handle;
-	/* We need a sink, no need to continue without one */
+	/* We need source and sink, no need to continue if any is not set */
+	source = coresight_get_source(path);
 	sink = coresight_get_sink(path);
-	if (WARN_ON_ONCE(!sink))
+	if (WARN_ON_ONCE(!source || !sink))
 		goto fail_end_stop;
 
 	/* Nothing will happen without a path */
@@ -533,7 +593,7 @@ static void etm_event_start(struct perf_event *event, int flags)
 		goto fail_end_stop;
 
 	/* Finally enable the tracer */
-	if (source_ops(csdev)->enable(csdev, event, CS_MODE_PERF, path))
+	if (coresight_enable_source(source, event, CS_MODE_PERF, path))
 		goto fail_disable_path;
 
 	/*
@@ -557,7 +617,7 @@ static void etm_event_start(struct perf_event *event, int flags)
 	/* Tell the perf core the event is alive */
 	event->hw.state = 0;
 	/* Save the event_data for this ETM */
-	ctxt->event_data = event_data;
+	WRITE_ONCE(ctxt->event_data, event_data);
 	return;
 
 fail_disable_path:
@@ -577,27 +637,26 @@ static void etm_event_start(struct perf_event *event, int flags)
 	return;
 }
 
-static void etm_event_pause(struct perf_event *event,
-			    struct coresight_device *csdev,
+static void etm_event_pause(struct coresight_path *path,
+			    struct perf_event *event,
 			    struct etm_ctxt *ctxt)
 {
-	int cpu = smp_processor_id();
-	struct coresight_device *sink;
 	struct perf_output_handle *handle = &ctxt->handle;
-	struct coresight_path *path;
+	struct coresight_device *source, *sink;
+	struct etm_event_data *event_data;
 	unsigned long size;
 
-	if (!ctxt->event_data)
+	if (!path)
 		return;
 
-	/* Stop tracer */
-	coresight_pause_source(csdev);
-
-	path = etm_event_cpu_path(ctxt->event_data, cpu);
+	source = coresight_get_source(path);
 	sink = coresight_get_sink(path);
-	if (WARN_ON_ONCE(!sink))
+	if (WARN_ON_ONCE(!source || !sink))
 		return;
 
+	/* Stop tracer */
+	coresight_pause_source(source);
+
 	/*
 	 * The per CPU sink has own interrupt handling, it might have
 	 * race condition with updating buffer on AUX trace pause if
@@ -613,8 +672,9 @@ static void etm_event_pause(struct perf_event *event,
 	if (!sink_ops(sink)->update_buffer)
 		return;
 
+	event_data = READ_ONCE(ctxt->event_data);
 	size = sink_ops(sink)->update_buffer(sink, handle,
-					     ctxt->event_data->snk_config);
+					     event_data->snk_config);
 	if (READ_ONCE(handle->event)) {
 		if (!size)
 			return;
@@ -630,14 +690,14 @@ static void etm_event_stop(struct perf_event *event, int mode)
 {
 	int cpu = smp_processor_id();
 	unsigned long size;
-	struct coresight_device *sink, *csdev = per_cpu(csdev_src, cpu);
+	struct coresight_device *source, *sink;
 	struct etm_ctxt *ctxt = this_cpu_ptr(&etm_ctxt);
 	struct perf_output_handle *handle = &ctxt->handle;
+	struct coresight_path *path = etm_event_get_ctxt_path(ctxt);
 	struct etm_event_data *event_data;
-	struct coresight_path *path;
 
 	if (mode & PERF_EF_PAUSE)
-		return etm_event_pause(event, csdev, ctxt);
+		return etm_event_pause(path, event, ctxt);
 
 	/*
 	 * If we still have access to the event_data via handle,
@@ -647,9 +707,9 @@ static void etm_event_stop(struct perf_event *event, int mode)
 	    WARN_ON(perf_get_aux(handle) != ctxt->event_data))
 		return;
 
-	event_data = ctxt->event_data;
+	event_data = READ_ONCE(ctxt->event_data);
 	/* Clear the event_data as this ETM is stopping the trace. */
-	ctxt->event_data = NULL;
+	WRITE_ONCE(ctxt->event_data, NULL);
 
 	if (event->hw.state == PERF_HES_STOPPED)
 		return;
@@ -671,19 +731,13 @@ static void etm_event_stop(struct perf_event *event, int mode)
 		return;
 	}
 
-	if (!csdev)
-		return;
-
-	path = etm_event_cpu_path(event_data, cpu);
-	if (!path)
-		return;
-
+	source = coresight_get_source(path);
 	sink = coresight_get_sink(path);
-	if (!sink)
+	if (!source || !sink)
 		return;
 
 	/* stop tracer */
-	coresight_disable_source(csdev, event);
+	coresight_disable_source(source, event);
 
 	/* tell the core */
 	event->hw.state = PERF_HES_STOPPED;
@@ -824,7 +878,7 @@ static void etm_addr_filters_sync(struct perf_event *event)
 int etm_perf_symlink(struct coresight_device *csdev, bool link)
 {
 	char entry[sizeof("cpu9999999")];
-	int ret = 0, cpu = source_ops(csdev)->cpu_id(csdev);
+	int ret = 0, cpu = csdev->cpu;
 	struct device *pmu_dev = etm_pmu.dev;
 	struct device *cs_dev = &csdev->dev;
 
@@ -833,17 +887,12 @@ int etm_perf_symlink(struct coresight_device *csdev, bool link)
 	if (!etm_perf_up)
 		return -EPROBE_DEFER;
 
-	if (link) {
+	if (link)
 		ret = sysfs_create_link(&pmu_dev->kobj, &cs_dev->kobj, entry);
-		if (ret)
-			return ret;
-		per_cpu(csdev_src, cpu) = csdev;
-	} else {
+	else
 		sysfs_remove_link(&pmu_dev->kobj, entry);
-		per_cpu(csdev_src, cpu) = NULL;
-	}
 
-	return 0;
+	return ret;
 }
 EXPORT_SYMBOL_GPL(etm_perf_symlink);
 
diff --git a/drivers/hwtracing/coresight/coresight-etm3x-core.c b/drivers/hwtracing/coresight/coresight-etm3x-core.c
index a547a6d2e0bde..862ad0786699c 100644
--- a/drivers/hwtracing/coresight/coresight-etm3x-core.c
+++ b/drivers/hwtracing/coresight/coresight-etm3x-core.c
@@ -441,6 +441,7 @@ static int etm_enable_hw(struct etm_drvdata *drvdata)
 
 struct etm_enable_arg {
 	struct etm_drvdata *drvdata;
+	struct coresight_path *path;
 	int rc;
 };
 
@@ -462,15 +463,12 @@ static void etm_enable_sysfs_smp_call(void *info)
 	arg->rc = etm_enable_hw(arg->drvdata);
 
 	/* The tracer didn't start */
-	if (arg->rc)
+	if (arg->rc) {
 		coresight_set_mode(csdev, CS_MODE_DISABLED);
-}
-
-static int etm_cpu_id(struct coresight_device *csdev)
-{
-	struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
+		return;
+	}
 
-	return drvdata->cpu;
+	csdev->path = arg->path;
 }
 
 void etm_release_trace_id(struct etm_drvdata *drvdata)
@@ -499,10 +497,13 @@ static int etm_enable_perf(struct coresight_device *csdev,
 	ret = etm_enable_hw(drvdata);
 
 	/* Failed to start tracer; roll back to DISABLED mode */
-	if (ret)
+	if (ret) {
 		coresight_set_mode(csdev, CS_MODE_DISABLED);
+		return ret;
+	}
 
-	return ret;
+	csdev->path = path;
+	return 0;
 }
 
 static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path)
@@ -521,6 +522,7 @@ static int etm_enable_sysfs(struct coresight_device *csdev, struct coresight_pat
 	 */
 	if (cpu_online(drvdata->cpu)) {
 		arg.drvdata = drvdata;
+		arg.path = path;
 		ret = smp_call_function_single(drvdata->cpu,
 					       etm_enable_sysfs_smp_call, &arg, 1);
 		if (!ret)
@@ -590,6 +592,7 @@ static void etm_disable_sysfs_smp_call(void *info)
 
 	etm_disable_hw(drvdata);
 
+	drvdata->csdev->path = NULL;
 	coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED);
 }
 
@@ -614,6 +617,7 @@ static void etm_disable_perf(struct coresight_device *csdev)
 
 	CS_LOCK(drvdata->csa.base);
 
+	drvdata->csdev->path = NULL;
 	coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED);
 
 	/*
@@ -627,13 +631,6 @@ static void etm_disable_sysfs(struct coresight_device *csdev)
 {
 	struct etm_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
-	/*
-	 * Taking hotplug lock here protects from clocks getting disabled
-	 * with tracing being left on (crash scenario) if user disable occurs
-	 * after cpu online mask indicates the cpu is offline but before the
-	 * DYING hotplug callback is serviced by the ETM driver.
-	 */
-	cpus_read_lock();
 	spin_lock(&drvdata->spinlock);
 
 	/*
@@ -644,7 +641,6 @@ static void etm_disable_sysfs(struct coresight_device *csdev)
 				 drvdata, 1);
 
 	spin_unlock(&drvdata->spinlock);
-	cpus_read_unlock();
 
 	/*
 	 * we only release trace IDs when resetting sysfs.
@@ -684,7 +680,6 @@ static void etm_disable(struct coresight_device *csdev,
 }
 
 static const struct coresight_ops_source etm_source_ops = {
-	.cpu_id		= etm_cpu_id,
 	.enable		= etm_enable,
 	.disable	= etm_disable,
 };
@@ -704,35 +699,6 @@ static int etm_online_cpu(unsigned int cpu)
 	return 0;
 }
 
-static int etm_starting_cpu(unsigned int cpu)
-{
-	if (!etmdrvdata[cpu])
-		return 0;
-
-	spin_lock(&etmdrvdata[cpu]->spinlock);
-	if (!etmdrvdata[cpu]->os_unlock) {
-		etm_os_unlock(etmdrvdata[cpu]);
-		etmdrvdata[cpu]->os_unlock = true;
-	}
-
-	if (coresight_get_mode(etmdrvdata[cpu]->csdev))
-		etm_enable_hw(etmdrvdata[cpu]);
-	spin_unlock(&etmdrvdata[cpu]->spinlock);
-	return 0;
-}
-
-static int etm_dying_cpu(unsigned int cpu)
-{
-	if (!etmdrvdata[cpu])
-		return 0;
-
-	spin_lock(&etmdrvdata[cpu]->spinlock);
-	if (coresight_get_mode(etmdrvdata[cpu]->csdev))
-		etm_disable_hw(etmdrvdata[cpu]);
-	spin_unlock(&etmdrvdata[cpu]->spinlock);
-	return 0;
-}
-
 static bool etm_arch_supported(u8 arch)
 {
 	switch (arch) {
@@ -800,13 +766,6 @@ static int __init etm_hp_setup(void)
 {
 	int ret;
 
-	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,
-					"arm/coresight:starting",
-					etm_starting_cpu, etm_dying_cpu);
-
-	if (ret)
-		return ret;
-
 	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
 					"arm/coresight:online",
 					etm_online_cpu, NULL);
@@ -817,15 +776,11 @@ static int __init etm_hp_setup(void)
 		return 0;
 	}
 
-	/* failed dyn state - remove others */
-	cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
-
 	return ret;
 }
 
 static void etm_hp_clear(void)
 {
-	cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
 	if (hp_online) {
 		cpuhp_remove_state_nocalls(hp_online);
 		hp_online = 0;
@@ -891,6 +846,8 @@ static int etm_probe(struct amba_device *adev, const struct amba_id *id)
 	desc.pdata = pdata;
 	desc.dev = dev;
 	desc.groups = coresight_etm_groups;
+	desc.cpu = drvdata->cpu;
+	desc.flags = CORESIGHT_DESC_CPU_BOUND;
 	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev))
 		return PTR_ERR(drvdata->csdev);
diff --git a/drivers/hwtracing/coresight/coresight-etm4x-core.c b/drivers/hwtracing/coresight/coresight-etm4x-core.c
index d565a73f0042e..14bb31bd6a0b9 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x-core.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x-core.c
@@ -56,10 +56,14 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
 #define PARAM_PM_SAVE_NEVER	  1 /* never save any state */
 #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
 
+/*
+ * Save option for ETM4. ETE, sysreg ETM4s and ACPI boots ignore this option and
+ * will always save.
+ */
 static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
 module_param(pm_save_enable, int, 0444);
 MODULE_PARM_DESC(pm_save_enable,
-	"Save/restore state on power down: 1 = never, 2 = self-hosted");
+	"Save/restore state on power down: 1 = never, 2 = self-hosted. MMIO and DT only.");
 
 static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
 static void etm4_set_default_config(struct etmv4_config *config);
@@ -227,13 +231,6 @@ static void etm4_cs_unlock(struct etmv4_drvdata *drvdata,
 		CS_UNLOCK(csa->base);
 }
 
-static int etm4_cpu_id(struct coresight_device *csdev)
-{
-	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
-
-	return drvdata->cpu;
-}
-
 void etm4_release_trace_id(struct etmv4_drvdata *drvdata)
 {
 	coresight_trace_id_put_cpu_id(drvdata->cpu);
@@ -241,6 +238,7 @@ void etm4_release_trace_id(struct etmv4_drvdata *drvdata)
 
 struct etm4_enable_arg {
 	struct etmv4_drvdata *drvdata;
+	struct coresight_path *path;
 	int rc;
 };
 
@@ -628,8 +626,12 @@ static void etm4_enable_sysfs_smp_call(void *info)
 	arg->rc = etm4_enable_hw(arg->drvdata);
 
 	/* The tracer didn't start */
-	if (arg->rc)
+	if (arg->rc) {
 		coresight_set_mode(csdev, CS_MODE_DISABLED);
+		return;
+	}
+
+	csdev->path = arg->path;
 }
 
 /*
@@ -897,9 +899,13 @@ static int etm4_enable_perf(struct coresight_device *csdev,
 
 out:
 	/* Failed to start tracer; roll back to DISABLED mode */
-	if (ret)
+	if (ret) {
 		coresight_set_mode(csdev, CS_MODE_DISABLED);
-	return ret;
+		return ret;
+	}
+
+	csdev->path = path;
+	return 0;
 }
 
 static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_path *path)
@@ -929,6 +935,7 @@ static int etm4_enable_sysfs(struct coresight_device *csdev, struct coresight_pa
 	 * ensures that register writes occur when cpu is powered.
 	 */
 	arg.drvdata = drvdata;
+	arg.path = path;
 	ret = smp_call_function_single(drvdata->cpu,
 				       etm4_enable_sysfs_smp_call, &arg, 1);
 	if (!ret)
@@ -1070,6 +1077,7 @@ static void etm4_disable_sysfs_smp_call(void *info)
 
 	etm4_disable_hw(drvdata);
 
+	drvdata->csdev->path = NULL;
 	coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED);
 }
 
@@ -1099,6 +1107,7 @@ static int etm4_disable_perf(struct coresight_device *csdev,
 	/* TRCVICTLR::SSSTATUS, bit[9] */
 	filters->ssstatus = (control & BIT(9));
 
+	drvdata->csdev->path = NULL;
 	coresight_set_mode(drvdata->csdev, CS_MODE_DISABLED);
 
 	/*
@@ -1113,13 +1122,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev)
 {
 	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 
-	/*
-	 * Taking hotplug lock here protects from clocks getting disabled
-	 * with tracing being left on (crash scenario) if user disable occurs
-	 * after cpu online mask indicates the cpu is offline but before the
-	 * DYING hotplug callback is serviced by the ETM driver.
-	 */
-	cpus_read_lock();
 	raw_spin_lock(&drvdata->spinlock);
 
 	/*
@@ -1133,8 +1135,6 @@ static void etm4_disable_sysfs(struct coresight_device *csdev)
 
 	cscfg_csdev_disable_active_config(csdev);
 
-	cpus_read_unlock();
-
 	/*
 	 * we only release trace IDs when resetting sysfs.
 	 * This permits sysfs users to read the trace ID after the trace
@@ -1201,18 +1201,12 @@ static void etm4_pause_perf(struct coresight_device *csdev)
 }
 
 static const struct coresight_ops_source etm4_source_ops = {
-	.cpu_id		= etm4_cpu_id,
 	.enable		= etm4_enable,
 	.disable	= etm4_disable,
 	.resume_perf	= etm4_resume_perf,
 	.pause_perf	= etm4_pause_perf,
 };
 
-static const struct coresight_ops etm4_cs_ops = {
-	.trace_id	= coresight_etm_get_trace_id,
-	.source_ops	= &etm4_source_ops,
-};
-
 static bool cpu_supports_sysreg_trace(void)
 {
 	u64 dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
@@ -1839,44 +1833,19 @@ static int etm4_online_cpu(unsigned int cpu)
 	return 0;
 }
 
-static int etm4_starting_cpu(unsigned int cpu)
+static inline bool etm4_pm_save_needed(struct etmv4_drvdata *drvdata)
 {
-	if (!etmdrvdata[cpu])
-		return 0;
-
-	raw_spin_lock(&etmdrvdata[cpu]->spinlock);
-	if (!etmdrvdata[cpu]->os_unlock)
-		etm4_os_unlock(etmdrvdata[cpu]);
-
-	if (coresight_get_mode(etmdrvdata[cpu]->csdev))
-		etm4_enable_hw(etmdrvdata[cpu]);
-	raw_spin_unlock(&etmdrvdata[cpu]->spinlock);
-	return 0;
-}
-
-static int etm4_dying_cpu(unsigned int cpu)
-{
-	if (!etmdrvdata[cpu])
-		return 0;
-
-	raw_spin_lock(&etmdrvdata[cpu]->spinlock);
-	if (coresight_get_mode(etmdrvdata[cpu]->csdev))
-		etm4_disable_hw(etmdrvdata[cpu]);
-	raw_spin_unlock(&etmdrvdata[cpu]->spinlock);
-	return 0;
+	return !!drvdata->save_state;
 }
 
-static int __etm4_cpu_save(struct etmv4_drvdata *drvdata)
+static int etm4_cpu_save(struct coresight_device *csdev)
 {
 	int i, ret = 0;
+	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 	struct etmv4_save_state *state;
-	struct coresight_device *csdev = drvdata->csdev;
 	struct csdev_access *csa;
 	struct device *etm_dev;
 
-	if (WARN_ON(!csdev))
-		return -ENODEV;
-
 	etm_dev = &csdev->dev;
 	csa = &csdev->access;
 
@@ -1979,7 +1948,7 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata)
 
 	state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR0);
 	if (drvdata->numvmidc > 4)
-		state->trcvmidcctlr0 = etm4x_read32(csa, TRCVMIDCCTLR1);
+		state->trcvmidcctlr1 = etm4x_read32(csa, TRCVMIDCCTLR1);
 
 	state->trcclaimset = etm4x_read32(csa, TRCCLAIMCLR);
 
@@ -2008,31 +1977,13 @@ static int __etm4_cpu_save(struct etmv4_drvdata *drvdata)
 	return ret;
 }
 
-static int etm4_cpu_save(struct etmv4_drvdata *drvdata)
-{
-	int ret = 0;
-
-	if (pm_save_enable != PARAM_PM_SAVE_SELF_HOSTED)
-		return 0;
-
-	/*
-	 * Save and restore the ETM Trace registers only if
-	 * the ETM is active.
-	 */
-	if (coresight_get_mode(drvdata->csdev))
-		ret = __etm4_cpu_save(drvdata);
-	return ret;
-}
-
-static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata)
+static void etm4_cpu_restore(struct coresight_device *csdev)
 {
 	int i;
+	struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 	struct etmv4_save_state *state = drvdata->save_state;
 	struct csdev_access *csa = &drvdata->csdev->access;
 
-	if (WARN_ON(!drvdata->csdev))
-		return;
-
 	etm4_cs_unlock(drvdata, csa);
 	etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
 
@@ -2102,7 +2053,7 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata)
 
 	etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR0);
 	if (drvdata->numvmidc > 4)
-		etm4x_relaxed_write32(csa, state->trcvmidcctlr0, TRCVMIDCCTLR1);
+		etm4x_relaxed_write32(csa, state->trcvmidcctlr1, TRCVMIDCCTLR1);
 
 	etm4x_relaxed_write32(csa, state->trcclaimset, TRCCLAIMSET);
 
@@ -2125,47 +2076,16 @@ static void __etm4_cpu_restore(struct etmv4_drvdata *drvdata)
 	etm4_cs_lock(drvdata, csa);
 }
 
-static void etm4_cpu_restore(struct etmv4_drvdata *drvdata)
-{
-	if (pm_save_enable != PARAM_PM_SAVE_SELF_HOSTED)
-		return;
-
-	if (coresight_get_mode(drvdata->csdev))
-		__etm4_cpu_restore(drvdata);
-}
-
-static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
-			      void *v)
-{
-	struct etmv4_drvdata *drvdata;
-	unsigned int cpu = smp_processor_id();
-
-	if (!etmdrvdata[cpu])
-		return NOTIFY_OK;
-
-	drvdata = etmdrvdata[cpu];
-
-	if (WARN_ON_ONCE(drvdata->cpu != cpu))
-		return NOTIFY_BAD;
-
-	switch (cmd) {
-	case CPU_PM_ENTER:
-		if (etm4_cpu_save(drvdata))
-			return NOTIFY_BAD;
-		break;
-	case CPU_PM_EXIT:
-	case CPU_PM_ENTER_FAILED:
-		etm4_cpu_restore(drvdata);
-		break;
-	default:
-		return NOTIFY_DONE;
-	}
-
-	return NOTIFY_OK;
-}
+static const struct coresight_ops etm4_cs_ops = {
+	.trace_id	= coresight_etm_get_trace_id,
+	.source_ops	= &etm4_source_ops,
+};
 
-static struct notifier_block etm4_cpu_pm_nb = {
-	.notifier_call = etm4_cpu_pm_notify,
+static const struct coresight_ops etm4_cs_pm_ops = {
+	.trace_id		= coresight_etm_get_trace_id,
+	.source_ops		= &etm4_source_ops,
+	.pm_save_disable	= etm4_cpu_save,
+	.pm_restore_enable	= etm4_cpu_restore,
 };
 
 /* Setup PM. Deals with error conditions and counts */
@@ -2173,17 +2093,6 @@ static int __init etm4_pm_setup(void)
 {
 	int ret;
 
-	ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb);
-	if (ret)
-		return ret;
-
-	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,
-					"arm/coresight4:starting",
-					etm4_starting_cpu, etm4_dying_cpu);
-
-	if (ret)
-		goto unregister_notifier;
-
 	ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
 					"arm/coresight4:online",
 					etm4_online_cpu, NULL);
@@ -2194,24 +2103,28 @@ static int __init etm4_pm_setup(void)
 		return 0;
 	}
 
-	/* failed dyn state - remove others */
-	cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
-
-unregister_notifier:
-	cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
 	return ret;
 }
 
 static void etm4_pm_clear(void)
 {
-	cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
-	cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
 	if (hp_online) {
 		cpuhp_remove_state_nocalls(hp_online);
 		hp_online = 0;
 	}
 }
 
+static bool etm4x_always_pm_save(struct device *dev, struct csdev_access *csa)
+{
+	/*
+	 * Only IO mem ETM devices will benefit from skipping PM save and only
+	 * DT has the option to control it, not ACPI. Otherwise system register
+	 * based ETMs and ETEs will always lose context on CPU power down, so
+	 * always save.
+	 */
+	return !csa->io_mem || is_acpi_device_node(dev_fwnode(dev));
+}
+
 static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg)
 {
 	int ret;
@@ -2221,6 +2134,7 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg)
 	struct coresight_desc desc = { 0 };
 	u8 major, minor;
 	char *type_name;
+	bool pm_save;
 
 	if (!drvdata)
 		return -EINVAL;
@@ -2248,6 +2162,21 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg)
 
 	etm4_set_default(&drvdata->config);
 
+	if (etm4x_always_pm_save(dev, init_arg->csa))
+		pm_save = true;
+	else if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE)
+		pm_save = coresight_loses_context_with_cpu(dev);
+	else
+		pm_save = pm_save_enable != PARAM_PM_SAVE_NEVER;
+
+	if (pm_save) {
+		drvdata->save_state = devm_kmalloc(dev,
+						   sizeof(struct etmv4_save_state),
+						   GFP_KERNEL);
+		if (!drvdata->save_state)
+			return -ENOMEM;
+	}
+
 	pdata = coresight_get_platform_data(dev);
 	if (IS_ERR(pdata))
 		return PTR_ERR(pdata);
@@ -2256,10 +2185,12 @@ static int etm4_add_coresight_dev(struct etm4_init_arg *init_arg)
 
 	desc.type = CORESIGHT_DEV_TYPE_SOURCE;
 	desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
-	desc.ops = &etm4_cs_ops;
+	desc.ops = etm4_pm_save_needed(drvdata) ? &etm4_cs_pm_ops : &etm4_cs_ops;
 	desc.pdata = pdata;
 	desc.dev = dev;
 	desc.groups = coresight_etmv4_groups;
+	desc.cpu = drvdata->cpu;
+	desc.flags = CORESIGHT_DESC_CPU_BOUND;
 	drvdata->csdev = coresight_register(&desc);
 	if (IS_ERR(drvdata->csdev))
 		return PTR_ERR(drvdata->csdev);
@@ -2305,17 +2236,6 @@ static int etm4_probe(struct device *dev)
 	if (ret)
 		return ret;
 
-	if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE)
-		pm_save_enable = coresight_loses_context_with_cpu(dev) ?
-			       PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER;
-
-	if (pm_save_enable != PARAM_PM_SAVE_NEVER) {
-		drvdata->save_state = devm_kmalloc(dev,
-				sizeof(struct etmv4_save_state), GFP_KERNEL);
-		if (!drvdata->save_state)
-			return -ENOMEM;
-	}
-
 	raw_spin_lock_init(&drvdata->spinlock);
 
 	drvdata->cpu = coresight_get_cpu(dev);
diff --git a/drivers/hwtracing/coresight/coresight-priv.h b/drivers/hwtracing/coresight/coresight-priv.h
index 1ea882dffd703..dddac946659fc 100644
--- a/drivers/hwtracing/coresight/coresight-priv.h
+++ b/drivers/hwtracing/coresight/coresight-priv.h
@@ -153,7 +153,7 @@ int coresight_make_links(struct coresight_device *orig,
 void coresight_remove_links(struct coresight_device *orig,
 			    struct coresight_connection *conn);
 u32 coresight_get_sink_id(struct coresight_device *csdev);
-void coresight_path_assign_trace_id(struct coresight_path *path,
+int coresight_path_assign_trace_id(struct coresight_path *path,
 				   enum cs_mode mode);
 
 #if IS_ENABLED(CONFIG_CORESIGHT_SOURCE_ETM3X)
@@ -248,6 +248,12 @@ void coresight_add_helper(struct coresight_device *csdev,
 
 void coresight_set_percpu_sink(int cpu, struct coresight_device *csdev);
 struct coresight_device *coresight_get_percpu_sink(int cpu);
+struct coresight_device *coresight_get_source(struct coresight_path *path);
+struct coresight_device *coresight_get_percpu_source_ref(int cpu);
+void coresight_put_percpu_source_ref(struct coresight_device *csdev);
+int coresight_enable_source(struct coresight_device *csdev,
+			    struct perf_event *event, enum cs_mode mode,
+			    struct coresight_path *path);
 void coresight_disable_source(struct coresight_device *csdev, void *data);
 void coresight_pause_source(struct coresight_device *csdev);
 int coresight_resume_source(struct coresight_device *csdev);
diff --git a/drivers/hwtracing/coresight/coresight-syscfg.c b/drivers/hwtracing/coresight/coresight-syscfg.c
index d7f5037953d6b..2bfdd7b45e49c 100644
--- a/drivers/hwtracing/coresight/coresight-syscfg.c
+++ b/drivers/hwtracing/coresight/coresight-syscfg.c
@@ -953,39 +953,41 @@ int cscfg_config_sysfs_activate(struct cscfg_config_desc *config_desc, bool acti
 	unsigned long cfg_hash;
 	int err = 0;
 
-	mutex_lock(&cscfg_mutex);
+	guard(mutex)(&cscfg_mutex);
 
 	cfg_hash = (unsigned long)config_desc->event_ea->var;
 
 	if (activate) {
 		/* cannot be a current active value to activate this */
-		if (cscfg_mgr->sysfs_active_config) {
-			err = -EBUSY;
-			goto exit_unlock;
-		}
-		err = _cscfg_activate_config(cfg_hash);
-		if (!err)
+		if (cscfg_mgr->sysfs_active_config)
+			return -EBUSY;
+
+		scoped_guard(raw_spinlock_irqsave, &cscfg_mgr->sysfs_store_lock) {
+			err = _cscfg_activate_config(cfg_hash);
+			if (err)
+				return err;
+
 			cscfg_mgr->sysfs_active_config = cfg_hash;
+		}
 	} else {
-		/* disable if matching current value */
-		if (cscfg_mgr->sysfs_active_config == cfg_hash) {
+		if (cscfg_mgr->sysfs_active_config != cfg_hash)
+			return -EINVAL;
+
+		scoped_guard(raw_spinlock_irqsave, &cscfg_mgr->sysfs_store_lock) {
+			/* disable if matching current value */
 			_cscfg_deactivate_config(cfg_hash);
 			cscfg_mgr->sysfs_active_config = 0;
-		} else
-			err = -EINVAL;
+		}
 	}
 
-exit_unlock:
-	mutex_unlock(&cscfg_mutex);
-	return err;
+	return 0;
 }
 
 /* set the sysfs preset value */
 void cscfg_config_sysfs_set_preset(int preset)
 {
-	mutex_lock(&cscfg_mutex);
+	guard(raw_spinlock_irqsave)(&cscfg_mgr->sysfs_store_lock);
 	cscfg_mgr->sysfs_active_preset = preset;
-	mutex_unlock(&cscfg_mutex);
 }
 
 /*
@@ -994,10 +996,9 @@ void cscfg_config_sysfs_set_preset(int preset)
  */
 void cscfg_config_sysfs_get_active_cfg(unsigned long *cfg_hash, int *preset)
 {
-	mutex_lock(&cscfg_mutex);
+	guard(raw_spinlock_irqsave)(&cscfg_mgr->sysfs_store_lock);
 	*preset = cscfg_mgr->sysfs_active_preset;
 	*cfg_hash = cscfg_mgr->sysfs_active_config;
-	mutex_unlock(&cscfg_mutex);
 }
 EXPORT_SYMBOL_GPL(cscfg_config_sysfs_get_active_cfg);
 
@@ -1201,6 +1202,7 @@ static int cscfg_create_device(void)
 	INIT_LIST_HEAD(&cscfg_mgr->load_order_list);
 	atomic_set(&cscfg_mgr->sys_active_cnt, 0);
 	cscfg_mgr->load_state = CSCFG_NONE;
+	raw_spin_lock_init(&cscfg_mgr->sysfs_store_lock);
 
 	/* setup the device */
 	dev = cscfg_device();
diff --git a/drivers/hwtracing/coresight/coresight-syscfg.h b/drivers/hwtracing/coresight/coresight-syscfg.h
index 66e2db890d820..658e93c3705f1 100644
--- a/drivers/hwtracing/coresight/coresight-syscfg.h
+++ b/drivers/hwtracing/coresight/coresight-syscfg.h
@@ -42,6 +42,7 @@ enum cscfg_load_ops {
  * @sysfs_active_config:Active config hash used if CoreSight controlled from sysfs.
  * @sysfs_active_preset:Active preset index used if CoreSight controlled from sysfs.
  * @load_state:		A multi-stage load/unload operation is in progress.
+ * @sysfs_store_lock:	Exclusive access sysfs stored variables.
  */
 struct cscfg_manager {
 	struct device dev;
@@ -54,6 +55,7 @@ struct cscfg_manager {
 	u32 sysfs_active_config;
 	int sysfs_active_preset;
 	enum cscfg_load_ops load_state;
+	raw_spinlock_t sysfs_store_lock;
 };
 
 /* get reference to dev in cscfg_manager */
diff --git a/drivers/hwtracing/coresight/coresight-sysfs.c b/drivers/hwtracing/coresight/coresight-sysfs.c
index d2a6ed8bcc74d..4b010f8bc4c08 100644
--- a/drivers/hwtracing/coresight/coresight-sysfs.c
+++ b/drivers/hwtracing/coresight/coresight-sysfs.c
@@ -5,26 +5,12 @@
  */
 
 #include <linux/device.h>
-#include <linux/idr.h>
 #include <linux/kernel.h>
 #include <linux/property.h>
 
 #include "coresight-priv.h"
 #include "coresight-trace-id.h"
 
-/*
- * Use IDR to map the hash of the source's device name
- * to the pointer of path for the source. The idr is for
- * the sources which aren't associated with CPU.
- */
-static DEFINE_IDR(path_idr);
-
-/*
- * When operating Coresight drivers from the sysFS interface, only a single
- * path can exist from a tracer (associated to a CPU) to a sink.
- */
-static DEFINE_PER_CPU(struct coresight_path *, tracer_path);
-
 ssize_t coresight_simple_show_pair(struct device *_dev,
 			      struct device_attribute *attr, char *buf)
 {
@@ -53,6 +39,26 @@ ssize_t coresight_simple_show32(struct device *_dev,
 }
 EXPORT_SYMBOL_GPL(coresight_simple_show32);
 
+static void coresight_source_get_refcnt(struct coresight_device *csdev)
+{
+	/*
+	 * There could be multiple applications driving the software
+	 * source. So keep the refcount for each such user when the
+	 * source is already enabled.
+	 *
+	 * No need to increment the reference counter for other source
+	 * types, as multiple enables are the same as a single enable.
+	 */
+	if (coresight_is_software_source(csdev))
+		csdev->refcnt++;
+}
+
+static void coresight_source_put_refcnt(struct coresight_device *csdev)
+{
+	if (coresight_is_software_source(csdev))
+		csdev->refcnt--;
+}
+
 static int coresight_enable_source_sysfs(struct coresight_device *csdev,
 					 enum cs_mode mode,
 					 struct coresight_path *path)
@@ -66,19 +72,19 @@ static int coresight_enable_source_sysfs(struct coresight_device *csdev,
 	 */
 	lockdep_assert_held(&coresight_mutex);
 	if (coresight_get_mode(csdev) != CS_MODE_SYSFS) {
-		ret = source_ops(csdev)->enable(csdev, NULL, mode, path);
+		ret = coresight_enable_source(csdev, NULL, mode, path);
 		if (ret)
 			return ret;
 	}
 
-	csdev->refcnt++;
+	coresight_source_get_refcnt(csdev);
 
 	return 0;
 }
 
 /**
- *  coresight_disable_source_sysfs - Drop the reference count by 1 and disable
- *  the device if there are no users left.
+ *  coresight_disable_source_sysfs - Drop the reference count by 1 for software
+ *  sources. Disable the device if there are no users left.
  *
  *  @csdev: The coresight device to disable
  *  @data: Opaque data to pass on to the disable function of the source device.
@@ -93,7 +99,7 @@ static bool coresight_disable_source_sysfs(struct coresight_device *csdev,
 	if (coresight_get_mode(csdev) != CS_MODE_SYSFS)
 		return false;
 
-	csdev->refcnt--;
+	coresight_source_put_refcnt(csdev);
 	if (csdev->refcnt == 0) {
 		coresight_disable_source(csdev, data);
 		return true;
@@ -162,18 +168,17 @@ static int coresight_validate_source_sysfs(struct coresight_device *csdev,
 		return -EINVAL;
 	}
 
+	if (coresight_is_percpu_source(csdev) && !cpu_online(csdev->cpu))
+		return -ENODEV;
+
 	return 0;
 }
 
 int coresight_enable_sysfs(struct coresight_device *csdev)
 {
-	int cpu, ret = 0;
+	int ret = 0;
 	struct coresight_device *sink;
 	struct coresight_path *path;
-	enum coresight_dev_subtype_source subtype;
-	u32 hash;
-
-	subtype = csdev->subtype.source_subtype;
 
 	mutex_lock(&coresight_mutex);
 
@@ -188,13 +193,7 @@ int coresight_enable_sysfs(struct coresight_device *csdev)
 	 * doesn't hold coresight_mutex.
 	 */
 	if (coresight_get_mode(csdev) == CS_MODE_SYSFS) {
-		/*
-		 * There could be multiple applications driving the software
-		 * source. So keep the refcount for each such user when the
-		 * source is already enabled.
-		 */
-		if (subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE)
-			csdev->refcnt++;
+		coresight_source_get_refcnt(csdev);
 		goto out;
 	}
 
@@ -211,8 +210,8 @@ int coresight_enable_sysfs(struct coresight_device *csdev)
 		goto out;
 	}
 
-	coresight_path_assign_trace_id(path, CS_MODE_SYSFS);
-	if (!IS_VALID_CS_TRACE_ID(path->trace_id))
+	ret = coresight_path_assign_trace_id(path, CS_MODE_SYSFS);
+	if (ret)
 		goto err_path;
 
 	ret = coresight_enable_path(path, CS_MODE_SYSFS);
@@ -223,35 +222,6 @@ int coresight_enable_sysfs(struct coresight_device *csdev)
 	if (ret)
 		goto err_source;
 
-	switch (subtype) {
-	case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC:
-		/*
-		 * When working from sysFS it is important to keep track
-		 * of the paths that were created so that they can be
-		 * undone in 'coresight_disable()'.  Since there can only
-		 * be a single session per tracer (when working from sysFS)
-		 * a per-cpu variable will do just fine.
-		 */
-		cpu = source_ops(csdev)->cpu_id(csdev);
-		per_cpu(tracer_path, cpu) = path;
-		break;
-	case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE:
-	case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM:
-	case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS:
-		/*
-		 * Use the hash of source's device name as ID
-		 * and map the ID to the pointer of the path.
-		 */
-		hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev)));
-		ret = idr_alloc_u32(&path_idr, path, &hash, hash, GFP_KERNEL);
-		if (ret)
-			goto err_source;
-		break;
-	default:
-		/* We can't be here */
-		break;
-	}
-
 out:
 	mutex_unlock(&coresight_mutex);
 	return ret;
@@ -267,9 +237,8 @@ EXPORT_SYMBOL_GPL(coresight_enable_sysfs);
 
 void coresight_disable_sysfs(struct coresight_device *csdev)
 {
-	int cpu, ret;
-	struct coresight_path *path = NULL;
-	u32 hash;
+	struct coresight_path *path;
+	int ret;
 
 	mutex_lock(&coresight_mutex);
 
@@ -277,32 +246,15 @@ void coresight_disable_sysfs(struct coresight_device *csdev)
 	if (ret)
 		goto out;
 
+	/*
+	 * coresight_disable_source_sysfs() clears the 'csdev->path' pointer
+	 * when disabling the source. Retrieve the path pointer here.
+	 */
+	path = csdev->path;
+
 	if (!coresight_disable_source_sysfs(csdev, NULL))
 		goto out;
 
-	switch (csdev->subtype.source_subtype) {
-	case CORESIGHT_DEV_SUBTYPE_SOURCE_PROC:
-		cpu = source_ops(csdev)->cpu_id(csdev);
-		path = per_cpu(tracer_path, cpu);
-		per_cpu(tracer_path, cpu) = NULL;
-		break;
-	case CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE:
-	case CORESIGHT_DEV_SUBTYPE_SOURCE_TPDM:
-	case CORESIGHT_DEV_SUBTYPE_SOURCE_OTHERS:
-		hash = hashlen_hash(hashlen_string(NULL, dev_name(&csdev->dev)));
-		/* Find the path by the hash. */
-		path = idr_find(&path_idr, hash);
-		if (path == NULL) {
-			pr_err("Path is not found for %s\n", dev_name(&csdev->dev));
-			goto out;
-		}
-		idr_remove(&path_idr, hash);
-		break;
-	default:
-		/* We can't be here */
-		break;
-	}
-
 	coresight_disable_path(path);
 	coresight_release_path(path);
 
@@ -360,6 +312,13 @@ static ssize_t enable_source_store(struct device *dev,
 	if (ret)
 		return ret;
 
+	/*
+	 * CoreSight hotplug callbacks in core layer control a activated path
+	 * from its source to sink. Taking hotplug lock here protects a race
+	 * condition with hotplug callbacks.
+	 */
+	guard(cpus_read_lock)();
+
 	if (val) {
 		ret = coresight_enable_sysfs(csdev);
 		if (ret)
diff --git a/drivers/hwtracing/coresight/coresight-tmc-etr.c b/drivers/hwtracing/coresight/coresight-tmc-etr.c
index 4dc1defe27a5f..361a433e6f0c5 100644
--- a/drivers/hwtracing/coresight/coresight-tmc-etr.c
+++ b/drivers/hwtracing/coresight/coresight-tmc-etr.c
@@ -154,7 +154,7 @@ tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
 	for (i = 0; i < tmc_pages->nr_pages; i++) {
 		page_start = tmc_pages->daddrs[i];
 		if (addr >= page_start && addr < (page_start + PAGE_SIZE))
-			return i * PAGE_SIZE + (addr - page_start);
+			return (long)i * PAGE_SIZE + (addr - page_start);
 	}
 
 	return -EINVAL;
@@ -1379,7 +1379,7 @@ alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
 	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(event->cpu);
 
 	/* Use the minimum limit if the required size is smaller */
-	size = nr_pages << PAGE_SHIFT;
+	size = (ssize_t)nr_pages << PAGE_SHIFT;
 	size = max_t(ssize_t, size, TMC_ETR_PERF_MIN_BUF_SIZE);
 
 	/*
diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c
index 1511f8eb95afb..c7cbca45f2deb 100644
--- a/drivers/hwtracing/coresight/coresight-trbe.c
+++ b/drivers/hwtracing/coresight/coresight-trbe.c
@@ -116,6 +116,20 @@ static int trbe_errata_cpucaps[] = {
  */
 #define TRBE_WORKAROUND_OVERWRITE_FILL_MODE_SKIP_BYTES	256
 
+/*
+ * struct trbe_save_state: Register values representing TRBE state
+ * @trblimitr		- Trace Buffer Limit Address Register value
+ * @trbbaser		- Trace Buffer Base Register value
+ * @trbptr		- Trace Buffer Write Pointer Register value
+ * @trbsr		- Trace Buffer Status Register value
+ */
+struct trbe_save_state {
+	u64 trblimitr;
+	u64 trbbaser;
+	u64 trbptr;
+	u64 trbsr;
+};
+
 /*
  * struct trbe_cpudata: TRBE instance specific data
  * @trbe_flag		- TRBE dirty/access flag support
@@ -134,6 +148,7 @@ struct trbe_cpudata {
 	enum cs_mode mode;
 	struct trbe_buf *buf;
 	struct trbe_drvdata *drvdata;
+	struct trbe_save_state save_state;
 	DECLARE_BITMAP(errata, TRBE_ERRATA_MAX);
 };
 
@@ -1189,6 +1204,46 @@ static irqreturn_t arm_trbe_irq_handler(int irq, void *dev)
 	return IRQ_HANDLED;
 }
 
+static int arm_trbe_save(struct coresight_device *csdev)
+{
+	struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
+	struct trbe_save_state *state = &cpudata->save_state;
+
+	state->trblimitr = read_sysreg_s(SYS_TRBLIMITR_EL1);
+
+	/* Disable the unit, ensure the writes to memory are complete */
+	if (state->trblimitr & TRBLIMITR_EL1_E)
+		trbe_drain_and_disable_local(cpudata);
+
+	state->trbbaser = read_sysreg_s(SYS_TRBBASER_EL1);
+	state->trbptr = read_sysreg_s(SYS_TRBPTR_EL1);
+	state->trbsr = read_sysreg_s(SYS_TRBSR_EL1);
+	return 0;
+}
+
+static void arm_trbe_restore(struct coresight_device *csdev)
+{
+	struct trbe_cpudata *cpudata = dev_get_drvdata(&csdev->dev);
+	struct trbe_save_state *state = &cpudata->save_state;
+
+	write_sysreg_s(state->trbbaser, SYS_TRBBASER_EL1);
+	write_sysreg_s(state->trbptr, SYS_TRBPTR_EL1);
+	write_sysreg_s(state->trbsr, SYS_TRBSR_EL1);
+
+	if (!(state->trblimitr & TRBLIMITR_EL1_E)) {
+		write_sysreg_s(state->trblimitr, SYS_TRBLIMITR_EL1);
+	} else {
+		/*
+		 * The section K5.5 Context switching, Arm ARM (ARM DDI 0487
+		 * L.a), S_PKLXF requires a Context synchronization event to
+		 * guarantee the Trace Buffer Unit will observe the new values
+		 * of the system registers.
+		 */
+		isb();
+		set_trbe_enabled(cpudata, state->trblimitr);
+	}
+}
+
 static const struct coresight_ops_sink arm_trbe_sink_ops = {
 	.enable		= arm_trbe_enable,
 	.disable	= arm_trbe_disable,
@@ -1198,7 +1253,9 @@ static const struct coresight_ops_sink arm_trbe_sink_ops = {
 };
 
 static const struct coresight_ops arm_trbe_cs_ops = {
-	.sink_ops	= &arm_trbe_sink_ops,
+	.pm_save_disable	= arm_trbe_save,
+	.pm_restore_enable	= arm_trbe_restore,
+	.sink_ops		= &arm_trbe_sink_ops,
 };
 
 static ssize_t align_show(struct device *dev, struct device_attribute *attr, char *buf)
@@ -1289,6 +1346,8 @@ static void arm_trbe_register_coresight_cpu(struct trbe_drvdata *drvdata, int cp
 	desc.ops = &arm_trbe_cs_ops;
 	desc.groups = arm_trbe_groups;
 	desc.dev = dev;
+	desc.cpu = cpu;
+	desc.flags = CORESIGHT_DESC_CPU_BOUND;
 	trbe_csdev = coresight_register(&desc);
 	if (IS_ERR(trbe_csdev))
 		goto cpu_clear;
diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
index 8c935f867a37a..ea3e7e92465dc 100644
--- a/drivers/i2c/busses/Kconfig
+++ b/drivers/i2c/busses/Kconfig
@@ -850,6 +850,17 @@ config I2C_LS2X
 	  This driver can also be built as a module. If so, the module
 	  will be called i2c-ls2x.
 
+config I2C_LS2X_V2
+	tristate "Loongson-2 Fast Speed I2C adapter"
+	depends on LOONGARCH || COMPILE_TEST
+	select REGMAP_MMIO
+	help
+	  If you say yes to this option, support will be included for the
+	  I2C interface on the Loongson-2K0300 SoCs.
+
+	  This driver can also be built as a module. If so, the module
+	  will be called i2c-ls2x-v2.
+
 config I2C_MLXBF
         tristate "Mellanox BlueField I2C controller"
         depends on (MELLANOX_PLATFORM && ARM64) || COMPILE_TEST
diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
index 547123ab351f1..3755c54b3d828 100644
--- a/drivers/i2c/busses/Makefile
+++ b/drivers/i2c/busses/Makefile
@@ -80,6 +80,7 @@ obj-$(CONFIG_I2C_KEBA)		+= i2c-keba.o
 obj-$(CONFIG_I2C_KEMPLD)	+= i2c-kempld.o
 obj-$(CONFIG_I2C_LPC2K)		+= i2c-lpc2k.o
 obj-$(CONFIG_I2C_LS2X)		+= i2c-ls2x.o
+obj-$(CONFIG_I2C_LS2X_V2)	+= i2c-ls2x-v2.o
 obj-$(CONFIG_I2C_MESON)		+= i2c-meson.o
 obj-$(CONFIG_I2C_MICROCHIP_CORE)	+= i2c-microchip-corei2c.o
 obj-$(CONFIG_I2C_MPC)		+= i2c-mpc.o
diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c
index 4dc57fd561703..4bb0abda0a91a 100644
--- a/drivers/i2c/busses/i2c-designware-common.c
+++ b/drivers/i2c/busses/i2c-designware-common.c
@@ -958,8 +958,8 @@ int i2c_dw_probe(struct dw_i2c_dev *dev)
 	 * registered to the device core and immediate resume in case bus has
 	 * registered I2C slaves that do I2C transfers in their probe.
 	 */
-	ACQUIRE(pm_runtime_noresume, pm)(dev->dev);
-	ret = ACQUIRE_ERR(pm_runtime_noresume, &pm);
+	PM_RUNTIME_ACQUIRE(dev->dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
 	if (ret)
 		return ret;
 
diff --git a/drivers/i2c/busses/i2c-ls2x-v2.c b/drivers/i2c/busses/i2c-ls2x-v2.c
new file mode 100644
index 0000000000000..517760d701690
--- /dev/null
+++ b/drivers/i2c/busses/i2c-ls2x-v2.c
@@ -0,0 +1,544 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Loongson-2K0300 I2C controller driver
+ *
+ * Copyright (C) 2025-2026 Loongson Technology Corporation Limited
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+#include <linux/time.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+/* Loongson-2 fast I2C offset registers */
+#define LOONGSON2_I2C_CR1	0x00	/* I2C control 1 register */
+#define LOONGSON2_I2C_CR2	0x04	/* I2C control 2 register */
+#define LOONGSON2_I2C_OAR	0x08	/* I2C slave address register */
+#define LOONGSON2_I2C_DR	0x10	/* I2C data register */
+#define LOONGSON2_I2C_SR1	0x14	/* I2C status 1 register */
+#define LOONGSON2_I2C_SR2	0x18	/* I2C status 2 register */
+#define LOONGSON2_I2C_CCR	0x1c	/* I2C clock control register */
+#define LOONGSON2_I2C_TRISE	0x20	/* I2C trise register */
+#define LOONGSON2_I2C_FLTR	0x24
+
+/* Bitfields of I2C control 1 register */
+#define LOONGSON2_I2C_CR1_PE		BIT(0)	/* Peripheral enable */
+#define LOONGSON2_I2C_CR1_START		BIT(8)	/* Start generation */
+#define LOONGSON2_I2C_CR1_STOP		BIT(9)	/* Stop generation */
+#define LOONGSON2_I2C_CR1_ACK		BIT(10)	/* Acknowledge enable */
+#define LOONGSON2_I2C_CR1_POS		BIT(11)	/* Acknowledge/PEC Position (for data reception) */
+
+#define LOONGSON2_I2C_CR1_OP_MASK	(LOONGSON2_I2C_CR1_START | LOONGSON2_I2C_CR1_STOP)
+
+/* Bitfields of I2C control 2 register */
+#define LOONGSON2_I2C_CR2_FREQ		GENMASK(5, 0)	/* APB Clock Frequency in MHz */
+#define LOONGSON2_I2C_CR2_ITERREN	BIT(8)	/* Fault-Class Interrupt Enable */
+#define LOONGSON2_I2C_CR2_ITEVTEN	BIT(9)	/* Event-Based Interrupt Enable */
+#define LOONGSON2_I2C_CR2_ITBUFEN	BIT(10)	/* Cache-Class Interrupt Enable */
+
+#define LOONGSON2_I2C_CR2_INT_MASK	\
+	(LOONGSON2_I2C_CR2_ITBUFEN | LOONGSON2_I2C_CR2_ITEVTEN | LOONGSON2_I2C_CR2_ITERREN)
+
+/* Bitfields of I2C status 1 register */
+#define LOONGSON2_I2C_SR1_SB		BIT(0)	/* Start bit (Master mode) */
+#define LOONGSON2_I2C_SR1_ADDR		BIT(1)	/* Address sent (master mode) */
+#define LOONGSON2_I2C_SR1_BTF		BIT(2)	/* Byte transfer finished */
+#define LOONGSON2_I2C_SR1_RXNE		BIT(6)	/* Data register not empty (receivers) */
+#define LOONGSON2_I2C_SR1_TXE		BIT(7)	/* Data register empty (transmitters) */
+#define LOONGSON2_I2C_SR1_BERR		BIT(8)	/* Bus error */
+#define LOONGSON2_I2C_SR1_ARLO		BIT(9)	/* Arbitration lost (master mode) */
+#define LOONGSON2_I2C_SR1_AF		BIT(10)	/* Acknowledge failure */
+
+#define LOONGSON2_I2C_SR1_ITEVTEN_MASK	\
+	(LOONGSON2_I2C_SR1_BTF | LOONGSON2_I2C_SR1_ADDR | LOONGSON2_I2C_SR1_SB)
+#define LOONGSON2_I2C_SR1_ITBUFEN_MASK	(LOONGSON2_I2C_SR1_TXE | LOONGSON2_I2C_SR1_RXNE)
+#define LOONGSON2_I2C_SR1_ITERREN_MASK	\
+	(LOONGSON2_I2C_SR1_AF | LOONGSON2_I2C_SR1_ARLO | LOONGSON2_I2C_SR1_BERR)
+
+/* Bitfields of I2C status 2 register */
+#define LOONGSON2_I2C_SR2_MSL		BIT(0)	/* Master/slave */
+#define LOONGSON2_I2C_SR2_BUSY		BIT(1)	/* Bus busy */
+#define LOONGSON2_I2C_SR2_TRA		BIT(2)	/* Transmitter/receiver */
+#define LOONGSON2_I2C_SR2_GENCALL	BIT(4)	/* General call address (Slave mode) */
+
+/* Bitfields of I2C clock control register */
+#define LOONGSON2_I2C_CCR_CCR		GENMASK(11, 0)
+#define LOONGSON2_I2C_CCR_DUTY		BIT(14)
+#define LOONGSON2_I2C_CCR_FS		BIT(15)
+
+/* Bitfields of I2C trise register */
+#define LOONGSON2_I2C_TRISE_SCL		GENMASK(5, 0)
+
+#define LOONGSON2_I2C_FREE_SLEEP_US	10
+#define LOONGSON2_I2C_FREE_TIMEOUT_US	(2 * USEC_PER_MSEC)
+
+/**
+ * struct loongson2_i2c_msg - client specific data
+ * @buf: data buffer
+ * @count: number of bytes to be transferred
+ * @result: result of the transfer
+ * @addr: 8-bit slave addr, including r/w bit
+ * @stop: last I2C msg to be sent, i.e. STOP to be generated
+ */
+struct loongson2_i2c_msg {
+	u8	*buf;
+	u32	count;
+	int	result;
+	u8      addr;
+	bool	stop;
+};
+
+/**
+ * struct loongson2_i2c_priv - private data of the controller
+ * @adapter: I2C adapter for this controller
+ * @complete: completion of I2C message
+ * @clk: hw i2c clock
+ * @regmap: regmap of the I2C device
+ * @parent_rate_MHz: I2C clock parent rate
+ * @msg: I2C transfer information
+ */
+struct loongson2_i2c_priv {
+	struct i2c_adapter		adapter;
+	struct completion		complete;
+	struct clk			*clk;
+	struct regmap			*regmap;
+	unsigned long			parent_rate_MHz;
+	struct loongson2_i2c_msg	msg;
+};
+
+static void loongson2_i2c_disable_irq(struct loongson2_i2c_priv *priv)
+{
+	regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR2, LOONGSON2_I2C_CR2_INT_MASK, 0);
+}
+
+static void loongson2_i2c_read_msg(struct loongson2_i2c_priv *priv)
+{
+	struct loongson2_i2c_msg *msg = &priv->msg;
+	u32 rbuf;
+
+	regmap_read(priv->regmap, LOONGSON2_I2C_DR, &rbuf);
+	*msg->buf++ = rbuf;
+	msg->count--;
+}
+
+static void loongson2_i2c_write_msg(struct loongson2_i2c_priv *priv, u8 byte)
+{
+	regmap_write(priv->regmap, LOONGSON2_I2C_DR, byte);
+}
+
+static void loongson2_i2c_terminate_xfer(struct loongson2_i2c_priv *priv)
+{
+	struct loongson2_i2c_msg *msg = &priv->msg;
+
+	loongson2_i2c_disable_irq(priv);
+	regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_OP_MASK,
+			   msg->stop ? LOONGSON2_I2C_CR1_STOP : LOONGSON2_I2C_CR1_START);
+	complete(&priv->complete);
+}
+
+static void loongson2_i2c_handle_write(struct loongson2_i2c_priv *priv)
+{
+	struct loongson2_i2c_msg *msg = &priv->msg;
+
+	if (msg->count) {
+		loongson2_i2c_write_msg(priv, *msg->buf++);
+		if (!--msg->count)
+			regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR2,
+					   LOONGSON2_I2C_CR2_ITBUFEN, 0);
+	} else {
+		loongson2_i2c_terminate_xfer(priv);
+	}
+}
+
+static void loongson2_i2c_handle_rx_addr(struct loongson2_i2c_priv *priv)
+{
+	struct loongson2_i2c_msg *msg = &priv->msg;
+
+	switch (msg->count) {
+	case 0:
+		loongson2_i2c_terminate_xfer(priv);
+		break;
+	case 1:
+		/* Enable NACK and reset POS (Acknowledge position) */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1,
+				   LOONGSON2_I2C_CR1_ACK | LOONGSON2_I2C_CR1_POS, 0);
+		/* Set STOP or RepSTART */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_OP_MASK,
+				   msg->stop ? LOONGSON2_I2C_CR1_STOP : LOONGSON2_I2C_CR1_START);
+		break;
+	case 2:
+		/* Enable NACK */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_ACK, 0);
+		/* Set POS (NACK position) */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_POS,
+				   LOONGSON2_I2C_CR1_POS);
+		break;
+
+	default:
+		/* Enable ACK */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_ACK,
+				   LOONGSON2_I2C_CR1_ACK);
+		/* Reset POS (ACK position) */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_POS, 0);
+		break;
+	}
+}
+
+static void loongson2_i2c_isr_error(u32 status, void *data)
+{
+	struct loongson2_i2c_priv *priv = data;
+	struct loongson2_i2c_msg *msg = &priv->msg;
+
+	/* Arbitration lost */
+	if (status & LOONGSON2_I2C_SR1_ARLO) {
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_SR1, LOONGSON2_I2C_SR1_ARLO, 0);
+		msg->result = -EAGAIN;
+		goto out;
+	}
+
+	/*
+	 * Acknowledge failure:
+	 * In master transmitter mode a Stop must be generated by software.
+	 */
+	if (status & LOONGSON2_I2C_SR1_AF) {
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_STOP,
+				   LOONGSON2_I2C_CR1_STOP);
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_SR1, LOONGSON2_I2C_SR1_AF, 0);
+		msg->result = -EIO;
+		goto out;
+	}
+
+	/* Bus error */
+	if (status & LOONGSON2_I2C_SR1_BERR) {
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_SR1, LOONGSON2_I2C_SR1_BERR, 0);
+		msg->result = -EIO;
+		goto out;
+	}
+
+out:
+	loongson2_i2c_disable_irq(priv);
+	complete(&priv->complete);
+}
+
+static void loongson2_i2c_handle_read(struct loongson2_i2c_priv *priv)
+{
+	struct loongson2_i2c_msg *msg = &priv->msg;
+
+	switch (msg->count) {
+	case 1:
+		loongson2_i2c_disable_irq(priv);
+		loongson2_i2c_read_msg(priv);
+		complete(&priv->complete);
+		break;
+	case 2:
+	case 3:
+		/*
+		 * For 2-byte/3-byte reception and for N-byte reception with N > 3, we have to
+		 * wait for byte transferred finished event before reading data.
+		 * Just disable buffer interrupt in order to avoid another system preemption due
+		 * to RX not empty event.
+		 */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR2, LOONGSON2_I2C_CR2_ITBUFEN, 0);
+		break;
+	default:
+		/*
+		 * For N byte reception with N > 3 we directly read data register
+		 * until N-2 data.
+		 */
+		loongson2_i2c_read_msg(priv);
+		break;
+	}
+}
+
+static void loongson2_i2c_handle_rx_done(struct loongson2_i2c_priv *priv)
+{
+	struct loongson2_i2c_msg *msg = &priv->msg;
+
+	switch (msg->count) {
+	case 2:
+		/*
+		 * The STOP/START bit has to be set before reading the last two bytes.
+		 * After that, we could read the last two bytes.
+		 */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_OP_MASK,
+				   msg->stop ? LOONGSON2_I2C_CR1_STOP : LOONGSON2_I2C_CR1_START);
+
+		for (unsigned int i = msg->count; i > 0; i--)
+			loongson2_i2c_read_msg(priv);
+
+		loongson2_i2c_disable_irq(priv);
+
+		complete(&priv->complete);
+		break;
+	case 3:
+		/*
+		 * In order to generate the NACK after the last received data byte, enable NACK
+		 * before reading N-2 data.
+		 */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_ACK, 0);
+		loongson2_i2c_read_msg(priv);
+		break;
+	default:
+		loongson2_i2c_read_msg(priv);
+		break;
+	}
+}
+
+static irqreturn_t loongson2_i2c_isr_event(int irq, void *data)
+{
+	struct loongson2_i2c_priv *priv = data;
+	struct device *dev = regmap_get_device(priv->regmap);
+	struct loongson2_i2c_msg *msg = &priv->msg;
+	u32 status, ien, event, cr2, possible_status;
+
+	regmap_read(priv->regmap, LOONGSON2_I2C_SR1, &status);
+	if (status & LOONGSON2_I2C_SR1_ITERREN_MASK) {
+		loongson2_i2c_isr_error(status, data);
+		return IRQ_NONE;
+	}
+
+	regmap_read(priv->regmap, LOONGSON2_I2C_CR2, &cr2);
+	ien = cr2 & LOONGSON2_I2C_CR2_INT_MASK;
+
+	/* Update possible_status if buffer interrupt is enabled */
+	possible_status = LOONGSON2_I2C_SR1_ITEVTEN_MASK;
+	if (ien & LOONGSON2_I2C_CR2_ITBUFEN)
+		possible_status |= LOONGSON2_I2C_SR1_ITBUFEN_MASK;
+
+	event = status & possible_status;
+	if (!event) {
+		dev_dbg(dev, "spurious evt IRQ (status=0x%08x, ien=0x%08x)\n", status, ien);
+		return IRQ_NONE;
+	}
+
+	/* Start condition generated */
+	if (event & LOONGSON2_I2C_SR1_SB)
+		loongson2_i2c_write_msg(priv, msg->addr);
+
+	/* I2C Address sent */
+	if (event & LOONGSON2_I2C_SR1_ADDR) {
+		if (msg->addr & I2C_M_RD)
+			loongson2_i2c_handle_rx_addr(priv);
+		/* Clear ADDR flag */
+		regmap_read(priv->regmap, LOONGSON2_I2C_SR2, &status);
+		/* Enable buffer interrupts for RX/TX not empty events */
+		regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR2, LOONGSON2_I2C_CR2_ITBUFEN,
+				   LOONGSON2_I2C_CR2_ITBUFEN);
+	}
+
+	/* TX empty */
+	if ((event & LOONGSON2_I2C_SR1_TXE) && !(msg->addr & I2C_M_RD))
+		loongson2_i2c_handle_write(priv);
+
+	/* RX not empty */
+	if ((event & LOONGSON2_I2C_SR1_RXNE) && (msg->addr & I2C_M_RD))
+		loongson2_i2c_handle_read(priv);
+
+	/*
+	 * The BTF (Byte Transfer finished) event occurs when:
+	 * - in reception: a new byte is received in the shift register
+	 * but the previous byte has not been read yet from data register
+	 * - in transmission: a new byte should be sent but the data register
+	 * has not been written yet
+	 */
+	if (event & LOONGSON2_I2C_SR1_BTF) {
+		if (msg->addr & I2C_M_RD)
+			loongson2_i2c_handle_rx_done(priv);
+		else
+			loongson2_i2c_handle_write(priv);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int loongson2_i2c_xfer_msg(struct loongson2_i2c_priv *priv, struct i2c_msg *msg,
+				  bool is_stop)
+{
+	struct loongson2_i2c_msg *l_msg = &priv->msg;
+	unsigned long timeout;
+
+	l_msg->addr   = i2c_8bit_addr_from_msg(msg);
+	l_msg->buf    = msg->buf;
+	l_msg->count  = msg->len;
+	l_msg->stop   = is_stop;
+	l_msg->result = 0;
+
+	reinit_completion(&priv->complete);
+
+	/* Enable events and errors interrupts */
+	regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR2,
+			   LOONGSON2_I2C_CR2_ITEVTEN | LOONGSON2_I2C_CR2_ITERREN,
+			   LOONGSON2_I2C_CR2_ITEVTEN | LOONGSON2_I2C_CR2_ITERREN);
+
+	timeout = wait_for_completion_timeout(&priv->complete, priv->adapter.timeout);
+	if (!timeout)
+		return -ETIMEDOUT;
+
+	return l_msg->result;
+}
+
+static int loongson2_i2c_xfer(struct i2c_adapter *i2c_adap, struct i2c_msg msgs[], int num)
+{
+	struct loongson2_i2c_priv *priv = i2c_get_adapdata(i2c_adap);
+	struct device *dev = regmap_get_device(priv->regmap);
+	unsigned int status;
+	int ret;
+
+	/* Wait I2C bus free */
+	ret = regmap_read_poll_timeout(priv->regmap, LOONGSON2_I2C_SR2, status,
+				       !(status & LOONGSON2_I2C_SR2_BUSY),
+				       LOONGSON2_I2C_FREE_SLEEP_US,
+				       LOONGSON2_I2C_FREE_TIMEOUT_US);
+	if (ret) {
+		dev_dbg(dev, "The I2C bus is busy now.\n");
+		return ret;
+	}
+
+	/* Start generation */
+	regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_START,
+			   LOONGSON2_I2C_CR1_START);
+
+	for (unsigned int i = 0; i < num; i++) {
+		ret = loongson2_i2c_xfer_msg(priv, &msgs[i], i == num - 1);
+		if (ret < 0)
+			return ret;
+	}
+
+	return num;
+}
+
+static u32 loongson2_i2c_func(struct i2c_adapter *adap)
+{
+	return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm loongson2_i2c_algo = {
+	.xfer		= loongson2_i2c_xfer,
+	.functionality	= loongson2_i2c_func,
+};
+
+static int loongson2_i2c_adjust_bus_speed(struct loongson2_i2c_priv *priv)
+{
+	struct device *dev = regmap_get_device(priv->regmap);
+	struct i2c_timings i2c_t;
+	u32 val, freq_MHz, ccr;
+
+	i2c_parse_fw_timings(dev, &i2c_t, true);
+	priv->parent_rate_MHz = clk_get_rate(priv->clk);
+
+	if (i2c_t.bus_freq_hz == I2C_MAX_STANDARD_MODE_FREQ) {
+		 /* Select Standard mode */
+		ccr = 0;
+		val = DIV_ROUND_UP(priv->parent_rate_MHz, i2c_t.bus_freq_hz * 2);
+	} else if (i2c_t.bus_freq_hz == I2C_MAX_FAST_MODE_FREQ) {
+		/* Select Fast mode */
+		ccr = LOONGSON2_I2C_CCR_FS;
+		val = DIV_ROUND_UP(priv->parent_rate_MHz, i2c_t.bus_freq_hz * 3);
+	} else {
+		return dev_err_probe(dev, -EINVAL, "Unsupported speed (%uHz)\n", i2c_t.bus_freq_hz);
+	}
+
+	FIELD_MODIFY(LOONGSON2_I2C_CCR_CCR, &ccr, val);
+	regmap_write(priv->regmap, LOONGSON2_I2C_CCR, ccr);
+
+	freq_MHz = DIV_ROUND_UP(priv->parent_rate_MHz, HZ_PER_MHZ);
+	regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR2, LOONGSON2_I2C_CR2_FREQ,
+			   FIELD_GET(LOONGSON2_I2C_CR2_FREQ, freq_MHz));
+
+	regmap_update_bits(priv->regmap, LOONGSON2_I2C_TRISE, LOONGSON2_I2C_TRISE_SCL,
+			   LOONGSON2_I2C_TRISE_SCL);
+
+	/* Enable I2C */
+	regmap_update_bits(priv->regmap, LOONGSON2_I2C_CR1, LOONGSON2_I2C_CR1_PE,
+			   LOONGSON2_I2C_CR1_PE);
+
+	return 0;
+}
+
+static const struct regmap_config loongson2_i2c_regmap_config = {
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.max_register = LOONGSON2_I2C_TRISE,
+};
+
+static int loongson2_i2c_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct loongson2_i2c_priv *priv;
+	struct i2c_adapter *adap;
+	void __iomem *base;
+	int irq, ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	priv->regmap = devm_regmap_init_mmio(dev, base, &loongson2_i2c_regmap_config);
+	if (IS_ERR(priv->regmap))
+		return dev_err_probe(dev, PTR_ERR(priv->regmap), "Failed to init regmap.\n");
+
+	priv->clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(priv->clk))
+		return dev_err_probe(dev, PTR_ERR(priv->clk), "Failed to enable clock.\n");
+
+	irq = platform_get_irq(pdev, 0);
+	if (irq < 0)
+		return irq;
+
+	adap = &priv->adapter;
+	adap->retries = 5;
+	adap->nr = pdev->id;
+	adap->dev.parent = dev;
+	adap->owner = THIS_MODULE;
+	adap->algo = &loongson2_i2c_algo;
+	adap->timeout = 2 * HZ;
+	device_set_node(&adap->dev, dev_fwnode(dev));
+	i2c_set_adapdata(adap, priv);
+	strscpy(adap->name, pdev->name);
+	init_completion(&priv->complete);
+	platform_set_drvdata(pdev, priv);
+
+	ret = loongson2_i2c_adjust_bus_speed(priv);
+	if (ret)
+		return ret;
+
+	ret = devm_request_irq(dev, irq, loongson2_i2c_isr_event, IRQF_SHARED, pdev->name, priv);
+	if (ret)
+		return ret;
+
+	return devm_i2c_add_adapter(dev, adap);
+}
+
+static const struct of_device_id loongson2_i2c_id_table[] = {
+	{ .compatible = "loongson,ls2k0300-i2c" },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, loongson2_i2c_id_table);
+
+static struct platform_driver loongson2_i2c_driver = {
+	.driver = {
+		.name = "loongson2-i2c-v2",
+		.of_match_table = loongson2_i2c_id_table,
+	},
+	.probe = loongson2_i2c_probe,
+};
+module_platform_driver(loongson2_i2c_driver);
+
+MODULE_DESCRIPTION("Loongson-2K0300 I2C bus driver");
+MODULE_AUTHOR("Loongson Technology Corporation Limited");
+MODULE_LICENSE("GPL");
diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c
index 9fd5ade774a0b..479a1667e88d5 100644
--- a/drivers/i2c/busses/i2c-tegra.c
+++ b/drivers/i2c/busses/i2c-tegra.c
@@ -589,25 +589,22 @@ static int tegra_i2c_mutex_lock(struct tegra_i2c_dev *i2c_dev)
 	return ret;
 }
 
-static int tegra_i2c_mutex_unlock(struct tegra_i2c_dev *i2c_dev)
+static void tegra_i2c_mutex_unlock(struct tegra_i2c_dev *i2c_dev)
 {
 	unsigned int reg = i2c_dev->hw->regs->sw_mutex;
 	u32 val, id;
 
 	if (!i2c_dev->hw->has_mutex)
-		return 0;
+		return;
 
 	val = readl(i2c_dev->base + reg);
 
 	id = FIELD_GET(I2C_SW_MUTEX_GRANT, val);
-	if (id && id != I2C_SW_MUTEX_ID_CCPLEX) {
-		dev_warn(i2c_dev->dev, "unable to unlock mutex, mutex is owned by: %u\n", id);
-		return -EPERM;
-	}
+	if (WARN(id && id != I2C_SW_MUTEX_ID_CCPLEX,
+		 "unable to unlock mutex, mutex is owned by: %u\n", id))
+		return;
 
 	writel(0, i2c_dev->base + reg);
-
-	return 0;
 }
 
 static void tegra_i2c_mask_irq(struct tegra_i2c_dev *i2c_dev, u32 mask)
@@ -1666,8 +1663,10 @@ static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[],
 	}
 
 	ret = tegra_i2c_mutex_lock(i2c_dev);
-	if (ret)
+	if (ret) {
+		pm_runtime_put(i2c_dev->dev);
 		return ret;
+	}
 
 	for (i = 0; i < num; i++) {
 		enum msg_end_type end_type = MSG_END_STOP;
@@ -1698,7 +1697,7 @@ static int tegra_i2c_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[],
 			break;
 	}
 
-	ret = tegra_i2c_mutex_unlock(i2c_dev);
+	tegra_i2c_mutex_unlock(i2c_dev);
 	pm_runtime_put(i2c_dev->dev);
 
 	return ret ?: i;
diff --git a/drivers/i2c/i2c-core-smbus.c b/drivers/i2c/i2c-core-smbus.c
index ad6acb5ebadc3..fa63bee0b345d 100644
--- a/drivers/i2c/i2c-core-smbus.c
+++ b/drivers/i2c/i2c-core-smbus.c
@@ -353,6 +353,7 @@ static s32 i2c_smbus_xfer_emulated(struct i2c_adapter *adapter, u16 addr,
 			  && size != I2C_SMBUS_I2C_BLOCK_DATA);
 
 	msgbuf0[0] = command;
+	msgbuf1[0] = 0;
 	switch (size) {
 	case I2C_SMBUS_QUICK:
 		msg[0].len = 0;
diff --git a/drivers/i3c/master.c b/drivers/i3c/master.c
index 5cd4e5da22333..6b8df8089a357 100644
--- a/drivers/i3c/master.c
+++ b/drivers/i3c/master.c
@@ -174,7 +174,7 @@ static ssize_t bcr_show(struct device *dev,
 
 	i3c_bus_normaluse_lock(bus);
 	desc = dev_to_i3cdesc(dev);
-	ret = sprintf(buf, "0x%02x\n", desc->info.bcr);
+	ret = sysfs_emit(buf, "0x%02x\n", desc->info.bcr);
 	i3c_bus_normaluse_unlock(bus);
 
 	return ret;
@@ -191,7 +191,7 @@ static ssize_t dcr_show(struct device *dev,
 
 	i3c_bus_normaluse_lock(bus);
 	desc = dev_to_i3cdesc(dev);
-	ret = sprintf(buf, "0x%02x\n", desc->info.dcr);
+	ret = sysfs_emit(buf, "0x%02x\n", desc->info.dcr);
 	i3c_bus_normaluse_unlock(bus);
 
 	return ret;
@@ -208,7 +208,7 @@ static ssize_t pid_show(struct device *dev,
 
 	i3c_bus_normaluse_lock(bus);
 	desc = dev_to_i3cdesc(dev);
-	ret = sprintf(buf, "%llx\n", desc->info.pid);
+	ret = sysfs_emit(buf, "%llx\n", desc->info.pid);
 	i3c_bus_normaluse_unlock(bus);
 
 	return ret;
@@ -225,7 +225,7 @@ static ssize_t dynamic_address_show(struct device *dev,
 
 	i3c_bus_normaluse_lock(bus);
 	desc = dev_to_i3cdesc(dev);
-	ret = sprintf(buf, "%02x\n", desc->info.dyn_addr);
+	ret = sysfs_emit(buf, "%02x\n", desc->info.dyn_addr);
 	i3c_bus_normaluse_unlock(bus);
 
 	return ret;
@@ -256,7 +256,7 @@ static ssize_t hdrcap_show(struct device *dev,
 		if (!hdrcap_strings[mode])
 			continue;
 
-		ret = sprintf(buf + offset, offset ? " %s" : "%s",
+		ret = sysfs_emit_at(buf, offset, offset ? " %s" : "%s",
 			      hdrcap_strings[mode]);
 		if (ret < 0)
 			goto out;
@@ -264,7 +264,7 @@ static ssize_t hdrcap_show(struct device *dev,
 		offset += ret;
 	}
 
-	ret = sprintf(buf + offset, "\n");
+	ret = sysfs_emit_at(buf, offset, "\n");
 	if (ret < 0)
 		goto out;
 
@@ -290,10 +290,10 @@ static ssize_t modalias_show(struct device *dev,
 	ext = I3C_PID_EXTRA_INFO(devinfo.pid);
 
 	if (I3C_PID_RND_LOWER_32BITS(devinfo.pid))
-		return sprintf(buf, "i3c:dcr%02Xmanuf%04X", devinfo.dcr,
+		return sysfs_emit(buf, "i3c:dcr%02Xmanuf%04X\n", devinfo.dcr,
 			       manuf);
 
-	return sprintf(buf, "i3c:dcr%02Xmanuf%04Xpart%04Xext%04X",
+	return sysfs_emit(buf, "i3c:dcr%02Xmanuf%04Xpart%04Xext%04X\n",
 		       devinfo.dcr, manuf, part, ext);
 }
 static DEVICE_ATTR_RO(modalias);
@@ -578,9 +578,9 @@ static ssize_t mode_show(struct device *dev,
 	if (i3cbus->mode < 0 ||
 	    i3cbus->mode >= ARRAY_SIZE(i3c_bus_mode_strings) ||
 	    !i3c_bus_mode_strings[i3cbus->mode])
-		ret = sprintf(buf, "unknown\n");
+		ret = sysfs_emit(buf, "unknown\n");
 	else
-		ret = sprintf(buf, "%s\n", i3c_bus_mode_strings[i3cbus->mode]);
+		ret = sysfs_emit(buf, "%s\n", i3c_bus_mode_strings[i3cbus->mode]);
 	i3c_bus_normaluse_unlock(i3cbus);
 
 	return ret;
@@ -595,7 +595,7 @@ static ssize_t current_master_show(struct device *dev,
 	ssize_t ret;
 
 	i3c_bus_normaluse_lock(i3cbus);
-	ret = sprintf(buf, "%d-%llx\n", i3cbus->id,
+	ret = sysfs_emit(buf, "%d-%llx\n", i3cbus->id,
 		      i3cbus->cur_master->info.pid);
 	i3c_bus_normaluse_unlock(i3cbus);
 
@@ -611,7 +611,7 @@ static ssize_t i3c_scl_frequency_show(struct device *dev,
 	ssize_t ret;
 
 	i3c_bus_normaluse_lock(i3cbus);
-	ret = sprintf(buf, "%ld\n", i3cbus->scl_rate.i3c);
+	ret = sysfs_emit(buf, "%ld\n", i3cbus->scl_rate.i3c);
 	i3c_bus_normaluse_unlock(i3cbus);
 
 	return ret;
@@ -626,7 +626,7 @@ static ssize_t i2c_scl_frequency_show(struct device *dev,
 	ssize_t ret;
 
 	i3c_bus_normaluse_lock(i3cbus);
-	ret = sprintf(buf, "%ld\n", i3cbus->scl_rate.i2c);
+	ret = sysfs_emit(buf, "%ld\n", i3cbus->scl_rate.i2c);
 	i3c_bus_normaluse_unlock(i3cbus);
 
 	return ret;
diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c
index 655693a2187e5..a7593d6efac5a 100644
--- a/drivers/i3c/master/dw-i3c-master.c
+++ b/drivers/i3c/master/dw-i3c-master.c
@@ -1435,7 +1435,11 @@ static void dw_i3c_master_irq_handle_ibis(struct dw_i3c_master *master)
 	u32 reg;
 
 	reg = readl(master->regs + QUEUE_STATUS_LEVEL);
-	n_ibis = QUEUE_STATUS_IBI_STATUS_CNT(reg);
+	if (master->has_ibi_data)
+		n_ibis = QUEUE_STATUS_IBI_STATUS_CNT(reg);
+	else
+		n_ibis = QUEUE_STATUS_IBI_BUF_BLR(reg);
+
 	if (!n_ibis)
 		return;
 
@@ -1566,6 +1570,7 @@ int dw_i3c_common_probe(struct dw_i3c_master *master,
 			struct platform_device *pdev)
 {
 	int ret, irq;
+	u32 thld_ctrl;
 	const struct dw_i3c_drvdata *drvdata;
 	unsigned long quirks = 0;
 
@@ -1623,6 +1628,20 @@ int dw_i3c_common_probe(struct dw_i3c_master *master,
 	master->maxdevs = ret >> 16;
 	master->free_pos = GENMASK(master->maxdevs - 1, 0);
 
+	/*
+	 * Detect IBI data capability (IC_HAS_IBI_DATA): write a non-zero value
+	 * to IBI_DATA_THLD and read back. On controllers like Versalnet
+	 * the field is hardwired to 0 and the write is ignored. Restore the
+	 * original register value after detection.
+	 */
+	thld_ctrl = readl(master->regs + QUEUE_THLD_CTRL);
+	ret = thld_ctrl | QUEUE_THLD_CTRL_IBI_DATA(2);
+	writel(ret, master->regs + QUEUE_THLD_CTRL);
+	ret = readl(master->regs + QUEUE_THLD_CTRL);
+	if (ret & QUEUE_THLD_CTRL_IBI_DATA_MASK)
+		master->has_ibi_data = true;
+	writel(thld_ctrl, master->regs + QUEUE_THLD_CTRL);
+
 	if (has_acpi_companion(&pdev->dev)) {
 		quirks = (unsigned long)device_get_match_data(&pdev->dev);
 	} else if (pdev->dev.of_node) {
diff --git a/drivers/i3c/master/dw-i3c-master.h b/drivers/i3c/master/dw-i3c-master.h
index c5cb695c16ab8..306e25a089371 100644
--- a/drivers/i3c/master/dw-i3c-master.h
+++ b/drivers/i3c/master/dw-i3c-master.h
@@ -51,6 +51,7 @@ struct dw_i3c_master {
 	u32 i2c_fm_timing;
 	u32 i2c_fmp_timing;
 	u32 quirks;
+	bool has_ibi_data;
 	/*
 	 * Per-device hardware data, used to manage the device address table
 	 * (DAT)
diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c
index e2d99a3ac07da..57d63d299e1ef 100644
--- a/drivers/i3c/master/svc-i3c-master.c
+++ b/drivers/i3c/master/svc-i3c-master.c
@@ -655,6 +655,15 @@ static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master)
 		break;
 	case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST:
 		svc_i3c_master_emit_stop(master);
+
+		/*
+		 * If a target gets stuck holding SDA low, the controller reports a MR.
+		 * On NPCM845, emitting STOP may spuriously set SLVSTART, retriggering
+		 * the interrupt and re-entering MR handling, leading to an IRQ storm.
+		 * Clear SLVSTART after STOP to break the loop.
+		 */
+		if (svc_has_quirk(master, SVC_I3C_QUIRK_FALSE_SLVSTART))
+			writel(SVC_I3C_MINT_SLVSTART, master->regs + SVC_I3C_MSTATUS);
 		break;
 	default:
 		break;
@@ -672,10 +681,18 @@ static irqreturn_t svc_i3c_master_irq_handler(int irq, void *dev_id)
 	/* Clear the interrupt status */
 	writel(SVC_I3C_MINT_SLVSTART, master->regs + SVC_I3C_MSTATUS);
 
-	/* Ignore the false event */
-	if (svc_has_quirk(master, SVC_I3C_QUIRK_FALSE_SLVSTART) &&
-	    !SVC_I3C_MSTATUS_STATE_SLVREQ(active))
-		return IRQ_HANDLED;
+	if (svc_has_quirk(master, SVC_I3C_QUIRK_FALSE_SLVSTART)) {
+		/*
+		 * Re-read MSTATUS to obtain the latest state and avoid
+		 * missing an IBI that arrives after MSTATUS is latched
+		 * but before SLVSTART is cleared.
+		 */
+		active = readl(master->regs + SVC_I3C_MSTATUS);
+
+		/* Ignore the false event */
+		if (!SVC_I3C_MSTATUS_STATE_SLVREQ(active))
+			return IRQ_HANDLED;
+	}
 
 	/*
 	 * The SDA line remains low until the request is processed.
diff --git a/drivers/iio/accel/adxl313_core.c b/drivers/iio/accel/adxl313_core.c
index bcc11dabdf22a..6dc918c4ae172 100644
--- a/drivers/iio/accel/adxl313_core.c
+++ b/drivers/iio/accel/adxl313_core.c
@@ -1240,7 +1240,9 @@ int adxl313_core_probe(struct device *dev,
 	data->regmap = regmap;
 	data->chip_info = chip_info;
 
-	mutex_init(&data->lock);
+	ret = devm_mutex_init(dev, &data->lock);
+	if (ret)
+		return ret;
 
 	indio_dev->name = chip_info->name;
 	indio_dev->info = &adxl313_info;
@@ -1250,10 +1252,8 @@ int adxl313_core_probe(struct device *dev,
 	indio_dev->available_scan_masks = adxl313_scan_masks;
 
 	ret = adxl313_setup(dev, data, setup);
-	if (ret) {
-		dev_err(dev, "ADXL313 setup failed\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "ADXL313 setup failed\n");
 
 	int_line = adxl313_get_int_type(dev, &irq);
 	if (int_line == ADXL313_INT_NONE) {
diff --git a/drivers/iio/accel/adxl313_i2c.c b/drivers/iio/accel/adxl313_i2c.c
index b67ff0b4dc547..6736b83f23bd6 100644
--- a/drivers/iio/accel/adxl313_i2c.c
+++ b/drivers/iio/accel/adxl313_i2c.c
@@ -65,6 +65,7 @@ MODULE_DEVICE_TABLE(of, adxl313_of_match);
 static int adxl313_i2c_probe(struct i2c_client *client)
 {
 	const struct adxl313_chip_info *chip_data;
+	struct device *dev = &client->dev;
 	struct regmap *regmap;
 
 	/*
@@ -75,13 +76,10 @@ static int adxl313_i2c_probe(struct i2c_client *client)
 
 	regmap = devm_regmap_init_i2c(client,
 				      &adxl31x_i2c_regmap_config[chip_data->type]);
-	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Error initializing i2c regmap: %ld\n",
-			PTR_ERR(regmap));
-		return PTR_ERR(regmap);
-	}
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Error initializing i2c regmap\n");
 
-	return adxl313_core_probe(&client->dev, regmap, chip_data, NULL);
+	return adxl313_core_probe(dev, regmap, chip_data, NULL);
 }
 
 static struct i2c_driver adxl313_i2c_driver = {
diff --git a/drivers/iio/accel/adxl313_spi.c b/drivers/iio/accel/adxl313_spi.c
index dedb0885c2772..d096ea0632ba7 100644
--- a/drivers/iio/accel/adxl313_spi.c
+++ b/drivers/iio/accel/adxl313_spi.c
@@ -70,6 +70,7 @@ static int adxl313_spi_setup(struct device *dev, struct regmap *regmap)
 static int adxl313_spi_probe(struct spi_device *spi)
 {
 	const struct adxl313_chip_info *chip_data;
+	struct device *dev = &spi->dev;
 	struct regmap *regmap;
 	int ret;
 
@@ -83,14 +84,10 @@ static int adxl313_spi_probe(struct spi_device *spi)
 	regmap = devm_regmap_init_spi(spi,
 				      &adxl31x_spi_regmap_config[chip_data->type]);
 
-	if (IS_ERR(regmap)) {
-		dev_err(&spi->dev, "Error initializing spi regmap: %ld\n",
-			PTR_ERR(regmap));
-		return PTR_ERR(regmap);
-	}
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Error initializing spi regmap\n");
 
-	return adxl313_core_probe(&spi->dev, regmap,
-				  chip_data, &adxl313_spi_setup);
+	return adxl313_core_probe(dev, regmap, chip_data, &adxl313_spi_setup);
 }
 
 static const struct spi_device_id adxl313_spi_id[] = {
diff --git a/drivers/iio/accel/adxl355_core.c b/drivers/iio/accel/adxl355_core.c
index 8f90c58f41008..89ac62ff4d04f 100644
--- a/drivers/iio/accel/adxl355_core.c
+++ b/drivers/iio/accel/adxl355_core.c
@@ -336,10 +336,8 @@ static int adxl355_setup(struct adxl355_data *data)
 		return ret;
 
 	do {
-		if (--retries == 0) {
-			dev_err(data->dev, "Shadow registers mismatch\n");
-			return -EIO;
-		}
+		if (--retries == 0)
+			return dev_err_probe(data->dev, -EIO, "Shadow registers mismatch\n");
 
 		/*
 		 * Perform a software reset to make sure the device is in a consistent
@@ -351,7 +349,7 @@ static int adxl355_setup(struct adxl355_data *data)
 			return ret;
 
 		/* Wait at least 5ms after software reset */
-		usleep_range(5000, 10000);
+		fsleep(5 * USEC_PER_MSEC);
 
 		/* Read shadow registers for comparison */
 		ret = regmap_bulk_read(data->regmap,
@@ -775,10 +773,8 @@ static int adxl355_probe_trigger(struct iio_dev *indio_dev, int irq)
 				     irq);
 
 	ret = devm_iio_trigger_register(data->dev, data->dready_trig);
-	if (ret) {
-		dev_err(data->dev, "iio trigger register failed\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(data->dev, ret, "iio trigger register failed\n");
 
 	indio_dev->trig = iio_trigger_get(data->dready_trig);
 
@@ -802,7 +798,9 @@ int adxl355_core_probe(struct device *dev, struct regmap *regmap,
 	data->dev = dev;
 	data->op_mode = ADXL355_STANDBY;
 	data->chip_info = chip_info;
-	mutex_init(&data->lock);
+	ret = devm_mutex_init(dev, &data->lock);
+	if (ret)
+		return ret;
 
 	indio_dev->name = chip_info->name;
 	indio_dev->info = &adxl355_info;
@@ -812,18 +810,14 @@ int adxl355_core_probe(struct device *dev, struct regmap *regmap,
 	indio_dev->available_scan_masks = adxl355_avail_scan_masks;
 
 	ret = adxl355_setup(data);
-	if (ret) {
-		dev_err(dev, "ADXL355 setup failed\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "ADXL355 setup failed\n");
 
 	ret = devm_iio_triggered_buffer_setup(dev, indio_dev,
 					      &iio_pollfunc_store_time,
 					      &adxl355_trigger_handler, NULL);
-	if (ret) {
-		dev_err(dev, "iio triggered buffer setup failed\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "iio triggered buffer setup failed\n");
 
 	irq = fwnode_irq_get_byname(dev_fwnode(dev), "DRDY");
 	if (irq > 0) {
diff --git a/drivers/iio/accel/adxl355_i2c.c b/drivers/iio/accel/adxl355_i2c.c
index 1a512c7b792b6..19490a6e1f35e 100644
--- a/drivers/iio/accel/adxl355_i2c.c
+++ b/drivers/iio/accel/adxl355_i2c.c
@@ -23,6 +23,7 @@ static const struct regmap_config adxl355_i2c_regmap_config = {
 static int adxl355_i2c_probe(struct i2c_client *client)
 {
 	struct regmap *regmap;
+	struct device *dev = &client->dev;
 	const struct adxl355_chip_info *chip_data;
 
 	chip_data = i2c_get_match_data(client);
@@ -30,14 +31,10 @@ static int adxl355_i2c_probe(struct i2c_client *client)
 		return -ENODEV;
 
 	regmap = devm_regmap_init_i2c(client, &adxl355_i2c_regmap_config);
-	if (IS_ERR(regmap)) {
-		dev_err(&client->dev, "Error initializing i2c regmap: %ld\n",
-			PTR_ERR(regmap));
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Error initializing i2c regmap\n");
 
-		return PTR_ERR(regmap);
-	}
-
-	return adxl355_core_probe(&client->dev, regmap, chip_data);
+	return adxl355_core_probe(dev, regmap, chip_data);
 }
 
 static const struct i2c_device_id adxl355_i2c_id[] = {
diff --git a/drivers/iio/accel/adxl355_spi.c b/drivers/iio/accel/adxl355_spi.c
index 869e3e57d6f72..347ed62b6582e 100644
--- a/drivers/iio/accel/adxl355_spi.c
+++ b/drivers/iio/accel/adxl355_spi.c
@@ -26,6 +26,7 @@ static const struct regmap_config adxl355_spi_regmap_config = {
 static int adxl355_spi_probe(struct spi_device *spi)
 {
 	const struct adxl355_chip_info *chip_data;
+	struct device *dev = &spi->dev;
 	struct regmap *regmap;
 
 	chip_data = spi_get_device_match_data(spi);
@@ -33,14 +34,10 @@ static int adxl355_spi_probe(struct spi_device *spi)
 		return -EINVAL;
 
 	regmap = devm_regmap_init_spi(spi, &adxl355_spi_regmap_config);
-	if (IS_ERR(regmap)) {
-		dev_err(&spi->dev, "Error initializing spi regmap: %ld\n",
-			PTR_ERR(regmap));
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap), "Error initializing spi regmap\n");
 
-		return PTR_ERR(regmap);
-	}
-
-	return adxl355_core_probe(&spi->dev, regmap, chip_data);
+	return adxl355_core_probe(dev, regmap, chip_data);
 }
 
 static const struct spi_device_id adxl355_spi_id[] = {
diff --git a/drivers/iio/accel/adxl367.c b/drivers/iio/accel/adxl367.c
index 0c04b2bb7efbf..63a0b182824f6 100644
--- a/drivers/iio/accel/adxl367.c
+++ b/drivers/iio/accel/adxl367.c
@@ -1445,7 +1445,9 @@ int adxl367_probe(struct device *dev, const struct adxl367_ops *ops,
 	st->context = context;
 	st->ops = ops;
 
-	mutex_init(&st->lock);
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
 
 	indio_dev->channels = adxl367_channels;
 	indio_dev->num_channels = ARRAY_SIZE(adxl367_channels);
diff --git a/drivers/iio/accel/adxl372.c b/drivers/iio/accel/adxl372.c
index 545a21e5a3084..e375d068a3f56 100644
--- a/drivers/iio/accel/adxl372.c
+++ b/drivers/iio/accel/adxl372.c
@@ -1299,7 +1299,9 @@ int adxl372_probe(struct device *dev, struct regmap *regmap,
 	st->irq = irq;
 	st->chip_info = chip_info;
 
-	mutex_init(&st->threshold_m);
+	ret = devm_mutex_init(dev, &st->threshold_m);
+	if (ret < 0)
+		return ret;
 
 	indio_dev->channels = adxl372_channels;
 	indio_dev->num_channels = ARRAY_SIZE(adxl372_channels);
@@ -1314,10 +1316,8 @@ int adxl372_probe(struct device *dev, struct regmap *regmap,
 	}
 
 	ret = adxl372_setup(st);
-	if (ret < 0) {
-		dev_err(dev, "ADXL372 setup failed\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "ADXL372 setup failed\n");
 
 	if (chip_info->fifo_supported) {
 		ret = adxl372_buffer_setup(indio_dev);
diff --git a/drivers/iio/accel/adxl380.c b/drivers/iio/accel/adxl380.c
index e7bb32fbc4752..7dca5523091fc 100644
--- a/drivers/iio/accel/adxl380.c
+++ b/drivers/iio/accel/adxl380.c
@@ -1967,7 +1967,9 @@ int adxl380_probe(struct device *dev, struct regmap *regmap,
 	st->chip_info = chip_info;
 	st->odr = ADXL380_ODR_DSM;
 
-	mutex_init(&st->lock);
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
 
 	indio_dev->channels = adxl380_channels;
 	indio_dev->num_channels = ARRAY_SIZE(adxl380_channels);
diff --git a/drivers/iio/adc/Kconfig b/drivers/iio/adc/Kconfig
index a9dedbb8eb461..0dd7800eacdba 100644
--- a/drivers/iio/adc/Kconfig
+++ b/drivers/iio/adc/Kconfig
@@ -78,6 +78,7 @@ config AD4030
 config AD4062
 	tristate "Analog Devices AD4062 Driver"
 	depends on I3C
+	depends on GPIOLIB
 	select REGMAP_I3C
 	select IIO_BUFFER
 	select IIO_TRIGGERED_BUFFER
@@ -144,6 +145,21 @@ config AD4170_4
 	  To compile this driver as a module, choose M here: the module will be
 	  called ad4170-4.
 
+config AD4691
+	tristate "Analog Devices AD4691 Family ADC Driver"
+	depends on SPI
+	select IIO_BUFFER
+	select IIO_BUFFER_DMAENGINE
+	select IIO_TRIGGERED_BUFFER
+	select REGMAP
+	select SPI_OFFLOAD
+	help
+	  Say yes here to build support for Analog Devices AD4691 Family MuxSAR
+	  SPI analog to digital converters (ADC).
+
+	  To compile this driver as a module, choose M here: the module will be
+	  called ad4691.
+
 config AD4695
 	tristate "Analog Device AD4695 ADC Driver"
 	depends on SPI
@@ -414,6 +430,7 @@ config AD7766
 config AD7768_1
 	tristate "Analog Devices AD7768-1 ADC driver"
 	depends on SPI
+	select GPIOLIB
 	select REGULATOR
 	select REGMAP_SPI
 	select RATIONAL
@@ -1354,7 +1371,7 @@ config QCOM_SPMI_VADC
 	  be called qcom-spmi-vadc.
 
 config QCOM_SPMI_ADC5
-	tristate "Qualcomm Technologies Inc. SPMI PMIC5 ADC"
+	tristate "Qualcomm SPMI PMIC5 ADC"
 	depends on SPMI
 	select REGMAP_SPMI
 	select QCOM_VADC_COMMON
@@ -1374,7 +1391,7 @@ config QCOM_SPMI_ADC5
 	  be called qcom-spmi-adc5.
 
 config QCOM_SPMI_ADC5_GEN3
-	tristate "Qualcomm Technologies Inc. SPMI PMIC5 GEN3 ADC"
+	tristate "Qualcomm SPMI PMIC5 GEN3 ADC"
 	depends on SPMI && THERMAL
 	select REGMAP_SPMI
 	select QCOM_VADC_COMMON
diff --git a/drivers/iio/adc/Makefile b/drivers/iio/adc/Makefile
index 097357d146baf..707dd708912fe 100644
--- a/drivers/iio/adc/Makefile
+++ b/drivers/iio/adc/Makefile
@@ -16,6 +16,7 @@ obj-$(CONFIG_AD4080) += ad4080.o
 obj-$(CONFIG_AD4130) += ad4130.o
 obj-$(CONFIG_AD4134) += ad4134.o
 obj-$(CONFIG_AD4170_4) += ad4170-4.o
+obj-$(CONFIG_AD4691) += ad4691.o
 obj-$(CONFIG_AD4695) += ad4695.o
 obj-$(CONFIG_AD4851) += ad4851.o
 obj-$(CONFIG_AD7091R) += ad7091r-base.o
diff --git a/drivers/iio/adc/ad4062.c b/drivers/iio/adc/ad4062.c
index 222499a45b9b4..8e5984055b159 100644
--- a/drivers/iio/adc/ad4062.c
+++ b/drivers/iio/adc/ad4062.c
@@ -436,10 +436,9 @@ static int ad4062_check_ids(struct ad4062_state *st)
 		return ret;
 
 	val = be16_to_cpu(st->buf.be16);
-	if (val != AD4062_I3C_VENDOR) {
-		dev_err(dev, "Vendor ID x%x does not match expected value\n", val);
-		return -ENODEV;
-	}
+	if (val != AD4062_I3C_VENDOR)
+		return dev_err_probe(dev, -ENODEV,
+				     "Vendor ID x%x does not match expected value\n", val);
 
 	return 0;
 }
diff --git a/drivers/iio/adc/ad4080.c b/drivers/iio/adc/ad4080.c
index 204ad198342ba..265d85ac171a9 100644
--- a/drivers/iio/adc/ad4080.c
+++ b/drivers/iio/adc/ad4080.c
@@ -16,6 +16,7 @@
 #include <linux/mod_devicetable.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
 #include <linux/spi/spi.h>
@@ -134,6 +135,9 @@
 #define AD4086_CHIP_ID						0x0056
 #define AD4087_CHIP_ID						0x0057
 #define AD4088_CHIP_ID						0x0058
+#define AD4880_CHIP_ID						0x0750
+
+#define AD4080_MAX_CHANNELS					2
 
 #define AD4080_LVDS_CNV_CLK_CNT_MAX				7
 
@@ -179,8 +183,9 @@ struct ad4080_chip_info {
 };
 
 struct ad4080_state {
-	struct regmap			*regmap;
-	struct iio_backend		*back;
+	struct spi_device		*spi[AD4080_MAX_CHANNELS];
+	struct regmap			*regmap[AD4080_MAX_CHANNELS];
+	struct iio_backend		*back[AD4080_MAX_CHANNELS];
 	const struct ad4080_chip_info	*info;
 	/*
 	 * Synchronize access to members the of driver state, and ensure
@@ -189,7 +194,7 @@ struct ad4080_state {
 	struct mutex			lock;
 	unsigned int			num_lanes;
 	unsigned long			clk_rate;
-	enum ad4080_filter_type		filter_type;
+	enum ad4080_filter_type		filter_type[AD4080_MAX_CHANNELS];
 	bool				lvds_cnv_en;
 };
 
@@ -206,9 +211,9 @@ static int ad4080_reg_access(struct iio_dev *indio_dev, unsigned int reg,
 	struct ad4080_state *st = iio_priv(indio_dev);
 
 	if (readval)
-		return regmap_read(st->regmap, reg, readval);
+		return regmap_read(st->regmap[0], reg, readval);
 
-	return regmap_write(st->regmap, reg, writeval);
+	return regmap_write(st->regmap[0], reg, writeval);
 }
 
 static int ad4080_get_scale(struct ad4080_state *st, int *val, int *val2)
@@ -229,8 +234,9 @@ static unsigned int ad4080_get_dec_rate(struct iio_dev *dev,
 	struct ad4080_state *st = iio_priv(dev);
 	int ret;
 	unsigned int data;
+	unsigned int ch = chan->channel;
 
-	ret = regmap_read(st->regmap, AD4080_REG_FILTER_CONFIG, &data);
+	ret = regmap_read(st->regmap[ch], AD4080_REG_FILTER_CONFIG, &data);
 	if (ret)
 		return ret;
 
@@ -242,13 +248,14 @@ static int ad4080_set_dec_rate(struct iio_dev *dev,
 			       unsigned int mode)
 {
 	struct ad4080_state *st = iio_priv(dev);
+	unsigned int ch = chan->channel;
 
 	guard(mutex)(&st->lock);
 
-	if ((st->filter_type >= SINC_5 && mode >= 512) || mode < 2)
+	if ((st->filter_type[ch] >= SINC_5 && mode >= 512) || mode < 2)
 		return -EINVAL;
 
-	return regmap_update_bits(st->regmap, AD4080_REG_FILTER_CONFIG,
+	return regmap_update_bits(st->regmap[ch], AD4080_REG_FILTER_CONFIG,
 				  AD4080_FILTER_CONFIG_SINC_DEC_RATE_MSK,
 				  FIELD_PREP(AD4080_FILTER_CONFIG_SINC_DEC_RATE_MSK,
 					     (ilog2(mode) - 1)));
@@ -268,15 +275,15 @@ static int ad4080_read_raw(struct iio_dev *indio_dev,
 		dec_rate = ad4080_get_dec_rate(indio_dev, chan);
 		if (dec_rate < 0)
 			return dec_rate;
-		if (st->filter_type == SINC_5_COMP)
+		if (st->filter_type[chan->channel] == SINC_5_COMP)
 			dec_rate *= 2;
-		if (st->filter_type)
+		if (st->filter_type[chan->channel])
 			*val = DIV_ROUND_CLOSEST(st->clk_rate, dec_rate);
 		else
 			*val = st->clk_rate;
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
-		if (st->filter_type == FILTER_NONE) {
+		if (st->filter_type[chan->channel] == FILTER_NONE) {
 			*val = 1;
 		} else {
 			*val = ad4080_get_dec_rate(indio_dev, chan);
@@ -297,7 +304,7 @@ static int ad4080_write_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
-		if (st->filter_type == FILTER_NONE && val > 1)
+		if (st->filter_type[chan->channel] == FILTER_NONE && val > 1)
 			return -EINVAL;
 
 		return ad4080_set_dec_rate(indio_dev, chan, val);
@@ -306,23 +313,23 @@ static int ad4080_write_raw(struct iio_dev *indio_dev,
 	}
 }
 
-static int ad4080_lvds_sync_write(struct ad4080_state *st)
+static int ad4080_lvds_sync_write(struct ad4080_state *st, unsigned int ch)
 {
-	struct device *dev = regmap_get_device(st->regmap);
+	struct device *dev = regmap_get_device(st->regmap[ch]);
 	int ret;
 
-	ret = regmap_set_bits(st->regmap, AD4080_REG_ADC_DATA_INTF_CONFIG_A,
+	ret = regmap_set_bits(st->regmap[ch], AD4080_REG_ADC_DATA_INTF_CONFIG_A,
 			      AD4080_ADC_DATA_INTF_CONFIG_A_INTF_CHK_EN);
 	if (ret)
 		return ret;
 
-	ret = iio_backend_interface_data_align(st->back, 10000);
+	ret = iio_backend_interface_data_align(st->back[ch], 10000);
 	if (ret)
 		return dev_err_probe(dev, ret,
 				     "Data alignment process failed\n");
 
 	dev_dbg(dev, "Success: Pattern correct and Locked!\n");
-	return regmap_clear_bits(st->regmap, AD4080_REG_ADC_DATA_INTF_CONFIG_A,
+	return regmap_clear_bits(st->regmap[ch], AD4080_REG_ADC_DATA_INTF_CONFIG_A,
 				 AD4080_ADC_DATA_INTF_CONFIG_A_INTF_CHK_EN);
 }
 
@@ -331,9 +338,10 @@ static int ad4080_get_filter_type(struct iio_dev *dev,
 {
 	struct ad4080_state *st = iio_priv(dev);
 	unsigned int data;
+	unsigned int ch = chan->channel;
 	int ret;
 
-	ret = regmap_read(st->regmap, AD4080_REG_FILTER_CONFIG, &data);
+	ret = regmap_read(st->regmap[ch], AD4080_REG_FILTER_CONFIG, &data);
 	if (ret)
 		return ret;
 
@@ -345,6 +353,7 @@ static int ad4080_set_filter_type(struct iio_dev *dev,
 				  unsigned int mode)
 {
 	struct ad4080_state *st = iio_priv(dev);
+	unsigned int ch = chan->channel;
 	int dec_rate;
 	int ret;
 
@@ -357,18 +366,18 @@ static int ad4080_set_filter_type(struct iio_dev *dev,
 	if (mode >= SINC_5 && dec_rate >= 512)
 		return -EINVAL;
 
-	ret = iio_backend_filter_type_set(st->back, mode);
+	ret = iio_backend_filter_type_set(st->back[ch], mode);
 	if (ret)
 		return ret;
 
-	ret = regmap_update_bits(st->regmap, AD4080_REG_FILTER_CONFIG,
+	ret = regmap_update_bits(st->regmap[ch], AD4080_REG_FILTER_CONFIG,
 				 AD4080_FILTER_CONFIG_FILTER_SEL_MSK,
 				 FIELD_PREP(AD4080_FILTER_CONFIG_FILTER_SEL_MSK,
 					    mode));
 	if (ret)
 		return ret;
 
-	st->filter_type = mode;
+	st->filter_type[ch] = mode;
 
 	return 0;
 }
@@ -382,14 +391,14 @@ static int ad4080_read_avail(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
-		switch (st->filter_type) {
+		switch (st->filter_type[chan->channel]) {
 		case FILTER_NONE:
 			*vals = ad4080_dec_rate_none;
 			*length = ARRAY_SIZE(ad4080_dec_rate_none);
 			break;
 		default:
 			*vals = ad4080_dec_rate_avail;
-			*length = st->filter_type >= SINC_5 ?
+			*length = st->filter_type[chan->channel] >= SINC_5 ?
 				  (ARRAY_SIZE(ad4080_dec_rate_avail) - 2) :
 				  ARRAY_SIZE(ad4080_dec_rate_avail);
 			break;
@@ -401,6 +410,28 @@ static int ad4080_read_avail(struct iio_dev *indio_dev,
 	}
 }
 
+static int ad4880_update_scan_mode(struct iio_dev *indio_dev,
+				   const unsigned long *scan_mask)
+{
+	struct ad4080_state *st = iio_priv(indio_dev);
+	int ret;
+
+	for (unsigned int ch = 0; ch < st->info->num_channels; ch++) {
+		/*
+		 * Each backend has a single channel (channel 0 from the
+		 * backend's perspective), so always use channel index 0.
+		 */
+		if (test_bit(ch, scan_mask))
+			ret = iio_backend_chan_enable(st->back[ch], 0);
+		else
+			ret = iio_backend_chan_disable(st->back[ch], 0);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static const struct iio_info ad4080_iio_info = {
 	.debugfs_reg_access = ad4080_reg_access,
 	.read_raw = ad4080_read_raw,
@@ -408,6 +439,19 @@ static const struct iio_info ad4080_iio_info = {
 	.read_avail = ad4080_read_avail,
 };
 
+/*
+ * AD4880 needs update_scan_mode to enable/disable individual backend channels.
+ * Single-channel devices don't need this as their backends may not implement
+ * chan_enable/chan_disable operations.
+ */
+static const struct iio_info ad4880_iio_info = {
+	.debugfs_reg_access = ad4080_reg_access,
+	.read_raw = ad4080_read_raw,
+	.write_raw = ad4080_write_raw,
+	.read_avail = ad4080_read_avail,
+	.update_scan_mode = ad4880_update_scan_mode,
+};
+
 static const struct iio_enum ad4080_filter_type_enum = {
 	.items = ad4080_filter_type_iio_enum,
 	.num_items = ARRAY_SIZE(ad4080_filter_type_iio_enum),
@@ -422,17 +466,28 @@ static struct iio_chan_spec_ext_info ad4080_ext_info[] = {
 	{ }
 };
 
-#define AD4080_CHANNEL_DEFINE(bits, storage) {				\
+/*
+ * AD4880 needs per-channel filter configuration since each channel has
+ * its own independent ADC with separate SPI interface.
+ */
+static struct iio_chan_spec_ext_info ad4880_ext_info[] = {
+	IIO_ENUM("filter_type", IIO_SEPARATE, &ad4080_filter_type_enum),
+	IIO_ENUM_AVAILABLE("filter_type", IIO_SEPARATE,
+			   &ad4080_filter_type_enum),
+	{ }
+};
+
+#define AD4080_CHANNEL_DEFINE(bits, storage, idx) {			\
 	.type = IIO_VOLTAGE,						\
 	.indexed = 1,							\
-	.channel = 0,							\
+	.channel = (idx),						\
 	.info_mask_separate = BIT(IIO_CHAN_INFO_SCALE),			\
 	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ) |	\
 			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),		\
 	.info_mask_shared_by_all_available =				\
 			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),		\
 	.ext_info = ad4080_ext_info,					\
-	.scan_index = 0,						\
+	.scan_index = (idx),						\
 	.scan_type = {							\
 		.sign = 's',						\
 		.realbits = (bits),					\
@@ -440,23 +495,51 @@ static struct iio_chan_spec_ext_info ad4080_ext_info[] = {
 	},								\
 }
 
-static const struct iio_chan_spec ad4080_channel = AD4080_CHANNEL_DEFINE(20, 32);
+/*
+ * AD4880 has per-channel attributes (filter_type, oversampling_ratio,
+ * sampling_frequency) since each channel has its own independent ADC
+ * with separate SPI configuration interface.
+ */
+#define AD4880_CHANNEL_DEFINE(bits, storage, idx) {		\
+	.type = IIO_VOLTAGE,						\
+	.indexed = 1,							\
+	.channel = (idx),						\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_SCALE) |		\
+			BIT(IIO_CHAN_INFO_SAMP_FREQ) |			\
+			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),		\
+	.info_mask_separate_available =					\
+			BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO),		\
+	.ext_info = ad4880_ext_info,				\
+	.scan_index = (idx),						\
+	.scan_type = {							\
+		.sign = 's',						\
+		.realbits = (bits),					\
+		.storagebits = (storage),				\
+	},								\
+}
 
-static const struct iio_chan_spec ad4081_channel = AD4080_CHANNEL_DEFINE(20, 32);
+static const struct iio_chan_spec ad4080_channel = AD4080_CHANNEL_DEFINE(20, 32, 0);
 
-static const struct iio_chan_spec ad4082_channel = AD4080_CHANNEL_DEFINE(20, 32);
+static const struct iio_chan_spec ad4081_channel = AD4080_CHANNEL_DEFINE(20, 32, 0);
 
-static const struct iio_chan_spec ad4083_channel = AD4080_CHANNEL_DEFINE(16, 16);
+static const struct iio_chan_spec ad4082_channel = AD4080_CHANNEL_DEFINE(20, 32, 0);
 
-static const struct iio_chan_spec ad4084_channel = AD4080_CHANNEL_DEFINE(16, 16);
+static const struct iio_chan_spec ad4083_channel = AD4080_CHANNEL_DEFINE(16, 16, 0);
 
-static const struct iio_chan_spec ad4085_channel = AD4080_CHANNEL_DEFINE(16, 16);
+static const struct iio_chan_spec ad4084_channel = AD4080_CHANNEL_DEFINE(16, 16, 0);
 
-static const struct iio_chan_spec ad4086_channel = AD4080_CHANNEL_DEFINE(14, 16);
+static const struct iio_chan_spec ad4085_channel = AD4080_CHANNEL_DEFINE(16, 16, 0);
 
-static const struct iio_chan_spec ad4087_channel = AD4080_CHANNEL_DEFINE(14, 16);
+static const struct iio_chan_spec ad4086_channel = AD4080_CHANNEL_DEFINE(14, 16, 0);
 
-static const struct iio_chan_spec ad4088_channel = AD4080_CHANNEL_DEFINE(14, 16);
+static const struct iio_chan_spec ad4087_channel = AD4080_CHANNEL_DEFINE(14, 16, 0);
+
+static const struct iio_chan_spec ad4088_channel = AD4080_CHANNEL_DEFINE(14, 16, 0);
+
+static const struct iio_chan_spec ad4880_channels[] = {
+	AD4880_CHANNEL_DEFINE(20, 32, 0),
+	AD4880_CHANNEL_DEFINE(20, 32, 1),
+};
 
 static const struct ad4080_chip_info ad4080_chip_info = {
 	.name = "ad4080",
@@ -548,25 +631,34 @@ static const struct ad4080_chip_info ad4088_chip_info = {
 	.lvds_cnv_clk_cnt_max = 8,
 };
 
-static int ad4080_setup(struct iio_dev *indio_dev)
+static const struct ad4080_chip_info ad4880_chip_info = {
+	.name = "ad4880",
+	.product_id = AD4880_CHIP_ID,
+	.scale_table = ad4080_scale_table,
+	.num_scales = ARRAY_SIZE(ad4080_scale_table),
+	.num_channels = 2,
+	.channels = ad4880_channels,
+	.lvds_cnv_clk_cnt_max = AD4080_LVDS_CNV_CLK_CNT_MAX,
+};
+
+static int ad4080_setup_channel(struct ad4080_state *st, unsigned int ch)
 {
-	struct ad4080_state *st = iio_priv(indio_dev);
-	struct device *dev = regmap_get_device(st->regmap);
+	struct device *dev = regmap_get_device(st->regmap[ch]);
 	__le16 id_le;
 	u16 id;
 	int ret;
 
-	ret = regmap_write(st->regmap, AD4080_REG_INTERFACE_CONFIG_A,
+	ret = regmap_write(st->regmap[ch], AD4080_REG_INTERFACE_CONFIG_A,
 			   AD4080_INTERFACE_CONFIG_A_SW_RESET);
 	if (ret)
 		return ret;
 
-	ret = regmap_write(st->regmap, AD4080_REG_INTERFACE_CONFIG_A,
+	ret = regmap_write(st->regmap[ch], AD4080_REG_INTERFACE_CONFIG_A,
 			   AD4080_INTERFACE_CONFIG_A_SDO_ENABLE);
 	if (ret)
 		return ret;
 
-	ret = regmap_bulk_read(st->regmap, AD4080_REG_PRODUCT_ID_L, &id_le,
+	ret = regmap_bulk_read(st->regmap[ch], AD4080_REG_PRODUCT_ID_L, &id_le,
 			       sizeof(id_le));
 	if (ret)
 		return ret;
@@ -575,18 +667,18 @@ static int ad4080_setup(struct iio_dev *indio_dev)
 	if (id != st->info->product_id)
 		dev_info(dev, "Unrecognized CHIP_ID 0x%X\n", id);
 
-	ret = regmap_set_bits(st->regmap, AD4080_REG_GPIO_CONFIG_A,
+	ret = regmap_set_bits(st->regmap[ch], AD4080_REG_GPIO_CONFIG_A,
 			      AD4080_GPIO_CONFIG_A_GPO_1_EN);
 	if (ret)
 		return ret;
 
-	ret = regmap_write(st->regmap, AD4080_REG_GPIO_CONFIG_B,
+	ret = regmap_write(st->regmap[ch], AD4080_REG_GPIO_CONFIG_B,
 			   FIELD_PREP(AD4080_GPIO_CONFIG_B_GPIO_1_SEL_MSK,
 				      AD4080_GPIO_CONFIG_B_GPIO_FILTER_RES_RDY));
 	if (ret)
 		return ret;
 
-	ret = iio_backend_num_lanes_set(st->back, st->num_lanes);
+	ret = iio_backend_num_lanes_set(st->back[ch], st->num_lanes);
 	if (ret)
 		return ret;
 
@@ -594,7 +686,7 @@ static int ad4080_setup(struct iio_dev *indio_dev)
 		return 0;
 
 	/* Set maximum LVDS Data Transfer Latency */
-	ret = regmap_update_bits(st->regmap,
+	ret = regmap_update_bits(st->regmap[ch],
 				 AD4080_REG_ADC_DATA_INTF_CONFIG_B,
 				 AD4080_ADC_DATA_INTF_CONFIG_B_LVDS_CNV_CLK_CNT_MSK,
 				 FIELD_PREP(AD4080_ADC_DATA_INTF_CONFIG_B_LVDS_CNV_CLK_CNT_MSK,
@@ -603,24 +695,38 @@ static int ad4080_setup(struct iio_dev *indio_dev)
 		return ret;
 
 	if (st->num_lanes > 1) {
-		ret = regmap_set_bits(st->regmap, AD4080_REG_ADC_DATA_INTF_CONFIG_A,
+		ret = regmap_set_bits(st->regmap[ch], AD4080_REG_ADC_DATA_INTF_CONFIG_A,
 				      AD4080_ADC_DATA_INTF_CONFIG_A_SPI_LVDS_LANES);
 		if (ret)
 			return ret;
 	}
 
-	ret = regmap_set_bits(st->regmap,
+	ret = regmap_set_bits(st->regmap[ch],
 			      AD4080_REG_ADC_DATA_INTF_CONFIG_B,
 			      AD4080_ADC_DATA_INTF_CONFIG_B_LVDS_CNV_EN);
 	if (ret)
 		return ret;
 
-	return ad4080_lvds_sync_write(st);
+	return ad4080_lvds_sync_write(st, ch);
 }
 
-static int ad4080_properties_parse(struct ad4080_state *st)
+static int ad4080_setup(struct iio_dev *indio_dev)
+{
+	struct ad4080_state *st = iio_priv(indio_dev);
+	int ret;
+
+	for (unsigned int ch = 0; ch < st->info->num_channels; ch++) {
+		ret = ad4080_setup_channel(st, ch);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int ad4080_properties_parse(struct ad4080_state *st,
+				   struct device *dev)
 {
-	struct device *dev = regmap_get_device(st->regmap);
 
 	st->lvds_cnv_en = device_property_read_bool(dev, "adi,lvds-cnv-enable");
 
@@ -655,14 +761,28 @@ static int ad4080_probe(struct spi_device *spi)
 		return dev_err_probe(dev, ret,
 				     "failed to get and enable supplies\n");
 
-	st->regmap = devm_regmap_init_spi(spi, &ad4080_regmap_config);
-	if (IS_ERR(st->regmap))
-		return PTR_ERR(st->regmap);
+	/* Setup primary SPI device (channel 0) */
+	st->spi[0] = spi;
+	st->regmap[0] = devm_regmap_init_spi(spi, &ad4080_regmap_config);
+	if (IS_ERR(st->regmap[0]))
+		return PTR_ERR(st->regmap[0]);
 
 	st->info = spi_get_device_match_data(spi);
 	if (!st->info)
 		return -ENODEV;
 
+	/* Setup ancillary SPI devices for additional channels */
+	for (unsigned int ch = 1; ch < st->info->num_channels; ch++) {
+		st->spi[ch] = devm_spi_new_ancillary_device(spi, spi_get_chipselect(spi, ch));
+		if (IS_ERR(st->spi[ch]))
+			return dev_err_probe(dev, PTR_ERR(st->spi[ch]),
+					     "failed to register ancillary device\n");
+
+		st->regmap[ch] = devm_regmap_init_spi(st->spi[ch], &ad4080_regmap_config);
+		if (IS_ERR(st->regmap[ch]))
+			return PTR_ERR(st->regmap[ch]);
+	}
+
 	ret = devm_mutex_init(dev, &st->lock);
 	if (ret)
 		return ret;
@@ -670,9 +790,10 @@ static int ad4080_probe(struct spi_device *spi)
 	indio_dev->name = st->info->name;
 	indio_dev->channels = st->info->channels;
 	indio_dev->num_channels = st->info->num_channels;
-	indio_dev->info = &ad4080_iio_info;
+	indio_dev->info = st->info->num_channels > 1 ?
+			  &ad4880_iio_info : &ad4080_iio_info;
 
-	ret = ad4080_properties_parse(st);
+	ret = ad4080_properties_parse(st, dev);
 	if (ret)
 		return ret;
 
@@ -682,15 +803,25 @@ static int ad4080_probe(struct spi_device *spi)
 
 	st->clk_rate = clk_get_rate(clk);
 
-	st->back = devm_iio_backend_get(dev, NULL);
-	if (IS_ERR(st->back))
-		return PTR_ERR(st->back);
+	/* Get backends for all channels */
+	for (unsigned int ch = 0; ch < st->info->num_channels; ch++) {
+		st->back[ch] = devm_iio_backend_get_by_index(dev, ch);
+		if (IS_ERR(st->back[ch]))
+			return PTR_ERR(st->back[ch]);
 
-	ret = devm_iio_backend_request_buffer(dev, st->back, indio_dev);
-	if (ret)
-		return ret;
+		ret = devm_iio_backend_enable(dev, st->back[ch]);
+		if (ret)
+			return ret;
+	}
 
-	ret = devm_iio_backend_enable(dev, st->back);
+	/*
+	 * Request buffer from the first backend only. For multi-channel
+	 * devices (e.g., AD4880), the FPGA uses two axi_ad408x IP instances
+	 * (one per ADC channel) whose outputs are combined by a packer block
+	 * that interleaves all channel data into a single DMA stream routed
+	 * through the first backend's clock domain.
+	 */
+	ret = devm_iio_backend_request_buffer(dev, st->back[0], indio_dev);
 	if (ret)
 		return ret;
 
@@ -711,6 +842,7 @@ static const struct spi_device_id ad4080_id[] = {
 	{ "ad4086", (kernel_ulong_t)&ad4086_chip_info },
 	{ "ad4087", (kernel_ulong_t)&ad4087_chip_info },
 	{ "ad4088", (kernel_ulong_t)&ad4088_chip_info },
+	{ "ad4880", (kernel_ulong_t)&ad4880_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(spi, ad4080_id);
@@ -725,6 +857,7 @@ static const struct of_device_id ad4080_of_match[] = {
 	{ .compatible = "adi,ad4086", &ad4086_chip_info },
 	{ .compatible = "adi,ad4087", &ad4087_chip_info },
 	{ .compatible = "adi,ad4088", &ad4088_chip_info },
+	{ .compatible = "adi,ad4880", &ad4880_chip_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ad4080_of_match);
diff --git a/drivers/iio/adc/ad4130.c b/drivers/iio/adc/ad4130.c
index 5567ae5dee88e..7f39f3484062a 100644
--- a/drivers/iio/adc/ad4130.c
+++ b/drivers/iio/adc/ad4130.c
@@ -9,6 +9,7 @@
 #include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
+#include <linux/completion.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/err.h>
@@ -21,6 +22,7 @@
 #include <linux/regmap.h>
 #include <linux/regulator/consumer.h>
 #include <linux/spi/spi.h>
+#include <linux/types.h>
 #include <linux/units.h>
 
 #include <asm/div64.h>
@@ -30,6 +32,9 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/kfifo_buf.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/trigger_consumer.h>
+#include <linux/iio/triggered_buffer.h>
 
 #define AD4130_NAME				"ad4130"
 
@@ -40,6 +45,7 @@
 #define AD4130_ADC_CONTROL_REG			0x01
 #define AD4130_ADC_CONTROL_BIPOLAR_MASK		BIT(14)
 #define AD4130_ADC_CONTROL_INT_REF_VAL_MASK	BIT(13)
+#define AD4130_ADC_CONTROL_CONT_READ_MASK	BIT(11)
 #define AD4130_INT_REF_2_5V			2500000
 #define AD4130_INT_REF_1_25V			1250000
 #define AD4130_ADC_CONTROL_CSB_EN_MASK		BIT(9)
@@ -54,7 +60,9 @@
 #define AD4130_IO_CONTROL_REG			0x03
 #define AD4130_IO_CONTROL_INT_PIN_SEL_MASK	GENMASK(9, 8)
 #define AD4130_IO_CONTROL_GPIO_DATA_MASK	GENMASK(7, 4)
+#define AD4130_4_IO_CONTROL_GPIO_DATA_MASK	GENMASK(7, 6)
 #define AD4130_IO_CONTROL_GPIO_CTRL_MASK	GENMASK(3, 0)
+#define AD4130_4_IO_CONTROL_GPIO_CTRL_MASK	GENMASK(3, 2)
 
 #define AD4130_VBIAS_REG			0x04
 
@@ -125,6 +133,28 @@
 
 #define AD4130_INVALID_SLOT			-1
 
+static const unsigned int ad4129_reg_size[] = {
+	[AD4130_STATUS_REG] = 1,
+	[AD4130_ADC_CONTROL_REG] = 2,
+	[AD4130_DATA_REG] = 2,
+	[AD4130_IO_CONTROL_REG] = 2,
+	[AD4130_VBIAS_REG] = 2,
+	[AD4130_ID_REG] = 1,
+	[AD4130_ERROR_REG] = 2,
+	[AD4130_ERROR_EN_REG] = 2,
+	[AD4130_MCLK_COUNT_REG] = 1,
+	[AD4130_CHANNEL_X_REG(0) ... AD4130_CHANNEL_X_REG(AD4130_MAX_CHANNELS - 1)] = 3,
+	[AD4130_CONFIG_X_REG(0) ... AD4130_CONFIG_X_REG(AD4130_MAX_SETUPS - 1)] = 2,
+	[AD4130_FILTER_X_REG(0) ... AD4130_FILTER_X_REG(AD4130_MAX_SETUPS - 1)] = 3,
+	[AD4130_OFFSET_X_REG(0) ... AD4130_OFFSET_X_REG(AD4130_MAX_SETUPS - 1)] = 2,
+	[AD4130_GAIN_X_REG(0) ... AD4130_GAIN_X_REG(AD4130_MAX_SETUPS - 1)] = 2,
+	[AD4130_MISC_REG] = 2,
+	[AD4130_FIFO_CONTROL_REG] = 3,
+	[AD4130_FIFO_STATUS_REG] = 1,
+	[AD4130_FIFO_THRESHOLD_REG] = 3,
+	[AD4130_FIFO_DATA_REG] = 2,
+};
+
 static const unsigned int ad4130_reg_size[] = {
 	[AD4130_STATUS_REG] = 1,
 	[AD4130_ADC_CONTROL_REG] = 2,
@@ -147,6 +177,24 @@ static const unsigned int ad4130_reg_size[] = {
 	[AD4130_FIFO_DATA_REG] = 3,
 };
 
+static const unsigned int ad4131_reg_size[] = {
+	[AD4130_STATUS_REG] = 1,
+	[AD4130_ADC_CONTROL_REG] = 2,
+	[AD4130_DATA_REG] = 2,
+	[AD4130_IO_CONTROL_REG] = 2,
+	[AD4130_VBIAS_REG] = 2,
+	[AD4130_ID_REG] = 1,
+	[AD4130_ERROR_REG] = 2,
+	[AD4130_ERROR_EN_REG] = 2,
+	[AD4130_MCLK_COUNT_REG] = 1,
+	[AD4130_CHANNEL_X_REG(0) ... AD4130_CHANNEL_X_REG(AD4130_MAX_CHANNELS - 1)] = 3,
+	[AD4130_CONFIG_X_REG(0) ... AD4130_CONFIG_X_REG(AD4130_MAX_SETUPS - 1)] = 2,
+	[AD4130_FILTER_X_REG(0) ... AD4130_FILTER_X_REG(AD4130_MAX_SETUPS - 1)] = 3,
+	[AD4130_OFFSET_X_REG(0) ... AD4130_OFFSET_X_REG(AD4130_MAX_SETUPS - 1)] = 2,
+	[AD4130_GAIN_X_REG(0) ... AD4130_GAIN_X_REG(AD4130_MAX_SETUPS - 1)] = 2,
+	[AD4130_MISC_REG] = 2,
+};
+
 enum ad4130_int_ref_val {
 	AD4130_INT_REF_VAL_2_5V,
 	AD4130_INT_REF_VAL_1_25V,
@@ -224,6 +272,28 @@ enum ad4130_pin_function {
 	AD4130_PIN_FN_VBIAS = BIT(3),
 };
 
+/* Pin mapping for AIN0..AIN7, VBIAS_0..VBIAS_7 */
+static const u8 ad4130_4_pin_map[] = {
+	0x00, 0x01, 0x04, 0x05, 0x0A, 0x0B, 0x0E, 0x0F, /* 0 - 7 */
+};
+
+/* Pin mapping for AIN0..AIN15, VBIAS_0..VBIAS_15 */
+static const u8 ad4130_8_pin_map[] = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0 - 7 */
+	0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 8 - 15 */
+};
+
+struct ad4130_chip_info {
+	const char *name;
+	unsigned int max_analog_pins;
+	unsigned int num_gpios;
+	const struct iio_info *info;
+	const unsigned int *reg_size;
+	const unsigned int reg_size_length;
+	const u8 *pin_map;
+	bool has_fifo;
+};
+
 /*
  * If you make adaptations in this struct, you most likely also have to adapt
  * ad4130_setup_info_eq(), too.
@@ -268,6 +338,7 @@ struct ad4130_state {
 	struct regmap			*regmap;
 	struct spi_device		*spi;
 	struct clk			*mclk;
+	const struct ad4130_chip_info	*chip_info;
 	struct regulator_bulk_data	regulators[4];
 	u32				irq_trigger;
 	u32				inv_irq_trigger;
@@ -294,6 +365,7 @@ struct ad4130_state {
 	u32			mclk_sel;
 	bool			int_ref_en;
 	bool			bipolar;
+	bool			buffer_wait_for_irq;
 
 	unsigned int		num_enabled_channels;
 	unsigned int		effective_watermark;
@@ -301,6 +373,7 @@ struct ad4130_state {
 
 	struct spi_message	fifo_msg;
 	struct spi_transfer	fifo_xfer[2];
+	struct iio_trigger	*trig;
 
 	/*
 	 * DMA (thus cache coherency maintenance) requires any transfer
@@ -312,9 +385,13 @@ struct ad4130_state {
 	u8			reg_write_tx_buf[4];
 	u8			reg_read_tx_buf[1];
 	u8			reg_read_rx_buf[3];
-	u8			fifo_tx_buf[2];
-	u8			fifo_rx_buf[AD4130_FIFO_SIZE *
-					    AD4130_FIFO_MAX_SAMPLE_SIZE];
+	union {
+		struct {
+			u8 fifo_tx_buf[2];
+			u8 fifo_rx_buf[AD4130_FIFO_SIZE * AD4130_FIFO_MAX_SAMPLE_SIZE];
+		};
+		IIO_DECLARE_BUFFER_WITH_TS(u32, scan_channels, AD4130_MAX_CHANNELS);
+	};
 };
 
 static const char * const ad4130_int_pin_names[] = {
@@ -394,10 +471,10 @@ static const char * const ad4130_filter_types_str[] = {
 static int ad4130_get_reg_size(struct ad4130_state *st, unsigned int reg,
 			       unsigned int *size)
 {
-	if (reg >= ARRAY_SIZE(ad4130_reg_size))
+	if (reg >= st->chip_info->reg_size_length)
 		return -EINVAL;
 
-	*size = ad4130_reg_size[reg];
+	*size = st->chip_info->reg_size[reg];
 
 	return 0;
 }
@@ -505,7 +582,8 @@ static int ad4130_gpio_init_valid_mask(struct gpio_chip *gc,
 
 	/*
 	 * Output-only GPIO functionality is available on pins AIN2 through
-	 * AIN5. If these pins are used for anything else, do not expose them.
+	 * AIN5 for some parts and AIN2 through AIN3 for others. If these pins
+	 * are used for anything else, do not expose them.
 	 */
 	for (i = 0; i < ngpios; i++) {
 		unsigned int pin = i + AD4130_AIN2_P1;
@@ -526,8 +604,14 @@ static int ad4130_gpio_set(struct gpio_chip *gc, unsigned int offset,
 			   int value)
 {
 	struct ad4130_state *st = gpiochip_get_data(gc);
-	unsigned int mask = FIELD_PREP(AD4130_IO_CONTROL_GPIO_DATA_MASK,
-				       BIT(offset));
+	unsigned int mask;
+
+	if (st->chip_info->num_gpios == AD4130_MAX_GPIOS)
+		mask = FIELD_PREP(AD4130_IO_CONTROL_GPIO_DATA_MASK,
+				  BIT(offset));
+	else
+		mask = FIELD_PREP(AD4130_4_IO_CONTROL_GPIO_DATA_MASK,
+				  BIT(offset));
 
 	return regmap_update_bits(st->regmap, AD4130_IO_CONTROL_REG, mask,
 				  value ? mask : 0);
@@ -588,10 +672,43 @@ static irqreturn_t ad4130_irq_handler(int irq, void *private)
 	struct iio_dev *indio_dev = private;
 	struct ad4130_state *st = iio_priv(indio_dev);
 
-	if (iio_buffer_enabled(indio_dev))
-		ad4130_push_fifo_data(indio_dev);
-	else
+	if (iio_buffer_enabled(indio_dev)) {
+		if (st->chip_info->has_fifo)
+			ad4130_push_fifo_data(indio_dev);
+		else if (st->buffer_wait_for_irq)
+			complete(&st->completion);
+		else
+			iio_trigger_poll(st->trig);
+	} else {
 		complete(&st->completion);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t ad4130_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct ad4130_state *st = iio_priv(indio_dev);
+	unsigned int data_reg_size = ad4130_data_reg_size(st);
+	struct spi_transfer xfer = { };
+	unsigned int num_en_chn;
+	int ret;
+
+	num_en_chn = bitmap_weight(indio_dev->active_scan_mask,
+				   iio_get_masklength(indio_dev));
+	xfer.rx_buf = st->scan_channels;
+	xfer.len = data_reg_size * num_en_chn;
+	ret = spi_sync_transfer(st->spi, &xfer, 1);
+	if (ret < 0)
+		goto err_out;
+
+	iio_push_to_buffers_with_timestamp(indio_dev, &st->scan_channels,
+					   iio_get_time_ns(indio_dev));
+
+err_out:
+	iio_trigger_notify_done(indio_dev->trig);
 
 	return IRQ_HANDLED;
 }
@@ -1291,6 +1408,79 @@ static const struct iio_info ad4130_info = {
 	.debugfs_reg_access = ad4130_reg_access,
 };
 
+static const struct iio_info ad4131_info = {
+	.read_raw = ad4130_read_raw,
+	.read_avail = ad4130_read_avail,
+	.write_raw_get_fmt = ad4130_write_raw_get_fmt,
+	.write_raw = ad4130_write_raw,
+	.update_scan_mode = ad4130_update_scan_mode,
+	.debugfs_reg_access = ad4130_reg_access,
+};
+
+static const struct ad4130_chip_info ad4129_4_chip_info = {
+	.name = "ad4129-4",
+	.max_analog_pins = 8,
+	.num_gpios = 2,
+	.info = &ad4130_info,
+	.reg_size = ad4129_reg_size,
+	.reg_size_length = ARRAY_SIZE(ad4129_reg_size),
+	.has_fifo = true,
+	.pin_map = ad4130_4_pin_map,
+};
+
+static const struct ad4130_chip_info ad4129_8_chip_info = {
+	.name = "ad4129-8",
+	.max_analog_pins = 16,
+	.num_gpios = 4,
+	.info = &ad4130_info,
+	.reg_size = ad4129_reg_size,
+	.reg_size_length = ARRAY_SIZE(ad4129_reg_size),
+	.has_fifo = true,
+	.pin_map = ad4130_8_pin_map,
+};
+
+static const struct ad4130_chip_info ad4130_4_chip_info = {
+	.name = "ad4130-4",
+	.max_analog_pins = 16,
+	.num_gpios = 2,
+	.info = &ad4130_info,
+	.reg_size = ad4130_reg_size,
+	.reg_size_length = ARRAY_SIZE(ad4130_reg_size),
+	.has_fifo = true,
+	.pin_map = ad4130_4_pin_map,
+};
+
+static const struct ad4130_chip_info ad4130_8_chip_info = {
+	.name = "ad4130-8",
+	.max_analog_pins = 16,
+	.num_gpios = 4,
+	.info = &ad4130_info,
+	.reg_size = ad4130_reg_size,
+	.reg_size_length = ARRAY_SIZE(ad4130_reg_size),
+	.has_fifo = true,
+	.pin_map = ad4130_8_pin_map,
+};
+
+static const struct ad4130_chip_info ad4131_4_chip_info = {
+	.name = "ad4131-4",
+	.max_analog_pins = 8,
+	.num_gpios = 2,
+	.info = &ad4131_info,
+	.reg_size = ad4131_reg_size,
+	.reg_size_length = ARRAY_SIZE(ad4131_reg_size),
+	.pin_map = ad4130_4_pin_map,
+};
+
+static const struct ad4130_chip_info ad4131_8_chip_info = {
+	.name = "ad4131-8",
+	.max_analog_pins = 16,
+	.num_gpios = 4,
+	.info = &ad4131_info,
+	.reg_size = ad4131_reg_size,
+	.reg_size_length = ARRAY_SIZE(ad4131_reg_size),
+	.pin_map = ad4130_8_pin_map,
+};
+
 static int ad4130_buffer_postenable(struct iio_dev *indio_dev)
 {
 	struct ad4130_state *st = iio_priv(indio_dev);
@@ -1298,44 +1488,89 @@ static int ad4130_buffer_postenable(struct iio_dev *indio_dev)
 
 	guard(mutex)(&st->lock);
 
-	ret = ad4130_set_watermark_interrupt_en(st, true);
-	if (ret)
-		return ret;
+	if (st->chip_info->has_fifo) {
+		ret = ad4130_set_watermark_interrupt_en(st, true);
+		if (ret)
+			return ret;
 
-	ret = irq_set_irq_type(st->spi->irq, st->inv_irq_trigger);
-	if (ret)
-		return ret;
+		ret = irq_set_irq_type(st->spi->irq, st->inv_irq_trigger);
+		if (ret)
+			return ret;
+
+		ret = ad4130_set_fifo_mode(st, AD4130_FIFO_MODE_WM);
+		if (ret)
+			return ret;
+	}
 
-	ret = ad4130_set_fifo_mode(st, AD4130_FIFO_MODE_WM);
+	ret = ad4130_set_mode(st, AD4130_MODE_CONTINUOUS);
 	if (ret)
 		return ret;
 
-	return ad4130_set_mode(st, AD4130_MODE_CONTINUOUS);
+	/*
+	 * When using triggered buffer, Entering continuous read mode must
+	 * be the last command sent. No configuration changes are allowed until
+	 * exiting this mode.
+	 */
+	if (!st->chip_info->has_fifo) {
+		ret = regmap_update_bits(st->regmap, AD4130_ADC_CONTROL_REG,
+					 AD4130_ADC_CONTROL_CONT_READ_MASK,
+					 FIELD_PREP(AD4130_ADC_CONTROL_CONT_READ_MASK, 1));
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
 static int ad4130_buffer_predisable(struct iio_dev *indio_dev)
 {
 	struct ad4130_state *st = iio_priv(indio_dev);
 	unsigned int i;
+	u32 temp;
 	int ret;
 
 	guard(mutex)(&st->lock);
 
+	if (!st->chip_info->has_fifo) {
+		temp = 0x42;
+		reinit_completion(&st->completion);
+
+		/*
+		 * In continuous read mode, when all samples are read, the data
+		 * ready signal returns high until the next conversion result is
+		 * ready. To exit this mode, the command must be sent when data
+		 * ready is low. In order to ensure that condition, wait for the
+		 * next interrupt (when the new conversion is finished), allowing
+		 * data ready to return low before sending the exit command.
+		 */
+		st->buffer_wait_for_irq = true;
+		if (!wait_for_completion_timeout(&st->completion, msecs_to_jiffies(1000)))
+			dev_warn(&st->spi->dev, "Conversion timed out\n");
+		st->buffer_wait_for_irq = false;
+
+		/* Perform a read data command to exit continuous read mode (0x42) */
+		ret = spi_write(st->spi, &temp, 1);
+		if (ret)
+			return ret;
+	}
+
 	ret = ad4130_set_mode(st, AD4130_MODE_IDLE);
 	if (ret)
 		return ret;
 
-	ret = irq_set_irq_type(st->spi->irq, st->irq_trigger);
-	if (ret)
-		return ret;
+	if (st->chip_info->has_fifo) {
+		ret = irq_set_irq_type(st->spi->irq, st->irq_trigger);
+		if (ret)
+			return ret;
 
-	ret = ad4130_set_fifo_mode(st, AD4130_FIFO_MODE_DISABLED);
-	if (ret)
-		return ret;
+		ret = ad4130_set_fifo_mode(st, AD4130_FIFO_MODE_DISABLED);
+		if (ret)
+			return ret;
 
-	ret = ad4130_set_watermark_interrupt_en(st, false);
-	if (ret)
-		return ret;
+		ret = ad4130_set_watermark_interrupt_en(st, false);
+		if (ret)
+			return ret;
+	}
 
 	/*
 	 * update_scan_mode() is not called in the disable path, disable all
@@ -1410,6 +1645,34 @@ static const struct iio_dev_attr *ad4130_fifo_attributes[] = {
 	NULL
 };
 
+static const struct iio_trigger_ops ad4130_trigger_ops = {
+	.validate_device = iio_trigger_validate_own_device,
+};
+
+static int ad4130_triggered_buffer_setup(struct iio_dev *indio_dev)
+{
+	struct ad4130_state *st = iio_priv(indio_dev);
+	int ret;
+
+	st->trig = devm_iio_trigger_alloc(indio_dev->dev.parent, "%s-dev%d",
+					  indio_dev->name, iio_device_id(indio_dev));
+	if (!st->trig)
+		return -ENOMEM;
+
+	st->trig->ops = &ad4130_trigger_ops;
+	iio_trigger_set_drvdata(st->trig, indio_dev);
+	ret = devm_iio_trigger_register(indio_dev->dev.parent, st->trig);
+	if (ret)
+		return ret;
+
+	indio_dev->trig = iio_trigger_get(st->trig);
+
+	return devm_iio_triggered_buffer_setup(indio_dev->dev.parent, indio_dev,
+					       &iio_pollfunc_store_time,
+					       &ad4130_trigger_handler,
+					       &ad4130_buffer_ops);
+}
+
 static int _ad4130_find_table_index(const unsigned int *tbl, size_t len,
 				    unsigned int val)
 {
@@ -1496,6 +1759,17 @@ static int ad4130_parse_fw_setup(struct ad4130_state *st,
 	return 0;
 }
 
+static unsigned int ad4130_translate_pin(struct ad4130_state *st,
+					 unsigned int logical_pin)
+{
+	/* For analog input pins, use the chip-specific pin mapping */
+	if (logical_pin < st->chip_info->max_analog_pins)
+		return st->chip_info->pin_map[logical_pin];
+
+	/* For internal channels, pass through unchanged */
+	return logical_pin;
+}
+
 static int ad4130_validate_diff_channel(struct ad4130_state *st, u32 pin)
 {
 	struct device *dev = &st->spi->dev;
@@ -1504,7 +1778,7 @@ static int ad4130_validate_diff_channel(struct ad4130_state *st, u32 pin)
 		return dev_err_probe(dev, -EINVAL,
 				     "Invalid differential channel %u\n", pin);
 
-	if (pin >= AD4130_MAX_ANALOG_PINS)
+	if (pin >= st->chip_info->max_analog_pins)
 		return 0;
 
 	if (st->pins_fn[pin] == AD4130_PIN_FN_SPECIAL)
@@ -1536,7 +1810,7 @@ static int ad4130_validate_excitation_pin(struct ad4130_state *st, u32 pin)
 {
 	struct device *dev = &st->spi->dev;
 
-	if (pin >= AD4130_MAX_ANALOG_PINS)
+	if (pin >= st->chip_info->max_analog_pins)
 		return dev_err_probe(dev, -EINVAL,
 				     "Invalid excitation pin %u\n", pin);
 
@@ -1554,7 +1828,7 @@ static int ad4130_validate_vbias_pin(struct ad4130_state *st, u32 pin)
 {
 	struct device *dev = &st->spi->dev;
 
-	if (pin >= AD4130_MAX_ANALOG_PINS)
+	if (pin >= st->chip_info->max_analog_pins)
 		return dev_err_probe(dev, -EINVAL, "Invalid vbias pin %u\n",
 				     pin);
 
@@ -1730,7 +2004,7 @@ static int ad4310_parse_fw(struct iio_dev *indio_dev)
 
 	ret = device_property_count_u32(dev, "adi,vbias-pins");
 	if (ret > 0) {
-		if (ret > AD4130_MAX_ANALOG_PINS)
+		if (ret > st->chip_info->max_analog_pins)
 			return dev_err_probe(dev, -EINVAL,
 					     "Too many vbias pins %u\n", ret);
 
@@ -1914,9 +2188,14 @@ static int ad4130_setup(struct iio_dev *indio_dev)
 	 * function of P2 takes priority over the GPIO out function.
 	 */
 	val = 0;
-	for (i = 0; i < AD4130_MAX_GPIOS; i++)
-		if (st->pins_fn[i + AD4130_AIN2_P1] == AD4130_PIN_FN_NONE)
-			val |= FIELD_PREP(AD4130_IO_CONTROL_GPIO_CTRL_MASK, BIT(i));
+	for (i = 0; i < st->chip_info->num_gpios; i++) {
+		if (st->pins_fn[i + AD4130_AIN2_P1] == AD4130_PIN_FN_NONE) {
+			if (st->chip_info->num_gpios == 2)
+				val |= FIELD_PREP(AD4130_4_IO_CONTROL_GPIO_CTRL_MASK, BIT(i));
+			else
+				val |= FIELD_PREP(AD4130_IO_CONTROL_GPIO_CTRL_MASK, BIT(i));
+		}
+	}
 
 	val |= FIELD_PREP(AD4130_IO_CONTROL_INT_PIN_SEL_MASK, st->int_pin_sel);
 
@@ -1926,21 +2205,23 @@ static int ad4130_setup(struct iio_dev *indio_dev)
 
 	val = 0;
 	for (i = 0; i < st->num_vbias_pins; i++)
-		val |= BIT(st->vbias_pins[i]);
+		val |= BIT(ad4130_translate_pin(st, st->vbias_pins[i]));
 
 	ret = regmap_write(st->regmap, AD4130_VBIAS_REG, val);
 	if (ret)
 		return ret;
 
-	ret = regmap_clear_bits(st->regmap, AD4130_FIFO_CONTROL_REG,
-				AD4130_FIFO_CONTROL_HEADER_MASK);
-	if (ret)
-		return ret;
+	if (st->chip_info->has_fifo) {
+		ret = regmap_clear_bits(st->regmap, AD4130_FIFO_CONTROL_REG,
+					AD4130_FIFO_CONTROL_HEADER_MASK);
+		if (ret)
+			return ret;
 
-	/* FIFO watermark interrupt starts out as enabled, disable it. */
-	ret = ad4130_set_watermark_interrupt_en(st, false);
-	if (ret)
-		return ret;
+		/* FIFO watermark interrupt starts out as enabled, disable it. */
+		ret = ad4130_set_watermark_interrupt_en(st, false);
+		if (ret)
+			return ret;
+	}
 
 	/* Setup channels. */
 	for (i = 0; i < indio_dev->num_channels; i++) {
@@ -1948,10 +2229,14 @@ static int ad4130_setup(struct iio_dev *indio_dev)
 		struct iio_chan_spec *chan = &st->chans[i];
 		unsigned int val;
 
-		val = FIELD_PREP(AD4130_CHANNEL_AINP_MASK, chan->channel) |
-		      FIELD_PREP(AD4130_CHANNEL_AINM_MASK, chan->channel2) |
-		      FIELD_PREP(AD4130_CHANNEL_IOUT1_MASK, chan_info->iout0) |
-		      FIELD_PREP(AD4130_CHANNEL_IOUT2_MASK, chan_info->iout1);
+		val = FIELD_PREP(AD4130_CHANNEL_AINP_MASK,
+				 ad4130_translate_pin(st, chan->channel)) |
+		      FIELD_PREP(AD4130_CHANNEL_AINM_MASK,
+				 ad4130_translate_pin(st, chan->channel2)) |
+		      FIELD_PREP(AD4130_CHANNEL_IOUT1_MASK,
+				 ad4130_translate_pin(st, chan_info->iout0)) |
+		      FIELD_PREP(AD4130_CHANNEL_IOUT2_MASK,
+				 ad4130_translate_pin(st, chan_info->iout1));
 
 		ret = regmap_write(st->regmap, AD4130_CHANNEL_X_REG(i), val);
 		if (ret)
@@ -1994,26 +2279,30 @@ static int ad4130_probe(struct spi_device *spi)
 
 	st = iio_priv(indio_dev);
 
+	st->chip_info = device_get_match_data(dev);
+
 	memset(st->reset_buf, 0xff, sizeof(st->reset_buf));
 	init_completion(&st->completion);
 	mutex_init(&st->lock);
 	st->spi = spi;
 
-	/*
-	 * Xfer:   [ XFR1 ] [         XFR2         ]
-	 * Master:  0x7D N   ......................
-	 * Slave:   ......   DATA1 DATA2 ... DATAN
-	 */
-	st->fifo_tx_buf[0] = AD4130_COMMS_READ_MASK | AD4130_FIFO_DATA_REG;
-	st->fifo_xfer[0].tx_buf = st->fifo_tx_buf;
-	st->fifo_xfer[0].len = sizeof(st->fifo_tx_buf);
-	st->fifo_xfer[1].rx_buf = st->fifo_rx_buf;
-	spi_message_init_with_transfers(&st->fifo_msg, st->fifo_xfer,
-					ARRAY_SIZE(st->fifo_xfer));
-
-	indio_dev->name = AD4130_NAME;
+	if (st->chip_info->has_fifo) {
+		/*
+		 * Xfer:   [ XFR1 ] [         XFR2         ]
+		 * Master:  0x7D N   ......................
+		 * Slave:   ......   DATA1 DATA2 ... DATAN
+		 */
+		st->fifo_tx_buf[0] = AD4130_COMMS_READ_MASK | AD4130_FIFO_DATA_REG;
+		st->fifo_xfer[0].tx_buf = st->fifo_tx_buf;
+		st->fifo_xfer[0].len = sizeof(st->fifo_tx_buf);
+		st->fifo_xfer[1].rx_buf = st->fifo_rx_buf;
+		spi_message_init_with_transfers(&st->fifo_msg, st->fifo_xfer,
+						ARRAY_SIZE(st->fifo_xfer));
+	}
+
+	indio_dev->name = st->chip_info->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->info = &ad4130_info;
+	indio_dev->info = st->chip_info->info;
 
 	st->regmap = devm_regmap_init(dev, NULL, st, &ad4130_regmap_config);
 	if (IS_ERR(st->regmap))
@@ -2056,9 +2345,9 @@ static int ad4130_probe(struct spi_device *spi)
 	ad4130_fill_scale_tbls(st);
 
 	st->gc.owner = THIS_MODULE;
-	st->gc.label = AD4130_NAME;
+	st->gc.label = st->chip_info->name;
 	st->gc.base = -1;
-	st->gc.ngpio = AD4130_MAX_GPIOS;
+	st->gc.ngpio = st->chip_info->num_gpios;
 	st->gc.parent = dev;
 	st->gc.can_sleep = true;
 	st->gc.init_valid_mask = ad4130_gpio_init_valid_mask;
@@ -2069,9 +2358,12 @@ static int ad4130_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = devm_iio_kfifo_buffer_setup_ext(dev, indio_dev,
-					      &ad4130_buffer_ops,
-					      ad4130_fifo_attributes);
+	if (st->chip_info->has_fifo)
+		ret = devm_iio_kfifo_buffer_setup_ext(dev, indio_dev,
+						      &ad4130_buffer_ops,
+						      ad4130_fifo_attributes);
+	else
+		ret = ad4130_triggered_buffer_setup(indio_dev);
 	if (ret)
 		return ret;
 
@@ -2081,40 +2373,75 @@ static int ad4130_probe(struct spi_device *spi)
 	if (ret)
 		return dev_err_probe(dev, ret, "Failed to request irq\n");
 
-	/*
-	 * When the chip enters FIFO mode, IRQ polarity is inverted.
-	 * When the chip exits FIFO mode, IRQ polarity returns to normal.
-	 * See datasheet pages: 65, FIFO Watermark Interrupt section,
-	 * and 71, Bit Descriptions for STATUS Register, RDYB.
-	 * Cache the normal and inverted IRQ triggers to set them when
-	 * entering and exiting FIFO mode.
-	 */
-	st->irq_trigger = irq_get_trigger_type(spi->irq);
-	if (st->irq_trigger & IRQF_TRIGGER_RISING)
-		st->inv_irq_trigger = IRQF_TRIGGER_FALLING;
-	else if (st->irq_trigger & IRQF_TRIGGER_FALLING)
-		st->inv_irq_trigger = IRQF_TRIGGER_RISING;
-	else
-		return dev_err_probe(dev, -EINVAL, "Invalid irq flags: %u\n",
-				     st->irq_trigger);
+	if (st->chip_info->has_fifo) {
+		/*
+		 * When the chip enters FIFO mode, IRQ polarity is inverted.
+		 * When the chip exits FIFO mode, IRQ polarity returns to normal.
+		 * See datasheet pages: 65, FIFO Watermark Interrupt section,
+		 * and 71, Bit Descriptions for STATUS Register, RDYB.
+		 * Cache the normal and inverted IRQ triggers to set them when
+		 * entering and exiting FIFO mode.
+		 */
+		st->irq_trigger = irq_get_trigger_type(spi->irq);
+		if (st->irq_trigger & IRQF_TRIGGER_RISING)
+			st->inv_irq_trigger = IRQF_TRIGGER_FALLING;
+		else if (st->irq_trigger & IRQF_TRIGGER_FALLING)
+			st->inv_irq_trigger = IRQF_TRIGGER_RISING;
+		else
+			return dev_err_probe(dev, -EINVAL, "Invalid irq flags: %u\n",
+					     st->irq_trigger);
+	}
 
 	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct of_device_id ad4130_of_match[] = {
+	{
+		.compatible = "adi,ad4129-4",
+		.data = &ad4129_4_chip_info
+	},
+	{
+		.compatible = "adi,ad4129-8",
+		.data = &ad4129_8_chip_info
+	},
+	{
+		.compatible = "adi,ad4130-4",
+		.data = &ad4130_4_chip_info
+	},
 	{
 		.compatible = "adi,ad4130",
+		.data = &ad4130_8_chip_info
+	},
+	{
+		.compatible = "adi,ad4131-4",
+		.data = &ad4131_4_chip_info
+	},
+	{
+		.compatible = "adi,ad4131-8",
+		.data = &ad4131_8_chip_info
 	},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, ad4130_of_match);
 
+static const struct spi_device_id ad4130_id_table[] = {
+	{ "ad4129-4", (kernel_ulong_t)&ad4129_4_chip_info },
+	{ "ad4129-8", (kernel_ulong_t)&ad4129_8_chip_info },
+	{ "ad4130-4", (kernel_ulong_t)&ad4130_4_chip_info },
+	{ "ad4130", (kernel_ulong_t)&ad4130_8_chip_info },
+	{ "ad4131-4", (kernel_ulong_t)&ad4131_4_chip_info },
+	{ "ad4131-8", (kernel_ulong_t)&ad4131_8_chip_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, ad4130_id_table);
+
 static struct spi_driver ad4130_driver = {
 	.driver = {
 		.name = AD4130_NAME,
 		.of_match_table = ad4130_of_match,
 	},
 	.probe = ad4130_probe,
+	.id_table = ad4130_id_table,
 };
 module_spi_driver(ad4130_driver);
 
diff --git a/drivers/iio/adc/ad4170-4.c b/drivers/iio/adc/ad4170-4.c
index 77af0e6b2c596..627cbf5a37b0b 100644
--- a/drivers/iio/adc/ad4170-4.c
+++ b/drivers/iio/adc/ad4170-4.c
@@ -1699,34 +1699,29 @@ static int ad4170_gpio_get_direction(struct gpio_chip *gc, unsigned int offset)
 	return ret;
 }
 
+static const unsigned long gpio_masks[] = {
+	AD4170_GPIO_MODE_GPIO0_MSK,
+	AD4170_GPIO_MODE_GPIO1_MSK,
+	AD4170_GPIO_MODE_GPIO2_MSK,
+	AD4170_GPIO_MODE_GPIO3_MSK,
+};
+
 static int ad4170_gpio_direction_input(struct gpio_chip *gc, unsigned int offset)
 {
 	struct iio_dev *indio_dev = gpiochip_get_data(gc);
 	struct ad4170_state *st = iio_priv(indio_dev);
-	unsigned long gpio_mask;
 	int ret;
 
 	if (!iio_device_claim_direct(indio_dev))
 		return -EBUSY;
 
-	switch (offset) {
-	case 0:
-		gpio_mask = AD4170_GPIO_MODE_GPIO0_MSK;
-		break;
-	case 1:
-		gpio_mask = AD4170_GPIO_MODE_GPIO1_MSK;
-		break;
-	case 2:
-		gpio_mask = AD4170_GPIO_MODE_GPIO2_MSK;
-		break;
-	case 3:
-		gpio_mask = AD4170_GPIO_MODE_GPIO3_MSK;
-		break;
-	default:
+	if (offset >= ARRAY_SIZE(gpio_masks)) {
 		ret = -EINVAL;
 		goto err_release;
 	}
-	ret = regmap_update_bits(st->regmap, AD4170_GPIO_MODE_REG, gpio_mask,
+
+	ret = regmap_update_bits(st->regmap, AD4170_GPIO_MODE_REG,
+				 gpio_masks[offset],
 				 AD4170_GPIO_MODE_GPIO_INPUT << (2 * offset));
 
 err_release:
@@ -1740,7 +1735,6 @@ static int ad4170_gpio_direction_output(struct gpio_chip *gc,
 {
 	struct iio_dev *indio_dev = gpiochip_get_data(gc);
 	struct ad4170_state *st = iio_priv(indio_dev);
-	unsigned long gpio_mask;
 	int ret;
 
 	ret = ad4170_gpio_set(gc, offset, value);
@@ -1750,24 +1744,13 @@ static int ad4170_gpio_direction_output(struct gpio_chip *gc,
 	if (!iio_device_claim_direct(indio_dev))
 		return -EBUSY;
 
-	switch (offset) {
-	case 0:
-		gpio_mask = AD4170_GPIO_MODE_GPIO0_MSK;
-		break;
-	case 1:
-		gpio_mask = AD4170_GPIO_MODE_GPIO1_MSK;
-		break;
-	case 2:
-		gpio_mask = AD4170_GPIO_MODE_GPIO2_MSK;
-		break;
-	case 3:
-		gpio_mask = AD4170_GPIO_MODE_GPIO3_MSK;
-		break;
-	default:
+	if (offset >= ARRAY_SIZE(gpio_masks)) {
 		ret = -EINVAL;
 		goto err_release;
 	}
-	ret = regmap_update_bits(st->regmap, AD4170_GPIO_MODE_REG, gpio_mask,
+
+	ret = regmap_update_bits(st->regmap, AD4170_GPIO_MODE_REG,
+				 gpio_masks[offset],
 				 AD4170_GPIO_MODE_GPIO_OUTPUT << (2 * offset));
 
 err_release:
diff --git a/drivers/iio/adc/ad4691.c b/drivers/iio/adc/ad4691.c
new file mode 100644
index 0000000000000..f000e4cde3398
--- /dev/null
+++ b/drivers/iio/adc/ad4691.c
@@ -0,0 +1,1792 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2024-2026 Analog Devices, Inc.
+ * Author: Radu Sabau <radu.sabau@analog.com>
+ */
+#include <linux/array_size.h>
+#include <linux/bitfield.h>
+#include <linux/bitmap.h>
+#include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/device/devres.h>
+#include <linux/dmaengine.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/limits.h>
+#include <linux/math.h>
+#include <linux/module.h>
+#include <linux/mod_devicetable.h>
+#include <linux/property.h>
+#include <linux/pwm.h>
+#include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
+#include <linux/reset.h>
+#include <linux/spi/spi.h>
+#include <linux/spi/offload/consumer.h>
+#include <linux/spi/offload/provider.h>
+#include <linux/units.h>
+#include <linux/unaligned.h>
+
+#include <linux/iio/buffer.h>
+#include <linux/iio/buffer-dma.h>
+#include <linux/iio/buffer-dmaengine.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/iio/trigger.h>
+#include <linux/iio/triggered_buffer.h>
+#include <linux/iio/trigger_consumer.h>
+
+#define AD4691_VREF_uV_MIN			2400000
+#define AD4691_VREF_uV_MAX			5250000
+#define AD4691_VREF_2P5_uV_MAX			2750000
+#define AD4691_VREF_3P0_uV_MAX			3250000
+#define AD4691_VREF_3P3_uV_MAX			3750000
+#define AD4691_VREF_4P096_uV_MAX		4500000
+
+#define AD4691_CNV_DUTY_CYCLE_NS		380
+#define AD4691_CNV_HIGH_TIME_NS			430
+/*
+ * Conservative default for the manual offload periodic trigger. Low enough
+ * to work safely out of the box across all OSR and channel count combinations.
+ */
+#define AD4691_OFFLOAD_INITIAL_TRIGGER_HZ	(100 * HZ_PER_KHZ)
+
+#define AD4691_SPI_CONFIG_A_REG			0x000
+#define AD4691_SW_RESET				(BIT(7) | BIT(0))
+
+#define AD4691_STATUS_REG			0x014
+#define AD4691_CLAMP_STATUS1_REG		0x01A
+#define AD4691_CLAMP_STATUS2_REG		0x01B
+#define AD4691_DEVICE_SETUP			0x020
+#define AD4691_MANUAL_MODE			BIT(2)
+#define AD4691_LDO_EN				BIT(4)
+#define AD4691_REF_CTRL				0x021
+#define AD4691_REF_CTRL_MASK			GENMASK(4, 2)
+#define AD4691_REFBUF_EN			BIT(0)
+#define AD4691_OSC_FREQ_REG			0x023
+#define AD4691_OSC_FREQ_MASK			GENMASK(3, 0)
+#define AD4691_STD_SEQ_CONFIG			0x025
+#define AD4691_SEQ_ALL_CHANNELS_OFF		0x00
+#define AD4691_SPARE_CONTROL			0x02A
+
+#define AD4691_NOOP				0x00
+#define AD4691_ADC_CHAN(ch)			((0x10 + (ch)) << 3)
+
+#define AD4691_OSC_EN_REG			0x180
+#define AD4691_STATE_RESET_REG			0x181
+#define AD4691_STATE_RESET_ALL			0x01
+#define AD4691_ADC_SETUP			0x182
+#define AD4691_ADC_MODE_MASK			GENMASK(1, 0)
+#define AD4691_CNV_BURST_MODE			0x01
+#define AD4691_AUTONOMOUS_MODE			0x02
+/*
+ * ACC_MASK_REG covers both mask bytes via ADDR_DESCENDING SPI: writing a
+ * 16-bit BE value to 0x185 auto-decrements to 0x184 for the second byte.
+ */
+#define AD4691_ACC_MASK_REG			0x185
+#define AD4691_ACC_DEPTH_IN(n)			(0x186 + (n))
+#define AD4691_GPIO_MODE1_REG			0x196
+#define AD4691_GPIO_MODE2_REG			0x197
+#define AD4691_GP_MODE_MASK			GENMASK(3, 0)
+#define AD4691_GP_MODE_DATA_READY		0x06
+#define AD4691_GPIO_READ			0x1A0
+#define AD4691_ACC_STATUS_FULL1_REG		0x1B0
+#define AD4691_ACC_STATUS_FULL2_REG		0x1B1
+#define AD4691_ACC_STATUS_OVERRUN1_REG		0x1B2
+#define AD4691_ACC_STATUS_OVERRUN2_REG		0x1B3
+#define AD4691_ACC_STATUS_SAT1_REG		0x1B4
+#define AD4691_ACC_STATUS_SAT2_REG		0x1BE
+#define AD4691_ACC_SAT_OVR_REG(n)		(0x1C0 + (n))
+#define AD4691_AVG_IN(n)			(0x201 + (2 * (n)))
+#define AD4691_AVG_STS_IN(n)			(0x222 + (3 * (n)))
+#define AD4691_ACC_IN(n)			(0x252 + (3 * (n)))
+#define AD4691_ACC_STS_DATA(n)			(0x283 + (4 * (n)))
+
+static const char * const ad4691_supplies[] = { "avdd", "vio" };
+
+enum ad4691_ref_ctrl {
+	AD4691_VREF_2P5   = 0,
+	AD4691_VREF_3P0   = 1,
+	AD4691_VREF_3P3   = 2,
+	AD4691_VREF_4P096 = 3,
+	AD4691_VREF_5P0   = 4,
+};
+
+struct ad4691_channel_info {
+	const struct iio_chan_spec *channels;
+	const struct iio_chan_spec *manual_channels;
+	unsigned int num_channels;
+};
+
+struct ad4691_chip_info {
+	const char *name;
+	unsigned int max_rate;
+	const struct ad4691_channel_info *sw_info;
+	const struct ad4691_channel_info *offload_info;
+};
+
+/* CNV burst mode channel — exposes oversampling ratio. */
+#define AD4691_CHANNEL(ch)						\
+	{								\
+		.type = IIO_VOLTAGE,					\
+		.indexed = 1,						\
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |		\
+				      BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO) | \
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),	\
+		.info_mask_separate_available =				\
+				      BIT(IIO_CHAN_INFO_OVERSAMPLING_RATIO) | \
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),	\
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SCALE),	\
+		.channel = ch,						\
+		.scan_index = ch,					\
+		.scan_type = {						\
+			.sign = 'u',					\
+			.realbits = 16,					\
+			.storagebits = 16,				\
+			.endianness = IIO_BE,				\
+		},							\
+	}
+
+/*
+ * Manual mode channel — no oversampling ratio attribute. OSR is not
+ * supported in manual mode; ACC_DEPTH_IN is not configured during manual
+ * buffer enable.
+ */
+#define AD4691_MANUAL_CHANNEL(ch)					\
+	{								\
+		.type = IIO_VOLTAGE,					\
+		.indexed = 1,						\
+		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |		\
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),	\
+		.info_mask_separate_available =				\
+				      BIT(IIO_CHAN_INFO_SAMP_FREQ),	\
+		.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SCALE),	\
+		.channel = ch,						\
+		.scan_index = ch,					\
+		.scan_type = {						\
+			.sign = 'u',					\
+			.realbits = 16,					\
+			.storagebits = 16,				\
+			.endianness = IIO_BE,				\
+		},							\
+	}
+
+static const struct iio_chan_spec ad4691_channels[] = {
+	AD4691_CHANNEL(0),
+	AD4691_CHANNEL(1),
+	AD4691_CHANNEL(2),
+	AD4691_CHANNEL(3),
+	AD4691_CHANNEL(4),
+	AD4691_CHANNEL(5),
+	AD4691_CHANNEL(6),
+	AD4691_CHANNEL(7),
+	AD4691_CHANNEL(8),
+	AD4691_CHANNEL(9),
+	AD4691_CHANNEL(10),
+	AD4691_CHANNEL(11),
+	AD4691_CHANNEL(12),
+	AD4691_CHANNEL(13),
+	AD4691_CHANNEL(14),
+	AD4691_CHANNEL(15),
+	IIO_CHAN_SOFT_TIMESTAMP(16),
+};
+
+static const struct iio_chan_spec ad4693_channels[] = {
+	AD4691_CHANNEL(0),
+	AD4691_CHANNEL(1),
+	AD4691_CHANNEL(2),
+	AD4691_CHANNEL(3),
+	AD4691_CHANNEL(4),
+	AD4691_CHANNEL(5),
+	AD4691_CHANNEL(6),
+	AD4691_CHANNEL(7),
+	IIO_CHAN_SOFT_TIMESTAMP(8),
+};
+
+static const struct iio_chan_spec ad4691_manual_channels[] = {
+	AD4691_MANUAL_CHANNEL(0),
+	AD4691_MANUAL_CHANNEL(1),
+	AD4691_MANUAL_CHANNEL(2),
+	AD4691_MANUAL_CHANNEL(3),
+	AD4691_MANUAL_CHANNEL(4),
+	AD4691_MANUAL_CHANNEL(5),
+	AD4691_MANUAL_CHANNEL(6),
+	AD4691_MANUAL_CHANNEL(7),
+	AD4691_MANUAL_CHANNEL(8),
+	AD4691_MANUAL_CHANNEL(9),
+	AD4691_MANUAL_CHANNEL(10),
+	AD4691_MANUAL_CHANNEL(11),
+	AD4691_MANUAL_CHANNEL(12),
+	AD4691_MANUAL_CHANNEL(13),
+	AD4691_MANUAL_CHANNEL(14),
+	AD4691_MANUAL_CHANNEL(15),
+	IIO_CHAN_SOFT_TIMESTAMP(16),
+};
+
+static const struct iio_chan_spec ad4693_manual_channels[] = {
+	AD4691_MANUAL_CHANNEL(0),
+	AD4691_MANUAL_CHANNEL(1),
+	AD4691_MANUAL_CHANNEL(2),
+	AD4691_MANUAL_CHANNEL(3),
+	AD4691_MANUAL_CHANNEL(4),
+	AD4691_MANUAL_CHANNEL(5),
+	AD4691_MANUAL_CHANNEL(6),
+	AD4691_MANUAL_CHANNEL(7),
+	IIO_CHAN_SOFT_TIMESTAMP(8),
+};
+
+static const int ad4691_oversampling_ratios[] = { 1, 2, 4, 8, 16, 32 };
+
+static const struct ad4691_channel_info ad4691_sw_info = {
+	.channels = ad4691_channels,
+	.manual_channels = ad4691_manual_channels,
+	.num_channels = ARRAY_SIZE(ad4691_channels),
+};
+
+static const struct ad4691_channel_info ad4693_sw_info = {
+	.channels = ad4693_channels,
+	.manual_channels = ad4693_manual_channels,
+	.num_channels = ARRAY_SIZE(ad4693_channels),
+};
+
+static const struct ad4691_channel_info ad4691_offload_info = {
+	.channels = ad4691_channels,
+	/*
+	 * Offload paths share the SW channel arrays. num_channels caps access
+	 * before the soft timestamp entry, so no separate array is needed.
+	 */
+	.manual_channels = ad4691_manual_channels,
+	.num_channels = ARRAY_SIZE(ad4691_channels) - 1,
+};
+
+static const struct ad4691_channel_info ad4693_offload_info = {
+	.channels = ad4693_channels,
+	.manual_channels = ad4693_manual_channels,
+	.num_channels = ARRAY_SIZE(ad4693_channels) - 1,
+};
+
+/*
+ * Internal oscillator frequency table. Index is the OSC_FREQ_REG[3:0] value.
+ * Index 0 (1 MHz) is only valid for AD4692/AD4694; AD4691/AD4693 support
+ * up to 500 kHz and use index 1 as their highest valid rate.
+ */
+static const int ad4691_osc_freqs_Hz[] = {
+	[0x0] = 1000000,
+	[0x1] = 500000,
+	[0x2] = 400000,
+	[0x3] = 250000,
+	[0x4] = 200000,
+	[0x5] = 167000,
+	[0x6] = 133000,
+	[0x7] = 125000,
+	[0x8] = 100000,
+	[0x9] = 50000,
+	[0xA] = 25000,
+	[0xB] = 12500,
+	[0xC] = 10000,
+	[0xD] = 5000,
+	[0xE] = 2500,
+	[0xF] = 1250,
+};
+
+static const char * const ad4691_gp_names[] = { "gp0", "gp1", "gp2", "gp3" };
+
+static const struct ad4691_chip_info ad4691_chip_info = {
+	.name = "ad4691",
+	.max_rate = 500 * HZ_PER_KHZ,
+	.sw_info = &ad4691_sw_info,
+	.offload_info = &ad4691_offload_info,
+};
+
+static const struct ad4691_chip_info ad4692_chip_info = {
+	.name = "ad4692",
+	.max_rate = 1 * HZ_PER_MHZ,
+	.sw_info = &ad4691_sw_info,
+	.offload_info = &ad4691_offload_info,
+};
+
+static const struct ad4691_chip_info ad4693_chip_info = {
+	.name = "ad4693",
+	.max_rate = 500 * HZ_PER_KHZ,
+	.sw_info = &ad4693_sw_info,
+	.offload_info = &ad4693_offload_info,
+};
+
+static const struct ad4691_chip_info ad4694_chip_info = {
+	.name = "ad4694",
+	.max_rate = 1 * HZ_PER_MHZ,
+	.sw_info = &ad4693_sw_info,
+	.offload_info = &ad4693_offload_info,
+};
+
+struct ad4691_offload_state {
+	struct spi_offload *offload;
+	struct spi_offload_trigger *trigger;
+	u64 trigger_hz;
+	u8 tx_cmd[17][2];
+	u8 tx_reset[4];
+};
+
+struct ad4691_state {
+	const struct ad4691_chip_info *info;
+	struct regmap *regmap;
+	struct spi_device *spi;
+
+	struct pwm_device *conv_trigger;
+	int irq;
+	int vref_uV;
+	u32 cnv_period_ns;
+	/*
+	 * Snapped oscillator frequency (Hz) shared by all channels. Set when
+	 * sampling_frequency or oversampling_ratio is written; written to
+	 * OSC_FREQ_REG at buffer enable and single-shot time so both attributes
+	 * can be set in any order. Reading in_voltageN_sampling_frequency
+	 * returns target_osc_freq_Hz / osr[N] — the effective rate for that
+	 * channel given its oversampling ratio.
+	 */
+	u32 target_osc_freq_Hz;
+	/* Per-channel oversampling ratio; always 1 in manual mode. */
+	u8 osr[16];
+	/* Scratch buffer for read_avail SAMP_FREQ; content is OSR-dependent. */
+	int samp_freq_avail[ARRAY_SIZE(ad4691_osc_freqs_Hz)];
+
+	bool manual_mode;
+	bool refbuf_en;
+	bool ldo_en;
+	/*
+	 * Synchronize access to members of the driver state, and ensure
+	 * atomicity of consecutive SPI operations.
+	 */
+	struct mutex lock;
+	/*
+	 * Per-buffer-enable lifetime resources:
+	 * Manual Mode - a pre-built SPI message that clocks out N+1
+	 *		 transfers in one go.
+	 * CNV Burst Mode - a pre-built SPI message that clocks out 2*N
+	 *		    transfers in one go.
+	 */
+	struct spi_message scan_msg;
+	/*
+	 * Non-offload: max 16 + 1 NOOP (manual) or 2*16 + 2 (CNV burst).
+	 * Offload reuses this array — both paths are mutually exclusive.
+	 */
+	struct spi_transfer scan_xfers[34];
+	/*
+	 * CNV burst: 16 AVG_IN addresses + state-reset address + state-reset
+	 * value = 18.  Manual: 16 channel cmds + 1 NOOP = 17.
+	 */
+	__be16 scan_tx[18] __aligned(IIO_DMA_MINALIGN);
+	/*
+	 * Scan buffer: one BE16 slot per active channel, plus timestamp.
+	 * DMA-aligned because scan_xfers point rx_buf directly into vals[].
+	 */
+	IIO_DECLARE_DMA_BUFFER_WITH_TS(__be16, vals, 16);
+	/* NULL when no SPI offload hardware is present */
+	struct ad4691_offload_state *offload;
+};
+
+/*
+ * Configure the given GP pin (0-3) as DATA_READY output.
+ * GP0/GP1 → GPIO_MODE1_REG, GP2/GP3 → GPIO_MODE2_REG.
+ * Even pins occupy bits [3:0], odd pins bits [7:4].
+ */
+static int ad4691_gpio_setup(struct ad4691_state *st, unsigned int gp_num)
+{
+	unsigned int bit_off = gp_num % 2;
+	unsigned int reg_off = gp_num / 2;
+	unsigned int shift = 4 * bit_off;
+
+	return regmap_update_bits(st->regmap,
+				  AD4691_GPIO_MODE1_REG + reg_off,
+				  AD4691_GP_MODE_MASK << shift,
+				  AD4691_GP_MODE_DATA_READY << shift);
+}
+
+static const struct spi_offload_config ad4691_offload_config = {
+	.capability_flags = SPI_OFFLOAD_CAP_TRIGGER |
+			    SPI_OFFLOAD_CAP_RX_STREAM_DMA,
+};
+
+static bool ad4691_offload_trigger_match(struct spi_offload_trigger *trigger,
+					 enum spi_offload_trigger_type type,
+					 u64 *args, u32 nargs)
+{
+	return type == SPI_OFFLOAD_TRIGGER_DATA_READY &&
+	       nargs == 1 && args[0] <= 3;
+}
+
+static int ad4691_offload_trigger_request(struct spi_offload_trigger *trigger,
+					  enum spi_offload_trigger_type type,
+					  u64 *args, u32 nargs)
+{
+	struct ad4691_state *st = spi_offload_trigger_get_priv(trigger);
+
+	if (nargs != 1)
+		return -EINVAL;
+
+	return ad4691_gpio_setup(st, args[0]);
+}
+
+static int ad4691_offload_trigger_validate(struct spi_offload_trigger *trigger,
+					   struct spi_offload_trigger_config *config)
+{
+	if (config->type != SPI_OFFLOAD_TRIGGER_DATA_READY)
+		return -EINVAL;
+
+	return 0;
+}
+
+static const struct spi_offload_trigger_ops ad4691_offload_trigger_ops = {
+	.match    = ad4691_offload_trigger_match,
+	.request  = ad4691_offload_trigger_request,
+	.validate = ad4691_offload_trigger_validate,
+};
+
+static int ad4691_reg_read(void *context, unsigned int reg, unsigned int *val)
+{
+	struct spi_device *spi = context;
+	u8 tx[2], rx[4];
+	int ret;
+
+	/* Set bit 15 to mark the operation as READ. */
+	put_unaligned_be16(0x8000 | reg, tx);
+
+	switch (reg) {
+	case 0 ... AD4691_OSC_FREQ_REG:
+	case AD4691_SPARE_CONTROL ... AD4691_ACC_SAT_OVR_REG(15):
+		ret = spi_write_then_read(spi, tx, sizeof(tx), rx, 1);
+		if (ret)
+			return ret;
+		*val = rx[0];
+		return 0;
+	case AD4691_STD_SEQ_CONFIG:
+	case AD4691_AVG_IN(0) ... AD4691_AVG_IN(15):
+		ret = spi_write_then_read(spi, tx, sizeof(tx), rx, 2);
+		if (ret)
+			return ret;
+		*val = get_unaligned_be16(rx);
+		return 0;
+	case AD4691_AVG_STS_IN(0) ... AD4691_AVG_STS_IN(15):
+	case AD4691_ACC_IN(0) ... AD4691_ACC_IN(15):
+		ret = spi_write_then_read(spi, tx, sizeof(tx), rx, 3);
+		if (ret)
+			return ret;
+		*val = get_unaligned_be24(rx);
+		return 0;
+	case AD4691_ACC_STS_DATA(0) ... AD4691_ACC_STS_DATA(15):
+		ret = spi_write_then_read(spi, tx, sizeof(tx), rx, 4);
+		if (ret)
+			return ret;
+		*val = get_unaligned_be32(rx);
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad4691_reg_write(void *context, unsigned int reg, unsigned int val)
+{
+	struct spi_device *spi = context;
+	u8 tx[4];
+
+	put_unaligned_be16(reg, tx);
+
+	switch (reg) {
+	case 0 ... AD4691_OSC_FREQ_REG:
+	case AD4691_SPARE_CONTROL ... AD4691_ACC_MASK_REG - 1:
+	case AD4691_ACC_MASK_REG + 1 ... AD4691_GPIO_MODE2_REG:
+		if (val > U8_MAX)
+			return -EINVAL;
+		tx[2] = val;
+		return spi_write_then_read(spi, tx, 3, NULL, 0);
+	case AD4691_ACC_MASK_REG:
+	case AD4691_STD_SEQ_CONFIG:
+		if (val > U16_MAX)
+			return -EINVAL;
+		put_unaligned_be16(val, &tx[2]);
+		return spi_write_then_read(spi, tx, 4, NULL, 0);
+	default:
+		return -EINVAL;
+	}
+}
+
+static bool ad4691_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case AD4691_STATUS_REG:
+	case AD4691_CLAMP_STATUS1_REG:
+	case AD4691_CLAMP_STATUS2_REG:
+	case AD4691_GPIO_READ:
+	case AD4691_ACC_STATUS_FULL1_REG ... AD4691_ACC_STATUS_SAT2_REG:
+	case AD4691_ACC_SAT_OVR_REG(0) ... AD4691_ACC_SAT_OVR_REG(15):
+	case AD4691_AVG_IN(0) ... AD4691_AVG_IN(15):
+	case AD4691_AVG_STS_IN(0) ... AD4691_AVG_STS_IN(15):
+	case AD4691_ACC_IN(0) ... AD4691_ACC_IN(15):
+	case AD4691_ACC_STS_DATA(0) ... AD4691_ACC_STS_DATA(15):
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool ad4691_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case 0 ... AD4691_OSC_FREQ_REG:
+	case AD4691_SPARE_CONTROL ... AD4691_ACC_SAT_OVR_REG(15):
+	case AD4691_STD_SEQ_CONFIG:
+	case AD4691_AVG_IN(0) ... AD4691_AVG_IN(15):
+	case AD4691_AVG_STS_IN(0) ... AD4691_AVG_STS_IN(15):
+	case AD4691_ACC_IN(0) ... AD4691_ACC_IN(15):
+	case AD4691_ACC_STS_DATA(0) ... AD4691_ACC_STS_DATA(15):
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool ad4691_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case 0 ... AD4691_OSC_FREQ_REG:
+	case AD4691_STD_SEQ_CONFIG:
+	case AD4691_SPARE_CONTROL ... AD4691_GPIO_MODE2_REG:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_config ad4691_regmap_config = {
+	.reg_bits = 16,
+	.val_bits = 32,
+	.reg_read = ad4691_reg_read,
+	.reg_write = ad4691_reg_write,
+	.volatile_reg = ad4691_volatile_reg,
+	.readable_reg = ad4691_readable_reg,
+	.writeable_reg = ad4691_writeable_reg,
+	.max_register = AD4691_ACC_STS_DATA(15),
+	.cache_type = REGCACHE_MAPLE,
+};
+
+/* Write target_osc_freq_Hz to OSC_FREQ_REG. Called at use time. */
+static int ad4691_write_osc_freq(struct ad4691_state *st)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(ad4691_osc_freqs_Hz); i++) {
+		if (ad4691_osc_freqs_Hz[i] == st->target_osc_freq_Hz)
+			return regmap_write(st->regmap, AD4691_OSC_FREQ_REG, i);
+	}
+	return -EINVAL;
+}
+
+/*
+ * Index 0 in ad4691_osc_freqs_Hz is 1 MHz — valid only for AD4692/AD4694
+ * (max_rate == 1 MHz). AD4691/AD4693 cap at 500 kHz so their valid range
+ * starts at index 1.
+ */
+static unsigned int ad4691_samp_freq_start(const struct ad4691_chip_info *info)
+{
+	return (info->max_rate == 1 * HZ_PER_MHZ) ? 0 : 1;
+}
+
+/*
+ * Find the largest oscillator table entry that is both <= needed_osc and
+ * evenly divisible by osr (guaranteeing an integer effective rate on
+ * read-back). Returns 0 if no such entry exists in the chip's valid range.
+ */
+static unsigned int ad4691_find_osc_freq(struct ad4691_state *st,
+					 unsigned int needed_osc,
+					 unsigned int osr)
+{
+	unsigned int start = ad4691_samp_freq_start(st->info);
+
+	for (unsigned int i = start; i < ARRAY_SIZE(ad4691_osc_freqs_Hz); i++) {
+		if ((unsigned int)ad4691_osc_freqs_Hz[i] > needed_osc)
+			continue;
+		if (ad4691_osc_freqs_Hz[i] % osr != 0)
+			continue;
+		return ad4691_osc_freqs_Hz[i];
+	}
+	return 0;
+}
+
+static int ad4691_get_sampling_freq(struct ad4691_state *st, u8 osr, int *val)
+{
+	*val = st->target_osc_freq_Hz / osr;
+	return IIO_VAL_INT;
+}
+
+static int ad4691_set_sampling_freq(struct iio_dev *indio_dev,
+				    struct iio_chan_spec const *chan, int freq)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	unsigned int osr = st->osr[chan->channel];
+	unsigned int found;
+
+	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	if (freq <= 0 || (unsigned int)freq > st->info->max_rate / osr)
+		return -EINVAL;
+
+	found = ad4691_find_osc_freq(st, (unsigned int)freq * osr, osr);
+	if (!found)
+		return -EINVAL;
+
+	/*
+	 * Store the snapped oscillator frequency; OSC_FREQ_REG is written at
+	 * buffer enable and single-shot time so that sampling_frequency and
+	 * oversampling_ratio can be set in any order.
+	 */
+	st->target_osc_freq_Hz = found;
+	return 0;
+}
+
+static int ad4691_read_avail(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     const int **vals, int *type,
+			     int *length, long mask)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	unsigned int start = ad4691_samp_freq_start(st->info);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		unsigned int osr = st->osr[chan->channel];
+		int n = 0;
+
+		/*
+		 * Only oscillator frequencies evenly divisible by the channel's
+		 * OSR yield an integer effective rate; expose those as effective
+		 * rates (osc / osr) so the user works entirely in output-sample
+		 * space.
+		 */
+		for (unsigned int i = start; i < ARRAY_SIZE(ad4691_osc_freqs_Hz); i++) {
+			if (ad4691_osc_freqs_Hz[i] % osr != 0)
+				continue;
+			st->samp_freq_avail[n++] = ad4691_osc_freqs_Hz[i] / osr;
+		}
+		*vals = st->samp_freq_avail;
+		*type = IIO_VAL_INT;
+		*length = n;
+		return IIO_AVAIL_LIST;
+	}
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		*vals = ad4691_oversampling_ratios;
+		*type = IIO_VAL_INT;
+		*length = ARRAY_SIZE(ad4691_oversampling_ratios);
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad4691_single_shot_read(struct iio_dev *indio_dev,
+				   struct iio_chan_spec const *chan, int *val)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	unsigned int reg_val, period_us;
+	int ret;
+
+	guard(mutex)(&st->lock);
+
+	/* Use AUTONOMOUS mode for single-shot reads. */
+	ret = regmap_write(st->regmap, AD4691_STATE_RESET_REG,
+			   AD4691_STATE_RESET_ALL);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(st->regmap, AD4691_STD_SEQ_CONFIG,
+			   BIT(chan->channel));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(st->regmap, AD4691_ACC_MASK_REG,
+			   ~BIT(chan->channel) & GENMASK(15, 0));
+	if (ret)
+		return ret;
+
+	ret = regmap_write(st->regmap, AD4691_ACC_DEPTH_IN(chan->channel),
+			   st->osr[chan->channel]);
+	if (ret)
+		return ret;
+
+	ret = ad4691_write_osc_freq(st);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(st->regmap, AD4691_OSC_EN_REG, 1);
+	if (ret)
+		return ret;
+
+	/*
+	 * Wait osr + 1 oscillator periods: osr for accumulation, +1 for the
+	 * pipeline margin (one extra period ensures the final result is ready).
+	 */
+	period_us = DIV_ROUND_UP((unsigned long)(st->osr[chan->channel] + 1) * USEC_PER_SEC,
+				 st->target_osc_freq_Hz);
+	fsleep(period_us);
+
+	ret = regmap_write(st->regmap, AD4691_OSC_EN_REG, 0);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(st->regmap, AD4691_AVG_IN(chan->channel), &reg_val);
+	if (ret)
+		return ret;
+
+	*val = reg_val;
+
+	ret = regmap_write(st->regmap, AD4691_STATE_RESET_REG, AD4691_STATE_RESET_ALL);
+	if (ret)
+		return ret;
+
+	return IIO_VAL_INT;
+}
+
+static int ad4691_read_raw(struct iio_dev *indio_dev,
+			   struct iio_chan_spec const *chan, int *val,
+			   int *val2, long info)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+
+	switch (info) {
+	case IIO_CHAN_INFO_RAW: {
+		IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+		if (IIO_DEV_ACQUIRE_FAILED(claim))
+			return -EBUSY;
+
+		return ad4691_single_shot_read(indio_dev, chan, val);
+	}
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return ad4691_get_sampling_freq(st, st->osr[chan->channel], val);
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO:
+		*val = st->osr[chan->channel];
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = st->vref_uV / (MICRO / MILLI);
+		*val2 = chan->scan_type.realbits;
+		return IIO_VAL_FRACTIONAL_LOG2;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad4691_write_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int val, int val2, long mask)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		return ad4691_set_sampling_freq(indio_dev, chan, val);
+	case IIO_CHAN_INFO_OVERSAMPLING_RATIO: {
+		IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+		if (IIO_DEV_ACQUIRE_FAILED(claim))
+			return -EBUSY;
+
+		for (unsigned int i = 0; i < ARRAY_SIZE(ad4691_oversampling_ratios); i++) {
+			if (ad4691_oversampling_ratios[i] != val)
+				continue;
+			/*
+			 * Store the new OSR; target_osc_freq_Hz is unchanged.
+			 * The effective rate read back via in_voltageN_sampling_frequency
+			 * becomes target_osc_freq_Hz / new_osr automatically.
+			 */
+			st->osr[chan->channel] = val;
+			return 0;
+		}
+		return -EINVAL;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad4691_reg_access(struct iio_dev *indio_dev, unsigned int reg,
+			     unsigned int writeval, unsigned int *readval)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+
+	guard(mutex)(&st->lock);
+
+	if (readval)
+		return regmap_read(st->regmap, reg, readval);
+
+	return regmap_write(st->regmap, reg, writeval);
+}
+
+static int ad4691_set_pwm_freq(struct ad4691_state *st, int freq)
+{
+	if (!freq)
+		return -EINVAL;
+
+	st->cnv_period_ns = DIV_ROUND_UP(NSEC_PER_SEC, freq);
+	return 0;
+}
+
+static int ad4691_sampling_enable(struct ad4691_state *st, bool enable)
+{
+	struct pwm_state conv_state = {
+		.period     = st->cnv_period_ns,
+		.duty_cycle = AD4691_CNV_DUTY_CYCLE_NS,
+		.polarity   = PWM_POLARITY_NORMAL,
+		.enabled    = enable,
+	};
+
+	return pwm_apply_might_sleep(st->conv_trigger, &conv_state);
+}
+
+/*
+ * ad4691_enter_conversion_mode - Switch the chip to its buffer conversion mode.
+ *
+ * Configures the ADC hardware registers for the mode selected at probe
+ * (CNV_BURST or MANUAL). Called from buffer preenable before starting
+ * sampling. The chip is in AUTONOMOUS mode during idle (for read_raw).
+ */
+static int ad4691_enter_conversion_mode(struct ad4691_state *st)
+{
+	int ret;
+
+	if (st->manual_mode)
+		return regmap_update_bits(st->regmap, AD4691_DEVICE_SETUP,
+					  AD4691_MANUAL_MODE, AD4691_MANUAL_MODE);
+
+	ret = ad4691_write_osc_freq(st);
+	if (ret)
+		return ret;
+
+	ret = regmap_update_bits(st->regmap, AD4691_ADC_SETUP,
+				 AD4691_ADC_MODE_MASK, AD4691_CNV_BURST_MODE);
+	if (ret)
+		return ret;
+
+	return regmap_write(st->regmap, AD4691_STATE_RESET_REG,
+			    AD4691_STATE_RESET_ALL);
+}
+
+/*
+ * ad4691_exit_conversion_mode - Return the chip to AUTONOMOUS mode.
+ *
+ * Called from buffer postdisable to restore the chip to the
+ * idle state used by read_raw. Clears the sequencer and resets state.
+ */
+static int ad4691_exit_conversion_mode(struct ad4691_state *st)
+{
+	if (st->manual_mode)
+		return regmap_update_bits(st->regmap, AD4691_DEVICE_SETUP,
+					  AD4691_MANUAL_MODE, 0);
+
+	return regmap_update_bits(st->regmap, AD4691_ADC_SETUP,
+				  AD4691_ADC_MODE_MASK, AD4691_AUTONOMOUS_MODE);
+}
+
+static int ad4691_manual_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	unsigned int prev_i, k, i;
+	bool first;
+	int ret;
+
+	memset(st->scan_xfers, 0, sizeof(st->scan_xfers));
+	memset(st->scan_tx, 0, sizeof(st->scan_tx));
+
+	spi_message_init(&st->scan_msg);
+
+	first = true;
+	prev_i = 0;
+	k = 0;
+	iio_for_each_active_channel(indio_dev, i) {
+		st->scan_tx[k] = cpu_to_be16(AD4691_ADC_CHAN(i));
+		st->scan_xfers[k].tx_buf = &st->scan_tx[k];
+		/*
+		 * The pipeline means xfer[0] receives the residual from the
+		 * previous sequence, not a valid sample for channel i. Point
+		 * it at vals[i] anyway; xfer[1] (or the NOOP when only one
+		 * channel is active) will overwrite that slot with the real
+		 * result, so no separate dummy buffer is needed.
+		 */
+		if (first) {
+			st->scan_xfers[k].rx_buf = &st->vals[i];
+			first = false;
+		} else {
+			st->scan_xfers[k].rx_buf = &st->vals[prev_i];
+		}
+		st->scan_xfers[k].len = sizeof(__be16);
+		st->scan_xfers[k].cs_change = 1;
+		spi_message_add_tail(&st->scan_xfers[k], &st->scan_msg);
+		prev_i = i;
+		k++;
+	}
+
+	/* Final NOOP transfer retrieves the last channel's result. */
+	st->scan_xfers[k].tx_buf = &st->scan_tx[k]; /* scan_tx[k] == 0 == NOOP */
+	st->scan_xfers[k].rx_buf = &st->vals[prev_i];
+	st->scan_xfers[k].len = sizeof(__be16);
+	spi_message_add_tail(&st->scan_xfers[k], &st->scan_msg);
+
+	ret = spi_optimize_message(st->spi, &st->scan_msg);
+	if (ret)
+		return ret;
+
+	ret = ad4691_enter_conversion_mode(st);
+	if (ret) {
+		spi_unoptimize_message(&st->scan_msg);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ad4691_manual_buffer_postdisable(struct iio_dev *indio_dev)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	int ret;
+
+	ret = ad4691_exit_conversion_mode(st);
+	spi_unoptimize_message(&st->scan_msg);
+	return ret;
+}
+
+static const struct iio_buffer_setup_ops ad4691_manual_buffer_setup_ops = {
+	.preenable = &ad4691_manual_buffer_preenable,
+	.postdisable = &ad4691_manual_buffer_postdisable,
+};
+
+static int ad4691_cnv_burst_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	unsigned int acc_mask;
+	unsigned int k, i;
+	int ret;
+
+	memset(st->scan_xfers, 0, sizeof(st->scan_xfers));
+	memset(st->scan_tx, 0, sizeof(st->scan_tx));
+
+	spi_message_init(&st->scan_msg);
+
+	/*
+	 * Each AVG_IN read needs two transfers: a 2-byte address write phase
+	 * followed by a 2-byte data read phase. CS toggles between channels
+	 * (cs_change=1 on the read phase of all but the last channel).
+	 */
+	k = 0;
+	iio_for_each_active_channel(indio_dev, i) {
+		st->scan_tx[k] = cpu_to_be16(0x8000 | AD4691_AVG_IN(i));
+		st->scan_xfers[2 * k].tx_buf = &st->scan_tx[k];
+		st->scan_xfers[2 * k].len = sizeof(__be16);
+		spi_message_add_tail(&st->scan_xfers[2 * k], &st->scan_msg);
+		st->scan_xfers[2 * k + 1].rx_buf = &st->vals[i];
+		st->scan_xfers[2 * k + 1].len = sizeof(__be16);
+		st->scan_xfers[2 * k + 1].cs_change = 1;
+		spi_message_add_tail(&st->scan_xfers[2 * k + 1], &st->scan_msg);
+		k++;
+	}
+
+	st->scan_tx[k] = cpu_to_be16(AD4691_STATE_RESET_REG);
+	st->scan_xfers[2 * k].tx_buf = &st->scan_tx[k];
+	st->scan_xfers[2 * k].len = sizeof(__be16);
+	spi_message_add_tail(&st->scan_xfers[2 * k], &st->scan_msg);
+	st->scan_tx[k + 1] = cpu_to_be16(AD4691_STATE_RESET_ALL << 8);
+	st->scan_xfers[2 * k + 1].tx_buf = &st->scan_tx[k + 1];
+	st->scan_xfers[2 * k + 1].len = sizeof(__be16);
+	st->scan_xfers[2 * k + 1].cs_change = 1;
+	spi_message_add_tail(&st->scan_xfers[2 * k + 1], &st->scan_msg);
+
+	ret = spi_optimize_message(st->spi, &st->scan_msg);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(st->regmap, AD4691_STD_SEQ_CONFIG,
+			   bitmap_read(indio_dev->active_scan_mask, 0,
+				       iio_get_masklength(indio_dev)));
+	if (ret)
+		goto err_unoptimize;
+
+	acc_mask = ~bitmap_read(indio_dev->active_scan_mask, 0,
+				iio_get_masklength(indio_dev)) & GENMASK(15, 0);
+	ret = regmap_write(st->regmap, AD4691_ACC_MASK_REG, acc_mask);
+	if (ret)
+		goto err_unoptimize;
+
+	iio_for_each_active_channel(indio_dev, i) {
+		ret = regmap_write(st->regmap, AD4691_ACC_DEPTH_IN(i), st->osr[i]);
+		if (ret)
+			goto err_unoptimize;
+	}
+
+	ret = ad4691_enter_conversion_mode(st);
+	if (ret)
+		goto err_unoptimize;
+
+	ret = ad4691_sampling_enable(st, true);
+	if (ret)
+		goto err_exit_conv;
+
+	enable_irq(st->irq);
+	return 0;
+
+err_exit_conv:
+	ad4691_exit_conversion_mode(st);
+err_unoptimize:
+	spi_unoptimize_message(&st->scan_msg);
+	return ret;
+}
+
+static int ad4691_cnv_burst_buffer_postdisable(struct iio_dev *indio_dev)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	int ret;
+
+	disable_irq(st->irq);
+
+	ret = ad4691_sampling_enable(st, false);
+	if (ret)
+		return ret;
+
+	ret = ad4691_exit_conversion_mode(st);
+	spi_unoptimize_message(&st->scan_msg);
+	return ret;
+}
+
+static const struct iio_buffer_setup_ops ad4691_cnv_burst_buffer_setup_ops = {
+	.preenable = &ad4691_cnv_burst_buffer_preenable,
+	.postdisable = &ad4691_cnv_burst_buffer_postdisable,
+};
+
+static int ad4691_manual_offload_buffer_postenable(struct iio_dev *indio_dev)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	struct ad4691_offload_state *offload = st->offload;
+	struct device *dev = regmap_get_device(st->regmap);
+	struct spi_device *spi = to_spi_device(dev);
+	struct spi_offload_trigger_config config = {
+		.type = SPI_OFFLOAD_TRIGGER_PERIODIC,
+	};
+	unsigned int bpw = indio_dev->channels[0].scan_type.realbits;
+	unsigned int bit, k;
+	int ret;
+
+	ret = ad4691_enter_conversion_mode(st);
+	if (ret)
+		return ret;
+
+	memset(st->scan_xfers, 0, sizeof(st->scan_xfers));
+
+	/*
+	 * N+1 transfers for N channels. Each CS-low period triggers
+	 * a conversion AND returns the previous result (pipelined).
+	 *   TX: [AD4691_ADC_CHAN(n), 0x00]
+	 *   RX: [data_hi, data_lo]     (storagebits=16, shift=0)
+	 * Transfer 0 RX is garbage; transfers 1..N carry real data.
+	 */
+	k = 0;
+	iio_for_each_active_channel(indio_dev, bit) {
+		offload->tx_cmd[k][0] = AD4691_ADC_CHAN(bit);
+		st->scan_xfers[k].tx_buf = offload->tx_cmd[k];
+		st->scan_xfers[k].len = sizeof(offload->tx_cmd[k]);
+		st->scan_xfers[k].bits_per_word = bpw;
+		st->scan_xfers[k].cs_change = 1;
+		st->scan_xfers[k].cs_change_delay.value = AD4691_CNV_HIGH_TIME_NS;
+		st->scan_xfers[k].cs_change_delay.unit = SPI_DELAY_UNIT_NSECS;
+		/* First transfer RX is garbage — skip it. */
+		if (k > 0)
+			st->scan_xfers[k].offload_flags = SPI_OFFLOAD_XFER_RX_STREAM;
+		k++;
+	}
+
+	/* Final NOOP to flush pipeline and capture last channel. */
+	offload->tx_cmd[k][0] = AD4691_NOOP;
+	st->scan_xfers[k].tx_buf = offload->tx_cmd[k];
+	st->scan_xfers[k].len = sizeof(offload->tx_cmd[k]);
+	st->scan_xfers[k].bits_per_word = bpw;
+	st->scan_xfers[k].offload_flags = SPI_OFFLOAD_XFER_RX_STREAM;
+	k++;
+
+	spi_message_init_with_transfers(&st->scan_msg, st->scan_xfers, k);
+	st->scan_msg.offload = offload->offload;
+
+	ret = spi_optimize_message(spi, &st->scan_msg);
+	if (ret)
+		goto err_exit_conversion;
+
+	config.periodic.frequency_hz = offload->trigger_hz;
+	ret = spi_offload_trigger_enable(offload->offload, offload->trigger, &config);
+	if (ret)
+		goto err_unoptimize;
+
+	return 0;
+
+err_unoptimize:
+	spi_unoptimize_message(&st->scan_msg);
+err_exit_conversion:
+	ad4691_exit_conversion_mode(st);
+	return ret;
+}
+
+static int ad4691_manual_offload_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	struct ad4691_offload_state *offload = st->offload;
+
+	spi_offload_trigger_disable(offload->offload, offload->trigger);
+	spi_unoptimize_message(&st->scan_msg);
+
+	return ad4691_exit_conversion_mode(st);
+}
+
+static const struct iio_buffer_setup_ops ad4691_manual_offload_buffer_setup_ops = {
+	.postenable = &ad4691_manual_offload_buffer_postenable,
+	.predisable = &ad4691_manual_offload_buffer_predisable,
+};
+
+static int ad4691_cnv_burst_offload_buffer_postenable(struct iio_dev *indio_dev)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	struct ad4691_offload_state *offload = st->offload;
+	struct device *dev = regmap_get_device(st->regmap);
+	struct spi_device *spi = to_spi_device(dev);
+	struct spi_offload_trigger_config config = {
+		.type = SPI_OFFLOAD_TRIGGER_DATA_READY,
+	};
+	unsigned int bpw = indio_dev->channels[0].scan_type.realbits;
+	unsigned int acc_mask;
+	unsigned int bit, k;
+	int ret;
+
+	ret = regmap_write(st->regmap, AD4691_STD_SEQ_CONFIG,
+			   bitmap_read(indio_dev->active_scan_mask, 0,
+				       iio_get_masklength(indio_dev)));
+	if (ret)
+		return ret;
+
+	acc_mask = ~bitmap_read(indio_dev->active_scan_mask, 0,
+				iio_get_masklength(indio_dev)) & GENMASK(15, 0);
+	ret = regmap_write(st->regmap, AD4691_ACC_MASK_REG, acc_mask);
+	if (ret)
+		return ret;
+
+	iio_for_each_active_channel(indio_dev, bit) {
+		ret = regmap_write(st->regmap, AD4691_ACC_DEPTH_IN(bit), st->osr[bit]);
+		if (ret)
+			return ret;
+	}
+
+	ret = ad4691_enter_conversion_mode(st);
+	if (ret)
+		return ret;
+
+	memset(st->scan_xfers, 0, sizeof(st->scan_xfers));
+
+	/*
+	 * Each AVG_IN register read uses two 16-bit transfers:
+	 *   TX: [reg_hi | 0x80, reg_lo]  (address, CS stays asserted)
+	 *   RX: [data_hi, data_lo]       (data, storagebits=16, shift=0)
+	 * The state reset is also split into two 16-bit transfers
+	 * (address then value) to keep bits_per_word uniform throughout.
+	 */
+	k = 0;
+	iio_for_each_active_channel(indio_dev, bit) {
+		put_unaligned_be16(0x8000 | AD4691_AVG_IN(bit), offload->tx_cmd[k]);
+
+		/* TX: address phase, CS stays asserted into data phase */
+		st->scan_xfers[2 * k].tx_buf = offload->tx_cmd[k];
+		st->scan_xfers[2 * k].len = sizeof(offload->tx_cmd[k]);
+		st->scan_xfers[2 * k].bits_per_word = bpw;
+
+		/* RX: data phase, CS toggles after to delimit the next register op */
+		st->scan_xfers[2 * k + 1].len = sizeof(offload->tx_cmd[k]);
+		st->scan_xfers[2 * k + 1].bits_per_word = bpw;
+		st->scan_xfers[2 * k + 1].offload_flags = SPI_OFFLOAD_XFER_RX_STREAM;
+		st->scan_xfers[2 * k + 1].cs_change = 1;
+		k++;
+	}
+
+	/* State reset to re-arm DATA_READY for the next scan. */
+	put_unaligned_be16(AD4691_STATE_RESET_REG, offload->tx_reset);
+	offload->tx_reset[2] = AD4691_STATE_RESET_ALL;
+
+	st->scan_xfers[2 * k].tx_buf = offload->tx_reset;
+	st->scan_xfers[2 * k].len = sizeof(offload->tx_cmd[k]);
+	st->scan_xfers[2 * k].bits_per_word = bpw;
+
+	st->scan_xfers[2 * k + 1].tx_buf = &offload->tx_reset[2];
+	st->scan_xfers[2 * k + 1].len = sizeof(offload->tx_cmd[k]);
+	st->scan_xfers[2 * k + 1].bits_per_word = bpw;
+	st->scan_xfers[2 * k + 1].cs_change = 1;
+
+	spi_message_init_with_transfers(&st->scan_msg, st->scan_xfers, 2 * k + 2);
+	st->scan_msg.offload = offload->offload;
+
+	ret = spi_optimize_message(spi, &st->scan_msg);
+	if (ret)
+		goto err_exit_conversion;
+
+	ret = ad4691_sampling_enable(st, true);
+	if (ret)
+		goto err_unoptimize;
+
+	ret = spi_offload_trigger_enable(offload->offload, offload->trigger, &config);
+	if (ret)
+		goto err_sampling_disable;
+
+	return 0;
+
+err_sampling_disable:
+	ad4691_sampling_enable(st, false);
+err_unoptimize:
+	spi_unoptimize_message(&st->scan_msg);
+err_exit_conversion:
+	ad4691_exit_conversion_mode(st);
+	return ret;
+}
+
+static int ad4691_cnv_burst_offload_buffer_predisable(struct iio_dev *indio_dev)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	struct ad4691_offload_state *offload = st->offload;
+	int ret;
+
+	spi_offload_trigger_disable(offload->offload, offload->trigger);
+
+	ret = ad4691_sampling_enable(st, false);
+	if (ret)
+		return ret;
+
+	spi_unoptimize_message(&st->scan_msg);
+
+	return ad4691_exit_conversion_mode(st);
+}
+
+static const struct iio_buffer_setup_ops ad4691_cnv_burst_offload_buffer_setup_ops = {
+	.postenable = &ad4691_cnv_burst_offload_buffer_postenable,
+	.predisable = &ad4691_cnv_burst_offload_buffer_predisable,
+};
+
+static ssize_t sampling_frequency_show(struct device *dev,
+				       struct device_attribute *attr,
+				       char *buf)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct ad4691_state *st = iio_priv(indio_dev);
+
+	if (st->manual_mode && st->offload)
+		return sysfs_emit(buf, "%llu\n", st->offload->trigger_hz);
+
+	return sysfs_emit(buf, "%lu\n", NSEC_PER_SEC / st->cnv_period_ns);
+}
+
+static ssize_t sampling_frequency_store(struct device *dev,
+					struct device_attribute *attr,
+					const char *buf, size_t len)
+{
+	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
+	struct ad4691_state *st = iio_priv(indio_dev);
+	int freq, ret;
+
+	ret = kstrtoint(buf, 10, &freq);
+	if (ret)
+		return ret;
+
+	IIO_DEV_ACQUIRE_DIRECT_MODE(indio_dev, claim);
+	if (IIO_DEV_ACQUIRE_FAILED(claim))
+		return -EBUSY;
+
+	if (st->manual_mode && st->offload) {
+		struct spi_offload_trigger_config config = {
+			.type = SPI_OFFLOAD_TRIGGER_PERIODIC,
+			.periodic = { .frequency_hz = freq },
+		};
+
+		ret = spi_offload_trigger_validate(st->offload->trigger, &config);
+		if (ret)
+			return ret;
+
+		st->offload->trigger_hz = config.periodic.frequency_hz;
+		return len;
+	}
+
+	ret = ad4691_set_pwm_freq(st, freq);
+	if (ret)
+		return ret;
+
+	return len;
+}
+
+static IIO_DEVICE_ATTR_RW(sampling_frequency, 0);
+
+static const struct iio_dev_attr *ad4691_buffer_attrs[] = {
+	&iio_dev_attr_sampling_frequency,
+	NULL
+};
+
+static irqreturn_t ad4691_irq(int irq, void *private)
+{
+	struct iio_dev *indio_dev = private;
+	struct ad4691_state *st = iio_priv(indio_dev);
+
+	iio_trigger_poll(indio_dev->trig);
+	/*
+	 * Keep the DATA_READY IRQ disabled until the trigger handler has
+	 * finished reading the scan, to prevent a new assertion mid-transfer.
+	 * The PWM continues running uninterrupted; the IRQ is re-enabled in
+	 * ad4691_trigger_handler once spi_sync completes.
+	 *
+	 * IRQF_ONESHOT already masks the hardware line during this threaded
+	 * handler, so disable_irq_nosync here ensures the IRQ stays disabled
+	 * even after IRQF_ONESHOT unmasks on return.
+	 */
+	disable_irq_nosync(st->irq);
+
+	return IRQ_HANDLED;
+}
+
+static const struct iio_trigger_ops ad4691_trigger_ops = {
+	.validate_device = iio_trigger_validate_own_device,
+};
+
+static int ad4691_read_scan(struct iio_dev *indio_dev, s64 timestamp)
+{
+	struct ad4691_state *st = iio_priv(indio_dev);
+	int ret;
+
+	guard(mutex)(&st->lock);
+
+	ret = spi_sync(st->spi, &st->scan_msg);
+	if (ret)
+		return ret;
+
+	/*
+	 * rx_buf pointers in scan_xfers point directly into scan.vals, so no
+	 * copy is needed. The scan_msg already includes a STATE_RESET at the
+	 * end (appended in preenable), so no explicit reset is needed here.
+	 */
+	iio_push_to_buffers_with_ts(indio_dev, st->vals, sizeof(st->vals),
+				    timestamp);
+	return 0;
+}
+
+static irqreturn_t ad4691_trigger_handler(int irq, void *p)
+{
+	struct iio_poll_func *pf = p;
+	struct iio_dev *indio_dev = pf->indio_dev;
+	struct ad4691_state *st = iio_priv(indio_dev);
+
+	ad4691_read_scan(indio_dev, pf->timestamp);
+	if (!st->manual_mode)
+		enable_irq(st->irq);
+	iio_trigger_notify_done(indio_dev->trig);
+	return IRQ_HANDLED;
+}
+
+static const struct iio_info ad4691_info = {
+	.read_raw = &ad4691_read_raw,
+	.write_raw = &ad4691_write_raw,
+	.read_avail = &ad4691_read_avail,
+	.debugfs_reg_access = &ad4691_reg_access,
+};
+
+static int ad4691_pwm_setup(struct ad4691_state *st)
+{
+	struct device *dev = regmap_get_device(st->regmap);
+
+	st->conv_trigger = devm_pwm_get(dev, "cnv");
+	if (IS_ERR(st->conv_trigger))
+		return dev_err_probe(dev, PTR_ERR(st->conv_trigger),
+				     "Failed to get cnv pwm\n");
+
+	return ad4691_set_pwm_freq(st, st->info->max_rate);
+}
+
+static int ad4691_regulator_setup(struct ad4691_state *st)
+{
+	struct device *dev = regmap_get_device(st->regmap);
+	int ret;
+
+	ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(ad4691_supplies),
+					     ad4691_supplies);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to get and enable supplies\n");
+
+	/*
+	 * vdd-supply and ldo-in-supply are mutually exclusive:
+	 *   vdd-supply present  → external 1.8V VDD; disable internal LDO.
+	 *   vdd-supply absent   → enable internal LDO fed from ldo-in-supply.
+	 * Having both simultaneously is strongly inadvisable per the datasheet.
+	 */
+	ret = devm_regulator_get_enable(dev, "vdd");
+	if (ret == -ENODEV) {
+		ret = devm_regulator_get_enable(dev, "ldo-in");
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to get and enable LDO-IN\n");
+		st->ldo_en = true;
+	} else if (ret) {
+		return dev_err_probe(dev, ret, "Failed to get and enable VDD\n");
+	}
+
+	st->vref_uV = devm_regulator_get_enable_read_voltage(dev, "ref");
+	if (st->vref_uV == -ENODEV) {
+		st->vref_uV = devm_regulator_get_enable_read_voltage(dev, "refin");
+		st->refbuf_en = true;
+	}
+	if (st->vref_uV < 0)
+		return dev_err_probe(dev, st->vref_uV,
+				     "Failed to get reference supply\n");
+
+	if (st->vref_uV < AD4691_VREF_uV_MIN || st->vref_uV > AD4691_VREF_uV_MAX)
+		return dev_err_probe(dev, -EINVAL,
+				     "vref(%d) must be in the range [%u...%u]\n",
+				     st->vref_uV, AD4691_VREF_uV_MIN,
+				     AD4691_VREF_uV_MAX);
+
+	return 0;
+}
+
+static int ad4691_reset(struct ad4691_state *st)
+{
+	struct device *dev = regmap_get_device(st->regmap);
+	struct reset_control *rst;
+
+	rst = devm_reset_control_get_optional_exclusive(dev, NULL);
+	if (IS_ERR(rst))
+		return dev_err_probe(dev, PTR_ERR(rst), "Failed to get reset\n");
+
+	if (rst) {
+		/*
+		 * reset_gpio_probe() already drives the pin asserted, so the
+		 * device is held in reset before we get here.
+		 * devm_reset_control_get_optional_exclusive_deasserted() cannot
+		 * be used because it deasserts immediately without delay; the
+		 * datasheet (Table 5) requires a ≥300 µs reset pulse width
+		 * before deassertion.
+		 */
+		fsleep(300);
+		return reset_control_deassert(rst);
+	}
+
+	/* No hardware reset available, fall back to software reset. */
+	return regmap_write(st->regmap, AD4691_SPI_CONFIG_A_REG,
+			    AD4691_SW_RESET);
+}
+
+static int ad4691_config(struct ad4691_state *st)
+{
+	struct device *dev = regmap_get_device(st->regmap);
+	enum ad4691_ref_ctrl ref_val;
+	unsigned int val;
+	int ret;
+
+	/*
+	 * Determine buffer conversion mode from DT: if a PWM is provided it
+	 * drives the CNV pin (CNV_BURST_MODE); otherwise CNV is tied to CS
+	 * and each SPI transfer triggers a conversion (MANUAL_MODE).
+	 * Both modes idle in AUTONOMOUS mode so that read_raw can use the
+	 * internal oscillator without disturbing the hardware configuration.
+	 */
+	if (device_property_present(dev, "pwms")) {
+		st->manual_mode = false;
+		ret = ad4691_pwm_setup(st);
+		if (ret)
+			return ret;
+	} else {
+		st->manual_mode = true;
+	}
+
+	switch (st->vref_uV) {
+	case AD4691_VREF_uV_MIN ... AD4691_VREF_2P5_uV_MAX:
+		ref_val = AD4691_VREF_2P5;
+		break;
+	case AD4691_VREF_2P5_uV_MAX + 1 ... AD4691_VREF_3P0_uV_MAX:
+		ref_val = AD4691_VREF_3P0;
+		break;
+	case AD4691_VREF_3P0_uV_MAX + 1 ... AD4691_VREF_3P3_uV_MAX:
+		ref_val = AD4691_VREF_3P3;
+		break;
+	case AD4691_VREF_3P3_uV_MAX + 1 ... AD4691_VREF_4P096_uV_MAX:
+		ref_val = AD4691_VREF_4P096;
+		break;
+	case AD4691_VREF_4P096_uV_MAX + 1 ... AD4691_VREF_uV_MAX:
+		ref_val = AD4691_VREF_5P0;
+		break;
+	default:
+		return dev_err_probe(dev, -EINVAL,
+				     "Unsupported vref voltage: %d uV\n",
+				     st->vref_uV);
+	}
+
+	val = FIELD_PREP(AD4691_REF_CTRL_MASK, ref_val);
+	if (st->refbuf_en)
+		val |= AD4691_REFBUF_EN;
+
+	ret = regmap_write(st->regmap, AD4691_REF_CTRL, val);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to write REF_CTRL\n");
+
+	ret = regmap_assign_bits(st->regmap, AD4691_DEVICE_SETUP,
+				 AD4691_LDO_EN, st->ldo_en);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to write DEVICE_SETUP\n");
+
+	/*
+	 * Set the internal oscillator to the highest rate this chip supports.
+	 * Index 0 (1 MHz) exceeds the 500 kHz max of AD4691/AD4693, so those
+	 * chips start at index 1 (500 kHz).
+	 */
+	ret = regmap_write(st->regmap, AD4691_OSC_FREQ_REG,
+			   ad4691_samp_freq_start(st->info));
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to write OSC_FREQ\n");
+
+	st->target_osc_freq_Hz = ad4691_osc_freqs_Hz[ad4691_samp_freq_start(st->info)];
+
+	ret = regmap_update_bits(st->regmap, AD4691_ADC_SETUP,
+				 AD4691_ADC_MODE_MASK, AD4691_AUTONOMOUS_MODE);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to write ADC_SETUP\n");
+
+	return 0;
+}
+
+static int ad4691_setup_triggered_buffer(struct iio_dev *indio_dev,
+					 struct ad4691_state *st)
+{
+	struct device *dev = regmap_get_device(st->regmap);
+	struct iio_trigger *trig;
+	unsigned int i;
+	int irq, ret;
+
+	trig = devm_iio_trigger_alloc(dev, "%s-dev%d", indio_dev->name,
+				      iio_device_id(indio_dev));
+	if (!trig)
+		return -ENOMEM;
+
+	trig->ops = &ad4691_trigger_ops;
+	iio_trigger_set_drvdata(trig, st);
+
+	ret = devm_iio_trigger_register(dev, trig);
+	if (ret)
+		return dev_err_probe(dev, ret, "IIO trigger register failed\n");
+
+	indio_dev->trig = iio_trigger_get(trig);
+
+	if (st->manual_mode)
+		return devm_iio_triggered_buffer_setup(dev, indio_dev,
+						       &iio_pollfunc_store_time,
+						       &ad4691_trigger_handler,
+						       &ad4691_manual_buffer_setup_ops);
+
+	/*
+	 * The GP pin named in interrupt-names asserts at end-of-conversion.
+	 * The IRQ handler stops conversions and fires the IIO trigger so
+	 * the trigger handler can read and push the sample to the buffer.
+	 * The IRQ is kept disabled until the buffer is enabled.
+	 */
+	irq = -ENXIO;
+	for (i = 0; i < ARRAY_SIZE(ad4691_gp_names); i++) {
+		irq = fwnode_irq_get_byname(dev_fwnode(dev),
+					    ad4691_gp_names[i]);
+		if (irq > 0)
+			break;
+	}
+	if (irq < 0)
+		return dev_err_probe(dev, irq, "failed to get GP interrupt\n");
+
+	st->irq = irq;
+
+	ret = ad4691_gpio_setup(st, i);
+	if (ret)
+		return ret;
+
+	/*
+	 * IRQ is kept disabled until the buffer is enabled to prevent
+	 * spurious DATA_READY events before the SPI message is set up.
+	 */
+	ret = devm_request_threaded_irq(dev, irq, NULL,
+					&ad4691_irq,
+					IRQF_ONESHOT | IRQF_NO_AUTOEN,
+					indio_dev->name, indio_dev);
+	if (ret)
+		return ret;
+
+	return devm_iio_triggered_buffer_setup_ext(dev, indio_dev,
+						   &iio_pollfunc_store_time,
+						   &ad4691_trigger_handler,
+						   IIO_BUFFER_DIRECTION_IN,
+						   &ad4691_cnv_burst_buffer_setup_ops,
+						   ad4691_buffer_attrs);
+}
+
+static int ad4691_setup_offload(struct iio_dev *indio_dev,
+				struct ad4691_state *st,
+				struct spi_offload *spi_offload)
+{
+	struct device *dev = regmap_get_device(st->regmap);
+	struct ad4691_offload_state *offload;
+	struct dma_chan *rx_dma;
+	int ret;
+
+	offload = devm_kzalloc(dev, sizeof(*offload), GFP_KERNEL);
+	if (!offload)
+		return -ENOMEM;
+
+	offload->offload = spi_offload;
+	st->offload = offload;
+
+	if (st->manual_mode) {
+		offload->trigger =
+			devm_spi_offload_trigger_get(dev, offload->offload,
+						     SPI_OFFLOAD_TRIGGER_PERIODIC);
+		if (IS_ERR(offload->trigger))
+			return dev_err_probe(dev, PTR_ERR(offload->trigger),
+					     "Failed to get periodic offload trigger\n");
+
+		offload->trigger_hz = AD4691_OFFLOAD_INITIAL_TRIGGER_HZ;
+	} else {
+		struct spi_offload_trigger_info trigger_info = {
+			.fwnode = dev_fwnode(dev),
+			.ops    = &ad4691_offload_trigger_ops,
+			.priv   = st,
+		};
+
+		ret = devm_spi_offload_trigger_register(dev, &trigger_info);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to register offload trigger\n");
+
+		offload->trigger =
+			devm_spi_offload_trigger_get(dev, offload->offload,
+						     SPI_OFFLOAD_TRIGGER_DATA_READY);
+		if (IS_ERR(offload->trigger))
+			return dev_err_probe(dev, PTR_ERR(offload->trigger),
+					     "Failed to get DATA_READY offload trigger\n");
+	}
+
+	rx_dma = devm_spi_offload_rx_stream_request_dma_chan(dev, offload->offload);
+	if (IS_ERR(rx_dma))
+		return dev_err_probe(dev, PTR_ERR(rx_dma),
+				     "Failed to get offload RX DMA channel\n");
+
+	if (st->manual_mode)
+		indio_dev->setup_ops = &ad4691_manual_offload_buffer_setup_ops;
+	else
+		indio_dev->setup_ops = &ad4691_cnv_burst_offload_buffer_setup_ops;
+
+	ret = devm_iio_dmaengine_buffer_setup_with_handle(dev, indio_dev, rx_dma,
+							  IIO_BUFFER_DIRECTION_IN);
+	if (ret)
+		return ret;
+
+	indio_dev->buffer->attrs = ad4691_buffer_attrs;
+
+	return 0;
+}
+
+static int ad4691_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct spi_offload *spi_offload;
+	struct iio_dev *indio_dev;
+	struct ad4691_state *st;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	st->spi = spi;
+	st->info = spi_get_device_match_data(spi);
+	memset(st->osr, 1, sizeof(st->osr));
+
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
+	st->regmap = devm_regmap_init(dev, NULL, spi, &ad4691_regmap_config);
+	if (IS_ERR(st->regmap))
+		return dev_err_probe(dev, PTR_ERR(st->regmap),
+				     "Failed to initialize regmap\n");
+
+	ret = ad4691_regulator_setup(st);
+	if (ret)
+		return ret;
+
+	ret = ad4691_reset(st);
+	if (ret)
+		return ret;
+
+	ret = ad4691_config(st);
+	if (ret)
+		return ret;
+
+	spi_offload = devm_spi_offload_get(dev, spi, &ad4691_offload_config);
+	ret = PTR_ERR_OR_ZERO(spi_offload);
+	if (ret == -ENODEV)
+		spi_offload = NULL;
+	else if (ret)
+		return dev_err_probe(dev, ret, "Failed to get SPI offload\n");
+
+	indio_dev->name = st->info->name;
+	indio_dev->info = &ad4691_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+
+	if (spi_offload) {
+		if (st->manual_mode)
+			indio_dev->channels = st->info->offload_info->manual_channels;
+		else
+			indio_dev->channels = st->info->offload_info->channels;
+		indio_dev->num_channels = st->info->offload_info->num_channels;
+		ret = ad4691_setup_offload(indio_dev, st, spi_offload);
+	} else {
+		if (st->manual_mode)
+			indio_dev->channels = st->info->sw_info->manual_channels;
+		else
+			indio_dev->channels = st->info->sw_info->channels;
+		indio_dev->num_channels = st->info->sw_info->num_channels;
+		ret = ad4691_setup_triggered_buffer(indio_dev, st);
+	}
+	if (ret)
+		return ret;
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct of_device_id ad4691_of_match[] = {
+	{ .compatible = "adi,ad4691", .data = &ad4691_chip_info },
+	{ .compatible = "adi,ad4692", .data = &ad4692_chip_info },
+	{ .compatible = "adi,ad4693", .data = &ad4693_chip_info },
+	{ .compatible = "adi,ad4694", .data = &ad4694_chip_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ad4691_of_match);
+
+static const struct spi_device_id ad4691_id[] = {
+	{ "ad4691", (kernel_ulong_t)&ad4691_chip_info },
+	{ "ad4692", (kernel_ulong_t)&ad4692_chip_info },
+	{ "ad4693", (kernel_ulong_t)&ad4693_chip_info },
+	{ "ad4694", (kernel_ulong_t)&ad4694_chip_info },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, ad4691_id);
+
+static struct spi_driver ad4691_driver = {
+	.driver = {
+		.name = "ad4691",
+		.of_match_table = ad4691_of_match,
+	},
+	.probe = ad4691_probe,
+	.id_table = ad4691_id,
+};
+module_spi_driver(ad4691_driver);
+
+MODULE_AUTHOR("Radu Sabau <radu.sabau@analog.com>");
+MODULE_DESCRIPTION("Analog Devices AD4691 Family ADC Driver");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("IIO_DMA_BUFFER");
+MODULE_IMPORT_NS("IIO_DMAENGINE_BUFFER");
diff --git a/drivers/iio/adc/ad4695.c b/drivers/iio/adc/ad4695.c
index cda419638d9a8..53642de7330dc 100644
--- a/drivers/iio/adc/ad4695.c
+++ b/drivers/iio/adc/ad4695.c
@@ -876,14 +876,14 @@ static int ad4695_offload_buffer_postenable(struct iio_dev *indio_dev)
 	if (ret)
 		goto err_unoptimize_message;
 
-	ret = spi_offload_trigger_enable(st->offload, st->offload_trigger,
-					 &config);
+	ret = ad4695_enter_advanced_sequencer_mode(st, num_slots);
 	if (ret)
 		goto err_disable_busy_output;
 
-	ret = ad4695_enter_advanced_sequencer_mode(st, num_slots);
+	ret = spi_offload_trigger_enable(st->offload, st->offload_trigger,
+					 &config);
 	if (ret)
-		goto err_offload_trigger_disable;
+		goto err_exit_conversion_mode;
 
 	mutex_lock(&st->cnv_pwm_lock);
 	pwm_get_state(st->cnv_pwm, &state);
@@ -895,23 +895,16 @@ static int ad4695_offload_buffer_postenable(struct iio_dev *indio_dev)
 	ret = pwm_apply_might_sleep(st->cnv_pwm, &state);
 	mutex_unlock(&st->cnv_pwm_lock);
 	if (ret)
-		goto err_offload_exit_conversion_mode;
+		goto err_offload_trigger_disable;
 
 	return 0;
 
-err_offload_exit_conversion_mode:
-	/*
-	 * We have to unwind in a different order to avoid triggering offload.
-	 * ad4695_exit_conversion_mode() triggers a conversion, so it has to be
-	 * done after spi_offload_trigger_disable().
-	 */
-	spi_offload_trigger_disable(st->offload, st->offload_trigger);
-	ad4695_exit_conversion_mode(st);
-	goto err_disable_busy_output;
-
 err_offload_trigger_disable:
 	spi_offload_trigger_disable(st->offload, st->offload_trigger);
 
+err_exit_conversion_mode:
+	ad4695_exit_conversion_mode(st);
+
 err_disable_busy_output:
 	regmap_clear_bits(st->regmap, AD4695_REG_GP_MODE,
 			  AD4695_REG_GP_MODE_BUSY_GP_EN);
diff --git a/drivers/iio/adc/ad7191.c b/drivers/iio/adc/ad7191.c
index d9cd903ffdd2c..51ec199fb06f4 100644
--- a/drivers/iio/adc/ad7191.c
+++ b/drivers/iio/adc/ad7191.c
@@ -154,27 +154,18 @@ static int ad7191_config_setup(struct iio_dev *indio_dev)
 	const u32 gain[4] = { 1, 8, 64, 128 };
 	static u32 scale_buffer[4][2];
 	int odr_value, odr_index = 0, pga_value, pga_index = 0, i, ret;
+	const char *propname;
 	u64 scale_uv;
 
 	st->samp_freq_index = 0;
 	st->scale_index = 0;
 
-	ret = device_property_read_u32(dev, "adi,odr-value", &odr_value);
-	if (ret && ret != -EINVAL)
-		return dev_err_probe(dev, ret, "Failed to get odr value.\n");
+	propname = "adi,odr-value";
+	if (device_property_present(dev, propname)) {
+		ret = device_property_read_u32(dev, propname, &odr_value);
+		if (ret)
+			return dev_err_probe(dev, ret, "Failed to get %s.\n", propname);
 
-	if (ret == -EINVAL) {
-		st->odr_gpios = devm_gpiod_get_array(dev, "odr", GPIOD_OUT_LOW);
-		if (IS_ERR(st->odr_gpios))
-			return dev_err_probe(dev, PTR_ERR(st->odr_gpios),
-					     "Failed to get odr gpios.\n");
-
-		if (st->odr_gpios->ndescs != 2)
-			return dev_err_probe(dev, -EINVAL, "Expected 2 odr gpio pins.\n");
-
-		st->samp_freq_avail = samp_freq;
-		st->samp_freq_avail_size = ARRAY_SIZE(samp_freq);
-	} else {
 		for (i = 0; i < ARRAY_SIZE(samp_freq); i++) {
 			if (odr_value != samp_freq[i])
 				continue;
@@ -186,6 +177,17 @@ static int ad7191_config_setup(struct iio_dev *indio_dev)
 		st->samp_freq_avail_size = 1;
 
 		st->odr_gpios = NULL;
+	} else {
+		st->odr_gpios = devm_gpiod_get_array(dev, "odr", GPIOD_OUT_LOW);
+		if (IS_ERR(st->odr_gpios))
+			return dev_err_probe(dev, PTR_ERR(st->odr_gpios),
+					     "Failed to get odr gpios.\n");
+
+		if (st->odr_gpios->ndescs != 2)
+			return dev_err_probe(dev, -EINVAL, "Expected 2 odr gpio pins.\n");
+
+		st->samp_freq_avail = samp_freq;
+		st->samp_freq_avail_size = ARRAY_SIZE(samp_freq);
 	}
 
 	mutex_lock(&st->lock);
@@ -200,22 +202,12 @@ static int ad7191_config_setup(struct iio_dev *indio_dev)
 
 	mutex_unlock(&st->lock);
 
-	ret = device_property_read_u32(dev, "adi,pga-value", &pga_value);
-	if (ret && ret != -EINVAL)
-		return dev_err_probe(dev, ret, "Failed to get pga value.\n");
+	propname = "adi,pga-value";
+	if (device_property_present(dev, propname)) {
+		ret = device_property_read_u32(dev, propname, &pga_value);
+		if (ret)
+			return dev_err_probe(dev, ret, "Failed to get %s.\n", propname);
 
-	if (ret == -EINVAL) {
-		st->pga_gpios = devm_gpiod_get_array(dev, "pga", GPIOD_OUT_LOW);
-		if (IS_ERR(st->pga_gpios))
-			return dev_err_probe(dev, PTR_ERR(st->pga_gpios),
-					     "Failed to get pga gpios.\n");
-
-		if (st->pga_gpios->ndescs != 2)
-			return dev_err_probe(dev, -EINVAL, "Expected 2 pga gpio pins.\n");
-
-		st->scale_avail = scale_buffer;
-		st->scale_avail_size = ARRAY_SIZE(scale_buffer);
-	} else {
 		for (i = 0; i < ARRAY_SIZE(gain); i++) {
 			if (pga_value != gain[i])
 				continue;
@@ -227,6 +219,17 @@ static int ad7191_config_setup(struct iio_dev *indio_dev)
 		st->scale_avail_size = 1;
 
 		st->pga_gpios = NULL;
+	} else {
+		st->pga_gpios = devm_gpiod_get_array(dev, "pga", GPIOD_OUT_LOW);
+		if (IS_ERR(st->pga_gpios))
+			return dev_err_probe(dev, PTR_ERR(st->pga_gpios),
+					     "Failed to get pga gpios.\n");
+
+		if (st->pga_gpios->ndescs != 2)
+			return dev_err_probe(dev, -EINVAL, "Expected 2 pga gpio pins.\n");
+
+		st->scale_avail = scale_buffer;
+		st->scale_avail_size = ARRAY_SIZE(scale_buffer);
 	}
 
 	st->temp_gpio = devm_gpiod_get(dev, "temp", GPIOD_OUT_LOW);
diff --git a/drivers/iio/adc/ad7192.c b/drivers/iio/adc/ad7192.c
index 8b1664f6b102c..caf4473ad643e 100644
--- a/drivers/iio/adc/ad7192.c
+++ b/drivers/iio/adc/ad7192.c
@@ -39,7 +39,7 @@
 #define AD7192_REG_CONF		2 /* Configuration Register  (RW, 24-bit) */
 #define AD7192_REG_DATA		3 /* Data Register	     (RO, 24/32-bit) */
 #define AD7192_REG_ID		4 /* ID Register	     (RO, 8-bit) */
-#define AD7192_REG_GPOCON	5 /* GPOCON Register	     (RO, 8-bit) */
+#define AD7192_REG_GPOCON	5 /* GPOCON Register	     (RW, 8-bit) */
 #define AD7192_REG_OFFSET	6 /* Offset Register	     (RW, 16-bit */
 				  /* (AD7792)/24-bit (AD7192)) */
 #define AD7192_REG_FULLSALE	7 /* Full-Scale Register */
@@ -576,7 +576,12 @@ static int ad7192_setup(struct iio_dev *indio_dev, struct device *dev)
 	ret = ad_sd_reset(&st->sd);
 	if (ret < 0)
 		return ret;
-	usleep_range(500, 1000); /* Wait for at least 500us */
+
+	/*
+	 * Per AD7192 datasheet (Rev. A, page 34, RESET section), allow
+	 * 500 us after a reset before accessing on-chip registers.
+	 */
+	fsleep(500);
 
 	/* write/read test for device presence */
 	ret = ad_sd_read_reg(&st->sd, AD7192_REG_ID, 1, &id);
diff --git a/drivers/iio/adc/ad7280a.c b/drivers/iio/adc/ad7280a.c
index ba12a3796e2b1..01c2f55a680cf 100644
--- a/drivers/iio/adc/ad7280a.c
+++ b/drivers/iio/adc/ad7280a.c
@@ -496,7 +496,8 @@ static ssize_t ad7280_store_balance_sw(struct iio_dev *indio_dev,
 	devaddr = chan->address >> 8;
 	ch = chan->address & 0xFF;
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
+
 	if (readin)
 		st->cb_mask[devaddr] |= BIT(ch);
 	else
@@ -505,7 +506,6 @@ static ssize_t ad7280_store_balance_sw(struct iio_dev *indio_dev,
 	ret = ad7280_write(st, devaddr, AD7280A_CELL_BALANCE_REG, 0,
 			   FIELD_PREP(AD7280A_CELL_BALANCE_CHAN_BITMAP_MSK,
 				      st->cb_mask[devaddr]));
-	mutex_unlock(&st->lock);
 
 	return ret ? ret : len;
 }
@@ -516,14 +516,13 @@ static ssize_t ad7280_show_balance_timer(struct iio_dev *indio_dev,
 					 char *buf)
 {
 	struct ad7280_state *st = iio_priv(indio_dev);
+	u8 devaddr = chan->address >> 8;
+	u8 ch = chan->address & 0xFF;
 	unsigned int msecs;
 	int ret;
 
-	mutex_lock(&st->lock);
-	ret = ad7280_read_reg(st, chan->address >> 8,
-			      (chan->address & 0xFF) + AD7280A_CB1_TIMER_REG);
-	mutex_unlock(&st->lock);
-
+	scoped_guard(mutex, &st->lock)
+		ret = ad7280_read_reg(st, devaddr, ch + AD7280A_CB1_TIMER_REG);
 	if (ret < 0)
 		return ret;
 
@@ -538,6 +537,8 @@ static ssize_t ad7280_store_balance_timer(struct iio_dev *indio_dev,
 					  const char *buf, size_t len)
 {
 	struct ad7280_state *st = iio_priv(indio_dev);
+	u8 devaddr = chan->address >> 8;
+	u8 ch = chan->address & 0xFF;
 	int val, val2;
 	int ret;
 
@@ -551,11 +552,10 @@ static ssize_t ad7280_store_balance_timer(struct iio_dev *indio_dev,
 	if (val > 31)
 		return -EINVAL;
 
-	mutex_lock(&st->lock);
-	ret = ad7280_write(st, chan->address >> 8,
-			   (chan->address & 0xFF) + AD7280A_CB1_TIMER_REG, 0,
+	guard(mutex)(&st->lock);
+
+	ret = ad7280_write(st, devaddr, ch + AD7280A_CB1_TIMER_REG, 0,
 			   FIELD_PREP(AD7280A_CB_TIMER_VAL_MSK, val));
-	mutex_unlock(&st->lock);
 
 	return ret ? ret : len;
 }
@@ -737,7 +737,8 @@ static int ad7280a_write_thresh(struct iio_dev *indio_dev,
 	if (val2 != 0)
 		return -EINVAL;
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
+
 	switch (chan->type) {
 	case IIO_VOLTAGE:
 		value = ((val - 1000) * 100) / 1568; /* LSB 15.68mV */
@@ -748,22 +749,20 @@ static int ad7280a_write_thresh(struct iio_dev *indio_dev,
 			ret = ad7280_write(st, AD7280A_DEVADDR_MASTER, addr,
 					   1, value);
 			if (ret)
-				break;
+				return ret;
 			st->cell_threshhigh = value;
-			break;
+			return 0;
 		case IIO_EV_DIR_FALLING:
 			addr = AD7280A_CELL_UNDERVOLTAGE_REG;
 			ret = ad7280_write(st, AD7280A_DEVADDR_MASTER, addr,
 					   1, value);
 			if (ret)
-				break;
+				return ret;
 			st->cell_threshlow = value;
-			break;
+			return 0;
 		default:
-			ret = -EINVAL;
-			goto err_unlock;
+			return -EINVAL;
 		}
-		break;
 	case IIO_TEMP:
 		value = (val * 10) / 196; /* LSB 19.6mV */
 		value = clamp(value, 0L, 0xFFL);
@@ -773,31 +772,23 @@ static int ad7280a_write_thresh(struct iio_dev *indio_dev,
 			ret = ad7280_write(st, AD7280A_DEVADDR_MASTER, addr,
 					   1, value);
 			if (ret)
-				break;
+				return ret;
 			st->aux_threshhigh = value;
-			break;
+			return 0;
 		case IIO_EV_DIR_FALLING:
 			addr = AD7280A_AUX_ADC_UNDERVOLTAGE_REG;
 			ret = ad7280_write(st, AD7280A_DEVADDR_MASTER, addr,
 					   1, value);
 			if (ret)
-				break;
+				return ret;
 			st->aux_threshlow = value;
-			break;
+			return 0;
 		default:
-			ret = -EINVAL;
-			goto err_unlock;
+			return -EINVAL;
 		}
-		break;
 	default:
-		ret = -EINVAL;
-		goto err_unlock;
+		return -EINVAL;
 	}
-
-err_unlock:
-	mutex_unlock(&st->lock);
-
-	return ret;
 }
 
 static irqreturn_t ad7280_event_handler(int irq, void *private)
@@ -881,17 +872,18 @@ static int ad7280_read_raw(struct iio_dev *indio_dev,
 			   long m)
 {
 	struct ad7280_state *st = iio_priv(indio_dev);
+	u8 devaddr = chan->address >> 8;
+	u8 ch = chan->address & 0xFF;
 	int ret;
 
 	switch (m) {
-	case IIO_CHAN_INFO_RAW:
-		mutex_lock(&st->lock);
+	case IIO_CHAN_INFO_RAW: {
+		guard(mutex)(&st->lock);
+
 		if (chan->address == AD7280A_ALL_CELLS)
 			ret = ad7280_read_all_channels(st, st->scan_cnt, NULL);
 		else
-			ret = ad7280_read_channel(st, chan->address >> 8,
-						  chan->address & 0xFF);
-		mutex_unlock(&st->lock);
+			ret = ad7280_read_channel(st, devaddr, ch);
 
 		if (ret < 0)
 			return ret;
@@ -899,8 +891,9 @@ static int ad7280_read_raw(struct iio_dev *indio_dev,
 		*val = ret;
 
 		return IIO_VAL_INT;
+	}
 	case IIO_CHAN_INFO_SCALE:
-		if ((chan->address & 0xFF) <= AD7280A_CELL_VOLTAGE_6_REG)
+		if (ch <= AD7280A_CELL_VOLTAGE_6_REG)
 			*val = 4000;
 		else
 			*val = 5000;
@@ -990,8 +983,8 @@ static int ad7280_probe(struct spi_device *spi)
 			st->acquisition_time = AD7280A_CTRL_LB_ACQ_TIME_1600ns;
 			break;
 		default:
-			dev_err(dev, "Firmware provided acquisition time is invalid\n");
-			return -EINVAL;
+			return dev_err_probe(dev, -EINVAL,
+					     "Firmware provided acquisition time is invalid\n");
 		}
 	} else {
 		st->acquisition_time = AD7280A_CTRL_LB_ACQ_TIME_400ns;
diff --git a/drivers/iio/adc/ad7292.c b/drivers/iio/adc/ad7292.c
index a398973f313d8..e5ad83d2240ae 100644
--- a/drivers/iio/adc/ad7292.c
+++ b/drivers/iio/adc/ad7292.c
@@ -253,12 +253,13 @@ static const struct iio_info ad7292_info = {
 
 static int ad7292_probe(struct spi_device *spi)
 {
+	struct device *dev = &spi->dev;
 	struct ad7292_state *st;
 	struct iio_dev *indio_dev;
 	bool diff_channels = false;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -266,12 +267,11 @@ static int ad7292_probe(struct spi_device *spi)
 	st->spi = spi;
 
 	ret = ad7292_spi_reg_read(st, AD7292_REG_VENDOR_ID);
-	if (ret != ADI_VENDOR_ID) {
-		dev_err(&spi->dev, "Wrong vendor id 0x%x\n", ret);
-		return -EINVAL;
-	}
+	if (ret != ADI_VENDOR_ID)
+		return dev_err_probe(dev, -EINVAL,
+				     "Wrong vendor id 0x%x\n", ret);
 
-	ret = devm_regulator_get_enable_read_voltage(&spi->dev, "vref");
+	ret = devm_regulator_get_enable_read_voltage(dev, "vref");
 	if (ret < 0 && ret != -ENODEV)
 		return ret;
 
@@ -281,7 +281,7 @@ static int ad7292_probe(struct spi_device *spi)
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->info = &ad7292_info;
 
-	device_for_each_child_node_scoped(&spi->dev, child) {
+	device_for_each_child_node_scoped(dev, child) {
 		diff_channels = fwnode_property_read_bool(child,
 							  "diff-channels");
 		if (diff_channels)
@@ -296,7 +296,7 @@ static int ad7292_probe(struct spi_device *spi)
 		indio_dev->channels = ad7292_channels;
 	}
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id ad7292_id_table[] = {
diff --git a/drivers/iio/adc/ad7625.c b/drivers/iio/adc/ad7625.c
index 0466c0c7eae4c..f1ee29f35fa85 100644
--- a/drivers/iio/adc/ad7625.c
+++ b/drivers/iio/adc/ad7625.c
@@ -175,12 +175,12 @@ enum ad7960_mode {
 
 static int ad7625_set_sampling_freq(struct ad7625_state *st, u32 freq)
 {
-	u32 target;
 	struct pwm_waveform clk_gate_wf = { }, cnv_wf = { };
+	unsigned long target;
 	int ret;
 
 	target = DIV_ROUND_UP(NSEC_PER_SEC, freq);
-	cnv_wf.period_length_ns = clamp(target, 100, 10 * KILO);
+	cnv_wf.period_length_ns = clamp(target, 100UL, 10UL * NSEC_PER_USEC);
 
 	/*
 	 * Use the maximum conversion time t_CNVH from the datasheet as
diff --git a/drivers/iio/adc/ad7768-1.c b/drivers/iio/adc/ad7768-1.c
index 73fb734d06b23..598936e47fd2d 100644
--- a/drivers/iio/adc/ad7768-1.c
+++ b/drivers/iio/adc/ad7768-1.c
@@ -1871,10 +1871,8 @@ static int ad7768_probe(struct spi_device *spi)
 		return ret;
 
 	ret = ad7768_setup(indio_dev);
-	if (ret < 0) {
-		dev_err(&spi->dev, "AD7768 setup failed\n");
-		return ret;
-	}
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "AD7768 setup failed\n");
 
 	init_completion(&st->completion);
 	ret = devm_mutex_init(&spi->dev, &st->pga_lock);
diff --git a/drivers/iio/adc/ad7780.c b/drivers/iio/adc/ad7780.c
index 24d2dcad8f4da..26382b1f8c9e4 100644
--- a/drivers/iio/adc/ad7780.c
+++ b/drivers/iio/adc/ad7780.c
@@ -264,16 +264,12 @@ static const struct iio_info ad7780_info = {
 
 static int ad7780_init_gpios(struct device *dev, struct ad7780_state *st)
 {
-	int ret;
-
 	st->powerdown_gpio = devm_gpiod_get_optional(dev,
 						     "powerdown",
 						     GPIOD_OUT_LOW);
-	if (IS_ERR(st->powerdown_gpio)) {
-		ret = PTR_ERR(st->powerdown_gpio);
-		dev_err(dev, "Failed to request powerdown GPIO: %d\n", ret);
-		return ret;
-	}
+	if (IS_ERR(st->powerdown_gpio))
+		return dev_err_probe(dev, PTR_ERR(st->powerdown_gpio),
+				     "Failed to request powerdown GPIO\n");
 
 	if (!st->chip_info->is_ad778x)
 		return 0;
@@ -282,20 +278,16 @@ static int ad7780_init_gpios(struct device *dev, struct ad7780_state *st)
 	st->gain_gpio = devm_gpiod_get_optional(dev,
 						"adi,gain",
 						GPIOD_OUT_HIGH);
-	if (IS_ERR(st->gain_gpio)) {
-		ret = PTR_ERR(st->gain_gpio);
-		dev_err(dev, "Failed to request gain GPIO: %d\n", ret);
-		return ret;
-	}
+	if (IS_ERR(st->gain_gpio))
+		return dev_err_probe(dev, PTR_ERR(st->gain_gpio),
+				     "Failed to request gain GPIO\n");
 
 	st->filter_gpio = devm_gpiod_get_optional(dev,
 						  "adi,filter",
 						  GPIOD_OUT_HIGH);
-	if (IS_ERR(st->filter_gpio)) {
-		ret = PTR_ERR(st->filter_gpio);
-		dev_err(dev, "Failed to request filter GPIO: %d\n", ret);
-		return ret;
-	}
+	if (IS_ERR(st->filter_gpio))
+		return dev_err_probe(dev, PTR_ERR(st->filter_gpio),
+				     "Failed to request filter GPIO\n");
 
 	return 0;
 }
@@ -307,11 +299,12 @@ static void ad7780_reg_disable(void *reg)
 
 static int ad7780_probe(struct spi_device *spi)
 {
+	struct device *dev = &spi->dev;
 	struct ad7780_state *st;
 	struct iio_dev *indio_dev;
 	int ret;
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
@@ -329,29 +322,28 @@ static int ad7780_probe(struct spi_device *spi)
 	indio_dev->num_channels = 1;
 	indio_dev->info = &ad7780_info;
 
-	ret = ad7780_init_gpios(&spi->dev, st);
+	ret = ad7780_init_gpios(dev, st);
 	if (ret)
 		return ret;
 
-	st->reg = devm_regulator_get(&spi->dev, "avdd");
+	st->reg = devm_regulator_get(dev, "avdd");
 	if (IS_ERR(st->reg))
 		return PTR_ERR(st->reg);
 
 	ret = regulator_enable(st->reg);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to enable specified AVdd supply\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "Failed to enable specified AVdd supply\n");
 
-	ret = devm_add_action_or_reset(&spi->dev, ad7780_reg_disable, st->reg);
+	ret = devm_add_action_or_reset(dev, ad7780_reg_disable, st->reg);
 	if (ret)
 		return ret;
 
-	ret = devm_ad_sd_setup_buffer_and_trigger(&spi->dev, indio_dev);
+	ret = devm_ad_sd_setup_buffer_and_trigger(dev, indio_dev);
 	if (ret)
 		return ret;
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id ad7780_id[] = {
diff --git a/drivers/iio/adc/ad7791.c b/drivers/iio/adc/ad7791.c
index 041fc25e3209b..bcdc19e799aaf 100644
--- a/drivers/iio/adc/ad7791.c
+++ b/drivers/iio/adc/ad7791.c
@@ -407,23 +407,22 @@ static void ad7791_reg_disable(void *reg)
 
 static int ad7791_probe(struct spi_device *spi)
 {
-	const struct ad7791_platform_data *pdata = dev_get_platdata(&spi->dev);
+	struct device *dev = &spi->dev;
+	const struct ad7791_platform_data *pdata = dev_get_platdata(dev);
 	struct iio_dev *indio_dev;
 	struct ad7791_state *st;
 	int ret;
 
-	if (!spi->irq) {
-		dev_err(&spi->dev, "Missing IRQ.\n");
-		return -ENXIO;
-	}
+	if (!spi->irq)
+		return dev_err_probe(dev, -ENXIO, "Missing IRQ.\n");
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (!indio_dev)
 		return -ENOMEM;
 
 	st = iio_priv(indio_dev);
 
-	st->reg = devm_regulator_get(&spi->dev, "refin");
+	st->reg = devm_regulator_get(dev, "refin");
 	if (IS_ERR(st->reg))
 		return PTR_ERR(st->reg);
 
@@ -431,7 +430,7 @@ static int ad7791_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	ret = devm_add_action_or_reset(&spi->dev, ad7791_reg_disable, st->reg);
+	ret = devm_add_action_or_reset(dev, ad7791_reg_disable, st->reg);
 	if (ret)
 		return ret;
 
@@ -447,7 +446,7 @@ static int ad7791_probe(struct spi_device *spi)
 	else
 		indio_dev->info = &ad7791_no_filter_info;
 
-	ret = devm_ad_sd_setup_buffer_and_trigger(&spi->dev, indio_dev);
+	ret = devm_ad_sd_setup_buffer_and_trigger(dev, indio_dev);
 	if (ret)
 		return ret;
 
@@ -455,7 +454,7 @@ static int ad7791_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id ad7791_spi_ids[] = {
diff --git a/drivers/iio/adc/ad7793.c b/drivers/iio/adc/ad7793.c
index 8ff7b70d6632e..ea1c192e21421 100644
--- a/drivers/iio/adc/ad7793.c
+++ b/drivers/iio/adc/ad7793.c
@@ -268,7 +268,12 @@ static int ad7793_setup(struct iio_dev *indio_dev,
 	ret = ad_sd_reset(&st->sd);
 	if (ret < 0)
 		goto out;
-	usleep_range(500, 2000); /* Wait for at least 500us */
+
+	/*
+	 * Per AD7792/AD7793 datasheet (Rev. B, page 25, RESET section),
+	 * allow 500 us after a reset before accessing on-chip registers.
+	 */
+	fsleep(500);
 
 	/* write/read test for device presence */
 	ret = ad_sd_read_reg(&st->sd, AD7793_REG_ID, 1, &id);
@@ -278,8 +283,8 @@ static int ad7793_setup(struct iio_dev *indio_dev,
 	id &= AD7793_ID_MASK;
 
 	if (id != st->chip_info->id) {
-		ret = -ENODEV;
-		dev_err(&st->sd.spi->dev, "device ID query failed\n");
+		ret = dev_err_probe(&st->sd.spi->dev, -ENODEV,
+				    "device ID query failed\n");
 		goto out;
 	}
 
@@ -338,8 +343,7 @@ static int ad7793_setup(struct iio_dev *indio_dev,
 
 	return 0;
 out:
-	dev_err(&st->sd.spi->dev, "setup failed\n");
-	return ret;
+	return dev_err_probe(&st->sd.spi->dev, ret, "setup failed\n");
 }
 
 static const u16 ad7793_sample_freq_avail[16] = {0, 470, 242, 123, 62, 50, 39,
@@ -774,22 +778,19 @@ static const struct ad7793_chip_info ad7793_chip_info_tbl[] = {
 
 static int ad7793_probe(struct spi_device *spi)
 {
-	const struct ad7793_platform_data *pdata = dev_get_platdata(&spi->dev);
+	struct device *dev = &spi->dev;
+	const struct ad7793_platform_data *pdata = dev_get_platdata(dev);
 	struct ad7793_state *st;
 	struct iio_dev *indio_dev;
 	int ret, vref_mv = 0;
 
-	if (!pdata) {
-		dev_err(&spi->dev, "no platform data?\n");
-		return -ENODEV;
-	}
+	if (!pdata)
+		return dev_err_probe(dev, -ENODEV, "no platform data?\n");
 
-	if (!spi->irq) {
-		dev_err(&spi->dev, "no IRQ?\n");
-		return -ENODEV;
-	}
+	if (!spi->irq)
+		return dev_err_probe(dev, -ENODEV, "no IRQ?\n");
 
-	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (indio_dev == NULL)
 		return -ENOMEM;
 
@@ -798,7 +799,7 @@ static int ad7793_probe(struct spi_device *spi)
 	ad_sd_init(&st->sd, indio_dev, spi, &ad7793_sigma_delta_info);
 
 	if (pdata->refsel != AD7793_REFSEL_INTERNAL) {
-		ret = devm_regulator_get_enable_read_voltage(&spi->dev, "refin");
+		ret = devm_regulator_get_enable_read_voltage(dev, "refin");
 		if (ret < 0)
 			return ret;
 
@@ -816,7 +817,7 @@ static int ad7793_probe(struct spi_device *spi)
 	indio_dev->num_channels = st->chip_info->num_channels;
 	indio_dev->info = st->chip_info->iio_info;
 
-	ret = devm_ad_sd_setup_buffer_and_trigger(&spi->dev, indio_dev);
+	ret = devm_ad_sd_setup_buffer_and_trigger(dev, indio_dev);
 	if (ret)
 		return ret;
 
@@ -824,7 +825,7 @@ static int ad7793_probe(struct spi_device *spi)
 	if (ret)
 		return ret;
 
-	return devm_iio_device_register(&spi->dev, indio_dev);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct spi_device_id ad7793_id[] = {
diff --git a/drivers/iio/adc/ad7949.c b/drivers/iio/adc/ad7949.c
index b35d299a3977f..ebc629bcfd4da 100644
--- a/drivers/iio/adc/ad7949.c
+++ b/drivers/iio/adc/ad7949.c
@@ -341,8 +341,8 @@ static int ad7949_spi_probe(struct spi_device *spi)
 	} else if (spi_is_bpw_supported(spi, 8)) {
 		spi->bits_per_word = 8;
 	} else {
-		dev_err(dev, "unable to find common BPW with spi controller\n");
-		return -EINVAL;
+		return dev_err_probe(dev, -EINVAL,
+				     "unable to find common BPW with spi controller\n");
 	}
 
 	/* Setup internal voltage reference */
@@ -357,8 +357,8 @@ static int ad7949_spi_probe(struct spi_device *spi)
 		ad7949_adc->refsel = AD7949_CFG_VAL_REF_INT_4096;
 		break;
 	default:
-		dev_err(dev, "unsupported internal voltage reference\n");
-		return -EINVAL;
+		return dev_err_probe(dev, -EINVAL,
+				     "unsupported internal voltage reference\n");
 	}
 
 	/* Setup external voltage reference, buffered? */
@@ -382,10 +382,9 @@ static int ad7949_spi_probe(struct spi_device *spi)
 
 	if (ad7949_adc->refsel & AD7949_CFG_VAL_REF_EXTERNAL) {
 		ret = regulator_enable(ad7949_adc->vref);
-		if (ret < 0) {
-			dev_err(dev, "fail to enable regulator\n");
-			return ret;
-		}
+		if (ret < 0)
+			return dev_err_probe(dev, ret,
+					     "fail to enable regulator\n");
 
 		ret = devm_add_action_or_reset(dev, ad7949_disable_reg,
 					       ad7949_adc->vref);
@@ -396,16 +395,10 @@ static int ad7949_spi_probe(struct spi_device *spi)
 	mutex_init(&ad7949_adc->lock);
 
 	ret = ad7949_spi_init(ad7949_adc);
-	if (ret) {
-		dev_err(dev, "fail to init this device: %d\n", ret);
-		return ret;
-	}
-
-	ret = devm_iio_device_register(dev, indio_dev);
 	if (ret)
-		dev_err(dev, "fail to register iio device: %d\n", ret);
+		return dev_err_probe(dev, ret, "fail to init this device\n");
 
-	return ret;
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static const struct of_device_id ad7949_spi_of_id[] = {
diff --git a/drivers/iio/adc/ad799x.c b/drivers/iio/adc/ad799x.c
index 108bb22162ef1..0ceedea5df0bc 100644
--- a/drivers/iio/adc/ad799x.c
+++ b/drivers/iio/adc/ad799x.c
@@ -18,27 +18,29 @@
  * ad7998 and similar chips.
  */
 
-#include <linux/interrupt.h>
+#include <linux/bitops.h>
 #include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/sysfs.h>
+#include <linux/err.h>
 #include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/regulator/consumer.h>
 #include <linux/slab.h>
+#include <linux/sysfs.h>
 #include <linux/types.h>
-#include <linux/err.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/bitops.h>
+#include <linux/units.h>
 
+#include <linux/iio/buffer.h>
+#include <linux/iio/events.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
-#include <linux/iio/events.h>
-#include <linux/iio/buffer.h>
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
 
 #define AD799X_CHANNEL_SHIFT			4
+#define AD799X_MAX_CHANNELS			8
 
 /*
  * AD7991, AD7995 and AD7999 defines
@@ -133,8 +135,11 @@ struct ad799x_state {
 	unsigned int			id;
 	u16				config;
 
-	u8				*rx_buf;
 	unsigned int			transfer_size;
+
+	int				vref_uV;
+
+	IIO_DECLARE_BUFFER_WITH_TS(__be16, rx_buf, AD799X_MAX_CHANNELS);
 };
 
 static int ad799x_write_config(struct ad799x_state *st, u16 val)
@@ -217,7 +222,8 @@ static irqreturn_t ad799x_trigger_handler(int irq, void *p)
 	}
 
 	b_sent = i2c_smbus_read_i2c_block_data(st->client,
-			cmd, st->transfer_size, st->rx_buf);
+						cmd, st->transfer_size,
+						(u8 *)st->rx_buf);
 	if (b_sent < 0)
 		goto out;
 
@@ -234,11 +240,6 @@ static int ad799x_update_scan_mode(struct iio_dev *indio_dev,
 {
 	struct ad799x_state *st = iio_priv(indio_dev);
 
-	kfree(st->rx_buf);
-	st->rx_buf = kmalloc(indio_dev->scan_bytes, GFP_KERNEL);
-	if (!st->rx_buf)
-		return -ENOMEM;
-
 	st->transfer_size = bitmap_weight(scan_mask,
 					  iio_get_masklength(indio_dev)) * 2;
 
@@ -306,14 +307,7 @@ static int ad799x_read_raw(struct iio_dev *indio_dev,
 			GENMASK(chan->scan_type.realbits - 1, 0);
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
-		if (st->vref)
-			ret = regulator_get_voltage(st->vref);
-		else
-			ret = regulator_get_voltage(st->reg);
-
-		if (ret < 0)
-			return ret;
-		*val = ret / 1000;
+		*val = st->vref_uV / (MICRO / MILLI);
 		*val2 = chan->scan_type.realbits;
 		return IIO_VAL_FRACTIONAL_LOG2;
 	}
@@ -781,8 +775,14 @@ static const struct ad799x_chip_info ad799x_chip_info_tbl[] = {
 	},
 };
 
+static void ad799x_reg_disable(void *reg)
+{
+	regulator_disable(reg);
+}
+
 static int ad799x_probe(struct i2c_client *client)
 {
+	struct device *dev = &client->dev;
 	const struct i2c_device_id *id = i2c_client_get_device_id(client);
 	int ret;
 	int extra_config = 0;
@@ -791,7 +791,7 @@ static int ad799x_probe(struct i2c_client *client)
 	const struct ad799x_chip_info *chip_info =
 		&ad799x_chip_info_tbl[id->driver_data];
 
-	indio_dev = devm_iio_device_alloc(&client->dev, sizeof(*st));
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
 	if (indio_dev == NULL)
 		return -ENOMEM;
 
@@ -799,6 +799,10 @@ static int ad799x_probe(struct i2c_client *client)
 	/* this is only used for device removal purposes */
 	i2c_set_clientdata(client, indio_dev);
 
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
 	st->id = id->driver_data;
 	if (client->irq > 0 && chip_info->irq_config.info)
 		st->chip_config = &chip_info->irq_config;
@@ -807,33 +811,53 @@ static int ad799x_probe(struct i2c_client *client)
 
 	/* TODO: Add pdata options for filtering and bit delay */
 
-	st->reg = devm_regulator_get(&client->dev, "vcc");
+	st->reg = devm_regulator_get(dev, "vcc");
 	if (IS_ERR(st->reg))
 		return PTR_ERR(st->reg);
 	ret = regulator_enable(st->reg);
 	if (ret)
 		return ret;
 
+	ret = devm_add_action_or_reset(dev, ad799x_reg_disable, st->reg);
+	if (ret)
+		return ret;
+
 	/* check if an external reference is supplied */
 	if (chip_info->has_vref) {
-		st->vref = devm_regulator_get_optional(&client->dev, "vref");
+		st->vref = devm_regulator_get_optional(dev, "vref");
 		ret = PTR_ERR_OR_ZERO(st->vref);
-		if (ret) {
-			if (ret != -ENODEV)
-				goto error_disable_reg;
+		if (ret == -ENODEV) {
 			st->vref = NULL;
-			dev_info(&client->dev, "Using VCC reference voltage\n");
+			dev_info(dev, "Using VCC reference voltage\n");
+		} else if (ret) {
+			return ret;
 		}
 
 		if (st->vref) {
-			dev_info(&client->dev, "Using external reference voltage\n");
+			dev_info(dev, "Using external reference voltage\n");
 			extra_config |= AD7991_REF_SEL;
 			ret = regulator_enable(st->vref);
 			if (ret)
-				goto error_disable_reg;
+				return ret;
+
+			ret = devm_add_action_or_reset(dev, ad799x_reg_disable, st->vref);
+			if (ret)
+				return ret;
+
+			ret = regulator_get_voltage(st->vref);
+			if (ret < 0)
+				return ret;
+			st->vref_uV = ret;
 		}
 	}
 
+	if (!st->vref) {
+		ret = regulator_get_voltage(st->reg);
+		if (ret < 0)
+			return ret;
+		st->vref_uV = ret;
+	}
+
 	st->client = client;
 
 	indio_dev->name = id->name;
@@ -845,15 +869,15 @@ static int ad799x_probe(struct i2c_client *client)
 
 	ret = ad799x_update_config(st, st->chip_config->default_config | extra_config);
 	if (ret)
-		goto error_disable_vref;
+		return ret;
 
-	ret = iio_triggered_buffer_setup(indio_dev, NULL,
+	ret = devm_iio_triggered_buffer_setup(dev, indio_dev, NULL,
 		&ad799x_trigger_handler, NULL);
 	if (ret)
-		goto error_disable_vref;
+		return ret;
 
 	if (client->irq > 0) {
-		ret = devm_request_threaded_irq(&client->dev,
+		ret = devm_request_threaded_irq(dev,
 						client->irq,
 						NULL,
 						ad799x_event_handler,
@@ -862,40 +886,10 @@ static int ad799x_probe(struct i2c_client *client)
 						client->name,
 						indio_dev);
 		if (ret)
-			goto error_cleanup_ring;
+			return ret;
 	}
 
-	mutex_init(&st->lock);
-
-	ret = iio_device_register(indio_dev);
-	if (ret)
-		goto error_cleanup_ring;
-
-	return 0;
-
-error_cleanup_ring:
-	iio_triggered_buffer_cleanup(indio_dev);
-error_disable_vref:
-	if (st->vref)
-		regulator_disable(st->vref);
-error_disable_reg:
-	regulator_disable(st->reg);
-
-	return ret;
-}
-
-static void ad799x_remove(struct i2c_client *client)
-{
-	struct iio_dev *indio_dev = i2c_get_clientdata(client);
-	struct ad799x_state *st = iio_priv(indio_dev);
-
-	iio_device_unregister(indio_dev);
-
-	iio_triggered_buffer_cleanup(indio_dev);
-	if (st->vref)
-		regulator_disable(st->vref);
-	regulator_disable(st->reg);
-	kfree(st->rx_buf);
+	return devm_iio_device_register(dev, indio_dev);
 }
 
 static int ad799x_suspend(struct device *dev)
@@ -965,7 +959,6 @@ static struct i2c_driver ad799x_driver = {
 		.pm = pm_sleep_ptr(&ad799x_pm_ops),
 	},
 	.probe = ad799x_probe,
-	.remove = ad799x_remove,
 	.id_table = ad799x_id,
 };
 module_i2c_driver(ad799x_driver);
diff --git a/drivers/iio/adc/ad9467.c b/drivers/iio/adc/ad9467.c
index 0bf67437508f7..0c377f9a7f256 100644
--- a/drivers/iio/adc/ad9467.c
+++ b/drivers/iio/adc/ad9467.c
@@ -1349,11 +1349,10 @@ static int ad9467_probe(struct spi_device *spi)
 		return ret;
 
 	id = ad9467_spi_read(st, AN877_ADC_REG_CHIP_ID);
-	if (id != st->info->id) {
-		dev_err(dev, "Mismatch CHIP_ID, got 0x%X, expected 0x%X\n",
-			id, st->info->id);
-		return -ENODEV;
-	}
+	if (id != st->info->id)
+		return dev_err_probe(dev, -ENODEV,
+				     "Mismatch CHIP_ID, got 0x%X, expected 0x%X\n",
+				     id, st->info->id);
 
 	if (st->info->num_scales > 1)
 		indio_dev->info = &ad9467_info;
diff --git a/drivers/iio/adc/hx711.c b/drivers/iio/adc/hx711.c
index 1db8b68a8f64e..86d2a70dd3de8 100644
--- a/drivers/iio/adc/hx711.c
+++ b/drivers/iio/adc/hx711.c
@@ -28,22 +28,20 @@
 struct hx711_gain_to_scale {
 	int			gain;
 	int			gain_pulse;
-	int			scale;
 	int			channel;
 };
 
 /*
  * .scale depends on AVDD which in turn is known as soon as the regulator
- * is available
- * therefore we set .scale in hx711_probe()
+ * is available; it is stored per device in hx711_data.gain_scale[]
  *
  * channel A in documentation is channel 0 in source code
  * channel B in documentation is channel 1 in source code
  */
-static struct hx711_gain_to_scale hx711_gain_to_scale[HX711_GAIN_MAX] = {
-	{ 128, 1, 0, 0 },
-	{  32, 2, 0, 1 },
-	{  64, 3, 0, 0 }
+static const struct hx711_gain_to_scale hx711_gain_to_scale[HX711_GAIN_MAX] = {
+	{ 128, 1, 0 },
+	{  32, 2, 1 },
+	{  64, 3, 0 },
 };
 
 static int hx711_get_gain_to_pulse(int gain)
@@ -56,22 +54,22 @@ static int hx711_get_gain_to_pulse(int gain)
 	return 1;
 }
 
-static int hx711_get_gain_to_scale(int gain)
+static int hx711_get_gain_to_scale(const int *gain_scale, int gain)
 {
 	int i;
 
 	for (i = 0; i < HX711_GAIN_MAX; i++)
 		if (hx711_gain_to_scale[i].gain == gain)
-			return hx711_gain_to_scale[i].scale;
+			return gain_scale[i];
 	return 0;
 }
 
-static int hx711_get_scale_to_gain(int scale)
+static int hx711_get_scale_to_gain(const int *gain_scale, int scale)
 {
 	int i;
 
 	for (i = 0; i < HX711_GAIN_MAX; i++)
-		if (hx711_gain_to_scale[i].scale == scale)
+		if (gain_scale[i] == scale)
 			return hx711_gain_to_scale[i].gain;
 	return -EINVAL;
 }
@@ -82,6 +80,7 @@ struct hx711_data {
 	struct gpio_desc	*gpiod_dout;
 	int			gain_set;	/* gain set on device */
 	int			gain_chan_a;	/* gain for channel A */
+	int			gain_scale[HX711_GAIN_MAX];
 	struct mutex		lock;
 	/*
 	 * triggered buffer
@@ -290,7 +289,8 @@ static int hx711_read_raw(struct iio_dev *indio_dev,
 		*val = 0;
 		mutex_lock(&hx711_data->lock);
 
-		*val2 = hx711_get_gain_to_scale(hx711_data->gain_set);
+		*val2 = hx711_get_gain_to_scale(hx711_data->gain_scale,
+						hx711_data->gain_set);
 
 		mutex_unlock(&hx711_data->lock);
 
@@ -321,7 +321,7 @@ static int hx711_write_raw(struct iio_dev *indio_dev,
 
 		mutex_lock(&hx711_data->lock);
 
-		gain = hx711_get_scale_to_gain(val2);
+		gain = hx711_get_scale_to_gain(hx711_data->gain_scale, val2);
 		if (gain < 0) {
 			mutex_unlock(&hx711_data->lock);
 			return gain;
@@ -386,6 +386,7 @@ static ssize_t hx711_scale_available_show(struct device *dev,
 				struct device_attribute *attr,
 				char *buf)
 {
+	struct hx711_data *hx711_data = iio_priv(dev_to_iio_dev(dev));
 	struct iio_dev_attr *iio_attr = to_iio_dev_attr(attr);
 	int channel = iio_attr->address;
 	int i, len = 0;
@@ -393,7 +394,7 @@ static ssize_t hx711_scale_available_show(struct device *dev,
 	for (i = 0; i < HX711_GAIN_MAX; i++)
 		if (hx711_gain_to_scale[i].channel == channel)
 			len += sprintf(buf + len, "0.%09d ",
-					hx711_gain_to_scale[i].scale);
+				       hx711_data->gain_scale[i]);
 
 	len += sprintf(buf + len, "\n");
 
@@ -511,7 +512,7 @@ static int hx711_probe(struct platform_device *pdev)
 	ret *= 100;
 
 	for (i = 0; i < HX711_GAIN_MAX; i++)
-		hx711_gain_to_scale[i].scale =
+		hx711_data->gain_scale[i] =
 			ret / hx711_gain_to_scale[i].gain / 1678;
 
 	hx711_data->gain_set = 128;
diff --git a/drivers/iio/adc/ingenic-adc.c b/drivers/iio/adc/ingenic-adc.c
index 1e802c8779a4c..414f69acab7b3 100644
--- a/drivers/iio/adc/ingenic-adc.c
+++ b/drivers/iio/adc/ingenic-adc.c
@@ -7,6 +7,8 @@
  */
 
 #include <dt-bindings/iio/adc/ingenic,adc.h>
+
+#include <linux/cleanup.h>
 #include <linux/clk.h>
 #include <linux/iio/buffer.h>
 #include <linux/iio/iio.h>
@@ -115,7 +117,7 @@ static void ingenic_adc_set_adcmd(struct iio_dev *iio_dev, unsigned long mask)
 {
 	struct ingenic_adc *adc = iio_priv(iio_dev);
 
-	mutex_lock(&adc->lock);
+	guard(mutex)(&adc->lock);
 
 	/* Init ADCMD */
 	readl(adc->base + JZ_ADC_REG_ADCMD);
@@ -162,8 +164,6 @@ static void ingenic_adc_set_adcmd(struct iio_dev *iio_dev, unsigned long mask)
 
 	/* We're done */
 	writel(0, adc->base + JZ_ADC_REG_ADCMD);
-
-	mutex_unlock(&adc->lock);
 }
 
 static void ingenic_adc_set_config(struct ingenic_adc *adc,
@@ -172,18 +172,15 @@ static void ingenic_adc_set_config(struct ingenic_adc *adc,
 {
 	uint32_t cfg;
 
-	mutex_lock(&adc->lock);
+	guard(mutex)(&adc->lock);
 
 	cfg = readl(adc->base + JZ_ADC_REG_CFG) & ~mask;
 	cfg |= val;
 	writel(cfg, adc->base + JZ_ADC_REG_CFG);
-
-	mutex_unlock(&adc->lock);
 }
 
-static void ingenic_adc_enable_unlocked(struct ingenic_adc *adc,
-					int engine,
-					bool enabled)
+static void __ingenic_adc_enable(struct ingenic_adc *adc, int engine,
+			      bool enabled)
 {
 	u8 val;
 
@@ -201,9 +198,8 @@ static void ingenic_adc_enable(struct ingenic_adc *adc,
 			       int engine,
 			       bool enabled)
 {
-	mutex_lock(&adc->lock);
-	ingenic_adc_enable_unlocked(adc, engine, enabled);
-	mutex_unlock(&adc->lock);
+	guard(mutex)(&adc->lock);
+	__ingenic_adc_enable(adc, engine, enabled);
 }
 
 static int ingenic_adc_capture(struct ingenic_adc *adc,
@@ -218,18 +214,17 @@ static int ingenic_adc_capture(struct ingenic_adc *adc,
 	 * probably due to the switch of VREF. We must keep the lock here to
 	 * avoid races with the buffer enable/disable functions.
 	 */
-	mutex_lock(&adc->lock);
+	guard(mutex)(&adc->lock);
 	cfg = readl(adc->base + JZ_ADC_REG_CFG);
 	writel(cfg & ~JZ_ADC_REG_CFG_CMD_SEL, adc->base + JZ_ADC_REG_CFG);
 
-	ingenic_adc_enable_unlocked(adc, engine, true);
+	__ingenic_adc_enable(adc, engine, true);
 	ret = readb_poll_timeout(adc->base + JZ_ADC_REG_ENABLE, val,
 				 !(val & BIT(engine)), 250, 1000);
 	if (ret)
-		ingenic_adc_enable_unlocked(adc, engine, false);
+		__ingenic_adc_enable(adc, engine, false);
 
 	writel(cfg, adc->base + JZ_ADC_REG_CFG);
-	mutex_unlock(&adc->lock);
 
 	return ret;
 }
@@ -628,22 +623,12 @@ static int ingenic_adc_read_avail(struct iio_dev *iio_dev,
 	}
 }
 
-static int ingenic_adc_read_chan_info_raw(struct iio_dev *iio_dev,
-					  struct iio_chan_spec const *chan,
-					  int *val)
+static int __ingenic_adc_read_chan(struct ingenic_adc *adc,
+				   struct iio_chan_spec const *chan,
+				   int *val)
 {
 	int cmd, ret, engine = (chan->channel == INGENIC_ADC_BATTERY);
-	struct ingenic_adc *adc = iio_priv(iio_dev);
-
-	ret = clk_enable(adc->clk);
-	if (ret) {
-		dev_err(iio_dev->dev.parent, "Failed to enable clock: %d\n",
-			ret);
-		return ret;
-	}
 
-	/* We cannot sample the aux channels in parallel. */
-	mutex_lock(&adc->aux_lock);
 	if (adc->soc_data->has_aux_md && engine == 0) {
 		switch (chan->channel) {
 		case INGENIC_ADC_AUX0:
@@ -662,7 +647,7 @@ static int ingenic_adc_read_chan_info_raw(struct iio_dev *iio_dev,
 
 	ret = ingenic_adc_capture(adc, engine);
 	if (ret)
-		goto out;
+		return ret;
 
 	switch (chan->channel) {
 	case INGENIC_ADC_AUX0:
@@ -675,9 +660,26 @@ static int ingenic_adc_read_chan_info_raw(struct iio_dev *iio_dev,
 		break;
 	}
 
-	ret = IIO_VAL_INT;
-out:
-	mutex_unlock(&adc->aux_lock);
+	return IIO_VAL_INT;
+}
+
+static int ingenic_adc_read_chan_info_raw(struct iio_dev *iio_dev,
+					  struct iio_chan_spec const *chan,
+					  int *val)
+{
+	struct ingenic_adc *adc = iio_priv(iio_dev);
+	int ret;
+
+	ret = clk_enable(adc->clk);
+	if (ret) {
+		dev_err(iio_dev->dev.parent, "Failed to enable clock: %d\n", ret);
+		return ret;
+	}
+
+	/* We cannot sample the aux channels in parallel. */
+	scoped_guard(mutex, &adc->aux_lock)
+		ret = __ingenic_adc_read_chan(adc, chan, val);
+
 	clk_disable(adc->clk);
 
 	return ret;
diff --git a/drivers/iio/adc/intel_dc_ti_adc.c b/drivers/iio/adc/intel_dc_ti_adc.c
index 0fe34f1c338e8..b5afad713e2d0 100644
--- a/drivers/iio/adc/intel_dc_ti_adc.c
+++ b/drivers/iio/adc/intel_dc_ti_adc.c
@@ -290,8 +290,8 @@ static int dc_ti_adc_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	info->vbat_zse = sign_extend32(FIELD_GET(DC_TI_VBAT_ZSE, val), 3);
-	info->vbat_ge = sign_extend32(FIELD_GET(DC_TI_VBAT_GE, val), 3);
+	info->vbat_zse = FIELD_GET_SIGNED(DC_TI_VBAT_ZSE, val);
+	info->vbat_ge = FIELD_GET_SIGNED(DC_TI_VBAT_GE, val);
 
 	dev_dbg(dev, "vbat-zse %d vbat-ge %d\n", info->vbat_zse, info->vbat_ge);
 
diff --git a/drivers/iio/adc/ltc2309.c b/drivers/iio/adc/ltc2309.c
index 316256edf150a..87b78d0353f19 100644
--- a/drivers/iio/adc/ltc2309.c
+++ b/drivers/iio/adc/ltc2309.c
@@ -9,7 +9,9 @@
  *
  * Copyright (c) 2023, Liam Beguin <liambeguin@gmail.com>
  */
+#include <linux/array_size.h>
 #include <linux/bitfield.h>
+#include <linux/delay.h>
 #include <linux/i2c.h>
 #include <linux/iio/iio.h>
 #include <linux/kernel.h>
@@ -34,12 +36,14 @@
  * @client:	I2C reference
  * @lock:	Lock to serialize data access
  * @vref_mv:	Internal voltage reference
+ * @read_delay_us:	Chip-specific read delay in microseconds
  */
 struct ltc2309 {
 	struct device		*dev;
 	struct i2c_client	*client;
 	struct mutex		lock; /* serialize data access */
 	int			vref_mv;
+	unsigned int		read_delay_us;
 };
 
 /* Order matches expected channel address, See datasheet Table 1. */
@@ -117,20 +121,22 @@ static const struct iio_chan_spec ltc2309_channels[] = {
 
 struct ltc2309_chip_info {
 	const char *name;
-	const struct iio_chan_spec *channels;
+	unsigned int read_delay_us;
 	int num_channels;
+	const struct iio_chan_spec *channels __counted_by_ptr(num_channels);
 };
 
 static const struct ltc2309_chip_info ltc2305_chip_info = {
 	.name = "ltc2305",
-	.channels = ltc2305_channels,
+	.read_delay_us = 2,
 	.num_channels = ARRAY_SIZE(ltc2305_channels),
+	.channels = ltc2305_channels,
 };
 
 static const struct ltc2309_chip_info ltc2309_chip_info = {
 	.name = "ltc2309",
-	.channels = ltc2309_channels,
 	.num_channels = ARRAY_SIZE(ltc2309_channels),
+	.channels = ltc2309_channels,
 };
 
 static int ltc2309_read_raw_channel(struct ltc2309 *ltc2309,
@@ -151,6 +157,9 @@ static int ltc2309_read_raw_channel(struct ltc2309 *ltc2309,
 		return ret;
 	}
 
+	if (ltc2309->read_delay_us)
+		fsleep(ltc2309->read_delay_us);
+
 	ret = i2c_master_recv(ltc2309->client, (char *)&buf, 2);
 	if (ret < 0) {
 		dev_err(ltc2309->dev, "i2c read failed: %pe\n", ERR_PTR(ret));
@@ -206,6 +215,7 @@ static int ltc2309_probe(struct i2c_client *client)
 
 	ltc2309->dev = &indio_dev->dev;
 	ltc2309->client = client;
+	ltc2309->read_delay_us = chip_info->read_delay_us;
 
 	indio_dev->name = chip_info->name;
 	indio_dev->modes = INDIO_DIRECT_MODE;
diff --git a/drivers/iio/adc/mcp3422.c b/drivers/iio/adc/mcp3422.c
index 50834fdcf7388..0cbd7a8747985 100644
--- a/drivers/iio/adc/mcp3422.c
+++ b/drivers/iio/adc/mcp3422.c
@@ -13,6 +13,8 @@
  * voltage unit is nV.
  */
 
+#include <linux/bitfield.h>
+#include <linux/bits.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
 #include <linux/module.h>
@@ -24,10 +26,10 @@
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
-/* Masks */
-#define MCP3422_CHANNEL_MASK	0x60
-#define MCP3422_PGA_MASK	0x03
-#define MCP3422_SRATE_MASK	0x0C
+#define MCP3422_CHANNEL_MASK	GENMASK(6, 5)
+#define MCP3422_SRATE_MASK	GENMASK(3, 2)
+#define MCP3422_PGA_MASK	GENMASK(1, 0)
+
 #define MCP3422_SRATE_240	0x0
 #define MCP3422_SRATE_60	0x1
 #define MCP3422_SRATE_15	0x2
@@ -36,15 +38,7 @@
 #define MCP3422_PGA_2	1
 #define MCP3422_PGA_4	2
 #define MCP3422_PGA_8	3
-#define MCP3422_CONT_SAMPLING	0x10
-
-#define MCP3422_CHANNEL(config)	(((config) & MCP3422_CHANNEL_MASK) >> 5)
-#define MCP3422_PGA(config)	((config) & MCP3422_PGA_MASK)
-#define MCP3422_SAMPLE_RATE(config)	(((config) & MCP3422_SRATE_MASK) >> 2)
-
-#define MCP3422_CHANNEL_VALUE(value) (((value) << 5) & MCP3422_CHANNEL_MASK)
-#define MCP3422_PGA_VALUE(value) ((value) & MCP3422_PGA_MASK)
-#define MCP3422_SAMPLE_RATE_VALUE(value) ((value << 2) & MCP3422_SRATE_MASK)
+#define MCP3422_CONT_SAMPLING	BIT(4)
 
 #define MCP3422_CHAN(_index) \
 	{ \
@@ -108,7 +102,7 @@ static int mcp3422_update_config(struct mcp3422 *adc, u8 newconfig)
 static int mcp3422_read(struct mcp3422 *adc, int *value, u8 *config)
 {
 	int ret = 0;
-	u8 sample_rate = MCP3422_SAMPLE_RATE(adc->config);
+	u8 sample_rate = FIELD_GET(MCP3422_SRATE_MASK, adc->config);
 	u8 buf[4] = {0, 0, 0, 0};
 	u32 temp;
 
@@ -136,18 +130,18 @@ static int mcp3422_read_channel(struct mcp3422 *adc,
 
 	mutex_lock(&adc->lock);
 
-	if (req_channel != MCP3422_CHANNEL(adc->config)) {
+	if (req_channel != FIELD_GET(MCP3422_CHANNEL_MASK, adc->config)) {
 		config = adc->config;
-		config &= ~MCP3422_CHANNEL_MASK;
-		config |= MCP3422_CHANNEL_VALUE(req_channel);
-		config &= ~MCP3422_PGA_MASK;
-		config |= MCP3422_PGA_VALUE(adc->pga[req_channel]);
+
+		FIELD_MODIFY(MCP3422_CHANNEL_MASK, &config, req_channel);
+		FIELD_MODIFY(MCP3422_PGA_MASK, &config, adc->pga[req_channel]);
+
 		ret = mcp3422_update_config(adc, config);
 		if (ret < 0) {
 			mutex_unlock(&adc->lock);
 			return ret;
 		}
-		msleep(mcp3422_read_times[MCP3422_SAMPLE_RATE(adc->config)]);
+		msleep(mcp3422_read_times[FIELD_GET(MCP3422_SRATE_MASK, adc->config)]);
 	}
 
 	ret = mcp3422_read(adc, value, &config);
@@ -163,9 +157,8 @@ static int mcp3422_read_raw(struct iio_dev *iio,
 {
 	struct mcp3422 *adc = iio_priv(iio);
 	int err;
-
-	u8 sample_rate = MCP3422_SAMPLE_RATE(adc->config);
-	u8 pga		 = MCP3422_PGA(adc->config);
+	u8 sample_rate = FIELD_GET(MCP3422_SRATE_MASK, adc->config);
+	u8 pga = FIELD_GET(MCP3422_PGA_MASK, adc->config);
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
@@ -181,7 +174,7 @@ static int mcp3422_read_raw(struct iio_dev *iio,
 		return IIO_VAL_INT_PLUS_NANO;
 
 	case IIO_CHAN_INFO_SAMP_FREQ:
-		*val1 = mcp3422_sample_rates[MCP3422_SAMPLE_RATE(adc->config)];
+		*val1 = mcp3422_sample_rates[FIELD_GET(MCP3422_SRATE_MASK, adc->config)];
 		return IIO_VAL_INT;
 
 	default:
@@ -199,7 +192,7 @@ static int mcp3422_write_raw(struct iio_dev *iio,
 	u8 temp;
 	u8 config = adc->config;
 	u8 req_channel = channel->channel;
-	u8 sample_rate = MCP3422_SAMPLE_RATE(config);
+	u8 sample_rate = FIELD_GET(MCP3422_SRATE_MASK, config);
 	u8 i;
 
 	switch (mask) {
@@ -211,10 +204,8 @@ static int mcp3422_write_raw(struct iio_dev *iio,
 			if (val2 == mcp3422_scales[sample_rate][i]) {
 				adc->pga[req_channel] = i;
 
-				config &= ~MCP3422_CHANNEL_MASK;
-				config |= MCP3422_CHANNEL_VALUE(req_channel);
-				config &= ~MCP3422_PGA_MASK;
-				config |= MCP3422_PGA_VALUE(adc->pga[req_channel]);
+				FIELD_MODIFY(MCP3422_CHANNEL_MASK, &config, req_channel);
+				FIELD_MODIFY(MCP3422_PGA_MASK, &config, adc->pga[req_channel]);
 
 				return mcp3422_update_config(adc, config);
 			}
@@ -241,10 +232,8 @@ static int mcp3422_write_raw(struct iio_dev *iio,
 			return -EINVAL;
 		}
 
-		config &= ~MCP3422_CHANNEL_MASK;
-		config |= MCP3422_CHANNEL_VALUE(req_channel);
-		config &= ~MCP3422_SRATE_MASK;
-		config |= MCP3422_SAMPLE_RATE_VALUE(temp);
+		FIELD_MODIFY(MCP3422_CHANNEL_MASK, &config, req_channel);
+		FIELD_MODIFY(MCP3422_SRATE_MASK, &config, temp);
 
 		return mcp3422_update_config(adc, config);
 
@@ -283,7 +272,7 @@ static ssize_t mcp3422_show_scales(struct device *dev,
 		struct device_attribute *attr, char *buf)
 {
 	struct mcp3422 *adc = iio_priv(dev_to_iio_dev(dev));
-	u8 sample_rate = MCP3422_SAMPLE_RATE(adc->config);
+	u8 sample_rate = FIELD_GET(MCP3422_SRATE_MASK, adc->config);
 
 	return sprintf(buf, "0.%09u 0.%09u 0.%09u 0.%09u\n",
 		mcp3422_scales[sample_rate][0],
@@ -376,10 +365,10 @@ static int mcp3422_probe(struct i2c_client *client)
 	}
 
 	/* meaningful default configuration */
-	config = (MCP3422_CONT_SAMPLING
-		| MCP3422_CHANNEL_VALUE(0)
-		| MCP3422_PGA_VALUE(MCP3422_PGA_1)
-		| MCP3422_SAMPLE_RATE_VALUE(MCP3422_SRATE_240));
+	config = MCP3422_CONT_SAMPLING |
+		 FIELD_PREP(MCP3422_CHANNEL_MASK, 0) |
+		 FIELD_PREP(MCP3422_PGA_MASK, MCP3422_PGA_1) |
+		 FIELD_PREP(MCP3422_SRATE_MASK, MCP3422_SRATE_240);
 	err = mcp3422_update_config(adc, config);
 	if (err < 0)
 		return err;
diff --git a/drivers/iio/adc/meson_saradc.c b/drivers/iio/adc/meson_saradc.c
index 23991a3612bdc..000e39ca5c624 100644
--- a/drivers/iio/adc/meson_saradc.c
+++ b/drivers/iio/adc/meson_saradc.c
@@ -817,9 +817,11 @@ static int meson_sar_adc_temp_sensor_init(struct iio_dev *indio_dev)
 	}
 
 	priv->tsc_regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "amlogic,hhi-sysctrl");
-	if (IS_ERR(priv->tsc_regmap))
+	if (IS_ERR(priv->tsc_regmap)) {
+		kfree(buf);
 		return dev_err_probe(dev, PTR_ERR(priv->tsc_regmap),
 				     "failed to get amlogic,hhi-sysctrl regmap\n");
+	}
 
 	trimming_bits = priv->param->temperature_trimming_bits;
 	trimming_mask = BIT(trimming_bits) - 1;
diff --git a/drivers/iio/adc/mt6359-auxadc.c b/drivers/iio/adc/mt6359-auxadc.c
index 6b9ed9b1fde2e..1d9724ef09838 100644
--- a/drivers/iio/adc/mt6359-auxadc.c
+++ b/drivers/iio/adc/mt6359-auxadc.c
@@ -497,6 +497,7 @@ static int mt6358_read_imp(struct mt6359_auxadc *adc_dev,
 		return ret;
 
 	/* Read the params before stopping */
+	val_v = 0;
 	regmap_read(regmap, reg_adc0 + (cinfo->imp_adc_num << 1), &val_v);
 
 	mt6358_stop_imp_conv(adc_dev);
diff --git a/drivers/iio/adc/npcm_adc.c b/drivers/iio/adc/npcm_adc.c
index ddabb9600d462..61c8b825bda12 100644
--- a/drivers/iio/adc/npcm_adc.c
+++ b/drivers/iio/adc/npcm_adc.c
@@ -231,7 +231,7 @@ static int npcm_adc_probe(struct platform_device *pdev)
 	if (IS_ERR(info->reset))
 		return PTR_ERR(info->reset);
 
-	info->adc_clk = devm_clk_get(&pdev->dev, NULL);
+	info->adc_clk = devm_clk_get_enabled(&pdev->dev, NULL);
 	if (IS_ERR(info->adc_clk)) {
 		dev_warn(&pdev->dev, "ADC clock failed: can't read clk\n");
 		return PTR_ERR(info->adc_clk);
@@ -244,17 +244,13 @@ static int npcm_adc_probe(struct platform_device *pdev)
 	info->adc_sample_hz = clk_get_rate(info->adc_clk) / ((div + 1) * 2);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = irq;
-		goto err_disable_clk;
-	}
+	if (irq < 0)
+		return irq;
 
 	ret = devm_request_irq(&pdev->dev, irq, npcm_adc_isr, 0,
 			       "NPCM_ADC", indio_dev);
-	if (ret < 0) {
-		dev_err(dev, "failed requesting interrupt\n");
-		goto err_disable_clk;
-	}
+	if (ret < 0)
+		return ret;
 
 	reg_con = ioread32(info->regs + NPCM_ADCCON);
 	info->vref = devm_regulator_get_optional(&pdev->dev, "vref");
@@ -262,7 +258,7 @@ static int npcm_adc_probe(struct platform_device *pdev)
 		ret = regulator_enable(info->vref);
 		if (ret) {
 			dev_err(&pdev->dev, "Can't enable ADC reference voltage\n");
-			goto err_disable_clk;
+			return ret;
 		}
 
 		iowrite32(reg_con & ~NPCM_ADCCON_REFSEL,
@@ -272,10 +268,8 @@ static int npcm_adc_probe(struct platform_device *pdev)
 		 * Any error which is not ENODEV indicates the regulator
 		 * has been specified and so is a failure case.
 		 */
-		if (PTR_ERR(info->vref) != -ENODEV) {
-			ret = PTR_ERR(info->vref);
-			goto err_disable_clk;
-		}
+		if (PTR_ERR(info->vref) != -ENODEV)
+			return PTR_ERR(info->vref);
 
 		/* Use internal reference */
 		iowrite32(reg_con | NPCM_ADCCON_REFSEL,
@@ -314,8 +308,6 @@ static int npcm_adc_probe(struct platform_device *pdev)
 	iowrite32(reg_con & ~NPCM_ADCCON_ADC_EN, info->regs + NPCM_ADCCON);
 	if (!IS_ERR(info->vref))
 		regulator_disable(info->vref);
-err_disable_clk:
-	clk_disable_unprepare(info->adc_clk);
 
 	return ret;
 }
@@ -332,7 +324,6 @@ static void npcm_adc_remove(struct platform_device *pdev)
 	iowrite32(regtemp & ~NPCM_ADCCON_ADC_EN, info->regs + NPCM_ADCCON);
 	if (!IS_ERR(info->vref))
 		regulator_disable(info->vref);
-	clk_disable_unprepare(info->adc_clk);
 }
 
 static struct platform_driver npcm_adc_driver = {
diff --git a/drivers/iio/adc/nxp-sar-adc.c b/drivers/iio/adc/nxp-sar-adc.c
index 9d9f2c76bed4f..01f3da56e9876 100644
--- a/drivers/iio/adc/nxp-sar-adc.c
+++ b/drivers/iio/adc/nxp-sar-adc.c
@@ -198,6 +198,15 @@ static void nxp_sar_adc_irq_cfg(struct nxp_sar_adc *info, bool enable)
 		writel(0, NXP_SAR_ADC_IMR(info->regs));
 }
 
+static void nxp_sar_adc_wait_for(struct nxp_sar_adc *info, unsigned int cycles)
+{
+	u64 rate;
+
+	rate = clk_get_rate(info->clk);
+	if (rate)
+		ndelay(div64_u64(NSEC_PER_SEC, rate * cycles));
+}
+
 static bool nxp_sar_adc_set_enabled(struct nxp_sar_adc *info, bool enable)
 {
 	u32 mcr;
@@ -221,7 +230,7 @@ static bool nxp_sar_adc_set_enabled(struct nxp_sar_adc *info, bool enable)
 	 * configuration of NCMR and the setting of NSTART.
 	 */
 	if (enable)
-		ndelay(div64_u64(NSEC_PER_SEC, clk_get_rate(info->clk) * 3));
+		nxp_sar_adc_wait_for(info, 3);
 
 	return pwdn;
 }
@@ -317,11 +326,7 @@ static int nxp_sar_adc_read_data(struct nxp_sar_adc *info, unsigned int chan)
 
 	ceocfr = readl(NXP_SAR_ADC_CEOCFR0(info->regs));
 
-	/*
-	 * FIELD_GET() can not be used here because EOC_CH is not constant.
-	 * TODO: Switch to field_get() when it will be available.
-	 */
-	if (!(NXP_SAR_ADC_EOC_CH(chan) & ceocfr))
+	if (!field_get(NXP_SAR_ADC_EOC_CH(chan), ceocfr))
 		return -EIO;
 
 	cdr = readl(NXP_SAR_ADC_CDR(info->regs, chan));
@@ -469,7 +474,7 @@ static void nxp_sar_adc_stop_conversion(struct nxp_sar_adc *info)
 	 * only when the capture finishes. The delay will be very
 	 * short, usec-ish, which is acceptable in the atomic context.
 	 */
-	ndelay(div64_u64(NSEC_PER_SEC, clk_get_rate(info->clk)) * 80);
+	nxp_sar_adc_wait_for(info, 80);
 }
 
 static int nxp_sar_adc_start_conversion(struct nxp_sar_adc *info, bool raw)
@@ -560,6 +565,9 @@ static int nxp_sar_adc_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec
 
 	switch (mask) {
 	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (val <= 0)
+			return -EINVAL;
+
 		/*
 		 * Configures the sample period duration in terms of the SAR
 		 * controller clock. The minimum acceptable value is 8.
@@ -568,7 +576,11 @@ static int nxp_sar_adc_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec
 		 * sampling timing which gives us the number of cycles expected.
 		 * The value is 8-bit wide, consequently the max value is 0xFF.
 		 */
-		inpsamp = clk_get_rate(info->clk) / val - NXP_SAR_ADC_CONV_TIME;
+		inpsamp = clk_get_rate(info->clk) / val;
+		if (inpsamp < NXP_SAR_ADC_CONV_TIME)
+			return -EINVAL;
+
+		inpsamp -= NXP_SAR_ADC_CONV_TIME;
 		nxp_sar_adc_conversion_timing_set(info, inpsamp);
 		return 0;
 
@@ -660,7 +672,7 @@ static void nxp_sar_adc_dma_cb(void *data)
 static int nxp_sar_adc_start_cyclic_dma(struct iio_dev *indio_dev)
 {
 	struct nxp_sar_adc *info = iio_priv(indio_dev);
-	struct dma_slave_config config;
+	struct dma_slave_config config = { };
 	struct dma_async_tx_descriptor *desc;
 	int ret;
 
diff --git a/drivers/iio/adc/pac1921.c b/drivers/iio/adc/pac1921.c
index bce7185953ecb..68bdd2f30bada 100644
--- a/drivers/iio/adc/pac1921.c
+++ b/drivers/iio/adc/pac1921.c
@@ -1310,7 +1310,7 @@ static int pac1921_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id pac1921_id[] = {
-	{ .name = "pac1921", 0 },
+	{ .name = "pac1921" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pac1921_id);
diff --git a/drivers/iio/adc/qcom-pm8xxx-xoadc.c b/drivers/iio/adc/qcom-pm8xxx-xoadc.c
index 31f88cf7f7f18..4a1a0cfb4699d 100644
--- a/drivers/iio/adc/qcom-pm8xxx-xoadc.c
+++ b/drivers/iio/adc/qcom-pm8xxx-xoadc.c
@@ -369,7 +369,7 @@ static const struct xoadc_channel pm8921_xoadc_channels[] = {
 
 /**
  * struct pm8xxx_chan_info - ADC channel information
- * @name: name of this channel
+ * @label: label of this channel from device tree (defaults to datasheet name if not specified)
  * @hwchan: pointer to hardware channel information (muxing & scaling settings)
  * @calibration: whether to use absolute or ratiometric calibration
  * @decimation: 0,1,2,3
@@ -377,7 +377,7 @@ static const struct xoadc_channel pm8921_xoadc_channels[] = {
  * calibration: 0, 1, 2, 4, 5.
  */
 struct pm8xxx_chan_info {
-	const char *name;
+	const char *label;
 	const struct xoadc_channel *hwchan;
 	enum vadc_calibration calibration;
 	u8 decimation:2;
@@ -446,7 +446,7 @@ static int pm8xxx_read_channel_rsv(struct pm8xxx_xoadc *adc,
 	u8 lsb, msb;
 
 	dev_dbg(adc->dev, "read channel \"%s\", amux %d, prescale/mux: %d, rsv %d\n",
-		ch->name, ch->hwchan->amux_channel, ch->hwchan->pre_scale_mux, rsv);
+		ch->label, ch->hwchan->amux_channel, ch->hwchan->pre_scale_mux, rsv);
 
 	mutex_lock(&adc->lock);
 
@@ -535,10 +535,7 @@ static int pm8xxx_read_channel_rsv(struct pm8xxx_xoadc *adc,
 		goto unlock;
 
 	/* Next the interrupt occurs */
-	ret = wait_for_completion_timeout(&adc->complete,
-					  VADC_CONV_TIME_MAX_US);
-	if (!ret) {
-		dev_err(adc->dev, "conversion timed out\n");
+	if (!wait_for_completion_timeout(&adc->complete, VADC_CONV_TIME_MAX_US)) {
 		ret = -ETIMEDOUT;
 		goto unlock;
 	}
@@ -657,11 +654,8 @@ static int pm8xxx_read_raw(struct iio_dev *indio_dev,
 	switch (mask) {
 	case IIO_CHAN_INFO_PROCESSED:
 		ch = pm8xxx_get_channel(adc, chan->address);
-		if (!ch) {
-			dev_err(adc->dev, "no such channel %lu\n",
-				chan->address);
+		if (!ch)
 			return -EINVAL;
-		}
 		ret = pm8xxx_read_channel(adc, ch, &adc_code);
 		if (ret)
 			return ret;
@@ -677,11 +671,8 @@ static int pm8xxx_read_raw(struct iio_dev *indio_dev,
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_RAW:
 		ch = pm8xxx_get_channel(adc, chan->address);
-		if (!ch) {
-			dev_err(adc->dev, "no such channel %lu\n",
-				chan->address);
+		if (!ch)
 			return -EINVAL;
-		}
 		ret = pm8xxx_read_channel(adc, ch, &adc_code);
 		if (ret)
 			return ret;
@@ -725,8 +716,21 @@ static int pm8xxx_fwnode_xlate(struct iio_dev *indio_dev,
 	return -EINVAL;
 }
 
+static int pm8xxx_read_label(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan, char *label)
+{
+	struct pm8xxx_xoadc *adc = iio_priv(indio_dev);
+	const struct pm8xxx_chan_info *ch;
+
+	ch = pm8xxx_get_channel(adc, chan->address);
+	if (!ch)
+		return -EINVAL;
+	return sysfs_emit(label, "%s\n", ch->label);
+}
+
 static const struct iio_info pm8xxx_xoadc_info = {
 	.fwnode_xlate = pm8xxx_fwnode_xlate,
+	.read_label = pm8xxx_read_label,
 	.read_raw = pm8xxx_read_raw,
 };
 
@@ -770,7 +774,8 @@ static int pm8xxx_xoadc_parse_channel(struct device *dev,
 			pre_scale_mux, amux_channel);
 		return -EINVAL;
 	}
-	ch->name = name;
+	ch->label = hwchan->datasheet_name;
+	fwnode_property_read_string(fwnode, "label", &ch->label);
 	ch->hwchan = hwchan;
 	/* Everyone seems to use absolute calibration except in special cases */
 	ch->calibration = VADC_CALIB_ABSOLUTE;
@@ -812,7 +817,7 @@ static int pm8xxx_xoadc_parse_channel(struct device *dev,
 
 	dev_dbg(dev,
 		"channel [PRESCALE/MUX: %02x AMUX: %02x] \"%s\" ref voltage: %d, decimation %d prescale %d/%d, scale function %d\n",
-		hwchan->pre_scale_mux, hwchan->amux_channel, ch->name,
+		hwchan->pre_scale_mux, hwchan->amux_channel, ch->label,
 		ch->amux_ip_rsv, ch->decimation, hwchan->prescale.numerator,
 		hwchan->prescale.denominator, hwchan->scale_fn_type);
 
diff --git a/drivers/iio/adc/qcom-spmi-adc5-gen3.c b/drivers/iio/adc/qcom-spmi-adc5-gen3.c
index f8168a14b9073..48c793b18d11e 100644
--- a/drivers/iio/adc/qcom-spmi-adc5-gen3.c
+++ b/drivers/iio/adc/qcom-spmi-adc5-gen3.c
@@ -482,7 +482,7 @@ static int adc5_gen3_get_fw_channel_data(struct adc5_chip *adc,
 	sid = FIELD_GET(ADC5_GEN3_VIRTUAL_SID_MASK, chan);
 	chan = FIELD_GET(ADC5_GEN3_CHANNEL_MASK, chan);
 
-	if (chan > ADC5_MAX_CHANNEL)
+	if (chan >= ADC5_MAX_CHANNEL)
 		return dev_err_probe(dev, -EINVAL,
 				     "%s invalid channel number %d\n",
 				     name, chan);
diff --git a/drivers/iio/adc/rtq6056.c b/drivers/iio/adc/rtq6056.c
index 2bf3a09ac6b04..e2b1da13c0d31 100644
--- a/drivers/iio/adc/rtq6056.c
+++ b/drivers/iio/adc/rtq6056.c
@@ -728,7 +728,7 @@ static int rtq6056_probe(struct i2c_client *i2c)
 	if (!i2c_check_functionality(i2c->adapter, I2C_FUNC_SMBUS_WORD_DATA))
 		return -EOPNOTSUPP;
 
-	devdata = device_get_match_data(dev);
+	devdata = i2c_get_match_data(i2c);
 	if (!devdata)
 		return dev_err_probe(dev, -EINVAL, "Invalid dev data\n");
 
@@ -871,6 +871,13 @@ static const struct richtek_dev_data rtq6059_devdata = {
 	.set_average = rtq6059_adc_set_average,
 };
 
+static const struct i2c_device_id rtq6056_id[] = {
+	{ "rtq6056", (kernel_ulong_t)&rtq6056_devdata },
+	{ "rtq6059", (kernel_ulong_t)&rtq6059_devdata },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, rtq6056_id);
+
 static const struct of_device_id rtq6056_device_match[] = {
 	{ .compatible = "richtek,rtq6056", .data = &rtq6056_devdata },
 	{ .compatible = "richtek,rtq6059", .data = &rtq6059_devdata },
@@ -885,6 +892,7 @@ static struct i2c_driver rtq6056_driver = {
 		.pm = pm_ptr(&rtq6056_pm_ops),
 	},
 	.probe = rtq6056_probe,
+	.id_table = rtq6056_id,
 };
 module_i2c_driver(rtq6056_driver);
 
diff --git a/drivers/iio/adc/sun20i-gpadc-iio.c b/drivers/iio/adc/sun20i-gpadc-iio.c
index 861c14da75add..81fc4610e15e6 100644
--- a/drivers/iio/adc/sun20i-gpadc-iio.c
+++ b/drivers/iio/adc/sun20i-gpadc-iio.c
@@ -139,8 +139,23 @@ static irqreturn_t sun20i_gpadc_irq_handler(int irq, void *data)
 	return IRQ_HANDLED;
 }
 
+static int
+sun20i_gpadc_fwnode_xlate(struct iio_dev *indio_dev,
+			  const struct fwnode_reference_args *iiospec)
+{
+	if (iiospec->nargs != 1)
+		return -EINVAL;
+
+	for (unsigned int i = 0; i < indio_dev->num_channels; i++)
+		if (indio_dev->channels[i].channel == iiospec->args[0])
+			return i;
+
+	return -EINVAL;
+}
+
 static const struct iio_info sun20i_gpadc_iio_info = {
 	.read_raw = sun20i_gpadc_read_raw,
+	.fwnode_xlate = sun20i_gpadc_fwnode_xlate,
 };
 
 static void sun20i_gpadc_reset_assert(void *data)
@@ -177,10 +192,10 @@ static int sun20i_gpadc_alloc_channels(struct iio_dev *indio_dev,
 static int sun20i_gpadc_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct iio_dev *indio_dev;
 	struct sun20i_gpadc_iio *info;
+	struct clk_bulk_data *clks;
 	struct reset_control *rst;
-	struct clk *clk;
+	struct iio_dev *indio_dev;
 	int irq;
 	int ret;
 
@@ -205,9 +220,11 @@ static int sun20i_gpadc_probe(struct platform_device *pdev)
 	if (IS_ERR(info->regs))
 		return PTR_ERR(info->regs);
 
-	clk = devm_clk_get_enabled(dev, NULL);
-	if (IS_ERR(clk))
-		return dev_err_probe(dev, PTR_ERR(clk), "failed to enable bus clock\n");
+	ret = devm_clk_bulk_get_all_enabled(dev, &clks);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "failed to enable clocks\n");
+	if (ret == 0)
+		return dev_err_probe(dev, -ENODEV, "needs at least one clocks\n");
 
 	rst = devm_reset_control_get_exclusive(dev, NULL);
 	if (IS_ERR(rst))
@@ -243,6 +260,7 @@ static int sun20i_gpadc_probe(struct platform_device *pdev)
 
 static const struct of_device_id sun20i_gpadc_of_id[] = {
 	{ .compatible = "allwinner,sun20i-d1-gpadc" },
+	{ .compatible = "allwinner,sun55i-a523-gpadc" },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, sun20i_gpadc_of_id);
diff --git a/drivers/iio/adc/ti-ads1298.c b/drivers/iio/adc/ti-ads1298.c
index ae30b47e45142..9e47ace16acd0 100644
--- a/drivers/iio/adc/ti-ads1298.c
+++ b/drivers/iio/adc/ti-ads1298.c
@@ -66,7 +66,7 @@
 #define ADS1298_MASK_CONFIG3_VREF_4V		BIT(5)
 
 #define ADS1298_REG_LOFF	0x04
-#define ADS1298_REG_CHnSET(n)	(0x05 + n)
+#define ADS1298_REG_CHnSET(n)	(0x05 + (n))
 #define ADS1298_MASK_CH_PD		BIT(7)
 #define ADS1298_MASK_CH_PGA		GENMASK(6, 4)
 #define ADS1298_MASK_CH_MUX		GENMASK(2, 0)
@@ -210,9 +210,11 @@ static int ads1298_read_one(struct ads1298_private *priv, int chan_index)
 		return ret;
 	}
 
-	/* Cannot take longer than 40ms (250Hz) */
-	ret = wait_for_completion_timeout(&priv->completion, msecs_to_jiffies(50));
-	if (!ret)
+	/*
+	 * One conversion takes at most 4ms at the lowest sample rate (250Hz).
+	 * Use 50ms to allow for kernel scheduling latency.
+	 */
+	if (!wait_for_completion_timeout(&priv->completion, msecs_to_jiffies(50)))
 		return -ETIMEDOUT;
 
 	return 0;
@@ -279,6 +281,7 @@ static const u8 ads1298_pga_settings[] = { 6, 1, 2, 3, 4, 8, 12 };
 static int ads1298_get_scale(struct ads1298_private *priv,
 			     int channel, int *val, int *val2)
 {
+	unsigned int pga_idx;
 	int ret;
 	unsigned int regval;
 	u8 gain;
@@ -302,7 +305,11 @@ static int ads1298_get_scale(struct ads1298_private *priv,
 	if (ret)
 		return ret;
 
-	gain = ads1298_pga_settings[FIELD_GET(ADS1298_MASK_CH_PGA, regval)];
+	pga_idx = FIELD_GET(ADS1298_MASK_CH_PGA, regval);
+	if (pga_idx >= ARRAY_SIZE(ads1298_pga_settings))
+		return -EINVAL;
+
+	gain = ads1298_pga_settings[pga_idx];
 	*val /= gain; /* Full scale is VREF / gain */
 
 	*val2 = ADS1298_BITS_PER_SAMPLE - 1; /* Signed, hence the -1 */
@@ -615,15 +622,6 @@ static int ads1298_init(struct iio_dev *indio_dev)
 	if (!indio_dev->name)
 		return -ENOMEM;
 
-	/* Enable internal test signal, double amplitude, double frequency */
-	ret = regmap_write(priv->regmap, ADS1298_REG_CONFIG2,
-			   ADS1298_MASK_CONFIG2_RESERVED |
-			   ADS1298_MASK_CONFIG2_INT_TEST |
-			   ADS1298_MASK_CONFIG2_TEST_AMP |
-			   ADS1298_MASK_CONFIG2_TEST_FREQ_FAST);
-	if (ret)
-		return ret;
-
 	val = ADS1298_MASK_CONFIG3_RESERVED; /* Must write 1 always */
 	if (!priv->reg_vref) {
 		/* Enable internal reference */
diff --git a/drivers/iio/adc/ti-ads7924.c b/drivers/iio/adc/ti-ads7924.c
index bbcc4fc22b6e3..5f294595a415f 100644
--- a/drivers/iio/adc/ti-ads7924.c
+++ b/drivers/iio/adc/ti-ads7924.c
@@ -12,6 +12,7 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/gpio/consumer.h>
 #include <linux/init.h>
@@ -198,6 +199,8 @@ static int ads7924_get_adc_result(struct ads7924_data *data,
 	if (chan->channel < 0 || chan->channel >= ADS7924_CHANNELS)
 		return -EINVAL;
 
+	guard(mutex)(&data->lock);
+
 	if (data->conv_invalid) {
 		int conv_time;
 
@@ -227,9 +230,7 @@ static int ads7924_read_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		mutex_lock(&data->lock);
 		ret = ads7924_get_adc_result(data, chan, val);
-		mutex_unlock(&data->lock);
 		if (ret < 0)
 			return ret;
 
diff --git a/drivers/iio/adc/ti-ads7950.c b/drivers/iio/adc/ti-ads7950.c
index 028acd42741f3..39a074bce6d55 100644
--- a/drivers/iio/adc/ti-ads7950.c
+++ b/drivers/iio/adc/ti-ads7950.c
@@ -267,31 +267,6 @@ static const struct ti_ads7950_chip_info ti_ads7961_chip_info = {
 	.num_channels	= ARRAY_SIZE(ti_ads7961_channels),
 };
 
-/*
- * ti_ads7950_update_scan_mode() setup the spi transfer buffer for the new
- * scan mask
- */
-static int ti_ads7950_update_scan_mode(struct iio_dev *indio_dev,
-				       const unsigned long *active_scan_mask)
-{
-	struct ti_ads7950_state *st = iio_priv(indio_dev);
-	int i, cmd, len;
-
-	len = 0;
-	for_each_set_bit(i, active_scan_mask, indio_dev->num_channels) {
-		cmd = TI_ADS7950_MAN_CMD(TI_ADS7950_CR_CHAN(i));
-		st->tx_buf[len++] = cmd;
-	}
-
-	/* Data for the 1st channel is not returned until the 3rd transfer */
-	st->tx_buf[len++] = 0;
-	st->tx_buf[len++] = 0;
-
-	st->ring_xfer.len = len * 2;
-
-	return 0;
-}
-
 static irqreturn_t ti_ads7950_trigger_handler(int irq, void *p)
 {
 	struct iio_poll_func *pf = p;
@@ -299,18 +274,19 @@ static irqreturn_t ti_ads7950_trigger_handler(int irq, void *p)
 	struct ti_ads7950_state *st = iio_priv(indio_dev);
 	int ret;
 
-	mutex_lock(&st->slock);
-	ret = spi_sync(st->spi, &st->ring_msg);
-	if (ret < 0)
-		goto out;
+	do {
+		guard(mutex)(&st->slock);
 
-	iio_push_to_buffers_with_ts_unaligned(indio_dev, &st->rx_buf[2],
-					      sizeof(*st->rx_buf) *
-					      TI_ADS7950_MAX_CHAN,
-					      iio_get_time_ns(indio_dev));
+		ret = spi_sync(st->spi, &st->ring_msg);
+		if (ret)
+			break;
+
+		iio_push_to_buffers_with_ts_unaligned(indio_dev, &st->rx_buf[2],
+						      sizeof(*st->rx_buf) *
+						      TI_ADS7950_MAX_CHAN,
+						      iio_get_time_ns(indio_dev));
+	} while (0);
 
-out:
-	mutex_unlock(&st->slock);
 	iio_trigger_notify_done(indio_dev->trig);
 
 	return IRQ_HANDLED;
@@ -321,35 +297,21 @@ static int ti_ads7950_scan_direct(struct iio_dev *indio_dev, unsigned int ch)
 	struct ti_ads7950_state *st = iio_priv(indio_dev);
 	int ret, cmd;
 
-	mutex_lock(&st->slock);
+	guard(mutex)(&st->slock);
+
 	cmd = TI_ADS7950_MAN_CMD(TI_ADS7950_CR_CHAN(ch));
 	st->single_tx = cmd;
 
 	ret = spi_sync(st->spi, &st->scan_single_msg);
 	if (ret)
-		goto out;
-
-	ret = st->single_rx;
-
-out:
-	mutex_unlock(&st->slock);
+		return ret;
 
-	return ret;
+	return st->single_rx;
 }
 
-static int ti_ads7950_get_range(struct ti_ads7950_state *st)
+static unsigned int ti_ads7950_get_range(struct ti_ads7950_state *st)
 {
-	int vref;
-
-	if (st->vref_mv) {
-		vref = st->vref_mv;
-	} else {
-		vref = regulator_get_voltage(st->reg);
-		if (vref < 0)
-			return vref;
-
-		vref /= 1000;
-	}
+	unsigned int vref = st->vref_mv;
 
 	if (st->cmd_settings_bitmask & TI_ADS7950_CR_RANGE_5V)
 		vref *= 2;
@@ -378,11 +340,7 @@ static int ti_ads7950_read_raw(struct iio_dev *indio_dev,
 
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
-		ret = ti_ads7950_get_range(st);
-		if (ret < 0)
-			return ret;
-
-		*val = ret;
+		*val = ti_ads7950_get_range(st);
 		*val2 = (1 << chan->scan_type.realbits) - 1;
 
 		return IIO_VAL_FRACTIONAL;
@@ -392,17 +350,50 @@ static int ti_ads7950_read_raw(struct iio_dev *indio_dev,
 }
 
 static const struct iio_info ti_ads7950_info = {
-	.read_raw		= &ti_ads7950_read_raw,
-	.update_scan_mode	= ti_ads7950_update_scan_mode,
+	.read_raw = &ti_ads7950_read_raw,
+};
+
+static int ti_ads7950_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct ti_ads7950_state *st = iio_priv(indio_dev);
+	u32 len = 0;
+	u32 i;
+	u16 cmd;
+
+	for_each_set_bit(i, indio_dev->active_scan_mask, indio_dev->num_channels) {
+		cmd = TI_ADS7950_MAN_CMD(TI_ADS7950_CR_CHAN(i));
+		st->tx_buf[len++] = cmd;
+	}
+
+	/* Data for the 1st channel is not returned until the 3rd transfer */
+	st->tx_buf[len++] = 0;
+	st->tx_buf[len++] = 0;
+
+	st->ring_xfer.len = len * 2;
+
+	return spi_optimize_message(st->spi, &st->ring_msg);
+}
+
+static int ti_ads7950_buffer_postdisable(struct iio_dev *indio_dev)
+{
+	struct ti_ads7950_state *st = iio_priv(indio_dev);
+
+	spi_unoptimize_message(&st->ring_msg);
+
+	return 0;
+}
+
+static const struct iio_buffer_setup_ops ti_ads7950_buffer_setup_ops = {
+	.preenable = ti_ads7950_buffer_preenable,
+	.postdisable = ti_ads7950_buffer_postdisable,
 };
 
 static int ti_ads7950_set(struct gpio_chip *chip, unsigned int offset,
 			  int value)
 {
 	struct ti_ads7950_state *st = gpiochip_get_data(chip);
-	int ret;
 
-	mutex_lock(&st->slock);
+	guard(mutex)(&st->slock);
 
 	if (value)
 		st->cmd_settings_bitmask |= BIT(offset);
@@ -410,11 +401,8 @@ static int ti_ads7950_set(struct gpio_chip *chip, unsigned int offset,
 		st->cmd_settings_bitmask &= ~BIT(offset);
 
 	st->single_tx = TI_ADS7950_MAN_CMD_SETTINGS(st);
-	ret = spi_sync(st->spi, &st->scan_single_msg);
 
-	mutex_unlock(&st->slock);
-
-	return ret;
+	return spi_sync(st->spi, &st->scan_single_msg);
 }
 
 static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset)
@@ -423,13 +411,12 @@ static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset)
 	bool state;
 	int ret;
 
-	mutex_lock(&st->slock);
+	guard(mutex)(&st->slock);
 
 	/* If set as output, return the output */
 	if (st->gpio_cmd_settings_bitmask & BIT(offset)) {
 		state = st->cmd_settings_bitmask & BIT(offset);
-		ret = 0;
-		goto out;
+		return state;
 	}
 
 	/* GPIO data bit sets SDO bits 12-15 to GPIO input */
@@ -437,7 +424,7 @@ static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset)
 	st->single_tx = TI_ADS7950_MAN_CMD_SETTINGS(st);
 	ret = spi_sync(st->spi, &st->scan_single_msg);
 	if (ret)
-		goto out;
+		return ret;
 
 	state = (st->single_rx >> 12) & BIT(offset);
 
@@ -446,12 +433,9 @@ static int ti_ads7950_get(struct gpio_chip *chip, unsigned int offset)
 	st->single_tx = TI_ADS7950_MAN_CMD_SETTINGS(st);
 	ret = spi_sync(st->spi, &st->scan_single_msg);
 	if (ret)
-		goto out;
-
-out:
-	mutex_unlock(&st->slock);
+		return ret;
 
-	return ret ?: state;
+	return state;
 }
 
 static int ti_ads7950_get_direction(struct gpio_chip *chip,
@@ -467,9 +451,8 @@ static int _ti_ads7950_set_direction(struct gpio_chip *chip, int offset,
 				     int input)
 {
 	struct ti_ads7950_state *st = gpiochip_get_data(chip);
-	int ret = 0;
 
-	mutex_lock(&st->slock);
+	guard(mutex)(&st->slock);
 
 	/* Only change direction if needed */
 	if (input && (st->gpio_cmd_settings_bitmask & BIT(offset)))
@@ -477,15 +460,11 @@ static int _ti_ads7950_set_direction(struct gpio_chip *chip, int offset,
 	else if (!input && !(st->gpio_cmd_settings_bitmask & BIT(offset)))
 		st->gpio_cmd_settings_bitmask |= BIT(offset);
 	else
-		goto out;
+		return 0;
 
 	st->single_tx = TI_ADS7950_GPIO_CMD_SETTINGS(st);
-	ret = spi_sync(st->spi, &st->scan_single_msg);
-
-out:
-	mutex_unlock(&st->slock);
 
-	return ret;
+	return spi_sync(st->spi, &st->scan_single_msg);
 }
 
 static int ti_ads7950_direction_input(struct gpio_chip *chip,
@@ -508,9 +487,9 @@ static int ti_ads7950_direction_output(struct gpio_chip *chip,
 
 static int ti_ads7950_init_hw(struct ti_ads7950_state *st)
 {
-	int ret = 0;
+	int ret;
 
-	mutex_lock(&st->slock);
+	guard(mutex)(&st->slock);
 
 	/* Settings for Manual/Auto1/Auto2 commands */
 	/* Default to 5v ref */
@@ -518,17 +497,12 @@ static int ti_ads7950_init_hw(struct ti_ads7950_state *st)
 	st->single_tx = TI_ADS7950_MAN_CMD_SETTINGS(st);
 	ret = spi_sync(st->spi, &st->scan_single_msg);
 	if (ret)
-		goto out;
+		return ret;
 
 	/* Settings for GPIO command */
 	st->gpio_cmd_settings_bitmask = 0x0;
 	st->single_tx = TI_ADS7950_GPIO_CMD_SETTINGS(st);
-	ret = spi_sync(st->spi, &st->scan_single_msg);
-
-out:
-	mutex_unlock(&st->slock);
-
-	return ret;
+	return spi_sync(st->spi, &st->scan_single_msg);
 }
 
 static int ti_ads7950_probe(struct spi_device *spi)
@@ -541,10 +515,8 @@ static int ti_ads7950_probe(struct spi_device *spi)
 	spi->bits_per_word = 16;
 	spi->mode |= SPI_CS_WORD;
 	ret = spi_setup(spi);
-	if (ret < 0) {
-		dev_err(&spi->dev, "Error in spi setup\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&spi->dev, ret, "Error in spi setup\n");
 
 	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
 	if (!indio_dev)
@@ -552,8 +524,6 @@ static int ti_ads7950_probe(struct spi_device *spi)
 
 	st = iio_priv(indio_dev);
 
-	spi_set_drvdata(spi, indio_dev);
-
 	st->spi = spi;
 
 	info = spi_get_device_match_data(spi);
@@ -594,43 +564,38 @@ static int ti_ads7950_probe(struct spi_device *spi)
 	spi_message_init_with_transfers(&st->scan_single_msg,
 					st->scan_single_xfer, 3);
 
+	ret = devm_mutex_init(&spi->dev, &st->slock);
+	if (ret)
+		return ret;
+
 	/* Use hard coded value for reference voltage in ACPI case */
-	if (ACPI_COMPANION(&spi->dev))
+	if (ACPI_COMPANION(&spi->dev)) {
 		st->vref_mv = TI_ADS7950_VA_MV_ACPI_DEFAULT;
+	} else {
+		ret = devm_regulator_get_enable_read_voltage(&spi->dev, "vref");
+		if (ret < 0)
+			return dev_err_probe(&spi->dev, ret,
+					     "Failed to get regulator \"vref\"\n");
 
-	mutex_init(&st->slock);
-
-	st->reg = devm_regulator_get(&spi->dev, "vref");
-	if (IS_ERR(st->reg)) {
-		ret = dev_err_probe(&spi->dev, PTR_ERR(st->reg),
-				     "Failed to get regulator \"vref\"\n");
-		goto error_destroy_mutex;
-	}
-
-	ret = regulator_enable(st->reg);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to enable regulator \"vref\"\n");
-		goto error_destroy_mutex;
+		st->vref_mv = ret / 1000;
 	}
 
-	ret = iio_triggered_buffer_setup(indio_dev, NULL,
-					 &ti_ads7950_trigger_handler, NULL);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to setup triggered buffer\n");
-		goto error_disable_reg;
-	}
+	ret = devm_iio_triggered_buffer_setup(&spi->dev, indio_dev, NULL,
+					      &ti_ads7950_trigger_handler,
+					      &ti_ads7950_buffer_setup_ops);
+	if (ret)
+		return dev_err_probe(&spi->dev, ret,
+				     "Failed to setup triggered buffer\n");
 
 	ret = ti_ads7950_init_hw(st);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to init adc chip\n");
-		goto error_cleanup_ring;
-	}
+	if (ret)
+		return dev_err_probe(&spi->dev, ret,
+				     "Failed to init adc chip\n");
 
-	ret = iio_device_register(indio_dev);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to register iio device\n");
-		goto error_cleanup_ring;
-	}
+	ret = devm_iio_device_register(&spi->dev, indio_dev);
+	if (ret)
+		return dev_err_probe(&spi->dev, ret,
+				     "Failed to register iio device\n");
 
 	/* Add GPIO chip */
 	st->chip.label = dev_name(&st->spi->dev);
@@ -645,36 +610,12 @@ static int ti_ads7950_probe(struct spi_device *spi)
 	st->chip.get = ti_ads7950_get;
 	st->chip.set = ti_ads7950_set;
 
-	ret = gpiochip_add_data(&st->chip, st);
-	if (ret) {
-		dev_err(&spi->dev, "Failed to init GPIOs\n");
-		goto error_iio_device;
-	}
+	ret = devm_gpiochip_add_data(&spi->dev, &st->chip, st);
+	if (ret)
+		return dev_err_probe(&spi->dev, ret,
+				     "Failed to init GPIOs\n");
 
 	return 0;
-
-error_iio_device:
-	iio_device_unregister(indio_dev);
-error_cleanup_ring:
-	iio_triggered_buffer_cleanup(indio_dev);
-error_disable_reg:
-	regulator_disable(st->reg);
-error_destroy_mutex:
-	mutex_destroy(&st->slock);
-
-	return ret;
-}
-
-static void ti_ads7950_remove(struct spi_device *spi)
-{
-	struct iio_dev *indio_dev = spi_get_drvdata(spi);
-	struct ti_ads7950_state *st = iio_priv(indio_dev);
-
-	gpiochip_remove(&st->chip);
-	iio_device_unregister(indio_dev);
-	iio_triggered_buffer_cleanup(indio_dev);
-	regulator_disable(st->reg);
-	mutex_destroy(&st->slock);
 }
 
 static const struct spi_device_id ti_ads7950_id[] = {
@@ -717,7 +658,6 @@ static struct spi_driver ti_ads7950_driver = {
 		.of_match_table = ads7950_of_table,
 	},
 	.probe		= ti_ads7950_probe,
-	.remove		= ti_ads7950_remove,
 	.id_table	= ti_ads7950_id,
 };
 module_spi_driver(ti_ads7950_driver);
diff --git a/drivers/iio/adc/ti-ads8688.c b/drivers/iio/adc/ti-ads8688.c
index b0bf46cae0b69..ebd2826a7ff6a 100644
--- a/drivers/iio/adc/ti-ads8688.c
+++ b/drivers/iio/adc/ti-ads8688.c
@@ -6,7 +6,6 @@
 #include <linux/device.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
-#include <linux/sysfs.h>
 #include <linux/spi/spi.h>
 #include <linux/regulator/consumer.h>
 #include <linux/err.h>
@@ -17,7 +16,6 @@
 #include <linux/iio/buffer.h>
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
-#include <linux/iio/sysfs.h>
 
 #define ADS8688_CMD_REG(x)		(x << 8)
 #define ADS8688_CMD_REG_NOOP		0x00
@@ -66,6 +64,7 @@ struct ads8688_state {
 	const struct ads8688_chip_info	*chip_info;
 	struct spi_device		*spi;
 	unsigned int			vref_mv;
+	int				scale_avail[3][2];
 	enum ads8688_range		range[8];
 	union {
 		__be32 d32;
@@ -114,37 +113,9 @@ static const struct ads8688_ranges ads8688_range_def[5] = {
 	}
 };
 
-static ssize_t ads8688_show_scales(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct ads8688_state *st = iio_priv(dev_to_iio_dev(dev));
-
-	return sprintf(buf, "0.%09u 0.%09u 0.%09u\n",
-		       ads8688_range_def[0].scale * st->vref_mv,
-		       ads8688_range_def[1].scale * st->vref_mv,
-		       ads8688_range_def[2].scale * st->vref_mv);
-}
-
-static ssize_t ads8688_show_offsets(struct device *dev,
-				    struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d %d\n", ads8688_range_def[0].offset,
-		       ads8688_range_def[3].offset);
-}
-
-static IIO_DEVICE_ATTR(in_voltage_scale_available, S_IRUGO,
-		       ads8688_show_scales, NULL, 0);
-static IIO_DEVICE_ATTR(in_voltage_offset_available, S_IRUGO,
-		       ads8688_show_offsets, NULL, 0);
-
-static struct attribute *ads8688_attributes[] = {
-	&iio_dev_attr_in_voltage_scale_available.dev_attr.attr,
-	&iio_dev_attr_in_voltage_offset_available.dev_attr.attr,
-	NULL,
-};
-
-static const struct attribute_group ads8688_attribute_group = {
-	.attrs = ads8688_attributes,
+static const int ads8688_offset_avail[] = {
+	-(1 << (ADS8688_REALBITS - 1)),
+	0
 };
 
 #define ADS8688_CHAN(index)					\
@@ -155,6 +126,9 @@ static const struct attribute_group ads8688_attribute_group = {
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW)		\
 			      | BIT(IIO_CHAN_INFO_SCALE)	\
 			      | BIT(IIO_CHAN_INFO_OFFSET),	\
+	.info_mask_shared_by_type_available =			\
+			      BIT(IIO_CHAN_INFO_SCALE)		\
+			      | BIT(IIO_CHAN_INFO_OFFSET),	\
 	.scan_index = index,					\
 	.scan_type = {						\
 		.sign = 'u',					\
@@ -369,11 +343,34 @@ static int ads8688_write_raw_get_fmt(struct iio_dev *indio_dev,
 	return -EINVAL;
 }
 
+static int ads8688_read_avail(struct iio_dev *indio_dev,
+			      struct iio_chan_spec const *chan,
+			      const int **vals, int *type, int *length,
+			      long mask)
+{
+	struct ads8688_state *st = iio_priv(indio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SCALE:
+		*vals = (const int *)st->scale_avail;
+		*type = IIO_VAL_INT_PLUS_NANO;
+		*length = ARRAY_SIZE(st->scale_avail) * 2;
+		return IIO_AVAIL_LIST;
+	case IIO_CHAN_INFO_OFFSET:
+		*vals = ads8688_offset_avail;
+		*type = IIO_VAL_INT;
+		*length = ARRAY_SIZE(ads8688_offset_avail);
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
+}
+
 static const struct iio_info ads8688_info = {
 	.read_raw = &ads8688_read_raw,
+	.read_avail = &ads8688_read_avail,
 	.write_raw = &ads8688_write_raw,
 	.write_raw_get_fmt = &ads8688_write_raw_get_fmt,
-	.attrs = &ads8688_attribute_group,
 };
 
 static irqreturn_t ads8688_trigger_handler(int irq, void *p)
@@ -426,6 +423,11 @@ static int ads8688_probe(struct spi_device *spi)
 
 	st->vref_mv = ret == -ENODEV ? ADS8688_VREF_MV : ret / 1000;
 
+	for (unsigned int i = 0; i < ARRAY_SIZE(st->scale_avail); i++) {
+		st->scale_avail[i][0] = 0;
+		st->scale_avail[i][1] = ads8688_range_def[i].scale * st->vref_mv;
+	}
+
 	st->chip_info =	&ads8688_chip_info_tbl[spi_get_device_id(spi)->driver_data];
 
 	spi->mode = SPI_MODE_1;
diff --git a/drivers/iio/adc/viperboard_adc.c b/drivers/iio/adc/viperboard_adc.c
index 9bb0b83c8f676..6efe1c618ef71 100644
--- a/drivers/iio/adc/viperboard_adc.c
+++ b/drivers/iio/adc/viperboard_adc.c
@@ -70,8 +70,10 @@ static int vprbrd_iio_read_raw(struct iio_dev *iio_dev,
 			VPRBRD_USB_TYPE_OUT, 0x0000, 0x0000, admsg,
 			sizeof(struct vprbrd_adc_msg), VPRBRD_USB_TIMEOUT_MS);
 		if (ret != sizeof(struct vprbrd_adc_msg)) {
-			dev_err(&iio_dev->dev, "usb send error on adc read\n");
+			mutex_unlock(&vb->lock);
 			error = -EREMOTEIO;
+			dev_err(&iio_dev->dev, "usb send error on adc read\n");
+			goto error;
 		}
 
 		ret = usb_control_msg(vb->usb_dev,
diff --git a/drivers/iio/adc/xilinx-ams.c b/drivers/iio/adc/xilinx-ams.c
index 124470c925297..d38c4401dfce1 100644
--- a/drivers/iio/adc/xilinx-ams.c
+++ b/drivers/iio/adc/xilinx-ams.c
@@ -102,6 +102,7 @@
 #define AMS_PS_SEQ_MASK			GENMASK(21, 0)
 #define AMS_PL_SEQ_MASK			GENMASK_ULL(59, 22)
 
+#define AMS_ALARM_NONE			0x000 /* not a real offset */
 #define AMS_ALARM_TEMP			0x140
 #define AMS_ALARM_SUPPLY1		0x144
 #define AMS_ALARM_SUPPLY2		0x148
@@ -720,22 +721,20 @@ static int ams_read_raw(struct iio_dev *indio_dev,
 	int ret;
 
 	switch (mask) {
-	case IIO_CHAN_INFO_RAW:
-		mutex_lock(&ams->lock);
+	case IIO_CHAN_INFO_RAW: {
+		guard(mutex)(&ams->lock);
 		if (chan->scan_index >= AMS_CTRL_SEQ_BASE) {
 			ret = ams_read_vcc_reg(ams, chan->address, val);
 			if (ret)
-				goto unlock_mutex;
+				return ret;
 			ams_enable_channel_sequence(indio_dev);
 		} else if (chan->scan_index >= AMS_PS_SEQ_MAX)
 			*val = readl(ams->pl_base + chan->address);
 		else
 			*val = readl(ams->ps_base + chan->address);
 
-		ret = IIO_VAL_INT;
-unlock_mutex:
-		mutex_unlock(&ams->lock);
-		return ret;
+		return IIO_VAL_INT;
+	}
 	case IIO_CHAN_INFO_SCALE:
 		switch (chan->type) {
 		case IIO_VOLTAGE:
@@ -765,9 +764,49 @@ static int ams_read_raw(struct iio_dev *indio_dev,
 	}
 }
 
+struct ams_alarm_map {
+	enum ams_ps_pl_seq scan_index;
+	unsigned int base_offset;
+};
+
+/*
+ * Array index matches enum ams_alarm_bit.
+ * Entries with base_offset == AMS_ALARM_NONE are unused/invalid
+ * (e.g. RESERVED) and must be skipped.
+ */
+static const struct ams_alarm_map alarm_map[] = {
+	[AMS_ALARM_BIT_TEMP] = { AMS_SEQ_TEMP, AMS_ALARM_TEMP },
+	[AMS_ALARM_BIT_SUPPLY1] = { AMS_SEQ_SUPPLY1, AMS_ALARM_SUPPLY1 },
+	[AMS_ALARM_BIT_SUPPLY2] = { AMS_SEQ_SUPPLY2, AMS_ALARM_SUPPLY2 },
+	[AMS_ALARM_BIT_SUPPLY3] = { AMS_SEQ_SUPPLY3, AMS_ALARM_SUPPLY3 },
+	[AMS_ALARM_BIT_SUPPLY4] = { AMS_SEQ_SUPPLY4, AMS_ALARM_SUPPLY4 },
+	[AMS_ALARM_BIT_SUPPLY5] = { AMS_SEQ_SUPPLY5, AMS_ALARM_SUPPLY5 },
+	[AMS_ALARM_BIT_SUPPLY6] = { AMS_SEQ_SUPPLY6, AMS_ALARM_SUPPLY6 },
+	[AMS_ALARM_BIT_RESERVED] = { 0, AMS_ALARM_NONE },
+	[AMS_ALARM_BIT_SUPPLY7] = { AMS_SEQ_SUPPLY7, AMS_ALARM_SUPPLY7 },
+	[AMS_ALARM_BIT_SUPPLY8] = { AMS_SEQ_SUPPLY8, AMS_ALARM_SUPPLY8 },
+	[AMS_ALARM_BIT_SUPPLY9] = { AMS_SEQ_SUPPLY9, AMS_ALARM_SUPPLY9 },
+	[AMS_ALARM_BIT_SUPPLY10] = { AMS_SEQ_SUPPLY10, AMS_ALARM_SUPPLY10 },
+	[AMS_ALARM_BIT_VCCAMS] = { AMS_SEQ_VCCAMS, AMS_ALARM_VCCAMS },
+	[AMS_ALARM_BIT_TEMP_REMOTE] = { AMS_SEQ_TEMP_REMOTE, AMS_ALARM_TEMP_REMOTE },
+};
+
+static int ams_scan_index_to_event(int scan_index)
+{
+	for (unsigned int i = 0; i < ARRAY_SIZE(alarm_map); i++) {
+		if (alarm_map[i].base_offset == AMS_ALARM_NONE)
+			continue;
+
+		if (alarm_map[i].scan_index == scan_index)
+			return i;
+	}
+
+	return -EINVAL;
+}
+
 static int ams_get_alarm_offset(int scan_index, enum iio_event_direction dir)
 {
-	int offset;
+	int offset, event;
 
 	if (scan_index >= AMS_PS_SEQ_MAX)
 		scan_index -= AMS_PS_SEQ_MAX;
@@ -781,36 +820,11 @@ static int ams_get_alarm_offset(int scan_index, enum iio_event_direction dir)
 		offset = 0;
 	}
 
-	switch (scan_index) {
-	case AMS_SEQ_TEMP:
-		return AMS_ALARM_TEMP + offset;
-	case AMS_SEQ_SUPPLY1:
-		return AMS_ALARM_SUPPLY1 + offset;
-	case AMS_SEQ_SUPPLY2:
-		return AMS_ALARM_SUPPLY2 + offset;
-	case AMS_SEQ_SUPPLY3:
-		return AMS_ALARM_SUPPLY3 + offset;
-	case AMS_SEQ_SUPPLY4:
-		return AMS_ALARM_SUPPLY4 + offset;
-	case AMS_SEQ_SUPPLY5:
-		return AMS_ALARM_SUPPLY5 + offset;
-	case AMS_SEQ_SUPPLY6:
-		return AMS_ALARM_SUPPLY6 + offset;
-	case AMS_SEQ_SUPPLY7:
-		return AMS_ALARM_SUPPLY7 + offset;
-	case AMS_SEQ_SUPPLY8:
-		return AMS_ALARM_SUPPLY8 + offset;
-	case AMS_SEQ_SUPPLY9:
-		return AMS_ALARM_SUPPLY9 + offset;
-	case AMS_SEQ_SUPPLY10:
-		return AMS_ALARM_SUPPLY10 + offset;
-	case AMS_SEQ_VCCAMS:
-		return AMS_ALARM_VCCAMS + offset;
-	case AMS_SEQ_TEMP_REMOTE:
-		return AMS_ALARM_TEMP_REMOTE + offset;
-	default:
+	event = ams_scan_index_to_event(scan_index);
+	if (event < 0 || alarm_map[event].base_offset == AMS_ALARM_NONE)
 		return 0;
-	}
+
+	return alarm_map[event].base_offset + offset;
 }
 
 static const struct iio_chan_spec *ams_event_to_channel(struct iio_dev *dev,
@@ -823,96 +837,38 @@ static const struct iio_chan_spec *ams_event_to_channel(struct iio_dev *dev,
 		scan_index = AMS_PS_SEQ_MAX;
 	}
 
-	switch (event) {
-	case AMS_ALARM_BIT_TEMP:
-		scan_index += AMS_SEQ_TEMP;
-		break;
-	case AMS_ALARM_BIT_SUPPLY1:
-		scan_index += AMS_SEQ_SUPPLY1;
-		break;
-	case AMS_ALARM_BIT_SUPPLY2:
-		scan_index += AMS_SEQ_SUPPLY2;
-		break;
-	case AMS_ALARM_BIT_SUPPLY3:
-		scan_index += AMS_SEQ_SUPPLY3;
-		break;
-	case AMS_ALARM_BIT_SUPPLY4:
-		scan_index += AMS_SEQ_SUPPLY4;
-		break;
-	case AMS_ALARM_BIT_SUPPLY5:
-		scan_index += AMS_SEQ_SUPPLY5;
-		break;
-	case AMS_ALARM_BIT_SUPPLY6:
-		scan_index += AMS_SEQ_SUPPLY6;
-		break;
-	case AMS_ALARM_BIT_SUPPLY7:
-		scan_index += AMS_SEQ_SUPPLY7;
-		break;
-	case AMS_ALARM_BIT_SUPPLY8:
-		scan_index += AMS_SEQ_SUPPLY8;
-		break;
-	case AMS_ALARM_BIT_SUPPLY9:
-		scan_index += AMS_SEQ_SUPPLY9;
-		break;
-	case AMS_ALARM_BIT_SUPPLY10:
-		scan_index += AMS_SEQ_SUPPLY10;
-		break;
-	case AMS_ALARM_BIT_VCCAMS:
-		scan_index += AMS_SEQ_VCCAMS;
-		break;
-	case AMS_ALARM_BIT_TEMP_REMOTE:
-		scan_index += AMS_SEQ_TEMP_REMOTE;
-		break;
-	default:
-		break;
-	}
+	if (event >= ARRAY_SIZE(alarm_map))
+		return NULL;
+
+	if (alarm_map[event].base_offset == AMS_ALARM_NONE)
+		return NULL;
+
+	scan_index += alarm_map[event].scan_index;
 
 	for (i = 0; i < dev->num_channels; i++)
 		if (dev->channels[i].scan_index == scan_index)
 			break;
 
+	if (i == dev->num_channels)
+		return NULL;
+
 	return &dev->channels[i];
 }
 
 static int ams_get_alarm_mask(int scan_index)
 {
-	int bit = 0;
+	int bit = 0, event;
 
 	if (scan_index >= AMS_PS_SEQ_MAX) {
 		bit = AMS_PL_ALARM_START;
 		scan_index -= AMS_PS_SEQ_MAX;
 	}
 
-	switch (scan_index) {
-	case AMS_SEQ_TEMP:
-		return BIT(AMS_ALARM_BIT_TEMP + bit);
-	case AMS_SEQ_SUPPLY1:
-		return BIT(AMS_ALARM_BIT_SUPPLY1 + bit);
-	case AMS_SEQ_SUPPLY2:
-		return BIT(AMS_ALARM_BIT_SUPPLY2 + bit);
-	case AMS_SEQ_SUPPLY3:
-		return BIT(AMS_ALARM_BIT_SUPPLY3 + bit);
-	case AMS_SEQ_SUPPLY4:
-		return BIT(AMS_ALARM_BIT_SUPPLY4 + bit);
-	case AMS_SEQ_SUPPLY5:
-		return BIT(AMS_ALARM_BIT_SUPPLY5 + bit);
-	case AMS_SEQ_SUPPLY6:
-		return BIT(AMS_ALARM_BIT_SUPPLY6 + bit);
-	case AMS_SEQ_SUPPLY7:
-		return BIT(AMS_ALARM_BIT_SUPPLY7 + bit);
-	case AMS_SEQ_SUPPLY8:
-		return BIT(AMS_ALARM_BIT_SUPPLY8 + bit);
-	case AMS_SEQ_SUPPLY9:
-		return BIT(AMS_ALARM_BIT_SUPPLY9 + bit);
-	case AMS_SEQ_SUPPLY10:
-		return BIT(AMS_ALARM_BIT_SUPPLY10 + bit);
-	case AMS_SEQ_VCCAMS:
-		return BIT(AMS_ALARM_BIT_VCCAMS + bit);
-	case AMS_SEQ_TEMP_REMOTE:
-		return BIT(AMS_ALARM_BIT_TEMP_REMOTE + bit);
-	default:
+	event = ams_scan_index_to_event(scan_index);
+	if (event < 0)
 		return 0;
-	}
+
+	return BIT(event + bit);
 }
 
 static int ams_read_event_config(struct iio_dev *indio_dev,
@@ -936,7 +892,7 @@ static int ams_write_event_config(struct iio_dev *indio_dev,
 
 	alarm = ams_get_alarm_mask(chan->scan_index);
 
-	mutex_lock(&ams->lock);
+	guard(mutex)(&ams->lock);
 
 	if (state)
 		ams->alarm_mask |= alarm;
@@ -945,8 +901,6 @@ static int ams_write_event_config(struct iio_dev *indio_dev,
 
 	ams_update_alarm(ams, ams->alarm_mask);
 
-	mutex_unlock(&ams->lock);
-
 	return 0;
 }
 
@@ -959,15 +913,13 @@ static int ams_read_event_value(struct iio_dev *indio_dev,
 	struct ams *ams = iio_priv(indio_dev);
 	unsigned int offset = ams_get_alarm_offset(chan->scan_index, dir);
 
-	mutex_lock(&ams->lock);
+	guard(mutex)(&ams->lock);
 
 	if (chan->scan_index >= AMS_PS_SEQ_MAX)
 		*val = readl(ams->pl_base + offset);
 	else
 		*val = readl(ams->ps_base + offset);
 
-	mutex_unlock(&ams->lock);
-
 	return IIO_VAL_INT;
 }
 
@@ -980,7 +932,7 @@ static int ams_write_event_value(struct iio_dev *indio_dev,
 	struct ams *ams = iio_priv(indio_dev);
 	unsigned int offset;
 
-	mutex_lock(&ams->lock);
+	guard(mutex)(&ams->lock);
 
 	/* Set temperature channel threshold to direct threshold */
 	if (chan->type == IIO_TEMP) {
@@ -1002,8 +954,6 @@ static int ams_write_event_value(struct iio_dev *indio_dev,
 	else
 		writel(val, ams->ps_base + offset);
 
-	mutex_unlock(&ams->lock);
-
 	return 0;
 }
 
@@ -1012,6 +962,8 @@ static void ams_handle_event(struct iio_dev *indio_dev, u32 event)
 	const struct iio_chan_spec *chan;
 
 	chan = ams_event_to_channel(indio_dev, event);
+	if (!chan)
+		return;
 
 	if (chan->type == IIO_TEMP) {
 		/*
diff --git a/drivers/iio/adc/xilinx-xadc-core.c b/drivers/iio/adc/xilinx-xadc-core.c
index e257c1b94a5f7..3980dfacbcd79 100644
--- a/drivers/iio/adc/xilinx-xadc-core.c
+++ b/drivers/iio/adc/xilinx-xadc-core.c
@@ -817,6 +817,7 @@ static int xadc_postdisable(struct iio_dev *indio_dev)
 {
 	struct xadc *xadc = iio_priv(indio_dev);
 	unsigned long scan_mask;
+	int seq_mode;
 	int ret;
 	int i;
 
@@ -824,6 +825,12 @@ static int xadc_postdisable(struct iio_dev *indio_dev)
 	for (i = 0; i < indio_dev->num_channels; i++)
 		scan_mask |= BIT(indio_dev->channels[i].scan_index);
 
+	/*
+	 * Use the correct sequencer mode for the idle state: simultaneous
+	 * mode for dual external mux configurations, continuous otherwise.
+	 */
+	seq_mode = xadc_get_seq_mode(xadc, scan_mask);
+
 	/* Enable all channels and calibration */
 	ret = xadc_write_adc_reg(xadc, XADC_REG_SEQ(0), scan_mask & 0xffff);
 	if (ret)
@@ -834,11 +841,11 @@ static int xadc_postdisable(struct iio_dev *indio_dev)
 		return ret;
 
 	ret = xadc_update_adc_reg(xadc, XADC_REG_CONF1, XADC_CONF1_SEQ_MASK,
-		XADC_CONF1_SEQ_CONTINUOUS);
+				  seq_mode);
 	if (ret)
 		return ret;
 
-	return xadc_power_adc_b(xadc, XADC_CONF1_SEQ_CONTINUOUS);
+	return xadc_power_adc_b(xadc, seq_mode);
 }
 
 static int xadc_preenable(struct iio_dev *indio_dev)
diff --git a/drivers/iio/addac/ad74115.c b/drivers/iio/addac/ad74115.c
index f8b04d86b01f3..41e0b1d334cc2 100644
--- a/drivers/iio/addac/ad74115.c
+++ b/drivers/iio/addac/ad74115.c
@@ -1835,7 +1835,10 @@ static int ad74115_probe(struct spi_device *spi)
 	st = iio_priv(indio_dev);
 
 	st->spi = spi;
-	mutex_init(&st->lock);
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
+
 	init_completion(&st->adc_data_completion);
 
 	indio_dev->name = AD74115_NAME;
diff --git a/drivers/iio/buffer/industrialio-hw-consumer.c b/drivers/iio/buffer/industrialio-hw-consumer.c
index 24d7df603760f..10e912bbf0c5f 100644
--- a/drivers/iio/buffer/industrialio-hw-consumer.c
+++ b/drivers/iio/buffer/industrialio-hw-consumer.c
@@ -40,6 +40,8 @@ static void iio_hw_buf_release(struct iio_buffer *buffer)
 {
 	struct hw_consumer_buffer *hw_buf =
 		iio_buffer_to_hw_consumer_buffer(buffer);
+
+	bitmap_free(buffer->scan_mask);
 	kfree(hw_buf);
 }
 
@@ -85,7 +87,7 @@ static struct hw_consumer_buffer *iio_hw_consumer_get_buffer(
  */
 struct iio_hw_consumer *iio_hw_consumer_alloc(struct device *dev)
 {
-	struct hw_consumer_buffer *buf;
+	struct hw_consumer_buffer *buf, *tmp;
 	struct iio_hw_consumer *hwc;
 	struct iio_channel *chan;
 	int ret;
@@ -116,7 +118,7 @@ struct iio_hw_consumer *iio_hw_consumer_alloc(struct device *dev)
 	return hwc;
 
 err_put_buffers:
-	list_for_each_entry(buf, &hwc->buffers, head)
+	list_for_each_entry_safe(buf, tmp, &hwc->buffers, head)
 		iio_buffer_put(&buf->buffer);
 	iio_channel_release_all(hwc->channels);
 err_free_hwc:
diff --git a/drivers/iio/chemical/bme680_core.c b/drivers/iio/chemical/bme680_core.c
index 6aeac132394c7..ac9a737481dc7 100644
--- a/drivers/iio/chemical/bme680_core.c
+++ b/drivers/iio/chemical/bme680_core.c
@@ -128,7 +128,7 @@ struct bme680_data {
 	u16 heater_temp;
 
 	struct {
-		s32 chan[4];
+		s32 chan[BME680_NUM_CHANNELS];
 		aligned_s64 ts;
 	} scan;
 
diff --git a/drivers/iio/chemical/mhz19b.c b/drivers/iio/chemical/mhz19b.c
index 3c64154918b19..9d4cf432919e6 100644
--- a/drivers/iio/chemical/mhz19b.c
+++ b/drivers/iio/chemical/mhz19b.c
@@ -52,6 +52,8 @@ struct mhz19b_state {
 	struct completion buf_ready;
 
 	u8 buf_idx;
+	bool buf_overflow;
+
 	/*
 	 * Serdev receive buffer.
 	 * When data is received from the MH-Z19B,
@@ -106,6 +108,10 @@ static int mhz19b_serdev_cmd(struct iio_dev *indio_dev, int cmd, u16 arg)
 	cmd_buf[8] = mhz19b_get_checksum(cmd_buf);
 
 	/* Write buf to uart ctrl synchronously */
+	st->buf_idx = 0;
+	st->buf_overflow = false;
+	reinit_completion(&st->buf_ready);
+
 	ret = serdev_device_write(serdev, cmd_buf, MHZ19B_CMD_SIZE, 0);
 	if (ret < 0)
 		return ret;
@@ -121,6 +127,9 @@ static int mhz19b_serdev_cmd(struct iio_dev *indio_dev, int cmd, u16 arg)
 		if (!ret)
 			return -ETIMEDOUT;
 
+		if (st->buf_overflow)
+			return -EMSGSIZE;
+
 		if (st->buf[8] != mhz19b_get_checksum(st->buf)) {
 			dev_err(dev, "checksum err");
 			return -EINVAL;
@@ -240,6 +249,14 @@ static size_t mhz19b_receive_buf(struct serdev_device *serdev,
 {
 	struct iio_dev *indio_dev = dev_get_drvdata(&serdev->dev);
 	struct mhz19b_state *st = iio_priv(indio_dev);
+	size_t remaining = MHZ19B_CMD_SIZE - st->buf_idx;
+
+	if (len > remaining) {
+		st->buf_idx = 0;
+		st->buf_overflow = true;
+		complete(&st->buf_ready);
+		return len;
+	}
 
 	memcpy(st->buf + st->buf_idx, data, len);
 	st->buf_idx += len;
diff --git a/drivers/iio/chemical/scd30_core.c b/drivers/iio/chemical/scd30_core.c
index a665fcb78806f..11d6bc1b63e67 100644
--- a/drivers/iio/chemical/scd30_core.c
+++ b/drivers/iio/chemical/scd30_core.c
@@ -256,7 +256,7 @@ static int scd30_write_raw(struct iio_dev *indio_dev, struct iio_chan_spec const
 	guard(mutex)(&state->lock);
 	switch (mask) {
 	case IIO_CHAN_INFO_SAMP_FREQ:
-		if (val)
+		if (val || !val2)
 			return -EINVAL;
 
 		val = 1000000000 / val2;
diff --git a/drivers/iio/chemical/scd30_i2c.c b/drivers/iio/chemical/scd30_i2c.c
index 436df9c61a719..9e841f5651499 100644
--- a/drivers/iio/chemical/scd30_i2c.c
+++ b/drivers/iio/chemical/scd30_i2c.c
@@ -20,7 +20,7 @@
 #define SCD30_I2C_MAX_BUF_SIZE 18
 #define SCD30_I2C_CRC8_POLYNOMIAL 0x31
 
-static u16 scd30_i2c_cmd_lookup_tbl[] = {
+static const u16 scd30_i2c_cmd_lookup_tbl[] = {
 	[CMD_START_MEAS] = 0x0010,
 	[CMD_STOP_MEAS] = 0x0104,
 	[CMD_MEAS_INTERVAL] = 0x4600,
@@ -71,6 +71,9 @@ static int scd30_i2c_command(struct scd30_state *state, enum scd30_cmd cmd, u16
 	int i, ret;
 	char crc;
 
+	if (!response && size != 0)
+		return -EINVAL;
+
 	put_unaligned_be16(scd30_i2c_cmd_lookup_tbl[cmd], buf);
 	i = 2;
 
diff --git a/drivers/iio/chemical/sgp30.c b/drivers/iio/chemical/sgp30.c
index 21730d62b5c8d..d42b62b7081e9 100644
--- a/drivers/iio/chemical/sgp30.c
+++ b/drivers/iio/chemical/sgp30.c
@@ -498,6 +498,7 @@ static const struct of_device_id sgp_dt_ids[] = {
 	{ .compatible = "sensirion,sgpc3", .data = &sgp_devices[SGPC3] },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, sgp_dt_ids);
 
 static int sgp_probe(struct i2c_client *client)
 {
@@ -566,9 +567,7 @@ static const struct i2c_device_id sgp_id[] = {
 	{ "sgpc3", (kernel_ulong_t)&sgp_devices[SGPC3] },
 	{ }
 };
-
 MODULE_DEVICE_TABLE(i2c, sgp_id);
-MODULE_DEVICE_TABLE(of, sgp_dt_ids);
 
 static struct i2c_driver sgp_driver = {
 	.driver = {
diff --git a/drivers/iio/common/ssp_sensors/ssp_dev.c b/drivers/iio/common/ssp_sensors/ssp_dev.c
index da09c9f3ceb6c..51730dae58714 100644
--- a/drivers/iio/common/ssp_sensors/ssp_dev.c
+++ b/drivers/iio/common/ssp_sensors/ssp_dev.c
@@ -28,7 +28,7 @@ struct ssp_instruction {
 	__le32 a;
 	__le32 b;
 	u8 c;
-} __attribute__((__packed__));
+} __packed__;
 
 static const u8 ssp_magnitude_table[] = {110, 85, 171, 71, 203, 195, 0, 67,
 	208, 56, 175, 244, 206, 213, 0, 92, 250, 0, 55, 48, 189, 252, 171,
@@ -590,6 +590,7 @@ static void ssp_remove(struct spi_device *spi)
 	ssp_clean_pending_list(data);
 
 	free_irq(data->spi->irq, data);
+	cancel_delayed_work_sync(&data->work_refresh);
 
 	timer_delete_sync(&data->wdt_timer);
 	cancel_work_sync(&data->work_wdt);
diff --git a/drivers/iio/common/ssp_sensors/ssp_spi.c b/drivers/iio/common/ssp_sensors/ssp_spi.c
index 6c81c0385fb59..08ed92859be08 100644
--- a/drivers/iio/common/ssp_sensors/ssp_spi.c
+++ b/drivers/iio/common/ssp_sensors/ssp_spi.c
@@ -6,7 +6,7 @@
 #include "ssp.h"
 
 #define SSP_DEV (&data->spi->dev)
-#define SSP_GET_MESSAGE_TYPE(data) (data & (3 << SSP_RW))
+#define SSP_GET_MESSAGE_TYPE(data) ((data) & (3 << SSP_RW))
 
 /*
  * SSP -> AP Instruction
@@ -29,7 +29,7 @@ struct ssp_msg_header {
 	__le16 length;
 	__le16 options;
 	__le32 data;
-} __attribute__((__packed__));
+} __packed;
 
 struct ssp_msg {
 	u16 length;
@@ -119,9 +119,9 @@ static inline void ssp_get_buffer(struct ssp_msg *m, unsigned int offset,
 }
 
 #define SSP_GET_BUFFER_AT_INDEX(m, index) \
-	(m->buffer[SSP_HEADER_SIZE_ALIGNED + index])
+	((m)->buffer[SSP_HEADER_SIZE_ALIGNED + (index)])
 #define SSP_SET_BUFFER_AT_INDEX(m, index, val) \
-	(m->buffer[SSP_HEADER_SIZE_ALIGNED + index] = val)
+	((m)->buffer[SSP_HEADER_SIZE_ALIGNED + (index)] = val)
 
 static void ssp_clean_msg(struct ssp_msg *m)
 {
diff --git a/drivers/iio/dac/Kconfig b/drivers/iio/dac/Kconfig
index cd4870b654153..657c68e75542e 100644
--- a/drivers/iio/dac/Kconfig
+++ b/drivers/iio/dac/Kconfig
@@ -178,6 +178,17 @@ config AD5624R_SPI
 	  Say yes here to build support for Analog Devices AD5624R, AD5644R and
 	  AD5664R converters (DAC). This driver uses the common SPI interface.
 
+config AD5706R
+	tristate "Analog Devices AD5706R DAC driver"
+	depends on SPI
+	select REGMAP
+	help
+	  Say yes here to build support for Analog Devices AD5706R 4-channel,
+	  16-bit current output DAC.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called ad5706r.
+
 config AD9739A
 	tristate "Analog Devices AD9739A RF DAC spi driver"
 	depends on SPI
diff --git a/drivers/iio/dac/Makefile b/drivers/iio/dac/Makefile
index 2a80bbf4e80ad..0034317984985 100644
--- a/drivers/iio/dac/Makefile
+++ b/drivers/iio/dac/Makefile
@@ -21,6 +21,7 @@ obj-$(CONFIG_AD5449) += ad5449.o
 obj-$(CONFIG_AD5592R_BASE) += ad5592r-base.o
 obj-$(CONFIG_AD5592R) += ad5592r.o
 obj-$(CONFIG_AD5593R) += ad5593r.o
+obj-$(CONFIG_AD5706R) += ad5706r.o
 obj-$(CONFIG_AD5755) += ad5755.o
 obj-$(CONFIG_AD5758) += ad5758.o
 obj-$(CONFIG_AD5761) += ad5761.o
diff --git a/drivers/iio/dac/ad3530r.c b/drivers/iio/dac/ad3530r.c
index b97b46090d808..d9db3226ecd64 100644
--- a/drivers/iio/dac/ad3530r.c
+++ b/drivers/iio/dac/ad3530r.c
@@ -105,6 +105,12 @@ static const char * const ad3530r_powerdown_modes[] = {
 	"32kohm_to_gnd",
 };
 
+static const char * const ad3531r_powerdown_modes[] = {
+	"500ohm_to_gnd",
+	"3.85kohm_to_gnd",
+	"16kohm_to_gnd",
+};
+
 static int ad3530r_get_powerdown_mode(struct iio_dev *indio_dev,
 				      const struct iio_chan_spec *chan)
 {
@@ -133,6 +139,13 @@ static const struct iio_enum ad3530r_powerdown_mode_enum = {
 	.set = ad3530r_set_powerdown_mode,
 };
 
+static const struct iio_enum ad3531r_powerdown_mode_enum = {
+	.items = ad3531r_powerdown_modes,
+	.num_items = ARRAY_SIZE(ad3531r_powerdown_modes),
+	.get = ad3530r_get_powerdown_mode,
+	.set = ad3530r_set_powerdown_mode,
+};
+
 static ssize_t ad3530r_get_dac_powerdown(struct iio_dev *indio_dev,
 					 uintptr_t private,
 					 const struct iio_chan_spec *chan,
@@ -276,7 +289,20 @@ static const struct iio_chan_spec_ext_info ad3530r_ext_info[] = {
 	{ }
 };
 
-#define AD3530R_CHAN(_chan)					\
+static const struct iio_chan_spec_ext_info ad3531r_ext_info[] = {
+	{
+		.name = "powerdown",
+		.shared = IIO_SEPARATE,
+		.read = ad3530r_get_dac_powerdown,
+		.write = ad3530r_set_dac_powerdown,
+	},
+	IIO_ENUM("powerdown_mode", IIO_SEPARATE, &ad3531r_powerdown_mode_enum),
+	IIO_ENUM_AVAILABLE("powerdown_mode", IIO_SHARED_BY_TYPE,
+			   &ad3531r_powerdown_mode_enum),
+	{ }
+};
+
+#define AD3530R_CHAN(_chan, _ext_info)				\
 {								\
 	.type = IIO_VOLTAGE,					\
 	.indexed = 1,						\
@@ -284,25 +310,25 @@ static const struct iio_chan_spec_ext_info ad3530r_ext_info[] = {
 	.output = 1,						\
 	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |		\
 			      BIT(IIO_CHAN_INFO_SCALE),		\
-	.ext_info = ad3530r_ext_info,				\
+	.ext_info = _ext_info,					\
 }
 
 static const struct iio_chan_spec ad3530r_channels[] = {
-	AD3530R_CHAN(0),
-	AD3530R_CHAN(1),
-	AD3530R_CHAN(2),
-	AD3530R_CHAN(3),
-	AD3530R_CHAN(4),
-	AD3530R_CHAN(5),
-	AD3530R_CHAN(6),
-	AD3530R_CHAN(7),
+	AD3530R_CHAN(0, ad3530r_ext_info),
+	AD3530R_CHAN(1, ad3530r_ext_info),
+	AD3530R_CHAN(2, ad3530r_ext_info),
+	AD3530R_CHAN(3, ad3530r_ext_info),
+	AD3530R_CHAN(4, ad3530r_ext_info),
+	AD3530R_CHAN(5, ad3530r_ext_info),
+	AD3530R_CHAN(6, ad3530r_ext_info),
+	AD3530R_CHAN(7, ad3530r_ext_info),
 };
 
 static const struct iio_chan_spec ad3531r_channels[] = {
-	AD3530R_CHAN(0),
-	AD3530R_CHAN(1),
-	AD3530R_CHAN(2),
-	AD3530R_CHAN(3),
+	AD3530R_CHAN(0, ad3531r_ext_info),
+	AD3530R_CHAN(1, ad3531r_ext_info),
+	AD3530R_CHAN(2, ad3531r_ext_info),
+	AD3530R_CHAN(3, ad3531r_ext_info),
 };
 
 static const struct ad3530r_chip_info ad3530_chip = {
diff --git a/drivers/iio/dac/ad3552r.c b/drivers/iio/dac/ad3552r.c
index 93c33bc3e1be4..16db94fef9d42 100644
--- a/drivers/iio/dac/ad3552r.c
+++ b/drivers/iio/dac/ad3552r.c
@@ -167,8 +167,7 @@ static int ad3552r_read_raw(struct iio_dev *indio_dev,
 		mutex_unlock(&dac->lock);
 		if (err < 0)
 			return err;
-		*val = !((tmp_val & AD3552R_MASK_CH_DAC_POWERDOWN(ch)) >>
-			  __ffs(AD3552R_MASK_CH_DAC_POWERDOWN(ch)));
+		*val = !field_get(AD3552R_MASK_CH_DAC_POWERDOWN(ch), tmp_val);
 		return IIO_VAL_INT;
 	case IIO_CHAN_INFO_SCALE:
 		*val = dac->ch_data[ch].scale_int;
@@ -629,7 +628,9 @@ static int ad3552r_probe(struct spi_device *spi)
 	if (!dac->model_data)
 		return -EINVAL;
 
-	mutex_init(&dac->lock);
+	err = devm_mutex_init(&spi->dev, &dac->lock);
+	if (err)
+		return err;
 
 	err = ad3552r_init(dac);
 	if (err)
diff --git a/drivers/iio/dac/ad5686.c b/drivers/iio/dac/ad5686.c
index 4b18498aa0749..19440b17645b1 100644
--- a/drivers/iio/dac/ad5686.c
+++ b/drivers/iio/dac/ad5686.c
@@ -25,22 +25,37 @@ static const char * const ad5686_powerdown_modes[] = {
 	"three_state"
 };
 
+static inline unsigned int ad5686_pd_mask_shift(const struct iio_chan_spec *chan)
+{
+	if (chan->channel == chan->address)
+		return chan->channel * 2;
+
+	/* one-hot encoding is used in dual/quad channel devices */
+	return __ffs(chan->address) * 2;
+}
+
 static int ad5686_get_powerdown_mode(struct iio_dev *indio_dev,
 				     const struct iio_chan_spec *chan)
 {
+	unsigned int shift = ad5686_pd_mask_shift(chan);
 	struct ad5686_state *st = iio_priv(indio_dev);
 
-	return ((st->pwr_down_mode >> (chan->channel * 2)) & 0x3) - 1;
+	guard(mutex)(&st->lock);
+
+	return ((st->pwr_down_mode >> shift) & 0x3U) - 1;
 }
 
 static int ad5686_set_powerdown_mode(struct iio_dev *indio_dev,
 				     const struct iio_chan_spec *chan,
 				     unsigned int mode)
 {
+	unsigned int shift = ad5686_pd_mask_shift(chan);
 	struct ad5686_state *st = iio_priv(indio_dev);
 
-	st->pwr_down_mode &= ~(0x3 << (chan->channel * 2));
-	st->pwr_down_mode |= ((mode + 1) << (chan->channel * 2));
+	guard(mutex)(&st->lock);
+
+	st->pwr_down_mode &= ~(0x3U << shift);
+	st->pwr_down_mode |= (mode + 1) << shift;
 
 	return 0;
 }
@@ -55,10 +70,12 @@ static const struct iio_enum ad5686_powerdown_mode_enum = {
 static ssize_t ad5686_read_dac_powerdown(struct iio_dev *indio_dev,
 		uintptr_t private, const struct iio_chan_spec *chan, char *buf)
 {
+	unsigned int shift = ad5686_pd_mask_shift(chan);
 	struct ad5686_state *st = iio_priv(indio_dev);
 
-	return sysfs_emit(buf, "%d\n", !!(st->pwr_down_mask &
-				       (0x3 << (chan->channel * 2))));
+	guard(mutex)(&st->lock);
+
+	return sysfs_emit(buf, "%d\n", !!(st->pwr_down_mask & (0x3U << shift)));
 }
 
 static ssize_t ad5686_write_dac_powerdown(struct iio_dev *indio_dev,
@@ -77,10 +94,12 @@ static ssize_t ad5686_write_dac_powerdown(struct iio_dev *indio_dev,
 	if (ret)
 		return ret;
 
+	guard(mutex)(&st->lock);
+
 	if (readin)
-		st->pwr_down_mask |= (0x3 << (chan->channel * 2));
+		st->pwr_down_mask |= 0x3U << ad5686_pd_mask_shift(chan);
 	else
-		st->pwr_down_mask &= ~(0x3 << (chan->channel * 2));
+		st->pwr_down_mask &= ~(0x3U << ad5686_pd_mask_shift(chan));
 
 	switch (st->chip_info->regmap_type) {
 	case AD5310_REGMAP:
@@ -154,7 +173,7 @@ static int ad5686_write_raw(struct iio_dev *indio_dev,
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		if (val > (1 << chan->scan_type.realbits) || val < 0)
+		if (val >= (1 << chan->scan_type.realbits) || val < 0)
 			return -EINVAL;
 
 		mutex_lock(&st->lock);
@@ -460,7 +479,7 @@ int ad5686_probe(struct device *dev,
 {
 	struct ad5686_state *st;
 	struct iio_dev *indio_dev;
-	unsigned int val, ref_bit_msk;
+	unsigned int val, ref_bit_msk, shift;
 	bool has_external_vref;
 	u8 cmd;
 	int ret, i;
@@ -484,9 +503,18 @@ int ad5686_probe(struct device *dev,
 	has_external_vref = ret != -ENODEV;
 	st->vref_mv = has_external_vref ? ret / 1000 : st->chip_info->int_vref_mv;
 
+	/* Initialize masks to all ones provided the max shift (last channel) */
+	shift = ad5686_pd_mask_shift(&st->chip_info->channels[st->chip_info->num_channels - 1]);
+	st->pwr_down_mask = GENMASK(shift + 1, 0);
+	st->pwr_down_mode = GENMASK(shift + 1, 0);
+
 	/* Set all the power down mode for all channels to 1K pulldown */
-	for (i = 0; i < st->chip_info->num_channels; i++)
-		st->pwr_down_mode |= (0x01 << (i * 2));
+	for (i = 0; i < st->chip_info->num_channels; i++) {
+		shift = ad5686_pd_mask_shift(&st->chip_info->channels[i]);
+		st->pwr_down_mask &= ~(0x3U << shift); /* powered up state */
+		st->pwr_down_mode &= ~(0x3U << shift);
+		st->pwr_down_mode |= 0x01U << shift;
+	}
 
 	indio_dev->name = name;
 	indio_dev->info = &ad5686_info;
@@ -494,7 +522,9 @@ int ad5686_probe(struct device *dev,
 	indio_dev->channels = st->chip_info->channels;
 	indio_dev->num_channels = st->chip_info->num_channels;
 
-	mutex_init(&st->lock);
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
 
 	switch (st->chip_info->regmap_type) {
 	case AD5310_REGMAP:
@@ -509,7 +539,7 @@ int ad5686_probe(struct device *dev,
 		break;
 	case AD5686_REGMAP:
 		cmd = AD5686_CMD_INTERNAL_REFER_SETUP;
-		ref_bit_msk = 0;
+		ref_bit_msk = AD5686_REF_BIT_MSK;
 		break;
 	case AD5693_REGMAP:
 		cmd = AD5686_CMD_CONTROL_REG;
@@ -520,9 +550,9 @@ int ad5686_probe(struct device *dev,
 		return -EINVAL;
 	}
 
-	val = (has_external_vref | ref_bit_msk);
+	val = has_external_vref ? ref_bit_msk : 0;
 
-	ret = st->write(st, cmd, 0, !!val);
+	ret = st->write(st, cmd, 0, val);
 	if (ret)
 		return ret;
 
diff --git a/drivers/iio/dac/ad5686.h b/drivers/iio/dac/ad5686.h
index e7d36bae3e593..36e16c5c4581b 100644
--- a/drivers/iio/dac/ad5686.h
+++ b/drivers/iio/dac/ad5686.h
@@ -46,6 +46,7 @@
 
 #define AD5310_REF_BIT_MSK			BIT(8)
 #define AD5683_REF_BIT_MSK			BIT(12)
+#define AD5686_REF_BIT_MSK			BIT(0)
 #define AD5693_REF_BIT_MSK			BIT(12)
 
 /**
diff --git a/drivers/iio/dac/ad5706r.c b/drivers/iio/dac/ad5706r.c
new file mode 100644
index 0000000000000..f7872e92dc01d
--- /dev/null
+++ b/drivers/iio/dac/ad5706r.c
@@ -0,0 +1,253 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AD5706R 16-bit Current Output Digital to Analog Converter
+ *
+ * Copyright 2026 Analog Devices Inc.
+ */
+
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/dev_printk.h>
+#include <linux/err.h>
+#include <linux/iio/iio.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
+
+/* SPI frame layout */
+#define AD5706R_RD_MASK			BIT(15)
+#define AD5706R_ADDR_MASK		GENMASK(11, 0)
+
+/* Registers */
+#define AD5706R_REG_DAC_INPUT_A_CH(x)		(0x60 + ((x) * 2))
+#define AD5706R_REG_DAC_DATA_READBACK_CH(x)	(0x68 + ((x) * 2))
+
+#define AD5706R_DAC_RESOLUTION		16
+#define AD5706R_DAC_MAX_CODE		GENMASK(15, 0)
+#define AD5706R_MULTIBYTE_REG_START	0x14
+#define AD5706R_MULTIBYTE_REG_END	0x71
+#define AD5706R_MAX_REG			0x77
+
+struct ad5706r_state {
+	struct spi_device *spi;
+	struct regmap *regmap;
+
+	u8 tx_buf[4] __aligned(IIO_DMA_MINALIGN);
+	u8 rx_buf[4];
+};
+
+static int ad5706r_reg_len(unsigned int reg)
+{
+	if (reg >= AD5706R_MULTIBYTE_REG_START && reg <= AD5706R_MULTIBYTE_REG_END)
+		return 2;
+
+	return 1;
+}
+
+static int ad5706r_regmap_write(void *context, const void *data, size_t count)
+{
+	struct ad5706r_state *st = context;
+	unsigned int num_bytes;
+	u16 reg, val;
+
+	if (count != 4)
+		return -EINVAL;
+
+	reg = get_unaligned_be16(data);
+	val = get_unaligned_be16(data + 2);
+	num_bytes = ad5706r_reg_len(reg);
+
+	struct spi_transfer xfer = {
+		.tx_buf = st->tx_buf,
+		.len = num_bytes + 2,
+	};
+
+	put_unaligned_be16(reg, &st->tx_buf[0]);
+
+	if (num_bytes == 1)
+		st->tx_buf[2] = (u8)val;
+	else if (num_bytes == 2)
+		put_unaligned_be16(val, &st->tx_buf[2]);
+	else
+		return -EINVAL;
+
+	return spi_sync_transfer(st->spi, &xfer, 1);
+}
+
+static int ad5706r_regmap_read(void *context, const void *reg_buf,
+			       size_t reg_size, void *val_buf, size_t val_size)
+{
+	struct ad5706r_state *st = context;
+	unsigned int num_bytes;
+	u16 reg, cmd, val;
+	int ret;
+
+	if (reg_size != 2 || val_size != 2)
+		return -EINVAL;
+
+	reg = get_unaligned_be16(reg_buf);
+	num_bytes = ad5706r_reg_len(reg);
+
+	/* Full duplex, device responds immediately after command */
+	struct spi_transfer xfer = {
+		.tx_buf = st->tx_buf,
+		.rx_buf = st->rx_buf,
+		.len = 2 + num_bytes,
+	};
+
+	cmd = AD5706R_RD_MASK | (reg & AD5706R_ADDR_MASK);
+	put_unaligned_be16(cmd, &st->tx_buf[0]);
+	put_unaligned_be16(0, &st->tx_buf[2]);
+
+	ret = spi_sync_transfer(st->spi, &xfer, 1);
+	if (ret)
+		return ret;
+
+	/* Extract value from response (skip 2-byte command echo) */
+	if (num_bytes == 1)
+		val = st->rx_buf[2];
+	else if (num_bytes == 2)
+		val = get_unaligned_be16(&st->rx_buf[2]);
+	else
+		return -EINVAL;
+
+	put_unaligned_be16(val, val_buf);
+
+	return 0;
+}
+
+static int ad5706r_read_raw(struct iio_dev *indio_dev,
+			    struct iio_chan_spec const *chan,
+			    int *val, int *val2, long mask)
+{
+	struct ad5706r_state *st = iio_priv(indio_dev);
+	unsigned int reg, reg_val;
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		reg = AD5706R_REG_DAC_DATA_READBACK_CH(chan->channel);
+		ret = regmap_read(st->regmap, reg, &reg_val);
+		if (ret)
+			return ret;
+
+		*val = reg_val;
+		return IIO_VAL_INT;
+	case IIO_CHAN_INFO_SCALE:
+		*val = 50;
+		*val2 = AD5706R_DAC_RESOLUTION;
+		return IIO_VAL_FRACTIONAL_LOG2;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int ad5706r_write_raw(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     int val, int val2, long mask)
+{
+	struct ad5706r_state *st = iio_priv(indio_dev);
+	unsigned int reg;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
+		if (val < 0 || val > AD5706R_DAC_MAX_CODE)
+			return -EINVAL;
+
+		reg = AD5706R_REG_DAC_INPUT_A_CH(chan->channel);
+		return regmap_write(st->regmap, reg, val);
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct regmap_bus ad5706r_regmap_bus = {
+	.write = ad5706r_regmap_write,
+	.read = ad5706r_regmap_read,
+	.reg_format_endian_default = REGMAP_ENDIAN_BIG,
+	.val_format_endian_default = REGMAP_ENDIAN_BIG,
+};
+
+static const struct regmap_config ad5706r_regmap_config = {
+	.reg_bits = 16,
+	.val_bits = 16,
+	.max_register = AD5706R_MAX_REG,
+};
+
+static const struct iio_info ad5706r_info = {
+	.read_raw = ad5706r_read_raw,
+	.write_raw = ad5706r_write_raw,
+};
+
+#define AD5706R_CHAN(_channel) {				\
+	.type = IIO_CURRENT,					\
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW) |		\
+			      BIT(IIO_CHAN_INFO_SCALE),		\
+	.output = 1,						\
+	.indexed = 1,						\
+	.channel = _channel,					\
+}
+
+static const struct iio_chan_spec ad5706r_channels[] = {
+	AD5706R_CHAN(0),
+	AD5706R_CHAN(1),
+	AD5706R_CHAN(2),
+	AD5706R_CHAN(3),
+};
+
+static int ad5706r_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct iio_dev *indio_dev;
+	struct ad5706r_state *st;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*st));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	st = iio_priv(indio_dev);
+	st->spi = spi;
+
+	st->regmap = devm_regmap_init(dev, &ad5706r_regmap_bus,
+				      st, &ad5706r_regmap_config);
+	if (IS_ERR(st->regmap))
+		return dev_err_probe(dev, PTR_ERR(st->regmap),
+				     "Failed to init regmap\n");
+
+	indio_dev->name = "ad5706r";
+	indio_dev->info = &ad5706r_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->channels = ad5706r_channels;
+	indio_dev->num_channels = ARRAY_SIZE(ad5706r_channels);
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct of_device_id ad5706r_of_match[] = {
+	{ .compatible = "adi,ad5706r" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, ad5706r_of_match);
+
+static const struct spi_device_id ad5706r_id[] = {
+	{ "ad5706r" },
+	{ }
+};
+MODULE_DEVICE_TABLE(spi, ad5706r_id);
+
+static struct spi_driver ad5706r_driver = {
+	.driver = {
+		.name = "ad5706r",
+		.of_match_table = ad5706r_of_match,
+	},
+	.probe = ad5706r_probe,
+	.id_table = ad5706r_id,
+};
+module_spi_driver(ad5706r_driver);
+
+MODULE_AUTHOR("Alexis Czezar Torreno <alexisczezar.torreno@analog.com>");
+MODULE_DESCRIPTION("AD5706R 16-bit Current Output DAC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/dac/ad5755.c b/drivers/iio/dac/ad5755.c
index d0e5f35462b1b..cc6d56140d667 100644
--- a/drivers/iio/dac/ad5755.c
+++ b/drivers/iio/dac/ad5755.c
@@ -827,8 +827,9 @@ static int ad5755_probe(struct spi_device *spi)
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->num_channels = AD5755_NUM_CHANNELS;
 
-	mutex_init(&st->lock);
-
+	ret = devm_mutex_init(&spi->dev, &st->lock);
+	if (ret)
+		return ret;
 
 	pdata = ad5755_parse_fw(&spi->dev);
 	if (!pdata) {
diff --git a/drivers/iio/dac/ad5758.c b/drivers/iio/dac/ad5758.c
index 4ed4fda76ea97..8e6fb46cce4dd 100644
--- a/drivers/iio/dac/ad5758.c
+++ b/drivers/iio/dac/ad5758.c
@@ -851,7 +851,9 @@ static int ad5758_probe(struct spi_device *spi)
 
 	st->spi = spi;
 
-	mutex_init(&st->lock);
+	ret = devm_mutex_init(&spi->dev, &st->lock);
+	if (ret)
+		return ret;
 
 	indio_dev->name = spi_get_device_id(spi)->name;
 	indio_dev->info = &ad5758_info;
diff --git a/drivers/iio/dac/ad7303.c b/drivers/iio/dac/ad7303.c
index a88cc639047df..1c2960fa97432 100644
--- a/drivers/iio/dac/ad7303.c
+++ b/drivers/iio/dac/ad7303.c
@@ -218,7 +218,9 @@ static int ad7303_probe(struct spi_device *spi)
 
 	st->spi = spi;
 
-	mutex_init(&st->lock);
+	ret = devm_mutex_init(&spi->dev, &st->lock);
+	if (ret)
+		return ret;
 
 	st->vdd_reg = devm_regulator_get(&spi->dev, "Vdd");
 	if (IS_ERR(st->vdd_reg))
diff --git a/drivers/iio/dac/ltc2664.c b/drivers/iio/dac/ltc2664.c
index 67f14046cf773..616806615d3de 100644
--- a/drivers/iio/dac/ltc2664.c
+++ b/drivers/iio/dac/ltc2664.c
@@ -675,7 +675,9 @@ static int ltc2664_probe(struct spi_device *spi)
 
 	st->chip_info = chip_info;
 
-	mutex_init(&st->lock);
+	ret = devm_mutex_init(dev, &st->lock);
+	if (ret)
+		return ret;
 
 	st->regmap = devm_regmap_init_spi(spi, &ltc2664_regmap_config);
 	if (IS_ERR(st->regmap))
diff --git a/drivers/iio/dac/max5821.c b/drivers/iio/dac/max5821.c
index e7e29359f8fe5..dd4e35460195a 100644
--- a/drivers/iio/dac/max5821.c
+++ b/drivers/iio/dac/max5821.c
@@ -90,6 +90,7 @@ static int max5821_sync_powerdown_mode(struct max5821_data *data,
 				       const struct iio_chan_spec *chan)
 {
 	u8 outbuf[2];
+	int ret;
 
 	outbuf[0] = MAX5821_EXTENDED_COMMAND_MODE;
 
@@ -103,7 +104,13 @@ static int max5821_sync_powerdown_mode(struct max5821_data *data,
 	else
 		outbuf[1] |= MAX5821_EXTENDED_POWER_UP;
 
-	return i2c_master_send(data->client, outbuf, 2);
+	ret = i2c_master_send(data->client, outbuf, sizeof(outbuf));
+	if (ret < 0)
+		return ret;
+	if (ret != sizeof(outbuf))
+		return -EIO;
+
+	return 0;
 }
 
 static ssize_t max5821_write_dac_powerdown(struct iio_dev *indio_dev,
diff --git a/drivers/iio/dac/mcp4821.c b/drivers/iio/dac/mcp4821.c
index 748bdca9a964d..18b5934fb8a20 100644
--- a/drivers/iio/dac/mcp4821.c
+++ b/drivers/iio/dac/mcp4821.c
@@ -12,13 +12,13 @@
  *	MCP48x2: https://ww1.microchip.com/downloads/en/DeviceDoc/20002249B.pdf
  *
  * TODO:
- *	- Configurable gain
  *	- Regulator control
  */
 
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
 #include <linux/spi/spi.h>
+#include <linux/units.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/types.h>
@@ -26,12 +26,32 @@
 #include <linux/unaligned.h>
 
 #define MCP4821_ACTIVE_MODE BIT(12)
+#define MCP4821_GAIN_ENABLE BIT(13)
 #define MCP4802_SECOND_CHAN BIT(15)
 
-/* DAC uses an internal Voltage reference of 4.096V at a gain of 2x */
-#define MCP4821_2X_GAIN_VREF_MV 4096
+/* DAC uses an internal Voltage reference of 2.048V */
+#define MCP4821_VREF_MV 2048
 
-enum mcp4821_supported_drvice_ids {
+/*
+ * MCP48xx DAC output:
+ *
+ * Vout = (Vref * D / 2^N) * G
+ *
+ * where:
+ *  - Vref = 2.048V (internal reference)
+ *  - N = DAC resolution (12 bits for MCP4821)
+ *  - G = gain selection:
+ *        1x when GA bit = 1
+ *        2x when GA bit = 0 (default)
+ *
+ * Therefore full-scale voltage is:
+ *  - 1x gain: 2.048V
+ *  - 2x gain: 4.096V
+ *
+ * Scale = Vfull-scale / 2^N
+ */
+
+enum mcp4821_supported_device_ids {
 	ID_MCP4801,
 	ID_MCP4802,
 	ID_MCP4811,
@@ -43,6 +63,8 @@ enum mcp4821_supported_drvice_ids {
 struct mcp4821_state {
 	struct spi_device *spi;
 	u16 dac_value[2];
+	int gain;
+	int scale_avail[4];
 };
 
 struct mcp4821_chip_info {
@@ -57,6 +79,7 @@ struct mcp4821_chip_info {
 		.channel = (channel_id),                              \
 		.info_mask_separate = BIT(IIO_CHAN_INFO_RAW),         \
 		.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
+		.info_mask_shared_by_type_available = BIT(IIO_CHAN_INFO_SCALE), \
 		.scan_type = {                                        \
 			.realbits = (resolution),                     \
 			.shift = 12 - (resolution),                   \
@@ -115,15 +138,15 @@ static int mcp4821_read_raw(struct iio_dev *indio_dev,
 			    struct iio_chan_spec const *chan, int *val,
 			    int *val2, long mask)
 {
-	struct mcp4821_state *state;
+	struct mcp4821_state *state = iio_priv(indio_dev);
 
 	switch (mask) {
 	case IIO_CHAN_INFO_RAW:
-		state = iio_priv(indio_dev);
 		*val = state->dac_value[chan->channel];
 		return IIO_VAL_INT;
+
 	case IIO_CHAN_INFO_SCALE:
-		*val = MCP4821_2X_GAIN_VREF_MV;
+		*val = MCP4821_VREF_MV * state->gain;
 		*val2 = chan->scan_type.realbits;
 		return IIO_VAL_FRACTIONAL_LOG2;
 	default:
@@ -131,6 +154,17 @@ static int mcp4821_read_raw(struct iio_dev *indio_dev,
 	}
 }
 
+static void mcp4821_calc_scale(int vref_mv, int resolution,
+				int *val, int *val2)
+{
+	s64 tmp;
+	int micro;
+
+	tmp = (s64)vref_mv * MICRO >> resolution;
+	*val = div_s64_rem(tmp, MICRO, &micro);
+	*val2 = micro;
+}
+
 static int mcp4821_write_raw(struct iio_dev *indio_dev,
 			     struct iio_chan_spec const *chan, int val,
 			     int val2, long mask)
@@ -139,35 +173,85 @@ static int mcp4821_write_raw(struct iio_dev *indio_dev,
 	u16 write_val;
 	__be16 write_buffer;
 	int ret;
+	int v, v2;
 
-	if (val2 != 0)
-		return -EINVAL;
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW:
 
-	if (val < 0 || val >= BIT(chan->scan_type.realbits))
-		return -EINVAL;
+		if (val2 != 0)
+			return -EINVAL;
 
-	if (mask != IIO_CHAN_INFO_RAW)
-		return -EINVAL;
+		if (val < 0 || val >= BIT(chan->scan_type.realbits))
+			return -EINVAL;
 
-	write_val = MCP4821_ACTIVE_MODE | val << chan->scan_type.shift;
-	if (chan->channel)
-		write_val |= MCP4802_SECOND_CHAN;
+		write_val = MCP4821_ACTIVE_MODE | val << chan->scan_type.shift;
+		if (chan->channel)
+			write_val |= MCP4802_SECOND_CHAN;
 
-	write_buffer = cpu_to_be16(write_val);
-	ret = spi_write(state->spi, &write_buffer, sizeof(write_buffer));
-	if (ret) {
-		dev_err(&state->spi->dev, "Failed to write to device: %d", ret);
-		return ret;
+		/* GA bit = 1 -> 1x gain */
+		if (state->gain == 1)
+			write_val |= MCP4821_GAIN_ENABLE;
+
+		write_buffer = cpu_to_be16(write_val);
+		ret = spi_write(state->spi, &write_buffer, sizeof(write_buffer));
+		if (ret) {
+			dev_err(&state->spi->dev, "Failed to write to device: %d", ret);
+			return ret;
+		}
+
+		state->dac_value[chan->channel] = val;
+		return 0;
+
+	case IIO_CHAN_INFO_SCALE:
+		mcp4821_calc_scale(MCP4821_VREF_MV, chan->scan_type.realbits, &v, &v2);
+		if (val == v && val2 == v2) {
+			state->gain = 1;
+			return 0;
+		}
+
+		mcp4821_calc_scale(MCP4821_VREF_MV * 2,
+				chan->scan_type.realbits, &v, &v2);
+		if (val == v && val2 == v2) {
+			state->gain = 2;
+			return 0;
+		}
+		return -EINVAL;
+	default:
+		return -EINVAL;
 	}
+}
 
-	state->dac_value[chan->channel] = val;
+static inline void mcp4821_init_avail_gain(struct mcp4821_state *state,
+				int resolution)
+{
+	state->scale_avail[0] = MCP4821_VREF_MV;
+	state->scale_avail[1] = resolution;
+	state->scale_avail[2] = MCP4821_VREF_MV * 2;
+	state->scale_avail[3] = resolution;
+}
+
+static int mcp4821_read_avail(struct iio_dev *indio_dev,
+			     struct iio_chan_spec const *chan,
+			     const int **vals, int *type, int *length,
+			     long info)
+{
+	struct mcp4821_state *state = iio_priv(indio_dev);
 
-	return 0;
+	switch (info) {
+	case IIO_CHAN_INFO_SCALE:
+		*vals = state->scale_avail;
+		*type = IIO_VAL_FRACTIONAL_LOG2;
+		*length = ARRAY_SIZE(state->scale_avail);
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
 }
 
 static const struct iio_info mcp4821_info = {
 	.read_raw = &mcp4821_read_raw,
 	.write_raw = &mcp4821_write_raw,
+	.read_avail = &mcp4821_read_avail,
 };
 
 static int mcp4821_probe(struct spi_device *spi)
@@ -183,12 +267,15 @@ static int mcp4821_probe(struct spi_device *spi)
 	state = iio_priv(indio_dev);
 	state->spi = spi;
 
+	/* default gain is 2x */
+	state->gain = 2;
 	info = spi_get_device_match_data(spi);
 	indio_dev->name = info->name;
 	indio_dev->info = &mcp4821_info;
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->channels = info->channels;
 	indio_dev->num_channels = info->num_channels;
+	mcp4821_init_avail_gain(state, info->channels[0].scan_type.realbits);
 
 	return devm_iio_device_register(&spi->dev, indio_dev);
 }
diff --git a/drivers/iio/frequency/adrf6780.c b/drivers/iio/frequency/adrf6780.c
index 257fd31a0b0e6..9911b5273b22e 100644
--- a/drivers/iio/frequency/adrf6780.c
+++ b/drivers/iio/frequency/adrf6780.c
@@ -188,8 +188,11 @@ static int adrf6780_read_adc_raw(struct adrf6780_state *st, unsigned int *read_v
 	if (ret)
 		goto exit;
 
-	/* Recommended delay for the ADC to be ready*/
-	usleep_range(200, 250);
+	/*
+	 * Per ADRF6780 datasheet (Rev. D, page 23, ADC section),
+	 * wait approximately 200 us for the ADC to be ready.
+	 */
+	fsleep(200);
 
 	ret = __adrf6780_spi_read(st, ADRF6780_REG_ADC_OUTPUT, read_val);
 	if (ret)
diff --git a/drivers/iio/gyro/adis16260.c b/drivers/iio/gyro/adis16260.c
index 586e6cfa14a95..91b9c5f18ec40 100644
--- a/drivers/iio/gyro/adis16260.c
+++ b/drivers/iio/gyro/adis16260.c
@@ -287,6 +287,9 @@ static int adis16260_write_raw(struct iio_dev *indio_dev,
 		addr = adis16260_addresses[chan->scan_index][1];
 		return adis_write_reg_16(adis, addr, val);
 	case IIO_CHAN_INFO_SAMP_FREQ:
+		if (val <= 0)
+			return -EINVAL;
+
 		if (spi_get_device_id(adis->spi)->driver_data)
 			t = 256 / val;
 		else
diff --git a/drivers/iio/gyro/adxrs290.c b/drivers/iio/gyro/adxrs290.c
index 3efe385ebedc4..35928383d4f25 100644
--- a/drivers/iio/gyro/adxrs290.c
+++ b/drivers/iio/gyro/adxrs290.c
@@ -8,6 +8,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/bitops.h>
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/kernel.h>
@@ -115,65 +116,53 @@ static const int adxrs290_hpf_3db_freq_hz_table[][2] = {
 static int adxrs290_get_rate_data(struct iio_dev *indio_dev, const u8 cmd, int *val)
 {
 	struct adxrs290_state *st = iio_priv(indio_dev);
-	int ret = 0;
 	int temp;
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
+
 	temp = spi_w8r16(st->spi, cmd);
-	if (temp < 0) {
-		ret = temp;
-		goto err_unlock;
-	}
+	if (temp < 0)
+		return temp;
 
 	*val = sign_extend32(temp, 15);
 
-err_unlock:
-	mutex_unlock(&st->lock);
-	return ret;
+	return 0;
 }
 
 static int adxrs290_get_temp_data(struct iio_dev *indio_dev, int *val)
 {
 	const u8 cmd = ADXRS290_READ_REG(ADXRS290_REG_TEMP0);
 	struct adxrs290_state *st = iio_priv(indio_dev);
-	int ret = 0;
 	int temp;
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
+
 	temp = spi_w8r16(st->spi, cmd);
-	if (temp < 0) {
-		ret = temp;
-		goto err_unlock;
-	}
+	if (temp < 0)
+		return temp;
 
 	/* extract lower 12 bits temperature reading */
 	*val = sign_extend32(temp, 11);
 
-err_unlock:
-	mutex_unlock(&st->lock);
-	return ret;
+	return 0;
 }
 
 static int adxrs290_get_3db_freq(struct iio_dev *indio_dev, u8 *val, u8 *val2)
 {
 	const u8 cmd = ADXRS290_READ_REG(ADXRS290_REG_FILTER);
 	struct adxrs290_state *st = iio_priv(indio_dev);
-	int ret = 0;
 	short temp;
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
+
 	temp = spi_w8r8(st->spi, cmd);
-	if (temp < 0) {
-		ret = temp;
-		goto err_unlock;
-	}
+	if (temp < 0)
+		return temp;
 
 	*val = FIELD_GET(ADXRS290_LPF_MASK, temp);
 	*val2 = FIELD_GET(ADXRS290_HPF_MASK, temp);
 
-err_unlock:
-	mutex_unlock(&st->lock);
-	return ret;
+	return 0;
 }
 
 static int adxrs290_spi_write_reg(struct spi_device *spi, const u8 reg,
@@ -220,11 +209,11 @@ static int adxrs290_set_mode(struct iio_dev *indio_dev, enum adxrs290_mode mode)
 	if (st->mode == mode)
 		return 0;
 
-	mutex_lock(&st->lock);
+	guard(mutex)(&st->lock);
 
 	ret = spi_w8r8(st->spi, ADXRS290_READ_REG(ADXRS290_REG_POWER_CTL));
 	if (ret < 0)
-		goto out_unlock;
+		return ret;
 
 	val = ret;
 
@@ -236,21 +225,18 @@ static int adxrs290_set_mode(struct iio_dev *indio_dev, enum adxrs290_mode mode)
 		val |= ADXRS290_MEASUREMENT;
 		break;
 	default:
-		ret = -EINVAL;
-		goto out_unlock;
+		return -EINVAL;
 	}
 
 	ret = adxrs290_spi_write_reg(st->spi, ADXRS290_REG_POWER_CTL, val);
 	if (ret < 0) {
 		dev_err(&st->spi->dev, "unable to set mode: %d\n", ret);
-		goto out_unlock;
+		return ret;
 	}
 
 	/* update cached mode */
 	st->mode = mode;
 
-out_unlock:
-	mutex_unlock(&st->lock);
 	return ret;
 }
 
@@ -506,19 +492,20 @@ static irqreturn_t adxrs290_trigger_handler(int irq, void *p)
 	u8 tx = ADXRS290_READ_REG(ADXRS290_REG_DATAX0);
 	int ret;
 
-	mutex_lock(&st->lock);
+	do {
+		guard(mutex)(&st->lock);
 
-	/* exercise a bulk data capture starting from reg DATAX0... */
-	ret = spi_write_then_read(st->spi, &tx, sizeof(tx), st->buffer.channels,
-				  sizeof(st->buffer.channels));
-	if (ret < 0)
-		goto out_unlock_notify;
+		/* exercise a bulk data capture starting from reg DATAX0... */
+		ret = spi_write_then_read(st->spi, &tx, sizeof(tx),
+					  st->buffer.channels,
+					  sizeof(st->buffer.channels));
+		if (ret < 0)
+			break;
 
-	iio_push_to_buffers_with_timestamp(indio_dev, &st->buffer,
-					   pf->timestamp);
+		iio_push_to_buffers_with_timestamp(indio_dev, &st->buffer,
+						   pf->timestamp);
+	} while (0);
 
-out_unlock_notify:
-	mutex_unlock(&st->lock);
 	iio_trigger_notify_done(indio_dev->trig);
 
 	return IRQ_HANDLED;
diff --git a/drivers/iio/gyro/bmg160_core.c b/drivers/iio/gyro/bmg160_core.c
index 38394b5f32755..d611341a0e2ad 100644
--- a/drivers/iio/gyro/bmg160_core.c
+++ b/drivers/iio/gyro/bmg160_core.c
@@ -201,6 +201,9 @@ static int bmg160_get_filter(struct bmg160_data *data, int *val)
 			break;
 	}
 
+	if (i == ARRAY_SIZE(bmg160_samp_freq_table))
+		return -EINVAL;
+
 	*val = bmg160_samp_freq_table[i].filter;
 
 	return ret ? ret : IIO_VAL_INT;
@@ -218,6 +221,9 @@ static int bmg160_set_filter(struct bmg160_data *data, int val)
 			break;
 	}
 
+	if (i == ARRAY_SIZE(bmg160_samp_freq_table))
+		return -EINVAL;
+
 	ret = regmap_write(data->regmap, BMG160_REG_PMU_BW,
 			   bmg160_samp_freq_table[i].bw_bits);
 	if (ret < 0) {
@@ -258,8 +264,14 @@ static int bmg160_chip_init(struct bmg160_data *data)
 	if (ret < 0)
 		return ret;
 
-	/* Wait upto 500 ms to be ready after changing mode */
-	usleep_range(500, 1000);
+	/*
+	 * Wait for the chip to be ready after switching to normal mode.
+	 * The BMG160 datasheet (BST-BMG160-DS000-07 Rev. 1.0, May 2013)
+	 * specifies a start-up / wake-up time (tsu, twusm) of 30 ms; use
+	 * BMG160_MAX_STARTUP_TIME_MS (80 ms) as a safety margin, matching
+	 * what bmg160_runtime_resume() already does.
+	 */
+	msleep(BMG160_MAX_STARTUP_TIME_MS);
 
 	/* Set Bandwidth */
 	ret = bmg160_set_bw(data, BMG160_DEF_BW);
diff --git a/drivers/iio/gyro/itg3200_buffer.c b/drivers/iio/gyro/itg3200_buffer.c
index cf97adfa97274..87efa2c74ca4e 100644
--- a/drivers/iio/gyro/itg3200_buffer.c
+++ b/drivers/iio/gyro/itg3200_buffer.c
@@ -34,7 +34,7 @@ static int itg3200_read_all_channels(struct i2c_client *i2c, __be16 *buf)
 			.addr = i2c->addr,
 			.flags = i2c->flags | I2C_M_RD,
 			.len = ITG3200_SCAN_ELEMENTS * sizeof(s16),
-			.buf = (char *)&buf,
+			.buf = (char *)buf,
 		},
 	};
 
diff --git a/drivers/iio/imu/adis16550.c b/drivers/iio/imu/adis16550.c
index 1f2af506f4bdd..75679612052f2 100644
--- a/drivers/iio/imu/adis16550.c
+++ b/drivers/iio/imu/adis16550.c
@@ -836,7 +836,7 @@ static irqreturn_t adis16550_trigger_handler(int irq, void *p)
 	u16 dummy;
 	bool valid;
 	struct iio_poll_func *pf = p;
-	__be32 data[ADIS16550_MAX_SCAN_DATA] __aligned(8);
+	__be32 data[ADIS16550_MAX_SCAN_DATA] __aligned(8) = { };
 	struct iio_dev *indio_dev = pf->indio_dev;
 	struct adis16550 *st = iio_priv(indio_dev);
 	struct adis *adis = iio_device_get_drvdata(indio_dev);
diff --git a/drivers/iio/imu/bno055/bno055.c b/drivers/iio/imu/bno055/bno055.c
index c96fec2ebb3e7..d0101607a9b3a 100644
--- a/drivers/iio/imu/bno055/bno055.c
+++ b/drivers/iio/imu/bno055/bno055.c
@@ -210,9 +210,30 @@ struct bno055_priv {
 	u8 uid[BNO055_UID_LEN];
 	struct gpio_desc *reset_gpio;
 	bool sw_reset;
-	struct {
-		__le16 chans[BNO055_SCAN_CH_COUNT];
-		aligned_s64 timestamp;
+	union {
+		IIO_DECLARE_BUFFER_WITH_TS(__le16, chans, BNO055_SCAN_CH_COUNT);
+		/*
+		 * This struct is not used, but it is here to ensure proper size
+		 * and alignment of the scan buffer above (because of the extra
+		 * requirements of the quaternion field). Technically it is not
+		 * needed in this case, because other fields just happen to make
+		 * things correctly aligned already. But it is better to be
+		 * explicit about the requirements anyway. The actual contents
+		 * of the scan buffer will vary depending on which channels are
+		 * enabled.
+		 */
+		struct {
+			__le16 acc[3];
+			__le16 magn[3];
+			__le16 gyr[3];
+			__le16 yaw;
+			__le16 pitch;
+			__le16 roll;
+			IIO_DECLARE_QUATERNION(__le16, quaternion);
+			__le16 lia[3];
+			__le16 gravity[3];
+			aligned_s64 timestamp;
+		};
 	} buf;
 	struct dentry *debugfs;
 };
diff --git a/drivers/iio/imu/st_lsm6dsx/Makefile b/drivers/iio/imu/st_lsm6dsx/Makefile
index 57cbcd67d64ff..19a488254de34 100644
--- a/drivers/iio/imu/st_lsm6dsx/Makefile
+++ b/drivers/iio/imu/st_lsm6dsx/Makefile
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0-only
 st_lsm6dsx-y := st_lsm6dsx_core.o st_lsm6dsx_buffer.o \
-		st_lsm6dsx_shub.o
+		st_lsm6dsx_shub.o st_lsm6dsx_fusion.o
 
 obj-$(CONFIG_IIO_ST_LSM6DSX) += st_lsm6dsx.o
 obj-$(CONFIG_IIO_ST_LSM6DSX_I2C) += st_lsm6dsx_i2c.o
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
index 07b1773c87bdb..767aadfe70617 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h
@@ -294,6 +294,7 @@ enum st_lsm6dsx_sensor_id {
 	ST_LSM6DSX_ID_EXT0,
 	ST_LSM6DSX_ID_EXT1,
 	ST_LSM6DSX_ID_EXT2,
+	ST_LSM6DSX_ID_FUSION,
 	ST_LSM6DSX_ID_MAX
 };
 
@@ -301,6 +302,17 @@ enum st_lsm6dsx_ext_sensor_id {
 	ST_LSM6DSX_ID_MAGN,
 };
 
+struct st_lsm6dsx_fusion_settings {
+	const struct iio_chan_spec *chan;
+	int chan_len;
+	struct st_lsm6dsx_reg odr_reg;
+	int odr_hz[ST_LSM6DSX_ODR_LIST_SIZE];
+	int odr_len;
+	struct st_lsm6dsx_reg fifo_enable;
+	struct st_lsm6dsx_reg page_mux;
+	struct st_lsm6dsx_reg enable;
+};
+
 /**
  * struct st_lsm6dsx_ext_dev_settings - i2c controller slave settings
  * @i2c_addr: I2c slave address list.
@@ -388,6 +400,7 @@ struct st_lsm6dsx_settings {
 	struct st_lsm6dsx_hw_ts_settings ts_settings;
 	struct st_lsm6dsx_shub_settings shub_settings;
 	struct st_lsm6dsx_event_settings event_settings;
+	struct st_lsm6dsx_fusion_settings fusion_settings;
 };
 
 enum st_lsm6dsx_fifo_mode {
@@ -510,6 +523,9 @@ int st_lsm6dsx_check_odr(struct st_lsm6dsx_sensor *sensor, u32 odr, u8 *val);
 int st_lsm6dsx_shub_probe(struct st_lsm6dsx_hw *hw, const char *name);
 int st_lsm6dsx_shub_set_enable(struct st_lsm6dsx_sensor *sensor, bool enable);
 int st_lsm6dsx_shub_read_output(struct st_lsm6dsx_hw *hw, u8 *data, int len);
+int st_lsm6dsx_fusion_probe(struct st_lsm6dsx_hw *hw, const char *name);
+int st_lsm6dsx_fusion_set_enable(struct st_lsm6dsx_sensor *sensor, bool enable);
+int st_lsm6dsx_fusion_set_odr(struct st_lsm6dsx_sensor *sensor, bool enable);
 int st_lsm6dsx_set_page(struct st_lsm6dsx_hw *hw, bool enable);
 
 static inline int
@@ -564,12 +580,14 @@ st_lsm6dsx_get_mount_matrix(const struct iio_dev *iio_dev,
 static inline int
 st_lsm6dsx_device_set_enable(struct st_lsm6dsx_sensor *sensor, bool enable)
 {
-	if (sensor->id == ST_LSM6DSX_ID_EXT0 ||
-	    sensor->id == ST_LSM6DSX_ID_EXT1 ||
-	    sensor->id == ST_LSM6DSX_ID_EXT2)
+	switch (sensor->id) {
+	case ST_LSM6DSX_ID_EXT0 ... ST_LSM6DSX_ID_EXT2:
 		return st_lsm6dsx_shub_set_enable(sensor, enable);
-
-	return st_lsm6dsx_sensor_set_enable(sensor, enable);
+	case ST_LSM6DSX_ID_FUSION:
+		return st_lsm6dsx_fusion_set_enable(sensor, enable);
+	default:
+		return st_lsm6dsx_sensor_set_enable(sensor, enable);
+	}
 }
 
 static const
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
index 5b28a3ffcc3d0..777d479f10c0e 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_buffer.c
@@ -88,6 +88,7 @@ enum st_lsm6dsx_fifo_tag {
 	ST_LSM6DSX_EXT0_TAG = 0x0f,
 	ST_LSM6DSX_EXT1_TAG = 0x10,
 	ST_LSM6DSX_EXT2_TAG = 0x11,
+	ST_LSM6DSX_ROT_TAG = 0x13,
 };
 
 static const
@@ -226,8 +227,11 @@ static int st_lsm6dsx_set_fifo_odr(struct st_lsm6dsx_sensor *sensor,
 	u8 data;
 
 	/* Only internal sensors have a FIFO ODR configuration register. */
-	if (sensor->id >= ARRAY_SIZE(hw->settings->batch))
+	if (sensor->id >= ARRAY_SIZE(hw->settings->batch)) {
+		if (sensor->id == ST_LSM6DSX_ID_FUSION)
+			return st_lsm6dsx_fusion_set_odr(sensor, enable);
 		return 0;
+	}
 
 	batch_reg = &hw->settings->batch[sensor->id];
 	if (batch_reg->addr) {
@@ -365,8 +369,6 @@ static inline int st_lsm6dsx_read_block(struct st_lsm6dsx_hw *hw, u8 addr,
 	return 0;
 }
 
-#define ST_LSM6DSX_IIO_BUFF_SIZE	(ALIGN(ST_LSM6DSX_SAMPLE_SIZE, \
-					       sizeof(s64)) + sizeof(s64))
 /**
  * st_lsm6dsx_read_fifo() - hw FIFO read routine
  * @hw: Pointer to instance of struct st_lsm6dsx_hw.
@@ -537,16 +539,23 @@ int st_lsm6dsx_read_fifo(struct st_lsm6dsx_hw *hw)
 }
 
 #define ST_LSM6DSX_INVALID_SAMPLE	0x7ffd
-static int
-st_lsm6dsx_push_tagged_data(struct st_lsm6dsx_hw *hw, u8 tag,
-			    u8 *data, s64 ts)
+static bool st_lsm6dsx_check_data(u8 tag, __le16 *data)
+{
+	if ((tag == ST_LSM6DSX_GYRO_TAG || tag == ST_LSM6DSX_ACC_TAG) &&
+	    (s16)le16_to_cpup(data) >= ST_LSM6DSX_INVALID_SAMPLE)
+		return false;
+
+	return true;
+}
+
+static int st_lsm6dsx_push_tagged_data(struct st_lsm6dsx_hw *hw, u8 tag,
+				       __le16 *data, s64 ts)
 {
-	s16 val = le16_to_cpu(*(__le16 *)data);
 	struct st_lsm6dsx_sensor *sensor;
 	struct iio_dev *iio_dev;
 
 	/* invalid sample during bootstrap phase */
-	if (val >= ST_LSM6DSX_INVALID_SAMPLE)
+	if (!st_lsm6dsx_check_data(tag, data))
 		return -EINVAL;
 
 	/*
@@ -580,6 +589,9 @@ st_lsm6dsx_push_tagged_data(struct st_lsm6dsx_hw *hw, u8 tag,
 	case ST_LSM6DSX_EXT2_TAG:
 		iio_dev = hw->iio_devs[ST_LSM6DSX_ID_EXT2];
 		break;
+	case ST_LSM6DSX_ROT_TAG:
+		iio_dev = hw->iio_devs[ST_LSM6DSX_ID_FUSION];
+		break;
 	default:
 		return -EINVAL;
 	}
@@ -609,7 +621,13 @@ int st_lsm6dsx_read_tagged_fifo(struct st_lsm6dsx_hw *hw)
 	 * must be passed a buffer that is aligned to 8 bytes so
 	 * as to allow insertion of a naturally aligned timestamp.
 	 */
-	u8 iio_buff[ST_LSM6DSX_IIO_BUFF_SIZE] __aligned(8);
+	struct {
+		union {
+			__le16 data[3];
+			__le32 fifo_ts;
+		};
+		aligned_s64 timestamp;
+	} iio_buff = { };
 	u8 tag;
 	bool reset_ts = false;
 	int i, err, read_len;
@@ -648,7 +666,7 @@ int st_lsm6dsx_read_tagged_fifo(struct st_lsm6dsx_hw *hw)
 
 		for (i = 0; i < pattern_len;
 		     i += ST_LSM6DSX_TAGGED_SAMPLE_SIZE) {
-			memcpy(iio_buff, &hw->buff[i + ST_LSM6DSX_TAG_SIZE],
+			memcpy(&iio_buff, &hw->buff[i + ST_LSM6DSX_TAG_SIZE],
 			       ST_LSM6DSX_SAMPLE_SIZE);
 
 			tag = hw->buff[i] >> 3;
@@ -659,7 +677,7 @@ int st_lsm6dsx_read_tagged_fifo(struct st_lsm6dsx_hw *hw)
 				 * B0 = ts[7:0], B1 = ts[15:8], B2 = ts[23:16],
 				 * B3 = ts[31:24]
 				 */
-				ts = le32_to_cpu(*((__le32 *)iio_buff));
+				ts = le32_to_cpu(iio_buff.fifo_ts);
 				/*
 				 * check if hw timestamp engine is going to
 				 * reset (the sensor generates an interrupt
@@ -670,7 +688,8 @@ int st_lsm6dsx_read_tagged_fifo(struct st_lsm6dsx_hw *hw)
 					reset_ts = true;
 				ts *= hw->ts_gain;
 			} else {
-				st_lsm6dsx_push_tagged_data(hw, tag, iio_buff,
+				st_lsm6dsx_push_tagged_data(hw, tag,
+							    iio_buff.data,
 							    ts);
 			}
 		}
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
index 450cb5b47346e..630e2cae6f19e 100644
--- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c
@@ -94,6 +94,26 @@
 
 #define ST_LSM6DSX_REG_WHOAMI_ADDR		0x0f
 
+/* Raw values from the IMU are 16-bit half-precision floating-point numbers. */
+#define ST_LSM6DSX_CHANNEL_ROT						\
+{									\
+	.type = IIO_ROT,						\
+	.modified = 1,							\
+	.channel2 = IIO_MOD_QUATERNION_AXIS,				\
+	.info_mask_shared_by_all = BIT(IIO_CHAN_INFO_SAMP_FREQ),	\
+	.info_mask_shared_by_all_available =				\
+		BIT(IIO_CHAN_INFO_SAMP_FREQ),				\
+	.scan_index = 0,						\
+	.scan_type = {							\
+		.format = IIO_SCAN_FORMAT_FLOAT,			\
+		.realbits = 16,						\
+		.storagebits = 16,					\
+		.endianness = IIO_LE,					\
+		.repeat = 3,						\
+	},								\
+	.ext_info = st_lsm6dsx_ext_info,				\
+}
+
 static const struct iio_event_spec st_lsm6dsx_ev_motion[] = {
 	{
 		.type = IIO_EV_TYPE_THRESH,
@@ -153,6 +173,11 @@ static const struct iio_chan_spec st_lsm6ds0_gyro_channels[] = {
 	IIO_CHAN_SOFT_TIMESTAMP(3),
 };
 
+static const struct iio_chan_spec st_lsm6dsx_fusion_channels[] = {
+	ST_LSM6DSX_CHANNEL_ROT,
+	IIO_CHAN_SOFT_TIMESTAMP(1),
+};
+
 static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 	{
 		.reset = {
@@ -1492,6 +1517,33 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = {
 				},
 			},
 		},
+		.fusion_settings = {
+			.chan = st_lsm6dsx_fusion_channels,
+			.chan_len = ARRAY_SIZE(st_lsm6dsx_fusion_channels),
+			.odr_reg = {
+				.addr = 0x5e,
+				.mask = GENMASK(5, 3),
+			},
+			.odr_hz[0] = 15,
+			.odr_hz[1] = 30,
+			.odr_hz[2] = 60,
+			.odr_hz[3] = 120,
+			.odr_hz[4] = 240,
+			.odr_hz[5] = 480,
+			.odr_len = 6,
+			.fifo_enable = {
+				.addr = 0x44,
+				.mask = BIT(1),
+			},
+			.page_mux = {
+				.addr = 0x01,
+				.mask = BIT(7),
+			},
+			.enable = {
+				.addr = 0x04,
+				.mask = BIT(1),
+			},
+		},
 	},
 	{
 		.reset = {
@@ -2899,6 +2951,12 @@ int st_lsm6dsx_probe(struct device *dev, int irq, int hw_id,
 			return err;
 	}
 
+	if (hw->settings->fusion_settings.chan) {
+		err = st_lsm6dsx_fusion_probe(hw, name);
+		if (err)
+			return err;
+	}
+
 	if (hw->irq > 0) {
 		err = st_lsm6dsx_irq_setup(hw);
 		if (err < 0)
diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_fusion.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_fusion.c
new file mode 100644
index 0000000000000..2e72c7ba94dd1
--- /dev/null
+++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_fusion.c
@@ -0,0 +1,241 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * STMicroelectronics st_lsm6dsx IMU sensor fusion
+ *
+ * Copyright 2026 BayLibre, SAS
+ */
+
+#include <linux/cleanup.h>
+#include <linux/errno.h>
+#include <linux/iio/iio.h>
+#include <linux/iio/sysfs.h>
+#include <linux/mutex.h>
+#include <linux/regmap.h>
+#include <linux/sprintf.h>
+#include <linux/types.h>
+#include <linux/units.h>
+
+#include "st_lsm6dsx.h"
+
+static int
+st_lsm6dsx_fusion_get_odr_val(const struct st_lsm6dsx_fusion_settings *settings,
+			      u32 odr_mHz, u8 *val)
+{
+	int odr_hz = odr_mHz / MILLI;
+	int i;
+
+	for (i = 0; i < settings->odr_len; i++) {
+		if (settings->odr_hz[i] == odr_hz)
+			break;
+	}
+	if (i == settings->odr_len)
+		return -EINVAL;
+
+	*val = i;
+	return 0;
+}
+
+/**
+ * st_lsm6dsx_fusion_page_enable - Enable access to sensor fusion configuration
+ * registers.
+ * @hw: Sensor hardware instance.
+ *
+ * Return: 0 on success, negative value on error.
+ */
+static int st_lsm6dsx_fusion_page_enable(struct st_lsm6dsx_hw *hw)
+{
+	const struct st_lsm6dsx_reg *mux;
+
+	mux = &hw->settings->fusion_settings.page_mux;
+
+	return regmap_set_bits(hw->regmap, mux->addr, mux->mask);
+}
+
+/**
+ * st_lsm6dsx_fusion_page_disable - Disable access to sensor fusion
+ * configuration registers.
+ * @hw: Sensor hardware instance.
+ *
+ * Return: 0 on success, negative value on error.
+ */
+static int st_lsm6dsx_fusion_page_disable(struct st_lsm6dsx_hw *hw)
+{
+	const struct st_lsm6dsx_reg *mux;
+
+	mux = &hw->settings->fusion_settings.page_mux;
+
+	return regmap_clear_bits(hw->regmap, mux->addr, mux->mask);
+}
+
+static int st_lsm6dsx_fusion_set_odr_locked(struct st_lsm6dsx_sensor *sensor,
+					    bool enable)
+{
+	const struct st_lsm6dsx_fusion_settings *settings;
+	struct st_lsm6dsx_hw *hw = sensor->hw;
+	int err;
+
+	settings = &hw->settings->fusion_settings;
+	if (enable) {
+		const struct st_lsm6dsx_reg *reg = &settings->odr_reg;
+		u8 odr_val;
+		u8 data;
+
+		st_lsm6dsx_fusion_get_odr_val(settings, sensor->hwfifo_odr_mHz,
+					      &odr_val);
+		data = ST_LSM6DSX_SHIFT_VAL(odr_val, reg->mask);
+		err = regmap_update_bits(hw->regmap, reg->addr, reg->mask,
+					 data);
+		if (err)
+			return err;
+	}
+
+	return regmap_assign_bits(hw->regmap, settings->fifo_enable.addr,
+				  settings->fifo_enable.mask, enable);
+}
+
+int st_lsm6dsx_fusion_set_enable(struct st_lsm6dsx_sensor *sensor, bool enable)
+{
+	struct st_lsm6dsx_hw *hw = sensor->hw;
+	const struct st_lsm6dsx_reg *en_reg;
+	int err;
+
+	guard(mutex)(&hw->page_lock);
+
+	en_reg = &hw->settings->fusion_settings.enable;
+	err = st_lsm6dsx_fusion_page_enable(hw);
+	if (err)
+		return err;
+
+	err = regmap_assign_bits(hw->regmap, en_reg->addr, en_reg->mask, enable);
+	if (err) {
+		st_lsm6dsx_fusion_page_disable(hw);
+		return err;
+	}
+
+	return st_lsm6dsx_fusion_page_disable(hw);
+}
+
+int st_lsm6dsx_fusion_set_odr(struct st_lsm6dsx_sensor *sensor, bool enable)
+{
+	struct st_lsm6dsx_hw *hw = sensor->hw;
+	int err;
+
+	guard(mutex)(&hw->page_lock);
+
+	err = st_lsm6dsx_fusion_page_enable(hw);
+	if (err)
+		return err;
+
+	err = st_lsm6dsx_fusion_set_odr_locked(sensor, enable);
+	if (err) {
+		st_lsm6dsx_fusion_page_disable(hw);
+		return err;
+	}
+
+	return st_lsm6dsx_fusion_page_disable(hw);
+}
+
+static int st_lsm6dsx_fusion_read_raw(struct iio_dev *iio_dev,
+				      struct iio_chan_spec const *ch,
+				      int *val, int *val2, long mask)
+{
+	struct st_lsm6dsx_sensor *sensor = iio_priv(iio_dev);
+
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*val = sensor->hwfifo_odr_mHz / MILLI;
+		*val2 = (sensor->hwfifo_odr_mHz % MILLI) * (MICRO / MILLI);
+		return IIO_VAL_INT_PLUS_MICRO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int st_lsm6dsx_fusion_write_raw(struct iio_dev *iio_dev,
+				       struct iio_chan_spec const *chan,
+				       int val, int val2, long mask)
+{
+	struct st_lsm6dsx_sensor *sensor = iio_priv(iio_dev);
+	const struct st_lsm6dsx_fusion_settings *settings;
+	int err;
+
+	settings = &sensor->hw->settings->fusion_settings;
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ: {
+		u32 odr_mHz = val * MILLI + val2 * (MILLI / MICRO);
+		u8 odr_val;
+
+		/* check that the requested frequency is supported */
+		err = st_lsm6dsx_fusion_get_odr_val(settings, odr_mHz, &odr_val);
+		if (err)
+			return err;
+
+		sensor->hwfifo_odr_mHz = odr_mHz;
+		return 0;
+	}
+	default:
+		return -EINVAL;
+	}
+}
+
+static int st_lsm6dsx_fusion_read_avail(struct iio_dev *indio_dev,
+					struct iio_chan_spec const *chan,
+					const int **vals, int *type,
+					int *length, long mask)
+{
+	struct st_lsm6dsx_sensor *sensor = iio_priv(indio_dev);
+	const struct st_lsm6dsx_fusion_settings *settings;
+
+	settings = &sensor->hw->settings->fusion_settings;
+	switch (mask) {
+	case IIO_CHAN_INFO_SAMP_FREQ:
+		*vals = settings->odr_hz;
+		*type = IIO_VAL_INT;
+		*length = settings->odr_len;
+		return IIO_AVAIL_LIST;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info st_lsm6dsx_fusion_info = {
+	.read_raw = st_lsm6dsx_fusion_read_raw,
+	.read_avail = st_lsm6dsx_fusion_read_avail,
+	.write_raw = st_lsm6dsx_fusion_write_raw,
+	.hwfifo_set_watermark = st_lsm6dsx_set_watermark,
+};
+
+int st_lsm6dsx_fusion_probe(struct st_lsm6dsx_hw *hw, const char *name)
+{
+	const struct st_lsm6dsx_fusion_settings *settings;
+	struct st_lsm6dsx_sensor *sensor;
+	struct iio_dev *iio_dev;
+	int ret;
+
+	iio_dev = devm_iio_device_alloc(hw->dev, sizeof(*sensor));
+	if (!iio_dev)
+		return -ENOMEM;
+
+	settings = &hw->settings->fusion_settings;
+	sensor = iio_priv(iio_dev);
+	sensor->id = ST_LSM6DSX_ID_FUSION;
+	sensor->hw = hw;
+	sensor->hwfifo_odr_mHz = settings->odr_hz[0] * MILLI;
+	sensor->watermark = 1;
+	iio_dev->modes = INDIO_DIRECT_MODE;
+	iio_dev->info = &st_lsm6dsx_fusion_info;
+	iio_dev->channels = settings->chan;
+	iio_dev->num_channels = settings->chan_len;
+	ret = snprintf(sensor->name, sizeof(sensor->name), "%s_fusion", name);
+	if (ret >= sizeof(sensor->name))
+		return -E2BIG;
+	iio_dev->name = sensor->name;
+
+	/*
+	 * Put the IIO device pointer in the iio_devs array so that the caller
+	 * can set up a buffer and register this IIO device.
+	 */
+	hw->iio_devs[ST_LSM6DSX_ID_FUSION] = iio_dev;
+
+	return 0;
+}
diff --git a/drivers/iio/industrialio-backend.c b/drivers/iio/industrialio-backend.c
index 10e689f494419..138ebebc9c0d0 100644
--- a/drivers/iio/industrialio-backend.c
+++ b/drivers/iio/industrialio-backend.c
@@ -964,23 +964,13 @@ int iio_backend_data_transfer_addr(struct iio_backend *back, u32 address)
 }
 EXPORT_SYMBOL_NS_GPL(iio_backend_data_transfer_addr, "IIO_BACKEND");
 
-static struct iio_backend *__devm_iio_backend_fwnode_get(struct device *dev, const char *name,
-							 struct fwnode_handle *fwnode)
+static struct iio_backend *__devm_iio_backend_fwnode_get_by_index(struct device *dev,
+								  struct fwnode_handle *fwnode,
+								  unsigned int index)
 {
 	struct iio_backend *back;
-	unsigned int index;
 	int ret;
 
-	if (name) {
-		ret = device_property_match_string(dev, "io-backend-names",
-						   name);
-		if (ret < 0)
-			return ERR_PTR(ret);
-		index = ret;
-	} else {
-		index = 0;
-	}
-
 	struct fwnode_handle *fwnode_back __free(fwnode_handle) =
 		fwnode_find_reference(fwnode, "io-backends", index);
 	if (IS_ERR(fwnode_back))
@@ -996,8 +986,7 @@ static struct iio_backend *__devm_iio_backend_fwnode_get(struct device *dev, con
 		if (ret)
 			return ERR_PTR(ret);
 
-		if (name)
-			back->idx = index;
+		back->idx = index;
 
 		return back;
 	}
@@ -1005,6 +994,24 @@ static struct iio_backend *__devm_iio_backend_fwnode_get(struct device *dev, con
 	return ERR_PTR(-EPROBE_DEFER);
 }
 
+static struct iio_backend *__devm_iio_backend_fwnode_get(struct device *dev, const char *name,
+							 struct fwnode_handle *fwnode)
+{
+	unsigned int index;
+	int ret;
+
+	if (name) {
+		ret = device_property_match_string(dev, "io-backend-names", name);
+		if (ret < 0)
+			return ERR_PTR(ret);
+		index = ret;
+	} else {
+		index = 0;
+	}
+
+	return __devm_iio_backend_fwnode_get_by_index(dev, fwnode, index);
+}
+
 /**
  * devm_iio_backend_get - Device managed backend device get
  * @dev: Consumer device for the backend
@@ -1021,6 +1028,22 @@ struct iio_backend *devm_iio_backend_get(struct device *dev, const char *name)
 }
 EXPORT_SYMBOL_NS_GPL(devm_iio_backend_get, "IIO_BACKEND");
 
+/**
+ * devm_iio_backend_get_by_index - Device managed backend device get by index
+ * @dev: Consumer device for the backend
+ * @index: Index of the backend in the io-backends property
+ *
+ * Gets the backend at @index associated with @dev.
+ *
+ * RETURNS:
+ * A backend pointer, negative error pointer otherwise.
+ */
+struct iio_backend *devm_iio_backend_get_by_index(struct device *dev, unsigned int index)
+{
+	return __devm_iio_backend_fwnode_get_by_index(dev, dev_fwnode(dev), index);
+}
+EXPORT_SYMBOL_NS_GPL(devm_iio_backend_get_by_index, "IIO_BACKEND");
+
 /**
  * devm_iio_backend_fwnode_get - Device managed backend firmware node get
  * @dev: Consumer device for the backend
diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c
index 46f36a6ed2710..531fc4ccc15de 100644
--- a/drivers/iio/industrialio-buffer.c
+++ b/drivers/iio/industrialio-buffer.c
@@ -750,7 +750,7 @@ static int iio_storage_bytes_for_si(struct iio_dev *indio_dev,
 	bytes = scan_type->storagebits / 8;
 
 	if (scan_type->repeat > 1)
-		bytes *= scan_type->repeat;
+		bytes *= roundup_pow_of_two(scan_type->repeat);
 
 	return bytes;
 }
@@ -764,7 +764,9 @@ static int iio_storage_bytes_for_timestamp(struct iio_dev *indio_dev)
 }
 
 static int iio_compute_scan_bytes(struct iio_dev *indio_dev,
-				  const unsigned long *mask, bool timestamp)
+				  const unsigned long *mask, bool timestamp,
+				  unsigned int *scan_bytes,
+				  unsigned int *timestamp_offset)
 {
 	unsigned int bytes = 0;
 	int length, i, largest = 0;
@@ -786,12 +788,17 @@ static int iio_compute_scan_bytes(struct iio_dev *indio_dev,
 			return length;
 
 		bytes = ALIGN(bytes, length);
+
+		if (timestamp_offset)
+			*timestamp_offset = bytes;
+
 		bytes += length;
 		largest = max(largest, length);
 	}
 
-	bytes = ALIGN(bytes, largest);
-	return bytes;
+	*scan_bytes = ALIGN(bytes, largest);
+
+	return 0;
 }
 
 static void iio_buffer_activate(struct iio_dev *indio_dev,
@@ -836,18 +843,23 @@ static int iio_buffer_disable(struct iio_buffer *buffer,
 	return buffer->access->disable(buffer, indio_dev);
 }
 
-static void iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,
-					      struct iio_buffer *buffer)
+static int iio_buffer_update_bytes_per_datum(struct iio_dev *indio_dev,
+					     struct iio_buffer *buffer)
 {
 	unsigned int bytes;
+	int ret;
 
 	if (!buffer->access->set_bytes_per_datum)
-		return;
+		return 0;
 
-	bytes = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,
-				       buffer->scan_timestamp);
+	ret = iio_compute_scan_bytes(indio_dev, buffer->scan_mask,
+				     buffer->scan_timestamp, &bytes, NULL);
+	if (ret)
+		return ret;
 
 	buffer->access->set_bytes_per_datum(buffer, bytes);
+
+	return 0;
 }
 
 static int iio_buffer_request_update(struct iio_dev *indio_dev,
@@ -855,7 +867,10 @@ static int iio_buffer_request_update(struct iio_dev *indio_dev,
 {
 	int ret;
 
-	iio_buffer_update_bytes_per_datum(indio_dev, buffer);
+	ret = iio_buffer_update_bytes_per_datum(indio_dev, buffer);
+	if (ret)
+		return ret;
+
 	if (buffer->access->request_update) {
 		ret = buffer->access->request_update(buffer);
 		if (ret) {
@@ -882,6 +897,7 @@ struct iio_device_config {
 	unsigned int watermark;
 	const unsigned long *scan_mask;
 	unsigned int scan_bytes;
+	unsigned int scan_timestamp_offset;
 	bool scan_timestamp;
 };
 
@@ -898,6 +914,7 @@ static int iio_verify_update(struct iio_dev *indio_dev,
 	struct iio_buffer *buffer;
 	bool scan_timestamp;
 	unsigned int modes;
+	int ret;
 
 	if (insert_buffer &&
 	    bitmap_empty(insert_buffer->scan_mask, masklength)) {
@@ -985,8 +1002,12 @@ static int iio_verify_update(struct iio_dev *indio_dev,
 		scan_mask = compound_mask;
 	}
 
-	config->scan_bytes = iio_compute_scan_bytes(indio_dev,
-						    scan_mask, scan_timestamp);
+	ret = iio_compute_scan_bytes(indio_dev, scan_mask, scan_timestamp,
+				     &config->scan_bytes,
+				     &config->scan_timestamp_offset);
+	if (ret)
+		return ret;
+
 	config->scan_mask = scan_mask;
 	config->scan_timestamp = scan_timestamp;
 
@@ -1141,6 +1162,7 @@ static int iio_enable_buffers(struct iio_dev *indio_dev,
 	indio_dev->active_scan_mask = config->scan_mask;
 	ACCESS_PRIVATE(indio_dev, scan_timestamp) = config->scan_timestamp;
 	indio_dev->scan_bytes = config->scan_bytes;
+	ACCESS_PRIVATE(indio_dev, scan_timestamp_offset) = config->scan_timestamp_offset;
 	iio_dev_opaque->currentmode = config->mode;
 
 	iio_update_demux(indio_dev);
@@ -1909,6 +1931,7 @@ static int iio_buffer_enqueue_dmabuf(struct iio_dev_buffer_pair *ib,
 
 	dma_resv_add_fence(dmabuf->resv, &fence->base,
 			   dma_to_ram ? DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ);
+	dma_fence_put(&fence->base);
 	dma_resv_unlock(dmabuf->resv);
 
 	cookie = dma_fence_begin_signalling();
@@ -2422,7 +2445,7 @@ EXPORT_SYMBOL_GPL(iio_push_to_buffers);
 int iio_push_to_buffers_with_ts_unaligned(struct iio_dev *indio_dev,
 					  const void *data,
 					  size_t data_sz,
-					  int64_t timestamp)
+					  s64 timestamp)
 {
 	struct iio_dev_opaque *iio_dev_opaque = to_iio_dev_opaque(indio_dev);
 
diff --git a/drivers/iio/industrialio-core.c b/drivers/iio/industrialio-core.c
index 22eefd048ba92..bd6f4f9f45330 100644
--- a/drivers/iio/industrialio-core.c
+++ b/drivers/iio/industrialio-core.c
@@ -157,6 +157,7 @@ static const char * const iio_modifier_names[] = {
 	[IIO_MOD_ACTIVE] = "active",
 	[IIO_MOD_REACTIVE] = "reactive",
 	[IIO_MOD_APPARENT] = "apparent",
+	[IIO_MOD_QUATERNION_AXIS] = "quaternionaxis",
 };
 
 /* relies on pairs of these shared then separate */
diff --git a/drivers/iio/inkern.c b/drivers/iio/inkern.c
index 0df0ab3de2709..9ce20cb05a9b6 100644
--- a/drivers/iio/inkern.c
+++ b/drivers/iio/inkern.c
@@ -738,7 +738,11 @@ int iio_read_channel_processed_scale(struct iio_channel *chan, int *val,
 		if (ret < 0)
 			return ret;
 
-		return iio_multiply_value(val, scale, ret, pval, pval2);
+		ret = iio_multiply_value(val, scale, ret, pval, pval2);
+		if (ret < 0)
+			return ret;
+
+		return 0;
 	} else {
 		ret = iio_channel_read(chan, val, NULL, IIO_CHAN_INFO_RAW);
 		if (ret < 0)
diff --git a/drivers/iio/light/acpi-als.c b/drivers/iio/light/acpi-als.c
index ab229318dce94..1983a7f17aa93 100644
--- a/drivers/iio/light/acpi-als.c
+++ b/drivers/iio/light/acpi-als.c
@@ -179,11 +179,15 @@ static irqreturn_t acpi_als_trigger_handler(int irq, void *p)
 static int acpi_als_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct acpi_device *device = ACPI_COMPANION(dev);
+	struct acpi_device *device;
 	struct iio_dev *indio_dev;
 	struct acpi_als *als;
 	int ret;
 
+	device = ACPI_COMPANION(dev);
+	if (!device)
+		return -ENODEV;
+
 	indio_dev = devm_iio_device_alloc(dev, sizeof(*als));
 	if (!indio_dev)
 		return -ENOMEM;
diff --git a/drivers/iio/light/apds9160.c b/drivers/iio/light/apds9160.c
index 3da0bdac04cf7..8dacb17304298 100644
--- a/drivers/iio/light/apds9160.c
+++ b/drivers/iio/light/apds9160.c
@@ -1572,7 +1572,7 @@ static const struct of_device_id apds9160_of_match[] = {
 MODULE_DEVICE_TABLE(of, apds9160_of_match);
 
 static const struct i2c_device_id apds9160_id[] = {
-	{ "apds9160", 0 },
+	{ .name = "apds9160" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, apds9160_id);
diff --git a/drivers/iio/light/cm3232.c b/drivers/iio/light/cm3232.c
index 3a3ad6b4c4686..90aa77e3a03b5 100644
--- a/drivers/iio/light/cm3232.c
+++ b/drivers/iio/light/cm3232.c
@@ -369,6 +369,7 @@ static const struct i2c_device_id cm3232_id[] = {
 	{ "cm3232" },
 	{ }
 };
+MODULE_DEVICE_TABLE(i2c, cm3232_id);
 
 static int cm3232_suspend(struct device *dev)
 {
@@ -400,8 +401,6 @@ static int cm3232_resume(struct device *dev)
 
 static DEFINE_SIMPLE_DEV_PM_OPS(cm3232_pm_ops, cm3232_suspend, cm3232_resume);
 
-MODULE_DEVICE_TABLE(i2c, cm3232_id);
-
 static const struct of_device_id cm3232_of_match[] = {
 	{.compatible = "capella,cm3232"},
 	{ }
diff --git a/drivers/iio/light/cm3323.c b/drivers/iio/light/cm3323.c
index 79ad6e2209cab..0fe61b8a70299 100644
--- a/drivers/iio/light/cm3323.c
+++ b/drivers/iio/light/cm3323.c
@@ -89,15 +89,14 @@ static int cm3323_init(struct iio_dev *indio_dev)
 
 	/* enable sensor and set auto force mode */
 	ret &= ~(CM3323_CONF_SD_BIT | CM3323_CONF_AF_BIT);
+	data->reg_conf = ret;
 
-	ret = i2c_smbus_write_word_data(data->client, CM3323_CMD_CONF, ret);
+	ret = i2c_smbus_write_word_data(data->client, CM3323_CMD_CONF, data->reg_conf);
 	if (ret < 0) {
 		dev_err(&data->client->dev, "Error writing reg_conf\n");
 		return ret;
 	}
 
-	data->reg_conf = ret;
-
 	return 0;
 }
 
diff --git a/drivers/iio/light/iqs621-als.c b/drivers/iio/light/iqs621-als.c
index b9f230210f073..cd5843e3e2c33 100644
--- a/drivers/iio/light/iqs621-als.c
+++ b/drivers/iio/light/iqs621-als.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2019 Jeff LaBundy <jeff@labundy.com>
  */
 
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/iio/events.h>
 #include <linux/iio/iio.h>
@@ -107,26 +108,20 @@ static int iqs621_als_notifier(struct notifier_block *notifier,
 	indio_dev = iqs621_als->indio_dev;
 	timestamp = iio_get_time_ns(indio_dev);
 
-	mutex_lock(&iqs621_als->lock);
+	guard(mutex)(&iqs621_als->lock);
 
 	if (event_flags & BIT(IQS62X_EVENT_SYS_RESET)) {
 		ret = iqs621_als_init(iqs621_als);
 		if (ret) {
 			dev_err(indio_dev->dev.parent,
 				"Failed to re-initialize device: %d\n", ret);
-			ret = NOTIFY_BAD;
-		} else {
-			ret = NOTIFY_OK;
+			return NOTIFY_BAD;
 		}
-
-		goto err_mutex;
+		return NOTIFY_OK;
 	}
 
-	if (!iqs621_als->light_en && !iqs621_als->range_en &&
-	    !iqs621_als->prox_en) {
-		ret = NOTIFY_DONE;
-		goto err_mutex;
-	}
+	if (!iqs621_als->light_en && !iqs621_als->range_en && !iqs621_als->prox_en)
+		return NOTIFY_DONE;
 
 	/* IQS621 only */
 	light_new = event_data->als_flags & IQS621_ALS_FLAGS_LIGHT;
@@ -181,12 +176,7 @@ static int iqs621_als_notifier(struct notifier_block *notifier,
 
 	iqs621_als->als_flags = event_data->als_flags;
 	iqs621_als->ir_flags = event_data->ir_flags;
-	ret = NOTIFY_OK;
-
-err_mutex:
-	mutex_unlock(&iqs621_als->lock);
-
-	return ret;
+	return NOTIFY_OK;
 }
 
 static void iqs621_als_notifier_unregister(void *context)
@@ -241,30 +231,22 @@ static int iqs621_als_read_event_config(struct iio_dev *indio_dev,
 					enum iio_event_direction dir)
 {
 	struct iqs621_als_private *iqs621_als = iio_priv(indio_dev);
-	int ret;
 
-	mutex_lock(&iqs621_als->lock);
+	guard(mutex)(&iqs621_als->lock);
 
 	switch (chan->type) {
 	case IIO_LIGHT:
-		ret = iqs621_als->light_en;
-		break;
+		return iqs621_als->light_en;
 
 	case IIO_INTENSITY:
-		ret = iqs621_als->range_en;
-		break;
+		return iqs621_als->range_en;
 
 	case IIO_PROXIMITY:
-		ret = iqs621_als->prox_en;
-		break;
+		return iqs621_als->prox_en;
 
 	default:
-		ret = -EINVAL;
+		return -EINVAL;
 	}
-
-	mutex_unlock(&iqs621_als->lock);
-
-	return ret;
 }
 
 static int iqs621_als_write_event_config(struct iio_dev *indio_dev,
@@ -278,11 +260,11 @@ static int iqs621_als_write_event_config(struct iio_dev *indio_dev,
 	unsigned int val;
 	int ret;
 
-	mutex_lock(&iqs621_als->lock);
+	guard(mutex)(&iqs621_als->lock);
 
 	ret = regmap_read(iqs62x->regmap, iqs62x->dev_desc->als_flags, &val);
 	if (ret)
-		goto err_mutex;
+		return ret;
 	iqs621_als->als_flags = val;
 
 	switch (chan->type) {
@@ -291,40 +273,41 @@ static int iqs621_als_write_event_config(struct iio_dev *indio_dev,
 					 iqs62x->dev_desc->als_mask,
 					 iqs621_als->range_en || state ? 0 :
 									 0xFF);
-		if (!ret)
-			iqs621_als->light_en = state;
-		break;
+		if (ret)
+			return ret;
+		iqs621_als->light_en = state;
+
+		return 0;
 
 	case IIO_INTENSITY:
 		ret = regmap_update_bits(iqs62x->regmap, IQS620_GLBL_EVENT_MASK,
 					 iqs62x->dev_desc->als_mask,
 					 iqs621_als->light_en || state ? 0 :
 									 0xFF);
-		if (!ret)
-			iqs621_als->range_en = state;
-		break;
+		if (ret)
+			return ret;
+		iqs621_als->range_en = state;
+
+		return 0;
 
 	case IIO_PROXIMITY:
 		ret = regmap_read(iqs62x->regmap, IQS622_IR_FLAGS, &val);
 		if (ret)
-			goto err_mutex;
+			return ret;
 		iqs621_als->ir_flags = val;
 
 		ret = regmap_update_bits(iqs62x->regmap, IQS620_GLBL_EVENT_MASK,
 					 iqs62x->dev_desc->ir_mask,
 					 state ? 0 : 0xFF);
-		if (!ret)
-			iqs621_als->prox_en = state;
-		break;
+		if (ret)
+			return ret;
+		iqs621_als->prox_en = state;
+
+		return 0;
 
 	default:
-		ret = -EINVAL;
+		return -EINVAL;
 	}
-
-err_mutex:
-	mutex_unlock(&iqs621_als->lock);
-
-	return ret;
 }
 
 static int iqs621_als_read_event_value(struct iio_dev *indio_dev,
@@ -335,33 +318,28 @@ static int iqs621_als_read_event_value(struct iio_dev *indio_dev,
 				       int *val, int *val2)
 {
 	struct iqs621_als_private *iqs621_als = iio_priv(indio_dev);
-	int ret = IIO_VAL_INT;
 
-	mutex_lock(&iqs621_als->lock);
+	guard(mutex)(&iqs621_als->lock);
 
 	switch (dir) {
 	case IIO_EV_DIR_RISING:
 		*val = iqs621_als->thresh_light * 16;
-		break;
+		return IIO_VAL_INT;
 
 	case IIO_EV_DIR_FALLING:
 		*val = iqs621_als->thresh_dark * 4;
-		break;
+		return IIO_VAL_INT;
 
 	case IIO_EV_DIR_EITHER:
 		if (iqs621_als->ir_flags_mask == IQS622_IR_FLAGS_TOUCH)
 			*val = iqs621_als->thresh_prox * 4;
 		else
 			*val = iqs621_als->thresh_prox;
-		break;
+		return IIO_VAL_INT;
 
 	default:
-		ret = -EINVAL;
+		return -EINVAL;
 	}
-
-	mutex_unlock(&iqs621_als->lock);
-
-	return ret;
 }
 
 static int iqs621_als_write_event_value(struct iio_dev *indio_dev,
@@ -375,9 +353,9 @@ static int iqs621_als_write_event_value(struct iio_dev *indio_dev,
 	struct iqs62x_core *iqs62x = iqs621_als->iqs62x;
 	unsigned int thresh_reg, thresh_val;
 	u8 ir_flags_mask, *thresh_cache;
-	int ret = -EINVAL;
+	int ret;
 
-	mutex_lock(&iqs621_als->lock);
+	guard(mutex)(&iqs621_als->lock);
 
 	switch (dir) {
 	case IIO_EV_DIR_RISING:
@@ -426,30 +404,27 @@ static int iqs621_als_write_event_value(struct iio_dev *indio_dev,
 			break;
 
 		default:
-			goto err_mutex;
+			return -EINVAL;
 		}
 
 		thresh_cache = &iqs621_als->thresh_prox;
 		break;
 
 	default:
-		goto err_mutex;
+		return -EINVAL;
 	}
 
 	if (thresh_val > 0xFF)
-		goto err_mutex;
+		return -EINVAL;
 
 	ret = regmap_write(iqs62x->regmap, thresh_reg, thresh_val);
 	if (ret)
-		goto err_mutex;
+		return ret;
 
 	*thresh_cache = thresh_val;
 	iqs621_als->ir_flags_mask = ir_flags_mask;
 
-err_mutex:
-	mutex_unlock(&iqs621_als->lock);
-
-	return ret;
+	return 0;
 }
 
 static const struct iio_info iqs621_als_info = {
diff --git a/drivers/iio/light/opt3001.c b/drivers/iio/light/opt3001.c
index 53bc455b7bad1..dac3d3818d0a0 100644
--- a/drivers/iio/light/opt3001.c
+++ b/drivers/iio/light/opt3001.c
@@ -8,18 +8,19 @@
  * Based on previous work from: Felipe Balbi <balbi@ti.com>
  */
 
-#include <linux/bitops.h>
+#include <linux/array_size.h>
+#include <linux/bits.h>
 #include <linux/delay.h>
-#include <linux/device.h>
+#include <linux/dev_printk.h>
+#include <linux/errno.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
+#include <linux/jiffies.h>
 #include <linux/mod_devicetable.h>
+#include <linux/module.h>
 #include <linux/mutex.h>
-#include <linux/slab.h>
 #include <linux/types.h>
+#include <linux/wait.h>
 
 #include <linux/iio/events.h>
 #include <linux/iio/iio.h>
@@ -32,17 +33,16 @@
 #define OPT3001_MANUFACTURER_ID	0x7e
 #define OPT3001_DEVICE_ID	0x7f
 
-#define OPT3001_CONFIGURATION_RN_MASK	(0xf << 12)
+#define OPT3001_CONFIGURATION_RN_MASK	GENMASK(15, 12)
 #define OPT3001_CONFIGURATION_RN_AUTO	(0xc << 12)
 
 #define OPT3001_CONFIGURATION_CT	BIT(11)
 
-#define OPT3001_CONFIGURATION_M_MASK	(3 << 9)
+#define OPT3001_CONFIGURATION_M_MASK	GENMASK(10, 9)
 #define OPT3001_CONFIGURATION_M_SHUTDOWN (0 << 9)
 #define OPT3001_CONFIGURATION_M_SINGLE	(1 << 9)
 #define OPT3001_CONFIGURATION_M_CONTINUOUS (2 << 9) /* also 3 << 9 */
 
-#define OPT3001_CONFIGURATION_OVF	BIT(8)
 #define OPT3001_CONFIGURATION_CRF	BIT(7)
 #define OPT3001_CONFIGURATION_FH	BIT(6)
 #define OPT3001_CONFIGURATION_FL	BIT(5)
@@ -50,7 +50,7 @@
 #define OPT3001_CONFIGURATION_POL	BIT(3)
 #define OPT3001_CONFIGURATION_ME	BIT(2)
 
-#define OPT3001_CONFIGURATION_FC_MASK	(3 << 0)
+#define OPT3001_CONFIGURATION_FC_MASK	GENMASK(1, 0)
 
 /* The end-of-conversion enable is located in the low-limit register */
 #define OPT3001_LOW_LIMIT_EOC_ENABLE	0xc000
diff --git a/drivers/iio/light/si1133.c b/drivers/iio/light/si1133.c
index 44fa152dbd24c..655fa778b4a63 100644
--- a/drivers/iio/light/si1133.c
+++ b/drivers/iio/light/si1133.c
@@ -6,19 +6,28 @@
  * Copyright 2018 Maxime Roussin-Belanger <maxime.roussinbelanger@gmail.com>
  */
 
+#include <linux/array_size.h>
+#include <linux/bitops.h>
+#include <linux/cleanup.h>
+#include <linux/completion.h>
 #include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/err.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
+#include <linux/jiffies.h>
+#include <linux/math.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/regmap.h>
+#include <linux/types.h>
+#include <linux/unaligned.h>
+#include <linux/util_macros.h>
 
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
-#include <linux/util_macros.h>
-
-#include <linux/unaligned.h>
-
 #define SI1133_REG_PART_ID		0x00
 #define SI1133_REG_REV_ID		0x01
 #define SI1133_REG_MFR_ID		0x02
@@ -50,23 +59,23 @@
 #define SI1133_MAX_CMD_CTR		0xF
 
 #define SI1133_PARAM_REG_CHAN_LIST	0x01
-#define SI1133_PARAM_REG_ADCCONFIG(x)	((x) * 4) + 2
-#define SI1133_PARAM_REG_ADCSENS(x)	((x) * 4) + 3
-#define SI1133_PARAM_REG_ADCPOST(x)	((x) * 4) + 4
+#define SI1133_PARAM_REG_ADCCONFIG(x)	(((x) * 4) + 2)
+#define SI1133_PARAM_REG_ADCSENS(x)	(((x) * 4) + 3)
+#define SI1133_PARAM_REG_ADCPOST(x)	(((x) * 4) + 4)
 
 #define SI1133_ADCMUX_MASK 0x1F
 
-#define SI1133_ADCCONFIG_DECIM_RATE(x)	(x) << 5
+#define SI1133_ADCCONFIG_DECIM_RATE(x)	((x) << 5)
 
 #define SI1133_ADCSENS_SCALE_MASK 0x70
 #define SI1133_ADCSENS_SCALE_SHIFT 4
 #define SI1133_ADCSENS_HSIG_MASK BIT(7)
 #define SI1133_ADCSENS_HSIG_SHIFT 7
 #define SI1133_ADCSENS_HW_GAIN_MASK 0xF
-#define SI1133_ADCSENS_NB_MEAS(x)	fls(x) << SI1133_ADCSENS_SCALE_SHIFT
+#define SI1133_ADCSENS_NB_MEAS(x)	(fls(x) << SI1133_ADCSENS_SCALE_SHIFT)
 
 #define SI1133_ADCPOST_24BIT_EN BIT(6)
-#define SI1133_ADCPOST_POSTSHIFT_BITQTY(x) (x & GENMASK(2, 0)) << 3
+#define SI1133_ADCPOST_POSTSHIFT_BITQTY(x) (((x) & GENMASK(2, 0)) << 3)
 
 #define SI1133_PARAM_ADCMUX_SMALL_IR	0x0
 #define SI1133_PARAM_ADCMUX_MED_IR	0x1
@@ -86,16 +95,12 @@
 #define SI1133_CMD_MINSLEEP_US_LOW	5000
 #define SI1133_CMD_MINSLEEP_US_HIGH	7500
 #define SI1133_CMD_TIMEOUT_MS		25
-#define SI1133_CMD_LUX_TIMEOUT_MS	5000
-#define SI1133_CMD_TIMEOUT_US		SI1133_CMD_TIMEOUT_MS * 1000
 
-#define SI1133_REG_HOSTOUT(x)		(x) + 0x13
-
-#define SI1133_MEASUREMENT_FREQUENCY 1250
+#define SI1133_REG_HOSTOUT(x)		((x) + 0x13)
 
 #define SI1133_X_ORDER_MASK            0x0070
 #define SI1133_Y_ORDER_MASK            0x0007
-#define si1133_get_x_order(m)          ((m) & SI1133_X_ORDER_MASK) >> 4
+#define si1133_get_x_order(m)          (((m) & SI1133_X_ORDER_MASK) >> 4)
 #define si1133_get_y_order(m)          ((m) & SI1133_Y_ORDER_MASK)
 
 #define SI1133_LUX_ADC_MASK		0xE
@@ -386,35 +391,42 @@ static int si1133_cmd_reset_counter(struct si1133_data *data)
 
 static int si1133_command(struct si1133_data *data, u8 cmd)
 {
+	unsigned long timeout;
 	struct device *dev = &data->client->dev;
 	u32 resp;
 	int err;
 	int expected_seq;
 
-	mutex_lock(&data->mutex);
+	guard(mutex)(&data->mutex);
 
 	expected_seq = (data->rsp_seq + 1) & SI1133_MAX_CMD_CTR;
 
-	if (cmd == SI1133_CMD_FORCE)
+	if (cmd == SI1133_CMD_FORCE) {
+		/* Flush pending IRQs from a previous timeout. */
+		regmap_read(data->regmap, SI1133_REG_IRQ_STATUS, &resp);
+		regmap_write(data->regmap, SI1133_REG_IRQ_ENABLE,
+			     SI1133_IRQ_CHANNEL_ENABLE);
+
 		reinit_completion(&data->completion);
+	}
 
 	err = regmap_write(data->regmap, SI1133_REG_COMMAND, cmd);
 	if (err) {
 		dev_warn(dev, "Failed to write command 0x%02x, ret=%d\n", cmd,
 			 err);
-		goto out;
+		return err;
 	}
 
 	if (cmd == SI1133_CMD_FORCE) {
 		/* wait for irq */
-		if (!wait_for_completion_timeout(&data->completion,
-			msecs_to_jiffies(SI1133_COMPLETION_TIMEOUT_MS))) {
-			err = -ETIMEDOUT;
-			goto out;
+		timeout = msecs_to_jiffies(SI1133_COMPLETION_TIMEOUT_MS);
+		if (!wait_for_completion_timeout(&data->completion, timeout)) {
+			regmap_write(data->regmap, SI1133_REG_IRQ_ENABLE, 0);
+			return -ETIMEDOUT;
 		}
 		err = regmap_read(data->regmap, SI1133_REG_RESPONSE0, &resp);
 		if (err)
-			goto out;
+			return err;
 	} else {
 		err = regmap_read_poll_timeout(data->regmap,
 					       SI1133_REG_RESPONSE0, resp,
@@ -427,7 +439,12 @@ static int si1133_command(struct si1133_data *data, u8 cmd)
 			dev_warn(dev,
 				 "Failed to read command 0x%02x, ret=%d\n",
 				 cmd, err);
-			goto out;
+			/*
+			 * Reset counter on err to prevent software and hardware
+			 * counters being out of sync.
+			 */
+			si1133_cmd_reset_counter(data);
+			return err;
 		}
 	}
 
@@ -438,9 +455,6 @@ static int si1133_command(struct si1133_data *data, u8 cmd)
 		data->rsp_seq = expected_seq;
 	}
 
-out:
-	mutex_unlock(&data->mutex);
-
 	return err;
 }
 
diff --git a/drivers/iio/light/stk3310.c b/drivers/iio/light/stk3310.c
index a75a83594a7ee..60407bc87ccf4 100644
--- a/drivers/iio/light/stk3310.c
+++ b/drivers/iio/light/stk3310.c
@@ -7,15 +7,28 @@
  * IIO driver for STK3310/STK3311. 7-bit I2C address: 0x48.
  */
 
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/dev_printk.h>
+#include <linux/err.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
 #include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pm.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
+#include <linux/sprintf.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+
 #include <linux/iio/events.h>
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
+#include <linux/iio/types.h>
+
+#include <asm/byteorder.h>
 
 #define STK3310_REG_STATE			0x00
 #define STK3310_REG_PSCTRL			0x01
@@ -115,9 +128,6 @@ static const int stk3310_it_table[][2] = {
 struct stk3310_data {
 	struct i2c_client *client;
 	struct mutex lock;
-	bool als_enabled;
-	bool ps_enabled;
-	uint32_t ps_near_level;
 	u64 timestamp;
 	struct regmap *regmap;
 	struct regmap_field *reg_state;
@@ -128,6 +138,12 @@ struct stk3310_data {
 	struct regmap_field *reg_int_ps;
 	struct regmap_field *reg_flag_psint;
 	struct regmap_field *reg_flag_nf;
+	u32 ps_thdl;
+	u32 ps_thdh;
+	u32 ps_near_level;
+	bool als_enabled;
+	bool ps_enabled;
+	bool ps_int_enabled;
 };
 
 static const struct iio_event_spec stk3310_events[] = {
@@ -255,7 +271,7 @@ static int stk3310_read_event(struct iio_dev *indio_dev,
 		return -EINVAL;
 
 	mutex_lock(&data->lock);
-	ret = regmap_bulk_read(data->regmap, reg, &buf, 2);
+	ret = regmap_bulk_read(data->regmap, reg, &buf, sizeof(buf));
 	mutex_unlock(&data->lock);
 	if (ret < 0) {
 		dev_err(&data->client->dev, "register read failed\n");
@@ -295,11 +311,18 @@ static int stk3310_write_event(struct iio_dev *indio_dev,
 		return -EINVAL;
 
 	buf = cpu_to_be16(val);
-	ret = regmap_bulk_write(data->regmap, reg, &buf, 2);
-	if (ret < 0)
+	ret = regmap_bulk_write(data->regmap, reg, &buf, sizeof(buf));
+	if (ret < 0) {
 		dev_err(&client->dev, "failed to set PS threshold!\n");
+		return ret;
+	}
 
-	return ret;
+	if (reg == STK3310_REG_THDH_PS)
+		data->ps_thdh = val;
+	else
+		data->ps_thdl = val;
+
+	return 0;
 }
 
 static int stk3310_read_event_config(struct iio_dev *indio_dev,
@@ -331,11 +354,17 @@ static int stk3310_write_event_config(struct iio_dev *indio_dev,
 	/* Set INT_PS value */
 	mutex_lock(&data->lock);
 	ret = regmap_field_write(data->reg_int_ps, state);
-	if (ret < 0)
+	if (ret < 0) {
 		dev_err(&client->dev, "failed to set interrupt mode\n");
+		mutex_unlock(&data->lock);
+		return ret;
+	}
+
+	data->ps_int_enabled = state;
+
 	mutex_unlock(&data->lock);
 
-	return ret;
+	return 0;
 }
 
 static int stk3310_read_raw(struct iio_dev *indio_dev,
@@ -360,7 +389,7 @@ static int stk3310_read_raw(struct iio_dev *indio_dev,
 			reg = STK3310_REG_PS_DATA_MSB;
 
 		mutex_lock(&data->lock);
-		ret = regmap_bulk_read(data->regmap, reg, &buf, 2);
+		ret = regmap_bulk_read(data->regmap, reg, &buf, sizeof(buf));
 		if (ret < 0) {
 			dev_err(&client->dev, "register read failed\n");
 			mutex_unlock(&data->lock);
@@ -504,10 +533,15 @@ static int stk3310_init(struct iio_dev *indio_dev)
 
 	/* Enable PS interrupts */
 	ret = regmap_field_write(data->reg_int_ps, STK3310_PSINT_EN);
-	if (ret < 0)
+	if (ret < 0) {
 		dev_err(&client->dev, "failed to enable interrupts!\n");
+		return ret;
+	}
 
-	return ret;
+	data->ps_int_enabled = true;
+	data->ps_thdh = STK3310_PS_MAX_VAL;
+
+	return 0;
 }
 
 static bool stk3310_is_volatile_reg(struct device *dev, unsigned int reg)
@@ -671,9 +705,18 @@ static void stk3310_remove(struct i2c_client *client)
 static int stk3310_suspend(struct device *dev)
 {
 	struct stk3310_data *data;
+	int ret;
 
 	data = iio_priv(i2c_get_clientdata(to_i2c_client(dev)));
 
+	if (data->ps_int_enabled) {
+		ret = regmap_field_write(data->reg_int_ps, 0x0);
+		if (ret < 0) {
+			dev_err(dev, "failed to disable ps int at suspend.\n");
+			return ret;
+		}
+	}
+
 	return stk3310_set_state(data, STK3310_STATE_STANDBY);
 }
 
@@ -681,6 +724,8 @@ static int stk3310_resume(struct device *dev)
 {
 	u8 state = 0;
 	struct stk3310_data *data;
+	__be16 buf;
+	int ret;
 
 	data = iio_priv(i2c_get_clientdata(to_i2c_client(dev)));
 	if (data->ps_enabled)
@@ -688,7 +733,37 @@ static int stk3310_resume(struct device *dev)
 	if (data->als_enabled)
 		state |= STK3310_STATE_EN_ALS;
 
-	return stk3310_set_state(data, state);
+	ret = stk3310_set_state(data, state);
+	if (ret < 0)
+		return ret;
+
+	if (data->ps_thdl != 0x0) {
+		buf = cpu_to_be16(data->ps_thdl);
+		ret = regmap_bulk_write(data->regmap, STK3310_REG_THDL_PS, &buf, sizeof(buf));
+		if (ret < 0) {
+			dev_err(dev, "failed to set reg THDL_PS at resume.\n");
+			return ret;
+		}
+	}
+
+	if (data->ps_thdh != STK3310_PS_MAX_VAL) {
+		buf = cpu_to_be16(data->ps_thdh);
+		ret = regmap_bulk_write(data->regmap, STK3310_REG_THDH_PS, &buf, sizeof(buf));
+		if (ret < 0) {
+			dev_err(dev, "failed to set reg THDH_PS at resume.\n");
+			return ret;
+		}
+	}
+
+	if (data->ps_int_enabled) {
+		ret = regmap_field_write(data->reg_int_ps, STK3310_PSINT_EN);
+		if (ret < 0) {
+			dev_err(dev, "failed to enable ps int at resume.\n");
+			return ret;
+		}
+	}
+
+	return 0;
 }
 
 static DEFINE_SIMPLE_DEV_PM_OPS(stk3310_pm_ops, stk3310_suspend,
diff --git a/drivers/iio/light/tsl2563.c b/drivers/iio/light/tsl2563.c
index f2af1cd7c2d1f..7e277bc6a8b1f 100644
--- a/drivers/iio/light/tsl2563.c
+++ b/drivers/iio/light/tsl2563.c
@@ -839,10 +839,10 @@ static DEFINE_SIMPLE_DEV_PM_OPS(tsl2563_pm_ops, tsl2563_suspend,
 				tsl2563_resume);
 
 static const struct i2c_device_id tsl2563_id[] = {
-	{ "tsl2560", 0 },
-	{ "tsl2561", 1 },
-	{ "tsl2562", 2 },
-	{ "tsl2563", 3 },
+	{ .name = "tsl2560" },
+	{ .name = "tsl2561" },
+	{ .name = "tsl2562" },
+	{ .name = "tsl2563" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tsl2563_id);
diff --git a/drivers/iio/light/tsl2583.c b/drivers/iio/light/tsl2583.c
index 8801a491de77b..a0dd122af2cfb 100644
--- a/drivers/iio/light/tsl2583.c
+++ b/drivers/iio/light/tsl2583.c
@@ -913,9 +913,9 @@ static DEFINE_RUNTIME_DEV_PM_OPS(tsl2583_pm_ops, tsl2583_suspend,
 				 tsl2583_resume, NULL);
 
 static const struct i2c_device_id tsl2583_idtable[] = {
-	{ "tsl2580", 0 },
-	{ "tsl2581", 1 },
-	{ "tsl2583", 2 },
+	{ .name = "tsl2580" },
+	{ .name = "tsl2581" },
+	{ .name = "tsl2583" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tsl2583_idtable);
diff --git a/drivers/iio/light/tsl2772.c b/drivers/iio/light/tsl2772.c
index c8f15ba952672..9ba8140c8bc15 100644
--- a/drivers/iio/light/tsl2772.c
+++ b/drivers/iio/light/tsl2772.c
@@ -127,6 +127,7 @@ enum {
 	tmd2672,
 	tsl2772,
 	tmd2772,
+	apds9900,
 	apds9930,
 };
 
@@ -221,6 +222,12 @@ static const struct tsl2772_lux tmd2x72_lux_table[TSL2772_DEF_LUX_TABLE_SZ] = {
 	{     0,      0 },
 };
 
+static const struct tsl2772_lux apds9900_lux_table[TSL2772_DEF_LUX_TABLE_SZ] = {
+	{ 52000,  115960 },
+	{ 36400,   73840 },
+	{     0,       0 },
+};
+
 static const struct tsl2772_lux apds9930_lux_table[TSL2772_DEF_LUX_TABLE_SZ] = {
 	{ 52000,  96824 },
 	{ 38792,  67132 },
@@ -238,6 +245,7 @@ static const struct tsl2772_lux *tsl2772_default_lux_table_group[] = {
 	[tmd2672] = tmd2x72_lux_table,
 	[tsl2772] = tsl2x72_lux_table,
 	[tmd2772] = tmd2x72_lux_table,
+	[apds9900] = apds9900_lux_table,
 	[apds9930] = apds9930_lux_table,
 };
 
@@ -289,6 +297,7 @@ static const int tsl2772_int_time_avail[][6] = {
 	[tmd2672] = { 0, 2730, 0, 2730, 0, 699000 },
 	[tsl2772] = { 0, 2730, 0, 2730, 0, 699000 },
 	[tmd2772] = { 0, 2730, 0, 2730, 0, 699000 },
+	[apds9900] = { 0, 2720, 0, 2720, 0, 696000 },
 	[apds9930] = { 0, 2730, 0, 2730, 0, 699000 },
 };
 
@@ -316,6 +325,7 @@ static const u8 device_channel_config[] = {
 	[tmd2672] = PRX2,
 	[tsl2772] = ALSPRX2,
 	[tmd2772] = ALSPRX2,
+	[apds9900] = ALSPRX,
 	[apds9930] = ALSPRX2,
 };
 
@@ -530,6 +540,7 @@ static int tsl2772_get_prox(struct iio_dev *indio_dev)
 	case tmd2672:
 	case tsl2772:
 	case tmd2772:
+	case apds9900:
 	case apds9930:
 		if (!(ret & TSL2772_STA_PRX_VALID)) {
 			ret = -EINVAL;
@@ -1367,6 +1378,7 @@ static int tsl2772_device_id_verif(int id, int target)
 		return (id & 0xf0) == TRITON_ID;
 	case tmd2671:
 	case tmd2771:
+	case apds9900:
 		return (id & 0xf0) == HALIBUT_ID;
 	case tsl2572:
 	case tsl2672:
@@ -1898,6 +1910,8 @@ static const struct i2c_device_id tsl2772_idtable[] = {
 	{ "tmd2672", tmd2672 },
 	{ "tsl2772", tsl2772 },
 	{ "tmd2772", tmd2772 },
+	{ "apds9900", apds9900 },
+	{ "apds9901", apds9900 },
 	{ "apds9930", apds9930 },
 	{ }
 };
@@ -1915,6 +1929,8 @@ static const struct of_device_id tsl2772_of_match[] = {
 	{ .compatible = "amstaos,tmd2672" },
 	{ .compatible = "amstaos,tsl2772" },
 	{ .compatible = "amstaos,tmd2772" },
+	{ .compatible = "avago,apds9900" },
+	{ .compatible = "avago,apds9901" },
 	{ .compatible = "avago,apds9930" },
 	{ }
 };
diff --git a/drivers/iio/light/vcnl4000.c b/drivers/iio/light/vcnl4000.c
index 9650dbc41f2ba..88fc7424ae353 100644
--- a/drivers/iio/light/vcnl4000.c
+++ b/drivers/iio/light/vcnl4000.c
@@ -18,6 +18,7 @@
  */
 
 #include <linux/bitfield.h>
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/err.h>
 #include <linux/i2c.h>
@@ -185,14 +186,6 @@ static const int vcnl4040_ps_oversampling_ratio[] = {1, 2, 4, 8};
 
 #define VCNL4000_SLEEP_DELAY_MS	2000 /* before we enter pm_runtime_suspend */
 
-enum vcnl4000_device_ids {
-	CM36672P,
-	VCNL4000,
-	VCNL4010,
-	VCNL4040,
-	VCNL4200,
-};
-
 struct vcnl4200_channel {
 	u8 reg;
 	ktime_t last_measurement;
@@ -202,7 +195,6 @@ struct vcnl4200_channel {
 
 struct vcnl4000_data {
 	struct i2c_client *client;
-	enum vcnl4000_device_ids id;
 	int rev;
 	int al_scale;
 	int ps_scale;
@@ -234,20 +226,9 @@ struct vcnl4000_chip_spec {
 	const int(*als_it_times)[][2];
 	const int num_als_it_times;
 	const unsigned int ulux_step;
+	const int prod_id;
 };
 
-static const struct i2c_device_id vcnl4000_id[] = {
-	{ "cm36672p", CM36672P },
-	{ "cm36686", VCNL4040 },
-	{ "vcnl4000", VCNL4000 },
-	{ "vcnl4010", VCNL4010 },
-	{ "vcnl4020", VCNL4010 },
-	{ "vcnl4040", VCNL4040 },
-	{ "vcnl4200", VCNL4200 },
-	{ }
-};
-MODULE_DEVICE_TABLE(i2c, vcnl4000_id);
-
 static int vcnl4000_set_power_state(struct vcnl4000_data *data, bool on)
 {
 	/* no suspend op */
@@ -265,12 +246,12 @@ static int vcnl4000_init(struct vcnl4000_data *data)
 	prod_id = ret >> 4;
 	switch (prod_id) {
 	case VCNL4000_PROD_ID:
-		if (data->id != VCNL4000)
+		if (data->chip_spec->prod_id != VCNL4000_PROD_ID)
 			dev_warn(&data->client->dev,
 					"wrong device id, use vcnl4000");
 		break;
 	case VCNL4010_PROD_ID:
-		if (data->id != VCNL4010)
+		if (data->chip_spec->prod_id != VCNL4010_PROD_ID)
 			dev_warn(&data->client->dev,
 					"wrong device id, use vcnl4010/4020");
 		break;
@@ -288,46 +269,36 @@ static ssize_t vcnl4000_write_als_enable(struct vcnl4000_data *data, bool en)
 {
 	int ret;
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	ret = i2c_smbus_read_word_data(data->client, VCNL4200_AL_CONF);
 	if (ret < 0)
-		goto out;
+		return ret;
 
 	if (en)
 		ret &= ~VCNL4040_ALS_CONF_ALS_SHUTDOWN;
 	else
 		ret |= VCNL4040_ALS_CONF_ALS_SHUTDOWN;
 
-	ret = i2c_smbus_write_word_data(data->client, VCNL4200_AL_CONF, ret);
-
-out:
-	mutex_unlock(&data->vcnl4000_lock);
-
-	return ret;
+	return i2c_smbus_write_word_data(data->client, VCNL4200_AL_CONF, ret);
 }
 
 static ssize_t vcnl4000_write_ps_enable(struct vcnl4000_data *data, bool en)
 {
 	int ret;
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	ret = i2c_smbus_read_word_data(data->client, VCNL4200_PS_CONF1);
 	if (ret < 0)
-		goto out;
+		return ret;
 
 	if (en)
 		ret &= ~VCNL4040_PS_CONF1_PS_SHUTDOWN;
 	else
 		ret |= VCNL4040_PS_CONF1_PS_SHUTDOWN;
 
-	ret = i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF1, ret);
-
-out:
-	mutex_unlock(&data->vcnl4000_lock);
-
-	return ret;
+	return i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF1, ret);
 }
 
 static int vcnl4200_set_power_state(struct vcnl4000_data *data, bool on)
@@ -462,18 +433,18 @@ static int vcnl4000_measure(struct vcnl4000_data *data, u8 req_mask,
 	int tries = 20;
 	int ret;
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	ret = i2c_smbus_write_byte_data(data->client, VCNL4000_COMMAND,
 					req_mask);
 	if (ret < 0)
-		goto fail;
+		return ret;
 
 	/* wait for data to become ready */
 	while (tries--) {
 		ret = i2c_smbus_read_byte_data(data->client, VCNL4000_COMMAND);
 		if (ret < 0)
-			goto fail;
+			return ret;
 		if (ret & rdy_mask)
 			break;
 		msleep(20); /* measurement takes up to 100 ms */
@@ -482,21 +453,10 @@ static int vcnl4000_measure(struct vcnl4000_data *data, u8 req_mask,
 	if (tries < 0) {
 		dev_err(&data->client->dev,
 			"vcnl4000_measure() failed, data not ready\n");
-		ret = -EIO;
-		goto fail;
+		return -EIO;
 	}
 
-	ret = vcnl4000_read_data(data, data_reg, val);
-	if (ret < 0)
-		goto fail;
-
-	mutex_unlock(&data->vcnl4000_lock);
-
-	return 0;
-
-fail:
-	mutex_unlock(&data->vcnl4000_lock);
-	return ret;
+	return vcnl4000_read_data(data, data_reg, val);
 }
 
 static int vcnl4200_measure(struct vcnl4000_data *data,
@@ -506,16 +466,14 @@ static int vcnl4200_measure(struct vcnl4000_data *data,
 	s64 delta;
 	ktime_t next_measurement;
 
-	mutex_lock(&chan->lock);
-
-	next_measurement = ktime_add(chan->last_measurement,
-			chan->sampling_rate);
-	delta = ktime_us_delta(next_measurement, ktime_get());
-	if (delta > 0)
-		usleep_range(delta, delta + 500);
-	chan->last_measurement = ktime_get();
-
-	mutex_unlock(&chan->lock);
+	scoped_guard(mutex, &chan->lock) {
+		next_measurement = ktime_add(chan->last_measurement,
+				chan->sampling_rate);
+		delta = ktime_us_delta(next_measurement, ktime_get());
+		if (delta > 0)
+			usleep_range(delta, delta + 500);
+		chan->last_measurement = ktime_get();
+	}
 
 	ret = i2c_smbus_read_word_data(data->client, chan->reg);
 	if (ret < 0)
@@ -626,21 +584,15 @@ static ssize_t vcnl4040_write_als_it(struct vcnl4000_data *data, int val)
 			 (*data->chip_spec->als_it_times)[0][1]),
 			 val);
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	ret = i2c_smbus_read_word_data(data->client, VCNL4200_AL_CONF);
 	if (ret < 0)
-		goto out_unlock;
+		return ret;
 
 	regval = FIELD_PREP(VCNL4040_ALS_CONF_IT, i);
 	regval |= (ret & ~VCNL4040_ALS_CONF_IT);
-	ret = i2c_smbus_write_word_data(data->client,
-					VCNL4200_AL_CONF,
-					regval);
-
-out_unlock:
-	mutex_unlock(&data->vcnl4000_lock);
-	return ret;
+	return i2c_smbus_write_word_data(data->client, VCNL4200_AL_CONF, regval);
 }
 
 static int vcnl4040_read_ps_it(struct vcnl4000_data *data, int *val, int *val2)
@@ -680,20 +632,15 @@ static ssize_t vcnl4040_write_ps_it(struct vcnl4000_data *data, int val)
 
 	data->vcnl4200_ps.sampling_rate = ktime_set(0, val * 60 * NSEC_PER_USEC);
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	ret = i2c_smbus_read_word_data(data->client, VCNL4200_PS_CONF1);
 	if (ret < 0)
-		goto out;
+		return ret;
 
 	regval = (ret & ~VCNL4040_PS_CONF2_PS_IT) |
 	    FIELD_PREP(VCNL4040_PS_CONF2_PS_IT, index);
-	ret = i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF1,
-					regval);
-
-out:
-	mutex_unlock(&data->vcnl4000_lock);
-	return ret;
+	return i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF1, regval);
 }
 
 static ssize_t vcnl4040_read_als_period(struct vcnl4000_data *data, int *val, int *val2)
@@ -741,20 +688,15 @@ static ssize_t vcnl4040_write_als_period(struct vcnl4000_data *data, int val, in
 			break;
 	}
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	ret = i2c_smbus_read_word_data(data->client, VCNL4200_AL_CONF);
 	if (ret < 0)
-		goto out_unlock;
+		return ret;
 
 	regval = FIELD_PREP(VCNL4040_ALS_CONF_PERS, i);
 	regval |= (ret & ~VCNL4040_ALS_CONF_PERS);
-	ret = i2c_smbus_write_word_data(data->client, VCNL4200_AL_CONF,
-					regval);
-
-out_unlock:
-	mutex_unlock(&data->vcnl4000_lock);
-	return ret;
+	return i2c_smbus_write_word_data(data->client, VCNL4200_AL_CONF, regval);
 }
 
 static ssize_t vcnl4040_read_ps_period(struct vcnl4000_data *data, int *val, int *val2)
@@ -803,20 +745,15 @@ static ssize_t vcnl4040_write_ps_period(struct vcnl4000_data *data, int val, int
 		}
 	}
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	ret = i2c_smbus_read_word_data(data->client, VCNL4200_PS_CONF1);
 	if (ret < 0)
-		goto out_unlock;
+		return ret;
 
 	regval = FIELD_PREP(VCNL4040_CONF1_PS_PERS, i);
 	regval |= (ret & ~VCNL4040_CONF1_PS_PERS);
-	ret = i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF1,
-					regval);
-
-out_unlock:
-	mutex_unlock(&data->vcnl4000_lock);
-	return ret;
+	return i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF1, regval);
 }
 
 static ssize_t vcnl4040_read_ps_oversampling_ratio(struct vcnl4000_data *data, int *val)
@@ -850,20 +787,15 @@ static ssize_t vcnl4040_write_ps_oversampling_ratio(struct vcnl4000_data *data,
 	if (i >= ARRAY_SIZE(vcnl4040_ps_oversampling_ratio))
 		return -EINVAL;
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	ret = i2c_smbus_read_word_data(data->client, VCNL4200_PS_CONF3);
 	if (ret < 0)
-		goto out_unlock;
+		return ret;
 
 	regval = FIELD_PREP(VCNL4040_PS_CONF3_MPS, i);
 	regval |= (ret & ~VCNL4040_PS_CONF3_MPS);
-	ret = i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF3,
-					regval);
-
-out_unlock:
-	mutex_unlock(&data->vcnl4000_lock);
-	return ret;
+	return i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF3, regval);
 }
 
 static ssize_t vcnl4040_read_ps_calibbias(struct vcnl4000_data *data, int *val, int *val2)
@@ -898,20 +830,15 @@ static ssize_t vcnl4040_write_ps_calibbias(struct vcnl4000_data *data, int val)
 	if (i >= ARRAY_SIZE(vcnl4040_ps_calibbias_ua))
 		return -EINVAL;
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	ret = i2c_smbus_read_word_data(data->client, VCNL4200_PS_CONF3);
 	if (ret < 0)
-		goto out_unlock;
+		return ret;
 
 	regval = (ret & ~VCNL4040_PS_MS_LED_I);
 	regval |= FIELD_PREP(VCNL4040_PS_MS_LED_I, i);
-	ret = i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF3,
-					regval);
-
-out_unlock:
-	mutex_unlock(&data->vcnl4000_lock);
-	return ret;
+	return i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF3, regval);
 }
 
 static int vcnl4000_read_raw(struct iio_dev *indio_dev,
@@ -1496,17 +1423,17 @@ static int vcnl4040_write_event_config(struct iio_dev *indio_dev,
 				       enum iio_event_direction dir,
 				       bool state)
 {
-	int ret = -EINVAL;
+	int ret;
 	u16 val, mask;
 	struct vcnl4000_data *data = iio_priv(indio_dev);
 
-	mutex_lock(&data->vcnl4000_lock);
+	guard(mutex)(&data->vcnl4000_lock);
 
 	switch (chan->type) {
 	case IIO_LIGHT:
 		ret = i2c_smbus_read_word_data(data->client, VCNL4200_AL_CONF);
 		if (ret < 0)
-			goto out;
+			return ret;
 
 		mask = VCNL4040_ALS_CONF_INT_EN;
 		if (state)
@@ -1515,13 +1442,11 @@ static int vcnl4040_write_event_config(struct iio_dev *indio_dev,
 			val = (ret & ~mask);
 
 		data->als_int = FIELD_GET(VCNL4040_ALS_CONF_INT_EN, val);
-		ret = i2c_smbus_write_word_data(data->client, VCNL4200_AL_CONF,
-						val);
-		break;
+		return i2c_smbus_write_word_data(data->client, VCNL4200_AL_CONF, val);
 	case IIO_PROXIMITY:
 		ret = i2c_smbus_read_word_data(data->client, VCNL4200_PS_CONF1);
 		if (ret < 0)
-			goto out;
+			return ret;
 
 		if (dir == IIO_EV_DIR_RISING)
 			mask = VCNL4040_PS_IF_AWAY;
@@ -1531,17 +1456,10 @@ static int vcnl4040_write_event_config(struct iio_dev *indio_dev,
 		val = state ? (ret | mask) : (ret & ~mask);
 
 		data->ps_int = FIELD_GET(VCNL4040_PS_CONF2_PS_INT, val);
-		ret = i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF1,
-						val);
-		break;
+		return i2c_smbus_write_word_data(data->client, VCNL4200_PS_CONF1, val);
 	default:
-		break;
+		return -EINVAL;
 	}
-
-out:
-	mutex_unlock(&data->vcnl4000_lock);
-
-	return ret;
 }
 
 static irqreturn_t vcnl4040_irq_thread(int irq, void *p)
@@ -1888,77 +1806,84 @@ static const struct iio_info vcnl4040_info = {
 	.read_avail = vcnl4040_read_avail,
 };
 
-static const struct vcnl4000_chip_spec vcnl4000_chip_spec_cfg[] = {
-	[CM36672P] = {
-		.prod = "CM36672P",
-		.init = vcnl4200_init,
-		.measure_proximity = vcnl4200_measure_proximity,
-		.set_power_state = vcnl4200_set_power_state,
-		.channels = cm36672p_channels,
-		.num_channels = ARRAY_SIZE(cm36672p_channels),
-		.info = &vcnl4040_info,
-		.irq_thread = vcnl4040_irq_thread,
-		.int_reg = VCNL4040_INT_FLAGS,
-		.ps_it_times = &vcnl4040_ps_it_times,
-		.num_ps_it_times = ARRAY_SIZE(vcnl4040_ps_it_times),
-	},
-	[VCNL4000] = {
-		.prod = "VCNL4000",
-		.init = vcnl4000_init,
-		.measure_light = vcnl4000_measure_light,
-		.measure_proximity = vcnl4000_measure_proximity,
-		.set_power_state = vcnl4000_set_power_state,
-		.channels = vcnl4000_channels,
-		.num_channels = ARRAY_SIZE(vcnl4000_channels),
-		.info = &vcnl4000_info,
-	},
-	[VCNL4010] = {
-		.prod = "VCNL4010/4020",
-		.init = vcnl4000_init,
-		.measure_light = vcnl4000_measure_light,
-		.measure_proximity = vcnl4000_measure_proximity,
-		.set_power_state = vcnl4000_set_power_state,
-		.channels = vcnl4010_channels,
-		.num_channels = ARRAY_SIZE(vcnl4010_channels),
-		.info = &vcnl4010_info,
-		.irq_thread = vcnl4010_irq_thread,
-		.trig_buffer_func = vcnl4010_trigger_handler,
-		.buffer_setup_ops = &vcnl4010_buffer_ops,
-	},
-	[VCNL4040] = {
-		.prod = "VCNL4040",
-		.init = vcnl4200_init,
-		.measure_light = vcnl4200_measure_light,
-		.measure_proximity = vcnl4200_measure_proximity,
-		.set_power_state = vcnl4200_set_power_state,
-		.channels = vcnl4040_channels,
-		.num_channels = ARRAY_SIZE(vcnl4040_channels),
-		.info = &vcnl4040_info,
-		.irq_thread = vcnl4040_irq_thread,
-		.int_reg = VCNL4040_INT_FLAGS,
-		.ps_it_times = &vcnl4040_ps_it_times,
-		.num_ps_it_times = ARRAY_SIZE(vcnl4040_ps_it_times),
-		.als_it_times = &vcnl4040_als_it_times,
-		.num_als_it_times = ARRAY_SIZE(vcnl4040_als_it_times),
-		.ulux_step = 100000,
-	},
-	[VCNL4200] = {
-		.prod = "VCNL4200",
-		.init = vcnl4200_init,
-		.measure_light = vcnl4200_measure_light,
-		.measure_proximity = vcnl4200_measure_proximity,
-		.set_power_state = vcnl4200_set_power_state,
-		.channels = vcnl4040_channels,
-		.num_channels = ARRAY_SIZE(vcnl4000_channels),
-		.info = &vcnl4040_info,
-		.irq_thread = vcnl4040_irq_thread,
-		.int_reg = VCNL4200_INT_FLAGS,
-		.ps_it_times = &vcnl4200_ps_it_times,
-		.num_ps_it_times = ARRAY_SIZE(vcnl4200_ps_it_times),
-		.als_it_times = &vcnl4200_als_it_times,
-		.num_als_it_times = ARRAY_SIZE(vcnl4200_als_it_times),
-		.ulux_step = 24000,
-	},
+static const struct vcnl4000_chip_spec cm36672p_spec = {
+	.prod = "CM36672P",
+	.init = vcnl4200_init,
+	.measure_proximity = vcnl4200_measure_proximity,
+	.set_power_state = vcnl4200_set_power_state,
+	.channels = cm36672p_channels,
+	.num_channels = ARRAY_SIZE(cm36672p_channels),
+	.info = &vcnl4040_info,
+	.irq_thread = vcnl4040_irq_thread,
+	.int_reg = VCNL4040_INT_FLAGS,
+	.ps_it_times = &vcnl4040_ps_it_times,
+	.num_ps_it_times = ARRAY_SIZE(vcnl4040_ps_it_times),
+	.prod_id = VCNL4040_PROD_ID,
+};
+
+static const struct vcnl4000_chip_spec vcnl4000_spec = {
+	.prod = "VCNL4000",
+	.init = vcnl4000_init,
+	.measure_light = vcnl4000_measure_light,
+	.measure_proximity = vcnl4000_measure_proximity,
+	.set_power_state = vcnl4000_set_power_state,
+	.channels = vcnl4000_channels,
+	.num_channels = ARRAY_SIZE(vcnl4000_channels),
+	.info = &vcnl4000_info,
+	.prod_id = VCNL4000_PROD_ID,
+};
+
+static const struct vcnl4000_chip_spec vcnl4010_spec = {
+	.prod = "VCNL4010/4020",
+	.init = vcnl4000_init,
+	.measure_light = vcnl4000_measure_light,
+	.measure_proximity = vcnl4000_measure_proximity,
+	.set_power_state = vcnl4000_set_power_state,
+	.channels = vcnl4010_channels,
+	.num_channels = ARRAY_SIZE(vcnl4010_channels),
+	.info = &vcnl4010_info,
+	.irq_thread = vcnl4010_irq_thread,
+	.trig_buffer_func = vcnl4010_trigger_handler,
+	.buffer_setup_ops = &vcnl4010_buffer_ops,
+	.prod_id = VCNL4010_PROD_ID,
+};
+
+static const struct vcnl4000_chip_spec vcnl4040_spec = {
+	.prod = "VCNL4040",
+	.init = vcnl4200_init,
+	.measure_light = vcnl4200_measure_light,
+	.measure_proximity = vcnl4200_measure_proximity,
+	.set_power_state = vcnl4200_set_power_state,
+	.channels = vcnl4040_channels,
+	.num_channels = ARRAY_SIZE(vcnl4040_channels),
+	.info = &vcnl4040_info,
+	.irq_thread = vcnl4040_irq_thread,
+	.int_reg = VCNL4040_INT_FLAGS,
+	.ps_it_times = &vcnl4040_ps_it_times,
+	.num_ps_it_times = ARRAY_SIZE(vcnl4040_ps_it_times),
+	.als_it_times = &vcnl4040_als_it_times,
+	.num_als_it_times = ARRAY_SIZE(vcnl4040_als_it_times),
+	.ulux_step = 100000,
+	.prod_id = VCNL4040_PROD_ID,
+};
+
+static const struct vcnl4000_chip_spec vcnl4200_spec = {
+	.prod = "VCNL4200",
+	.init = vcnl4200_init,
+	.measure_light = vcnl4200_measure_light,
+	.measure_proximity = vcnl4200_measure_proximity,
+	.set_power_state = vcnl4200_set_power_state,
+	.channels = vcnl4040_channels,
+	.num_channels = ARRAY_SIZE(vcnl4000_channels),
+	.info = &vcnl4040_info,
+	.irq_thread = vcnl4040_irq_thread,
+	.int_reg = VCNL4200_INT_FLAGS,
+	.ps_it_times = &vcnl4200_ps_it_times,
+	.num_ps_it_times = ARRAY_SIZE(vcnl4200_ps_it_times),
+	.als_it_times = &vcnl4200_als_it_times,
+	.num_als_it_times = ARRAY_SIZE(vcnl4200_als_it_times),
+	.ulux_step = 24000,
+	.prod_id = VCNL4200_PROD_ID,
 };
 
 static const struct iio_trigger_ops vcnl4010_trigger_ops = {
@@ -1983,9 +1908,20 @@ static int vcnl4010_probe_trigger(struct iio_dev *indio_dev)
 	return devm_iio_trigger_register(&client->dev, trigger);
 }
 
+static void vcnl4000_cleanup(void *data)
+{
+	struct iio_dev *indio_dev = data;
+	struct vcnl4000_data *chip = iio_priv(indio_dev);
+	struct device *dev = &chip->client->dev;
+	int ret;
+
+	ret = chip->chip_spec->set_power_state(chip, false);
+	if (ret)
+		dev_warn(dev, "Failed to power down (%pe)", ERR_PTR(ret));
+}
+
 static int vcnl4000_probe(struct i2c_client *client)
 {
-	const struct i2c_device_id *id = i2c_client_get_device_id(client);
 	const char * const regulator_names[] = { "vdd", "vio", "vled" };
 	struct device *dev = &client->dev;
 	struct vcnl4000_data *data;
@@ -1999,8 +1935,7 @@ static int vcnl4000_probe(struct i2c_client *client)
 	data = iio_priv(indio_dev);
 	i2c_set_clientdata(client, indio_dev);
 	data->client = client;
-	data->id = id->driver_data;
-	data->chip_spec = &vcnl4000_chip_spec_cfg[data->id];
+	data->chip_spec = i2c_get_match_data(client);
 
 	ret = devm_regulator_bulk_get_enable(dev, ARRAY_SIZE(regulator_names),
 					     regulator_names);
@@ -2019,6 +1954,10 @@ static int vcnl4000_probe(struct i2c_client *client)
 	if (ret)
 		return ret;
 
+	ret = devm_add_action_or_reset(dev, vcnl4000_cleanup, indio_dev);
+	if (ret)
+		return ret;
+
 	dev_dbg(dev, "%s Ambient light/proximity sensor, Rev: %02x\n",
 		data->chip_spec->prod, data->rev);
 
@@ -2054,75 +1993,33 @@ static int vcnl4000_probe(struct i2c_client *client)
 			return ret;
 	}
 
-	ret = pm_runtime_set_active(dev);
-	if (ret < 0)
-		goto fail_poweroff;
+	ret = devm_pm_runtime_set_active_enabled(dev);
+	if (ret)
+		return ret;
 
-	ret = iio_device_register(indio_dev);
-	if (ret < 0)
-		goto fail_poweroff;
+	ret = devm_iio_device_register(dev, indio_dev);
+	if (ret)
+		return ret;
 
-	pm_runtime_enable(dev);
 	pm_runtime_set_autosuspend_delay(dev, VCNL4000_SLEEP_DELAY_MS);
 	pm_runtime_use_autosuspend(dev);
 
 	return 0;
-fail_poweroff:
-	data->chip_spec->set_power_state(data, false);
-	return ret;
 }
 
 static const struct of_device_id vcnl_4000_of_match[] = {
-	{
-		.compatible = "capella,cm36672p",
-		.data = (void *)CM36672P,
-	},
+	{ .compatible = "capella,cm36672p", .data = &cm36672p_spec },
 	/* Capella CM36686 is fully compatible with Vishay VCNL4040 */
-	{
-		.compatible = "capella,cm36686",
-		.data = (void *)VCNL4040,
-	},
-	{
-		.compatible = "vishay,vcnl4000",
-		.data = (void *)VCNL4000,
-	},
-	{
-		.compatible = "vishay,vcnl4010",
-		.data = (void *)VCNL4010,
-	},
-	{
-		.compatible = "vishay,vcnl4020",
-		.data = (void *)VCNL4010,
-	},
-	{
-		.compatible = "vishay,vcnl4040",
-		.data = (void *)VCNL4040,
-	},
-	{
-		.compatible = "vishay,vcnl4200",
-		.data = (void *)VCNL4200,
-	},
+	{ .compatible = "capella,cm36686", .data = &vcnl4040_spec },
+	{ .compatible = "vishay,vcnl4000", .data = &vcnl4000_spec },
+	{ .compatible = "vishay,vcnl4010", .data = &vcnl4010_spec },
+	{ .compatible = "vishay,vcnl4020", .data = &vcnl4010_spec },
+	{ .compatible = "vishay,vcnl4040", .data = &vcnl4040_spec },
+	{ .compatible = "vishay,vcnl4200", .data = &vcnl4200_spec },
 	{ }
 };
 MODULE_DEVICE_TABLE(of, vcnl_4000_of_match);
 
-static void vcnl4000_remove(struct i2c_client *client)
-{
-	struct iio_dev *indio_dev = i2c_get_clientdata(client);
-	struct vcnl4000_data *data = iio_priv(indio_dev);
-	int ret;
-
-	pm_runtime_dont_use_autosuspend(&client->dev);
-	pm_runtime_disable(&client->dev);
-	iio_device_unregister(indio_dev);
-	pm_runtime_set_suspended(&client->dev);
-
-	ret = data->chip_spec->set_power_state(data, false);
-	if (ret)
-		dev_warn(&client->dev, "Failed to power down (%pe)\n",
-			 ERR_PTR(ret));
-}
-
 static int vcnl4000_runtime_suspend(struct device *dev)
 {
 	struct iio_dev *indio_dev = i2c_get_clientdata(to_i2c_client(dev));
@@ -2142,6 +2039,18 @@ static int vcnl4000_runtime_resume(struct device *dev)
 static DEFINE_RUNTIME_DEV_PM_OPS(vcnl4000_pm_ops, vcnl4000_runtime_suspend,
 				 vcnl4000_runtime_resume, NULL);
 
+static const struct i2c_device_id vcnl4000_id[] = {
+	{ "cm36672p", (kernel_ulong_t)&cm36672p_spec },
+	{ "cm36686", (kernel_ulong_t)&vcnl4040_spec },
+	{ "vcnl4000", (kernel_ulong_t)&vcnl4000_spec },
+	{ "vcnl4010", (kernel_ulong_t)&vcnl4010_spec },
+	{ "vcnl4020", (kernel_ulong_t)&vcnl4010_spec },
+	{ "vcnl4040", (kernel_ulong_t)&vcnl4040_spec },
+	{ "vcnl4200", (kernel_ulong_t)&vcnl4200_spec },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, vcnl4000_id);
+
 static struct i2c_driver vcnl4000_driver = {
 	.driver = {
 		.name   = VCNL4000_DRV_NAME,
@@ -2150,7 +2059,6 @@ static struct i2c_driver vcnl4000_driver = {
 	},
 	.probe = vcnl4000_probe,
 	.id_table = vcnl4000_id,
-	.remove	= vcnl4000_remove,
 };
 
 module_i2c_driver(vcnl4000_driver);
diff --git a/drivers/iio/light/veml6030.c b/drivers/iio/light/veml6030.c
index 6bcacae3863c0..2f9a25d59b717 100644
--- a/drivers/iio/light/veml6030.c
+++ b/drivers/iio/light/veml6030.c
@@ -521,13 +521,9 @@ static int veml6030_write_persistence(struct iio_dev *indio_dev,
 
 static int veml6030_set_scale(struct iio_dev *indio_dev, int val, int val2)
 {
-	int ret, gain_sel, it_idx, it_sel;
+	int ret, gain_sel, it_sel;
 	struct veml6030_data *data = iio_priv(indio_dev);
 
-	ret = regmap_field_read(data->rf.it, &it_idx);
-	if (ret)
-		return ret;
-
 	ret = iio_gts_find_gain_time_sel_for_scale(&data->gts, val, val2,
 						   &gain_sel, &it_sel);
 	if (ret)
@@ -875,9 +871,11 @@ static irqreturn_t veml6030_event_handler(int irq, void *private)
 	else
 		evtdir = IIO_EV_DIR_FALLING;
 
-	iio_push_event(indio_dev, IIO_UNMOD_EVENT_CODE(IIO_INTENSITY,
-					0, IIO_EV_TYPE_THRESH, evtdir),
-					iio_get_time_ns(indio_dev));
+	iio_push_event(indio_dev, IIO_UNMOD_EVENT_CODE(IIO_LIGHT,
+						       0,
+						       IIO_EV_TYPE_THRESH,
+						       evtdir),
+		       iio_get_time_ns(indio_dev));
 
 	return IRQ_HANDLED;
 }
diff --git a/drivers/iio/light/veml6070.c b/drivers/iio/light/veml6070.c
index 74d7246e5225e..4bbd86d0cb46a 100644
--- a/drivers/iio/light/veml6070.c
+++ b/drivers/iio/light/veml6070.c
@@ -245,13 +245,6 @@ static const struct iio_info veml6070_info = {
 	.write_raw = veml6070_write_raw,
 };
 
-static void veml6070_i2c_unreg(void *p)
-{
-	struct veml6070_data *data = p;
-
-	i2c_unregister_device(data->client2);
-}
-
 static int veml6070_probe(struct i2c_client *client)
 {
 	struct veml6070_data *data;
@@ -281,7 +274,8 @@ static int veml6070_probe(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
-	data->client2 = i2c_new_dummy_device(client->adapter, VEML6070_ADDR_DATA_LSB);
+	data->client2 = devm_i2c_new_dummy_device(&client->dev, client->adapter,
+						  VEML6070_ADDR_DATA_LSB);
 	if (IS_ERR(data->client2))
 		return dev_err_probe(&client->dev, PTR_ERR(data->client2),
 				     "i2c device for second chip address failed\n");
@@ -292,10 +286,6 @@ static int veml6070_probe(struct i2c_client *client)
 	if (ret < 0)
 		return ret;
 
-	ret = devm_add_action_or_reset(&client->dev, veml6070_i2c_unreg, data);
-	if (ret < 0)
-		return ret;
-
 	return devm_iio_device_register(&client->dev, indio_dev);
 }
 
diff --git a/drivers/iio/light/veml6075.c b/drivers/iio/light/veml6075.c
index edbb434070543..f7eb159e5cb40 100644
--- a/drivers/iio/light/veml6075.c
+++ b/drivers/iio/light/veml6075.c
@@ -100,7 +100,7 @@ static const struct iio_chan_spec veml6075_channels[] = {
 
 static int veml6075_request_measurement(struct veml6075_data *data)
 {
-	int ret, conf, int_time;
+	int ret, conf, int_time, int_index;
 
 	ret = regmap_read(data->regmap, VEML6075_CMD_CONF, &conf);
 	if (ret < 0)
@@ -117,7 +117,11 @@ static int veml6075_request_measurement(struct veml6075_data *data)
 	 * time for all possible configurations. Using a 1.50 factor simplifies
 	 * operations and ensures reliability under all circumstances.
 	 */
-	int_time = veml6075_it_ms[FIELD_GET(VEML6075_CONF_IT, conf)];
+	int_index = FIELD_GET(VEML6075_CONF_IT, conf);
+	if (int_index >= ARRAY_SIZE(veml6075_it_ms))
+		return -EINVAL;
+
+	int_time = veml6075_it_ms[int_index];
 	msleep(int_time + (int_time / 2));
 
 	/* shutdown again, data registers are still accessible */
diff --git a/drivers/iio/magnetometer/Kconfig b/drivers/iio/magnetometer/Kconfig
index fb313e591e850..ea2697fb5ab61 100644
--- a/drivers/iio/magnetometer/Kconfig
+++ b/drivers/iio/magnetometer/Kconfig
@@ -151,6 +151,17 @@ config MMC5633
 	  To compile this driver as a module, choose M here: the module
 	  will be called mmc5633
 
+config MMC5983
+	tristate "MEMSIC MMC5983 3-axis magnetic sensor"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Say yes here to build support for the MEMSIC MMC5983 3-axis
+	  magnetic sensor.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called mmc5983
+
 config IIO_ST_MAGN_3AXIS
 	tristate "STMicroelectronics magnetometers 3-Axis Driver"
 	depends on (I2C || SPI_MASTER) && SYSFS
diff --git a/drivers/iio/magnetometer/Makefile b/drivers/iio/magnetometer/Makefile
index 5bd227f8c1204..7fd9b3fd914e3 100644
--- a/drivers/iio/magnetometer/Makefile
+++ b/drivers/iio/magnetometer/Makefile
@@ -16,6 +16,7 @@ obj-$(CONFIG_MAG3110)	+= mag3110.o
 obj-$(CONFIG_HID_SENSOR_MAGNETOMETER_3D) += hid-sensor-magn-3d.o
 obj-$(CONFIG_MMC35240)	+= mmc35240.o
 obj-$(CONFIG_MMC5633)	+= mmc5633.o
+obj-$(CONFIG_MMC5983)	+= mmc5983.o
 
 obj-$(CONFIG_IIO_ST_MAGN_3AXIS) += st_magn.o
 st_magn-y := st_magn_core.o
diff --git a/drivers/iio/magnetometer/ak8975.c b/drivers/iio/magnetometer/ak8975.c
index b648b0afa5733..bb74abb648f13 100644
--- a/drivers/iio/magnetometer/ak8975.c
+++ b/drivers/iio/magnetometer/ak8975.c
@@ -7,24 +7,31 @@
  * Copyright (c) 2010, NVIDIA Corporation.
  */
 
-#include <linux/module.h>
-#include <linux/mod_devicetable.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
+#include <linux/array_size.h>
+#include <linux/bitops.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/err.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
-#include <linux/err.h>
+#include <linux/iopoll.h>
+#include <linux/jiffies.h>
+#include <linux/minmax.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
 #include <linux/mutex.h>
-#include <linux/delay.h>
-#include <linux/bitops.h>
-#include <linux/gpio/consumer.h>
-#include <linux/regulator/consumer.h>
 #include <linux/pm_runtime.h>
+#include <linux/property.h>
+#include <linux/regulator/consumer.h>
+#include <linux/time.h>
+#include <linux/types.h>
+#include <linux/wait.h>
+
+#include <asm/byteorder.h>
 
-#include <linux/iio/iio.h>
-#include <linux/iio/sysfs.h>
 #include <linux/iio/buffer.h>
-#include <linux/iio/trigger.h>
+#include <linux/iio/iio.h>
 #include <linux/iio/trigger_consumer.h>
 #include <linux/iio/triggered_buffer.h>
 
@@ -127,13 +134,6 @@
 
 #define AK09912_MAX_REGS		AK09912_REG_ASAZ
 
-/*
- * Miscellaneous values.
- */
-#define AK8975_MAX_CONVERSION_TIMEOUT	500
-#define AK8975_CONVERSION_DONE_POLL_TIME 10
-#define AK8975_DATA_READY_TIMEOUT	((100*HZ)/1000)
-
 /*
  * Precalculate scale factor (in Gauss units) for each axis and
  * store in the device data.
@@ -456,7 +456,8 @@ static int ak8975_power_on(const struct ak8975_data *data)
 	 * and the minimum wait time before mode setting is 100us, in
 	 * total 300us. Add some margin and say minimum 500us here.
 	 */
-	usleep_range(500, 1000);
+	fsleep(500);
+
 	return 0;
 }
 
@@ -473,9 +474,10 @@ static void ak8975_power_off(const struct ak8975_data *data)
  * Return 0 if the i2c device is the one we expect.
  * return a negative error number otherwise
  */
-static int ak8975_who_i_am(struct i2c_client *client,
+static int ak8975_who_i_am(const struct ak8975_data *data,
 			   enum asahi_compass_chipset type)
 {
+	struct i2c_client *client = data->client;
 	u8 wia_val[2];
 	int ret;
 
@@ -489,12 +491,18 @@ static int ak8975_who_i_am(struct i2c_client *client,
 	 * AK8975   |  DEVICE_ID |  NA
 	 * AK8963   |  DEVICE_ID |  NA
 	 */
-	ret = i2c_smbus_read_i2c_block_data_or_emulated(
-			client, AK09912_REG_WIA1, 2, wia_val);
+	ret = i2c_smbus_read_i2c_block_data_or_emulated(client,
+							AK09912_REG_WIA1,
+							sizeof(wia_val),
+							wia_val);
 	if (ret < 0) {
 		dev_err(&client->dev, "Error reading WIA\n");
 		return ret;
 	}
+	if (ret != sizeof(wia_val)) {
+		dev_err(&client->dev, "Error reading WIA\n");
+		return -EIO;
+	}
 
 	if (wia_val[0] != AK8975_DEVICE_ID)
 		return -ENODEV;
@@ -541,12 +549,12 @@ static int ak8975_set_mode(struct ak8975_data *data, enum ak_ctrl_mode mode)
 		 data->def->ctrl_modes[mode];
 	ret = i2c_smbus_write_byte_data(data->client,
 					data->def->ctrl_regs[CNTL], regval);
-	if (ret < 0) {
+	if (ret < 0)
 		return ret;
-	}
+
 	data->cntl_cache = regval;
 	/* After mode change wait at least 100us */
-	usleep_range(100, 500);
+	fsleep(100);
 
 	return 0;
 }
@@ -570,8 +578,8 @@ static irqreturn_t ak8975_irq_handler(int irq, void *data)
 static int ak8975_setup_irq(struct ak8975_data *data)
 {
 	struct i2c_client *client = data->client;
-	int rc;
 	int irq;
+	int ret;
 
 	init_waitqueue_head(&data->data_ready_queue);
 	clear_bit(0, &data->flags);
@@ -580,28 +588,24 @@ static int ak8975_setup_irq(struct ak8975_data *data)
 	else
 		irq = gpiod_to_irq(data->eoc_gpiod);
 
-	rc = devm_request_irq(&client->dev, irq, ak8975_irq_handler,
-			      IRQF_TRIGGER_RISING,
-			      dev_name(&client->dev), data);
-	if (rc < 0) {
-		dev_err(&client->dev, "irq %d request failed: %d\n", irq, rc);
-		return rc;
-	}
+	ret = devm_request_irq(&client->dev, irq, ak8975_irq_handler,
+			       IRQF_TRIGGER_RISING,
+			       dev_name(&client->dev), data);
+	if (ret)
+		return ret;
 
 	data->eoc_irq = irq;
 
-	return rc;
+	return 0;
 }
 
-
 /*
  * Perform some start-of-day setup, including reading the asa calibration
  * values and caching them.
  */
-static int ak8975_setup(struct i2c_client *client)
+static int ak8975_setup(struct ak8975_data *data)
 {
-	struct iio_dev *indio_dev = i2c_get_clientdata(client);
-	struct ak8975_data *data = iio_priv(indio_dev);
+	struct i2c_client *client = data->client;
 	int ret;
 
 	/* Write the fused rom access mode. */
@@ -612,13 +616,18 @@ static int ak8975_setup(struct i2c_client *client)
 	}
 
 	/* Get asa data and store in the device data. */
-	ret = i2c_smbus_read_i2c_block_data_or_emulated(
-			client, data->def->ctrl_regs[ASA_BASE],
-			3, data->asa);
+	ret = i2c_smbus_read_i2c_block_data_or_emulated(client,
+							data->def->ctrl_regs[ASA_BASE],
+							sizeof(data->asa),
+							data->asa);
 	if (ret < 0) {
 		dev_err(&client->dev, "Not able to read asa data\n");
 		return ret;
 	}
+	if (ret != sizeof(data->asa)) {
+		dev_err(&client->dev, "Error reading asa data\n");
+		return -EIO;
+	}
 
 	/* After reading fuse ROM data set power-down mode */
 	ret = ak8975_set_mode(data, POWER_DOWN);
@@ -643,22 +652,23 @@ static int ak8975_setup(struct i2c_client *client)
 	return 0;
 }
 
-static int wait_conversion_complete_gpio(struct ak8975_data *data)
+static int wait_conversion_complete_gpio(struct ak8975_data *data,
+					 unsigned int poll_ms,
+					 unsigned int timeout_ms)
 {
 	struct i2c_client *client = data->client;
-	u32 timeout_ms = AK8975_MAX_CONVERSION_TIMEOUT;
 	int ret;
+	int val;
 
 	/* Wait for the conversion to complete. */
-	while (timeout_ms) {
-		msleep(AK8975_CONVERSION_DONE_POLL_TIME);
-		if (gpiod_get_value(data->eoc_gpiod))
-			break;
-		timeout_ms -= AK8975_CONVERSION_DONE_POLL_TIME;
-	}
-	if (!timeout_ms) {
-		dev_err(&client->dev, "Conversion timeout happened\n");
-		return -EINVAL;
+	ret = readx_poll_timeout(gpiod_get_value, data->eoc_gpiod, val, val != 0,
+				 poll_ms * USEC_PER_MSEC,
+				 timeout_ms * USEC_PER_MSEC);
+	if (ret)
+		return ret;
+	if (val < 0) {
+		dev_err(&client->dev, "Error in reading GPIOD\n");
+		return val;
 	}
 
 	ret = i2c_smbus_read_byte_data(client, data->def->ctrl_regs[ST1]);
@@ -668,54 +678,49 @@ static int wait_conversion_complete_gpio(struct ak8975_data *data)
 	return ret;
 }
 
-static int wait_conversion_complete_polled(struct ak8975_data *data)
+static int wait_conversion_complete_polled(struct ak8975_data *data,
+					   unsigned int poll_ms,
+					   unsigned int timeout_ms)
 {
 	struct i2c_client *client = data->client;
-	u8 read_status;
-	u32 timeout_ms = AK8975_MAX_CONVERSION_TIMEOUT;
 	int ret;
+	int val;
 
 	/* Wait for the conversion to complete. */
-	while (timeout_ms) {
-		msleep(AK8975_CONVERSION_DONE_POLL_TIME);
-		ret = i2c_smbus_read_byte_data(client,
-					       data->def->ctrl_regs[ST1]);
-		if (ret < 0) {
-			dev_err(&client->dev, "Error in reading ST1\n");
-			return ret;
-		}
-		read_status = ret;
-		if (read_status)
-			break;
-		timeout_ms -= AK8975_CONVERSION_DONE_POLL_TIME;
-	}
-	if (!timeout_ms) {
-		dev_err(&client->dev, "Conversion timeout happened\n");
-		return -EINVAL;
-	}
+	ret = read_poll_timeout(i2c_smbus_read_byte_data, val, val != 0,
+				poll_ms * USEC_PER_MSEC,
+				timeout_ms * USEC_PER_MSEC,
+				true,
+				client, data->def->ctrl_regs[ST1]);
+	if (ret)
+		return ret;
+	if (val < 0)
+		dev_err(&client->dev, "Error in reading ST1\n");
 
-	return read_status;
+	return val;
 }
 
-/* Returns 0 if the end of conversion interrupt occurred or -ETIME otherwise */
-static int wait_conversion_complete_interrupt(struct ak8975_data *data)
+/* Returns 0 if the end of conversion interrupt occurred or -ETIMEDOUT otherwise */
+static int wait_conversion_complete_interrupt(struct ak8975_data *data,
+					      unsigned int timeout_ms)
 {
 	int ret;
 
 	ret = wait_event_timeout(data->data_ready_queue,
 				 test_bit(0, &data->flags),
-				 AK8975_DATA_READY_TIMEOUT);
+				 msecs_to_jiffies(timeout_ms));
 	clear_bit(0, &data->flags);
 
-	return ret > 0 ? 0 : -ETIME;
+	return ret > 0 ? 0 : -ETIMEDOUT;
 }
 
-static int ak8975_start_read_axis(struct ak8975_data *data,
-				  const struct i2c_client *client)
+static int ak8975_start_read_axis(struct ak8975_data *data)
 {
-	/* Set up the device for taking a sample. */
-	int ret = ak8975_set_mode(data, MODE_ONCE);
+	struct i2c_client *client = data->client;
+	int ret;
 
+	/* Set up the device for taking a sample. */
+	ret = ak8975_set_mode(data, MODE_ONCE);
 	if (ret < 0) {
 		dev_err(&client->dev, "Error in setting operating mode\n");
 		return ret;
@@ -723,11 +728,11 @@ static int ak8975_start_read_axis(struct ak8975_data *data,
 
 	/* Wait for the conversion to complete. */
 	if (data->eoc_irq)
-		ret = wait_conversion_complete_interrupt(data);
+		ret = wait_conversion_complete_interrupt(data, 100);
 	else if (data->eoc_gpiod)
-		ret = wait_conversion_complete_gpio(data);
+		ret = wait_conversion_complete_gpio(data, 10, 500);
 	else
-		ret = wait_conversion_complete_polled(data);
+		ret = wait_conversion_complete_polled(data, 10, 500);
 	if (ret < 0)
 		return ret;
 
@@ -742,22 +747,26 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val)
 	const struct i2c_client *client = data->client;
 	const struct ak_def *def = data->def;
 	__le16 rval;
-	u16 buff;
 	int ret;
 
 	pm_runtime_get_sync(&data->client->dev);
 
 	mutex_lock(&data->lock);
 
-	ret = ak8975_start_read_axis(data, client);
+	ret = ak8975_start_read_axis(data);
 	if (ret)
 		goto exit;
 
-	ret = i2c_smbus_read_i2c_block_data_or_emulated(
-			client, def->data_regs[index],
-			sizeof(rval), (u8*)&rval);
+	ret = i2c_smbus_read_i2c_block_data_or_emulated(client,
+							def->data_regs[index],
+							sizeof(rval),
+							(u8 *)&rval);
 	if (ret < 0)
 		goto exit;
+	if (ret != sizeof(rval)) {
+		ret = -EIO;
+		goto exit;
+	}
 
 	/* Read out ST2 for release lock on measurement data. */
 	ret = i2c_smbus_read_byte_data(client, data->def->ctrl_regs[ST2]);
@@ -778,12 +787,13 @@ static int ak8975_read_axis(struct iio_dev *indio_dev, int index, int *val)
 	pm_runtime_put_autosuspend(&data->client->dev);
 
 	/* Swap bytes and convert to valid range. */
-	buff = le16_to_cpu(rval);
-	*val = clamp_t(s16, buff, -def->range, def->range);
+	*val = clamp_t(s16, le16_to_cpu(rval), -def->range, def->range);
+
 	return IIO_VAL_INT;
 
 exit:
 	mutex_unlock(&data->lock);
+	pm_runtime_put_autosuspend(&data->client->dev);
 	dev_err(&client->dev, "Error in reading axis\n");
 	return ret;
 }
@@ -859,7 +869,7 @@ static void ak8975_fill_buffer(struct iio_dev *indio_dev)
 
 	mutex_lock(&data->lock);
 
-	ret = ak8975_start_read_axis(data, client);
+	ret = ak8975_start_read_axis(data);
 	if (ret)
 		goto unlock;
 
@@ -869,10 +879,12 @@ static void ak8975_fill_buffer(struct iio_dev *indio_dev)
 	 */
 	ret = i2c_smbus_read_i2c_block_data_or_emulated(client,
 							def->data_regs[0],
-							3 * sizeof(fval[0]),
+							sizeof(fval),
 							(u8 *)fval);
 	if (ret < 0)
 		goto unlock;
+	if (ret != sizeof(fval))
+		goto unlock;
 
 	mutex_unlock(&data->lock);
 
@@ -901,6 +913,28 @@ static irqreturn_t ak8975_handle_trigger(int irq, void *p)
 	return IRQ_HANDLED;
 }
 
+static int ak8975_buffer_preenable(struct iio_dev *indio_dev)
+{
+	struct ak8975_data *data = iio_priv(indio_dev);
+	struct device *dev = &data->client->dev;
+
+	return pm_runtime_resume_and_get(dev);
+}
+
+static int ak8975_buffer_postdisable(struct iio_dev *indio_dev)
+{
+	struct ak8975_data *data = iio_priv(indio_dev);
+	struct device *dev = &data->client->dev;
+
+	pm_runtime_put_autosuspend(dev);
+
+	return 0;
+}
+
+static const struct iio_buffer_setup_ops ak8975_buffer_setup_ops = {
+	.preenable = ak8975_buffer_preenable,
+	.postdisable = ak8975_buffer_postdisable,
+};
 static int ak8975_probe(struct i2c_client *client)
 {
 	const struct i2c_device_id *id = i2c_client_get_device_id(client);
@@ -908,8 +942,8 @@ static int ak8975_probe(struct i2c_client *client)
 	struct iio_dev *indio_dev;
 	struct gpio_desc *eoc_gpiod;
 	struct gpio_desc *reset_gpiod;
-	int err;
 	const char *name = NULL;
+	int ret;
 
 	/*
 	 * Grab and set up the supplied GPIO.
@@ -919,8 +953,7 @@ static int ak8975_probe(struct i2c_client *client)
 	eoc_gpiod = devm_gpiod_get_optional(&client->dev, NULL, GPIOD_IN);
 	if (IS_ERR(eoc_gpiod))
 		return PTR_ERR(eoc_gpiod);
-	if (eoc_gpiod)
-		gpiod_set_consumer_name(eoc_gpiod, "ak_8975");
+	gpiod_set_consumer_name(eoc_gpiod, "ak_8975");
 
 	/*
 	 * According to AK09911 datasheet, if reset GPIO is provided then
@@ -945,9 +978,9 @@ static int ak8975_probe(struct i2c_client *client)
 	data->reset_gpiod = reset_gpiod;
 	data->eoc_irq = 0;
 
-	err = iio_read_mount_matrix(&client->dev, &data->orientation);
-	if (err)
-		return err;
+	ret = iio_read_mount_matrix(&client->dev, &data->orientation);
+	if (ret)
+		return ret;
 
 	/* id will be NULL when enumerated via ACPI */
 	data->def = i2c_get_match_data(client);
@@ -968,20 +1001,20 @@ static int ak8975_probe(struct i2c_client *client)
 	if (IS_ERR(data->vid))
 		return PTR_ERR(data->vid);
 
-	err = ak8975_power_on(data);
-	if (err)
-		return err;
+	ret = ak8975_power_on(data);
+	if (ret)
+		return ret;
 
-	err = ak8975_who_i_am(client, data->def->type);
-	if (err < 0) {
+	ret = ak8975_who_i_am(data, data->def->type);
+	if (ret) {
 		dev_err(&client->dev, "Unexpected device\n");
 		goto power_off;
 	}
 	dev_dbg(&client->dev, "Asahi compass chip %s\n", name);
 
 	/* Perform some basic start-of-day setup of the device. */
-	err = ak8975_setup(client);
-	if (err < 0) {
+	ret = ak8975_setup(data);
+	if (ret) {
 		dev_err(&client->dev, "%s initialization fails\n", name);
 		goto power_off;
 	}
@@ -994,15 +1027,15 @@ static int ak8975_probe(struct i2c_client *client)
 	indio_dev->modes = INDIO_DIRECT_MODE;
 	indio_dev->name = name;
 
-	err = iio_triggered_buffer_setup(indio_dev, NULL, ak8975_handle_trigger,
-					 NULL);
-	if (err) {
+	ret = iio_triggered_buffer_setup(indio_dev, NULL, ak8975_handle_trigger,
+					 &ak8975_buffer_setup_ops);
+	if (ret) {
 		dev_err(&client->dev, "triggered buffer setup failed\n");
 		goto power_off;
 	}
 
-	err = iio_device_register(indio_dev);
-	if (err) {
+	ret = iio_device_register(indio_dev);
+	if (ret) {
 		dev_err(&client->dev, "device register failed\n");
 		goto cleanup_buffer;
 	}
@@ -1025,7 +1058,7 @@ static int ak8975_probe(struct i2c_client *client)
 	iio_triggered_buffer_cleanup(indio_dev);
 power_off:
 	ak8975_power_off(data);
-	return err;
+	return ret;
 }
 
 static void ak8975_remove(struct i2c_client *client)
diff --git a/drivers/iio/magnetometer/hid-sensor-magn-3d.c b/drivers/iio/magnetometer/hid-sensor-magn-3d.c
index c673f9323e476..b01dd53eb1009 100644
--- a/drivers/iio/magnetometer/hid-sensor-magn-3d.c
+++ b/drivers/iio/magnetometer/hid-sensor-magn-3d.c
@@ -280,7 +280,7 @@ static const struct iio_info magn_3d_info = {
 
 /* Callback handler to send event after all samples are received and captured */
 static int magn_3d_proc_event(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
+				u32 usage_id,
 				void *priv)
 {
 	struct iio_dev *indio_dev = platform_get_drvdata(priv);
@@ -302,7 +302,7 @@ static int magn_3d_proc_event(struct hid_sensor_hub_device *hsdev,
 
 /* Capture samples in local storage */
 static int magn_3d_capture_sample(struct hid_sensor_hub_device *hsdev,
-				unsigned usage_id,
+				u32 usage_id,
 				size_t raw_len, char *raw_data,
 				void *priv)
 {
@@ -350,7 +350,7 @@ static int magn_3d_parse_report(struct platform_device *pdev,
 				struct hid_sensor_hub_device *hsdev,
 				struct iio_chan_spec **channels,
 				int *chan_count,
-				unsigned usage_id,
+				u32 usage_id,
 				struct magn_3d_state *st)
 {
 	int i;
diff --git a/drivers/iio/magnetometer/mmc5983.c b/drivers/iio/magnetometer/mmc5983.c
new file mode 100644
index 0000000000000..a67b13393b6b4
--- /dev/null
+++ b/drivers/iio/magnetometer/mmc5983.c
@@ -0,0 +1,348 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * MMC5983 - MEMSIC 3-axis Magnetic Sensor
+ *
+ * Copyright (c) 2026, Vlad Kulikov <vlad.kulikov.c@gmail.com>
+ *
+ * IIO driver for MMC5983
+ */
+
+#include <linux/array_size.h>
+#include <linux/bits.h>
+#include <linux/cleanup.h>
+#include <linux/delay.h>
+#include <linux/dev_printk.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/iio/iio.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/regmap.h>
+#include <linux/time.h>
+#include <linux/types.h>
+
+#define MMC5983_REG_XOUT0	0x00
+#define MMC5983_REG_XOUT1	0x01
+#define MMC5983_REG_YOUT0	0x02
+#define MMC5983_REG_YOUT1	0x03
+#define MMC5983_REG_ZOUT0	0x04
+#define MMC5983_REG_ZOUT1	0x05
+#define MMC5983_REG_XYZOUT2	0x06
+
+#define MMC5983_REG_STATUS	0x08
+
+#define MMC5983_REG_CTRL0	0x09
+#define MMC5983_REG_CTRL1	0x0A
+#define MMC5983_REG_CTRL2	0x0B
+#define MMC5983_REG_CTRL3	0x0C
+
+#define MMC5983_REG_ID		0x2F
+
+#define MMC5983_PRODUCT_ID	0x30
+
+#define MMC5983_STATUS_MEAS_M_DONE_BIT	BIT(0)
+#define MMC5983_STATUS_OTP_RD_DONE_BIT	BIT(4)
+
+#define MMC5983_CTRL0_TM_M_BIT		BIT(0)
+#define MMC5983_CTRL0_SET_BIT		BIT(3)
+#define MMC5983_CTRL0_RESET_BIT		BIT(4)
+#define MMC5983_CTRL0_OTP_RD_BIT	BIT(6)
+
+#define MMC5983_CTRL1_SW_RST_BIT	BIT(7)
+
+enum mmc5983_axis {
+	MMC5983_AXIS_X,
+	MMC5983_AXIS_Y,
+	MMC5983_AXIS_Z,
+};
+
+struct mmc5983_data {
+	struct regmap *regmap;
+	/* Protects chip access during SET/RESET measurement sequence */
+	struct mutex mutex;
+};
+
+#define MMC5983_CHANNEL(_axis) { \
+	.type = IIO_MAGN, \
+	.modified = 1, \
+	.channel2 = IIO_MOD_##_axis, \
+	.address = MMC5983_AXIS_##_axis, \
+	.info_mask_separate = BIT(IIO_CHAN_INFO_RAW), \
+	.info_mask_shared_by_type = BIT(IIO_CHAN_INFO_SCALE), \
+}
+
+static const struct iio_chan_spec mmc5983_channels[] = {
+	MMC5983_CHANNEL(X),
+	MMC5983_CHANNEL(Y),
+	MMC5983_CHANNEL(Z),
+};
+
+static int mmc5983_pulse_coil(struct mmc5983_data *data, unsigned int coil_bit)
+{
+	int ret;
+
+	ret = regmap_write(data->regmap, MMC5983_REG_CTRL0, coil_bit);
+	if (ret)
+		return ret;
+
+	/*
+	 * Datasheet page 15: SET/RESET coil pulse is 500 ns.
+	 * Vendor sample code waits 500 us before the next operation.
+	 */
+	fsleep(500);
+
+	return 0;
+}
+
+static int mmc5983_take_measurement(struct mmc5983_data *data, int m[3])
+{
+	unsigned int status;
+	u8 buf[7];
+	int ret;
+
+	ret = regmap_write(data->regmap, MMC5983_REG_CTRL0,
+			   MMC5983_CTRL0_TM_M_BIT);
+	if (ret)
+		return ret;
+
+	/*
+	 * Datasheet page 15: measurement time is 8 ms at BW=00 (default,
+	 * slowest setting). Use a 50 ms timeout for margin.
+	 */
+	ret = regmap_read_poll_timeout(data->regmap, MMC5983_REG_STATUS,
+				       status,
+				       status & MMC5983_STATUS_MEAS_M_DONE_BIT,
+				       10 * USEC_PER_MSEC,
+				       50 * USEC_PER_MSEC);
+	if (ret)
+		return ret;
+
+	ret = regmap_bulk_read(data->regmap, MMC5983_REG_XOUT0, buf,
+			       sizeof(buf));
+	if (ret)
+		return ret;
+
+	m[0] = (buf[0] << 10) | (buf[1] << 2) | ((buf[6] >> 6) & 0x3);
+	m[1] = (buf[2] << 10) | (buf[3] << 2) | ((buf[6] >> 4) & 0x3);
+	m[2] = (buf[4] << 10) | (buf[5] << 2) | ((buf[6] >> 2) & 0x3);
+
+	return 0;
+}
+
+static int mmc5983_read_raw(struct iio_dev *indio_dev,
+			     const struct iio_chan_spec *chan, int *val,
+			     int *val2, long mask)
+{
+	struct mmc5983_data *data = iio_priv(indio_dev);
+	int m1[3], m2[3];
+	int ret;
+
+	switch (mask) {
+	case IIO_CHAN_INFO_RAW: {
+		guard(mutex)(&data->mutex);
+
+		/* SET: magnetize sensor elements in forward direction */
+		ret = mmc5983_pulse_coil(data, MMC5983_CTRL0_SET_BIT);
+		if (ret)
+			return ret;
+
+		ret = mmc5983_take_measurement(data, m1);
+		if (ret)
+			return ret;
+
+		/* RESET: magnetize sensor elements in reverse direction */
+		ret = mmc5983_pulse_coil(data, MMC5983_CTRL0_RESET_BIT);
+		if (ret)
+			return ret;
+
+		ret = mmc5983_take_measurement(data, m2);
+		if (ret)
+			return ret;
+
+		*val = (m1[chan->address] - m2[chan->address]) / 2;
+		return IIO_VAL_INT;
+	}
+	case IIO_CHAN_INFO_SCALE:
+		*val = 0;
+		*val2 = 61035;
+		return IIO_VAL_INT_PLUS_NANO;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct iio_info mmc5983_info = {
+	.read_raw = mmc5983_read_raw,
+};
+
+static bool mmc5983_is_writeable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case MMC5983_REG_CTRL0:
+	case MMC5983_REG_CTRL1:
+	case MMC5983_REG_CTRL2:
+	case MMC5983_REG_CTRL3:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool mmc5983_is_readable_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case MMC5983_REG_XOUT0:
+	case MMC5983_REG_XOUT1:
+	case MMC5983_REG_YOUT0:
+	case MMC5983_REG_YOUT1:
+	case MMC5983_REG_ZOUT0:
+	case MMC5983_REG_ZOUT1:
+	case MMC5983_REG_XYZOUT2:
+	case MMC5983_REG_STATUS:
+	case MMC5983_REG_CTRL0:
+	case MMC5983_REG_CTRL1:
+	case MMC5983_REG_CTRL2:
+	case MMC5983_REG_CTRL3:
+	case MMC5983_REG_ID:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool mmc5983_is_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case MMC5983_REG_XOUT0:
+	case MMC5983_REG_XOUT1:
+	case MMC5983_REG_YOUT0:
+	case MMC5983_REG_YOUT1:
+	case MMC5983_REG_ZOUT0:
+	case MMC5983_REG_ZOUT1:
+	case MMC5983_REG_XYZOUT2:
+	case MMC5983_REG_STATUS:
+	case MMC5983_REG_CTRL0:
+	case MMC5983_REG_CTRL1:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_config mmc5983_regmap_config = {
+	.name = "mmc5983_regmap",
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = MMC5983_REG_ID,
+	.writeable_reg = mmc5983_is_writeable_reg,
+	.readable_reg = mmc5983_is_readable_reg,
+	.volatile_reg = mmc5983_is_volatile_reg,
+};
+
+static int mmc5983_init(struct mmc5983_data *data)
+{
+	struct regmap *regmap = data->regmap;
+	struct device *dev = regmap_get_device(regmap);
+	unsigned int reg_id, status;
+	int ret;
+
+	ret = regmap_read(regmap, MMC5983_REG_ID, &reg_id);
+	if (ret)
+		return dev_err_probe(dev, ret, "Error reading product id\n");
+
+	if (reg_id != MMC5983_PRODUCT_ID)
+		dev_info(dev, "unexpected product id 0x%02x\n", reg_id);
+
+	ret = regmap_write(regmap, MMC5983_REG_CTRL1, MMC5983_CTRL1_SW_RST_BIT);
+	if (ret)
+		return ret;
+
+	/* Datasheet page 15: power-on time after SW_RST is 10 ms */
+	fsleep(10 * USEC_PER_MSEC);
+
+	ret = regmap_write(regmap, MMC5983_REG_CTRL0, MMC5983_CTRL0_OTP_RD_BIT);
+	if (ret)
+		return ret;
+
+	/*
+	 * Datasheet page 15: OTP read completes and self-clears. No separate
+	 * OTP refresh timeout is specified, so use the 10 ms power-on time as
+	 * a conservative upper bound.
+	 */
+	ret = regmap_read_poll_timeout(regmap, MMC5983_REG_STATUS, status,
+				       status & MMC5983_STATUS_OTP_RD_DONE_BIT,
+				       USEC_PER_MSEC,
+				       10 * USEC_PER_MSEC);
+	if (ret)
+		return ret;
+
+	/* SET: magnetize sensor elements in forward direction */
+	ret = mmc5983_pulse_coil(data, MMC5983_CTRL0_SET_BIT);
+	if (ret)
+		return ret;
+
+	/* RESET: magnetize sensor elements in reverse direction */
+	return mmc5983_pulse_coil(data, MMC5983_CTRL0_RESET_BIT);
+}
+
+static int mmc5983_probe(struct i2c_client *i2c)
+{
+	struct device *dev = &i2c->dev;
+	struct mmc5983_data *data;
+	struct iio_dev *indio_dev;
+	int ret;
+
+	indio_dev = devm_iio_device_alloc(dev, sizeof(*data));
+	if (!indio_dev)
+		return -ENOMEM;
+
+	data = iio_priv(indio_dev);
+
+	ret = devm_mutex_init(dev, &data->mutex);
+	if (ret)
+		return ret;
+
+	data->regmap = devm_regmap_init_i2c(i2c, &mmc5983_regmap_config);
+	if (IS_ERR(data->regmap))
+		return dev_err_probe(dev, PTR_ERR(data->regmap),
+				     "failed to allocate register map\n");
+
+	indio_dev->info = &mmc5983_info;
+	indio_dev->modes = INDIO_DIRECT_MODE;
+	indio_dev->name = "mmc5983";
+	indio_dev->channels = mmc5983_channels;
+	indio_dev->num_channels = ARRAY_SIZE(mmc5983_channels);
+
+	ret = mmc5983_init(data);
+	if (ret)
+		return dev_err_probe(dev, ret, "mmc5983 chip init failed\n");
+
+	return devm_iio_device_register(dev, indio_dev);
+}
+
+static const struct of_device_id mmc5983_of_match[] = {
+	{ .compatible = "memsic,mmc5983" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mmc5983_of_match);
+
+static const struct i2c_device_id mmc5983_id[] = {
+	{ "mmc5983" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, mmc5983_id);
+
+static struct i2c_driver mmc5983_driver = {
+	.driver = {
+		.name = "mmc5983",
+		.of_match_table = mmc5983_of_match,
+	},
+	.probe = mmc5983_probe,
+	.id_table = mmc5983_id,
+};
+module_i2c_driver(mmc5983_driver);
+
+MODULE_AUTHOR("Vladislav Kulikov <vlad.kulikov.c@gmail.com>");
+MODULE_DESCRIPTION("MEMSIC MMC5983 magnetic sensor driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/iio/magnetometer/rm3100-core.c b/drivers/iio/magnetometer/rm3100-core.c
index 2b28844257460..ac3f9f7fc8089 100644
--- a/drivers/iio/magnetometer/rm3100-core.c
+++ b/drivers/iio/magnetometer/rm3100-core.c
@@ -204,27 +204,23 @@ static int rm3100_read_mag(struct rm3100_data *data, int idx, int *val)
 	u8 buffer[3];
 	int ret;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
+
 	ret = regmap_write(regmap, RM3100_REG_POLL, BIT(4 + idx));
 	if (ret < 0)
-		goto unlock_return;
+		return ret;
 
 	ret = rm3100_wait_measurement(data);
 	if (ret < 0)
-		goto unlock_return;
+		return ret;
 
 	ret = regmap_bulk_read(regmap, RM3100_REG_MX2 + 3 * idx, buffer, 3);
 	if (ret < 0)
-		goto unlock_return;
-	mutex_unlock(&data->lock);
+		return ret;
 
 	*val = sign_extend32(get_unaligned_be24(&buffer[0]), 23);
 
 	return IIO_VAL_INT;
-
-unlock_return:
-	mutex_unlock(&data->lock);
-	return ret;
 }
 
 #define RM3100_CHANNEL(axis, idx)					\
@@ -284,11 +280,12 @@ static int rm3100_get_samp_freq(struct rm3100_data *data, int *val, int *val2)
 	unsigned int tmp;
 	int ret;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
+
 	ret = regmap_read(data->regmap, RM3100_REG_TMRC, &tmp);
-	mutex_unlock(&data->lock);
 	if (ret < 0)
 		return ret;
+
 	*val = rm3100_samp_rates[tmp - RM3100_TMRC_OFFSET][0];
 	*val2 = rm3100_samp_rates[tmp - RM3100_TMRC_OFFSET][1];
 
@@ -338,56 +335,50 @@ static int rm3100_set_samp_freq(struct iio_dev *indio_dev, int val, int val2)
 	int ret;
 	int i;
 
-	mutex_lock(&data->lock);
+	guard(mutex)(&data->lock);
+
 	/* All cycle count registers use the same value. */
 	ret = regmap_read(regmap, RM3100_REG_CC_X, &cycle_count);
 	if (ret < 0)
-		goto unlock_return;
+		return ret;
 
 	for (i = 0; i < RM3100_SAMP_NUM; i++) {
 		if (val == rm3100_samp_rates[i][0] &&
 		    val2 == rm3100_samp_rates[i][1])
 			break;
 	}
-	if (i == RM3100_SAMP_NUM) {
-		ret = -EINVAL;
-		goto unlock_return;
-	}
+	if (i == RM3100_SAMP_NUM)
+		return -EINVAL;
 
 	ret = regmap_write(regmap, RM3100_REG_TMRC, i + RM3100_TMRC_OFFSET);
 	if (ret < 0)
-		goto unlock_return;
+		return ret;
 
 	/* Checking if cycle count registers need changing. */
 	if (val == 600 && cycle_count == 200) {
 		ret = rm3100_set_cycle_count(data, 100);
 		if (ret < 0)
-			goto unlock_return;
+			return ret;
 	} else if (val != 600 && cycle_count == 100) {
 		ret = rm3100_set_cycle_count(data, 200);
 		if (ret < 0)
-			goto unlock_return;
+			return ret;
 	}
 
 	if (iio_buffer_enabled(indio_dev)) {
 		/* Writing TMRC registers requires CMM reset. */
 		ret = regmap_write(regmap, RM3100_REG_CMM, 0);
 		if (ret < 0)
-			goto unlock_return;
+			return ret;
 		ret = regmap_write(data->regmap, RM3100_REG_CMM,
 			(*indio_dev->active_scan_mask & 0x7) <<
 			RM3100_CMM_AXIS_SHIFT | RM3100_CMM_START);
 		if (ret < 0)
-			goto unlock_return;
+			return ret;
 	}
-	mutex_unlock(&data->lock);
 
 	data->conversion_time = rm3100_samp_rates[i][2] * 2;
 	return 0;
-
-unlock_return:
-	mutex_unlock(&data->lock);
-	return ret;
 }
 
 static int rm3100_read_raw(struct iio_dev *indio_dev,
@@ -458,6 +449,27 @@ static const struct iio_buffer_setup_ops rm3100_buffer_ops = {
 	.postdisable = rm3100_buffer_postdisable,
 };
 
+/**
+ * rm3100_regmap_bulk_read_locked() - Wrapper around regmap_bulk_read() with a mutex
+ *
+ * @data: Data structure containing regmap and mutex
+ * @reg: First register to be read from, passed to regmap_bulk_read()
+ * @val: Pointer to store read value, in native register size for device,
+ * passed to regmap_bulk_read()
+ * @val_count: Number of registers to read, passed to regmap_bulk_read()
+ *
+ * Intended for use only in rm3100_trigger_handler().
+ *
+ * Return:
+ * A value of zero on success, a negative errno in error cases.
+ */
+static int rm3100_regmap_bulk_read_locked(struct rm3100_data *data, unsigned int reg,
+					  void *val, size_t val_count)
+{
+	guard(mutex)(&data->lock);
+	return regmap_bulk_read(data->regmap, reg, val, val_count);
+}
+
 static irqreturn_t rm3100_trigger_handler(int irq, void *p)
 {
 	struct iio_poll_func *pf = p;
@@ -465,14 +477,12 @@ static irqreturn_t rm3100_trigger_handler(int irq, void *p)
 	unsigned long scan_mask = *indio_dev->active_scan_mask;
 	unsigned int mask_len = iio_get_masklength(indio_dev);
 	struct rm3100_data *data = iio_priv(indio_dev);
-	struct regmap *regmap = data->regmap;
 	int ret, i, bit;
 
-	mutex_lock(&data->lock);
 	switch (scan_mask) {
 	case BIT(0) | BIT(1) | BIT(2):
-		ret = regmap_bulk_read(regmap, RM3100_REG_MX2, data->buffer, 9);
-		mutex_unlock(&data->lock);
+		ret = rm3100_regmap_bulk_read_locked(data, RM3100_REG_MX2,
+						     data->buffer, 9);
 		if (ret < 0)
 			goto done;
 		/* Convert XXXYYYZZZxxx to XXXxYYYxZZZx. x for paddings. */
@@ -480,36 +490,34 @@ static irqreturn_t rm3100_trigger_handler(int irq, void *p)
 			memmove(data->buffer + i * 4, data->buffer + i * 3, 3);
 		break;
 	case BIT(0) | BIT(1):
-		ret = regmap_bulk_read(regmap, RM3100_REG_MX2, data->buffer, 6);
-		mutex_unlock(&data->lock);
+		ret = rm3100_regmap_bulk_read_locked(data, RM3100_REG_MX2,
+						     data->buffer, 6);
 		if (ret < 0)
 			goto done;
 		memmove(data->buffer + 4, data->buffer + 3, 3);
 		break;
 	case BIT(1) | BIT(2):
-		ret = regmap_bulk_read(regmap, RM3100_REG_MY2, data->buffer, 6);
-		mutex_unlock(&data->lock);
+		ret = rm3100_regmap_bulk_read_locked(data, RM3100_REG_MY2,
+						     data->buffer, 6);
 		if (ret < 0)
 			goto done;
 		memmove(data->buffer + 4, data->buffer + 3, 3);
 		break;
 	case BIT(0) | BIT(2):
-		ret = regmap_bulk_read(regmap, RM3100_REG_MX2, data->buffer, 9);
-		mutex_unlock(&data->lock);
+		ret = rm3100_regmap_bulk_read_locked(data, RM3100_REG_MX2,
+						     data->buffer, 9);
 		if (ret < 0)
 			goto done;
 		memmove(data->buffer + 4, data->buffer + 6, 3);
 		break;
 	default:
 		for_each_set_bit(bit, &scan_mask, mask_len) {
-			ret = regmap_bulk_read(regmap, RM3100_REG_MX2 + 3 * bit,
-					       data->buffer, 3);
-			if (ret < 0) {
-				mutex_unlock(&data->lock);
+			ret = rm3100_regmap_bulk_read_locked(data,
+							     RM3100_REG_MX2 + 3 * bit,
+							     data->buffer, 3);
+			if (ret < 0)
 				goto done;
-			}
 		}
-		mutex_unlock(&data->lock);
 	}
 	/*
 	 * Always using the same buffer so that we wouldn't need to set the
diff --git a/drivers/iio/magnetometer/st_magn_core.c b/drivers/iio/magnetometer/st_magn_core.c
index ef348d316c001..7644bd04654b2 100644
--- a/drivers/iio/magnetometer/st_magn_core.c
+++ b/drivers/iio/magnetometer/st_magn_core.c
@@ -506,6 +506,11 @@ static const struct st_sensors_platform_data default_magn_pdata = {
 	.drdy_int_pin = 2,
 };
 
+/* LIS2MDL only supports DRDY on INT1 */
+static const struct st_sensors_platform_data alt_magn_pdata = {
+	.drdy_int_pin = 1,
+};
+
 static int st_magn_read_raw(struct iio_dev *indio_dev,
 			struct iio_chan_spec const *ch, int *val,
 							int *val2, long mask)
@@ -628,8 +633,12 @@ int st_magn_common_probe(struct iio_dev *indio_dev)
 	mdata->current_fullscale = &mdata->sensor_settings->fs.fs_avl[0];
 	mdata->odr = mdata->sensor_settings->odr.odr_avl[0].hz;
 
-	if (!pdata)
-		pdata = (struct st_sensors_platform_data *)&default_magn_pdata;
+	if (!pdata) {
+		if (mdata->sensor_settings->drdy_irq.int2.mask)
+			pdata = (struct st_sensors_platform_data *)&default_magn_pdata;
+		else
+			pdata = (struct st_sensors_platform_data *)&alt_magn_pdata;
+	}
 
 	err = st_sensors_init_sensor(indio_dev, pdata);
 	if (err < 0)
diff --git a/drivers/iio/magnetometer/yamaha-yas530.c b/drivers/iio/magnetometer/yamaha-yas530.c
index 140c422773f6a..75ff2ba0c23f7 100644
--- a/drivers/iio/magnetometer/yamaha-yas530.c
+++ b/drivers/iio/magnetometer/yamaha-yas530.c
@@ -168,6 +168,7 @@ struct yas5xx;
 /**
  * struct yas5xx_chip_info - device-specific data and function pointers
  * @devid: device ID number
+ * @product_label: product label used in Linux
  * @product_name: product name of the YAS variant
  * @version_names: version letters or namings
  * @volatile_reg: device-specific volatile registers
@@ -189,6 +190,7 @@ struct yas5xx;
  */
 struct yas5xx_chip_info {
 	unsigned int devid;
+	const char *product_label;
 	const char *product_name;
 	const char *version_names[2];
 	const int *volatile_reg;
@@ -859,9 +861,9 @@ static int yas530_get_calibration_data(struct yas5xx *yas5xx)
 	c->f[0] = FIELD_GET(GENMASK(22, 21), val);
 	c->f[1] = FIELD_GET(GENMASK(14, 13), val);
 	c->f[2] = FIELD_GET(GENMASK(6, 5), val);
-	c->r[0] = sign_extend32(FIELD_GET(GENMASK(28, 23), val), 5);
-	c->r[1] = sign_extend32(FIELD_GET(GENMASK(20, 15), val), 5);
-	c->r[2] = sign_extend32(FIELD_GET(GENMASK(12, 7), val), 5);
+	c->r[0] = FIELD_GET_SIGNED(GENMASK(28, 23), val);
+	c->r[1] = FIELD_GET_SIGNED(GENMASK(20, 15), val);
+	c->r[2] = FIELD_GET_SIGNED(GENMASK(12, 7), val);
 
 	return 0;
 }
@@ -914,9 +916,9 @@ static int yas532_get_calibration_data(struct yas5xx *yas5xx)
 	c->f[0] = FIELD_GET(GENMASK(24, 23), val);
 	c->f[1] = FIELD_GET(GENMASK(16, 15), val);
 	c->f[2] = FIELD_GET(GENMASK(8, 7), val);
-	c->r[0] = sign_extend32(FIELD_GET(GENMASK(30, 25), val), 5);
-	c->r[1] = sign_extend32(FIELD_GET(GENMASK(22, 17), val), 5);
-	c->r[2] = sign_extend32(FIELD_GET(GENMASK(14, 7), val), 5);
+	c->r[0] = FIELD_GET_SIGNED(GENMASK(30, 25), val);
+	c->r[1] = FIELD_GET_SIGNED(GENMASK(22, 17), val);
+	c->r[2] = FIELD_GET_SIGNED(GENMASK(14, 7), val);
 
 	return 0;
 }
@@ -1315,7 +1317,7 @@ static int yas537_power_on(struct yas5xx *yas5xx)
 		return ret;
 
 	/* Wait until the coil has ramped up */
-	usleep_range(YAS537_MAG_RCOIL_TIME_US, YAS537_MAG_RCOIL_TIME_US + 100);
+	fsleep(YAS537_MAG_RCOIL_TIME_US);
 
 	return 0;
 }
@@ -1323,6 +1325,7 @@ static int yas537_power_on(struct yas5xx *yas5xx)
 static const struct yas5xx_chip_info yas5xx_chip_info_tbl[] = {
 	[yas530] = {
 		.devid = YAS530_DEVICE_ID,
+		.product_label = "yas530",
 		.product_name = "YAS530 MS-3E",
 		.version_names = { "A", "B" },
 		.volatile_reg = yas530_volatile_reg,
@@ -1338,6 +1341,7 @@ static const struct yas5xx_chip_info yas5xx_chip_info_tbl[] = {
 	},
 	[yas532] = {
 		.devid = YAS532_DEVICE_ID,
+		.product_label = "yas532",
 		.product_name = "YAS532 MS-3R",
 		.version_names = { "AB", "AC" },
 		.volatile_reg = yas530_volatile_reg,
@@ -1353,6 +1357,7 @@ static const struct yas5xx_chip_info yas5xx_chip_info_tbl[] = {
 	},
 	[yas533] = {
 		.devid = YAS532_DEVICE_ID,
+		.product_label = "yas533",
 		.product_name = "YAS533 MS-3F",
 		.version_names = { "AB", "AC" },
 		.volatile_reg = yas530_volatile_reg,
@@ -1368,6 +1373,7 @@ static const struct yas5xx_chip_info yas5xx_chip_info_tbl[] = {
 	},
 	[yas537] = {
 		.devid = YAS537_DEVICE_ID,
+		.product_label = "yas537",
 		.product_name = "YAS537 MS-3T",
 		.version_names = { "v0", "v1" }, /* version naming unknown */
 		.volatile_reg = yas537_volatile_reg,
@@ -1385,7 +1391,6 @@ static const struct yas5xx_chip_info yas5xx_chip_info_tbl[] = {
 
 static int yas5xx_probe(struct i2c_client *i2c)
 {
-	const struct i2c_device_id *id = i2c_client_get_device_id(i2c);
 	struct iio_dev *indio_dev;
 	struct device *dev = &i2c->dev;
 	struct yas5xx *yas5xx;
@@ -1400,7 +1405,10 @@ static int yas5xx_probe(struct i2c_client *i2c)
 	yas5xx = iio_priv(indio_dev);
 	i2c_set_clientdata(i2c, indio_dev);
 	yas5xx->dev = dev;
-	mutex_init(&yas5xx->lock);
+
+	ret = devm_mutex_init(dev, &yas5xx->lock);
+	if (ret)
+		return ret;
 
 	ret = iio_read_mount_matrix(dev, &yas5xx->orientation);
 	if (ret)
@@ -1418,7 +1426,7 @@ static int yas5xx_probe(struct i2c_client *i2c)
 		return dev_err_probe(dev, ret, "cannot enable regulators\n");
 
 	/* See comment in runtime resume callback */
-	usleep_range(31000, 40000);
+	fsleep(31 * USEC_PER_MSEC);
 
 	/* This will take the device out of reset if need be */
 	yas5xx->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
@@ -1443,7 +1451,7 @@ static int yas5xx_probe(struct i2c_client *i2c)
 	if (id_check != ci->devid) {
 		ret = dev_err_probe(dev, -ENODEV,
 				    "device ID %02x doesn't match %s\n",
-				    id_check, id->name);
+				    id_check, ci->product_label);
 		goto assert_reset;
 	}
 
@@ -1469,7 +1477,7 @@ static int yas5xx_probe(struct i2c_client *i2c)
 	indio_dev->info = &yas5xx_info;
 	indio_dev->available_scan_masks = yas5xx_scan_masks;
 	indio_dev->modes = INDIO_DIRECT_MODE;
-	indio_dev->name = id->name;
+	indio_dev->name = ci->product_label;
 	indio_dev->channels = yas5xx_channels;
 	indio_dev->num_channels = ARRAY_SIZE(yas5xx_channels);
 
@@ -1557,7 +1565,7 @@ static int yas5xx_runtime_resume(struct device *dev)
 	 * for all voltages to settle. The YAS532 is 10ms then 4ms for the
 	 * I2C to come online. Let's keep it safe and put this at 31ms.
 	 */
-	usleep_range(31000, 40000);
+	fsleep(31 * USEC_PER_MSEC);
 	gpiod_set_value_cansleep(yas5xx->reset, 0);
 
 	ret = ci->power_on(yas5xx);
diff --git a/drivers/iio/orientation/hid-sensor-rotation.c b/drivers/iio/orientation/hid-sensor-rotation.c
index 5a5e6e4fbe34b..4a11e45550991 100644
--- a/drivers/iio/orientation/hid-sensor-rotation.c
+++ b/drivers/iio/orientation/hid-sensor-rotation.c
@@ -39,6 +39,27 @@ static const u32 rotation_sensitivity_addresses[] = {
 	HID_USAGE_SENSOR_ORIENT_QUATERNION,
 };
 
+enum {
+	DEV_ROT_SCAN_TYPE_16BIT,
+	DEV_ROT_SCAN_TYPE_32BIT,
+};
+
+static const struct iio_scan_type dev_rot_scan_types[] = {
+	[DEV_ROT_SCAN_TYPE_16BIT] = {
+		.sign = 's',
+		.realbits = 16,
+		/* Storage bits has to stay 32 to not break userspace. */
+		.storagebits = 32,
+		.repeat = 4,
+	},
+	[DEV_ROT_SCAN_TYPE_32BIT] = {
+		.sign = 's',
+		.realbits = 32,
+		.storagebits = 32,
+		.repeat = 4,
+	},
+};
+
 /* Channel definitions */
 static const struct iio_chan_spec dev_rot_channels[] = {
 	{
@@ -50,23 +71,14 @@ static const struct iio_chan_spec dev_rot_channels[] = {
 					BIT(IIO_CHAN_INFO_OFFSET) |
 					BIT(IIO_CHAN_INFO_SCALE) |
 					BIT(IIO_CHAN_INFO_HYSTERESIS),
-		.scan_index = 0
+		.scan_index = 0,
+		.has_ext_scan_type = 1,
+		.ext_scan_type = dev_rot_scan_types,
+		.num_ext_scan_type = ARRAY_SIZE(dev_rot_scan_types),
 	},
 	IIO_CHAN_SOFT_TIMESTAMP(1)
 };
 
-/* Adjust channel real bits based on report descriptor */
-static void dev_rot_adjust_channel_bit_mask(struct iio_chan_spec *chan,
-						int size)
-{
-	chan->scan_type.sign = 's';
-	/* Real storage bits will change based on the report desc. */
-	chan->scan_type.realbits = size * 8;
-	/* Maximum size of a sample to capture is u32 */
-	chan->scan_type.storagebits = sizeof(u32) * 8;
-	chan->scan_type.repeat = 4;
-}
-
 /* Channel read_raw handler */
 static int dev_rot_read_raw(struct iio_dev *indio_dev,
 				struct iio_chan_spec const *chan,
@@ -141,9 +153,25 @@ static int dev_rot_write_raw(struct iio_dev *indio_dev,
 	return ret;
 }
 
+static int dev_rot_get_current_scan_type(const struct iio_dev *indio_dev,
+					 const struct iio_chan_spec *chan)
+{
+	struct dev_rot_state *rot_state = iio_priv(indio_dev);
+
+	switch (rot_state->quaternion.size / 4) {
+	case sizeof(s16):
+		return DEV_ROT_SCAN_TYPE_16BIT;
+	case sizeof(s32):
+		return DEV_ROT_SCAN_TYPE_32BIT;
+	default:
+		return -EINVAL;
+	}
+}
+
 static const struct iio_info dev_rot_info = {
 	.read_raw_multi = &dev_rot_read_raw,
 	.write_raw = &dev_rot_write_raw,
+	.get_current_scan_type = &dev_rot_get_current_scan_type,
 };
 
 /* Callback handler to send event after all samples are received and captured */
@@ -212,7 +240,6 @@ static int dev_rot_capture_sample(struct hid_sensor_hub_device *hsdev,
 /* Parse report which is specific to an usage id*/
 static int dev_rot_parse_report(struct platform_device *pdev,
 				struct hid_sensor_hub_device *hsdev,
-				struct iio_chan_spec *channels,
 				unsigned usage_id,
 				struct dev_rot_state *st)
 {
@@ -226,9 +253,6 @@ static int dev_rot_parse_report(struct platform_device *pdev,
 	if (ret)
 		return ret;
 
-	dev_rot_adjust_channel_bit_mask(&channels[0],
-		st->quaternion.size / 4);
-
 	dev_dbg(&pdev->dev, "dev_rot %x:%x\n", st->quaternion.index,
 		st->quaternion.report_id);
 
@@ -287,22 +311,13 @@ static int hid_dev_rot_probe(struct platform_device *pdev)
 		return ret;
 	}
 
-	indio_dev->channels = devm_kmemdup(&pdev->dev, dev_rot_channels,
-					   sizeof(dev_rot_channels),
-					   GFP_KERNEL);
-	if (!indio_dev->channels) {
-		dev_err(&pdev->dev, "failed to duplicate channels\n");
-		return -ENOMEM;
-	}
-
-	ret = dev_rot_parse_report(pdev, hsdev,
-				   (struct iio_chan_spec *)indio_dev->channels,
-					hsdev->usage, rot_state);
+	ret = dev_rot_parse_report(pdev, hsdev, hsdev->usage, rot_state);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to setup attributes\n");
 		return ret;
 	}
 
+	indio_dev->channels = dev_rot_channels;
 	indio_dev->num_channels = ARRAY_SIZE(dev_rot_channels);
 	indio_dev->info = &dev_rot_info;
 	indio_dev->name = name;
diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c
index d983ce9c0b99f..990340a9b10c5 100644
--- a/drivers/iio/pressure/bmp280-core.c
+++ b/drivers/iio/pressure/bmp280-core.c
@@ -392,7 +392,7 @@ static int bme280_read_calib(struct bmp280_data *data)
 	h4_lower = FIELD_GET(BME280_COMP_H4_MASK_LOW, tmp_1);
 	calib->H4 = sign_extend32(h4_upper | h4_lower, 11);
 	tmp_3 = get_unaligned_le16(&data->bme280_humid_cal_buf[H5]);
-	calib->H5 = sign_extend32(FIELD_GET(BME280_COMP_H5_MASK, tmp_3), 11);
+	calib->H5 = FIELD_GET_SIGNED(BME280_COMP_H5_MASK, tmp_3);
 	calib->H6 = data->bme280_humid_cal_buf[H6];
 
 	return 0;
@@ -1910,7 +1910,7 @@ static irqreturn_t bmp380_trigger_handler(int irq, void *p)
 		u32 comp_press;
 		s32 comp_temp;
 		aligned_s64 timestamp;
-	} buffer;
+	} buffer = { };
 	int ret;
 
 	guard(mutex)(&data->lock);
@@ -2616,7 +2616,7 @@ static irqreturn_t bmp580_trigger_handler(int irq, void *p)
 		__le32 comp_temp;
 		__le32 comp_press;
 		aligned_s64 timestamp;
-	} buffer;
+	} buffer = { };
 	int ret;
 
 	guard(mutex)(&data->lock);
diff --git a/drivers/iio/proximity/srf08.c b/drivers/iio/proximity/srf08.c
index d7e4cc48cfbf7..92a37ba331f6a 100644
--- a/drivers/iio/proximity/srf08.c
+++ b/drivers/iio/proximity/srf08.c
@@ -226,7 +226,7 @@ static int srf08_read_raw(struct iio_dev *indio_dev,
 static ssize_t srf08_show_range_mm_available(struct device *dev,
 				struct device_attribute *attr, char *buf)
 {
-	return sprintf(buf, "[0.043 0.043 11.008]\n");
+	return sysfs_emit(buf, "[0.043 0.043 11.008]\n");
 }
 
 static IIO_DEVICE_ATTR(sensor_max_range_available, S_IRUGO,
@@ -238,8 +238,8 @@ static ssize_t srf08_show_range_mm(struct device *dev,
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct srf08_data *data = iio_priv(indio_dev);
 
-	return sprintf(buf, "%d.%03d\n", data->range_mm / 1000,
-						data->range_mm % 1000);
+	return sysfs_emit(buf, "%d.%03d\n",
+			  data->range_mm / 1000, data->range_mm % 1000);
 }
 
 /*
@@ -316,10 +316,10 @@ static ssize_t srf08_show_sensitivity_available(struct device *dev,
 
 	for (i = 0; i < data->chip_info->num_sensitivity_avail; i++)
 		if (data->chip_info->sensitivity_avail[i])
-			len += sprintf(buf + len, "%d ",
-				data->chip_info->sensitivity_avail[i]);
+			len += sysfs_emit_at(buf, len, "%d ",
+					     data->chip_info->sensitivity_avail[i]);
 
-	len += sprintf(buf + len, "\n");
+	len += sysfs_emit_at(buf, len, "\n");
 
 	return len;
 }
@@ -332,11 +332,8 @@ static ssize_t srf08_show_sensitivity(struct device *dev,
 {
 	struct iio_dev *indio_dev = dev_to_iio_dev(dev);
 	struct srf08_data *data = iio_priv(indio_dev);
-	int len;
 
-	len = sprintf(buf, "%d\n", data->sensitivity);
-
-	return len;
+	return sysfs_emit(buf, "%d\n", data->sensitivity);
 }
 
 static ssize_t srf08_write_sensitivity(struct srf08_data *data,
diff --git a/drivers/iio/proximity/vl53l0x-i2c.c b/drivers/iio/proximity/vl53l0x-i2c.c
index ad3e46d47fa8d..6c6e6dab045f7 100644
--- a/drivers/iio/proximity/vl53l0x-i2c.c
+++ b/drivers/iio/proximity/vl53l0x-i2c.c
@@ -87,15 +87,14 @@ static irqreturn_t vl53l0x_trigger_handler(int irq, void *priv)
 	ret = i2c_smbus_read_i2c_block_data(data->client,
 					VL_REG_RESULT_RANGE_STATUS,
 					sizeof(buffer), buffer);
-	if (ret < 0)
-		return ret;
-	else if (ret != 12)
-		return -EREMOTEIO;
+	if (ret != 12)
+		goto done;
 
 	scan.chan = get_unaligned_be16(&buffer[10]);
 	iio_push_to_buffers_with_ts(indio_dev, &scan, sizeof(scan),
 				    iio_get_time_ns(indio_dev));
 
+done:
 	iio_trigger_notify_done(indio_dev->trig);
 	vl53l0x_clear_irq(data);
 
diff --git a/drivers/iio/resolver/ad2s1210.c b/drivers/iio/resolver/ad2s1210.c
index 774728c804c01..1be19fe8aa3f6 100644
--- a/drivers/iio/resolver/ad2s1210.c
+++ b/drivers/iio/resolver/ad2s1210.c
@@ -1334,7 +1334,7 @@ static irqreturn_t ad2s1210_trigger_handler(int irq, void *p)
 
 		ret = regmap_read(st->regmap, AD2S1210_REG_FAULT, &reg_val);
 		if (ret < 0)
-			return ret;
+			goto error_ret;
 
 		st->sample.fault = reg_val;
 	}
diff --git a/drivers/iio/temperature/maxim_thermocouple.c b/drivers/iio/temperature/maxim_thermocouple.c
index 205939680fd4f..e898f56d11965 100644
--- a/drivers/iio/temperature/maxim_thermocouple.c
+++ b/drivers/iio/temperature/maxim_thermocouple.c
@@ -277,8 +277,8 @@ static const struct spi_device_id maxim_thermocouple_id[] = {
 MODULE_DEVICE_TABLE(spi, maxim_thermocouple_id);
 
 static const struct of_device_id maxim_thermocouple_of_match[] = {
-        { .compatible = "maxim,max6675" },
-        { .compatible = "maxim,max31855" },
+	{ .compatible = "maxim,max6675" },
+	{ .compatible = "maxim,max31855" },
 	{ .compatible = "maxim,max31855k" },
 	{ .compatible = "maxim,max31855j" },
 	{ .compatible = "maxim,max31855n" },
@@ -286,7 +286,7 @@ static const struct of_device_id maxim_thermocouple_of_match[] = {
 	{ .compatible = "maxim,max31855t" },
 	{ .compatible = "maxim,max31855e" },
 	{ .compatible = "maxim,max31855r" },
-        { },
+	{ },
 };
 MODULE_DEVICE_TABLE(of, maxim_thermocouple_of_match);
 
diff --git a/drivers/iio/temperature/mcp9600.c b/drivers/iio/temperature/mcp9600.c
index aa42c2b1a369e..69baf654c9c0a 100644
--- a/drivers/iio/temperature/mcp9600.c
+++ b/drivers/iio/temperature/mcp9600.c
@@ -297,7 +297,7 @@ static int mcp9600_read_thresh(struct iio_dev *indio_dev,
 		 * Temperature is stored in two’s complement format in
 		 * bits(15:2), LSB is 0.25 degree celsius.
 		 */
-		*val = sign_extend32(FIELD_GET(MCP9600_ALERT_LIMIT_MASK, ret), 13);
+		*val = FIELD_GET_SIGNED(MCP9600_ALERT_LIMIT_MASK, ret);
 		*val2 = 4;
 		return IIO_VAL_FRACTIONAL;
 	case IIO_EV_INFO_HYSTERESIS:
diff --git a/drivers/iio/temperature/tmp006.c b/drivers/iio/temperature/tmp006.c
index d8d8c8936d177..bf62143fa7191 100644
--- a/drivers/iio/temperature/tmp006.c
+++ b/drivers/iio/temperature/tmp006.c
@@ -350,7 +350,7 @@ static int tmp006_probe(struct i2c_client *client)
 
 		data->drdy_trig->ops = &tmp006_trigger_ops;
 		iio_trigger_set_drvdata(data->drdy_trig, indio_dev);
-		ret = iio_trigger_register(data->drdy_trig);
+		ret = devm_iio_trigger_register(&client->dev, data->drdy_trig);
 		if (ret)
 			return ret;
 
diff --git a/drivers/iio/temperature/tsys01.c b/drivers/iio/temperature/tsys01.c
index 334bba6fdae6c..104dd45598b0c 100644
--- a/drivers/iio/temperature/tsys01.c
+++ b/drivers/iio/temperature/tsys01.c
@@ -119,7 +119,7 @@ static bool tsys01_crc_valid(u16 *n_prom)
 	u8 sum = 0;
 
 	for (cnt = 0; cnt < TSYS01_PROM_WORDS_NB; cnt++)
-		sum += ((n_prom[0] >> 8) + (n_prom[0] & 0xFF));
+		sum += ((n_prom[cnt] >> 8) + (n_prom[cnt] & 0xFF));
 
 	return (sum == 0);
 }
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 27992c38ad902..7e62b5b1ffaa3 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -850,7 +850,7 @@ static struct notifier_block nb = {
 
 int addr_init(void)
 {
-	addr_wq = alloc_ordered_workqueue("ib_addr", 0);
+	addr_wq = alloc_workqueue("ib_addr", WQ_UNBOUND, 0);
 	if (!addr_wq)
 		return -ENOMEM;
 
diff --git a/drivers/infiniband/core/cma_configfs.c b/drivers/infiniband/core/cma_configfs.c
index 819927ce4f0ec..891e52afb8f4b 100644
--- a/drivers/infiniband/core/cma_configfs.c
+++ b/drivers/infiniband/core/cma_configfs.c
@@ -255,7 +255,7 @@ static void release_cma_ports_group(struct config_item  *item)
 	cma_dev_group->ports = NULL;
 };
 
-static struct configfs_item_operations cma_ports_item_ops = {
+static const struct configfs_item_operations cma_ports_item_ops = {
 	.release = release_cma_ports_group
 };
 
@@ -264,7 +264,7 @@ static const struct config_item_type cma_ports_group_type = {
 	.ct_owner	= THIS_MODULE
 };
 
-static struct configfs_item_operations cma_device_item_ops = {
+static const struct configfs_item_operations cma_device_item_ops = {
 	.release = release_cma_dev
 };
 
@@ -327,7 +327,7 @@ static void drop_cma_dev(struct config_group *cgroup, struct config_item *item)
 	config_item_put(item);
 }
 
-static struct configfs_group_operations cma_subsys_group_ops = {
+static const struct configfs_group_operations cma_subsys_group_ops = {
 	.make_group	= make_cma_dev,
 	.drop_item	= drop_cma_dev,
 };
diff --git a/drivers/infiniband/core/ib_core_uverbs.c b/drivers/infiniband/core/ib_core_uverbs.c
index 1f7a5c119cc9c..8a0e6fa2a5283 100644
--- a/drivers/infiniband/core/ib_core_uverbs.c
+++ b/drivers/infiniband/core/ib_core_uverbs.c
@@ -9,6 +9,7 @@
 #include <linux/dma-resv.h>
 #include "uverbs.h"
 #include "core_priv.h"
+#include "rdma_core.h"
 
 MODULE_IMPORT_NS("DMA_BUF");
 
@@ -416,3 +417,89 @@ struct ib_device *rdma_udata_to_dev(struct ib_udata *udata)
 }
 EXPORT_SYMBOL(rdma_udata_to_dev);
 
+#if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)
+uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata)
+{
+	struct uverbs_attr_bundle *bundle =
+		rdma_udata_to_uverbs_attr_bundle(udata);
+
+	lockdep_assert_held(&bundle->ufile->device->disassociate_srcu);
+
+	return srcu_dereference(bundle->method_elm->handler,
+				&bundle->ufile->device->disassociate_srcu);
+}
+
+int _ib_copy_validate_udata_in(struct ib_udata *udata, void *req,
+			       size_t kernel_size, size_t minimum_size)
+{
+	int err;
+
+	if (udata->inlen < minimum_size) {
+		ibdev_dbg(
+			rdma_udata_to_dev(udata),
+			"System call driver input udata too small (%zu < %zu) for ioctl %ps called by %pSR\n",
+			udata->inlen, minimum_size,
+			uverbs_get_handler_fn(udata),
+			__builtin_return_address(0));
+		return -EINVAL;
+	}
+
+	err = copy_struct_from_user(req, kernel_size, udata->inbuf,
+				    udata->inlen);
+	if (err) {
+		if (err == -E2BIG) {
+			ibdev_dbg(
+				rdma_udata_to_dev(udata),
+				"System call driver input udata not zero from %zu -> %zu for ioctl %ps called by %pSR\n",
+				minimum_size, udata->inlen,
+				uverbs_get_handler_fn(udata),
+				__builtin_return_address(0));
+			return -EOPNOTSUPP;
+		}
+		ibdev_dbg(
+			rdma_udata_to_dev(udata),
+			"System call driver input udata EFAULT for ioctl %ps called by %pSR\n",
+			uverbs_get_handler_fn(udata),
+			__builtin_return_address(0));
+		return err;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(_ib_copy_validate_udata_in);
+
+int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm,
+				    u64 valid_cm)
+{
+	ibdev_dbg(
+		rdma_udata_to_dev(udata),
+		"System call driver input udata has unsupported comp_mask %llx & ~%llx = %llx for ioctl %ps called by %pSR\n",
+		req_cm, valid_cm, req_cm & ~valid_cm,
+		uverbs_get_handler_fn(udata), __builtin_return_address(0));
+	return -EOPNOTSUPP;
+}
+EXPORT_SYMBOL(_ib_copy_validate_udata_cm_fail);
+
+int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len)
+{
+	size_t copy_len;
+
+	/* 0 length copy_len is a NOP for copy_to_user() and doesn't fail. */
+	copy_len = min(len, udata->outlen);
+	if (copy_to_user(udata->outbuf, src, copy_len))
+		goto err_fault;
+	if (copy_len < udata->outlen) {
+		if (clear_user(udata->outbuf + copy_len,
+			       udata->outlen - copy_len))
+			goto err_fault;
+	}
+	return 0;
+err_fault:
+	ibdev_dbg(
+		rdma_udata_to_dev(udata),
+		"System call driver out udata has EFAULT (%zu into %zu) for ioctl %ps called by %pSR\n",
+		len, udata->outlen, uverbs_get_handler_fn(udata),
+		__builtin_return_address(0));
+	return -EFAULT;
+}
+EXPORT_SYMBOL(_ib_respond_udata);
+#endif
diff --git a/drivers/infiniband/core/nldev.c b/drivers/infiniband/core/nldev.c
index 96c745d5bac45..5aaba2b9746ba 100644
--- a/drivers/infiniband/core/nldev.c
+++ b/drivers/infiniband/core/nldev.c
@@ -51,6 +51,7 @@
  * a controlled QKEY.
  */
 static bool privileged_qkey;
+static DEFINE_MUTEX(nldev_dellink_mutex);
 
 typedef int (*res_fill_func_t)(struct sk_buff*, bool,
 			       struct rdma_restrack_entry*, uint32_t);
@@ -1846,7 +1847,9 @@ static int nldev_dellink(struct sk_buff *skb, struct nlmsghdr *nlh,
 	 * implicitly scoped to the driver supporting dynamic link deletion like RXE.
 	 */
 	if (device->link_ops && device->link_ops->dellink) {
+		mutex_lock(&nldev_dellink_mutex);
 		err = device->link_ops->dellink(device);
+		mutex_unlock(&nldev_dellink_mutex);
 		if (err)
 			return err;
 	}
diff --git a/drivers/infiniband/core/uverbs.h b/drivers/infiniband/core/uverbs.h
index 6d4295277e0e5..f2e192b51e609 100644
--- a/drivers/infiniband/core/uverbs.h
+++ b/drivers/infiniband/core/uverbs.h
@@ -229,6 +229,40 @@ int uverbs_dealloc_mw(struct ib_mw *mw);
 void ib_uverbs_detach_umcast(struct ib_qp *qp,
 			     struct ib_uqp_object *uobj);
 
+struct bundle_alloc_head {
+	struct_group_tagged(bundle_alloc_head_hdr, hdr,
+		struct bundle_alloc_head *next;
+	);
+	u8 data[];
+};
+
+struct bundle_priv {
+	/* Must be first */
+	struct bundle_alloc_head_hdr alloc_head;
+	struct bundle_alloc_head *allocated_mem;
+	size_t internal_avail;
+	size_t internal_used;
+
+	struct radix_tree_root *radix;
+	void __rcu **radix_slots;
+	unsigned long radix_slots_len;
+	u32 method_key;
+
+	struct ib_uverbs_attr __user *user_attrs;
+	struct ib_uverbs_attr *uattrs;
+
+	DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
+	DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
+	DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
+
+	/*
+	 * Must be last. bundle ends in a flex array which overlaps
+	 * internal_buffer.
+	 */
+	struct uverbs_attr_bundle_hdr bundle;
+	u64 internal_buffer[32];
+};
+
 long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 
 struct ib_uverbs_flow_spec {
diff --git a/drivers/infiniband/core/uverbs_ioctl.c b/drivers/infiniband/core/uverbs_ioctl.c
index b61af625e679b..2552a7efe2fbe 100644
--- a/drivers/infiniband/core/uverbs_ioctl.c
+++ b/drivers/infiniband/core/uverbs_ioctl.c
@@ -35,54 +35,6 @@
 #include "rdma_core.h"
 #include "uverbs.h"
 
-struct bundle_alloc_head {
-	struct_group_tagged(bundle_alloc_head_hdr, hdr,
-		struct bundle_alloc_head *next;
-	);
-	u8 data[];
-};
-
-struct bundle_priv {
-	/* Must be first */
-	struct bundle_alloc_head_hdr alloc_head;
-	struct bundle_alloc_head *allocated_mem;
-	size_t internal_avail;
-	size_t internal_used;
-
-	struct radix_tree_root *radix;
-	const struct uverbs_api_ioctl_method *method_elm;
-	void __rcu **radix_slots;
-	unsigned long radix_slots_len;
-	u32 method_key;
-
-	struct ib_uverbs_attr __user *user_attrs;
-	struct ib_uverbs_attr *uattrs;
-
-	DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
-	DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
-	DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
-
-	/*
-	 * Must be last. bundle ends in a flex array which overlaps
-	 * internal_buffer.
-	 */
-	struct uverbs_attr_bundle_hdr bundle;
-	u64 internal_buffer[32];
-};
-
-uverbs_api_ioctl_handler_fn uverbs_get_handler_fn(struct ib_udata *udata)
-{
-	struct uverbs_attr_bundle *bundle =
-		rdma_udata_to_uverbs_attr_bundle(udata);
-	struct bundle_priv *pbundle =
-		container_of(&bundle->hdr, struct bundle_priv, bundle);
-
-	lockdep_assert_held(&bundle->ufile->device->disassociate_srcu);
-
-	return srcu_dereference(pbundle->method_elm->handler,
-				&bundle->ufile->device->disassociate_srcu);
-}
-
 /*
  * Each method has an absolute minimum amount of memory it needs to allocate,
  * precompute that amount and determine if the onstack memory can be used or
@@ -445,13 +397,13 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,
 	struct uverbs_attr_bundle *bundle =
 		container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
 	size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
-	unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
+	unsigned int destroy_bkey = bundle->method_elm->destroy_bkey;
 	unsigned int i;
 	int ret;
 
 	/* See uverbs_disassociate_api() */
 	handler = srcu_dereference(
-		pbundle->method_elm->handler,
+		bundle->method_elm->handler,
 		&pbundle->bundle.ufile->device->disassociate_srcu);
 	if (!handler)
 		return -EIO;
@@ -469,12 +421,12 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,
 	}
 
 	/* User space did not provide all the mandatory attributes */
-	if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
+	if (unlikely(!bitmap_subset(bundle->method_elm->attr_mandatory,
 				    pbundle->bundle.attr_present,
-				    pbundle->method_elm->key_bitmap_len)))
+				    bundle->method_elm->key_bitmap_len)))
 		return -EINVAL;
 
-	if (pbundle->method_elm->has_udata)
+	if (bundle->method_elm->has_udata)
 		uverbs_fill_udata(bundle, &pbundle->bundle.driver_udata,
 				  UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
 	else
@@ -499,7 +451,7 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,
 	 * assume that the driver wrote to its UHW_OUT and flag userspace
 	 * appropriately.
 	 */
-	if (!ret && pbundle->method_elm->has_udata) {
+	if (!ret && bundle->method_elm->has_udata) {
 		const struct uverbs_attr *attr =
 			uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT);
 
@@ -520,7 +472,7 @@ static int ib_uverbs_run_method(struct bundle_priv *pbundle,
 
 static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
 {
-	unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
+	unsigned int key_bitmap_len = pbundle->bundle.method_elm->key_bitmap_len;
 	struct uverbs_attr_bundle *bundle =
 		container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr);
 	struct bundle_alloc_head *memblock;
@@ -608,7 +560,7 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
 	}
 
 	/* Space for the pbundle->bundle.attrs flex array */
-	pbundle->method_elm = method_elm;
+	pbundle->bundle.method_elm = method_elm;
 	pbundle->method_key = attrs_iter.index;
 	pbundle->bundle.ufile = ufile;
 	pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
@@ -617,10 +569,12 @@ static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
 	pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
 	pbundle->user_attrs = user_attrs;
 
-	pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
-					       sizeof(*container_of(&pbundle->bundle,
-							struct uverbs_attr_bundle, hdr)->attrs),
-					       sizeof(*pbundle->internal_buffer));
+	pbundle->internal_used = ALIGN(
+		pbundle->bundle.method_elm->key_bitmap_len *
+			sizeof(*container_of(&pbundle->bundle,
+					     struct uverbs_attr_bundle, hdr)
+					->attrs),
+		sizeof(*pbundle->internal_buffer));
 	memset(pbundle->bundle.attr_present, 0,
 	       sizeof(pbundle->bundle.attr_present));
 	memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
@@ -860,77 +814,3 @@ void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
 		  pbundle->uobj_hw_obj_valid);
 }
 EXPORT_SYMBOL(uverbs_finalize_uobj_create);
-
-int _ib_copy_validate_udata_in(struct ib_udata *udata, void *req,
-			       size_t kernel_size, size_t minimum_size)
-{
-	int err;
-
-	if (udata->inlen < minimum_size) {
-		ibdev_dbg(
-			rdma_udata_to_dev(udata),
-			"System call driver input udata too small (%zu < %zu) for ioctl %ps called by %pSR\n",
-			udata->inlen, minimum_size,
-			uverbs_get_handler_fn(udata),
-			__builtin_return_address(0));
-		return -EINVAL;
-	}
-
-	err = copy_struct_from_user(req, kernel_size, udata->inbuf,
-				    udata->inlen);
-	if (err) {
-		if (err == -E2BIG) {
-			ibdev_dbg(
-				rdma_udata_to_dev(udata),
-				"System call driver input udata not zero from %zu -> %zu for ioctl %ps called by %pSR\n",
-				minimum_size, udata->inlen,
-				uverbs_get_handler_fn(udata),
-				__builtin_return_address(0));
-			return -EOPNOTSUPP;
-		}
-		ibdev_dbg(
-			rdma_udata_to_dev(udata),
-			"System call driver input udata EFAULT for ioctl %ps called by %pSR\n",
-			uverbs_get_handler_fn(udata),
-			__builtin_return_address(0));
-		return err;
-	}
-	return 0;
-}
-EXPORT_SYMBOL(_ib_copy_validate_udata_in);
-
-int _ib_copy_validate_udata_cm_fail(struct ib_udata *udata, u64 req_cm,
-				    u64 valid_cm)
-{
-	ibdev_dbg(
-		rdma_udata_to_dev(udata),
-		"System call driver input udata has unsupported comp_mask %llx & ~%llx = %llx for ioctl %ps called by %pSR\n",
-		req_cm, valid_cm, req_cm & ~valid_cm,
-		uverbs_get_handler_fn(udata), __builtin_return_address(0));
-	return -EOPNOTSUPP;
-}
-EXPORT_SYMBOL(_ib_copy_validate_udata_cm_fail);
-
-int _ib_respond_udata(struct ib_udata *udata, const void *src, size_t len)
-{
-	size_t copy_len;
-
-	/* 0 length copy_len is a NOP for copy_to_user() and doesn't fail. */
-	copy_len = min(len, udata->outlen);
-	if (copy_to_user(udata->outbuf, src, copy_len))
-		goto err_fault;
-	if (copy_len < udata->outlen) {
-		if (clear_user(udata->outbuf + copy_len,
-			       udata->outlen - copy_len))
-			goto err_fault;
-	}
-	return 0;
-err_fault:
-	ibdev_dbg(
-		rdma_udata_to_dev(udata),
-		"System call driver out udata has EFAULT (%zu into %zu) for ioctl %ps called by %pSR\n",
-		len, udata->outlen, uverbs_get_handler_fn(udata),
-		__builtin_return_address(0));
-	return -EFAULT;
-}
-EXPORT_SYMBOL(_ib_respond_udata);
diff --git a/drivers/infiniband/hw/bng_re/bng_fw.c b/drivers/infiniband/hw/bng_re/bng_fw.c
index 17d7cc3aa11d5..50156c300b335 100644
--- a/drivers/infiniband/hw/bng_re/bng_fw.c
+++ b/drivers/infiniband/hw/bng_re/bng_fw.c
@@ -123,7 +123,6 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw,
 	bool is_waiter_alive;
 	struct pci_dev *pdev;
 	u32 wait_cmds = 0;
-	int rc = 0;
 
 	pdev = rcfw->pdev;
 	switch (qp_event->event) {
@@ -152,7 +151,7 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw,
 				 "rcfw timedout: cookie = %#x, free_slots = %d",
 				 cookie, crsqe->free_slots);
 			spin_unlock(&hwq->lock);
-			return rc;
+			return 0;
 		}
 
 		if (crsqe->is_waiter_alive) {
@@ -182,7 +181,7 @@ static int bng_re_process_qp_event(struct bng_re_rcfw *rcfw,
 		spin_unlock(&hwq->lock);
 	}
 	*num_wait += wait_cmds;
-	return rc;
+	return 0;
 }
 
 /* function events */
diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
index 7ed294516b7ed..f325d977b1147 100644
--- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
+++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
@@ -1884,7 +1884,7 @@ int bnxt_re_create_qp(struct ib_qp *ib_qp, struct ib_qp_init_attr *qp_init_attr,
 		}
 
 		if (udata) {
-			struct bnxt_re_qp_resp resp;
+			struct bnxt_re_qp_resp resp = {};
 
 			resp.qpid = qp->qplib_qp.id;
 			resp.rsvd = 0;
@@ -4638,7 +4638,7 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata)
 
 	uctx->rdev = rdev;
 
-	uctx->shpg = (void *)__get_free_page(GFP_KERNEL);
+	uctx->shpg = (void *)get_zeroed_page(GFP_KERNEL);
 	if (!uctx->shpg) {
 		rc = -ENOMEM;
 		goto fail;
diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c
index e31fb9134aa81..d1517f2560b98 100644
--- a/drivers/infiniband/hw/cxgb4/cq.c
+++ b/drivers/infiniband/hw/cxgb4/cq.c
@@ -1004,7 +1004,7 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	struct c4iw_dev *rhp = to_c4iw_dev(ibcq->device);
 	struct c4iw_cq *chp = to_c4iw_cq(ibcq);
 	struct c4iw_create_cq ucmd;
-	struct c4iw_create_cq_resp uresp;
+	struct c4iw_create_cq_resp uresp = {};
 	int ret, wr_len;
 	size_t memsize, hwentries;
 	struct c4iw_mm_entry *mm, *mm2;
@@ -1102,7 +1102,6 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		if (!mm2)
 			goto err_free_mm;
 
-		memset(&uresp, 0, sizeof(uresp));
 		uresp.qid_mask = rhp->rdev.cqmask;
 		uresp.cqid = chp->cq.cqid;
 		uresp.size = chp->cq.size;
@@ -1115,13 +1114,11 @@ int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		/* communicate to the userspace that
 		 * kernel driver supports 64B CQE
 		 */
-		uresp.flags |= C4IW_64B_CQE;
+		if (!ucontext->is_32b_cqe)
+			uresp.flags |= C4IW_64B_CQE;
 
 		spin_unlock(&ucontext->mmap_lock);
-		ret = ib_copy_to_udata(udata, &uresp,
-				       ucontext->is_32b_cqe ?
-				       sizeof(uresp) - sizeof(uresp.flags) :
-				       sizeof(uresp));
+		ret = ib_respond_udata(udata, uresp);
 		if (ret)
 			goto err_free_mm2;
 
diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c
index 616019ac1da50..0e3827022c63d 100644
--- a/drivers/infiniband/hw/cxgb4/provider.c
+++ b/drivers/infiniband/hw/cxgb4/provider.c
@@ -52,6 +52,7 @@
 #include <rdma/ib_smi.h>
 #include <rdma/ib_umem.h>
 #include <rdma/ib_user_verbs.h>
+#include <rdma/uverbs_ioctl.h>
 
 #include "iw_cxgb4.h"
 
@@ -79,7 +80,7 @@ static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext,
 	struct ib_device *ibdev = ucontext->device;
 	struct c4iw_ucontext *context = to_c4iw_ucontext(ucontext);
 	struct c4iw_dev *rhp = to_c4iw_dev(ibdev);
-	struct c4iw_alloc_ucontext_resp uresp;
+	struct c4iw_alloc_ucontext_resp uresp = {};
 	int ret = 0;
 	struct c4iw_mm_entry *mm = NULL;
 
@@ -105,8 +106,7 @@ static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext,
 		context->key += PAGE_SIZE;
 		spin_unlock(&context->mmap_lock);
 
-		ret = ib_copy_to_udata(udata, &uresp,
-				       sizeof(uresp) - sizeof(uresp.reserved));
+		ret = ib_respond_udata(udata, uresp);
 		if (ret)
 			goto err_mm;
 
@@ -209,8 +209,9 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)
 {
 	struct c4iw_pd *php = to_c4iw_pd(pd);
 	struct ib_device *ibdev = pd->device;
-	u32 pdid;
 	struct c4iw_dev *rhp;
+	u32 pdid;
+	int ret;
 
 	pr_debug("ibdev %p\n", ibdev);
 	rhp = (struct c4iw_dev *) ibdev;
@@ -223,9 +224,10 @@ static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)
 	if (udata) {
 		struct c4iw_alloc_pd_resp uresp = {.pdid = php->pdid};
 
-		if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
+		ret = ib_respond_udata(udata, uresp);
+		if (ret) {
 			c4iw_deallocate_pd(&php->ibpd, udata);
-			return -EFAULT;
+			return ret;
 		}
 	}
 	mutex_lock(&rhp->rdev.stats.lock);
diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c
index d9a86e4c54618..e295f79e0cd3e 100644
--- a/drivers/infiniband/hw/cxgb4/qp.c
+++ b/drivers/infiniband/hw/cxgb4/qp.c
@@ -2120,7 +2120,7 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,
 	struct c4iw_pd *php;
 	struct c4iw_cq *schp;
 	struct c4iw_cq *rchp;
-	struct c4iw_create_qp_resp uresp;
+	struct c4iw_create_qp_resp uresp = {};
 	unsigned int sqsize, rqsize = 0;
 	struct c4iw_ucontext *ucontext = rdma_udata_to_drv_context(
 		udata, struct c4iw_ucontext, ibucontext);
@@ -2242,7 +2242,6 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,
 				goto err_free_sq_db_key;
 			}
 		}
-		memset(&uresp, 0, sizeof(uresp));
 		if (t4_sq_onchip(&qhp->wq.sq)) {
 			ma_sync_key_mm = kmalloc_obj(*ma_sync_key_mm);
 			if (!ma_sync_key_mm) {
@@ -2280,7 +2279,7 @@ int c4iw_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *attrs,
 			ucontext->key += PAGE_SIZE;
 		}
 		spin_unlock(&ucontext->mmap_lock);
-		ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+		ret = ib_respond_udata(udata, uresp);
 		if (ret)
 			goto err_free_ma_sync_key;
 		sq_key_mm->key = uresp.sq_key;
@@ -2686,7 +2685,7 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,
 	struct c4iw_dev *rhp;
 	struct c4iw_srq *srq = to_c4iw_srq(ib_srq);
 	struct c4iw_pd *php;
-	struct c4iw_create_srq_resp uresp;
+	struct c4iw_create_srq_resp uresp = {};
 	struct c4iw_ucontext *ucontext;
 	struct c4iw_mm_entry *srq_key_mm, *srq_db_key_mm;
 	int rqsize;
@@ -2764,7 +2763,6 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,
 			ret = -ENOMEM;
 			goto err_free_srq_key_mm;
 		}
-		memset(&uresp, 0, sizeof(uresp));
 		uresp.flags = srq->flags;
 		uresp.qid_mask = rhp->rdev.qpmask;
 		uresp.srqid = srq->wq.qid;
@@ -2777,7 +2775,7 @@ int c4iw_create_srq(struct ib_srq *ib_srq, struct ib_srq_init_attr *attrs,
 		uresp.srq_db_gts_key = ucontext->key;
 		ucontext->key += PAGE_SIZE;
 		spin_unlock(&ucontext->mmap_lock);
-		ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+		ret = ib_respond_udata(udata, uresp);
 		if (ret)
 			goto err_free_srq_db_key_mm;
 		srq_key_mm->key = uresp.srq_key;
diff --git a/drivers/infiniband/hw/efa/Kconfig b/drivers/infiniband/hw/efa/Kconfig
index 457e18ba1d570..ff7f7c0870b3a 100644
--- a/drivers/infiniband/hw/efa/Kconfig
+++ b/drivers/infiniband/hw/efa/Kconfig
@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
-# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
+# Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
 #
 # Amazon fabric device configuration
 #
@@ -8,6 +8,7 @@ config INFINIBAND_EFA
 	tristate "Amazon Elastic Fabric Adapter (EFA) support"
 	depends on PCI_MSI && 64BIT && !CPU_BIG_ENDIAN
 	depends on INFINIBAND_USER_ACCESS
+	select CRC16
 	help
 	  This driver supports Amazon Elastic Fabric Adapter (EFA).
 
diff --git a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
index ad34ea5da6b07..826790ca9d839 100644
--- a/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
@@ -6,9 +6,6 @@
 #ifndef _EFA_ADMIN_CMDS_H_
 #define _EFA_ADMIN_CMDS_H_
 
-#define EFA_ADMIN_API_VERSION_MAJOR          0
-#define EFA_ADMIN_API_VERSION_MINOR          1
-
 /* EFA admin queue opcodes */
 enum efa_admin_aq_opcode {
 	EFA_ADMIN_CREATE_QP                         = 1,
diff --git a/drivers/infiniband/hw/efa/efa_admin_defs.h b/drivers/infiniband/hw/efa/efa_admin_defs.h
index 35700c93e639e..02f86edabed86 100644
--- a/drivers/infiniband/hw/efa/efa_admin_defs.h
+++ b/drivers/infiniband/hw/efa/efa_admin_defs.h
@@ -1,18 +1,20 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_ADMIN_H_
 #define _EFA_ADMIN_H_
 
+#define EFA_ADMIN_API_VERSION_MAJOR          0
+#define EFA_ADMIN_API_VERSION_MINOR          2
+
 enum efa_admin_aq_completion_status {
 	EFA_ADMIN_SUCCESS                           = 0,
 	EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE       = 1,
 	EFA_ADMIN_BAD_OPCODE                        = 2,
 	EFA_ADMIN_UNSUPPORTED_OPCODE                = 3,
 	EFA_ADMIN_MALFORMED_REQUEST                 = 4,
-	/* Additional status is provided in ACQ entry extended_status */
 	EFA_ADMIN_ILLEGAL_PARAMETER                 = 5,
 	EFA_ADMIN_UNKNOWN_ERROR                     = 6,
 	EFA_ADMIN_RESOURCE_BUSY                     = 7,
@@ -78,13 +80,10 @@ struct efa_admin_acq_common_desc {
 	 */
 	u8 flags;
 
-	u16 extended_status;
+	/* Poly 0x8005 CRC16 with initial value 0xFFFF and final XOR of 0xFFFF */
+	u16 checksum;
 
-	/*
-	 * indicates to the driver which AQ entry has been consumed by the
-	 * device and could be reused
-	 */
-	u16 sq_head_indx;
+	u16 reserved;
 };
 
 struct efa_admin_acq_entry {
diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c
index e97b5f0d70038..7cc3f4af0bb9d 100644
--- a/drivers/infiniband/hw/efa/efa_com.c
+++ b/drivers/infiniband/hw/efa/efa_com.c
@@ -3,6 +3,7 @@
  * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
+#include <linux/crc16.h>
 #include <linux/log2.h>
 
 #include "efa_com.h"
@@ -22,6 +23,14 @@
 #define EFA_CTRL_MINOR          0
 #define EFA_CTRL_SUB_MINOR      1
 
+#define EFA_CRC16_INIT_VAL 0xffff
+
+#define EFA_CRC_MIN_ADMIN_API_VERSION_MAJOR 0
+#define EFA_CRC_MIN_ADMIN_API_VERSION_MINOR 2
+
+#define EFA_MIN_ADMIN_API_VERSION_MAJOR 0
+#define EFA_MIN_ADMIN_API_VERSION_MINOR 1
+
 enum efa_cmd_status {
 	EFA_CMD_UNUSED,
 	EFA_CMD_ALLOCATED,
@@ -167,9 +176,8 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev)
 	struct efa_com_admin_queue *aq = &edev->aq;
 	struct efa_com_admin_cq *cq = &aq->cq;
 	u16 size = aq->depth * sizeof(*cq->entries);
-	u32 acq_caps = 0;
-	u32 addr_high;
-	u32 addr_low;
+	u32 acq_caps = 0, crc_min_ver = 0;
+	u32 addr_high, addr_low;
 
 	cq->entries =
 		dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL);
@@ -178,6 +186,11 @@ static int efa_com_admin_init_cq(struct efa_com_dev *edev)
 
 	spin_lock_init(&cq->lock);
 
+	EFA_SET(&crc_min_ver, EFA_REGS_VERSION_MAJOR_VERSION, EFA_CRC_MIN_ADMIN_API_VERSION_MAJOR);
+	EFA_SET(&crc_min_ver, EFA_REGS_VERSION_MINOR_VERSION, EFA_CRC_MIN_ADMIN_API_VERSION_MINOR);
+	if (edev->dev_api_ver >= crc_min_ver)
+		cq->validate_checksum = true;
+
 	cq->cc = 0;
 	cq->phase = 1;
 
@@ -409,12 +422,35 @@ static int efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
 	return 0;
 }
 
+static bool efa_com_cqe_checksum_valid(struct efa_com_admin_queue *aq,
+				       struct efa_admin_acq_entry *cqe)
+{
+	u16 cqe_checksum = cqe->acq_common_descriptor.checksum;
+	u16 calc_checksum;
+
+	cqe->acq_common_descriptor.checksum = 0;
+
+	calc_checksum = crc16(EFA_CRC16_INIT_VAL, (u8 *)cqe, sizeof(*cqe)) ^ EFA_CRC16_INIT_VAL;
+	if (calc_checksum != cqe_checksum) {
+		ibdev_err(aq->efa_dev,
+			  "Received completion with invalid checksum, cqe[%u], calc[%u], sq producer[%d], sq consumer[%d], cq consumer[%d]\n",
+			  cqe_checksum, calc_checksum, aq->sq.pc, aq->sq.cc,
+			  aq->cq.cc);
+		return false;
+	}
+
+	return true;
+}
+
 static int efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,
 						  struct efa_admin_acq_entry *cqe)
 {
 	struct efa_comp_ctx *comp_ctx;
 	u16 cmd_id;
 
+	if (aq->cq.validate_checksum && !efa_com_cqe_checksum_valid(aq, cqe))
+		return -EINVAL;
+
 	cmd_id = EFA_GET(&cqe->acq_common_descriptor.command,
 			 EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);
 
@@ -954,16 +990,16 @@ int efa_com_validate_version(struct efa_com_dev *edev)
 		  EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION),
 		  EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION));
 
-	EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION,
-		EFA_ADMIN_API_VERSION_MAJOR);
-	EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION,
-		EFA_ADMIN_API_VERSION_MINOR);
+	EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION, EFA_MIN_ADMIN_API_VERSION_MAJOR);
+	EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION, EFA_MIN_ADMIN_API_VERSION_MINOR);
 	if (ver < min_ver) {
 		ibdev_err(edev->efa_dev,
 			  "EFA version is lower than the minimal version the driver supports\n");
 		return -EOPNOTSUPP;
 	}
 
+	edev->dev_api_ver = ver;
+
 	ibdev_dbg(
 		edev->efa_dev,
 		"efa controller version: %d.%d.%d implementation version %d\n",
diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h
index 4d9ca97e42962..f8c692b0e092e 100644
--- a/drivers/infiniband/hw/efa/efa_com.h
+++ b/drivers/infiniband/hw/efa/efa_com.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2026 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_COM_H_
@@ -25,6 +25,7 @@ struct efa_com_admin_cq {
 	struct efa_admin_acq_entry *entries;
 	dma_addr_t dma_addr;
 	spinlock_t lock; /* Protects ACQ */
+	bool validate_checksum;
 
 	u16 cc; /* consumer counter */
 	u8 phase;
@@ -112,6 +113,7 @@ struct efa_com_dev {
 	u32 supported_features;
 	u32 dma_addr_bits;
 
+	u32 dev_api_ver;
 	struct efa_com_mmio_read mmio_read;
 };
 
diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c
index 7bd0838ebc99e..395290ab05847 100644
--- a/drivers/infiniband/hw/efa/efa_verbs.c
+++ b/drivers/infiniband/hw/efa/efa_verbs.c
@@ -218,12 +218,9 @@ int efa_query_device(struct ib_device *ibdev,
 	struct efa_dev *dev = to_edev(ibdev);
 	int err;
 
-	if (udata && udata->inlen &&
-	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
-		ibdev_dbg(ibdev,
-			  "Incompatible ABI params, udata not cleared\n");
-		return -EINVAL;
-	}
+	err = ib_is_udata_in_empty(udata);
+	if (err)
+		return err;
 
 	dev_attr = &dev->dev_attr;
 
@@ -273,13 +270,9 @@ int efa_query_device(struct ib_device *ibdev,
 		if (dev->neqs)
 			resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_NOTIFICATIONS;
 
-		err = ib_copy_to_udata(udata, &resp,
-				       min(sizeof(resp), udata->outlen));
-		if (err) {
-			ibdev_dbg(ibdev,
-				  "Failed to copy udata for query_device\n");
+		err = ib_respond_udata(udata, resp);
+		if (err)
 			return err;
-		}
 	}
 
 	return 0;
@@ -433,13 +426,9 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	struct efa_pd *pd = to_epd(ibpd);
 	int err;
 
-	if (udata->inlen &&
-	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
-		ibdev_dbg(&dev->ibdev,
-			  "Incompatible ABI params, udata not cleared\n");
-		err = -EINVAL;
+	err = ib_is_udata_in_empty(udata);
+	if (err)
 		goto err_out;
-	}
 
 	err = efa_com_alloc_pd(&dev->edev, &result);
 	if (err)
@@ -449,13 +438,9 @@ int efa_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	resp.pdn = result.pdn;
 
 	if (udata->outlen) {
-		err = ib_copy_to_udata(udata, &resp,
-				       min(sizeof(resp), udata->outlen));
-		if (err) {
-			ibdev_dbg(&dev->ibdev,
-				  "Failed to copy udata for alloc_pd\n");
+		err = ib_respond_udata(udata, resp);
+		if (err)
 			goto err_dealloc_pd;
-		}
 	}
 
 	ibdev_dbg(&dev->ibdev, "Allocated pd[%d]\n", pd->pdn);
@@ -789,14 +774,9 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 	qp->max_inline_data = init_attr->cap.max_inline_data;
 
 	if (udata->outlen) {
-		err = ib_copy_to_udata(udata, &resp,
-				       min(sizeof(resp), udata->outlen));
-		if (err) {
-			ibdev_dbg(&dev->ibdev,
-				  "Failed to copy udata for qp[%u]\n",
-				  create_qp_resp.qp_num);
+		err = ib_respond_udata(udata, resp);
+		if (err)
 			goto err_remove_mmap_entries;
-		}
 	}
 
 	ibdev_dbg(&dev->ibdev, "Created qp[%d]\n", qp->ibqp.qp_num);
@@ -982,12 +962,9 @@ int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 	if (qp_attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
 		return -EOPNOTSUPP;
 
-	if (udata->inlen &&
-	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
-		ibdev_dbg(&dev->ibdev,
-			  "Incompatible ABI params, udata not cleared\n");
-		return -EINVAL;
-	}
+	err = ib_is_udata_in_empty(udata);
+	if (err)
+		return err;
 
 	cur_state = qp_attr_mask & IB_QP_CUR_STATE ? qp_attr->cur_qp_state :
 						     qp->state;
@@ -1236,13 +1213,9 @@ int efa_create_user_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	}
 
 	if (udata->outlen) {
-		err = ib_copy_to_udata(udata, &resp,
-				       min(sizeof(resp), udata->outlen));
-		if (err) {
-			ibdev_dbg(ibdev,
-				  "Failed to copy udata for create_cq\n");
+		err = ib_respond_udata(udata, resp);
+		if (err)
 			goto err_xa_erase;
-		}
 	}
 
 	ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n",
@@ -1612,13 +1585,11 @@ static struct efa_mr *efa_alloc_mr(struct ib_pd *ibpd, int access_flags,
 	struct efa_dev *dev = to_edev(ibpd->device);
 	int supp_access_flags;
 	struct efa_mr *mr;
+	int ret;
 
-	if (udata && udata->inlen &&
-	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
-		ibdev_dbg(&dev->ibdev,
-			  "Incompatible ABI params, udata not cleared\n");
-		return ERR_PTR(-EINVAL);
-	}
+	ret = ib_is_udata_in_empty(udata);
+	if (ret)
+		return ERR_PTR(ret);
 
 	supp_access_flags =
 		IB_ACCESS_LOCAL_WRITE |
@@ -1947,8 +1918,7 @@ int efa_alloc_ucontext(struct ib_ucontext *ibucontext, struct ib_udata *udata)
 	resp.max_tx_batch = dev->dev_attr.max_tx_batch;
 	resp.min_sq_wr = dev->dev_attr.min_sq_depth;
 
-	err = ib_copy_to_udata(udata, &resp,
-			       min(sizeof(resp), udata->outlen));
+	err = ib_respond_udata(udata, resp);
 	if (err)
 		goto err_dealloc_uar;
 
@@ -2082,12 +2052,9 @@ int efa_create_ah(struct ib_ah *ibah,
 		goto err_out;
 	}
 
-	if (udata->inlen &&
-	    !ib_is_udata_cleared(udata, 0, udata->inlen)) {
-		ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n");
-		err = -EINVAL;
+	err = ib_is_udata_in_empty(udata);
+	if (err)
 		goto err_out;
-	}
 
 	memcpy(params.dest_addr, ah_attr->grh.dgid.raw,
 	       sizeof(params.dest_addr));
@@ -2102,13 +2069,9 @@ int efa_create_ah(struct ib_ah *ibah,
 	resp.efa_address_handle = result.ah;
 
 	if (udata->outlen) {
-		err = ib_copy_to_udata(udata, &resp,
-				       min(sizeof(resp), udata->outlen));
-		if (err) {
-			ibdev_dbg(&dev->ibdev,
-				  "Failed to copy udata for create_ah response\n");
+		err = ib_respond_udata(udata, resp);
+		if (err)
 			goto err_destroy_ah;
-		}
 	}
 	ibdev_dbg(&dev->ibdev, "Created ah[%d]\n", ah->ah);
 
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 5523b4e151e1f..b59c2e3a5306d 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -996,7 +996,7 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 	struct erdma_ucontext *uctx = rdma_udata_to_drv_context(
 		udata, struct erdma_ucontext, ibucontext);
 	struct erdma_ureq_create_qp ureq;
-	struct erdma_uresp_create_qp uresp;
+	struct erdma_uresp_create_qp uresp = {};
 	void *old_entry;
 	int ret = 0;
 
@@ -1048,14 +1048,12 @@ int erdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 		if (ret)
 			goto err_out_xa;
 
-		memset(&uresp, 0, sizeof(uresp));
-
 		uresp.num_sqe = qp->attrs.sq_size;
 		uresp.num_rqe = qp->attrs.rq_size;
 		uresp.qp_id = QP_ID(qp);
 		uresp.rq_offset = qp->user_qp.rq_offset;
 
-		ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+		ret = ib_respond_udata(udata, uresp);
 		if (ret)
 			goto err_out_cmd;
 	} else {
@@ -1571,7 +1569,7 @@ int erdma_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
 
 	uresp.dev_id = dev->pdev->device;
 
-	ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+	ret = ib_respond_udata(udata, uresp);
 	if (ret)
 		goto err_put_mmap_entries;
 
@@ -1977,7 +1975,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
 	if (!rdma_is_kernel_res(&ibcq->res)) {
 		struct erdma_ureq_create_cq ureq;
-		struct erdma_uresp_create_cq uresp;
+		struct erdma_uresp_create_cq uresp = {};
 
 		ret = ib_copy_validate_udata_in(udata, ureq, rsvd0);
 		if (ret)
@@ -1990,8 +1988,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		uresp.cq_id = cq->cqn;
 		uresp.num_cqe = depth;
 
-		ret = ib_copy_to_udata(udata, &uresp,
-				       min(sizeof(uresp), udata->outlen));
+		ret = ib_respond_udata(udata, uresp);
 		if (ret)
 			goto err_free_res;
 	} else {
diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c
index 8a605da8a93c9..925ddf15b6810 100644
--- a/drivers/infiniband/hw/hns/hns_roce_ah.c
+++ b/drivers/infiniband/hw/hns/hns_roce_ah.c
@@ -32,6 +32,7 @@
 
 #include <rdma/ib_addr.h>
 #include <rdma/ib_cache.h>
+#include <rdma/uverbs_ioctl.h>
 #include "hns_roce_device.h"
 #include "hns_roce_hw_v2.h"
 
@@ -112,8 +113,7 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 		resp.priority = ah->av.sl;
 		resp.tc_mode = tc_mode;
 		memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN);
-		ret = ib_copy_to_udata(udata, &resp,
-				       min(udata->outlen, sizeof(resp)));
+		ret = ib_respond_udata(udata, resp);
 	}
 
 err_out:
diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c
index 621568e114054..24de651f735e0 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_cq.c
@@ -452,8 +452,7 @@ int hns_roce_create_cq(struct ib_cq *ib_cq, const struct ib_cq_init_attr *attr,
 
 	if (udata) {
 		resp.cqn = hr_cq->cqn;
-		ret = ib_copy_to_udata(udata, &resp,
-				       min(udata->outlen, sizeof(resp)));
+		ret = ib_respond_udata(udata, resp);
 		if (ret)
 			goto err_cqc;
 	}
diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.c b/drivers/infiniband/hw/hns/hns_roce_debugfs.c
index b869cdc541189..05630f7c91551 100644
--- a/drivers/infiniband/hw/hns/hns_roce_debugfs.c
+++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.c
@@ -3,11 +3,13 @@
  * Copyright (c) 2023 Hisilicon Limited.
  */
 
+#include <linux/cleanup.h>
 #include <linux/debugfs.h>
 #include <linux/device.h>
 #include <linux/pci.h>
 
 #include "hns_roce_device.h"
+#include "hns_roce_hw_v2.h"
 
 static struct dentry *hns_roce_dbgfs_root;
 
@@ -18,25 +20,34 @@ static int hns_debugfs_seqfile_open(struct inode *inode, struct file *f)
 	return single_open(f, seqfile->read, seqfile->data);
 }
 
+static ssize_t hns_debugfs_seqfile_write(struct file *file,
+					 const char __user *buffer,
+					 size_t count, loff_t *ppos)
+{
+	struct hns_debugfs_seqfile *seqfile = file_inode(file)->i_private;
+	char buf[16] = {};
+
+	if (!seqfile->write)
+		return -EOPNOTSUPP;
+
+	if (count >= sizeof(buf))
+		return -EINVAL;
+
+	if (copy_from_user(buf, buffer, count))
+		return -EFAULT;
+
+	return seqfile->write(buf, count, seqfile->data);
+}
+
 static const struct file_operations hns_debugfs_seqfile_fops = {
 	.owner = THIS_MODULE,
 	.open = hns_debugfs_seqfile_open,
 	.release = single_release,
 	.read = seq_read,
+	.write = hns_debugfs_seqfile_write,
 	.llseek = seq_lseek
 };
 
-static void init_debugfs_seqfile(struct hns_debugfs_seqfile *seq,
-				 const char *name, struct dentry *parent,
-				 int (*read_fn)(struct seq_file *, void *),
-				 void *data)
-{
-	debugfs_create_file(name, 0400, parent, seq, &hns_debugfs_seqfile_fops);
-
-	seq->read = read_fn;
-	seq->data = data;
-}
-
 static const char * const sw_stat_info[] = {
 	[HNS_ROCE_DFX_AEQE_CNT] = "aeqe",
 	[HNS_ROCE_DFX_CEQE_CNT] = "ceqe",
@@ -76,10 +87,256 @@ static void create_sw_stat_debugfs(struct hns_roce_dev *hr_dev,
 {
 	struct hns_sw_stat_debugfs *dbgfs = &hr_dev->dbgfs.sw_stat_root;
 
+	dbgfs->sw_stat.read = sw_stat_debugfs_show;
+	dbgfs->sw_stat.data = hr_dev;
+
 	dbgfs->root = debugfs_create_dir("sw_stat", parent);
+	debugfs_create_file("sw_stat", 0400, dbgfs->root, &dbgfs->sw_stat,
+			    &hns_debugfs_seqfile_fops);
+}
+
+#define __HNS_SCC_ATTR(_name, _type, _offset, _size, _min, _max) {	\
+	.name = _name,							\
+	.algo_type = _type,						\
+	.offset = _offset,						\
+	.size = _size,							\
+	.min = _min,							\
+	.max = _max,							\
+}
+
+#define HNS_DCQCN_CC_ATTR_RW(_name, NAME)				\
+	__HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DCQCN,			\
+			HNS_ROCE_DCQCN_##NAME##_OFS,			\
+			HNS_ROCE_DCQCN_##NAME##_SZ,			\
+			0, HNS_ROCE_DCQCN_##NAME##_MAX)
+
+#define HNS_LDCP_CC_ATTR_RW(_name, NAME)				\
+	__HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_LDCP,			\
+			HNS_ROCE_LDCP_##NAME##_OFS,			\
+			HNS_ROCE_LDCP_##NAME##_SZ,			\
+			0, HNS_ROCE_LDCP_##NAME##_MAX)
+
+#define HNS_HC3_CC_ATTR_RW(_name, NAME)				\
+	__HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_HC3,			\
+			HNS_ROCE_HC3_##NAME##_OFS,			\
+			HNS_ROCE_HC3_##NAME##_SZ,			\
+			HNS_ROCE_HC3_##NAME##_MIN,			\
+			HNS_ROCE_HC3_##NAME##_MAX)
+
+#define HNS_DIP_CC_ATTR_RW(_name, NAME)				\
+	__HNS_SCC_ATTR(_name, HNS_ROCE_SCC_ALGO_DIP,			\
+			HNS_ROCE_DIP_##NAME##_OFS,			\
+			HNS_ROCE_DIP_##NAME##_SZ,			\
+			0, HNS_ROCE_DIP_##NAME##_MAX)
+
+static const struct hns_roce_cong_attr {
+	enum hns_roce_cong_type cong_type;
+	const char *name;
+	struct hns_roce_cc_param_attr params[HNS_ROCE_CC_PARAM_MAX_NUM];
+} cong_attrs[] = {
+	{ CONG_TYPE_DCQCN, "dcqcn_cc_param",
+		{
+			HNS_DCQCN_CC_ATTR_RW("ai", AI),
+			HNS_DCQCN_CC_ATTR_RW("f", F),
+			HNS_DCQCN_CC_ATTR_RW("tkp", TKP),
+			HNS_DCQCN_CC_ATTR_RW("tmp", TMP),
+			HNS_DCQCN_CC_ATTR_RW("alp", ALP),
+			HNS_DCQCN_CC_ATTR_RW("max_speed", MAX_SPEED),
+			HNS_DCQCN_CC_ATTR_RW("g", G),
+			HNS_DCQCN_CC_ATTR_RW("al", AL),
+			HNS_DCQCN_CC_ATTR_RW("cnp_time", CNP_TIME),
+			HNS_DCQCN_CC_ATTR_RW("ashift", ASHIFT),
+		}
+	},
+	{ CONG_TYPE_LDCP, "ldcp_cc_param",
+		{
+			HNS_LDCP_CC_ATTR_RW("cwd0", CWD0),
+			HNS_LDCP_CC_ATTR_RW("alpha", ALPHA),
+			HNS_LDCP_CC_ATTR_RW("gamma", GAMMA),
+			HNS_LDCP_CC_ATTR_RW("beta", BETA),
+			HNS_LDCP_CC_ATTR_RW("eta", ETA),
+		}
+	},
+	{ CONG_TYPE_HC3, "hc3_cc_param",
+		{
+			HNS_HC3_CC_ATTR_RW("initial_window", INITIAL_WINDOW),
+			HNS_HC3_CC_ATTR_RW("bandwidth", BANDWIDTH),
+			HNS_HC3_CC_ATTR_RW("qlen_shift", QLEN_SHIFT),
+			HNS_HC3_CC_ATTR_RW("port_usage_shift", PORT_USAGE_SHIFT),
+			HNS_HC3_CC_ATTR_RW("over_period", OVER_PERIOD),
+			HNS_HC3_CC_ATTR_RW("max_stage", MAX_STAGE),
+			HNS_HC3_CC_ATTR_RW("gamma_shift", GAMMA_SHIFT),
+		}
+	},
+	{ CONG_TYPE_DIP, "dip_cc_param",
+		{
+			HNS_DIP_CC_ATTR_RW("ai", AI),
+			HNS_DIP_CC_ATTR_RW("f", F),
+			HNS_DIP_CC_ATTR_RW("tkp", TKP),
+			HNS_DIP_CC_ATTR_RW("tmp", TMP),
+			HNS_DIP_CC_ATTR_RW("alp", ALP),
+			HNS_DIP_CC_ATTR_RW("max_speed", MAX_SPEED),
+			HNS_DIP_CC_ATTR_RW("g", G),
+			HNS_DIP_CC_ATTR_RW("al", AL),
+			HNS_DIP_CC_ATTR_RW("cnp_time", CNP_TIME),
+			HNS_DIP_CC_ATTR_RW("ashift", ASHIFT),
+		}
+	}
+};
+
+static int cc_param_debugfs_show(struct seq_file *file, void *offset)
+{
+	struct hns_cc_param_seqfile *param_seqfile = file->private;
+	const struct hns_roce_cc_param_attr *param_attr = param_seqfile->param_attr;
+	int algo_type = param_attr->algo_type;
+	int index = param_seqfile->index;
+	struct hns_roce_dev *hr_dev =
+		container_of(param_seqfile, struct hns_roce_dev,
+			     dbgfs.cc_param_root[algo_type].params[index]);
+	struct hns_roce_scc_param *scc_param;
+	__le32 val = 0;
+
+	scc_param = &hr_dev->scc_param[algo_type];
+
+	scoped_guard(mutex, &scc_param->scc_mutex) {
+		memcpy(&val,
+		       (void *)scc_param->param + param_attr->offset,
+		       param_attr->size);
+	}
+
+	seq_printf(file, "%u\n", le32_to_cpu(val));
+
+	return 0;
+}
+
+static ssize_t cc_param_debugfs_store(char *buf, size_t count, void *data)
+{
+	struct hns_cc_param_seqfile *param_seqfile = data;
+	const struct hns_roce_cc_param_attr *param_attr = param_seqfile->param_attr;
+	int algo_type = param_attr->algo_type;
+	int index = param_seqfile->index;
+	struct hns_roce_dev *hr_dev =
+		container_of(param_seqfile, struct hns_roce_dev,
+			     dbgfs.cc_param_root[algo_type].params[index]);
+	struct hns_roce_scc_param *scc_param;
+	__le32 attr_val, old_val = 0;
+	u32 val;
+	int ret;
+
+	if (kstrtou32(buf, 0, &val))
+		return -EINVAL;
+
+	if (val > param_attr->max || val < param_attr->min)
+		return -EINVAL;
+
+	attr_val = cpu_to_le32(val);
+	scc_param = &hr_dev->scc_param[algo_type];
+	guard(mutex)(&scc_param->scc_mutex);
+
+	memcpy(&old_val, (void *)scc_param->param + param_attr->offset,
+	       param_attr->size);
+	memcpy((void *)scc_param->param + param_attr->offset, &attr_val,
+	       param_attr->size);
+
+	ret = hr_dev->hw->config_scc_param(hr_dev, algo_type);
+	if (ret) {
+		memcpy((void *)scc_param->param + param_attr->offset, &old_val,
+		       param_attr->size);
+		return ret;
+	}
+
+	return count;
+}
+
+static void get_default_scc_param(struct hns_roce_dev *hr_dev)
+{
+	int ret;
+	int i;
+
+	for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) {
+		ret = hr_dev->hw->query_scc_param(hr_dev, i);
+		if (ret && ret != -EOPNOTSUPP)
+			ibdev_warn_ratelimited(&hr_dev->ib_dev,
+					       "failed to get default parameters of scc algo %d, ret = %d.\n",
+					       i, ret);
+	}
+}
+
+static int hns_roce_alloc_scc_param(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_scc_param *scc_param;
+	int i;
+
+	scc_param = kvcalloc(HNS_ROCE_SCC_ALGO_TOTAL, sizeof(*scc_param),
+			     GFP_KERNEL);
+	if (!scc_param)
+		return -ENOMEM;
+
+	for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++) {
+		scc_param[i].algo_type = i;
+		scc_param[i].hr_dev = hr_dev;
+		mutex_init(&scc_param[i].scc_mutex);
+	}
+
+	hr_dev->scc_param = scc_param;
+
+	get_default_scc_param(hr_dev);
+
+	return 0;
+}
+
+static void hns_roce_dealloc_scc_param(struct hns_roce_dev *hr_dev)
+{
+	int i;
+
+	if (!hr_dev->scc_param)
+		return;
+
+	for (i = 0; i < HNS_ROCE_SCC_ALGO_TOTAL; i++)
+		mutex_destroy(&hr_dev->scc_param[i].scc_mutex);
+
+	kvfree(hr_dev->scc_param);
+}
+
+static void create_cc_param_debugfs(struct hns_roce_dev *hr_dev,
+				    struct dentry *parent)
+{
+	const struct hns_roce_cong_attr *cong_attr;
+	struct hns_cc_param_debugfs *dbgfs;
+	int i, j;
+	int ret;
+
+	if (hr_dev->pci_dev->revision <= PCI_REVISION_ID_HIP08 ||
+	    !(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) ||
+	    hr_dev->is_vf)
+		return;
+
+	ret = hns_roce_alloc_scc_param(hr_dev);
+	if (ret)
+		return;
+
+	for (i = 0; i < CONG_TYPE_MAX_NUM; i++) {
+		cong_attr = &cong_attrs[i];
+		if (!test_bit(cong_attr->cong_type,
+			      (unsigned long *)&hr_dev->caps.cong_cap))
+			continue;
 
-	init_debugfs_seqfile(&dbgfs->sw_stat, "sw_stat", dbgfs->root,
-			     sw_stat_debugfs_show, hr_dev);
+		dbgfs = &hr_dev->dbgfs.cc_param_root[i];
+		dbgfs->root = debugfs_create_dir(cong_attr->name, parent);
+		for (j = 0; j < HNS_ROCE_CC_PARAM_MAX_NUM; j++) {
+			if (!cong_attr->params[j].name)
+				break;
+			dbgfs->params[j].param_attr = &cong_attr->params[j];
+			dbgfs->params[j].index = j;
+			dbgfs->params[j].seqfile.read = cc_param_debugfs_show;
+			dbgfs->params[j].seqfile.write = cc_param_debugfs_store;
+			dbgfs->params[j].seqfile.data = &dbgfs->params[j];
+			debugfs_create_file(cong_attr->params[j].name, 0600,
+					    dbgfs->root,
+					    &dbgfs->params[j].seqfile,
+					    &hns_debugfs_seqfile_fops);
+		}
+	}
 }
 
 /* debugfs for device */
@@ -91,11 +348,13 @@ void hns_roce_register_debugfs(struct hns_roce_dev *hr_dev)
 					 hns_roce_dbgfs_root);
 
 	create_sw_stat_debugfs(hr_dev, dbgfs->root);
+	create_cc_param_debugfs(hr_dev, dbgfs->root);
 }
 
 void hns_roce_unregister_debugfs(struct hns_roce_dev *hr_dev)
 {
 	debugfs_remove_recursive(hr_dev->dbgfs.root);
+	hns_roce_dealloc_scc_param(hr_dev);
 }
 
 /* debugfs for hns module */
diff --git a/drivers/infiniband/hw/hns/hns_roce_debugfs.h b/drivers/infiniband/hw/hns/hns_roce_debugfs.h
index 98e87bd3161ee..116b1e8b66772 100644
--- a/drivers/infiniband/hw/hns/hns_roce_debugfs.h
+++ b/drivers/infiniband/hw/hns/hns_roce_debugfs.h
@@ -9,6 +9,7 @@
 /* debugfs seqfile */
 struct hns_debugfs_seqfile {
 	int (*read)(struct seq_file *seq, void *data);
+	ssize_t (*write)(char *buf, size_t count, void *data);
 	void *data;
 };
 
@@ -17,10 +18,35 @@ struct hns_sw_stat_debugfs {
 	struct hns_debugfs_seqfile sw_stat;
 };
 
+struct hns_roce_cc_param_attr {
+	const char *name;
+	int algo_type;
+	u32 offset;
+	u32 size;
+	u32 max;
+	u32 min;
+};
+
+struct hns_cc_param_seqfile {
+	struct hns_debugfs_seqfile seqfile;
+	const struct hns_roce_cc_param_attr *param_attr;
+	int index;
+};
+
+#define HNS_ROCE_CC_PARAM_MAX_NUM 11
+
+struct hns_cc_param_debugfs {
+	struct dentry *root;
+	struct hns_cc_param_seqfile params[HNS_ROCE_CC_PARAM_MAX_NUM];
+};
+
+#define CONG_TYPE_MAX_NUM 4
+
 /* Debugfs for device */
 struct hns_roce_dev_debugfs {
 	struct dentry *root;
 	struct hns_sw_stat_debugfs sw_stat_root;
+	struct hns_cc_param_debugfs cc_param_root[CONG_TYPE_MAX_NUM];
 };
 
 struct hns_roce_dev;
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 3f032b8038af0..eb7b1865e4c79 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -726,6 +726,14 @@ struct hns_roce_eq_table {
 	struct hns_roce_eq	*eq;
 };
 
+enum hns_roce_scc_algo {
+	HNS_ROCE_SCC_ALGO_DCQCN = 0,
+	HNS_ROCE_SCC_ALGO_LDCP,
+	HNS_ROCE_SCC_ALGO_HC3,
+	HNS_ROCE_SCC_ALGO_DIP,
+	HNS_ROCE_SCC_ALGO_TOTAL,
+};
+
 struct hns_roce_caps {
 	u64		fw_ver;
 	u8		num_ports;
@@ -963,6 +971,18 @@ struct hns_roce_hw {
 			u8 *tc_mode, u8 *priority);
 	const struct ib_device_ops *hns_roce_dev_ops;
 	const struct ib_device_ops *hns_roce_dev_srq_ops;
+	int (*config_scc_param)(struct hns_roce_dev *hr_dev,
+				enum hns_roce_scc_algo algo);
+	int (*query_scc_param)(struct hns_roce_dev *hr_dev,
+			       enum hns_roce_scc_algo alog);
+};
+
+#define HNS_ROCE_SCC_PARAM_SIZE 4
+struct hns_roce_scc_param {
+	__le32 param[HNS_ROCE_SCC_PARAM_SIZE];
+	enum hns_roce_scc_algo algo_type;
+	struct hns_roce_dev *hr_dev;
+	struct mutex scc_mutex; /* protect @param */
 };
 
 struct hns_roce_dev {
@@ -1026,6 +1046,7 @@ struct hns_roce_dev {
 	u64 dwqe_page;
 	struct hns_roce_dev_debugfs dbgfs;
 	atomic64_t *dfx_cnt;
+	struct hns_roce_scc_param *scc_param;
 };
 
 enum hns_roce_trace_type {
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c
index e7c9e30ad2d8b..ccb40f8a48b72 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.c
@@ -314,14 +314,14 @@ static int calc_hem_config(struct hns_roce_dev *hr_dev,
 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
 	switch (bt_num) {
 	case 3:
-		index->l1 = l0_idx * chunk_ba_num + l1_idx;
+		index->l1 = (u64)l0_idx * chunk_ba_num + l1_idx;
 		index->l0 = l0_idx;
-		index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
-			     l1_idx * chunk_ba_num + l2_idx;
+		index->buf = (u64)l0_idx * chunk_ba_num * chunk_ba_num +
+					 (u64)l1_idx * chunk_ba_num + l2_idx;
 		break;
 	case 2:
 		index->l0 = l0_idx;
-		index->buf = l0_idx * chunk_ba_num + l1_idx;
+		index->buf = (u64)l0_idx * chunk_ba_num + l1_idx;
 		break;
 	case 1:
 		index->buf = l0_idx;
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index fa36700d0db2b..4afd7d6ae3caa 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -31,6 +31,7 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/cleanup.h>
 #include <linux/etherdevice.h>
 #include <linux/interrupt.h>
 #include <linux/iopoll.h>
@@ -7196,6 +7197,57 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
 	kfree(eq_table->eq);
 }
 
+static const enum hns_roce_opcode_type scc_opcode[] = {
+	HNS_ROCE_OPC_CFG_DCQCN_PARAM,
+	HNS_ROCE_OPC_CFG_LDCP_PARAM,
+	HNS_ROCE_OPC_CFG_HC3_PARAM,
+	HNS_ROCE_OPC_CFG_DIP_PARAM,
+};
+
+static int hns_roce_v2_config_scc_param(struct hns_roce_dev *hr_dev,
+					enum hns_roce_scc_algo algo)
+{
+	struct hns_roce_scc_param *scc_param = &hr_dev->scc_param[algo];
+	struct hns_roce_cmq_desc desc;
+	int ret;
+
+	lockdep_assert_held(&scc_param->scc_mutex);
+
+	hns_roce_cmq_setup_basic_desc(&desc, scc_opcode[algo], false);
+	memcpy(&desc.data, scc_param->param, sizeof(scc_param->param));
+
+	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	if (ret)
+		ibdev_err_ratelimited(&hr_dev->ib_dev,
+				      "failed to configure scc param, opcode: 0x%x, ret = %d.\n",
+				      le16_to_cpu(desc.opcode), ret);
+
+	return ret;
+}
+
+static int hns_roce_v2_query_scc_param(struct hns_roce_dev *hr_dev,
+				       enum hns_roce_scc_algo algo)
+{
+	struct hns_roce_scc_param *scc_param;
+	struct hns_roce_cmq_desc desc;
+	int ret;
+
+	hns_roce_cmq_setup_basic_desc(&desc, scc_opcode[algo], true);
+	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	if (ret) {
+		ibdev_err_ratelimited(&hr_dev->ib_dev,
+				      "failed to query scc param, opcode: 0x%x, ret = %d.\n",
+				      le16_to_cpu(desc.opcode), ret);
+		return ret;
+	}
+
+	scc_param = &hr_dev->scc_param[algo];
+	scoped_guard(mutex, &scc_param->scc_mutex)
+		memcpy(scc_param->param, &desc.data, sizeof(scc_param->param));
+
+	return 0;
+}
+
 static const struct ib_device_ops hns_roce_v2_dev_ops = {
 	.destroy_qp = hns_roce_v2_destroy_qp,
 	.modify_cq = hns_roce_v2_modify_cq,
@@ -7246,19 +7298,35 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
 	.get_dscp = hns_roce_hw_v2_get_dscp,
 	.hns_roce_dev_ops = &hns_roce_v2_dev_ops,
 	.hns_roce_dev_srq_ops = &hns_roce_v2_dev_srq_ops,
+	.config_scc_param = hns_roce_v2_config_scc_param,
+	.query_scc_param = hns_roce_v2_query_scc_param,
 };
 
 static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
-	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
+	{
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
+		.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
+	},
 	/* required last entry */
-	{0, }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, hns_roce_hw_v2_pci_tbl);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index 285fe0875fac0..8c7856aa24dcc 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -202,6 +202,10 @@ enum {
 /* CMQ command */
 enum hns_roce_opcode_type {
 	HNS_QUERY_FW_VER				= 0x0001,
+	HNS_ROCE_OPC_CFG_DCQCN_PARAM			= 0x1A80,
+	HNS_ROCE_OPC_CFG_LDCP_PARAM			= 0x1A81,
+	HNS_ROCE_OPC_CFG_HC3_PARAM			= 0x1A82,
+	HNS_ROCE_OPC_CFG_DIP_PARAM			= 0x1A83,
 	HNS_ROCE_OPC_QUERY_HW_VER			= 0x8000,
 	HNS_ROCE_OPC_CFG_GLOBAL_PARAM			= 0x8001,
 	HNS_ROCE_OPC_ALLOC_PF_RES			= 0x8004,
@@ -1459,6 +1463,127 @@ struct hns_roce_wqe_atomic_seg {
 	__le64          cmp_data;
 };
 
+#define HNS_ROCE_DCQCN_AI_OFS 0
+#define HNS_ROCE_DCQCN_AI_SZ sizeof(u16)
+#define HNS_ROCE_DCQCN_AI_MAX ((u16)(~0U))
+#define HNS_ROCE_DCQCN_F_OFS (HNS_ROCE_DCQCN_AI_OFS + HNS_ROCE_DCQCN_AI_SZ)
+#define HNS_ROCE_DCQCN_F_SZ sizeof(u8)
+#define HNS_ROCE_DCQCN_F_MAX ((u8)(~0U))
+#define HNS_ROCE_DCQCN_TKP_OFS (HNS_ROCE_DCQCN_F_OFS + HNS_ROCE_DCQCN_F_SZ)
+#define HNS_ROCE_DCQCN_TKP_SZ sizeof(u8)
+#define HNS_ROCE_DCQCN_TKP_MAX 10
+#define HNS_ROCE_DCQCN_TMP_OFS (HNS_ROCE_DCQCN_TKP_OFS + HNS_ROCE_DCQCN_TKP_SZ)
+#define HNS_ROCE_DCQCN_TMP_SZ sizeof(u16)
+#define HNS_ROCE_DCQCN_TMP_MAX 15
+#define HNS_ROCE_DCQCN_ALP_OFS (HNS_ROCE_DCQCN_TMP_OFS + HNS_ROCE_DCQCN_TMP_SZ)
+#define HNS_ROCE_DCQCN_ALP_SZ sizeof(u16)
+#define HNS_ROCE_DCQCN_ALP_MAX ((u16)(~0U))
+#define HNS_ROCE_DCQCN_MAX_SPEED_OFS (HNS_ROCE_DCQCN_ALP_OFS + \
+					HNS_ROCE_DCQCN_ALP_SZ)
+#define HNS_ROCE_DCQCN_MAX_SPEED_SZ sizeof(u32)
+#define HNS_ROCE_DCQCN_MAX_SPEED_MAX ((u32)(~0U))
+#define HNS_ROCE_DCQCN_G_OFS (HNS_ROCE_DCQCN_MAX_SPEED_OFS + \
+					HNS_ROCE_DCQCN_MAX_SPEED_SZ)
+#define HNS_ROCE_DCQCN_G_SZ sizeof(u8)
+#define HNS_ROCE_DCQCN_G_MAX 15
+#define HNS_ROCE_DCQCN_AL_OFS (HNS_ROCE_DCQCN_G_OFS + HNS_ROCE_DCQCN_G_SZ)
+#define HNS_ROCE_DCQCN_AL_SZ sizeof(u8)
+#define HNS_ROCE_DCQCN_AL_MAX ((u8)(~0U))
+#define HNS_ROCE_DCQCN_CNP_TIME_OFS (HNS_ROCE_DCQCN_AL_OFS + \
+					HNS_ROCE_DCQCN_AL_SZ)
+#define HNS_ROCE_DCQCN_CNP_TIME_SZ sizeof(u8)
+#define HNS_ROCE_DCQCN_CNP_TIME_MAX ((u8)(~0U))
+#define HNS_ROCE_DCQCN_ASHIFT_OFS (HNS_ROCE_DCQCN_CNP_TIME_OFS + \
+					HNS_ROCE_DCQCN_CNP_TIME_SZ)
+#define HNS_ROCE_DCQCN_ASHIFT_SZ sizeof(u8)
+#define HNS_ROCE_DCQCN_ASHIFT_MAX 15
+
+#define HNS_ROCE_LDCP_CWD0_OFS 0
+#define HNS_ROCE_LDCP_CWD0_SZ sizeof(u32)
+#define HNS_ROCE_LDCP_CWD0_MAX ((u32)(~0U))
+#define HNS_ROCE_LDCP_ALPHA_OFS (HNS_ROCE_LDCP_CWD0_OFS + HNS_ROCE_LDCP_CWD0_SZ)
+#define HNS_ROCE_LDCP_ALPHA_SZ sizeof(u8)
+#define HNS_ROCE_LDCP_ALPHA_MAX ((u8)(~0U))
+#define HNS_ROCE_LDCP_GAMMA_OFS (HNS_ROCE_LDCP_ALPHA_OFS + \
+					HNS_ROCE_LDCP_ALPHA_SZ)
+#define HNS_ROCE_LDCP_GAMMA_SZ sizeof(u8)
+#define HNS_ROCE_LDCP_GAMMA_MAX 7
+#define HNS_ROCE_LDCP_BETA_OFS (HNS_ROCE_LDCP_GAMMA_OFS + \
+					HNS_ROCE_LDCP_GAMMA_SZ)
+#define HNS_ROCE_LDCP_BETA_SZ sizeof(u8)
+#define HNS_ROCE_LDCP_BETA_MAX 7
+#define HNS_ROCE_LDCP_ETA_OFS (HNS_ROCE_LDCP_BETA_OFS + HNS_ROCE_LDCP_BETA_SZ)
+#define HNS_ROCE_LDCP_ETA_SZ sizeof(u8)
+#define HNS_ROCE_LDCP_ETA_MAX 7
+
+#define HNS_ROCE_HC3_INITIAL_WINDOW_OFS 0
+#define HNS_ROCE_HC3_INITIAL_WINDOW_SZ sizeof(u32)
+#define HNS_ROCE_HC3_INITIAL_WINDOW_MIN 0
+#define HNS_ROCE_HC3_INITIAL_WINDOW_MAX ((u32)(~0U))
+#define HNS_ROCE_HC3_BANDWIDTH_OFS (HNS_ROCE_HC3_INITIAL_WINDOW_OFS + \
+					HNS_ROCE_HC3_INITIAL_WINDOW_SZ)
+#define HNS_ROCE_HC3_BANDWIDTH_SZ sizeof(u32)
+#define HNS_ROCE_HC3_BANDWIDTH_MIN 1000
+#define HNS_ROCE_HC3_BANDWIDTH_MAX ((u32)(~0U))
+#define HNS_ROCE_HC3_QLEN_SHIFT_OFS (HNS_ROCE_HC3_BANDWIDTH_OFS + \
+					HNS_ROCE_HC3_BANDWIDTH_SZ)
+#define HNS_ROCE_HC3_QLEN_SHIFT_SZ sizeof(u8)
+#define HNS_ROCE_HC3_QLEN_SHIFT_MIN 0
+#define HNS_ROCE_HC3_QLEN_SHIFT_MAX 31
+#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_OFS (HNS_ROCE_HC3_QLEN_SHIFT_OFS + \
+						HNS_ROCE_HC3_QLEN_SHIFT_SZ)
+#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_SZ sizeof(u8)
+#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_MIN 0
+#define HNS_ROCE_HC3_PORT_USAGE_SHIFT_MAX 100
+#define HNS_ROCE_HC3_OVER_PERIOD_OFS (HNS_ROCE_HC3_PORT_USAGE_SHIFT_OFS + \
+					HNS_ROCE_HC3_PORT_USAGE_SHIFT_SZ)
+#define HNS_ROCE_HC3_OVER_PERIOD_SZ sizeof(u8)
+#define HNS_ROCE_HC3_OVER_PERIOD_MIN 0
+#define HNS_ROCE_HC3_OVER_PERIOD_MAX ((u8)(~0U))
+#define HNS_ROCE_HC3_MAX_STAGE_OFS (HNS_ROCE_HC3_OVER_PERIOD_OFS + \
+					HNS_ROCE_HC3_OVER_PERIOD_SZ)
+#define HNS_ROCE_HC3_MAX_STAGE_SZ sizeof(u8)
+#define HNS_ROCE_HC3_MAX_STAGE_MIN 0
+#define HNS_ROCE_HC3_MAX_STAGE_MAX ((u8)(~0U))
+#define HNS_ROCE_HC3_GAMMA_SHIFT_OFS (HNS_ROCE_HC3_MAX_STAGE_OFS + \
+					HNS_ROCE_HC3_MAX_STAGE_SZ)
+#define HNS_ROCE_HC3_GAMMA_SHIFT_SZ sizeof(u8)
+#define HNS_ROCE_HC3_GAMMA_SHIFT_MIN 0
+#define HNS_ROCE_HC3_GAMMA_SHIFT_MAX 15
+
+#define HNS_ROCE_DIP_AI_OFS 0
+#define HNS_ROCE_DIP_AI_SZ sizeof(u16)
+#define HNS_ROCE_DIP_AI_MAX ((u16)(~0U))
+#define HNS_ROCE_DIP_F_OFS (HNS_ROCE_DIP_AI_OFS + HNS_ROCE_DIP_AI_SZ)
+#define HNS_ROCE_DIP_F_SZ sizeof(u8)
+#define HNS_ROCE_DIP_F_MAX ((u8)(~0U))
+#define HNS_ROCE_DIP_TKP_OFS (HNS_ROCE_DIP_F_OFS + HNS_ROCE_DIP_F_SZ)
+#define HNS_ROCE_DIP_TKP_SZ sizeof(u8)
+#define HNS_ROCE_DIP_TKP_MAX 10
+#define HNS_ROCE_DIP_TMP_OFS (HNS_ROCE_DIP_TKP_OFS + HNS_ROCE_DIP_TKP_SZ)
+#define HNS_ROCE_DIP_TMP_SZ sizeof(u16)
+#define HNS_ROCE_DIP_TMP_MAX 15
+#define HNS_ROCE_DIP_ALP_OFS (HNS_ROCE_DIP_TMP_OFS + HNS_ROCE_DIP_TMP_SZ)
+#define HNS_ROCE_DIP_ALP_SZ sizeof(u16)
+#define HNS_ROCE_DIP_ALP_MAX ((u16)(~0U))
+#define HNS_ROCE_DIP_MAX_SPEED_OFS (HNS_ROCE_DIP_ALP_OFS + HNS_ROCE_DIP_ALP_SZ)
+#define HNS_ROCE_DIP_MAX_SPEED_SZ sizeof(u32)
+#define HNS_ROCE_DIP_MAX_SPEED_MAX ((u32)(~0U))
+#define HNS_ROCE_DIP_G_OFS (HNS_ROCE_DIP_MAX_SPEED_OFS + \
+				HNS_ROCE_DIP_MAX_SPEED_SZ)
+#define HNS_ROCE_DIP_G_SZ sizeof(u8)
+#define HNS_ROCE_DIP_G_MAX 15
+#define HNS_ROCE_DIP_AL_OFS (HNS_ROCE_DIP_G_OFS + HNS_ROCE_DIP_G_SZ)
+#define HNS_ROCE_DIP_AL_SZ sizeof(u8)
+#define HNS_ROCE_DIP_AL_MAX ((u8)(~0U))
+#define HNS_ROCE_DIP_CNP_TIME_OFS (HNS_ROCE_DIP_AL_OFS + HNS_ROCE_DIP_AL_SZ)
+#define HNS_ROCE_DIP_CNP_TIME_SZ sizeof(u8)
+#define HNS_ROCE_DIP_CNP_TIME_MAX ((u8)(~0U))
+#define HNS_ROCE_DIP_ASHIFT_OFS (HNS_ROCE_DIP_CNP_TIME_OFS + \
+					HNS_ROCE_DIP_CNP_TIME_SZ)
+#define HNS_ROCE_DIP_ASHIFT_SZ sizeof(u8)
+#define HNS_ROCE_DIP_ASHIFT_MAX 15
+
 struct hns_roce_sccc_clr {
 	__le32 qpn;
 	__le32 rsv[5];
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 0dbe99aab6ad2..c17ff5347a014 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -477,8 +477,7 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
 
 	resp.cqe_size = hr_dev->caps.cqe_sz;
 
-	ret = ib_copy_to_udata(udata, &resp,
-			       min(udata->outlen, sizeof(resp)));
+	ret = ib_respond_udata(udata, resp);
 	if (ret)
 		goto error_fail_copy_to_udata;
 
diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c
index 225c3e328e0e0..73bb000574c50 100644
--- a/drivers/infiniband/hw/hns/hns_roce_pd.c
+++ b/drivers/infiniband/hw/hns/hns_roce_pd.c
@@ -30,6 +30,7 @@
  * SOFTWARE.
  */
 
+#include <rdma/uverbs_ioctl.h>
 #include "hns_roce_device.h"
 
 void hns_roce_init_pd_table(struct hns_roce_dev *hr_dev)
@@ -61,12 +62,9 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	if (udata) {
 		struct hns_roce_ib_alloc_pd_resp resp = {.pdn = pd->pdn};
 
-		ret = ib_copy_to_udata(udata, &resp,
-				       min(udata->outlen, sizeof(resp)));
-		if (ret) {
+		ret = ib_respond_udata(udata, resp);
+		if (ret)
 			ida_free(&pd_ida->ida, id);
-			ibdev_err(ib_dev, "failed to copy to udata, ret = %d\n", ret);
-		}
 	}
 
 	return ret;
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index bf04ee84a9439..e333a8c4acb52 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -1236,12 +1236,9 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 
 	if (udata) {
 		resp.cap_flags = hr_qp->en_flags;
-		ret = ib_copy_to_udata(udata, &resp,
-				       min(udata->outlen, sizeof(resp)));
-		if (ret) {
-			ibdev_err(ibdev, "copy qp resp failed!\n");
+		ret = ib_respond_udata(udata, resp);
+		if (ret)
 			goto err_flow_ctrl;
-		}
 	}
 
 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
@@ -1494,11 +1491,7 @@ int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	if (udata && udata->outlen) {
 		resp.tc_mode = hr_qp->tc_mode;
 		resp.priority = hr_qp->sl;
-		ret = ib_copy_to_udata(udata, &resp,
-				       min(udata->outlen, sizeof(resp)));
-		if (ret)
-			ibdev_err_ratelimited(&hr_dev->ib_dev,
-					      "failed to copy modify qp resp.\n");
+		ret = ib_respond_udata(udata, resp);
 	}
 
 out:
diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c
index 8b94cbdfa54df..241fc9980f4f5 100644
--- a/drivers/infiniband/hw/hns/hns_roce_srq.c
+++ b/drivers/infiniband/hw/hns/hns_roce_srq.c
@@ -477,11 +477,9 @@ int hns_roce_create_srq(struct ib_srq *ib_srq,
 	if (udata) {
 		resp.cap_flags = srq->cap_flags;
 		resp.srqn = srq->srqn;
-		if (ib_copy_to_udata(udata, &resp,
-				     min(udata->outlen, sizeof(resp)))) {
-			ret = -EFAULT;
+		ret = ib_respond_udata(udata, resp);
+		if (ret)
 			goto err_srqc;
-		}
 	}
 
 	return 0;
diff --git a/drivers/infiniband/hw/ionic/ionic_controlpath.c b/drivers/infiniband/hw/ionic/ionic_controlpath.c
index 7051a81cca942..2b01345848ddb 100644
--- a/drivers/infiniband/hw/ionic/ionic_controlpath.c
+++ b/drivers/infiniband/hw/ionic/ionic_controlpath.c
@@ -414,7 +414,7 @@ int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata)
 	if (dev->lif_cfg.rq_expdb)
 		resp.expdb_qtypes |= IONIC_EXPDB_RQ;
 
-	rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
+	rc = ib_respond_udata(udata, resp);
 	if (rc)
 		goto err_resp;
 
@@ -752,7 +752,7 @@ int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 	if (udata) {
 		resp.ahid = ah->ahid;
 
-		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
+		rc = ib_respond_udata(udata, resp);
 		if (rc)
 			goto err_resp;
 	}
@@ -1263,7 +1263,7 @@ int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	if (udata) {
 		resp.udma_mask = vcq->udma_mask;
 
-		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
+		rc = ib_respond_udata(udata, resp);
 		if (rc)
 			goto err_resp;
 	}
@@ -2315,7 +2315,7 @@ int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 			resp.rq_cmb = qp->rq_cmb;
 		}
 
-		rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
+		rc = ib_respond_udata(udata, resp);
 		if (rc)
 			goto err_resp;
 	}
diff --git a/drivers/infiniband/hw/irdma/verbs.c b/drivers/infiniband/hw/irdma/verbs.c
index 17086048d2d7f..79e72a457e798 100644
--- a/drivers/infiniband/hw/irdma/verbs.c
+++ b/drivers/infiniband/hw/irdma/verbs.c
@@ -325,9 +325,9 @@ static int irdma_alloc_ucontext(struct ib_ucontext *uctx,
 		uresp.max_pds = iwdev->rf->sc_dev.hw_attrs.max_hw_pds;
 		uresp.wq_size = iwdev->rf->sc_dev.hw_attrs.max_qp_wr * 2;
 		uresp.kernel_ver = req.userspace_ver;
-		if (ib_copy_to_udata(udata, &uresp,
-				     min(sizeof(uresp), udata->outlen)))
-			return -EFAULT;
+		ret = ib_respond_udata(udata, uresp);
+		if (ret)
+			return ret;
 	} else {
 		u64 bar_off = (uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
 
@@ -354,10 +354,10 @@ static int irdma_alloc_ucontext(struct ib_ucontext *uctx,
 		uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
 		uresp.max_hw_srq_quanta = uk_attrs->max_hw_srq_quanta;
 		uresp.comp_mask |= IRDMA_ALLOC_UCTX_MAX_HW_SRQ_QUANTA;
-		if (ib_copy_to_udata(udata, &uresp,
-				     min(sizeof(uresp), udata->outlen))) {
+		ret = ib_respond_udata(udata, uresp);
+		if (ret) {
 			rdma_user_mmap_entry_remove(ucontext->db_mmap_entry);
-			return -EFAULT;
+			return ret;
 		}
 	}
 
@@ -420,11 +420,9 @@ static int irdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 						  ibucontext);
 		irdma_sc_pd_init(dev, sc_pd, pd_id, ucontext->abi_ver);
 		uresp.pd_id = pd_id;
-		if (ib_copy_to_udata(udata, &uresp,
-				     min(sizeof(uresp), udata->outlen))) {
-			err = -EFAULT;
+		err = ib_respond_udata(udata, uresp);
+		if (err)
 			goto error;
-		}
 	} else {
 		irdma_sc_pd_init(dev, sc_pd, pd_id, IRDMA_ABI_VER);
 	}
@@ -1124,10 +1122,8 @@ static int irdma_create_qp(struct ib_qp *ibqp,
 		uresp.qp_id = qp_num;
 		uresp.qp_caps = qp->qp_uk.qp_caps;
 
-		err_code = ib_copy_to_udata(udata, &uresp,
-					    min(sizeof(uresp), udata->outlen));
+		err_code = ib_respond_udata(udata, uresp);
 		if (err_code) {
-			ibdev_dbg(&iwdev->ibdev, "VERBS: copy_to_udata failed\n");
 			irdma_destroy_qp(&iwqp->ibqp, udata);
 			return err_code;
 		}
@@ -1612,12 +1608,9 @@ int irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 				uresp.push_valid = 1;
 				uresp.push_offset = iwqp->sc_qp.push_offset;
 			}
-			ret = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp),
-					       udata->outlen));
+			ret = ib_respond_udata(udata, uresp);
 			if (ret) {
 				irdma_remove_push_mmap_entries(iwqp);
-				ibdev_dbg(&iwdev->ibdev,
-					  "VERBS: copy_to_udata failed\n");
 				return ret;
 			}
 		}
@@ -1860,12 +1853,9 @@ int irdma_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
 			uresp.push_offset = iwqp->sc_qp.push_offset;
 		}
 
-		err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp),
-				       udata->outlen));
+		err = ib_respond_udata(udata, uresp);
 		if (err) {
 			irdma_remove_push_mmap_entries(iwqp);
-			ibdev_dbg(&iwdev->ibdev,
-				  "VERBS: copy_to_udata failed\n");
 			return err;
 		}
 	}
@@ -2418,11 +2408,9 @@ static int irdma_create_srq(struct ib_srq *ibsrq,
 
 		resp.srq_id = iwsrq->srq_num;
 		resp.srq_size = ukinfo->srq_size;
-		if (ib_copy_to_udata(udata, &resp,
-				     min(sizeof(resp), udata->outlen))) {
-			err_code = -EPROTO;
+		err_code = ib_respond_udata(udata, resp);
+		if (err_code)
 			goto srq_destroy;
-		}
 	}
 
 	return 0;
@@ -2664,13 +2652,9 @@ static int irdma_create_cq(struct ib_cq *ibcq,
 
 		resp.cq_id = info.cq_uk_init_info.cq_id;
 		resp.cq_size = info.cq_uk_init_info.cq_size;
-		if (ib_copy_to_udata(udata, &resp,
-				     min(sizeof(resp), udata->outlen))) {
-			ibdev_dbg(&iwdev->ibdev,
-				  "VERBS: copy to user data\n");
-			err_code = -EPROTO;
+		err_code = ib_respond_udata(udata, resp);
+		if (err_code)
 			goto cq_destroy;
-		}
 	}
 
 	init_completion(&iwcq->free_cq);
@@ -5330,7 +5314,7 @@ static int irdma_create_user_ah(struct ib_ah *ibah,
 	mutex_unlock(&iwdev->rf->ah_tbl_lock);
 
 	uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
-	err = ib_copy_to_udata(udata, &uresp, min(sizeof(uresp), udata->outlen));
+	err = ib_respond_udata(udata, uresp);
 	if (err)
 		irdma_destroy_ah(ibah, attr->flags);
 
diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index 2d682428ef202..f2547989f4229 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -79,11 +79,9 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 
 	if (udata) {
 		resp.cqid = cq->queue.id;
-		err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
-		if (err) {
-			ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
+		err = ib_respond_udata(udata, resp);
+		if (err)
 			goto err_remove_cq_cb;
-		}
 	}
 
 	spin_lock_init(&cq->cq_lock);
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index ac5e75dd3494d..afc2fc124fee3 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -606,6 +606,7 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
 	if (mana_ib_is_rnic(dev)) {
 		props->gid_tbl_len = 16;
 		props->ip_gids = true;
+		props->max_msg_sz = SZ_16M;
 		if (port == 1)
 			props->port_cap_flags = IB_PORT_CM_SUP;
 	}
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index 0fbcf449c134b..39d9cdcc5df45 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -212,13 +212,9 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
 	if (ret)
 		goto fail;
 
-	ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
-	if (ret) {
-		ibdev_dbg(&mdev->ib_dev,
-			  "Failed to copy to udata create rss-qp, %d\n",
-			  ret);
+	ret = ib_respond_udata(udata, resp);
+	if (ret)
 		goto err_disable_vport_rx;
-	}
 
 	kfree(mana_ind_table);
 
@@ -353,13 +349,9 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
 	resp.cqid = send_cq->queue.id;
 	resp.tx_vp_offset = pd->tx_vp_offset;
 
-	err = ib_copy_to_udata(udata, &resp, sizeof(resp));
-	if (err) {
-		ibdev_dbg(&mdev->ib_dev,
-			  "Failed copy udata for create qp-raw, %d\n",
-			  err);
+	err = ib_respond_udata(udata, resp);
+	if (err)
 		goto err_remove_cq_cb;
-	}
 
 	return 0;
 
@@ -557,11 +549,9 @@ static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
 			resp.queue_id[j] = qp->rc_qp.queues[i].id;
 			j++;
 		}
-		err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
-		if (err) {
-			ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
+		err = ib_respond_udata(udata, resp);
+		if (err)
 			goto destroy_qp;
-		}
 	}
 
 	err = mana_table_store_qp(mdev, qp);
diff --git a/drivers/infiniband/hw/mlx4/alias_GUID.c b/drivers/infiniband/hw/mlx4/alias_GUID.c
index c6e1b9e4122a6..fc23960cf1fd0 100644
--- a/drivers/infiniband/hw/mlx4/alias_GUID.c
+++ b/drivers/infiniband/hw/mlx4/alias_GUID.c
@@ -437,7 +437,7 @@ static void aliasguid_query_handler(int status,
 		queue_delayed_work(dev->sriov.alias_guid.ports_guid[port_index].wq,
 				   &dev->sriov.alias_guid.ports_guid[port_index].
 				   alias_guid_work,
-				   msecs_to_jiffies(resched_delay_sec * 1000));
+				   secs_to_jiffies(resched_delay_sec));
 	}
 	if (cb_ctx->sa_query) {
 		list_del(&cb_ctx->list);
diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c
index 7a6eb602d4a6d..7e4505f6c78b3 100644
--- a/drivers/infiniband/hw/mlx4/cq.c
+++ b/drivers/infiniband/hw/mlx4/cq.c
@@ -142,6 +142,7 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
 {
 	struct ib_udata *udata = &attrs->driver_udata;
 	struct ib_device *ibdev = ibcq->device;
+	struct mlx4_ib_create_cq_resp uresp = {};
 	int entries = attr->cqe;
 	int vector = attr->comp_vector;
 	struct mlx4_ib_dev *dev = to_mdev(ibdev);
@@ -219,10 +220,10 @@ int mlx4_ib_create_user_cq(struct ib_cq *ibcq,
 	cq->mcq.event = mlx4_ib_cq_event;
 	cq->mcq.usage = MLX4_RES_USAGE_USER_VERBS;
 
-	if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
-		err = -EFAULT;
+	uresp.cqn = cq->mcq.cqn;
+	err = ib_respond_udata(udata, uresp);
+	if (err)
 		goto err_cq_free;
-	}
 
 	return 0;
 
diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 464c9ab425163..d50743f090bf2 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -626,7 +626,7 @@ static int mlx4_ib_query_device(struct ib_device *ibdev,
 	}
 
 	if (uhw->outlen) {
-		err = ib_copy_to_udata(uhw, &resp, resp.response_length);
+		err = ib_respond_udata(uhw, resp);
 		if (err)
 			goto out;
 	}
@@ -1090,8 +1090,8 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,
 	struct ib_device *ibdev = uctx->device;
 	struct mlx4_ib_dev *dev = to_mdev(ibdev);
 	struct mlx4_ib_ucontext *context = to_mucontext(uctx);
-	struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3;
-	struct mlx4_ib_alloc_ucontext_resp resp;
+	struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3 = {};
+	struct mlx4_ib_alloc_ucontext_resp resp = {};
 	int err;
 
 	if (!dev->ib_active)
@@ -1121,16 +1121,16 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,
 	mutex_init(&context->wqn_ranges_mutex);
 
 	if (ibdev->ops.uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)
-		err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3));
+		err = ib_respond_udata(udata, resp_v3);
 	else
-		err = ib_copy_to_udata(udata, &resp, sizeof(resp));
+		err = ib_respond_udata(udata, resp);
 
 	if (err) {
 		mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar);
-		return -EFAULT;
+		return err;
 	}
 
-	return err;
+	return 0;
 }
 
 static void mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
@@ -1199,9 +1199,14 @@ static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	if (err)
 		return err;
 
-	if (udata && ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) {
-		mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
-		return -EFAULT;
+	if (udata) {
+		struct mlx4_ib_alloc_pd_resp uresp = { .pdn = pd->pdn };
+
+		err = ib_respond_udata(udata, uresp);
+		if (err) {
+			mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
+			return err;
+		}
 	}
 	return 0;
 }
@@ -1696,9 +1701,9 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
 	    (flow_attr->type != IB_FLOW_ATTR_NORMAL))
 		return ERR_PTR(-EOPNOTSUPP);
 
-	if (udata &&
-	    udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen))
-		return ERR_PTR(-EOPNOTSUPP);
+	err = ib_is_udata_in_empty(udata);
+	if (err)
+		return ERR_PTR(err);
 
 	memset(type, 0, sizeof(type));
 
diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c
index 790be09d985a1..8dc4196218bf0 100644
--- a/drivers/infiniband/hw/mlx4/qp.c
+++ b/drivers/infiniband/hw/mlx4/qp.c
@@ -4297,10 +4297,9 @@ int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table,
 	size_t min_resp_len;
 	int i, err = 0;
 
-	if (udata->inlen > 0 &&
-	    !ib_is_udata_cleared(udata, 0,
-				 udata->inlen))
-		return -EOPNOTSUPP;
+	err = ib_is_udata_in_empty(udata);
+	if (err)
+		return err;
 
 	min_resp_len = offsetof(typeof(resp), reserved) + sizeof(resp.reserved);
 	if (udata->outlen && udata->outlen < min_resp_len)
@@ -4332,7 +4331,7 @@ int mlx4_ib_create_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table,
 	if (udata->outlen) {
 		resp.response_length = offsetof(typeof(resp), response_length) +
 					sizeof(resp.response_length);
-		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		err = ib_respond_udata(udata, resp);
 	}
 
 	return err;
diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c
index 767840736d583..dd868f9b893d7 100644
--- a/drivers/infiniband/hw/mlx4/srq.c
+++ b/drivers/infiniband/hw/mlx4/srq.c
@@ -191,11 +191,15 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq,
 	srq->msrq.event = mlx4_ib_srq_event;
 	srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
 
-	if (udata)
-		if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) {
-			err = -EFAULT;
+	if (udata) {
+		struct mlx4_ib_create_srq_resp uresp = {
+			.srqn = srq->msrq.srqn
+		};
+
+		err = ib_respond_udata(udata, uresp);
+		if (err)
 			goto err_srq;
-		}
+	}
 
 	init_attr->attr.max_wr = srq->msrq.max - 1;
 
diff --git a/drivers/infiniband/hw/mlx5/ah.c b/drivers/infiniband/hw/mlx5/ah.c
index 531a57f9ee7e8..a3aa700d08355 100644
--- a/drivers/infiniband/hw/mlx5/ah.c
+++ b/drivers/infiniband/hw/mlx5/ah.c
@@ -121,7 +121,7 @@ int mlx5_ib_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 		resp.response_length = min_resp_len;
 
 		memcpy(resp.dmac, ah_attr->roce.dmac, ETH_ALEN);
-		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		err = ib_respond_udata(udata, resp);
 		if (err)
 			return err;
 	}
diff --git a/drivers/infiniband/hw/mlx5/counters.c b/drivers/infiniband/hw/mlx5/counters.c
index 5b4482dd62743..5a79e834ddea0 100644
--- a/drivers/infiniband/hw/mlx5/counters.c
+++ b/drivers/infiniband/hw/mlx5/counters.c
@@ -697,7 +697,7 @@ static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
 				  u32 port_num)
 {
 	bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
-			port_num != MLX5_VPORT_PF;
+			port_num != MLX5_VPORT_HOST_PF;
 	const struct mlx5_ib_counter *names;
 	int j = 0, i, size;
 
@@ -802,7 +802,7 @@ static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
 				    struct mlx5_ib_counters *cnts, u32 port_num)
 {
 	bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
-			port_num != MLX5_VPORT_PF;
+			port_num != MLX5_VPORT_HOST_PF;
 	u32 num_counters, num_op_counters = 0, size;
 
 	size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :
diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c
index a76b7a36087d9..64fad78cb256d 100644
--- a/drivers/infiniband/hw/mlx5/cq.c
+++ b/drivers/infiniband/hw/mlx5/cq.c
@@ -169,7 +169,8 @@ enum {
 static void handle_responder(struct ib_wc *wc, struct mlx5_cqe64 *cqe,
 			     struct mlx5_ib_qp *qp)
 {
-	enum rdma_link_layer ll = rdma_port_get_link_layer(qp->ibqp.device, 1);
+	enum rdma_link_layer ll =
+		rdma_port_get_link_layer(qp->ibqp.device, qp->port);
 	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
 	struct mlx5_ib_srq *srq = NULL;
 	struct mlx5_ib_wq *wq;
@@ -949,6 +950,7 @@ int mlx5_ib_create_user_cq(struct ib_cq *ibcq,
 {
 	struct ib_udata *udata = &attrs->driver_udata;
 	struct ib_device *ibdev = ibcq->device;
+	struct mlx5_ib_create_cq_resp uresp = {};
 	int entries = attr->cqe;
 	int vector = attr->comp_vector;
 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
@@ -1015,10 +1017,10 @@ int mlx5_ib_create_user_cq(struct ib_cq *ibcq,
 
 	INIT_LIST_HEAD(&cq->wc_list);
 
-	if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
-		err = -EFAULT;
+	uresp.cqn = cq->mcq.cqn;
+	err = ib_respond_udata(udata, uresp);
+	if (err)
 		goto err_cmd;
-	}
 
 	kvfree(cqb);
 	return 0;
diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c
index 645ebcc0832d7..c5929d023ee38 100644
--- a/drivers/infiniband/hw/mlx5/devx.c
+++ b/drivers/infiniband/hw/mlx5/devx.c
@@ -1913,6 +1913,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
 	return err;
 }
 
+static bool devx_key_in_sub_list(struct list_head *list, u32 key_level1)
+{
+	struct devx_event_subscription *s;
+
+	list_for_each_entry(s, list, event_list)
+		if (s->xa_key_level1 == key_level1)
+			return true;
+
+	return false;
+}
+
 static void
 subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
 			   u32 key_level1,
@@ -2160,10 +2171,17 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
 
 		event_sub = kzalloc_obj(*event_sub);
 		if (!event_sub) {
+			if (!devx_key_in_sub_list(&sub_list, key_level1))
+				subscribe_event_xa_dealloc(devx_event_table,
+							   key_level1,
+							   obj,
+							   obj_id);
 			err = -ENOMEM;
 			goto err;
 		}
 
+		event_sub->ev_file = ev_file;
+		event_sub->xa_key_level1 = key_level1;
 		list_add_tail(&event_sub->event_list, &sub_list);
 		uverbs_uobject_get(&ev_file->uobj);
 		if (use_eventfd) {
@@ -2178,9 +2196,6 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
 		}
 
 		event_sub->cookie = cookie;
-		event_sub->ev_file = ev_file;
-		/* May be needed upon cleanup the devx object/subscription */
-		event_sub->xa_key_level1 = key_level1;
 		event_sub->xa_key_level2 = obj_id;
 		INIT_LIST_HEAD(&event_sub->obj_list);
 	}
@@ -2225,10 +2240,11 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
 	list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
 		list_del(&event_sub->event_list);
 
-		subscribe_event_xa_dealloc(devx_event_table,
-					   event_sub->xa_key_level1,
-					   obj,
-					   obj_id);
+		if (!devx_key_in_sub_list(&sub_list, event_sub->xa_key_level1))
+			subscribe_event_xa_dealloc(devx_event_table,
+						   event_sub->xa_key_level1,
+						   obj,
+						   obj_id);
 
 		if (event_sub->eventfd)
 			eventfd_ctx_put(event_sub->eventfd);
@@ -2512,7 +2528,7 @@ static int deliver_event(struct devx_event_subscription *event_sub,
 			 const void *data)
 {
 	struct devx_async_event_file *ev_file;
-	struct devx_async_event_data *event_data;
+	struct devx_async_event_data *event_data, *to_free;
 	unsigned long flags;
 
 	ev_file = event_sub->ev_file;
@@ -2543,12 +2559,17 @@ static int deliver_event(struct devx_event_subscription *event_sub,
 	event_data->hdr.cookie = event_sub->cookie;
 	memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
 
+	to_free = NULL;
+
 	spin_lock_irqsave(&ev_file->lock, flags);
 	if (!ev_file->is_destroyed)
 		list_add_tail(&event_data->list, &ev_file->event_list);
 	else
-		kfree(event_data);
+		to_free = event_data;
 	spin_unlock_irqrestore(&ev_file->lock, flags);
+
+	kfree(to_free);
+
 	wake_up_interruptible(&ev_file->poll_wait);
 
 	return 0;
@@ -2942,6 +2963,7 @@ static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
 			     uobj);
 	struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
 	struct devx_async_data *entry, *tmp;
+	LIST_HEAD(tmp_list);
 
 	spin_lock_irq(&ev_queue->lock);
 	ev_queue->is_destroyed = 1;
@@ -2951,12 +2973,15 @@ static void devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
 	mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
 
 	spin_lock_irq(&comp_ev_file->ev_queue.lock);
-	list_for_each_entry_safe(entry, tmp,
-				 &comp_ev_file->ev_queue.event_list, list) {
+	/* Move all entries to a temporary list and free them outside lock */
+	list_splice_init(&comp_ev_file->ev_queue.event_list, &tmp_list);
+	spin_unlock_irq(&comp_ev_file->ev_queue.lock);
+
+	/* Free memory outside of critical section */
+	list_for_each_entry_safe(entry, tmp, &tmp_list, list) {
 		list_del(&entry->list);
 		kvfree(entry);
 	}
-	spin_unlock_irq(&comp_ev_file->ev_queue.lock);
 };
 
 static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
@@ -2966,7 +2991,9 @@ static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
 		container_of(uobj, struct devx_async_event_file,
 			     uobj);
 	struct devx_event_subscription *event_sub, *event_sub_tmp;
+	struct devx_async_event_data *entry, *tmp;
 	struct mlx5_ib_dev *dev = ev_file->dev;
+	LIST_HEAD(tmp_list);
 
 	spin_lock_irq(&ev_file->lock);
 	ev_file->is_destroyed = 1;
@@ -2980,18 +3007,19 @@ static void devx_async_event_destroy_uobj(struct ib_uobject *uobj,
 			list_del_init(&event_sub->event_list);
 
 	} else {
-		struct devx_async_event_data *entry, *tmp;
-
-		list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
-					 list) {
-			list_del(&entry->list);
-			kfree(entry);
-		}
+		/* Move all entries to a temporary list */
+		list_splice_init(&ev_file->event_list, &tmp_list);
 	}
 
 	spin_unlock_irq(&ev_file->lock);
 	wake_up_interruptible(&ev_file->poll_wait);
 
+	/* Free event data outside of critical section */
+	list_for_each_entry_safe(entry, tmp, &tmp_list, list) {
+		list_del(&entry->list);
+		kfree(entry);
+	}
+
 	mutex_lock(&dev->devx_event_table.event_xa_lock);
 	/* delete the subscriptions which are related to this FD */
 	list_for_each_entry_safe(event_sub, event_sub_tmp,
diff --git a/drivers/infiniband/hw/mlx5/ib_rep.c b/drivers/infiniband/hw/mlx5/ib_rep.c
index 1709b628702ee..65d8767d1830c 100644
--- a/drivers/infiniband/hw/mlx5/ib_rep.c
+++ b/drivers/infiniband/hw/mlx5/ib_rep.c
@@ -262,9 +262,10 @@ mlx5_ib_vport_rep_unload(struct mlx5_eswitch_rep *rep)
 			struct mlx5_core_dev *peer_mdev;
 			struct mlx5_eswitch *esw;
 
+			/* Called while the master E-Switch reps_lock is held. */
 			mlx5_lag_for_each_peer_mdev(mdev, peer_mdev, i) {
 				esw = peer_mdev->priv.eswitch;
-				mlx5_eswitch_unregister_vport_reps(esw, REP_IB);
+				mlx5_eswitch_unregister_vport_reps_nested(esw, REP_IB);
 			}
 			mlx5_ib_release_transport(mdev);
 		}
@@ -284,9 +285,10 @@ static void mlx5_ib_register_peer_vport_reps(struct mlx5_core_dev *mdev)
 	struct mlx5_eswitch *esw;
 	int i;
 
+	/* Called while the master E-Switch reps_lock is held. */
 	mlx5_lag_for_each_peer_mdev(mdev, peer_mdev, i) {
 		esw = peer_mdev->priv.eswitch;
-		mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_IB);
+		mlx5_eswitch_register_vport_reps_nested(esw, &rep_ops, REP_IB);
 	}
 }
 
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 61078281953d6..419489bd21996 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -965,8 +965,9 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 
 	resp.response_length = resp_len;
 
-	if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
-		return -EINVAL;
+	err = ib_is_udata_in_empty(uhw);
+	if (err)
+		return err;
 
 	memset(props, 0, sizeof(*props));
 	err = mlx5_query_system_image_guid(ibdev,
@@ -1355,7 +1356,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev,
 	}
 
 	if (uhw_outlen) {
-		err = ib_copy_to_udata(uhw, &resp, resp.response_length);
+		err = ib_respond_udata(uhw, resp);
 
 		if (err)
 			return err;
@@ -2069,9 +2070,13 @@ static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
 	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
 	    (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
 	     !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
-		return err;
+		return 0;
+
+	err = mlx5_ib_enable_lb(dev, true, false);
+	if (err)
+		mlx5_cmd_dealloc_transport_domain(dev->mdev, *tdn, uid);
 
-	return mlx5_ib_enable_lb(dev, true, false);
+	return err;
 }
 
 static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
@@ -2280,7 +2285,7 @@ static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
 		goto out_mdev;
 
 	resp.response_length = min(udata->outlen, sizeof(resp));
-	err = ib_copy_to_udata(udata, &resp, resp.response_length);
+	err = ib_respond_udata(udata, resp);
 	if (err)
 		goto out_mdev;
 
@@ -2772,7 +2777,7 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 {
 	struct mlx5_ib_pd *pd = to_mpd(ibpd);
 	struct ib_device *ibdev = ibpd->device;
-	struct mlx5_ib_alloc_pd_resp resp;
+	struct mlx5_ib_alloc_pd_resp resp = {};
 	int err;
 	u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
 	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
@@ -2791,9 +2796,10 @@ static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	pd->uid = uid;
 	if (udata) {
 		resp.pdn = pd->pdn;
-		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
+		err = ib_respond_udata(udata, resp);
+		if (err) {
 			mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
-			return -EFAULT;
+			return err;
 		}
 	}
 
@@ -4488,6 +4494,8 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
 		dev->port[i].roce.last_port_state = IB_PORT_DOWN;
 	}
 
+	mutex_init(&dev->lb.mutex);
+
 	err = mlx5r_cmd_query_special_mkeys(dev);
 	if (err)
 		return err;
@@ -4788,11 +4796,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
 	if (err)
 		return err;
 
-	if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
-	    (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
-	     MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
-		mutex_init(&dev->lb.mutex);
-
 	if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
 			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
 		err = mlx5_ib_init_var_region(dev);
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 3b6da45061a55..d7d8f3ae8b647 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1809,7 +1809,7 @@ int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
 	resp.response_length =
 		min(offsetofend(typeof(resp), response_length), udata->outlen);
 	if (resp.response_length) {
-		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		err = ib_respond_udata(udata, resp);
 		if (err)
 			goto free_mkey;
 	}
diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 8fd05532c09cc..043b12cf7c1fd 100644
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -703,12 +703,8 @@ static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
 static int num_med_bfreg(struct mlx5_ib_dev *dev,
 			 struct mlx5_bfreg_info *bfregi)
 {
-	int n;
-
-	n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
-	    NUM_NON_BLUE_FLAME_BFREGS;
-
-	return n >= 0 ? n : 0;
+	return max(0, max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
+			      NUM_NON_BLUE_FLAME_BFREGS);
 }
 
 static int first_med_bfreg(struct mlx5_ib_dev *dev,
@@ -3332,7 +3328,7 @@ int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
 		 * including MLX5_IB_QPT_DCT, which doesn't need it.
 		 * In that case, resp will be filled with zeros.
 		 */
-		err = ib_copy_to_udata(udata, &params.resp, params.outlen);
+		err = ib_respond_udata(udata, params.resp);
 	if (err)
 		goto destroy_qp;
 
@@ -4631,7 +4627,7 @@ static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		resp.dctn = qp->dct.mdct.mqp.qpn;
 		if (MLX5_CAP_GEN(dev->mdev, ece_support))
 			resp.ece_options = MLX5_GET(create_dct_out, out, ece);
-		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		err = ib_respond_udata(udata, resp);
 		if (err) {
 			mlx5_core_destroy_dct(dev, &qp->dct.mdct);
 			return err;
@@ -4790,7 +4786,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	if (!err && resp.response_length &&
 	    udata->outlen >= resp.response_length)
 		/* Return -EFAULT to the user and expect him to destroy QP. */
-		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		err = ib_respond_udata(udata, resp);
 
 out:
 	mutex_unlock(&qp->mutex);
@@ -5490,7 +5486,7 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
 	if (udata->outlen) {
 		resp.response_length = offsetofend(
 			struct mlx5_ib_create_wq_resp, response_length);
-		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		err = ib_respond_udata(udata, resp);
 		if (err)
 			goto err_copy;
 	}
@@ -5538,10 +5534,9 @@ int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
 	u32 *in;
 	void *rqtc;
 
-	if (udata->inlen > 0 &&
-	    !ib_is_udata_cleared(udata, 0,
-				 udata->inlen))
-		return -EOPNOTSUPP;
+	err = ib_is_udata_in_empty(udata);
+	if (err)
+		return err;
 
 	if (init_attr->log_ind_tbl_size >
 	    MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) {
@@ -5582,7 +5577,7 @@ int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
 		resp.response_length =
 			offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp,
 				    response_length);
-		err = ib_copy_to_udata(udata, &resp, resp.response_length);
+		err = ib_respond_udata(udata, resp);
 		if (err)
 			goto err_copy;
 	}
diff --git a/drivers/infiniband/hw/mlx5/srq.c b/drivers/infiniband/hw/mlx5/srq.c
index 852f6f502d14d..3fb8519a4ce0d 100644
--- a/drivers/infiniband/hw/mlx5/srq.c
+++ b/drivers/infiniband/hw/mlx5/srq.c
@@ -292,12 +292,9 @@ int mlx5_ib_create_srq(struct ib_srq *ib_srq,
 			.srqn = srq->msrq.srqn,
 		};
 
-		if (ib_copy_to_udata(udata, &resp, min(udata->outlen,
-				     sizeof(resp)))) {
-			mlx5_ib_dbg(dev, "copy to user failed\n");
-			err = -EFAULT;
+		err = ib_respond_udata(udata, resp);
+		if (err)
 			goto err_core;
-		}
 	}
 
 	init_attr->attr.max_wr = srq->msrq.max - 1;
diff --git a/drivers/infiniband/hw/mlx5/umr.c b/drivers/infiniband/hw/mlx5/umr.c
index f2139474be375..688d5246f284e 100644
--- a/drivers/infiniband/hw/mlx5/umr.c
+++ b/drivers/infiniband/hw/mlx5/umr.c
@@ -915,7 +915,7 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
 		 */
 		err = mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
 		if (err)
-			return err;
+			break;
 		dma_sync_single_for_device(ddev, sg.addr, sg.length,
 					   DMA_TO_DEVICE);
 		sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT);
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index e8d5d865c1f1f..afa97d3801f78 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -311,10 +311,11 @@ static int mthca_alloc_ucontext(struct ib_ucontext *uctx,
 		return err;
 	}
 
-	if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
+	err = ib_respond_udata(udata, uresp);
+	if (err) {
 		mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab);
 		mthca_uar_free(to_mdev(ibdev), &context->uar);
-		return -EFAULT;
+		return err;
 	}
 
 	context->reg_mr_warned = 0;
@@ -356,9 +357,12 @@ static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 		return err;
 
 	if (udata) {
-		if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) {
+		struct mthca_alloc_pd_resp uresp = { .pdn = pd->pd_num };
+
+		err = ib_respond_udata(udata, uresp);
+		if (err) {
 			mthca_pd_free(to_mdev(ibdev), pd);
-			return -EFAULT;
+			return err;
 		}
 	}
 
@@ -427,11 +431,17 @@ static int mthca_create_srq(struct ib_srq *ibsrq,
 	if (err)
 		return err;
 
-	if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) {
-		mthca_free_srq(to_mdev(ibsrq->device), srq);
-		mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar,
-				    context->db_tab, ucmd.db_index);
-		return -EFAULT;
+	if (context) {
+		struct mthca_create_srq_resp uresp = { .srqn = srq->srqn };
+
+		err = ib_respond_udata(udata, uresp);
+		if (err) {
+			mthca_free_srq(to_mdev(ibsrq->device), srq);
+			mthca_unmap_user_db(to_mdev(ibsrq->device),
+					    &context->uar, context->db_tab,
+					    ucmd.db_index);
+			return err;
+		}
 	}
 
 	return 0;
@@ -630,10 +640,14 @@ static int mthca_create_cq(struct ib_cq *ibcq,
 	if (err)
 		goto err_unmap_arm;
 
-	if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) {
-		mthca_free_cq(to_mdev(ibdev), cq);
-		err = -EFAULT;
-		goto err_unmap_arm;
+	if (udata) {
+		struct mthca_create_cq_resp uresp = { .cqn = cq->cqn };
+
+		err = ib_respond_udata(udata, uresp);
+		if (err) {
+			mthca_free_cq(to_mdev(ibdev), cq);
+			goto err_unmap_arm;
+		}
 	}
 
 	cq->resize_buf = NULL;
diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
index a88cc5d84af82..383f1d9c15d15 100644
--- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
+++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
@@ -502,7 +502,7 @@ int ocrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 	resp.dpp_wqe_size = dev->attr.wqe_size;
 
 	memcpy(resp.fw_ver, dev->attr.fw_ver, sizeof(resp.fw_ver));
-	status = ib_copy_to_udata(udata, &resp, sizeof(resp));
+	status = ib_respond_udata(udata, resp);
 	if (status)
 		goto cpy_err;
 	return 0;
@@ -586,11 +586,10 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 	u64 db_page_addr;
 	u64 dpp_page_addr = 0;
 	u32 db_page_size;
-	struct ocrdma_alloc_pd_uresp rsp;
+	struct ocrdma_alloc_pd_uresp rsp = {};
 	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 		udata, struct ocrdma_ucontext, ibucontext);
 
-	memset(&rsp, 0, sizeof(rsp));
 	rsp.id = pd->id;
 	rsp.dpp_enabled = pd->dpp_enabled;
 	db_page_addr = ocrdma_get_db_addr(dev, pd->id);
@@ -611,7 +610,7 @@ static int ocrdma_copy_pd_uresp(struct ocrdma_dev *dev, struct ocrdma_pd *pd,
 		rsp.dpp_page_addr_lo = dpp_page_addr;
 	}
 
-	status = ib_copy_to_udata(udata, &rsp, sizeof(rsp));
+	status = ib_respond_udata(udata, rsp);
 	if (status)
 		goto ucopy_err;
 
@@ -930,13 +929,12 @@ static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
 	int status;
 	struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context(
 		udata, struct ocrdma_ucontext, ibucontext);
-	struct ocrdma_create_cq_uresp uresp;
+	struct ocrdma_create_cq_uresp uresp = {};
 
 	/* this must be user flow! */
 	if (!udata)
 		return -EINVAL;
 
-	memset(&uresp, 0, sizeof(uresp));
 	uresp.cq_id = cq->id;
 	uresp.page_size = PAGE_ALIGN(cq->len);
 	uresp.num_pages = 1;
@@ -945,12 +943,9 @@ static int ocrdma_copy_cq_uresp(struct ocrdma_dev *dev, struct ocrdma_cq *cq,
 	uresp.db_page_addr =  ocrdma_get_db_addr(dev, uctx->cntxt_pd->id);
 	uresp.db_page_size = dev->nic_info.db_page_size;
 	uresp.phase_change = cq->phase_change ? 1 : 0;
-	status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
-	if (status) {
-		pr_err("%s(%d) copy error cqid=0x%x.\n",
-		       __func__, dev->id, cq->id);
+	status = ib_respond_udata(udata, uresp);
+	if (status)
 		goto err;
-	}
 	status = ocrdma_add_mmap(uctx, uresp.db_page_addr, uresp.db_page_size);
 	if (status)
 		goto err;
@@ -1176,11 +1171,10 @@ static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
 {
 	int status;
 	u64 usr_db;
-	struct ocrdma_create_qp_uresp uresp;
+	struct ocrdma_create_qp_uresp uresp = {};
 	struct ocrdma_pd *pd = qp->pd;
 	struct ocrdma_dev *dev = get_ocrdma_dev(pd->ibpd.device);
 
-	memset(&uresp, 0, sizeof(uresp));
 	usr_db = dev->nic_info.unmapped_db +
 			(pd->id * dev->nic_info.db_page_size);
 	uresp.qp_id = qp->id;
@@ -1206,11 +1200,9 @@ static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
 		uresp.dpp_credit = dpp_credit_lmt;
 		uresp.dpp_offset = dpp_offset;
 	}
-	status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
-	if (status) {
-		pr_err("%s(%d) user copy error.\n", __func__, dev->id);
+	status = ib_respond_udata(udata, uresp);
+	if (status)
 		goto err;
-	}
 	status = ocrdma_add_mmap(pd->uctx, uresp.sq_page_addr[0],
 				 uresp.sq_page_size);
 	if (status)
@@ -1735,9 +1727,8 @@ static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
 				struct ib_udata *udata)
 {
 	int status;
-	struct ocrdma_create_srq_uresp uresp;
+	struct ocrdma_create_srq_uresp uresp = {};
 
-	memset(&uresp, 0, sizeof(uresp));
 	uresp.rq_dbid = srq->rq.dbid;
 	uresp.num_rq_pages = 1;
 	uresp.rq_page_addr[0] = virt_to_phys(srq->rq.va);
@@ -1754,7 +1745,7 @@ static int ocrdma_copy_srq_uresp(struct ocrdma_dev *dev, struct ocrdma_srq *srq,
 		uresp.db_shift = 16;
 	}
 
-	status = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+	status = ib_respond_udata(udata, uresp);
 	if (status)
 		return status;
 	status = ocrdma_add_mmap(srq->pd->uctx, uresp.rq_page_addr[0],
diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c
index 679aa6f3a63bc..1af908275ca72 100644
--- a/drivers/infiniband/hw/qedr/verbs.c
+++ b/drivers/infiniband/hw/qedr/verbs.c
@@ -64,14 +64,6 @@ enum {
 	QEDR_USER_MMAP_PHYS_PAGE,
 };
 
-static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
-					size_t len)
-{
-	size_t min_len = min_t(size_t, len, udata->outlen);
-
-	return ib_copy_to_udata(udata, src, min_len);
-}
-
 int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
 {
 	if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
@@ -340,7 +332,7 @@ int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 	uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 	uresp.max_cqes = QEDR_MAX_CQES;
 
-	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+	rc = ib_respond_udata(udata, uresp);
 	if (rc)
 		goto err;
 
@@ -459,9 +451,8 @@ int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 		struct qedr_ucontext *context = rdma_udata_to_drv_context(
 			udata, struct qedr_ucontext, ibucontext);
 
-		rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+		rc = ib_respond_udata(udata, uresp);
 		if (rc) {
-			DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 			dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 			return rc;
 		}
@@ -696,14 +687,10 @@ static void qedr_db_recovery_del(struct qedr_dev *dev,
 	dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
 }
 
-static int qedr_copy_cq_uresp(struct qedr_dev *dev,
-			      struct qedr_cq *cq, struct ib_udata *udata,
+static int qedr_copy_cq_uresp(struct qedr_cq *cq, struct ib_udata *udata,
 			      u32 db_offset)
 {
-	struct qedr_create_cq_uresp uresp;
-	int rc;
-
-	memset(&uresp, 0, sizeof(uresp));
+	struct qedr_create_cq_uresp uresp = {};
 
 	uresp.db_offset = db_offset;
 	uresp.icid = cq->icid;
@@ -711,11 +698,7 @@ static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 		uresp.db_rec_addr =
 			rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
 
-	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
-	if (rc)
-		DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
-
-	return rc;
+	return ib_respond_udata(udata, uresp);
 }
 
 static void consume_cqe(struct qedr_cq *cq)
@@ -994,7 +977,7 @@ int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 	spin_lock_init(&cq->cq_lock);
 
 	if (udata) {
-		rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
+		rc = qedr_copy_cq_uresp(cq, udata, db_offset);
 		if (rc)
 			goto err2;
 
@@ -1251,15 +1234,10 @@ static int qedr_copy_srq_uresp(struct qedr_dev *dev,
 			       struct qedr_srq *srq, struct ib_udata *udata)
 {
 	struct qedr_create_srq_uresp uresp = {};
-	int rc;
 
 	uresp.srq_id = srq->srq_id;
 
-	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
-	if (rc)
-		DP_ERR(dev, "create srq: problem copying data to user space\n");
-
-	return rc;
+	return ib_respond_udata(udata, uresp);
 }
 
 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
@@ -1303,10 +1281,6 @@ static int qedr_copy_qp_uresp(struct qedr_dev *dev,
 			      struct qedr_qp *qp, struct ib_udata *udata,
 			      struct qedr_create_qp_uresp *uresp)
 {
-	int rc;
-
-	memset(uresp, 0, sizeof(*uresp));
-
 	if (qedr_qp_has_sq(qp))
 		qedr_copy_sq_uresp(dev, uresp, qp);
 
@@ -1316,13 +1290,7 @@ static int qedr_copy_qp_uresp(struct qedr_dev *dev,
 	uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
 	uresp->qp_id = qp->qp_id;
 
-	rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
-	if (rc)
-		DP_ERR(dev,
-		       "create qp: failed a copy to user space with qp icid=0x%x.\n",
-		       qp->icid);
-
-	return rc;
+	return ib_respond_udata(udata, *uresp);
 }
 
 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
index 615de9c4209bf..261f18a836854 100644
--- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
+++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c
@@ -82,17 +82,13 @@ static void usnic_ib_fw_string_to_u64(char *fw_ver_str, u64 *fw_ver)
 static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp,
 					struct ib_udata *udata)
 {
-	struct usnic_ib_dev *us_ibdev;
-	struct usnic_ib_create_qp_resp resp;
+	struct usnic_ib_create_qp_resp resp = {};
 	struct pci_dev *pdev;
 	struct vnic_dev_bar *bar;
 	struct usnic_vnic_res_chunk *chunk;
 	struct usnic_ib_qp_grp_flow *default_flow;
 	int i, err;
 
-	memset(&resp, 0, sizeof(resp));
-
-	us_ibdev = qp_grp->vf->pf;
 	pdev = usnic_vnic_get_pdev(qp_grp->vf->vnic);
 	if (!pdev) {
 		usnic_err("Failed to get pdev of qp_grp %d\n",
@@ -157,12 +153,9 @@ static int usnic_ib_fill_create_qp_resp(struct usnic_ib_qp_grp *qp_grp,
 					struct usnic_ib_qp_grp_flow, link);
 	resp.transport = default_flow->trans_type;
 
-	err = ib_copy_to_udata(udata, &resp, sizeof(resp));
-	if (err) {
-		usnic_err("Failed to copy udata for %s",
-			  dev_name(&us_ibdev->ib_dev.dev));
+	err = ib_respond_udata(udata, resp);
+	if (err)
 		return err;
-	}
 
 	return 0;
 }
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
index bc3adcc1ae67c..d5bfdbfe1376d 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c
@@ -203,11 +203,10 @@ int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 		cq->uar = &context->uar;
 
 		/* Copy udata back. */
-		if (ib_copy_to_udata(udata, &cq_resp, sizeof(cq_resp))) {
-			dev_warn(&dev->pdev->dev,
-				 "failed to copy back udata\n");
+		ret = ib_respond_udata(udata, cq_resp);
+		if (ret) {
 			pvrdma_destroy_cq(&cq->ibcq, udata);
-			return -EINVAL;
+			return ret;
 		}
 	}
 
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
index 16aab967a2030..cefcb243c3a6f 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c
@@ -406,12 +406,10 @@ int pvrdma_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 		qp_resp.qpn = qp->ibqp.qp_num;
 		qp_resp.qp_handle = qp->qp_handle;
 
-		if (ib_copy_to_udata(udata, &qp_resp,
-				     min(udata->outlen, sizeof(qp_resp)))) {
-			dev_warn(&dev->pdev->dev,
-				 "failed to copy back udata\n");
+		ret = ib_respond_udata(udata, qp_resp);
+		if (ret) {
 			__pvrdma_destroy_qp(dev, qp);
-			return -EINVAL;
+			return ret;
 		}
 	}
 
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
index d31fb692fcaaf..e69eadde6c26e 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_srq.c
@@ -195,10 +195,10 @@ int pvrdma_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
 	spin_unlock_irqrestore(&dev->srq_tbl_lock, flags);
 
 	/* Copy udata back. */
-	if (ib_copy_to_udata(udata, &srq_resp, sizeof(srq_resp))) {
-		dev_warn(&dev->pdev->dev, "failed to copy back udata\n");
+	ret = ib_respond_udata(udata, srq_resp);
+	if (ret) {
 		pvrdma_destroy_srq(&srq->ibsrq, udata);
-		return -EINVAL;
+		return ret;
 	}
 
 	return 0;
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
index c7c2b41060e52..b9c3202b9545e 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -320,11 +320,11 @@ int pvrdma_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 
 	/* copy back to user */
 	uresp.qp_tab_size = vdev->dsr->caps.max_qp;
-	ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+	ret = ib_respond_udata(udata, uresp);
 	if (ret) {
 		/* pvrdma_dealloc_ucontext() also frees the UAR */
 		pvrdma_dealloc_ucontext(&context->ibucontext);
-		return -EFAULT;
+		return ret;
 	}
 
 	return 0;
@@ -430,11 +430,10 @@ int pvrdma_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 	pd_resp.pdn = resp->pd_handle;
 
 	if (udata) {
-		if (ib_copy_to_udata(udata, &pd_resp, sizeof(pd_resp))) {
-			dev_warn(&dev->pdev->dev,
-				 "failed to copy back protection domain\n");
+		ret = ib_respond_udata(udata, pd_resp);
+		if (ret) {
 			pvrdma_dealloc_pd(&pd->ibpd, udata);
-			return -EFAULT;
+			return ret;
 		}
 	}
 
diff --git a/drivers/infiniband/sw/rdmavt/cq.c b/drivers/infiniband/sw/rdmavt/cq.c
index 30904c6ae852d..45404611c9ce5 100644
--- a/drivers/infiniband/sw/rdmavt/cq.c
+++ b/drivers/infiniband/sw/rdmavt/cq.c
@@ -372,7 +372,7 @@ int rvt_resize_cq(struct ib_cq *ibcq, unsigned int cqe, struct ib_udata *udata)
 	if (udata && udata->outlen >= sizeof(__u64)) {
 		__u64 offset = 0;
 
-		ret = ib_copy_to_udata(udata, &offset, sizeof(offset));
+		ret = ib_respond_udata(udata, offset);
 		if (ret)
 			goto bail_free;
 	}
diff --git a/drivers/infiniband/sw/rdmavt/qp.c b/drivers/infiniband/sw/rdmavt/qp.c
index 816624e0991a0..70e7d08fdce69 100644
--- a/drivers/infiniband/sw/rdmavt/qp.c
+++ b/drivers/infiniband/sw/rdmavt/qp.c
@@ -1192,8 +1192,7 @@ int rvt_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
 		if (!qp->r_rq.wq) {
 			__u64 offset = 0;
 
-			ret = ib_copy_to_udata(udata, &offset,
-					       sizeof(offset));
+			ret = ib_respond_udata(udata, offset);
 			if (ret)
 				goto bail_qpn;
 		} else {
diff --git a/drivers/infiniband/sw/rdmavt/srq.c b/drivers/infiniband/sw/rdmavt/srq.c
index fe125bf85b272..d022aa56c5bfd 100644
--- a/drivers/infiniband/sw/rdmavt/srq.c
+++ b/drivers/infiniband/sw/rdmavt/srq.c
@@ -128,6 +128,7 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 	struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
 	struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
 	struct rvt_rq tmp_rq = {};
+	__u64 offset_addr;
 	int ret = 0;
 
 	if (attr_mask & IB_SRQ_MAX_WR) {
@@ -149,19 +150,17 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 			return -ENOMEM;
 		/* Check that we can write the offset to mmap. */
 		if (udata && udata->inlen >= sizeof(__u64)) {
-			__u64 offset_addr;
 			__u64 offset = 0;
 
 			ret = ib_copy_from_udata(&offset_addr, udata,
 						 sizeof(offset_addr));
 			if (ret)
 				goto bail_free;
-			udata->outbuf = (void __user *)
-					(unsigned long)offset_addr;
-			ret = ib_copy_to_udata(udata, &offset,
-					       sizeof(offset));
-			if (ret)
+			if (copy_to_user(u64_to_user_ptr(offset_addr), &offset,
+					 sizeof(offset))) {
+				ret = -EFAULT;
 				goto bail_free;
+			}
 		}
 
 		spin_lock_irq(&srq->rq.kwq->c_lock);
@@ -236,10 +235,10 @@ int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 			 * See rvt_mmap() for details.
 			 */
 			if (udata && udata->inlen >= sizeof(__u64)) {
-				ret = ib_copy_to_udata(udata, &ip->offset,
-						       sizeof(ip->offset));
-				if (ret)
-					return ret;
+				if (copy_to_user(u64_to_user_ptr(offset_addr),
+						 &ip->offset,
+						 sizeof(ip->offset)))
+					return -EFAULT;
 			}
 
 			/*
diff --git a/drivers/infiniband/sw/rxe/Makefile b/drivers/infiniband/sw/rxe/Makefile
index 3977f4f13258b..e097c1ca1874f 100644
--- a/drivers/infiniband/sw/rxe/Makefile
+++ b/drivers/infiniband/sw/rxe/Makefile
@@ -23,6 +23,7 @@ rdma_rxe-y := \
 	rxe_task.o \
 	rxe_net.o \
 	rxe_hw_counters.o \
+	rxe_mad.o \
 	rxe_ns.o
 
 rdma_rxe-$(CONFIG_INFINIBAND_ON_DEMAND_PAGING) += rxe_odp.o
diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.c b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
index 437917a7d8f2c..17edaa9a9b9b3 100644
--- a/drivers/infiniband/sw/rxe/rxe_hw_counters.c
+++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.c
@@ -22,6 +22,8 @@ static const struct rdma_stat_desc rxe_counter_descs[] = {
 	[RXE_CNT_LINK_DOWNED].name         =  "link_downed",
 	[RXE_CNT_RDMA_SEND].name           =  "rdma_sends",
 	[RXE_CNT_RDMA_RECV].name           =  "rdma_recvs",
+	[RXE_CNT_SENT_BYTES].name          =  "sent_bytes",
+	[RXE_CNT_RCVD_BYTES].name          =  "rcvd_bytes",
 };
 
 int rxe_ib_get_hw_stats(struct ib_device *ibdev,
diff --git a/drivers/infiniband/sw/rxe/rxe_hw_counters.h b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
index 051f9e1c3852c..01b355103cbc3 100644
--- a/drivers/infiniband/sw/rxe/rxe_hw_counters.h
+++ b/drivers/infiniband/sw/rxe/rxe_hw_counters.h
@@ -26,6 +26,8 @@ enum rxe_counters {
 	RXE_CNT_LINK_DOWNED,
 	RXE_CNT_RDMA_SEND,
 	RXE_CNT_RDMA_RECV,
+	RXE_CNT_SENT_BYTES,
+	RXE_CNT_RCVD_BYTES,
 	RXE_NUM_OF_COUNTERS
 };
 
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index e095c12699cb0..64d636bf80fd2 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -242,4 +242,10 @@ static inline int rxe_ib_advise_mr(struct ib_pd *pd,
 
 #endif /* CONFIG_INFINIBAND_ON_DEMAND_PAGING */
 
+/* rxe-mad.c */
+int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
+		    const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+		    const struct ib_mad *in, struct ib_mad *out,
+		    size_t *out_mad_size, u16 *out_mad_pkey_index);
+
 #endif /* RXE_LOC_H */
diff --git a/drivers/infiniband/sw/rxe/rxe_mad.c b/drivers/infiniband/sw/rxe/rxe_mad.c
new file mode 100644
index 0000000000000..cb2d55830f423
--- /dev/null
+++ b/drivers/infiniband/sw/rxe/rxe_mad.c
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/*
+ * Copyright (c) 2026 zhenwei pi <zhenwei.pi@linux.dev>
+ */
+
+#include <rdma/ib_pma.h>
+#include "rxe.h"
+#include "rxe_hw_counters.h"
+
+static int rxe_get_pma_info(struct ib_mad *out)
+{
+	struct ib_class_port_info cpi = {};
+
+	cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH;
+	memcpy((out->data + 40), &cpi, sizeof(cpi));
+
+	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static int rxe_get_pma_counters(struct rxe_dev *rxe, struct ib_mad *out)
+{
+	struct ib_pma_portcounters *pma_cnt = (struct ib_pma_portcounters *)(out->data + 40);
+	s64 val;
+
+	/* IBA release 1.8, 16.1.3.5: During operation, instead of overflowing, they shall stop
+	 * at all ones.
+	 */
+	val = atomic64_read(&rxe->stats_counters[RXE_CNT_LINK_DOWNED]);
+	pma_cnt->link_downed_counter = clamp(val, 0, U8_MAX);
+	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static int rxe_get_pma_counters_ext(struct rxe_dev *rxe, struct ib_mad *out)
+{
+	struct ib_pma_portcounters_ext *pma_cnt_ext =
+		(struct ib_pma_portcounters_ext *)(out->data + 40);
+	s64 val;
+
+	val = atomic64_read(&rxe->stats_counters[RXE_CNT_SENT_BYTES]);
+	pma_cnt_ext->port_xmit_data = cpu_to_be64(val >> 2);
+
+	val = atomic64_read(&rxe->stats_counters[RXE_CNT_RCVD_BYTES]);
+	pma_cnt_ext->port_rcv_data = cpu_to_be64(val >> 2);
+
+	val = atomic64_read(&rxe->stats_counters[RXE_CNT_SENT_PKTS]);
+	pma_cnt_ext->port_xmit_packets = cpu_to_be64(val);
+
+	val = atomic64_read(&rxe->stats_counters[RXE_CNT_RCVD_PKTS]);
+	pma_cnt_ext->port_rcv_packets = cpu_to_be64(val);
+
+	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
+
+static int rxe_get_perf_mgmt(struct rxe_dev *rxe, const struct ib_mad *in, struct ib_mad *out)
+{
+	switch (in->mad_hdr.attr_id) {
+	case IB_PMA_CLASS_PORT_INFO:
+		return rxe_get_pma_info(out);
+
+	case IB_PMA_PORT_COUNTERS:
+		return rxe_get_pma_counters(rxe, out);
+
+	case IB_PMA_PORT_COUNTERS_EXT:
+		return rxe_get_pma_counters_ext(rxe, out);
+
+	default:
+		out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB);
+		return IB_MAD_RESULT_SUCCESS;
+	}
+}
+
+int rxe_process_mad(struct ib_device *ibdev, int mad_flags, u32 port_num,
+		    const struct ib_wc *in_wc, const struct ib_grh *in_grh,
+		    const struct ib_mad *in, struct ib_mad *out,
+		    size_t *out_mad_size, u16 *out_mad_pkey_index)
+{
+	struct rxe_dev *rxe = to_rdev(ibdev);
+	u8 mgmt_class = in->mad_hdr.mgmt_class;
+	u8 method = in->mad_hdr.method;
+
+	if (port_num != 1)
+		return IB_MAD_RESULT_FAILURE;
+
+	memset(out, 0, sizeof(*out));
+	switch (mgmt_class) {
+	case IB_MGMT_CLASS_PERF_MGMT:
+		if (method == IB_MGMT_METHOD_GET)
+			return rxe_get_perf_mgmt(rxe, in, out);
+		break;
+
+	default:
+		out->mad_hdr.status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD);
+		return IB_MAD_RESULT_SUCCESS;
+	}
+
+	return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY;
+}
diff --git a/drivers/infiniband/sw/rxe/rxe_mcast.c b/drivers/infiniband/sw/rxe/rxe_mcast.c
index 5cad72073eca1..acd03bd87794e 100644
--- a/drivers/infiniband/sw/rxe/rxe_mcast.c
+++ b/drivers/infiniband/sw/rxe/rxe_mcast.c
@@ -34,7 +34,7 @@ static int rxe_mcast_add(struct rxe_dev *rxe, union ib_gid *mgid)
 	struct net_device *ndev;
 	int ret;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(&rxe->ib_dev, 1);
 	if (!ndev)
 		return -ENODEV;
 
@@ -59,7 +59,7 @@ static int rxe_mcast_del(struct rxe_dev *rxe, union ib_gid *mgid)
 	struct net_device *ndev;
 	int ret;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(&rxe->ib_dev, 1);
 	if (!ndev)
 		return -ENODEV;
 
diff --git a/drivers/infiniband/sw/rxe/rxe_net.c b/drivers/infiniband/sw/rxe/rxe_net.c
index 50a2cb5405e22..3741b2c4b0bbf 100644
--- a/drivers/infiniband/sw/rxe/rxe_net.c
+++ b/drivers/infiniband/sw/rxe/rxe_net.c
@@ -283,12 +283,12 @@ static struct socket *rxe_setup_udp_tunnel(struct net *net, __be16 port,
 	tnl_cfg.encap_rcv = rxe_udp_encap_recv;
 
 	/* Setup UDP tunnel */
-	setup_udp_tunnel_sock(net, sock, &tnl_cfg);
+	setup_udp_tunnel_sock(net, sock->sk, &tnl_cfg);
 
 	return sock;
 }
 
-static void rxe_release_udp_tunnel(struct socket *sk)
+static void rxe_release_udp_tunnel(struct sock *sk)
 {
 	if (sk)
 		udp_tunnel_sock_release(sk);
@@ -501,6 +501,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
 	int err;
 	int is_request = pkt->mask & RXE_REQ_MASK;
 	struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
+	unsigned int skblen = skb->len;
 	unsigned long flags;
 
 	spin_lock_irqsave(&qp->state_lock, flags);
@@ -524,6 +525,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
 	}
 
 	rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);
+	rxe_counter_add(rxe, RXE_CNT_SENT_BYTES, skblen);
 	goto done;
 
 drop:
@@ -600,7 +602,7 @@ const char *rxe_parent_name(struct rxe_dev *rxe, unsigned int port_num)
 	struct net_device *ndev;
 	char *ndev_name;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(&rxe->ib_dev, 1);
 	if (!ndev)
 		return NULL;
 	ndev_name = ndev->name;
@@ -636,7 +638,7 @@ static void rxe_sock_put(struct sock *sk,
 	if (refcount_read(&sk->sk_refcnt) > SK_REF_FOR_TUNNEL) {
 		__sock_put(sk);
 	} else {
-		rxe_release_udp_tunnel(sk->sk_socket);
+		rxe_release_udp_tunnel(sk);
 		sk = NULL;
 		set_sk(net, sk);
 	}
@@ -644,12 +646,11 @@ static void rxe_sock_put(struct sock *sk,
 
 void rxe_net_del(struct ib_device *dev)
 {
-	struct rxe_dev *rxe = container_of(dev, struct rxe_dev, ib_dev);
 	struct net_device *ndev;
 	struct sock *sk;
 	struct net *net;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(dev, 1);
 	if (!ndev)
 		return;
 
@@ -697,7 +698,7 @@ void rxe_set_port_state(struct rxe_dev *rxe)
 {
 	struct net_device *ndev;
 
-	ndev = rxe_ib_device_get_netdev(&rxe->ib_dev);
+	ndev = ib_device_get_netdev(&rxe->ib_dev, 1);
 	if (!ndev)
 		return;
 
diff --git a/drivers/infiniband/sw/rxe/rxe_ns.c b/drivers/infiniband/sw/rxe/rxe_ns.c
index 8b9d734229b24..64621c89f8bf1 100644
--- a/drivers/infiniband/sw/rxe/rxe_ns.c
+++ b/drivers/infiniband/sw/rxe/rxe_ns.c
@@ -47,7 +47,7 @@ static void rxe_ns_exit(struct net *net)
 	rcu_read_unlock();
 	if (sk) {
 		rcu_assign_pointer(ns_sk->rxe_sk4, NULL);
-		udp_tunnel_sock_release(sk->sk_socket);
+		udp_tunnel_sock_release(sk);
 	}
 
 #if IS_ENABLED(CONFIG_IPV6)
@@ -56,7 +56,7 @@ static void rxe_ns_exit(struct net *net)
 	rcu_read_unlock();
 	if (sk) {
 		rcu_assign_pointer(ns_sk->rxe_sk6, NULL);
-		udp_tunnel_sock_release(sk->sk_socket);
+		udp_tunnel_sock_release(sk);
 	}
 #endif
 }
diff --git a/drivers/infiniband/sw/rxe/rxe_recv.c b/drivers/infiniband/sw/rxe/rxe_recv.c
index 2d5e701ff961a..2c90387aa1266 100644
--- a/drivers/infiniband/sw/rxe/rxe_recv.c
+++ b/drivers/infiniband/sw/rxe/rxe_recv.c
@@ -318,6 +318,7 @@ void rxe_rcv(struct sk_buff *skb)
 	int err;
 	struct rxe_pkt_info *pkt = SKB_TO_PKT(skb);
 	struct rxe_dev *rxe = pkt->rxe;
+	unsigned int skblen = skb->len - skb_network_offset(skb);
 
 	if (unlikely(skb->len < RXE_BTH_BYTES))
 		goto drop;
@@ -353,6 +354,7 @@ void rxe_rcv(struct sk_buff *skb)
 	if (unlikely(err))
 		goto drop;
 
+	rxe_counter_add(rxe, RXE_CNT_RCVD_BYTES, skblen);
 	rxe_counter_inc(rxe, RXE_CNT_RCVD_PKTS);
 
 	if (unlikely(bth_qpn(pkt) == IB_MULTICAST_QPN))
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index 4d4891dc28846..8edd4dd1f031f 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -50,7 +50,7 @@ static int rxe_query_port(struct ib_device *ibdev,
 		goto err_out;
 	}
 
-	ndev = rxe_ib_device_get_netdev(ibdev);
+	ndev = ib_device_get_netdev(ibdev, 1);
 	if (!ndev) {
 		err = -ENODEV;
 		goto err_out;
@@ -1441,7 +1441,7 @@ static int rxe_enable_driver(struct ib_device *ib_dev)
 	struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev);
 	struct net_device *ndev;
 
-	ndev = rxe_ib_device_get_netdev(ib_dev);
+	ndev = ib_device_get_netdev(ib_dev, 1);
 	if (!ndev)
 		return -ENODEV;
 
@@ -1496,6 +1496,7 @@ static const struct ib_device_ops rxe_dev_ops = {
 	.post_recv = rxe_post_recv,
 	.post_send = rxe_post_send,
 	.post_srq_recv = rxe_post_srq_recv,
+	.process_mad = rxe_process_mad,
 	.query_ah = rxe_query_ah,
 	.query_device = rxe_query_device,
 	.query_pkey = rxe_query_pkey,
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index d92f80d16f784..0f5ffd94643f9 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -415,7 +415,6 @@ struct rxe_port {
 	u32			qp_gsi_index;
 };
 
-#define	RXE_PORT	1
 struct rxe_dev {
 	struct ib_device	ib_dev;
 	struct ib_device_attr	attr;
@@ -451,14 +450,15 @@ struct rxe_dev {
 	struct rxe_port		port;
 };
 
-static inline struct net_device *rxe_ib_device_get_netdev(struct ib_device *dev)
+static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
 {
-	return ib_device_get_netdev(dev, RXE_PORT);
+	atomic64_inc(&rxe->stats_counters[index]);
 }
 
-static inline void rxe_counter_inc(struct rxe_dev *rxe, enum rxe_counters index)
+static inline void rxe_counter_add(struct rxe_dev *rxe, enum rxe_counters index,
+				   s64 val)
 {
-	atomic64_inc(&rxe->stats_counters[index]);
+	atomic64_add(val, &rxe->stats_counters[index]);
 }
 
 static inline struct rxe_dev *to_rdev(struct ib_device *dev)
diff --git a/drivers/infiniband/sw/siw/siw.h b/drivers/infiniband/sw/siw/siw.h
index f5fd71717b807..dbc998248b1e9 100644
--- a/drivers/infiniband/sw/siw/siw.h
+++ b/drivers/infiniband/sw/siw/siw.h
@@ -119,9 +119,10 @@ struct siw_page_chunk {
 
 struct siw_umem {
 	struct ib_umem *base_mem;
-	struct siw_page_chunk *page_chunk;
-	int num_pages;
+	unsigned int num_pages;
+	unsigned int num_chunks;
 	u64 fp_addr; /* First page base address */
+	struct siw_page_chunk page_chunk[] __counted_by(num_chunks);
 };
 
 struct siw_pble {
diff --git a/drivers/infiniband/sw/siw/siw_mem.c b/drivers/infiniband/sw/siw/siw_mem.c
index 98c802b3ed723..4df792ebe02fa 100644
--- a/drivers/infiniband/sw/siw/siw_mem.c
+++ b/drivers/infiniband/sw/siw/siw_mem.c
@@ -41,16 +41,14 @@ struct siw_mem *siw_mem_id2obj(struct siw_device *sdev, int stag_index)
 
 void siw_umem_release(struct siw_umem *umem)
 {
-	int i, num_pages = umem->num_pages;
+	unsigned int i, num_chunks = umem->num_chunks;
 
 	if (umem->base_mem)
 		ib_umem_release(umem->base_mem);
 
-	for (i = 0; num_pages > 0; i++) {
+	for (i = 0; i < num_chunks; i++)
 		kfree(umem->page_chunk[i].plist);
-		num_pages -= PAGES_PER_CHUNK;
-	}
-	kfree(umem->page_chunk);
+
 	kfree(umem);
 }
 
@@ -188,7 +186,7 @@ int siw_check_mem(struct ib_pd *pd, struct siw_mem *mem, u64 addr,
  * lookup is being done and mem is not released it check fails.
  */
 int siw_check_sge(struct ib_pd *pd, struct siw_sge *sge, struct siw_mem *mem[],
-		  enum ib_access_flags perms, u32 off, int len)
+		  enum ib_access_flags perms, u32 off, u32 len)
 {
 	struct siw_device *sdev = to_siw_dev(pd->device);
 	struct siw_mem *new = NULL;
@@ -338,25 +336,20 @@ struct siw_umem *siw_umem_get(struct ib_device *base_dev, u64 start,
 	struct sg_page_iter sg_iter;
 	struct sg_table *sgt;
 	u64 first_page_va;
-	int num_pages, num_chunks, i, rv = 0;
+	unsigned int num_pages, num_chunks, i;
+	int rv = 0;
 
 	if (!len)
 		return ERR_PTR(-EINVAL);
 
 	first_page_va = start & PAGE_MASK;
 	num_pages = PAGE_ALIGN(start + len - first_page_va) >> PAGE_SHIFT;
-	num_chunks = (num_pages >> CHUNK_SHIFT) + 1;
+	num_chunks = ((num_pages - 1) >> CHUNK_SHIFT) + 1;
 
-	umem = kzalloc_obj(*umem);
+	umem = kzalloc_flex(*umem, page_chunk, num_chunks);
 	if (!umem)
 		return ERR_PTR(-ENOMEM);
 
-	umem->page_chunk =
-		kzalloc_objs(struct siw_page_chunk, num_chunks);
-	if (!umem->page_chunk) {
-		rv = -ENOMEM;
-		goto err_out;
-	}
 	base_mem = ib_umem_get(base_dev, start, len, rights);
 	if (IS_ERR(base_mem)) {
 		rv = PTR_ERR(base_mem);
@@ -365,33 +358,40 @@ struct siw_umem *siw_umem_get(struct ib_device *base_dev, u64 start,
 	}
 	umem->fp_addr = first_page_va;
 	umem->base_mem = base_mem;
+	umem->num_pages = num_pages;
+	umem->num_chunks = num_chunks;
 
 	sgt = &base_mem->sgt_append.sgt;
 	__sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0);
 
-	if (!__sg_page_iter_next(&sg_iter)) {
-		rv = -EINVAL;
-		goto err_out;
-	}
-	for (i = 0; num_pages > 0; i++) {
-		int nents = min_t(int, num_pages, PAGES_PER_CHUNK);
-		struct page **plist =
-			kzalloc_objs(struct page *, nents);
+	for (i = 0; i < num_chunks; i++) {
+		struct page **plist;
+		unsigned int pix, nents = min(num_pages, PAGES_PER_CHUNK);
 
+		plist = kzalloc_objs(struct page *, nents);
 		if (!plist) {
 			rv = -ENOMEM;
 			goto err_out;
 		}
 		umem->page_chunk[i].plist = plist;
-		while (nents--) {
-			*plist = sg_page_iter_page(&sg_iter);
-			umem->num_pages++;
-			num_pages--;
-			plist++;
+
+		for (pix = 0; pix < nents; pix++) {
 			if (!__sg_page_iter_next(&sg_iter))
 				break;
+			plist[pix] = sg_page_iter_page(&sg_iter);
+			num_pages--;
 		}
 	}
+
+	if (num_pages) {
+		/*
+		 * Unexpected size of sg list provided by ib_umem_get()
+		 */
+		siw_dbg(base_dev, "Short SG list, missing %u pages\n",
+			num_pages);
+		rv = -EINVAL;
+		goto err_out;
+	}
 	return umem;
 err_out:
 	siw_umem_release(umem);
diff --git a/drivers/infiniband/sw/siw/siw_mem.h b/drivers/infiniband/sw/siw/siw_mem.h
index 8e769d30e2acd..e4f3a5a4f81df 100644
--- a/drivers/infiniband/sw/siw/siw_mem.h
+++ b/drivers/infiniband/sw/siw/siw_mem.h
@@ -17,7 +17,7 @@ int siw_check_mem(struct ib_pd *pd, struct siw_mem *mem, u64 addr,
 		  enum ib_access_flags perms, int len);
 int siw_check_sge(struct ib_pd *pd, struct siw_sge *sge,
 		  struct siw_mem *mem[], enum ib_access_flags perms,
-		  u32 off, int len);
+		  u32 off, u32 len);
 void siw_wqe_put_mem(struct siw_wqe *wqe, enum siw_opcode op);
 int siw_mr_add_mem(struct siw_mr *mr, struct ib_pd *pd, void *mem_obj,
 		   u64 start, u64 len, int rights);
@@ -45,7 +45,6 @@ static inline void siw_unref_mem_sgl(struct siw_mem **mem, unsigned int num_sge)
 #define CHUNK_SHIFT 9 /* sets number of pages per chunk */
 #define PAGES_PER_CHUNK (_AC(1, UL) << CHUNK_SHIFT)
 #define CHUNK_MASK (~(PAGES_PER_CHUNK - 1))
-#define PAGE_CHUNK_SIZE (PAGES_PER_CHUNK * sizeof(struct page *))
 
 /*
  * siw_get_upage()
@@ -61,7 +60,7 @@ static inline struct page *siw_get_upage(struct siw_umem *umem, u64 addr)
 		     chunk_idx = page_idx >> CHUNK_SHIFT,
 		     page_in_chunk = page_idx & ~CHUNK_MASK;
 
-	if (likely(page_idx < umem->num_pages))
+	if (page_idx < umem->num_pages)
 		return umem->page_chunk[chunk_idx].plist[page_in_chunk];
 
 	return NULL;
diff --git a/drivers/infiniband/sw/siw/siw_qp_rx.c b/drivers/infiniband/sw/siw/siw_qp_rx.c
index e8a88b378d51d..34d03584160c2 100644
--- a/drivers/infiniband/sw/siw/siw_qp_rx.c
+++ b/drivers/infiniband/sw/siw/siw_qp_rx.c
@@ -1081,6 +1081,21 @@ static int siw_get_hdr(struct siw_rx_stream *srx)
 			return -EAGAIN;
 	}
 
+	/*
+	 * Peer-controlled mpa_len must not underflow srx->fpdu_part_rem
+	 * in siw_tcp_rx_data(); a negative value flows as a signed copy
+	 * length into siw_check_mem() and skb_copy_bits().
+	 */
+	if (unlikely(be16_to_cpu(c_hdr->mpa_len) + MPA_HDR_SIZE <
+		     iwarp_pktinfo[opcode].hdr_len)) {
+		pr_warn_ratelimited("siw: short mpa_len %u for opcode %u (hdr_len %u)\n",
+				    be16_to_cpu(c_hdr->mpa_len), opcode,
+				    iwarp_pktinfo[opcode].hdr_len);
+		siw_init_terminate(rx_qp(srx), TERM_ERROR_LAYER_LLP,
+				   LLP_ETYPE_MPA, LLP_ECODE_FPDU_START, 0);
+		return -EINVAL;
+	}
+
 	/*
 	 * DDP/RDMAP header receive completed. Check if the current
 	 * DDP segment starts a new RDMAP message or continues a previously
diff --git a/drivers/infiniband/sw/siw/siw_verbs.c b/drivers/infiniband/sw/siw/siw_verbs.c
index 1e1d262a4ae2d..b34f3d6547ffc 100644
--- a/drivers/infiniband/sw/siw/siw_verbs.c
+++ b/drivers/infiniband/sw/siw/siw_verbs.c
@@ -102,7 +102,7 @@ int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
 		rv = -EINVAL;
 		goto err_out;
 	}
-	rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+	rv = ib_respond_udata(udata, uresp);
 	if (rv)
 		goto err_out;
 
@@ -472,7 +472,7 @@ int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
 			rv = -EINVAL;
 			goto err_out_xa;
 		}
-		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+		rv = ib_respond_udata(udata, uresp);
 		if (rv)
 			goto err_out_xa;
 	}
@@ -1205,7 +1205,7 @@ int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
 			rv = -EINVAL;
 			goto err_out;
 		}
-		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+		rv = ib_respond_udata(udata, uresp);
 		if (rv)
 			goto err_out;
 	}
@@ -1386,7 +1386,7 @@ struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
 			rv = -EINVAL;
 			goto err_out;
 		}
-		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+		rv = ib_respond_udata(udata, uresp);
 		if (rv)
 			goto err_out;
 	}
@@ -1646,7 +1646,7 @@ int siw_create_srq(struct ib_srq *base_srq,
 			rv = -EINVAL;
 			goto err_out;
 		}
-		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
+		rv = ib_respond_udata(udata, uresp);
 		if (rv)
 			goto err_out;
 	}
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index 4026715677362..3e1e1e8617394 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1297,7 +1297,9 @@ static int ipoib_hard_header(struct sk_buff *skb,
 	return IPOIB_HARD_LEN;
 }
 
-static void ipoib_set_mcast_list(struct net_device *dev)
+static void ipoib_set_rx_mode_async(struct net_device *dev,
+				    struct netdev_hw_addr_list *uc,
+				    struct netdev_hw_addr_list *mc)
 {
 	struct ipoib_dev_priv *priv = ipoib_priv(dev);
 
@@ -2160,7 +2162,7 @@ static const struct net_device_ops ipoib_netdev_ops_pf = {
 	.ndo_fix_features	 = ipoib_fix_features,
 	.ndo_start_xmit		 = ipoib_start_xmit,
 	.ndo_tx_timeout		 = ipoib_timeout,
-	.ndo_set_rx_mode	 = ipoib_set_mcast_list,
+	.ndo_set_rx_mode_async	 = ipoib_set_rx_mode_async,
 	.ndo_get_iflink		 = ipoib_get_iflink,
 	.ndo_set_vf_link_state	 = ipoib_set_vf_link_state,
 	.ndo_get_vf_config	 = ipoib_get_vf_config,
@@ -2183,7 +2185,7 @@ static const struct net_device_ops ipoib_netdev_ops_vf = {
 	.ndo_fix_features	 = ipoib_fix_features,
 	.ndo_start_xmit	 	 = ipoib_start_xmit,
 	.ndo_tx_timeout		 = ipoib_timeout,
-	.ndo_set_rx_mode	 = ipoib_set_mcast_list,
+	.ndo_set_rx_mode_async	 = ipoib_set_rx_mode_async,
 	.ndo_get_iflink		 = ipoib_get_iflink,
 	.ndo_get_stats64	 = ipoib_get_stats,
 	.ndo_eth_ioctl		 = ipoib_ioctl,
diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c
index 51727c7d710c3..9dd9141c86a54 100644
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv-sysfs.c
@@ -295,8 +295,8 @@ int rtrs_srv_create_path_files(struct rtrs_srv_path *srv_path)
 put_kobj:
 	kobject_del(&srv_path->kobj);
 destroy_root:
-	kobject_put(&srv_path->kobj);
 	rtrs_srv_destroy_once_sysfs_root_folders(srv_path);
+	kobject_put(&srv_path->kobj);
 
 	return err;
 }
diff --git a/drivers/infiniband/ulp/srpt/ib_srpt.c b/drivers/infiniband/ulp/srpt/ib_srpt.c
index 9aec5d80117f1..f66cfd70c2636 100644
--- a/drivers/infiniband/ulp/srpt/ib_srpt.c
+++ b/drivers/infiniband/ulp/srpt/ib_srpt.c
@@ -1129,9 +1129,10 @@ static int srpt_get_desc_tbl(struct srpt_recv_ioctx *recv_ioctx,
 		struct srp_imm_buf *imm_buf = srpt_get_desc_buf(srp_cmd);
 		void *data = (void *)srp_cmd + imm_data_offset;
 		uint32_t len = be32_to_cpu(imm_buf->len);
-		uint32_t req_size = imm_data_offset + len;
+		uint32_t req_size;
 
-		if (req_size > srp_max_req_size) {
+		if (check_add_overflow((uint32_t)imm_data_offset, len, &req_size) ||
+		    req_size > srp_max_req_size) {
 			pr_err("Immediate data (length %d + %d) exceeds request size %d\n",
 			       imm_data_offset, len, srp_max_req_size);
 			return -EINVAL;
diff --git a/drivers/input/gameport/fm801-gp.c b/drivers/input/gameport/fm801-gp.c
index 423cccdea34f5..1e8c6c0448446 100644
--- a/drivers/input/gameport/fm801-gp.c
+++ b/drivers/input/gameport/fm801-gp.c
@@ -125,8 +125,8 @@ static void fm801_gp_remove(struct pci_dev *pci)
 }
 
 static const struct pci_device_id fm801_gp_id_table[] = {
-	{ PCI_VENDOR_ID_FORTEMEDIA, PCI_DEVICE_ID_FM801_GP, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0  },
-	{ 0 }
+	{ PCI_VDEVICE(FORTEMEDIA, PCI_DEVICE_ID_FM801_GP) },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, fm801_gp_id_table);
 
diff --git a/drivers/input/joystick/xpad.c b/drivers/input/joystick/xpad.c
index 0549fdc5a9851..feb8f368f834e 100644
--- a/drivers/input/joystick/xpad.c
+++ b/drivers/input/joystick/xpad.c
@@ -186,6 +186,10 @@ static const struct xpad_device {
 	{ 0x07ff, 0xffff, "Mad Catz GamePad", 0, XTYPE_XBOX360 },
 	{ 0x0b05, 0x1a38, "ASUS ROG RAIKIRI", MAP_SHARE_BUTTON, XTYPE_XBOXONE },
 	{ 0x0b05, 0x1abb, "ASUS ROG RAIKIRI PRO", 0, XTYPE_XBOXONE },
+	{ 0x0b05, 0x1c91, "ASUS ROG RAIKIRI II", 0, XTYPE_XBOX360 },
+	{ 0x0b05, 0x1c92, "ASUS ROG RAIKIRI II WIRELESS", 0, XTYPE_XBOX360 },
+	{ 0x0b05, 0x1c96, "ASUS ROG RAIKIRI II XBOX", MAP_SHARE_BUTTON, XTYPE_XBOXONE },
+	{ 0x0b05, 0x1d04, "ASUS ROG RAIKIRI II XBOX WIRELESS", MAP_SHARE_BUTTON, XTYPE_XBOXONE },
 	{ 0x0c12, 0x0005, "Intec wireless", 0, XTYPE_XBOX },
 	{ 0x0c12, 0x8801, "Nyko Xbox Controller", 0, XTYPE_XBOX },
 	{ 0x0c12, 0x8802, "Zeroplus Xbox Controller", 0, XTYPE_XBOX },
@@ -391,6 +395,7 @@ static const struct xpad_device {
 	{ 0x3285, 0x0662, "Nacon Revolution5 Pro", 0, XTYPE_XBOX360 },
 	{ 0x3285, 0x0663, "Nacon Evol-X", 0, XTYPE_XBOXONE },
 	{ 0x3537, 0x1004, "GameSir T4 Kaleid", 0, XTYPE_XBOX360 },
+	{ 0x3537, 0x100f, "GameSir Nova 2 Lite", 0, XTYPE_XBOX360 },
 	{ 0x3537, 0x1010, "GameSir G7 SE", 0, XTYPE_XBOXONE },
 	{ 0x3651, 0x1000, "CRKD SG", 0, XTYPE_XBOX360 },
 	{ 0x366c, 0x0005, "ByoWave Proteus Controller", MAP_SHARE_BUTTON, XTYPE_XBOXONE, FLAG_DELAY_INIT },
@@ -507,6 +512,7 @@ static const struct usb_device_id xpad_table[] = {
 	{ USB_DEVICE(0x0738, 0x4540) },		/* Mad Catz Beat Pad */
 	XPAD_XBOXONE_VENDOR(0x0738),		/* Mad Catz FightStick TE 2 */
 	XPAD_XBOX360_VENDOR(0x07ff),		/* Mad Catz Gamepad */
+	XPAD_XBOX360_VENDOR(0x0b05),		/* ASUS controllers */
 	XPAD_XBOXONE_VENDOR(0x0b05),		/* ASUS controllers */
 	XPAD_XBOX360_VENDOR(0x0c12),		/* Zeroplus X-Box 360 controllers */
 	XPAD_XBOX360_VENDOR(0x0db0),		/* Micro Star International X-Box 360 controllers */
@@ -1077,10 +1083,10 @@ static void xpadone_process_packet(struct usb_xpad *xpad, u16 cmd, unsigned char
 		input_report_key(dev, BTN_START,  data[4] & BIT(2));
 		input_report_key(dev, BTN_SELECT, data[4] & BIT(3));
 		if (xpad->mapping & MAP_SHARE_BUTTON) {
-			if (xpad->mapping & MAP_SHARE_OFFSET)
-				input_report_key(dev, KEY_RECORD, data[len - 26] & BIT(0));
-			else
-				input_report_key(dev, KEY_RECORD, data[len - 18] & BIT(0));
+			u32 offset = (xpad->mapping & MAP_SHARE_OFFSET) ? 26 : 18;
+
+			if (len >= offset)
+				input_report_key(dev, KEY_RECORD, data[len - offset] & BIT(0));
 		}
 
 		/* buttons A,B,X,Y */
diff --git a/drivers/input/keyboard/lm8323.c b/drivers/input/keyboard/lm8323.c
index e19442c6f80f5..d9df10484755d 100644
--- a/drivers/input/keyboard/lm8323.c
+++ b/drivers/input/keyboard/lm8323.c
@@ -259,7 +259,7 @@ static void process_keys(struct lm8323_chip *lm)
 	ret = lm8323_read(lm, LM8323_CMD_READ_FIFO, key_fifo, LM8323_FIFO_LEN);
 
 	if (ret < 0) {
-		dev_err(&lm->client->dev, "Failed reading fifo \n");
+		dev_err(&lm->client->dev, "Failed reading fifo\n");
 		return;
 	}
 	key_fifo[ret] = 0;
diff --git a/drivers/input/misc/Kconfig b/drivers/input/misc/Kconfig
index 94a753fcb64fa..1f6c57dba0308 100644
--- a/drivers/input/misc/Kconfig
+++ b/drivers/input/misc/Kconfig
@@ -745,16 +745,6 @@ config INPUT_WM831X_ON
 	  To compile this driver as a module, choose M here: the module
 	  will be called wm831x_on.
 
-config INPUT_PCAP
-	tristate "Motorola EZX PCAP misc input events"
-	depends on EZX_PCAP
-	help
-	  Say Y here if you want to use Power key and Headphone button
-	  on Motorola EZX phones.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called pcap_keys.
-
 config INPUT_ADXL34X
 	tristate "Analog Devices ADXL34x Three-Axis Digital Accelerometer"
 	default n
diff --git a/drivers/input/misc/Makefile b/drivers/input/misc/Makefile
index 415fc4e2918be..2281d6803fce9 100644
--- a/drivers/input/misc/Makefile
+++ b/drivers/input/misc/Makefile
@@ -60,7 +60,6 @@ obj-$(CONFIG_INPUT_MAX8997_HAPTIC)	+= max8997_haptic.o
 obj-$(CONFIG_INPUT_MC13783_PWRBUTTON)	+= mc13783-pwrbutton.o
 obj-$(CONFIG_INPUT_MMA8450)		+= mma8450.o
 obj-$(CONFIG_INPUT_PALMAS_PWRBUTTON)	+= palmas-pwrbutton.o
-obj-$(CONFIG_INPUT_PCAP)		+= pcap_keys.o
 obj-$(CONFIG_INPUT_PCF8574)		+= pcf8574_keypad.o
 obj-$(CONFIG_INPUT_PCSPKR)		+= pcspkr.o
 obj-$(CONFIG_INPUT_PF1550_ONKEY)	+= pf1550-onkey.o
diff --git a/drivers/input/misc/atlas_btns.c b/drivers/input/misc/atlas_btns.c
index 47b31725e8507..835ad45a9d65c 100644
--- a/drivers/input/misc/atlas_btns.c
+++ b/drivers/input/misc/atlas_btns.c
@@ -60,11 +60,15 @@ static acpi_status acpi_atlas_button_handler(u32 function,
 
 static int atlas_acpi_button_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *device;
 	acpi_status status;
 	int i;
 	int err;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	input_dev = input_allocate_device();
 	if (!input_dev) {
 		pr_err("unable to allocate input device\n");
diff --git a/drivers/input/misc/pcap_keys.c b/drivers/input/misc/pcap_keys.c
deleted file mode 100644
index b19899b50d0ba..0000000000000
--- a/drivers/input/misc/pcap_keys.c
+++ /dev/null
@@ -1,125 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- *  Input driver for PCAP events:
- *   * Power key
- *   * Headphone button
- *
- *  Copyright (c) 2008,2009 Ilya Petrov <ilya.muromec@gmail.com>
- */
-
-#include <linux/module.h>
-#include <linux/interrupt.h>
-#include <linux/platform_device.h>
-#include <linux/input.h>
-#include <linux/mfd/ezx-pcap.h>
-#include <linux/slab.h>
-
-struct pcap_keys {
-	struct pcap_chip *pcap;
-	struct input_dev *input;
-};
-
-/* PCAP2 interrupts us on keypress */
-static irqreturn_t pcap_keys_handler(int irq, void *_pcap_keys)
-{
-	struct pcap_keys *pcap_keys = _pcap_keys;
-	int pirq = irq_to_pcap(pcap_keys->pcap, irq);
-	u32 pstat;
-
-	ezx_pcap_read(pcap_keys->pcap, PCAP_REG_PSTAT, &pstat);
-	pstat &= 1 << pirq;
-
-	switch (pirq) {
-	case PCAP_IRQ_ONOFF:
-		input_report_key(pcap_keys->input, KEY_POWER, !pstat);
-		break;
-	case PCAP_IRQ_MIC:
-		input_report_key(pcap_keys->input, KEY_HP, !pstat);
-		break;
-	}
-
-	input_sync(pcap_keys->input);
-
-	return IRQ_HANDLED;
-}
-
-static int pcap_keys_probe(struct platform_device *pdev)
-{
-	int err = -ENOMEM;
-	struct pcap_keys *pcap_keys;
-	struct input_dev *input_dev;
-
-	pcap_keys = kmalloc_obj(*pcap_keys);
-	if (!pcap_keys)
-		return err;
-
-	pcap_keys->pcap = dev_get_drvdata(pdev->dev.parent);
-
-	input_dev = input_allocate_device();
-	if (!input_dev)
-		goto fail;
-
-	pcap_keys->input = input_dev;
-
-	platform_set_drvdata(pdev, pcap_keys);
-	input_dev->name = pdev->name;
-	input_dev->phys = "pcap-keys/input0";
-	input_dev->id.bustype = BUS_HOST;
-	input_dev->dev.parent = &pdev->dev;
-
-	__set_bit(EV_KEY, input_dev->evbit);
-	__set_bit(KEY_POWER, input_dev->keybit);
-	__set_bit(KEY_HP, input_dev->keybit);
-
-	err = input_register_device(input_dev);
-	if (err)
-		goto fail_allocate;
-
-	err = request_irq(pcap_to_irq(pcap_keys->pcap, PCAP_IRQ_ONOFF),
-			pcap_keys_handler, 0, "Power key", pcap_keys);
-	if (err)
-		goto fail_register;
-
-	err = request_irq(pcap_to_irq(pcap_keys->pcap, PCAP_IRQ_MIC),
-			pcap_keys_handler, 0, "Headphone button", pcap_keys);
-	if (err)
-		goto fail_pwrkey;
-
-	return 0;
-
-fail_pwrkey:
-	free_irq(pcap_to_irq(pcap_keys->pcap, PCAP_IRQ_ONOFF), pcap_keys);
-fail_register:
-	input_unregister_device(input_dev);
-	goto fail;
-fail_allocate:
-	input_free_device(input_dev);
-fail:
-	kfree(pcap_keys);
-	return err;
-}
-
-static void pcap_keys_remove(struct platform_device *pdev)
-{
-	struct pcap_keys *pcap_keys = platform_get_drvdata(pdev);
-
-	free_irq(pcap_to_irq(pcap_keys->pcap, PCAP_IRQ_ONOFF), pcap_keys);
-	free_irq(pcap_to_irq(pcap_keys->pcap, PCAP_IRQ_MIC), pcap_keys);
-
-	input_unregister_device(pcap_keys->input);
-	kfree(pcap_keys);
-}
-
-static struct platform_driver pcap_keys_device_driver = {
-	.probe		= pcap_keys_probe,
-	.remove		= pcap_keys_remove,
-	.driver		= {
-		.name	= "pcap-keys",
-	}
-};
-module_platform_driver(pcap_keys_device_driver);
-
-MODULE_DESCRIPTION("Motorola PCAP2 input events driver");
-MODULE_AUTHOR("Ilya Petrov <ilya.muromec@gmail.com>");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:pcap_keys");
diff --git a/drivers/input/mouse/elan_i2c_core.c b/drivers/input/mouse/elan_i2c_core.c
index fee1796da3d08..5cba02a156ce4 100644
--- a/drivers/input/mouse/elan_i2c_core.c
+++ b/drivers/input/mouse/elan_i2c_core.c
@@ -162,6 +162,9 @@ static int elan_get_fwinfo(u16 ic_type, u8 iap_version, u16 *validpage_count,
 	case 0x15:
 		*validpage_count = 1024;
 		break;
+	case 0x19:
+		*validpage_count = 2032;
+		break;
 	default:
 		/* unknown ic type clear value */
 		*validpage_count = 0;
@@ -645,6 +648,11 @@ static ssize_t elan_sysfs_update_fw(struct device *dev,
 		return error;
 	}
 
+	if (fw->size < data->fw_signature_address + sizeof(signature)) {
+		dev_err(dev, "firmware file too small\n");
+		return -EBADF;
+	}
+
 	/* Firmware file must match signature data */
 	fw_signature = &fw->data[data->fw_signature_address];
 	if (memcmp(fw_signature, signature, sizeof(signature)) != 0) {
diff --git a/drivers/input/mouse/elan_i2c_i2c.c b/drivers/input/mouse/elan_i2c_i2c.c
index a9057d124a888..88d4070d4b44f 100644
--- a/drivers/input/mouse/elan_i2c_i2c.c
+++ b/drivers/input/mouse/elan_i2c_i2c.c
@@ -690,7 +690,7 @@ static int elan_i2c_finish_fw_update(struct i2c_client *client,
 	if (error) {
 		dev_err(dev, "device reset failed: %d\n", error);
 	} else if (!wait_for_completion_timeout(completion,
-						msecs_to_jiffies(300))) {
+						msecs_to_jiffies(700))) {
 		dev_err(dev, "timeout waiting for device reset\n");
 		error = -ETIMEDOUT;
 	}
diff --git a/drivers/input/mouse/synaptics.c b/drivers/input/mouse/synaptics.c
index 26071128f43a7..c70502e24031a 100644
--- a/drivers/input/mouse/synaptics.c
+++ b/drivers/input/mouse/synaptics.c
@@ -190,6 +190,7 @@ static const char * const smbus_pnp_ids[] = {
 	"LEN2044", /* L470  */
 	"LEN2054", /* E480 */
 	"LEN2055", /* E580 */
+	"LEN2058", /* E490 */
 	"LEN2068", /* T14 Gen 1 */
 	"SYN1221", /* TUXEDO InfinityBook Pro 14 v5 */
 	"SYN3003", /* HP EliteBook 850 G1 */
diff --git a/drivers/input/touchscreen/Kconfig b/drivers/input/touchscreen/Kconfig
index aeaf9a9cbb416..484522d8d675a 100644
--- a/drivers/input/touchscreen/Kconfig
+++ b/drivers/input/touchscreen/Kconfig
@@ -1178,16 +1178,6 @@ config TOUCHSCREEN_TSC2007_IIO
 	  or ambient light monitoring), temperature and raw input
 	  values.
 
-config TOUCHSCREEN_PCAP
-	tristate "Motorola PCAP touchscreen"
-	depends on EZX_PCAP
-	help
-	  Say Y here if you have a Motorola EZX telephone and
-	  want to enable support for the built-in touchscreen.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called pcap_ts.
-
 config TOUCHSCREEN_RM_TS
 	tristate "Raydium I2C Touchscreen"
 	depends on I2C
diff --git a/drivers/input/touchscreen/Makefile b/drivers/input/touchscreen/Makefile
index f2b002abebe8a..6d397268d2e3a 100644
--- a/drivers/input/touchscreen/Makefile
+++ b/drivers/input/touchscreen/Makefile
@@ -74,7 +74,6 @@ obj-$(CONFIG_TOUCHSCREEN_HP7XX)		+= jornada720_ts.o
 obj-$(CONFIG_TOUCHSCREEN_IPAQ_MICRO)	+= ipaq-micro-ts.o
 obj-$(CONFIG_TOUCHSCREEN_HTCPEN)	+= htcpen.o
 obj-$(CONFIG_TOUCHSCREEN_USB_COMPOSITE)	+= usbtouchscreen.o
-obj-$(CONFIG_TOUCHSCREEN_PCAP)		+= pcap_ts.o
 obj-$(CONFIG_TOUCHSCREEN_PENMOUNT)	+= penmount.o
 obj-$(CONFIG_TOUCHSCREEN_PIXCIR)	+= pixcir_i2c_ts.o
 obj-$(CONFIG_TOUCHSCREEN_RM_TS)		+= raydium_i2c_ts.o
diff --git a/drivers/input/touchscreen/atmel_mxt_ts.c b/drivers/input/touchscreen/atmel_mxt_ts.c
index 87c6a10381f2d..04d3b12490f4a 100644
--- a/drivers/input/touchscreen/atmel_mxt_ts.c
+++ b/drivers/input/touchscreen/atmel_mxt_ts.c
@@ -275,8 +275,8 @@ struct mxt_cfg {
 	off_t raw_pos;
 
 	u8 *mem;
-	size_t mem_size;
-	int start_ofs;
+	u16 mem_size;
+	u16 start_ofs;
 
 	struct mxt_info info;
 };
@@ -1397,7 +1397,8 @@ static int mxt_prepare_cfg_mem(struct mxt_data *data, struct mxt_cfg *cfg)
 {
 	struct device *dev = &data->client->dev;
 	struct mxt_object *object;
-	unsigned int type, instance, size, byte_offset;
+	unsigned int type, instance, size;
+	int byte_offset;
 	int offset;
 	int ret;
 	int i;
@@ -1473,7 +1474,7 @@ static int mxt_prepare_cfg_mem(struct mxt_data *data, struct mxt_cfg *cfg)
 			}
 			cfg->raw_pos += offset;
 
-			if (i > mxt_obj_size(object))
+			if (i >= mxt_obj_size(object))
 				continue;
 
 			byte_offset = reg + i - cfg->start_ofs;
@@ -1627,6 +1628,13 @@ static int mxt_update_cfg(struct mxt_data *data, const struct firmware *fw)
 	cfg.start_ofs = MXT_OBJECT_START +
 			data->info->object_num * sizeof(struct mxt_object) +
 			MXT_INFO_CHECKSUM_SIZE;
+
+	if (data->mem_size <= cfg.start_ofs) {
+		dev_err(dev, "Memory size too small: %u < %u\n",
+			data->mem_size, cfg.start_ofs);
+		return -EINVAL;
+	}
+
 	cfg.mem_size = data->mem_size - cfg.start_ofs;
 
 	u8 *mem_buf __free(kfree) = cfg.mem = kzalloc(cfg.mem_size, GFP_KERNEL);
@@ -2833,14 +2841,12 @@ static ssize_t mxt_object_show(struct device *dev,
 	int count = 0;
 	int i, j;
 	int error;
-	u8 *obuf;
 
 	/* Pre-allocate buffer large enough to hold max sized object. */
-	obuf = kmalloc(256, GFP_KERNEL);
+	u8 *obuf __free(kfree) = kmalloc(256, GFP_KERNEL);
 	if (!obuf)
 		return -ENOMEM;
 
-	error = 0;
 	for (i = 0; i < data->info->object_num; i++) {
 		object = data->object_table + i;
 
@@ -2855,15 +2861,13 @@ static ssize_t mxt_object_show(struct device *dev,
 
 			error = __mxt_read_reg(data->client, addr, size, obuf);
 			if (error)
-				goto done;
+				return error;
 
 			count = mxt_show_instance(buf, count, object, j, obuf);
 		}
 	}
 
-done:
-	kfree(obuf);
-	return error ?: count;
+	return count;
 }
 
 static int mxt_check_firmware_format(struct device *dev,
diff --git a/drivers/input/touchscreen/pcap_ts.c b/drivers/input/touchscreen/pcap_ts.c
deleted file mode 100644
index 7b89eb74b9de7..0000000000000
--- a/drivers/input/touchscreen/pcap_ts.c
+++ /dev/null
@@ -1,252 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Driver for Motorola PCAP2 touchscreen as found in the EZX phone platform.
- *
- *  Copyright (C) 2006 Harald Welte <laforge@openezx.org>
- *  Copyright (C) 2009 Daniel Ribeiro <drwyrm@gmail.com>
- */
-
-#include <linux/module.h>
-#include <linux/fs.h>
-#include <linux/string.h>
-#include <linux/slab.h>
-#include <linux/pm.h>
-#include <linux/timer.h>
-#include <linux/interrupt.h>
-#include <linux/platform_device.h>
-#include <linux/input.h>
-#include <linux/mfd/ezx-pcap.h>
-
-struct pcap_ts {
-	struct pcap_chip *pcap;
-	struct input_dev *input;
-	struct delayed_work work;
-	u16 x, y;
-	u16 pressure;
-	u8 read_state;
-};
-
-#define SAMPLE_DELAY	20 /* msecs */
-
-#define X_AXIS_MIN	0
-#define X_AXIS_MAX	1023
-#define Y_AXIS_MAX	X_AXIS_MAX
-#define Y_AXIS_MIN	X_AXIS_MIN
-#define PRESSURE_MAX	X_AXIS_MAX
-#define PRESSURE_MIN	X_AXIS_MIN
-
-static void pcap_ts_read_xy(void *data, u16 res[2])
-{
-	struct pcap_ts *pcap_ts = data;
-
-	switch (pcap_ts->read_state) {
-	case PCAP_ADC_TS_M_PRESSURE:
-		/* pressure reading is unreliable */
-		if (res[0] > PRESSURE_MIN && res[0] < PRESSURE_MAX)
-			pcap_ts->pressure = res[0];
-		pcap_ts->read_state = PCAP_ADC_TS_M_XY;
-		schedule_delayed_work(&pcap_ts->work, 0);
-		break;
-	case PCAP_ADC_TS_M_XY:
-		pcap_ts->y = res[0];
-		pcap_ts->x = res[1];
-		if (pcap_ts->x <= X_AXIS_MIN || pcap_ts->x >= X_AXIS_MAX ||
-		    pcap_ts->y <= Y_AXIS_MIN || pcap_ts->y >= Y_AXIS_MAX) {
-			/* pen has been released */
-			input_report_abs(pcap_ts->input, ABS_PRESSURE, 0);
-			input_report_key(pcap_ts->input, BTN_TOUCH, 0);
-
-			pcap_ts->read_state = PCAP_ADC_TS_M_STANDBY;
-			schedule_delayed_work(&pcap_ts->work, 0);
-		} else {
-			/* pen is touching the screen */
-			input_report_abs(pcap_ts->input, ABS_X, pcap_ts->x);
-			input_report_abs(pcap_ts->input, ABS_Y, pcap_ts->y);
-			input_report_key(pcap_ts->input, BTN_TOUCH, 1);
-			input_report_abs(pcap_ts->input, ABS_PRESSURE,
-						pcap_ts->pressure);
-
-			/* switch back to pressure read mode */
-			pcap_ts->read_state = PCAP_ADC_TS_M_PRESSURE;
-			schedule_delayed_work(&pcap_ts->work,
-					msecs_to_jiffies(SAMPLE_DELAY));
-		}
-		input_sync(pcap_ts->input);
-		break;
-	default:
-		dev_warn(&pcap_ts->input->dev,
-				"pcap_ts: Warning, unhandled read_state %d\n",
-				pcap_ts->read_state);
-		break;
-	}
-}
-
-static void pcap_ts_work(struct work_struct *work)
-{
-	struct delayed_work *dw = to_delayed_work(work);
-	struct pcap_ts *pcap_ts = container_of(dw, struct pcap_ts, work);
-	u8 ch[2];
-
-	pcap_set_ts_bits(pcap_ts->pcap,
-			pcap_ts->read_state << PCAP_ADC_TS_M_SHIFT);
-
-	if (pcap_ts->read_state == PCAP_ADC_TS_M_STANDBY)
-		return;
-
-	/* start adc conversion */
-	ch[0] = PCAP_ADC_CH_TS_X1;
-	ch[1] = PCAP_ADC_CH_TS_Y1;
-	pcap_adc_async(pcap_ts->pcap, PCAP_ADC_BANK_1, 0, ch,
-						pcap_ts_read_xy, pcap_ts);
-}
-
-static irqreturn_t pcap_ts_event_touch(int pirq, void *data)
-{
-	struct pcap_ts *pcap_ts = data;
-
-	if (pcap_ts->read_state == PCAP_ADC_TS_M_STANDBY) {
-		pcap_ts->read_state = PCAP_ADC_TS_M_PRESSURE;
-		schedule_delayed_work(&pcap_ts->work, 0);
-	}
-	return IRQ_HANDLED;
-}
-
-static int pcap_ts_open(struct input_dev *dev)
-{
-	struct pcap_ts *pcap_ts = input_get_drvdata(dev);
-
-	pcap_ts->read_state = PCAP_ADC_TS_M_STANDBY;
-	schedule_delayed_work(&pcap_ts->work, 0);
-
-	return 0;
-}
-
-static void pcap_ts_close(struct input_dev *dev)
-{
-	struct pcap_ts *pcap_ts = input_get_drvdata(dev);
-
-	cancel_delayed_work_sync(&pcap_ts->work);
-
-	pcap_ts->read_state = PCAP_ADC_TS_M_NONTS;
-	pcap_set_ts_bits(pcap_ts->pcap,
-				pcap_ts->read_state << PCAP_ADC_TS_M_SHIFT);
-}
-
-static int pcap_ts_probe(struct platform_device *pdev)
-{
-	struct input_dev *input_dev;
-	struct pcap_ts *pcap_ts;
-	int err = -ENOMEM;
-
-	pcap_ts = kzalloc_obj(*pcap_ts);
-	if (!pcap_ts)
-		return err;
-
-	pcap_ts->pcap = dev_get_drvdata(pdev->dev.parent);
-	platform_set_drvdata(pdev, pcap_ts);
-
-	input_dev = input_allocate_device();
-	if (!input_dev)
-		goto fail;
-
-	INIT_DELAYED_WORK(&pcap_ts->work, pcap_ts_work);
-
-	pcap_ts->read_state = PCAP_ADC_TS_M_NONTS;
-	pcap_set_ts_bits(pcap_ts->pcap,
-				pcap_ts->read_state << PCAP_ADC_TS_M_SHIFT);
-
-	pcap_ts->input = input_dev;
-	input_set_drvdata(input_dev, pcap_ts);
-
-	input_dev->name = "pcap-touchscreen";
-	input_dev->phys = "pcap_ts/input0";
-	input_dev->id.bustype = BUS_HOST;
-	input_dev->id.vendor = 0x0001;
-	input_dev->id.product = 0x0002;
-	input_dev->id.version = 0x0100;
-	input_dev->dev.parent = &pdev->dev;
-	input_dev->open = pcap_ts_open;
-	input_dev->close = pcap_ts_close;
-
-	input_dev->evbit[0] = BIT_MASK(EV_KEY) | BIT_MASK(EV_ABS);
-	input_dev->keybit[BIT_WORD(BTN_TOUCH)] = BIT_MASK(BTN_TOUCH);
-	input_set_abs_params(input_dev, ABS_X, X_AXIS_MIN, X_AXIS_MAX, 0, 0);
-	input_set_abs_params(input_dev, ABS_Y, Y_AXIS_MIN, Y_AXIS_MAX, 0, 0);
-	input_set_abs_params(input_dev, ABS_PRESSURE, PRESSURE_MIN,
-			     PRESSURE_MAX, 0, 0);
-
-	err = input_register_device(pcap_ts->input);
-	if (err)
-		goto fail_allocate;
-
-	err = request_irq(pcap_to_irq(pcap_ts->pcap, PCAP_IRQ_TS),
-			pcap_ts_event_touch, 0, "Touch Screen", pcap_ts);
-	if (err)
-		goto fail_register;
-
-	return 0;
-
-fail_register:
-	input_unregister_device(input_dev);
-	goto fail;
-fail_allocate:
-	input_free_device(input_dev);
-fail:
-	kfree(pcap_ts);
-
-	return err;
-}
-
-static void pcap_ts_remove(struct platform_device *pdev)
-{
-	struct pcap_ts *pcap_ts = platform_get_drvdata(pdev);
-
-	free_irq(pcap_to_irq(pcap_ts->pcap, PCAP_IRQ_TS), pcap_ts);
-	cancel_delayed_work_sync(&pcap_ts->work);
-
-	input_unregister_device(pcap_ts->input);
-
-	kfree(pcap_ts);
-}
-
-#ifdef CONFIG_PM
-static int pcap_ts_suspend(struct device *dev)
-{
-	struct pcap_ts *pcap_ts = dev_get_drvdata(dev);
-
-	pcap_set_ts_bits(pcap_ts->pcap, PCAP_ADC_TS_REF_LOWPWR);
-	return 0;
-}
-
-static int pcap_ts_resume(struct device *dev)
-{
-	struct pcap_ts *pcap_ts = dev_get_drvdata(dev);
-
-	pcap_set_ts_bits(pcap_ts->pcap,
-				pcap_ts->read_state << PCAP_ADC_TS_M_SHIFT);
-	return 0;
-}
-
-static const struct dev_pm_ops pcap_ts_pm_ops = {
-	.suspend	= pcap_ts_suspend,
-	.resume		= pcap_ts_resume,
-};
-#define PCAP_TS_PM_OPS (&pcap_ts_pm_ops)
-#else
-#define PCAP_TS_PM_OPS NULL
-#endif
-
-static struct platform_driver pcap_ts_driver = {
-	.probe		= pcap_ts_probe,
-	.remove		= pcap_ts_remove,
-	.driver		= {
-		.name	= "pcap-ts",
-		.pm	= PCAP_TS_PM_OPS,
-	},
-};
-module_platform_driver(pcap_ts_driver);
-
-MODULE_DESCRIPTION("Motorola PCAP2 touchscreen driver");
-MODULE_AUTHOR("Daniel Ribeiro / Harald Welte");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS("platform:pcap_ts");
diff --git a/drivers/input/touchscreen/stmfts.c b/drivers/input/touchscreen/stmfts.c
index 8af87d0b6eb64..454f7e822e9cb 100644
--- a/drivers/input/touchscreen/stmfts.c
+++ b/drivers/input/touchscreen/stmfts.c
@@ -5,6 +5,7 @@
 // Copyright (c) 2017 Andi Shyti <andi@etezian.org>
 
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/input/mt.h>
 #include <linux/input/touchscreen.h>
@@ -69,20 +70,21 @@
 #define STMFTS_MAX_FINGERS	10
 #define STMFTS_DEV_NAME		"stmfts"
 
-enum stmfts_regulators {
-	STMFTS_REGULATOR_VDD,
-	STMFTS_REGULATOR_AVDD,
+static const struct regulator_bulk_data stmfts_supplies[] = {
+	{ .supply = "vdd" },
+	{ .supply = "avdd" },
 };
 
 struct stmfts_data {
 	struct i2c_client *client;
 	struct input_dev *input;
+	struct gpio_desc *reset_gpio;
 	struct led_classdev led_cdev;
 	struct mutex mutex;
 
 	struct touchscreen_properties prop;
 
-	struct regulator_bulk_data regulators[2];
+	struct regulator_bulk_data *supplies;
 
 	/*
 	 * Presence of ledvdd will be used also to check
@@ -107,7 +109,7 @@ struct stmfts_data {
 };
 
 static int stmfts_brightness_set(struct led_classdev *led_cdev,
-					enum led_brightness value)
+				 enum led_brightness value)
 {
 	struct stmfts_data *sdata = container_of(led_cdev,
 					struct stmfts_data, led_cdev);
@@ -250,7 +252,6 @@ static void stmfts_parse_events(struct stmfts_data *sdata)
 		u8 *event = &sdata->data[i * STMFTS_EVENT_SIZE];
 
 		switch (event[0]) {
-
 		case STMFTS_EV_CONTROLLER_READY:
 		case STMFTS_EV_SLEEP_OUT_CONTROLLER_READY:
 		case STMFTS_EV_STATUS:
@@ -263,7 +264,6 @@ static void stmfts_parse_events(struct stmfts_data *sdata)
 		}
 
 		switch (event[0] & STMFTS_MASK_EVENT_ID) {
-
 		case STMFTS_EV_MULTI_TOUCH_ENTER:
 		case STMFTS_EV_MULTI_TOUCH_MOTION:
 			stmfts_report_contact_event(sdata, event);
@@ -285,9 +285,9 @@ static void stmfts_parse_events(struct stmfts_data *sdata)
 
 		case STMFTS_EV_ERROR:
 			dev_warn(&sdata->client->dev,
-					"error code: 0x%x%x%x%x%x%x",
-					event[6], event[5], event[4],
-					event[3], event[2], event[1]);
+				 "error code: 0x%x%x%x%x%x%x",
+				 event[6], event[5], event[4],
+				 event[3], event[2], event[1]);
 			break;
 
 		default:
@@ -404,7 +404,7 @@ static void stmfts_input_close(struct input_dev *dev)
 }
 
 static ssize_t stmfts_sysfs_chip_id(struct device *dev,
-				struct device_attribute *attr, char *buf)
+				    struct device_attribute *attr, char *buf)
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
@@ -412,7 +412,8 @@ static ssize_t stmfts_sysfs_chip_id(struct device *dev,
 }
 
 static ssize_t stmfts_sysfs_chip_version(struct device *dev,
-				struct device_attribute *attr, char *buf)
+					 struct device_attribute *attr,
+					 char *buf)
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
@@ -420,7 +421,7 @@ static ssize_t stmfts_sysfs_chip_version(struct device *dev,
 }
 
 static ssize_t stmfts_sysfs_fw_ver(struct device *dev,
-				struct device_attribute *attr, char *buf)
+				   struct device_attribute *attr, char *buf)
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
@@ -428,7 +429,7 @@ static ssize_t stmfts_sysfs_fw_ver(struct device *dev,
 }
 
 static ssize_t stmfts_sysfs_config_id(struct device *dev,
-				struct device_attribute *attr, char *buf)
+				      struct device_attribute *attr, char *buf)
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
@@ -436,7 +437,8 @@ static ssize_t stmfts_sysfs_config_id(struct device *dev,
 }
 
 static ssize_t stmfts_sysfs_config_version(struct device *dev,
-				struct device_attribute *attr, char *buf)
+					   struct device_attribute *attr,
+					   char *buf)
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
@@ -444,7 +446,8 @@ static ssize_t stmfts_sysfs_config_version(struct device *dev,
 }
 
 static ssize_t stmfts_sysfs_read_status(struct device *dev,
-				struct device_attribute *attr, char *buf)
+					struct device_attribute *attr,
+					char *buf)
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 	u8 status[4];
@@ -459,7 +462,8 @@ static ssize_t stmfts_sysfs_read_status(struct device *dev,
 }
 
 static ssize_t stmfts_sysfs_hover_enable_read(struct device *dev,
-				struct device_attribute *attr, char *buf)
+					      struct device_attribute *attr,
+					      char *buf)
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 
@@ -467,8 +471,8 @@ static ssize_t stmfts_sysfs_hover_enable_read(struct device *dev,
 }
 
 static ssize_t stmfts_sysfs_hover_enable_write(struct device *dev,
-				struct device_attribute *attr,
-				const char *buf, size_t len)
+					       struct device_attribute *attr,
+					       const char *buf, size_t len)
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
 	unsigned long value;
@@ -485,8 +489,8 @@ static ssize_t stmfts_sysfs_hover_enable_write(struct device *dev,
 	if (hover != sdata->hover_enabled) {
 		if (sdata->running) {
 			err = i2c_smbus_write_byte(sdata->client,
-					   value ? STMFTS_SS_HOVER_SENSE_ON :
-						   STMFTS_SS_HOVER_SENSE_OFF);
+						   value ? STMFTS_SS_HOVER_SENSE_ON :
+							   STMFTS_SS_HOVER_SENSE_OFF);
 			if (err)
 				return err;
 		}
@@ -518,22 +522,11 @@ static struct attribute *stmfts_sysfs_attrs[] = {
 };
 ATTRIBUTE_GROUPS(stmfts_sysfs);
 
-static int stmfts_power_on(struct stmfts_data *sdata)
+static int stmfts_read_system_info(struct stmfts_data *sdata)
 {
 	int err;
 	u8 reg[8];
 
-	err = regulator_bulk_enable(ARRAY_SIZE(sdata->regulators),
-				    sdata->regulators);
-	if (err)
-		return err;
-
-	/*
-	 * The datasheet does not specify the power on time, but considering
-	 * that the reset time is < 10ms, I sleep 20ms to be sure
-	 */
-	msleep(20);
-
 	err = i2c_smbus_read_i2c_block_data(sdata->client, STMFTS_READ_INFO,
 					    sizeof(reg), reg);
 	if (err < 0)
@@ -547,9 +540,21 @@ static int stmfts_power_on(struct stmfts_data *sdata)
 	sdata->config_id = reg[4];
 	sdata->config_ver = reg[5];
 
-	enable_irq(sdata->client->irq);
+	return 0;
+}
 
+static void stmfts_reset(struct stmfts_data *sdata)
+{
+	gpiod_set_value_cansleep(sdata->reset_gpio, 1);
+	msleep(20);
+
+	gpiod_set_value_cansleep(sdata->reset_gpio, 0);
 	msleep(50);
+}
+
+static int stmfts_configure(struct stmfts_data *sdata)
+{
+	int err;
 
 	err = stmfts_command(sdata, STMFTS_SYSTEM_RESET);
 	if (err)
@@ -575,13 +580,52 @@ static int stmfts_power_on(struct stmfts_data *sdata)
 	if (err)
 		return err;
 
+	return 0;
+}
+
+static int stmfts_power_on(struct stmfts_data *sdata)
+{
+	int err;
+
+	err = regulator_bulk_enable(ARRAY_SIZE(stmfts_supplies),
+				    sdata->supplies);
+	if (err)
+		return err;
+
+	/*
+	 * The datasheet does not specify the power on time, but considering
+	 * that the reset time is < 10ms, I sleep 20ms to be sure
+	 */
+	msleep(20);
+
+	if (sdata->reset_gpio)
+		stmfts_reset(sdata);
+
+	err = stmfts_read_system_info(sdata);
+	if (err)
+		goto err_disable_regulators;
+
+	enable_irq(sdata->client->irq);
+
+	msleep(50);
+
+	err = stmfts_configure(sdata);
+	if (err)
+		goto err_disable_irq;
+
 	/*
 	 * At this point no one is using the touchscreen
 	 * and I don't really care about the return value
 	 */
-	(void) i2c_smbus_write_byte(sdata->client, STMFTS_SLEEP_IN);
+	(void)i2c_smbus_write_byte(sdata->client, STMFTS_SLEEP_IN);
 
 	return 0;
+
+err_disable_irq:
+	disable_irq(sdata->client->irq);
+err_disable_regulators:
+	regulator_bulk_disable(ARRAY_SIZE(stmfts_supplies), sdata->supplies);
+	return err;
 }
 
 static void stmfts_power_off(void *data)
@@ -589,8 +633,11 @@ static void stmfts_power_off(void *data)
 	struct stmfts_data *sdata = data;
 
 	disable_irq(sdata->client->irq);
-	regulator_bulk_disable(ARRAY_SIZE(sdata->regulators),
-						sdata->regulators);
+
+	if (sdata->reset_gpio)
+		gpiod_set_value_cansleep(sdata->reset_gpio, 1);
+
+	regulator_bulk_disable(ARRAY_SIZE(stmfts_supplies), sdata->supplies);
 }
 
 static int stmfts_enable_led(struct stmfts_data *sdata)
@@ -619,6 +666,7 @@ static int stmfts_enable_led(struct stmfts_data *sdata)
 
 static int stmfts_probe(struct i2c_client *client)
 {
+	struct device *dev = &client->dev;
 	int err;
 	struct stmfts_data *sdata;
 
@@ -627,7 +675,7 @@ static int stmfts_probe(struct i2c_client *client)
 						I2C_FUNC_SMBUS_I2C_BLOCK))
 		return -ENODEV;
 
-	sdata = devm_kzalloc(&client->dev, sizeof(*sdata), GFP_KERNEL);
+	sdata = devm_kzalloc(dev, sizeof(*sdata), GFP_KERNEL);
 	if (!sdata)
 		return -ENOMEM;
 
@@ -637,15 +685,19 @@ static int stmfts_probe(struct i2c_client *client)
 	mutex_init(&sdata->mutex);
 	init_completion(&sdata->cmd_done);
 
-	sdata->regulators[STMFTS_REGULATOR_VDD].supply = "vdd";
-	sdata->regulators[STMFTS_REGULATOR_AVDD].supply = "avdd";
-	err = devm_regulator_bulk_get(&client->dev,
-				      ARRAY_SIZE(sdata->regulators),
-				      sdata->regulators);
+	err = devm_regulator_bulk_get_const(dev,
+					    ARRAY_SIZE(stmfts_supplies),
+					    stmfts_supplies,
+					    &sdata->supplies);
 	if (err)
 		return err;
 
-	sdata->input = devm_input_allocate_device(&client->dev);
+	sdata->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
+	if (IS_ERR(sdata->reset_gpio))
+		return dev_err_probe(dev, PTR_ERR(sdata->reset_gpio),
+				     "Failed to get GPIO 'reset'\n");
+
+	sdata->input = devm_input_allocate_device(dev);
 	if (!sdata->input)
 		return -ENOMEM;
 
@@ -664,8 +716,7 @@ static int stmfts_probe(struct i2c_client *client)
 	input_set_abs_params(sdata->input, ABS_MT_PRESSURE, 0, 255, 0, 0);
 	input_set_abs_params(sdata->input, ABS_DISTANCE, 0, 255, 0, 0);
 
-	sdata->use_key = device_property_read_bool(&client->dev,
-						   "touch-key-connected");
+	sdata->use_key = device_property_read_bool(dev, "touch-key-connected");
 	if (sdata->use_key) {
 		input_set_capability(sdata->input, EV_KEY, KEY_MENU);
 		input_set_capability(sdata->input, EV_KEY, KEY_BACK);
@@ -685,20 +736,20 @@ static int stmfts_probe(struct i2c_client *client)
 	 * interrupts. To be on the safe side it's better to not enable
 	 * the interrupts during their request.
 	 */
-	err = devm_request_threaded_irq(&client->dev, client->irq,
+	err = devm_request_threaded_irq(dev, client->irq,
 					NULL, stmfts_irq_handler,
 					IRQF_ONESHOT | IRQF_NO_AUTOEN,
 					"stmfts_irq", sdata);
 	if (err)
 		return err;
 
-	dev_dbg(&client->dev, "initializing ST-Microelectronics FTS...\n");
+	dev_dbg(dev, "initializing ST-Microelectronics FTS...\n");
 
 	err = stmfts_power_on(sdata);
 	if (err)
 		return err;
 
-	err = devm_add_action_or_reset(&client->dev, stmfts_power_off, sdata);
+	err = devm_add_action_or_reset(dev, stmfts_power_off, sdata);
 	if (err)
 		return err;
 
@@ -715,13 +766,13 @@ static int stmfts_probe(struct i2c_client *client)
 			 * without LEDs. The ledvdd regulator pointer will be
 			 * used as a flag.
 			 */
-			dev_warn(&client->dev, "unable to use touchkey leds\n");
+			dev_warn(dev, "unable to use touchkey leds\n");
 			sdata->ledvdd = NULL;
 		}
 	}
 
-	pm_runtime_enable(&client->dev);
-	device_enable_async_suspend(&client->dev);
+	pm_runtime_enable(dev);
+	device_enable_async_suspend(dev);
 
 	return 0;
 }
@@ -746,9 +797,10 @@ static int stmfts_runtime_suspend(struct device *dev)
 static int stmfts_runtime_resume(struct device *dev)
 {
 	struct stmfts_data *sdata = dev_get_drvdata(dev);
+	struct i2c_client *client = sdata->client;
 	int ret;
 
-	ret = i2c_smbus_write_byte(sdata->client, STMFTS_SLEEP_OUT);
+	ret = i2c_smbus_write_byte(client, STMFTS_SLEEP_OUT);
 	if (ret)
 		dev_err(dev, "failed to resume device: %d\n", ret);
 
@@ -807,4 +859,4 @@ module_i2c_driver(stmfts_driver);
 
 MODULE_AUTHOR("Andi Shyti <andi.shyti@samsung.com>");
 MODULE_DESCRIPTION("STMicroelectronics FTS Touch Screen");
-MODULE_LICENSE("GPL v2");
+MODULE_LICENSE("GPL");
diff --git a/drivers/input/touchscreen/tsc2007_core.c b/drivers/input/touchscreen/tsc2007_core.c
index 524f14eb3da2f..f9b3d2598933c 100644
--- a/drivers/input/touchscreen/tsc2007_core.c
+++ b/drivers/input/touchscreen/tsc2007_core.c
@@ -61,10 +61,8 @@ static void tsc2007_read_values(struct tsc2007 *tsc, struct ts_event *tc)
 
 	/* turn y+ off, x- on; we'll use formula #1 */
 	tc->z1 = tsc2007_xfer(tsc, READ_Z1);
-	tc->z2 = tsc2007_xfer(tsc, READ_Z2);
-
-	/* Prepare for next touch reading - power down ADC, enable PENIRQ */
-	tsc2007_xfer(tsc, PWRDOWN);
+	/* Read Z2 and power down ADC after A/D conversion, enable PENIRQ */
+	tc->z2 = tsc2007_xfer(tsc, (TSC2007_POWER_OFF_IRQ_EN | TSC2007_MEASURE_Z2));
 }
 
 u32 tsc2007_calculate_resistance(struct tsc2007 *tsc, struct ts_event *tc)
diff --git a/drivers/input/touchscreen/usbtouchscreen.c b/drivers/input/touchscreen/usbtouchscreen.c
index daa28135f887e..0bbacb517c286 100644
--- a/drivers/input/touchscreen/usbtouchscreen.c
+++ b/drivers/input/touchscreen/usbtouchscreen.c
@@ -1067,6 +1067,11 @@ static int nexio_read_data(struct usbtouch_usb *usbtouch, unsigned char *pkt)
 	if (x_len > 0xff)
 		x_len -= 0x80;
 
+	if (data_len > usbtouch->data_size - sizeof(*packet))
+		data_len = usbtouch->data_size - sizeof(*packet);
+	if (x_len > data_len)
+		x_len = data_len;
+
 	/* send ACK */
 	ret = usb_submit_urb(priv->ack, GFP_ATOMIC);
 	if (ret)
diff --git a/drivers/interconnect/core.c b/drivers/interconnect/core.c
index 8569b78a18517..ce572c62f58f9 100644
--- a/drivers/interconnect/core.c
+++ b/drivers/interconnect/core.c
@@ -432,7 +432,7 @@ struct icc_path *devm_of_icc_get(struct device *dev, const char *name)
 		return ERR_PTR(-ENOMEM);
 
 	path = of_icc_get(dev, name);
-	if (!IS_ERR(path)) {
+	if (!IS_ERR_OR_NULL(path)) {
 		*ptr = path;
 		devres_add(dev, ptr);
 	} else {
diff --git a/drivers/interconnect/qcom/Kconfig b/drivers/interconnect/qcom/Kconfig
index 786b4eda44b4f..5b8a094ca4ede 100644
--- a/drivers/interconnect/qcom/Kconfig
+++ b/drivers/interconnect/qcom/Kconfig
@@ -9,26 +9,38 @@ config INTERCONNECT_QCOM_BCM_VOTER
 	tristate
 
 config INTERCONNECT_QCOM_ELIZA
-       tristate "Qualcomm Eliza interconnect driver"
-       depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
-       select INTERCONNECT_QCOM_RPMH
-       select INTERCONNECT_QCOM_BCM_VOTER
-       help
-         This is a driver for the Qualcomm Network-on-Chip on Eliza-based
-         platforms.
+	tristate "Qualcomm Eliza interconnect driver"
+	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
+	select INTERCONNECT_QCOM_RPMH
+	select INTERCONNECT_QCOM_BCM_VOTER
+	help
+	  This is a driver for the Qualcomm Network-on-Chip on Eliza-based
+	  platforms.
 
 config INTERCONNECT_QCOM_GLYMUR
-       tristate "Qualcomm Glymur interconnect driver"
-       depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
-       select INTERCONNECT_QCOM_RPMH
-       select INTERCONNECT_QCOM_BCM_VOTER
-       help
-         This is a driver for the Qualcomm Network-on-Chip on glymur-based
-         platforms.
+	tristate "Qualcomm Glymur interconnect driver"
+	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
+	select INTERCONNECT_QCOM_RPMH
+	select INTERCONNECT_QCOM_BCM_VOTER
+	help
+	  This is a driver for the Qualcomm Network-on-Chip on glymur-based
+	  platforms.
+
+config INTERCONNECT_QCOM_HAWI
+	tristate "Qualcomm HAWI interconnect driver"
+	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	select INTERCONNECT_QCOM_RPMH
+	select INTERCONNECT_QCOM_BCM_VOTER
+	help
+	  This is a driver for the Qualcomm Network-on-Chip on hawi-based
+	  platforms.
 
 config INTERCONNECT_QCOM_KAANAPALI
 	tristate "Qualcomm Kaanapali interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -39,6 +51,7 @@ config INTERCONNECT_QCOM_MSM8909
 	tristate "Qualcomm MSM8909 interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	depends on ARM || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on msm8909-based
@@ -75,6 +88,7 @@ config INTERCONNECT_QCOM_MSM8953
 	tristate "Qualcomm MSM8953 interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on msm8953-based
@@ -84,6 +98,7 @@ config INTERCONNECT_QCOM_MSM8974
 	tristate "Qualcomm MSM8974 interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	depends on ARM || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	 This is a driver for the Qualcomm Network-on-Chip on msm8974-based
@@ -93,6 +108,7 @@ config INTERCONNECT_QCOM_MSM8976
 	tristate "Qualcomm MSM8976 interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on msm8976-based
@@ -102,14 +118,27 @@ config INTERCONNECT_QCOM_MSM8996
 	tristate "Qualcomm MSM8996 interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on msm8996-based
 	  platforms.
 
+config INTERCONNECT_QCOM_NORD
+	tristate "Qualcomm Nord interconnect driver"
+	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	select INTERCONNECT_QCOM_RPMH
+	select INTERCONNECT_QCOM_BCM_VOTER
+	help
+	  This is a driver for the Qualcomm Network-on-Chip on Nord-based
+	  platforms. The topology consists of several NoCs controlled by
+	  the RPMh hardware and communicates via Bus Clock Manager (BCM)
+	  through the Resource State Coordinator (RSC).
+
 config INTERCONNECT_QCOM_OSM_L3
 	tristate "Qualcomm OSM L3 interconnect driver"
 	depends on INTERCONNECT_QCOM || COMPILE_TEST
+	depends on ARM64 || COMPILE_TEST
 	help
 	  Say y here to support the Operating State Manager (OSM) interconnect
 	  driver which controls the scaling of L3 caches on Qualcomm SoCs.
@@ -118,6 +147,7 @@ config INTERCONNECT_QCOM_QCM2290
 	tristate "Qualcomm QCM2290 interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on qcm2290-based
@@ -127,6 +157,7 @@ config INTERCONNECT_QCOM_QCS404
 	tristate "Qualcomm QCS404 interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on qcs404-based
@@ -135,6 +166,7 @@ config INTERCONNECT_QCOM_QCS404
 config INTERCONNECT_QCOM_QCS615
 	tristate "Qualcomm QCS615 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -144,6 +176,7 @@ config INTERCONNECT_QCOM_QCS615
 config INTERCONNECT_QCOM_QCS8300
 	tristate "Qualcomm QCS8300 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -155,6 +188,7 @@ config INTERCONNECT_QCOM_QCS8300
 config INTERCONNECT_QCOM_QDU1000
 	tristate "Qualcomm QDU1000/QRU1000 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -178,6 +212,7 @@ config INTERCONNECT_QCOM_RPMH
 config INTERCONNECT_QCOM_SA8775P
 	tristate "Qualcomm SA8775P interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -187,6 +222,7 @@ config INTERCONNECT_QCOM_SA8775P
 config INTERCONNECT_QCOM_SAR2130P
 	tristate "Qualcomm SAR2130P interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -196,6 +232,7 @@ config INTERCONNECT_QCOM_SAR2130P
 config INTERCONNECT_QCOM_SC7180
 	tristate "Qualcomm SC7180 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -205,6 +242,7 @@ config INTERCONNECT_QCOM_SC7180
 config INTERCONNECT_QCOM_SC7280
 	tristate "Qualcomm SC7280 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -214,6 +252,7 @@ config INTERCONNECT_QCOM_SC7280
 config INTERCONNECT_QCOM_SC8180X
 	tristate "Qualcomm SC8180X interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -223,6 +262,7 @@ config INTERCONNECT_QCOM_SC8180X
 config INTERCONNECT_QCOM_SC8280XP
 	tristate "Qualcomm SC8280XP interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -232,6 +272,7 @@ config INTERCONNECT_QCOM_SC8280XP
 config INTERCONNECT_QCOM_SDM660
 	tristate "Qualcomm SDM660 interconnect driver"
 	depends on INTERCONNECT_QCOM
+	depends on ARM64 || COMPILE_TEST
 	depends on QCOM_SMD_RPM
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
@@ -241,6 +282,7 @@ config INTERCONNECT_QCOM_SDM660
 config INTERCONNECT_QCOM_SDM670
 	tristate "Qualcomm SDM670 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -250,6 +292,7 @@ config INTERCONNECT_QCOM_SDM670
 config INTERCONNECT_QCOM_SDM845
 	tristate "Qualcomm SDM845 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -259,6 +302,7 @@ config INTERCONNECT_QCOM_SDM845
 config INTERCONNECT_QCOM_SDX55
 	tristate "Qualcomm SDX55 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -268,6 +312,7 @@ config INTERCONNECT_QCOM_SDX55
 config INTERCONNECT_QCOM_SDX65
 	tristate "Qualcomm SDX65 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -277,16 +322,27 @@ config INTERCONNECT_QCOM_SDX65
 config INTERCONNECT_QCOM_SDX75
 	tristate "Qualcomm SDX75 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sdx75-based
 	  platforms.
 
+config INTERCONNECT_QCOM_SHIKRA
+	tristate "Qualcomm SHIKRA interconnect driver"
+	depends on INTERCONNECT_QCOM
+	depends on QCOM_SMD_RPM
+	select INTERCONNECT_QCOM_SMD_RPM
+	help
+	  This is a driver for the Qualcomm Network-on-Chip on shikra-based
+	  platforms.
+
 config INTERCONNECT_QCOM_SM6115
 	tristate "Qualcomm SM6115 interconnect driver"
 	depends on INTERCONNECT_QCOM
 	depends on QCOM_SMD_RPM
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_SMD_RPM
 	help
 	  This is a driver for the Qualcomm Network-on-Chip on sm6115-based
@@ -295,6 +351,7 @@ config INTERCONNECT_QCOM_SM6115
 config INTERCONNECT_QCOM_SM6350
 	tristate "Qualcomm SM6350 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -304,6 +361,7 @@ config INTERCONNECT_QCOM_SM6350
 config INTERCONNECT_QCOM_SM7150
 	tristate "Qualcomm SM7150 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -313,6 +371,7 @@ config INTERCONNECT_QCOM_SM7150
 config INTERCONNECT_QCOM_MILOS
 	tristate "Qualcomm Milos interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -322,6 +381,7 @@ config INTERCONNECT_QCOM_MILOS
 config INTERCONNECT_QCOM_SM8150
 	tristate "Qualcomm SM8150 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -331,6 +391,7 @@ config INTERCONNECT_QCOM_SM8150
 config INTERCONNECT_QCOM_SM8250
 	tristate "Qualcomm SM8250 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -340,6 +401,7 @@ config INTERCONNECT_QCOM_SM8250
 config INTERCONNECT_QCOM_SM8350
 	tristate "Qualcomm SM8350 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -349,6 +411,7 @@ config INTERCONNECT_QCOM_SM8350
 config INTERCONNECT_QCOM_SM8450
 	tristate "Qualcomm SM8450 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -358,6 +421,7 @@ config INTERCONNECT_QCOM_SM8450
 config INTERCONNECT_QCOM_SM8550
 	tristate "Qualcomm SM8550 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -367,6 +431,7 @@ config INTERCONNECT_QCOM_SM8550
 config INTERCONNECT_QCOM_SM8650
 	tristate "Qualcomm SM8650 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -376,6 +441,7 @@ config INTERCONNECT_QCOM_SM8650
 config INTERCONNECT_QCOM_SM8750
 	tristate "Qualcomm SM8750 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
@@ -385,6 +451,7 @@ config INTERCONNECT_QCOM_SM8750
 config INTERCONNECT_QCOM_X1E80100
 	tristate "Qualcomm X1E80100 interconnect driver"
 	depends on INTERCONNECT_QCOM_RPMH_POSSIBLE
+	depends on ARM64 || COMPILE_TEST
 	select INTERCONNECT_QCOM_RPMH
 	select INTERCONNECT_QCOM_BCM_VOTER
 	help
diff --git a/drivers/interconnect/qcom/Makefile b/drivers/interconnect/qcom/Makefile
index cdf2c6c9fbf32..1c7d410b40cc9 100644
--- a/drivers/interconnect/qcom/Makefile
+++ b/drivers/interconnect/qcom/Makefile
@@ -6,6 +6,7 @@ interconnect_qcom-y			:= icc-common.o
 icc-bcm-voter-objs			:= bcm-voter.o
 qnoc-eliza-objs				:= eliza.o
 qnoc-glymur-objs			:= glymur.o
+qnoc-hawi-objs				:= hawi.o
 qnoc-kaanapali-objs			:= kaanapali.o
 qnoc-milos-objs				:= milos.o
 qnoc-msm8909-objs			:= msm8909.o
@@ -16,6 +17,7 @@ qnoc-msm8953-objs			:= msm8953.o
 qnoc-msm8974-objs			:= msm8974.o
 qnoc-msm8976-objs			:= msm8976.o
 qnoc-msm8996-objs			:= msm8996.o
+qnoc-nord-objs				:= nord.o
 icc-osm-l3-objs				:= osm-l3.o
 qnoc-qcm2290-objs			:= qcm2290.o
 qnoc-qcs404-objs			:= qcs404.o
@@ -35,6 +37,7 @@ qnoc-sdm845-objs			:= sdm845.o
 qnoc-sdx55-objs				:= sdx55.o
 qnoc-sdx65-objs				:= sdx65.o
 qnoc-sdx75-objs				:= sdx75.o
+qnoc-shikra-objs			:= shikra.o
 qnoc-sm6115-objs			:= sm6115.o
 qnoc-sm6350-objs			:= sm6350.o
 qnoc-sm7150-objs			:= sm7150.o
@@ -51,6 +54,7 @@ icc-smd-rpm-objs			:= smd-rpm.o icc-rpm.o icc-rpm-clocks.o
 obj-$(CONFIG_INTERCONNECT_QCOM_BCM_VOTER) += icc-bcm-voter.o
 obj-$(CONFIG_INTERCONNECT_QCOM_ELIZA) += qnoc-eliza.o
 obj-$(CONFIG_INTERCONNECT_QCOM_GLYMUR) += qnoc-glymur.o
+obj-$(CONFIG_INTERCONNECT_QCOM_HAWI) += qnoc-hawi.o
 obj-$(CONFIG_INTERCONNECT_QCOM_KAANAPALI) += qnoc-kaanapali.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MILOS) += qnoc-milos.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MSM8909) += qnoc-msm8909.o
@@ -61,6 +65,7 @@ obj-$(CONFIG_INTERCONNECT_QCOM_MSM8953) += qnoc-msm8953.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MSM8974) += qnoc-msm8974.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MSM8976) += qnoc-msm8976.o
 obj-$(CONFIG_INTERCONNECT_QCOM_MSM8996) += qnoc-msm8996.o
+obj-$(CONFIG_INTERCONNECT_QCOM_NORD) += qnoc-nord.o
 obj-$(CONFIG_INTERCONNECT_QCOM_OSM_L3) += icc-osm-l3.o
 obj-$(CONFIG_INTERCONNECT_QCOM_QCM2290) += qnoc-qcm2290.o
 obj-$(CONFIG_INTERCONNECT_QCOM_QCS404) += qnoc-qcs404.o
@@ -80,6 +85,7 @@ obj-$(CONFIG_INTERCONNECT_QCOM_SDM845) += qnoc-sdm845.o
 obj-$(CONFIG_INTERCONNECT_QCOM_SDX55) += qnoc-sdx55.o
 obj-$(CONFIG_INTERCONNECT_QCOM_SDX65) += qnoc-sdx65.o
 obj-$(CONFIG_INTERCONNECT_QCOM_SDX75) += qnoc-sdx75.o
+obj-$(CONFIG_INTERCONNECT_QCOM_SHIKRA) += qnoc-shikra.o
 obj-$(CONFIG_INTERCONNECT_QCOM_SM6115) += qnoc-sm6115.o
 obj-$(CONFIG_INTERCONNECT_QCOM_SM6350) += qnoc-sm6350.o
 obj-$(CONFIG_INTERCONNECT_QCOM_SM7150) += qnoc-sm7150.o
diff --git a/drivers/interconnect/qcom/eliza.c b/drivers/interconnect/qcom/eliza.c
index a4f7903f0524d..891e4e6e8ba8f 100644
--- a/drivers/interconnect/qcom/eliza.c
+++ b/drivers/interconnect/qcom/eliza.c
@@ -127,6 +127,12 @@ static struct qcom_icc_node qhs_qup2 = {
 	.buswidth = 4,
 };
 
+static struct qcom_icc_node qhs_sdc1 = {
+	.name = "qhs_sdc1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
 static struct qcom_icc_node qhs_sdc2 = {
 	.name = "qhs_sdc2",
 	.channels = 1,
@@ -383,7 +389,7 @@ static struct qcom_icc_node qsm_cfg = {
 	.name = "qsm_cfg",
 	.channels = 1,
 	.buswidth = 4,
-	.num_links = 29,
+	.num_links = 30,
 	.link_nodes = { &qhs_ahb2phy0, &qhs_ahb2phy1,
 			&qhs_camera_cfg, &qhs_clk_ctl,
 			&qhs_crypto0_cfg, &qhs_display_cfg,
@@ -392,7 +398,7 @@ static struct qcom_icc_node qsm_cfg = {
 			&qhs_mss_cfg, &qhs_pcie_0_cfg,
 			&qhs_prng, &qhs_qdss_cfg,
 			&qhs_qspi, &qhs_qup1,
-			&qhs_qup2, &qhs_sdc2,
+			&qhs_qup2, &qhs_sdc1, &qhs_sdc2,
 			&qhs_tcsr, &qhs_tlmm,
 			&qhs_ufs_mem_cfg, &qhs_usb3_0,
 			&qhs_venus_cfg, &qhs_vsense_ctrl_cfg,
@@ -1111,7 +1117,7 @@ static struct qcom_icc_bcm bcm_cn0 = {
 	.name = "CN0",
 	.enable_mask = BIT(0),
 	.keepalive = true,
-	.num_nodes = 43,
+	.num_nodes = 44,
 	.nodes = { &qsm_cfg, &qhs_ahb2phy0,
 		   &qhs_ahb2phy1, &qhs_camera_cfg,
 		   &qhs_clk_ctl, &qhs_crypto0_cfg,
@@ -1119,7 +1125,7 @@ static struct qcom_icc_bcm bcm_cn0 = {
 		   &qhs_i3c_ibi1_cfg, &qhs_imem_cfg,
 		   &qhs_mss_cfg, &qhs_pcie_0_cfg,
 		   &qhs_prng, &qhs_qdss_cfg,
-		   &qhs_qspi, &qhs_sdc2,
+		   &qhs_qspi, &qhs_sdc1, &qhs_sdc2,
 		   &qhs_tcsr, &qhs_tlmm,
 		   &qhs_ufs_mem_cfg, &qhs_usb3_0,
 		   &qhs_venus_cfg, &qhs_vsense_ctrl_cfg,
@@ -1321,6 +1327,7 @@ static struct qcom_icc_node * const cnoc_cfg_nodes[] = {
 	[SLAVE_QSPI_0] = &qhs_qspi,
 	[SLAVE_QUP_1] = &qhs_qup1,
 	[SLAVE_QUP_2] = &qhs_qup2,
+	[SLAVE_SDCC_1] = &qhs_sdc1,
 	[SLAVE_SDCC_2] = &qhs_sdc2,
 	[SLAVE_TCSR] = &qhs_tcsr,
 	[SLAVE_TLMM] = &qhs_tlmm,
diff --git a/drivers/interconnect/qcom/hawi.c b/drivers/interconnect/qcom/hawi.c
new file mode 100644
index 0000000000000..6ece9828c62d3
--- /dev/null
+++ b/drivers/interconnect/qcom/hawi.c
@@ -0,0 +1,2028 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ */
+
+#include <linux/device.h>
+#include <linux/interconnect.h>
+#include <linux/interconnect-provider.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <dt-bindings/interconnect/qcom,hawi-rpmh.h>
+
+#include "bcm-voter.h"
+#include "icc-rpmh.h"
+
+static struct qcom_icc_node ddr_eff_veto_slave = {
+	.name = "ddr_eff_veto_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup0_core_slave = {
+	.name = "qup0_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup1_core_slave = {
+	.name = "qup1_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup2_core_slave = {
+	.name = "qup2_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup3_core_slave = {
+	.name = "qup3_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup4_core_slave = {
+	.name = "qup4_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_aoss = {
+	.name = "qhs_aoss",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipa = {
+	.name = "qhs_ipa",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipc_router_fence = {
+	.name = "qhs_ipc_router_fence",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_soccp = {
+	.name = "qhs_soccp",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tme_cfg = {
+	.name = "qhs_tme_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_ddrss_cfg = {
+	.name = "qss_ddrss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qxs_imem = {
+	.name = "qxs_imem",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node xs_pcie = {
+	.name = "xs_pcie",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node xs_pcie_g4x1 = {
+	.name = "xs_pcie_g4x1",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qhs_lpi_cc = {
+	.name = "qhs_lpi_cc",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qns_lb = {
+	.name = "qns_lb",
+	.channels = 4,
+	.buswidth = 32,
+};
+
+static struct qcom_icc_node srvc_llclpi_noc = {
+	.name = "srvc_llclpi_noc",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node srvc_llclpi_noc_chipcx = {
+	.name = "srvc_llclpi_noc_chipcx",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node ebi = {
+	.name = "ebi",
+	.channels = 4,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node ddr_rt_slave = {
+	.name = "ddr_rt_slave",
+	.channels = 4,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node srvc_pcie_aggre_noc = {
+	.name = "srvc_pcie_aggre_noc",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy0 = {
+	.name = "qhs_ahb2phy0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_boot_rom = {
+	.name = "qhs_boot_rom",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_camera_cfg = {
+	.name = "qhs_camera_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_clk_ctl = {
+	.name = "qhs_clk_ctl",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_crypto_cfg = {
+	.name = "qhs_crypto_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_display_cfg = {
+	.name = "qhs_display_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_eva_cfg = {
+	.name = "qhs_eva_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_gpuss_cfg = {
+	.name = "qhs_gpuss_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qhs_i2c = {
+	.name = "qhs_i2c",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_imem_cfg = {
+	.name = "qhs_imem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipc_router = {
+	.name = "qhs_ipc_router",
+	.channels = 4,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_iris_cfg = {
+	.name = "qhs_iris_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_mss_cfg = {
+	.name = "qhs_mss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_cfg = {
+	.name = "qhs_pcie_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_g4x1_cfg = {
+	.name = "qhs_pcie_g4x1_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_prng = {
+	.name = "qhs_prng",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qspi = {
+	.name = "qhs_qspi",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup1 = {
+	.name = "qhs_qup1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup2 = {
+	.name = "qhs_qup2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup3 = {
+	.name = "qhs_qup3",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup4 = {
+	.name = "qhs_qup4",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_sdc2 = {
+	.name = "qhs_sdc2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_sdc4 = {
+	.name = "qhs_sdc4",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tlmm = {
+	.name = "qhs_tlmm",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ufs_mem_cfg = {
+	.name = "qhs_ufs_mem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_usb3 = {
+	.name = "qhs_usb3",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_vsense_ctrl_cfg = {
+	.name = "qhs_vsense_ctrl_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_qdss_cfg = {
+	.name = "qss_qdss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_qdss_stm = {
+	.name = "qss_qdss_stm",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_tcsr = {
+	.name = "qss_tcsr",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node xs_sys_tcu_cfg = {
+	.name = "xs_sys_tcu_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node ddr_eff_veto_master = {
+	.name = "ddr_eff_veto_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &ddr_eff_veto_slave },
+};
+
+static struct qcom_icc_node qup0_core_master = {
+	.name = "qup0_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup0_core_slave },
+};
+
+static struct qcom_icc_node qup1_core_master = {
+	.name = "qup1_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup1_core_slave },
+};
+
+static struct qcom_icc_node qup2_core_master = {
+	.name = "qup2_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup2_core_slave },
+};
+
+static struct qcom_icc_node qup3_core_master = {
+	.name = "qup3_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup3_core_slave },
+};
+
+static struct qcom_icc_node qup4_core_master = {
+	.name = "qup4_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup4_core_slave },
+};
+
+static struct qcom_icc_node qnm_gemnoc_pcie = {
+	.name = "qnm_gemnoc_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 2,
+	.link_nodes = { &xs_pcie, &xs_pcie_g4x1 },
+};
+
+static struct qcom_icc_node qnm_lpiaon_noc_llclpi_noc = {
+	.name = "qnm_lpiaon_noc_llclpi_noc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 4,
+	.link_nodes = { &qhs_lpi_cc, &qns_lb,
+			&srvc_llclpi_noc, &srvc_llclpi_noc_chipcx },
+};
+
+static struct qcom_icc_node llcc_mc = {
+	.name = "llcc_mc",
+	.channels = 4,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &ebi },
+};
+
+static struct qcom_icc_node ddr_rt_mc = {
+	.name = "ddr_rt_mc",
+	.channels = 4,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &ddr_rt_slave },
+};
+
+static struct qcom_icc_node qsm_pcie_anoc_cfg = {
+	.name = "qsm_pcie_anoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &srvc_pcie_aggre_noc },
+};
+
+static struct qcom_icc_node qsm_cfg_east = {
+	.name = "qsm_cfg_east",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 4,
+	.link_nodes = { &qhs_crypto_cfg, &qhs_gpuss_cfg,
+			&qhs_qup2, &qhs_vsense_ctrl_cfg },
+};
+
+static struct qcom_icc_node qsm_cfg_mm = {
+	.name = "qsm_cfg_mm",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 5,
+	.link_nodes = { &qhs_boot_rom, &qhs_camera_cfg,
+			&qhs_display_cfg, &qhs_eva_cfg,
+			&qhs_iris_cfg },
+};
+
+static struct qcom_icc_node qsm_cfg_north = {
+	.name = "qsm_cfg_north",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 5,
+	.link_nodes = { &qhs_pcie_cfg, &qhs_pcie_g4x1_cfg,
+			&qhs_qup3, &qhs_qup4,
+			&qhs_sdc2 },
+};
+
+static struct qcom_icc_node qsm_cfg_south = {
+	.name = "qsm_cfg_south",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 6,
+	.link_nodes = { &qhs_ahb2phy0, &qhs_qspi,
+			&qhs_qup1, &qhs_sdc4,
+			&qhs_ufs_mem_cfg, &qhs_usb3 },
+};
+
+static struct qcom_icc_node qsm_cfg_southwest = {
+	.name = "qsm_cfg_southwest",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 2,
+	.link_nodes = { &qhs_ipc_router, &qhs_mss_cfg },
+};
+
+static struct qcom_icc_node qns_llcc = {
+	.name = "qns_llcc",
+	.channels = 4,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &llcc_mc },
+};
+
+static struct qcom_icc_node qns_pcie = {
+	.name = "qns_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qnm_gemnoc_pcie },
+};
+
+static struct qcom_icc_node qns_llc_lpinoc = {
+	.name = "qns_llc_lpinoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpiaon_noc_llclpi_noc },
+};
+
+static struct qcom_icc_node qss_pcie_anoc_cfg = {
+	.name = "qss_pcie_anoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_pcie_anoc_cfg },
+};
+
+static struct qcom_icc_node qss_stdst_east_cfg = {
+	.name = "qss_stdst_east_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_east },
+};
+
+static struct qcom_icc_node qss_stdst_mm_cfg = {
+	.name = "qss_stdst_mm_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_mm },
+};
+
+static struct qcom_icc_node qss_stdst_north_cfg = {
+	.name = "qss_stdst_north_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_north },
+};
+
+static struct qcom_icc_node qss_stdst_south_cfg = {
+	.name = "qss_stdst_south_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_south },
+};
+
+static struct qcom_icc_node qss_stdst_southwest_cfg = {
+	.name = "qss_stdst_southwest_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_southwest },
+};
+
+static struct qcom_icc_node alm_gic = {
+	.name = "alm_gic",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14d000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_node qnm_qpace = {
+	.name = "qnm_qpace",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x153000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_node qsm_cfg_center = {
+	.name = "qsm_cfg_center",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 10,
+	.link_nodes = { &qhs_clk_ctl, &qhs_i2c,
+			&qhs_imem_cfg, &qhs_prng,
+			&qhs_tlmm, &qss_pcie_anoc_cfg,
+			&qss_qdss_cfg, &qss_qdss_stm,
+			&qss_tcsr, &xs_sys_tcu_cfg },
+};
+
+static struct qcom_icc_node qss_stdst_center_cfg = {
+	.name = "qss_stdst_center_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg_center },
+};
+
+static struct qcom_icc_node qsm_cnoc_main = {
+	.name = "qsm_cnoc_main",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 6,
+	.link_nodes = { &qss_stdst_center_cfg, &qss_stdst_east_cfg,
+			&qss_stdst_mm_cfg, &qss_stdst_north_cfg,
+			&qss_stdst_south_cfg, &qss_stdst_southwest_cfg },
+};
+
+static struct qcom_icc_node qss_cfg = {
+	.name = "qss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cnoc_main },
+};
+
+static struct qcom_icc_node qnm_gemnoc_cnoc = {
+	.name = "qnm_gemnoc_cnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 8,
+	.link_nodes = { &qhs_aoss, &qhs_ipa,
+			&qhs_ipc_router_fence, &qhs_soccp,
+			&qhs_tme_cfg, &qss_cfg,
+			&qss_ddrss_cfg, &qxs_imem },
+};
+
+static struct qcom_icc_node qns_gem_noc_cnoc = {
+	.name = "qns_gem_noc_cnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_gemnoc_cnoc },
+};
+
+static struct qcom_icc_node alm_gpu_tcu = {
+	.name = "alm_gpu_tcu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x145000 },
+		.prio = 1,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node alm_sys_tcu = {
+	.name = "alm_sys_tcu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x147000 },
+		.prio = 6,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node chm_apps = {
+	.name = "chm_apps",
+	.channels = 4,
+	.buswidth = 32,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_gpu = {
+	.name = "qnm_gpu",
+	.channels = 4,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 4,
+		.port_offsets = { 0x51000, 0x53000, 0xd1000, 0xd3000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_lpass_gemnoc = {
+	.name = "qnm_lpass_gemnoc",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x149000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_mdsp = {
+	.name = "qnm_mdsp",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_mnoc_hf = {
+	.name = "qnm_mnoc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x55000, 0xd5000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_mnoc_sf = {
+	.name = "qnm_mnoc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x57000, 0xd7000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_nsp_gemnoc = {
+	.name = "qnm_nsp_gemnoc",
+	.channels = 4,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 4,
+		.port_offsets = { 0x59000, 0x5b000, 0xd9000, 0xdb000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_pcie = {
+	.name = "qnm_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14b000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node qnm_snoc_sf = {
+	.name = "qnm_snoc_sf",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14f000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_wlan_q6 = {
+	.name = "qnm_wlan_q6",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 3,
+	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qns_lpass_ag_noc_gemnoc = {
+	.name = "qns_lpass_ag_noc_gemnoc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpass_gemnoc },
+};
+
+static struct qcom_icc_node qns_mem_noc_hf = {
+	.name = "qns_mem_noc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_mnoc_hf },
+};
+
+static struct qcom_icc_node qns_mem_noc_sf = {
+	.name = "qns_mem_noc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_mnoc_sf },
+};
+
+static struct qcom_icc_node qns_nsp_gemnoc = {
+	.name = "qns_nsp_gemnoc",
+	.channels = 4,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_nsp_gemnoc },
+};
+
+static struct qcom_icc_node qns_pcie_gemnoc = {
+	.name = "qns_pcie_gemnoc",
+	.channels = 1,
+	.buswidth = 8,
+	.num_links = 1,
+	.link_nodes = { &qnm_pcie },
+};
+
+static struct qcom_icc_node qns_gemnoc_sf = {
+	.name = "qns_gemnoc_sf",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_snoc_sf },
+};
+
+static struct qcom_icc_node qnm_lpiaon_noc = {
+	.name = "qnm_lpiaon_noc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_lpass_ag_noc_gemnoc },
+};
+
+static struct qcom_icc_node qnm_camnoc_hf = {
+	.name = "qnm_camnoc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x2a000, 0x2b000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_camnoc_nrt_icp_sf = {
+	.name = "qnm_camnoc_nrt_icp_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2c000 },
+		.prio = 4,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_camnoc_rt_cdm_sf = {
+	.name = "qnm_camnoc_rt_cdm_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x38000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_camnoc_sf = {
+	.name = "qnm_camnoc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x2d000, 0x2e000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_mdp = {
+	.name = "qnm_mdp",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x2f000, 0x30000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_mdss_dcp = {
+	.name = "qnm_mdss_dcp",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x39000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_vapss_hcp = {
+	.name = "qnm_vapss_hcp",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_cv_cpu = {
+	.name = "qnm_video_cv_cpu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x34000 },
+		.prio = 4,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_eva = {
+	.name = "qnm_video_eva",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x35000, 0x36000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_mvp = {
+	.name = "qnm_video_mvp",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x32000, 0x33000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_v_cpu = {
+	.name = "qnm_video_v_cpu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x37000 },
+		.prio = 4,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_nsp = {
+	.name = "qnm_nsp",
+	.channels = 4,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_nsp_gemnoc },
+};
+
+static struct qcom_icc_node xm_pcie = {
+	.name = "xm_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 3,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_gemnoc },
+};
+
+static struct qcom_icc_node xm_pcie_g4x1 = {
+	.name = "xm_pcie_g4x1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_gemnoc },
+};
+
+static struct qcom_icc_node qnm_aggre_noc = {
+	.name = "qnm_aggre_noc",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x20000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qnm_apss_noc = {
+	.name = "qnm_apss_noc",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1e000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qnm_cnoc_data = {
+	.name = "qnm_cnoc_data",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1f000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_node qns_a1noc_snoc = {
+	.name = "qns_a1noc_snoc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_aggre_noc },
+};
+
+static struct qcom_icc_node qns_lpass_aggnoc = {
+	.name = "qns_lpass_aggnoc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpiaon_noc },
+};
+
+static struct qcom_icc_node qhm_qspi = {
+	.name = "qhm_qspi",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x49000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qhm_qup2 = {
+	.name = "qhm_qup2",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x48000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qhm_qup3 = {
+	.name = "qhm_qup3",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x46000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qhm_qup4 = {
+	.name = "qhm_qup4",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x47000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_crypto = {
+	.name = "qxm_crypto",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x40000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_ipa = {
+	.name = "qxm_ipa",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x41000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_qup1 = {
+	.name = "qxm_qup1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4d000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_soccp = {
+	.name = "qxm_soccp",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x45000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_qdss_etr_0 = {
+	.name = "xm_qdss_etr_0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x42000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_qdss_etr_1 = {
+	.name = "xm_qdss_etr_1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x43000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_sdc2 = {
+	.name = "xm_sdc2",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x44000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_sdc4 = {
+	.name = "xm_sdc4",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4a000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_ufs_mem = {
+	.name = "xm_ufs_mem",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4b000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node xm_usb3 = {
+	.name = "xm_usb3",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4c000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qnm_lpass_lpinoc = {
+	.name = "qnm_lpass_lpinoc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 2,
+	.link_nodes = { &qns_llc_lpinoc, &qns_lpass_aggnoc },
+};
+
+static struct qcom_icc_node qns_lpi_aon_noc = {
+	.name = "qns_lpi_aon_noc",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_lpass_lpinoc },
+};
+
+static struct qcom_icc_node qnm_lpinoc_dsp_qns4m = {
+	.name = "qnm_lpinoc_dsp_qns4m",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_lpi_aon_noc },
+};
+
+static struct qcom_icc_bcm bcm_acv = {
+	.name = "ACV",
+	.enable_mask = BIT(3),
+	.num_nodes = 1,
+	.nodes = { &ebi },
+};
+
+static struct qcom_icc_bcm bcm_ce0 = {
+	.name = "CE0",
+	.num_nodes = 1,
+	.nodes = { &qxm_crypto },
+};
+
+static struct qcom_icc_bcm bcm_cn0 = {
+	.name = "CN0",
+	.enable_mask = BIT(0),
+	.keepalive = true,
+	.num_nodes = 24,
+	.nodes = { &qnm_gemnoc_cnoc, &qnm_gemnoc_pcie,
+		   &qhs_aoss, &qhs_ipa,
+		   &qhs_ipc_router_fence, &qhs_soccp,
+		   &qhs_tme_cfg, &qss_cfg,
+		   &qss_ddrss_cfg, &qxs_imem,
+		   &xs_pcie, &xs_pcie_g4x1,
+		   &qsm_cfg_center, &qsm_cfg_east,
+		   &qsm_cfg_mm, &qsm_cfg_north,
+		   &qsm_cfg_south, &qsm_cfg_southwest,
+		   &qhs_ahb2phy0, &qhs_boot_rom,
+		   &qhs_camera_cfg, &qhs_clk_ctl,
+		   &qhs_crypto_cfg, &qhs_eva_cfg },
+};
+
+static struct qcom_icc_bcm bcm_cn1 = {
+	.name = "CN1",
+	.num_nodes = 1,
+	.nodes = { &qhs_display_cfg },
+};
+
+static struct qcom_icc_bcm bcm_co0 = {
+	.name = "CO0",
+	.enable_mask = BIT(0),
+	.num_nodes = 2,
+	.nodes = { &qnm_nsp, &qns_nsp_gemnoc },
+};
+
+static struct qcom_icc_bcm bcm_de0 = {
+	.name = "DE0",
+	.enable_mask = BIT(0),
+	.num_nodes = 1,
+	.nodes = { &ddr_eff_veto_slave },
+};
+
+static struct qcom_icc_bcm bcm_lp0 = {
+	.name = "LP0",
+	.num_nodes = 5,
+	.nodes = { &qnm_lpiaon_noc_llclpi_noc, &qns_lb,
+		   &qnm_lpass_lpinoc, &qns_llc_lpinoc,
+		   &qns_lpass_aggnoc },
+};
+
+static struct qcom_icc_bcm bcm_mc0 = {
+	.name = "MC0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &ebi },
+};
+
+static struct qcom_icc_bcm bcm_mc5 = {
+	.name = "MC5",
+	.num_nodes = 1,
+	.nodes = { &ddr_rt_slave },
+};
+
+static struct qcom_icc_bcm bcm_mm0 = {
+	.name = "MM0",
+	.num_nodes = 1,
+	.nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_bcm bcm_mm1 = {
+	.name = "MM1",
+	.enable_mask = BIT(0),
+	.num_nodes = 9,
+	.nodes = { &qnm_camnoc_hf, &qnm_camnoc_nrt_icp_sf,
+		   &qnm_camnoc_rt_cdm_sf, &qnm_camnoc_sf,
+		   &qnm_vapss_hcp, &qnm_video_cv_cpu,
+		   &qnm_video_mvp, &qnm_video_v_cpu,
+		   &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_bcm bcm_qpc0 = {
+	.name = "QPC0",
+	.num_nodes = 1,
+	.nodes = { &qnm_qpace },
+};
+
+static struct qcom_icc_bcm bcm_qup0 = {
+	.name = "QUP0",
+	.keepalive = true,
+	.vote_scale = 1,
+	.num_nodes = 1,
+	.nodes = { &qup0_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup1 = {
+	.name = "QUP1",
+	.keepalive = true,
+	.vote_scale = 1,
+	.num_nodes = 1,
+	.nodes = { &qup1_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup2 = {
+	.name = "QUP2",
+	.keepalive = true,
+	.vote_scale = 1,
+	.num_nodes = 1,
+	.nodes = { &qup2_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup3 = {
+	.name = "QUP3",
+	.keepalive = true,
+	.vote_scale = 1,
+	.num_nodes = 1,
+	.nodes = { &qup3_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup4 = {
+	.name = "QUP4",
+	.keepalive = true,
+	.vote_scale = 1,
+	.num_nodes = 1,
+	.nodes = { &qup4_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_sh0 = {
+	.name = "SH0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_bcm bcm_sh1 = {
+	.name = "SH1",
+	.enable_mask = BIT(0),
+	.num_nodes = 15,
+	.nodes = { &alm_gic, &alm_gpu_tcu,
+		   &alm_sys_tcu, &chm_apps,
+		   &qnm_gpu, &qnm_lpass_gemnoc,
+		   &qnm_mdsp, &qnm_mnoc_hf,
+		   &qnm_mnoc_sf, &qnm_nsp_gemnoc,
+		   &qnm_pcie, &qnm_snoc_sf,
+		   &qnm_wlan_q6, &qns_gem_noc_cnoc,
+		   &qns_pcie },
+};
+
+static struct qcom_icc_bcm bcm_sn0 = {
+	.name = "SN0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_gemnoc_sf },
+};
+
+static struct qcom_icc_bcm bcm_sn2 = {
+	.name = "SN2",
+	.num_nodes = 1,
+	.nodes = { &qnm_aggre_noc },
+};
+
+static struct qcom_icc_bcm bcm_sn3 = {
+	.name = "SN3",
+	.num_nodes = 1,
+	.nodes = { &qns_pcie_gemnoc },
+};
+
+static struct qcom_icc_bcm * const aggre1_noc_bcms[] = {
+	&bcm_ce0,
+};
+
+static struct qcom_icc_node * const aggre1_noc_nodes[] = {
+	[MASTER_QSPI_0] = &qhm_qspi,
+	[MASTER_QUP_2] = &qhm_qup2,
+	[MASTER_QUP_3] = &qhm_qup3,
+	[MASTER_QUP_4] = &qhm_qup4,
+	[MASTER_CRYPTO] = &qxm_crypto,
+	[MASTER_IPA] = &qxm_ipa,
+	[MASTER_QUP_1] = &qxm_qup1,
+	[MASTER_SOCCP_PROC] = &qxm_soccp,
+	[MASTER_QDSS_ETR] = &xm_qdss_etr_0,
+	[MASTER_QDSS_ETR_1] = &xm_qdss_etr_1,
+	[MASTER_SDCC_2] = &xm_sdc2,
+	[MASTER_SDCC_4] = &xm_sdc4,
+	[MASTER_UFS_MEM] = &xm_ufs_mem,
+	[MASTER_USB3] = &xm_usb3,
+	[SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc,
+};
+
+static const struct regmap_config hawi_aggre1_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x54400,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_aggre1_noc = {
+	.config = &hawi_aggre1_noc_regmap_config,
+	.nodes = aggre1_noc_nodes,
+	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
+	.bcms = aggre1_noc_bcms,
+	.num_bcms = ARRAY_SIZE(aggre1_noc_bcms),
+	.qos_requires_clocks = true,
+};
+
+static struct qcom_icc_bcm * const clk_virt_bcms[] = {
+	&bcm_de0,
+	&bcm_qup0,
+	&bcm_qup1,
+	&bcm_qup2,
+	&bcm_qup3,
+	&bcm_qup4,
+};
+
+static struct qcom_icc_node * const clk_virt_nodes[] = {
+	[MASTER_DDR_EFF_VETO] = &ddr_eff_veto_master,
+	[MASTER_QUP_CORE_0] = &qup0_core_master,
+	[MASTER_QUP_CORE_1] = &qup1_core_master,
+	[MASTER_QUP_CORE_2] = &qup2_core_master,
+	[MASTER_QUP_CORE_3] = &qup3_core_master,
+	[MASTER_QUP_CORE_4] = &qup4_core_master,
+	[SLAVE_DDR_EFF_VETO] = &ddr_eff_veto_slave,
+	[SLAVE_QUP_CORE_0] = &qup0_core_slave,
+	[SLAVE_QUP_CORE_1] = &qup1_core_slave,
+	[SLAVE_QUP_CORE_2] = &qup2_core_slave,
+	[SLAVE_QUP_CORE_3] = &qup3_core_slave,
+	[SLAVE_QUP_CORE_4] = &qup4_core_slave,
+};
+
+static const struct qcom_icc_desc hawi_clk_virt = {
+	.nodes = clk_virt_nodes,
+	.num_nodes = ARRAY_SIZE(clk_virt_nodes),
+	.bcms = clk_virt_bcms,
+	.num_bcms = ARRAY_SIZE(clk_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const cnoc_main_bcms[] = {
+	&bcm_cn0,
+};
+
+static struct qcom_icc_node * const cnoc_main_nodes[] = {
+	[MASTER_GEM_NOC_CNOC] = &qnm_gemnoc_cnoc,
+	[MASTER_GEM_NOC_PCIE_SNOC] = &qnm_gemnoc_pcie,
+	[SLAVE_AOSS] = &qhs_aoss,
+	[SLAVE_IPA_CFG] = &qhs_ipa,
+	[SLAVE_IPC_ROUTER_FENCE] = &qhs_ipc_router_fence,
+	[SLAVE_SOCCP] = &qhs_soccp,
+	[SLAVE_TME_CFG] = &qhs_tme_cfg,
+	[SLAVE_CNOC_CFG] = &qss_cfg,
+	[SLAVE_DDRSS_CFG] = &qss_ddrss_cfg,
+	[SLAVE_IMEM] = &qxs_imem,
+	[SLAVE_PCIE_0] = &xs_pcie,
+	[SLAVE_PCIE_1] = &xs_pcie_g4x1,
+};
+
+static const struct regmap_config hawi_cnoc_main_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x20000,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_cnoc_main = {
+	.config = &hawi_cnoc_main_regmap_config,
+	.nodes = cnoc_main_nodes,
+	.num_nodes = ARRAY_SIZE(cnoc_main_nodes),
+	.bcms = cnoc_main_bcms,
+	.num_bcms = ARRAY_SIZE(cnoc_main_bcms),
+};
+
+static struct qcom_icc_bcm * const gem_noc_bcms[] = {
+	&bcm_qpc0,
+	&bcm_sh0,
+	&bcm_sh1,
+};
+
+static struct qcom_icc_node * const gem_noc_nodes[] = {
+	[MASTER_GIC] = &alm_gic,
+	[MASTER_GPU_TCU] = &alm_gpu_tcu,
+	[MASTER_SYS_TCU] = &alm_sys_tcu,
+	[MASTER_APPSS_PROC] = &chm_apps,
+	[MASTER_GFX3D] = &qnm_gpu,
+	[MASTER_LPASS_GEM_NOC] = &qnm_lpass_gemnoc,
+	[MASTER_MSS_PROC] = &qnm_mdsp,
+	[MASTER_MNOC_HF_MEM_NOC] = &qnm_mnoc_hf,
+	[MASTER_MNOC_SF_MEM_NOC] = &qnm_mnoc_sf,
+	[MASTER_COMPUTE_NOC] = &qnm_nsp_gemnoc,
+	[MASTER_ANOC_PCIE_GEM_NOC] = &qnm_pcie,
+	[MASTER_QPACE] = &qnm_qpace,
+	[MASTER_SNOC_SF_MEM_NOC] = &qnm_snoc_sf,
+	[MASTER_WLAN_Q6] = &qnm_wlan_q6,
+	[SLAVE_GEM_NOC_CNOC] = &qns_gem_noc_cnoc,
+	[SLAVE_LLCC] = &qns_llcc,
+	[SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie,
+};
+
+static const struct regmap_config hawi_gem_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x160200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_gem_noc = {
+	.config = &hawi_gem_noc_regmap_config,
+	.nodes = gem_noc_nodes,
+	.num_nodes = ARRAY_SIZE(gem_noc_nodes),
+	.bcms = gem_noc_bcms,
+	.num_bcms = ARRAY_SIZE(gem_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const llclpi_noc_bcms[] = {
+	&bcm_lp0,
+};
+
+static struct qcom_icc_node * const llclpi_noc_nodes[] = {
+	[MASTER_LPIAON_NOC_LLCLPI_NOC] = &qnm_lpiaon_noc_llclpi_noc,
+	[SLAVE_LPASS_LPI_CC] = &qhs_lpi_cc,
+	[SLAVE_LLCC_ISLAND] = &qns_lb,
+	[SLAVE_SERVICE_LLCLPI_NOC] = &srvc_llclpi_noc,
+	[SLAVE_SERVICE_LLCLPI_NOC_CHIPCX] = &srvc_llclpi_noc_chipcx,
+};
+
+static const struct regmap_config hawi_llclpi_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x17200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_llclpi_noc = {
+	.config = &hawi_llclpi_noc_regmap_config,
+	.nodes = llclpi_noc_nodes,
+	.num_nodes = ARRAY_SIZE(llclpi_noc_nodes),
+	.bcms = llclpi_noc_bcms,
+	.num_bcms = ARRAY_SIZE(llclpi_noc_bcms),
+};
+
+static struct qcom_icc_node * const lpass_ag_noc_nodes[] = {
+	[MASTER_LPIAON_NOC] = &qnm_lpiaon_noc,
+	[SLAVE_LPASS_GEM_NOC] = &qns_lpass_ag_noc_gemnoc,
+};
+
+static const struct regmap_config hawi_lpass_ag_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xc080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_lpass_ag_noc = {
+	.config = &hawi_lpass_ag_noc_regmap_config,
+	.nodes = lpass_ag_noc_nodes,
+	.num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const lpass_lpiaon_noc_bcms[] = {
+	&bcm_lp0,
+};
+
+static struct qcom_icc_node * const lpass_lpiaon_noc_nodes[] = {
+	[MASTER_LPASS_LPINOC] = &qnm_lpass_lpinoc,
+	[SLAVE_LPIAON_NOC_LLCLPI_NOC] = &qns_llc_lpinoc,
+	[SLAVE_LPIAON_NOC_LPASS_AG_NOC] = &qns_lpass_aggnoc,
+};
+
+static const struct regmap_config hawi_lpass_lpiaon_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x19080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_lpass_lpiaon_noc = {
+	.config = &hawi_lpass_lpiaon_noc_regmap_config,
+	.nodes = lpass_lpiaon_noc_nodes,
+	.num_nodes = ARRAY_SIZE(lpass_lpiaon_noc_nodes),
+	.bcms = lpass_lpiaon_noc_bcms,
+	.num_bcms = ARRAY_SIZE(lpass_lpiaon_noc_bcms),
+};
+
+static struct qcom_icc_node * const lpass_lpicx_noc_nodes[] = {
+	[MASTER_LPASS_PROC] = &qnm_lpinoc_dsp_qns4m,
+	[SLAVE_LPICX_NOC_LPIAON_NOC] = &qns_lpi_aon_noc,
+};
+
+static const struct regmap_config hawi_lpass_lpicx_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x46080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_lpass_lpicx_noc = {
+	.config = &hawi_lpass_lpicx_noc_regmap_config,
+	.nodes = lpass_lpicx_noc_nodes,
+	.num_nodes = ARRAY_SIZE(lpass_lpicx_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const mc_virt_bcms[] = {
+	&bcm_acv,
+	&bcm_mc0,
+	&bcm_mc5,
+};
+
+static struct qcom_icc_node * const mc_virt_nodes[] = {
+	[MASTER_LLCC] = &llcc_mc,
+	[MASTER_DDR_RT] = &ddr_rt_mc,
+	[SLAVE_EBI1] = &ebi,
+	[SLAVE_DDR_RT] = &ddr_rt_slave,
+};
+
+static const struct qcom_icc_desc hawi_mc_virt = {
+	.nodes = mc_virt_nodes,
+	.num_nodes = ARRAY_SIZE(mc_virt_nodes),
+	.bcms = mc_virt_bcms,
+	.num_bcms = ARRAY_SIZE(mc_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const mmss_noc_bcms[] = {
+	&bcm_mm0,
+	&bcm_mm1,
+};
+
+static struct qcom_icc_node * const mmss_noc_nodes[] = {
+	[MASTER_CAMNOC_HF] = &qnm_camnoc_hf,
+	[MASTER_CAMNOC_NRT_ICP_SF] = &qnm_camnoc_nrt_icp_sf,
+	[MASTER_CAMNOC_RT_CDM_SF] = &qnm_camnoc_rt_cdm_sf,
+	[MASTER_CAMNOC_SF] = &qnm_camnoc_sf,
+	[MASTER_MDP] = &qnm_mdp,
+	[MASTER_MDSS_DCP] = &qnm_mdss_dcp,
+	[MASTER_CDSP_HCP] = &qnm_vapss_hcp,
+	[MASTER_VIDEO_CV_PROC] = &qnm_video_cv_cpu,
+	[MASTER_VIDEO_EVA] = &qnm_video_eva,
+	[MASTER_VIDEO_MVP] = &qnm_video_mvp,
+	[MASTER_VIDEO_V_PROC] = &qnm_video_v_cpu,
+	[SLAVE_MNOC_HF_MEM_NOC] = &qns_mem_noc_hf,
+	[SLAVE_MNOC_SF_MEM_NOC] = &qns_mem_noc_sf,
+};
+
+static const struct regmap_config hawi_mmss_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x5f800,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_mmss_noc = {
+	.config = &hawi_mmss_noc_regmap_config,
+	.nodes = mmss_noc_nodes,
+	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
+	.bcms = mmss_noc_bcms,
+	.num_bcms = ARRAY_SIZE(mmss_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const nsp_noc_bcms[] = {
+	&bcm_co0,
+};
+
+static struct qcom_icc_node * const nsp_noc_nodes[] = {
+	[MASTER_CDSP_PROC] = &qnm_nsp,
+	[SLAVE_CDSP_MEM_NOC] = &qns_nsp_gemnoc,
+};
+
+static const struct regmap_config hawi_nsp_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x21280,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_nsp_noc = {
+	.config = &hawi_nsp_noc_regmap_config,
+	.nodes = nsp_noc_nodes,
+	.num_nodes = ARRAY_SIZE(nsp_noc_nodes),
+	.bcms = nsp_noc_bcms,
+	.num_bcms = ARRAY_SIZE(nsp_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const pcie_anoc_bcms[] = {
+	&bcm_sn3,
+};
+
+static struct qcom_icc_node * const pcie_anoc_nodes[] = {
+	[MASTER_PCIE_ANOC_CFG] = &qsm_pcie_anoc_cfg,
+	[MASTER_PCIE_0] = &xm_pcie,
+	[MASTER_PCIE_1] = &xm_pcie_g4x1,
+	[SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_gemnoc,
+	[SLAVE_SERVICE_PCIE_ANOC] = &srvc_pcie_aggre_noc,
+};
+
+static const struct regmap_config hawi_pcie_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x12400,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_pcie_anoc = {
+	.config = &hawi_pcie_anoc_regmap_config,
+	.nodes = pcie_anoc_nodes,
+	.num_nodes = ARRAY_SIZE(pcie_anoc_nodes),
+	.bcms = pcie_anoc_bcms,
+	.num_bcms = ARRAY_SIZE(pcie_anoc_bcms),
+	.qos_requires_clocks = true,
+};
+
+static struct qcom_icc_bcm * const stdst_cfg_bcms[] = {
+	&bcm_cn0,
+	&bcm_cn1,
+};
+
+static struct qcom_icc_node * const stdst_cfg_nodes[] = {
+	[MASTER_CFG_CENTER] = &qsm_cfg_center,
+	[MASTER_CFG_EAST] = &qsm_cfg_east,
+	[MASTER_CFG_MM] = &qsm_cfg_mm,
+	[MASTER_CFG_NORTH] = &qsm_cfg_north,
+	[MASTER_CFG_SOUTH] = &qsm_cfg_south,
+	[MASTER_CFG_SOUTHWEST] = &qsm_cfg_southwest,
+	[SLAVE_AHB2PHY_SOUTH] = &qhs_ahb2phy0,
+	[SLAVE_BOOT_ROM] = &qhs_boot_rom,
+	[SLAVE_CAMERA_CFG] = &qhs_camera_cfg,
+	[SLAVE_CLK_CTL] = &qhs_clk_ctl,
+	[SLAVE_CRYPTO_CFG] = &qhs_crypto_cfg,
+	[SLAVE_DISPLAY_CFG] = &qhs_display_cfg,
+	[SLAVE_EVA_CFG] = &qhs_eva_cfg,
+	[SLAVE_GFX3D_CFG] = &qhs_gpuss_cfg,
+	[SLAVE_I2C] = &qhs_i2c,
+	[SLAVE_IMEM_CFG] = &qhs_imem_cfg,
+	[SLAVE_IPC_ROUTER_CFG] = &qhs_ipc_router,
+	[SLAVE_IRIS_CFG] = &qhs_iris_cfg,
+	[SLAVE_CNOC_MSS] = &qhs_mss_cfg,
+	[SLAVE_PCIE_0_CFG] = &qhs_pcie_cfg,
+	[SLAVE_PCIE_1_CFG] = &qhs_pcie_g4x1_cfg,
+	[SLAVE_PRNG] = &qhs_prng,
+	[SLAVE_QSPI_0] = &qhs_qspi,
+	[SLAVE_QUP_1] = &qhs_qup1,
+	[SLAVE_QUP_2] = &qhs_qup2,
+	[SLAVE_QUP_3] = &qhs_qup3,
+	[SLAVE_QUP_4] = &qhs_qup4,
+	[SLAVE_SDCC_2] = &qhs_sdc2,
+	[SLAVE_SDCC_4] = &qhs_sdc4,
+	[SLAVE_TLMM] = &qhs_tlmm,
+	[SLAVE_UFS_MEM_CFG] = &qhs_ufs_mem_cfg,
+	[SLAVE_USB3] = &qhs_usb3,
+	[SLAVE_VSENSE_CTRL_CFG] = &qhs_vsense_ctrl_cfg,
+	[SLAVE_PCIE_ANOC_CFG] = &qss_pcie_anoc_cfg,
+	[SLAVE_QDSS_CFG] = &qss_qdss_cfg,
+	[SLAVE_QDSS_STM] = &qss_qdss_stm,
+	[SLAVE_TCSR] = &qss_tcsr,
+	[SLAVE_TCU] = &xs_sys_tcu_cfg,
+};
+
+static const struct regmap_config hawi_stdst_cfg_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xb1000,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_stdst_cfg = {
+	.config = &hawi_stdst_cfg_regmap_config,
+	.nodes = stdst_cfg_nodes,
+	.num_nodes = ARRAY_SIZE(stdst_cfg_nodes),
+	.bcms = stdst_cfg_bcms,
+	.num_bcms = ARRAY_SIZE(stdst_cfg_bcms),
+};
+
+static struct qcom_icc_node * const stdst_main_nodes[] = {
+	[MASTER_CNOC_STARDUST] = &qsm_cnoc_main,
+	[SLAVE_STARDUST_CENTER_CFG] = &qss_stdst_center_cfg,
+	[SLAVE_STARDUST_EAST_CFG] = &qss_stdst_east_cfg,
+	[SLAVE_STARDUST_MM_CFG] = &qss_stdst_mm_cfg,
+	[SLAVE_STARDUST_NORTH_CFG] = &qss_stdst_north_cfg,
+	[SLAVE_STARDUST_SOUTH_CFG] = &qss_stdst_south_cfg,
+	[SLAVE_STARDUST_SOUTHWEST_CFG] = &qss_stdst_southwest_cfg,
+};
+
+static const struct regmap_config hawi_stdst_main_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x10000,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_stdst_main = {
+	.config = &hawi_stdst_main_regmap_config,
+	.nodes = stdst_main_nodes,
+	.num_nodes = ARRAY_SIZE(stdst_main_nodes),
+};
+
+static struct qcom_icc_bcm * const system_noc_bcms[] = {
+	&bcm_sn0,
+	&bcm_sn2,
+};
+
+static struct qcom_icc_node * const system_noc_nodes[] = {
+	[MASTER_A1NOC_SNOC] = &qnm_aggre_noc,
+	[MASTER_APSS_NOC] = &qnm_apss_noc,
+	[MASTER_CNOC_SNOC] = &qnm_cnoc_data,
+	[SLAVE_SNOC_GEM_NOC_SF] = &qns_gemnoc_sf,
+};
+
+static const struct regmap_config hawi_system_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x20080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc hawi_system_noc = {
+	.config = &hawi_system_noc_regmap_config,
+	.nodes = system_noc_nodes,
+	.num_nodes = ARRAY_SIZE(system_noc_nodes),
+	.bcms = system_noc_bcms,
+	.num_bcms = ARRAY_SIZE(system_noc_bcms),
+};
+
+static const struct of_device_id qnoc_of_match[] = {
+	{ .compatible = "qcom,hawi-aggre1-noc", .data = &hawi_aggre1_noc },
+	{ .compatible = "qcom,hawi-clk-virt", .data = &hawi_clk_virt },
+	{ .compatible = "qcom,hawi-cnoc-main", .data = &hawi_cnoc_main },
+	{ .compatible = "qcom,hawi-gem-noc", .data = &hawi_gem_noc },
+	{ .compatible = "qcom,hawi-llclpi-noc", .data = &hawi_llclpi_noc },
+	{ .compatible = "qcom,hawi-lpass-ag-noc", .data = &hawi_lpass_ag_noc },
+	{ .compatible = "qcom,hawi-lpass-lpiaon-noc", .data = &hawi_lpass_lpiaon_noc },
+	{ .compatible = "qcom,hawi-lpass-lpicx-noc", .data = &hawi_lpass_lpicx_noc },
+	{ .compatible = "qcom,hawi-mc-virt", .data = &hawi_mc_virt },
+	{ .compatible = "qcom,hawi-mmss-noc", .data = &hawi_mmss_noc },
+	{ .compatible = "qcom,hawi-nsp-noc", .data = &hawi_nsp_noc },
+	{ .compatible = "qcom,hawi-pcie-anoc", .data = &hawi_pcie_anoc },
+	{ .compatible = "qcom,hawi-stdst-cfg", .data = &hawi_stdst_cfg },
+	{ .compatible = "qcom,hawi-stdst-main", .data = &hawi_stdst_main },
+	{ .compatible = "qcom,hawi-system-noc", .data = &hawi_system_noc },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qnoc_of_match);
+
+static struct platform_driver qnoc_driver = {
+	.probe = qcom_icc_rpmh_probe,
+	.remove = qcom_icc_rpmh_remove,
+	.driver = {
+		.name = "qnoc-hawi",
+		.of_match_table = qnoc_of_match,
+		.sync_state = icc_sync_state,
+	},
+};
+
+static int __init qnoc_driver_init(void)
+{
+	return platform_driver_register(&qnoc_driver);
+}
+core_initcall(qnoc_driver_init);
+
+static void __exit qnoc_driver_exit(void)
+{
+	platform_driver_unregister(&qnoc_driver);
+}
+module_exit(qnoc_driver_exit);
+
+MODULE_DESCRIPTION("Qualcomm Hawi NoC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/interconnect/qcom/msm8953.c b/drivers/interconnect/qcom/msm8953.c
index be2b1a606612c..94a9773d29702 100644
--- a/drivers/interconnect/qcom/msm8953.c
+++ b/drivers/interconnect/qcom/msm8953.c
@@ -1307,6 +1307,7 @@ static const struct of_device_id msm8953_noc_of_match[] = {
 	{ .compatible = "qcom,msm8953-snoc-mm", .data = &msm8953_snoc_mm },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, msm8953_noc_of_match);
 
 static struct platform_driver msm8953_noc_driver = {
 	.probe = qnoc_probe,
@@ -1318,6 +1319,5 @@ static struct platform_driver msm8953_noc_driver = {
 };
 
 module_platform_driver(msm8953_noc_driver);
-MODULE_DEVICE_TABLE(of, msm8953_noc_of_match);
 MODULE_DESCRIPTION("Qualcomm MSM8953 NoC driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/interconnect/qcom/nord.c b/drivers/interconnect/qcom/nord.c
new file mode 100644
index 0000000000000..b3ca5675e7c86
--- /dev/null
+++ b/drivers/interconnect/qcom/nord.c
@@ -0,0 +1,2682 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ */
+
+#include <linux/device.h>
+#include <linux/interconnect.h>
+#include <linux/interconnect-provider.h>
+#include <linux/module.h>
+#include <linux/of_platform.h>
+#include <dt-bindings/interconnect/qcom,nord-rpmh.h>
+
+#include "bcm-voter.h"
+#include "icc-rpmh.h"
+
+static struct qcom_icc_node qup0_core_slave = {
+	.name = "qup0_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup1_core_slave = {
+	.name = "qup1_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup2_core_slave = {
+	.name = "qup2_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qup3_core_slave = {
+	.name = "qup3_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node ps_eth_0 = {
+	.name = "ps_eth_0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node ps_eth_1 = {
+	.name = "ps_eth_1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node ps_shs_server = {
+	.name = "ps_shs_server",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy0 = {
+	.name = "qhs_ahb2phy0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy1 = {
+	.name = "qhs_ahb2phy1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy2 = {
+	.name = "qhs_ahb2phy2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy3 = {
+	.name = "qhs_ahb2phy3",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy_eth_0 = {
+	.name = "qhs_ahb2phy_eth_0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ahb2phy_eth_1 = {
+	.name = "qhs_ahb2phy_eth_1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_camera_cfg = {
+	.name = "qhs_camera_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_clk_ctl = {
+	.name = "qhs_clk_ctl",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_crypto0_cfg = {
+	.name = "qhs_crypto0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_crypto1_cfg = {
+	.name = "qhs_crypto1_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_crypto2_cfg = {
+	.name = "qhs_crypto2_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_display_1_cfg = {
+	.name = "qhs_display_1_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_display_cfg = {
+	.name = "qhs_display_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_dprx0 = {
+	.name = "qhs_dprx0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_dprx1 = {
+	.name = "qhs_dprx1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_eva_cfg = {
+	.name = "qhs_eva_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_gpuss_0_cfg = {
+	.name = "qhs_gpuss_0_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qhs_gpuss_1_cfg = {
+	.name = "qhs_gpuss_1_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qhs_i2c = {
+	.name = "qhs_i2c",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_imem_cfg = {
+	.name = "qhs_imem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_mcw_pcie = {
+	.name = "qhs_mcw_pcie",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_mm_rscc = {
+	.name = "qhs_mm_rscc",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ne_clk_ctl = {
+	.name = "qhs_ne_clk_ctl",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_nspss0_cfg = {
+	.name = "qhs_nspss0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_nspss1_cfg = {
+	.name = "qhs_nspss1_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_nspss2_cfg = {
+	.name = "qhs_nspss2_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_nspss3_cfg = {
+	.name = "qhs_nspss3_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_nw_clk_ctl = {
+	.name = "qhs_nw_clk_ctl",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_prng = {
+	.name = "qhs_prng",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qdss_cfg = {
+	.name = "qhs_qdss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qspi = {
+	.name = "qhs_qspi",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup0 = {
+	.name = "qhs_qup0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup3 = {
+	.name = "qhs_qup3",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup1 = {
+	.name = "qhs_qup1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_qup2 = {
+	.name = "qhs_qup2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_safedma_cfg = {
+	.name = "qhs_safedma_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_sdc4 = {
+	.name = "qhs_sdc4",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_se_clk_ctl = {
+	.name = "qhs_se_clk_ctl",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tcsr = {
+	.name = "qhs_tcsr",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tlmm = {
+	.name = "qhs_tlmm",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tsc_cfg = {
+	.name = "qhs_tsc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ufs_mem_cfg = {
+	.name = "qhs_ufs_mem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_usb2 = {
+	.name = "qhs_usb2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_usb3_0 = {
+	.name = "qhs_usb3_0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_usb3_1 = {
+	.name = "qhs_usb3_1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_venus_cfg = {
+	.name = "qhs_venus_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_computenoc_cfg = {
+	.name = "qss_computenoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_qtc_cfg = {
+	.name = "qss_qtc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node xs_qdss_stm = {
+	.name = "xs_qdss_stm",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node xs_sys_tcu0_cfg = {
+	.name = "xs_sys_tcu0_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node xs_sys_tcu1_cfg = {
+	.name = "xs_sys_tcu1_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node xs_sys_tcu2_cfg = {
+	.name = "xs_sys_tcu2_cfg",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qhs_aoss = {
+	.name = "qhs_aoss",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_hbcu = {
+	.name = "qhs_hbcu",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipa = {
+	.name = "qhs_ipa",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_ipc_router = {
+	.name = "qhs_ipc_router",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_soccp = {
+	.name = "qhs_soccp",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_tme_cfg = {
+	.name = "qhs_tme_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qss_ddrss_cfg = {
+	.name = "qss_ddrss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qxs_imem = {
+	.name = "qxs_imem",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node ebi = {
+	.name = "ebi",
+	.channels = 16,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_ahb2phy_cfg = {
+	.name = "qhs_pcie_ahb2phy_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_cfg_0 = {
+	.name = "qhs_pcie_cfg_0",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_cfg_1 = {
+	.name = "qhs_pcie_cfg_1",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_cfg_2 = {
+	.name = "qhs_pcie_cfg_2",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_cfg_3 = {
+	.name = "qhs_pcie_cfg_3",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_dma_0_cfg = {
+	.name = "qhs_pcie_dma_0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_dma_1_cfg = {
+	.name = "qhs_pcie_dma_1_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qhs_pcie_dma_2_cfg = {
+	.name = "qhs_pcie_dma_2_cfg",
+	.channels = 1,
+	.buswidth = 4,
+};
+
+static struct qcom_icc_node qxs_pcie_dma_0 = {
+	.name = "qxs_pcie_dma_0",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qxs_pcie_dma_1 = {
+	.name = "qxs_pcie_dma_1",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qxs_pcie_dma_2 = {
+	.name = "qxs_pcie_dma_2",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node xs_pcie_0 = {
+	.name = "xs_pcie_0",
+	.channels = 1,
+	.buswidth = 32,
+};
+
+static struct qcom_icc_node xs_pcie_1 = {
+	.name = "xs_pcie_1",
+	.channels = 1,
+	.buswidth = 32,
+};
+
+static struct qcom_icc_node xs_pcie_2 = {
+	.name = "xs_pcie_2",
+	.channels = 1,
+	.buswidth = 16,
+};
+
+static struct qcom_icc_node xs_pcie_3 = {
+	.name = "xs_pcie_3",
+	.channels = 1,
+	.buswidth = 8,
+};
+
+static struct qcom_icc_node qup0_core_master = {
+	.name = "qup0_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup0_core_slave },
+};
+
+static struct qcom_icc_node qup1_core_master = {
+	.name = "qup1_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup1_core_slave },
+};
+
+static struct qcom_icc_node qup2_core_master = {
+	.name = "qup2_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup2_core_slave },
+};
+
+static struct qcom_icc_node qup3_core_master = {
+	.name = "qup3_core_master",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qup3_core_slave },
+};
+
+static struct qcom_icc_node llcc_mc = {
+	.name = "llcc_mc",
+	.channels = 16,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &ebi },
+};
+
+static struct qcom_icc_node qsm_pcie_noc_cfg = {
+	.name = "qsm_pcie_noc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 8,
+	.link_nodes = { &qhs_pcie_ahb2phy_cfg, &qhs_pcie_cfg_0,
+			&qhs_pcie_cfg_1, &qhs_pcie_cfg_2,
+			&qhs_pcie_cfg_3, &qhs_pcie_dma_0_cfg,
+			&qhs_pcie_dma_1_cfg, &qhs_pcie_dma_2_cfg },
+};
+
+static struct qcom_icc_node qnm_cnoc_pcie_dma = {
+	.name = "qnm_cnoc_pcie_dma",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 3,
+	.link_nodes = { &qxs_pcie_dma_0, &qxs_pcie_dma_1,
+			&qxs_pcie_dma_2 },
+};
+
+static struct qcom_icc_node qnm_hscnoc_pcie = {
+	.name = "qnm_hscnoc_pcie",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 4,
+	.link_nodes = { &xs_pcie_0, &xs_pcie_1,
+			&xs_pcie_2, &xs_pcie_3 },
+};
+
+static struct qcom_icc_node qnm_pcie_ibnoc_dma = {
+	.name = "qnm_pcie_ibnoc_dma",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &xs_pcie_0 },
+};
+
+static struct qcom_icc_node qss_pcie_noc_cfg = {
+	.name = "qss_pcie_noc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_pcie_noc_cfg },
+};
+
+static struct qcom_icc_node qns_pcie_dma = {
+	.name = "qns_pcie_dma",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_cnoc_pcie_dma },
+};
+
+static struct qcom_icc_node qns_llcc = {
+	.name = "qns_llcc",
+	.channels = 16,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &llcc_mc },
+};
+
+static struct qcom_icc_node qns_pcie = {
+	.name = "qns_pcie",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_hscnoc_pcie },
+};
+
+static struct qcom_icc_node qns_pcie_obnoc_dma = {
+	.name = "qns_pcie_obnoc_dma",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_pcie_ibnoc_dma },
+};
+
+static struct qcom_icc_node qsm_cfg = {
+	.name = "qsm_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 56,
+	.link_nodes = { &ps_eth_0, &ps_eth_1,
+			&ps_shs_server, &qhs_ahb2phy0,
+			&qhs_ahb2phy1, &qhs_ahb2phy2,
+			&qhs_ahb2phy3, &qhs_ahb2phy_eth_0,
+			&qhs_ahb2phy_eth_1, &qhs_camera_cfg,
+			&qhs_clk_ctl, &qhs_crypto0_cfg,
+			&qhs_crypto1_cfg, &qhs_crypto2_cfg,
+			&qhs_display_1_cfg, &qhs_display_cfg,
+			&qhs_dprx0, &qhs_dprx1,
+			&qhs_eva_cfg, &qhs_gpuss_0_cfg,
+			&qhs_gpuss_1_cfg, &qhs_i2c,
+			&qhs_imem_cfg, &qhs_mcw_pcie,
+			&qhs_mm_rscc, &qhs_ne_clk_ctl,
+			&qhs_nspss0_cfg, &qhs_nspss1_cfg,
+			&qhs_nspss2_cfg, &qhs_nspss3_cfg,
+			&qhs_nw_clk_ctl, &qhs_prng,
+			&qhs_qdss_cfg, &qhs_qspi,
+			&qhs_qup0, &qhs_qup3,
+			&qhs_qup1, &qhs_qup2,
+			&qhs_safedma_cfg, &qhs_sdc4,
+			&qhs_se_clk_ctl, &qhs_tcsr,
+			&qhs_tlmm, &qhs_tsc_cfg,
+			&qhs_ufs_mem_cfg, &qhs_usb2,
+			&qhs_usb3_0, &qhs_usb3_1,
+			&qhs_venus_cfg, &qss_computenoc_cfg,
+			&qss_pcie_noc_cfg, &qss_qtc_cfg,
+			&xs_qdss_stm, &xs_sys_tcu0_cfg,
+			&xs_sys_tcu1_cfg, &xs_sys_tcu2_cfg },
+};
+
+static struct qcom_icc_node xm_gic = {
+	.name = "xm_gic",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa44000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_node qss_cfg = {
+	.name = "qss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qsm_cfg },
+};
+
+static struct qcom_icc_node qhm_mm_rscc = {
+	.name = "qhm_mm_rscc",
+	.channels = 1,
+	.buswidth = 4,
+	.num_links = 1,
+	.link_nodes = { &qss_cfg },
+};
+
+static struct qcom_icc_node qnm_hscnoc = {
+	.name = "qnm_hscnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 10,
+	.link_nodes = { &qhs_aoss, &qhs_hbcu,
+			&qhs_ipa, &qhs_ipc_router,
+			&qhs_soccp, &qhs_tme_cfg,
+			&qns_pcie_dma, &qss_cfg,
+			&qss_ddrss_cfg, &qxs_imem },
+};
+
+static struct qcom_icc_node qns_hscnoc_cnoc = {
+	.name = "qns_hscnoc_cnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_hscnoc },
+};
+
+static struct qcom_icc_node alm_gpu_tcu = {
+	.name = "alm_gpu_tcu",
+	.channels = 2,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x930000, 0xa45000 },
+		.prio = 1,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node alm_qtc = {
+	.name = "alm_qtc",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2c0000 },
+		.prio = 3,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node alm_sys_tcu0 = {
+	.name = "alm_sys_tcu0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa42000 },
+		.prio = 6,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node alm_sys_tcu1 = {
+	.name = "alm_sys_tcu1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x81f000 },
+		.prio = 6,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node alm_sys_tcu2 = {
+	.name = "alm_sys_tcu2",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x30000 },
+		.prio = 6,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node chm_apps = {
+	.name = "chm_apps",
+	.channels = 6,
+	.buswidth = 32,
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_aggre_north = {
+	.name = "qnm_aggre_north",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x935000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_aggre_south = {
+	.name = "qnm_aggre_south",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x31000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_gpu0 = {
+	.name = "qnm_gpu0",
+	.channels = 4,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 4,
+		.port_offsets = { 0x931000, 0x932000, 0x933000, 0x934000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_gpu1 = {
+	.name = "qnm_gpu1",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0xa40000, 0xa41000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_hpass_adas_hscnoc = {
+	.name = "qnm_hpass_adas_hscnoc",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x240000, 0x245000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_hpass_audio_hscnoc = {
+	.name = "qnm_hpass_audio_hscnoc",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x241000 },
+		.prio = 3,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_mnoc_hf = {
+	.name = "qnm_mnoc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x81d000, 0x820000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_mnoc_sf = {
+	.name = "qnm_mnoc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x81e000, 0x821000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_nsp0_hscnoc = {
+	.name = "qnm_nsp0_hscnoc",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x32000, 0x33000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_nsp1_hscnoc = {
+	.name = "qnm_nsp1_hscnoc",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x34000, 0x35000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_nsp2_hscnoc = {
+	.name = "qnm_nsp2_hscnoc",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x36000, 0x37000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_nsp3_hscnoc = {
+	.name = "qnm_nsp3_hscnoc",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x38000, 0x39000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_pcie = {
+	.name = "qnm_pcie",
+	.channels = 1,
+	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2c1000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc },
+};
+
+static struct qcom_icc_node qnm_sailss_md0_hscnoc = {
+	.name = "qnm_sailss_md0_hscnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x243000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qnm_snoc_sf = {
+	.name = "qnm_snoc_sf",
+	.channels = 1,
+	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa43000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 3,
+	.link_nodes = { &qns_hscnoc_cnoc, &qns_llcc,
+			&qns_pcie },
+};
+
+static struct qcom_icc_node qns_a1noc_hscnoc = {
+	.name = "qns_a1noc_hscnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_aggre_north },
+};
+
+static struct qcom_icc_node qns_a2noc_hscnoc = {
+	.name = "qns_a2noc_hscnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_aggre_south },
+};
+
+static struct qcom_icc_node qns_hpass_agnoc_audio = {
+	.name = "qns_hpass_agnoc_audio",
+	.channels = 1,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_hpass_audio_hscnoc },
+};
+
+static struct qcom_icc_node qns_mem_noc_hf = {
+	.name = "qns_mem_noc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_mnoc_hf },
+};
+
+static struct qcom_icc_node qns_mem_noc_sf = {
+	.name = "qns_mem_noc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_mnoc_sf },
+};
+
+static struct qcom_icc_node qns_nsp0_hsc_noc = {
+	.name = "qns_nsp0_hsc_noc",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_nsp0_hscnoc },
+};
+
+static struct qcom_icc_node qns_nsp1_hsc_noc = {
+	.name = "qns_nsp1_hsc_noc",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_nsp1_hscnoc },
+};
+
+static struct qcom_icc_node qns_nsp2_hsc_noc = {
+	.name = "qns_nsp2_hsc_noc",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_nsp2_hscnoc },
+};
+
+static struct qcom_icc_node qns_nsp3_hsc_noc = {
+	.name = "qns_nsp3_hsc_noc",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qnm_nsp3_hscnoc },
+};
+
+static struct qcom_icc_node qns_pcie_hscnoc = {
+	.name = "qns_pcie_hscnoc",
+	.channels = 1,
+	.buswidth = 64,
+	.num_links = 1,
+	.link_nodes = { &qnm_pcie },
+};
+
+static struct qcom_icc_node qns_hscnoc_sf = {
+	.name = "qns_hscnoc_sf",
+	.channels = 1,
+	.buswidth = 64,
+	.num_links = 1,
+	.link_nodes = { &qnm_snoc_sf },
+};
+
+static struct qcom_icc_node qhm_qup2 = {
+	.name = "qhm_qup2",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1b000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_hscnoc },
+};
+
+static struct qcom_icc_node qxm_crypto_0 = {
+	.name = "qxm_crypto_0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1c000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_hscnoc },
+};
+
+static struct qcom_icc_node qxm_crypto_1 = {
+	.name = "qxm_crypto_1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1d000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_hscnoc },
+};
+
+static struct qcom_icc_node qxm_crypto_2 = {
+	.name = "qxm_crypto_2",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1e000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_hscnoc },
+};
+
+static struct qcom_icc_node xm_sdc4 = {
+	.name = "xm_sdc4",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_hscnoc },
+};
+
+static struct qcom_icc_node xm_ufs_mem = {
+	.name = "xm_ufs_mem",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x17000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_hscnoc },
+};
+
+static struct qcom_icc_node xm_usb2 = {
+	.name = "xm_usb2",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x19000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_hscnoc },
+};
+
+static struct qcom_icc_node xm_usb3_0 = {
+	.name = "xm_usb3_0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x18000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_hscnoc },
+};
+
+static struct qcom_icc_node xm_usb3_1 = {
+	.name = "xm_usb3_1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1a000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_hscnoc },
+};
+
+static struct qcom_icc_node qhm_qup0 = {
+	.name = "qhm_qup0",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x13000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_hscnoc },
+};
+
+static struct qcom_icc_node qhm_qup1 = {
+	.name = "qhm_qup1",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_hscnoc },
+};
+
+static struct qcom_icc_node xm_emac_0 = {
+	.name = "xm_emac_0",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_hscnoc },
+};
+
+static struct qcom_icc_node xm_emac_1 = {
+	.name = "xm_emac_1",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x17000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_hscnoc },
+};
+
+static struct qcom_icc_node qnm_hpass_dsp0 = {
+	.name = "qnm_hpass_dsp0",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_hpass_agnoc_audio },
+};
+
+static struct qcom_icc_node qnm_hpass_dsp1 = {
+	.name = "qnm_hpass_dsp1",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_hpass_agnoc_audio },
+};
+
+static struct qcom_icc_node qnm_hpass_dsp2 = {
+	.name = "qnm_hpass_dsp2",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_hpass_agnoc_audio },
+};
+
+static struct qcom_icc_node qnm_camnoc_hf = {
+	.name = "qnm_camnoc_hf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x57000, 0x58000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_camnoc_nrt_icp_sf = {
+	.name = "qnm_camnoc_nrt_icp_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1a000 },
+		.prio = 4,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_camnoc_rt_cdm_sf = {
+	.name = "qnm_camnoc_rt_cdm_sf",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x5b000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_camnoc_sf = {
+	.name = "qnm_camnoc_sf",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x1b000, 0x1c000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_dprx0 = {
+	.name = "qnm_dprx0",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x5c000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_dprx1 = {
+	.name = "qnm_dprx1",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x5d000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_mdp0 = {
+	.name = "qnm_mdp0",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x59000, 0x5a000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_mdp1 = {
+	.name = "qnm_mdp1",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x5e000, 0x5f000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_node qnm_video_cv_cpu = {
+	.name = "qnm_video_cv_cpu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x21000 },
+		.prio = 4,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_eva = {
+	.name = "qnm_video_eva",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x22000, 0x23000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_mvp0 = {
+	.name = "qnm_video_mvp0",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x1d000, 0x1e000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_mvp1 = {
+	.name = "qnm_video_mvp1",
+	.channels = 2,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x1f000, 0x20000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_video_v_cpu = {
+	.name = "qnm_video_v_cpu",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x24000 },
+		.prio = 4,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_node qnm_nsp_data00 = {
+	.name = "qnm_nsp_data00",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_nsp0_hsc_noc },
+};
+
+static struct qcom_icc_node qnm_nsp_data01 = {
+	.name = "qnm_nsp_data01",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_nsp1_hsc_noc },
+};
+
+static struct qcom_icc_node qnm_nsp_data02 = {
+	.name = "qnm_nsp_data02",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_nsp2_hsc_noc },
+};
+
+static struct qcom_icc_node qnm_nsp_data03 = {
+	.name = "qnm_nsp_data03",
+	.channels = 2,
+	.buswidth = 32,
+	.num_links = 1,
+	.link_nodes = { &qns_nsp3_hsc_noc },
+};
+
+static struct qcom_icc_node qxm_pcie_dma_0 = {
+	.name = "qxm_pcie_dma_0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x49000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_pcie_hscnoc, &qns_pcie_obnoc_dma },
+};
+
+static struct qcom_icc_node qxm_pcie_dma_1 = {
+	.name = "qxm_pcie_dma_1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4a000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_pcie_hscnoc, &qns_pcie_obnoc_dma },
+};
+
+static struct qcom_icc_node qxm_pcie_dma_2 = {
+	.name = "qxm_pcie_dma_2",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x4b000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 2,
+	.link_nodes = { &qns_pcie_hscnoc, &qns_pcie_obnoc_dma },
+};
+
+static struct qcom_icc_node xm_pcie_0 = {
+	.name = "xm_pcie_0",
+	.channels = 1,
+	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x18000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_hscnoc },
+};
+
+static struct qcom_icc_node xm_pcie_1 = {
+	.name = "xm_pcie_1",
+	.channels = 1,
+	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x19000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_hscnoc },
+};
+
+static struct qcom_icc_node xm_pcie_2 = {
+	.name = "xm_pcie_2",
+	.channels = 1,
+	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1a000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_hscnoc },
+};
+
+static struct qcom_icc_node xm_pcie_3 = {
+	.name = "xm_pcie_3",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1b000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_pcie_hscnoc },
+};
+
+static struct qcom_icc_node qnm_aggre1_noc = {
+	.name = "qnm_aggre1_noc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qns_hscnoc_sf },
+};
+
+static struct qcom_icc_node qnm_aggre2_noc = {
+	.name = "qnm_aggre2_noc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qns_hscnoc_sf },
+};
+
+static struct qcom_icc_node qnm_cnoc_data = {
+	.name = "qnm_cnoc_data",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1a000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_hscnoc_sf },
+};
+
+static struct qcom_icc_node qnm_nsi_noc = {
+	.name = "qnm_nsi_noc",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x19000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_hscnoc_sf },
+};
+
+static struct qcom_icc_node qnm_safe_dma = {
+	.name = "qnm_safe_dma",
+	.channels = 1,
+	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1b000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_hscnoc_sf },
+};
+
+static struct qcom_icc_node qns_a1noc_snoc = {
+	.name = "qns_a1noc_snoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_aggre1_noc },
+};
+
+static struct qcom_icc_node qns_a2noc_snoc = {
+	.name = "qns_a2noc_snoc",
+	.channels = 1,
+	.buswidth = 16,
+	.num_links = 1,
+	.link_nodes = { &qnm_aggre2_noc },
+};
+
+static struct qcom_icc_node qhm_qspi = {
+	.name = "qhm_qspi",
+	.channels = 1,
+	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x18000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qnm_sailss_md1 = {
+	.name = "qnm_sailss_md1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_qup3 = {
+	.name = "qxm_qup3",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x17000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a1noc_snoc },
+};
+
+static struct qcom_icc_node qxm_ipa = {
+	.name = "qxm_ipa",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x13000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node qxm_soccp = {
+	.name = "qxm_soccp",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node xm_qdss_etr_0 = {
+	.name = "xm_qdss_etr_0",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_node xm_qdss_etr_1 = {
+	.name = "xm_qdss_etr_1",
+	.channels = 1,
+	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
+	.num_links = 1,
+	.link_nodes = { &qns_a2noc_snoc },
+};
+
+static struct qcom_icc_bcm bcm_c0n0 = {
+	.name = "C0N0",
+	.enable_mask = BIT(0),
+	.num_nodes = 2,
+	.nodes = { &qnm_nsp_data00, &qns_nsp0_hsc_noc },
+};
+
+static struct qcom_icc_bcm bcm_c1n0 = {
+	.name = "C1N0",
+	.enable_mask = BIT(0),
+	.num_nodes = 2,
+	.nodes = { &qnm_nsp_data01, &qns_nsp1_hsc_noc },
+};
+
+static struct qcom_icc_bcm bcm_c2n0 = {
+	.name = "C2N0",
+	.enable_mask = BIT(0),
+	.num_nodes = 2,
+	.nodes = { &qnm_nsp_data02, &qns_nsp2_hsc_noc },
+};
+
+static struct qcom_icc_bcm bcm_c3n0 = {
+	.name = "C3N0",
+	.enable_mask = BIT(0),
+	.num_nodes = 2,
+	.nodes = { &qnm_nsp_data03, &qns_nsp3_hsc_noc },
+};
+
+static struct qcom_icc_bcm bcm_ce0 = {
+	.name = "CE0",
+	.num_nodes = 1,
+	.nodes = { &qxm_crypto_0 },
+};
+
+static struct qcom_icc_bcm bcm_ce1 = {
+	.name = "CE1",
+	.num_nodes = 1,
+	.nodes = { &qxm_crypto_1 },
+};
+
+static struct qcom_icc_bcm bcm_ce2 = {
+	.name = "CE2",
+	.num_nodes = 1,
+	.nodes = { &qxm_crypto_2 },
+};
+
+static struct qcom_icc_bcm bcm_cn0 = {
+	.name = "CN0",
+	.keepalive = true,
+	.enable_mask = BIT(0),
+	.num_nodes = 6,
+	.nodes = { &qsm_cfg, &qhm_mm_rscc,
+		   &qnm_hscnoc, &qnm_cnoc_pcie_dma,
+		   &qnm_hscnoc_pcie, &qnm_pcie_ibnoc_dma },
+};
+
+static struct qcom_icc_bcm bcm_cn1 = {
+	.name = "CN1",
+	.num_nodes = 2,
+	.nodes = { &qhs_display_1_cfg, &qhs_display_cfg },
+};
+
+static struct qcom_icc_bcm bcm_mc0 = {
+	.name = "MC0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &ebi },
+};
+
+static struct qcom_icc_bcm bcm_mm0 = {
+	.name = "MM0",
+	.num_nodes = 1,
+	.nodes = { &qns_mem_noc_hf },
+};
+
+static struct qcom_icc_bcm bcm_mm1 = {
+	.name = "MM1",
+	.enable_mask = BIT(0),
+	.num_nodes = 14,
+	.nodes = { &qnm_camnoc_hf, &qnm_camnoc_nrt_icp_sf,
+		   &qnm_camnoc_rt_cdm_sf, &qnm_camnoc_sf,
+		   &qnm_dprx0, &qnm_dprx1,
+		   &qnm_mdp0, &qnm_mdp1,
+		   &qnm_video_cv_cpu, &qnm_video_eva,
+		   &qnm_video_mvp0, &qnm_video_mvp1,
+		   &qnm_video_v_cpu, &qns_mem_noc_sf },
+};
+
+static struct qcom_icc_bcm bcm_qup0 = {
+	.name = "QUP0",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup0_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup1 = {
+	.name = "QUP1",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup1_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup2 = {
+	.name = "QUP2",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup2_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_qup3 = {
+	.name = "QUP3",
+	.vote_scale = 1,
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qup3_core_slave },
+};
+
+static struct qcom_icc_bcm bcm_sh0 = {
+	.name = "SH0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_llcc },
+};
+
+static struct qcom_icc_bcm bcm_sh1 = {
+	.name = "SH1",
+	.enable_mask = BIT(0),
+	.num_nodes = 24,
+	.nodes = { &alm_gpu_tcu, &alm_qtc,
+		   &alm_sys_tcu0, &alm_sys_tcu1,
+		   &alm_sys_tcu2, &chm_apps,
+		   &qnm_aggre_north, &qnm_aggre_south,
+		   &qnm_gpu0, &qnm_gpu1,
+		   &qnm_hpass_adas_hscnoc, &qnm_hpass_audio_hscnoc,
+		   &qnm_mnoc_hf, &qnm_mnoc_sf,
+		   &qnm_nsp0_hscnoc, &qnm_nsp1_hscnoc,
+		   &qnm_nsp2_hscnoc, &qnm_nsp3_hscnoc,
+		   &qnm_pcie, &qnm_sailss_md0_hscnoc,
+		   &qnm_snoc_sf, &xm_gic,
+		   &qns_hscnoc_cnoc, &qns_pcie },
+};
+
+static struct qcom_icc_bcm bcm_sn0 = {
+	.name = "SN0",
+	.keepalive = true,
+	.num_nodes = 1,
+	.nodes = { &qns_hscnoc_sf },
+};
+
+static struct qcom_icc_bcm bcm_sn1 = {
+	.name = "SN1",
+	.enable_mask = BIT(0),
+	.num_nodes = 14,
+	.nodes = { &qns_a1noc_hscnoc, &qns_a2noc_hscnoc,
+		   &qxm_pcie_dma_0, &qxm_pcie_dma_1,
+		   &qxm_pcie_dma_2, &xm_pcie_0,
+		   &xm_pcie_1, &xm_pcie_2,
+		   &xm_pcie_3, &qns_pcie_hscnoc,
+		   &qns_pcie_obnoc_dma, &qnm_cnoc_data,
+		   &qnm_nsi_noc, &qnm_safe_dma },
+};
+
+static struct qcom_icc_bcm bcm_sn2 = {
+	.name = "SN2",
+	.num_nodes = 1,
+	.nodes = { &qnm_aggre1_noc },
+};
+
+static struct qcom_icc_bcm bcm_sn3 = {
+	.name = "SN3",
+	.num_nodes = 1,
+	.nodes = { &qnm_aggre2_noc },
+};
+
+static struct qcom_icc_node * const aggre1_noc_nodes[] = {
+	[MASTER_QSPI_0] = &qhm_qspi,
+	[MASTER_SAILSS_MD1] = &qnm_sailss_md1,
+	[MASTER_QUP_3] = &qxm_qup3,
+	[SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc,
+};
+
+static const struct regmap_config nord_aggre1_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1c400,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_aggre1_noc = {
+	.config = &nord_aggre1_noc_regmap_config,
+	.nodes = aggre1_noc_nodes,
+	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const aggre1_noc_tile_bcms[] = {
+	&bcm_ce0,
+	&bcm_ce1,
+	&bcm_ce2,
+	&bcm_sn1,
+};
+
+static struct qcom_icc_node * const aggre1_noc_tile_nodes[] = {
+	[MASTER_QUP_2] = &qhm_qup2,
+	[MASTER_CRYPTO_CORE0] = &qxm_crypto_0,
+	[MASTER_CRYPTO_CORE1] = &qxm_crypto_1,
+	[MASTER_CRYPTO_CORE2] = &qxm_crypto_2,
+	[MASTER_SDCC_4] = &xm_sdc4,
+	[MASTER_UFS_MEM] = &xm_ufs_mem,
+	[MASTER_USB2] = &xm_usb2,
+	[MASTER_USB3_0] = &xm_usb3_0,
+	[MASTER_USB3_1] = &xm_usb3_1,
+	[SLAVE_A1NOC_HSCNOC] = &qns_a1noc_hscnoc,
+};
+
+static const struct regmap_config nord_aggre1_noc_tile_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x23400,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_aggre1_noc_tile = {
+	.config = &nord_aggre1_noc_tile_regmap_config,
+	.nodes = aggre1_noc_tile_nodes,
+	.num_nodes = ARRAY_SIZE(aggre1_noc_tile_nodes),
+	.bcms = aggre1_noc_tile_bcms,
+	.num_bcms = ARRAY_SIZE(aggre1_noc_tile_bcms),
+};
+
+static struct qcom_icc_node * const aggre2_noc_nodes[] = {
+	[MASTER_IPA] = &qxm_ipa,
+	[MASTER_SOCCP_AGGR_NOC] = &qxm_soccp,
+	[MASTER_QDSS_ETR] = &xm_qdss_etr_0,
+	[MASTER_QDSS_ETR_1] = &xm_qdss_etr_1,
+	[SLAVE_A2NOC_SNOC] = &qns_a2noc_snoc,
+};
+
+static const struct regmap_config nord_aggre2_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1b400,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_aggre2_noc = {
+	.config = &nord_aggre2_noc_regmap_config,
+	.nodes = aggre2_noc_nodes,
+	.num_nodes = ARRAY_SIZE(aggre2_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const aggre2_noc_tile_bcms[] = {
+	&bcm_sn1,
+};
+
+static struct qcom_icc_node * const aggre2_noc_tile_nodes[] = {
+	[MASTER_QUP_0] = &qhm_qup0,
+	[MASTER_QUP_1] = &qhm_qup1,
+	[MASTER_EMAC_0] = &xm_emac_0,
+	[MASTER_EMAC_1] = &xm_emac_1,
+	[SLAVE_A2NOC_HSCNOC] = &qns_a2noc_hscnoc,
+};
+
+static const struct regmap_config nord_aggre2_noc_tile_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1b400,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_aggre2_noc_tile = {
+	.config = &nord_aggre2_noc_tile_regmap_config,
+	.nodes = aggre2_noc_tile_nodes,
+	.num_nodes = ARRAY_SIZE(aggre2_noc_tile_nodes),
+	.bcms = aggre2_noc_tile_bcms,
+	.num_bcms = ARRAY_SIZE(aggre2_noc_tile_bcms),
+};
+
+static struct qcom_icc_bcm * const clk_virt_bcms[] = {
+	&bcm_qup0,
+	&bcm_qup1,
+	&bcm_qup2,
+	&bcm_qup3,
+};
+
+static struct qcom_icc_node * const clk_virt_nodes[] = {
+	[MASTER_QUP_CORE_0] = &qup0_core_master,
+	[MASTER_QUP_CORE_1] = &qup1_core_master,
+	[MASTER_QUP_CORE_2] = &qup2_core_master,
+	[MASTER_QUP_CORE_3] = &qup3_core_master,
+	[SLAVE_QUP_CORE_0] = &qup0_core_slave,
+	[SLAVE_QUP_CORE_1] = &qup1_core_slave,
+	[SLAVE_QUP_CORE_2] = &qup2_core_slave,
+	[SLAVE_QUP_CORE_3] = &qup3_core_slave,
+};
+
+static const struct qcom_icc_desc nord_clk_virt = {
+	.nodes = clk_virt_nodes,
+	.num_nodes = ARRAY_SIZE(clk_virt_nodes),
+	.bcms = clk_virt_bcms,
+	.num_bcms = ARRAY_SIZE(clk_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const cnoc_cfg_bcms[] = {
+	&bcm_cn0,
+	&bcm_cn1,
+};
+
+static struct qcom_icc_node * const cnoc_cfg_nodes[] = {
+	[MASTER_CNOC_CFG] = &qsm_cfg,
+	[SLAVE_PS_ETH_0] = &ps_eth_0,
+	[SLAVE_PS_ETH_1] = &ps_eth_1,
+	[SLAVE_SHS_SERVER] = &ps_shs_server,
+	[SLAVE_AHB2PHY_0] = &qhs_ahb2phy0,
+	[SLAVE_AHB2PHY_1] = &qhs_ahb2phy1,
+	[SLAVE_AHB2PHY_2] = &qhs_ahb2phy2,
+	[SLAVE_AHB2PHY_3] = &qhs_ahb2phy3,
+	[SLAVE_AHB2PHY_ETH_0] = &qhs_ahb2phy_eth_0,
+	[SLAVE_AHB2PHY_ETH_1] = &qhs_ahb2phy_eth_1,
+	[SLAVE_CAMERA_CFG] = &qhs_camera_cfg,
+	[SLAVE_CLK_CTL] = &qhs_clk_ctl,
+	[SLAVE_CRYPTO_0_CFG] = &qhs_crypto0_cfg,
+	[SLAVE_CRYPTO_1_CFG] = &qhs_crypto1_cfg,
+	[SLAVE_CRYPTO_2_CFG] = &qhs_crypto2_cfg,
+	[SLAVE_DISPLAY_1_CFG] = &qhs_display_1_cfg,
+	[SLAVE_DISPLAY_CFG] = &qhs_display_cfg,
+	[SLAVE_DPRX0] = &qhs_dprx0,
+	[SLAVE_DPRX1] = &qhs_dprx1,
+	[SLAVE_EVA_CFG] = &qhs_eva_cfg,
+	[SLAVE_GFX3D_CFG] = &qhs_gpuss_0_cfg,
+	[SLAVE_GFX3D_1_CFG] = &qhs_gpuss_1_cfg,
+	[SLAVE_I2C] = &qhs_i2c,
+	[SLAVE_IMEM_CFG] = &qhs_imem_cfg,
+	[SLAVE_MCW_PCIE] = &qhs_mcw_pcie,
+	[SLAVE_MM_RSCC] = &qhs_mm_rscc,
+	[SLAVE_NE_CLK_CTL] = &qhs_ne_clk_ctl,
+	[SLAVE_NSPSS0_CFG] = &qhs_nspss0_cfg,
+	[SLAVE_NSPSS1_CFG] = &qhs_nspss1_cfg,
+	[SLAVE_NSPSS2_CFG] = &qhs_nspss2_cfg,
+	[SLAVE_NSPSS3_CFG] = &qhs_nspss3_cfg,
+	[SLAVE_NW_CLK_CTL] = &qhs_nw_clk_ctl,
+	[SLAVE_PRNG] = &qhs_prng,
+	[SLAVE_QDSS_CFG] = &qhs_qdss_cfg,
+	[SLAVE_QSPI_0] = &qhs_qspi,
+	[SLAVE_QUP_0] = &qhs_qup0,
+	[SLAVE_QUP_3] = &qhs_qup3,
+	[SLAVE_QUP_1] = &qhs_qup1,
+	[SLAVE_QUP_2] = &qhs_qup2,
+	[SLAVE_SAFEDMA_CFG] = &qhs_safedma_cfg,
+	[SLAVE_SDCC_4] = &qhs_sdc4,
+	[SLAVE_SE_CLK_CTL] = &qhs_se_clk_ctl,
+	[SLAVE_TCSR] = &qhs_tcsr,
+	[SLAVE_TLMM] = &qhs_tlmm,
+	[SLAVE_TSC_CFG] = &qhs_tsc_cfg,
+	[SLAVE_UFS_MEM_CFG] = &qhs_ufs_mem_cfg,
+	[SLAVE_USB2] = &qhs_usb2,
+	[SLAVE_USB3_0] = &qhs_usb3_0,
+	[SLAVE_USB3_1] = &qhs_usb3_1,
+	[SLAVE_VENUS_CFG] = &qhs_venus_cfg,
+	[SLAVE_COMPUTENOC_CFG] = &qss_computenoc_cfg,
+	[SLAVE_PCIE_NOC_CFG] = &qss_pcie_noc_cfg,
+	[SLAVE_QTC_CFG] = &qss_qtc_cfg,
+	[SLAVE_QDSS_STM] = &xs_qdss_stm,
+	[SLAVE_SYS_TCU0_CFG] = &xs_sys_tcu0_cfg,
+	[SLAVE_SYS_TCU1_CFG] = &xs_sys_tcu1_cfg,
+	[SLAVE_SYS_TCU2_CFG] = &xs_sys_tcu2_cfg,
+};
+
+static const struct regmap_config nord_cnoc_cfg_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xd200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_cnoc_cfg = {
+	.config = &nord_cnoc_cfg_regmap_config,
+	.nodes = cnoc_cfg_nodes,
+	.num_nodes = ARRAY_SIZE(cnoc_cfg_nodes),
+	.bcms = cnoc_cfg_bcms,
+	.num_bcms = ARRAY_SIZE(cnoc_cfg_bcms),
+};
+
+static struct qcom_icc_bcm * const cnoc_main_bcms[] = {
+	&bcm_cn0,
+};
+
+static struct qcom_icc_node * const cnoc_main_nodes[] = {
+	[MASTER_MM_RSCC] = &qhm_mm_rscc,
+	[MASTER_HSCNOC_CNOC] = &qnm_hscnoc,
+	[SLAVE_AOSS] = &qhs_aoss,
+	[SLAVE_HBCU] = &qhs_hbcu,
+	[SLAVE_IPA_CFG] = &qhs_ipa,
+	[SLAVE_IPC_ROUTER_CFG] = &qhs_ipc_router,
+	[SLAVE_SOCCP] = &qhs_soccp,
+	[SLAVE_TME_CFG] = &qhs_tme_cfg,
+	[SLAVE_PCIE_DMA] = &qns_pcie_dma,
+	[SLAVE_CNOC_CFG] = &qss_cfg,
+	[SLAVE_DDRSS_CFG] = &qss_ddrss_cfg,
+	[SLAVE_IMEM] = &qxs_imem,
+};
+
+static const struct regmap_config nord_cnoc_main_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1d200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_cnoc_main = {
+	.config = &nord_cnoc_main_regmap_config,
+	.nodes = cnoc_main_nodes,
+	.num_nodes = ARRAY_SIZE(cnoc_main_nodes),
+	.bcms = cnoc_main_bcms,
+	.num_bcms = ARRAY_SIZE(cnoc_main_bcms),
+};
+
+static struct qcom_icc_node * const hpass_ag_noc_nodes[] = {
+	[MASTER_HPASS_PROC_0] = &qnm_hpass_dsp0,
+	[MASTER_HPASS_PROC_1] = &qnm_hpass_dsp1,
+	[MASTER_HPASS_PROC_2] = &qnm_hpass_dsp2,
+	[SLAVE_HPASS_AGNOC_AUDIO] = &qns_hpass_agnoc_audio,
+};
+
+static const struct regmap_config nord_hpass_ag_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x37080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_hpass_ag_noc = {
+	.config = &nord_hpass_ag_noc_regmap_config,
+	.nodes = hpass_ag_noc_nodes,
+	.num_nodes = ARRAY_SIZE(hpass_ag_noc_nodes),
+};
+
+static struct qcom_icc_bcm * const hscnoc_bcms[] = {
+	&bcm_sh0,
+	&bcm_sh1,
+};
+
+static struct qcom_icc_node * const hscnoc_nodes[] = {
+	[MASTER_GPU_TCU] = &alm_gpu_tcu,
+	[MASTER_QTC_TCU] = &alm_qtc,
+	[MASTER_SYS_TCU_0] = &alm_sys_tcu0,
+	[MASTER_SYS_TCU_1] = &alm_sys_tcu1,
+	[MASTER_SYS_TCU_2] = &alm_sys_tcu2,
+	[MASTER_APPSS_PROC] = &chm_apps,
+	[MASTER_A1NOC_TILE_HSCNOC] = &qnm_aggre_north,
+	[MASTER_A2NOC_TILE_HSCNOC] = &qnm_aggre_south,
+	[MASTER_GFX3D] = &qnm_gpu0,
+	[MASTER_GFX3D_1] = &qnm_gpu1,
+	[MASTER_HPASS_ADAS_HSCNOC] = &qnm_hpass_adas_hscnoc,
+	[MASTER_HPASS_AUDIO_HSCNOC] = &qnm_hpass_audio_hscnoc,
+	[MASTER_MNOC_HF_MEM_NOC] = &qnm_mnoc_hf,
+	[MASTER_MNOC_SF_MEM_NOC] = &qnm_mnoc_sf,
+	[MASTER_NSP0_HSCNOC] = &qnm_nsp0_hscnoc,
+	[MASTER_NSP1_HSCNOC] = &qnm_nsp1_hscnoc,
+	[MASTER_NSP2_HSCNOC] = &qnm_nsp2_hscnoc,
+	[MASTER_NSP3_HSCNOC] = &qnm_nsp3_hscnoc,
+	[MASTER_ANOC_PCIE_GEM_NOC] = &qnm_pcie,
+	[MASTER_SAILSS_MD0_HSCNOC] = &qnm_sailss_md0_hscnoc,
+	[MASTER_SNOC_SF_MEM_NOC] = &qnm_snoc_sf,
+	[MASTER_GIC] = &xm_gic,
+	[SLAVE_HSCNOC_CNOC] = &qns_hscnoc_cnoc,
+	[SLAVE_LLCC] = &qns_llcc,
+	[SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie,
+};
+
+static const struct regmap_config nord_hscnoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xa45080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_hscnoc = {
+	.config = &nord_hscnoc_regmap_config,
+	.nodes = hscnoc_nodes,
+	.num_nodes = ARRAY_SIZE(hscnoc_nodes),
+	.bcms = hscnoc_bcms,
+	.num_bcms = ARRAY_SIZE(hscnoc_bcms),
+};
+
+static struct qcom_icc_bcm * const mc_virt_bcms[] = {
+	&bcm_mc0,
+};
+
+static struct qcom_icc_node * const mc_virt_nodes[] = {
+	[MASTER_LLCC] = &llcc_mc,
+	[SLAVE_EBI1] = &ebi,
+};
+
+static const struct qcom_icc_desc nord_mc_virt = {
+	.nodes = mc_virt_nodes,
+	.num_nodes = ARRAY_SIZE(mc_virt_nodes),
+	.bcms = mc_virt_bcms,
+	.num_bcms = ARRAY_SIZE(mc_virt_bcms),
+};
+
+static struct qcom_icc_bcm * const mmss_noc_bcms[] = {
+	&bcm_mm0,
+	&bcm_mm1,
+};
+
+static struct qcom_icc_node * const mmss_noc_nodes[] = {
+	[MASTER_CAMNOC_HF] = &qnm_camnoc_hf,
+	[MASTER_CAMNOC_NRT_ICP_SF] = &qnm_camnoc_nrt_icp_sf,
+	[MASTER_CAMNOC_RT_CDM_SF] = &qnm_camnoc_rt_cdm_sf,
+	[MASTER_CAMNOC_SF] = &qnm_camnoc_sf,
+	[MASTER_DPRX0] = &qnm_dprx0,
+	[MASTER_DPRX1] = &qnm_dprx1,
+	[MASTER_MDP0] = &qnm_mdp0,
+	[MASTER_MDP1] = &qnm_mdp1,
+	[MASTER_VIDEO_CV_PROC] = &qnm_video_cv_cpu,
+	[MASTER_VIDEO_EVA] = &qnm_video_eva,
+	[MASTER_VIDEO_MVP0] = &qnm_video_mvp0,
+	[MASTER_VIDEO_MVP1] = &qnm_video_mvp1,
+	[MASTER_VIDEO_V_PROC] = &qnm_video_v_cpu,
+	[SLAVE_MNOC_HF_MEM_NOC] = &qns_mem_noc_hf,
+	[SLAVE_MNOC_SF_MEM_NOC] = &qns_mem_noc_sf,
+};
+
+static const struct regmap_config nord_mmss_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x72800,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_mmss_noc = {
+	.config = &nord_mmss_noc_regmap_config,
+	.nodes = mmss_noc_nodes,
+	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
+	.bcms = mmss_noc_bcms,
+	.num_bcms = ARRAY_SIZE(mmss_noc_bcms),
+};
+
+static struct qcom_icc_bcm * const nsp_data_noc_0_bcms[] = {
+	&bcm_c0n0,
+};
+
+static struct qcom_icc_node * const nsp_data_noc_0_nodes[] = {
+	[MASTER_NSP0_PROC] = &qnm_nsp_data00,
+	[SLAVE_NSP0_HSC_NOC] = &qns_nsp0_hsc_noc,
+};
+
+static const struct regmap_config nord_nsp_data_noc_0_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x2a200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_nsp_data_noc_0 = {
+	.config = &nord_nsp_data_noc_0_regmap_config,
+	.nodes = nsp_data_noc_0_nodes,
+	.num_nodes = ARRAY_SIZE(nsp_data_noc_0_nodes),
+	.bcms = nsp_data_noc_0_bcms,
+	.num_bcms = ARRAY_SIZE(nsp_data_noc_0_bcms),
+};
+
+static struct qcom_icc_bcm * const nsp_data_noc_1_bcms[] = {
+	&bcm_c1n0,
+};
+
+static struct qcom_icc_node * const nsp_data_noc_1_nodes[] = {
+	[MASTER_NSP1_PROC] = &qnm_nsp_data01,
+	[SLAVE_NSP1_HSC_NOC] = &qns_nsp1_hsc_noc,
+};
+
+static const struct regmap_config nord_nsp_data_noc_1_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x2a200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_nsp_data_noc_1 = {
+	.config = &nord_nsp_data_noc_1_regmap_config,
+	.nodes = nsp_data_noc_1_nodes,
+	.num_nodes = ARRAY_SIZE(nsp_data_noc_1_nodes),
+	.bcms = nsp_data_noc_1_bcms,
+	.num_bcms = ARRAY_SIZE(nsp_data_noc_1_bcms),
+};
+
+static struct qcom_icc_bcm * const nsp_data_noc_2_bcms[] = {
+	&bcm_c2n0,
+};
+
+static struct qcom_icc_node * const nsp_data_noc_2_nodes[] = {
+	[MASTER_NSP2_PROC] = &qnm_nsp_data02,
+	[SLAVE_NSP2_HSC_NOC] = &qns_nsp2_hsc_noc,
+};
+
+static const struct regmap_config nord_nsp_data_noc_2_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x2a200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_nsp_data_noc_2 = {
+	.config = &nord_nsp_data_noc_2_regmap_config,
+	.nodes = nsp_data_noc_2_nodes,
+	.num_nodes = ARRAY_SIZE(nsp_data_noc_2_nodes),
+	.bcms = nsp_data_noc_2_bcms,
+	.num_bcms = ARRAY_SIZE(nsp_data_noc_2_bcms),
+};
+
+static struct qcom_icc_bcm * const nsp_data_noc_3_bcms[] = {
+	&bcm_c3n0,
+};
+
+static struct qcom_icc_node * const nsp_data_noc_3_nodes[] = {
+	[MASTER_NSP3_PROC] = &qnm_nsp_data03,
+	[SLAVE_NSP3_HSC_NOC] = &qns_nsp3_hsc_noc,
+};
+
+static const struct regmap_config nord_nsp_data_noc_3_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x2a200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_nsp_data_noc_3 = {
+	.config = &nord_nsp_data_noc_3_regmap_config,
+	.nodes = nsp_data_noc_3_nodes,
+	.num_nodes = ARRAY_SIZE(nsp_data_noc_3_nodes),
+	.bcms = nsp_data_noc_3_bcms,
+	.num_bcms = ARRAY_SIZE(nsp_data_noc_3_bcms),
+};
+
+static struct qcom_icc_node * const pcie_cfg_nodes[] = {
+	[MASTER_PCIE_NOC_CFG] = &qsm_pcie_noc_cfg,
+	[SLAVE_PCIE_AHB2PHY_CFG] = &qhs_pcie_ahb2phy_cfg,
+	[SLAVE_PCIE_CFG_0] = &qhs_pcie_cfg_0,
+	[SLAVE_PCIE_CFG_1] = &qhs_pcie_cfg_1,
+	[SLAVE_PCIE_CFG_2] = &qhs_pcie_cfg_2,
+	[SLAVE_PCIE_CFG_3] = &qhs_pcie_cfg_3,
+	[SLAVE_PCIE_DMA_0_CFG] = &qhs_pcie_dma_0_cfg,
+	[SLAVE_PCIE_DMA_1_CFG] = &qhs_pcie_dma_1_cfg,
+	[SLAVE_PCIE_DMA_2_CFG] = &qhs_pcie_dma_2_cfg,
+};
+
+static const struct regmap_config nord_pcie_cfg_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x7200,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_pcie_cfg = {
+	.config = &nord_pcie_cfg_regmap_config,
+	.nodes = pcie_cfg_nodes,
+	.num_nodes = ARRAY_SIZE(pcie_cfg_nodes),
+};
+
+static struct qcom_icc_bcm * const pcie_data_inbound_bcms[] = {
+	&bcm_sn1,
+};
+
+static struct qcom_icc_node * const pcie_data_inbound_nodes[] = {
+	[MASTER_PCIE_DMA_0] = &qxm_pcie_dma_0,
+	[MASTER_PCIE_DMA_1] = &qxm_pcie_dma_1,
+	[MASTER_PCIE_DMA_2] = &qxm_pcie_dma_2,
+	[MASTER_PCIE_0] = &xm_pcie_0,
+	[MASTER_PCIE_1] = &xm_pcie_1,
+	[MASTER_PCIE_2] = &xm_pcie_2,
+	[MASTER_PCIE_3] = &xm_pcie_3,
+	[SLAVE_PCIE_HSCNOC] = &qns_pcie_hscnoc,
+	[SLAVE_PCIE_OBNOC_DMA] = &qns_pcie_obnoc_dma,
+};
+
+static const struct regmap_config nord_pcie_data_inbound_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x4b080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_pcie_data_inbound = {
+	.config = &nord_pcie_data_inbound_regmap_config,
+	.nodes = pcie_data_inbound_nodes,
+	.num_nodes = ARRAY_SIZE(pcie_data_inbound_nodes),
+	.bcms = pcie_data_inbound_bcms,
+	.num_bcms = ARRAY_SIZE(pcie_data_inbound_bcms),
+};
+
+static struct qcom_icc_bcm * const pcie_data_outbound_bcms[] = {
+	&bcm_cn0,
+};
+
+static struct qcom_icc_node * const pcie_data_outbound_nodes[] = {
+	[MASTER_CNOC_PCIE_DMA] = &qnm_cnoc_pcie_dma,
+	[MASTER_ANOC_PCIE_HSCNOC] = &qnm_hscnoc_pcie,
+	[MASTER_PCIE_IBNOC_DMA] = &qnm_pcie_ibnoc_dma,
+	[SLAVE_PCIE_DMA_0] = &qxs_pcie_dma_0,
+	[SLAVE_PCIE_DMA_1] = &qxs_pcie_dma_1,
+	[SLAVE_PCIE_DMA_2] = &qxs_pcie_dma_2,
+	[SLAVE_PCIE_0] = &xs_pcie_0,
+	[SLAVE_PCIE_1] = &xs_pcie_1,
+	[SLAVE_PCIE_2] = &xs_pcie_2,
+	[SLAVE_PCIE_3] = &xs_pcie_3,
+};
+
+static const struct regmap_config nord_pcie_data_outbound_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x17000,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_pcie_data_outbound = {
+	.config = &nord_pcie_data_outbound_regmap_config,
+	.nodes = pcie_data_outbound_nodes,
+	.num_nodes = ARRAY_SIZE(pcie_data_outbound_nodes),
+	.bcms = pcie_data_outbound_bcms,
+	.num_bcms = ARRAY_SIZE(pcie_data_outbound_bcms),
+};
+
+static struct qcom_icc_bcm * const system_noc_bcms[] = {
+	&bcm_sn0,
+	&bcm_sn1,
+	&bcm_sn2,
+	&bcm_sn3,
+};
+
+static struct qcom_icc_node * const system_noc_nodes[] = {
+	[MASTER_A1NOC_SNOC] = &qnm_aggre1_noc,
+	[MASTER_A2NOC_SNOC] = &qnm_aggre2_noc,
+	[MASTER_CNOC_SNOC] = &qnm_cnoc_data,
+	[MASTER_NSINOC_SNOC] = &qnm_nsi_noc,
+	[MASTER_SAFE_DMA] = &qnm_safe_dma,
+	[SLAVE_SNOC_HSCNOC_SF] = &qns_hscnoc_sf,
+};
+
+static const struct regmap_config nord_system_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1c080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc nord_system_noc = {
+	.config = &nord_system_noc_regmap_config,
+	.nodes = system_noc_nodes,
+	.num_nodes = ARRAY_SIZE(system_noc_nodes),
+	.bcms = system_noc_bcms,
+	.num_bcms = ARRAY_SIZE(system_noc_bcms),
+};
+
+static const struct of_device_id qnoc_of_match[] = {
+	{ .compatible = "qcom,nord-aggre1-noc", .data = &nord_aggre1_noc },
+	{ .compatible = "qcom,nord-aggre1-noc-tile", .data = &nord_aggre1_noc_tile },
+	{ .compatible = "qcom,nord-aggre2-noc", .data = &nord_aggre2_noc },
+	{ .compatible = "qcom,nord-aggre2-noc-tile", .data = &nord_aggre2_noc_tile },
+	{ .compatible = "qcom,nord-clk-virt", .data = &nord_clk_virt },
+	{ .compatible = "qcom,nord-cnoc-cfg", .data = &nord_cnoc_cfg },
+	{ .compatible = "qcom,nord-cnoc-main", .data = &nord_cnoc_main },
+	{ .compatible = "qcom,nord-hpass-ag-noc", .data = &nord_hpass_ag_noc },
+	{ .compatible = "qcom,nord-hscnoc", .data = &nord_hscnoc },
+	{ .compatible = "qcom,nord-mc-virt", .data = &nord_mc_virt },
+	{ .compatible = "qcom,nord-mmss-noc", .data = &nord_mmss_noc },
+	{ .compatible = "qcom,nord-nsp-data-noc-0", .data = &nord_nsp_data_noc_0 },
+	{ .compatible = "qcom,nord-nsp-data-noc-1", .data = &nord_nsp_data_noc_1 },
+	{ .compatible = "qcom,nord-nsp-data-noc-2", .data = &nord_nsp_data_noc_2 },
+	{ .compatible = "qcom,nord-nsp-data-noc-3", .data = &nord_nsp_data_noc_3 },
+	{ .compatible = "qcom,nord-pcie-cfg", .data = &nord_pcie_cfg },
+	{ .compatible = "qcom,nord-pcie-data-inbound", .data = &nord_pcie_data_inbound },
+	{ .compatible = "qcom,nord-pcie-data-outbound", .data = &nord_pcie_data_outbound },
+	{ .compatible = "qcom,nord-system-noc", .data = &nord_system_noc },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, qnoc_of_match);
+
+static struct platform_driver qnoc_driver = {
+	.probe = qcom_icc_rpmh_probe,
+	.remove = qcom_icc_rpmh_remove,
+	.driver = {
+		.name = "qnoc-nord",
+		.of_match_table = qnoc_of_match,
+		.sync_state = icc_sync_state,
+	},
+};
+
+static int __init qnoc_driver_init(void)
+{
+	return platform_driver_register(&qnoc_driver);
+}
+core_initcall(qnoc_driver_init);
+
+static void __exit qnoc_driver_exit(void)
+{
+	platform_driver_unregister(&qnoc_driver);
+}
+module_exit(qnoc_driver_exit);
+
+MODULE_DESCRIPTION("Qualcomm Nord NoC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/interconnect/qcom/shikra.c b/drivers/interconnect/qcom/shikra.c
new file mode 100644
index 0000000000000..bc40d1592fb3d
--- /dev/null
+++ b/drivers/interconnect/qcom/shikra.c
@@ -0,0 +1,1837 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ *
+ */
+
+#include <dt-bindings/interconnect/qcom,shikra.h>
+#include <linux/device.h>
+#include <linux/interconnect-provider.h>
+#include <linux/io.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include "icc-rpm.h"
+
+static const char * const sys_noc_intf_clocks[] = {
+	"emac0_axi",
+	"emac1_axi",
+	"usb2_axi",
+	"usb3_axi",
+};
+
+static const char * const memnoc_intf_clocks[] = {
+	"gpu_axi",
+};
+
+enum {
+	SHIKRA_MASTER_QUP_CORE_0 = 1,
+	SHIKRA_SNOC_CNOC_MAS,
+	SHIKRA_MASTER_QDSS_DAP,
+	SHIKRA_MASTER_LLCC,
+	SHIKRA_MASTER_GRAPHICS_3D,
+	SHIKRA_MASTER_MNOC_HF_MEM_NOC,
+	SHIKRA_MASTER_ANOC_PCIE_MEM_NOC,
+	SHIKRA_MASTER_SNOC_SF_MEM_NOC,
+	SHIKRA_MASTER_AMPSS_M0,
+	SHIKRA_MASTER_SYS_TCU,
+	SHIKRA_MASTER_CAMNOC_SF,
+	SHIKRA_MASTER_VIDEO_P0,
+	SHIKRA_MASTER_VIDEO_PROC,
+	SHIKRA_MASTER_CAMNOC_HF,
+	SHIKRA_MASTER_MDP_PORT0,
+	SHIKRA_MASTER_MMRT_VIRT,
+	SHIKRA_MASTER_SNOC_CFG,
+	SHIKRA_MASTER_TIC,
+	SHIKRA_MASTER_ANOC_SNOC,
+	SHIKRA_MASTER_MEMNOC_PCIE,
+	SHIKRA_MASTER_MEMNOC_SNOC,
+	SHIKRA_MASTER_PIMEM,
+	SHIKRA_MASTER_PCIE2_0,
+	SHIKRA_MASTER_QDSS_BAM,
+	SHIKRA_MASTER_QPIC,
+	SHIKRA_MASTER_QUP_0,
+	SHIKRA_CNOC_SNOC_MAS,
+	SHIKRA_MASTER_AUDIO,
+	SHIKRA_MASTER_EMAC_0,
+	SHIKRA_MASTER_EMAC_1,
+	SHIKRA_MASTER_QDSS_ETR,
+	SHIKRA_MASTER_SDCC_1,
+	SHIKRA_MASTER_SDCC_2,
+	SHIKRA_MASTER_USB2_0,
+	SHIKRA_MASTER_USB3,
+	SHIKRA_MASTER_CRYPTO_CORE0,
+
+	SHIKRA_SLAVE_QUP_CORE_0,
+	SHIKRA_SLAVE_AHB2PHY_USB,
+	SHIKRA_SLAVE_APSS_THROTTLE_CFG,
+	SHIKRA_SLAVE_AUDIO,
+	SHIKRA_SLAVE_BOOT_ROM,
+	SHIKRA_SLAVE_CAMERA_NRT_THROTTLE_CFG,
+	SHIKRA_SLAVE_CAMERA_CFG,
+	SHIKRA_SLAVE_CDSP_THROTTLE_CFG,
+	SHIKRA_SLAVE_CLK_CTL,
+	SHIKRA_SLAVE_DSP_CFG,
+	SHIKRA_SLAVE_RBCPR_CX_CFG,
+	SHIKRA_SLAVE_RBCPR_MX_CFG,
+	SHIKRA_SLAVE_CRYPTO_0_CFG,
+	SHIKRA_SLAVE_DDR_SS_CFG,
+	SHIKRA_SLAVE_DISPLAY_CFG,
+	SHIKRA_SLAVE_EMAC0_CFG,
+	SHIKRA_SLAVE_EMAC1_CFG,
+	SHIKRA_SLAVE_GPU_CFG,
+	SHIKRA_SLAVE_GPU_THROTTLE_CFG,
+	SHIKRA_SLAVE_HWKM,
+	SHIKRA_SLAVE_IMEM_CFG,
+	SHIKRA_SLAVE_MAPSS,
+	SHIKRA_SLAVE_MDSP_MPU_CFG,
+	SHIKRA_SLAVE_MESSAGE_RAM,
+	SHIKRA_SLAVE_MSS,
+	SHIKRA_SLAVE_PCIE_CFG,
+	SHIKRA_SLAVE_PDM,
+	SHIKRA_SLAVE_PIMEM_CFG,
+	SHIKRA_SLAVE_PKA_WRAPPER_CFG,
+	SHIKRA_SLAVE_PMIC_ARB,
+	SHIKRA_SLAVE_QDSS_CFG,
+	SHIKRA_SLAVE_QM_CFG,
+	SHIKRA_SLAVE_QM_MPU_CFG,
+	SHIKRA_SLAVE_QPIC,
+	SHIKRA_SLAVE_QUP_0,
+	SHIKRA_SLAVE_RPM,
+	SHIKRA_SLAVE_SDCC_1,
+	SHIKRA_SLAVE_SDCC_2,
+	SHIKRA_SLAVE_SECURITY,
+	SHIKRA_SLAVE_SNOC_CFG,
+	SHIKRA_SNOC_SF_THROTTLE_CFG,
+	SHIKRA_SLAVE_TLMM,
+	SHIKRA_SLAVE_TSCSS,
+	SHIKRA_SLAVE_USB2,
+	SHIKRA_SLAVE_USB3,
+	SHIKRA_SLAVE_VENUS_CFG,
+	SHIKRA_SLAVE_VENUS_THROTTLE_CFG,
+	SHIKRA_SLAVE_VSENSE_CTRL_CFG,
+	SHIKRA_SLAVE_SERVICE_CNOC,
+	SHIKRA_SLAVE_EBI_CH0,
+	SHIKRA_SLAVE_LLCC,
+	SHIKRA_SLAVE_MEMNOC_SNOC,
+	SHIKRA_SLAVE_MEM_NOC_PCIE_SNOC,
+	SHIKRA_SLAVE_MMNRT_VIRT,
+	SHIKRA_SLAVE_MM_MEMNOC,
+	SHIKRA_SLAVE_APPSS,
+	SHIKRA_SLAVE_MCUSS,
+	SHIKRA_SLAVE_WCSS,
+	SHIKRA_SLAVE_MEMNOC_SF,
+	SHIKRA_SNOC_CNOC_SLV,
+	SHIKRA_SLAVE_BOOTIMEM,
+	SHIKRA_SLAVE_OCIMEM,
+	SHIKRA_SLAVE_PIMEM,
+	SHIKRA_SLAVE_SERVICE_SNOC,
+	SHIKRA_SLAVE_PCIE2_0,
+	SHIKRA_SLAVE_QDSS_STM,
+	SHIKRA_SLAVE_TCU,
+	SHIKRA_SLAVE_PCIE_MEMNOC,
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+/* Master nodes */
+static const u16 qup0_core_master_links[] = {
+	SHIKRA_SLAVE_QUP_CORE_0,
+};
+
+static struct qcom_icc_node qup0_core_master = {
+	.id = SHIKRA_MASTER_QUP_CORE_0,
+	.name = "qup0_core_master",
+	.buswidth = 4,
+	.mas_rpm_id = 170,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qup0_core_master_links),
+	.links = qup0_core_master_links,
+};
+
+static const u16 qnm_snoc_cnoc_links[] = {
+	SHIKRA_SLAVE_AHB2PHY_USB,
+	SHIKRA_SLAVE_APSS_THROTTLE_CFG,
+	SHIKRA_SLAVE_AUDIO,
+	SHIKRA_SLAVE_BOOT_ROM,
+	SHIKRA_SLAVE_CAMERA_NRT_THROTTLE_CFG,
+	SHIKRA_SLAVE_CAMERA_CFG,
+	SHIKRA_SLAVE_CDSP_THROTTLE_CFG,
+	SHIKRA_SLAVE_CLK_CTL,
+	SHIKRA_SLAVE_DSP_CFG,
+	SHIKRA_SLAVE_RBCPR_CX_CFG,
+	SHIKRA_SLAVE_RBCPR_MX_CFG,
+	SHIKRA_SLAVE_CRYPTO_0_CFG,
+	SHIKRA_SLAVE_DDR_SS_CFG,
+	SHIKRA_SLAVE_DISPLAY_CFG,
+	SHIKRA_SLAVE_EMAC0_CFG,
+	SHIKRA_SLAVE_EMAC1_CFG,
+	SHIKRA_SLAVE_GPU_CFG,
+	SHIKRA_SLAVE_GPU_THROTTLE_CFG,
+	SHIKRA_SLAVE_HWKM,
+	SHIKRA_SLAVE_IMEM_CFG,
+	SHIKRA_SLAVE_MAPSS,
+	SHIKRA_SLAVE_MDSP_MPU_CFG,
+	SHIKRA_SLAVE_MESSAGE_RAM,
+	SHIKRA_SLAVE_MSS,
+	SHIKRA_SLAVE_PCIE_CFG,
+	SHIKRA_SLAVE_PDM,
+	SHIKRA_SLAVE_PIMEM_CFG,
+	SHIKRA_SLAVE_PKA_WRAPPER_CFG,
+	SHIKRA_SLAVE_PMIC_ARB,
+	SHIKRA_SLAVE_QDSS_CFG,
+	SHIKRA_SLAVE_QM_CFG,
+	SHIKRA_SLAVE_QM_MPU_CFG,
+	SHIKRA_SLAVE_QPIC,
+	SHIKRA_SLAVE_QUP_0,
+	SHIKRA_SLAVE_RPM,
+	SHIKRA_SLAVE_SDCC_1,
+	SHIKRA_SLAVE_SDCC_2,
+	SHIKRA_SLAVE_SECURITY,
+	SHIKRA_SLAVE_SNOC_CFG,
+	SHIKRA_SNOC_SF_THROTTLE_CFG,
+	SHIKRA_SLAVE_TLMM,
+	SHIKRA_SLAVE_TSCSS,
+	SHIKRA_SLAVE_USB2,
+	SHIKRA_SLAVE_USB3,
+	SHIKRA_SLAVE_VENUS_CFG,
+	SHIKRA_SLAVE_VENUS_THROTTLE_CFG,
+	SHIKRA_SLAVE_VSENSE_CTRL_CFG,
+	SHIKRA_SLAVE_SERVICE_CNOC,
+};
+
+static struct qcom_icc_node qnm_snoc_cnoc = {
+	.id = SHIKRA_SNOC_CNOC_MAS,
+	.name = "qnm_snoc_cnoc",
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_snoc_cnoc_links),
+	.links = qnm_snoc_cnoc_links,
+};
+
+static const u16 xm_dap_links[] = {
+	SHIKRA_SLAVE_AHB2PHY_USB,
+	SHIKRA_SLAVE_APSS_THROTTLE_CFG,
+	SHIKRA_SLAVE_AUDIO,
+	SHIKRA_SLAVE_BOOT_ROM,
+	SHIKRA_SLAVE_CAMERA_NRT_THROTTLE_CFG,
+	SHIKRA_SLAVE_CAMERA_CFG,
+	SHIKRA_SLAVE_CDSP_THROTTLE_CFG,
+	SHIKRA_SLAVE_CLK_CTL,
+	SHIKRA_SLAVE_DSP_CFG,
+	SHIKRA_SLAVE_RBCPR_CX_CFG,
+	SHIKRA_SLAVE_RBCPR_MX_CFG,
+	SHIKRA_SLAVE_CRYPTO_0_CFG,
+	SHIKRA_SLAVE_DDR_SS_CFG,
+	SHIKRA_SLAVE_DISPLAY_CFG,
+	SHIKRA_SLAVE_EMAC0_CFG,
+	SHIKRA_SLAVE_EMAC1_CFG,
+	SHIKRA_SLAVE_GPU_CFG,
+	SHIKRA_SLAVE_GPU_THROTTLE_CFG,
+	SHIKRA_SLAVE_HWKM,
+	SHIKRA_SLAVE_IMEM_CFG,
+	SHIKRA_SLAVE_MAPSS,
+	SHIKRA_SLAVE_MDSP_MPU_CFG,
+	SHIKRA_SLAVE_MESSAGE_RAM,
+	SHIKRA_SLAVE_MSS,
+	SHIKRA_SLAVE_PCIE_CFG,
+	SHIKRA_SLAVE_PDM,
+	SHIKRA_SLAVE_PIMEM_CFG,
+	SHIKRA_SLAVE_PKA_WRAPPER_CFG,
+	SHIKRA_SLAVE_PMIC_ARB,
+	SHIKRA_SLAVE_QDSS_CFG,
+	SHIKRA_SLAVE_QM_CFG,
+	SHIKRA_SLAVE_QM_MPU_CFG,
+	SHIKRA_SLAVE_QPIC,
+	SHIKRA_SLAVE_QUP_0,
+	SHIKRA_SLAVE_RPM,
+	SHIKRA_SLAVE_SDCC_1,
+	SHIKRA_SLAVE_SDCC_2,
+	SHIKRA_SLAVE_SECURITY,
+	SHIKRA_SLAVE_SNOC_CFG,
+	SHIKRA_SNOC_SF_THROTTLE_CFG,
+	SHIKRA_SLAVE_TLMM,
+	SHIKRA_SLAVE_TSCSS,
+	SHIKRA_SLAVE_USB2,
+	SHIKRA_SLAVE_USB3,
+	SHIKRA_SLAVE_VENUS_CFG,
+	SHIKRA_SLAVE_VENUS_THROTTLE_CFG,
+	SHIKRA_SLAVE_VSENSE_CTRL_CFG,
+	SHIKRA_SLAVE_SERVICE_CNOC,
+};
+
+static struct qcom_icc_node xm_dap = {
+	.id = SHIKRA_MASTER_QDSS_DAP,
+	.name = "xm_dap",
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_dap_links),
+	.links = xm_dap_links,
+};
+
+static const u16 llcc_mc_links[] = {
+	SHIKRA_SLAVE_EBI_CH0,
+};
+
+static struct qcom_icc_node llcc_mc = {
+	.id = SHIKRA_MASTER_LLCC,
+	.name = "llcc_mc",
+	.buswidth = 4,
+	.channels = 2,
+	.mas_rpm_id = 190,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(llcc_mc_links),
+	.links = llcc_mc_links,
+};
+
+static const u16 qnm_gpu_links[] = {
+	SHIKRA_SLAVE_LLCC,
+	SHIKRA_SLAVE_MEMNOC_SNOC,
+	SHIKRA_SLAVE_MEM_NOC_PCIE_SNOC,
+};
+
+static struct qcom_icc_node qnm_gpu = {
+	.id = SHIKRA_MASTER_GRAPHICS_3D,
+	.name = "qnm_gpu",
+	.buswidth = 16,
+	.qos.ap_owned = true,
+	.qos.qos_port = 6,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 0,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_gpu_links),
+	.links = qnm_gpu_links,
+};
+
+static const u16 qnm_mnoc_hf_links[] = {
+	SHIKRA_SLAVE_LLCC,
+	SHIKRA_SLAVE_MEMNOC_SNOC,
+	SHIKRA_SLAVE_MEM_NOC_PCIE_SNOC,
+};
+
+static struct qcom_icc_node qnm_mnoc_hf = {
+	.id = SHIKRA_MASTER_MNOC_HF_MEM_NOC,
+	.name = "qnm_mnoc_hf",
+	.buswidth = 16,
+	.qos.ap_owned = true,
+	.qos.qos_port = 7,
+	.qos.urg_fwd_en = true,
+	.qos.qos_mode = NOC_QOS_MODE_BYPASS,
+	.qos.areq_prio = 0,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_mnoc_hf_links),
+	.links = qnm_mnoc_hf_links,
+};
+
+static const u16 qnm_pcie_links[] = {
+	SHIKRA_SLAVE_LLCC,
+	SHIKRA_SLAVE_MEMNOC_SNOC,
+};
+
+static struct qcom_icc_node qnm_pcie = {
+	.id = SHIKRA_MASTER_ANOC_PCIE_MEM_NOC,
+	.name = "qnm_pcie",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 4,
+	.qos.urg_fwd_en = true,
+	.qos.qos_mode = NOC_QOS_MODE_BYPASS,
+	.qos.areq_prio = 0,
+	.mas_rpm_id = 185,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_pcie_links),
+	.links = qnm_pcie_links,
+};
+
+static const u16 qnm_snoc_sf_links[] = {
+	SHIKRA_SLAVE_LLCC,
+	SHIKRA_SLAVE_MEMNOC_SNOC,
+	SHIKRA_SLAVE_MEM_NOC_PCIE_SNOC,
+};
+
+static struct qcom_icc_node qnm_snoc_sf = {
+	.id = SHIKRA_MASTER_SNOC_SF_MEM_NOC,
+	.name = "qnm_snoc_sf",
+	.buswidth = 16,
+	.qos.ap_owned = true,
+	.qos.qos_port = 3,
+	.qos.urg_fwd_en = true,
+	.qos.qos_mode = NOC_QOS_MODE_BYPASS,
+	.qos.areq_prio = 0,
+	.mas_rpm_id = 76,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_snoc_sf_links),
+	.links = qnm_snoc_sf_links,
+};
+
+static const u16 xm_apps_links[] = {
+	SHIKRA_SLAVE_LLCC,
+	SHIKRA_SLAVE_MEMNOC_SNOC,
+	SHIKRA_SLAVE_MEM_NOC_PCIE_SNOC,
+};
+
+static struct qcom_icc_node xm_apps = {
+	.id = SHIKRA_MASTER_AMPSS_M0,
+	.name = "xm_apps",
+	.buswidth = 16,
+	.qos.ap_owned = true,
+	.qos.qos_port = 5,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 0,
+	.mas_rpm_id = 0,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_apps_links),
+	.links = xm_apps_links,
+};
+
+static const u16 xm_tcu_links[] = {
+	SHIKRA_SLAVE_LLCC,
+	SHIKRA_SLAVE_MEMNOC_SNOC,
+};
+
+static struct qcom_icc_node xm_tcu = {
+	.id = SHIKRA_MASTER_SYS_TCU,
+	.name = "xm_tcu",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 2,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 6,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_tcu_links),
+	.links = xm_tcu_links,
+};
+
+static const u16 qnm_camera_nrt_links[] = {
+	SHIKRA_SLAVE_MMNRT_VIRT,
+};
+
+static struct qcom_icc_node qnm_camera_nrt = {
+	.id = SHIKRA_MASTER_CAMNOC_SF,
+	.name = "qnm_camera_nrt",
+	.buswidth = 32,
+	.qos.ap_owned = true,
+	.qos.qos_port = 3,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 3,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_camera_nrt_links),
+	.links = qnm_camera_nrt_links,
+};
+
+static const u16 qxm_venus0_links[] = {
+	SHIKRA_SLAVE_MMNRT_VIRT,
+};
+
+static struct qcom_icc_node qxm_venus0 = {
+	.id = SHIKRA_MASTER_VIDEO_P0,
+	.name = "qxm_venus0",
+	.buswidth = 16,
+	.qos.ap_owned = true,
+	.qos.qos_port = 8,
+	.qos.urg_fwd_en = true,
+	.qos.qos_mode = NOC_QOS_MODE_BYPASS,
+	.qos.areq_prio = 0,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qxm_venus0_links),
+	.links = qxm_venus0_links,
+};
+
+static const u16 qxm_venus_cpu_links[] = {
+	SHIKRA_SLAVE_MMNRT_VIRT,
+};
+
+static struct qcom_icc_node qxm_venus_cpu = {
+	.id = SHIKRA_MASTER_VIDEO_PROC,
+	.name = "qxm_venus_cpu",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 12,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qxm_venus_cpu_links),
+	.links = qxm_venus_cpu_links,
+};
+
+static const u16 qnm_camera_rt_links[] = {
+	SHIKRA_SLAVE_MM_MEMNOC,
+};
+
+static struct qcom_icc_node qnm_camera_rt = {
+	.id = SHIKRA_MASTER_CAMNOC_HF,
+	.name = "qnm_camera_rt",
+	.buswidth = 32,
+	.qos.ap_owned = true,
+	.qos.qos_port = 9,
+	.qos.urg_fwd_en = true,
+	.qos.qos_mode = NOC_QOS_MODE_BYPASS,
+	.qos.areq_prio = 0,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_camera_rt_links),
+	.links = qnm_camera_rt_links,
+};
+
+static const u16 qxm_mdp0_links[] = {
+	SHIKRA_SLAVE_MM_MEMNOC,
+};
+
+static struct qcom_icc_node qxm_mdp0 = {
+	.id = SHIKRA_MASTER_MDP_PORT0,
+	.name = "qxm_mdp0",
+	.buswidth = 16,
+	.qos.ap_owned = true,
+	.qos.qos_port = 4,
+	.qos.urg_fwd_en = true,
+	.qos.qos_mode = NOC_QOS_MODE_BYPASS,
+	.qos.areq_prio = 0,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qxm_mdp0_links),
+	.links = qxm_mdp0_links,
+};
+
+static const u16 mmrt_virt_master_links[] = {
+	SHIKRA_SLAVE_MM_MEMNOC,
+};
+
+static struct qcom_icc_node mmrt_virt_master = {
+	.id = SHIKRA_MASTER_MMRT_VIRT,
+	.name = "mmrt_virt_master",
+	.buswidth = 16,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(mmrt_virt_master_links),
+	.links = mmrt_virt_master_links,
+};
+
+static const u16 qhm_snoc_cfg_links[] = {
+	SHIKRA_SLAVE_SERVICE_SNOC,
+};
+
+static struct qcom_icc_node qhm_snoc_cfg = {
+	.id = SHIKRA_MASTER_SNOC_CFG,
+	.name = "qhm_snoc_cfg",
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qhm_snoc_cfg_links),
+	.links = qhm_snoc_cfg_links,
+};
+
+static const u16 qhm_tic_links[] = {
+	SHIKRA_SLAVE_APPSS,
+	SHIKRA_SLAVE_MCUSS,
+	SHIKRA_SLAVE_WCSS,
+	SHIKRA_SLAVE_MEMNOC_SF,
+	SHIKRA_SNOC_CNOC_SLV,
+	SHIKRA_SLAVE_BOOTIMEM,
+	SHIKRA_SLAVE_OCIMEM,
+	SHIKRA_SLAVE_PIMEM,
+	SHIKRA_SLAVE_QDSS_STM,
+	SHIKRA_SLAVE_TCU,
+};
+
+static struct qcom_icc_node qhm_tic = {
+	.id = SHIKRA_MASTER_TIC,
+	.name = "qhm_tic",
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qhm_tic_links),
+	.links = qhm_tic_links,
+};
+
+static const u16 qnm_anoc_snoc_links[] = {
+	SHIKRA_SLAVE_MEMNOC_SF,
+};
+
+static struct qcom_icc_node qnm_anoc_snoc = {
+	.id = SHIKRA_MASTER_ANOC_SNOC,
+	.name = "qnm_anoc_snoc",
+	.buswidth = 16,
+	.mas_rpm_id = 110,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_anoc_snoc_links),
+	.links = qnm_anoc_snoc_links,
+};
+
+static const u16 qnm_memnoc_pcie_links[] = {
+	SHIKRA_SLAVE_PCIE2_0,
+};
+
+static struct qcom_icc_node qnm_memnoc_pcie = {
+	.id = SHIKRA_MASTER_MEMNOC_PCIE,
+	.name = "qnm_memnoc_pcie",
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_memnoc_pcie_links),
+	.links = qnm_memnoc_pcie_links,
+};
+
+static const u16 qnm_memnoc_snoc_links[] = {
+	SHIKRA_SLAVE_APPSS,
+	SHIKRA_SLAVE_MCUSS,
+	SHIKRA_SLAVE_WCSS,
+	SHIKRA_SNOC_CNOC_SLV,
+	SHIKRA_SLAVE_BOOTIMEM,
+	SHIKRA_SLAVE_OCIMEM,
+	SHIKRA_SLAVE_PIMEM,
+	SHIKRA_SLAVE_QDSS_STM,
+	SHIKRA_SLAVE_TCU,
+};
+
+static struct qcom_icc_node qnm_memnoc_snoc = {
+	.id = SHIKRA_MASTER_MEMNOC_SNOC,
+	.name = "qnm_memnoc_snoc",
+	.buswidth = 8,
+	.mas_rpm_id = 184,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_memnoc_snoc_links),
+	.links = qnm_memnoc_snoc_links,
+};
+
+static const u16 qxm_pimem_links[] = {
+	SHIKRA_SLAVE_MEMNOC_SF,
+	SHIKRA_SLAVE_OCIMEM,
+};
+
+static struct qcom_icc_node qxm_pimem = {
+	.id = SHIKRA_MASTER_PIMEM,
+	.name = "qxm_pimem",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 14,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qxm_pimem_links),
+	.links = qxm_pimem_links,
+};
+
+static const u16 xm_pcie2_0_links[] = {
+	SHIKRA_SLAVE_PCIE_MEMNOC,
+};
+
+static struct qcom_icc_node xm_pcie2_0 = {
+	.id = SHIKRA_MASTER_PCIE2_0,
+	.name = "xm_pcie2_0",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 21,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = 186,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_pcie2_0_links),
+	.links = xm_pcie2_0_links,
+};
+
+static const u16 qhm_qdss_bam_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node qhm_qdss_bam = {
+	.id = SHIKRA_MASTER_QDSS_BAM,
+	.name = "qhm_qdss_bam",
+	.buswidth = 4,
+	.qos.ap_owned = true,
+	.qos.qos_port = 2,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qhm_qdss_bam_links),
+	.links = qhm_qdss_bam_links,
+};
+
+static const u16 qhm_qpic_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node qhm_qpic = {
+	.id = SHIKRA_MASTER_QPIC,
+	.name = "qhm_qpic",
+	.buswidth = 4,
+	.qos.ap_owned = true,
+	.qos.qos_port = 1,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qhm_qpic_links),
+	.links = qhm_qpic_links,
+};
+
+static const u16 qhm_qup0_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node qhm_qup0 = {
+	.id = SHIKRA_MASTER_QUP_0,
+	.name = "qhm_qup0",
+	.buswidth = 4,
+	.qos.ap_owned = true,
+	.qos.qos_port = 0,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = 166,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qhm_qup0_links),
+	.links = qhm_qup0_links,
+};
+
+static const u16 qnm_cnoc_snoc_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node qnm_cnoc_snoc = {
+	.id = SHIKRA_CNOC_SNOC_MAS,
+	.name = "qnm_cnoc_snoc",
+	.buswidth = 4,
+	.qos.ap_owned = true,
+	.qos.qos_port = 7,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qnm_cnoc_snoc_links),
+	.links = qnm_cnoc_snoc_links,
+};
+
+static const u16 qxm_audio_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node qxm_audio = {
+	.id = SHIKRA_MASTER_AUDIO,
+	.name = "qxm_audio",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 22,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 3,
+	.mas_rpm_id = 78,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(qxm_audio_links),
+	.links = qxm_audio_links,
+};
+
+static const u16 xm_emac_0_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node xm_emac_0 = {
+	.id = SHIKRA_MASTER_EMAC_0,
+	.name = "xm_emac_0",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 19,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_emac_0_links),
+	.links = xm_emac_0_links,
+};
+
+static const u16 xm_emac_1_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node xm_emac_1 = {
+	.id = SHIKRA_MASTER_EMAC_1,
+	.name = "xm_emac_1",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 20,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_emac_1_links),
+	.links = xm_emac_1_links,
+};
+
+static const u16 xm_qdss_etr_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node xm_qdss_etr = {
+	.id = SHIKRA_MASTER_QDSS_ETR,
+	.name = "xm_qdss_etr",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 11,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_qdss_etr_links),
+	.links = xm_qdss_etr_links,
+};
+
+static const u16 xm_sdc1_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node xm_sdc1 = {
+	.id = SHIKRA_MASTER_SDCC_1,
+	.name = "xm_sdc1",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 13,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_sdc1_links),
+	.links = xm_sdc1_links,
+};
+
+static const u16 xm_sdc2_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node xm_sdc2 = {
+	.id = SHIKRA_MASTER_SDCC_2,
+	.name = "xm_sdc2",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 17,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_sdc2_links),
+	.links = xm_sdc2_links,
+};
+
+static const u16 xm_usb2_0_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node xm_usb2_0 = {
+	.id = SHIKRA_MASTER_USB2_0,
+	.name = "xm_usb2_0",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 24,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_usb2_0_links),
+	.links = xm_usb2_0_links,
+};
+
+static const u16 xm_usb3_0_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node xm_usb3_0 = {
+	.id = SHIKRA_MASTER_USB3,
+	.name = "xm_usb3_0",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 18,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(xm_usb3_0_links),
+	.links = xm_usb3_0_links,
+};
+
+static const u16 crypto_c0_links[] = {
+	SHIKRA_SLAVE_ANOC_SNOC,
+};
+
+static struct qcom_icc_node crypto_c0 = {
+	.id = SHIKRA_MASTER_CRYPTO_CORE0,
+	.name = "crypto_c0",
+	.buswidth = 8,
+	.qos.ap_owned = true,
+	.qos.qos_port = 16,
+	.qos.urg_fwd_en = false,
+	.qos.qos_mode = NOC_QOS_MODE_FIXED,
+	.qos.areq_prio = 2,
+	.mas_rpm_id = 23,
+	.slv_rpm_id = -1,
+	.num_links = ARRAY_SIZE(crypto_c0_links),
+	.links = crypto_c0_links,
+};
+
+/* Slave nodes */
+static struct qcom_icc_node qup0_core_slave = {
+	.id = SHIKRA_SLAVE_QUP_CORE_0,
+	.name = "qup0_core_slave",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 264,
+};
+
+static struct qcom_icc_node qhs_ahb2phy_usb = {
+	.id = SHIKRA_SLAVE_AHB2PHY_USB,
+	.name = "qhs_ahb2phy_usb",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_apss_throttle_cfg = {
+	.id = SHIKRA_SLAVE_APSS_THROTTLE_CFG,
+	.name = "qhs_apss_throttle_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_audio = {
+	.id = SHIKRA_SLAVE_AUDIO,
+	.name = "qhs_audio",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_boot_rom = {
+	.id = SHIKRA_SLAVE_BOOT_ROM,
+	.name = "qhs_boot_rom",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_camera_nrt_throttle_cfg = {
+	.id = SHIKRA_SLAVE_CAMERA_NRT_THROTTLE_CFG,
+	.name = "qhs_camera_nrt_throttle_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_camera_ss_cfg = {
+	.id = SHIKRA_SLAVE_CAMERA_CFG,
+	.name = "qhs_camera_ss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_cdsp_throttle_cfg = {
+	.id = SHIKRA_SLAVE_CDSP_THROTTLE_CFG,
+	.name = "qhs_cdsp_throttle_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_clk_ctl = {
+	.id = SHIKRA_SLAVE_CLK_CTL,
+	.name = "qhs_clk_ctl",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_compute_dsp_cfg = {
+	.id = SHIKRA_SLAVE_DSP_CFG,
+	.name = "qhs_compute_dsp_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_cpr_cx = {
+	.id = SHIKRA_SLAVE_RBCPR_CX_CFG,
+	.name = "qhs_cpr_cx",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_cpr_mx = {
+	.id = SHIKRA_SLAVE_RBCPR_MX_CFG,
+	.name = "qhs_cpr_mx",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_crypto0_cfg = {
+	.id = SHIKRA_SLAVE_CRYPTO_0_CFG,
+	.name = "qhs_crypto0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_ddr_ss_cfg = {
+	.id = SHIKRA_SLAVE_DDR_SS_CFG,
+	.name = "qhs_ddr_ss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_disp_ss_cfg = {
+	.id = SHIKRA_SLAVE_DISPLAY_CFG,
+	.name = "qhs_disp_ss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_emac0_cfg = {
+	.id = SHIKRA_SLAVE_EMAC0_CFG,
+	.name = "qhs_emac0_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_emac1_cfg = {
+	.id = SHIKRA_SLAVE_EMAC1_CFG,
+	.name = "qhs_emac1_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_gpu_cfg = {
+	.id = SHIKRA_SLAVE_GPU_CFG,
+	.name = "qhs_gpu_cfg",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_gpu_throttle_cfg = {
+	.id = SHIKRA_SLAVE_GPU_THROTTLE_CFG,
+	.name = "qhs_gpu_throttle_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_hwkm = {
+	.id = SHIKRA_SLAVE_HWKM,
+	.name = "qhs_hwkm",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_imem_cfg = {
+	.id = SHIKRA_SLAVE_IMEM_CFG,
+	.name = "qhs_imem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_mapss = {
+	.id = SHIKRA_SLAVE_MAPSS,
+	.name = "qhs_mapss",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_mdsp_mpu_cfg = {
+	.id = SHIKRA_SLAVE_MDSP_MPU_CFG,
+	.name = "qhs_mdsp_mpu_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_mesg_ram = {
+	.id = SHIKRA_SLAVE_MESSAGE_RAM,
+	.name = "qhs_mesg_ram",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_mss = {
+	.id = SHIKRA_SLAVE_MSS,
+	.name = "qhs_mss",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_pcie_cfg = {
+	.id = SHIKRA_SLAVE_PCIE_CFG,
+	.name = "qhs_pcie_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_pdm = {
+	.id = SHIKRA_SLAVE_PDM,
+	.name = "qhs_pdm",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_pimem_cfg = {
+	.id = SHIKRA_SLAVE_PIMEM_CFG,
+	.name = "qhs_pimem_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_pka_wrapper = {
+	.id = SHIKRA_SLAVE_PKA_WRAPPER_CFG,
+	.name = "qhs_pka_wrapper",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_pmic_arb = {
+	.id = SHIKRA_SLAVE_PMIC_ARB,
+	.name = "qhs_pmic_arb",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_qdss_cfg = {
+	.id = SHIKRA_SLAVE_QDSS_CFG,
+	.name = "qhs_qdss_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_qm_cfg = {
+	.id = SHIKRA_SLAVE_QM_CFG,
+	.name = "qhs_qm_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_qm_mpu_cfg = {
+	.id = SHIKRA_SLAVE_QM_MPU_CFG,
+	.name = "qhs_qm_mpu_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_qpic = {
+	.id = SHIKRA_SLAVE_QPIC,
+	.name = "qhs_qpic",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_qup0 = {
+	.id = SHIKRA_SLAVE_QUP_0,
+	.name = "qhs_qup0",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_rpm = {
+	.id = SHIKRA_SLAVE_RPM,
+	.name = "qhs_rpm",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_sdc1 = {
+	.id = SHIKRA_SLAVE_SDCC_1,
+	.name = "qhs_sdc1",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_sdc2 = {
+	.id = SHIKRA_SLAVE_SDCC_2,
+	.name = "qhs_sdc2",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_security = {
+	.id = SHIKRA_SLAVE_SECURITY,
+	.name = "qhs_security",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static const u16 qhs_snoc_cfg_links[] = {
+	SHIKRA_MASTER_SNOC_CFG,
+};
+
+static struct qcom_icc_node qhs_snoc_cfg = {
+	.id = SHIKRA_SLAVE_SNOC_CFG,
+	.name = "qhs_snoc_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = 1,
+	.links = qhs_snoc_cfg_links,
+};
+
+static struct qcom_icc_node qhs_snoc_sf_throttle_cfg = {
+	.id = SHIKRA_SNOC_SF_THROTTLE_CFG,
+	.name = "qhs_snoc_sf_throttle_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_tlmm = {
+	.id = SHIKRA_SLAVE_TLMM,
+	.name = "qhs_tlmm",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_tscss = {
+	.id = SHIKRA_SLAVE_TSCSS,
+	.name = "qhs_tscss",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_usb2 = {
+	.id = SHIKRA_SLAVE_USB2,
+	.name = "qhs_usb2",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_usb3 = {
+	.id = SHIKRA_SLAVE_USB3,
+	.name = "qhs_usb3",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_venus_cfg = {
+	.id = SHIKRA_SLAVE_VENUS_CFG,
+	.name = "qhs_venus_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_venus_throttle_cfg = {
+	.id = SHIKRA_SLAVE_VENUS_THROTTLE_CFG,
+	.name = "qhs_venus_throttle_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_vsense_ctrl_cfg = {
+	.id = SHIKRA_SLAVE_VSENSE_CTRL_CFG,
+	.name = "qhs_vsense_ctrl_cfg",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node srvc_cnoc = {
+	.id = SHIKRA_SLAVE_SERVICE_CNOC,
+	.name = "srvc_cnoc",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node ebi = {
+	.id = SHIKRA_SLAVE_EBI_CH0,
+	.name = "ebi",
+	.channels = 2,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 0,
+};
+
+static const u16 qns_llcc_links[] = {
+	SHIKRA_MASTER_LLCC,
+};
+
+static struct qcom_icc_node qns_llcc = {
+	.id = SHIKRA_SLAVE_LLCC,
+	.name = "qns_llcc",
+	.channels = 2,
+	.buswidth = 16,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 312,
+	.num_links = 1,
+	.links = qns_llcc_links,
+};
+
+static const u16 qns_memnoc_snoc_links[] = {
+	SHIKRA_MASTER_MEMNOC_SNOC,
+};
+
+static struct qcom_icc_node qns_memnoc_snoc = {
+	.id = SHIKRA_SLAVE_MEMNOC_SNOC,
+	.name = "qns_memnoc_snoc",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 314,
+	.num_links = 1,
+	.links = qns_memnoc_snoc_links,
+};
+
+static const u16 qns_pcie_links[] = {
+	SHIKRA_MASTER_MEMNOC_PCIE,
+};
+
+static struct qcom_icc_node qns_pcie = {
+	.id = SHIKRA_SLAVE_MEM_NOC_PCIE_SNOC,
+	.name = "qns_pcie",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = 1,
+	.links = qns_pcie_links,
+};
+
+static const u16 mmnrt_virt_slave_links[] = {
+	SHIKRA_MASTER_MMRT_VIRT,
+};
+
+static struct qcom_icc_node mmnrt_virt_slave = {
+	.id = SHIKRA_SLAVE_MMNRT_VIRT,
+	.name = "mmnrt_virt_slave",
+	.channels = 1,
+	.buswidth = 16,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = 1,
+	.links = mmnrt_virt_slave_links,
+};
+
+static const u16 qns_mm_memnoc_links[] = {
+	SHIKRA_MASTER_MNOC_HF_MEM_NOC,
+};
+
+static struct qcom_icc_node qns_mm_memnoc = {
+	.id = SHIKRA_SLAVE_MM_MEMNOC,
+	.name = "qns_mm_memnoc",
+	.channels = 1,
+	.buswidth = 16,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+	.num_links = 1,
+	.links = qns_mm_memnoc_links,
+};
+
+static struct qcom_icc_node qhs_apss = {
+	.id = SHIKRA_SLAVE_APPSS,
+	.name = "qhs_apss",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qhs_mcuss = {
+	.id = SHIKRA_SLAVE_MCUSS,
+	.name = "qhs_mcuss",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 319,
+};
+
+static struct qcom_icc_node qhs_wcss = {
+	.id = SHIKRA_SLAVE_WCSS,
+	.name = "qhs_wcss",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 23,
+};
+
+static const u16 qns_memnoc_sf_links[] = {
+	SHIKRA_MASTER_SNOC_SF_MEM_NOC,
+};
+
+static struct qcom_icc_node qns_memnoc_sf = {
+	.id = SHIKRA_SLAVE_MEMNOC_SF,
+	.name = "qns_memnoc_sf",
+	.channels = 1,
+	.buswidth = 16,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 313,
+	.num_links = 1,
+	.links = qns_memnoc_sf_links,
+};
+
+static const u16 qns_snoc_cnoc_links[] = {
+	SHIKRA_SNOC_CNOC_MAS,
+};
+
+static struct qcom_icc_node qns_snoc_cnoc = {
+	.id = SHIKRA_SNOC_CNOC_SLV,
+	.name = "qns_snoc_cnoc",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 25,
+	.num_links = 1,
+	.links = qns_snoc_cnoc_links,
+};
+
+static struct qcom_icc_node qxs_bootimem = {
+	.id = SHIKRA_SLAVE_BOOTIMEM,
+	.name = "qxs_bootimem",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node qxs_imem = {
+	.id = SHIKRA_SLAVE_OCIMEM,
+	.name = "qxs_imem",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 26,
+};
+
+static struct qcom_icc_node qxs_pimem = {
+	.id = SHIKRA_SLAVE_PIMEM,
+	.name = "qxs_pimem",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node srvc_snoc = {
+	.id = SHIKRA_SLAVE_SERVICE_SNOC,
+	.name = "srvc_snoc",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node xs_pcie2_0 = {
+	.id = SHIKRA_SLAVE_PCIE2_0,
+	.name = "xs_pcie2_0",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static struct qcom_icc_node xs_qdss_stm = {
+	.id = SHIKRA_SLAVE_QDSS_STM,
+	.name = "xs_qdss_stm",
+	.channels = 1,
+	.buswidth = 4,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 30,
+};
+
+static struct qcom_icc_node xs_sys_tcu_cfg = {
+	.id = SHIKRA_SLAVE_TCU,
+	.name = "xs_sys_tcu_cfg",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = -1,
+};
+
+static const u16 qns_pcie_memnoc_links[] = {
+	SHIKRA_MASTER_ANOC_PCIE_MEM_NOC,
+};
+
+static struct qcom_icc_node qns_pcie_memnoc = {
+	.id = SHIKRA_SLAVE_PCIE_MEMNOC,
+	.name = "qns_pcie_memnoc",
+	.channels = 1,
+	.buswidth = 8,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 317,
+	.num_links = 1,
+	.links = qns_pcie_memnoc_links,
+};
+
+static const u16 qns_anoc_snoc_links[] = {
+	SHIKRA_MASTER_ANOC_SNOC,
+};
+
+static struct qcom_icc_node qns_anoc_snoc = {
+	.id = SHIKRA_SLAVE_ANOC_SNOC,
+	.name = "qns_anoc_snoc",
+	.channels = 1,
+	.buswidth = 16,
+	.mas_rpm_id = -1,
+	.slv_rpm_id = 141,
+	.num_links = 1,
+	.links = qns_anoc_snoc_links,
+};
+
+/* NoC descriptors */
+static struct qcom_icc_node * const shikra_clk_virt_nodes[] = {
+	[MASTER_QUP_CORE_0] = &qup0_core_master,
+	[SLAVE_QUP_CORE_0] = &qup0_core_slave,
+};
+
+static const struct qcom_icc_desc shikra_clk_virt = {
+	.type = QCOM_ICC_QNOC,
+	.nodes = shikra_clk_virt_nodes,
+	.num_nodes = ARRAY_SIZE(shikra_clk_virt_nodes),
+	.bus_clk_desc = &qup_clk,
+	.keep_alive = true,
+};
+
+static struct qcom_icc_node * const shikra_config_noc_nodes[] = {
+	[SNOC_CNOC_MAS] = &qnm_snoc_cnoc,
+	[MASTER_QDSS_DAP] = &xm_dap,
+	[SLAVE_AHB2PHY_USB] = &qhs_ahb2phy_usb,
+	[SLAVE_APSS_THROTTLE_CFG] = &qhs_apss_throttle_cfg,
+	[SLAVE_AUDIO] = &qhs_audio,
+	[SLAVE_BOOT_ROM] = &qhs_boot_rom,
+	[SLAVE_CAMERA_NRT_THROTTLE_CFG] = &qhs_camera_nrt_throttle_cfg,
+	[SLAVE_CAMERA_CFG] = &qhs_camera_ss_cfg,
+	[SLAVE_CDSP_THROTTLE_CFG] = &qhs_cdsp_throttle_cfg,
+	[SLAVE_CLK_CTL] = &qhs_clk_ctl,
+	[SLAVE_DSP_CFG] = &qhs_compute_dsp_cfg,
+	[SLAVE_RBCPR_CX_CFG] = &qhs_cpr_cx,
+	[SLAVE_RBCPR_MX_CFG] = &qhs_cpr_mx,
+	[SLAVE_CRYPTO_0_CFG] = &qhs_crypto0_cfg,
+	[SLAVE_DDR_SS_CFG] = &qhs_ddr_ss_cfg,
+	[SLAVE_DISPLAY_CFG] = &qhs_disp_ss_cfg,
+	[SLAVE_EMAC0_CFG] = &qhs_emac0_cfg,
+	[SLAVE_EMAC1_CFG] = &qhs_emac1_cfg,
+	[SLAVE_GPU_CFG] = &qhs_gpu_cfg,
+	[SLAVE_GPU_THROTTLE_CFG] = &qhs_gpu_throttle_cfg,
+	[SLAVE_HWKM] = &qhs_hwkm,
+	[SLAVE_IMEM_CFG] = &qhs_imem_cfg,
+	[SLAVE_MAPSS] = &qhs_mapss,
+	[SLAVE_MDSP_MPU_CFG] = &qhs_mdsp_mpu_cfg,
+	[SLAVE_MESSAGE_RAM] = &qhs_mesg_ram,
+	[SLAVE_MSS] = &qhs_mss,
+	[SLAVE_PCIE_CFG] = &qhs_pcie_cfg,
+	[SLAVE_PDM] = &qhs_pdm,
+	[SLAVE_PIMEM_CFG] = &qhs_pimem_cfg,
+	[SLAVE_PKA_WRAPPER_CFG] = &qhs_pka_wrapper,
+	[SLAVE_PMIC_ARB] = &qhs_pmic_arb,
+	[SLAVE_QDSS_CFG] = &qhs_qdss_cfg,
+	[SLAVE_QM_CFG] = &qhs_qm_cfg,
+	[SLAVE_QM_MPU_CFG] = &qhs_qm_mpu_cfg,
+	[SLAVE_QPIC] = &qhs_qpic,
+	[SLAVE_QUP_0] = &qhs_qup0,
+	[SLAVE_RPM] = &qhs_rpm,
+	[SLAVE_SDCC_1] = &qhs_sdc1,
+	[SLAVE_SDCC_2] = &qhs_sdc2,
+	[SLAVE_SECURITY] = &qhs_security,
+	[SLAVE_SNOC_CFG] = &qhs_snoc_cfg,
+	[SNOC_SF_THROTTLE_CFG] = &qhs_snoc_sf_throttle_cfg,
+	[SLAVE_TLMM] = &qhs_tlmm,
+	[SLAVE_TSCSS] = &qhs_tscss,
+	[SLAVE_USB2] = &qhs_usb2,
+	[SLAVE_USB3] = &qhs_usb3,
+	[SLAVE_VENUS_CFG] = &qhs_venus_cfg,
+	[SLAVE_VENUS_THROTTLE_CFG] = &qhs_venus_throttle_cfg,
+	[SLAVE_VSENSE_CTRL_CFG] = &qhs_vsense_ctrl_cfg,
+	[SLAVE_SERVICE_CNOC] = &srvc_cnoc,
+};
+
+static const struct regmap_config shikra_config_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x8080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc shikra_config_noc = {
+	.type = QCOM_ICC_QNOC,
+	.nodes = shikra_config_noc_nodes,
+	.num_nodes = ARRAY_SIZE(shikra_config_noc_nodes),
+	.bus_clk_desc = &bus_1_clk,
+	.regmap_cfg = &shikra_config_noc_regmap_config,
+	.keep_alive = true,
+};
+
+static struct qcom_icc_node * const shikra_mc_virt_nodes[] = {
+	[MASTER_LLCC] = &llcc_mc,
+	[SLAVE_EBI_CH0] = &ebi,
+};
+
+static const struct qcom_icc_desc shikra_mc_virt = {
+	.type = QCOM_ICC_QNOC,
+	.nodes = shikra_mc_virt_nodes,
+	.num_nodes = ARRAY_SIZE(shikra_mc_virt_nodes),
+	.bus_clk_desc = &bimc_clk,
+	.keep_alive = true,
+	.ab_coeff = 152,
+};
+
+static struct qcom_icc_node * const shikra_mem_noc_core_nodes[] = {
+	[MASTER_GRAPHICS_3D] = &qnm_gpu,
+	[MASTER_MNOC_HF_MEM_NOC] = &qnm_mnoc_hf,
+	[MASTER_ANOC_PCIE_MEM_NOC] = &qnm_pcie,
+	[MASTER_SNOC_SF_MEM_NOC] = &qnm_snoc_sf,
+	[MASTER_AMPSS_M0] = &xm_apps,
+	[MASTER_SYS_TCU] = &xm_tcu,
+	[SLAVE_LLCC] = &qns_llcc,
+	[SLAVE_MEMNOC_SNOC] = &qns_memnoc_snoc,
+	[SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie,
+};
+
+static const struct regmap_config shikra_mem_noc_core_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x43080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc shikra_mem_noc_core = {
+	.type = QCOM_ICC_QNOC,
+	.nodes = shikra_mem_noc_core_nodes,
+	.num_nodes = ARRAY_SIZE(shikra_mem_noc_core_nodes),
+	.bus_clk_desc = &mem_1_clk,
+	.regmap_cfg = &shikra_mem_noc_core_regmap_config,
+	.intf_clocks = memnoc_intf_clocks,
+	.num_intf_clocks = ARRAY_SIZE(memnoc_intf_clocks),
+	.qos_offset = 0x28000,
+	.keep_alive = true,
+	.ab_coeff = 142,
+};
+
+static struct qcom_icc_node * const shikra_mmnrt_virt_nodes[] = {
+	[MASTER_CAMNOC_SF] = &qnm_camera_nrt,
+	[MASTER_VIDEO_P0] = &qxm_venus0,
+	[MASTER_VIDEO_PROC] = &qxm_venus_cpu,
+	[SLAVE_MMNRT_VIRT] = &mmnrt_virt_slave,
+};
+
+static const struct regmap_config shikra_sys_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x6a080,
+	.fast_io = true,
+};
+
+static const struct qcom_icc_desc shikra_mmnrt_virt = {
+	.type = QCOM_ICC_QNOC,
+	.nodes = shikra_mmnrt_virt_nodes,
+	.num_nodes = ARRAY_SIZE(shikra_mmnrt_virt_nodes),
+	.bus_clk_desc = &mmaxi_0_clk,
+	.regmap_cfg = &shikra_sys_noc_regmap_config,
+	.qos_offset = 0x51000,
+	.keep_alive = true,
+	.ab_coeff = 142,
+};
+
+static struct qcom_icc_node * const shikra_mmrt_virt_nodes[] = {
+	[MASTER_CAMNOC_HF] = &qnm_camera_rt,
+	[MASTER_MDP_PORT0] = &qxm_mdp0,
+	[MASTER_MMRT_VIRT] = &mmrt_virt_master,
+	[SLAVE_MM_MEMNOC] = &qns_mm_memnoc,
+};
+
+static const struct qcom_icc_desc shikra_mmrt_virt = {
+	.type = QCOM_ICC_QNOC,
+	.nodes = shikra_mmrt_virt_nodes,
+	.num_nodes = ARRAY_SIZE(shikra_mmrt_virt_nodes),
+	.bus_clk_desc = &mmaxi_1_clk,
+	.regmap_cfg = &shikra_sys_noc_regmap_config,
+	.qos_offset = 0x51000,
+	.keep_alive = true,
+	.ab_coeff = 142,
+};
+
+static struct qcom_icc_node * const shikra_sys_noc_nodes[] = {
+	[MASTER_SNOC_CFG] = &qhm_snoc_cfg,
+	[MASTER_TIC] = &qhm_tic,
+	[MASTER_ANOC_SNOC] = &qnm_anoc_snoc,
+	[MASTER_MEMNOC_PCIE] = &qnm_memnoc_pcie,
+	[MASTER_MEMNOC_SNOC] = &qnm_memnoc_snoc,
+	[MASTER_PIMEM] = &qxm_pimem,
+	[MASTER_PCIE2_0] = &xm_pcie2_0,
+	[MASTER_QDSS_BAM] = &qhm_qdss_bam,
+	[MASTER_QPIC] = &qhm_qpic,
+	[MASTER_QUP_0] = &qhm_qup0,
+	[CNOC_SNOC_MAS] = &qnm_cnoc_snoc,
+	[MASTER_AUDIO] = &qxm_audio,
+	[MASTER_EMAC_0] = &xm_emac_0,
+	[MASTER_EMAC_1] = &xm_emac_1,
+	[MASTER_QDSS_ETR] = &xm_qdss_etr,
+	[MASTER_SDCC_1] = &xm_sdc1,
+	[MASTER_SDCC_2] = &xm_sdc2,
+	[MASTER_USB2_0] = &xm_usb2_0,
+	[MASTER_USB3] = &xm_usb3_0,
+	[MASTER_CRYPTO_CORE0] = &crypto_c0,
+	[SLAVE_APPSS] = &qhs_apss,
+	[SLAVE_MCUSS] = &qhs_mcuss,
+	[SLAVE_WCSS] = &qhs_wcss,
+	[SLAVE_MEMNOC_SF] = &qns_memnoc_sf,
+	[SNOC_CNOC_SLV] = &qns_snoc_cnoc,
+	[SLAVE_BOOTIMEM] = &qxs_bootimem,
+	[SLAVE_OCIMEM] = &qxs_imem,
+	[SLAVE_PIMEM] = &qxs_pimem,
+	[SLAVE_SERVICE_SNOC] = &srvc_snoc,
+	[SLAVE_PCIE2_0] = &xs_pcie2_0,
+	[SLAVE_QDSS_STM] = &xs_qdss_stm,
+	[SLAVE_TCU] = &xs_sys_tcu_cfg,
+	[SLAVE_PCIE_MEMNOC] = &qns_pcie_memnoc,
+	[SLAVE_ANOC_SNOC] = &qns_anoc_snoc,
+};
+
+static const struct qcom_icc_desc shikra_sys_noc = {
+	.type = QCOM_ICC_QNOC,
+	.nodes = shikra_sys_noc_nodes,
+	.num_nodes = ARRAY_SIZE(shikra_sys_noc_nodes),
+	.bus_clk_desc = &bus_2_clk,
+	.regmap_cfg = &shikra_sys_noc_regmap_config,
+	.intf_clocks = sys_noc_intf_clocks,
+	.num_intf_clocks = ARRAY_SIZE(sys_noc_intf_clocks),
+	.qos_offset = 0x51000,
+	.keep_alive = true,
+};
+
+static const struct of_device_id shikra_qnoc_of_match[] = {
+	{ .compatible = "qcom,shikra-clk-virt", .data = &shikra_clk_virt },
+	{ .compatible = "qcom,shikra-config-noc", .data = &shikra_config_noc },
+	{ .compatible = "qcom,shikra-mc-virt", .data = &shikra_mc_virt },
+	{ .compatible = "qcom,shikra-mem-noc-core", .data = &shikra_mem_noc_core },
+	{ .compatible = "qcom,shikra-mmnrt-virt", .data = &shikra_mmnrt_virt },
+	{ .compatible = "qcom,shikra-mmrt-virt", .data = &shikra_mmrt_virt },
+	{ .compatible = "qcom,shikra-sys-noc", .data = &shikra_sys_noc },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, shikra_qnoc_of_match);
+
+static struct platform_driver shikra_qnoc_driver = {
+	.probe = qnoc_probe,
+	.remove = qnoc_remove,
+	.driver = {
+		.name = "qnoc-shikra",
+		.of_match_table = shikra_qnoc_of_match,
+		.sync_state = icc_sync_state,
+	},
+};
+
+static int __init qnoc_driver_init(void)
+{
+	return platform_driver_register(&shikra_qnoc_driver);
+}
+core_initcall(qnoc_driver_init);
+
+static void __exit qnoc_driver_exit(void)
+{
+	platform_driver_unregister(&shikra_qnoc_driver);
+}
+module_exit(qnoc_driver_exit);
+
+MODULE_DESCRIPTION("Qualcomm Shikra NoC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/interconnect/qcom/x1e80100.c b/drivers/interconnect/qcom/x1e80100.c
index 2ba2823c7860e..8075e0ff20597 100644
--- a/drivers/interconnect/qcom/x1e80100.c
+++ b/drivers/interconnect/qcom/x1e80100.c
@@ -173,6 +173,13 @@ static struct qcom_icc_node qhm_qspi = {
 	.name = "qhm_qspi",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -181,6 +188,13 @@ static struct qcom_icc_node qhm_qup1 = {
 	.name = "qhm_qup1",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -189,6 +203,13 @@ static struct qcom_icc_node xm_sdc4 = {
 	.name = "xm_sdc4",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -197,6 +218,13 @@ static struct qcom_icc_node xm_ufs_mem = {
 	.name = "xm_ufs_mem",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a1noc_snoc },
 };
@@ -205,6 +233,13 @@ static struct qcom_icc_node qhm_qup0 = {
 	.name = "qhm_qup0",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x16000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -213,6 +248,13 @@ static struct qcom_icc_node qhm_qup2 = {
 	.name = "qhm_qup2",
 	.channels = 1,
 	.buswidth = 4,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x11000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -221,6 +263,13 @@ static struct qcom_icc_node qxm_crypto = {
 	.name = "qxm_crypto",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x12000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -237,6 +286,13 @@ static struct qcom_icc_node xm_qdss_etr_0 = {
 	.name = "xm_qdss_etr_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x13000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -245,6 +301,13 @@ static struct qcom_icc_node xm_qdss_etr_1 = {
 	.name = "xm_qdss_etr_1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x14000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -253,6 +316,13 @@ static struct qcom_icc_node xm_sdc2 = {
 	.name = "xm_sdc2",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x15000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_a2noc_snoc },
 };
@@ -337,6 +407,13 @@ static struct qcom_icc_node alm_gpu_tcu = {
 	.name = "alm_gpu_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x271000 },
+		.prio = 1,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -345,6 +422,13 @@ static struct qcom_icc_node alm_pcie_tcu = {
 	.name = "alm_pcie_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x27d000 },
+		.prio = 3,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -353,6 +437,13 @@ static struct qcom_icc_node alm_sys_tcu = {
 	.name = "alm_sys_tcu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x273000 },
+		.prio = 6,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -370,6 +461,13 @@ static struct qcom_icc_node qnm_gpu = {
 	.name = "qnm_gpu",
 	.channels = 4,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 4,
+		.port_offsets = { 0x51000, 0x58000, 0xd1000, 0xd8000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -378,6 +476,13 @@ static struct qcom_icc_node qnm_lpass = {
 	.name = "qnm_lpass",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x275000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -387,6 +492,13 @@ static struct qcom_icc_node qnm_mnoc_hf = {
 	.name = "qnm_mnoc_hf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x53000, 0xd3000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -395,6 +507,13 @@ static struct qcom_icc_node qnm_mnoc_sf = {
 	.name = "qnm_mnoc_sf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x55000, 0xd5000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -403,6 +522,13 @@ static struct qcom_icc_node qnm_nsp_noc = {
 	.name = "qnm_nsp_noc",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x57000, 0xd7000 },
+		.prio = 0,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -412,6 +538,13 @@ static struct qcom_icc_node qnm_pcie = {
 	.name = "qnm_pcie",
 	.channels = 1,
 	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x277000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 2,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc },
 };
@@ -420,6 +553,13 @@ static struct qcom_icc_node qnm_snoc_sf = {
 	.name = "qnm_snoc_sf",
 	.channels = 1,
 	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x27b000 },
+		.prio = 2,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 3,
 	.link_nodes = { &qns_gem_noc_cnoc, &qns_llcc,
 			&qns_pcie },
@@ -429,6 +569,13 @@ static struct qcom_icc_node xm_gic = {
 	.name = "xm_gic",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x27f000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_llcc },
 };
@@ -469,6 +616,13 @@ static struct qcom_icc_node qnm_av1_enc = {
 	.name = "qnm_av1_enc",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2f000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -477,6 +631,13 @@ static struct qcom_icc_node qnm_camnoc_hf = {
 	.name = "qnm_camnoc_hf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x28000, 0x29000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -485,6 +646,13 @@ static struct qcom_icc_node qnm_camnoc_icp = {
 	.name = "qnm_camnoc_icp",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x2a000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -493,6 +661,13 @@ static struct qcom_icc_node qnm_camnoc_sf = {
 	.name = "qnm_camnoc_sf",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x2b000, 0x2c000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -501,6 +676,13 @@ static struct qcom_icc_node qnm_eva = {
 	.name = "qnm_eva",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x33000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -509,6 +691,13 @@ static struct qcom_icc_node qnm_mdp = {
 	.name = "qnm_mdp",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x2d000, 0x2e000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_hf },
 };
@@ -517,6 +706,13 @@ static struct qcom_icc_node qnm_video = {
 	.name = "qnm_video",
 	.channels = 2,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 2,
+		.port_offsets = { 0x30000, 0x31000 },
+		.prio = 0,
+		.urg_fwd = 1,
+		.prio_fwd_disable = 0,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -525,6 +721,13 @@ static struct qcom_icc_node qnm_video_cv_cpu = {
 	.name = "qnm_video_cv_cpu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x32000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -533,6 +736,13 @@ static struct qcom_icc_node qnm_video_v_cpu = {
 	.name = "qnm_video_v_cpu",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x34000 },
+		.prio = 4,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_mem_noc_sf },
 };
@@ -573,6 +783,13 @@ static struct qcom_icc_node xm_pcie_3 = {
 	.name = "xm_pcie_3",
 	.channels = 1,
 	.buswidth = 64,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x7000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_north_gem_noc },
 };
@@ -581,6 +798,13 @@ static struct qcom_icc_node xm_pcie_4 = {
 	.name = "xm_pcie_4",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x8000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_north_gem_noc },
 };
@@ -589,6 +813,13 @@ static struct qcom_icc_node xm_pcie_5 = {
 	.name = "xm_pcie_5",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x9000 },
+		.prio = 3,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_north_gem_noc },
 };
@@ -597,6 +828,13 @@ static struct qcom_icc_node xm_pcie_0 = {
 	.name = "xm_pcie_0",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x9000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -605,6 +843,13 @@ static struct qcom_icc_node xm_pcie_1 = {
 	.name = "xm_pcie_1",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -613,6 +858,13 @@ static struct qcom_icc_node xm_pcie_2 = {
 	.name = "xm_pcie_2",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -621,6 +873,13 @@ static struct qcom_icc_node xm_pcie_6a = {
 	.name = "xm_pcie_6a",
 	.channels = 1,
 	.buswidth = 32,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -629,6 +888,13 @@ static struct qcom_icc_node xm_pcie_6b = {
 	.name = "xm_pcie_6b",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_pcie_south_gem_noc },
 };
@@ -653,6 +919,13 @@ static struct qcom_icc_node qnm_gic = {
 	.name = "qnm_gic",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x1c000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_gemnoc_sf },
 };
@@ -685,6 +958,13 @@ static struct qcom_icc_node xm_usb2_0 = {
 	.name = "xm_usb2_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x6000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_north_snoc },
 };
@@ -693,6 +973,13 @@ static struct qcom_icc_node xm_usb3_mp = {
 	.name = "xm_usb3_mp",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0x7000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_north_snoc },
 };
@@ -701,6 +988,13 @@ static struct qcom_icc_node xm_usb3_0 = {
 	.name = "xm_usb3_0",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xa000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -709,6 +1003,13 @@ static struct qcom_icc_node xm_usb3_1 = {
 	.name = "xm_usb3_1",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xb000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -717,6 +1018,13 @@ static struct qcom_icc_node xm_usb3_2 = {
 	.name = "xm_usb3_2",
 	.channels = 1,
 	.buswidth = 8,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xc000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -725,6 +1033,13 @@ static struct qcom_icc_node xm_usb4_0 = {
 	.name = "xm_usb4_0",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xd000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -733,6 +1048,13 @@ static struct qcom_icc_node xm_usb4_1 = {
 	.name = "xm_usb4_1",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xe000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -741,6 +1063,13 @@ static struct qcom_icc_node xm_usb4_2 = {
 	.name = "xm_usb4_2",
 	.channels = 1,
 	.buswidth = 16,
+	.qosbox = &(const struct qcom_icc_qosbox) {
+		.num_ports = 1,
+		.port_offsets = { 0xf000 },
+		.prio = 2,
+		.urg_fwd = 0,
+		.prio_fwd_disable = 1,
+	},
 	.num_links = 1,
 	.link_nodes = { &qns_aggre_usb_south_snoc },
 };
@@ -1466,11 +1795,21 @@ static struct qcom_icc_node * const aggre1_noc_nodes[] = {
 	[SLAVE_A1NOC_SNOC] = &qns_a1noc_snoc,
 };
 
+static const struct regmap_config x1e80100_aggre1_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x14400,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_aggre1_noc = {
+	.config = &x1e80100_aggre1_noc_regmap_config,
 	.nodes = aggre1_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre1_noc_nodes),
 	.bcms = aggre1_noc_bcms,
 	.num_bcms = ARRAY_SIZE(aggre1_noc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const aggre2_noc_bcms[] = {
@@ -1488,7 +1827,16 @@ static struct qcom_icc_node * const aggre2_noc_nodes[] = {
 	[SLAVE_A2NOC_SNOC] = &qns_a2noc_snoc,
 };
 
+static const struct regmap_config x1e80100_aggre2_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1c400,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_aggre2_noc = {
+	.config = &x1e80100_aggre2_noc_regmap_config,
 	.nodes = aggre2_noc_nodes,
 	.num_nodes = ARRAY_SIZE(aggre2_noc_nodes),
 	.bcms = aggre2_noc_bcms,
@@ -1573,7 +1921,16 @@ static struct qcom_icc_node * const cnoc_cfg_nodes[] = {
 	[SLAVE_TCU] = &xs_sys_tcu_cfg,
 };
 
+static const struct regmap_config x1e80100_cnoc_cfg_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x6600,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_cnoc_cfg = {
+	.config = &x1e80100_cnoc_cfg_regmap_config,
 	.nodes = cnoc_cfg_nodes,
 	.num_nodes = ARRAY_SIZE(cnoc_cfg_nodes),
 	.bcms = cnoc_cfg_bcms,
@@ -1603,7 +1960,16 @@ static struct qcom_icc_node * const cnoc_main_nodes[] = {
 	[SLAVE_PCIE_6B] = &xs_pcie_6b,
 };
 
+static const struct regmap_config x1e80100_cnoc_main_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x14400,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_cnoc_main = {
+	.config = &x1e80100_cnoc_main_regmap_config,
 	.nodes = cnoc_main_nodes,
 	.num_nodes = ARRAY_SIZE(cnoc_main_nodes),
 	.bcms = cnoc_main_bcms,
@@ -1633,7 +1999,16 @@ static struct qcom_icc_node * const gem_noc_nodes[] = {
 	[SLAVE_MEM_NOC_PCIE_SNOC] = &qns_pcie,
 };
 
+static const struct regmap_config x1e80100_gem_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x311200,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_gem_noc = {
+	.config = &x1e80100_gem_noc_regmap_config,
 	.nodes = gem_noc_nodes,
 	.num_nodes = ARRAY_SIZE(gem_noc_nodes),
 	.bcms = gem_noc_bcms,
@@ -1648,7 +2023,16 @@ static struct qcom_icc_node * const lpass_ag_noc_nodes[] = {
 	[SLAVE_LPASS_GEM_NOC] = &qns_lpass_ag_noc_gemnoc,
 };
 
+static const struct regmap_config x1e80100_lpass_ag_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xe080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_lpass_ag_noc = {
+	.config = &x1e80100_lpass_ag_noc_regmap_config,
 	.nodes = lpass_ag_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_ag_noc_nodes),
 	.bcms = lpass_ag_noc_bcms,
@@ -1664,7 +2048,16 @@ static struct qcom_icc_node * const lpass_lpiaon_noc_nodes[] = {
 	[SLAVE_LPIAON_NOC_LPASS_AG_NOC] = &qns_lpass_aggnoc,
 };
 
+static const struct regmap_config x1e80100_lpass_lpiaon_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x19080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_lpass_lpiaon_noc = {
+	.config = &x1e80100_lpass_lpiaon_noc_regmap_config,
 	.nodes = lpass_lpiaon_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_lpiaon_noc_nodes),
 	.bcms = lpass_lpiaon_noc_bcms,
@@ -1679,7 +2072,16 @@ static struct qcom_icc_node * const lpass_lpicx_noc_nodes[] = {
 	[SLAVE_LPICX_NOC_LPIAON_NOC] = &qns_lpi_aon_noc,
 };
 
+static const struct regmap_config x1e80100_lpass_lpicx_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x3a200,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_lpass_lpicx_noc = {
+	.config = &x1e80100_lpass_lpicx_noc_regmap_config,
 	.nodes = lpass_lpicx_noc_nodes,
 	.num_nodes = ARRAY_SIZE(lpass_lpicx_noc_nodes),
 	.bcms = lpass_lpicx_noc_bcms,
@@ -1724,7 +2126,16 @@ static struct qcom_icc_node * const mmss_noc_nodes[] = {
 	[SLAVE_SERVICE_MNOC] = &srvc_mnoc,
 };
 
+static const struct regmap_config x1e80100_mmss_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x5b800,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_mmss_noc = {
+	.config = &x1e80100_mmss_noc_regmap_config,
 	.nodes = mmss_noc_nodes,
 	.num_nodes = ARRAY_SIZE(mmss_noc_nodes),
 	.bcms = mmss_noc_bcms,
@@ -1740,7 +2151,16 @@ static struct qcom_icc_node * const nsp_noc_nodes[] = {
 	[SLAVE_CDSP_MEM_NOC] = &qns_nsp_gemnoc,
 };
 
+static const struct regmap_config x1e80100_nsp_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xe080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_nsp_noc = {
+	.config = &x1e80100_nsp_noc_regmap_config,
 	.nodes = nsp_noc_nodes,
 	.num_nodes = ARRAY_SIZE(nsp_noc_nodes),
 	.bcms = nsp_noc_bcms,
@@ -1757,7 +2177,16 @@ static struct qcom_icc_node * const pcie_center_anoc_nodes[] = {
 	[SLAVE_ANOC_PCIE_GEM_NOC] = &qns_pcie_mem_noc,
 };
 
+static const struct regmap_config x1e80100_pcie_center_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x7000,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_pcie_center_anoc = {
+	.config = &x1e80100_pcie_center_anoc_regmap_config,
 	.nodes = pcie_center_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_center_anoc_nodes),
 	.bcms = pcie_center_anoc_bcms,
@@ -1774,7 +2203,16 @@ static struct qcom_icc_node * const pcie_north_anoc_nodes[] = {
 	[SLAVE_PCIE_NORTH] = &qns_pcie_north_gem_noc,
 };
 
+static const struct regmap_config x1e80100_pcie_north_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x9080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_pcie_north_anoc = {
+	.config = &x1e80100_pcie_north_anoc_regmap_config,
 	.nodes = pcie_north_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_north_anoc_nodes),
 	.bcms = pcie_north_anoc_bcms,
@@ -1793,7 +2231,16 @@ static struct qcom_icc_node * const pcie_south_anoc_nodes[] = {
 	[SLAVE_PCIE_SOUTH] = &qns_pcie_south_gem_noc,
 };
 
+static const struct regmap_config x1e80100_pcie_south_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xd080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_pcie_south_anoc = {
+	.config = &x1e80100_pcie_south_anoc_regmap_config,
 	.nodes = pcie_south_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(pcie_south_anoc_nodes),
 	.bcms = pcie_south_anoc_bcms,
@@ -1815,7 +2262,16 @@ static struct qcom_icc_node * const system_noc_nodes[] = {
 	[SLAVE_SNOC_GEM_NOC_SF] = &qns_gemnoc_sf,
 };
 
+static const struct regmap_config x1e80100_system_noc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x1c080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_system_noc = {
+	.config = &x1e80100_system_noc_regmap_config,
 	.nodes = system_noc_nodes,
 	.num_nodes = ARRAY_SIZE(system_noc_nodes),
 	.bcms = system_noc_bcms,
@@ -1831,7 +2287,16 @@ static struct qcom_icc_node * const usb_center_anoc_nodes[] = {
 	[SLAVE_USB_NOC_SNOC] = &qns_aggre_usb_snoc,
 };
 
+static const struct regmap_config x1e80100_usb_center_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x8800,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_usb_center_anoc = {
+	.config = &x1e80100_usb_center_anoc_regmap_config,
 	.nodes = usb_center_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(usb_center_anoc_nodes),
 	.bcms = usb_center_anoc_bcms,
@@ -1847,11 +2312,21 @@ static struct qcom_icc_node * const usb_north_anoc_nodes[] = {
 	[SLAVE_AGGRE_USB_NORTH] = &qns_aggre_usb_north_snoc,
 };
 
+static const struct regmap_config x1e80100_usb_north_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0x7080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_usb_north_anoc = {
+	.config = &x1e80100_usb_north_anoc_regmap_config,
 	.nodes = usb_north_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(usb_north_anoc_nodes),
 	.bcms = usb_north_anoc_bcms,
 	.num_bcms = ARRAY_SIZE(usb_north_anoc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static struct qcom_icc_bcm * const usb_south_anoc_bcms[] = {
@@ -1867,11 +2342,21 @@ static struct qcom_icc_node * const usb_south_anoc_nodes[] = {
 	[SLAVE_AGGRE_USB_SOUTH] = &qns_aggre_usb_south_snoc,
 };
 
+static const struct regmap_config x1e80100_usb_south_anoc_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+	.max_register = 0xf080,
+	.fast_io = true,
+};
+
 static const struct qcom_icc_desc x1e80100_usb_south_anoc = {
+	.config = &x1e80100_usb_south_anoc_regmap_config,
 	.nodes = usb_south_anoc_nodes,
 	.num_nodes = ARRAY_SIZE(usb_south_anoc_nodes),
 	.bcms = usb_south_anoc_bcms,
 	.num_bcms = ARRAY_SIZE(usb_south_anoc_bcms),
+	.qos_requires_clocks = true,
 };
 
 static const struct of_device_id qnoc_of_match[] = {
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index f86262b11416d..18d3d68af7cdf 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -384,6 +384,17 @@ config SPRD_IOMMU
 
 	  Say Y here if you want to use the multimedia devices listed above.
 
+config VSI_IOMMU
+	tristate "Verisilicon IOMMU Support"
+	depends on (ARCH_ROCKCHIP && ARM64) || COMPILE_TEST
+	select IOMMU_API
+	help
+	  Support for IOMMUs used by Verisilicon sub-systems like video
+	  decoders or encoder hardware blocks.
+
+	  Say Y here if you want to use this IOMMU in front of these
+	  hardware blocks.
+
 config IOMMU_DEBUG_PAGEALLOC
 	bool "Debug IOMMU mappings against page allocations"
 	depends on DEBUG_PAGEALLOC && IOMMU_API && PAGE_EXTENSION
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 0275821f4ef98..887af357a7c96 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -36,4 +36,5 @@ obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o
 obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o
 obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o
 obj-$(CONFIG_APPLE_DART) += apple-dart.o
+obj-$(CONFIG_VSI_IOMMU) += vsi-iommu.o
 obj-$(CONFIG_IOMMU_DEBUG_PAGEALLOC) += iommu-debug-pagealloc.o
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h
index 834d8fabfba38..af720bf14914d 100644
--- a/drivers/iommu/amd/amd_iommu.h
+++ b/drivers/iommu/amd/amd_iommu.h
@@ -44,7 +44,7 @@ int amd_iommu_enable_faulting(unsigned int cpu);
 extern int amd_iommu_guest_ir;
 extern enum protection_domain_mode amd_iommu_pgtable;
 extern int amd_iommu_gpt_level;
-extern u8 amd_iommu_hpt_level;
+extern u8 amd_iommu_hpt_vasize;
 extern unsigned long amd_iommu_pgsize_bitmap;
 extern bool amd_iommu_hatdis;
 
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index f9f7180878930..d2c64e2e9f056 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -51,10 +51,6 @@
 #define MMIO_GET_BUS(x) (((x) & MMIO_RANGE_BUS_MASK) >> MMIO_RANGE_BUS_SHIFT)
 #define MMIO_MSI_NUM(x)	((x) & 0x1f)
 
-/* Flag masks for the AMD IOMMU exclusion range */
-#define MMIO_EXCL_ENABLE_MASK 0x01ULL
-#define MMIO_EXCL_ALLOW_MASK  0x02ULL
-
 /* Used offsets into the MMIO space */
 #define MMIO_DEV_TABLE_OFFSET   0x0000
 #define MMIO_CMD_BUF_OFFSET     0x0008
@@ -231,7 +227,6 @@
 #define DEV_ENTRY_IR            0x3d
 #define DEV_ENTRY_IW            0x3e
 #define DEV_ENTRY_NO_PAGE_FAULT	0x62
-#define DEV_ENTRY_EX            0x67
 #define DEV_ENTRY_SYSMGT1       0x68
 #define DEV_ENTRY_SYSMGT2       0x69
 #define DTE_DATA1_SYSMGT_MASK	GENMASK_ULL(41, 40)
@@ -305,9 +300,6 @@
 
 #define GA_GUEST_NR		0x1
 
-#define IOMMU_IN_ADDR_BIT_SIZE  52
-#define IOMMU_OUT_ADDR_BIT_SIZE 52
-
 /*
  * This bitmap is used to advertise the page sizes our hardware support
  * to the IOMMU core, which will then use this information to split
@@ -389,8 +381,6 @@
 #define IOMMU_PROT_IR 0x01
 #define IOMMU_PROT_IW 0x02
 
-#define IOMMU_UNITY_MAP_FLAG_EXCL_RANGE	(1 << 2)
-
 /* IOMMU capabilities */
 #define IOMMU_CAP_IOTLB   24
 #define IOMMU_CAP_NPCACHE 26
@@ -400,6 +390,7 @@
 #define IOMMU_IVINFO_OFFSET     36
 #define IOMMU_IVINFO_EFRSUP     BIT(0)
 #define IOMMU_IVINFO_DMA_REMAP  BIT(1)
+#define IOMMU_IVINFO_VASIZE	GENMASK_ULL(21, 15)
 
 /* IOMMU Feature Reporting Field (for IVHD type 10h */
 #define IOMMU_FEAT_GASUP_SHIFT	6
@@ -685,11 +676,6 @@ struct amd_iommu {
 	/* pci domain of this IOMMU */
 	struct amd_iommu_pci_seg *pci_seg;
 
-	/* start of exclusion range of that IOMMU */
-	u64 exclusion_start;
-	/* length of exclusion range of that IOMMU */
-	u64 exclusion_length;
-
 	/* command buffer virtual address */
 	u8 *cmd_buf;
 	u32 cmd_buf_head;
@@ -948,12 +934,13 @@ static inline int get_hpet_devid(int id)
 }
 
 enum amd_iommu_intr_mode_type {
-	AMD_IOMMU_GUEST_IR_LEGACY,
-
-	/* This mode is not visible to users. It is used when
-	 * we cannot fully enable vAPIC and fallback to only support
-	 * legacy interrupt remapping via 128-bit IRTE.
+	/*
+	 * The legacy format mode is not visible to users to prevent the user
+	 * from crashing x2APIC systems, which for all intents and purposes
+	 * require 128-bit IRTEs.   The legacy format will be forced as needed
+	 * when hardware doesn't support 128-bit IRTEs.
 	 */
+	AMD_IOMMU_GUEST_IR_LEGACY,
 	AMD_IOMMU_GUEST_IR_LEGACY_GA,
 	AMD_IOMMU_GUEST_IR_VAPIC,
 };
diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c
index 3bdb380d23e9a..d4dc9b2a50f38 100644
--- a/drivers/iommu/amd/init.c
+++ b/drivers/iommu/amd/init.c
@@ -155,8 +155,8 @@ bool amd_iommu_dump;
 bool amd_iommu_irq_remap __read_mostly;
 
 enum protection_domain_mode amd_iommu_pgtable = PD_MODE_V1;
-/* Host page table level */
-u8 amd_iommu_hpt_level;
+/* Virtual address size */
+u8 amd_iommu_hpt_vasize;
 /* Guest page table level */
 int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL;
 
@@ -355,28 +355,6 @@ static void iommu_write_l2(struct amd_iommu *iommu, u8 address, u32 val)
  *
  ****************************************************************************/
 
-/*
- * This function set the exclusion range in the IOMMU. DMA accesses to the
- * exclusion range are passed through untranslated
- */
-static void iommu_set_exclusion_range(struct amd_iommu *iommu)
-{
-	u64 start = iommu->exclusion_start & PAGE_MASK;
-	u64 limit = (start + iommu->exclusion_length - 1) & PAGE_MASK;
-	u64 entry;
-
-	if (!iommu->exclusion_start)
-		return;
-
-	entry = start | MMIO_EXCL_ENABLE_MASK;
-	memcpy_toio(iommu->mmio_base + MMIO_EXCL_BASE_OFFSET,
-			&entry, sizeof(entry));
-
-	entry = limit;
-	memcpy_toio(iommu->mmio_base + MMIO_EXCL_LIMIT_OFFSET,
-			&entry, sizeof(entry));
-}
-
 static void iommu_set_cwwb_range(struct amd_iommu *iommu)
 {
 	u64 start = iommu_virt_to_phys((void *)iommu->cmd_sem);
@@ -972,8 +950,8 @@ static int iommu_init_ga_log(struct amd_iommu *iommu)
 {
 	int nid = iommu->dev ? dev_to_node(&iommu->dev->dev) : NUMA_NO_NODE;
 
-	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
-		return 0;
+	if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)))
+		return -EINVAL;
 
 	iommu->ga_log = iommu_alloc_pages_node_sz(nid, GFP_KERNEL, GA_LOG_SIZE);
 	if (!iommu->ga_log)
@@ -2905,7 +2883,6 @@ static void early_enable_iommu(struct amd_iommu *iommu)
 	iommu_init_flags(iommu);
 	iommu_set_device_table(iommu);
 	iommu_enable_command_buffer(iommu);
-	iommu_set_exclusion_range(iommu);
 	iommu_enable_gt(iommu);
 	iommu_enable_ga(iommu);
 	iommu_enable_xt(iommu);
@@ -3022,8 +2999,10 @@ static void enable_iommus_vapic(void)
 			return;
 	}
 
-	if (AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir) &&
-	    !check_feature(FEATURE_GAM_VAPIC)) {
+	if (!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir))
+		return;
+
+	if (!check_feature(FEATURE_GAM_VAPIC)) {
 		amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_LEGACY_GA;
 		return;
 	}
@@ -3110,6 +3089,9 @@ static void __init free_iommu_resources(void)
 /* SB IOAPIC is always on this device in AMD systems */
 #define IOAPIC_SB_DEVID		((0x00 << 8) | PCI_DEVFN(0x14, 0))
 
+/* SB IOAPIC for Hygon family 18h model 4h is on the device 0xb */
+#define IOAPIC_SB_DEVID_FAM18H_M4H	((0x00 << 8) | PCI_DEVFN(0xb, 0))
+
 static bool __init check_ioapic_information(void)
 {
 	const char *fw_bug = FW_BUG;
@@ -3135,7 +3117,12 @@ static bool __init check_ioapic_information(void)
 			pr_err("%s: IOAPIC[%d] not in IVRS table\n",
 				fw_bug, id);
 			ret = false;
-		} else if (devid == IOAPIC_SB_DEVID) {
+		} else if (devid == IOAPIC_SB_DEVID ||
+			   (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON &&
+			    boot_cpu_data.x86 == 0x18 &&
+			    boot_cpu_data.x86_model >= 0x4 &&
+			    boot_cpu_data.x86_model <= 0xf &&
+			    devid == IOAPIC_SB_DEVID_FAM18H_M4H)) {
 			has_sb_ioapic = true;
 			ret           = true;
 		}
@@ -3202,7 +3189,7 @@ static int __init early_amd_iommu_init(void)
 	struct acpi_table_header *ivrs_base;
 	int ret;
 	acpi_status status;
-	u8 efr_hats;
+	u8 efr_hats, max_vasize;
 
 	if (!amd_iommu_detected)
 		return -ENODEV;
@@ -3232,6 +3219,10 @@ static int __init early_amd_iommu_init(void)
 
 	ivinfo_init(ivrs_base);
 
+	max_vasize = FIELD_GET(IOMMU_IVINFO_VASIZE, amd_iommu_ivinfo);
+	if (!max_vasize)
+		max_vasize = 64;
+
 	amd_iommu_target_ivhd_type = get_highest_supported_ivhd_type(ivrs_base);
 	DUMP_printk("Using IVHD type %#x\n", amd_iommu_target_ivhd_type);
 
@@ -3254,7 +3245,8 @@ static int __init early_amd_iommu_init(void)
 		 * efr[HATS] bits specify the maximum host translation level
 		 * supported, with LEVEL 4 being initial max level.
 		 */
-		amd_iommu_hpt_level = efr_hats + PAGE_MODE_4_LEVEL;
+		amd_iommu_hpt_vasize = min_t(unsigned int, max_vasize,
+					 (efr_hats + PAGE_MODE_4_LEVEL - 1) * 9 + 21);
 	} else {
 		pr_warn_once(FW_BUG "Disable host address translation due to invalid translation level (%#x).\n",
 			     efr_hats);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 57dc8fabc7d9b..84cad43dc188c 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -2718,8 +2718,7 @@ static struct iommu_domain *amd_iommu_domain_alloc_paging_v1(struct device *dev,
 	else
 		cfg.common.features |= BIT(PT_FEAT_FLUSH_RANGE);
 
-	cfg.common.hw_max_vasz_lg2 =
-		min(64, (amd_iommu_hpt_level - 1) * 9 + 21);
+	cfg.common.hw_max_vasz_lg2 = amd_iommu_hpt_vasize;
 	cfg.common.hw_max_oasz_lg2 = 52;
 	cfg.starting_level = 2;
 	domain->domain.ops = &amdv1_ops;
@@ -3086,9 +3085,6 @@ static void amd_iommu_get_resv_regions(struct device *dev,
 			prot |= IOMMU_READ;
 		if (entry->prot & IOMMU_PROT_IW)
 			prot |= IOMMU_WRITE;
-		if (entry->prot & IOMMU_UNITY_MAP_FLAG_EXCL_RANGE)
-			/* Exclusion range */
-			type = IOMMU_RESV_RESERVED;
 
 		region = iommu_alloc_resv_region(entry->address_start,
 						 length, prot, type,
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 54d96e847f161..3fdcbbf273fa2 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -2142,18 +2142,21 @@ EXPORT_SYMBOL_GPL(dma_iova_destroy);
 
 void iommu_setup_dma_ops(struct device *dev, struct iommu_domain *domain)
 {
+	bool dma_iommu;
+
 	if (dev_is_pci(dev))
 		dev->iommu->pci_32bit_workaround = !iommu_dma_forcedac;
 
-	dev->dma_iommu = iommu_is_dma_domain(domain);
-	if (dev->dma_iommu && iommu_dma_init_domain(domain, dev))
+	dma_iommu = iommu_is_dma_domain(domain);
+	dev_assign_dma_iommu(dev, dma_iommu);
+	if (dma_iommu && iommu_dma_init_domain(domain, dev))
 		goto out_err;
 
 	return;
 out_err:
 	pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
 		dev_name(dev));
-	dev->dma_iommu = false;
+	dev_clear_dma_iommu(dev);
 }
 
 static bool has_msi_cookie(const struct iommu_domain *domain)
diff --git a/drivers/iommu/generic_pt/fmt/amdv1.h b/drivers/iommu/generic_pt/fmt/amdv1.h
index 8d11b08291d73..1426a6758ff25 100644
--- a/drivers/iommu/generic_pt/fmt/amdv1.h
+++ b/drivers/iommu/generic_pt/fmt/amdv1.h
@@ -404,6 +404,7 @@ amdv1pt_iommu_fmt_hw_info(struct pt_iommu_amdv1 *table,
 static const struct pt_iommu_amdv1_cfg amdv1_kunit_fmt_cfgs[] = {
 	/* Matches what io_pgtable does */
 	[0] = { .starting_level = 2 },
+	[1] = { .starting_level = 2, .common.hw_max_vasz_lg2 = 32 },
 };
 #define kunit_fmt_cfgs amdv1_kunit_fmt_cfgs
 enum { KUNIT_FMT_FEATURES = 0 };
diff --git a/drivers/iommu/generic_pt/fmt/iommu_riscv64.c b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c
index cbf60fffa9bf7..b18fc4d109f53 100644
--- a/drivers/iommu/generic_pt/fmt/iommu_riscv64.c
+++ b/drivers/iommu/generic_pt/fmt/iommu_riscv64.c
@@ -6,6 +6,8 @@
 #define PT_FMT_VARIANT 64
 #define PT_SUPPORTED_FEATURES                                  \
 	(BIT(PT_FEAT_SIGN_EXTEND) | BIT(PT_FEAT_FLUSH_RANGE) | \
-	 BIT(PT_FEAT_RISCV_SVNAPOT_64K))
+	 BIT(PT_FEAT_RISCV_SVNAPOT_64K) |                      \
+	 BIT(PT_FEAT_DETAILED_GATHER))
+#define PT_FORCE_ENABLED_FEATURES BIT(PT_FEAT_DETAILED_GATHER)
 
 #include "iommu_template.h"
diff --git a/drivers/iommu/generic_pt/fmt/riscv.h b/drivers/iommu/generic_pt/fmt/riscv.h
index a7fef6266a36a..ae9a76514416c 100644
--- a/drivers/iommu/generic_pt/fmt/riscv.h
+++ b/drivers/iommu/generic_pt/fmt/riscv.h
@@ -64,6 +64,8 @@ enum {
 	RISCVPT_PPN64 = GENMASK_ULL(53, 10),
 	RISCVPT_PPN64_64K = GENMASK_ULL(53, 14),
 	RISCVPT_PBMT = GENMASK_ULL(62, 61),
+	RISCVPT_NC = BIT_ULL(61),
+	RISCVPT_IO = BIT_ULL(62),
 	RISCVPT_N = BIT_ULL(63),
 
 	/* Svnapot encodings for ppn[0] */
@@ -201,7 +203,8 @@ static inline void riscvpt_attr_from_entry(const struct pt_state *pts,
 {
 	attrs->descriptor_bits =
 		pts->entry & (RISCVPT_R | RISCVPT_W | RISCVPT_X | RISCVPT_U |
-			      RISCVPT_G | RISCVPT_A | RISCVPT_D);
+			      RISCVPT_G | RISCVPT_A | RISCVPT_D | RISCVPT_NC |
+			      RISCVPT_IO);
 }
 #define pt_attr_from_entry riscvpt_attr_from_entry
 
@@ -237,6 +240,12 @@ static inline int riscvpt_iommu_set_prot(struct pt_common *common,
 		pte |= RISCVPT_R;
 	if (!(iommu_prot & IOMMU_NOEXEC))
 		pte |= RISCVPT_X;
+	if (common->features & BIT(PT_FEAT_RISCV_SVPBMT)) {
+		if (iommu_prot & IOMMU_MMIO)
+			pte |= RISCVPT_IO;
+		else if (!(iommu_prot & IOMMU_CACHE))
+			pte |= RISCVPT_NC;
+	}
 
 	/* Caller must specify a supported combination of flags */
 	if (unlikely((pte & (RISCVPT_X | RISCVPT_W | RISCVPT_R)) == 0))
diff --git a/drivers/iommu/generic_pt/iommu_pt.h b/drivers/iommu/generic_pt/iommu_pt.h
index dc91fb4e2f61c..c2752151c80af 100644
--- a/drivers/iommu/generic_pt/iommu_pt.h
+++ b/drivers/iommu/generic_pt/iommu_pt.h
@@ -40,15 +40,40 @@ static void flush_writes_item(const struct pt_state *pts)
 			PT_ITEM_WORD_SIZE);
 }
 
-static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather,
-			       struct pt_iommu *iommu_table, pt_vaddr_t iova,
-			       pt_vaddr_t len,
-			       struct iommu_pages_list *free_list)
+struct iommupt_pending_gather {
+	struct iommu_iotlb_gather *iotlb_gather;
+	struct iommu_pages_list free_list;
+	u8 leaf_levels_bitmap;
+	u8 table_levels_bitmap;
+};
+
+static void gather_add_table(struct iommupt_pending_gather *pending,
+			     const struct pt_state *pts,
+			     struct pt_table_p *table)
+{
+	iommu_pages_list_add(&pending->free_list, table);
+	if (pts_feature(pts, PT_FEAT_DETAILED_GATHER))
+		pending->table_levels_bitmap |= BIT(pts->level);
+}
+
+static void gather_add_leaf(struct iommupt_pending_gather *pending,
+			    const struct pt_state *pts)
+{
+	if (!pts_feature(pts, PT_FEAT_DETAILED_GATHER))
+		return;
+
+	pending->leaf_levels_bitmap |= BIT(pts->level);
+}
+
+static void gather_range_pending(struct iommupt_pending_gather *pending,
+				 struct pt_iommu *iommu_table, pt_vaddr_t iova,
+				 pt_vaddr_t len)
 {
+	struct iommu_iotlb_gather *iotlb_gather = pending->iotlb_gather;
 	struct pt_common *common = common_from_iommu(iommu_table);
 
 	if (pt_feature(common, PT_FEAT_DMA_INCOHERENT))
-		iommu_pages_stop_incoherent_list(free_list,
+		iommu_pages_stop_incoherent_list(&pending->free_list,
 						 iommu_table->iommu_device);
 
 	/*
@@ -72,7 +97,17 @@ static void gather_range_pages(struct iommu_iotlb_gather *iotlb_gather,
 		iommu_iotlb_gather_add_range(iotlb_gather, iova, len);
 	}
 
-	iommu_pages_list_splice(free_list, &iotlb_gather->freelist);
+	iommu_pages_list_splice(&pending->free_list, &iotlb_gather->freelist);
+	INIT_LIST_HEAD(&pending->free_list.pages);
+
+	if (pt_feature(common, PT_FEAT_DETAILED_GATHER)) {
+		iotlb_gather->pt.leaf_levels_bitmap |=
+			pending->leaf_levels_bitmap;
+		iotlb_gather->pt.table_levels_bitmap |=
+			pending->table_levels_bitmap;
+		pending->leaf_levels_bitmap = 0;
+		pending->table_levels_bitmap = 0;
+	}
 }
 
 #define DOMAIN_NS(op) CONCATENATE(CONCATENATE(pt_iommu_, PTPFX), op)
@@ -341,7 +376,7 @@ static int __maybe_unused NS(set_dirty)(struct pt_iommu *iommu_table,
 }
 
 struct pt_iommu_collect_args {
-	struct iommu_pages_list free_list;
+	struct iommupt_pending_gather pending;
 	/* Fail if any OAs are within the range */
 	u8 check_mapped : 1;
 };
@@ -358,7 +393,8 @@ static int __collect_tables(struct pt_range *range, void *arg,
 
 	for_each_pt_level_entry(&pts) {
 		if (pts.type == PT_ENTRY_TABLE) {
-			iommu_pages_list_add(&collect->free_list, pts.table_lower);
+			gather_add_table(&collect->pending, &pts,
+					 pts.table_lower);
 			ret = pt_descend(&pts, arg, __collect_tables);
 			if (ret)
 				return ret;
@@ -493,15 +529,18 @@ static int clear_contig(const struct pt_state *start_pts,
 	struct pt_range range = *start_pts->range;
 	struct pt_state pts =
 		pt_init(&range, start_pts->level, start_pts->table);
-	struct pt_iommu_collect_args collect = { .check_mapped = true };
+	struct pt_iommu_collect_args collect = {
+		.check_mapped = true,
+		.pending.iotlb_gather = iotlb_gather,
+		.pending.free_list = IOMMU_PAGES_LIST_INIT(
+			collect.pending.free_list),
+	};
 	int ret;
 
 	pts.index = start_pts->index;
 	pts.end_index = start_pts->index + step;
 	for (; _pt_iter_load(&pts); pt_next_entry(&pts)) {
 		if (pts.type == PT_ENTRY_TABLE) {
-			collect.free_list =
-				IOMMU_PAGES_LIST_INIT(collect.free_list);
 			ret = pt_walk_descend_all(&pts, __collect_tables,
 						  &collect);
 			if (ret)
@@ -514,12 +553,11 @@ static int clear_contig(const struct pt_state *start_pts,
 			pt_clear_entries(&pts, ilog2(1));
 			flush_writes_item(&pts);
 
-			iommu_pages_list_add(&collect.free_list,
-					     pt_table_ptr(&pts));
-			gather_range_pages(
-				iotlb_gather, iommu_table, range.va,
-				log2_to_int(pt_table_item_lg2sz(&pts)),
-				&collect.free_list);
+			gather_add_table(&collect.pending, &pts,
+					 pts.table_lower);
+			gather_range_pending(
+				&collect.pending, iommu_table, range.va,
+				log2_to_int(pt_table_item_lg2sz(&pts)));
 		} else if (pts.type != PT_ENTRY_EMPTY) {
 			return -EADDRINUSE;
 		}
@@ -968,7 +1006,7 @@ static int NS(map_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
 }
 
 struct pt_unmap_args {
-	struct iommu_pages_list free_list;
+	struct iommupt_pending_gather pending;
 	pt_vaddr_t unmapped;
 };
 
@@ -1031,8 +1069,8 @@ static __maybe_unused int __unmap_range(struct pt_range *range, void *arg,
 			 * succeed in clearing the lower table levels.
 			 */
 			if (fully_covered) {
-				iommu_pages_list_add(&unmap->free_list,
-						     pts.table_lower);
+				gather_add_table(&unmap->pending, &pts,
+						 pts.table_lower);
 				pt_clear_entries(&pts, ilog2(1));
 				if (pts.index < flush_start_index)
 					flush_start_index = pts.index;
@@ -1049,6 +1087,7 @@ static __maybe_unused int __unmap_range(struct pt_range *range, void *arg,
 			 */
 			num_contig_lg2 = pt_entry_num_contig_lg2(&pts);
 			pt_clear_entries(&pts, num_contig_lg2);
+			gather_add_leaf(&unmap->pending, &pts);
 			num_oas += log2_to_int(num_contig_lg2);
 			if (pts.index < flush_start_index)
 				flush_start_index = pts.index;
@@ -1071,8 +1110,11 @@ static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
 			      dma_addr_t len,
 			      struct iommu_iotlb_gather *iotlb_gather)
 {
-	struct pt_unmap_args unmap = { .free_list = IOMMU_PAGES_LIST_INIT(
-					       unmap.free_list) };
+	struct pt_unmap_args unmap = {
+		.pending.iotlb_gather = iotlb_gather,
+		.pending.free_list = IOMMU_PAGES_LIST_INIT(
+			unmap.pending.free_list),
+	};
 	struct pt_range range;
 	int ret;
 
@@ -1082,8 +1124,7 @@ static size_t NS(unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova,
 
 	pt_walk_range(&range, __unmap_range, &unmap);
 
-	gather_range_pages(iotlb_gather, iommu_table, iova, unmap.unmapped,
-			   &unmap.free_list);
+	gather_range_pending(&unmap.pending, iommu_table, iova, unmap.unmapped);
 
 	return unmap.unmapped;
 }
@@ -1108,8 +1149,12 @@ static void NS(get_info)(struct pt_iommu *iommu_table,
 			pgsize_bitmap |= pt_possible_sizes(&pts);
 	}
 
-	/* Hide page sizes larger than the maximum OA */
-	info->pgsize_bitmap = oalog2_mod(pgsize_bitmap, common->max_oasz_lg2);
+	/*
+	 * Hide page sizes larger than the maximum. -1 because a whole table
+	 * pgsize is not allowed
+	 */
+	info->pgsize_bitmap = log2_mod(pgsize_bitmap, common->max_vasz_lg2 - 1);
+	info->pgsize_bitmap = oalog2_mod(info->pgsize_bitmap, common->max_oasz_lg2);
 }
 
 static void NS(deinit)(struct pt_iommu *iommu_table)
@@ -1117,10 +1162,11 @@ static void NS(deinit)(struct pt_iommu *iommu_table)
 	struct pt_common *common = common_from_iommu(iommu_table);
 	struct pt_range range = pt_all_range(common);
 	struct pt_iommu_collect_args collect = {
-		.free_list = IOMMU_PAGES_LIST_INIT(collect.free_list),
+		.pending.free_list = IOMMU_PAGES_LIST_INIT(
+			collect.pending.free_list),
 	};
 
-	iommu_pages_list_add(&collect.free_list, range.top_table);
+	iommu_pages_list_add(&collect.pending.free_list, range.top_table);
 	pt_walk_range(&range, __collect_tables, &collect);
 
 	/*
@@ -1128,9 +1174,9 @@ static void NS(deinit)(struct pt_iommu *iommu_table)
 	 * and invalidated any caching referring to this memory.
 	 */
 	if (pt_feature(common, PT_FEAT_DMA_INCOHERENT))
-		iommu_pages_stop_incoherent_list(&collect.free_list,
+		iommu_pages_stop_incoherent_list(&collect.pending.free_list,
 						 iommu_table->iommu_device);
-	iommu_put_pages_list(&collect.free_list);
+	iommu_put_pages_list(&collect.pending.free_list);
 }
 
 static const struct pt_iommu_ops NS(ops) = {
@@ -1151,10 +1197,6 @@ static int pt_init_common(struct pt_common *common)
 	if (PT_WARN_ON(top_range.top_level > PT_MAX_TOP_LEVEL))
 		return -EINVAL;
 
-	if (top_range.top_level == PT_MAX_TOP_LEVEL ||
-	    common->max_vasz_lg2 == top_range.max_vasz_lg2)
-		common->features &= ~BIT(PT_FEAT_DYNAMIC_TOP);
-
 	if (top_range.max_vasz_lg2 == PT_VADDR_MAX_LG2)
 		common->features |= BIT(PT_FEAT_FULL_VA);
 
diff --git a/drivers/iommu/generic_pt/kunit_generic_pt.h b/drivers/iommu/generic_pt/kunit_generic_pt.h
index 374e475f591e1..ef2c90b6d6af3 100644
--- a/drivers/iommu/generic_pt/kunit_generic_pt.h
+++ b/drivers/iommu/generic_pt/kunit_generic_pt.h
@@ -438,6 +438,9 @@ static void test_lvl_possible_sizes(struct kunit *test, struct pt_state *pts,
 {
 	unsigned int num_items_lg2 = safe_pt_num_items_lg2(pts);
 	pt_vaddr_t pgsize_bitmap = pt_possible_sizes(pts);
+	/* Matches get_info() */
+	pt_vaddr_t limited_pgsize_bitmap =
+		log2_mod(pgsize_bitmap, pts->range->common->max_vasz_lg2 - 1);
 	unsigned int isz_lg2 = pt_table_item_lg2sz(pts);
 
 	if (!pt_can_have_leaf(pts)) {
@@ -448,7 +451,8 @@ static void test_lvl_possible_sizes(struct kunit *test, struct pt_state *pts,
 	/* No bits for sizes that would be outside this table */
 	KUNIT_ASSERT_EQ(test, log2_mod(pgsize_bitmap, isz_lg2), 0);
 	KUNIT_ASSERT_EQ(
-		test, fvalog2_div(pgsize_bitmap, num_items_lg2 + isz_lg2), 0);
+		test,
+		fvalog2_div(limited_pgsize_bitmap, num_items_lg2 + isz_lg2), 0);
 
 	/*
 	 * Non contiguous must be supported. AMDv1 has a HW bug where it does
@@ -463,8 +467,8 @@ static void test_lvl_possible_sizes(struct kunit *test, struct pt_state *pts,
 	/* A contiguous entry should not span the whole table */
 	if (num_items_lg2 + isz_lg2 != PT_VADDR_MAX_LG2)
 		KUNIT_ASSERT_FALSE(
-			test,
-			pgsize_bitmap & log2_to_int(num_items_lg2 + isz_lg2));
+			test, limited_pgsize_bitmap &
+					log2_to_int(num_items_lg2 + isz_lg2));
 }
 
 static void test_entry_possible_sizes(struct kunit *test)
diff --git a/drivers/iommu/generic_pt/kunit_iommu_pt.h b/drivers/iommu/generic_pt/kunit_iommu_pt.h
index e8a63c8ea850e..ece1c9b8c55d3 100644
--- a/drivers/iommu/generic_pt/kunit_iommu_pt.h
+++ b/drivers/iommu/generic_pt/kunit_iommu_pt.h
@@ -112,8 +112,9 @@ static void test_increase_level(struct kunit *test)
 	if (IS_32BIT)
 		kunit_skip(test, "Unable to test on 32bit");
 
-	KUNIT_ASSERT_GT(test, common->max_vasz_lg2,
-			pt_top_range(common).max_vasz_lg2);
+	if (common->max_vasz_lg2 <= pt_top_range(common).max_vasz_lg2)
+		kunit_skip(test,
+			   "max_vasz_lg2 fits in starting level, no growth possible");
 
 	/* Add every possible level to the max */
 	while (common->max_vasz_lg2 != pt_top_range(common).max_vasz_lg2) {
diff --git a/drivers/iommu/intel/cache.c b/drivers/iommu/intel/cache.c
index be8410f0e8414..fdc88817709f2 100644
--- a/drivers/iommu/intel/cache.c
+++ b/drivers/iommu/intel/cache.c
@@ -254,37 +254,29 @@ void cache_tag_unassign_domain(struct dmar_domain *domain,
 }
 
 static unsigned long calculate_psi_aligned_address(unsigned long start,
-						   unsigned long end,
-						   unsigned long *_mask)
+						   unsigned long last,
+						   unsigned long *size_order)
 {
-	unsigned long pages = aligned_nrpages(start, end - start + 1);
-	unsigned long aligned_pages = __roundup_pow_of_two(pages);
-	unsigned long bitmask = aligned_pages - 1;
-	unsigned long mask = ilog2(aligned_pages);
-	unsigned long pfn = IOVA_PFN(start);
-
-	/*
-	 * PSI masks the low order bits of the base address. If the
-	 * address isn't aligned to the mask, then compute a mask value
-	 * needed to ensure the target range is flushed.
-	 */
-	if (unlikely(bitmask & pfn)) {
-		unsigned long end_pfn = pfn + pages - 1, shared_bits;
-
+	unsigned int sz_lg2;
+
+	/* Compute a sz_lg2 that spans start and last */
+	start &= GENMASK(BITS_PER_LONG - 1, VTD_PAGE_SHIFT);
+	sz_lg2 = fls_long(start ^ last);
+	if (sz_lg2 <= 12) {
+		*size_order = 0;
+		return start;
+	}
+	if (unlikely(sz_lg2 >= BITS_PER_LONG)) {
 		/*
-		 * Since end_pfn <= pfn + bitmask, the only way bits
-		 * higher than bitmask can differ in pfn and end_pfn is
-		 * by carrying. This means after masking out bitmask,
-		 * high bits starting with the first set bit in
-		 * shared_bits are all equal in both pfn and end_pfn.
+		 * MAX_AGAW_PFN_WIDTH triggers full invalidation in all
+		 * downstream users.
 		 */
-		shared_bits = ~(pfn ^ end_pfn) & ~bitmask;
-		mask = shared_bits ? __ffs(shared_bits) : MAX_AGAW_PFN_WIDTH;
+		*size_order = MAX_AGAW_PFN_WIDTH;
+		return 0;
 	}
 
-	*_mask = mask;
-
-	return ALIGN_DOWN(start, VTD_PAGE_SIZE << mask);
+	*size_order = sz_lg2 - VTD_PAGE_SHIFT;
+	return start & GENMASK(BITS_PER_LONG - 1, sz_lg2);
 }
 
 static void qi_batch_flush_descs(struct intel_iommu *iommu, struct qi_batch *batch)
@@ -441,12 +433,7 @@ void cache_tag_flush_range(struct dmar_domain *domain, unsigned long start,
 	struct cache_tag *tag;
 	unsigned long flags;
 
-	if (start == 0 && end == ULONG_MAX) {
-		addr = 0;
-		mask = MAX_AGAW_PFN_WIDTH;
-	} else {
-		addr = calculate_psi_aligned_address(start, end, &mask);
-	}
+	addr = calculate_psi_aligned_address(start, end, &mask);
 
 	spin_lock_irqsave(&domain->cache_lock, flags);
 	list_for_each_entry(tag, &domain->cache_tags, node) {
diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c
index 0208e5897c299..476c0e25631af 100644
--- a/drivers/iommu/io-pgtable-arm.c
+++ b/drivers/iommu/io-pgtable-arm.c
@@ -143,7 +143,7 @@
 #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL
 
 /* IOPTE accessors */
-#define iopte_deref(pte,d) __va(iopte_to_paddr(pte, d))
+#define iopte_deref(pte, d) phys_to_virt(iopte_to_paddr(pte, d))
 
 #define iopte_type(pte)					\
 	(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)
@@ -248,26 +248,15 @@ static dma_addr_t __arm_lpae_dma_addr(void *pages)
 	return (dma_addr_t)virt_to_phys(pages);
 }
 
-static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
-				    struct io_pgtable_cfg *cfg,
-				    void *cookie)
+static void *__arm_lpae_cfg_alloc(size_t size, gfp_t gfp,
+				  struct io_pgtable_cfg *cfg,
+				  void *cookie)
 {
 	struct device *dev = cfg->iommu_dev;
-	size_t alloc_size;
 	dma_addr_t dma;
 	void *pages;
 
-	/*
-	 * For very small starting-level translation tables the HW requires a
-	 * minimum alignment of at least 64 to cover all cases.
-	 */
-	alloc_size = max(size, 64);
-	if (cfg->alloc)
-		pages = cfg->alloc(cookie, alloc_size, gfp);
-	else
-		pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp,
-						  alloc_size);
-
+	pages = cfg->alloc(cookie, size, gfp);
 	if (!pages)
 		return NULL;
 
@@ -291,24 +280,69 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
 	dma_unmap_single(dev, dma, size, DMA_TO_DEVICE);
 
 out_free:
-	if (cfg->free)
-		cfg->free(cookie, pages, size);
-	else
-		iommu_free_pages(pages);
-
+	cfg->free(cookie, pages, size);
 	return NULL;
 }
 
-static void __arm_lpae_free_pages(void *pages, size_t size,
-				  struct io_pgtable_cfg *cfg,
-				  void *cookie)
+static void __arm_lpae_cfg_free(void *pages, size_t size,
+				struct io_pgtable_cfg *cfg,
+				void *cookie)
 {
 	if (!cfg->coherent_walk)
 		dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages),
 				 size, DMA_TO_DEVICE);
 
-	if (cfg->free)
-		cfg->free(cookie, pages, size);
+	cfg->free(cookie, pages, size);
+}
+
+static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp,
+				    struct io_pgtable_cfg *cfg,
+				    void *cookie)
+{
+	struct device *dev = cfg->iommu_dev;
+	void *pages;
+
+	/*
+	 * For very small starting-level translation tables the HW requires a
+	 * minimum alignment of at least 64 to cover all cases.
+	 */
+	size = max(size, 64);
+
+	if (cfg->alloc)
+		return __arm_lpae_cfg_alloc(size, gfp, cfg, cookie);
+
+	pages = iommu_alloc_pages_node_sz(dev_to_node(dev), gfp, size);
+	if (!pages)
+		return NULL;
+
+	if (!cfg->coherent_walk) {
+		int ret = iommu_pages_start_incoherent(pages, dev);
+
+		if (ret) {
+			if (ret == -EOPNOTSUPP)
+				dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n");
+			iommu_free_pages(pages);
+			return NULL;
+		}
+	}
+
+	return pages;
+}
+
+static void __arm_lpae_free_pages(void *pages, size_t size,
+				  struct io_pgtable_cfg *cfg,
+				  void *cookie)
+{
+	/* See __arm_lpae_alloc_pages(). */
+	size = max(size, 64);
+
+	if (cfg->free) {
+		__arm_lpae_cfg_free(pages, size, cfg, cookie);
+		return;
+	}
+
+	if (!cfg->coherent_walk)
+		iommu_pages_free_incoherent(pages, cfg->iommu_dev);
 	else
 		iommu_free_pages(pages);
 }
@@ -395,7 +429,7 @@ static arm_lpae_iopte arm_lpae_install_table(arm_lpae_iopte *table,
 	arm_lpae_iopte old, new;
 	struct io_pgtable_cfg *cfg = &data->iop.cfg;
 
-	new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
+	new = paddr_to_iopte(virt_to_phys(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
 	if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)
 		new |= ARM_LPAE_PTE_NSTABLE;
 
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index d1a9e713d3a05..e8f13dcebbde5 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -611,9 +611,8 @@ static void iommu_deinit_device(struct device *dev)
 	dev->iommu_group = NULL;
 	module_put(ops->owner);
 	dev_iommu_free(dev);
-#ifdef CONFIG_IOMMU_DMA
-	dev->dma_iommu = false;
-#endif
+	if (IS_ENABLED(CONFIG_IOMMU_DMA))
+		dev_clear_dma_iommu(dev);
 }
 
 static struct iommu_domain *pasid_array_entry_to_domain(void *entry)
diff --git a/drivers/iommu/iommufd/eventq.c b/drivers/iommu/iommufd/eventq.c
index 710eef0b60045..78689fb52d24c 100644
--- a/drivers/iommu/iommufd/eventq.c
+++ b/drivers/iommu/iommufd/eventq.c
@@ -321,7 +321,7 @@ static ssize_t iommufd_veventq_fops_read(struct file *filep, char __user *buf,
 
 		/* If being a normal vEVENT, validate against the full size */
 		if (!vevent_for_lost_events_header(cur) &&
-		    sizeof(hdr) + cur->data_len > count - done) {
+		    sizeof(*hdr) + cur->data_len > count - done) {
 			iommufd_veventq_deliver_restore(veventq, cur);
 			break;
 		}
diff --git a/drivers/iommu/riscv/iommu-bits.h b/drivers/iommu/riscv/iommu-bits.h
index 29a0040b1c32e..f2ef9bd3cde96 100644
--- a/drivers/iommu/riscv/iommu-bits.h
+++ b/drivers/iommu/riscv/iommu-bits.h
@@ -63,6 +63,8 @@
 #define RISCV_IOMMU_CAPABILITIES_PD8		BIT_ULL(38)
 #define RISCV_IOMMU_CAPABILITIES_PD17		BIT_ULL(39)
 #define RISCV_IOMMU_CAPABILITIES_PD20		BIT_ULL(40)
+#define RISCV_IOMMU_CAPABILITIES_NL		BIT_ULL(42)
+#define RISCV_IOMMU_CAPABILITIES_S		BIT_ULL(43)
 
 /**
  * enum riscv_iommu_igs_settings - Interrupt Generation Support Settings
@@ -460,31 +462,33 @@ struct riscv_iommu_command {
 };
 
 /* Fields on dword0, common for all commands */
-#define RISCV_IOMMU_CMD_OPCODE	GENMASK_ULL(6, 0)
-#define	RISCV_IOMMU_CMD_FUNC	GENMASK_ULL(9, 7)
+#define RISCV_IOMMU_CMD0_OPCODE	GENMASK_ULL(6, 0)
+#define RISCV_IOMMU_CMD0_FUNC	GENMASK_ULL(9, 7)
 
 /* 3.1.1 IOMMU Page-table cache invalidation */
 /* Fields on dword0 */
 #define RISCV_IOMMU_CMD_IOTINVAL_OPCODE		1
 #define RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA	0
 #define RISCV_IOMMU_CMD_IOTINVAL_FUNC_GVMA	1
-#define RISCV_IOMMU_CMD_IOTINVAL_AV		BIT_ULL(10)
-#define RISCV_IOMMU_CMD_IOTINVAL_PSCID		GENMASK_ULL(31, 12)
-#define RISCV_IOMMU_CMD_IOTINVAL_PSCV		BIT_ULL(32)
-#define RISCV_IOMMU_CMD_IOTINVAL_GV		BIT_ULL(33)
-#define RISCV_IOMMU_CMD_IOTINVAL_GSCID		GENMASK_ULL(59, 44)
+#define RISCV_IOMMU_CMD0_IOTINVAL_AV		BIT_ULL(10)
+#define RISCV_IOMMU_CMD0_IOTINVAL_PSCID		GENMASK_ULL(31, 12)
+#define RISCV_IOMMU_CMD0_IOTINVAL_PSCV		BIT_ULL(32)
+#define RISCV_IOMMU_CMD0_IOTINVAL_GV		BIT_ULL(33)
+#define RISCV_IOMMU_CMD0_IOTINVAL_GSCID		GENMASK_ULL(59, 44)
+#define RISCV_IOMMU_CMD0_IOTINVAL_NL		BIT_ULL(34)
+#define RISCV_IOMMU_CMD1_IOTINVAL_S		BIT_ULL(9)
 /* dword1[61:10] is the 4K-aligned page address */
-#define RISCV_IOMMU_CMD_IOTINVAL_ADDR		GENMASK_ULL(61, 10)
+#define RISCV_IOMMU_CMD1_IOTINVAL_ADDR		GENMASK_ULL(61, 10)
 
 /* 3.1.2 IOMMU Command Queue Fences */
 /* Fields on dword0 */
 #define RISCV_IOMMU_CMD_IOFENCE_OPCODE		2
 #define RISCV_IOMMU_CMD_IOFENCE_FUNC_C		0
-#define RISCV_IOMMU_CMD_IOFENCE_AV		BIT_ULL(10)
-#define RISCV_IOMMU_CMD_IOFENCE_WSI		BIT_ULL(11)
-#define RISCV_IOMMU_CMD_IOFENCE_PR		BIT_ULL(12)
-#define RISCV_IOMMU_CMD_IOFENCE_PW		BIT_ULL(13)
-#define RISCV_IOMMU_CMD_IOFENCE_DATA		GENMASK_ULL(63, 32)
+#define RISCV_IOMMU_CMD0_IOFENCE_AV		BIT_ULL(10)
+#define RISCV_IOMMU_CMD0_IOFENCE_WSI		BIT_ULL(11)
+#define RISCV_IOMMU_CMD0_IOFENCE_PR		BIT_ULL(12)
+#define RISCV_IOMMU_CMD0_IOFENCE_PW		BIT_ULL(13)
+#define RISCV_IOMMU_CMD0_IOFENCE_DATA		GENMASK_ULL(63, 32)
 /* dword1 is the address, word-size aligned and shifted to the right by two bits. */
 
 /* 3.1.3 IOMMU Directory cache invalidation */
@@ -492,9 +496,9 @@ struct riscv_iommu_command {
 #define RISCV_IOMMU_CMD_IODIR_OPCODE		3
 #define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT	0
 #define RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT	1
-#define RISCV_IOMMU_CMD_IODIR_PID		GENMASK_ULL(31, 12)
-#define RISCV_IOMMU_CMD_IODIR_DV		BIT_ULL(33)
-#define RISCV_IOMMU_CMD_IODIR_DID		GENMASK_ULL(63, 40)
+#define RISCV_IOMMU_CMD0_IODIR_PID		GENMASK_ULL(31, 12)
+#define RISCV_IOMMU_CMD0_IODIR_DV		BIT_ULL(33)
+#define RISCV_IOMMU_CMD0_IODIR_DID		GENMASK_ULL(63, 40)
 /* dword1 is reserved for standard use */
 
 /* 3.1.4 IOMMU PCIe ATS */
@@ -502,25 +506,25 @@ struct riscv_iommu_command {
 #define RISCV_IOMMU_CMD_ATS_OPCODE		4
 #define RISCV_IOMMU_CMD_ATS_FUNC_INVAL		0
 #define RISCV_IOMMU_CMD_ATS_FUNC_PRGR		1
-#define RISCV_IOMMU_CMD_ATS_PID			GENMASK_ULL(31, 12)
-#define RISCV_IOMMU_CMD_ATS_PV			BIT_ULL(32)
-#define RISCV_IOMMU_CMD_ATS_DSV			BIT_ULL(33)
-#define RISCV_IOMMU_CMD_ATS_RID			GENMASK_ULL(55, 40)
-#define RISCV_IOMMU_CMD_ATS_DSEG		GENMASK_ULL(63, 56)
+#define RISCV_IOMMU_CMD0_ATS_PID		GENMASK_ULL(31, 12)
+#define RISCV_IOMMU_CMD0_ATS_PV			BIT_ULL(32)
+#define RISCV_IOMMU_CMD0_ATS_DSV		BIT_ULL(33)
+#define RISCV_IOMMU_CMD0_ATS_RID		GENMASK_ULL(55, 40)
+#define RISCV_IOMMU_CMD0_ATS_DSEG		GENMASK_ULL(63, 56)
 /* dword1 is the ATS payload, two different payload types for INVAL and PRGR */
 
 /* ATS.INVAL payload*/
-#define RISCV_IOMMU_CMD_ATS_INVAL_G		BIT_ULL(0)
+#define RISCV_IOMMU_CMD1_ATS_INVAL_G		BIT_ULL(0)
 /* Bits 1 - 10 are zeroed */
-#define RISCV_IOMMU_CMD_ATS_INVAL_S		BIT_ULL(11)
-#define RISCV_IOMMU_CMD_ATS_INVAL_UADDR		GENMASK_ULL(63, 12)
+#define RISCV_IOMMU_CMD1_ATS_INVAL_S		BIT_ULL(11)
+#define RISCV_IOMMU_CMD1_ATS_INVAL_UADDR	GENMASK_ULL(63, 12)
 
 /* ATS.PRGR payload */
 /* Bits 0 - 31 are zeroed */
-#define RISCV_IOMMU_CMD_ATS_PRGR_PRG_INDEX	GENMASK_ULL(40, 32)
+#define RISCV_IOMMU_CMD1_ATS_PRGR_PRG_INDEX	GENMASK_ULL(40, 32)
 /* Bits 41 - 43 are zeroed */
-#define RISCV_IOMMU_CMD_ATS_PRGR_RESP_CODE	GENMASK_ULL(47, 44)
-#define RISCV_IOMMU_CMD_ATS_PRGR_DST_ID		GENMASK_ULL(63, 48)
+#define RISCV_IOMMU_CMD1_ATS_PRGR_RESP_CODE	GENMASK_ULL(47, 44)
+#define RISCV_IOMMU_CMD1_ATS_PRGR_DST_ID	GENMASK_ULL(63, 48)
 
 /**
  * struct riscv_iommu_fq_record - Fault/Event Queue Record
@@ -711,8 +715,8 @@ struct riscv_iommu_msipte {
 
 static inline void riscv_iommu_cmd_inval_vma(struct riscv_iommu_command *cmd)
 {
-	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOTINVAL_OPCODE) |
-		      FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA);
+	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IOTINVAL_OPCODE) |
+		      FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IOTINVAL_FUNC_VMA);
 	cmd->dword1 = 0;
 }
 
@@ -720,67 +724,88 @@ static inline void riscv_iommu_cmd_inval_set_addr(struct riscv_iommu_command *cm
 						  u64 addr)
 {
 	cmd->dword1 =
-		FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_ADDR, PHYS_PFN(addr));
-	cmd->dword0 |= RISCV_IOMMU_CMD_IOTINVAL_AV;
+		FIELD_PREP(RISCV_IOMMU_CMD1_IOTINVAL_ADDR, PHYS_PFN(addr));
+	cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_AV;
+}
+
+static inline void riscv_iommu_cmd_inval_set_nl(struct riscv_iommu_command *cmd)
+{
+	cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_NL;
+}
+
+/*
+ * Set NAPOT-encoded address for range invalidation (S=1).
+ * sz_lg2: log2 of total range in bytes, must be >= 13 (8KiB, 2 pages).
+ * addr must be naturally aligned to 2^sz_lg2.
+ */
+static inline void riscv_iommu_cmd_inval_set_napot(
+	struct riscv_iommu_command *cmd, u64 addr, unsigned int sz_lg2)
+{
+	u64 pfn = addr >> 12;
+
+	pfn |= BIT_U64(sz_lg2 - 13) - 1;
+	cmd->dword1 = FIELD_PREP(RISCV_IOMMU_CMD1_IOTINVAL_ADDR, pfn) |
+		      RISCV_IOMMU_CMD1_IOTINVAL_S;
+	cmd->dword0 |= RISCV_IOMMU_CMD0_IOTINVAL_AV;
 }
 
 static inline void riscv_iommu_cmd_inval_set_pscid(struct riscv_iommu_command *cmd,
 						   int pscid)
 {
-	cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_PSCID, pscid) |
-		       RISCV_IOMMU_CMD_IOTINVAL_PSCV;
+	cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IOTINVAL_PSCID, pscid) |
+		       RISCV_IOMMU_CMD0_IOTINVAL_PSCV;
 }
 
 static inline void riscv_iommu_cmd_inval_set_gscid(struct riscv_iommu_command *cmd,
 						   int gscid)
 {
-	cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IOTINVAL_GSCID, gscid) |
-		       RISCV_IOMMU_CMD_IOTINVAL_GV;
+	cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IOTINVAL_GSCID, gscid) |
+		       RISCV_IOMMU_CMD0_IOTINVAL_GV;
 }
 
 static inline void riscv_iommu_cmd_iofence(struct riscv_iommu_command *cmd)
 {
-	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
-		      FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
-		      RISCV_IOMMU_CMD_IOFENCE_PR | RISCV_IOMMU_CMD_IOFENCE_PW;
+	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
+		      FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
+		      RISCV_IOMMU_CMD0_IOFENCE_PR | RISCV_IOMMU_CMD0_IOFENCE_PW;
 	cmd->dword1 = 0;
 }
 
 static inline void riscv_iommu_cmd_iofence_set_av(struct riscv_iommu_command *cmd,
 						  u64 addr, u32 data)
 {
-	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
-		      FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
-		      FIELD_PREP(RISCV_IOMMU_CMD_IOFENCE_DATA, data) |
-		      RISCV_IOMMU_CMD_IOFENCE_AV;
+	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IOFENCE_OPCODE) |
+		      FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IOFENCE_FUNC_C) |
+		      FIELD_PREP(RISCV_IOMMU_CMD0_IOFENCE_DATA, data) |
+		      RISCV_IOMMU_CMD0_IOFENCE_AV;
 	cmd->dword1 = addr >> 2;
 }
 
 static inline void riscv_iommu_cmd_iodir_inval_ddt(struct riscv_iommu_command *cmd)
 {
-	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
-		      FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT);
+	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
+		      FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_DDT);
 	cmd->dword1 = 0;
 }
 
 static inline void riscv_iommu_cmd_iodir_inval_pdt(struct riscv_iommu_command *cmd)
 {
-	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
-		      FIELD_PREP(RISCV_IOMMU_CMD_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT);
+	cmd->dword0 = FIELD_PREP(RISCV_IOMMU_CMD0_OPCODE, RISCV_IOMMU_CMD_IODIR_OPCODE) |
+		      FIELD_PREP(RISCV_IOMMU_CMD0_FUNC, RISCV_IOMMU_CMD_IODIR_FUNC_INVAL_PDT);
 	cmd->dword1 = 0;
 }
 
 static inline void riscv_iommu_cmd_iodir_set_did(struct riscv_iommu_command *cmd,
 						 unsigned int devid)
 {
-	cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_DID, devid) |
-		       RISCV_IOMMU_CMD_IODIR_DV;
+	cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IODIR_DID, devid) |
+		       RISCV_IOMMU_CMD0_IODIR_DV;
 }
 
 static inline void riscv_iommu_cmd_iodir_set_pid(struct riscv_iommu_command *cmd,
 						 unsigned int pasid)
 {
-	cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD_IODIR_PID, pasid);
+	cmd->dword0 |= FIELD_PREP(RISCV_IOMMU_CMD0_IODIR_PID, pasid);
 }
 
 #endif /* _RISCV_IOMMU_BITS_H_ */
diff --git a/drivers/iommu/riscv/iommu.c b/drivers/iommu/riscv/iommu.c
index a31f50bbad353..cec3ddd7ab103 100644
--- a/drivers/iommu/riscv/iommu.c
+++ b/drivers/iommu/riscv/iommu.c
@@ -920,20 +920,120 @@ static void riscv_iommu_bond_unlink(struct riscv_iommu_domain *domain,
 	}
 }
 
-/*
- * Send IOTLB.INVAL for whole address space for ranges larger than 2MB.
- * This limit will be replaced with range invalidations, if supported by
- * the hardware, when RISC-V IOMMU architecture specification update for
- * range invalidations update will be available.
- */
-#define RISCV_IOMMU_IOTLB_INVAL_LIMIT	(2 << 20)
+struct riscv_iommu_tlbi {
+	u64 start;
+	u64 last;
+	bool non_leaf;
+	struct {
+		bool use_global;
+		u8 stride_lg2;
+		unsigned int num;
+	} single;
+	struct {
+		u8 sz_lg2;
+		u64 addr;
+	} range;
+};
+
+static void riscv_iommu_tlbi_calc(struct riscv_iommu_tlbi *tlbi,
+				  struct iommu_iotlb_gather *gather)
+{
+	u8 combined = gather->pt.leaf_levels_bitmap |
+		      gather->pt.table_levels_bitmap;
+	u64 num;
+
+	tlbi->non_leaf = gather->pt.table_levels_bitmap != 0;
+	tlbi->start = gather->start;
+	tlbi->last = gather->end;
+
+	/* No level information available */
+	if (!combined) {
+		tlbi->single.use_global = true;
+		tlbi->range.sz_lg2 = 0;
+		return;
+	}
+
+	/*
+	 * Calculate the smallest NAPOT range containing [start, last].
+	 * NAPOT encoding requires a power-of-two sized, naturally aligned
+	 * range. Over-invalidation is always safe.
+	 */
+	tlbi->range.sz_lg2 = fls64(tlbi->start ^ tlbi->last);
+	if (unlikely(tlbi->range.sz_lg2 >= 64)) {
+		tlbi->single.use_global = true;
+		tlbi->range.sz_lg2 = 0;
+		return;
+	}
+	tlbi->range.addr = tlbi->start & ~(BIT_U64(tlbi->range.sz_lg2) - 1);
+
+	/*
+	 * Calculate stride from the lowest changed level. RISC-V uses 4KiB
+	 * granule with 9 bits per level.
+	 */
+	tlbi->single.stride_lg2 = 9 * __ffs(combined) + 12;
+	num = (tlbi->last - tlbi->start + 1) >> tlbi->single.stride_lg2;
+	if (!num || num > 512) {
+		tlbi->single.use_global = true;
+	} else {
+		tlbi->single.num = num;
+		tlbi->single.use_global = false;
+	}
+}
+
+static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu,
+					  int pscid,
+					  struct riscv_iommu_tlbi *tlbi)
+{
+	bool use_nl = tlbi->non_leaf &&
+		      (iommu->caps & RISCV_IOMMU_CAPABILITIES_NL);
+	struct riscv_iommu_command cmd;
+	unsigned int i;
+
+	riscv_iommu_cmd_inval_vma(&cmd);
+	riscv_iommu_cmd_inval_set_pscid(&cmd, pscid);
+
+	/*
+	 * If non-leaf entries were changed and the IOMMU doesn't
+	 * support NL, we must fall back to global invalidation (AV=0).
+	 */
+	if (tlbi->non_leaf && !use_nl)
+		goto global;
+
+	if (iommu->caps & RISCV_IOMMU_CAPABILITIES_S &&
+	    tlbi->range.sz_lg2 >= 13) {
+		riscv_iommu_cmd_inval_set_napot(&cmd, tlbi->range.addr,
+						tlbi->range.sz_lg2);
+		if (use_nl)
+			riscv_iommu_cmd_inval_set_nl(&cmd);
+		riscv_iommu_cmd_send(iommu, &cmd);
+	} else {
+		unsigned long iova;
+
+		if (tlbi->single.use_global)
+			goto global;
+
+		iova = tlbi->start;
+		for (i = 0; i < tlbi->single.num; i++) {
+			riscv_iommu_cmd_inval_set_addr(&cmd, iova);
+			if (use_nl)
+				riscv_iommu_cmd_inval_set_nl(&cmd);
+			riscv_iommu_cmd_send(iommu, &cmd);
+			iova += 1ULL << tlbi->single.stride_lg2;
+		}
+	}
+	return;
+global:
+	riscv_iommu_cmd_send(iommu, &cmd);
+}
 
 static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain,
-				    unsigned long start, unsigned long end)
+				    struct iommu_iotlb_gather *gather)
 {
-	struct riscv_iommu_bond *bond;
 	struct riscv_iommu_device *iommu, *prev;
-	struct riscv_iommu_command cmd;
+	struct riscv_iommu_bond *bond;
+	struct riscv_iommu_tlbi tlbi;
+
+	riscv_iommu_tlbi_calc(&tlbi, gather);
 
 	/*
 	 * For each IOMMU linked with this protection domain (via bonds->dev),
@@ -974,19 +1074,7 @@ static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain,
 		if (iommu == prev)
 			continue;
 
-		riscv_iommu_cmd_inval_vma(&cmd);
-		riscv_iommu_cmd_inval_set_pscid(&cmd, domain->pscid);
-		if (end - start < RISCV_IOMMU_IOTLB_INVAL_LIMIT - 1) {
-			unsigned long iova = start;
-
-			do {
-				riscv_iommu_cmd_inval_set_addr(&cmd, iova);
-				riscv_iommu_cmd_send(iommu, &cmd);
-			} while (!check_add_overflow(iova, PAGE_SIZE, &iova) &&
-				 iova < end);
-		} else {
-			riscv_iommu_cmd_send(iommu, &cmd);
-		}
+		riscv_iommu_iotlb_inval_iommu(iommu, domain->pscid, &tlbi);
 		prev = iommu;
 	}
 
@@ -1145,8 +1233,14 @@ static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,
 static void riscv_iommu_iotlb_flush_all(struct iommu_domain *iommu_domain)
 {
 	struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
+	struct iommu_iotlb_gather gather = {
+		.start = 0,
+		.end = ULONG_MAX,
+		.pt.leaf_levels_bitmap = 0xFF,
+		.pt.table_levels_bitmap = 0xFE,
+	};
 
-	riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX);
+	riscv_iommu_iotlb_inval(domain, &gather);
 }
 
 static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain,
@@ -1154,19 +1248,8 @@ static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain,
 {
 	struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
 
-	if (iommu_pages_list_empty(&gather->freelist)) {
-		riscv_iommu_iotlb_inval(domain, gather->start, gather->end);
-	} else {
-		/*
-		 * In 1.0 spec version, the smallest scope we can use to
-		 * invalidate all levels of page table (i.e. leaf and non-leaf)
-		 * is an invalidate-all-PSCID IOTINVAL.VMA with AV=0.
-		 * This will be updated with hardware support for
-		 * capability.NL (non-leaf) IOTINVAL command.
-		 */
-		riscv_iommu_iotlb_inval(domain, 0, ULONG_MAX);
-		iommu_put_pages_list(&gather->freelist);
-	}
+	riscv_iommu_iotlb_inval(domain, gather);
+	iommu_put_pages_list(&gather->freelist);
 }
 
 static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain)
@@ -1267,7 +1350,10 @@ static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev)
 	 */
 	cfg.common.features = BIT(PT_FEAT_SIGN_EXTEND) |
 			      BIT(PT_FEAT_FLUSH_RANGE) |
-			      BIT(PT_FEAT_RISCV_SVNAPOT_64K);
+			      BIT(PT_FEAT_RISCV_SVNAPOT_64K) |
+			      BIT(PT_FEAT_DETAILED_GATHER);
+	if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SVPBMT)
+		cfg.common.features |= BIT(PT_FEAT_RISCV_SVPBMT);
 	domain->riscvpt.iommu.nid = dev_to_node(iommu->dev);
 	domain->domain.ops = &riscv_iommu_paging_domain_ops;
 
diff --git a/drivers/iommu/vsi-iommu.c b/drivers/iommu/vsi-iommu.c
new file mode 100644
index 0000000000000..5d0721bd2c7af
--- /dev/null
+++ b/drivers/iommu/vsi-iommu.c
@@ -0,0 +1,796 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (C) 2025 Collabora Ltd.
+ *
+ * IOMMU API for Verisilicon
+ *
+ * Module Authors:	Yandong Lin <yandong.lin@rock-chips.com>
+ *			Simon Xue <xxm@rock-chips.com>
+ *			Benjamin Gaignard <benjamin.gaignard@collabora.com>
+ *
+ * This hardware block is using a 2 pages tables allocation structure.
+ * That make very similar to Rockhip iommu hardware blocks but it has
+ * it own driver because the registers offset and configuration bits
+ * are completely different. An additional reason is that this hardware
+ * has been developed by Verisilicon to be used by their hardware video
+ * decoders and not for a general purpose like Rockchip iommus.
+ */
+
+#include <linux/clk.h>
+#include <linux/compiler.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/errno.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iommu.h>
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_iommu.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "iommu-pages.h"
+
+struct vsi_iommu {
+	struct device *dev;
+	void __iomem *regs;
+	struct clk_bulk_data *clocks;
+	int num_clocks;
+	struct iommu_device iommu;
+	struct list_head node; /* entry in vsi_iommu_domain.iommus */
+	struct iommu_domain *domain; /* domain to which iommu is attached */
+	spinlock_t lock; /* lock to protect vsi_iommu fields */
+	int irq;
+	bool enable;
+};
+
+struct vsi_iommu_domain {
+	struct list_head iommus;
+	struct device *dev;
+	u32 *dt;
+	dma_addr_t dt_dma;
+	struct iommu_domain domain;
+	u64 *pta;
+	dma_addr_t pta_dma;
+	spinlock_t lock; /* lock to protect vsi_iommu_domain fields */
+};
+
+static struct iommu_domain vsi_identity_domain;
+
+#define NUM_DT_ENTRIES	1024
+#define NUM_PT_ENTRIES	1024
+
+#define SPAGE_SIZE	BIT(12)
+
+/* vsi iommu regs address */
+#define VSI_MMU_CONFIG1_BASE			0x1ac
+#define VSI_MMU_AHB_EXCEPTION_BASE		0x380
+#define VSI_MMU_AHB_CONTROL_BASE		0x388
+#define VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE	0x38C
+
+/* MMU register offsets */
+#define VSI_MMU_FLUSH_BASE		0x184
+#define VSI_MMU_BIT_FLUSH		BIT(4)
+
+#define VSI_MMU_PAGE_FAULT_ADDR		0x380
+#define VSI_MMU_STATUS_BASE		0x384	/* IRQ status */
+
+#define VSI_MMU_BIT_ENABLE		BIT(0)
+
+#define VSI_MMU_OUT_OF_BOUND		BIT(28)
+/* Irq mask */
+#define VSI_MMU_IRQ_MASK		0x7
+
+#define VSI_DTE_PT_ADDRESS_MASK		0xffffffc0
+#define VSI_DTE_PT_VALID		BIT(0)
+
+#define VSI_PAGE_DESC_LO_MASK		0xfffff000
+#define VSI_PAGE_DESC_HI_MASK		GENMASK_ULL(39, 32)
+#define VSI_PAGE_DESC_HI_SHIFT		(32 - 4)
+
+static inline phys_addr_t vsi_dte_pt_address(u32 dte)
+{
+	return (phys_addr_t)dte & VSI_DTE_PT_ADDRESS_MASK;
+}
+
+static inline u32 vsi_mk_dte(u32 dte)
+{
+	return (phys_addr_t)dte | VSI_DTE_PT_VALID;
+}
+
+#define VSI_PTE_PAGE_WRITABLE		BIT(2)
+#define VSI_PTE_PAGE_VALID		BIT(0)
+
+static inline phys_addr_t vsi_pte_page_address(u64 pte)
+{
+	return ((pte << VSI_PAGE_DESC_HI_SHIFT) & VSI_PAGE_DESC_HI_MASK) |
+	       (pte & VSI_PAGE_DESC_LO_MASK);
+}
+
+static u32 vsi_mk_pte(phys_addr_t page, int prot)
+{
+	u32 flags = 0;
+
+	flags |= (prot & IOMMU_WRITE) ? VSI_PTE_PAGE_WRITABLE : 0;
+
+	page = (page & VSI_PAGE_DESC_LO_MASK) |
+	       ((page & VSI_PAGE_DESC_HI_MASK) >> VSI_PAGE_DESC_HI_SHIFT);
+
+	return page | flags | VSI_PTE_PAGE_VALID;
+}
+
+#define VSI_DTE_PT_VALID	BIT(0)
+
+static inline bool vsi_dte_is_pt_valid(u32 dte)
+{
+	return dte & VSI_DTE_PT_VALID;
+}
+
+static inline bool vsi_pte_is_page_valid(u32 pte)
+{
+	return pte & VSI_PTE_PAGE_VALID;
+}
+
+static u32 vsi_mk_pte_invalid(u32 pte)
+{
+	return pte & ~VSI_PTE_PAGE_VALID;
+}
+
+#define VSI_MASTER_TLB_MASK	GENMASK_ULL(31, 10)
+/* mode 0 : 4k */
+#define VSI_PTA_4K_MODE	0
+
+static u64 vsi_mk_pta(dma_addr_t dt_dma)
+{
+	u64 val = (dt_dma & VSI_MASTER_TLB_MASK) | VSI_PTA_4K_MODE;
+
+	return val;
+}
+
+static struct vsi_iommu_domain *to_vsi_domain(struct iommu_domain *dom)
+{
+	return container_of(dom, struct vsi_iommu_domain, domain);
+}
+
+static inline void vsi_table_flush(struct vsi_iommu_domain *vsi_domain, dma_addr_t dma,
+				   unsigned int count)
+{
+	size_t size = count * sizeof(u32); /* count of u32 entry */
+
+	dma_sync_single_for_device(vsi_domain->dev, dma, size, DMA_TO_DEVICE);
+}
+
+#define VSI_IOVA_DTE_MASK	0xffc00000
+#define VSI_IOVA_DTE_SHIFT	22
+#define VSI_IOVA_PTE_MASK	0x003ff000
+#define VSI_IOVA_PTE_SHIFT	12
+#define VSI_IOVA_PAGE_MASK	0x00000fff
+#define VSI_IOVA_PAGE_SHIFT	0
+
+static u32 vsi_iova_dte_index(u32 iova)
+{
+	return (iova & VSI_IOVA_DTE_MASK) >> VSI_IOVA_DTE_SHIFT;
+}
+
+static u32 vsi_iova_pte_index(u32 iova)
+{
+	return (iova & VSI_IOVA_PTE_MASK) >> VSI_IOVA_PTE_SHIFT;
+}
+
+static u32 vsi_iova_page_offset(u32 iova)
+{
+	return (iova & VSI_IOVA_PAGE_MASK) >> VSI_IOVA_PAGE_SHIFT;
+}
+
+static irqreturn_t vsi_iommu_irq(int irq, void *dev_id)
+{
+	struct vsi_iommu *iommu = dev_id;
+	unsigned long flags;
+	dma_addr_t iova;
+	u32 status;
+
+	if (pm_runtime_resume_and_get(iommu->dev) < 0)
+		return IRQ_NONE;
+
+	spin_lock_irqsave(&iommu->lock, flags);
+
+	status = readl(iommu->regs + VSI_MMU_STATUS_BASE);
+	if (status & VSI_MMU_IRQ_MASK) {
+		dev_err(iommu->dev, "unexpected int_status=%08x\n", status);
+		iova = readl(iommu->regs + VSI_MMU_PAGE_FAULT_ADDR);
+		report_iommu_fault(iommu->domain, iommu->dev, iova, status);
+	}
+	writel(0, iommu->regs + VSI_MMU_STATUS_BASE);
+
+	spin_unlock_irqrestore(&iommu->lock, flags);
+	pm_runtime_put_autosuspend(iommu->dev);
+
+	return IRQ_HANDLED;
+}
+
+static struct vsi_iommu *vsi_iommu_get_from_dev(struct device *dev)
+{
+	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
+	struct device *iommu_dev = bus_find_device_by_fwnode(&platform_bus_type,
+							     fwspec->iommu_fwnode);
+
+	put_device(iommu_dev);
+
+	return iommu_dev ? dev_get_drvdata(iommu_dev) : NULL;
+}
+
+static struct iommu_domain *vsi_iommu_domain_alloc_paging(struct device *dev)
+{
+	struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
+	struct vsi_iommu_domain *vsi_domain;
+
+	vsi_domain = kzalloc(sizeof(*vsi_domain), GFP_KERNEL);
+	if (!vsi_domain)
+		return NULL;
+
+	vsi_domain->dev = iommu->dev;
+	spin_lock_init(&vsi_domain->lock);
+
+	/*
+	 * iommu use a 2 level pagetable.
+	 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries.
+	 * Allocate one 4 KiB page for each table.
+	 */
+	vsi_domain->dt = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32,
+					      SPAGE_SIZE);
+	if (!vsi_domain->dt)
+		goto err_free_domain;
+
+	vsi_domain->dt_dma = dma_map_single(vsi_domain->dev, vsi_domain->dt,
+					    SPAGE_SIZE, DMA_TO_DEVICE);
+	if (dma_mapping_error(vsi_domain->dev, vsi_domain->dt_dma)) {
+		dev_err(dev, "DMA map error for DT\n");
+		goto err_free_dt;
+	}
+
+	vsi_domain->pta = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32,
+					       SPAGE_SIZE);
+	if (!vsi_domain->pta)
+		goto err_unmap_dt;
+
+	vsi_domain->pta[0] = vsi_mk_pta(vsi_domain->dt_dma);
+	vsi_domain->pta_dma = dma_map_single(vsi_domain->dev, vsi_domain->pta,
+					     SPAGE_SIZE, DMA_TO_DEVICE);
+	if (dma_mapping_error(vsi_domain->dev, vsi_domain->pta_dma)) {
+		dev_err(dev, "DMA map error for PTA\n");
+		goto err_free_pta;
+	}
+
+	INIT_LIST_HEAD(&vsi_domain->iommus);
+
+	vsi_domain->domain.geometry.aperture_start = 0;
+	vsi_domain->domain.geometry.aperture_end   = DMA_BIT_MASK(32);
+	vsi_domain->domain.geometry.force_aperture = true;
+	vsi_domain->domain.pgsize_bitmap	   = SZ_4K;
+
+	return &vsi_domain->domain;
+
+err_free_pta:
+	iommu_free_pages(vsi_domain->pta);
+err_unmap_dt:
+	dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma,
+			 SPAGE_SIZE, DMA_TO_DEVICE);
+err_free_dt:
+	iommu_free_pages(vsi_domain->dt);
+err_free_domain:
+	kfree(vsi_domain);
+
+	return NULL;
+}
+
+static phys_addr_t vsi_iommu_iova_to_phys(struct iommu_domain *domain,
+					  dma_addr_t iova)
+{
+	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
+	phys_addr_t pt_phys, phys = 0;
+	unsigned long flags;
+	u32 dte, pte;
+	u32 *page_table;
+
+	spin_lock_irqsave(&vsi_domain->lock, flags);
+	dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
+	if (!vsi_dte_is_pt_valid(dte))
+		goto unlock;
+
+	pt_phys = vsi_dte_pt_address(dte);
+	page_table = (u32 *)phys_to_virt(pt_phys);
+	pte = page_table[vsi_iova_pte_index(iova)];
+	if (!vsi_pte_is_page_valid(pte))
+		goto unlock;
+
+	phys = vsi_pte_page_address(pte) + vsi_iova_page_offset(iova);
+
+unlock:
+	spin_unlock_irqrestore(&vsi_domain->lock, flags);
+	return phys;
+}
+
+static size_t vsi_iommu_unmap_iova(struct vsi_iommu_domain *vsi_domain,
+				   u32 *pte_addr, dma_addr_t pte_dma,
+				   size_t size)
+{
+	unsigned int pte_count;
+	unsigned int pte_total = size / SPAGE_SIZE;
+
+	for (pte_count = 0;
+	     pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) {
+		u32 pte = pte_addr[pte_count];
+
+		if (!vsi_pte_is_page_valid(pte))
+			break;
+
+		pte_addr[pte_count] = vsi_mk_pte_invalid(pte);
+	}
+
+	vsi_table_flush(vsi_domain, pte_dma, pte_total);
+
+	return pte_count * SPAGE_SIZE;
+}
+
+static int vsi_iommu_map_iova(struct vsi_iommu_domain *vsi_domain, u32 *pte_addr,
+			      dma_addr_t pte_dma, dma_addr_t iova,
+			      phys_addr_t paddr, size_t size, int prot)
+{
+	unsigned int pte_count;
+	unsigned int pte_total = size / SPAGE_SIZE;
+
+	for (pte_count = 0;
+	     pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) {
+		u32 pte = pte_addr[pte_count];
+
+		if (vsi_pte_is_page_valid(pte))
+			return (pte_count - 1) * SPAGE_SIZE;
+
+		pte_addr[pte_count] = vsi_mk_pte(paddr, prot);
+
+		paddr += SPAGE_SIZE;
+	}
+
+	vsi_table_flush(vsi_domain, pte_dma, pte_total);
+
+	return 0;
+}
+
+static void vsi_iommu_flush_tlb(struct iommu_domain *domain)
+{
+	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
+	struct list_head *pos;
+
+	list_for_each(pos, &vsi_domain->iommus) {
+		struct vsi_iommu *iommu;
+
+		iommu = list_entry(pos, struct vsi_iommu, node);
+		if (!iommu)
+			continue;
+
+		if (pm_runtime_get(iommu->dev) < 0)
+			continue;
+
+		spin_lock(&iommu->lock);
+
+		if (iommu->enable) {
+			writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);
+			writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);
+		}
+
+		spin_unlock(&iommu->lock);
+
+		pm_runtime_put_autosuspend(iommu->dev);
+	}
+}
+
+static size_t vsi_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
+			      size_t size, size_t count, struct iommu_iotlb_gather *gather)
+{
+	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
+	dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
+	unsigned long flags;
+	phys_addr_t pt_phys;
+	u32 dte;
+	u32 *pte_addr;
+	size_t unmap_size = 0;
+
+	spin_lock_irqsave(&vsi_domain->lock, flags);
+
+	dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
+	/* Just return 0 if iova is unmapped */
+	if (!vsi_dte_is_pt_valid(dte))
+		goto unlock;
+
+	pt_phys = vsi_dte_pt_address(dte);
+	pte_addr = (u32 *)phys_to_virt(pt_phys) + vsi_iova_pte_index(iova);
+	pte_dma = pt_phys + vsi_iova_pte_index(iova) * sizeof(u32);
+	unmap_size = vsi_iommu_unmap_iova(vsi_domain, pte_addr, pte_dma, size);
+	if (!unmap_size)
+		goto unlock;
+
+	vsi_iommu_flush_tlb(domain);
+unlock:
+	spin_unlock_irqrestore(&vsi_domain->lock, flags);
+
+	return unmap_size;
+}
+
+static u32 *vsi_dte_get_page_table(struct vsi_iommu_domain *vsi_domain,
+				   dma_addr_t iova, gfp_t gfp)
+{
+	u32 *page_table, *dte_addr;
+	u32 dte_index, dte;
+	phys_addr_t pt_phys;
+	dma_addr_t pt_dma;
+	gfp_t flags;
+
+	dte_index = vsi_iova_dte_index(iova);
+	dte_addr = &vsi_domain->dt[dte_index];
+	dte = *dte_addr;
+	if (vsi_dte_is_pt_valid(dte))
+		goto done;
+
+	/* Do not allow to sleep while allocating the buffer */
+	flags = (gfp & ~GFP_KERNEL) | GFP_ATOMIC | GFP_DMA32;
+	page_table = iommu_alloc_pages_sz(flags, PAGE_SIZE);
+	if (!page_table)
+		return ERR_PTR(-ENOMEM);
+
+	pt_dma = dma_map_single(vsi_domain->dev, page_table, PAGE_SIZE, DMA_TO_DEVICE);
+	if (dma_mapping_error(vsi_domain->dev, pt_dma)) {
+		dev_err(vsi_domain->dev, "DMA mapping error while allocating page table\n");
+		iommu_free_pages(page_table);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	dte = vsi_mk_dte(pt_dma);
+	*dte_addr = dte;
+
+	vsi_table_flush(vsi_domain,
+			vsi_domain->dt_dma + dte_index * sizeof(u32), 1);
+done:
+	pt_phys = vsi_dte_pt_address(dte);
+	return (u32 *)phys_to_virt(pt_phys);
+}
+
+static int vsi_iommu_map(struct iommu_domain *domain, unsigned long _iova,
+			 phys_addr_t paddr, size_t size, size_t count,
+			 int prot, gfp_t gfp, size_t *mapped)
+{
+	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
+	dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
+	u32 *page_table, *pte_addr;
+	u32 dte, pte_index;
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&vsi_domain->lock, flags);
+
+	page_table = vsi_dte_get_page_table(vsi_domain, iova, gfp);
+	if (IS_ERR(page_table)) {
+		spin_unlock_irqrestore(&vsi_domain->lock, flags);
+		return PTR_ERR(page_table);
+	}
+
+	dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
+	pte_index = vsi_iova_pte_index(iova);
+	pte_addr = &page_table[pte_index];
+	pte_dma = vsi_dte_pt_address(dte) + pte_index * sizeof(u32);
+	ret = vsi_iommu_map_iova(vsi_domain, pte_addr, pte_dma, iova,
+				 paddr, size, prot);
+	if (!ret)
+		*mapped = size;
+
+	vsi_iommu_flush_tlb(domain);
+
+	spin_unlock_irqrestore(&vsi_domain->lock, flags);
+
+	return ret;
+}
+
+static void vsi_iommu_disable(struct vsi_iommu *iommu)
+{
+	writel(0, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);
+	iommu->enable = false;
+}
+
+static int vsi_iommu_identity_attach(struct iommu_domain *domain,
+				     struct device *dev, struct iommu_domain *old)
+{
+	struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
+	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
+	unsigned long flags;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(iommu->dev);
+	if (ret < 0)
+		return ret;
+
+	spin_lock_irqsave(&vsi_domain->lock, flags);
+	spin_lock(&iommu->lock);
+	if (iommu->domain == domain)
+		goto unlock;
+
+	vsi_iommu_disable(iommu);
+	list_del_init(&iommu->node);
+
+	iommu->domain = domain;
+
+unlock:
+	spin_unlock(&iommu->lock);
+	spin_unlock_irqrestore(&vsi_domain->lock, flags);
+	pm_runtime_put_autosuspend(iommu->dev);
+	return 0;
+}
+
+static const struct iommu_domain_ops vsi_identity_ops = {
+	.attach_dev = vsi_iommu_identity_attach,
+};
+
+static struct iommu_domain vsi_identity_domain = {
+	.type = IOMMU_DOMAIN_IDENTITY,
+	.ops = &vsi_identity_ops,
+};
+
+static void vsi_iommu_enable(struct vsi_iommu *iommu, struct iommu_domain *domain)
+{
+	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
+
+	if (domain == &vsi_identity_domain)
+		return;
+
+	writel(vsi_domain->pta_dma, iommu->regs + VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE);
+	writel(VSI_MMU_OUT_OF_BOUND, iommu->regs + VSI_MMU_CONFIG1_BASE);
+	writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_EXCEPTION_BASE);
+	writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);
+	iommu->enable = true;
+}
+
+static int vsi_iommu_attach_device(struct iommu_domain *domain,
+				   struct device *dev, struct iommu_domain *old)
+{
+	struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
+	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
+	unsigned long flags;
+	int ret = 0;
+
+	ret = pm_runtime_resume_and_get(iommu->dev);
+	if (ret < 0)
+		return ret;
+
+	spin_lock_irqsave(&vsi_domain->lock, flags);
+	spin_lock(&iommu->lock);
+
+	vsi_iommu_enable(iommu, domain);
+	writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);
+	writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);
+
+	list_del_init(&iommu->node);
+	list_add_tail(&iommu->node, &vsi_domain->iommus);
+
+	iommu->domain = domain;
+
+	spin_unlock(&iommu->lock);
+	spin_unlock_irqrestore(&vsi_domain->lock, flags);
+	pm_runtime_put_autosuspend(iommu->dev);
+	return ret;
+}
+
+static void vsi_iommu_domain_free(struct iommu_domain *domain)
+{
+	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
+	unsigned long flags;
+	int i;
+
+	spin_lock_irqsave(&vsi_domain->lock, flags);
+
+	WARN_ON(!list_empty(&vsi_domain->iommus));
+
+	for (i = 0; i < NUM_DT_ENTRIES; i++) {
+		u32 dte = vsi_domain->dt[i];
+
+		if (vsi_dte_is_pt_valid(dte)) {
+			phys_addr_t pt_phys = vsi_dte_pt_address(dte);
+			u32 *page_table = phys_to_virt(pt_phys);
+
+			dma_unmap_single(vsi_domain->dev, pt_phys,
+					 SPAGE_SIZE, DMA_TO_DEVICE);
+			iommu_free_pages(page_table);
+		}
+	}
+
+	dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma,
+			 SPAGE_SIZE, DMA_TO_DEVICE);
+	iommu_free_pages(vsi_domain->dt);
+
+	dma_unmap_single(vsi_domain->dev, vsi_domain->pta_dma,
+			 SPAGE_SIZE, DMA_TO_DEVICE);
+	iommu_free_pages(vsi_domain->pta);
+
+	spin_unlock_irqrestore(&vsi_domain->lock, flags);
+
+	kfree(vsi_domain);
+}
+
+static struct iommu_device *vsi_iommu_probe_device(struct device *dev)
+{
+	struct vsi_iommu *iommu = vsi_iommu_get_from_dev(dev);
+	struct device_link *link;
+
+	link = device_link_add(dev, iommu->dev,
+			       DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
+	if (!link)
+		dev_err(dev, "Unable to link %s\n", dev_name(iommu->dev));
+
+	dev_iommu_priv_set(dev, iommu);
+	return &iommu->iommu;
+}
+
+static void vsi_iommu_release_device(struct device *dev)
+{
+	struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
+
+	device_link_remove(dev, iommu->dev);
+}
+
+static int vsi_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
+{
+	return iommu_fwspec_add_ids(dev, args->args, 1);
+}
+
+static const struct iommu_ops vsi_iommu_ops = {
+	.identity_domain = &vsi_identity_domain,
+	.release_domain = &vsi_identity_domain,
+	.domain_alloc_paging = vsi_iommu_domain_alloc_paging,
+	.of_xlate = vsi_iommu_of_xlate,
+	.probe_device = vsi_iommu_probe_device,
+	.release_device = vsi_iommu_release_device,
+	.device_group = generic_single_device_group,
+	.owner = THIS_MODULE,
+	.default_domain_ops = &(const struct iommu_domain_ops) {
+		.attach_dev		= vsi_iommu_attach_device,
+		.map_pages		= vsi_iommu_map,
+		.unmap_pages		= vsi_iommu_unmap,
+		.iova_to_phys		= vsi_iommu_iova_to_phys,
+		.free			= vsi_iommu_domain_free,
+	}
+};
+
+static const struct of_device_id vsi_iommu_dt_ids[] = {
+	{
+		.compatible = "verisilicon,iommu-1.2",
+	},
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, vsi_iommu_dt_ids);
+
+static int vsi_iommu_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct vsi_iommu *iommu;
+	int err;
+
+	iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
+	if (!iommu)
+		return -ENOMEM;
+
+	iommu->dev = dev;
+	spin_lock_init(&iommu->lock);
+	INIT_LIST_HEAD(&iommu->node);
+
+	iommu->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(iommu->regs))
+		return -ENOMEM;
+
+	iommu->num_clocks = devm_clk_bulk_get_all(dev, &iommu->clocks);
+	if  (iommu->num_clocks < 0)
+		return iommu->num_clocks;
+
+	err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);
+	if (err)
+		return err;
+
+	iommu->irq = platform_get_irq(pdev, 0);
+	if (iommu->irq < 0)
+		return iommu->irq;
+
+	err = devm_request_irq(iommu->dev, iommu->irq, vsi_iommu_irq,
+			       IRQF_SHARED, dev_name(dev), iommu);
+	if (err)
+		goto err_unprepare_clocks;
+
+	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
+	platform_set_drvdata(pdev, iommu);
+
+	pm_runtime_set_autosuspend_delay(dev, 100);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_enable(dev);
+
+	err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, dev_name(dev));
+	if (err)
+		goto err_runtime_disable;
+
+	err = iommu_device_register(&iommu->iommu, &vsi_iommu_ops, dev);
+	if (err)
+		goto err_remove_sysfs;
+
+	return 0;
+
+err_remove_sysfs:
+	iommu_device_sysfs_remove(&iommu->iommu);
+err_runtime_disable:
+	pm_runtime_disable(dev);
+err_unprepare_clocks:
+	clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);
+	return err;
+}
+
+static void vsi_iommu_shutdown(struct platform_device *pdev)
+{
+	struct vsi_iommu *iommu = platform_get_drvdata(pdev);
+
+	disable_irq(iommu->irq);
+	pm_runtime_force_suspend(&pdev->dev);
+}
+
+static int __maybe_unused vsi_iommu_suspend(struct device *dev)
+{
+	struct vsi_iommu *iommu = dev_get_drvdata(dev);
+
+	vsi_iommu_disable(iommu);
+
+	clk_bulk_disable(iommu->num_clocks, iommu->clocks);
+
+	return 0;
+}
+
+static int __maybe_unused vsi_iommu_resume(struct device *dev)
+{
+	struct vsi_iommu *iommu = dev_get_drvdata(dev);
+	unsigned long flags;
+	int ret;
+
+	ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);
+	if (ret)
+		return ret;
+
+	if (iommu->domain) {
+		struct vsi_iommu_domain *vsi_domain = to_vsi_domain(iommu->domain);
+
+		spin_lock_irqsave(&vsi_domain->lock, flags);
+		spin_lock(&iommu->lock);
+		vsi_iommu_enable(iommu, iommu->domain);
+		spin_unlock(&iommu->lock);
+		spin_unlock_irqrestore(&vsi_domain->lock, flags);
+	}
+
+	return 0;
+}
+
+static DEFINE_RUNTIME_DEV_PM_OPS(vsi_iommu_pm_ops,
+				 vsi_iommu_suspend, vsi_iommu_resume,
+				 NULL);
+
+static struct platform_driver rockchip_vsi_iommu_driver = {
+	.probe = vsi_iommu_probe,
+	.shutdown = vsi_iommu_shutdown,
+	.driver = {
+		   .name = "vsi_iommu",
+		   .of_match_table = vsi_iommu_dt_ids,
+		   .pm = pm_sleep_ptr(&vsi_iommu_pm_ops),
+		   .suppress_bind_attrs = true,
+	},
+};
+module_platform_driver(rockchip_vsi_iommu_driver);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Benjamin Gaignard <benjamin.gaignard@collabora.com>");
+MODULE_DESCRIPTION("Verisilicon IOMMU driver");
diff --git a/drivers/irqchip/.kunitconfig b/drivers/irqchip/.kunitconfig
new file mode 100644
index 0000000000000..00a12703f6354
--- /dev/null
+++ b/drivers/irqchip/.kunitconfig
@@ -0,0 +1,5 @@
+CONFIG_KUNIT=y
+CONFIG_OF=y
+CONFIG_COMPILE_TEST=y
+CONFIG_ASPEED_AST2700_INTC=y
+CONFIG_ASPEED_AST2700_INTC_TEST=y
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index e755a2a052096..42f2278a702d0 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -51,12 +51,6 @@ config ARM_GIC_V3_ITS
 	default ARM_GIC_V3
 	select IRQ_MSI_IOMMU
 
-config ARM_GIC_V3_ITS_FSL_MC
-	bool
-	depends on ARM_GIC_V3_ITS
-	depends on FSL_MC_BUS
-	default ARM_GIC_V3_ITS
-
 config ARM_GIC_V5
 	bool
 	select IRQ_DOMAIN_HIERARCHY
@@ -110,6 +104,29 @@ config AL_FIC
 	help
 	  Support Amazon's Annapurna Labs Fabric Interrupt Controller.
 
+config ASPEED_AST2700_INTC
+	bool "ASPEED AST2700 Interrupt Controller support"
+	depends on OF
+	depends on ARCH_ASPEED || COMPILE_TEST
+	select IRQ_DOMAIN_HIERARCHY
+	help
+	  Enable support for the ASPEED AST2700 interrupt controller.
+	  This driver handles interrupt, routing and merged interrupt
+	  sources to upstream parent interrupt controllers.
+
+	  If unsure, say N.
+
+config ASPEED_AST2700_INTC_TEST
+	bool "Tests for the ASPEED AST2700 Interrupt Controller"
+	depends on ASPEED_AST2700_INTC && KUNIT=y
+	default KUNIT_ALL_TESTS
+	help
+	  Enable KUnit tests for AST2700 INTC route resolution.
+	  The tests exercise error handling and route selection paths.
+	  This option is intended for test builds.
+
+	  If unsure, say N.
+
 config ATMEL_AIC_IRQ
 	bool
 	select GENERIC_IRQ_CHIP
@@ -476,7 +493,7 @@ config STM32_EXTI
 	select GENERIC_IRQ_CHIP
 
 config QCOM_IRQ_COMBINER
-	bool "QCOM IRQ combiner support"
+	bool "Qualcomm IRQ combiner support"
 	depends on ARCH_QCOM && ACPI
 	select IRQ_DOMAIN_HIERARCHY
 	help
@@ -509,7 +526,7 @@ config GOLDFISH_PIC
          for Goldfish based virtual platforms.
 
 config QCOM_PDC
-	tristate "QCOM PDC"
+	tristate "Qualcomm PDC"
 	depends on ARCH_QCOM
 	select IRQ_DOMAIN_HIERARCHY
 	help
@@ -517,7 +534,7 @@ config QCOM_PDC
 	  IRQs for Qualcomm Technologies Inc (QTI) mobile chips.
 
 config QCOM_MPM
-	tristate "QCOM MPM"
+	tristate "Qualcomm MPM"
 	depends on ARCH_QCOM
 	depends on MAILBOX
 	select IRQ_DOMAIN_HIERARCHY
@@ -654,13 +671,13 @@ config SIFIVE_PLIC
 	select IRQ_DOMAIN_HIERARCHY
 	select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP
 
-config STARFIVE_JH8100_INTC
-	bool "StarFive JH8100 External Interrupt Controller"
+config STARFIVE_JHB100_INTC
+	bool "StarFive JHB100 External Interrupt Controller"
 	depends on ARCH_STARFIVE || COMPILE_TEST
 	default ARCH_STARFIVE
 	select IRQ_DOMAIN_HIERARCHY
 	help
-	  This enables support for the INTC chip found in StarFive JH8100
+	  This enables support for the INTC chip found in StarFive JHB100
 	  SoC.
 
 	  If you don't know what to do here, say Y.
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index 26aa3b6ec99fd..99f48187a33c6 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -35,7 +35,6 @@ obj-$(CONFIG_ARM_GIC_V2M)		+= irq-gic-v2m.o
 obj-$(CONFIG_ARM_GIC_V3)		+= irq-gic-v3.o irq-gic-v3-mbi.o irq-gic-common.o
 obj-$(CONFIG_ARM_GIC_ITS_PARENT)	+= irq-gic-its-msi-parent.o
 obj-$(CONFIG_ARM_GIC_V3_ITS)		+= irq-gic-v3-its.o irq-gic-v4.o
-obj-$(CONFIG_ARM_GIC_V3_ITS_FSL_MC)	+= irq-gic-v3-its-fsl-mc-msi.o
 obj-$(CONFIG_ARM_GIC_V5)		+= irq-gic-v5.o irq-gic-v5-irs.o irq-gic-v5-its.o \
 					   irq-gic-v5-iwb.o
 obj-$(CONFIG_HISILICON_IRQ_MBIGEN)	+= irq-mbigen.o
@@ -89,8 +88,9 @@ obj-$(CONFIG_MVEBU_PIC)			+= irq-mvebu-pic.o
 obj-$(CONFIG_MVEBU_SEI)			+= irq-mvebu-sei.o
 obj-$(CONFIG_LS_EXTIRQ)			+= irq-ls-extirq.o
 obj-$(CONFIG_LS_SCFG_MSI)		+= irq-ls-scfg-msi.o
+obj-$(CONFIG_ASPEED_AST2700_INTC)	+= irq-ast2700.o irq-ast2700-intc0.o irq-ast2700-intc1.o
+obj-$(CONFIG_ASPEED_AST2700_INTC_TEST)	+= irq-ast2700-intc0-test.o
 obj-$(CONFIG_ARCH_ASPEED)		+= irq-aspeed-vic.o irq-aspeed-i2c-ic.o irq-aspeed-scu-ic.o
-obj-$(CONFIG_ARCH_ASPEED)		+= irq-aspeed-intc.o
 obj-$(CONFIG_STM32MP_EXTI)		+= irq-stm32mp-exti.o
 obj-$(CONFIG_STM32_EXTI) 		+= irq-stm32-exti.o
 obj-$(CONFIG_QCOM_IRQ_COMBINER)		+= qcom-irq-combiner.o
@@ -108,7 +108,7 @@ obj-$(CONFIG_RISCV_APLIC_MSI)		+= irq-riscv-aplic-msi.o
 obj-$(CONFIG_RISCV_IMSIC)		+= irq-riscv-imsic-state.o irq-riscv-imsic-early.o irq-riscv-imsic-platform.o
 obj-$(CONFIG_RISCV_RPMI_SYSMSI)		+= irq-riscv-rpmi-sysmsi.o
 obj-$(CONFIG_SIFIVE_PLIC)		+= irq-sifive-plic.o
-obj-$(CONFIG_STARFIVE_JH8100_INTC)	+= irq-starfive-jh8100-intc.o
+obj-$(CONFIG_STARFIVE_JHB100_INTC)	+= irq-starfive-jhb100-intc.o
 obj-$(CONFIG_ACLINT_SSWI)		+= irq-aclint-sswi.o
 obj-$(CONFIG_IMX_IRQSTEER)		+= irq-imx-irqsteer.o
 obj-$(CONFIG_IMX_INTMUX)		+= irq-imx-intmux.o
diff --git a/drivers/irqchip/irq-aspeed-intc.c b/drivers/irqchip/irq-aspeed-intc.c
deleted file mode 100644
index 4fb0dd8349da0..0000000000000
--- a/drivers/irqchip/irq-aspeed-intc.c
+++ /dev/null
@@ -1,139 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- *  Aspeed Interrupt Controller.
- *
- *  Copyright (C) 2023 ASPEED Technology Inc.
- */
-
-#include <linux/bitops.h>
-#include <linux/irq.h>
-#include <linux/irqchip.h>
-#include <linux/irqchip/chained_irq.h>
-#include <linux/irqdomain.h>
-#include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/io.h>
-#include <linux/spinlock.h>
-
-#define INTC_INT_ENABLE_REG	0x00
-#define INTC_INT_STATUS_REG	0x04
-#define INTC_IRQS_PER_WORD	32
-
-struct aspeed_intc_ic {
-	void __iomem		*base;
-	raw_spinlock_t		gic_lock;
-	raw_spinlock_t		intc_lock;
-	struct irq_domain	*irq_domain;
-};
-
-static void aspeed_intc_ic_irq_handler(struct irq_desc *desc)
-{
-	struct aspeed_intc_ic *intc_ic = irq_desc_get_handler_data(desc);
-	struct irq_chip *chip = irq_desc_get_chip(desc);
-
-	chained_irq_enter(chip, desc);
-
-	scoped_guard(raw_spinlock, &intc_ic->gic_lock) {
-		unsigned long bit, status;
-
-		status = readl(intc_ic->base + INTC_INT_STATUS_REG);
-		for_each_set_bit(bit, &status, INTC_IRQS_PER_WORD) {
-			generic_handle_domain_irq(intc_ic->irq_domain, bit);
-			writel(BIT(bit), intc_ic->base + INTC_INT_STATUS_REG);
-		}
-	}
-
-	chained_irq_exit(chip, desc);
-}
-
-static void aspeed_intc_irq_mask(struct irq_data *data)
-{
-	struct aspeed_intc_ic *intc_ic = irq_data_get_irq_chip_data(data);
-	unsigned int mask = readl(intc_ic->base + INTC_INT_ENABLE_REG) & ~BIT(data->hwirq);
-
-	guard(raw_spinlock)(&intc_ic->intc_lock);
-	writel(mask, intc_ic->base + INTC_INT_ENABLE_REG);
-}
-
-static void aspeed_intc_irq_unmask(struct irq_data *data)
-{
-	struct aspeed_intc_ic *intc_ic = irq_data_get_irq_chip_data(data);
-	unsigned int unmask = readl(intc_ic->base + INTC_INT_ENABLE_REG) | BIT(data->hwirq);
-
-	guard(raw_spinlock)(&intc_ic->intc_lock);
-	writel(unmask, intc_ic->base + INTC_INT_ENABLE_REG);
-}
-
-static struct irq_chip aspeed_intc_chip = {
-	.name			= "ASPEED INTC",
-	.irq_mask		= aspeed_intc_irq_mask,
-	.irq_unmask		= aspeed_intc_irq_unmask,
-};
-
-static int aspeed_intc_ic_map_irq_domain(struct irq_domain *domain, unsigned int irq,
-					 irq_hw_number_t hwirq)
-{
-	irq_set_chip_and_handler(irq, &aspeed_intc_chip, handle_level_irq);
-	irq_set_chip_data(irq, domain->host_data);
-
-	return 0;
-}
-
-static const struct irq_domain_ops aspeed_intc_ic_irq_domain_ops = {
-	.map = aspeed_intc_ic_map_irq_domain,
-};
-
-static int __init aspeed_intc_ic_of_init(struct device_node *node,
-					 struct device_node *parent)
-{
-	struct aspeed_intc_ic *intc_ic;
-	int irq, i, ret = 0;
-
-	intc_ic = kzalloc_obj(*intc_ic);
-	if (!intc_ic)
-		return -ENOMEM;
-
-	intc_ic->base = of_iomap(node, 0);
-	if (!intc_ic->base) {
-		pr_err("Failed to iomap intc_ic base\n");
-		ret = -ENOMEM;
-		goto err_free_ic;
-	}
-	writel(0xffffffff, intc_ic->base + INTC_INT_STATUS_REG);
-	writel(0x0, intc_ic->base + INTC_INT_ENABLE_REG);
-
-	intc_ic->irq_domain = irq_domain_create_linear(of_fwnode_handle(node), INTC_IRQS_PER_WORD,
-						    &aspeed_intc_ic_irq_domain_ops, intc_ic);
-	if (!intc_ic->irq_domain) {
-		ret = -ENOMEM;
-		goto err_iounmap;
-	}
-
-	raw_spin_lock_init(&intc_ic->gic_lock);
-	raw_spin_lock_init(&intc_ic->intc_lock);
-
-	/* Check all the irq numbers valid. If not, unmaps all the base and frees the data. */
-	for (i = 0; i < of_irq_count(node); i++) {
-		irq = irq_of_parse_and_map(node, i);
-		if (!irq) {
-			pr_err("Failed to get irq number\n");
-			ret = -EINVAL;
-			goto err_iounmap;
-		}
-	}
-
-	for (i = 0; i < of_irq_count(node); i++) {
-		irq = irq_of_parse_and_map(node, i);
-		irq_set_chained_handler_and_data(irq, aspeed_intc_ic_irq_handler, intc_ic);
-	}
-
-	return 0;
-
-err_iounmap:
-	iounmap(intc_ic->base);
-err_free_ic:
-	kfree(intc_ic);
-	return ret;
-}
-
-IRQCHIP_DECLARE(ast2700_intc_ic, "aspeed,ast2700-intc-ic", aspeed_intc_ic_of_init);
diff --git a/drivers/irqchip/irq-ast2700-intc0-test.c b/drivers/irqchip/irq-ast2700-intc0-test.c
new file mode 100644
index 0000000000000..d49784509ac7d
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700-intc0-test.c
@@ -0,0 +1,473 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *  Copyright (C) 2026 Code Construct
+ */
+#include <kunit/test.h>
+
+#include "irq-ast2700.h"
+
+static void aspeed_intc0_resolve_route_bad_args(struct kunit *test)
+{
+	static const struct aspeed_intc_interrupt_range c1ranges[] = { 0 };
+	static const u32 c1outs[] = { 0 };
+	struct aspeed_intc_interrupt_range resolved;
+	const struct irq_domain c0domain = { 0 };
+	int rc;
+
+	rc = aspeed_intc0_resolve_route(NULL, 0, c1outs, 0, c1ranges, NULL);
+	KUNIT_EXPECT_EQ(test, rc, -EINVAL);
+
+	rc = aspeed_intc0_resolve_route(&c0domain, 0, c1outs,
+					ARRAY_SIZE(c1ranges), c1ranges,
+					&resolved);
+	KUNIT_EXPECT_EQ(test, rc, -ENOENT);
+
+	rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+					0, c1ranges, &resolved);
+	KUNIT_EXPECT_EQ(test, rc, -ENOENT);
+}
+
+static int gicv3_fwnode_read_string_array(const struct fwnode_handle *fwnode,
+					  const char *propname, const char **val, size_t nval)
+{
+	if (!propname)
+		return -EINVAL;
+
+	if (!val)
+		return 1;
+
+	if (WARN_ON(nval != 1))
+		return -EOVERFLOW;
+
+	*val = "arm,gic-v3";
+	return 1;
+}
+
+static const struct fwnode_operations arm_gicv3_fwnode_ops = {
+	.property_read_string_array = gicv3_fwnode_read_string_array,
+};
+
+static void aspeed_intc_resolve_route_invalid_c0domain(struct kunit *test)
+{
+	struct device_node intc0_node = {
+		.fwnode = { .ops = &arm_gicv3_fwnode_ops },
+	};
+	const struct irq_domain c0domain = { .fwnode = &intc0_node.fwnode };
+	static const struct aspeed_intc_interrupt_range c1ranges[] = { 0 };
+	static const u32 c1outs[] = { 0 };
+	struct aspeed_intc_interrupt_range resolved;
+	int rc;
+
+	rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+					ARRAY_SIZE(c1ranges), c1ranges,
+					&resolved);
+	KUNIT_EXPECT_NE(test, rc, 0);
+}
+
+static int
+aspeed_intc0_fwnode_read_string_array(const struct fwnode_handle *fwnode_handle,
+				      const char *propname, const char **val,
+				      size_t nval)
+{
+	if (!propname)
+		return -EINVAL;
+
+	if (!val)
+		return 1;
+
+	if (WARN_ON(nval != 1))
+		return -EOVERFLOW;
+
+	*val = "aspeed,ast2700-intc0";
+	return nval;
+}
+
+static const struct fwnode_operations intc0_fwnode_ops = {
+	.property_read_string_array = aspeed_intc0_fwnode_read_string_array,
+};
+
+static void
+aspeed_intc0_resolve_route_c1i1o1c0i1o1_connected(struct kunit *test)
+{
+	struct device_node intc0_node = {
+		.fwnode = { .ops = &intc0_fwnode_ops },
+	};
+	struct aspeed_intc_interrupt_range c1ranges[] = {
+		{
+			.start = 0,
+			.count = 1,
+			.upstream = {
+				.fwnode = &intc0_node.fwnode,
+				.param_count = 1,
+				.param = { 128 }
+			}
+		}
+	};
+	static const u32 c1outs[] = { 0 };
+	struct aspeed_intc_interrupt_range resolved;
+	struct aspeed_intc_interrupt_range intc0_ranges[] = {
+		{
+			.start = 128,
+			.count = 1,
+			.upstream = {
+				.fwnode = NULL,
+				.param_count = 0,
+				.param = { 0 },
+			}
+		}
+	};
+	struct aspeed_intc0 intc0 = {
+		.ranges = { .ranges = intc0_ranges, .nranges = ARRAY_SIZE(intc0_ranges), }
+	};
+	const struct irq_domain c0domain = {
+		.host_data = &intc0,
+		.fwnode = &intc0_node.fwnode
+	};
+	int rc;
+
+	rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+					ARRAY_SIZE(c1ranges), c1ranges,
+					&resolved);
+	KUNIT_EXPECT_EQ(test, rc, 0);
+	KUNIT_EXPECT_EQ(test, resolved.start, 0);
+	KUNIT_EXPECT_EQ(test, resolved.count, 1);
+	KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 128);
+}
+
+static void
+aspeed_intc0_resolve_route_c1i1o1c0i1o1_disconnected(struct kunit *test)
+{
+	struct device_node intc0_node = {
+		.fwnode = { .ops = &intc0_fwnode_ops },
+	};
+	struct aspeed_intc_interrupt_range c1ranges[] = {
+		{
+			.start = 0,
+			.count = 1,
+			.upstream = {
+				.fwnode = &intc0_node.fwnode,
+				.param_count = 1,
+				.param = { 128 }
+			}
+		}
+	};
+	static const u32 c1outs[] = { 0 };
+	struct aspeed_intc_interrupt_range resolved;
+	struct aspeed_intc_interrupt_range intc0_ranges[] = {
+		{
+			.start = 129,
+			.count = 1,
+			.upstream = {
+				.fwnode = NULL,
+				.param_count = 0,
+				.param = { 0 },
+			}
+		}
+	};
+	struct aspeed_intc0 intc0 = {
+		.ranges = {
+			.ranges = intc0_ranges,
+			.nranges = ARRAY_SIZE(intc0_ranges),
+		}
+	};
+	const struct irq_domain c0domain = {
+		.host_data = &intc0,
+		.fwnode = &intc0_node.fwnode
+	};
+	int rc;
+
+	rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+					ARRAY_SIZE(c1ranges), c1ranges,
+					&resolved);
+	KUNIT_EXPECT_NE(test, rc, 0);
+}
+
+static void aspeed_intc0_resolve_route_c1i1o1mc0i1o1(struct kunit *test)
+{
+	struct device_node intc0_node = {
+		.fwnode = { .ops = &intc0_fwnode_ops },
+	};
+	struct aspeed_intc_interrupt_range c1ranges[] = {
+		{
+			.start = 0,
+			.count = 1,
+			.upstream = {
+				.fwnode = &intc0_node.fwnode,
+				.param_count = 1,
+				.param = { 480 }
+			}
+		}
+	};
+	static const u32 c1outs[] = { 0 };
+	struct aspeed_intc_interrupt_range resolved;
+	struct aspeed_intc_interrupt_range intc0_ranges[] = {
+		{
+			.start = 192,
+			.count = 1,
+			.upstream = {
+				.fwnode = NULL,
+				.param_count = 0,
+				.param = { 0 },
+			}
+		}
+	};
+	struct aspeed_intc0 intc0 = {
+		.ranges = {
+			.ranges = intc0_ranges,
+			.nranges = ARRAY_SIZE(intc0_ranges),
+		}
+	};
+	const struct irq_domain c0domain = {
+		.host_data = &intc0,
+		.fwnode = &intc0_node.fwnode
+	};
+	int rc;
+
+	rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+					ARRAY_SIZE(c1ranges), c1ranges,
+					&resolved);
+	KUNIT_EXPECT_EQ(test, rc, 0);
+	KUNIT_EXPECT_EQ(test, resolved.start, 0);
+	KUNIT_EXPECT_EQ(test, resolved.count, 1);
+	KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 480);
+}
+
+static void aspeed_intc0_resolve_route_c1i2o2mc0i1o1(struct kunit *test)
+{
+	struct device_node intc0_node = {
+		.fwnode = { .ops = &intc0_fwnode_ops },
+	};
+	struct aspeed_intc_interrupt_range c1ranges[] = {
+		{
+			.start = 0,
+			.count = 1,
+			.upstream = {
+				.fwnode = &intc0_node.fwnode,
+				.param_count = 1,
+				.param = { 480 }
+			}
+		},
+		{
+			.start = 1,
+			.count = 1,
+			.upstream = {
+				.fwnode = &intc0_node.fwnode,
+				.param_count = 1,
+				.param = { 510 }
+			}
+		}
+	};
+	static const u32 c1outs[] = { 1 };
+	struct aspeed_intc_interrupt_range resolved;
+	static struct aspeed_intc_interrupt_range intc0_ranges[] = {
+		{
+			.start = 208,
+			.count = 1,
+			.upstream = {
+				.fwnode = NULL,
+				.param_count = 0,
+				.param = { 0 },
+			}
+		}
+	};
+	struct aspeed_intc0 intc0 = {
+		.ranges = {
+			.ranges = intc0_ranges,
+			.nranges = ARRAY_SIZE(intc0_ranges),
+		}
+	};
+	const struct irq_domain c0domain = {
+		.host_data = &intc0,
+		.fwnode = &intc0_node.fwnode
+	};
+	int rc;
+
+	rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+					ARRAY_SIZE(c1ranges), c1ranges,
+					&resolved);
+	KUNIT_EXPECT_EQ(test, rc, 0);
+	KUNIT_EXPECT_EQ(test, resolved.start, 1);
+	KUNIT_EXPECT_EQ(test, resolved.count, 1);
+	KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 510);
+}
+
+static void aspeed_intc0_resolve_route_c1i1o1mc0i2o1(struct kunit *test)
+{
+	struct device_node intc0_node = {
+		.fwnode = { .ops = &intc0_fwnode_ops },
+	};
+	struct aspeed_intc_interrupt_range c1ranges[] = {
+		{
+			.start = 0,
+			.count = 1,
+			.upstream = {
+				.fwnode = &intc0_node.fwnode,
+				.param_count = 1,
+				.param = { 510 }
+			}
+		},
+	};
+	static const u32 c1outs[] = { 0 };
+	struct aspeed_intc_interrupt_range resolved;
+	static struct aspeed_intc_interrupt_range intc0_ranges[] = {
+		{
+			.start = 192,
+			.count = 1,
+			.upstream = {
+				.fwnode = NULL,
+				.param_count = 0,
+				.param = {0},
+			}
+		},
+		{
+			.start = 208,
+			.count = 1,
+			.upstream = {
+				.fwnode = NULL,
+				.param_count = 0,
+				.param = {0},
+			}
+		}
+	};
+	struct aspeed_intc0 intc0 = {
+		.ranges = {
+			.ranges = intc0_ranges,
+			.nranges = ARRAY_SIZE(intc0_ranges),
+		}
+	};
+	const struct irq_domain c0domain = {
+		.host_data = &intc0,
+		.fwnode = &intc0_node.fwnode
+	};
+	int rc;
+
+	rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+					ARRAY_SIZE(c1ranges), c1ranges,
+					&resolved);
+	KUNIT_EXPECT_EQ(test, rc, 0);
+	KUNIT_EXPECT_EQ(test, resolved.start, 0);
+	KUNIT_EXPECT_EQ(test, resolved.count, 1);
+	KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 510);
+}
+
+static void aspeed_intc0_resolve_route_c1i1o2mc0i1o1_invalid(struct kunit *test)
+{
+	struct device_node intc0_node = {
+		.fwnode = { .ops = &intc0_fwnode_ops },
+	};
+	struct aspeed_intc_interrupt_range c1ranges[] = {
+		{
+			.start = 0,
+			.count = 1,
+			.upstream = {
+				.fwnode = &intc0_node.fwnode,
+				.param_count = 1,
+				.param = { 480 }
+			}
+		}
+	};
+	static const u32 c1outs[] = {
+		AST2700_INTC_INVALID_ROUTE, 0
+	};
+	struct aspeed_intc_interrupt_range resolved;
+	struct aspeed_intc_interrupt_range intc0_ranges[] = {
+		{
+			.start = 192,
+			.count = 1,
+			.upstream = {
+				.fwnode = NULL,
+				.param_count = 0,
+				.param = { 0 },
+			}
+		}
+	};
+	struct aspeed_intc0 intc0 = {
+		.ranges = {
+			.ranges = intc0_ranges,
+			.nranges = ARRAY_SIZE(intc0_ranges),
+		}
+	};
+	const struct irq_domain c0domain = {
+		.host_data = &intc0,
+		.fwnode = &intc0_node.fwnode
+	};
+	int rc;
+
+	rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+					ARRAY_SIZE(c1ranges), c1ranges,
+					&resolved);
+	KUNIT_EXPECT_EQ(test, rc, 1);
+	KUNIT_EXPECT_EQ(test, resolved.start, 0);
+	KUNIT_EXPECT_EQ(test, resolved.count, 1);
+	KUNIT_EXPECT_EQ(test, resolved.upstream.param[0], 480);
+}
+
+static void
+aspeed_intc0_resolve_route_c1i1o1mc0i1o1_bad_range_upstream(struct kunit *test)
+{
+	struct device_node intc0_node = {
+		.fwnode = { .ops = &intc0_fwnode_ops },
+	};
+	struct aspeed_intc_interrupt_range c1ranges[] = {
+		{
+			.start = 0,
+			.count = 1,
+			.upstream = {
+				.fwnode = &intc0_node.fwnode,
+				.param_count = 0,
+				.param = { 0 }
+			}
+		}
+	};
+	static const u32 c1outs[] = { 0 };
+	struct aspeed_intc_interrupt_range resolved;
+	struct aspeed_intc_interrupt_range intc0_ranges[] = {
+		{
+			.start = 0,
+			.count = 0,
+			.upstream = {
+				.fwnode = NULL,
+				.param_count = 0,
+				.param = { 0 },
+			}
+		}
+	};
+	struct aspeed_intc0 intc0 = {
+		.ranges = {
+			.ranges = intc0_ranges,
+			.nranges = ARRAY_SIZE(intc0_ranges),
+		}
+	};
+	const struct irq_domain c0domain = {
+		.host_data = &intc0,
+		.fwnode = &intc0_node.fwnode
+	};
+	int rc;
+
+	rc = aspeed_intc0_resolve_route(&c0domain, ARRAY_SIZE(c1outs), c1outs,
+					ARRAY_SIZE(c1ranges), c1ranges,
+					&resolved);
+	KUNIT_EXPECT_NE(test, rc, 0);
+}
+
+static struct kunit_case ast2700_intc0_test_cases[] = {
+	KUNIT_CASE(aspeed_intc0_resolve_route_bad_args),
+	KUNIT_CASE(aspeed_intc_resolve_route_invalid_c0domain),
+	KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1c0i1o1_connected),
+	KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1c0i1o1_disconnected),
+	KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1mc0i1o1),
+	KUNIT_CASE(aspeed_intc0_resolve_route_c1i2o2mc0i1o1),
+	KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1mc0i2o1),
+	KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o2mc0i1o1_invalid),
+	KUNIT_CASE(aspeed_intc0_resolve_route_c1i1o1mc0i1o1_bad_range_upstream),
+	{},
+};
+
+static struct kunit_suite ast2700_intc0_test_suite = {
+	.name = "ast2700-intc0",
+	.test_cases = ast2700_intc0_test_cases,
+};
+
+kunit_test_suite(ast2700_intc0_test_suite);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/irqchip/irq-ast2700-intc0.c b/drivers/irqchip/irq-ast2700-intc0.c
new file mode 100644
index 0000000000000..14b8b88f1179b
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700-intc0.c
@@ -0,0 +1,582 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *  Aspeed AST2700 Interrupt Controller.
+ *
+ *  Copyright (C) 2026 ASPEED Technology Inc.
+ */
+
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/fwnode.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/kconfig.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/overflow.h>
+#include <linux/property.h>
+#include <linux/spinlock.h>
+
+#include "irq-ast2700.h"
+
+#define INT_NUM		480
+#define INTM_NUM	50
+#define SWINT_NUM	16
+
+#define INTM_BASE	(INT_NUM)
+#define SWINT_BASE	(INT_NUM + INTM_NUM)
+#define INT0_NUM	(INT_NUM + INTM_NUM + SWINT_NUM)
+
+#define INTC0_IN_NUM		480
+#define INTC0_ROUTE_NUM		5
+#define INTC0_INTM_NUM		50
+#define INTC0_ROUTE_BITS	3
+
+#define GIC_P2P_SPI_END		128
+#define INTC0_SWINT_OUT_BASE	144
+
+#define INTC0_SWINT_IER		0x10
+#define INTC0_SWINT_ISR		0x14
+#define INTC0_INTBANKX_IER	0x1000
+#define INTC0_INTBANK_SIZE	0x100
+#define INTC0_INTBANK_GROUPS	11
+#define INTC0_INTBANKS_PER_GRP	3
+#define INTC0_INTMX_IER		0x1b00
+#define INTC0_INTMX_ISR		0x1b04
+#define INTC0_INTMX_BANK_SIZE	0x10
+#define INTC0_INTM_BANK_NUM	3
+#define INTC0_IRQS_PER_BANK	32
+#define INTM_IRQS_PER_BANK	10
+#define INTC0_SEL_BASE			0x200
+#define INTC0_SEL_BANK_SIZE		0x4
+#define INTC0_SEL_ROUTE_SIZE	0x100
+
+static void aspeed_swint_irq_mask(struct irq_data *data)
+{
+	struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+	int bit = data->hwirq - SWINT_BASE;
+	u32 ier;
+
+	guard(raw_spinlock)(&intc0->intc_lock);
+	ier = readl(intc0->base + INTC0_SWINT_IER) & ~BIT(bit);
+	writel(ier, intc0->base + INTC0_SWINT_IER);
+	irq_chip_mask_parent(data);
+}
+
+static void aspeed_swint_irq_unmask(struct irq_data *data)
+{
+	struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+	int bit = data->hwirq - SWINT_BASE;
+	u32 ier;
+
+	guard(raw_spinlock)(&intc0->intc_lock);
+	ier = readl(intc0->base + INTC0_SWINT_IER) | BIT(bit);
+	writel(ier, intc0->base + INTC0_SWINT_IER);
+	irq_chip_unmask_parent(data);
+}
+
+static void aspeed_swint_irq_eoi(struct irq_data *data)
+{
+	struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+	int bit = data->hwirq - SWINT_BASE;
+
+	writel(BIT(bit), intc0->base + INTC0_SWINT_ISR);
+	irq_chip_eoi_parent(data);
+}
+
+static struct irq_chip aspeed_swint_chip = {
+	.name			= "ast2700-swint",
+	.irq_eoi		= aspeed_swint_irq_eoi,
+	.irq_mask		= aspeed_swint_irq_mask,
+	.irq_unmask		= aspeed_swint_irq_unmask,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.flags			= IRQCHIP_SET_TYPE_MASKED,
+};
+
+static void aspeed_intc0_irq_mask(struct irq_data *data)
+{
+	struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+	int bank = (data->hwirq - INTM_BASE) / INTM_IRQS_PER_BANK;
+	int bit = (data->hwirq - INTM_BASE) % INTM_IRQS_PER_BANK;
+	u32 ier;
+
+	guard(raw_spinlock)(&intc0->intc_lock);
+	ier = readl(intc0->base + INTC0_INTMX_IER + bank * INTC0_INTMX_BANK_SIZE) & ~BIT(bit);
+	writel(ier, intc0->base + INTC0_INTMX_IER + bank * INTC0_INTMX_BANK_SIZE);
+	irq_chip_mask_parent(data);
+}
+
+static void aspeed_intc0_irq_unmask(struct irq_data *data)
+{
+	struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+	int bank = (data->hwirq - INTM_BASE) / INTM_IRQS_PER_BANK;
+	int bit = (data->hwirq - INTM_BASE) % INTM_IRQS_PER_BANK;
+	u32 ier;
+
+	guard(raw_spinlock)(&intc0->intc_lock);
+	ier = readl(intc0->base + INTC0_INTMX_IER + bank * INTC0_INTMX_BANK_SIZE) | BIT(bit);
+	writel(ier, intc0->base + INTC0_INTMX_IER + bank * INTC0_INTMX_BANK_SIZE);
+	irq_chip_unmask_parent(data);
+}
+
+static void aspeed_intc0_irq_eoi(struct irq_data *data)
+{
+	struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+	int bank = (data->hwirq - INTM_BASE) / INTM_IRQS_PER_BANK;
+	int bit = (data->hwirq - INTM_BASE) % INTM_IRQS_PER_BANK;
+
+	writel(BIT(bit), intc0->base + INTC0_INTMX_ISR + bank * INTC0_INTMX_BANK_SIZE);
+	irq_chip_eoi_parent(data);
+}
+
+static struct irq_chip aspeed_intm_chip = {
+	.name			= "ast2700-intmerge",
+	.irq_eoi		= aspeed_intc0_irq_eoi,
+	.irq_mask		= aspeed_intc0_irq_mask,
+	.irq_unmask		= aspeed_intc0_irq_unmask,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.flags			= IRQCHIP_SET_TYPE_MASKED,
+};
+
+static struct irq_chip linear_intr_irq_chip = {
+	.name			= "ast2700-int",
+	.irq_eoi		= irq_chip_eoi_parent,
+	.irq_mask		= irq_chip_mask_parent,
+	.irq_unmask		= irq_chip_unmask_parent,
+	.irq_set_affinity	= irq_chip_set_affinity_parent,
+	.flags			= IRQCHIP_SET_TYPE_MASKED,
+};
+
+static const u32 aspeed_intc0_routes[INTC0_IN_NUM / INTC0_IRQS_PER_BANK][INTC0_ROUTE_NUM] = {
+	{ 0, 256, 426, AST2700_INTC_INVALID_ROUTE, AST2700_INTC_INVALID_ROUTE },
+	{ 32, 288, 458, AST2700_INTC_INVALID_ROUTE, AST2700_INTC_INVALID_ROUTE },
+	{ 64, 320, 490, AST2700_INTC_INVALID_ROUTE, AST2700_INTC_INVALID_ROUTE },
+	{ 96, 352, 522, AST2700_INTC_INVALID_ROUTE, AST2700_INTC_INVALID_ROUTE },
+	{ 128, 384, 554, 160, 176 },
+	{ 129, 385, 555, 161, 177 },
+	{ 130, 386, 556, 162, 178 },
+	{ 131, 387, 557, 163, 179 },
+	{ 132, 388, 558, 164, 180 },
+	{ 133, 544, 714, 165, 181 },
+	{ 134, 545, 715, 166, 182 },
+	{ 135, 546, 706, 167, 183 },
+	{ 136, 547, 707, 168, 184 },
+	{ 137, 548, 708, 169, 185 },
+	{ 138, 549, 709, 170, 186 },
+};
+
+static const u32 aspeed_intc0_intm_routes[INTC0_INTM_NUM / INTM_IRQS_PER_BANK] = {
+	192, 416, 586, 208, 224
+};
+
+static int resolve_input_from_child_ranges(const struct aspeed_intc0 *intc0,
+					   const struct aspeed_intc_interrupt_range *range,
+					   u32 outpin, u32 *input)
+{
+	u32 offset, base;
+
+	if (!in_range32(outpin, range->start, range->count))
+		return -ENOENT;
+
+	if (range->upstream.param_count == 0)
+		return -EINVAL;
+
+	base = range->upstream.param[ASPEED_INTC_RANGES_BASE];
+	offset = outpin - range->start;
+	if (check_add_overflow(base, offset, input)) {
+		dev_warn(intc0->dev, "%s: Arithmetic overflow for input derivation: %u + %u\n",
+			 __func__, base, offset);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static int resolve_parent_range_for_output(const struct aspeed_intc0 *intc0,
+					   const struct fwnode_handle *parent, u32 output,
+					   struct aspeed_intc_interrupt_range *resolved)
+{
+	for (size_t i = 0; i < intc0->ranges.nranges; i++) {
+		struct aspeed_intc_interrupt_range range = intc0->ranges.ranges[i];
+
+		if (!in_range32(output, range.start, range.count))
+			continue;
+
+		if (range.upstream.fwnode != parent)
+			continue;
+
+		if (resolved) {
+			resolved->start = output;
+			resolved->count = 1;
+			resolved->upstream = range.upstream;
+			resolved->upstream.param[ASPEED_INTC_RANGES_COUNT] +=
+				output - range.start;
+		}
+
+		return 0;
+	}
+
+	return -ENOENT;
+}
+
+static int resolve_parent_route_for_input(const struct aspeed_intc0 *intc0,
+					  const struct fwnode_handle *parent, u32 input,
+					  struct aspeed_intc_interrupt_range *resolved)
+{
+	int rc = -ENOENT;
+	u32 c0o;
+
+	if (input < INT_NUM) {
+		static_assert(INTC0_ROUTE_NUM < INT_MAX, "Broken cast");
+		for (size_t i = 0; rc == -ENOENT && i < INTC0_ROUTE_NUM; i++) {
+			c0o = aspeed_intc0_routes[input / INTC0_IRQS_PER_BANK][i];
+			if (c0o == AST2700_INTC_INVALID_ROUTE)
+				continue;
+
+			if (input < GIC_P2P_SPI_END)
+				c0o += input % INTC0_IRQS_PER_BANK;
+
+			rc = resolve_parent_range_for_output(intc0, parent, c0o, resolved);
+			if (!rc)
+				return (int)i;
+		}
+	} else if (input < (INT_NUM + INTM_NUM)) {
+		c0o = aspeed_intc0_intm_routes[(input - INT_NUM) / INTM_IRQS_PER_BANK];
+		c0o += ((input - INT_NUM) % INTM_IRQS_PER_BANK);
+		return resolve_parent_range_for_output(intc0, parent, c0o, resolved);
+	} else if (input < (INT_NUM + INTM_NUM + SWINT_NUM)) {
+		c0o = input - SWINT_BASE + INTC0_SWINT_OUT_BASE;
+		return resolve_parent_range_for_output(intc0, parent, c0o, resolved);
+	} else {
+		return -ENOENT;
+	}
+
+	return rc;
+}
+
+/**
+ * aspeed_intc0_resolve_route - Determine the necessary interrupt output at intc1
+ * @c0domain: The pointer to intc0's irq_domain
+ * @nc1outs: The number of valid intc1 outputs available for the input
+ * @c1outs: The array of available intc1 output indices for the input
+ * @nc1ranges: The number of interrupt range entries for intc1
+ * @c1ranges: The array of configured intc1 interrupt ranges
+ * @resolved: The fully resolved range entry after applying the resolution
+ *            algorithm
+ *
+ * Returns: The intc1 route index associated with the intc1 output identified in
+ * @resolved on success. Otherwise, a negative errno value.
+ *
+ * The AST2700 interrupt architecture allows any peripheral interrupt source
+ * to be routed to one of up to four processors running in the SoC. A processor
+ * binding a driver for a peripheral that requests an interrupt is (without
+ * further design and effort) the destination for the requested interrupt.
+ *
+ * Routing a peripheral interrupt to its destination processor requires
+ * coordination between INTC0 on the CPU die and one or more INTC1 instances.
+ * At least one INTC1 instance exists in the SoC on the IO-die, however up
+ * to two more instances may be integrated via LTPI (LVDS Tunneling Protocol
+ * & Interface).
+ *
+ * Between the multiple destinations, various route constraints, and the
+ * devicetree binding design, some information that's needed at INTC1 instances
+ * to route inbound interrupts correctly to the destination processor is only
+ * available at INTC0.
+ *
+ * aspeed_intc0_resolve_route() is to be invoked by INTC1 driver instances to
+ * perform the route resolution. The implementation in INTC0 allows INTC0 to
+ * encapsulate the information used to perform route selection, and provides it
+ * with an opportunity to apply policy as part of the selection process. Such
+ * policy may, for instance, choose to de-prioritise some interrupts destined
+ * for the PSP (Primary Service Processor) GIC.
+ */
+int aspeed_intc0_resolve_route(const struct irq_domain *c0domain, size_t nc1outs,
+			       const u32 *c1outs, size_t nc1ranges,
+			       const struct aspeed_intc_interrupt_range *c1ranges,
+			       struct aspeed_intc_interrupt_range *resolved)
+{
+	struct fwnode_handle *parent_fwnode;
+	struct aspeed_intc0 *intc0;
+	int ret;
+
+	if (!c0domain || !resolved)
+		return -EINVAL;
+
+	if (nc1outs > INT_MAX)
+		return -EINVAL;
+
+	if (nc1outs == 0 || nc1ranges == 0)
+		return -ENOENT;
+
+	if (!IS_ENABLED(CONFIG_ASPEED_AST2700_INTC_TEST) &&
+	    !fwnode_device_is_compatible(c0domain->fwnode, "aspeed,ast2700-intc0"))
+		return -ENODEV;
+
+	intc0 = c0domain->host_data;
+	if (!intc0)
+		return -EINVAL;
+
+	parent_fwnode = of_fwnode_handle(intc0->parent);
+
+	for (size_t i = 0; i < nc1outs; i++) {
+		u32 c1o = c1outs[i];
+
+		if (c1o == AST2700_INTC_INVALID_ROUTE)
+			continue;
+
+		for (size_t j = 0; j < nc1ranges; j++) {
+			struct aspeed_intc_interrupt_range c1r = c1ranges[j];
+			u32 input;
+
+			/*
+			 * Range match for intc1 output pin
+			 *
+			 * Assume a failed match is still a match for the purpose of testing,
+			 * saves a bunch of mess in the test fixtures
+			 */
+			if (!(c0domain == c1r.domain ||
+			      IS_ENABLED(CONFIG_ASPEED_AST2700_INTC_TEST)))
+				continue;
+
+			ret = resolve_input_from_child_ranges(intc0, &c1r, c1o, &input);
+			if (ret)
+				continue;
+
+			/*
+			 * INTC1 should never request routes for peripheral interrupt sources
+			 * directly attached to INTC0.
+			 */
+			if (input < GIC_P2P_SPI_END)
+				continue;
+
+			ret = resolve_parent_route_for_input(intc0, parent_fwnode, input, NULL);
+			if (ret < 0)
+				continue;
+
+			/* Route resolution succeeded */
+			resolved->start = c1o;
+			resolved->count = 1;
+			resolved->upstream = c1r.upstream;
+			resolved->upstream.param[ASPEED_INTC_RANGES_BASE] = input;
+			/* Cast protected by prior test against nc1outs */
+			return (int)i;
+		}
+	}
+
+	return -ENOENT;
+}
+
+static int aspeed_intc0_irq_domain_map(struct irq_domain *domain,
+				       unsigned int irq, irq_hw_number_t hwirq)
+{
+	if (hwirq < GIC_P2P_SPI_END)
+		irq_set_chip_and_handler(irq, &linear_intr_irq_chip, handle_level_irq);
+	else if (hwirq < INTM_BASE)
+		return -EINVAL;
+	else if (hwirq < SWINT_BASE)
+		irq_set_chip_and_handler(irq, &aspeed_intm_chip, handle_level_irq);
+	else if (hwirq < INT0_NUM)
+		irq_set_chip_and_handler(irq, &aspeed_swint_chip, handle_level_irq);
+	else
+		return -EINVAL;
+
+	irq_set_chip_data(irq, domain->host_data);
+	return 0;
+}
+
+static int aspeed_intc0_irq_domain_translate(struct irq_domain *domain,
+					     struct irq_fwspec *fwspec,
+					     unsigned long *hwirq,
+					     unsigned int *type)
+{
+	if (fwspec->param_count != 1)
+		return -EINVAL;
+
+	*hwirq = fwspec->param[0];
+	*type = IRQ_TYPE_NONE;
+	return 0;
+}
+
+static int aspeed_intc0_irq_domain_alloc(struct irq_domain *domain,
+					 unsigned int virq,
+					 unsigned int nr_irqs, void *data)
+{
+	struct aspeed_intc0 *intc0 = domain->host_data;
+	struct aspeed_intc_interrupt_range resolved;
+	struct irq_fwspec *fwspec = data;
+	struct irq_fwspec parent_fwspec;
+	struct irq_chip *chip;
+	unsigned long hwirq;
+	unsigned int type;
+	int ret;
+
+	ret = aspeed_intc0_irq_domain_translate(domain, fwspec, &hwirq, &type);
+	if (ret)
+		return ret;
+
+	if (hwirq >= GIC_P2P_SPI_END && hwirq < INT_NUM)
+		return -EINVAL;
+
+	if (hwirq < INTM_BASE)
+		chip = &linear_intr_irq_chip;
+	else if (hwirq < SWINT_BASE)
+		chip = &aspeed_intm_chip;
+	else
+		chip = &aspeed_swint_chip;
+
+	ret = resolve_parent_route_for_input(intc0, domain->parent->fwnode,
+					     (u32)hwirq, &resolved);
+	if (ret)
+		return ret;
+
+	parent_fwspec = resolved.upstream;
+	ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs,
+					   &parent_fwspec);
+	if (ret)
+		return ret;
+
+	for (int i = 0; i < nr_irqs; ++i, ++hwirq, ++virq) {
+		ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, chip,
+						    domain->host_data);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int aspeed_intc0_irq_domain_activate(struct irq_domain *domain,
+					    struct irq_data *data, bool reserve)
+{
+	struct aspeed_intc0 *intc0 = irq_data_get_irq_chip_data(data);
+	unsigned long hwirq = data->hwirq;
+	int route, bank, bit;
+	u32 mask;
+
+	if (hwirq >= INT0_NUM)
+		return -EINVAL;
+
+	if (in_range32(hwirq, INTM_BASE, INTM_NUM + SWINT_NUM))
+		return 0;
+
+	bank = hwirq / INTC0_IRQS_PER_BANK;
+	bit = hwirq % INTC0_IRQS_PER_BANK;
+	mask = BIT(bit);
+
+	route = resolve_parent_route_for_input(intc0, intc0->local->parent->fwnode,
+					       hwirq, NULL);
+	if (route < 0)
+		return route;
+
+	guard(raw_spinlock)(&intc0->intc_lock);
+	for (int i = 0; i < INTC0_ROUTE_BITS; i++) {
+		void __iomem *sel = intc0->base + INTC0_SEL_BASE +
+				    (bank * INTC0_SEL_BANK_SIZE) +
+				    (INTC0_SEL_ROUTE_SIZE * i);
+		u32 reg = readl(sel);
+
+		if (route & BIT(i))
+			reg |= mask;
+		else
+			reg &= ~mask;
+
+		writel(reg, sel);
+		if (readl(sel) != reg)
+			return -EACCES;
+	}
+
+	return 0;
+}
+
+static const struct irq_domain_ops aspeed_intc0_irq_domain_ops = {
+	.translate	= aspeed_intc0_irq_domain_translate,
+	.activate	= aspeed_intc0_irq_domain_activate,
+	.alloc		= aspeed_intc0_irq_domain_alloc,
+	.free		= irq_domain_free_irqs_common,
+	.map		= aspeed_intc0_irq_domain_map,
+};
+
+static void aspeed_intc0_disable_swint(struct aspeed_intc0 *intc0)
+{
+	writel(0, intc0->base + INTC0_SWINT_IER);
+}
+
+static void aspeed_intc0_disable_intbank(struct aspeed_intc0 *intc0)
+{
+	for (int i = 0; i < INTC0_INTBANK_GROUPS; i++) {
+		for (int j = 0; j < INTC0_INTBANKS_PER_GRP; j++) {
+			u32 base = INTC0_INTBANKX_IER +
+				   (INTC0_INTBANK_SIZE * i) +
+				   (INTC0_INTMX_BANK_SIZE * j);
+
+			writel(0, intc0->base + base);
+		}
+	}
+}
+
+static void aspeed_intc0_disable_intm(struct aspeed_intc0 *intc0)
+{
+	for (int i = 0; i < INTC0_INTM_BANK_NUM; i++)
+		writel(0, intc0->base + INTC0_INTMX_IER + (INTC0_INTMX_BANK_SIZE * i));
+}
+
+static int aspeed_intc0_probe(struct platform_device *pdev,
+			      struct device_node *parent)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct irq_domain *parent_domain;
+	struct aspeed_intc0 *intc0;
+	int ret;
+
+	if (!parent) {
+		pr_err("missing parent interrupt node\n");
+		return -ENODEV;
+	}
+
+	intc0 = devm_kzalloc(&pdev->dev, sizeof(*intc0), GFP_KERNEL);
+	if (!intc0)
+		return -ENOMEM;
+
+	intc0->dev = &pdev->dev;
+	intc0->parent = parent;
+	intc0->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(intc0->base))
+		return PTR_ERR(intc0->base);
+
+	aspeed_intc0_disable_swint(intc0);
+	aspeed_intc0_disable_intbank(intc0);
+	aspeed_intc0_disable_intm(intc0);
+
+	raw_spin_lock_init(&intc0->intc_lock);
+
+	parent_domain = irq_find_host(parent);
+	if (!parent_domain) {
+		pr_err("unable to obtain parent domain\n");
+		return -ENODEV;
+	}
+
+	if (!of_device_is_compatible(parent, "arm,gic-v3"))
+		return -ENODEV;
+
+	intc0->local = irq_domain_create_hierarchy(parent_domain, 0, INT0_NUM,
+						   of_fwnode_handle(node),
+						   &aspeed_intc0_irq_domain_ops,
+						   intc0);
+	if (!intc0->local)
+		return -ENOMEM;
+
+	ret = aspeed_intc_populate_ranges(&pdev->dev, &intc0->ranges);
+	if (ret < 0) {
+		irq_domain_remove(intc0->local);
+		return ret;
+	}
+
+	return 0;
+}
+
+IRQCHIP_PLATFORM_DRIVER_BEGIN(ast2700_intc0)
+IRQCHIP_MATCH("aspeed,ast2700-intc0", aspeed_intc0_probe)
+IRQCHIP_PLATFORM_DRIVER_END(ast2700_intc0)
diff --git a/drivers/irqchip/irq-ast2700-intc1.c b/drivers/irqchip/irq-ast2700-intc1.c
new file mode 100644
index 0000000000000..59e8f0d5ddcd2
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700-intc1.c
@@ -0,0 +1,280 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *  Aspeed AST2700 Interrupt Controller.
+ *
+ *  Copyright (C) 2026 ASPEED Technology Inc.
+ */
+
+#include <linux/bitops.h>
+#include <linux/device.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/spinlock.h>
+
+#include "irq-ast2700.h"
+
+#define INTC1_IER			0x100
+#define INTC1_ISR			0x104
+#define INTC1_BANK_SIZE		0x10
+#define INTC1_SEL_BASE			0x80
+#define INTC1_SEL_BANK_SIZE		0x4
+#define INTC1_SEL_ROUTE_SIZE	0x20
+#define INTC1_IRQS_PER_BANK		32
+#define INTC1_BANK_NUM			6
+#define INTC1_ROUTE_NUM			7
+#define INTC1_IN_NUM			192
+#define INTC1_BOOTMCU_ROUTE		6
+#define INTC1_ROUTE_SELECTOR_BITS	3
+#define INTC1_ROUTE_IRQS_PER_GROUP	32
+#define INTC1_ROUTE_SHIFT		5
+
+struct aspeed_intc1 {
+	struct device				*dev;
+	void __iomem				*base;
+	raw_spinlock_t				intc_lock;
+	struct irq_domain			*local;
+	struct irq_domain			*upstream;
+	struct aspeed_intc_interrupt_ranges	ranges;
+};
+
+static void aspeed_intc1_disable_int(struct aspeed_intc1 *intc1)
+{
+	for (int i = 0; i < INTC1_BANK_NUM; i++)
+		writel(0, intc1->base + INTC1_IER + (INTC1_BANK_SIZE * i));
+}
+
+static void aspeed_intc1_irq_handler(struct irq_desc *desc)
+{
+	struct aspeed_intc1 *intc1 = irq_desc_get_handler_data(desc);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	unsigned long bit, status;
+
+	chained_irq_enter(chip, desc);
+
+	for (int bank = 0; bank < INTC1_BANK_NUM; bank++) {
+		status = readl(intc1->base + INTC1_ISR + (INTC1_BANK_SIZE * bank));
+		if (!status)
+			continue;
+
+		for_each_set_bit(bit, &status, INTC1_IRQS_PER_BANK) {
+			generic_handle_domain_irq(intc1->local, (bank * INTC1_IRQS_PER_BANK) + bit);
+			writel(BIT(bit), intc1->base + INTC1_ISR + (INTC1_BANK_SIZE * bank));
+		}
+	}
+
+	chained_irq_exit(chip, desc);
+}
+
+static void aspeed_intc1_irq_mask(struct irq_data *data)
+{
+	struct aspeed_intc1 *intc1 = irq_data_get_irq_chip_data(data);
+	int bank = data->hwirq / INTC1_IRQS_PER_BANK;
+	int bit = data->hwirq % INTC1_IRQS_PER_BANK;
+	u32 ier;
+
+	guard(raw_spinlock)(&intc1->intc_lock);
+	ier = readl(intc1->base + INTC1_IER + (INTC1_BANK_SIZE * bank)) & ~BIT(bit);
+	writel(ier, intc1->base + INTC1_IER + (INTC1_BANK_SIZE * bank));
+}
+
+static void aspeed_intc1_irq_unmask(struct irq_data *data)
+{
+	struct aspeed_intc1 *intc1 = irq_data_get_irq_chip_data(data);
+	int bank = data->hwirq / INTC1_IRQS_PER_BANK;
+	int bit = data->hwirq % INTC1_IRQS_PER_BANK;
+	u32 ier;
+
+	guard(raw_spinlock)(&intc1->intc_lock);
+	ier = readl(intc1->base + INTC1_IER + (INTC1_BANK_SIZE * bank)) | BIT(bit);
+	writel(ier, intc1->base + INTC1_IER + (INTC1_BANK_SIZE * bank));
+}
+
+static struct irq_chip aspeed_intc_chip = {
+	.name		= "ASPEED INTC1",
+	.irq_mask	= aspeed_intc1_irq_mask,
+	.irq_unmask	= aspeed_intc1_irq_unmask,
+};
+
+static int aspeed_intc1_irq_domain_translate(struct irq_domain *domain,
+					     struct irq_fwspec *fwspec,
+					     unsigned long *hwirq,
+					     unsigned int *type)
+{
+	if (fwspec->param_count != 1)
+		return -EINVAL;
+
+	*hwirq = fwspec->param[0];
+	*type = IRQ_TYPE_LEVEL_HIGH;
+	return 0;
+}
+
+static int aspeed_intc1_map_irq_domain(struct irq_domain *domain,
+				       unsigned int irq,
+				       irq_hw_number_t hwirq)
+{
+	irq_domain_set_info(domain, irq, hwirq, &aspeed_intc_chip,
+			    domain->host_data, handle_level_irq, NULL, NULL);
+	return 0;
+}
+
+/*
+ * In-bound interrupts are progressively merged into one out-bound interrupt in
+ * groups of 32. Apply this fact to compress the route table in corresponding
+ * groups of 32.
+ */
+static const u32
+aspeed_intc1_routes[INTC1_IN_NUM / INTC1_ROUTE_IRQS_PER_GROUP][INTC1_ROUTE_NUM] = {
+	{ 0, AST2700_INTC_INVALID_ROUTE, 10, 20, 30, 40, 50 },
+	{ 1, AST2700_INTC_INVALID_ROUTE, 11, 21, 31, 41, 50 },
+	{ 2, AST2700_INTC_INVALID_ROUTE, 12, 22, 32, 42, 50 },
+	{ 3, AST2700_INTC_INVALID_ROUTE, 13, 23, 33, 43, 50 },
+	{ 4, AST2700_INTC_INVALID_ROUTE, 14, 24, 34, 44, 50 },
+	{ 5, AST2700_INTC_INVALID_ROUTE, 15, 25, 35, 45, 50 },
+};
+
+static int aspeed_intc1_irq_domain_activate(struct irq_domain *domain,
+					    struct irq_data *data, bool reserve)
+{
+	struct aspeed_intc1 *intc1 = irq_data_get_irq_chip_data(data);
+	struct aspeed_intc_interrupt_range resolved;
+	int rc, bank, bit;
+	u32 mask;
+
+	if (WARN_ON_ONCE((data->hwirq >> INTC1_ROUTE_SHIFT) >= ARRAY_SIZE(aspeed_intc1_routes)))
+		return -EINVAL;
+
+	/*
+	 * outpin may be an error if the upstream is the BootMCU APLIC node, or
+	 * anything except a valid intc0 driver instance
+	 */
+	rc = aspeed_intc0_resolve_route(intc1->upstream, INTC1_ROUTE_NUM,
+					aspeed_intc1_routes[data->hwirq >> INTC1_ROUTE_SHIFT],
+					intc1->ranges.nranges,
+					intc1->ranges.ranges, &resolved);
+	if (rc < 0) {
+		if (!fwnode_device_is_compatible(intc1->upstream->fwnode, "riscv,aplic")) {
+			dev_warn(intc1->dev,
+				 "Failed to resolve interrupt route for hwirq %lu in domain %s\n",
+				 data->hwirq, domain->name);
+			return rc;
+		}
+		rc = INTC1_BOOTMCU_ROUTE;
+	}
+
+	bank = data->hwirq / INTC1_IRQS_PER_BANK;
+	bit = data->hwirq % INTC1_IRQS_PER_BANK;
+	mask = BIT(bit);
+
+	guard(raw_spinlock)(&intc1->intc_lock);
+	for (int i = 0; i < INTC1_ROUTE_SELECTOR_BITS; i++) {
+		void __iomem *sel = intc1->base + INTC1_SEL_BASE +
+				    (bank * INTC1_SEL_BANK_SIZE) +
+				    (INTC1_SEL_ROUTE_SIZE * i);
+		u32 reg = readl(sel);
+
+		if (rc & BIT(i))
+			reg |= mask;
+		else
+			reg &= ~mask;
+
+		writel(reg, sel);
+		if (readl(sel) != reg)
+			return -EACCES;
+	}
+
+	return 0;
+}
+
+static const struct irq_domain_ops aspeed_intc1_irq_domain_ops = {
+	.map		= aspeed_intc1_map_irq_domain,
+	.translate	= aspeed_intc1_irq_domain_translate,
+	.activate	= aspeed_intc1_irq_domain_activate,
+};
+
+static void aspeed_intc1_request_interrupts(struct aspeed_intc1 *intc1)
+{
+	for (unsigned int i = 0; i < intc1->ranges.nranges; i++) {
+		struct aspeed_intc_interrupt_range *r =
+			&intc1->ranges.ranges[i];
+
+		if (intc1->upstream != r->domain)
+			continue;
+
+		for (u32 k = 0; k < r->count; k++) {
+			struct of_phandle_args parent_irq;
+			int irq;
+
+			parent_irq.np = to_of_node(r->upstream.fwnode);
+			parent_irq.args_count = 1;
+			parent_irq.args[0] =
+				intc1->ranges.ranges[i].upstream.param[ASPEED_INTC_RANGES_BASE] + k;
+
+			irq = irq_create_of_mapping(&parent_irq);
+			if (!irq)
+				continue;
+
+			irq_set_chained_handler_and_data(irq,
+							 aspeed_intc1_irq_handler, intc1);
+		}
+	}
+}
+
+static int aspeed_intc1_probe(struct platform_device *pdev,
+			      struct device_node *parent)
+{
+	struct device_node *node = pdev->dev.of_node;
+	struct aspeed_intc1 *intc1;
+	struct irq_domain *host;
+	int ret;
+
+	if (!parent) {
+		dev_err(&pdev->dev, "missing parent interrupt node\n");
+		return -ENODEV;
+	}
+
+	if (!of_device_is_compatible(parent, "aspeed,ast2700-intc0"))
+		return -ENODEV;
+
+	host = irq_find_host(parent);
+	if (!host)
+		return -ENODEV;
+
+	intc1 = devm_kzalloc(&pdev->dev, sizeof(*intc1), GFP_KERNEL);
+	if (!intc1)
+		return -ENOMEM;
+
+	intc1->dev = &pdev->dev;
+	intc1->upstream = host;
+	intc1->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(intc1->base))
+		return PTR_ERR(intc1->base);
+
+	aspeed_intc1_disable_int(intc1);
+
+	raw_spin_lock_init(&intc1->intc_lock);
+
+	intc1->local = irq_domain_create_linear(of_fwnode_handle(node),
+						INTC1_BANK_NUM * INTC1_IRQS_PER_BANK,
+						&aspeed_intc1_irq_domain_ops, intc1);
+	if (!intc1->local)
+		return -ENOMEM;
+
+	ret = aspeed_intc_populate_ranges(&pdev->dev, &intc1->ranges);
+	if (ret < 0) {
+		irq_domain_remove(intc1->local);
+		return ret;
+	}
+
+	aspeed_intc1_request_interrupts(intc1);
+
+	return 0;
+}
+
+IRQCHIP_PLATFORM_DRIVER_BEGIN(ast2700_intc1)
+IRQCHIP_MATCH("aspeed,ast2700-intc1", aspeed_intc1_probe)
+IRQCHIP_PLATFORM_DRIVER_END(ast2700_intc1)
diff --git a/drivers/irqchip/irq-ast2700.c b/drivers/irqchip/irq-ast2700.c
new file mode 100644
index 0000000000000..1e4c4a624dbf5
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700.c
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ *  Aspeed AST2700 Interrupt Controller.
+ *
+ *  Copyright (C) 2026 ASPEED Technology Inc.
+ */
+#include "irq-ast2700.h"
+
+#define ASPEED_INTC_RANGE_FIXED_CELLS	3U
+#define ASPEED_INTC_RANGE_OFF_START	0U
+#define ASPEED_INTC_RANGE_OFF_COUNT	1U
+#define ASPEED_INTC_RANGE_OFF_PHANDLE	2U
+
+/**
+ * aspeed_intc_populate_ranges
+ * @dev: Device owning the interrupt controller node.
+ * @ranges: Destination for parsed range descriptors.
+ *
+ * Return: 0 on success, negative errno on error.
+ */
+int aspeed_intc_populate_ranges(struct device *dev,
+				struct aspeed_intc_interrupt_ranges *ranges)
+{
+	struct aspeed_intc_interrupt_range *arr;
+	const __be32 *pvs, *pve;
+	struct device_node *dn;
+	int len;
+
+	if (!dev || !ranges)
+		return -EINVAL;
+
+	dn = dev->of_node;
+
+	pvs = of_get_property(dn, "aspeed,interrupt-ranges", &len);
+	if (!pvs)
+		return -EINVAL;
+
+	if (len % sizeof(__be32))
+		return -EINVAL;
+
+	/* Over-estimate the range entry count for now */
+	ranges->ranges = devm_kmalloc_array(dev,
+					    len / (ASPEED_INTC_RANGE_FIXED_CELLS * sizeof(__be32)),
+					    sizeof(*ranges->ranges),
+					    GFP_KERNEL);
+	if (!ranges->ranges)
+		return -ENOMEM;
+
+	pve = pvs + (len / sizeof(__be32));
+	for (unsigned int i = 0; pve - pvs >= ASPEED_INTC_RANGE_FIXED_CELLS; i++) {
+		struct aspeed_intc_interrupt_range *r;
+		struct device_node *target;
+		u32 target_cells;
+
+		target = of_find_node_by_phandle(be32_to_cpu(pvs[ASPEED_INTC_RANGE_OFF_PHANDLE]));
+		if (!target)
+			return -EINVAL;
+
+		if (of_property_read_u32(target, "#interrupt-cells",
+					 &target_cells)) {
+			of_node_put(target);
+			return -EINVAL;
+		}
+
+		if (!target_cells || target_cells > IRQ_DOMAIN_IRQ_SPEC_PARAMS) {
+			of_node_put(target);
+			return -EINVAL;
+		}
+
+		if (pve - pvs < ASPEED_INTC_RANGE_FIXED_CELLS + target_cells) {
+			of_node_put(target);
+			return -EINVAL;
+		}
+
+		r = &ranges->ranges[i];
+		r->start = be32_to_cpu(pvs[ASPEED_INTC_RANGE_OFF_START]);
+		r->count = be32_to_cpu(pvs[ASPEED_INTC_RANGE_OFF_COUNT]);
+
+		{
+			struct of_phandle_args args = {
+				.np = target,
+				.args_count = target_cells,
+			};
+
+			for (u32 j = 0; j < target_cells; j++)
+				args.args[j] = be32_to_cpu(pvs[ASPEED_INTC_RANGE_FIXED_CELLS + j]);
+
+			of_phandle_args_to_fwspec(target, args.args,
+						  args.args_count,
+						  &r->upstream);
+		}
+
+		of_node_put(target);
+		r->domain = irq_find_matching_fwspec(&r->upstream, DOMAIN_BUS_ANY);
+		pvs += ASPEED_INTC_RANGE_FIXED_CELLS + target_cells;
+		ranges->nranges++;
+	}
+
+	/* Re-fit the range array now we know the entry count */
+	arr = devm_krealloc_array(dev, ranges->ranges, ranges->nranges,
+				  sizeof(*ranges->ranges), GFP_KERNEL);
+	if (!arr)
+		return -ENOMEM;
+	ranges->ranges = arr;
+
+	return 0;
+}
diff --git a/drivers/irqchip/irq-ast2700.h b/drivers/irqchip/irq-ast2700.h
new file mode 100644
index 0000000000000..318296638445e
--- /dev/null
+++ b/drivers/irqchip/irq-ast2700.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ *  Aspeed AST2700 Interrupt Controller.
+ *
+ *  Copyright (C) 2026 ASPEED Technology Inc.
+ */
+#ifndef DRIVERS_IRQCHIP_AST2700
+#define DRIVERS_IRQCHIP_AST2700
+
+#include <linux/device.h>
+#include <linux/irqdomain.h>
+
+#define AST2700_INTC_INVALID_ROUTE (~0U)
+#define ASPEED_INTC_RANGES_BASE		0U
+#define ASPEED_INTC_RANGES_COUNT	1U
+
+struct aspeed_intc_interrupt_range {
+	u32               start;
+	u32               count;
+	struct irq_fwspec upstream;
+	struct irq_domain *domain;
+};
+
+struct aspeed_intc_interrupt_ranges {
+	struct aspeed_intc_interrupt_range *ranges;
+	unsigned int                       nranges;
+};
+
+struct aspeed_intc0 {
+	struct device				*dev;
+	void __iomem				*base;
+	raw_spinlock_t				intc_lock;
+	struct irq_domain			*local;
+	struct device_node			*parent;
+	struct aspeed_intc_interrupt_ranges	ranges;
+};
+
+int aspeed_intc_populate_ranges(struct device *dev,
+				struct aspeed_intc_interrupt_ranges *ranges);
+
+int aspeed_intc0_resolve_route(const struct irq_domain *c0domain,
+			       size_t nc1outs,
+			       const u32 *c1outs,
+			       size_t nc1ranges,
+			       const struct aspeed_intc_interrupt_range *c1ranges,
+			       struct aspeed_intc_interrupt_range *resolved);
+
+#endif
diff --git a/drivers/irqchip/irq-econet-en751221.c b/drivers/irqchip/irq-econet-en751221.c
index d83d5eb127950..2ca5d901866fe 100644
--- a/drivers/irqchip/irq-econet-en751221.c
+++ b/drivers/irqchip/irq-econet-en751221.c
@@ -30,6 +30,8 @@
 #include <linux/irqchip.h>
 #include <linux/irqchip/chained_irq.h>
 
+#include <asm/setup.h>
+
 #define IRQ_COUNT		40
 
 #define NOT_PERCPU		0xff
@@ -41,15 +43,19 @@
 #define REG_PENDING1		0x54
 
 /**
- * @membase: Base address of the interrupt controller registers
- * @interrupt_shadows: Array of all interrupts, for each value,
- *	- NOT_PERCPU: This interrupt is not per-cpu, so it has no shadow
- *	- IS_SHADOW: This interrupt is a shadow of another per-cpu interrupt
- *	- else: This is a per-cpu interrupt whose shadow is the value
+ * @membase:		Base address of the interrupt controller registers
+ * @domain:		The irq_domain for direct dispatch
+ * @ipi_domain:		The irq_domain for inter-process dispatch
+ * @interrupt_shadows:	Array of all interrupts, for each value,
+ *	- NOT_PERCPU:	This interrupt is not per-cpu, so it has no shadow
+ *	- IS_SHADOW:	This interrupt is a shadow of another per-cpu interrupt
+ *	- else:		This is a per-cpu interrupt whose shadow is the value
  */
 static struct {
-	void __iomem	*membase;
-	u8		interrupt_shadows[IRQ_COUNT];
+	void __iomem		*membase;
+	struct irq_domain	*domain;
+	struct irq_domain	*ipi_domain;
+	u8			interrupt_shadows[IRQ_COUNT];
 } econet_intc __ro_after_init;
 
 static DEFINE_RAW_SPINLOCK(irq_lock);
@@ -150,6 +156,56 @@ static void econet_intc_from_parent(struct irq_desc *desc)
 	chained_irq_exit(chip, desc);
 }
 
+/*
+ * When in VEIC mode, the CPU jumps to a handler in the vector table.
+ * The only way to know which interrupt is being triggered is from the vector table offset that
+ * has been jumped to. Reading REG_PENDING(0|1) will tell you which interrupts are currently
+ * pending in the intc, but that will not tell you which one the intc wants you to process
+ * right now. And if you are not processing the exact interrupt that the intc wants you to be
+ * processing, you might be on the wrong VPE. You can't tell which VPE any given REG_PENDING
+ * interrupt is intended for (shadow IRQ numbers are for masking only, they never flag as
+ * pending).
+ *
+ * Consequently, this little ritual of generating n handler functions and registering one per
+ * interrupt is unavoidable.
+ */
+#define X(irq) \
+	static void econet_irq_dispatch ## irq (void) \
+	{ \
+		do_domain_IRQ(econet_intc.domain, irq); \
+	}
+
+ X(0)  X(1)  X(2)  X(3)  X(4)  X(5)  X(6)  X(7)  X(8)  X(9)
+X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) X(18) X(19)
+X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) X(28) X(29)
+X(30) X(31) X(32) X(33) X(34) X(35) X(36) X(37) X(38) X(39)
+
+#undef X
+#define X(irq) econet_irq_dispatch ## irq,
+
+static void (* const econet_irq_dispatchers[])(void) = {
+	X(0)  X(1)  X(2)  X(3)  X(4)  X(5)  X(6)  X(7)  X(8)  X(9)
+	X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) X(18) X(19)
+	X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) X(28) X(29)
+	X(30) X(31) X(32) X(33) X(34) X(35) X(36) X(37) X(38) X(39)
+};
+
+/* Likewise, we do the same for the 2 IPI IRQs so that we can route them back */
+static void econet_cpu_dispatch0(void)
+{
+	do_domain_IRQ(econet_intc.ipi_domain, 0);
+}
+
+static void econet_cpu_dispatch1(void)
+{
+	do_domain_IRQ(econet_intc.ipi_domain, 1);
+}
+
+static void (* const econet_cpu_dispatchers[])(void) = {
+	econet_cpu_dispatch0,
+	econet_cpu_dispatch1,
+};
+
 static const struct irq_chip econet_irq_chip;
 
 static int econet_intc_map(struct irq_domain *d, u32 irq, irq_hw_number_t hwirq)
@@ -174,6 +230,10 @@ static int econet_intc_map(struct irq_domain *d, u32 irq, irq_hw_number_t hwirq)
 	}
 
 	irq_set_chip_data(irq, NULL);
+
+	if (cpu_has_veic)
+		set_vi_handler(hwirq + 1, econet_irq_dispatchers[hwirq]);
+
 	return 0;
 }
 
@@ -249,6 +309,100 @@ static int __init get_shadow_interrupts(struct device_node *node)
 	return 0;
 }
 
+/**
+ * econet_cpu_init() - configure routing of CPU interrupts to the correct domain.
+ * @node: The devicetree node of this interrupt controller.
+ *
+ * Interrupts that originate from the CPU are unconditionally unmasked here and are re-routed back
+ * to the IPI irq_domain in the CPU intc. Masking still takes place but the CPU intc is in charge
+ * of it, using the mask bits of the c0_status register.
+ *
+ * Note that because IP2 ... IP7 are repurposed as Interrupt Priority Level, only the two IPI
+ * interrupts are actually supported.
+ */
+static int __init econet_cpu_init(struct device_node *node)
+{
+	const char *field = "econet,cpu-interrupt-map";
+	struct device_node *parent_intc;
+	int map_size;
+	u32 mask;
+
+	map_size = of_property_count_u32_elems(node, field);
+
+	if (map_size <= 0) {
+		return 0;
+	} else if (map_size % 2) {
+		pr_err("%pOF: %s count is odd, ignoring\n", node, field);
+		return 0;
+	}
+
+	u32 *maps __free(kfree) = kmalloc_array(map_size, sizeof(u32), GFP_KERNEL);
+	if (!maps)
+		return -ENOMEM;
+
+	if (of_property_read_u32_array(node, field, maps, map_size)) {
+		pr_err("%pOF: Failed to read %s\n", node, field);
+		return -EINVAL;
+	}
+
+	/* Validation */
+	for (int i = 0; i < map_size; i += 2) {
+		u32 receive = maps[i];
+		u32 dispatch = maps[i + 1];
+		u8 shadow;
+
+		if (receive >= IRQ_COUNT) {
+			pr_err("%pOF: Entry %d:%d in %s (%u) is out of bounds\n",
+			       node, i, 0, field, receive);
+			return -EINVAL;
+		}
+
+		shadow = econet_intc.interrupt_shadows[receive];
+		if (shadow != NOT_PERCPU && shadow >= IRQ_COUNT) {
+			pr_err("%pOF: Entry %d:%d in %s (%u) has invalid shadow (%d)\n",
+			       node, i, 0, field, receive, shadow);
+			return -EINVAL;
+		}
+
+		if (dispatch >= ARRAY_SIZE(econet_cpu_dispatchers)) {
+			pr_err("%pOF: Entry %d:%d in %s (%u) is out of bounds only IPI interrupts are supported\n",
+			       node, i, 1, field, dispatch);
+			return -EINVAL;
+		}
+	}
+
+	parent_intc = of_irq_find_parent(node);
+	if (!parent_intc) {
+		pr_err("%pOF: Failed to find parent %s\n", node, "IRQ device");
+		return -ENODEV;
+	}
+
+	econet_intc.ipi_domain = irq_find_matching_host(parent_intc, DOMAIN_BUS_IPI);
+	if (!econet_intc.ipi_domain) {
+		pr_err("%pOF: Failed to find parent %s\n", node, "IPI domain");
+		return -ENODEV;
+	}
+
+	mask = 0;
+	for (int i = 0; i < map_size; i += 2) {
+		u32 receive = maps[i];
+		u32 dispatch = maps[i + 1];
+		u8 shadow;
+
+		set_vi_handler(receive + 1, econet_cpu_dispatchers[dispatch]);
+
+		mask |= BIT(receive);
+
+		shadow = econet_intc.interrupt_shadows[receive];
+		if (shadow != NOT_PERCPU)
+			mask |= BIT(shadow);
+	}
+
+	econet_wreg(REG_MASK0, mask, mask);
+
+	return 0;
+}
+
 static int __init econet_intc_of_init(struct device_node *node, struct device_node *parent)
 {
 	struct irq_domain *domain;
@@ -294,7 +448,23 @@ static int __init econet_intc_of_init(struct device_node *node, struct device_no
 		goto err_unmap;
 	}
 
-	irq_set_chained_handler_and_data(irq, econet_intc_from_parent, domain);
+	/*
+	 * 34K Manual (MD00534) Section 6.3.1.3 rev 1.13 page 136:
+	 * In VEIC mode, IP2 ... IP7 are repurposed as Interrupt Priority Level. The controller
+	 * will filter incoming interrupts whose priority is lower than the IPL number. Therefore
+	 * we must not set any of these bits. We avoid setting IP2 by not actually chaining this
+	 * intc to the CPU intc.
+	 */
+	if (cpu_has_veic) {
+		ret = econet_cpu_init(node);
+
+		if (ret)
+			return ret;
+	} else {
+		irq_set_chained_handler_and_data(irq, econet_intc_from_parent, domain);
+	}
+
+	econet_intc.domain = domain;
 
 	return 0;
 
diff --git a/drivers/irqchip/irq-gic-its-msi-parent.c b/drivers/irqchip/irq-gic-its-msi-parent.c
index a832cdb2e6978..d36b278ae66c5 100644
--- a/drivers/irqchip/irq-gic-its-msi-parent.c
+++ b/drivers/irqchip/irq-gic-its-msi-parent.c
@@ -5,6 +5,7 @@
 // Copyright (C) 2022 Intel
 
 #include <linux/acpi_iort.h>
+#include <linux/fsl/mc.h>
 #include <linux/of_address.h>
 #include <linux/pci.h>
 
@@ -187,9 +188,11 @@ static int its_pmsi_prepare(struct irq_domain *domain, struct device *dev,
 {
 	struct msi_domain_info *msi_info;
 	u32 dev_id;
-	int ret;
+	int ret = 0;
 
-	if (dev->of_node)
+	if (dev_is_fsl_mc(dev))
+		dev_id = fsl_mc_get_msi_id(dev);
+	else if (dev->of_node)
 		ret = of_pmsi_get_msi_info(domain->parent, dev, &dev_id, NULL);
 	else
 		ret = iort_pmsi_get_msi_info(dev, &dev_id, NULL);
diff --git a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c b/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c
deleted file mode 100644
index b5785472765a3..0000000000000
--- a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c
+++ /dev/null
@@ -1,168 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Freescale Management Complex (MC) bus driver MSI support
- *
- * Copyright (C) 2015-2016 Freescale Semiconductor, Inc.
- * Author: German Rivera <German.Rivera@freescale.com>
- *
- */
-
-#include <linux/acpi.h>
-#include <linux/acpi_iort.h>
-#include <linux/irq.h>
-#include <linux/msi.h>
-#include <linux/of.h>
-#include <linux/of_irq.h>
-#include <linux/fsl/mc.h>
-
-static struct irq_chip its_msi_irq_chip = {
-	.name = "ITS-fMSI",
-	.irq_mask = irq_chip_mask_parent,
-	.irq_unmask = irq_chip_unmask_parent,
-	.irq_eoi = irq_chip_eoi_parent,
-	.irq_set_affinity = msi_domain_set_affinity
-};
-
-static u32 fsl_mc_msi_domain_get_msi_id(struct irq_domain *domain,
-					struct fsl_mc_device *mc_dev)
-{
-	struct device_node *of_node;
-	u32 out_id;
-
-	of_node = irq_domain_get_of_node(domain);
-	out_id = of_node ? of_msi_xlate(&mc_dev->dev, &of_node, mc_dev->icid) :
-			iort_msi_map_id(&mc_dev->dev, mc_dev->icid);
-
-	return out_id;
-}
-
-static int its_fsl_mc_msi_prepare(struct irq_domain *msi_domain,
-				  struct device *dev,
-				  int nvec, msi_alloc_info_t *info)
-{
-	struct fsl_mc_device *mc_bus_dev;
-	struct msi_domain_info *msi_info;
-
-	if (!dev_is_fsl_mc(dev))
-		return -EINVAL;
-
-	mc_bus_dev = to_fsl_mc_device(dev);
-	if (!(mc_bus_dev->flags & FSL_MC_IS_DPRC))
-		return -EINVAL;
-
-	/*
-	 * Set the device Id to be passed to the GIC-ITS:
-	 *
-	 * NOTE: This device id corresponds to the IOMMU stream ID
-	 * associated with the DPRC object (ICID).
-	 */
-	info->scratchpad[0].ul = fsl_mc_msi_domain_get_msi_id(msi_domain,
-							      mc_bus_dev);
-	msi_info = msi_get_domain_info(msi_domain->parent);
-
-	/* Allocate at least 32 MSIs, and always as a power of 2 */
-	nvec = max_t(int, 32, roundup_pow_of_two(nvec));
-	return msi_info->ops->msi_prepare(msi_domain->parent, dev, nvec, info);
-}
-
-static struct msi_domain_ops its_fsl_mc_msi_ops __ro_after_init = {
-	.msi_prepare = its_fsl_mc_msi_prepare,
-};
-
-static struct msi_domain_info its_fsl_mc_msi_domain_info = {
-	.flags	= (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
-	.ops	= &its_fsl_mc_msi_ops,
-	.chip	= &its_msi_irq_chip,
-};
-
-static const struct of_device_id its_device_id[] = {
-	{	.compatible	= "arm,gic-v3-its",	},
-	{},
-};
-
-static void __init its_fsl_mc_msi_init_one(struct fwnode_handle *handle,
-					  const char *name)
-{
-	struct irq_domain *parent;
-	struct irq_domain *mc_msi_domain;
-
-	parent = irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS);
-	if (!parent || !msi_get_domain_info(parent)) {
-		pr_err("%s: unable to locate ITS domain\n", name);
-		return;
-	}
-
-	mc_msi_domain = fsl_mc_msi_create_irq_domain(handle,
-						&its_fsl_mc_msi_domain_info,
-						parent);
-	if (!mc_msi_domain) {
-		pr_err("%s: unable to create fsl-mc domain\n", name);
-		return;
-	}
-
-	pr_info("fsl-mc MSI: %s domain created\n", name);
-}
-
-#ifdef CONFIG_ACPI
-static int __init
-its_fsl_mc_msi_parse_madt(union acpi_subtable_headers *header,
-			  const unsigned long end)
-{
-	struct acpi_madt_generic_translator *its_entry;
-	struct fwnode_handle *dom_handle;
-	const char *node_name;
-	int err = 0;
-
-	its_entry = (struct acpi_madt_generic_translator *)header;
-	node_name = kasprintf(GFP_KERNEL, "ITS@0x%lx",
-			      (long)its_entry->base_address);
-
-	dom_handle = iort_find_domain_token(its_entry->translation_id);
-	if (!dom_handle) {
-		pr_err("%s: Unable to locate ITS domain handle\n", node_name);
-		err = -ENXIO;
-		goto out;
-	}
-
-	its_fsl_mc_msi_init_one(dom_handle, node_name);
-
-out:
-	kfree(node_name);
-	return err;
-}
-
-
-static void __init its_fsl_mc_acpi_msi_init(void)
-{
-	acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
-			      its_fsl_mc_msi_parse_madt, 0);
-}
-#else
-static inline void its_fsl_mc_acpi_msi_init(void) { }
-#endif
-
-static void __init its_fsl_mc_of_msi_init(void)
-{
-	struct device_node *np;
-
-	for (np = of_find_matching_node(NULL, its_device_id); np;
-	     np = of_find_matching_node(np, its_device_id)) {
-		if (!of_device_is_available(np))
-			continue;
-		if (!of_property_read_bool(np, "msi-controller"))
-			continue;
-
-		its_fsl_mc_msi_init_one(of_fwnode_handle(np),
-					np->full_name);
-	}
-}
-
-static int __init its_fsl_mc_msi_init(void)
-{
-	its_fsl_mc_of_msi_init();
-	its_fsl_mc_acpi_msi_init();
-
-	return 0;
-}
-
-early_initcall(its_fsl_mc_msi_init);
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 291d7668cc8da..e78795b7acfd6 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -4784,8 +4784,7 @@ static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
 	struct its_node *its = data;
 
 	/* erratum 22375: only alloc 8MB table size (20 bits) */
-	its->typer &= ~GITS_TYPER_DEVBITS;
-	its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1);
+	FIELD_MODIFY(GITS_TYPER_DEVBITS, &its->typer, 20 - 1);
 	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
 
 	return true;
@@ -4805,8 +4804,7 @@ static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
 	struct its_node *its = data;
 
 	/* On QDF2400, the size of the ITE is 16Bytes */
-	its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE;
-	its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1);
+	FIELD_MODIFY(GITS_TYPER_ITT_ENTRY_SIZE, &its->typer, 16 - 1);
 
 	return true;
 }
@@ -4840,10 +4838,8 @@ static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
 		its->get_msi_base = its_irq_get_msi_base_pre_its;
 
 		ids = ilog2(pre_its_window[1]) - 2;
-		if (device_ids(its) > ids) {
-			its->typer &= ~GITS_TYPER_DEVBITS;
-			its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1);
-		}
+		if (device_ids(its) > ids)
+			FIELD_MODIFY(GITS_TYPER_DEVBITS, &its->typer, ids - 1);
 
 		/* the pre-ITS breaks isolation, so disable MSI remapping */
 		its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI;
diff --git a/drivers/irqchip/irq-meson-gpio.c b/drivers/irqchip/irq-meson-gpio.c
index 74a376ef452e2..91a9c337fe6d9 100644
--- a/drivers/irqchip/irq-meson-gpio.c
+++ b/drivers/irqchip/irq-meson-gpio.c
@@ -27,6 +27,10 @@
 /* use for A1 like chips */
 #define REG_PIN_A1_SEL	0x04
 
+/* use for A9 like chips */
+#define REG_A9_AO_POL	0x00
+#define REG_A9_AO_EDGE	0x30
+
 /*
  * Note: The S905X3 datasheet reports that BOTH_EDGE is controlled by
  * bits 24 to 31. Tests on the actual HW show that these bits are
@@ -53,6 +57,8 @@ static void meson_a1_gpio_irq_sel_pin(struct meson_gpio_irq_controller *ctl,
 static void meson_a1_gpio_irq_init(struct meson_gpio_irq_controller *ctl);
 static int meson8_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
 				    unsigned int type, u32 *channel_hwirq);
+static int meson_a9_ao_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
+					 unsigned int type, u32 *channel_hwirq);
 static int meson_s4_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
 				      unsigned int type, u32 *channel_hwirq);
 
@@ -116,6 +122,18 @@ struct meson_gpio_irq_params {
 	.pin_sel_mask = 0xff,					\
 	.nr_channels = 2,					\
 
+#define INIT_MESON_A9_AO_COMMON_DATA(irqs)			\
+	INIT_MESON_COMMON(irqs, meson_a1_gpio_irq_init,		\
+			  meson_a1_gpio_irq_sel_pin,		\
+			  meson_a9_ao_gpio_irq_set_type)	\
+	.support_edge_both = true,				\
+	.edge_both_offset = 0,					\
+	.edge_single_offset = 0,				\
+	.edge_pol_reg = 0x2c,					\
+	.pol_low_offset = 0,					\
+	.pin_sel_mask = 0xff,					\
+	.nr_channels = 20,					\
+
 #define INIT_MESON_S4_COMMON_DATA(irqs)				\
 	INIT_MESON_COMMON(irqs, meson_a1_gpio_irq_init,		\
 			  meson_a1_gpio_irq_sel_pin,		\
@@ -170,6 +188,14 @@ static const struct meson_gpio_irq_params a5_params = {
 	INIT_MESON_S4_COMMON_DATA(99)
 };
 
+static const struct meson_gpio_irq_params a9_params = {
+	INIT_MESON_S4_COMMON_DATA(96)
+};
+
+static const struct meson_gpio_irq_params a9_ao_params = {
+	INIT_MESON_A9_AO_COMMON_DATA(39)
+};
+
 static const struct meson_gpio_irq_params s4_params = {
 	INIT_MESON_S4_COMMON_DATA(82)
 };
@@ -203,6 +229,8 @@ static const struct of_device_id meson_irq_gpio_matches[] __maybe_unused = {
 	{ .compatible = "amlogic,a4-gpio-ao-intc", .data = &a4_ao_params },
 	{ .compatible = "amlogic,a4-gpio-intc", .data = &a4_params },
 	{ .compatible = "amlogic,a5-gpio-intc", .data = &a5_params },
+	{ .compatible = "amlogic,a9-gpio-ao-intc", .data = &a9_ao_params },
+	{ .compatible = "amlogic,a9-gpio-intc", .data = &a9_params },
 	{ .compatible = "amlogic,s6-gpio-intc", .data = &s6_params },
 	{ .compatible = "amlogic,s7-gpio-intc", .data = &s7_params },
 	{ .compatible = "amlogic,s7d-gpio-intc", .data = &s7_params },
@@ -375,6 +403,55 @@ static int meson8_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
 	return 0;
 }
 
+/*
+ * gpio irq relative registers for a9_ao
+ * -PADCTRL_GPIO_IRQ_CTRL0
+ * bit[31]:    enable/disable all the irq lines
+ * bit[0-19]:  polarity trigger
+ *
+ * -PADCTRL_GPIO_IRQ_CTRL[X]
+ * bit[0-5]: 6 bits to choose gpio source for irq line 2*[X] - 2
+ * bit[16-21]:6 bits to choose gpio source for irq line 2*[X] - 1
+ * where X = 1-10
+ *
+ * -PADCTRL_GPIO_IRQ_CTRL[11]
+ * bit[0-19]: both edge trigger
+ *
+ * -PADCTRL_GPIO_IRQ_CTRL[12]
+ * bit[0-19]: single edge trigger
+ */
+static int meson_a9_ao_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl,
+					 unsigned int type, u32 *channel_hwirq)
+{
+	const struct meson_gpio_irq_params *params = ctl->params;
+	unsigned int idx;
+	u32 val;
+
+	idx = meson_gpio_irq_get_channel_idx(ctl, channel_hwirq);
+
+	type &= IRQ_TYPE_SENSE_MASK;
+
+	meson_gpio_irq_update_bits(ctl, params->edge_pol_reg, BIT(idx), 0);
+
+	if (type == IRQ_TYPE_EDGE_BOTH) {
+		val = BIT(ctl->params->edge_both_offset + idx);
+		meson_gpio_irq_update_bits(ctl, params->edge_pol_reg, val, val);
+		return 0;
+	}
+
+	val = 0;
+	if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_EDGE_FALLING))
+		val = BIT(idx);
+	meson_gpio_irq_update_bits(ctl, REG_A9_AO_POL, BIT(idx), val);
+
+	val = 0;
+	if (type & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING))
+		val = BIT(idx);
+	meson_gpio_irq_update_bits(ctl, REG_A9_AO_EDGE, BIT(idx), val);
+
+	return 0;
+};
+
 /*
  * gpio irq relative registers for s4
  * -PADCTRL_GPIO_IRQ_CTRL0
diff --git a/drivers/irqchip/irq-starfive-jh8100-intc.c b/drivers/irqchip/irq-starfive-jh8100-intc.c
deleted file mode 100644
index bb62ef363d0bd..0000000000000
--- a/drivers/irqchip/irq-starfive-jh8100-intc.c
+++ /dev/null
@@ -1,207 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * StarFive JH8100 External Interrupt Controller driver
- *
- * Copyright (C) 2023 StarFive Technology Co., Ltd.
- *
- * Author: Changhuang Liang <changhuang.liang@starfivetech.com>
- */
-
-#define pr_fmt(fmt) "irq-starfive-jh8100: " fmt
-
-#include <linux/bitops.h>
-#include <linux/clk.h>
-#include <linux/irq.h>
-#include <linux/irqchip.h>
-#include <linux/irqchip/chained_irq.h>
-#include <linux/irqdomain.h>
-#include <linux/of_address.h>
-#include <linux/of_irq.h>
-#include <linux/reset.h>
-#include <linux/spinlock.h>
-
-#define STARFIVE_INTC_SRC0_CLEAR	0x10
-#define STARFIVE_INTC_SRC0_MASK		0x14
-#define STARFIVE_INTC_SRC0_INT		0x1c
-
-#define STARFIVE_INTC_SRC_IRQ_NUM	32
-
-struct starfive_irq_chip {
-	void __iomem		*base;
-	struct irq_domain	*domain;
-	raw_spinlock_t		lock;
-};
-
-static void starfive_intc_bit_set(struct starfive_irq_chip *irqc,
-				  u32 reg, u32 bit_mask)
-{
-	u32 value;
-
-	value = ioread32(irqc->base + reg);
-	value |= bit_mask;
-	iowrite32(value, irqc->base + reg);
-}
-
-static void starfive_intc_bit_clear(struct starfive_irq_chip *irqc,
-				    u32 reg, u32 bit_mask)
-{
-	u32 value;
-
-	value = ioread32(irqc->base + reg);
-	value &= ~bit_mask;
-	iowrite32(value, irqc->base + reg);
-}
-
-static void starfive_intc_unmask(struct irq_data *d)
-{
-	struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
-
-	raw_spin_lock(&irqc->lock);
-	starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC0_MASK, BIT(d->hwirq));
-	raw_spin_unlock(&irqc->lock);
-}
-
-static void starfive_intc_mask(struct irq_data *d)
-{
-	struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
-
-	raw_spin_lock(&irqc->lock);
-	starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC0_MASK, BIT(d->hwirq));
-	raw_spin_unlock(&irqc->lock);
-}
-
-static struct irq_chip intc_dev = {
-	.name		= "StarFive JH8100 INTC",
-	.irq_unmask	= starfive_intc_unmask,
-	.irq_mask	= starfive_intc_mask,
-};
-
-static int starfive_intc_map(struct irq_domain *d, unsigned int irq,
-			     irq_hw_number_t hwirq)
-{
-	irq_domain_set_info(d, irq, hwirq, &intc_dev, d->host_data,
-			    handle_level_irq, NULL, NULL);
-
-	return 0;
-}
-
-static const struct irq_domain_ops starfive_intc_domain_ops = {
-	.xlate	= irq_domain_xlate_onecell,
-	.map	= starfive_intc_map,
-};
-
-static void starfive_intc_irq_handler(struct irq_desc *desc)
-{
-	struct starfive_irq_chip *irqc = irq_data_get_irq_handler_data(&desc->irq_data);
-	struct irq_chip *chip = irq_desc_get_chip(desc);
-	unsigned long value;
-	int hwirq;
-
-	chained_irq_enter(chip, desc);
-
-	value = ioread32(irqc->base + STARFIVE_INTC_SRC0_INT);
-	while (value) {
-		hwirq = ffs(value) - 1;
-
-		generic_handle_domain_irq(irqc->domain, hwirq);
-
-		starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC0_CLEAR, BIT(hwirq));
-		starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC0_CLEAR, BIT(hwirq));
-
-		__clear_bit(hwirq, &value);
-	}
-
-	chained_irq_exit(chip, desc);
-}
-
-static int starfive_intc_probe(struct platform_device *pdev, struct device_node *parent)
-{
-	struct device_node *intc = pdev->dev.of_node;
-	struct starfive_irq_chip *irqc;
-	struct reset_control *rst;
-	struct clk *clk;
-	int parent_irq;
-	int ret;
-
-	irqc = kzalloc_obj(*irqc);
-	if (!irqc)
-		return -ENOMEM;
-
-	irqc->base = of_iomap(intc, 0);
-	if (!irqc->base) {
-		pr_err("Unable to map registers\n");
-		ret = -ENXIO;
-		goto err_free;
-	}
-
-	rst = of_reset_control_get_exclusive(intc, NULL);
-	if (IS_ERR(rst)) {
-		pr_err("Unable to get reset control %pe\n", rst);
-		ret = PTR_ERR(rst);
-		goto err_unmap;
-	}
-
-	clk = of_clk_get(intc, 0);
-	if (IS_ERR(clk)) {
-		pr_err("Unable to get clock %pe\n", clk);
-		ret = PTR_ERR(clk);
-		goto err_reset_put;
-	}
-
-	ret = reset_control_deassert(rst);
-	if (ret)
-		goto err_clk_put;
-
-	ret = clk_prepare_enable(clk);
-	if (ret)
-		goto err_reset_assert;
-
-	raw_spin_lock_init(&irqc->lock);
-
-	irqc->domain = irq_domain_create_linear(of_fwnode_handle(intc), STARFIVE_INTC_SRC_IRQ_NUM,
-						&starfive_intc_domain_ops, irqc);
-	if (!irqc->domain) {
-		pr_err("Unable to create IRQ domain\n");
-		ret = -EINVAL;
-		goto err_clk_disable;
-	}
-
-	parent_irq = of_irq_get(intc, 0);
-	if (parent_irq < 0) {
-		pr_err("Failed to get main IRQ: %d\n", parent_irq);
-		ret = parent_irq;
-		goto err_remove_domain;
-	}
-
-	irq_set_chained_handler_and_data(parent_irq, starfive_intc_irq_handler,
-					 irqc);
-
-	pr_info("Interrupt controller register, nr_irqs %d\n",
-		STARFIVE_INTC_SRC_IRQ_NUM);
-
-	return 0;
-
-err_remove_domain:
-	irq_domain_remove(irqc->domain);
-err_clk_disable:
-	clk_disable_unprepare(clk);
-err_reset_assert:
-	reset_control_assert(rst);
-err_clk_put:
-	clk_put(clk);
-err_reset_put:
-	reset_control_put(rst);
-err_unmap:
-	iounmap(irqc->base);
-err_free:
-	kfree(irqc);
-	return ret;
-}
-
-IRQCHIP_PLATFORM_DRIVER_BEGIN(starfive_intc)
-IRQCHIP_MATCH("starfive,jh8100-intc", starfive_intc_probe)
-IRQCHIP_PLATFORM_DRIVER_END(starfive_intc)
-
-MODULE_DESCRIPTION("StarFive JH8100 External Interrupt Controller");
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Changhuang Liang <changhuang.liang@starfivetech.com>");
diff --git a/drivers/irqchip/irq-starfive-jhb100-intc.c b/drivers/irqchip/irq-starfive-jhb100-intc.c
new file mode 100644
index 0000000000000..838885b02f343
--- /dev/null
+++ b/drivers/irqchip/irq-starfive-jhb100-intc.c
@@ -0,0 +1,254 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * StarFive JHB100 External Interrupt Controller driver
+ *
+ * Copyright (C) 2023 StarFive Technology Co., Ltd.
+ *
+ * Author: Changhuang Liang <changhuang.liang@starfivetech.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/cleanup.h>
+#include <linux/clk.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of_irq.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/spinlock.h>
+
+#define STARFIVE_INTC_SRC_TYPE(n)	(0x04 + ((n) * 0x20))
+#define STARFIVE_INTC_SRC_CLEAR(n)	(0x10 + ((n) * 0x20))
+#define STARFIVE_INTC_SRC_MASK(n)	(0x14 + ((n) * 0x20))
+#define STARFIVE_INTC_SRC_INT(n)	(0x1c + ((n) * 0x20))
+
+#define STARFIVE_INTC_TRIGGER_MASK	0x3
+#define STARFIVE_INTC_TRIGGER_HIGH	0
+#define STARFIVE_INTC_TRIGGER_LOW	1
+#define STARFIVE_INTC_TRIGGER_POSEDGE	2
+#define STARFIVE_INTC_TRIGGER_NEGEDGE	3
+
+#define STARFIVE_INTC_NUM		2
+#define STARFIVE_INTC_SRC_IRQ_NUM	32
+#define STARFIVE_INTC_TYPE_NUM		16
+
+struct starfive_irq_chip {
+	void __iomem		*base;
+	struct irq_domain	*domain;
+	raw_spinlock_t		lock;
+};
+
+static void starfive_intc_mod(struct starfive_irq_chip *irqc, u32 reg, u32 mask, u32 data)
+{
+	u32 value;
+
+	value = ioread32(irqc->base + reg) & ~mask;
+	data &= mask;
+	data |= value;
+	iowrite32(data, irqc->base + reg);
+}
+
+static void starfive_intc_bit_set(struct starfive_irq_chip *irqc,
+				  u32 reg, u32 bit_mask)
+{
+	u32 value;
+
+	value = ioread32(irqc->base + reg);
+	value |= bit_mask;
+	iowrite32(value, irqc->base + reg);
+}
+
+static void starfive_intc_bit_clear(struct starfive_irq_chip *irqc,
+				    u32 reg, u32 bit_mask)
+{
+	u32 value;
+
+	value = ioread32(irqc->base + reg);
+	value &= ~bit_mask;
+	iowrite32(value, irqc->base + reg);
+}
+
+static void starfive_intc_unmask(struct irq_data *d)
+{
+	struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
+	int i, bitpos;
+
+	i = d->hwirq / STARFIVE_INTC_SRC_IRQ_NUM;
+	bitpos = d->hwirq % STARFIVE_INTC_SRC_IRQ_NUM;
+
+	guard(raw_spinlock)(&irqc->lock);
+	starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC_MASK(i), BIT(bitpos));
+}
+
+static void starfive_intc_mask(struct irq_data *d)
+{
+	struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
+	int i, bitpos;
+
+	i = d->hwirq / STARFIVE_INTC_SRC_IRQ_NUM;
+	bitpos = d->hwirq % STARFIVE_INTC_SRC_IRQ_NUM;
+
+	guard(raw_spinlock)(&irqc->lock);
+	starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC_MASK(i), BIT(bitpos));
+}
+
+static void starfive_intc_ack(struct irq_data *d)
+{
+	/* for handle_edge_irq, nothing to do */
+}
+
+static int starfive_intc_set_type(struct irq_data *d, unsigned int type)
+{
+	struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d);
+	u32 i, bitpos, ty_pos, ty_shift, trigger, typeval;
+	irq_flow_handler_t handler;
+
+	i = d->hwirq / STARFIVE_INTC_SRC_IRQ_NUM;
+	bitpos = d->hwirq % STARFIVE_INTC_SRC_IRQ_NUM;
+	ty_pos = bitpos / STARFIVE_INTC_TYPE_NUM;
+	ty_shift = (bitpos % STARFIVE_INTC_TYPE_NUM) * 2;
+
+	switch (type) {
+	case IRQF_TRIGGER_LOW:
+		trigger = STARFIVE_INTC_TRIGGER_LOW;
+		handler = handle_level_irq;
+		break;
+	case IRQF_TRIGGER_HIGH:
+		trigger = STARFIVE_INTC_TRIGGER_HIGH;
+		handler = handle_level_irq;
+		break;
+	case IRQF_TRIGGER_FALLING:
+		trigger = STARFIVE_INTC_TRIGGER_NEGEDGE;
+		handler = handle_edge_irq;
+		break;
+	case IRQF_TRIGGER_RISING:
+		trigger = STARFIVE_INTC_TRIGGER_POSEDGE;
+		handler = handle_edge_irq;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	irq_set_handler_locked(d, handler);
+	typeval = trigger << ty_shift;
+
+	guard(raw_spinlock)(&irqc->lock);
+
+	starfive_intc_mod(irqc, STARFIVE_INTC_SRC_TYPE(i) + 4 * ty_pos,
+			  STARFIVE_INTC_TRIGGER_MASK << ty_shift, typeval);
+
+	/* Once the type is updated, clear interrupt can help to reset the type value */
+	starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC_CLEAR(i), BIT(bitpos));
+	starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC_CLEAR(i), BIT(bitpos));
+
+	return 0;
+}
+
+static struct irq_chip intc_dev = {
+	.name		= "StarFive JHB100 INTC",
+	.irq_unmask	= starfive_intc_unmask,
+	.irq_mask	= starfive_intc_mask,
+	.irq_ack	= starfive_intc_ack,
+	.irq_set_type	= starfive_intc_set_type,
+};
+
+static int starfive_intc_map(struct irq_domain *d, unsigned int irq,
+			     irq_hw_number_t hwirq)
+{
+	irq_domain_set_info(d, irq, hwirq, &intc_dev, d->host_data,
+			    handle_level_irq, NULL, NULL);
+
+	return 0;
+}
+
+static const struct irq_domain_ops starfive_intc_domain_ops = {
+	.xlate	= irq_domain_xlate_onecell,
+	.map	= starfive_intc_map,
+};
+
+static void starfive_intc_irq_handler(struct irq_desc *desc)
+{
+	struct starfive_irq_chip *irqc = irq_data_get_irq_handler_data(&desc->irq_data);
+	struct irq_chip *chip = irq_desc_get_chip(desc);
+	unsigned long value;
+	int hwirq;
+
+	chained_irq_enter(chip, desc);
+
+	for (int i = 0; i < STARFIVE_INTC_NUM; i++) {
+		value = ioread32(irqc->base + STARFIVE_INTC_SRC_INT(i));
+		while (value) {
+			hwirq = ffs(value) - 1;
+
+			generic_handle_domain_irq(irqc->domain,
+						  hwirq + i * STARFIVE_INTC_SRC_IRQ_NUM);
+
+			starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC_CLEAR(i), BIT(hwirq));
+			starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC_CLEAR(i), BIT(hwirq));
+
+			__clear_bit(hwirq, &value);
+		}
+	}
+
+	chained_irq_exit(chip, desc);
+}
+
+static int starfive_intc_probe(struct platform_device *pdev, struct device_node *parent)
+{
+	struct device_node *intc = pdev->dev.of_node;
+	struct reset_control *rst;
+	struct clk *clk;
+	int parent_irq;
+
+	struct starfive_irq_chip *irqc __free(kfree) = kzalloc_obj(*irqc);
+	if (!irqc)
+		return -ENOMEM;
+
+	irqc->base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(irqc->base))
+		return dev_err_probe(&pdev->dev, PTR_ERR(irqc->base), "unable to map registers\n");
+
+	rst = devm_reset_control_get_optional_exclusive_deasserted(&pdev->dev, NULL);
+	if (IS_ERR(rst))
+		return dev_err_probe(&pdev->dev, PTR_ERR(rst),
+				     "Unable to get and deassert reset control\n");
+
+	clk = devm_clk_get_optional_enabled(&pdev->dev, NULL);
+	if (IS_ERR(clk))
+		return dev_err_probe(&pdev->dev, PTR_ERR(clk), "Unable to get and enable clock\n");
+
+
+	raw_spin_lock_init(&irqc->lock);
+
+	irqc->domain = irq_domain_create_linear(of_fwnode_handle(intc),
+						STARFIVE_INTC_SRC_IRQ_NUM * STARFIVE_INTC_NUM,
+						&starfive_intc_domain_ops, irqc);
+	if (!irqc->domain)
+		return dev_err_probe(&pdev->dev, -EINVAL, "Unable to create IRQ domain\n");
+
+	parent_irq = of_irq_get(intc, 0);
+	if (parent_irq < 0) {
+		irq_domain_remove(irqc->domain);
+		return dev_err_probe(&pdev->dev, parent_irq, "Failed to get main IRQ\n");
+	}
+
+	irq_set_chained_handler_and_data(parent_irq, starfive_intc_irq_handler,
+					 irqc);
+
+	dev_info(&pdev->dev, "Interrupt controller register, nr_irqs %d\n",
+		 STARFIVE_INTC_SRC_IRQ_NUM * STARFIVE_INTC_NUM);
+
+	retain_and_null_ptr(irqc);
+	return 0;
+}
+
+IRQCHIP_PLATFORM_DRIVER_BEGIN(starfive_intc)
+IRQCHIP_MATCH("starfive,jhb100-intc", starfive_intc_probe)
+IRQCHIP_PLATFORM_DRIVER_END(starfive_intc)
+
+MODULE_DESCRIPTION("StarFive JHB100 External Interrupt Controller");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Changhuang Liang <changhuang.liang@starfivetech.com>");
diff --git a/drivers/leds/Makefile b/drivers/leds/Makefile
index 8fdb45d5b4393..7db3768912ca5 100644
--- a/drivers/leds/Makefile
+++ b/drivers/leds/Makefile
@@ -15,7 +15,7 @@ obj-$(CONFIG_LEDS_ADP5520)		+= leds-adp5520.o
 obj-$(CONFIG_LEDS_AN30259A)		+= leds-an30259a.o
 obj-$(CONFIG_LEDS_APU)			+= leds-apu.o
 obj-$(CONFIG_LEDS_ARIEL)		+= leds-ariel.o
-obj-$(CONFIG_LEDS_AS3668)		+= leds-as3668.o
+obj-$(CONFIG_LEDS_OSRAM_AMS_AS3668)	+= leds-as3668.o
 obj-$(CONFIG_LEDS_AW200XX)		+= leds-aw200xx.o
 obj-$(CONFIG_LEDS_AW2013)		+= leds-aw2013.o
 obj-$(CONFIG_LEDS_BCM6328)		+= leds-bcm6328.o
diff --git a/drivers/leds/blink/leds-bcm63138.c b/drivers/leds/blink/leds-bcm63138.c
index ef2e511438ccb..45c0662df9338 100644
--- a/drivers/leds/blink/leds-bcm63138.c
+++ b/drivers/leds/blink/leds-bcm63138.c
@@ -296,6 +296,7 @@ static const struct of_device_id bcm63138_leds_of_match_table[] = {
 	{ .compatible = "brcm,bcm63138-leds", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, bcm63138_leds_of_match_table);
 
 static struct platform_driver bcm63138_leds_driver = {
 	.probe = bcm63138_leds_probe,
@@ -310,4 +311,3 @@ module_platform_driver(bcm63138_leds_driver);
 MODULE_AUTHOR("Rafał Miłecki");
 MODULE_DESCRIPTION("Broadcom BCM63138 SoC LED driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, bcm63138_leds_of_match_table);
diff --git a/drivers/leds/flash/Kconfig b/drivers/leds/flash/Kconfig
index 5e08102a67841..31c996651f2a4 100644
--- a/drivers/leds/flash/Kconfig
+++ b/drivers/leds/flash/Kconfig
@@ -76,7 +76,7 @@ config LEDS_MT6370_FLASH
 	  will be called "leds-mt6370-flash".
 
 config LEDS_QCOM_FLASH
-	tristate "LED support for flash module inside Qualcomm Technologies, Inc. PMIC"
+	tristate "LED support for flash module inside Qualcomm PMIC"
 	depends on MFD_SPMI_PMIC || COMPILE_TEST
 	depends on LEDS_CLASS && OF
 	depends on V4L2_FLASH_LED_CLASS || !V4L2_FLASH_LED_CLASS
diff --git a/drivers/leds/led-class-multicolor.c b/drivers/leds/led-class-multicolor.c
index 6b671f3f9c617..59d46f8a90d5a 100644
--- a/drivers/leds/led-class-multicolor.c
+++ b/drivers/leds/led-class-multicolor.c
@@ -7,10 +7,29 @@
 #include <linux/init.h>
 #include <linux/led-class-multicolor.h>
 #include <linux/math.h>
+#include <linux/minmax.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 
+static unsigned int led_mc_get_max_intensity(struct led_classdev_mc *mcled_cdev, size_t index)
+{
+	unsigned int max_intensity;
+
+	/*
+	 * The maximum global brightness value might still be changed by
+	 * led_classdev_register_ext() using devicetree properties. This
+	 * prevents us from changing subled_info[X].max_intensity when
+	 * registering a multicolor LED class device, so we have to do
+	 * this during runtime.
+	 */
+	max_intensity = mcled_cdev->subled_info[index].max_intensity;
+	if (max_intensity)
+		return max_intensity;
+
+	return mcled_cdev->led_cdev.max_brightness;
+}
+
 int led_mc_calc_color_components(struct led_classdev_mc *mcled_cdev,
 				 enum led_brightness brightness)
 {
@@ -27,6 +46,26 @@ int led_mc_calc_color_components(struct led_classdev_mc *mcled_cdev,
 }
 EXPORT_SYMBOL_GPL(led_mc_calc_color_components);
 
+static ssize_t multi_max_intensity_show(struct device *dev,
+					struct device_attribute *intensity_attr, char *buf)
+{
+	struct led_classdev *led_cdev = dev_get_drvdata(dev);
+	struct led_classdev_mc *mcled_cdev = lcdev_to_mccdev(led_cdev);
+	unsigned int max_intensity;
+	int len = 0;
+
+	for (int i = 0; i < mcled_cdev->num_colors; i++) {
+		max_intensity = led_mc_get_max_intensity(mcled_cdev, i);
+		len += sysfs_emit_at(buf, len, "%u", max_intensity);
+		if (i < mcled_cdev->num_colors - 1)
+			len += sprintf(buf + len, " ");
+	}
+
+	buf[len++] = '\n';
+	return len;
+}
+static DEVICE_ATTR_RO(multi_max_intensity);
+
 static ssize_t multi_intensity_store(struct device *dev,
 				struct device_attribute *intensity_attr,
 				const char *buf, size_t size)
@@ -35,6 +74,7 @@ static ssize_t multi_intensity_store(struct device *dev,
 	struct led_classdev_mc *mcled_cdev = lcdev_to_mccdev(led_cdev);
 	int nrchars, offset = 0;
 	unsigned int intensity_value[LED_COLOR_ID_MAX];
+	unsigned int max_intensity;
 	int i;
 	ssize_t ret;
 
@@ -56,8 +96,10 @@ static ssize_t multi_intensity_store(struct device *dev,
 		goto err_out;
 	}
 
-	for (i = 0; i < mcled_cdev->num_colors; i++)
-		mcled_cdev->subled_info[i].intensity = intensity_value[i];
+	for (int i = 0; i < mcled_cdev->num_colors; i++) {
+		max_intensity = led_mc_get_max_intensity(mcled_cdev, i);
+		mcled_cdev->subled_info[i].intensity = min(intensity_value[i], max_intensity);
+	}
 
 	if (!test_bit(LED_BLINK_SW, &led_cdev->work_flags))
 		led_set_brightness(led_cdev, led_cdev->brightness);
@@ -111,6 +153,7 @@ static ssize_t multi_index_show(struct device *dev,
 static DEVICE_ATTR_RO(multi_index);
 
 static struct attribute *led_multicolor_attrs[] = {
+	&dev_attr_multi_max_intensity.attr,
 	&dev_attr_multi_intensity.attr,
 	&dev_attr_multi_index.attr,
 	NULL,
diff --git a/drivers/leds/leds-blinkm.c b/drivers/leds/leds-blinkm.c
index 577497b9d426b..1978bbda47dbc 100644
--- a/drivers/leds/leds-blinkm.c
+++ b/drivers/leds/leds-blinkm.c
@@ -356,7 +356,8 @@ static int blinkm_transfer_hw(struct i2c_client *client, int cmd)
 	struct blinkm_data *data = i2c_get_clientdata(client);
 
 	/* We start hardware transfers which are not to be
-	 * mixed with other commands. Aquire a lock now. */
+	 * mixed with other commands. Acquire a lock now.
+	 */
 	if (mutex_lock_interruptible(&data->update_lock) < 0)
 		return -EAGAIN;
 
diff --git a/drivers/leds/leds-cros_ec.c b/drivers/leds/leds-cros_ec.c
index bea3cc3fbfd2d..6592ceee866a2 100644
--- a/drivers/leds/leds-cros_ec.c
+++ b/drivers/leds/leds-cros_ec.c
@@ -249,6 +249,7 @@ static const struct platform_device_id cros_ec_led_id[] = {
 	{ "cros-ec-led", 0 },
 	{}
 };
+MODULE_DEVICE_TABLE(platform, cros_ec_led_id);
 
 static struct platform_driver cros_ec_led_driver = {
 	.driver.name	= "cros-ec-led",
@@ -257,7 +258,6 @@ static struct platform_driver cros_ec_led_driver = {
 };
 module_platform_driver(cros_ec_led_driver);
 
-MODULE_DEVICE_TABLE(platform, cros_ec_led_id);
 MODULE_DESCRIPTION("ChromeOS EC LED Driver");
 MODULE_AUTHOR("Thomas Weißschuh <linux@weissschuh.net");
 MODULE_LICENSE("GPL");
diff --git a/drivers/leds/leds-dac124s085.c b/drivers/leds/leds-dac124s085.c
index cf5fb1195f87f..192b43333aeef 100644
--- a/drivers/leds/leds-dac124s085.c
+++ b/drivers/leds/leds-dac124s085.c
@@ -35,7 +35,7 @@ static int dac124s085_set_brightness(struct led_classdev *ldev,
 {
 	struct dac124s085_led *led = container_of(ldev, struct dac124s085_led,
 						  ldev);
-	u16 word;
+	__le16 word;
 	int ret;
 
 	mutex_lock(&led->mutex);
diff --git a/drivers/leds/leds-lp50xx.c b/drivers/leds/leds-lp50xx.c
index e2a9c8592953c..69c3550f1a31c 100644
--- a/drivers/leds/leds-lp50xx.c
+++ b/drivers/leds/leds-lp50xx.c
@@ -525,6 +525,7 @@ static int lp50xx_probe_dt(struct lp50xx *priv)
 			}
 
 			mc_led_info[multi_index].color_index = color_id;
+			mc_led_info[multi_index].max_intensity = 255;
 			num_colors++;
 		}
 
diff --git a/drivers/leds/leds-pca9532.c b/drivers/leds/leds-pca9532.c
index 0344189bb991c..dae7c67605084 100644
--- a/drivers/leds/leds-pca9532.c
+++ b/drivers/leds/leds-pca9532.c
@@ -182,11 +182,13 @@ static int pca9532_set_brightness(struct led_classdev *led_cdev,
 	int err = 0;
 	struct pca9532_led *led = ldev_to_led(led_cdev);
 
-	if (value == LED_OFF)
+	if (value == LED_OFF) {
 		led->state = PCA9532_OFF;
-	else if (value == LED_FULL)
+	} else if (led->state == PCA9532_PWM1) {
+		return 0; /* Non-zero brightness shall not stop HW blinking */
+	} else if (value == LED_FULL) {
 		led->state = PCA9532_ON;
-	else {
+	} else {
 		led->state = PCA9532_PWM0; /* Thecus: hardcode one pwm */
 		err = pca9532_calcpwm(led->client, PCA9532_PWM_ID_0, 0, value);
 		if (err)
diff --git a/drivers/leds/leds-st1202.c b/drivers/leds/leds-st1202.c
index 4e5dd76d714d8..2b68cd3c45f8d 100644
--- a/drivers/leds/leds-st1202.c
+++ b/drivers/leds/leds-st1202.c
@@ -9,7 +9,6 @@
 #include <linux/ctype.h>
 #include <linux/delay.h>
 #include <linux/err.h>
-#include <linux/gpio.h>
 #include <linux/i2c.h>
 #include <linux/leds.h>
 #include <linux/module.h>
diff --git a/drivers/leds/rgb/Kconfig b/drivers/leds/rgb/Kconfig
index 28ef4c487367c..9a4ba6531cf87 100644
--- a/drivers/leds/rgb/Kconfig
+++ b/drivers/leds/rgb/Kconfig
@@ -39,6 +39,31 @@ config LEDS_LP5812
 
 	  If unsure, say N.
 
+config LEDS_LP5860_CORE
+	tristate "Core Driver for TI LP5860"
+	depends on LEDS_CLASS
+	depends on OF
+	select REGMAP
+	help
+	  This option supports common operations for LP5860 devices.
+	  The LP5860 is a LED matrix driver with 18 constant current
+	  sinks and 11 scan switches for 198 LED dots. Each dot can be
+	  controlled individually and supports 8/16-bit PWM dimming.
+	  The chip supports individual LED open and short detection.
+
+	  The device can be used with SPI or I2C bus.
+
+config LEDS_LP5860_SPI
+	tristate "LED Support for TI LP5860 SPI LED driver chip"
+	depends on SPI
+	select LEDS_LP5860_CORE
+	help
+	  If you say yes here you get support for the Texas Instruments
+	  LP5860 LED driver for SPI bus connections.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called leds-lp5860-spi.
+
 config LEDS_NCP5623
 	tristate "LED support for NCP5623"
 	depends on I2C
diff --git a/drivers/leds/rgb/Makefile b/drivers/leds/rgb/Makefile
index be45991f63f50..f3b365ea082d1 100644
--- a/drivers/leds/rgb/Makefile
+++ b/drivers/leds/rgb/Makefile
@@ -3,6 +3,8 @@
 obj-$(CONFIG_LEDS_GROUP_MULTICOLOR)	+= leds-group-multicolor.o
 obj-$(CONFIG_LEDS_KTD202X)		+= leds-ktd202x.o
 obj-$(CONFIG_LEDS_LP5812)		+= leds-lp5812.o
+obj-$(CONFIG_LEDS_LP5860_CORE)		+= leds-lp5860-core.o
+obj-$(CONFIG_LEDS_LP5860_SPI)		+= leds-lp5860-spi.o
 obj-$(CONFIG_LEDS_NCP5623)		+= leds-ncp5623.o
 obj-$(CONFIG_LEDS_PWM_MULTICOLOR)	+= leds-pwm-multicolor.o
 obj-$(CONFIG_LEDS_QCOM_LPG)		+= leds-qcom-lpg.o
diff --git a/drivers/leds/rgb/leds-lp5860-core.c b/drivers/leds/rgb/leds-lp5860-core.c
new file mode 100644
index 0000000000000..fd0e2f6e6e0f4
--- /dev/null
+++ b/drivers/leds/rgb/leds-lp5860-core.c
@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2025 Pengutronix
+ *
+ * Author: Steffen Trumtrar <kernel@pengutronix.de>
+ */
+
+#include <linux/led-class-multicolor.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of_platform.h>
+#include <linux/property.h>
+#include <linux/regmap.h>
+
+#include "leds-lp5860.h"
+
+static struct lp5860_led *mcled_cdev_to_led(struct led_classdev_mc *mc_cdev)
+{
+	return container_of(mc_cdev, struct lp5860_led, mc_cdev);
+}
+
+static int lp5860_set_dot_onoff(struct lp5860_led *led, unsigned int dot, bool enable)
+{
+	unsigned int offset = dot / LP5860_MAX_DOT_ONOFF_GROUP_NUM;
+	unsigned int mask = BIT(dot % LP5860_MAX_DOT_ONOFF_GROUP_NUM);
+
+	if (dot > LP5860_MAX_LED)
+		return -EINVAL;
+
+	return regmap_update_bits(led->chip->regmap,
+				  LP5860_REG_DOT_ONOFF_START + offset, mask,
+				  enable ? LP5860_DOT_ALL_ON : LP5860_DOT_ALL_OFF);
+}
+
+static int lp5860_set_mc_brightness(struct led_classdev *cdev,
+				    enum led_brightness brightness)
+{
+	struct led_classdev_mc *mc_cdev = lcdev_to_mccdev(cdev);
+	struct lp5860_led *led = mcled_cdev_to_led(mc_cdev);
+
+	led_mc_calc_color_components(mc_cdev, brightness);
+
+	guard(mutex)(&led->chip->lock);
+	for (int i = 0; i < led->mc_cdev.num_colors; i++) {
+		unsigned int channel = mc_cdev->subled_info[i].channel;
+		unsigned int led_brightness = mc_cdev->subled_info[i].brightness;
+		int ret;
+
+		ret = lp5860_set_dot_onoff(led, channel, !!led_brightness);
+		if (ret)
+			return ret;
+
+		ret = regmap_write(led->chip->regmap,
+				   LP5860_REG_PWM_BRI_START + channel, led_brightness);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int lp5860_chip_enable(struct lp5860 *lp, bool enable)
+{
+	guard(mutex)(&lp->lock);
+	return regmap_write(lp->regmap, LP5860_REG_CHIP_EN, enable);
+}
+
+static int lp5860_led_init(struct lp5860_led *led, struct fwnode_handle *fwnode,
+			   unsigned int channel)
+{
+	enum led_default_state default_state;
+	unsigned int brightness;
+	int ret;
+
+	guard(mutex)(&led->chip->lock);
+	ret = regmap_read(led->chip->regmap, LP5860_REG_PWM_BRI_START + channel, &brightness);
+	if (ret)
+		return ret;
+
+	default_state = led_init_default_state_get(fwnode);
+
+	switch (default_state) {
+	case LEDS_DEFSTATE_ON:
+		led->brightness = LP5860_MAX_BRIGHTNESS;
+		break;
+	case LEDS_DEFSTATE_KEEP:
+		led->brightness = min(brightness, LP5860_MAX_BRIGHTNESS);
+		break;
+	default:
+		led->brightness = 0;
+		break;
+	}
+
+	return 0;
+}
+
+static int lp5860_iterate_subleds(struct lp5860_led *led, struct led_init_data *init_data)
+{
+	struct fwnode_handle *led_node = NULL;
+	struct fwnode_handle *multi_led = init_data->fwnode;
+	int subled = 0;
+
+	fwnode_for_each_child_node(multi_led, led_node) {
+		u32 channel;
+		u32 color_index;
+		int ret;
+
+		ret = fwnode_property_read_u32(led_node, "color", &color_index);
+		if (ret) {
+			dev_err_probe(led->chip->dev, ret,
+				      "%pfwP: Cannot read 'color' property. Skipping.\n", led_node);
+			fwnode_handle_put(led_node);
+			return ret;
+		}
+
+		ret = fwnode_property_read_u32(led_node, "reg", &channel);
+		if (ret < 0 || channel > LP5860_MAX_LED) {
+			dev_err_probe(led->chip->dev, ret,
+				      "%pfwP: 'reg' property is missing. Skipping.\n", led_node);
+			fwnode_handle_put(led_node);
+			return ret;
+		}
+
+		led->mc_cdev.subled_info[subled].color_index = color_index;
+		led->mc_cdev.subled_info[subled].channel = channel;
+		ret = lp5860_led_init(led, init_data->fwnode, channel);
+		if (ret) {
+			dev_err_probe(led->chip->dev, ret,
+				      "%pfwP: Failed to init LED\n", led_node);
+			fwnode_handle_put(led_node);
+			return ret;
+		}
+
+		subled++;
+	}
+
+	return 0;
+}
+
+static int lp5860_init_dt(struct lp5860 *lp)
+{
+	struct led_init_data init_data = {};
+	struct led_classdev *led_cdev;
+	struct mc_subled *mc_led_info;
+	struct lp5860_led *led;
+	int led_index = 0;
+	int chan;
+	int ret;
+
+	device_for_each_child_node_scoped(lp->dev, multi_led) {
+		led = &lp->leds[led_index];
+
+		init_data.fwnode = multi_led;
+
+		/* Count the number of channels in this multi_led */
+		chan = fwnode_get_child_node_count(multi_led);
+		if (!chan || chan > LP5860_MAX_LED_CHANNELS)
+			return -EINVAL;
+
+		led->mc_cdev.num_colors = chan;
+
+		mc_led_info = devm_kcalloc(lp->dev, chan, sizeof(*mc_led_info), GFP_KERNEL);
+		if (!mc_led_info)
+			return -ENOMEM;
+
+		led->chip = lp;
+		led->mc_cdev.subled_info = mc_led_info;
+		led_cdev = &led->mc_cdev.led_cdev;
+		led_cdev->max_brightness = LP5860_MAX_BRIGHTNESS;
+		led_cdev->brightness_set_blocking = lp5860_set_mc_brightness;
+
+		ret = lp5860_iterate_subleds(led, &init_data);
+		if (ret)
+			continue;
+
+		ret = lp5860_set_mc_brightness(&led->mc_cdev.led_cdev, led->brightness);
+		if (ret)
+			return dev_err_probe(lp->dev, ret, "%pfwP: Failed to set Multi-Color brightness\n",
+					     multi_led);
+
+		ret = devm_led_classdev_multicolor_register_ext(lp->dev, &led->mc_cdev, &init_data);
+		if (ret)
+			return dev_err_probe(lp->dev, ret, "%pfwP: Failed to register Multi-Color LEDs\n",
+					     multi_led);
+		led_index++;
+	}
+
+	return 0;
+}
+
+int lp5860_device_init(struct device *dev)
+{
+	struct lp5860 *lp = dev_get_drvdata(dev);
+	int ret;
+
+	ret = lp5860_chip_enable(lp, LP5860_CHIP_ENABLE);
+	if (ret)
+		return ret;
+
+	/*
+	 * Set to 8-bit PWM data without VSYNC.
+	 * Data is sent out for display instantly after received.
+	 */
+	mutex_lock(&lp->lock);
+	ret = regmap_update_bits(lp->regmap, LP5860_REG_DEV_INITIAL, LP5860_MODE_MASK,
+				 LP5860_MODE_1 << LP5860_MODE_SHIFT);
+	if (ret)
+		goto err_disable;
+	mutex_unlock(&lp->lock);
+
+	ret = lp5860_init_dt(lp);
+	if (ret)
+		goto err_disable;
+
+	return 0;
+
+err_disable:
+	mutex_unlock(&lp->lock);
+	lp5860_chip_enable(lp, LP5860_CHIP_DISABLE);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(lp5860_device_init);
+
+void lp5860_device_remove(struct device *dev)
+{
+	struct lp5860 *lp = dev_get_drvdata(dev);
+
+	lp5860_chip_enable(lp, LP5860_CHIP_DISABLE);
+}
+EXPORT_SYMBOL_GPL(lp5860_device_remove);
+
+MODULE_AUTHOR("Steffen Trumtrar <kernel@pengutronix.de>");
+MODULE_DESCRIPTION("TI LP5860 RGB LED core driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/leds/rgb/leds-lp5860-spi.c b/drivers/leds/rgb/leds-lp5860-spi.c
new file mode 100644
index 0000000000000..5e0c44854a680
--- /dev/null
+++ b/drivers/leds/rgb/leds-lp5860-spi.c
@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2025 Pengutronix
+ *
+ * Author: Steffen Trumtrar <kernel@pengutronix.de>
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/spi/spi.h>
+
+#include "leds-lp5860.h"
+
+#define LP5860_SPI_WRITE_FLAG BIT(13)
+
+/*
+ * The lp5860 uses a rather uncommon SPI data format: The R/W flag is on BIT(5) in the two address
+ * bytes; BIT(4) to BIT(0) are don't care. Therefore it has 10 bits for the address and 6 bits for
+ * padding the address. The address bytes are sent MSB first. Matching the cores registers to regmap
+ * results in write_flag_mask being BIT(13).
+ */
+static const struct regmap_config lp5860_regmap_config = {
+	.name = "lp5860",
+	.reg_bits = 10,
+	.pad_bits = 6,
+	.val_bits = 8,
+	.write_flag_mask = LP5860_SPI_WRITE_FLAG,
+	.reg_format_endian = REGMAP_ENDIAN_BIG,
+	.max_register = LP5860_MAX_REG,
+};
+
+static int lp5860_probe(struct spi_device *spi)
+{
+	struct device *dev = &spi->dev;
+	struct lp5860 *lp5860;
+	unsigned int multi_leds;
+
+	multi_leds = device_get_child_node_count(dev);
+	if (!multi_leds) {
+		dev_err(dev, "LEDs are not defined in Device Tree!");
+		return -ENODEV;
+	}
+
+	if (multi_leds > LP5860_MAX_LED) {
+		dev_err(dev, "Too many LEDs specified.\n");
+		return -EINVAL;
+	}
+
+	lp5860 = devm_kzalloc(dev, struct_size(lp5860, leds, multi_leds),
+			      GFP_KERNEL);
+	if (!lp5860)
+		return -ENOMEM;
+
+	lp5860->regmap = devm_regmap_init_spi(spi, &lp5860_regmap_config);
+	if (IS_ERR(lp5860->regmap))
+		return dev_err_probe(&spi->dev, PTR_ERR(lp5860->regmap),
+				     "Failed to initialise Regmap.\n");
+
+	lp5860->dev = dev;
+	mutex_init(&lp5860->lock);
+
+	spi_set_drvdata(spi, lp5860);
+
+	return lp5860_device_init(dev);
+}
+
+static void lp5860_remove(struct spi_device *spi)
+{
+	struct lp5860 *lp5860 = spi_get_drvdata(spi);
+
+	mutex_destroy(&lp5860->lock);
+
+	lp5860_device_remove(&spi->dev);
+}
+
+static const struct of_device_id lp5860_of_match[] = {
+	{ .compatible = "ti,lp5860" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, lp5860_of_match);
+
+static struct spi_driver lp5860_driver = {
+	.driver = {
+		.name = "lp5860-spi",
+		.of_match_table = lp5860_of_match,
+	},
+	.probe	= lp5860_probe,
+	.remove = lp5860_remove,
+};
+module_spi_driver(lp5860_driver);
+
+MODULE_AUTHOR("Steffen Trumtrar <kernel@pengutronix.de>");
+MODULE_DESCRIPTION("TI LP5860 RGB LED SPI driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/leds/rgb/leds-lp5860.h b/drivers/leds/rgb/leds-lp5860.h
new file mode 100644
index 0000000000000..940be0c6e8daa
--- /dev/null
+++ b/drivers/leds/rgb/leds-lp5860.h
@@ -0,0 +1,268 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) 2025 Pengutronix
+ *
+ * Author: Steffen Trumtrar <kernel@pengutronix.de>
+ */
+
+#ifndef _DRIVERS_LEDS_RGB_LP5860_H
+#define _DRIVERS_LEDS_RGB_LP5860_H
+
+#include <linux/led-class-multicolor.h>
+#include <linux/regmap.h>
+
+#define LP5860_REG_CHIP_EN		0x00
+#define LP5860_REG_DEV_INITIAL		0x01
+#define LP5860_REG_DEV_CONFIG1		0x02
+#define LP5860_REG_DEV_CONFIG2		0x03
+#define LP5860_REG_DEV_CONFIG3		0x04
+#define LP5860_REG_GLOBAL_BRI		0x05
+#define LP5860_REG_GROUP0_BRI		0x06
+#define LP5860_REG_GROUP1_BRI		0x07
+#define LP5860_REG_GROUP2_BRI		0x08
+#define LP5860_REG_R_CURRENT_SET	0x09
+#define LP5860_REG_G_CURRENT_SET	0x0A
+#define LP5860_REG_B_CURRENT_SET	0x0B
+#define LP5860_REG_GRP_SEL_START	0x0C
+#define LP5860_REG_DOT_ONOFF_START	0x43
+#define LP5860_REG_DOT_ONOFF_MAX	0x63
+#define LP5860_REG_FAULT_STATE		0x64
+#define LP5860_REG_DOT_LOD_START	0x65
+#define LP5860_REG_DOT_LSD_START	0x86
+#define LP5860_REG_LOD_CLEAR		0xA7
+#define LP5860_REG_LSD_CLEAR		0xA8
+#define LP5860_REG_RESET		0xA9
+#define LP5860_REG_DC_START		0x0100
+#define LP5860_REG_PWM_BRI_START	0x0200
+#define LP5860_MAX_REG			0x038B
+
+/* Register chip_enable value */
+#define LP5860_CHIP_SHIFT		0
+#define LP5860_CHIP_MASK		BIT(0)
+#define LP5860_CHIP_DISABLE		false
+#define LP5860_CHIP_ENABLE		true
+
+/* Register dev_initial value */
+#define LP5860_MAX_LINE_SHIFT		3
+#define LP5860_MAX_LINE_MASK		GENMASK(6, 3)
+#define LP5860_MAX_LINE_11		0x0B
+#define LP5860_MAX_LINE_10		0x0A
+#define LP5860_MAX_LINE_9		0x09
+#define LP5860_MAX_LINE_8		0x08
+#define LP5860_MAX_LINE_7		0x07
+#define LP5860_MAX_LINE_6		0x06
+#define LP5860_MAX_LINE_5		0x05
+#define LP5860_MAX_LINE_4		0x04
+#define LP5860_MAX_LINE_3		0x03
+#define LP5860_MAX_LINE_2		0x02
+#define LP5860_MAX_LINE_1		0x01
+
+#define LP5860_MODE_SHIFT		1
+#define LP5860_MODE_MASK		GENMASK(2, 1)
+#define LP5860_MODE_3_1			0x03
+#define LP5860_MODE_3			0x02
+#define LP5860_MODE_2			0x01
+#define LP5860_MODE_1			0x00
+
+#define LP5860_PWM_FREQUENCY_SHIFT	0
+#define LP5860_PWM_FREQUENCY_MASK	BIT(0)
+#define LP5860_PWM_FREQUENCY_62_5K	0x01
+#define LP5860_PWM_FREQUENCY_125K	0x00
+
+/* Register dev_config1 value */
+#define LP5860_SW_BLK_SHIFT		3
+#define LP5860_SW_BLK_MASK		BIT(3)
+#define LP5860_SW_BLK_05US		0x01
+#define LP5860_SW_BLK_1US		0x00
+
+#define LP5860_PWM_SCALE_MODE_SHIFT	2
+#define LP5860_PWM_SCALE_MODE_MASK	BIT(2)
+#define LP5860_PWM_SCALE_EXPONENTIAL	0x01
+#define LP5860_PWM_SCALE_LINEAR		0x00
+
+#define LP5860_PWM_PHASESHIFT_SHIFT	1
+#define LP5860_PWM_PHASESHIFT_MASK	BIT(1)
+#define LP5860_PWM_PHASESHIFT_ON	0x01
+#define LP5860_PWM_PHASESHIFT_OFF	0x00
+
+#define LP5860_CS_ON_SHIFT_SHIFT	0
+#define LP5860_CS_ON_SHIFT_MASK		BIT(0)
+#define LP5860_CS_DELAY_ON		0x01
+#define LP5860_CS_DELAY_OFF		0x00
+
+/* Register dev_config2 value */
+#define LP5860_COMP_GROUP3_SHIFT	6
+#define LP5860_COMP_GROUP3_MASK		GENMASK(7, 6)
+#define LP5860_COMP_GROUP3_3CLOCK	0x03
+#define LP5860_COMP_GROUP3_2CLOCK	0x02
+#define LP5860_COMP_GROUP3_1CLOCK	0x01
+#define LP5860_COMP_GROUP3_OFF		0x00
+
+#define LP5860_COMP_GROUP2_SHIFT	4
+#define LP5860_COMP_GROUP2_MASK		GENMASK(5, 4)
+#define LP5860_COMP_GROUP2_3CLOCK	0x03
+#define LP5860_COMP_GROUP2_2CLOCK	0x02
+#define LP5860_COMP_GROUP2_1CLOCK	0x01
+#define LP5860_COMP_GROUP2_OFF		0x00
+
+#define LP5860_COMP_GROUP1_SHIFT	2
+#define LP5860_COMP_GROUP1_MASK		GENMASK(3, 2)
+#define LP5860_COMP_GROUP1_3CLOCK	0x03
+#define LP5860_COMP_GROUP1_2CLOCK	0x02
+#define LP5860_COMP_GROUP1_1CLOCK	0x01
+#define LP5860_COMP_GROUP1_OFF		0x00
+
+#define LP5860_LOD_REMOVAL_SHIFT	1
+#define LP5860_LOD_REMOVAL_MASK		BIT(1)
+#define LP5860_LOD_REMOVAL_EN		0x01
+#define LP5860_LOD_REMOVAL_OFF		0x00
+
+#define LP5860_LSD_REMOVAL_SHIFT	0
+#define LP5860_LSD_REMOVAL_MASK		BIT(0)
+#define LP5860_LSD_REMOVAL_EN		0x01
+#define LP5860_LSD_REMOVAL_OFF		0x00
+
+/* Register dev_config3 value */
+#define LP5860_DOWN_DEGHOST_SHIFT	6
+#define LP5860_DOWN_DEGHOST_MASK	GENMASK(7, 6)
+#define LP5860_DOWN_DEGHOST_STRONG	0x03
+#define LP5860_DOWN_DEGHOST_MEDIUM	0x02
+#define LP5860_DOWN_DEGHOST_WEAK	0x01
+#define LP5860_DOWN_DEGHOST_OFF		0x00
+
+#define LP5860_UP_DEGHOST_SHIFT	4
+#define LP5860_UP_DEGHOST_MASK		GENMASK(5, 4)
+#define LP5860_UP_DEGHOST_GND		0x03
+#define LP5860_UP_DEGHOST_3		0x02
+#define LP5860_UP_DEGHOST_2_5		0x01
+#define LP5860_UP_DEGHOST_2		0x00
+
+#define LP5860_MAXIMUM_CURRENT_SHIFT	1
+#define LP5860_MAXIMUM_CURRENT_MASK	GENMASK(3, 1)
+#define LP5860_MAXIMUM_CURRENT_50	0x07
+#define LP5860_MAXIMUM_CURRENT_40	0x06
+#define LP5860_MAXIMUM_CURRENT_30	0x05
+#define LP5860_MAXIMUM_CURRENT_20	0x04
+#define LP5860_MAXIMUM_CURRENT_15	0x03
+#define LP5860_MAXIMUM_CURRENT_10	0x02
+#define LP5860_MAXIMUM_CURRENT_5	0x01
+#define LP5860_MAXIMUM_CURRENT_3	0x00
+
+#define LP5860_UP_DEGHOST_ENABLE_SHIFT	0
+#define LP5860_UP_DEGHOST_ENABLE_MASK	BIT(0)
+#define LP5860_UP_DEGHOST_ENABLE_EN	0x01
+#define LP5860_UP_DEGHOST_ENABLE_OFF	0x00
+
+/* Register PWM */
+#define LP5860_PWM_GLOBAL_MAX		0xff
+#define LP5860_PWM_GROUP_MAX		0xff
+
+/* Register CC group select */
+#define LP5860_CC_GROUP_MASK		GENMASK(7, 0)
+#define LP5860_CC_GROUP_MAX		0x7F
+
+/* Register dot group select */
+#define LP5860_DOT_0_SHIFT		0
+#define LP5860_DOT_1_SHIFT		2
+#define LP5860_DOT_2_SHIFT		4
+#define LP5860_DOT_3_SHIFT		6
+
+#define LP5860_DOT_GROUP3		0x03
+#define LP5860_DOT_GROUP2		0x02
+#define LP5860_DOT_GROUP1		0x01
+#define LP5860_DOT_GROUP_NONE		0x00
+
+#define LP5860_DOT_ALL_ON		0xff
+#define LP5860_DOT_ALL_OFF		0x0
+#define LP5860_PWM_DOT_MAX		0xff
+/* Dot onoff value */
+#define LP5860_DOT_CS0_SHIFT		0
+#define LP5860_DOT_CS1_SHIFT		1
+#define LP5860_DOT_CS2_SHIFT		2
+#define LP5860_DOT_CS3_SHIFT		3
+#define LP5860_DOT_CS4_SHIFT		4
+#define LP5860_DOT_CS5_SHIFT		5
+#define LP5860_DOT_CS6_SHIFT		6
+#define LP5860_DOT_CS7_SHIFT		7
+
+#define LP5860_DOT_CS_ON		0x01
+#define LP5860_DOT_CS_OFF		0x00
+
+/* Dot lod value */
+#define LP5860_DOT_LOD0_SHIFT		0
+#define LP5860_DOT_LOD1_SHIFT		1
+#define LP5860_DOT_LOD2_SHIFT		2
+#define LP5860_DOT_LOD3_SHIFT		3
+#define LP5860_DOT_LOD4_SHIFT		4
+#define LP5860_DOT_LOD5_SHIFT		5
+#define LP5860_DOT_LOD6_SHIFT		6
+#define LP5860_DOT_LOD7_SHIFT		7
+
+#define LP5860_DOT_LOD_ON		0x01
+#define LP5860_DOT_LOD_OFF		0x00
+
+/* dot lsd value */
+#define LP5860_DOT_LSD0_SHIFT		0
+#define LP5860_DOT_LSD1_SHIFT		1
+#define LP5860_DOT_LSD2_SHIFT		2
+#define LP5860_DOT_LSD3_SHIFT		3
+#define LP5860_DOT_LSD4_SHIFT		4
+#define LP5860_DOT_LSD5_SHIFT		5
+#define LP5860_DOT_LSD6_SHIFT		6
+#define LP5860_DOT_LSD7_SHIFT		7
+
+#define LP5860_DOT_LSD_ON		0x01
+#define LP5860_DOT_LSD_OFF		0x00
+
+/* Register lod state */
+#define LP5860_GLOBAL_LOD_SHIFT	1
+#define LP5860_GLOBAL_LOD_STATE		BIT(1)
+#define LP5860_GLOBAL_LSD_SHIFT	0
+#define LP5860_GLOBAL_LSD_STATE		BIT(0)
+
+#define LP5860_FAULT_STATE_ON		0x01
+#define LP5860_FAULT_STATE_OFF		0x00
+
+#define LP5860_GLOBAL_LOD_CLEAR		0x00
+#define LP5860_GLOBAL_LSD_CLEAR		0x00
+
+
+#define LP5860_LOD_CLEAR_EN		0xff
+#define LP5860_LSD_CLEAR_EN		0xff
+#define LP5860_RESET_EN			0xff
+
+#define LP5860_MAX_BRIGHTNESS		255
+#define LP5860_REG_R_PWM		0x0
+#define LP5860_REG_G_PWM		0x1
+#define LP5860_REG_B_PWM		0x2
+
+#define LP5860_MAX_LED_CONSTANT		18
+#define LP5860_MAX_LED_SCAN		11
+#define LP5860_MAX_LED			(LP5860_MAX_LED_CONSTANT * LP5860_MAX_LED_SCAN)
+
+#define LP5860_MAX_DOT_ONOFF_GROUP_NUM	8
+
+/*
+ * Theoretically, there is no max channel per LED,
+ * limit this to a reasonable value for RGBW LEDs
+ */
+#define LP5860_MAX_LED_CHANNELS		4
+
+struct lp5860_led {
+	struct lp5860 *chip;
+	struct led_classdev_mc mc_cdev;
+	u8 brightness;
+};
+
+struct lp5860 {
+	struct device *dev;
+	struct regmap *regmap;
+	struct mutex lock;
+
+	DECLARE_FLEX_ARRAY(struct lp5860_led, leds);
+};
+
+int lp5860_device_init(struct device *dev);
+void lp5860_device_remove(struct device *dev);
+
+#endif /* _DRIVERS_LEDS_RGB_LP5860_H */
diff --git a/drivers/leds/rgb/leds-ncp5623.c b/drivers/leds/rgb/leds-ncp5623.c
index 85d6be6fff2bc..f2528f06507dd 100644
--- a/drivers/leds/rgb/leds-ncp5623.c
+++ b/drivers/leds/rgb/leds-ncp5623.c
@@ -56,8 +56,7 @@ static int ncp5623_brightness_set(struct led_classdev *cdev,
 	for (int i = 0; i < mc_cdev->num_colors; i++) {
 		ret = ncp5623_write(ncp->client,
 				    NCP5623_PWM_REG(mc_cdev->subled_info[i].channel),
-				    min(mc_cdev->subled_info[i].intensity,
-					NCP5623_MAX_BRIGHTNESS));
+				    mc_cdev->subled_info[i].intensity);
 		if (ret)
 			return ret;
 	}
@@ -190,6 +189,7 @@ static int ncp5623_probe(struct i2c_client *client)
 			goto release_led_node;
 
 		subled_info[ncp->mc_dev.num_colors].channel = reg;
+		subled_info[ncp->mc_dev.num_colors].max_intensity = NCP5623_MAX_BRIGHTNESS;
 		subled_info[ncp->mc_dev.num_colors++].color_index = color_index;
 	}
 
diff --git a/drivers/leds/rgb/leds-qcom-lpg.c b/drivers/leds/rgb/leds-qcom-lpg.c
index f6061c47f863d..d7d6518de30f2 100644
--- a/drivers/leds/rgb/leds-qcom-lpg.c
+++ b/drivers/leds/rgb/leds-qcom-lpg.c
@@ -80,58 +80,8 @@
 #define SDAM_PAUSE_HI_MULTIPLIER_OFFSET	0x8
 #define SDAM_PAUSE_LO_MULTIPLIER_OFFSET	0x9
 
-struct lpg_channel;
 struct lpg_data;
 
-/**
- * struct lpg - LPG device context
- * @dev:	pointer to LPG device
- * @map:	regmap for register access
- * @lock:	used to synchronize LED and pwm callback requests
- * @pwm:	PWM-chip object, if operating in PWM mode
- * @data:	reference to version specific data
- * @lut_base:	base address of the LUT block (optional)
- * @lut_size:	number of entries in the LUT block
- * @lut_bitmap:	allocation bitmap for LUT entries
- * @pbs_dev:	PBS device
- * @lpg_chan_sdam:	LPG SDAM peripheral device
- * @lut_sdam:	LUT SDAM peripheral device
- * @pbs_en_bitmap:	bitmap for tracking PBS triggers
- * @triled_base: base address of the TRILED block (optional)
- * @triled_src:	power-source for the TRILED
- * @triled_has_atc_ctl:	true if there is TRI_LED_ATC_CTL register
- * @triled_has_src_sel:	true if there is TRI_LED_SRC_SEL register
- * @channels:	list of PWM channels
- * @num_channels: number of @channels
- */
-struct lpg {
-	struct device *dev;
-	struct regmap *map;
-
-	struct mutex lock;
-
-	struct pwm_chip *pwm;
-
-	const struct lpg_data *data;
-
-	u32 lut_base;
-	u32 lut_size;
-	unsigned long *lut_bitmap;
-
-	struct pbs_dev *pbs_dev;
-	struct nvmem_device *lpg_chan_sdam;
-	struct nvmem_device *lut_sdam;
-	unsigned long pbs_en_bitmap;
-
-	u32 triled_base;
-	u32 triled_src;
-	bool triled_has_atc_ctl;
-	bool triled_has_src_sel;
-
-	struct lpg_channel *channels;
-	unsigned int num_channels;
-};
-
 /**
  * struct lpg_channel - per channel data
  * @lpg:	reference to parent lpg
@@ -203,8 +153,8 @@ struct lpg_channel {
  * @lpg:		lpg context reference
  * @cdev:		LED class device
  * @mcdev:		Multicolor LED class device
- * @num_channels:	number of @channels
  * @channels:		list of channels associated with the LED
+ * @num_channels:	number of @channels
  */
 struct lpg_led {
 	struct lpg *lpg;
@@ -216,6 +166,55 @@ struct lpg_led {
 	struct lpg_channel *channels[] __counted_by(num_channels);
 };
 
+/**
+ * struct lpg - LPG device context
+ * @dev:	pointer to LPG device
+ * @map:	regmap for register access
+ * @lock:	used to synchronize LED and pwm callback requests
+ * @pwm:	PWM-chip object, if operating in PWM mode
+ * @data:	reference to version specific data
+ * @lut_base:	base address of the LUT block (optional)
+ * @lut_size:	number of entries in the LUT block
+ * @lut_bitmap:	allocation bitmap for LUT entries
+ * @pbs_dev:	PBS device
+ * @lpg_chan_sdam:	LPG SDAM peripheral device
+ * @lut_sdam:	LUT SDAM peripheral device
+ * @pbs_en_bitmap:	bitmap for tracking PBS triggers
+ * @triled_base: base address of the TRILED block (optional)
+ * @triled_src:	power-source for the TRILED
+ * @triled_has_atc_ctl:	true if there is TRI_LED_ATC_CTL register
+ * @triled_has_src_sel:	true if there is TRI_LED_SRC_SEL register
+ * @num_channels: number of @channels
+ * @channels:	list of PWM channels
+ */
+struct lpg {
+	struct device *dev;
+	struct regmap *map;
+
+	struct mutex lock;
+
+	struct pwm_chip *pwm;
+
+	const struct lpg_data *data;
+
+	u32 lut_base;
+	u32 lut_size;
+	unsigned long *lut_bitmap;
+
+	struct pbs_dev *pbs_dev;
+	struct nvmem_device *lpg_chan_sdam;
+	struct nvmem_device *lut_sdam;
+	unsigned long pbs_en_bitmap;
+
+	u32 triled_base;
+	u32 triled_src;
+	bool triled_has_atc_ctl;
+	bool triled_has_src_sel;
+
+	unsigned int num_channels;
+	struct lpg_channel channels[] __counted_by(num_channels);
+};
+
 /**
  * struct lpg_channel_data - per channel initialization data
  * @sdam_offset:	Channel offset in LPG SDAM
@@ -1475,12 +1474,6 @@ static int lpg_init_channels(struct lpg *lpg)
 	struct lpg_channel *chan;
 	int i;
 
-	lpg->num_channels = data->num_channels;
-	lpg->channels = devm_kcalloc(lpg->dev, data->num_channels,
-				     sizeof(struct lpg_channel), GFP_KERNEL);
-	if (!lpg->channels)
-		return -ENOMEM;
-
 	for (i = 0; i < data->num_channels; i++) {
 		chan = &lpg->channels[i];
 
@@ -1603,18 +1596,21 @@ static int lpg_init_sdam(struct lpg *lpg)
 
 static int lpg_probe(struct platform_device *pdev)
 {
+	const struct lpg_data *data;
 	struct lpg *lpg;
 	int ret;
 	int i;
 
-	lpg = devm_kzalloc(&pdev->dev, sizeof(*lpg), GFP_KERNEL);
+	data = of_device_get_match_data(&pdev->dev);
+	if (!data)
+		return -EINVAL;
+
+	lpg = devm_kzalloc(&pdev->dev, struct_size(lpg, channels, data->num_channels), GFP_KERNEL);
 	if (!lpg)
 		return -ENOMEM;
 
-	lpg->data = of_device_get_match_data(&pdev->dev);
-	if (!lpg->data)
-		return -EINVAL;
-
+	lpg->num_channels = data->num_channels;
+	lpg->data = data;
 	lpg->dev = &pdev->dev;
 	mutex_init(&lpg->lock);
 
diff --git a/drivers/leds/trigger/ledtrig-gpio.c b/drivers/leds/trigger/ledtrig-gpio.c
index fc911dfec0ef9..3b3e6869e8561 100644
--- a/drivers/leds/trigger/ledtrig-gpio.c
+++ b/drivers/leds/trigger/ledtrig-gpio.c
@@ -86,7 +86,8 @@ static int gpio_trig_activate(struct led_classdev *led)
 	 * The generic property "trigger-sources" is followed,
 	 * and we hope that this is a GPIO.
 	 */
-	gpio_data->gpiod = gpiod_get_optional(dev, "trigger-sources", GPIOD_IN);
+	gpio_data->gpiod = gpiod_get_optional(dev, "trigger-sources",
+					      GPIOD_IN | GPIOD_FLAGS_BIT_NONEXCLUSIVE);
 	if (IS_ERR(gpio_data->gpiod)) {
 		ret = PTR_ERR(gpio_data->gpiod);
 		kfree(gpio_data);
diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig
index 5bf4155b090a4..3062ee352f78f 100644
--- a/drivers/mailbox/Kconfig
+++ b/drivers/mailbox/Kconfig
@@ -341,7 +341,7 @@ config SPRD_MBOX
 	  you want to build the Spreatrum mailbox controller driver.
 
 config QCOM_CPUCP_MBOX
-	tristate "Qualcomm Technologies, Inc. CPUCP mailbox driver"
+	tristate "Qualcomm CPUCP mailbox driver"
 	depends on (ARCH_QCOM || COMPILE_TEST) && 64BIT
 	help
 	  Qualcomm Technologies, Inc. CPUSS Control Processor (CPUCP) mailbox
@@ -349,7 +349,7 @@ config QCOM_CPUCP_MBOX
 	  Y here if you want to build this driver.
 
 config QCOM_IPCC
-	tristate "Qualcomm Technologies, Inc. IPCC driver"
+	tristate "Qualcomm IPCC driver"
 	depends on ARCH_QCOM || COMPILE_TEST
 	help
 	  Qualcomm Technologies, Inc. Inter-Processor Communication Controller
diff --git a/drivers/mailbox/exynos-mailbox.c b/drivers/mailbox/exynos-mailbox.c
index d2355b128ba43..fa02f18948cfb 100644
--- a/drivers/mailbox/exynos-mailbox.c
+++ b/drivers/mailbox/exynos-mailbox.c
@@ -16,15 +16,8 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 
-#define EXYNOS_MBOX_MCUCTRL		0x0	/* Mailbox Control Register */
-#define EXYNOS_MBOX_INTCR0		0x24	/* Interrupt Clear Register 0 */
 #define EXYNOS_MBOX_INTMR0		0x28	/* Interrupt Mask Register 0 */
-#define EXYNOS_MBOX_INTSR0		0x2c	/* Interrupt Status Register 0 */
-#define EXYNOS_MBOX_INTMSR0		0x30	/* Interrupt Mask Status Register 0 */
 #define EXYNOS_MBOX_INTGR1		0x40	/* Interrupt Generation Register 1 */
-#define EXYNOS_MBOX_INTMR1		0x48	/* Interrupt Mask Register 1 */
-#define EXYNOS_MBOX_INTSR1		0x4c	/* Interrupt Status Register 1 */
-#define EXYNOS_MBOX_INTMSR1		0x50	/* Interrupt Mask Status Register 1 */
 
 #define EXYNOS_MBOX_INTMR0_MASK		GENMASK(15, 0)
 #define EXYNOS_MBOX_INTGR1_MASK		GENMASK(15, 0)
diff --git a/drivers/mailbox/mailbox-mpfs.c b/drivers/mailbox/mailbox-mpfs.c
index d5d9effece979..ef40fe2be30d6 100644
--- a/drivers/mailbox/mailbox-mpfs.c
+++ b/drivers/mailbox/mailbox-mpfs.c
@@ -201,7 +201,7 @@ static irqreturn_t mpfs_mbox_inbox_isr(int irq, void *data)
 	struct mbox_chan *chan = data;
 	struct mpfs_mbox *mbox = (struct mpfs_mbox *)chan->con_priv;
 
-	if (mbox->control_scb)
+	if (mbox->sysreg_scb)
 		regmap_write(mbox->sysreg_scb, MESSAGE_INT_OFFSET, 0);
 	else
 		writel_relaxed(0, mbox->int_reg);
diff --git a/drivers/mailbox/mailbox.c b/drivers/mailbox/mailbox.c
index bbc9fd75a95f7..efacd24a085d1 100644
--- a/drivers/mailbox/mailbox.c
+++ b/drivers/mailbox/mailbox.c
@@ -7,6 +7,7 @@
  */
 
 #include <linux/cleanup.h>
+#include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/err.h>
@@ -16,6 +17,7 @@
 #include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/property.h>
+#include <linux/seq_file.h>
 #include <linux/spinlock.h>
 
 static LIST_HEAD(mbox_cons);
@@ -98,8 +100,10 @@ static void tx_tick(struct mbox_chan *chan, int r)
 	if (chan->cl->tx_done)
 		chan->cl->tx_done(chan->cl, mssg, r);
 
-	if (r != -ETIME && chan->cl->tx_block)
+	if (r != -ETIME && chan->cl->tx_block) {
+		chan->tx_status = r;
 		complete(&chan->tx_complete);
+	}
 }
 
 static enum hrtimer_restart txdone_hrtimer(struct hrtimer *hrtimer)
@@ -258,6 +262,10 @@ EXPORT_SYMBOL_GPL(mbox_chan_tx_slots_available);
  * over the chan, i.e, tx_done() is made.
  * This function could be called from atomic context as it simply
  * queues the data and returns a token against the request.
+ *  In blocking mode, it is caller's responsibility to serialize threads'
+ * access to a channel if multi-threads are to send messages through the
+ * same channel, i.e. caller should not call this function until any
+ * previous call returns.
  *
  * Return: Non-negative integer for successful submission (non-blocking mode)
  *	or transmission over chan (blocking mode).
@@ -291,6 +299,8 @@ int mbox_send_message(struct mbox_chan *chan, void *mssg)
 		if (ret == 0) {
 			t = -ETIME;
 			tx_tick(chan, t);
+		} else if (chan->tx_status < 0) {
+			t = chan->tx_status;
 		}
 	}
 
@@ -327,6 +337,19 @@ int mbox_flush(struct mbox_chan *chan, unsigned long timeout)
 }
 EXPORT_SYMBOL_GPL(mbox_flush);
 
+static void mbox_clean_and_put_channel(struct mbox_chan *chan)
+{
+	/* The queued TX requests are simply aborted, no callbacks are made */
+	scoped_guard(spinlock_irqsave, &chan->lock) {
+		chan->cl = NULL;
+		chan->active_req = MBOX_NO_MSG;
+		if (chan->txdone_method == MBOX_TXDONE_BY_ACK)
+			chan->txdone_method = MBOX_TXDONE_BY_POLL;
+	}
+
+	module_put(chan->mbox->dev->driver->owner);
+}
+
 static int __mbox_bind_client(struct mbox_chan *chan, struct mbox_client *cl)
 {
 	struct device *dev = cl->dev;
@@ -350,10 +373,9 @@ static int __mbox_bind_client(struct mbox_chan *chan, struct mbox_client *cl)
 
 	if (chan->mbox->ops->startup) {
 		ret = chan->mbox->ops->startup(chan);
-
 		if (ret) {
 			dev_err(dev, "Unable to startup the chan (%d)\n", ret);
-			mbox_free_channel(chan);
+			mbox_clean_and_put_channel(chan);
 			return ret;
 		}
 	}
@@ -495,15 +517,7 @@ void mbox_free_channel(struct mbox_chan *chan)
 	if (chan->mbox->ops->shutdown)
 		chan->mbox->ops->shutdown(chan);
 
-	/* The queued TX requests are simply aborted, no callbacks are made */
-	scoped_guard(spinlock_irqsave, &chan->lock) {
-		chan->cl = NULL;
-		chan->active_req = MBOX_NO_MSG;
-		if (chan->txdone_method == MBOX_TXDONE_BY_ACK)
-			chan->txdone_method = MBOX_TXDONE_BY_POLL;
-	}
-
-	module_put(chan->mbox->dev->driver->owner);
+	mbox_clean_and_put_channel(chan);
 }
 EXPORT_SYMBOL_GPL(mbox_free_channel);
 
@@ -632,3 +646,66 @@ int devm_mbox_controller_register(struct device *dev,
 	return 0;
 }
 EXPORT_SYMBOL_GPL(devm_mbox_controller_register);
+
+#ifdef CONFIG_DEBUG_FS
+static void *mbox_seq_start(struct seq_file *s, loff_t *pos)
+{
+	mutex_lock(&con_mutex);
+	return seq_list_start(&mbox_cons, *pos);
+}
+
+static void *mbox_seq_next(struct seq_file *s, void *v, loff_t *pos)
+{
+	return seq_list_next(v, &mbox_cons, pos);
+}
+
+static void mbox_seq_stop(struct seq_file *s, void *v)
+{
+	mutex_unlock(&con_mutex);
+}
+
+static int mbox_seq_show(struct seq_file *seq, void *v)
+{
+	const struct mbox_controller *mbox = list_entry(v, struct mbox_controller, node);
+
+	seq_printf(seq, "%s:\n", dev_name(mbox->dev));
+
+	for (unsigned int i = 0; i < mbox->num_chans; i++) {
+		struct mbox_chan *chan = &mbox->chans[i];
+
+		scoped_guard(spinlock_irqsave, &chan->lock) {
+			if (chan->cl) {
+				struct device *cl_dev = chan->cl->dev;
+
+				seq_printf(seq, " %3u: %s\n", i,
+					   cl_dev ? dev_name(cl_dev) : "NULL device");
+			}
+		}
+	}
+
+	return 0;
+}
+
+static const struct seq_operations mbox_sops = {
+	.start = mbox_seq_start,
+	.next = mbox_seq_next,
+	.stop = mbox_seq_stop,
+	.show = mbox_seq_show,
+};
+DEFINE_SEQ_ATTRIBUTE(mbox);
+
+/*
+ * subsys_initcall() is used here but controllers may already have been
+ * registered earlier or will be later. The rationale is that debugfs is
+ * accessed only late, i.e. from userspace. So, files created here must make no
+ * assumptions about initcall ordering.
+ */
+static int __init mbox_init(void)
+{
+	struct dentry *mbox_debugfs = debugfs_create_dir("mailbox", NULL);
+
+	debugfs_create_file("mailbox_summary", 0444, mbox_debugfs, NULL, &mbox_fops);
+	return 0;
+}
+subsys_initcall(mbox_init);
+#endif	/* DEBUG_FS */
diff --git a/drivers/mailbox/mtk-adsp-mailbox.c b/drivers/mailbox/mtk-adsp-mailbox.c
index 91487aa4d7da0..8bcecddee0eb5 100644
--- a/drivers/mailbox/mtk-adsp-mailbox.c
+++ b/drivers/mailbox/mtk-adsp-mailbox.c
@@ -19,6 +19,7 @@ struct mtk_adsp_mbox_priv {
 	struct mbox_controller mbox;
 	void __iomem *va_mboxreg;
 	const struct mtk_adsp_mbox_cfg *cfg;
+	int irq;
 };
 
 struct mtk_adsp_mbox_cfg {
@@ -67,6 +68,8 @@ static int mtk_adsp_mbox_startup(struct mbox_chan *chan)
 	writel(0xFFFFFFFF, priv->va_mboxreg + priv->cfg->clr_in);
 	writel(0xFFFFFFFF, priv->va_mboxreg + priv->cfg->clr_out);
 
+	enable_irq(priv->irq);
+
 	return 0;
 }
 
@@ -74,6 +77,8 @@ static void mtk_adsp_mbox_shutdown(struct mbox_chan *chan)
 {
 	struct mtk_adsp_mbox_priv *priv = get_mtk_adsp_mbox_priv(chan->mbox);
 
+	disable_irq(priv->irq);
+
 	/* Clear ADSP mbox command */
 	writel(0xFFFFFFFF, priv->va_mboxreg + priv->cfg->clr_in);
 	writel(0xFFFFFFFF, priv->va_mboxreg + priv->cfg->clr_out);
@@ -139,8 +144,10 @@ static int mtk_adsp_mbox_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
+	priv->irq = irq;
 	ret = devm_request_threaded_irq(dev, irq, mtk_adsp_mbox_irq,
-					mtk_adsp_mbox_isr, IRQF_TRIGGER_NONE,
+					mtk_adsp_mbox_isr,
+					IRQF_TRIGGER_NONE | IRQF_NO_AUTOEN,
 					dev_name(dev), mbox->chans);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/mailbox/qcom-cpucp-mbox.c b/drivers/mailbox/qcom-cpucp-mbox.c
index 44f4ed15f8184..862e45e8fbd58 100644
--- a/drivers/mailbox/qcom-cpucp-mbox.c
+++ b/drivers/mailbox/qcom-cpucp-mbox.c
@@ -12,7 +12,6 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 
-#define APSS_CPUCP_IPC_CHAN_SUPPORTED		3
 #define APSS_CPUCP_MBOX_CMD_OFF			0x4
 
 /* Tx Registers */
@@ -26,6 +25,14 @@
 #define APSS_CPUCP_RX_MBOX_EN			0x4c00
 #define APSS_CPUCP_RX_MBOX_CMD_MASK		GENMASK_ULL(63, 0)
 
+/**
+ * struct qcom_cpucp_mbox_data - Per-hardware mailbox configuration data
+ * @num_chans:			Number of IPC channels supported by this hardware
+ */
+struct qcom_cpucp_mbox_data {
+	int num_chans;
+};
+
 /**
  * struct qcom_cpucp_mbox - Holder for the mailbox driver
  * @chans:			The mailbox channel
@@ -34,7 +41,7 @@
  * @rx_base:			Base address of the CPUCP rx registers
  */
 struct qcom_cpucp_mbox {
-	struct mbox_chan chans[APSS_CPUCP_IPC_CHAN_SUPPORTED];
+	struct mbox_chan *chans;
 	struct mbox_controller mbox;
 	void __iomem *tx_base;
 	void __iomem *rx_base;
@@ -53,7 +60,7 @@ static irqreturn_t qcom_cpucp_mbox_irq_fn(int irq, void *data)
 
 	status = readq(cpucp->rx_base + APSS_CPUCP_RX_MBOX_STAT);
 
-	for_each_set_bit(i, (unsigned long *)&status, APSS_CPUCP_IPC_CHAN_SUPPORTED) {
+	for_each_set_bit(i, (unsigned long *)&status, cpucp->mbox.num_chans) {
 		u32 val = readl(cpucp->rx_base + APSS_CPUCP_RX_MBOX_CMD(i) + APSS_CPUCP_MBOX_CMD_OFF);
 		struct mbox_chan *chan = &cpucp->chans[i];
 		unsigned long flags;
@@ -112,15 +119,24 @@ static const struct mbox_chan_ops qcom_cpucp_mbox_chan_ops = {
 
 static int qcom_cpucp_mbox_probe(struct platform_device *pdev)
 {
+	const struct qcom_cpucp_mbox_data *data;
 	struct device *dev = &pdev->dev;
 	struct qcom_cpucp_mbox *cpucp;
 	struct mbox_controller *mbox;
 	int irq, ret;
 
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return dev_err_probe(dev, -EINVAL, "No match data found\n");
+
 	cpucp = devm_kzalloc(dev, sizeof(*cpucp), GFP_KERNEL);
 	if (!cpucp)
 		return -ENOMEM;
 
+	cpucp->chans = devm_kcalloc(dev, data->num_chans, sizeof(*cpucp->chans), GFP_KERNEL);
+	if (!cpucp->chans)
+		return -ENOMEM;
+
 	cpucp->rx_base = devm_of_iomap(dev, dev->of_node, 0, NULL);
 	if (IS_ERR(cpucp->rx_base))
 		return PTR_ERR(cpucp->rx_base);
@@ -146,7 +162,7 @@ static int qcom_cpucp_mbox_probe(struct platform_device *pdev)
 
 	mbox = &cpucp->mbox;
 	mbox->dev = dev;
-	mbox->num_chans = APSS_CPUCP_IPC_CHAN_SUPPORTED;
+	mbox->num_chans = data->num_chans;
 	mbox->chans = cpucp->chans;
 	mbox->ops = &qcom_cpucp_mbox_chan_ops;
 
@@ -157,8 +173,17 @@ static int qcom_cpucp_mbox_probe(struct platform_device *pdev)
 	return 0;
 }
 
+static const struct qcom_cpucp_mbox_data qcom_x1e80100_mbox_data = {
+	.num_chans = 3,
+};
+
+static const struct qcom_cpucp_mbox_data qcom_nord_mbox_data = {
+	.num_chans = 16,
+};
+
 static const struct of_device_id qcom_cpucp_mbox_of_match[] = {
-	{ .compatible = "qcom,x1e80100-cpucp-mbox" },
+	{ .compatible = "qcom,nord-cpucp-mbox", .data = &qcom_nord_mbox_data },
+	{ .compatible = "qcom,x1e80100-cpucp-mbox", .data = &qcom_x1e80100_mbox_data },
 	{}
 };
 MODULE_DEVICE_TABLE(of, qcom_cpucp_mbox_of_match);
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index a3fcdca7e6db3..df27c7d066d23 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -315,6 +315,17 @@ config DM_CRYPT
 
 	  If unsure, say N.
 
+config DM_INLINECRYPT
+	tristate "Inline encryption target support"
+	depends on BLK_DEV_DM
+	depends on (KEYS || KEYS=n)
+	depends on BLK_INLINE_ENCRYPTION
+	help
+	  This device-mapper target is similar to dm-crypt, but it uses the
+	  blk-crypto API instead of the regular crypto API. This allows it to
+	  take advantage of inline encryption hardware such as that commonly
+	  built into UFS host controllers.
+
 config DM_SNAPSHOT
        tristate "Snapshot target"
        depends on BLK_DEV_DM
diff --git a/drivers/md/Makefile b/drivers/md/Makefile
index c338cc6fbe2eb..517d1f7d82888 100644
--- a/drivers/md/Makefile
+++ b/drivers/md/Makefile
@@ -55,6 +55,7 @@ obj-$(CONFIG_DM_UNSTRIPED)	+= dm-unstripe.o
 obj-$(CONFIG_DM_BUFIO)		+= dm-bufio.o
 obj-$(CONFIG_DM_BIO_PRISON)	+= dm-bio-prison.o
 obj-$(CONFIG_DM_CRYPT)		+= dm-crypt.o
+obj-$(CONFIG_DM_INLINECRYPT)	+= dm-inlinecrypt.o
 obj-$(CONFIG_DM_DELAY)		+= dm-delay.o
 obj-$(CONFIG_DM_DUST)		+= dm-dust.o
 obj-$(CONFIG_DM_FLAKEY)		+= dm-flakey.o
diff --git a/drivers/md/dm-ima.c b/drivers/md/dm-ima.c
index 9495ca0350563..5e7232582feb7 100644
--- a/drivers/md/dm-ima.c
+++ b/drivers/md/dm-ima.c
@@ -21,25 +21,32 @@
  * character, so that they don't interfere with the construction of key-value pairs,
  * and clients can split the key1=val1,key2=val2,key3=val3; pairs properly.
  */
-static void fix_separator_chars(char **buf)
+static void fix_separator_chars(char *buf)
 {
-	int l = strlen(*buf);
+	int l = strlen(buf);
 	int i, j, sp = 0;
 
 	for (i = 0; i < l; i++)
-		if ((*buf)[i] == '\\' || (*buf)[i] == ';' || (*buf)[i] == '=' || (*buf)[i] == ',')
+		if (buf[i] == '\\' || buf[i] == ';' || buf[i] == '=' || buf[i] == ',')
 			sp++;
 
 	if (!sp)
 		return;
 
+	buf[l + sp] = '\0';
 	for (i = l-1, j = i+sp; i >= 0; i--) {
-		(*buf)[j--] = (*buf)[i];
-		if ((*buf)[i] == '\\' || (*buf)[i] == ';' || (*buf)[i] == '=' || (*buf)[i] == ',')
-			(*buf)[j--] = '\\';
+		buf[j--] = buf[i];
+		if (buf[i] == '\\' || buf[i] == ';' || buf[i] == '=' || buf[i] == ',')
+			buf[j--] = '\\';
 	}
 }
 
+static void fix_context_strings(struct dm_ima_context *context)
+{
+	fix_separator_chars(context->dev_name);
+	fix_separator_chars(context->dev_uuid);
+}
+
 /*
  * Internal function to allocate memory for IMA measurements.
  */
@@ -59,68 +66,85 @@ static void *dm_ima_alloc(size_t len, bool noio)
 	return ptr;
 }
 
-/*
- * Internal function to allocate and copy name and uuid for IMA measurements.
- */
-static int dm_ima_alloc_and_copy_name_uuid(struct mapped_device *md, char **dev_name,
-					   char **dev_uuid, bool noio)
+void dm_ima_init(struct mapped_device *md)
 {
-	int r;
-	*dev_name = dm_ima_alloc(DM_NAME_LEN*2, noio);
-	if (!(*dev_name)) {
-		r = -ENOMEM;
-		goto error;
-	}
+	md->ima.update_idx = 0;
+	md->ima.measure_idx = 0;
+	init_waitqueue_head(&md->ima.ima_wq);
+	spin_lock_init(&md->ima.ima_lock);
+}
 
-	*dev_uuid = dm_ima_alloc(DM_UUID_LEN*2, noio);
-	if (!(*dev_uuid)) {
-		r = -ENOMEM;
-		goto error;
-	}
+void dm_ima_alloc_context(struct dm_ima_context **context, bool noio)
+{
+	*context = dm_ima_alloc(sizeof(struct dm_ima_context), noio);
+}
 
-	r = dm_copy_name_and_uuid(md, *dev_name, *dev_uuid);
-	if (r)
-		goto error;
+void dm_ima_free_context(struct dm_ima_context *context)
+{
+	if (likely(context)) {
+		kfree(context->table.device_metadata);
+		kfree(context->table.hash);
+		kfree(context);
+	}
+}
 
-	fix_separator_chars(dev_name);
-	fix_separator_chars(dev_uuid);
+static void wait_to_measure(struct dm_ima_measurements *ima,
+			    unsigned int update_idx)
+{
+	spin_lock_irq(&ima->ima_lock);
+	wait_event_lock_irq(ima->ima_wq,
+			    ima->measure_idx == update_idx,
+			    ima->ima_lock);
+	spin_unlock_irq(&ima->ima_lock);
+}
 
-	return 0;
-error:
-	kfree(*dev_name);
-	kfree(*dev_uuid);
-	*dev_name = NULL;
-	*dev_uuid = NULL;
-	return r;
+static void wake_next_measure(struct dm_ima_measurements *ima)
+{
+	spin_lock_irq(&ima->ima_lock);
+	ima->measure_idx++;
+	spin_unlock_irq(&ima->ima_lock);
+	wake_up_all(&ima->ima_wq);
 }
 
 /*
- * Internal function to allocate and copy device data for IMA measurements.
+ * Helper function for swapping the table, to make sure that the
+ * correct table metadata is saved and restored.
  */
-static int dm_ima_alloc_and_copy_device_data(struct mapped_device *md, char **device_data,
-					     unsigned int num_targets, bool noio)
+void dm_ima_context_table_op(struct mapped_device *md,
+			     struct dm_ima_context *context,
+			     enum dm_ima_table_op op)
 {
-	char *dev_name = NULL, *dev_uuid = NULL;
-	int r;
+	struct dm_ima_measurements *ima = &md->ima;
 
-	r = dm_ima_alloc_and_copy_name_uuid(md, &dev_name, &dev_uuid, noio);
-	if (r)
-		return r;
+	if (unlikely(!context))
+		return;
 
-	*device_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio);
-	if (!(*device_data)) {
-		r = -ENOMEM;
-		goto error;
+	wait_to_measure(ima, context->update_idx);
+
+	if (op == DM_IMA_TABLE_SAVE) {
+		context->table = ima->inactive_table;
+		memset(&ima->inactive_table, 0, sizeof(ima->inactive_table));
+	} else {
+		ima->inactive_table = context->table;
+		memset(&context->table, 0, sizeof(context->table));
 	}
 
-	scnprintf(*device_data, DM_IMA_DEVICE_BUF_LEN,
+	wake_next_measure(ima);
+}
+
+/*
+ * Internal function to copy device data for IMA measurements.
+ */
+static void dm_ima_copy_device_data(struct mapped_device *md, char *device_data,
+				    struct dm_ima_context *context,
+				    unsigned int num_targets)
+{
+	memset(device_data, 0, DM_IMA_DEVICE_BUF_LEN);
+	scnprintf(device_data, DM_IMA_DEVICE_BUF_LEN,
 		  "name=%s,uuid=%s,major=%d,minor=%d,minor_count=%d,num_targets=%u;",
-		  dev_name, dev_uuid, md->disk->major, md->disk->first_minor,
-		  md->disk->minors, num_targets);
-error:
-	kfree(dev_name);
-	kfree(dev_uuid);
-	return r;
+		  context->dev_name, context->dev_uuid, md->disk->major,
+		  md->disk->first_minor, md->disk->minors, num_targets);
+
 }
 
 /*
@@ -141,42 +165,21 @@ static void dm_ima_measure_data(const char *event_name, const void *buf, size_t
 		memalloc_noio_restore(noio_flag);
 }
 
-/*
- * Internal function to allocate and copy current device capacity for IMA measurements.
- */
-static int dm_ima_alloc_and_copy_capacity_str(struct mapped_device *md, char **capacity_str,
-					      bool noio)
-{
-	sector_t capacity;
-
-	capacity = get_capacity(md->disk);
-
-	*capacity_str = dm_ima_alloc(DM_IMA_DEVICE_CAPACITY_BUF_LEN, noio);
-	if (!(*capacity_str))
-		return -ENOMEM;
-
-	return scnprintf(*capacity_str, DM_IMA_DEVICE_BUF_LEN, "current_device_capacity=%llu;",
-			 capacity);
-}
-
-/*
- * Initialize/reset the dm ima related data structure variables.
- */
-void dm_ima_reset_data(struct mapped_device *md)
+static sector_t dm_ima_capacity(struct mapped_device *md)
 {
-	memset(&(md->ima), 0, sizeof(md->ima));
-	md->ima.dm_version_str_len = strlen(DM_IMA_VERSION_STR);
+	return (md->ima.active_table.device_metadata) ?
+		md->ima.active_table.capacity : get_capacity(md->disk);
 }
 
 /*
  * Build up the IMA data for each target, and finally measure.
  */
-void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_flags)
+void dm_ima_measure_on_table_load(struct dm_table *table,
+				  struct dm_ima_context *context)
 {
 	size_t device_data_buf_len, target_metadata_buf_len, target_data_buf_len, l = 0;
 	char *target_metadata_buf = NULL, *target_data_buf = NULL, *digest_buf = NULL;
 	char *ima_buf = NULL, *device_data_buf = NULL;
-	int last_target_measured = -1;
 	status_type_t type = STATUSTYPE_IMA;
 	size_t cur_total_buf_len = 0;
 	unsigned int num_targets, i;
@@ -185,9 +188,14 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 	bool noio = false;
 	char table_load_event_name[] = "dm_table_load";
 
+	if (unlikely(!context))
+		return;
+
+	wait_to_measure(&table->md->ima, context->update_idx);
+
 	ima_buf = dm_ima_alloc(DM_IMA_MEASUREMENT_BUF_LEN, noio);
 	if (!ima_buf)
-		return;
+		goto error;
 
 	target_metadata_buf = dm_ima_alloc(DM_IMA_TARGET_METADATA_BUF_LEN, noio);
 	if (!target_metadata_buf)
@@ -199,13 +207,18 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 
 	num_targets = table->num_targets;
 
-	if (dm_ima_alloc_and_copy_device_data(table->md, &device_data_buf, num_targets, noio))
+	device_data_buf = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio);
+	if (!device_data_buf)
 		goto error;
 
+	fix_context_strings(context);
+	dm_ima_copy_device_data(table->md, device_data_buf, context,
+				num_targets);
+
 	sha256_init(&hash_ctx);
 
-	memcpy(ima_buf + l, DM_IMA_VERSION_STR, table->md->ima.dm_version_str_len);
-	l += table->md->ima.dm_version_str_len;
+	memcpy(ima_buf + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR));
+	l += strlen(DM_IMA_VERSION_STR);
 
 	device_data_buf_len = strlen(device_data_buf);
 	memcpy(ima_buf + l, device_data_buf, device_data_buf_len);
@@ -214,8 +227,6 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 	for (i = 0; i < num_targets; i++) {
 		struct dm_target *ti = dm_table_get_target(table, i);
 
-		last_target_measured = 0;
-
 		/*
 		 * First retrieve the target metadata.
 		 */
@@ -229,7 +240,7 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 		 * Then retrieve the actual target data.
 		 */
 		if (ti->type->status)
-			ti->type->status(ti, type, status_flags, target_data_buf,
+			ti->type->status(ti, type, 0, target_data_buf,
 					 DM_IMA_TARGET_DATA_BUF_LEN);
 		else
 			target_data_buf[0] = '\0';
@@ -260,19 +271,11 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 			 * prefix, so that multiple records from the same "dm_table_load" for
 			 * a given device can be linked together.
 			 */
-			memcpy(ima_buf + l, DM_IMA_VERSION_STR, table->md->ima.dm_version_str_len);
-			l += table->md->ima.dm_version_str_len;
+			memcpy(ima_buf + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR));
+			l += strlen(DM_IMA_VERSION_STR);
 
 			memcpy(ima_buf + l, device_data_buf, device_data_buf_len);
 			l += device_data_buf_len;
-
-			/*
-			 * If this iteration of the for loop turns out to be the last target
-			 * in the table, dm_ima_measure_data("dm_table_load", ...) doesn't need
-			 * to be called again, just the hash needs to be finalized.
-			 * "last_target_measured" tracks this state.
-			 */
-			last_target_measured = 1;
 		}
 
 		/*
@@ -286,11 +289,8 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 		l += target_data_buf_len;
 	}
 
-	if (!last_target_measured) {
-		dm_ima_measure_data(table_load_event_name, ima_buf, l, noio);
-
-		sha256_update(&hash_ctx, (const u8 *)ima_buf, l);
-	}
+	dm_ima_measure_data(table_load_event_name, ima_buf, l, noio);
+	sha256_update(&hash_ctx, (const u8 *)ima_buf, l);
 
 	/*
 	 * Finalize the table hash, and store it in table->md->ima.inactive_table.hash,
@@ -304,17 +304,14 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 	if (!digest_buf)
 		goto error;
 
-	if (table->md->ima.active_table.hash != table->md->ima.inactive_table.hash)
-		kfree(table->md->ima.inactive_table.hash);
-
+	kfree(table->md->ima.inactive_table.hash);
 	table->md->ima.inactive_table.hash = digest_buf;
 	table->md->ima.inactive_table.hash_len = strlen(digest_buf);
 	table->md->ima.inactive_table.num_targets = num_targets;
+	table->md->ima.inactive_table.capacity = dm_table_get_size(table);
 
-	if (table->md->ima.active_table.device_metadata !=
-	    table->md->ima.inactive_table.device_metadata)
-		kfree(table->md->ima.inactive_table.device_metadata);
 
+	kfree(table->md->ima.inactive_table.device_metadata);
 	table->md->ima.inactive_table.device_metadata = device_data_buf;
 	table->md->ima.inactive_table.device_metadata_len = device_data_buf_len;
 
@@ -326,66 +323,55 @@ void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_fl
 	kfree(ima_buf);
 	kfree(target_metadata_buf);
 	kfree(target_data_buf);
+
+	wake_next_measure(&table->md->ima);
 }
 
 /*
  * Measure IMA data on device resume.
  */
-void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap)
+void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap,
+				     struct dm_ima_context *context)
 {
-	char *device_table_data, *dev_name = NULL, *dev_uuid = NULL, *capacity_str = NULL;
+	char *device_table_data = NULL;
 	char active[] = "active_table_hash=";
 	unsigned int active_len = strlen(active);
 	unsigned int l = 0;
 	bool noio = true;
 	bool nodata = true;
-	int capacity_len;
 
-	device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio);
-	if (!device_table_data)
+	if (unlikely(!context))
 		return;
 
-	capacity_len = dm_ima_alloc_and_copy_capacity_str(md, &capacity_str, noio);
-	if (capacity_len < 0)
-		goto error;
-
-	memcpy(device_table_data + l, DM_IMA_VERSION_STR, md->ima.dm_version_str_len);
-	l += md->ima.dm_version_str_len;
+	wait_to_measure(&md->ima, context->update_idx);
 
 	if (swap) {
-		if (md->ima.active_table.hash != md->ima.inactive_table.hash)
-			kfree(md->ima.active_table.hash);
-
-		md->ima.active_table.hash = NULL;
-		md->ima.active_table.hash_len = 0;
-
-		if (md->ima.active_table.device_metadata !=
-		    md->ima.inactive_table.device_metadata)
-			kfree(md->ima.active_table.device_metadata);
-
-		md->ima.active_table.device_metadata = NULL;
-		md->ima.active_table.device_metadata_len = 0;
-		md->ima.active_table.num_targets = 0;
-
-		if (md->ima.inactive_table.hash) {
-			md->ima.active_table.hash = md->ima.inactive_table.hash;
-			md->ima.active_table.hash_len = md->ima.inactive_table.hash_len;
-			md->ima.inactive_table.hash = NULL;
-			md->ima.inactive_table.hash_len = 0;
-		}
-
-		if (md->ima.inactive_table.device_metadata) {
-			md->ima.active_table.device_metadata =
-				md->ima.inactive_table.device_metadata;
-			md->ima.active_table.device_metadata_len =
-				md->ima.inactive_table.device_metadata_len;
-			md->ima.active_table.num_targets = md->ima.inactive_table.num_targets;
-			md->ima.inactive_table.device_metadata = NULL;
-			md->ima.inactive_table.device_metadata_len = 0;
-			md->ima.inactive_table.num_targets = 0;
+		kfree(md->ima.active_table.hash);
+		kfree(md->ima.active_table.device_metadata);
+		md->ima.active_table = context->table;
+		memset(&context->table, 0, sizeof(context->table));
+		if (md->ima.active_table.device_metadata) {
+			/*
+			 * A rename could have happened while the swap was
+			 * going on. In that case, the saved table info would
+			 * still have the old name. Update the metadata to be
+			 * sure that it has the current name
+			 */
+			struct dm_ima_device_table_metadata *table = &md->ima.active_table;
+			fix_context_strings(context);
+			dm_ima_copy_device_data(md, table->device_metadata,
+						context, table->num_targets);
+			table->device_metadata_len = strlen(table->device_metadata);
 		}
 	}
 
+	device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio);
+	if (!device_table_data)
+		goto error;
+
+	memcpy(device_table_data + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR));
+	l += strlen(DM_IMA_VERSION_STR);
+
 	if (md->ima.active_table.device_metadata) {
 		memcpy(device_table_data + l, md->ima.active_table.device_metadata,
 		       md->ima.active_table.device_metadata_len);
@@ -409,59 +395,54 @@ void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap)
 	}
 
 	if (nodata) {
-		if (dm_ima_alloc_and_copy_name_uuid(md, &dev_name, &dev_uuid, noio))
-			goto error;
-
+		fix_context_strings(context);
 		l = scnprintf(device_table_data, DM_IMA_DEVICE_BUF_LEN,
 			      "%sname=%s,uuid=%s;device_resume=no_data;",
-			      DM_IMA_VERSION_STR, dev_name, dev_uuid);
+			      DM_IMA_VERSION_STR, context->dev_name,
+			      context->dev_uuid);
 	}
-
-	memcpy(device_table_data + l, capacity_str, capacity_len);
-	l += capacity_len;
+	l += scnprintf(device_table_data + l, DM_IMA_DEVICE_BUF_LEN - l,
+		       "current_device_capacity=%llu;", dm_ima_capacity(md));
 
 	dm_ima_measure_data("dm_device_resume", device_table_data, l, noio);
 
-	kfree(dev_name);
-	kfree(dev_uuid);
 error:
-	kfree(capacity_str);
 	kfree(device_table_data);
+
+	wake_next_measure(&md->ima);
 }
 
 /*
  * Measure IMA data on remove.
  */
-void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all)
+void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all,
+				     struct dm_ima_context *context,
+				     unsigned int idx)
 {
-	char *device_table_data, *dev_name = NULL, *dev_uuid = NULL, *capacity_str = NULL;
+	char *device_table_data;
 	char active_table_str[] = "active_table_hash=";
 	char inactive_table_str[] = "inactive_table_hash=";
 	char device_active_str[] = "device_active_metadata=";
 	char device_inactive_str[] = "device_inactive_metadata=";
-	char remove_all_str[] = "remove_all=";
 	unsigned int active_table_len = strlen(active_table_str);
 	unsigned int inactive_table_len = strlen(inactive_table_str);
 	unsigned int device_active_len = strlen(device_active_str);
 	unsigned int device_inactive_len = strlen(device_inactive_str);
-	unsigned int remove_all_len = strlen(remove_all_str);
 	unsigned int l = 0;
 	bool noio = true;
 	bool nodata = true;
-	int capacity_len;
 
-	device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN*2, noio);
-	if (!device_table_data)
+	wait_to_measure(&md->ima, idx);
+
+	if (unlikely(!context))
 		goto exit;
 
-	capacity_len = dm_ima_alloc_and_copy_capacity_str(md, &capacity_str, noio);
-	if (capacity_len < 0) {
-		kfree(device_table_data);
+	device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN*2, noio);
+	if (!device_table_data)
 		goto exit;
-	}
 
-	memcpy(device_table_data + l, DM_IMA_VERSION_STR, md->ima.dm_version_str_len);
-	l += md->ima.dm_version_str_len;
+	memcpy(device_table_data + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR));
+	l += strlen(DM_IMA_VERSION_STR);
 
 	if (md->ima.active_table.device_metadata) {
 		memcpy(device_table_data + l, device_active_str, device_active_len);
@@ -518,68 +499,57 @@ void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all)
 	 * in IMA measurements.
 	 */
 	if (nodata) {
-		if (dm_ima_alloc_and_copy_name_uuid(md, &dev_name, &dev_uuid, noio))
-			goto error;
-
+		fix_context_strings(context);
 		l = scnprintf(device_table_data, DM_IMA_DEVICE_BUF_LEN,
 			      "%sname=%s,uuid=%s;device_remove=no_data;",
-			      DM_IMA_VERSION_STR, dev_name, dev_uuid);
+			      DM_IMA_VERSION_STR, context->dev_name,
+			      context->dev_uuid);
 	}
 
-	memcpy(device_table_data + l, remove_all_str, remove_all_len);
-	l += remove_all_len;
-	memcpy(device_table_data + l, remove_all ? "y;" : "n;", 2);
-	l += 2;
-
-	memcpy(device_table_data + l, capacity_str, capacity_len);
-	l += capacity_len;
+	l += scnprintf(device_table_data + l, (DM_IMA_DEVICE_BUF_LEN * 2) - l,
+		       "remove_all=%c;current_device_capacity=%llu;",
+		       remove_all ? 'y' : 'n', dm_ima_capacity(md));
 
 	dm_ima_measure_data("dm_device_remove", device_table_data, l, noio);
 
-error:
 	kfree(device_table_data);
-	kfree(capacity_str);
 exit:
 	kfree(md->ima.active_table.device_metadata);
-
-	if (md->ima.active_table.device_metadata !=
-	    md->ima.inactive_table.device_metadata)
-		kfree(md->ima.inactive_table.device_metadata);
+	kfree(md->ima.inactive_table.device_metadata);
 
 	kfree(md->ima.active_table.hash);
+	kfree(md->ima.inactive_table.hash);
 
-	if (md->ima.active_table.hash != md->ima.inactive_table.hash)
-		kfree(md->ima.inactive_table.hash);
-
-	dm_ima_reset_data(md);
+	memset(&md->ima.active_table, 0, sizeof(md->ima.active_table));
+	memset(&md->ima.inactive_table, 0, sizeof(md->ima.inactive_table));
 
-	kfree(dev_name);
-	kfree(dev_uuid);
+	wake_next_measure(&md->ima);
 }
 
 /*
  * Measure ima data on table clear.
  */
-void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map)
+void dm_ima_measure_on_table_clear(struct mapped_device *md,
+				   struct dm_ima_context *context)
 {
 	unsigned int l = 0;
-	char *device_table_data = NULL, *dev_name = NULL, *dev_uuid = NULL, *capacity_str = NULL;
+	char *device_table_data = NULL;
 	char inactive_str[] = "inactive_table_hash=";
 	unsigned int inactive_len = strlen(inactive_str);
 	bool noio = true;
 	bool nodata = true;
-	int capacity_len;
 
-	device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio);
-	if (!device_table_data)
+	if (unlikely(!context))
 		return;
 
-	capacity_len = dm_ima_alloc_and_copy_capacity_str(md, &capacity_str, noio);
-	if (capacity_len < 0)
-		goto error1;
+	wait_to_measure(&md->ima, context->update_idx);
 
-	memcpy(device_table_data + l, DM_IMA_VERSION_STR, md->ima.dm_version_str_len);
-	l += md->ima.dm_version_str_len;
+	device_table_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN, noio);
+	if (!device_table_data)
+		goto error;
+
+	memcpy(device_table_data + l, DM_IMA_VERSION_STR, strlen(DM_IMA_VERSION_STR));
+	l += strlen(DM_IMA_VERSION_STR);
 
 	if (md->ima.inactive_table.device_metadata_len &&
 	    md->ima.inactive_table.hash_len) {
@@ -602,101 +572,79 @@ void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map)
 	}
 
 	if (nodata) {
-		if (dm_ima_alloc_and_copy_name_uuid(md, &dev_name, &dev_uuid, noio))
-			goto error2;
-
+		fix_context_strings(context);
 		l = scnprintf(device_table_data, DM_IMA_DEVICE_BUF_LEN,
 			      "%sname=%s,uuid=%s;table_clear=no_data;",
-			      DM_IMA_VERSION_STR, dev_name, dev_uuid);
+			      DM_IMA_VERSION_STR, context->dev_name,
+			      context->dev_uuid);
 	}
 
-	memcpy(device_table_data + l, capacity_str, capacity_len);
-	l += capacity_len;
+	l += scnprintf(device_table_data + l, DM_IMA_DEVICE_BUF_LEN - l,
+		       "current_device_capacity=%llu;", dm_ima_capacity(md));
 
 	dm_ima_measure_data("dm_table_clear", device_table_data, l, noio);
 
-	if (new_map) {
-		if (md->ima.inactive_table.hash &&
-		    md->ima.inactive_table.hash != md->ima.active_table.hash)
-			kfree(md->ima.inactive_table.hash);
-
-		md->ima.inactive_table.hash = NULL;
-		md->ima.inactive_table.hash_len = 0;
-
-		if (md->ima.inactive_table.device_metadata &&
-		    md->ima.inactive_table.device_metadata != md->ima.active_table.device_metadata)
-			kfree(md->ima.inactive_table.device_metadata);
-
-		md->ima.inactive_table.device_metadata = NULL;
-		md->ima.inactive_table.device_metadata_len = 0;
-		md->ima.inactive_table.num_targets = 0;
-
-		if (md->ima.active_table.hash) {
-			md->ima.inactive_table.hash = md->ima.active_table.hash;
-			md->ima.inactive_table.hash_len = md->ima.active_table.hash_len;
-		}
-
-		if (md->ima.active_table.device_metadata) {
-			md->ima.inactive_table.device_metadata =
-				md->ima.active_table.device_metadata;
-			md->ima.inactive_table.device_metadata_len =
-				md->ima.active_table.device_metadata_len;
-			md->ima.inactive_table.num_targets =
-				md->ima.active_table.num_targets;
-		}
-	}
+error:
+	kfree(md->ima.inactive_table.hash);
+	kfree(md->ima.inactive_table.device_metadata);
+	memset(&md->ima.inactive_table, 0, sizeof(md->ima.inactive_table));
 
-	kfree(dev_name);
-	kfree(dev_uuid);
-error2:
-	kfree(capacity_str);
-error1:
 	kfree(device_table_data);
+
+	wake_next_measure(&md->ima);
 }
 
 /*
  * Measure IMA data on device rename.
  */
-void dm_ima_measure_on_device_rename(struct mapped_device *md)
+void dm_ima_measure_on_device_rename(struct mapped_device *md,
+				     struct dm_ima_context *context)
 {
-	char *old_device_data = NULL, *new_device_data = NULL, *combined_device_data = NULL;
-	char *new_dev_name = NULL, *new_dev_uuid = NULL, *capacity_str = NULL;
+	char *old_device_data = NULL;
+	char *combined_device_data = NULL;
 	bool noio = true;
 	int len;
+	struct dm_ima_device_table_metadata *table;
 
-	if (dm_ima_alloc_and_copy_device_data(md, &new_device_data,
-					      md->ima.active_table.num_targets, noio))
+	if (unlikely(!context))
 		return;
 
-	if (dm_ima_alloc_and_copy_name_uuid(md, &new_dev_name, &new_dev_uuid, noio))
-		goto error;
+	wait_to_measure(&md->ima, context->update_idx);
+
+	fix_context_strings(context);
 
 	combined_device_data = dm_ima_alloc(DM_IMA_DEVICE_BUF_LEN * 2, noio);
 	if (!combined_device_data)
-		goto error;
-
-	if (dm_ima_alloc_and_copy_capacity_str(md, &capacity_str, noio) < 0)
-		goto error;
-
-	old_device_data = md->ima.active_table.device_metadata;
-
-	md->ima.active_table.device_metadata = new_device_data;
-	md->ima.active_table.device_metadata_len = strlen(new_device_data);
+		goto exit;
 
+	if (md->ima.active_table.device_metadata)
+		old_device_data = md->ima.active_table.device_metadata;
+	else if (md->ima.inactive_table.device_metadata)
+		old_device_data = md->ima.inactive_table.device_metadata;
+	else
+		old_device_data = "device_rename=no_data;";
 	len = scnprintf(combined_device_data, DM_IMA_DEVICE_BUF_LEN * 2,
-			"%s%snew_name=%s,new_uuid=%s;%s", DM_IMA_VERSION_STR, old_device_data,
-			new_dev_name, new_dev_uuid, capacity_str);
+			"%s%snew_name=%s,new_uuid=%s;current_device_capacity=%llu;",
+			DM_IMA_VERSION_STR, old_device_data, context->dev_name,
+			context->dev_uuid, dm_ima_capacity(md));
 
 	dm_ima_measure_data("dm_device_rename", combined_device_data, len, noio);
+	kfree(combined_device_data);
 
-	goto exit;
-
-error:
-	kfree(new_device_data);
 exit:
-	kfree(capacity_str);
-	kfree(combined_device_data);
-	kfree(old_device_data);
-	kfree(new_dev_name);
-	kfree(new_dev_uuid);
+	if (md->ima.active_table.device_metadata) {
+		table = &md->ima.active_table;
+		dm_ima_copy_device_data(md, table->device_metadata, context,
+					table->num_targets);
+		table->device_metadata_len = strlen(table->device_metadata);
+	}
+
+	if (md->ima.inactive_table.device_metadata) {
+		table = &md->ima.inactive_table;
+		dm_ima_copy_device_data(md, table->device_metadata, context,
+					table->num_targets);
+		table->device_metadata_len = strlen(table->device_metadata);
+	}
+
+	wake_next_measure(&md->ima);
 }
diff --git a/drivers/md/dm-ima.h b/drivers/md/dm-ima.h
index a403deca6093e..0ec013d1545cc 100644
--- a/drivers/md/dm-ima.h
+++ b/drivers/md/dm-ima.h
@@ -24,6 +24,11 @@
 	__dm_ima_str(DM_VERSION_MINOR) "."	\
 	__dm_ima_str(DM_VERSION_PATCHLEVEL) ";"
 
+enum dm_ima_table_op {
+	DM_IMA_TABLE_SAVE,
+	DM_IMA_TABLE_RESTORE,
+};
+
 #ifdef CONFIG_IMA
 
 struct dm_ima_device_table_metadata {
@@ -36,6 +41,7 @@ struct dm_ima_device_table_metadata {
 	char *device_metadata;
 	unsigned int device_metadata_len;
 	unsigned int num_targets;
+	sector_t capacity;
 
 	/*
 	 * Contains the sha256 hashes of the IMA measurements of the target
@@ -45,31 +51,67 @@ struct dm_ima_device_table_metadata {
 	unsigned int hash_len;
 };
 
+struct dm_ima_context {
+	struct dm_ima_device_table_metadata table;
+	unsigned int update_idx;
+	char dev_name[DM_NAME_LEN*2];
+	char dev_uuid[DM_UUID_LEN*2];
+};
+
 /*
  * This structure contains device metadata, and table hash for
  * active and inactive tables for ima measurements.
  */
 struct dm_ima_measurements {
+	unsigned int update_idx;
+	unsigned int measure_idx;
+	struct wait_queue_head ima_wq;
+	spinlock_t ima_lock;
 	struct dm_ima_device_table_metadata active_table;
 	struct dm_ima_device_table_metadata inactive_table;
-	unsigned int dm_version_str_len;
 };
 
-void dm_ima_reset_data(struct mapped_device *md);
-void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_flags);
-void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap);
-void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all);
-void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map);
-void dm_ima_measure_on_device_rename(struct mapped_device *md);
+void dm_ima_init(struct mapped_device *md);
+void dm_ima_alloc_context(struct dm_ima_context **context, bool noio);
+void dm_ima_free_context(struct dm_ima_context *context);
+void dm_ima_context_table_op(struct mapped_device *md,
+			     struct dm_ima_context *context,
+			     enum dm_ima_table_op op);
+void dm_ima_measure_on_table_load(struct dm_table *table,
+				  struct dm_ima_context *context);
+void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap,
+				     struct dm_ima_context *context);
+void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all,
+				     struct dm_ima_context *context,
+				     unsigned int idx);
+void dm_ima_measure_on_table_clear(struct mapped_device *md,
+				   struct dm_ima_context *context);
+void dm_ima_measure_on_device_rename(struct mapped_device *md,
+				     struct dm_ima_context *context);
 
 #else
 
-static inline void dm_ima_reset_data(struct mapped_device *md) {}
-static inline void dm_ima_measure_on_table_load(struct dm_table *table, unsigned int status_flags) {}
-static inline void dm_ima_measure_on_device_resume(struct mapped_device *md, bool swap) {}
-static inline void dm_ima_measure_on_device_remove(struct mapped_device *md, bool remove_all) {}
-static inline void dm_ima_measure_on_table_clear(struct mapped_device *md, bool new_map) {}
-static inline void dm_ima_measure_on_device_rename(struct mapped_device *md) {}
+struct dm_ima_context;
+
+static inline void dm_ima_init(struct mapped_device *md) {}
+static inline void dm_ima_alloc_context(struct dm_ima_context **context, bool noio) {}
+static inline void dm_ima_free_context(struct dm_ima_context *context) {}
+static inline void dm_ima_context_table_op(struct mapped_device *md,
+					   struct dm_ima_context *context,
+					   enum dm_ima_table_op op) {}
+static inline void dm_ima_measure_on_table_load(struct dm_table *table,
+						struct dm_ima_context *context) {}
+static inline void dm_ima_measure_on_device_resume(struct mapped_device *md,
+						   bool swap,
+						   struct dm_ima_context *context) {}
+static inline void dm_ima_measure_on_device_remove(struct mapped_device *md,
+						   bool remove_all,
+						   struct dm_ima_context *context,
+						   unsigned int idx) {}
+static inline void dm_ima_measure_on_table_clear(struct mapped_device *md,
+						 struct dm_ima_context *context) {}
+static inline void dm_ima_measure_on_device_rename(struct mapped_device *md,
+						   struct dm_ima_context *context) {}
 
 #endif /* CONFIG_IMA */
 
diff --git a/drivers/md/dm-inlinecrypt.c b/drivers/md/dm-inlinecrypt.c
new file mode 100644
index 0000000000000..bd8e58a028c58
--- /dev/null
+++ b/drivers/md/dm-inlinecrypt.c
@@ -0,0 +1,599 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2024 Google LLC
+ */
+
+#include <linux/blk-crypto.h>
+#include <linux/ctype.h>
+#include <linux/device-mapper.h>
+#include <linux/hex.h>
+#include <linux/module.h>
+#include <keys/user-type.h>
+
+#define DM_MSG_PREFIX	"inlinecrypt"
+
+static const struct dm_inlinecrypt_cipher {
+	const char *name;
+	enum blk_crypto_mode_num mode_num;
+} dm_inlinecrypt_ciphers[] = {
+	{
+		.name = "aes-xts-plain64",
+		.mode_num = BLK_ENCRYPTION_MODE_AES_256_XTS,
+	},
+};
+
+/**
+ * struct inlinecrypt_ctx - private data of an inlinecrypt target
+ * @dev: the underlying device
+ * @start: starting sector of the range of @dev which this target actually maps.
+ *	   For this purpose a "sector" is 512 bytes.
+ * @cipher_string: the name of the encryption algorithm being used
+ * @key_size: size of the encryption key in bytes
+ * @iv_offset: starting offset for IVs.  IVs are generated as if the target were
+ *	       preceded by @iv_offset 512-byte sectors.
+ * @sector_size: crypto sector size in bytes (usually 4096)
+ * @sector_bits: log2(sector_size)
+ * @key: the encryption key to use
+ * @max_dun: the maximum DUN that may be used (computed from other params)
+ */
+struct inlinecrypt_ctx {
+	struct dm_dev *dev;
+	sector_t start;
+	const char *cipher_string;
+	unsigned int key_size;
+	u64 iv_offset;
+	unsigned int sector_size;
+	unsigned int sector_bits;
+	struct blk_crypto_key key;
+	u64 max_dun;
+};
+
+static const struct dm_inlinecrypt_cipher *
+lookup_cipher(const char *cipher_string)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(dm_inlinecrypt_ciphers); i++) {
+		if (strcmp(cipher_string, dm_inlinecrypt_ciphers[i].name) == 0)
+			return &dm_inlinecrypt_ciphers[i];
+	}
+	return NULL;
+}
+
+static void inlinecrypt_dtr(struct dm_target *ti)
+{
+	struct inlinecrypt_ctx *ctx = ti->private;
+
+	if (ctx->dev) {
+		if (ctx->key.size)
+			blk_crypto_evict_key(ctx->dev->bdev, &ctx->key);
+		dm_put_device(ti, ctx->dev);
+	}
+	kfree_sensitive(ctx->cipher_string);
+	kfree_sensitive(ctx);
+}
+
+#ifdef CONFIG_KEYS
+
+static bool contains_whitespace(const char *str)
+{
+	while (*str)
+		if (isspace(*str++))
+			return true;
+	return false;
+}
+
+static int set_key_user(struct key *key, char *bin_key,
+			const unsigned int bin_key_size)
+{
+	const struct user_key_payload *ukp;
+
+	ukp = user_key_payload_locked(key);
+	if (!ukp)
+		return -EKEYREVOKED;
+
+	if (bin_key_size != ukp->datalen)
+		return -EINVAL;
+
+	memcpy(bin_key, ukp->data, bin_key_size);
+
+	return 0;
+}
+
+static int inlinecrypt_get_keyring_key(const char *key_string, u8 *bin_key,
+					const unsigned int bin_key_size)
+{
+	char *key_desc;
+	int ret;
+	struct key_type *type;
+	struct key *key;
+	int (*set_key)(struct key *key, char *bin_key,
+				   const unsigned int bin_key_size);
+
+	/*
+	 * Reject key_string with whitespace. dm core currently lacks code for
+	 * proper whitespace escaping in arguments on DM_TABLE_STATUS path.
+	 */
+	if (contains_whitespace(key_string)) {
+		DMERR("whitespace chars not allowed in key string");
+		return -EINVAL;
+	}
+
+	/* look for next ':' separating key_type from key_description */
+	key_desc = strchr(key_string, ':');
+	if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
+		return -EINVAL;
+
+	if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) {
+		type = &key_type_logon;
+		set_key = set_key_user;
+	} else {
+		return -EINVAL;
+	}
+
+	key = request_key(type, key_desc + 1, NULL);
+	if (IS_ERR(key))
+		return PTR_ERR(key);
+
+	down_read(&key->sem);
+
+	ret = set_key(key, (char *)bin_key, bin_key_size);
+
+	up_read(&key->sem);
+	key_put(key);
+
+	return ret;
+}
+
+static int get_key_size(char **key_string)
+{
+	char *colon, dummy;
+	int ret;
+
+	if (*key_string[0] != ':') {
+		ret = strlen(*key_string);
+
+		if (ret > 2 * BLK_CRYPTO_MAX_ANY_KEY_SIZE
+			|| ret  % 2
+			|| !ret) {
+			DMERR("Invalid keysize");
+			return -EINVAL;
+		}
+		return ret >> 1;
+	}
+
+	/* look for next ':' in key string */
+	colon = strpbrk(*key_string + 1, ":");
+	if (!colon)
+		return -EINVAL;
+
+	if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')
+		return -EINVAL;
+
+	/* remaining key string should be :<logon|user>:<key_desc> */
+	*key_string = colon;
+
+	return ret;
+}
+
+#else
+
+static int inlinecrypt_get_keyring_key(const char *key_string, u8 *bin_key,
+					const unsigned int bin_key_size)
+{
+	return -EINVAL;
+}
+
+static int get_key_size(char **key_string)
+{
+	int key_hex_size = strlen(*key_string);
+
+	if (*key_string[0] == ':')
+		return -EINVAL;
+
+	if (key_hex_size > 2 * BLK_CRYPTO_MAX_ANY_KEY_SIZE
+		|| key_hex_size  % 2
+		|| !key_hex_size) {
+		DMERR("Invalid keysize");
+		return -EINVAL;
+	}
+
+	return key_hex_size >> 1;
+}
+
+#endif /* CONFIG_KEYS */
+
+static int inlinecrypt_get_key(const char *key_string,
+				u8 key[BLK_CRYPTO_MAX_ANY_KEY_SIZE],
+				const unsigned int key_size)
+{
+	int ret = 0;
+
+	if (key_size > BLK_CRYPTO_MAX_ANY_KEY_SIZE) {
+		DMERR("Invalid keysize");
+		return -EINVAL;
+	}
+
+	/* ':' means the key is in kernel keyring, short-circuit normal key processing */
+	if (key_string[0] == ':') {
+		/* key string should be :<logon|user>:<key_desc> */
+		ret = inlinecrypt_get_keyring_key(key_string + 1, key, key_size);
+		goto out;
+	}
+
+	if (hex2bin(key, key_string, key_size) != 0)
+		ret = -EINVAL;
+
+out:
+	return ret;
+}
+
+static int inlinecrypt_ctr_optional(struct dm_target *ti,
+				    unsigned int argc, char **argv)
+{
+	struct inlinecrypt_ctx *ctx = ti->private;
+	struct dm_arg_set as;
+	static const struct dm_arg _args[] = {
+		{0, 3, "Invalid number of feature args"},
+	};
+	unsigned int opt_params;
+	const char *opt_string;
+	bool iv_large_sectors = false;
+	char dummy;
+	int err;
+
+	as.argc = argc;
+	as.argv = argv;
+
+	err = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
+	if (err)
+		return err;
+
+	while (opt_params--) {
+		opt_string = dm_shift_arg(&as);
+		if (!opt_string) {
+			ti->error = "Not enough feature arguments";
+			return -EINVAL;
+		}
+		if (!strcmp(opt_string, "allow_discards")) {
+			ti->num_discard_bios = 1;
+		} else if (sscanf(opt_string, "sector_size:%u%c",
+				  &ctx->sector_size, &dummy) == 1) {
+			if (ctx->sector_size < SECTOR_SIZE ||
+			    ctx->sector_size > 4096 ||
+			    !is_power_of_2(ctx->sector_size)) {
+				ti->error = "Invalid sector_size";
+				return -EINVAL;
+			}
+		} else if (!strcmp(opt_string, "iv_large_sectors")) {
+			iv_large_sectors = true;
+		} else {
+			ti->error = "Invalid feature arguments";
+			return -EINVAL;
+		}
+	}
+
+	/* dm-inlinecrypt doesn't implement iv_large_sectors=false. */
+	if (ctx->sector_size != SECTOR_SIZE && !iv_large_sectors) {
+		ti->error = "iv_large_sectors must be specified";
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+/*
+ * Construct an inlinecrypt mapping:
+ * <cipher> [<key>|:<key_size>:<logon>:<key_description>] <iv_offset> <dev_path> <start>
+ *
+ * This syntax matches dm-crypt's, but the set of supported functionality has
+ * been stripped down.
+ */
+static int inlinecrypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
+{
+	struct inlinecrypt_ctx *ctx;
+	const struct dm_inlinecrypt_cipher *cipher;
+	u8 raw_key[BLK_CRYPTO_MAX_ANY_KEY_SIZE];
+	unsigned int dun_bytes;
+	unsigned long long tmpll;
+	char dummy;
+	int err;
+
+	if (argc < 5) {
+		ti->error = "Not enough arguments";
+		return -EINVAL;
+	}
+
+	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx) {
+		ti->error = "Out of memory";
+		return -ENOMEM;
+	}
+	ti->private = ctx;
+
+	/* <cipher> */
+	ctx->cipher_string = kstrdup(argv[0], GFP_KERNEL);
+	if (!ctx->cipher_string) {
+		ti->error = "Out of memory";
+		err = -ENOMEM;
+		goto bad;
+	}
+	cipher = lookup_cipher(ctx->cipher_string);
+	if (!cipher) {
+		ti->error = "Unsupported cipher";
+		err = -EINVAL;
+		goto bad;
+	}
+
+	/* <key> */
+	err = get_key_size(&argv[1]);
+	if (err < 0) {
+		ti->error = "Cannot parse key size";
+		return -EINVAL;
+	}
+	ctx->key_size = err;
+
+	err = inlinecrypt_get_key(argv[1], raw_key, ctx->key_size);
+	if (err) {
+		ti->error = "Malformed key string";
+		goto bad;
+	}
+
+	/* <iv_offset> */
+	if (sscanf(argv[2], "%llu%c", &ctx->iv_offset, &dummy) != 1) {
+		ti->error = "Invalid iv_offset sector";
+		err = -EINVAL;
+		goto bad;
+	}
+
+	/* <dev_path> */
+	err = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table),
+			    &ctx->dev);
+	if (err) {
+		ti->error = "Device lookup failed";
+		goto bad;
+	}
+
+	/* <start> */
+	if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 ||
+	    tmpll != (sector_t)tmpll) {
+		ti->error = "Invalid start sector";
+		err = -EINVAL;
+		goto bad;
+	}
+	ctx->start = tmpll;
+
+	/* optional arguments */
+	ctx->sector_size = SECTOR_SIZE;
+	if (argc > 5) {
+		err = inlinecrypt_ctr_optional(ti, argc - 5, &argv[5]);
+		if (err)
+			goto bad;
+	}
+	ctx->sector_bits = ilog2(ctx->sector_size);
+	if (ti->len & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) {
+		ti->error = "Device size is not a multiple of sector_size";
+		err = -EINVAL;
+		goto bad;
+	}
+	if (ctx->iv_offset & ((ctx->sector_size >> SECTOR_SHIFT) - 1)) {
+		ti->error = "Wrong alignment of iv_offset sector";
+		err = -EINVAL;
+	}
+
+	ctx->max_dun = (ctx->iv_offset + ti->len - 1) >>
+		       (ctx->sector_bits - SECTOR_SHIFT);
+	dun_bytes = DIV_ROUND_UP(fls64(ctx->max_dun), 8);
+
+	err = blk_crypto_init_key(&ctx->key, raw_key, ctx->key_size,
+				  BLK_CRYPTO_KEY_TYPE_RAW,
+				  cipher->mode_num, dun_bytes,
+				  ctx->sector_size);
+	if (err) {
+		ti->error = "Error initializing blk-crypto key";
+		goto bad;
+	}
+
+	err = blk_crypto_start_using_key(ctx->dev->bdev, &ctx->key);
+	if (err) {
+		ti->error = "Error starting to use blk-crypto";
+		goto bad;
+	}
+
+	ti->num_flush_bios = 1;
+
+	err = 0;
+	goto out;
+
+bad:
+	inlinecrypt_dtr(ti);
+out:
+	memzero_explicit(raw_key, sizeof(raw_key));
+	return err;
+}
+
+static int inlinecrypt_map(struct dm_target *ti, struct bio *bio)
+{
+	const struct inlinecrypt_ctx *ctx = ti->private;
+	sector_t sector_in_target;
+	u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE] = {};
+
+	bio_set_dev(bio, ctx->dev->bdev);
+
+	/*
+	 * If the bio is a device-level request which doesn't target a specific
+	 * sector, there's nothing more to do.
+	 */
+	if (bio_sectors(bio) == 0)
+		return DM_MAPIO_REMAPPED;
+
+	/*
+	 * The bio should never have an encryption context already, since
+	 * dm-inlinecrypt doesn't pass through any inline encryption
+	 * capabilities to the layer above it.
+	 */
+	if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
+		return DM_MAPIO_KILL;
+
+	/* Map the bio's sector to the underlying device. (512-byte sectors) */
+	sector_in_target = dm_target_offset(ti, bio->bi_iter.bi_sector);
+	bio->bi_iter.bi_sector = ctx->start + sector_in_target;
+	/*
+	 * If the bio doesn't have any data (e.g. if it's a DISCARD request),
+	 * there's nothing more to do.
+	 */
+	if (!bio_has_data(bio))
+		return DM_MAPIO_REMAPPED;
+
+	/* Calculate the DUN and enforce data-unit (crypto sector) alignment. */
+	dun[0] = ctx->iv_offset + sector_in_target; /* 512-byte sectors */
+	if (dun[0] & ((ctx->sector_size >> SECTOR_SHIFT) - 1))
+		return DM_MAPIO_KILL;
+	dun[0] >>= ctx->sector_bits - SECTOR_SHIFT; /* crypto sectors */
+
+	/*
+	 * This check isn't necessary as we should have calculated max_dun
+	 * correctly, but be safe.
+	 */
+	if (WARN_ON_ONCE(dun[0] > ctx->max_dun))
+		return DM_MAPIO_KILL;
+
+	bio_crypt_set_ctx(bio, &ctx->key, dun, GFP_NOIO);
+
+	/*
+	 * Since we've added an encryption context to the bio and
+	 * blk-crypto-fallback may be needed to process it, it's necessary to
+	 * use the fallback-aware bio submission code rather than
+	 * unconditionally returning DM_MAPIO_REMAPPED.
+	 *
+	 * To get the correct accounting for a dm target in the case where
+	 * __blk_crypto_submit_bio() doesn't take ownership of the bio (returns
+	 * true), call __blk_crypto_submit_bio() directly and return
+	 * DM_MAPIO_REMAPPED in that case, rather than relying on
+	 * blk_crypto_submit_bio() which calls submit_bio() in that case.
+	 *
+	 * TODO: blk-crypto fallback write slow-path currently double-accounts
+	 * IO in vmstat, as encrypted bios are submitted via submit_bio().
+	 * This does not affect data correctness. Consider fixing this if
+	 * a cleaner accounting model for derived bios is introduced.
+	 */
+	if (__blk_crypto_submit_bio(bio))
+		return DM_MAPIO_REMAPPED;
+	return DM_MAPIO_SUBMITTED;
+}
+
+static void inlinecrypt_status(struct dm_target *ti, status_type_t type,
+			       unsigned int status_flags, char *result,
+			       unsigned int maxlen)
+{
+	const struct inlinecrypt_ctx *ctx = ti->private;
+	unsigned int sz = 0;
+	int num_feature_args = 0;
+
+	switch (type) {
+	case STATUSTYPE_INFO:
+	case STATUSTYPE_IMA:
+		result[0] = '\0';
+		break;
+
+	case STATUSTYPE_TABLE:
+		/*
+		 * Warning: like dm-crypt, dm-inlinecrypt includes the key in
+		 * the returned table.  Userspace is responsible for redacting
+		 * the key when needed.
+		 */
+		DMEMIT("%s %*phN %llu %s %llu", ctx->cipher_string,
+		       ctx->key.size, ctx->key.bytes, ctx->iv_offset,
+		       ctx->dev->name, ctx->start);
+		num_feature_args += !!ti->num_discard_bios;
+		if (ctx->sector_size != SECTOR_SIZE)
+			num_feature_args += 2;
+		if (num_feature_args != 0) {
+			DMEMIT(" %d", num_feature_args);
+			if (ti->num_discard_bios)
+				DMEMIT(" allow_discards");
+			if (ctx->sector_size != SECTOR_SIZE) {
+				DMEMIT(" sector_size:%u", ctx->sector_size);
+				DMEMIT(" iv_large_sectors");
+			}
+		}
+		break;
+	}
+}
+
+static int inlinecrypt_prepare_ioctl(struct dm_target *ti,
+				     struct block_device **bdev, unsigned int cmd,
+				     unsigned long arg, bool *forward)
+{
+	const struct inlinecrypt_ctx *ctx = ti->private;
+	const struct dm_dev *dev = ctx->dev;
+
+	*bdev = dev->bdev;
+
+	/* Only pass ioctls through if the device sizes match exactly. */
+	return ctx->start != 0 || ti->len != bdev_nr_sectors(dev->bdev);
+}
+
+static int inlinecrypt_iterate_devices(struct dm_target *ti,
+				       iterate_devices_callout_fn fn,
+				       void *data)
+{
+	const struct inlinecrypt_ctx *ctx = ti->private;
+
+	return fn(ti, ctx->dev, ctx->start, ti->len, data);
+}
+
+#ifdef CONFIG_BLK_DEV_ZONED
+static int inlinecrypt_report_zones(struct dm_target *ti,
+				    struct dm_report_zones_args *args,
+				    unsigned int nr_zones)
+{
+	const struct inlinecrypt_ctx *ctx = ti->private;
+
+	return dm_report_zones(ctx->dev->bdev, ctx->start,
+			ctx->start + dm_target_offset(ti, args->next_sector),
+			args, nr_zones);
+}
+#else
+#define inlinecrypt_report_zones NULL
+#endif
+
+static void inlinecrypt_io_hints(struct dm_target *ti,
+				 struct queue_limits *limits)
+{
+	const struct inlinecrypt_ctx *ctx = ti->private;
+	const unsigned int sector_size = ctx->sector_size;
+
+	limits->logical_block_size =
+		max_t(unsigned int, limits->logical_block_size, sector_size);
+	limits->physical_block_size =
+		max_t(unsigned int, limits->physical_block_size, sector_size);
+	limits->io_min = max_t(unsigned int, limits->io_min, sector_size);
+	limits->dma_alignment = limits->logical_block_size - 1;
+}
+
+static struct target_type inlinecrypt_target = {
+	.name			= "inlinecrypt",
+	.version		= {1, 0, 0},
+	/*
+	 * Do not set DM_TARGET_PASSES_CRYPTO, since dm-inlinecrypt consumes the
+	 * crypto capability itself.
+	 */
+	.features		= DM_TARGET_ZONED_HM,
+	.module			= THIS_MODULE,
+	.ctr			= inlinecrypt_ctr,
+	.dtr			= inlinecrypt_dtr,
+	.map			= inlinecrypt_map,
+	.status			= inlinecrypt_status,
+	.prepare_ioctl		= inlinecrypt_prepare_ioctl,
+	.iterate_devices	= inlinecrypt_iterate_devices,
+	.report_zones		= inlinecrypt_report_zones,
+	.io_hints		= inlinecrypt_io_hints,
+};
+
+module_dm(inlinecrypt);
+
+MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
+MODULE_AUTHOR("Linlin Zhang <linlin.zhang@oss.qualcomm.com>");
+MODULE_DESCRIPTION(DM_NAME " target for inline encryption");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/dm-ioctl.c b/drivers/md/dm-ioctl.c
index a529174c94cf3..ac77dc0ca2255 100644
--- a/drivers/md/dm-ioctl.c
+++ b/drivers/md/dm-ioctl.c
@@ -259,6 +259,80 @@ static void free_cell(struct hash_cell *hc)
 	}
 }
 
+#ifdef CONFIG_IMA
+
+/*
+ * Called while holding to _hash_lock, to guarantee the ordering of the
+ * following dm_ima_measure_on_* functions, which should be called
+ * right after dropping the _hash_lock
+ */
+static unsigned int dm_ima_init_context(struct hash_cell *hc,
+					struct dm_ima_context *context,
+				        bool need_idx)
+{
+	lockdep_assert_held(&_hash_lock);
+
+	if (unlikely(!context))
+		return need_idx ? hc->md->ima.update_idx++ : 0;
+
+	context->update_idx = hc->md->ima.update_idx++;
+	strcpy(context->dev_name, hc->name);
+	strcpy(context->dev_uuid, hc->uuid ? : "");
+
+	return context->update_idx;
+}
+
+/*
+ * Called by do_resume() to guarantee correct ordering, since do_resume()
+ * does not grab the _hash_lock when the table is not getting swapped or
+ * when actually swapping the active table
+ */
+static bool dm_ima_need_measure(struct mapped_device *md,
+				struct dm_table *table,
+				struct dm_ima_context *context)
+{
+	int srcu_idx;
+	struct hash_cell *hc;
+	bool need_measure = false;
+
+	if (unlikely(!context))
+		return false;
+
+	down_write(&_hash_lock);
+	/* Check if the device has been removed */
+	hc = dm_get_mdptr(md);
+	if (hc) {
+		/*
+		 * If we have a table, we need to make sure that it's the
+		 * active table. Otherwise we raced with another process
+		 * setting the active table and it will do the measurement
+		 */
+		if (!table || dm_get_live_table(md, &srcu_idx) == table) {
+			dm_ima_init_context(hc, context, false);
+			need_measure = true;
+		}
+		if (table)
+			dm_put_live_table(md, srcu_idx);
+	}
+	up_write(&_hash_lock);
+
+	return need_measure;
+}
+#else
+static inline unsigned int dm_ima_init_context(struct hash_cell *hc,
+					       struct dm_ima_context *context,
+					       bool neex_idx)
+{
+	return 0;
+}
+static inline bool dm_ima_need_measure(struct mapped_device *md,
+				       struct dm_table *table,
+				       struct dm_ima_context *context)
+{
+	return false;
+}
+#endif
+
 /*
  * The kdev_t and uuid of a device can never change once it is
  * initially inserted.
@@ -344,7 +418,10 @@ static int dm_hash_remove_all(unsigned flags)
 	struct hash_cell *hc;
 	struct mapped_device *md;
 	struct dm_table *t;
+	struct dm_ima_context *ima_context = NULL;
+	unsigned int ima_idx;
 
+	dm_ima_alloc_context(&ima_context, true);
 retry:
 	dev_skipped = 0;
 
@@ -353,6 +430,7 @@ static int dm_hash_remove_all(unsigned flags)
 	for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) {
 		if (flags & DM_REMOVE_INTERRUPTIBLE && fatal_signal_pending(current)) {
 			up_write(&_hash_lock);
+			dm_ima_free_context(ima_context);
 			return -EINTR;
 		}
 
@@ -367,6 +445,7 @@ static int dm_hash_remove_all(unsigned flags)
 			continue;
 		}
 
+		ima_idx = dm_ima_init_context(hc, ima_context, true);
 		t = __hash_remove(hc);
 
 		up_write(&_hash_lock);
@@ -375,7 +454,7 @@ static int dm_hash_remove_all(unsigned flags)
 			dm_sync_table(md);
 			dm_table_destroy(t);
 		}
-		dm_ima_measure_on_device_remove(md, true);
+		dm_ima_measure_on_device_remove(md, true, ima_context, ima_idx);
 		dm_put(md);
 		if (likely(flags & DM_REMOVE_KEEP_OPEN_DEVICES))
 			dm_destroy(md);
@@ -396,6 +475,7 @@ static int dm_hash_remove_all(unsigned flags)
 	if (dev_skipped && !(flags & DM_REMOVE_ONLY_DEFERRED))
 		DMWARN("remove_all left %d open device(s)", dev_skipped);
 
+	dm_ima_free_context(ima_context);
 	return 0;
 }
 
@@ -443,6 +523,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
 	struct mapped_device *md;
 	unsigned int change_uuid = (param->flags & DM_UUID_FLAG) ? 1 : 0;
 	int srcu_idx;
+	struct dm_ima_context *ima_context = NULL;
 
 	/*
 	 * duplicate new.
@@ -451,6 +532,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
 	if (!new_data)
 		return ERR_PTR(-ENOMEM);
 
+	dm_ima_alloc_context(&ima_context, true);
 	down_write(&_hash_lock);
 
 	/*
@@ -467,6 +549,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
 		      param->name, new);
 		dm_put(hc->md);
 		up_write(&_hash_lock);
+		dm_ima_free_context(ima_context);
 		kfree(new_data);
 		return ERR_PTR(-EBUSY);
 	}
@@ -479,6 +562,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
 		DMERR("Unable to rename non-existent device, %s to %s%s",
 		      param->name, change_uuid ? "uuid " : "", new);
 		up_write(&_hash_lock);
+		dm_ima_free_context(ima_context);
 		kfree(new_data);
 		return ERR_PTR(-ENXIO);
 	}
@@ -492,6 +576,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
 		      param->name, new, hc->uuid);
 		dm_put(hc->md);
 		up_write(&_hash_lock);
+		dm_ima_free_context(ima_context);
 		kfree(new_data);
 		return ERR_PTR(-EINVAL);
 	}
@@ -514,9 +599,11 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
 
 	md = hc->md;
 
-	dm_ima_measure_on_device_rename(md);
+	dm_ima_init_context(hc, ima_context, false);
 
 	up_write(&_hash_lock);
+	dm_ima_measure_on_device_rename(md, ima_context);
+	dm_ima_free_context(ima_context);
 	kfree(old_name);
 
 	return md;
@@ -995,13 +1082,17 @@ static int dev_remove(struct file *filp, struct dm_ioctl *param, size_t param_si
 	struct mapped_device *md;
 	int r;
 	struct dm_table *t;
+	struct dm_ima_context *ima_context = NULL;
+	unsigned int ima_idx;
 
+	dm_ima_alloc_context(&ima_context, true);
 	down_write(&_hash_lock);
 	hc = __find_device_hash_cell(param);
 
 	if (!hc) {
 		DMDEBUG_LIMIT("device doesn't appear to be in the dev hash table.");
 		up_write(&_hash_lock);
+		dm_ima_free_context(ima_context);
 		return -ENXIO;
 	}
 
@@ -1015,14 +1106,17 @@ static int dev_remove(struct file *filp, struct dm_ioctl *param, size_t param_si
 		if (r == -EBUSY && param->flags & DM_DEFERRED_REMOVE) {
 			up_write(&_hash_lock);
 			dm_put(md);
+			dm_ima_free_context(ima_context);
 			return 0;
 		}
 		DMDEBUG_LIMIT("unable to remove open device %s", hc->name);
 		up_write(&_hash_lock);
 		dm_put(md);
+		dm_ima_free_context(ima_context);
 		return r;
 	}
 
+	ima_idx = dm_ima_init_context(hc, ima_context, true);
 	t = __hash_remove(hc);
 	up_write(&_hash_lock);
 
@@ -1033,7 +1127,8 @@ static int dev_remove(struct file *filp, struct dm_ioctl *param, size_t param_si
 
 	param->flags &= ~DM_DEFERRED_REMOVE;
 
-	dm_ima_measure_on_device_remove(md, false);
+	dm_ima_measure_on_device_remove(md, false, ima_context, ima_idx);
+	dm_ima_free_context(ima_context);
 
 	if (!dm_kobject_uevent(md, KOBJ_REMOVE, param->event_nr, false))
 		param->flags |= DM_UEVENT_GENERATED_FLAG;
@@ -1169,13 +1264,16 @@ static int do_resume(struct dm_ioctl *param)
 	struct mapped_device *md;
 	struct dm_table *new_map, *old_map = NULL;
 	bool need_resize_uevent = false;
+	struct dm_ima_context *ima_context = NULL;
 
+	dm_ima_alloc_context(&ima_context, true);
 	down_write(&_hash_lock);
 
 	hc = __find_device_hash_cell(param);
 	if (!hc) {
 		DMDEBUG_LIMIT("device doesn't appear to be in the dev hash table.");
 		up_write(&_hash_lock);
+		dm_ima_free_context(ima_context);
 		return -ENXIO;
 	}
 
@@ -1184,13 +1282,15 @@ static int do_resume(struct dm_ioctl *param)
 	new_map = hc->new_map;
 	hc->new_map = NULL;
 	param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
-
+	if (new_map)
+		dm_ima_init_context(hc, ima_context, false);
 	up_write(&_hash_lock);
 
 	/* Do we need to load a new map ? */
 	if (new_map) {
 		sector_t old_size, new_size;
 
+		dm_ima_context_table_op(md, ima_context, DM_IMA_TABLE_SAVE);
 		/* Suspend if it isn't already suspended */
 		if (param->flags & DM_SKIP_LOCKFS_FLAG)
 			suspend_flags &= ~DM_SUSPEND_LOCKFS_FLAG;
@@ -1204,6 +1304,8 @@ static int do_resume(struct dm_ioctl *param)
 				if (hc && !hc->new_map) {
 					hc->new_map = new_map;
 					new_map = NULL;
+					dm_ima_init_context(hc, ima_context,
+							    false);
 				} else {
 					r = -ENXIO;
 				}
@@ -1211,7 +1313,9 @@ static int do_resume(struct dm_ioctl *param)
 				if (new_map) {
 					dm_sync_table(md);
 					dm_table_destroy(new_map);
-				}
+				} else
+					dm_ima_context_table_op(md, ima_context, DM_IMA_TABLE_RESTORE);
+				dm_ima_free_context(ima_context);
 				dm_put(md);
 				return r;
 			}
@@ -1222,9 +1326,12 @@ static int do_resume(struct dm_ioctl *param)
 		if (IS_ERR(old_map)) {
 			dm_sync_table(md);
 			dm_table_destroy(new_map);
+			dm_ima_free_context(ima_context);
 			dm_put(md);
 			return PTR_ERR(old_map);
 		}
+		if (dm_ima_need_measure(md, new_map, ima_context))
+			dm_ima_measure_on_device_resume(md, true, ima_context);
 		new_size = dm_get_size(md);
 		if (old_size && new_size && old_size != new_size)
 			need_resize_uevent = true;
@@ -1238,7 +1345,10 @@ static int do_resume(struct dm_ioctl *param)
 	if (dm_suspended_md(md)) {
 		r = dm_resume(md);
 		if (!r) {
-			dm_ima_measure_on_device_resume(md, new_map ? true : false);
+			if (!new_map && dm_ima_need_measure(md, NULL,
+							    ima_context))
+				dm_ima_measure_on_device_resume(md, false,
+								ima_context);
 
 			if (!dm_kobject_uevent(md, KOBJ_CHANGE, param->event_nr, need_resize_uevent))
 				param->flags |= DM_UEVENT_GENERATED_FLAG;
@@ -1255,6 +1365,7 @@ static int do_resume(struct dm_ioctl *param)
 	if (!r)
 		__dev_status(md, param);
 
+	dm_ima_free_context(ima_context);
 	dm_put(md);
 	return r;
 }
@@ -1532,11 +1643,12 @@ static bool is_valid_type(enum dm_queue_mode cur, enum dm_queue_mode new)
 
 static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_size)
 {
-	int r;
+	int r, srcu_idx;
 	struct hash_cell *hc;
 	struct dm_table *t, *old_map = NULL;
 	struct mapped_device *md;
 	struct target_type *immutable_target_type;
+	struct dm_ima_context *ima_context = NULL;
 
 	md = find_device(param);
 	if (!md)
@@ -1552,8 +1664,6 @@ static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_si
 	if (r)
 		goto err_unlock_md_type;
 
-	dm_ima_measure_on_table_load(t, STATUSTYPE_IMA);
-
 	immutable_target_type = dm_get_immutable_target_type(md);
 	if (immutable_target_type &&
 	    (immutable_target_type != dm_table_get_immutable_target_type(t)) &&
@@ -1580,12 +1690,14 @@ static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_si
 
 	dm_unlock_md_type(md);
 
+	dm_ima_alloc_context(&ima_context, false);
 	/* stage inactive table */
 	down_write(&_hash_lock);
 	hc = dm_get_mdptr(md);
 	if (!hc) {
 		DMERR("device has been removed from the dev hash table.");
 		up_write(&_hash_lock);
+		dm_ima_free_context(ima_context);
 		r = -ENXIO;
 		goto err_destroy_table;
 	}
@@ -1593,8 +1705,15 @@ static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_si
 	if (hc->new_map)
 		old_map = hc->new_map;
 	hc->new_map = t;
+	dm_ima_init_context(hc, ima_context, false);
+	/* Make sure new_map doesn't get freed before we measure it*/
+	dm_get_live_table(md, &srcu_idx);
 	up_write(&_hash_lock);
 
+	dm_ima_measure_on_table_load(t, ima_context);
+	dm_ima_free_context(ima_context);
+	dm_put_live_table(md, srcu_idx);
+
 	param->flags |= DM_INACTIVE_PRESENT_FLAG;
 	__dev_status(md, param);
 
@@ -1622,25 +1741,29 @@ static int table_clear(struct file *filp, struct dm_ioctl *param, size_t param_s
 	struct hash_cell *hc;
 	struct mapped_device *md;
 	struct dm_table *old_map = NULL;
-	bool has_new_map = false;
+	struct dm_ima_context *ima_context = NULL;
 
+	dm_ima_alloc_context(&ima_context, true);
 	down_write(&_hash_lock);
 
 	hc = __find_device_hash_cell(param);
 	if (!hc) {
 		DMDEBUG_LIMIT("device doesn't appear to be in the dev hash table.");
 		up_write(&_hash_lock);
+		dm_ima_free_context(ima_context);
 		return -ENXIO;
 	}
 
 	if (hc->new_map) {
 		old_map = hc->new_map;
 		hc->new_map = NULL;
-		has_new_map = true;
 	}
 
+	dm_ima_init_context(hc, ima_context, false);
 	md = hc->md;
 	up_write(&_hash_lock);
+	dm_ima_measure_on_table_clear(md, ima_context);
+	dm_ima_free_context(ima_context);
 
 	param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
 	__dev_status(md, param);
@@ -1649,7 +1772,6 @@ static int table_clear(struct file *filp, struct dm_ioctl *param, size_t param_s
 		dm_sync_table(md);
 		dm_table_destroy(old_map);
 	}
-	dm_ima_measure_on_table_clear(md, has_new_map);
 	dm_put(md);
 
 	return 0;
@@ -1816,8 +1938,11 @@ static int target_message(struct file *filp, struct dm_ioctl *param, size_t para
 		goto out_argv;
 
 	table = dm_get_live_table(md, &srcu_idx);
-	if (!table)
+	if (!table) {
+		DMERR("The device has no table.");
+		r = -EINVAL;
 		goto out_table;
+	}
 
 	if (dm_deleting_md(md)) {
 		r = -ENXIO;
diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c
index c5dc083c72441..8f5a5e1342a95 100644
--- a/drivers/md/dm-raid.c
+++ b/drivers/md/dm-raid.c
@@ -3831,6 +3831,7 @@ static void raid_presuspend(struct dm_target *ti)
 	 * resume, raid_postsuspend() is too late.
 	 */
 	set_bit(RT_FLAG_RS_FROZEN, &rs->runtime_flags);
+	set_bit(MD_DM_SUSPENDING, &mddev->flags);
 
 	if (!reshape_interrupted(mddev))
 		return;
@@ -3847,13 +3848,16 @@ static void raid_presuspend(struct dm_target *ti)
 static void raid_presuspend_undo(struct dm_target *ti)
 {
 	struct raid_set *rs = ti->private;
+	struct mddev *mddev = &rs->md;
 
+	clear_bit(MD_DM_SUSPENDING, &mddev->flags);
 	clear_bit(RT_FLAG_RS_FROZEN, &rs->runtime_flags);
 }
 
 static void raid_postsuspend(struct dm_target *ti)
 {
 	struct raid_set *rs = ti->private;
+	struct mddev *mddev = &rs->md;
 
 	if (!test_and_set_bit(RT_FLAG_RS_SUSPENDED, &rs->runtime_flags)) {
 		/*
@@ -3864,6 +3868,8 @@ static void raid_postsuspend(struct dm_target *ti)
 		mddev_suspend(&rs->md, false);
 		rs->md.ro = MD_RDONLY;
 	}
+	clear_bit(MD_DM_SUSPENDING, &mddev->flags);
+
 }
 
 static void attempt_restore_of_faulty_devices(struct raid_set *rs)
diff --git a/drivers/md/dm-vdo/indexer/index-layout.c b/drivers/md/dm-vdo/indexer/index-layout.c
index 5f4ce4ab1b1ea..2d529250000eb 100644
--- a/drivers/md/dm-vdo/indexer/index-layout.c
+++ b/drivers/md/dm-vdo/indexer/index-layout.c
@@ -282,7 +282,7 @@ static void create_unique_nonce_data(u8 *buffer)
 	u32 rand;
 	size_t offset = 0;
 
-	get_random_bytes(&rand, sizeof(u32));
+	rand = get_random_u32();
 	memcpy(buffer + offset, &now, sizeof(now));
 	offset += sizeof(now);
 	memcpy(buffer + offset, &rand, sizeof(rand));
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index e178fe19973ea..7287bed6eb642 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -2098,8 +2098,17 @@ static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob,
 	WARN_ON_ONCE(!dm_tio_is_normal(&io->tio));
 
 	/* don't poll if the mapped io is done */
-	if (atomic_read(&io->io_count) > 1)
-		bio_poll(&io->tio.clone, iob, flags);
+	if (atomic_read(&io->io_count) > 1) {
+		/*
+		 * DM hides the target queues from the upper poller, which may
+		 * decide it is safe to spin on a single stacked queue.  Do not
+		 * pass that spinning policy down to a target queue: one slow
+		 * clone could keep the task inside dm_poll_bio() for a long
+		 * time.  Poll target bios once and let the caller decide
+		 * whether to keep polling, reap completions or reschedule.
+		 */
+		bio_poll(&io->tio.clone, iob, flags | BLK_POLL_ONESHOT);
+	}
 
 	/* bio_poll holds the last reference */
 	return atomic_read(&io->io_count) == 1;
@@ -2546,7 +2555,7 @@ int dm_create(int minor, struct mapped_device **result)
 	if (!md)
 		return -ENXIO;
 
-	dm_ima_reset_data(md);
+	dm_ima_init(md);
 
 	*result = md;
 	return 0;
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 52c3780860464..9e5100609d120 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -346,6 +346,7 @@ struct md_cluster_operations;
  * @MD_HAS_SUPERBLOCK: There is persistence sb in member disks.
  * @MD_FAILLAST_DEV: Allow last rdev to be removed.
  * @MD_SERIALIZE_POLICY: Enforce write IO is not reordered, just used by raid1.
+ * @MD_DM_SUSPENDING: This DM raid device is suspending.
  *
  * change UNSUPPORTED_MDDEV_FLAGS for each array type if new flag is added
  */
@@ -365,6 +366,7 @@ enum mddev_flags {
 	MD_HAS_SUPERBLOCK,
 	MD_FAILLAST_DEV,
 	MD_SERIALIZE_POLICY,
+	MD_DM_SUSPENDING,
 };
 
 enum mddev_sb_flags {
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 0d76e82f4506e..d0d7f96fd7cd9 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -6042,8 +6042,11 @@ static enum stripe_result make_stripe_request(struct mddev *mddev,
 	raid5_release_stripe(sh);
 out:
 	if (ret == STRIPE_SCHEDULE_AND_RETRY && reshape_interrupted(mddev)) {
-		bi->bi_status = BLK_STS_RESOURCE;
-		ret = STRIPE_WAIT_RESHAPE;
+		if (!mddev_is_dm(mddev) ||
+		    test_bit(MD_DM_SUSPENDING, &mddev->flags)) {
+			bi->bi_status = BLK_STS_RESOURCE;
+			ret = STRIPE_WAIT_RESHAPE;
+		}
 		pr_err_ratelimited("dm-raid456: io across reshape position while reshape can't make progress");
 	}
 	return ret;
@@ -6955,7 +6958,7 @@ raid5_store_rmw_level(struct mddev  *mddev, const char *page, size_t len)
 	if (kstrtoul(page, 10, &new))
 		return -EINVAL;
 
-	if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)
+	if (new != PARITY_DISABLE_RMW && !raid6_can_xor_syndrome())
 		return -EINVAL;
 
 	if (new != PARITY_DISABLE_RMW &&
@@ -7646,7 +7649,7 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 	conf->level = mddev->new_level;
 	if (conf->level == 6) {
 		conf->max_degraded = 2;
-		if (raid6_call.xor_syndrome)
+		if (raid6_can_xor_syndrome())
 			conf->rmw_level = PARITY_ENABLE_RMW;
 		else
 			conf->rmw_level = PARITY_DISABLE_RMW;
diff --git a/drivers/media/cec/core/cec-adap.c b/drivers/media/cec/core/cec-adap.c
index 8f7244ac1d43e..a90cb84a4b4d0 100644
--- a/drivers/media/cec/core/cec-adap.c
+++ b/drivers/media/cec/core/cec-adap.c
@@ -1098,6 +1098,15 @@ static const u8 cec_msg_size[256] = {
 	[CEC_MSG_REQUEST_CURRENT_LATENCY] = 4 | BCAST,
 	[CEC_MSG_REPORT_CURRENT_LATENCY] = 6 | BCAST,
 	[CEC_MSG_CDC_MESSAGE] = 2 | BCAST,
+	[CEC_MSG_REQUEST_LIP_SUPPORT] = 4 | DIRECTED,
+	[CEC_MSG_REPORT_LIP_SUPPORT] = 6 | DIRECTED,
+	[CEC_MSG_REQUEST_AUDIO_AND_VIDEO_LATENCY] = 6 | DIRECTED,
+	[CEC_MSG_REPORT_AUDIO_AND_VIDEO_LATENCY] = 6 | DIRECTED,
+	[CEC_MSG_REQUEST_AUDIO_LATENCY] = 3 | DIRECTED,
+	[CEC_MSG_REPORT_AUDIO_LATENCY] = 4 | DIRECTED,
+	[CEC_MSG_REQUEST_VIDEO_LATENCY] = 5 | DIRECTED,
+	[CEC_MSG_REPORT_VIDEO_LATENCY] = 4 | DIRECTED,
+	[CEC_MSG_UPDATE_SQID] = 6 | DIRECTED,
 };
 
 /* Called by the CEC adapter if a message is received */
diff --git a/drivers/media/cec/i2c/tda9950.c b/drivers/media/cec/i2c/tda9950.c
index cbff851e0c85f..2bece4e636872 100644
--- a/drivers/media/cec/i2c/tda9950.c
+++ b/drivers/media/cec/i2c/tda9950.c
@@ -486,7 +486,7 @@ static void tda9950_remove(struct i2c_client *client)
 }
 
 static struct i2c_device_id tda9950_ids[] = {
-	{ "tda9950" },
+	{ .name = "tda9950" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tda9950_ids);
diff --git a/drivers/media/cec/platform/cros-ec/cros-ec-cec.c b/drivers/media/cec/platform/cros-ec/cros-ec-cec.c
index 419b9a7abcceb..ec1da9cd3674b 100644
--- a/drivers/media/cec/platform/cros-ec/cros-ec-cec.c
+++ b/drivers/media/cec/platform/cros-ec/cros-ec-cec.c
@@ -334,6 +334,12 @@ static const struct cec_dmi_match cec_dmi_match_table[] = {
 	{ "Google", "Dirks", "0000:00:02.0", port_ab_conns },
 	/* Google Moxie */
 	{ "Google", "Moxie", "0000:00:02.0", port_b_conns },
+	/* Google Kulnex */
+	{ "Google", "Kulnex", "0000:00:02.0", port_b_conns },
+	/* Google Moxoe */
+	{ "Google", "Moxoe", "0000:00:02.0", port_b_conns },
+	/* Google Dirkson */
+	{ "Google", "Dirkson", "0000:00:02.0", port_ab_conns },
 };
 
 static struct device *cros_ec_cec_find_hdmi_dev(struct device *dev,
diff --git a/drivers/media/cec/platform/seco/seco-cec.c b/drivers/media/cec/platform/seco/seco-cec.c
index b7bb49f023957..97ed9654c78a1 100644
--- a/drivers/media/cec/platform/seco/seco-cec.c
+++ b/drivers/media/cec/platform/seco/seco-cec.c
@@ -649,7 +649,7 @@ static int secocec_probe(struct platform_device *pdev)
 
 	ret = secocec_ir_probe(secocec);
 	if (ret)
-		goto err_notifier;
+		goto err_unregister_adapter;
 
 	platform_set_drvdata(pdev, secocec);
 
@@ -657,6 +657,10 @@ static int secocec_probe(struct platform_device *pdev)
 
 	return ret;
 
+err_unregister_adapter:
+	cec_notifier_cec_adap_unregister(secocec->notifier, secocec->cec_adap);
+	cec_unregister_adapter(secocec->cec_adap);
+	goto err;
 err_notifier:
 	cec_notifier_cec_adap_unregister(secocec->notifier, secocec->cec_adap);
 err_delete_adapter:
diff --git a/drivers/media/common/videobuf2/videobuf2-core.c b/drivers/media/common/videobuf2/videobuf2-core.c
index adf668b213c2e..b0a6084f17573 100644
--- a/drivers/media/common/videobuf2/videobuf2-core.c
+++ b/drivers/media/common/videobuf2/videobuf2-core.c
@@ -2990,8 +2990,8 @@ static int __vb2_cleanup_fileio(struct vb2_queue *q)
  * @nonblock:	mode selector (1 means blocking calls, 0 means nonblocking)
  * @read:	access mode selector (1 means read, 0 means write)
  */
-static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count,
-		loff_t *ppos, int nonblock, int read)
+static ssize_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_t count,
+				    loff_t *ppos, int nonblock, int read)
 {
 	struct vb2_fileio_data *fileio;
 	struct vb2_fileio_buf *buf;
@@ -3154,15 +3154,15 @@ static size_t __vb2_perform_fileio(struct vb2_queue *q, char __user *data, size_
 	return ret;
 }
 
-size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
-		loff_t *ppos, int nonblocking)
+ssize_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
+		 loff_t *ppos, int nonblocking)
 {
 	return __vb2_perform_fileio(q, data, count, ppos, nonblocking, 1);
 }
 EXPORT_SYMBOL_GPL(vb2_read);
 
-size_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
-		loff_t *ppos, int nonblocking)
+ssize_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
+		  loff_t *ppos, int nonblocking)
 {
 	return __vb2_perform_fileio(q, (char __user *) data, count,
 							ppos, nonblocking, 0);
diff --git a/drivers/media/dvb-frontends/a8293.c b/drivers/media/dvb-frontends/a8293.c
index 7c0963054a8f8..52e3dc9283270 100644
--- a/drivers/media/dvb-frontends/a8293.c
+++ b/drivers/media/dvb-frontends/a8293.c
@@ -256,8 +256,8 @@ static void a8293_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id a8293_id_table[] = {
-	{ "a8293" },
-	{}
+	{ .name = "a8293" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, a8293_id_table);
 
diff --git a/drivers/media/dvb-frontends/af9013.c b/drivers/media/dvb-frontends/af9013.c
index 75f3063c193e7..d335bfd18e440 100644
--- a/drivers/media/dvb-frontends/af9013.c
+++ b/drivers/media/dvb-frontends/af9013.c
@@ -1553,8 +1553,8 @@ static void af9013_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id af9013_id_table[] = {
-	{ "af9013" },
-	{}
+	{ .name = "af9013" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, af9013_id_table);
 
diff --git a/drivers/media/dvb-frontends/af9033.c b/drivers/media/dvb-frontends/af9033.c
index 9a3a4d6513dde..01761861b01f4 100644
--- a/drivers/media/dvb-frontends/af9033.c
+++ b/drivers/media/dvb-frontends/af9033.c
@@ -1173,8 +1173,8 @@ static void af9033_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id af9033_id_table[] = {
-	{ "af9033" },
-	{}
+	{ .name = "af9033" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, af9033_id_table);
 
diff --git a/drivers/media/dvb-frontends/au8522_decoder.c b/drivers/media/dvb-frontends/au8522_decoder.c
index 58b959b272c6f..9ae47ea3976b8 100644
--- a/drivers/media/dvb-frontends/au8522_decoder.c
+++ b/drivers/media/dvb-frontends/au8522_decoder.c
@@ -768,8 +768,8 @@ static void au8522_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id au8522_id[] = {
-	{ "au8522" },
-	{}
+	{ .name = "au8522" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, au8522_id);
diff --git a/drivers/media/dvb-frontends/cxd2099.c b/drivers/media/dvb-frontends/cxd2099.c
index f95950a613074..e6110af1edcdf 100644
--- a/drivers/media/dvb-frontends/cxd2099.c
+++ b/drivers/media/dvb-frontends/cxd2099.c
@@ -672,8 +672,8 @@ static void cxd2099_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id cxd2099_id[] = {
-	{ "cxd2099" },
-	{}
+	{ .name = "cxd2099" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, cxd2099_id);
 
diff --git a/drivers/media/dvb-frontends/cxd2820r_core.c b/drivers/media/dvb-frontends/cxd2820r_core.c
index 3deefeff4bd19..fbbffe9e72515 100644
--- a/drivers/media/dvb-frontends/cxd2820r_core.c
+++ b/drivers/media/dvb-frontends/cxd2820r_core.c
@@ -723,8 +723,8 @@ static void cxd2820r_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id cxd2820r_id_table[] = {
-	{ "cxd2820r" },
-	{}
+	{ .name = "cxd2820r" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, cxd2820r_id_table);
 
diff --git a/drivers/media/dvb-frontends/dvb-pll.c b/drivers/media/dvb-frontends/dvb-pll.c
index f8fd23f60e3fb..d36c163b772fc 100644
--- a/drivers/media/dvb-frontends/dvb-pll.c
+++ b/drivers/media/dvb-frontends/dvb-pll.c
@@ -911,28 +911,28 @@ static void dvb_pll_remove(struct i2c_client *client)
 
 
 static const struct i2c_device_id dvb_pll_id[] = {
-	{"dtt7579",		DVB_PLL_THOMSON_DTT7579},
-	{"dtt759x",		DVB_PLL_THOMSON_DTT759X},
-	{"z201",		DVB_PLL_LG_Z201},
-	{"unknown_1",		DVB_PLL_UNKNOWN_1},
-	{"tua6010xs",		DVB_PLL_TUA6010XS},
-	{"env57h1xd5",		DVB_PLL_ENV57H1XD5},
-	{"tua6034",		DVB_PLL_TUA6034},
-	{"tda665x",		DVB_PLL_TDA665X},
-	{"tded4",		DVB_PLL_TDED4},
-	{"tdhu2",		DVB_PLL_TDHU2},
-	{"tbmv",		DVB_PLL_SAMSUNG_TBMV},
-	{"sd1878_tda8261",	DVB_PLL_PHILIPS_SD1878_TDA8261},
-	{"opera1",		DVB_PLL_OPERA1},
-	{"dtos403ih102a",	DVB_PLL_SAMSUNG_DTOS403IH102A},
-	{"tdtc9251dh0",		DVB_PLL_SAMSUNG_TDTC9251DH0},
-	{"tbdu18132",		DVB_PLL_SAMSUNG_TBDU18132},
-	{"tbmu24112",		DVB_PLL_SAMSUNG_TBMU24112},
-	{"tdee4",		DVB_PLL_TDEE4},
-	{"dtt7520x",		DVB_PLL_THOMSON_DTT7520X},
-	{"tua6034_friio",	DVB_PLL_TUA6034_FRIIO},
-	{"tda665x_earthpt1",	DVB_PLL_TDA665X_EARTH_PT1},
-	{}
+	{ .name = "dtt7579",             .driver_data = DVB_PLL_THOMSON_DTT7579 },
+	{ .name = "dtt759x",             .driver_data = DVB_PLL_THOMSON_DTT759X },
+	{ .name = "z201",                .driver_data = DVB_PLL_LG_Z201 },
+	{ .name = "unknown_1",           .driver_data = DVB_PLL_UNKNOWN_1 },
+	{ .name = "tua6010xs",           .driver_data = DVB_PLL_TUA6010XS },
+	{ .name = "env57h1xd5",          .driver_data = DVB_PLL_ENV57H1XD5 },
+	{ .name = "tua6034",             .driver_data = DVB_PLL_TUA6034 },
+	{ .name = "tda665x",             .driver_data = DVB_PLL_TDA665X },
+	{ .name = "tded4",               .driver_data = DVB_PLL_TDED4 },
+	{ .name = "tdhu2",               .driver_data = DVB_PLL_TDHU2 },
+	{ .name = "tbmv",                .driver_data = DVB_PLL_SAMSUNG_TBMV },
+	{ .name = "sd1878_tda8261",      .driver_data = DVB_PLL_PHILIPS_SD1878_TDA8261 },
+	{ .name = "opera1",              .driver_data = DVB_PLL_OPERA1 },
+	{ .name = "dtos403ih102a",       .driver_data = DVB_PLL_SAMSUNG_DTOS403IH102A },
+	{ .name = "tdtc9251dh0",         .driver_data = DVB_PLL_SAMSUNG_TDTC9251DH0 },
+	{ .name = "tbdu18132",           .driver_data = DVB_PLL_SAMSUNG_TBDU18132 },
+	{ .name = "tbmu24112",           .driver_data = DVB_PLL_SAMSUNG_TBMU24112 },
+	{ .name = "tdee4",               .driver_data = DVB_PLL_TDEE4 },
+	{ .name = "dtt7520x",            .driver_data = DVB_PLL_THOMSON_DTT7520X },
+	{ .name = "tua6034_friio",       .driver_data = DVB_PLL_TUA6034_FRIIO },
+	{ .name = "tda665x_earthpt1",    .driver_data = DVB_PLL_TDA665X_EARTH_PT1 },
+	{ }
 };
 
 
diff --git a/drivers/media/dvb-frontends/helene.c b/drivers/media/dvb-frontends/helene.c
index 1402d124544ef..993280fefc2ce 100644
--- a/drivers/media/dvb-frontends/helene.c
+++ b/drivers/media/dvb-frontends/helene.c
@@ -1101,8 +1101,8 @@ static int helene_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id helene_id[] = {
-	{ "helene", },
-	{}
+	{ .name = "helene" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, helene_id);
 
diff --git a/drivers/media/dvb-frontends/lgdt3306a.c b/drivers/media/dvb-frontends/lgdt3306a.c
index b6a66e122ed5a..f4a3136baea3a 100644
--- a/drivers/media/dvb-frontends/lgdt3306a.c
+++ b/drivers/media/dvb-frontends/lgdt3306a.c
@@ -2244,8 +2244,8 @@ static void lgdt3306a_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lgdt3306a_id_table[] = {
-	{ "lgdt3306a" },
-	{}
+	{ .name = "lgdt3306a" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lgdt3306a_id_table);
 
diff --git a/drivers/media/dvb-frontends/lgdt330x.c b/drivers/media/dvb-frontends/lgdt330x.c
index 19a4a05b34995..d90f642e9237c 100644
--- a/drivers/media/dvb-frontends/lgdt330x.c
+++ b/drivers/media/dvb-frontends/lgdt330x.c
@@ -983,8 +983,8 @@ static void lgdt330x_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lgdt330x_id_table[] = {
-	{ "lgdt330x" },
-	{}
+	{ .name = "lgdt330x" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, lgdt330x_id_table);
 
diff --git a/drivers/media/dvb-frontends/m88ds3103.c b/drivers/media/dvb-frontends/m88ds3103.c
index 44bee1f3c5e96..d79b88848a8c3 100644
--- a/drivers/media/dvb-frontends/m88ds3103.c
+++ b/drivers/media/dvb-frontends/m88ds3103.c
@@ -2221,11 +2221,11 @@ static void m88ds3103_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id m88ds3103_id_table[] = {
-	{"m88ds3103",  M88DS3103_CHIPTYPE_3103},
-	{"m88rs6000",  M88DS3103_CHIPTYPE_RS6000},
-	{"m88ds3103b", M88DS3103_CHIPTYPE_3103B},
-	{"m88ds3103c", M88DS3103_CHIPTYPE_3103C},
-	{}
+	{ .name = "m88ds3103", .driver_data = M88DS3103_CHIPTYPE_3103 },
+	{ .name = "m88rs6000", .driver_data = M88DS3103_CHIPTYPE_RS6000 },
+	{ .name = "m88ds3103b", .driver_data = M88DS3103_CHIPTYPE_3103B },
+	{ .name = "m88ds3103c", .driver_data = M88DS3103_CHIPTYPE_3103C },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, m88ds3103_id_table);
 
diff --git a/drivers/media/dvb-frontends/mn88443x.c b/drivers/media/dvb-frontends/mn88443x.c
index 818c4e67364c7..cc6bd17daf73b 100644
--- a/drivers/media/dvb-frontends/mn88443x.c
+++ b/drivers/media/dvb-frontends/mn88443x.c
@@ -787,10 +787,10 @@ static const struct of_device_id mn88443x_of_match[] = {
 MODULE_DEVICE_TABLE(of, mn88443x_of_match);
 
 static const struct i2c_device_id mn88443x_i2c_id[] = {
-	{ "mn884433",   (kernel_ulong_t)&mn88443x_spec_pri },
-	{ "mn884434-0", (kernel_ulong_t)&mn88443x_spec_pri },
-	{ "mn884434-1", (kernel_ulong_t)&mn88443x_spec_sec },
-	{}
+	{ .name = "mn884433", .driver_data = (kernel_ulong_t)&mn88443x_spec_pri },
+	{ .name = "mn884434-0", .driver_data = (kernel_ulong_t)&mn88443x_spec_pri },
+	{ .name = "mn884434-1", .driver_data = (kernel_ulong_t)&mn88443x_spec_sec },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mn88443x_i2c_id);
 
diff --git a/drivers/media/dvb-frontends/mn88472.c b/drivers/media/dvb-frontends/mn88472.c
index 275c404ce2864..42b78b9aba205 100644
--- a/drivers/media/dvb-frontends/mn88472.c
+++ b/drivers/media/dvb-frontends/mn88472.c
@@ -708,8 +708,8 @@ static void mn88472_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mn88472_id_table[] = {
-	{ "mn88472" },
-	{}
+	{ .name = "mn88472" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mn88472_id_table);
 
diff --git a/drivers/media/dvb-frontends/mn88473.c b/drivers/media/dvb-frontends/mn88473.c
index 40a0cb1d9c675..b7b11a000969a 100644
--- a/drivers/media/dvb-frontends/mn88473.c
+++ b/drivers/media/dvb-frontends/mn88473.c
@@ -743,8 +743,8 @@ static void mn88473_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mn88473_id_table[] = {
-	{ "mn88473" },
-	{}
+	{ .name = "mn88473" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mn88473_id_table);
 
diff --git a/drivers/media/dvb-frontends/mxl692.c b/drivers/media/dvb-frontends/mxl692.c
index 2d8eaa20723f0..bca4a4c3dd663 100644
--- a/drivers/media/dvb-frontends/mxl692.c
+++ b/drivers/media/dvb-frontends/mxl692.c
@@ -1346,8 +1346,8 @@ static void mxl692_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mxl692_id_table[] = {
-	{ "mxl692" },
-	{}
+	{ .name = "mxl692" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mxl692_id_table);
 
diff --git a/drivers/media/dvb-frontends/rtl2830.c b/drivers/media/dvb-frontends/rtl2830.c
index f0ee7c38ee79a..4c44e2d695f82 100644
--- a/drivers/media/dvb-frontends/rtl2830.c
+++ b/drivers/media/dvb-frontends/rtl2830.c
@@ -876,8 +876,8 @@ static void rtl2830_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id rtl2830_id_table[] = {
-	{ "rtl2830" },
-	{}
+	{ .name = "rtl2830" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, rtl2830_id_table);
 
diff --git a/drivers/media/dvb-frontends/rtl2832.c b/drivers/media/dvb-frontends/rtl2832.c
index d8e1546aea5ec..b31cdb44b383e 100644
--- a/drivers/media/dvb-frontends/rtl2832.c
+++ b/drivers/media/dvb-frontends/rtl2832.c
@@ -1115,18 +1115,18 @@ static void rtl2832_remove(struct i2c_client *client)
 
 	dev_dbg(&client->dev, "\n");
 
-	cancel_delayed_work_sync(&dev->i2c_gate_work);
-
 	i2c_mux_del_adapters(dev->muxc);
 
+	cancel_delayed_work_sync(&dev->i2c_gate_work);
+
 	regmap_exit(dev->regmap);
 
 	kfree(dev);
 }
 
 static const struct i2c_device_id rtl2832_id_table[] = {
-	{ "rtl2832" },
-	{}
+	{ .name = "rtl2832" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, rtl2832_id_table);
 
diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c
index 422d1a7b5456e..c564485e3bbb5 100644
--- a/drivers/media/dvb-frontends/rtl2832_sdr.c
+++ b/drivers/media/dvb-frontends/rtl2832_sdr.c
@@ -399,7 +399,8 @@ static int rtl2832_sdr_alloc_urbs(struct rtl2832_sdr_dev *dev)
 }
 
 /* Must be called with vb_queue_lock hold */
-static void rtl2832_sdr_cleanup_queued_bufs(struct rtl2832_sdr_dev *dev)
+static void rtl2832_sdr_cleanup_queued_bufs(struct rtl2832_sdr_dev *dev,
+					    enum vb2_buffer_state state)
 {
 	struct platform_device *pdev = dev->pdev;
 	unsigned long flags;
@@ -413,7 +414,7 @@ static void rtl2832_sdr_cleanup_queued_bufs(struct rtl2832_sdr_dev *dev)
 		buf = list_entry(dev->queued_bufs.next,
 				struct rtl2832_sdr_frame_buf, list);
 		list_del(&buf->list);
-		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
 	}
 	spin_unlock_irqrestore(&dev->queued_bufs_lock, flags);
 }
@@ -855,11 +856,15 @@ static int rtl2832_sdr_start_streaming(struct vb2_queue *vq, unsigned int count)
 
 	dev_dbg(&pdev->dev, "\n");
 
-	if (!dev->udev)
+	if (!dev->udev) {
+		rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED);
 		return -ENODEV;
+	}
 
-	if (mutex_lock_interruptible(&dev->v4l2_lock))
+	if (mutex_lock_interruptible(&dev->v4l2_lock)) {
+		rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED);
 		return -ERESTARTSYS;
+	}
 
 	if (d->props->power_ctrl)
 		d->props->power_ctrl(d, 1);
@@ -900,7 +905,11 @@ static int rtl2832_sdr_start_streaming(struct vb2_queue *vq, unsigned int count)
 	if (ret)
 		goto err;
 
+	mutex_unlock(&dev->v4l2_lock);
+	return 0;
+
 err:
+	rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED);
 	mutex_unlock(&dev->v4l2_lock);
 
 	return ret;
@@ -920,7 +929,7 @@ static void rtl2832_sdr_stop_streaming(struct vb2_queue *vq)
 	rtl2832_sdr_kill_urbs(dev);
 	rtl2832_sdr_free_urbs(dev);
 	rtl2832_sdr_free_stream_bufs(dev);
-	rtl2832_sdr_cleanup_queued_bufs(dev);
+	rtl2832_sdr_cleanup_queued_bufs(dev, VB2_BUF_STATE_ERROR);
 	rtl2832_sdr_unset_adc(dev);
 
 	/* sleep tuner */
diff --git a/drivers/media/dvb-frontends/si2165.c b/drivers/media/dvb-frontends/si2165.c
index 4170696af9f05..f1241b63aa5ce 100644
--- a/drivers/media/dvb-frontends/si2165.c
+++ b/drivers/media/dvb-frontends/si2165.c
@@ -1281,8 +1281,8 @@ static void si2165_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id si2165_id_table[] = {
-	{ "si2165" },
-	{}
+	{ .name = "si2165" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, si2165_id_table);
 
diff --git a/drivers/media/dvb-frontends/si2168.c b/drivers/media/dvb-frontends/si2168.c
index 9c5bac8cda477..8bc3b6eb1dd3b 100644
--- a/drivers/media/dvb-frontends/si2168.c
+++ b/drivers/media/dvb-frontends/si2168.c
@@ -790,8 +790,8 @@ static void si2168_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id si2168_id_table[] = {
-	{ "si2168" },
-	{}
+	{ .name = "si2168" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, si2168_id_table);
 
diff --git a/drivers/media/dvb-frontends/sp2.c b/drivers/media/dvb-frontends/sp2.c
index 071c7371c699f..63dec4b4fd2fe 100644
--- a/drivers/media/dvb-frontends/sp2.c
+++ b/drivers/media/dvb-frontends/sp2.c
@@ -407,8 +407,8 @@ static void sp2_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id sp2_id[] = {
-	{ "sp2" },
-	{}
+	{ .name = "sp2" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sp2_id);
 
diff --git a/drivers/media/dvb-frontends/stv090x.c b/drivers/media/dvb-frontends/stv090x.c
index 657df713865e6..932bbed5497aa 100644
--- a/drivers/media/dvb-frontends/stv090x.c
+++ b/drivers/media/dvb-frontends/stv090x.c
@@ -5079,8 +5079,8 @@ struct dvb_frontend *stv090x_attach(struct stv090x_config *config,
 EXPORT_SYMBOL_GPL(stv090x_attach);
 
 static const struct i2c_device_id stv090x_id_table[] = {
-	{ "stv090x" },
-	{}
+	{ .name = "stv090x" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, stv090x_id_table);
 
diff --git a/drivers/media/dvb-frontends/stv6110x.c b/drivers/media/dvb-frontends/stv6110x.c
index 7506f39ead55d..5075333d7ffcd 100644
--- a/drivers/media/dvb-frontends/stv6110x.c
+++ b/drivers/media/dvb-frontends/stv6110x.c
@@ -470,8 +470,8 @@ const struct stv6110x_devctl *stv6110x_attach(struct dvb_frontend *fe,
 EXPORT_SYMBOL_GPL(stv6110x_attach);
 
 static const struct i2c_device_id stv6110x_id_table[] = {
-	{ "stv6110x" },
-	{}
+	{ .name = "stv6110x" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, stv6110x_id_table);
 
diff --git a/drivers/media/dvb-frontends/tc90522.c b/drivers/media/dvb-frontends/tc90522.c
index f1343ba67b970..9a5ffb5f70fc0 100644
--- a/drivers/media/dvb-frontends/tc90522.c
+++ b/drivers/media/dvb-frontends/tc90522.c
@@ -830,9 +830,9 @@ static void tc90522_remove(struct i2c_client *client)
 
 
 static const struct i2c_device_id tc90522_id[] = {
-	{ TC90522_I2C_DEV_SAT, 0 },
-	{ TC90522_I2C_DEV_TER, 1 },
-	{}
+	{ .name = TC90522_I2C_DEV_SAT, .driver_data = 0 },
+	{ .name = TC90522_I2C_DEV_TER, .driver_data = 1 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tc90522_id);
 
diff --git a/drivers/media/dvb-frontends/tda10071.c b/drivers/media/dvb-frontends/tda10071.c
index 6e6c2e5c427e7..7a635c86b86cc 100644
--- a/drivers/media/dvb-frontends/tda10071.c
+++ b/drivers/media/dvb-frontends/tda10071.c
@@ -1230,8 +1230,8 @@ static void tda10071_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tda10071_id_table[] = {
-	{ "tda10071_cx24118" },
-	{}
+	{ .name = "tda10071_cx24118" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tda10071_id_table);
 
diff --git a/drivers/media/dvb-frontends/ts2020.c b/drivers/media/dvb-frontends/ts2020.c
index 8b6006635e491..8775083f4dd6e 100644
--- a/drivers/media/dvb-frontends/ts2020.c
+++ b/drivers/media/dvb-frontends/ts2020.c
@@ -716,9 +716,9 @@ static void ts2020_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ts2020_id_table[] = {
-	{ "ts2020" },
-	{ "ts2022" },
-	{}
+	{ .name = "ts2020" },
+	{ .name = "ts2022" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ts2020_id_table);
 
diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig
index 8f2ba4121586e..a3ab48607dcf0 100644
--- a/drivers/media/i2c/Kconfig
+++ b/drivers/media/i2c/Kconfig
@@ -237,7 +237,7 @@ config VIDEO_IMX319
 
 config VIDEO_IMX334
 	tristate "Sony IMX334 sensor support"
-	depends on OF_GPIO
+	depends on OF
 	select V4L2_CCI_I2C
 	help
 	  This is a Video4Linux2 sensor driver for the Sony
@@ -248,7 +248,7 @@ config VIDEO_IMX334
 
 config VIDEO_IMX335
 	tristate "Sony IMX335 sensor support"
-	depends on OF_GPIO
+	depends on OF
 	select V4L2_CCI_I2C
 	help
 	  This is a Video4Linux2 sensor driver for the Sony
@@ -268,7 +268,7 @@ config VIDEO_IMX355
 
 config VIDEO_IMX412
 	tristate "Sony IMX412 sensor support"
-	depends on OF_GPIO
+	depends on OF
 	help
 	  This is a Video4Linux2 sensor driver for the Sony
 	  IMX412 camera.
@@ -278,7 +278,7 @@ config VIDEO_IMX412
 
 config VIDEO_IMX415
 	tristate "Sony IMX415 sensor support"
-	depends on OF_GPIO
+	depends on OF
 	select V4L2_CCI_I2C
 	help
 	  This is a Video4Linux2 sensor driver for the Sony
@@ -703,7 +703,7 @@ config VIDEO_OV8865
 
 config VIDEO_OV9282
 	tristate "OmniVision OV9282 sensor support"
-	depends on OF_GPIO
+	depends on OF
 	select V4L2_CCI_I2C
 	help
 	  This is a Video4Linux2 sensor driver for the OmniVision
@@ -1271,7 +1271,6 @@ config VIDEO_BT866
 config VIDEO_ISL7998X
 	tristate "Intersil ISL7998x video decoder"
 	depends on VIDEO_DEV && I2C
-	depends on OF_GPIO
 	select MEDIA_CONTROLLER
 	select VIDEO_V4L2_SUBDEV_API
 	select V4L2_FWNODE
@@ -1309,7 +1308,6 @@ config VIDEO_MAX9286
 	tristate "Maxim MAX9286 GMSL deserializer support"
 	depends on I2C && I2C_MUX
 	depends on VIDEO_DEV
-	depends on OF_GPIO
 	select V4L2_FWNODE
 	select VIDEO_V4L2_SUBDEV_API
 	select MEDIA_CONTROLLER
diff --git a/drivers/media/i2c/ad5820.c b/drivers/media/i2c/ad5820.c
index f60271082fb5b..fcce2857b69ef 100644
--- a/drivers/media/i2c/ad5820.c
+++ b/drivers/media/i2c/ad5820.c
@@ -347,8 +347,8 @@ static void ad5820_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ad5820_id_table[] = {
-	{ "ad5820" },
-	{ "ad5821" },
+	{ .name = "ad5820" },
+	{ .name = "ad5821" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ad5820_id_table);
diff --git a/drivers/media/i2c/adp1653.c b/drivers/media/i2c/adp1653.c
index 391bc75bfcd0f..177a6e8d7fb8b 100644
--- a/drivers/media/i2c/adp1653.c
+++ b/drivers/media/i2c/adp1653.c
@@ -522,7 +522,7 @@ static void adp1653_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adp1653_id_table[] = {
-	{ ADP1653_NAME },
+	{ .name = ADP1653_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adp1653_id_table);
diff --git a/drivers/media/i2c/adv7170.c b/drivers/media/i2c/adv7170.c
index ef8682b980b44..812998729207b 100644
--- a/drivers/media/i2c/adv7170.c
+++ b/drivers/media/i2c/adv7170.c
@@ -377,8 +377,8 @@ static void adv7170_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id adv7170_id[] = {
-	{ "adv7170" },
-	{ "adv7171" },
+	{ .name = "adv7170" },
+	{ .name = "adv7171" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adv7170_id);
diff --git a/drivers/media/i2c/adv7175.c b/drivers/media/i2c/adv7175.c
index 384da1ec5bf9d..f1caab8e2abd9 100644
--- a/drivers/media/i2c/adv7175.c
+++ b/drivers/media/i2c/adv7175.c
@@ -432,8 +432,8 @@ static void adv7175_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id adv7175_id[] = {
-	{ "adv7175" },
-	{ "adv7176" },
+	{ .name = "adv7175" },
+	{ .name = "adv7176" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adv7175_id);
diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c
index 669b0b3165b1c..e5d11a6e6766d 100644
--- a/drivers/media/i2c/adv7180.c
+++ b/drivers/media/i2c/adv7180.c
@@ -1626,18 +1626,18 @@ static SIMPLE_DEV_PM_OPS(adv7180_pm_ops, adv7180_suspend, adv7180_resume);
 #endif
 
 static const struct i2c_device_id adv7180_id[] = {
-	{ "adv7180", (kernel_ulong_t)&adv7180_info },
-	{ "adv7180cp", (kernel_ulong_t)&adv7180_info },
-	{ "adv7180st", (kernel_ulong_t)&adv7180_info },
-	{ "adv7182", (kernel_ulong_t)&adv7182_info },
-	{ "adv7280", (kernel_ulong_t)&adv7280_info },
-	{ "adv7280-m", (kernel_ulong_t)&adv7280_m_info },
-	{ "adv7281", (kernel_ulong_t)&adv7281_info },
-	{ "adv7281-m", (kernel_ulong_t)&adv7281_m_info },
-	{ "adv7281-ma", (kernel_ulong_t)&adv7281_ma_info },
-	{ "adv7282", (kernel_ulong_t)&adv7282_info },
-	{ "adv7282-m", (kernel_ulong_t)&adv7282_m_info },
-	{}
+	{ .name = "adv7180", .driver_data = (kernel_ulong_t)&adv7180_info },
+	{ .name = "adv7180cp", .driver_data = (kernel_ulong_t)&adv7180_info },
+	{ .name = "adv7180st", .driver_data = (kernel_ulong_t)&adv7180_info },
+	{ .name = "adv7182", .driver_data = (kernel_ulong_t)&adv7182_info },
+	{ .name = "adv7280", .driver_data = (kernel_ulong_t)&adv7280_info },
+	{ .name = "adv7280-m", .driver_data = (kernel_ulong_t)&adv7280_m_info },
+	{ .name = "adv7281", .driver_data = (kernel_ulong_t)&adv7281_info },
+	{ .name = "adv7281-m", .driver_data = (kernel_ulong_t)&adv7281_m_info },
+	{ .name = "adv7281-ma", .driver_data = (kernel_ulong_t)&adv7281_ma_info },
+	{ .name = "adv7282", .driver_data = (kernel_ulong_t)&adv7282_info },
+	{ .name = "adv7282-m", .driver_data = (kernel_ulong_t)&adv7282_m_info },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adv7180_id);
 
diff --git a/drivers/media/i2c/adv7183.c b/drivers/media/i2c/adv7183.c
index 25a31a6dd456d..a04a1a205fe0d 100644
--- a/drivers/media/i2c/adv7183.c
+++ b/drivers/media/i2c/adv7183.c
@@ -619,8 +619,8 @@ static void adv7183_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adv7183_id[] = {
-	{ "adv7183" },
-	{}
+	{ .name = "adv7183" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, adv7183_id);
diff --git a/drivers/media/i2c/adv7343.c b/drivers/media/i2c/adv7343.c
index b96443404a269..9b91d7073d3c8 100644
--- a/drivers/media/i2c/adv7343.c
+++ b/drivers/media/i2c/adv7343.c
@@ -502,8 +502,8 @@ static void adv7343_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adv7343_id[] = {
-	{ "adv7343" },
-	{}
+	{ .name = "adv7343" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, adv7343_id);
diff --git a/drivers/media/i2c/adv7393.c b/drivers/media/i2c/adv7393.c
index c7994bd0bbd46..6f948ba02f864 100644
--- a/drivers/media/i2c/adv7393.c
+++ b/drivers/media/i2c/adv7393.c
@@ -446,8 +446,8 @@ static void adv7393_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id adv7393_id[] = {
-	{ "adv7393" },
-	{}
+	{ .name = "adv7393" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adv7393_id);
 
diff --git a/drivers/media/i2c/adv7511-v4l2.c b/drivers/media/i2c/adv7511-v4l2.c
index 853c7806de928..860cff50c522d 100644
--- a/drivers/media/i2c/adv7511-v4l2.c
+++ b/drivers/media/i2c/adv7511-v4l2.c
@@ -2008,7 +2008,7 @@ static void adv7511_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id adv7511_id[] = {
-	{ "adv7511-v4l2" },
+	{ .name = "adv7511-v4l2" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adv7511_id);
diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c
index 67116a4ef134e..ac9c69ce438f1 100644
--- a/drivers/media/i2c/adv7604.c
+++ b/drivers/media/i2c/adv7604.c
@@ -3236,10 +3236,10 @@ static const struct adv76xx_chip_info adv76xx_chip_info[] = {
 };
 
 static const struct i2c_device_id adv76xx_i2c_id[] = {
-	{ "adv7604", (kernel_ulong_t)&adv76xx_chip_info[ADV7604] },
-	{ "adv7610", (kernel_ulong_t)&adv76xx_chip_info[ADV7611] },
-	{ "adv7611", (kernel_ulong_t)&adv76xx_chip_info[ADV7611] },
-	{ "adv7612", (kernel_ulong_t)&adv76xx_chip_info[ADV7612] },
+	{ .name = "adv7604", .driver_data = (kernel_ulong_t)&adv76xx_chip_info[ADV7604] },
+	{ .name = "adv7610", .driver_data = (kernel_ulong_t)&adv76xx_chip_info[ADV7611] },
+	{ .name = "adv7611", .driver_data = (kernel_ulong_t)&adv76xx_chip_info[ADV7611] },
+	{ .name = "adv7612", .driver_data = (kernel_ulong_t)&adv76xx_chip_info[ADV7612] },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adv76xx_i2c_id);
@@ -3668,6 +3668,12 @@ static int adv76xx_probe(struct i2c_client *client)
 
 	state->source_pad = state->info->num_dv_ports
 			  + (state->info->has_afe ? 2 : 0);
+	if (WARN_ON(state->source_pad >= ADV76XX_PAD_MAX)) {
+		err = -EINVAL;
+		v4l2_err(sd, "invalid chip info\n");
+		goto err_i2c;
+	}
+
 	for (i = 0; i < state->source_pad; ++i)
 		state->pads[i].flags = MEDIA_PAD_FL_SINK;
 	state->pads[state->source_pad].flags = MEDIA_PAD_FL_SOURCE;
diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c
index ea6966c0605e5..3cfae89ce944a 100644
--- a/drivers/media/i2c/adv7842.c
+++ b/drivers/media/i2c/adv7842.c
@@ -3675,7 +3675,7 @@ static void adv7842_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id adv7842_id[] = {
-	{ "adv7842" },
+	{ .name = "adv7842" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, adv7842_id);
diff --git a/drivers/media/i2c/ak881x.c b/drivers/media/i2c/ak881x.c
index ee575d01a6761..cea46f01997da 100644
--- a/drivers/media/i2c/ak881x.c
+++ b/drivers/media/i2c/ak881x.c
@@ -304,8 +304,8 @@ static void ak881x_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ak881x_id[] = {
-	{ "ak8813" },
-	{ "ak8814" },
+	{ .name = "ak8813" },
+	{ .name = "ak8814" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ak881x_id);
diff --git a/drivers/media/i2c/alvium-csi2.c b/drivers/media/i2c/alvium-csi2.c
index 955b7072a5605..dd991c2ee7002 100644
--- a/drivers/media/i2c/alvium-csi2.c
+++ b/drivers/media/i2c/alvium-csi2.c
@@ -2100,20 +2100,21 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
 					      V4L2_CID_PIXEL_RATE, 0,
 					      ALVIUM_DEFAULT_PIXEL_RATE_MHZ, 1,
 					      ALVIUM_DEFAULT_PIXEL_RATE_MHZ);
-	ctrls->pixel_rate->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
 	/* Link freq is fixed */
 	ctrls->link_freq = v4l2_ctrl_new_int_menu(hdl, ops,
 						  V4L2_CID_LINK_FREQ,
 						  0, 0, &alvium->link_freq);
-	ctrls->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
+	if (ctrls->link_freq)
+		ctrls->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY;
 
 	/* Auto/manual white balance */
 	if (alvium->avail_ft.auto_whiteb) {
 		ctrls->auto_wb = v4l2_ctrl_new_std(hdl, ops,
 						   V4L2_CID_AUTO_WHITE_BALANCE,
 						   0, 1, 1, 1);
-		v4l2_ctrl_auto_cluster(3, &ctrls->auto_wb, 0, false);
+		if (ctrls->auto_wb)
+			v4l2_ctrl_auto_cluster(3, &ctrls->auto_wb, 0, false);
 	}
 
 	ctrls->blue_balance = v4l2_ctrl_new_std(hdl, ops,
@@ -2122,6 +2123,7 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
 						alvium->max_bbalance,
 						alvium->inc_bbalance,
 						alvium->dft_bbalance);
+
 	ctrls->red_balance = v4l2_ctrl_new_std(hdl, ops,
 					       V4L2_CID_RED_BALANCE,
 					       alvium->min_rbalance,
@@ -2136,7 +2138,9 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
 					       V4L2_CID_EXPOSURE_AUTO,
 					       V4L2_EXPOSURE_MANUAL, 0,
 					       V4L2_EXPOSURE_AUTO);
-		v4l2_ctrl_auto_cluster(2, &ctrls->auto_exp, 1, true);
+		if (ctrls->auto_exp)
+			v4l2_ctrl_auto_cluster(2, &ctrls->auto_exp,
+					       V4L2_EXPOSURE_MANUAL, true);
 	}
 
 	ctrls->exposure = v4l2_ctrl_new_std(hdl, ops,
@@ -2145,14 +2149,16 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
 					    alvium->max_exp,
 					    alvium->inc_exp,
 					    alvium->dft_exp);
-	ctrls->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE;
+	if (ctrls->exposure)
+		ctrls->exposure->flags |= V4L2_CTRL_FLAG_VOLATILE;
 
 	/* Auto/manual gain */
 	if (alvium->avail_ft.auto_gain) {
 		ctrls->auto_gain = v4l2_ctrl_new_std(hdl, ops,
 						     V4L2_CID_AUTOGAIN,
 						     0, 1, 1, 1);
-		v4l2_ctrl_auto_cluster(2, &ctrls->auto_gain, 0, true);
+		if (ctrls->auto_gain)
+			v4l2_ctrl_auto_cluster(2, &ctrls->auto_gain, 0, true);
 	}
 
 	if (alvium->avail_ft.gain) {
@@ -2162,7 +2168,8 @@ static int alvium_ctrl_init(struct alvium_dev *alvium)
 						alvium->max_gain,
 						alvium->inc_gain,
 						alvium->dft_gain);
-		ctrls->gain->flags |= V4L2_CTRL_FLAG_VOLATILE;
+		if (ctrls->gain)
+			ctrls->gain->flags |= V4L2_CTRL_FLAG_VOLATILE;
 	}
 
 	if (alvium->avail_ft.sat)
diff --git a/drivers/media/i2c/bt819.c b/drivers/media/i2c/bt819.c
index f97245f91f881..da44100062fa2 100644
--- a/drivers/media/i2c/bt819.c
+++ b/drivers/media/i2c/bt819.c
@@ -457,9 +457,9 @@ static void bt819_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id bt819_id[] = {
-	{ "bt819a" },
-	{ "bt817a" },
-	{ "bt815a" },
+	{ .name = "bt819a" },
+	{ .name = "bt817a" },
+	{ .name = "bt815a" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, bt819_id);
diff --git a/drivers/media/i2c/bt856.c b/drivers/media/i2c/bt856.c
index 6852aa47cafba..656719158bb45 100644
--- a/drivers/media/i2c/bt856.c
+++ b/drivers/media/i2c/bt856.c
@@ -230,7 +230,7 @@ static void bt856_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id bt856_id[] = {
-	{ "bt856" },
+	{ .name = "bt856" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, bt856_id);
diff --git a/drivers/media/i2c/bt866.c b/drivers/media/i2c/bt866.c
index a2cc34d35ed24..f5104c7d9f900 100644
--- a/drivers/media/i2c/bt866.c
+++ b/drivers/media/i2c/bt866.c
@@ -197,7 +197,7 @@ static void bt866_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id bt866_id[] = {
-	{ "bt866" },
+	{ .name = "bt866" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, bt866_id);
diff --git a/drivers/media/i2c/cs3308.c b/drivers/media/i2c/cs3308.c
index 3f2f993e16a67..ad37aeecf05e5 100644
--- a/drivers/media/i2c/cs3308.c
+++ b/drivers/media/i2c/cs3308.c
@@ -109,7 +109,7 @@ static void cs3308_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id cs3308_id[] = {
-	{ "cs3308" },
+	{ .name = "cs3308" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, cs3308_id);
diff --git a/drivers/media/i2c/cs5345.c b/drivers/media/i2c/cs5345.c
index 3a9797a50e828..49b8020fbea29 100644
--- a/drivers/media/i2c/cs5345.c
+++ b/drivers/media/i2c/cs5345.c
@@ -189,7 +189,7 @@ static void cs5345_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id cs5345_id[] = {
-	{ "cs5345" },
+	{ .name = "cs5345" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, cs5345_id);
diff --git a/drivers/media/i2c/cs53l32a.c b/drivers/media/i2c/cs53l32a.c
index c4cad3293905c..a894dcf6b5a96 100644
--- a/drivers/media/i2c/cs53l32a.c
+++ b/drivers/media/i2c/cs53l32a.c
@@ -200,7 +200,7 @@ static void cs53l32a_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id cs53l32a_id[] = {
-	{ "cs53l32a" },
+	{ .name = "cs53l32a" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, cs53l32a_id);
diff --git a/drivers/media/i2c/cx25840/cx25840-core.c b/drivers/media/i2c/cx25840/cx25840-core.c
index 69d5cc648c0fc..8110d40931d98 100644
--- a/drivers/media/i2c/cx25840/cx25840-core.c
+++ b/drivers/media/i2c/cx25840/cx25840-core.c
@@ -3989,7 +3989,7 @@ static void cx25840_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id cx25840_id[] = {
-	{ "cx25840" },
+	{ .name = "cx25840" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, cx25840_id);
diff --git a/drivers/media/i2c/ds90ub913.c b/drivers/media/i2c/ds90ub913.c
index 49aa5f4a172c0..6abb5e324ae19 100644
--- a/drivers/media/i2c/ds90ub913.c
+++ b/drivers/media/i2c/ds90ub913.c
@@ -872,8 +872,8 @@ static void ub913_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ub913_id[] = {
-	{ "ds90ub913a-q1" },
-	{}
+	{ .name = "ds90ub913a-q1" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ub913_id);
 
diff --git a/drivers/media/i2c/ds90ub953.c b/drivers/media/i2c/ds90ub953.c
index a8ab67f4137f3..d4228e1134ff8 100644
--- a/drivers/media/i2c/ds90ub953.c
+++ b/drivers/media/i2c/ds90ub953.c
@@ -1347,9 +1347,9 @@ static const struct ub953_hw_data ds90ub971_hw = {
 };
 
 static const struct i2c_device_id ub953_id[] = {
-	{ "ds90ub953-q1", (kernel_ulong_t)&ds90ub953_hw },
-	{ "ds90ub971-q1", (kernel_ulong_t)&ds90ub971_hw },
-	{}
+	{ .name = "ds90ub953-q1", .driver_data = (kernel_ulong_t)&ds90ub953_hw },
+	{ .name = "ds90ub971-q1", .driver_data = (kernel_ulong_t)&ds90ub971_hw },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ub953_id);
 
diff --git a/drivers/media/i2c/ds90ub960.c b/drivers/media/i2c/ds90ub960.c
index d50e977cf6ce4..15a9797b47ac9 100644
--- a/drivers/media/i2c/ds90ub960.c
+++ b/drivers/media/i2c/ds90ub960.c
@@ -5266,10 +5266,10 @@ static const struct ub960_hw_data ds90ub9702_hw = {
 };
 
 static const struct i2c_device_id ub960_id[] = {
-	{ "ds90ub954-q1", (kernel_ulong_t)&ds90ub954_hw },
-	{ "ds90ub960-q1", (kernel_ulong_t)&ds90ub960_hw },
-	{ "ds90ub9702-q1", (kernel_ulong_t)&ds90ub9702_hw },
-	{}
+	{ .name = "ds90ub954-q1", .driver_data = (kernel_ulong_t)&ds90ub954_hw },
+	{ .name = "ds90ub960-q1", .driver_data = (kernel_ulong_t)&ds90ub960_hw },
+	{ .name = "ds90ub9702-q1", .driver_data = (kernel_ulong_t)&ds90ub9702_hw },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ub960_id);
 
diff --git a/drivers/media/i2c/dw9714.c b/drivers/media/i2c/dw9714.c
index 3288de539452e..1d686b1d2fd78 100644
--- a/drivers/media/i2c/dw9714.c
+++ b/drivers/media/i2c/dw9714.c
@@ -307,7 +307,7 @@ static int __maybe_unused dw9714_vcm_resume(struct device *dev)
 }
 
 static const struct i2c_device_id dw9714_id_table[] = {
-	{ DW9714_NAME },
+	{ .name = DW9714_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, dw9714_id_table);
diff --git a/drivers/media/i2c/dw9719.c b/drivers/media/i2c/dw9719.c
index 59558335989ed..3b7ba88fd67c0 100644
--- a/drivers/media/i2c/dw9719.c
+++ b/drivers/media/i2c/dw9719.c
@@ -439,6 +439,15 @@ static void dw9719_remove(struct i2c_client *client)
 	pm_runtime_set_suspended(&client->dev);
 }
 
+static const struct i2c_device_id dw9719_id_table[] = {
+	{ .name = "dw9718s", .driver_data = (kernel_ulong_t)DW9718S },
+	{ .name = "dw9719", .driver_data = (kernel_ulong_t)DW9719 },
+	{ .name = "dw9761", .driver_data = (kernel_ulong_t)DW9761 },
+	{ .name = "dw9800k", .driver_data = (kernel_ulong_t)DW9800K },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, dw9719_id_table);
+
 static const struct of_device_id dw9719_of_table[] = {
 	{ .compatible = "dongwoon,dw9718s", .data = (const void *)DW9718S },
 	{ .compatible = "dongwoon,dw9719", .data = (const void *)DW9719 },
@@ -459,6 +468,7 @@ static struct i2c_driver dw9719_i2c_driver = {
 	},
 	.probe = dw9719_probe,
 	.remove = dw9719_remove,
+	.id_table = dw9719_id_table,
 };
 module_i2c_driver(dw9719_i2c_driver);
 
diff --git a/drivers/media/i2c/et8ek8/et8ek8_driver.c b/drivers/media/i2c/et8ek8/et8ek8_driver.c
index 50121c3e5b480..738e2801016a6 100644
--- a/drivers/media/i2c/et8ek8/et8ek8_driver.c
+++ b/drivers/media/i2c/et8ek8/et8ek8_driver.c
@@ -1485,7 +1485,7 @@ static const struct of_device_id et8ek8_of_table[] = {
 MODULE_DEVICE_TABLE(of, et8ek8_of_table);
 
 static const struct i2c_device_id et8ek8_id_table[] = {
-	{ ET8EK8_NAME },
+	{ .name = ET8EK8_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, et8ek8_id_table);
diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c
index 7da02ce5da154..223d3753cc93b 100644
--- a/drivers/media/i2c/imx219.c
+++ b/drivers/media/i2c/imx219.c
@@ -72,7 +72,7 @@
 
 /* V_TIMING internal */
 #define IMX219_REG_FRM_LENGTH_A		CCI_REG16(0x0160)
-#define IMX219_FLL_MAX			0xffff
+#define IMX219_FLL_MAX			0xfffe
 #define IMX219_VBLANK_MIN		32
 #define IMX219_REG_LINE_LENGTH_A	CCI_REG16(0x0162)
 #define IMX219_LLP_MIN			0x0d78
@@ -142,10 +142,10 @@
 /* IMX219 native and active pixel array size. */
 #define IMX219_NATIVE_WIDTH		3296U
 #define IMX219_NATIVE_HEIGHT		2480U
-#define IMX219_PIXEL_ARRAY_LEFT		8U
-#define IMX219_PIXEL_ARRAY_TOP		8U
-#define IMX219_PIXEL_ARRAY_WIDTH	3280U
-#define IMX219_PIXEL_ARRAY_HEIGHT	2464U
+#define IMX219_ACTIVE_AREA_LEFT		8U
+#define IMX219_ACTIVE_AREA_TOP		8U
+#define IMX219_ACTIVE_AREA_WIDTH	3280U
+#define IMX219_ACTIVE_AREA_HEIGHT	2464U
 
 /* Mode : resolution and related config&values */
 struct imx219_mode {
@@ -675,13 +675,13 @@ static int imx219_set_framefmt(struct imx219 *imx219,
 	bpp = imx219_get_format_bpp(format);
 
 	cci_write(imx219->regmap, IMX219_REG_X_ADD_STA_A,
-		  crop->left - IMX219_PIXEL_ARRAY_LEFT, &ret);
+		  crop->left - IMX219_ACTIVE_AREA_LEFT, &ret);
 	cci_write(imx219->regmap, IMX219_REG_X_ADD_END_A,
-		  crop->left - IMX219_PIXEL_ARRAY_LEFT + crop->width - 1, &ret);
+		  crop->left - IMX219_ACTIVE_AREA_LEFT + crop->width - 1, &ret);
 	cci_write(imx219->regmap, IMX219_REG_Y_ADD_STA_A,
-		  crop->top - IMX219_PIXEL_ARRAY_TOP, &ret);
+		  crop->top - IMX219_ACTIVE_AREA_TOP, &ret);
 	cci_write(imx219->regmap, IMX219_REG_Y_ADD_END_A,
-		  crop->top - IMX219_PIXEL_ARRAY_TOP + crop->height - 1, &ret);
+		  crop->top - IMX219_ACTIVE_AREA_TOP + crop->height - 1, &ret);
 
 	imx219_get_binning(state, &bin_h, &bin_v);
 	cci_write(imx219->regmap, IMX219_REG_BINNING_MODE_H, bin_h, &ret);
@@ -837,11 +837,9 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *crop;
 	u8 bin_h, bin_v, binning;
-	u32 prev_line_len;
 	int ret;
 
 	format = v4l2_subdev_state_get_format(state, 0);
-	prev_line_len = format->width + imx219->hblank->val;
 
 	/*
 	 * Adjust the requested format to match the closest mode. The Bayer
@@ -867,8 +865,8 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 	 * Use binning to maximize the crop rectangle size, and centre it in the
 	 * sensor.
 	 */
-	bin_h = min(IMX219_PIXEL_ARRAY_WIDTH / format->width, 2U);
-	bin_v = min(IMX219_PIXEL_ARRAY_HEIGHT / format->height, 2U);
+	bin_h = min(IMX219_ACTIVE_AREA_WIDTH / format->width, 2U);
+	bin_v = min(IMX219_ACTIVE_AREA_HEIGHT / format->height, 2U);
 
 	/* Ensure bin_h and bin_v are same to avoid 1:2 or 2:1 stretching */
 	binning = min(bin_h, bin_v);
@@ -882,7 +880,7 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 	if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
 		int exposure_max;
 		int exposure_def;
-		int hblank, llp_min;
+		int llp_min;
 		int pixel_rate;
 
 		/* Update limits and set FPS to default */
@@ -924,15 +922,8 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
 					       llp_min - mode->width);
 		if (ret)
 			return ret;
-		/*
-		 * Retain PPL setting from previous mode so that the
-		 * line time does not change on a mode change.
-		 * Limits have to be recomputed as the controls define
-		 * the blanking only, so PPL values need to have the
-		 * mode width subtracted.
-		 */
-		hblank = prev_line_len - mode->width;
-		ret = __v4l2_ctrl_s_ctrl(imx219->hblank, hblank);
+
+		ret = __v4l2_ctrl_s_ctrl(imx219->hblank, llp_min - mode->width);
 		if (ret)
 			return ret;
 
@@ -967,10 +958,10 @@ static int imx219_get_selection(struct v4l2_subdev *sd,
 
 	case V4L2_SEL_TGT_CROP_DEFAULT:
 	case V4L2_SEL_TGT_CROP_BOUNDS:
-		sel->r.top = IMX219_PIXEL_ARRAY_TOP;
-		sel->r.left = IMX219_PIXEL_ARRAY_LEFT;
-		sel->r.width = IMX219_PIXEL_ARRAY_WIDTH;
-		sel->r.height = IMX219_PIXEL_ARRAY_HEIGHT;
+		sel->r.top = IMX219_ACTIVE_AREA_TOP;
+		sel->r.left = IMX219_ACTIVE_AREA_LEFT;
+		sel->r.width = IMX219_ACTIVE_AREA_WIDTH;
+		sel->r.height = IMX219_ACTIVE_AREA_HEIGHT;
 
 		return 0;
 	}
diff --git a/drivers/media/i2c/imx274.c b/drivers/media/i2c/imx274.c
index 8ec78b60bea6d..76da647f9cf96 100644
--- a/drivers/media/i2c/imx274.c
+++ b/drivers/media/i2c/imx274.c
@@ -182,7 +182,7 @@ struct imx274_mode {
 };
 
 /*
- * imx274 test pattern related structure
+ * imx274 test pattern related enum
  */
 enum {
 	TEST_PATTERN_DISABLED = 0,
@@ -533,7 +533,7 @@ static const struct imx274_mode imx274_modes[] = {
 /*
  * struct imx274_ctrls - imx274 ctrl structure
  * @handler: V4L2 ctrl handler structure
- * @exposure: Pointer to expsure ctrl structure
+ * @exposure: Pointer to exposure ctrl structure
  * @gain: Pointer to gain ctrl structure
  * @vflip: Pointer to vflip ctrl structure
  * @test_pattern: Pointer to test pattern ctrl structure
@@ -547,7 +547,7 @@ struct imx274_ctrls {
 };
 
 /*
- * struct stim274 - imx274 device structure
+ * struct stimx274 - imx274 device structure
  * @sd: V4L2 subdevice structure
  * @pad: Media pad structure
  * @client: Pointer to I2C client
@@ -587,9 +587,6 @@ struct stimx274 {
 	    ? rounddown((dim), (step))			\
 	    : rounddown((dim) + (step) / 2, (step))))
 
-/*
- * Function declaration
- */
 static int imx274_set_gain(struct stimx274 *priv, struct v4l2_ctrl *ctrl);
 static int imx274_set_exposure(struct stimx274 *priv, int val);
 static int imx274_set_vflip(struct stimx274 *priv, int val);
@@ -640,9 +637,9 @@ static int imx274_write_table(struct stimx274 *priv, const struct reg_8 table[])
 
 	for (next = table;; next++) {
 		if ((next->addr != range_start + range_count) ||
-		    (next->addr == IMX274_TABLE_END) ||
-		    (next->addr == IMX274_TABLE_WAIT_MS) ||
-		    (range_count == max_range_vals)) {
+		    next->addr == IMX274_TABLE_END ||
+		    next->addr == IMX274_TABLE_WAIT_MS ||
+		    range_count == max_range_vals) {
 			if (range_count == 1)
 				err = regmap_write(regmap,
 						   range_start, range_vals[0]);
@@ -650,8 +647,6 @@ static int imx274_write_table(struct stimx274 *priv, const struct reg_8 table[])
 				err = regmap_bulk_write(regmap, range_start,
 							&range_vals[0],
 							range_count);
-			else
-				err = 0;
 
 			if (err)
 				return err;
@@ -897,14 +892,6 @@ static int imx274_regulators_get(struct device *dev, struct stimx274 *imx274)
 					imx274->supplies);
 }
 
-/**
- * imx274_s_ctrl - This is used to set the imx274 V4L2 controls
- * @ctrl: V4L2 control to be set
- *
- * This function is used to set the V4L2 controls for the imx274 sensor.
- *
- * Return: 0 on success, errors otherwise
- */
 static int imx274_s_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct v4l2_subdev *sd = ctrl_to_sd(ctrl);
@@ -1059,16 +1046,6 @@ static int __imx274_change_compose(struct stimx274 *imx274,
 	return 0;
 }
 
-/**
- * imx274_get_fmt - Get the pad format
- * @sd: Pointer to V4L2 Sub device structure
- * @sd_state: Pointer to sub device state structure
- * @fmt: Pointer to pad level media bus format
- *
- * This function is used to get the pad format information.
- *
- * Return: 0 on success
- */
 static int imx274_get_fmt(struct v4l2_subdev *sd,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *fmt)
@@ -1081,16 +1058,6 @@ static int imx274_get_fmt(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * imx274_set_fmt - This is used to set the pad format
- * @sd: Pointer to V4L2 Sub device structure
- * @sd_state: Pointer to sub device state information structure
- * @format: Pointer to pad level media bus format
- *
- * This function is used to set the pad format.
- *
- * Return: 0 on success
- */
 static int imx274_set_fmt(struct v4l2_subdev *sd,
 			  struct v4l2_subdev_state *sd_state,
 			  struct v4l2_subdev_format *format)
@@ -1423,16 +1390,6 @@ static void imx274_load_default(struct stimx274 *priv)
 	priv->ctrls.test_pattern->val = TEST_PATTERN_DISABLED;
 }
 
-/**
- * imx274_s_stream - It is used to start/stop the streaming.
- * @sd: V4L2 Sub device
- * @on: Flag (True / False)
- *
- * This function controls the start or stop of streaming for the
- * imx274 sensor.
- *
- * Return: 0 on success, errors otherwise
- */
 static int imx274_s_stream(struct v4l2_subdev *sd, int on)
 {
 	struct stimx274 *imx274 = to_imx274(sd);
@@ -1951,7 +1908,7 @@ static const struct of_device_id imx274_of_id_table[] = {
 MODULE_DEVICE_TABLE(of, imx274_of_id_table);
 
 static const struct i2c_device_id imx274_id[] = {
-	{ "IMX274" },
+	{ .name = "IMX274" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, imx274_id);
@@ -1998,7 +1955,6 @@ static int imx274_probe(struct i2c_client *client)
 	struct device *dev = &client->dev;
 	int ret;
 
-	/* initialize imx274 */
 	imx274 = devm_kzalloc(dev, sizeof(*imx274), GFP_KERNEL);
 	if (!imx274)
 		return -ENOMEM;
@@ -2163,7 +2119,7 @@ static const struct dev_pm_ops imx274_pm_ops = {
 
 static struct i2c_driver imx274_i2c_driver = {
 	.driver = {
-		.name	= DRIVER_NAME,
+		.name = DRIVER_NAME,
 		.pm = &imx274_pm_ops,
 		.of_match_table	= imx274_of_id_table,
 	},
diff --git a/drivers/media/i2c/imx334.c b/drivers/media/i2c/imx334.c
index 9654f9268056f..553a16b84f4d2 100644
--- a/drivers/media/i2c/imx334.c
+++ b/drivers/media/i2c/imx334.c
@@ -566,18 +566,6 @@ static int imx334_update_exp_gain(struct imx334 *imx334, u32 exposure, u32 gain)
 	return ret;
 }
 
-/**
- * imx334_set_ctrl() - Set subdevice control
- * @ctrl: pointer to v4l2_ctrl structure
- *
- * Supported controls:
- * - V4L2_CID_VBLANK
- * - cluster controls:
- *   - V4L2_CID_ANALOGUE_GAIN
- *   - V4L2_CID_EXPOSURE
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_set_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct imx334 *imx334 =
@@ -678,14 +666,6 @@ static int imx334_get_format_code(struct imx334 *imx334, u32 code)
 	return imx334_mbus_codes[0];
 }
 
-/**
- * imx334_enum_mbus_code() - Enumerate V4L2 sub-device mbus codes
- * @sd: pointer to imx334 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device state
- * @code: V4L2 sub-device code enumeration need to be filled
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_enum_mbus_code(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_mbus_code_enum *code)
@@ -698,14 +678,6 @@ static int imx334_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * imx334_enum_frame_size() - Enumerate V4L2 sub-device frame sizes
- * @sd: pointer to imx334 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device state
- * @fsize: V4L2 sub-device size enumeration need to be filled
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_enum_frame_size(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_frame_size_enum *fsize)
@@ -749,14 +721,6 @@ static void imx334_fill_pad_format(struct imx334 *imx334,
 	fmt->format.xfer_func = V4L2_XFER_FUNC_NONE;
 }
 
-/**
- * imx334_get_pad_format() - Get subdevice pad format
- * @sd: pointer to imx334 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device state
- * @fmt: V4L2 sub-device format need to be set
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_get_pad_format(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
@@ -776,14 +740,6 @@ static int imx334_get_pad_format(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * imx334_set_pad_format() - Set subdevice pad format
- * @sd: pointer to imx334 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device state
- * @fmt: V4L2 sub-device format need to be set
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_set_pad_format(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
@@ -815,13 +771,6 @@ static int imx334_set_pad_format(struct v4l2_subdev *sd,
 	return ret;
 }
 
-/**
- * imx334_init_state() - Initialize sub-device state
- * @sd: pointer to imx334 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device state
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_init_state(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_state *sd_state)
 {
@@ -856,15 +805,6 @@ static int imx334_set_framefmt(struct imx334 *imx334)
 	return -EINVAL;
 }
 
-/**
- * imx334_enable_streams() - Enable specified streams for the sensor
- * @sd: pointer to the V4L2 subdevice
- * @state: pointer to the subdevice state
- * @pad: pad number for which streams are enabled
- * @streams_mask: bitmask specifying the streams to enable
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_enable_streams(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *state, u32 pad,
 				 u64 streams_mask)
@@ -929,15 +869,6 @@ static int imx334_enable_streams(struct v4l2_subdev *sd,
 	return ret;
 }
 
-/**
- * imx334_disable_streams() - Enable specified streams for the sensor
- * @sd: pointer to the V4L2 subdevice
- * @state: pointer to the subdevice state
- * @pad: pad number for which streams are disabled
- * @streams_mask: bitmask specifying the streams to disable
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_disable_streams(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *state, u32 pad,
 				  u64 streams_mask)
@@ -1067,12 +998,6 @@ static const struct v4l2_subdev_internal_ops imx334_internal_ops = {
 	.init_state = imx334_init_state,
 };
 
-/**
- * imx334_power_on() - Sensor power on sequence
- * @dev: pointer to i2c device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_power_on(struct device *dev)
 {
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
@@ -1101,12 +1026,6 @@ static int imx334_power_on(struct device *dev)
 	return ret;
 }
 
-/**
- * imx334_power_off() - Sensor power off sequence
- * @dev: pointer to i2c device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_power_off(struct device *dev)
 {
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
@@ -1206,12 +1125,6 @@ static int imx334_init_controls(struct imx334 *imx334)
 	return 0;
 }
 
-/**
- * imx334_probe() - I2C client device binding
- * @client: pointer to i2c client device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx334_probe(struct i2c_client *client)
 {
 	struct imx334 *imx334;
@@ -1311,12 +1224,6 @@ static int imx334_probe(struct i2c_client *client)
 	return ret;
 }
 
-/**
- * imx334_remove() - I2C client device unbinding
- * @client: pointer to I2C client device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static void imx334_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
diff --git a/drivers/media/i2c/imx335.c b/drivers/media/i2c/imx335.c
index 5790aa4fabeb7..1f777a1a81925 100644
--- a/drivers/media/i2c/imx335.c
+++ b/drivers/media/i2c/imx335.c
@@ -698,18 +698,6 @@ static int imx335_update_test_pattern(struct imx335 *imx335, u32 pattern_index)
 	return ret;
 }
 
-/**
- * imx335_set_ctrl() - Set subdevice control
- * @ctrl: pointer to v4l2_ctrl structure
- *
- * Supported controls:
- * - V4L2_CID_VBLANK
- * - cluster controls:
- *   - V4L2_CID_ANALOGUE_GAIN
- *   - V4L2_CID_EXPOSURE
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_set_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct imx335 *imx335 =
@@ -800,14 +788,6 @@ static int imx335_get_format_code(struct imx335 *imx335, u32 code)
 	return imx335_mbus_codes[0];
 }
 
-/**
- * imx335_enum_mbus_code() - Enumerate V4L2 sub-device mbus codes
- * @sd: pointer to imx335 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @code: V4L2 sub-device code enumeration need to be filled
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_enum_mbus_code(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_mbus_code_enum *code)
@@ -820,14 +800,6 @@ static int imx335_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * imx335_enum_frame_size() - Enumerate V4L2 sub-device frame sizes
- * @sd: pointer to imx335 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @fsize: V4L2 sub-device size enumeration need to be filled
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_enum_frame_size(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_frame_size_enum *fsize)
@@ -871,14 +843,6 @@ static void imx335_fill_pad_format(struct imx335 *imx335,
 	fmt->format.xfer_func = V4L2_XFER_FUNC_NONE;
 }
 
-/**
- * imx335_set_pad_format() - Set subdevice pad format
- * @sd: pointer to imx335 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @fmt: V4L2 sub-device format need to be set
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_set_pad_format(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
@@ -923,13 +887,6 @@ static int imx335_set_pad_format(struct v4l2_subdev *sd,
 	return ret;
 }
 
-/**
- * imx335_init_state() - Initialize sub-device state
- * @sd: pointer to imx335 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_init_state(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_state *sd_state)
 {
@@ -947,14 +904,6 @@ static int imx335_init_state(struct v4l2_subdev *sd,
 	return imx335_set_pad_format(sd, sd_state, &fmt);
 }
 
-/**
- * imx335_get_selection() - Selection API
- * @sd: pointer to imx335 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @sel: V4L2 selection info
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_get_selection(struct v4l2_subdev *sd,
 				struct v4l2_subdev_state *sd_state,
 				struct v4l2_subdev_selection *sel)
@@ -1011,15 +960,6 @@ static int imx335_set_framefmt(struct imx335 *imx335)
 	return ret;
 }
 
-/**
- * imx335_enable_streams() - Enable sensor streams
- * @sd: V4L2 subdevice
- * @state: V4L2 subdevice state
- * @pad: The pad to enable
- * @streams_mask: Bitmask of streams to enable
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_enable_streams(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *state, u32 pad,
 				 u64 streams_mask)
@@ -1097,15 +1037,6 @@ static int imx335_enable_streams(struct v4l2_subdev *sd,
 	return ret;
 }
 
-/**
- * imx335_disable_streams() - Disable sensor streams
- * @sd: V4L2 subdevice
- * @state: V4L2 subdevice state
- * @pad: The pad to disable
- * @streams_mask: Bitmask of streams to disable
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_disable_streams(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *state, u32 pad,
 				  u64 streams_mask)
@@ -1299,12 +1230,6 @@ static int imx335_power_on(struct device *dev)
 	return ret;
 }
 
-/**
- * imx335_power_off() - Sensor power off sequence
- * @dev: pointer to i2c device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_power_off(struct device *dev)
 {
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
@@ -1430,12 +1355,6 @@ static int imx335_init_controls(struct imx335 *imx335)
 	return 0;
 }
 
-/**
- * imx335_probe() - I2C client device binding
- * @client: pointer to i2c client device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx335_probe(struct i2c_client *client)
 {
 	struct imx335 *imx335;
@@ -1530,12 +1449,6 @@ static int imx335_probe(struct i2c_client *client)
 	return ret;
 }
 
-/**
- * imx335_remove() - I2C client device unbinding
- * @client: pointer to I2C client device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static void imx335_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
diff --git a/drivers/media/i2c/imx412.c b/drivers/media/i2c/imx412.c
index e25e0a9ff65c3..2705af2f16c0b 100644
--- a/drivers/media/i2c/imx412.c
+++ b/drivers/media/i2c/imx412.c
@@ -570,18 +570,6 @@ static int imx412_update_exp_gain(struct imx412 *imx412, u32 exposure, u32 gain)
 	return ret;
 }
 
-/**
- * imx412_set_ctrl() - Set subdevice control
- * @ctrl: pointer to v4l2_ctrl structure
- *
- * Supported controls:
- * - V4L2_CID_VBLANK
- * - cluster controls:
- *   - V4L2_CID_ANALOGUE_GAIN
- *   - V4L2_CID_EXPOSURE
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_set_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct imx412 *imx412 =
@@ -634,14 +622,6 @@ static const struct v4l2_ctrl_ops imx412_ctrl_ops = {
 	.s_ctrl = imx412_set_ctrl,
 };
 
-/**
- * imx412_enum_mbus_code() - Enumerate V4L2 sub-device mbus codes
- * @sd: pointer to imx412 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @code: V4L2 sub-device code enumeration need to be filled
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_enum_mbus_code(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_mbus_code_enum *code)
@@ -654,14 +634,6 @@ static int imx412_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * imx412_enum_frame_size() - Enumerate V4L2 sub-device frame sizes
- * @sd: pointer to imx412 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @fsize: V4L2 sub-device size enumeration need to be filled
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_enum_frame_size(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_frame_size_enum *fsize)
@@ -701,14 +673,6 @@ static void imx412_fill_pad_format(struct imx412 *imx412,
 	fmt->format.xfer_func = V4L2_XFER_FUNC_NONE;
 }
 
-/**
- * imx412_get_pad_format() - Get subdevice pad format
- * @sd: pointer to imx412 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @fmt: V4L2 sub-device format need to be set
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_get_pad_format(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
@@ -731,14 +695,6 @@ static int imx412_get_pad_format(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * imx412_set_pad_format() - Set subdevice pad format
- * @sd: pointer to imx412 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @fmt: V4L2 sub-device format need to be set
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_set_pad_format(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
@@ -768,13 +724,6 @@ static int imx412_set_pad_format(struct v4l2_subdev *sd,
 	return ret;
 }
 
-/**
- * imx412_init_state() - Initialize sub-device state
- * @sd: pointer to imx412 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_init_state(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_state *sd_state)
 {
@@ -840,13 +789,6 @@ static int imx412_stop_streaming(struct imx412 *imx412)
 				1, IMX412_MODE_STANDBY);
 }
 
-/**
- * imx412_set_stream() - Enable sensor streaming
- * @sd: pointer to imx412 subdevice
- * @enable: set to enable sensor streaming
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_set_stream(struct v4l2_subdev *sd, int enable)
 {
 	struct imx412 *imx412 = to_imx412(sd);
@@ -1010,12 +952,6 @@ static const struct v4l2_subdev_internal_ops imx412_internal_ops = {
 	.init_state = imx412_init_state,
 };
 
-/**
- * imx412_power_on() - Sensor power on sequence
- * @dev: pointer to i2c device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_power_on(struct device *dev)
 {
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
@@ -1053,12 +989,6 @@ static int imx412_power_on(struct device *dev)
 	return ret;
 }
 
-/**
- * imx412_power_off() - Sensor power off sequence
- * @dev: pointer to i2c device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_power_off(struct device *dev)
 {
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
@@ -1159,12 +1089,6 @@ static int imx412_init_controls(struct imx412 *imx412)
 	return 0;
 }
 
-/**
- * imx412_probe() - I2C client device binding
- * @client: pointer to i2c client device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int imx412_probe(struct i2c_client *client)
 {
 	struct imx412 *imx412;
@@ -1254,12 +1178,6 @@ static int imx412_probe(struct i2c_client *client)
 	return ret;
 }
 
-/**
- * imx412_remove() - I2C client device unbinding
- * @client: pointer to I2C client device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static void imx412_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c
index 6047453170043..f2bf2b3540007 100644
--- a/drivers/media/i2c/ir-kbd-i2c.c
+++ b/drivers/media/i2c/ir-kbd-i2c.c
@@ -978,10 +978,10 @@ static void ir_remove(struct i2c_client *client)
 
 static const struct i2c_device_id ir_kbd_id[] = {
 	/* Generic entry for any IR receiver */
-	{ "ir_video", 0 },
+	{ .name = "ir_video", .driver_data = 0 },
 	/* IR device specific entries should be added here */
-	{ "ir_z8f0811_haup", FLAG_TX },
-	{ "ir_z8f0811_hdpvr", FLAG_TX | FLAG_HDPVR },
+	{ .name = "ir_z8f0811_haup", .driver_data = FLAG_TX },
+	{ .name = "ir_z8f0811_hdpvr", .driver_data = FLAG_TX | FLAG_HDPVR },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ir_kbd_id);
diff --git a/drivers/media/i2c/isl7998x.c b/drivers/media/i2c/isl7998x.c
index 5ffd53e005ee5..a77538d2343c1 100644
--- a/drivers/media/i2c/isl7998x.c
+++ b/drivers/media/i2c/isl7998x.c
@@ -1561,7 +1561,7 @@ static const struct of_device_id isl7998x_of_match[] = {
 MODULE_DEVICE_TABLE(of, isl7998x_of_match);
 
 static const struct i2c_device_id isl7998x_id[] = {
-	{ "isl79987" },
+	{ .name = "isl79987" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, isl7998x_id);
diff --git a/drivers/media/i2c/ks0127.c b/drivers/media/i2c/ks0127.c
index f3fba9179684e..8c66be38adc30 100644
--- a/drivers/media/i2c/ks0127.c
+++ b/drivers/media/i2c/ks0127.c
@@ -677,9 +677,9 @@ static void ks0127_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ks0127_id[] = {
-	{ "ks0127" },
-	{ "ks0127b" },
-	{ "ks0122s" },
+	{ .name = "ks0127" },
+	{ .name = "ks0127b" },
+	{ .name = "ks0122s" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ks0127_id);
diff --git a/drivers/media/i2c/lm3560.c b/drivers/media/i2c/lm3560.c
index f4cc844f4e3cc..c3c90d830ee24 100644
--- a/drivers/media/i2c/lm3560.c
+++ b/drivers/media/i2c/lm3560.c
@@ -9,16 +9,22 @@
  *			Ldd-Mlp <ldd-mlp@list.ti.com>
  */
 
+#include <linux/bitmap.h>
 #include <linux/delay.h>
 #include <linux/module.h>
+#include <linux/gpio/consumer.h>
 #include <linux/i2c.h>
 #include <linux/slab.h>
+#include <linux/mod_devicetable.h>
 #include <linux/mutex.h>
+#include <linux/pm_runtime.h>
+#include <linux/property.h>
 #include <linux/regmap.h>
+#include <linux/regulator/consumer.h>
 #include <linux/videodev2.h>
-#include <media/i2c/lm3560.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
 
 /* registers definitions */
 #define REG_ENABLE		0x10
@@ -33,32 +39,83 @@
 #define FAULT_OVERTEMP	(1<<1)
 #define FAULT_SHORT_CIRCUIT	(1<<2)
 
+#define LM3560_FLASH_TOUT_MIN			32
+#define LM3560_FLASH_TOUT_STEP			32
+#define LM3560_FLASH_TOUT_MAX			1024
+#define LM3560_FLASH_TOUT_ms_TO_REG(a)		\
+	((a) < LM3560_FLASH_TOUT_MIN ? 0 :	\
+	 (((a) - LM3560_FLASH_TOUT_MIN) / LM3560_FLASH_TOUT_STEP))
+#define LM3560_FLASH_TOUT_REG_TO_ms(a)		\
+	((a) * LM3560_FLASH_TOUT_STEP + LM3560_FLASH_TOUT_MIN)
+
+enum lm3560_led_id {
+	LM3560_LED0 = 0,
+	LM3560_LED1,
+	LM3560_LED_MAX
+};
+
+enum lm3560_peak_current {
+	LM3560_PEAK_1600mA = 0x00,
+	LM3560_PEAK_2300mA = 0x20,
+	LM3560_PEAK_3000mA = 0x40,
+	LM3560_PEAK_3600mA = 0x60
+};
+
 enum led_enable {
 	MODE_SHDN = 0x0,
 	MODE_TORCH = 0x2,
 	MODE_FLASH = 0x3,
 };
 
+struct lm3560_flash_config {
+	u32 flash_brt_min_ua;
+	u32 flash_brt_max_ua;
+	u32 flash_brt_step_ua;
+
+	u32 torch_brt_min_ua;
+	u32 torch_brt_max_ua;
+	u32 torch_brt_step_ua;
+};
+
 /**
  * struct lm3560_flash
  *
  * @dev: pointer to &struct device
- * @pdata: platform data
  * @regmap: reg. map for i2c
  * @lock: muxtex for serial access.
+ * @hwen_gpio: line connected to HWEN pin
+ * @vin_supply: line connected to IN supply (2.5V - 5.5V)
  * @led_mode: V4L2 LED mode
  * @ctrls_led: V4L2 controls
  * @subdev_led: V4L2 subdev
+ * @led_id: LED status holder
+ * @peak: peak current
+ * @max_flash_timeout: flash timeout
+ * @max_flash_brt: flash mode led brightness
+ * @max_torch_brt: torch mode led brightness
+ * @config: device specific current configuration
  */
 struct lm3560_flash {
 	struct device *dev;
-	struct lm3560_platform_data *pdata;
 	struct regmap *regmap;
 	struct mutex lock;
 
+	struct gpio_desc *hwen_gpio;
+	struct regulator *vin_supply;
+
 	enum v4l2_flash_led_mode led_mode;
 	struct v4l2_ctrl_handler ctrls_led[LM3560_LED_MAX];
 	struct v4l2_subdev subdev_led[LM3560_LED_MAX];
+
+	DECLARE_BITMAP(led_id, LM3560_LED_MAX);
+
+	enum lm3560_peak_current peak;
+	u32 max_flash_timeout;
+
+	u32 max_flash_brt[LM3560_LED_MAX];
+	u32 max_torch_brt[LM3560_LED_MAX];
+
+	const struct lm3560_flash_config *config;
 };
 
 #define to_lm3560_flash(_ctrl, _no)	\
@@ -114,15 +171,20 @@ static int lm3560_enable_ctrl(struct lm3560_flash *flash,
 static int lm3560_torch_brt_ctrl(struct lm3560_flash *flash,
 				 enum lm3560_led_id led_no, unsigned int brt)
 {
+	const struct lm3560_flash_config *config = flash->config;
 	int rval;
-	u8 br_bits;
+	u32 br_bits;
 
-	if (brt < LM3560_TORCH_BRT_MIN)
+	if (brt < config->torch_brt_min_ua)
 		return lm3560_enable_ctrl(flash, led_no, false);
 	else
 		rval = lm3560_enable_ctrl(flash, led_no, true);
 
-	br_bits = LM3560_TORCH_BRT_uA_TO_REG(brt);
+	br_bits = clamp(brt, config->torch_brt_min_ua,
+			config->torch_brt_max_ua);
+	br_bits = (br_bits - config->torch_brt_min_ua) /
+		  config->torch_brt_step_ua;
+
 	if (led_no == LM3560_LED0)
 		rval = regmap_update_bits(flash->regmap,
 					  REG_TORCH_BR, 0x07, br_bits);
@@ -137,15 +199,20 @@ static int lm3560_torch_brt_ctrl(struct lm3560_flash *flash,
 static int lm3560_flash_brt_ctrl(struct lm3560_flash *flash,
 				 enum lm3560_led_id led_no, unsigned int brt)
 {
+	const struct lm3560_flash_config *config = flash->config;
 	int rval;
-	u8 br_bits;
+	u32 br_bits;
 
-	if (brt < LM3560_FLASH_BRT_MIN)
+	if (brt < config->flash_brt_min_ua)
 		return lm3560_enable_ctrl(flash, led_no, false);
 	else
 		rval = lm3560_enable_ctrl(flash, led_no, true);
 
-	br_bits = LM3560_FLASH_BRT_uA_TO_REG(brt);
+	br_bits = clamp(brt, config->flash_brt_min_ua,
+			config->flash_brt_max_ua);
+	br_bits = (br_bits - config->flash_brt_min_ua) /
+		  config->flash_brt_step_ua;
+
 	if (led_no == LM3560_LED0)
 		rval = regmap_update_bits(flash->regmap,
 					  REG_FLASH_BR, 0x0f, br_bits);
@@ -162,14 +229,17 @@ static int lm3560_get_ctrl(struct v4l2_ctrl *ctrl, enum lm3560_led_id led_no)
 	struct lm3560_flash *flash = to_lm3560_flash(ctrl, led_no);
 	int rval = -EINVAL;
 
-	mutex_lock(&flash->lock);
+	if (!pm_runtime_get_if_active(flash->dev))
+		return 0;
 
 	if (ctrl->id == V4L2_CID_FLASH_FAULT) {
 		s32 fault = 0;
 		unsigned int reg_val;
 		rval = regmap_read(flash->regmap, REG_FLAG, &reg_val);
-		if (rval < 0)
-			goto out;
+		if (rval < 0) {
+			pm_runtime_put(flash->dev);
+			return rval;
+		}
 		if (reg_val & FAULT_SHORT_CIRCUIT)
 			fault |= V4L2_FLASH_FAULT_SHORT_CIRCUIT;
 		if (reg_val & FAULT_OVERTEMP)
@@ -179,8 +249,8 @@ static int lm3560_get_ctrl(struct v4l2_ctrl *ctrl, enum lm3560_led_id led_no)
 		ctrl->cur.val = fault;
 	}
 
-out:
-	mutex_unlock(&flash->lock);
+	pm_runtime_put(flash->dev);
+
 	return rval;
 }
 
@@ -190,7 +260,8 @@ static int lm3560_set_ctrl(struct v4l2_ctrl *ctrl, enum lm3560_led_id led_no)
 	u8 tout_bits;
 	int rval = -EINVAL;
 
-	mutex_lock(&flash->lock);
+	if (!pm_runtime_get_if_active(flash->dev))
+		return 0;
 
 	switch (ctrl->id) {
 	case V4L2_CID_FLASH_LED_MODE:
@@ -202,14 +273,12 @@ static int lm3560_set_ctrl(struct v4l2_ctrl *ctrl, enum lm3560_led_id led_no)
 	case V4L2_CID_FLASH_STROBE_SOURCE:
 		rval = regmap_update_bits(flash->regmap,
 					  REG_CONFIG1, 0x04, (ctrl->val) << 2);
-		if (rval < 0)
-			goto err_out;
 		break;
 
 	case V4L2_CID_FLASH_STROBE:
 		if (flash->led_mode != V4L2_FLASH_LED_MODE_FLASH) {
 			rval = -EBUSY;
-			goto err_out;
+			break;
 		}
 		flash->led_mode = V4L2_FLASH_LED_MODE_FLASH;
 		rval = lm3560_mode_ctrl(flash);
@@ -218,7 +287,7 @@ static int lm3560_set_ctrl(struct v4l2_ctrl *ctrl, enum lm3560_led_id led_no)
 	case V4L2_CID_FLASH_STROBE_STOP:
 		if (flash->led_mode != V4L2_FLASH_LED_MODE_FLASH) {
 			rval = -EBUSY;
-			goto err_out;
+			break;
 		}
 		flash->led_mode = V4L2_FLASH_LED_MODE_NONE;
 		rval = lm3560_mode_ctrl(flash);
@@ -239,8 +308,8 @@ static int lm3560_set_ctrl(struct v4l2_ctrl *ctrl, enum lm3560_led_id led_no)
 		break;
 	}
 
-err_out:
-	mutex_unlock(&flash->lock);
+	pm_runtime_put(flash->dev);
+
 	return rval;
 }
 
@@ -279,10 +348,11 @@ static int lm3560_init_controls(struct lm3560_flash *flash,
 				enum lm3560_led_id led_no)
 {
 	struct v4l2_ctrl *fault;
-	u32 max_flash_brt = flash->pdata->max_flash_brt[led_no];
-	u32 max_torch_brt = flash->pdata->max_torch_brt[led_no];
+	u32 max_flash_brt = flash->max_flash_brt[led_no];
+	u32 max_torch_brt = flash->max_torch_brt[led_no];
 	struct v4l2_ctrl_handler *hdl = &flash->ctrls_led[led_no];
 	const struct v4l2_ctrl_ops *ops = &lm3560_led_ctrl_ops[led_no];
+	const struct lm3560_flash_config *config = flash->config;
 
 	v4l2_ctrl_handler_init(hdl, 8);
 
@@ -305,19 +375,19 @@ static int lm3560_init_controls(struct lm3560_flash *flash,
 	/* flash strobe timeout */
 	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_FLASH_TIMEOUT,
 			  LM3560_FLASH_TOUT_MIN,
-			  flash->pdata->max_flash_timeout,
+			  flash->max_flash_timeout,
 			  LM3560_FLASH_TOUT_STEP,
-			  flash->pdata->max_flash_timeout);
+			  flash->max_flash_timeout);
 
 	/* flash brt */
 	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_FLASH_INTENSITY,
-			  LM3560_FLASH_BRT_MIN, max_flash_brt,
-			  LM3560_FLASH_BRT_STEP, max_flash_brt);
+			  config->flash_brt_min_ua, max_flash_brt,
+			  config->flash_brt_step_ua, max_flash_brt);
 
 	/* torch brt */
 	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_FLASH_TORCH_INTENSITY,
-			  LM3560_TORCH_BRT_MIN, max_torch_brt,
-			  LM3560_TORCH_BRT_STEP, max_torch_brt);
+			  config->torch_brt_min_ua, max_torch_brt,
+			  config->torch_brt_step_ua, max_torch_brt);
 
 	/* fault */
 	fault = v4l2_ctrl_new_std(hdl, ops, V4L2_CID_FLASH_FAULT, 0,
@@ -328,6 +398,8 @@ static int lm3560_init_controls(struct lm3560_flash *flash,
 	if (fault != NULL)
 		fault->flags |= V4L2_CTRL_FLAG_VOLATILE;
 
+	hdl->lock = &flash->lock;
+
 	if (hdl->error)
 		return hdl->error;
 
@@ -347,15 +419,18 @@ static const struct regmap_config lm3560_regmap = {
 };
 
 static int lm3560_subdev_init(struct lm3560_flash *flash,
-			      enum lm3560_led_id led_no, char *led_name)
+			      enum lm3560_led_id led_no,
+			      struct fwnode_handle *fwnode)
 {
 	struct i2c_client *client = to_i2c_client(flash->dev);
 	int rval;
 
 	v4l2_i2c_subdev_init(&flash->subdev_led[led_no], client, &lm3560_ops);
 	flash->subdev_led[led_no].flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-	strscpy(flash->subdev_led[led_no].name, led_name,
-		sizeof(flash->subdev_led[led_no].name));
+	snprintf(flash->subdev_led[led_no].name,
+		 sizeof(flash->subdev_led[led_no].name),
+		 "lm3560-led%d", led_no);
+	flash->subdev_led[led_no].fwnode = fwnode_handle_get(fwnode);
 	rval = lm3560_init_controls(flash, led_no);
 	if (rval)
 		goto err_out;
@@ -363,9 +438,17 @@ static int lm3560_subdev_init(struct lm3560_flash *flash,
 	if (rval < 0)
 		goto err_out;
 	flash->subdev_led[led_no].entity.function = MEDIA_ENT_F_FLASH;
+	flash->subdev_led[led_no].state_lock = &flash->lock;
 
-	return rval;
+	rval = v4l2_async_register_subdev(&flash->subdev_led[led_no]);
+	if (rval < 0) {
+		dev_err(flash->dev, "failed to register V4L2 subdev");
+		goto error_out_media;
+	}
 
+	return rval;
+error_out_media:
+	media_entity_cleanup(&flash->subdev_led[led_no].entity);
 err_out:
 	v4l2_ctrl_handler_free(&flash->ctrls_led[led_no]);
 	return rval;
@@ -378,7 +461,7 @@ static int lm3560_init_device(struct lm3560_flash *flash)
 
 	/* set peak current */
 	rval = regmap_update_bits(flash->regmap,
-				  REG_FLASH_TOUT, 0x60, flash->pdata->peak);
+				  REG_FLASH_TOUT, 0x60, flash->peak);
 	if (rval < 0)
 		return rval;
 	/* output disable */
@@ -391,81 +474,247 @@ static int lm3560_init_device(struct lm3560_flash *flash)
 	return rval;
 }
 
+static int __maybe_unused lm3560_power_off(struct device *dev)
+{
+	struct lm3560_flash *flash = dev_get_drvdata(dev);
+
+	gpiod_set_value_cansleep(flash->hwen_gpio, 0);
+	regulator_disable(flash->vin_supply);
+
+	return 0;
+}
+
+static int __maybe_unused lm3560_power_on(struct device *dev)
+{
+	struct lm3560_flash *flash = dev_get_drvdata(dev);
+	int rval;
+
+	rval = regulator_enable(flash->vin_supply);
+	if (rval < 0) {
+		dev_err(flash->dev, "failed to enable vin power supply\n");
+		return rval;
+	}
+
+	gpiod_set_value_cansleep(flash->hwen_gpio, 1);
+
+	rval = lm3560_init_device(flash);
+	if (rval < 0) {
+		lm3560_power_off(dev);
+		return rval;
+	}
+
+	return 0;
+}
+
+static void lm3560_subdev_cleanup(struct lm3560_flash *flash)
+{
+	int led_no;
+
+	for_each_set_bit(led_no, flash->led_id, LM3560_LED_MAX) {
+		v4l2_async_unregister_subdev(&flash->subdev_led[led_no]);
+		v4l2_ctrl_handler_free(&flash->ctrls_led[led_no]);
+		media_entity_cleanup(&flash->subdev_led[led_no].entity);
+	}
+}
+
 static int lm3560_probe(struct i2c_client *client)
 {
 	struct lm3560_flash *flash;
-	struct lm3560_platform_data *pdata = dev_get_platdata(&client->dev);
+	u32 peak_ua;
 	int rval;
 
 	flash = devm_kzalloc(&client->dev, sizeof(*flash), GFP_KERNEL);
 	if (flash == NULL)
 		return -ENOMEM;
 
+	flash->config = device_get_match_data(&client->dev);
+	if (!flash->config)
+		return -ENODEV;
+
 	flash->regmap = devm_regmap_init_i2c(client, &lm3560_regmap);
 	if (IS_ERR(flash->regmap)) {
 		rval = PTR_ERR(flash->regmap);
 		return rval;
 	}
 
-	/* if there is no platform data, use chip default value */
-	if (pdata == NULL) {
-		pdata = devm_kzalloc(&client->dev, sizeof(*pdata), GFP_KERNEL);
-		if (pdata == NULL)
-			return -ENODEV;
-		pdata->peak = LM3560_PEAK_3600mA;
-		pdata->max_flash_timeout = LM3560_FLASH_TOUT_MAX;
-		/* led 1 */
-		pdata->max_flash_brt[LM3560_LED0] = LM3560_FLASH_BRT_MAX;
-		pdata->max_torch_brt[LM3560_LED0] = LM3560_TORCH_BRT_MAX;
-		/* led 2 */
-		pdata->max_flash_brt[LM3560_LED1] = LM3560_FLASH_BRT_MAX;
-		pdata->max_torch_brt[LM3560_LED1] = LM3560_TORCH_BRT_MAX;
-	}
-	flash->pdata = pdata;
 	flash->dev = &client->dev;
 	mutex_init(&flash->lock);
 
-	rval = lm3560_subdev_init(flash, LM3560_LED0, "lm3560-led0");
-	if (rval < 0)
-		return rval;
+	bitmap_zero(flash->led_id, LM3560_LED_MAX);
+
+	flash->hwen_gpio = devm_gpiod_get_optional(flash->dev, "enable",
+						   GPIOD_OUT_LOW);
+	if (IS_ERR(flash->hwen_gpio))
+		return dev_err_probe(flash->dev, PTR_ERR(flash->hwen_gpio),
+				     "failed to get hwen gpio\n");
+
+	flash->vin_supply = devm_regulator_get(flash->dev, "vin");
+	if (IS_ERR(flash->vin_supply))
+		return dev_err_probe(flash->dev, PTR_ERR(flash->vin_supply),
+				     "failed to get vin-supply\n");
+
+	flash->peak = LM3560_PEAK_1600mA;
+	rval = device_property_read_u32(flash->dev,
+					"ti,peak-current-microamp", &peak_ua);
+	if (!rval) {
+		/*
+		 * LM3559 has lower peak current limit, but
+		 * bit configuration matches LM3560.
+		 * Correct current restrictions are enforced
+		 * by the LM3560 schema.
+		 */
+		switch (peak_ua) {
+		case 1400000:
+		case 1600000:
+			flash->peak = LM3560_PEAK_1600mA;
+			break;
+		case 2100000:
+		case 2300000:
+			flash->peak = LM3560_PEAK_2300mA;
+			break;
+		case 2700000:
+		case 3000000:
+			flash->peak = LM3560_PEAK_3000mA;
+			break;
+		case 3200000:
+		case 3600000:
+			flash->peak = LM3560_PEAK_3600mA;
+			break;
+		default:
+			return -EINVAL;
+		}
+	}
+
+	flash->max_flash_timeout = LM3560_FLASH_TOUT_MIN * 1000;
+	device_property_read_u32(flash->dev, "flash-max-timeout-us",
+				 &flash->max_flash_timeout);
+	flash->max_flash_timeout /= 1000;
 
-	rval = lm3560_subdev_init(flash, LM3560_LED1, "lm3560-led1");
+	rval = regulator_enable(flash->vin_supply);
 	if (rval < 0)
-		return rval;
+		return dev_err_probe(flash->dev, rval,
+				     "failed to enable vin power supply\n");
+
+	gpiod_set_value_cansleep(flash->hwen_gpio, 1);
 
 	rval = lm3560_init_device(flash);
 	if (rval < 0)
-		return rval;
+		goto error_disable;
+
+	pm_runtime_set_active(flash->dev);
+	pm_runtime_enable(flash->dev);
+
+	device_for_each_child_node_scoped(flash->dev, node) {
+		const struct lm3560_flash_config *config = flash->config;
+		u32 reg;
+
+		rval = fwnode_property_read_u32(node, "reg", &reg);
+		if (rval < 0)
+			/* We care only about nodes with reg property */
+			continue;
+
+		if (reg == LM3560_LED0 || reg == LM3560_LED1) {
+			flash->max_flash_brt[reg] = config->flash_brt_min_ua;
+			fwnode_property_read_u32(node, "flash-max-microamp",
+						 &flash->max_flash_brt[reg]);
+
+			flash->max_torch_brt[reg] = config->torch_brt_min_ua;
+			fwnode_property_read_u32(node, "led-max-microamp",
+						 &flash->max_torch_brt[reg]);
+
+			rval = lm3560_subdev_init(flash, reg, node);
+			if (rval < 0) {
+				dev_err(flash->dev,
+					"failed to register led%d: %d\n",
+					reg, rval);
+				goto error_clean;
+			}
+
+			set_bit(reg, flash->led_id);
+		}
+	}
 
 	i2c_set_clientdata(client, flash);
 
+	pm_runtime_set_autosuspend_delay(flash->dev, 1000);
+	pm_runtime_use_autosuspend(flash->dev);
+	pm_runtime_idle(flash->dev);
+
 	return 0;
+
+error_clean:
+	pm_runtime_disable(flash->dev);
+	pm_runtime_set_suspended(flash->dev);
+
+	lm3560_subdev_cleanup(flash);
+
+error_disable:
+	gpiod_set_value_cansleep(flash->hwen_gpio, 0);
+	regulator_disable(flash->vin_supply);
+
+	return rval;
 }
 
 static void lm3560_remove(struct i2c_client *client)
 {
 	struct lm3560_flash *flash = i2c_get_clientdata(client);
-	unsigned int i;
 
-	for (i = LM3560_LED0; i < LM3560_LED_MAX; i++) {
-		v4l2_device_unregister_subdev(&flash->subdev_led[i]);
-		v4l2_ctrl_handler_free(&flash->ctrls_led[i]);
-		media_entity_cleanup(&flash->subdev_led[i].entity);
+	lm3560_subdev_cleanup(flash);
+
+	/*
+	 * Disable runtime PM. In case runtime PM is disabled in the kernel,
+	 * make sure to turn power off manually.
+	 */
+	pm_runtime_disable(&client->dev);
+	if (!pm_runtime_status_suspended(&client->dev)) {
+		lm3560_power_off(&client->dev);
+		pm_runtime_set_suspended(&client->dev);
 	}
 }
 
-static const struct i2c_device_id lm3560_id_table[] = {
-	{ LM3559_NAME },
-	{ LM3560_NAME },
-	{}
+static const struct dev_pm_ops lm3560_pm_ops = {
+	SET_RUNTIME_PM_OPS(lm3560_power_off, lm3560_power_on, NULL)
 };
 
+static const struct lm3560_flash_config lm3559_config = {
+	.flash_brt_min_ua = 56250,
+	.flash_brt_max_ua = 900000,
+	.flash_brt_step_ua = 56250,
+
+	.torch_brt_min_ua = 28125,
+	.torch_brt_max_ua = 225000,
+	.torch_brt_step_ua = 28125,
+};
+
+static const struct lm3560_flash_config lm3560_config = {
+	.flash_brt_min_ua = 62500,
+	.flash_brt_max_ua = 1000000,
+	.flash_brt_step_ua = 62500,
+
+	.torch_brt_min_ua = 31250,
+	.torch_brt_max_ua = 250000,
+	.torch_brt_step_ua = 31250,
+};
+
+static const struct of_device_id lm3560_of_match[] = {
+	{ .compatible = "ti,lm3559", .data = &lm3559_config },
+	{ .compatible = "ti,lm3560", .data = &lm3560_config },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lm3560_of_match);
+
+static const struct i2c_device_id lm3560_id_table[] = {
+	{ .name = "lm3559", .driver_data = (kernel_ulong_t)&lm3559_config },
+	{ .name = "lm3560", .driver_data = (kernel_ulong_t)&lm3560_config },
+	{ }
+};
 MODULE_DEVICE_TABLE(i2c, lm3560_id_table);
 
 static struct i2c_driver lm3560_i2c_driver = {
 	.driver = {
-		   .name = LM3560_NAME,
-		   .pm = NULL,
+		   .name = "lm3560",
+		   .pm = pm_ptr(&lm3560_pm_ops),
+		   .of_match_table = lm3560_of_match,
 		   },
 	.probe = lm3560_probe,
 	.remove = lm3560_remove,
diff --git a/drivers/media/i2c/lm3646.c b/drivers/media/i2c/lm3646.c
index 2d16e42ec224e..9030cbe32afca 100644
--- a/drivers/media/i2c/lm3646.c
+++ b/drivers/media/i2c/lm3646.c
@@ -386,8 +386,8 @@ static void lm3646_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id lm3646_id_table[] = {
-	{ LM3646_NAME },
-	{}
+	{ .name = LM3646_NAME },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, lm3646_id_table);
diff --git a/drivers/media/i2c/m52790.c b/drivers/media/i2c/m52790.c
index 9e1ecfd01e2a7..f3adf8c2b27f1 100644
--- a/drivers/media/i2c/m52790.c
+++ b/drivers/media/i2c/m52790.c
@@ -163,7 +163,7 @@ static void m52790_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id m52790_id[] = {
-	{ "m52790" },
+	{ .name = "m52790" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, m52790_id);
diff --git a/drivers/media/i2c/max2175.c b/drivers/media/i2c/max2175.c
index bf02ca23a284c..1cc388b52902c 100644
--- a/drivers/media/i2c/max2175.c
+++ b/drivers/media/i2c/max2175.c
@@ -1413,8 +1413,8 @@ static void max2175_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id max2175_id[] = {
-	{ DRIVER_NAME },
-	{}
+	{ .name = DRIVER_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, max2175_id);
 
diff --git a/drivers/media/i2c/ml86v7667.c b/drivers/media/i2c/ml86v7667.c
index 57ba3693649ab..48b7d589df318 100644
--- a/drivers/media/i2c/ml86v7667.c
+++ b/drivers/media/i2c/ml86v7667.c
@@ -424,8 +424,8 @@ static void ml86v7667_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ml86v7667_id[] = {
-	{ DRV_NAME },
-	{}
+	{ .name = DRV_NAME },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ml86v7667_id);
 
diff --git a/drivers/media/i2c/msp3400-driver.c b/drivers/media/i2c/msp3400-driver.c
index 4c0b0ad68c084..413cfbc2dd94e 100644
--- a/drivers/media/i2c/msp3400-driver.c
+++ b/drivers/media/i2c/msp3400-driver.c
@@ -874,7 +874,7 @@ static const struct dev_pm_ops msp3400_pm_ops = {
 };
 
 static const struct i2c_device_id msp_id[] = {
-	{ "msp3400" },
+	{ .name = "msp3400" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, msp_id);
diff --git a/drivers/media/i2c/mt9m001.c b/drivers/media/i2c/mt9m001.c
index 7a6114d18dfc5..0ade967b357bc 100644
--- a/drivers/media/i2c/mt9m001.c
+++ b/drivers/media/i2c/mt9m001.c
@@ -855,7 +855,7 @@ static void mt9m001_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mt9m001_id[] = {
-	{ "mt9m001" },
+	{ .name = "mt9m001" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mt9m001_id);
diff --git a/drivers/media/i2c/mt9m111.c b/drivers/media/i2c/mt9m111.c
index 3532c7c38becc..4e748080b7986 100644
--- a/drivers/media/i2c/mt9m111.c
+++ b/drivers/media/i2c/mt9m111.c
@@ -1384,7 +1384,7 @@ static const struct of_device_id mt9m111_of_match[] = {
 MODULE_DEVICE_TABLE(of, mt9m111_of_match);
 
 static const struct i2c_device_id mt9m111_id[] = {
-	{ "mt9m111" },
+	{ .name = "mt9m111" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mt9m111_id);
diff --git a/drivers/media/i2c/mt9p031.c b/drivers/media/i2c/mt9p031.c
index ea5d43d925ffa..8dc57eeba6060 100644
--- a/drivers/media/i2c/mt9p031.c
+++ b/drivers/media/i2c/mt9p031.c
@@ -796,7 +796,8 @@ static int mt9p031_s_ctrl(struct v4l2_ctrl *ctrl)
 			data = (1 << 6) | (ctrl->val >> 1);
 		} else {
 			ctrl->val &= ~7;
-			data = ((ctrl->val - 64) << 5) | (1 << 6) | 32;
+			data = ((ctrl->val - 64) >> 3) & 0x7f;
+			data = (data << 8) | (1 << 6) | 32;
 		}
 
 		return mt9p031_write(client, MT9P031_GLOBAL_GAIN, data);
diff --git a/drivers/media/i2c/mt9t112.c b/drivers/media/i2c/mt9t112.c
index 2d2c840fc0021..bd2268154ca7a 100644
--- a/drivers/media/i2c/mt9t112.c
+++ b/drivers/media/i2c/mt9t112.c
@@ -1108,7 +1108,7 @@ static void mt9t112_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mt9t112_id[] = {
-	{ "mt9t112" },
+	{ .name = "mt9t112" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mt9t112_id);
diff --git a/drivers/media/i2c/mt9v011.c b/drivers/media/i2c/mt9v011.c
index 055b7915260a2..985517f1cff7e 100644
--- a/drivers/media/i2c/mt9v011.c
+++ b/drivers/media/i2c/mt9v011.c
@@ -582,7 +582,7 @@ static void mt9v011_remove(struct i2c_client *c)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id mt9v011_id[] = {
-	{ "mt9v011" },
+	{ .name = "mt9v011" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mt9v011_id);
diff --git a/drivers/media/i2c/ov13858.c b/drivers/media/i2c/ov13858.c
index 162b49046990a..09d58e8b1c7fb 100644
--- a/drivers/media/i2c/ov13858.c
+++ b/drivers/media/i2c/ov13858.c
@@ -1747,8 +1747,8 @@ static void ov13858_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ov13858_id_table[] = {
-	{ "ov13858" },
-	{}
+	{ .name = "ov13858" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, ov13858_id_table);
diff --git a/drivers/media/i2c/ov13b10.c b/drivers/media/i2c/ov13b10.c
index 5421874732bcd..b0d34141a13a9 100644
--- a/drivers/media/i2c/ov13b10.c
+++ b/drivers/media/i2c/ov13b10.c
@@ -8,6 +8,7 @@
 #include <linux/i2c.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
+#include <linux/regulator/consumer.h>
 #include <media/v4l2-ctrls.h>
 #include <media/v4l2-device.h>
 #include <media/v4l2-fwnode.h>
@@ -699,6 +700,12 @@ static const struct ov13b10_mode supported_2_lanes_modes[] = {
 	},
 };
 
+static const char * const ov13b10_supply_names[] = {
+	"dovdd",        /* Digital I/O power */
+	"avdd",         /* Analog power */
+	"dvdd",         /* Digital core power */
+};
+
 struct ov13b10 {
 	struct device *dev;
 
@@ -708,7 +715,7 @@ struct ov13b10 {
 	struct v4l2_ctrl_handler ctrl_handler;
 
 	struct clk *img_clk;
-	struct regulator *avdd;
+	struct regulator_bulk_data supplies[ARRAY_SIZE(ov13b10_supply_names)];
 	struct gpio_desc *reset;
 
 	/* V4L2 Controls */
@@ -1194,9 +1201,8 @@ static int ov13b10_power_off(struct device *dev)
 	struct ov13b10 *ov13b10 = to_ov13b10(sd);
 
 	gpiod_set_value_cansleep(ov13b10->reset, 1);
-
-	if (ov13b10->avdd)
-		regulator_disable(ov13b10->avdd);
+	regulator_bulk_disable(ARRAY_SIZE(ov13b10_supply_names),
+			       ov13b10->supplies);
 
 	clk_disable_unprepare(ov13b10->img_clk);
 
@@ -1214,14 +1220,12 @@ static int ov13b10_power_on(struct device *dev)
 		dev_err(dev, "failed to enable imaging clock: %d", ret);
 		return ret;
 	}
-
-	if (ov13b10->avdd) {
-		ret = regulator_enable(ov13b10->avdd);
-		if (ret < 0) {
-			dev_err(dev, "failed to enable avdd: %d", ret);
-			clk_disable_unprepare(ov13b10->img_clk);
-			return ret;
-		}
+	ret = regulator_bulk_enable(ARRAY_SIZE(ov13b10_supply_names),
+				    ov13b10->supplies);
+	if (ret < 0) {
+		dev_err(dev, "failed to enable regulators\n");
+		clk_disable_unprepare(ov13b10->img_clk);
+		return ret;
 	}
 
 	gpiod_set_value_cansleep(ov13b10->reset, 0);
@@ -1475,7 +1479,8 @@ static int ov13b10_get_pm_resources(struct ov13b10 *ov13b)
 	unsigned long freq;
 	int ret;
 
-	ov13b->reset = devm_gpiod_get_optional(ov13b->dev, "reset", GPIOD_OUT_LOW);
+	ov13b->reset = devm_gpiod_get_optional(ov13b->dev, "reset",
+					       GPIOD_OUT_LOW);
 	if (IS_ERR(ov13b->reset))
 		return dev_err_probe(ov13b->dev, PTR_ERR(ov13b->reset),
 				     "failed to get reset gpio\n");
@@ -1491,15 +1496,15 @@ static int ov13b10_get_pm_resources(struct ov13b10 *ov13b)
 				     "external clock %lu is not supported\n",
 				     freq);
 
-	ov13b->avdd = devm_regulator_get_optional(ov13b->dev, "avdd");
-	if (IS_ERR(ov13b->avdd)) {
-		ret = PTR_ERR(ov13b->avdd);
-		ov13b->avdd = NULL;
-		if (ret != -ENODEV)
-			return dev_err_probe(ov13b->dev, ret,
-					     "failed to get avdd regulator\n");
-	}
+	for (unsigned int i = 0; i < ARRAY_SIZE(ov13b10_supply_names); i++)
+		ov13b->supplies[i].supply = ov13b10_supply_names[i];
 
+	ret = devm_regulator_bulk_get(ov13b->dev,
+				      ARRAY_SIZE(ov13b10_supply_names),
+				      ov13b->supplies);
+	if (ret)
+		return dev_err_probe(ov13b->dev, ret,
+				     "failed to get regulators\n");
 	return 0;
 }
 
diff --git a/drivers/media/i2c/ov2640.c b/drivers/media/i2c/ov2640.c
index d27fc2df64e6d..50feb608b92b6 100644
--- a/drivers/media/i2c/ov2640.c
+++ b/drivers/media/i2c/ov2640.c
@@ -1271,7 +1271,7 @@ static void ov2640_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ov2640_id[] = {
-	{ "ov2640" },
+	{ .name = "ov2640" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ov2640_id);
diff --git a/drivers/media/i2c/ov2659.c b/drivers/media/i2c/ov2659.c
index 061401b020fcb..7d8c7c3465a4c 100644
--- a/drivers/media/i2c/ov2659.c
+++ b/drivers/media/i2c/ov2659.c
@@ -1553,7 +1553,7 @@ static const struct dev_pm_ops ov2659_pm_ops = {
 };
 
 static const struct i2c_device_id ov2659_id[] = {
-	{ "ov2659" },
+	{ .name = "ov2659" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, ov2659_id);
diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c
index 85ecc23b35877..92d2d6cd4ba48 100644
--- a/drivers/media/i2c/ov5640.c
+++ b/drivers/media/i2c/ov5640.c
@@ -3999,8 +3999,8 @@ static const struct dev_pm_ops ov5640_pm_ops = {
 };
 
 static const struct i2c_device_id ov5640_id[] = {
-	{ "ov5640" },
-	{}
+	{ .name = "ov5640" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ov5640_id);
 
diff --git a/drivers/media/i2c/ov5645.c b/drivers/media/i2c/ov5645.c
index b10d408034a18..c772ef6e51d20 100644
--- a/drivers/media/i2c/ov5645.c
+++ b/drivers/media/i2c/ov5645.c
@@ -1219,8 +1219,8 @@ static void ov5645_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ov5645_id[] = {
-	{ "ov5645" },
-	{}
+	{ .name = "ov5645" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ov5645_id);
 
diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c
index db9bd2892140d..3facf92b3841e 100644
--- a/drivers/media/i2c/ov5647.c
+++ b/drivers/media/i2c/ov5647.c
@@ -1278,7 +1278,7 @@ static const struct dev_pm_ops ov5647_pm_ops = {
 };
 
 static const struct i2c_device_id ov5647_id[] = {
-	{ "ov5647" },
+	{ .name = "ov5647" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, ov5647_id);
diff --git a/drivers/media/i2c/ov7640.c b/drivers/media/i2c/ov7640.c
index 9f68d89936eb7..0fd90bc67e29d 100644
--- a/drivers/media/i2c/ov7640.c
+++ b/drivers/media/i2c/ov7640.c
@@ -77,7 +77,7 @@ static void ov7640_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ov7640_id[] = {
-	{ "ov7640" },
+	{ .name = "ov7640" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ov7640_id);
diff --git a/drivers/media/i2c/ov7670.c b/drivers/media/i2c/ov7670.c
index 0cb96b6c99905..b6d238ba0d53f 100644
--- a/drivers/media/i2c/ov7670.c
+++ b/drivers/media/i2c/ov7670.c
@@ -1997,8 +1997,8 @@ static const struct ov7670_devtype ov7675_devdata = {
 };
 
 static const struct i2c_device_id ov7670_id[] = {
-	{ "ov7670", (kernel_ulong_t)&ov7670_devdata },
-	{ "ov7675", (kernel_ulong_t)&ov7675_devdata },
+	{ .name = "ov7670", .driver_data = (kernel_ulong_t)&ov7670_devdata },
+	{ .name = "ov7675", .driver_data = (kernel_ulong_t)&ov7675_devdata },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, ov7670_id);
diff --git a/drivers/media/i2c/ov772x.c b/drivers/media/i2c/ov772x.c
index 062e1023a411e..be3ba284ee0ba 100644
--- a/drivers/media/i2c/ov772x.c
+++ b/drivers/media/i2c/ov772x.c
@@ -1546,7 +1546,7 @@ static void ov772x_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ov772x_id[] = {
-	{ "ov772x" },
+	{ .name = "ov772x" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ov772x_id);
diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c
index 632fb80469bef..c2e02f1918169 100644
--- a/drivers/media/i2c/ov7740.c
+++ b/drivers/media/i2c/ov7740.c
@@ -1149,7 +1149,7 @@ static int __maybe_unused ov7740_runtime_resume(struct device *dev)
 }
 
 static const struct i2c_device_id ov7740_id[] = {
-	{ "ov7740" },
+	{ .name = "ov7740" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, ov7740_id);
diff --git a/drivers/media/i2c/ov9282.c b/drivers/media/i2c/ov9282.c
index 2167fb73ea415..5b6f897a74fcd 100644
--- a/drivers/media/i2c/ov9282.c
+++ b/drivers/media/i2c/ov9282.c
@@ -586,18 +586,6 @@ static u32 ov9282_flash_duration_to_us(struct ov9282 *ov9282, u32 value)
 	return DIV_ROUND_UP(value * frame_width, OV9282_STROBE_SPAN_FACTOR);
 }
 
-/**
- * ov9282_set_ctrl() - Set subdevice control
- * @ctrl: pointer to v4l2_ctrl structure
- *
- * Supported controls:
- * - V4L2_CID_VBLANK
- * - cluster controls:
- *   - V4L2_CID_ANALOGUE_GAIN
- *   - V4L2_CID_EXPOSURE
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int ov9282_set_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct ov9282 *ov9282 =
@@ -704,14 +692,6 @@ static const struct v4l2_ctrl_ops ov9282_ctrl_ops = {
 	.try_ctrl = ov9282_try_ctrl,
 };
 
-/**
- * ov9282_enum_mbus_code() - Enumerate V4L2 sub-device mbus codes
- * @sd: pointer to ov9282 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @code: V4L2 sub-device code enumeration need to be filled
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int ov9282_enum_mbus_code(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_mbus_code_enum *code)
@@ -780,14 +760,6 @@ static void ov9282_fill_pad_format(struct ov9282 *ov9282,
 	fmt->format.xfer_func = V4L2_XFER_FUNC_NONE;
 }
 
-/**
- * ov9282_get_pad_format() - Get subdevice pad format
- * @sd: pointer to ov9282 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @fmt: V4L2 sub-device format need to be set
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int ov9282_get_pad_format(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
@@ -807,14 +779,6 @@ static int ov9282_get_pad_format(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * ov9282_set_pad_format() - Set subdevice pad format
- * @sd: pointer to ov9282 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- * @fmt: V4L2 sub-device format need to be set
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int ov9282_set_pad_format(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
 				 struct v4l2_subdev_format *fmt)
@@ -852,13 +816,6 @@ static int ov9282_set_pad_format(struct v4l2_subdev *sd,
 	return ret;
 }
 
-/**
- * ov9282_init_state() - Initialize sub-device state
- * @sd: pointer to ov9282 V4L2 sub-device structure
- * @sd_state: V4L2 sub-device configuration
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int ov9282_init_state(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_state *sd_state)
 {
@@ -1157,12 +1114,6 @@ static const struct v4l2_subdev_internal_ops ov9282_internal_ops = {
 	.init_state = ov9282_init_state,
 };
 
-/**
- * ov9282_power_on() - Sensor power on sequence
- * @dev: pointer to i2c device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int ov9282_power_on(struct device *dev)
 {
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
@@ -1206,12 +1157,6 @@ static int ov9282_power_on(struct device *dev)
 	return ret;
 }
 
-/**
- * ov9282_power_off() - Sensor power off sequence
- * @dev: pointer to i2c device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int ov9282_power_off(struct device *dev)
 {
 	struct v4l2_subdev *sd = dev_get_drvdata(dev);
@@ -1333,12 +1278,6 @@ static int ov9282_init_controls(struct ov9282 *ov9282)
 	return 0;
 }
 
-/**
- * ov9282_probe() - I2C client device binding
- * @client: pointer to i2c client device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static int ov9282_probe(struct i2c_client *client)
 {
 	struct ov9282 *ov9282;
@@ -1435,12 +1374,6 @@ static int ov9282_probe(struct i2c_client *client)
 	return ret;
 }
 
-/**
- * ov9282_remove() - I2C client device unbinding
- * @client: pointer to I2C client device
- *
- * Return: 0 if successful, error code otherwise.
- */
 static void ov9282_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
diff --git a/drivers/media/i2c/ov9640.c b/drivers/media/i2c/ov9640.c
index 2190c52b14332..122f411044ce5 100644
--- a/drivers/media/i2c/ov9640.c
+++ b/drivers/media/i2c/ov9640.c
@@ -752,7 +752,7 @@ static void ov9640_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ov9640_id[] = {
-	{ "ov9640" },
+	{ .name = "ov9640" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ov9640_id);
diff --git a/drivers/media/i2c/ov9650.c b/drivers/media/i2c/ov9650.c
index c94e8fe29f226..5c85db8a4a383 100644
--- a/drivers/media/i2c/ov9650.c
+++ b/drivers/media/i2c/ov9650.c
@@ -1567,8 +1567,8 @@ static void ov965x_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ov965x_id[] = {
-	{ "OV9650" },
-	{ "OV9652" },
+	{ .name = "OV9650" },
+	{ .name = "OV9652" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, ov965x_id);
diff --git a/drivers/media/i2c/rj54n1cb0c.c b/drivers/media/i2c/rj54n1cb0c.c
index e95342d706c39..23352d71a108d 100644
--- a/drivers/media/i2c/rj54n1cb0c.c
+++ b/drivers/media/i2c/rj54n1cb0c.c
@@ -1413,7 +1413,7 @@ static void rj54n1_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id rj54n1_id[] = {
-	{ "rj54n1cb0c" },
+	{ .name = "rj54n1cb0c" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, rj54n1_id);
diff --git a/drivers/media/i2c/s5c73m3/s5c73m3-core.c b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
index ab31ee2b596b2..551387cea521e 100644
--- a/drivers/media/i2c/s5c73m3/s5c73m3-core.c
+++ b/drivers/media/i2c/s5c73m3/s5c73m3-core.c
@@ -1728,7 +1728,7 @@ static void s5c73m3_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id s5c73m3_id[] = {
-	{ DRIVER_NAME },
+	{ .name = DRIVER_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, s5c73m3_id);
diff --git a/drivers/media/i2c/s5k5baf.c b/drivers/media/i2c/s5k5baf.c
index d1d00eca87083..378d273055eec 100644
--- a/drivers/media/i2c/s5k5baf.c
+++ b/drivers/media/i2c/s5k5baf.c
@@ -2007,7 +2007,7 @@ static void s5k5baf_remove(struct i2c_client *c)
 }
 
 static const struct i2c_device_id s5k5baf_id[] = {
-	{ S5K5BAF_DRIVER_NAME },
+	{ .name = S5K5BAF_DRIVER_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, s5k5baf_id);
diff --git a/drivers/media/i2c/saa6588.c b/drivers/media/i2c/saa6588.c
index 90ae4121a68a7..56bfa3d2604b4 100644
--- a/drivers/media/i2c/saa6588.c
+++ b/drivers/media/i2c/saa6588.c
@@ -495,7 +495,7 @@ static void saa6588_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id saa6588_id[] = {
-	{ "saa6588" },
+	{ .name = "saa6588" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, saa6588_id);
diff --git a/drivers/media/i2c/saa6752hs.c b/drivers/media/i2c/saa6752hs.c
index 1c0031ba43b44..c6bf0b0902e84 100644
--- a/drivers/media/i2c/saa6752hs.c
+++ b/drivers/media/i2c/saa6752hs.c
@@ -770,7 +770,7 @@ static void saa6752hs_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id saa6752hs_id[] = {
-	{ "saa6752hs" },
+	{ .name = "saa6752hs" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, saa6752hs_id);
diff --git a/drivers/media/i2c/saa7110.c b/drivers/media/i2c/saa7110.c
index 942aeeb40c524..652058b8f7660 100644
--- a/drivers/media/i2c/saa7110.c
+++ b/drivers/media/i2c/saa7110.c
@@ -439,7 +439,7 @@ static void saa7110_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id saa7110_id[] = {
-	{ "saa7110" },
+	{ .name = "saa7110" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, saa7110_id);
diff --git a/drivers/media/i2c/saa7115.c b/drivers/media/i2c/saa7115.c
index 48d6730d92714..7cce90750c934 100644
--- a/drivers/media/i2c/saa7115.c
+++ b/drivers/media/i2c/saa7115.c
@@ -1928,13 +1928,13 @@ static void saa711x_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id saa711x_id[] = {
-	{ "saa7115_auto", 1 }, /* autodetect */
-	{ "saa7111", 0 },
-	{ "saa7113", 0 },
-	{ "saa7114", 0 },
-	{ "saa7115", 0 },
-	{ "saa7118", 0 },
-	{ "gm7113c", 0 },
+	{ .name = "saa7115_auto", .driver_data = 1 }, /* autodetect */
+	{ .name = "saa7111", .driver_data = 0 },
+	{ .name = "saa7113", .driver_data = 0 },
+	{ .name = "saa7114", .driver_data = 0 },
+	{ .name = "saa7115", .driver_data = 0 },
+	{ .name = "saa7118", .driver_data = 0 },
+	{ .name = "gm7113c", .driver_data = 0 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, saa711x_id);
diff --git a/drivers/media/i2c/saa7127.c b/drivers/media/i2c/saa7127.c
index a42a7ffe37684..fdf17f6fae679 100644
--- a/drivers/media/i2c/saa7127.c
+++ b/drivers/media/i2c/saa7127.c
@@ -797,11 +797,11 @@ static void saa7127_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id saa7127_id[] = {
-	{ "saa7127_auto", 0 },	/* auto-detection */
-	{ "saa7126", SAA7127 },
-	{ "saa7127", SAA7127 },
-	{ "saa7128", SAA7129 },
-	{ "saa7129", SAA7129 },
+	{ .name = "saa7127_auto", .driver_data = 0 },	/* auto-detection */
+	{ .name = "saa7126", .driver_data = SAA7127 },
+	{ .name = "saa7127", .driver_data = SAA7127 },
+	{ .name = "saa7128", .driver_data = SAA7129 },
+	{ .name = "saa7129", .driver_data = SAA7129 },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, saa7127_id);
diff --git a/drivers/media/i2c/saa717x.c b/drivers/media/i2c/saa717x.c
index 713331be947c3..0536ceb546506 100644
--- a/drivers/media/i2c/saa717x.c
+++ b/drivers/media/i2c/saa717x.c
@@ -1334,7 +1334,7 @@ static void saa717x_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id saa717x_id[] = {
-	{ "saa717x" },
+	{ .name = "saa717x" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, saa717x_id);
diff --git a/drivers/media/i2c/saa7185.c b/drivers/media/i2c/saa7185.c
index c04e452a332bc..8e5c0eab907db 100644
--- a/drivers/media/i2c/saa7185.c
+++ b/drivers/media/i2c/saa7185.c
@@ -334,7 +334,7 @@ static void saa7185_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id saa7185_id[] = {
-	{ "saa7185" },
+	{ .name = "saa7185" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, saa7185_id);
diff --git a/drivers/media/i2c/sony-btf-mpx.c b/drivers/media/i2c/sony-btf-mpx.c
index 16072a9f82477..4c87de0fe1f0a 100644
--- a/drivers/media/i2c/sony-btf-mpx.c
+++ b/drivers/media/i2c/sony-btf-mpx.c
@@ -366,7 +366,7 @@ static void sony_btf_mpx_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id sony_btf_mpx_id[] = {
-	{ "sony-btf-mpx" },
+	{ .name = "sony-btf-mpx" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, sony_btf_mpx_id);
diff --git a/drivers/media/i2c/tc358743.c b/drivers/media/i2c/tc358743.c
index a0ca19359c431..fbd38bbfee038 100644
--- a/drivers/media/i2c/tc358743.c
+++ b/drivers/media/i2c/tc358743.c
@@ -2358,8 +2358,8 @@ static void tc358743_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tc358743_id[] = {
-	{ "tc358743" },
-	{}
+	{ .name = "tc358743" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, tc358743_id);
diff --git a/drivers/media/i2c/tda1997x.c b/drivers/media/i2c/tda1997x.c
index 5c6dda5338f5f..afa1d6f34c9c5 100644
--- a/drivers/media/i2c/tda1997x.c
+++ b/drivers/media/i2c/tda1997x.c
@@ -2274,9 +2274,9 @@ static int tda1997x_set_power(struct tda1997x_state *state, bool on)
 }
 
 static const struct i2c_device_id tda1997x_i2c_id[] = {
-	{"tda19971", (kernel_ulong_t)&tda1997x_chip_info[TDA19971]},
-	{"tda19973", (kernel_ulong_t)&tda1997x_chip_info[TDA19973]},
-	{ },
+	{ .name = "tda19971", .driver_data = (kernel_ulong_t)&tda1997x_chip_info[TDA19971] },
+	{ .name = "tda19973", .driver_data = (kernel_ulong_t)&tda1997x_chip_info[TDA19973] },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tda1997x_i2c_id);
 
diff --git a/drivers/media/i2c/tda7432.c b/drivers/media/i2c/tda7432.c
index 76ef0fdddf761..a0b65a595ba82 100644
--- a/drivers/media/i2c/tda7432.c
+++ b/drivers/media/i2c/tda7432.c
@@ -400,7 +400,7 @@ static void tda7432_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tda7432_id[] = {
-	{ "tda7432" },
+	{ .name = "tda7432" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tda7432_id);
diff --git a/drivers/media/i2c/tda9840.c b/drivers/media/i2c/tda9840.c
index e3b266db571fe..b34d992777fba 100644
--- a/drivers/media/i2c/tda9840.c
+++ b/drivers/media/i2c/tda9840.c
@@ -182,7 +182,7 @@ static void tda9840_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tda9840_id[] = {
-	{ "tda9840" },
+	{ .name = "tda9840" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tda9840_id);
diff --git a/drivers/media/i2c/tea6415c.c b/drivers/media/i2c/tea6415c.c
index 0cd2e6c52e208..ea7730a7ee2c1 100644
--- a/drivers/media/i2c/tea6415c.c
+++ b/drivers/media/i2c/tea6415c.c
@@ -141,7 +141,7 @@ static void tea6415c_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tea6415c_id[] = {
-	{ "tea6415c" },
+	{ .name = "tea6415c" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tea6415c_id);
diff --git a/drivers/media/i2c/tea6420.c b/drivers/media/i2c/tea6420.c
index 400883fc0c0fe..bebb9a8095dbc 100644
--- a/drivers/media/i2c/tea6420.c
+++ b/drivers/media/i2c/tea6420.c
@@ -123,7 +123,7 @@ static void tea6420_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tea6420_id[] = {
-	{ "tea6420" },
+	{ .name = "tea6420" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tea6420_id);
diff --git a/drivers/media/i2c/ths7303.c b/drivers/media/i2c/ths7303.c
index ff268ebeb4d93..fa5b9c884c1d7 100644
--- a/drivers/media/i2c/ths7303.c
+++ b/drivers/media/i2c/ths7303.c
@@ -369,9 +369,9 @@ static void ths7303_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ths7303_id[] = {
-	{ "ths7303" },
-	{ "ths7353" },
-	{}
+	{ .name = "ths7303" },
+	{ .name = "ths7353" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, ths7303_id);
diff --git a/drivers/media/i2c/ths8200.c b/drivers/media/i2c/ths8200.c
index 686f10641c7a6..808ef16ec3b3f 100644
--- a/drivers/media/i2c/ths8200.c
+++ b/drivers/media/i2c/ths8200.c
@@ -487,8 +487,8 @@ static void ths8200_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ths8200_id[] = {
-	{ "ths8200" },
-	{}
+	{ .name = "ths8200" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ths8200_id);
 
diff --git a/drivers/media/i2c/tlv320aic23b.c b/drivers/media/i2c/tlv320aic23b.c
index 6f6bc5236565c..7deeef3177147 100644
--- a/drivers/media/i2c/tlv320aic23b.c
+++ b/drivers/media/i2c/tlv320aic23b.c
@@ -188,7 +188,7 @@ static void tlv320aic23b_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id tlv320aic23b_id[] = {
-	{ "tlv320aic23b" },
+	{ .name = "tlv320aic23b" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tlv320aic23b_id);
diff --git a/drivers/media/i2c/tvaudio.c b/drivers/media/i2c/tvaudio.c
index 6267e9ad39c01..f136310b899d5 100644
--- a/drivers/media/i2c/tvaudio.c
+++ b/drivers/media/i2c/tvaudio.c
@@ -2086,7 +2086,7 @@ static void tvaudio_remove(struct i2c_client *client)
    detect which device is present. So rather than listing all supported
    devices here, we pretend to support a single, fake device type. */
 static const struct i2c_device_id tvaudio_id[] = {
-	{ "tvaudio" },
+	{ .name = "tvaudio" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tvaudio_id);
diff --git a/drivers/media/i2c/tvp514x.c b/drivers/media/i2c/tvp514x.c
index f9c9c80c33ac1..376eecb0b6730 100644
--- a/drivers/media/i2c/tvp514x.c
+++ b/drivers/media/i2c/tvp514x.c
@@ -686,13 +686,6 @@ static int tvp514x_s_routing(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * tvp514x_s_ctrl() - V4L2 decoder interface handler for s_ctrl
- * @ctrl: pointer to v4l2_ctrl structure
- *
- * If the requested control is supported, sets the control's current
- * value in HW. Otherwise, returns -EINVAL if the control is not supported.
- */
 static int tvp514x_s_ctrl(struct v4l2_ctrl *ctrl)
 {
 	struct v4l2_subdev *sd = to_sd(ctrl);
@@ -789,13 +782,6 @@ tvp514x_set_frame_interval(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * tvp514x_s_stream() - V4L2 decoder i/f handler for s_stream
- * @sd: pointer to standard V4L2 sub-device structure
- * @enable: streaming enable or disable
- *
- * Sets streaming to enable or disable, if possible.
- */
 static int tvp514x_s_stream(struct v4l2_subdev *sd, int enable)
 {
 	int err = 0;
@@ -850,14 +836,6 @@ static const struct v4l2_ctrl_ops tvp514x_ctrl_ops = {
 	.s_ctrl = tvp514x_s_ctrl,
 };
 
-/**
- * tvp514x_enum_mbus_code() - V4L2 decoder interface handler for enum_mbus_code
- * @sd: pointer to standard V4L2 sub-device structure
- * @sd_state: subdev state
- * @code: pointer to v4l2_subdev_mbus_code_enum structure
- *
- * Enumertaes mbus codes supported
- */
 static int tvp514x_enum_mbus_code(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_mbus_code_enum *code)
@@ -877,14 +855,6 @@ static int tvp514x_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * tvp514x_get_pad_format() - V4L2 decoder interface handler for get pad format
- * @sd: pointer to standard V4L2 sub-device structure
- * @sd_state: subdev state
- * @format: pointer to v4l2_subdev_format structure
- *
- * Retrieves pad format which is active or tried based on requirement
- */
 static int tvp514x_get_pad_format(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_format *format)
@@ -909,14 +879,6 @@ static int tvp514x_get_pad_format(struct v4l2_subdev *sd,
 	return 0;
 }
 
-/**
- * tvp514x_set_pad_format() - V4L2 decoder interface handler for set pad format
- * @sd: pointer to standard V4L2 sub-device structure
- * @sd_state: subdev state
- * @fmt: pointer to v4l2_subdev_format structure
- *
- * Set pad format for the output pad
- */
 static int tvp514x_set_pad_format(struct v4l2_subdev *sd,
 				  struct v4l2_subdev_state *sd_state,
 				  struct v4l2_subdev_format *fmt)
@@ -1014,15 +976,7 @@ tvp514x_get_pdata(struct i2c_client *client)
 	return pdata;
 }
 
-/**
- * tvp514x_probe() - decoder driver i2c probe handler
- * @client: i2c driver client device structure
- *
- * Register decoder as an i2c client device and V4L2
- * device.
- */
-static int
-tvp514x_probe(struct i2c_client *client)
+static int tvp514x_probe(struct i2c_client *client)
 {
 	struct tvp514x_platform_data *pdata = tvp514x_get_pdata(client);
 	struct tvp514x_decoder *decoder;
@@ -1113,13 +1067,6 @@ tvp514x_probe(struct i2c_client *client)
 	return ret;
 }
 
-/**
- * tvp514x_remove() - decoder driver i2c remove handler
- * @client: i2c driver client device structure
- *
- * Unregister decoder as an i2c client device and V4L2
- * device. Complement of tvp514x_probe().
- */
 static void tvp514x_remove(struct i2c_client *client)
 {
 	struct v4l2_subdev *sd = i2c_get_clientdata(client);
@@ -1182,10 +1129,10 @@ static const struct tvp514x_reg tvp514xm_init_reg_seq[] = {
  * driver_data - Driver data
  */
 static const struct i2c_device_id tvp514x_id[] = {
-	{"tvp5146", (kernel_ulong_t)tvp5146_init_reg_seq },
-	{"tvp5146m2", (kernel_ulong_t)tvp514xm_init_reg_seq },
-	{"tvp5147", (kernel_ulong_t)tvp5147_init_reg_seq },
-	{"tvp5147m1", (kernel_ulong_t)tvp514xm_init_reg_seq },
+	{ .name = "tvp5146", .driver_data = (kernel_ulong_t)tvp5146_init_reg_seq },
+	{ .name = "tvp5146m2", .driver_data = (kernel_ulong_t)tvp514xm_init_reg_seq },
+	{ .name = "tvp5147", .driver_data = (kernel_ulong_t)tvp5147_init_reg_seq },
+	{ .name = "tvp5147m1", .driver_data = (kernel_ulong_t)tvp514xm_init_reg_seq },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, tvp514x_id);
diff --git a/drivers/media/i2c/tvp5150.c b/drivers/media/i2c/tvp5150.c
index e3675c744d9ed..9c204f38935dd 100644
--- a/drivers/media/i2c/tvp5150.c
+++ b/drivers/media/i2c/tvp5150.c
@@ -2265,7 +2265,7 @@ static const struct dev_pm_ops tvp5150_pm_ops = {
 };
 
 static const struct i2c_device_id tvp5150_id[] = {
-	{ "tvp5150" },
+	{ .name = "tvp5150" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tvp5150_id);
diff --git a/drivers/media/i2c/tvp7002.c b/drivers/media/i2c/tvp7002.c
index c09a5bd71fd0b..3979ccde5a954 100644
--- a/drivers/media/i2c/tvp7002.c
+++ b/drivers/media/i2c/tvp7002.c
@@ -1070,7 +1070,7 @@ static void tvp7002_remove(struct i2c_client *c)
 
 /* I2C Device ID table */
 static const struct i2c_device_id tvp7002_id[] = {
-	{ "tvp7002" },
+	{ .name = "tvp7002" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tvp7002_id);
diff --git a/drivers/media/i2c/tw2804.c b/drivers/media/i2c/tw2804.c
index 3d154f4fb5f9d..713e078ff3dac 100644
--- a/drivers/media/i2c/tw2804.c
+++ b/drivers/media/i2c/tw2804.c
@@ -414,7 +414,7 @@ static void tw2804_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tw2804_id[] = {
-	{ "tw2804" },
+	{ .name = "tw2804" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tw2804_id);
diff --git a/drivers/media/i2c/tw9900.c b/drivers/media/i2c/tw9900.c
index 53efdeaed1dba..617fcf7f0b453 100644
--- a/drivers/media/i2c/tw9900.c
+++ b/drivers/media/i2c/tw9900.c
@@ -753,7 +753,7 @@ static const struct dev_pm_ops tw9900_pm_ops = {
 };
 
 static const struct i2c_device_id tw9900_id[] = {
-	{ "tw9900" },
+	{ .name = "tw9900" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tw9900_id);
diff --git a/drivers/media/i2c/tw9903.c b/drivers/media/i2c/tw9903.c
index c3eafd5d5dc82..db063b885b09c 100644
--- a/drivers/media/i2c/tw9903.c
+++ b/drivers/media/i2c/tw9903.c
@@ -246,7 +246,7 @@ static void tw9903_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id tw9903_id[] = {
-	{ "tw9903" },
+	{ .name = "tw9903" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tw9903_id);
diff --git a/drivers/media/i2c/tw9906.c b/drivers/media/i2c/tw9906.c
index 0ab43fe42d7f4..079e2f49f38f3 100644
--- a/drivers/media/i2c/tw9906.c
+++ b/drivers/media/i2c/tw9906.c
@@ -214,7 +214,7 @@ static void tw9906_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id tw9906_id[] = {
-	{ "tw9906" },
+	{ .name = "tw9906" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tw9906_id);
diff --git a/drivers/media/i2c/tw9910.c b/drivers/media/i2c/tw9910.c
index f3e400304e042..872207e688bf6 100644
--- a/drivers/media/i2c/tw9910.c
+++ b/drivers/media/i2c/tw9910.c
@@ -996,7 +996,7 @@ static void tw9910_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tw9910_id[] = {
-	{ "tw9910" },
+	{ .name = "tw9910" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tw9910_id);
diff --git a/drivers/media/i2c/uda1342.c b/drivers/media/i2c/uda1342.c
index 2e4540ee2df2d..437726788ba02 100644
--- a/drivers/media/i2c/uda1342.c
+++ b/drivers/media/i2c/uda1342.c
@@ -79,7 +79,7 @@ static void uda1342_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id uda1342_id[] = {
-	{ "uda1342" },
+	{ .name = "uda1342" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, uda1342_id);
diff --git a/drivers/media/i2c/upd64031a.c b/drivers/media/i2c/upd64031a.c
index a178af46e6951..670118c168724 100644
--- a/drivers/media/i2c/upd64031a.c
+++ b/drivers/media/i2c/upd64031a.c
@@ -219,7 +219,7 @@ static void upd64031a_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id upd64031a_id[] = {
-	{ "upd64031a" },
+	{ .name = "upd64031a" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, upd64031a_id);
diff --git a/drivers/media/i2c/upd64083.c b/drivers/media/i2c/upd64083.c
index 5421dc5e32c92..e610dffa9e10d 100644
--- a/drivers/media/i2c/upd64083.c
+++ b/drivers/media/i2c/upd64083.c
@@ -190,7 +190,7 @@ static void upd64083_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id upd64083_id[] = {
-	{ "upd64083" },
+	{ .name = "upd64083" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, upd64083_id);
diff --git a/drivers/media/i2c/video-i2c.c b/drivers/media/i2c/video-i2c.c
index fef3993f4e2d1..56b99eea54a17 100644
--- a/drivers/media/i2c/video-i2c.c
+++ b/drivers/media/i2c/video-i2c.c
@@ -454,8 +454,9 @@ static int video_i2c_thread_vid_cap(void *priv)
 		spin_lock(&data->slock);
 
 		if (!list_empty(&data->vid_cap_active)) {
-			vid_cap_buf = list_last_entry(&data->vid_cap_active,
-						 struct video_i2c_buffer, list);
+			vid_cap_buf = list_first_entry(&data->vid_cap_active,
+						       struct video_i2c_buffer,
+						       list);
 			list_del(&vid_cap_buf->list);
 		}
 
@@ -888,7 +889,7 @@ static void video_i2c_remove(struct i2c_client *client)
 	if (data->chip->set_power)
 		data->chip->set_power(data, false);
 
-	video_unregister_device(&data->vdev);
+	vb2_video_unregister_device(&data->vdev);
 }
 
 #ifdef CONFIG_PM
@@ -921,9 +922,9 @@ static const struct dev_pm_ops video_i2c_pm_ops = {
 };
 
 static const struct i2c_device_id video_i2c_id_table[] = {
-	{ "amg88xx", (kernel_ulong_t)&video_i2c_chip[AMG88XX] },
-	{ "mlx90640", (kernel_ulong_t)&video_i2c_chip[MLX90640] },
-	{}
+	{ .name = "amg88xx", .driver_data = (kernel_ulong_t)&video_i2c_chip[AMG88XX] },
+	{ .name = "mlx90640", .driver_data = (kernel_ulong_t)&video_i2c_chip[MLX90640] },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, video_i2c_id_table);
 
diff --git a/drivers/media/i2c/vp27smpx.c b/drivers/media/i2c/vp27smpx.c
index df21950be24f2..21fcfdd4c1630 100644
--- a/drivers/media/i2c/vp27smpx.c
+++ b/drivers/media/i2c/vp27smpx.c
@@ -172,7 +172,7 @@ static void vp27smpx_remove(struct i2c_client *client)
 /* ----------------------------------------------------------------------- */
 
 static const struct i2c_device_id vp27smpx_id[] = {
-	{ "vp27smpx" },
+	{ .name = "vp27smpx" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, vp27smpx_id);
diff --git a/drivers/media/i2c/vpx3220.c b/drivers/media/i2c/vpx3220.c
index 5f1a22284168d..29bcbb5a5fbbe 100644
--- a/drivers/media/i2c/vpx3220.c
+++ b/drivers/media/i2c/vpx3220.c
@@ -535,9 +535,9 @@ static void vpx3220_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id vpx3220_id[] = {
-	{ "vpx3220a" },
-	{ "vpx3216b" },
-	{ "vpx3214c" },
+	{ .name = "vpx3220a" },
+	{ .name = "vpx3216b" },
+	{ .name = "vpx3214c" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, vpx3220_id);
diff --git a/drivers/media/i2c/wm8739.c b/drivers/media/i2c/wm8739.c
index 72eb10339d062..db62bafb447c4 100644
--- a/drivers/media/i2c/wm8739.c
+++ b/drivers/media/i2c/wm8739.c
@@ -243,7 +243,7 @@ static void wm8739_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id wm8739_id[] = {
-	{ "wm8739" },
+	{ .name = "wm8739" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, wm8739_id);
diff --git a/drivers/media/i2c/wm8775.c b/drivers/media/i2c/wm8775.c
index 56778d3bc28a7..5db197bb6fdaa 100644
--- a/drivers/media/i2c/wm8775.c
+++ b/drivers/media/i2c/wm8775.c
@@ -289,7 +289,7 @@ static void wm8775_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id wm8775_id[] = {
-	{ "wm8775" },
+	{ .name = "wm8775" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, wm8775_id);
diff --git a/drivers/media/pci/Kconfig b/drivers/media/pci/Kconfig
index eebb16c58f3d1..bfdb200f85a33 100644
--- a/drivers/media/pci/Kconfig
+++ b/drivers/media/pci/Kconfig
@@ -13,6 +13,7 @@ if MEDIA_PCI_SUPPORT
 if MEDIA_CAMERA_SUPPORT
 	comment "Media capture support"
 
+source "drivers/media/pci/hws/Kconfig"
 source "drivers/media/pci/mgb4/Kconfig"
 source "drivers/media/pci/solo6x10/Kconfig"
 source "drivers/media/pci/tw5864/Kconfig"
diff --git a/drivers/media/pci/Makefile b/drivers/media/pci/Makefile
index 02763ad885115..c4508b6723a98 100644
--- a/drivers/media/pci/Makefile
+++ b/drivers/media/pci/Makefile
@@ -29,6 +29,7 @@ obj-$(CONFIG_VIDEO_CX23885) += cx23885/
 obj-$(CONFIG_VIDEO_CX25821) += cx25821/
 obj-$(CONFIG_VIDEO_CX88) += cx88/
 obj-$(CONFIG_VIDEO_DT3155) += dt3155/
+obj-$(CONFIG_VIDEO_HWS) += hws/
 obj-$(CONFIG_VIDEO_IVTV) += ivtv/
 obj-$(CONFIG_VIDEO_MGB4) += mgb4/
 obj-$(CONFIG_VIDEO_SAA7134) += saa7134/
diff --git a/drivers/media/pci/cx23885/cx23885-core.c b/drivers/media/pci/cx23885/cx23885-core.c
index 4a8af8b88d84d..9b92e8db494c5 100644
--- a/drivers/media/pci/cx23885/cx23885-core.c
+++ b/drivers/media/pci/cx23885/cx23885-core.c
@@ -1002,8 +1002,12 @@ static int cx23885_dev_setup(struct cx23885_dev *dev)
 	}
 
 	/* PCIe stuff */
-	dev->lmmio = ioremap(pci_resource_start(dev->pci, 0),
-			     pci_resource_len(dev->pci, 0));
+	dev->lmmio = pci_ioremap_bar(dev->pci, 0);
+	if (!dev->lmmio) {
+		dev_err(&dev->pci->dev, "CORE %s: can't ioremap MMIO memory\n",
+			dev->name);
+		goto err_release_region;
+	}
 
 	dev->bmmio = (u8 __iomem *)dev->lmmio;
 
@@ -1109,6 +1113,12 @@ static int cx23885_dev_setup(struct cx23885_dev *dev)
 	}
 
 	return 0;
+
+err_release_region:
+	release_mem_region(pci_resource_start(dev->pci, 0),
+			   pci_resource_len(dev->pci, 0));
+	cx23885_devcount--;
+	return -ENODEV;
 }
 
 static void cx23885_dev_unregister(struct cx23885_dev *dev)
diff --git a/drivers/media/pci/ddbridge/ddbridge-core.c b/drivers/media/pci/ddbridge/ddbridge-core.c
index 40e6c873c36d2..d240e291ba4fe 100644
--- a/drivers/media/pci/ddbridge/ddbridge-core.c
+++ b/drivers/media/pci/ddbridge/ddbridge-core.c
@@ -3430,7 +3430,7 @@ int ddb_init_ddbridge(void)
 
 	if (ddb_class_create() < 0)
 		return -1;
-	ddb_wq = alloc_workqueue("ddbridge", 0, 0);
+	ddb_wq = alloc_workqueue("ddbridge", WQ_PERCPU, 0);
 	if (!ddb_wq)
 		return ddb_exit_ddbridge(1, -1);
 
diff --git a/drivers/media/pci/dm1105/dm1105.c b/drivers/media/pci/dm1105/dm1105.c
index bbd24769ae56b..80d2e143384b4 100644
--- a/drivers/media/pci/dm1105/dm1105.c
+++ b/drivers/media/pci/dm1105/dm1105.c
@@ -768,6 +768,8 @@ static void dm1105_ir_exit(struct dm1105_dev *dm1105)
 
 static int dm1105_hw_init(struct dm1105_dev *dev)
 {
+	int ret;
+
 	dm1105_disable_irqs(dev);
 
 	dm_writeb(DM1105_HOST_CTR, 0);
@@ -778,7 +780,10 @@ static int dm1105_hw_init(struct dm1105_dev *dev)
 	dm_writew(DM1105_TSCTR, 0xc10a);
 
 	/* map DMA and set address */
-	dm1105_dma_map(dev);
+	ret = dm1105_dma_map(dev);
+	if (ret)
+		return -ENOMEM;
+
 	dm1105_set_dma_addr(dev);
 	/* big buffer */
 	dm_writel(DM1105_RLEN, 5 * DM1105_DMA_BYTES);
@@ -1194,6 +1199,7 @@ static void dm1105_remove(struct pci_dev *pdev)
 
 	dm1105_hw_exit(dev);
 	free_irq(pdev->irq, dev);
+	destroy_workqueue(dev->wq);
 	pci_iounmap(pdev, dev->io_mem);
 	pci_release_regions(pdev);
 	pci_disable_device(pdev);
diff --git a/drivers/media/pci/hws/Kconfig b/drivers/media/pci/hws/Kconfig
new file mode 100644
index 0000000000000..ab0bbf49ca716
--- /dev/null
+++ b/drivers/media/pci/hws/Kconfig
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config VIDEO_HWS
+	tristate "AVMatrix HWS PCIe capture devices"
+	depends on PCI && VIDEO_DEV
+	select VIDEOBUF2_DMA_CONTIG
+	help
+	  Enable support for AVMatrix HWS series multi-channel PCIe capture
+	  devices that provide HDMI video capture.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be named hws.
diff --git a/drivers/media/pci/hws/Makefile b/drivers/media/pci/hws/Makefile
new file mode 100644
index 0000000000000..f9c7dc4f2d8d4
--- /dev/null
+++ b/drivers/media/pci/hws/Makefile
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
+hws-objs := hws_pci.o hws_irq.o hws_video.o hws_v4l2_ioctl.o
+
+obj-$(CONFIG_VIDEO_HWS) += hws.o
diff --git a/drivers/media/pci/hws/hws.h b/drivers/media/pci/hws/hws.h
new file mode 100644
index 0000000000000..8fbe1fe27844d
--- /dev/null
+++ b/drivers/media/pci/hws/hws.h
@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef HWS_PCIE_H
+#define HWS_PCIE_H
+
+#include <linux/types.h>
+#include <linux/compiler.h>
+#include <linux/dma-mapping.h>
+#include <linux/kthread.h>
+#include <linux/pci.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/sizes.h>
+#include <linux/atomic.h>
+
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-dv-timings.h>
+#include <media/videobuf2-dma-sg.h>
+
+#include "hws_reg.h"
+
+struct hwsmem_param {
+	u32 index;
+	u32 type;
+	u32 status;
+};
+
+struct hws_pix_state {
+	u32 width;
+	u32 height;
+	u32 fourcc;		/* V4L2_PIX_FMT_* (YUYV only here) */
+	u32 bytesperline;	/* stride */
+	u32 sizeimage;		/* full frame */
+	enum v4l2_field field;	/* V4L2_FIELD_NONE or INTERLACED */
+	enum v4l2_colorspace colorspace;	/* e.g., REC709 */
+	enum v4l2_ycbcr_encoding ycbcr_enc;	/* V4L2_YCBCR_ENC_DEFAULT */
+	enum v4l2_quantization quantization;	/* V4L2_QUANTIZATION_LIM_RANGE */
+	enum v4l2_xfer_func xfer_func;	/* V4L2_XFER_FUNC_DEFAULT */
+	bool interlaced;	/* cached hardware state */
+	u32 half_size;		/* hardware half-frame size */
+};
+
+#define	UNSET	(-1U)
+
+struct hws_pcie_dev;
+struct hws_adapter;
+struct hws_video;
+
+struct hwsvideo_buffer {
+	struct vb2_v4l2_buffer vb;
+	struct list_head list;
+	int slot;
+};
+
+struct hws_video {
+	/* Linkage */
+	struct hws_pcie_dev *parent;
+	struct video_device *video_device;
+
+	struct vb2_queue buffer_queue;
+	struct list_head capture_queue;
+	struct hwsvideo_buffer *active;
+	struct hwsvideo_buffer *next_prepared;
+
+	/* Locking */
+	struct mutex state_lock;
+	spinlock_t irq_lock;	/* Protects capture_queue and active buffers. */
+
+	/* Indices */
+	int channel_index;
+
+	/* Color controls */
+	int current_brightness;
+	int current_contrast;
+	int current_saturation;
+	int current_hue;
+
+	/* V4L2 controls */
+	struct v4l2_ctrl_handler control_handler;
+	struct v4l2_ctrl *ctrl_brightness;
+	struct v4l2_ctrl *ctrl_contrast;
+	struct v4l2_ctrl *ctrl_saturation;
+	struct v4l2_ctrl *ctrl_hue;
+
+	/* Capture queue status */
+	struct hws_pix_state pix;
+	struct v4l2_dv_timings cur_dv_timings; /* last configured/notified DV timings */
+	u32 current_fps; /* Hz, updated by mode changes, not by read-only queries */
+
+	/* Per-channel capture state */
+	bool cap_active;
+	bool stop_requested;
+	u8 last_buf_half_toggle;
+	bool half_seen;
+	atomic_t sequence_number;
+	u32 queued_count;
+
+	/* Timeout and error handling */
+	u32 timeout_count;
+	u32 error_count;
+
+	bool window_valid;
+	u32 last_dma_hi;
+	u32 last_dma_page;
+	u32 last_pci_addr;
+	u32 last_half16;
+
+	/* Misc counters */
+	int signal_loss_cnt;
+};
+
+static inline void hws_set_current_dv_timings(struct hws_video *vid,
+					      u32 width, u32 height,
+					      bool interlaced)
+{
+	if (!vid)
+		return;
+
+	vid->cur_dv_timings = (struct v4l2_dv_timings) {
+		.type = V4L2_DV_BT_656_1120,
+		.bt = {
+			.width = width,
+			.height = height,
+			.interlaced = interlaced,
+		},
+	};
+}
+
+struct hws_scratch_dma {
+	void *cpu;
+	dma_addr_t dma;
+	size_t size;
+};
+
+struct hws_pcie_dev {
+	/* Core objects */
+	struct pci_dev *pdev;
+	struct hws_video video[MAX_VID_CHANNELS];
+
+	/* BAR and workqueues */
+	void __iomem *bar0_base;
+
+	/* Device identity and capabilities */
+	u16 vendor_id;
+	u16 device_id;
+	u16 device_ver;
+	u16 hw_ver;
+	u32 sub_ver;
+	u32 port_id;
+	/* Tri-state support flag used by set_video_format_size(). */
+	u32 support_yv12;
+	u32 max_hw_video_buf_sz;
+	u8 max_channels;
+	u8 cur_max_video_ch;
+	bool start_run;
+
+	bool buf_allocated;
+
+	/* V4L2 framework objects */
+	struct v4l2_device v4l2_device;
+
+	/* Kernel thread */
+	struct task_struct *main_task;
+	struct hws_scratch_dma scratch_vid[MAX_VID_CHANNELS];
+
+	bool suspended;
+	int irq;
+
+	/* Error flags */
+	int pci_lost;
+};
+
+#endif
diff --git a/drivers/media/pci/hws/hws_irq.c b/drivers/media/pci/hws/hws_irq.c
new file mode 100644
index 0000000000000..eebb4b8a5cd53
--- /dev/null
+++ b/drivers/media/pci/hws/hws_irq.c
@@ -0,0 +1,269 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/compiler.h>
+#include <linux/moduleparam.h>
+#include <linux/io.h>
+#include <linux/dma-mapping.h>
+#include <linux/interrupt.h>
+#include <linux/minmax.h>
+#include <linux/string.h>
+
+#include <media/videobuf2-dma-contig.h>
+
+#include "hws_irq.h"
+#include "hws_reg.h"
+#include "hws_video.h"
+#include "hws.h"
+
+#define MAX_INT_LOOPS 100
+
+static bool hws_toggle_debug;
+module_param_named(toggle_debug, hws_toggle_debug, bool, 0644);
+MODULE_PARM_DESC(toggle_debug,
+		 "Read toggle registers in IRQ handler for debug logging");
+
+static int hws_arm_next(struct hws_pcie_dev *hws, u32 ch)
+{
+	struct hws_video *v = &hws->video[ch];
+	unsigned long flags;
+	struct hwsvideo_buffer *buf;
+
+	dev_dbg(&hws->pdev->dev,
+		"arm_next(ch=%u): stop=%d cap=%d queued=%d\n",
+		ch, READ_ONCE(v->stop_requested), READ_ONCE(v->cap_active),
+		!list_empty(&v->capture_queue));
+
+	if (READ_ONCE(hws->suspended)) {
+		dev_dbg(&hws->pdev->dev, "arm_next(ch=%u): suspended\n", ch);
+		return -EBUSY;
+	}
+
+	if (READ_ONCE(v->stop_requested) || !READ_ONCE(v->cap_active)) {
+		dev_dbg(&hws->pdev->dev,
+			"arm_next(ch=%u): stop=%d cap=%d -> cancel\n", ch,
+			v->stop_requested, v->cap_active);
+		return -ECANCELED;
+	}
+
+	spin_lock_irqsave(&v->irq_lock, flags);
+	if (list_empty(&v->capture_queue)) {
+		spin_unlock_irqrestore(&v->irq_lock, flags);
+		dev_dbg(&hws->pdev->dev, "arm_next(ch=%u): queue empty\n", ch);
+		return -EAGAIN;
+	}
+
+	buf = list_first_entry(&v->capture_queue, struct hwsvideo_buffer, list);
+	list_del_init(&buf->list);	/* keep buffer safe for later cleanup */
+	if (v->queued_count)
+		v->queued_count--;
+	v->active = buf;
+	spin_unlock_irqrestore(&v->irq_lock, flags);
+	dev_dbg(&hws->pdev->dev, "arm_next(ch=%u): picked buffer %p\n", ch,
+		buf);
+
+	/* Publish descriptor(s) before doorbell/MMIO kicks. */
+	wmb();
+
+	/* Avoid MMIO during suspend */
+	if (READ_ONCE(hws->suspended)) {
+		unsigned long f;
+
+		dev_dbg(&hws->pdev->dev,
+			"arm_next(ch=%u): suspended after pick\n", ch);
+		spin_lock_irqsave(&v->irq_lock, f);
+		if (v->active) {
+			list_add(&buf->list, &v->capture_queue);
+			v->queued_count++;
+			v->active = NULL;
+		}
+		spin_unlock_irqrestore(&v->irq_lock, f);
+		return -EBUSY;
+	}
+
+	/* Also program the DMA address register directly */
+	{
+		dma_addr_t dma_addr =
+		    vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
+		hws_program_dma_for_addr(hws, ch, dma_addr);
+		iowrite32(lower_32_bits(dma_addr),
+			  hws->bar0_base + HWS_REG_DMA_ADDR(ch));
+	}
+
+	dev_dbg(&hws->pdev->dev, "arm_next(ch=%u): programmed buffer %p\n", ch,
+		buf);
+	spin_lock_irqsave(&v->irq_lock, flags);
+	hws_prime_next_locked(v);
+	spin_unlock_irqrestore(&v->irq_lock, flags);
+	return 0;
+}
+
+static void hws_video_handle_vdone(struct hws_video *v)
+{
+	struct hws_pcie_dev *hws = v->parent;
+	unsigned int ch = v->channel_index;
+	struct hwsvideo_buffer *done;
+	unsigned long flags;
+	bool promoted = false;
+
+	dev_dbg(&hws->pdev->dev,
+		"bh_video(ch=%u): stop=%d cap=%d active=%p\n",
+		ch, READ_ONCE(v->stop_requested), READ_ONCE(v->cap_active),
+		v->active);
+
+	int ret;
+
+	dev_dbg(&hws->pdev->dev,
+		"bh_video(ch=%u): entry stop=%d cap=%d\n", ch,
+		v->stop_requested, v->cap_active);
+	if (READ_ONCE(hws->suspended))
+		return;
+
+	if (READ_ONCE(v->stop_requested) || !READ_ONCE(v->cap_active))
+		return;
+
+	spin_lock_irqsave(&v->irq_lock, flags);
+	done = v->active;
+	if (done && v->next_prepared) {
+		v->active = v->next_prepared;
+		v->next_prepared = NULL;
+		promoted = true;
+	}
+	spin_unlock_irqrestore(&v->irq_lock, flags);
+
+	/* 1) Complete the buffer the HW just finished (if any) */
+	if (done) {
+		struct vb2_v4l2_buffer *vb2v = &done->vb;
+		size_t expected = v->pix.sizeimage;
+		size_t plane_size = vb2_plane_size(&vb2v->vb2_buf, 0);
+
+		if (expected > plane_size) {
+			dev_warn_ratelimited(&hws->pdev->dev,
+					     "bh_video(ch=%u): sizeimage %zu > plane %zu, dropping seq=%u\n",
+					     ch, expected, plane_size,
+					     (u32)atomic_read(&v->sequence_number) + 1);
+			vb2_buffer_done(&vb2v->vb2_buf, VB2_BUF_STATE_ERROR);
+			goto arm_next;
+		}
+		vb2_set_plane_payload(&vb2v->vb2_buf, 0, expected);
+
+		dma_rmb();	/* device writes visible before userspace sees it */
+
+		vb2v->sequence = (u32)atomic_inc_return(&v->sequence_number);
+		vb2v->vb2_buf.timestamp = ktime_get_ns();
+		dev_dbg(&hws->pdev->dev,
+			"bh_video(ch=%u): DONE buf=%p seq=%u half_seen=%d toggle=%u\n",
+			ch, done, vb2v->sequence, v->half_seen,
+			v->last_buf_half_toggle);
+
+		if (!promoted)
+			v->active = NULL;	/* channel no longer owns this buffer */
+		vb2_buffer_done(&vb2v->vb2_buf, VB2_BUF_STATE_DONE);
+	}
+
+	if (READ_ONCE(hws->suspended))
+		return;
+
+	if (promoted) {
+		dev_dbg(&hws->pdev->dev,
+			"bh_video(ch=%u): promoted pre-armed buffer active=%p\n",
+			ch, v->active);
+		spin_lock_irqsave(&v->irq_lock, flags);
+		hws_prime_next_locked(v);
+		spin_unlock_irqrestore(&v->irq_lock, flags);
+		return;
+	}
+
+arm_next:
+	/* 2) Immediately arm the next queued buffer (if present) */
+	ret = hws_arm_next(hws, ch);
+	if (ret == -EAGAIN) {
+		dev_dbg(&hws->pdev->dev,
+			"bh_video(ch=%u): no queued buffer to arm\n", ch);
+		return;
+	}
+	dev_dbg(&hws->pdev->dev,
+		"bh_video(ch=%u): armed next buffer, active=%p\n", ch,
+		v->active);
+	/* On success the engine now points at v->active's DMA address */
+}
+
+irqreturn_t hws_irq_handler(int irq, void *info)
+{
+	struct hws_pcie_dev *pdx = info;
+	u32 int_state;
+
+	dev_dbg(&pdx->pdev->dev, "irq: entry\n");
+	if (pdx->bar0_base) {
+		dev_dbg(&pdx->pdev->dev,
+			"irq: INT_EN=0x%08x INT_STATUS=0x%08x\n",
+			readl(pdx->bar0_base + INT_EN_REG_BASE),
+			readl(pdx->bar0_base + HWS_REG_INT_STATUS));
+	}
+
+	/* Fast path: if suspended, quietly ack and exit */
+	if (READ_ONCE(pdx->suspended)) {
+		int_state = readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS);
+		if (int_state) {
+			writel(int_state, pdx->bar0_base + HWS_REG_INT_STATUS);
+			(void)readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS);
+		}
+		return int_state ? IRQ_HANDLED : IRQ_NONE;
+	}
+	int_state = readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS);
+	if (!int_state || int_state == 0xFFFFFFFF) {
+		dev_dbg(&pdx->pdev->dev,
+			"irq: spurious or device-gone int_state=0x%08x\n",
+			int_state);
+		return IRQ_NONE;
+	}
+	dev_dbg(&pdx->pdev->dev, "irq: entry INT_STATUS=0x%08x\n", int_state);
+
+	/* Loop until all pending bits are serviced (max 100 iterations) */
+	for (u32 cnt = 0; int_state && cnt < MAX_INT_LOOPS; ++cnt) {
+		for (unsigned int ch = 0; ch < pdx->cur_max_video_ch; ++ch) {
+			u32 vbit = HWS_INT_VDONE_BIT(ch);
+
+			if (!(int_state & vbit))
+				continue;
+
+			if (READ_ONCE(pdx->video[ch].cap_active) &&
+			    !READ_ONCE(pdx->video[ch].stop_requested)) {
+				if (hws_toggle_debug) {
+					u32 toggle =
+					    readl_relaxed(pdx->bar0_base +
+						  HWS_REG_VBUF_TOGGLE(ch)) & 0x01;
+					WRITE_ONCE(pdx->video[ch].last_buf_half_toggle,
+						   toggle);
+				}
+				dma_rmb();
+				WRITE_ONCE(pdx->video[ch].half_seen, true);
+				dev_dbg(&pdx->pdev->dev,
+					"irq: VDONE ch=%u toggle=%u handling inline (cap=%d)\n",
+					ch,
+					READ_ONCE(pdx->video[ch].last_buf_half_toggle),
+					READ_ONCE(pdx->video[ch].cap_active));
+				hws_video_handle_vdone(&pdx->video[ch]);
+			} else {
+				dev_dbg(&pdx->pdev->dev,
+					"irq: VDONE ch=%u ignored (cap=%d stop=%d)\n",
+					ch,
+					READ_ONCE(pdx->video[ch].cap_active),
+					READ_ONCE(pdx->video[ch].stop_requested));
+			}
+
+			writel(vbit, pdx->bar0_base + HWS_REG_INT_STATUS);
+			(void)readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS);
+		}
+
+		/* Re-read in case new interrupt bits popped while processing */
+		int_state = readl_relaxed(pdx->bar0_base + HWS_REG_INT_STATUS);
+		dev_dbg(&pdx->pdev->dev,
+			"irq: loop cnt=%u new INT_STATUS=0x%08x\n", cnt,
+			int_state);
+		if (cnt + 1 == MAX_INT_LOOPS)
+			dev_warn_ratelimited(&pdx->pdev->dev,
+					     "IRQ storm? status=0x%08x\n",
+					     int_state);
+	}
+
+	return IRQ_HANDLED;
+}
diff --git a/drivers/media/pci/hws/hws_irq.h b/drivers/media/pci/hws/hws_irq.h
new file mode 100644
index 0000000000000..a42867aa0c462
--- /dev/null
+++ b/drivers/media/pci/hws/hws_irq.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef HWS_INTERRUPT_H
+#define HWS_INTERRUPT_H
+
+#include <linux/pci.h>
+#include "hws.h"
+
+irqreturn_t hws_irq_handler(int irq, void *info);
+
+#endif /* HWS_INTERRUPT_H */
diff --git a/drivers/media/pci/hws/hws_pci.c b/drivers/media/pci/hws/hws_pci.c
new file mode 100644
index 0000000000000..30bb7d34465b8
--- /dev/null
+++ b/drivers/media/pci/hws/hws_pci.c
@@ -0,0 +1,865 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/pci.h>
+#include <linux/types.h>
+#include <linux/iopoll.h>
+#include <linux/bitfield.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kthread.h>
+#include <linux/interrupt.h>
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/ktime.h>
+#include <linux/math64.h>
+#include <linux/pm.h>
+#include <linux/freezer.h>
+#include <linux/pci_regs.h>
+
+#include <media/v4l2-ctrls.h>
+
+#include "hws.h"
+#include "hws_reg.h"
+#include "hws_video.h"
+#include "hws_irq.h"
+#include "hws_v4l2_ioctl.h"
+
+#define DRV_NAME "hws"
+#define HWS_BUSY_POLL_DELAY_US 10
+#define HWS_BUSY_POLL_TIMEOUT_US 1000000
+
+static unsigned long long hws_elapsed_us(u64 start_ns)
+{
+	return div_u64(ktime_get_mono_fast_ns() - start_ns, 1000);
+}
+
+/* register layout inside HWS_REG_DEVICE_INFO */
+#define DEVINFO_VER GENMASK(7, 0)
+#define DEVINFO_SUBVER GENMASK(15, 8)
+#define DEVINFO_YV12 GENMASK(31, 28)
+#define DEVINFO_HWKEY GENMASK(27, 24)
+#define DEVINFO_PORTID GENMASK(25, 24) /* low 2 bits of HW-key */
+
+#define MAKE_ENTRY(__vend, __chip, __subven, __subdev, __configptr) \
+	{ .vendor = (__vend),                                       \
+	  .device = (__chip),                                       \
+	  .subvendor = (__subven),                                  \
+	  .subdevice = (__subdev),                                  \
+	  .driver_data = (unsigned long)(__configptr) }
+
+/*
+ * PCI IDs for HWS family cards.
+ *
+ * The subsystem IDs are fixed at 0x8888:0x0007 for this family. Some boards
+ * enumerate with vendor ID 0x8888 or 0x1f33. Exact SKU names are not fully
+ * pinned down yet; update these comments when vendor documentation or INF
+ * strings are available.
+ */
+static const struct pci_device_id hws_pci_table[] = {
+	/* HWS family, SKU unknown. */
+	MAKE_ENTRY(0x8888, 0x9534, 0x8888, 0x0007, NULL),
+	MAKE_ENTRY(0x1F33, 0x8534, 0x8888, 0x0007, NULL),
+	MAKE_ENTRY(0x1F33, 0x8554, 0x8888, 0x0007, NULL),
+
+	/* HWS 2x2 HDMI family. */
+	MAKE_ENTRY(0x8888, 0x8524, 0x8888, 0x0007, NULL),
+	/* HWS 2x2 SDI family. */
+	MAKE_ENTRY(0x1F33, 0x6524, 0x8888, 0x0007, NULL),
+
+	/* HWS X4 HDMI family. */
+	MAKE_ENTRY(0x8888, 0x8504, 0x8888, 0x0007, NULL),
+	/* HWS X4 SDI family. */
+	MAKE_ENTRY(0x8888, 0x6504, 0x8888, 0x0007, NULL),
+
+	/* HWS family, SKU unknown. */
+	MAKE_ENTRY(0x8888, 0x8532, 0x8888, 0x0007, NULL),
+	MAKE_ENTRY(0x8888, 0x8512, 0x8888, 0x0007, NULL),
+	MAKE_ENTRY(0x8888, 0x8501, 0x8888, 0x0007, NULL),
+	MAKE_ENTRY(0x1F33, 0x6502, 0x8888, 0x0007, NULL),
+
+	/* HWS X4 HDMI family (alternate vendor ID). */
+	MAKE_ENTRY(0x1F33, 0x8504, 0x8888, 0x0007, NULL),
+	/* HWS 2x2 HDMI family (alternate vendor ID). */
+	MAKE_ENTRY(0x1F33, 0x8524, 0x8888, 0x0007, NULL),
+
+	{}
+};
+
+static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
+{
+	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
+}
+
+static void hws_configure_hardware_capabilities(struct hws_pcie_dev *hdev)
+{
+	u16 id = hdev->device_id;
+
+	/* select per-chip channel counts */
+	switch (id) {
+	case 0x9534:
+	case 0x6524:
+	case 0x8524:
+	case 0x8504:
+	case 0x6504:
+		hdev->cur_max_video_ch = 4;
+		break;
+	case 0x8532:
+		hdev->cur_max_video_ch = 2;
+		break;
+	case 0x8512:
+	case 0x6502:
+		hdev->cur_max_video_ch = 2;
+		break;
+	case 0x8501:
+		hdev->cur_max_video_ch = 1;
+		break;
+	default:
+		hdev->cur_max_video_ch = 4;
+		break;
+	}
+
+	/* universal buffer capacity */
+	hdev->max_hw_video_buf_sz = MAX_MM_VIDEO_SIZE;
+
+	/* decide hardware-version and program DMA max size if needed */
+	if (hdev->device_ver > 121) {
+		if (id == 0x8501 && hdev->device_ver == 122) {
+			hdev->hw_ver = 0;
+		} else {
+			hdev->hw_ver = 1;
+			u32 dma_max = (u32)(MAX_VIDEO_SCALER_SIZE / 16);
+
+			writel(dma_max, hdev->bar0_base + HWS_REG_DMA_MAX_SIZE);
+			/* readback to flush posted MMIO write */
+			(void)readl(hdev->bar0_base + HWS_REG_DMA_MAX_SIZE);
+		}
+	} else {
+		hdev->hw_ver = 0;
+	}
+}
+
+static void hws_stop_device(struct hws_pcie_dev *hws);
+
+static void hws_log_lifecycle_snapshot(struct hws_pcie_dev *hws,
+				       const char *action,
+				       const char *phase)
+{
+	struct device *dev;
+	u32 int_en, int_status, vcap, sys_status, dec_mode;
+
+	if (!hws || !hws->pdev)
+		return;
+
+	dev = &hws->pdev->dev;
+	if (!hws->bar0_base) {
+		dev_dbg(dev,
+			"lifecycle:%s:%s bar0-unmapped suspended=%d start_run=%d pci_lost=%d irq=%d\n",
+			action, phase, READ_ONCE(hws->suspended), hws->start_run,
+			hws->pci_lost, hws->irq);
+		return;
+	}
+
+	int_en = readl(hws->bar0_base + INT_EN_REG_BASE);
+	int_status = readl(hws->bar0_base + HWS_REG_INT_STATUS);
+	vcap = readl(hws->bar0_base + HWS_REG_VCAP_ENABLE);
+	sys_status = readl(hws->bar0_base + HWS_REG_SYS_STATUS);
+	dec_mode = readl(hws->bar0_base + HWS_REG_DEC_MODE);
+
+	dev_dbg(dev,
+		"lifecycle:%s:%s suspended=%d start_run=%d pci_lost=%d irq=%d INT_EN=0x%08x INT_STATUS=0x%08x VCAP=0x%08x SYS=0x%08x DEC=0x%08x\n",
+		action, phase, READ_ONCE(hws->suspended), hws->start_run,
+		hws->pci_lost, hws->irq, int_en, int_status, vcap,
+		sys_status, dec_mode);
+}
+
+static int read_chip_id(struct hws_pcie_dev *hdev)
+{
+	u32 reg;
+	/* mirror PCI IDs for later switches */
+	hdev->device_id = hdev->pdev->device;
+	hdev->vendor_id = hdev->pdev->vendor;
+
+	reg = readl(hdev->bar0_base + HWS_REG_DEVICE_INFO);
+
+	hdev->device_ver = FIELD_GET(DEVINFO_VER, reg);
+	hdev->sub_ver = FIELD_GET(DEVINFO_SUBVER, reg);
+	hdev->support_yv12 = FIELD_GET(DEVINFO_YV12, reg);
+	hdev->port_id = FIELD_GET(DEVINFO_PORTID, reg);
+
+	hdev->max_hw_video_buf_sz = MAX_MM_VIDEO_SIZE;
+	hdev->max_channels = 4;
+	hdev->buf_allocated = false;
+	hdev->main_task = NULL;
+	hdev->start_run = false;
+	hdev->pci_lost = 0;
+
+	writel(0x00, hdev->bar0_base + HWS_REG_DEC_MODE);
+	writel(0x10, hdev->bar0_base + HWS_REG_DEC_MODE);
+
+	hws_configure_hardware_capabilities(hdev);
+
+	dev_info(&hdev->pdev->dev,
+		 "chip detected: ver=%u subver=%u port=%u yv12=%u\n",
+		 hdev->device_ver, hdev->sub_ver, hdev->port_id,
+		 hdev->support_yv12);
+
+	return 0;
+}
+
+static int main_ks_thread_handle(void *data)
+{
+	struct hws_pcie_dev *pdx = data;
+
+	set_freezable();
+
+	while (!kthread_should_stop()) {
+		/* If we're suspending, don't touch hardware; just sleep/freeze. */
+		if (READ_ONCE(pdx->suspended)) {
+			try_to_freeze();
+			schedule_timeout_interruptible(msecs_to_jiffies(1000));
+			continue;
+		}
+
+		/* avoid MMIO when suspended (guarded above) */
+		check_video_format(pdx);
+
+		try_to_freeze(); /* cooperate with freezer each loop */
+
+		/* Sleep 1s or until signaled to wake/stop */
+		schedule_timeout_interruptible(msecs_to_jiffies(1000));
+	}
+
+	dev_dbg(&pdx->pdev->dev, "%s: exiting\n", __func__);
+	return 0;
+}
+
+static void hws_stop_kthread_action(void *data)
+{
+	struct hws_pcie_dev *hws = data;
+	struct task_struct *t;
+	u64 start_ns;
+
+	if (!hws)
+		return;
+
+	t = READ_ONCE(hws->main_task);
+	if (!IS_ERR_OR_NULL(t)) {
+		start_ns = ktime_get_mono_fast_ns();
+		dev_dbg(&hws->pdev->dev,
+			"lifecycle:kthread-stop:begin task=%s[%d]\n",
+			t->comm, t->pid);
+		WRITE_ONCE(hws->main_task, NULL);
+		kthread_stop(t);
+		dev_dbg(&hws->pdev->dev,
+			"lifecycle:kthread-stop:done (%lluus)\n",
+			hws_elapsed_us(start_ns));
+	}
+}
+
+static int hws_alloc_seed_buffers(struct hws_pcie_dev *hws)
+{
+	int ch;
+	/* 64 KiB is plenty for a safe dummy; hardware needs 64-byte alignment. */
+	const size_t need = ALIGN(64 * 1024, 64);
+
+	for (ch = 0; ch < hws->cur_max_video_ch; ch++) {
+#if defined(CONFIG_HAS_DMA) /* normal on PCIe platforms */
+		void *cpu = dma_alloc_coherent(&hws->pdev->dev, need,
+					       &hws->scratch_vid[ch].dma,
+					       GFP_KERNEL);
+#else
+		void *cpu = NULL;
+#endif
+		if (!cpu) {
+			dev_warn(&hws->pdev->dev,
+				 "scratch: dma_alloc_coherent failed ch=%d\n", ch);
+			/* not fatal: free earlier ones and continue without seeding */
+			while (--ch >= 0) {
+				if (hws->scratch_vid[ch].cpu)
+					dma_free_coherent(&hws->pdev->dev,
+							  hws->scratch_vid[ch].size,
+							  hws->scratch_vid[ch].cpu,
+							  hws->scratch_vid[ch].dma);
+				hws->scratch_vid[ch].cpu = NULL;
+				hws->scratch_vid[ch].size = 0;
+			}
+			return -ENOMEM;
+		}
+		hws->scratch_vid[ch].cpu  = cpu;
+		hws->scratch_vid[ch].size = need;
+	}
+	return 0;
+}
+
+static void hws_free_seed_buffers(struct hws_pcie_dev *hws)
+{
+	int ch;
+
+	for (ch = 0; ch < hws->cur_max_video_ch; ch++) {
+		if (hws->scratch_vid[ch].cpu) {
+			dma_free_coherent(&hws->pdev->dev,
+					  hws->scratch_vid[ch].size,
+					  hws->scratch_vid[ch].cpu,
+					  hws->scratch_vid[ch].dma);
+			hws->scratch_vid[ch].cpu = NULL;
+			hws->scratch_vid[ch].size = 0;
+		}
+	}
+}
+
+static void hws_seed_channel(struct hws_pcie_dev *hws, int ch)
+{
+	dma_addr_t paddr = hws->scratch_vid[ch].dma;
+	u32 lo = lower_32_bits(paddr);
+	u32 hi = upper_32_bits(paddr);
+	u32 pci_addr = lo & PCI_E_BAR_ADD_LOWMASK;
+
+	lo &= PCI_E_BAR_ADD_MASK;
+
+	/* Program 64-bit BAR remap entry for this channel (table @ 0x208 + ch * 8) */
+	writel_relaxed(hi, hws->bar0_base +
+			    PCI_ADDR_TABLE_BASE + 0x208 + ch * 8);
+	writel_relaxed(lo, hws->bar0_base +
+			    PCI_ADDR_TABLE_BASE + 0x208 + ch * 8 +
+			    PCIE_BARADDROFSIZE);
+
+	/* Program capture engine per-channel base/half */
+	writel_relaxed((ch + 1) * PCIEBAR_AXI_BASE + pci_addr,
+		       hws->bar0_base + CVBS_IN_BUF_BASE +
+		       ch * PCIE_BARADDROFSIZE);
+
+	/* Half size: use either the current format's half or half of scratch. */
+	{
+		u32 half = hws->video[ch].pix.half_size ?
+			hws->video[ch].pix.half_size :
+			(u32)(hws->scratch_vid[ch].size / 2);
+
+		writel_relaxed(half / 16,
+			       hws->bar0_base + CVBS_IN_BUF_BASE2 +
+			       ch * PCIE_BARADDROFSIZE);
+	}
+
+	(void)readl(hws->bar0_base + HWS_REG_INT_STATUS); /* flush posted writes */
+}
+
+static void hws_seed_all_channels(struct hws_pcie_dev *hws)
+{
+	int ch;
+
+	for (ch = 0; ch < hws->cur_max_video_ch; ch++) {
+		if (hws->scratch_vid[ch].cpu)
+			hws_seed_channel(hws, ch);
+	}
+}
+
+static void hws_irq_mask_gate(struct hws_pcie_dev *hws)
+{
+	writel(0x00000000, hws->bar0_base + INT_EN_REG_BASE);
+	(void)readl(hws->bar0_base + INT_EN_REG_BASE);
+}
+
+static void hws_irq_unmask_gate(struct hws_pcie_dev *hws)
+{
+	writel(HWS_INT_EN_MASK, hws->bar0_base + INT_EN_REG_BASE);
+	(void)readl(hws->bar0_base + INT_EN_REG_BASE);
+}
+
+static void hws_irq_clear_pending(struct hws_pcie_dev *hws)
+{
+	u32 st = readl(hws->bar0_base + HWS_REG_INT_STATUS);
+
+	if (st) {
+		writel(st, hws->bar0_base + HWS_REG_INT_STATUS); /* W1C */
+		(void)readl(hws->bar0_base + HWS_REG_INT_STATUS);
+	}
+}
+
+static void hws_block_hotpaths(struct hws_pcie_dev *hws)
+{
+	WRITE_ONCE(hws->suspended, true);
+	if (hws->irq >= 0)
+		disable_irq(hws->irq);
+
+	if (!hws->bar0_base)
+		return;
+
+	hws_irq_mask_gate(hws);
+	hws_irq_clear_pending(hws);
+}
+
+static int hws_probe(struct pci_dev *pdev, const struct pci_device_id *pci_id)
+{
+	struct hws_pcie_dev *hws;
+	int i, ret, irq;
+	unsigned long irqf = 0;
+	bool v4l2_registered = false;
+
+	/* devres-backed device object */
+	hws = devm_kzalloc(&pdev->dev, sizeof(*hws), GFP_KERNEL);
+	if (!hws)
+		return -ENOMEM;
+
+	hws->pdev = pdev;
+	hws->irq = -1;
+	hws->suspended = false;
+	pci_set_drvdata(pdev, hws);
+
+	/* 1) Enable device + bus mastering (managed) */
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "pcim_enable_device\n");
+	pci_set_master(pdev);
+
+	/* 2) Map BAR0 (managed) */
+	ret = pcim_iomap_regions(pdev, BIT(0), KBUILD_MODNAME);
+	if (ret)
+		return dev_err_probe(&pdev->dev, ret, "pcim_iomap_regions BAR0\n");
+	hws->bar0_base = pcim_iomap_table(pdev)[0];
+
+	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	if (ret) {
+		dev_warn(&pdev->dev,
+			 "64-bit DMA mask unavailable, falling back to 32-bit (%d)\n",
+			 ret);
+		ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
+		if (ret)
+			return dev_err_probe(&pdev->dev, ret,
+					     "No suitable DMA configuration\n");
+	} else {
+		dev_dbg(&pdev->dev, "Using 64-bit DMA mask\n");
+	}
+
+	/* 3) Apply optional PCIe tuning. */
+	enable_pcie_relaxed_ordering(pdev);
+#ifdef CONFIG_ARCH_TI816X
+	pcie_set_readrq(pdev, 128);
+#endif
+
+	/* 4) Identify chip & capabilities */
+	read_chip_id(hws);
+	dev_info(&pdev->dev, "Device VID=0x%04x DID=0x%04x\n",
+		 pdev->vendor, pdev->device);
+	hws_init_video_sys(hws, false);
+
+	/* 5) Init channels (video state, locks, vb2, ctrls) */
+	for (i = 0; i < hws->max_channels; i++) {
+		ret = hws_video_init_channel(hws, i);
+		if (ret) {
+			dev_err(&pdev->dev, "video channel init failed (ch=%d)\n", i);
+			goto err_unwind_channels;
+		}
+	}
+
+	/* 6) Allocate scratch DMA and seed BAR table + channel base/half (legacy SetDMAAddress) */
+	ret = hws_alloc_seed_buffers(hws);
+	if (!ret)
+		hws_seed_all_channels(hws);
+
+	/* 7) Start-run sequence. */
+	hws_init_video_sys(hws, false);
+
+	/* A) Force legacy INTx; legacy used request_irq(pdev->irq, ..., IRQF_SHARED) */
+	pci_intx(pdev, 1);
+	irqf = IRQF_SHARED;
+	irq = pdev->irq;
+	hws->irq = irq;
+	dev_info(&pdev->dev, "IRQ mode: legacy INTx (shared), irq=%d\n", irq);
+
+	/* B) Mask the device's global/bridge gate (INT_EN_REG_BASE) */
+	hws_irq_mask_gate(hws);
+
+	/* C) Clear any sticky pending interrupt status (W1C) before we arm the line */
+	hws_irq_clear_pending(hws);
+
+	/* D) Request the legacy shared interrupt line (no vectors/MSI/MSI-X) */
+	ret = devm_request_irq(&pdev->dev, irq, hws_irq_handler, irqf,
+			       dev_name(&pdev->dev), hws);
+	if (ret) {
+		dev_err(&pdev->dev, "request_irq(%d) failed: %d\n", irq, ret);
+		goto err_unwind_channels;
+	}
+
+	/* E) Set the global interrupt enable bit in main control register */
+	{
+		u32 ctl_reg = readl(hws->bar0_base + HWS_REG_CTL);
+
+		ctl_reg |= HWS_CTL_IRQ_ENABLE_BIT;
+		writel(ctl_reg, hws->bar0_base + HWS_REG_CTL);
+		(void)readl(hws->bar0_base + HWS_REG_CTL); /* flush write */
+		dev_info(&pdev->dev, "Global IRQ enable bit set in control register\n");
+	}
+
+	/* F) Open the global gate just like legacy did */
+	hws_irq_unmask_gate(hws);
+	dev_info(&pdev->dev, "INT_EN_GATE readback=0x%08x\n",
+		 readl(hws->bar0_base + INT_EN_REG_BASE));
+
+	/* 11) Register V4L2 */
+	ret = hws_video_register(hws);
+	if (ret) {
+		dev_err(&pdev->dev, "video_register: %d\n", ret);
+		goto err_unwind_channels;
+	}
+	v4l2_registered = true;
+
+	/* 12) Background monitor thread (managed) */
+	hws->main_task = kthread_run(main_ks_thread_handle, hws, "hws-mon");
+	if (IS_ERR(hws->main_task)) {
+		ret = PTR_ERR(hws->main_task);
+		hws->main_task = NULL;
+		dev_err(&pdev->dev, "kthread_run: %d\n", ret);
+		goto err_unregister_va;
+	}
+	ret = devm_add_action_or_reset(&pdev->dev, hws_stop_kthread_action, hws);
+	if (ret) {
+		dev_err(&pdev->dev, "devm_add_action kthread_stop: %d\n", ret);
+		goto err_unregister_va; /* reset already stopped the thread */
+	}
+
+	/* 13) Final: show the line is armed */
+	dev_info(&pdev->dev, "irq handler installed on irq=%d\n", irq);
+	return 0;
+
+err_unregister_va:
+	hws_stop_device(hws);
+	hws_video_unregister(hws);
+	hws_free_seed_buffers(hws);
+	return ret;
+err_unwind_channels:
+	hws_free_seed_buffers(hws);
+	if (!v4l2_registered) {
+		while (--i >= 0)
+			hws_video_cleanup_channel(hws, i);
+	}
+	return ret;
+}
+
+static int hws_check_busy(struct hws_pcie_dev *pdx)
+{
+	void __iomem *reg = pdx->bar0_base + HWS_REG_SYS_STATUS;
+	u32 val;
+	int ret;
+
+	/* poll until !(val & BUSY_BIT), sleeping HWS_BUSY_POLL_DELAY_US between reads */
+	ret = readl_poll_timeout(reg, val, !(val & HWS_SYS_DMA_BUSY_BIT),
+				 HWS_BUSY_POLL_DELAY_US,
+				 HWS_BUSY_POLL_TIMEOUT_US);
+	if (ret) {
+		dev_err(&pdx->pdev->dev,
+			"SYS_STATUS busy bit never cleared (0x%08x)\n", val);
+		return -ETIMEDOUT;
+	}
+
+	return 0;
+}
+
+static void hws_stop_dsp(struct hws_pcie_dev *hws)
+{
+	u32 status;
+
+	/* Read the decoder mode/status register */
+	status = readl(hws->bar0_base + HWS_REG_DEC_MODE);
+	dev_dbg(&hws->pdev->dev, "%s: status=0x%08x\n", __func__, status);
+
+	/* If the device looks unplugged/stuck, bail out */
+	if (status == 0xFFFFFFFF)
+		return;
+
+	/* Tell the DSP to stop */
+	writel(0x10, hws->bar0_base + HWS_REG_DEC_MODE);
+
+	if (hws_check_busy(hws))
+		dev_warn(&hws->pdev->dev, "DSP busy timeout on stop\n");
+	/* Disable video capture engine in the DSP */
+	writel(0x0, hws->bar0_base + HWS_REG_VCAP_ENABLE);
+}
+
+/* Publish stop so ISR/BH will not touch video buffers anymore. */
+static void hws_publish_stop_flags(struct hws_pcie_dev *hws)
+{
+	unsigned int i;
+
+	for (i = 0; i < hws->cur_max_video_ch; ++i) {
+		struct hws_video *v = &hws->video[i];
+
+		WRITE_ONCE(v->cap_active,     false);
+		WRITE_ONCE(v->stop_requested, true);
+	}
+
+	smp_wmb(); /* make flags visible before we touch MMIO/queues */
+}
+
+/* Drain engines + ISR/BH after flags are published. */
+static void hws_drain_after_stop(struct hws_pcie_dev *hws)
+{
+	u32 ackmask = 0;
+	unsigned int i;
+	u64 start_ns = ktime_get_mono_fast_ns();
+
+	/* Mask device enables: no new DMA starts. */
+	writel(0x0, hws->bar0_base + HWS_REG_VCAP_ENABLE);
+	(void)readl(hws->bar0_base + HWS_REG_INT_STATUS); /* flush */
+
+	/* Let any in-flight DMAs finish (best-effort). */
+	(void)hws_check_busy(hws);
+
+	/* Ack any latched VDONE. */
+	for (i = 0; i < hws->cur_max_video_ch; ++i)
+		ackmask |= HWS_INT_VDONE_BIT(i);
+	if (ackmask) {
+		writel(ackmask, hws->bar0_base + HWS_REG_INT_STATUS);
+		(void)readl(hws->bar0_base + HWS_REG_INT_STATUS);
+	}
+
+	/* Ensure no hard IRQ is still running. */
+	if (hws->irq >= 0)
+		synchronize_irq(hws->irq);
+
+	dev_dbg(&hws->pdev->dev, "lifecycle:drain-after-stop:done (%lluus)\n",
+		hws_elapsed_us(start_ns));
+}
+
+static void hws_stop_device(struct hws_pcie_dev *hws)
+{
+	u32 status = readl(hws->bar0_base + HWS_REG_SYS_STATUS);
+	u64 start_ns = ktime_get_mono_fast_ns();
+	bool live = status != 0xFFFFFFFF;
+
+	dev_dbg(&hws->pdev->dev, "%s: status=0x%08x\n", __func__, status);
+	if (!live) {
+		hws->pci_lost = true;
+		goto out;
+	}
+	hws_log_lifecycle_snapshot(hws, "stop-device", "begin");
+
+	/* Make ISR/BH a no-op, then drain engines/IRQ. */
+	hws_publish_stop_flags(hws);
+	hws_drain_after_stop(hws);
+
+	/* 1) Stop the on-board DSP */
+	hws_stop_dsp(hws);
+
+out:
+	hws->start_run = false;
+	if (live)
+		hws_log_lifecycle_snapshot(hws, "stop-device", "end");
+	else
+		dev_dbg(&hws->pdev->dev, "lifecycle:stop-device:device-lost\n");
+	dev_dbg(&hws->pdev->dev, "lifecycle:stop-device:done (%lluus)\n",
+		hws_elapsed_us(start_ns));
+	dev_dbg(&hws->pdev->dev, "%s: complete\n", __func__);
+}
+
+static int hws_quiesce_for_transition(struct hws_pcie_dev *hws,
+				      const char *action,
+				      bool stop_thread)
+{
+	struct device *dev = &hws->pdev->dev;
+	u64 start_ns = ktime_get_mono_fast_ns();
+	u64 step_ns;
+	int vret;
+
+	hws_log_lifecycle_snapshot(hws, action, "begin");
+
+	step_ns = ktime_get_mono_fast_ns();
+	hws_block_hotpaths(hws);
+	dev_dbg(dev, "lifecycle:%s:block-hotpaths (%lluus)\n", action,
+		hws_elapsed_us(step_ns));
+	hws_log_lifecycle_snapshot(hws, action, "blocked");
+
+	if (stop_thread) {
+		step_ns = ktime_get_mono_fast_ns();
+		hws_stop_kthread_action(hws);
+		dev_dbg(dev, "lifecycle:%s:stop-kthread (%lluus)\n", action,
+			hws_elapsed_us(step_ns));
+	}
+
+	step_ns = ktime_get_mono_fast_ns();
+	vret = hws_video_quiesce(hws, action);
+	dev_dbg(dev, "lifecycle:%s:video-quiesce ret=%d (%lluus)\n", action,
+		vret, hws_elapsed_us(step_ns));
+	if (vret)
+		dev_warn(dev, "lifecycle:%s video quiesce returned %d\n",
+			 action, vret);
+
+	step_ns = ktime_get_mono_fast_ns();
+	hws_stop_device(hws);
+	dev_dbg(dev, "lifecycle:%s:stop-device (%lluus)\n", action,
+		hws_elapsed_us(step_ns));
+	hws_log_lifecycle_snapshot(hws, action, "end");
+	dev_dbg(dev, "lifecycle:%s:quiesce-done ret=%d (%lluus)\n", action,
+		vret, hws_elapsed_us(start_ns));
+
+	return vret;
+}
+
+static void hws_remove(struct pci_dev *pdev)
+{
+	struct hws_pcie_dev *hws = pci_get_drvdata(pdev);
+	u64 start_ns;
+
+	if (!hws)
+		return;
+
+	start_ns = ktime_get_mono_fast_ns();
+	dev_info(&pdev->dev, "lifecycle:remove begin\n");
+	hws_log_lifecycle_snapshot(hws, "remove", "begin");
+
+	/* Stop the monitor thread before tearing down V4L2/vb2 objects. */
+	hws_block_hotpaths(hws);
+	hws_stop_kthread_action(hws);
+
+	/* Stop hardware and capture cleanly. */
+	hws_stop_device(hws);
+
+	/* Unregister V4L2 resources. */
+	hws_video_unregister(hws);
+
+	/* Release seeded DMA buffers */
+	hws_free_seed_buffers(hws);
+	/* kthread is stopped by the devm action registered in probe. */
+	hws_log_lifecycle_snapshot(hws, "remove", "end");
+	dev_info(&pdev->dev, "lifecycle:remove done (%lluus)\n",
+		 hws_elapsed_us(start_ns));
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int hws_pm_suspend(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct hws_pcie_dev *hws = pci_get_drvdata(pdev);
+	int vret;
+	u64 start_ns = ktime_get_mono_fast_ns();
+	u64 step_ns;
+
+	dev_info(dev, "lifecycle:pm_suspend begin\n");
+	vret = hws_quiesce_for_transition(hws, "pm_suspend", false);
+
+	step_ns = ktime_get_mono_fast_ns();
+	pci_save_state(pdev);
+	pci_clear_master(pdev);
+	pci_disable_device(pdev);
+	pci_set_power_state(pdev, PCI_D3hot);
+	dev_dbg(dev, "lifecycle:pm_suspend:pci-d3hot (%lluus)\n",
+		hws_elapsed_us(step_ns));
+	dev_info(dev, "lifecycle:pm_suspend done ret=%d (%lluus)\n", vret,
+		 hws_elapsed_us(start_ns));
+
+	return 0;
+}
+
+static int hws_pm_resume(struct device *dev)
+{
+	struct pci_dev *pdev = to_pci_dev(dev);
+	struct hws_pcie_dev *hws = pci_get_drvdata(pdev);
+	int ret;
+	u64 start_ns = ktime_get_mono_fast_ns();
+	u64 step_ns;
+
+	dev_info(dev, "lifecycle:pm_resume begin\n");
+
+	/* Back to D0 and re-enable the function */
+	step_ns = ktime_get_mono_fast_ns();
+	pci_set_power_state(pdev, PCI_D0);
+
+	ret = pci_enable_device(pdev);
+	if (ret) {
+		dev_err(dev, "pci_enable_device: %d\n", ret);
+		return ret;
+	}
+	pci_restore_state(pdev);
+	pci_set_master(pdev);
+	dev_dbg(dev, "lifecycle:pm_resume:pci-enable (%lluus)\n",
+		hws_elapsed_us(step_ns));
+
+	/* Reapply any PCIe tuning lost across D3 */
+	enable_pcie_relaxed_ordering(pdev);
+
+	/* Reinitialize chip-side capabilities / registers */
+	step_ns = ktime_get_mono_fast_ns();
+	read_chip_id(hws);
+	/* Re-seed BAR remaps/DMA windows and restart the capture core */
+	hws_seed_all_channels(hws);
+	hws_init_video_sys(hws, true);
+	hws_irq_clear_pending(hws);
+	dev_dbg(dev, "lifecycle:pm_resume:chip-reinit (%lluus)\n",
+		hws_elapsed_us(step_ns));
+
+	/* IRQs can be re-enabled now that MMIO is sane */
+	step_ns = ktime_get_mono_fast_ns();
+	if (hws->irq >= 0)
+		enable_irq(hws->irq);
+
+	WRITE_ONCE(hws->suspended, false);
+	dev_dbg(dev, "lifecycle:pm_resume:irq-unsuspend (%lluus)\n",
+		hws_elapsed_us(step_ns));
+
+	/* vb2: nothing mandatory; userspace will STREAMON again when ready */
+	step_ns = ktime_get_mono_fast_ns();
+	hws_video_pm_resume(hws);
+	dev_dbg(dev, "lifecycle:pm_resume:video-resume (%lluus)\n",
+		hws_elapsed_us(step_ns));
+	hws_log_lifecycle_snapshot(hws, "pm_resume", "end");
+	dev_info(dev, "lifecycle:pm_resume done (%lluus)\n",
+		 hws_elapsed_us(start_ns));
+
+	return 0;
+}
+
+static SIMPLE_DEV_PM_OPS(hws_pm_ops, hws_pm_suspend, hws_pm_resume);
+# define HWS_PM_OPS (&hws_pm_ops)
+#else
+# define HWS_PM_OPS NULL
+#endif
+
+static void hws_shutdown(struct pci_dev *pdev)
+{
+	struct hws_pcie_dev *hws = pci_get_drvdata(pdev);
+	int vret = 0;
+	u64 start_ns = ktime_get_mono_fast_ns();
+	u64 step_ns;
+
+	if (!hws)
+		return;
+
+	dev_info(&pdev->dev, "lifecycle:pci_shutdown begin\n");
+	vret = hws_quiesce_for_transition(hws, "pci_shutdown", true);
+
+	step_ns = ktime_get_mono_fast_ns();
+	pci_clear_master(pdev);
+	dev_dbg(&pdev->dev, "lifecycle:pci_shutdown:clear-master (%lluus)\n",
+		hws_elapsed_us(step_ns));
+	dev_info(&pdev->dev, "lifecycle:pci_shutdown done ret=%d (%lluus)\n",
+		 vret, hws_elapsed_us(start_ns));
+}
+
+static struct pci_driver hws_pci_driver = {
+	.name = KBUILD_MODNAME,
+	.id_table = hws_pci_table,
+	.probe = hws_probe,
+	.remove = hws_remove,
+	.shutdown = hws_shutdown,
+	.driver = {
+		.pm = HWS_PM_OPS,
+	},
+};
+
+MODULE_DEVICE_TABLE(pci, hws_pci_table);
+
+static int __init pcie_hws_init(void)
+{
+	return pci_register_driver(&hws_pci_driver);
+}
+
+static void __exit pcie_hws_exit(void)
+{
+	pci_unregister_driver(&hws_pci_driver);
+}
+
+module_init(pcie_hws_init);
+module_exit(pcie_hws_exit);
+
+MODULE_DESCRIPTION(DRV_NAME);
+MODULE_AUTHOR("Ben Hoff <hoff.benjamin.k@gmail.com>");
+MODULE_AUTHOR("Sales <sales@avmatrix.com>");
+MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("DMA_BUF");
diff --git a/drivers/media/pci/hws/hws_reg.h b/drivers/media/pci/hws/hws_reg.h
new file mode 100644
index 0000000000000..e4fb4af444340
--- /dev/null
+++ b/drivers/media/pci/hws/hws_reg.h
@@ -0,0 +1,136 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef _HWS_PCIE_REG_H
+#define _HWS_PCIE_REG_H
+
+#include <linux/bits.h>
+#include <linux/sizes.h>
+
+#define XDMA_CHANNEL_NUM_MAX (1)
+#define MAX_NUM_ENGINES (XDMA_CHANNEL_NUM_MAX * 2)
+
+#define  PCIE_BARADDROFSIZE 4u
+
+#define PCI_BUS_ACCESS_BASE       0x00000000U
+#define INT_EN_REG_BASE           (PCI_BUS_ACCESS_BASE + 0x0134U)
+#define PCIEBR_EN_REG_BASE        (PCI_BUS_ACCESS_BASE + 0x0148U)
+#define PCIE_INT_DEC_REG_BASE     (PCI_BUS_ACCESS_BASE + 0x0138U)
+
+#define HWS_INT_EN_MASK           0x0003FFFFU
+
+#define PCIEBAR_AXI_BASE 0x20000000U
+
+#define CTL_REG_ACC_BASE 0x0
+#define PCI_ADDR_TABLE_BASE CTL_REG_ACC_BASE
+
+#define CVBS_IN_BASE              0x00004000U
+#define CVBS_IN_BUF_BASE          (CVBS_IN_BASE + (16U * PCIE_BARADDROFSIZE))
+#define CVBS_IN_BUF_BASE2         (CVBS_IN_BASE + (50U * PCIE_BARADDROFSIZE))
+
+/* 2 Mib */
+#define MAX_L_VIDEO_SIZE            0x200000U
+
+#define PCI_E_BAR_PAGE_SIZE 0x20000000
+#define PCI_E_BAR_ADD_MASK 0xE0000000
+#define PCI_E_BAR_ADD_LOWMASK 0x1FFFFFFF
+
+#define MAX_VID_CHANNELS            4
+
+#define MAX_MM_VIDEO_SIZE            SZ_4M
+
+#define MAX_VIDEO_HW_W 1920
+#define MAX_VIDEO_HW_H 1080
+#define MAX_VIDEO_SCALER_SIZE     (1920U * 1080U * 2U)
+
+#define MIN_VAMP_BRIGHTNESS_UNITS   0
+#define MAX_VAMP_BRIGHTNESS_UNITS   0xff
+
+#define MIN_VAMP_CONTRAST_UNITS     0
+#define MAX_VAMP_CONTRAST_UNITS     0xff
+
+#define MIN_VAMP_SATURATION_UNITS   0
+#define MAX_VAMP_SATURATION_UNITS   0xff
+
+#define MIN_VAMP_HUE_UNITS          0
+#define MAX_VAMP_HUE_UNITS          0xff
+
+#define HWS_BRIGHTNESS_DEFAULT       0x80
+#define HWS_CONTRAST_DEFAULT         0x80
+#define HWS_SATURATION_DEFAULT       0x80
+#define HWS_HUE_DEFAULT              0x00
+
+/* Core/global status. */
+#define HWS_REG_SYS_STATUS            (CVBS_IN_BASE +  0 * PCIE_BARADDROFSIZE)
+/* bit3: DMA busy, bit2: int, ... */
+
+#define HWS_SYS_DMA_BUSY_BIT          BIT(3) /* 0x08 = DMA busy flag */
+
+#define HWS_REG_DEC_MODE       (CVBS_IN_BASE +  0 * PCIE_BARADDROFSIZE)
+/* Main control register */
+#define HWS_REG_CTL            (CVBS_IN_BASE +  4 * PCIE_BARADDROFSIZE)
+#define HWS_CTL_IRQ_ENABLE_BIT BIT(0)   /* Global interrupt enable bit */
+/*  Write 0x00 to fully reset decoder,
+ *  set bit 31=1 to "start run",
+ *  low byte=0x13 selects YUYV/BT.709/etc,
+ *  in ReadChipId() we also write 0x00 and 0x10 here for chip-ID sequencing.
+ */
+
+/* Per-channel done flags. */
+#define HWS_REG_INT_STATUS            (CVBS_IN_BASE +  1 * PCIE_BARADDROFSIZE)
+#define HWS_SYS_BUSY_BIT          BIT(2)      /* matches old 0x04 test   */
+
+/* Capture enable switches. */
+/* bit0-3: CH0-CH3 video enable */
+#define HWS_REG_VCAP_ENABLE           (CVBS_IN_BASE +  2 * PCIE_BARADDROFSIZE)
+/* bits0-3: signal present, bits8-11: interlace */
+#define HWS_REG_ACTIVE_STATUS          (CVBS_IN_BASE +  5  * PCIE_BARADDROFSIZE)
+/* bits0-3: HDCP detected */
+#define HWS_REG_HDCP_STATUS            (CVBS_IN_BASE +  8  * PCIE_BARADDROFSIZE)
+#define HWS_REG_DMA_MAX_SIZE   (CVBS_IN_BASE +  9 * PCIE_BARADDROFSIZE)
+
+/* Buffer addresses (written once during init/reset). */
+/* Base of host-visible buffer. */
+#define HWS_REG_VBUF1_ADDR            (CVBS_IN_BASE + 25 * PCIE_BARADDROFSIZE)
+/* Per-channel DMA address. */
+#define HWS_REG_DMA_ADDR(ch)          (CVBS_IN_BASE + (26 + (ch)) * PCIE_BARADDROFSIZE)
+
+/* Per-channel live buffer toggles (read-only). */
+#define HWS_REG_VBUF_TOGGLE(ch)       (CVBS_IN_BASE + (32 + (ch)) * PCIE_BARADDROFSIZE)
+/*
+ * Returns 0 or 1 = which half of the video ring the DMA engine is
+ * currently filling for channel *ch* (0-3).
+ */
+
+/* Per-interrupt bits (video 0-3). */
+#define HWS_INT_VDONE_BIT(ch)     BIT(ch)         /* 0x01,0x02,0x04,0x08  */
+
+#define HWS_REG_INT_ACK           (CVBS_IN_BASE + 0x4000 + 1 * PCIE_BARADDROFSIZE)
+
+/* 16-bit W | 16-bit H. */
+#define HWS_REG_IN_RES(ch)             (CVBS_IN_BASE + (90  + (ch) * 2) * PCIE_BARADDROFSIZE)
+/* B|C|H|S packed bytes. */
+#define HWS_REG_BCHS(ch)               (CVBS_IN_BASE + (91  + (ch) * 2) * PCIE_BARADDROFSIZE)
+
+/* Input fps. */
+#define HWS_REG_FRAME_RATE(ch)         (CVBS_IN_BASE + (110 + (ch))    * PCIE_BARADDROFSIZE)
+/* Programmed out W|H. */
+#define HWS_REG_OUT_RES(ch)            (CVBS_IN_BASE + (120 + (ch))    * PCIE_BARADDROFSIZE)
+/* Programmed out fps. */
+#define HWS_REG_OUT_FRAME_RATE(ch)     (CVBS_IN_BASE + (130 + (ch))    * PCIE_BARADDROFSIZE)
+
+/* Device version/port ID/subversion register. */
+#define HWS_REG_DEVICE_INFO   (CVBS_IN_BASE +  88 * PCIE_BARADDROFSIZE)
+/*
+ * Reading this 32-bit word returns:
+ *   bits 7:0   = "device version"
+ *   bits 15:8  = "device sub-version"
+ *   bits 23:24 = "HW key / port ID" etc.
+ *   bits 31:28 = "support YV12" flags
+ */
+
+/* Convenience aliases for individual channels. */
+#define HWS_REG_VBUF_TOGGLE_CH0       HWS_REG_VBUF_TOGGLE(0)
+#define HWS_REG_VBUF_TOGGLE_CH1       HWS_REG_VBUF_TOGGLE(1)
+#define HWS_REG_VBUF_TOGGLE_CH2       HWS_REG_VBUF_TOGGLE(2)
+#define HWS_REG_VBUF_TOGGLE_CH3       HWS_REG_VBUF_TOGGLE(3)
+
+#endif /* _HWS_PCIE_REG_H */
diff --git a/drivers/media/pci/hws/hws_v4l2_ioctl.c b/drivers/media/pci/hws/hws_v4l2_ioctl.c
new file mode 100644
index 0000000000000..0303e311ee4a8
--- /dev/null
+++ b/drivers/media/pci/hws/hws_v4l2_ioctl.c
@@ -0,0 +1,919 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/kernel.h>
+#include <linux/string.h>
+#include <linux/pci.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/math64.h>
+
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-dv-timings.h>
+#include <media/videobuf2-core.h>
+#include <media/videobuf2-v4l2.h>
+
+#include "hws.h"
+#include "hws_reg.h"
+#include "hws_video.h"
+#include "hws_v4l2_ioctl.h"
+
+struct hws_dv_mode {
+	struct v4l2_dv_timings timings;
+	u32 refresh_hz;
+};
+
+static const struct hws_dv_mode *
+hws_find_dv_by_wh(u32 w, u32 h, bool interlaced);
+static const struct hws_dv_mode *
+hws_find_dv_by_wh_fps(u32 w, u32 h, bool interlaced, u32 fps);
+static u32 hws_get_live_fps(struct hws_video *vid);
+static u32 hws_input_status(struct hws_video *vid);
+static int hws_fill_dv_timings(u32 w, u32 h, bool interlace, u32 fps,
+			       struct v4l2_dv_timings *timings);
+
+static const struct hws_dv_mode hws_dv_modes[] = {
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1920,
+				.height = 1080,
+				.hfrontporch = 88,
+				.hsync = 44,
+				.hbackporch = 148,
+				.vfrontporch = 4,
+				.vsync = 5,
+				.vbackporch = 36,
+				.pixelclock = 148500000,
+				.polarities = V4L2_DV_VSYNC_POS_POL |
+					      V4L2_DV_HSYNC_POS_POL,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1920,
+				.height = 1080,
+				.hfrontporch = 88,
+				.hsync = 44,
+				.hbackporch = 148,
+				.vfrontporch = 4,
+				.vsync = 5,
+				.vbackporch = 36,
+				.pixelclock = 74250000,
+				.polarities = V4L2_DV_VSYNC_POS_POL |
+					      V4L2_DV_HSYNC_POS_POL,
+				.interlaced = 0,
+			},
+		},
+		30,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1280,
+				.height = 720,
+				.hfrontporch = 110,
+				.hsync = 40,
+				.hbackporch = 220,
+				.vfrontporch = 5,
+				.vsync = 5,
+				.vbackporch = 20,
+				.pixelclock = 74250000,
+				.polarities = V4L2_DV_VSYNC_POS_POL |
+					      V4L2_DV_HSYNC_POS_POL,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 720,
+				.height = 480,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 720,
+				.height = 576,
+				.interlaced = 0,
+			},
+		},
+		50,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 800,
+				.height = 600,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 640,
+				.height = 480,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1024,
+				.height = 768,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1280,
+				.height = 768,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1280,
+				.height = 800,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1280,
+				.height = 1024,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1360,
+				.height = 768,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1440,
+				.height = 900,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1680,
+				.height = 1050,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+	/* Portrait */
+	{
+		{
+			.type = V4L2_DV_BT_656_1120,
+			.bt = {
+				.width = 1080,
+				.height = 1920,
+				.interlaced = 0,
+			},
+		},
+		60,
+	},
+};
+
+static const size_t hws_dv_modes_cnt = ARRAY_SIZE(hws_dv_modes);
+
+/* YUYV: 16 bpp; align to 64 as you did elsewhere */
+static inline u32 hws_calc_bpl_yuyv(u32 w)     { return ALIGN(w * 2, 64); }
+static inline u32 hws_calc_size_yuyv(u32 w, u32 h) { return hws_calc_bpl_yuyv(w) * h; }
+static inline u32 hws_calc_half_size(u32 sizeimage)
+{
+	return sizeimage / 2;
+}
+
+static inline void hws_hw_write_bchs(struct hws_pcie_dev *hws, unsigned int ch,
+				     u8 br, u8 co, u8 hu, u8 sa)
+{
+	u32 packed = (sa << 24) | (hu << 16) | (co << 8) | br;
+
+	if (!hws || !hws->bar0_base || ch >= hws->max_channels)
+		return;
+	writel_relaxed(packed, hws->bar0_base + HWS_REG_BCHS(ch));
+	(void)readl(hws->bar0_base + HWS_REG_BCHS(ch)); /* post write */
+}
+
+/* Helper: find a supported DV mode by W/H + interlace flag */
+static const struct hws_dv_mode *
+hws_match_supported_dv(const struct v4l2_dv_timings *req)
+{
+	const struct v4l2_bt_timings *bt;
+	u32 fps;
+
+	if (!req || req->type != V4L2_DV_BT_656_1120)
+		return NULL;
+
+	bt = &req->bt;
+	fps = 0;
+	if (bt->pixelclock) {
+		u32 total_w = bt->width + bt->hfrontporch + bt->hsync +
+			      bt->hbackporch;
+		u32 total_h = bt->height + bt->vfrontporch + bt->vsync +
+			      bt->vbackporch;
+
+		if (total_w && total_h)
+			fps = DIV_ROUND_CLOSEST_ULL((u64)bt->pixelclock,
+						    (u64)total_w * total_h);
+	}
+	if (fps) {
+		const struct hws_dv_mode *exact =
+			hws_find_dv_by_wh_fps(bt->width, bt->height,
+					      !!bt->interlaced, fps);
+		if (exact)
+			return exact;
+	}
+	return hws_find_dv_by_wh(bt->width, bt->height, !!bt->interlaced);
+}
+
+/* Helper: find a supported DV mode by W/H + interlace flag */
+static const struct hws_dv_mode *
+hws_find_dv_by_wh(u32 w, u32 h, bool interlaced)
+{
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(hws_dv_modes); i++) {
+		const struct hws_dv_mode *t = &hws_dv_modes[i];
+		const struct v4l2_bt_timings *bt = &t->timings.bt;
+
+		if (t->timings.type != V4L2_DV_BT_656_1120)
+			continue;
+
+		if (bt->width == w && bt->height == h &&
+		    !!bt->interlaced == interlaced)
+			return t;
+	}
+	return NULL;
+}
+
+static const struct hws_dv_mode *
+hws_find_dv_by_wh_fps(u32 w, u32 h, bool interlaced, u32 fps)
+{
+	size_t i;
+
+	for (i = 0; i < ARRAY_SIZE(hws_dv_modes); i++) {
+		const struct hws_dv_mode *t = &hws_dv_modes[i];
+		const struct v4l2_bt_timings *bt = &t->timings.bt;
+
+		if (t->timings.type != V4L2_DV_BT_656_1120)
+			continue;
+
+		if (bt->width == w && bt->height == h &&
+		    !!bt->interlaced == interlaced &&
+		    t->refresh_hz == fps)
+			return t;
+	}
+	return NULL;
+}
+
+static bool hws_get_live_dv_geometry(struct hws_video *vid,
+				     u32 *w, u32 *h, bool *interlaced)
+{
+	struct hws_pcie_dev *pdx;
+	u32 reg;
+
+	if (!vid)
+		return false;
+
+	pdx = vid->parent;
+	if (!pdx || !pdx->bar0_base)
+		return false;
+
+	reg = readl(pdx->bar0_base + HWS_REG_IN_RES(vid->channel_index));
+	if (!reg || reg == 0xFFFFFFFF)
+		return false;
+
+	if (w)
+		*w = reg & 0xFFFF;
+	if (h)
+		*h = (reg >> 16) & 0xFFFF;
+	if (interlaced) {
+		reg = readl(pdx->bar0_base + HWS_REG_ACTIVE_STATUS);
+		*interlaced = !!(reg & BIT(8 + vid->channel_index));
+	}
+	return true;
+}
+
+static u32 hws_get_live_fps(struct hws_video *vid)
+{
+	struct hws_pcie_dev *pdx;
+	u32 fps;
+
+	if (!vid)
+		return 0;
+
+	pdx = vid->parent;
+	if (!pdx || !pdx->bar0_base)
+		return 0;
+
+	fps = readl(pdx->bar0_base + HWS_REG_FRAME_RATE(vid->channel_index));
+	if (!fps || fps == 0xFFFFFFFF || fps > 240)
+		return 0;
+
+	return fps;
+}
+
+static u32 hws_pick_fps_from_mode(u32 w, u32 h, bool interlaced)
+{
+	const struct hws_dv_mode *m = hws_find_dv_by_wh(w, h, interlaced);
+
+	if (m && m->refresh_hz)
+		return m->refresh_hz;
+	/* Fallback to a sane default */
+	return 60;
+}
+
+static int hws_fill_dv_timings(u32 w, u32 h, bool interlace, u32 fps,
+			       struct v4l2_dv_timings *timings)
+{
+	const struct hws_dv_mode *m;
+
+	m = fps ? hws_find_dv_by_wh_fps(w, h, interlace, fps) : NULL;
+	if (!m)
+		m = hws_find_dv_by_wh(w, h, interlace);
+	if (!m)
+		return -ENOLINK;
+
+	*timings = m->timings;
+	return 0;
+}
+
+static u32 hws_input_status(struct hws_video *vid)
+{
+	struct hws_pcie_dev *pdx;
+	u32 reg;
+
+	if (!vid)
+		return V4L2_IN_ST_NO_SIGNAL;
+
+	pdx = vid->parent;
+	if (!pdx || !pdx->bar0_base)
+		return V4L2_IN_ST_NO_SIGNAL;
+
+	reg = readl(pdx->bar0_base + HWS_REG_ACTIVE_STATUS);
+	if (reg == 0xffffffff)
+		return V4L2_IN_ST_NO_SIGNAL;
+
+	return (reg & BIT(vid->channel_index)) ? 0 : V4L2_IN_ST_NO_SIGNAL;
+}
+
+/* Query the *current detected* DV timings on the input.
+ * If you have a real hardware detector, call it here; otherwise we
+ * derive from the cached pix state and map to the closest supported DV mode.
+ */
+int hws_vidioc_query_dv_timings(struct file *file, void *fh,
+				struct v4l2_dv_timings *timings)
+{
+	struct hws_video *vid = video_drvdata(file);
+	u32 w, h;
+	u32 fps;
+	bool interlace;
+
+	if (!timings)
+		return -EINVAL;
+
+	w = vid->pix.width;
+	h = vid->pix.height;
+	interlace = vid->pix.interlaced;
+	hws_get_live_dv_geometry(vid, &w, &h, &interlace);
+	fps = hws_get_live_fps(vid);
+	if (!fps)
+		fps = vid->current_fps ? vid->current_fps :
+		      hws_pick_fps_from_mode(w, h, interlace);
+
+	return hws_fill_dv_timings(w, h, interlace, fps, timings);
+}
+
+/* Enumerate the Nth supported DV timings from our static table. */
+int hws_vidioc_enum_dv_timings(struct file *file, void *fh,
+			       struct v4l2_enum_dv_timings *edv)
+{
+	struct hws_video *vid = video_drvdata(file);
+	const struct hws_dv_mode *m;
+	u32 w, h;
+	u32 fps;
+	bool interlace;
+
+	if (!edv)
+		return -EINVAL;
+
+	if (edv->pad)
+		return -EINVAL;
+
+	w = 0;
+	h = 0;
+	interlace = false;
+	if (hws_get_live_dv_geometry(vid, &w, &h, &interlace)) {
+		fps = hws_get_live_fps(vid);
+		if (!fps)
+			fps = vid->current_fps ? vid->current_fps :
+			      hws_pick_fps_from_mode(w, h, interlace);
+		m = fps ? hws_find_dv_by_wh_fps(w, h, interlace, fps) : NULL;
+		if (!m)
+			m = hws_find_dv_by_wh(w, h, interlace);
+		if (m) {
+			if (edv->index)
+				return -EINVAL;
+			edv->timings = m->timings;
+			return 0;
+		}
+	}
+
+	if (edv->index >= hws_dv_modes_cnt)
+		return -EINVAL;
+
+	edv->timings = hws_dv_modes[edv->index].timings;
+	return 0;
+}
+
+/* Get the *currently configured* DV timings. */
+int hws_vidioc_g_dv_timings(struct file *file, void *fh,
+			    struct v4l2_dv_timings *timings)
+{
+	struct hws_video *vid = video_drvdata(file);
+	u32 w, h;
+	u32 fps;
+	bool interlace;
+
+	if (!timings)
+		return -EINVAL;
+
+	w = vid->pix.width;
+	h = vid->pix.height;
+	interlace = vid->pix.interlaced;
+	if (hws_get_live_dv_geometry(vid, &w, &h, &interlace)) {
+		fps = hws_get_live_fps(vid);
+		if (!fps)
+			fps = vid->current_fps ? vid->current_fps :
+			      hws_pick_fps_from_mode(w, h, interlace);
+		return hws_fill_dv_timings(w, h, interlace, fps, timings);
+	}
+
+	*timings = vid->cur_dv_timings;
+	return 0;
+}
+
+static inline void hws_set_colorimetry_state(struct hws_pix_state *p)
+{
+	bool sd = p->height <= 576;
+
+	p->colorspace   = sd ? V4L2_COLORSPACE_SMPTE170M : V4L2_COLORSPACE_REC709;
+	p->ycbcr_enc    = V4L2_YCBCR_ENC_DEFAULT;
+	p->quantization = V4L2_QUANTIZATION_FULL_RANGE;
+	p->xfer_func    = V4L2_XFER_FUNC_DEFAULT;
+}
+
+/* Set DV timings: must match one of our supported modes.
+ * If buffers are queued and this implies a size change, we reject with -EBUSY.
+ * Otherwise we update pix state and (optionally) reprogram the HW.
+ */
+int hws_vidioc_s_dv_timings(struct file *file, void *fh,
+			    struct v4l2_dv_timings *timings)
+{
+	struct hws_video *vid = video_drvdata(file);
+	const struct hws_dv_mode *m;
+	const struct v4l2_bt_timings *bt;
+	u32 new_w, new_h;
+	bool interlaced;
+	int ret = 0;
+	unsigned long was_busy;
+	u32 live_w, live_h;
+	u32 live_fps;
+	bool live_interlaced;
+	bool live_present;
+
+	if (!timings)
+		return -EINVAL;
+
+	m = hws_match_supported_dv(timings);
+	if (!m)
+		return -EINVAL;
+
+	bt = &m->timings.bt;
+	if (bt->interlaced)
+		return -EINVAL; /* only progressive modes are advertised */
+	new_w = bt->width;
+	new_h = bt->height;
+	interlaced = false;
+
+	lockdep_assert_held(&vid->state_lock);
+	live_present = hws_get_live_dv_geometry(vid, &live_w, &live_h,
+						&live_interlaced);
+
+	/* If vb2 has active buffers and size would change, reject. */
+	was_busy = vb2_is_busy(&vid->buffer_queue);
+	if (was_busy &&
+	    (new_w != vid->pix.width || new_h != vid->pix.height ||
+	     interlaced != vid->pix.interlaced)) {
+		ret = -EBUSY;
+		return ret;
+	}
+
+	/* When a live input signal is present, the receiver owns the timing.
+	 * Allow setting the already-active timings so v4l2-compliance can
+	 * round-trip them, but reject attempts to retime the live source.
+	 */
+	if (live_present) {
+		live_fps = hws_get_live_fps(vid);
+		if (!live_fps)
+			live_fps = vid->current_fps ? vid->current_fps :
+					hws_pick_fps_from_mode(live_w, live_h,
+							       live_interlaced);
+		if (live_w == new_w && live_h == new_h &&
+		    live_interlaced == interlaced &&
+		    m->refresh_hz == live_fps)
+			return 0;
+		return -EBUSY;
+	}
+
+	/* Update software pixel state (and recalc sizes) */
+	vid->pix.width      = new_w;
+	vid->pix.height     = new_h;
+	vid->pix.field      = interlaced ? V4L2_FIELD_INTERLACED
+					 : V4L2_FIELD_NONE;
+	vid->pix.interlaced = interlaced;
+	vid->pix.fourcc     = V4L2_PIX_FMT_YUYV;
+
+	hws_set_colorimetry_state(&vid->pix);
+
+	/* Recompute stride, sizeimage, and half_size. */
+	vid->pix.bytesperline = hws_calc_bpl_yuyv(new_w);
+	vid->pix.sizeimage    = hws_calc_size_yuyv(new_w, new_h);
+	vid->pix.half_size    = hws_calc_half_size(vid->pix.sizeimage);
+	vid->cur_dv_timings   = m->timings;
+	vid->current_fps      = m->refresh_hz;
+	return ret;
+}
+
+/* Report DV timings capability: advertise BT.656/1120 with
+ * the min/max WxH derived from our table and basic progressive support.
+ */
+int hws_vidioc_dv_timings_cap(struct file *file, void *fh,
+			      struct v4l2_dv_timings_cap *cap)
+{
+	u32 min_w = ~0U, min_h = ~0U;
+	u32 max_w = 0,       max_h = 0;
+	size_t i, n = 0;
+
+	if (!cap)
+		return -EINVAL;
+
+	memset(cap, 0, sizeof(*cap));
+	cap->type = V4L2_DV_BT_656_1120;
+
+	for (i = 0; i < ARRAY_SIZE(hws_dv_modes); i++) {
+		const struct v4l2_bt_timings *bt = &hws_dv_modes[i].timings.bt;
+
+		if (hws_dv_modes[i].timings.type != V4L2_DV_BT_656_1120)
+			continue;
+		n++;
+
+		if (bt->width  < min_w)
+			min_w = bt->width;
+		if (bt->height < min_h)
+			min_h = bt->height;
+		if (bt->width  > max_w)
+			max_w = bt->width;
+		if (bt->height > max_h)
+			max_h = bt->height;
+	}
+
+	/* If the table was empty, fail gracefully. */
+	if (!n || min_w == U32_MAX)
+		return -ENODATA;
+
+	cap->bt.min_width  = min_w;
+	cap->bt.max_width  = max_w;
+	cap->bt.min_height = min_h;
+	cap->bt.max_height = max_h;
+
+	/* We support both CEA-861- and VESA-style modes in the list. */
+	cap->bt.standards =
+		V4L2_DV_BT_STD_CEA861 | V4L2_DV_BT_STD_DMT | V4L2_DV_BT_STD_CVT;
+
+	/* Only progressive modes are advertised. */
+	cap->bt.capabilities = V4L2_DV_BT_CAP_PROGRESSIVE;
+
+	/* Leave pixelclock/porch limits unconstrained (0) for now. */
+	return 0;
+}
+
+static int hws_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct hws_video *vid =
+		container_of(ctrl->handler, struct hws_video, control_handler);
+	struct hws_pcie_dev *pdx = vid->parent;
+	bool program = false;
+
+	switch (ctrl->id) {
+	case V4L2_CID_BRIGHTNESS:
+		vid->current_brightness = ctrl->val;
+		program = true;
+		break;
+	case V4L2_CID_CONTRAST:
+		vid->current_contrast = ctrl->val;
+		program = true;
+		break;
+	case V4L2_CID_SATURATION:
+		vid->current_saturation = ctrl->val;
+		program = true;
+		break;
+	case V4L2_CID_HUE:
+		vid->current_hue = ctrl->val;
+		program = true;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (program) {
+		hws_hw_write_bchs(pdx, vid->channel_index,
+				  (u8)vid->current_brightness,
+				  (u8)vid->current_contrast,
+				  (u8)vid->current_hue,
+				  (u8)vid->current_saturation);
+	}
+	return 0;
+}
+
+const struct v4l2_ctrl_ops hws_ctrl_ops = {
+	.s_ctrl = hws_s_ctrl,
+};
+
+int hws_vidioc_querycap(struct file *file, void *priv, struct v4l2_capability *cap)
+{
+	struct hws_video *vid = video_drvdata(file);
+	int vi_index = vid->channel_index + 1; /* keep it simple */
+
+	strscpy(cap->driver, KBUILD_MODNAME, sizeof(cap->driver));
+	snprintf(cap->card, sizeof(cap->card),
+		 "AVMatrix HWS Capture %d", vi_index);
+	return 0;
+}
+
+int hws_vidioc_enum_fmt_vid_cap(struct file *file, void *priv_fh, struct v4l2_fmtdesc *f)
+{
+	if (f->index != 0)
+		return -EINVAL; /* only one format */
+
+	f->pixelformat = V4L2_PIX_FMT_YUYV;
+	return 0;
+}
+
+int hws_vidioc_g_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *fmt)
+{
+	struct hws_video *vid = video_drvdata(file);
+
+	fmt->fmt.pix.width        = vid->pix.width;
+	fmt->fmt.pix.height       = vid->pix.height;
+	fmt->fmt.pix.pixelformat  = V4L2_PIX_FMT_YUYV;
+	fmt->fmt.pix.field        = vid->pix.field;
+	fmt->fmt.pix.bytesperline = vid->pix.bytesperline;
+	fmt->fmt.pix.sizeimage    = vid->pix.sizeimage;
+	fmt->fmt.pix.colorspace   = vid->pix.colorspace;
+	fmt->fmt.pix.ycbcr_enc    = vid->pix.ycbcr_enc;
+	fmt->fmt.pix.quantization = vid->pix.quantization;
+	fmt->fmt.pix.xfer_func    = vid->pix.xfer_func;
+	return 0;
+}
+
+static inline void hws_set_colorimetry_fmt(struct v4l2_pix_format *p)
+{
+	bool sd = p->height <= 576;
+
+	p->colorspace   = sd ? V4L2_COLORSPACE_SMPTE170M : V4L2_COLORSPACE_REC709;
+	p->ycbcr_enc    = V4L2_YCBCR_ENC_DEFAULT;
+	p->quantization = V4L2_QUANTIZATION_FULL_RANGE;
+	p->xfer_func    = V4L2_XFER_FUNC_DEFAULT;
+}
+
+int hws_vidioc_try_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f)
+{
+	struct hws_video *vid = file ? video_drvdata(file) : NULL;
+	struct hws_pcie_dev *pdev = vid ? vid->parent : NULL;
+	struct v4l2_pix_format *pix = &f->fmt.pix;
+	u32 req_w = pix->width, req_h = pix->height;
+	u32 w, h, min_bpl, bpl;
+	size_t size; /* wider than u32 for overflow check */
+	size_t max_frame = pdev ? pdev->max_hw_video_buf_sz : MAX_MM_VIDEO_SIZE;
+
+	/* Only YUYV */
+	pix->pixelformat = V4L2_PIX_FMT_YUYV;
+
+	/* Defaults then clamp */
+	w = (req_w ? req_w : 640);
+	h = (req_h ? req_h : 480);
+	if (w > MAX_VIDEO_HW_W)
+		w = MAX_VIDEO_HW_W;
+	if (h > MAX_VIDEO_HW_H)
+		h = MAX_VIDEO_HW_H;
+	if (!w)
+		w = 640; /* hard fallback in case macros are odd */
+	if (!h)
+		h = 480;
+
+	/* Field policy */
+	pix->field = V4L2_FIELD_NONE;
+
+	/* Stride policy for packed 16bpp, 64B align */
+	min_bpl = ALIGN(w * 2, 64);
+
+	/* Bound requested bpl to something sane, then align */
+	bpl = pix->bytesperline;
+	if (bpl < min_bpl) {
+		bpl = min_bpl;
+	} else {
+		/* Cap at 16x width to avoid silly values that overflow sizeimage */
+		u32 max_bpl = ALIGN(w * 2 * 16, 64);
+
+		if (bpl > max_bpl)
+			bpl = max_bpl;
+		bpl = ALIGN(bpl, 64);
+	}
+	if (h && max_frame) {
+		size_t max_bpl_hw = max_frame / h;
+
+		if (max_bpl_hw < min_bpl)
+			return -ERANGE;
+		max_bpl_hw = rounddown(max_bpl_hw, 64);
+		if (!max_bpl_hw)
+			return -ERANGE;
+		if (bpl > max_bpl_hw) {
+			if (pdev)
+				dev_dbg(&pdev->pdev->dev,
+					"try_fmt: clamp bpl %u -> %zu due to hw buf cap %zu\n",
+					bpl, max_bpl_hw, max_frame);
+			bpl = (u32)max_bpl_hw;
+		}
+	}
+	size = (size_t)bpl * (size_t)h;
+	if (size > max_frame)
+		return -ERANGE;
+
+	pix->width        = w;
+	pix->height       = h;
+	pix->bytesperline = bpl;
+	pix->sizeimage    = (u32)size; /* logical size, not page-aligned */
+
+	hws_set_colorimetry_fmt(pix);
+	if (pdev)
+		dev_dbg(&pdev->pdev->dev,
+			"try_fmt: w=%u h=%u bpl=%u size=%u field=%u\n",
+			pix->width, pix->height, pix->bytesperline,
+			pix->sizeimage, pix->field);
+	return 0;
+}
+
+int hws_vidioc_s_fmt_vid_cap(struct file *file, void *priv, struct v4l2_format *f)
+{
+	struct hws_video *vid = video_drvdata(file);
+	int ret;
+
+	if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	/* Normalize the request */
+	ret = hws_vidioc_try_fmt_vid_cap(file, priv, f);
+	if (ret)
+		return ret;
+
+	/* Don't allow buffer layout changes while buffers are queued. */
+	if (vb2_is_busy(&vid->buffer_queue)) {
+		if (f->fmt.pix.width  != vid->pix.width  ||
+		    f->fmt.pix.height != vid->pix.height ||
+		    f->fmt.pix.bytesperline != vid->pix.bytesperline)
+			return -EBUSY;
+	}
+
+	/* Apply to driver state */
+	vid->pix.width        = f->fmt.pix.width;
+	vid->pix.height       = f->fmt.pix.height;
+	vid->pix.fourcc       = V4L2_PIX_FMT_YUYV;
+	vid->pix.field        = f->fmt.pix.field;
+	vid->pix.colorspace   = f->fmt.pix.colorspace;
+	vid->pix.ycbcr_enc    = f->fmt.pix.ycbcr_enc;
+	vid->pix.quantization = f->fmt.pix.quantization;
+	vid->pix.xfer_func    = f->fmt.pix.xfer_func;
+
+	/* Update negotiated buffer sizes. */
+	vid->pix.bytesperline = f->fmt.pix.bytesperline; /* aligned */
+	vid->pix.sizeimage    = f->fmt.pix.sizeimage;    /* logical */
+	vid->pix.half_size    = hws_calc_half_size(vid->pix.sizeimage);
+	vid->pix.interlaced   = false;
+	/* S_FMT negotiates buffer layout only. Keep detector-owned DV timing
+	 * state unchanged so a harmless restart cannot clobber the live FPS.
+	 */
+	/* Or:
+	 * hws_calc_sizeimage(vid, vid->pix.width, vid->pix.height, false);
+	 */
+
+	dev_dbg(&vid->parent->pdev->dev,
+		"s_fmt:   w=%u h=%u bpl=%u size=%u\n",
+		vid->pix.width, vid->pix.height, vid->pix.bytesperline,
+		vid->pix.sizeimage);
+
+	return 0;
+}
+
+int hws_vidioc_g_parm(struct file *file, void *fh, struct v4l2_streamparm *param)
+{
+	struct hws_video *vid = video_drvdata(file);
+	u32 fps;
+
+	if (param->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	fps = hws_get_live_fps(vid);
+	if (!fps)
+		fps = vid->current_fps ? vid->current_fps : 60;
+
+	/* HDMI receivers report the detected frame period, they don't set it. */
+	param->parm.capture.capability           = 0;
+	param->parm.capture.capturemode          = 0;
+	param->parm.capture.timeperframe.numerator   = 1;
+	param->parm.capture.timeperframe.denominator = fps;
+	param->parm.capture.extendedmode         = 0;
+	param->parm.capture.readbuffers          = 0;
+
+	return 0;
+}
+
+int hws_vidioc_enum_input(struct file *file, void *priv,
+			  struct v4l2_input *input)
+{
+	struct hws_video *vid = video_drvdata(file);
+
+	if (input->index)
+		return -EINVAL;
+	input->type         = V4L2_INPUT_TYPE_CAMERA;
+	strscpy(input->name, KBUILD_MODNAME, sizeof(input->name));
+	input->capabilities = V4L2_IN_CAP_DV_TIMINGS;
+	input->status       = hws_input_status(vid);
+
+	return 0;
+}
+
+int hws_vidioc_g_input(struct file *file, void *priv, unsigned int *index)
+{
+	*index = 0;
+	return 0;
+}
+
+int hws_vidioc_s_input(struct file *file, void *priv, unsigned int i)
+{
+	return i ? -EINVAL : 0;
+}
diff --git a/drivers/media/pci/hws/hws_v4l2_ioctl.h b/drivers/media/pci/hws/hws_v4l2_ioctl.h
new file mode 100644
index 0000000000000..53044f78d6fa3
--- /dev/null
+++ b/drivers/media/pci/hws/hws_v4l2_ioctl.h
@@ -0,0 +1,36 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef HWS_V4L2_IOCTL_H
+#define HWS_V4L2_IOCTL_H
+
+#include <media/v4l2-ctrls.h>
+#include <linux/fs.h>
+
+extern const struct v4l2_ctrl_ops hws_ctrl_ops;
+
+int hws_vidioc_querycap(struct file *file, void *priv, struct v4l2_capability *cap);
+int hws_vidioc_enum_fmt_vid_cap(struct file *file, void *priv_fh, struct v4l2_fmtdesc *f);
+int hws_vidioc_g_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *fmt);
+int hws_vidioc_try_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format *f);
+int hws_vidioc_g_std(struct file *file, void *priv, v4l2_std_id *tvnorms);
+int hws_vidioc_s_std(struct file *file, void *priv, v4l2_std_id tvnorms);
+int hws_vidioc_g_parm(struct file *file, void *fh, struct v4l2_streamparm *setfps);
+int hws_vidioc_enum_input(struct file *file, void *priv, struct v4l2_input *i);
+int hws_vidioc_g_input(struct file *file, void *priv, unsigned int *i);
+int hws_vidioc_s_input(struct file *file, void *priv, unsigned int i);
+int hws_vidioc_g_ctrl(struct file *file, void *fh, struct v4l2_control *a);
+int hws_vidioc_s_ctrl(struct file *file, void *fh, struct v4l2_control *a);
+int hws_vidioc_dv_timings_cap(struct file *file, void *fh,
+			      struct v4l2_dv_timings_cap *cap);
+int hws_vidioc_s_dv_timings(struct file *file, void *fh,
+			    struct v4l2_dv_timings *timings);
+
+int hws_vidioc_queryctrl(struct file *file, void *fh, struct v4l2_queryctrl *a);
+int hws_vidioc_g_dv_timings(struct file *file, void *fh,
+			    struct v4l2_dv_timings *timings);
+int hws_vidioc_enum_dv_timings(struct file *file, void *fh,
+			       struct v4l2_enum_dv_timings *edv);
+int hws_vidioc_query_dv_timings(struct file *file, void *fh,
+				struct v4l2_dv_timings *timings);
+int hws_vidioc_s_fmt_vid_cap(struct file *file, void *priv, struct v4l2_format *f);
+
+#endif
diff --git a/drivers/media/pci/hws/hws_video.c b/drivers/media/pci/hws/hws_video.c
new file mode 100644
index 0000000000000..18e4bc6901d30
--- /dev/null
+++ b/drivers/media/pci/hws/hws_video.c
@@ -0,0 +1,1490 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/pci.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/compiler.h>
+#include <linux/overflow.h>
+#include <linux/delay.h>
+#include <linux/bits.h>
+#include <linux/jiffies.h>
+#include <linux/ktime.h>
+#include <linux/math64.h>
+#include <linux/interrupt.h>
+#include <linux/moduleparam.h>
+
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-dev.h>
+#include <media/v4l2-event.h>
+#include <media/videobuf2-v4l2.h>
+#include <media/v4l2-device.h>
+#include <media/videobuf2-dma-contig.h>
+
+#include "hws.h"
+#include "hws_reg.h"
+#include "hws_video.h"
+#include "hws_irq.h"
+#include "hws_v4l2_ioctl.h"
+
+#define HWS_REMAP_SLOT_OFF(ch)   (0x208 + (ch) * 8)	/* one 64-bit slot per ch */
+#define HWS_BUF_BASE_OFF(ch)     (CVBS_IN_BUF_BASE  + (ch) * PCIE_BARADDROFSIZE)
+#define HWS_HALF_SZ_OFF(ch)      (CVBS_IN_BUF_BASE2 + (ch) * PCIE_BARADDROFSIZE)
+
+static void update_live_resolution(struct hws_pcie_dev *pdx, unsigned int ch,
+				   bool interlace);
+static bool hws_read_active_state(struct hws_pcie_dev *pdx, unsigned int ch,
+				  bool *interlace);
+static void handle_hwv2_path(struct hws_pcie_dev *hws, unsigned int ch);
+static void handle_legacy_path(struct hws_pcie_dev *hws, unsigned int ch);
+static u32 hws_calc_sizeimage(struct hws_video *v, u16 w, u16 h,
+			      bool interlaced);
+
+/* DMA helper functions */
+static void hws_program_dma_window(struct hws_video *vid, dma_addr_t dma);
+static struct hwsvideo_buffer *
+hws_take_queued_buffer_locked(struct hws_video *vid);
+
+static unsigned long long hws_elapsed_us(u64 start_ns)
+{
+	return div_u64(ktime_get_mono_fast_ns() - start_ns, 1000);
+}
+
+static inline bool list_node_unlinked(const struct list_head *n)
+{
+	return n->next == LIST_POISON1 || n->prev == LIST_POISON2;
+}
+
+static bool dma_window_verify;
+module_param_named(dma_window_verify, dma_window_verify, bool, 0644);
+MODULE_PARM_DESC(dma_window_verify,
+		 "Read back DMA window registers after programming (debug)");
+
+void hws_set_dma_doorbell(struct hws_pcie_dev *hws, unsigned int ch,
+			  dma_addr_t dma, const char *tag)
+{
+	iowrite32(lower_32_bits(dma), hws->bar0_base + HWS_REG_DMA_ADDR(ch));
+	dev_dbg(&hws->pdev->dev, "dma_doorbell ch%u: dma=0x%llx tag=%s\n", ch,
+		(u64)dma, tag ? tag : "");
+}
+
+static void hws_program_dma_window(struct hws_video *vid, dma_addr_t dma)
+{
+	const u32 addr_mask = PCI_E_BAR_ADD_MASK;
+	const u32 addr_low_mask = PCI_E_BAR_ADD_LOWMASK;
+	struct hws_pcie_dev *hws = vid->parent;
+	unsigned int ch = vid->channel_index;
+	u32 table_off = HWS_REMAP_SLOT_OFF(ch);
+	u32 lo = lower_32_bits(dma);
+	u32 hi = upper_32_bits(dma);
+	u32 pci_addr = lo & addr_low_mask;
+	u32 page_lo = lo & addr_mask;
+
+	bool wrote = false;
+
+	/* Remap entry only when DMA crosses into a new 512 MB page */
+	if (!vid->window_valid || vid->last_dma_hi != hi ||
+	    vid->last_dma_page != page_lo) {
+		writel(hi, hws->bar0_base + PCI_ADDR_TABLE_BASE + table_off);
+		writel(page_lo,
+		       hws->bar0_base + PCI_ADDR_TABLE_BASE + table_off +
+		       PCIE_BARADDROFSIZE);
+		vid->last_dma_hi = hi;
+		vid->last_dma_page = page_lo;
+		wrote = true;
+	}
+
+	/* Base pointer only needs low 29 bits */
+	if (!vid->window_valid || vid->last_pci_addr != pci_addr) {
+		writel((ch + 1) * PCIEBAR_AXI_BASE + pci_addr,
+		       hws->bar0_base + HWS_BUF_BASE_OFF(ch));
+		vid->last_pci_addr = pci_addr;
+		wrote = true;
+	}
+
+	/* Half-size only changes when resolution changes */
+	if (!vid->window_valid || vid->last_half16 != vid->pix.half_size / 16) {
+		writel(vid->pix.half_size / 16,
+		       hws->bar0_base + HWS_HALF_SZ_OFF(ch));
+		vid->last_half16 = vid->pix.half_size / 16;
+		wrote = true;
+	}
+
+	vid->window_valid = true;
+
+	if (dma_window_verify && wrote) {
+		u32 r_hi =
+		    readl(hws->bar0_base + PCI_ADDR_TABLE_BASE + table_off);
+		u32 r_lo =
+		    readl(hws->bar0_base + PCI_ADDR_TABLE_BASE + table_off +
+			  PCIE_BARADDROFSIZE);
+		u32 r_base = readl(hws->bar0_base + HWS_BUF_BASE_OFF(ch));
+		u32 r_half = readl(hws->bar0_base + HWS_HALF_SZ_OFF(ch));
+
+		dev_dbg(&hws->pdev->dev,
+			"ch%u remap verify: hi=0x%08x page_lo=0x%08x exp_page=0x%08x base=0x%08x exp_base=0x%08x half16B=0x%08x exp_half=0x%08x\n",
+			ch, r_hi, r_lo, page_lo, r_base,
+			(ch + 1) * PCIEBAR_AXI_BASE + pci_addr, r_half,
+			vid->pix.half_size / 16);
+	} else if (wrote) {
+		/* Flush posted writes before arming DMA */
+		readl_relaxed(hws->bar0_base + HWS_HALF_SZ_OFF(ch));
+	}
+}
+
+static struct hwsvideo_buffer *
+hws_take_queued_buffer_locked(struct hws_video *vid)
+{
+	struct hwsvideo_buffer *buf;
+
+	if (!vid || list_empty(&vid->capture_queue))
+		return NULL;
+
+	buf = list_first_entry(&vid->capture_queue,
+			       struct hwsvideo_buffer, list);
+	list_del_init(&buf->list);
+	if (vid->queued_count)
+		vid->queued_count--;
+	return buf;
+}
+
+void hws_prime_next_locked(struct hws_video *vid)
+{
+	struct hws_pcie_dev *hws;
+	struct hwsvideo_buffer *next;
+	dma_addr_t dma;
+
+	if (!vid)
+		return;
+
+	hws = vid->parent;
+	if (!hws || !hws->bar0_base)
+		return;
+
+	if (!READ_ONCE(vid->cap_active) || !vid->active || vid->next_prepared)
+		return;
+
+	next = hws_take_queued_buffer_locked(vid);
+	if (!next)
+		return;
+
+	vid->next_prepared = next;
+	dma = vb2_dma_contig_plane_dma_addr(&next->vb.vb2_buf, 0);
+	hws_program_dma_for_addr(hws, vid->channel_index, dma);
+	iowrite32(lower_32_bits(dma),
+		  hws->bar0_base + HWS_REG_DMA_ADDR(vid->channel_index));
+	dev_dbg(&hws->pdev->dev,
+		"ch%u pre-armed next buffer %p dma=0x%llx\n",
+		vid->channel_index, next, (u64)dma);
+}
+
+static bool hws_force_no_signal_frame(struct hws_video *v, const char *tag)
+{
+	struct hws_pcie_dev *hws;
+	unsigned long flags;
+	struct hwsvideo_buffer *buf = NULL, *next = NULL;
+	bool have_next = false;
+	bool doorbell = false;
+
+	if (!v)
+		return false;
+	hws = v->parent;
+	if (!hws || READ_ONCE(v->stop_requested) || !READ_ONCE(v->cap_active))
+		return false;
+	spin_lock_irqsave(&v->irq_lock, flags);
+	if (v->active) {
+		buf = v->active;
+		v->active = NULL;
+		buf->slot = 0;
+	} else if (!list_empty(&v->capture_queue)) {
+		buf = list_first_entry(&v->capture_queue,
+				       struct hwsvideo_buffer, list);
+		list_del_init(&buf->list);
+		if (v->queued_count)
+			v->queued_count--;
+		buf->slot = 0;
+	}
+	if (v->next_prepared) {
+		next = v->next_prepared;
+		v->next_prepared = NULL;
+		next->slot = 0;
+		v->active = next;
+		have_next = true;
+	} else if (!list_empty(&v->capture_queue)) {
+		next = list_first_entry(&v->capture_queue,
+					struct hwsvideo_buffer, list);
+		list_del_init(&next->list);
+		if (v->queued_count)
+			v->queued_count--;
+		next->slot = 0;
+		v->active = next;
+		have_next = true;
+	} else {
+		v->active = NULL;
+	}
+	spin_unlock_irqrestore(&v->irq_lock, flags);
+	if (!buf)
+		return false;
+	/* Complete buffer with a neutral frame so dequeuers keep running. */
+	{
+		struct vb2_v4l2_buffer *vb2v = &buf->vb;
+		void *dst = vb2_plane_vaddr(&vb2v->vb2_buf, 0);
+
+		if (dst)
+			memset(dst, 0x10, v->pix.sizeimage);
+		vb2_set_plane_payload(&vb2v->vb2_buf, 0, v->pix.sizeimage);
+		vb2v->sequence = (u32)atomic_inc_return(&v->sequence_number);
+		vb2v->vb2_buf.timestamp = ktime_get_ns();
+		vb2_buffer_done(&vb2v->vb2_buf, VB2_BUF_STATE_DONE);
+	}
+	if (have_next && next) {
+		dma_addr_t dma =
+		    vb2_dma_contig_plane_dma_addr(&next->vb.vb2_buf, 0);
+		hws_program_dma_for_addr(hws, v->channel_index, dma);
+		hws_set_dma_doorbell(hws, v->channel_index, dma,
+				     tag ? tag : "nosignal_zero");
+		doorbell = true;
+	}
+	if (doorbell) {
+		wmb(); /* ensure descriptors visible before enabling capture */
+		hws_enable_video_capture(hws, v->channel_index, true);
+	}
+	return true;
+}
+
+static int hws_ctrls_init(struct hws_video *vid)
+{
+	struct v4l2_ctrl_handler *hdl = &vid->control_handler;
+
+	/* Create BCHS controls. */
+	v4l2_ctrl_handler_init(hdl, 4);
+
+	vid->ctrl_brightness = v4l2_ctrl_new_std(hdl, &hws_ctrl_ops,
+						 V4L2_CID_BRIGHTNESS,
+						 MIN_VAMP_BRIGHTNESS_UNITS,
+						 MAX_VAMP_BRIGHTNESS_UNITS, 1,
+						 HWS_BRIGHTNESS_DEFAULT);
+
+	vid->ctrl_contrast =
+	    v4l2_ctrl_new_std(hdl, &hws_ctrl_ops, V4L2_CID_CONTRAST,
+			      MIN_VAMP_CONTRAST_UNITS, MAX_VAMP_CONTRAST_UNITS,
+			      1, HWS_CONTRAST_DEFAULT);
+
+	vid->ctrl_saturation = v4l2_ctrl_new_std(hdl, &hws_ctrl_ops,
+						 V4L2_CID_SATURATION,
+						 MIN_VAMP_SATURATION_UNITS,
+						 MAX_VAMP_SATURATION_UNITS, 1,
+						 HWS_SATURATION_DEFAULT);
+
+	vid->ctrl_hue = v4l2_ctrl_new_std(hdl, &hws_ctrl_ops, V4L2_CID_HUE,
+					  MIN_VAMP_HUE_UNITS,
+					  MAX_VAMP_HUE_UNITS, 1,
+					  HWS_HUE_DEFAULT);
+	if (hdl->error) {
+		int err = hdl->error;
+
+		v4l2_ctrl_handler_free(hdl);
+		return err;
+	}
+	return 0;
+}
+
+int hws_video_init_channel(struct hws_pcie_dev *pdev, int ch)
+{
+	struct hws_video *vid;
+
+	/* basic sanity */
+	if (!pdev || ch < 0 || ch >= pdev->max_channels)
+		return -EINVAL;
+
+	vid = &pdev->video[ch];
+
+	/* hard reset the per-channel struct (safe here since we init everything next) */
+	memset(vid, 0, sizeof(*vid));
+
+	/* identity */
+	vid->parent = pdev;
+	vid->channel_index = ch;
+
+	/* locks & lists */
+	mutex_init(&vid->state_lock);
+	spin_lock_init(&vid->irq_lock);
+	INIT_LIST_HEAD(&vid->capture_queue);
+	atomic_set(&vid->sequence_number, 0);
+	vid->active = NULL;
+
+	/* DMA watchdog removed; retain counters for diagnostics */
+	vid->timeout_count = 0;
+	vid->error_count = 0;
+
+	vid->queued_count = 0;
+	vid->window_valid = false;
+
+	/* Default format. */
+	vid->pix.width = 1920;
+	vid->pix.height = 1080;
+	vid->pix.fourcc = V4L2_PIX_FMT_YUYV;
+	vid->pix.bytesperline = ALIGN(vid->pix.width * 2, 64);
+	vid->pix.sizeimage = vid->pix.bytesperline * vid->pix.height;
+	vid->pix.field = V4L2_FIELD_NONE;
+	vid->pix.colorspace = V4L2_COLORSPACE_REC709;
+	vid->pix.ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
+	vid->pix.quantization = V4L2_QUANTIZATION_FULL_RANGE;
+	vid->pix.xfer_func = V4L2_XFER_FUNC_DEFAULT;
+	vid->pix.interlaced = false;
+	vid->pix.half_size = vid->pix.sizeimage / 2;
+	hws_set_current_dv_timings(vid, vid->pix.width,
+				   vid->pix.height, vid->pix.interlaced);
+	vid->current_fps = 60;
+
+	/* color controls default (mid-scale) */
+	vid->current_brightness = 0x80;
+	vid->current_contrast = 0x80;
+	vid->current_saturation = 0x80;
+	vid->current_hue = 0x80;
+
+	/* capture state */
+	vid->cap_active = false;
+	vid->stop_requested = false;
+	vid->last_buf_half_toggle = 0;
+	vid->half_seen = false;
+	vid->signal_loss_cnt = 0;
+
+	/* Create BCHS + DV power-present as modern controls */
+	{
+		int err = hws_ctrls_init(vid);
+
+		if (err) {
+			dev_err(&pdev->pdev->dev,
+				"v4l2 ctrl init failed on ch%d: %d\n", ch, err);
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+static void hws_video_drain_queue_locked(struct hws_video *vid)
+{
+	/* Return in-flight first */
+	if (vid->active) {
+		vb2_buffer_done(&vid->active->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+		vid->active = NULL;
+	}
+
+	/* Then everything queued */
+	while (!list_empty(&vid->capture_queue)) {
+		struct hwsvideo_buffer *b =
+		    list_first_entry(&vid->capture_queue,
+				     struct hwsvideo_buffer,
+				     list);
+		list_del_init(&b->list);
+		vb2_buffer_done(&b->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+	}
+}
+
+static void hws_video_release_registration(struct hws_video *vid)
+{
+	if (vid->buffer_queue.ops) {
+		vb2_queue_release(&vid->buffer_queue);
+		vid->buffer_queue.ops = NULL;
+	}
+
+	if (!vid->video_device)
+		return;
+
+	if (video_is_registered(vid->video_device))
+		vb2_video_unregister_device(vid->video_device);
+	else
+		video_device_release(vid->video_device);
+	vid->video_device = NULL;
+}
+
+static void hws_video_collect_done_locked(struct hws_video *vid,
+					  struct list_head *done)
+{
+	struct hwsvideo_buffer *b;
+
+	if (vid->active) {
+		if (!list_node_unlinked(&vid->active->list)) {
+			list_move_tail(&vid->active->list, done);
+		} else {
+			INIT_LIST_HEAD(&vid->active->list);
+			list_add_tail(&vid->active->list, done);
+		}
+		vid->active = NULL;
+	}
+
+	if (vid->next_prepared) {
+		if (!list_node_unlinked(&vid->next_prepared->list)) {
+			list_move_tail(&vid->next_prepared->list, done);
+		} else {
+			INIT_LIST_HEAD(&vid->next_prepared->list);
+			list_add_tail(&vid->next_prepared->list, done);
+		}
+		vid->next_prepared = NULL;
+	}
+
+	while (!list_empty(&vid->capture_queue)) {
+		b = list_first_entry(&vid->capture_queue, struct hwsvideo_buffer,
+				     list);
+		list_move_tail(&b->list, done);
+	}
+
+	vid->queued_count = 0;
+}
+
+void hws_video_cleanup_channel(struct hws_pcie_dev *pdev, int ch)
+{
+	struct hws_video *vid;
+	unsigned long flags;
+
+	if (!pdev || ch < 0 || ch >= pdev->max_channels)
+		return;
+
+	vid = &pdev->video[ch];
+
+	/* 1) Stop HW best-effort for this channel */
+	hws_enable_video_capture(vid->parent, vid->channel_index, false);
+
+	/* 2) Flip software state so IRQ/BH will be no-ops if they run */
+	WRITE_ONCE(vid->stop_requested, true);
+	WRITE_ONCE(vid->cap_active, false);
+
+	/* 3) Ensure the IRQ handler finished any in-flight completions */
+	if (vid->parent && vid->parent->irq >= 0)
+		synchronize_irq(vid->parent->irq);
+
+	/* 4) Drain SW capture queue & in-flight under lock */
+	spin_lock_irqsave(&vid->irq_lock, flags);
+	hws_video_drain_queue_locked(vid);
+	spin_unlock_irqrestore(&vid->irq_lock, flags);
+
+	/* 5) Release VB2 queue if initialized */
+	hws_video_release_registration(vid);
+
+	/* 6) Free V4L2 controls */
+	v4l2_ctrl_handler_free(&vid->control_handler);
+
+	/* 8) Reset simple state; do not memset the whole struct here. */
+	mutex_destroy(&vid->state_lock);
+	INIT_LIST_HEAD(&vid->capture_queue);
+	vid->active = NULL;
+	vid->stop_requested = false;
+	vid->last_buf_half_toggle = 0;
+	vid->half_seen = false;
+	vid->signal_loss_cnt = 0;
+}
+
+/* Convenience cast */
+static inline struct hwsvideo_buffer *to_hwsbuf(struct vb2_buffer *vb)
+{
+	return container_of(to_vb2_v4l2_buffer(vb), struct hwsvideo_buffer, vb);
+}
+
+static int hws_buf_init(struct vb2_buffer *vb)
+{
+	struct hwsvideo_buffer *b = to_hwsbuf(vb);
+
+	INIT_LIST_HEAD(&b->list);
+	return 0;
+}
+
+static void hws_buf_finish(struct vb2_buffer *vb)
+{
+	/* vb2 core handles cache maintenance for dma-contig buffers */
+	(void)vb;
+}
+
+static void hws_buf_cleanup(struct vb2_buffer *vb)
+{
+	struct hwsvideo_buffer *b = to_hwsbuf(vb);
+
+	if (!list_empty(&b->list))
+		list_del_init(&b->list);
+}
+
+void hws_program_dma_for_addr(struct hws_pcie_dev *hws, unsigned int ch,
+			      dma_addr_t dma)
+{
+	struct hws_video *vid = &hws->video[ch];
+
+	hws_program_dma_window(vid, dma);
+}
+
+void hws_enable_video_capture(struct hws_pcie_dev *hws, unsigned int chan,
+			      bool on)
+{
+	u32 status;
+
+	if (!hws || hws->pci_lost || chan >= hws->max_channels)
+		return;
+
+	status = readl(hws->bar0_base + HWS_REG_VCAP_ENABLE);
+	status = on ? (status | BIT(chan)) : (status & ~BIT(chan));
+	writel(status, hws->bar0_base + HWS_REG_VCAP_ENABLE);
+	(void)readl(hws->bar0_base + HWS_REG_VCAP_ENABLE);
+
+	WRITE_ONCE(hws->video[chan].cap_active, on);
+
+	dev_dbg(&hws->pdev->dev, "vcap %s ch%u (reg=0x%08x)\n",
+		on ? "ON" : "OFF", chan, status);
+}
+
+static void hws_seed_dma_windows(struct hws_pcie_dev *hws)
+{
+	const u32 addr_mask = PCI_E_BAR_ADD_MASK;
+	const u32 addr_low_mask = PCI_E_BAR_ADD_LOWMASK;
+	u32 table = 0x208;	/* one 64-bit entry per channel */
+	unsigned int ch;
+
+	if (!hws || !hws->bar0_base)
+		return;
+
+	/* If cur_max_video_ch is not set yet, default to max_channels. */
+	if (!hws->cur_max_video_ch || hws->cur_max_video_ch > hws->max_channels)
+		hws->cur_max_video_ch = hws->max_channels;
+
+	for (ch = 0; ch < hws->cur_max_video_ch; ch++, table += 8) {
+		if (!hws->scratch_vid[ch].cpu)
+			continue;
+
+		/* Program 64-bit BAR remap entry for this channel */
+		{
+			dma_addr_t p = hws->scratch_vid[ch].dma;
+			u32 lo = lower_32_bits(p) & addr_mask;
+			u32 hi = upper_32_bits(p);
+			u32 pci_addr_low = lower_32_bits(p) & addr_low_mask;
+
+			writel_relaxed(hi,
+				       hws->bar0_base + PCI_ADDR_TABLE_BASE +
+				       table);
+			writel_relaxed(lo,
+				       hws->bar0_base + PCI_ADDR_TABLE_BASE +
+				       table + PCIE_BARADDROFSIZE);
+
+			/* Per-channel AXI base + PCI low */
+			writel_relaxed((ch + 1) * PCIEBAR_AXI_BASE +
+				       pci_addr_low,
+				       hws->bar0_base + CVBS_IN_BUF_BASE +
+				       ch * PCIE_BARADDROFSIZE);
+
+			/* Half-frame length in /16 units.
+			 * Prefer the current channel's computed half_size if available.
+			 * Fall back to half of the probe-owned scratch buffer.
+			 */
+			{
+				u32 half_bytes = hws->video[ch].pix.half_size ?
+				    hws->video[ch].pix.half_size :
+				    (u32)(hws->scratch_vid[ch].size / 2);
+				writel_relaxed(half_bytes / 16,
+					       hws->bar0_base +
+					       CVBS_IN_BUF_BASE2 +
+					       ch * PCIE_BARADDROFSIZE);
+			}
+		}
+	}
+
+	/* Post writes so device sees them before we move on */
+	(void)readl(hws->bar0_base + HWS_REG_INT_STATUS);
+}
+
+static void hws_ack_all_irqs(struct hws_pcie_dev *hws)
+{
+	u32 st = readl(hws->bar0_base + HWS_REG_INT_STATUS);
+
+	if (st) {
+		writel(st, hws->bar0_base + HWS_REG_INT_STATUS);	/* W1C */
+		(void)readl(hws->bar0_base + HWS_REG_INT_STATUS);
+	}
+}
+
+static void hws_open_irq_fabric(struct hws_pcie_dev *hws)
+{
+	/* Route all sources to vector 0. */
+	writel(0x00000000, hws->bar0_base + PCIE_INT_DEC_REG_BASE);
+	(void)readl(hws->bar0_base + PCIE_INT_DEC_REG_BASE);
+
+	/* Enable the PCIe bridge. */
+	writel(0x00000001, hws->bar0_base + PCIEBR_EN_REG_BASE);
+	(void)readl(hws->bar0_base + PCIEBR_EN_REG_BASE);
+
+	/* Open the global/bridge gate (legacy 0x3FFFF) */
+	writel(HWS_INT_EN_MASK, hws->bar0_base + INT_EN_REG_BASE);
+	(void)readl(hws->bar0_base + INT_EN_REG_BASE);
+}
+
+void hws_init_video_sys(struct hws_pcie_dev *hws, bool enable)
+{
+	int i;
+
+	if (hws->start_run && !enable)
+		return;
+
+	/* 1) reset the decoder mode register to 0 */
+	writel(0x00000000, hws->bar0_base + HWS_REG_DEC_MODE);
+	hws_seed_dma_windows(hws);
+
+	/* 3) on a full reset, clear all per-channel status and indices */
+	if (!enable) {
+		for (i = 0; i < hws->max_channels; i++) {
+			/* helpers to arm/disable capture engines */
+			hws_enable_video_capture(hws, i, false);
+		}
+	}
+
+	/* 4) Start run: set bit31, wait a bit, then program low 24 bits. */
+	writel(0x80000000, hws->bar0_base + HWS_REG_DEC_MODE);
+	writel(0x80FFFFFF, hws->bar0_base + HWS_REG_DEC_MODE);
+	writel(0x13, hws->bar0_base + HWS_REG_DEC_MODE);
+	hws_ack_all_irqs(hws);
+	hws_open_irq_fabric(hws);
+	/* 6) record that we're now running */
+	hws->start_run = true;
+}
+
+int hws_check_card_status(struct hws_pcie_dev *hws)
+{
+	u32 status;
+
+	if (!hws || !hws->bar0_base)
+		return -ENODEV;
+
+	status = readl(hws->bar0_base + HWS_REG_SYS_STATUS);
+
+	/* Common device-missing pattern. */
+	if (status == 0xFFFFFFFF) {
+		hws->pci_lost = true;
+		dev_err(&hws->pdev->dev, "PCIe device not responding\n");
+		return -ENODEV;
+	}
+
+	/* If RUN/READY bit (bit0) is not set, reinitialize the video core. */
+	if (!(status & BIT(0))) {
+		dev_dbg(&hws->pdev->dev,
+			"SYS_STATUS not ready (0x%08x), reinitializing\n",
+			status);
+		hws_init_video_sys(hws, true);
+	}
+
+	return 0;
+}
+
+void check_video_format(struct hws_pcie_dev *pdx)
+{
+	int i;
+
+	for (i = 0; i < pdx->cur_max_video_ch; i++) {
+		bool interlace = false;
+
+		if (!hws_read_active_state(pdx, i, &interlace)) {
+			/* No active video; optionally feed neutral frames to keep streaming. */
+			if (pdx->video[i].signal_loss_cnt == 0)
+				pdx->video[i].signal_loss_cnt = 1;
+			if (READ_ONCE(pdx->video[i].cap_active))
+				hws_force_no_signal_frame(&pdx->video[i],
+							  "monitor_nosignal");
+		} else {
+			if (pdx->hw_ver > 0)
+				handle_hwv2_path(pdx, i);
+			else
+				/* Legacy path stub; see handle_legacy_path() comment. */
+				handle_legacy_path(pdx, i);
+
+			update_live_resolution(pdx, i, interlace);
+			pdx->video[i].signal_loss_cnt = 0;
+		}
+	}
+}
+
+static inline void hws_write_if_diff(struct hws_pcie_dev *hws, u32 reg_off,
+				     u32 new_val)
+{
+	void __iomem *addr;
+	u32 old;
+
+	if (!hws || !hws->bar0_base)
+		return;
+
+	addr = hws->bar0_base + reg_off;
+
+	old = readl(addr);
+	/* Treat all-ones as device gone; avoid writing garbage. */
+	if (old == 0xFFFFFFFF) {
+		hws->pci_lost = true;
+		return;
+	}
+
+	if (old != new_val) {
+		writel(new_val, addr);
+		/* Post the write on some bridges / enforce ordering. */
+		(void)readl(addr);
+	}
+}
+
+static bool hws_read_active_state(struct hws_pcie_dev *pdx, unsigned int ch,
+				  bool *interlace)
+{
+	u32 reg;
+	bool active;
+
+	if (ch >= pdx->cur_max_video_ch)
+		return false;
+
+	reg = readl(pdx->bar0_base + HWS_REG_ACTIVE_STATUS);
+	active = !!(reg & BIT(ch));
+	if (interlace)
+		*interlace = !!(reg & BIT(8 + ch));
+	return active;
+}
+
+/* Modern hardware path: keep HW registers in sync with current per-channel
+ * software state.
+ */
+static void handle_hwv2_path(struct hws_pcie_dev *hws, unsigned int ch)
+{
+	struct hws_video *vid;
+	u32 reg, in_fps, cur_out_res, want_out_res;
+
+	if (!hws || !hws->bar0_base || ch >= hws->max_channels)
+		return;
+
+	vid = &hws->video[ch];
+
+	/* 1) Input frame rate (read-only; log or export via debugfs if wanted) */
+	in_fps = readl(hws->bar0_base + HWS_REG_FRAME_RATE(ch));
+	/* dev_dbg(&hws->pdev->dev, "ch%u input fps=%u\n", ch, in_fps); */
+	(void)in_fps;
+
+	/* 2) Output resolution programming.
+	 * For now, mirror the current format to OUT_RES.
+	 */
+	want_out_res = (vid->pix.height << 16) | vid->pix.width;
+	cur_out_res = readl(hws->bar0_base + HWS_REG_OUT_RES(ch));
+	if (cur_out_res != want_out_res)
+		hws_write_if_diff(hws, HWS_REG_OUT_RES(ch), want_out_res);
+
+	/* 3) Output FPS: only program if you actually track a target.
+	 * Example heuristic (disabled by default):
+	 *
+	 *   u32 out_fps = (vid->fmt_curr.height >= 1080) ? 60 : 30;
+	 *   hws_write_if_diff(hws, HWS_REG_OUT_FRAME_RATE(ch), out_fps);
+	 */
+
+	/* 4) BCHS controls: pack from per-channel current_* fields */
+	reg = readl(hws->bar0_base + HWS_REG_BCHS(ch));
+	{
+		u8 br = reg & 0xFF;
+		u8 co = (reg >> 8) & 0xFF;
+		u8 hu = (reg >> 16) & 0xFF;
+		u8 sa = (reg >> 24) & 0xFF;
+
+		if (br != vid->current_brightness ||
+		    co != vid->current_contrast || hu != vid->current_hue ||
+		    sa != vid->current_saturation) {
+			u32 packed = (vid->current_saturation << 24) |
+			    (vid->current_hue << 16) |
+			    (vid->current_contrast << 8) |
+			    vid->current_brightness;
+			hws_write_if_diff(hws, HWS_REG_BCHS(ch), packed);
+		}
+	}
+
+	/* 5) HDCP detect: read only (no cache field in your structs today) */
+	reg = readl(hws->bar0_base + HWS_REG_HDCP_STATUS);
+	/* bool hdcp = !!(reg & BIT(ch)); // use if you later add a field/control */
+}
+
+static void handle_legacy_path(struct hws_pcie_dev *hws, unsigned int ch)
+{
+	/*
+	 * Legacy (hw_ver == 0) expected behavior:
+	 * - A per-channel SW FPS accumulator incremented on each VDONE.
+	 * - A once-per-second poll mapped the count to discrete FPS:
+	 *   >55*2 => 60, >45*2 => 50, >25*2 => 30, >20*2 => 25, else 60,
+	 *   then reset the accumulator to 0.
+	 * - The *2 factor assumed VDONE fired per-field; if legacy VDONE is
+	 *   per-frame, drop the factor.
+	 *
+	 * Current code keeps this path as a no-op; vid->current_fps stays at the
+	 * default or mode-derived value. If accurate legacy FPS reporting is
+	 * needed (V4L2 g_parm/timeperframe), reintroduce the accumulator in the
+	 * IRQ path and perform the mapping/reset here.
+	 *
+	 * No-op by default. If you introduce a SW FPS accumulator, map it here.
+	 *
+	 * Example skeleton:
+	 *
+	 *   u32 sw_rate = READ_ONCE(hws->sw_fps[ch]); // incremented elsewhere
+	 *   if (sw_rate > THRESHOLD) {
+	 *       u32 fps = pick_fps_from_rate(sw_rate);
+	 *       hws_write_if_diff(hws, HWS_REG_OUT_FRAME_RATE(ch), fps);
+	 *       WRITE_ONCE(hws->sw_fps[ch], 0);
+	 *   }
+	 */
+	(void)hws;
+	(void)ch;
+}
+
+static void hws_video_apply_mode_change(struct hws_pcie_dev *pdx,
+					unsigned int ch, u16 w, u16 h,
+					bool interlaced, u32 fps)
+{
+	struct hws_video *v = &pdx->video[ch];
+	unsigned long flags;
+	bool queue_busy;
+	bool geometry_changed;
+	struct list_head done;
+	struct hwsvideo_buffer *b, *tmp;
+
+	if (!pdx || !pdx->bar0_base)
+		return;
+	if (ch >= pdx->max_channels)
+		return;
+	if (!w || !h || w > MAX_VIDEO_HW_W ||
+	    (!interlaced && h > MAX_VIDEO_HW_H) ||
+	    (interlaced && (h * 2) > MAX_VIDEO_HW_H))
+		return;
+	if (!fps || fps == 0xFFFFFFFF || fps > 240)
+		fps = (h == 576) ? 50 : 60;
+
+	geometry_changed = w != v->pix.width || h != v->pix.height ||
+		interlaced != v->pix.interlaced;
+	if (!geometry_changed && fps == v->current_fps)
+		return;
+
+	if (!geometry_changed) {
+		/* Refresh cached live timing state, but don't emit a resolution
+		 * change event when only the frame rate changes.
+		 */
+		mutex_lock(&v->state_lock);
+		v->pix.interlaced = interlaced;
+		v->pix.field = interlaced ? V4L2_FIELD_INTERLACED :
+					    V4L2_FIELD_NONE;
+		hws_set_current_dv_timings(v, w, h, interlaced);
+		v->current_fps = fps;
+		mutex_unlock(&v->state_lock);
+		return;
+	}
+
+	if (!mutex_trylock(&v->state_lock))
+		return;
+
+	INIT_LIST_HEAD(&done);
+	queue_busy = vb2_is_busy(&v->buffer_queue);
+
+	WRITE_ONCE(v->stop_requested, true);
+	WRITE_ONCE(v->cap_active, false);
+	/* Publish software stop first so the IRQ completion path sees the stop
+	 * before we touch MMIO or the lists. Pairs with READ_ONCE() checks in the
+	 * VDONE handler and hws_arm_next() to prevent completions while modes
+	 * change.
+	 */
+	smp_wmb();
+
+	hws_enable_video_capture(pdx, ch, false);
+	readl(pdx->bar0_base + HWS_REG_INT_STATUS);
+
+	if (v->parent && v->parent->irq >= 0)
+		synchronize_irq(v->parent->irq);
+
+	spin_lock_irqsave(&v->irq_lock, flags);
+	hws_video_collect_done_locked(v, &done);
+	spin_unlock_irqrestore(&v->irq_lock, flags);
+
+	/* Update software pixel state */
+	v->pix.width = w;
+	v->pix.height = h;
+	v->pix.interlaced = interlaced;
+	hws_set_current_dv_timings(v, w, h, interlaced);
+	v->current_fps = fps;
+
+	hws_calc_sizeimage(v, w, h, interlaced);
+	v->window_valid = false;
+
+	/* Geometry changes require userspace renegotiation once buffers exist.
+	 * Emit SOURCE_CHANGE, mark the queue in error, and let userspace
+	 * STREAMOFF/REQBUFS/STREAMON rather than trying to restart capture
+	 * with partially drained in-flight state.
+	 */
+	if (queue_busy) {
+		struct v4l2_event ev = {
+			.type = V4L2_EVENT_SOURCE_CHANGE,
+		};
+
+		ev.u.src_change.changes = V4L2_EVENT_SRC_CH_RESOLUTION;
+		v4l2_event_queue(v->video_device, &ev);
+		vb2_queue_error(&v->buffer_queue);
+	} else {
+		WRITE_ONCE(v->stop_requested, false);
+	}
+
+	/* Program HW with new resolution */
+	hws_write_if_diff(pdx, HWS_REG_OUT_RES(ch), (h << 16) | w);
+
+	/* Legacy half-buffer programming */
+	writel(v->pix.half_size / 16,
+	       pdx->bar0_base + CVBS_IN_BUF_BASE2 + ch * PCIE_BARADDROFSIZE);
+	(void)readl(pdx->bar0_base + CVBS_IN_BUF_BASE2 +
+		    ch * PCIE_BARADDROFSIZE);
+
+	/* Reset per-channel toggles/counters */
+	WRITE_ONCE(v->last_buf_half_toggle, 0);
+	atomic_set(&v->sequence_number, 0);
+
+	mutex_unlock(&v->state_lock);
+
+	list_for_each_entry_safe(b, tmp, &done, list) {
+		list_del_init(&b->list);
+		vb2_buffer_done(&b->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+	}
+}
+
+static void update_live_resolution(struct hws_pcie_dev *pdx, unsigned int ch,
+				   bool interlace)
+{
+	u32 reg = readl(pdx->bar0_base + HWS_REG_IN_RES(ch));
+	u32 fps = readl(pdx->bar0_base + HWS_REG_FRAME_RATE(ch));
+	u16 res_w = reg & 0xFFFF;
+	u16 res_h = (reg >> 16) & 0xFFFF;
+	struct hws_video *vid = &pdx->video[ch];
+	bool geometry_changed;
+	bool fps_changed;
+
+	bool within_hw = (res_w <= MAX_VIDEO_HW_W) &&
+	    ((!interlace && res_h <= MAX_VIDEO_HW_H) ||
+	     (interlace && (res_h * 2) <= MAX_VIDEO_HW_H));
+
+	if (!within_hw)
+		return;
+
+	geometry_changed = res_w != vid->pix.width ||
+		res_h != vid->pix.height ||
+		interlace != vid->pix.interlaced;
+	fps_changed = fps && fps != 0xFFFFFFFF && fps <= 240 &&
+		fps != vid->current_fps;
+
+	if (geometry_changed || fps_changed)
+		hws_video_apply_mode_change(pdx, ch, res_w, res_h, interlace,
+					    fps);
+}
+
+static int hws_open(struct file *file)
+{
+	return v4l2_fh_open(file);
+}
+
+static const struct v4l2_file_operations hws_fops = {
+	.owner = THIS_MODULE,
+	.open = hws_open,
+	.release = vb2_fop_release,
+	.poll = vb2_fop_poll,
+	.unlocked_ioctl = video_ioctl2,
+	.mmap = vb2_fop_mmap,
+};
+
+static int hws_subscribe_event(struct v4l2_fh *fh,
+			       const struct v4l2_event_subscription *sub)
+{
+	switch (sub->type) {
+	case V4L2_EVENT_SOURCE_CHANGE:
+		return v4l2_src_change_event_subscribe(fh, sub);
+	case V4L2_EVENT_CTRL:
+		return v4l2_ctrl_subscribe_event(fh, sub);
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct v4l2_ioctl_ops hws_ioctl_fops = {
+	/* Core caps/info */
+	.vidioc_querycap = hws_vidioc_querycap,
+
+	/* Pixel format: still needed to report YUYV etc. */
+	.vidioc_enum_fmt_vid_cap = hws_vidioc_enum_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap = hws_vidioc_g_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap = hws_vidioc_s_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap = hws_vidioc_try_fmt_vid_cap,
+
+	/* Buffer queueing / streaming */
+	.vidioc_reqbufs = vb2_ioctl_reqbufs,
+	.vidioc_prepare_buf = vb2_ioctl_prepare_buf,
+	.vidioc_create_bufs = vb2_ioctl_create_bufs,
+	.vidioc_querybuf = vb2_ioctl_querybuf,
+	.vidioc_qbuf = vb2_ioctl_qbuf,
+	.vidioc_dqbuf = vb2_ioctl_dqbuf,
+	.vidioc_expbuf = vb2_ioctl_expbuf,
+	.vidioc_streamon = vb2_ioctl_streamon,
+	.vidioc_streamoff = vb2_ioctl_streamoff,
+
+	/* Inputs */
+	.vidioc_enum_input = hws_vidioc_enum_input,
+	.vidioc_g_input = hws_vidioc_g_input,
+	.vidioc_s_input = hws_vidioc_s_input,
+
+	/* DV timings (HDMI/DVI/VESA modes) */
+	.vidioc_query_dv_timings = hws_vidioc_query_dv_timings,
+	.vidioc_enum_dv_timings = hws_vidioc_enum_dv_timings,
+	.vidioc_g_dv_timings = hws_vidioc_g_dv_timings,
+	.vidioc_s_dv_timings = hws_vidioc_s_dv_timings,
+	.vidioc_dv_timings_cap = hws_vidioc_dv_timings_cap,
+
+	.vidioc_log_status = v4l2_ctrl_log_status,
+	.vidioc_subscribe_event = hws_subscribe_event,
+	.vidioc_unsubscribe_event = v4l2_event_unsubscribe,
+	.vidioc_g_parm = hws_vidioc_g_parm,
+};
+
+static u32 hws_calc_sizeimage(struct hws_video *v, u16 w, u16 h,
+			      bool interlaced)
+{
+	/* HWS captures packed YUYV only; stride is 16 bpp aligned to 64 bytes. */
+	u32 lines = h;		/* full frame lines for sizeimage */
+	u32 bytesperline = ALIGN(w * 2, 64);
+	u32 sizeimage, half0;
+
+	/* publish into pix, since we now carry these in-state */
+	v->pix.bytesperline = bytesperline;
+	sizeimage = bytesperline * lines;
+
+	half0 = sizeimage / 2;
+
+	v->pix.sizeimage = sizeimage;
+	v->pix.half_size = half0;	/* first half; second = sizeimage - half0 */
+	v->pix.field = interlaced ? V4L2_FIELD_INTERLACED : V4L2_FIELD_NONE;
+
+	return v->pix.sizeimage;
+}
+
+static int hws_queue_setup(struct vb2_queue *q, unsigned int *num_buffers,
+			   unsigned int *nplanes, unsigned int sizes[],
+			   struct device *alloc_devs[])
+{
+	struct hws_video *vid = q->drv_priv;
+
+	if (*nplanes) {
+		if (sizes[0] < vid->pix.sizeimage)
+			return -EINVAL;
+	} else {
+		*nplanes = 1;
+		sizes[0] = vid->pix.sizeimage;
+	}
+
+	return 0;
+}
+
+static int hws_buffer_prepare(struct vb2_buffer *vb)
+{
+	struct hws_video *vid = vb->vb2_queue->drv_priv;
+	struct hws_pcie_dev *hws = vid->parent;
+	size_t need = vid->pix.sizeimage;
+	dma_addr_t dma_addr;
+
+	if (vb2_plane_size(vb, 0) < need)
+		return -EINVAL;
+
+	/* Validate DMA address alignment */
+	dma_addr = vb2_dma_contig_plane_dma_addr(vb, 0);
+	if (dma_addr & 0x3F) {	/* 64-byte alignment required */
+		dev_err(&hws->pdev->dev,
+			"Buffer DMA address 0x%llx not 64-byte aligned\n",
+			(unsigned long long)dma_addr);
+		return -EINVAL;
+	}
+
+	vb2_set_plane_payload(vb, 0, need);
+	return 0;
+}
+
+static void hws_buffer_queue(struct vb2_buffer *vb)
+{
+	struct hws_video *vid = vb->vb2_queue->drv_priv;
+	struct hwsvideo_buffer *buf = to_hwsbuf(vb);
+	struct hws_pcie_dev *hws = vid->parent;
+	unsigned long flags;
+
+	dev_dbg(&hws->pdev->dev,
+		"buffer_queue(ch=%u): vb=%p sizeimage=%u q_active=%d\n",
+		vid->channel_index, vb, vid->pix.sizeimage,
+		READ_ONCE(vid->cap_active));
+
+	/* Initialize buffer slot */
+	buf->slot = 0;
+
+	spin_lock_irqsave(&vid->irq_lock, flags);
+	list_add_tail(&buf->list, &vid->capture_queue);
+	vid->queued_count++;
+
+	/* If streaming and no in-flight buffer, prime HW immediately */
+	if (READ_ONCE(vid->cap_active) && !vid->active) {
+		dma_addr_t dma_addr;
+
+		dev_dbg(&hws->pdev->dev,
+			"buffer_queue(ch=%u): priming first vb=%p\n",
+			vid->channel_index, &buf->vb.vb2_buf);
+		list_del_init(&buf->list);
+		vid->queued_count--;
+		vid->active = buf;
+
+		dma_addr = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
+		hws_program_dma_for_addr(vid->parent, vid->channel_index,
+					 dma_addr);
+		iowrite32(lower_32_bits(dma_addr),
+			  hws->bar0_base + HWS_REG_DMA_ADDR(vid->channel_index));
+
+		wmb(); /* ensure descriptors visible before enabling capture */
+		hws_enable_video_capture(hws, vid->channel_index, true);
+		hws_prime_next_locked(vid);
+	} else if (READ_ONCE(vid->cap_active) && vid->active) {
+		hws_prime_next_locked(vid);
+	}
+	spin_unlock_irqrestore(&vid->irq_lock, flags);
+}
+
+static int hws_start_streaming(struct vb2_queue *q, unsigned int count)
+{
+	struct hws_video *v = q->drv_priv;
+	struct hws_pcie_dev *hws = v->parent;
+	struct hwsvideo_buffer *to_program = NULL;	/* local copy */
+	struct vb2_buffer *prog_vb2 = NULL;
+	unsigned long flags;
+	int ret;
+
+	dev_dbg(&hws->pdev->dev, "start_streaming: ch=%u count=%u\n",
+		v->channel_index, count);
+
+	ret = hws_check_card_status(hws);
+	if (ret) {
+		struct hwsvideo_buffer *b, *tmp;
+		unsigned long f;
+		LIST_HEAD(queued);
+
+		spin_lock_irqsave(&v->irq_lock, f);
+		if (v->active) {
+			list_add_tail(&v->active->list, &queued);
+			v->active = NULL;
+		}
+		if (v->next_prepared) {
+			list_add_tail(&v->next_prepared->list, &queued);
+			v->next_prepared = NULL;
+		}
+		while (!list_empty(&v->capture_queue)) {
+			b = list_first_entry(&v->capture_queue,
+					     struct hwsvideo_buffer, list);
+			list_move_tail(&b->list, &queued);
+		}
+		spin_unlock_irqrestore(&v->irq_lock, f);
+
+		list_for_each_entry_safe(b, tmp, &queued, list) {
+			list_del_init(&b->list);
+			vb2_buffer_done(&b->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
+		}
+		return ret;
+	}
+	(void)hws_read_active_state(hws, v->channel_index,
+				       &v->pix.interlaced);
+
+	lockdep_assert_held(&v->state_lock);
+	/* init per-stream state */
+	WRITE_ONCE(v->stop_requested, false);
+	WRITE_ONCE(v->cap_active, true);
+	WRITE_ONCE(v->half_seen, false);
+	WRITE_ONCE(v->last_buf_half_toggle, 0);
+
+	/* Try to prime a buffer, but it's OK if none are queued yet */
+	spin_lock_irqsave(&v->irq_lock, flags);
+	if (!v->active && !list_empty(&v->capture_queue)) {
+		to_program = list_first_entry(&v->capture_queue,
+					      struct hwsvideo_buffer, list);
+		list_del_init(&to_program->list);
+		v->queued_count--;
+		v->active = to_program;
+		prog_vb2 = &to_program->vb.vb2_buf;
+		dev_dbg(&hws->pdev->dev,
+			"start_streaming: ch=%u took buffer %p\n",
+			v->channel_index, to_program);
+	}
+	spin_unlock_irqrestore(&v->irq_lock, flags);
+
+	/* Only program/enable HW if we actually have a buffer */
+	if (to_program) {
+		if (!prog_vb2)
+			prog_vb2 = &to_program->vb.vb2_buf;
+		{
+			dma_addr_t dma_addr;
+
+			dma_addr = vb2_dma_contig_plane_dma_addr(prog_vb2, 0);
+			hws_program_dma_for_addr(hws, v->channel_index, dma_addr);
+			iowrite32(lower_32_bits(dma_addr),
+				  hws->bar0_base +
+				  HWS_REG_DMA_ADDR(v->channel_index));
+			dev_dbg(&hws->pdev->dev,
+				"start_streaming: ch=%u programmed buffer %p dma=0x%08x\n",
+				v->channel_index, to_program,
+				lower_32_bits(dma_addr));
+			(void)readl(hws->bar0_base + HWS_REG_INT_STATUS);
+		}
+
+		wmb(); /* ensure descriptors visible before enabling capture */
+		hws_enable_video_capture(hws, v->channel_index, true);
+		{
+			unsigned long pf;
+
+			spin_lock_irqsave(&v->irq_lock, pf);
+			hws_prime_next_locked(v);
+			spin_unlock_irqrestore(&v->irq_lock, pf);
+		}
+	} else {
+		dev_dbg(&hws->pdev->dev,
+			"start_streaming: ch=%u no buffer yet (will arm on QBUF)\n",
+			v->channel_index);
+	}
+
+	return 0;
+}
+
+static void hws_log_video_state(struct hws_video *v, const char *action,
+				const char *phase)
+{
+	struct hws_pcie_dev *hws = v->parent;
+	unsigned long flags;
+	unsigned int queued = 0;
+	unsigned int tracked = 0;
+	unsigned int seq = 0;
+	struct hwsvideo_buffer *b;
+	bool streaming = vb2_is_streaming(&v->buffer_queue);
+	bool cap_active;
+	bool stop_requested;
+	struct hwsvideo_buffer *active;
+	struct hwsvideo_buffer *next_prepared;
+
+	spin_lock_irqsave(&v->irq_lock, flags);
+	list_for_each_entry(b, &v->capture_queue, list)
+		queued++;
+	cap_active = READ_ONCE(v->cap_active);
+	stop_requested = READ_ONCE(v->stop_requested);
+	active = v->active;
+	next_prepared = v->next_prepared;
+	tracked = v->queued_count;
+	seq = (u32)atomic_read(&v->sequence_number);
+	spin_unlock_irqrestore(&v->irq_lock, flags);
+
+	dev_dbg(&hws->pdev->dev,
+		"video:%s:%s ch=%u streaming=%d cap=%d stop=%d active=%p next=%p queued=%u tracked=%u seq=%u\n",
+		action, phase, v->channel_index, streaming, cap_active,
+		stop_requested, active, next_prepared, queued, tracked, seq);
+}
+
+static void hws_stop_streaming(struct vb2_queue *q)
+{
+	struct hws_video *v = q->drv_priv;
+	struct hws_pcie_dev *hws = v->parent;
+	unsigned long flags;
+	struct hwsvideo_buffer *b, *tmp;
+	LIST_HEAD(done);
+	unsigned int done_cnt = 0;
+	u64 start_ns = ktime_get_mono_fast_ns();
+
+	hws_log_video_state(v, "streamoff", "begin");
+
+	/* 1) Quiesce SW/HW first */
+	lockdep_assert_held(&v->state_lock);
+	WRITE_ONCE(v->cap_active, false);
+	WRITE_ONCE(v->stop_requested, true);
+
+	hws_enable_video_capture(v->parent, v->channel_index, false);
+
+	/* 2) Collect in-flight + queued under the IRQ lock */
+	spin_lock_irqsave(&v->irq_lock, flags);
+	hws_video_collect_done_locked(v, &done);
+	spin_unlock_irqrestore(&v->irq_lock, flags);
+
+	/* 3) Complete outside the lock */
+	list_for_each_entry_safe(b, tmp, &done, list) {
+		/* Unlink from 'done' before completing */
+		list_del_init(&b->list);
+		vb2_buffer_done(&b->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+		done_cnt++;
+	}
+	dev_dbg(&hws->pdev->dev,
+		"video:streamoff:done ch=%u completed=%u (%lluus)\n",
+		v->channel_index, done_cnt, hws_elapsed_us(start_ns));
+	hws_log_video_state(v, "streamoff", "end");
+}
+
+static const struct vb2_ops hwspcie_video_qops = {
+	.queue_setup = hws_queue_setup,
+	.buf_prepare = hws_buffer_prepare,
+	.buf_init = hws_buf_init,
+	.buf_finish = hws_buf_finish,
+	.buf_cleanup = hws_buf_cleanup,
+	.buf_queue = hws_buffer_queue,
+	.start_streaming = hws_start_streaming,
+	.stop_streaming = hws_stop_streaming,
+};
+
+int hws_video_register(struct hws_pcie_dev *dev)
+{
+	int i, ret;
+
+	ret = v4l2_device_register(&dev->pdev->dev, &dev->v4l2_device);
+	if (ret) {
+		dev_err(&dev->pdev->dev, "v4l2_device_register failed: %d\n",
+			ret);
+		return ret;
+	}
+
+	for (i = 0; i < dev->cur_max_video_ch; i++) {
+		struct hws_video *ch = &dev->video[i];
+		struct video_device *vdev;
+		struct vb2_queue *q;
+
+		/* hws_video_init_channel() should have set:
+		 * - ch->parent, ch->channel_index
+		 * - locks (state_lock, irq_lock)
+		 * - capture_queue (INIT_LIST_HEAD)
+		 * - control_handler + controls
+		 * - fmt_curr (width/height)
+		 * Do not reinitialize any of those here.
+		 */
+
+		vdev = video_device_alloc();
+		if (!vdev) {
+			dev_err(&dev->pdev->dev,
+				"video_device_alloc ch%u failed\n", i);
+			ret = -ENOMEM;
+			goto err_unwind;
+		}
+		ch->video_device = vdev;
+
+		/* Basic V4L2 node setup */
+		snprintf(vdev->name, sizeof(vdev->name), "%s-hdmi%u",
+			 KBUILD_MODNAME, i);
+		vdev->v4l2_dev = &dev->v4l2_device;
+		vdev->fops = &hws_fops;
+		vdev->ioctl_ops = &hws_ioctl_fops;
+		vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING;
+		vdev->lock = &ch->state_lock;	/* serialize file ops */
+		vdev->ctrl_handler = &ch->control_handler;
+		vdev->vfl_dir = VFL_DIR_RX;
+		vdev->release = video_device_release;
+		if (ch->control_handler.error) {
+			ret = ch->control_handler.error;
+			goto err_unwind;
+		}
+		video_set_drvdata(vdev, ch);
+
+		/* vb2 queue init (dma-contig) */
+		q = &ch->buffer_queue;
+		memset(q, 0, sizeof(*q));
+		q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+		q->io_modes = VB2_MMAP | VB2_DMABUF;
+		q->drv_priv = ch;
+		q->buf_struct_size = sizeof(struct hwsvideo_buffer);
+		q->ops = &hwspcie_video_qops;
+		q->mem_ops = &vb2_dma_contig_memops;
+		q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+		q->lock = &ch->state_lock;
+		q->min_queued_buffers = 1;
+		q->dev = &dev->pdev->dev;
+
+		ret = vb2_queue_init(q);
+		vdev->queue = q;
+		if (ret) {
+			dev_err(&dev->pdev->dev,
+				"vb2_queue_init ch%u failed: %d\n", i, ret);
+			goto err_unwind;
+		}
+
+		/* Make controls live (no-op if none or already set up) */
+		if (ch->control_handler.error) {
+			ret = ch->control_handler.error;
+			dev_err(&dev->pdev->dev,
+				"ctrl handler ch%u error: %d\n", i, ret);
+			goto err_unwind;
+		}
+		v4l2_ctrl_handler_setup(&ch->control_handler);
+		ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
+		if (ret) {
+			dev_err(&dev->pdev->dev,
+				"video_register_device ch%u failed: %d\n", i,
+				ret);
+			goto err_unwind;
+		}
+	}
+
+	return 0;
+
+err_unwind:
+	for (; i >= 0; i--) {
+		struct hws_video *ch = &dev->video[i];
+
+		hws_video_release_registration(ch);
+	}
+	v4l2_device_unregister(&dev->v4l2_device);
+	return ret;
+}
+
+void hws_video_unregister(struct hws_pcie_dev *dev)
+{
+	int i;
+
+	if (!dev)
+		return;
+
+	for (i = 0; i < dev->cur_max_video_ch; i++) {
+		struct hws_video *ch = &dev->video[i];
+
+		hws_video_release_registration(ch);
+		v4l2_ctrl_handler_free(&ch->control_handler);
+	}
+	v4l2_device_unregister(&dev->v4l2_device);
+}
+
+int hws_video_quiesce(struct hws_pcie_dev *hws, const char *reason)
+{
+	int i, ret = 0;
+	u64 start_ns = ktime_get_mono_fast_ns();
+
+	dev_dbg(&hws->pdev->dev, "video:%s:begin channels=%u\n", reason,
+		hws->cur_max_video_ch);
+	for (i = 0; i < hws->cur_max_video_ch; i++) {
+		struct hws_video *vid = &hws->video[i];
+		struct vb2_queue *q = &vid->buffer_queue;
+		u64 ch_start_ns = ktime_get_mono_fast_ns();
+		bool streaming;
+
+		if (!q || !q->ops) {
+			dev_dbg(&hws->pdev->dev,
+				"video:%s:ch=%d skipped queue-unavailable\n",
+				reason, i);
+			continue;
+		}
+
+		streaming = vb2_is_streaming(q);
+		hws_log_video_state(vid, reason, "channel");
+		if (streaming) {
+			/* Stop via vb2, which runs .stop_streaming. */
+			int r = vb2_streamoff(q, q->type);
+
+			dev_dbg(&hws->pdev->dev,
+				"video:%s:ch=%d streamoff ret=%d (%lluus)\n",
+				reason, i, r, hws_elapsed_us(ch_start_ns));
+			if (r && !ret)
+				ret = r;
+		} else {
+			dev_dbg(&hws->pdev->dev,
+				"video:%s:ch=%d idle (%lluus)\n",
+				reason, i, hws_elapsed_us(ch_start_ns));
+		}
+	}
+	dev_dbg(&hws->pdev->dev, "video:%s:done ret=%d (%lluus)\n", reason,
+		ret, hws_elapsed_us(start_ns));
+	return ret;
+}
+
+void hws_video_pm_resume(struct hws_pcie_dev *hws)
+{
+	/* Nothing mandatory to do here for vb2; userspace will STREAMON
+	 * again when ready.
+	 */
+}
diff --git a/drivers/media/pci/hws/hws_video.h b/drivers/media/pci/hws/hws_video.h
new file mode 100644
index 0000000000000..4feaf5b2f5a9e
--- /dev/null
+++ b/drivers/media/pci/hws/hws_video.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef HWS_VIDEO_H
+#define HWS_VIDEO_H
+
+struct hws_video;
+
+int hws_video_register(struct hws_pcie_dev *dev);
+void hws_video_unregister(struct hws_pcie_dev *dev);
+void hws_enable_video_capture(struct hws_pcie_dev *hws,
+			      unsigned int chan,
+			      bool on);
+void hws_prime_next_locked(struct hws_video *vid);
+
+int hws_video_init_channel(struct hws_pcie_dev *pdev, int ch);
+void hws_video_cleanup_channel(struct hws_pcie_dev *pdev, int ch);
+void check_video_format(struct hws_pcie_dev *pdx);
+int hws_check_card_status(struct hws_pcie_dev *hws);
+void hws_init_video_sys(struct hws_pcie_dev *hws, bool enable);
+
+void hws_program_dma_for_addr(struct hws_pcie_dev *hws,
+			      unsigned int ch,
+			      dma_addr_t dma);
+void hws_set_dma_doorbell(struct hws_pcie_dev *hws, unsigned int ch,
+			  dma_addr_t dma, const char *tag);
+
+int hws_video_quiesce(struct hws_pcie_dev *hws, const char *reason);
+void hws_video_pm_resume(struct hws_pcie_dev *hws);
+
+#endif
diff --git a/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c b/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c
index db28748434530..237906cb131ac 100644
--- a/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c
+++ b/drivers/media/pci/intel/ipu6/ipu6-isys-dwc-phy.c
@@ -288,15 +288,27 @@ static const struct dwc_dphy_freq_range freqranges[DPHY_FREQ_RANGE_NUM] = {
 
 static u16 get_hsfreq_by_mbps(u32 mbps)
 {
-	unsigned int i = DPHY_FREQ_RANGE_NUM;
-
-	while (i--) {
-		if (freqranges[i].default_mbps == mbps ||
-		    (mbps >= freqranges[i].min && mbps <= freqranges[i].max))
-			return i;
+	u16 best = DPHY_FREQ_RANGE_INVALID_INDEX;
+	unsigned int i;
+
+	for (i = 0; i < DPHY_FREQ_RANGE_NUM; i++) {
+		if (mbps > freqranges[i].max)
+			continue;
+
+		if (mbps < freqranges[i].min)
+			break;
+
+		if (best == DPHY_FREQ_RANGE_INVALID_INDEX ||
+		    freqranges[i].osc_freq_target >
+		    freqranges[best].osc_freq_target ||
+		    (freqranges[i].osc_freq_target ==
+		     freqranges[best].osc_freq_target &&
+		     abs((int)mbps - (int)freqranges[i].default_mbps) <
+		     abs((int)mbps - (int)freqranges[best].default_mbps)))
+			best = i;
 	}
 
-	return DPHY_FREQ_RANGE_INVALID_INDEX;
+	return best;
 }
 
 static int ipu6_isys_dwc_phy_config(struct ipu6_isys *isys,
diff --git a/drivers/media/pci/ivtv/ivtv-ioctl.c b/drivers/media/pci/ivtv/ivtv-ioctl.c
index 8d5ea3aec06f3..fc95f0bf48d55 100644
--- a/drivers/media/pci/ivtv/ivtv-ioctl.c
+++ b/drivers/media/pci/ivtv/ivtv-ioctl.c
@@ -467,15 +467,13 @@ static int ivtv_try_fmt_vid_cap(struct file *file, void *fh, struct v4l2_format
 	int h = fmt->fmt.pix.height;
 	int min_h = 2;
 
-	w = min(w, 720);
-	w = max(w, 2);
+	w = clamp(w, 2, 720);
 	if (id->type == IVTV_ENC_STREAM_TYPE_YUV) {
 		/* YUV height must be a multiple of 32 */
 		h &= ~0x1f;
 		min_h = 32;
 	}
-	h = min(h, itv->is_50hz ? 576 : 480);
-	h = max(h, min_h);
+	h = clamp(h, min_h, itv->is_50hz ? 576 : 480);
 	ivtv_g_fmt_vid_cap(file, fh, fmt);
 	fmt->fmt.pix.width = w;
 	fmt->fmt.pix.height = h;
@@ -516,8 +514,7 @@ static int ivtv_try_fmt_vid_out(struct file *file, void *fh, struct v4l2_format
 	int field = fmt->fmt.pix.field;
 	int ret = ivtv_g_fmt_vid_out(file, fh, fmt);
 
-	w = min(w, 720);
-	w = max(w, 2);
+	w = clamp(w, 2, 720);
 	/* Why can the height be 576 even when the output is NTSC?
 
 	   Internally the buffers of the PVR350 are always set to 720x576. The
@@ -533,8 +530,7 @@ static int ivtv_try_fmt_vid_out(struct file *file, void *fh, struct v4l2_format
 	   resolution is locked to the broadcast standard and not scaled.
 
 	   Thanks to Ian Armstrong for this explanation. */
-	h = min(h, 576);
-	h = max(h, 2);
+	h = clamp(h, 2, 576);
 	if (id->type == IVTV_DEC_STREAM_TYPE_YUV)
 		fmt->fmt.pix.field = field;
 	fmt->fmt.pix.width = w;
diff --git a/drivers/media/pci/mgb4/mgb4_vin.c b/drivers/media/pci/mgb4/mgb4_vin.c
index 74fb00c4a4082..b247854556919 100644
--- a/drivers/media/pci/mgb4/mgb4_vin.c
+++ b/drivers/media/pci/mgb4/mgb4_vin.c
@@ -85,12 +85,12 @@ static const struct mgb4_i2c_kv gmsl1_i2c[] = {
 static const struct v4l2_dv_timings_cap video_timings_cap = {
 	.type = V4L2_DV_BT_656_1120,
 	.bt = {
-		.min_width = 240,
+		.min_width = 64,
 		.max_width = 4096,
-		.min_height = 240,
+		.min_height = 64,
 		.max_height = 4096,
-		.min_pixelclock = 1843200, /* 320 x 240 x 24Hz */
-		.max_pixelclock = 530841600, /* 4096 x 2160 x 60Hz */
+		.min_pixelclock = 20000000,
+		.max_pixelclock = 200000000,
 		.standards = V4L2_DV_BT_STD_CEA861 | V4L2_DV_BT_STD_DMT |
 			V4L2_DV_BT_STD_CVT | V4L2_DV_BT_STD_GTF,
 		.capabilities = V4L2_DV_BT_CAP_PROGRESSIVE |
diff --git a/drivers/media/pci/mgb4/mgb4_vout.c b/drivers/media/pci/mgb4/mgb4_vout.c
index 7725bcd55e4cb..22fbc9e538193 100644
--- a/drivers/media/pci/mgb4/mgb4_vout.c
+++ b/drivers/media/pci/mgb4/mgb4_vout.c
@@ -53,12 +53,12 @@ static const struct mgb4_i2c_kv gmsl1_i2c[] = {
 static const struct v4l2_dv_timings_cap video_timings_cap = {
 	.type = V4L2_DV_BT_656_1120,
 	.bt = {
-		.min_width = 240,
+		.min_width = 64,
 		.max_width = 4096,
-		.min_height = 240,
+		.min_height = 64,
 		.max_height = 4096,
-		.min_pixelclock = 1843200, /* 320 x 240 x 24Hz */
-		.max_pixelclock = 530841600, /* 4096 x 2160 x 60Hz */
+		.min_pixelclock = 25000000,
+		.max_pixelclock = 189284000,
 		.standards = V4L2_DV_BT_STD_CEA861 | V4L2_DV_BT_STD_DMT |
 			V4L2_DV_BT_STD_CVT | V4L2_DV_BT_STD_GTF,
 		.capabilities = V4L2_DV_BT_CAP_PROGRESSIVE |
diff --git a/drivers/media/pci/saa7134/saa7134-video.c b/drivers/media/pci/saa7134/saa7134-video.c
index 4a51b873e47ad..2b1672737d841 100644
--- a/drivers/media/pci/saa7134/saa7134-video.c
+++ b/drivers/media/pci/saa7134/saa7134-video.c
@@ -1714,8 +1714,10 @@ int saa7134_video_init1(struct saa7134_dev *dev)
 	q->dev = &dev->pci->dev;
 	ret = vb2_queue_init(q);
 	if (ret)
-		return ret;
-	saa7134_pgtable_alloc(dev->pci, &dev->video_q.pt);
+		goto err_free_ctrl;
+	ret = saa7134_pgtable_alloc(dev->pci, &dev->video_q.pt);
+	if (ret)
+		goto err_free_ctrl;
 
 	q = &dev->vbi_vbq;
 	q->type = V4L2_BUF_TYPE_VBI_CAPTURE;
@@ -1732,11 +1734,24 @@ int saa7134_video_init1(struct saa7134_dev *dev)
 	q->lock = &dev->lock;
 	q->dev = &dev->pci->dev;
 	ret = vb2_queue_init(q);
-	if (ret)
-		return ret;
-	saa7134_pgtable_alloc(dev->pci, &dev->vbi_q.pt);
+	if (ret) {
+		saa7134_pgtable_free(dev->pci, &dev->video_q.pt);
+		goto err_free_ctrl;
+	}
+
+	ret = saa7134_pgtable_alloc(dev->pci, &dev->vbi_q.pt);
+	if (ret) {
+		saa7134_pgtable_free(dev->pci, &dev->video_q.pt);
+		goto err_free_ctrl;
+	}
 
 	return 0;
+
+err_free_ctrl:
+	v4l2_ctrl_handler_free(&dev->ctrl_handler);
+	if (card_has_radio(dev))
+		v4l2_ctrl_handler_free(&dev->radio_ctrl_handler);
+	return ret;
 }
 
 void saa7134_video_fini(struct saa7134_dev *dev)
diff --git a/drivers/media/pci/solo6x10/Kconfig b/drivers/media/pci/solo6x10/Kconfig
index adb247847e004..cc4a7b0883708 100644
--- a/drivers/media/pci/solo6x10/Kconfig
+++ b/drivers/media/pci/solo6x10/Kconfig
@@ -8,7 +8,6 @@ config VIDEO_SOLO6X10
 	select VIDEOBUF2_DMA_SG
 	select VIDEOBUF2_DMA_CONTIG
 	select SND_PCM
-	select FONT_8x16
 	help
 	  This driver supports the Bluecherry H.264 and MPEG-4 hardware
 	  compression capture cards and other Softlogic-based ones.
diff --git a/drivers/media/platform/Kconfig b/drivers/media/platform/Kconfig
index 3f0b7bb68cc94..0b33e927bd59c 100644
--- a/drivers/media/platform/Kconfig
+++ b/drivers/media/platform/Kconfig
@@ -63,6 +63,7 @@ config VIDEO_MUX
 
 # Platform drivers - Please keep it alphabetically sorted
 source "drivers/media/platform/allegro-dvt/Kconfig"
+source "drivers/media/platform/amd/Kconfig"
 source "drivers/media/platform/amlogic/Kconfig"
 source "drivers/media/platform/amphion/Kconfig"
 source "drivers/media/platform/arm/Kconfig"
diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile
index 6d5f79ddfcc37..16c185752474e 100644
--- a/drivers/media/platform/Makefile
+++ b/drivers/media/platform/Makefile
@@ -6,6 +6,7 @@
 # Place here, alphabetically sorted by directory
 # (e. g. LC_ALL=C sort Makefile)
 obj-y += allegro-dvt/
+obj-y += amd/
 obj-y += amlogic/
 obj-y += amphion/
 obj-y += arm/
diff --git a/drivers/media/platform/amd/Kconfig b/drivers/media/platform/amd/Kconfig
new file mode 100644
index 0000000000000..25af49f246b26
--- /dev/null
+++ b/drivers/media/platform/amd/Kconfig
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+source "drivers/media/platform/amd/isp4/Kconfig"
diff --git a/drivers/media/platform/amd/Makefile b/drivers/media/platform/amd/Makefile
new file mode 100644
index 0000000000000..8bfc1955f22e6
--- /dev/null
+++ b/drivers/media/platform/amd/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+obj-y += isp4/
diff --git a/drivers/media/platform/amd/isp4/Kconfig b/drivers/media/platform/amd/isp4/Kconfig
new file mode 100644
index 0000000000000..9d1927af1cb8c
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/Kconfig
@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: GPL-2.0+
+
+config VIDEO_AMD_ISP4_CAPTURE
+	tristate "AMD ISP4 and camera driver"
+	depends on DRM_AMDGPU && DRM_AMD_ISP
+	depends on HAS_DMA
+	depends on VIDEO_DEV
+	select VIDEOBUF2_CORE
+	select VIDEOBUF2_MEMOPS
+	select VIDEOBUF2_V4L2
+	select VIDEOBUF2_VMALLOC
+	select VIDEO_V4L2_SUBDEV_API
+	help
+	  This is support for AMD ISP4 and camera subsystem driver.
+	  Say Y here to enable the ISP4 and camera device for video capture.
+	  To compile this driver as a module, choose M here. The module will
+	  be called amd_isp4_capture.
diff --git a/drivers/media/platform/amd/isp4/Makefile b/drivers/media/platform/amd/isp4/Makefile
new file mode 100644
index 0000000000000..3849062e17f3d
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/Makefile
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0+
+#
+# Copyright (C) 2025 Advanced Micro Devices, Inc.
+
+obj-$(CONFIG_VIDEO_AMD_ISP4_CAPTURE) += amd_isp4_capture.o
+amd_isp4_capture-objs := isp4.o \
+                         isp4_debug.o \
+                         isp4_interface.o \
+                         isp4_subdev.o \
+                         isp4_video.o
diff --git a/drivers/media/platform/amd/isp4/isp4.c b/drivers/media/platform/amd/isp4/isp4.c
new file mode 100644
index 0000000000000..bf6b8e26c2c0f
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4.c
@@ -0,0 +1,240 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#include <linux/irq.h>
+#include <linux/pm_runtime.h>
+#include <linux/vmalloc.h>
+#include <media/v4l2-ioctl.h>
+
+#include "isp4.h"
+#include "isp4_debug.h"
+#include "isp4_hw_reg.h"
+
+#define ISP4_DRV_NAME "amd_isp_capture"
+#define ISP4_FW_RESP_RB_IRQ_STATUS_MASK \
+	(ISP_SYS_INT0_STATUS__SYS_INT_RINGBUFFER_WPT9_INT_MASK  | \
+	 ISP_SYS_INT0_STATUS__SYS_INT_RINGBUFFER_WPT12_INT_MASK)
+
+static const struct {
+	const char *name;
+	u32 status_mask;
+	u32 en_mask;
+	u32 ack_mask;
+	u32 rb_int_num;
+} isp4_irq[ISP4SD_MAX_FW_RESP_STREAM_NUM] = {
+	/* The IRQ order is aligned with the isp4_subdev.fw_resp_thread order */
+	{
+		.name = "isp_irq_global",
+		.status_mask =
+		ISP_SYS_INT0_STATUS__SYS_INT_RINGBUFFER_WPT12_INT_MASK,
+		.en_mask = ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT12_EN_MASK,
+		.ack_mask = ISP_SYS_INT0_ACK__SYS_INT_RINGBUFFER_WPT12_ACK_MASK,
+		.rb_int_num = 4, /* ISP_4_1__SRCID__ISP_RINGBUFFER_WPT12 */
+	},
+	{
+		.name = "isp_irq_stream1",
+		.status_mask =
+		ISP_SYS_INT0_STATUS__SYS_INT_RINGBUFFER_WPT9_INT_MASK,
+		.en_mask = ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT9_EN_MASK,
+		.ack_mask = ISP_SYS_INT0_ACK__SYS_INT_RINGBUFFER_WPT9_ACK_MASK,
+		.rb_int_num = 0, /* ISP_4_1__SRCID__ISP_RINGBUFFER_WPT9 */
+	},
+};
+
+void isp4_intr_enable(struct isp4_subdev *isp_subdev, u32 index, bool enable)
+{
+	u32 intr_en;
+
+	/* Synchronize ISP_SYS_INT0_EN writes with the IRQ handler's writes */
+	spin_lock_irq(&isp_subdev->irq_lock);
+	intr_en = isp4hw_rreg(isp_subdev->mmio, ISP_SYS_INT0_EN);
+	if (enable)
+		intr_en |= isp4_irq[index].en_mask;
+	else
+		intr_en &= ~isp4_irq[index].en_mask;
+
+	isp4hw_wreg(isp_subdev->mmio, ISP_SYS_INT0_EN, intr_en);
+	spin_unlock_irq(&isp_subdev->irq_lock);
+}
+
+static void isp4_wake_up_resp_thread(struct isp4_subdev *isp_subdev, u32 index)
+{
+	struct isp4sd_thread_handler *thread_ctx =
+			&isp_subdev->fw_resp_thread[index];
+
+	thread_ctx->resp_ready = true;
+	wake_up_interruptible(&thread_ctx->waitq);
+}
+
+static irqreturn_t isp4_irq_handler(int irq, void *arg)
+{
+	struct isp4_subdev *isp_subdev = arg;
+	u32 intr_ack = 0, intr_en = 0, intr_status;
+	int seen = 0;
+
+	/* Get the ISP_SYS interrupt status */
+	intr_status = isp4hw_rreg(isp_subdev->mmio, ISP_SYS_INT0_STATUS);
+	intr_status &= ISP4_FW_RESP_RB_IRQ_STATUS_MASK;
+
+	/* Find which ISP_SYS interrupts fired */
+	for (size_t i = 0; i < ARRAY_SIZE(isp4_irq); i++) {
+		if (intr_status & isp4_irq[i].status_mask) {
+			intr_ack |= isp4_irq[i].ack_mask;
+			intr_en |= isp4_irq[i].en_mask;
+			seen |= BIT(i);
+		}
+	}
+
+	/*
+	 * Disable the ISP_SYS interrupts that fired. Must be done before waking
+	 * the response threads, since they re-enable interrupts when finished.
+	 * The lock synchronizes RMW of INT0_EN with isp4_enable_interrupt().
+	 */
+	spin_lock(&isp_subdev->irq_lock);
+	intr_en = isp4hw_rreg(isp_subdev->mmio, ISP_SYS_INT0_EN) & ~intr_en;
+	isp4hw_wreg(isp_subdev->mmio, ISP_SYS_INT0_EN, intr_en);
+	spin_unlock(&isp_subdev->irq_lock);
+
+	/*
+	 * Clear the ISP_SYS interrupts. This must be done after the interrupts
+	 * are disabled, so that ISP FW won't flag any new interrupts on these
+	 * streams, and thus we don't need to clear interrupts again before
+	 * re-enabling them in the response thread.
+	 */
+	isp4hw_wreg(isp_subdev->mmio, ISP_SYS_INT0_ACK, intr_ack);
+
+	/*
+	 * The operation `(seen >> i) << i` is logically equivalent to
+	 * `seen &= ~BIT(i)`, with fewer instructions after compilation.
+	 */
+	for (int i; (i = ffs(seen)); seen = (seen >> i) << i)
+		isp4_wake_up_resp_thread(isp_subdev, i - 1);
+
+	return IRQ_HANDLED;
+}
+
+static int isp4_capture_probe(struct platform_device *pdev)
+{
+	int irq[ISP4SD_MAX_FW_RESP_STREAM_NUM];
+	struct device *dev = &pdev->dev;
+	struct isp4_subdev *isp_subdev;
+	struct isp4_device *isp_dev;
+	int ret;
+
+	isp_dev = devm_kzalloc(dev, sizeof(*isp_dev), GFP_KERNEL);
+	if (!isp_dev)
+		return -ENOMEM;
+
+	dev->init_name = ISP4_DRV_NAME;
+
+	isp_subdev = &isp_dev->isp_subdev;
+	isp_subdev->mmio = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(isp_subdev->mmio))
+		return dev_err_probe(dev, PTR_ERR(isp_subdev->mmio),
+				     "isp ioremap fail\n");
+
+	for (size_t i = 0; i < ARRAY_SIZE(isp4_irq); i++) {
+		irq[i] = platform_get_irq(pdev, isp4_irq[i].rb_int_num);
+		if (irq[i] < 0)
+			return dev_err_probe(dev, irq[i],
+					     "fail to get irq %d\n",
+					     isp4_irq[i].rb_int_num);
+
+		ret = devm_request_irq(dev, irq[i], isp4_irq_handler,
+				       IRQF_NO_AUTOEN, isp4_irq[i].name,
+				       isp_subdev);
+		if (ret)
+			return dev_err_probe(dev, ret, "fail to req irq %d\n",
+					     irq[i]);
+	}
+
+	isp_dev->v4l2_dev.mdev = &isp_dev->mdev;
+
+	strscpy(isp_dev->mdev.model, "amd_isp41_mdev",
+		sizeof(isp_dev->mdev.model));
+	isp_dev->mdev.dev = dev;
+	media_device_init(&isp_dev->mdev);
+
+	snprintf(isp_dev->v4l2_dev.name, sizeof(isp_dev->v4l2_dev.name),
+		 "AMD-V4L2-ROOT");
+	ret = v4l2_device_register(dev, &isp_dev->v4l2_dev);
+	if (ret) {
+		dev_err_probe(dev, ret, "fail register v4l2 device\n");
+		goto err_clean_media;
+	}
+
+	pm_runtime_set_suspended(dev);
+	pm_runtime_enable(dev);
+	spin_lock_init(&isp_subdev->irq_lock);
+	ret = isp4sd_init(&isp_dev->isp_subdev, &isp_dev->v4l2_dev, irq);
+	if (ret) {
+		dev_err_probe(dev, ret, "fail init isp4 sub dev\n");
+		goto err_pm_disable;
+	}
+
+	ret = media_create_pad_link(&isp_dev->isp_subdev.sdev.entity,
+				    0,
+				    &isp_dev->isp_subdev.isp_vdev.vdev.entity,
+				    0,
+				    MEDIA_LNK_FL_ENABLED |
+				    MEDIA_LNK_FL_IMMUTABLE);
+	if (ret) {
+		dev_err_probe(dev, ret, "fail to create pad link\n");
+		goto err_isp4_deinit;
+	}
+
+	ret = media_device_register(&isp_dev->mdev);
+	if (ret) {
+		dev_err_probe(dev, ret, "fail to register media device\n");
+		goto err_isp4_deinit;
+	}
+
+	platform_set_drvdata(pdev, isp_dev);
+	isp_debugfs_create(isp_dev);
+
+	return 0;
+
+err_isp4_deinit:
+	isp4sd_deinit(&isp_dev->isp_subdev);
+err_pm_disable:
+	pm_runtime_disable(dev);
+	v4l2_device_unregister(&isp_dev->v4l2_dev);
+err_clean_media:
+	media_device_cleanup(&isp_dev->mdev);
+
+	return ret;
+}
+
+static void isp4_capture_remove(struct platform_device *pdev)
+{
+	struct isp4_device *isp_dev = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+
+	isp_debugfs_remove(isp_dev);
+
+	media_device_unregister(&isp_dev->mdev);
+	isp4sd_deinit(&isp_dev->isp_subdev);
+	pm_runtime_disable(dev);
+	v4l2_device_unregister(&isp_dev->v4l2_dev);
+	media_device_cleanup(&isp_dev->mdev);
+}
+
+static struct platform_driver isp4_capture_drv = {
+	.probe = isp4_capture_probe,
+	.remove = isp4_capture_remove,
+	.driver = {
+		.name = ISP4_DRV_NAME,
+	}
+};
+
+module_platform_driver(isp4_capture_drv);
+
+MODULE_ALIAS("platform:" ISP4_DRV_NAME);
+MODULE_IMPORT_NS("DMA_BUF");
+
+MODULE_DESCRIPTION("AMD ISP4 Driver");
+MODULE_AUTHOR("Bin Du <bin.du@amd.com>");
+MODULE_AUTHOR("Pratap Nirujogi <pratap.nirujogi@amd.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/media/platform/amd/isp4/isp4.h b/drivers/media/platform/amd/isp4/isp4.h
new file mode 100644
index 0000000000000..2db6683d6d8b5
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4.h
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _ISP4_H_
+#define _ISP4_H_
+
+#include <drm/amd/isp.h>
+#include "isp4_subdev.h"
+
+struct isp4_device {
+	struct v4l2_device v4l2_dev;
+	struct isp4_subdev isp_subdev;
+	struct media_device mdev;
+};
+
+void isp4_intr_enable(struct isp4_subdev *isp_subdev, u32 index, bool enable);
+
+#endif /* _ISP4_H_ */
diff --git a/drivers/media/platform/amd/isp4/isp4_debug.c b/drivers/media/platform/amd/isp4/isp4_debug.c
new file mode 100644
index 0000000000000..2fc00fc9a1940
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_debug.c
@@ -0,0 +1,271 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#include "isp4.h"
+#include "isp4_debug.h"
+#include "isp4_hw_reg.h"
+#include "isp4_interface.h"
+
+#define ISP4DBG_FW_LOG_RINGBUF_SIZE (2 * 1024 * 1024)
+#define ISP4DBG_MACRO_2_STR(X) #X
+#define ISP4DBG_ONE_TIME_LOG_LEN 510
+
+#ifdef CONFIG_DEBUG_FS
+
+void isp_debugfs_create(struct isp4_device *isp_dev)
+{
+	isp_dev->isp_subdev.debugfs_dir = debugfs_create_dir("amd_isp4", NULL);
+	debugfs_create_bool("fw_log_enable", 0644,
+			    isp_dev->isp_subdev.debugfs_dir,
+			    &isp_dev->isp_subdev.enable_fw_log);
+	isp_dev->isp_subdev.fw_log_output =
+		devm_kzalloc(isp_dev->isp_subdev.dev,
+			     ISP4DBG_FW_LOG_RINGBUF_SIZE + 32,
+			     GFP_KERNEL);
+}
+
+void isp_debugfs_remove(struct isp4_device *isp_dev)
+{
+	debugfs_remove_recursive(isp_dev->isp_subdev.debugfs_dir);
+	isp_dev->isp_subdev.debugfs_dir = NULL;
+}
+
+static u32 isp_fw_fill_rb_log(struct isp4_subdev *isp, void *sys, u32 rb_size)
+{
+	struct isp4_interface *ispif = &isp->ispif;
+	char *buf = isp->fw_log_output;
+	struct device *dev = isp->dev;
+	u32 rd_ptr, wr_ptr;
+	u32 total_cnt = 0;
+	u32 offset = 0;
+	u32 cnt;
+
+	if (!sys || !rb_size)
+		return 0;
+
+	guard(mutex)(&ispif->isp4if_mutex);
+
+	rd_ptr = isp4hw_rreg(isp->mmio, ISP_LOG_RB_RPTR0);
+	wr_ptr = isp4hw_rreg(isp->mmio, ISP_LOG_RB_WPTR0);
+
+	do {
+		if (wr_ptr > rd_ptr)
+			cnt = wr_ptr - rd_ptr;
+		else if (wr_ptr < rd_ptr)
+			cnt = rb_size - rd_ptr;
+		else
+			goto quit;
+
+		if (cnt > rb_size) {
+			dev_err(dev, "fail bad fw log size %u\n", cnt);
+			goto quit;
+		}
+
+		memcpy(buf + offset, sys + rd_ptr, cnt);
+
+		offset += cnt;
+		total_cnt += cnt;
+		rd_ptr = (rd_ptr + cnt) % rb_size;
+	} while (rd_ptr < wr_ptr);
+
+	isp4hw_wreg(isp->mmio, ISP_LOG_RB_RPTR0, rd_ptr);
+
+quit:
+	return total_cnt;
+}
+
+void isp_fw_log_print(struct isp4_subdev *isp)
+{
+	struct isp4_interface *ispif = &isp->ispif;
+	char *fw_log_buf = isp->fw_log_output;
+	u32 cnt;
+
+	if (!isp->enable_fw_log || !fw_log_buf)
+		return;
+
+	cnt = isp_fw_fill_rb_log(isp, ispif->fw_log_buf->sys_addr,
+				 ispif->fw_log_buf->mem_size);
+
+	if (cnt) {
+		char temp_ch;
+		char *str;
+		char *end;
+		/* line end */
+		char *le;
+
+		str = (char *)fw_log_buf;
+		end = ((char *)fw_log_buf + cnt);
+		fw_log_buf[cnt] = 0;
+
+		while (str < end) {
+			le = strchr(str, 0x0A);
+			if ((le && str + ISP4DBG_ONE_TIME_LOG_LEN >= le) ||
+			    (!le && str + ISP4DBG_ONE_TIME_LOG_LEN >= end)) {
+				if (le)
+					*le = 0;
+
+				if (*str != '\0')
+					dev_dbg(isp->dev, "%s", str);
+
+				if (le) {
+					*le = 0x0A;
+					str = le + 1;
+				} else {
+					break;
+				}
+			} else {
+				u32 tmp_len = ISP4DBG_ONE_TIME_LOG_LEN;
+
+				temp_ch = str[tmp_len];
+				str[tmp_len] = 0;
+				dev_dbg(isp->dev, "%s", str);
+				str[tmp_len] = temp_ch;
+				str = &str[tmp_len];
+			}
+		}
+	}
+}
+#endif
+
+char *isp4dbg_get_buf_src_str(u32 src)
+{
+	switch (src) {
+	case ISP4FW_BUFFER_SOURCE_STREAM:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_SOURCE_STREAM);
+	default:
+		return "Unknown buf source";
+	}
+}
+
+char *isp4dbg_get_buf_done_str(u32 status)
+{
+	switch (status) {
+	case ISP4FW_BUFFER_STATUS_INVALID:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_STATUS_INVALID);
+	case ISP4FW_BUFFER_STATUS_SKIPPED:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_STATUS_SKIPPED);
+	case ISP4FW_BUFFER_STATUS_EXIST:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_STATUS_EXIST);
+	case ISP4FW_BUFFER_STATUS_DONE:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_STATUS_DONE);
+	case ISP4FW_BUFFER_STATUS_LACK:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_STATUS_LACK);
+	case ISP4FW_BUFFER_STATUS_DIRTY:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_STATUS_DIRTY);
+	case ISP4FW_BUFFER_STATUS_MAX:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_STATUS_MAX);
+	default:
+		return "Unknown Buf Done Status";
+	}
+}
+
+char *isp4dbg_get_img_fmt_str(int fmt /* enum isp4fw_image_format * */)
+{
+	switch (fmt) {
+	case ISP4FW_IMAGE_FORMAT_NV12:
+		return "NV12";
+	case ISP4FW_IMAGE_FORMAT_YUV422INTERLEAVED:
+		return "YUV422INTERLEAVED";
+	default:
+		return "unknown fmt";
+	}
+}
+
+void isp4dbg_show_bufmeta_info(struct device *dev, char *pre,
+			       void *in, void *orig_buf)
+{
+	struct isp4fw_buffer_meta_info *p;
+	struct isp4if_img_buf_info *orig;
+
+	if (!in)
+		return;
+
+	if (!pre)
+		pre = "";
+
+	p = in;
+	orig = orig_buf;
+
+	dev_dbg(dev, "%s(%s) en:%d,stat:%s(%u),src:%s\n", pre,
+		isp4dbg_get_img_fmt_str(p->image_prop.image_format),
+		p->enabled, isp4dbg_get_buf_done_str(p->status), p->status,
+		isp4dbg_get_buf_src_str(p->source));
+
+	dev_dbg(dev, "%p,0x%llx(%u) %p,0x%llx(%u) %p,0x%llx(%u)\n",
+		orig->planes[0].sys_addr, orig->planes[0].mc_addr,
+		orig->planes[0].len, orig->planes[1].sys_addr,
+		orig->planes[1].mc_addr, orig->planes[1].len,
+		orig->planes[2].sys_addr, orig->planes[2].mc_addr,
+		orig->planes[2].len);
+}
+
+char *isp4dbg_get_buf_type(u32 type)
+{
+	/* enum isp4fw_buffer_type */
+	switch (type) {
+	case ISP4FW_BUFFER_TYPE_PREVIEW:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_TYPE_PREVIEW);
+	case ISP4FW_BUFFER_TYPE_META_INFO:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_TYPE_META_INFO);
+	case ISP4FW_BUFFER_TYPE_MEM_POOL:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_BUFFER_TYPE_MEM_POOL);
+	default:
+		return "unknown type";
+	}
+}
+
+char *isp4dbg_get_cmd_str(u32 cmd)
+{
+	switch (cmd) {
+	case ISP4FW_CMD_ID_START_STREAM:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_CMD_ID_START_STREAM);
+	case ISP4FW_CMD_ID_STOP_STREAM:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_CMD_ID_STOP_STREAM);
+	case ISP4FW_CMD_ID_SEND_BUFFER:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_CMD_ID_SEND_BUFFER);
+	case ISP4FW_CMD_ID_SET_STREAM_CONFIG:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_CMD_ID_SET_STREAM_CONFIG);
+	case ISP4FW_CMD_ID_SET_OUT_CHAN_PROP:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_CMD_ID_SET_OUT_CHAN_PROP);
+	case ISP4FW_CMD_ID_ENABLE_OUT_CHAN:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_CMD_ID_ENABLE_OUT_CHAN);
+	default:
+		return "unknown cmd";
+	}
+}
+
+char *isp4dbg_get_resp_str(u32 cmd)
+{
+	switch (cmd) {
+	case ISP4FW_RESP_ID_CMD_DONE:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_RESP_ID_CMD_DONE);
+	case ISP4FW_RESP_ID_NOTI_FRAME_DONE:
+		return ISP4DBG_MACRO_2_STR(ISP4FW_RESP_ID_NOTI_FRAME_DONE);
+	default:
+		return "unknown respid";
+	}
+}
+
+char *isp4dbg_get_if_stream_str(u32 stream /* enum fw_cmd_resp_stream_id */)
+{
+	switch (stream) {
+	case ISP4IF_STREAM_ID_GLOBAL:
+		return "STREAM_GLOBAL";
+	case ISP4IF_STREAM_ID_1:
+		return "STREAM1";
+	default:
+		return "unknown streamID";
+	}
+}
+
+char *isp4dbg_get_out_ch_str(int ch /* enum isp4fw_pipe_out_ch */)
+{
+	switch ((enum isp4fw_pipe_out_ch)ch) {
+	case ISP4FW_ISP_PIPE_OUT_CH_PREVIEW:
+		return "prev";
+	default:
+		return "unknown channel";
+	}
+}
diff --git a/drivers/media/platform/amd/isp4/isp4_debug.h b/drivers/media/platform/amd/isp4/isp4_debug.h
new file mode 100644
index 0000000000000..d1262e03ae640
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_debug.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _ISP4_DEBUG_H_
+#define _ISP4_DEBUG_H_
+
+#include <linux/dev_printk.h>
+#include <linux/printk.h>
+
+#include "isp4_subdev.h"
+
+#ifdef CONFIG_DEBUG_FS
+struct isp4_device;
+
+void isp_debugfs_create(struct isp4_device *isp_dev);
+void isp_debugfs_remove(struct isp4_device *isp_dev);
+void isp_fw_log_print(struct isp4_subdev *isp);
+
+#else
+
+/* to avoid checkpatch warning */
+#define isp_debugfs_create(cam) ((void)(cam))
+#define isp_debugfs_remove(cam) ((void)(cam))
+#define isp_fw_log_print(isp) ((void)(isp))
+
+#endif /* CONFIG_DEBUG_FS */
+
+void isp4dbg_show_bufmeta_info(struct device *dev, char *pre, void *p,
+			       void *orig_buf /* struct sys_img_buf_handle */);
+char *isp4dbg_get_img_fmt_str(int fmt /* enum _image_format_t */);
+char *isp4dbg_get_out_ch_str(int ch /* enum _isp_pipe_out_ch_t */);
+char *isp4dbg_get_cmd_str(u32 cmd);
+char *isp4dbg_get_buf_type(u32 type);/* enum _buffer_type_t */
+char *isp4dbg_get_resp_str(u32 resp);
+char *isp4dbg_get_buf_src_str(u32 src);
+char *isp4dbg_get_buf_done_str(u32 status);
+char *isp4dbg_get_if_stream_str(u32 stream);
+
+#endif /* _ISP4_DEBUG_H_ */
diff --git a/drivers/media/platform/amd/isp4/isp4_fw_cmd_resp.h b/drivers/media/platform/amd/isp4/isp4_fw_cmd_resp.h
new file mode 100644
index 0000000000000..88bacb00355c4
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_fw_cmd_resp.h
@@ -0,0 +1,318 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _ISP4_FW_CMD_RESP_H_
+#define _ISP4_FW_CMD_RESP_H_
+
+/*
+ *        Two types of command/response channel.
+ *          Type Global Command has one command/response channel.
+ *          Type Stream Command has one command/response channel.
+ *-----------                                        ------------
+ *|         |       ---------------------------      |          |
+ *|         |  ---->|  Global Command         |----> |          |
+ *|         |       ---------------------------      |          |
+ *|         |                                        |          |
+ *|         |                                        |          |
+ *|         |       ---------------------------      |          |
+ *|         |  ---->|   Stream Command        |----> |          |
+ *|         |       ---------------------------      |          |
+ *|         |                                        |          |
+ *|         |                                        |          |
+ *|         |                                        |          |
+ *|  HOST   |                                        | Firmware |
+ *|         |                                        |          |
+ *|         |                                        |          |
+ *|         |       --------------------------       |          |
+ *|         |  <----|  Global Response       |<----  |          |
+ *|         |       --------------------------       |          |
+ *|         |                                        |          |
+ *|         |                                        |          |
+ *|         |       --------------------------       |          |
+ *|         |  <----|  Stream Response       |<----  |          |
+ *|         |       --------------------------       |          |
+ *|         |                                        |          |
+ *|         |                                        |          |
+ *-----------                                        ------------
+ */
+
+/*
+ *        cmd_id is in the format of following type:
+ *        type: indicate command type, global/stream commands.
+ *        group: indicate the command group.
+ *        id: A unique command identification in one type and group.
+ *        |<-Bit31 ~ Bit24->|<-Bit23 ~ Bit16->|<-Bit15 ~ Bit0->|
+ *        |      type       |      group      |       id       |
+ */
+
+#define ISP4FW_CMD_TYPE_SHIFT            24
+#define ISP4FW_CMD_GROUP_SHIFT           16
+#define ISP4FW_CMD_TYPE_STREAM_CTRL      (0x2U << ISP4FW_CMD_TYPE_SHIFT)
+
+#define ISP4FW_CMD_GROUP_STREAM_CTRL     (0x1U << ISP4FW_CMD_GROUP_SHIFT)
+#define ISP4FW_CMD_GROUP_STREAM_BUFFER   (0x4U << ISP4FW_CMD_GROUP_SHIFT)
+
+/* Stream  Command */
+#define ISP4FW_CMD_ID_SET_STREAM_CONFIG  (ISP4FW_CMD_TYPE_STREAM_CTRL\
+					 | ISP4FW_CMD_GROUP_STREAM_CTRL | 0x1)
+#define ISP4FW_CMD_ID_SET_OUT_CHAN_PROP  (ISP4FW_CMD_TYPE_STREAM_CTRL\
+					 | ISP4FW_CMD_GROUP_STREAM_CTRL | 0x3)
+#define ISP4FW_CMD_ID_ENABLE_OUT_CHAN    (ISP4FW_CMD_TYPE_STREAM_CTRL\
+					 | ISP4FW_CMD_GROUP_STREAM_CTRL | 0x5)
+#define ISP4FW_CMD_ID_START_STREAM       (ISP4FW_CMD_TYPE_STREAM_CTRL\
+					 | ISP4FW_CMD_GROUP_STREAM_CTRL | 0x7)
+#define ISP4FW_CMD_ID_STOP_STREAM        (ISP4FW_CMD_TYPE_STREAM_CTRL\
+					 | ISP4FW_CMD_GROUP_STREAM_CTRL | 0x8)
+
+/* Stream Buffer Command */
+#define ISP4FW_CMD_ID_SEND_BUFFER        (ISP4FW_CMD_TYPE_STREAM_CTRL\
+					 | ISP4FW_CMD_GROUP_STREAM_BUFFER | 0x1)
+
+/*
+ *        resp_id is in the format of following type:
+ *        type: indicate command type, global/stream commands.
+ *        group: indicate the command group.
+ *        id: A unique command identification in one type and group.
+ *        |<-Bit31 ~ Bit24->|<-Bit23 ~ Bit16->|<-Bit15 ~ Bit0->|
+ *        |      type       |      group      |       id       |
+ */
+
+#define ISP4FW_RESP_GROUP_SHIFT          16
+
+#define ISP4FW_RESP_GROUP_GENERAL        (0x1 << ISP4FW_RESP_GROUP_SHIFT)
+#define ISP4FW_RESP_GROUP_NOTIFICATION   (0x3 << ISP4FW_RESP_GROUP_SHIFT)
+
+/* General Response */
+#define ISP4FW_RESP_ID_CMD_DONE          (ISP4FW_RESP_GROUP_GENERAL | 0x1)
+
+/* Notification */
+#define ISP4FW_RESP_ID_NOTI_FRAME_DONE   (ISP4FW_RESP_GROUP_NOTIFICATION | 0x1)
+
+#define ISP4FW_CMD_STATUS_SUCCESS        0
+#define ISP4FW_CMD_STATUS_FAIL           1
+#define ISP4FW_CMD_STATUS_SKIPPED        2
+
+#define ISP4FW_ADDR_SPACE_TYPE_GPU_VA    4
+
+#define ISP4FW_MEMORY_POOL_SIZE          (100 * 1024 * 1024)
+
+/*
+ * standard ISP pipeline: mipicsi=>isp
+ */
+#define ISP4FW_MIPI0_ISP_PIPELINE_ID     0x5f91
+
+enum isp4fw_sensor_id {
+	/* Sensor id for ISP input from MIPI port 0 */
+	ISP4FW_SENSOR_ID_ON_MIPI0  = 0,
+};
+
+enum isp4fw_stream_id {
+	ISP4FW_STREAM_ID_INVALID = -1,
+	ISP4FW_STREAM_ID_1 = 0,
+	ISP4FW_STREAM_ID_2 = 1,
+	ISP4FW_STREAM_ID_3 = 2,
+	ISP4FW_STREAM_ID_MAXIMUM
+};
+
+enum isp4fw_image_format {
+	/* 4:2:0,semi-planar, 8-bit */
+	ISP4FW_IMAGE_FORMAT_NV12 = 1,
+	/* interleave, 4:2:2, 8-bit */
+	ISP4FW_IMAGE_FORMAT_YUV422INTERLEAVED = 7,
+};
+
+enum isp4fw_pipe_out_ch {
+	ISP4FW_ISP_PIPE_OUT_CH_PREVIEW = 0,
+};
+
+enum isp4fw_yuv_range {
+	ISP4FW_ISP_YUV_RANGE_FULL = 0,     /* YUV value range in 0~255 */
+	ISP4FW_ISP_YUV_RANGE_NARROW = 1,   /* YUV value range in 16~235 */
+	ISP4FW_ISP_YUV_RANGE_MAX
+};
+
+enum isp4fw_buffer_type {
+	ISP4FW_BUFFER_TYPE_PREVIEW = 8,
+	ISP4FW_BUFFER_TYPE_META_INFO = 10,
+	ISP4FW_BUFFER_TYPE_MEM_POOL = 15,
+};
+
+enum isp4fw_buffer_status {
+	/* The buffer is INVALID */
+	ISP4FW_BUFFER_STATUS_INVALID,
+	/* The buffer is not filled with image data */
+	ISP4FW_BUFFER_STATUS_SKIPPED,
+	/* The buffer is available and awaiting to be filled */
+	ISP4FW_BUFFER_STATUS_EXIST,
+	/* The buffer is filled with image data */
+	ISP4FW_BUFFER_STATUS_DONE,
+	/* The buffer is unavailable */
+	ISP4FW_BUFFER_STATUS_LACK,
+	/* The buffer is dirty, probably caused by LMI leakage */
+	ISP4FW_BUFFER_STATUS_DIRTY,
+	ISP4FW_BUFFER_STATUS_MAX
+};
+
+enum isp4fw_buffer_source {
+	/* The buffer is from the stream buffer queue */
+	ISP4FW_BUFFER_SOURCE_STREAM,
+};
+
+struct isp4fw_error_code {
+	u32 code1;
+	u32 code2;
+	u32 code3;
+	u32 code4;
+	u32 code5;
+};
+
+/* Command Structure for FW */
+
+struct isp4fw_cmd {
+	u32 cmd_seq_num;
+	u32 cmd_id;
+	u32 cmd_param[12];
+	u16 cmd_stream_id;
+	u8 cmd_silent_resp;
+	u8 reserved;
+	u32 cmd_check_sum;
+};
+
+struct isp4fw_resp_cmd_done {
+	/*
+	 * The host2fw command seqNum.
+	 * To indicate which command this response refers to.
+	 */
+	u32 cmd_seq_num;
+	/* The host2fw command id for host double check. */
+	u32 cmd_id;
+	/*
+	 * Indicate the command process status.
+	 * 0 means success. 1 means fail. 2 means skipped
+	 */
+	u16 cmd_status;
+	/*
+	 * If cmd_status is 1, the command failed. The host can check
+	 * isp4fw_error_code for details.
+	 */
+	u16 isp4fw_error_code;
+	/* The response payload type varies by cmd. */
+	u8 payload[36];
+};
+
+struct isp4fw_resp_param_package {
+	u32 package_addr_lo;	/* The low 32 bit of the pkg address. */
+	u32 package_addr_hi;	/* The high 32 bit of the pkg address. */
+	u32 package_size;	/* The total pkg size in bytes. */
+	u32 package_check_sum;	/* The byte sum of the pkg. */
+};
+
+struct isp4fw_resp {
+	u32 resp_seq_num;
+	u32 resp_id;
+	union {
+		struct isp4fw_resp_cmd_done cmd_done;
+		struct isp4fw_resp_param_package frame_done;
+		u32 resp_param[12];
+	} param;
+	u8  reserved[4];
+	u32 resp_check_sum;
+};
+
+struct isp4fw_mipi_pipe_path_cfg {
+	u32 b_enable;
+	enum isp4fw_sensor_id isp4fw_sensor_id;
+};
+
+struct isp4fw_isp_pipe_path_cfg {
+	u32  isp_pipe_id;	/* pipe ids for pipeline construction */
+};
+
+struct isp4fw_isp_stream_cfg {
+	/* Isp mipi path */
+	struct isp4fw_mipi_pipe_path_cfg mipi_pipe_path_cfg;
+	/* Isp pipe path */
+	struct isp4fw_isp_pipe_path_cfg  isp_pipe_path_cfg;
+	/* enable TNR */
+	u32 b_enable_tnr;
+	/*
+	 * Number of frames for RTA processing.
+	 * Set to 0 to use the firmware's default value.
+	 */
+	u32 rta_frames_per_proc;
+};
+
+struct isp4fw_image_prop {
+	enum isp4fw_image_format image_format;
+	u32 width;
+	u32 height;
+	u32 luma_pitch;
+	u32 chroma_pitch;
+	enum isp4fw_yuv_range yuv_range;
+};
+
+struct isp4fw_buffer {
+	/*
+	 * A check num for debug usage, host can set the buf_tags
+	 * to different numbers
+	 */
+	u32 buf_tags;
+	union {
+		u32 value;
+		struct {
+			u32 space : 16;
+			u32 vmid  : 16;
+		} bit;
+	} vmid_space;
+	u32 buf_base_a_lo;		/* Low address of buffer A */
+	u32 buf_base_a_hi;		/* High address of buffer A */
+	u32 buf_size_a;			/* Buffer size of buffer A */
+
+	u32 buf_base_b_lo;		/* Low address of buffer B */
+	u32 buf_base_b_hi;		/* High address of buffer B */
+	u32 buf_size_b;			/* Buffer size of buffer B */
+
+	u32 buf_base_c_lo;		/* Low address of buffer C */
+	u32 buf_base_c_hi;		/* High address of buffer C */
+	u32 buf_size_c;			/* Buffer size of buffer C */
+};
+
+struct isp4fw_buffer_meta_info {
+	u32 enabled;					/* enabled flag */
+	enum isp4fw_buffer_status status;		/* BufferStatus */
+	struct isp4fw_error_code err;			/* err code */
+	enum isp4fw_buffer_source source;		/* BufferSource */
+	struct isp4fw_image_prop image_prop;		/* image_prop */
+	struct isp4fw_buffer buffer;			/* buffer info */
+};
+
+struct isp4fw_meta_info {
+	u32 poc;				/* frame id */
+	u32 fc_id;				/* frame ctl id */
+	u32 time_stamp_lo;			/* timestamp low 32 bits */
+	u32 time_stamp_hi;			/* timestamp_high 32 bits */
+	struct isp4fw_buffer_meta_info preview;	/* preview BufferMetaInfo */
+};
+
+struct isp4fw_cmd_send_buffer {
+	enum isp4fw_buffer_type buffer_type;
+	struct isp4fw_buffer buffer;		/* buffer info */
+};
+
+struct isp4fw_cmd_set_out_ch_prop {
+	enum isp4fw_pipe_out_ch ch;	/* ISP output channel */
+	struct isp4fw_image_prop image_prop;	/* image property */
+};
+
+struct isp4fw_cmd_enable_out_ch {
+	enum isp4fw_pipe_out_ch ch;	/* ISP output channel */
+	u32 is_enable;			/* If channel is enabled or not */
+};
+
+struct isp4fw_cmd_set_stream_cfg {
+	struct isp4fw_isp_stream_cfg stream_cfg; /* stream path config */
+};
+
+#endif /* _ISP4_FW_CMD_RESP_H_ */
diff --git a/drivers/media/platform/amd/isp4/isp4_hw_reg.h b/drivers/media/platform/amd/isp4/isp4_hw_reg.h
new file mode 100644
index 0000000000000..09c76f75c5ee9
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_hw_reg.h
@@ -0,0 +1,124 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _ISP4_HW_REG_H_
+#define _ISP4_HW_REG_H_
+
+#include <linux/io.h>
+
+#define ISP_SOFT_RESET			0x62000
+#define ISP_SYS_INT0_EN			0x62010
+#define ISP_SYS_INT0_STATUS		0x62014
+#define ISP_SYS_INT0_ACK		0x62018
+#define ISP_CCPU_CNTL			0x62054
+#define ISP_STATUS			0x62058
+#define ISP_LOG_RB_BASE_LO0		0x62148
+#define ISP_LOG_RB_BASE_HI0		0x6214c
+#define ISP_LOG_RB_SIZE0		0x62150
+#define ISP_LOG_RB_RPTR0		0x62154
+#define ISP_LOG_RB_WPTR0		0x62158
+#define ISP_RB_BASE_LO1			0x62170
+#define ISP_RB_BASE_HI1			0x62174
+#define ISP_RB_SIZE1			0x62178
+#define ISP_RB_RPTR1			0x6217c
+#define ISP_RB_WPTR1			0x62180
+#define ISP_RB_BASE_LO2			0x62184
+#define ISP_RB_BASE_HI2			0x62188
+#define ISP_RB_SIZE2			0x6218c
+#define ISP_RB_RPTR2			0x62190
+#define ISP_RB_WPTR2			0x62194
+#define ISP_RB_BASE_LO3			0x62198
+#define ISP_RB_BASE_HI3			0x6219c
+#define ISP_RB_SIZE3			0x621a0
+#define ISP_RB_RPTR3			0x621a4
+#define ISP_RB_WPTR3			0x621a8
+#define ISP_RB_BASE_LO4			0x621ac
+#define ISP_RB_BASE_HI4			0x621b0
+#define ISP_RB_SIZE4			0x621b4
+#define ISP_RB_RPTR4			0x621b8
+#define ISP_RB_WPTR4			0x621bc
+#define ISP_RB_BASE_LO5			0x621c0
+#define ISP_RB_BASE_HI5			0x621c4
+#define ISP_RB_SIZE5			0x621c8
+#define ISP_RB_RPTR5			0x621cc
+#define ISP_RB_WPTR5			0x621d0
+#define ISP_RB_BASE_LO6			0x621d4
+#define ISP_RB_BASE_HI6			0x621d8
+#define ISP_RB_SIZE6			0x621dc
+#define ISP_RB_RPTR6			0x621e0
+#define ISP_RB_WPTR6			0x621e4
+#define ISP_RB_BASE_LO7			0x621e8
+#define ISP_RB_BASE_HI7			0x621ec
+#define ISP_RB_SIZE7			0x621f0
+#define ISP_RB_RPTR7			0x621f4
+#define ISP_RB_WPTR7			0x621f8
+#define ISP_RB_BASE_LO8			0x621fc
+#define ISP_RB_BASE_HI8			0x62200
+#define ISP_RB_SIZE8			0x62204
+#define ISP_RB_RPTR8			0x62208
+#define ISP_RB_WPTR8			0x6220c
+#define ISP_RB_BASE_LO9			0x62210
+#define ISP_RB_BASE_HI9			0x62214
+#define ISP_RB_SIZE9			0x62218
+#define ISP_RB_RPTR9			0x6221c
+#define ISP_RB_WPTR9			0x62220
+#define ISP_RB_BASE_LO10		0x62224
+#define ISP_RB_BASE_HI10		0x62228
+#define ISP_RB_SIZE10			0x6222c
+#define ISP_RB_RPTR10			0x62230
+#define ISP_RB_WPTR10			0x62234
+#define ISP_RB_BASE_LO11		0x62238
+#define ISP_RB_BASE_HI11		0x6223c
+#define ISP_RB_SIZE11			0x62240
+#define ISP_RB_RPTR11			0x62244
+#define ISP_RB_WPTR11			0x62248
+#define ISP_RB_BASE_LO12		0x6224c
+#define ISP_RB_BASE_HI12		0x62250
+#define ISP_RB_SIZE12			0x62254
+#define ISP_RB_RPTR12			0x62258
+#define ISP_RB_WPTR12			0x6225c
+
+#define ISP_POWER_STATUS		0x60000
+
+/* ISP_SOFT_RESET */
+#define ISP_SOFT_RESET__CCPU_SOFT_RESET_MASK			0x00000001UL
+
+/* ISP_CCPU_CNTL */
+#define ISP_CCPU_CNTL__CCPU_HOST_SOFT_RST_MASK			0x00040000UL
+
+/* ISP_STATUS */
+#define ISP_STATUS__CCPU_REPORT_MASK				0x000000feUL
+
+/* ISP_SYS_INT0_STATUS */
+#define ISP_SYS_INT0_STATUS__SYS_INT_RINGBUFFER_WPT9_INT_MASK	0x00010000UL
+#define ISP_SYS_INT0_STATUS__SYS_INT_RINGBUFFER_WPT10_INT_MASK	0x00040000UL
+#define ISP_SYS_INT0_STATUS__SYS_INT_RINGBUFFER_WPT11_INT_MASK	0x00100000UL
+#define ISP_SYS_INT0_STATUS__SYS_INT_RINGBUFFER_WPT12_INT_MASK	0x00400000UL
+
+/* ISP_SYS_INT0_EN */
+#define ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT9_EN_MASK	0x00010000UL
+#define ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT10_EN_MASK	0x00040000UL
+#define ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT11_EN_MASK	0x00100000UL
+#define ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT12_EN_MASK	0x00400000UL
+
+/* ISP_SYS_INT0_ACK */
+#define ISP_SYS_INT0_ACK__SYS_INT_RINGBUFFER_WPT9_ACK_MASK	0x00010000UL
+#define ISP_SYS_INT0_ACK__SYS_INT_RINGBUFFER_WPT10_ACK_MASK	0x00040000UL
+#define ISP_SYS_INT0_ACK__SYS_INT_RINGBUFFER_WPT11_ACK_MASK	0x00100000UL
+#define ISP_SYS_INT0_ACK__SYS_INT_RINGBUFFER_WPT12_ACK_MASK	0x00400000UL
+
+/* Helper functions for reading isp registers */
+static inline u32 isp4hw_rreg(void __iomem *base, u32 reg)
+{
+	return readl(base + reg);
+}
+
+/* Helper functions for writing isp registers */
+static inline void isp4hw_wreg(void __iomem *base, u32 reg, u32 val)
+{
+	return writel(val, base + reg);
+}
+
+#endif /* _ISP4_HW_REG_H_ */
diff --git a/drivers/media/platform/amd/isp4/isp4_interface.c b/drivers/media/platform/amd/isp4/isp4_interface.c
new file mode 100644
index 0000000000000..8d73f66bb42cb
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_interface.c
@@ -0,0 +1,832 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#include <linux/iopoll.h>
+
+#include "isp4_debug.h"
+#include "isp4_fw_cmd_resp.h"
+#include "isp4_hw_reg.h"
+#include "isp4_interface.h"
+
+#define ISP4IF_FW_RESP_RB_IRQ_EN_MASK \
+	(ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT9_EN_MASK\
+	 | ISP_SYS_INT0_EN__SYS_INT_RINGBUFFER_WPT12_EN_MASK)
+
+#define ISP4IF_FW_CMD_TIMEOUT (HZ / 2)
+
+struct isp4if_rb_config {
+	const char *name;
+	u32 index;
+	u32 reg_rptr;
+	u32 reg_wptr;
+	u32 reg_base_lo;
+	u32 reg_base_hi;
+	u32 reg_size;
+	u32 val_size;
+	u64 base_mc_addr;
+	void *base_sys_addr;
+};
+
+/* FW cmd ring buffer configuration */
+static struct isp4if_rb_config isp4if_cmd_rb_config[ISP4IF_STREAM_ID_MAX] = {
+	{
+		.name = "CMD_RB_GBL0",
+		.index = 3,
+		.reg_rptr = ISP_RB_RPTR4,
+		.reg_wptr = ISP_RB_WPTR4,
+		.reg_base_lo = ISP_RB_BASE_LO4,
+		.reg_base_hi = ISP_RB_BASE_HI4,
+		.reg_size = ISP_RB_SIZE4,
+	},
+	{
+		.name = "CMD_RB_STR1",
+		.index = 0,
+		.reg_rptr = ISP_RB_RPTR1,
+		.reg_wptr = ISP_RB_WPTR1,
+		.reg_base_lo = ISP_RB_BASE_LO1,
+		.reg_base_hi = ISP_RB_BASE_HI1,
+		.reg_size = ISP_RB_SIZE1,
+	},
+	{
+		.name = "CMD_RB_STR2",
+		.index = 1,
+		.reg_rptr = ISP_RB_RPTR2,
+		.reg_wptr = ISP_RB_WPTR2,
+		.reg_base_lo = ISP_RB_BASE_LO2,
+		.reg_base_hi = ISP_RB_BASE_HI2,
+		.reg_size = ISP_RB_SIZE2,
+	},
+	{
+		.name = "CMD_RB_STR3",
+		.index = 2,
+		.reg_rptr = ISP_RB_RPTR3,
+		.reg_wptr = ISP_RB_WPTR3,
+		.reg_base_lo = ISP_RB_BASE_LO3,
+		.reg_base_hi = ISP_RB_BASE_HI3,
+		.reg_size = ISP_RB_SIZE3,
+	},
+};
+
+/* FW resp ring buffer configuration */
+static struct isp4if_rb_config isp4if_resp_rb_config[ISP4IF_STREAM_ID_MAX] = {
+	{
+		.name = "RES_RB_GBL0",
+		.index = 3,
+		.reg_rptr = ISP_RB_RPTR12,
+		.reg_wptr = ISP_RB_WPTR12,
+		.reg_base_lo = ISP_RB_BASE_LO12,
+		.reg_base_hi = ISP_RB_BASE_HI12,
+		.reg_size = ISP_RB_SIZE12,
+	},
+	{
+		.name = "RES_RB_STR1",
+		.index = 0,
+		.reg_rptr = ISP_RB_RPTR9,
+		.reg_wptr = ISP_RB_WPTR9,
+		.reg_base_lo = ISP_RB_BASE_LO9,
+		.reg_base_hi = ISP_RB_BASE_HI9,
+		.reg_size = ISP_RB_SIZE9,
+	},
+	{
+		.name = "RES_RB_STR2",
+		.index = 1,
+		.reg_rptr = ISP_RB_RPTR10,
+		.reg_wptr = ISP_RB_WPTR10,
+		.reg_base_lo = ISP_RB_BASE_LO10,
+		.reg_base_hi = ISP_RB_BASE_HI10,
+		.reg_size = ISP_RB_SIZE10,
+	},
+	{
+		.name = "RES_RB_STR3",
+		.index = 2,
+		.reg_rptr = ISP_RB_RPTR11,
+		.reg_wptr = ISP_RB_WPTR11,
+		.reg_base_lo = ISP_RB_BASE_LO11,
+		.reg_base_hi = ISP_RB_BASE_HI11,
+		.reg_size = ISP_RB_SIZE11,
+	},
+};
+
+/* FW log ring buffer configuration */
+static struct isp4if_rb_config isp4if_log_rb_config = {
+	.name = "LOG_RB",
+	.index = 0,
+	.reg_rptr = ISP_LOG_RB_RPTR0,
+	.reg_wptr = ISP_LOG_RB_WPTR0,
+	.reg_base_lo = ISP_LOG_RB_BASE_LO0,
+	.reg_base_hi = ISP_LOG_RB_BASE_HI0,
+	.reg_size = ISP_LOG_RB_SIZE0,
+};
+
+static struct isp4if_gpu_mem_info *
+isp4if_gpu_mem_alloc(struct isp4_interface *ispif, u32 mem_size)
+{
+	struct isp4if_gpu_mem_info *mem_info;
+	struct device *dev = ispif->dev;
+	int ret;
+
+	mem_info = kmalloc_obj(*mem_info, GFP_KERNEL);
+	if (!mem_info)
+		return NULL;
+
+	mem_info->mem_size = mem_size;
+	ret = isp_kernel_buffer_alloc(dev, mem_info->mem_size,
+				      &mem_info->mem_handle,
+				      &mem_info->gpu_mc_addr,
+				      &mem_info->sys_addr);
+	if (ret) {
+		kfree(mem_info);
+		return NULL;
+	}
+
+	return mem_info;
+}
+
+static void isp4if_gpu_mem_free(struct isp4_interface *ispif,
+				struct isp4if_gpu_mem_info **mem_info_ptr)
+{
+	struct isp4if_gpu_mem_info *mem_info = *mem_info_ptr;
+	struct device *dev = ispif->dev;
+
+	if (!mem_info) {
+		dev_err(dev, "invalid mem_info\n");
+		return;
+	}
+
+	*mem_info_ptr = NULL;
+	isp_kernel_buffer_free(&mem_info->mem_handle, &mem_info->gpu_mc_addr,
+			       &mem_info->sys_addr);
+	kfree(mem_info);
+}
+
+static void isp4if_dealloc_fw_gpumem(struct isp4_interface *ispif)
+{
+	isp4if_gpu_mem_free(ispif, &ispif->fw_mem_pool);
+	isp4if_gpu_mem_free(ispif, &ispif->fw_cmd_resp_buf);
+	isp4if_gpu_mem_free(ispif, &ispif->fw_log_buf);
+
+	for (unsigned int i = 0; i < ISP4IF_MAX_STREAM_BUF_COUNT; i++)
+		isp4if_gpu_mem_free(ispif, &ispif->meta_info_buf[i]);
+}
+
+static int isp4if_alloc_fw_gpumem(struct isp4_interface *ispif)
+{
+	struct device *dev = ispif->dev;
+
+	ispif->fw_mem_pool = isp4if_gpu_mem_alloc(ispif,
+						  ISP4FW_MEMORY_POOL_SIZE);
+	if (!ispif->fw_mem_pool)
+		goto error_no_memory;
+
+	ispif->fw_cmd_resp_buf =
+		isp4if_gpu_mem_alloc(ispif, ISP4IF_RB_PMBMAP_MEM_SIZE);
+	if (!ispif->fw_cmd_resp_buf)
+		goto error_no_memory;
+
+	ispif->fw_log_buf =
+		isp4if_gpu_mem_alloc(ispif, ISP4IF_FW_LOG_RINGBUF_SIZE);
+	if (!ispif->fw_log_buf)
+		goto error_no_memory;
+
+	for (unsigned int i = 0; i < ISP4IF_MAX_STREAM_BUF_COUNT; i++) {
+		ispif->meta_info_buf[i] =
+			isp4if_gpu_mem_alloc(ispif, ISP4IF_META_INFO_BUF_SIZE);
+		if (!ispif->meta_info_buf[i])
+			goto error_no_memory;
+	}
+
+	return 0;
+
+error_no_memory:
+	dev_err(dev, "failed to allocate gpu memory\n");
+	return -ENOMEM;
+}
+
+static u32 isp4if_compute_check_sum(const void *buf, size_t buf_size)
+{
+	const u8 *surplus_ptr;
+	const u32 *buffer;
+	u32 checksum = 0;
+	size_t i;
+
+	buffer = (const u32 *)buf;
+	for (i = 0; i < buf_size / sizeof(u32); i++)
+		checksum += buffer[i];
+
+	surplus_ptr = (const u8 *)&buffer[i];
+	/* add surplus data crc checksum */
+	for (i = 0; i < buf_size % sizeof(u32); i++)
+		checksum += surplus_ptr[i];
+
+	return checksum;
+}
+
+void isp4if_clear_cmdq(struct isp4_interface *ispif)
+{
+	struct isp4if_cmd_element *buf_node, *tmp_node;
+	LIST_HEAD(free_list);
+
+	scoped_guard(spinlock, &ispif->cmdq_lock)
+		list_splice_init(&ispif->cmdq, &free_list);
+
+	list_for_each_entry_safe(buf_node, tmp_node, &free_list, list)
+		kfree(buf_node);
+}
+
+static bool isp4if_is_cmdq_rb_full(struct isp4_interface *ispif,
+				   enum isp4if_stream_id stream)
+{
+	struct isp4if_rb_config *rb_config = &isp4if_cmd_rb_config[stream];
+	u32 rreg = rb_config->reg_rptr, wreg = rb_config->reg_wptr;
+	u32 len = rb_config->val_size;
+	u32 rd_ptr, wr_ptr;
+	u32 bytes_free;
+
+	rd_ptr = isp4hw_rreg(ispif->mmio, rreg);
+	wr_ptr = isp4hw_rreg(ispif->mmio, wreg);
+
+	/*
+	 * Read and write pointers are equal, indicating the ring buffer
+	 * is empty
+	 */
+	if (wr_ptr == rd_ptr)
+		return false;
+
+	if (wr_ptr > rd_ptr)
+		bytes_free = len - (wr_ptr - rd_ptr);
+	else
+		bytes_free = rd_ptr - wr_ptr;
+
+	/*
+	 * Ignore one byte from the bytes free to prevent rd_ptr from equaling
+	 * wr_ptr when the ring buffer is full, because rd_ptr == wr_ptr is
+	 * supposed to indicate that the ring buffer is empty.
+	 */
+	return bytes_free <= sizeof(struct isp4fw_cmd);
+}
+
+struct isp4if_cmd_element *isp4if_rm_cmd_from_cmdq(struct isp4_interface *ispif,
+						   u32 seq_num, u32 cmd_id)
+{
+	struct isp4if_cmd_element *ele;
+
+	guard(spinlock)(&ispif->cmdq_lock);
+
+	list_for_each_entry(ele, &ispif->cmdq, list) {
+		if (ele->seq_num == seq_num && ele->cmd_id == cmd_id) {
+			list_del(&ele->list);
+			return ele;
+		}
+	}
+
+	return NULL;
+}
+
+/* Must check that isp4if_is_cmdq_rb_full() == false before calling */
+static int isp4if_insert_isp_fw_cmd(struct isp4_interface *ispif,
+				    enum isp4if_stream_id stream,
+				    const struct isp4fw_cmd *cmd)
+{
+	struct isp4if_rb_config *rb_config = &isp4if_cmd_rb_config[stream];
+	u32 rreg = rb_config->reg_rptr, wreg = rb_config->reg_wptr;
+	void *mem_sys = rb_config->base_sys_addr;
+	const u32 cmd_sz = sizeof(*cmd);
+	struct device *dev = ispif->dev;
+	u32 len = rb_config->val_size;
+	const void *src = cmd;
+	u32 rd_ptr, wr_ptr;
+	u32 bytes_to_end;
+
+	rd_ptr = isp4hw_rreg(ispif->mmio, rreg);
+	wr_ptr = isp4hw_rreg(ispif->mmio, wreg);
+	if (rd_ptr >= len || wr_ptr >= len) {
+		dev_err(dev,
+			"rb invalid: stream=%u(%s), rd=%u, wr=%u, len=%u, cmd_sz=%u\n",
+			stream, isp4dbg_get_if_stream_str(stream), rd_ptr,
+			wr_ptr, len, cmd_sz);
+		return -EINVAL;
+	}
+
+	bytes_to_end = len - wr_ptr;
+	if (bytes_to_end >= cmd_sz) {
+		/* FW cmd is just a straight copy to the write pointer */
+		memcpy(mem_sys + wr_ptr, src, cmd_sz);
+		isp4hw_wreg(ispif->mmio, wreg, (wr_ptr + cmd_sz) % len);
+	} else {
+		/*
+		 * FW cmd is split because the ring buffer needs to wrap
+		 * around
+		 */
+		memcpy(mem_sys + wr_ptr, src, bytes_to_end);
+		memcpy(mem_sys, src + bytes_to_end, cmd_sz - bytes_to_end);
+		isp4hw_wreg(ispif->mmio, wreg, cmd_sz - bytes_to_end);
+	}
+
+	return 0;
+}
+
+static inline enum isp4if_stream_id isp4if_get_fw_stream(u32 cmd_id)
+{
+	return ISP4IF_STREAM_ID_1;
+}
+
+static int isp4if_send_fw_cmd(struct isp4_interface *ispif, u32 cmd_id,
+			      const void *package,
+			      u32 package_size, bool sync)
+{
+	enum isp4if_stream_id stream = isp4if_get_fw_stream(cmd_id);
+	struct isp4if_cmd_element *ele = NULL;
+	struct device *dev = ispif->dev;
+	struct isp4fw_cmd cmd;
+	u32 seq_num;
+	int ret;
+
+	if (package_size > sizeof(cmd.cmd_param)) {
+		dev_err(dev, "fail pkgsize(%u) > %zu cmd:0x%x, stream %d\n",
+			package_size, sizeof(cmd.cmd_param), cmd_id, stream);
+		return -EINVAL;
+	}
+
+	/*
+	 * The struct will be shared with ISP FW, use memset() to guarantee
+	 * padding bits are zeroed, since this is not guaranteed on all
+	 * compilers.
+	 */
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.cmd_id = cmd_id;
+	switch (stream) {
+	case ISP4IF_STREAM_ID_GLOBAL:
+		cmd.cmd_stream_id = ISP4FW_STREAM_ID_INVALID;
+		break;
+	case ISP4IF_STREAM_ID_1:
+		cmd.cmd_stream_id = ISP4FW_STREAM_ID_1;
+		break;
+	default:
+		dev_err(dev, "fail bad stream id %d\n", stream);
+		return -EINVAL;
+	}
+
+	/* Allocate the sync command object early and outside of the lock */
+	if (sync) {
+		ele = kmalloc_obj(*ele, GFP_KERNEL);
+		if (!ele)
+			return -ENOMEM;
+
+		/* Get two references: one for the resp thread, one for us */
+		atomic_set(&ele->refcnt, 2);
+		init_completion(&ele->cmd_done);
+	}
+
+	if (package && package_size)
+		memcpy(cmd.cmd_param, package, package_size);
+
+	scoped_guard(mutex, &ispif->isp4if_mutex) {
+		ret = read_poll_timeout(isp4if_is_cmdq_rb_full, ret, !ret,
+					ISP4IF_RB_FULL_SLEEP_US,
+					ISP4IF_RB_FULL_TIMEOUT_US, false, ispif,
+					stream);
+		if (ret) {
+			struct isp4if_rb_config *rb_config =
+					&isp4if_resp_rb_config[stream];
+			u32 rd_ptr = isp4hw_rreg(ispif->mmio,
+						 rb_config->reg_rptr);
+			u32 wr_ptr = isp4hw_rreg(ispif->mmio,
+						 rb_config->reg_wptr);
+
+			dev_err(dev,
+				"failed to get free cmdq slot, stream %s(%d),rd %u, wr %u\n",
+				isp4dbg_get_if_stream_str(stream), stream,
+				rd_ptr, wr_ptr);
+			ret = -ETIMEDOUT;
+			goto free_ele;
+		}
+
+		seq_num = ispif->host2fw_seq_num++;
+		cmd.cmd_seq_num = seq_num;
+		cmd.cmd_check_sum = isp4if_compute_check_sum(&cmd, sizeof(cmd)
+							     - sizeof(u32));
+
+		/*
+		 * only append the fw cmd to queue when its response needs to
+		 * be waited for, currently there are only two such commands,
+		 * disable channel and stop stream which are only sent after
+		 * close camera
+		 */
+		if (ele) {
+			ele->seq_num = seq_num;
+			ele->cmd_id = cmd_id;
+			scoped_guard(spinlock, &ispif->cmdq_lock)
+				list_add_tail(&ele->list, &ispif->cmdq);
+		}
+
+		ret = isp4if_insert_isp_fw_cmd(ispif, stream, &cmd);
+		if (ret) {
+			dev_err(dev,
+				"fail for insert_isp_fw_cmd cmd_id %s(0x%08x)\n",
+				isp4dbg_get_cmd_str(cmd_id), cmd_id);
+			goto err_dequeue_ele;
+		}
+	}
+
+	if (ele) {
+		ret = wait_for_completion_timeout(&ele->cmd_done,
+						  ISP4IF_FW_CMD_TIMEOUT);
+		if (!ret) {
+			ret = -ETIMEDOUT;
+			goto err_dequeue_ele;
+		}
+
+		ret = 0;
+		goto put_ele_ref;
+	}
+
+	return 0;
+
+err_dequeue_ele:
+	/*
+	 * Try to remove the command from the queue. If that fails, then it
+	 * means the response thread is currently using the object, and we need
+	 * to use the refcount to avoid a use-after-free by either side.
+	 */
+	if (ele && isp4if_rm_cmd_from_cmdq(ispif, seq_num, cmd_id))
+		goto free_ele;
+
+put_ele_ref:
+	/* Don't free the command if we didn't put the last reference */
+	if (ele && atomic_dec_return(&ele->refcnt))
+		ele = NULL;
+
+free_ele:
+	kfree(ele);
+	return ret;
+}
+
+static int isp4if_send_buffer(struct isp4_interface *ispif,
+			      struct isp4if_img_buf_info *buf_info)
+{
+	struct isp4fw_cmd_send_buffer cmd;
+
+	/*
+	 * The struct will be shared with ISP FW, use memset() to guarantee
+	 * padding bits are zeroed, since this is not guaranteed on all
+	 * compilers.
+	 */
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.buffer_type = ISP4FW_BUFFER_TYPE_PREVIEW;
+	cmd.buffer.vmid_space.bit.space = ISP4FW_ADDR_SPACE_TYPE_GPU_VA;
+	isp4if_split_addr64(buf_info->planes[0].mc_addr,
+			    &cmd.buffer.buf_base_a_lo,
+			    &cmd.buffer.buf_base_a_hi);
+	cmd.buffer.buf_size_a = buf_info->planes[0].len;
+
+	isp4if_split_addr64(buf_info->planes[1].mc_addr,
+			    &cmd.buffer.buf_base_b_lo,
+			    &cmd.buffer.buf_base_b_hi);
+	cmd.buffer.buf_size_b = buf_info->planes[1].len;
+
+	isp4if_split_addr64(buf_info->planes[2].mc_addr,
+			    &cmd.buffer.buf_base_c_lo,
+			    &cmd.buffer.buf_base_c_hi);
+	cmd.buffer.buf_size_c = buf_info->planes[2].len;
+
+	return isp4if_send_fw_cmd(ispif, ISP4FW_CMD_ID_SEND_BUFFER, &cmd,
+				  sizeof(cmd), false);
+}
+
+static void isp4if_init_rb_config(struct isp4_interface *ispif,
+				  struct isp4if_rb_config *rb_config)
+{
+	isp4hw_wreg(ispif->mmio, rb_config->reg_rptr, 0x0);
+	isp4hw_wreg(ispif->mmio, rb_config->reg_wptr, 0x0);
+	isp4hw_wreg(ispif->mmio, rb_config->reg_base_lo,
+		    rb_config->base_mc_addr);
+	isp4hw_wreg(ispif->mmio, rb_config->reg_base_hi,
+		    rb_config->base_mc_addr >> 32);
+	isp4hw_wreg(ispif->mmio, rb_config->reg_size, rb_config->val_size);
+}
+
+static int isp4if_fw_init(struct isp4_interface *ispif)
+{
+	u32 aligned_rb_chunk_size = ISP4IF_RB_PMBMAP_MEM_CHUNK & 0xffffffc0;
+	struct isp4if_rb_config *rb_config;
+	u32 offset;
+	unsigned int i;
+
+	/* initialize CMD_RB streams */
+	for (i = 0; i < ISP4IF_STREAM_ID_MAX; i++) {
+		rb_config = (isp4if_cmd_rb_config + i);
+		offset = aligned_rb_chunk_size * rb_config->index;
+
+		rb_config->val_size = ISP4IF_FW_CMD_BUF_SIZE;
+		rb_config->base_sys_addr =
+			ispif->fw_cmd_resp_buf->sys_addr + offset;
+		rb_config->base_mc_addr =
+			ispif->fw_cmd_resp_buf->gpu_mc_addr + offset;
+
+		isp4if_init_rb_config(ispif, rb_config);
+	}
+
+	/* initialize RESP_RB streams */
+	for (i = 0; i < ISP4IF_STREAM_ID_MAX; i++) {
+		rb_config = (isp4if_resp_rb_config + i);
+		offset = aligned_rb_chunk_size *
+			 (rb_config->index + ISP4IF_RESP_CHAN_TO_RB_OFFSET - 1);
+
+		rb_config->val_size = ISP4IF_FW_CMD_BUF_SIZE;
+		rb_config->base_sys_addr =
+			ispif->fw_cmd_resp_buf->sys_addr + offset;
+		rb_config->base_mc_addr =
+			ispif->fw_cmd_resp_buf->gpu_mc_addr + offset;
+
+		isp4if_init_rb_config(ispif, rb_config);
+	}
+
+	/* initialize LOG_RB stream */
+	rb_config = &isp4if_log_rb_config;
+	rb_config->val_size = ISP4IF_FW_LOG_RINGBUF_SIZE;
+	rb_config->base_mc_addr = ispif->fw_log_buf->gpu_mc_addr;
+	rb_config->base_sys_addr = ispif->fw_log_buf->sys_addr;
+
+	isp4if_init_rb_config(ispif, rb_config);
+
+	return 0;
+}
+
+static int isp4if_wait_fw_ready(struct isp4_interface *ispif,
+				u32 isp_status_addr)
+{
+	struct device *dev = ispif->dev;
+	u32 timeout_ms = 100;
+	u32 interval_ms = 1;
+	u32 reg_val;
+
+	/* wait for FW initialize done! */
+	if (!read_poll_timeout(isp4hw_rreg, reg_val, reg_val
+			       & ISP_STATUS__CCPU_REPORT_MASK,
+			       interval_ms * 1000, timeout_ms * 1000, false,
+			       ispif->mmio, isp_status_addr))
+		return 0;
+
+	dev_err(dev, "ISP CCPU FW boot failed\n");
+
+	return -ETIME;
+}
+
+static void isp4if_enable_ccpu(struct isp4_interface *ispif)
+{
+	u32 reg_val;
+
+	reg_val = isp4hw_rreg(ispif->mmio, ISP_SOFT_RESET);
+	reg_val &= (~ISP_SOFT_RESET__CCPU_SOFT_RESET_MASK);
+	isp4hw_wreg(ispif->mmio, ISP_SOFT_RESET, reg_val);
+
+	usleep_range(100, 150);
+
+	reg_val = isp4hw_rreg(ispif->mmio, ISP_CCPU_CNTL);
+	reg_val &= (~ISP_CCPU_CNTL__CCPU_HOST_SOFT_RST_MASK);
+	isp4hw_wreg(ispif->mmio, ISP_CCPU_CNTL, reg_val);
+}
+
+static void isp4if_disable_ccpu(struct isp4_interface *ispif)
+{
+	u32 reg_val;
+
+	reg_val = isp4hw_rreg(ispif->mmio, ISP_CCPU_CNTL);
+	reg_val |= ISP_CCPU_CNTL__CCPU_HOST_SOFT_RST_MASK;
+	isp4hw_wreg(ispif->mmio, ISP_CCPU_CNTL, reg_val);
+
+	usleep_range(100, 150);
+
+	reg_val = isp4hw_rreg(ispif->mmio, ISP_SOFT_RESET);
+	reg_val |= ISP_SOFT_RESET__CCPU_SOFT_RESET_MASK;
+	isp4hw_wreg(ispif->mmio, ISP_SOFT_RESET, reg_val);
+}
+
+static int isp4if_fw_boot(struct isp4_interface *ispif)
+{
+	struct device *dev = ispif->dev;
+
+	if (ispif->status != ISP4IF_STATUS_PWR_ON) {
+		dev_err(dev, "invalid isp power status %d\n", ispif->status);
+		return -EINVAL;
+	}
+
+	isp4if_disable_ccpu(ispif);
+
+	isp4if_fw_init(ispif);
+
+	/* clear ccpu status */
+	isp4hw_wreg(ispif->mmio, ISP_STATUS, 0x0);
+
+	isp4if_enable_ccpu(ispif);
+
+	if (isp4if_wait_fw_ready(ispif, ISP_STATUS)) {
+		isp4if_disable_ccpu(ispif);
+		return -EINVAL;
+	}
+
+	/* enable interrupts */
+	isp4hw_wreg(ispif->mmio, ISP_SYS_INT0_EN,
+		    ISP4IF_FW_RESP_RB_IRQ_EN_MASK);
+
+	ispif->status = ISP4IF_STATUS_FW_RUNNING;
+
+	dev_dbg(dev, "ISP CCPU FW boot success\n");
+
+	return 0;
+}
+
+int isp4if_f2h_resp(struct isp4_interface *ispif, enum isp4if_stream_id stream,
+		    struct isp4fw_resp *resp)
+{
+	struct isp4if_rb_config *rb_config = &isp4if_resp_rb_config[stream];
+	u32 rreg = rb_config->reg_rptr, wreg = rb_config->reg_wptr;
+	void *mem_sys = rb_config->base_sys_addr;
+	const u32 resp_sz = sizeof(*resp);
+	struct device *dev = ispif->dev;
+	u32 len = rb_config->val_size;
+	u32 rd_ptr, wr_ptr;
+	u32 bytes_to_end;
+	void *dst = resp;
+	u32 checksum;
+
+	rd_ptr = isp4hw_rreg(ispif->mmio, rreg);
+	wr_ptr = isp4hw_rreg(ispif->mmio, wreg);
+	if (rd_ptr >= len || wr_ptr >= len)
+		goto err_rb_invalid;
+
+	/*
+	 * Read and write pointers are equal, indicating the ring buffer is
+	 * empty
+	 */
+	if (rd_ptr == wr_ptr)
+		return -ENODATA;
+
+	bytes_to_end = len - rd_ptr;
+	if (bytes_to_end >= resp_sz) {
+		/* FW response is just a straight copy from the read pointer */
+		if (wr_ptr > rd_ptr && wr_ptr - rd_ptr < resp_sz)
+			goto err_rb_invalid;
+
+		memcpy(dst, mem_sys + rd_ptr, resp_sz);
+		isp4hw_wreg(ispif->mmio, rreg, (rd_ptr + resp_sz) % len);
+	} else {
+		/*
+		 * FW response is split because the ring buffer wrapped
+		 * around
+		 */
+		if (wr_ptr > rd_ptr || wr_ptr < resp_sz - bytes_to_end)
+			goto err_rb_invalid;
+
+		memcpy(dst, mem_sys + rd_ptr, bytes_to_end);
+		memcpy(dst + bytes_to_end, mem_sys, resp_sz - bytes_to_end);
+		isp4hw_wreg(ispif->mmio, rreg, resp_sz - bytes_to_end);
+	}
+
+	checksum = isp4if_compute_check_sum(resp, resp_sz - sizeof(u32));
+	if (checksum != resp->resp_check_sum) {
+		dev_err(dev, "resp checksum 0x%x,should 0x%x,rptr %u,wptr %u\n",
+			checksum, resp->resp_check_sum, rd_ptr, wr_ptr);
+		dev_err(dev, "%s(%u), seqNo %u, resp_id %s(0x%x)\n",
+			isp4dbg_get_if_stream_str(stream), stream,
+			resp->resp_seq_num, isp4dbg_get_resp_str(resp->resp_id),
+			resp->resp_id);
+		return -EINVAL;
+	}
+
+	return 0;
+
+err_rb_invalid:
+	dev_err(dev,
+		"rb invalid: stream=%u(%s), rd=%u, wr=%u, len=%u, resp_sz=%u\n",
+		stream, isp4dbg_get_if_stream_str(stream), rd_ptr, wr_ptr, len,
+		resp_sz);
+	return -EINVAL;
+}
+
+int isp4if_send_command(struct isp4_interface *ispif, u32 cmd_id,
+			const void *package, u32 package_size)
+{
+	return isp4if_send_fw_cmd(ispif, cmd_id, package, package_size, false);
+}
+
+int isp4if_send_command_sync(struct isp4_interface *ispif, u32 cmd_id,
+			     const void *package, u32 package_size)
+{
+	return isp4if_send_fw_cmd(ispif, cmd_id, package, package_size, true);
+}
+
+void isp4if_clear_bufq(struct isp4_interface *ispif)
+{
+	struct isp4if_img_buf_node *buf_node, *tmp_node;
+	LIST_HEAD(free_list);
+
+	scoped_guard(spinlock, &ispif->bufq_lock)
+		list_splice_init(&ispif->bufq, &free_list);
+
+	list_for_each_entry_safe(buf_node, tmp_node, &free_list, node)
+		kfree(buf_node);
+}
+
+void isp4if_dealloc_buffer_node(struct isp4if_img_buf_node *buf_node)
+{
+	kfree(buf_node);
+}
+
+struct isp4if_img_buf_node *
+isp4if_alloc_buffer_node(struct isp4if_img_buf_info *buf_info)
+{
+	struct isp4if_img_buf_node *node;
+
+	node = kmalloc_obj(*node, GFP_KERNEL);
+	if (node)
+		node->buf_info = *buf_info;
+
+	return node;
+}
+
+struct isp4if_img_buf_node *isp4if_dequeue_buffer(struct isp4_interface *ispif)
+{
+	struct isp4if_img_buf_node *buf_node;
+
+	guard(spinlock)(&ispif->bufq_lock);
+
+	buf_node = list_first_entry_or_null(&ispif->bufq, typeof(*buf_node),
+					    node);
+	if (buf_node)
+		list_del(&buf_node->node);
+
+	return buf_node;
+}
+
+int isp4if_queue_buffer(struct isp4_interface *ispif,
+			struct isp4if_img_buf_node *buf_node)
+{
+	int ret;
+
+	ret = isp4if_send_buffer(ispif, &buf_node->buf_info);
+	if (ret)
+		return ret;
+
+	scoped_guard(spinlock, &ispif->bufq_lock)
+		list_add_tail(&buf_node->node, &ispif->bufq);
+
+	return 0;
+}
+
+int isp4if_stop(struct isp4_interface *ispif)
+{
+	isp4if_disable_ccpu(ispif);
+
+	isp4if_dealloc_fw_gpumem(ispif);
+
+	return 0;
+}
+
+int isp4if_start(struct isp4_interface *ispif)
+{
+	int ret;
+
+	ret = isp4if_alloc_fw_gpumem(ispif);
+	if (ret)
+		return ret;
+
+	ret = isp4if_fw_boot(ispif);
+	if (ret)
+		goto failed_fw_boot;
+
+	return 0;
+
+failed_fw_boot:
+	isp4if_dealloc_fw_gpumem(ispif);
+	return ret;
+}
+
+int isp4if_deinit(struct isp4_interface *ispif)
+{
+	isp4if_clear_cmdq(ispif);
+
+	isp4if_clear_bufq(ispif);
+
+	mutex_destroy(&ispif->isp4if_mutex);
+
+	return 0;
+}
+
+int isp4if_init(struct isp4_interface *ispif, struct device *dev,
+		void __iomem *isp_mmio)
+{
+	ispif->dev = dev;
+	ispif->mmio = isp_mmio;
+
+	spin_lock_init(&ispif->cmdq_lock); /* used for cmdq access */
+	spin_lock_init(&ispif->bufq_lock); /* used for bufq access */
+	mutex_init(&ispif->isp4if_mutex); /* used for commands sent to ispfw */
+
+	INIT_LIST_HEAD(&ispif->cmdq);
+	INIT_LIST_HEAD(&ispif->bufq);
+
+	return 0;
+}
diff --git a/drivers/media/platform/amd/isp4/isp4_interface.h b/drivers/media/platform/amd/isp4/isp4_interface.h
new file mode 100644
index 0000000000000..ce3ac9b9e5cda
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_interface.h
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _ISP4_INTERFACE_H_
+#define _ISP4_INTERFACE_H_
+
+#include <drm/amd/isp.h>
+#include <linux/mutex.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+
+struct isp4fw_resp;
+
+#define ISP4IF_RB_MAX 25
+#define ISP4IF_RESP_CHAN_TO_RB_OFFSET 9
+#define ISP4IF_RB_PMBMAP_MEM_SIZE (SZ_16M - 1)
+#define ISP4IF_RB_PMBMAP_MEM_CHUNK \
+	(ISP4IF_RB_PMBMAP_MEM_SIZE / (ISP4IF_RB_MAX - 1))
+#define ISP4IF_HOST2FW_COMMAND_SIZE sizeof(struct isp4fw_cmd)
+#define ISP4IF_MAX_NUM_HOST2FW_COMMAND 40
+#define ISP4IF_FW_CMD_BUF_SIZE \
+	(ISP4IF_MAX_NUM_HOST2FW_COMMAND * ISP4IF_HOST2FW_COMMAND_SIZE)
+#define ISP4IF_RB_FULL_SLEEP_US (33 * USEC_PER_MSEC)
+#define ISP4IF_RB_FULL_TIMEOUT_US (10 * ISP4IF_RB_FULL_SLEEP_US)
+
+#define ISP4IF_META_INFO_BUF_SIZE ALIGN(sizeof(struct isp4fw_meta_info), 0x8000)
+#define ISP4IF_MAX_STREAM_BUF_COUNT 8
+
+#define ISP4IF_FW_LOG_RINGBUF_SIZE SZ_2M
+
+enum isp4if_stream_id {
+	ISP4IF_STREAM_ID_GLOBAL = 0,
+	ISP4IF_STREAM_ID_1 = 1,
+	ISP4IF_STREAM_ID_MAX = 4
+};
+
+enum isp4if_status {
+	ISP4IF_STATUS_PWR_OFF,
+	ISP4IF_STATUS_PWR_ON,
+	ISP4IF_STATUS_FW_RUNNING,
+	ISP4IF_FSM_STATUS_MAX
+};
+
+struct isp4if_gpu_mem_info {
+	u64 mem_size;
+	u64 gpu_mc_addr;
+	void *sys_addr;
+	void *mem_handle;
+};
+
+struct isp4if_img_buf_info {
+	struct {
+		void *sys_addr;
+		u64 mc_addr;
+		u32 len;
+	} planes[3];
+};
+
+struct isp4if_img_buf_node {
+	struct list_head node;
+	struct isp4if_img_buf_info buf_info;
+};
+
+struct isp4if_cmd_element {
+	struct list_head list;
+	u32 seq_num;
+	u32 cmd_id;
+	struct completion cmd_done;
+	atomic_t refcnt;
+};
+
+struct isp4_interface {
+	struct device *dev;
+	void __iomem *mmio;
+
+	spinlock_t cmdq_lock; /* used for cmdq access */
+	spinlock_t bufq_lock; /* used for bufq access */
+	struct mutex isp4if_mutex; /* used to send fw cmd and read fw log */
+
+	struct list_head cmdq; /* commands sent to fw */
+	struct list_head bufq; /* buffers sent to fw */
+
+	enum isp4if_status status;
+	u32 host2fw_seq_num;
+
+	/* ISP fw buffers */
+	struct isp4if_gpu_mem_info *fw_log_buf;
+	struct isp4if_gpu_mem_info *fw_cmd_resp_buf;
+	struct isp4if_gpu_mem_info *fw_mem_pool;
+	struct isp4if_gpu_mem_info *meta_info_buf[ISP4IF_MAX_STREAM_BUF_COUNT];
+};
+
+static inline void isp4if_split_addr64(u64 addr, u32 *lo, u32 *hi)
+{
+	if (lo)
+		*lo = addr & 0xffffffff;
+
+	if (hi)
+		*hi = addr >> 32;
+}
+
+static inline u64 isp4if_join_addr64(u32 lo, u32 hi)
+{
+	return (((u64)hi) << 32) | (u64)lo;
+}
+
+int isp4if_f2h_resp(struct isp4_interface *ispif, enum isp4if_stream_id stream,
+		    struct isp4fw_resp *resp);
+
+int isp4if_send_command(struct isp4_interface *ispif, u32 cmd_id,
+			const void *package, u32 package_size);
+
+int isp4if_send_command_sync(struct isp4_interface *ispif, u32 cmd_id,
+			     const void *package, u32 package_size);
+
+struct isp4if_cmd_element *isp4if_rm_cmd_from_cmdq(struct isp4_interface *ispif,
+						   u32 seq_num, u32 cmd_id);
+
+void isp4if_clear_cmdq(struct isp4_interface *ispif);
+
+void isp4if_clear_bufq(struct isp4_interface *ispif);
+
+void isp4if_dealloc_buffer_node(struct isp4if_img_buf_node *buf_node);
+
+struct isp4if_img_buf_node *
+isp4if_alloc_buffer_node(struct isp4if_img_buf_info *buf_info);
+
+struct isp4if_img_buf_node *isp4if_dequeue_buffer(struct isp4_interface *ispif);
+
+int isp4if_queue_buffer(struct isp4_interface *ispif,
+			struct isp4if_img_buf_node *buf_node);
+
+int isp4if_stop(struct isp4_interface *ispif);
+
+int isp4if_start(struct isp4_interface *ispif);
+
+int isp4if_deinit(struct isp4_interface *ispif);
+
+int isp4if_init(struct isp4_interface *ispif, struct device *dev,
+		void __iomem *isp_mmio);
+
+#endif /* _ISP4_INTERFACE_H_ */
diff --git a/drivers/media/platform/amd/isp4/isp4_subdev.c b/drivers/media/platform/amd/isp4/isp4_subdev.c
new file mode 100644
index 0000000000000..48deea79ce6c2
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_subdev.c
@@ -0,0 +1,1047 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#include <linux/pm_domain.h>
+#include <linux/units.h>
+
+#include "isp4.h"
+#include "isp4_debug.h"
+#include "isp4_fw_cmd_resp.h"
+#include "isp4_interface.h"
+
+#define ISP4SD_MIN_BUF_CNT_BEF_START_STREAM 4
+
+#define ISP4SD_PERFORMANCE_STATE_LOW 0
+#define ISP4SD_PERFORMANCE_STATE_HIGH 1
+
+/* align 32KB */
+#define ISP4SD_META_BUF_SIZE ALIGN(sizeof(struct isp4fw_meta_info), 0x8000)
+
+#define to_isp4_subdev(sd) container_of(sd, struct isp4_subdev, sdev)
+
+static const char *isp4sd_entity_name = "amd isp4";
+
+static const char *isp4sd_thread_name[ISP4SD_MAX_FW_RESP_STREAM_NUM] = {
+	"amd_isp4_thread_global",
+	"amd_isp4_thread_stream1",
+};
+
+static void isp4sd_module_enable(struct isp4_subdev *isp_subdev, bool enable)
+{
+	if (isp_subdev->enable_gpio) {
+		gpiod_set_value(isp_subdev->enable_gpio, enable ? 1 : 0);
+		dev_dbg(isp_subdev->dev, "%s isp_subdev module\n",
+			enable ? "enable" : "disable");
+	}
+}
+
+static int isp4sd_setup_fw_mem_pool(struct isp4_subdev *isp_subdev)
+{
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct isp4fw_cmd_send_buffer buf_type;
+	struct device *dev = isp_subdev->dev;
+	int ret;
+
+	if (!ispif->fw_mem_pool) {
+		dev_err(dev, "fail to alloc mem pool\n");
+		return -ENOMEM;
+	}
+
+	/*
+	 * The struct will be shared with ISP FW, use memset() to guarantee
+	 * padding bits are zeroed, since this is not guaranteed on all
+	 * compilers.
+	 */
+	memset(&buf_type, 0, sizeof(buf_type));
+	buf_type.buffer_type = ISP4FW_BUFFER_TYPE_MEM_POOL;
+	buf_type.buffer.vmid_space.bit.space = ISP4FW_ADDR_SPACE_TYPE_GPU_VA;
+	isp4if_split_addr64(ispif->fw_mem_pool->gpu_mc_addr,
+			    &buf_type.buffer.buf_base_a_lo,
+			    &buf_type.buffer.buf_base_a_hi);
+	buf_type.buffer.buf_size_a = ispif->fw_mem_pool->mem_size;
+
+	ret = isp4if_send_command(ispif, ISP4FW_CMD_ID_SEND_BUFFER,
+				  &buf_type, sizeof(buf_type));
+	if (ret) {
+		dev_err(dev, "send fw mem pool 0x%llx(%u) fail %d\n",
+			ispif->fw_mem_pool->gpu_mc_addr,
+			buf_type.buffer.buf_size_a, ret);
+		return ret;
+	}
+
+	dev_dbg(dev, "send fw mem pool 0x%llx(%u) suc\n",
+		ispif->fw_mem_pool->gpu_mc_addr, buf_type.buffer.buf_size_a);
+
+	return 0;
+}
+
+static int isp4sd_set_stream_path(struct isp4_subdev *isp_subdev)
+{
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct isp4fw_cmd_set_stream_cfg cmd;
+	struct device *dev = isp_subdev->dev;
+
+	/*
+	 * The struct will be shared with ISP FW, use memset() to guarantee
+	 * padding bits are zeroed, since this is not guaranteed on all
+	 * compilers.
+	 */
+	memset(&cmd, 0, sizeof(cmd));
+	cmd.stream_cfg.mipi_pipe_path_cfg.isp4fw_sensor_id =
+		ISP4FW_SENSOR_ID_ON_MIPI0;
+	cmd.stream_cfg.mipi_pipe_path_cfg.b_enable = true;
+	cmd.stream_cfg.isp_pipe_path_cfg.isp_pipe_id =
+		ISP4FW_MIPI0_ISP_PIPELINE_ID;
+
+	cmd.stream_cfg.b_enable_tnr = true;
+	dev_dbg(dev, "isp4fw_sensor_id %d, pipeId 0x%x EnableTnr %u\n",
+		cmd.stream_cfg.mipi_pipe_path_cfg.isp4fw_sensor_id,
+		cmd.stream_cfg.isp_pipe_path_cfg.isp_pipe_id,
+		cmd.stream_cfg.b_enable_tnr);
+
+	return isp4if_send_command(ispif, ISP4FW_CMD_ID_SET_STREAM_CONFIG,
+				   &cmd, sizeof(cmd));
+}
+
+static int isp4sd_send_meta_buf(struct isp4_subdev *isp_subdev)
+{
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct isp4fw_cmd_send_buffer buf_type;
+	struct device *dev = isp_subdev->dev;
+
+	/*
+	 * The struct will be shared with ISP FW, use memset() to guarantee
+	 * padding bits are zeroed, since this is not guaranteed on all
+	 * compilers.
+	 */
+	memset(&buf_type, 0, sizeof(buf_type));
+	for (unsigned int i = 0; i < ISP4IF_MAX_STREAM_BUF_COUNT; i++) {
+		struct isp4if_gpu_mem_info *meta_info_buf =
+				isp_subdev->ispif.meta_info_buf[i];
+		int ret;
+
+		if (!meta_info_buf) {
+			dev_err(dev, "fail for no meta info buf(%u)\n", i);
+			return -ENOMEM;
+		}
+
+		buf_type.buffer_type = ISP4FW_BUFFER_TYPE_META_INFO;
+		buf_type.buffer.vmid_space.bit.space =
+			ISP4FW_ADDR_SPACE_TYPE_GPU_VA;
+		isp4if_split_addr64(meta_info_buf->gpu_mc_addr,
+				    &buf_type.buffer.buf_base_a_lo,
+				    &buf_type.buffer.buf_base_a_hi);
+		buf_type.buffer.buf_size_a = meta_info_buf->mem_size;
+		ret = isp4if_send_command(ispif, ISP4FW_CMD_ID_SEND_BUFFER,
+					  &buf_type, sizeof(buf_type));
+		if (ret) {
+			dev_err(dev, "send meta info(%u) fail\n", i);
+			return ret;
+		}
+	}
+
+	dev_dbg(dev, "send meta info suc\n");
+	return 0;
+}
+
+static bool isp4sd_get_str_out_prop(struct isp4_subdev *isp_subdev,
+				    struct isp4fw_image_prop *out_prop,
+				    struct v4l2_subdev_state *state, u32 pad)
+{
+	struct device *dev = isp_subdev->dev;
+	struct v4l2_mbus_framefmt *format;
+
+	format = v4l2_subdev_state_get_format(state, pad, 0);
+	if (!format) {
+		dev_err(dev, "fail get subdev state format\n");
+		return false;
+	}
+
+	switch (format->code) {
+	case MEDIA_BUS_FMT_YUYV8_1_5X8:
+		out_prop->image_format = ISP4FW_IMAGE_FORMAT_NV12;
+		out_prop->width = format->width;
+		out_prop->height = format->height;
+		out_prop->luma_pitch = format->width;
+		out_prop->chroma_pitch = out_prop->width;
+		break;
+	case MEDIA_BUS_FMT_YUYV8_1X16:
+		out_prop->image_format = ISP4FW_IMAGE_FORMAT_YUV422INTERLEAVED;
+		out_prop->width = format->width;
+		out_prop->height = format->height;
+		out_prop->luma_pitch = format->width * 2;
+		out_prop->chroma_pitch = 0;
+		break;
+	default:
+		dev_err(dev, "fail for bad image format:0x%x\n",
+			format->code);
+		return false;
+	}
+
+	if (!out_prop->width || !out_prop->height)
+		return false;
+
+	return true;
+}
+
+static int isp4sd_kickoff_stream(struct isp4_subdev *isp_subdev, u32 w, u32 h)
+{
+	struct isp4sd_sensor_info *sensor_info = &isp_subdev->sensor_info;
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct device *dev = isp_subdev->dev;
+
+	if (sensor_info->status == ISP4SD_START_STATUS_STARTED)
+		return 0;
+
+	if (sensor_info->status == ISP4SD_START_STATUS_START_FAIL) {
+		dev_err(dev, "fail for previous start fail\n");
+		return -EINVAL;
+	}
+
+	dev_dbg(dev, "w:%u,h:%u\n", w, h);
+
+	if (isp4sd_send_meta_buf(isp_subdev)) {
+		dev_err(dev, "fail to send meta buf\n");
+		sensor_info->status = ISP4SD_START_STATUS_START_FAIL;
+		return -EINVAL;
+	}
+
+	sensor_info->status = ISP4SD_START_STATUS_OFF;
+
+	if (!sensor_info->start_stream_cmd_sent &&
+	    sensor_info->buf_sent_cnt >= ISP4SD_MIN_BUF_CNT_BEF_START_STREAM) {
+		int ret = isp4if_send_command(ispif, ISP4FW_CMD_ID_START_STREAM,
+					      NULL, 0);
+		if (ret) {
+			dev_err(dev, "fail to start stream\n");
+			return ret;
+		}
+
+		sensor_info->start_stream_cmd_sent = true;
+	} else {
+		dev_dbg(dev,
+			"no send START_STREAM, start_sent %u, buf_sent %u\n",
+			sensor_info->start_stream_cmd_sent,
+			sensor_info->buf_sent_cnt);
+	}
+
+	return 0;
+}
+
+static int isp4sd_setup_output(struct isp4_subdev *isp_subdev,
+			       struct v4l2_subdev_state *state, u32 pad)
+{
+	struct isp4sd_output_info *output_info =
+			&isp_subdev->sensor_info.output_info;
+	struct isp4sd_sensor_info *sensor_info = &isp_subdev->sensor_info;
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct isp4fw_cmd_set_out_ch_prop cmd_ch_prop;
+	struct isp4fw_cmd_enable_out_ch cmd_ch_en;
+	struct device *dev = isp_subdev->dev;
+	int ret;
+
+	if (output_info->start_status == ISP4SD_START_STATUS_STARTED)
+		return 0;
+
+	if (output_info->start_status == ISP4SD_START_STATUS_START_FAIL) {
+		dev_err(dev, "fail for previous start fail\n");
+		return -EINVAL;
+	}
+
+	/*
+	 * The struct will be shared with ISP FW, use memset() to guarantee
+	 * padding bits are zeroed, since this is not guaranteed on all
+	 * compilers.
+	 */
+	memset(&cmd_ch_prop, 0, sizeof(cmd_ch_prop));
+	cmd_ch_prop.ch = ISP4FW_ISP_PIPE_OUT_CH_PREVIEW;
+
+	if (!isp4sd_get_str_out_prop(isp_subdev,
+				     &cmd_ch_prop.image_prop, state, pad)) {
+		dev_err(dev, "fail to get out prop\n");
+		return -EINVAL;
+	}
+
+	dev_dbg(dev, "channel:%s,fmt %s,w:h=%u:%u,lp:%u,cp%u\n",
+		isp4dbg_get_out_ch_str(cmd_ch_prop.ch),
+		isp4dbg_get_img_fmt_str(cmd_ch_prop.image_prop.image_format),
+		cmd_ch_prop.image_prop.width, cmd_ch_prop.image_prop.height,
+		cmd_ch_prop.image_prop.luma_pitch,
+		cmd_ch_prop.image_prop.chroma_pitch);
+
+	ret = isp4if_send_command(ispif, ISP4FW_CMD_ID_SET_OUT_CHAN_PROP,
+				  &cmd_ch_prop, sizeof(cmd_ch_prop));
+	if (ret) {
+		output_info->start_status = ISP4SD_START_STATUS_START_FAIL;
+		dev_err(dev, "fail to set out prop\n");
+		return ret;
+	}
+
+	/*
+	 * The struct will be shared with ISP FW, use memset() to guarantee
+	 * padding bits are zeroed, since this is not guaranteed on all
+	 * compilers.
+	 */
+	memset(&cmd_ch_en, 0, sizeof(cmd_ch_en));
+	cmd_ch_en.ch = ISP4FW_ISP_PIPE_OUT_CH_PREVIEW;
+	cmd_ch_en.is_enable = true;
+	ret = isp4if_send_command(ispif, ISP4FW_CMD_ID_ENABLE_OUT_CHAN,
+				  &cmd_ch_en, sizeof(cmd_ch_en));
+	if (ret) {
+		output_info->start_status = ISP4SD_START_STATUS_START_FAIL;
+		dev_err(dev, "fail to enable channel\n");
+		return ret;
+	}
+
+	dev_dbg(dev, "enable channel %s\n",
+		isp4dbg_get_out_ch_str(cmd_ch_en.ch));
+
+	if (!sensor_info->start_stream_cmd_sent) {
+		ret = isp4sd_kickoff_stream(isp_subdev,
+					    cmd_ch_prop.image_prop.width,
+					    cmd_ch_prop.image_prop.height);
+		if (ret) {
+			dev_err(dev, "kickoff stream fail %d\n", ret);
+			return ret;
+		}
+		/*
+		 * sensor_info->start_stream_cmd_sent will be set to true
+		 * 1. in isp4sd_kickoff_stream, if app first send buffer then
+		 * start stream
+		 * 2. in isp_set_stream_buf, if app first start stream, then
+		 * send buffer because ISP FW has the requirement, host needs
+		 * to send buffer before send start stream cmd
+		 */
+		if (sensor_info->start_stream_cmd_sent) {
+			sensor_info->status = ISP4SD_START_STATUS_STARTED;
+			output_info->start_status = ISP4SD_START_STATUS_STARTED;
+			dev_dbg(dev, "kickoff stream suc,start cmd sent\n");
+		}
+	} else {
+		dev_dbg(dev, "stream running, no need kickoff\n");
+		output_info->start_status = ISP4SD_START_STATUS_STARTED;
+	}
+
+	dev_dbg(dev, "setup output suc\n");
+	return 0;
+}
+
+static int isp4sd_init_stream(struct isp4_subdev *isp_subdev)
+{
+	struct device *dev = isp_subdev->dev;
+	int ret;
+
+	ret = isp4sd_setup_fw_mem_pool(isp_subdev);
+	if (ret) {
+		dev_err(dev, "fail to setup fw mem pool\n");
+		return ret;
+	}
+
+	ret = isp4sd_set_stream_path(isp_subdev);
+	if (ret) {
+		dev_err(dev, "fail to setup stream path\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static void isp4sd_uninit_stream(struct isp4_subdev *isp_subdev,
+				 struct v4l2_subdev_state *state, u32 pad)
+{
+	struct isp4sd_sensor_info *sensor_info = &isp_subdev->sensor_info;
+	struct isp4sd_output_info *output_info = &sensor_info->output_info;
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct v4l2_mbus_framefmt *format;
+
+	format = v4l2_subdev_state_get_format(state, pad, 0);
+	if (!format) {
+		dev_err(isp_subdev->dev, "fail to get v4l2 format\n");
+	} else {
+		memset(format, 0, sizeof(*format));
+		format->code = MEDIA_BUS_FMT_YUYV8_1_5X8;
+	}
+
+	isp4if_clear_bufq(ispif);
+	isp4if_clear_cmdq(ispif);
+
+	sensor_info->start_stream_cmd_sent = false;
+	sensor_info->buf_sent_cnt = 0;
+
+	sensor_info->status = ISP4SD_START_STATUS_OFF;
+	output_info->start_status = ISP4SD_START_STATUS_OFF;
+}
+
+static void isp4sd_fw_resp_cmd_done(struct isp4_subdev *isp_subdev,
+				    enum isp4if_stream_id stream_id,
+				    struct isp4fw_resp_cmd_done *para)
+{
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct isp4if_cmd_element *ele =
+		isp4if_rm_cmd_from_cmdq(ispif, para->cmd_seq_num, para->cmd_id);
+	struct device *dev = isp_subdev->dev;
+
+	dev_dbg(dev, "stream %d,cmd %s(0x%08x)(%d),seq %u, ele %p\n",
+		stream_id,
+		isp4dbg_get_cmd_str(para->cmd_id),
+		para->cmd_id, para->cmd_status, para->cmd_seq_num,
+		ele);
+
+	if (ele) {
+		complete(&ele->cmd_done);
+		if (atomic_dec_and_test(&ele->refcnt))
+			kfree(ele);
+	}
+}
+
+static struct isp4fw_meta_info *
+isp4sd_get_meta_by_mc(struct isp4_subdev *isp_subdev, u64 mc)
+{
+	for (unsigned int i = 0; i < ISP4IF_MAX_STREAM_BUF_COUNT; i++) {
+		struct isp4if_gpu_mem_info *meta_info_buf =
+				isp_subdev->ispif.meta_info_buf[i];
+
+		if (meta_info_buf->gpu_mc_addr == mc)
+			return meta_info_buf->sys_addr;
+	}
+
+	return NULL;
+}
+
+static void isp4sd_send_meta_info(struct isp4_subdev *isp_subdev,
+				  u64 meta_info_mc)
+{
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct isp4fw_cmd_send_buffer buf_type;
+	struct device *dev = isp_subdev->dev;
+
+	if (isp_subdev->sensor_info.status != ISP4SD_START_STATUS_STARTED) {
+		dev_warn(dev, "not working status %i, meta_info 0x%llx\n",
+			 isp_subdev->sensor_info.status, meta_info_mc);
+		return;
+	}
+
+	/*
+	 * The struct will be shared with ISP FW, use memset() to guarantee
+	 * padding bits are zeroed, since this is not guaranteed on all
+	 * compilers.
+	 */
+	memset(&buf_type, 0, sizeof(buf_type));
+	buf_type.buffer_type = ISP4FW_BUFFER_TYPE_META_INFO;
+	buf_type.buffer.vmid_space.bit.space = ISP4FW_ADDR_SPACE_TYPE_GPU_VA;
+	isp4if_split_addr64(meta_info_mc,
+			    &buf_type.buffer.buf_base_a_lo,
+			    &buf_type.buffer.buf_base_a_hi);
+	buf_type.buffer.buf_size_a = ISP4SD_META_BUF_SIZE;
+
+	if (isp4if_send_command(ispif, ISP4FW_CMD_ID_SEND_BUFFER,
+				&buf_type, sizeof(buf_type)))
+		dev_err(dev, "fail send meta_info 0x%llx\n",
+			meta_info_mc);
+	else
+		dev_dbg(dev, "resend meta_info 0x%llx\n", meta_info_mc);
+}
+
+static void isp4sd_fw_resp_frame_done(struct isp4_subdev *isp_subdev,
+				      enum isp4if_stream_id stream_id,
+				      struct isp4fw_resp_param_package *para)
+{
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct device *dev = isp_subdev->dev;
+	struct isp4if_img_buf_node *prev;
+	struct isp4fw_meta_info *meta;
+	u64 mc;
+
+	mc = isp4if_join_addr64(para->package_addr_lo, para->package_addr_hi);
+	meta = isp4sd_get_meta_by_mc(isp_subdev, mc);
+	if (!meta) {
+		dev_err(dev, "fail to get meta from mc %llx\n", mc);
+		return;
+	}
+
+	dev_dbg(dev, "ts:%llu,streamId:%d,poc:%u,preview_en:%u,status:%s(%i)\n",
+		ktime_get_ns(), stream_id, meta->poc, meta->preview.enabled,
+		isp4dbg_get_buf_done_str(meta->preview.status),
+		meta->preview.status);
+
+	if (meta->preview.enabled &&
+	    (meta->preview.status == ISP4FW_BUFFER_STATUS_SKIPPED ||
+	     meta->preview.status == ISP4FW_BUFFER_STATUS_DONE ||
+	     meta->preview.status == ISP4FW_BUFFER_STATUS_DIRTY)) {
+		prev = isp4if_dequeue_buffer(ispif);
+		if (prev) {
+			isp4dbg_show_bufmeta_info(dev, "prev", &meta->preview,
+						  &prev->buf_info);
+			isp4vid_handle_frame_done(&isp_subdev->isp_vdev,
+						  &prev->buf_info);
+			isp4if_dealloc_buffer_node(prev);
+		} else {
+			dev_err(dev, "fail null prev buf\n");
+		}
+	} else if (meta->preview.enabled) {
+		dev_err(dev, "fail bad preview status %u(%s)\n",
+			meta->preview.status,
+			isp4dbg_get_buf_done_str(meta->preview.status));
+	}
+
+	if (isp_subdev->sensor_info.status == ISP4SD_START_STATUS_STARTED)
+		isp4sd_send_meta_info(isp_subdev, mc);
+
+	dev_dbg(dev, "stream_id:%d, status:%d\n", stream_id,
+		isp_subdev->sensor_info.status);
+}
+
+static void isp4sd_fw_resp_func(struct isp4_subdev *isp_subdev,
+				enum isp4if_stream_id stream_id)
+{
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct device *dev = isp_subdev->dev;
+	struct isp4fw_resp resp;
+
+	if (stream_id == ISP4IF_STREAM_ID_1)
+		isp_fw_log_print(isp_subdev);
+
+	while (true) {
+		if (isp4if_f2h_resp(ispif, stream_id, &resp)) {
+			/* Re-enable the interrupt */
+			isp4_intr_enable(isp_subdev, stream_id, true);
+			/*
+			 * Recheck to see if there is a new response.
+			 * To ensure that an in-flight interrupt is not lost,
+			 * enabling the interrupt must occur _before_ checking
+			 * for a new response, hence a memory barrier is needed.
+			 * Disable the interrupt again if there was a new
+			 * response.
+			 */
+			mb();
+			if (likely(isp4if_f2h_resp(ispif, stream_id, &resp)))
+				break;
+
+			isp4_intr_enable(isp_subdev, stream_id, false);
+		}
+
+		switch (resp.resp_id) {
+		case ISP4FW_RESP_ID_CMD_DONE:
+			isp4sd_fw_resp_cmd_done(isp_subdev, stream_id,
+						&resp.param.cmd_done);
+			break;
+		case ISP4FW_RESP_ID_NOTI_FRAME_DONE:
+			isp4sd_fw_resp_frame_done(isp_subdev, stream_id,
+						  &resp.param.frame_done);
+			break;
+		default:
+			dev_err(dev, "-><- fail respid %s(0x%x)\n",
+				isp4dbg_get_resp_str(resp.resp_id),
+				resp.resp_id);
+			break;
+		}
+	}
+}
+
+static s32 isp4sd_fw_resp_thread(void *context)
+{
+	struct isp4_subdev_thread_param *para = context;
+	struct isp4_subdev *isp_subdev = para->isp_subdev;
+	struct isp4sd_thread_handler *thread_ctx =
+			&isp_subdev->fw_resp_thread[para->idx];
+	struct device *dev = isp_subdev->dev;
+
+	dev_dbg(dev, "[%u] fw resp thread started\n", para->idx);
+	while (true) {
+		wait_event_interruptible(thread_ctx->waitq,
+					 thread_ctx->resp_ready);
+		thread_ctx->resp_ready = false;
+
+		if (kthread_should_stop()) {
+			dev_dbg(dev, "[%u] fw resp thread quit\n", para->idx);
+			break;
+		}
+
+		isp4sd_fw_resp_func(isp_subdev, para->idx);
+	}
+
+	return 0;
+}
+
+static int isp4sd_stop_resp_proc_threads(struct isp4_subdev *isp_subdev)
+{
+	for (unsigned int i = 0; i < ISP4SD_MAX_FW_RESP_STREAM_NUM; i++) {
+		struct isp4sd_thread_handler *thread_ctx =
+				&isp_subdev->fw_resp_thread[i];
+
+		if (thread_ctx->thread) {
+			kthread_stop(thread_ctx->thread);
+			thread_ctx->thread = NULL;
+		}
+	}
+
+	return 0;
+}
+
+static int isp4sd_start_resp_proc_threads(struct isp4_subdev *isp_subdev)
+{
+	struct device *dev = isp_subdev->dev;
+
+	for (unsigned int i = 0; i < ISP4SD_MAX_FW_RESP_STREAM_NUM; i++) {
+		struct isp4sd_thread_handler *thread_ctx =
+				&isp_subdev->fw_resp_thread[i];
+
+		isp_subdev->isp_resp_para[i].idx = i;
+		isp_subdev->isp_resp_para[i].isp_subdev = isp_subdev;
+		init_waitqueue_head(&thread_ctx->waitq);
+		thread_ctx->resp_ready = false;
+
+		thread_ctx->thread = kthread_run(isp4sd_fw_resp_thread,
+						 &isp_subdev->isp_resp_para[i],
+						 isp4sd_thread_name[i]);
+		if (IS_ERR(thread_ctx->thread)) {
+			dev_err(dev, "create thread [%d] fail\n", i);
+			thread_ctx->thread = NULL;
+			isp4sd_stop_resp_proc_threads(isp_subdev);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+int isp4sd_pwroff_and_deinit(struct v4l2_subdev *sd)
+{
+	struct isp4_subdev *isp_subdev = to_isp4_subdev(sd);
+	struct isp4sd_sensor_info *sensor_info = &isp_subdev->sensor_info;
+	unsigned int perf_state = ISP4SD_PERFORMANCE_STATE_LOW;
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct device *dev = isp_subdev->dev;
+	int ret;
+
+	guard(mutex)(&isp_subdev->ops_mutex);
+	if (sensor_info->status == ISP4SD_START_STATUS_STARTED) {
+		dev_err(dev, "fail for stream still running\n");
+		return -EINVAL;
+	}
+
+	sensor_info->status = ISP4SD_START_STATUS_OFF;
+
+	if (isp_subdev->irq_enabled) {
+		for (unsigned int i = 0; i < ISP4SD_MAX_FW_RESP_STREAM_NUM; i++)
+			disable_irq(isp_subdev->irq[i]);
+		isp_subdev->irq_enabled = false;
+	}
+
+	isp4sd_stop_resp_proc_threads(isp_subdev);
+	dev_dbg(dev, "isp_subdev stop resp proc threads suc\n");
+
+	isp4if_stop(ispif);
+
+	ret = dev_pm_genpd_set_performance_state(dev, perf_state);
+	if (ret)
+		dev_err(dev,
+			"fail to set isp_subdev performance state %u,ret %d\n",
+			perf_state, ret);
+
+	/* hold ccpu reset */
+	isp4hw_wreg(isp_subdev->mmio, ISP_SOFT_RESET, 0);
+	isp4hw_wreg(isp_subdev->mmio, ISP_POWER_STATUS, 0);
+	ret = pm_runtime_put_sync(dev);
+	if (ret)
+		dev_err(dev, "power off isp_subdev fail %d\n", ret);
+	else
+		dev_dbg(dev, "power off isp_subdev suc\n");
+
+	ispif->status = ISP4IF_STATUS_PWR_OFF;
+	isp4if_clear_cmdq(ispif);
+	isp4sd_module_enable(isp_subdev, false);
+
+	/*
+	 * When opening the camera, isp4sd_module_enable(isp_subdev, true) is
+	 * called. Hardware requires at least a 20ms delay between disabling
+	 * and enabling the module, so a sleep is added to ensure ISP stability
+	 * during quick reopen scenarios.
+	 */
+	msleep(20);
+
+	return 0;
+}
+
+int isp4sd_pwron_and_init(struct v4l2_subdev *sd)
+{
+	struct isp4_subdev *isp_subdev = to_isp4_subdev(sd);
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct device *dev = isp_subdev->dev;
+	int ret;
+
+	guard(mutex)(&isp_subdev->ops_mutex);
+	if (ispif->status == ISP4IF_STATUS_FW_RUNNING) {
+		dev_dbg(dev, "camera already opened, do nothing\n");
+		return 0;
+	}
+
+	isp4sd_module_enable(isp_subdev, true);
+
+	if (ispif->status < ISP4IF_STATUS_PWR_ON) {
+		unsigned int perf_state = ISP4SD_PERFORMANCE_STATE_HIGH;
+
+		ret = pm_runtime_resume_and_get(dev);
+		if (ret) {
+			dev_err(dev, "fail to power on isp_subdev ret %d\n",
+				ret);
+			goto err_deinit;
+		}
+
+		/* ISPPG ISP Power Status */
+		isp4hw_wreg(isp_subdev->mmio, ISP_POWER_STATUS, 0x7FF);
+		ret = dev_pm_genpd_set_performance_state(dev, perf_state);
+		if (ret) {
+			dev_err(dev,
+				"fail to set performance state %u, ret %d\n",
+				perf_state, ret);
+			goto err_deinit;
+		}
+
+		ispif->status = ISP4IF_STATUS_PWR_ON;
+	}
+
+	isp_subdev->sensor_info.start_stream_cmd_sent = false;
+	isp_subdev->sensor_info.buf_sent_cnt = 0;
+
+	ret = isp4if_start(ispif);
+	if (ret) {
+		dev_err(dev, "fail to start isp_subdev interface\n");
+		goto err_deinit;
+	}
+
+	if (isp4sd_start_resp_proc_threads(isp_subdev)) {
+		dev_err(dev, "isp_start_resp_proc_threads fail\n");
+		goto err_deinit;
+	}
+
+	dev_dbg(dev, "create resp threads ok\n");
+
+	for (unsigned int i = 0; i < ISP4SD_MAX_FW_RESP_STREAM_NUM; i++)
+		enable_irq(isp_subdev->irq[i]);
+	isp_subdev->irq_enabled = true;
+
+	return 0;
+err_deinit:
+	isp4sd_pwroff_and_deinit(sd);
+	return -EINVAL;
+}
+
+static int isp4sd_stop_stream(struct isp4_subdev *isp_subdev,
+			      struct v4l2_subdev_state *state, u32 pad)
+{
+	struct isp4sd_sensor_info *sensor_info = &isp_subdev->sensor_info;
+	struct isp4sd_output_info *output_info = &sensor_info->output_info;
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct device *dev = isp_subdev->dev;
+
+	guard(mutex)(&isp_subdev->ops_mutex);
+	dev_dbg(dev, "status %i\n", output_info->start_status);
+
+	if (output_info->start_status == ISP4SD_START_STATUS_STARTED) {
+		struct isp4fw_cmd_enable_out_ch cmd_ch_disable;
+		int ret;
+
+		/*
+		 * The struct will be shared with ISP FW, use memset() to
+		 * guarantee padding bits are zeroed, since this is not
+		 * guaranteed on all compilers.
+		 */
+		memset(&cmd_ch_disable, 0, sizeof(cmd_ch_disable));
+		cmd_ch_disable.ch = ISP4FW_ISP_PIPE_OUT_CH_PREVIEW;
+		/* `cmd_ch_disable.is_enable` is already false */
+		ret = isp4if_send_command_sync(ispif,
+					       ISP4FW_CMD_ID_ENABLE_OUT_CHAN,
+					       &cmd_ch_disable,
+					       sizeof(cmd_ch_disable));
+		if (ret)
+			dev_err(dev, "fail to disable stream\n");
+		else
+			dev_dbg(dev, "wait disable stream suc\n");
+
+		ret = isp4if_send_command_sync(ispif, ISP4FW_CMD_ID_STOP_STREAM,
+					       NULL, 0);
+		if (ret)
+			dev_err(dev, "fail to stop stream\n");
+		else
+			dev_dbg(dev, "wait stop stream suc\n");
+	}
+
+	isp4sd_uninit_stream(isp_subdev, state, pad);
+
+	/*
+	 * Return success to ensure the stop process proceeds,
+	 * and disregard any errors since they are not fatal.
+	 */
+	return 0;
+}
+
+static int isp4sd_start_stream(struct isp4_subdev *isp_subdev,
+			       struct v4l2_subdev_state *state, u32 pad)
+{
+	struct isp4sd_output_info *output_info =
+			&isp_subdev->sensor_info.output_info;
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct device *dev = isp_subdev->dev;
+	int ret;
+
+	guard(mutex)(&isp_subdev->ops_mutex);
+
+	if (ispif->status != ISP4IF_STATUS_FW_RUNNING) {
+		dev_err(dev, "fail, bad fsm %d\n", ispif->status);
+		return -EINVAL;
+	}
+
+	switch (output_info->start_status) {
+	case ISP4SD_START_STATUS_OFF:
+		break;
+	case ISP4SD_START_STATUS_STARTED:
+		dev_dbg(dev, "stream already started, do nothing\n");
+		return 0;
+	case ISP4SD_START_STATUS_START_FAIL:
+		dev_err(dev, "stream previously failed to start\n");
+		return -EINVAL;
+	}
+
+	ret = isp4sd_init_stream(isp_subdev);
+	if (ret) {
+		dev_err(dev, "fail to init isp_subdev stream\n");
+		goto err_stop_stream;
+	}
+
+	ret = isp4sd_setup_output(isp_subdev, state, pad);
+	if (ret) {
+		dev_err(dev, "fail to setup output\n");
+		goto err_stop_stream;
+	}
+
+	return 0;
+
+err_stop_stream:
+	isp4sd_stop_stream(isp_subdev, state, pad);
+	return ret;
+}
+
+int isp4sd_ioc_send_img_buf(struct v4l2_subdev *sd,
+			    struct isp4if_img_buf_info *buf_info)
+{
+	struct isp4_subdev *isp_subdev = to_isp4_subdev(sd);
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct isp4if_img_buf_node *buf_node;
+	struct device *dev = isp_subdev->dev;
+	int ret;
+
+	guard(mutex)(&isp_subdev->ops_mutex);
+
+	if (ispif->status != ISP4IF_STATUS_FW_RUNNING) {
+		dev_err(dev, "fail send img buf for bad fsm %d\n",
+			ispif->status);
+		return -EINVAL;
+	}
+
+	buf_node = isp4if_alloc_buffer_node(buf_info);
+	if (!buf_node) {
+		dev_err(dev, "fail alloc sys img buf info node\n");
+		return -ENOMEM;
+	}
+
+	ret = isp4if_queue_buffer(ispif, buf_node);
+	if (ret) {
+		dev_err(dev, "fail to queue image buf, %d\n", ret);
+		goto error_release_buf_node;
+	}
+
+	if (!isp_subdev->sensor_info.start_stream_cmd_sent) {
+		isp_subdev->sensor_info.buf_sent_cnt++;
+
+		if (isp_subdev->sensor_info.buf_sent_cnt >=
+		    ISP4SD_MIN_BUF_CNT_BEF_START_STREAM) {
+			ret = isp4if_send_command(ispif,
+						  ISP4FW_CMD_ID_START_STREAM,
+						  NULL, 0);
+			if (ret) {
+				dev_err(dev, "fail to START_STREAM");
+				goto error_release_buf_node;
+			}
+			isp_subdev->sensor_info.start_stream_cmd_sent = true;
+			isp_subdev->sensor_info.output_info.start_status =
+				ISP4SD_START_STATUS_STARTED;
+			isp_subdev->sensor_info.status =
+				ISP4SD_START_STATUS_STARTED;
+		} else {
+			dev_dbg(dev,
+				"no send start, required %u, buf sent %u\n",
+				ISP4SD_MIN_BUF_CNT_BEF_START_STREAM,
+				isp_subdev->sensor_info.buf_sent_cnt);
+		}
+	}
+
+	return 0;
+
+error_release_buf_node:
+	isp4if_dealloc_buffer_node(buf_node);
+	return ret;
+}
+
+static const struct v4l2_subdev_video_ops isp4sd_video_ops = {
+	.s_stream = v4l2_subdev_s_stream_helper,
+};
+
+static int isp4sd_set_pad_format(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *sd_state,
+				 struct v4l2_subdev_format *format)
+{
+	struct isp4sd_output_info *stream_info =
+		&(to_isp4_subdev(sd)->sensor_info.output_info);
+	struct v4l2_mbus_framefmt *fmt;
+
+	fmt = v4l2_subdev_state_get_format(sd_state, format->pad);
+
+	if (!fmt) {
+		dev_err(sd->dev, "fail to get state format\n");
+		return -EINVAL;
+	}
+
+	*fmt = format->format;
+	switch (fmt->code) {
+	case MEDIA_BUS_FMT_YUYV8_1X16:
+		stream_info->image_size = fmt->width * fmt->height * 2;
+		break;
+	case MEDIA_BUS_FMT_YUYV8_1_5X8:
+	default:
+		stream_info->image_size = fmt->width * fmt->height * 3 / 2;
+		break;
+	}
+
+	if (!stream_info->image_size) {
+		dev_err(sd->dev,
+			"fail set pad format,code 0x%x,width %u, height %u\n",
+			fmt->code, fmt->width, fmt->height);
+		return -EINVAL;
+	}
+
+	dev_dbg(sd->dev, "set pad format suc, code:%x w:%u h:%u size:%u\n",
+		fmt->code, fmt->width, fmt->height,
+		stream_info->image_size);
+
+	return 0;
+}
+
+static int isp4sd_enable_streams(struct v4l2_subdev *sd,
+				 struct v4l2_subdev_state *state, u32 pad,
+				 u64 streams_mask)
+{
+	struct isp4_subdev *isp_subdev = to_isp4_subdev(sd);
+
+	return isp4sd_start_stream(isp_subdev, state, pad);
+}
+
+static int isp4sd_disable_streams(struct v4l2_subdev *sd,
+				  struct v4l2_subdev_state *state, u32 pad,
+				  u64 streams_mask)
+{
+	struct isp4_subdev *isp_subdev = to_isp4_subdev(sd);
+
+	return isp4sd_stop_stream(isp_subdev, state, pad);
+}
+
+static const struct v4l2_subdev_pad_ops isp4sd_pad_ops = {
+	.get_fmt = v4l2_subdev_get_fmt,
+	.set_fmt = isp4sd_set_pad_format,
+	.enable_streams = isp4sd_enable_streams,
+	.disable_streams = isp4sd_disable_streams,
+};
+
+static const struct v4l2_subdev_ops isp4sd_subdev_ops = {
+	.video = &isp4sd_video_ops,
+	.pad = &isp4sd_pad_ops,
+};
+
+int isp4sd_init(struct isp4_subdev *isp_subdev, struct v4l2_device *v4l2_dev,
+		int irq[ISP4SD_MAX_FW_RESP_STREAM_NUM])
+{
+	struct isp4sd_sensor_info *sensor_info = &isp_subdev->sensor_info;
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+	struct device *dev = v4l2_dev->dev;
+	int ret;
+
+	isp_subdev->dev = dev;
+	v4l2_subdev_init(&isp_subdev->sdev, &isp4sd_subdev_ops);
+	isp_subdev->sdev.owner = THIS_MODULE;
+	isp_subdev->sdev.dev = dev;
+	snprintf(isp_subdev->sdev.name, sizeof(isp_subdev->sdev.name), "%s",
+		 dev_name(dev));
+
+	isp_subdev->sdev.entity.name = isp4sd_entity_name;
+	isp_subdev->sdev.entity.function = MEDIA_ENT_F_PROC_VIDEO_ISP;
+	isp_subdev->sdev_pad.flags = MEDIA_PAD_FL_SOURCE;
+	ret = media_entity_pads_init(&isp_subdev->sdev.entity, 1,
+				     &isp_subdev->sdev_pad);
+	if (ret) {
+		dev_err(dev, "fail to init isp4 subdev entity pad %d\n", ret);
+		return ret;
+	}
+
+	ret = v4l2_subdev_init_finalize(&isp_subdev->sdev);
+	if (ret < 0) {
+		dev_err(dev, "fail to init finalize isp4 subdev %d\n",
+			ret);
+		return ret;
+	}
+
+	ret = v4l2_device_register_subdev(v4l2_dev, &isp_subdev->sdev);
+	if (ret) {
+		dev_err(dev, "fail to register isp4 subdev to V4L2 device %d\n",
+			ret);
+		goto err_media_clean_up;
+	}
+
+	isp4if_init(ispif, dev, isp_subdev->mmio);
+
+	mutex_init(&isp_subdev->ops_mutex);
+	sensor_info->status = ISP4SD_START_STATUS_OFF;
+
+	/* create ISP enable gpio control */
+	isp_subdev->enable_gpio = devm_gpiod_get(isp_subdev->dev,
+						 "enable_isp",
+						 GPIOD_OUT_LOW);
+	if (IS_ERR(isp_subdev->enable_gpio)) {
+		ret = PTR_ERR(isp_subdev->enable_gpio);
+		dev_err(dev, "fail to get gpiod %d\n", ret);
+		goto err_subdev_unreg;
+	}
+
+	for (unsigned int i = 0; i < ISP4SD_MAX_FW_RESP_STREAM_NUM; i++)
+		isp_subdev->irq[i] = irq[i];
+
+	isp_subdev->host2fw_seq_num = 1;
+	ispif->status = ISP4IF_STATUS_PWR_OFF;
+
+	ret = isp4vid_dev_init(&isp_subdev->isp_vdev, &isp_subdev->sdev);
+	if (ret)
+		goto err_subdev_unreg;
+
+	return 0;
+
+err_subdev_unreg:
+	v4l2_device_unregister_subdev(&isp_subdev->sdev);
+err_media_clean_up:
+	v4l2_subdev_cleanup(&isp_subdev->sdev);
+	media_entity_cleanup(&isp_subdev->sdev.entity);
+	return ret;
+}
+
+void isp4sd_deinit(struct isp4_subdev *isp_subdev)
+{
+	struct isp4_interface *ispif = &isp_subdev->ispif;
+
+	isp4vid_dev_deinit(&isp_subdev->isp_vdev);
+	v4l2_device_unregister_subdev(&isp_subdev->sdev);
+	media_entity_cleanup(&isp_subdev->sdev.entity);
+	isp4if_deinit(ispif);
+	isp4sd_module_enable(isp_subdev, false);
+
+	ispif->status = ISP4IF_STATUS_PWR_OFF;
+}
diff --git a/drivers/media/platform/amd/isp4/isp4_subdev.h b/drivers/media/platform/amd/isp4/isp4_subdev.h
new file mode 100644
index 0000000000000..20ea08a830afd
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_subdev.h
@@ -0,0 +1,127 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _ISP4_SUBDEV_H_
+#define _ISP4_SUBDEV_H_
+
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/firmware.h>
+#include <linux/pm_runtime.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <media/v4l2-device.h>
+
+#include "isp4_fw_cmd_resp.h"
+#include "isp4_hw_reg.h"
+#include "isp4_interface.h"
+#include "isp4_video.h"
+
+/*
+ * One is for none sensor specific response which is not used now.
+ * Another is for sensor specific response
+ */
+#define ISP4SD_MAX_FW_RESP_STREAM_NUM 2
+
+/* Indicates the ISP status */
+enum isp4sd_status {
+	ISP4SD_STATUS_PWR_OFF,
+	ISP4SD_STATUS_PWR_ON,
+	ISP4SD_STATUS_FW_RUNNING,
+	ISP4SD_STATUS_MAX
+};
+
+/* Indicates sensor and output stream status */
+enum isp4sd_start_status {
+	ISP4SD_START_STATUS_OFF,
+	ISP4SD_START_STATUS_STARTED,
+	ISP4SD_START_STATUS_START_FAIL,
+};
+
+struct isp4sd_img_buf_node {
+	struct list_head node;
+	struct isp4if_img_buf_info buf_info;
+};
+
+/* This is ISP output after processing Bayer raw sensor input */
+struct isp4sd_output_info {
+	enum isp4sd_start_status start_status;
+	u32 image_size;
+};
+
+/*
+ * Struct for sensor info used as ISP input or source.
+ * status: sensor status.
+ * output_info: ISP output after processing the sensor input.
+ * start_stream_cmd_sent: indicates if ISP4FW_CMD_ID_START_STREAM was sent
+ * to firmware.
+ * buf_sent_cnt: number of buffers sent to receive images.
+ */
+struct isp4sd_sensor_info {
+	struct isp4sd_output_info output_info;
+	enum isp4sd_start_status status;
+	bool start_stream_cmd_sent;
+	u32 buf_sent_cnt;
+};
+
+/*
+ * The thread is created by the driver to handle firmware responses which will
+ * be waken up when a firmware-to-driver response interrupt occurs.
+ */
+struct isp4sd_thread_handler {
+	struct task_struct *thread;
+	wait_queue_head_t waitq;
+	bool resp_ready;
+};
+
+struct isp4_subdev_thread_param {
+	u32 idx;
+	struct isp4_subdev *isp_subdev;
+};
+
+struct isp4_subdev {
+	struct v4l2_subdev sdev;
+	struct isp4_interface ispif;
+	struct isp4vid_dev isp_vdev;
+
+	struct media_pad sdev_pad;
+
+	enum isp4sd_status isp_status;
+	/* mutex used to synchronize the operation with firmware */
+	struct mutex ops_mutex;
+
+	struct isp4sd_thread_handler
+		fw_resp_thread[ISP4SD_MAX_FW_RESP_STREAM_NUM];
+
+	u32 host2fw_seq_num;
+
+	struct isp4sd_sensor_info sensor_info;
+
+	/* gpio descriptor */
+	struct gpio_desc *enable_gpio;
+	struct device *dev;
+	void __iomem *mmio;
+	struct isp4_subdev_thread_param
+		isp_resp_para[ISP4SD_MAX_FW_RESP_STREAM_NUM];
+	int irq[ISP4SD_MAX_FW_RESP_STREAM_NUM];
+	bool irq_enabled;
+	/* spin lock to access ISP_SYS_INT0_EN exclusively */
+	spinlock_t irq_lock;
+#ifdef CONFIG_DEBUG_FS
+	bool enable_fw_log;
+	struct dentry *debugfs_dir;
+	char *fw_log_output;
+#endif
+};
+
+int isp4sd_init(struct isp4_subdev *isp_subdev, struct v4l2_device *v4l2_dev,
+		int irq[ISP4SD_MAX_FW_RESP_STREAM_NUM]);
+void isp4sd_deinit(struct isp4_subdev *isp_subdev);
+int isp4sd_ioc_send_img_buf(struct v4l2_subdev *sd,
+			    struct isp4if_img_buf_info *buf_info);
+int isp4sd_pwron_and_init(struct v4l2_subdev *sd);
+int isp4sd_pwroff_and_deinit(struct v4l2_subdev *sd);
+
+#endif /* _ISP4_SUBDEV_H_ */
diff --git a/drivers/media/platform/amd/isp4/isp4_video.c b/drivers/media/platform/amd/isp4/isp4_video.c
new file mode 100644
index 0000000000000..0cebb39f98e1d
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_video.c
@@ -0,0 +1,797 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#include <media/v4l2-ioctl.h>
+#include <media/v4l2-mc.h>
+#include <media/videobuf2-vmalloc.h>
+
+#include "isp4_interface.h"
+#include "isp4_subdev.h"
+#include "isp4_video.h"
+
+#define ISP4VID_ISP_DRV_NAME "amd_isp_capture"
+#define ISP4VID_MAX_PREVIEW_FPS 30
+#define ISP4VID_DEFAULT_FMT V4L2_PIX_FMT_NV12
+
+#define ISP4VID_PAD_VIDEO_OUTPUT 0
+
+/* time perframe default */
+#define ISP4VID_ISP_TPF_DEFAULT isp4vid_tpfs[0]
+
+static const char *const isp4vid_video_dev_name = "Preview";
+
+/* Sizes must be in increasing order */
+static const struct v4l2_frmsize_discrete isp4vid_frmsize[] = {
+	{640, 360},
+	{640, 480},
+	{1280, 720},
+	{1280, 960},
+	{1920, 1080},
+	{1920, 1440},
+	{2560, 1440},
+	{2880, 1620},
+	{2880, 1624},
+	{2888, 1808},
+};
+
+static const u32 isp4vid_formats[] = {
+	V4L2_PIX_FMT_NV12,
+	V4L2_PIX_FMT_YUYV
+};
+
+/* time perframe list */
+static const struct v4l2_fract isp4vid_tpfs[] = {
+	{ 1, ISP4VID_MAX_PREVIEW_FPS }
+};
+
+void isp4vid_handle_frame_done(struct isp4vid_dev *isp_vdev,
+			       const struct isp4if_img_buf_info *img_buf)
+{
+	struct isp4vid_capture_buffer *isp4vid_buf;
+	void *vbuf;
+
+	scoped_guard(mutex, &isp_vdev->buf_list_lock) {
+		isp4vid_buf = list_first_entry_or_null(&isp_vdev->buf_list,
+						       typeof(*isp4vid_buf),
+						       list);
+		if (!isp4vid_buf)
+			return;
+
+		vbuf = vb2_plane_vaddr(&isp4vid_buf->vb2.vb2_buf, 0);
+
+		if (vbuf != img_buf->planes[0].sys_addr) {
+			dev_err(isp_vdev->dev, "Invalid vbuf\n");
+			return;
+		}
+
+		list_del(&isp4vid_buf->list);
+	}
+
+	/* Fill the buffer */
+	isp4vid_buf->vb2.vb2_buf.timestamp = ktime_get_ns();
+	isp4vid_buf->vb2.sequence = isp_vdev->sequence++;
+	isp4vid_buf->vb2.field = V4L2_FIELD_ANY;
+
+	vb2_set_plane_payload(&isp4vid_buf->vb2.vb2_buf,
+			      0, isp_vdev->format.sizeimage);
+
+	vb2_buffer_done(&isp4vid_buf->vb2.vb2_buf, VB2_BUF_STATE_DONE);
+
+	dev_dbg(isp_vdev->dev, "call vb2_buffer_done(size=%u)\n",
+		isp_vdev->format.sizeimage);
+}
+
+static const struct v4l2_pix_format isp4vid_fmt_default = {
+	.width = 1920,
+	.height = 1080,
+	.pixelformat = ISP4VID_DEFAULT_FMT,
+	.field = V4L2_FIELD_NONE,
+	.colorspace = V4L2_COLORSPACE_SRGB,
+};
+
+static void isp4vid_capture_return_all_buffers(struct isp4vid_dev *isp_vdev,
+					       enum vb2_buffer_state state)
+{
+	struct isp4vid_capture_buffer *vbuf, *node;
+
+	scoped_guard(mutex, &isp_vdev->buf_list_lock) {
+		list_for_each_entry_safe(vbuf, node, &isp_vdev->buf_list, list)
+			vb2_buffer_done(&vbuf->vb2.vb2_buf, state);
+		INIT_LIST_HEAD(&isp_vdev->buf_list);
+	}
+
+	dev_dbg(isp_vdev->dev, "call vb2_buffer_done(%d)\n", state);
+}
+
+static int isp4vid_vdev_link_validate(struct media_link *link)
+{
+	return 0;
+}
+
+static const struct media_entity_operations isp4vid_vdev_ent_ops = {
+	.link_validate = isp4vid_vdev_link_validate,
+};
+
+static const struct v4l2_file_operations isp4vid_vdev_fops = {
+	.owner = THIS_MODULE,
+	.open = v4l2_fh_open,
+	.release = vb2_fop_release,
+	.read = vb2_fop_read,
+	.poll = vb2_fop_poll,
+	.unlocked_ioctl = video_ioctl2,
+	.mmap = vb2_fop_mmap,
+};
+
+static int isp4vid_ioctl_querycap(struct file *file, void *fh,
+				  struct v4l2_capability *cap)
+{
+	struct isp4vid_dev *isp_vdev = video_drvdata(file);
+
+	strscpy(cap->driver, ISP4VID_ISP_DRV_NAME, sizeof(cap->driver));
+	snprintf(cap->card, sizeof(cap->card), "%s", ISP4VID_ISP_DRV_NAME);
+	cap->capabilities |= V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE;
+
+	dev_dbg(isp_vdev->dev, "%s|capabilities=0x%X\n", isp_vdev->vdev.name,
+		cap->capabilities);
+
+	return 0;
+}
+
+static int isp4vid_g_fmt_vid_cap(struct file *file, void *priv,
+				 struct v4l2_format *f)
+{
+	struct isp4vid_dev *isp_vdev = video_drvdata(file);
+
+	f->fmt.pix = isp_vdev->format;
+
+	return 0;
+}
+
+static int isp4vid_fill_buffer_size(struct v4l2_pix_format *fmt)
+{
+	int ret = 0;
+
+	switch (fmt->pixelformat) {
+	case V4L2_PIX_FMT_NV12:
+		fmt->bytesperline = fmt->width;
+		fmt->sizeimage = fmt->bytesperline * fmt->height * 3 / 2;
+		break;
+	case V4L2_PIX_FMT_YUYV:
+		fmt->bytesperline = fmt->width * 2;
+		fmt->sizeimage = fmt->bytesperline * fmt->height;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static int isp4vid_try_fmt_vid_cap(struct file *file, void *priv,
+				   struct v4l2_format *f)
+{
+	struct isp4vid_dev *isp_vdev = video_drvdata(file);
+	struct v4l2_pix_format *format = &f->fmt.pix;
+	const struct v4l2_frmsize_discrete *fsz;
+	size_t i;
+
+	/*
+	 * Check if the hardware supports the requested format, use the default
+	 * format otherwise.
+	 */
+	for (i = 0; i < ARRAY_SIZE(isp4vid_formats); i++)
+		if (isp4vid_formats[i] == format->pixelformat)
+			break;
+
+	if (i == ARRAY_SIZE(isp4vid_formats))
+		format->pixelformat = ISP4VID_DEFAULT_FMT;
+
+	switch (format->pixelformat) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_YUYV:
+		fsz = v4l2_find_nearest_size(isp4vid_frmsize,
+					     ARRAY_SIZE(isp4vid_frmsize),
+					     width, height, format->width,
+					     format->height);
+		format->width = fsz->width;
+		format->height = fsz->height;
+		break;
+	default:
+		dev_err(isp_vdev->dev, "%s|unsupported fmt=%u\n",
+			isp_vdev->vdev.name,
+			format->pixelformat);
+		return -EINVAL;
+	}
+
+	/*
+	 * There is no need to check the return value, as failure will never
+	 * happen here
+	 */
+	isp4vid_fill_buffer_size(format);
+
+	if (format->field == V4L2_FIELD_ANY)
+		format->field = isp4vid_fmt_default.field;
+
+	if (format->colorspace == V4L2_COLORSPACE_DEFAULT)
+		format->colorspace = isp4vid_fmt_default.colorspace;
+
+	return 0;
+}
+
+static int isp4vid_set_fmt_2_isp(struct v4l2_subdev *sdev,
+				 struct v4l2_pix_format *pix_fmt)
+{
+	struct v4l2_subdev_format fmt = {};
+
+	switch (pix_fmt->pixelformat) {
+	case V4L2_PIX_FMT_NV12:
+		fmt.format.code = MEDIA_BUS_FMT_YUYV8_1_5X8;
+		break;
+	case V4L2_PIX_FMT_YUYV:
+		fmt.format.code = MEDIA_BUS_FMT_YUYV8_1X16;
+		break;
+	default:
+		return -EINVAL;
+	}
+	fmt.which = V4L2_SUBDEV_FORMAT_ACTIVE;
+	fmt.pad = ISP4VID_PAD_VIDEO_OUTPUT;
+	fmt.format.width = pix_fmt->width;
+	fmt.format.height = pix_fmt->height;
+	return v4l2_subdev_call(sdev, pad, set_fmt, NULL, &fmt);
+}
+
+static int isp4vid_s_fmt_vid_cap(struct file *file, void *priv,
+				 struct v4l2_format *f)
+{
+	struct isp4vid_dev *isp_vdev = video_drvdata(file);
+	int ret;
+
+	/* Do not change the format while stream is on */
+	if (vb2_is_busy(&isp_vdev->vbq))
+		return -EBUSY;
+
+	ret = isp4vid_try_fmt_vid_cap(file, priv, f);
+	if (ret)
+		return ret;
+
+	dev_dbg(isp_vdev->dev, "%s|width height:%ux%u->%ux%u\n",
+		isp_vdev->vdev.name,
+		isp_vdev->format.width, isp_vdev->format.height,
+		f->fmt.pix.width, f->fmt.pix.height);
+	dev_dbg(isp_vdev->dev, "%s|pixelformat:0x%x-0x%x\n",
+		isp_vdev->vdev.name, isp_vdev->format.pixelformat,
+		f->fmt.pix.pixelformat);
+	dev_dbg(isp_vdev->dev, "%s|bytesperline:%u->%u\n",
+		isp_vdev->vdev.name, isp_vdev->format.bytesperline,
+		f->fmt.pix.bytesperline);
+	dev_dbg(isp_vdev->dev, "%s|sizeimage:%u->%u\n",
+		isp_vdev->vdev.name, isp_vdev->format.sizeimage,
+		f->fmt.pix.sizeimage);
+
+	isp_vdev->format = f->fmt.pix;
+	ret = isp4vid_set_fmt_2_isp(isp_vdev->isp_sdev, &isp_vdev->format);
+
+	return ret;
+}
+
+static int isp4vid_enum_fmt_vid_cap(struct file *file, void *priv,
+				    struct v4l2_fmtdesc *f)
+{
+	struct isp4vid_dev *isp_vdev = video_drvdata(file);
+
+	switch (f->index) {
+	case 0:
+		f->pixelformat = V4L2_PIX_FMT_NV12;
+		break;
+	case 1:
+		f->pixelformat = V4L2_PIX_FMT_YUYV;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	dev_dbg(isp_vdev->dev, "%s|index=%d, pixelformat=0x%X\n",
+		isp_vdev->vdev.name, f->index, f->pixelformat);
+
+	return 0;
+}
+
+static int isp4vid_enum_framesizes(struct file *file, void *fh,
+				   struct v4l2_frmsizeenum *fsize)
+{
+	struct isp4vid_dev *isp_vdev = video_drvdata(file);
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(isp4vid_formats); i++) {
+		if (isp4vid_formats[i] == fsize->pixel_format)
+			break;
+	}
+
+	if (i == ARRAY_SIZE(isp4vid_formats))
+		return -EINVAL;
+
+	if (fsize->index < ARRAY_SIZE(isp4vid_frmsize)) {
+		fsize->type = V4L2_FRMSIZE_TYPE_DISCRETE;
+		fsize->discrete = isp4vid_frmsize[fsize->index];
+		dev_dbg(isp_vdev->dev, "%s|size[%d]=%dx%d\n",
+			isp_vdev->vdev.name, fsize->index,
+			fsize->discrete.width, fsize->discrete.height);
+	} else {
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int isp4vid_ioctl_enum_frameintervals(struct file *file, void *priv,
+					     struct v4l2_frmivalenum *fival)
+{
+	struct isp4vid_dev *isp_vdev = video_drvdata(file);
+	size_t i;
+
+	if (fival->index >= ARRAY_SIZE(isp4vid_tpfs))
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(isp4vid_formats); i++)
+		if (isp4vid_formats[i] == fival->pixel_format)
+			break;
+
+	if (i == ARRAY_SIZE(isp4vid_formats))
+		return -EINVAL;
+
+	for (i = 0; i < ARRAY_SIZE(isp4vid_frmsize); i++)
+		if (isp4vid_frmsize[i].width == fival->width &&
+		    isp4vid_frmsize[i].height == fival->height)
+			break;
+
+	if (i == ARRAY_SIZE(isp4vid_frmsize))
+		return -EINVAL;
+
+	fival->type = V4L2_FRMIVAL_TYPE_DISCRETE;
+	fival->discrete = isp4vid_tpfs[fival->index];
+	v4l2_simplify_fraction(&fival->discrete.numerator,
+			       &fival->discrete.denominator, 8, 333);
+
+	dev_dbg(isp_vdev->dev, "%s|interval[%d]=%d/%d\n",
+		isp_vdev->vdev.name, fival->index,
+		fival->discrete.numerator,
+		fival->discrete.denominator);
+
+	return 0;
+}
+
+static int isp4vid_ioctl_g_param(struct file *file, void *priv,
+				 struct v4l2_streamparm *param)
+{
+	struct v4l2_captureparm *capture = &param->parm.capture;
+	struct isp4vid_dev *isp_vdev = video_drvdata(file);
+
+	if (param->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
+		return -EINVAL;
+
+	capture->capability   = V4L2_CAP_TIMEPERFRAME;
+	capture->timeperframe = isp_vdev->timeperframe;
+	capture->readbuffers  = 0;
+
+	dev_dbg(isp_vdev->dev, "%s|timeperframe=%d/%d\n", isp_vdev->vdev.name,
+		capture->timeperframe.numerator,
+		capture->timeperframe.denominator);
+
+	return 0;
+}
+
+static const struct v4l2_ioctl_ops isp4vid_vdev_ioctl_ops = {
+	.vidioc_querycap            = isp4vid_ioctl_querycap,
+	.vidioc_enum_fmt_vid_cap    = isp4vid_enum_fmt_vid_cap,
+	.vidioc_g_fmt_vid_cap       = isp4vid_g_fmt_vid_cap,
+	.vidioc_s_fmt_vid_cap       = isp4vid_s_fmt_vid_cap,
+	.vidioc_try_fmt_vid_cap     = isp4vid_try_fmt_vid_cap,
+	.vidioc_reqbufs             = vb2_ioctl_reqbufs,
+	.vidioc_querybuf            = vb2_ioctl_querybuf,
+	.vidioc_qbuf                = vb2_ioctl_qbuf,
+	.vidioc_expbuf              = vb2_ioctl_expbuf,
+	.vidioc_dqbuf               = vb2_ioctl_dqbuf,
+	.vidioc_create_bufs         = vb2_ioctl_create_bufs,
+	.vidioc_prepare_buf         = vb2_ioctl_prepare_buf,
+	.vidioc_streamon            = vb2_ioctl_streamon,
+	.vidioc_streamoff           = vb2_ioctl_streamoff,
+	.vidioc_g_parm              = isp4vid_ioctl_g_param,
+	.vidioc_s_parm              = isp4vid_ioctl_g_param,
+	.vidioc_enum_framesizes     = isp4vid_enum_framesizes,
+	.vidioc_enum_frameintervals = isp4vid_ioctl_enum_frameintervals,
+};
+
+static unsigned int isp4vid_get_image_size(struct v4l2_pix_format *fmt)
+{
+	switch (fmt->pixelformat) {
+	case V4L2_PIX_FMT_NV12:
+		return fmt->width * fmt->height * 3 / 2;
+	case V4L2_PIX_FMT_YUYV:
+		return fmt->width * fmt->height * 2;
+	default:
+		return 0;
+	}
+}
+
+static int isp4vid_qops_queue_setup(struct vb2_queue *vq,
+				    unsigned int *nbuffers,
+				    unsigned int *nplanes, unsigned int sizes[],
+				    struct device *alloc_devs[])
+{
+	struct isp4vid_dev *isp_vdev = vb2_get_drv_priv(vq);
+	unsigned int q_num_bufs = vb2_get_num_buffers(vq);
+
+	if (*nplanes > 1) {
+		dev_err(isp_vdev->dev,
+			"fail to setup queue, no mplane supported %u\n",
+			*nplanes);
+		return -EINVAL;
+	}
+
+	if (*nplanes == 1) {
+		unsigned int size;
+
+		size = isp4vid_get_image_size(&isp_vdev->format);
+		if (sizes[0] < size) {
+			dev_err(isp_vdev->dev,
+				"fail for small plane size %u, %u expected\n",
+				sizes[0], size);
+			return -EINVAL;
+		}
+	}
+
+	if (q_num_bufs + *nbuffers < ISP4IF_MAX_STREAM_BUF_COUNT)
+		*nbuffers = ISP4IF_MAX_STREAM_BUF_COUNT - q_num_bufs;
+
+	switch (isp_vdev->format.pixelformat) {
+	case V4L2_PIX_FMT_NV12:
+	case V4L2_PIX_FMT_YUYV: {
+		*nplanes = 1;
+		sizes[0] = max(sizes[0], isp_vdev->format.sizeimage);
+		isp_vdev->format.sizeimage = sizes[0];
+	}
+	break;
+	default:
+		dev_err(isp_vdev->dev, "%s|unsupported fmt=%u\n",
+			isp_vdev->vdev.name, isp_vdev->format.pixelformat);
+		return -EINVAL;
+	}
+
+	dev_dbg(isp_vdev->dev, "%s|*nbuffers=%u *nplanes=%u sizes[0]=%u\n",
+		isp_vdev->vdev.name,
+		*nbuffers, *nplanes, sizes[0]);
+
+	return 0;
+}
+
+static void isp4vid_qops_buffer_queue(struct vb2_buffer *vb)
+{
+	struct isp4vid_capture_buffer *buf =
+		container_of(vb, struct isp4vid_capture_buffer, vb2.vb2_buf);
+	struct isp4vid_dev *isp_vdev = vb2_get_drv_priv(vb->vb2_queue);
+	struct isp4if_img_buf_info *img_buf = &buf->img_buf;
+	void *vaddr = vb2_plane_vaddr(vb, 0);
+
+	dev_dbg(isp_vdev->dev, "queue buf, vaddr %p, gpuva 0x%llx, size %u\n",
+		vaddr, buf->gpu_addr, vb->planes[0].length);
+
+	switch (isp_vdev->format.pixelformat) {
+	case V4L2_PIX_FMT_NV12: {
+		u32 y_size = isp_vdev->format.sizeimage / 3 * 2;
+		u32 uv_size = isp_vdev->format.sizeimage / 3;
+
+		img_buf->planes[0].len = y_size;
+		img_buf->planes[0].sys_addr = vaddr;
+		img_buf->planes[0].mc_addr = buf->gpu_addr;
+
+		dev_dbg(isp_vdev->dev, "img_buf[0]: mc=0x%llx size=%u\n",
+			img_buf->planes[0].mc_addr,
+			img_buf->planes[0].len);
+
+		img_buf->planes[1].len = uv_size;
+		img_buf->planes[1].sys_addr = vaddr + y_size;
+		img_buf->planes[1].mc_addr = buf->gpu_addr + y_size;
+
+		dev_dbg(isp_vdev->dev, "img_buf[1]: mc=0x%llx size=%u\n",
+			img_buf->planes[1].mc_addr,
+			img_buf->planes[1].len);
+
+		img_buf->planes[2].len = 0;
+	}
+	break;
+	case V4L2_PIX_FMT_YUYV: {
+		img_buf->planes[0].len = isp_vdev->format.sizeimage;
+		img_buf->planes[0].sys_addr = vaddr;
+		img_buf->planes[0].mc_addr = buf->gpu_addr;
+
+		dev_dbg(isp_vdev->dev, "img_buf[0]: mc=0x%llx size=%u\n",
+			img_buf->planes[0].mc_addr,
+			img_buf->planes[0].len);
+
+		img_buf->planes[1].len = 0;
+		img_buf->planes[2].len = 0;
+	}
+	break;
+	default:
+		dev_err(isp_vdev->dev, "%s|unsupported fmt=%u\n",
+			isp_vdev->vdev.name, isp_vdev->format.pixelformat);
+		return;
+	}
+
+	if (isp_vdev->stream_started)
+		isp4sd_ioc_send_img_buf(isp_vdev->isp_sdev, img_buf);
+
+	scoped_guard(mutex, &isp_vdev->buf_list_lock)
+		list_add_tail(&buf->list, &isp_vdev->buf_list);
+}
+
+static int isp4vid_qops_start_streaming(struct vb2_queue *vq,
+					unsigned int count)
+{
+	struct isp4vid_dev *isp_vdev = vb2_get_drv_priv(vq);
+	struct isp4vid_capture_buffer *isp4vid_buf;
+	struct media_entity *entity;
+	struct v4l2_subdev *subdev;
+	struct media_pad *pad;
+	int ret = 0;
+
+	isp_vdev->sequence = 0;
+
+	ret = isp4sd_pwron_and_init(isp_vdev->isp_sdev);
+	if (ret) {
+		dev_err(isp_vdev->dev, "power up isp fail %d\n", ret);
+		goto release_buffers;
+	}
+
+	entity = &isp_vdev->vdev.entity;
+	while (1) {
+		pad = &entity->pads[0];
+		if (!(pad->flags & MEDIA_PAD_FL_SINK))
+			break;
+
+		pad = media_pad_remote_pad_first(pad);
+		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+			break;
+
+		entity = pad->entity;
+		subdev = media_entity_to_v4l2_subdev(entity);
+
+		ret = v4l2_subdev_call(subdev, video, s_stream, 1);
+		if (ret < 0 && ret != -ENOIOCTLCMD) {
+			dev_dbg(isp_vdev->dev, "fail start streaming: %s %d\n",
+				subdev->name, ret);
+			goto release_buffers;
+		}
+	}
+
+	list_for_each_entry(isp4vid_buf, &isp_vdev->buf_list, list)
+		isp4sd_ioc_send_img_buf(isp_vdev->isp_sdev,
+					&isp4vid_buf->img_buf);
+
+	isp_vdev->stream_started = true;
+
+	return 0;
+
+release_buffers:
+	isp4vid_capture_return_all_buffers(isp_vdev, VB2_BUF_STATE_QUEUED);
+	return ret;
+}
+
+static void isp4vid_qops_stop_streaming(struct vb2_queue *vq)
+{
+	struct isp4vid_dev *isp_vdev = vb2_get_drv_priv(vq);
+	struct media_entity *entity;
+	struct v4l2_subdev *subdev;
+	struct media_pad *pad;
+	int ret;
+
+	entity = &isp_vdev->vdev.entity;
+	while (1) {
+		pad = &entity->pads[0];
+		if (!(pad->flags & MEDIA_PAD_FL_SINK))
+			break;
+
+		pad = media_pad_remote_pad_first(pad);
+		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
+			break;
+
+		entity = pad->entity;
+		subdev = media_entity_to_v4l2_subdev(entity);
+
+		ret = v4l2_subdev_call(subdev, video, s_stream, 0);
+
+		if (ret < 0 && ret != -ENOIOCTLCMD)
+			dev_dbg(isp_vdev->dev, "fail stop streaming: %s %d\n",
+				subdev->name, ret);
+	}
+
+	isp_vdev->stream_started = false;
+	isp4sd_pwroff_and_deinit(isp_vdev->isp_sdev);
+
+	/* Release all active buffers */
+	isp4vid_capture_return_all_buffers(isp_vdev, VB2_BUF_STATE_ERROR);
+}
+
+static int isp4vid_qops_buf_init(struct vb2_buffer *vb)
+{
+	struct isp4vid_capture_buffer *buf =
+		container_of(vb, struct isp4vid_capture_buffer, vb2.vb2_buf);
+	struct isp4vid_dev *isp_vdev = vb2_get_drv_priv(vb->vb2_queue);
+	void *mem_priv = vb->planes[0].mem_priv;
+	struct device *dev = isp_vdev->dev;
+	u64 gpu_addr;
+	void *bo;
+	int ret;
+
+	if (vb->planes[0].dbuf) {
+		buf->dbuf = vb->planes[0].dbuf;
+	} else {
+		/*
+		 * HAS_DMA is a Kconfig dependency so CONFIG_HAS_DMA is always
+		 * defined when this driver is compiled. The #else branch is
+		 * kept as a safeguard in case the dependency is ever removed.
+		 */
+#ifdef CONFIG_HAS_DMA
+		buf->dbuf = vb2_vmalloc_memops.get_dmabuf(vb, mem_priv, 0);
+		if (IS_ERR_OR_NULL(buf->dbuf)) {
+			dev_err(dev, "fail to get dma buf\n");
+			return -EINVAL;
+		}
+#else
+		dev_err(dev, "get dmabuf fail -- CONFIG_HAS_DMA not defined\n");
+		buf->dbuf = NULL;
+		return -EINVAL;
+#endif
+	}
+
+	/* create isp user BO and obtain gpu_addr */
+	ret = isp_user_buffer_alloc(dev, buf->dbuf, &bo, &gpu_addr);
+	if (ret) {
+		dev_err(dev, "fail to create isp user BO\n");
+		if (!vb->planes[0].dbuf) {
+			dma_buf_put(buf->dbuf);
+			buf->dbuf = NULL;
+		}
+
+		return ret;
+	}
+
+	buf->bo = bo;
+	buf->gpu_addr = gpu_addr;
+	return 0;
+}
+
+static void isp4vid_qops_buf_cleanup(struct vb2_buffer *vb)
+{
+	struct isp4vid_capture_buffer *buf =
+		container_of(vb, struct isp4vid_capture_buffer, vb2.vb2_buf);
+
+	if (buf->bo) {
+		isp_user_buffer_free(buf->bo);
+		buf->bo = NULL;
+	}
+
+	/*
+	 * Only put dmabufs we obtained ourselves via get_dmabuf, not ones
+	 * provided by the framework for DMABUF import
+	 */
+	if (buf->dbuf && buf->dbuf != vb->planes[0].dbuf)
+		dma_buf_put(buf->dbuf);
+
+	buf->dbuf = NULL;
+}
+
+static const struct vb2_ops isp4vid_qops = {
+	.queue_setup = isp4vid_qops_queue_setup,
+	.buf_init = isp4vid_qops_buf_init,
+	.buf_cleanup = isp4vid_qops_buf_cleanup,
+	.start_streaming = isp4vid_qops_start_streaming,
+	.stop_streaming = isp4vid_qops_stop_streaming,
+	.buf_queue = isp4vid_qops_buffer_queue,
+};
+
+int isp4vid_dev_init(struct isp4vid_dev *isp_vdev, struct v4l2_subdev *isp_sd)
+{
+	const char *vdev_name = isp4vid_video_dev_name;
+	struct v4l2_device *v4l2_dev;
+	struct video_device *vdev;
+	struct vb2_queue *q;
+	int ret;
+
+	if (!isp_vdev || !isp_sd || !isp_sd->v4l2_dev)
+		return -EINVAL;
+
+	v4l2_dev = isp_sd->v4l2_dev;
+	vdev = &isp_vdev->vdev;
+
+	isp_vdev->isp_sdev = isp_sd;
+	isp_vdev->dev = v4l2_dev->dev;
+
+	/* Initialize the vb2_queue struct */
+	mutex_init(&isp_vdev->vbq_lock);
+	q = &isp_vdev->vbq;
+	q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
+	q->io_modes = VB2_MMAP | VB2_DMABUF;
+	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
+	q->buf_struct_size = sizeof(struct isp4vid_capture_buffer);
+	q->min_queued_buffers = 2;
+	q->ops = &isp4vid_qops;
+	q->drv_priv = isp_vdev;
+	q->mem_ops = &vb2_vmalloc_memops;
+	q->lock = &isp_vdev->vbq_lock;
+	q->dev = v4l2_dev->dev;
+	ret = vb2_queue_init(q);
+	if (ret) {
+		dev_err(v4l2_dev->dev, "vb2_queue_init error:%d\n", ret);
+		return ret;
+	}
+
+	/* Initialize buffer list and its lock */
+	mutex_init(&isp_vdev->buf_list_lock);
+	INIT_LIST_HEAD(&isp_vdev->buf_list);
+
+	/* Set default frame format */
+	isp_vdev->format = isp4vid_fmt_default;
+	isp_vdev->timeperframe = ISP4VID_ISP_TPF_DEFAULT;
+	v4l2_simplify_fraction(&isp_vdev->timeperframe.numerator,
+			       &isp_vdev->timeperframe.denominator, 8, 333);
+
+	ret = isp4vid_fill_buffer_size(&isp_vdev->format);
+	if (ret) {
+		dev_err(v4l2_dev->dev, "fail to fill buffer size: %d\n", ret);
+		goto err_release_vb2_queue;
+	}
+
+	ret = isp4vid_set_fmt_2_isp(isp_sd, &isp_vdev->format);
+	if (ret) {
+		dev_err(v4l2_dev->dev, "fail init format :%d\n", ret);
+		goto err_release_vb2_queue;
+	}
+
+	/* Initialize the video_device struct */
+	isp_vdev->vdev.entity.name = vdev_name;
+	isp_vdev->vdev.entity.function = MEDIA_ENT_F_IO_V4L;
+	isp_vdev->vdev_pad.flags = MEDIA_PAD_FL_SINK;
+	ret = media_entity_pads_init(&isp_vdev->vdev.entity, 1,
+				     &isp_vdev->vdev_pad);
+
+	if (ret) {
+		dev_err(v4l2_dev->dev, "init media entity pad fail:%d\n", ret);
+		goto err_release_vb2_queue;
+	}
+
+	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE |
+			    V4L2_CAP_STREAMING | V4L2_CAP_IO_MC;
+	vdev->entity.ops = &isp4vid_vdev_ent_ops;
+	vdev->release = video_device_release_empty;
+	vdev->fops = &isp4vid_vdev_fops;
+	vdev->ioctl_ops = &isp4vid_vdev_ioctl_ops;
+	vdev->lock = NULL;
+	vdev->queue = q;
+	vdev->v4l2_dev = v4l2_dev;
+	vdev->vfl_dir = VFL_DIR_RX;
+	strscpy(vdev->name, vdev_name, sizeof(vdev->name));
+	video_set_drvdata(vdev, isp_vdev);
+
+	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
+	if (ret) {
+		dev_err(v4l2_dev->dev, "register video device fail:%d\n", ret);
+		goto err_entity_cleanup;
+	}
+
+	return 0;
+
+err_entity_cleanup:
+	media_entity_cleanup(&isp_vdev->vdev.entity);
+err_release_vb2_queue:
+	vb2_queue_release(q);
+	return ret;
+}
+
+void isp4vid_dev_deinit(struct isp4vid_dev *isp_vdev)
+{
+	vb2_video_unregister_device(&isp_vdev->vdev);
+}
diff --git a/drivers/media/platform/amd/isp4/isp4_video.h b/drivers/media/platform/amd/isp4/isp4_video.h
new file mode 100644
index 0000000000000..c66451e261664
--- /dev/null
+++ b/drivers/media/platform/amd/isp4/isp4_video.h
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (C) 2025 Advanced Micro Devices, Inc.
+ */
+
+#ifndef _ISP4_VIDEO_H_
+#define _ISP4_VIDEO_H_
+
+#include <media/v4l2-dev.h>
+#include <media/videobuf2-memops.h>
+
+#include "isp4_interface.h"
+
+struct isp4vid_capture_buffer {
+	/*
+	 * struct vb2_v4l2_buffer must be the first element
+	 * the videobuf2 framework will allocate this struct based on
+	 * buf_struct_size and use the first sizeof(struct vb2_buffer) bytes of
+	 * memory as a vb2_buffer
+	 */
+	struct vb2_v4l2_buffer vb2;
+	struct isp4if_img_buf_info img_buf;
+	struct list_head list;
+	struct dma_buf *dbuf;
+	void *bo;
+	u64 gpu_addr;
+};
+
+struct isp4vid_dev {
+	struct video_device vdev;
+	struct media_pad vdev_pad;
+	struct v4l2_pix_format format;
+
+	/* mutex that protects vbq */
+	struct mutex vbq_lock;
+	struct vb2_queue vbq;
+
+	/* mutex that protects buf_list */
+	struct mutex buf_list_lock;
+	struct list_head buf_list;
+
+	u32 sequence;
+	bool stream_started;
+
+	struct device *dev;
+	struct v4l2_subdev *isp_sdev;
+	struct v4l2_fract timeperframe;
+};
+
+int isp4vid_dev_init(struct isp4vid_dev *isp_vdev, struct v4l2_subdev *isp_sd);
+
+void isp4vid_dev_deinit(struct isp4vid_dev *isp_vdev);
+
+void isp4vid_handle_frame_done(struct isp4vid_dev *isp_vdev,
+			       const struct isp4if_img_buf_info *img_buf);
+
+#endif /* _ISP4_VIDEO_H_ */
diff --git a/drivers/media/platform/amlogic/c3/isp/c3-isp-params.c b/drivers/media/platform/amlogic/c3/isp/c3-isp-params.c
index 6f9ca7a7dd882..aec3eed0e443e 100644
--- a/drivers/media/platform/amlogic/c3/isp/c3-isp-params.c
+++ b/drivers/media/platform/amlogic/c3/isp/c3-isp-params.c
@@ -104,6 +104,8 @@ static void c3_isp_params_awb_wt(struct c3_isp_device *isp,
 	c3_isp_write(isp, ISP_AWB_BLK_WT_ADDR, 0);
 
 	zones_num = cfg->horiz_zones_num * cfg->vert_zones_num;
+	if (zones_num > C3_ISP_AWB_MAX_ZONES)
+		zones_num = C3_ISP_AWB_MAX_ZONES;
 
 	/* Need to write 8 weights at once */
 	for (i = 0; i < zones_num / 8; i++) {
@@ -220,6 +222,8 @@ static void c3_isp_params_ae_wt(struct c3_isp_device *isp,
 	c3_isp_write(isp, ISP_AE_BLK_WT_ADDR, 0);
 
 	zones_num = cfg->horiz_zones_num * cfg->vert_zones_num;
+	if (zones_num > C3_ISP_AE_MAX_ZONES)
+		zones_num = C3_ISP_AE_MAX_ZONES;
 
 	/* Need to write 8 weights at once */
 	for (i = 0; i < zones_num / 8; i++) {
diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-core.c b/drivers/media/platform/arm/mali-c55/mali-c55-core.c
index c1a562cd214ec..ee4a4267415e5 100644
--- a/drivers/media/platform/arm/mali-c55/mali-c55-core.c
+++ b/drivers/media/platform/arm/mali-c55/mali-c55-core.c
@@ -458,7 +458,7 @@ static int mali_c55_media_frameworks_init(struct mali_c55 *mali_c55)
 	if (ret) {
 		dev_err(mali_c55->dev, "failed to register V4L2 device\n");
 		goto err_unregister_media_device;
-	};
+	}
 
 	mali_c55->notifier.ops = &mali_c55_notifier_ops;
 	v4l2_async_nf_init(&mali_c55->notifier, &mali_c55->v4l2_dev);
@@ -806,8 +806,10 @@ static int mali_c55_probe(struct platform_device *pdev)
 	vb2_dma_contig_set_max_seg_size(dev, UINT_MAX);
 
 	ret = __mali_c55_power_on(mali_c55);
-	if (ret)
-		return dev_err_probe(dev, ret, "failed to power on\n");
+	if (ret) {
+		dev_err_probe(dev, ret, "failed to power on\n");
+		goto err_release_mem;
+	}
 
 	ret = mali_c55_check_hwcfg(mali_c55);
 	if (ret)
@@ -826,14 +828,13 @@ static int mali_c55_probe(struct platform_device *pdev)
 
 	ret = mali_c55_media_frameworks_init(mali_c55);
 	if (ret)
-		goto err_free_context_registers;
+		goto err_pm_runtime_disable;
 
 	pm_runtime_idle(&pdev->dev);
 
 	mali_c55->irqnum = platform_get_irq(pdev, 0);
 	if (mali_c55->irqnum < 0) {
 		ret = mali_c55->irqnum;
-		dev_err(dev, "failed to get interrupt\n");
 		goto err_deinit_media_frameworks;
 	}
 
@@ -841,11 +842,14 @@ static int mali_c55_probe(struct platform_device *pdev)
 
 err_deinit_media_frameworks:
 	mali_c55_media_frameworks_deinit(mali_c55);
+err_pm_runtime_disable:
+	pm_runtime_set_suspended(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-err_free_context_registers:
 	kfree(mali_c55->context.registers);
 err_power_off:
 	__mali_c55_power_off(mali_c55);
+err_release_mem:
+	of_reserved_mem_device_release(dev);
 
 	return ret;
 }
@@ -854,8 +858,14 @@ static void mali_c55_remove(struct platform_device *pdev)
 {
 	struct mali_c55 *mali_c55 = platform_get_drvdata(pdev);
 
-	kfree(mali_c55->context.registers);
 	mali_c55_media_frameworks_deinit(mali_c55);
+	if (!pm_runtime_suspended(&pdev->dev)) {
+		__mali_c55_power_off(mali_c55);
+		pm_runtime_set_suspended(&pdev->dev);
+	}
+	pm_runtime_disable(&pdev->dev);
+	kfree(mali_c55->context.registers);
+	of_reserved_mem_device_release(&pdev->dev);
 }
 
 static const struct of_device_id mali_c55_of_match[] = {
diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-isp.c b/drivers/media/platform/arm/mali-c55/mali-c55-isp.c
index 4c0fd1ec741c3..e128adf6ee37e 100644
--- a/drivers/media/platform/arm/mali-c55/mali-c55-isp.c
+++ b/drivers/media/platform/arm/mali-c55/mali-c55-isp.c
@@ -333,6 +333,13 @@ static int mali_c55_isp_enable_streams(struct v4l2_subdev *sd,
 
 	sink_pad = &isp->pads[MALI_C55_ISP_PAD_SINK_VIDEO];
 	isp->remote_src = media_pad_remote_pad_unique(sink_pad);
+	if (IS_ERR(isp->remote_src))  {
+		ret = PTR_ERR(isp->remote_src);
+		dev_err(mali_c55->dev, "Failed to get remote source pad: %d\n", ret);
+		isp->remote_src = NULL;
+		return ret;
+	}
+
 	src_sd = media_entity_to_v4l2_subdev(isp->remote_src->entity);
 
 	isp->frame_sequence = 0;
@@ -583,6 +590,7 @@ int mali_c55_register_isp(struct mali_c55 *mali_c55)
 	sd->entity.ops = &mali_c55_isp_media_ops;
 	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_ISP;
 	sd->internal_ops = &mali_c55_isp_internal_ops;
+	sd->dev = mali_c55->dev;
 	strscpy(sd->name, MALI_C55_DRIVER_NAME " isp", sizeof(sd->name));
 
 	isp->pads[MALI_C55_ISP_PAD_SINK_VIDEO].flags = MEDIA_PAD_FL_SINK |
diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c b/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c
index a8d739af74b6c..c4f46651dcee3 100644
--- a/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c
+++ b/drivers/media/platform/arm/mali-c55/mali-c55-resizer.c
@@ -1070,6 +1070,7 @@ static int mali_c55_register_resizer(struct mali_c55 *mali_c55,
 	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_STREAMS;
 	sd->entity.function = MEDIA_ENT_F_PROC_VIDEO_SCALER;
 	sd->internal_ops = &mali_c55_resizer_internal_ops;
+	sd->dev = mali_c55->dev;
 
 	rsz->pads[MALI_C55_RSZ_SINK_PAD].flags = MEDIA_PAD_FL_SINK;
 	rsz->pads[MALI_C55_RSZ_SOURCE_PAD].flags = MEDIA_PAD_FL_SOURCE;
diff --git a/drivers/media/platform/arm/mali-c55/mali-c55-tpg.c b/drivers/media/platform/arm/mali-c55/mali-c55-tpg.c
index 1af5d2759a839..894f4cf377afc 100644
--- a/drivers/media/platform/arm/mali-c55/mali-c55-tpg.c
+++ b/drivers/media/platform/arm/mali-c55/mali-c55-tpg.c
@@ -370,6 +370,7 @@ int mali_c55_register_tpg(struct mali_c55 *mali_c55)
 	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_HAS_EVENTS;
 	sd->entity.function = MEDIA_ENT_F_CAM_SENSOR;
 	sd->internal_ops = &mali_c55_tpg_internal_ops;
+	sd->dev = mali_c55->dev;
 	strscpy(sd->name, MALI_C55_DRIVER_NAME " tpg", sizeof(sd->name));
 
 	pad->flags = MEDIA_PAD_FL_SOURCE;
diff --git a/drivers/media/platform/aspeed/aspeed-video.c b/drivers/media/platform/aspeed/aspeed-video.c
index 41cb96f601102..a292275f6b7b9 100644
--- a/drivers/media/platform/aspeed/aspeed-video.c
+++ b/drivers/media/platform/aspeed/aspeed-video.c
@@ -2343,6 +2343,7 @@ static int aspeed_video_probe(struct platform_device *pdev)
 	rc = aspeed_video_setup_video(video);
 	if (rc) {
 		aspeed_video_free_buf(video, &video->jpeg);
+		of_reserved_mem_device_release(&pdev->dev);
 		clk_unprepare(video->vclk);
 		clk_unprepare(video->eclk);
 		return rc;
diff --git a/drivers/media/platform/cadence/Kconfig b/drivers/media/platform/cadence/Kconfig
index 1aa608c00dbce..ea85ef82760e6 100644
--- a/drivers/media/platform/cadence/Kconfig
+++ b/drivers/media/platform/cadence/Kconfig
@@ -5,6 +5,7 @@ comment "Cadence media platform drivers"
 config VIDEO_CADENCE_CSI2RX
 	tristate "Cadence MIPI-CSI2 RX Controller"
 	depends on VIDEO_DEV
+	depends on PM
 	select MEDIA_CONTROLLER
 	select VIDEO_V4L2_SUBDEV_API
 	select V4L2_FWNODE
diff --git a/drivers/media/platform/cadence/cdns-csi2rx.c b/drivers/media/platform/cadence/cdns-csi2rx.c
index cde690c6fdee6..1ff2d8f78d5b4 100644
--- a/drivers/media/platform/cadence/cdns-csi2rx.c
+++ b/drivers/media/platform/cadence/cdns-csi2rx.c
@@ -125,12 +125,6 @@ struct csi2rx_priv {
 	unsigned int			count;
 	int				error_irq;
 
-	/*
-	 * Used to prevent race conditions between multiple,
-	 * concurrent calls to start and stop.
-	 */
-	struct mutex			lock;
-
 	void __iomem			*base;
 	struct clk			*sys_clk;
 	struct clk			*p_clk;
@@ -141,6 +135,7 @@ struct csi2rx_priv {
 	struct phy			*dphy;
 
 	u8				num_pixels[CSI2RX_STREAMS_MAX];
+	u32				vc_select[CSI2RX_STREAMS_MAX];
 	u8				lanes[CSI2RX_LANES_MAX];
 	u8				num_lanes;
 	u8				max_lanes;
@@ -235,6 +230,21 @@ static const struct csi2rx_fmt *csi2rx_get_fmt_by_code(u32 code)
 	return NULL;
 }
 
+static int csi2rx_get_frame_desc_from_source(struct csi2rx_priv *csi2rx,
+					     struct v4l2_mbus_frame_desc *fd)
+{
+	struct media_pad *remote_pad;
+
+	remote_pad = media_entity_remote_source_pad_unique(&csi2rx->subdev.entity);
+	if (IS_ERR(remote_pad)) {
+		dev_err(csi2rx->dev, "No remote pad found for sink\n");
+		return PTR_ERR(remote_pad);
+	}
+
+	return v4l2_subdev_call(csi2rx->source_subdev, pad, get_frame_desc,
+				remote_pad->index, fd);
+}
+
 static inline
 struct csi2rx_priv *v4l2_subdev_to_csi2rx(struct v4l2_subdev *subdev)
 {
@@ -264,29 +274,46 @@ static void csi2rx_reset(struct csi2rx_priv *csi2rx)
 
 static int csi2rx_configure_ext_dphy(struct csi2rx_priv *csi2rx)
 {
-	struct media_pad *src_pad =
-		&csi2rx->source_subdev->entity.pads[csi2rx->source_pad];
 	union phy_configure_opts opts = { };
 	struct phy_configure_opts_mipi_dphy *cfg = &opts.mipi_dphy;
-	struct v4l2_subdev_format sd_fmt = {
-		.which	= V4L2_SUBDEV_FORMAT_ACTIVE,
-		.pad	= CSI2RX_PAD_SINK,
-	};
+	struct v4l2_mbus_framefmt *framefmt;
+	struct v4l2_subdev_state *state;
 	const struct csi2rx_fmt *fmt;
+	struct v4l2_subdev_route *route;
+	int source_pad = csi2rx->source_pad;
+	struct media_pad *pad = &csi2rx->source_subdev->entity.pads[source_pad];
 	s64 link_freq;
 	int ret;
+	u32 bpp;
 
-	ret = v4l2_subdev_call_state_active(&csi2rx->subdev, pad, get_fmt,
-					    &sd_fmt);
-	if (ret < 0)
-		return ret;
+	state = v4l2_subdev_get_locked_active_state(&csi2rx->subdev);
+
+	/*
+	 * For multi-stream transmitters there is no single pixel rate.
+	 *
+	 * In multistream usecase pass bpp as 0 so that v4l2_get_link_freq()
+	 * returns an error if it falls back to V4L2_CID_PIXEL_RATE.
+	 */
+	if (state->routing.num_routes > 1) {
+		bpp = 0;
+	} else {
+		route = &state->routing.routes[0];
+		framefmt = v4l2_subdev_state_get_format(state, CSI2RX_PAD_SINK,
+							route->sink_stream);
+		if (!framefmt) {
+			dev_err(csi2rx->dev, "Did not find active sink format\n");
+			return -EINVAL;
+		}
 
-	fmt = csi2rx_get_fmt_by_code(sd_fmt.format.code);
+		fmt = csi2rx_get_fmt_by_code(framefmt->code);
+		bpp = fmt->bpp;
+	}
 
-	link_freq = v4l2_get_link_freq(src_pad,
-				       fmt->bpp, 2 * csi2rx->num_lanes);
-	if (link_freq < 0)
+	link_freq = v4l2_get_link_freq(pad, bpp, 2 * csi2rx->num_lanes);
+	if (link_freq < 0) {
+		dev_err(csi2rx->dev, "Unable to calculate link frequency\n");
 		return link_freq;
+	}
 
 	ret = phy_mipi_dphy_get_default_config_for_hsclk(link_freq,
 							 csi2rx->num_lanes, cfg);
@@ -313,11 +340,6 @@ static int csi2rx_start(struct csi2rx_priv *csi2rx)
 	u32 reg;
 	int ret;
 
-	ret = clk_prepare_enable(csi2rx->p_clk);
-	if (ret)
-		return ret;
-
-	reset_control_deassert(csi2rx->p_rst);
 	csi2rx_reset(csi2rx);
 
 	if (csi2rx->error_irq >= 0)
@@ -358,7 +380,7 @@ static int csi2rx_start(struct csi2rx_priv *csi2rx)
 		if (ret) {
 			dev_err(csi2rx->dev,
 				"Failed to configure external DPHY: %d\n", ret);
-			goto err_disable_pclk;
+			return ret;
 		}
 	}
 
@@ -373,58 +395,20 @@ static int csi2rx_start(struct csi2rx_priv *csi2rx)
 	 * hence the reference counting.
 	 */
 	for (i = 0; i < csi2rx->max_streams; i++) {
-		ret = clk_prepare_enable(csi2rx->pixel_clk[i]);
-		if (ret)
-			goto err_disable_pixclk;
-
-		reset_control_deassert(csi2rx->pixel_rst[i]);
-
 		writel(CSI2RX_STREAM_CFG_FIFO_MODE_LARGE_BUF |
 			       FIELD_PREP(CSI2RX_STREAM_CFG_NUM_PIXELS_MASK,
 					  csi2rx->num_pixels[i]),
 		       csi2rx->base + CSI2RX_STREAM_CFG_REG(i));
 
-		/*
-		 * Enable one virtual channel. When multiple virtual channels
-		 * are supported this will have to be changed.
-		 */
-		writel(CSI2RX_STREAM_DATA_CFG_VC_SELECT(0),
+		writel(csi2rx->vc_select[i],
 		       csi2rx->base + CSI2RX_STREAM_DATA_CFG_REG(i));
 
 		writel(CSI2RX_STREAM_CTRL_START,
 		       csi2rx->base + CSI2RX_STREAM_CTRL_REG(i));
 	}
 
-	ret = clk_prepare_enable(csi2rx->sys_clk);
-	if (ret)
-		goto err_disable_pixclk;
-
-	reset_control_deassert(csi2rx->sys_rst);
-
-	ret = v4l2_subdev_call(csi2rx->source_subdev, video, s_stream, true);
-	if (ret)
-		goto err_disable_sysclk;
-
-	clk_disable_unprepare(csi2rx->p_clk);
 
 	return 0;
-
-err_disable_sysclk:
-	clk_disable_unprepare(csi2rx->sys_clk);
-err_disable_pixclk:
-	for (; i > 0; i--) {
-		reset_control_assert(csi2rx->pixel_rst[i - 1]);
-		clk_disable_unprepare(csi2rx->pixel_clk[i - 1]);
-	}
-
-	if (csi2rx->dphy) {
-		writel(0, csi2rx->base + CSI2RX_DPHY_LANE_CTRL_REG);
-		phy_power_off(csi2rx->dphy);
-	}
-err_disable_pclk:
-	clk_disable_unprepare(csi2rx->p_clk);
-
-	return ret;
 }
 
 static void csi2rx_stop(struct csi2rx_priv *csi2rx)
@@ -433,10 +417,6 @@ static void csi2rx_stop(struct csi2rx_priv *csi2rx)
 	u32 val;
 	int ret;
 
-	clk_prepare_enable(csi2rx->p_clk);
-	reset_control_assert(csi2rx->sys_rst);
-	clk_disable_unprepare(csi2rx->sys_clk);
-
 	writel(0, csi2rx->base + CSI2RX_ERROR_IRQS_MASK_REG);
 
 	for (i = 0; i < csi2rx->max_streams; i++) {
@@ -451,17 +431,8 @@ static void csi2rx_stop(struct csi2rx_priv *csi2rx)
 		if (ret)
 			dev_warn(csi2rx->dev,
 				 "Failed to stop streaming on pad%u\n", i);
-
-		reset_control_assert(csi2rx->pixel_rst[i]);
-		clk_disable_unprepare(csi2rx->pixel_clk[i]);
 	}
 
-	reset_control_assert(csi2rx->p_rst);
-	clk_disable_unprepare(csi2rx->p_clk);
-
-	if (v4l2_subdev_call(csi2rx->source_subdev, video, s_stream, false))
-		dev_warn(csi2rx->dev, "Couldn't disable our subdev\n");
-
 	if (csi2rx->dphy) {
 		writel(0, csi2rx->base + CSI2RX_DPHY_LANE_CTRL_REG);
 
@@ -485,40 +456,117 @@ static int csi2rx_log_status(struct v4l2_subdev *sd)
 	return 0;
 }
 
-static int csi2rx_s_stream(struct v4l2_subdev *subdev, int enable)
+static void csi2rx_update_vc_select(struct csi2rx_priv *csi2rx,
+				    struct v4l2_subdev_state *state)
 {
-	struct csi2rx_priv *csi2rx = v4l2_subdev_to_csi2rx(subdev);
-	int ret = 0;
+	struct v4l2_mbus_frame_desc fd = {0};
+	struct v4l2_subdev_route *route;
+	unsigned int i;
+	int ret;
+
+	ret = csi2rx_get_frame_desc_from_source(csi2rx, &fd);
+	if (ret || fd.type != V4L2_MBUS_FRAME_DESC_TYPE_CSI2) {
+		dev_dbg(csi2rx->dev,
+			"Failed to get source frame desc, allowing only VC=0\n");
+		for (i = 0; i < CSI2RX_STREAMS_MAX; i++)
+			csi2rx->vc_select[i] = CSI2RX_STREAM_DATA_CFG_VC_SELECT(0);
+		return;
+	}
+
+	/* If source provides per-stream VC info, use it to filter by VC */
+	memset(csi2rx->vc_select, 0, sizeof(csi2rx->vc_select));
+
+	for_each_active_route(&state->routing, route) {
+		u32 cdns_stream = route->source_pad - CSI2RX_PAD_SOURCE_STREAM0;
+
+		for (i = 0; i < fd.num_entries; i++) {
+			if (fd.entry[i].stream != route->sink_stream)
+				continue;
 
-	mutex_lock(&csi2rx->lock);
-
-	if (enable) {
-		/*
-		 * If we're not the first users, there's no need to
-		 * enable the whole controller.
-		 */
-		if (!csi2rx->count) {
-			ret = csi2rx_start(csi2rx);
-			if (ret)
-				goto out;
+			csi2rx->vc_select[cdns_stream] |=
+				CSI2RX_STREAM_DATA_CFG_VC_SELECT(fd.entry[i].bus.csi2.vc);
 		}
+	}
+}
 
-		csi2rx->count++;
-	} else {
-		csi2rx->count--;
+static int csi2rx_enable_streams(struct v4l2_subdev *subdev,
+				 struct v4l2_subdev_state *state, u32 pad,
+				 u64 streams_mask)
+{
+	struct csi2rx_priv *csi2rx = v4l2_subdev_to_csi2rx(subdev);
+	u64 sink_streams;
+	int ret;
 
-		/*
-		 * Let the last user turn off the lights.
-		 */
-		if (!csi2rx->count)
-			csi2rx_stop(csi2rx);
+	sink_streams = v4l2_subdev_state_xlate_streams(state, pad,
+						       CSI2RX_PAD_SINK,
+						       &streams_mask);
+
+	/*
+	 * If we're not the first users, there's no need to
+	 * enable the whole controller.
+	 */
+	if (!csi2rx->count) {
+		ret = pm_runtime_resume_and_get(csi2rx->dev);
+		if (ret < 0)
+			goto err;
+
+		csi2rx_update_vc_select(csi2rx, state);
+
+		ret = csi2rx_start(csi2rx);
+		if (ret)
+			goto err_put_pm;
 	}
 
-out:
-	mutex_unlock(&csi2rx->lock);
+	/* Start streaming on the source */
+	ret = v4l2_subdev_enable_streams(csi2rx->source_subdev, csi2rx->source_pad,
+					 sink_streams);
+	if (ret) {
+		dev_err(csi2rx->dev,
+			"Failed to start streams %#llx on subdev\n",
+			sink_streams);
+		goto err_stop_csi;
+	}
+
+	csi2rx->count++;
+	return 0;
+
+err_stop_csi:
+	if (!csi2rx->count)
+		csi2rx_stop(csi2rx);
+err_put_pm:
+	if (!csi2rx->count)
+		pm_runtime_put(csi2rx->dev);
+err:
 	return ret;
 }
 
+static int csi2rx_disable_streams(struct v4l2_subdev *subdev,
+				  struct v4l2_subdev_state *state, u32 pad,
+				  u64 streams_mask)
+{
+	struct csi2rx_priv *csi2rx = v4l2_subdev_to_csi2rx(subdev);
+	u64 sink_streams;
+
+	sink_streams = v4l2_subdev_state_xlate_streams(state, pad,
+						       CSI2RX_PAD_SINK,
+						       &streams_mask);
+
+	if (v4l2_subdev_disable_streams(csi2rx->source_subdev,
+						 csi2rx->source_pad, sink_streams)) {
+		dev_err(csi2rx->dev, "Couldn't disable our subdev\n");
+	}
+
+	csi2rx->count--;
+
+	/* Let the last user turn off the lights. */
+	if (!csi2rx->count) {
+		csi2rx_stop(csi2rx);
+		pm_runtime_put(csi2rx->dev);
+	}
+
+	return 0;
+}
+
 static int csi2rx_enum_mbus_code(struct v4l2_subdev *subdev,
 				 struct v4l2_subdev_state *state,
 				 struct v4l2_subdev_mbus_code_enum *code_enum)
@@ -531,12 +579,53 @@ static int csi2rx_enum_mbus_code(struct v4l2_subdev *subdev,
 	return 0;
 }
 
+static int _csi2rx_set_routing(struct v4l2_subdev *subdev,
+			       struct v4l2_subdev_state *state,
+			       struct v4l2_subdev_krouting *routing)
+{
+	static const struct v4l2_mbus_framefmt format = {
+		.width = 640,
+		.height = 480,
+		.code = MEDIA_BUS_FMT_UYVY8_1X16,
+		.field = V4L2_FIELD_NONE,
+		.colorspace = V4L2_COLORSPACE_SRGB,
+		.ycbcr_enc = V4L2_YCBCR_ENC_601,
+		.quantization = V4L2_QUANTIZATION_LIM_RANGE,
+		.xfer_func = V4L2_XFER_FUNC_SRGB,
+	};
+	int ret;
+
+	ret = v4l2_subdev_routing_validate(subdev, routing,
+					   V4L2_SUBDEV_ROUTING_ONLY_1_TO_1);
+	if (ret)
+		return ret;
+
+	return v4l2_subdev_set_routing_with_fmt(subdev, state, routing, &format);
+}
+
+static int csi2rx_set_routing(struct v4l2_subdev *subdev,
+			      struct v4l2_subdev_state *state,
+			      enum v4l2_subdev_format_whence which,
+			      struct v4l2_subdev_krouting *routing)
+{
+	struct csi2rx_priv *csi2rx = v4l2_subdev_to_csi2rx(subdev);
+	int ret;
+
+	if (which == V4L2_SUBDEV_FORMAT_ACTIVE && csi2rx->count)
+		return -EBUSY;
+
+	ret = _csi2rx_set_routing(subdev, state, routing);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
 static int csi2rx_set_fmt(struct v4l2_subdev *subdev,
 			  struct v4l2_subdev_state *state,
 			  struct v4l2_subdev_format *format)
 {
 	struct v4l2_mbus_framefmt *fmt;
-	unsigned int i;
 
 	/* No transcoding, source and sink formats must match. */
 	if (format->pad != CSI2RX_PAD_SINK)
@@ -548,14 +637,16 @@ static int csi2rx_set_fmt(struct v4l2_subdev *subdev,
 	format->format.field = V4L2_FIELD_NONE;
 
 	/* Set sink format */
-	fmt = v4l2_subdev_state_get_format(state, format->pad);
+	fmt = v4l2_subdev_state_get_format(state, format->pad, format->stream);
 	*fmt = format->format;
 
-	/* Propagate to source formats */
-	for (i = CSI2RX_PAD_SOURCE_STREAM0; i < CSI2RX_PAD_MAX; i++) {
-		fmt = v4l2_subdev_state_get_format(state, i);
-		*fmt = format->format;
-	}
+	/* Propagate to source format */
+	fmt = v4l2_subdev_state_get_opposite_stream_format(state, format->pad,
+							   format->stream);
+	if (!fmt)
+		return -EINVAL;
+
+	*fmt = format->format;
 
 	return 0;
 }
@@ -563,21 +654,22 @@ static int csi2rx_set_fmt(struct v4l2_subdev *subdev,
 static int csi2rx_init_state(struct v4l2_subdev *subdev,
 			     struct v4l2_subdev_state *state)
 {
-	struct v4l2_subdev_format format = {
-		.pad = CSI2RX_PAD_SINK,
-		.format = {
-			.width = 640,
-			.height = 480,
-			.code = MEDIA_BUS_FMT_UYVY8_1X16,
-			.field = V4L2_FIELD_NONE,
-			.colorspace = V4L2_COLORSPACE_SRGB,
-			.ycbcr_enc = V4L2_YCBCR_ENC_601,
-			.quantization = V4L2_QUANTIZATION_LIM_RANGE,
-			.xfer_func = V4L2_XFER_FUNC_SRGB,
+	struct v4l2_subdev_route routes[] = {
+		{
+			.sink_pad = CSI2RX_PAD_SINK,
+			.sink_stream = 0,
+			.source_pad = CSI2RX_PAD_SOURCE_STREAM0,
+			.source_stream = 0,
+			.flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE,
 		},
 	};
 
-	return csi2rx_set_fmt(subdev, state, &format);
+	struct v4l2_subdev_krouting routing = {
+		.num_routes = ARRAY_SIZE(routes),
+		.routes = routes,
+	};
+
+	return _csi2rx_set_routing(subdev, state, &routing);
 }
 
 int cdns_csi2rx_negotiate_ppc(struct v4l2_subdev *subdev, unsigned int pad,
@@ -585,36 +677,55 @@ int cdns_csi2rx_negotiate_ppc(struct v4l2_subdev *subdev, unsigned int pad,
 {
 	struct csi2rx_priv *csi2rx = v4l2_subdev_to_csi2rx(subdev);
 	const struct csi2rx_fmt *csi_fmt;
+	struct v4l2_subdev_route *route;
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *fmt;
+	int ret = 0;
 
 	if (!ppc || pad < CSI2RX_PAD_SOURCE_STREAM0 || pad >= CSI2RX_PAD_MAX)
 		return -EINVAL;
 
 	state = v4l2_subdev_lock_and_get_active_state(subdev);
-	fmt = v4l2_subdev_state_get_format(state, pad);
-	csi_fmt = csi2rx_get_fmt_by_code(fmt->code);
+	/* Check all streams on requested pad */
+	for_each_active_route(&state->routing, route) {
+		if (route->source_pad != pad)
+			continue;
+
+		fmt = v4l2_subdev_state_get_format(state, route->source_pad,
+						   route->source_stream);
+		if (!fmt) {
+			ret = -EPIPE;
+			*ppc = 1;
+			break;
+		}
 
-	/* Reduce requested PPC if it is too high */
-	*ppc = min(*ppc, csi_fmt->max_pixels);
+		csi_fmt = csi2rx_get_fmt_by_code(fmt->code);
+		if (!csi_fmt) {
+			ret = -EINVAL;
+			*ppc = 1;
+			break;
+		}
 
+		/* Reduce requested PPC if it is too high for this stream */
+		*ppc = min(*ppc, csi_fmt->max_pixels);
+	}
 	v4l2_subdev_unlock_state(state);
 
 	csi2rx->num_pixels[pad - CSI2RX_PAD_SOURCE_STREAM0] =
 		CSI2RX_STREAM_CFG_NUM_PIXELS(*ppc);
 
-	return 0;
+	return ret;
 }
 EXPORT_SYMBOL_FOR_MODULES(cdns_csi2rx_negotiate_ppc, "j721e-csi2rx");
 
 static const struct v4l2_subdev_pad_ops csi2rx_pad_ops = {
-	.enum_mbus_code	= csi2rx_enum_mbus_code,
-	.get_fmt	= v4l2_subdev_get_fmt,
-	.set_fmt	= csi2rx_set_fmt,
-};
-
-static const struct v4l2_subdev_video_ops csi2rx_video_ops = {
-	.s_stream	= csi2rx_s_stream,
+	.enum_mbus_code		= csi2rx_enum_mbus_code,
+	.get_fmt		= v4l2_subdev_get_fmt,
+	.set_fmt		= csi2rx_set_fmt,
+	.get_frame_desc		= v4l2_subdev_get_frame_desc_passthrough,
+	.set_routing		= csi2rx_set_routing,
+	.enable_streams		= csi2rx_enable_streams,
+	.disable_streams	= csi2rx_disable_streams,
 };
 
 static const struct v4l2_subdev_core_ops csi2rx_core_ops = {
@@ -623,7 +734,6 @@ static const struct v4l2_subdev_core_ops csi2rx_core_ops = {
 
 static const struct v4l2_subdev_ops csi2rx_subdev_ops = {
 	.core		= &csi2rx_core_ops,
-	.video		= &csi2rx_video_ops,
 	.pad		= &csi2rx_pad_ops,
 };
 
@@ -634,6 +744,7 @@ static const struct v4l2_subdev_internal_ops csi2rx_internal_ops = {
 static const struct media_entity_operations csi2rx_media_ops = {
 	.link_validate = v4l2_subdev_link_validate,
 	.get_fwnode_pad = v4l2_subdev_get_fwnode_pad_1_to_1,
+	.has_pad_interdep = v4l2_subdev_has_pad_interdep,
 };
 
 static int csi2rx_async_bound(struct v4l2_async_notifier *notifier,
@@ -829,7 +940,6 @@ static int csi2rx_probe(struct platform_device *pdev)
 		return -ENOMEM;
 	platform_set_drvdata(pdev, csi2rx);
 	csi2rx->dev = &pdev->dev;
-	mutex_init(&csi2rx->lock);
 
 	ret = csi2rx_get_resources(csi2rx, pdev);
 	if (ret)
@@ -852,7 +962,8 @@ static int csi2rx_probe(struct platform_device *pdev)
 	csi2rx->pads[CSI2RX_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
 	for (i = CSI2RX_PAD_SOURCE_STREAM0; i < CSI2RX_PAD_MAX; i++)
 		csi2rx->pads[i].flags = MEDIA_PAD_FL_SOURCE;
-	csi2rx->subdev.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
+	csi2rx->subdev.flags = V4L2_SUBDEV_FL_HAS_DEVNODE |
+		V4L2_SUBDEV_FL_STREAMS;
 	csi2rx->subdev.entity.ops = &csi2rx_media_ops;
 
 	ret = media_entity_pads_init(&csi2rx->subdev.entity, CSI2RX_PAD_MAX,
@@ -879,6 +990,7 @@ static int csi2rx_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_cleanup;
 
+	pm_runtime_enable(csi2rx->dev);
 	ret = v4l2_async_register_subdev(&csi2rx->subdev);
 	if (ret < 0)
 		goto err_free_state;
@@ -893,6 +1005,7 @@ static int csi2rx_probe(struct platform_device *pdev)
 
 err_free_state:
 	v4l2_subdev_cleanup(&csi2rx->subdev);
+	pm_runtime_disable(csi2rx->dev);
 err_cleanup:
 	v4l2_async_nf_unregister(&csi2rx->notifier);
 	v4l2_async_nf_cleanup(&csi2rx->notifier);
@@ -911,9 +1024,72 @@ static void csi2rx_remove(struct platform_device *pdev)
 	v4l2_async_unregister_subdev(&csi2rx->subdev);
 	v4l2_subdev_cleanup(&csi2rx->subdev);
 	media_entity_cleanup(&csi2rx->subdev.entity);
+	pm_runtime_disable(csi2rx->dev);
 	kfree(csi2rx);
 }
 
+static int csi2rx_runtime_suspend(struct device *dev)
+{
+	struct csi2rx_priv *csi2rx = dev_get_drvdata(dev);
+
+	reset_control_assert(csi2rx->sys_rst);
+	clk_disable_unprepare(csi2rx->sys_clk);
+
+	for (unsigned int i = 0; i < csi2rx->max_streams; i++) {
+		reset_control_assert(csi2rx->pixel_rst[i]);
+		clk_disable_unprepare(csi2rx->pixel_clk[i]);
+	}
+
+	reset_control_assert(csi2rx->p_rst);
+	clk_disable_unprepare(csi2rx->p_clk);
+
+	return 0;
+}
+
+static int csi2rx_runtime_resume(struct device *dev)
+{
+	struct csi2rx_priv *csi2rx = dev_get_drvdata(dev);
+	unsigned int i;
+	int ret;
+
+	ret = clk_prepare_enable(csi2rx->p_clk);
+	if (ret)
+		return ret;
+
+	reset_control_deassert(csi2rx->p_rst);
+
+	for (i = 0; i < csi2rx->max_streams; i++) {
+		ret = clk_prepare_enable(csi2rx->pixel_clk[i]);
+		if (ret)
+			goto err_disable_pixclk;
+
+		reset_control_deassert(csi2rx->pixel_rst[i]);
+	}
+
+	ret = clk_prepare_enable(csi2rx->sys_clk);
+	if (ret)
+		goto err_disable_pixclk;
+
+	reset_control_deassert(csi2rx->sys_rst);
+
+	return 0;
+
+err_disable_pixclk:
+	while (i--) {
+		reset_control_assert(csi2rx->pixel_rst[i]);
+		clk_disable_unprepare(csi2rx->pixel_clk[i]);
+	}
+
+	reset_control_assert(csi2rx->p_rst);
+	clk_disable_unprepare(csi2rx->p_clk);
+
+	return ret;
+}
+
+static const struct dev_pm_ops csi2rx_pm_ops = {
+	RUNTIME_PM_OPS(csi2rx_runtime_suspend, csi2rx_runtime_resume, NULL)
+};
+
 static const struct of_device_id csi2rx_of_table[] = {
 	{ .compatible = "starfive,jh7110-csi2rx" },
 	{ .compatible = "cdns,csi2rx" },
@@ -928,6 +1104,7 @@ static struct platform_driver csi2rx_driver = {
 	.driver	= {
 		.name		= "cdns-csi2rx",
 		.of_match_table	= csi2rx_of_table,
+		.pm		= &csi2rx_pm_ops,
 	},
 };
 module_platform_driver(csi2rx_driver);
diff --git a/drivers/media/platform/chips-media/wave5/wave5-helper.c b/drivers/media/platform/chips-media/wave5/wave5-helper.c
index 53a0ac068c2e2..c3d34be833ff0 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-helper.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-helper.c
@@ -68,7 +68,6 @@ int wave5_vpu_release_device(struct file *filp,
 	int ret = 0;
 	unsigned long flags;
 
-	v4l2_m2m_ctx_release(inst->v4l2_fh.m2m_ctx);
 	/*
 	 * To prevent Null reference exception, the existing irq handler were
 	 * separated to two modules.
@@ -89,6 +88,9 @@ int wave5_vpu_release_device(struct file *filp,
 	list_del_init(&inst->list);
 	spin_unlock_irqrestore(&inst->dev->irq_spinlock, flags);
 	mutex_unlock(&inst->dev->irq_lock);
+
+	v4l2_m2m_ctx_release(inst->v4l2_fh.m2m_ctx);
+
 	if (inst->state != VPU_INST_STATE_NONE) {
 		u32 fail_res;
 
diff --git a/drivers/media/platform/chips-media/wave5/wave5-helper.h b/drivers/media/platform/chips-media/wave5/wave5-helper.h
index d61fdbda359d6..e6f241012c3bc 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-helper.h
+++ b/drivers/media/platform/chips-media/wave5/wave5-helper.h
@@ -11,7 +11,7 @@
 #include "wave5-vpu.h"
 
 #define FMT_TYPES	2
-#define MAX_FMTS	12
+#define MAX_FMTS	16
 
 const char *state_to_str(enum vpu_instance_state state);
 void wave5_cleanup_instance(struct vpu_instance *inst, struct file *filp);
diff --git a/drivers/media/platform/chips-media/wave5/wave5-hw.c b/drivers/media/platform/chips-media/wave5/wave5-hw.c
index 687ce6ccf3ae2..2392bce8d840b 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-hw.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-hw.c
@@ -49,6 +49,7 @@
 
 #define FASTIO_ADDRESS_MASK		GENMASK(15, 0)
 #define SEQ_PARAM_PROFILE_MASK		GENMASK(30, 24)
+#define SEQ_BG_PARAM_REG_DATA		0x3800410
 
 static void _wave5_print_reg_err(struct vpu_device *vpu_dev, u32 reg_fail_reason,
 				 const char *func);
@@ -1762,6 +1763,9 @@ int wave5_vpu_enc_init_seq(struct vpu_instance *inst)
 			(p_param->skip_intra_trans << 25) |
 			(p_param->strong_intra_smooth_enable << 27) |
 			(p_param->en_still_picture << 30);
+	else if (inst->std == W_AVC_ENC)
+		reg_val |= (p_param->constraint_set1_flag << 29);
+
 	vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_SPS_PARAM, reg_val);
 
 	reg_val = (p_param->lossless_enable) |
@@ -1838,7 +1842,8 @@ int wave5_vpu_enc_init_seq(struct vpu_instance *inst)
 	vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_RC_BIT_RATIO_LAYER_4_7, 0);
 	vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_ROT_PARAM, rot_mir_mode);
 
-	vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_BG_PARAM, 0);
+	vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_BG_PARAM,
+		      SEQ_BG_PARAM_REG_DATA | p_param->bg_detection);
 	vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_CUSTOM_LAMBDA_ADDR, 0);
 	vpu_write_reg(inst->dev, W5_CMD_ENC_SEQ_CONF_WIN_TOP_BOT,
 		      p_param->conf_win_bot << 16 | p_param->conf_win_top);
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
index d419076d7052c..bb2ba9204a836 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-dec.c
@@ -283,10 +283,23 @@ static void send_eos_event(struct vpu_instance *inst)
 	inst->sent_eos = true;
 }
 
+static void wave5_update_min_bufs_ctrl(struct vpu_instance *inst, u32 fbc_buf_count)
+{
+	struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx;
+	struct v4l2_ctrl *ctrl;
+
+	if (!fbc_buf_count || fbc_buf_count == v4l2_m2m_num_dst_bufs_ready(m2m_ctx))
+		return;
+
+	ctrl = v4l2_ctrl_find(&inst->v4l2_ctrl_hdl,
+			      V4L2_CID_MIN_BUFFERS_FOR_CAPTURE);
+	if (ctrl)
+		v4l2_ctrl_s_ctrl(ctrl, fbc_buf_count);
+}
+
 static int handle_dynamic_resolution_change(struct vpu_instance *inst)
 {
 	struct v4l2_fh *fh = &inst->v4l2_fh;
-	struct v4l2_m2m_ctx *m2m_ctx = inst->v4l2_fh.m2m_ctx;
 
 	static const struct v4l2_event vpu_event_src_ch = {
 		.type = V4L2_EVENT_SOURCE_CHANGE,
@@ -305,14 +318,6 @@ static int handle_dynamic_resolution_change(struct vpu_instance *inst)
 
 	inst->needs_reallocation = true;
 	inst->fbc_buf_count = initial_info->min_frame_buffer_count + 1;
-	if (inst->fbc_buf_count != v4l2_m2m_num_dst_bufs_ready(m2m_ctx)) {
-		struct v4l2_ctrl *ctrl;
-
-		ctrl = v4l2_ctrl_find(&inst->v4l2_ctrl_hdl,
-				      V4L2_CID_MIN_BUFFERS_FOR_CAPTURE);
-		if (ctrl)
-			v4l2_ctrl_s_ctrl(ctrl, inst->fbc_buf_count);
-	}
 
 	if (p_dec_info->initial_info_obtained) {
 		const struct vpu_format *vpu_fmt;
@@ -439,19 +444,24 @@ static void wave5_vpu_dec_finish_decode(struct vpu_instance *inst)
 	if ((dec_info.index_frame_display == DISPLAY_IDX_FLAG_SEQ_END ||
 	     dec_info.sequence_changed)) {
 		unsigned long flags;
+		u32 fbc_buf_count = 0;
 
 		spin_lock_irqsave(&inst->state_spinlock, flags);
 		if (!v4l2_m2m_has_stopped(m2m_ctx)) {
 			switch_state(inst, VPU_INST_STATE_STOP);
 
-			if (dec_info.sequence_changed)
+			if (dec_info.sequence_changed) {
 				handle_dynamic_resolution_change(inst);
-			else
+				fbc_buf_count = inst->fbc_buf_count;
+			} else {
 				send_eos_event(inst);
+			}
 
 			flag_last_buffer_done(inst);
 		}
 		spin_unlock_irqrestore(&inst->state_spinlock, flags);
+
+		wave5_update_min_bufs_ctrl(inst, fbc_buf_count);
 	}
 
 	if (inst->sent_eos &&
@@ -1592,8 +1602,9 @@ static const struct vpu_instance_ops wave5_vpu_dec_inst_ops = {
 static int initialize_sequence(struct vpu_instance *inst)
 {
 	struct dec_initial_info initial_info;
-	int ret = 0;
 	unsigned long flags;
+	u32 fbc_buf_count;
+	int ret = 0;
 
 	memset(&initial_info, 0, sizeof(struct dec_initial_info));
 
@@ -1617,8 +1628,11 @@ static int initialize_sequence(struct vpu_instance *inst)
 
 	spin_lock_irqsave(&inst->state_spinlock, flags);
 	handle_dynamic_resolution_change(inst);
+	fbc_buf_count = inst->fbc_buf_count;
 	spin_unlock_irqrestore(&inst->state_spinlock, flags);
 
+	wave5_update_min_bufs_ctrl(inst, fbc_buf_count);
+
 	return 0;
 }
 
@@ -1659,6 +1673,7 @@ static void wave5_vpu_dec_device_run(void *priv)
 		ret = initialize_sequence(inst);
 		if (ret) {
 			unsigned long flags;
+			u32 fbc_buf_count = 0;
 
 			spin_lock_irqsave(&inst->state_spinlock, flags);
 			if (wave5_is_draining_or_eos(inst) &&
@@ -1667,14 +1682,18 @@ static void wave5_vpu_dec_device_run(void *priv)
 
 				switch_state(inst, VPU_INST_STATE_STOP);
 
-				if (vb2_is_streaming(dst_vq))
+				if (vb2_is_streaming(dst_vq)) {
 					send_eos_event(inst);
-				else
+				} else {
 					handle_dynamic_resolution_change(inst);
+					fbc_buf_count = inst->fbc_buf_count;
+				}
 
 				flag_last_buffer_done(inst);
 			}
 			spin_unlock_irqrestore(&inst->state_spinlock, flags);
+
+			wave5_update_min_bufs_ctrl(inst, fbc_buf_count);
 		} else {
 			set_instance_state(inst, VPU_INST_STATE_INIT_SEQ);
 		}
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
index 7613fcdbafedb..e6c94b6f26711 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpu-enc.c
@@ -90,6 +90,22 @@ static const struct vpu_format enc_fmt_list[FMT_TYPES][MAX_FMTS] = {
 			.v4l2_pix_fmt = V4L2_PIX_FMT_NV61M,
 			.v4l2_frmsize = &enc_frmsize[VPU_FMT_TYPE_RAW],
 		},
+		{
+			.v4l2_pix_fmt = V4L2_PIX_FMT_YUYV,
+			.v4l2_frmsize = &enc_frmsize[VPU_FMT_TYPE_RAW],
+		},
+		{
+			.v4l2_pix_fmt = V4L2_PIX_FMT_YVYU,
+			.v4l2_frmsize = &enc_frmsize[VPU_FMT_TYPE_RAW],
+		},
+		{
+			.v4l2_pix_fmt = V4L2_PIX_FMT_UYVY,
+			.v4l2_frmsize = &enc_frmsize[VPU_FMT_TYPE_RAW],
+		},
+		{
+			.v4l2_pix_fmt = V4L2_PIX_FMT_VYUY,
+			.v4l2_frmsize = &enc_frmsize[VPU_FMT_TYPE_RAW],
+		},
 	}
 };
 
@@ -226,13 +242,6 @@ static int start_encode(struct vpu_instance *inst, u32 *fail_res)
 	} else {
 		dev_dbg(inst->dev->dev, "%s: wave5_vpu_enc_start_one_frame success\n",
 			__func__);
-		/*
-		 * Remove the source buffer from the ready-queue now and finish
-		 * it in the videobuf2 framework once the index is returned by the
-		 * firmware in finish_encode
-		 */
-		if (src_buf)
-			v4l2_m2m_src_buf_remove_by_idx(m2m_ctx, src_buf->vb2_buf.index);
 	}
 
 	return 0;
@@ -259,27 +268,13 @@ static void wave5_vpu_enc_finish_encode(struct vpu_instance *inst)
 		__func__,  enc_output_info.pic_type, enc_output_info.recon_frame_index,
 		enc_output_info.enc_src_idx, enc_output_info.enc_pic_byte, enc_output_info.pts);
 
-	/*
-	 * The source buffer will not be found in the ready-queue as it has been
-	 * dropped after sending of the encode firmware command, locate it in
-	 * the videobuf2 queue directly
-	 */
 	if (enc_output_info.enc_src_idx >= 0) {
-		struct vb2_buffer *vb = vb2_get_buffer(v4l2_m2m_get_src_vq(m2m_ctx),
-						       enc_output_info.enc_src_idx);
-		if (vb->state != VB2_BUF_STATE_ACTIVE)
-			dev_warn(inst->dev->dev,
-				 "%s: encoded buffer (%d) was not in ready queue %i.",
-				 __func__, enc_output_info.enc_src_idx, vb->state);
-		else
-			src_buf = to_vb2_v4l2_buffer(vb);
-
-		if (src_buf) {
+		src_buf = v4l2_m2m_src_buf_remove_by_idx(m2m_ctx, enc_output_info.enc_src_idx);
+		if (!src_buf) {
+			dev_warn(inst->dev->dev, "%s: no source buffer found\n", __func__);
+		} else {
 			inst->timestamp = src_buf->vb2_buf.timestamp;
 			v4l2_m2m_buf_done(src_buf, VB2_BUF_STATE_DONE);
-		} else {
-			dev_warn(inst->dev->dev, "%s: no source buffer with index: %d found\n",
-				 __func__, enc_output_info.enc_src_idx);
 		}
 	}
 
@@ -780,6 +775,9 @@ static int wave5_vpu_enc_s_ctrl(struct v4l2_ctrl *ctrl)
 	case V4L2_CID_MPEG_VIDEO_BITRATE:
 		inst->bit_rate = ctrl->val;
 		break;
+	case V4L2_CID_MPEG_VIDEO_BACKGROUND_DETECTION:
+		inst->enc_param.bg_detection = ctrl->val;
+		break;
 	case V4L2_CID_MPEG_VIDEO_GOP_SIZE:
 		inst->enc_param.avc_idr_period = ctrl->val;
 		break;
@@ -936,6 +934,8 @@ static int wave5_vpu_enc_s_ctrl(struct v4l2_ctrl *ctrl)
 		case V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE:
 			inst->enc_param.profile = H264_PROFILE_BP;
 			inst->bit_depth = 8;
+			if (ctrl->val == V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE)
+				inst->enc_param.constraint_set1_flag = 1;
 			break;
 		case V4L2_MPEG_VIDEO_H264_PROFILE_MAIN:
 			inst->enc_param.profile = H264_PROFILE_MP;
@@ -1156,6 +1156,22 @@ static int wave5_set_enc_openparam(struct enc_open_param *open_param,
 	else
 		open_param->src_format = FORMAT_420;
 
+	switch (info->format) {
+	case V4L2_PIX_FMT_YUYV:
+		open_param->packed_format = PACKED_YUYV;
+		break;
+	case V4L2_PIX_FMT_YVYU:
+		open_param->packed_format = PACKED_YVYU;
+		break;
+	case V4L2_PIX_FMT_UYVY:
+		open_param->packed_format = PACKED_UYVY;
+		break;
+	case V4L2_PIX_FMT_VYUY:
+		open_param->packed_format = PACKED_VYUY;
+		break;
+	default:
+		break;
+	}
 	open_param->wave_param.gop_preset_idx = PRESET_IDX_IPP_SINGLE;
 	open_param->wave_param.hvs_qp_scale = 2;
 	open_param->wave_param.hvs_max_delta_qp = 10;
@@ -1205,12 +1221,14 @@ static int wave5_set_enc_openparam(struct enc_open_param *open_param,
 	open_param->wave_param.beta_offset_div2 = input.beta_offset_div2;
 	open_param->wave_param.decoding_refresh_type = input.decoding_refresh_type;
 	open_param->wave_param.intra_period = input.intra_period;
+	open_param->wave_param.bg_detection = input.bg_detection;
 	if (inst->std == W_HEVC_ENC) {
 		if (input.intra_period == 0) {
 			open_param->wave_param.decoding_refresh_type = DEC_REFRESH_TYPE_IDR;
 			open_param->wave_param.intra_period = input.avc_idr_period;
 		}
 	} else {
+		open_param->wave_param.constraint_set1_flag = input.constraint_set1_flag;
 		open_param->wave_param.avc_idr_period = input.avc_idr_period;
 	}
 	open_param->wave_param.entropy_coding_mode = input.entropy_coding_mode;
@@ -1683,7 +1701,7 @@ static int wave5_vpu_open_enc(struct file *filp)
 			  -6, 6, 1, 0);
 	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
 			  V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM,
-			  0, 1, 1, 1);
+			  0, 1, 1, 0);
 	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
 			  V4L2_CID_MPEG_VIDEO_H264_CONSTRAINED_INTRA_PREDICTION,
 			  0, 1, 1, 0);
@@ -1700,6 +1718,9 @@ static int wave5_vpu_open_enc(struct file *filp)
 	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
 			  V4L2_CID_MPEG_VIDEO_AU_DELIMITER,
 			  0, 1, 1, 1);
+	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
+			  V4L2_CID_MPEG_VIDEO_BACKGROUND_DETECTION,
+			  0, 1, 1, 0);
 	v4l2_ctrl_new_std(v4l2_ctrl_hdl, &wave5_vpu_enc_ctrl_ops,
 			  V4L2_CID_HFLIP,
 			  0, 1, 1, 0);
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
index d26ffc9422190..f77abd5e122a2 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.c
@@ -584,8 +584,15 @@ int wave5_vpu_dec_get_output_info(struct vpu_instance *inst, struct dec_output_i
 		p_dec_info->num_of_decoding_fbs : p_dec_info->num_of_display_fbs;
 
 	if (info->index_frame_display >= 0 &&
-	    info->index_frame_display < (int)max_dec_index)
-		info->disp_frame = inst->frame_buf[val + info->index_frame_display];
+	    info->index_frame_display < (int)max_dec_index) {
+		u32 idx = val + info->index_frame_display;
+
+		if (WARN_ON(idx >= MAX_REG_FRAME)) {
+			ret = -EINVAL;
+			goto err_out;
+		}
+		info->disp_frame = inst->frame_buf[idx];
+	}
 
 	info->rd_ptr = p_dec_info->stream_rd_ptr;
 	info->wr_ptr = p_dec_info->stream_wr_ptr;
diff --git a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
index c641357698691..7b08fef582171 100644
--- a/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
+++ b/drivers/media/platform/chips-media/wave5/wave5-vpuapi.h
@@ -570,6 +570,8 @@ struct enc_wave_param {
 	u32 transform8x8_enable: 1; /* enable 8x8 intra prediction and 8x8 transform */
 	u32 mb_level_rc_enable: 1; /* enable MB-level rate control */
 	u32 forced_idr_header_enable: 1; /* enable header encoding before IDR frame */
+	u32 constraint_set1_flag: 1; /* enable CBP */
+	u32 bg_detection: 1; /* enable background detection */
 };
 
 struct enc_open_param {
diff --git a/drivers/media/platform/marvell/cafe-driver.c b/drivers/media/platform/marvell/cafe-driver.c
index 632c15572aa8f..22034df6cba96 100644
--- a/drivers/media/platform/marvell/cafe-driver.c
+++ b/drivers/media/platform/marvell/cafe-driver.c
@@ -609,6 +609,7 @@ static void cafe_pci_remove(struct pci_dev *pdev)
 		return;
 	}
 	cafe_shutdown(cam);
+	pci_disable_device(pdev);
 	kfree(cam);
 }
 
diff --git a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c
index 8c684756d5fc2..d147ec4830811 100644
--- a/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c
+++ b/drivers/media/platform/mediatek/jpeg/mtk_jpeg_core.c
@@ -1202,7 +1202,8 @@ static int mtk_jpeg_release(struct file *file)
 	struct mtk_jpeg_dev *jpeg = video_drvdata(file);
 	struct mtk_jpeg_ctx *ctx = mtk_jpeg_file_to_ctx(file);
 
-	cancel_work_sync(&ctx->jpeg_work);
+	if (jpeg->variant->jpeg_worker)
+		cancel_work_sync(&ctx->jpeg_work);
 	mutex_lock(&jpeg->lock);
 	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
 	v4l2_ctrl_handler_free(&ctx->ctrl_hdl);
diff --git a/drivers/media/platform/nuvoton/npcm-video.c b/drivers/media/platform/nuvoton/npcm-video.c
index b2a562e1ee1cb..52505af35c087 100644
--- a/drivers/media/platform/nuvoton/npcm-video.c
+++ b/drivers/media/platform/nuvoton/npcm-video.c
@@ -1720,10 +1720,12 @@ static int npcm_video_init(struct npcm_video *video)
 	if (rc) {
 		dev_err(dev, "Failed to set DMA mask\n");
 		of_reserved_mem_device_release(dev);
+		return rc;
 	}
 
 	rc = npcm_video_ece_init(video);
 	if (rc) {
+		of_reserved_mem_device_release(dev);
 		dev_err(dev, "Failed to initialize ECE\n");
 		return rc;
 	}
@@ -1748,42 +1750,55 @@ static int npcm_video_probe(struct platform_device *pdev)
 	regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(regs)) {
 		dev_err(&pdev->dev, "Failed to parse VCD reg in DTS\n");
-		return PTR_ERR(regs);
+		rc = PTR_ERR(regs);
+		goto err_free;
 	}
 
 	video->vcd_regmap = devm_regmap_init_mmio(&pdev->dev, regs,
 						  &npcm_video_regmap_cfg);
 	if (IS_ERR(video->vcd_regmap)) {
 		dev_err(&pdev->dev, "Failed to initialize VCD regmap\n");
-		return PTR_ERR(video->vcd_regmap);
+		rc = PTR_ERR(video->vcd_regmap);
+		goto err_free;
 	}
 
 	video->reset = devm_reset_control_get(&pdev->dev, NULL);
 	if (IS_ERR(video->reset)) {
 		dev_err(&pdev->dev, "Failed to get VCD reset control in DTS\n");
-		return PTR_ERR(video->reset);
+		rc = PTR_ERR(video->reset);
+		goto err_free;
 	}
 
 	video->gcr_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 							    "nuvoton,sysgcr");
-	if (IS_ERR(video->gcr_regmap))
-		return PTR_ERR(video->gcr_regmap);
+	if (IS_ERR(video->gcr_regmap)) {
+		rc = PTR_ERR(video->gcr_regmap);
+		goto err_free;
+	}
 
 	video->gfx_regmap = syscon_regmap_lookup_by_phandle(pdev->dev.of_node,
 							    "nuvoton,sysgfxi");
-	if (IS_ERR(video->gfx_regmap))
-		return PTR_ERR(video->gfx_regmap);
+	if (IS_ERR(video->gfx_regmap)) {
+		rc = PTR_ERR(video->gfx_regmap);
+		goto err_free;
+	}
 
 	rc = npcm_video_init(video);
 	if (rc)
-		return rc;
+		goto err_free;
 
 	rc = npcm_video_setup_video(video);
 	if (rc)
-		return rc;
+		goto err_release_mem;
 
 	dev_info(video->dev, "NPCM video driver probed\n");
 	return 0;
+
+err_release_mem:
+	of_reserved_mem_device_release(&pdev->dev);
+err_free:
+	kfree(video);
+	return rc;
 }
 
 static void npcm_video_remove(struct platform_device *pdev)
@@ -1798,6 +1813,7 @@ static void npcm_video_remove(struct platform_device *pdev)
 	v4l2_device_unregister(v4l2_dev);
 	if (video->ece.enable)
 		npcm_video_ece_stop(video);
+	kfree(video);
 	of_reserved_mem_device_release(dev);
 }
 
diff --git a/drivers/media/platform/nvidia/tegra-vde/Kconfig b/drivers/media/platform/nvidia/tegra-vde/Kconfig
index 2fe13f39c95bb..f05fcc94decaf 100644
--- a/drivers/media/platform/nvidia/tegra-vde/Kconfig
+++ b/drivers/media/platform/nvidia/tegra-vde/Kconfig
@@ -2,6 +2,7 @@ config VIDEO_TEGRA_VDE
 	tristate "NVIDIA Tegra Video Decoder Engine driver"
 	depends on V4L_MEM2MEM_DRIVERS
 	depends on ARCH_TEGRA || COMPILE_TEST
+	depends on HAS_IOMEM
 	depends on VIDEO_DEV
 	select DMA_SHARED_BUFFER
 	select IOMMU_IOVA
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c
index 4bf8570e1b9e9..e8545761b5ff9 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.c
@@ -538,6 +538,8 @@ static int mxc_isi_probe(struct platform_device *pdev)
 	return 0;
 
 err_xbar:
+	while (i--)
+		mxc_isi_pipe_cleanup(&isi->pipes[i]);
 	mxc_isi_crossbar_cleanup(&isi->crossbar);
 
 	return ret;
@@ -556,8 +558,8 @@ static void mxc_isi_remove(struct platform_device *pdev)
 		mxc_isi_pipe_cleanup(pipe);
 	}
 
-	mxc_isi_crossbar_cleanup(&isi->crossbar);
 	mxc_isi_v4l2_cleanup(isi);
+	mxc_isi_crossbar_cleanup(&isi->crossbar);
 }
 
 static const struct of_device_id mxc_isi_of_match[] = {
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h
index 14d63ec36416e..7547a6559d4c1 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-core.h
@@ -11,6 +11,7 @@
 #define __MXC_ISI_CORE_H__
 
 #include <linux/list.h>
+#include <linux/math.h>
 #include <linux/mutex.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
@@ -414,4 +415,19 @@ static inline void mxc_isi_debug_cleanup(struct mxc_isi_dev *isi)
 }
 #endif
 
+/*
+ * ISI scaling engine works in two parts: it performs pre-decimation of
+ * the image followed by bilinear filtering to achieve the desired
+ * downscaling factor.
+ *
+ * The decimation filter provides a maximum downscaling factor of 8, and
+ * the subsequent bilinear filter provides a maximum downscaling factor
+ * of 2. Combined, the maximum scaling factor can be up to 16.
+ */
+static inline unsigned int
+mxc_isi_clamp_downscale_16(unsigned int val, unsigned int max_val)
+{
+	return clamp(val, max(1U, DIV_ROUND_UP(max_val, 16)), max_val);
+}
+
 #endif /* __MXC_ISI_CORE_H__ */
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
index 605a451241035..c580c831972ec 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-crossbar.c
@@ -491,6 +491,7 @@ int mxc_isi_crossbar_init(struct mxc_isi_dev *isi)
 
 void mxc_isi_crossbar_cleanup(struct mxc_isi_crossbar *xbar)
 {
+	v4l2_subdev_cleanup(&xbar->sd);
 	media_entity_cleanup(&xbar->sd.entity);
 	kfree(xbar->pads);
 	kfree(xbar->inputs);
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c
index 0187d4ab97e8e..16b20ea2d1db7 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-hw.c
@@ -112,7 +112,14 @@ static u32 mxc_isi_channel_scaling_ratio(unsigned int from, unsigned int to,
 	else
 		*dec = 8;
 
-	return min_t(u32, from * 0x1000 / (to * *dec), ISI_DOWNSCALE_THRESHOLD);
+	/*
+	 * The ISI rounds output dimensions up to the next integer (i.MX93 RM
+	 * section 57.7.8). Calculate the scale factor such that the theoretical
+	 * output (input / scale_factor) rounds up to exactly the desired
+	 * output.
+	 */
+	return min_t(u32, DIV_ROUND_UP(from * 0x1000, to * *dec),
+		     ISI_DOWNSCALE_THRESHOLD);
 }
 
 static void mxc_isi_channel_set_scaling(struct mxc_isi_pipe *pipe,
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c
index a39ad7a1ab18d..de398b232d747 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-m2m.c
@@ -509,9 +509,14 @@ __mxc_isi_m2m_try_fmt_vid(struct mxc_isi_m2m_ctx *ctx,
 			  const enum mxc_isi_video_type type)
 {
 	if (type == MXC_ISI_VIDEO_M2M_CAP) {
-		/* Downscaling only  */
-		pix->width = min(pix->width, ctx->queues.out.format.width);
-		pix->height = min(pix->height, ctx->queues.out.format.height);
+		const struct v4l2_pix_format_mplane *format =
+			&ctx->queues.out.format;
+
+		/* Downscaling only, by up to 16. */
+		pix->width = mxc_isi_clamp_downscale_16(pix->width,
+							format->width);
+		pix->height = mxc_isi_clamp_downscale_16(pix->height,
+							 format->height);
 	}
 
 	return mxc_isi_format_try(ctx->m2m->pipe, pix, type);
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
index a41c51dd9ce0f..2d0843c86534c 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-pipe.c
@@ -641,16 +641,19 @@ static int mxc_isi_pipe_set_selection(struct v4l2_subdev *sd,
 			/* Composing is supported on the sink only. */
 			return -EINVAL;
 
-		/* The sink crop is bound by the sink format downscaling only). */
+		/*
+		 * The ISI supports downscaling only, with a factor up to 16.
+		 * Clamp the compose rectangle size accordingly.
+		 */
 		format = mxc_isi_pipe_get_pad_format(pipe, state,
 						     MXC_ISI_PIPE_PAD_SINK);
 
 		sel->r.left = 0;
 		sel->r.top = 0;
-		sel->r.width = clamp(sel->r.width, MXC_ISI_MIN_WIDTH,
-				     format->width);
-		sel->r.height = clamp(sel->r.height, MXC_ISI_MIN_HEIGHT,
-				      format->height);
+		sel->r.width = mxc_isi_clamp_downscale_16(sel->r.width,
+							  format->width);
+		sel->r.height = mxc_isi_clamp_downscale_16(sel->r.height,
+							   format->height);
 
 		rect = mxc_isi_pipe_get_pad_compose(pipe, state,
 						    MXC_ISI_PIPE_PAD_SINK);
@@ -796,18 +799,20 @@ int mxc_isi_pipe_init(struct mxc_isi_dev *isi, unsigned int id)
 	irq = platform_get_irq(to_platform_device(isi->dev), id);
 	if (irq < 0) {
 		ret = irq;
-		goto error;
+		goto error_subdev;
 	}
 
 	ret = devm_request_irq(isi->dev, irq, mxc_isi_pipe_irq_handler,
 			       0, dev_name(isi->dev), pipe);
 	if (ret < 0) {
 		dev_err(isi->dev, "failed to request IRQ (%d)\n", ret);
-		goto error;
+		goto error_subdev;
 	}
 
 	return 0;
 
+error_subdev:
+	v4l2_subdev_cleanup(sd);
 error:
 	media_entity_cleanup(&sd->entity);
 	mutex_destroy(&pipe->lock);
@@ -819,6 +824,7 @@ void mxc_isi_pipe_cleanup(struct mxc_isi_pipe *pipe)
 {
 	struct v4l2_subdev *sd = &pipe->sd;
 
+	v4l2_subdev_cleanup(sd);
 	media_entity_cleanup(&sd->entity);
 	mutex_destroy(&pipe->lock);
 }
diff --git a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c
index 1be3a728f32f8..fe4adfa3a1f01 100644
--- a/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c
+++ b/drivers/media/platform/nxp/imx8-isi/imx8-isi-video.c
@@ -792,7 +792,11 @@ static void mxc_isi_video_queue_first_buffers(struct mxc_isi_video *video)
 		struct mxc_isi_buffer *buf;
 		struct list_head *list;
 
-		list = i < discard ? &video->out_discard : &video->out_pending;
+		/*
+		 * Queue buffers: prioritize pending buffers, then discard
+		 * buffers.
+		 */
+		list = (i < 2 - discard) ? &video->out_pending : &video->out_discard;
 		buf = list_first_entry(list, struct mxc_isi_buffer, list);
 
 		mxc_isi_channel_set_outbuf(video->pipe, buf->dma_addrs, buf_id);
diff --git a/drivers/media/platform/qcom/camss/Makefile b/drivers/media/platform/qcom/camss/Makefile
index 5e349b4915130..27898b3cc7d3c 100644
--- a/drivers/media/platform/qcom/camss/Makefile
+++ b/drivers/media/platform/qcom/camss/Makefile
@@ -10,10 +10,14 @@ qcom-camss-objs += \
 		camss-csid-680.o \
 		camss-csid-gen2.o \
 		camss-csid-gen3.o \
+		camss-csiphy.o \
 		camss-csiphy-2ph-1-0.o \
 		camss-csiphy-3ph-1-0.o \
-		camss-csiphy.o \
+		camss-format.o \
 		camss-ispif.o \
+		camss-tpg.o \
+		camss-tpg-gen1.o \
+		camss-vfe.o \
 		camss-vfe-4-1.o \
 		camss-vfe-4-7.o \
 		camss-vfe-4-8.o \
@@ -21,11 +25,9 @@ qcom-camss-objs += \
 		camss-vfe-340.o \
 		camss-vfe-480.o \
 		camss-vfe-680.o \
-		camss-vfe-gen3.o \
 		camss-vfe-gen1.o \
+		camss-vfe-gen3.o \
 		camss-vfe-vbif.o \
-		camss-vfe.o \
-		camss-video.o \
-		camss-format.o \
+		camss-video.o
 
 obj-$(CONFIG_VIDEO_QCOM_CAMSS) += qcom-camss.o
diff --git a/drivers/media/platform/qcom/camss/camss-csid-340.c b/drivers/media/platform/qcom/camss/camss-csid-340.c
index 2b50f9b96a34e..0231985746edf 100644
--- a/drivers/media/platform/qcom/camss/camss-csid-340.c
+++ b/drivers/media/platform/qcom/camss/camss-csid-340.c
@@ -74,9 +74,9 @@ static void __csid_ctrl_rdi(struct csid_device *csid, int enable, u8 rdi)
 	writel_relaxed(!!enable, csid->base + CSID_RDI_CTRL(rdi));
 }
 
-static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 vc)
+static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 port, u8 vc)
 {
-	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc];
+	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port];
 	const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats,
 								   csid->res->formats->nformats,
 								   input_format->code);
@@ -88,14 +88,14 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
 	 * the four least significant bits of the five bit VC
 	 * bitfield to generate an internal CID value.
 	 *
-	 * CSID_RDI_CFG0(vc)
+	 * CSID_RDI_CFG0(port)
 	 * DT_ID : 28:27
 	 * VC    : 26:22
 	 * DT    : 21:16
 	 *
 	 * CID   : VC 3:0 << 2 | DT_ID 1:0
 	 */
-	dt_id = vc & 0x03;
+	dt_id = port & 0x03;
 
 	val = CSID_RDI_CFG0_DECODE_FORMAT_NOP; /* only for RDI path */
 	val |= FIELD_PREP(CSID_RDI_CFG0_DT_MASK, format->data_type);
@@ -105,10 +105,11 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
 	if (enable)
 		val |= CSID_RDI_CFG0_ENABLE;
 
-	dev_dbg(csid->camss->dev, "CSID%u: Stream %s (dt:0x%x vc=%u)\n",
-		csid->id, enable ? "enable" : "disable", format->data_type, vc);
+	dev_dbg(csid->camss->dev, "CSID%u: Stream %s (dt:0x%x port=%u vc=%u)\n",
+		csid->id, enable ? "enable" : "disable", format->data_type,
+		port, vc);
 
-	writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_CFG0(port));
 }
 
 static void csid_configure_stream(struct csid_device *csid, u8 enable)
@@ -117,9 +118,10 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable)
 
 	__csid_configure_rx(csid, &csid->phy);
 
+	/* Loop through all enabled ports and configure a stream for each */
 	for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++) {
 		if (csid->phy.en_vc & BIT(i)) {
-			__csid_configure_rdi_stream(csid, enable, i);
+			__csid_configure_rdi_stream(csid, enable, i, 0);
 			__csid_ctrl_rdi(csid, enable, i);
 		}
 	}
diff --git a/drivers/media/platform/qcom/camss/camss-csid-680.c b/drivers/media/platform/qcom/camss/camss-csid-680.c
index 3ad3a174bcfb8..345a67c8fb947 100644
--- a/drivers/media/platform/qcom/camss/camss-csid-680.c
+++ b/drivers/media/platform/qcom/camss/camss-csid-680.c
@@ -103,6 +103,8 @@
 #define		CSI2_RX_CFG0_PHY_NUM_SEL			20
 #define		CSI2_RX_CFG0_PHY_SEL_BASE_IDX			1
 #define		CSI2_RX_CFG0_PHY_TYPE_SEL			24
+#define		CSI2_RX_CFG0_TPG_MUX_EN				BIT(27)
+#define		CSI2_RX_CFG0_TPG_MUX_SEL			GENMASK(29, 28)
 
 #define CSID_CSI2_RX_CFG1					0x204
 #define		CSI2_RX_CFG1_PACKET_ECC_CORRECTION_EN		BIT(0)
@@ -185,10 +187,20 @@ static void __csid_configure_rx(struct csid_device *csid,
 				struct csid_phy_config *phy, int vc)
 {
 	u32 val;
+	struct camss *camss;
 
+	camss = csid->camss;
 	val = (phy->lane_cnt - 1) << CSI2_RX_CFG0_NUM_ACTIVE_LANES;
 	val |= phy->lane_assign << CSI2_RX_CFG0_DL0_INPUT_SEL;
-	val |= (phy->csiphy_id + CSI2_RX_CFG0_PHY_SEL_BASE_IDX) << CSI2_RX_CFG0_PHY_NUM_SEL;
+
+	if (camss->tpg && csid->tpg_linked &&
+	    camss->tpg[phy->csiphy_id].testgen.mode != TPG_PAYLOAD_MODE_DISABLED) {
+		val |= FIELD_PREP(CSI2_RX_CFG0_TPG_MUX_SEL, phy->csiphy_id + 1);
+		val |= CSI2_RX_CFG0_TPG_MUX_EN;
+	} else {
+		val |= (phy->csiphy_id + CSI2_RX_CFG0_PHY_SEL_BASE_IDX)
+			<< CSI2_RX_CFG0_PHY_NUM_SEL;
+	}
 
 	writel(val, csid->base + CSID_CSI2_RX_CFG0);
 
@@ -219,9 +231,9 @@ static void __csid_configure_top(struct csid_device *csid)
 	    CSID_TOP_IO_PATH_CFG0(csid->id));
 }
 
-static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 vc)
+static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 port, u8 vc)
 {
-	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc];
+	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port];
 	const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats,
 								   csid->res->formats->nformats,
 								   input_format->code);
@@ -233,28 +245,28 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
 		lane_cnt = 4;
 
 	val = 0;
-	writel(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(vc));
+	writel(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(port));
 
 	/*
 	 * DT_ID is a two bit bitfield that is concatenated with
 	 * the four least significant bits of the five bit VC
 	 * bitfield to generate an internal CID value.
 	 *
-	 * CSID_RDI_CFG0(vc)
+	 * CSID_RDI_CFG0(port)
 	 * DT_ID : 28:27
 	 * VC    : 26:22
 	 * DT    : 21:16
 	 *
 	 * CID   : VC 3:0 << 2 | DT_ID 1:0
 	 */
-	dt_id = vc & 0x03;
+	dt_id = port & 0x03;
 
 	/* note: for non-RDI path, this should be format->decode_format */
 	val |= DECODE_FORMAT_PAYLOAD_ONLY << RDI_CFG0_DECODE_FORMAT;
 	val |= format->data_type << RDI_CFG0_DATA_TYPE;
 	val |= vc << RDI_CFG0_VIRTUAL_CHANNEL;
 	val |= dt_id << RDI_CFG0_DT_ID;
-	writel(val, csid->base + CSID_RDI_CFG0(vc));
+	writel(val, csid->base + CSID_RDI_CFG0(port));
 
 	val = RDI_CFG1_TIMESTAMP_STB_FRAME;
 	val |= RDI_CFG1_BYTE_CNTR_EN;
@@ -265,23 +277,23 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
 	val |= RDI_CFG1_CROP_V_EN;
 	val |= RDI_CFG1_PACKING_MIPI;
 
-	writel(val, csid->base + CSID_RDI_CFG1(vc));
+	writel(val, csid->base + CSID_RDI_CFG1(port));
 
 	val = 0;
-	writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(vc));
+	writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(port));
 
 	val = 1;
-	writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(vc));
+	writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(port));
 
 	val = 0;
-	writel(val, csid->base + CSID_RDI_CTRL(vc));
+	writel(val, csid->base + CSID_RDI_CTRL(port));
 
-	val = readl(csid->base + CSID_RDI_CFG0(vc));
+	val = readl(csid->base + CSID_RDI_CFG0(port));
 	if (enable)
 		val |= RDI_CFG0_ENABLE;
 	else
 		val &= ~RDI_CFG0_ENABLE;
-	writel(val, csid->base + CSID_RDI_CFG0(vc));
+	writel(val, csid->base + CSID_RDI_CFG0(port));
 }
 
 static void csid_configure_stream(struct csid_device *csid, u8 enable)
@@ -290,11 +302,11 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable)
 
 	__csid_configure_top(csid);
 
-       /* Loop through all enabled VCs and configure stream for each */
+	/* Loop through all enabled ports and configure a stream for each */
 	for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++) {
 		if (csid->phy.en_vc & BIT(i)) {
-			__csid_configure_rdi_stream(csid, enable, i);
-			__csid_configure_rx(csid, &csid->phy, i);
+			__csid_configure_rdi_stream(csid, enable, i, 0);
+			__csid_configure_rx(csid, &csid->phy, 0);
 			__csid_ctrl_rdi(csid, enable, i);
 		}
 	}
diff --git a/drivers/media/platform/qcom/camss/camss-csid-gen2.c b/drivers/media/platform/qcom/camss/camss-csid-gen2.c
index 2a1746dcc1c5b..eadcb2f7e3aaa 100644
--- a/drivers/media/platform/qcom/camss/camss-csid-gen2.c
+++ b/drivers/media/platform/qcom/camss/camss-csid-gen2.c
@@ -203,10 +203,10 @@ static void __csid_ctrl_rdi(struct csid_device *csid, int enable, u8 rdi)
 	writel_relaxed(val, csid->base + CSID_RDI_CTRL(rdi));
 }
 
-static void __csid_configure_testgen(struct csid_device *csid, u8 enable, u8 vc)
+static void __csid_configure_testgen(struct csid_device *csid, u8 enable, u8 port, u8 vc)
 {
 	struct csid_testgen_config *tg = &csid->testgen;
-	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc];
+	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port];
 	const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats,
 								   csid->res->formats->nformats,
 								   input_format->code);
@@ -253,10 +253,10 @@ static void __csid_configure_testgen(struct csid_device *csid, u8 enable, u8 vc)
 	writel_relaxed(val, csid->base + CSID_TPG_CTRL);
 }
 
-static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 vc)
+static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 port, u8 vc)
 {
 	/* Source pads matching RDI channels on hardware. Pad 1 -> RDI0, Pad 2 -> RDI1, etc. */
-	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc];
+	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port];
 	const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats,
 								   csid->res->formats->nformats,
 								   input_format->code);
@@ -267,14 +267,14 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
 	 * the four least significant bits of the five bit VC
 	 * bitfield to generate an internal CID value.
 	 *
-	 * CSID_RDI_CFG0(vc)
+	 * CSID_RDI_CFG0(port)
 	 * DT_ID : 28:27
 	 * VC    : 26:22
 	 * DT    : 21:16
 	 *
 	 * CID   : VC 3:0 << 2 | DT_ID 1:0
 	 */
-	u8 dt_id = vc & 0x03;
+	u8 dt_id = port & 0x03;
 
 	val = 1 << RDI_CFG0_BYTE_CNTR_EN;
 	val |= 1 << RDI_CFG0_FORMAT_MEASURE_EN;
@@ -284,56 +284,57 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
 	val |= format->data_type << RDI_CFG0_DATA_TYPE;
 	val |= vc << RDI_CFG0_VIRTUAL_CHANNEL;
 	val |= dt_id << RDI_CFG0_DT_ID;
-	writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_CFG0(port));
 
 	/* CSID_TIMESTAMP_STB_POST_IRQ */
 	val = 2 << RDI_CFG1_TIMESTAMP_STB_SEL;
-	writel_relaxed(val, csid->base + CSID_RDI_CFG1(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_CFG1(port));
 
 	val = 1;
-	writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PERIOD(port));
 
 	val = 0;
-	writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PATTERN(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_FRM_DROP_PATTERN(port));
 
 	val = 1;
-	writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(port));
 
 	val = 0;
-	writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(port));
 
 	val = 1;
-	writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PERIOD(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PERIOD(port));
 
 	val = 0;
-	writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PATTERN(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_RPP_PIX_DROP_PATTERN(port));
 
 	val = 1;
-	writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PERIOD(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PERIOD(port));
 
 	val = 0;
-	writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PATTERN(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_RPP_LINE_DROP_PATTERN(port));
 
 	val = 0;
-	writel_relaxed(val, csid->base + CSID_RDI_CTRL(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_CTRL(port));
 
-	val = readl_relaxed(csid->base + CSID_RDI_CFG0(vc));
+	val = readl_relaxed(csid->base + CSID_RDI_CFG0(port));
 	val |=  enable << RDI_CFG0_ENABLE;
-	writel_relaxed(val, csid->base + CSID_RDI_CFG0(vc));
+	writel_relaxed(val, csid->base + CSID_RDI_CFG0(port));
 }
 
 static void csid_configure_stream(struct csid_device *csid, u8 enable)
 {
 	struct csid_testgen_config *tg = &csid->testgen;
 	u8 i;
-	/* Loop through all enabled VCs and configure stream for each */
+
+	/* Loop through all enabled ports and configure a stream for each */
 	for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++)
 		if (csid->phy.en_vc & BIT(i)) {
 			if (tg->enabled)
-				__csid_configure_testgen(csid, enable, i);
+				__csid_configure_testgen(csid, enable, i, 0);
 
-			__csid_configure_rdi_stream(csid, enable, i);
-			__csid_configure_rx(csid, &csid->phy, i);
+			__csid_configure_rdi_stream(csid, enable, i, 0);
+			__csid_configure_rx(csid, &csid->phy, 0);
 			__csid_ctrl_rdi(csid, enable, i);
 		}
 }
diff --git a/drivers/media/platform/qcom/camss/camss-csid-gen3.c b/drivers/media/platform/qcom/camss/camss-csid-gen3.c
index bd059243790ed..0fdbf75fb27db 100644
--- a/drivers/media/platform/qcom/camss/camss-csid-gen3.c
+++ b/drivers/media/platform/qcom/camss/camss-csid-gen3.c
@@ -66,6 +66,8 @@
 #define		CSI2_RX_CFG0_VC_MODE		3
 #define		CSI2_RX_CFG0_DL0_INPUT_SEL	4
 #define		CSI2_RX_CFG0_PHY_NUM_SEL	20
+#define		CSI2_RX_CFG0_TPG_MUX_EN		BIT(27)
+#define		CSI2_RX_CFG0_TPG_MUX_SEL	GENMASK(29, 28)
 
 #define CSID_CSI2_RX_CFG1		0x204
 #define		CSI2_RX_CFG1_ECC_CORRECTION_EN	BIT(0)
@@ -109,10 +111,20 @@ static void __csid_configure_rx(struct csid_device *csid,
 				struct csid_phy_config *phy, int vc)
 {
 	int val;
+	struct camss *camss;
 
+	camss = csid->camss;
 	val = (phy->lane_cnt - 1) << CSI2_RX_CFG0_NUM_ACTIVE_LANES;
 	val |= phy->lane_assign << CSI2_RX_CFG0_DL0_INPUT_SEL;
-	val |= (phy->csiphy_id + CSI2_RX_CFG0_PHY_SEL_BASE_IDX) << CSI2_RX_CFG0_PHY_NUM_SEL;
+
+	if (camss->tpg && csid->tpg_linked &&
+	    camss->tpg[phy->csiphy_id].testgen.mode != TPG_PAYLOAD_MODE_DISABLED) {
+		val |= FIELD_PREP(CSI2_RX_CFG0_TPG_MUX_SEL, phy->csiphy_id + 1);
+		val |= CSI2_RX_CFG0_TPG_MUX_EN;
+	} else {
+		val |= (phy->csiphy_id + CSI2_RX_CFG0_PHY_SEL_BASE_IDX)
+			<< CSI2_RX_CFG0_PHY_NUM_SEL;
+	}
 
 	writel(val, csid->base + CSID_CSI2_RX_CFG0);
 
@@ -145,12 +157,12 @@ static void __csid_configure_wrapper(struct csid_device *csid)
 	writel(val, csid->camss->csid_wrapper_base + CSID_IO_PATH_CFG0(csid->id));
 }
 
-static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 vc)
+static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8 port, u8 vc)
 {
 	u32 val;
 	u8 lane_cnt = csid->phy.lane_cnt;
 	/* Source pads matching RDI channels on hardware. Pad 1 -> RDI0, Pad 2 -> RDI1, etc. */
-	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + vc];
+	struct v4l2_mbus_framefmt *input_format = &csid->fmt[MSM_CSID_PAD_FIRST_SRC + port];
 	const struct csid_format_info *format = csid_get_fmt_entry(csid->res->formats->formats,
 								   csid->res->formats->nformats,
 								   input_format->code);
@@ -163,14 +175,14 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
 	 * the four least significant bits of the five bit VC
 	 * bitfield to generate an internal CID value.
 	 *
-	 * CSID_RDI_CFG0(vc)
+	 * CSID_RDI_CFG0(port)
 	 * DT_ID : 28:27
 	 * VC    : 26:22
 	 * DT    : 21:16
 	 *
 	 * CID   : VC 3:0 << 2 | DT_ID 1:0
 	 */
-	u8 dt_id = vc & 0x03;
+	u8 dt_id = port & 0x03;
 
 	val = RDI_CFG0_TIMESTAMP_EN;
 	val |= RDI_CFG0_TIMESTAMP_STB_SEL;
@@ -180,7 +192,7 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
 	val |= format->data_type << RDI_CFG0_DT;
 	val |= dt_id << RDI_CFG0_DT_ID;
 
-	writel(val, csid->base + CSID_RDI_CFG0(vc));
+	writel(val, csid->base + CSID_RDI_CFG0(port));
 
 	val = RDI_CFG1_PACKING_FORMAT_MIPI;
 	val |= RDI_CFG1_PIX_STORE;
@@ -189,22 +201,22 @@ static void __csid_configure_rdi_stream(struct csid_device *csid, u8 enable, u8
 	val |= RDI_CFG1_CROP_H_EN;
 	val |= RDI_CFG1_CROP_V_EN;
 
-	writel(val, csid->base + CSID_RDI_CFG1(vc));
+	writel(val, csid->base + CSID_RDI_CFG1(port));
 
 	val = 0;
-	writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(vc));
+	writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PERIOD(port));
 
 	val = 1;
-	writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(vc));
+	writel(val, csid->base + CSID_RDI_IRQ_SUBSAMPLE_PATTERN(port));
 
 	val = 0;
-	writel(val, csid->base + CSID_RDI_CTRL(vc));
+	writel(val, csid->base + CSID_RDI_CTRL(port));
 
-	val = readl(csid->base + CSID_RDI_CFG0(vc));
+	val = readl(csid->base + CSID_RDI_CFG0(port));
 
 	if (enable)
 		val |= RDI_CFG0_EN;
-	writel(val, csid->base + CSID_RDI_CFG0(vc));
+	writel(val, csid->base + CSID_RDI_CFG0(port));
 }
 
 static void csid_configure_stream(struct csid_device *csid, u8 enable)
@@ -213,11 +225,11 @@ static void csid_configure_stream(struct csid_device *csid, u8 enable)
 
 	__csid_configure_wrapper(csid);
 
-	/* Loop through all enabled VCs and configure stream for each */
+	/* Loop through all enabled ports and configure a stream for each */
 	for (i = 0; i < MSM_CSID_MAX_SRC_STREAMS; i++)
 		if (csid->phy.en_vc & BIT(i)) {
-			__csid_configure_rdi_stream(csid, enable, i);
-			__csid_configure_rx(csid, &csid->phy, i);
+			__csid_configure_rdi_stream(csid, enable, i, 0);
+			__csid_configure_rx(csid, &csid->phy, 0);
 			__csid_ctrl_rdi(csid, enable, i);
 		}
 }
diff --git a/drivers/media/platform/qcom/camss/camss-csid.c b/drivers/media/platform/qcom/camss/camss-csid.c
index ed1820488c987..48459b46a981b 100644
--- a/drivers/media/platform/qcom/camss/camss-csid.c
+++ b/drivers/media/platform/qcom/camss/camss-csid.c
@@ -35,6 +35,8 @@
 #define		HW_VERSION_REVISION	16
 #define		HW_VERSION_GENERATION	28
 
+#define LANE_CFG_BITWIDTH 4
+
 #define MSM_CSID_NAME "msm_csid"
 
 const char * const csid_testgen_modes[] = {
@@ -1215,18 +1217,22 @@ void msm_csid_get_csid_id(struct media_entity *entity, u8 *id)
 }
 
 /*
- * csid_get_lane_assign - Calculate CSI2 lane assign configuration parameter
- * @lane_cfg - CSI2 lane configuration
+ * csid_get_lane_assign - Calculate lane assign by csiphy/tpg lane num
+ * @lane_cfg: CSI2 lane configuration
+ * @num_lanes: lane num
  *
  * Return lane assign
  */
-static u32 csid_get_lane_assign(struct csiphy_lanes_cfg *lane_cfg)
+static u32 csid_get_lane_assign(struct csiphy_lanes_cfg *lane_cfg, int num_lanes)
 {
 	u32 lane_assign = 0;
+	int pos;
 	int i;
 
-	for (i = 0; i < lane_cfg->num_data; i++)
-		lane_assign |= lane_cfg->data[i].pos << (i * 4);
+	for (i = 0; i < num_lanes; i++) {
+		pos = lane_cfg ? lane_cfg->data[i].pos : i;
+		lane_assign |= pos << (i * LANE_CFG_BITWIDTH);
+	}
 
 	return lane_assign;
 }
@@ -1251,6 +1257,7 @@ static int csid_link_setup(struct media_entity *entity,
 	if ((local->flags & MEDIA_PAD_FL_SINK) &&
 	    (flags & MEDIA_LNK_FL_ENABLED)) {
 		struct v4l2_subdev *sd;
+		struct tpg_device *tpg;
 		struct csid_device *csid;
 		struct csiphy_device *csiphy;
 		struct csiphy_lanes_cfg *lane_cfg;
@@ -1265,18 +1272,28 @@ static int csid_link_setup(struct media_entity *entity,
 			return -EBUSY;
 
 		sd = media_entity_to_v4l2_subdev(remote->entity);
-		csiphy = v4l2_get_subdevdata(sd);
+		if (sd->grp_id == TPG_GRP_ID) {
+			tpg = v4l2_get_subdevdata(sd);
 
-		/* If a sensor is not linked to CSIPHY */
-		/* do no allow a link from CSIPHY to CSID */
-		if (!csiphy->cfg.csi2)
-			return -EPERM;
+			csid->phy.lane_cnt = tpg->res->lane_cnt;
+			csid->phy.csiphy_id = tpg->id;
+			csid->phy.lane_assign = csid_get_lane_assign(NULL, csid->phy.lane_cnt);
+			csid->tpg_linked = true;
+		} else {
+			csiphy = v4l2_get_subdevdata(sd);
 
-		csid->phy.csiphy_id = csiphy->id;
+			/* If a sensor is not linked to CSIPHY */
+			/* do no allow a link from CSIPHY to CSID */
+			if (!csiphy->cfg.csi2)
+				return -EPERM;
 
-		lane_cfg = &csiphy->cfg.csi2->lane_cfg;
-		csid->phy.lane_cnt = lane_cfg->num_data;
-		csid->phy.lane_assign = csid_get_lane_assign(lane_cfg);
+			csid->phy.csiphy_id = csiphy->id;
+
+			lane_cfg = &csiphy->cfg.csi2->lane_cfg;
+			csid->phy.lane_cnt = lane_cfg->num_data;
+			csid->phy.lane_assign = csid_get_lane_assign(lane_cfg, lane_cfg->num_data);
+			csid->tpg_linked = false;
+		}
 	}
 	/* Decide which virtual channels to enable based on which source pads are enabled */
 	if (local->flags & MEDIA_PAD_FL_SOURCE) {
diff --git a/drivers/media/platform/qcom/camss/camss-csid.h b/drivers/media/platform/qcom/camss/camss-csid.h
index aedc96ed84b2f..5296b10f6bac8 100644
--- a/drivers/media/platform/qcom/camss/camss-csid.h
+++ b/drivers/media/platform/qcom/camss/camss-csid.h
@@ -161,6 +161,7 @@ struct csid_device {
 	int num_supplies;
 	struct completion reset_complete;
 	struct csid_testgen_config testgen;
+	bool tpg_linked;
 	struct csid_phy_config phy;
 	struct v4l2_mbus_framefmt fmt[MSM_CSID_PADS_NUM];
 	struct v4l2_ctrl_handler ctrls;
diff --git a/drivers/media/platform/qcom/camss/camss-csiphy-3ph-1-0.c b/drivers/media/platform/qcom/camss/camss-csiphy-3ph-1-0.c
index 4154832745525..dac8d2ecf7995 100644
--- a/drivers/media/platform/qcom/camss/camss-csiphy-3ph-1-0.c
+++ b/drivers/media/platform/qcom/camss/camss-csiphy-3ph-1-0.c
@@ -399,6 +399,126 @@ csiphy_lane_regs lane_regs_sm8250[] = {
 	{0x0884, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS},
 };
 
+/* GEN2 1.2.3 2PH */
+static const struct
+csiphy_lane_regs lane_regs_sm6350[] = {
+	{0x0030, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0904, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0910, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0900, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0908, 0x06, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0904, 0x07, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x002c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0034, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0010, 0x50, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x001c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0014, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0028, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x003c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0000, 0x91, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0004, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0020, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0008, 0x10, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{0x0010, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0038, 0xfe, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x005c, 0xc0, 0x00, CSIPHY_SKEW_CAL},
+	{0x0060, 0x0d, 0x00, CSIPHY_SKEW_CAL},
+	{0x0800, 0x02, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0000, 0x00, 0x00, CSIPHY_DNP_PARAMS},
+	{0x0730, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c84, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c90, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c80, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c88, 0x06, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c84, 0x07, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x072c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0734, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0710, 0x50, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x071c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0714, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0728, 0x04, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x073c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0700, 0x80, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0704, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0720, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0708, 0x04, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{0x070c, 0xff, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0710, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0738, 0x1f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0800, 0x02, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0000, 0x00, 0x00, CSIPHY_DNP_PARAMS},
+	{0x0000, 0x00, 0x00, CSIPHY_DNP_PARAMS},
+	{0x0230, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0a04, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0a10, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0a00, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0a08, 0x06, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0a04, 0x07, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x022c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0234, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0210, 0x50, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x021c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0214, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0228, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x023c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0200, 0x91, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0204, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0220, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0208, 0x04, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{0x0210, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0238, 0xfe, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x025c, 0xc0, 0x00, CSIPHY_SKEW_CAL},
+	{0x0260, 0x0d, 0x00, CSIPHY_SKEW_CAL},
+	{0x0800, 0x02, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0000, 0x00, 0x00, CSIPHY_DNP_PARAMS},
+	{0x0430, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0b04, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0b10, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0b00, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0b08, 0x06, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0b04, 0x07, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x042c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0434, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0410, 0x50, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x041c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0414, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0428, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x043c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0400, 0x91, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0404, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0420, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0408, 0x04, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{0x0410, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0438, 0xfe, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x045c, 0xc0, 0x00, CSIPHY_SKEW_CAL},
+	{0x0460, 0x0d, 0x00, CSIPHY_SKEW_CAL},
+	{0x0800, 0x02, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0000, 0x00, 0x00, CSIPHY_DNP_PARAMS},
+	{0x0630, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c04, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c10, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c00, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c08, 0x06, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0c04, 0x07, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x062c, 0x01, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0634, 0x0f, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0610, 0x50, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x061c, 0x0a, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0614, 0x60, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0628, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x063c, 0xb8, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0600, 0x91, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0604, 0x0c, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0620, 0x00, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0608, 0x04, 0x00, CSIPHY_SETTLE_CNT_LOWER_BYTE},
+	{0x0610, 0x52, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0638, 0xfe, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x065c, 0xc0, 0x00, CSIPHY_SKEW_CAL},
+	{0x0660, 0x0d, 0x00, CSIPHY_SKEW_CAL},
+	{0x0800, 0x02, 0x00, CSIPHY_DEFAULT_PARAMS},
+	{0x0000, 0x00, 0x00, CSIPHY_DNP_PARAMS},
+};
+
 /* 14nm 2PH v 2.0.1 2p5Gbps 4 lane DPHY mode */
 static const struct
 csiphy_lane_regs lane_regs_qcm2290[] = {
@@ -1011,6 +1131,7 @@ static bool csiphy_is_gen2(u32 version)
 	switch (version) {
 	case CAMSS_2290:
 	case CAMSS_6150:
+	case CAMSS_6350:
 	case CAMSS_7280:
 	case CAMSS_8250:
 	case CAMSS_8280XP:
@@ -1105,6 +1226,10 @@ static int csiphy_init(struct csiphy_device *csiphy)
 		regs->lane_regs = &lane_regs_qcm2290[0];
 		regs->lane_array_size = ARRAY_SIZE(lane_regs_qcm2290);
 		break;
+	case CAMSS_6350:
+		regs->lane_regs = &lane_regs_sm6350[0];
+		regs->lane_array_size = ARRAY_SIZE(lane_regs_sm6350);
+		break;
 	case CAMSS_7280:
 	case CAMSS_8250:
 		regs->lane_regs = &lane_regs_sm8250[0];
diff --git a/drivers/media/platform/qcom/camss/camss-csiphy.c b/drivers/media/platform/qcom/camss/camss-csiphy.c
index 78a1b568dbae6..539ac4888b608 100644
--- a/drivers/media/platform/qcom/camss/camss-csiphy.c
+++ b/drivers/media/platform/qcom/camss/camss-csiphy.c
@@ -793,6 +793,7 @@ int msm_csiphy_register_entity(struct csiphy_device *csiphy,
 	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 	snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d",
 		 MSM_CSIPHY_NAME, csiphy->id);
+	sd->grp_id = CSIPHY_GRP_ID;
 	v4l2_set_subdevdata(sd, csiphy);
 
 	ret = csiphy_init_formats(sd, NULL);
diff --git a/drivers/media/platform/qcom/camss/camss-csiphy.h b/drivers/media/platform/qcom/camss/camss-csiphy.h
index 2d5054819df7f..9d9657b82f748 100644
--- a/drivers/media/platform/qcom/camss/camss-csiphy.h
+++ b/drivers/media/platform/qcom/camss/camss-csiphy.h
@@ -21,6 +21,8 @@
 #define MSM_CSIPHY_PAD_SRC 1
 #define MSM_CSIPHY_PADS_NUM 2
 
+#define CSIPHY_GRP_ID 1
+
 struct csiphy_lane {
 	u8 pos;
 	u8 pol;
diff --git a/drivers/media/platform/qcom/camss/camss-format.c b/drivers/media/platform/qcom/camss/camss-format.c
index 4a3d5549615cf..52cb01306ee0b 100644
--- a/drivers/media/platform/qcom/camss/camss-format.c
+++ b/drivers/media/platform/qcom/camss/camss-format.c
@@ -7,8 +7,10 @@
  * Copyright (c) 2023, The Linux Foundation. All rights reserved.
  * Copyright (c) 2023 Qualcomm Technologies, Inc.
  */
+#include <linux/bits.h>
 #include <linux/bug.h>
 #include <linux/errno.h>
+#include <linux/lcm.h>
 
 #include "camss-format.h"
 
@@ -33,6 +35,18 @@ u8 camss_format_get_bpp(const struct camss_format_info *formats, unsigned int nf
 	return formats[0].mbus_bpp;
 }
 
+/*
+ * camss_format_get_bpl_alignment - Retrieve required BPL alignment for a given format.
+ * @format: a pointer to the format
+ *
+ * Return the required alignment, in bytes.
+ */
+unsigned int camss_format_get_bpl_alignment(const struct camss_format_info *format)
+{
+	/* Minimal number of bytes required to keep the line length an integer number of pixels */
+	return lcm_not_zero(format->mbus_bpp, BITS_PER_BYTE) / BITS_PER_BYTE;
+}
+
 /*
  * camss_format_find_code - Find a format code in an array
  * @code: a pointer to media bus format codes array
diff --git a/drivers/media/platform/qcom/camss/camss-format.h b/drivers/media/platform/qcom/camss/camss-format.h
index 923a48c9c3fb6..4f87ac8c49751 100644
--- a/drivers/media/platform/qcom/camss/camss-format.h
+++ b/drivers/media/platform/qcom/camss/camss-format.h
@@ -55,6 +55,7 @@ struct camss_formats {
 };
 
 u8 camss_format_get_bpp(const struct camss_format_info *formats, unsigned int nformats, u32 code);
+unsigned int camss_format_get_bpl_alignment(const struct camss_format_info *f);
 u32 camss_format_find_code(u32 *code, unsigned int n_code, unsigned int index, u32 req_code);
 int camss_format_find_format(u32 code, u32 pixelformat, const struct camss_format_info *formats,
 			     unsigned int nformats);
diff --git a/drivers/media/platform/qcom/camss/camss-tpg-gen1.c b/drivers/media/platform/qcom/camss/camss-tpg-gen1.c
new file mode 100644
index 0000000000000..d29de5f93c18e
--- /dev/null
+++ b/drivers/media/platform/qcom/camss/camss-tpg-gen1.c
@@ -0,0 +1,231 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *
+ * Qualcomm MSM Camera Subsystem - TPG (Test Pattern Generator) Module
+ *
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#include <linux/io.h>
+#include <linux/kernel.h>
+
+#include "camss-tpg.h"
+#include "camss.h"
+
+/* TPG global registers */
+#define TPG_HW_VERSION		0x0
+# define HW_VERSION_STEPPING		GENMASK(15, 0)
+# define HW_VERSION_REVISION		GENMASK(27, 16)
+# define HW_VERSION_GENERATION		GENMASK(31, 28)
+
+#define TPG_HW_VER(gen, rev, step) \
+	(((u32)(gen) << 28) | ((u32)(rev) << 16) | (u32)(step))
+
+#define TPG_HW_VER_2_0_0                TPG_HW_VER(2, 0, 0)
+#define TPG_HW_VER_2_1_0                TPG_HW_VER(2, 1, 0)
+
+#define TPG_HW_STATUS		0x4
+
+#define TPG_CTRL		0x64
+# define TPG_CTRL_TEST_EN		BIT(0)
+# define TPG_CTRL_PHY_SEL		BIT(3)
+# define TPG_CTRL_NUM_ACTIVE_LANES	GENMASK(5, 4)
+# define TPG_CTRL_VC_DT_PATTERN_ID	GENMASK(8, 6)
+# define TPG_CTRL_OVERLAP_SHDR_EN	BIT(10)
+# define TPG_CTRL_NUM_ACTIVE_VC		GENMASK(31, 30)
+
+#define TPG_CLEAR		0x1F4
+
+/* TPG VC-based registers */
+#define TPG_VC_n_GAIN_CFG(n)		(0x60 + (n) * 0x60)
+
+#define TPG_VC_n_CFG0(n)	(0x68 + (n) * 0x60)
+# define TPG_VC_n_CFG0_VC_NUM			GENMASK(4, 0)
+# define TPG_VC_n_CFG0_NUM_ACTIVE_DT		GENMASK(9, 8)
+# define TPG_VC_n_CFG0_NUM_BATCH		GENMASK(15, 12)
+# define TPG_VC_n_CFG0_NUM_FRAMES		GENMASK(31, 16)
+
+#define TPG_VC_n_LSFR_SEED(n)	(0x6C + (n) * 0x60)
+#define TPG_VC_n_HBI_CFG(n)	(0x70 + (n) * 0x60)
+#define TPG_VC_n_VBI_CFG(n)	(0x74 + (n) * 0x60)
+
+#define TPG_VC_n_COLOR_BARS_CFG(n)		(0x78 + (n) * 0x60)
+# define TPG_VC_n_COLOR_BARS_CFG_PIX_PATTERN		GENMASK(2, 0)
+# define TPG_VC_n_COLOR_BARS_CFG_QCFA_EN		BIT(3)
+# define TPG_VC_n_COLOR_BARS_CFG_SPLIT_EN		BIT(4)
+# define TPG_VC_n_COLOR_BARS_CFG_NOISE_EN		BIT(5)
+# define TPG_VC_n_COLOR_BARS_CFG_ROTATE_PERIOD		GENMASK(13, 8)
+# define TPG_VC_n_COLOR_BARS_CFG_XCFA_EN		BIT(16)
+# define TPG_VC_n_COLOR_BARS_CFG_SIZE_X			GENMASK(26, 24)
+# define TPG_VC_n_COLOR_BARS_CFG_SIZE_Y			GENMASK(30, 28)
+
+/* TPG DT-based registers */
+#define TPG_VC_m_DT_n_CFG_0(m, n)		(0x7C + (m) * 0x60 + (n) * 0xC)
+# define TPG_VC_m_DT_n_CFG_0_FRAME_HEIGHT	GENMASK(15, 0)
+# define TPG_VC_m_DT_n_CFG_0_FRAME_WIDTH	GENMASK(31, 16)
+
+#define TPG_VC_m_DT_n_CFG_1(m, n)		(0x80 + (m) * 0x60 + (n) * 0xC)
+# define TPG_VC_m_DT_n_CFG_1_DATA_TYPE		GENMASK(5, 0)
+# define TPG_VC_m_DT_n_CFG_1_ECC_XOR_MASK	GENMASK(13, 8)
+# define TPG_VC_m_DT_n_CFG_1_CRC_XOR_MASK	GENMASK(31, 16)
+
+#define TPG_VC_m_DT_n_CFG_2(m, n)		(0x84 + (m) * 0x60 + (n) * 0xC)
+# define TPG_VC_m_DT_n_CFG_2_PAYLOAD_MODE		GENMASK(3, 0)
+/* v2.0.0: USER[19:4], ENC[23:20] */
+# define TPG_V2_0_0_VC_m_DT_n_CFG_2_USER_SPECIFIED_PAYLOAD		GENMASK(19, 4)
+# define TPG_V2_0_0_VC_m_DT_n_CFG_2_ENCODE_FORMAT			GENMASK(23, 20)
+/* v2.1.0: USER[27:4], ENC[31:28] */
+# define TPG_V2_1_0_VC_m_DT_n_CFG_2_USER_SPECIFIED_PAYLOAD	GENMASK(27, 4)
+# define TPG_V2_1_0_VC_m_DT_n_CFG_2_ENCODE_FORMAT			GENMASK(31, 28)
+
+#define TPG_HBI_PCT_DEFAULT			545	/* 545% */
+#define TPG_VBI_PCT_DEFAULT			10	/* 10% */
+#define PERCENT_BASE				100
+
+/* Default user-specified payload for TPG test generator.
+ * Keep consistent with CSID TPG default: 0xBE.
+ */
+#define TPG_USER_SPECIFIED_PAYLOAD_DEFAULT	0xBE
+#define TPG_LFSR_SEED_DEFAULT			0x12345678
+#define TPG_COLOR_BARS_CFG_STANDARD \
+	FIELD_PREP(TPG_VC_n_COLOR_BARS_CFG_ROTATE_PERIOD, 0xA)
+
+static const char * const testgen_payload_modes[] = {
+	[TPG_PAYLOAD_MODE_DISABLED]		= "Disabled",
+	[TPG_PAYLOAD_MODE_INCREMENTING]		= "Incrementing",
+	[TPG_PAYLOAD_MODE_ALTERNATING_55_AA]	= "Alternating 0x55/0xAA",
+	[TPG_PAYLOAD_MODE_RANDOM]		= "Pseudo-random Data",
+	[TPG_PAYLOAD_MODE_USER_SPECIFIED]	= "User Specified",
+	[TPG_PAYLOAD_MODE_COLOR_BARS]		= "Color bars",
+};
+
+static int tpg_stream_on(struct tpg_device *tpg)
+{
+	struct tpg_testgen_config *tg = &tpg->testgen;
+	struct v4l2_mbus_framefmt *input_format;
+	const struct tpg_format_info *format;
+	u8 payload_mode = (tg->mode > TPG_PAYLOAD_MODE_DISABLED) ?
+			   tg->mode - 1 : 0;
+	u8 lane_cnt = tpg->res->lane_cnt;
+	u8 vc, dt, last_vc = 0;
+	u32 val;
+
+	for (vc = 0; vc <= MSM_TPG_ACTIVE_VC; vc++) {
+		last_vc = vc;
+
+		input_format = &tpg->fmt;
+		format = tpg_get_fmt_entry(tpg->res->formats->formats,
+					   tpg->res->formats->nformats,
+					   input_format->code);
+		if (IS_ERR(format))
+			return -EINVAL;
+
+		/* VC configuration */
+		val = FIELD_PREP(TPG_VC_n_CFG0_NUM_ACTIVE_DT, MSM_TPG_ACTIVE_DT) |
+		      FIELD_PREP(TPG_VC_n_CFG0_NUM_FRAMES, 0);
+		writel(val, tpg->base + TPG_VC_n_CFG0(vc));
+
+		writel(TPG_LFSR_SEED_DEFAULT, tpg->base + TPG_VC_n_LSFR_SEED(vc));
+
+		val = DIV_ROUND_UP(input_format->width * format->bpp * TPG_HBI_PCT_DEFAULT,
+				   BITS_PER_BYTE * lane_cnt * PERCENT_BASE);
+		writel(val, tpg->base + TPG_VC_n_HBI_CFG(vc));
+
+		val = input_format->height * TPG_VBI_PCT_DEFAULT / PERCENT_BASE;
+		writel(val, tpg->base + TPG_VC_n_VBI_CFG(vc));
+
+		writel(TPG_COLOR_BARS_CFG_STANDARD, tpg->base + TPG_VC_n_COLOR_BARS_CFG(vc));
+
+		/* DT configuration */
+		for (dt = 0; dt <= MSM_TPG_ACTIVE_DT; dt++) {
+			val = FIELD_PREP(TPG_VC_m_DT_n_CFG_0_FRAME_HEIGHT,
+					 input_format->height & 0xffff) |
+			      FIELD_PREP(TPG_VC_m_DT_n_CFG_0_FRAME_WIDTH,
+					 input_format->width & 0xffff);
+			writel(val, tpg->base + TPG_VC_m_DT_n_CFG_0(vc, dt));
+
+			val = FIELD_PREP(TPG_VC_m_DT_n_CFG_1_DATA_TYPE, format->data_type);
+			writel(val, tpg->base + TPG_VC_m_DT_n_CFG_1(vc, dt));
+
+			if (tpg->hw_version == TPG_HW_VER_2_0_0) {
+				val = FIELD_PREP(TPG_VC_m_DT_n_CFG_2_PAYLOAD_MODE, payload_mode) |
+				      FIELD_PREP(TPG_V2_0_0_VC_m_DT_n_CFG_2_USER_SPECIFIED_PAYLOAD,
+						 TPG_USER_SPECIFIED_PAYLOAD_DEFAULT) |
+				      FIELD_PREP(TPG_V2_0_0_VC_m_DT_n_CFG_2_ENCODE_FORMAT,
+						 format->encode_format);
+			} else if (tpg->hw_version >= TPG_HW_VER_2_1_0) {
+				val = FIELD_PREP(TPG_VC_m_DT_n_CFG_2_PAYLOAD_MODE, payload_mode) |
+				      FIELD_PREP(TPG_V2_1_0_VC_m_DT_n_CFG_2_USER_SPECIFIED_PAYLOAD,
+						 TPG_USER_SPECIFIED_PAYLOAD_DEFAULT) |
+				      FIELD_PREP(TPG_V2_1_0_VC_m_DT_n_CFG_2_ENCODE_FORMAT,
+						 format->encode_format);
+			}
+			writel(val, tpg->base + TPG_VC_m_DT_n_CFG_2(vc, dt));
+		}
+	}
+
+	/* Global TPG control */
+	val = FIELD_PREP(TPG_CTRL_TEST_EN, 1) |
+	      FIELD_PREP(TPG_CTRL_NUM_ACTIVE_LANES, lane_cnt - 1) |
+	      FIELD_PREP(TPG_CTRL_NUM_ACTIVE_VC, last_vc);
+	writel(val, tpg->base + TPG_CTRL);
+
+	return 0;
+}
+
+static int tpg_reset(struct tpg_device *tpg)
+{
+	writel(0, tpg->base + TPG_CTRL);
+	writel(1, tpg->base + TPG_CLEAR);
+
+	return 0;
+}
+
+static void tpg_stream_off(struct tpg_device *tpg)
+{
+	tpg_reset(tpg);
+}
+
+static int tpg_configure_stream(struct tpg_device *tpg, u8 enable)
+{
+	if (enable)
+		return tpg_stream_on(tpg);
+
+	tpg_stream_off(tpg);
+
+	return 0;
+}
+
+static int tpg_configure_testgen_pattern(struct tpg_device *tpg, s32 val)
+{
+	if (val >= 0 && val <= TPG_PAYLOAD_MODE_COLOR_BARS)
+		tpg->testgen.mode = val;
+
+	return 0;
+}
+
+static u32 tpg_hw_version(struct tpg_device *tpg)
+{
+	u32 hw_version = readl(tpg->base + TPG_HW_VERSION);
+
+	tpg->hw_version = hw_version;
+	dev_dbg(tpg->camss->dev, "tpg HW Version = %u.%u.%u\n",
+		(u32)FIELD_GET(HW_VERSION_GENERATION, hw_version),
+		(u32)FIELD_GET(HW_VERSION_REVISION, hw_version),
+		(u32)FIELD_GET(HW_VERSION_STEPPING, hw_version));
+
+	return hw_version;
+}
+
+static void tpg_subdev_init(struct tpg_device *tpg)
+{
+	tpg->testgen.modes = testgen_payload_modes;
+	tpg->testgen.nmodes = TPG_PAYLOAD_MODE_NUM_SUPPORTED_GEN1;
+}
+
+const struct tpg_hw_ops tpg_ops_gen1 = {
+	.configure_stream = tpg_configure_stream,
+	.configure_testgen_pattern = tpg_configure_testgen_pattern,
+	.hw_version = tpg_hw_version,
+	.reset = tpg_reset,
+	.subdev_init = tpg_subdev_init,
+};
diff --git a/drivers/media/platform/qcom/camss/camss-tpg.c b/drivers/media/platform/qcom/camss/camss-tpg.c
new file mode 100644
index 0000000000000..c5b75132add44
--- /dev/null
+++ b/drivers/media/platform/qcom/camss/camss-tpg.c
@@ -0,0 +1,519 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *
+ * Qualcomm MSM Camera Subsystem - TPG Module
+ *
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
+#include <media/media-entity.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-subdev.h>
+#include <media/mipi-csi2.h>
+
+#include "camss-tpg.h"
+#include "camss.h"
+
+static const struct tpg_format_info formats_gen1[] = {
+	{
+		MEDIA_BUS_FMT_SBGGR8_1X8,
+		MIPI_CSI2_DT_RAW8,
+		ENCODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG8_1X8,
+		MIPI_CSI2_DT_RAW8,
+		ENCODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG8_1X8,
+		MIPI_CSI2_DT_RAW8,
+		ENCODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB8_1X8,
+		MIPI_CSI2_DT_RAW8,
+		ENCODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR10_1X10,
+		MIPI_CSI2_DT_RAW10,
+		ENCODE_FORMAT_UNCOMPRESSED_10_BIT,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG10_1X10,
+		MIPI_CSI2_DT_RAW10,
+		ENCODE_FORMAT_UNCOMPRESSED_10_BIT,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG10_1X10,
+		MIPI_CSI2_DT_RAW10,
+		ENCODE_FORMAT_UNCOMPRESSED_10_BIT,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB10_1X10,
+		MIPI_CSI2_DT_RAW10,
+		ENCODE_FORMAT_UNCOMPRESSED_10_BIT,
+		10,
+	},
+	{
+		MEDIA_BUS_FMT_SBGGR12_1X12,
+		MIPI_CSI2_DT_RAW12,
+		ENCODE_FORMAT_UNCOMPRESSED_12_BIT,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_SGBRG12_1X12,
+		MIPI_CSI2_DT_RAW12,
+		ENCODE_FORMAT_UNCOMPRESSED_12_BIT,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_SGRBG12_1X12,
+		MIPI_CSI2_DT_RAW12,
+		ENCODE_FORMAT_UNCOMPRESSED_12_BIT,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_SRGGB12_1X12,
+		MIPI_CSI2_DT_RAW12,
+		ENCODE_FORMAT_UNCOMPRESSED_12_BIT,
+		12,
+	},
+	{
+		MEDIA_BUS_FMT_Y8_1X8,
+		MIPI_CSI2_DT_RAW8,
+		ENCODE_FORMAT_UNCOMPRESSED_8_BIT,
+		8,
+	},
+	{
+		MEDIA_BUS_FMT_Y10_1X10,
+		MIPI_CSI2_DT_RAW10,
+		ENCODE_FORMAT_UNCOMPRESSED_10_BIT,
+		10,
+	},
+};
+
+const struct tpg_formats tpg_formats_gen1 = {
+	.nformats = ARRAY_SIZE(formats_gen1),
+	.formats = formats_gen1
+};
+
+const struct tpg_format_info *tpg_get_fmt_entry(const struct tpg_format_info *formats,
+						unsigned int nformats,
+						u32 code)
+{
+	unsigned int i;
+
+	for (i = 0; i < nformats; i++)
+		if (code == formats[i].code)
+			return &formats[i];
+
+	return ERR_PTR(-EINVAL);
+}
+
+static int tpg_set_clock_rates(struct tpg_device *tpg)
+{
+	struct device *dev = tpg->camss->dev;
+	int i, ret;
+
+	for (i = 0; i < tpg->nclocks; i++) {
+		struct camss_clock *clock = &tpg->clock[i];
+		long round_rate;
+
+		if (clock->freq) {
+			round_rate = clk_round_rate(clock->clk, clock->freq[0]);
+			if (round_rate < 0) {
+				dev_err(dev, "clk round rate failed: %ld\n",
+					round_rate);
+				return -EINVAL;
+			}
+
+			ret = clk_set_rate(clock->clk, round_rate);
+			if (ret < 0) {
+				dev_err(dev, "clk set rate failed: %d\n", ret);
+				return ret;
+			}
+		}
+	}
+
+	return 0;
+}
+
+static int tpg_set_power(struct v4l2_subdev *sd, int on)
+{
+	struct tpg_device *tpg = v4l2_get_subdevdata(sd);
+	struct device *dev = tpg->camss->dev;
+
+	if (on) {
+		int ret;
+
+		ret = pm_runtime_resume_and_get(dev);
+		if (ret < 0)
+			return ret;
+
+		ret = tpg_set_clock_rates(tpg);
+		if (ret < 0) {
+			pm_runtime_put_sync(dev);
+			return ret;
+		}
+
+		ret = camss_enable_clocks(tpg->nclocks, tpg->clock, dev);
+		if (ret < 0) {
+			pm_runtime_put_sync(dev);
+			return ret;
+		}
+
+		tpg->res->hw_ops->reset(tpg);
+
+		tpg->res->hw_ops->hw_version(tpg);
+	} else {
+		camss_disable_clocks(tpg->nclocks, tpg->clock);
+
+		pm_runtime_put_sync(dev);
+	}
+
+	return 0;
+}
+
+static int tpg_set_stream(struct v4l2_subdev *sd, int enable)
+{
+	struct tpg_device *tpg = v4l2_get_subdevdata(sd);
+	int ret;
+
+	if (enable) {
+		ret = v4l2_ctrl_handler_setup(&tpg->ctrls);
+		if (ret < 0) {
+			dev_err(tpg->camss->dev,
+				"could not sync v4l2 controls: %d\n", ret);
+			return ret;
+		}
+	}
+
+	return tpg->res->hw_ops->configure_stream(tpg, enable);
+}
+
+static struct v4l2_mbus_framefmt *
+__tpg_get_format(struct tpg_device *tpg,
+		 struct v4l2_subdev_state *sd_state,
+		 unsigned int pad,
+		 enum v4l2_subdev_format_whence which)
+{
+	if (which == V4L2_SUBDEV_FORMAT_TRY)
+		return v4l2_subdev_state_get_format(sd_state,
+						    pad);
+
+	return &tpg->fmt;
+}
+
+static void tpg_try_format(struct tpg_device *tpg,
+			   struct v4l2_mbus_framefmt *fmt)
+{
+	unsigned int i;
+
+	for (i = 0; i < tpg->res->formats->nformats; i++)
+		if (tpg->res->formats->formats[i].code == fmt->code)
+			break;
+
+	if (i >= tpg->res->formats->nformats)
+		fmt->code = MEDIA_BUS_FMT_SBGGR8_1X8;
+
+	fmt->width = clamp_t(u32, fmt->width, TPG_MIN_WIDTH, TPG_MAX_WIDTH);
+	fmt->height = clamp_t(u32, fmt->height, TPG_MIN_HEIGHT, TPG_MAX_HEIGHT);
+	fmt->field = V4L2_FIELD_NONE;
+	fmt->colorspace = V4L2_COLORSPACE_SRGB;
+}
+
+static int tpg_enum_mbus_code(struct v4l2_subdev *sd,
+			      struct v4l2_subdev_state *sd_state,
+			      struct v4l2_subdev_mbus_code_enum *code)
+{
+	struct tpg_device *tpg = v4l2_get_subdevdata(sd);
+
+	if (code->index >= tpg->res->formats->nformats)
+		return -EINVAL;
+
+	code->code = tpg->res->formats->formats[code->index].code;
+
+	return 0;
+}
+
+static int tpg_enum_frame_size(struct v4l2_subdev *sd,
+			       struct v4l2_subdev_state *sd_state,
+			       struct v4l2_subdev_frame_size_enum *fse)
+{
+	struct tpg_device *tpg = v4l2_get_subdevdata(sd);
+	unsigned int i;
+
+	if (fse->index != 0)
+		return -EINVAL;
+
+	for (i = 0; i < tpg->res->formats->nformats; i++)
+		if (tpg->res->formats->formats[i].code == fse->code)
+			break;
+
+	if (i >= tpg->res->formats->nformats)
+		return -EINVAL;
+
+	fse->min_width = TPG_MIN_WIDTH;
+	fse->min_height = TPG_MIN_HEIGHT;
+	fse->max_width = TPG_MAX_WIDTH;
+	fse->max_height = TPG_MAX_HEIGHT;
+
+	return 0;
+}
+
+static int tpg_get_format(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_state *sd_state,
+			  struct v4l2_subdev_format *fmt)
+{
+	struct tpg_device *tpg = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __tpg_get_format(tpg, sd_state, fmt->pad, fmt->which);
+	if (!format)
+		return -EINVAL;
+
+	fmt->format = *format;
+
+	return 0;
+}
+
+static int tpg_set_format(struct v4l2_subdev *sd,
+			  struct v4l2_subdev_state *sd_state,
+			  struct v4l2_subdev_format *fmt)
+{
+	struct tpg_device *tpg = v4l2_get_subdevdata(sd);
+	struct v4l2_mbus_framefmt *format;
+
+	format = __tpg_get_format(tpg, sd_state, fmt->pad, fmt->which);
+	if (!format)
+		return -EINVAL;
+
+	tpg_try_format(tpg, &fmt->format);
+	*format = fmt->format;
+
+	return 0;
+}
+
+static int tpg_init_formats(struct v4l2_subdev *sd,
+			    struct v4l2_subdev_fh *fh)
+{
+	struct v4l2_subdev_format format = {
+		.pad = MSM_TPG_PAD_SRC,
+		.which = fh ? V4L2_SUBDEV_FORMAT_TRY :
+			      V4L2_SUBDEV_FORMAT_ACTIVE,
+		.format = {
+			.code = MEDIA_BUS_FMT_SBGGR8_1X8,
+			.width = 1920,
+			.height = 1080,
+		}
+	};
+
+	return tpg_set_format(sd, fh ? fh->state : NULL, &format);
+}
+
+static int tpg_s_ctrl(struct v4l2_ctrl *ctrl)
+{
+	struct tpg_device *tpg = container_of(ctrl->handler,
+					      struct tpg_device, ctrls);
+	int ret = -EINVAL;
+
+	switch (ctrl->id) {
+	case V4L2_CID_TEST_PATTERN:
+		ret = tpg->res->hw_ops->configure_testgen_pattern(tpg, ctrl->val);
+		break;
+	}
+
+	return ret;
+}
+
+static const struct v4l2_ctrl_ops tpg_ctrl_ops = {
+	.s_ctrl = tpg_s_ctrl,
+};
+
+int msm_tpg_subdev_init(struct camss *camss,
+			struct tpg_device *tpg,
+			const struct camss_subdev_resources *res, u8 id)
+{
+	struct platform_device *pdev;
+	struct device *dev;
+	int i, j;
+
+	dev  = camss->dev;
+	pdev = to_platform_device(dev);
+
+	tpg->camss = camss;
+	tpg->id = id;
+	tpg->res = &res->tpg;
+	tpg->res->hw_ops->subdev_init(tpg);
+
+	tpg->base = devm_platform_ioremap_resource_byname(pdev, res->reg[0]);
+	if (IS_ERR(tpg->base))
+		return PTR_ERR(tpg->base);
+
+	tpg->nclocks = 0;
+	while (res->clock[tpg->nclocks])
+		tpg->nclocks++;
+
+	if (!tpg->nclocks)
+		return 0;
+
+	tpg->clock = devm_kcalloc(dev, tpg->nclocks,
+				  sizeof(*tpg->clock), GFP_KERNEL);
+	if (!tpg->clock)
+		return -ENOMEM;
+
+	for (i = 0; i < tpg->nclocks; i++) {
+		struct camss_clock *clock = &tpg->clock[i];
+
+		clock->clk = devm_clk_get(dev, res->clock[i]);
+		if (IS_ERR(clock->clk))
+			return PTR_ERR(clock->clk);
+
+		clock->name = res->clock[i];
+
+		clock->nfreqs = 0;
+		while (res->clock_rate[i][clock->nfreqs])
+			clock->nfreqs++;
+
+		if (!clock->nfreqs) {
+			clock->freq = NULL;
+			continue;
+		}
+
+		clock->freq = devm_kcalloc(dev, clock->nfreqs,
+					   sizeof(*clock->freq), GFP_KERNEL);
+		if (!clock->freq)
+			return -ENOMEM;
+
+		for (j = 0; j < clock->nfreqs; j++)
+			clock->freq[j] = res->clock_rate[i][j];
+	}
+
+	return 0;
+}
+
+static int tpg_link_setup(struct media_entity *entity,
+			  const struct media_pad *local,
+			  const struct media_pad *remote, u32 flags)
+{
+	if (flags & MEDIA_LNK_FL_ENABLED)
+		if (media_pad_remote_pad_first(local))
+			return -EBUSY;
+
+	return 0;
+}
+
+static const struct v4l2_subdev_core_ops tpg_core_ops = {
+	.s_power = tpg_set_power,
+};
+
+static const struct v4l2_subdev_video_ops tpg_video_ops = {
+	.s_stream = tpg_set_stream,
+};
+
+static const struct v4l2_subdev_pad_ops tpg_pad_ops = {
+	.enum_mbus_code = tpg_enum_mbus_code,
+	.enum_frame_size = tpg_enum_frame_size,
+	.get_fmt = tpg_get_format,
+	.set_fmt = tpg_set_format,
+};
+
+static const struct v4l2_subdev_ops tpg_v4l2_ops = {
+	.core = &tpg_core_ops,
+	.video = &tpg_video_ops,
+	.pad = &tpg_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops tpg_v4l2_internal_ops = {
+	.open = tpg_init_formats,
+};
+
+static const struct media_entity_operations tpg_media_ops = {
+	.link_setup = tpg_link_setup,
+	.link_validate = v4l2_subdev_link_validate,
+};
+
+int msm_tpg_register_entity(struct tpg_device *tpg,
+			    struct v4l2_device *v4l2_dev)
+{
+	struct v4l2_subdev *sd = &tpg->subdev;
+	struct device *dev = tpg->camss->dev;
+	int ret;
+
+	v4l2_subdev_init(sd, &tpg_v4l2_ops);
+	sd->internal_ops = &tpg_v4l2_internal_ops;
+	sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE |
+		     V4L2_SUBDEV_FL_HAS_EVENTS;
+	snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d",
+		 "msm_tpg", tpg->id);
+	sd->grp_id = TPG_GRP_ID;
+	v4l2_set_subdevdata(sd, tpg);
+
+	ret = v4l2_ctrl_handler_init(&tpg->ctrls, 1);
+	if (ret < 0) {
+		dev_err(dev, "Failed to init ctrl handler: %d\n", ret);
+		return ret;
+	}
+
+	tpg->testgen_mode = v4l2_ctrl_new_std_menu_items(&tpg->ctrls,
+							 &tpg_ctrl_ops, V4L2_CID_TEST_PATTERN,
+							 tpg->testgen.nmodes, 0, 0,
+							 tpg->testgen.modes);
+	if (tpg->ctrls.error) {
+		dev_err(dev, "Failed to init ctrl: %d\n", tpg->ctrls.error);
+		ret = tpg->ctrls.error;
+		goto free_ctrl;
+	}
+
+	tpg->subdev.ctrl_handler = &tpg->ctrls;
+
+	ret = tpg_init_formats(sd, NULL);
+	if (ret < 0) {
+		dev_err(dev, "Failed to init format: %d\n", ret);
+		goto free_ctrl;
+	}
+
+	tpg->pad.flags = MEDIA_PAD_FL_SOURCE;
+
+	sd->entity.ops = &tpg_media_ops;
+	ret = media_entity_pads_init(&sd->entity, 1, &tpg->pad);
+	if (ret < 0) {
+		dev_err(dev, "Failed to init media entity: %d\n", ret);
+		goto free_ctrl;
+	}
+
+	ret = v4l2_device_register_subdev(v4l2_dev, sd);
+	if (ret < 0) {
+		dev_err(dev, "Failed to register subdev: %d\n", ret);
+		media_entity_cleanup(&sd->entity);
+		goto free_ctrl;
+	}
+
+	return 0;
+
+free_ctrl:
+	v4l2_ctrl_handler_free(&tpg->ctrls);
+
+	return ret;
+}
+
+void msm_tpg_unregister_entity(struct tpg_device *tpg)
+{
+	v4l2_device_unregister_subdev(&tpg->subdev);
+	media_entity_cleanup(&tpg->subdev.entity);
+	v4l2_ctrl_handler_free(&tpg->ctrls);
+}
diff --git a/drivers/media/platform/qcom/camss/camss-tpg.h b/drivers/media/platform/qcom/camss/camss-tpg.h
new file mode 100644
index 0000000000000..7fb35a97dd068
--- /dev/null
+++ b/drivers/media/platform/qcom/camss/camss-tpg.h
@@ -0,0 +1,118 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * camss-tpg.h
+ *
+ * Qualcomm MSM Camera Subsystem - TPG Module
+ *
+ * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+#ifndef QC_MSM_CAMSS_TPG_H
+#define QC_MSM_CAMSS_TPG_H
+
+#include <linux/clk.h>
+#include <linux/bitfield.h>
+#include <media/media-entity.h>
+#include <media/v4l2-ctrls.h>
+#include <media/v4l2-device.h>
+#include <media/v4l2-mediabus.h>
+#include <media/v4l2-subdev.h>
+
+#define ENCODE_FORMAT_UNCOMPRESSED_8_BIT	0x1
+#define ENCODE_FORMAT_UNCOMPRESSED_10_BIT	0x2
+#define ENCODE_FORMAT_UNCOMPRESSED_12_BIT	0x3
+#define ENCODE_FORMAT_UNCOMPRESSED_14_BIT	0x4
+#define ENCODE_FORMAT_UNCOMPRESSED_16_BIT	0x5
+#define ENCODE_FORMAT_UNCOMPRESSED_20_BIT	0x6
+#define ENCODE_FORMAT_UNCOMPRESSED_24_BIT	0x7
+
+#define MSM_TPG_PAD_SRC		0
+#define MSM_TPG_ACTIVE_VC		0
+#define MSM_TPG_ACTIVE_DT		0
+
+#define TPG_MIN_WIDTH   1
+#define TPG_MIN_HEIGHT  1
+#define TPG_MAX_WIDTH   8191
+#define TPG_MAX_HEIGHT  8191
+
+#define TPG_GRP_ID 0
+
+enum tpg_testgen_mode {
+	TPG_PAYLOAD_MODE_DISABLED = 0,
+	TPG_PAYLOAD_MODE_INCREMENTING = 1,
+	TPG_PAYLOAD_MODE_ALTERNATING_55_AA = 2,
+	TPG_PAYLOAD_MODE_RANDOM = 5,
+	TPG_PAYLOAD_MODE_USER_SPECIFIED = 6,
+	TPG_PAYLOAD_MODE_COLOR_BARS = 9,
+	TPG_PAYLOAD_MODE_NUM_SUPPORTED_GEN1 = 9,
+};
+
+struct tpg_testgen_config {
+	enum tpg_testgen_mode mode;
+	const char * const*modes;
+	u8 nmodes;
+};
+
+struct tpg_format_info {
+	u32 code;
+	u8 data_type;
+	u8 encode_format;
+	u8 bpp;
+};
+
+struct tpg_formats {
+	unsigned int nformats;
+	const struct tpg_format_info *formats;
+};
+
+struct tpg_device;
+
+struct tpg_hw_ops {
+	int (*configure_stream)(struct tpg_device *tpg, u8 enable);
+	int (*configure_testgen_pattern)(struct tpg_device *tpg, s32 val);
+	u32 (*hw_version)(struct tpg_device *tpg);
+	int (*reset)(struct tpg_device *tpg);
+	void (*subdev_init)(struct tpg_device *tpg);
+};
+
+struct tpg_subdev_resources {
+	u8 lane_cnt;
+	const struct tpg_formats *formats;
+	const struct tpg_hw_ops *hw_ops;
+};
+
+struct tpg_device {
+	struct camss *camss;
+	u8 id;
+	struct v4l2_subdev subdev;
+	struct media_pad pad;
+	void __iomem *base;
+	struct camss_clock *clock;
+	int nclocks;
+	struct tpg_testgen_config testgen;
+	struct v4l2_mbus_framefmt fmt;
+	struct v4l2_ctrl_handler ctrls;
+	struct v4l2_ctrl *testgen_mode;
+	const struct tpg_subdev_resources *res;
+	u32 hw_version;
+};
+
+struct camss_subdev_resources;
+
+const struct tpg_format_info *tpg_get_fmt_entry(const struct tpg_format_info *formats,
+						unsigned int nformats,
+						u32 code);
+
+int msm_tpg_subdev_init(struct camss *camss,
+			struct tpg_device *tpg,
+			const struct camss_subdev_resources *res, u8 id);
+
+int msm_tpg_register_entity(struct tpg_device *tpg,
+			    struct v4l2_device *v4l2_dev);
+
+void msm_tpg_unregister_entity(struct tpg_device *tpg);
+
+extern const struct tpg_formats tpg_formats_gen1;
+
+extern const struct tpg_hw_ops tpg_ops_gen1;
+
+#endif /* QC_MSM_CAMSS_TPG_H */
diff --git a/drivers/media/platform/qcom/camss/camss-vfe.c b/drivers/media/platform/qcom/camss/camss-vfe.c
index 5baf0e3d4bc46..319d191589884 100644
--- a/drivers/media/platform/qcom/camss/camss-vfe.c
+++ b/drivers/media/platform/qcom/camss/camss-vfe.c
@@ -343,6 +343,7 @@ static u32 vfe_src_pad_code(struct vfe_line *line, u32 sink_code,
 	case CAMSS_660:
 	case CAMSS_2290:
 	case CAMSS_6150:
+	case CAMSS_6350:
 	case CAMSS_7280:
 	case CAMSS_8x96:
 	case CAMSS_8250:
@@ -1997,12 +1998,13 @@ static const struct media_entity_operations vfe_media_ops = {
 	.link_validate = v4l2_subdev_link_validate,
 };
 
-static int vfe_bpl_align(struct vfe_device *vfe)
+static int vfe_bpl_align_rdi(struct vfe_device *vfe)
 {
 	int ret = 8;
 
 	switch (vfe->camss->res->version) {
 	case CAMSS_6150:
+	case CAMSS_6350:
 	case CAMSS_7280:
 	case CAMSS_8250:
 	case CAMSS_8280XP:
@@ -2021,6 +2023,24 @@ static int vfe_bpl_align(struct vfe_device *vfe)
 	return ret;
 }
 
+static int vfe_bpl_align_pix(struct vfe_device *vfe)
+{
+	int ret = 16;
+
+	switch (vfe->camss->res->version) {
+	case CAMSS_2290:
+		/* The alignment/bpl depends solely on the pixel format and is
+		 * computed dynamically in camss_format_get_bpl_alignment().
+		 */
+		ret = 0;
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
 /*
  * msm_vfe_register_entities - Register subdev node for VFE module
  * @vfe: VFE device
@@ -2053,7 +2073,7 @@ int msm_vfe_register_entities(struct vfe_device *vfe,
 		v4l2_subdev_init(sd, &vfe_v4l2_ops);
 		sd->internal_ops = &vfe_v4l2_internal_ops;
 		sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
-		if (i == VFE_LINE_PIX)
+		if (i == VFE_LINE_PIX && vfe->res->is_lite == false)
 			snprintf(sd->name, ARRAY_SIZE(sd->name), "%s%d_%s",
 				 MSM_VFE_NAME, vfe->id, "pix");
 		else
@@ -2087,11 +2107,12 @@ int msm_vfe_register_entities(struct vfe_device *vfe,
 		}
 
 		video_out->ops = &vfe->video_ops;
-		video_out->bpl_alignment = vfe_bpl_align(vfe);
-		video_out->line_based = 0;
 		if (i == VFE_LINE_PIX) {
-			video_out->bpl_alignment = 16;
+			video_out->bpl_alignment = vfe_bpl_align_pix(vfe);
 			video_out->line_based = 1;
+		} else {
+			video_out->bpl_alignment = vfe_bpl_align_rdi(vfe);
+			video_out->line_based = 0;
 		}
 
 		video_out->nformats = vfe->line[i].nformats;
diff --git a/drivers/media/platform/qcom/camss/camss-video.c b/drivers/media/platform/qcom/camss/camss-video.c
index 831486e14754a..0852eb6f13154 100644
--- a/drivers/media/platform/qcom/camss/camss-video.c
+++ b/drivers/media/platform/qcom/camss/camss-video.c
@@ -47,6 +47,9 @@ static int video_mbus_to_pix_mp(const struct v4l2_mbus_framefmt *mbus,
 	unsigned int i;
 	u32 bytesperline;
 
+	if (!alignment)
+		alignment = camss_format_get_bpl_alignment(f);
+
 	memset(pix, 0, sizeof(*pix));
 	v4l2_fill_pix_format_mplane(pix, mbus);
 	pix->pixelformat = f->pixelformat;
@@ -54,7 +57,7 @@ static int video_mbus_to_pix_mp(const struct v4l2_mbus_framefmt *mbus,
 	for (i = 0; i < pix->num_planes; i++) {
 		bytesperline = pix->width / f->hsub[i].numerator *
 			f->hsub[i].denominator * f->bpp[i] / 8;
-		bytesperline = ALIGN(bytesperline, alignment);
+		bytesperline = roundup(bytesperline, alignment);
 		pix->plane_fmt[i].bytesperline = bytesperline;
 		pix->plane_fmt[i].sizeimage = pix->height /
 				f->vsub[i].numerator * f->vsub[i].denominator *
@@ -215,6 +218,12 @@ static int video_check_format(struct camss_video *video)
 	if (ret < 0)
 		return ret;
 
+	dev_dbg(video->camss->dev,
+		"%s: format is (%ux%u %p4cc/%up field:%u), trying (%ux%u %p4cc/%up field:%u)",
+		 video->vdev.name, sd_pix->width, sd_pix->height, &sd_pix->pixelformat,
+		 sd_pix->num_planes, sd_pix->field, pix->width, pix->height, &pix->pixelformat,
+		 pix->num_planes, pix->field);
+
 	if (pix->pixelformat != sd_pix->pixelformat ||
 	    pix->height != sd_pix->height ||
 	    pix->width != sd_pix->width ||
@@ -453,6 +462,7 @@ static int video_g_fmt(struct file *file, void *fh, struct v4l2_format *f)
 
 static int __video_try_fmt(struct camss_video *video, struct v4l2_format *f)
 {
+	unsigned int alignment = video->bpl_alignment;
 	struct v4l2_pix_format_mplane *pix_mp;
 	const struct camss_format_info *fi;
 	struct v4l2_plane_pix_format *p;
@@ -485,6 +495,9 @@ static int __video_try_fmt(struct camss_video *video, struct v4l2_format *f)
 	width = pix_mp->width;
 	height = pix_mp->height;
 
+	if (!alignment)
+		alignment = camss_format_get_bpl_alignment(fi);
+
 	memset(pix_mp, 0, sizeof(*pix_mp));
 
 	pix_mp->pixelformat = fi->pixelformat;
@@ -494,7 +507,7 @@ static int __video_try_fmt(struct camss_video *video, struct v4l2_format *f)
 	for (i = 0; i < pix_mp->num_planes; i++) {
 		bpl = pix_mp->width / fi->hsub[i].numerator *
 			fi->hsub[i].denominator * fi->bpp[i] / 8;
-		bpl = ALIGN(bpl, video->bpl_alignment);
+		bpl = roundup(bpl, alignment);
 		pix_mp->plane_fmt[i].bytesperline = bpl;
 		pix_mp->plane_fmt[i].sizeimage = pix_mp->height /
 			fi->vsub[i].numerator * fi->vsub[i].denominator * bpl;
@@ -519,7 +532,7 @@ static int __video_try_fmt(struct camss_video *video, struct v4l2_format *f)
 			lines = p->sizeimage / p->bytesperline;
 
 			if (p->bytesperline < bytesperline[i])
-				p->bytesperline = ALIGN(bytesperline[i], 8);
+				p->bytesperline = roundup(bytesperline[i], alignment);
 
 			if (p->sizeimage < p->bytesperline * lines)
 				p->sizeimage = p->bytesperline * lines;
diff --git a/drivers/media/platform/qcom/camss/camss.c b/drivers/media/platform/qcom/camss/camss.c
index 9335636d7c4df..2123f6388e3d7 100644
--- a/drivers/media/platform/qcom/camss/camss.c
+++ b/drivers/media/platform/qcom/camss/camss.c
@@ -1703,6 +1703,253 @@ static const struct resources_icc icc_res_sm6150[] = {
 	},
 };
 
+static const struct camss_subdev_resources csiphy_res_sm6350[] = {
+	/* CSIPHY0 */
+	{
+		.regulators = {
+			{ .supply = "vdd-csiphy0-0p9", .init_load_uA = 80000 },
+			{ .supply = "vdd-csiphy0-1p25", .init_load_uA = 80000 },
+		},
+		.clock = { "csiphy0", "csiphy0_timer" },
+		.clock_rate = { { 300000000, 384000000, 400000000 },
+				{ 300000000 } },
+		.reg = { "csiphy0" },
+		.interrupt = { "csiphy0" },
+		.csiphy = {
+			.id = 0,
+			.hw_ops = &csiphy_ops_3ph_1_0,
+			.formats = &csiphy_formats_sdm845
+		}
+	},
+	/* CSIPHY1 */
+	{
+		.regulators = {
+			{ .supply = "vdd-csiphy1-0p9", .init_load_uA = 80000 },
+			{ .supply = "vdd-csiphy1-1p25", .init_load_uA = 80000 },
+		},
+		.clock = { "csiphy1", "csiphy1_timer" },
+		.clock_rate = { { 300000000, 384000000, 400000000 },
+				{ 300000000 } },
+		.reg = { "csiphy1" },
+		.interrupt = { "csiphy1" },
+		.csiphy = {
+			.id = 1,
+			.hw_ops = &csiphy_ops_3ph_1_0,
+			.formats = &csiphy_formats_sdm845
+		}
+	},
+	/* CSIPHY2 */
+	{
+		.regulators = {
+			{ .supply = "vdd-csiphy2-0p9", .init_load_uA = 80000 },
+			{ .supply = "vdd-csiphy2-1p25", .init_load_uA = 80000 },
+		},
+		.clock = { "csiphy2", "csiphy2_timer" },
+		.clock_rate = { { 300000000, 384000000, 400000000 },
+				{ 300000000 } },
+		.reg = { "csiphy2" },
+		.interrupt = { "csiphy2" },
+		.csiphy = {
+			.id = 2,
+			.hw_ops = &csiphy_ops_3ph_1_0,
+			.formats = &csiphy_formats_sdm845
+		}
+	},
+	/* CSIPHY3 */
+	{
+		.regulators = {
+			{ .supply = "vdd-csiphy3-0p9", .init_load_uA = 80000 },
+			{ .supply = "vdd-csiphy3-1p25", .init_load_uA = 80000 },
+		},
+		.clock = { "csiphy3", "csiphy3_timer" },
+		.clock_rate = { { 300000000, 384000000, 400000000 },
+				{ 300000000 } },
+		.reg = { "csiphy3" },
+		.interrupt = { "csiphy3" },
+		.csiphy = {
+			.id = 3,
+			.hw_ops = &csiphy_ops_3ph_1_0,
+			.formats = &csiphy_formats_sdm845
+		}
+	}
+};
+
+static const struct camss_subdev_resources csid_res_sm6350[] = {
+	/* CSID0 */
+	{
+		.regulators = {},
+		.clock = { "vfe0_csid", "vfe0_cphy_rx", "vfe0" },
+		.clock_rate = { { 300000000, 384000000, 400000000 },
+				{ 0 },
+				{ 320000000, 404000000, 480000000, 600000000 } },
+		.reg = { "csid0" },
+		.interrupt = { "csid0" },
+		.csid = {
+			.hw_ops = &csid_ops_gen2,
+			.parent_dev_ops = &vfe_parent_dev_ops,
+			.formats = &csid_formats_gen2
+		}
+	},
+	/* CSID1 */
+	{
+		.regulators = {},
+		.clock = { "vfe1_csid", "vfe1_cphy_rx", "vfe1" },
+		.clock_rate = { { 300000000, 384000000, 400000000 },
+				{ 0 },
+				{ 320000000, 404000000, 480000000, 600000000 } },
+		.reg = { "csid1" },
+		.interrupt = { "csid1" },
+		.csid = {
+			.hw_ops = &csid_ops_gen2,
+			.parent_dev_ops = &vfe_parent_dev_ops,
+			.formats = &csid_formats_gen2
+		}
+	},
+	/* CSID2 */
+	{
+		.regulators = {},
+		.clock = { "vfe2_csid", "vfe2_cphy_rx", "vfe2" },
+		.clock_rate = { { 300000000, 384000000, 400000000 },
+				{ 0 },
+				{ 320000000, 404000000, 480000000, 600000000 } },
+		.reg = { "csid2" },
+		.interrupt = { "csid2" },
+		.csid = {
+			.hw_ops = &csid_ops_gen2,
+			.parent_dev_ops = &vfe_parent_dev_ops,
+			.formats = &csid_formats_gen2
+		}
+	},
+	/* CSID3 (lite) */
+	{
+		.regulators = {},
+		.clock = { "vfe_lite_csid", "vfe_lite_cphy_rx", "vfe_lite" },
+		.clock_rate = { { 300000000, 384000000, 400000000 },
+				{ 0 },
+				{ 400000000, 480000000 } },
+		.reg = { "csid_lite" },
+		.interrupt = { "csid_lite" },
+		.csid = {
+			.is_lite = true,
+			.hw_ops = &csid_ops_gen2,
+			.parent_dev_ops = &vfe_parent_dev_ops,
+			.formats = &csid_formats_gen2
+		}
+	}
+};
+
+static const struct camss_subdev_resources vfe_res_sm6350[] = {
+	/* VFE0 */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "camnoc_axi", "vfe0",
+			   "vfe0_axi", "cam_axi", "soc_ahb" },
+		.clock_rate = { { 19200000 },
+				{ 0 },
+				{ 320000000, 404000000, 480000000, 600000000 },
+				{ 0 },
+				{ 0 },
+				{ 0 } },
+		.reg = { "vfe0" },
+		.interrupt = { "vfe0" },
+		.vfe = {
+			.line_num = 3,
+			.has_pd = true,
+			.pd_name = "ife0",
+			.hw_ops = &vfe_ops_170,
+			.formats_rdi = &vfe_formats_rdi_845,
+			.formats_pix = &vfe_formats_pix_845
+		}
+	},
+	/* VFE1 */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "camnoc_axi", "vfe1",
+			   "vfe1_axi", "cam_axi", "soc_ahb" },
+		.clock_rate = { { 19200000 },
+				{ 0 },
+				{ 320000000, 404000000, 480000000, 600000000 },
+				{ 0 },
+				{ 0 },
+				{ 0 } },
+		.reg = { "vfe1" },
+		.interrupt = { "vfe1" },
+		.vfe = {
+			.line_num = 3,
+			.has_pd = true,
+			.pd_name = "ife1",
+			.hw_ops = &vfe_ops_170,
+			.formats_rdi = &vfe_formats_rdi_845,
+			.formats_pix = &vfe_formats_pix_845
+		}
+	},
+	/* VFE2 */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "camnoc_axi", "vfe2",
+			   "vfe2_axi", "cam_axi", "soc_ahb" },
+		.clock_rate = { { 19200000 },
+				{ 0 },
+				{ 320000000, 404000000, 480000000, 600000000 },
+				{ 0 },
+				{ 0 },
+				{ 0 } },
+		.reg = { "vfe2" },
+		.interrupt = { "vfe2" },
+		.vfe = {
+			.line_num = 3,
+			.has_pd = true,
+			.pd_name = "ife2",
+			.hw_ops = &vfe_ops_170,
+			.formats_rdi = &vfe_formats_rdi_845,
+			.formats_pix = &vfe_formats_pix_845
+		}
+	},
+	/* VFE3 (lite) */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "camnoc_axi", "vfe_lite",
+			   "cam_axi", "soc_ahb" },
+		.clock_rate = { { 19200000 },
+				{ 0 },
+				{ 400000000, 480000000 },
+				{ 0 },
+				{ 0 } },
+		.reg = { "vfe_lite" },
+		.interrupt = { "vfe_lite" },
+		.vfe = {
+			.is_lite = true,
+			.line_num = 4,
+			.hw_ops = &vfe_ops_170,
+			.formats_rdi = &vfe_formats_rdi_845,
+			.formats_pix = &vfe_formats_pix_845
+		}
+	},
+};
+
+static const struct resources_icc icc_res_sm6350[] = {
+	{
+		.name = "ahb",
+		.icc_bw_tbl.avg = 0,
+		.icc_bw_tbl.peak = 300000,
+	},
+	{
+		.name = "hf_mnoc",
+		.icc_bw_tbl.avg = 2097152,
+		.icc_bw_tbl.peak = 2097152,
+	},
+	{
+		.name = "sf_mnoc",
+		.icc_bw_tbl.avg = 2097152,
+		.icc_bw_tbl.peak = 2097152,
+	},
+	{
+		.name = "sf_icp_mnoc",
+		.icc_bw_tbl.avg = 2097152,
+		.icc_bw_tbl.peak = 2097152,
+	},
+};
+
 static const struct camss_subdev_resources csiphy_res_8250[] = {
 	/* CSIPHY0 */
 	{
@@ -3559,6 +3806,54 @@ static const struct camss_subdev_resources csiphy_res_8775p[] = {
 	},
 };
 
+static const struct camss_subdev_resources tpg_res_8775p[] = {
+	/* TPG0 */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "csiphy_rx" },
+		.clock_rate = {
+			{ 0 },
+			{ 400000000 },
+		},
+		.reg = { "tpg0" },
+		.tpg = {
+			.lane_cnt = 4,
+			.formats = &tpg_formats_gen1,
+			.hw_ops = &tpg_ops_gen1
+		}
+	},
+	/* TPG1 */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "csiphy_rx" },
+		.clock_rate = {
+			{ 0 },
+			{ 400000000 },
+		},
+		.reg = { "tpg1" },
+		.tpg = {
+			.lane_cnt = 4,
+			.formats = &tpg_formats_gen1,
+			.hw_ops = &tpg_ops_gen1
+		}
+	},
+	/* TPG2 */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "csiphy_rx" },
+		.clock_rate = {
+			{ 0 },
+			{ 400000000 },
+		},
+		.reg = { "tpg2" },
+		.tpg = {
+			.lane_cnt = 4,
+			.formats = &tpg_formats_gen1,
+			.hw_ops = &tpg_ops_gen1
+		}
+	},
+};
+
 static const struct camss_subdev_resources csid_res_8775p[] = {
 	/* CSID0 */
 	{
@@ -3963,6 +4258,54 @@ static const struct camss_subdev_resources csiphy_res_x1e80100[] = {
 	},
 };
 
+static const struct camss_subdev_resources tpg_res_x1e80100[] = {
+	/* TPG0 */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "csid_csiphy_rx" },
+		.clock_rate = {
+			{ 0 },
+			{ 400000000 },
+		},
+		.reg = { "csitpg0" },
+		.tpg = {
+			.lane_cnt = 4,
+			.formats = &tpg_formats_gen1,
+			.hw_ops = &tpg_ops_gen1
+		}
+	},
+	/* TPG1 */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "csid_csiphy_rx" },
+		.clock_rate = {
+			{ 0 },
+			{ 400000000 },
+		},
+		.reg = { "csitpg1" },
+		.tpg = {
+			.lane_cnt = 4,
+			.formats = &tpg_formats_gen1,
+			.hw_ops = &tpg_ops_gen1
+		}
+	},
+	/* TPG2 */
+	{
+		.regulators = {},
+		.clock = { "cpas_ahb", "csid_csiphy_rx" },
+		.clock_rate = {
+			{ 0 },
+			{ 400000000 },
+		},
+		.reg = { "csitpg2" },
+		.tpg = {
+			.lane_cnt = 4,
+			.formats = &tpg_formats_gen1,
+			.hw_ops = &tpg_ops_gen1
+		}
+	},
+};
+
 static const struct camss_subdev_resources csid_res_x1e80100[] = {
 	/* CSID0 */
 	{
@@ -4076,7 +4419,7 @@ static const struct camss_subdev_resources vfe_res_x1e80100[] = {
 		.clock = {"camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb",
 			  "cpas_fast_ahb", "cpas_vfe0", "vfe0_fast_ahb",
 			  "vfe0" },
-		.clock_rate = { { 0 },
+		.clock_rate = { { 400000000 },
 				{ 0 },
 				{ 0 },
 				{ 0 },
@@ -4100,7 +4443,7 @@ static const struct camss_subdev_resources vfe_res_x1e80100[] = {
 		.clock = { "camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb",
 			   "cpas_fast_ahb", "cpas_vfe1", "vfe1_fast_ahb",
 			   "vfe1"  },
-		.clock_rate = { { 0 },
+		.clock_rate = { { 400000000 },
 				{ 0 },
 				{ 0 },
 				{ 0 },
@@ -4124,7 +4467,7 @@ static const struct camss_subdev_resources vfe_res_x1e80100[] = {
 		.clock = { "camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb",
 			   "vfe_lite_ahb", "cpas_vfe_lite", "vfe_lite",
 			   "vfe_lite_csid" },
-		.clock_rate = { { 0 },
+		.clock_rate = { { 400000000 },
 				{ 0 },
 				{ 0 },
 				{ 0 },
@@ -4147,7 +4490,7 @@ static const struct camss_subdev_resources vfe_res_x1e80100[] = {
 		.clock = { "camnoc_rt_axi", "camnoc_nrt_axi", "cpas_ahb",
 			   "vfe_lite_ahb", "cpas_vfe_lite", "vfe_lite",
 			   "vfe_lite_csid" },
-		.clock_rate = { { 0 },
+		.clock_rate = { { 400000000 },
 				{ 0 },
 				{ 0 },
 				{ 0 },
@@ -4501,6 +4844,19 @@ static int camss_init_subdevices(struct camss *camss)
 		}
 	}
 
+	if (camss->tpg) {
+		for (i = 0; i < camss->res->tpg_num; i++) {
+			ret = msm_tpg_subdev_init(camss, &camss->tpg[i],
+						  &res->tpg_res[i], i);
+			if (ret < 0) {
+				dev_err(camss->dev,
+					"Failed to init tpg%d sub-device: %d\n",
+					i, ret);
+				return ret;
+			}
+		}
+	}
+
 	/* note: SM8250 requires VFE to be initialized before CSID */
 	for (i = 0; i < camss->res->vfe_num; i++) {
 		ret = msm_vfe_subdev_init(camss, &camss->vfe[i],
@@ -4589,6 +4945,23 @@ static int camss_link_entities(struct camss *camss)
 		}
 	}
 
+	for (i = 0; i < camss->res->tpg_num; i++) {
+		for (j = 0; j < camss->res->csid_num; j++) {
+			ret = media_create_pad_link(&camss->tpg[i].subdev.entity,
+						    MSM_TPG_PAD_SRC,
+						    &camss->csid[j].subdev.entity,
+						    MSM_CSID_PAD_SINK,
+						    0);
+			if (ret < 0) {
+				camss_link_err(camss,
+					       camss->tpg[i].subdev.entity.name,
+					       camss->csid[j].subdev.entity.name,
+					       ret);
+				return ret;
+			}
+		}
+	}
+
 	if (camss->ispif) {
 		for (i = 0; i < camss->res->csid_num; i++) {
 			for (j = 0; j < camss->ispif->line_num; j++) {
@@ -4693,6 +5066,19 @@ static int camss_register_entities(struct camss *camss)
 		}
 	}
 
+	if (camss->tpg) {
+		for (i = 0; i < camss->res->tpg_num; i++) {
+			ret = msm_tpg_register_entity(&camss->tpg[i],
+						      &camss->v4l2_dev);
+			if (ret < 0) {
+				dev_err(camss->dev,
+					"Failed to register tpg%d entity: %d\n",
+					i, ret);
+				goto err_reg_tpg;
+			}
+		}
+	}
+
 	for (i = 0; i < camss->res->csid_num; i++) {
 		ret = msm_csid_register_entity(&camss->csid[i],
 					       &camss->v4l2_dev);
@@ -4736,6 +5122,13 @@ static int camss_register_entities(struct camss *camss)
 	for (i--; i >= 0; i--)
 		msm_csid_unregister_entity(&camss->csid[i]);
 
+	i = camss->res->tpg_num;
+err_reg_tpg:
+	if (camss->tpg) {
+		for (i--; i >= 0; i--)
+			msm_tpg_unregister_entity(&camss->tpg[i]);
+	}
+
 	i = camss->res->csiphy_num;
 err_reg_csiphy:
 	for (i--; i >= 0; i--)
@@ -4757,6 +5150,11 @@ static void camss_unregister_entities(struct camss *camss)
 	for (i = 0; i < camss->res->csiphy_num; i++)
 		msm_csiphy_unregister_entity(&camss->csiphy[i]);
 
+	if (camss->tpg) {
+		for (i = 0; i < camss->res->tpg_num; i++)
+			msm_tpg_unregister_entity(&camss->tpg[i]);
+	}
+
 	for (i = 0; i < camss->res->csid_num; i++)
 		msm_csid_unregister_entity(&camss->csid[i]);
 
@@ -4975,6 +5373,13 @@ static int camss_probe(struct platform_device *pdev)
 	if (!camss->csiphy)
 		return -ENOMEM;
 
+	if (camss->res->tpg_num > 0) {
+		camss->tpg = devm_kcalloc(dev, camss->res->tpg_num,
+					  sizeof(*camss->tpg), GFP_KERNEL);
+		if (!camss->tpg)
+			return -ENOMEM;
+	}
+
 	camss->csid = devm_kcalloc(dev, camss->res->csid_num, sizeof(*camss->csid),
 				   GFP_KERNEL);
 	if (!camss->csid)
@@ -5164,11 +5569,13 @@ static const struct camss_resources qcs8300_resources = {
 	.version = CAMSS_8300,
 	.pd_name = "top",
 	.csiphy_res = csiphy_res_8300,
+	.tpg_res = tpg_res_8775p,
 	.csid_res = csid_res_8775p,
 	.csid_wrapper_res = &csid_wrapper_res_sm8550,
 	.vfe_res = vfe_res_8775p,
 	.icc_res = icc_res_qcs8300,
 	.csiphy_num = ARRAY_SIZE(csiphy_res_8300),
+	.tpg_num = ARRAY_SIZE(tpg_res_8775p),
 	.csid_num = ARRAY_SIZE(csid_res_8775p),
 	.vfe_num = ARRAY_SIZE(vfe_res_8775p),
 	.icc_path_num = ARRAY_SIZE(icc_res_qcs8300),
@@ -5178,11 +5585,13 @@ static const struct camss_resources sa8775p_resources = {
 	.version = CAMSS_8775P,
 	.pd_name = "top",
 	.csiphy_res = csiphy_res_8775p,
+	.tpg_res = tpg_res_8775p,
 	.csid_res = csid_res_8775p,
 	.csid_wrapper_res = &csid_wrapper_res_sm8550,
 	.vfe_res = vfe_res_8775p,
 	.icc_res = icc_res_sa8775p,
 	.csiphy_num = ARRAY_SIZE(csiphy_res_8775p),
+	.tpg_num = ARRAY_SIZE(tpg_res_8775p),
 	.csid_num = ARRAY_SIZE(csid_res_8775p),
 	.vfe_num = ARRAY_SIZE(vfe_res_8775p),
 	.icc_path_num = ARRAY_SIZE(icc_res_sa8775p),
@@ -5233,6 +5642,19 @@ static const struct camss_resources sm6150_resources = {
 	.vfe_num = ARRAY_SIZE(vfe_res_sm6150),
 };
 
+static const struct camss_resources sm6350_resources = {
+	.version = CAMSS_6350,
+	.pd_name = "top",
+	.csiphy_res = csiphy_res_sm6350,
+	.csid_res = csid_res_sm6350,
+	.vfe_res = vfe_res_sm6350,
+	.icc_res = icc_res_sm6350,
+	.icc_path_num = ARRAY_SIZE(icc_res_sm6350),
+	.csiphy_num = ARRAY_SIZE(csiphy_res_sm6350),
+	.csid_num = ARRAY_SIZE(csid_res_sm6350),
+	.vfe_num = ARRAY_SIZE(vfe_res_sm6350),
+};
+
 static const struct camss_resources sm8250_resources = {
 	.version = CAMSS_8250,
 	.pd_name = "top",
@@ -5305,12 +5727,14 @@ static const struct camss_resources x1e80100_resources = {
 	.version = CAMSS_X1E80100,
 	.pd_name = "top",
 	.csiphy_res = csiphy_res_x1e80100,
+	.tpg_res = tpg_res_x1e80100,
 	.csid_res = csid_res_x1e80100,
 	.vfe_res = vfe_res_x1e80100,
 	.csid_wrapper_res = &csid_wrapper_res_x1e80100,
 	.icc_res = icc_res_x1e80100,
 	.icc_path_num = ARRAY_SIZE(icc_res_x1e80100),
 	.csiphy_num = ARRAY_SIZE(csiphy_res_x1e80100),
+	.tpg_num = ARRAY_SIZE(tpg_res_x1e80100),
 	.csid_num = ARRAY_SIZE(csid_res_x1e80100),
 	.vfe_num = ARRAY_SIZE(vfe_res_x1e80100),
 };
@@ -5329,6 +5753,7 @@ static const struct of_device_id camss_dt_match[] = {
 	{ .compatible = "qcom,sdm670-camss", .data = &sdm670_resources },
 	{ .compatible = "qcom,sdm845-camss", .data = &sdm845_resources },
 	{ .compatible = "qcom,sm6150-camss", .data = &sm6150_resources },
+	{ .compatible = "qcom,sm6350-camss", .data = &sm6350_resources },
 	{ .compatible = "qcom,sm8250-camss", .data = &sm8250_resources },
 	{ .compatible = "qcom,sm8550-camss", .data = &sm8550_resources },
 	{ .compatible = "qcom,sm8650-camss", .data = &sm8650_resources },
diff --git a/drivers/media/platform/qcom/camss/camss.h b/drivers/media/platform/qcom/camss/camss.h
index 6d048414c919e..93d691c8ac63b 100644
--- a/drivers/media/platform/qcom/camss/camss.h
+++ b/drivers/media/platform/qcom/camss/camss.h
@@ -21,6 +21,7 @@
 #include "camss-csid.h"
 #include "camss-csiphy.h"
 #include "camss-ispif.h"
+#include "camss-tpg.h"
 #include "camss-vfe.h"
 #include "camss-format.h"
 
@@ -52,6 +53,7 @@ struct camss_subdev_resources {
 	char *interrupt[CAMSS_RES_MAX];
 	union {
 		struct csiphy_subdev_resources csiphy;
+		struct tpg_subdev_resources tpg;
 		struct csid_subdev_resources csid;
 		struct vfe_subdev_resources vfe;
 	};
@@ -81,6 +83,7 @@ enum camss_version {
 	CAMSS_660,
 	CAMSS_2290,
 	CAMSS_6150,
+	CAMSS_6350,
 	CAMSS_7280,
 	CAMSS_8x16,
 	CAMSS_8x39,
@@ -105,6 +108,7 @@ struct camss_resources {
 	enum camss_version version;
 	const char *pd_name;
 	const struct camss_subdev_resources *csiphy_res;
+	const struct camss_subdev_resources *tpg_res;
 	const struct camss_subdev_resources *csid_res;
 	const struct camss_subdev_resources *ispif_res;
 	const struct camss_subdev_resources *vfe_res;
@@ -112,6 +116,7 @@ struct camss_resources {
 	const struct resources_icc *icc_res;
 	const unsigned int icc_path_num;
 	const unsigned int csiphy_num;
+	const unsigned int tpg_num;
 	const unsigned int csid_num;
 	const unsigned int vfe_num;
 };
@@ -122,6 +127,7 @@ struct camss {
 	struct media_device media_dev;
 	struct device *dev;
 	struct csiphy_device *csiphy;
+	struct tpg_device *tpg;
 	struct csid_device *csid;
 	struct ispif_device *ispif;
 	struct vfe_device *vfe;
diff --git a/drivers/media/platform/qcom/iris/Kconfig b/drivers/media/platform/qcom/iris/Kconfig
index 5498f48362d15..af78a1775937b 100644
--- a/drivers/media/platform/qcom/iris/Kconfig
+++ b/drivers/media/platform/qcom/iris/Kconfig
@@ -5,6 +5,7 @@ config VIDEO_QCOM_IRIS
         select V4L2_MEM2MEM_DEV
         select QCOM_MDT_LOADER
         select QCOM_SCM
+        select QCOM_UBWC_CONFIG
         select VIDEOBUF2_DMA_CONTIG
         help
           This is a V4L2 driver for Qualcomm iris video accelerator
diff --git a/drivers/media/platform/qcom/iris/Makefile b/drivers/media/platform/qcom/iris/Makefile
index 2abbd3aeb4af0..48e415cbc4390 100644
--- a/drivers/media/platform/qcom/iris/Makefile
+++ b/drivers/media/platform/qcom/iris/Makefile
@@ -4,13 +4,16 @@ qcom-iris-objs += iris_buffer.o \
              iris_ctrls.o \
              iris_firmware.o \
              iris_hfi_common.o \
+             iris_hfi_gen1.o \
              iris_hfi_gen1_command.o \
              iris_hfi_gen1_response.o \
+             iris_hfi_gen2.o \
              iris_hfi_gen2_command.o \
              iris_hfi_gen2_packet.o \
              iris_hfi_gen2_response.o \
              iris_hfi_queue.o \
-             iris_platform_gen2.o \
+             iris_platform_vpu2.o \
+             iris_platform_vpu3x.o \
              iris_power.o \
              iris_probe.o \
              iris_resources.o \
@@ -26,8 +29,4 @@ qcom-iris-objs += iris_buffer.o \
              iris_vpu_buffer.o \
              iris_vpu_common.o \
 
-ifeq ($(CONFIG_VIDEO_QCOM_VENUS),)
-qcom-iris-objs += iris_platform_gen1.o
-endif
-
 obj-$(CONFIG_VIDEO_QCOM_IRIS) += qcom-iris.o
diff --git a/drivers/media/platform/qcom/iris/iris_buffer.c b/drivers/media/platform/qcom/iris/iris_buffer.c
index 1d53c7414b754..60e3d14450c8b 100644
--- a/drivers/media/platform/qcom/iris/iris_buffer.c
+++ b/drivers/media/platform/qcom/iris/iris_buffer.c
@@ -295,37 +295,37 @@ static void iris_fill_internal_buf_info(struct iris_inst *inst,
 {
 	struct iris_buffers *buffers = &inst->buffers[buffer_type];
 
-	buffers->size = inst->core->iris_platform_data->get_vpu_buffer_size(inst, buffer_type);
+	buffers->size = inst->core->iris_firmware_desc->get_vpu_buffer_size(inst, buffer_type);
 	buffers->min_count = iris_vpu_buf_count(inst, buffer_type);
 }
 
 void iris_get_internal_buffers(struct iris_inst *inst, u32 plane)
 {
-	const struct iris_platform_data *platform_data = inst->core->iris_platform_data;
+	const struct iris_firmware_data *firmware_data = inst->core->iris_firmware_data;
 	const u32 *internal_buf_type;
 	u32 internal_buffer_count, i;
 
 	if (inst->domain == DECODER) {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
-			internal_buf_type = platform_data->dec_ip_int_buf_tbl;
-			internal_buffer_count = platform_data->dec_ip_int_buf_tbl_size;
+			internal_buf_type = firmware_data->dec_ip_int_buf_tbl;
+			internal_buffer_count = firmware_data->dec_ip_int_buf_tbl_size;
 			for (i = 0; i < internal_buffer_count; i++)
 				iris_fill_internal_buf_info(inst, internal_buf_type[i]);
 		} else {
-			internal_buf_type = platform_data->dec_op_int_buf_tbl;
-			internal_buffer_count = platform_data->dec_op_int_buf_tbl_size;
+			internal_buf_type = firmware_data->dec_op_int_buf_tbl;
+			internal_buffer_count = firmware_data->dec_op_int_buf_tbl_size;
 			for (i = 0; i < internal_buffer_count; i++)
 				iris_fill_internal_buf_info(inst, internal_buf_type[i]);
 		}
 	} else {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
-			internal_buf_type = platform_data->enc_ip_int_buf_tbl;
-			internal_buffer_count = platform_data->enc_ip_int_buf_tbl_size;
+			internal_buf_type = firmware_data->enc_ip_int_buf_tbl;
+			internal_buffer_count = firmware_data->enc_ip_int_buf_tbl_size;
 			for (i = 0; i < internal_buffer_count; i++)
 				iris_fill_internal_buf_info(inst, internal_buf_type[i]);
 		} else {
-			internal_buf_type = platform_data->enc_op_int_buf_tbl;
-			internal_buffer_count = platform_data->enc_op_int_buf_tbl_size;
+			internal_buf_type = firmware_data->enc_op_int_buf_tbl;
+			internal_buffer_count = firmware_data->enc_op_int_buf_tbl_size;
 			for (i = 0; i < internal_buffer_count; i++)
 				iris_fill_internal_buf_info(inst, internal_buf_type[i]);
 		}
@@ -366,7 +366,7 @@ static int iris_create_internal_buffer(struct iris_inst *inst,
 
 int iris_create_internal_buffers(struct iris_inst *inst, u32 plane)
 {
-	const struct iris_platform_data *platform_data = inst->core->iris_platform_data;
+	const struct iris_firmware_data *firmware_data = inst->core->iris_firmware_data;
 	u32 internal_buffer_count, i, j;
 	struct iris_buffers *buffers;
 	const u32 *internal_buf_type;
@@ -374,19 +374,19 @@ int iris_create_internal_buffers(struct iris_inst *inst, u32 plane)
 
 	if (inst->domain == DECODER) {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
-			internal_buf_type = platform_data->dec_ip_int_buf_tbl;
-			internal_buffer_count = platform_data->dec_ip_int_buf_tbl_size;
+			internal_buf_type = firmware_data->dec_ip_int_buf_tbl;
+			internal_buffer_count = firmware_data->dec_ip_int_buf_tbl_size;
 		} else {
-			internal_buf_type = platform_data->dec_op_int_buf_tbl;
-			internal_buffer_count = platform_data->dec_op_int_buf_tbl_size;
+			internal_buf_type = firmware_data->dec_op_int_buf_tbl;
+			internal_buffer_count = firmware_data->dec_op_int_buf_tbl_size;
 		}
 	} else {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
-			internal_buf_type = platform_data->enc_ip_int_buf_tbl;
-			internal_buffer_count = platform_data->enc_ip_int_buf_tbl_size;
+			internal_buf_type = firmware_data->enc_ip_int_buf_tbl;
+			internal_buffer_count = firmware_data->enc_ip_int_buf_tbl_size;
 		} else {
-			internal_buf_type = platform_data->enc_op_int_buf_tbl;
-			internal_buffer_count = platform_data->enc_op_int_buf_tbl_size;
+			internal_buf_type = firmware_data->enc_op_int_buf_tbl;
+			internal_buffer_count = firmware_data->enc_op_int_buf_tbl_size;
 		}
 	}
 
@@ -404,7 +404,7 @@ int iris_create_internal_buffers(struct iris_inst *inst, u32 plane)
 
 int iris_queue_buffer(struct iris_inst *inst, struct iris_buffer *buf)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	int ret;
 
 	ret = hfi_ops->session_queue_buf(inst, buf);
@@ -442,7 +442,7 @@ int iris_queue_internal_deferred_buffers(struct iris_inst *inst, enum iris_buffe
 
 int iris_queue_internal_buffers(struct iris_inst *inst, u32 plane)
 {
-	const struct iris_platform_data *platform_data = inst->core->iris_platform_data;
+	const struct iris_firmware_data *firmware_data = inst->core->iris_firmware_data;
 	struct iris_buffer *buffer, *next;
 	struct iris_buffers *buffers;
 	const u32 *internal_buf_type;
@@ -451,19 +451,19 @@ int iris_queue_internal_buffers(struct iris_inst *inst, u32 plane)
 
 	if (inst->domain == DECODER) {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
-			internal_buf_type = platform_data->dec_ip_int_buf_tbl;
-			internal_buffer_count = platform_data->dec_ip_int_buf_tbl_size;
+			internal_buf_type = firmware_data->dec_ip_int_buf_tbl;
+			internal_buffer_count = firmware_data->dec_ip_int_buf_tbl_size;
 		} else {
-			internal_buf_type = platform_data->dec_op_int_buf_tbl;
-			internal_buffer_count = platform_data->dec_op_int_buf_tbl_size;
+			internal_buf_type = firmware_data->dec_op_int_buf_tbl;
+			internal_buffer_count = firmware_data->dec_op_int_buf_tbl_size;
 		}
 	} else {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
-			internal_buf_type = platform_data->enc_ip_int_buf_tbl;
-			internal_buffer_count = platform_data->enc_ip_int_buf_tbl_size;
+			internal_buf_type = firmware_data->enc_ip_int_buf_tbl;
+			internal_buffer_count = firmware_data->enc_ip_int_buf_tbl_size;
 		} else {
-			internal_buf_type = platform_data->enc_op_int_buf_tbl;
-			internal_buffer_count = platform_data->enc_op_int_buf_tbl_size;
+			internal_buf_type = firmware_data->enc_op_int_buf_tbl;
+			internal_buffer_count = firmware_data->enc_op_int_buf_tbl_size;
 		}
 	}
 
@@ -501,7 +501,7 @@ int iris_destroy_internal_buffer(struct iris_inst *inst, struct iris_buffer *buf
 
 static int iris_destroy_internal_buffers(struct iris_inst *inst, u32 plane, bool force)
 {
-	const struct iris_platform_data *platform_data = inst->core->iris_platform_data;
+	const struct iris_firmware_data *firmware_data = inst->core->iris_firmware_data;
 	struct iris_buffer *buf, *next;
 	struct iris_buffers *buffers;
 	const u32 *internal_buf_type;
@@ -510,19 +510,19 @@ static int iris_destroy_internal_buffers(struct iris_inst *inst, u32 plane, bool
 
 	if (inst->domain == DECODER) {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
-			internal_buf_type = platform_data->dec_ip_int_buf_tbl;
-			len = platform_data->dec_ip_int_buf_tbl_size;
+			internal_buf_type = firmware_data->dec_ip_int_buf_tbl;
+			len = firmware_data->dec_ip_int_buf_tbl_size;
 		} else {
-			internal_buf_type = platform_data->dec_op_int_buf_tbl;
-			len = platform_data->dec_op_int_buf_tbl_size;
+			internal_buf_type = firmware_data->dec_op_int_buf_tbl;
+			len = firmware_data->dec_op_int_buf_tbl_size;
 		}
 	} else {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
-			internal_buf_type = platform_data->enc_ip_int_buf_tbl;
-			len = platform_data->enc_ip_int_buf_tbl_size;
+			internal_buf_type = firmware_data->enc_ip_int_buf_tbl;
+			len = firmware_data->enc_ip_int_buf_tbl_size;
 		} else {
-			internal_buf_type = platform_data->enc_op_int_buf_tbl;
-			len = platform_data->enc_op_int_buf_tbl_size;
+			internal_buf_type = firmware_data->enc_op_int_buf_tbl;
+			len = firmware_data->enc_op_int_buf_tbl_size;
 		}
 	}
 
@@ -572,7 +572,7 @@ int iris_destroy_dequeued_internal_buffers(struct iris_inst *inst, u32 plane)
 static int iris_release_internal_buffers(struct iris_inst *inst,
 					 enum iris_buffer_type buffer_type)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	struct iris_buffers *buffers = &inst->buffers[buffer_type];
 	struct iris_buffer *buffer, *next;
 	int ret;
@@ -595,17 +595,17 @@ static int iris_release_internal_buffers(struct iris_inst *inst,
 
 static int iris_release_input_internal_buffers(struct iris_inst *inst)
 {
-	const struct iris_platform_data *platform_data = inst->core->iris_platform_data;
+	const struct iris_firmware_data *firmware_data = inst->core->iris_firmware_data;
 	const u32 *internal_buf_type;
 	u32 internal_buffer_count, i;
 	int ret;
 
 	if (inst->domain == DECODER) {
-		internal_buf_type = platform_data->dec_ip_int_buf_tbl;
-		internal_buffer_count = platform_data->dec_ip_int_buf_tbl_size;
+		internal_buf_type = firmware_data->dec_ip_int_buf_tbl;
+		internal_buffer_count = firmware_data->dec_ip_int_buf_tbl_size;
 	} else {
-		internal_buf_type = platform_data->enc_ip_int_buf_tbl;
-		internal_buffer_count = platform_data->enc_ip_int_buf_tbl_size;
+		internal_buf_type = firmware_data->enc_ip_int_buf_tbl;
+		internal_buffer_count = firmware_data->enc_ip_int_buf_tbl_size;
 	}
 
 	for (i = 0; i < internal_buffer_count; i++) {
diff --git a/drivers/media/platform/qcom/iris/iris_common.c b/drivers/media/platform/qcom/iris/iris_common.c
index 7f1c7fe144f70..25836561bcf3e 100644
--- a/drivers/media/platform/qcom/iris/iris_common.c
+++ b/drivers/media/platform/qcom/iris/iris_common.c
@@ -48,7 +48,7 @@ void iris_set_ts_metadata(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
 
 int iris_process_streamon_input(struct iris_inst *inst)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	enum iris_inst_sub_state set_sub_state = 0;
 	int ret;
 
@@ -90,7 +90,7 @@ int iris_process_streamon_input(struct iris_inst *inst)
 
 int iris_process_streamon_output(struct iris_inst *inst)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	enum iris_inst_sub_state clear_sub_state = 0;
 	bool drain_active, drc_active, first_ipsc;
 	int ret = 0;
@@ -189,7 +189,7 @@ static void iris_flush_deferred_buffers(struct iris_inst *inst,
 
 static void iris_kill_session(struct iris_inst *inst)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 
 	if (!inst->session_id)
 		return;
@@ -200,7 +200,7 @@ static void iris_kill_session(struct iris_inst *inst)
 
 int iris_session_streamoff(struct iris_inst *inst, u32 plane)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	enum iris_buffer_type buffer_type;
 	int ret;
 
diff --git a/drivers/media/platform/qcom/iris/iris_core.c b/drivers/media/platform/qcom/iris/iris_core.c
index dbaac01eb15a0..1f326f696d080 100644
--- a/drivers/media/platform/qcom/iris/iris_core.c
+++ b/drivers/media/platform/qcom/iris/iris_core.c
@@ -28,14 +28,13 @@ void iris_core_deinit(struct iris_core *core)
 
 static int iris_wait_for_system_response(struct iris_core *core)
 {
-	u32 hw_response_timeout_val = core->iris_platform_data->hw_response_timeout;
 	int ret;
 
 	if (core->state == IRIS_CORE_ERROR)
 		return -EIO;
 
 	ret = wait_for_completion_timeout(&core->core_init_done,
-					  msecs_to_jiffies(hw_response_timeout_val));
+					  msecs_to_jiffies(HW_RESPONSE_TIMEOUT_VALUE));
 	if (!ret) {
 		core->state = IRIS_CORE_ERROR;
 		return -ETIMEDOUT;
diff --git a/drivers/media/platform/qcom/iris/iris_core.h b/drivers/media/platform/qcom/iris/iris_core.h
index fb194c967ad4f..24da60448cf24 100644
--- a/drivers/media/platform/qcom/iris/iris_core.h
+++ b/drivers/media/platform/qcom/iris/iris_core.h
@@ -30,6 +30,8 @@ enum domain_type {
 	DECODER	= BIT(1),
 };
 
+struct qcom_ubwc_cfg_data;
+
 /**
  * struct iris_core - holds core parameters valid for all instances
  *
@@ -52,6 +54,9 @@ enum domain_type {
  * @resets: table of iris reset clocks
  * @controller_resets: table of controller reset clocks
  * @iris_platform_data: a structure for platform data
+ * @iris_firmware_data: a pointer to the firmware (or HFI) specific data
+ * @iris_firmware_desc: a pointer to the firmware-specific descriptive data
+ * @ubwc_cfg: UBWC configuration for the platform
  * @state: current state of core
  * @iface_q_table_daddr: device address for interface queue table memory
  * @sfr_daddr: device address for SFR (Sub System Failure Reason) register memory
@@ -65,8 +70,7 @@ enum domain_type {
  * @header_id: id of packet header
  * @packet_id: id of packet
  * @power: a structure for clock and bw information
- * @hfi_ops: iris hfi command ops
- * @hfi_response_ops: iris hfi response ops
+ * @hfi_sys_ops: iris HFI system ops
  * @core_init_done: structure of signal completion for system response
  * @intr_status: interrupt status
  * @sys_error_handler: a delayed work for handling system fatal error
@@ -95,6 +99,9 @@ struct iris_core {
 	struct reset_control_bulk_data		*resets;
 	struct reset_control_bulk_data		*controller_resets;
 	const struct iris_platform_data		*iris_platform_data;
+	const struct iris_firmware_data		*iris_firmware_data;
+	const struct iris_firmware_desc		*iris_firmware_desc;
+	const struct qcom_ubwc_cfg_data		*ubwc_cfg;
 	enum iris_core_state			state;
 	dma_addr_t				iface_q_table_daddr;
 	dma_addr_t				sfr_daddr;
@@ -108,8 +115,7 @@ struct iris_core {
 	u32					header_id;
 	u32					packet_id;
 	struct iris_core_power			power;
-	const struct iris_hfi_command_ops	*hfi_ops;
-	const struct iris_hfi_response_ops	*hfi_response_ops;
+	const struct iris_hfi_sys_ops		*hfi_sys_ops;
 	struct completion			core_init_done;
 	u32					intr_status;
 	struct delayed_work			sys_error_handler;
diff --git a/drivers/media/platform/qcom/iris/iris_ctrls.c b/drivers/media/platform/qcom/iris/iris_ctrls.c
index 3cec957580f5e..ef7adac3764d7 100644
--- a/drivers/media/platform/qcom/iris/iris_ctrls.c
+++ b/drivers/media/platform/qcom/iris/iris_ctrls.c
@@ -332,8 +332,8 @@ void iris_session_init_caps(struct iris_core *core)
 	const struct platform_inst_fw_cap *caps;
 	u32 i, num_cap, cap_id;
 
-	caps = core->iris_platform_data->inst_fw_caps_dec;
-	num_cap = core->iris_platform_data->inst_fw_caps_dec_size;
+	caps = core->iris_firmware_data->inst_fw_caps_dec;
+	num_cap = core->iris_firmware_data->inst_fw_caps_dec_size;
 
 	for (i = 0; i < num_cap; i++) {
 		cap_id = caps[i].cap_id;
@@ -360,8 +360,8 @@ void iris_session_init_caps(struct iris_core *core)
 		}
 	}
 
-	caps = core->iris_platform_data->inst_fw_caps_enc;
-	num_cap = core->iris_platform_data->inst_fw_caps_enc_size;
+	caps = core->iris_firmware_data->inst_fw_caps_enc;
+	num_cap = core->iris_firmware_data->inst_fw_caps_enc_size;
 
 	for (i = 0; i < num_cap; i++) {
 		cap_id = caps[i].cap_id;
@@ -399,7 +399,7 @@ static u32 iris_get_port_info(struct iris_inst *inst,
 
 int iris_set_u32_enum(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 hfi_value = inst->fw_caps[cap_id].value;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 
@@ -412,7 +412,7 @@ int iris_set_u32_enum(struct iris_inst *inst, enum platform_inst_fw_cap_type cap
 
 int iris_set_u32(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 hfi_value = inst->fw_caps[cap_id].value;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 
@@ -425,7 +425,7 @@ int iris_set_u32(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 
 int iris_set_stage(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	struct v4l2_format *inp_f = inst->fmt_src;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 	u32 height = inp_f->fmt.pix_mp.height;
@@ -446,7 +446,7 @@ int iris_set_stage(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id
 
 int iris_set_pipe(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 work_route = inst->fw_caps[PIPE].value;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 
@@ -459,7 +459,7 @@ int iris_set_pipe(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 
 int iris_set_profile(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 hfi_id, hfi_value;
 
 	if (inst->codec == V4L2_PIX_FMT_H264) {
@@ -479,7 +479,7 @@ int iris_set_profile(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_
 
 int iris_set_level(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 hfi_id, hfi_value;
 
 	if (inst->codec == V4L2_PIX_FMT_H264) {
@@ -499,7 +499,7 @@ int iris_set_level(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id
 
 int iris_set_profile_level_gen1(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 	struct hfi_profile_level pl;
 
@@ -520,7 +520,7 @@ int iris_set_profile_level_gen1(struct iris_inst *inst, enum platform_inst_fw_ca
 
 int iris_set_header_mode_gen1(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 header_mode = inst->fw_caps[cap_id].value;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 	u32 hfi_val;
@@ -539,7 +539,7 @@ int iris_set_header_mode_gen1(struct iris_inst *inst, enum platform_inst_fw_cap_
 
 int iris_set_header_mode_gen2(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 prepend_sps_pps = inst->fw_caps[PREPEND_SPSPPS_TO_IDR].value;
 	u32 header_mode = inst->fw_caps[cap_id].value;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
@@ -561,7 +561,7 @@ int iris_set_header_mode_gen2(struct iris_inst *inst, enum platform_inst_fw_cap_
 
 int iris_set_bitrate(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 entropy_mode = inst->fw_caps[ENTROPY_MODE].value;
 	u32 bitrate = inst->fw_caps[cap_id].value;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
@@ -586,7 +586,7 @@ int iris_set_bitrate(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_
 
 int iris_set_peak_bitrate(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 rc_mode = inst->fw_caps[BITRATE_MODE].value;
 	u32 peak_bitrate = inst->fw_caps[cap_id].value;
 	u32 bitrate = inst->fw_caps[BITRATE].value;
@@ -613,7 +613,7 @@ int iris_set_peak_bitrate(struct iris_inst *inst, enum platform_inst_fw_cap_type
 
 int iris_set_bitrate_mode_gen1(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 bitrate_mode = inst->fw_caps[BITRATE_MODE].value;
 	u32 frame_rc = inst->fw_caps[FRAME_RC_ENABLE].value;
 	u32 frame_skip = inst->fw_caps[FRAME_SKIP_MODE].value;
@@ -640,7 +640,7 @@ int iris_set_bitrate_mode_gen1(struct iris_inst *inst, enum platform_inst_fw_cap
 
 int iris_set_bitrate_mode_gen2(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 bitrate_mode = inst->fw_caps[BITRATE_MODE].value;
 	u32 frame_rc = inst->fw_caps[FRAME_RC_ENABLE].value;
 	u32 frame_skip = inst->fw_caps[FRAME_SKIP_MODE].value;
@@ -667,7 +667,7 @@ int iris_set_bitrate_mode_gen2(struct iris_inst *inst, enum platform_inst_fw_cap
 
 int iris_set_entropy_mode_gen1(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 entropy_mode = inst->fw_caps[cap_id].value;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 	u32 hfi_val;
@@ -687,7 +687,7 @@ int iris_set_entropy_mode_gen1(struct iris_inst *inst, enum platform_inst_fw_cap
 
 int iris_set_entropy_mode_gen2(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 entropy_mode = inst->fw_caps[cap_id].value;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 	u32 profile;
@@ -712,7 +712,7 @@ int iris_set_entropy_mode_gen2(struct iris_inst *inst, enum platform_inst_fw_cap
 
 int iris_set_min_qp(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 i_qp_enable = 0, p_qp_enable = 0, b_qp_enable = 0;
 	u32 i_frame_qp = 0, p_frame_qp = 0, b_frame_qp = 0;
 	u32 min_qp_enable = 0, client_qp_enable = 0;
@@ -776,7 +776,7 @@ int iris_set_min_qp(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_i
 
 int iris_set_max_qp(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 i_qp_enable = 0, p_qp_enable = 0, b_qp_enable = 0;
 	u32 max_qp_enable = 0, client_qp_enable;
 	u32 i_frame_qp, p_frame_qp, b_frame_qp;
@@ -841,7 +841,7 @@ int iris_set_max_qp(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_i
 
 int iris_set_frame_qp(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 i_qp_enable = 0, p_qp_enable = 0, b_qp_enable = 0, client_qp_enable;
 	u32 i_frame_qp, p_frame_qp, b_frame_qp;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
@@ -902,7 +902,7 @@ int iris_set_frame_qp(struct iris_inst *inst, enum platform_inst_fw_cap_type cap
 
 int iris_set_qp_range(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	struct hfi_quantization_range_v2 range;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 
@@ -923,7 +923,7 @@ int iris_set_qp_range(struct iris_inst *inst, enum platform_inst_fw_cap_type cap
 
 int iris_set_rotation(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 	u32 hfi_val;
 
@@ -953,7 +953,7 @@ int iris_set_rotation(struct iris_inst *inst, enum platform_inst_fw_cap_type cap
 
 int iris_set_flip(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	u32 hfi_id = inst->fw_caps[cap_id].hfi_id;
 	u32 hfi_val = HFI_DISABLE_FLIP;
 
@@ -972,7 +972,7 @@ int iris_set_flip(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 
 int iris_set_ir_period(struct iris_inst *inst, enum platform_inst_fw_cap_type cap_id)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	struct vb2_queue *q = v4l2_m2m_get_dst_vq(inst->m2m_ctx);
 	u32 ir_period = inst->fw_caps[cap_id].value;
 	u32 ir_type = 0;
@@ -998,7 +998,7 @@ int iris_set_ir_period(struct iris_inst *inst, enum platform_inst_fw_cap_type ca
 
 int iris_set_properties(struct iris_inst *inst, u32 plane)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	struct platform_inst_fw_cap *cap;
 	int ret;
 	u32 i;
diff --git a/drivers/media/platform/qcom/iris/iris_firmware.c b/drivers/media/platform/qcom/iris/iris_firmware.c
index 5f408024e967f..1a476146d7580 100644
--- a/drivers/media/platform/qcom/iris/iris_firmware.c
+++ b/drivers/media/platform/qcom/iris/iris_firmware.c
@@ -12,11 +12,12 @@
 #include "iris_core.h"
 #include "iris_firmware.h"
 
+#define IRIS_PAS_ID				9
+
 #define MAX_FIRMWARE_NAME_SIZE	128
 
 static int iris_load_fw_to_memory(struct iris_core *core, const char *fw_name)
 {
-	u32 pas_id = core->iris_platform_data->pas_id;
 	const struct firmware *firmware = NULL;
 	struct device *dev = core->dev;
 	struct resource res;
@@ -53,7 +54,7 @@ static int iris_load_fw_to_memory(struct iris_core *core, const char *fw_name)
 	}
 
 	ret = qcom_mdt_load(dev, firmware, fw_name,
-			    pas_id, mem_virt, mem_phys, res_size, NULL);
+			    IRIS_PAS_ID, mem_virt, mem_phys, res_size, NULL);
 
 	memunmap(mem_virt);
 err_release_fw:
@@ -71,7 +72,7 @@ int iris_fw_load(struct iris_core *core)
 	ret = of_property_read_string_index(core->dev->of_node, "firmware-name", 0,
 					    &fwpath);
 	if (ret)
-		fwpath = core->iris_platform_data->fwname;
+		fwpath = core->iris_firmware_desc->fwname;
 
 	ret = iris_load_fw_to_memory(core, fwpath);
 	if (ret) {
@@ -79,7 +80,7 @@ int iris_fw_load(struct iris_core *core)
 		return -ENOMEM;
 	}
 
-	ret = qcom_scm_pas_auth_and_reset(core->iris_platform_data->pas_id);
+	ret = qcom_scm_pas_auth_and_reset(IRIS_PAS_ID);
 	if (ret)  {
 		dev_err(core->dev, "auth and reset failed: %d\n", ret);
 		return ret;
@@ -93,7 +94,7 @@ int iris_fw_load(struct iris_core *core)
 						     cp_config->cp_nonpixel_size);
 		if (ret) {
 			dev_err(core->dev, "qcom_scm_mem_protect_video_var failed: %d\n", ret);
-			qcom_scm_pas_shutdown(core->iris_platform_data->pas_id);
+			qcom_scm_pas_shutdown(IRIS_PAS_ID);
 			return ret;
 		}
 	}
@@ -103,7 +104,7 @@ int iris_fw_load(struct iris_core *core)
 
 int iris_fw_unload(struct iris_core *core)
 {
-	return qcom_scm_pas_shutdown(core->iris_platform_data->pas_id);
+	return qcom_scm_pas_shutdown(IRIS_PAS_ID);
 }
 
 int iris_set_hw_state(struct iris_core *core, bool resume)
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_common.c b/drivers/media/platform/qcom/iris/iris_hfi_common.c
index 621c66593d88d..8769ec61f1176 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_common.c
+++ b/drivers/media/platform/qcom/iris/iris_hfi_common.c
@@ -76,7 +76,7 @@ u32 iris_hfi_get_v4l2_matrix_coefficients(u32 hfi_coefficients)
 
 int iris_hfi_core_init(struct iris_core *core)
 {
-	const struct iris_hfi_command_ops *hfi_ops = core->hfi_ops;
+	const struct iris_hfi_sys_ops *hfi_ops = core->hfi_sys_ops;
 	int ret;
 
 	ret = hfi_ops->sys_init(core);
@@ -109,7 +109,7 @@ irqreturn_t iris_hfi_isr_handler(int irq, void *data)
 	iris_vpu_clear_interrupt(core);
 	mutex_unlock(&core->lock);
 
-	core->hfi_response_ops->hfi_response_handler(core);
+	core->hfi_sys_ops->sys_hfi_response_handler(core);
 
 	if (!iris_vpu_watchdog(core, core->intr_status))
 		enable_irq(irq);
@@ -144,7 +144,7 @@ int iris_hfi_pm_suspend(struct iris_core *core)
 
 int iris_hfi_pm_resume(struct iris_core *core)
 {
-	const struct iris_hfi_command_ops *ops = core->hfi_ops;
+	const struct iris_hfi_sys_ops *ops = core->hfi_sys_ops;
 	int ret;
 
 	ret = iris_vpu_power_on(core);
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_common.h b/drivers/media/platform/qcom/iris/iris_hfi_common.h
index 3edb5ae582b49..a27447eb25199 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_common.h
+++ b/drivers/media/platform/qcom/iris/iris_hfi_common.h
@@ -105,11 +105,18 @@ struct iris_hfi_prop_type_handle {
 	int (*handle)(struct iris_inst *inst, u32 plane);
 };
 
-struct iris_hfi_command_ops {
+struct iris_hfi_sys_ops {
 	int (*sys_init)(struct iris_core *core);
 	int (*sys_image_version)(struct iris_core *core);
 	int (*sys_interframe_powercollapse)(struct iris_core *core);
 	int (*sys_pc_prep)(struct iris_core *core);
+
+	void (*sys_hfi_response_handler)(struct iris_core *core);
+
+	struct iris_inst *(*sys_get_instance)(void);
+};
+
+struct iris_hfi_session_ops {
 	int (*session_set_config_params)(struct iris_inst *inst, u32 plane);
 	int (*session_set_property)(struct iris_inst *inst,
 				    u32 packet_type, u32 flag, u32 plane, u32 payload_type,
@@ -126,10 +133,6 @@ struct iris_hfi_command_ops {
 	int (*session_close)(struct iris_inst *inst);
 };
 
-struct iris_hfi_response_ops {
-	void (*hfi_response_handler)(struct iris_core *core);
-};
-
 struct hfi_subscription_params {
 	u32	bitstream_resolution;
 	u32	crop_offsets[2];
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen1.c b/drivers/media/platform/qcom/iris/iris_hfi_gen1.c
new file mode 100644
index 0000000000000..60f51a1ba9412
--- /dev/null
+++ b/drivers/media/platform/qcom/iris/iris_hfi_gen1.c
@@ -0,0 +1,286 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include "iris_ctrls.h"
+#include "iris_platform_common.h"
+#include "iris_hfi_gen1.h"
+#include "iris_hfi_gen1_defines.h"
+#include "iris_vpu_buffer.h"
+
+#define BITRATE_MIN		32000
+#define BITRATE_MAX		160000000
+#define BITRATE_STEP		100
+
+static struct platform_inst_fw_cap inst_fw_cap_sm8250_dec[] = {
+	{
+		.cap_id = PIPE,
+		/* .max, .min and .value are set via platform data */
+		.step_or_mask = 1,
+		.hfi_id = HFI_PROPERTY_PARAM_WORK_ROUTE,
+		.set = iris_set_pipe,
+	},
+	{
+		.cap_id = STAGE,
+		.min = STAGE_1,
+		.max = STAGE_2,
+		.step_or_mask = 1,
+		.value = STAGE_2,
+		.hfi_id = HFI_PROPERTY_PARAM_WORK_MODE,
+		.set = iris_set_stage,
+	},
+};
+
+static const struct platform_inst_fw_cap inst_fw_cap_sm8250_enc[] = {
+	{
+		.cap_id = STAGE,
+		.min = STAGE_1,
+		.max = STAGE_2,
+		.step_or_mask = 1,
+		.value = STAGE_2,
+		.hfi_id = HFI_PROPERTY_PARAM_WORK_MODE,
+		.set = iris_set_stage,
+	},
+	{
+		.cap_id = PROFILE_H264,
+		.min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
+		.max = V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_STEREO_HIGH) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH),
+		.value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH,
+		.hfi_id = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_profile_level_gen1,
+	},
+	{
+		.cap_id = PROFILE_HEVC,
+		.min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
+		.max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10),
+		.value = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
+		.hfi_id = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_profile_level_gen1,
+	},
+	{
+		.cap_id = LEVEL_H264,
+		.min = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
+		.max = V4L2_MPEG_VIDEO_H264_LEVEL_5_1,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1B) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_3) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_1),
+		.value = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
+		.hfi_id = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_profile_level_gen1,
+	},
+	{
+		.cap_id = LEVEL_HEVC,
+		.min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
+		.max = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2),
+		.value = V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
+		.hfi_id = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_profile_level_gen1,
+	},
+	{
+		.cap_id = HEADER_MODE,
+		.min = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE,
+		.max = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE) |
+				BIT(V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME),
+		.value = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
+		.hfi_id = HFI_PROPERTY_CONFIG_VENC_SYNC_FRAME_SEQUENCE_HEADER,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_header_mode_gen1,
+	},
+	{
+		.cap_id = BITRATE,
+		.min = BITRATE_MIN,
+		.max = BITRATE_MAX,
+		.step_or_mask = BITRATE_STEP,
+		.value = BITRATE_DEFAULT,
+		.hfi_id = HFI_PROPERTY_CONFIG_VENC_TARGET_BITRATE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_bitrate,
+	},
+	{
+		.cap_id = BITRATE_MODE,
+		.min = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR,
+		.max = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_VBR) |
+				BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR),
+		.value = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR,
+		.hfi_id = HFI_PROPERTY_PARAM_VENC_RATE_CONTROL,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_bitrate_mode_gen1,
+	},
+	{
+		.cap_id = FRAME_SKIP_MODE,
+		.min = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED,
+		.max = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED) |
+				BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT),
+		.value = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+	},
+	{
+		.cap_id = FRAME_RC_ENABLE,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 1,
+	},
+	{
+		.cap_id = GOP_SIZE,
+		.min = 0,
+		.max = (1 << 16) - 1,
+		.step_or_mask = 1,
+		.value = 30,
+		.set = iris_set_u32
+	},
+	{
+		.cap_id = ENTROPY_MODE,
+		.min = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC,
+		.max = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC) |
+				BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC),
+		.value = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC,
+		.hfi_id = HFI_PROPERTY_PARAM_VENC_H264_ENTROPY_CONTROL,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_entropy_mode_gen1,
+	},
+	{
+		.cap_id = MIN_FRAME_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+		.hfi_id = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_qp_range,
+	},
+	{
+		.cap_id = MIN_FRAME_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP_HEVC,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+		.hfi_id = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_qp_range,
+	},
+	{
+		.cap_id = MAX_FRAME_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MAX_QP,
+		.hfi_id = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_qp_range,
+	},
+	{
+		.cap_id = MAX_FRAME_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP_HEVC,
+		.step_or_mask = 1,
+		.value = MAX_QP_HEVC,
+		.hfi_id = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_qp_range,
+	},
+};
+
+static const u32 sm8250_vdec_input_config_param_default[] = {
+	HFI_PROPERTY_CONFIG_VIDEOCORES_USAGE,
+	HFI_PROPERTY_PARAM_UNCOMPRESSED_FORMAT_SELECT,
+	HFI_PROPERTY_PARAM_UNCOMPRESSED_PLANE_ACTUAL_CONSTRAINTS_INFO,
+	HFI_PROPERTY_PARAM_BUFFER_COUNT_ACTUAL,
+	HFI_PROPERTY_PARAM_VDEC_MULTI_STREAM,
+	HFI_PROPERTY_PARAM_FRAME_SIZE,
+	HFI_PROPERTY_PARAM_BUFFER_SIZE_ACTUAL,
+	HFI_PROPERTY_PARAM_BUFFER_ALLOC_MODE,
+};
+
+static const u32 sm8250_venc_input_config_param[] = {
+	HFI_PROPERTY_CONFIG_FRAME_RATE,
+	HFI_PROPERTY_PARAM_UNCOMPRESSED_PLANE_ACTUAL_INFO,
+	HFI_PROPERTY_PARAM_FRAME_SIZE,
+	HFI_PROPERTY_PARAM_UNCOMPRESSED_FORMAT_SELECT,
+	HFI_PROPERTY_PARAM_BUFFER_COUNT_ACTUAL,
+};
+
+static const u32 sm8250_dec_ip_int_buf_tbl[] = {
+	BUF_BIN,
+	BUF_SCRATCH_1,
+};
+
+static const u32 sm8250_dec_op_int_buf_tbl[] = {
+	BUF_DPB,
+};
+
+static const u32 sm8250_enc_ip_int_buf_tbl[] = {
+	BUF_BIN,
+	BUF_SCRATCH_1,
+	BUF_SCRATCH_2,
+};
+
+const struct iris_firmware_data iris_hfi_gen1_data = {
+	.init_hfi_ops = &iris_hfi_gen1_sys_ops_init,
+
+	.inst_fw_caps_dec = inst_fw_cap_sm8250_dec,
+	.inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8250_dec),
+	.inst_fw_caps_enc = inst_fw_cap_sm8250_enc,
+	.inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8250_enc),
+
+	.dec_input_config_params_default =
+		sm8250_vdec_input_config_param_default,
+	.dec_input_config_params_default_size =
+		ARRAY_SIZE(sm8250_vdec_input_config_param_default),
+	.enc_input_config_params = sm8250_venc_input_config_param,
+	.enc_input_config_params_size =
+		ARRAY_SIZE(sm8250_venc_input_config_param),
+
+	.dec_ip_int_buf_tbl = sm8250_dec_ip_int_buf_tbl,
+	.dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8250_dec_ip_int_buf_tbl),
+	.dec_op_int_buf_tbl = sm8250_dec_op_int_buf_tbl,
+	.dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8250_dec_op_int_buf_tbl),
+
+	.enc_ip_int_buf_tbl = sm8250_enc_ip_int_buf_tbl,
+	.enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8250_enc_ip_int_buf_tbl),
+};
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen1.h b/drivers/media/platform/qcom/iris/iris_hfi_gen1.h
index 19b8e9054a757..c37adf65055a5 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_gen1.h
+++ b/drivers/media/platform/qcom/iris/iris_hfi_gen1.h
@@ -7,10 +7,8 @@
 #define __IRIS_HFI_GEN1_H__
 
 struct iris_core;
-struct iris_inst;
 
-void iris_hfi_gen1_command_ops_init(struct iris_core *core);
-void iris_hfi_gen1_response_ops_init(struct iris_core *core);
-struct iris_inst *iris_hfi_gen1_get_instance(void);
+void iris_hfi_gen1_sys_ops_init(struct iris_core *core);
+void iris_hfi_gen1_response_handler(struct iris_core *core);
 
 #endif
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c
index e42d17653c2c3..83373862655f7 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c
+++ b/drivers/media/platform/qcom/iris/iris_hfi_gen1_command.c
@@ -918,7 +918,7 @@ static int iris_hfi_gen1_set_bufsize(struct iris_inst *inst, u32 plane)
 
 	if (iris_split_mode_enabled(inst)) {
 		bufsz.type = HFI_BUFFER_OUTPUT;
-		bufsz.size = inst->core->iris_platform_data->get_vpu_buffer_size(inst, BUF_DPB);
+		bufsz.size = inst->core->iris_firmware_desc->get_vpu_buffer_size(inst, BUF_DPB);
 
 		ret = hfi_gen1_set_property(inst, ptype, &bufsz, sizeof(bufsz));
 		if (ret)
@@ -1033,8 +1033,8 @@ static int iris_hfi_gen1_session_set_config_params(struct iris_inst *inst, u32 p
 	};
 
 	if (inst->domain == DECODER) {
-		config_params = core->iris_platform_data->dec_input_config_params_default;
-		config_params_size = core->iris_platform_data->dec_input_config_params_default_size;
+		config_params = core->iris_firmware_data->dec_input_config_params_default;
+		config_params_size = core->iris_firmware_data->dec_input_config_params_default_size;
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
 			handler = vdec_prop_type_handle_inp_arr;
 			handler_size = ARRAY_SIZE(vdec_prop_type_handle_inp_arr);
@@ -1043,8 +1043,8 @@ static int iris_hfi_gen1_session_set_config_params(struct iris_inst *inst, u32 p
 			handler_size = ARRAY_SIZE(vdec_prop_type_handle_out_arr);
 		}
 	} else {
-		config_params = core->iris_platform_data->enc_input_config_params;
-		config_params_size = core->iris_platform_data->enc_input_config_params_size;
+		config_params = core->iris_firmware_data->enc_input_config_params;
+		config_params_size = core->iris_firmware_data->enc_input_config_params_size;
 		handler = venc_prop_type_handle_inp_arr;
 		handler_size = ARRAY_SIZE(venc_prop_type_handle_inp_arr);
 	}
@@ -1063,11 +1063,7 @@ static int iris_hfi_gen1_session_set_config_params(struct iris_inst *inst, u32 p
 	return 0;
 }
 
-static const struct iris_hfi_command_ops iris_hfi_gen1_command_ops = {
-	.sys_init = iris_hfi_gen1_sys_init,
-	.sys_image_version = iris_hfi_gen1_sys_image_version,
-	.sys_interframe_powercollapse = iris_hfi_gen1_sys_interframe_powercollapse,
-	.sys_pc_prep = iris_hfi_gen1_sys_pc_prep,
+static const struct iris_hfi_session_ops iris_hfi_gen1_session_ops = {
 	.session_open = iris_hfi_gen1_session_open,
 	.session_set_config_params = iris_hfi_gen1_session_set_config_params,
 	.session_set_property = iris_hfi_gen1_session_set_property,
@@ -1080,12 +1076,31 @@ static const struct iris_hfi_command_ops iris_hfi_gen1_command_ops = {
 	.session_close = iris_hfi_gen1_session_close,
 };
 
-void iris_hfi_gen1_command_ops_init(struct iris_core *core)
+static struct iris_inst *iris_hfi_gen1_get_instance(void)
 {
-	core->hfi_ops = &iris_hfi_gen1_command_ops;
+	struct iris_inst *out;
+
+	out = kzalloc_obj(*out);
+	if (!out)
+		return NULL;
+
+	out->hfi_session_ops = &iris_hfi_gen1_session_ops;
+
+	return out;
 }
 
-struct iris_inst *iris_hfi_gen1_get_instance(void)
+static const struct iris_hfi_sys_ops iris_hfi_gen1_sys_ops = {
+	.sys_init = iris_hfi_gen1_sys_init,
+	.sys_image_version = iris_hfi_gen1_sys_image_version,
+	.sys_interframe_powercollapse = iris_hfi_gen1_sys_interframe_powercollapse,
+	.sys_pc_prep = iris_hfi_gen1_sys_pc_prep,
+
+	.sys_hfi_response_handler = iris_hfi_gen1_response_handler,
+
+	.sys_get_instance = iris_hfi_gen1_get_instance,
+};
+
+void iris_hfi_gen1_sys_ops_init(struct iris_core *core)
 {
-	return kzalloc_obj(struct iris_inst);
+	core->hfi_sys_ops = &iris_hfi_gen1_sys_ops;
 }
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen1_response.c b/drivers/media/platform/qcom/iris/iris_hfi_gen1_response.c
index 8e864c239e293..bfd7495bf44f0 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_gen1_response.c
+++ b/drivers/media/platform/qcom/iris/iris_hfi_gen1_response.c
@@ -688,7 +688,7 @@ static void iris_hfi_gen1_flush_debug_queue(struct iris_core *core, u8 *packet)
 	}
 }
 
-static void iris_hfi_gen1_response_handler(struct iris_core *core)
+void iris_hfi_gen1_response_handler(struct iris_core *core)
 {
 	memset(core->response_packet, 0, sizeof(struct hfi_pkt_hdr));
 	while (!iris_hfi_queue_msg_read(core, core->response_packet)) {
@@ -698,12 +698,3 @@ static void iris_hfi_gen1_response_handler(struct iris_core *core)
 
 	iris_hfi_gen1_flush_debug_queue(core, core->response_packet);
 }
-
-static const struct iris_hfi_response_ops iris_hfi_gen1_response_ops = {
-	.hfi_response_handler = iris_hfi_gen1_response_handler,
-};
-
-void iris_hfi_gen1_response_ops_init(struct iris_core *core)
-{
-	core->hfi_response_ops = &iris_hfi_gen1_response_ops;
-}
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2.c
new file mode 100644
index 0000000000000..ce8490d64854c
--- /dev/null
+++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2.c
@@ -0,0 +1,896 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2025 Linaro Ltd
+ */
+
+#include "iris_ctrls.h"
+#include "iris_hfi_gen2.h"
+#include "iris_hfi_gen2_defines.h"
+#include "iris_platform_common.h"
+#include "iris_vpu_buffer.h"
+
+#define VIDEO_ARCH_LX 1
+#define BITRATE_MAX				245000000
+
+static const struct platform_inst_fw_cap inst_fw_cap_sm8550_dec[] = {
+	{
+		.cap_id = PROFILE_H264,
+		.min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
+		.max = V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH),
+		.value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH,
+		.hfi_id = HFI_PROP_PROFILE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_u32_enum,
+	},
+	{
+		.cap_id = PROFILE_HEVC,
+		.min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
+		.max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE),
+		.value = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
+		.hfi_id = HFI_PROP_PROFILE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_u32_enum,
+	},
+	{
+		.cap_id = PROFILE_AV1,
+		.min = V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN,
+		.max = V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN),
+		.value = V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN,
+		.hfi_id = HFI_PROP_PROFILE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_u32_enum,
+	},
+	{
+		.cap_id = PROFILE_VP9,
+		.min = V4L2_MPEG_VIDEO_VP9_PROFILE_0,
+		.max = V4L2_MPEG_VIDEO_VP9_PROFILE_2,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_VP9_PROFILE_0) |
+				BIT(V4L2_MPEG_VIDEO_VP9_PROFILE_2),
+		.value = V4L2_MPEG_VIDEO_VP9_PROFILE_0,
+		.hfi_id = HFI_PROP_PROFILE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_u32_enum,
+	},
+	{
+		.cap_id = LEVEL_H264,
+		.min = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
+		.max = V4L2_MPEG_VIDEO_H264_LEVEL_6_2,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1B) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_3) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_2),
+		.value = V4L2_MPEG_VIDEO_H264_LEVEL_6_1,
+		.hfi_id = HFI_PROP_LEVEL,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_u32_enum,
+	},
+	{
+		.cap_id = LEVEL_HEVC,
+		.min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
+		.max = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2),
+		.value = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1,
+		.hfi_id = HFI_PROP_LEVEL,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_u32_enum,
+	},
+	{
+		.cap_id = LEVEL_VP9,
+		.min = V4L2_MPEG_VIDEO_VP9_LEVEL_1_0,
+		.max = V4L2_MPEG_VIDEO_VP9_LEVEL_6_0,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_1_0) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_1_1) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_2_0) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_2_1) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_3_0) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_3_1) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_4_0) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_4_1) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_5_0) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_5_1) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_5_2) |
+				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_6_0),
+		.value = V4L2_MPEG_VIDEO_VP9_LEVEL_6_0,
+		.hfi_id = HFI_PROP_LEVEL,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_u32_enum,
+	},
+	{
+		.cap_id = LEVEL_AV1,
+		.min = V4L2_MPEG_VIDEO_AV1_LEVEL_2_0,
+		.max = V4L2_MPEG_VIDEO_AV1_LEVEL_6_1,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_0) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_1) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_2) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_3) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_0) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_1) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_2) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_3) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_0) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_1) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_2) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_3) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_0) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_1) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_2) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_3) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_6_0) |
+				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_6_1),
+		.value = V4L2_MPEG_VIDEO_AV1_LEVEL_6_1,
+		.hfi_id = HFI_PROP_LEVEL,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_u32_enum,
+	},
+	{
+		.cap_id = TIER,
+		.min = V4L2_MPEG_VIDEO_HEVC_TIER_MAIN,
+		.max = V4L2_MPEG_VIDEO_HEVC_TIER_HIGH,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_TIER_MAIN) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_TIER_HIGH),
+		.value = V4L2_MPEG_VIDEO_HEVC_TIER_HIGH,
+		.hfi_id = HFI_PROP_TIER,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_u32_enum,
+	},
+	{
+		.cap_id = TIER_AV1,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 0,
+		.hfi_id = HFI_PROP_TIER,
+		.flags = CAP_FLAG_INPUT_PORT,
+		.set = iris_set_u32,
+	},
+	{
+		.cap_id = DRAP,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 0,
+		.hfi_id = HFI_PROP_AV1_DRAP_CONFIG,
+		.flags = CAP_FLAG_INPUT_PORT,
+		.set = iris_set_u32,
+	},
+	{
+		.cap_id = FILM_GRAIN,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 0,
+		.hfi_id = HFI_PROP_AV1_FILM_GRAIN_PRESENT,
+		.flags = CAP_FLAG_VOLATILE,
+	},
+	{
+		.cap_id = SUPER_BLOCK,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 0,
+		.hfi_id = HFI_PROP_AV1_SUPER_BLOCK_ENABLED,
+	},
+	{
+		.cap_id = ENH_LAYER_COUNT,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 0,
+		.hfi_id = HFI_PROP_AV1_OP_POINT,
+		.flags = CAP_FLAG_INPUT_PORT,
+		.set = iris_set_u32,
+	},
+	{
+		.cap_id = INPUT_BUF_HOST_MAX_COUNT,
+		.min = DEFAULT_MAX_HOST_BUF_COUNT,
+		.max = DEFAULT_MAX_HOST_BURST_BUF_COUNT,
+		.step_or_mask = 1,
+		.value = DEFAULT_MAX_HOST_BUF_COUNT,
+		.hfi_id = HFI_PROP_BUFFER_HOST_MAX_COUNT,
+		.flags = CAP_FLAG_INPUT_PORT,
+		.set = iris_set_u32,
+	},
+	{
+		.cap_id = STAGE,
+		.min = STAGE_1,
+		.max = STAGE_2,
+		.step_or_mask = 1,
+		.value = STAGE_2,
+		.hfi_id = HFI_PROP_STAGE,
+		.set = iris_set_stage,
+	},
+	{
+		.cap_id = PIPE,
+		/* .max, .min and .value are set via platform data */
+		.step_or_mask = 1,
+		.hfi_id = HFI_PROP_PIPE,
+		.set = iris_set_pipe,
+	},
+	{
+		.cap_id = POC,
+		.min = 0,
+		.max = 2,
+		.step_or_mask = 1,
+		.value = 1,
+		.hfi_id = HFI_PROP_PIC_ORDER_CNT_TYPE,
+	},
+	{
+		.cap_id = CODED_FRAMES,
+		.min = CODED_FRAMES_PROGRESSIVE,
+		.max = CODED_FRAMES_PROGRESSIVE,
+		.step_or_mask = 0,
+		.value = CODED_FRAMES_PROGRESSIVE,
+		.hfi_id = HFI_PROP_CODED_FRAMES,
+	},
+	{
+		.cap_id = BIT_DEPTH,
+		.min = BIT_DEPTH_8,
+		.max = BIT_DEPTH_8,
+		.step_or_mask = 1,
+		.value = BIT_DEPTH_8,
+		.hfi_id = HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
+	},
+	{
+		.cap_id = RAP_FRAME,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 1,
+		.hfi_id = HFI_PROP_DEC_START_FROM_RAP_FRAME,
+		.flags = CAP_FLAG_INPUT_PORT,
+		.set = iris_set_u32,
+	},
+};
+
+static const struct platform_inst_fw_cap inst_fw_cap_sm8550_enc[] = {
+	{
+		.cap_id = PROFILE_H264,
+		.min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
+		.max = V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) |
+				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH),
+		.value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH,
+		.hfi_id = HFI_PROP_PROFILE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_profile,
+	},
+	{
+		.cap_id = PROFILE_HEVC,
+		.min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
+		.max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10),
+		.value = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
+		.hfi_id = HFI_PROP_PROFILE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_profile,
+	},
+	{
+		.cap_id = LEVEL_H264,
+		.min = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
+		.max = V4L2_MPEG_VIDEO_H264_LEVEL_6_0,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1B) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_3) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_0) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_1) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_2) |
+				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_0),
+		.value = V4L2_MPEG_VIDEO_H264_LEVEL_5_0,
+		.hfi_id = HFI_PROP_LEVEL,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_level,
+	},
+	{
+		.cap_id = LEVEL_HEVC,
+		.min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
+		.max = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1) |
+				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2),
+		.value = V4L2_MPEG_VIDEO_HEVC_LEVEL_5,
+		.hfi_id = HFI_PROP_LEVEL,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_level,
+	},
+	{
+		.cap_id = STAGE,
+		.min = STAGE_1,
+		.max = STAGE_2,
+		.step_or_mask = 1,
+		.value = STAGE_2,
+		.hfi_id = HFI_PROP_STAGE,
+		.set = iris_set_stage,
+	},
+	{
+		.cap_id = HEADER_MODE,
+		.min = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE,
+		.max = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE) |
+				BIT(V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME),
+		.value = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
+		.hfi_id = HFI_PROP_SEQ_HEADER_MODE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_header_mode_gen2,
+	},
+	{
+		.cap_id = PREPEND_SPSPPS_TO_IDR,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 0,
+	},
+	{
+		.cap_id = BITRATE,
+		.min = 1,
+		.max = BITRATE_MAX,
+		.step_or_mask = 1,
+		.value = BITRATE_DEFAULT,
+		.hfi_id = HFI_PROP_TOTAL_BITRATE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_bitrate,
+	},
+	{
+		.cap_id = BITRATE_PEAK,
+		.min = 1,
+		.max = BITRATE_MAX,
+		.step_or_mask = 1,
+		.value = BITRATE_DEFAULT,
+		.hfi_id = HFI_PROP_TOTAL_PEAK_BITRATE,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_peak_bitrate,
+	},
+	{
+		.cap_id = BITRATE_MODE,
+		.min = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR,
+		.max = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_VBR) |
+				BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR),
+		.value = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR,
+		.hfi_id = HFI_PROP_RATE_CONTROL,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_bitrate_mode_gen2,
+	},
+	{
+		.cap_id = FRAME_SKIP_MODE,
+		.min = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED,
+		.max = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED) |
+				BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_LEVEL_LIMIT) |
+				BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT),
+		.value = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+	},
+	{
+		.cap_id = FRAME_RC_ENABLE,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 1,
+	},
+	{
+		.cap_id = GOP_SIZE,
+		.min = 0,
+		.max = INT_MAX,
+		.step_or_mask = 1,
+		.value = 2 * DEFAULT_FPS - 1,
+		.hfi_id = HFI_PROP_MAX_GOP_FRAMES,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_u32,
+	},
+	{
+		.cap_id = ENTROPY_MODE,
+		.min = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC,
+		.max = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC,
+		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC) |
+				BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC),
+		.value = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC,
+		.hfi_id = HFI_PROP_CABAC_SESSION,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+		.set = iris_set_entropy_mode_gen2,
+	},
+	{
+		.cap_id = MIN_FRAME_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+		.hfi_id = HFI_PROP_MIN_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_min_qp,
+	},
+	{
+		.cap_id = MIN_FRAME_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+		.hfi_id = HFI_PROP_MIN_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_min_qp,
+	},
+	{
+		.cap_id = MAX_FRAME_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MAX_QP,
+		.hfi_id = HFI_PROP_MAX_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_max_qp,
+	},
+	{
+		.cap_id = MAX_FRAME_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MAX_QP,
+		.hfi_id = HFI_PROP_MAX_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_max_qp,
+	},
+	{
+		.cap_id = I_FRAME_MIN_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+	},
+	{
+		.cap_id = I_FRAME_MIN_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+	},
+	{
+		.cap_id = P_FRAME_MIN_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+	},
+	{
+		.cap_id = P_FRAME_MIN_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+	},
+	{
+		.cap_id = B_FRAME_MIN_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+	},
+	{
+		.cap_id = B_FRAME_MIN_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MIN_QP_8BIT,
+	},
+	{
+		.cap_id = I_FRAME_MAX_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MAX_QP,
+	},
+	{
+		.cap_id = I_FRAME_MAX_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MAX_QP,
+	},
+	{
+		.cap_id = P_FRAME_MAX_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MAX_QP,
+	},
+	{
+		.cap_id = P_FRAME_MAX_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MAX_QP,
+	},
+	{
+		.cap_id = B_FRAME_MAX_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MAX_QP,
+	},
+	{
+		.cap_id = B_FRAME_MAX_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = MAX_QP,
+	},
+	{
+		.cap_id = I_FRAME_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = DEFAULT_QP,
+		.hfi_id = HFI_PROP_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_frame_qp,
+	},
+	{
+		.cap_id = I_FRAME_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = DEFAULT_QP,
+		.hfi_id = HFI_PROP_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_frame_qp,
+	},
+	{
+		.cap_id = P_FRAME_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = DEFAULT_QP,
+		.hfi_id = HFI_PROP_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_frame_qp,
+	},
+	{
+		.cap_id = P_FRAME_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = DEFAULT_QP,
+		.hfi_id = HFI_PROP_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_frame_qp,
+	},
+	{
+		.cap_id = B_FRAME_QP_H264,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = DEFAULT_QP,
+		.hfi_id = HFI_PROP_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_frame_qp,
+	},
+	{
+		.cap_id = B_FRAME_QP_HEVC,
+		.min = MIN_QP_8BIT,
+		.max = MAX_QP,
+		.step_or_mask = 1,
+		.value = DEFAULT_QP,
+		.hfi_id = HFI_PROP_QP_PACKED,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_frame_qp,
+	},
+	{
+		.cap_id = INPUT_BUF_HOST_MAX_COUNT,
+		.min = DEFAULT_MAX_HOST_BUF_COUNT,
+		.max = DEFAULT_MAX_HOST_BURST_BUF_COUNT,
+		.step_or_mask = 1,
+		.value = DEFAULT_MAX_HOST_BUF_COUNT,
+		.hfi_id = HFI_PROP_BUFFER_HOST_MAX_COUNT,
+		.flags = CAP_FLAG_INPUT_PORT,
+		.set = iris_set_u32,
+	},
+	{
+		.cap_id = OUTPUT_BUF_HOST_MAX_COUNT,
+		.min = DEFAULT_MAX_HOST_BUF_COUNT,
+		.max = DEFAULT_MAX_HOST_BURST_BUF_COUNT,
+		.step_or_mask = 1,
+		.value = DEFAULT_MAX_HOST_BUF_COUNT,
+		.hfi_id = HFI_PROP_BUFFER_HOST_MAX_COUNT,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_u32,
+	},
+	{
+		.cap_id = ROTATION,
+		.min = 0,
+		.max = 270,
+		.step_or_mask = 90,
+		.value = 0,
+		.hfi_id = HFI_PROP_ROTATION,
+		.flags = CAP_FLAG_OUTPUT_PORT,
+		.set = iris_set_rotation,
+	},
+	{
+		.cap_id = HFLIP,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 0,
+		.hfi_id = HFI_PROP_FLIP,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_flip,
+	},
+	{
+		.cap_id = VFLIP,
+		.min = 0,
+		.max = 1,
+		.step_or_mask = 1,
+		.value = 0,
+		.hfi_id = HFI_PROP_FLIP,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_flip,
+	},
+	{
+		.cap_id = IR_TYPE,
+		.min = V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM,
+		.max = V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_CYCLIC,
+		.step_or_mask = BIT(V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM) |
+			BIT(V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_CYCLIC),
+		.value = V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM,
+		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
+	},
+	{
+		.cap_id = IR_PERIOD,
+		.min = 0,
+		.max = INT_MAX,
+		.step_or_mask = 1,
+		.value = 0,
+		.flags = CAP_FLAG_OUTPUT_PORT |
+			CAP_FLAG_DYNAMIC_ALLOWED,
+		.set = iris_set_ir_period,
+	},
+};
+
+static const u32 sm8550_vdec_input_config_params_default[] = {
+	HFI_PROP_BITSTREAM_RESOLUTION,
+	HFI_PROP_CROP_OFFSETS,
+	HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
+	HFI_PROP_CODED_FRAMES,
+	HFI_PROP_BUFFER_FW_MIN_OUTPUT_COUNT,
+	HFI_PROP_PIC_ORDER_CNT_TYPE,
+	HFI_PROP_PROFILE,
+	HFI_PROP_LEVEL,
+	HFI_PROP_SIGNAL_COLOR_INFO,
+};
+
+static const u32 sm8550_vdec_input_config_param_hevc[] = {
+	HFI_PROP_BITSTREAM_RESOLUTION,
+	HFI_PROP_CROP_OFFSETS,
+	HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
+	HFI_PROP_BUFFER_FW_MIN_OUTPUT_COUNT,
+	HFI_PROP_PROFILE,
+	HFI_PROP_LEVEL,
+	HFI_PROP_TIER,
+	HFI_PROP_SIGNAL_COLOR_INFO,
+};
+
+static const u32 sm8550_vdec_input_config_param_vp9[] = {
+	HFI_PROP_BITSTREAM_RESOLUTION,
+	HFI_PROP_CROP_OFFSETS,
+	HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
+	HFI_PROP_BUFFER_FW_MIN_OUTPUT_COUNT,
+	HFI_PROP_PROFILE,
+	HFI_PROP_LEVEL,
+};
+
+static const u32 sm8550_vdec_input_config_param_av1[] = {
+	HFI_PROP_BITSTREAM_RESOLUTION,
+	HFI_PROP_CROP_OFFSETS,
+	HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
+	HFI_PROP_BUFFER_FW_MIN_OUTPUT_COUNT,
+	HFI_PROP_PROFILE,
+	HFI_PROP_LEVEL,
+	HFI_PROP_TIER,
+	HFI_PROP_AV1_FILM_GRAIN_PRESENT,
+	HFI_PROP_AV1_SUPER_BLOCK_ENABLED,
+	HFI_PROP_SIGNAL_COLOR_INFO,
+};
+
+static const u32 sm8550_venc_input_config_params[] = {
+	HFI_PROP_COLOR_FORMAT,
+	HFI_PROP_RAW_RESOLUTION,
+	HFI_PROP_CROP_OFFSETS,
+	HFI_PROP_LINEAR_STRIDE_SCANLINE,
+	HFI_PROP_SIGNAL_COLOR_INFO,
+};
+
+static const u32 sm8550_vdec_output_config_params[] = {
+	HFI_PROP_OPB_ENABLE,
+	HFI_PROP_COLOR_FORMAT,
+	HFI_PROP_LINEAR_STRIDE_SCANLINE,
+};
+
+static const u32 sm8550_venc_output_config_params[] = {
+	HFI_PROP_BITSTREAM_RESOLUTION,
+	HFI_PROP_CROP_OFFSETS,
+	HFI_PROP_FRAME_RATE,
+};
+
+static const u32 sm8550_vdec_subscribe_input_properties[] = {
+	HFI_PROP_NO_OUTPUT,
+};
+
+static const u32 sm8550_vdec_subscribe_output_properties_avc[] = {
+	HFI_PROP_PICTURE_TYPE,
+	HFI_PROP_CABAC_SESSION,
+};
+
+static const u32 sm8550_vdec_subscribe_output_properties_hevc[] = {
+	HFI_PROP_PICTURE_TYPE,
+};
+
+static const u32 sm8550_vdec_subscribe_output_properties_vp9[] = {
+	HFI_PROP_PICTURE_TYPE,
+};
+
+static const u32 sm8550_vdec_subscribe_output_properties_av1[] = {
+	HFI_PROP_PICTURE_TYPE,
+	HFI_PROP_WORST_COMPRESSION_RATIO,
+	HFI_PROP_WORST_COMPLEXITY_FACTOR,
+};
+
+static const u32 sm8550_dec_ip_int_buf_tbl[] = {
+	BUF_BIN,
+	BUF_COMV,
+	BUF_NON_COMV,
+	BUF_LINE,
+	BUF_PARTIAL,
+};
+
+static const u32 sm8550_dec_op_int_buf_tbl[] = {
+	BUF_DPB,
+};
+
+static const u32 sm8550_enc_ip_int_buf_tbl[] = {
+	BUF_VPSS,
+};
+
+static const u32 sm8550_enc_op_int_buf_tbl[] = {
+	BUF_BIN,
+	BUF_COMV,
+	BUF_NON_COMV,
+	BUF_LINE,
+	BUF_SCRATCH_2,
+};
+
+const struct iris_firmware_data iris_hfi_gen2_data = {
+	.init_hfi_ops = iris_hfi_gen2_sys_ops_init,
+
+	.core_arch = VIDEO_ARCH_LX,
+
+	.inst_fw_caps_dec = inst_fw_cap_sm8550_dec,
+	.inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec),
+	.inst_fw_caps_enc = inst_fw_cap_sm8550_enc,
+	.inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc),
+
+	.dec_input_config_params_default =
+		sm8550_vdec_input_config_params_default,
+	.dec_input_config_params_default_size =
+		ARRAY_SIZE(sm8550_vdec_input_config_params_default),
+	.dec_input_config_params_hevc =
+		sm8550_vdec_input_config_param_hevc,
+	.dec_input_config_params_hevc_size =
+		ARRAY_SIZE(sm8550_vdec_input_config_param_hevc),
+	.dec_input_config_params_vp9 =
+		sm8550_vdec_input_config_param_vp9,
+	.dec_input_config_params_vp9_size =
+		ARRAY_SIZE(sm8550_vdec_input_config_param_vp9),
+	.dec_input_config_params_av1 =
+		sm8550_vdec_input_config_param_av1,
+	.dec_input_config_params_av1_size =
+		ARRAY_SIZE(sm8550_vdec_input_config_param_av1),
+	.dec_output_config_params =
+		sm8550_vdec_output_config_params,
+	.dec_output_config_params_size =
+		ARRAY_SIZE(sm8550_vdec_output_config_params),
+
+	.enc_input_config_params =
+		sm8550_venc_input_config_params,
+	.enc_input_config_params_size =
+		ARRAY_SIZE(sm8550_venc_input_config_params),
+	.enc_output_config_params =
+		sm8550_venc_output_config_params,
+	.enc_output_config_params_size =
+		ARRAY_SIZE(sm8550_venc_output_config_params),
+
+	.dec_input_prop = sm8550_vdec_subscribe_input_properties,
+	.dec_input_prop_size = ARRAY_SIZE(sm8550_vdec_subscribe_input_properties),
+	.dec_output_prop_avc = sm8550_vdec_subscribe_output_properties_avc,
+	.dec_output_prop_avc_size =
+		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_avc),
+	.dec_output_prop_hevc = sm8550_vdec_subscribe_output_properties_hevc,
+	.dec_output_prop_hevc_size =
+		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_hevc),
+	.dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9,
+	.dec_output_prop_vp9_size =
+		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9),
+	.dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1,
+	.dec_output_prop_av1_size =
+		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1),
+
+	.dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl,
+	.dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl),
+	.dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl,
+	.dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl),
+
+	.enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl,
+	.enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl),
+	.enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl,
+	.enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl),
+};
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2.h b/drivers/media/platform/qcom/iris/iris_hfi_gen2.h
index b9d3749a10efe..21ab58e0aa840 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_gen2.h
+++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2.h
@@ -34,8 +34,7 @@ struct iris_inst_hfi_gen2 {
 	struct hfi_subscription_params	dst_subcr_params;
 };
 
-void iris_hfi_gen2_command_ops_init(struct iris_core *core);
-void iris_hfi_gen2_response_ops_init(struct iris_core *core);
-struct iris_inst *iris_hfi_gen2_get_instance(void);
+void iris_hfi_gen2_sys_ops_init(struct iris_core *core);
+void iris_hfi_gen2_response_handler(struct iris_core *core);
 
 #endif
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c
index 30bfd90d423ba..c90b22a75bc56 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c
+++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_command.c
@@ -601,7 +601,7 @@ static int iris_hfi_gen2_set_super_block(struct iris_inst *inst, u32 plane)
 
 static int iris_hfi_gen2_session_set_config_params(struct iris_inst *inst, u32 plane)
 {
-	const struct iris_platform_data *pdata = inst->core->iris_platform_data;
+	const struct iris_firmware_data *fdata = inst->core->iris_firmware_data;
 	u32 config_params_size = 0, i, j;
 	const u32 *config_params = NULL;
 	int ret;
@@ -630,31 +630,31 @@ static int iris_hfi_gen2_session_set_config_params(struct iris_inst *inst, u32 p
 	if (inst->domain == DECODER) {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
 			if (inst->codec == V4L2_PIX_FMT_H264) {
-				config_params = pdata->dec_input_config_params_default;
-				config_params_size = pdata->dec_input_config_params_default_size;
+				config_params = fdata->dec_input_config_params_default;
+				config_params_size = fdata->dec_input_config_params_default_size;
 			} else if (inst->codec == V4L2_PIX_FMT_HEVC) {
-				config_params = pdata->dec_input_config_params_hevc;
-				config_params_size = pdata->dec_input_config_params_hevc_size;
+				config_params = fdata->dec_input_config_params_hevc;
+				config_params_size = fdata->dec_input_config_params_hevc_size;
 			} else if (inst->codec == V4L2_PIX_FMT_VP9) {
-				config_params = pdata->dec_input_config_params_vp9;
-				config_params_size = pdata->dec_input_config_params_vp9_size;
+				config_params = fdata->dec_input_config_params_vp9;
+				config_params_size = fdata->dec_input_config_params_vp9_size;
 			} else if (inst->codec == V4L2_PIX_FMT_AV1) {
-				config_params = pdata->dec_input_config_params_av1;
-				config_params_size = pdata->dec_input_config_params_av1_size;
+				config_params = fdata->dec_input_config_params_av1;
+				config_params_size = fdata->dec_input_config_params_av1_size;
 			} else {
 				return -EINVAL;
 			}
 		} else {
-			config_params = pdata->dec_output_config_params;
-			config_params_size = pdata->dec_output_config_params_size;
+			config_params = fdata->dec_output_config_params;
+			config_params_size = fdata->dec_output_config_params_size;
 		}
 	} else {
 		if (V4L2_TYPE_IS_OUTPUT(plane)) {
-			config_params = pdata->enc_input_config_params;
-			config_params_size = pdata->enc_input_config_params_size;
+			config_params = fdata->enc_input_config_params;
+			config_params_size = fdata->enc_input_config_params_size;
 		} else {
-			config_params = pdata->enc_output_config_params;
-			config_params_size = pdata->enc_output_config_params_size;
+			config_params = fdata->enc_output_config_params;
+			config_params_size = fdata->enc_output_config_params_size;
 		}
 	}
 
@@ -849,24 +849,24 @@ static int iris_hfi_gen2_subscribe_change_param(struct iris_inst *inst, u32 plan
 
 	switch (inst->codec) {
 	case V4L2_PIX_FMT_H264:
-		change_param = core->iris_platform_data->dec_input_config_params_default;
+		change_param = core->iris_firmware_data->dec_input_config_params_default;
 		change_param_size =
-			core->iris_platform_data->dec_input_config_params_default_size;
+			core->iris_firmware_data->dec_input_config_params_default_size;
 		break;
 	case V4L2_PIX_FMT_HEVC:
-		change_param = core->iris_platform_data->dec_input_config_params_hevc;
+		change_param = core->iris_firmware_data->dec_input_config_params_hevc;
 		change_param_size =
-			core->iris_platform_data->dec_input_config_params_hevc_size;
+			core->iris_firmware_data->dec_input_config_params_hevc_size;
 		break;
 	case V4L2_PIX_FMT_VP9:
-		change_param = core->iris_platform_data->dec_input_config_params_vp9;
+		change_param = core->iris_firmware_data->dec_input_config_params_vp9;
 		change_param_size =
-			core->iris_platform_data->dec_input_config_params_vp9_size;
+			core->iris_firmware_data->dec_input_config_params_vp9_size;
 		break;
 	case V4L2_PIX_FMT_AV1:
-		change_param = core->iris_platform_data->dec_input_config_params_av1;
+		change_param = core->iris_firmware_data->dec_input_config_params_av1;
 		change_param_size =
-			core->iris_platform_data->dec_input_config_params_av1_size;
+			core->iris_firmware_data->dec_input_config_params_av1_size;
 		break;
 	}
 
@@ -996,29 +996,29 @@ static int iris_hfi_gen2_subscribe_property(struct iris_inst *inst, u32 plane)
 		return 0;
 
 	if (V4L2_TYPE_IS_OUTPUT(plane)) {
-		subscribe_prop_size = core->iris_platform_data->dec_input_prop_size;
-		subcribe_prop = core->iris_platform_data->dec_input_prop;
+		subscribe_prop_size = core->iris_firmware_data->dec_input_prop_size;
+		subcribe_prop = core->iris_firmware_data->dec_input_prop;
 	} else {
 		switch (inst->codec) {
 		case V4L2_PIX_FMT_H264:
-			subcribe_prop = core->iris_platform_data->dec_output_prop_avc;
+			subcribe_prop = core->iris_firmware_data->dec_output_prop_avc;
 			subscribe_prop_size =
-				core->iris_platform_data->dec_output_prop_avc_size;
+				core->iris_firmware_data->dec_output_prop_avc_size;
 			break;
 		case V4L2_PIX_FMT_HEVC:
-			subcribe_prop = core->iris_platform_data->dec_output_prop_hevc;
+			subcribe_prop = core->iris_firmware_data->dec_output_prop_hevc;
 			subscribe_prop_size =
-				core->iris_platform_data->dec_output_prop_hevc_size;
+				core->iris_firmware_data->dec_output_prop_hevc_size;
 			break;
 		case V4L2_PIX_FMT_VP9:
-			subcribe_prop = core->iris_platform_data->dec_output_prop_vp9;
+			subcribe_prop = core->iris_firmware_data->dec_output_prop_vp9;
 			subscribe_prop_size =
-				core->iris_platform_data->dec_output_prop_vp9_size;
+				core->iris_firmware_data->dec_output_prop_vp9_size;
 			break;
 		case V4L2_PIX_FMT_AV1:
-			subcribe_prop = core->iris_platform_data->dec_output_prop_av1;
+			subcribe_prop = core->iris_firmware_data->dec_output_prop_av1;
 			subscribe_prop_size =
-				core->iris_platform_data->dec_output_prop_av1_size;
+				core->iris_firmware_data->dec_output_prop_av1_size;
 			break;
 		}
 	}
@@ -1205,7 +1205,7 @@ static u32 iris_hfi_gen2_buf_type_from_driver(u32 domain, enum iris_buffer_type
 	}
 }
 
-static int iris_set_num_comv(struct iris_inst *inst)
+static int iris_hfi_gen2_set_num_comv(struct iris_inst *inst)
 {
 	struct platform_inst_caps *caps;
 	struct iris_core *core = inst->core;
@@ -1220,12 +1220,12 @@ static int iris_set_num_comv(struct iris_inst *inst)
 	num_comv = (inst->codec == V4L2_PIX_FMT_AV1) ?
 				NUM_COMV_AV1 : caps->num_comv;
 
-	return core->hfi_ops->session_set_property(inst,
-						   HFI_PROP_COMV_BUFFER_COUNT,
-						   HFI_HOST_FLAGS_NONE,
-						   HFI_PORT_BITSTREAM,
-						   HFI_PAYLOAD_U32,
-						   &num_comv, sizeof(u32));
+	return iris_hfi_gen2_session_set_property(inst,
+						  HFI_PROP_COMV_BUFFER_COUNT,
+						  HFI_HOST_FLAGS_NONE,
+						  HFI_PORT_BITSTREAM,
+						  HFI_PAYLOAD_U32,
+						  &num_comv, sizeof(u32));
 }
 
 static void iris_hfi_gen2_get_buffer(u32 domain, struct iris_buffer *buffer,
@@ -1257,7 +1257,7 @@ static int iris_hfi_gen2_session_queue_buffer(struct iris_inst *inst, struct iri
 
 	iris_hfi_gen2_get_buffer(inst->domain, buffer, &hfi_buffer);
 	if (buffer->type == BUF_COMV) {
-		ret = iris_set_num_comv(inst);
+		ret = iris_hfi_gen2_set_num_comv(inst);
 		if (ret)
 			return ret;
 	}
@@ -1300,11 +1300,7 @@ static int iris_hfi_gen2_session_release_buffer(struct iris_inst *inst, struct i
 					inst_hfi_gen2->packet->size);
 }
 
-static const struct iris_hfi_command_ops iris_hfi_gen2_command_ops = {
-	.sys_init = iris_hfi_gen2_sys_init,
-	.sys_image_version = iris_hfi_gen2_sys_image_version,
-	.sys_interframe_powercollapse = iris_hfi_gen2_sys_interframe_powercollapse,
-	.sys_pc_prep = iris_hfi_gen2_sys_pc_prep,
+static const struct iris_hfi_session_ops iris_hfi_gen2_session_ops = {
 	.session_open = iris_hfi_gen2_session_open,
 	.session_set_config_params = iris_hfi_gen2_session_set_config_params,
 	.session_set_property = iris_hfi_gen2_session_set_property,
@@ -1319,17 +1315,32 @@ static const struct iris_hfi_command_ops iris_hfi_gen2_command_ops = {
 	.session_close = iris_hfi_gen2_session_close,
 };
 
-void iris_hfi_gen2_command_ops_init(struct iris_core *core)
-{
-	core->hfi_ops = &iris_hfi_gen2_command_ops;
-}
-
-struct iris_inst *iris_hfi_gen2_get_instance(void)
+static struct iris_inst *iris_hfi_gen2_get_instance(void)
 {
 	struct iris_inst_hfi_gen2 *out;
 
 	/* The allocation is intentionally larger than struct iris_inst. */
 	out = kzalloc_obj(*out);
+	if (!out)
+		return NULL;
+
+	out->inst.hfi_session_ops = &iris_hfi_gen2_session_ops;
 
 	return &out->inst;
 }
+
+static const struct iris_hfi_sys_ops iris_hfi_gen2_sys_ops = {
+	.sys_init = iris_hfi_gen2_sys_init,
+	.sys_image_version = iris_hfi_gen2_sys_image_version,
+	.sys_interframe_powercollapse = iris_hfi_gen2_sys_interframe_powercollapse,
+	.sys_pc_prep = iris_hfi_gen2_sys_pc_prep,
+
+	.sys_hfi_response_handler = iris_hfi_gen2_response_handler,
+
+	.sys_get_instance = iris_hfi_gen2_get_instance,
+};
+
+void iris_hfi_gen2_sys_ops_init(struct iris_core *core)
+{
+	core->hfi_sys_ops = &iris_hfi_gen2_sys_ops;
+}
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_packet.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2_packet.c
index d77fa29f44fc6..0d05dd2afc07d 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_packet.c
+++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_packet.c
@@ -3,6 +3,9 @@
  * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
+#include <linux/printk.h>
+#include <linux/soc/qcom/ubwc.h>
+
 #include "iris_hfi_common.h"
 #include "iris_hfi_gen2.h"
 #include "iris_hfi_gen2_packet.h"
@@ -120,6 +123,7 @@ static void iris_hfi_gen2_create_packet(struct iris_hfi_header *hdr, u32 pkt_typ
 
 void iris_hfi_gen2_packet_sys_init(struct iris_core *core, struct iris_hfi_header *hdr)
 {
+	const struct qcom_ubwc_cfg_data *ubwc = core->ubwc_cfg;
 	u32 payload = 0;
 
 	iris_hfi_gen2_create_header(hdr, 0, core->header_id++);
@@ -136,7 +140,7 @@ void iris_hfi_gen2_packet_sys_init(struct iris_core *core, struct iris_hfi_heade
 				    &payload,
 				    sizeof(u32));
 
-	payload = core->iris_platform_data->ubwc_config->max_channels;
+	payload = qcom_ubwc_macrotile_mode(ubwc) ? 8 : 4;
 	iris_hfi_gen2_create_packet(hdr,
 				    HFI_PROP_UBWC_MAX_CHANNELS,
 				    HFI_HOST_FLAGS_NONE,
@@ -146,7 +150,7 @@ void iris_hfi_gen2_packet_sys_init(struct iris_core *core, struct iris_hfi_heade
 				    &payload,
 				    sizeof(u32));
 
-	payload = core->iris_platform_data->ubwc_config->mal_length;
+	payload = qcom_ubwc_min_acc_length_64b(ubwc) ? 64 : 32;
 	iris_hfi_gen2_create_packet(hdr,
 				    HFI_PROP_UBWC_MAL_LENGTH,
 				    HFI_HOST_FLAGS_NONE,
@@ -156,7 +160,7 @@ void iris_hfi_gen2_packet_sys_init(struct iris_core *core, struct iris_hfi_heade
 				    &payload,
 				    sizeof(u32));
 
-	payload = core->iris_platform_data->ubwc_config->highest_bank_bit;
+	payload = ubwc->highest_bank_bit;
 	iris_hfi_gen2_create_packet(hdr,
 				    HFI_PROP_UBWC_HBB,
 				    HFI_HOST_FLAGS_NONE,
@@ -166,7 +170,7 @@ void iris_hfi_gen2_packet_sys_init(struct iris_core *core, struct iris_hfi_heade
 				    &payload,
 				    sizeof(u32));
 
-	payload = core->iris_platform_data->ubwc_config->bank_swzl_level;
+	payload = !!(qcom_ubwc_swizzle(ubwc) & UBWC_SWIZZLE_ENABLE_LVL1);
 	iris_hfi_gen2_create_packet(hdr,
 				    HFI_PROP_UBWC_BANK_SWZL_LEVEL1,
 				    HFI_HOST_FLAGS_NONE,
@@ -176,7 +180,7 @@ void iris_hfi_gen2_packet_sys_init(struct iris_core *core, struct iris_hfi_heade
 				    &payload,
 				    sizeof(u32));
 
-	payload = core->iris_platform_data->ubwc_config->bank_swz2_level;
+	payload = !!(qcom_ubwc_swizzle(ubwc) & UBWC_SWIZZLE_ENABLE_LVL2);
 	iris_hfi_gen2_create_packet(hdr,
 				    HFI_PROP_UBWC_BANK_SWZL_LEVEL2,
 				    HFI_HOST_FLAGS_NONE,
@@ -186,7 +190,7 @@ void iris_hfi_gen2_packet_sys_init(struct iris_core *core, struct iris_hfi_heade
 				    &payload,
 				    sizeof(u32));
 
-	payload = core->iris_platform_data->ubwc_config->bank_swz3_level;
+	payload = !!(qcom_ubwc_swizzle(ubwc) & UBWC_SWIZZLE_ENABLE_LVL3);
 	iris_hfi_gen2_create_packet(hdr,
 				    HFI_PROP_UBWC_BANK_SWZL_LEVEL3,
 				    HFI_HOST_FLAGS_NONE,
@@ -196,7 +200,7 @@ void iris_hfi_gen2_packet_sys_init(struct iris_core *core, struct iris_hfi_heade
 				    &payload,
 				    sizeof(u32));
 
-	payload = core->iris_platform_data->ubwc_config->bank_spreading;
+	payload = qcom_ubwc_bank_spread(ubwc);
 	iris_hfi_gen2_create_packet(hdr,
 				    HFI_PROP_UBWC_BANK_SPREADING,
 				    HFI_HOST_FLAGS_NONE,
diff --git a/drivers/media/platform/qcom/iris/iris_hfi_gen2_response.c b/drivers/media/platform/qcom/iris/iris_hfi_gen2_response.c
index 8e19f61bbbf9e..c350d231265e5 100644
--- a/drivers/media/platform/qcom/iris/iris_hfi_gen2_response.c
+++ b/drivers/media/platform/qcom/iris/iris_hfi_gen2_response.c
@@ -977,7 +977,7 @@ static void iris_hfi_gen2_flush_debug_queue(struct iris_core *core, u8 *packet)
 	}
 }
 
-static void iris_hfi_gen2_response_handler(struct iris_core *core)
+void iris_hfi_gen2_response_handler(struct iris_core *core)
 {
 	if (iris_vpu_watchdog(core, core->intr_status)) {
 		struct iris_hfi_packet pkt = {.type = HFI_SYS_ERROR_WD_TIMEOUT};
@@ -997,12 +997,3 @@ static void iris_hfi_gen2_response_handler(struct iris_core *core)
 
 	iris_hfi_gen2_flush_debug_queue(core, core->response_packet);
 }
-
-static const struct iris_hfi_response_ops iris_hfi_gen2_response_ops = {
-	.hfi_response_handler = iris_hfi_gen2_response_handler,
-};
-
-void iris_hfi_gen2_response_ops_init(struct iris_core *core)
-{
-	core->hfi_response_ops = &iris_hfi_gen2_response_ops;
-}
diff --git a/drivers/media/platform/qcom/iris/iris_instance.h b/drivers/media/platform/qcom/iris/iris_instance.h
index 16965150f427b..352af99699dd5 100644
--- a/drivers/media/platform/qcom/iris/iris_instance.h
+++ b/drivers/media/platform/qcom/iris/iris_instance.h
@@ -15,6 +15,8 @@
 #define DEFAULT_WIDTH 320
 #define DEFAULT_HEIGHT 240
 
+struct iris_hfi_session_ops;
+
 enum iris_fmt_type_out {
 	IRIS_FMT_H264,
 	IRIS_FMT_HEVC,
@@ -38,6 +40,7 @@ struct iris_fmt {
  * @list: used for attach an instance to the core
  * @core: pointer to core structure
  * @session_id: id of current video session
+ * @hfi_session_ops: iris HFI session ops
  * @ctx_q_lock: lock to serialize queues related ioctls
  * @lock: lock to seralise forward and reverse threads
  * @fh: reference of v4l2 file handler
@@ -80,6 +83,7 @@ struct iris_inst {
 	struct list_head		list;
 	struct iris_core		*core;
 	u32				session_id;
+	const struct iris_hfi_session_ops	*hfi_session_ops;
 	struct mutex			ctx_q_lock;/* lock to serialize queues related ioctls */
 	struct mutex			lock; /* lock to serialize forward and reverse threads */
 	struct v4l2_fh			fh;
diff --git a/drivers/media/platform/qcom/iris/iris_platform_common.h b/drivers/media/platform/qcom/iris/iris_platform_common.h
index 5a489917580eb..6a108173be354 100644
--- a/drivers/media/platform/qcom/iris/iris_platform_common.h
+++ b/drivers/media/platform/qcom/iris/iris_platform_common.h
@@ -12,7 +12,6 @@
 struct iris_core;
 struct iris_inst;
 
-#define IRIS_PAS_ID				9
 #define HW_RESPONSE_TIMEOUT_VALUE               (1000) /* milliseconds */
 #define AUTOSUSPEND_DELAY_VALUE			(HW_RESPONSE_TIMEOUT_VALUE + 500) /* milliseconds */
 
@@ -41,6 +40,9 @@ enum pipe_type {
 	PIPE_4 = 4,
 };
 
+extern const struct iris_firmware_data iris_hfi_gen1_data;
+extern const struct iris_firmware_data iris_hfi_gen2_data;
+
 extern const struct iris_platform_data qcs8300_data;
 extern const struct iris_platform_data sc7280_data;
 extern const struct iris_platform_data sm8250_data;
@@ -75,16 +77,6 @@ struct tz_cp_config {
 	u32 cp_nonpixel_size;
 };
 
-struct ubwc_config_data {
-	u32	max_channels;
-	u32	mal_length;
-	u32	highest_bank_bit;
-	u32	bank_swzl_level;
-	u32	bank_swz2_level;
-	u32	bank_swz3_level;
-	u32	bank_spreading;
-};
-
 struct platform_inst_caps {
 	u32 min_frame_width;
 	u32 max_frame_width;
@@ -212,50 +204,16 @@ enum platform_pm_domain_type {
 	IRIS_APV_HW_POWER_DOMAIN,
 };
 
-struct iris_platform_data {
-	void (*init_hfi_command_ops)(struct iris_core *core);
-	void (*init_hfi_response_ops)(struct iris_core *core);
-	struct iris_inst *(*get_instance)(void);
-	u32 (*get_vpu_buffer_size)(struct iris_inst *inst, enum iris_buffer_type buffer_type);
-	const struct vpu_ops *vpu_ops;
-	void (*set_preset_registers)(struct iris_core *core);
-	const struct icc_info *icc_tbl;
-	unsigned int icc_tbl_size;
-	const struct bw_info *bw_tbl_dec;
-	unsigned int bw_tbl_dec_size;
-	const char * const *pmdomain_tbl;
-	unsigned int pmdomain_tbl_size;
-	const char * const *opp_pd_tbl;
-	unsigned int opp_pd_tbl_size;
-	const struct platform_clk_data *clk_tbl;
-	const char * const *opp_clk_tbl;
-	unsigned int clk_tbl_size;
-	const char * const *clk_rst_tbl;
-	unsigned int clk_rst_tbl_size;
-	const char * const *controller_rst_tbl;
-	unsigned int controller_rst_tbl_size;
-	u64 dma_mask;
-	const char *fwname;
-	u32 pas_id;
-	struct iris_fmt *inst_iris_fmts;
-	u32 inst_iris_fmts_size;
-	struct platform_inst_caps *inst_caps;
+struct iris_firmware_data {
+	void (*init_hfi_ops)(struct iris_core *core);
+
+	u32 core_arch;
+
 	const struct platform_inst_fw_cap *inst_fw_caps_dec;
 	u32 inst_fw_caps_dec_size;
 	const struct platform_inst_fw_cap *inst_fw_caps_enc;
 	u32 inst_fw_caps_enc_size;
-	const struct tz_cp_config *tz_cp_config_data;
-	u32 tz_cp_config_data_size;
-	u32 core_arch;
-	u32 hw_response_timeout;
-	struct ubwc_config_data *ubwc_config;
-	u32 num_vpp_pipe;
-	bool no_aon;
-	u32 max_session_count;
-	/* max number of macroblocks per frame supported */
-	u32 max_core_mbpf;
-	/* max number of macroblocks per second supported */
-	u32 max_core_mbps;
+
 	const u32 *dec_input_config_params_default;
 	unsigned int dec_input_config_params_default_size;
 	const u32 *dec_input_config_params_hevc;
@@ -270,6 +228,7 @@ struct iris_platform_data {
 	unsigned int enc_input_config_params_size;
 	const u32 *enc_output_config_params;
 	unsigned int enc_output_config_params_size;
+
 	const u32 *dec_input_prop;
 	unsigned int dec_input_prop_size;
 	const u32 *dec_output_prop_avc;
@@ -280,6 +239,7 @@ struct iris_platform_data {
 	unsigned int dec_output_prop_vp9_size;
 	const u32 *dec_output_prop_av1;
 	unsigned int dec_output_prop_av1_size;
+
 	const u32 *dec_ip_int_buf_tbl;
 	unsigned int dec_ip_int_buf_tbl_size;
 	const u32 *dec_op_int_buf_tbl;
@@ -290,4 +250,48 @@ struct iris_platform_data {
 	unsigned int enc_op_int_buf_tbl_size;
 };
 
+struct iris_firmware_desc {
+	const struct iris_firmware_data *firmware_data;
+	u32 (*get_vpu_buffer_size)(struct iris_inst *inst, enum iris_buffer_type buffer_type);
+	const char *fwname;
+};
+
+struct iris_platform_data {
+	/*
+	 * XXX: replace with gen1 / gen2 pointers once we have platforms
+	 * supporting both firmware kinds.
+	 */
+	const struct iris_firmware_desc *firmware_desc;
+
+	const struct vpu_ops *vpu_ops;
+	const struct icc_info *icc_tbl;
+	unsigned int icc_tbl_size;
+	const struct bw_info *bw_tbl_dec;
+	unsigned int bw_tbl_dec_size;
+	const char * const *pmdomain_tbl;
+	unsigned int pmdomain_tbl_size;
+	const char * const *opp_pd_tbl;
+	unsigned int opp_pd_tbl_size;
+	const struct platform_clk_data *clk_tbl;
+	const char * const *opp_clk_tbl;
+	unsigned int clk_tbl_size;
+	const char * const *clk_rst_tbl;
+	unsigned int clk_rst_tbl_size;
+	const char * const *controller_rst_tbl;
+	unsigned int controller_rst_tbl_size;
+	u64 dma_mask;
+	struct iris_fmt *inst_iris_fmts;
+	u32 inst_iris_fmts_size;
+	struct platform_inst_caps *inst_caps;
+	const struct tz_cp_config *tz_cp_config_data;
+	u32 tz_cp_config_data_size;
+	u32 num_vpp_pipe;
+	bool no_aon;
+	u32 max_session_count;
+	/* max number of macroblocks per frame supported */
+	u32 max_core_mbpf;
+	/* max number of macroblocks per second supported */
+	u32 max_core_mbps;
+};
+
 #endif
diff --git a/drivers/media/platform/qcom/iris/iris_platform_gen1.c b/drivers/media/platform/qcom/iris/iris_platform_gen1.c
deleted file mode 100644
index df8e6bf9430ed..0000000000000
--- a/drivers/media/platform/qcom/iris/iris_platform_gen1.c
+++ /dev/null
@@ -1,448 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved.
- */
-
-#include "iris_core.h"
-#include "iris_ctrls.h"
-#include "iris_platform_common.h"
-#include "iris_resources.h"
-#include "iris_hfi_gen1.h"
-#include "iris_hfi_gen1_defines.h"
-#include "iris_vpu_buffer.h"
-#include "iris_vpu_common.h"
-#include "iris_instance.h"
-
-#include "iris_platform_sc7280.h"
-
-#define BITRATE_MIN		32000
-#define BITRATE_MAX		160000000
-#define BITRATE_PEAK_DEFAULT	(BITRATE_DEFAULT * 2)
-#define BITRATE_STEP		100
-
-static struct iris_fmt platform_fmts_sm8250_dec[] = {
-	[IRIS_FMT_H264] = {
-		.pixfmt = V4L2_PIX_FMT_H264,
-		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
-	},
-	[IRIS_FMT_HEVC] = {
-		.pixfmt = V4L2_PIX_FMT_HEVC,
-		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
-	},
-	[IRIS_FMT_VP9] = {
-		.pixfmt = V4L2_PIX_FMT_VP9,
-		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
-	},
-};
-
-static struct platform_inst_fw_cap inst_fw_cap_sm8250_dec[] = {
-	{
-		.cap_id = PIPE,
-		/* .max, .min and .value are set via platform data */
-		.step_or_mask = 1,
-		.hfi_id = HFI_PROPERTY_PARAM_WORK_ROUTE,
-		.set = iris_set_pipe,
-	},
-	{
-		.cap_id = STAGE,
-		.min = STAGE_1,
-		.max = STAGE_2,
-		.step_or_mask = 1,
-		.value = STAGE_2,
-		.hfi_id = HFI_PROPERTY_PARAM_WORK_MODE,
-		.set = iris_set_stage,
-	},
-};
-
-static const struct platform_inst_fw_cap inst_fw_cap_sm8250_enc[] = {
-	{
-		.cap_id = STAGE,
-		.min = STAGE_1,
-		.max = STAGE_2,
-		.step_or_mask = 1,
-		.value = STAGE_2,
-		.hfi_id = HFI_PROPERTY_PARAM_WORK_MODE,
-		.set = iris_set_stage,
-	},
-	{
-		.cap_id = PROFILE_H264,
-		.min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
-		.max = V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_STEREO_HIGH) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MULTIVIEW_HIGH),
-		.value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH,
-		.hfi_id = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_profile_level_gen1,
-	},
-	{
-		.cap_id = PROFILE_HEVC,
-		.min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
-		.max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10),
-		.value = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
-		.hfi_id = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_profile_level_gen1,
-	},
-	{
-		.cap_id = LEVEL_H264,
-		.min = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
-		.max = V4L2_MPEG_VIDEO_H264_LEVEL_5_1,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1B) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_3) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_1),
-		.value = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
-		.hfi_id = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_profile_level_gen1,
-	},
-	{
-		.cap_id = LEVEL_HEVC,
-		.min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
-		.max = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2),
-		.value = V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
-		.hfi_id = HFI_PROPERTY_PARAM_PROFILE_LEVEL_CURRENT,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_profile_level_gen1,
-	},
-	{
-		.cap_id = HEADER_MODE,
-		.min = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE,
-		.max = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE) |
-				BIT(V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME),
-		.value = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
-		.hfi_id = HFI_PROPERTY_CONFIG_VENC_SYNC_FRAME_SEQUENCE_HEADER,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_header_mode_gen1,
-	},
-	{
-		.cap_id = BITRATE,
-		.min = BITRATE_MIN,
-		.max = BITRATE_MAX,
-		.step_or_mask = BITRATE_STEP,
-		.value = BITRATE_DEFAULT,
-		.hfi_id = HFI_PROPERTY_CONFIG_VENC_TARGET_BITRATE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_bitrate,
-	},
-	{
-		.cap_id = BITRATE_MODE,
-		.min = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR,
-		.max = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_VBR) |
-				BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR),
-		.value = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR,
-		.hfi_id = HFI_PROPERTY_PARAM_VENC_RATE_CONTROL,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_bitrate_mode_gen1,
-	},
-	{
-		.cap_id = FRAME_SKIP_MODE,
-		.min = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED,
-		.max = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED) |
-				BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT),
-		.value = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-	},
-	{
-		.cap_id = FRAME_RC_ENABLE,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 1,
-	},
-	{
-		.cap_id = GOP_SIZE,
-		.min = 0,
-		.max = (1 << 16) - 1,
-		.step_or_mask = 1,
-		.value = 30,
-		.set = iris_set_u32
-	},
-	{
-		.cap_id = ENTROPY_MODE,
-		.min = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC,
-		.max = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC) |
-				BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC),
-		.value = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC,
-		.hfi_id = HFI_PROPERTY_PARAM_VENC_H264_ENTROPY_CONTROL,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_entropy_mode_gen1,
-	},
-	{
-		.cap_id = MIN_FRAME_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-		.hfi_id = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_qp_range,
-	},
-	{
-		.cap_id = MIN_FRAME_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP_HEVC,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-		.hfi_id = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_qp_range,
-	},
-	{
-		.cap_id = MAX_FRAME_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MAX_QP,
-		.hfi_id = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_qp_range,
-	},
-	{
-		.cap_id = MAX_FRAME_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP_HEVC,
-		.step_or_mask = 1,
-		.value = MAX_QP_HEVC,
-		.hfi_id = HFI_PROPERTY_PARAM_VENC_SESSION_QP_RANGE_V2,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_qp_range,
-	},
-};
-
-static struct platform_inst_caps platform_inst_cap_sm8250 = {
-	.min_frame_width = 128,
-	.max_frame_width = 8192,
-	.min_frame_height = 128,
-	.max_frame_height = 8192,
-	.max_mbpf = 138240,
-	.mb_cycles_vsp = 25,
-	.mb_cycles_vpp = 200,
-	.max_frame_rate = MAXIMUM_FPS,
-	.max_operating_rate = MAXIMUM_FPS,
-};
-
-static void iris_set_sm8250_preset_registers(struct iris_core *core)
-{
-	writel(0x0, core->reg_base + 0xB0088);
-}
-
-static const struct icc_info sm8250_icc_table[] = {
-	{ "cpu-cfg",    1000, 1000     },
-	{ "video-mem",  1000, 15000000 },
-};
-
-static const char * const sm8250_clk_reset_table[] = { "bus", "core" };
-
-static const struct bw_info sm8250_bw_table_dec[] = {
-	{ ((4096 * 2160) / 256) * 60, 2403000 },
-	{ ((4096 * 2160) / 256) * 30, 1224000 },
-	{ ((1920 * 1080) / 256) * 60,  812000 },
-	{ ((1920 * 1080) / 256) * 30,  416000 },
-};
-
-static const char * const sm8250_pmdomain_table[] = { "venus", "vcodec0" };
-
-static const char * const sm8250_opp_pd_table[] = { "mx" };
-
-static const struct platform_clk_data sm8250_clk_table[] = {
-	{IRIS_AXI_CLK,  "iface"        },
-	{IRIS_CTRL_CLK, "core"         },
-	{IRIS_HW_CLK,   "vcodec0_core" },
-};
-
-static const char * const sm8250_opp_clk_table[] = {
-	"vcodec0_core",
-	NULL,
-};
-
-static const struct tz_cp_config tz_cp_config_sm8250[] = {
-	{
-		.cp_start = 0,
-		.cp_size = 0x25800000,
-		.cp_nonpixel_start = 0x01000000,
-		.cp_nonpixel_size = 0x24800000,
-	},
-};
-
-static const u32 sm8250_vdec_input_config_param_default[] = {
-	HFI_PROPERTY_CONFIG_VIDEOCORES_USAGE,
-	HFI_PROPERTY_PARAM_UNCOMPRESSED_FORMAT_SELECT,
-	HFI_PROPERTY_PARAM_UNCOMPRESSED_PLANE_ACTUAL_CONSTRAINTS_INFO,
-	HFI_PROPERTY_PARAM_BUFFER_COUNT_ACTUAL,
-	HFI_PROPERTY_PARAM_VDEC_MULTI_STREAM,
-	HFI_PROPERTY_PARAM_FRAME_SIZE,
-	HFI_PROPERTY_PARAM_BUFFER_SIZE_ACTUAL,
-	HFI_PROPERTY_PARAM_BUFFER_ALLOC_MODE,
-};
-
-static const u32 sm8250_venc_input_config_param[] = {
-	HFI_PROPERTY_CONFIG_FRAME_RATE,
-	HFI_PROPERTY_PARAM_UNCOMPRESSED_PLANE_ACTUAL_INFO,
-	HFI_PROPERTY_PARAM_FRAME_SIZE,
-	HFI_PROPERTY_PARAM_UNCOMPRESSED_FORMAT_SELECT,
-	HFI_PROPERTY_PARAM_BUFFER_COUNT_ACTUAL,
-};
-
-static const u32 sm8250_dec_ip_int_buf_tbl[] = {
-	BUF_BIN,
-	BUF_SCRATCH_1,
-};
-
-static const u32 sm8250_dec_op_int_buf_tbl[] = {
-	BUF_DPB,
-};
-
-static const u32 sm8250_enc_ip_int_buf_tbl[] = {
-	BUF_BIN,
-	BUF_SCRATCH_1,
-	BUF_SCRATCH_2,
-};
-
-const struct iris_platform_data sm8250_data = {
-	.get_instance = iris_hfi_gen1_get_instance,
-	.init_hfi_command_ops = &iris_hfi_gen1_command_ops_init,
-	.init_hfi_response_ops = iris_hfi_gen1_response_ops_init,
-	.get_vpu_buffer_size = iris_vpu_buf_size,
-	.vpu_ops = &iris_vpu2_ops,
-	.set_preset_registers = iris_set_sm8250_preset_registers,
-	.icc_tbl = sm8250_icc_table,
-	.icc_tbl_size = ARRAY_SIZE(sm8250_icc_table),
-	.clk_rst_tbl = sm8250_clk_reset_table,
-	.clk_rst_tbl_size = ARRAY_SIZE(sm8250_clk_reset_table),
-	.bw_tbl_dec = sm8250_bw_table_dec,
-	.bw_tbl_dec_size = ARRAY_SIZE(sm8250_bw_table_dec),
-	.pmdomain_tbl = sm8250_pmdomain_table,
-	.pmdomain_tbl_size = ARRAY_SIZE(sm8250_pmdomain_table),
-	.opp_pd_tbl = sm8250_opp_pd_table,
-	.opp_pd_tbl_size = ARRAY_SIZE(sm8250_opp_pd_table),
-	.clk_tbl = sm8250_clk_table,
-	.clk_tbl_size = ARRAY_SIZE(sm8250_clk_table),
-	.opp_clk_tbl = sm8250_opp_clk_table,
-	/* Upper bound of DMA address range */
-	.dma_mask = 0xe0000000 - 1,
-	.fwname = "qcom/vpu-1.0/venus.mbn",
-	.pas_id = IRIS_PAS_ID,
-	.inst_iris_fmts = platform_fmts_sm8250_dec,
-	.inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8250_dec),
-	.inst_caps = &platform_inst_cap_sm8250,
-	.inst_fw_caps_dec = inst_fw_cap_sm8250_dec,
-	.inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8250_dec),
-	.inst_fw_caps_enc = inst_fw_cap_sm8250_enc,
-	.inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8250_enc),
-	.tz_cp_config_data = tz_cp_config_sm8250,
-	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8250),
-	.hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE,
-	.num_vpp_pipe = 4,
-	.max_session_count = 16,
-	.max_core_mbpf = NUM_MBS_8K,
-	.max_core_mbps = ((7680 * 4320) / 256) * 60,
-	.dec_input_config_params_default =
-		sm8250_vdec_input_config_param_default,
-	.dec_input_config_params_default_size =
-		ARRAY_SIZE(sm8250_vdec_input_config_param_default),
-	.enc_input_config_params = sm8250_venc_input_config_param,
-	.enc_input_config_params_size =
-		ARRAY_SIZE(sm8250_venc_input_config_param),
-
-	.dec_ip_int_buf_tbl = sm8250_dec_ip_int_buf_tbl,
-	.dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8250_dec_ip_int_buf_tbl),
-	.dec_op_int_buf_tbl = sm8250_dec_op_int_buf_tbl,
-	.dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8250_dec_op_int_buf_tbl),
-
-	.enc_ip_int_buf_tbl = sm8250_enc_ip_int_buf_tbl,
-	.enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8250_enc_ip_int_buf_tbl),
-};
-
-const struct iris_platform_data sc7280_data = {
-	.get_instance = iris_hfi_gen1_get_instance,
-	.init_hfi_command_ops = &iris_hfi_gen1_command_ops_init,
-	.init_hfi_response_ops = iris_hfi_gen1_response_ops_init,
-	.get_vpu_buffer_size = iris_vpu_buf_size,
-	.vpu_ops = &iris_vpu2_ops,
-	.set_preset_registers = iris_set_sm8250_preset_registers,
-	.icc_tbl = sm8250_icc_table,
-	.icc_tbl_size = ARRAY_SIZE(sm8250_icc_table),
-	.bw_tbl_dec = sc7280_bw_table_dec,
-	.bw_tbl_dec_size = ARRAY_SIZE(sc7280_bw_table_dec),
-	.pmdomain_tbl = sm8250_pmdomain_table,
-	.pmdomain_tbl_size = ARRAY_SIZE(sm8250_pmdomain_table),
-	.opp_pd_tbl = sc7280_opp_pd_table,
-	.opp_pd_tbl_size = ARRAY_SIZE(sc7280_opp_pd_table),
-	.clk_tbl = sc7280_clk_table,
-	.clk_tbl_size = ARRAY_SIZE(sc7280_clk_table),
-	.opp_clk_tbl = sc7280_opp_clk_table,
-	/* Upper bound of DMA address range */
-	.dma_mask = 0xe0000000 - 1,
-	.fwname = "qcom/vpu/vpu20_p1.mbn",
-	.pas_id = IRIS_PAS_ID,
-	.inst_iris_fmts = platform_fmts_sm8250_dec,
-	.inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8250_dec),
-	.inst_caps = &platform_inst_cap_sm8250,
-	.inst_fw_caps_dec = inst_fw_cap_sm8250_dec,
-	.inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8250_dec),
-	.inst_fw_caps_enc = inst_fw_cap_sm8250_enc,
-	.inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8250_enc),
-	.tz_cp_config_data = tz_cp_config_sm8250,
-	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8250),
-	.hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE,
-	.num_vpp_pipe = 1,
-	.no_aon = true,
-	.max_session_count = 16,
-	.max_core_mbpf = 4096 * 2176 / 256 * 2 + 1920 * 1088 / 256,
-	/* max spec for SC7280 is 4096x2176@60fps */
-	.max_core_mbps = 4096 * 2176 / 256 * 60,
-	.dec_input_config_params_default =
-		sm8250_vdec_input_config_param_default,
-	.dec_input_config_params_default_size =
-		ARRAY_SIZE(sm8250_vdec_input_config_param_default),
-	.enc_input_config_params = sm8250_venc_input_config_param,
-	.enc_input_config_params_size =
-		ARRAY_SIZE(sm8250_venc_input_config_param),
-
-	.dec_ip_int_buf_tbl = sm8250_dec_ip_int_buf_tbl,
-	.dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8250_dec_ip_int_buf_tbl),
-	.dec_op_int_buf_tbl = sm8250_dec_op_int_buf_tbl,
-	.dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8250_dec_op_int_buf_tbl),
-
-	.enc_ip_int_buf_tbl = sm8250_enc_ip_int_buf_tbl,
-	.enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8250_enc_ip_int_buf_tbl),
-};
diff --git a/drivers/media/platform/qcom/iris/iris_platform_gen2.c b/drivers/media/platform/qcom/iris/iris_platform_gen2.c
deleted file mode 100644
index 5da90d47f9c6e..0000000000000
--- a/drivers/media/platform/qcom/iris/iris_platform_gen2.c
+++ /dev/null
@@ -1,1319 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (c) 2022-2024 Qualcomm Innovation Center, Inc. All rights reserved.
- * Copyright (c) 2025 Linaro Ltd
- */
-
-#include "iris_core.h"
-#include "iris_ctrls.h"
-#include "iris_hfi_gen2.h"
-#include "iris_hfi_gen2_defines.h"
-#include "iris_platform_common.h"
-#include "iris_vpu_buffer.h"
-#include "iris_vpu_common.h"
-
-#include "iris_platform_qcs8300.h"
-#include "iris_platform_sm8650.h"
-#include "iris_platform_sm8750.h"
-
-#define VIDEO_ARCH_LX 1
-#define BITRATE_MAX				245000000
-
-static struct iris_fmt platform_fmts_sm8550_dec[] = {
-	[IRIS_FMT_H264] = {
-		.pixfmt = V4L2_PIX_FMT_H264,
-		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
-	},
-	[IRIS_FMT_HEVC] = {
-		.pixfmt = V4L2_PIX_FMT_HEVC,
-		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
-	},
-	[IRIS_FMT_VP9] = {
-		.pixfmt = V4L2_PIX_FMT_VP9,
-		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
-	},
-	[IRIS_FMT_AV1] = {
-		.pixfmt = V4L2_PIX_FMT_AV1,
-		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
-	},
-};
-
-static const struct platform_inst_fw_cap inst_fw_cap_sm8550_dec[] = {
-	{
-		.cap_id = PROFILE_H264,
-		.min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
-		.max = V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH),
-		.value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH,
-		.hfi_id = HFI_PROP_PROFILE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_u32_enum,
-	},
-	{
-		.cap_id = PROFILE_HEVC,
-		.min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
-		.max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE),
-		.value = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
-		.hfi_id = HFI_PROP_PROFILE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_u32_enum,
-	},
-	{
-		.cap_id = PROFILE_AV1,
-		.min = V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN,
-		.max = V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN),
-		.value = V4L2_MPEG_VIDEO_AV1_PROFILE_MAIN,
-		.hfi_id = HFI_PROP_PROFILE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_u32_enum,
-	},
-	{
-		.cap_id = PROFILE_VP9,
-		.min = V4L2_MPEG_VIDEO_VP9_PROFILE_0,
-		.max = V4L2_MPEG_VIDEO_VP9_PROFILE_2,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_VP9_PROFILE_0) |
-				BIT(V4L2_MPEG_VIDEO_VP9_PROFILE_2),
-		.value = V4L2_MPEG_VIDEO_VP9_PROFILE_0,
-		.hfi_id = HFI_PROP_PROFILE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_u32_enum,
-	},
-	{
-		.cap_id = LEVEL_H264,
-		.min = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
-		.max = V4L2_MPEG_VIDEO_H264_LEVEL_6_2,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1B) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_3) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_2),
-		.value = V4L2_MPEG_VIDEO_H264_LEVEL_6_1,
-		.hfi_id = HFI_PROP_LEVEL,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_u32_enum,
-	},
-	{
-		.cap_id = LEVEL_HEVC,
-		.min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
-		.max = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2),
-		.value = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1,
-		.hfi_id = HFI_PROP_LEVEL,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_u32_enum,
-	},
-	{
-		.cap_id = LEVEL_VP9,
-		.min = V4L2_MPEG_VIDEO_VP9_LEVEL_1_0,
-		.max = V4L2_MPEG_VIDEO_VP9_LEVEL_6_0,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_1_0) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_1_1) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_2_0) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_2_1) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_3_0) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_3_1) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_4_0) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_4_1) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_5_0) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_5_1) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_5_2) |
-				BIT(V4L2_MPEG_VIDEO_VP9_LEVEL_6_0),
-		.value = V4L2_MPEG_VIDEO_VP9_LEVEL_6_0,
-		.hfi_id = HFI_PROP_LEVEL,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_u32_enum,
-	},
-	{
-		.cap_id = LEVEL_AV1,
-		.min = V4L2_MPEG_VIDEO_AV1_LEVEL_2_0,
-		.max = V4L2_MPEG_VIDEO_AV1_LEVEL_6_1,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_0) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_1) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_2) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_2_3) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_0) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_1) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_2) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_3_3) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_0) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_1) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_2) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_4_3) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_0) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_1) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_2) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_5_3) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_6_0) |
-				BIT(V4L2_MPEG_VIDEO_AV1_LEVEL_6_1),
-		.value = V4L2_MPEG_VIDEO_AV1_LEVEL_6_1,
-		.hfi_id = HFI_PROP_LEVEL,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_u32_enum,
-	},
-	{
-		.cap_id = TIER,
-		.min = V4L2_MPEG_VIDEO_HEVC_TIER_MAIN,
-		.max = V4L2_MPEG_VIDEO_HEVC_TIER_HIGH,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_TIER_MAIN) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_TIER_HIGH),
-		.value = V4L2_MPEG_VIDEO_HEVC_TIER_HIGH,
-		.hfi_id = HFI_PROP_TIER,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_u32_enum,
-	},
-	{
-		.cap_id = TIER_AV1,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 0,
-		.hfi_id = HFI_PROP_TIER,
-		.flags = CAP_FLAG_INPUT_PORT,
-		.set = iris_set_u32,
-	},
-	{
-		.cap_id = DRAP,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 0,
-		.hfi_id = HFI_PROP_AV1_DRAP_CONFIG,
-		.flags = CAP_FLAG_INPUT_PORT,
-		.set = iris_set_u32,
-	},
-	{
-		.cap_id = FILM_GRAIN,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 0,
-		.hfi_id = HFI_PROP_AV1_FILM_GRAIN_PRESENT,
-		.flags = CAP_FLAG_VOLATILE,
-	},
-	{
-		.cap_id = SUPER_BLOCK,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 0,
-		.hfi_id = HFI_PROP_AV1_SUPER_BLOCK_ENABLED,
-	},
-	{
-		.cap_id = ENH_LAYER_COUNT,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 0,
-		.hfi_id = HFI_PROP_AV1_OP_POINT,
-		.flags = CAP_FLAG_INPUT_PORT,
-		.set = iris_set_u32,
-	},
-	{
-		.cap_id = INPUT_BUF_HOST_MAX_COUNT,
-		.min = DEFAULT_MAX_HOST_BUF_COUNT,
-		.max = DEFAULT_MAX_HOST_BURST_BUF_COUNT,
-		.step_or_mask = 1,
-		.value = DEFAULT_MAX_HOST_BUF_COUNT,
-		.hfi_id = HFI_PROP_BUFFER_HOST_MAX_COUNT,
-		.flags = CAP_FLAG_INPUT_PORT,
-		.set = iris_set_u32,
-	},
-	{
-		.cap_id = STAGE,
-		.min = STAGE_1,
-		.max = STAGE_2,
-		.step_or_mask = 1,
-		.value = STAGE_2,
-		.hfi_id = HFI_PROP_STAGE,
-		.set = iris_set_stage,
-	},
-	{
-		.cap_id = PIPE,
-		/* .max, .min and .value are set via platform data */
-		.step_or_mask = 1,
-		.hfi_id = HFI_PROP_PIPE,
-		.set = iris_set_pipe,
-	},
-	{
-		.cap_id = POC,
-		.min = 0,
-		.max = 2,
-		.step_or_mask = 1,
-		.value = 1,
-		.hfi_id = HFI_PROP_PIC_ORDER_CNT_TYPE,
-	},
-	{
-		.cap_id = CODED_FRAMES,
-		.min = CODED_FRAMES_PROGRESSIVE,
-		.max = CODED_FRAMES_PROGRESSIVE,
-		.step_or_mask = 0,
-		.value = CODED_FRAMES_PROGRESSIVE,
-		.hfi_id = HFI_PROP_CODED_FRAMES,
-	},
-	{
-		.cap_id = BIT_DEPTH,
-		.min = BIT_DEPTH_8,
-		.max = BIT_DEPTH_8,
-		.step_or_mask = 1,
-		.value = BIT_DEPTH_8,
-		.hfi_id = HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
-	},
-	{
-		.cap_id = RAP_FRAME,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 1,
-		.hfi_id = HFI_PROP_DEC_START_FROM_RAP_FRAME,
-		.flags = CAP_FLAG_INPUT_PORT,
-		.set = iris_set_u32,
-	},
-};
-
-static const struct platform_inst_fw_cap inst_fw_cap_sm8550_enc[] = {
-	{
-		.cap_id = PROFILE_H264,
-		.min = V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE,
-		.max = V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_HIGH) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_CONSTRAINED_BASELINE) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_MAIN) |
-				BIT(V4L2_MPEG_VIDEO_H264_PROFILE_HIGH),
-		.value = V4L2_MPEG_VIDEO_H264_PROFILE_HIGH,
-		.hfi_id = HFI_PROP_PROFILE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_profile,
-	},
-	{
-		.cap_id = PROFILE_HEVC,
-		.min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
-		.max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_STILL_PICTURE) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10),
-		.value = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN,
-		.hfi_id = HFI_PROP_PROFILE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_profile,
-	},
-	{
-		.cap_id = LEVEL_H264,
-		.min = V4L2_MPEG_VIDEO_H264_LEVEL_1_0,
-		.max = V4L2_MPEG_VIDEO_H264_LEVEL_6_0,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1B) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_1_3) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_2_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_3_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_4_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_0) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_1) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_5_2) |
-				BIT(V4L2_MPEG_VIDEO_H264_LEVEL_6_0),
-		.value = V4L2_MPEG_VIDEO_H264_LEVEL_5_0,
-		.hfi_id = HFI_PROP_LEVEL,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_level,
-	},
-	{
-		.cap_id = LEVEL_HEVC,
-		.min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1,
-		.max = V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_2_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_3_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_4_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_5_2) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_1) |
-				BIT(V4L2_MPEG_VIDEO_HEVC_LEVEL_6_2),
-		.value = V4L2_MPEG_VIDEO_HEVC_LEVEL_5,
-		.hfi_id = HFI_PROP_LEVEL,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_level,
-	},
-	{
-		.cap_id = STAGE,
-		.min = STAGE_1,
-		.max = STAGE_2,
-		.step_or_mask = 1,
-		.value = STAGE_2,
-		.hfi_id = HFI_PROP_STAGE,
-		.set = iris_set_stage,
-	},
-	{
-		.cap_id = HEADER_MODE,
-		.min = V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE,
-		.max = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_HEADER_MODE_SEPARATE) |
-				BIT(V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME),
-		.value = V4L2_MPEG_VIDEO_HEADER_MODE_JOINED_WITH_1ST_FRAME,
-		.hfi_id = HFI_PROP_SEQ_HEADER_MODE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_header_mode_gen2,
-	},
-	{
-		.cap_id = PREPEND_SPSPPS_TO_IDR,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 0,
-	},
-	{
-		.cap_id = BITRATE,
-		.min = 1,
-		.max = BITRATE_MAX,
-		.step_or_mask = 1,
-		.value = BITRATE_DEFAULT,
-		.hfi_id = HFI_PROP_TOTAL_BITRATE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_bitrate,
-	},
-	{
-		.cap_id = BITRATE_PEAK,
-		.min = 1,
-		.max = BITRATE_MAX,
-		.step_or_mask = 1,
-		.value = BITRATE_DEFAULT,
-		.hfi_id = HFI_PROP_TOTAL_PEAK_BITRATE,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_peak_bitrate,
-	},
-	{
-		.cap_id = BITRATE_MODE,
-		.min = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR,
-		.max = V4L2_MPEG_VIDEO_BITRATE_MODE_CBR,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_VBR) |
-				BIT(V4L2_MPEG_VIDEO_BITRATE_MODE_CBR),
-		.value = V4L2_MPEG_VIDEO_BITRATE_MODE_VBR,
-		.hfi_id = HFI_PROP_RATE_CONTROL,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_bitrate_mode_gen2,
-	},
-	{
-		.cap_id = FRAME_SKIP_MODE,
-		.min = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED,
-		.max = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED) |
-				BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_LEVEL_LIMIT) |
-				BIT(V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_BUF_LIMIT),
-		.value = V4L2_MPEG_VIDEO_FRAME_SKIP_MODE_DISABLED,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-	},
-	{
-		.cap_id = FRAME_RC_ENABLE,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 1,
-	},
-	{
-		.cap_id = GOP_SIZE,
-		.min = 0,
-		.max = INT_MAX,
-		.step_or_mask = 1,
-		.value = 2 * DEFAULT_FPS - 1,
-		.hfi_id = HFI_PROP_MAX_GOP_FRAMES,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_u32,
-	},
-	{
-		.cap_id = ENTROPY_MODE,
-		.min = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC,
-		.max = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC,
-		.step_or_mask = BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC) |
-				BIT(V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC),
-		.value = V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CABAC,
-		.hfi_id = HFI_PROP_CABAC_SESSION,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-		.set = iris_set_entropy_mode_gen2,
-	},
-	{
-		.cap_id = MIN_FRAME_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-		.hfi_id = HFI_PROP_MIN_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_min_qp,
-	},
-	{
-		.cap_id = MIN_FRAME_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-		.hfi_id = HFI_PROP_MIN_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_min_qp,
-	},
-	{
-		.cap_id = MAX_FRAME_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MAX_QP,
-		.hfi_id = HFI_PROP_MAX_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_max_qp,
-	},
-	{
-		.cap_id = MAX_FRAME_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MAX_QP,
-		.hfi_id = HFI_PROP_MAX_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_max_qp,
-	},
-	{
-		.cap_id = I_FRAME_MIN_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-	},
-	{
-		.cap_id = I_FRAME_MIN_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-	},
-	{
-		.cap_id = P_FRAME_MIN_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-	},
-	{
-		.cap_id = P_FRAME_MIN_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-	},
-	{
-		.cap_id = B_FRAME_MIN_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-	},
-	{
-		.cap_id = B_FRAME_MIN_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MIN_QP_8BIT,
-	},
-	{
-		.cap_id = I_FRAME_MAX_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MAX_QP,
-	},
-	{
-		.cap_id = I_FRAME_MAX_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MAX_QP,
-	},
-	{
-		.cap_id = P_FRAME_MAX_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MAX_QP,
-	},
-	{
-		.cap_id = P_FRAME_MAX_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MAX_QP,
-	},
-	{
-		.cap_id = B_FRAME_MAX_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MAX_QP,
-	},
-	{
-		.cap_id = B_FRAME_MAX_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = MAX_QP,
-	},
-	{
-		.cap_id = I_FRAME_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = DEFAULT_QP,
-		.hfi_id = HFI_PROP_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_frame_qp,
-	},
-	{
-		.cap_id = I_FRAME_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = DEFAULT_QP,
-		.hfi_id = HFI_PROP_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_frame_qp,
-	},
-	{
-		.cap_id = P_FRAME_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = DEFAULT_QP,
-		.hfi_id = HFI_PROP_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_frame_qp,
-	},
-	{
-		.cap_id = P_FRAME_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = DEFAULT_QP,
-		.hfi_id = HFI_PROP_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_frame_qp,
-	},
-	{
-		.cap_id = B_FRAME_QP_H264,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = DEFAULT_QP,
-		.hfi_id = HFI_PROP_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_frame_qp,
-	},
-	{
-		.cap_id = B_FRAME_QP_HEVC,
-		.min = MIN_QP_8BIT,
-		.max = MAX_QP,
-		.step_or_mask = 1,
-		.value = DEFAULT_QP,
-		.hfi_id = HFI_PROP_QP_PACKED,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_frame_qp,
-	},
-	{
-		.cap_id = INPUT_BUF_HOST_MAX_COUNT,
-		.min = DEFAULT_MAX_HOST_BUF_COUNT,
-		.max = DEFAULT_MAX_HOST_BURST_BUF_COUNT,
-		.step_or_mask = 1,
-		.value = DEFAULT_MAX_HOST_BUF_COUNT,
-		.hfi_id = HFI_PROP_BUFFER_HOST_MAX_COUNT,
-		.flags = CAP_FLAG_INPUT_PORT,
-		.set = iris_set_u32,
-	},
-	{
-		.cap_id = OUTPUT_BUF_HOST_MAX_COUNT,
-		.min = DEFAULT_MAX_HOST_BUF_COUNT,
-		.max = DEFAULT_MAX_HOST_BURST_BUF_COUNT,
-		.step_or_mask = 1,
-		.value = DEFAULT_MAX_HOST_BUF_COUNT,
-		.hfi_id = HFI_PROP_BUFFER_HOST_MAX_COUNT,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_u32,
-	},
-	{
-		.cap_id = ROTATION,
-		.min = 0,
-		.max = 270,
-		.step_or_mask = 90,
-		.value = 0,
-		.hfi_id = HFI_PROP_ROTATION,
-		.flags = CAP_FLAG_OUTPUT_PORT,
-		.set = iris_set_rotation,
-	},
-	{
-		.cap_id = HFLIP,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 0,
-		.hfi_id = HFI_PROP_FLIP,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_flip,
-	},
-	{
-		.cap_id = VFLIP,
-		.min = 0,
-		.max = 1,
-		.step_or_mask = 1,
-		.value = 0,
-		.hfi_id = HFI_PROP_FLIP,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_INPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_flip,
-	},
-	{
-		.cap_id = IR_TYPE,
-		.min = V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM,
-		.max = V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_CYCLIC,
-		.step_or_mask = BIT(V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM) |
-			BIT(V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_CYCLIC),
-		.value = V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_RANDOM,
-		.flags = CAP_FLAG_OUTPUT_PORT | CAP_FLAG_MENU,
-	},
-	{
-		.cap_id = IR_PERIOD,
-		.min = 0,
-		.max = INT_MAX,
-		.step_or_mask = 1,
-		.value = 0,
-		.flags = CAP_FLAG_OUTPUT_PORT |
-			CAP_FLAG_DYNAMIC_ALLOWED,
-		.set = iris_set_ir_period,
-	},
-};
-
-static struct platform_inst_caps platform_inst_cap_sm8550 = {
-	.min_frame_width = 96,
-	.max_frame_width = 8192,
-	.min_frame_height = 96,
-	.max_frame_height = 8192,
-	.max_mbpf = (8192 * 4352) / 256,
-	.mb_cycles_vpp = 200,
-	.mb_cycles_fw = 489583,
-	.mb_cycles_fw_vpp = 66234,
-	.num_comv = 0,
-	.max_frame_rate = MAXIMUM_FPS,
-	.max_operating_rate = MAXIMUM_FPS,
-};
-
-static void iris_set_sm8550_preset_registers(struct iris_core *core)
-{
-	writel(0x0, core->reg_base + 0xB0088);
-}
-
-static const struct icc_info sm8550_icc_table[] = {
-	{ "cpu-cfg",    1000, 1000     },
-	{ "video-mem",  1000, 15000000 },
-};
-
-static const char * const sm8550_clk_reset_table[] = { "bus" };
-
-static const struct bw_info sm8550_bw_table_dec[] = {
-	{ ((4096 * 2160) / 256) * 60, 1608000 },
-	{ ((4096 * 2160) / 256) * 30,  826000 },
-	{ ((1920 * 1080) / 256) * 60,  567000 },
-	{ ((1920 * 1080) / 256) * 30,  294000 },
-};
-
-static const char * const sm8550_pmdomain_table[] = { "venus", "vcodec0" };
-
-static const char * const sm8550_opp_pd_table[] = { "mxc", "mmcx" };
-
-static const struct platform_clk_data sm8550_clk_table[] = {
-	{IRIS_AXI_CLK,  "iface"        },
-	{IRIS_CTRL_CLK, "core"         },
-	{IRIS_HW_CLK,   "vcodec0_core" },
-};
-
-static const char * const sm8550_opp_clk_table[] = {
-	"vcodec0_core",
-	NULL,
-};
-
-static struct ubwc_config_data ubwc_config_sm8550 = {
-	.max_channels = 8,
-	.mal_length = 32,
-	.highest_bank_bit = 16,
-	.bank_swzl_level = 0,
-	.bank_swz2_level = 1,
-	.bank_swz3_level = 1,
-	.bank_spreading = 1,
-};
-
-static const struct tz_cp_config tz_cp_config_sm8550[] = {
-	{
-		.cp_start = 0,
-		.cp_size = 0x25800000,
-		.cp_nonpixel_start = 0x01000000,
-		.cp_nonpixel_size = 0x24800000,
-	},
-};
-
-static const u32 sm8550_vdec_input_config_params_default[] = {
-	HFI_PROP_BITSTREAM_RESOLUTION,
-	HFI_PROP_CROP_OFFSETS,
-	HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
-	HFI_PROP_CODED_FRAMES,
-	HFI_PROP_BUFFER_FW_MIN_OUTPUT_COUNT,
-	HFI_PROP_PIC_ORDER_CNT_TYPE,
-	HFI_PROP_PROFILE,
-	HFI_PROP_LEVEL,
-	HFI_PROP_SIGNAL_COLOR_INFO,
-};
-
-static const u32 sm8550_vdec_input_config_param_hevc[] = {
-	HFI_PROP_BITSTREAM_RESOLUTION,
-	HFI_PROP_CROP_OFFSETS,
-	HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
-	HFI_PROP_BUFFER_FW_MIN_OUTPUT_COUNT,
-	HFI_PROP_PROFILE,
-	HFI_PROP_LEVEL,
-	HFI_PROP_TIER,
-	HFI_PROP_SIGNAL_COLOR_INFO,
-};
-
-static const u32 sm8550_vdec_input_config_param_vp9[] = {
-	HFI_PROP_BITSTREAM_RESOLUTION,
-	HFI_PROP_CROP_OFFSETS,
-	HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
-	HFI_PROP_BUFFER_FW_MIN_OUTPUT_COUNT,
-	HFI_PROP_PROFILE,
-	HFI_PROP_LEVEL,
-};
-
-static const u32 sm8550_vdec_input_config_param_av1[] = {
-	HFI_PROP_BITSTREAM_RESOLUTION,
-	HFI_PROP_CROP_OFFSETS,
-	HFI_PROP_LUMA_CHROMA_BIT_DEPTH,
-	HFI_PROP_BUFFER_FW_MIN_OUTPUT_COUNT,
-	HFI_PROP_PROFILE,
-	HFI_PROP_LEVEL,
-	HFI_PROP_TIER,
-	HFI_PROP_AV1_FILM_GRAIN_PRESENT,
-	HFI_PROP_AV1_SUPER_BLOCK_ENABLED,
-	HFI_PROP_SIGNAL_COLOR_INFO,
-};
-
-static const u32 sm8550_venc_input_config_params[] = {
-	HFI_PROP_COLOR_FORMAT,
-	HFI_PROP_RAW_RESOLUTION,
-	HFI_PROP_CROP_OFFSETS,
-	HFI_PROP_LINEAR_STRIDE_SCANLINE,
-	HFI_PROP_SIGNAL_COLOR_INFO,
-};
-
-static const u32 sm8550_vdec_output_config_params[] = {
-	HFI_PROP_OPB_ENABLE,
-	HFI_PROP_COLOR_FORMAT,
-	HFI_PROP_LINEAR_STRIDE_SCANLINE,
-};
-
-static const u32 sm8550_venc_output_config_params[] = {
-	HFI_PROP_BITSTREAM_RESOLUTION,
-	HFI_PROP_CROP_OFFSETS,
-	HFI_PROP_FRAME_RATE,
-};
-
-static const u32 sm8550_vdec_subscribe_input_properties[] = {
-	HFI_PROP_NO_OUTPUT,
-};
-
-static const u32 sm8550_vdec_subscribe_output_properties_avc[] = {
-	HFI_PROP_PICTURE_TYPE,
-	HFI_PROP_CABAC_SESSION,
-};
-
-static const u32 sm8550_vdec_subscribe_output_properties_hevc[] = {
-	HFI_PROP_PICTURE_TYPE,
-};
-
-static const u32 sm8550_vdec_subscribe_output_properties_vp9[] = {
-	HFI_PROP_PICTURE_TYPE,
-};
-
-static const u32 sm8550_vdec_subscribe_output_properties_av1[] = {
-	HFI_PROP_PICTURE_TYPE,
-	HFI_PROP_WORST_COMPRESSION_RATIO,
-	HFI_PROP_WORST_COMPLEXITY_FACTOR,
-};
-
-static const u32 sm8550_dec_ip_int_buf_tbl[] = {
-	BUF_BIN,
-	BUF_COMV,
-	BUF_NON_COMV,
-	BUF_LINE,
-	BUF_PARTIAL,
-};
-
-static const u32 sm8550_dec_op_int_buf_tbl[] = {
-	BUF_DPB,
-};
-
-static const u32 sm8550_enc_ip_int_buf_tbl[] = {
-	BUF_VPSS,
-};
-
-static const u32 sm8550_enc_op_int_buf_tbl[] = {
-	BUF_BIN,
-	BUF_COMV,
-	BUF_NON_COMV,
-	BUF_LINE,
-	BUF_SCRATCH_2,
-};
-
-const struct iris_platform_data sm8550_data = {
-	.get_instance = iris_hfi_gen2_get_instance,
-	.init_hfi_command_ops = iris_hfi_gen2_command_ops_init,
-	.init_hfi_response_ops = iris_hfi_gen2_response_ops_init,
-	.get_vpu_buffer_size = iris_vpu_buf_size,
-	.vpu_ops = &iris_vpu3_ops,
-	.set_preset_registers = iris_set_sm8550_preset_registers,
-	.icc_tbl = sm8550_icc_table,
-	.icc_tbl_size = ARRAY_SIZE(sm8550_icc_table),
-	.clk_rst_tbl = sm8550_clk_reset_table,
-	.clk_rst_tbl_size = ARRAY_SIZE(sm8550_clk_reset_table),
-	.bw_tbl_dec = sm8550_bw_table_dec,
-	.bw_tbl_dec_size = ARRAY_SIZE(sm8550_bw_table_dec),
-	.pmdomain_tbl = sm8550_pmdomain_table,
-	.pmdomain_tbl_size = ARRAY_SIZE(sm8550_pmdomain_table),
-	.opp_pd_tbl = sm8550_opp_pd_table,
-	.opp_pd_tbl_size = ARRAY_SIZE(sm8550_opp_pd_table),
-	.clk_tbl = sm8550_clk_table,
-	.clk_tbl_size = ARRAY_SIZE(sm8550_clk_table),
-	.opp_clk_tbl = sm8550_opp_clk_table,
-	/* Upper bound of DMA address range */
-	.dma_mask = 0xe0000000 - 1,
-	.fwname = "qcom/vpu/vpu30_p4.mbn",
-	.pas_id = IRIS_PAS_ID,
-	.inst_iris_fmts = platform_fmts_sm8550_dec,
-	.inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8550_dec),
-	.inst_caps = &platform_inst_cap_sm8550,
-	.inst_fw_caps_dec = inst_fw_cap_sm8550_dec,
-	.inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec),
-	.inst_fw_caps_enc = inst_fw_cap_sm8550_enc,
-	.inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc),
-	.tz_cp_config_data = tz_cp_config_sm8550,
-	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8550),
-	.core_arch = VIDEO_ARCH_LX,
-	.hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE,
-	.ubwc_config = &ubwc_config_sm8550,
-	.num_vpp_pipe = 4,
-	.max_session_count = 16,
-	.max_core_mbpf = NUM_MBS_8K * 2,
-	.max_core_mbps = ((7680 * 4320) / 256) * 60,
-	.dec_input_config_params_default =
-		sm8550_vdec_input_config_params_default,
-	.dec_input_config_params_default_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_params_default),
-	.dec_input_config_params_hevc =
-		sm8550_vdec_input_config_param_hevc,
-	.dec_input_config_params_hevc_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_hevc),
-	.dec_input_config_params_vp9 =
-		sm8550_vdec_input_config_param_vp9,
-	.dec_input_config_params_vp9_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_vp9),
-	.dec_input_config_params_av1 =
-		sm8550_vdec_input_config_param_av1,
-	.dec_input_config_params_av1_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_av1),
-	.dec_output_config_params =
-		sm8550_vdec_output_config_params,
-	.dec_output_config_params_size =
-		ARRAY_SIZE(sm8550_vdec_output_config_params),
-
-	.enc_input_config_params =
-		sm8550_venc_input_config_params,
-	.enc_input_config_params_size =
-		ARRAY_SIZE(sm8550_venc_input_config_params),
-	.enc_output_config_params =
-		sm8550_venc_output_config_params,
-	.enc_output_config_params_size =
-		ARRAY_SIZE(sm8550_venc_output_config_params),
-
-	.dec_input_prop = sm8550_vdec_subscribe_input_properties,
-	.dec_input_prop_size = ARRAY_SIZE(sm8550_vdec_subscribe_input_properties),
-	.dec_output_prop_avc = sm8550_vdec_subscribe_output_properties_avc,
-	.dec_output_prop_avc_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_avc),
-	.dec_output_prop_hevc = sm8550_vdec_subscribe_output_properties_hevc,
-	.dec_output_prop_hevc_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_hevc),
-	.dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9,
-	.dec_output_prop_vp9_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9),
-	.dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1,
-	.dec_output_prop_av1_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1),
-
-	.dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl,
-	.dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl),
-	.dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl,
-	.dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl),
-
-	.enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl,
-	.enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl),
-	.enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl,
-	.enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl),
-};
-
-/*
- * Shares most of SM8550 data except:
- * - vpu_ops to iris_vpu33_ops
- * - clk_rst_tbl to sm8650_clk_reset_table
- * - controller_rst_tbl to sm8650_controller_reset_table
- * - fwname to "qcom/vpu/vpu33_p4.mbn"
- */
-const struct iris_platform_data sm8650_data = {
-	.get_instance = iris_hfi_gen2_get_instance,
-	.init_hfi_command_ops = iris_hfi_gen2_command_ops_init,
-	.init_hfi_response_ops = iris_hfi_gen2_response_ops_init,
-	.get_vpu_buffer_size = iris_vpu33_buf_size,
-	.vpu_ops = &iris_vpu33_ops,
-	.set_preset_registers = iris_set_sm8550_preset_registers,
-	.icc_tbl = sm8550_icc_table,
-	.icc_tbl_size = ARRAY_SIZE(sm8550_icc_table),
-	.clk_rst_tbl = sm8650_clk_reset_table,
-	.clk_rst_tbl_size = ARRAY_SIZE(sm8650_clk_reset_table),
-	.controller_rst_tbl = sm8650_controller_reset_table,
-	.controller_rst_tbl_size = ARRAY_SIZE(sm8650_controller_reset_table),
-	.bw_tbl_dec = sm8550_bw_table_dec,
-	.bw_tbl_dec_size = ARRAY_SIZE(sm8550_bw_table_dec),
-	.pmdomain_tbl = sm8550_pmdomain_table,
-	.pmdomain_tbl_size = ARRAY_SIZE(sm8550_pmdomain_table),
-	.opp_pd_tbl = sm8550_opp_pd_table,
-	.opp_pd_tbl_size = ARRAY_SIZE(sm8550_opp_pd_table),
-	.clk_tbl = sm8550_clk_table,
-	.clk_tbl_size = ARRAY_SIZE(sm8550_clk_table),
-	.opp_clk_tbl = sm8550_opp_clk_table,
-	/* Upper bound of DMA address range */
-	.dma_mask = 0xe0000000 - 1,
-	.fwname = "qcom/vpu/vpu33_p4.mbn",
-	.pas_id = IRIS_PAS_ID,
-	.inst_iris_fmts = platform_fmts_sm8550_dec,
-	.inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8550_dec),
-	.inst_caps = &platform_inst_cap_sm8550,
-	.inst_fw_caps_dec = inst_fw_cap_sm8550_dec,
-	.inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec),
-	.inst_fw_caps_enc = inst_fw_cap_sm8550_enc,
-	.inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc),
-	.tz_cp_config_data = tz_cp_config_sm8550,
-	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8550),
-	.core_arch = VIDEO_ARCH_LX,
-	.hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE,
-	.ubwc_config = &ubwc_config_sm8550,
-	.num_vpp_pipe = 4,
-	.max_session_count = 16,
-	.max_core_mbpf = NUM_MBS_8K * 2,
-	.max_core_mbps = ((7680 * 4320) / 256) * 60,
-	.dec_input_config_params_default =
-		sm8550_vdec_input_config_params_default,
-	.dec_input_config_params_default_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_params_default),
-	.dec_input_config_params_hevc =
-		sm8550_vdec_input_config_param_hevc,
-	.dec_input_config_params_hevc_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_hevc),
-	.dec_input_config_params_vp9 =
-		sm8550_vdec_input_config_param_vp9,
-	.dec_input_config_params_vp9_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_vp9),
-	.dec_input_config_params_av1 =
-		sm8550_vdec_input_config_param_av1,
-	.dec_input_config_params_av1_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_av1),
-	.dec_output_config_params =
-		sm8550_vdec_output_config_params,
-	.dec_output_config_params_size =
-		ARRAY_SIZE(sm8550_vdec_output_config_params),
-
-	.enc_input_config_params =
-		sm8550_venc_input_config_params,
-	.enc_input_config_params_size =
-		ARRAY_SIZE(sm8550_venc_input_config_params),
-	.enc_output_config_params =
-		sm8550_venc_output_config_params,
-	.enc_output_config_params_size =
-		ARRAY_SIZE(sm8550_venc_output_config_params),
-
-	.dec_input_prop = sm8550_vdec_subscribe_input_properties,
-	.dec_input_prop_size = ARRAY_SIZE(sm8550_vdec_subscribe_input_properties),
-	.dec_output_prop_avc = sm8550_vdec_subscribe_output_properties_avc,
-	.dec_output_prop_avc_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_avc),
-	.dec_output_prop_hevc = sm8550_vdec_subscribe_output_properties_hevc,
-	.dec_output_prop_hevc_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_hevc),
-	.dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9,
-	.dec_output_prop_vp9_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9),
-	.dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1,
-	.dec_output_prop_av1_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1),
-
-	.dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl,
-	.dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl),
-	.dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl,
-	.dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl),
-
-	.enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl,
-	.enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl),
-	.enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl,
-	.enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl),
-};
-
-const struct iris_platform_data sm8750_data = {
-	.get_instance = iris_hfi_gen2_get_instance,
-	.init_hfi_command_ops = iris_hfi_gen2_command_ops_init,
-	.init_hfi_response_ops = iris_hfi_gen2_response_ops_init,
-	.get_vpu_buffer_size = iris_vpu33_buf_size,
-	.vpu_ops = &iris_vpu35_ops,
-	.set_preset_registers = iris_set_sm8550_preset_registers,
-	.icc_tbl = sm8550_icc_table,
-	.icc_tbl_size = ARRAY_SIZE(sm8550_icc_table),
-	.clk_rst_tbl = sm8750_clk_reset_table,
-	.clk_rst_tbl_size = ARRAY_SIZE(sm8750_clk_reset_table),
-	.bw_tbl_dec = sm8550_bw_table_dec,
-	.bw_tbl_dec_size = ARRAY_SIZE(sm8550_bw_table_dec),
-	.pmdomain_tbl = sm8550_pmdomain_table,
-	.pmdomain_tbl_size = ARRAY_SIZE(sm8550_pmdomain_table),
-	.opp_pd_tbl = sm8550_opp_pd_table,
-	.opp_pd_tbl_size = ARRAY_SIZE(sm8550_opp_pd_table),
-	.clk_tbl = sm8750_clk_table,
-	.clk_tbl_size = ARRAY_SIZE(sm8750_clk_table),
-	.opp_clk_tbl = sm8550_opp_clk_table,
-	/* Upper bound of DMA address range */
-	.dma_mask = 0xe0000000 - 1,
-	.fwname = "qcom/vpu/vpu35_p4.mbn",
-	.pas_id = IRIS_PAS_ID,
-	.inst_iris_fmts = platform_fmts_sm8550_dec,
-	.inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8550_dec),
-	.inst_caps = &platform_inst_cap_sm8550,
-	.inst_fw_caps_dec = inst_fw_cap_sm8550_dec,
-	.inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec),
-	.inst_fw_caps_enc = inst_fw_cap_sm8550_enc,
-	.inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc),
-	.tz_cp_config_data = tz_cp_config_sm8550,
-	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8550),
-	.core_arch = VIDEO_ARCH_LX,
-	.hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE,
-	.ubwc_config = &ubwc_config_sm8550,
-	.num_vpp_pipe = 4,
-	.max_session_count = 16,
-	.max_core_mbpf = NUM_MBS_8K * 2,
-	.max_core_mbps = ((7680 * 4320) / 256) * 60,
-	.dec_input_config_params_default =
-		sm8550_vdec_input_config_params_default,
-	.dec_input_config_params_default_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_params_default),
-	.dec_input_config_params_hevc =
-		sm8550_vdec_input_config_param_hevc,
-	.dec_input_config_params_hevc_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_hevc),
-	.dec_input_config_params_vp9 =
-		sm8550_vdec_input_config_param_vp9,
-	.dec_input_config_params_vp9_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_vp9),
-	.dec_input_config_params_av1 =
-		sm8550_vdec_input_config_param_av1,
-	.dec_input_config_params_av1_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_av1),
-	.dec_output_config_params =
-		sm8550_vdec_output_config_params,
-	.dec_output_config_params_size =
-		ARRAY_SIZE(sm8550_vdec_output_config_params),
-
-	.enc_input_config_params =
-		sm8550_venc_input_config_params,
-	.enc_input_config_params_size =
-		ARRAY_SIZE(sm8550_venc_input_config_params),
-	.enc_output_config_params =
-		sm8550_venc_output_config_params,
-	.enc_output_config_params_size =
-		ARRAY_SIZE(sm8550_venc_output_config_params),
-
-	.dec_input_prop = sm8550_vdec_subscribe_input_properties,
-	.dec_input_prop_size = ARRAY_SIZE(sm8550_vdec_subscribe_input_properties),
-	.dec_output_prop_avc = sm8550_vdec_subscribe_output_properties_avc,
-	.dec_output_prop_avc_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_avc),
-	.dec_output_prop_hevc = sm8550_vdec_subscribe_output_properties_hevc,
-	.dec_output_prop_hevc_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_hevc),
-	.dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9,
-	.dec_output_prop_vp9_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9),
-	.dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1,
-	.dec_output_prop_av1_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1),
-
-	.dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl,
-	.dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl),
-	.dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl,
-	.dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl),
-
-	.enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl,
-	.enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl),
-	.enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl,
-	.enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl),
-};
-
-/*
- * Shares most of SM8550 data except:
- * - inst_caps to platform_inst_cap_qcs8300
- */
-const struct iris_platform_data qcs8300_data = {
-	.get_instance = iris_hfi_gen2_get_instance,
-	.init_hfi_command_ops = iris_hfi_gen2_command_ops_init,
-	.init_hfi_response_ops = iris_hfi_gen2_response_ops_init,
-	.get_vpu_buffer_size = iris_vpu_buf_size,
-	.vpu_ops = &iris_vpu3_ops,
-	.set_preset_registers = iris_set_sm8550_preset_registers,
-	.icc_tbl = sm8550_icc_table,
-	.icc_tbl_size = ARRAY_SIZE(sm8550_icc_table),
-	.clk_rst_tbl = sm8550_clk_reset_table,
-	.clk_rst_tbl_size = ARRAY_SIZE(sm8550_clk_reset_table),
-	.bw_tbl_dec = sm8550_bw_table_dec,
-	.bw_tbl_dec_size = ARRAY_SIZE(sm8550_bw_table_dec),
-	.pmdomain_tbl = sm8550_pmdomain_table,
-	.pmdomain_tbl_size = ARRAY_SIZE(sm8550_pmdomain_table),
-	.opp_pd_tbl = sm8550_opp_pd_table,
-	.opp_pd_tbl_size = ARRAY_SIZE(sm8550_opp_pd_table),
-	.clk_tbl = sm8550_clk_table,
-	.clk_tbl_size = ARRAY_SIZE(sm8550_clk_table),
-	.opp_clk_tbl = sm8550_opp_clk_table,
-	/* Upper bound of DMA address range */
-	.dma_mask = 0xe0000000 - 1,
-	.fwname = "qcom/vpu/vpu30_p4_s6.mbn",
-	.pas_id = IRIS_PAS_ID,
-	.inst_iris_fmts = platform_fmts_sm8550_dec,
-	.inst_iris_fmts_size = ARRAY_SIZE(platform_fmts_sm8550_dec),
-	.inst_caps = &platform_inst_cap_qcs8300,
-	.inst_fw_caps_dec = inst_fw_cap_sm8550_dec,
-	.inst_fw_caps_dec_size = ARRAY_SIZE(inst_fw_cap_sm8550_dec),
-	.inst_fw_caps_enc = inst_fw_cap_sm8550_enc,
-	.inst_fw_caps_enc_size = ARRAY_SIZE(inst_fw_cap_sm8550_enc),
-	.tz_cp_config_data = tz_cp_config_sm8550,
-	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_sm8550),
-	.core_arch = VIDEO_ARCH_LX,
-	.hw_response_timeout = HW_RESPONSE_TIMEOUT_VALUE,
-	.ubwc_config = &ubwc_config_sm8550,
-	.num_vpp_pipe = 2,
-	.max_session_count = 16,
-	.max_core_mbpf = ((4096 * 2176) / 256) * 4,
-	.max_core_mbps = (((3840 * 2176) / 256) * 120),
-	.dec_input_config_params_default =
-		sm8550_vdec_input_config_params_default,
-	.dec_input_config_params_default_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_params_default),
-	.dec_input_config_params_hevc =
-		sm8550_vdec_input_config_param_hevc,
-	.dec_input_config_params_hevc_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_hevc),
-	.dec_input_config_params_vp9 =
-		sm8550_vdec_input_config_param_vp9,
-	.dec_input_config_params_vp9_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_vp9),
-	.dec_input_config_params_av1 =
-		sm8550_vdec_input_config_param_av1,
-	.dec_input_config_params_av1_size =
-		ARRAY_SIZE(sm8550_vdec_input_config_param_av1),
-	.dec_output_config_params =
-		sm8550_vdec_output_config_params,
-	.dec_output_config_params_size =
-		ARRAY_SIZE(sm8550_vdec_output_config_params),
-
-	.enc_input_config_params =
-		sm8550_venc_input_config_params,
-	.enc_input_config_params_size =
-		ARRAY_SIZE(sm8550_venc_input_config_params),
-	.enc_output_config_params =
-		sm8550_venc_output_config_params,
-	.enc_output_config_params_size =
-		ARRAY_SIZE(sm8550_venc_output_config_params),
-
-	.dec_input_prop = sm8550_vdec_subscribe_input_properties,
-	.dec_input_prop_size = ARRAY_SIZE(sm8550_vdec_subscribe_input_properties),
-	.dec_output_prop_avc = sm8550_vdec_subscribe_output_properties_avc,
-	.dec_output_prop_avc_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_avc),
-	.dec_output_prop_hevc = sm8550_vdec_subscribe_output_properties_hevc,
-	.dec_output_prop_hevc_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_hevc),
-	.dec_output_prop_vp9 = sm8550_vdec_subscribe_output_properties_vp9,
-	.dec_output_prop_vp9_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_vp9),
-	.dec_output_prop_av1 = sm8550_vdec_subscribe_output_properties_av1,
-	.dec_output_prop_av1_size =
-		ARRAY_SIZE(sm8550_vdec_subscribe_output_properties_av1),
-
-	.dec_ip_int_buf_tbl = sm8550_dec_ip_int_buf_tbl,
-	.dec_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_ip_int_buf_tbl),
-	.dec_op_int_buf_tbl = sm8550_dec_op_int_buf_tbl,
-	.dec_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_dec_op_int_buf_tbl),
-
-	.enc_ip_int_buf_tbl = sm8550_enc_ip_int_buf_tbl,
-	.enc_ip_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_ip_int_buf_tbl),
-	.enc_op_int_buf_tbl = sm8550_enc_op_int_buf_tbl,
-	.enc_op_int_buf_tbl_size = ARRAY_SIZE(sm8550_enc_op_int_buf_tbl),
-};
diff --git a/drivers/media/platform/qcom/iris/iris_platform_sm8250.h b/drivers/media/platform/qcom/iris/iris_platform_sm8250.h
new file mode 100644
index 0000000000000..50306043eb8ec
--- /dev/null
+++ b/drivers/media/platform/qcom/iris/iris_platform_sm8250.h
@@ -0,0 +1,29 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __IRIS_PLATFORM_SM8250_H__
+#define __IRIS_PLATFORM_SM8250_H__
+
+static const struct bw_info sm8250_bw_table_dec[] = {
+	{ ((4096 * 2160) / 256) * 60, 2403000 },
+	{ ((4096 * 2160) / 256) * 30, 1224000 },
+	{ ((1920 * 1080) / 256) * 60,  812000 },
+	{ ((1920 * 1080) / 256) * 30,  416000 },
+};
+
+static const char * const sm8250_opp_pd_table[] = { "mx", "mmcx" };
+
+static const struct platform_clk_data sm8250_clk_table[] = {
+	{IRIS_AXI_CLK,  "iface"        },
+	{IRIS_CTRL_CLK, "core"         },
+	{IRIS_HW_CLK,   "vcodec0_core" },
+};
+
+static const char * const sm8250_opp_clk_table[] = {
+	"vcodec0_core",
+	NULL,
+};
+
+#endif
diff --git a/drivers/media/platform/qcom/iris/iris_platform_sm8550.h b/drivers/media/platform/qcom/iris/iris_platform_sm8550.h
new file mode 100644
index 0000000000000..a9d9709c2e352
--- /dev/null
+++ b/drivers/media/platform/qcom/iris/iris_platform_sm8550.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Copyright (c) Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#ifndef __IRIS_PLATFORM_SM8550_H__
+#define __IRIS_PLATFORM_SM8550_H__
+
+static const char * const sm8550_clk_reset_table[] = { "bus" };
+
+static const struct platform_clk_data sm8550_clk_table[] = {
+	{IRIS_AXI_CLK,  "iface"        },
+	{IRIS_CTRL_CLK, "core"         },
+	{IRIS_HW_CLK,   "vcodec0_core" },
+};
+
+static struct platform_inst_caps platform_inst_cap_sm8550 = {
+	.min_frame_width = 96,
+	.max_frame_width = 8192,
+	.min_frame_height = 96,
+	.max_frame_height = 8192,
+	.max_mbpf = (8192 * 4352) / 256,
+	.mb_cycles_vpp = 200,
+	.mb_cycles_fw = 489583,
+	.mb_cycles_fw_vpp = 66234,
+	.num_comv = 0,
+	.max_frame_rate = MAXIMUM_FPS,
+	.max_operating_rate = MAXIMUM_FPS,
+};
+
+#endif
diff --git a/drivers/media/platform/qcom/iris/iris_platform_vpu2.c b/drivers/media/platform/qcom/iris/iris_platform_vpu2.c
new file mode 100644
index 0000000000000..41986af8313b1
--- /dev/null
+++ b/drivers/media/platform/qcom/iris/iris_platform_vpu2.c
@@ -0,0 +1,132 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include "iris_core.h"
+#include "iris_ctrls.h"
+#include "iris_platform_common.h"
+#include "iris_resources.h"
+#include "iris_hfi_gen1.h"
+#include "iris_hfi_gen1_defines.h"
+#include "iris_vpu_buffer.h"
+#include "iris_vpu_common.h"
+#include "iris_instance.h"
+
+#include "iris_platform_sc7280.h"
+#include "iris_platform_sm8250.h"
+
+static const struct iris_firmware_desc iris_vpu20_p1_gen1_desc = {
+	.firmware_data = &iris_hfi_gen1_data,
+	.get_vpu_buffer_size = iris_vpu_buf_size,
+	.fwname = "qcom/vpu/vpu20_p1.mbn",
+};
+
+static const struct iris_firmware_desc iris_vpu20_p4_gen1_desc = {
+	.firmware_data = &iris_hfi_gen1_data,
+	.get_vpu_buffer_size = iris_vpu_buf_size,
+	.fwname = "qcom/vpu/vpu20_p4.mbn",
+};
+
+static struct iris_fmt iris_fmts_vpu2_dec[] = {
+	[IRIS_FMT_H264] = {
+		.pixfmt = V4L2_PIX_FMT_H264,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+	},
+	[IRIS_FMT_HEVC] = {
+		.pixfmt = V4L2_PIX_FMT_HEVC,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+	},
+	[IRIS_FMT_VP9] = {
+		.pixfmt = V4L2_PIX_FMT_VP9,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+	},
+};
+
+static struct platform_inst_caps platform_inst_cap_vpu2 = {
+	.min_frame_width = 128,
+	.max_frame_width = 8192,
+	.min_frame_height = 128,
+	.max_frame_height = 8192,
+	.max_mbpf = 138240,
+	.mb_cycles_vsp = 25,
+	.mb_cycles_vpp = 200,
+	.max_frame_rate = MAXIMUM_FPS,
+	.max_operating_rate = MAXIMUM_FPS,
+};
+
+static const struct icc_info iris_icc_info_vpu2[] = {
+	{ "cpu-cfg",    1000, 1000     },
+	{ "video-mem",  1000, 15000000 },
+};
+
+static const char * const iris_clk_reset_table_vpu2[] = { "bus", "core" };
+
+static const char * const iris_pmdomain_table_vpu2[] = { "venus", "vcodec0" };
+
+static const struct tz_cp_config tz_cp_config_vpu2[] = {
+	{
+		.cp_start = 0,
+		.cp_size = 0x25800000,
+		.cp_nonpixel_start = 0x01000000,
+		.cp_nonpixel_size = 0x24800000,
+	},
+};
+
+const struct iris_platform_data sc7280_data = {
+	.firmware_desc = &iris_vpu20_p1_gen1_desc,
+	.vpu_ops = &iris_vpu2_ops,
+	.icc_tbl = iris_icc_info_vpu2,
+	.icc_tbl_size = ARRAY_SIZE(iris_icc_info_vpu2),
+	.bw_tbl_dec = sc7280_bw_table_dec,
+	.bw_tbl_dec_size = ARRAY_SIZE(sc7280_bw_table_dec),
+	.pmdomain_tbl = iris_pmdomain_table_vpu2,
+	.pmdomain_tbl_size = ARRAY_SIZE(iris_pmdomain_table_vpu2),
+	.opp_pd_tbl = sc7280_opp_pd_table,
+	.opp_pd_tbl_size = ARRAY_SIZE(sc7280_opp_pd_table),
+	.clk_tbl = sc7280_clk_table,
+	.clk_tbl_size = ARRAY_SIZE(sc7280_clk_table),
+	.opp_clk_tbl = sc7280_opp_clk_table,
+	/* Upper bound of DMA address range */
+	.dma_mask = 0xe0000000 - 1,
+	.inst_iris_fmts = iris_fmts_vpu2_dec,
+	.inst_iris_fmts_size = ARRAY_SIZE(iris_fmts_vpu2_dec),
+	.inst_caps = &platform_inst_cap_vpu2,
+	.tz_cp_config_data = tz_cp_config_vpu2,
+	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_vpu2),
+	.num_vpp_pipe = 1,
+	.no_aon = true,
+	.max_session_count = 16,
+	.max_core_mbpf = 4096 * 2176 / 256 * 2 + 1920 * 1088 / 256,
+	/* max spec for SC7280 is 4096x2176@60fps */
+	.max_core_mbps = 4096 * 2176 / 256 * 60,
+};
+
+const struct iris_platform_data sm8250_data = {
+	.firmware_desc = &iris_vpu20_p4_gen1_desc,
+	.vpu_ops = &iris_vpu2_ops,
+	.icc_tbl = iris_icc_info_vpu2,
+	.icc_tbl_size = ARRAY_SIZE(iris_icc_info_vpu2),
+	.clk_rst_tbl = iris_clk_reset_table_vpu2,
+	.clk_rst_tbl_size = ARRAY_SIZE(iris_clk_reset_table_vpu2),
+	.bw_tbl_dec = sm8250_bw_table_dec,
+	.bw_tbl_dec_size = ARRAY_SIZE(sm8250_bw_table_dec),
+	.pmdomain_tbl = iris_pmdomain_table_vpu2,
+	.pmdomain_tbl_size = ARRAY_SIZE(iris_pmdomain_table_vpu2),
+	.opp_pd_tbl = sm8250_opp_pd_table,
+	.opp_pd_tbl_size = ARRAY_SIZE(sm8250_opp_pd_table),
+	.clk_tbl = sm8250_clk_table,
+	.clk_tbl_size = ARRAY_SIZE(sm8250_clk_table),
+	.opp_clk_tbl = sm8250_opp_clk_table,
+	/* Upper bound of DMA address range */
+	.dma_mask = 0xe0000000 - 1,
+	.inst_iris_fmts = iris_fmts_vpu2_dec,
+	.inst_iris_fmts_size = ARRAY_SIZE(iris_fmts_vpu2_dec),
+	.inst_caps = &platform_inst_cap_vpu2,
+	.tz_cp_config_data = tz_cp_config_vpu2,
+	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_vpu2),
+	.num_vpp_pipe = 4,
+	.max_session_count = 16,
+	.max_core_mbpf = NUM_MBS_8K,
+	.max_core_mbps = ((7680 * 4320) / 256) * 60,
+};
diff --git a/drivers/media/platform/qcom/iris/iris_platform_vpu3x.c b/drivers/media/platform/qcom/iris/iris_platform_vpu3x.c
new file mode 100644
index 0000000000000..c249ff8275414
--- /dev/null
+++ b/drivers/media/platform/qcom/iris/iris_platform_vpu3x.c
@@ -0,0 +1,219 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2025 Linaro Ltd
+ */
+
+#include "iris_core.h"
+#include "iris_ctrls.h"
+#include "iris_hfi_gen2.h"
+#include "iris_hfi_gen2_defines.h"
+#include "iris_platform_common.h"
+#include "iris_vpu_buffer.h"
+#include "iris_vpu_common.h"
+
+#include "iris_platform_qcs8300.h"
+#include "iris_platform_sm8550.h"
+#include "iris_platform_sm8650.h"
+#include "iris_platform_sm8750.h"
+
+static const struct iris_firmware_desc iris_vpu30_p4_s6_gen2_desc = {
+	.firmware_data = &iris_hfi_gen2_data,
+	.get_vpu_buffer_size = iris_vpu_buf_size,
+	.fwname = "qcom/vpu/vpu30_p4_s6.mbn",
+};
+
+static const struct iris_firmware_desc iris_vpu30_p4_gen2_desc = {
+	.firmware_data = &iris_hfi_gen2_data,
+	.get_vpu_buffer_size = iris_vpu_buf_size,
+	.fwname = "qcom/vpu/vpu30_p4.mbn",
+};
+
+static const struct iris_firmware_desc iris_vpu33_p4_gen2_desc = {
+	.firmware_data = &iris_hfi_gen2_data,
+	.get_vpu_buffer_size = iris_vpu33_buf_size,
+	.fwname = "qcom/vpu/vpu33_p4.mbn",
+};
+
+static const struct iris_firmware_desc iris_vpu35_p4_gen2_desc = {
+	.firmware_data = &iris_hfi_gen2_data,
+	.get_vpu_buffer_size = iris_vpu33_buf_size,
+	.fwname = "qcom/vpu/vpu35_p4.mbn",
+};
+
+static struct iris_fmt iris_fmts_vpu3x_dec[] = {
+	[IRIS_FMT_H264] = {
+		.pixfmt = V4L2_PIX_FMT_H264,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+	},
+	[IRIS_FMT_HEVC] = {
+		.pixfmt = V4L2_PIX_FMT_HEVC,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+	},
+	[IRIS_FMT_VP9] = {
+		.pixfmt = V4L2_PIX_FMT_VP9,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+	},
+	[IRIS_FMT_AV1] = {
+		.pixfmt = V4L2_PIX_FMT_AV1,
+		.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE,
+	},
+};
+
+static const struct icc_info iris_icc_info_vpu3x[] = {
+	{ "cpu-cfg",    1000, 1000     },
+	{ "video-mem",  1000, 15000000 },
+};
+
+static const struct bw_info iris_bw_table_dec_vpu3x[] = {
+	{ ((4096 * 2160) / 256) * 60, 1608000 },
+	{ ((4096 * 2160) / 256) * 30,  826000 },
+	{ ((1920 * 1080) / 256) * 60,  567000 },
+	{ ((1920 * 1080) / 256) * 30,  294000 },
+};
+
+static const char * const iris_pmdomain_table_vpu3x[] = { "venus", "vcodec0" };
+
+static const char * const iris_opp_pd_table_vpu3x[] = { "mxc", "mmcx" };
+
+static const char * const iris_opp_clk_table_vpu3x[] = {
+	"vcodec0_core",
+	NULL,
+};
+
+static const struct tz_cp_config tz_cp_config_vpu3[] = {
+	{
+		.cp_start = 0,
+		.cp_size = 0x25800000,
+		.cp_nonpixel_start = 0x01000000,
+		.cp_nonpixel_size = 0x24800000,
+	},
+};
+
+/*
+ * Shares most of SM8550 data except:
+ * - inst_caps to platform_inst_cap_qcs8300
+ */
+const struct iris_platform_data qcs8300_data = {
+	.firmware_desc = &iris_vpu30_p4_s6_gen2_desc,
+	.vpu_ops = &iris_vpu3_ops,
+	.icc_tbl = iris_icc_info_vpu3x,
+	.icc_tbl_size = ARRAY_SIZE(iris_icc_info_vpu3x),
+	.clk_rst_tbl = sm8550_clk_reset_table,
+	.clk_rst_tbl_size = ARRAY_SIZE(sm8550_clk_reset_table),
+	.bw_tbl_dec = iris_bw_table_dec_vpu3x,
+	.bw_tbl_dec_size = ARRAY_SIZE(iris_bw_table_dec_vpu3x),
+	.pmdomain_tbl = iris_pmdomain_table_vpu3x,
+	.pmdomain_tbl_size = ARRAY_SIZE(iris_pmdomain_table_vpu3x),
+	.opp_pd_tbl = iris_opp_pd_table_vpu3x,
+	.opp_pd_tbl_size = ARRAY_SIZE(iris_opp_pd_table_vpu3x),
+	.clk_tbl = sm8550_clk_table,
+	.clk_tbl_size = ARRAY_SIZE(sm8550_clk_table),
+	.opp_clk_tbl = iris_opp_clk_table_vpu3x,
+	/* Upper bound of DMA address range */
+	.dma_mask = 0xe0000000 - 1,
+	.inst_iris_fmts = iris_fmts_vpu3x_dec,
+	.inst_iris_fmts_size = ARRAY_SIZE(iris_fmts_vpu3x_dec),
+	.inst_caps = &platform_inst_cap_qcs8300,
+	.tz_cp_config_data = tz_cp_config_vpu3,
+	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_vpu3),
+	.num_vpp_pipe = 2,
+	.max_session_count = 16,
+	.max_core_mbpf = ((4096 * 2176) / 256) * 4,
+	.max_core_mbps = (((3840 * 2176) / 256) * 120),
+};
+
+const struct iris_platform_data sm8550_data = {
+	.firmware_desc = &iris_vpu30_p4_gen2_desc,
+	.vpu_ops = &iris_vpu3_ops,
+	.icc_tbl = iris_icc_info_vpu3x,
+	.icc_tbl_size = ARRAY_SIZE(iris_icc_info_vpu3x),
+	.clk_rst_tbl = sm8550_clk_reset_table,
+	.clk_rst_tbl_size = ARRAY_SIZE(sm8550_clk_reset_table),
+	.bw_tbl_dec = iris_bw_table_dec_vpu3x,
+	.bw_tbl_dec_size = ARRAY_SIZE(iris_bw_table_dec_vpu3x),
+	.pmdomain_tbl = iris_pmdomain_table_vpu3x,
+	.pmdomain_tbl_size = ARRAY_SIZE(iris_pmdomain_table_vpu3x),
+	.opp_pd_tbl = iris_opp_pd_table_vpu3x,
+	.opp_pd_tbl_size = ARRAY_SIZE(iris_opp_pd_table_vpu3x),
+	.clk_tbl = sm8550_clk_table,
+	.clk_tbl_size = ARRAY_SIZE(sm8550_clk_table),
+	.opp_clk_tbl = iris_opp_clk_table_vpu3x,
+	/* Upper bound of DMA address range */
+	.dma_mask = 0xe0000000 - 1,
+	.inst_iris_fmts = iris_fmts_vpu3x_dec,
+	.inst_iris_fmts_size = ARRAY_SIZE(iris_fmts_vpu3x_dec),
+	.inst_caps = &platform_inst_cap_sm8550,
+	.tz_cp_config_data = tz_cp_config_vpu3,
+	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_vpu3),
+	.num_vpp_pipe = 4,
+	.max_session_count = 16,
+	.max_core_mbpf = NUM_MBS_8K * 2,
+	.max_core_mbps = ((7680 * 4320) / 256) * 60,
+};
+
+/*
+ * Shares most of SM8550 data except:
+ * - vpu_ops to iris_vpu33_ops
+ * - clk_rst_tbl to sm8650_clk_reset_table
+ * - controller_rst_tbl to sm8650_controller_reset_table
+ */
+const struct iris_platform_data sm8650_data = {
+	.firmware_desc = &iris_vpu33_p4_gen2_desc,
+	.vpu_ops = &iris_vpu33_ops,
+	.icc_tbl = iris_icc_info_vpu3x,
+	.icc_tbl_size = ARRAY_SIZE(iris_icc_info_vpu3x),
+	.clk_rst_tbl = sm8650_clk_reset_table,
+	.clk_rst_tbl_size = ARRAY_SIZE(sm8650_clk_reset_table),
+	.controller_rst_tbl = sm8650_controller_reset_table,
+	.controller_rst_tbl_size = ARRAY_SIZE(sm8650_controller_reset_table),
+	.bw_tbl_dec = iris_bw_table_dec_vpu3x,
+	.bw_tbl_dec_size = ARRAY_SIZE(iris_bw_table_dec_vpu3x),
+	.pmdomain_tbl = iris_pmdomain_table_vpu3x,
+	.pmdomain_tbl_size = ARRAY_SIZE(iris_pmdomain_table_vpu3x),
+	.opp_pd_tbl = iris_opp_pd_table_vpu3x,
+	.opp_pd_tbl_size = ARRAY_SIZE(iris_opp_pd_table_vpu3x),
+	.clk_tbl = sm8550_clk_table,
+	.clk_tbl_size = ARRAY_SIZE(sm8550_clk_table),
+	.opp_clk_tbl = iris_opp_clk_table_vpu3x,
+	/* Upper bound of DMA address range */
+	.dma_mask = 0xe0000000 - 1,
+	.inst_iris_fmts = iris_fmts_vpu3x_dec,
+	.inst_iris_fmts_size = ARRAY_SIZE(iris_fmts_vpu3x_dec),
+	.inst_caps = &platform_inst_cap_sm8550,
+	.tz_cp_config_data = tz_cp_config_vpu3,
+	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_vpu3),
+	.num_vpp_pipe = 4,
+	.max_session_count = 16,
+	.max_core_mbpf = NUM_MBS_8K * 2,
+	.max_core_mbps = ((7680 * 4320) / 256) * 60,
+};
+
+const struct iris_platform_data sm8750_data = {
+	.firmware_desc = &iris_vpu35_p4_gen2_desc,
+	.vpu_ops = &iris_vpu35_ops,
+	.icc_tbl = iris_icc_info_vpu3x,
+	.icc_tbl_size = ARRAY_SIZE(iris_icc_info_vpu3x),
+	.clk_rst_tbl = sm8750_clk_reset_table,
+	.clk_rst_tbl_size = ARRAY_SIZE(sm8750_clk_reset_table),
+	.bw_tbl_dec = iris_bw_table_dec_vpu3x,
+	.bw_tbl_dec_size = ARRAY_SIZE(iris_bw_table_dec_vpu3x),
+	.pmdomain_tbl = iris_pmdomain_table_vpu3x,
+	.pmdomain_tbl_size = ARRAY_SIZE(iris_pmdomain_table_vpu3x),
+	.opp_pd_tbl = iris_opp_pd_table_vpu3x,
+	.opp_pd_tbl_size = ARRAY_SIZE(iris_opp_pd_table_vpu3x),
+	.clk_tbl = sm8750_clk_table,
+	.clk_tbl_size = ARRAY_SIZE(sm8750_clk_table),
+	.opp_clk_tbl = iris_opp_clk_table_vpu3x,
+	/* Upper bound of DMA address range */
+	.dma_mask = 0xe0000000 - 1,
+	.inst_iris_fmts = iris_fmts_vpu3x_dec,
+	.inst_iris_fmts_size = ARRAY_SIZE(iris_fmts_vpu3x_dec),
+	.inst_caps = &platform_inst_cap_sm8550,
+	.tz_cp_config_data = tz_cp_config_vpu3,
+	.tz_cp_config_data_size = ARRAY_SIZE(tz_cp_config_vpu3),
+	.num_vpp_pipe = 4,
+	.max_session_count = 16,
+	.max_core_mbpf = NUM_MBS_8K * 2,
+	.max_core_mbps = ((7680 * 4320) / 256) * 60,
+};
diff --git a/drivers/media/platform/qcom/iris/iris_probe.c b/drivers/media/platform/qcom/iris/iris_probe.c
index ddaacda523ecb..d36f0c0e785b7 100644
--- a/drivers/media/platform/qcom/iris/iris_probe.c
+++ b/drivers/media/platform/qcom/iris/iris_probe.c
@@ -10,6 +10,7 @@
 #include <linux/pm_opp.h>
 #include <linux/pm_runtime.h>
 #include <linux/reset.h>
+#include <linux/soc/qcom/ubwc.h>
 
 #include "iris_core.h"
 #include "iris_ctrls.h"
@@ -64,6 +65,13 @@ static int iris_init_power_domains(struct iris_core *core)
 		return ret;
 
 	ret =  devm_pm_domain_attach_list(core->dev, &iris_opp_pd_data, &core->opp_pmdomain_tbl);
+	/* backwards compatibility for incomplete ABI SM8250 */
+	if (ret == -ENODEV &&
+	    of_device_is_compatible(core->dev->of_node, "qcom,sm8250-venus")) {
+		iris_opp_pd_data.num_pd_names--;
+		ret = devm_pm_domain_attach_list(core->dev, &iris_opp_pd_data,
+						 &core->opp_pmdomain_tbl);
+	}
 	if (ret < 0)
 		return ret;
 
@@ -243,6 +251,12 @@ static int iris_probe(struct platform_device *pdev)
 		return core->irq;
 
 	core->iris_platform_data = of_device_get_match_data(core->dev);
+	core->iris_firmware_desc = core->iris_platform_data->firmware_desc;
+	core->iris_firmware_data = core->iris_firmware_desc->firmware_data;
+
+	core->ubwc_cfg = qcom_ubwc_config_get_data();
+	if (IS_ERR(core->ubwc_cfg))
+		return PTR_ERR(core->ubwc_cfg);
 
 	ret = devm_request_threaded_irq(core->dev, core->irq, iris_hfi_isr,
 					iris_hfi_isr_handler, IRQF_TRIGGER_HIGH, "iris", core);
@@ -252,8 +266,7 @@ static int iris_probe(struct platform_device *pdev)
 	disable_irq_nosync(core->irq);
 
 	iris_init_ops(core);
-	core->iris_platform_data->init_hfi_command_ops(core);
-	core->iris_platform_data->init_hfi_response_ops(core);
+	core->iris_firmware_data->init_hfi_ops(core);
 
 	ret = iris_init_resources(core);
 	if (ret)
@@ -352,7 +365,6 @@ static const struct of_device_id iris_dt_match[] = {
 		.compatible = "qcom,qcs8300-iris",
 		.data = &qcs8300_data,
 	},
-#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_VENUS))
 	{
 		.compatible = "qcom,sc7280-venus",
 		.data = &sc7280_data,
@@ -361,7 +373,6 @@ static const struct of_device_id iris_dt_match[] = {
 		.compatible = "qcom,sm8250-venus",
 		.data = &sm8250_data,
 	},
-#endif
 	{
 		.compatible = "qcom,sm8550-iris",
 		.data = &sm8550_data,
diff --git a/drivers/media/platform/qcom/iris/iris_utils.c b/drivers/media/platform/qcom/iris/iris_utils.c
index cfc5b576ec56b..29b07d88507eb 100644
--- a/drivers/media/platform/qcom/iris/iris_utils.c
+++ b/drivers/media/platform/qcom/iris/iris_utils.c
@@ -55,16 +55,13 @@ void iris_helper_buffers_done(struct iris_inst *inst, unsigned int type,
 
 int iris_wait_for_session_response(struct iris_inst *inst, bool is_flush)
 {
-	struct iris_core *core = inst->core;
-	u32 hw_response_timeout_val;
 	struct completion *done;
 	int ret;
 
-	hw_response_timeout_val = core->iris_platform_data->hw_response_timeout;
 	done = is_flush ? &inst->flush_completion : &inst->completion;
 
 	mutex_unlock(&inst->lock);
-	ret = wait_for_completion_timeout(done, msecs_to_jiffies(hw_response_timeout_val));
+	ret = wait_for_completion_timeout(done, msecs_to_jiffies(HW_RESPONSE_TIMEOUT_VALUE));
 	mutex_lock(&inst->lock);
 	if (!ret) {
 		iris_inst_change_state(inst, IRIS_INST_ERROR);
diff --git a/drivers/media/platform/qcom/iris/iris_vb2.c b/drivers/media/platform/qcom/iris/iris_vb2.c
index bf0b8400996ec..a2ea2d67f60d0 100644
--- a/drivers/media/platform/qcom/iris/iris_vb2.c
+++ b/drivers/media/platform/qcom/iris/iris_vb2.c
@@ -129,7 +129,7 @@ int iris_vb2_queue_setup(struct vb2_queue *q,
 	if (!inst->once_per_session_set) {
 		inst->once_per_session_set = true;
 
-		ret = core->hfi_ops->session_open(inst);
+		ret = inst->hfi_session_ops->session_open(inst);
 		if (ret) {
 			ret = -EINVAL;
 			dev_err(core->dev, "session open failed\n");
diff --git a/drivers/media/platform/qcom/iris/iris_vdec.c b/drivers/media/platform/qcom/iris/iris_vdec.c
index 99d544e2af4f9..dfccfb274492f 100644
--- a/drivers/media/platform/qcom/iris/iris_vdec.c
+++ b/drivers/media/platform/qcom/iris/iris_vdec.c
@@ -368,7 +368,7 @@ int iris_vdec_streamon_input(struct iris_inst *inst)
 
 int iris_vdec_streamon_output(struct iris_inst *inst)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	int ret;
 
 	ret = hfi_ops->session_set_config_params(inst, V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE);
@@ -428,7 +428,7 @@ int iris_vdec_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
 
 int iris_vdec_start_cmd(struct iris_inst *inst)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	enum iris_inst_sub_state clear_sub_state = 0;
 	struct vb2_queue *dst_vq;
 	int ret;
@@ -491,7 +491,7 @@ int iris_vdec_start_cmd(struct iris_inst *inst)
 
 int iris_vdec_stop_cmd(struct iris_inst *inst)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	int ret;
 
 	ret = hfi_ops->session_drain(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
diff --git a/drivers/media/platform/qcom/iris/iris_venc.c b/drivers/media/platform/qcom/iris/iris_venc.c
index 4d886769d958b..c11fee4799cdd 100644
--- a/drivers/media/platform/qcom/iris/iris_venc.c
+++ b/drivers/media/platform/qcom/iris/iris_venc.c
@@ -575,7 +575,7 @@ int iris_venc_qbuf(struct iris_inst *inst, struct vb2_v4l2_buffer *vbuf)
 
 int iris_venc_start_cmd(struct iris_inst *inst)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	enum iris_inst_sub_state clear_sub_state = 0;
 	struct vb2_queue *dst_vq;
 	int ret;
@@ -617,7 +617,7 @@ int iris_venc_start_cmd(struct iris_inst *inst)
 
 int iris_venc_stop_cmd(struct iris_inst *inst)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	int ret;
 
 	ret = hfi_ops->session_drain(inst, V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE);
diff --git a/drivers/media/platform/qcom/iris/iris_vidc.c b/drivers/media/platform/qcom/iris/iris_vidc.c
index 5eb1786b07371..14d63dc76c9ba 100644
--- a/drivers/media/platform/qcom/iris/iris_vidc.c
+++ b/drivers/media/platform/qcom/iris/iris_vidc.c
@@ -156,7 +156,7 @@ int iris_open(struct file *filp)
 
 	pm_runtime_put_sync(core->dev);
 
-	inst = core->iris_platform_data->get_instance();
+	inst = core->hfi_sys_ops->sys_get_instance();
 	if (!inst)
 		return -ENOMEM;
 
@@ -224,7 +224,7 @@ int iris_open(struct file *filp)
 
 static void iris_session_close(struct iris_inst *inst)
 {
-	const struct iris_hfi_command_ops *hfi_ops = inst->core->hfi_ops;
+	const struct iris_hfi_session_ops *hfi_ops = inst->hfi_session_ops;
 	bool wait_for_response = true;
 	int ret;
 
@@ -243,7 +243,7 @@ static void iris_session_close(struct iris_inst *inst)
 
 static void iris_check_num_queued_internal_buffers(struct iris_inst *inst, u32 plane)
 {
-	const struct iris_platform_data *platform_data = inst->core->iris_platform_data;
+	const struct iris_firmware_data *firmware_data = inst->core->iris_firmware_data;
 	struct iris_buffer *buf, *next;
 	struct iris_buffers *buffers;
 	const u32 *internal_buf_type;
@@ -251,11 +251,11 @@ static void iris_check_num_queued_internal_buffers(struct iris_inst *inst, u32 p
 	u32 count = 0;
 
 	if (V4L2_TYPE_IS_OUTPUT(plane)) {
-		internal_buf_type = platform_data->dec_ip_int_buf_tbl;
-		internal_buffer_count = platform_data->dec_ip_int_buf_tbl_size;
+		internal_buf_type = firmware_data->dec_ip_int_buf_tbl;
+		internal_buffer_count = firmware_data->dec_ip_int_buf_tbl_size;
 	} else {
-		internal_buf_type = platform_data->dec_op_int_buf_tbl;
-		internal_buffer_count = platform_data->dec_op_int_buf_tbl_size;
+		internal_buf_type = firmware_data->dec_op_int_buf_tbl;
+		internal_buffer_count = firmware_data->dec_op_int_buf_tbl_size;
 	}
 
 	for (i = 0; i < internal_buffer_count; i++) {
diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.c b/drivers/media/platform/qcom/iris/iris_vpu_common.c
index 69e6126dc4d95..7bba3b6209c20 100644
--- a/drivers/media/platform/qcom/iris/iris_vpu_common.c
+++ b/drivers/media/platform/qcom/iris/iris_vpu_common.c
@@ -63,7 +63,7 @@ static void iris_vpu_setup_ucregion_memory_map(struct iris_core *core)
 	writel(QTBL_ENABLE, core->reg_base + QTBL_INFO);
 
 	if (core->sfr_daddr) {
-		value = (u32)core->sfr_daddr + core->iris_platform_data->core_arch;
+		value = (u32)core->sfr_daddr + core->iris_firmware_data->core_arch;
 		writel(value, core->reg_base + SFR_ADDR);
 	}
 
@@ -149,7 +149,7 @@ int iris_vpu_prepare_pc(struct iris_core *core)
 	if (!wfi_status || !idle_status)
 		goto skip_power_off;
 
-	ret = core->hfi_ops->sys_pc_prep(core);
+	ret = core->hfi_sys_ops->sys_pc_prep(core);
 	if (ret)
 		goto skip_power_off;
 
@@ -472,7 +472,7 @@ int iris_vpu_power_on(struct iris_core *core)
 
 	iris_opp_set_rate(core->dev, freq);
 
-	core->iris_platform_data->set_preset_registers(core);
+	iris_vpu_set_preset_registers(core);
 
 	iris_vpu_interrupt_init(core);
 	core->intr_status = 0;
@@ -489,3 +489,8 @@ int iris_vpu_power_on(struct iris_core *core)
 
 	return ret;
 }
+
+void iris_vpu_set_preset_registers(struct iris_core *core)
+{
+	writel(0x0, core->reg_base + 0xb0088);
+}
diff --git a/drivers/media/platform/qcom/iris/iris_vpu_common.h b/drivers/media/platform/qcom/iris/iris_vpu_common.h
index dee3b1349c5e8..09799a375c142 100644
--- a/drivers/media/platform/qcom/iris/iris_vpu_common.h
+++ b/drivers/media/platform/qcom/iris/iris_vpu_common.h
@@ -42,4 +42,6 @@ int iris_vpu35_vpu4x_power_on_controller(struct iris_core *core);
 void iris_vpu35_vpu4x_program_bootup_registers(struct iris_core *core);
 u64 iris_vpu3x_vpu4x_calculate_frequency(struct iris_inst *inst, size_t data_size);
 
+void iris_vpu_set_preset_registers(struct iris_core *core);
+
 #endif
diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c
index 7e639760c41d9..a87e8afb23dfb 100644
--- a/drivers/media/platform/qcom/venus/core.c
+++ b/drivers/media/platform/qcom/venus/core.c
@@ -882,6 +882,7 @@ static const struct venus_resources sdm845_res_v2 = {
 	.vcodec_pmdomains = (const char *[]) { "venus", "vcodec0", "vcodec1" },
 	.vcodec_pmdomains_num = 3,
 	.opp_pmdomain = (const char *[]) { "cx" },
+	.opp_pmdomain_num = 1,
 	.vcodec_num = 2,
 	.max_load = 3110400,	/* 4096x2160@90 */
 	.hfi_version = HFI_VERSION_4XX,
@@ -933,6 +934,7 @@ static const struct venus_resources sc7180_res = {
 	.vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" },
 	.vcodec_pmdomains_num = 2,
 	.opp_pmdomain = (const char *[]) { "cx" },
+	.opp_pmdomain_num = 1,
 	.vcodec_num = 1,
 	.hfi_version = HFI_VERSION_4XX,
 	.vpu_version = VPU_VERSION_AR50,
@@ -949,6 +951,7 @@ static const struct venus_resources sc7180_res = {
 	.enc_nodename = "video-encoder",
 };
 
+#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS))
 static const struct freq_tbl sm8250_freq_table[] = {
 	{ 0, 444000000 },
 	{ 0, 366000000 },
@@ -991,7 +994,8 @@ static const struct venus_resources sm8250_res = {
 	.vcodec_clks_num = 1,
 	.vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" },
 	.vcodec_pmdomains_num = 2,
-	.opp_pmdomain = (const char *[]) { "mx" },
+	.opp_pmdomain = (const char *[]) { "mx", "mmcx" },
+	.opp_pmdomain_num = 2,
 	.vcodec_num = 1,
 	.max_load = 7833600,
 	.hfi_version = HFI_VERSION_6XX,
@@ -1053,6 +1057,7 @@ static const struct venus_resources sc7280_res = {
 	.vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" },
 	.vcodec_pmdomains_num = 2,
 	.opp_pmdomain = (const char *[]) { "cx" },
+	.opp_pmdomain_num = 1,
 	.vcodec_num = 1,
 	.hfi_version = HFI_VERSION_6XX,
 	.vpu_version = VPU_VERSION_IRIS2_1,
@@ -1069,6 +1074,7 @@ static const struct venus_resources sc7280_res = {
 	.dec_nodename = "video-decoder",
 	.enc_nodename = "video-encoder",
 };
+#endif
 
 static const struct bw_tbl qcm2290_bw_table_dec[] = {
 	{ 352800, 597000, 0, 746000, 0 }, /* 1080p@30 + 720p@30 */
@@ -1100,6 +1106,7 @@ static const struct venus_resources qcm2290_res = {
 	.vcodec_pmdomains = (const char *[]) { "venus", "vcodec0" },
 	.vcodec_pmdomains_num = 2,
 	.opp_pmdomain = (const char *[]) { "cx" },
+	.opp_pmdomain_num = 1,
 	.vcodec_num = 1,
 	.hfi_version = HFI_VERSION_4XX,
 	.vpu_version = VPU_VERSION_AR50_LITE,
@@ -1125,11 +1132,13 @@ static const struct of_device_id venus_dt_match[] = {
 	{ .compatible = "qcom,msm8998-venus", .data = &msm8998_res, },
 	{ .compatible = "qcom,qcm2290-venus", .data = &qcm2290_res, },
 	{ .compatible = "qcom,sc7180-venus", .data = &sc7180_res, },
-	{ .compatible = "qcom,sc7280-venus", .data = &sc7280_res, },
 	{ .compatible = "qcom,sdm660-venus", .data = &sdm660_res, },
 	{ .compatible = "qcom,sdm845-venus", .data = &sdm845_res, },
 	{ .compatible = "qcom,sdm845-venus-v2", .data = &sdm845_res_v2, },
+#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS))
+	{ .compatible = "qcom,sc7280-venus", .data = &sc7280_res, },
 	{ .compatible = "qcom,sm8250-venus", .data = &sm8250_res, },
+#endif
 	{ }
 };
 MODULE_DEVICE_TABLE(of, venus_dt_match);
diff --git a/drivers/media/platform/qcom/venus/core.h b/drivers/media/platform/qcom/venus/core.h
index 7506f5d0f609a..03804c30808eb 100644
--- a/drivers/media/platform/qcom/venus/core.h
+++ b/drivers/media/platform/qcom/venus/core.h
@@ -54,8 +54,10 @@ enum vpu_version {
 	VPU_VERSION_AR50,
 	VPU_VERSION_AR50_LITE,
 	VPU_VERSION_IRIS1,
+#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS))
 	VPU_VERSION_IRIS2,
 	VPU_VERSION_IRIS2_1,
+#endif
 };
 
 struct firmware_version {
@@ -83,6 +85,7 @@ struct venus_resources {
 	const char **vcodec_pmdomains;
 	unsigned int vcodec_pmdomains_num;
 	const char **opp_pmdomain;
+	unsigned int opp_pmdomain_num;
 	unsigned int vcodec_num;
 	const char * const resets[VIDC_RESETS_NUM_MAX];
 	unsigned int resets_num;
@@ -525,13 +528,22 @@ struct venus_inst {
 #define IS_V1(core)	((core)->res->hfi_version == HFI_VERSION_1XX)
 #define IS_V3(core)	((core)->res->hfi_version == HFI_VERSION_3XX)
 #define IS_V4(core)	((core)->res->hfi_version == HFI_VERSION_4XX)
+#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS))
 #define IS_V6(core)	((core)->res->hfi_version == HFI_VERSION_6XX)
+#else
+#define IS_V6(core)	(((void)(core), 0))
+#endif
 
 #define IS_AR50(core)		((core)->res->vpu_version == VPU_VERSION_AR50)
 #define IS_AR50_LITE(core)	((core)->res->vpu_version == VPU_VERSION_AR50_LITE)
 #define IS_IRIS1(core)		((core)->res->vpu_version == VPU_VERSION_IRIS1)
+#if (!IS_ENABLED(CONFIG_VIDEO_QCOM_IRIS))
 #define IS_IRIS2(core)		((core)->res->vpu_version == VPU_VERSION_IRIS2)
 #define IS_IRIS2_1(core)	((core)->res->vpu_version == VPU_VERSION_IRIS2_1)
+#else
+#define IS_IRIS2(core)		(((void)(core), 0))
+#define IS_IRIS2_1(core)	(((void)(core), 0))
+#endif
 
 static inline bool is_lite(struct venus_core *core)
 {
diff --git a/drivers/media/platform/qcom/venus/pm_helpers.c b/drivers/media/platform/qcom/venus/pm_helpers.c
index f0269524ac70e..14a4e8311a643 100644
--- a/drivers/media/platform/qcom/venus/pm_helpers.c
+++ b/drivers/media/platform/qcom/venus/pm_helpers.c
@@ -887,7 +887,7 @@ static int vcodec_domains_get(struct venus_core *core)
 	};
 	struct dev_pm_domain_attach_data opp_pd_data = {
 		.pd_names = res->opp_pmdomain,
-		.num_pd_names = 1,
+		.num_pd_names = res->opp_pmdomain_num,
 		.pd_flags = PD_FLAG_DEV_LINK_ON | PD_FLAG_REQUIRED_OPP,
 	};
 
@@ -904,6 +904,12 @@ static int vcodec_domains_get(struct venus_core *core)
 
 	/* Attach the power domain for setting performance state */
 	ret = devm_pm_domain_attach_list(dev, &opp_pd_data, &core->opp_pmdomain);
+	/* backwards compatibility for incomplete ABI SM8250 */
+	if (ret == -ENODEV &&
+	    of_device_is_compatible(dev->of_node, "qcom,sm8250-venus")) {
+		opp_pd_data.num_pd_names--;
+		ret = devm_pm_domain_attach_list(dev, &opp_pd_data, &core->opp_pmdomain);
+	}
 	if (ret < 0)
 		return ret;
 
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru-regs.h b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru-regs.h
index a5a57369ef0eb..23cb50ee8e57f 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru-regs.h
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru-regs.h
@@ -14,12 +14,11 @@
 
 #define CRUnIE_EFE			BIT(17)
 
-#define CRUnIE2_FSxE(x)			BIT(((x) * 3))
 #define CRUnIE2_FExE(x)			BIT(((x) * 3) + 1)
 
 #define CRUnINTS_SFS			BIT(16)
 
-#define CRUnINTS2_FSxS(x)		BIT(((x) * 3))
+#define CRUnINTS2_FExS(x)		BIT(((x) * 3) + 1)
 
 #define CRUnRST_VRESETN			BIT(0)
 
@@ -60,6 +59,7 @@
 #define ICnMC_CSCTHR			BIT(5)
 #define ICnMC_INF(x)			((x) << 16)
 #define ICnMC_VCSEL(x)			((x) << 22)
+#define ICnMC_VCSEL_MASK		GENMASK(23, 22)
 #define ICnMC_INF_MASK			GENMASK(21, 16)
 
 #define ICnMS_IA			BIT(2)
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h
index 3a200db157308..5bf334e173d2b 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-cru.h
@@ -9,7 +9,9 @@
 #define __RZG2L_CRU__
 
 #include <linux/irqreturn.h>
+#include <linux/mutex.h>
 #include <linux/reset.h>
+#include <linux/spinlock.h>
 
 #include <media/v4l2-async.h>
 #include <media/v4l2-dev.h>
@@ -36,20 +38,6 @@ enum rzg2l_csi2_pads {
 
 struct rzg2l_cru_dev;
 
-/**
- * enum rzg2l_cru_dma_state - DMA states
- * @RZG2L_CRU_DMA_STOPPED:   No operation in progress
- * @RZG2L_CRU_DMA_STARTING:  Capture starting up
- * @RZG2L_CRU_DMA_RUNNING:   Operation in progress have buffers
- * @RZG2L_CRU_DMA_STOPPING:  Stopping operation
- */
-enum rzg2l_cru_dma_state {
-	RZG2L_CRU_DMA_STOPPED = 0,
-	RZG2L_CRU_DMA_STARTING,
-	RZG2L_CRU_DMA_RUNNING,
-	RZG2L_CRU_DMA_STOPPING,
-};
-
 struct rzg2l_cru_csi {
 	struct v4l2_async_connection *asd;
 	struct v4l2_subdev *subdev;
@@ -109,7 +97,6 @@ struct rzg2l_cru_info {
  * @v4l2_dev:		V4L2 device
  * @num_buf:		Holds the current number of buffers enabled
  * @svc_channel:	SVC0/1/2/3 to use for RZ/G3E
- * @buf_addr:		Memory addresses where current video data is written.
  * @notifier:		V4L2 asynchronous subdevs notifier
  *
  * @ip:			Image processing subdev info
@@ -118,6 +105,11 @@ struct rzg2l_cru_info {
  * @mdev_lock:		protects the count, notifier and csi members
  * @pad:		media pad for the video device entity
  *
+ * @hw_lock:		protects the @active_slot counter, hardware programming
+ *			of slot addresses and the @buf_addr[] list
+ * @buf_addr:		Memory addresses where current video data is written
+ * @active_slot:	The slot in use
+ *
  * @lock:		protects @queue
  * @queue:		vb2 buffers queue
  * @scratch:		cpu address for scratch buffer
@@ -147,8 +139,6 @@ struct rzg2l_cru_dev {
 	u8 num_buf;
 
 	u8 svc_channel;
-	dma_addr_t buf_addr[RZG2L_CRU_HW_BUFFER_DEFAULT];
-
 	struct v4l2_async_notifier notifier;
 
 	struct rzg2l_cru_ip ip;
@@ -157,6 +147,10 @@ struct rzg2l_cru_dev {
 	struct mutex mdev_lock;
 	struct media_pad pad;
 
+	spinlock_t hw_lock;
+	dma_addr_t buf_addr[RZG2L_CRU_HW_BUFFER_DEFAULT];
+	unsigned int active_slot;
+
 	struct mutex lock;
 	struct vb2_queue queue;
 	void *scratch;
@@ -166,7 +160,6 @@ struct rzg2l_cru_dev {
 	struct vb2_v4l2_buffer *queue_buf[RZG2L_CRU_HW_BUFFER_MAX];
 	struct list_head buf_list;
 	unsigned int sequence;
-	enum rzg2l_cru_dma_state state;
 
 	struct v4l2_pix_format format;
 };
diff --git a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c
index 162e2ace69318..5185a547461d7 100644
--- a/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c
+++ b/drivers/media/platform/renesas/rzg2l-cru/rzg2l-video.c
@@ -42,6 +42,24 @@ struct rzg2l_cru_buffer {
 #define to_buf_list(vb2_buffer) \
 	(&container_of(vb2_buffer, struct rzg2l_cru_buffer, vb)->list)
 
+/*
+ * The CRU hardware cycles over its slots when transferring frames. All drivers
+ * structure that contains programming data for the slots, such as the memory
+ * destination addresses have to be iterated as they were circular buffers.
+ *
+ * Provide here utilities to iterate over slots and the associated data.
+ */
+static inline unsigned int rzg2l_cru_slot_next(struct rzg2l_cru_dev *cru,
+					       unsigned int slot)
+{
+	return (slot + 1) % cru->num_buf;
+}
+
+/* Start cycling on cru slots from the one after 'start'. */
+#define for_each_cru_slot_from(cru, slot, start)			\
+	for ((slot) = rzg2l_cru_slot_next((cru), (start));			\
+	     (slot) != (start); (slot) = rzg2l_cru_slot_next((cru), (slot)))
+
 /* -----------------------------------------------------------------------------
  * DMA operations
  */
@@ -105,28 +123,36 @@ __rzg2l_cru_read_constant(struct rzg2l_cru_dev *cru, u32 offset)
 	 __rzg2l_cru_read_constant(cru, offset) : \
 	 __rzg2l_cru_read(cru, offset))
 
-/* Need to hold qlock before calling */
-static void return_unused_buffers(struct rzg2l_cru_dev *cru,
-				  enum vb2_buffer_state state)
+static void rzg2l_cru_return_buffers(struct rzg2l_cru_dev *cru,
+				     enum vb2_buffer_state state)
 {
 	struct rzg2l_cru_buffer *buf, *node;
-	unsigned long flags;
-	unsigned int i;
 
-	spin_lock_irqsave(&cru->qlock, flags);
-	for (i = 0; i < cru->num_buf; i++) {
-		if (cru->queue_buf[i]) {
-			vb2_buffer_done(&cru->queue_buf[i]->vb2_buf,
-					state);
-			cru->queue_buf[i] = NULL;
+	scoped_guard(spinlock_irq, &cru->hw_lock) {
+		/* Return the buffer in progress first, if not completed yet. */
+		unsigned int slot = cru->active_slot;
+
+		if (cru->queue_buf[slot]) {
+			vb2_buffer_done(&cru->queue_buf[slot]->vb2_buf, state);
+			cru->queue_buf[slot] = NULL;
+		}
+
+		/* Return all the pending buffers after the active one. */
+		for_each_cru_slot_from(cru, slot, cru->active_slot) {
+			if (!cru->queue_buf[slot])
+				continue;
+
+			vb2_buffer_done(&cru->queue_buf[slot]->vb2_buf, state);
+			cru->queue_buf[slot] = NULL;
 		}
 	}
 
+	guard(spinlock_irq)(&cru->qlock);
+
 	list_for_each_entry_safe(buf, node, &cru->buf_list, list) {
 		vb2_buffer_done(&buf->vb.vb2_buf, state);
 		list_del(&buf->list);
 	}
-	spin_unlock_irqrestore(&cru->qlock, flags);
 }
 
 static int rzg2l_cru_queue_setup(struct vb2_queue *vq, unsigned int *nbuffers,
@@ -165,13 +191,9 @@ static void rzg2l_cru_buffer_queue(struct vb2_buffer *vb)
 {
 	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
 	struct rzg2l_cru_dev *cru = vb2_get_drv_priv(vb->vb2_queue);
-	unsigned long flags;
-
-	spin_lock_irqsave(&cru->qlock, flags);
 
+	guard(spinlock_irq)(&cru->qlock);
 	list_add_tail(to_buf_list(vbuf), &cru->buf_list);
-
-	spin_unlock_irqrestore(&cru->qlock, flags);
 }
 
 static void rzg2l_cru_set_slot_addr(struct rzg2l_cru_dev *cru,
@@ -192,45 +214,52 @@ static void rzg2l_cru_set_slot_addr(struct rzg2l_cru_dev *cru,
 }
 
 /*
- * Moves a buffer from the queue to the HW slot. If no buffer is
- * available use the scratch buffer. The scratch buffer is never
- * returned to userspace, its only function is to enable the capture
- * loop to keep running.
+ * Move as many buffers as possible from the queue to HW slots If no buffer is
+ * available use the scratch buffer. The scratch buffer is never returned to
+ * userspace, its only function is to enable the capture loop to keep running.
+ *
+ * @cru: the CRU device
+ * @slot: the slot that has just completed
  */
 static void rzg2l_cru_fill_hw_slot(struct rzg2l_cru_dev *cru, int slot)
 {
-	struct vb2_v4l2_buffer *vbuf;
 	struct rzg2l_cru_buffer *buf;
+	struct vb2_v4l2_buffer *vbuf;
+	unsigned int next_slot;
 	dma_addr_t phys_addr;
 
-	/* A already populated slot shall never be overwritten. */
-	if (WARN_ON(cru->queue_buf[slot]))
-		return;
+	lockdep_assert_held(&cru->hw_lock);
 
-	dev_dbg(cru->dev, "Filling HW slot: %d\n", slot);
+	/* Find the next slot which hasn't a valid address programmed. */
+	for_each_cru_slot_from(cru, next_slot, slot) {
+		if (cru->queue_buf[next_slot])
+			continue;
 
-	if (list_empty(&cru->buf_list)) {
-		cru->queue_buf[slot] = NULL;
-		phys_addr = cru->scratch_phys;
-	} else {
-		/* Keep track of buffer we give to HW */
-		buf = list_entry(cru->buf_list.next,
-				 struct rzg2l_cru_buffer, list);
-		vbuf = &buf->vb;
-		list_del_init(to_buf_list(vbuf));
-		cru->queue_buf[slot] = vbuf;
+		scoped_guard(spinlock_irqsave, &cru->qlock) {
+			buf = list_first_entry_or_null(&cru->buf_list,
+						       struct rzg2l_cru_buffer, list);
+			if (buf)
+				list_del_init(&buf->list);
+		}
 
-		/* Setup DMA */
+		if (!buf) {
+			/* Direct frames to the scratch buffer. */
+			phys_addr = cru->scratch_phys;
+			cru->queue_buf[next_slot] = NULL;
+			rzg2l_cru_set_slot_addr(cru, next_slot, phys_addr);
+			return;
+		}
+
+		vbuf = &buf->vb;
+		cru->queue_buf[next_slot] = vbuf;
 		phys_addr = vb2_dma_contig_plane_dma_addr(&vbuf->vb2_buf, 0);
+		rzg2l_cru_set_slot_addr(cru, next_slot, phys_addr);
 	}
-
-	rzg2l_cru_set_slot_addr(cru, slot, phys_addr);
 }
 
 static void rzg2l_cru_initialize_axi(struct rzg2l_cru_dev *cru)
 {
 	const struct rzg2l_cru_info *info = cru->info;
-	unsigned int slot;
 	u32 amnaxiattr;
 
 	/*
@@ -239,8 +268,14 @@ static void rzg2l_cru_initialize_axi(struct rzg2l_cru_dev *cru)
 	 */
 	rzg2l_cru_write(cru, AMnMBVALID, AMnMBVALID_MBVALID(cru->num_buf - 1));
 
-	for (slot = 0; slot < cru->num_buf; slot++)
-		rzg2l_cru_fill_hw_slot(cru, slot);
+	/*
+	 * Program slot#0 with the first available buffer, if any. Pass to the
+	 * function 'num_buf - 1' as rzg2l_cru_fill_hw_slot() calculates which
+	 * is the next slot to program.
+	 */
+	scoped_guard(spinlock_irq, &cru->hw_lock) {
+		rzg2l_cru_fill_hw_slot(cru, cru->num_buf - 1);
+	}
 
 	if (info->has_stride) {
 		u32 stride = cru->format.bytesperline;
@@ -262,19 +297,24 @@ static void rzg2l_cru_csi2_setup(struct rzg2l_cru_dev *cru,
 				 u8 csi_vc)
 {
 	const struct rzg2l_cru_info *info = cru->info;
-	u32 icnmc = ICnMC_INF(ip_fmt->datatype);
+	u32 icnmc = rzg2l_cru_read(cru, info->image_conv) & ~(ICnMC_INF_MASK |
+							      ICnMC_VCSEL_MASK);
+	icnmc |= ICnMC_INF(ip_fmt->datatype);
 
+	/*
+	 * VC filtering goes through SVC register on G3E/V2H.
+	 *
+	 * FIXME: virtual channel filtering is likely broken and only VC=0
+	 * works.
+	 */
 	if (cru->info->regs[ICnSVC]) {
 		rzg2l_cru_write(cru, ICnSVCNUM, csi_vc);
 		rzg2l_cru_write(cru, ICnSVC, ICnSVC_SVC0(0) | ICnSVC_SVC1(1) |
 				ICnSVC_SVC2(2) | ICnSVC_SVC3(3));
+	} else {
+		icnmc |= ICnMC_VCSEL(csi_vc);
 	}
 
-	icnmc |= rzg2l_cru_read(cru, info->image_conv) & ~ICnMC_INF_MASK;
-
-	/* Set virtual channel CSI2 */
-	icnmc |= ICnMC_VCSEL(csi_vc);
-
 	rzg2l_cru_write(cru, info->image_conv, icnmc);
 }
 
@@ -340,30 +380,24 @@ bool rzg2l_fifo_empty(struct rzg2l_cru_dev *cru)
 void rzg2l_cru_stop_image_processing(struct rzg2l_cru_dev *cru)
 {
 	unsigned int retries = 0;
-	unsigned long flags;
 	u32 icnms;
 
-	spin_lock_irqsave(&cru->qlock, flags);
-
-	/* Disable and clear the interrupt */
-	cru->info->disable_interrupts(cru);
+	scoped_guard(spinlock_irq, &cru->hw_lock) {
+		/* Disable and clear the interrupt */
+		cru->info->disable_interrupts(cru);
+	}
 
 	/* Stop the operation of image conversion */
 	rzg2l_cru_write(cru, ICnEN, 0);
 
 	/* Wait for streaming to stop */
-	while ((rzg2l_cru_read(cru, ICnMS) & ICnMS_IA) && retries++ < RZG2L_RETRIES) {
-		spin_unlock_irqrestore(&cru->qlock, flags);
+	while ((rzg2l_cru_read(cru, ICnMS) & ICnMS_IA) && retries++ < RZG2L_RETRIES)
 		msleep(RZG2L_TIMEOUT_MS);
-		spin_lock_irqsave(&cru->qlock, flags);
-	}
 
 	icnms = rzg2l_cru_read(cru, ICnMS) & ICnMS_IA;
 	if (icnms)
 		dev_err(cru->dev, "Failed stop HW, something is seriously broken\n");
 
-	cru->state = RZG2L_CRU_DMA_STOPPED;
-
 	/* Wait until the FIFO becomes empty */
 	for (retries = 5; retries > 0; retries--) {
 		if (cru->info->fifo_empty(cru))
@@ -400,8 +434,6 @@ void rzg2l_cru_stop_image_processing(struct rzg2l_cru_dev *cru)
 
 	/* Resets the image processing module */
 	rzg2l_cru_write(cru, CRUnRST, 0);
-
-	spin_unlock_irqrestore(&cru->qlock, flags);
 }
 
 static int rzg2l_cru_get_virtual_channel(struct rzg2l_cru_dev *cru)
@@ -435,7 +467,6 @@ static int rzg2l_cru_get_virtual_channel(struct rzg2l_cru_dev *cru)
 
 void rzg3e_cru_enable_interrupts(struct rzg2l_cru_dev *cru)
 {
-	rzg2l_cru_write(cru, CRUnIE2, CRUnIE2_FSxE(cru->svc_channel));
 	rzg2l_cru_write(cru, CRUnIE2, CRUnIE2_FExE(cru->svc_channel));
 }
 
@@ -461,7 +492,6 @@ void rzg2l_cru_disable_interrupts(struct rzg2l_cru_dev *cru)
 int rzg2l_cru_start_image_processing(struct rzg2l_cru_dev *cru)
 {
 	struct v4l2_mbus_framefmt *fmt = rzg2l_cru_ip_get_src_fmt(cru);
-	unsigned long flags;
 	u8 csi_vc;
 	int ret;
 
@@ -471,8 +501,6 @@ int rzg2l_cru_start_image_processing(struct rzg2l_cru_dev *cru)
 	csi_vc = ret;
 	cru->svc_channel = csi_vc;
 
-	spin_lock_irqsave(&cru->qlock, flags);
-
 	/* Select a video input */
 	rzg2l_cru_write(cru, CRUnCTRL, CRUnCTRL_VINSEL(0));
 
@@ -488,7 +516,6 @@ int rzg2l_cru_start_image_processing(struct rzg2l_cru_dev *cru)
 	/* Initialize image convert */
 	ret = rzg2l_cru_initialize_image_conv(cru, fmt, csi_vc);
 	if (ret) {
-		spin_unlock_irqrestore(&cru->qlock, flags);
 		return ret;
 	}
 
@@ -498,8 +525,6 @@ int rzg2l_cru_start_image_processing(struct rzg2l_cru_dev *cru)
 	/* Enable image processing reception */
 	rzg2l_cru_write(cru, ICnEN, ICnEN_ICEN);
 
-	spin_unlock_irqrestore(&cru->qlock, flags);
-
 	return 0;
 }
 
@@ -560,69 +585,36 @@ static int rzg2l_cru_set_stream(struct rzg2l_cru_dev *cru, int on)
 
 static void rzg2l_cru_stop_streaming(struct rzg2l_cru_dev *cru)
 {
-	cru->state = RZG2L_CRU_DMA_STOPPING;
-
 	rzg2l_cru_set_stream(cru, 0);
 }
 
 irqreturn_t rzg2l_cru_irq(int irq, void *data)
 {
 	struct rzg2l_cru_dev *cru = data;
-	unsigned int handled = 0;
-	unsigned long flags;
 	u32 irq_status;
 	u32 amnmbs;
 	int slot;
 
-	spin_lock_irqsave(&cru->qlock, flags);
-
 	irq_status = rzg2l_cru_read(cru, CRUnINTS);
 	if (!irq_status)
-		goto done;
-
-	handled = 1;
+		return IRQ_NONE;
 
 	rzg2l_cru_write(cru, CRUnINTS, rzg2l_cru_read(cru, CRUnINTS));
 
-	/* Nothing to do if capture status is 'RZG2L_CRU_DMA_STOPPED' */
-	if (cru->state == RZG2L_CRU_DMA_STOPPED) {
-		dev_dbg(cru->dev, "IRQ while state stopped\n");
-		goto done;
-	}
-
-	/* Increase stop retries if capture status is 'RZG2L_CRU_DMA_STOPPING' */
-	if (cru->state == RZG2L_CRU_DMA_STOPPING) {
-		if (irq_status & CRUnINTS_SFS)
-			dev_dbg(cru->dev, "IRQ while state stopping\n");
-		goto done;
-	}
+	/* Calculate slot and prepare for new capture. */
+	guard(spinlock_irqsave)(&cru->hw_lock);
 
-	/* Prepare for capture and update state */
 	amnmbs = rzg2l_cru_read(cru, AMnMBS);
-	slot = amnmbs & AMnMBS_MBSTS;
+	cru->active_slot = amnmbs & AMnMBS_MBSTS;
 
 	/*
 	 * AMnMBS.MBSTS indicates the destination of Memory Bank (MB).
 	 * Recalculate to get the current transfer complete MB.
 	 */
-	if (slot == 0)
+	if (cru->active_slot == 0)
 		slot = cru->num_buf - 1;
 	else
-		slot--;
-
-	/*
-	 * To hand buffers back in a known order to userspace start
-	 * to capture first from slot 0.
-	 */
-	if (cru->state == RZG2L_CRU_DMA_STARTING) {
-		if (slot != 0) {
-			dev_dbg(cru->dev, "Starting sync slot: %d\n", slot);
-			goto done;
-		}
-
-		dev_dbg(cru->dev, "Capture start synced!\n");
-		cru->state = RZG2L_CRU_DMA_RUNNING;
-	}
+		slot = cru->active_slot - 1;
 
 	/* Capture frame */
 	if (cru->queue_buf[slot]) {
@@ -632,9 +624,6 @@ irqreturn_t rzg2l_cru_irq(int irq, void *data)
 		vb2_buffer_done(&cru->queue_buf[slot]->vb2_buf,
 				VB2_BUF_STATE_DONE);
 		cru->queue_buf[slot] = NULL;
-	} else {
-		/* Scratch buffer was used, dropping frame. */
-		dev_dbg(cru->dev, "Dropping frame %u\n", cru->sequence);
 	}
 
 	cru->sequence++;
@@ -642,35 +631,7 @@ irqreturn_t rzg2l_cru_irq(int irq, void *data)
 	/* Prepare for next frame */
 	rzg2l_cru_fill_hw_slot(cru, slot);
 
-done:
-	spin_unlock_irqrestore(&cru->qlock, flags);
-
-	return IRQ_RETVAL(handled);
-}
-
-static int rzg3e_cru_get_current_slot(struct rzg2l_cru_dev *cru)
-{
-	u64 amnmadrs;
-	int slot;
-
-	/*
-	 * When AMnMADRSL is read, AMnMADRSH of the higher-order
-	 * address also latches the address.
-	 *
-	 * AMnMADRSH must be read after AMnMADRSL has been read.
-	 */
-	amnmadrs = rzg2l_cru_read(cru, AMnMADRSL);
-	amnmadrs |= (u64)rzg2l_cru_read(cru, AMnMADRSH) << 32;
-
-	/* Ensure amnmadrs is within this buffer range */
-	for (slot = 0; slot < cru->num_buf; slot++) {
-		if (amnmadrs >= cru->buf_addr[slot] &&
-		    amnmadrs < cru->buf_addr[slot] + cru->format.sizeimage)
-			return slot;
-	}
-
-	dev_err(cru->dev, "Invalid MB address 0x%llx (out of range)\n", amnmadrs);
-	return -EINVAL;
+	return IRQ_HANDLED;
 }
 
 irqreturn_t rzg3e_cru_irq(int irq, void *data)
@@ -679,69 +640,31 @@ irqreturn_t rzg3e_cru_irq(int irq, void *data)
 	u32 irq_status;
 	int slot;
 
-	scoped_guard(spinlock, &cru->qlock) {
-		irq_status = rzg2l_cru_read(cru, CRUnINTS2);
-		if (!irq_status)
-			return IRQ_NONE;
-
-		dev_dbg(cru->dev, "CRUnINTS2 0x%x\n", irq_status);
-
-		rzg2l_cru_write(cru, CRUnINTS2, rzg2l_cru_read(cru, CRUnINTS2));
-
-		/* Nothing to do if capture status is 'RZG2L_CRU_DMA_STOPPED' */
-		if (cru->state == RZG2L_CRU_DMA_STOPPED) {
-			dev_dbg(cru->dev, "IRQ while state stopped\n");
-			return IRQ_HANDLED;
-		}
+	irq_status = rzg2l_cru_read(cru, CRUnINTS2);
+	if (!irq_status)
+		return IRQ_NONE;
 
-		if (cru->state == RZG2L_CRU_DMA_STOPPING) {
-			if (irq_status & CRUnINTS2_FSxS(0) ||
-			    irq_status & CRUnINTS2_FSxS(1) ||
-			    irq_status & CRUnINTS2_FSxS(2) ||
-			    irq_status & CRUnINTS2_FSxS(3))
-				dev_dbg(cru->dev, "IRQ while state stopping\n");
-			return IRQ_HANDLED;
-		}
+	rzg2l_cru_write(cru, CRUnINTS2, rzg2l_cru_read(cru, CRUnINTS2));
 
-		slot = rzg3e_cru_get_current_slot(cru);
-		if (slot < 0)
-			return IRQ_HANDLED;
-
-		dev_dbg(cru->dev, "Current written slot: %d\n", slot);
-		cru->buf_addr[slot] = 0;
-
-		/*
-		 * To hand buffers back in a known order to userspace start
-		 * to capture first from slot 0.
-		 */
-		if (cru->state == RZG2L_CRU_DMA_STARTING) {
-			if (slot != 0) {
-				dev_dbg(cru->dev, "Starting sync slot: %d\n", slot);
-				return IRQ_HANDLED;
-			}
-			dev_dbg(cru->dev, "Capture start synced!\n");
-			cru->state = RZG2L_CRU_DMA_RUNNING;
-		}
+	guard(spinlock)(&cru->hw_lock);
+	slot = cru->active_slot;
+	cru->active_slot = rzg2l_cru_slot_next(cru, cru->active_slot);
 
-		/* Capture frame */
-		if (cru->queue_buf[slot]) {
-			struct vb2_v4l2_buffer *buf = cru->queue_buf[slot];
+	/* Capture frame */
+	if (cru->queue_buf[slot]) {
+		struct vb2_v4l2_buffer *buf = cru->queue_buf[slot];
 
-			buf->field = cru->format.field;
-			buf->sequence = cru->sequence;
-			buf->vb2_buf.timestamp = ktime_get_ns();
-			vb2_buffer_done(&buf->vb2_buf, VB2_BUF_STATE_DONE);
-			cru->queue_buf[slot] = NULL;
-		} else {
-			/* Scratch buffer was used, dropping frame. */
-			dev_dbg(cru->dev, "Dropping frame %u\n", cru->sequence);
-		}
+		buf->field = cru->format.field;
+		buf->sequence = cru->sequence;
+		buf->vb2_buf.timestamp = ktime_get_ns();
+		vb2_buffer_done(&buf->vb2_buf, VB2_BUF_STATE_DONE);
+		cru->queue_buf[slot] = NULL;
+	}
 
-		cru->sequence++;
+	cru->sequence++;
 
-		/* Prepare for next frame */
-		rzg2l_cru_fill_hw_slot(cru, slot);
-	}
+	/* Prepare for next frame */
+	rzg2l_cru_fill_hw_slot(cru, slot);
 
 	return IRQ_HANDLED;
 }
@@ -777,21 +700,21 @@ static int rzg2l_cru_start_streaming_vq(struct vb2_queue *vq, unsigned int count
 	cru->scratch = dma_alloc_coherent(cru->dev, cru->format.sizeimage,
 					  &cru->scratch_phys, GFP_KERNEL);
 	if (!cru->scratch) {
-		return_unused_buffers(cru, VB2_BUF_STATE_QUEUED);
+		rzg2l_cru_return_buffers(cru, VB2_BUF_STATE_QUEUED);
 		dev_err(cru->dev, "Failed to allocate scratch buffer\n");
 		ret = -ENOMEM;
 		goto assert_presetn;
 	}
 
+	cru->active_slot = 0;
 	cru->sequence = 0;
 
 	ret = rzg2l_cru_set_stream(cru, 1);
 	if (ret) {
-		return_unused_buffers(cru, VB2_BUF_STATE_QUEUED);
+		rzg2l_cru_return_buffers(cru, VB2_BUF_STATE_QUEUED);
 		goto out;
 	}
 
-	cru->state = RZG2L_CRU_DMA_STARTING;
 	dev_dbg(cru->dev, "Starting to capture\n");
 	return 0;
 
@@ -824,7 +747,7 @@ static void rzg2l_cru_stop_streaming_vq(struct vb2_queue *vq)
 	dma_free_coherent(cru->dev, cru->format.sizeimage,
 			  cru->scratch, cru->scratch_phys);
 
-	return_unused_buffers(cru, VB2_BUF_STATE_ERROR);
+	rzg2l_cru_return_buffers(cru, VB2_BUF_STATE_ERROR);
 
 	reset_control_assert(cru->presetn);
 	clk_disable_unprepare(cru->vclk);
@@ -861,10 +784,9 @@ int rzg2l_cru_dma_register(struct rzg2l_cru_dev *cru)
 	mutex_init(&cru->lock);
 	INIT_LIST_HEAD(&cru->buf_list);
 
+	spin_lock_init(&cru->hw_lock);
 	spin_lock_init(&cru->qlock);
 
-	cru->state = RZG2L_CRU_DMA_STOPPED;
-
 	for (i = 0; i < RZG2L_CRU_HW_BUFFER_MAX; i++)
 		cru->queue_buf[i] = NULL;
 
diff --git a/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-video.c b/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-video.c
index b167f1bab7ef2..932fed38cf3fc 100644
--- a/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-video.c
+++ b/drivers/media/platform/renesas/rzv2h-ivc/rzv2h-ivc-video.c
@@ -297,12 +297,33 @@ static int rzv2h_ivc_start_streaming(struct vb2_queue *q, unsigned int count)
 static void rzv2h_ivc_stop_streaming(struct vb2_queue *q)
 {
 	struct rzv2h_ivc *ivc = vb2_get_drv_priv(q);
-	u32 val = 0;
+	unsigned int loop = 5;
 
-	rzv2h_ivc_write(ivc, RZV2H_IVC_REG_FM_STOP, RZV2H_IVC_REG_FM_STOP_FSTOP);
-	readl_poll_timeout(ivc->base + RZV2H_IVC_REG_FM_STOP,
-			   val, !(val & RZV2H_IVC_REG_FM_STOP_FSTOP),
-			   10 * USEC_PER_MSEC, 250 * USEC_PER_MSEC);
+	/*
+	 * If no frame transfer is in progress, we're done, otherwise, wait for
+	 * the transfer to complete.
+	 *
+	 * Transferring a 1920x1080@10bit frame to the ISP takes less than 5
+	 * msec so sleep for 2.5 msec (+- 25%) and give up after 5 attempts.
+	 */
+	for (; loop > 0; loop--) {
+		unsigned int vvalid_ifp;
+
+		/*
+		 * Inspect the ivc->vvalid_ifp variable holding the spinlock not
+		 * to the race with the rzv2h_ivc_buffer_done() call in the irq
+		 * handler.
+		 */
+		scoped_guard(spinlock_irq, &ivc->spinlock) {
+			vvalid_ifp = ivc->vvalid_ifp;
+		}
+		if (vvalid_ifp < 2)
+			break;
+
+		fsleep(2500);
+	}
+	if (!loop)
+		dev_err(ivc->dev, "Failed to stop streaming\n");
 
 	rzv2h_ivc_return_buffers(ivc, VB2_BUF_STATE_ERROR);
 	video_device_pipeline_stop(&ivc->vdev.dev);
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_brx.c b/drivers/media/platform/renesas/vsp1/vsp1_brx.c
index b1a2c68e99446..d150a92b26179 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_brx.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_brx.c
@@ -130,15 +130,12 @@ static int brx_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_brx *brx = to_brx(subdev);
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
 
-	mutex_lock(&brx->entity.lock);
+	guard(mutex)(&brx->entity.lock);
 
 	state = vsp1_entity_get_state(&brx->entity, sd_state, fmt->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	brx_try_format(brx, state, fmt->pad, &fmt->format);
 
@@ -172,9 +169,7 @@ static int brx_set_format(struct v4l2_subdev *subdev,
 		*format = fmt->format;
 	}
 
-done:
-	mutex_unlock(&brx->entity.lock);
-	return ret;
+	return 0;
 }
 
 static int brx_get_selection(struct v4l2_subdev *subdev,
@@ -201,9 +196,10 @@ static int brx_get_selection(struct v4l2_subdev *subdev,
 		if (!state)
 			return -EINVAL;
 
-		mutex_lock(&brx->entity.lock);
-		sel->r = *v4l2_subdev_state_get_compose(state, sel->pad);
-		mutex_unlock(&brx->entity.lock);
+		scoped_guard(mutex, &brx->entity.lock) {
+			sel->r = *v4l2_subdev_state_get_compose(state, sel->pad);
+		}
+
 		return 0;
 
 	default:
@@ -219,7 +215,6 @@ static int brx_set_selection(struct v4l2_subdev *subdev,
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *compose;
-	int ret = 0;
 
 	if (sel->pad == brx->entity.source_pad)
 		return -EINVAL;
@@ -227,13 +222,11 @@ static int brx_set_selection(struct v4l2_subdev *subdev,
 	if (sel->target != V4L2_SEL_TGT_COMPOSE)
 		return -EINVAL;
 
-	mutex_lock(&brx->entity.lock);
+	guard(mutex)(&brx->entity.lock);
 
 	state = vsp1_entity_get_state(&brx->entity, sd_state, sel->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	/*
 	 * The compose rectangle top left corner must be inside the output
@@ -254,9 +247,7 @@ static int brx_set_selection(struct v4l2_subdev *subdev,
 	compose = v4l2_subdev_state_get_compose(state, sel->pad);
 	*compose = sel->r;
 
-done:
-	mutex_unlock(&brx->entity.lock);
-	return ret;
+	return 0;
 }
 
 static const struct v4l2_subdev_pad_ops brx_pad_ops = {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_clu.c b/drivers/media/platform/renesas/vsp1/vsp1_clu.c
index 04c466c4da814..a6e4bcab5101a 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_clu.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_clu.c
@@ -53,9 +53,9 @@ static int clu_set_table(struct vsp1_clu *clu, struct v4l2_ctrl *ctrl)
 	for (i = 0; i < CLU_SIZE; ++i)
 		vsp1_dl_body_write(dlb, VI6_CLU_DATA, ctrl->p_new.p_u32[i]);
 
-	spin_lock_irq(&clu->lock);
-	swap(clu->clu, dlb);
-	spin_unlock_irq(&clu->lock);
+	scoped_guard(spinlock_irq, &clu->lock) {
+		swap(clu->clu, dlb);
+	}
 
 	vsp1_dl_body_put(dlb);
 	return 0;
@@ -162,7 +162,6 @@ static void clu_configure_frame(struct vsp1_entity *entity,
 {
 	struct vsp1_clu *clu = to_clu(&entity->subdev);
 	struct vsp1_dl_body *clu_dlb;
-	unsigned long flags;
 	u32 ctrl = VI6_CLU_CTRL_AAI | VI6_CLU_CTRL_MVS | VI6_CLU_CTRL_EN;
 
 	/* 2D mode can only be used with the YCbCr pixel encoding. */
@@ -173,10 +172,10 @@ static void clu_configure_frame(struct vsp1_entity *entity,
 
 	vsp1_clu_write(clu, dlb, VI6_CLU_CTRL, ctrl);
 
-	spin_lock_irqsave(&clu->lock, flags);
-	clu_dlb = clu->clu;
-	clu->clu = NULL;
-	spin_unlock_irqrestore(&clu->lock, flags);
+	scoped_guard(spinlock_irqsave, &clu->lock) {
+		clu_dlb = clu->clu;
+		clu->clu = NULL;
+	}
 
 	if (clu_dlb) {
 		vsp1_dl_list_add_body(dl, clu_dlb);
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_dl.c b/drivers/media/platform/renesas/vsp1/vsp1_dl.c
index 6c5578d9d2dea..3dc74fed91dcd 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_dl.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_dl.c
@@ -336,9 +336,8 @@ void vsp1_dl_body_pool_destroy(struct vsp1_dl_body_pool *pool)
 struct vsp1_dl_body *vsp1_dl_body_get(struct vsp1_dl_body_pool *pool)
 {
 	struct vsp1_dl_body *dlb = NULL;
-	unsigned long flags;
 
-	spin_lock_irqsave(&pool->lock, flags);
+	guard(spinlock_irqsave)(&pool->lock);
 
 	if (!list_empty(&pool->free)) {
 		dlb = list_first_entry(&pool->free, struct vsp1_dl_body, free);
@@ -346,8 +345,6 @@ struct vsp1_dl_body *vsp1_dl_body_get(struct vsp1_dl_body_pool *pool)
 		refcount_set(&dlb->refcnt, 1);
 	}
 
-	spin_unlock_irqrestore(&pool->lock, flags);
-
 	return dlb;
 }
 
@@ -359,8 +356,6 @@ struct vsp1_dl_body *vsp1_dl_body_get(struct vsp1_dl_body_pool *pool)
  */
 void vsp1_dl_body_put(struct vsp1_dl_body *dlb)
 {
-	unsigned long flags;
-
 	if (!dlb)
 		return;
 
@@ -369,9 +364,9 @@ void vsp1_dl_body_put(struct vsp1_dl_body *dlb)
 
 	dlb->num_entries = 0;
 
-	spin_lock_irqsave(&dlb->pool->lock, flags);
+	guard(spinlock_irqsave)(&dlb->pool->lock);
+
 	list_add_tail(&dlb->free, &dlb->pool->free);
-	spin_unlock_irqrestore(&dlb->pool->lock, flags);
 }
 
 /**
@@ -493,9 +488,8 @@ static
 struct vsp1_dl_ext_cmd *vsp1_dl_ext_cmd_get(struct vsp1_dl_cmd_pool *pool)
 {
 	struct vsp1_dl_ext_cmd *cmd = NULL;
-	unsigned long flags;
 
-	spin_lock_irqsave(&pool->lock, flags);
+	guard(spinlock_irqsave)(&pool->lock);
 
 	if (!list_empty(&pool->free)) {
 		cmd = list_first_entry(&pool->free, struct vsp1_dl_ext_cmd,
@@ -503,24 +497,20 @@ struct vsp1_dl_ext_cmd *vsp1_dl_ext_cmd_get(struct vsp1_dl_cmd_pool *pool)
 		list_del(&cmd->free);
 	}
 
-	spin_unlock_irqrestore(&pool->lock, flags);
-
 	return cmd;
 }
 
 static void vsp1_dl_ext_cmd_put(struct vsp1_dl_ext_cmd *cmd)
 {
-	unsigned long flags;
-
 	if (!cmd)
 		return;
 
 	/* Reset flags, these mark data usage. */
 	cmd->flags = 0;
 
-	spin_lock_irqsave(&cmd->pool->lock, flags);
+	guard(spinlock_irqsave)(&cmd->pool->lock);
+
 	list_add_tail(&cmd->free, &cmd->pool->free);
-	spin_unlock_irqrestore(&cmd->pool->lock, flags);
 }
 
 static void vsp1_dl_ext_cmd_pool_destroy(struct vsp1_dl_cmd_pool *pool)
@@ -611,11 +601,10 @@ static void vsp1_dl_list_free(struct vsp1_dl_list *dl)
 struct vsp1_dl_list *vsp1_dl_list_get(struct vsp1_dl_manager *dlm)
 {
 	struct vsp1_dl_list *dl = NULL;
-	unsigned long flags;
 
 	lockdep_assert_not_held(&dlm->lock);
 
-	spin_lock_irqsave(&dlm->lock, flags);
+	guard(spinlock_irqsave)(&dlm->lock);
 
 	if (!list_empty(&dlm->free)) {
 		dl = list_first_entry(&dlm->free, struct vsp1_dl_list, list);
@@ -629,8 +618,6 @@ struct vsp1_dl_list *vsp1_dl_list_get(struct vsp1_dl_manager *dlm)
 		dl->allocated = true;
 	}
 
-	spin_unlock_irqrestore(&dlm->lock, flags);
-
 	return dl;
 }
 
@@ -690,14 +677,12 @@ static void __vsp1_dl_list_put(struct vsp1_dl_list *dl)
  */
 void vsp1_dl_list_put(struct vsp1_dl_list *dl)
 {
-	unsigned long flags;
-
 	if (!dl)
 		return;
 
-	spin_lock_irqsave(&dl->dlm->lock, flags);
+	guard(spinlock_irqsave)(&dl->dlm->lock);
+
 	__vsp1_dl_list_put(dl);
-	spin_unlock_irqrestore(&dl->dlm->lock, flags);
 }
 
 /**
@@ -937,7 +922,6 @@ void vsp1_dl_list_commit(struct vsp1_dl_list *dl, unsigned int dl_flags)
 {
 	struct vsp1_dl_manager *dlm = dl->dlm;
 	struct vsp1_dl_list *dl_next;
-	unsigned long flags;
 
 	/* Fill the header for the head and chained display lists. */
 	vsp1_dl_list_fill_header(dl, list_empty(&dl->chain));
@@ -950,14 +934,12 @@ void vsp1_dl_list_commit(struct vsp1_dl_list *dl, unsigned int dl_flags)
 
 	dl->flags = dl_flags & ~VSP1_DL_FRAME_END_COMPLETED;
 
-	spin_lock_irqsave(&dlm->lock, flags);
+	guard(spinlock_irqsave)(&dlm->lock);
 
 	if (dlm->singleshot)
 		vsp1_dl_list_commit_singleshot(dl);
 	else
 		vsp1_dl_list_commit_continuous(dl);
-
-	spin_unlock_irqrestore(&dlm->lock, flags);
 }
 
 /* -----------------------------------------------------------------------------
@@ -991,7 +973,7 @@ unsigned int vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 	u32 status = vsp1_read(vsp1, VI6_STATUS);
 	unsigned int flags = 0;
 
-	spin_lock(&dlm->lock);
+	guard(spinlock)(&dlm->lock);
 
 	/*
 	 * The mem-to-mem pipelines work in single-shot mode. No new display
@@ -1001,7 +983,7 @@ unsigned int vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 		__vsp1_dl_list_put(dlm->active);
 		dlm->active = NULL;
 		flags |= VSP1_DL_FRAME_END_COMPLETED;
-		goto done;
+		return flags;
 	}
 
 	/*
@@ -1011,7 +993,7 @@ unsigned int vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 	 * and retry.
 	 */
 	if (vsp1_dl_list_hw_update_pending(dlm))
-		goto done;
+		return flags;
 
 	/*
 	 * Progressive streams report only TOP fields. If we have a BOTTOM
@@ -1019,7 +1001,7 @@ unsigned int vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 	 * next frame end interrupt.
 	 */
 	if (status & VI6_STATUS_FLD_STD(dlm->index))
-		goto done;
+		return flags;
 
 	/*
 	 * If the active display list has the writeback flag set, the frame
@@ -1058,9 +1040,6 @@ unsigned int vsp1_dlm_irq_frame_end(struct vsp1_dl_manager *dlm)
 		dlm->pending = NULL;
 	}
 
-done:
-	spin_unlock(&dlm->lock);
-
 	return flags;
 }
 
@@ -1085,17 +1064,15 @@ void vsp1_dlm_setup(struct vsp1_device *vsp1)
 
 void vsp1_dlm_reset(struct vsp1_dl_manager *dlm)
 {
-	unsigned long flags;
 	size_t list_count;
 
-	spin_lock_irqsave(&dlm->lock, flags);
-
-	__vsp1_dl_list_put(dlm->active);
-	__vsp1_dl_list_put(dlm->queued);
-	__vsp1_dl_list_put(dlm->pending);
+	scoped_guard(spinlock_irqsave, &dlm->lock) {
+		__vsp1_dl_list_put(dlm->active);
+		__vsp1_dl_list_put(dlm->queued);
+		__vsp1_dl_list_put(dlm->pending);
 
-	list_count = list_count_nodes(&dlm->free);
-	spin_unlock_irqrestore(&dlm->lock, flags);
+		list_count = list_count_nodes(&dlm->free);
+	}
 
 	WARN_ON_ONCE(list_count != dlm->list_count);
 
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_drm.c b/drivers/media/platform/renesas/vsp1/vsp1_drm.c
index 15d266439564e..f6fbd34753293 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_drm.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_drm.c
@@ -57,6 +57,50 @@ static void vsp1_du_pipeline_frame_end(struct vsp1_pipeline *pipe,
  * Pipeline Configuration
  */
 
+/* Configure all entities in the pipeline. */
+static void vsp1_du_pipeline_configure(struct vsp1_pipeline *pipe)
+{
+	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
+	struct vsp1_entity *entity;
+	struct vsp1_entity *next;
+	struct vsp1_dl_list *dl;
+	struct vsp1_dl_body *dlb;
+	unsigned int dl_flags = 0;
+
+	vsp1_pipeline_calculate_partition(pipe, &pipe->part_table[0],
+					  drm_pipe->width, 0);
+
+	if (drm_pipe->force_brx_release)
+		dl_flags |= VSP1_DL_FRAME_END_INTERNAL;
+	if (pipe->output->writeback)
+		dl_flags |= VSP1_DL_FRAME_END_WRITEBACK;
+
+	dl = vsp1_dl_list_get(pipe->output->dlm);
+	dlb = vsp1_dl_list_get_body0(dl);
+
+	list_for_each_entry_safe(entity, next, &pipe->entities, list_pipe) {
+		/* Disconnect unused entities from the pipeline. */
+		if (!entity->pipe) {
+			vsp1_dl_body_write(dlb, entity->route->reg,
+					   VI6_DPR_NODE_UNUSED);
+
+			entity->sink = NULL;
+			list_del(&entity->list_pipe);
+
+			continue;
+		}
+
+		vsp1_entity_route_setup(entity, pipe, dlb);
+		vsp1_entity_configure_stream(entity, entity->state, pipe,
+					     dl, dlb);
+		vsp1_entity_configure_frame(entity, pipe, dl, dlb);
+		vsp1_entity_configure_partition(entity, pipe,
+						&pipe->part_table[0], dl, dlb);
+	}
+
+	vsp1_dl_list_commit(dl, dl_flags);
+}
+
 /*
  * Insert the UIF in the pipeline between the prev and next entities. If no UIF
  * is available connect the two entities directly.
@@ -224,8 +268,6 @@ static int vsp1_du_pipeline_setup_rpf(struct vsp1_device *vsp1,
 /* Setup the BRx source pad. */
 static int vsp1_du_pipeline_setup_inputs(struct vsp1_device *vsp1,
 					 struct vsp1_pipeline *pipe);
-static void vsp1_du_pipeline_configure(struct vsp1_pipeline *pipe);
-
 static int vsp1_du_pipeline_setup_brx(struct vsp1_device *vsp1,
 				      struct vsp1_pipeline *pipe)
 {
@@ -541,50 +583,6 @@ static int vsp1_du_pipeline_setup_output(struct vsp1_device *vsp1,
 	return 0;
 }
 
-/* Configure all entities in the pipeline. */
-static void vsp1_du_pipeline_configure(struct vsp1_pipeline *pipe)
-{
-	struct vsp1_drm_pipeline *drm_pipe = to_vsp1_drm_pipeline(pipe);
-	struct vsp1_entity *entity;
-	struct vsp1_entity *next;
-	struct vsp1_dl_list *dl;
-	struct vsp1_dl_body *dlb;
-	unsigned int dl_flags = 0;
-
-	vsp1_pipeline_calculate_partition(pipe, &pipe->part_table[0],
-					  drm_pipe->width, 0);
-
-	if (drm_pipe->force_brx_release)
-		dl_flags |= VSP1_DL_FRAME_END_INTERNAL;
-	if (pipe->output->writeback)
-		dl_flags |= VSP1_DL_FRAME_END_WRITEBACK;
-
-	dl = vsp1_dl_list_get(pipe->output->dlm);
-	dlb = vsp1_dl_list_get_body0(dl);
-
-	list_for_each_entry_safe(entity, next, &pipe->entities, list_pipe) {
-		/* Disconnect unused entities from the pipeline. */
-		if (!entity->pipe) {
-			vsp1_dl_body_write(dlb, entity->route->reg,
-					   VI6_DPR_NODE_UNUSED);
-
-			entity->sink = NULL;
-			list_del(&entity->list_pipe);
-
-			continue;
-		}
-
-		vsp1_entity_route_setup(entity, pipe, dlb);
-		vsp1_entity_configure_stream(entity, entity->state, pipe,
-					     dl, dlb);
-		vsp1_entity_configure_frame(entity, pipe, dl, dlb);
-		vsp1_entity_configure_partition(entity, pipe,
-						&pipe->part_table[0], dl, dlb);
-	}
-
-	vsp1_dl_list_commit(dl, dl_flags);
-}
-
 static int vsp1_du_pipeline_set_rwpf_format(struct vsp1_device *vsp1,
 					    struct vsp1_rwpf *rwpf,
 					    u32 pixelformat, unsigned int pitch)
@@ -631,14 +629,14 @@ int vsp1_du_init(struct device *dev)
 EXPORT_SYMBOL_GPL(vsp1_du_init);
 
 /**
- * vsp1_du_setup_lif - Setup the output part of the VSP pipeline
+ * vsp1_du_enable - Setup and enable a DU pipeline
  * @dev: the VSP device
  * @pipe_index: the DRM pipeline index
  * @cfg: the LIF configuration
  *
  * Configure the output part of VSP DRM pipeline for the given frame @cfg.width
  * and @cfg.height. This sets up formats on the BRx source pad, the WPF sink and
- * source pads, and the LIF sink pad.
+ * source pads, and the LIF sink pad, and then starts the pipeline.
  *
  * The @pipe_index argument selects which DRM pipeline to setup. The number of
  * available pipelines depend on the VSP instance.
@@ -651,14 +649,12 @@ EXPORT_SYMBOL_GPL(vsp1_du_init);
  *
  * Return 0 on success or a negative error code on failure.
  */
-int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
-		      const struct vsp1_du_lif_config *cfg)
+int vsp1_du_enable(struct device *dev, unsigned int pipe_index,
+		   const struct vsp1_du_lif_config *cfg)
 {
 	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
 	struct vsp1_drm_pipeline *drm_pipe;
 	struct vsp1_pipeline *pipe;
-	unsigned long flags;
-	unsigned int i;
 	int ret;
 
 	if (pipe_index >= vsp1->info->lif_count)
@@ -667,17 +663,87 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 	drm_pipe = &vsp1->drm->pipe[pipe_index];
 	pipe = &drm_pipe->pipe;
 
-	if (!cfg) {
-		struct vsp1_brx *brx;
+	/* Reset the underrun counter */
+	pipe->underrun_count = 0;
+
+	drm_pipe->width = cfg->width;
+	drm_pipe->height = cfg->height;
+	pipe->interlaced = cfg->interlaced;
+
+	dev_dbg(vsp1->dev, "%s: configuring LIF%u with format %ux%u%s\n",
+		__func__, pipe_index, cfg->width, cfg->height,
+		pipe->interlaced ? "i" : "");
+
+	scoped_guard(mutex, &vsp1->drm->lock) {
+		/* Setup formats through the pipeline. */
+		ret = vsp1_du_pipeline_setup_inputs(vsp1, pipe);
+		if (ret < 0)
+			return ret;
 
-		mutex_lock(&vsp1->drm->lock);
+		ret = vsp1_du_pipeline_setup_output(vsp1, pipe);
+		if (ret < 0)
+			return ret;
 
-		brx = to_brx(&pipe->brx->subdev);
+		vsp1_pipeline_dump(pipe, "DU enable");
+
+		/* Enable the VSP1. */
+		ret = vsp1_device_get(vsp1);
+		if (ret < 0)
+			return ret;
 
 		/*
-		 * NULL configuration means the CRTC is being disabled, stop
-		 * the pipeline and turn the light off.
+		 * Register a callback to allow us to notify the DRM driver of frame
+		 * completion events.
 		 */
+		drm_pipe->du_complete = cfg->callback;
+		drm_pipe->du_private = cfg->callback_data;
+
+		/* Disable the display interrupts. */
+		vsp1_write(vsp1, VI6_DISP_IRQ_STA(pipe_index), 0);
+		vsp1_write(vsp1, VI6_DISP_IRQ_ENB(pipe_index), 0);
+
+		/* Configure all entities in the pipeline. */
+		vsp1_du_pipeline_configure(pipe);
+	}
+
+	/* Start the pipeline. */
+	scoped_guard(spinlock_irqsave, &pipe->irqlock) {
+		vsp1_pipeline_run(pipe);
+	}
+
+	dev_dbg(vsp1->dev, "%s: pipeline enabled\n", __func__);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(vsp1_du_enable);
+
+/**
+ * vsp1_du_disable - Disable and stop a DU pipeline
+ * @dev: the VSP device
+ * @pipe_index: the DRM pipeline index
+ *
+ * The @pipe_index argument selects which DRM pipeline to disable. The number
+ * of available pipelines depend on the VSP instance.
+ *
+ * Return 0 on success or a negative error code on failure.
+ */
+int vsp1_du_disable(struct device *dev, unsigned int pipe_index)
+{
+	struct vsp1_device *vsp1 = dev_get_drvdata(dev);
+	struct vsp1_drm_pipeline *drm_pipe;
+	struct vsp1_pipeline *pipe;
+	unsigned int i;
+	int ret;
+
+	if (pipe_index >= vsp1->info->lif_count)
+		return -EINVAL;
+
+	drm_pipe = &vsp1->drm->pipe[pipe_index];
+	pipe = &drm_pipe->pipe;
+
+	scoped_guard(mutex, &vsp1->drm->lock) {
+		struct vsp1_brx *brx = to_brx(&pipe->brx->subdev);
+
 		ret = vsp1_pipeline_stop(pipe);
 		if (ret == -ETIMEDOUT)
 			dev_err(vsp1->dev, "DRM pipeline stop timeout\n");
@@ -710,76 +776,16 @@ int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
 		list_del(&pipe->brx->list_pipe);
 		pipe->brx->pipe = NULL;
 		pipe->brx = NULL;
-
-		mutex_unlock(&vsp1->drm->lock);
-
-		vsp1_dlm_reset(pipe->output->dlm);
-		vsp1_device_put(vsp1);
-
-		dev_dbg(vsp1->dev, "%s: pipeline disabled\n", __func__);
-
-		return 0;
 	}
 
-	/* Reset the underrun counter */
-	pipe->underrun_count = 0;
-
-	drm_pipe->width = cfg->width;
-	drm_pipe->height = cfg->height;
-	pipe->interlaced = cfg->interlaced;
-
-	dev_dbg(vsp1->dev, "%s: configuring LIF%u with format %ux%u%s\n",
-		__func__, pipe_index, cfg->width, cfg->height,
-		pipe->interlaced ? "i" : "");
-
-	mutex_lock(&vsp1->drm->lock);
-
-	/* Setup formats through the pipeline. */
-	ret = vsp1_du_pipeline_setup_inputs(vsp1, pipe);
-	if (ret < 0)
-		goto unlock;
-
-	ret = vsp1_du_pipeline_setup_output(vsp1, pipe);
-	if (ret < 0)
-		goto unlock;
-
-	vsp1_pipeline_dump(pipe, "LIF setup");
-
-	/* Enable the VSP1. */
-	ret = vsp1_device_get(vsp1);
-	if (ret < 0)
-		goto unlock;
-
-	/*
-	 * Register a callback to allow us to notify the DRM driver of frame
-	 * completion events.
-	 */
-	drm_pipe->du_complete = cfg->callback;
-	drm_pipe->du_private = cfg->callback_data;
-
-	/* Disable the display interrupts. */
-	vsp1_write(vsp1, VI6_DISP_IRQ_STA(pipe_index), 0);
-	vsp1_write(vsp1, VI6_DISP_IRQ_ENB(pipe_index), 0);
-
-	/* Configure all entities in the pipeline. */
-	vsp1_du_pipeline_configure(pipe);
-
-unlock:
-	mutex_unlock(&vsp1->drm->lock);
+	vsp1_dlm_reset(pipe->output->dlm);
+	vsp1_device_put(vsp1);
 
-	if (ret < 0)
-		return ret;
-
-	/* Start the pipeline. */
-	spin_lock_irqsave(&pipe->irqlock, flags);
-	vsp1_pipeline_run(pipe);
-	spin_unlock_irqrestore(&pipe->irqlock, flags);
-
-	dev_dbg(vsp1->dev, "%s: pipeline enabled\n", __func__);
+	dev_dbg(vsp1->dev, "%s: pipeline disabled\n", __func__);
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(vsp1_du_setup_lif);
+EXPORT_SYMBOL_GPL(vsp1_du_disable);
 
 /**
  * vsp1_du_atomic_begin - Prepare for an atomic update
@@ -904,7 +910,7 @@ void vsp1_du_atomic_flush(struct device *dev, unsigned int pipe_index,
 
 	drm_pipe->crc = cfg->crc;
 
-	mutex_lock(&vsp1->drm->lock);
+	guard(mutex)(&vsp1->drm->lock);
 
 	if (cfg->writeback.pixelformat) {
 		const struct vsp1_du_writeback_config *wb_cfg = &cfg->writeback;
@@ -913,7 +919,7 @@ void vsp1_du_atomic_flush(struct device *dev, unsigned int pipe_index,
 						       wb_cfg->pixelformat,
 						       wb_cfg->pitch);
 		if (WARN_ON(ret < 0))
-			goto done;
+			return;
 
 		pipe->output->mem.addr[0] = wb_cfg->mem[0];
 		pipe->output->mem.addr[1] = wb_cfg->mem[1];
@@ -926,9 +932,6 @@ void vsp1_du_atomic_flush(struct device *dev, unsigned int pipe_index,
 	vsp1_pipeline_dump(pipe, "atomic update");
 
 	vsp1_du_pipeline_configure(pipe);
-
-done:
-	mutex_unlock(&vsp1->drm->lock);
 }
 EXPORT_SYMBOL_GPL(vsp1_du_atomic_flush);
 
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_entity.c b/drivers/media/platform/renesas/vsp1/vsp1_entity.c
index 1dad9589768c2..50aae8ee724f5 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_entity.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_entity.c
@@ -172,9 +172,9 @@ int vsp1_subdev_get_pad_format(struct v4l2_subdev *subdev,
 	if (!state)
 		return -EINVAL;
 
-	mutex_lock(&entity->lock);
+	guard(mutex)(&entity->lock);
+
 	fmt->format = *v4l2_subdev_state_get_format(state, fmt->pad);
-	mutex_unlock(&entity->lock);
 
 	return 0;
 }
@@ -216,10 +216,10 @@ int vsp1_subdev_enum_mbus_code(struct v4l2_subdev *subdev,
 		if (!state)
 			return -EINVAL;
 
-		mutex_lock(&entity->lock);
-		format = v4l2_subdev_state_get_format(state, 0);
-		code->code = format->code;
-		mutex_unlock(&entity->lock);
+		scoped_guard(mutex, &entity->lock) {
+			format = v4l2_subdev_state_get_format(state, 0);
+			code->code = format->code;
+		}
 	}
 
 	return 0;
@@ -308,22 +308,19 @@ int vsp1_subdev_set_pad_format(struct v4l2_subdev *subdev,
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *selection;
 	unsigned int i;
-	int ret = 0;
 
-	mutex_lock(&entity->lock);
+	guard(mutex)(&entity->lock);
 
 	state = vsp1_entity_get_state(entity, sd_state, fmt->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	format = v4l2_subdev_state_get_format(state, fmt->pad);
 
 	if (fmt->pad == entity->source_pad) {
 		/* The output format can't be modified. */
 		fmt->format = *format;
-		goto done;
+		return 0;
 	}
 
 	/*
@@ -369,9 +366,7 @@ int vsp1_subdev_set_pad_format(struct v4l2_subdev *subdev,
 	selection->width = format->width;
 	selection->height = format->height;
 
-done:
-	mutex_unlock(&entity->lock);
-	return ret;
+	return 0;
 }
 
 static int vsp1_entity_init_state(struct v4l2_subdev *subdev,
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_hgo.c b/drivers/media/platform/renesas/vsp1/vsp1_hgo.c
index 2c8ce7175a4e7..0ef512e3a94b7 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_hgo.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_hgo.c
@@ -153,11 +153,11 @@ static void hgo_configure_stream(struct vsp1_entity *entity,
 		       (crop->width << VI6_HGO_SIZE_HSIZE_SHIFT) |
 		       (crop->height << VI6_HGO_SIZE_VSIZE_SHIFT));
 
-	mutex_lock(hgo->ctrls.handler.lock);
-	hgo->max_rgb = hgo->ctrls.max_rgb->cur.val;
-	if (hgo->ctrls.num_bins)
-		hgo->num_bins = hgo_num_bins[hgo->ctrls.num_bins->cur.val];
-	mutex_unlock(hgo->ctrls.handler.lock);
+	scoped_guard(mutex, hgo->ctrls.handler.lock) {
+		hgo->max_rgb = hgo->ctrls.max_rgb->cur.val;
+		if (hgo->ctrls.num_bins)
+			hgo->num_bins = hgo_num_bins[hgo->ctrls.num_bins->cur.val];
+	}
 
 	hratio = crop->width * 2 / compose->width / 3;
 	vratio = crop->height * 2 / compose->height / 3;
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_hgt.c b/drivers/media/platform/renesas/vsp1/vsp1_hgt.c
index 858f330d44fa4..78b5a9201c70a 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_hgt.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_hgt.c
@@ -152,15 +152,15 @@ static void hgt_configure_stream(struct vsp1_entity *entity,
 		       (crop->width << VI6_HGT_SIZE_HSIZE_SHIFT) |
 		       (crop->height << VI6_HGT_SIZE_VSIZE_SHIFT));
 
-	mutex_lock(hgt->ctrls.lock);
-	for (i = 0; i < HGT_NUM_HUE_AREAS; ++i) {
-		lower = hgt->hue_areas[i*2 + 0];
-		upper = hgt->hue_areas[i*2 + 1];
-		vsp1_hgt_write(hgt, dlb, VI6_HGT_HUE_AREA(i),
-			       (lower << VI6_HGT_HUE_AREA_LOWER_SHIFT) |
-			       (upper << VI6_HGT_HUE_AREA_UPPER_SHIFT));
+	scoped_guard(mutex, hgt->ctrls.lock) {
+		for (i = 0; i < HGT_NUM_HUE_AREAS; ++i) {
+			lower = hgt->hue_areas[i*2 + 0];
+			upper = hgt->hue_areas[i*2 + 1];
+			vsp1_hgt_write(hgt, dlb, VI6_HGT_HUE_AREA(i),
+				       (lower << VI6_HGT_HUE_AREA_LOWER_SHIFT) |
+				       (upper << VI6_HGT_HUE_AREA_UPPER_SHIFT));
+		}
 	}
-	mutex_unlock(hgt->ctrls.lock);
 
 	hratio = crop->width * 2 / compose->width / 3;
 	vratio = crop->height * 2 / compose->height / 3;
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_histo.c b/drivers/media/platform/renesas/vsp1/vsp1_histo.c
index 3f87a2c9df0ef..97dbfb93abe97 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_histo.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_histo.c
@@ -35,20 +35,18 @@ to_vsp1_histogram_buffer(struct vb2_v4l2_buffer *vbuf)
 struct vsp1_histogram_buffer *
 vsp1_histogram_buffer_get(struct vsp1_histogram *histo)
 {
-	struct vsp1_histogram_buffer *buf = NULL;
+	struct vsp1_histogram_buffer *buf;
 
-	spin_lock(&histo->irqlock);
+	guard(spinlock)(&histo->irqlock);
 
 	if (list_empty(&histo->irqqueue))
-		goto done;
+		return NULL;
 
 	buf = list_first_entry(&histo->irqqueue, struct vsp1_histogram_buffer,
 			       queue);
 	list_del(&buf->queue);
 	histo->readout = true;
 
-done:
-	spin_unlock(&histo->irqlock);
 	return buf;
 }
 
@@ -68,10 +66,10 @@ void vsp1_histogram_buffer_complete(struct vsp1_histogram *histo,
 	vb2_set_plane_payload(&buf->buf.vb2_buf, 0, size);
 	vb2_buffer_done(&buf->buf.vb2_buf, VB2_BUF_STATE_DONE);
 
-	spin_lock(&histo->irqlock);
+	guard(spinlock)(&histo->irqlock);
+
 	histo->readout = false;
 	wake_up(&histo->wait_queue);
-	spin_unlock(&histo->irqlock);
 }
 
 /* -----------------------------------------------------------------------------
@@ -123,9 +121,9 @@ static void histo_buffer_queue(struct vb2_buffer *vb)
 	struct vsp1_histogram *histo = vb2_get_drv_priv(vb->vb2_queue);
 	struct vsp1_histogram_buffer *buf = to_vsp1_histogram_buffer(vbuf);
 
-	spin_lock_irq(&histo->irqlock);
+	guard(spinlock_irq)(&histo->irqlock);
+
 	list_add_tail(&buf->queue, &histo->irqqueue);
-	spin_unlock_irq(&histo->irqlock);
 }
 
 static int histo_start_streaming(struct vb2_queue *vq, unsigned int count)
@@ -138,7 +136,7 @@ static void histo_stop_streaming(struct vb2_queue *vq)
 	struct vsp1_histogram *histo = vb2_get_drv_priv(vq);
 	struct vsp1_histogram_buffer *buffer;
 
-	spin_lock_irq(&histo->irqlock);
+	guard(spinlock_irq)(&histo->irqlock);
 
 	/* Remove all buffers from the IRQ queue. */
 	list_for_each_entry(buffer, &histo->irqqueue, queue)
@@ -147,8 +145,6 @@ static void histo_stop_streaming(struct vb2_queue *vq)
 
 	/* Wait for the buffer being read out (if any) to complete. */
 	wait_event_lock_irq(histo->wait_queue, !histo->readout, histo->irqlock);
-
-	spin_unlock_irq(&histo->irqlock);
 }
 
 static const struct vb2_ops histo_video_queue_qops = {
@@ -196,18 +192,15 @@ static int histo_get_selection(struct v4l2_subdev *subdev,
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *crop;
-	int ret = 0;
 
 	if (sel->pad != HISTO_PAD_SINK)
 		return -EINVAL;
 
-	mutex_lock(&histo->entity.lock);
+	guard(mutex)(&histo->entity.lock);
 
 	state = vsp1_entity_get_state(&histo->entity, sd_state, sel->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	switch (sel->target) {
 	case V4L2_SEL_TGT_COMPOSE_BOUNDS:
@@ -237,13 +230,10 @@ static int histo_get_selection(struct v4l2_subdev *subdev,
 		break;
 
 	default:
-		ret = -EINVAL;
-		break;
+		return -EINVAL;
 	}
 
-done:
-	mutex_unlock(&histo->entity.lock);
-	return ret;
+	return 0;
 }
 
 static int histo_set_crop(struct v4l2_subdev *subdev,
@@ -321,29 +311,22 @@ static int histo_set_selection(struct v4l2_subdev *subdev,
 {
 	struct vsp1_histogram *histo = subdev_to_histo(subdev);
 	struct v4l2_subdev_state *state;
-	int ret;
 
 	if (sel->pad != HISTO_PAD_SINK)
 		return -EINVAL;
 
-	mutex_lock(&histo->entity.lock);
+	guard(mutex)(&histo->entity.lock);
 
 	state = vsp1_entity_get_state(&histo->entity, sd_state, sel->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	if (sel->target == V4L2_SEL_TGT_CROP)
-		ret = histo_set_crop(subdev, state, sel);
+		return histo_set_crop(subdev, state, sel);
 	else if (sel->target == V4L2_SEL_TGT_COMPOSE)
-		ret = histo_set_compose(subdev, state, sel);
+		return histo_set_compose(subdev, state, sel);
 	else
-		ret = -EINVAL;
-
-done:
-	mutex_unlock(&histo->entity.lock);
-	return ret;
+		return -EINVAL;
 }
 
 static int histo_set_format(struct v4l2_subdev *subdev,
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_hsit.c b/drivers/media/platform/renesas/vsp1/vsp1_hsit.c
index 830e124beb7bc..df069c228243a 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_hsit.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_hsit.c
@@ -115,15 +115,12 @@ static int hsit_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_hsit *hsit = to_hsit(subdev);
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
 
-	mutex_lock(&hsit->entity.lock);
+	guard(mutex)(&hsit->entity.lock);
 
 	state = vsp1_entity_get_state(&hsit->entity, sd_state, fmt->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	format = v4l2_subdev_state_get_format(state, fmt->pad);
 
@@ -133,7 +130,7 @@ static int hsit_set_format(struct v4l2_subdev *subdev,
 		 * modified.
 		 */
 		fmt->format = *format;
-		goto done;
+		return 0;
 	}
 
 	format->code = hsit->inverse ? MEDIA_BUS_FMT_AHSV8888_1X32
@@ -161,9 +158,7 @@ static int hsit_set_format(struct v4l2_subdev *subdev,
 
 	vsp1_entity_adjust_color_space(format);
 
-done:
-	mutex_unlock(&hsit->entity.lock);
-	return ret;
+	return 0;
 }
 
 static const struct v4l2_subdev_pad_ops hsit_pad_ops = {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_lut.c b/drivers/media/platform/renesas/vsp1/vsp1_lut.c
index 94bdedcc5c923..a22c31e17cb7a 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_lut.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_lut.c
@@ -50,9 +50,9 @@ static int lut_set_table(struct vsp1_lut *lut, struct v4l2_ctrl *ctrl)
 		vsp1_dl_body_write(dlb, VI6_LUT_TABLE + 4 * i,
 				       ctrl->p_new.p_u32[i]);
 
-	spin_lock_irq(&lut->lock);
-	swap(lut->lut, dlb);
-	spin_unlock_irq(&lut->lock);
+	scoped_guard(spinlock_irq, &lut->lock) {
+		swap(lut->lut, dlb);
+	}
 
 	vsp1_dl_body_put(dlb);
 	return 0;
@@ -132,12 +132,11 @@ static void lut_configure_frame(struct vsp1_entity *entity,
 {
 	struct vsp1_lut *lut = to_lut(&entity->subdev);
 	struct vsp1_dl_body *lut_dlb;
-	unsigned long flags;
 
-	spin_lock_irqsave(&lut->lock, flags);
-	lut_dlb = lut->lut;
-	lut->lut = NULL;
-	spin_unlock_irqrestore(&lut->lock, flags);
+	scoped_guard(spinlock_irqsave, &lut->lock) {
+		lut_dlb = lut->lut;
+		lut->lut = NULL;
+	}
 
 	if (lut_dlb) {
 		vsp1_dl_list_add_body(dl, lut_dlb);
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_pipe.c b/drivers/media/platform/renesas/vsp1/vsp1_pipe.c
index 5d769cc42fe1a..32bb02ce0366a 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_pipe.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_pipe.c
@@ -229,6 +229,10 @@ static const struct vsp1_format_info vsp1_video_hsit_formats[] = {
 	  1, { 32, 0, 0 }, false, false, 1, 1, false },
 };
 
+#define vsp1_for_each_format(info, formats) \
+	for (const struct vsp1_format_info *info = &formats[0]; \
+	     info < formats + ARRAY_SIZE(formats); ++info)
+
 /**
  * vsp1_get_format_info - Retrieve format information for a 4CC
  * @vsp1: the VSP1 device
@@ -240,30 +244,20 @@ static const struct vsp1_format_info vsp1_video_hsit_formats[] = {
 const struct vsp1_format_info *vsp1_get_format_info(struct vsp1_device *vsp1,
 						    u32 fourcc)
 {
-	unsigned int i;
-
-	for (i = 0; i < ARRAY_SIZE(vsp1_video_formats); ++i) {
-		const struct vsp1_format_info *info = &vsp1_video_formats[i];
-
+	vsp1_for_each_format(info, vsp1_video_formats) {
 		if (info->fourcc == fourcc)
 			return info;
 	}
 
 	if (vsp1->info->gen == 2) {
-		for (i = 0; i < ARRAY_SIZE(vsp1_video_gen2_formats); ++i) {
-			const struct vsp1_format_info *info =
-				&vsp1_video_gen2_formats[i];
-
+		vsp1_for_each_format(info, vsp1_video_gen2_formats) {
 			if (info->fourcc == fourcc)
 				return info;
 		}
 	}
 
 	if (vsp1_feature(vsp1, VSP1_HAS_HSIT)) {
-		for (i = 0; i < ARRAY_SIZE(vsp1_video_hsit_formats); ++i) {
-			const struct vsp1_format_info *info =
-				&vsp1_video_hsit_formats[i];
-
+		vsp1_for_each_format(info, vsp1_video_hsit_formats) {
 			if (info->fourcc == fourcc)
 				return info;
 		}
@@ -287,8 +281,6 @@ const struct vsp1_format_info *
 vsp1_get_format_info_by_index(struct vsp1_device *vsp1, unsigned int index,
 			      u32 code)
 {
-	unsigned int i;
-
 	if (!code) {
 		if (index < ARRAY_SIZE(vsp1_video_formats))
 			return &vsp1_video_formats[index];
@@ -308,9 +300,7 @@ vsp1_get_format_info_by_index(struct vsp1_device *vsp1, unsigned int index,
 		return NULL;
 	}
 
-	for (i = 0; i < ARRAY_SIZE(vsp1_video_formats); ++i) {
-		const struct vsp1_format_info *info = &vsp1_video_formats[i];
-
+	vsp1_for_each_format(info, vsp1_video_formats) {
 		if (info->mbus == code) {
 			if (!index)
 				return info;
@@ -319,10 +309,7 @@ vsp1_get_format_info_by_index(struct vsp1_device *vsp1, unsigned int index,
 	}
 
 	if (vsp1->info->gen == 2) {
-		for (i = 0; i < ARRAY_SIZE(vsp1_video_gen2_formats); ++i) {
-			const struct vsp1_format_info *info =
-				&vsp1_video_gen2_formats[i];
-
+		vsp1_for_each_format(info, vsp1_video_gen2_formats) {
 			if (info->mbus == code) {
 				if (!index)
 					return info;
@@ -332,10 +319,7 @@ vsp1_get_format_info_by_index(struct vsp1_device *vsp1, unsigned int index,
 	}
 
 	if (vsp1_feature(vsp1, VSP1_HAS_HSIT)) {
-		for (i = 0; i < ARRAY_SIZE(vsp1_video_hsit_formats); ++i) {
-			const struct vsp1_format_info *info =
-				&vsp1_video_hsit_formats[i];
-
+		vsp1_for_each_format(info, vsp1_video_hsit_formats) {
 			if (info->mbus == code) {
 				if (!index)
 					return info;
@@ -487,21 +471,15 @@ void vsp1_pipeline_run(struct vsp1_pipeline *pipe)
 
 bool vsp1_pipeline_stopped(struct vsp1_pipeline *pipe)
 {
-	unsigned long flags;
-	bool stopped;
+	guard(spinlock_irqsave)(&pipe->irqlock);
 
-	spin_lock_irqsave(&pipe->irqlock, flags);
-	stopped = pipe->state == VSP1_PIPELINE_STOPPED;
-	spin_unlock_irqrestore(&pipe->irqlock, flags);
-
-	return stopped;
+	return pipe->state == VSP1_PIPELINE_STOPPED;
 }
 
 int vsp1_pipeline_stop(struct vsp1_pipeline *pipe)
 {
 	struct vsp1_device *vsp1 = pipe->output->entity.vsp1;
 	struct vsp1_entity *entity;
-	unsigned long flags;
 	int ret;
 
 	if (pipe->lif) {
@@ -511,16 +489,16 @@ int vsp1_pipeline_stop(struct vsp1_pipeline *pipe)
 		 */
 		ret = vsp1_reset_wpf(vsp1, pipe->output->entity.index);
 		if (ret == 0) {
-			spin_lock_irqsave(&pipe->irqlock, flags);
-			pipe->state = VSP1_PIPELINE_STOPPED;
-			spin_unlock_irqrestore(&pipe->irqlock, flags);
+			scoped_guard(spinlock_irqsave, &pipe->irqlock) {
+				pipe->state = VSP1_PIPELINE_STOPPED;
+			}
 		}
 	} else {
 		/* Otherwise just request a stop and wait. */
-		spin_lock_irqsave(&pipe->irqlock, flags);
-		if (pipe->state == VSP1_PIPELINE_RUNNING)
-			pipe->state = VSP1_PIPELINE_STOPPING;
-		spin_unlock_irqrestore(&pipe->irqlock, flags);
+		scoped_guard(spinlock_irqsave, &pipe->irqlock) {
+			if (pipe->state == VSP1_PIPELINE_RUNNING)
+				pipe->state = VSP1_PIPELINE_STOPPING;
+		}
 
 		ret = wait_event_timeout(pipe->wq, vsp1_pipeline_stopped(pipe),
 					 msecs_to_jiffies(500));
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c b/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c
index c72518b29f84b..ced01870acd6e 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_rwpf.c
@@ -116,15 +116,12 @@ static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_rwpf *rwpf = to_rwpf(subdev);
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
 
-	mutex_lock(&rwpf->entity.lock);
+	guard(mutex)(&rwpf->entity.lock);
 
 	state = vsp1_entity_get_state(&rwpf->entity, sd_state, fmt->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	/* Default to YUV if the requested format is not supported. */
 	if (fmt->format.code != MEDIA_BUS_FMT_ARGB8888_1X32 &&
@@ -174,7 +171,7 @@ static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev,
 		fmt->format = *format;
 		fmt->format.flags = flags;
 
-		goto done;
+		return 0;
 	}
 
 	format->code = fmt->format.code;
@@ -213,9 +210,7 @@ static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev,
 		format->height = fmt->format.width;
 	}
 
-done:
-	mutex_unlock(&rwpf->entity.lock);
-	return ret;
+	return 0;
 }
 
 static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev,
@@ -225,7 +220,6 @@ static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev,
 	struct vsp1_rwpf *rwpf = to_rwpf(subdev);
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
 
 	/*
 	 * Cropping is only supported on the RPF and is implemented on the sink
@@ -234,13 +228,11 @@ static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev,
 	if (rwpf->entity.type == VSP1_ENTITY_WPF || sel->pad != RWPF_PAD_SINK)
 		return -EINVAL;
 
-	mutex_lock(&rwpf->entity.lock);
+	guard(mutex)(&rwpf->entity.lock);
 
 	state = vsp1_entity_get_state(&rwpf->entity, sd_state, sel->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	switch (sel->target) {
 	case V4L2_SEL_TGT_CROP:
@@ -256,13 +248,10 @@ static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev,
 		break;
 
 	default:
-		ret = -EINVAL;
-		break;
+		return -EINVAL;
 	}
 
-done:
-	mutex_unlock(&rwpf->entity.lock);
-	return ret;
+	return 0;
 }
 
 static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
@@ -275,7 +264,6 @@ static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *crop;
-	int ret = 0;
 
 	/*
 	 * Cropping is only supported on the RPF and is implemented on the sink
@@ -287,13 +275,11 @@ static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
 	if (sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 
-	mutex_lock(&rwpf->entity.lock);
+	guard(mutex)(&rwpf->entity.lock);
 
 	state = vsp1_entity_get_state(&rwpf->entity, sd_state, sel->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	/* Make sure the crop rectangle is entirely contained in the image. */
 	format = v4l2_subdev_state_get_format(state, RWPF_PAD_SINK);
@@ -342,9 +328,7 @@ static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev,
 	format->width = crop->width;
 	format->height = crop->height;
 
-done:
-	mutex_unlock(&rwpf->entity.lock);
-	return ret;
+	return 0;
 }
 
 static const struct v4l2_subdev_pad_ops vsp1_rwpf_pad_ops = {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_sru.c b/drivers/media/platform/renesas/vsp1/vsp1_sru.c
index 94149da0c9003..3fd9fde5c7242 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_sru.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_sru.c
@@ -216,15 +216,12 @@ static int sru_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_sru *sru = to_sru(subdev);
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
 
-	mutex_lock(&sru->entity.lock);
+	guard(mutex)(&sru->entity.lock);
 
 	state = vsp1_entity_get_state(&sru->entity, sd_state, fmt->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	sru_try_format(sru, state, fmt->pad, &fmt->format);
 
@@ -239,9 +236,7 @@ static int sru_set_format(struct v4l2_subdev *subdev,
 		sru_try_format(sru, state, SRU_PAD_SOURCE, format);
 	}
 
-done:
-	mutex_unlock(&sru->entity.lock);
-	return ret;
+	return 0;
 }
 
 static const struct v4l2_subdev_pad_ops sru_pad_ops = {
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_uds.c b/drivers/media/platform/renesas/vsp1/vsp1_uds.c
index dd4722315c564..9f7bb112929ed 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_uds.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_uds.c
@@ -199,15 +199,12 @@ static int uds_set_format(struct v4l2_subdev *subdev,
 	struct vsp1_uds *uds = to_uds(subdev);
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
 
-	mutex_lock(&uds->entity.lock);
+	guard(mutex)(&uds->entity.lock);
 
 	state = vsp1_entity_get_state(&uds->entity, sd_state, fmt->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	uds_try_format(uds, state, fmt->pad, &fmt->format);
 
@@ -222,9 +219,7 @@ static int uds_set_format(struct v4l2_subdev *subdev,
 		uds_try_format(uds, state, UDS_PAD_SOURCE, format);
 	}
 
-done:
-	mutex_unlock(&uds->entity.lock);
-	return ret;
+	return 0;
 }
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_uif.c b/drivers/media/platform/renesas/vsp1/vsp1_uif.c
index 3aefe5c9d421a..52dbfe58a70d2 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_uif.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_uif.c
@@ -60,18 +60,15 @@ static int uif_get_selection(struct v4l2_subdev *subdev,
 	struct vsp1_uif *uif = to_uif(subdev);
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
-	int ret = 0;
 
 	if (sel->pad != UIF_PAD_SINK)
 		return -EINVAL;
 
-	mutex_lock(&uif->entity.lock);
+	guard(mutex)(&uif->entity.lock);
 
 	state = vsp1_entity_get_state(&uif->entity, sd_state, sel->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	switch (sel->target) {
 	case V4L2_SEL_TGT_CROP_BOUNDS:
@@ -88,13 +85,10 @@ static int uif_get_selection(struct v4l2_subdev *subdev,
 		break;
 
 	default:
-		ret = -EINVAL;
-		break;
+		return -EINVAL;
 	}
 
-done:
-	mutex_unlock(&uif->entity.lock);
-	return ret;
+	return 0;
 }
 
 static int uif_set_selection(struct v4l2_subdev *subdev,
@@ -105,19 +99,16 @@ static int uif_set_selection(struct v4l2_subdev *subdev,
 	struct v4l2_subdev_state *state;
 	struct v4l2_mbus_framefmt *format;
 	struct v4l2_rect *selection;
-	int ret = 0;
 
 	if (sel->pad != UIF_PAD_SINK ||
 	    sel->target != V4L2_SEL_TGT_CROP)
 		return -EINVAL;
 
-	mutex_lock(&uif->entity.lock);
+	guard(mutex)(&uif->entity.lock);
 
 	state = vsp1_entity_get_state(&uif->entity, sd_state, sel->which);
-	if (!state) {
-		ret = -EINVAL;
-		goto done;
-	}
+	if (!state)
+		return -EINVAL;
 
 	/* The crop rectangle must be inside the input frame. */
 	format = v4l2_subdev_state_get_format(state, UIF_PAD_SINK);
@@ -133,9 +124,7 @@ static int uif_set_selection(struct v4l2_subdev *subdev,
 	selection = v4l2_subdev_state_get_crop(state, sel->pad);
 	*selection = sel->r;
 
-done:
-	mutex_unlock(&uif->entity.lock);
-	return ret;
+	return 0;
 }
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_video.c b/drivers/media/platform/renesas/vsp1/vsp1_video.c
index fe1dac11d4aea..5a1d284213adf 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_video.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_video.c
@@ -209,26 +209,21 @@ vsp1_video_complete_buffer(struct vsp1_video *video)
 	struct vsp1_pipeline *pipe = video->rwpf->entity.pipe;
 	struct vsp1_vb2_buffer *next = NULL;
 	struct vsp1_vb2_buffer *done;
-	unsigned long flags;
 	unsigned int i;
 
-	spin_lock_irqsave(&video->irqlock, flags);
-
-	if (list_empty(&video->irqqueue)) {
-		spin_unlock_irqrestore(&video->irqlock, flags);
-		return NULL;
-	}
-
-	done = list_first_entry(&video->irqqueue,
-				struct vsp1_vb2_buffer, queue);
-
-	list_del(&done->queue);
+	scoped_guard(spinlock_irqsave, &video->irqlock) {
+		if (list_empty(&video->irqqueue))
+			return NULL;
 
-	if (!list_empty(&video->irqqueue))
-		next = list_first_entry(&video->irqqueue,
+		done = list_first_entry(&video->irqqueue,
 					struct vsp1_vb2_buffer, queue);
 
-	spin_unlock_irqrestore(&video->irqlock, flags);
+		list_del(&done->queue);
+
+		if (!list_empty(&video->irqqueue))
+			next = list_first_entry(&video->irqqueue,
+						struct vsp1_vb2_buffer, queue);
+	}
 
 	done->buf.sequence = pipe->sequence;
 	done->buf.vb2_buf.timestamp = ktime_get_ns();
@@ -595,9 +590,9 @@ static void vsp1_video_pipeline_put(struct vsp1_pipeline *pipe)
 {
 	struct media_device *mdev = &pipe->output->entity.vsp1->media_dev;
 
-	mutex_lock(&mdev->graph_mutex);
+	guard(mutex)(&mdev->graph_mutex);
+
 	kref_put(&pipe->kref, vsp1_video_pipeline_release);
-	mutex_unlock(&mdev->graph_mutex);
 }
 
 /* -----------------------------------------------------------------------------
@@ -661,18 +656,17 @@ static void vsp1_video_buffer_queue(struct vb2_buffer *vb)
 	struct vsp1_video *video = vb2_get_drv_priv(vb->vb2_queue);
 	struct vsp1_pipeline *pipe = video->rwpf->entity.pipe;
 	struct vsp1_vb2_buffer *buf = to_vsp1_vb2_buffer(vbuf);
-	unsigned long flags;
 	bool empty;
 
-	spin_lock_irqsave(&video->irqlock, flags);
-	empty = list_empty(&video->irqqueue);
-	list_add_tail(&buf->queue, &video->irqqueue);
-	spin_unlock_irqrestore(&video->irqlock, flags);
+	scoped_guard(spinlock_irqsave, &video->irqlock) {
+		empty = list_empty(&video->irqqueue);
+		list_add_tail(&buf->queue, &video->irqqueue);
+	}
 
 	if (!empty)
 		return;
 
-	spin_lock_irqsave(&pipe->irqlock, flags);
+	guard(spinlock_irqsave)(&pipe->irqlock);
 
 	video->rwpf->mem = buf->mem;
 	pipe->buffers_ready |= 1 << video->pipe_index;
@@ -680,8 +674,6 @@ static void vsp1_video_buffer_queue(struct vb2_buffer *vb)
 	if (vb2_start_streaming_called(&video->queue) &&
 	    vsp1_pipeline_ready(pipe))
 		vsp1_video_pipeline_run(pipe);
-
-	spin_unlock_irqrestore(&pipe->irqlock, flags);
 }
 
 static int vsp1_video_pipeline_setup_partitions(struct vsp1_pipeline *pipe)
@@ -783,14 +775,13 @@ static int vsp1_video_setup_pipeline(struct vsp1_pipeline *pipe)
 static void vsp1_video_release_buffers(struct vsp1_video *video)
 {
 	struct vsp1_vb2_buffer *buffer;
-	unsigned long flags;
 
 	/* Remove all buffers from the IRQ queue. */
-	spin_lock_irqsave(&video->irqlock, flags);
+	guard(spinlock_irqsave)(&video->irqlock);
+
 	list_for_each_entry(buffer, &video->irqqueue, queue)
 		vb2_buffer_done(&buffer->buf.vb2_buf, VB2_BUF_STATE_ERROR);
 	INIT_LIST_HEAD(&video->irqqueue);
-	spin_unlock_irqrestore(&video->irqlock, flags);
 }
 
 static void vsp1_video_cleanup_pipeline(struct vsp1_pipeline *pipe)
@@ -812,25 +803,23 @@ static int vsp1_video_start_streaming(struct vb2_queue *vq, unsigned int count)
 	struct vsp1_video *video = vb2_get_drv_priv(vq);
 	struct vsp1_pipeline *pipe = video->rwpf->entity.pipe;
 	bool start_pipeline = false;
-	unsigned long flags;
 	int ret;
 
-	mutex_lock(&pipe->lock);
-	if (pipe->stream_count == pipe->num_inputs) {
-		ret = vsp1_video_setup_pipeline(pipe);
-		if (ret < 0) {
-			vsp1_video_release_buffers(video);
-			vsp1_video_cleanup_pipeline(pipe);
-			mutex_unlock(&pipe->lock);
-			return ret;
+	scoped_guard(mutex, &pipe->lock) {
+		if (pipe->stream_count == pipe->num_inputs) {
+			ret = vsp1_video_setup_pipeline(pipe);
+			if (ret < 0) {
+				vsp1_video_release_buffers(video);
+				vsp1_video_cleanup_pipeline(pipe);
+				return ret;
+			}
+
+			start_pipeline = true;
 		}
 
-		start_pipeline = true;
+		pipe->stream_count++;
 	}
 
-	pipe->stream_count++;
-	mutex_unlock(&pipe->lock);
-
 	/*
 	 * vsp1_pipeline_ready() is not sufficient to establish that all streams
 	 * are prepared and the pipeline is configured, as multiple streams
@@ -841,10 +830,10 @@ static int vsp1_video_start_streaming(struct vb2_queue *vq, unsigned int count)
 	if (!start_pipeline)
 		return 0;
 
-	spin_lock_irqsave(&pipe->irqlock, flags);
+	guard(spinlock_irqsave)(&pipe->irqlock);
+
 	if (vsp1_pipeline_ready(pipe))
 		vsp1_video_pipeline_run(pipe);
-	spin_unlock_irqrestore(&pipe->irqlock, flags);
 
 	return 0;
 }
@@ -853,27 +842,27 @@ static void vsp1_video_stop_streaming(struct vb2_queue *vq)
 {
 	struct vsp1_video *video = vb2_get_drv_priv(vq);
 	struct vsp1_pipeline *pipe = video->rwpf->entity.pipe;
-	unsigned long flags;
 	int ret;
 
 	/*
 	 * Clear the buffers ready flag to make sure the device won't be started
 	 * by a QBUF on the video node on the other side of the pipeline.
 	 */
-	spin_lock_irqsave(&video->irqlock, flags);
-	pipe->buffers_ready &= ~(1 << video->pipe_index);
-	spin_unlock_irqrestore(&video->irqlock, flags);
-
-	mutex_lock(&pipe->lock);
-	if (--pipe->stream_count == pipe->num_inputs) {
-		/* Stop the pipeline. */
-		ret = vsp1_pipeline_stop(pipe);
-		if (ret == -ETIMEDOUT)
-			dev_err(video->vsp1->dev, "pipeline stop timeout\n");
-
-		vsp1_video_cleanup_pipeline(pipe);
+	scoped_guard(spinlock_irqsave, &video->irqlock) {
+		pipe->buffers_ready &= ~(1 << video->pipe_index);
+	}
+
+	scoped_guard(mutex, &pipe->lock) {
+		if (--pipe->stream_count == pipe->num_inputs) {
+			/* Stop the pipeline. */
+			ret = vsp1_pipeline_stop(pipe);
+			if (ret == -ETIMEDOUT)
+				dev_err(video->vsp1->dev,
+					"pipeline stop timeout\n");
+
+			vsp1_video_cleanup_pipeline(pipe);
+		}
 	}
-	mutex_unlock(&pipe->lock);
 
 	video_device_pipeline_stop(&video->video);
 	vsp1_video_release_buffers(video);
@@ -938,9 +927,9 @@ vsp1_video_get_format(struct file *file, void *fh, struct v4l2_format *format)
 	if (format->type != video->queue.type)
 		return -EINVAL;
 
-	mutex_lock(&video->lock);
+	guard(mutex)(&video->lock);
+
 	format->fmt.pix_mp = video->rwpf->format;
-	mutex_unlock(&video->lock);
 
 	return 0;
 }
@@ -972,19 +961,15 @@ vsp1_video_set_format(struct file *file, void *fh, struct v4l2_format *format)
 	if (ret < 0)
 		return ret;
 
-	mutex_lock(&video->lock);
+	guard(mutex)(&video->lock);
 
-	if (vb2_is_busy(&video->queue)) {
-		ret = -EBUSY;
-		goto done;
-	}
+	if (vb2_is_busy(&video->queue))
+		return -EBUSY;
 
 	video->rwpf->format = format->fmt.pix_mp;
 	video->rwpf->fmtinfo = info;
 
-done:
-	mutex_unlock(&video->lock);
-	return ret;
+	return 0;
 }
 
 static int
@@ -1004,22 +989,16 @@ vsp1_video_streamon(struct file *file, void *fh, enum v4l2_buf_type type)
 	 * touching an entity in the pipeline can be activated or deactivated
 	 * once streaming is started.
 	 */
-	mutex_lock(&mdev->graph_mutex);
-
-	pipe = vsp1_video_pipeline_get(video);
-	if (IS_ERR(pipe)) {
-		mutex_unlock(&mdev->graph_mutex);
-		return PTR_ERR(pipe);
-	}
+	scoped_guard(mutex, &mdev->graph_mutex) {
+		pipe = vsp1_video_pipeline_get(video);
+		if (IS_ERR(pipe))
+			return PTR_ERR(pipe);
 
-	ret = __video_device_pipeline_start(&video->video, &pipe->pipe);
-	if (ret < 0) {
-		mutex_unlock(&mdev->graph_mutex);
-		goto err_pipe;
+		ret = __video_device_pipeline_start(&video->video, &pipe->pipe);
+		if (ret < 0)
+			goto err_pipe;
 	}
 
-	mutex_unlock(&mdev->graph_mutex);
-
 	/*
 	 * Verify that the configured format matches the output of the connected
 	 * subdev.
@@ -1137,7 +1116,6 @@ static const struct media_entity_operations vsp1_video_media_ops = {
 
 void vsp1_video_suspend(struct vsp1_device *vsp1)
 {
-	unsigned long flags;
 	unsigned int i;
 	int ret;
 
@@ -1157,10 +1135,10 @@ void vsp1_video_suspend(struct vsp1_device *vsp1)
 		if (pipe == NULL)
 			continue;
 
-		spin_lock_irqsave(&pipe->irqlock, flags);
-		if (pipe->state == VSP1_PIPELINE_RUNNING)
-			pipe->state = VSP1_PIPELINE_STOPPING;
-		spin_unlock_irqrestore(&pipe->irqlock, flags);
+		scoped_guard(spinlock_irqsave, &pipe->irqlock) {
+			if (pipe->state == VSP1_PIPELINE_RUNNING)
+				pipe->state = VSP1_PIPELINE_STOPPING;
+		}
 	}
 
 	for (i = 0; i < vsp1->info->wpf_count; ++i) {
@@ -1184,7 +1162,6 @@ void vsp1_video_suspend(struct vsp1_device *vsp1)
 
 void vsp1_video_resume(struct vsp1_device *vsp1)
 {
-	unsigned long flags;
 	unsigned int i;
 
 	/* Resume all running pipelines. */
@@ -1205,10 +1182,10 @@ void vsp1_video_resume(struct vsp1_device *vsp1)
 		 */
 		pipe->configured = false;
 
-		spin_lock_irqsave(&pipe->irqlock, flags);
-		if (vsp1_pipeline_ready(pipe))
-			vsp1_video_pipeline_run(pipe);
-		spin_unlock_irqrestore(&pipe->irqlock, flags);
+		scoped_guard(spinlock_irqsave, &pipe->irqlock) {
+			if (vsp1_pipeline_ready(pipe))
+				vsp1_video_pipeline_run(pipe);
+		}
 	}
 }
 
diff --git a/drivers/media/platform/renesas/vsp1/vsp1_wpf.c b/drivers/media/platform/renesas/vsp1/vsp1_wpf.c
index cd6c5592221b6..0ec707d2913f1 100644
--- a/drivers/media/platform/renesas/vsp1/vsp1_wpf.c
+++ b/drivers/media/platform/renesas/vsp1/vsp1_wpf.c
@@ -47,7 +47,6 @@ static int vsp1_wpf_set_rotation(struct vsp1_rwpf *wpf, unsigned int rotation)
 	struct v4l2_mbus_framefmt *sink_format;
 	struct v4l2_mbus_framefmt *source_format;
 	bool rotate;
-	int ret = 0;
 
 	/*
 	 * Only consider the 0°/180° from/to 90°/270° modifications, the rest
@@ -58,19 +57,17 @@ static int vsp1_wpf_set_rotation(struct vsp1_rwpf *wpf, unsigned int rotation)
 		return 0;
 
 	/* Changing rotation isn't allowed when buffers are allocated. */
-	mutex_lock(&video->lock);
+	guard(mutex)(&video->lock);
 
-	if (vb2_is_busy(&video->queue)) {
-		ret = -EBUSY;
-		goto done;
-	}
+	if (vb2_is_busy(&video->queue))
+		return -EBUSY;
 
 	sink_format = v4l2_subdev_state_get_format(wpf->entity.state,
 						   RWPF_PAD_SINK);
 	source_format = v4l2_subdev_state_get_format(wpf->entity.state,
 						     RWPF_PAD_SOURCE);
 
-	mutex_lock(&wpf->entity.lock);
+	guard(mutex)(&wpf->entity.lock);
 
 	if (rotate) {
 		source_format->width = sink_format->height;
@@ -82,11 +79,7 @@ static int vsp1_wpf_set_rotation(struct vsp1_rwpf *wpf, unsigned int rotation)
 
 	wpf->flip.rotate = rotate;
 
-	mutex_unlock(&wpf->entity.lock);
-
-done:
-	mutex_unlock(&video->lock);
-	return ret;
+	return 0;
 }
 
 static int vsp1_wpf_s_ctrl(struct v4l2_ctrl *ctrl)
@@ -118,9 +111,9 @@ static int vsp1_wpf_s_ctrl(struct v4l2_ctrl *ctrl)
 	if (rotation == 180 || rotation == 270)
 		flip ^= BIT(WPF_CTRL_VFLIP) | BIT(WPF_CTRL_HFLIP);
 
-	spin_lock_irq(&wpf->flip.lock);
+	guard(spinlock_irq)(&wpf->flip.lock);
+
 	wpf->flip.pending = flip;
-	spin_unlock_irq(&wpf->flip.lock);
 
 	return 0;
 }
@@ -373,13 +366,12 @@ static void wpf_configure_frame(struct vsp1_entity *entity,
 	const unsigned int mask = BIT(WPF_CTRL_VFLIP)
 				| BIT(WPF_CTRL_HFLIP);
 	struct vsp1_rwpf *wpf = to_rwpf(&entity->subdev);
-	unsigned long flags;
 	u32 outfmt;
 
-	spin_lock_irqsave(&wpf->flip.lock, flags);
-	wpf->flip.active = (wpf->flip.active & ~mask)
-			 | (wpf->flip.pending & mask);
-	spin_unlock_irqrestore(&wpf->flip.lock, flags);
+	scoped_guard(spinlock_irqsave, &wpf->flip.lock) {
+		wpf->flip.active = (wpf->flip.active & ~mask)
+				 | (wpf->flip.pending & mask);
+	}
 
 	outfmt = (wpf->alpha << VI6_WPF_OUTFMT_PDV_SHIFT) | wpf->outfmt;
 
diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c
index 6442436a5e428..042b759eba625 100644
--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c
+++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c
@@ -64,6 +64,7 @@ union rkisp1_ext_params_config {
 	struct rkisp1_ext_params_compand_bls_config compand_bls;
 	struct rkisp1_ext_params_compand_curve_config compand_curve;
 	struct rkisp1_ext_params_wdr_config wdr;
+	struct rkisp1_ext_params_cac_config cac;
 };
 
 enum rkisp1_params_formats {
@@ -1413,6 +1414,47 @@ static void rkisp1_wdr_config(struct rkisp1_params *params,
 				     RKISP1_CIF_ISP_WDR_TONE_CURVE_YM_MASK);
 }
 
+static void rkisp1_cac_config(struct rkisp1_params *params,
+			      const struct rkisp1_cif_isp_cac_config *arg)
+{
+	u32 val;
+
+	/*
+	 * The enable bit is in the same register (RKISP1_CIF_ISP_CAC_CTRL),
+	 * so only set the clipping mode, and do not modify the other bits.
+	 */
+	val = rkisp1_read(params->rkisp1, RKISP1_CIF_ISP_CAC_CTRL);
+	val &= ~(RKISP1_CIF_ISP_CAC_CTRL_H_CLIP_MODE |
+		 RKISP1_CIF_ISP_CAC_CTRL_V_CLIP_MODE);
+	val |= FIELD_PREP(RKISP1_CIF_ISP_CAC_CTRL_H_CLIP_MODE, arg->h_clip_mode) |
+	       FIELD_PREP(RKISP1_CIF_ISP_CAC_CTRL_V_CLIP_MODE, arg->v_clip_mode);
+	rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_CAC_CTRL, val);
+
+	val = FIELD_PREP(RKISP1_CIF_ISP_CAC_COUNT_START_H_MASK, arg->h_count_start) |
+	      FIELD_PREP(RKISP1_CIF_ISP_CAC_COUNT_START_V_MASK, arg->v_count_start);
+	rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_CAC_COUNT_START, val);
+
+	val = FIELD_PREP(RKISP1_CIF_ISP_CAC_RED_MASK, arg->red[0]) |
+	      FIELD_PREP(RKISP1_CIF_ISP_CAC_BLUE_MASK, arg->blue[0]);
+	rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_CAC_A, val);
+
+	val = FIELD_PREP(RKISP1_CIF_ISP_CAC_RED_MASK, arg->red[1]) |
+	      FIELD_PREP(RKISP1_CIF_ISP_CAC_BLUE_MASK, arg->blue[1]);
+	rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_CAC_B, val);
+
+	val = FIELD_PREP(RKISP1_CIF_ISP_CAC_RED_MASK, arg->red[2]) |
+	      FIELD_PREP(RKISP1_CIF_ISP_CAC_BLUE_MASK, arg->blue[2]);
+	rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_CAC_C, val);
+
+	val = FIELD_PREP(RKISP1_CIF_ISP_CAC_NF_MASK, arg->x_nf) |
+	      FIELD_PREP(RKISP1_CIF_ISP_CAC_NS_MASK, arg->x_ns);
+	rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_CAC_X_NORM, val);
+
+	val = FIELD_PREP(RKISP1_CIF_ISP_CAC_NF_MASK, arg->y_nf) |
+	      FIELD_PREP(RKISP1_CIF_ISP_CAC_NS_MASK, arg->y_ns);
+	rkisp1_write(params->rkisp1, RKISP1_CIF_ISP_CAC_Y_NORM, val);
+}
+
 static void
 rkisp1_isp_isr_other_config(struct rkisp1_params *params,
 			    const struct rkisp1_params_cfg *new_params)
@@ -2089,6 +2131,25 @@ static void rkisp1_ext_params_wdr(struct rkisp1_params *params,
 				      RKISP1_CIF_ISP_WDR_CTRL_ENABLE);
 }
 
+static void rkisp1_ext_params_cac(struct rkisp1_params *params,
+				  const union rkisp1_ext_params_config *block)
+{
+	const struct rkisp1_ext_params_cac_config *cac = &block->cac;
+
+	if (cac->header.flags & RKISP1_EXT_PARAMS_FL_BLOCK_DISABLE) {
+		rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_CAC_CTRL,
+					RKISP1_CIF_ISP_CAC_CTRL_ENABLE);
+		return;
+	}
+
+	rkisp1_cac_config(params, &cac->config);
+
+	if ((cac->header.flags & RKISP1_EXT_PARAMS_FL_BLOCK_ENABLE) &&
+	    !(params->enabled_blocks & BIT(cac->header.type)))
+		rkisp1_param_set_bits(params, RKISP1_CIF_ISP_CAC_CTRL,
+				      RKISP1_CIF_ISP_CAC_CTRL_ENABLE);
+}
+
 typedef void (*rkisp1_block_handler)(struct rkisp1_params *params,
 			     const union rkisp1_ext_params_config *config);
 
@@ -2185,6 +2246,10 @@ static const struct rkisp1_ext_params_handler {
 		.handler	= rkisp1_ext_params_wdr,
 		.group		= RKISP1_EXT_PARAMS_BLOCK_GROUP_OTHERS,
 	},
+	[RKISP1_EXT_PARAMS_BLOCK_TYPE_CAC] = {
+		.handler	= rkisp1_ext_params_cac,
+		.group		= RKISP1_EXT_PARAMS_BLOCK_GROUP_OTHERS,
+	},
 };
 
 #define RKISP1_PARAMS_BLOCK_INFO(block, data) \
@@ -2215,6 +2280,7 @@ rkisp1_ext_params_block_types_info[] = {
 	RKISP1_PARAMS_BLOCK_INFO(COMPAND_EXPAND, compand_curve),
 	RKISP1_PARAMS_BLOCK_INFO(COMPAND_COMPRESS, compand_curve),
 	RKISP1_PARAMS_BLOCK_INFO(WDR, wdr),
+	RKISP1_PARAMS_BLOCK_INFO(CAC, cac),
 };
 
 static_assert(ARRAY_SIZE(rkisp1_ext_params_handlers) ==
@@ -2474,6 +2540,8 @@ void rkisp1_params_disable(struct rkisp1_params *params)
 	rkisp1_ie_enable(params, false);
 	rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_DPF_MODE,
 				RKISP1_CIF_ISP_DPF_MODE_EN);
+	rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_CAC_CTRL,
+				RKISP1_CIF_ISP_CAC_CTRL_ENABLE);
 }
 
 static const struct rkisp1_params_ops rkisp1_v10_params_ops = {
diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-regs.h b/drivers/media/platform/rockchip/rkisp1/rkisp1-regs.h
index fbeb186cde0d5..2b842194de8f7 100644
--- a/drivers/media/platform/rockchip/rkisp1/rkisp1-regs.h
+++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-regs.h
@@ -724,6 +724,17 @@
 #define RKISP1_CIF_ISP_WDR_DMIN_STRENGTH_MASK		GENMASK(20, 16)
 #define RKISP1_CIF_ISP_WDR_DMIN_STRENGTH_MAX		16U
 
+/* CAC */
+#define RKISP1_CIF_ISP_CAC_CTRL_ENABLE		BIT(0)
+#define RKISP1_CIF_ISP_CAC_CTRL_V_CLIP_MODE	GENMASK(2, 1)
+#define RKISP1_CIF_ISP_CAC_CTRL_H_CLIP_MODE	BIT(3)
+#define RKISP1_CIF_ISP_CAC_COUNT_START_H_MASK	GENMASK(12, 0)
+#define RKISP1_CIF_ISP_CAC_COUNT_START_V_MASK	GENMASK(28, 16)
+#define RKISP1_CIF_ISP_CAC_RED_MASK		GENMASK(8, 0)
+#define RKISP1_CIF_ISP_CAC_BLUE_MASK		GENMASK(24, 16)
+#define RKISP1_CIF_ISP_CAC_NF_MASK		GENMASK(4, 0)
+#define RKISP1_CIF_ISP_CAC_NS_MASK		GENMASK(19, 16)
+
 /* =================================================================== */
 /*                            CIF Registers                            */
 /* =================================================================== */
@@ -1196,8 +1207,8 @@
 #define RKISP1_CIF_ISP_CAC_A			(RKISP1_CIF_ISP_CAC_BASE + 0x00000008)
 #define RKISP1_CIF_ISP_CAC_B			(RKISP1_CIF_ISP_CAC_BASE + 0x0000000c)
 #define RKISP1_CIF_ISP_CAC_C			(RKISP1_CIF_ISP_CAC_BASE + 0x00000010)
-#define RKISP1_CIF_ISP_X_NORM			(RKISP1_CIF_ISP_CAC_BASE + 0x00000014)
-#define RKISP1_CIF_ISP_Y_NORM			(RKISP1_CIF_ISP_CAC_BASE + 0x00000018)
+#define RKISP1_CIF_ISP_CAC_X_NORM		(RKISP1_CIF_ISP_CAC_BASE + 0x00000014)
+#define RKISP1_CIF_ISP_CAC_Y_NORM		(RKISP1_CIF_ISP_CAC_BASE + 0x00000018)
 
 #define RKISP1_CIF_ISP_EXP_BASE			0x00002600
 #define RKISP1_CIF_ISP_EXP_CTRL			(RKISP1_CIF_ISP_EXP_BASE + 0x00000000)
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-bitwriter.h b/drivers/media/platform/rockchip/rkvdec/rkvdec-bitwriter.h
new file mode 100644
index 0000000000000..288467941abfa
--- /dev/null
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-bitwriter.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Rockchip Video Decoder bit writer
+ *
+ * Copyright (C) 2026 Collabora, Ltd.
+ *	Detlev Casanova <detlev.casanova@collabora.com>
+ * Copyright (C) 2019 Collabora, Ltd.
+ *	Boris Brezillon <boris.brezillon@collabora.com>
+ */
+
+#ifndef RKVDEC_BIT_WRITER_H_
+#define RKVDEC_BIT_WRITER_H_
+
+#include <linux/types.h>
+#include <linux/bits.h>
+
+struct rkvdec_bw_field {
+	u16 offset;
+	u8 len;
+};
+
+#define BW_FIELD(_offset, _len) ((struct rkvdec_bw_field){ _offset, _len })
+
+static inline void rkvdec_set_bw_field(u32 *buf, struct rkvdec_bw_field field, u32 value)
+{
+	u8 bit = field.offset % 32;
+	u16 word = field.offset / 32;
+	u64 mask = GENMASK_ULL(bit + field.len - 1, bit);
+	u64 val = ((u64)value << bit) & mask;
+
+	buf[word] &= ~mask;
+	buf[word] |= val;
+	if (bit + field.len > 32) {
+		buf[word + 1] &= ~(mask >> 32);
+		buf[word + 1] |= val >> 32;
+	}
+}
+
+#endif /* RKVDEC_BIT_WRITER_H_ */
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-h264-common.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-h264-common.c
index e28f063944705..54639512e456c 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-h264-common.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-h264-common.c
@@ -21,51 +21,6 @@
 
 #define RKVDEC_NUM_REFLIST		3
 
-static void set_dpb_info(struct rkvdec_rps_entry *entries,
-			 u8 reflist,
-			 u8 refnum,
-			 u8 info,
-			 bool bottom)
-{
-	struct rkvdec_rps_entry *entry = &entries[(reflist * 4) + refnum / 8];
-	u8 idx = refnum % 8;
-
-	switch (idx) {
-	case 0:
-		entry->dpb_info0 = info;
-		entry->bottom_flag0 = bottom;
-		break;
-	case 1:
-		entry->dpb_info1 = info;
-		entry->bottom_flag1 = bottom;
-		break;
-	case 2:
-		entry->dpb_info2 = info;
-		entry->bottom_flag2 = bottom;
-		break;
-	case 3:
-		entry->dpb_info3 = info;
-		entry->bottom_flag3 = bottom;
-		break;
-	case 4:
-		entry->dpb_info4 = info;
-		entry->bottom_flag4 = bottom;
-		break;
-	case 5:
-		entry->dpb_info5 = info;
-		entry->bottom_flag5 = bottom;
-		break;
-	case 6:
-		entry->dpb_info6 = info;
-		entry->bottom_flag6 = bottom;
-		break;
-	case 7:
-		entry->dpb_info7 = info;
-		entry->bottom_flag7 = bottom;
-		break;
-	}
-}
-
 void lookup_ref_buf_idx(struct rkvdec_ctx *ctx,
 			struct rkvdec_h264_run *run)
 {
@@ -111,7 +66,7 @@ void assemble_hw_rps(struct v4l2_h264_reflist_builder *builder,
 		if (!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE))
 			continue;
 
-		hw_rps->frame_num[i] = builder->refs[i].frame_num;
+		rkvdec_set_bw_field(hw_rps->info, RPS_FRAME_NUM(i), builder->refs[i].frame_num);
 	}
 
 	for (j = 0; j < RKVDEC_NUM_REFLIST; j++) {
@@ -138,7 +93,9 @@ void assemble_hw_rps(struct v4l2_h264_reflist_builder *builder,
 			dpb_valid = !!(run->ref_buf[ref->index]);
 			bottom = ref->fields == V4L2_H264_BOTTOM_FIELD_REF;
 
-			set_dpb_info(hw_rps->entries, j, i, ref->index | (dpb_valid << 4), bottom);
+			rkvdec_set_bw_field(hw_rps->info, RPS_ENTRY_DPB_INFO(j, i),
+					    ref->index | (dpb_valid << 4));
+			rkvdec_set_bw_field(hw_rps->info, RPS_ENTRY_BOTTOM_FLAG(j, i), bottom);
 		}
 	}
 }
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-h264-common.h b/drivers/media/platform/rockchip/rkvdec/rkvdec-h264-common.h
index 5336370507d66..f04b700b863cb 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-h264-common.h
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-h264-common.h
@@ -16,6 +16,7 @@
 #include <media/v4l2-mem2mem.h>
 
 #include "rkvdec.h"
+#include "rkvdec-bitwriter.h"
 
 struct rkvdec_h264_scaling_list {
 	u8 scaling_list_4x4[6][16];
@@ -38,39 +39,16 @@ struct rkvdec_h264_run {
 	struct vb2_buffer *ref_buf[V4L2_H264_NUM_DPB_ENTRIES];
 };
 
-struct rkvdec_rps_entry {
-	u32 dpb_info0:          5;
-	u32 bottom_flag0:       1;
-	u32 view_index_off0:    1;
-	u32 dpb_info1:          5;
-	u32 bottom_flag1:       1;
-	u32 view_index_off1:    1;
-	u32 dpb_info2:          5;
-	u32 bottom_flag2:       1;
-	u32 view_index_off2:    1;
-	u32 dpb_info3:          5;
-	u32 bottom_flag3:       1;
-	u32 view_index_off3:    1;
-	u32 dpb_info4:          5;
-	u32 bottom_flag4:       1;
-	u32 view_index_off4:    1;
-	u32 dpb_info5:          5;
-	u32 bottom_flag5:       1;
-	u32 view_index_off5:    1;
-	u32 dpb_info6:          5;
-	u32 bottom_flag6:       1;
-	u32 view_index_off6:    1;
-	u32 dpb_info7:          5;
-	u32 bottom_flag7:       1;
-	u32 view_index_off7:    1;
-} __packed;
+#define RPS_FRAME_NUM(i)		BW_FIELD((i) * 16, 16)
+#define RPS_ENTRY_DPB_INFO(l, e)	BW_FIELD(288 + (l) * 7 * 32 + (e) * 7, 5) //l: 0-2, e: 0-31
+#define RPS_ENTRY_BOTTOM_FLAG(l, e)	BW_FIELD(293 + (l) * 7 * 32 + (e) * 7, 1) //l: 0-2, e: 0-31
+#define RPS_ENTRY_VIEW_INDEX_OFF(l, e)	BW_FIELD(294 + (l) * 7 * 32 + (e) * 7, 1) //l: 0-2, e: 0-31
+
+#define RKVDEC_H264_RPS_SIZE		ALIGN(288 + 3 * 7 * 32, 128)
 
 struct rkvdec_rps {
-	u16 frame_num[16];
-	u32 reserved0;
-	struct rkvdec_rps_entry entries[12];
-	u32 reserved1[66];
-} __packed;
+	u32 info[RKVDEC_H264_RPS_SIZE / 8 / 4];
+};
 
 void lookup_ref_buf_idx(struct rkvdec_ctx *ctx, struct rkvdec_h264_run *run);
 void assemble_hw_rps(struct v4l2_h264_reflist_builder *builder,
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-h264.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-h264.c
index d3202cecb9880..ffa6060381929 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-h264.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-h264.c
@@ -16,6 +16,7 @@
 #include "rkvdec-regs.h"
 #include "rkvdec-cabac.h"
 #include "rkvdec-h264-common.h"
+#include "rkvdec-bitwriter.h"
 
 /* Size with u32 units. */
 #define RKV_CABAC_INIT_BUFFER_SIZE	(3680 + 128)
@@ -25,56 +26,48 @@ struct rkvdec_sps_pps_packet {
 	u32 info[8];
 };
 
-struct rkvdec_ps_field {
-	u16 offset;
-	u8 len;
-};
-
-#define PS_FIELD(_offset, _len) \
-	((struct rkvdec_ps_field){ _offset, _len })
-
-#define SEQ_PARAMETER_SET_ID				PS_FIELD(0, 4)
-#define PROFILE_IDC					PS_FIELD(4, 8)
-#define CONSTRAINT_SET3_FLAG				PS_FIELD(12, 1)
-#define CHROMA_FORMAT_IDC				PS_FIELD(13, 2)
-#define BIT_DEPTH_LUMA					PS_FIELD(15, 3)
-#define BIT_DEPTH_CHROMA				PS_FIELD(18, 3)
-#define QPPRIME_Y_ZERO_TRANSFORM_BYPASS_FLAG		PS_FIELD(21, 1)
-#define LOG2_MAX_FRAME_NUM_MINUS4			PS_FIELD(22, 4)
-#define MAX_NUM_REF_FRAMES				PS_FIELD(26, 5)
-#define PIC_ORDER_CNT_TYPE				PS_FIELD(31, 2)
-#define LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4		PS_FIELD(33, 4)
-#define DELTA_PIC_ORDER_ALWAYS_ZERO_FLAG		PS_FIELD(37, 1)
-#define PIC_WIDTH_IN_MBS				PS_FIELD(38, 9)
-#define PIC_HEIGHT_IN_MBS				PS_FIELD(47, 9)
-#define FRAME_MBS_ONLY_FLAG				PS_FIELD(56, 1)
-#define MB_ADAPTIVE_FRAME_FIELD_FLAG			PS_FIELD(57, 1)
-#define DIRECT_8X8_INFERENCE_FLAG			PS_FIELD(58, 1)
-#define MVC_EXTENSION_ENABLE				PS_FIELD(59, 1)
-#define NUM_VIEWS					PS_FIELD(60, 2)
-#define VIEW_ID(i)					PS_FIELD(62 + ((i) * 10), 10)
-#define NUM_ANCHOR_REFS_L(i)				PS_FIELD(82 + ((i) * 11), 1)
-#define ANCHOR_REF_L(i)				PS_FIELD(83 + ((i) * 11), 10)
-#define NUM_NON_ANCHOR_REFS_L(i)			PS_FIELD(104 + ((i) * 11), 1)
-#define NON_ANCHOR_REFS_L(i)				PS_FIELD(105 + ((i) * 11), 10)
-#define PIC_PARAMETER_SET_ID				PS_FIELD(128, 8)
-#define PPS_SEQ_PARAMETER_SET_ID			PS_FIELD(136, 5)
-#define ENTROPY_CODING_MODE_FLAG			PS_FIELD(141, 1)
-#define BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT_FLAG	PS_FIELD(142, 1)
-#define NUM_REF_IDX_L_DEFAULT_ACTIVE_MINUS1(i)		PS_FIELD(143 + ((i) * 5), 5)
-#define WEIGHTED_PRED_FLAG				PS_FIELD(153, 1)
-#define WEIGHTED_BIPRED_IDC				PS_FIELD(154, 2)
-#define PIC_INIT_QP_MINUS26				PS_FIELD(156, 7)
-#define PIC_INIT_QS_MINUS26				PS_FIELD(163, 6)
-#define CHROMA_QP_INDEX_OFFSET				PS_FIELD(169, 5)
-#define DEBLOCKING_FILTER_CONTROL_PRESENT_FLAG		PS_FIELD(174, 1)
-#define CONSTRAINED_INTRA_PRED_FLAG			PS_FIELD(175, 1)
-#define REDUNDANT_PIC_CNT_PRESENT			PS_FIELD(176, 1)
-#define TRANSFORM_8X8_MODE_FLAG			PS_FIELD(177, 1)
-#define SECOND_CHROMA_QP_INDEX_OFFSET			PS_FIELD(178, 5)
-#define SCALING_LIST_ENABLE_FLAG			PS_FIELD(183, 1)
-#define SCALING_LIST_ADDRESS				PS_FIELD(184, 32)
-#define IS_LONG_TERM(i)				PS_FIELD(216 + (i), 1)
+#define SEQ_PARAMETER_SET_ID				BW_FIELD(0, 4)
+#define PROFILE_IDC					BW_FIELD(4, 8)
+#define CONSTRAINT_SET3_FLAG				BW_FIELD(12, 1)
+#define CHROMA_FORMAT_IDC				BW_FIELD(13, 2)
+#define BIT_DEPTH_LUMA					BW_FIELD(15, 3)
+#define BIT_DEPTH_CHROMA				BW_FIELD(18, 3)
+#define QPPRIME_Y_ZERO_TRANSFORM_BYPASS_FLAG		BW_FIELD(21, 1)
+#define LOG2_MAX_FRAME_NUM_MINUS4			BW_FIELD(22, 4)
+#define MAX_NUM_REF_FRAMES				BW_FIELD(26, 5)
+#define PIC_ORDER_CNT_TYPE				BW_FIELD(31, 2)
+#define LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4		BW_FIELD(33, 4)
+#define DELTA_PIC_ORDER_ALWAYS_ZERO_FLAG		BW_FIELD(37, 1)
+#define PIC_WIDTH_IN_MBS				BW_FIELD(38, 9)
+#define PIC_HEIGHT_IN_MBS				BW_FIELD(47, 9)
+#define FRAME_MBS_ONLY_FLAG				BW_FIELD(56, 1)
+#define MB_ADAPTIVE_FRAME_FIELD_FLAG			BW_FIELD(57, 1)
+#define DIRECT_8X8_INFERENCE_FLAG			BW_FIELD(58, 1)
+#define MVC_EXTENSION_ENABLE				BW_FIELD(59, 1)
+#define NUM_VIEWS					BW_FIELD(60, 2)
+#define VIEW_ID(i)					BW_FIELD(62 + ((i) * 10), 10)
+#define NUM_ANCHOR_REFS_L(i)				BW_FIELD(82 + ((i) * 11), 1)
+#define ANCHOR_REF_L(i)				BW_FIELD(83 + ((i) * 11), 10)
+#define NUM_NON_ANCHOR_REFS_L(i)			BW_FIELD(104 + ((i) * 11), 1)
+#define NON_ANCHOR_REFS_L(i)				BW_FIELD(105 + ((i) * 11), 10)
+#define PIC_PARAMETER_SET_ID				BW_FIELD(128, 8)
+#define PPS_SEQ_PARAMETER_SET_ID			BW_FIELD(136, 5)
+#define ENTROPY_CODING_MODE_FLAG			BW_FIELD(141, 1)
+#define BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT_FLAG	BW_FIELD(142, 1)
+#define NUM_REF_IDX_L_DEFAULT_ACTIVE_MINUS1(i)		BW_FIELD(143 + ((i) * 5), 5)
+#define WEIGHTED_PRED_FLAG				BW_FIELD(153, 1)
+#define WEIGHTED_BIPRED_IDC				BW_FIELD(154, 2)
+#define PIC_INIT_QP_MINUS26				BW_FIELD(156, 7)
+#define PIC_INIT_QS_MINUS26				BW_FIELD(163, 6)
+#define CHROMA_QP_INDEX_OFFSET				BW_FIELD(169, 5)
+#define DEBLOCKING_FILTER_CONTROL_PRESENT_FLAG		BW_FIELD(174, 1)
+#define CONSTRAINED_INTRA_PRED_FLAG			BW_FIELD(175, 1)
+#define REDUNDANT_PIC_CNT_PRESENT			BW_FIELD(176, 1)
+#define TRANSFORM_8X8_MODE_FLAG			BW_FIELD(177, 1)
+#define SECOND_CHROMA_QP_INDEX_OFFSET			BW_FIELD(178, 5)
+#define SCALING_LIST_ENABLE_FLAG			BW_FIELD(183, 1)
+#define SCALING_LIST_ADDRESS				BW_FIELD(184, 32)
+#define IS_LONG_TERM(i)				BW_FIELD(216 + (i), 1)
 
 /* Data structure describing auxiliary buffer format. */
 struct rkvdec_h264_priv_tbl {
@@ -91,20 +84,6 @@ struct rkvdec_h264_ctx {
 	struct rkvdec_regs regs;
 };
 
-static void set_ps_field(u32 *buf, struct rkvdec_ps_field field, u32 value)
-{
-	u8 bit = field.offset % 32, word = field.offset / 32;
-	u64 mask = GENMASK_ULL(bit + field.len - 1, bit);
-	u64 val = ((u64)value << bit) & mask;
-
-	buf[word] &= ~mask;
-	buf[word] |= val;
-	if (bit + field.len > 32) {
-		buf[word + 1] &= ~(mask >> 32);
-		buf[word + 1] |= val >> 32;
-	}
-}
-
 static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 			    struct rkvdec_h264_run *run)
 {
@@ -128,7 +107,7 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 	hw_ps = &priv_tbl->param_set[pps->pic_parameter_set_id];
 	memset(hw_ps, 0, sizeof(*hw_ps));
 
-#define WRITE_PPS(value, field) set_ps_field(hw_ps->info, field, value)
+#define WRITE_PPS(value, field) rkvdec_set_bw_field(hw_ps->info, field, value)
 	/* write sps */
 	WRITE_PPS(sps->seq_parameter_set_id, SEQ_PARAMETER_SET_ID);
 	WRITE_PPS(sps->profile_idc, PROFILE_IDC);
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
index 3119f3bc9f98b..f89602075121b 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
@@ -74,72 +74,6 @@ void compute_tiles_non_uniform(struct rkvdec_hevc_run *run, u16 log2_min_cb_size
 	row_height[i] = pic_in_cts_height - sum;
 }
 
-static void set_ref_poc(struct rkvdec_rps_short_term_ref_set *set, int poc, int value, int flag)
-{
-	switch (poc) {
-	case 0:
-		set->delta_poc0 = value;
-		set->used_flag0 = flag;
-		break;
-	case 1:
-		set->delta_poc1 = value;
-		set->used_flag1 = flag;
-		break;
-	case 2:
-		set->delta_poc2 = value;
-		set->used_flag2 = flag;
-		break;
-	case 3:
-		set->delta_poc3 = value;
-		set->used_flag3 = flag;
-		break;
-	case 4:
-		set->delta_poc4 = value;
-		set->used_flag4 = flag;
-		break;
-	case 5:
-		set->delta_poc5 = value;
-		set->used_flag5 = flag;
-		break;
-	case 6:
-		set->delta_poc6 = value;
-		set->used_flag6 = flag;
-		break;
-	case 7:
-		set->delta_poc7 = value;
-		set->used_flag7 = flag;
-		break;
-	case 8:
-		set->delta_poc8 = value;
-		set->used_flag8 = flag;
-		break;
-	case 9:
-		set->delta_poc9 = value;
-		set->used_flag9 = flag;
-		break;
-	case 10:
-		set->delta_poc10 = value;
-		set->used_flag10 = flag;
-		break;
-	case 11:
-		set->delta_poc11 = value;
-		set->used_flag11 = flag;
-		break;
-	case 12:
-		set->delta_poc12 = value;
-		set->used_flag12 = flag;
-		break;
-	case 13:
-		set->delta_poc13 = value;
-		set->used_flag13 = flag;
-		break;
-	case 14:
-		set->delta_poc14 = value;
-		set->used_flag14 = flag;
-		break;
-	}
-}
-
 static void assemble_scalingfactor0(struct rkvdec_ctx *ctx, u8 *output,
 				    const struct v4l2_ctrl_hevc_scaling_matrix *input)
 {
@@ -218,10 +152,11 @@ static void rkvdec_hevc_assemble_hw_lt_rps(struct rkvdec_hevc_run *run, struct r
 		return;
 
 	for (int i = 0; i < sps->num_long_term_ref_pics_sps; i++) {
-		rps->refs[i].lt_ref_pic_poc_lsb =
-			run->ext_sps_lt_rps[i].lt_ref_pic_poc_lsb_sps;
-		rps->refs[i].used_by_curr_pic_lt_flag =
-			!!(run->ext_sps_lt_rps[i].flags & V4L2_HEVC_EXT_SPS_LT_RPS_FLAG_USED_LT);
+		rkvdec_set_bw_field(rps->info, RPS_LT_REF_PIC_POC_LSB(i),
+				    run->ext_sps_lt_rps[i].lt_ref_pic_poc_lsb_sps);
+		rkvdec_set_bw_field(rps->info, RPS_LT_REF_USED_BY_CURR_PIC(i),
+				    !!(run->ext_sps_lt_rps[i].flags &
+				       V4L2_HEVC_EXT_SPS_LT_RPS_FLAG_USED_LT));
 	}
 }
 
@@ -235,18 +170,24 @@ static void rkvdec_hevc_assemble_hw_st_rps(struct rkvdec_hevc_run *run, struct r
 		int j = 0;
 		const struct calculated_rps_st_set *set = &calculated_rps_st_sets[i];
 
-		rps->short_term_ref_sets[i].num_negative = set->num_negative_pics;
-		rps->short_term_ref_sets[i].num_positive = set->num_positive_pics;
+		rkvdec_set_bw_field(rps->info, RPS_ST_REF_SET_NUM_NEGATIVE(i),
+				    set->num_negative_pics);
+		rkvdec_set_bw_field(rps->info, RPS_ST_REF_SET_NUM_POSITIVE(i),
+				    set->num_positive_pics);
 
 		for (; j < set->num_negative_pics; j++) {
-			set_ref_poc(&rps->short_term_ref_sets[i], j,
-				    set->delta_poc_s0[j], set->used_by_curr_pic_s0[j]);
+			rkvdec_set_bw_field(rps->info, RPS_ST_REF_SET_DELTA_POC(i, j),
+					    set->delta_poc_s0[j]);
+			rkvdec_set_bw_field(rps->info, RPS_ST_REF_SET_USED(i, j),
+					    set->used_by_curr_pic_s0[j]);
 		}
 		poc = j;
 
 		for (j = 0; j < set->num_positive_pics; j++) {
-			set_ref_poc(&rps->short_term_ref_sets[i], poc + j,
-				    set->delta_poc_s1[j], set->used_by_curr_pic_s1[j]);
+			rkvdec_set_bw_field(rps->info, RPS_ST_REF_SET_DELTA_POC(i, poc + j),
+					    set->delta_poc_s1[j]);
+			rkvdec_set_bw_field(rps->info, RPS_ST_REF_SET_USED(i, poc + j),
+					    set->used_by_curr_pic_s1[j]);
 		}
 	}
 }
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h
index 6f4faca4c091c..2a9b7719ab2d5 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h
@@ -19,53 +19,24 @@
 #include <linux/types.h>
 
 #include "rkvdec.h"
+#include "rkvdec-bitwriter.h"
 
-struct rkvdec_rps_refs {
-	u16 lt_ref_pic_poc_lsb;
-	u16 used_by_curr_pic_lt_flag	: 1;
-	u16 reserved			: 15;
-} __packed;
+#define RPS_LT_REF_PIC_POC_LSB(i)	BW_FIELD(0 + (i) * 32, 16) // i: 0-31
+#define RPS_LT_REF_USED_BY_CURR_PIC(i)	BW_FIELD(16 + (i) * 32, 1) // i: 0-31
 
-struct rkvdec_rps_short_term_ref_set {
-	u32 num_negative	: 4;
-	u32 num_positive	: 4;
-	u32 delta_poc0		: 16;
-	u32 used_flag0		: 1;
-	u32 delta_poc1		: 16;
-	u32 used_flag1		: 1;
-	u32 delta_poc2		: 16;
-	u32 used_flag2		: 1;
-	u32 delta_poc3		: 16;
-	u32 used_flag3		: 1;
-	u32 delta_poc4		: 16;
-	u32 used_flag4		: 1;
-	u32 delta_poc5		: 16;
-	u32 used_flag5		: 1;
-	u32 delta_poc6		: 16;
-	u32 used_flag6		: 1;
-	u32 delta_poc7		: 16;
-	u32 used_flag7		: 1;
-	u32 delta_poc8		: 16;
-	u32 used_flag8		: 1;
-	u32 delta_poc9		: 16;
-	u32 used_flag9		: 1;
-	u32 delta_poc10		: 16;
-	u32 used_flag10		: 1;
-	u32 delta_poc11		: 16;
-	u32 used_flag11		: 1;
-	u32 delta_poc12		: 16;
-	u32 used_flag12		: 1;
-	u32 delta_poc13		: 16;
-	u32 used_flag13		: 1;
-	u32 delta_poc14		: 16;
-	u32 used_flag14		: 1;
-	u32 reserved_bits	: 25;
-	u32 reserved[3];
-} __packed;
+#define RPS_ST_REF_SET_NUM_NEGATIVE(i)	BW_FIELD(1024 + ((i) * 384), 4) // i: 0-63
+#define RPS_ST_REF_SET_NUM_POSITIVE(i)	BW_FIELD(1028 + ((i) * 384), 4) // i: 0-63
+
+// i: 0-63, j: 0-14
+#define RPS_ST_REF_SET_DELTA_POC(i, j)	BW_FIELD(1032 + ((i) * 384) + ((j) * 17), 16)
+
+// i: 0-63, j: 0-14
+#define RPS_ST_REF_SET_USED(i, j)	BW_FIELD(1048 + ((i) * 384) + ((j) * 17), 1)
+
+#define RKVDEC_RPS_HEVC_SIZE		ALIGN(1032 + 64 * 384, 128)
 
 struct rkvdec_rps {
-	struct rkvdec_rps_refs refs[32];
-	struct rkvdec_rps_short_term_ref_set short_term_ref_sets[64];
+	u32 info[RKVDEC_RPS_HEVC_SIZE / 8 / 4];
 } __packed;
 
 struct rkvdec_hevc_run {
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c
index ac8b825d080a2..87abf93dfd5e2 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc.c
@@ -18,6 +18,7 @@
 #include "rkvdec-regs.h"
 #include "rkvdec-cabac.h"
 #include "rkvdec-hevc-common.h"
+#include "rkvdec-bitwriter.h"
 
 /* Size in u8/u32 units. */
 #define RKV_SCALING_LIST_SIZE		1360
@@ -34,80 +35,72 @@ struct rkvdec_rps_packet {
 	u32 info[RKV_RPS_SIZE];
 };
 
-struct rkvdec_ps_field {
-	u16 offset;
-	u8 len;
-};
-
-#define PS_FIELD(_offset, _len) \
-	((struct rkvdec_ps_field){ _offset, _len })
-
 /* SPS */
-#define VIDEO_PARAMETER_SET_ID				PS_FIELD(0, 4)
-#define SEQ_PARAMETER_SET_ID				PS_FIELD(4, 4)
-#define CHROMA_FORMAT_IDC				PS_FIELD(8, 2)
-#define PIC_WIDTH_IN_LUMA_SAMPLES			PS_FIELD(10, 13)
-#define PIC_HEIGHT_IN_LUMA_SAMPLES			PS_FIELD(23, 13)
-#define BIT_DEPTH_LUMA					PS_FIELD(36, 4)
-#define BIT_DEPTH_CHROMA				PS_FIELD(40, 4)
-#define LOG2_MAX_PIC_ORDER_CNT_LSB			PS_FIELD(44, 5)
-#define LOG2_DIFF_MAX_MIN_LUMA_CODING_BLOCK_SIZE	PS_FIELD(49, 2)
-#define LOG2_MIN_LUMA_CODING_BLOCK_SIZE			PS_FIELD(51, 3)
-#define LOG2_MIN_TRANSFORM_BLOCK_SIZE			PS_FIELD(54, 3)
-#define LOG2_DIFF_MAX_MIN_LUMA_TRANSFORM_BLOCK_SIZE	PS_FIELD(57, 2)
-#define MAX_TRANSFORM_HIERARCHY_DEPTH_INTER		PS_FIELD(59, 3)
-#define MAX_TRANSFORM_HIERARCHY_DEPTH_INTRA		PS_FIELD(62, 3)
-#define SCALING_LIST_ENABLED_FLAG			PS_FIELD(65, 1)
-#define AMP_ENABLED_FLAG				PS_FIELD(66, 1)
-#define SAMPLE_ADAPTIVE_OFFSET_ENABLED_FLAG		PS_FIELD(67, 1)
-#define PCM_ENABLED_FLAG				PS_FIELD(68, 1)
-#define PCM_SAMPLE_BIT_DEPTH_LUMA			PS_FIELD(69, 4)
-#define PCM_SAMPLE_BIT_DEPTH_CHROMA			PS_FIELD(73, 4)
-#define PCM_LOOP_FILTER_DISABLED_FLAG			PS_FIELD(77, 1)
-#define LOG2_DIFF_MAX_MIN_PCM_LUMA_CODING_BLOCK_SIZE	PS_FIELD(78, 3)
-#define LOG2_MIN_PCM_LUMA_CODING_BLOCK_SIZE		PS_FIELD(81, 3)
-#define NUM_SHORT_TERM_REF_PIC_SETS			PS_FIELD(84, 7)
-#define LONG_TERM_REF_PICS_PRESENT_FLAG			PS_FIELD(91, 1)
-#define NUM_LONG_TERM_REF_PICS_SPS			PS_FIELD(92, 6)
-#define SPS_TEMPORAL_MVP_ENABLED_FLAG			PS_FIELD(98, 1)
-#define STRONG_INTRA_SMOOTHING_ENABLED_FLAG		PS_FIELD(99, 1)
+#define VIDEO_PARAMETER_SET_ID				BW_FIELD(0, 4)
+#define SEQ_PARAMETER_SET_ID				BW_FIELD(4, 4)
+#define CHROMA_FORMAT_IDC				BW_FIELD(8, 2)
+#define PIC_WIDTH_IN_LUMA_SAMPLES			BW_FIELD(10, 13)
+#define PIC_HEIGHT_IN_LUMA_SAMPLES			BW_FIELD(23, 13)
+#define BIT_DEPTH_LUMA					BW_FIELD(36, 4)
+#define BIT_DEPTH_CHROMA				BW_FIELD(40, 4)
+#define LOG2_MAX_PIC_ORDER_CNT_LSB			BW_FIELD(44, 5)
+#define LOG2_DIFF_MAX_MIN_LUMA_CODING_BLOCK_SIZE	BW_FIELD(49, 2)
+#define LOG2_MIN_LUMA_CODING_BLOCK_SIZE			BW_FIELD(51, 3)
+#define LOG2_MIN_TRANSFORM_BLOCK_SIZE			BW_FIELD(54, 3)
+#define LOG2_DIFF_MAX_MIN_LUMA_TRANSFORM_BLOCK_SIZE	BW_FIELD(57, 2)
+#define MAX_TRANSFORM_HIERARCHY_DEPTH_INTER		BW_FIELD(59, 3)
+#define MAX_TRANSFORM_HIERARCHY_DEPTH_INTRA		BW_FIELD(62, 3)
+#define SCALING_LIST_ENABLED_FLAG			BW_FIELD(65, 1)
+#define AMP_ENABLED_FLAG				BW_FIELD(66, 1)
+#define SAMPLE_ADAPTIVE_OFFSET_ENABLED_FLAG		BW_FIELD(67, 1)
+#define PCM_ENABLED_FLAG				BW_FIELD(68, 1)
+#define PCM_SAMPLE_BIT_DEPTH_LUMA			BW_FIELD(69, 4)
+#define PCM_SAMPLE_BIT_DEPTH_CHROMA			BW_FIELD(73, 4)
+#define PCM_LOOP_FILTER_DISABLED_FLAG			BW_FIELD(77, 1)
+#define LOG2_DIFF_MAX_MIN_PCM_LUMA_CODING_BLOCK_SIZE	BW_FIELD(78, 3)
+#define LOG2_MIN_PCM_LUMA_CODING_BLOCK_SIZE		BW_FIELD(81, 3)
+#define NUM_SHORT_TERM_REF_PIC_SETS			BW_FIELD(84, 7)
+#define LONG_TERM_REF_PICS_PRESENT_FLAG			BW_FIELD(91, 1)
+#define NUM_LONG_TERM_REF_PICS_SPS			BW_FIELD(92, 6)
+#define SPS_TEMPORAL_MVP_ENABLED_FLAG			BW_FIELD(98, 1)
+#define STRONG_INTRA_SMOOTHING_ENABLED_FLAG		BW_FIELD(99, 1)
 /* PPS */
-#define PIC_PARAMETER_SET_ID				PS_FIELD(128, 6)
-#define PPS_SEQ_PARAMETER_SET_ID			PS_FIELD(134, 4)
-#define DEPENDENT_SLICE_SEGMENTS_ENABLED_FLAG		PS_FIELD(138, 1)
-#define OUTPUT_FLAG_PRESENT_FLAG			PS_FIELD(139, 1)
-#define NUM_EXTRA_SLICE_HEADER_BITS			PS_FIELD(140, 13)
-#define SIGN_DATA_HIDING_ENABLED_FLAG			PS_FIELD(153, 1)
-#define CABAC_INIT_PRESENT_FLAG				PS_FIELD(154, 1)
-#define NUM_REF_IDX_L0_DEFAULT_ACTIVE			PS_FIELD(155, 4)
-#define NUM_REF_IDX_L1_DEFAULT_ACTIVE			PS_FIELD(159, 4)
-#define INIT_QP_MINUS26					PS_FIELD(163, 7)
-#define CONSTRAINED_INTRA_PRED_FLAG			PS_FIELD(170, 1)
-#define TRANSFORM_SKIP_ENABLED_FLAG			PS_FIELD(171, 1)
-#define CU_QP_DELTA_ENABLED_FLAG			PS_FIELD(172, 1)
-#define LOG2_MIN_CU_QP_DELTA_SIZE			PS_FIELD(173, 3)
-#define PPS_CB_QP_OFFSET				PS_FIELD(176, 5)
-#define PPS_CR_QP_OFFSET				PS_FIELD(181, 5)
-#define PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT_FLAG	PS_FIELD(186, 1)
-#define WEIGHTED_PRED_FLAG				PS_FIELD(187, 1)
-#define WEIGHTED_BIPRED_FLAG				PS_FIELD(188, 1)
-#define TRANSQUANT_BYPASS_ENABLED_FLAG			PS_FIELD(189, 1)
-#define TILES_ENABLED_FLAG				PS_FIELD(190, 1)
-#define ENTROPY_CODING_SYNC_ENABLED_FLAG		PS_FIELD(191, 1)
-#define PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED_FLAG	PS_FIELD(192, 1)
-#define LOOP_FILTER_ACROSS_TILES_ENABLED_FLAG		PS_FIELD(193, 1)
-#define DEBLOCKING_FILTER_OVERRIDE_ENABLED_FLAG		PS_FIELD(194, 1)
-#define PPS_DEBLOCKING_FILTER_DISABLED_FLAG		PS_FIELD(195, 1)
-#define PPS_BETA_OFFSET_DIV2				PS_FIELD(196, 4)
-#define PPS_TC_OFFSET_DIV2				PS_FIELD(200, 4)
-#define LISTS_MODIFICATION_PRESENT_FLAG			PS_FIELD(204, 1)
-#define LOG2_PARALLEL_MERGE_LEVEL			PS_FIELD(205, 3)
-#define SLICE_SEGMENT_HEADER_EXTENSION_PRESENT_FLAG	PS_FIELD(208, 1)
-#define NUM_TILE_COLUMNS				PS_FIELD(212, 5)
-#define NUM_TILE_ROWS					PS_FIELD(217, 5)
-#define COLUMN_WIDTH(i)					PS_FIELD(256 + ((i) * 8), 8)
-#define ROW_HEIGHT(i)					PS_FIELD(416 + ((i) * 8), 8)
-#define SCALING_LIST_ADDRESS				PS_FIELD(592, 32)
+#define PIC_PARAMETER_SET_ID				BW_FIELD(128, 6)
+#define PPS_SEQ_PARAMETER_SET_ID			BW_FIELD(134, 4)
+#define DEPENDENT_SLICE_SEGMENTS_ENABLED_FLAG		BW_FIELD(138, 1)
+#define OUTPUT_FLAG_PRESENT_FLAG			BW_FIELD(139, 1)
+#define NUM_EXTRA_SLICE_HEADER_BITS			BW_FIELD(140, 13)
+#define SIGN_DATA_HIDING_ENABLED_FLAG			BW_FIELD(153, 1)
+#define CABAC_INIT_PRESENT_FLAG				BW_FIELD(154, 1)
+#define NUM_REF_IDX_L0_DEFAULT_ACTIVE			BW_FIELD(155, 4)
+#define NUM_REF_IDX_L1_DEFAULT_ACTIVE			BW_FIELD(159, 4)
+#define INIT_QP_MINUS26					BW_FIELD(163, 7)
+#define CONSTRAINED_INTRA_PRED_FLAG			BW_FIELD(170, 1)
+#define TRANSFORM_SKIP_ENABLED_FLAG			BW_FIELD(171, 1)
+#define CU_QP_DELTA_ENABLED_FLAG			BW_FIELD(172, 1)
+#define LOG2_MIN_CU_QP_DELTA_SIZE			BW_FIELD(173, 3)
+#define PPS_CB_QP_OFFSET				BW_FIELD(176, 5)
+#define PPS_CR_QP_OFFSET				BW_FIELD(181, 5)
+#define PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT_FLAG	BW_FIELD(186, 1)
+#define WEIGHTED_PRED_FLAG				BW_FIELD(187, 1)
+#define WEIGHTED_BIPRED_FLAG				BW_FIELD(188, 1)
+#define TRANSQUANT_BYPASS_ENABLED_FLAG			BW_FIELD(189, 1)
+#define TILES_ENABLED_FLAG				BW_FIELD(190, 1)
+#define ENTROPY_CODING_SYNC_ENABLED_FLAG		BW_FIELD(191, 1)
+#define PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED_FLAG	BW_FIELD(192, 1)
+#define LOOP_FILTER_ACROSS_TILES_ENABLED_FLAG		BW_FIELD(193, 1)
+#define DEBLOCKING_FILTER_OVERRIDE_ENABLED_FLAG		BW_FIELD(194, 1)
+#define PPS_DEBLOCKING_FILTER_DISABLED_FLAG		BW_FIELD(195, 1)
+#define PPS_BETA_OFFSET_DIV2				BW_FIELD(196, 4)
+#define PPS_TC_OFFSET_DIV2				BW_FIELD(200, 4)
+#define LISTS_MODIFICATION_PRESENT_FLAG			BW_FIELD(204, 1)
+#define LOG2_PARALLEL_MERGE_LEVEL			BW_FIELD(205, 3)
+#define SLICE_SEGMENT_HEADER_EXTENSION_PRESENT_FLAG	BW_FIELD(208, 1)
+#define NUM_TILE_COLUMNS				BW_FIELD(212, 5)
+#define NUM_TILE_ROWS					BW_FIELD(217, 5)
+#define COLUMN_WIDTH(i)					BW_FIELD(256 + ((i) * 8), 8)
+#define ROW_HEIGHT(i)					BW_FIELD(416 + ((i) * 8), 8)
+#define SCALING_LIST_ADDRESS				BW_FIELD(592, 32)
 
 /* Data structure describing auxiliary buffer format. */
 struct rkvdec_hevc_priv_tbl {
@@ -123,20 +116,6 @@ struct rkvdec_hevc_ctx {
 	struct rkvdec_regs regs;
 };
 
-static void set_ps_field(u32 *buf, struct rkvdec_ps_field field, u32 value)
-{
-	u8 bit = field.offset % 32, word = field.offset / 32;
-	u64 mask = GENMASK_ULL(bit + field.len - 1, bit);
-	u64 val = ((u64)value << bit) & mask;
-
-	buf[word] &= ~mask;
-	buf[word] |= val;
-	if (bit + field.len > 32) {
-		buf[word + 1] &= ~(mask >> 32);
-		buf[word + 1] |= val >> 32;
-	}
-}
-
 static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 			    struct rkvdec_hevc_run *run)
 {
@@ -159,7 +138,7 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 	hw_ps = &priv_tbl->param_set[pps->pic_parameter_set_id];
 	memset(hw_ps, 0, sizeof(*hw_ps));
 
-#define WRITE_PPS(value, field) set_ps_field(hw_ps->info, field, value)
+#define WRITE_PPS(value, field) rkvdec_set_bw_field(hw_ps->info, field, value)
 	/* write sps */
 	WRITE_PPS(sps->video_parameter_set_id, VIDEO_PARAMETER_SET_ID);
 	WRITE_PPS(sps->seq_parameter_set_id, SEQ_PARAMETER_SET_ID);
@@ -321,17 +300,17 @@ static void assemble_sw_rps(struct rkvdec_ctx *ctx,
 	int i, j;
 	unsigned int lowdelay;
 
-#define WRITE_RPS(value, field) set_ps_field(hw_ps->info, field, value)
+#define WRITE_RPS(value, field) rkvdec_set_bw_field(hw_ps->info, field, value)
 
-#define REF_PIC_LONG_TERM_L0(i)			PS_FIELD((i) * 5, 1)
-#define REF_PIC_IDX_L0(i)			PS_FIELD(1 + ((i) * 5), 4)
-#define REF_PIC_LONG_TERM_L1(i)			PS_FIELD(((i) < 5 ? 75 : 132) + ((i) * 5), 1)
-#define REF_PIC_IDX_L1(i)			PS_FIELD(((i) < 4 ? 76 : 128) + ((i) * 5), 4)
+#define REF_PIC_LONG_TERM_L0(n)			BW_FIELD((n) * 5, 1)
+#define REF_PIC_IDX_L0(n)			BW_FIELD(1 + ((n) * 5), 4)
+#define REF_PIC_LONG_TERM_L1(n)			BW_FIELD(((n) < 5 ? 75 : 132) + ((n) * 5), 1)
+#define REF_PIC_IDX_L1(n)			BW_FIELD(((n) < 4 ? 76 : 128) + ((n) * 5), 4)
 
-#define LOWDELAY				PS_FIELD(182, 1)
-#define LONG_TERM_RPS_BIT_OFFSET		PS_FIELD(183, 10)
-#define SHORT_TERM_RPS_BIT_OFFSET		PS_FIELD(193, 9)
-#define NUM_RPS_POC				PS_FIELD(202, 4)
+#define LOWDELAY				BW_FIELD(182, 1)
+#define LONG_TERM_RPS_BIT_OFFSET		BW_FIELD(183, 10)
+#define SHORT_TERM_RPS_BIT_OFFSET		BW_FIELD(193, 9)
+#define NUM_RPS_POC				BW_FIELD(202, 4)
 
 	for (j = 0; j < run->num_slices; j++) {
 		uint st_bit_offset = 0;
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c
index fb4f849d73668..5ec7557339162 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-h264.c
@@ -15,105 +15,64 @@
 #include "rkvdec-cabac.h"
 #include "rkvdec-vdpu383-regs.h"
 #include "rkvdec-h264-common.h"
-
-struct rkvdec_sps {
-	u16 seq_parameter_set_id:			4;
-	u16 profile_idc:				8;
-	u16 constraint_set3_flag:			1;
-	u16 chroma_format_idc:				2;
-	u16 bit_depth_luma:				3;
-	u16 bit_depth_chroma:				3;
-	u16 qpprime_y_zero_transform_bypass_flag:	1;
-	u16 log2_max_frame_num_minus4:			4;
-	u16 max_num_ref_frames:				5;
-	u16 pic_order_cnt_type:				2;
-	u16 log2_max_pic_order_cnt_lsb_minus4:		4;
-	u16 delta_pic_order_always_zero_flag:		1;
-
-	u16 pic_width_in_mbs:				16;
-	u16 pic_height_in_mbs:				16;
-
-	u16 frame_mbs_only_flag:			1;
-	u16 mb_adaptive_frame_field_flag:		1;
-	u16 direct_8x8_inference_flag:			1;
-	u16 mvc_extension_enable:			1;
-	u16 num_views:					2;
-	u16 view_id0:                                   10;
-	u16 view_id1:                                   10;
-} __packed;
-
-struct rkvdec_pps {
-	u32 pic_parameter_set_id:				8;
-	u32 pps_seq_parameter_set_id:				5;
-	u32 entropy_coding_mode_flag:				1;
-	u32 bottom_field_pic_order_in_frame_present_flag:	1;
-	u32 num_ref_idx_l0_default_active_minus1:		5;
-	u32 num_ref_idx_l1_default_active_minus1:		5;
-	u32 weighted_pred_flag:					1;
-	u32 weighted_bipred_idc:				2;
-	u32 pic_init_qp_minus26:				7;
-	u32 pic_init_qs_minus26:				6;
-	u32 chroma_qp_index_offset:				5;
-	u32 deblocking_filter_control_present_flag:		1;
-	u32 constrained_intra_pred_flag:			1;
-	u32 redundant_pic_cnt_present:				1;
-	u32 transform_8x8_mode_flag:				1;
-	u32 second_chroma_qp_index_offset:			5;
-	u32 scaling_list_enable_flag:				1;
-	u32 is_longterm:					16;
-	u32 voidx:						16;
-
-	// dpb
-	u32 pic_field_flag:                                     1;
-	u32 pic_associated_flag:                                1;
-	u32 cur_top_field:					32;
-	u32 cur_bot_field:					32;
-
-	u32 top_field_order_cnt0:				32;
-	u32 bot_field_order_cnt0:				32;
-	u32 top_field_order_cnt1:				32;
-	u32 bot_field_order_cnt1:				32;
-	u32 top_field_order_cnt2:				32;
-	u32 bot_field_order_cnt2:				32;
-	u32 top_field_order_cnt3:				32;
-	u32 bot_field_order_cnt3:				32;
-	u32 top_field_order_cnt4:				32;
-	u32 bot_field_order_cnt4:				32;
-	u32 top_field_order_cnt5:				32;
-	u32 bot_field_order_cnt5:				32;
-	u32 top_field_order_cnt6:				32;
-	u32 bot_field_order_cnt6:				32;
-	u32 top_field_order_cnt7:				32;
-	u32 bot_field_order_cnt7:				32;
-	u32 top_field_order_cnt8:				32;
-	u32 bot_field_order_cnt8:				32;
-	u32 top_field_order_cnt9:				32;
-	u32 bot_field_order_cnt9:				32;
-	u32 top_field_order_cnt10:				32;
-	u32 bot_field_order_cnt10:				32;
-	u32 top_field_order_cnt11:				32;
-	u32 bot_field_order_cnt11:				32;
-	u32 top_field_order_cnt12:				32;
-	u32 bot_field_order_cnt12:				32;
-	u32 top_field_order_cnt13:				32;
-	u32 bot_field_order_cnt13:				32;
-	u32 top_field_order_cnt14:				32;
-	u32 bot_field_order_cnt14:				32;
-	u32 top_field_order_cnt15:				32;
-	u32 bot_field_order_cnt15:				32;
-
-	u32 ref_field_flags:					16;
-	u32 ref_topfield_used:					16;
-	u32 ref_botfield_used:					16;
-	u32 ref_colmv_use_flag:					16;
-
-	u32 reserved0:						30;
-	u32 reserved[3];
-} __packed;
+#include "rkvdec-bitwriter.h"
+
+#define SEQ_PARAMETER_SET_ID				BW_FIELD(0, 4)
+#define PROFILE_IDC					BW_FIELD(4, 8)
+#define CONSTRAINT_SET3_FLAG				BW_FIELD(12, 1)
+#define CHROMA_FORMAT_IDC				BW_FIELD(13, 2)
+#define BIT_DEPTH_LUMA					BW_FIELD(15, 3)
+#define BIT_DEPTH_CHROMA				BW_FIELD(18, 3)
+#define QPPRIME_Y_ZERO_TRANSFORM_BYPASS_FLAG		BW_FIELD(21, 1)
+#define LOG2_MAX_FRAME_NUM_MINUS4			BW_FIELD(22, 4)
+#define MAX_NUM_REF_FRAMES				BW_FIELD(26, 5)
+#define PIC_ORDER_CNT_TYPE				BW_FIELD(31, 2)
+#define LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4		BW_FIELD(33, 4)
+#define DELTA_PIC_ORDER_ALWAYS_ZERO_FLAG		BW_FIELD(37, 1)
+#define PIC_WIDTH_IN_MBS				BW_FIELD(38, 16)
+#define PIC_HEIGHT_IN_MBS				BW_FIELD(54, 16)
+#define FRAME_MBS_ONLY_FLAG				BW_FIELD(70, 1)
+#define MB_ADAPTIVE_FRAME_FIELD_FLAG			BW_FIELD(71, 1)
+#define DIRECT_8X8_INFERENCE_FLAG			BW_FIELD(72, 1)
+#define MVC_EXTENSION_ENABLE				BW_FIELD(73, 1)
+#define NUM_VIEWS					BW_FIELD(74, 2)
+#define VIEW_ID(i)					BW_FIELD(76 + ((i) * 10), 10) // i: 0-1
+
+#define PIC_PARAMETER_SET_ID				BW_FIELD(96, 8)
+#define PPS_SEQ_PARAMETER_SET_ID			BW_FIELD(104, 5)
+#define ENTROPY_CODING_MODE_FLAG			BW_FIELD(109, 1)
+#define BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT_FLAG	BW_FIELD(110, 1)
+#define NUM_REF_IDX_L_DEFAULT_ACTIVE_MINUS1(i)		BW_FIELD(111 + ((i) * 5), 5) // i: 0-1
+#define WEIGHTED_PRED_FLAG				BW_FIELD(121, 1)
+#define WEIGHTED_BIPRED_IDC				BW_FIELD(122, 2)
+#define PIC_INIT_QP_MINUS26				BW_FIELD(124, 7)
+#define PIC_INIT_QS_MINUS26				BW_FIELD(131, 6)
+#define CHROMA_QP_INDEX_OFFSET				BW_FIELD(137, 5)
+#define DEBLOCKING_FILTER_CONTROL_PRESENT_FLAG		BW_FIELD(142, 1)
+#define CONSTRAINED_INTRA_PRED_FLAG			BW_FIELD(143, 1)
+#define REDUNDANT_PIC_CNT_PRESENT			BW_FIELD(144, 1)
+#define TRANSFORM_8X8_MODE_FLAG				BW_FIELD(145, 1)
+#define SECOND_CHROMA_QP_INDEX_OFFSET			BW_FIELD(146, 5)
+#define SCALING_LIST_ENABLE_FLAG			BW_FIELD(151, 1)
+#define IS_LONG_TERM(i)					BW_FIELD(152 + (i), 1) // i: 0-15
+
+#define PIC_FIELD_FLAG					BW_FIELD(184, 1)
+#define PIC_ASSOCIATED_FLAG				BW_FIELD(185, 1)
+#define CUR_TOP_FIELD					BW_FIELD(186, 32)
+#define CUR_BOT_FIELD					BW_FIELD(218, 32)
+
+#define TOP_FIELD_ORDER_CNT(i)				BW_FIELD(250 + (i) * 64, 32) // i: 0-15
+#define BOT_FIELD_ORDER_CNT(i)				BW_FIELD(282 + (i) * 64, 32) // i: 0-15
+
+#define REF_FIELD_FLAGS(i)				BW_FIELD(1274 + (i), 1) // i: 0-15
+#define REF_TOPFIELD_USED(i)				BW_FIELD(1290 + (i), 1) // i: 0-15
+#define REF_BOTFIELD_USED(i)				BW_FIELD(1306 + (i), 1) // i: 0-15
+#define REF_COLMV_USE_FLAG(i)				BW_FIELD(1322 + (i), 1) // i: 0-15
+
+#define SPS_SIZE					ALIGN(1322 + 16, 128)
 
 struct rkvdec_sps_pps {
-	struct rkvdec_sps sps;
-	struct rkvdec_pps pps;
+	u32 info[SPS_SIZE / 8 / 4];
 } __packed;
 
 /* Data structure describing auxiliary buffer format. */
@@ -130,67 +89,6 @@ struct rkvdec_h264_ctx {
 	struct vdpu383_regs_h26x regs;
 };
 
-static noinline_for_stack void set_field_order_cnt(struct rkvdec_pps *pps, const struct v4l2_h264_dpb_entry *dpb)
-{
-	pps->top_field_order_cnt0 = dpb[0].top_field_order_cnt;
-	pps->bot_field_order_cnt0 = dpb[0].bottom_field_order_cnt;
-	pps->top_field_order_cnt1 = dpb[1].top_field_order_cnt;
-	pps->bot_field_order_cnt1 = dpb[1].bottom_field_order_cnt;
-	pps->top_field_order_cnt2 = dpb[2].top_field_order_cnt;
-	pps->bot_field_order_cnt2 = dpb[2].bottom_field_order_cnt;
-	pps->top_field_order_cnt3 = dpb[3].top_field_order_cnt;
-	pps->bot_field_order_cnt3 = dpb[3].bottom_field_order_cnt;
-	pps->top_field_order_cnt4 = dpb[4].top_field_order_cnt;
-	pps->bot_field_order_cnt4 = dpb[4].bottom_field_order_cnt;
-	pps->top_field_order_cnt5 = dpb[5].top_field_order_cnt;
-	pps->bot_field_order_cnt5 = dpb[5].bottom_field_order_cnt;
-	pps->top_field_order_cnt6 = dpb[6].top_field_order_cnt;
-	pps->bot_field_order_cnt6 = dpb[6].bottom_field_order_cnt;
-	pps->top_field_order_cnt7 = dpb[7].top_field_order_cnt;
-	pps->bot_field_order_cnt7 = dpb[7].bottom_field_order_cnt;
-	pps->top_field_order_cnt8 = dpb[8].top_field_order_cnt;
-	pps->bot_field_order_cnt8 = dpb[8].bottom_field_order_cnt;
-	pps->top_field_order_cnt9 = dpb[9].top_field_order_cnt;
-	pps->bot_field_order_cnt9 = dpb[9].bottom_field_order_cnt;
-	pps->top_field_order_cnt10 = dpb[10].top_field_order_cnt;
-	pps->bot_field_order_cnt10 = dpb[10].bottom_field_order_cnt;
-	pps->top_field_order_cnt11 = dpb[11].top_field_order_cnt;
-	pps->bot_field_order_cnt11 = dpb[11].bottom_field_order_cnt;
-	pps->top_field_order_cnt12 = dpb[12].top_field_order_cnt;
-	pps->bot_field_order_cnt12 = dpb[12].bottom_field_order_cnt;
-	pps->top_field_order_cnt13 = dpb[13].top_field_order_cnt;
-	pps->bot_field_order_cnt13 = dpb[13].bottom_field_order_cnt;
-	pps->top_field_order_cnt14 = dpb[14].top_field_order_cnt;
-	pps->bot_field_order_cnt14 = dpb[14].bottom_field_order_cnt;
-	pps->top_field_order_cnt15 = dpb[15].top_field_order_cnt;
-	pps->bot_field_order_cnt15 = dpb[15].bottom_field_order_cnt;
-}
-
-static noinline_for_stack void set_dec_params(struct rkvdec_pps *pps, const struct v4l2_ctrl_h264_decode_params *dec_params)
-{
-	const struct v4l2_h264_dpb_entry *dpb = dec_params->dpb;
-
-	for (int i = 0; i < ARRAY_SIZE(dec_params->dpb); i++) {
-		if (dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM)
-			pps->is_longterm |= (1 << i);
-		pps->ref_field_flags |=
-		 (!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD)) << i;
-		pps->ref_colmv_use_flag |=
-		 (!!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE)) << i;
-		pps->ref_topfield_used |=
-		 (!!(dpb[i].fields & V4L2_H264_TOP_FIELD_REF)) << i;
-		pps->ref_botfield_used |=
-			(!!(dpb[i].fields & V4L2_H264_BOTTOM_FIELD_REF)) << i;
-	}
-	pps->pic_field_flag =
-		!!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC);
-	pps->pic_associated_flag =
-		!!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD);
-
-	pps->cur_top_field = dec_params->top_field_order_cnt;
-	pps->cur_bot_field = dec_params->bottom_field_order_cnt;
-}
-
 static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 			    struct rkvdec_h264_run *run)
 {
@@ -202,6 +100,7 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 	struct rkvdec_h264_priv_tbl *priv_tbl = h264_ctx->priv_tbl.cpu;
 	struct rkvdec_sps_pps *hw_ps;
 	u32 pic_width, pic_height;
+	int i;
 
 	/*
 	 * HW read the SPS/PPS information from PPS packet index by PPS id.
@@ -213,23 +112,25 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 	memset(hw_ps, 0, sizeof(*hw_ps));
 
 	/* write sps */
-	hw_ps->sps.seq_parameter_set_id = sps->seq_parameter_set_id;
-	hw_ps->sps.profile_idc = sps->profile_idc;
-	hw_ps->sps.constraint_set3_flag = !!(sps->constraint_set_flags & (1 << 3));
-	hw_ps->sps.chroma_format_idc = sps->chroma_format_idc;
-	hw_ps->sps.bit_depth_luma = sps->bit_depth_luma_minus8;
-	hw_ps->sps.bit_depth_chroma = sps->bit_depth_chroma_minus8;
-	hw_ps->sps.qpprime_y_zero_transform_bypass_flag =
-		!!(sps->flags & V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS);
-	hw_ps->sps.log2_max_frame_num_minus4 = sps->log2_max_frame_num_minus4;
-	hw_ps->sps.max_num_ref_frames = sps->max_num_ref_frames;
-	hw_ps->sps.pic_order_cnt_type = sps->pic_order_cnt_type;
-	hw_ps->sps.log2_max_pic_order_cnt_lsb_minus4 =
-		sps->log2_max_pic_order_cnt_lsb_minus4;
-	hw_ps->sps.delta_pic_order_always_zero_flag =
-		!!(sps->flags & V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO);
-	hw_ps->sps.mvc_extension_enable = 0;
-	hw_ps->sps.num_views = 0;
+	rkvdec_set_bw_field(hw_ps->info, SEQ_PARAMETER_SET_ID, sps->seq_parameter_set_id);
+	rkvdec_set_bw_field(hw_ps->info, PROFILE_IDC, sps->profile_idc);
+	rkvdec_set_bw_field(hw_ps->info, CONSTRAINT_SET3_FLAG,
+			    !!(sps->constraint_set_flags & (1 << 3)));
+	rkvdec_set_bw_field(hw_ps->info, CHROMA_FORMAT_IDC, sps->chroma_format_idc);
+	rkvdec_set_bw_field(hw_ps->info, BIT_DEPTH_LUMA, sps->bit_depth_luma_minus8);
+	rkvdec_set_bw_field(hw_ps->info, BIT_DEPTH_CHROMA, sps->bit_depth_chroma_minus8);
+	rkvdec_set_bw_field(hw_ps->info, QPPRIME_Y_ZERO_TRANSFORM_BYPASS_FLAG,
+			    !!(sps->flags & V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS));
+	rkvdec_set_bw_field(hw_ps->info, LOG2_MAX_FRAME_NUM_MINUS4,
+			    sps->log2_max_frame_num_minus4);
+	rkvdec_set_bw_field(hw_ps->info, MAX_NUM_REF_FRAMES, sps->max_num_ref_frames);
+	rkvdec_set_bw_field(hw_ps->info, PIC_ORDER_CNT_TYPE, sps->pic_order_cnt_type);
+	rkvdec_set_bw_field(hw_ps->info, LOG2_MAX_PIC_ORDER_CNT_LSB_MINUS4,
+			    sps->log2_max_pic_order_cnt_lsb_minus4);
+	rkvdec_set_bw_field(hw_ps->info, DELTA_PIC_ORDER_ALWAYS_ZERO_FLAG,
+			    !!(sps->flags & V4L2_H264_SPS_FLAG_DELTA_PIC_ORDER_ALWAYS_ZERO));
+	rkvdec_set_bw_field(hw_ps->info, MVC_EXTENSION_ENABLE, 0);
+	rkvdec_set_bw_field(hw_ps->info, NUM_VIEWS, 0);
 
 	/*
 	 * Use the SPS values since they are already in macroblocks
@@ -245,48 +146,72 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 	if (!!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC))
 		pic_height /= 2;
 
-	hw_ps->sps.pic_width_in_mbs = pic_width;
-	hw_ps->sps.pic_height_in_mbs = pic_height;
+	rkvdec_set_bw_field(hw_ps->info, PIC_WIDTH_IN_MBS, pic_width);
+	rkvdec_set_bw_field(hw_ps->info, PIC_HEIGHT_IN_MBS, pic_height);
 
-	hw_ps->sps.frame_mbs_only_flag =
-		!!(sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY);
-	hw_ps->sps.mb_adaptive_frame_field_flag =
-		!!(sps->flags & V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD);
-	hw_ps->sps.direct_8x8_inference_flag =
-		!!(sps->flags & V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE);
+	rkvdec_set_bw_field(hw_ps->info, FRAME_MBS_ONLY_FLAG,
+			    !!(sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY));
+	rkvdec_set_bw_field(hw_ps->info, MB_ADAPTIVE_FRAME_FIELD_FLAG,
+			    !!(sps->flags & V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD));
+	rkvdec_set_bw_field(hw_ps->info, DIRECT_8X8_INFERENCE_FLAG,
+			    !!(sps->flags & V4L2_H264_SPS_FLAG_DIRECT_8X8_INFERENCE));
 
 	/* write pps */
-	hw_ps->pps.pic_parameter_set_id = pps->pic_parameter_set_id;
-	hw_ps->pps.pps_seq_parameter_set_id = pps->seq_parameter_set_id;
-	hw_ps->pps.entropy_coding_mode_flag =
-		!!(pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE);
-	hw_ps->pps.bottom_field_pic_order_in_frame_present_flag =
-		!!(pps->flags & V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT);
-	hw_ps->pps.num_ref_idx_l0_default_active_minus1 =
-		pps->num_ref_idx_l0_default_active_minus1;
-	hw_ps->pps.num_ref_idx_l1_default_active_minus1 =
-		pps->num_ref_idx_l1_default_active_minus1;
-	hw_ps->pps.weighted_pred_flag =
-		!!(pps->flags & V4L2_H264_PPS_FLAG_WEIGHTED_PRED);
-	hw_ps->pps.weighted_bipred_idc = pps->weighted_bipred_idc;
-	hw_ps->pps.pic_init_qp_minus26 = pps->pic_init_qp_minus26;
-	hw_ps->pps.pic_init_qs_minus26 = pps->pic_init_qs_minus26;
-	hw_ps->pps.chroma_qp_index_offset = pps->chroma_qp_index_offset;
-	hw_ps->pps.deblocking_filter_control_present_flag =
-		!!(pps->flags & V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT);
-	hw_ps->pps.constrained_intra_pred_flag =
-		!!(pps->flags & V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED);
-	hw_ps->pps.redundant_pic_cnt_present =
-		!!(pps->flags & V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT);
-	hw_ps->pps.transform_8x8_mode_flag =
-		!!(pps->flags & V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE);
-	hw_ps->pps.second_chroma_qp_index_offset = pps->second_chroma_qp_index_offset;
-	hw_ps->pps.scaling_list_enable_flag =
-		!!(pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT);
-
-	set_field_order_cnt(&hw_ps->pps, dpb);
-	set_dec_params(&hw_ps->pps, dec_params);
+	rkvdec_set_bw_field(hw_ps->info, PIC_PARAMETER_SET_ID, pps->pic_parameter_set_id);
+	rkvdec_set_bw_field(hw_ps->info, PPS_SEQ_PARAMETER_SET_ID, pps->seq_parameter_set_id);
+	rkvdec_set_bw_field(hw_ps->info, ENTROPY_CODING_MODE_FLAG,
+			    !!(pps->flags & V4L2_H264_PPS_FLAG_ENTROPY_CODING_MODE));
+	rkvdec_set_bw_field(hw_ps->info, BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT_FLAG,
+			    !!(pps->flags &
+			       V4L2_H264_PPS_FLAG_BOTTOM_FIELD_PIC_ORDER_IN_FRAME_PRESENT));
+	rkvdec_set_bw_field(hw_ps->info, NUM_REF_IDX_L_DEFAULT_ACTIVE_MINUS1(0),
+			    pps->num_ref_idx_l0_default_active_minus1);
+	rkvdec_set_bw_field(hw_ps->info, NUM_REF_IDX_L_DEFAULT_ACTIVE_MINUS1(1),
+			    pps->num_ref_idx_l1_default_active_minus1);
+	rkvdec_set_bw_field(hw_ps->info, WEIGHTED_PRED_FLAG,
+			    !!(pps->flags & V4L2_H264_PPS_FLAG_WEIGHTED_PRED));
+	rkvdec_set_bw_field(hw_ps->info, WEIGHTED_BIPRED_IDC, pps->weighted_bipred_idc);
+	rkvdec_set_bw_field(hw_ps->info, PIC_INIT_QP_MINUS26, pps->pic_init_qp_minus26);
+	rkvdec_set_bw_field(hw_ps->info, PIC_INIT_QS_MINUS26, pps->pic_init_qs_minus26);
+	rkvdec_set_bw_field(hw_ps->info, CHROMA_QP_INDEX_OFFSET, pps->chroma_qp_index_offset);
+	rkvdec_set_bw_field(hw_ps->info, DEBLOCKING_FILTER_CONTROL_PRESENT_FLAG,
+			    !!(pps->flags & V4L2_H264_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT));
+	rkvdec_set_bw_field(hw_ps->info, CONSTRAINED_INTRA_PRED_FLAG,
+			    !!(pps->flags & V4L2_H264_PPS_FLAG_CONSTRAINED_INTRA_PRED));
+	rkvdec_set_bw_field(hw_ps->info, REDUNDANT_PIC_CNT_PRESENT,
+			    !!(pps->flags & V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT));
+	rkvdec_set_bw_field(hw_ps->info, TRANSFORM_8X8_MODE_FLAG,
+			    !!(pps->flags & V4L2_H264_PPS_FLAG_TRANSFORM_8X8_MODE));
+	rkvdec_set_bw_field(hw_ps->info, SECOND_CHROMA_QP_INDEX_OFFSET,
+			    pps->second_chroma_qp_index_offset);
+	rkvdec_set_bw_field(hw_ps->info, SCALING_LIST_ENABLE_FLAG,
+			    !!(pps->flags & V4L2_H264_PPS_FLAG_SCALING_MATRIX_PRESENT));
+
+	for (i = 0; i < ARRAY_SIZE(dec_params->dpb); i++) {
+		rkvdec_set_bw_field(hw_ps->info, TOP_FIELD_ORDER_CNT(i),
+				    dpb[i].top_field_order_cnt);
+		rkvdec_set_bw_field(hw_ps->info, BOT_FIELD_ORDER_CNT(i),
+				    dpb[i].bottom_field_order_cnt);
+
+		rkvdec_set_bw_field(hw_ps->info, IS_LONG_TERM(i),
+				    !!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_LONG_TERM));
+		rkvdec_set_bw_field(hw_ps->info, REF_FIELD_FLAGS(i),
+				    !!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_FIELD));
+		rkvdec_set_bw_field(hw_ps->info, REF_COLMV_USE_FLAG(i),
+				    !!(dpb[i].flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE));
+		rkvdec_set_bw_field(hw_ps->info, REF_TOPFIELD_USED(i),
+				    !!(dpb[i].fields & V4L2_H264_TOP_FIELD_REF));
+		rkvdec_set_bw_field(hw_ps->info, REF_BOTFIELD_USED(i),
+				    !!(dpb[i].fields & V4L2_H264_BOTTOM_FIELD_REF));
+	}
+
+	rkvdec_set_bw_field(hw_ps->info, PIC_FIELD_FLAG,
+			    !!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_FIELD_PIC));
+	rkvdec_set_bw_field(hw_ps->info, PIC_ASSOCIATED_FLAG,
+			    !!(dec_params->flags & V4L2_H264_DECODE_PARAM_FLAG_BOTTOM_FIELD));
 
+	rkvdec_set_bw_field(hw_ps->info, CUR_TOP_FIELD, dec_params->top_field_order_cnt);
+	rkvdec_set_bw_field(hw_ps->info, CUR_BOT_FIELD, dec_params->bottom_field_order_cnt);
 }
 
 static void rkvdec_write_regs(struct rkvdec_ctx *ctx)
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
index 96d938ee70b04..3575338a531a4 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
@@ -13,149 +13,106 @@
 #include "rkvdec-rcb.h"
 #include "rkvdec-hevc-common.h"
 #include "rkvdec-vdpu383-regs.h"
+#include "rkvdec-bitwriter.h"
+
+#define VIDEO_PARAMETER_SET_ID				BW_FIELD(0, 4)
+#define SEQ_PARAMETER_SET_ID				BW_FIELD(4, 4)
+#define CHROMA_FORMAT_IDC				BW_FIELD(8, 2)
+#define PIC_WIDTH_IN_LUMA_SAMPLES			BW_FIELD(10, 16)
+#define PIC_HEIGHT_IN_LUMA_SAMPLES			BW_FIELD(26, 16)
+#define BIT_DEPTH_LUMA					BW_FIELD(42, 3)
+#define BIT_DEPTH_CHROMA				BW_FIELD(45, 3)
+#define LOG2_MAX_PIC_ORDER_CNT_LSB			BW_FIELD(48, 5)
+#define LOG2_DIFF_MAX_MIN_LUMA_CODING_BLOCK_SIZE	BW_FIELD(53, 2)
+#define LOG2_MIN_LUMA_CODING_BLOCK_SIZE			BW_FIELD(55, 3)
+#define LOG2_MIN_TRANSFORM_BLOCK_SIZE			BW_FIELD(58, 3)
+#define LOG2_DIFF_MAX_MIN_LUMA_TRANSFORM_BLOCK_SIZE	BW_FIELD(61, 2)
+#define MAX_TRANSFORM_HIERARCHY_DEPTH_INTER		BW_FIELD(63, 3)
+#define MAX_TRANSFORM_HIERARCHY_DEPTH_INTRA		BW_FIELD(66, 3)
+#define SCALING_LIST_ENABLED_FLAG			BW_FIELD(69, 1)
+#define AMP_ENABLED_FLAG				BW_FIELD(70, 1)
+#define SAMPLE_ADAPTIVE_OFFSET_ENABLED_FLAG		BW_FIELD(71, 1)
+#define PCM_ENABLED_FLAG				BW_FIELD(72, 1)
+#define PCM_SAMPLE_BIT_DEPTH_LUMA			BW_FIELD(73, 4)
+#define PCM_SAMPLE_BIT_DEPTH_CHROMA			BW_FIELD(77, 4)
+#define PCM_LOOP_FILTER_DISABLED_FLAG			BW_FIELD(81, 1)
+#define LOG2_DIFF_MAX_MIN_PCM_LUMA_CODING_BLOCK_SIZE	BW_FIELD(82, 3)
+#define LOG2_MIN_PCM_LUMA_CODING_BLOCK_SIZE		BW_FIELD(85, 3)
+#define NUM_SHORT_TERM_REF_PIC_SETS			BW_FIELD(88, 7)
+#define LONG_TERM_REF_PICS_PRESENT_FLAG			BW_FIELD(95, 1)
+#define NUM_LONG_TERM_REF_PICS_SPS			BW_FIELD(96, 6)
+#define SPS_TEMPORAL_MVP_ENABLED_FLAG			BW_FIELD(102, 1)
+#define STRONG_INTRA_SMOOTHING_ENABLED_FLAG		BW_FIELD(103, 1)
+#define SPS_MAX_DEC_PIC_BUFFERING_MINUS1		BW_FIELD(111, 4)
+#define SEPARATE_COLOUR_PLANE_FLAG			BW_FIELD(115, 1)
+#define HIGH_PRECISION_OFFSETS_ENABLED_FLAG		BW_FIELD(116, 1)
+#define PERSISTENT_RICE_ADAPTATION_ENABLED_FLAG		BW_FIELD(117, 1)
+
+/* PPS */
+#define PIC_PARAMETER_SET_ID				BW_FIELD(118, 6)
+#define PPS_SEQ_PARAMETER_SET_ID			BW_FIELD(124, 4)
+#define DEPENDENT_SLICE_SEGMENTS_ENABLED_FLAG		BW_FIELD(128, 1)
+#define OUTPUT_FLAG_PRESENT_FLAG			BW_FIELD(129, 1)
+#define NUM_EXTRA_SLICE_HEADER_BITS			BW_FIELD(130, 13)
+#define SIGN_DATA_HIDING_ENABLED_FLAG			BW_FIELD(143, 1)
+#define CABAC_INIT_PRESENT_FLAG				BW_FIELD(144, 1)
+#define NUM_REF_IDX_L0_DEFAULT_ACTIVE			BW_FIELD(145, 4)
+#define NUM_REF_IDX_L1_DEFAULT_ACTIVE			BW_FIELD(149, 4)
+#define INIT_QP_MINUS26					BW_FIELD(153, 7)
+#define CONSTRAINED_INTRA_PRED_FLAG			BW_FIELD(160, 1)
+#define TRANSFORM_SKIP_ENABLED_FLAG			BW_FIELD(161, 1)
+#define CU_QP_DELTA_ENABLED_FLAG			BW_FIELD(162, 1)
+#define LOG2_MIN_CU_QP_DELTA_SIZE			BW_FIELD(163, 3)
+#define PPS_CB_QP_OFFSET				BW_FIELD(166, 5)
+#define PPS_CR_QP_OFFSET				BW_FIELD(171, 5)
+#define PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT_FLAG	BW_FIELD(176, 1)
+#define WEIGHTED_PRED_FLAG				BW_FIELD(177, 1)
+#define WEIGHTED_BIPRED_FLAG				BW_FIELD(178, 1)
+#define TRANSQUANT_BYPASS_ENABLED_FLAG			BW_FIELD(179, 1)
+#define TILES_ENABLED_FLAG				BW_FIELD(180, 1)
+#define ENTROPY_CODING_SYNC_ENABLED_FLAG		BW_FIELD(181, 1)
+#define PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED_FLAG	BW_FIELD(182, 1)
+#define LOOP_FILTER_ACROSS_TILES_ENABLED_FLAG		BW_FIELD(183, 1)
+#define DEBLOCKING_FILTER_OVERRIDE_ENABLED_FLAG		BW_FIELD(184, 1)
+#define PPS_DEBLOCKING_FILTER_DISABLED_FLAG		BW_FIELD(185, 1)
+#define PPS_BETA_OFFSET_DIV2				BW_FIELD(186, 4)
+#define PPS_TC_OFFSET_DIV2				BW_FIELD(190, 4)
+#define LISTS_MODIFICATION_PRESENT_FLAG			BW_FIELD(194, 1)
+#define LOG2_PARALLEL_MERGE_LEVEL			BW_FIELD(195, 3)
+#define SLICE_SEGMENT_HEADER_EXTENSION_PRESENT_FLAG	BW_FIELD(198, 1)
+
+/* pps extensions */
+#define LOG2_MAX_TRANSFORM_SKIP_BLOCK_SIZE		BW_FIELD(202, 2)
+#define CROSS_COMPONENT_PREDICTION_ENABLED_FLAG		BW_FIELD(204, 1)
+#define CHROMA_QP_OFFSET_LIST_ENABLED_FLAG		BW_FIELD(205, 1)
+#define LOG2_MIN_CU_CHROMA_QP_DELTA_SIZE		BW_FIELD(206, 3)
+#define CB_QP_OFFSET_LIST(i)				BW_FIELD(209 + (i) * 5, 5) // i: 0-5
+#define CB_CR_OFFSET_LIST(i)				BW_FIELD(239 + (i) * 5, 5) // i: 0-5
+#define CHROMA_QP_OFFSET_LIST_LEN_MINUS1		BW_FIELD(269, 3)
+
+/* mvc0 && mvc1 */
+#define MVC_FF						BW_FIELD(272, 16)
+#define MVC_00						BW_FIELD(288, 9)
+
+/* poc info */
+#define RESERVED2					BW_FIELD(297, 3)
+#define CURRENT_POC					BW_FIELD(300, 32)
+#define REF_PIC_POC(i)					BW_FIELD(332 + (i) * 32, 32) // i: 0-14
+#define RESERVED3					BW_FIELD(812, 32)
+#define REF_IS_VALID(i)					BW_FIELD(844 + (i), 1) // i: 0-14
+#define RESERVED4					BW_FIELD(859, 1)
+
+/* tile info*/
+#define NUM_TILE_COLUMNS				BW_FIELD(860, 5)
+#define NUM_TILE_ROWS					BW_FIELD(865, 5)
+#define COLUMN_WIDTH(i)					BW_FIELD(870 + (i) * 12, 12) // i: 0-19
+#define ROW_HEIGHT(i)					BW_FIELD(1110 + (i) * 12, 12) // i: 0-21
+
+#define HEVC_SPS_SIZE					ALIGN(1110 + 22 * 12, 256)
 
 struct rkvdec_hevc_sps_pps {
-	// SPS
-	u16 video_parameters_set_id			: 4;
-	u16 seq_parameters_set_id_sps			: 4;
-	u16 chroma_format_idc				: 2;
-	u16 width					: 16;
-	u16 height					: 16;
-	u16 bit_depth_luma				: 3;
-	u16 bit_depth_chroma				: 3;
-	u16 max_pic_order_count_lsb			: 5;
-	u16 diff_max_min_luma_coding_block_size		: 2;
-	u16 min_luma_coding_block_size			: 3;
-	u16 min_transform_block_size			: 3;
-	u16 diff_max_min_transform_block_size		: 2;
-	u16 max_transform_hierarchy_depth_inter		: 3;
-	u16 max_transform_hierarchy_depth_intra		: 3;
-	u16 scaling_list_enabled_flag			: 1;
-	u16 amp_enabled_flag				: 1;
-	u16 sample_adaptive_offset_enabled_flag		: 1;
-	u16 pcm_enabled_flag				: 1;
-	u16 pcm_sample_bit_depth_luma			: 4;
-	u16 pcm_sample_bit_depth_chroma			: 4;
-	u16 pcm_loop_filter_disabled_flag		: 1;
-	u16 diff_max_min_pcm_luma_coding_block_size	: 3;
-	u16 min_pcm_luma_coding_block_size		: 3;
-	u16 num_short_term_ref_pic_sets			: 7;
-	u16 long_term_ref_pics_present_flag		: 1;
-	u16 num_long_term_ref_pics_sps			: 6;
-	u16 sps_temporal_mvp_enabled_flag		: 1;
-	u16 strong_intra_smoothing_enabled_flag		: 1;
-	u16 reserved0					: 7;
-	u16 sps_max_dec_pic_buffering_minus1		: 4;
-	u16 separate_colour_plane_flag			: 1;
-	u16 high_precision_offsets_enabled_flag		: 1;
-	u16 persistent_rice_adaptation_enabled_flag	: 1;
-
-	// PPS
-	u16 picture_parameters_set_id			: 6;
-	u16 seq_parameters_set_id_pps			: 4;
-	u16 dependent_slice_segments_enabled_flag	: 1;
-	u16 output_flag_present_flag			: 1;
-	u16 num_extra_slice_header_bits			: 13;
-	u16 sign_data_hiding_enabled_flag		: 1;
-	u16 cabac_init_present_flag			: 1;
-	u16 num_ref_idx_l0_default_active		: 4;
-	u16 num_ref_idx_l1_default_active		: 4;
-	u16 init_qp_minus26				: 7;
-	u16 constrained_intra_pred_flag			: 1;
-	u16 transform_skip_enabled_flag			: 1;
-	u16 cu_qp_delta_enabled_flag			: 1;
-	u16 log2_min_cb_size				: 3;
-	u16 pps_cb_qp_offset				: 5;
-	u16 pps_cr_qp_offset				: 5;
-	u16 pps_slice_chroma_qp_offsets_present_flag	: 1;
-	u16 weighted_pred_flag				: 1;
-	u16 weighted_bipred_flag			: 1;
-	u16 transquant_bypass_enabled_flag		: 1;
-	u16 tiles_enabled_flag				: 1;
-	u16 entropy_coding_sync_enabled_flag		: 1;
-	u16 pps_loop_filter_across_slices_enabled_flag	: 1;
-	u16 loop_filter_across_tiles_enabled_flag	: 1;
-	u16 deblocking_filter_override_enabled_flag	: 1;
-	u16 pps_deblocking_filter_disabled_flag		: 1;
-	u16 pps_beta_offset_div2			: 4;
-	u16 pps_tc_offset_div2				: 4;
-	u16 lists_modification_present_flag		: 1;
-	u16 log2_parallel_merge_level			: 3;
-	u16 slice_segment_header_extension_present_flag	: 1;
-	u16 reserved1					: 3;
-
-	// pps extensions
-	u16 log2_max_transform_skip_block_size		: 2;
-	u16 cross_component_prediction_enabled_flag	: 1;
-	u16 chroma_qp_offset_list_enabled_flag		: 1;
-	u16 log2_min_cu_chroma_qp_delta_size		: 3;
-	u16 cb_qp_offset_list0				: 5;
-	u16 cb_qp_offset_list1				: 5;
-	u16 cb_qp_offset_list2				: 5;
-	u16 cb_qp_offset_list3				: 5;
-	u16 cb_qp_offset_list4				: 5;
-	u16 cb_qp_offset_list5				: 5;
-	u16 cb_cr_offset_list0				: 5;
-	u16 cb_cr_offset_list1				: 5;
-	u16 cb_cr_offset_list2				: 5;
-	u16 cb_cr_offset_list3				: 5;
-	u16 cb_cr_offset_list4				: 5;
-	u16 cb_cr_offset_list5				: 5;
-	u16 chroma_qp_offset_list_len_minus1		: 3;
-
-	/* mvc0 && mvc1 */
-	u16 mvc_ff					: 16;
-	u16 mvc_00					: 9;
-
-	/* poc info */
-	u16 reserved2					: 3;
-	u32 current_poc					: 32;
-	u32 ref_pic_poc0				: 32;
-	u32 ref_pic_poc1				: 32;
-	u32 ref_pic_poc2				: 32;
-	u32 ref_pic_poc3				: 32;
-	u32 ref_pic_poc4				: 32;
-	u32 ref_pic_poc5				: 32;
-	u32 ref_pic_poc6				: 32;
-	u32 ref_pic_poc7				: 32;
-	u32 ref_pic_poc8				: 32;
-	u32 ref_pic_poc9				: 32;
-	u32 ref_pic_poc10				: 32;
-	u32 ref_pic_poc11				: 32;
-	u32 ref_pic_poc12				: 32;
-	u32 ref_pic_poc13				: 32;
-	u32 ref_pic_poc14				: 32;
-	u32 reserved3					: 32;
-	u32 ref_is_valid				: 15;
-	u32 reserved4					: 1;
-
-	/* tile info*/
-	u16 num_tile_columns				: 5;
-	u16 num_tile_rows				: 5;
-	u32 column_width0				: 24;
-	u32 column_width1				: 24;
-	u32 column_width2				: 24;
-	u32 column_width3				: 24;
-	u32 column_width4				: 24;
-	u32 column_width5				: 24;
-	u32 column_width6				: 24;
-	u32 column_width7				: 24;
-	u32 column_width8				: 24;
-	u32 column_width9				: 24;
-	u32 row_height0					: 24;
-	u32 row_height1					: 24;
-	u32 row_height2					: 24;
-	u32 row_height3					: 24;
-	u32 row_height4					: 24;
-	u32 row_height5					: 24;
-	u32 row_height6					: 24;
-	u32 row_height7					: 24;
-	u32 row_height8					: 24;
-	u32 row_height9					: 24;
-	u32 row_height10				: 24;
-	u32 reserved5					: 2;
-	u32 padding;
-} __packed;
+	u32 info[HEVC_SPS_SIZE / 8 / 4];
+};
 
 struct rkvdec_hevc_priv_tbl {
 	struct rkvdec_hevc_sps_pps param_set;
@@ -171,51 +128,6 @@ struct rkvdec_hevc_ctx {
 	struct vdpu383_regs_h26x		regs;
 };
 
-static void set_column_row(struct rkvdec_hevc_sps_pps *hw_ps, u16 *column, u16 *row)
-{
-	hw_ps->column_width0 = column[0] | (column[1] << 12);
-	hw_ps->row_height0 = row[0] | (row[1] << 12);
-	hw_ps->column_width1 = column[2] | (column[3] << 12);
-	hw_ps->row_height1 = row[2] | (row[3] << 12);
-	hw_ps->column_width2 = column[4] | (column[5] << 12);
-	hw_ps->row_height2 = row[4] | (row[5] << 12);
-	hw_ps->column_width3 = column[6] | (column[7] << 12);
-	hw_ps->row_height3 = row[6] | (row[7] << 12);
-	hw_ps->column_width4 = column[8] | (column[9] << 12);
-	hw_ps->row_height4 = row[8] | (row[9] << 12);
-	hw_ps->column_width5 = column[10] | (column[11] << 12);
-	hw_ps->row_height5 = row[10] | (row[11] << 12);
-	hw_ps->column_width6 = column[12] | (column[13] << 12);
-	hw_ps->row_height6 = row[12] | (row[13] << 12);
-	hw_ps->column_width7 = column[14] | (column[15] << 12);
-	hw_ps->row_height7 = row[14] | (row[15] << 12);
-	hw_ps->column_width8 = column[16] | (column[17] << 12);
-	hw_ps->row_height8 = row[16] | (row[17] << 12);
-	hw_ps->column_width9 = column[18] | (column[19] << 12);
-	hw_ps->row_height9 = row[18] | (row[19] << 12);
-
-	hw_ps->row_height10 = row[20] | (row[21] << 12);
-}
-
-static void set_pps_ref_pic_poc(struct rkvdec_hevc_sps_pps *hw_ps, const struct v4l2_hevc_dpb_entry *dpb)
-{
-	hw_ps->ref_pic_poc0 = dpb[0].pic_order_cnt_val;
-	hw_ps->ref_pic_poc1 = dpb[1].pic_order_cnt_val;
-	hw_ps->ref_pic_poc2 = dpb[2].pic_order_cnt_val;
-	hw_ps->ref_pic_poc3 = dpb[3].pic_order_cnt_val;
-	hw_ps->ref_pic_poc4 = dpb[4].pic_order_cnt_val;
-	hw_ps->ref_pic_poc5 = dpb[5].pic_order_cnt_val;
-	hw_ps->ref_pic_poc6 = dpb[6].pic_order_cnt_val;
-	hw_ps->ref_pic_poc7 = dpb[7].pic_order_cnt_val;
-	hw_ps->ref_pic_poc8 = dpb[8].pic_order_cnt_val;
-	hw_ps->ref_pic_poc9 = dpb[9].pic_order_cnt_val;
-	hw_ps->ref_pic_poc10 = dpb[10].pic_order_cnt_val;
-	hw_ps->ref_pic_poc11 = dpb[11].pic_order_cnt_val;
-	hw_ps->ref_pic_poc12 = dpb[12].pic_order_cnt_val;
-	hw_ps->ref_pic_poc13 = dpb[13].pic_order_cnt_val;
-	hw_ps->ref_pic_poc14 = dpb[14].pic_order_cnt_val;
-}
-
 static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 			    struct rkvdec_hevc_run *run)
 {
@@ -245,104 +157,130 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 	memset(hw_ps, 0, sizeof(*hw_ps));
 
 	/* write sps */
-	hw_ps->video_parameters_set_id = sps->video_parameter_set_id;
-	hw_ps->seq_parameters_set_id_sps = sps->seq_parameter_set_id;
-	hw_ps->chroma_format_idc = sps->chroma_format_idc;
+	rkvdec_set_bw_field(hw_ps->info, VIDEO_PARAMETER_SET_ID, sps->video_parameter_set_id);
+	rkvdec_set_bw_field(hw_ps->info, SEQ_PARAMETER_SET_ID, sps->seq_parameter_set_id);
+	rkvdec_set_bw_field(hw_ps->info, CHROMA_FORMAT_IDC, sps->chroma_format_idc);
 
 	log2_min_cb_size = sps->log2_min_luma_coding_block_size_minus3 + 3;
 	width = sps->pic_width_in_luma_samples;
 	height = sps->pic_height_in_luma_samples;
-	hw_ps->width = width;
-	hw_ps->height = height;
-	hw_ps->bit_depth_luma = sps->bit_depth_luma_minus8 + 8;
-	hw_ps->bit_depth_chroma = sps->bit_depth_chroma_minus8 + 8;
-	hw_ps->max_pic_order_count_lsb = sps->log2_max_pic_order_cnt_lsb_minus4 + 4;
-	hw_ps->diff_max_min_luma_coding_block_size = sps->log2_diff_max_min_luma_coding_block_size;
-	hw_ps->min_luma_coding_block_size = sps->log2_min_luma_coding_block_size_minus3 + 3;
-	hw_ps->min_transform_block_size = sps->log2_min_luma_transform_block_size_minus2 + 2;
-	hw_ps->diff_max_min_transform_block_size =
-		sps->log2_diff_max_min_luma_transform_block_size;
-	hw_ps->max_transform_hierarchy_depth_inter = sps->max_transform_hierarchy_depth_inter;
-	hw_ps->max_transform_hierarchy_depth_intra = sps->max_transform_hierarchy_depth_intra;
-	hw_ps->scaling_list_enabled_flag =
-		!!(sps->flags & V4L2_HEVC_SPS_FLAG_SCALING_LIST_ENABLED);
-	hw_ps->amp_enabled_flag = !!(sps->flags & V4L2_HEVC_SPS_FLAG_AMP_ENABLED);
-	hw_ps->sample_adaptive_offset_enabled_flag =
-		!!(sps->flags & V4L2_HEVC_SPS_FLAG_SAMPLE_ADAPTIVE_OFFSET);
+
+	rkvdec_set_bw_field(hw_ps->info, PIC_WIDTH_IN_LUMA_SAMPLES, width);
+	rkvdec_set_bw_field(hw_ps->info, PIC_HEIGHT_IN_LUMA_SAMPLES, height);
+	rkvdec_set_bw_field(hw_ps->info, BIT_DEPTH_LUMA, sps->bit_depth_luma_minus8 + 8);
+	rkvdec_set_bw_field(hw_ps->info, BIT_DEPTH_CHROMA, sps->bit_depth_chroma_minus8 + 8);
+	rkvdec_set_bw_field(hw_ps->info, LOG2_MAX_PIC_ORDER_CNT_LSB,
+			    sps->log2_max_pic_order_cnt_lsb_minus4 + 4);
+	rkvdec_set_bw_field(hw_ps->info, LOG2_DIFF_MAX_MIN_LUMA_CODING_BLOCK_SIZE,
+			    sps->log2_diff_max_min_luma_coding_block_size);
+	rkvdec_set_bw_field(hw_ps->info, LOG2_MIN_LUMA_CODING_BLOCK_SIZE,
+			    sps->log2_min_luma_coding_block_size_minus3 + 3);
+	rkvdec_set_bw_field(hw_ps->info, LOG2_MIN_TRANSFORM_BLOCK_SIZE,
+			    sps->log2_min_luma_transform_block_size_minus2 + 2);
+	rkvdec_set_bw_field(hw_ps->info, LOG2_DIFF_MAX_MIN_LUMA_TRANSFORM_BLOCK_SIZE,
+			    sps->log2_diff_max_min_luma_transform_block_size);
+	rkvdec_set_bw_field(hw_ps->info, MAX_TRANSFORM_HIERARCHY_DEPTH_INTER,
+			    sps->max_transform_hierarchy_depth_inter);
+	rkvdec_set_bw_field(hw_ps->info, MAX_TRANSFORM_HIERARCHY_DEPTH_INTRA,
+			    sps->max_transform_hierarchy_depth_intra);
+	rkvdec_set_bw_field(hw_ps->info, SCALING_LIST_ENABLED_FLAG,
+			    !!(sps->flags & V4L2_HEVC_SPS_FLAG_SCALING_LIST_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, AMP_ENABLED_FLAG,
+			    !!(sps->flags & V4L2_HEVC_SPS_FLAG_AMP_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, SAMPLE_ADAPTIVE_OFFSET_ENABLED_FLAG,
+			    !!(sps->flags & V4L2_HEVC_SPS_FLAG_SAMPLE_ADAPTIVE_OFFSET));
 
 	pcm_enabled = !!(sps->flags & V4L2_HEVC_SPS_FLAG_PCM_ENABLED);
-	hw_ps->pcm_enabled_flag = pcm_enabled;
-	hw_ps->pcm_sample_bit_depth_luma =
-		pcm_enabled ? sps->pcm_sample_bit_depth_luma_minus1 + 1 : 0;
-	hw_ps->pcm_sample_bit_depth_chroma =
-		pcm_enabled ? sps->pcm_sample_bit_depth_chroma_minus1 + 1 : 0;
-	hw_ps->pcm_loop_filter_disabled_flag =
-		!!(sps->flags & V4L2_HEVC_SPS_FLAG_PCM_LOOP_FILTER_DISABLED);
-	hw_ps->diff_max_min_pcm_luma_coding_block_size =
-		sps->log2_diff_max_min_pcm_luma_coding_block_size;
-	hw_ps->min_pcm_luma_coding_block_size =
-		pcm_enabled ? sps->log2_min_pcm_luma_coding_block_size_minus3 + 3 : 0;
-	hw_ps->num_short_term_ref_pic_sets = sps->num_short_term_ref_pic_sets;
-	hw_ps->long_term_ref_pics_present_flag =
-		!!(sps->flags & V4L2_HEVC_SPS_FLAG_LONG_TERM_REF_PICS_PRESENT);
-	hw_ps->num_long_term_ref_pics_sps = sps->num_long_term_ref_pics_sps;
-	hw_ps->sps_temporal_mvp_enabled_flag =
-		!!(sps->flags & V4L2_HEVC_SPS_FLAG_SPS_TEMPORAL_MVP_ENABLED);
-	hw_ps->strong_intra_smoothing_enabled_flag =
-		!!(sps->flags & V4L2_HEVC_SPS_FLAG_STRONG_INTRA_SMOOTHING_ENABLED);
-	hw_ps->sps_max_dec_pic_buffering_minus1 = sps->sps_max_dec_pic_buffering_minus1;
+	rkvdec_set_bw_field(hw_ps->info, PCM_ENABLED_FLAG, pcm_enabled);
+	rkvdec_set_bw_field(hw_ps->info, PCM_SAMPLE_BIT_DEPTH_LUMA,
+			    pcm_enabled ? sps->pcm_sample_bit_depth_luma_minus1 + 1 : 0);
+	rkvdec_set_bw_field(hw_ps->info, PCM_SAMPLE_BIT_DEPTH_CHROMA,
+			    pcm_enabled ? sps->pcm_sample_bit_depth_chroma_minus1 + 1 : 0);
+	rkvdec_set_bw_field(hw_ps->info, PCM_LOOP_FILTER_DISABLED_FLAG,
+			    !!(sps->flags & V4L2_HEVC_SPS_FLAG_PCM_LOOP_FILTER_DISABLED));
+	rkvdec_set_bw_field(hw_ps->info, LOG2_DIFF_MAX_MIN_PCM_LUMA_CODING_BLOCK_SIZE,
+			    sps->log2_diff_max_min_pcm_luma_coding_block_size);
+	rkvdec_set_bw_field(hw_ps->info, LOG2_MIN_PCM_LUMA_CODING_BLOCK_SIZE,
+			    pcm_enabled ? sps->log2_min_pcm_luma_coding_block_size_minus3 + 3 : 0);
+	rkvdec_set_bw_field(hw_ps->info, NUM_SHORT_TERM_REF_PIC_SETS,
+			    sps->num_short_term_ref_pic_sets);
+	rkvdec_set_bw_field(hw_ps->info, LONG_TERM_REF_PICS_PRESENT_FLAG,
+			    !!(sps->flags & V4L2_HEVC_SPS_FLAG_LONG_TERM_REF_PICS_PRESENT));
+	rkvdec_set_bw_field(hw_ps->info, NUM_LONG_TERM_REF_PICS_SPS,
+			    sps->num_long_term_ref_pics_sps);
+	rkvdec_set_bw_field(hw_ps->info, SPS_TEMPORAL_MVP_ENABLED_FLAG,
+			    !!(sps->flags & V4L2_HEVC_SPS_FLAG_SPS_TEMPORAL_MVP_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, STRONG_INTRA_SMOOTHING_ENABLED_FLAG,
+			    !!(sps->flags & V4L2_HEVC_SPS_FLAG_STRONG_INTRA_SMOOTHING_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, SPS_MAX_DEC_PIC_BUFFERING_MINUS1,
+			    sps->sps_max_dec_pic_buffering_minus1);
 
 	/* write pps */
-	hw_ps->picture_parameters_set_id = pps->pic_parameter_set_id;
-	hw_ps->seq_parameters_set_id_pps = sps->seq_parameter_set_id;
-	hw_ps->dependent_slice_segments_enabled_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT_ENABLED);
-	hw_ps->output_flag_present_flag = !!(pps->flags & V4L2_HEVC_PPS_FLAG_OUTPUT_FLAG_PRESENT);
-	hw_ps->num_extra_slice_header_bits = pps->num_extra_slice_header_bits;
-	hw_ps->sign_data_hiding_enabled_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_SIGN_DATA_HIDING_ENABLED);
-	hw_ps->cabac_init_present_flag = !!(pps->flags & V4L2_HEVC_PPS_FLAG_CABAC_INIT_PRESENT);
-	hw_ps->num_ref_idx_l0_default_active = pps->num_ref_idx_l0_default_active_minus1 + 1;
-	hw_ps->num_ref_idx_l1_default_active = pps->num_ref_idx_l1_default_active_minus1 + 1;
-	hw_ps->init_qp_minus26 = pps->init_qp_minus26;
-	hw_ps->constrained_intra_pred_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_CONSTRAINED_INTRA_PRED);
-	hw_ps->transform_skip_enabled_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_TRANSFORM_SKIP_ENABLED);
-	hw_ps->cu_qp_delta_enabled_flag = !!(pps->flags & V4L2_HEVC_PPS_FLAG_CU_QP_DELTA_ENABLED);
-	hw_ps->log2_min_cb_size = log2_min_cb_size +
-				  sps->log2_diff_max_min_luma_coding_block_size -
-				  pps->diff_cu_qp_delta_depth;
-	hw_ps->pps_cb_qp_offset = pps->pps_cb_qp_offset;
-	hw_ps->pps_cr_qp_offset = pps->pps_cr_qp_offset;
-	hw_ps->pps_slice_chroma_qp_offsets_present_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT);
-	hw_ps->weighted_pred_flag = !!(pps->flags & V4L2_HEVC_PPS_FLAG_WEIGHTED_PRED);
-	hw_ps->weighted_bipred_flag = !!(pps->flags & V4L2_HEVC_PPS_FLAG_WEIGHTED_BIPRED);
-	hw_ps->transquant_bypass_enabled_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_TRANSQUANT_BYPASS_ENABLED);
+	rkvdec_set_bw_field(hw_ps->info, PIC_PARAMETER_SET_ID, pps->pic_parameter_set_id);
+	rkvdec_set_bw_field(hw_ps->info, SEQ_PARAMETER_SET_ID, sps->seq_parameter_set_id);
+	rkvdec_set_bw_field(hw_ps->info, DEPENDENT_SLICE_SEGMENTS_ENABLED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, OUTPUT_FLAG_PRESENT_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_OUTPUT_FLAG_PRESENT));
+	rkvdec_set_bw_field(hw_ps->info, NUM_EXTRA_SLICE_HEADER_BITS,
+			    pps->num_extra_slice_header_bits);
+	rkvdec_set_bw_field(hw_ps->info, SIGN_DATA_HIDING_ENABLED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_SIGN_DATA_HIDING_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, CABAC_INIT_PRESENT_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_CABAC_INIT_PRESENT));
+	rkvdec_set_bw_field(hw_ps->info, NUM_REF_IDX_L0_DEFAULT_ACTIVE,
+			    pps->num_ref_idx_l0_default_active_minus1 + 1);
+	rkvdec_set_bw_field(hw_ps->info, NUM_REF_IDX_L1_DEFAULT_ACTIVE,
+			    pps->num_ref_idx_l1_default_active_minus1 + 1);
+	rkvdec_set_bw_field(hw_ps->info, INIT_QP_MINUS26, pps->init_qp_minus26);
+	rkvdec_set_bw_field(hw_ps->info, CONSTRAINED_INTRA_PRED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_CONSTRAINED_INTRA_PRED));
+	rkvdec_set_bw_field(hw_ps->info, TRANSFORM_SKIP_ENABLED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_TRANSFORM_SKIP_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, CU_QP_DELTA_ENABLED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_CU_QP_DELTA_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, LOG2_MIN_CU_QP_DELTA_SIZE, log2_min_cb_size +
+			    sps->log2_diff_max_min_luma_coding_block_size -
+			    pps->diff_cu_qp_delta_depth);
+	rkvdec_set_bw_field(hw_ps->info, PPS_CB_QP_OFFSET, pps->pps_cb_qp_offset);
+	rkvdec_set_bw_field(hw_ps->info, PPS_CR_QP_OFFSET, pps->pps_cr_qp_offset);
+	rkvdec_set_bw_field(hw_ps->info, PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT_FLAG,
+			    !!(pps->flags &
+			       V4L2_HEVC_PPS_FLAG_PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT));
+	rkvdec_set_bw_field(hw_ps->info, WEIGHTED_PRED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_WEIGHTED_PRED));
+	rkvdec_set_bw_field(hw_ps->info, WEIGHTED_BIPRED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_WEIGHTED_BIPRED));
+	rkvdec_set_bw_field(hw_ps->info, TRANSQUANT_BYPASS_ENABLED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_TRANSQUANT_BYPASS_ENABLED));
 	tiles_enabled = !!(pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED);
-	hw_ps->tiles_enabled_flag = tiles_enabled;
-	hw_ps->entropy_coding_sync_enabled_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_ENTROPY_CODING_SYNC_ENABLED);
-	hw_ps->pps_loop_filter_across_slices_enabled_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED);
-	hw_ps->loop_filter_across_tiles_enabled_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED);
-	hw_ps->deblocking_filter_override_enabled_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_OVERRIDE_ENABLED);
-	hw_ps->pps_deblocking_filter_disabled_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_PPS_DISABLE_DEBLOCKING_FILTER);
-	hw_ps->pps_beta_offset_div2 = pps->pps_beta_offset_div2;
-	hw_ps->pps_tc_offset_div2 = pps->pps_tc_offset_div2;
-	hw_ps->lists_modification_present_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_LISTS_MODIFICATION_PRESENT);
-	hw_ps->log2_parallel_merge_level = pps->log2_parallel_merge_level_minus2 + 2;
-	hw_ps->slice_segment_header_extension_present_flag =
-		!!(pps->flags & V4L2_HEVC_PPS_FLAG_SLICE_SEGMENT_HEADER_EXTENSION_PRESENT);
-	hw_ps->num_tile_columns = tiles_enabled ? pps->num_tile_columns_minus1 + 1 : 1;
-	hw_ps->num_tile_rows = tiles_enabled ? pps->num_tile_rows_minus1 + 1 : 1;
-	hw_ps->mvc_ff = 0xffff;
+	rkvdec_set_bw_field(hw_ps->info, TILES_ENABLED_FLAG, tiles_enabled);
+	rkvdec_set_bw_field(hw_ps->info, ENTROPY_CODING_SYNC_ENABLED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_ENTROPY_CODING_SYNC_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED_FLAG,
+			    !!(pps->flags &
+			       V4L2_HEVC_PPS_FLAG_PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, LOOP_FILTER_ACROSS_TILES_ENABLED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, DEBLOCKING_FILTER_OVERRIDE_ENABLED_FLAG,
+			    !!(pps->flags &
+			       V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_OVERRIDE_ENABLED));
+	rkvdec_set_bw_field(hw_ps->info, PPS_DEBLOCKING_FILTER_DISABLED_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_PPS_DISABLE_DEBLOCKING_FILTER));
+	rkvdec_set_bw_field(hw_ps->info, PPS_BETA_OFFSET_DIV2, pps->pps_beta_offset_div2);
+	rkvdec_set_bw_field(hw_ps->info, PPS_TC_OFFSET_DIV2, pps->pps_tc_offset_div2);
+	rkvdec_set_bw_field(hw_ps->info, LISTS_MODIFICATION_PRESENT_FLAG,
+			    !!(pps->flags & V4L2_HEVC_PPS_FLAG_LISTS_MODIFICATION_PRESENT));
+	rkvdec_set_bw_field(hw_ps->info, LOG2_PARALLEL_MERGE_LEVEL,
+			    pps->log2_parallel_merge_level_minus2 + 2);
+	rkvdec_set_bw_field(hw_ps->info, SLICE_SEGMENT_HEADER_EXTENSION_PRESENT_FLAG,
+			    !!(pps->flags &
+			       V4L2_HEVC_PPS_FLAG_SLICE_SEGMENT_HEADER_EXTENSION_PRESENT));
+	rkvdec_set_bw_field(hw_ps->info, NUM_TILE_COLUMNS,
+			    tiles_enabled ? pps->num_tile_columns_minus1 + 1 : 1);
+	rkvdec_set_bw_field(hw_ps->info, NUM_TILE_ROWS,
+			    tiles_enabled ? pps->num_tile_rows_minus1 + 1 : 1);
+	rkvdec_set_bw_field(hw_ps->info, MVC_FF, 0xffff);
 
 	// Setup tiles information
 	memset(column_width, 0, sizeof(column_width));
@@ -367,15 +305,19 @@ static void assemble_hw_pps(struct rkvdec_ctx *ctx,
 		row_height[0] = (height + max_cu_width - 1) / max_cu_width;
 	}
 
-	set_column_row(hw_ps, column_width, row_height);
+	for (i = 0; i < 20; i++)
+		rkvdec_set_bw_field(hw_ps->info, COLUMN_WIDTH(i), column_width[i]);
+	for (i = 0; i < 22; i++)
+		rkvdec_set_bw_field(hw_ps->info, ROW_HEIGHT(i), row_height[i]);
 
 	// Setup POC information
-	hw_ps->current_poc = dec_params->pic_order_cnt_val;
+	rkvdec_set_bw_field(hw_ps->info, CURRENT_POC, dec_params->pic_order_cnt_val);
 
-	set_pps_ref_pic_poc(hw_ps, dec_params->dpb);
 	for (i = 0; i < ARRAY_SIZE(dec_params->dpb); i++) {
-		u32 valid = !!(dec_params->num_active_dpb_entries > i);
-		hw_ps->ref_is_valid |= valid << i;
+		rkvdec_set_bw_field(hw_ps->info, REF_IS_VALID(i),
+				    !!(dec_params->num_active_dpb_entries > i));
+		rkvdec_set_bw_field(hw_ps->info, REF_PIC_POC(i),
+				    dec_params->dpb[i].pic_order_cnt_val);
 	}
 }
 
diff --git a/drivers/media/platform/st/stm32/stm32-dcmi.c b/drivers/media/platform/st/stm32/stm32-dcmi.c
index e5663fbe64220..eeb0199864dd1 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmi.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmi.c
@@ -2195,6 +2195,7 @@ static int dcmi_probe(struct platform_device *pdev)
 	return 0;
 
 err_cleanup:
+	v4l2_async_nf_unregister(&dcmi->notifier);
 	v4l2_async_nf_cleanup(&dcmi->notifier);
 err_media_entity_cleanup:
 	media_entity_cleanup(&dcmi->vdev->entity);
diff --git a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c
index a42f43d19f9ef..f0e809458489f 100644
--- a/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c
+++ b/drivers/media/platform/st/stm32/stm32-dcmipp/dcmipp-bytecap.c
@@ -401,8 +401,10 @@ static int dcmipp_bytecap_start_streaming(struct vb2_queue *vq,
 	 */
 	if (!vcap->s_subdev) {
 		pad = media_pad_remote_pad_first(&vcap->vdev.entity.pads[0]);
-		if (!pad || !is_media_entity_v4l2_subdev(pad->entity))
-			return -EINVAL;
+		if (!pad || !is_media_entity_v4l2_subdev(pad->entity)) {
+			ret = -EINVAL;
+			goto err_buffer_done;
+		}
 		vcap->s_subdev = media_entity_to_v4l2_subdev(pad->entity);
 		vcap->s_subdev_pad_nb = pad->index;
 	}
diff --git a/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c b/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c
index e911c7f7acc52..4781db21c2055 100644
--- a/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c
+++ b/drivers/media/platform/sunxi/sun4i-csi/sun4i_dma.c
@@ -234,8 +234,10 @@ static int sun4i_csi_start_streaming(struct vb2_queue *vq, unsigned int count)
 	int ret;
 
 	csi_fmt = sun4i_csi_find_format(&csi->fmt.pixelformat, NULL);
-	if (!csi_fmt)
-		return -EINVAL;
+	if (!csi_fmt) {
+		ret = -EINVAL;
+		goto err_clear_dma_queue;
+	}
 
 	dev_dbg(csi->dev, "Starting capture\n");
 
diff --git a/drivers/media/platform/synopsys/dw-mipi-csi2rx.c b/drivers/media/platform/synopsys/dw-mipi-csi2rx.c
index 02eb4a6cafade..41e48365167e5 100644
--- a/drivers/media/platform/synopsys/dw-mipi-csi2rx.c
+++ b/drivers/media/platform/synopsys/dw-mipi-csi2rx.c
@@ -11,6 +11,7 @@
 #include <linux/clk.h>
 #include <linux/delay.h>
 #include <linux/io.h>
+#include <linux/iopoll.h>
 #include <linux/module.h>
 #include <linux/of.h>
 #include <linux/phy/phy.h>
@@ -35,6 +36,8 @@
 #define DW_REG_EXIST		BIT(31)
 #define DW_REG(x)		(DW_REG_EXIST | (x))
 
+#define DPHY_STOPSTATE_CLK_LANE		BIT(16)
+
 #define DPHY_TEST_CTRL0_TEST_CLR	BIT(0)
 
 #define IPI_VCID_VC(x)			FIELD_PREP(GENMASK(1, 0), (x))
@@ -65,6 +68,7 @@ enum dw_mipi_csi2rx_regs_index {
 	DW_MIPI_CSI2RX_PHY_TST_CTRL0,
 	DW_MIPI_CSI2RX_PHY_TST_CTRL1,
 	DW_MIPI_CSI2RX_PHY_SHUTDOWNZ,
+	DW_MIPI_CSI2RX_PHY_STOPSTATE,
 	DW_MIPI_CSI2RX_IPI_DATATYPE,
 	DW_MIPI_CSI2RX_IPI_MEM_FLUSH,
 	DW_MIPI_CSI2RX_IPI_MODE,
@@ -87,6 +91,7 @@ struct dw_mipi_csi2rx_drvdata {
 	void (*dphy_assert_reset)(struct dw_mipi_csi2rx_device *csi2);
 	void (*dphy_deassert_reset)(struct dw_mipi_csi2rx_device *csi2);
 	void (*ipi_enable)(struct dw_mipi_csi2rx_device *csi2);
+	int (*wait_for_phy_stopstate)(struct dw_mipi_csi2rx_device *csi2);
 };
 
 struct dw_mipi_csi2rx_format {
@@ -113,6 +118,7 @@ struct dw_mipi_csi2rx_device {
 
 	enum v4l2_mbus_type bus_type;
 	u32 lanes_num;
+	u64 enabled_streams;
 
 	const struct dw_mipi_csi2rx_drvdata *drvdata;
 };
@@ -138,6 +144,7 @@ static const u32 imx93_regs[DW_MIPI_CSI2RX_MAX] = {
 	[DW_MIPI_CSI2RX_PHY_SHUTDOWNZ] = DW_REG(0x40),
 	[DW_MIPI_CSI2RX_DPHY_RSTZ] = DW_REG(0x44),
 	[DW_MIPI_CSI2RX_PHY_STATE] = DW_REG(0x48),
+	[DW_MIPI_CSI2RX_PHY_STOPSTATE] = DW_REG(0x4c),
 	[DW_MIPI_CSI2RX_PHY_TST_CTRL0] = DW_REG(0x50),
 	[DW_MIPI_CSI2RX_PHY_TST_CTRL1] = DW_REG(0x54),
 	[DW_MIPI_CSI2RX_IPI_MODE] = DW_REG(0x80),
@@ -147,6 +154,17 @@ static const u32 imx93_regs[DW_MIPI_CSI2RX_MAX] = {
 	[DW_MIPI_CSI2RX_IPI_SOFTRSTN] = DW_REG(0xa0),
 };
 
+static const u32 imx95_regs[DW_MIPI_CSI2RX_MAX] = {
+	[DW_MIPI_CSI2RX_N_LANES] = DW_REG(0x4),
+	[DW_MIPI_CSI2RX_RESETN] = DW_REG(0x8),
+	[DW_MIPI_CSI2RX_PHY_SHUTDOWNZ] = DW_REG(0x40),
+	[DW_MIPI_CSI2RX_DPHY_RSTZ] = DW_REG(0x44),
+	[DW_MIPI_CSI2RX_PHY_STATE] = DW_REG(0x48),
+	[DW_MIPI_CSI2RX_PHY_STOPSTATE] = DW_REG(0x4c),
+	[DW_MIPI_CSI2RX_PHY_TST_CTRL0] = DW_REG(0x50),
+	[DW_MIPI_CSI2RX_PHY_TST_CTRL1] = DW_REG(0x54),
+};
+
 static const struct v4l2_mbus_framefmt default_format = {
 	.width = 3840,
 	.height = 2160,
@@ -252,6 +270,26 @@ static const struct dw_mipi_csi2rx_format formats[] = {
 		.depth = 12,
 		.csi_dt = MIPI_CSI2_DT_RAW12,
 	},
+	{
+		.code = MEDIA_BUS_FMT_SBGGR16_1X16,
+		.depth = 16,
+		.csi_dt = MIPI_CSI2_DT_RAW16,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGBRG16_1X16,
+		.depth = 16,
+		.csi_dt = MIPI_CSI2_DT_RAW16,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SGRBG16_1X16,
+		.depth = 16,
+		.csi_dt = MIPI_CSI2_DT_RAW16,
+	},
+	{
+		.code = MEDIA_BUS_FMT_SRGGB16_1X16,
+		.depth = 16,
+		.csi_dt = MIPI_CSI2_DT_RAW16,
+	},
 };
 
 static inline struct dw_mipi_csi2rx_device *to_csi2(struct v4l2_subdev *sd)
@@ -311,7 +349,7 @@ dw_mipi_csi2rx_find_format(struct dw_mipi_csi2rx_device *csi2, u32 mbus_code)
 	WARN_ON(csi2->formats_num == 0);
 
 	for (unsigned int i = 0; i < csi2->formats_num; i++) {
-		const struct dw_mipi_csi2rx_format *format = &csi2->formats[i];
+		const struct dw_mipi_csi2rx_format *format = &formats[i];
 
 		if (format->code == mbus_code)
 			return format;
@@ -433,7 +471,7 @@ dw_mipi_csi2rx_enum_mbus_code(struct v4l2_subdev *sd,
 		if (code->index >= csi2->formats_num)
 			return -EINVAL;
 
-		code->code = csi2->formats[code->index].code;
+		code->code = formats[code->index].code;
 		return 0;
 	default:
 		return -EINVAL;
@@ -470,6 +508,17 @@ static int dw_mipi_csi2rx_set_fmt(struct v4l2_subdev *sd,
 
 	*src = *sink;
 
+	/* Store the CSIS format descriptor for active formats. */
+	if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
+		csi2->formats = fmt ? :
+			dw_mipi_csi2rx_find_format(csi2, default_format.code);
+
+		if (!csi2->formats) {
+			dev_err(csi2->dev, "Failed to find valid format\n");
+			return -EINVAL;
+		}
+	}
+
 	return 0;
 }
 
@@ -508,26 +557,42 @@ static int dw_mipi_csi2rx_enable_streams(struct v4l2_subdev *sd,
 					       DW_MIPI_CSI2RX_PAD_SRC,
 					       &streams_mask);
 
-	ret = pm_runtime_resume_and_get(dev);
-	if (ret)
-		goto err;
+	if (!csi2->enabled_streams) {
+		ret = pm_runtime_resume_and_get(dev);
+		if (ret)
+			goto err;
 
-	ret = dw_mipi_csi2rx_start(csi2);
-	if (ret) {
-		dev_err(dev, "failed to enable CSI hardware\n");
-		goto err_pm_runtime_put;
+		ret = dw_mipi_csi2rx_start(csi2);
+		if (ret) {
+			dev_err(dev, "failed to enable CSI hardware\n");
+			goto err_pm_runtime_put;
+		}
 	}
 
 	ret = v4l2_subdev_enable_streams(remote_sd, remote_pad->index, mask);
 	if (ret)
 		goto err_csi_stop;
 
+	if (!csi2->enabled_streams &&
+	    csi2->drvdata->wait_for_phy_stopstate) {
+		ret = csi2->drvdata->wait_for_phy_stopstate(csi2);
+		if (ret)
+			goto err_disable_streams;
+	}
+
+	csi2->enabled_streams |= streams_mask;
+
 	return 0;
 
+err_disable_streams:
+	v4l2_subdev_disable_streams(remote_sd, remote_pad->index, mask);
 err_csi_stop:
-	dw_mipi_csi2rx_stop(csi2);
+	/* Stop CSI hardware if no streams are enabled */
+	if (!csi2->enabled_streams)
+		dw_mipi_csi2rx_stop(csi2);
 err_pm_runtime_put:
-	pm_runtime_put(dev);
+	if (!csi2->enabled_streams)
+		pm_runtime_put(dev);
 err:
 	return ret;
 }
@@ -552,10 +617,15 @@ static int dw_mipi_csi2rx_disable_streams(struct v4l2_subdev *sd,
 					       &streams_mask);
 
 	ret = v4l2_subdev_disable_streams(remote_sd, remote_pad->index, mask);
+	if (ret)
+		dev_err(dev, "failed to disable streams on remote subdev: %d\n", ret);
 
-	dw_mipi_csi2rx_stop(csi2);
+	csi2->enabled_streams &= ~streams_mask;
 
-	pm_runtime_put(dev);
+	if (!csi2->enabled_streams) {
+		dw_mipi_csi2rx_stop(csi2);
+		pm_runtime_put(dev);
+	}
 
 	return ret;
 }
@@ -827,11 +897,39 @@ static void imx93_csi2rx_dphy_ipi_enable(struct dw_mipi_csi2rx_device *csi2)
 	dw_mipi_csi2rx_write(csi2, DW_MIPI_CSI2RX_IPI_MODE, val);
 }
 
+static int imx93_csi2rx_wait_for_phy_stopstate(struct dw_mipi_csi2rx_device *csi2)
+{
+	struct device *dev = csi2->dev;
+	u32 stopstate_mask;
+	u32 val;
+	int ret;
+
+	stopstate_mask = DPHY_STOPSTATE_CLK_LANE | GENMASK(csi2->lanes_num - 1, 0);
+
+	ret = read_poll_timeout(dw_mipi_csi2rx_read, val,
+				(val & stopstate_mask) == stopstate_mask,
+				 10, 1000, true,
+				 csi2, DW_MIPI_CSI2RX_PHY_STOPSTATE);
+	if (ret)
+		dev_err(dev, "lanes are not in stop state: %#x, expected %#x\n",
+			val, stopstate_mask);
+
+	return ret;
+}
+
 static const struct dw_mipi_csi2rx_drvdata imx93_drvdata = {
 	.regs = imx93_regs,
 	.dphy_assert_reset = imx93_csi2rx_dphy_assert_reset,
 	.dphy_deassert_reset = imx93_csi2rx_dphy_deassert_reset,
 	.ipi_enable = imx93_csi2rx_dphy_ipi_enable,
+	.wait_for_phy_stopstate = imx93_csi2rx_wait_for_phy_stopstate,
+};
+
+static const struct dw_mipi_csi2rx_drvdata imx95_drvdata = {
+	.regs = imx95_regs,
+	.dphy_assert_reset = imx93_csi2rx_dphy_assert_reset,
+	.dphy_deassert_reset = imx93_csi2rx_dphy_deassert_reset,
+	.wait_for_phy_stopstate = imx93_csi2rx_wait_for_phy_stopstate,
 };
 
 static const struct of_device_id dw_mipi_csi2rx_of_match[] = {
@@ -839,6 +937,10 @@ static const struct of_device_id dw_mipi_csi2rx_of_match[] = {
 		.compatible = "fsl,imx93-mipi-csi2",
 		.data = &imx93_drvdata,
 	},
+	{
+		.compatible = "fsl,imx95-mipi-csi2",
+		.data = &imx95_drvdata,
+	},
 	{
 		.compatible = "rockchip,rk3568-mipi-csi2",
 		.data = &rk3568_drvdata,
diff --git a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
index 61ad20b18b8d6..25f8ca0d6d946 100644
--- a/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
+++ b/drivers/media/platform/synopsys/hdmirx/snps_hdmirx.c
@@ -506,9 +506,9 @@ static void hdmirx_hpd_ctrl(struct snps_hdmirx_dev *hdmirx_dev, bool en)
 	hdmirx_writel(hdmirx_dev, CORE_CONFIG,
 		      hdmirx_dev->hpd_trigger_level_high ? en : !en);
 
-	/* 100ms delay as per HDMI spec */
+	/* 100ms delay as per HDMI spec + extra 50ms to cover internal delay */
 	if (!en)
-		msleep(100);
+		msleep(100 + 50);
 }
 
 static void hdmirx_write_edid_data(struct snps_hdmirx_dev *hdmirx_dev,
@@ -1776,7 +1776,7 @@ static void process_signal_change(struct snps_hdmirx_dev *hdmirx_dev)
 			   FIFO_UNDERFLOW_INT_EN |
 			   HDMIRX_AXI_ERROR_INT_EN, 0);
 	hdmirx_reset_dma(hdmirx_dev);
-	queue_delayed_work(system_unbound_wq,
+	queue_delayed_work(system_dfl_wq,
 			   &hdmirx_dev->delayed_work_res_change,
 			   msecs_to_jiffies(50));
 }
@@ -2238,7 +2238,7 @@ static void hdmirx_delayed_work_res_change(struct work_struct *work)
 
 		if (hdmirx_wait_signal_lock(hdmirx_dev)) {
 			hdmirx_plugout(hdmirx_dev);
-			queue_delayed_work(system_unbound_wq,
+			queue_delayed_work(system_dfl_wq,
 					   &hdmirx_dev->delayed_work_hotplug,
 					   msecs_to_jiffies(200));
 		} else {
@@ -2253,7 +2253,7 @@ static irqreturn_t hdmirx_5v_det_irq_handler(int irq, void *dev_id)
 {
 	struct snps_hdmirx_dev *hdmirx_dev = dev_id;
 
-	queue_delayed_work(system_unbound_wq,
+	queue_delayed_work(system_dfl_wq,
 			   &hdmirx_dev->delayed_work_hotplug,
 			   msecs_to_jiffies(10));
 
@@ -2518,7 +2518,7 @@ static void hdmirx_enable_irq(struct device *dev)
 	enable_irq(hdmirx_dev->dma_irq);
 	enable_irq(hdmirx_dev->det_irq);
 
-	queue_delayed_work(system_unbound_wq,
+	queue_delayed_work(system_dfl_wq,
 			   &hdmirx_dev->delayed_work_hotplug,
 			   msecs_to_jiffies(110));
 }
diff --git a/drivers/media/platform/ti/Kconfig b/drivers/media/platform/ti/Kconfig
index da33facf44678..d0cb05481bd85 100644
--- a/drivers/media/platform/ti/Kconfig
+++ b/drivers/media/platform/ti/Kconfig
@@ -83,6 +83,7 @@ config VIDEO_TI_J721E_CSI2RX
 	depends on VIDEO_CADENCE_CSI2RX
 	depends on PHY_CADENCE_DPHY_RX || COMPILE_TEST
 	depends on ARCH_K3 || COMPILE_TEST
+	depends on PM
 	select VIDEOBUF2_DMA_CONTIG
 	select V4L2_FWNODE
 	help
diff --git a/drivers/media/platform/ti/davinci/vpif_capture.c b/drivers/media/platform/ti/davinci/vpif_capture.c
index 15df3ea2f77e1..91cb6223561a8 100644
--- a/drivers/media/platform/ti/davinci/vpif_capture.c
+++ b/drivers/media/platform/ti/davinci/vpif_capture.c
@@ -1498,7 +1498,7 @@ vpif_capture_get_pdata(struct platform_device *pdev,
 	 * video ports & endpoints data.
 	 */
 	if (pdev->dev.parent && pdev->dev.parent->of_node)
-		pdev->dev.of_node = pdev->dev.parent->of_node;
+		device_set_of_node_from_dev(&pdev->dev, pdev->dev.parent);
 	if (!IS_ENABLED(CONFIG_OF) || !pdev->dev.of_node)
 		return pdev->dev.platform_data;
 
diff --git a/drivers/media/platform/ti/j721e-csi2rx/j721e-csi2rx.c b/drivers/media/platform/ti/j721e-csi2rx/j721e-csi2rx.c
index b75aa363d1bf2..21388284cbaab 100644
--- a/drivers/media/platform/ti/j721e-csi2rx/j721e-csi2rx.c
+++ b/drivers/media/platform/ti/j721e-csi2rx/j721e-csi2rx.c
@@ -13,6 +13,7 @@
 #include <linux/module.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/property.h>
 
 #include <media/cadence/cdns-csi2rx.h>
@@ -27,23 +28,25 @@
 #define SHIM_CNTL			0x10
 #define SHIM_CNTL_PIX_RST		BIT(0)
 
-#define SHIM_DMACNTX			0x20
+#define SHIM_DMACNTX(i)			(0x20 + ((i) * 0x20))
 #define SHIM_DMACNTX_EN			BIT(31)
 #define SHIM_DMACNTX_YUV422		GENMASK(27, 26)
 #define SHIM_DMACNTX_DUAL_PCK_CFG	BIT(24)
 #define SHIM_DMACNTX_SIZE		GENMASK(21, 20)
+#define SHIM_DMACNTX_VC			GENMASK(9, 6)
 #define SHIM_DMACNTX_FMT		GENMASK(5, 0)
 #define SHIM_DMACNTX_YUV422_MODE_11	3
 #define SHIM_DMACNTX_SIZE_8		0
 #define SHIM_DMACNTX_SIZE_16		1
 #define SHIM_DMACNTX_SIZE_32		2
 
-#define SHIM_PSI_CFG0			0x24
+#define SHIM_PSI_CFG0(i)		(0x24 + ((i) * 0x20))
 #define SHIM_PSI_CFG0_SRC_TAG		GENMASK(15, 0)
 #define SHIM_PSI_CFG0_DST_TAG		GENMASK(31, 16)
 
 #define TI_CSI2RX_MAX_PIX_PER_CLK	4
-#define PSIL_WORD_SIZE_BYTES		16
+#define TI_CSI2RX_MAX_CTX		32
+
 /*
  * There are no hard limits on the width or height. The DMA engine can handle
  * all sizes. The max width and height are arbitrary numbers for this driver.
@@ -53,6 +56,11 @@
 #define MAX_WIDTH_BYTES			SZ_16K
 #define MAX_HEIGHT_LINES		SZ_16K
 
+#define TI_CSI2RX_PAD_SINK		0
+#define TI_CSI2RX_PAD_FIRST_SOURCE	1
+#define TI_CSI2RX_MAX_SOURCE_PADS	TI_CSI2RX_MAX_CTX
+#define TI_CSI2RX_MAX_PADS		(1 + TI_CSI2RX_MAX_SOURCE_PADS)
+
 #define DRAIN_TIMEOUT_MS		50
 #define DRAIN_BUFFER_SIZE		SZ_32K
 
@@ -70,13 +78,13 @@ struct ti_csi2rx_buffer {
 	/* Common v4l2 buffer. Must be first. */
 	struct vb2_v4l2_buffer		vb;
 	struct list_head		list;
-	struct ti_csi2rx_dev		*csi;
+	struct ti_csi2rx_ctx		*ctx;
 };
 
 enum ti_csi2rx_dma_state {
 	TI_CSI2RX_DMA_STOPPED,	/* Streaming not started yet. */
-	TI_CSI2RX_DMA_IDLE,	/* Streaming but no pending DMA operation. */
 	TI_CSI2RX_DMA_ACTIVE,	/* Streaming and pending DMA operation. */
+	TI_CSI2RX_DMA_DRAINING, /* Dumping all the data in drain buffer */
 };
 
 struct ti_csi2rx_dma {
@@ -90,32 +98,54 @@ struct ti_csi2rx_dma {
 	 * Queue of buffers submitted to DMA engine.
 	 */
 	struct list_head		submitted;
-	/* Buffer to drain stale data from PSI-L endpoint */
-	struct {
-		void			*vaddr;
-		dma_addr_t		paddr;
-		size_t			len;
-	} drain;
+};
+
+struct ti_csi2rx_dev;
+
+struct ti_csi2rx_ctx {
+	struct ti_csi2rx_dev		*csi;
+	struct video_device		vdev;
+	struct vb2_queue		vidq;
+	struct mutex			mutex; /* To serialize ioctls. */
+	struct v4l2_format		v_fmt;
+	struct ti_csi2rx_dma		dma;
+	struct media_pad		pad;
+	struct completion		drain_complete;
+	u32				sequence;
+	u32				idx;
+	u32				vc;
+	u32				dt;
+	u32				stream;
 };
 
 struct ti_csi2rx_dev {
 	struct device			*dev;
 	void __iomem			*shim;
+	unsigned int			enable_count;
+	unsigned int			num_ctx;
 	struct v4l2_device		v4l2_dev;
-	struct video_device		vdev;
 	struct media_device		mdev;
 	struct media_pipeline		pipe;
-	struct media_pad		pad;
+	struct media_pad		pads[TI_CSI2RX_MAX_PADS];
 	struct v4l2_async_notifier	notifier;
 	struct v4l2_subdev		*source;
-	struct vb2_queue		vidq;
-	struct mutex			mutex; /* To serialize ioctls. */
-	struct v4l2_format		v_fmt;
-	struct ti_csi2rx_dma		dma;
-	u32				sequence;
+	struct v4l2_subdev		subdev;
+	struct ti_csi2rx_ctx		ctx[TI_CSI2RX_MAX_CTX];
+	struct notifier_block		pm_notifier;
 	u8				pix_per_clk;
+	/* Buffer to drain stale data from PSI-L endpoint */
+	struct {
+		void			*vaddr;
+		dma_addr_t		paddr;
+		size_t			len;
+	} drain;
 };
 
+static inline struct ti_csi2rx_dev *to_csi2rx_dev(struct v4l2_subdev *sd)
+{
+	return container_of(sd, struct ti_csi2rx_dev, subdev);
+}
+
 static const struct ti_csi2rx_fmt ti_csi2rx_formats[] = {
 	{
 		.fourcc			= V4L2_PIX_FMT_YUYV,
@@ -219,9 +249,13 @@ static const struct ti_csi2rx_fmt ti_csi2rx_formats[] = {
 };
 
 /* Forward declaration needed by ti_csi2rx_dma_callback. */
-static int ti_csi2rx_start_dma(struct ti_csi2rx_dev *csi,
+static int ti_csi2rx_start_dma(struct ti_csi2rx_ctx *ctx,
 			       struct ti_csi2rx_buffer *buf);
 
+/* Forward declarations needed by ti_csi2rx_drain_callback. */
+static int ti_csi2rx_drain_dma(struct ti_csi2rx_ctx *ctx);
+static int ti_csi2rx_dma_submit_pending(struct ti_csi2rx_ctx *ctx);
+
 static const struct ti_csi2rx_fmt *find_format_by_fourcc(u32 pixelformat)
 {
 	unsigned int i;
@@ -250,19 +284,12 @@ static void ti_csi2rx_fill_fmt(const struct ti_csi2rx_fmt *csi_fmt,
 			       struct v4l2_format *v4l2_fmt)
 {
 	struct v4l2_pix_format *pix = &v4l2_fmt->fmt.pix;
-	unsigned int pixels_in_word;
-
-	pixels_in_word = PSIL_WORD_SIZE_BYTES * 8 / csi_fmt->bpp;
 
 	/* Clamp width and height to sensible maximums (16K x 16K) */
 	pix->width = clamp_t(unsigned int, pix->width,
-			     pixels_in_word,
-			     MAX_WIDTH_BYTES * 8 / csi_fmt->bpp);
+			     1, MAX_WIDTH_BYTES * 8 / csi_fmt->bpp);
 	pix->height = clamp_t(unsigned int, pix->height, 1, MAX_HEIGHT_LINES);
 
-	/* Width should be a multiple of transfer word-size */
-	pix->width = rounddown(pix->width, pixels_in_word);
-
 	v4l2_fmt->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 	pix->pixelformat = csi_fmt->fourcc;
 	pix->bytesperline = pix->width * (csi_fmt->bpp / 8);
@@ -309,7 +336,7 @@ static int ti_csi2rx_enum_fmt_vid_cap(struct file *file, void *priv,
 static int ti_csi2rx_g_fmt_vid_cap(struct file *file, void *priv,
 				   struct v4l2_format *f)
 {
-	struct ti_csi2rx_dev *csi = video_drvdata(file);
+	struct ti_csi2rx_ctx *csi = video_drvdata(file);
 
 	*f = csi->v_fmt;
 
@@ -340,7 +367,7 @@ static int ti_csi2rx_try_fmt_vid_cap(struct file *file, void *priv,
 static int ti_csi2rx_s_fmt_vid_cap(struct file *file, void *priv,
 				   struct v4l2_format *f)
 {
-	struct ti_csi2rx_dev *csi = video_drvdata(file);
+	struct ti_csi2rx_ctx *csi = video_drvdata(file);
 	struct vb2_queue *q = &csi->vidq;
 	int ret;
 
@@ -360,23 +387,15 @@ static int ti_csi2rx_enum_framesizes(struct file *file, void *fh,
 				     struct v4l2_frmsizeenum *fsize)
 {
 	const struct ti_csi2rx_fmt *fmt;
-	unsigned int pixels_in_word;
 
 	fmt = find_format_by_fourcc(fsize->pixel_format);
 	if (!fmt || fsize->index != 0)
 		return -EINVAL;
 
-	/*
-	 * Number of pixels in one PSI-L word. The transfer happens in multiples
-	 * of PSI-L word sizes.
-	 */
-	pixels_in_word = PSIL_WORD_SIZE_BYTES * 8 / fmt->bpp;
-
 	fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE;
-	fsize->stepwise.min_width = pixels_in_word;
-	fsize->stepwise.max_width = rounddown(MAX_WIDTH_BYTES * 8 / fmt->bpp,
-					      pixels_in_word);
-	fsize->stepwise.step_width = pixels_in_word;
+	fsize->stepwise.min_width = 1;
+	fsize->stepwise.max_width = MAX_WIDTH_BYTES * 8 / fmt->bpp;
+	fsize->stepwise.step_width = 1;
 	fsize->stepwise.min_height = 1;
 	fsize->stepwise.max_height = MAX_HEIGHT_LINES;
 	fsize->stepwise.step_height = 1;
@@ -426,26 +445,51 @@ static int csi_async_notifier_bound(struct v4l2_async_notifier *notifier,
 static int csi_async_notifier_complete(struct v4l2_async_notifier *notifier)
 {
 	struct ti_csi2rx_dev *csi = dev_get_drvdata(notifier->v4l2_dev->dev);
-	struct video_device *vdev = &csi->vdev;
-	int ret;
+	int ret, i;
+
+	/* Create link from source to subdev */
+	ret = media_create_pad_link(&csi->source->entity,
+				    CSI2RX_BRIDGE_SOURCE_PAD,
+				    &csi->subdev.entity,
+				    TI_CSI2RX_PAD_SINK,
+				    MEDIA_LNK_FL_IMMUTABLE |
+				    MEDIA_LNK_FL_ENABLED);
 
-	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
 	if (ret)
 		return ret;
 
-	ret = media_create_pad_link(&csi->source->entity, CSI2RX_BRIDGE_SOURCE_PAD,
-				    &vdev->entity, csi->pad.index,
-				    MEDIA_LNK_FL_IMMUTABLE | MEDIA_LNK_FL_ENABLED);
+	/* Create and link video nodes for all DMA contexts */
+	for (i = 0; i < csi->num_ctx; i++) {
+		struct ti_csi2rx_ctx *ctx = &csi->ctx[i];
+		struct video_device *vdev = &ctx->vdev;
 
-	if (ret) {
-		video_unregister_device(vdev);
-		return ret;
+		ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
+		if (ret)
+			goto unregister_dev;
+
+		ret = media_create_pad_link(&csi->subdev.entity,
+					    TI_CSI2RX_PAD_FIRST_SOURCE + ctx->idx,
+					    &vdev->entity, 0,
+					    MEDIA_LNK_FL_IMMUTABLE |
+					    MEDIA_LNK_FL_ENABLED);
+		if (ret) {
+			video_unregister_device(vdev);
+			goto unregister_dev;
+		}
 	}
 
 	ret = v4l2_device_register_subdev_nodes(&csi->v4l2_dev);
 	if (ret)
-		video_unregister_device(vdev);
+		goto unregister_dev;
 
+	return 0;
+
+unregister_dev:
+	i--;
+	for (; i >= 0; i--) {
+		media_entity_remove_links(&csi->ctx[i].vdev.entity);
+		video_unregister_device(&csi->ctx[i].vdev);
+	}
 	return ret;
 }
 
@@ -496,7 +540,7 @@ static void ti_csi2rx_request_max_ppc(struct ti_csi2rx_dev *csi)
 	struct media_pad *pad;
 	int ret;
 
-	pad = media_entity_remote_source_pad_unique(&csi->vdev.entity);
+	pad = media_entity_remote_source_pad_unique(&csi->subdev.entity);
 	if (IS_ERR(pad))
 		return;
 
@@ -509,22 +553,19 @@ static void ti_csi2rx_request_max_ppc(struct ti_csi2rx_dev *csi)
 	}
 }
 
-static void ti_csi2rx_setup_shim(struct ti_csi2rx_dev *csi)
+static void ti_csi2rx_setup_shim(struct ti_csi2rx_ctx *ctx)
 {
+	struct ti_csi2rx_dev *csi = ctx->csi;
 	const struct ti_csi2rx_fmt *fmt;
 	unsigned int reg;
 
-	fmt = find_format_by_fourcc(csi->v_fmt.fmt.pix.pixelformat);
-
-	/* De-assert the pixel interface reset. */
-	reg = SHIM_CNTL_PIX_RST;
-	writel(reg, csi->shim + SHIM_CNTL);
+	fmt = find_format_by_fourcc(ctx->v_fmt.fmt.pix.pixelformat);
 
 	/* Negotiate pixel count from the source */
 	ti_csi2rx_request_max_ppc(csi);
 
 	reg = SHIM_DMACNTX_EN;
-	reg |= FIELD_PREP(SHIM_DMACNTX_FMT, fmt->csi_dt);
+	reg |= FIELD_PREP(SHIM_DMACNTX_FMT, ctx->dt);
 
 	/*
 	 * The hardware assumes incoming YUV422 8-bit data on MIPI CSI2 bus
@@ -563,18 +604,43 @@ static void ti_csi2rx_setup_shim(struct ti_csi2rx_dev *csi)
 		break;
 	}
 
-	writel(reg, csi->shim + SHIM_DMACNTX);
+	reg |= FIELD_PREP(SHIM_DMACNTX_VC, ctx->vc);
+
+	writel(reg, csi->shim + SHIM_DMACNTX(ctx->idx));
 
 	reg = FIELD_PREP(SHIM_PSI_CFG0_SRC_TAG, 0) |
 	      FIELD_PREP(SHIM_PSI_CFG0_DST_TAG, 0);
-	writel(reg, csi->shim + SHIM_PSI_CFG0);
+	writel(reg, csi->shim + SHIM_PSI_CFG0(ctx->idx));
 }
 
 static void ti_csi2rx_drain_callback(void *param)
 {
-	struct completion *drain_complete = param;
+	struct ti_csi2rx_ctx *ctx = param;
+	struct ti_csi2rx_dma *dma = &ctx->dma;
+	unsigned long flags;
+
+	spin_lock_irqsave(&dma->lock, flags);
+
+	if (dma->state == TI_CSI2RX_DMA_STOPPED) {
+		complete(&ctx->drain_complete);
+		spin_unlock_irqrestore(&dma->lock, flags);
+		return;
+	}
 
-	complete(drain_complete);
+	/*
+	 * If dma->queue is empty, it indicates that no buffer has been
+	 * provided by user space. In this case, initiate a transactions
+	 * to drain the DMA. Since one drain of size DRAIN_BUFFER_SIZE
+	 * will be done here, the subsequent frame will be a
+	 * partial frame, with a size of frame_size - DRAIN_BUFFER_SIZE
+	 */
+	if (list_empty(&dma->queue)) {
+		if (ti_csi2rx_drain_dma(ctx))
+			dev_warn(ctx->csi->dev, "DMA drain failed\n");
+	} else {
+		ti_csi2rx_dma_submit_pending(ctx);
+	}
+	spin_unlock_irqrestore(&dma->lock, flags);
 }
 
 /*
@@ -588,17 +654,15 @@ static void ti_csi2rx_drain_callback(void *param)
  * To prevent that stale data corrupting the subsequent transactions, it is
  * required to issue DMA requests to drain it out.
  */
-static int ti_csi2rx_drain_dma(struct ti_csi2rx_dev *csi)
+static int ti_csi2rx_drain_dma(struct ti_csi2rx_ctx *ctx)
 {
+	struct ti_csi2rx_dev *csi = ctx->csi;
 	struct dma_async_tx_descriptor *desc;
-	struct completion drain_complete;
 	dma_cookie_t cookie;
 	int ret;
 
-	init_completion(&drain_complete);
-
-	desc = dmaengine_prep_slave_single(csi->dma.chan, csi->dma.drain.paddr,
-					   csi->dma.drain.len, DMA_DEV_TO_MEM,
+	desc = dmaengine_prep_slave_single(ctx->dma.chan, csi->drain.paddr,
+					   csi->drain.len, DMA_DEV_TO_MEM,
 					   DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 	if (!desc) {
 		ret = -EIO;
@@ -606,31 +670,46 @@ static int ti_csi2rx_drain_dma(struct ti_csi2rx_dev *csi)
 	}
 
 	desc->callback = ti_csi2rx_drain_callback;
-	desc->callback_param = &drain_complete;
+	desc->callback_param = ctx;
 
 	cookie = dmaengine_submit(desc);
 	ret = dma_submit_error(cookie);
 	if (ret)
 		goto out;
 
-	dma_async_issue_pending(csi->dma.chan);
+	dma_async_issue_pending(ctx->dma.chan);
 
-	if (!wait_for_completion_timeout(&drain_complete,
-					 msecs_to_jiffies(DRAIN_TIMEOUT_MS))) {
-		dmaengine_terminate_sync(csi->dma.chan);
-		dev_dbg(csi->dev, "DMA transfer timed out for drain buffer\n");
-		ret = -ETIMEDOUT;
-		goto out;
-	}
 out:
 	return ret;
 }
 
+static int ti_csi2rx_dma_submit_pending(struct ti_csi2rx_ctx *ctx)
+{
+	struct ti_csi2rx_dma *dma = &ctx->dma;
+	struct ti_csi2rx_buffer *buf;
+	int ret = 0;
+
+	/* If there are more buffers to process then start their transfer. */
+	while (!list_empty(&dma->queue)) {
+		buf = list_entry(dma->queue.next, struct ti_csi2rx_buffer, list);
+		ret = ti_csi2rx_start_dma(ctx, buf);
+		if (ret) {
+			dev_err(ctx->csi->dev,
+				"Failed to queue the next buffer for DMA\n");
+			vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+			list_del(&buf->list);
+		} else {
+			list_move_tail(&buf->list, &dma->submitted);
+		}
+	}
+	return ret;
+}
+
 static void ti_csi2rx_dma_callback(void *param)
 {
 	struct ti_csi2rx_buffer *buf = param;
-	struct ti_csi2rx_dev *csi = buf->csi;
-	struct ti_csi2rx_dma *dma = &csi->dma;
+	struct ti_csi2rx_ctx *ctx = buf->ctx;
+	struct ti_csi2rx_dma *dma = &ctx->dma;
 	unsigned long flags;
 
 	/*
@@ -638,44 +717,43 @@ static void ti_csi2rx_dma_callback(void *param)
 	 * hardware monitor registers.
 	 */
 	buf->vb.vb2_buf.timestamp = ktime_get_ns();
-	buf->vb.sequence = csi->sequence++;
+	buf->vb.sequence = ctx->sequence++;
 
 	spin_lock_irqsave(&dma->lock, flags);
 
 	WARN_ON(!list_is_first(&buf->list, &dma->submitted));
-	vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
-	list_del(&buf->list);
 
-	/* If there are more buffers to process then start their transfer. */
-	while (!list_empty(&dma->queue)) {
-		buf = list_entry(dma->queue.next, struct ti_csi2rx_buffer, list);
-
-		if (ti_csi2rx_start_dma(csi, buf)) {
-			dev_err(csi->dev, "Failed to queue the next buffer for DMA\n");
-			list_del(&buf->list);
-			vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
-		} else {
-			list_move_tail(&buf->list, &dma->submitted);
-		}
+	if (dma->state == TI_CSI2RX_DMA_DRAINING) {
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+		dma->state = TI_CSI2RX_DMA_ACTIVE;
+	} else {
+		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_DONE);
 	}
 
-	if (list_empty(&dma->submitted))
-		dma->state = TI_CSI2RX_DMA_IDLE;
+	list_del(&buf->list);
+
+	ti_csi2rx_dma_submit_pending(ctx);
 
+	if (list_empty(&dma->submitted)) {
+		dma->state = TI_CSI2RX_DMA_DRAINING;
+		if (ti_csi2rx_drain_dma(ctx))
+			dev_warn(ctx->csi->dev,
+				 "DMA drain failed on one of the transactions\n");
+	}
 	spin_unlock_irqrestore(&dma->lock, flags);
 }
 
-static int ti_csi2rx_start_dma(struct ti_csi2rx_dev *csi,
+static int ti_csi2rx_start_dma(struct ti_csi2rx_ctx *ctx,
 			       struct ti_csi2rx_buffer *buf)
 {
 	unsigned long addr;
 	struct dma_async_tx_descriptor *desc;
-	size_t len = csi->v_fmt.fmt.pix.sizeimage;
+	size_t len = ctx->v_fmt.fmt.pix.sizeimage;
 	dma_cookie_t cookie;
 	int ret = 0;
 
 	addr = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0);
-	desc = dmaengine_prep_slave_single(csi->dma.chan, addr, len,
+	desc = dmaengine_prep_slave_single(ctx->dma.chan, addr, len,
 					   DMA_DEV_TO_MEM,
 					   DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 	if (!desc)
@@ -689,23 +767,25 @@ static int ti_csi2rx_start_dma(struct ti_csi2rx_dev *csi,
 	if (ret)
 		return ret;
 
-	dma_async_issue_pending(csi->dma.chan);
+	dma_async_issue_pending(ctx->dma.chan);
 
 	return 0;
 }
 
-static void ti_csi2rx_stop_dma(struct ti_csi2rx_dev *csi)
+static void ti_csi2rx_stop_dma(struct ti_csi2rx_ctx *ctx)
 {
-	struct ti_csi2rx_dma *dma = &csi->dma;
+	struct ti_csi2rx_dma *dma = &ctx->dma;
 	enum ti_csi2rx_dma_state state;
 	unsigned long flags;
 	int ret;
 
 	spin_lock_irqsave(&dma->lock, flags);
-	state = csi->dma.state;
+	state = ctx->dma.state;
 	dma->state = TI_CSI2RX_DMA_STOPPED;
 	spin_unlock_irqrestore(&dma->lock, flags);
 
+	init_completion(&ctx->drain_complete);
+
 	if (state != TI_CSI2RX_DMA_STOPPED) {
 		/*
 		 * Normal DMA termination does not clean up pending data on
@@ -713,30 +793,38 @@ static void ti_csi2rx_stop_dma(struct ti_csi2rx_dev *csi)
 		 * is stopped, as the module-level pixel reset cannot be
 		 * enforced before terminating DMA.
 		 */
-		ret = ti_csi2rx_drain_dma(csi);
-		if (ret && ret != -ETIMEDOUT)
-			dev_warn(csi->dev,
+		ret = ti_csi2rx_drain_dma(ctx);
+		if (ret)
+			dev_warn(ctx->csi->dev,
 				 "Failed to drain DMA. Next frame might be bogus\n");
 	}
 
-	ret = dmaengine_terminate_sync(csi->dma.chan);
+	/* We wait for the drain to complete so that the stream stops
+	 * cleanly, making sure the shared hardware FIFO is cleared of
+	 * data from the current stream. No more data will be coming from
+	 * the source after this.
+	 */
+	wait_for_completion_timeout(&ctx->drain_complete,
+				    msecs_to_jiffies(DRAIN_TIMEOUT_MS));
+
+	ret = dmaengine_terminate_sync(ctx->dma.chan);
 	if (ret)
-		dev_err(csi->dev, "Failed to stop DMA: %d\n", ret);
+		dev_err(ctx->csi->dev, "Failed to stop DMA: %d\n", ret);
 }
 
-static void ti_csi2rx_cleanup_buffers(struct ti_csi2rx_dev *csi,
+static void ti_csi2rx_cleanup_buffers(struct ti_csi2rx_ctx *ctx,
 				      enum vb2_buffer_state state)
 {
-	struct ti_csi2rx_dma *dma = &csi->dma;
+	struct ti_csi2rx_dma *dma = &ctx->dma;
 	struct ti_csi2rx_buffer *buf, *tmp;
 	unsigned long flags;
 
 	spin_lock_irqsave(&dma->lock, flags);
-	list_for_each_entry_safe(buf, tmp, &csi->dma.queue, list) {
+	list_for_each_entry_safe(buf, tmp, &ctx->dma.queue, list) {
 		list_del(&buf->list);
 		vb2_buffer_done(&buf->vb.vb2_buf, state);
 	}
-	list_for_each_entry_safe(buf, tmp, &csi->dma.submitted, list) {
+	list_for_each_entry_safe(buf, tmp, &ctx->dma.submitted, list) {
 		list_del(&buf->list);
 		vb2_buffer_done(&buf->vb.vb2_buf, state);
 	}
@@ -747,8 +835,8 @@ static int ti_csi2rx_queue_setup(struct vb2_queue *q, unsigned int *nbuffers,
 				 unsigned int *nplanes, unsigned int sizes[],
 				 struct device *alloc_devs[])
 {
-	struct ti_csi2rx_dev *csi = vb2_get_drv_priv(q);
-	unsigned int size = csi->v_fmt.fmt.pix.sizeimage;
+	struct ti_csi2rx_ctx *ctx = vb2_get_drv_priv(q);
+	unsigned int size = ctx->v_fmt.fmt.pix.sizeimage;
 
 	if (*nplanes) {
 		if (sizes[0] < size)
@@ -764,11 +852,11 @@ static int ti_csi2rx_queue_setup(struct vb2_queue *q, unsigned int *nbuffers,
 
 static int ti_csi2rx_buffer_prepare(struct vb2_buffer *vb)
 {
-	struct ti_csi2rx_dev *csi = vb2_get_drv_priv(vb->vb2_queue);
-	unsigned long size = csi->v_fmt.fmt.pix.sizeimage;
+	struct ti_csi2rx_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
+	unsigned long size = ctx->v_fmt.fmt.pix.sizeimage;
 
 	if (vb2_plane_size(vb, 0) < size) {
-		dev_err(csi->dev, "Data will not fit into plane\n");
+		dev_err(ctx->csi->dev, "Data will not fit into plane\n");
 		return -EINVAL;
 	}
 
@@ -778,132 +866,161 @@ static int ti_csi2rx_buffer_prepare(struct vb2_buffer *vb)
 
 static void ti_csi2rx_buffer_queue(struct vb2_buffer *vb)
 {
-	struct ti_csi2rx_dev *csi = vb2_get_drv_priv(vb->vb2_queue);
+	struct ti_csi2rx_ctx *ctx = vb2_get_drv_priv(vb->vb2_queue);
 	struct ti_csi2rx_buffer *buf;
-	struct ti_csi2rx_dma *dma = &csi->dma;
-	bool restart_dma = false;
+	struct ti_csi2rx_dma *dma = &ctx->dma;
 	unsigned long flags = 0;
-	int ret;
 
 	buf = container_of(vb, struct ti_csi2rx_buffer, vb.vb2_buf);
-	buf->csi = csi;
+	buf->ctx = ctx;
 
 	spin_lock_irqsave(&dma->lock, flags);
-	/*
-	 * Usually the DMA callback takes care of queueing the pending buffers.
-	 * But if DMA has stalled due to lack of buffers, restart it now.
-	 */
-	if (dma->state == TI_CSI2RX_DMA_IDLE) {
-		/*
-		 * Do not restart DMA with the lock held because
-		 * ti_csi2rx_drain_dma() might block for completion.
-		 * There won't be a race on queueing DMA anyway since the
-		 * callback is not being fired.
-		 */
-		restart_dma = true;
-		dma->state = TI_CSI2RX_DMA_ACTIVE;
-	} else {
-		list_add_tail(&buf->list, &dma->queue);
-	}
+	list_add_tail(&buf->list, &dma->queue);
 	spin_unlock_irqrestore(&dma->lock, flags);
+}
 
-	if (restart_dma) {
-		/*
-		 * Once frames start dropping, some data gets stuck in the DMA
-		 * pipeline somewhere. So the first DMA transfer after frame
-		 * drops gives a partial frame. This is obviously not useful to
-		 * the application and will only confuse it. Issue a DMA
-		 * transaction to drain that up.
-		 */
-		ret = ti_csi2rx_drain_dma(csi);
-		if (ret && ret != -ETIMEDOUT)
-			dev_warn(csi->dev,
-				 "Failed to drain DMA. Next frame might be bogus\n");
+static int ti_csi2rx_get_stream(struct ti_csi2rx_ctx *ctx)
+{
+	struct ti_csi2rx_dev *csi = ctx->csi;
+	struct media_pad *pad;
+	struct v4l2_subdev_state *state;
+	struct v4l2_subdev_route *r;
+
+	/* Get the source pad connected to this ctx */
+	pad = media_entity_remote_source_pad_unique(ctx->pad.entity);
+	if (!pad) {
+		dev_err(csi->dev, "No pad connected to ctx %d\n", ctx->idx);
+		return -ENODEV;
+	}
 
-		spin_lock_irqsave(&dma->lock, flags);
-		ret = ti_csi2rx_start_dma(csi, buf);
-		if (ret) {
-			vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
-			dma->state = TI_CSI2RX_DMA_IDLE;
-			spin_unlock_irqrestore(&dma->lock, flags);
-			dev_err(csi->dev, "Failed to start DMA: %d\n", ret);
-		} else {
-			list_add_tail(&buf->list, &dma->submitted);
-			spin_unlock_irqrestore(&dma->lock, flags);
+	state = v4l2_subdev_get_locked_active_state(&csi->subdev);
+
+	for_each_active_route(&state->routing, r) {
+		if (r->source_pad == pad->index) {
+			ctx->stream = r->sink_stream;
+			return 0;
 		}
 	}
+
+	/* No route found for this ctx */
+	return -ENODEV;
+}
+
+static int ti_csi2rx_get_vc_and_dt(struct ti_csi2rx_ctx *ctx)
+{
+	struct ti_csi2rx_dev *csi = ctx->csi;
+	struct ti_csi2rx_ctx *curr_ctx;
+	struct v4l2_mbus_frame_desc fd;
+	struct media_pad *source_pad;
+	const struct ti_csi2rx_fmt *fmt;
+	int ret;
+	unsigned int i, j;
+
+	/* Get the frame desc from source */
+	source_pad = media_entity_remote_pad_unique(&csi->subdev.entity, MEDIA_PAD_FL_SOURCE);
+	if (IS_ERR(source_pad))
+		return PTR_ERR(source_pad);
+
+	ret = v4l2_subdev_call(csi->source, pad, get_frame_desc, source_pad->index, &fd);
+	if (ret) {
+		if (ret == -ENOIOCTLCMD) {
+			ctx->vc = 0;
+			fmt = find_format_by_fourcc(ctx->v_fmt.fmt.pix.pixelformat);
+			ctx->dt = fmt->csi_dt;
+		}
+		return ret;
+	}
+
+	if (fd.type != V4L2_MBUS_FRAME_DESC_TYPE_CSI2)
+		return -EINVAL;
+
+	for (i = 0; i < csi->num_ctx; i++) {
+		curr_ctx = &csi->ctx[i];
+
+		/* Capture VC 0 by default */
+		curr_ctx->vc = 0;
+
+		ret = ti_csi2rx_get_stream(curr_ctx);
+		if (ret)
+			continue;
+
+		for (j = 0; j < fd.num_entries; j++) {
+			if (curr_ctx->stream == fd.entry[j].stream) {
+				curr_ctx->vc = fd.entry[j].bus.csi2.vc;
+				curr_ctx->dt = fd.entry[j].bus.csi2.dt;
+				break;
+			}
+
+			/* Return error if no matching stream found */
+			if (j == fd.num_entries)
+				return -EINVAL;
+		}
+	}
+
+	return 0;
 }
 
 static int ti_csi2rx_start_streaming(struct vb2_queue *vq, unsigned int count)
 {
-	struct ti_csi2rx_dev *csi = vb2_get_drv_priv(vq);
-	struct ti_csi2rx_dma *dma = &csi->dma;
-	struct ti_csi2rx_buffer *buf;
+	struct ti_csi2rx_ctx *ctx = vb2_get_drv_priv(vq);
+	struct ti_csi2rx_dev *csi = ctx->csi;
+	struct ti_csi2rx_dma *dma = &ctx->dma;
 	unsigned long flags;
-	int ret = 0;
+	int ret;
+
+	ret = pm_runtime_resume_and_get(csi->dev);
+	if (ret)
+		return ret;
 
 	spin_lock_irqsave(&dma->lock, flags);
 	if (list_empty(&dma->queue))
 		ret = -EIO;
 	spin_unlock_irqrestore(&dma->lock, flags);
 	if (ret)
-		return ret;
+		goto err;
 
-	ret = video_device_pipeline_start(&csi->vdev, &csi->pipe);
+	ret = video_device_pipeline_start(&ctx->vdev, &csi->pipe);
 	if (ret)
 		goto err;
 
-	ti_csi2rx_setup_shim(csi);
-
-	csi->sequence = 0;
-
-	spin_lock_irqsave(&dma->lock, flags);
-	buf = list_entry(dma->queue.next, struct ti_csi2rx_buffer, list);
-
-	ret = ti_csi2rx_start_dma(csi, buf);
-	if (ret) {
-		dev_err(csi->dev, "Failed to start DMA: %d\n", ret);
-		spin_unlock_irqrestore(&dma->lock, flags);
-		goto err_pipeline;
-	}
-
-	list_move_tail(&buf->list, &dma->submitted);
-	dma->state = TI_CSI2RX_DMA_ACTIVE;
-	spin_unlock_irqrestore(&dma->lock, flags);
-
-	ret = v4l2_subdev_call(csi->source, video, s_stream, 1);
+	/* Start stream 0, we don't allow multiple streams on the source pad */
+	ret = v4l2_subdev_enable_streams(&csi->subdev,
+					 TI_CSI2RX_PAD_FIRST_SOURCE + ctx->idx,
+					 BIT_U64(0));
 	if (ret)
 		goto err_dma;
 
 	return 0;
 
 err_dma:
-	ti_csi2rx_stop_dma(csi);
-err_pipeline:
-	video_device_pipeline_stop(&csi->vdev);
+	ti_csi2rx_stop_dma(ctx);
+	video_device_pipeline_stop(&ctx->vdev);
 	writel(0, csi->shim + SHIM_CNTL);
-	writel(0, csi->shim + SHIM_DMACNTX);
+	writel(0, csi->shim + SHIM_DMACNTX(ctx->idx));
 err:
-	ti_csi2rx_cleanup_buffers(csi, VB2_BUF_STATE_QUEUED);
+	ti_csi2rx_cleanup_buffers(ctx, VB2_BUF_STATE_QUEUED);
+	pm_runtime_put(csi->dev);
+
 	return ret;
 }
 
 static void ti_csi2rx_stop_streaming(struct vb2_queue *vq)
 {
-	struct ti_csi2rx_dev *csi = vb2_get_drv_priv(vq);
+	struct ti_csi2rx_ctx *ctx = vb2_get_drv_priv(vq);
+	struct ti_csi2rx_dev *csi = ctx->csi;
 	int ret;
 
-	video_device_pipeline_stop(&csi->vdev);
+	video_device_pipeline_stop(&ctx->vdev);
 
-	writel(0, csi->shim + SHIM_CNTL);
-	writel(0, csi->shim + SHIM_DMACNTX);
-
-	ret = v4l2_subdev_call(csi->source, video, s_stream, 0);
+	ret = v4l2_subdev_disable_streams(&csi->subdev,
+					  TI_CSI2RX_PAD_FIRST_SOURCE + ctx->idx,
+					  BIT_U64(0));
 	if (ret)
 		dev_err(csi->dev, "Failed to stop subdev stream\n");
 
-	ti_csi2rx_stop_dma(csi);
-	ti_csi2rx_cleanup_buffers(csi, VB2_BUF_STATE_ERROR);
+	ti_csi2rx_stop_dma(ctx);
+	ti_csi2rx_cleanup_buffers(ctx, VB2_BUF_STATE_ERROR);
+	pm_runtime_put(csi->dev);
 }
 
 static const struct vb2_ops csi_vb2_qops = {
@@ -914,20 +1031,265 @@ static const struct vb2_ops csi_vb2_qops = {
 	.stop_streaming = ti_csi2rx_stop_streaming,
 };
 
-static int ti_csi2rx_init_vb2q(struct ti_csi2rx_dev *csi)
+static int ti_csi2rx_enum_mbus_code(struct v4l2_subdev *subdev,
+				    struct v4l2_subdev_state *state,
+				    struct v4l2_subdev_mbus_code_enum *code_enum)
+{
+	if (code_enum->index >= ARRAY_SIZE(ti_csi2rx_formats))
+		return -EINVAL;
+
+	code_enum->code = ti_csi2rx_formats[code_enum->index].code;
+
+	return 0;
+}
+
+static int ti_csi2rx_sd_set_fmt(struct v4l2_subdev *sd,
+				struct v4l2_subdev_state *state,
+				struct v4l2_subdev_format *format)
+{
+	struct v4l2_mbus_framefmt *fmt;
+
+	/* No transcoding, don't allow setting source fmt */
+	if (format->pad > TI_CSI2RX_PAD_SINK)
+		return v4l2_subdev_get_fmt(sd, state, format);
+
+	if (!find_format_by_code(format->format.code))
+		format->format.code = ti_csi2rx_formats[0].code;
+
+	format->format.field = V4L2_FIELD_NONE;
+
+	fmt = v4l2_subdev_state_get_format(state, format->pad, format->stream);
+	*fmt = format->format;
+
+	fmt = v4l2_subdev_state_get_opposite_stream_format(state, format->pad,
+							   format->stream);
+	if (!fmt)
+		return -EINVAL;
+
+	*fmt = format->format;
+
+	return 0;
+}
+
+static int _ti_csi2rx_sd_set_routing(struct v4l2_subdev *sd,
+				     struct v4l2_subdev_state *state,
+				     struct v4l2_subdev_krouting *routing)
 {
-	struct vb2_queue *q = &csi->vidq;
+	int ret;
+
+	static const struct v4l2_mbus_framefmt format = {
+		.width = 640,
+		.height = 480,
+		.code = MEDIA_BUS_FMT_UYVY8_1X16,
+		.field = V4L2_FIELD_NONE,
+		.colorspace = V4L2_COLORSPACE_SRGB,
+		.ycbcr_enc = V4L2_YCBCR_ENC_601,
+		.quantization = V4L2_QUANTIZATION_LIM_RANGE,
+		.xfer_func = V4L2_XFER_FUNC_SRGB,
+	};
+
+	ret = v4l2_subdev_routing_validate(sd, routing,
+					   V4L2_SUBDEV_ROUTING_ONLY_1_TO_1 |
+					   V4L2_SUBDEV_ROUTING_NO_SOURCE_MULTIPLEXING);
+
+	if (ret)
+		return ret;
+
+	/* Only stream ID 0 allowed on source pads */
+	for (unsigned int i = 0; i < routing->num_routes; ++i) {
+		const struct v4l2_subdev_route *route = &routing->routes[i];
+
+		if (route->source_stream != 0)
+			return -EINVAL;
+	}
+
+	ret = v4l2_subdev_set_routing_with_fmt(sd, state, routing, &format);
+
+	return ret;
+}
+
+static int ti_csi2rx_sd_set_routing(struct v4l2_subdev *sd,
+				    struct v4l2_subdev_state *state,
+				    enum v4l2_subdev_format_whence which,
+				    struct v4l2_subdev_krouting *routing)
+{
+	struct ti_csi2rx_dev *csi = to_csi2rx_dev(sd);
+
+	if (csi->enable_count > 0)
+		return -EBUSY;
+
+	return _ti_csi2rx_sd_set_routing(sd, state, routing);
+}
+
+static int ti_csi2rx_sd_init_state(struct v4l2_subdev *sd,
+				   struct v4l2_subdev_state *state)
+{
+	struct v4l2_subdev_route routes[] = { {
+		.sink_pad = 0,
+		.sink_stream = 0,
+		.source_pad = TI_CSI2RX_PAD_FIRST_SOURCE,
+		.source_stream = 0,
+		.flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE,
+	} };
+
+	struct v4l2_subdev_krouting routing = {
+		.num_routes = 1,
+		.routes = routes,
+	};
+
+	/* Initialize routing to single route to the fist source pad */
+	return _ti_csi2rx_sd_set_routing(sd, state, &routing);
+}
+
+static int ti_csi2rx_sd_enable_streams(struct v4l2_subdev *sd,
+				       struct v4l2_subdev_state *state,
+				       u32 pad, u64 streams_mask)
+{
+	struct ti_csi2rx_dev *csi = to_csi2rx_dev(sd);
+	struct ti_csi2rx_ctx *ctx = &csi->ctx[pad - TI_CSI2RX_PAD_FIRST_SOURCE];
+	struct ti_csi2rx_dma *dma = &ctx->dma;
+	struct media_pad *remote_pad;
+	unsigned long flags;
+	u64 sink_streams;
+	int ret = 0;
+	unsigned int reg;
+
+	ret = ti_csi2rx_get_stream(ctx);
+	if (ret)
+		return ret;
+
+	/* Get the VC and DT for all enabled ctx on first stream start */
+	if (!csi->enable_count) {
+		ret = ti_csi2rx_get_vc_and_dt(ctx);
+		if (ret < 0 && ret != -ENOIOCTLCMD)
+			return ret;
+
+		/* De-assert the pixel interface reset. */
+		reg = SHIM_CNTL_PIX_RST;
+		writel(reg, csi->shim + SHIM_CNTL);
+	}
+
+	ti_csi2rx_setup_shim(ctx);
+	ctx->sequence = 0;
+
+	spin_lock_irqsave(&dma->lock, flags);
+
+	ret = ti_csi2rx_dma_submit_pending(ctx);
+	if (ret) {
+		spin_unlock_irqrestore(&dma->lock, flags);
+		return ret;
+	}
+
+	dma->state = TI_CSI2RX_DMA_ACTIVE;
+	spin_unlock_irqrestore(&dma->lock, flags);
+
+	remote_pad = media_entity_remote_source_pad_unique(&csi->subdev.entity);
+	if (!remote_pad)
+		return -ENODEV;
+	sink_streams = v4l2_subdev_state_xlate_streams(state, pad,
+						       TI_CSI2RX_PAD_SINK,
+						       &streams_mask);
+
+	ret = v4l2_subdev_enable_streams(csi->source, remote_pad->index,
+					 sink_streams);
+	if (ret)
+		return ret;
+
+	csi->enable_count++;
+
+	return 0;
+}
+
+static int ti_csi2rx_sd_disable_streams(struct v4l2_subdev *sd,
+					struct v4l2_subdev_state *state,
+					u32 pad, u64 streams_mask)
+{
+	struct ti_csi2rx_dev *csi = to_csi2rx_dev(sd);
+	struct ti_csi2rx_ctx *ctx = &csi->ctx[pad - TI_CSI2RX_PAD_FIRST_SOURCE];
+	struct media_pad *remote_pad;
+	u64 sink_streams;
+	int ret = 0;
+
+	WARN_ON(csi->enable_count == 0);
+
+	writel(0, csi->shim + SHIM_DMACNTX(ctx->idx));
+
+	/* assert pixel reset to prevent stale data */
+	if (csi->enable_count == 1)
+		writel(0, csi->shim + SHIM_CNTL);
+
+	remote_pad = media_entity_remote_source_pad_unique(&csi->subdev.entity);
+	if (!remote_pad)
+		return -ENODEV;
+	sink_streams = v4l2_subdev_state_xlate_streams(state, pad,
+						       TI_CSI2RX_PAD_SINK,
+						       &streams_mask);
+
+	ret = v4l2_subdev_disable_streams(csi->source, remote_pad->index,
+					  sink_streams);
+	if (!ret)
+		--csi->enable_count;
+
+	return 0;
+}
+
+static const struct v4l2_subdev_pad_ops ti_csi2rx_subdev_pad_ops = {
+	.enum_mbus_code	= ti_csi2rx_enum_mbus_code,
+	.set_routing = ti_csi2rx_sd_set_routing,
+	.get_fmt = v4l2_subdev_get_fmt,
+	.set_fmt = ti_csi2rx_sd_set_fmt,
+	.enable_streams = ti_csi2rx_sd_enable_streams,
+	.disable_streams = ti_csi2rx_sd_disable_streams,
+};
+
+static const struct v4l2_subdev_ops ti_csi2rx_subdev_ops = {
+	.pad = &ti_csi2rx_subdev_pad_ops,
+};
+
+static const struct v4l2_subdev_internal_ops ti_csi2rx_internal_ops = {
+	.init_state = ti_csi2rx_sd_init_state,
+};
+
+static void ti_csi2rx_cleanup_v4l2(struct ti_csi2rx_dev *csi)
+{
+	v4l2_subdev_cleanup(&csi->subdev);
+	media_device_unregister(&csi->mdev);
+	v4l2_device_unregister(&csi->v4l2_dev);
+	media_device_cleanup(&csi->mdev);
+}
+
+static void ti_csi2rx_cleanup_notifier(struct ti_csi2rx_dev *csi)
+{
+	v4l2_async_nf_unregister(&csi->notifier);
+	v4l2_async_nf_cleanup(&csi->notifier);
+}
+
+static void ti_csi2rx_cleanup_ctx(struct ti_csi2rx_ctx *ctx)
+{
+	if (!pm_runtime_status_suspended(ctx->csi->dev))
+		dma_release_channel(ctx->dma.chan);
+
+	vb2_queue_release(&ctx->vidq);
+
+	video_unregister_device(&ctx->vdev);
+
+	mutex_destroy(&ctx->mutex);
+}
+
+static int ti_csi2rx_init_vb2q(struct ti_csi2rx_ctx *ctx)
+{
+	struct vb2_queue *q = &ctx->vidq;
 	int ret;
 
 	q->type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
 	q->io_modes = VB2_MMAP | VB2_DMABUF;
-	q->drv_priv = csi;
+	q->drv_priv = ctx;
 	q->buf_struct_size = sizeof(struct ti_csi2rx_buffer);
 	q->ops = &csi_vb2_qops;
 	q->mem_ops = &vb2_dma_contig_memops;
 	q->timestamp_flags = V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
-	q->dev = dmaengine_get_dma_device(csi->dma.chan);
-	q->lock = &csi->mutex;
+	q->dev = dmaengine_get_dma_device(ctx->dma.chan);
+	q->lock = &ctx->mutex;
 	q->min_queued_buffers = 1;
 	q->allow_cache_hints = 1;
 
@@ -935,7 +1297,7 @@ static int ti_csi2rx_init_vb2q(struct ti_csi2rx_dev *csi)
 	if (ret)
 		return ret;
 
-	csi->vdev.queue = q;
+	ctx->vdev.queue = q;
 
 	return 0;
 }
@@ -944,50 +1306,55 @@ static int ti_csi2rx_link_validate(struct media_link *link)
 {
 	struct media_entity *entity = link->sink->entity;
 	struct video_device *vdev = media_entity_to_video_device(entity);
-	struct ti_csi2rx_dev *csi = container_of(vdev, struct ti_csi2rx_dev, vdev);
-	struct v4l2_pix_format *csi_fmt = &csi->v_fmt.fmt.pix;
-	struct v4l2_subdev_format source_fmt = {
-		.which	= V4L2_SUBDEV_FORMAT_ACTIVE,
-		.pad	= link->source->index,
-	};
+	struct ti_csi2rx_ctx *ctx = container_of(vdev, struct ti_csi2rx_ctx, vdev);
+	struct ti_csi2rx_dev *csi = ctx->csi;
+	struct v4l2_pix_format *csi_fmt = &ctx->v_fmt.fmt.pix;
+	struct v4l2_mbus_framefmt *format;
+	struct v4l2_subdev_state *state;
 	const struct ti_csi2rx_fmt *ti_fmt;
-	int ret;
 
-	ret = v4l2_subdev_call_state_active(csi->source, pad,
-					    get_fmt, &source_fmt);
-	if (ret)
-		return ret;
+	state = v4l2_subdev_lock_and_get_active_state(&csi->subdev);
+	format = v4l2_subdev_state_get_format(state, link->source->index, 0);
+	v4l2_subdev_unlock_state(state);
 
-	if (source_fmt.format.width != csi_fmt->width) {
+	if (!format) {
+		dev_err(csi->dev,
+			"No format present on \"%s\":%u:0\n",
+			link->source->entity->name, link->source->index);
+		return 0;
+	}
+
+	if (format->width != csi_fmt->width) {
 		dev_dbg(csi->dev, "Width does not match (source %u, sink %u)\n",
-			source_fmt.format.width, csi_fmt->width);
+			format->width, csi_fmt->width);
 		return -EPIPE;
 	}
 
-	if (source_fmt.format.height != csi_fmt->height) {
+	if (format->height != csi_fmt->height) {
 		dev_dbg(csi->dev, "Height does not match (source %u, sink %u)\n",
-			source_fmt.format.height, csi_fmt->height);
+			format->height, csi_fmt->height);
 		return -EPIPE;
 	}
 
-	if (source_fmt.format.field != csi_fmt->field &&
+	if (format->field != csi_fmt->field &&
 	    csi_fmt->field != V4L2_FIELD_NONE) {
 		dev_dbg(csi->dev, "Field does not match (source %u, sink %u)\n",
-			source_fmt.format.field, csi_fmt->field);
+			format->field, csi_fmt->field);
 		return -EPIPE;
 	}
 
-	ti_fmt = find_format_by_code(source_fmt.format.code);
+	ti_fmt = find_format_by_code(format->code);
 	if (!ti_fmt) {
 		dev_dbg(csi->dev, "Media bus format 0x%x not supported\n",
-			source_fmt.format.code);
+			format->code);
 		return -EPIPE;
 	}
 
 	if (ti_fmt->fourcc != csi_fmt->pixelformat) {
 		dev_dbg(csi->dev,
-			"Cannot transform source fmt 0x%x to sink fmt 0x%x\n",
-			ti_fmt->fourcc, csi_fmt->pixelformat);
+			"Cannot transform \"%s\":%u format %p4cc to %p4cc\n",
+			link->source->entity->name, link->source->index,
+			&ti_fmt->fourcc, &csi_fmt->pixelformat);
 		return -EPIPE;
 	}
 
@@ -998,47 +1365,107 @@ static const struct media_entity_operations ti_csi2rx_video_entity_ops = {
 	.link_validate = ti_csi2rx_link_validate,
 };
 
-static int ti_csi2rx_init_dma(struct ti_csi2rx_dev *csi)
+static const struct media_entity_operations ti_csi2rx_subdev_entity_ops = {
+	.link_validate = v4l2_subdev_link_validate,
+	.has_pad_interdep = v4l2_subdev_has_pad_interdep,
+};
+
+static int ti_csi2rx_init_dma(struct ti_csi2rx_ctx *ctx)
 {
 	struct dma_slave_config cfg = {
 		.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES,
 	};
+	char name[5];
 	int ret;
 
-	INIT_LIST_HEAD(&csi->dma.queue);
-	INIT_LIST_HEAD(&csi->dma.submitted);
-	spin_lock_init(&csi->dma.lock);
-
-	csi->dma.state = TI_CSI2RX_DMA_STOPPED;
-
-	csi->dma.chan = dma_request_chan(csi->dev, "rx0");
-	if (IS_ERR(csi->dma.chan))
-		return PTR_ERR(csi->dma.chan);
+	snprintf(name, sizeof(name), "rx%u", ctx->idx);
+	ctx->dma.chan = dma_request_chan(ctx->csi->dev, name);
+	if (IS_ERR(ctx->dma.chan))
+		return PTR_ERR(ctx->dma.chan);
 
-	ret = dmaengine_slave_config(csi->dma.chan, &cfg);
+	ret = dmaengine_slave_config(ctx->dma.chan, &cfg);
 	if (ret) {
-		dma_release_channel(csi->dma.chan);
+		dma_release_channel(ctx->dma.chan);
 		return ret;
 	}
 
-	csi->dma.drain.len = DRAIN_BUFFER_SIZE;
-	csi->dma.drain.vaddr = dma_alloc_coherent(csi->dev, csi->dma.drain.len,
-						  &csi->dma.drain.paddr,
-						  GFP_KERNEL);
-	if (!csi->dma.drain.vaddr)
-		return -ENOMEM;
-
 	return 0;
 }
 
 static int ti_csi2rx_v4l2_init(struct ti_csi2rx_dev *csi)
 {
 	struct media_device *mdev = &csi->mdev;
-	struct video_device *vdev = &csi->vdev;
+	struct v4l2_subdev *sd = &csi->subdev;
+	int ret;
+
+	mdev->dev = csi->dev;
+	mdev->hw_revision = 1;
+	strscpy(mdev->model, "TI-CSI2RX", sizeof(mdev->model));
+
+	media_device_init(mdev);
+
+	csi->v4l2_dev.mdev = mdev;
+
+	ret = v4l2_device_register(csi->dev, &csi->v4l2_dev);
+	if (ret)
+		goto cleanup_media;
+
+	ret = media_device_register(mdev);
+	if (ret)
+		goto unregister_v4l2;
+
+	v4l2_subdev_init(sd, &ti_csi2rx_subdev_ops);
+	sd->internal_ops = &ti_csi2rx_internal_ops;
+	sd->entity.function = MEDIA_ENT_F_VID_IF_BRIDGE;
+	sd->flags = V4L2_SUBDEV_FL_HAS_DEVNODE | V4L2_SUBDEV_FL_STREAMS;
+	strscpy(sd->name, dev_name(csi->dev), sizeof(sd->name));
+	sd->dev = csi->dev;
+	sd->entity.ops = &ti_csi2rx_subdev_entity_ops;
+
+	csi->pads[TI_CSI2RX_PAD_SINK].flags = MEDIA_PAD_FL_SINK;
+
+	for (unsigned int i = TI_CSI2RX_PAD_FIRST_SOURCE;
+	     i < TI_CSI2RX_PAD_FIRST_SOURCE + csi->num_ctx; i++)
+		csi->pads[i].flags = MEDIA_PAD_FL_SOURCE;
+
+	ret = media_entity_pads_init(&sd->entity,
+				     TI_CSI2RX_PAD_FIRST_SOURCE + csi->num_ctx,
+				     csi->pads);
+	if (ret)
+		goto unregister_media;
+
+	ret = v4l2_subdev_init_finalize(sd);
+	if (ret)
+		goto unregister_media;
+
+	ret = v4l2_device_register_subdev(&csi->v4l2_dev, sd);
+	if (ret)
+		goto cleanup_subdev;
+
+	return 0;
+
+cleanup_subdev:
+	v4l2_subdev_cleanup(sd);
+unregister_media:
+	media_device_unregister(mdev);
+unregister_v4l2:
+	v4l2_device_unregister(&csi->v4l2_dev);
+cleanup_media:
+	media_device_cleanup(mdev);
+
+	return ret;
+}
+
+static int ti_csi2rx_init_ctx(struct ti_csi2rx_ctx *ctx)
+{
+	struct ti_csi2rx_dev *csi = ctx->csi;
+	struct video_device *vdev = &ctx->vdev;
 	const struct ti_csi2rx_fmt *fmt;
-	struct v4l2_pix_format *pix_fmt = &csi->v_fmt.fmt.pix;
+	struct v4l2_pix_format *pix_fmt = &ctx->v_fmt.fmt.pix;
 	int ret;
 
+	mutex_init(&ctx->mutex);
+
 	fmt = find_format_by_fourcc(V4L2_PIX_FMT_UYVY);
 	if (!fmt)
 		return -EINVAL;
@@ -1047,19 +1474,20 @@ static int ti_csi2rx_v4l2_init(struct ti_csi2rx_dev *csi)
 	pix_fmt->height = 480;
 	pix_fmt->field = V4L2_FIELD_NONE;
 	pix_fmt->colorspace = V4L2_COLORSPACE_SRGB;
-	pix_fmt->ycbcr_enc = V4L2_YCBCR_ENC_601;
-	pix_fmt->quantization = V4L2_QUANTIZATION_LIM_RANGE;
-	pix_fmt->xfer_func = V4L2_XFER_FUNC_SRGB;
-
-	ti_csi2rx_fill_fmt(fmt, &csi->v_fmt);
+	pix_fmt->ycbcr_enc = V4L2_YCBCR_ENC_601,
+	pix_fmt->quantization = V4L2_QUANTIZATION_LIM_RANGE,
+	pix_fmt->xfer_func = V4L2_XFER_FUNC_SRGB,
 
-	mdev->dev = csi->dev;
-	mdev->hw_revision = 1;
-	strscpy(mdev->model, "TI-CSI2RX", sizeof(mdev->model));
+	ti_csi2rx_fill_fmt(fmt, &ctx->v_fmt);
 
-	media_device_init(mdev);
+	ctx->pad.flags = MEDIA_PAD_FL_SINK;
+	vdev->entity.ops = &ti_csi2rx_video_entity_ops;
+	ret = media_entity_pads_init(&ctx->vdev.entity, 1, &ctx->pad);
+	if (ret)
+		return ret;
 
-	strscpy(vdev->name, TI_CSI2RX_MODULE_NAME, sizeof(vdev->name));
+	snprintf(vdev->name, sizeof(vdev->name), "%s context %u",
+		 dev_name(csi->dev), ctx->idx);
 	vdev->v4l2_dev = &csi->v4l2_dev;
 	vdev->vfl_dir = VFL_DIR_RX;
 	vdev->fops = &csi_fops;
@@ -1067,61 +1495,184 @@ static int ti_csi2rx_v4l2_init(struct ti_csi2rx_dev *csi)
 	vdev->release = video_device_release_empty;
 	vdev->device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING |
 			    V4L2_CAP_IO_MC;
-	vdev->lock = &csi->mutex;
-	video_set_drvdata(vdev, csi);
+	vdev->lock = &ctx->mutex;
+	video_set_drvdata(vdev, ctx);
 
-	csi->pad.flags = MEDIA_PAD_FL_SINK;
-	vdev->entity.ops = &ti_csi2rx_video_entity_ops;
-	ret = media_entity_pads_init(&csi->vdev.entity, 1, &csi->pad);
+	INIT_LIST_HEAD(&ctx->dma.queue);
+	INIT_LIST_HEAD(&ctx->dma.submitted);
+	spin_lock_init(&ctx->dma.lock);
+	ctx->dma.state = TI_CSI2RX_DMA_STOPPED;
+
+	ret = ti_csi2rx_init_dma(ctx);
 	if (ret)
 		return ret;
 
-	csi->v4l2_dev.mdev = mdev;
-
-	ret = v4l2_device_register(csi->dev, &csi->v4l2_dev);
+	ret = ti_csi2rx_init_vb2q(ctx);
 	if (ret)
-		return ret;
+		goto cleanup_dma;
 
-	ret = media_device_register(mdev);
-	if (ret) {
-		v4l2_device_unregister(&csi->v4l2_dev);
-		media_device_cleanup(mdev);
-		return ret;
-	}
+	return 0;
+
+cleanup_dma:
+	dma_release_channel(ctx->dma.chan);
+	return ret;
+}
+
+static int ti_csi2rx_runtime_suspend(struct device *dev)
+{
+	struct ti_csi2rx_dev *csi = dev_get_drvdata(dev);
+
+	if (csi->enable_count != 0)
+		return -EBUSY;
+
+	for (unsigned int i = 0; i < csi->num_ctx; i++)
+		dma_release_channel(csi->ctx[i].dma.chan);
 
 	return 0;
 }
 
-static void ti_csi2rx_cleanup_dma(struct ti_csi2rx_dev *csi)
+static int ti_csi2rx_runtime_resume(struct device *dev)
 {
-	dma_free_coherent(csi->dev, csi->dma.drain.len,
-			  csi->dma.drain.vaddr, csi->dma.drain.paddr);
-	csi->dma.drain.vaddr = NULL;
-	dma_release_channel(csi->dma.chan);
+	struct ti_csi2rx_dev *csi = dev_get_drvdata(dev);
+	int ret;
+
+	for (unsigned int i = 0; i < csi->num_ctx; i++) {
+		ret = ti_csi2rx_init_dma(&csi->ctx[i]);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
 }
 
-static void ti_csi2rx_cleanup_v4l2(struct ti_csi2rx_dev *csi)
+static int ti_csi2rx_suspend(struct device *dev)
 {
-	media_device_unregister(&csi->mdev);
-	v4l2_device_unregister(&csi->v4l2_dev);
-	media_device_cleanup(&csi->mdev);
+	struct ti_csi2rx_dev *csi = dev_get_drvdata(dev);
+	enum ti_csi2rx_dma_state state;
+	struct ti_csi2rx_ctx *ctx;
+	struct ti_csi2rx_dma *dma;
+	unsigned long flags = 0;
+	int i, ret = 0;
+
+	/* If device was not in use we can simply suspend */
+	if (pm_runtime_status_suspended(dev))
+		return 0;
+
+	/*
+	 * If device is running, assert the pixel reset to cleanly stop any
+	 * on-going streams before we suspend.
+	 */
+	writel(0, csi->shim + SHIM_CNTL);
+
+	for (i = 0; i < csi->num_ctx; i++) {
+		ctx = &csi->ctx[i];
+		dma = &ctx->dma;
+
+		spin_lock_irqsave(&dma->lock, flags);
+		state = dma->state;
+		spin_unlock_irqrestore(&dma->lock, flags);
+
+		if (state != TI_CSI2RX_DMA_STOPPED) {
+			/* Disable source */
+			ret = v4l2_subdev_disable_streams(&csi->subdev,
+							  TI_CSI2RX_PAD_FIRST_SOURCE + ctx->idx,
+							  BIT(0));
+			if (ret)
+				dev_err(csi->dev, "Failed to stop subdev stream\n");
+		}
+
+		/* Stop any on-going streams */
+		writel(0, csi->shim + SHIM_DMACNTX(ctx->idx));
+
+		/* Drain DMA */
+		ti_csi2rx_drain_dma(ctx);
+
+		/* Terminate DMA */
+		ret = dmaengine_terminate_sync(ctx->dma.chan);
+		if (ret)
+			dev_err(csi->dev, "Failed to stop DMA\n");
+	}
+
+	return ret;
 }
 
-static void ti_csi2rx_cleanup_subdev(struct ti_csi2rx_dev *csi)
+static int ti_csi2rx_resume(struct device *dev)
 {
-	v4l2_async_nf_unregister(&csi->notifier);
-	v4l2_async_nf_cleanup(&csi->notifier);
+	struct ti_csi2rx_dev *csi = dev_get_drvdata(dev);
+	struct ti_csi2rx_ctx *ctx;
+	struct ti_csi2rx_dma *dma;
+	struct ti_csi2rx_buffer *buf;
+	unsigned long flags = 0;
+	unsigned int reg;
+	int i, ret = 0;
+
+	/* If device was not in use, we can simply wakeup */
+	if (pm_runtime_status_suspended(dev))
+		return 0;
+
+	/* If device was in use before, restore all the running streams */
+	reg = SHIM_CNTL_PIX_RST;
+	writel(reg, csi->shim + SHIM_CNTL);
+
+	for (i = 0; i < csi->num_ctx; i++) {
+		ctx = &csi->ctx[i];
+		dma = &ctx->dma;
+		spin_lock_irqsave(&dma->lock, flags);
+		if (dma->state != TI_CSI2RX_DMA_STOPPED) {
+			/* Re-submit all previously submitted buffers to DMA */
+			list_for_each_entry(buf, &ctx->dma.submitted, list) {
+				ti_csi2rx_start_dma(ctx, buf);
+			}
+			spin_unlock_irqrestore(&dma->lock, flags);
+
+			/* Restore stream config */
+			ti_csi2rx_setup_shim(ctx);
+
+			ret = v4l2_subdev_enable_streams(&csi->subdev,
+							 TI_CSI2RX_PAD_FIRST_SOURCE + ctx->idx,
+							 BIT(0));
+			if (ret)
+				dev_err(ctx->csi->dev, "Failed to start subdev\n");
+		} else {
+			spin_unlock_irqrestore(&dma->lock, flags);
+		}
+	}
+
+	return ret;
 }
 
-static void ti_csi2rx_cleanup_vb2q(struct ti_csi2rx_dev *csi)
+static int ti_csi2rx_pm_notifier(struct notifier_block *nb,
+				 unsigned long action, void *data)
 {
-	vb2_queue_release(&csi->vidq);
+	struct ti_csi2rx_dev *csi =
+		container_of(nb, struct ti_csi2rx_dev, pm_notifier);
+
+	switch (action) {
+	case PM_HIBERNATION_PREPARE:
+	case PM_SUSPEND_PREPARE:
+	case PM_RESTORE_PREPARE:
+		ti_csi2rx_suspend(csi->dev);
+		break;
+	case PM_POST_SUSPEND:
+	case PM_POST_HIBERNATION:
+	case PM_POST_RESTORE:
+		ti_csi2rx_resume(csi->dev);
+		break;
+	}
+
+	return NOTIFY_DONE;
 }
 
+static const struct dev_pm_ops ti_csi2rx_pm_ops = {
+	RUNTIME_PM_OPS(ti_csi2rx_runtime_suspend, ti_csi2rx_runtime_resume,
+		       NULL)
+};
+
 static int ti_csi2rx_probe(struct platform_device *pdev)
 {
+	struct device_node *np = pdev->dev.of_node;
 	struct ti_csi2rx_dev *csi;
-	int ret;
+	int ret = 0, i, count;
 
 	csi = devm_kzalloc(&pdev->dev, sizeof(*csi), GFP_KERNEL);
 	if (!csi)
@@ -1130,62 +1681,108 @@ static int ti_csi2rx_probe(struct platform_device *pdev)
 	csi->dev = &pdev->dev;
 	platform_set_drvdata(pdev, csi);
 
-	mutex_init(&csi->mutex);
 	csi->shim = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(csi->shim)) {
 		ret = PTR_ERR(csi->shim);
-		goto err_mutex;
+		return ret;
 	}
 
-	ret = ti_csi2rx_init_dma(csi);
-	if (ret)
-		goto err_mutex;
+	csi->drain.len = DRAIN_BUFFER_SIZE;
+	csi->drain.vaddr = dma_alloc_coherent(csi->dev, csi->drain.len,
+					      &csi->drain.paddr,
+					      GFP_KERNEL);
+	if (!csi->drain.vaddr)
+		return -ENOMEM;
+
+	/* Only use as many contexts as the number of DMA channels allocated. */
+	count = of_property_count_strings(np, "dma-names");
+	if (count < 0) {
+		dev_err(csi->dev, "Failed to get DMA channel count: %d\n", count);
+		ret = count;
+		goto err_dma_chan;
+	}
+
+	csi->num_ctx = count;
+	if (csi->num_ctx > TI_CSI2RX_MAX_CTX) {
+		dev_err(csi->dev,
+			"%u DMA channels passed. Maximum is %u.\n",
+			csi->num_ctx, TI_CSI2RX_MAX_CTX);
+		ret = -EINVAL;
+		goto err_dma_chan;
+	}
 
 	ret = ti_csi2rx_v4l2_init(csi);
 	if (ret)
-		goto err_dma;
+		goto err_dma_chan;
+
+	for (i = 0; i < csi->num_ctx; i++) {
+		csi->ctx[i].idx = i;
+		csi->ctx[i].csi = csi;
+		ret = ti_csi2rx_init_ctx(&csi->ctx[i]);
+		if (ret)
+			goto err_ctx;
+	}
 
-	ret = ti_csi2rx_init_vb2q(csi);
-	if (ret)
-		goto err_v4l2;
+	pm_runtime_set_active(csi->dev);
+	pm_runtime_enable(csi->dev);
 
 	ret = ti_csi2rx_notifier_register(csi);
 	if (ret)
-		goto err_vb2q;
+		goto err_ctx;
 
 	ret = devm_of_platform_populate(csi->dev);
 	if (ret) {
 		dev_err(csi->dev, "Failed to create children: %d\n", ret);
-		goto err_subdev;
+		goto err_notifier;
+	}
+
+	/*
+	 * Use PM notifier instead of .suspend/.resume callbacks because the
+	 * ordering of callbacks among camera pipeline devices (sensor, serdes,
+	 * CSI bridge) cannot be enforced even with device links. The notifier
+	 * is called when the system is fully functional, ensuring all
+	 * dependencies are available when stopping/starting streams.
+	 */
+	csi->pm_notifier.notifier_call = ti_csi2rx_pm_notifier;
+	ret = register_pm_notifier(&csi->pm_notifier);
+	if (ret) {
+		dev_err(csi->dev, "Failed to create PM notifier: %d\n", ret);
+		goto err_notifier;
 	}
 
 	return 0;
 
-err_subdev:
-	ti_csi2rx_cleanup_subdev(csi);
-err_vb2q:
-	ti_csi2rx_cleanup_vb2q(csi);
-err_v4l2:
+err_notifier:
+	ti_csi2rx_cleanup_notifier(csi);
+err_ctx:
+	i--;
+	for (; i >= 0; i--)
+		ti_csi2rx_cleanup_ctx(&csi->ctx[i]);
 	ti_csi2rx_cleanup_v4l2(csi);
-err_dma:
-	ti_csi2rx_cleanup_dma(csi);
-err_mutex:
-	mutex_destroy(&csi->mutex);
+err_dma_chan:
+	dma_free_coherent(csi->dev, csi->drain.len, csi->drain.vaddr,
+			  csi->drain.paddr);
 	return ret;
 }
 
 static void ti_csi2rx_remove(struct platform_device *pdev)
 {
 	struct ti_csi2rx_dev *csi = platform_get_drvdata(pdev);
+	unsigned int i;
 
-	video_unregister_device(&csi->vdev);
+	if (!pm_runtime_status_suspended(&pdev->dev))
+		pm_runtime_set_suspended(&pdev->dev);
 
-	ti_csi2rx_cleanup_vb2q(csi);
-	ti_csi2rx_cleanup_subdev(csi);
-	ti_csi2rx_cleanup_v4l2(csi);
-	ti_csi2rx_cleanup_dma(csi);
+	for (i = 0; i < csi->num_ctx; i++)
+		ti_csi2rx_cleanup_ctx(&csi->ctx[i]);
+
+	ti_csi2rx_cleanup_notifier(csi);
+	unregister_pm_notifier(&csi->pm_notifier);
 
-	mutex_destroy(&csi->mutex);
+	ti_csi2rx_cleanup_v4l2(csi);
+	dma_free_coherent(csi->dev, csi->drain.len, csi->drain.vaddr,
+			  csi->drain.paddr);
+	pm_runtime_disable(&pdev->dev);
 }
 
 static const struct of_device_id ti_csi2rx_of_match[] = {
@@ -1200,6 +1797,7 @@ static struct platform_driver ti_csi2rx_pdrv = {
 	.driver = {
 		.name = TI_CSI2RX_MODULE_NAME,
 		.of_match_table = ti_csi2rx_of_match,
+		.pm		= &ti_csi2rx_pm_ops,
 	},
 };
 
diff --git a/drivers/media/platform/ti/vpe/vip.c b/drivers/media/platform/ti/vpe/vip.c
index 0e91e87bda9b9..cb0a5a07a3d46 100644
--- a/drivers/media/platform/ti/vpe/vip.c
+++ b/drivers/media/platform/ti/vpe/vip.c
@@ -9,6 +9,7 @@
  */
 
 #include <linux/clk.h>
+#include <linux/cleanup.h>
 #include <linux/delay.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
@@ -3389,7 +3390,6 @@ static int vip_probe_complete(struct platform_device *pdev)
 	struct vip_port *port;
 	struct vip_dev *dev;
 	struct device_node *parent = pdev->dev.of_node;
-	struct fwnode_handle *ep = NULL;
 	unsigned int syscon_args[5];
 	int ret, i, slice_id, port_id, p;
 
@@ -3411,8 +3411,9 @@ static int vip_probe_complete(struct platform_device *pdev)
 		ctrl->syscon_bit_field[i] = syscon_args[i + 1];
 
 	for (p = 0; p < (VIP_NUM_PORTS * VIP_NUM_SLICES); p++) {
-		ep = fwnode_graph_get_next_endpoint_by_regs(of_fwnode_handle(parent),
-							    p, 0);
+		struct fwnode_handle *ep __free(fwnode_handle) =
+			fwnode_graph_get_next_endpoint_by_regs(
+				of_fwnode_handle(parent), p, 0);
 		if (!ep)
 			continue;
 
@@ -3447,7 +3448,6 @@ static int vip_probe_complete(struct platform_device *pdev)
 		port = dev->ports[port_id];
 
 		vip_register_subdev_notify(port, ep);
-		fwnode_handle_put(ep);
 	}
 	return 0;
 }
@@ -3472,7 +3472,7 @@ static int vip_probe_slice(struct platform_device *pdev, int slice)
 	ret = devm_request_irq(&pdev->dev, dev->irq, vip_irq,
 			       0, VIP_MODULE_NAME, dev);
 	if (ret < 0)
-		return -ENOMEM;
+		return ret;
 
 	spin_lock_init(&dev->slock);
 	mutex_init(&dev->mutex);
@@ -3490,7 +3490,7 @@ static int vip_probe_slice(struct platform_device *pdev, int slice)
 
 	parser = devm_kzalloc(&pdev->dev, sizeof(*dev->parser), GFP_KERNEL);
 	if (!parser)
-		return PTR_ERR_OR_ZERO(parser);
+		return -ENOMEM;
 
 	parser->base = dev->base + (slice ? VIP_SLICE1_PARSER : VIP_SLICE0_PARSER);
 	if (IS_ERR(parser->base))
@@ -3502,7 +3502,7 @@ static int vip_probe_slice(struct platform_device *pdev, int slice)
 	dev->sc_assigned = VIP_NOT_ASSIGNED;
 	sc = devm_kzalloc(&pdev->dev, sizeof(*dev->sc), GFP_KERNEL);
 	if (!sc)
-		return PTR_ERR_OR_ZERO(sc);
+		return -ENOMEM;
 
 	sc->base = dev->base + (slice ? VIP_SLICE1_SC : VIP_SLICE0_SC);
 	if (IS_ERR(sc->base))
@@ -3514,7 +3514,7 @@ static int vip_probe_slice(struct platform_device *pdev, int slice)
 	dev->csc_assigned = VIP_NOT_ASSIGNED;
 	csc = devm_kzalloc(&pdev->dev, sizeof(*dev->csc), GFP_KERNEL);
 	if (!csc)
-		return PTR_ERR_OR_ZERO(csc);
+		return -ENOMEM;
 
 	csc->base = dev->base + (slice ? VIP_SLICE1_CSC : VIP_SLICE0_CSC);
 	if (IS_ERR(csc->base))
diff --git a/drivers/media/platform/ti/vpe/vpe.c b/drivers/media/platform/ti/vpe/vpe.c
index a7e5a85e72a14..81bd1f9cee302 100644
--- a/drivers/media/platform/ti/vpe/vpe.c
+++ b/drivers/media/platform/ti/vpe/vpe.c
@@ -2539,7 +2539,8 @@ static int vpe_probe(struct platform_device *pdev)
 						"vpe_top");
 	if (!dev->res) {
 		dev_err(&pdev->dev, "missing 'vpe_top' resources data\n");
-		return -ENODEV;
+		ret = -ENODEV;
+		goto v4l2_dev_unreg;
 	}
 
 	/*
diff --git a/drivers/media/platform/verisilicon/hantro_hw.h b/drivers/media/platform/verisilicon/hantro_hw.h
index 5f2011529f02d..13e573f1f19de 100644
--- a/drivers/media/platform/verisilicon/hantro_hw.h
+++ b/drivers/media/platform/verisilicon/hantro_hw.h
@@ -427,7 +427,6 @@ extern const struct hantro_postproc_ops rockchip_vpu981_postproc_ops;
 extern const u32 hantro_vp8_dec_mc_filter[8][6];
 
 void hantro_watchdog(struct work_struct *work);
-void hantro_run(struct hantro_ctx *ctx);
 void hantro_irq_done(struct hantro_dev *vpu,
 		     enum vb2_buffer_state result);
 void hantro_start_prepare_run(struct hantro_ctx *ctx);
diff --git a/drivers/media/platform/verisilicon/hantro_v4l2.c b/drivers/media/platform/verisilicon/hantro_v4l2.c
index fcf3bd9bcda2d..83af9fa1ce949 100644
--- a/drivers/media/platform/verisilicon/hantro_v4l2.c
+++ b/drivers/media/platform/verisilicon/hantro_v4l2.c
@@ -222,6 +222,7 @@ static int vidioc_enum_fmt(struct file *file, void *priv,
 	unsigned int num_fmts, i, j = 0;
 	bool skip_mode_none, enum_all_formats;
 	u32 index = f->index & ~V4L2_FMTDESC_FLAG_ENUM_ALL;
+	bool need_postproc = ctx->need_postproc;
 
 	/*
 	 * If the V4L2_FMTDESC_FLAG_ENUM_ALL flag is set, we want to enumerate all
@@ -230,6 +231,9 @@ static int vidioc_enum_fmt(struct file *file, void *priv,
 	enum_all_formats = !!(f->index & V4L2_FMTDESC_FLAG_ENUM_ALL);
 	f->index = index;
 
+	if (enum_all_formats)
+		need_postproc = HANTRO_AUTO_POSTPROC;
+
 	/*
 	 * When dealing with an encoder:
 	 *  - on the capture side we want to filter out all MODE_NONE formats.
@@ -242,7 +246,7 @@ static int vidioc_enum_fmt(struct file *file, void *priv,
 	 */
 	skip_mode_none = capture == ctx->is_encoder;
 
-	formats = hantro_get_formats(ctx, &num_fmts, HANTRO_AUTO_POSTPROC);
+	formats = hantro_get_formats(ctx, &num_fmts, need_postproc);
 	for (i = 0; i < num_fmts; i++) {
 		bool mode_none = formats[i].codec_mode == HANTRO_MODE_NONE;
 		fmt = &formats[i];
diff --git a/drivers/media/radio/radio-si476x.c b/drivers/media/radio/radio-si476x.c
index 9980346cb5ea3..bfe89782dce43 100644
--- a/drivers/media/radio/radio-si476x.c
+++ b/drivers/media/radio/radio-si476x.c
@@ -1493,6 +1493,7 @@ static int si476x_radio_probe(struct platform_device *pdev)
 	return 0;
 exit:
 	v4l2_ctrl_handler_free(radio->videodev.ctrl_handler);
+	v4l2_device_unregister(&radio->v4l2dev);
 	return rval;
 }
 
diff --git a/drivers/media/radio/radio-tea5764.c b/drivers/media/radio/radio-tea5764.c
index 156cca2866aa5..d9547f625e30d 100644
--- a/drivers/media/radio/radio-tea5764.c
+++ b/drivers/media/radio/radio-tea5764.c
@@ -502,7 +502,7 @@ static void tea5764_i2c_remove(struct i2c_client *client)
 
 /* I2C subsystem interface */
 static const struct i2c_device_id tea5764_id[] = {
-	{ "radio-tea5764" },
+	{ .name = "radio-tea5764" },
 	{ }					/* Terminating entry */
 };
 MODULE_DEVICE_TABLE(i2c, tea5764_id);
diff --git a/drivers/media/radio/saa7706h.c b/drivers/media/radio/saa7706h.c
index 9572a866defba..bd8bd295a9ec2 100644
--- a/drivers/media/radio/saa7706h.c
+++ b/drivers/media/radio/saa7706h.c
@@ -395,8 +395,8 @@ static void saa7706h_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id saa7706h_id[] = {
-	{ DRIVER_NAME },
-	{}
+	{ .name = DRIVER_NAME },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, saa7706h_id);
diff --git a/drivers/media/radio/si470x/radio-si470x-i2c.c b/drivers/media/radio/si470x/radio-si470x-i2c.c
index 3932a449a1b1f..a1e570af9c59b 100644
--- a/drivers/media/radio/si470x/radio-si470x-i2c.c
+++ b/drivers/media/radio/si470x/radio-si470x-i2c.c
@@ -28,7 +28,7 @@
 /* I2C Device ID List */
 static const struct i2c_device_id si470x_i2c_id[] = {
 	/* Generic Entry */
-	{ "si470x" },
+	{ .name = "si470x" },
 	/* Terminating entry */
 	{ }
 };
diff --git a/drivers/media/radio/si470x/radio-si470x-usb.c b/drivers/media/radio/si470x/radio-si470x-usb.c
index 318b5f6d42021..869b1e7e34b97 100644
--- a/drivers/media/radio/si470x/radio-si470x-usb.c
+++ b/drivers/media/radio/si470x/radio-si470x-usb.c
@@ -565,8 +565,7 @@ static int si470x_usb_driver_probe(struct usb_interface *intf,
 {
 	struct si470x_device *radio;
 	struct usb_host_interface *iface_desc;
-	struct usb_endpoint_descriptor *endpoint;
-	int i, int_end_size, retval;
+	int int_end_size, retval;
 	unsigned char version_warning = 0;
 
 	/* private data allocation and initialization */
@@ -595,12 +594,8 @@ static int si470x_usb_driver_probe(struct usb_interface *intf,
 	iface_desc = intf->cur_altsetting;
 
 	/* Set up interrupt endpoint information. */
-	for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) {
-		endpoint = &iface_desc->endpoint[i].desc;
-		if (usb_endpoint_is_int_in(endpoint))
-			radio->int_in_endpoint = endpoint;
-	}
-	if (!radio->int_in_endpoint) {
+	retval = usb_find_int_in_endpoint(iface_desc, &radio->int_in_endpoint);
+	if (retval) {
 		dev_info(&intf->dev, "could not find interrupt in endpoint\n");
 		retval = -EIO;
 		goto err_usbbuf;
diff --git a/drivers/media/radio/si4713/si4713.c b/drivers/media/radio/si4713/si4713.c
index e71272c6de37a..0c0354566b0a7 100644
--- a/drivers/media/radio/si4713/si4713.c
+++ b/drivers/media/radio/si4713/si4713.c
@@ -1639,7 +1639,7 @@ static void si4713_remove(struct i2c_client *client)
 
 /* si4713_i2c_driver - i2c driver interface */
 static const struct i2c_device_id si4713_id[] = {
-	{ "si4713" },
+	{ .name = "si4713" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, si4713_id);
diff --git a/drivers/media/radio/tef6862.c b/drivers/media/radio/tef6862.c
index 3a6d7926e8565..2596d7f4f3f1a 100644
--- a/drivers/media/radio/tef6862.c
+++ b/drivers/media/radio/tef6862.c
@@ -173,8 +173,8 @@ static void tef6862_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tef6862_id[] = {
-	{ DRIVER_NAME },
-	{}
+	{ .name = DRIVER_NAME },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, tef6862_id);
diff --git a/drivers/media/rc/imon.c b/drivers/media/rc/imon.c
index 9bb27ba8240f8..049a73b5f8829 100644
--- a/drivers/media/rc/imon.c
+++ b/drivers/media/rc/imon.c
@@ -290,6 +290,10 @@ static const struct imon_usb_dev_descr imon_OEM_VFD = {
 		{ 0x000100000000ffeell, KEY_VOLUMEUP },
 		{ 0x010000000000ffeell, KEY_VOLUMEDOWN },
 		{ 0x000000000100ffeell, KEY_MUTE },
+		/* iMON VFD HID OEM v1.2 */
+		{ 0x000000000a00ffeell, KEY_VOLUMEUP },
+		{ 0x000000000b00ffeell, KEY_VOLUMEDOWN },
+		{ 0x000000000c00ffeell, KEY_MUTE },
 		/* 0xffdc iMON MCE VFD */
 		{ 0x00010000ffffffeell, KEY_VOLUMEUP },
 		{ 0x01000000ffffffeell, KEY_VOLUMEDOWN },
diff --git a/drivers/media/rc/imon_raw.c b/drivers/media/rc/imon_raw.c
index 3a526dea6532c..295acd6ba9e14 100644
--- a/drivers/media/rc/imon_raw.c
+++ b/drivers/media/rc/imon_raw.c
@@ -105,26 +105,16 @@ static void imon_ir_rx(struct urb *urb)
 static int imon_probe(struct usb_interface *intf,
 		      const struct usb_device_id *id)
 {
-	struct usb_endpoint_descriptor *ir_ep = NULL;
-	struct usb_host_interface *idesc;
+	struct usb_endpoint_descriptor *ir_ep;
 	struct usb_device *udev;
 	struct rc_dev *rcdev;
 	struct imon *imon;
-	int i, ret;
+	int ret;
 
 	udev = interface_to_usbdev(intf);
-	idesc = intf->cur_altsetting;
-
-	for (i = 0; i < idesc->desc.bNumEndpoints; i++) {
-		struct usb_endpoint_descriptor *ep = &idesc->endpoint[i].desc;
-
-		if (usb_endpoint_is_int_in(ep)) {
-			ir_ep = ep;
-			break;
-		}
-	}
 
-	if (!ir_ep) {
+	ret = usb_find_int_in_endpoint(intf->cur_altsetting, &ir_ep);
+	if (ret) {
 		dev_err(&intf->dev, "IR endpoint missing");
 		return -ENODEV;
 	}
diff --git a/drivers/media/rc/ir_toy.c b/drivers/media/rc/ir_toy.c
index 089833e411786..ee645882dd5a6 100644
--- a/drivers/media/rc/ir_toy.c
+++ b/drivers/media/rc/ir_toy.c
@@ -393,27 +393,15 @@ static int irtoy_probe(struct usb_interface *intf,
 {
 	struct usb_host_interface *idesc = intf->cur_altsetting;
 	struct usb_device *usbdev = interface_to_usbdev(intf);
-	struct usb_endpoint_descriptor *ep_in = NULL;
-	struct usb_endpoint_descriptor *ep_out = NULL;
-	struct usb_endpoint_descriptor *ep = NULL;
+	struct usb_endpoint_descriptor *ep_in, *ep_out;
 	struct irtoy *irtoy;
 	struct rc_dev *rc;
 	struct urb *urb;
-	int i, pipe, err = -ENOMEM;
+	int pipe, err;
 
-	for (i = 0; i < idesc->desc.bNumEndpoints; i++) {
-		ep = &idesc->endpoint[i].desc;
-
-		if (!ep_in && usb_endpoint_is_bulk_in(ep) &&
-		    usb_endpoint_maxp(ep) == MAX_PACKET)
-			ep_in = ep;
-
-		if (!ep_out && usb_endpoint_is_bulk_out(ep) &&
-		    usb_endpoint_maxp(ep) == MAX_PACKET)
-			ep_out = ep;
-	}
-
-	if (!ep_in || !ep_out) {
+	err = usb_find_common_endpoints(idesc, &ep_in, &ep_out, NULL, NULL);
+	if (err || usb_endpoint_maxp(ep_in) != MAX_PACKET ||
+	    usb_endpoint_maxp(ep_out) != MAX_PACKET) {
 		dev_err(&intf->dev, "required endpoints not found\n");
 		return -ENODEV;
 	}
@@ -422,6 +410,7 @@ static int irtoy_probe(struct usb_interface *intf,
 	if (!irtoy)
 		return -ENOMEM;
 
+	err = -ENOMEM;
 	irtoy->in = kmalloc(MAX_PACKET,  GFP_KERNEL);
 	if (!irtoy->in)
 		goto free_irtoy;
diff --git a/drivers/media/rc/mceusb.c b/drivers/media/rc/mceusb.c
index 6a9e4382a2243..39ba7f6a2549e 100644
--- a/drivers/media/rc/mceusb.c
+++ b/drivers/media/rc/mceusb.c
@@ -397,6 +397,8 @@ static const struct usb_device_id mceusb_dev_table[] = {
 	{ USB_DEVICE(VENDOR_COMPRO, 0x3082) },
 	/* Northstar Systems, Inc. eHome Infrared Transceiver */
 	{ USB_DEVICE(VENDOR_NORTHSTAR, 0xe004) },
+	/* Northstar Systems, Inc. eHome Infrared Transceiver - variant */
+	{ USB_DEVICE(VENDOR_NORTHSTAR, 0xe033) },
 	/* TiVo PC IR Receiver */
 	{ USB_DEVICE(VENDOR_TIVO, 0x2000),
 	  .driver_info = TIVO_KIT },
diff --git a/drivers/media/rc/ttusbir.c b/drivers/media/rc/ttusbir.c
index 3848ad3a6b85f..db2f6698a6c06 100644
--- a/drivers/media/rc/ttusbir.c
+++ b/drivers/media/rc/ttusbir.c
@@ -191,7 +191,7 @@ static int ttusbir_probe(struct usb_interface *intf,
 	tt = kzalloc_obj(*tt);
 	buffer = kzalloc(5, GFP_KERNEL);
 	rc = rc_allocate_device(RC_DRIVER_IR_RAW);
-	if (!tt || !rc || buffer) {
+	if (!tt || !rc || !buffer) {
 		ret = -ENOMEM;
 		goto out;
 	}
diff --git a/drivers/media/test-drivers/vidtv/vidtv_bridge.c b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
index a8a76434989c2..fd69b4ee16f4b 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_bridge.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_bridge.c
@@ -594,8 +594,10 @@ static int __init vidtv_bridge_init(void)
 	int ret;
 
 	ret = platform_device_register(&vidtv_bridge_dev);
-	if (ret)
+	if (ret) {
+		platform_device_put(&vidtv_bridge_dev);
 		return ret;
+	}
 
 	ret = platform_driver_register(&vidtv_bridge_driver);
 	if (ret)
diff --git a/drivers/media/test-drivers/vidtv/vidtv_demod.c b/drivers/media/test-drivers/vidtv/vidtv_demod.c
index c382e9e94c328..6e5fe402976bb 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_demod.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_demod.c
@@ -407,8 +407,8 @@ static const struct dvb_frontend_ops vidtv_demod_ops = {
 };
 
 static const struct i2c_device_id vidtv_demod_i2c_id_table[] = {
-	{ "dvb_vidtv_demod" },
-	{}
+	{ .name = "dvb_vidtv_demod" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, vidtv_demod_i2c_id_table);
 
diff --git a/drivers/media/test-drivers/vidtv/vidtv_mux.c b/drivers/media/test-drivers/vidtv/vidtv_mux.c
index f0134e38a1fb8..ea2ea53102f9f 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_mux.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_mux.c
@@ -101,7 +101,8 @@ static int vidtv_mux_pid_ctx_init(struct vidtv_mux *m)
 	/* add a ctx for all PMT sections */
 	while (p) {
 		pid = vidtv_psi_get_pat_program_pid(p);
-		vidtv_mux_create_pid_ctx_once(m, pid);
+		if (!vidtv_mux_create_pid_ctx_once(m, pid))
+			goto free;
 		p = p->next;
 	}
 
@@ -170,6 +171,9 @@ static u32 vidtv_mux_push_si(struct vidtv_mux *m)
 	nit_ctx = vidtv_mux_get_pid_ctx(m, VIDTV_NIT_PID);
 	eit_ctx = vidtv_mux_get_pid_ctx(m, VIDTV_EIT_PID);
 
+	if (!pat_ctx || !sdt_ctx || !nit_ctx || !eit_ctx)
+		return 0;
+
 	pat_args.offset             = m->mux_buf_offset;
 	pat_args.continuity_counter = &pat_ctx->cc;
 
@@ -186,6 +190,8 @@ static u32 vidtv_mux_push_si(struct vidtv_mux *m)
 		}
 
 		pmt_ctx = vidtv_mux_get_pid_ctx(m, pmt_pid);
+		if (!pmt_ctx)
+			continue;
 
 		pmt_args.offset             = m->mux_buf_offset;
 		pmt_args.pmt                = m->si.pmt_secs[i];
diff --git a/drivers/media/test-drivers/vidtv/vidtv_tuner.c b/drivers/media/test-drivers/vidtv/vidtv_tuner.c
index ee55df4029bc9..bd50b86e927c1 100644
--- a/drivers/media/test-drivers/vidtv/vidtv_tuner.c
+++ b/drivers/media/test-drivers/vidtv/vidtv_tuner.c
@@ -385,8 +385,8 @@ static const struct dvb_tuner_ops vidtv_tuner_ops = {
 };
 
 static const struct i2c_device_id vidtv_tuner_i2c_id_table[] = {
-	{ "dvb_vidtv_tuner" },
-	{}
+	{ .name = "dvb_vidtv_tuner" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, vidtv_tuner_i2c_id_table);
 
diff --git a/drivers/media/test-drivers/vimc/vimc-core.c b/drivers/media/test-drivers/vimc/vimc-core.c
index 15167e127461a..fee0c7a09c4fc 100644
--- a/drivers/media/test-drivers/vimc/vimc-core.c
+++ b/drivers/media/test-drivers/vimc/vimc-core.c
@@ -421,6 +421,7 @@ static int __init vimc_init(void)
 	if (ret) {
 		dev_err(&vimc_pdev.dev,
 			"platform device registration failed (err=%d)\n", ret);
+		platform_device_put(&vimc_pdev);
 		return ret;
 	}
 
diff --git a/drivers/media/test-drivers/visl/visl-core.c b/drivers/media/test-drivers/visl/visl-core.c
index 127ab18bce99f..5e9bdd425d4a3 100644
--- a/drivers/media/test-drivers/visl/visl-core.c
+++ b/drivers/media/test-drivers/visl/visl-core.c
@@ -339,6 +339,10 @@ static int visl_open(struct file *file)
 	}
 
 	ctx->tpg_str_buf = kzalloc(TPG_STR_BUF_SZ, GFP_KERNEL);
+	if (!ctx->tpg_str_buf) {
+		rc = -ENOMEM;
+		goto free_ctx;
+	}
 
 	v4l2_fh_init(&ctx->fh, video_devdata(file));
 	ctx->dev = dev;
diff --git a/drivers/media/test-drivers/vivid/vivid-core.c b/drivers/media/test-drivers/vivid/vivid-core.c
index c8bf9b4d406c2..62cfb5feb2cf5 100644
--- a/drivers/media/test-drivers/vivid/vivid-core.c
+++ b/drivers/media/test-drivers/vivid/vivid-core.c
@@ -2289,8 +2289,10 @@ static int __init vivid_init(void)
 		}
 	}
 	ret = platform_device_register(&vivid_pdev);
-	if (ret)
+	if (ret) {
+		platform_device_put(&vivid_pdev);
 		goto free_output_strings;
+	}
 	ret = platform_driver_register(&vivid_pdrv);
 	if (ret)
 		goto unreg_device;
@@ -2311,7 +2313,7 @@ static int __init vivid_init(void)
 destroy_hdmi_wq:
 	destroy_workqueue(update_hdmi_ctrls_workqueue);
 unreg_driver:
-	platform_driver_register(&vivid_pdrv);
+	platform_driver_unregister(&vivid_pdrv);
 unreg_device:
 	platform_device_unregister(&vivid_pdev);
 free_output_strings:
diff --git a/drivers/media/test-drivers/vivid/vivid-ctrls.c b/drivers/media/test-drivers/vivid/vivid-ctrls.c
index f94c15ff84f78..a8a134b36720e 100644
--- a/drivers/media/test-drivers/vivid/vivid-ctrls.c
+++ b/drivers/media/test-drivers/vivid/vivid-ctrls.c
@@ -609,17 +609,24 @@ static int vivid_vid_cap_s_ctrl(struct v4l2_ctrl *ctrl)
 		break;
 	case VIVID_CID_REDUCED_FPS:
 		dev->reduced_fps = ctrl->val;
-		vivid_update_format_cap(dev, true);
+		if (dev->input_type[dev->input] == HDMI)
+			vivid_update_reduced_fps(dev);
 		break;
 	case VIVID_CID_HAS_CROP_CAP:
+		if (vb2_is_busy(&dev->vb_vid_cap_q))
+			return -EBUSY;
 		dev->has_crop_cap = ctrl->val;
 		vivid_update_format_cap(dev, true);
 		break;
 	case VIVID_CID_HAS_COMPOSE_CAP:
+		if (vb2_is_busy(&dev->vb_vid_cap_q))
+			return -EBUSY;
 		dev->has_compose_cap = ctrl->val;
 		vivid_update_format_cap(dev, true);
 		break;
 	case VIVID_CID_HAS_SCALER_CAP:
+		if (vb2_is_busy(&dev->vb_vid_cap_q))
+			return -EBUSY;
 		dev->has_scaler_cap = ctrl->val;
 		vivid_update_format_cap(dev, true);
 		break;
@@ -1116,14 +1123,20 @@ static int vivid_vid_out_s_ctrl(struct v4l2_ctrl *ctrl)
 
 	switch (ctrl->id) {
 	case VIVID_CID_HAS_CROP_OUT:
+		if (vb2_is_busy(&dev->vb_vid_out_q))
+			return -EBUSY;
 		dev->has_crop_out = ctrl->val;
 		vivid_update_format_out(dev);
 		break;
 	case VIVID_CID_HAS_COMPOSE_OUT:
+		if (vb2_is_busy(&dev->vb_vid_out_q))
+			return -EBUSY;
 		dev->has_compose_out = ctrl->val;
 		vivid_update_format_out(dev);
 		break;
 	case VIVID_CID_HAS_SCALER_OUT:
+		if (vb2_is_busy(&dev->vb_vid_out_q))
+			return -EBUSY;
 		dev->has_scaler_out = ctrl->val;
 		vivid_update_format_out(dev);
 		break;
diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
index b95f06a9b5ae9..e204490847095 100644
--- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c
+++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c
@@ -364,6 +364,24 @@ static enum tpg_pixel_aspect vivid_get_pixel_aspect(const struct vivid_dev *dev)
 	return TPG_PIXEL_ASPECT_SQUARE;
 }
 
+void vivid_update_reduced_fps(struct vivid_dev *dev)
+{
+	struct v4l2_bt_timings *bt = &dev->dv_timings_cap[dev->input].bt;
+	unsigned int size = V4L2_DV_BT_FRAME_WIDTH(bt) * V4L2_DV_BT_FRAME_HEIGHT(bt);
+	u64 pixelclock;
+
+	if (dev->reduced_fps && can_reduce_fps(bt)) {
+		pixelclock = div_u64(bt->pixelclock * 1000, 1001);
+		bt->flags |= V4L2_DV_FL_REDUCED_FPS;
+	} else {
+		pixelclock = bt->pixelclock;
+		bt->flags &= ~V4L2_DV_FL_REDUCED_FPS;
+	}
+	dev->timeperframe_vid_cap = (struct v4l2_fract) {
+		size / 100, (u32)pixelclock / 100
+	};
+}
+
 /*
  * Called whenever the format has to be reset which can occur when
  * changing inputs, standard, timings, etc.
@@ -372,8 +390,12 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls)
 {
 	struct v4l2_bt_timings *bt = &dev->dv_timings_cap[dev->input].bt;
 	u32 dims[V4L2_CTRL_MAX_DIMS] = {};
-	unsigned size;
-	u64 pixelclock;
+
+	/*
+	 * This resets the format, so must never be called while vb2_is_busy().
+	 */
+	if (WARN_ON(vb2_is_busy(&dev->vb_vid_cap_q)))
+		return;
 
 	switch (dev->input_type[dev->input]) {
 	case WEBCAM:
@@ -402,17 +424,7 @@ void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls)
 	case HDMI:
 		dev->src_rect.width = bt->width;
 		dev->src_rect.height = bt->height;
-		size = V4L2_DV_BT_FRAME_WIDTH(bt) * V4L2_DV_BT_FRAME_HEIGHT(bt);
-		if (dev->reduced_fps && can_reduce_fps(bt)) {
-			pixelclock = div_u64(bt->pixelclock * 1000, 1001);
-			bt->flags |= V4L2_DV_FL_REDUCED_FPS;
-		} else {
-			pixelclock = bt->pixelclock;
-			bt->flags &= ~V4L2_DV_FL_REDUCED_FPS;
-		}
-		dev->timeperframe_vid_cap = (struct v4l2_fract) {
-			size / 100, (u32)pixelclock / 100
-		};
+		vivid_update_reduced_fps(dev);
 		if (bt->interlaced)
 			dev->field_cap = V4L2_FIELD_ALTERNATE;
 		else
diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.h b/drivers/media/test-drivers/vivid/vivid-vid-cap.h
index 38a99f7e038ea..d08a85927510f 100644
--- a/drivers/media/test-drivers/vivid/vivid-vid-cap.h
+++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.h
@@ -9,6 +9,7 @@
 #define _VIVID_VID_CAP_H_
 
 void vivid_update_quality(struct vivid_dev *dev);
+void vivid_update_reduced_fps(struct vivid_dev *dev);
 void vivid_update_format_cap(struct vivid_dev *dev, bool keep_controls);
 void vivid_update_outputs(struct vivid_dev *dev);
 void vivid_update_connected_outputs(struct vivid_dev *dev);
diff --git a/drivers/media/test-drivers/vivid/vivid-vid-out.c b/drivers/media/test-drivers/vivid/vivid-vid-out.c
index 8c037b90833e7..23e1d5a189eed 100644
--- a/drivers/media/test-drivers/vivid/vivid-vid-out.c
+++ b/drivers/media/test-drivers/vivid/vivid-vid-out.c
@@ -214,6 +214,12 @@ void vivid_update_format_out(struct vivid_dev *dev)
 	unsigned size, p;
 	u64 pixelclock;
 
+	/*
+	 * This resets the format, so must never be called while vb2_is_busy().
+	 */
+	if (WARN_ON(vb2_is_busy(&dev->vb_vid_out_q)))
+		return;
+
 	switch (dev->output_type[dev->output]) {
 	case SVID:
 	default:
diff --git a/drivers/media/tuners/e4000.c b/drivers/media/tuners/e4000.c
index b83f37a77224d..94abb37154018 100644
--- a/drivers/media/tuners/e4000.c
+++ b/drivers/media/tuners/e4000.c
@@ -719,8 +719,8 @@ static void e4000_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id e4000_id_table[] = {
-	{ "e4000" },
-	{}
+	{ .name = "e4000" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, e4000_id_table);
 
diff --git a/drivers/media/tuners/fc2580.c b/drivers/media/tuners/fc2580.c
index 75087d9b224f0..b76fa1320f5fb 100644
--- a/drivers/media/tuners/fc2580.c
+++ b/drivers/media/tuners/fc2580.c
@@ -600,8 +600,8 @@ static void fc2580_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id fc2580_id_table[] = {
-	{ "fc2580" },
-	{}
+	{ .name = "fc2580" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, fc2580_id_table);
 
diff --git a/drivers/media/tuners/m88rs6000t.c b/drivers/media/tuners/m88rs6000t.c
index 0a724cdf0f6d8..1addb3d229cfe 100644
--- a/drivers/media/tuners/m88rs6000t.c
+++ b/drivers/media/tuners/m88rs6000t.c
@@ -709,8 +709,8 @@ static void m88rs6000t_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id m88rs6000t_id[] = {
-	{ "m88rs6000t" },
-	{}
+	{ .name = "m88rs6000t" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, m88rs6000t_id);
 
diff --git a/drivers/media/tuners/mt2060.c b/drivers/media/tuners/mt2060.c
index ef3196e6bd303..c57233c7441ac 100644
--- a/drivers/media/tuners/mt2060.c
+++ b/drivers/media/tuners/mt2060.c
@@ -514,8 +514,8 @@ static void mt2060_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id mt2060_id_table[] = {
-	{ "mt2060" },
-	{}
+	{ .name = "mt2060" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mt2060_id_table);
 
diff --git a/drivers/media/tuners/mxl301rf.c b/drivers/media/tuners/mxl301rf.c
index cfc78891ce03d..1d84456cb19ba 100644
--- a/drivers/media/tuners/mxl301rf.c
+++ b/drivers/media/tuners/mxl301rf.c
@@ -317,8 +317,8 @@ static void mxl301rf_remove(struct i2c_client *client)
 
 
 static const struct i2c_device_id mxl301rf_id[] = {
-	{ "mxl301rf" },
-	{}
+	{ .name = "mxl301rf" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mxl301rf_id);
 
diff --git a/drivers/media/tuners/qm1d1b0004.c b/drivers/media/tuners/qm1d1b0004.c
index 07ad84f42c9f1..59d98681e6748 100644
--- a/drivers/media/tuners/qm1d1b0004.c
+++ b/drivers/media/tuners/qm1d1b0004.c
@@ -243,8 +243,8 @@ static void qm1d1b0004_remove(struct i2c_client *client)
 
 
 static const struct i2c_device_id qm1d1b0004_id[] = {
-	{ "qm1d1b0004" },
-	{}
+	{ .name = "qm1d1b0004" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, qm1d1b0004_id);
diff --git a/drivers/media/tuners/qm1d1c0042.c b/drivers/media/tuners/qm1d1c0042.c
index db60562ad6989..2d19cfdb67b99 100644
--- a/drivers/media/tuners/qm1d1c0042.c
+++ b/drivers/media/tuners/qm1d1c0042.c
@@ -434,8 +434,8 @@ static void qm1d1c0042_remove(struct i2c_client *client)
 
 
 static const struct i2c_device_id qm1d1c0042_id[] = {
-	{ "qm1d1c0042" },
-	{}
+	{ .name = "qm1d1c0042" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, qm1d1c0042_id);
 
diff --git a/drivers/media/tuners/si2157.c b/drivers/media/tuners/si2157.c
index 4d67e347c22f0..d517a91e6fbc7 100644
--- a/drivers/media/tuners/si2157.c
+++ b/drivers/media/tuners/si2157.c
@@ -1097,11 +1097,11 @@ static void si2157_remove(struct i2c_client *client)
  * all SiLabs TER tuners, as the driver should auto-detect it.
  */
 static const struct i2c_device_id si2157_id_table[] = {
-	{"si2157", SI2157},
-	{"si2146", SI2146},
-	{"si2141", SI2141},
-	{"si2177", SI2177},
-	{}
+	{ .name = "si2157", .driver_data = SI2157 },
+	{ .name = "si2146", .driver_data = SI2146 },
+	{ .name = "si2141", .driver_data = SI2141 },
+	{ .name = "si2177", .driver_data = SI2177 },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, si2157_id_table);
 
diff --git a/drivers/media/tuners/tda18212.c b/drivers/media/tuners/tda18212.c
index 5f583c010408f..18d1850691fb9 100644
--- a/drivers/media/tuners/tda18212.c
+++ b/drivers/media/tuners/tda18212.c
@@ -254,8 +254,8 @@ static void tda18212_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tda18212_id[] = {
-	{ "tda18212" },
-	{}
+	{ .name = "tda18212" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tda18212_id);
 
diff --git a/drivers/media/tuners/tda18250.c b/drivers/media/tuners/tda18250.c
index caaf5e0d0c9bc..7bb945ba09898 100644
--- a/drivers/media/tuners/tda18250.c
+++ b/drivers/media/tuners/tda18250.c
@@ -868,8 +868,8 @@ static void tda18250_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tda18250_id_table[] = {
-	{ "tda18250" },
-	{}
+	{ .name = "tda18250" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tda18250_id_table);
 
diff --git a/drivers/media/tuners/tua9001.c b/drivers/media/tuners/tua9001.c
index c0aed1b441e27..fcdbf1a0c1d8a 100644
--- a/drivers/media/tuners/tua9001.c
+++ b/drivers/media/tuners/tua9001.c
@@ -245,8 +245,8 @@ static void tua9001_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tua9001_id_table[] = {
-	{ "tua9001" },
-	{}
+	{ .name = "tua9001" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tua9001_id_table);
 
diff --git a/drivers/media/usb/airspy/airspy.c b/drivers/media/usb/airspy/airspy.c
index 8f6b721ba107c..57edb42463e81 100644
--- a/drivers/media/usb/airspy/airspy.c
+++ b/drivers/media/usb/airspy/airspy.c
@@ -522,11 +522,13 @@ static int airspy_start_streaming(struct vb2_queue *vq, unsigned int count)
 
 	dev_dbg(s->dev, "\n");
 
-	if (!s->udev)
-		return -ENODEV;
-
 	mutex_lock(&s->v4l2_lock);
 
+	if (!s->udev) {
+		ret = -ENODEV;
+		goto err_clear_bit;
+	}
+
 	s->sequence = 0;
 
 	set_bit(POWER_ON, &s->flags);
diff --git a/drivers/media/usb/cx231xx/cx231xx-cards.c b/drivers/media/usb/cx231xx/cx231xx-cards.c
index b75535d6abafe..69b24205bc567 100644
--- a/drivers/media/usb/cx231xx/cx231xx-cards.c
+++ b/drivers/media/usb/cx231xx/cx231xx-cards.c
@@ -1573,7 +1573,8 @@ static int cx231xx_init_v4l2(struct cx231xx *dev,
 		 dev->video_mode.end_point_addr,
 		 dev->video_mode.num_alt);
 
-	dev->video_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->video_mode.num_alt, GFP_KERNEL);
+	dev->video_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32,
+							      dev->video_mode.num_alt, GFP_KERNEL);
 	if (dev->video_mode.alt_max_pkt_size == NULL)
 		return -ENOMEM;
 
@@ -1614,7 +1615,8 @@ static int cx231xx_init_v4l2(struct cx231xx *dev,
 		 dev->vbi_mode.num_alt);
 
 	/* compute alternate max packet sizes for vbi */
-	dev->vbi_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->vbi_mode.num_alt, GFP_KERNEL);
+	dev->vbi_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32,
+							    dev->vbi_mode.num_alt, GFP_KERNEL);
 	if (dev->vbi_mode.alt_max_pkt_size == NULL)
 		return -ENOMEM;
 
@@ -1656,7 +1658,9 @@ static int cx231xx_init_v4l2(struct cx231xx *dev,
 		 "sliced CC EndPoint Addr 0x%x, Alternate settings: %i\n",
 		 dev->sliced_cc_mode.end_point_addr,
 		 dev->sliced_cc_mode.num_alt);
-	dev->sliced_cc_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->sliced_cc_mode.num_alt, GFP_KERNEL);
+	dev->sliced_cc_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32,
+								  dev->sliced_cc_mode.num_alt,
+								  GFP_KERNEL);
 	if (dev->sliced_cc_mode.alt_max_pkt_size == NULL)
 		return -ENOMEM;
 
@@ -1720,7 +1724,7 @@ static int cx231xx_usb_probe(struct usb_interface *interface,
 	udev = interface_to_usbdev(interface);
 
 	/* allocate memory for our device state and initialize it */
-	dev = devm_kzalloc(&udev->dev, sizeof(*dev), GFP_KERNEL);
+	dev = devm_kzalloc(&interface->dev, sizeof(*dev), GFP_KERNEL);
 	if (dev == NULL) {
 		retval = -ENOMEM;
 		goto err_if;
@@ -1850,7 +1854,9 @@ static int cx231xx_usb_probe(struct usb_interface *interface,
 			 dev->ts1_mode.end_point_addr,
 			 dev->ts1_mode.num_alt);
 
-		dev->ts1_mode.alt_max_pkt_size = devm_kmalloc_array(&udev->dev, 32, dev->ts1_mode.num_alt, GFP_KERNEL);
+		dev->ts1_mode.alt_max_pkt_size = devm_kmalloc_array(&interface->dev, 32,
+								    dev->ts1_mode.num_alt,
+								    GFP_KERNEL);
 		if (dev->ts1_mode.alt_max_pkt_size == NULL) {
 			retval = -ENOMEM;
 			goto err_video_alt;
diff --git a/drivers/media/usb/em28xx/em28xx-video.c b/drivers/media/usb/em28xx/em28xx-video.c
index 4a0ce9c5ee4b4..da0422c65e5f4 100644
--- a/drivers/media/usb/em28xx/em28xx-video.c
+++ b/drivers/media/usb/em28xx/em28xx-video.c
@@ -1248,8 +1248,10 @@ int em28xx_start_analog_streaming(struct vb2_queue *vq, unsigned int count)
 					  dev->max_pkt_size,
 					  dev->packet_multiplier,
 					  em28xx_urb_data_copy);
-		if (rc < 0)
+		if (rc < 0) {
+			res_free(dev, vq->type);
 			return rc;
+		}
 
 		/*
 		 * djh: it's not clear whether this code is still needed.  I'm
diff --git a/drivers/media/usb/go7007/s2250-board.c b/drivers/media/usb/go7007/s2250-board.c
index 567f851d58968..0901d79e827d9 100644
--- a/drivers/media/usb/go7007/s2250-board.c
+++ b/drivers/media/usb/go7007/s2250-board.c
@@ -611,7 +611,7 @@ static void s2250_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id s2250_id[] = {
-	{ "s2250" },
+	{ .name = "s2250" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, s2250_id);
diff --git a/drivers/media/usb/gspca/gspca.c b/drivers/media/usb/gspca/gspca.c
index f3d3f441c8510..594d73e50b9fa 100644
--- a/drivers/media/usb/gspca/gspca.c
+++ b/drivers/media/usb/gspca/gspca.c
@@ -208,22 +208,17 @@ static int alloc_and_submit_int_urb(struct gspca_dev *gspca_dev,
 static void gspca_input_create_urb(struct gspca_dev *gspca_dev)
 {
 	struct usb_interface *intf;
-	struct usb_host_interface *intf_desc;
 	struct usb_endpoint_descriptor *ep;
-	int i;
+	int ret;
 
 	if (gspca_dev->sd_desc->int_pkt_scan)  {
 		intf = usb_ifnum_to_if(gspca_dev->dev, gspca_dev->iface);
-		intf_desc = intf->cur_altsetting;
-		for (i = 0; i < intf_desc->desc.bNumEndpoints; i++) {
-			ep = &intf_desc->endpoint[i].desc;
-			if (usb_endpoint_dir_in(ep) &&
-			    usb_endpoint_xfer_int(ep)) {
 
-				alloc_and_submit_int_urb(gspca_dev, ep);
-				break;
-			}
-		}
+		ret = usb_find_int_in_endpoint(intf->cur_altsetting, &ep);
+		if (ret)
+			return;
+
+		alloc_and_submit_int_urb(gspca_dev, ep);
 	}
 }
 
diff --git a/drivers/media/usb/gspca/sonixb.c b/drivers/media/usb/gspca/sonixb.c
index 4d655e2da9cba..f251a886024f8 100644
--- a/drivers/media/usb/gspca/sonixb.c
+++ b/drivers/media/usb/gspca/sonixb.c
@@ -943,7 +943,7 @@ static int sd_config(struct gspca_dev *gspca_dev,
 /* this function is called at probe and resume time */
 static int sd_init(struct gspca_dev *gspca_dev)
 {
-	const __u8 stop = 0x09; /* Disable stream turn of LED */
+	const __u8 stop = 0x09; /* Disable stream, turn off LED */
 
 	reg_w(gspca_dev, 0x01, &stop, 1);
 
diff --git a/drivers/media/usb/gspca/touptek.c b/drivers/media/usb/gspca/touptek.c
index dde311c25d9bd..734644d928abc 100644
--- a/drivers/media/usb/gspca/touptek.c
+++ b/drivers/media/usb/gspca/touptek.c
@@ -709,19 +709,4 @@ static struct usb_driver sd_driver = {
 #endif
 };
 
-static int __init sd_mod_init(void)
-{
-	int ret;
-
-	ret = usb_register(&sd_driver);
-	if (ret < 0)
-		return ret;
-	return 0;
-}
-static void __exit sd_mod_exit(void)
-{
-	usb_deregister(&sd_driver);
-}
-
-module_init(sd_mod_init);
-module_exit(sd_mod_exit);
+module_usb_driver(sd_driver);
diff --git a/drivers/media/usb/hdpvr/hdpvr-core.c b/drivers/media/usb/hdpvr/hdpvr-core.c
index d42336836b185..849a2be416bdf 100644
--- a/drivers/media/usb/hdpvr/hdpvr-core.c
+++ b/drivers/media/usb/hdpvr/hdpvr-core.c
@@ -265,13 +265,10 @@ static int hdpvr_probe(struct usb_interface *interface,
 		       const struct usb_device_id *id)
 {
 	struct hdpvr_device *dev;
-	struct usb_host_interface *iface_desc;
 	struct usb_endpoint_descriptor *endpoint;
 #if IS_ENABLED(CONFIG_I2C)
 	struct i2c_client *client;
 #endif
-	size_t buffer_size;
-	int i;
 	int dev_num;
 	int retval = -ENOMEM;
 
@@ -321,25 +318,18 @@ static int hdpvr_probe(struct usb_interface *interface,
 
 	/* set up the endpoint information */
 	/* use only the first bulk-in and bulk-out endpoints */
-	iface_desc = interface->cur_altsetting;
-	for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) {
-		endpoint = &iface_desc->endpoint[i].desc;
-
-		if (!dev->bulk_in_endpointAddr &&
-		    usb_endpoint_is_bulk_in(endpoint)) {
-			/* USB interface description is buggy, reported max
-			 * packet size is 512 bytes, windows driver uses 8192 */
-			buffer_size = 8192;
-			dev->bulk_in_size = buffer_size;
-			dev->bulk_in_endpointAddr = endpoint->bEndpointAddress;
-		}
-
-	}
-	if (!dev->bulk_in_endpointAddr) {
+	if (usb_find_bulk_in_endpoint(interface->cur_altsetting, &endpoint)) {
 		v4l2_err(&dev->v4l2_dev, "Could not find bulk-in endpoint\n");
 		goto error_put_usb;
 	}
 
+	/*
+	 * USB interface description is buggy, reported max packet size is 512
+	 * bytes, windows driver uses 8192
+	 */
+	dev->bulk_in_size = 8192;
+	dev->bulk_in_endpointAddr = endpoint->bEndpointAddress;
+
 	/* init the device */
 	if (hdpvr_device_init(dev)) {
 		v4l2_err(&dev->v4l2_dev, "device init failed\n");
diff --git a/drivers/media/usb/msi2500/msi2500.c b/drivers/media/usb/msi2500/msi2500.c
index 1ff98956b680b..0614087c3c3cd 100644
--- a/drivers/media/usb/msi2500/msi2500.c
+++ b/drivers/media/usb/msi2500/msi2500.c
@@ -541,7 +541,8 @@ static int msi2500_isoc_init(struct msi2500_dev *dev)
 }
 
 /* Must be called with vb_queue_lock hold */
-static void msi2500_cleanup_queued_bufs(struct msi2500_dev *dev)
+static void msi2500_cleanup_queued_bufs(struct msi2500_dev *dev,
+					enum vb2_buffer_state state)
 {
 	unsigned long flags;
 
@@ -554,7 +555,7 @@ static void msi2500_cleanup_queued_bufs(struct msi2500_dev *dev)
 		buf = list_entry(dev->queued_bufs.next,
 				 struct msi2500_frame_buf, list);
 		list_del(&buf->list);
-		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
+		vb2_buffer_done(&buf->vb.vb2_buf, state);
 	}
 	spin_unlock_irqrestore(&dev->queued_bufs_lock, flags);
 }
@@ -830,25 +831,40 @@ static int msi2500_start_streaming(struct vb2_queue *vq, unsigned int count)
 
 	dev_dbg(dev->dev, "\n");
 
-	if (!dev->udev)
-		return -ENODEV;
+	if (!dev->udev) {
+		ret = -ENODEV;
+		goto err_cleanup;
+	}
 
-	if (mutex_lock_interruptible(&dev->v4l2_lock))
-		return -ERESTARTSYS;
+	if (mutex_lock_interruptible(&dev->v4l2_lock)) {
+		ret = -ERESTARTSYS;
+		goto err_cleanup;
+	}
 
 	/* wake-up tuner */
 	v4l2_subdev_call(dev->v4l2_subdev, core, s_power, 1);
 
 	ret = msi2500_set_usb_adc(dev);
+	if (ret)
+		goto err_unlock_cleanup;
 
 	ret = msi2500_isoc_init(dev);
 	if (ret)
-		msi2500_cleanup_queued_bufs(dev);
+		goto err_unlock_cleanup;
 
 	ret = msi2500_ctrl_msg(dev, CMD_START_STREAMING, 0);
+	if (ret)
+		goto err_isoc_cleanup;
 
 	mutex_unlock(&dev->v4l2_lock);
+	return 0;
 
+err_isoc_cleanup:
+	msi2500_isoc_cleanup(dev);
+err_unlock_cleanup:
+	mutex_unlock(&dev->v4l2_lock);
+err_cleanup:
+	msi2500_cleanup_queued_bufs(dev, VB2_BUF_STATE_QUEUED);
 	return ret;
 }
 
@@ -863,7 +879,7 @@ static void msi2500_stop_streaming(struct vb2_queue *vq)
 	if (dev->udev)
 		msi2500_isoc_cleanup(dev);
 
-	msi2500_cleanup_queued_bufs(dev);
+	msi2500_cleanup_queued_bufs(dev, VB2_BUF_STATE_ERROR);
 
 	/* according to tests, at least 700us delay is required  */
 	msleep(20);
diff --git a/drivers/media/usb/pwc/pwc-if.c b/drivers/media/usb/pwc/pwc-if.c
index c416e2fc57548..e2884b04d952f 100644
--- a/drivers/media/usb/pwc/pwc-if.c
+++ b/drivers/media/usb/pwc/pwc-if.c
@@ -710,11 +710,15 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count)
 	struct pwc_device *pdev = vb2_get_drv_priv(vq);
 	int r;
 
-	if (!pdev->udev)
+	if (!pdev->udev) {
+		pwc_cleanup_queued_bufs(pdev, VB2_BUF_STATE_QUEUED);
 		return -ENODEV;
+	}
 
-	if (mutex_lock_interruptible(&pdev->v4l2_lock))
+	if (mutex_lock_interruptible(&pdev->v4l2_lock)) {
+		pwc_cleanup_queued_bufs(pdev, VB2_BUF_STATE_QUEUED);
 		return -ERESTARTSYS;
+	}
 	/* Turn on camera and set LEDS on */
 	pwc_camera_power(pdev, 1);
 	pwc_set_leds(pdev, leds[0], leds[1]);
@@ -726,6 +730,11 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count)
 		pwc_camera_power(pdev, 0);
 		/* And cleanup any queued bufs!! */
 		pwc_cleanup_queued_bufs(pdev, VB2_BUF_STATE_QUEUED);
+		if (pdev->fill_buf) {
+			vb2_buffer_done(&pdev->fill_buf->vb.vb2_buf,
+					VB2_BUF_STATE_QUEUED);
+			pdev->fill_buf = NULL;
+		}
 	}
 	mutex_unlock(&pdev->v4l2_lock);
 
diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c
index 2c02873d09b5f..0b8182edf8e4b 100644
--- a/drivers/media/usb/s2255/s2255drv.c
+++ b/drivers/media/usb/s2255/s2255drv.c
@@ -2240,18 +2240,14 @@ static int s2255_probe(struct usb_interface *interface,
 	iface_desc = interface->cur_altsetting;
 	dev_dbg(&interface->dev, "num EP: %d\n",
 		iface_desc->desc.bNumEndpoints);
-	for (i = 0; i < iface_desc->desc.bNumEndpoints; ++i) {
-		endpoint = &iface_desc->endpoint[i].desc;
-		if (!dev->read_endpoint && usb_endpoint_is_bulk_in(endpoint)) {
-			/* we found the bulk in endpoint */
-			dev->read_endpoint = endpoint->bEndpointAddress;
-		}
-	}
 
-	if (!dev->read_endpoint) {
+	if (usb_find_bulk_in_endpoint(iface_desc, &endpoint)) {
 		dev_err(&interface->dev, "Could not find bulk-in endpoint\n");
 		goto errorEP;
 	}
+
+	dev->read_endpoint = endpoint->bEndpointAddress;
+
 	timer_setup(&dev->timer, s2255_timer, 0);
 	init_waitqueue_head(&dev->fw_data->wait_fw);
 	for (i = 0; i < MAX_CHANNELS; i++) {
diff --git a/drivers/media/usb/uvc/uvc_ctrl.c b/drivers/media/usb/uvc/uvc_ctrl.c
index b9a3d9257a11b..3ca108b83f1da 100644
--- a/drivers/media/usb/uvc/uvc_ctrl.c
+++ b/drivers/media/usb/uvc/uvc_ctrl.c
@@ -385,6 +385,99 @@ static const struct uvc_control_info uvc_ctrls[] = {
 				| UVC_CTRL_FLAG_GET_RANGE
 				| UVC_CTRL_FLAG_RESTORE,
 	},
+	/*
+	 * Allows the control of pan/tilt motor movements for camera models
+	 * that support mechanical pan/tilt.
+	 *
+	 * Bits 0 to 15 control pan, bits 16 to 31 control tilt.
+	 * The unit of the pan/tilt values is 1/64th of a degree and the
+	 * resolution is 1 degree.
+	 */
+	{
+		.entity         = UVC_GUID_LOGITECH_MOTOR_CONTROL_V1,
+		.selector       = 1,
+		.index          = 0,
+		.size           = 4,
+		.flags          = UVC_CTRL_FLAG_GET_DEF
+				| UVC_CTRL_FLAG_GET_MAX
+				| UVC_CTRL_FLAG_GET_MIN
+				| UVC_CTRL_FLAG_SET_CUR,
+	},
+	/*
+	 * Reset the pan/tilt motors to their original position for camera
+	 * models that support mechanical pan/tilt.
+	 *
+	 * Setting bit 0 resets the pan position.
+	 * Setting bit 1 resets the tilt position.
+	 *
+	 * Both bits can be set at the same time to reset both, pan and tilt,
+	 * at the same time.
+	 */
+	{
+		.entity         = UVC_GUID_LOGITECH_MOTOR_CONTROL_V1,
+		.selector       = 2,
+		.index          = 1,
+		.size           = 1,
+		.flags          = UVC_CTRL_FLAG_GET_DEF
+				| UVC_CTRL_FLAG_GET_MAX
+				| UVC_CTRL_FLAG_GET_MIN
+				| UVC_CTRL_FLAG_SET_CUR,
+	},
+	/*
+	 * Allows the control of focus motor movements for camera models that
+	 * support mechanical focus.
+	 *
+	 * Bits 0 to 7 allow selection of the desired lens position.
+	 * There are no physical units, instead, the focus range is spread over
+	 * 256 logical units with 0 representing infinity focus and 255 being
+	 * macro focus.
+	 */
+	{
+		.entity         = UVC_GUID_LOGITECH_MOTOR_CONTROL_V1,
+		.selector       = 3,
+		.index          = 2,
+		.size           = 6,
+		.flags          = UVC_CTRL_FLAG_GET_CUR
+				| UVC_CTRL_FLAG_GET_DEF
+				| UVC_CTRL_FLAG_GET_MAX
+				| UVC_CTRL_FLAG_GET_MIN
+				| UVC_CTRL_FLAG_SET_CUR,
+	},
+	/*
+	 * Allows the control of pan/tilt motor movements for camera models
+	 * that support mechanical pan/tilt.
+	 *
+	 * Bits 0 to 15 control pan, bits 16 to 31 control tilt.
+	 */
+	{
+		.entity         = UVC_GUID_LOGITECH_PERIPHERAL,
+		.selector       = 1,
+		.index          = 0,
+		.size           = 4,
+		.flags          = UVC_CTRL_FLAG_GET_DEF
+				| UVC_CTRL_FLAG_GET_MAX
+				| UVC_CTRL_FLAG_GET_MIN
+				| UVC_CTRL_FLAG_GET_RES
+				| UVC_CTRL_FLAG_SET_CUR,
+	},
+	/*
+	 * Reset the pan/tilt motors to their original position for camera
+	 * models that support mechanical pan/tilt.
+	 *
+	 * Setting bit 0 resets the pan position.
+	 * Setting bit 1 resets the tilt position.
+	 */
+	{
+		.entity         = UVC_GUID_LOGITECH_PERIPHERAL,
+		.selector       = 2,
+		.index          = 1,
+		.size           = 1,
+		.flags          = UVC_CTRL_FLAG_GET_DEF
+				| UVC_CTRL_FLAG_GET_MAX
+				| UVC_CTRL_FLAG_GET_MIN
+				| UVC_CTRL_FLAG_GET_RES
+				| UVC_CTRL_FLAG_SET_CUR,
+	},
 };
 
 static const u32 uvc_control_classes[] = {
@@ -1009,6 +1102,87 @@ static const struct uvc_control_mapping uvc_ctrl_mappings[] = {
 		.menu_mask	= BIT(V4L2_COLORFX_VIVID) |
 				  BIT(V4L2_COLORFX_NONE),
 	},
+	{
+		.id             = V4L2_CID_PAN_RELATIVE,
+		.entity         = UVC_GUID_LOGITECH_MOTOR_CONTROL_V1,
+		.selector       = 1,
+		.size           = 16,
+		.offset         = 0,
+		.v4l2_type      = V4L2_CTRL_TYPE_INTEGER,
+		.data_type      = UVC_CTRL_DATA_TYPE_SIGNED,
+	},
+	{
+		.id             = V4L2_CID_TILT_RELATIVE,
+		.entity         = UVC_GUID_LOGITECH_MOTOR_CONTROL_V1,
+		.selector       = 1,
+		.size           = 16,
+		.offset         = 16,
+		.v4l2_type      = V4L2_CTRL_TYPE_INTEGER,
+		.data_type      = UVC_CTRL_DATA_TYPE_SIGNED,
+	},
+	{
+		.id             = V4L2_CID_PAN_RESET,
+		.entity         = UVC_GUID_LOGITECH_MOTOR_CONTROL_V1,
+		.selector       = 2,
+		.size           = 1,
+		.offset         = 0,
+		.v4l2_type      = V4L2_CTRL_TYPE_BUTTON,
+		.data_type      = UVC_CTRL_DATA_TYPE_UNSIGNED,
+	},
+	{
+		.id             = V4L2_CID_TILT_RESET,
+		.entity         = UVC_GUID_LOGITECH_MOTOR_CONTROL_V1,
+		.selector       = 2,
+		.size           = 1,
+		.offset         = 1,
+		.v4l2_type      = V4L2_CTRL_TYPE_BUTTON,
+		.data_type      = UVC_CTRL_DATA_TYPE_UNSIGNED,
+	},
+	{
+		.id             = V4L2_CID_PAN_RELATIVE,
+		.entity         = UVC_GUID_LOGITECH_PERIPHERAL,
+		.selector       = 1,
+		.size           = 16,
+		.offset         = 0,
+		.v4l2_type      = V4L2_CTRL_TYPE_INTEGER,
+		.data_type      = UVC_CTRL_DATA_TYPE_SIGNED,
+	},
+	{
+		.id             = V4L2_CID_TILT_RELATIVE,
+		.entity         = UVC_GUID_LOGITECH_PERIPHERAL,
+		.selector       = 1,
+		.size           = 16,
+		.offset         = 16,
+		.v4l2_type      = V4L2_CTRL_TYPE_INTEGER,
+		.data_type      = UVC_CTRL_DATA_TYPE_SIGNED,
+	},
+	{
+		.id             = V4L2_CID_PAN_RESET,
+		.entity         = UVC_GUID_LOGITECH_PERIPHERAL,
+		.selector       = 2,
+		.size           = 1,
+		.offset         = 0,
+		.v4l2_type      = V4L2_CTRL_TYPE_BUTTON,
+		.data_type      = UVC_CTRL_DATA_TYPE_UNSIGNED,
+	},
+	{
+		.id             = V4L2_CID_TILT_RESET,
+		.entity         = UVC_GUID_LOGITECH_PERIPHERAL,
+		.selector       = 2,
+		.size           = 1,
+		.offset         = 1,
+		.v4l2_type      = V4L2_CTRL_TYPE_BUTTON,
+		.data_type      = UVC_CTRL_DATA_TYPE_UNSIGNED,
+	},
+	{
+		.id             = V4L2_CID_FOCUS_ABSOLUTE,
+		.entity         = UVC_GUID_LOGITECH_MOTOR_CONTROL_V1,
+		.selector       = 3,
+		.size           = 8,
+		.offset         = 0,
+		.v4l2_type      = V4L2_CTRL_TYPE_INTEGER,
+		.data_type      = UVC_CTRL_DATA_TYPE_UNSIGNED,
+	},
 };
 
 /* ------------------------------------------------------------------------
@@ -2827,6 +3001,35 @@ static int uvc_ctrl_init_xu_ctrl(struct uvc_device *dev,
 	return ret;
 }
 
+bool uvc_ctrl_is_privacy_control(u8 entity[16], u8 selector)
+{
+	/*
+	 * This list is not exhaustive, it is a best effort to block access to
+	 * non documented controls that can affect user's privacy.
+	 */
+	struct privacy_control {
+		u8 entity[16];
+		u8 selector;
+	} privacy_control[] = {
+		{
+			.entity = UVC_GUID_LOGITECH_USER_HW_CONTROL_V1,
+			.selector = 1,
+		},
+		{
+			.entity = UVC_GUID_LOGITECH_PERIPHERAL,
+			.selector = 9,
+		},
+	};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(privacy_control); i++)
+		if (!memcmp(entity, privacy_control[i].entity, 16) &&
+		    selector == privacy_control[i].selector)
+			return true;
+
+	return false;
+}
+
 int uvc_xu_ctrl_query(struct uvc_video_chain *chain,
 	struct uvc_xu_control_query *xqry)
 {
@@ -2871,6 +3074,15 @@ int uvc_xu_ctrl_query(struct uvc_video_chain *chain,
 		return -ENOENT;
 	}
 
+	if (uvc_ctrl_is_privacy_control(entity->guid, xqry->selector) &&
+	    !uvc_allow_privacy_override_param) {
+		dev_warn_once(&chain->dev->intf->dev,
+			      "Privacy related controls can only be accessed if module parameter allow_privacy_override is true\n");
+		uvc_dbg(chain->dev, CONTROL, "Blocking access to privacy related Control %pUl/%u\n",
+			entity->guid, xqry->selector);
+		return -EACCES;
+	}
+
 	if (mutex_lock_interruptible(&chain->ctrl_mutex))
 		return -ERESTARTSYS;
 
diff --git a/drivers/media/usb/uvc/uvc_driver.c b/drivers/media/usb/uvc/uvc_driver.c
index 31b4ac3b48c1f..e289cc71ba986 100644
--- a/drivers/media/usb/uvc/uvc_driver.c
+++ b/drivers/media/usb/uvc/uvc_driver.c
@@ -36,6 +36,7 @@ unsigned int uvc_no_drop_param = 1;
 static unsigned int uvc_quirks_param = -1;
 unsigned int uvc_dbg_param;
 unsigned int uvc_timeout_param = UVC_CTRL_STREAMING_TIMEOUT;
+bool uvc_allow_privacy_override_param;
 
 static struct usb_driver uvc_driver;
 
@@ -2504,6 +2505,9 @@ module_param_named(trace, uvc_dbg_param, uint, 0644);
 MODULE_PARM_DESC(trace, "Trace level bitmask");
 module_param_named(timeout, uvc_timeout_param, uint, 0644);
 MODULE_PARM_DESC(timeout, "Streaming control requests timeout");
+module_param_named(allow_privacy_override, uvc_allow_privacy_override_param, bool, 0644);
+MODULE_PARM_DESC(allow_privacy_override,
+		 "Allow access to privacy related controls");
 
 /* ------------------------------------------------------------------------
  * Driver initialization and cleanup
diff --git a/drivers/media/usb/uvc/uvc_v4l2.c b/drivers/media/usb/uvc/uvc_v4l2.c
index cda1697204eaf..644c2f1cd8e63 100644
--- a/drivers/media/usb/uvc/uvc_v4l2.c
+++ b/drivers/media/usb/uvc/uvc_v4l2.c
@@ -133,6 +133,13 @@ static int uvc_ioctl_xu_ctrl_map(struct uvc_video_chain *chain,
 		return -EINVAL;
 	}
 
+	if (uvc_ctrl_is_privacy_control(xmap->entity, xmap->selector) &&
+	    !uvc_allow_privacy_override_param) {
+		dev_warn_once(&chain->dev->intf->dev,
+			      "Privacy related controls can only be mapped if module parameter allow_privacy_override is true\n");
+		return -EACCES;
+	}
+
 	map = kzalloc_obj(*map);
 	if (map == NULL)
 		return -ENOMEM;
@@ -1049,6 +1056,8 @@ static long uvc_ioctl_default(struct file *file, void *priv, bool valid_prio,
 	switch (cmd) {
 	/* Dynamic controls. */
 	case UVCIOC_CTRL_MAP:
+		pr_warn_once("uvcvideo: " DEPRECATED
+			     "UVCIOC_CTRL_MAP ioctl will be eventually removed.\n");
 		return uvc_ioctl_xu_ctrl_map(chain, arg);
 
 	case UVCIOC_CTRL_QUERY:
@@ -1163,6 +1172,8 @@ static long uvc_v4l2_compat_ioctl32(struct file *file,
 
 	switch (cmd) {
 	case UVCIOC_CTRL_MAP32:
+		pr_warn_once("uvcvideo: " DEPRECATED
+			     "UVCIOC_CTRL_MAP32 ioctl will be eventually removed.\n");
 		ret = uvc_v4l2_get_xu_mapping(&karg.xmap, up);
 		if (ret)
 			break;
diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c
index f6c8e32237968..62db4db4e565c 100644
--- a/drivers/media/usb/uvc/uvc_video.c
+++ b/drivers/media/usb/uvc/uvc_video.c
@@ -1168,6 +1168,53 @@ static int uvc_video_decode_start(struct uvc_streaming *stream,
 	header_len = data[0];
 	fid = data[1] & UVC_STREAM_FID;
 
+	/*
+	 * Mark the buffer as done if we're at the beginning of a new frame.
+	 * End of frame detection is better implemented by checking the EOF
+	 * bit (FID bit toggling is delayed by one frame compared to the EOF
+	 * bit), but some devices don't set the bit at end of frame (and the
+	 * last payload can be lost anyway). We thus must check if the FID has
+	 * been toggled.
+	 *
+	 * stream->last_fid is initialized to -1, and buf->bytesused to 0,
+	 * so the first isochronous frame will never trigger an end of frame
+	 * detection.
+	 *
+	 * Empty buffers (bytesused == 0) don't trigger end of frame detection
+	 * as it doesn't make sense to return an empty buffer. This also
+	 * avoids detecting end of frame conditions at FID toggling if the
+	 * previous payload had the EOF bit set.
+	 */
+	if (fid != stream->last_fid && buf && buf->bytesused != 0) {
+		uvc_dbg(stream->dev, FRAME,
+			"Frame complete (FID bit toggled)\n");
+		buf->state = UVC_BUF_STATE_READY;
+
+		return -EAGAIN;
+	}
+
+	/*
+	 * Some cameras, when running two parallel streams (one MJPEG alongside
+	 * another non-MJPEG stream), are known to lose the EOF packet for a frame.
+	 * We can detect the end of a frame by checking for a new SOI marker, as
+	 * the SOI always lies on the packet boundary between two frames for
+	 * these devices.
+	 */
+	if (stream->dev->quirks & UVC_QUIRK_MJPEG_NO_EOF &&
+	    (stream->cur_format->fcc == V4L2_PIX_FMT_MJPEG ||
+	     stream->cur_format->fcc == V4L2_PIX_FMT_JPEG) &&
+	    buf && buf->bytesused != 0) {
+		const u8 *packet = data + header_len;
+
+		if (len >= header_len + 2 &&
+		    packet[0] == 0xff && packet[1] == JPEG_MARKER_SOI) {
+			buf->state = UVC_BUF_STATE_READY;
+			buf->error = 1;
+			stream->last_fid ^= UVC_STREAM_FID;
+			return -EAGAIN;
+		}
+	}
+
 	/*
 	 * Increase the sequence number regardless of any buffer states, so
 	 * that discontinuous sequence numbers always indicate lost frames.
@@ -1176,6 +1223,19 @@ static int uvc_video_decode_start(struct uvc_streaming *stream,
 		stream->sequence++;
 		if (stream->sequence)
 			uvc_video_stats_update(stream);
+
+		/*
+		 * On a FID flip initialize sequence number and timestamp.
+		 *
+		 * The driver already takes care of injecting FID flips for
+		 * UVC_QUIRK_STREAM_NO_FID and UVC_QUIRK_MJPEG_NO_EOF.
+		 */
+		if (buf) {
+			buf->buf.field = V4L2_FIELD_NONE;
+			buf->buf.sequence = stream->sequence;
+			buf->buf.vb2_buf.timestamp =
+					ktime_to_ns(uvc_video_get_time());
+		}
 	}
 
 	uvc_video_clock_decode(stream, buf, data, len);
@@ -1216,59 +1276,10 @@ static int uvc_video_decode_start(struct uvc_streaming *stream,
 			return -ENODATA;
 		}
 
-		buf->buf.field = V4L2_FIELD_NONE;
-		buf->buf.sequence = stream->sequence;
-		buf->buf.vb2_buf.timestamp = ktime_to_ns(uvc_video_get_time());
-
 		/* TODO: Handle PTS and SCR. */
 		buf->state = UVC_BUF_STATE_ACTIVE;
 	}
 
-	/*
-	 * Mark the buffer as done if we're at the beginning of a new frame.
-	 * End of frame detection is better implemented by checking the EOF
-	 * bit (FID bit toggling is delayed by one frame compared to the EOF
-	 * bit), but some devices don't set the bit at end of frame (and the
-	 * last payload can be lost anyway). We thus must check if the FID has
-	 * been toggled.
-	 *
-	 * stream->last_fid is initialized to -1, so the first isochronous
-	 * frame will never trigger an end of frame detection.
-	 *
-	 * Empty buffers (bytesused == 0) don't trigger end of frame detection
-	 * as it doesn't make sense to return an empty buffer. This also
-	 * avoids detecting end of frame conditions at FID toggling if the
-	 * previous payload had the EOF bit set.
-	 */
-	if (fid != stream->last_fid && buf->bytesused != 0) {
-		uvc_dbg(stream->dev, FRAME,
-			"Frame complete (FID bit toggled)\n");
-		buf->state = UVC_BUF_STATE_READY;
-		return -EAGAIN;
-	}
-
-	/*
-	 * Some cameras, when running two parallel streams (one MJPEG alongside
-	 * another non-MJPEG stream), are known to lose the EOF packet for a frame.
-	 * We can detect the end of a frame by checking for a new SOI marker, as
-	 * the SOI always lies on the packet boundary between two frames for
-	 * these devices.
-	 */
-	if (stream->dev->quirks & UVC_QUIRK_MJPEG_NO_EOF &&
-	    (stream->cur_format->fcc == V4L2_PIX_FMT_MJPEG ||
-	    stream->cur_format->fcc == V4L2_PIX_FMT_JPEG)) {
-		const u8 *packet = data + header_len;
-
-		if (len >= header_len + 2 &&
-		    packet[0] == 0xff && packet[1] == JPEG_MARKER_SOI &&
-		    buf->bytesused != 0) {
-			buf->state = UVC_BUF_STATE_READY;
-			buf->error = 1;
-			stream->last_fid ^= UVC_STREAM_FID;
-			return -EAGAIN;
-		}
-	}
-
 	stream->last_fid = fid;
 
 	return header_len;
diff --git a/drivers/media/usb/uvc/uvcvideo.h b/drivers/media/usb/uvc/uvcvideo.h
index 0a0c01b2420fc..4ba35727e954e 100644
--- a/drivers/media/usb/uvc/uvcvideo.h
+++ b/drivers/media/usb/uvc/uvcvideo.h
@@ -666,6 +666,7 @@ extern unsigned int uvc_no_drop_param;
 extern unsigned int uvc_dbg_param;
 extern unsigned int uvc_timeout_param;
 extern unsigned int uvc_hw_timestamps_param;
+extern bool uvc_allow_privacy_override_param;
 
 #define uvc_dbg(_dev, flag, fmt, ...)					\
 do {									\
@@ -791,6 +792,7 @@ int uvc_xu_ctrl_query(struct uvc_video_chain *chain,
 		      struct uvc_xu_control_query *xqry);
 
 void uvc_ctrl_cleanup_fh(struct uvc_fh *handle);
+bool uvc_ctrl_is_privacy_control(u8 entity[16], u8 selector);
 
 /* Utility functions */
 struct usb_host_endpoint *uvc_find_endpoint(struct usb_host_interface *alts,
diff --git a/drivers/media/v4l2-core/tuner-core.c b/drivers/media/v4l2-core/tuner-core.c
index 004ec4d7beeae..1e130e6f903f9 100644
--- a/drivers/media/v4l2-core/tuner-core.c
+++ b/drivers/media/v4l2-core/tuner-core.c
@@ -1401,7 +1401,7 @@ static const struct dev_pm_ops tuner_pm_ops = {
 };
 
 static const struct i2c_device_id tuner_id[] = {
-	{ "tuner", }, /* autodetect */
+	{ .name = "tuner" }, /* autodetect */
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tuner_id);
diff --git a/drivers/media/v4l2-core/v4l2-common.c b/drivers/media/v4l2-core/v4l2-common.c
index 554c591e11133..bceafc4e92c81 100644
--- a/drivers/media/v4l2-core/v4l2-common.c
+++ b/drivers/media/v4l2-core/v4l2-common.c
@@ -281,6 +281,7 @@ const struct v4l2_format_info *v4l2_format_info(u32 format)
 		{ .format = V4L2_PIX_FMT_Y212,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_Y216,    .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 4, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 2, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_YUV48_12, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 6, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
+		{ .format = V4L2_PIX_FMT_YUV24,   .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 1, .comp_planes = 1, .bpp = { 3, 0, 0, 0 }, .bpp_div = { 1, 1, 1, 1 }, .hdiv = 1, .vdiv = 1 },
 		{ .format = V4L2_PIX_FMT_MT2110T, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 5, 10, 0, 0 }, .bpp_div = { 4, 4, 1, 1 }, .hdiv = 2, .vdiv = 2,
 		  .block_w = { 16, 8, 0, 0 }, .block_h = { 32, 16, 0, 0 }},
 		{ .format = V4L2_PIX_FMT_MT2110R, .pixel_enc = V4L2_PIXEL_ENC_YUV, .mem_planes = 2, .comp_planes = 2, .bpp = { 5, 10, 0, 0 }, .bpp_div = { 4, 4, 1, 1 }, .hdiv = 2, .vdiv = 2,
@@ -792,14 +793,15 @@ struct clk *__devm_v4l2_sensor_clk_get(struct device *dev, const char *id,
 	if (ret)
 		return ERR_PTR(ret == -EINVAL ? -EPROBE_DEFER : ret);
 
-	if (!id) {
+	if (id)
+		clk_id = kasprintf(GFP_KERNEL, "clk-%s-%s", dev_name(dev), id);
+	else
 		clk_id = kasprintf(GFP_KERNEL, "clk-%s", dev_name(dev));
-		if (!clk_id)
-			return ERR_PTR(-ENOMEM);
-		id = clk_id;
-	}
 
-	clk_hw = devm_clk_hw_register_fixed_rate(dev, id, NULL, 0, rate);
+	if (!clk_id)
+		return ERR_PTR(-ENOMEM);
+
+	clk_hw = devm_clk_hw_register_fixed_rate(dev, clk_id, NULL, 0, rate);
 	if (IS_ERR(clk_hw))
 		return ERR_CAST(clk_hw);
 
diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c
index 6b375720e395c..ba047d7d86010 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls-core.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c
@@ -971,6 +971,7 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx,
 	struct v4l2_ctrl_hevc_ext_sps_st_rps *p_hevc_st_rps;
 	struct v4l2_ctrl_hevc_sps *p_hevc_sps;
 	struct v4l2_ctrl_hevc_pps *p_hevc_pps;
+	struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params;
 	struct v4l2_ctrl_hdr10_mastering_display *p_hdr10_mastering;
 	struct v4l2_ctrl_hevc_decode_params *p_hevc_decode_params;
 	struct v4l2_area *area;
@@ -1260,6 +1261,18 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx,
 		break;
 
 	case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS:
+		p_hevc_slice_params = p;
+
+		if (p_hevc_slice_params->num_ref_idx_l0_active_minus1 >=
+		    V4L2_HEVC_DPB_ENTRIES_NUM_MAX)
+			return -EINVAL;
+
+		if (p_hevc_slice_params->slice_type != V4L2_HEVC_SLICE_TYPE_B)
+			break;
+
+		if (p_hevc_slice_params->num_ref_idx_l1_active_minus1 >=
+		    V4L2_HEVC_DPB_ENTRIES_NUM_MAX)
+			return -EINVAL;
 		break;
 
 	case V4L2_CTRL_TYPE_HEVC_EXT_SPS_ST_RPS:
diff --git a/drivers/media/v4l2-core/v4l2-ctrls-defs.c b/drivers/media/v4l2-core/v4l2-ctrls-defs.c
index 551426c4cd016..e062f20884904 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls-defs.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls-defs.c
@@ -889,6 +889,7 @@ const char *v4l2_ctrl_get_name(u32 id)
 	case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY:		return "Display Delay";
 	case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY_ENABLE:	return "Display Delay Enable";
 	case V4L2_CID_MPEG_VIDEO_AU_DELIMITER:			return "Generate Access Unit Delimiters";
+	case V4L2_CID_MPEG_VIDEO_BACKGROUND_DETECTION:		return "Background Detection";
 	case V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP:		return "H263 I-Frame QP Value";
 	case V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP:		return "H263 P-Frame QP Value";
 	case V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP:		return "H263 B-Frame QP Value";
@@ -1296,6 +1297,7 @@ void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type,
 	case V4L2_CID_MPEG_VIDEO_MPEG4_QPEL:
 	case V4L2_CID_MPEG_VIDEO_REPEAT_SEQ_HEADER:
 	case V4L2_CID_MPEG_VIDEO_AU_DELIMITER:
+	case V4L2_CID_MPEG_VIDEO_BACKGROUND_DETECTION:
 	case V4L2_CID_WIDE_DYNAMIC_RANGE:
 	case V4L2_CID_IMAGE_STABILIZATION:
 	case V4L2_CID_RDS_RECEPTION:
diff --git a/drivers/media/v4l2-core/v4l2-ctrls-request.c b/drivers/media/v4l2-core/v4l2-ctrls-request.c
index 4b7e6981b8d6f..5c2fc767c6a61 100644
--- a/drivers/media/v4l2-core/v4l2-ctrls-request.c
+++ b/drivers/media/v4l2-core/v4l2-ctrls-request.c
@@ -348,13 +348,12 @@ void v4l2_ctrl_request_complete(struct media_request *req,
 		ret = v4l2_ctrl_handler_init(hdl, (main_hdl->nr_of_buckets - 1) * 8);
 		if (!ret)
 			ret = v4l2_ctrl_request_bind(req, hdl, main_hdl);
-		if (ret) {
-			v4l2_ctrl_handler_free(hdl);
-			kfree(hdl);
-			return;
-		}
+		if (ret)
+			goto error;
 		hdl->request_is_queued = true;
 		obj = media_request_object_find(req, &req_ops, main_hdl);
+		if (!obj)
+			goto error;
 	}
 	hdl = container_of(obj, struct v4l2_ctrl_handler, req_obj);
 
@@ -389,6 +388,11 @@ void v4l2_ctrl_request_complete(struct media_request *req,
 	mutex_unlock(main_hdl->lock);
 	media_request_object_complete(obj);
 	media_request_object_put(obj);
+	return;
+
+error:
+	v4l2_ctrl_handler_free(hdl);
+	kfree(hdl);
 }
 EXPORT_SYMBOL(v4l2_ctrl_request_complete);
 
diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c
index 6ce623a1245a4..5516b2bbb08f0 100644
--- a/drivers/media/v4l2-core/v4l2-dev.c
+++ b/drivers/media/v4l2-core/v4l2-dev.c
@@ -1032,6 +1032,11 @@ int __video_register_device(struct video_device *vdev,
 	vdev->minor = i + minor_offset;
 	vdev->num = nr;
 
+	if (WARN_ON(vdev->minor >= VIDEO_NUM_DEVICES)) {
+		mutex_unlock(&videodev_lock);
+		return -EINVAL;
+	}
+
 	/* Should not happen since we thought this minor was free */
 	if (WARN_ON(video_devices[vdev->minor])) {
 		mutex_unlock(&videodev_lock);
diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c
index 831c69c958b8a..e9f81b9be9e24 100644
--- a/drivers/media/v4l2-core/v4l2-subdev.c
+++ b/drivers/media/v4l2-core/v4l2-subdev.c
@@ -629,6 +629,18 @@ subdev_ioctl_get_state(struct v4l2_subdev *sd, struct v4l2_subdev_fh *subdev_fh,
 			     v4l2_subdev_get_unlocked_active_state(sd);
 }
 
+static void v4l2_subdev_copy_routes(struct v4l2_subdev_routing *routing,
+				    const struct v4l2_subdev_state *state)
+{
+	struct v4l2_subdev_route *routes =
+		(struct v4l2_subdev_route *)(uintptr_t)routing->routes;
+	u32 copy_routes = min(routing->len_routes, state->routing.num_routes);
+
+	memcpy(routes, state->routing.routes, sizeof(*routes) * copy_routes);
+
+	routing->num_routes = state->routing.num_routes;
+}
+
 static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 			    struct v4l2_subdev_state *state)
 {
@@ -1000,7 +1012,6 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 
 	case VIDIOC_SUBDEV_G_ROUTING: {
 		struct v4l2_subdev_routing *routing = arg;
-		struct v4l2_subdev_krouting *krouting;
 
 		if (!v4l2_subdev_enable_streams_api)
 			return -ENOIOCTLCMD;
@@ -1008,15 +1019,12 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		if (!(sd->flags & V4L2_SUBDEV_FL_STREAMS))
 			return -ENOIOCTLCMD;
 
-		memset(routing->reserved, 0, sizeof(routing->reserved));
+		if (!client_supports_streams)
+			return -EINVAL;
 
-		krouting = &state->routing;
+		memset(routing->reserved, 0, sizeof(routing->reserved));
 
-		memcpy((struct v4l2_subdev_route *)(uintptr_t)routing->routes,
-		       krouting->routes,
-		       min(krouting->num_routes, routing->len_routes) *
-		       sizeof(*krouting->routes));
-		routing->num_routes = krouting->num_routes;
+		v4l2_subdev_copy_routes(routing, state);
 
 		return 0;
 	}
@@ -1035,6 +1043,9 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		if (!(sd->flags & V4L2_SUBDEV_FL_STREAMS))
 			return -ENOIOCTLCMD;
 
+		if (!client_supports_streams)
+			return -EINVAL;
+
 		if (routing->which != V4L2_SUBDEV_FORMAT_TRY && ro_subdev)
 			return -EPERM;
 
@@ -1084,11 +1095,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		 * the routing table.
 		 */
 		if (!v4l2_subdev_has_op(sd, pad, set_routing)) {
-			memcpy((struct v4l2_subdev_route *)(uintptr_t)routing->routes,
-			       state->routing.routes,
-			       min(state->routing.num_routes, routing->len_routes) *
-			       sizeof(*state->routing.routes));
-			routing->num_routes = state->routing.num_routes;
+			v4l2_subdev_copy_routes(routing, state);
 
 			return 0;
 		}
@@ -1102,11 +1109,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg,
 		if (rval < 0)
 			return rval;
 
-		memcpy((struct v4l2_subdev_route *)(uintptr_t)routing->routes,
-		       state->routing.routes,
-		       min(state->routing.num_routes, routing->len_routes) *
-		       sizeof(*state->routing.routes));
-		routing->num_routes = state->routing.num_routes;
+		v4l2_subdev_copy_routes(routing, state);
 
 		return 0;
 	}
@@ -2504,6 +2507,10 @@ int v4l2_subdev_s_stream_helper(struct v4l2_subdev *sd, int enable)
 	u64 source_mask = 0;
 	int pad_index = -1;
 
+	if (WARN_ON(!v4l2_subdev_has_op(sd, pad, enable_streams) ||
+		    !v4l2_subdev_has_op(sd, pad, disable_streams)))
+		return -ENOIOCTLCMD;
+
 	/*
 	 * Find the source pad. This helper is meant for subdevs that have a
 	 * single source pad, so failures shouldn't happen, but catch them
diff --git a/drivers/memory/atmel-ebi.c b/drivers/memory/atmel-ebi.c
index 8db970da9af96..1e8e8aba2542d 100644
--- a/drivers/memory/atmel-ebi.c
+++ b/drivers/memory/atmel-ebi.c
@@ -628,10 +628,11 @@ static __maybe_unused int atmel_ebi_resume(struct device *dev)
 static SIMPLE_DEV_PM_OPS(atmel_ebi_pm_ops, NULL, atmel_ebi_resume);
 
 static struct platform_driver atmel_ebi_driver = {
+	.probe = atmel_ebi_probe,
 	.driver = {
 		.name = "atmel-ebi",
 		.of_match_table	= atmel_ebi_id_table,
 		.pm = &atmel_ebi_pm_ops,
 	},
 };
-builtin_platform_driver_probe(atmel_ebi_driver, atmel_ebi_probe);
+builtin_platform_driver(atmel_ebi_driver);
diff --git a/drivers/memory/tegra/Kconfig b/drivers/memory/tegra/Kconfig
index fc5a277918267..cf0bfead8c456 100644
--- a/drivers/memory/tegra/Kconfig
+++ b/drivers/memory/tegra/Kconfig
@@ -17,6 +17,7 @@ config TEGRA20_EMC
 	select DEVFREQ_GOV_SIMPLE_ONDEMAND
 	select PM_DEVFREQ
 	select DDR
+	select TEGRA_EMC_COMMON
 	help
 	  This driver is for the External Memory Controller (EMC) found on
 	  Tegra20 chips. The EMC controls the external DRAM on the board.
@@ -29,18 +30,33 @@ config TEGRA30_EMC
 	depends on ARCH_TEGRA_3x_SOC || COMPILE_TEST
 	select PM_OPP
 	select DDR
+	select TEGRA_EMC_COMMON
 	help
 	  This driver is for the External Memory Controller (EMC) found on
 	  Tegra30 chips. The EMC controls the external DRAM on the board.
 	  This driver is required to change memory timings / clock rate for
 	  external memory.
 
+config TEGRA114_EMC
+	tristate "NVIDIA Tegra114 External Memory Controller driver"
+	default ARCH_TEGRA_114_SOC
+	depends on ARCH_TEGRA_114_SOC || COMPILE_TEST
+	select TEGRA124_CLK_EMC if ARCH_TEGRA
+	select PM_OPP
+	select TEGRA_EMC_COMMON
+	help
+	  This driver is for the External Memory Controller (EMC) found on
+	  Tegra114 chips. The EMC controls the external DRAM on the board.
+	  This driver is required to change memory timings / clock rate for
+	  external memory.
+
 config TEGRA124_EMC
 	tristate "NVIDIA Tegra124 External Memory Controller driver"
 	default ARCH_TEGRA_124_SOC
 	depends on ARCH_TEGRA_124_SOC || COMPILE_TEST
 	select TEGRA124_CLK_EMC if ARCH_TEGRA
 	select PM_OPP
+	select TEGRA_EMC_COMMON
 	help
 	  This driver is for the External Memory Controller (EMC) found on
 	  Tegra124 chips. The EMC controls the external DRAM on the board.
@@ -61,4 +77,7 @@ config TEGRA210_EMC
 	  This driver is required to change memory timings / clock rate for
 	  external memory.
 
+config TEGRA_EMC_COMMON
+	tristate
+
 endif
diff --git a/drivers/memory/tegra/Makefile b/drivers/memory/tegra/Makefile
index 6334601e6120d..908a5046454f6 100644
--- a/drivers/memory/tegra/Makefile
+++ b/drivers/memory/tegra/Makefile
@@ -10,12 +10,15 @@ tegra-mc-$(CONFIG_ARCH_TEGRA_210_SOC) += tegra210.o
 tegra-mc-$(CONFIG_ARCH_TEGRA_186_SOC) += tegra186.o
 tegra-mc-$(CONFIG_ARCH_TEGRA_194_SOC) += tegra186.o tegra194.o
 tegra-mc-$(CONFIG_ARCH_TEGRA_234_SOC) += tegra186.o tegra234.o
+tegra-mc-$(CONFIG_ARCH_TEGRA_238_SOC) += tegra186.o tegra234.o tegra238.o
 tegra-mc-$(CONFIG_ARCH_TEGRA_264_SOC) += tegra186.o tegra264.o
 
 obj-$(CONFIG_TEGRA_MC) += tegra-mc.o
 
+obj-$(CONFIG_TEGRA_EMC_COMMON) += tegra-emc-common.o
 obj-$(CONFIG_TEGRA20_EMC)  += tegra20-emc.o
 obj-$(CONFIG_TEGRA30_EMC)  += tegra30-emc.o
+obj-$(CONFIG_TEGRA114_EMC) += tegra114-emc.o
 obj-$(CONFIG_TEGRA124_EMC) += tegra124-emc.o
 obj-$(CONFIG_TEGRA210_EMC_TABLE) += tegra210-emc-table.o
 obj-$(CONFIG_TEGRA210_EMC) += tegra210-emc.o
diff --git a/drivers/memory/tegra/mc.c b/drivers/memory/tegra/mc.c
index d620660da3311..ec80ea9cc1736 100644
--- a/drivers/memory/tegra/mc.c
+++ b/drivers/memory/tegra/mc.c
@@ -13,6 +13,7 @@
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/pm.h>
 #include <linux/slab.h>
 #include <linux/sort.h>
 #include <linux/tegra-icc.h>
@@ -49,6 +50,9 @@ static const struct of_device_id tegra_mc_of_match[] = {
 #ifdef CONFIG_ARCH_TEGRA_234_SOC
 	{ .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc },
 #endif
+#ifdef CONFIG_ARCH_TEGRA_238_SOC
+	{ .compatible = "nvidia,tegra238-mc", .data = &tegra238_mc_soc },
+#endif
 #ifdef CONFIG_ARCH_TEGRA_264_SOC
 	{ .compatible = "nvidia,tegra264-mc", .data = &tegra264_mc_soc },
 #endif
@@ -910,6 +914,19 @@ static void tegra_mc_num_channel_enabled(struct tegra_mc *mc)
 	}
 }
 
+static void tegra_mc_setup_intmask(struct tegra_mc *mc)
+{
+	unsigned int i;
+
+	for (i = 0; i < mc->soc->num_intmasks; i++) {
+		if (mc->soc->num_channels)
+			mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmasks[i].mask,
+				     mc->soc->intmasks[i].reg);
+		else
+			mc_writel(mc, mc->soc->intmasks[i].mask, mc->soc->intmasks[i].reg);
+	}
+}
+
 static int tegra_mc_probe(struct platform_device *pdev)
 {
 	struct tegra_mc *mc;
@@ -970,13 +987,7 @@ static int tegra_mc_probe(struct platform_device *pdev)
 			}
 		}
 
-		for (i = 0; i < mc->soc->num_intmasks; i++) {
-			if (mc->soc->num_channels)
-				mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmasks[i].mask,
-					     mc->soc->intmasks[i].reg);
-			else
-				mc_writel(mc, mc->soc->intmasks[i].mask, mc->soc->intmasks[i].reg);
-		}
+		tegra_mc_setup_intmask(mc);
 	}
 
 	if (mc->soc->reset_ops) {
@@ -1010,10 +1021,25 @@ static void tegra_mc_sync_state(struct device *dev)
 		icc_sync_state(dev);
 }
 
+static int tegra_mc_resume(struct device *dev)
+{
+	struct tegra_mc *mc = dev_get_drvdata(dev);
+
+	if (mc->soc->ops && mc->soc->ops->resume)
+		mc->soc->ops->resume(mc);
+
+	tegra_mc_setup_intmask(mc);
+
+	return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(tegra_mc_pm_ops, NULL, tegra_mc_resume);
+
 static struct platform_driver tegra_mc_driver = {
 	.driver = {
 		.name = "tegra-mc",
 		.of_match_table = tegra_mc_of_match,
+		.pm = pm_sleep_ptr(&tegra_mc_pm_ops),
 		.suppress_bind_attrs = true,
 		.sync_state = tegra_mc_sync_state,
 	},
diff --git a/drivers/memory/tegra/mc.h b/drivers/memory/tegra/mc.h
index 649b54369263a..e94d265d7b678 100644
--- a/drivers/memory/tegra/mc.h
+++ b/drivers/memory/tegra/mc.h
@@ -240,6 +240,14 @@ extern const struct tegra_mc_soc tegra194_mc_soc;
 extern const struct tegra_mc_soc tegra234_mc_soc;
 #endif
 
+#if defined(CONFIG_ARCH_TEGRA_234_SOC) || defined(CONFIG_ARCH_TEGRA_238_SOC)
+extern const struct tegra_mc_icc_ops tegra234_mc_icc_ops;
+#endif
+
+#ifdef CONFIG_ARCH_TEGRA_238_SOC
+extern const struct tegra_mc_soc tegra238_mc_soc;
+#endif
+
 #ifdef CONFIG_ARCH_TEGRA_264_SOC
 extern const struct tegra_mc_soc tegra264_mc_soc;
 #endif
@@ -256,6 +264,7 @@ extern const struct tegra_mc_ops tegra30_mc_ops;
 #if defined(CONFIG_ARCH_TEGRA_186_SOC) || \
     defined(CONFIG_ARCH_TEGRA_194_SOC) || \
     defined(CONFIG_ARCH_TEGRA_234_SOC) || \
+    defined(CONFIG_ARCH_TEGRA_238_SOC) || \
     defined(CONFIG_ARCH_TEGRA_264_SOC)
 extern const struct tegra_mc_ops tegra186_mc_ops;
 #endif
diff --git a/drivers/memory/tegra/tegra-emc-common.c b/drivers/memory/tegra/tegra-emc-common.c
new file mode 100644
index 0000000000000..bbe86cb4942c5
--- /dev/null
+++ b/drivers/memory/tegra/tegra-emc-common.c
@@ -0,0 +1,143 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/device.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/pm_opp.h>
+
+#include "tegra-emc-common.h"
+
+/**
+ * tegra_emc_rate_requests_init() - Initialize EMC rate request tracking
+ * @reqs: struct tegra_emc_rate_requests to initialize.
+ * @dev: EMC device.
+ *
+ * Initializes the rate request tracking state with default state
+ * (no active requests). Must be called before using @reqs with
+ * other functions.
+ */
+void tegra_emc_rate_requests_init(struct tegra_emc_rate_requests *reqs,
+				  struct device *dev)
+{
+	unsigned int i;
+
+	mutex_init(&reqs->rate_lock);
+	reqs->dev = dev;
+
+	for (i = 0; i < TEGRA_EMC_RATE_TYPE_MAX; i++) {
+		reqs->requested_rate[i].min_rate = 0;
+		reqs->requested_rate[i].max_rate = ULONG_MAX;
+	}
+}
+EXPORT_SYMBOL_GPL(tegra_emc_rate_requests_init);
+
+/* Caller must hold reqs->rate_lock. */
+static int tegra_emc_request_rate(struct tegra_emc_rate_requests *reqs,
+				  unsigned long new_min_rate,
+				  unsigned long new_max_rate,
+				  enum tegra_emc_rate_request_type type)
+{
+	struct tegra_emc_rate_request *req = reqs->requested_rate;
+	unsigned long min_rate = 0, max_rate = ULONG_MAX;
+	unsigned int i;
+	int err;
+
+	lockdep_assert_held(&reqs->rate_lock);
+
+	/* select minimum and maximum rates among the requested rates */
+	for (i = 0; i < TEGRA_EMC_RATE_TYPE_MAX; i++, req++) {
+		if (i == type) {
+			min_rate = max(new_min_rate, min_rate);
+			max_rate = min(new_max_rate, max_rate);
+		} else {
+			min_rate = max(req->min_rate, min_rate);
+			max_rate = min(req->max_rate, max_rate);
+		}
+	}
+
+	if (min_rate > max_rate) {
+		dev_err_ratelimited(reqs->dev, "%s: type %u: out of range: %lu %lu\n",
+				    __func__, type, min_rate, max_rate);
+		return -ERANGE;
+	}
+
+	/*
+	 * EMC rate-changes should go via OPP API because it manages voltage
+	 * changes.
+	 */
+	err = dev_pm_opp_set_rate(reqs->dev, min_rate);
+	if (err)
+		return err;
+
+	reqs->requested_rate[type].min_rate = new_min_rate;
+	reqs->requested_rate[type].max_rate = new_max_rate;
+
+	return 0;
+}
+
+/**
+ * tegra_emc_set_min_rate() - Update minimum rate request for a request type
+ * @reqs: rate request tracking state
+ * @rate: new minimum rate in Hz requested by @type
+ * @type: type of request
+ *
+ * Records @rate as the new minimum rate request for @type, recalculates target
+ * rate based on all requests and applies new rate through the OPP API.
+ *
+ * Context: Sleeps. Requests to same @reqs are synchronized via mutex.
+ *
+ * Return:
+ * * %0 - success
+ * * %-ERANGE - request would cause minimum rate request to be higher than
+ *              maximum rate request
+ * * other - setting new rate failed
+ */
+int tegra_emc_set_min_rate(struct tegra_emc_rate_requests *reqs,
+			   unsigned long rate,
+			   enum tegra_emc_rate_request_type type)
+{
+	struct tegra_emc_rate_request *req = &reqs->requested_rate[type];
+	int ret;
+
+	mutex_lock(&reqs->rate_lock);
+	ret = tegra_emc_request_rate(reqs, rate, req->max_rate, type);
+	mutex_unlock(&reqs->rate_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(tegra_emc_set_min_rate);
+
+/**
+ * tegra_emc_set_max_rate() - Update maximum rate request for a request type
+ * @reqs: rate request tracking state
+ * @rate: new maximum rate in Hz requested by @type
+ * @type: type of request
+ *
+ * Records @rate as the new maximum rate request for @type, recalculates target
+ * rate based on all requests and applies new rate through the OPP API.
+ *
+ * Context: Sleeps. Requests to same @reqs are synchronized via mutex.
+ *
+ * Return:
+ * * %0 - success
+ * * %-ERANGE - request would cause minimum rate request to be higher than
+ *              maximum rate request
+ * * other - setting new rate failed
+ */
+int tegra_emc_set_max_rate(struct tegra_emc_rate_requests *reqs,
+			   unsigned long rate,
+			   enum tegra_emc_rate_request_type type)
+{
+	struct tegra_emc_rate_request *req = &reqs->requested_rate[type];
+	int ret;
+
+	mutex_lock(&reqs->rate_lock);
+	ret = tegra_emc_request_rate(reqs, req->min_rate, rate, type);
+	mutex_unlock(&reqs->rate_lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(tegra_emc_set_max_rate);
+
+MODULE_DESCRIPTION("NVIDIA Tegra EMC common code");
+MODULE_LICENSE("GPL");
diff --git a/drivers/memory/tegra/tegra-emc-common.h b/drivers/memory/tegra/tegra-emc-common.h
new file mode 100644
index 0000000000000..157e27024342b
--- /dev/null
+++ b/drivers/memory/tegra/tegra-emc-common.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef TEGRA_EMC_COMMON_H
+#define TEGRA_EMC_COMMON_H
+
+#include <linux/device.h>
+#include <linux/mutex.h>
+
+/**
+ * enum tegra_emc_rate_request_type - source of rate request
+ * @TEGRA_EMC_RATE_DEVFREQ: rate requested by devfreq governor
+ * @TEGRA_EMC_RATE_DEBUG: rate requested through debugfs knobs
+ * @TEGRA_EMC_RATE_ICC: rate requested by ICC framework
+ * @TEGRA_EMC_RATE_TYPE_MAX: number of valid request types
+ */
+enum tegra_emc_rate_request_type {
+	TEGRA_EMC_RATE_DEVFREQ,
+	TEGRA_EMC_RATE_DEBUG,
+	TEGRA_EMC_RATE_ICC,
+	TEGRA_EMC_RATE_TYPE_MAX,
+};
+
+struct tegra_emc_rate_request {
+	unsigned long min_rate;
+	unsigned long max_rate;
+};
+
+struct tegra_emc_rate_requests {
+	struct tegra_emc_rate_request requested_rate[TEGRA_EMC_RATE_TYPE_MAX];
+	/* Protects @requested_rate. */
+	struct mutex rate_lock;
+	struct device *dev;
+};
+
+void tegra_emc_rate_requests_init(struct tegra_emc_rate_requests *reqs,
+				  struct device *dev);
+
+int tegra_emc_set_min_rate(struct tegra_emc_rate_requests *reqs,
+			   unsigned long rate,
+			   enum tegra_emc_rate_request_type type);
+
+int tegra_emc_set_max_rate(struct tegra_emc_rate_requests *reqs,
+			   unsigned long rate,
+			   enum tegra_emc_rate_request_type type);
+
+#endif /* TEGRA_EMC_COMMON_H */
diff --git a/drivers/memory/tegra/tegra114-emc.c b/drivers/memory/tegra/tegra114-emc.c
new file mode 100644
index 0000000000000..e50d71842d3aa
--- /dev/null
+++ b/drivers/memory/tegra/tegra114-emc.c
@@ -0,0 +1,1351 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Tegra114 External Memory Controller driver
+ *
+ * Based on downstream driver from NVIDIA and tegra124-emc.c
+ * Copyright (C) 2011-2014 NVIDIA Corporation
+ *
+ * Copyright (C) 2024 Svyatoslav Ryhel <clamor95@gmail.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/clk/tegra.h>
+#include <linux/debugfs.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/pm_opp.h>
+#include <linux/sort.h>
+#include <linux/string.h>
+
+#include <soc/tegra/common.h>
+#include <soc/tegra/fuse.h>
+#include <soc/tegra/mc.h>
+
+#include "mc.h"
+#include "tegra-emc-common.h"
+
+#define EMC_INTSTATUS				0x0
+#define EMC_INTSTATUS_REFRESH_OVERFLOW		BIT(3)
+#define EMC_INTSTATUS_CLKCHANGE_COMPLETE	BIT(4)
+
+#define EMC_INTMASK				0x4
+
+#define EMC_DBG					0x8
+#define EMC_DBG_READ_MUX_ASSEMBLY		BIT(0)
+#define EMC_DBG_WRITE_MUX_ACTIVE		BIT(1)
+#define EMC_DBG_FORCE_UPDATE			BIT(2)
+#define EMC_DBG_CFG_PRIORITY			BIT(24)
+
+#define EMC_CFG					0xc
+#define EMC_CFG_DSR_VTTGEN_DRV_EN		BIT(18)
+#define EMC_CFG_PWR_MASK			((0xF << 28) | BIT(18))
+#define EMC_CFG_DYN_SREF			BIT(28)
+#define EMC_CFG_DRAM_ACPD			BIT(29)
+#define EMC_CFG_DRAM_CLKSTOP_SR			BIT(30)
+#define EMC_CFG_DRAM_CLKSTOP_PD			BIT(31)
+
+#define EMC_ADR_CFG				0x10
+#define EMC_ADR_CFG_EMEM_NUMDEV			BIT(0)
+
+#define EMC_REFCTRL				0x20
+#define EMC_REFCTRL_DEV_SEL_SHIFT		0
+#define EMC_REFCTRL_ENABLE			BIT(31)
+
+#define EMC_TIMING_CONTROL			0x28
+#define EMC_RC					0x2c
+#define EMC_RFC					0x30
+#define EMC_RAS					0x34
+#define EMC_RP					0x38
+#define EMC_R2W					0x3c
+#define EMC_W2R					0x40
+#define EMC_R2P					0x44
+#define EMC_W2P					0x48
+#define EMC_RD_RCD				0x4c
+#define EMC_WR_RCD				0x50
+#define EMC_RRD					0x54
+#define EMC_REXT				0x58
+#define EMC_WDV					0x5c
+#define EMC_QUSE				0x60
+#define EMC_QRST				0x64
+#define EMC_QSAFE				0x68
+#define EMC_RDV					0x6c
+#define EMC_REFRESH				0x70
+#define EMC_BURST_REFRESH_NUM			0x74
+#define EMC_PDEX2WR				0x78
+#define EMC_PDEX2RD				0x7c
+#define EMC_PCHG2PDEN				0x80
+#define EMC_ACT2PDEN				0x84
+#define EMC_AR2PDEN				0x88
+#define EMC_RW2PDEN				0x8c
+#define EMC_TXSR				0x90
+#define EMC_TCKE				0x94
+#define EMC_TFAW				0x98
+#define EMC_TRPAB				0x9c
+#define EMC_TCLKSTABLE				0xa0
+#define EMC_TCLKSTOP				0xa4
+#define EMC_TREFBW				0xa8
+#define EMC_QUSE_EXTRA				0xac
+#define EMC_ODT_WRITE				0xb0
+#define EMC_ODT_READ				0xb4
+#define EMC_WEXT				0xb8
+#define EMC_CTT					0xbc
+#define EMC_RFC_SLR				0xc0
+#define EMC_MRS_WAIT_CNT2			0xc4
+
+#define EMC_MRS_WAIT_CNT			0xc8
+#define EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT	0
+#define EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK	\
+	(0x3FF << EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)
+#define EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT	16
+#define EMC_MRS_WAIT_CNT_LONG_WAIT_MASK		\
+	(0x3FF << EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT)
+
+#define EMC_MRS					0xcc
+#define EMC_MODE_SET_DLL_RESET			BIT(8)
+#define EMC_MODE_SET_LONG_CNT			BIT(26)
+#define EMC_EMRS				0xd0
+#define EMC_REF					0xd4
+#define EMC_PRE					0xd8
+
+#define EMC_SELF_REF				0xe0
+#define EMC_SELF_REF_CMD_ENABLED		BIT(0)
+#define EMC_SELF_REF_DEV_SEL_SHIFT		30
+
+#define EMC_MRW					0xe8
+
+#define EMC_MRR					0xec
+#define EMC_MRR_MA_SHIFT			16
+#define LPDDR2_MR4_TEMP_SHIFT			0
+
+#define EMC_XM2DQSPADCTRL3			0xf8
+#define EMC_FBIO_SPARE				0x100
+
+#define EMC_FBIO_CFG5				0x104
+#define EMC_FBIO_CFG5_DRAM_TYPE_MASK		0x3
+#define EMC_FBIO_CFG5_DRAM_TYPE_SHIFT		0
+
+#define EMC_FBIO_CFG6				0x114
+#define EMC_EMRS2				0x12c
+#define EMC_MRW2				0x134
+#define EMC_MRW4				0x13c
+#define EMC_EINPUT				0x14c
+#define EMC_EINPUT_DURATION			0x150
+#define EMC_PUTERM_EXTRA			0x154
+#define EMC_TCKESR				0x158
+#define EMC_TPD					0x15c
+
+#define EMC_AUTO_CAL_CONFIG			0x2a4
+#define EMC_AUTO_CAL_CONFIG_AUTO_CAL_START	BIT(31)
+#define EMC_AUTO_CAL_INTERVAL			0x2a8
+#define EMC_AUTO_CAL_STATUS			0x2ac
+#define EMC_AUTO_CAL_STATUS_ACTIVE		BIT(31)
+#define EMC_STATUS				0x2b4
+#define EMC_STATUS_TIMING_UPDATE_STALLED	BIT(23)
+
+#define EMC_CFG_2				0x2b8
+#define EMC_CFG_2_CLKCHANGE_REQ_ENABLE		BIT(0)
+#define EMC_CFG_2_CLKCHANGE_PD_ENABLE		BIT(1)
+#define EMC_CFG_2_CLKCHANGE_SR_ENABLE		BIT(2)
+
+#define EMC_CFG_DIG_DLL				0x2bc
+#define EMC_CFG_DIG_DLL_PERIOD			0x2c0
+#define EMC_RDV_MASK				0x2cc
+#define EMC_WDV_MASK				0x2d0
+#define EMC_CTT_DURATION			0x2d8
+#define EMC_CTT_TERM_CTRL			0x2dc
+#define EMC_ZCAL_INTERVAL			0x2e0
+#define EMC_ZCAL_WAIT_CNT			0x2e4
+
+#define EMC_ZQ_CAL				0x2ec
+#define EMC_ZQ_CAL_CMD				BIT(0)
+#define EMC_ZQ_CAL_LONG				BIT(4)
+#define EMC_ZQ_CAL_LONG_CMD_DEV0		\
+	(DRAM_DEV_SEL_0 | EMC_ZQ_CAL_LONG | EMC_ZQ_CAL_CMD)
+#define EMC_ZQ_CAL_LONG_CMD_DEV1		\
+	(DRAM_DEV_SEL_1 | EMC_ZQ_CAL_LONG | EMC_ZQ_CAL_CMD)
+
+#define EMC_XM2CMDPADCTRL			0x2f0
+#define EMC_XM2DQSPADCTRL			0x2f8
+#define EMC_XM2DQSPADCTRL2			0x2fc
+#define EMC_XM2DQSPADCTRL2_RX_FT_REC_ENABLE	BIT(0)
+#define EMC_XM2DQSPADCTRL2_VREF_ENABLE		BIT(5)
+#define EMC_XM2DQPADCTRL			0x300
+#define EMC_XM2DQPADCTRL2			0x304
+#define EMC_XM2CLKPADCTRL			0x308
+#define EMC_XM2COMPPADCTRL			0x30c
+#define EMC_XM2VTTGENPADCTRL			0x310
+#define EMC_XM2VTTGENPADCTRL2			0x314
+#define EMC_XM2QUSEPADCTRL			0x318
+#define EMC_XM2DQSPADCTRL4			0x320
+#define EMC_DLL_XFORM_DQS0			0x328
+#define EMC_DLL_XFORM_DQS1			0x32c
+#define EMC_DLL_XFORM_DQS2			0x330
+#define EMC_DLL_XFORM_DQS3			0x334
+#define EMC_DLL_XFORM_DQS4			0x338
+#define EMC_DLL_XFORM_DQS5			0x33c
+#define EMC_DLL_XFORM_DQS6			0x340
+#define EMC_DLL_XFORM_DQS7			0x344
+#define EMC_DLL_XFORM_QUSE0			0x348
+#define EMC_DLL_XFORM_QUSE1			0x34c
+#define EMC_DLL_XFORM_QUSE2			0x350
+#define EMC_DLL_XFORM_QUSE3			0x354
+#define EMC_DLL_XFORM_QUSE4			0x358
+#define EMC_DLL_XFORM_QUSE5			0x35c
+#define EMC_DLL_XFORM_QUSE6			0x360
+#define EMC_DLL_XFORM_QUSE7			0x364
+#define EMC_DLL_XFORM_DQ0			0x368
+#define EMC_DLL_XFORM_DQ1			0x36c
+#define EMC_DLL_XFORM_DQ2			0x370
+#define EMC_DLL_XFORM_DQ3			0x374
+#define EMC_DLI_TRIM_TXDQS0			0x3a8
+#define EMC_DLI_TRIM_TXDQS1			0x3ac
+#define EMC_DLI_TRIM_TXDQS2			0x3b0
+#define EMC_DLI_TRIM_TXDQS3			0x3b4
+#define EMC_DLI_TRIM_TXDQS4			0x3b8
+#define EMC_DLI_TRIM_TXDQS5			0x3bc
+#define EMC_DLI_TRIM_TXDQS6			0x3c0
+#define EMC_DLI_TRIM_TXDQS7			0x3c4
+#define EMC_STALL_THEN_EXE_AFTER_CLKCHANGE	0x3cc
+#define EMC_SEL_DPD_CTRL			0x3d8
+#define EMC_SEL_DPD_CTRL_DATA_SEL_DPD		BIT(8)
+#define EMC_SEL_DPD_CTRL_ODT_SEL_DPD		BIT(5)
+#define EMC_SEL_DPD_CTRL_RESET_SEL_DPD		BIT(4)
+#define EMC_SEL_DPD_CTRL_CA_SEL_DPD		BIT(3)
+#define EMC_SEL_DPD_CTRL_CLK_SEL_DPD		BIT(2)
+#define EMC_SEL_DPD_CTRL_DDR3_MASK	\
+	((0xf << 2) | BIT(8))
+#define EMC_SEL_DPD_CTRL_MASK \
+	((0x3 << 2) | BIT(5) | BIT(8))
+#define EMC_PRE_REFRESH_REQ_CNT			0x3dc
+#define EMC_DYN_SELF_REF_CONTROL		0x3e0
+#define EMC_TXSRDLL				0x3e4
+#define EMC_CCFIFO_ADDR				0x3e8
+#define EMC_CCFIFO_DATA				0x3ec
+#define EMC_CCFIFO_STATUS			0x3f0
+#define EMC_CDB_CNTL_1				0x3f4
+#define EMC_CDB_CNTL_2				0x3f8
+#define EMC_XM2CLKPADCTRL2			0x3fc
+#define EMC_AUTO_CAL_CONFIG2			0x458
+#define EMC_AUTO_CAL_CONFIG3			0x45c
+#define EMC_IBDLY				0x468
+#define EMC_DLL_XFORM_ADDR0			0x46c
+#define EMC_DLL_XFORM_ADDR1			0x470
+#define EMC_DLL_XFORM_ADDR2			0x474
+#define EMC_DSR_VTTGEN_DRV			0x47c
+#define EMC_TXDSRVTTGEN				0x480
+#define EMC_XM2CMDPADCTRL4			0x484
+
+#define DRAM_DEV_SEL_ALL			0
+#define DRAM_DEV_SEL_0				BIT(31)
+#define DRAM_DEV_SEL_1				BIT(30)
+
+#define EMC_CFG_POWER_FEATURES_MASK		\
+	(EMC_CFG_DYN_SREF | EMC_CFG_DRAM_ACPD | EMC_CFG_DRAM_CLKSTOP_SR | \
+	EMC_CFG_DRAM_CLKSTOP_PD | EMC_CFG_DSR_VTTGEN_DRV_EN)
+#define EMC_REFCTRL_DEV_SEL(n) ((((n) > 1) ? 0 : 2) << EMC_REFCTRL_DEV_SEL_SHIFT)
+#define EMC_DRAM_DEV_SEL(n) (((n) > 1) ? DRAM_DEV_SEL_ALL : DRAM_DEV_SEL_0)
+
+/* Maximum amount of time in us. to wait for changes to become effective */
+#define EMC_STATUS_UPDATE_TIMEOUT		1000
+
+enum emc_dram_type {
+	DRAM_TYPE_DDR3,
+	DRAM_TYPE_DDR1,
+	DRAM_TYPE_LPDDR2,
+	DRAM_TYPE_DDR2
+};
+
+enum emc_dll_change {
+	DLL_CHANGE_NONE,
+	DLL_CHANGE_ON,
+	DLL_CHANGE_OFF
+};
+
+static const unsigned long emc_burst_regs[] = {
+	EMC_RC,
+	EMC_RFC,
+	EMC_RAS,
+	EMC_RP,
+	EMC_R2W,
+	EMC_W2R,
+	EMC_R2P,
+	EMC_W2P,
+	EMC_RD_RCD,
+	EMC_WR_RCD,
+	EMC_RRD,
+	EMC_REXT,
+	EMC_WEXT,
+	EMC_WDV,
+	EMC_WDV_MASK,
+	EMC_QUSE,
+	EMC_IBDLY,
+	EMC_EINPUT,
+	EMC_EINPUT_DURATION,
+	EMC_PUTERM_EXTRA,
+	EMC_CDB_CNTL_1,
+	EMC_CDB_CNTL_2,
+	EMC_QRST,
+	EMC_QSAFE,
+	EMC_RDV,
+	EMC_RDV_MASK,
+	EMC_REFRESH,
+	EMC_BURST_REFRESH_NUM,
+	EMC_PRE_REFRESH_REQ_CNT,
+	EMC_PDEX2WR,
+	EMC_PDEX2RD,
+	EMC_PCHG2PDEN,
+	EMC_ACT2PDEN,
+	EMC_AR2PDEN,
+	EMC_RW2PDEN,
+	EMC_TXSR,
+	EMC_TXSRDLL,
+	EMC_TCKE,
+	EMC_TCKESR,
+	EMC_TPD,
+	EMC_TFAW,
+	EMC_TRPAB,
+	EMC_TCLKSTABLE,
+	EMC_TCLKSTOP,
+	EMC_TREFBW,
+	EMC_QUSE_EXTRA,
+	EMC_FBIO_CFG6,
+	EMC_ODT_WRITE,
+	EMC_ODT_READ,
+	EMC_FBIO_CFG5,
+	EMC_CFG_DIG_DLL,
+	EMC_CFG_DIG_DLL_PERIOD,
+	EMC_DLL_XFORM_DQS0,
+	EMC_DLL_XFORM_DQS1,
+	EMC_DLL_XFORM_DQS2,
+	EMC_DLL_XFORM_DQS3,
+	EMC_DLL_XFORM_DQS4,
+	EMC_DLL_XFORM_DQS5,
+	EMC_DLL_XFORM_DQS6,
+	EMC_DLL_XFORM_DQS7,
+	EMC_DLL_XFORM_QUSE0,
+	EMC_DLL_XFORM_QUSE1,
+	EMC_DLL_XFORM_QUSE2,
+	EMC_DLL_XFORM_QUSE3,
+	EMC_DLL_XFORM_QUSE4,
+	EMC_DLL_XFORM_QUSE5,
+	EMC_DLL_XFORM_QUSE6,
+	EMC_DLL_XFORM_QUSE7,
+	EMC_DLI_TRIM_TXDQS0,
+	EMC_DLI_TRIM_TXDQS1,
+	EMC_DLI_TRIM_TXDQS2,
+	EMC_DLI_TRIM_TXDQS3,
+	EMC_DLI_TRIM_TXDQS4,
+	EMC_DLI_TRIM_TXDQS5,
+	EMC_DLI_TRIM_TXDQS6,
+	EMC_DLI_TRIM_TXDQS7,
+	EMC_DLL_XFORM_DQ0,
+	EMC_DLL_XFORM_DQ1,
+	EMC_DLL_XFORM_DQ2,
+	EMC_DLL_XFORM_DQ3,
+	EMC_XM2CMDPADCTRL,
+	EMC_XM2CMDPADCTRL4,
+	EMC_XM2DQPADCTRL2,
+	EMC_XM2CLKPADCTRL,
+	EMC_XM2COMPPADCTRL,
+	EMC_XM2VTTGENPADCTRL,
+	EMC_XM2VTTGENPADCTRL2,
+	EMC_XM2DQSPADCTRL3,
+	EMC_XM2DQSPADCTRL4,
+	EMC_DSR_VTTGEN_DRV,
+	EMC_TXDSRVTTGEN,
+	EMC_FBIO_SPARE,
+	EMC_ZCAL_WAIT_CNT,
+	EMC_MRS_WAIT_CNT2,
+	EMC_CTT,
+	EMC_CTT_DURATION,
+	EMC_DYN_SELF_REF_CONTROL,
+};
+
+struct emc_timing {
+	unsigned long rate;
+
+	u32 emc_burst_data[ARRAY_SIZE(emc_burst_regs)];
+
+	u32 emc_auto_cal_config;
+	u32 emc_auto_cal_config2;
+	u32 emc_auto_cal_config3;
+	u32 emc_auto_cal_interval;
+	u32 emc_cfg;
+	u32 emc_ctt_term_ctrl;
+	u32 emc_mode_1;
+	u32 emc_mode_2;
+	u32 emc_mode_4;
+	u32 emc_mode_reset;
+	u32 emc_mrs_wait_cnt;
+	u32 emc_sel_dpd_ctrl;
+	u32 emc_xm2dqspadctrl2;
+	u32 emc_zcal_cnt_long;
+	u32 emc_zcal_interval;
+};
+
+struct tegra_emc {
+	struct device *dev;
+	struct tegra_mc *mc;
+	void __iomem *regs;
+	unsigned int irq;
+	struct clk *clk;
+
+	enum emc_dram_type dram_type;
+	unsigned int dram_num;
+
+	struct emc_timing last_timing;
+	struct emc_timing *timings;
+	unsigned int num_timings;
+
+	struct {
+		struct dentry *root;
+		unsigned long min_rate;
+		unsigned long max_rate;
+	} debugfs;
+
+	struct icc_provider provider;
+	struct tegra_emc_rate_requests reqs;
+};
+
+static irqreturn_t tegra114_emc_isr(int irq, void *data)
+{
+	struct tegra_emc *emc = data;
+	u32 intmask = EMC_INTSTATUS_REFRESH_OVERFLOW;
+	u32 status;
+
+	status = readl_relaxed(emc->regs + EMC_INTSTATUS) & intmask;
+	if (!status)
+		return IRQ_NONE;
+
+	/* notify about HW problem */
+	if (status & EMC_INTSTATUS_REFRESH_OVERFLOW)
+		dev_err_ratelimited(emc->dev,
+				    "refresh request overflow timeout\n");
+
+	/* clear interrupts */
+	writel_relaxed(status, emc->regs + EMC_INTSTATUS);
+
+	return IRQ_HANDLED;
+}
+
+/* Timing change sequence functions */
+
+static void emc_ccfifo_writel(struct tegra_emc *emc, u32 value,
+			      unsigned long offset)
+{
+	writel(value, emc->regs + EMC_CCFIFO_DATA);
+	writel(offset, emc->regs + EMC_CCFIFO_ADDR);
+}
+
+static void emc_seq_update_timing(struct tegra_emc *emc)
+{
+	int ret;
+	u32 value;
+
+	writel(1, emc->regs + EMC_TIMING_CONTROL);
+
+	ret = readl_poll_timeout_atomic(emc->regs + EMC_STATUS, value,
+					!(value & EMC_STATUS_TIMING_UPDATE_STALLED),
+					1, EMC_STATUS_UPDATE_TIMEOUT);
+	if (ret)
+		dev_err(emc->dev, "timing update timed out\n");
+}
+
+static void emc_seq_disable_auto_cal(struct tegra_emc *emc)
+{
+	int ret;
+	u32 value;
+
+	writel(0, emc->regs + EMC_AUTO_CAL_INTERVAL);
+
+	ret = readl_poll_timeout_atomic(emc->regs + EMC_AUTO_CAL_STATUS, value,
+					!(value & EMC_AUTO_CAL_STATUS_ACTIVE),
+					1, EMC_STATUS_UPDATE_TIMEOUT);
+	if (ret)
+		dev_err(emc->dev, "auto cal disable timed out\n");
+}
+
+static void emc_seq_wait_clkchange(struct tegra_emc *emc)
+{
+	int ret;
+	u32 value;
+
+	ret = readl_poll_timeout_atomic(emc->regs + EMC_INTSTATUS, value,
+					value & EMC_INTSTATUS_CLKCHANGE_COMPLETE,
+					1, EMC_STATUS_UPDATE_TIMEOUT);
+	if (ret)
+		dev_err(emc->dev, "clock change timed out\n");
+}
+
+static struct emc_timing *tegra114_emc_find_timing(struct tegra_emc *emc,
+						   unsigned long rate)
+{
+	struct emc_timing *timing = NULL;
+	unsigned int i;
+
+	for (i = 0; i < emc->num_timings; i++) {
+		if (emc->timings[i].rate == rate) {
+			timing = &emc->timings[i];
+			break;
+		}
+	}
+
+	if (!timing) {
+		dev_err(emc->dev, "no timing for rate %lu\n", rate);
+		return NULL;
+	}
+
+	return timing;
+}
+
+static int tegra114_emc_prepare_timing_change(struct tegra_emc *emc,
+					      unsigned long rate)
+{
+	struct emc_timing *timing = tegra114_emc_find_timing(emc, rate);
+	struct emc_timing *last = &emc->last_timing;
+	enum emc_dll_change dll_change;
+	unsigned int pre_wait = 0;
+	u32 val, mask;
+	bool last_dll_enabled = !(last->emc_mode_1 & 0x1);
+	bool update = false;
+	bool next_dll_enabled;
+	unsigned int i;
+
+	if (!timing)
+		return -ENOENT;
+
+	next_dll_enabled = !(timing->emc_mode_1 & 0x1);
+
+	if (next_dll_enabled == last_dll_enabled)
+		dll_change = DLL_CHANGE_NONE;
+	else if (next_dll_enabled)
+		dll_change = DLL_CHANGE_ON;
+	else
+		dll_change = DLL_CHANGE_OFF;
+
+	/* Clear CLKCHANGE_COMPLETE interrupts */
+	writel(EMC_INTSTATUS_CLKCHANGE_COMPLETE, emc->regs + EMC_INTSTATUS);
+
+	/* Disable dynamic self-refresh */
+	val = readl(emc->regs + EMC_CFG);
+	if (val & EMC_CFG_PWR_MASK) {
+		val &= ~EMC_CFG_POWER_FEATURES_MASK;
+		writel(val, emc->regs + EMC_CFG);
+
+		pre_wait = 5;
+	}
+
+	/* Disable SEL_DPD_CTRL for clock change */
+	if (emc->dram_type == DRAM_TYPE_DDR3)
+		mask = EMC_SEL_DPD_CTRL_DDR3_MASK;
+	else
+		mask = EMC_SEL_DPD_CTRL_MASK;
+
+	val = readl(emc->regs + EMC_SEL_DPD_CTRL);
+	if (val & mask) {
+		val &= ~mask;
+		writel(val, emc->regs + EMC_SEL_DPD_CTRL);
+	}
+
+	/* Prepare DQ/DQS for clock change */
+	val = readl(emc->regs + EMC_XM2DQSPADCTRL2);
+	if (timing->emc_xm2dqspadctrl2 & EMC_XM2DQSPADCTRL2_VREF_ENABLE &&
+	    !(val & EMC_XM2DQSPADCTRL2_VREF_ENABLE)) {
+		val |= EMC_XM2DQSPADCTRL2_VREF_ENABLE;
+		update = true;
+	}
+
+	if (timing->emc_xm2dqspadctrl2 & EMC_XM2DQSPADCTRL2_RX_FT_REC_ENABLE &&
+	    !(val & EMC_XM2DQSPADCTRL2_RX_FT_REC_ENABLE)) {
+		val |= EMC_XM2DQSPADCTRL2_RX_FT_REC_ENABLE;
+		update = true;
+	}
+
+	if (update) {
+		writel(val, emc->regs + EMC_XM2DQSPADCTRL2);
+		if (pre_wait < 30)
+			pre_wait = 30;
+	}
+
+	/* Wait to settle */
+	if (pre_wait) {
+		emc_seq_update_timing(emc);
+		udelay(pre_wait);
+	}
+
+	/* Program CTT_TERM control */
+	if (last->emc_ctt_term_ctrl != timing->emc_ctt_term_ctrl) {
+		emc_seq_disable_auto_cal(emc);
+		writel(timing->emc_ctt_term_ctrl,
+		       emc->regs + EMC_CTT_TERM_CTRL);
+		emc_seq_update_timing(emc);
+	}
+
+	/* Program burst shadow registers */
+	for (i = 0; i < ARRAY_SIZE(timing->emc_burst_data); ++i)
+		writel(timing->emc_burst_data[i],
+		       emc->regs + emc_burst_regs[i]);
+
+	writel(timing->emc_xm2dqspadctrl2, emc->regs + EMC_XM2DQSPADCTRL2);
+	writel(timing->emc_zcal_interval, emc->regs + EMC_ZCAL_INTERVAL);
+
+	tegra_mc_write_emem_configuration(emc->mc, timing->rate);
+
+	val = timing->emc_cfg & ~EMC_CFG_POWER_FEATURES_MASK;
+	emc_ccfifo_writel(emc, val, EMC_CFG);
+
+	/* Program AUTO_CAL_CONFIG */
+	if (timing->emc_auto_cal_config2 != last->emc_auto_cal_config2)
+		emc_ccfifo_writel(emc, timing->emc_auto_cal_config2,
+				  EMC_AUTO_CAL_CONFIG2);
+
+	if (timing->emc_auto_cal_config3 != last->emc_auto_cal_config3)
+		emc_ccfifo_writel(emc, timing->emc_auto_cal_config3,
+				  EMC_AUTO_CAL_CONFIG3);
+
+	if (timing->emc_auto_cal_config != last->emc_auto_cal_config) {
+		val = timing->emc_auto_cal_config;
+		val &= EMC_AUTO_CAL_CONFIG_AUTO_CAL_START;
+		emc_ccfifo_writel(emc, val, EMC_AUTO_CAL_CONFIG);
+	}
+
+	/* DDR3: predict MRS long wait count */
+	if (emc->dram_type == DRAM_TYPE_DDR3 &&
+	    dll_change == DLL_CHANGE_ON) {
+		u32 cnt = 512;
+
+		if (timing->emc_zcal_interval != 0 &&
+		    last->emc_zcal_interval == 0)
+			cnt -= emc->dram_num * 256;
+
+		val = (timing->emc_mrs_wait_cnt
+			& EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK)
+			>> EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT;
+		if (cnt < val)
+			cnt = val;
+
+		val = timing->emc_mrs_wait_cnt
+			& ~EMC_MRS_WAIT_CNT_LONG_WAIT_MASK;
+		val |= (cnt << EMC_MRS_WAIT_CNT_LONG_WAIT_SHIFT)
+			& EMC_MRS_WAIT_CNT_LONG_WAIT_MASK;
+
+		writel(val, emc->regs + EMC_MRS_WAIT_CNT);
+	}
+
+	/* DDR3: Turn off DLL and enter self-refresh */
+	if (emc->dram_type == DRAM_TYPE_DDR3 && dll_change == DLL_CHANGE_OFF)
+		emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_EMRS);
+
+	/* Disable refresh controller */
+	emc_ccfifo_writel(emc, EMC_REFCTRL_DEV_SEL(emc->dram_num),
+			  EMC_REFCTRL);
+	if (emc->dram_type == DRAM_TYPE_DDR3)
+		emc_ccfifo_writel(emc, EMC_DRAM_DEV_SEL(emc->dram_num) |
+				       EMC_SELF_REF_CMD_ENABLED,
+				  EMC_SELF_REF);
+
+	/* Flow control marker */
+	emc_ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE);
+
+	/* DDR3: Exit self-refresh */
+	if (emc->dram_type == DRAM_TYPE_DDR3)
+		emc_ccfifo_writel(emc, EMC_DRAM_DEV_SEL(emc->dram_num),
+				  EMC_SELF_REF);
+	emc_ccfifo_writel(emc, EMC_REFCTRL_DEV_SEL(emc->dram_num) |
+			       EMC_REFCTRL_ENABLE,
+			  EMC_REFCTRL);
+
+	/* Set DRAM mode registers */
+	if (emc->dram_type == DRAM_TYPE_DDR3) {
+		if (timing->emc_mode_1 != last->emc_mode_1)
+			emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_EMRS);
+		if (timing->emc_mode_2 != last->emc_mode_2)
+			emc_ccfifo_writel(emc, timing->emc_mode_2, EMC_EMRS2);
+
+		if (timing->emc_mode_reset != last->emc_mode_reset ||
+		    dll_change == DLL_CHANGE_ON) {
+			val = timing->emc_mode_reset;
+			if (dll_change == DLL_CHANGE_ON) {
+				val |= EMC_MODE_SET_DLL_RESET;
+				val |= EMC_MODE_SET_LONG_CNT;
+			} else {
+				val &= ~EMC_MODE_SET_DLL_RESET;
+			}
+			emc_ccfifo_writel(emc, val, EMC_MRS);
+		}
+	} else {
+		if (timing->emc_mode_2 != last->emc_mode_2)
+			emc_ccfifo_writel(emc, timing->emc_mode_2, EMC_MRW2);
+		if (timing->emc_mode_1 != last->emc_mode_1)
+			emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_MRW);
+		if (timing->emc_mode_4 != last->emc_mode_4)
+			emc_ccfifo_writel(emc, timing->emc_mode_4, EMC_MRW4);
+	}
+
+	/*  Issue ZCAL command if turning ZCAL on */
+	if (timing->emc_zcal_interval != 0 && last->emc_zcal_interval == 0) {
+		emc_ccfifo_writel(emc, EMC_ZQ_CAL_LONG_CMD_DEV0, EMC_ZQ_CAL);
+		if (emc->dram_num > 1)
+			emc_ccfifo_writel(emc, EMC_ZQ_CAL_LONG_CMD_DEV1,
+					  EMC_ZQ_CAL);
+	}
+
+	/*  Write to RO register to remove stall after change */
+	emc_ccfifo_writel(emc, 0, EMC_CCFIFO_STATUS);
+
+	/* Disable AUTO_CAL for clock change */
+	emc_seq_disable_auto_cal(emc);
+
+	/* Read register to wait until programming has settled */
+	mc_readl(emc->mc, MC_EMEM_ADR_CFG);
+
+	return 0;
+}
+
+static void tegra114_emc_complete_timing_change(struct tegra_emc *emc,
+						unsigned long rate)
+{
+	struct emc_timing *timing = tegra114_emc_find_timing(emc, rate);
+	struct emc_timing *last = &emc->last_timing;
+
+	if (!timing)
+		return;
+
+	/* Wait until the state machine has settled */
+	emc_seq_wait_clkchange(emc);
+
+	/* Restore AUTO_CAL */
+	if (timing->emc_ctt_term_ctrl != last->emc_ctt_term_ctrl)
+		writel(timing->emc_auto_cal_interval,
+		       emc->regs + EMC_AUTO_CAL_INTERVAL);
+
+	/* Restore dynamic self-refresh */
+	if (timing->emc_cfg & EMC_CFG_PWR_MASK)
+		writel(timing->emc_cfg, emc->regs + EMC_CFG);
+
+	/* Set ZCAL wait count */
+	writel(timing->emc_zcal_cnt_long, emc->regs + EMC_ZCAL_WAIT_CNT);
+
+	/* Wait for timing to settle */
+	udelay(2);
+
+	/* Reprogram SEL_DPD_CTRL */
+	writel(timing->emc_sel_dpd_ctrl, emc->regs + EMC_SEL_DPD_CTRL);
+	emc_seq_update_timing(emc);
+
+	emc->last_timing = *timing;
+}
+
+/* Initialization and deinitialization */
+
+static void emc_read_current_timing(struct tegra_emc *emc,
+				    struct emc_timing *timing)
+{
+	unsigned int i;
+
+	for (i = 0; i < ARRAY_SIZE(emc_burst_regs); ++i)
+		timing->emc_burst_data[i] =
+			readl(emc->regs + emc_burst_regs[i]);
+
+	timing->emc_cfg = readl(emc->regs + EMC_CFG);
+
+	timing->emc_auto_cal_interval = 0;
+	timing->emc_zcal_cnt_long = 0;
+	timing->emc_mode_1 = 0;
+	timing->emc_mode_2 = 0;
+	timing->emc_mode_4 = 0;
+	timing->emc_mode_reset = 0;
+}
+
+static void emc_init(struct tegra_emc *emc)
+{
+	u32 emc_cfg, emc_dbg;
+	u32 intmask = EMC_INTSTATUS_REFRESH_OVERFLOW;
+	const char *dram_type_str;
+
+	emc->dram_type = readl(emc->regs + EMC_FBIO_CFG5);
+
+	emc->dram_type &= EMC_FBIO_CFG5_DRAM_TYPE_MASK;
+	emc->dram_type >>= EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;
+
+	emc->dram_num = tegra_mc_get_emem_device_count(emc->mc);
+
+	emc_cfg = readl_relaxed(emc->regs + EMC_CFG_2);
+
+	/* enable EMC and CAR to handshake on PLL divider/source changes */
+	emc_cfg |= EMC_CFG_2_CLKCHANGE_REQ_ENABLE;
+
+	/* configure clock change mode accordingly to DRAM type */
+	if (emc->dram_type == DRAM_TYPE_LPDDR2)
+		emc_cfg |= EMC_CFG_2_CLKCHANGE_PD_ENABLE;
+	else
+		emc_cfg &= ~EMC_CFG_2_CLKCHANGE_PD_ENABLE;
+
+	writel_relaxed(emc_cfg, emc->regs + EMC_CFG_2);
+
+	/* initialize interrupt */
+	writel_relaxed(intmask, emc->regs + EMC_INTMASK);
+	writel_relaxed(0xffffffff, emc->regs + EMC_INTSTATUS);
+
+	/* ensure that unwanted debug features are disabled */
+	emc_dbg = readl_relaxed(emc->regs + EMC_DBG);
+	emc_dbg |= EMC_DBG_CFG_PRIORITY;
+	emc_dbg &= ~EMC_DBG_READ_MUX_ASSEMBLY;
+	emc_dbg &= ~EMC_DBG_WRITE_MUX_ACTIVE;
+	emc_dbg &= ~EMC_DBG_FORCE_UPDATE;
+	writel_relaxed(emc_dbg, emc->regs + EMC_DBG);
+
+	switch (emc->dram_type) {
+	case DRAM_TYPE_DDR1:
+		dram_type_str = "DDR1";
+		break;
+	case DRAM_TYPE_LPDDR2:
+		dram_type_str = "LPDDR2";
+		break;
+	case DRAM_TYPE_DDR2:
+		dram_type_str = "DDR2";
+		break;
+	case DRAM_TYPE_DDR3:
+		dram_type_str = "DDR3";
+		break;
+	}
+
+	dev_info_once(emc->dev, "%u %s %s attached\n", emc->dram_num,
+		      dram_type_str, emc->dram_num == 2 ? "devices" : "device");
+
+	emc_read_current_timing(emc, &emc->last_timing);
+}
+
+static int load_one_timing_from_dt(struct tegra_emc *emc,
+				   struct emc_timing *timing,
+				   struct device_node *node)
+{
+	u32 value;
+	int err;
+
+	err = of_property_read_u32(node, "clock-frequency", &value);
+	if (err) {
+		dev_err(emc->dev, "timing %pOFn: failed to read rate: %d\n",
+			node, err);
+		return err;
+	}
+
+	timing->rate = value;
+
+	err = of_property_read_u32_array(node, "nvidia,emc-configuration",
+					 timing->emc_burst_data,
+					 ARRAY_SIZE(timing->emc_burst_data));
+	if (err) {
+		dev_err(emc->dev,
+			"timing %pOFn: failed to read emc burst data: %d\n",
+			node, err);
+		return err;
+	}
+
+#define EMC_READ_PROP(prop, dtprop) { \
+	err = of_property_read_u32(node, dtprop, &timing->prop); \
+	if (err) { \
+		dev_err(emc->dev, "timing %pOFn: failed to read " #prop ": %d\n", \
+			node, err); \
+		return err; \
+	} \
+}
+
+	EMC_READ_PROP(emc_auto_cal_config, "nvidia,emc-auto-cal-config")
+	EMC_READ_PROP(emc_auto_cal_config2, "nvidia,emc-auto-cal-config2")
+	EMC_READ_PROP(emc_auto_cal_config3, "nvidia,emc-auto-cal-config3")
+	EMC_READ_PROP(emc_auto_cal_interval, "nvidia,emc-auto-cal-interval")
+	EMC_READ_PROP(emc_cfg, "nvidia,emc-cfg")
+	EMC_READ_PROP(emc_ctt_term_ctrl, "nvidia,emc-ctt-term-ctrl")
+	EMC_READ_PROP(emc_mode_1, "nvidia,emc-mode-1")
+	EMC_READ_PROP(emc_mode_2, "nvidia,emc-mode-2")
+	EMC_READ_PROP(emc_mode_4, "nvidia,emc-mode-4")
+	EMC_READ_PROP(emc_mode_reset, "nvidia,emc-mode-reset")
+	EMC_READ_PROP(emc_mrs_wait_cnt, "nvidia,emc-mrs-wait-cnt")
+	EMC_READ_PROP(emc_sel_dpd_ctrl, "nvidia,emc-sel-dpd-ctrl")
+	EMC_READ_PROP(emc_xm2dqspadctrl2, "nvidia,emc-xm2dqspadctrl2")
+	EMC_READ_PROP(emc_zcal_cnt_long, "nvidia,emc-zcal-cnt-long")
+	EMC_READ_PROP(emc_zcal_interval, "nvidia,emc-zcal-interval")
+
+#undef EMC_READ_PROP
+
+	return 0;
+}
+
+static int cmp_timings(const void *_a, const void *_b)
+{
+	const struct emc_timing *a = _a;
+	const struct emc_timing *b = _b;
+
+	if (a->rate < b->rate)
+		return -1;
+	else if (a->rate == b->rate)
+		return 0;
+	else
+		return 1;
+}
+
+static int emc_check_mc_timings(struct tegra_emc *emc)
+{
+	struct tegra_mc *mc = emc->mc;
+	unsigned int i;
+
+	if (emc->num_timings != mc->num_timings) {
+		dev_err(emc->dev, "emc/mc timings number mismatch: %u %u\n",
+			emc->num_timings, mc->num_timings);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < mc->num_timings; i++) {
+		if (emc->timings[i].rate != mc->timings[i].rate) {
+			dev_err(emc->dev,
+				"emc/mc timing rate mismatch: %lu %lu\n",
+				emc->timings[i].rate, mc->timings[i].rate);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int tegra114_emc_load_timings_from_dt(struct tegra_emc *emc,
+					     struct device_node *node)
+{
+	int child_count = of_get_child_count(node);
+	struct emc_timing *timing;
+	unsigned int i = 0;
+	int err;
+
+	emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing),
+				    GFP_KERNEL);
+	if (!emc->timings)
+		return -ENOMEM;
+
+	emc->num_timings = child_count;
+
+	for_each_child_of_node_scoped(node, child) {
+		timing = &emc->timings[i++];
+
+		err = load_one_timing_from_dt(emc, timing, child);
+		if (err)
+			return err;
+	}
+
+	sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings,
+	     NULL);
+
+	err = emc_check_mc_timings(emc);
+	if (err)
+		return err;
+
+	dev_info_once(emc->dev,
+		      "got %u timings for RAM code %u (min %luMHz max %luMHz)\n",
+		      emc->num_timings,
+		      tegra_read_ram_code(),
+		      emc->timings[0].rate / 1000000,
+		      emc->timings[emc->num_timings - 1].rate / 1000000);
+
+	return 0;
+}
+
+static struct device_node *
+tegra114_emc_find_node_by_ram_code(struct device_node *node, u32 ram_code)
+{
+	struct device_node *np;
+	int err;
+
+	for_each_child_of_node(node, np) {
+		u32 value;
+
+		err = of_property_read_u32(np, "nvidia,ram-code", &value);
+		if (err || value != ram_code)
+			continue;
+
+		return np;
+	}
+
+	return NULL;
+}
+
+/*
+ * debugfs interface
+ *
+ * The memory controller driver exposes some files in debugfs that can be used
+ * to control the EMC frequency. The top-level directory can be found here:
+ *
+ *   /sys/kernel/debug/emc
+ *
+ * It contains the following files:
+ *
+ *   - available_rates: This file contains a list of valid, space-separated
+ *     EMC frequencies.
+ *
+ *   - min_rate: Writing a value to this file sets the given frequency as the
+ *       floor of the permitted range. If this is higher than the currently
+ *       configured EMC frequency, this will cause the frequency to be
+ *       increased so that it stays within the valid range.
+ *
+ *   - max_rate: Similarly to the min_rate file, writing a value to this file
+ *       sets the given frequency as the ceiling of the permitted range. If
+ *       the value is lower than the currently configured EMC frequency, this
+ *       will cause the frequency to be decreased so that it stays within the
+ *       valid range.
+ */
+
+static bool tegra114_emc_validate_rate(struct tegra_emc *emc, unsigned long rate)
+{
+	unsigned int i;
+
+	for (i = 0; i < emc->num_timings; i++)
+		if (rate == emc->timings[i].rate)
+			return true;
+
+	return false;
+}
+
+static int tegra114_emc_debug_available_rates_show(struct seq_file *s,
+						   void *data)
+{
+	struct tegra_emc *emc = s->private;
+	const char *prefix = "";
+	unsigned int i;
+
+	for (i = 0; i < emc->num_timings; i++) {
+		seq_printf(s, "%s%lu", prefix, emc->timings[i].rate);
+		prefix = " ";
+	}
+
+	seq_puts(s, "\n");
+
+	return 0;
+}
+
+DEFINE_SHOW_ATTRIBUTE(tegra114_emc_debug_available_rates);
+
+static int tegra114_emc_debug_min_rate_get(void *data, u64 *rate)
+{
+	struct tegra_emc *emc = data;
+
+	*rate = emc->debugfs.min_rate;
+
+	return 0;
+}
+
+static int tegra114_emc_debug_min_rate_set(void *data, u64 rate)
+{
+	struct tegra_emc *emc = data;
+	int err;
+
+	if (!tegra114_emc_validate_rate(emc, rate))
+		return -EINVAL;
+
+	err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG);
+	if (err < 0)
+		return err;
+
+	emc->debugfs.min_rate = rate;
+
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(tegra114_emc_debug_min_rate_fops,
+			 tegra114_emc_debug_min_rate_get,
+			 tegra114_emc_debug_min_rate_set, "%llu\n");
+
+static int tegra114_emc_debug_max_rate_get(void *data, u64 *rate)
+{
+	struct tegra_emc *emc = data;
+
+	*rate = emc->debugfs.max_rate;
+
+	return 0;
+}
+
+static int tegra114_emc_debug_max_rate_set(void *data, u64 rate)
+{
+	struct tegra_emc *emc = data;
+	int err;
+
+	if (!tegra114_emc_validate_rate(emc, rate))
+		return -EINVAL;
+
+	err = tegra_emc_set_max_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG);
+	if (err < 0)
+		return err;
+
+	emc->debugfs.max_rate = rate;
+
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(tegra114_emc_debug_max_rate_fops,
+			 tegra114_emc_debug_max_rate_get,
+			 tegra114_emc_debug_max_rate_set, "%llu\n");
+
+static void emc_debugfs_init(struct device *dev, struct tegra_emc *emc)
+{
+	unsigned int i;
+	int err;
+
+	emc->debugfs.min_rate = ULONG_MAX;
+	emc->debugfs.max_rate = 0;
+
+	for (i = 0; i < emc->num_timings; i++) {
+		if (emc->timings[i].rate < emc->debugfs.min_rate)
+			emc->debugfs.min_rate = emc->timings[i].rate;
+
+		if (emc->timings[i].rate > emc->debugfs.max_rate)
+			emc->debugfs.max_rate = emc->timings[i].rate;
+	}
+
+	if (!emc->num_timings) {
+		emc->debugfs.min_rate = clk_get_rate(emc->clk);
+		emc->debugfs.max_rate = emc->debugfs.min_rate;
+	}
+
+	err = clk_set_rate_range(emc->clk, emc->debugfs.min_rate,
+				 emc->debugfs.max_rate);
+	if (err < 0) {
+		dev_err(dev, "failed to set rate range [%lu-%lu] for %pC\n",
+			emc->debugfs.min_rate, emc->debugfs.max_rate,
+			emc->clk);
+		return;
+	}
+
+	emc->debugfs.root = debugfs_create_dir("emc", NULL);
+
+	debugfs_create_file("available_rates", 0444, emc->debugfs.root, emc,
+			    &tegra114_emc_debug_available_rates_fops);
+	debugfs_create_file("min_rate", 0644, emc->debugfs.root,
+			    emc, &tegra114_emc_debug_min_rate_fops);
+	debugfs_create_file("max_rate", 0644, emc->debugfs.root,
+			    emc, &tegra114_emc_debug_max_rate_fops);
+}
+
+static inline struct tegra_emc *
+to_tegra_emc_provider(struct icc_provider *provider)
+{
+	return container_of(provider, struct tegra_emc, provider);
+}
+
+static struct icc_node_data *
+emc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data)
+{
+	struct icc_provider *provider = data;
+	struct icc_node_data *ndata;
+	struct icc_node *node;
+
+	/* External Memory is the only possible ICC route */
+	list_for_each_entry(node, &provider->nodes, node_list) {
+		if (node->id != TEGRA_ICC_EMEM)
+			continue;
+
+		ndata = kzalloc_obj(*ndata);
+		if (!ndata)
+			return ERR_PTR(-ENOMEM);
+
+		/*
+		 * SRC and DST nodes should have matching TAG in order to have
+		 * it set by default for a requested path.
+		 */
+		ndata->tag = TEGRA_MC_ICC_TAG_ISO;
+		ndata->node = node;
+
+		return ndata;
+	}
+
+	return ERR_PTR(-EPROBE_DEFER);
+}
+
+static int emc_icc_set(struct icc_node *src, struct icc_node *dst)
+{
+	struct tegra_emc *emc = to_tegra_emc_provider(dst->provider);
+	unsigned long long peak_bw = icc_units_to_bps(dst->peak_bw);
+	unsigned long long avg_bw = icc_units_to_bps(dst->avg_bw);
+	unsigned long long rate = max(avg_bw, peak_bw);
+	unsigned int dram_data_bus_width_bytes = 4;
+	const unsigned int ddr = 2;
+	int err;
+
+	/*
+	 * Tegra114 EMC runs on a clock rate of SDRAM bus. This means that
+	 * EMC clock rate is twice smaller than the peak data rate because
+	 * data is sampled on both EMC clock edges.
+	 */
+	do_div(rate, ddr * dram_data_bus_width_bytes);
+	rate = min_t(u64, rate, U32_MAX);
+
+	err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_ICC);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static int tegra114_emc_interconnect_init(struct tegra_emc *emc)
+{
+	const struct tegra_mc_soc *soc = emc->mc->soc;
+	struct icc_node *node;
+	int err;
+
+	emc->provider.dev = emc->dev;
+	emc->provider.set = emc_icc_set;
+	emc->provider.data = &emc->provider;
+	emc->provider.aggregate = soc->icc_ops->aggregate;
+	emc->provider.xlate_extended = emc_of_icc_xlate_extended;
+
+	icc_provider_init(&emc->provider);
+
+	/* create External Memory Controller node */
+	node = icc_node_create(TEGRA_ICC_EMC);
+	if (IS_ERR(node))
+		return PTR_ERR(node);
+
+	node->name = "External Memory Controller";
+	icc_node_add(node, &emc->provider);
+
+	/* link External Memory Controller to External Memory (DRAM) */
+	err = icc_link_create(node, TEGRA_ICC_EMEM);
+	if (err)
+		goto remove_nodes;
+
+	/* create External Memory node */
+	node = icc_node_create(TEGRA_ICC_EMEM);
+	if (IS_ERR(node)) {
+		err = PTR_ERR(node);
+		goto remove_nodes;
+	}
+
+	node->name = "External Memory (DRAM)";
+	icc_node_add(node, &emc->provider);
+
+	err = icc_provider_register(&emc->provider);
+	if (err)
+		goto remove_nodes;
+
+	return 0;
+
+remove_nodes:
+	icc_nodes_remove(&emc->provider);
+
+	return dev_err_probe(emc->dev, err, "failed to initialize ICC");
+}
+
+static void devm_tegra114_emc_unset_callback(void *data)
+{
+	tegra124_clk_set_emc_callbacks(NULL, NULL);
+}
+
+static int tegra114_emc_probe(struct platform_device *pdev)
+{
+	struct tegra_core_opp_params opp_params = {};
+	struct device *dev = &pdev->dev;
+	struct device_node *np;
+	struct tegra_emc *emc;
+	u32 ram_code;
+	int err;
+
+	emc = devm_kzalloc(dev, sizeof(*emc), GFP_KERNEL);
+	if (!emc)
+		return -ENOMEM;
+
+	emc->dev = dev;
+
+	emc->regs = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(emc->regs))
+		return PTR_ERR(emc->regs);
+
+	emc->mc = devm_tegra_memory_controller_get(dev);
+	if (IS_ERR(emc->mc))
+		return PTR_ERR(emc->mc);
+
+	ram_code = tegra_read_ram_code();
+
+	np = tegra114_emc_find_node_by_ram_code(dev->of_node, ram_code);
+	if (np) {
+		err = tegra114_emc_load_timings_from_dt(emc, np);
+		of_node_put(np);
+		if (err)
+			return err;
+	} else {
+		dev_info_once(dev, "no memory timings for RAM code %u found in DT\n",
+			      ram_code);
+	}
+
+	emc_init(emc);
+
+	platform_set_drvdata(pdev, emc);
+
+	tegra124_clk_set_emc_callbacks(tegra114_emc_prepare_timing_change,
+				       tegra114_emc_complete_timing_change);
+
+	err = devm_add_action_or_reset(dev, devm_tegra114_emc_unset_callback,
+				       NULL);
+	if (err)
+		return err;
+
+	err = platform_get_irq(pdev, 0);
+	if (err < 0)
+		return err;
+
+	emc->irq = err;
+
+	err = devm_request_irq(dev, emc->irq, tegra114_emc_isr, 0,
+			       dev_name(dev), emc);
+	if (err)
+		return dev_err_probe(dev, err, "failed to request irq\n");
+
+	emc->clk = devm_clk_get(dev, "emc");
+	if (IS_ERR(emc->clk))
+		return dev_err_probe(dev, PTR_ERR(emc->clk),
+				     "failed to get EMC clock\n");
+
+	opp_params.init_state = true;
+
+	err = devm_tegra_core_dev_init_opp_table(dev, &opp_params);
+	if (err)
+		return err;
+
+	tegra_emc_rate_requests_init(&emc->reqs, dev);
+
+	if (IS_ENABLED(CONFIG_DEBUG_FS))
+		emc_debugfs_init(dev, emc);
+
+	tegra114_emc_interconnect_init(emc);
+
+	/*
+	 * Don't allow the kernel module to be unloaded. Unloading adds some
+	 * extra complexity which doesn't really worth the effort in a case of
+	 * this driver.
+	 */
+	try_module_get(THIS_MODULE);
+
+	return 0;
+};
+
+static const struct of_device_id tegra114_emc_of_match[] = {
+	{ .compatible = "nvidia,tegra114-emc" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tegra114_emc_of_match);
+
+static struct platform_driver tegra114_emc_driver = {
+	.probe = tegra114_emc_probe,
+	.driver = {
+		.name = "tegra114-emc",
+		.of_match_table = tegra114_emc_of_match,
+		.suppress_bind_attrs = true,
+		.sync_state = icc_sync_state,
+	},
+};
+module_platform_driver(tegra114_emc_driver);
+
+MODULE_AUTHOR("Svyatoslav Ryhel <clamor95@gmail.com>");
+MODULE_DESCRIPTION("NVIDIA Tegra114 EMC driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/memory/tegra/tegra114.c b/drivers/memory/tegra/tegra114.c
index 02dd4e26288a1..2cb153091228a 100644
--- a/drivers/memory/tegra/tegra114.c
+++ b/drivers/memory/tegra/tegra114.c
@@ -3,6 +3,7 @@
  * Copyright (C) 2014-2026 NVIDIA CORPORATION.  All rights reserved.
  */
 
+#include <linux/device.h>
 #include <linux/of.h>
 #include <linux/mm.h>
 
@@ -1101,6 +1102,195 @@ static const struct tegra_mc_reset tegra114_mc_resets[] = {
 	TEGRA114_MC_RESET(VI,       0x200, 0x204, 17),
 };
 
+static void tegra114_mc_tune_client_latency(struct tegra_mc *mc,
+					    const struct tegra_mc_client *client,
+					    unsigned int bandwidth_mbytes_sec)
+{
+	u32 arb_tolerance_compensation_nsec, arb_tolerance_compensation_div;
+	unsigned int fifo_size = client->fifo_size;
+	u32 arb_nsec, la_ticks, value;
+
+	/* see 20.3.1.1 Client Configuration in Tegra4 TRM v01p */
+	if (bandwidth_mbytes_sec)
+		arb_nsec = fifo_size * NSEC_PER_USEC / bandwidth_mbytes_sec;
+	else
+		arb_nsec = U32_MAX;
+
+	/*
+	 * Latency allowness should be set with consideration for the module's
+	 * latency tolerance and internal buffering capabilities.
+	 *
+	 * Display memory clients use isochronous transfers and have very low
+	 * tolerance to a belated transfers. Hence we need to compensate the
+	 * memory arbitration imperfection for them in order to prevent FIFO
+	 * underflow condition when memory bus is busy.
+	 *
+	 * VI clients also need a stronger compensation.
+	 */
+	switch (client->swgroup) {
+	case TEGRA_SWGROUP_MPCORE:
+	case TEGRA_SWGROUP_PTC:
+		/*
+		 * We always want lower latency for these clients, hence
+		 * don't touch them.
+		 */
+		return;
+
+	case TEGRA_SWGROUP_DC:
+	case TEGRA_SWGROUP_DCB:
+		arb_tolerance_compensation_nsec = 1050;
+		arb_tolerance_compensation_div = 2;
+		break;
+
+	case TEGRA_SWGROUP_VI:
+		arb_tolerance_compensation_nsec = 1050;
+		arb_tolerance_compensation_div = 1;
+		break;
+
+	default:
+		arb_tolerance_compensation_nsec = 150;
+		arb_tolerance_compensation_div = 1;
+		break;
+	}
+
+	if (arb_nsec > arb_tolerance_compensation_nsec)
+		arb_nsec -= arb_tolerance_compensation_nsec;
+	else
+		arb_nsec = 0;
+
+	arb_nsec /= arb_tolerance_compensation_div;
+
+	/*
+	 * Latency allowance is a number of ticks a request from a particular
+	 * client may wait in the EMEM arbiter before it becomes a high-priority
+	 * request.
+	 */
+	la_ticks = arb_nsec / mc->tick;
+	la_ticks = min(la_ticks, client->regs.la.mask);
+
+	value = mc_readl(mc, client->regs.la.reg);
+	value &= ~(client->regs.la.mask << client->regs.la.shift);
+	value |= la_ticks << client->regs.la.shift;
+	mc_writel(mc, value, client->regs.la.reg);
+}
+
+static int tegra114_mc_icc_set(struct icc_node *src, struct icc_node *dst)
+{
+	struct tegra_mc *mc = icc_provider_to_tegra_mc(src->provider);
+	const struct tegra_mc_client *client = &mc->soc->clients[src->id];
+	u64 peak_bandwidth = icc_units_to_bps(src->peak_bw);
+
+	/*
+	 * Skip pre-initialization that is done by icc_node_add(), which sets
+	 * bandwidth to maximum for all clients before drivers are loaded.
+	 *
+	 * This doesn't make sense for us because we don't have drivers for all
+	 * clients and it's okay to keep configuration left from bootloader
+	 * during boot, at least for today.
+	 */
+	if (src == dst)
+		return 0;
+
+	/* convert bytes/sec to megabytes/sec */
+	do_div(peak_bandwidth, 1000000);
+
+	tegra114_mc_tune_client_latency(mc, client, peak_bandwidth);
+
+	return 0;
+}
+
+static int tegra114_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw,
+				    u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
+{
+	/*
+	 * ISO clients need to reserve extra bandwidth up-front because
+	 * there could be high bandwidth pressure during initial filling
+	 * of the client's FIFO buffers.  Secondly, we need to take into
+	 * account impurities of the memory subsystem.
+	 */
+	if (tag & TEGRA_MC_ICC_TAG_ISO)
+		peak_bw = tegra_mc_scale_percents(peak_bw, 400);
+
+	*agg_avg += avg_bw;
+	*agg_peak = max(*agg_peak, peak_bw);
+
+	return 0;
+}
+
+static struct icc_node_data *
+tegra114_mc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data)
+{
+	struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
+	const struct tegra_mc_client *client;
+	unsigned int i, idx = spec->args[0];
+	struct icc_node_data *ndata;
+	struct icc_node *node;
+
+	list_for_each_entry(node, &mc->provider.nodes, node_list) {
+		if (node->id != idx)
+			continue;
+
+		ndata = kzalloc_obj(*ndata);
+		if (!ndata)
+			return ERR_PTR(-ENOMEM);
+
+		client = &mc->soc->clients[idx];
+		ndata->node = node;
+
+		switch (client->swgroup) {
+		case TEGRA_SWGROUP_DC:
+		case TEGRA_SWGROUP_DCB:
+		case TEGRA_SWGROUP_PTC:
+		case TEGRA_SWGROUP_VI:
+			/* these clients are isochronous by default */
+			ndata->tag = TEGRA_MC_ICC_TAG_ISO;
+			break;
+
+		default:
+			ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT;
+			break;
+		}
+
+		return ndata;
+	}
+
+	for (i = 0; i < mc->soc->num_clients; i++) {
+		if (mc->soc->clients[i].id == idx)
+			return ERR_PTR(-EPROBE_DEFER);
+	}
+
+	dev_err(mc->dev, "invalid ICC client ID %u\n", idx);
+
+	return ERR_PTR(-EINVAL);
+}
+
+static const struct tegra_mc_icc_ops tegra114_mc_icc_ops = {
+	.xlate_extended = tegra114_mc_of_icc_xlate_extended,
+	.aggregate = tegra114_mc_icc_aggreate,
+	.set = tegra114_mc_icc_set,
+};
+
+static const unsigned long tegra114_mc_emem_regs[] = {
+	MC_EMEM_ARB_CFG,
+	MC_EMEM_ARB_OUTSTANDING_REQ,
+	MC_EMEM_ARB_TIMING_RCD,
+	MC_EMEM_ARB_TIMING_RP,
+	MC_EMEM_ARB_TIMING_RC,
+	MC_EMEM_ARB_TIMING_RAS,
+	MC_EMEM_ARB_TIMING_FAW,
+	MC_EMEM_ARB_TIMING_RRD,
+	MC_EMEM_ARB_TIMING_RAP2PRE,
+	MC_EMEM_ARB_TIMING_WAP2PRE,
+	MC_EMEM_ARB_TIMING_R2R,
+	MC_EMEM_ARB_TIMING_W2W,
+	MC_EMEM_ARB_TIMING_R2W,
+	MC_EMEM_ARB_TIMING_W2R,
+	MC_EMEM_ARB_DA_TURNS,
+	MC_EMEM_ARB_DA_COVERS,
+	MC_EMEM_ARB_MISC0,
+	MC_EMEM_ARB_RING1_THROTTLE,
+};
+
 static const struct tegra_mc_intmask tegra114_mc_intmasks[] = {
 	{
 		.reg = MC_INTMASK,
@@ -1116,11 +1306,14 @@ const struct tegra_mc_soc tegra114_mc_soc = {
 	.atom_size = 32,
 	.client_id_mask = 0x7f,
 	.smmu = &tegra114_smmu_soc,
+	.emem_regs = tegra114_mc_emem_regs,
+	.num_emem_regs = ARRAY_SIZE(tegra114_mc_emem_regs),
 	.intmasks = tegra114_mc_intmasks,
 	.num_intmasks = ARRAY_SIZE(tegra114_mc_intmasks),
 	.reset_ops = &tegra_mc_reset_ops_common,
 	.resets = tegra114_mc_resets,
 	.num_resets = ARRAY_SIZE(tegra114_mc_resets),
+	.icc_ops = &tegra114_mc_icc_ops,
 	.ops = &tegra30_mc_ops,
 	.regs = &tegra20_mc_regs,
 	.handle_irq = tegra30_mc_irq_handlers,
diff --git a/drivers/memory/tegra/tegra124-emc.c b/drivers/memory/tegra/tegra124-emc.c
index 5cfbc169c5f98..555a6c753be76 100644
--- a/drivers/memory/tegra/tegra124-emc.c
+++ b/drivers/memory/tegra/tegra124-emc.c
@@ -27,6 +27,7 @@
 #include <soc/tegra/mc.h>
 
 #include "mc.h"
+#include "tegra-emc-common.h"
 
 #define EMC_FBIO_CFG5				0x104
 #define	EMC_FBIO_CFG5_DRAM_TYPE_MASK		0x3
@@ -467,17 +468,6 @@ struct emc_timing {
 	u32 emc_zcal_interval;
 };
 
-enum emc_rate_request_type {
-	EMC_RATE_DEBUG,
-	EMC_RATE_ICC,
-	EMC_RATE_TYPE_MAX,
-};
-
-struct emc_rate_request {
-	unsigned long min_rate;
-	unsigned long max_rate;
-};
-
 struct tegra_emc {
 	struct device *dev;
 
@@ -503,14 +493,7 @@ struct tegra_emc {
 
 	struct icc_provider provider;
 
-	/*
-	 * There are multiple sources in the EMC driver which could request
-	 * a min/max clock rate, these rates are contained in this array.
-	 */
-	struct emc_rate_request requested_rate[EMC_RATE_TYPE_MAX];
-
-	/* protect shared rate-change code path */
-	struct mutex rate_lock;
+	struct tegra_emc_rate_requests reqs;
 };
 
 /* Timing change sequence functions */
@@ -1041,83 +1024,6 @@ tegra124_emc_find_node_by_ram_code(struct device_node *node, u32 ram_code)
 	return NULL;
 }
 
-static void tegra124_emc_rate_requests_init(struct tegra_emc *emc)
-{
-	unsigned int i;
-
-	for (i = 0; i < EMC_RATE_TYPE_MAX; i++) {
-		emc->requested_rate[i].min_rate = 0;
-		emc->requested_rate[i].max_rate = ULONG_MAX;
-	}
-}
-
-static int emc_request_rate(struct tegra_emc *emc,
-			    unsigned long new_min_rate,
-			    unsigned long new_max_rate,
-			    enum emc_rate_request_type type)
-{
-	struct emc_rate_request *req = emc->requested_rate;
-	unsigned long min_rate = 0, max_rate = ULONG_MAX;
-	unsigned int i;
-	int err;
-
-	/* select minimum and maximum rates among the requested rates */
-	for (i = 0; i < EMC_RATE_TYPE_MAX; i++, req++) {
-		if (i == type) {
-			min_rate = max(new_min_rate, min_rate);
-			max_rate = min(new_max_rate, max_rate);
-		} else {
-			min_rate = max(req->min_rate, min_rate);
-			max_rate = min(req->max_rate, max_rate);
-		}
-	}
-
-	if (min_rate > max_rate) {
-		dev_err_ratelimited(emc->dev, "%s: type %u: out of range: %lu %lu\n",
-				    __func__, type, min_rate, max_rate);
-		return -ERANGE;
-	}
-
-	/*
-	 * EMC rate-changes should go via OPP API because it manages voltage
-	 * changes.
-	 */
-	err = dev_pm_opp_set_rate(emc->dev, min_rate);
-	if (err)
-		return err;
-
-	emc->requested_rate[type].min_rate = new_min_rate;
-	emc->requested_rate[type].max_rate = new_max_rate;
-
-	return 0;
-}
-
-static int emc_set_min_rate(struct tegra_emc *emc, unsigned long rate,
-			    enum emc_rate_request_type type)
-{
-	struct emc_rate_request *req = &emc->requested_rate[type];
-	int ret;
-
-	mutex_lock(&emc->rate_lock);
-	ret = emc_request_rate(emc, rate, req->max_rate, type);
-	mutex_unlock(&emc->rate_lock);
-
-	return ret;
-}
-
-static int emc_set_max_rate(struct tegra_emc *emc, unsigned long rate,
-			    enum emc_rate_request_type type)
-{
-	struct emc_rate_request *req = &emc->requested_rate[type];
-	int ret;
-
-	mutex_lock(&emc->rate_lock);
-	ret = emc_request_rate(emc, req->min_rate, rate, type);
-	mutex_unlock(&emc->rate_lock);
-
-	return ret;
-}
-
 /*
  * debugfs interface
  *
@@ -1190,7 +1096,7 @@ static int tegra124_emc_debug_min_rate_set(void *data, u64 rate)
 	if (!tegra124_emc_validate_rate(emc, rate))
 		return -EINVAL;
 
-	err = emc_set_min_rate(emc, rate, EMC_RATE_DEBUG);
+	err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG);
 	if (err < 0)
 		return err;
 
@@ -1220,7 +1126,7 @@ static int tegra124_emc_debug_max_rate_set(void *data, u64 rate)
 	if (!tegra124_emc_validate_rate(emc, rate))
 		return -EINVAL;
 
-	err = emc_set_max_rate(emc, rate, EMC_RATE_DEBUG);
+	err = tegra_emc_set_max_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG);
 	if (err < 0)
 		return err;
 
@@ -1327,7 +1233,7 @@ static int emc_icc_set(struct icc_node *src, struct icc_node *dst)
 	do_div(rate, ddr * dram_data_bus_width_bytes);
 	rate = min_t(u64, rate, U32_MAX);
 
-	err = emc_set_min_rate(emc, rate, EMC_RATE_ICC);
+	err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_ICC);
 	if (err)
 		return err;
 
@@ -1441,7 +1347,6 @@ static int tegra124_emc_probe(struct platform_device *pdev)
 	if (!emc)
 		return -ENOMEM;
 
-	mutex_init(&emc->rate_lock);
 	emc->dev = &pdev->dev;
 
 	emc->regs = devm_platform_ioremap_resource(pdev, 0);
@@ -1487,7 +1392,7 @@ static int tegra124_emc_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	tegra124_emc_rate_requests_init(emc);
+	tegra_emc_rate_requests_init(&emc->reqs, &pdev->dev);
 
 	if (IS_ENABLED(CONFIG_DEBUG_FS))
 		emc_debugfs_init(&pdev->dev, emc);
diff --git a/drivers/memory/tegra/tegra186.c b/drivers/memory/tegra/tegra186.c
index 91d56165605f6..579d058da220c 100644
--- a/drivers/memory/tegra/tegra186.c
+++ b/drivers/memory/tegra/tegra186.c
@@ -154,7 +154,7 @@ static int tegra186_mc_probe_device(struct tegra_mc *mc, struct device *dev)
 	return 0;
 }
 
-static int tegra186_mc_resume(struct tegra_mc *mc)
+static void tegra186_mc_resume(struct tegra_mc *mc)
 {
 #if IS_ENABLED(CONFIG_IOMMU_API)
 	unsigned int i;
@@ -165,8 +165,6 @@ static int tegra186_mc_resume(struct tegra_mc *mc)
 		tegra186_mc_client_sid_override(mc, client, client->sid);
 	}
 #endif
-
-	return 0;
 }
 
 const struct tegra_mc_ops tegra186_mc_ops = {
diff --git a/drivers/memory/tegra/tegra20-emc.c b/drivers/memory/tegra/tegra20-emc.c
index a1fadefee7fdb..1d564b80e2bb9 100644
--- a/drivers/memory/tegra/tegra20-emc.c
+++ b/drivers/memory/tegra/tegra20-emc.c
@@ -32,6 +32,7 @@
 #include "../of_memory.h"
 
 #include "mc.h"
+#include "tegra-emc-common.h"
 
 #define EMC_INTSTATUS				0x000
 #define EMC_INTMASK				0x004
@@ -182,18 +183,6 @@ struct emc_timing {
 	u32 data[ARRAY_SIZE(emc_timing_registers)];
 };
 
-enum emc_rate_request_type {
-	EMC_RATE_DEVFREQ,
-	EMC_RATE_DEBUG,
-	EMC_RATE_ICC,
-	EMC_RATE_TYPE_MAX,
-};
-
-struct emc_rate_request {
-	unsigned long min_rate;
-	unsigned long max_rate;
-};
-
 struct tegra_emc {
 	struct device *dev;
 	struct tegra_mc *mc;
@@ -212,14 +201,7 @@ struct tegra_emc {
 		unsigned long max_rate;
 	} debugfs;
 
-	/*
-	 * There are multiple sources in the EMC driver which could request
-	 * a min/max clock rate, these rates are contained in this array.
-	 */
-	struct emc_rate_request requested_rate[EMC_RATE_TYPE_MAX];
-
-	/* protect shared rate-change code path */
-	struct mutex rate_lock;
+	struct tegra_emc_rate_requests reqs;
 
 	struct devfreq_simple_ondemand_data ondemand_data;
 
@@ -710,83 +692,6 @@ static long emc_round_rate(unsigned long rate,
 	return timing->rate;
 }
 
-static void tegra20_emc_rate_requests_init(struct tegra_emc *emc)
-{
-	unsigned int i;
-
-	for (i = 0; i < EMC_RATE_TYPE_MAX; i++) {
-		emc->requested_rate[i].min_rate = 0;
-		emc->requested_rate[i].max_rate = ULONG_MAX;
-	}
-}
-
-static int emc_request_rate(struct tegra_emc *emc,
-			    unsigned long new_min_rate,
-			    unsigned long new_max_rate,
-			    enum emc_rate_request_type type)
-{
-	struct emc_rate_request *req = emc->requested_rate;
-	unsigned long min_rate = 0, max_rate = ULONG_MAX;
-	unsigned int i;
-	int err;
-
-	/* select minimum and maximum rates among the requested rates */
-	for (i = 0; i < EMC_RATE_TYPE_MAX; i++, req++) {
-		if (i == type) {
-			min_rate = max(new_min_rate, min_rate);
-			max_rate = min(new_max_rate, max_rate);
-		} else {
-			min_rate = max(req->min_rate, min_rate);
-			max_rate = min(req->max_rate, max_rate);
-		}
-	}
-
-	if (min_rate > max_rate) {
-		dev_err_ratelimited(emc->dev, "%s: type %u: out of range: %lu %lu\n",
-				    __func__, type, min_rate, max_rate);
-		return -ERANGE;
-	}
-
-	/*
-	 * EMC rate-changes should go via OPP API because it manages voltage
-	 * changes.
-	 */
-	err = dev_pm_opp_set_rate(emc->dev, min_rate);
-	if (err)
-		return err;
-
-	emc->requested_rate[type].min_rate = new_min_rate;
-	emc->requested_rate[type].max_rate = new_max_rate;
-
-	return 0;
-}
-
-static int emc_set_min_rate(struct tegra_emc *emc, unsigned long rate,
-			    enum emc_rate_request_type type)
-{
-	struct emc_rate_request *req = &emc->requested_rate[type];
-	int ret;
-
-	mutex_lock(&emc->rate_lock);
-	ret = emc_request_rate(emc, rate, req->max_rate, type);
-	mutex_unlock(&emc->rate_lock);
-
-	return ret;
-}
-
-static int emc_set_max_rate(struct tegra_emc *emc, unsigned long rate,
-			    enum emc_rate_request_type type)
-{
-	struct emc_rate_request *req = &emc->requested_rate[type];
-	int ret;
-
-	mutex_lock(&emc->rate_lock);
-	ret = emc_request_rate(emc, req->min_rate, rate, type);
-	mutex_unlock(&emc->rate_lock);
-
-	return ret;
-}
-
 /*
  * debugfs interface
  *
@@ -857,7 +762,7 @@ static int tegra20_emc_debug_min_rate_set(void *data, u64 rate)
 	if (!tegra20_emc_validate_rate(emc, rate))
 		return -EINVAL;
 
-	err = emc_set_min_rate(emc, rate, EMC_RATE_DEBUG);
+	err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG);
 	if (err < 0)
 		return err;
 
@@ -887,7 +792,7 @@ static int tegra20_emc_debug_max_rate_set(void *data, u64 rate)
 	if (!tegra20_emc_validate_rate(emc, rate))
 		return -EINVAL;
 
-	err = emc_set_max_rate(emc, rate, EMC_RATE_DEBUG);
+	err = tegra_emc_set_max_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG);
 	if (err < 0)
 		return err;
 
@@ -993,7 +898,7 @@ static int emc_icc_set(struct icc_node *src, struct icc_node *dst)
 	do_div(rate, dram_data_bus_width_bytes);
 	rate = min_t(u64, rate, U32_MAX);
 
-	err = emc_set_min_rate(emc, rate, EMC_RATE_ICC);
+	err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_ICC);
 	if (err)
 		return err;
 
@@ -1111,7 +1016,7 @@ static int tegra20_emc_devfreq_target(struct device *dev, unsigned long *freq,
 	rate = dev_pm_opp_get_freq(opp);
 	dev_pm_opp_put(opp);
 
-	return emc_set_min_rate(emc, rate, EMC_RATE_DEVFREQ);
+	return tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEVFREQ);
 }
 
 static int tegra20_emc_devfreq_get_dev_status(struct device *dev,
@@ -1190,7 +1095,6 @@ static int tegra20_emc_probe(struct platform_device *pdev)
 	if (!emc)
 		return -ENOMEM;
 
-	mutex_init(&emc->rate_lock);
 	emc->clk_nb.notifier_call = tegra20_emc_clk_change_notify;
 	emc->dev = &pdev->dev;
 
@@ -1228,7 +1132,7 @@ static int tegra20_emc_probe(struct platform_device *pdev)
 		return err;
 
 	platform_set_drvdata(pdev, emc);
-	tegra20_emc_rate_requests_init(emc);
+	tegra_emc_rate_requests_init(&emc->reqs, &pdev->dev);
 	tegra20_emc_debugfs_init(emc);
 	tegra20_emc_interconnect_init(emc);
 	tegra20_emc_devfreq_init(emc);
diff --git a/drivers/memory/tegra/tegra234.c b/drivers/memory/tegra/tegra234.c
index 87b22038a5fbf..9fbd34d4abe04 100644
--- a/drivers/memory/tegra/tegra234.c
+++ b/drivers/memory/tegra/tegra234.c
@@ -1125,7 +1125,7 @@ static int tegra234_mc_icc_get_init_bw(struct icc_node *node, u32 *avg, u32 *pea
 	return 0;
 }
 
-static const struct tegra_mc_icc_ops tegra234_mc_icc_ops = {
+const struct tegra_mc_icc_ops tegra234_mc_icc_ops = {
 	.xlate = tegra_mc_icc_xlate,
 	.aggregate = tegra234_mc_icc_aggregate,
 	.get_bw = tegra234_mc_icc_get_init_bw,
diff --git a/drivers/memory/tegra/tegra238.c b/drivers/memory/tegra/tegra238.c
new file mode 100644
index 0000000000000..938c8985600e6
--- /dev/null
+++ b/drivers/memory/tegra/tegra238.c
@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2026, NVIDIA CORPORATION.  All rights reserved.
+ */
+
+#include <soc/tegra/mc.h>
+
+#include <dt-bindings/memory/tegra234-mc.h>
+#include <dt-bindings/memory/nvidia,tegra238-mc.h>
+#include <linux/interconnect.h>
+#include <linux/tegra-icc.h>
+
+#include <soc/tegra/bpmp.h>
+#include "mc.h"
+
+static const struct tegra_mc_client tegra238_mc_clients[] = {
+	{
+		.id = TEGRA234_MEMORY_CLIENT_HDAR,
+		.name = "hdar",
+		.bpmp_id = TEGRA_ICC_BPMP_HDA,
+		.type = TEGRA_ICC_ISO_AUDIO,
+		.sid = TEGRA238_SID_HDA,
+		.regs = {
+			.sid = {
+				.override = 0xa8,
+				.security = 0xac,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_HDAW,
+		.name = "hdaw",
+		.bpmp_id = TEGRA_ICC_BPMP_HDA,
+		.type = TEGRA_ICC_ISO_AUDIO,
+		.sid = TEGRA238_SID_HDA,
+		.regs = {
+			.sid = {
+				.override = 0x1a8,
+				.security = 0x1ac,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_SDMMCRAB,
+		.name = "sdmmcrab",
+		.bpmp_id = TEGRA_ICC_BPMP_SDMMC_4,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_SDMMC4A,
+		.regs = {
+			.sid = {
+				.override = 0x318,
+				.security = 0x31c,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_SDMMCWAB,
+		.name = "sdmmcwab",
+		.bpmp_id = TEGRA_ICC_BPMP_SDMMC_4,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_SDMMC4A,
+		.regs = {
+			.sid = {
+				.override = 0x338,
+				.security = 0x33c,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_APER,
+		.name = "aper",
+		.bpmp_id = TEGRA_ICC_BPMP_APE,
+		.type = TEGRA_ICC_ISO_AUDIO,
+		.sid = TEGRA238_SID_ISO_APE0,
+		.regs = {
+			.sid = {
+				.override = 0x3d0,
+				.security = 0x3d4,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_APEW,
+		.name = "apew",
+		.bpmp_id = TEGRA_ICC_BPMP_APE,
+		.type = TEGRA_ICC_ISO_AUDIO,
+		.sid = TEGRA238_SID_ISO_APE0,
+		.regs = {
+			.sid = {
+				.override = 0x3d8,
+				.security = 0x3dc,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_NVDISPLAYR,
+		.name = "nvdisplayr",
+		.bpmp_id = TEGRA_ICC_BPMP_DISPLAY,
+		.type = TEGRA_ICC_ISO_DISPLAY,
+		.sid = TEGRA238_SID_ISO_NVDISPLAY,
+		.regs = {
+			.sid = {
+				.override = 0x490,
+				.security = 0x494,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_NVDISPLAYR1,
+		.name = "nvdisplayr1",
+		.bpmp_id = TEGRA_ICC_BPMP_DISPLAY,
+		.type = TEGRA_ICC_ISO_DISPLAY,
+		.sid = TEGRA238_SID_ISO_NVDISPLAY,
+		.regs = {
+			.sid = {
+				.override = 0x508,
+				.security = 0x50c,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_BPMPR,
+		.name = "bpmpr",
+		.sid = TEGRA238_SID_BPMP,
+		.regs = {
+			.sid = {
+				.override = 0x498,
+				.security = 0x49c,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_BPMPW,
+		.name = "bpmpw",
+		.sid = TEGRA238_SID_BPMP,
+		.regs = {
+			.sid = {
+				.override = 0x4a0,
+				.security = 0x4a4,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_BPMPDMAR,
+		.name = "bpmpdmar",
+		.sid = TEGRA238_SID_BPMP,
+		.regs = {
+			.sid = {
+				.override = 0x4a8,
+				.security = 0x4ac,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_BPMPDMAW,
+		.name = "bpmpdmaw",
+		.sid = TEGRA238_SID_BPMP,
+		.regs = {
+			.sid = {
+				.override = 0x4b0,
+				.security = 0x4b4,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_APEDMAR,
+		.name = "apedmar",
+		.bpmp_id = TEGRA_ICC_BPMP_APEDMA,
+		.type = TEGRA_ICC_ISO_AUDIO,
+		.sid = TEGRA238_SID_ISO_APE1,
+		.regs = {
+			.sid = {
+				.override = 0x4f8,
+				.security = 0x4fc,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_APEDMAW,
+		.name = "apedmaw",
+		.bpmp_id = TEGRA_ICC_BPMP_APEDMA,
+		.type = TEGRA_ICC_ISO_AUDIO,
+		.sid = TEGRA238_SID_ISO_APE1,
+		.regs = {
+			.sid = {
+				.override = 0x500,
+				.security = 0x504,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_VICSRD,
+		.name = "vicsrd",
+		.bpmp_id = TEGRA_ICC_BPMP_VIC,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_VIC,
+		.regs = {
+			.sid = {
+				.override = 0x360,
+				.security = 0x364,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_VICSWR,
+		.name = "vicswr",
+		.bpmp_id = TEGRA_ICC_BPMP_VIC,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_VIC,
+		.regs = {
+			.sid = {
+				.override = 0x368,
+				.security = 0x36c,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_NVDECSRD,
+		.name = "nvdecsrd",
+		.bpmp_id = TEGRA_ICC_BPMP_NVDEC,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_NVDEC,
+		.regs = {
+			.sid = {
+				.override = 0x3c0,
+				.security = 0x3c4,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_NVDECSWR,
+		.name = "nvdecswr",
+		.bpmp_id = TEGRA_ICC_BPMP_NVDEC,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_NVDEC,
+		.regs = {
+			.sid = {
+				.override = 0x3c8,
+				.security = 0x3cc,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_NVENCSRD,
+		.name = "nvencsrd",
+		.bpmp_id = TEGRA_ICC_BPMP_NVENC,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_NVENC,
+		.regs = {
+			.sid = {
+				.override = 0xe0,
+				.security = 0xe4,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_NVENCSWR,
+		.name = "nvencswr",
+		.bpmp_id = TEGRA_ICC_BPMP_NVENC,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_NVENC,
+		.regs = {
+			.sid = {
+				.override = 0x158,
+				.security = 0x15c,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_PCIE0R,
+		.name = "pcie0r",
+		.bpmp_id = TEGRA_ICC_BPMP_PCIE_0,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_PCIE0,
+		.regs = {
+			.sid = {
+				.override = 0x6c0,
+				.security = 0x6c4,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_PCIE0W,
+		.name = "pcie0w",
+		.bpmp_id = TEGRA_ICC_BPMP_PCIE_0,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_PCIE0,
+		.regs = {
+			.sid = {
+				.override = 0x6c8,
+				.security = 0x6cc,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_PCIE1R,
+		.name = "pcie1r",
+		.bpmp_id = TEGRA_ICC_BPMP_PCIE_1,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_PCIE1,
+		.regs = {
+			.sid = {
+				.override = 0x6d0,
+				.security = 0x6d4,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_PCIE1W,
+		.name = "pcie1w",
+		.bpmp_id = TEGRA_ICC_BPMP_PCIE_1,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_PCIE1,
+		.regs = {
+			.sid = {
+				.override = 0x6d8,
+				.security = 0x6dc,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_PCIE2AR,
+		.name = "pcie2ar",
+		.bpmp_id = TEGRA_ICC_BPMP_PCIE_2,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_PCIE2,
+		.regs = {
+			.sid = {
+				.override = 0x6e0,
+				.security = 0x6e4,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_PCIE2AW,
+		.name = "pcie2aw",
+		.bpmp_id = TEGRA_ICC_BPMP_PCIE_2,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_PCIE2,
+		.regs = {
+			.sid = {
+				.override = 0x6e8,
+				.security = 0x6ec,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_PCIE3R,
+		.name = "pcie3r",
+		.bpmp_id = TEGRA_ICC_BPMP_PCIE_3,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_PCIE3,
+		.regs = {
+			.sid = {
+				.override = 0x6f0,
+				.security = 0x6f4,
+			},
+		},
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_PCIE3W,
+		.name = "pcie3w",
+		.bpmp_id = TEGRA_ICC_BPMP_PCIE_3,
+		.type = TEGRA_ICC_NISO,
+		.sid = TEGRA238_SID_PCIE3,
+		.regs = {
+			.sid = {
+				.override = 0x6f8,
+				.security = 0x6fc,
+			},
+		},
+	}, {
+		.id = TEGRA_ICC_MC_CPU_CLUSTER0,
+		.name = "sw_cluster0",
+		.bpmp_id = TEGRA_ICC_BPMP_CPU_CLUSTER0,
+		.type = TEGRA_ICC_NISO,
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_NVL1R,
+		.name = "nvl1r",
+		.bpmp_id = TEGRA_ICC_BPMP_GPU,
+		.type = TEGRA_ICC_NISO,
+	}, {
+		.id = TEGRA234_MEMORY_CLIENT_NVL1W,
+		.name = "nvl1w",
+		.bpmp_id = TEGRA_ICC_BPMP_GPU,
+		.type = TEGRA_ICC_NISO,
+	}
+};
+
+static const struct tegra_mc_intmask tegra238_mc_intmasks[] = {
+	{
+		.reg = MC_INTMASK,
+		.mask = MC_INT_DECERR_ROUTE_SANITY | MC_INT_DECERR_GENERALIZED_CARVEOUT |
+			MC_INT_DECERR_MTS | MC_INT_SECERR_SEC | MC_INT_DECERR_VPR |
+			MC_INT_SECURITY_VIOLATION | MC_INT_DECERR_EMEM,
+	},
+};
+
+const struct tegra_mc_soc tegra238_mc_soc = {
+	.num_clients = ARRAY_SIZE(tegra238_mc_clients),
+	.clients = tegra238_mc_clients,
+	.num_address_bits = 40,
+	.num_channels = 8,
+	.client_id_mask = 0x1ff,
+	.intmasks = tegra238_mc_intmasks,
+	.num_intmasks = ARRAY_SIZE(tegra238_mc_intmasks),
+	.has_addr_hi_reg = true,
+	.ops = &tegra186_mc_ops,
+	.icc_ops = &tegra234_mc_icc_ops,
+	.ch_intmask = 0x0000ff00,
+	.global_intstatus_channel_shift = 8,
+	.num_carveouts = 32,
+	.regs = &tegra20_mc_regs,
+	.handle_irq = tegra30_mc_irq_handlers,
+	.num_interrupts = ARRAY_SIZE(tegra30_mc_irq_handlers),
+	.mc_addr_hi_mask = 0x3,
+	.mc_err_status_type_mask = (0x7 << 28),
+};
diff --git a/drivers/memory/tegra/tegra30-emc.c b/drivers/memory/tegra/tegra30-emc.c
index 5812c8cd6ce4f..c1eacd66ce49f 100644
--- a/drivers/memory/tegra/tegra30-emc.c
+++ b/drivers/memory/tegra/tegra30-emc.c
@@ -36,6 +36,7 @@
 #include "../of_memory.h"
 
 #include "mc.h"
+#include "tegra-emc-common.h"
 
 #define EMC_INTSTATUS				0x000
 #define EMC_INTMASK				0x004
@@ -341,17 +342,6 @@ struct emc_timing {
 	bool emc_cfg_dyn_self_ref;
 };
 
-enum emc_rate_request_type {
-	EMC_RATE_DEBUG,
-	EMC_RATE_ICC,
-	EMC_RATE_TYPE_MAX,
-};
-
-struct emc_rate_request {
-	unsigned long min_rate;
-	unsigned long max_rate;
-};
-
 struct tegra_emc {
 	struct device *dev;
 	struct tegra_mc *mc;
@@ -383,14 +373,7 @@ struct tegra_emc {
 		unsigned long max_rate;
 	} debugfs;
 
-	/*
-	 * There are multiple sources in the EMC driver which could request
-	 * a min/max clock rate, these rates are contained in this array.
-	 */
-	struct emc_rate_request requested_rate[EMC_RATE_TYPE_MAX];
-
-	/* protect shared rate-change code path */
-	struct mutex rate_lock;
+	struct tegra_emc_rate_requests reqs;
 
 	bool mrr_error;
 };
@@ -1228,83 +1211,6 @@ static long emc_round_rate(unsigned long rate,
 	return timing->rate;
 }
 
-static void tegra30_emc_rate_requests_init(struct tegra_emc *emc)
-{
-	unsigned int i;
-
-	for (i = 0; i < EMC_RATE_TYPE_MAX; i++) {
-		emc->requested_rate[i].min_rate = 0;
-		emc->requested_rate[i].max_rate = ULONG_MAX;
-	}
-}
-
-static int emc_request_rate(struct tegra_emc *emc,
-			    unsigned long new_min_rate,
-			    unsigned long new_max_rate,
-			    enum emc_rate_request_type type)
-{
-	struct emc_rate_request *req = emc->requested_rate;
-	unsigned long min_rate = 0, max_rate = ULONG_MAX;
-	unsigned int i;
-	int err;
-
-	/* select minimum and maximum rates among the requested rates */
-	for (i = 0; i < EMC_RATE_TYPE_MAX; i++, req++) {
-		if (i == type) {
-			min_rate = max(new_min_rate, min_rate);
-			max_rate = min(new_max_rate, max_rate);
-		} else {
-			min_rate = max(req->min_rate, min_rate);
-			max_rate = min(req->max_rate, max_rate);
-		}
-	}
-
-	if (min_rate > max_rate) {
-		dev_err_ratelimited(emc->dev, "%s: type %u: out of range: %lu %lu\n",
-				    __func__, type, min_rate, max_rate);
-		return -ERANGE;
-	}
-
-	/*
-	 * EMC rate-changes should go via OPP API because it manages voltage
-	 * changes.
-	 */
-	err = dev_pm_opp_set_rate(emc->dev, min_rate);
-	if (err)
-		return err;
-
-	emc->requested_rate[type].min_rate = new_min_rate;
-	emc->requested_rate[type].max_rate = new_max_rate;
-
-	return 0;
-}
-
-static int emc_set_min_rate(struct tegra_emc *emc, unsigned long rate,
-			    enum emc_rate_request_type type)
-{
-	struct emc_rate_request *req = &emc->requested_rate[type];
-	int ret;
-
-	mutex_lock(&emc->rate_lock);
-	ret = emc_request_rate(emc, rate, req->max_rate, type);
-	mutex_unlock(&emc->rate_lock);
-
-	return ret;
-}
-
-static int emc_set_max_rate(struct tegra_emc *emc, unsigned long rate,
-			    enum emc_rate_request_type type)
-{
-	struct emc_rate_request *req = &emc->requested_rate[type];
-	int ret;
-
-	mutex_lock(&emc->rate_lock);
-	ret = emc_request_rate(emc, req->min_rate, rate, type);
-	mutex_unlock(&emc->rate_lock);
-
-	return ret;
-}
-
 /*
  * debugfs interface
  *
@@ -1375,7 +1281,7 @@ static int tegra30_emc_debug_min_rate_set(void *data, u64 rate)
 	if (!tegra30_emc_validate_rate(emc, rate))
 		return -EINVAL;
 
-	err = emc_set_min_rate(emc, rate, EMC_RATE_DEBUG);
+	err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG);
 	if (err < 0)
 		return err;
 
@@ -1405,7 +1311,7 @@ static int tegra30_emc_debug_max_rate_set(void *data, u64 rate)
 	if (!tegra30_emc_validate_rate(emc, rate))
 		return -EINVAL;
 
-	err = emc_set_max_rate(emc, rate, EMC_RATE_DEBUG);
+	err = tegra_emc_set_max_rate(&emc->reqs, rate, TEGRA_EMC_RATE_DEBUG);
 	if (err < 0)
 		return err;
 
@@ -1511,7 +1417,7 @@ static int emc_icc_set(struct icc_node *src, struct icc_node *dst)
 	do_div(rate, ddr * dram_data_bus_width_bytes);
 	rate = min_t(u64, rate, U32_MAX);
 
-	err = emc_set_min_rate(emc, rate, EMC_RATE_ICC);
+	err = tegra_emc_set_min_rate(&emc->reqs, rate, TEGRA_EMC_RATE_ICC);
 	if (err)
 		return err;
 
@@ -1622,7 +1528,6 @@ static int tegra30_emc_probe(struct platform_device *pdev)
 	if (IS_ERR(emc->mc))
 		return PTR_ERR(emc->mc);
 
-	mutex_init(&emc->rate_lock);
 	emc->clk_nb.notifier_call = emc_clk_change_notify;
 	emc->dev = &pdev->dev;
 
@@ -1664,7 +1569,7 @@ static int tegra30_emc_probe(struct platform_device *pdev)
 		return err;
 
 	platform_set_drvdata(pdev, emc);
-	tegra30_emc_rate_requests_init(emc);
+	tegra_emc_rate_requests_init(&emc->reqs, &pdev->dev);
 	tegra30_emc_debugfs_init(emc);
 	tegra30_emc_interconnect_init(emc);
 
diff --git a/drivers/memstick/host/tifm_ms.c b/drivers/memstick/host/tifm_ms.c
index 0b6a90661eee5..0a54586aa54e8 100644
--- a/drivers/memstick/host/tifm_ms.c
+++ b/drivers/memstick/host/tifm_ms.c
@@ -647,7 +647,7 @@ static int tifm_ms_resume(struct tifm_dev *sock)
 
 #endif /* CONFIG_PM */
 
-static struct tifm_device_id tifm_ms_id_tbl[] = {
+static const struct tifm_device_id tifm_ms_id_tbl[] = {
 	{ TIFM_TYPE_MS }, { 0 }
 };
 
diff --git a/drivers/mfd/Kconfig b/drivers/mfd/Kconfig
index 7192c9d1d268e..763ce6a34782b 100644
--- a/drivers/mfd/Kconfig
+++ b/drivers/mfd/Kconfig
@@ -1199,13 +1199,6 @@ config MFD_OCELOT
 
 	  If unsure, say N.
 
-config EZX_PCAP
-	bool "Motorola EZXPCAP Support"
-	depends on SPI_MASTER
-	help
-	  This enables the PCAP ASIC present on EZX Phones. This is
-	  needed for MMC, TouchScreen, Sound, USB, etc..
-
 config MFD_CPCAP
 	tristate "Support for Motorola CPCAP"
 	depends on SPI
@@ -2388,7 +2381,7 @@ config MFD_ACER_A500_EC
 	  customized for the specific needs of the Acer A500 hardware.
 
 config MFD_QCOM_PM8008
-	tristate "QCOM PM8008 Power Management IC"
+	tristate "Qualcomm PM8008 Power Management IC"
 	depends on I2C && OF
 	select MFD_CORE
 	select REGMAP_I2C
diff --git a/drivers/mfd/Makefile b/drivers/mfd/Makefile
index e75e8045c28af..dd4bb7e77c336 100644
--- a/drivers/mfd/Makefile
+++ b/drivers/mfd/Makefile
@@ -131,7 +131,6 @@ obj-$(CONFIG_MFD_CORE)		+= mfd-core.o
 ocelot-soc-objs			:= ocelot-core.o ocelot-spi.o
 obj-$(CONFIG_MFD_OCELOT)	+= ocelot-soc.o
 
-obj-$(CONFIG_EZX_PCAP)		+= ezx-pcap.o
 obj-$(CONFIG_MFD_CPCAP)		+= motorola-cpcap.o
 
 obj-$(CONFIG_MCP)		+= mcp-core.o
diff --git a/drivers/mfd/bq257xx.c b/drivers/mfd/bq257xx.c
index e9d49dac0a167..054342c60b731 100644
--- a/drivers/mfd/bq257xx.c
+++ b/drivers/mfd/bq257xx.c
@@ -39,6 +39,39 @@ static const struct regmap_config bq25703_regmap_config = {
 	.val_format_endian = REGMAP_ENDIAN_LITTLE,
 };
 
+static const struct regmap_range bq25792_writeable_reg_ranges[] = {
+	regmap_reg_range(BQ25792_REG00_MIN_SYS_VOLTAGE,
+			 BQ25792_REG18_NTC_CONTROL_1),
+	regmap_reg_range(BQ25792_REG28_CHARGER_MASK_0,
+			 BQ25792_REG30_ADC_FUNCTION_DISABLE_1),
+};
+
+static const struct regmap_access_table bq25792_writeable_regs = {
+	.yes_ranges = bq25792_writeable_reg_ranges,
+	.n_yes_ranges = ARRAY_SIZE(bq25792_writeable_reg_ranges),
+};
+
+static const struct regmap_range bq25792_volatile_reg_ranges[] = {
+	regmap_reg_range(BQ25792_REG19_ICO_CURRENT_LIMIT,
+			 BQ25792_REG27_FAULT_FLAG_1),
+	regmap_reg_range(BQ25792_REG31_IBUS_ADC,
+			 BQ25792_REG47_DPDM_DRIVER),
+};
+
+static const struct regmap_access_table bq25792_volatile_regs = {
+	.yes_ranges = bq25792_volatile_reg_ranges,
+	.n_yes_ranges = ARRAY_SIZE(bq25792_volatile_reg_ranges),
+};
+
+static const struct regmap_config bq25792_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = BQ25792_REG48_PART_INFORMATION,
+	.cache_type = REGCACHE_MAPLE,
+	.wr_table = &bq25792_writeable_regs,
+	.volatile_table = &bq25792_volatile_regs,
+};
+
 static const struct mfd_cell cells[] = {
 	MFD_CELL_NAME("bq257xx-regulator"),
 	MFD_CELL_NAME("bq257xx-charger"),
@@ -46,6 +79,7 @@ static const struct mfd_cell cells[] = {
 
 static int bq257xx_probe(struct i2c_client *client)
 {
+	const struct regmap_config *rcfg;
 	struct bq257xx_device *ddata;
 	int ret;
 
@@ -53,9 +87,21 @@ static int bq257xx_probe(struct i2c_client *client)
 	if (!ddata)
 		return -ENOMEM;
 
+	ddata->type = (uintptr_t)i2c_get_match_data(client);
 	ddata->client = client;
 
-	ddata->regmap = devm_regmap_init_i2c(client, &bq25703_regmap_config);
+	switch (ddata->type) {
+	case BQ25703A:
+		rcfg = &bq25703_regmap_config;
+		break;
+	case BQ25792:
+		rcfg = &bq25792_regmap_config;
+		break;
+	default:
+		return dev_err_probe(&client->dev, -ENODEV, "Unsupported device type\n");
+	}
+
+	ddata->regmap = devm_regmap_init_i2c(client, rcfg);
 	if (IS_ERR(ddata->regmap)) {
 		return dev_err_probe(&client->dev, PTR_ERR(ddata->regmap),
 				     "Failed to allocate register map\n");
@@ -73,13 +119,15 @@ static int bq257xx_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id bq257xx_i2c_ids[] = {
-	{ "bq25703a" },
+	{ "bq25703a", BQ25703A },
+	{ "bq25792", BQ25792 },
 	{}
 };
 MODULE_DEVICE_TABLE(i2c, bq257xx_i2c_ids);
 
 static const struct of_device_id bq257xx_of_match[] = {
-	{ .compatible = "ti,bq25703a" },
+	{ .compatible = "ti,bq25703a", .data = (void *)BQ25703A },
+	{ .compatible = "ti,bq25792", .data = (void *)BQ25792 },
 	{}
 };
 MODULE_DEVICE_TABLE(of, bq257xx_of_match);
diff --git a/drivers/mfd/cros_ec_dev.c b/drivers/mfd/cros_ec_dev.c
index 39430dd44e30c..11ee1146cf716 100644
--- a/drivers/mfd/cros_ec_dev.c
+++ b/drivers/mfd/cros_ec_dev.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2014 Google, Inc.
  */
 
+#include <linux/acpi.h>
 #include <linux/dmi.h>
 #include <linux/kconfig.h>
 #include <linux/mfd/core.h>
@@ -131,11 +132,6 @@ static const struct cros_feature_to_cells cros_subdevices[] = {
 		.mfd_cells	= cros_ec_rtc_cells,
 		.num_cells	= ARRAY_SIZE(cros_ec_rtc_cells),
 	},
-	{
-		.id		= EC_FEATURE_UCSI_PPM,
-		.mfd_cells	= cros_ec_ucsi_cells,
-		.num_cells	= ARRAY_SIZE(cros_ec_ucsi_cells),
-	},
 	{
 		.id		= EC_FEATURE_HANG_DETECT,
 		.mfd_cells	= cros_ec_wdt_cells,
@@ -195,7 +191,6 @@ static int ec_device_probe(struct platform_device *pdev)
 	if (!ec)
 		return retval;
 
-	dev_set_drvdata(dev, ec);
 	ec->ec_dev = dev_get_drvdata(dev->parent);
 	ec->dev = dev;
 	ec->cmd_offset = ec_platform->cmd_offset;
@@ -237,6 +232,8 @@ static int ec_device_probe(struct platform_device *pdev)
 	if (retval)
 		goto failed;
 
+	dev_set_drvdata(dev, ec);
+
 	/* check whether this EC is a sensor hub. */
 	if (cros_ec_get_sensor_count(ec) > 0) {
 		retval = mfd_add_hotplug_devices(ec->dev,
@@ -264,6 +261,23 @@ static int ec_device_probe(struct platform_device *pdev)
 		}
 	}
 
+	/*
+	 * FW nodes can load cros_ec_ucsi, but early PDC devices did not define
+	 * the required nodes. On PDC systems without FW nodes for cros_ec_ucsi,
+	 * the driver should be added as an mfd subdevice.
+	 */
+	if (cros_ec_check_features(ec, EC_FEATURE_USB_PD) &&
+	    cros_ec_check_features(ec, EC_FEATURE_UCSI_PPM) &&
+	    !acpi_dev_found("GOOG0021") &&
+	    !of_find_compatible_node(NULL, NULL, "google,cros-ec-ucsi")) {
+		retval = mfd_add_hotplug_devices(ec->dev,
+						 cros_ec_ucsi_cells,
+						 ARRAY_SIZE(cros_ec_ucsi_cells));
+
+		if (retval)
+			dev_warn(ec->dev, "failed to add cros_ec_ucsi: %d\n", retval);
+	}
+
 	/*
 	 * UCSI provides power supply information so we don't need to separately
 	 * load the cros_usbpd_charger driver.
diff --git a/drivers/mfd/cs42l43.c b/drivers/mfd/cs42l43.c
index 166881751e698..ed6d93893de04 100644
--- a/drivers/mfd/cs42l43.c
+++ b/drivers/mfd/cs42l43.c
@@ -722,7 +722,7 @@ static void cs42l43_mcu_load_firmware(const struct firmware *firmware, void *con
 	unsigned int loadaddr, val;
 	int ret;
 
-	if (!firmware) {
+	if (!firmware || firmware->size < sizeof(*hdr)) {
 		dev_err(cs42l43->dev, "Failed to load firmware\n");
 		cs42l43->firmware_error = -ENODEV;
 		goto err;
diff --git a/drivers/mfd/cs5535-mfd.c b/drivers/mfd/cs5535-mfd.c
index d0fb2e52ee76a..f3becbef19f5d 100644
--- a/drivers/mfd/cs5535-mfd.c
+++ b/drivers/mfd/cs5535-mfd.c
@@ -12,8 +12,11 @@
 
 #include <linux/kernel.h>
 #include <linux/mfd/core.h>
+#include <linux/mfd/cs5535.h>
 #include <linux/module.h>
 #include <linux/pci.h>
+#include <linux/property.h>
+
 #include <asm/olpc.h>
 
 #define DRV_NAME "cs5535-mfd"
@@ -29,6 +32,11 @@ enum cs5535_mfd_bars {
 
 static struct resource cs5535_mfd_resources[NR_BARS];
 
+const struct software_node cs5535_gpio_swnode = {
+	.name = "cs5535-gpio",
+};
+EXPORT_SYMBOL_NS(cs5535_gpio_swnode, "CS5535");
+
 static struct mfd_cell cs5535_mfd_cells[] = {
 	{
 		.name = "cs5535-smb",
@@ -39,6 +47,7 @@ static struct mfd_cell cs5535_mfd_cells[] = {
 		.name = "cs5535-gpio",
 		.num_resources = 1,
 		.resources = &cs5535_mfd_resources[GPIO_BAR],
+		.swnode = &cs5535_gpio_swnode,
 	},
 	{
 		.name = "cs5535-mfgpt",
diff --git a/drivers/mfd/ezx-pcap.c b/drivers/mfd/ezx-pcap.c
deleted file mode 100644
index 9a685ff8cd157..0000000000000
--- a/drivers/mfd/ezx-pcap.c
+++ /dev/null
@@ -1,491 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Driver for Motorola PCAP2 as present in EZX phones
- *
- * Copyright (C) 2006 Harald Welte <laforge@openezx.org>
- * Copyright (C) 2009 Daniel Ribeiro <drwyrm@gmail.com>
- */
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/platform_device.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/mfd/ezx-pcap.h>
-#include <linux/spi/spi.h>
-#include <linux/gpio.h>
-#include <linux/slab.h>
-
-#define PCAP_ADC_MAXQ		8
-struct pcap_adc_request {
-	u8 bank;
-	u8 ch[2];
-	u32 flags;
-	void (*callback)(void *, u16[]);
-	void *data;
-};
-
-struct pcap_chip {
-	struct spi_device *spi;
-
-	/* IO */
-	u32 buf;
-	spinlock_t io_lock;
-
-	/* IRQ */
-	unsigned int irq_base;
-	u32 msr;
-	struct work_struct isr_work;
-	struct work_struct msr_work;
-	struct workqueue_struct *workqueue;
-
-	/* ADC */
-	struct pcap_adc_request *adc_queue[PCAP_ADC_MAXQ];
-	u8 adc_head;
-	u8 adc_tail;
-	spinlock_t adc_lock;
-};
-
-/* IO */
-static int ezx_pcap_putget(struct pcap_chip *pcap, u32 *data)
-{
-	struct spi_transfer t;
-	struct spi_message m;
-	int status;
-
-	memset(&t, 0, sizeof(t));
-	spi_message_init(&m);
-	t.len = sizeof(u32);
-	spi_message_add_tail(&t, &m);
-
-	pcap->buf = *data;
-	t.tx_buf = (u8 *) &pcap->buf;
-	t.rx_buf = (u8 *) &pcap->buf;
-	status = spi_sync(pcap->spi, &m);
-
-	if (status == 0)
-		*data = pcap->buf;
-
-	return status;
-}
-
-int ezx_pcap_write(struct pcap_chip *pcap, u8 reg_num, u32 value)
-{
-	unsigned long flags;
-	int ret;
-
-	spin_lock_irqsave(&pcap->io_lock, flags);
-	value &= PCAP_REGISTER_VALUE_MASK;
-	value |= PCAP_REGISTER_WRITE_OP_BIT
-		| (reg_num << PCAP_REGISTER_ADDRESS_SHIFT);
-	ret = ezx_pcap_putget(pcap, &value);
-	spin_unlock_irqrestore(&pcap->io_lock, flags);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(ezx_pcap_write);
-
-int ezx_pcap_read(struct pcap_chip *pcap, u8 reg_num, u32 *value)
-{
-	unsigned long flags;
-	int ret;
-
-	spin_lock_irqsave(&pcap->io_lock, flags);
-	*value = PCAP_REGISTER_READ_OP_BIT
-		| (reg_num << PCAP_REGISTER_ADDRESS_SHIFT);
-
-	ret = ezx_pcap_putget(pcap, value);
-	spin_unlock_irqrestore(&pcap->io_lock, flags);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(ezx_pcap_read);
-
-int ezx_pcap_set_bits(struct pcap_chip *pcap, u8 reg_num, u32 mask, u32 val)
-{
-	unsigned long flags;
-	int ret;
-	u32 tmp = PCAP_REGISTER_READ_OP_BIT |
-		(reg_num << PCAP_REGISTER_ADDRESS_SHIFT);
-
-	spin_lock_irqsave(&pcap->io_lock, flags);
-	ret = ezx_pcap_putget(pcap, &tmp);
-	if (ret)
-		goto out_unlock;
-
-	tmp &= (PCAP_REGISTER_VALUE_MASK & ~mask);
-	tmp |= (val & mask) | PCAP_REGISTER_WRITE_OP_BIT |
-		(reg_num << PCAP_REGISTER_ADDRESS_SHIFT);
-
-	ret = ezx_pcap_putget(pcap, &tmp);
-out_unlock:
-	spin_unlock_irqrestore(&pcap->io_lock, flags);
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(ezx_pcap_set_bits);
-
-/* IRQ */
-int irq_to_pcap(struct pcap_chip *pcap, int irq)
-{
-	return irq - pcap->irq_base;
-}
-EXPORT_SYMBOL_GPL(irq_to_pcap);
-
-int pcap_to_irq(struct pcap_chip *pcap, int irq)
-{
-	return pcap->irq_base + irq;
-}
-EXPORT_SYMBOL_GPL(pcap_to_irq);
-
-static void pcap_mask_irq(struct irq_data *d)
-{
-	struct pcap_chip *pcap = irq_data_get_irq_chip_data(d);
-
-	pcap->msr |= 1 << irq_to_pcap(pcap, d->irq);
-	queue_work(pcap->workqueue, &pcap->msr_work);
-}
-
-static void pcap_unmask_irq(struct irq_data *d)
-{
-	struct pcap_chip *pcap = irq_data_get_irq_chip_data(d);
-
-	pcap->msr &= ~(1 << irq_to_pcap(pcap, d->irq));
-	queue_work(pcap->workqueue, &pcap->msr_work);
-}
-
-static struct irq_chip pcap_irq_chip = {
-	.name		= "pcap",
-	.irq_disable	= pcap_mask_irq,
-	.irq_mask	= pcap_mask_irq,
-	.irq_unmask	= pcap_unmask_irq,
-};
-
-static void pcap_msr_work(struct work_struct *work)
-{
-	struct pcap_chip *pcap = container_of(work, struct pcap_chip, msr_work);
-
-	ezx_pcap_write(pcap, PCAP_REG_MSR, pcap->msr);
-}
-
-static void pcap_isr_work(struct work_struct *work)
-{
-	struct pcap_chip *pcap = container_of(work, struct pcap_chip, isr_work);
-	struct pcap_platform_data *pdata = dev_get_platdata(&pcap->spi->dev);
-	u32 msr, isr, int_sel, service;
-	int irq;
-
-	do {
-		ezx_pcap_read(pcap, PCAP_REG_MSR, &msr);
-		ezx_pcap_read(pcap, PCAP_REG_ISR, &isr);
-
-		/* We can't service/ack irqs that are assigned to port 2 */
-		if (!(pdata->config & PCAP_SECOND_PORT)) {
-			ezx_pcap_read(pcap, PCAP_REG_INT_SEL, &int_sel);
-			isr &= ~int_sel;
-		}
-
-		ezx_pcap_write(pcap, PCAP_REG_MSR, isr | msr);
-		ezx_pcap_write(pcap, PCAP_REG_ISR, isr);
-
-		service = isr & ~msr;
-		for (irq = pcap->irq_base; service; service >>= 1, irq++) {
-			if (service & 1)
-				generic_handle_irq_safe(irq);
-		}
-		ezx_pcap_write(pcap, PCAP_REG_MSR, pcap->msr);
-	} while (gpio_get_value(pdata->gpio));
-}
-
-static void pcap_irq_handler(struct irq_desc *desc)
-{
-	struct pcap_chip *pcap = irq_desc_get_handler_data(desc);
-
-	desc->irq_data.chip->irq_ack(&desc->irq_data);
-	queue_work(pcap->workqueue, &pcap->isr_work);
-}
-
-/* ADC */
-void pcap_set_ts_bits(struct pcap_chip *pcap, u32 bits)
-{
-	unsigned long flags;
-	u32 tmp;
-
-	spin_lock_irqsave(&pcap->adc_lock, flags);
-	ezx_pcap_read(pcap, PCAP_REG_ADC, &tmp);
-	tmp &= ~(PCAP_ADC_TS_M_MASK | PCAP_ADC_TS_REF_LOWPWR);
-	tmp |= bits & (PCAP_ADC_TS_M_MASK | PCAP_ADC_TS_REF_LOWPWR);
-	ezx_pcap_write(pcap, PCAP_REG_ADC, tmp);
-	spin_unlock_irqrestore(&pcap->adc_lock, flags);
-}
-EXPORT_SYMBOL_GPL(pcap_set_ts_bits);
-
-static void pcap_disable_adc(struct pcap_chip *pcap)
-{
-	u32 tmp;
-
-	ezx_pcap_read(pcap, PCAP_REG_ADC, &tmp);
-	tmp &= ~(PCAP_ADC_ADEN|PCAP_ADC_BATT_I_ADC|PCAP_ADC_BATT_I_POLARITY);
-	ezx_pcap_write(pcap, PCAP_REG_ADC, tmp);
-}
-
-static void pcap_adc_trigger(struct pcap_chip *pcap)
-{
-	unsigned long flags;
-	u32 tmp;
-	u8 head;
-
-	spin_lock_irqsave(&pcap->adc_lock, flags);
-	head = pcap->adc_head;
-	if (!pcap->adc_queue[head]) {
-		/* queue is empty, save power */
-		pcap_disable_adc(pcap);
-		spin_unlock_irqrestore(&pcap->adc_lock, flags);
-		return;
-	}
-	/* start conversion on requested bank, save TS_M bits */
-	ezx_pcap_read(pcap, PCAP_REG_ADC, &tmp);
-	tmp &= (PCAP_ADC_TS_M_MASK | PCAP_ADC_TS_REF_LOWPWR);
-	tmp |= pcap->adc_queue[head]->flags | PCAP_ADC_ADEN;
-
-	if (pcap->adc_queue[head]->bank == PCAP_ADC_BANK_1)
-		tmp |= PCAP_ADC_AD_SEL1;
-
-	ezx_pcap_write(pcap, PCAP_REG_ADC, tmp);
-	spin_unlock_irqrestore(&pcap->adc_lock, flags);
-	ezx_pcap_write(pcap, PCAP_REG_ADR, PCAP_ADR_ASC);
-}
-
-static irqreturn_t pcap_adc_irq(int irq, void *_pcap)
-{
-	struct pcap_chip *pcap = _pcap;
-	struct pcap_adc_request *req;
-	u16 res[2];
-	u32 tmp;
-
-	spin_lock(&pcap->adc_lock);
-	req = pcap->adc_queue[pcap->adc_head];
-
-	if (WARN(!req, "adc irq without pending request\n")) {
-		spin_unlock(&pcap->adc_lock);
-		return IRQ_HANDLED;
-	}
-
-	/* read requested channels results */
-	ezx_pcap_read(pcap, PCAP_REG_ADC, &tmp);
-	tmp &= ~(PCAP_ADC_ADA1_MASK | PCAP_ADC_ADA2_MASK);
-	tmp |= (req->ch[0] << PCAP_ADC_ADA1_SHIFT);
-	tmp |= (req->ch[1] << PCAP_ADC_ADA2_SHIFT);
-	ezx_pcap_write(pcap, PCAP_REG_ADC, tmp);
-	ezx_pcap_read(pcap, PCAP_REG_ADR, &tmp);
-	res[0] = (tmp & PCAP_ADR_ADD1_MASK) >> PCAP_ADR_ADD1_SHIFT;
-	res[1] = (tmp & PCAP_ADR_ADD2_MASK) >> PCAP_ADR_ADD2_SHIFT;
-
-	pcap->adc_queue[pcap->adc_head] = NULL;
-	pcap->adc_head = (pcap->adc_head + 1) & (PCAP_ADC_MAXQ - 1);
-	spin_unlock(&pcap->adc_lock);
-
-	/* pass the results and release memory */
-	req->callback(req->data, res);
-	kfree(req);
-
-	/* trigger next conversion (if any) on queue */
-	pcap_adc_trigger(pcap);
-
-	return IRQ_HANDLED;
-}
-
-int pcap_adc_async(struct pcap_chip *pcap, u8 bank, u32 flags, u8 ch[],
-						void *callback, void *data)
-{
-	struct pcap_adc_request *req;
-	unsigned long irq_flags;
-
-	/* This will be freed after we have a result */
-	req = kmalloc_obj(struct pcap_adc_request);
-	if (!req)
-		return -ENOMEM;
-
-	req->bank = bank;
-	req->flags = flags;
-	req->ch[0] = ch[0];
-	req->ch[1] = ch[1];
-	req->callback = callback;
-	req->data = data;
-
-	spin_lock_irqsave(&pcap->adc_lock, irq_flags);
-	if (pcap->adc_queue[pcap->adc_tail]) {
-		spin_unlock_irqrestore(&pcap->adc_lock, irq_flags);
-		kfree(req);
-		return -EBUSY;
-	}
-	pcap->adc_queue[pcap->adc_tail] = req;
-	pcap->adc_tail = (pcap->adc_tail + 1) & (PCAP_ADC_MAXQ - 1);
-	spin_unlock_irqrestore(&pcap->adc_lock, irq_flags);
-
-	/* start conversion */
-	pcap_adc_trigger(pcap);
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(pcap_adc_async);
-
-/* subdevs */
-static int pcap_remove_subdev(struct device *dev, void *unused)
-{
-	platform_device_unregister(to_platform_device(dev));
-	return 0;
-}
-
-static int pcap_add_subdev(struct pcap_chip *pcap,
-						struct pcap_subdev *subdev)
-{
-	struct platform_device *pdev;
-	int ret;
-
-	pdev = platform_device_alloc(subdev->name, subdev->id);
-	if (!pdev)
-		return -ENOMEM;
-
-	pdev->dev.parent = &pcap->spi->dev;
-	pdev->dev.platform_data = subdev->platform_data;
-
-	ret = platform_device_add(pdev);
-	if (ret)
-		platform_device_put(pdev);
-
-	return ret;
-}
-
-static void ezx_pcap_remove(struct spi_device *spi)
-{
-	struct pcap_chip *pcap = spi_get_drvdata(spi);
-	unsigned long flags;
-	int i;
-
-	/* remove all registered subdevs */
-	device_for_each_child(&spi->dev, NULL, pcap_remove_subdev);
-
-	/* cleanup ADC */
-	spin_lock_irqsave(&pcap->adc_lock, flags);
-	for (i = 0; i < PCAP_ADC_MAXQ; i++)
-		kfree(pcap->adc_queue[i]);
-	spin_unlock_irqrestore(&pcap->adc_lock, flags);
-
-	/* cleanup irqchip */
-	for (i = pcap->irq_base; i < (pcap->irq_base + PCAP_NIRQS); i++)
-		irq_set_chip_and_handler(i, NULL, NULL);
-}
-
-static int ezx_pcap_probe(struct spi_device *spi)
-{
-	struct pcap_platform_data *pdata = dev_get_platdata(&spi->dev);
-	struct pcap_chip *pcap;
-	int i, adc_irq;
-	int ret;
-
-	/* platform data is required */
-	if (!pdata)
-		return -ENODEV;
-
-	pcap = devm_kzalloc(&spi->dev, sizeof(*pcap), GFP_KERNEL);
-	if (!pcap)
-		return -ENOMEM;
-
-	spin_lock_init(&pcap->io_lock);
-	spin_lock_init(&pcap->adc_lock);
-	INIT_WORK(&pcap->isr_work, pcap_isr_work);
-	INIT_WORK(&pcap->msr_work, pcap_msr_work);
-	spi_set_drvdata(spi, pcap);
-
-	/* setup spi */
-	spi->bits_per_word = 32;
-	spi->mode = SPI_MODE_0 | (pdata->config & PCAP_CS_AH ? SPI_CS_HIGH : 0);
-	ret = spi_setup(spi);
-	if (ret)
-		return ret;
-
-	pcap->spi = spi;
-
-	/* setup irq */
-	pcap->irq_base = pdata->irq_base;
-	pcap->workqueue = devm_alloc_ordered_workqueue(&spi->dev, "pcapd", 0);
-	if (!pcap->workqueue)
-		return -ENOMEM;
-
-	/* redirect interrupts to AP, except adcdone2 */
-	if (!(pdata->config & PCAP_SECOND_PORT))
-		ezx_pcap_write(pcap, PCAP_REG_INT_SEL,
-					(1 << PCAP_IRQ_ADCDONE2));
-
-	/* setup irq chip */
-	for (i = pcap->irq_base; i < (pcap->irq_base + PCAP_NIRQS); i++) {
-		irq_set_chip_and_handler(i, &pcap_irq_chip, handle_simple_irq);
-		irq_set_chip_data(i, pcap);
-		irq_clear_status_flags(i, IRQ_NOREQUEST | IRQ_NOPROBE);
-	}
-
-	/* mask/ack all PCAP interrupts */
-	ezx_pcap_write(pcap, PCAP_REG_MSR, PCAP_MASK_ALL_INTERRUPT);
-	ezx_pcap_write(pcap, PCAP_REG_ISR, PCAP_CLEAR_INTERRUPT_REGISTER);
-	pcap->msr = PCAP_MASK_ALL_INTERRUPT;
-
-	irq_set_irq_type(spi->irq, IRQ_TYPE_EDGE_RISING);
-	irq_set_chained_handler_and_data(spi->irq, pcap_irq_handler, pcap);
-	irq_set_irq_wake(spi->irq, 1);
-
-	/* ADC */
-	adc_irq = pcap_to_irq(pcap, (pdata->config & PCAP_SECOND_PORT) ?
-					PCAP_IRQ_ADCDONE2 : PCAP_IRQ_ADCDONE);
-
-	ret = devm_request_irq(&spi->dev, adc_irq, pcap_adc_irq, 0, "ADC",
-				pcap);
-	if (ret)
-		goto free_irqchip;
-
-	/* setup subdevs */
-	for (i = 0; i < pdata->num_subdevs; i++) {
-		ret = pcap_add_subdev(pcap, &pdata->subdevs[i]);
-		if (ret)
-			goto remove_subdevs;
-	}
-
-	/* board specific quirks */
-	if (pdata->init)
-		pdata->init(pcap);
-
-	return 0;
-
-remove_subdevs:
-	device_for_each_child(&spi->dev, NULL, pcap_remove_subdev);
-free_irqchip:
-	for (i = pcap->irq_base; i < (pcap->irq_base + PCAP_NIRQS); i++)
-		irq_set_chip_and_handler(i, NULL, NULL);
-
-	return ret;
-}
-
-static struct spi_driver ezxpcap_driver = {
-	.probe	= ezx_pcap_probe,
-	.remove = ezx_pcap_remove,
-	.driver = {
-		.name	= "ezx-pcap",
-	},
-};
-
-static int __init ezx_pcap_init(void)
-{
-	return spi_register_driver(&ezxpcap_driver);
-}
-
-static void __exit ezx_pcap_exit(void)
-{
-	spi_unregister_driver(&ezxpcap_driver);
-}
-
-subsys_initcall(ezx_pcap_init);
-module_exit(ezx_pcap_exit);
-
-MODULE_AUTHOR("Daniel Ribeiro / Harald Welte");
-MODULE_DESCRIPTION("Motorola PCAP2 ASIC Driver");
-MODULE_ALIAS("spi:ezx-pcap");
diff --git a/drivers/mfd/intel-lpss-pci.c b/drivers/mfd/intel-lpss-pci.c
index a9452ac92fb2d..f7c592dd7e875 100644
--- a/drivers/mfd/intel-lpss-pci.c
+++ b/drivers/mfd/intel-lpss-pci.c
@@ -247,431 +247,431 @@ static const struct intel_lpss_platform_info tgl_spi_info = {
 
 static const struct pci_device_id intel_lpss_pci_ids[] = {
 	/* CML-LP */
-	{ PCI_VDEVICE(INTEL, 0x02a8), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x02a9), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x02aa), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x02ab), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x02c5), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x02c6), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x02c7), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x02e8), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x02e9), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x02ea), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x02eb), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x02fb), (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x02a8), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x02a9), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x02aa), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x02ab), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x02c5), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x02c6), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x02c7), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x02e8), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x02e9), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x02ea), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x02eb), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x02fb), .driver_data = (kernel_ulong_t)&cnl_spi_info },
 	/* CML-H */
-	{ PCI_VDEVICE(INTEL, 0x06a8), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x06a9), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x06aa), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x06ab), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x06c7), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x06e8), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x06e9), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x06ea), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x06eb), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x06fb), (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x06a8), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x06a9), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x06aa), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x06ab), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x06c7), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x06e8), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x06e9), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x06ea), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x06eb), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x06fb), .driver_data = (kernel_ulong_t)&cnl_spi_info },
 	/* BXT A-Step */
-	{ PCI_VDEVICE(INTEL, 0x0aac), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x0aae), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x0ab0), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x0ab2), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x0ab4), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x0ab6), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x0ab8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x0aba), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x0abc), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x0abe), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x0ac0), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x0ac2), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x0ac4), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x0ac6), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x0aee), (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x0aac), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x0aae), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x0ab0), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x0ab2), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x0ab4), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x0ab6), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x0ab8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x0aba), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x0abc), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x0abe), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x0ac0), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x0ac2), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x0ac4), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x0ac6), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x0aee), .driver_data = (kernel_ulong_t)&bxt_uart_info },
 	/* BXT B-Step */
-	{ PCI_VDEVICE(INTEL, 0x1aac), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x1aae), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x1ab0), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x1ab2), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x1ab4), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x1ab6), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x1ab8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x1aba), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x1abc), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x1abe), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x1ac0), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x1ac2), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x1ac4), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x1ac6), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x1aee), (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x1aac), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x1aae), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x1ab0), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x1ab2), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x1ab4), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x1ab6), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x1ab8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x1aba), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x1abc), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x1abe), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x1ac0), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x1ac2), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x1ac4), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x1ac6), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x1aee), .driver_data = (kernel_ulong_t)&bxt_uart_info },
 	/* EBG */
-	{ PCI_VDEVICE(INTEL, 0x1bad), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x1bae), (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x1bad), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x1bae), .driver_data = (kernel_ulong_t)&bxt_uart_info },
 	/* GLK */
-	{ PCI_VDEVICE(INTEL, 0x31ac), (kernel_ulong_t)&glk_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x31ae), (kernel_ulong_t)&glk_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x31b0), (kernel_ulong_t)&glk_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x31b2), (kernel_ulong_t)&glk_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x31b4), (kernel_ulong_t)&glk_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x31b6), (kernel_ulong_t)&glk_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x31b8), (kernel_ulong_t)&glk_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x31ba), (kernel_ulong_t)&glk_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x31bc), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x31be), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x31c0), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x31c2), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x31c4), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x31c6), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x31ee), (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x31ac), .driver_data = (kernel_ulong_t)&glk_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x31ae), .driver_data = (kernel_ulong_t)&glk_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x31b0), .driver_data = (kernel_ulong_t)&glk_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x31b2), .driver_data = (kernel_ulong_t)&glk_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x31b4), .driver_data = (kernel_ulong_t)&glk_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x31b6), .driver_data = (kernel_ulong_t)&glk_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x31b8), .driver_data = (kernel_ulong_t)&glk_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x31ba), .driver_data = (kernel_ulong_t)&glk_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x31bc), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x31be), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x31c0), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x31c2), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x31c4), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x31c6), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x31ee), .driver_data = (kernel_ulong_t)&bxt_uart_info },
 	/* ICL-LP */
-	{ PCI_VDEVICE(INTEL, 0x34a8), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x34a9), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x34aa), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x34ab), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x34c5), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x34c6), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x34c7), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x34e8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x34e9), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x34ea), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x34eb), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x34fb), (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x34a8), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x34a9), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x34aa), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x34ab), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x34c5), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x34c6), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x34c7), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x34e8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x34e9), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x34ea), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x34eb), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x34fb), .driver_data = (kernel_ulong_t)&cnl_spi_info },
 	/* ICL-N */
-	{ PCI_VDEVICE(INTEL, 0x38a8), (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x38a8), .driver_data = (kernel_ulong_t)&spt_uart_info },
 	/* TGL-H */
-	{ PCI_VDEVICE(INTEL, 0x43a7), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x43a8), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x43a9), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x43aa), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x43ab), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x43ad), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x43ae), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x43d8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x43da), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x43e8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x43e9), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x43ea), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x43eb), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x43fb), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x43fd), (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x43a7), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x43a8), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x43a9), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x43aa), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x43ab), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x43ad), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x43ae), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x43d8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x43da), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x43e8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x43e9), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x43ea), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x43eb), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x43fb), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x43fd), .driver_data = (kernel_ulong_t)&tgl_spi_info },
 	/* EHL */
-	{ PCI_VDEVICE(INTEL, 0x4b28), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x4b29), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x4b2a), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x4b2b), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x4b37), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x4b44), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4b45), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4b4b), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4b4c), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4b4d), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x4b78), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4b79), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4b7a), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4b7b), (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4b28), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x4b29), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x4b2a), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x4b2b), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x4b37), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x4b44), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4b45), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4b4b), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4b4c), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4b4d), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x4b78), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4b79), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4b7a), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4b7b), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
 	/* WCL */
-	{ PCI_VDEVICE(INTEL, 0x4d25), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x4d26), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x4d27), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x4d30), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x4d46), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x4d50), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4d51), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4d52), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x4d78), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4d79), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4d7a), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4d7b), (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4d25), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x4d26), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x4d27), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x4d30), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x4d46), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x4d50), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4d51), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4d52), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x4d78), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4d79), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4d7a), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4d7b), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
 	/* JSL */
-	{ PCI_VDEVICE(INTEL, 0x4da8), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x4da9), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x4daa), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x4dab), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x4dc5), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4dc6), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4dc7), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x4de8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4de9), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4dea), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4deb), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x4dfb), (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x4da8), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x4da9), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x4daa), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x4dab), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x4dc5), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4dc6), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4dc7), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x4de8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4de9), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4dea), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4deb), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x4dfb), .driver_data = (kernel_ulong_t)&cnl_spi_info },
 	/* ADL-P */
-	{ PCI_VDEVICE(INTEL, 0x51a8), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x51a9), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x51aa), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x51ab), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x51c5), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x51c6), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x51c7), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x51d8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x51d9), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x51e8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x51e9), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x51ea), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x51eb), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x51fb), (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x51a8), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x51a9), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x51aa), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x51ab), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x51c5), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x51c6), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x51c7), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x51d8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x51d9), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x51e8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x51e9), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x51ea), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x51eb), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x51fb), .driver_data = (kernel_ulong_t)&tgl_spi_info },
 	/* ADL-M */
-	{ PCI_VDEVICE(INTEL, 0x54a8), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x54a9), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x54aa), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x54ab), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x54c5), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x54c6), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x54c7), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x54e8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x54e9), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x54ea), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x54eb), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x54fb), (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x54a8), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x54a9), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x54aa), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x54ab), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x54c5), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x54c6), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x54c7), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x54e8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x54e9), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x54ea), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x54eb), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x54fb), .driver_data = (kernel_ulong_t)&tgl_spi_info },
 	/* APL */
-	{ PCI_VDEVICE(INTEL, 0x5aac), (kernel_ulong_t)&apl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5aae), (kernel_ulong_t)&apl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab0), (kernel_ulong_t)&apl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab2), (kernel_ulong_t)&apl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab4), (kernel_ulong_t)&apl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab6), (kernel_ulong_t)&apl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5ab8), (kernel_ulong_t)&apl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5aba), (kernel_ulong_t)&apl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x5abc), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x5abe), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x5ac0), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x5ac2), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x5ac4), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x5ac6), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x5aee), (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x5aac), .driver_data = (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5aae), .driver_data = (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab0), .driver_data = (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab2), .driver_data = (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab4), .driver_data = (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab6), .driver_data = (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5ab8), .driver_data = (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5aba), .driver_data = (kernel_ulong_t)&apl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x5abc), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x5abe), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x5ac0), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x5ac2), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x5ac4), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x5ac6), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x5aee), .driver_data = (kernel_ulong_t)&bxt_uart_info },
 	/* NVL-S */
-	{ PCI_VDEVICE(INTEL, 0x6e28), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x6e29), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x6e2a), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x6e2b), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x6e4c), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x6e4d), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x6e4e), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x6e4f), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x6e5c), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x6e5e), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x6e7a), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x6e7b), (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x6e28), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x6e29), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x6e2a), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x6e2b), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x6e4c), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x6e4d), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x6e4e), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x6e4f), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x6e5c), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x6e5e), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x6e7a), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x6e7b), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
 	/* ARL-H */
-	{ PCI_VDEVICE(INTEL, 0x7725), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7726), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7727), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7730), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7746), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7750), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7751), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7752), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7778), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7779), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x777a), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x777b), (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7725), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7726), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7727), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7730), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7746), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7750), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7751), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7752), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7778), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7779), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x777a), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x777b), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
 	/* RPL-S */
-	{ PCI_VDEVICE(INTEL, 0x7a28), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7a29), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7a2a), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7a2b), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7a4c), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7a4d), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7a4e), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7a4f), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7a5c), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7a79), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7a7b), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7a7c), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7a7d), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7a7e), (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7a28), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7a29), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7a2a), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7a2b), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7a4c), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7a4d), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7a4e), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7a4f), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7a5c), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7a79), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7a7b), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7a7c), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7a7d), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7a7e), .driver_data = (kernel_ulong_t)&bxt_uart_info },
 	/* ADL-S */
-	{ PCI_VDEVICE(INTEL, 0x7aa8), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7aa9), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7aaa), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7aab), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7acc), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7acd), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7ace), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7acf), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7adc), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7af9), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7afb), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7afc), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7afd), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7afe), (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7aa8), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7aa9), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7aaa), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7aab), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7acc), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7acd), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7ace), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7acf), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7adc), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7af9), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7afb), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7afc), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7afd), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7afe), .driver_data = (kernel_ulong_t)&bxt_uart_info },
 	/* MTL-P */
-	{ PCI_VDEVICE(INTEL, 0x7e25), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7e26), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7e27), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7e30), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7e46), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7e50), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7e51), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7e52), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7e78), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7e79), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7e7a), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7e7b), (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7e25), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7e26), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7e27), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7e30), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7e46), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7e50), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7e51), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7e52), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7e78), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7e79), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7e7a), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7e7b), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
 	/* MTP-S */
-	{ PCI_VDEVICE(INTEL, 0x7f28), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7f29), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7f2a), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7f2b), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7f4c), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7f4d), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7f4e), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7f4f), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7f5c), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7f5d), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x7f5e), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7f5f), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x7f7a), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x7f7b), (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7f28), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7f29), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7f2a), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7f2b), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7f4c), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7f4d), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7f4e), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7f4f), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7f5c), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7f5d), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x7f5e), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7f5f), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x7f7a), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x7f7b), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
 	/* LKF */
-	{ PCI_VDEVICE(INTEL, 0x98a8), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x98a9), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x98aa), (kernel_ulong_t)&bxt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x98c5), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x98c6), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x98c7), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x98e8), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x98e9), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x98ea), (kernel_ulong_t)&bxt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x98eb), (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x98a8), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x98a9), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x98aa), .driver_data = (kernel_ulong_t)&bxt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x98c5), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x98c6), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x98c7), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x98e8), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x98e9), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x98ea), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x98eb), .driver_data = (kernel_ulong_t)&bxt_i2c_info },
 	/* SPT-LP */
-	{ PCI_VDEVICE(INTEL, 0x9d27), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x9d28), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x9d29), (kernel_ulong_t)&spt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x9d2a), (kernel_ulong_t)&spt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x9d60), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9d61), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9d62), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9d63), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9d64), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9d65), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9d66), (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x9d27), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x9d28), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x9d29), .driver_data = (kernel_ulong_t)&spt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x9d2a), .driver_data = (kernel_ulong_t)&spt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x9d60), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9d61), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9d62), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9d63), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9d64), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9d65), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9d66), .driver_data = (kernel_ulong_t)&spt_uart_info },
 	/* CNL-LP */
-	{ PCI_VDEVICE(INTEL, 0x9da8), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x9da9), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x9daa), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x9dab), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0x9dc5), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9dc6), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9dc7), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0x9de8), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9de9), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9dea), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9deb), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0x9dfb), (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x9da8), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x9da9), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x9daa), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x9dab), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0x9dc5), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9dc6), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9dc7), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0x9de8), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9de9), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9dea), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9deb), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0x9dfb), .driver_data = (kernel_ulong_t)&cnl_spi_info },
 	/* TGL-LP */
-	{ PCI_VDEVICE(INTEL, 0xa0a8), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa0a9), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa0aa), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa0ab), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa0c5), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa0c6), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa0c7), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa0d8), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa0d9), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa0da), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa0db), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa0dc), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa0dd), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa0de), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa0df), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa0e8), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa0e9), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa0ea), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa0eb), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa0fb), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa0fd), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa0fe), (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa0a8), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa0a9), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa0aa), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa0ab), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa0c5), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa0c6), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa0c7), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa0d8), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa0d9), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa0da), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa0db), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa0dc), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa0dd), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa0de), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa0df), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa0e8), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa0e9), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa0ea), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa0eb), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa0fb), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa0fd), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa0fe), .driver_data = (kernel_ulong_t)&cnl_spi_info },
 	/* SPT-H */
-	{ PCI_VDEVICE(INTEL, 0xa127), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa128), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa129), (kernel_ulong_t)&spt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa12a), (kernel_ulong_t)&spt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa160), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa161), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa162), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa166), (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa127), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa128), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa129), .driver_data = (kernel_ulong_t)&spt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa12a), .driver_data = (kernel_ulong_t)&spt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa160), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa161), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa162), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa166), .driver_data = (kernel_ulong_t)&spt_uart_info },
 	/* KBL-H */
-	{ PCI_VDEVICE(INTEL, 0xa2a7), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa2a8), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa2a9), (kernel_ulong_t)&spt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa2aa), (kernel_ulong_t)&spt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa2e0), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa2e1), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa2e2), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa2e3), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa2e6), (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa2a7), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa2a8), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa2a9), .driver_data = (kernel_ulong_t)&spt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa2aa), .driver_data = (kernel_ulong_t)&spt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e0), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e1), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e2), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e3), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa2e6), .driver_data = (kernel_ulong_t)&spt_uart_info },
 	/* CNL-H */
-	{ PCI_VDEVICE(INTEL, 0xa328), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa329), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa32a), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa32b), (kernel_ulong_t)&cnl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa347), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa368), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa369), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa36a), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa36b), (kernel_ulong_t)&cnl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa37b), (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa328), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa329), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa32a), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa32b), .driver_data = (kernel_ulong_t)&cnl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa347), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa368), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa369), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa36a), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa36b), .driver_data = (kernel_ulong_t)&cnl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa37b), .driver_data = (kernel_ulong_t)&cnl_spi_info },
 	/* CML-V */
-	{ PCI_VDEVICE(INTEL, 0xa3a7), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa3a8), (kernel_ulong_t)&spt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa3a9), (kernel_ulong_t)&spt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa3aa), (kernel_ulong_t)&spt_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa3e0), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa3e1), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa3e2), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa3e3), (kernel_ulong_t)&spt_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa3e6), (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa3a7), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa3a8), .driver_data = (kernel_ulong_t)&spt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa3a9), .driver_data = (kernel_ulong_t)&spt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa3aa), .driver_data = (kernel_ulong_t)&spt_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa3e0), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa3e1), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa3e2), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa3e3), .driver_data = (kernel_ulong_t)&spt_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa3e6), .driver_data = (kernel_ulong_t)&spt_uart_info },
 	/* LNL-M */
-	{ PCI_VDEVICE(INTEL, 0xa825), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa826), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa827), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa830), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa846), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xa850), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa851), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa852), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xa878), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa879), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa87a), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xa87b), (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa825), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa826), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa827), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa830), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa846), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xa850), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa851), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa852), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xa878), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa879), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa87a), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xa87b), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
 	/* NVL-H */
-	{ PCI_VDEVICE(INTEL, 0xd325), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xd326), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xd327), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xd330), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xd347), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xd350), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xd351), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xd352), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xd378), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xd379), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xd37a), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xd37b), (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xd325), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xd326), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xd327), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xd330), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xd347), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xd350), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xd351), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xd352), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xd378), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xd379), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xd37a), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xd37b), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
 	/* PTL-H */
-	{ PCI_VDEVICE(INTEL, 0xe325), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xe326), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xe327), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xe330), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xe346), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xe350), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe351), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe352), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xe378), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe379), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe37a), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe37b), (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe325), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xe326), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xe327), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xe330), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xe346), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xe350), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe351), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe352), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xe378), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe379), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe37a), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe37b), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
 	/* PTL-P */
-	{ PCI_VDEVICE(INTEL, 0xe425), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xe426), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xe427), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xe430), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xe446), (kernel_ulong_t)&tgl_spi_info },
-	{ PCI_VDEVICE(INTEL, 0xe450), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe451), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe452), (kernel_ulong_t)&bxt_uart_info },
-	{ PCI_VDEVICE(INTEL, 0xe478), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe479), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe47a), (kernel_ulong_t)&ehl_i2c_info },
-	{ PCI_VDEVICE(INTEL, 0xe47b), (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe425), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xe426), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xe427), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xe430), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xe446), .driver_data = (kernel_ulong_t)&tgl_spi_info },
+	{ PCI_VDEVICE(INTEL, 0xe450), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe451), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe452), .driver_data = (kernel_ulong_t)&bxt_uart_info },
+	{ PCI_VDEVICE(INTEL, 0xe478), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe479), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe47a), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
+	{ PCI_VDEVICE(INTEL, 0xe47b), .driver_data = (kernel_ulong_t)&ehl_i2c_info },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, intel_lpss_pci_ids);
diff --git a/drivers/mfd/lpc_ich.c b/drivers/mfd/lpc_ich.c
index 5a3d79f339dd2..d2efcc92bb59b 100644
--- a/drivers/mfd/lpc_ich.c
+++ b/drivers/mfd/lpc_ich.c
@@ -721,244 +721,244 @@ static struct lpc_ich_info lpc_chipset_info[] = {
  * functions that probably will be registered by other drivers.
  */
 static const struct pci_device_id lpc_ich_ids[] = {
-	{ PCI_VDEVICE(INTEL, 0x0f1c), LPC_BAYTRAIL},
-	{ PCI_VDEVICE(INTEL, 0x19dc), LPC_DNV},
-	{ PCI_VDEVICE(INTEL, 0x1c41), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c42), LPC_CPTD},
-	{ PCI_VDEVICE(INTEL, 0x1c43), LPC_CPTM},
-	{ PCI_VDEVICE(INTEL, 0x1c44), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c45), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c46), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c47), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c48), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c49), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c4a), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c4b), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c4c), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c4d), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c4e), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c4f), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c50), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c51), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c52), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c53), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c54), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c55), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c56), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c57), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c58), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c59), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c5a), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c5b), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c5c), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c5d), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c5e), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1c5f), LPC_CPT},
-	{ PCI_VDEVICE(INTEL, 0x1d40), LPC_PBG},
-	{ PCI_VDEVICE(INTEL, 0x1d41), LPC_PBG},
-	{ PCI_VDEVICE(INTEL, 0x1e40), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e41), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e42), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e43), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e44), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e45), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e46), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e47), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e48), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e49), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e4a), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e4b), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e4c), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e4d), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e4e), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e4f), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e50), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e51), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e52), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e53), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e54), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e55), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e56), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e57), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e58), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e59), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e5a), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e5b), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e5c), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e5d), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e5e), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1e5f), LPC_PPT},
-	{ PCI_VDEVICE(INTEL, 0x1f38), LPC_AVN},
-	{ PCI_VDEVICE(INTEL, 0x1f39), LPC_AVN},
-	{ PCI_VDEVICE(INTEL, 0x1f3a), LPC_AVN},
-	{ PCI_VDEVICE(INTEL, 0x1f3b), LPC_AVN},
-	{ PCI_VDEVICE(INTEL, 0x229c), LPC_BRASWELL},
-	{ PCI_VDEVICE(INTEL, 0x2310), LPC_DH89XXCC},
-	{ PCI_VDEVICE(INTEL, 0x2390), LPC_COLETO},
-	{ PCI_VDEVICE(INTEL, 0x2410), LPC_ICH},
-	{ PCI_VDEVICE(INTEL, 0x2420), LPC_ICH0},
-	{ PCI_VDEVICE(INTEL, 0x2440), LPC_ICH2},
-	{ PCI_VDEVICE(INTEL, 0x244c), LPC_ICH2M},
-	{ PCI_VDEVICE(INTEL, 0x2450), LPC_CICH},
-	{ PCI_VDEVICE(INTEL, 0x2480), LPC_ICH3},
-	{ PCI_VDEVICE(INTEL, 0x248c), LPC_ICH3M},
-	{ PCI_VDEVICE(INTEL, 0x24c0), LPC_ICH4},
-	{ PCI_VDEVICE(INTEL, 0x24cc), LPC_ICH4M},
-	{ PCI_VDEVICE(INTEL, 0x24d0), LPC_ICH5},
-	{ PCI_VDEVICE(INTEL, 0x25a1), LPC_6300ESB},
-	{ PCI_VDEVICE(INTEL, 0x2640), LPC_ICH6},
-	{ PCI_VDEVICE(INTEL, 0x2641), LPC_ICH6M},
-	{ PCI_VDEVICE(INTEL, 0x2642), LPC_ICH6W},
-	{ PCI_VDEVICE(INTEL, 0x2670), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x2671), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x2672), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x2673), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x2674), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x2675), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x2676), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x2677), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x2678), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x2679), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x267a), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x267b), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x267c), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x267d), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x267e), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x267f), LPC_631XESB},
-	{ PCI_VDEVICE(INTEL, 0x27b0), LPC_ICH7DH},
-	{ PCI_VDEVICE(INTEL, 0x27b8), LPC_ICH7},
-	{ PCI_VDEVICE(INTEL, 0x27b9), LPC_ICH7M},
-	{ PCI_VDEVICE(INTEL, 0x27bc), LPC_NM10},
-	{ PCI_VDEVICE(INTEL, 0x27bd), LPC_ICH7MDH},
-	{ PCI_VDEVICE(INTEL, 0x2810), LPC_ICH8},
-	{ PCI_VDEVICE(INTEL, 0x2811), LPC_ICH8ME},
-	{ PCI_VDEVICE(INTEL, 0x2812), LPC_ICH8DH},
-	{ PCI_VDEVICE(INTEL, 0x2814), LPC_ICH8DO},
-	{ PCI_VDEVICE(INTEL, 0x2815), LPC_ICH8M},
-	{ PCI_VDEVICE(INTEL, 0x2912), LPC_ICH9DH},
-	{ PCI_VDEVICE(INTEL, 0x2914), LPC_ICH9DO},
-	{ PCI_VDEVICE(INTEL, 0x2916), LPC_ICH9R},
-	{ PCI_VDEVICE(INTEL, 0x2917), LPC_ICH9ME},
-	{ PCI_VDEVICE(INTEL, 0x2918), LPC_ICH9},
-	{ PCI_VDEVICE(INTEL, 0x2919), LPC_ICH9M},
-	{ PCI_VDEVICE(INTEL, 0x2b9c), LPC_COUGARMOUNTAIN},
-	{ PCI_VDEVICE(INTEL, 0x3197), LPC_GLK},
-	{ PCI_VDEVICE(INTEL, 0x31e8), LPC_GLK},
-	{ PCI_VDEVICE(INTEL, 0x3a14), LPC_ICH10DO},
-	{ PCI_VDEVICE(INTEL, 0x3a16), LPC_ICH10R},
-	{ PCI_VDEVICE(INTEL, 0x3a18), LPC_ICH10},
-	{ PCI_VDEVICE(INTEL, 0x3a1a), LPC_ICH10D},
-	{ PCI_VDEVICE(INTEL, 0x3b00), LPC_PCH},
-	{ PCI_VDEVICE(INTEL, 0x3b01), LPC_PCHM},
-	{ PCI_VDEVICE(INTEL, 0x3b02), LPC_P55},
-	{ PCI_VDEVICE(INTEL, 0x3b03), LPC_PM55},
-	{ PCI_VDEVICE(INTEL, 0x3b06), LPC_H55},
-	{ PCI_VDEVICE(INTEL, 0x3b07), LPC_QM57},
-	{ PCI_VDEVICE(INTEL, 0x3b08), LPC_H57},
-	{ PCI_VDEVICE(INTEL, 0x3b09), LPC_HM55},
-	{ PCI_VDEVICE(INTEL, 0x3b0a), LPC_Q57},
-	{ PCI_VDEVICE(INTEL, 0x3b0b), LPC_HM57},
-	{ PCI_VDEVICE(INTEL, 0x3b0d), LPC_PCHMSFF},
-	{ PCI_VDEVICE(INTEL, 0x3b0f), LPC_QS57},
-	{ PCI_VDEVICE(INTEL, 0x3b12), LPC_3400},
-	{ PCI_VDEVICE(INTEL, 0x3b14), LPC_3420},
-	{ PCI_VDEVICE(INTEL, 0x3b16), LPC_3450},
-	{ PCI_VDEVICE(INTEL, 0x5031), LPC_EP80579},
-	{ PCI_VDEVICE(INTEL, 0x5ae8), LPC_APL},
-	{ PCI_VDEVICE(INTEL, 0x8c40), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c41), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c42), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c43), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c44), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c45), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c46), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c47), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c48), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c49), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c4a), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c4b), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c4c), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c4d), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c4e), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c4f), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c50), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c51), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c52), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c53), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c54), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c55), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c56), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c57), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c58), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c59), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c5a), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c5b), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c5c), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c5d), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c5e), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8c5f), LPC_LPT},
-	{ PCI_VDEVICE(INTEL, 0x8cc1), LPC_9S},
-	{ PCI_VDEVICE(INTEL, 0x8cc2), LPC_9S},
-	{ PCI_VDEVICE(INTEL, 0x8cc3), LPC_9S},
-	{ PCI_VDEVICE(INTEL, 0x8cc4), LPC_9S},
-	{ PCI_VDEVICE(INTEL, 0x8cc6), LPC_9S},
-	{ PCI_VDEVICE(INTEL, 0x8d40), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d41), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d42), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d43), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d44), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d45), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d46), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d47), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d48), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d49), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d4a), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d4b), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d4c), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d4d), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d4e), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d4f), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d50), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d51), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d52), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d53), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d54), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d55), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d56), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d57), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d58), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d59), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d5a), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d5b), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d5c), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d5d), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d5e), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x8d5f), LPC_WBG},
-	{ PCI_VDEVICE(INTEL, 0x9c40), LPC_LPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9c41), LPC_LPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9c42), LPC_LPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9c43), LPC_LPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9c44), LPC_LPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9c45), LPC_LPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9c46), LPC_LPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9c47), LPC_LPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9cc1), LPC_WPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9cc2), LPC_WPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9cc3), LPC_WPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9cc5), LPC_WPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9cc6), LPC_WPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9cc7), LPC_WPT_LP},
-	{ PCI_VDEVICE(INTEL, 0x9cc9), LPC_WPT_LP},
-	{ PCI_VDEVICE(INTEL, 0xa1c1), LPC_LEWISBURG},
-	{ PCI_VDEVICE(INTEL, 0xa1c2), LPC_LEWISBURG},
-	{ PCI_VDEVICE(INTEL, 0xa1c3), LPC_LEWISBURG},
-	{ PCI_VDEVICE(INTEL, 0xa1c4), LPC_LEWISBURG},
-	{ PCI_VDEVICE(INTEL, 0xa1c5), LPC_LEWISBURG},
-	{ PCI_VDEVICE(INTEL, 0xa1c6), LPC_LEWISBURG},
-	{ PCI_VDEVICE(INTEL, 0xa1c7), LPC_LEWISBURG},
-	{ PCI_VDEVICE(INTEL, 0xa242), LPC_LEWISBURG},
-	{ PCI_VDEVICE(INTEL, 0xa243), LPC_LEWISBURG},
-	{ 0, },			/* End of list */
+	{ PCI_VDEVICE(INTEL, 0x0f1c), .driver_data = LPC_BAYTRAIL },
+	{ PCI_VDEVICE(INTEL, 0x19dc), .driver_data = LPC_DNV },
+	{ PCI_VDEVICE(INTEL, 0x1c41), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c42), .driver_data = LPC_CPTD },
+	{ PCI_VDEVICE(INTEL, 0x1c43), .driver_data = LPC_CPTM },
+	{ PCI_VDEVICE(INTEL, 0x1c44), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c45), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c46), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c47), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c48), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c49), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c4a), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c4b), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c4c), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c4d), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c4e), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c4f), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c50), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c51), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c52), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c53), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c54), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c55), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c56), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c57), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c58), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c59), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c5a), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c5b), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c5c), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c5d), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c5e), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1c5f), .driver_data = LPC_CPT },
+	{ PCI_VDEVICE(INTEL, 0x1d40), .driver_data = LPC_PBG },
+	{ PCI_VDEVICE(INTEL, 0x1d41), .driver_data = LPC_PBG },
+	{ PCI_VDEVICE(INTEL, 0x1e40), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e41), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e42), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e43), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e44), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e45), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e46), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e47), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e48), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e49), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e4a), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e4b), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e4c), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e4d), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e4e), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e4f), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e50), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e51), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e52), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e53), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e54), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e55), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e56), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e57), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e58), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e59), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e5a), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e5b), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e5c), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e5d), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e5e), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1e5f), .driver_data = LPC_PPT },
+	{ PCI_VDEVICE(INTEL, 0x1f38), .driver_data = LPC_AVN },
+	{ PCI_VDEVICE(INTEL, 0x1f39), .driver_data = LPC_AVN },
+	{ PCI_VDEVICE(INTEL, 0x1f3a), .driver_data = LPC_AVN },
+	{ PCI_VDEVICE(INTEL, 0x1f3b), .driver_data = LPC_AVN },
+	{ PCI_VDEVICE(INTEL, 0x229c), .driver_data = LPC_BRASWELL },
+	{ PCI_VDEVICE(INTEL, 0x2310), .driver_data = LPC_DH89XXCC },
+	{ PCI_VDEVICE(INTEL, 0x2390), .driver_data = LPC_COLETO },
+	{ PCI_VDEVICE(INTEL, 0x2410), .driver_data = LPC_ICH },
+	{ PCI_VDEVICE(INTEL, 0x2420), .driver_data = LPC_ICH0 },
+	{ PCI_VDEVICE(INTEL, 0x2440), .driver_data = LPC_ICH2 },
+	{ PCI_VDEVICE(INTEL, 0x244c), .driver_data = LPC_ICH2M },
+	{ PCI_VDEVICE(INTEL, 0x2450), .driver_data = LPC_CICH },
+	{ PCI_VDEVICE(INTEL, 0x2480), .driver_data = LPC_ICH3 },
+	{ PCI_VDEVICE(INTEL, 0x248c), .driver_data = LPC_ICH3M },
+	{ PCI_VDEVICE(INTEL, 0x24c0), .driver_data = LPC_ICH4 },
+	{ PCI_VDEVICE(INTEL, 0x24cc), .driver_data = LPC_ICH4M },
+	{ PCI_VDEVICE(INTEL, 0x24d0), .driver_data = LPC_ICH5 },
+	{ PCI_VDEVICE(INTEL, 0x25a1), .driver_data = LPC_6300ESB },
+	{ PCI_VDEVICE(INTEL, 0x2640), .driver_data = LPC_ICH6 },
+	{ PCI_VDEVICE(INTEL, 0x2641), .driver_data = LPC_ICH6M },
+	{ PCI_VDEVICE(INTEL, 0x2642), .driver_data = LPC_ICH6W },
+	{ PCI_VDEVICE(INTEL, 0x2670), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x2671), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x2672), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x2673), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x2674), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x2675), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x2676), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x2677), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x2678), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x2679), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x267a), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x267b), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x267c), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x267d), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x267e), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x267f), .driver_data = LPC_631XESB },
+	{ PCI_VDEVICE(INTEL, 0x27b0), .driver_data = LPC_ICH7DH },
+	{ PCI_VDEVICE(INTEL, 0x27b8), .driver_data = LPC_ICH7 },
+	{ PCI_VDEVICE(INTEL, 0x27b9), .driver_data = LPC_ICH7M },
+	{ PCI_VDEVICE(INTEL, 0x27bc), .driver_data = LPC_NM10 },
+	{ PCI_VDEVICE(INTEL, 0x27bd), .driver_data = LPC_ICH7MDH },
+	{ PCI_VDEVICE(INTEL, 0x2810), .driver_data = LPC_ICH8 },
+	{ PCI_VDEVICE(INTEL, 0x2811), .driver_data = LPC_ICH8ME },
+	{ PCI_VDEVICE(INTEL, 0x2812), .driver_data = LPC_ICH8DH },
+	{ PCI_VDEVICE(INTEL, 0x2814), .driver_data = LPC_ICH8DO },
+	{ PCI_VDEVICE(INTEL, 0x2815), .driver_data = LPC_ICH8M },
+	{ PCI_VDEVICE(INTEL, 0x2912), .driver_data = LPC_ICH9DH },
+	{ PCI_VDEVICE(INTEL, 0x2914), .driver_data = LPC_ICH9DO },
+	{ PCI_VDEVICE(INTEL, 0x2916), .driver_data = LPC_ICH9R },
+	{ PCI_VDEVICE(INTEL, 0x2917), .driver_data = LPC_ICH9ME },
+	{ PCI_VDEVICE(INTEL, 0x2918), .driver_data = LPC_ICH9 },
+	{ PCI_VDEVICE(INTEL, 0x2919), .driver_data = LPC_ICH9M },
+	{ PCI_VDEVICE(INTEL, 0x2b9c), .driver_data = LPC_COUGARMOUNTAIN },
+	{ PCI_VDEVICE(INTEL, 0x3197), .driver_data = LPC_GLK },
+	{ PCI_VDEVICE(INTEL, 0x31e8), .driver_data = LPC_GLK },
+	{ PCI_VDEVICE(INTEL, 0x3a14), .driver_data = LPC_ICH10DO },
+	{ PCI_VDEVICE(INTEL, 0x3a16), .driver_data = LPC_ICH10R },
+	{ PCI_VDEVICE(INTEL, 0x3a18), .driver_data = LPC_ICH10 },
+	{ PCI_VDEVICE(INTEL, 0x3a1a), .driver_data = LPC_ICH10D },
+	{ PCI_VDEVICE(INTEL, 0x3b00), .driver_data = LPC_PCH },
+	{ PCI_VDEVICE(INTEL, 0x3b01), .driver_data = LPC_PCHM },
+	{ PCI_VDEVICE(INTEL, 0x3b02), .driver_data = LPC_P55 },
+	{ PCI_VDEVICE(INTEL, 0x3b03), .driver_data = LPC_PM55 },
+	{ PCI_VDEVICE(INTEL, 0x3b06), .driver_data = LPC_H55 },
+	{ PCI_VDEVICE(INTEL, 0x3b07), .driver_data = LPC_QM57 },
+	{ PCI_VDEVICE(INTEL, 0x3b08), .driver_data = LPC_H57 },
+	{ PCI_VDEVICE(INTEL, 0x3b09), .driver_data = LPC_HM55 },
+	{ PCI_VDEVICE(INTEL, 0x3b0a), .driver_data = LPC_Q57 },
+	{ PCI_VDEVICE(INTEL, 0x3b0b), .driver_data = LPC_HM57 },
+	{ PCI_VDEVICE(INTEL, 0x3b0d), .driver_data = LPC_PCHMSFF },
+	{ PCI_VDEVICE(INTEL, 0x3b0f), .driver_data = LPC_QS57 },
+	{ PCI_VDEVICE(INTEL, 0x3b12), .driver_data = LPC_3400 },
+	{ PCI_VDEVICE(INTEL, 0x3b14), .driver_data = LPC_3420 },
+	{ PCI_VDEVICE(INTEL, 0x3b16), .driver_data = LPC_3450 },
+	{ PCI_VDEVICE(INTEL, 0x5031), .driver_data = LPC_EP80579 },
+	{ PCI_VDEVICE(INTEL, 0x5ae8), .driver_data = LPC_APL },
+	{ PCI_VDEVICE(INTEL, 0x8c40), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c41), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c42), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c43), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c44), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c45), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c46), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c47), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c48), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c49), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c4a), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c4b), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c4c), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c4d), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c4e), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c4f), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c50), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c51), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c52), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c53), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c54), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c55), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c56), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c57), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c58), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c59), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c5a), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c5b), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c5c), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c5d), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c5e), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8c5f), .driver_data = LPC_LPT },
+	{ PCI_VDEVICE(INTEL, 0x8cc1), .driver_data = LPC_9S },
+	{ PCI_VDEVICE(INTEL, 0x8cc2), .driver_data = LPC_9S },
+	{ PCI_VDEVICE(INTEL, 0x8cc3), .driver_data = LPC_9S },
+	{ PCI_VDEVICE(INTEL, 0x8cc4), .driver_data = LPC_9S },
+	{ PCI_VDEVICE(INTEL, 0x8cc6), .driver_data = LPC_9S },
+	{ PCI_VDEVICE(INTEL, 0x8d40), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d41), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d42), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d43), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d44), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d45), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d46), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d47), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d48), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d49), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d4a), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d4b), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d4c), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d4d), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d4e), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d4f), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d50), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d51), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d52), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d53), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d54), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d55), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d56), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d57), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d58), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d59), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d5a), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d5b), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d5c), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d5d), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d5e), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x8d5f), .driver_data = LPC_WBG },
+	{ PCI_VDEVICE(INTEL, 0x9c40), .driver_data = LPC_LPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9c41), .driver_data = LPC_LPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9c42), .driver_data = LPC_LPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9c43), .driver_data = LPC_LPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9c44), .driver_data = LPC_LPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9c45), .driver_data = LPC_LPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9c46), .driver_data = LPC_LPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9c47), .driver_data = LPC_LPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9cc1), .driver_data = LPC_WPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9cc2), .driver_data = LPC_WPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9cc3), .driver_data = LPC_WPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9cc5), .driver_data = LPC_WPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9cc6), .driver_data = LPC_WPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9cc7), .driver_data = LPC_WPT_LP },
+	{ PCI_VDEVICE(INTEL, 0x9cc9), .driver_data = LPC_WPT_LP },
+	{ PCI_VDEVICE(INTEL, 0xa1c1), .driver_data = LPC_LEWISBURG },
+	{ PCI_VDEVICE(INTEL, 0xa1c2), .driver_data = LPC_LEWISBURG },
+	{ PCI_VDEVICE(INTEL, 0xa1c3), .driver_data = LPC_LEWISBURG },
+	{ PCI_VDEVICE(INTEL, 0xa1c4), .driver_data = LPC_LEWISBURG },
+	{ PCI_VDEVICE(INTEL, 0xa1c5), .driver_data = LPC_LEWISBURG },
+	{ PCI_VDEVICE(INTEL, 0xa1c6), .driver_data = LPC_LEWISBURG },
+	{ PCI_VDEVICE(INTEL, 0xa1c7), .driver_data = LPC_LEWISBURG },
+	{ PCI_VDEVICE(INTEL, 0xa242), .driver_data = LPC_LEWISBURG },
+	{ PCI_VDEVICE(INTEL, 0xa243), .driver_data = LPC_LEWISBURG },
+	{ },			/* End of list */
 };
 MODULE_DEVICE_TABLE(pci, lpc_ich_ids);
 
diff --git a/drivers/mfd/lpc_sch.c b/drivers/mfd/lpc_sch.c
index 9ab9adce06fdd..d069165f9b860 100644
--- a/drivers/mfd/lpc_sch.c
+++ b/drivers/mfd/lpc_sch.c
@@ -64,11 +64,11 @@ static struct lpc_sch_info sch_chipset_info[] = {
 };
 
 static const struct pci_device_id lpc_sch_ids[] = {
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_SCH_LPC), LPC_SCH },
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ITC_LPC), LPC_ITC },
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_CENTERTON_ILB), LPC_CENTERTON },
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QUARK_X1000_ILB), LPC_QUARK_X1000 },
-	{ 0, }
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_SCH_LPC), .driver_data = LPC_SCH },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ITC_LPC), .driver_data = LPC_ITC },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_CENTERTON_ILB), .driver_data = LPC_CENTERTON },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QUARK_X1000_ILB), .driver_data = LPC_QUARK_X1000 },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, lpc_sch_ids);
 
diff --git a/drivers/mfd/max77620.c b/drivers/mfd/max77620.c
index 3af2974b30232..3b1f3a2f35664 100644
--- a/drivers/mfd/max77620.c
+++ b/drivers/mfd/max77620.c
@@ -31,11 +31,10 @@
 #include <linux/init.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/reboot.h>
 #include <linux/regmap.h>
 #include <linux/slab.h>
 
-static struct max77620_chip *max77620_scratch;
-
 static const struct resource gpio_resources[] = {
 	DEFINE_RES_IRQ(MAX77620_IRQ_TOP_GPIO),
 };
@@ -484,13 +483,27 @@ static int max77620_read_es_version(struct max77620_chip *chip)
 	return ret;
 }
 
-static void max77620_pm_power_off(void)
+static int max77620_pm_power_off(struct sys_off_data *data)
 {
-	struct max77620_chip *chip = max77620_scratch;
+	struct max77620_chip *chip = data->cb_data;
 
 	regmap_update_bits(chip->rmap, MAX77620_REG_ONOFFCNFG1,
 			   MAX77620_ONOFFCNFG1_SFT_RST,
 			   MAX77620_ONOFFCNFG1_SFT_RST);
+
+	return NOTIFY_DONE;
+}
+
+static int max77620_power_off_priority(void)
+{
+	/*
+	 * For Smaug  we need to override the PSCI poweroff handler
+	 * which is registered at priority SYS_OFF_PRIO_FIRMWARE.
+	 */
+	if (of_machine_is_compatible("google,smaug"))
+		return SYS_OFF_PRIO_FIRMWARE + 1;
+
+	return SYS_OFF_PRIO_DEFAULT;
 }
 
 static int max77620_probe(struct i2c_client *client)
@@ -501,7 +514,7 @@ static int max77620_probe(struct i2c_client *client)
 	struct regmap_irq_chip *chip_desc;
 	const struct mfd_cell *mfd_cells;
 	int n_mfd_cells;
-	bool pm_off;
+	int priority;
 	int ret;
 
 	chip = devm_kzalloc(&client->dev, sizeof(*chip), GFP_KERNEL);
@@ -573,10 +586,15 @@ static int max77620_probe(struct i2c_client *client)
 		return ret;
 	}
 
-	pm_off = of_device_is_system_power_controller(client->dev.of_node);
-	if (pm_off && !pm_power_off) {
-		max77620_scratch = chip;
-		pm_power_off = max77620_pm_power_off;
+	if (of_device_is_system_power_controller(client->dev.of_node)) {
+		priority = max77620_power_off_priority();
+		ret = devm_register_sys_off_handler(&client->dev,
+						    SYS_OFF_MODE_POWER_OFF,
+						    priority, max77620_pm_power_off,
+						    chip);
+		if (ret)
+			return dev_err_probe(&client->dev, ret,
+					"failed to register power-off handler\n");
 	}
 
 	return 0;
diff --git a/drivers/mfd/max77759.c b/drivers/mfd/max77759.c
index 9fa6027a92c40..b50433e7b3d35 100644
--- a/drivers/mfd/max77759.c
+++ b/drivers/mfd/max77759.c
@@ -286,38 +286,22 @@ static const struct regmap_irq max77759_topsys_irqs[] = {
 };
 
 static const struct regmap_irq max77759_chgr_irqs[] = {
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_AICL, 0,
-		       MAX77759_CHGR_REG_CHG_INT_AICL),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_CHGIN, 0,
-		       MAX77759_CHGR_REG_CHG_INT_CHGIN),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_WCIN, 0,
-		       MAX77759_CHGR_REG_CHG_INT_WCIN),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_CHG, 0,
-		       MAX77759_CHGR_REG_CHG_INT_CHG),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_BAT, 0,
-		       MAX77759_CHGR_REG_CHG_INT_BAT),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_INLIM, 0,
-		       MAX77759_CHGR_REG_CHG_INT_INLIM),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_THM2, 0,
-		       MAX77759_CHGR_REG_CHG_INT_THM2),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_BYP, 0,
-		       MAX77759_CHGR_REG_CHG_INT_BYP),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_INSEL, 1,
-		       MAX77759_CHGR_REG_CHG_INT2_INSEL),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_SYS_UVLO1, 1,
-		       MAX77759_CHGR_REG_CHG_INT2_SYS_UVLO1),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_SYS_UVLO2, 1,
-		       MAX77759_CHGR_REG_CHG_INT2_SYS_UVLO2),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_BAT_OILO, 1,
-		       MAX77759_CHGR_REG_CHG_INT2_BAT_OILO),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_CHG_STA_CC, 1,
-		       MAX77759_CHGR_REG_CHG_INT2_CHG_STA_CC),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_CHG_STA_CV, 1,
-		       MAX77759_CHGR_REG_CHG_INT2_CHG_STA_CV),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_CHG_STA_TO, 1,
-		       MAX77759_CHGR_REG_CHG_INT2_CHG_STA_TO),
-	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_CHG_STA_DONE, 1,
-		       MAX77759_CHGR_REG_CHG_INT2_CHG_STA_DONE),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_AICL, 0, MAX77759_CHGR_REG_CHG_INT_AICL),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_CHGIN, 0, MAX77759_CHGR_REG_CHG_INT_CHGIN),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_WCIN, 0, MAX77759_CHGR_REG_CHG_INT_WCIN),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_CHG, 0, MAX77759_CHGR_REG_CHG_INT_CHG),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_BAT, 0, MAX77759_CHGR_REG_CHG_INT_BAT),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_INLIM, 0, MAX77759_CHGR_REG_CHG_INT_INLIM),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_THM2, 0, MAX77759_CHGR_REG_CHG_INT_THM2),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT1_BYP, 0, MAX77759_CHGR_REG_CHG_INT_BYP),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_INSEL, 1, MAX77759_CHGR_REG_CHG_INT2_INSEL),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_SYS_UVLO1, 1, MAX77759_CHGR_REG_CHG_INT2_SYS_UVLO1),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_SYS_UVLO2, 1, MAX77759_CHGR_REG_CHG_INT2_SYS_UVLO2),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_BAT_OILO, 1, MAX77759_CHGR_REG_CHG_INT2_BAT_OILO),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_CHG_STA_CC, 1, MAX77759_CHGR_REG_CHG_INT2_CHG_STA_CC),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_CHG_STA_CV, 1, MAX77759_CHGR_REG_CHG_INT2_CHG_STA_CV),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_CHG_STA_TO, 1, MAX77759_CHGR_REG_CHG_INT2_CHG_STA_TO),
+	REGMAP_IRQ_REG(MAX77759_CHGR_INT2_CHG_STA_DONE, 1, MAX77759_CHGR_REG_CHG_INT2_CHG_STA_DONE),
 };
 
 static const struct regmap_irq_chip max77759_pmic_irq_chip = {
diff --git a/drivers/mfd/menf21bmc.c b/drivers/mfd/menf21bmc.c
index 1d36095155e04..0f24de516d724 100644
--- a/drivers/mfd/menf21bmc.c
+++ b/drivers/mfd/menf21bmc.c
@@ -54,11 +54,9 @@ menf21bmc_probe(struct i2c_client *client)
 	int rev_major, rev_minor, rev_main;
 	int ret;
 
-	ret = i2c_check_functionality(client->adapter,
-				      I2C_FUNC_SMBUS_BYTE_DATA |
-				      I2C_FUNC_SMBUS_WORD_DATA |
-				      I2C_FUNC_SMBUS_BYTE);
-	if (!ret)
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_SMBUS_BYTE_DATA |
+						      I2C_FUNC_SMBUS_WORD_DATA |
+						      I2C_FUNC_SMBUS_BYTE))
 		return -ENODEV;
 
 	rev_major = i2c_smbus_read_word_data(client, BMC_CMD_REV_MAJOR);
diff --git a/drivers/mfd/rsmu_i2c.c b/drivers/mfd/rsmu_i2c.c
index cba64f107a2fd..c57795ed20f4e 100644
--- a/drivers/mfd/rsmu_i2c.c
+++ b/drivers/mfd/rsmu_i2c.c
@@ -134,14 +134,10 @@ static int rsmu_i2c_write_device(struct rsmu_ddata *rsmu, u8 reg, u8 *buf, u8 by
 static int rsmu_write_page_register(struct rsmu_ddata *rsmu, u32 reg,
 				    rsmu_rw_device rsmu_write_device)
 {
-	u32 page = reg & RSMU_CM_PAGE_MASK;
+	u32 page = (reg | RSMU_CM_SCSR_BASE) & RSMU_CM_PAGE_MASK;
 	u8 buf[4];
 	int err;
 
-	/* Do not modify offset register for none-scsr registers */
-	if (reg < RSMU_CM_SCSR_BASE)
-		return 0;
-
 	/* Simply return if we are on the same page */
 	if (rsmu->page == page)
 		return 0;
@@ -334,6 +330,7 @@ static void rsmu_i2c_remove(struct i2c_client *client)
 static const struct i2c_device_id rsmu_i2c_id[] = {
 	{ "8a34000",  RSMU_CM },
 	{ "8a34001",  RSMU_CM },
+	{ "8a34002",  RSMU_CM },
 	{ "82p33810", RSMU_SABRE },
 	{ "82p33811", RSMU_SABRE },
 	{ "8v19n850", RSMU_SL },
@@ -345,6 +342,7 @@ MODULE_DEVICE_TABLE(i2c, rsmu_i2c_id);
 static const struct of_device_id rsmu_i2c_of_match[] = {
 	{ .compatible = "idt,8a34000",  .data = (void *)RSMU_CM },
 	{ .compatible = "idt,8a34001",  .data = (void *)RSMU_CM },
+	{ .compatible = "idt,8a34002",  .data = (void *)RSMU_CM },
 	{ .compatible = "idt,82p33810", .data = (void *)RSMU_SABRE },
 	{ .compatible = "idt,82p33811", .data = (void *)RSMU_SABRE },
 	{ .compatible = "idt,8v19n850", .data = (void *)RSMU_SL },
diff --git a/drivers/mfd/rsmu_spi.c b/drivers/mfd/rsmu_spi.c
index 39d9be1e141fb..e07f21482439e 100644
--- a/drivers/mfd/rsmu_spi.c
+++ b/drivers/mfd/rsmu_spi.c
@@ -101,11 +101,8 @@ static int rsmu_write_page_register(struct rsmu_ddata *rsmu, u32 reg)
 
 	switch (rsmu->type) {
 	case RSMU_CM:
-		/* Do not modify page register for none-scsr registers */
-		if (reg < RSMU_CM_SCSR_BASE)
-			return 0;
 		page_reg = RSMU_CM_PAGE_ADDR;
-		page = reg & RSMU_PAGE_MASK;
+		page = (reg | RSMU_CM_SCSR_BASE) & RSMU_PAGE_MASK;
 		buf[0] = (u8)(page & 0xFF);
 		buf[1] = (u8)((page >> 8) & 0xFF);
 		buf[2] = (u8)((page >> 16) & 0xFF);
@@ -244,6 +241,7 @@ static void rsmu_spi_remove(struct spi_device *client)
 static const struct spi_device_id rsmu_spi_id[] = {
 	{ "8a34000",  RSMU_CM },
 	{ "8a34001",  RSMU_CM },
+	{ "8a34002",  RSMU_CM },
 	{ "82p33810", RSMU_SABRE },
 	{ "82p33811", RSMU_SABRE },
 	{}
@@ -253,6 +251,7 @@ MODULE_DEVICE_TABLE(spi, rsmu_spi_id);
 static const struct of_device_id rsmu_spi_of_match[] = {
 	{ .compatible = "idt,8a34000",  .data = (void *)RSMU_CM },
 	{ .compatible = "idt,8a34001",  .data = (void *)RSMU_CM },
+	{ .compatible = "idt,8a34002",  .data = (void *)RSMU_CM },
 	{ .compatible = "idt,82p33810", .data = (void *)RSMU_SABRE },
 	{ .compatible = "idt,82p33811", .data = (void *)RSMU_SABRE },
 	{}
diff --git a/drivers/mfd/si476x-i2c.c b/drivers/mfd/si476x-i2c.c
index 899c0b5ea3aab..7ddc97dfc9402 100644
--- a/drivers/mfd/si476x-i2c.c
+++ b/drivers/mfd/si476x-i2c.c
@@ -30,7 +30,7 @@
  *
  * Configure the functions of the pins of the radio chip.
  *
- * The function returns zero in case of succes or negative error code
+ * The function returns zero in case of success or negative error code
  * otherwise.
  */
 static int si476x_core_config_pinmux(struct si476x_core *core)
@@ -121,7 +121,7 @@ static inline void si476x_core_schedule_polling_work(struct si476x_core *core)
  *       4. Configures, pin multiplexor, disables digital audio and
  *          configures interrupt sources.
  *
- * The function returns zero in case of succes or negative error code
+ * The function returns zero in case of success or negative error code
  * otherwise.
  */
 int si476x_core_start(struct si476x_core *core, bool soft)
@@ -215,7 +215,7 @@ EXPORT_SYMBOL_GPL(si476x_core_start);
  * 2. Send the POWER_DOWN command if the power down is soft or bring
  *    reset line low if not.
  *
- * The function returns zero in case of succes or negative error code
+ * The function returns zero in case of success or negative error code
  * otherwise.
  */
 int si476x_core_stop(struct si476x_core *core, bool soft)
@@ -428,7 +428,7 @@ static void si476x_core_pronounce_dead(struct si476x_core *core)
  * of I/O errors. If the error counter rises above the threshold
  * pronounce device dead.
  *
- * The function returns zero on succes or negative error code on
+ * The function returns zero on success or negative error code on
  * failure.
  */
 int si476x_core_i2c_xfer(struct si476x_core *core,
diff --git a/drivers/mfd/simple-mfd-i2c.c b/drivers/mfd/simple-mfd-i2c.c
index 7315fad618e44..52c81b18750e1 100644
--- a/drivers/mfd/simple-mfd-i2c.c
+++ b/drivers/mfd/simple-mfd-i2c.c
@@ -105,6 +105,7 @@ static const struct regmap_config spacemit_p1_regmap_config = {
 };
 
 static const struct mfd_cell spacemit_p1_cells[] = {
+	{ .name = "spacemit-p1-reboot", },
 	{ .name = "spacemit-p1-regulator", },
 	{ .name = "spacemit-p1-rtc", },
 };
diff --git a/drivers/mfd/sm501.c b/drivers/mfd/sm501.c
index 0ee6d8940e690..b5bda477ebfc2 100644
--- a/drivers/mfd/sm501.c
+++ b/drivers/mfd/sm501.c
@@ -704,9 +704,11 @@ static int sm501_register_device(struct sm501_devdata *sm,
 	if (ret >= 0) {
 		dev_dbg(sm->dev, "registered %s\n", pdev->name);
 		list_add_tail(&smdev->list, &sm->devices);
-	} else
+	} else {
 		dev_err(sm->dev, "error registering %s (%d)\n",
 			pdev->name, ret);
+		platform_device_put(pdev);
+	}
 
 	return ret;
 }
@@ -1638,8 +1640,8 @@ static void sm501_plat_remove(struct platform_device *dev)
 }
 
 static const struct pci_device_id sm501_pci_tbl[] = {
-	{ 0x126f, 0x0501, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ 0, },
+	{ PCI_DEVICE(0x126f, 0x0501) },
+	{ },
 };
 
 MODULE_DEVICE_TABLE(pci, sm501_pci_tbl);
diff --git a/drivers/mfd/sprd-sc27xx-spi.c b/drivers/mfd/sprd-sc27xx-spi.c
index d6b4350779e6a..aa052f6466231 100644
--- a/drivers/mfd/sprd-sc27xx-spi.c
+++ b/drivers/mfd/sprd-sc27xx-spi.c
@@ -14,6 +14,11 @@
 #include <linux/spi/spi.h>
 #include <uapi/linux/usb/charger.h>
 
+enum sprd_pmic_type {
+	PMIC_TYPE_SC2730 = 1,
+	PMIC_TYPE_SC2731,
+};
+
 #define SPRD_PMIC_INT_MASK_STATUS	0x0
 #define SPRD_PMIC_INT_RAW_STATUS	0x4
 #define SPRD_PMIC_INT_EN		0x8
@@ -50,6 +55,29 @@ struct sprd_pmic_data {
 	u32 charger_det;
 };
 
+static const struct mfd_cell sc2730_devices[] = {
+	MFD_CELL_OF("sc2730-adc", NULL, NULL, 0, 0, "sprd,sc2730-adc"),
+	MFD_CELL_OF("sc2730-bltc", NULL, NULL, 0, 0, "sprd,sc2730-bltc"),
+	MFD_CELL_OF("sc2730-efuse", NULL, NULL, 0, 0, "sprd,sc2730-efuse"),
+	MFD_CELL_OF("sc2730-eic", NULL, NULL, 0, 0, "sprd,sc2730-eic"),
+	MFD_CELL_OF("sc2730-fgu", NULL, NULL, 0, 0, "sprd,sc2730-fgu"),
+	MFD_CELL_OF("sc2730-rtc", NULL, NULL, 0, 0, "sprd,sc2730-rtc"),
+	MFD_CELL_OF("sc2730-vibrator", NULL, NULL, 0, 0, "sprd,sc2730-vibrator"),
+};
+
+static const struct mfd_cell sc2731_devices[] = {
+	MFD_CELL_OF("sc2731-adc", NULL, NULL, 0, 0, "sprd,sc2731-adc"),
+	MFD_CELL_OF("sc2731-bltc", NULL, NULL, 0, 0, "sprd,sc2731-bltc"),
+	MFD_CELL_OF("sc2731-charger", NULL, NULL, 0, 0, "sprd,sc2731-charger"),
+	MFD_CELL_OF("sc2731-efuse", NULL, NULL, 0, 0, "sprd,sc2731-efuse"),
+	MFD_CELL_OF("sc2731-eic", NULL, NULL, 0, 0, "sprd,sc2731-eic"),
+	MFD_CELL_OF("sc2731-fgu", NULL, NULL, 0, 0, "sprd,sc2731-fgu"),
+	MFD_CELL_NAME("sc2731-poweroff"),
+	MFD_CELL_NAME("sc2731-regulator"),
+	MFD_CELL_OF("sc2731-rtc", NULL, NULL, 0, 0, "sprd,sc2731-rtc"),
+	MFD_CELL_OF("sc2731-vibrator", NULL, NULL, 0, 0, "sprd,sc2731-vibrator"),
+};
+
 /*
  * Since different PMICs of SC27xx series can have different interrupt
  * base address and irq number, we should save irq number and irq base
@@ -152,12 +180,26 @@ static const struct regmap_config sprd_pmic_config = {
 static int sprd_pmic_probe(struct spi_device *spi)
 {
 	struct sprd_pmic *ddata;
+	enum sprd_pmic_type pmic_type;
 	const struct sprd_pmic_data *pdata;
-	int ret, i;
+	const struct mfd_cell *cells;
+	int ret, i, num_cells;
+
+	pmic_type = (uintptr_t)of_device_get_match_data(&spi->dev);
 
-	pdata = of_device_get_match_data(&spi->dev);
-	if (!pdata) {
-		dev_err(&spi->dev, "No matching driver data found\n");
+	switch (pmic_type) {
+	case PMIC_TYPE_SC2730:
+		pdata = &sc2730_data;
+		cells = sc2730_devices;
+		num_cells = ARRAY_SIZE(sc2730_devices);
+		break;
+	case PMIC_TYPE_SC2731:
+		pdata = &sc2731_data;
+		cells = sc2731_devices;
+		num_cells = ARRAY_SIZE(sc2731_devices);
+		break;
+	default:
+		dev_err(&spi->dev, "Invalid device ID\n");
 		return -EINVAL;
 	}
 
@@ -204,7 +246,9 @@ static int sprd_pmic_probe(struct spi_device *spi)
 		return ret;
 	}
 
-	ret = devm_of_platform_populate(&spi->dev);
+	ret = devm_mfd_add_devices(&spi->dev, PLATFORM_DEVID_AUTO,
+				   cells, num_cells, NULL, 0,
+				   regmap_irq_get_domain(ddata->irq_data));
 	if (ret) {
 		dev_err(&spi->dev, "Failed to populate sub-devices %d\n", ret);
 		return ret;
@@ -241,15 +285,15 @@ static DEFINE_SIMPLE_DEV_PM_OPS(sprd_pmic_pm_ops,
 				sprd_pmic_suspend, sprd_pmic_resume);
 
 static const struct of_device_id sprd_pmic_match[] = {
-	{ .compatible = "sprd,sc2730", .data = &sc2730_data },
-	{ .compatible = "sprd,sc2731", .data = &sc2731_data },
+	{ .compatible = "sprd,sc2730", .data = (void *)PMIC_TYPE_SC2730 },
+	{ .compatible = "sprd,sc2731", .data = (void *)PMIC_TYPE_SC2731 },
 	{},
 };
 MODULE_DEVICE_TABLE(of, sprd_pmic_match);
 
 static const struct spi_device_id sprd_pmic_spi_ids[] = {
-	{ .name = "sc2730", .driver_data = (unsigned long)&sc2730_data },
-	{ .name = "sc2731", .driver_data = (unsigned long)&sc2731_data },
+	{ .name = "sc2730", .driver_data = PMIC_TYPE_SC2730 },
+	{ .name = "sc2731", .driver_data = PMIC_TYPE_SC2731 },
 	{},
 };
 MODULE_DEVICE_TABLE(spi, sprd_pmic_spi_ids);
diff --git a/drivers/mfd/timberdale.c b/drivers/mfd/timberdale.c
index a4d9c070d481a..e75e1d6851abe 100644
--- a/drivers/mfd/timberdale.c
+++ b/drivers/mfd/timberdale.c
@@ -15,8 +15,6 @@
 #include <linux/property.h>
 #include <linux/slab.h>
 
-#include <linux/timb_gpio.h>
-
 #include <linux/i2c.h>
 #include <linux/platform_data/i2c-ocores.h>
 #include <linux/platform_data/i2c-xiic.h>
@@ -37,6 +35,10 @@
 
 #define DRIVER_NAME "timberdale"
 
+#define GPIO_NR_PINS	16
+#define GPIO_BASE	0
+#define IRQ_BASE	200
+
 struct timberdale_device {
 	resource_size_t		ctl_mapbase;
 	unsigned char __iomem   *ctl_membase;
@@ -174,11 +176,16 @@ static const struct resource timberdale_eth_resources[] = {
 	},
 };
 
-static struct timbgpio_platform_data
-	timberdale_gpio_platform_data = {
-	.gpio_base = 0,
-	.nr_pins = GPIO_NR_PINS,
-	.irq_base = 200,
+static const struct property_entry timberdale_gpio_properties[] = {
+	PROPERTY_ENTRY_U32("ngpios", GPIO_NR_PINS),
+	PROPERTY_ENTRY_U32("gpio-base", GPIO_BASE),
+	PROPERTY_ENTRY_U32("irq-base", IRQ_BASE),
+	{ }
+};
+
+static const struct software_node timberdale_gpio_swnode = {
+	.name = "timb-gpio",
+	.properties = timberdale_gpio_properties,
 };
 
 static const struct resource timberdale_gpio_resources[] = {
@@ -390,8 +397,7 @@ static const struct mfd_cell timberdale_cells_bar0_cfg0[] = {
 		.name = "timb-gpio",
 		.num_resources = ARRAY_SIZE(timberdale_gpio_resources),
 		.resources = timberdale_gpio_resources,
-		.platform_data = &timberdale_gpio_platform_data,
-		.pdata_size = sizeof(timberdale_gpio_platform_data),
+		.swnode = &timberdale_gpio_swnode,
 	},
 	{
 		.name = "timb-video",
@@ -452,8 +458,7 @@ static const struct mfd_cell timberdale_cells_bar0_cfg1[] = {
 		.name = "timb-gpio",
 		.num_resources = ARRAY_SIZE(timberdale_gpio_resources),
 		.resources = timberdale_gpio_resources,
-		.platform_data = &timberdale_gpio_platform_data,
-		.pdata_size = sizeof(timberdale_gpio_platform_data),
+		.swnode = &timberdale_gpio_swnode,
 	},
 	{
 		.name = "timb-mlogicore",
@@ -514,8 +519,7 @@ static const struct mfd_cell timberdale_cells_bar0_cfg2[] = {
 		.name = "timb-gpio",
 		.num_resources = ARRAY_SIZE(timberdale_gpio_resources),
 		.resources = timberdale_gpio_resources,
-		.platform_data = &timberdale_gpio_platform_data,
-		.pdata_size = sizeof(timberdale_gpio_platform_data),
+		.swnode = &timberdale_gpio_swnode,
 	},
 	{
 		.name = "timb-video",
@@ -564,8 +568,7 @@ static const struct mfd_cell timberdale_cells_bar0_cfg3[] = {
 		.name = "timb-gpio",
 		.num_resources = ARRAY_SIZE(timberdale_gpio_resources),
 		.resources = timberdale_gpio_resources,
-		.platform_data = &timberdale_gpio_platform_data,
-		.pdata_size = sizeof(timberdale_gpio_platform_data),
+		.swnode = &timberdale_gpio_swnode,
 	},
 	{
 		.name = "timb-video",
diff --git a/drivers/mfd/timberdale.h b/drivers/mfd/timberdale.h
index b01d2388e1afe..db7b434f766d0 100644
--- a/drivers/mfd/timberdale.h
+++ b/drivers/mfd/timberdale.h
@@ -113,7 +113,6 @@
 #define GPIO_PIN_ASCB		8
 #define GPIO_PIN_INIC_RST	14
 #define GPIO_PIN_BT_RST		15
-#define GPIO_NR_PINS		16
 
 /* DMA Channels */
 #define DMA_UART_RX         0
diff --git a/drivers/mfd/tps65219.c b/drivers/mfd/tps65219.c
index 7275dcdb7c44f..e52fbf1481fef 100644
--- a/drivers/mfd/tps65219.c
+++ b/drivers/mfd/tps65219.c
@@ -541,13 +541,15 @@ static int tps65219_probe(struct i2c_client *client)
 		return ret;
 	}
 
-	ret = devm_register_power_off_handler(tps->dev,
-					      tps65219_power_off_handler,
-					      tps);
-	if (ret) {
-		dev_err(tps->dev, "failed to register power-off handler: %d\n", ret);
-		return ret;
+	if (of_device_is_system_power_controller(tps->dev->of_node)) {
+		ret = devm_register_power_off_handler(tps->dev,
+						      tps65219_power_off_handler,
+						      tps);
+		if (ret)
+			return dev_err_probe(tps->dev, ret,
+					     "Failed to register power-off handler\n");
 	}
+
 	return 0;
 }
 
diff --git a/drivers/mfd/tps65910.c b/drivers/mfd/tps65910.c
index 6a7b7a697fb78..6243b66b1b6f2 100644
--- a/drivers/mfd/tps65910.c
+++ b/drivers/mfd/tps65910.c
@@ -21,6 +21,9 @@
 #include <linux/of.h>
 #include <linux/property.h>
 
+/* Dummy I2C transfer length for SWCZ010 workaround */
+#define TPS65910_DUMMY_XFER_LEN 1
+
 static const struct resource rtc_resources[] = {
 	{
 		.start  = TPS65910_IRQ_RTC_ALARM,
@@ -472,7 +475,18 @@ static int tps65910_i2c_probe(struct i2c_client *i2c)
 	 * first I2C transfer. So issue a dummy transfer before the first
 	 * real transfer.
 	 */
-	i2c_master_send(i2c, "", 1);
+	ret = i2c_master_send(i2c, "", TPS65910_DUMMY_XFER_LEN);
+	if (ret != TPS65910_DUMMY_XFER_LEN) {
+		int err;
+
+		if (ret < 0)
+			err = ret;
+		else
+			err = -EIO;
+
+		return dev_err_probe(&i2c->dev, err, "dummy transfer failed\n");
+	}
+
 	tps65910->regmap = devm_regmap_init_i2c(i2c, &tps65910_regmap_config);
 	if (IS_ERR(tps65910->regmap)) {
 		ret = PTR_ERR(tps65910->regmap);
diff --git a/drivers/mfd/twl-core.c b/drivers/mfd/twl-core.c
index f89eda4a17fe4..c024a28b057e3 100644
--- a/drivers/mfd/twl-core.c
+++ b/drivers/mfd/twl-core.c
@@ -754,7 +754,7 @@ twl_probe(struct i2c_client *client)
 		return status;
 	}
 
-	if (i2c_check_functionality(client->adapter, I2C_FUNC_I2C) == 0) {
+	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
 		dev_dbg(&client->dev, "can't talk I2C?\n");
 		status = -EIO;
 		goto free;
diff --git a/drivers/mfd/wm8994-irq.c b/drivers/mfd/wm8994-irq.c
index 1475b1ac6983b..a46cea9487632 100644
--- a/drivers/mfd/wm8994-irq.c
+++ b/drivers/mfd/wm8994-irq.c
@@ -135,53 +135,9 @@ static const struct regmap_irq_chip wm8994_irq_chip = {
 	.runtime_pm = true,
 };
 
-static void wm8994_edge_irq_enable(struct irq_data *data)
-{
-}
-
-static void wm8994_edge_irq_disable(struct irq_data *data)
-{
-}
-
-static struct irq_chip wm8994_edge_irq_chip = {
-	.name			= "wm8994_edge",
-	.irq_disable		= wm8994_edge_irq_disable,
-	.irq_enable		= wm8994_edge_irq_enable,
-};
-
-static irqreturn_t wm8994_edge_irq(int irq, void *data)
-{
-	struct wm8994 *wm8994 = data;
-
-	while (gpio_get_value_cansleep(wm8994->pdata.irq_gpio))
-		handle_nested_irq(irq_find_mapping(wm8994->edge_irq, 0));
-
-	return IRQ_HANDLED;
-}
-
-static int wm8994_edge_irq_map(struct irq_domain *h, unsigned int virq,
-			       irq_hw_number_t hw)
-{
-	struct wm8994 *wm8994 = h->host_data;
-
-	irq_set_chip_data(virq, wm8994);
-	irq_set_chip_and_handler(virq, &wm8994_edge_irq_chip, handle_edge_irq);
-	irq_set_nested_thread(virq, 1);
-	irq_set_noprobe(virq);
-
-	return 0;
-}
-
-static const struct irq_domain_ops wm8994_edge_irq_ops = {
-	.map	= wm8994_edge_irq_map,
-	.xlate	= irq_domain_xlate_twocell,
-};
-
 int wm8994_irq_init(struct wm8994 *wm8994)
 {
 	int ret;
-	unsigned long irqflags;
-	struct wm8994_pdata *pdata = &wm8994->pdata;
 
 	if (!wm8994->irq) {
 		dev_warn(wm8994->dev,
@@ -190,53 +146,11 @@ int wm8994_irq_init(struct wm8994 *wm8994)
 		return 0;
 	}
 
-	/* select user or default irq flags */
-	irqflags = IRQF_TRIGGER_HIGH | IRQF_ONESHOT;
-	if (pdata->irq_flags)
-		irqflags = pdata->irq_flags;
-
 	/* use a GPIO for edge triggered controllers */
-	if (irqflags & (IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING)) {
-		if (gpio_to_irq(pdata->irq_gpio) != wm8994->irq) {
-			dev_warn(wm8994->dev, "IRQ %d is not GPIO %d (%d)\n",
-				 wm8994->irq, pdata->irq_gpio,
-				 gpio_to_irq(pdata->irq_gpio));
-			wm8994->irq = gpio_to_irq(pdata->irq_gpio);
-		}
-
-		ret = devm_gpio_request_one(wm8994->dev, pdata->irq_gpio,
-					    GPIOF_IN, "WM8994 IRQ");
-
-		if (ret != 0) {
-			dev_err(wm8994->dev, "Failed to get IRQ GPIO: %d\n",
-				ret);
-			return ret;
-		}
-
-		wm8994->edge_irq = irq_domain_create_linear(NULL, 1, &wm8994_edge_irq_ops, wm8994);
-
-		ret = regmap_add_irq_chip(wm8994->regmap,
-					  irq_create_mapping(wm8994->edge_irq,
-							     0),
-					  IRQF_ONESHOT,
-					  wm8994->irq_base, &wm8994_irq_chip,
-					  &wm8994->irq_data);
-		if (ret != 0) {
-			dev_err(wm8994->dev, "Failed to get IRQ: %d\n",
-				ret);
-			return ret;
-		}
-
-		ret = request_threaded_irq(wm8994->irq,
-					   NULL, wm8994_edge_irq,
-					   irqflags,
-					   "WM8994 edge", wm8994);
-	} else {
-		ret = regmap_add_irq_chip(wm8994->regmap, wm8994->irq,
-					  irqflags,
-					  wm8994->irq_base, &wm8994_irq_chip,
-					  &wm8994->irq_data);
-	}
+	ret = regmap_add_irq_chip(wm8994->regmap, wm8994->irq,
+				  IRQF_TRIGGER_HIGH | IRQF_ONESHOT,
+				  wm8994->irq_base, &wm8994_irq_chip,
+				  &wm8994->irq_data);
 
 	if (ret != 0) {
 		dev_err(wm8994->dev, "Failed to register IRQ chip: %d\n", ret);
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 00683bf06258f..390256ed91f41 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -381,16 +381,6 @@ config SENSORS_BH1770
 	   To compile this driver as a module, choose M here: the
 	   module will be called bh1770glc. If unsure, say N here.
 
-config SENSORS_APDS990X
-	 tristate "APDS990X combined als and proximity sensors"
-	 depends on I2C
-	help
-	   Say Y here if you want to build a driver for Avago APDS990x
-	   combined ambient light and proximity sensor chip.
-
-	   To compile this driver as a module, choose M here: the
-	   module will be called apds990x. If unsure, say N here.
-
 config HMC6352
 	tristate "Honeywell HMC6352 compass"
 	depends on I2C
diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
index b32a2597d2467..fed47c7672b95 100644
--- a/drivers/misc/Makefile
+++ b/drivers/misc/Makefile
@@ -20,7 +20,6 @@ obj-$(CONFIG_RPMB)		+= rpmb-core.o
 obj-$(CONFIG_QCOM_COINCELL)	+= qcom-coincell.o
 obj-$(CONFIG_QCOM_FASTRPC)	+= fastrpc.o
 obj-$(CONFIG_SENSORS_BH1770)	+= bh1770glc.o
-obj-$(CONFIG_SENSORS_APDS990X)	+= apds990x.o
 obj-$(CONFIG_ENCLOSURE_SERVICES) += enclosure.o
 obj-$(CONFIG_KGDB_TESTS)	+= kgdbts.o
 obj-$(CONFIG_SGI_XP)		+= sgi-xp/
diff --git a/drivers/misc/apds990x.c b/drivers/misc/apds990x.c
deleted file mode 100644
index b69c3a1c94d1c..0000000000000
--- a/drivers/misc/apds990x.c
+++ /dev/null
@@ -1,1284 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * This file is part of the APDS990x sensor driver.
- * Chip is combined proximity and ambient light sensor.
- *
- * Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
- *
- * Contact: Samu Onkalo <samu.p.onkalo@nokia.com>
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/i2c.h>
-#include <linux/interrupt.h>
-#include <linux/mutex.h>
-#include <linux/regulator/consumer.h>
-#include <linux/pm_runtime.h>
-#include <linux/delay.h>
-#include <linux/wait.h>
-#include <linux/slab.h>
-#include <linux/platform_data/apds990x.h>
-
-/* Register map */
-#define APDS990X_ENABLE	 0x00 /* Enable of states and interrupts */
-#define APDS990X_ATIME	 0x01 /* ALS ADC time  */
-#define APDS990X_PTIME	 0x02 /* Proximity ADC time  */
-#define APDS990X_WTIME	 0x03 /* Wait time  */
-#define APDS990X_AILTL	 0x04 /* ALS interrupt low threshold low byte */
-#define APDS990X_AILTH	 0x05 /* ALS interrupt low threshold hi byte */
-#define APDS990X_AIHTL	 0x06 /* ALS interrupt hi threshold low byte */
-#define APDS990X_AIHTH	 0x07 /* ALS interrupt hi threshold hi byte */
-#define APDS990X_PILTL	 0x08 /* Proximity interrupt low threshold low byte */
-#define APDS990X_PILTH	 0x09 /* Proximity interrupt low threshold hi byte */
-#define APDS990X_PIHTL	 0x0a /* Proximity interrupt hi threshold low byte */
-#define APDS990X_PIHTH	 0x0b /* Proximity interrupt hi threshold hi byte */
-#define APDS990X_PERS	 0x0c /* Interrupt persistence filters */
-#define APDS990X_CONFIG	 0x0d /* Configuration */
-#define APDS990X_PPCOUNT 0x0e /* Proximity pulse count */
-#define APDS990X_CONTROL 0x0f /* Gain control register */
-#define APDS990X_REV	 0x11 /* Revision Number */
-#define APDS990X_ID	 0x12 /* Device ID */
-#define APDS990X_STATUS	 0x13 /* Device status */
-#define APDS990X_CDATAL	 0x14 /* Clear ADC low data register */
-#define APDS990X_CDATAH	 0x15 /* Clear ADC high data register */
-#define APDS990X_IRDATAL 0x16 /* IR ADC low data register */
-#define APDS990X_IRDATAH 0x17 /* IR ADC high data register */
-#define APDS990X_PDATAL	 0x18 /* Proximity ADC low data register */
-#define APDS990X_PDATAH	 0x19 /* Proximity ADC high data register */
-
-/* Control */
-#define APDS990X_MAX_AGAIN	3
-
-/* Enable register */
-#define APDS990X_EN_PIEN	(0x1 << 5)
-#define APDS990X_EN_AIEN	(0x1 << 4)
-#define APDS990X_EN_WEN		(0x1 << 3)
-#define APDS990X_EN_PEN		(0x1 << 2)
-#define APDS990X_EN_AEN		(0x1 << 1)
-#define APDS990X_EN_PON		(0x1 << 0)
-#define APDS990X_EN_DISABLE_ALL 0
-
-/* Status register */
-#define APDS990X_ST_PINT	(0x1 << 5)
-#define APDS990X_ST_AINT	(0x1 << 4)
-
-/* I2C access types */
-#define APDS990x_CMD_TYPE_MASK	(0x03 << 5)
-#define APDS990x_CMD_TYPE_RB	(0x00 << 5) /* Repeated byte */
-#define APDS990x_CMD_TYPE_INC	(0x01 << 5) /* Auto increment */
-#define APDS990x_CMD_TYPE_SPE	(0x03 << 5) /* Special function */
-
-#define APDS990x_ADDR_SHIFT	0
-#define APDS990x_CMD		0x80
-
-/* Interrupt ack commands */
-#define APDS990X_INT_ACK_ALS	0x6
-#define APDS990X_INT_ACK_PS	0x5
-#define APDS990X_INT_ACK_BOTH	0x7
-
-/* ptime */
-#define APDS990X_PTIME_DEFAULT	0xff /* Recommended conversion time 2.7ms*/
-
-/* wtime */
-#define APDS990X_WTIME_DEFAULT	0xee /* ~50ms wait time */
-
-#define APDS990X_TIME_TO_ADC	1024 /* One timetick as ADC count value */
-
-/* Persistence */
-#define APDS990X_APERS_SHIFT	0
-#define APDS990X_PPERS_SHIFT	4
-
-/* Supported ID:s */
-#define APDS990X_ID_0		0x0
-#define APDS990X_ID_4		0x4
-#define APDS990X_ID_29		0x29
-
-/* pgain and pdiode settings */
-#define APDS_PGAIN_1X	       0x0
-#define APDS_PDIODE_IR	       0x2
-
-#define APDS990X_LUX_OUTPUT_SCALE 10
-
-/* Reverse chip factors for threshold calculation */
-struct reverse_factors {
-	u32 afactor;
-	int cf1;
-	int irf1;
-	int cf2;
-	int irf2;
-};
-
-struct apds990x_chip {
-	struct apds990x_platform_data	*pdata;
-	struct i2c_client		*client;
-	struct mutex			mutex; /* avoid parallel access */
-	struct regulator_bulk_data	regs[2];
-	wait_queue_head_t		wait;
-
-	int	prox_en;
-	bool	prox_continuous_mode;
-	bool	lux_wait_fresh_res;
-
-	/* Chip parameters */
-	struct	apds990x_chip_factors	cf;
-	struct	reverse_factors		rcf;
-	u16	atime;		/* als integration time */
-	u16	arate;		/* als reporting rate */
-	u16	a_max_result;	/* Max possible ADC value with current atime */
-	u8	again_meas;	/* Gain used in last measurement */
-	u8	again_next;	/* Next calculated gain */
-	u8	pgain;
-	u8	pdiode;
-	u8	pdrive;
-	u8	lux_persistence;
-	u8	prox_persistence;
-
-	u32	lux_raw;
-	u32	lux;
-	u16	lux_clear;
-	u16	lux_ir;
-	u16	lux_calib;
-	u32	lux_thres_hi;
-	u32	lux_thres_lo;
-
-	u32	prox_thres;
-	u16	prox_data;
-	u16	prox_calib;
-
-	char	chipname[10];
-	u8	revision;
-};
-
-#define APDS_CALIB_SCALER		8192
-#define APDS_LUX_NEUTRAL_CALIB_VALUE	(1 * APDS_CALIB_SCALER)
-#define APDS_PROX_NEUTRAL_CALIB_VALUE	(1 * APDS_CALIB_SCALER)
-
-#define APDS_PROX_DEF_THRES		600
-#define APDS_PROX_HYSTERESIS		50
-#define APDS_LUX_DEF_THRES_HI		101
-#define APDS_LUX_DEF_THRES_LO		100
-#define APDS_DEFAULT_PROX_PERS		1
-
-#define APDS_TIMEOUT			2000
-#define APDS_STARTUP_DELAY		25000 /* us */
-#define APDS_RANGE			65535
-#define APDS_PROX_RANGE			1023
-#define APDS_LUX_GAIN_LO_LIMIT		100
-#define APDS_LUX_GAIN_LO_LIMIT_STRICT	25
-
-#define TIMESTEP			87 /* 2.7ms is about 87 / 32 */
-#define TIME_STEP_SCALER		32
-
-#define APDS_LUX_AVERAGING_TIME		50 /* tolerates 50/60Hz ripple */
-#define APDS_LUX_DEFAULT_RATE		200
-
-static const u8 again[]	= {1, 8, 16, 120}; /* ALS gain steps */
-
-/* Following two tables must match i.e 10Hz rate means 1 as persistence value */
-static const u16 arates_hz[] = {10, 5, 2, 1};
-static const u8 apersis[] = {1, 2, 4, 5};
-
-/* Regulators */
-static const char reg_vcc[] = "Vdd";
-static const char reg_vled[] = "Vled";
-
-static int apds990x_read_byte(struct apds990x_chip *chip, u8 reg, u8 *data)
-{
-	struct i2c_client *client = chip->client;
-	s32 ret;
-
-	reg &= ~APDS990x_CMD_TYPE_MASK;
-	reg |= APDS990x_CMD | APDS990x_CMD_TYPE_RB;
-
-	ret = i2c_smbus_read_byte_data(client, reg);
-	*data = ret;
-	return (int)ret;
-}
-
-static int apds990x_read_word(struct apds990x_chip *chip, u8 reg, u16 *data)
-{
-	struct i2c_client *client = chip->client;
-	s32 ret;
-
-	reg &= ~APDS990x_CMD_TYPE_MASK;
-	reg |= APDS990x_CMD | APDS990x_CMD_TYPE_INC;
-
-	ret = i2c_smbus_read_word_data(client, reg);
-	*data = ret;
-	return (int)ret;
-}
-
-static int apds990x_write_byte(struct apds990x_chip *chip, u8 reg, u8 data)
-{
-	struct i2c_client *client = chip->client;
-	s32 ret;
-
-	reg &= ~APDS990x_CMD_TYPE_MASK;
-	reg |= APDS990x_CMD | APDS990x_CMD_TYPE_RB;
-
-	ret = i2c_smbus_write_byte_data(client, reg, data);
-	return (int)ret;
-}
-
-static int apds990x_write_word(struct apds990x_chip *chip, u8 reg, u16 data)
-{
-	struct i2c_client *client = chip->client;
-	s32 ret;
-
-	reg &= ~APDS990x_CMD_TYPE_MASK;
-	reg |= APDS990x_CMD | APDS990x_CMD_TYPE_INC;
-
-	ret = i2c_smbus_write_word_data(client, reg, data);
-	return (int)ret;
-}
-
-static int apds990x_mode_on(struct apds990x_chip *chip)
-{
-	/* ALS is mandatory, proximity optional */
-	u8 reg = APDS990X_EN_AIEN | APDS990X_EN_PON | APDS990X_EN_AEN |
-		APDS990X_EN_WEN;
-
-	if (chip->prox_en)
-		reg |= APDS990X_EN_PIEN | APDS990X_EN_PEN;
-
-	return apds990x_write_byte(chip, APDS990X_ENABLE, reg);
-}
-
-static u16 apds990x_lux_to_threshold(struct apds990x_chip *chip, u32 lux)
-{
-	u32 thres;
-	u32 cpl;
-	u32 ir;
-
-	if (lux == 0)
-		return 0;
-	else if (lux == APDS_RANGE)
-		return APDS_RANGE;
-
-	/*
-	 * Reported LUX value is a combination of the IR and CLEAR channel
-	 * values. However, interrupt threshold is only for clear channel.
-	 * This function approximates needed HW threshold value for a given
-	 * LUX value in the current lightning type.
-	 * IR level compared to visible light varies heavily depending on the
-	 * source of the light
-	 *
-	 * Calculate threshold value for the next measurement period.
-	 * Math: threshold = lux * cpl where
-	 * cpl = atime * again / (glass_attenuation * device_factor)
-	 * (count-per-lux)
-	 *
-	 * First remove calibration. Division by four is to avoid overflow
-	 */
-	lux = lux * (APDS_CALIB_SCALER / 4) / (chip->lux_calib / 4);
-
-	/* Multiplication by 64 is to increase accuracy */
-	cpl = ((u32)chip->atime * (u32)again[chip->again_next] *
-		APDS_PARAM_SCALE * 64) / (chip->cf.ga * chip->cf.df);
-
-	thres = lux * cpl / 64;
-	/*
-	 * Convert IR light from the latest result to match with
-	 * new gain step. This helps to adapt with the current
-	 * source of light.
-	 */
-	ir = (u32)chip->lux_ir * (u32)again[chip->again_next] /
-		(u32)again[chip->again_meas];
-
-	/*
-	 * Compensate count with IR light impact
-	 * IAC1 > IAC2 (see apds990x_get_lux for formulas)
-	 */
-	if (chip->lux_clear * APDS_PARAM_SCALE >=
-		chip->rcf.afactor * chip->lux_ir)
-		thres = (chip->rcf.cf1 * thres + chip->rcf.irf1 * ir) /
-			APDS_PARAM_SCALE;
-	else
-		thres = (chip->rcf.cf2 * thres + chip->rcf.irf2 * ir) /
-			APDS_PARAM_SCALE;
-
-	if (thres >= chip->a_max_result)
-		thres = chip->a_max_result - 1;
-	return thres;
-}
-
-static inline int apds990x_set_atime(struct apds990x_chip *chip, u32 time_ms)
-{
-	u8 reg_value;
-
-	chip->atime = time_ms;
-	/* Formula is specified in the data sheet */
-	reg_value = 256 - ((time_ms * TIME_STEP_SCALER) / TIMESTEP);
-	/* Calculate max ADC value for given integration time */
-	chip->a_max_result = (u16)(256 - reg_value) * APDS990X_TIME_TO_ADC;
-	return apds990x_write_byte(chip, APDS990X_ATIME, reg_value);
-}
-
-/* Called always with mutex locked */
-static int apds990x_refresh_pthres(struct apds990x_chip *chip, int data)
-{
-	int ret, lo, hi;
-
-	/* If the chip is not in use, don't try to access it */
-	if (pm_runtime_suspended(&chip->client->dev))
-		return 0;
-
-	if (data < chip->prox_thres) {
-		lo = 0;
-		hi = chip->prox_thres;
-	} else {
-		lo = chip->prox_thres - APDS_PROX_HYSTERESIS;
-		if (chip->prox_continuous_mode)
-			hi = chip->prox_thres;
-		else
-			hi = APDS_RANGE;
-	}
-
-	ret = apds990x_write_word(chip, APDS990X_PILTL, lo);
-	ret |= apds990x_write_word(chip, APDS990X_PIHTL, hi);
-	return ret;
-}
-
-/* Called always with mutex locked */
-static int apds990x_refresh_athres(struct apds990x_chip *chip)
-{
-	int ret;
-	/* If the chip is not in use, don't try to access it */
-	if (pm_runtime_suspended(&chip->client->dev))
-		return 0;
-
-	ret = apds990x_write_word(chip, APDS990X_AILTL,
-			apds990x_lux_to_threshold(chip, chip->lux_thres_lo));
-	ret |= apds990x_write_word(chip, APDS990X_AIHTL,
-			apds990x_lux_to_threshold(chip, chip->lux_thres_hi));
-
-	return ret;
-}
-
-/* Called always with mutex locked */
-static void apds990x_force_a_refresh(struct apds990x_chip *chip)
-{
-	/* This will force ALS interrupt after the next measurement. */
-	apds990x_write_word(chip, APDS990X_AILTL, APDS_LUX_DEF_THRES_LO);
-	apds990x_write_word(chip, APDS990X_AIHTL, APDS_LUX_DEF_THRES_HI);
-}
-
-/* Called always with mutex locked */
-static void apds990x_force_p_refresh(struct apds990x_chip *chip)
-{
-	/* This will force proximity interrupt after the next measurement. */
-	apds990x_write_word(chip, APDS990X_PILTL, APDS_PROX_DEF_THRES - 1);
-	apds990x_write_word(chip, APDS990X_PIHTL, APDS_PROX_DEF_THRES);
-}
-
-/* Called always with mutex locked */
-static int apds990x_calc_again(struct apds990x_chip *chip)
-{
-	int curr_again = chip->again_meas;
-	int next_again = chip->again_meas;
-	int ret = 0;
-
-	/* Calculate suitable als gain */
-	if (chip->lux_clear == chip->a_max_result)
-		next_again -= 2; /* ALS saturated. Decrease gain by 2 steps */
-	else if (chip->lux_clear > chip->a_max_result / 2)
-		next_again--;
-	else if (chip->lux_clear < APDS_LUX_GAIN_LO_LIMIT_STRICT)
-		next_again += 2; /* Too dark. Increase gain by 2 steps */
-	else if (chip->lux_clear < APDS_LUX_GAIN_LO_LIMIT)
-		next_again++;
-
-	/* Limit gain to available range */
-	if (next_again < 0)
-		next_again = 0;
-	else if (next_again > APDS990X_MAX_AGAIN)
-		next_again = APDS990X_MAX_AGAIN;
-
-	/* Let's check can we trust the measured result */
-	if (chip->lux_clear == chip->a_max_result)
-		/* Result can be totally garbage due to saturation */
-		ret = -ERANGE;
-	else if (next_again != curr_again &&
-		chip->lux_clear < APDS_LUX_GAIN_LO_LIMIT_STRICT)
-		/*
-		 * Gain is changed and measurement result is very small.
-		 * Result can be totally garbage due to underflow
-		 */
-		ret = -ERANGE;
-
-	chip->again_next = next_again;
-	apds990x_write_byte(chip, APDS990X_CONTROL,
-			(chip->pdrive << 6) |
-			(chip->pdiode << 4) |
-			(chip->pgain << 2) |
-			(chip->again_next << 0));
-
-	/*
-	 * Error means bad result -> re-measurement is needed. The forced
-	 * refresh uses fastest possible persistence setting to get result
-	 * as soon as possible.
-	 */
-	if (ret < 0)
-		apds990x_force_a_refresh(chip);
-	else
-		apds990x_refresh_athres(chip);
-
-	return ret;
-}
-
-/* Called always with mutex locked */
-static int apds990x_get_lux(struct apds990x_chip *chip, int clear, int ir)
-{
-	int iac, iac1, iac2; /* IR adjusted counts */
-	u32 lpc; /* Lux per count */
-
-	/* Formulas:
-	 * iac1 = CF1 * CLEAR_CH - IRF1 * IR_CH
-	 * iac2 = CF2 * CLEAR_CH - IRF2 * IR_CH
-	 */
-	iac1 = (chip->cf.cf1 * clear - chip->cf.irf1 * ir) / APDS_PARAM_SCALE;
-	iac2 = (chip->cf.cf2 * clear - chip->cf.irf2 * ir) / APDS_PARAM_SCALE;
-
-	iac = max(iac1, iac2);
-	iac = max(iac, 0);
-
-	lpc = APDS990X_LUX_OUTPUT_SCALE * (chip->cf.df * chip->cf.ga) /
-		(u32)(again[chip->again_meas] * (u32)chip->atime);
-
-	return (iac * lpc) / APDS_PARAM_SCALE;
-}
-
-static int apds990x_ack_int(struct apds990x_chip *chip, u8 mode)
-{
-	struct i2c_client *client = chip->client;
-	s32 ret;
-	u8 reg = APDS990x_CMD | APDS990x_CMD_TYPE_SPE;
-
-	switch (mode & (APDS990X_ST_AINT | APDS990X_ST_PINT)) {
-	case APDS990X_ST_AINT:
-		reg |= APDS990X_INT_ACK_ALS;
-		break;
-	case APDS990X_ST_PINT:
-		reg |= APDS990X_INT_ACK_PS;
-		break;
-	default:
-		reg |= APDS990X_INT_ACK_BOTH;
-		break;
-	}
-
-	ret = i2c_smbus_read_byte_data(client, reg);
-	return (int)ret;
-}
-
-static irqreturn_t apds990x_irq(int irq, void *data)
-{
-	struct apds990x_chip *chip = data;
-	u8 status;
-
-	apds990x_read_byte(chip, APDS990X_STATUS, &status);
-	apds990x_ack_int(chip, status);
-
-	mutex_lock(&chip->mutex);
-	if (!pm_runtime_suspended(&chip->client->dev)) {
-		if (status & APDS990X_ST_AINT) {
-			apds990x_read_word(chip, APDS990X_CDATAL,
-					&chip->lux_clear);
-			apds990x_read_word(chip, APDS990X_IRDATAL,
-					&chip->lux_ir);
-			/* Store used gain for calculations */
-			chip->again_meas = chip->again_next;
-
-			chip->lux_raw = apds990x_get_lux(chip,
-							chip->lux_clear,
-							chip->lux_ir);
-
-			if (apds990x_calc_again(chip) == 0) {
-				/* Result is valid */
-				chip->lux = chip->lux_raw;
-				chip->lux_wait_fresh_res = false;
-				wake_up(&chip->wait);
-				sysfs_notify(&chip->client->dev.kobj,
-					NULL, "lux0_input");
-			}
-		}
-
-		if ((status & APDS990X_ST_PINT) && chip->prox_en) {
-			u16 clr_ch;
-
-			apds990x_read_word(chip, APDS990X_CDATAL, &clr_ch);
-			/*
-			 * If ALS channel is saturated at min gain,
-			 * proximity gives false posivite values.
-			 * Just ignore them.
-			 */
-			if (chip->again_meas == 0 &&
-				clr_ch == chip->a_max_result)
-				chip->prox_data = 0;
-			else
-				apds990x_read_word(chip,
-						APDS990X_PDATAL,
-						&chip->prox_data);
-
-			apds990x_refresh_pthres(chip, chip->prox_data);
-			if (chip->prox_data < chip->prox_thres)
-				chip->prox_data = 0;
-			else if (!chip->prox_continuous_mode)
-				chip->prox_data = APDS_PROX_RANGE;
-			sysfs_notify(&chip->client->dev.kobj,
-				NULL, "prox0_raw");
-		}
-	}
-	mutex_unlock(&chip->mutex);
-	return IRQ_HANDLED;
-}
-
-static int apds990x_configure(struct apds990x_chip *chip)
-{
-	/* It is recommended to use disabled mode during these operations */
-	apds990x_write_byte(chip, APDS990X_ENABLE, APDS990X_EN_DISABLE_ALL);
-
-	/* conversion and wait times for different state machince states */
-	apds990x_write_byte(chip, APDS990X_PTIME, APDS990X_PTIME_DEFAULT);
-	apds990x_write_byte(chip, APDS990X_WTIME, APDS990X_WTIME_DEFAULT);
-	apds990x_set_atime(chip, APDS_LUX_AVERAGING_TIME);
-
-	apds990x_write_byte(chip, APDS990X_CONFIG, 0);
-
-	/* Persistence levels */
-	apds990x_write_byte(chip, APDS990X_PERS,
-			(chip->lux_persistence << APDS990X_APERS_SHIFT) |
-			(chip->prox_persistence << APDS990X_PPERS_SHIFT));
-
-	apds990x_write_byte(chip, APDS990X_PPCOUNT, chip->pdata->ppcount);
-
-	/* Start with relatively small gain */
-	chip->again_meas = 1;
-	chip->again_next = 1;
-	apds990x_write_byte(chip, APDS990X_CONTROL,
-			(chip->pdrive << 6) |
-			(chip->pdiode << 4) |
-			(chip->pgain << 2) |
-			(chip->again_next << 0));
-	return 0;
-}
-
-static int apds990x_detect(struct apds990x_chip *chip)
-{
-	struct i2c_client *client = chip->client;
-	int ret;
-	u8 id;
-
-	ret = apds990x_read_byte(chip, APDS990X_ID, &id);
-	if (ret < 0) {
-		dev_err(&client->dev, "ID read failed\n");
-		return ret;
-	}
-
-	ret = apds990x_read_byte(chip, APDS990X_REV, &chip->revision);
-	if (ret < 0) {
-		dev_err(&client->dev, "REV read failed\n");
-		return ret;
-	}
-
-	switch (id) {
-	case APDS990X_ID_0:
-	case APDS990X_ID_4:
-	case APDS990X_ID_29:
-		snprintf(chip->chipname, sizeof(chip->chipname), "APDS-990x");
-		break;
-	default:
-		ret = -ENODEV;
-		break;
-	}
-	return ret;
-}
-
-#ifdef CONFIG_PM
-static int apds990x_chip_on(struct apds990x_chip *chip)
-{
-	int err	 = regulator_bulk_enable(ARRAY_SIZE(chip->regs),
-					chip->regs);
-	if (err < 0)
-		return err;
-
-	usleep_range(APDS_STARTUP_DELAY, 2 * APDS_STARTUP_DELAY);
-
-	/* Refresh all configs in case of regulators were off */
-	chip->prox_data = 0;
-	apds990x_configure(chip);
-	apds990x_mode_on(chip);
-	return 0;
-}
-#endif
-
-static int apds990x_chip_off(struct apds990x_chip *chip)
-{
-	apds990x_write_byte(chip, APDS990X_ENABLE, APDS990X_EN_DISABLE_ALL);
-	regulator_bulk_disable(ARRAY_SIZE(chip->regs), chip->regs);
-	return 0;
-}
-
-static ssize_t apds990x_lux_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	struct apds990x_chip *chip = dev_get_drvdata(dev);
-	ssize_t ret;
-	u32 result;
-	long time_left;
-
-	if (pm_runtime_suspended(dev))
-		return -EIO;
-
-	time_left = wait_event_interruptible_timeout(chip->wait,
-						     !chip->lux_wait_fresh_res,
-						     msecs_to_jiffies(APDS_TIMEOUT));
-	if (!time_left)
-		return -EIO;
-
-	mutex_lock(&chip->mutex);
-	result = (chip->lux * chip->lux_calib) / APDS_CALIB_SCALER;
-	if (result > (APDS_RANGE * APDS990X_LUX_OUTPUT_SCALE))
-		result = APDS_RANGE * APDS990X_LUX_OUTPUT_SCALE;
-
-	ret = sprintf(buf, "%d.%d\n",
-		result / APDS990X_LUX_OUTPUT_SCALE,
-		result % APDS990X_LUX_OUTPUT_SCALE);
-	mutex_unlock(&chip->mutex);
-	return ret;
-}
-
-static DEVICE_ATTR(lux0_input, S_IRUGO, apds990x_lux_show, NULL);
-
-static ssize_t apds990x_lux_range_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%u\n", APDS_RANGE);
-}
-
-static DEVICE_ATTR(lux0_sensor_range, S_IRUGO, apds990x_lux_range_show, NULL);
-
-static ssize_t apds990x_lux_calib_format_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%u\n", APDS_CALIB_SCALER);
-}
-
-static DEVICE_ATTR(lux0_calibscale_default, S_IRUGO,
-		apds990x_lux_calib_format_show, NULL);
-
-static ssize_t apds990x_lux_calib_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	struct apds990x_chip *chip = dev_get_drvdata(dev);
-
-	return sprintf(buf, "%u\n", chip->lux_calib);
-}
-
-static ssize_t apds990x_lux_calib_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct apds990x_chip *chip = dev_get_drvdata(dev);
-	unsigned long value;
-	int ret;
-
-	ret = kstrtoul(buf, 0, &value);
-	if (ret)
-		return ret;
-
-	chip->lux_calib = value;
-
-	return len;
-}
-
-static DEVICE_ATTR(lux0_calibscale, S_IRUGO | S_IWUSR, apds990x_lux_calib_show,
-		apds990x_lux_calib_store);
-
-static ssize_t apds990x_rate_avail(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	int i;
-	int pos = 0;
-
-	for (i = 0; i < ARRAY_SIZE(arates_hz); i++)
-		pos += sprintf(buf + pos, "%d ", arates_hz[i]);
-	sprintf(buf + pos - 1, "\n");
-	return pos;
-}
-
-static ssize_t apds990x_rate_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-
-	return sprintf(buf, "%d\n", chip->arate);
-}
-
-static int apds990x_set_arate(struct apds990x_chip *chip, int rate)
-{
-	int i;
-
-	for (i = 0; i < ARRAY_SIZE(arates_hz); i++)
-		if (rate >= arates_hz[i])
-			break;
-
-	if (i == ARRAY_SIZE(arates_hz))
-		return -EINVAL;
-
-	/* Pick up corresponding persistence value */
-	chip->lux_persistence = apersis[i];
-	chip->arate = arates_hz[i];
-
-	/* If the chip is not in use, don't try to access it */
-	if (pm_runtime_suspended(&chip->client->dev))
-		return 0;
-
-	/* Persistence levels */
-	return apds990x_write_byte(chip, APDS990X_PERS,
-			(chip->lux_persistence << APDS990X_APERS_SHIFT) |
-			(chip->prox_persistence << APDS990X_PPERS_SHIFT));
-}
-
-static ssize_t apds990x_rate_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-	unsigned long value;
-	int ret;
-
-	ret = kstrtoul(buf, 0, &value);
-	if (ret)
-		return ret;
-
-	mutex_lock(&chip->mutex);
-	ret = apds990x_set_arate(chip, value);
-	mutex_unlock(&chip->mutex);
-
-	if (ret < 0)
-		return ret;
-	return len;
-}
-
-static DEVICE_ATTR(lux0_rate_avail, S_IRUGO, apds990x_rate_avail, NULL);
-
-static DEVICE_ATTR(lux0_rate, S_IRUGO | S_IWUSR, apds990x_rate_show,
-						 apds990x_rate_store);
-
-static ssize_t apds990x_prox_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	ssize_t ret;
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-
-	if (pm_runtime_suspended(dev) || !chip->prox_en)
-		return -EIO;
-
-	mutex_lock(&chip->mutex);
-	ret = sprintf(buf, "%d\n", chip->prox_data);
-	mutex_unlock(&chip->mutex);
-	return ret;
-}
-
-static DEVICE_ATTR(prox0_raw, S_IRUGO, apds990x_prox_show, NULL);
-
-static ssize_t apds990x_prox_range_show(struct device *dev,
-				 struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%u\n", APDS_PROX_RANGE);
-}
-
-static DEVICE_ATTR(prox0_sensor_range, S_IRUGO, apds990x_prox_range_show, NULL);
-
-static ssize_t apds990x_prox_enable_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-
-	return sprintf(buf, "%d\n", chip->prox_en);
-}
-
-static ssize_t apds990x_prox_enable_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-	unsigned long value;
-	int ret;
-
-	ret = kstrtoul(buf, 0, &value);
-	if (ret)
-		return ret;
-
-	mutex_lock(&chip->mutex);
-
-	if (!chip->prox_en)
-		chip->prox_data = 0;
-
-	if (value)
-		chip->prox_en++;
-	else if (chip->prox_en > 0)
-		chip->prox_en--;
-
-	if (!pm_runtime_suspended(dev))
-		apds990x_mode_on(chip);
-	mutex_unlock(&chip->mutex);
-	return len;
-}
-
-static DEVICE_ATTR(prox0_raw_en, S_IRUGO | S_IWUSR, apds990x_prox_enable_show,
-						   apds990x_prox_enable_store);
-
-static const char *reporting_modes[] = {"trigger", "periodic"};
-
-static ssize_t apds990x_prox_reporting_mode_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-
-	return sprintf(buf, "%s\n",
-		reporting_modes[!!chip->prox_continuous_mode]);
-}
-
-static ssize_t apds990x_prox_reporting_mode_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-	int ret;
-
-	ret = sysfs_match_string(reporting_modes, buf);
-	if (ret < 0)
-		return ret;
-
-	chip->prox_continuous_mode = ret;
-	return len;
-}
-
-static DEVICE_ATTR(prox0_reporting_mode, S_IRUGO | S_IWUSR,
-		apds990x_prox_reporting_mode_show,
-		apds990x_prox_reporting_mode_store);
-
-static ssize_t apds990x_prox_reporting_avail_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%s %s\n", reporting_modes[0], reporting_modes[1]);
-}
-
-static DEVICE_ATTR(prox0_reporting_mode_avail, S_IRUGO | S_IWUSR,
-		apds990x_prox_reporting_avail_show, NULL);
-
-
-static ssize_t apds990x_lux_thresh_above_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-
-	return sprintf(buf, "%d\n", chip->lux_thres_hi);
-}
-
-static ssize_t apds990x_lux_thresh_below_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-
-	return sprintf(buf, "%d\n", chip->lux_thres_lo);
-}
-
-static ssize_t apds990x_set_lux_thresh(struct apds990x_chip *chip, u32 *target,
-				const char *buf)
-{
-	unsigned long thresh;
-	int ret;
-
-	ret = kstrtoul(buf, 0, &thresh);
-	if (ret)
-		return ret;
-
-	if (thresh > APDS_RANGE)
-		return -EINVAL;
-
-	mutex_lock(&chip->mutex);
-	*target = thresh;
-	/*
-	 * Don't update values in HW if we are still waiting for
-	 * first interrupt to come after device handle open call.
-	 */
-	if (!chip->lux_wait_fresh_res)
-		apds990x_refresh_athres(chip);
-	mutex_unlock(&chip->mutex);
-	return ret;
-
-}
-
-static ssize_t apds990x_lux_thresh_above_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-	int ret = apds990x_set_lux_thresh(chip, &chip->lux_thres_hi, buf);
-
-	if (ret < 0)
-		return ret;
-	return len;
-}
-
-static ssize_t apds990x_lux_thresh_below_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-	int ret = apds990x_set_lux_thresh(chip, &chip->lux_thres_lo, buf);
-
-	if (ret < 0)
-		return ret;
-	return len;
-}
-
-static DEVICE_ATTR(lux0_thresh_above_value, S_IRUGO | S_IWUSR,
-		apds990x_lux_thresh_above_show,
-		apds990x_lux_thresh_above_store);
-
-static DEVICE_ATTR(lux0_thresh_below_value, S_IRUGO | S_IWUSR,
-		apds990x_lux_thresh_below_show,
-		apds990x_lux_thresh_below_store);
-
-static ssize_t apds990x_prox_threshold_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-
-	return sprintf(buf, "%d\n", chip->prox_thres);
-}
-
-static ssize_t apds990x_prox_threshold_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-	unsigned long value;
-	int ret;
-
-	ret = kstrtoul(buf, 0, &value);
-	if (ret)
-		return ret;
-
-	if ((value > APDS_RANGE) || (value == 0) ||
-		(value < APDS_PROX_HYSTERESIS))
-		return -EINVAL;
-
-	mutex_lock(&chip->mutex);
-	chip->prox_thres = value;
-
-	apds990x_force_p_refresh(chip);
-	mutex_unlock(&chip->mutex);
-	return len;
-}
-
-static DEVICE_ATTR(prox0_thresh_above_value, S_IRUGO | S_IWUSR,
-		apds990x_prox_threshold_show,
-		apds990x_prox_threshold_store);
-
-static ssize_t apds990x_power_state_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	return sprintf(buf, "%d\n", !pm_runtime_suspended(dev));
-}
-
-static ssize_t apds990x_power_state_store(struct device *dev,
-				  struct device_attribute *attr,
-				  const char *buf, size_t len)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-	unsigned long value;
-	int ret;
-
-	ret = kstrtoul(buf, 0, &value);
-	if (ret)
-		return ret;
-
-	if (value) {
-		pm_runtime_get_sync(dev);
-		mutex_lock(&chip->mutex);
-		chip->lux_wait_fresh_res = true;
-		apds990x_force_a_refresh(chip);
-		apds990x_force_p_refresh(chip);
-		mutex_unlock(&chip->mutex);
-	} else {
-		if (!pm_runtime_suspended(dev))
-			pm_runtime_put(dev);
-	}
-	return len;
-}
-
-static DEVICE_ATTR(power_state, S_IRUGO | S_IWUSR,
-		apds990x_power_state_show,
-		apds990x_power_state_store);
-
-static ssize_t apds990x_chip_id_show(struct device *dev,
-				   struct device_attribute *attr, char *buf)
-{
-	struct apds990x_chip *chip =  dev_get_drvdata(dev);
-
-	return sprintf(buf, "%s %d\n", chip->chipname, chip->revision);
-}
-
-static DEVICE_ATTR(chip_id, S_IRUGO, apds990x_chip_id_show, NULL);
-
-static struct attribute *sysfs_attrs_ctrl[] = {
-	&dev_attr_lux0_calibscale.attr,
-	&dev_attr_lux0_calibscale_default.attr,
-	&dev_attr_lux0_input.attr,
-	&dev_attr_lux0_sensor_range.attr,
-	&dev_attr_lux0_rate.attr,
-	&dev_attr_lux0_rate_avail.attr,
-	&dev_attr_lux0_thresh_above_value.attr,
-	&dev_attr_lux0_thresh_below_value.attr,
-	&dev_attr_prox0_raw_en.attr,
-	&dev_attr_prox0_raw.attr,
-	&dev_attr_prox0_sensor_range.attr,
-	&dev_attr_prox0_thresh_above_value.attr,
-	&dev_attr_prox0_reporting_mode.attr,
-	&dev_attr_prox0_reporting_mode_avail.attr,
-	&dev_attr_chip_id.attr,
-	&dev_attr_power_state.attr,
-	NULL
-};
-
-static const struct attribute_group apds990x_attribute_group[] = {
-	{.attrs = sysfs_attrs_ctrl },
-};
-
-static int apds990x_probe(struct i2c_client *client)
-{
-	struct apds990x_chip *chip;
-	int err;
-
-	chip = kzalloc_obj(*chip);
-	if (!chip)
-		return -ENOMEM;
-
-	i2c_set_clientdata(client, chip);
-	chip->client  = client;
-
-	init_waitqueue_head(&chip->wait);
-	mutex_init(&chip->mutex);
-	chip->pdata	= client->dev.platform_data;
-
-	if (chip->pdata == NULL) {
-		dev_err(&client->dev, "platform data is mandatory\n");
-		err = -EINVAL;
-		goto fail1;
-	}
-
-	if (chip->pdata->cf.ga == 0) {
-		/* set uncovered sensor default parameters */
-		chip->cf.ga = 1966; /* 0.48 * APDS_PARAM_SCALE */
-		chip->cf.cf1 = 4096; /* 1.00 * APDS_PARAM_SCALE */
-		chip->cf.irf1 = 9134; /* 2.23 * APDS_PARAM_SCALE */
-		chip->cf.cf2 = 2867; /* 0.70 * APDS_PARAM_SCALE */
-		chip->cf.irf2 = 5816; /* 1.42 * APDS_PARAM_SCALE */
-		chip->cf.df = 52;
-	} else {
-		chip->cf = chip->pdata->cf;
-	}
-
-	/* precalculate inverse chip factors for threshold control */
-	chip->rcf.afactor =
-		(chip->cf.irf1 - chip->cf.irf2) * APDS_PARAM_SCALE /
-		(chip->cf.cf1 - chip->cf.cf2);
-	chip->rcf.cf1 = APDS_PARAM_SCALE * APDS_PARAM_SCALE /
-		chip->cf.cf1;
-	chip->rcf.irf1 = chip->cf.irf1 * APDS_PARAM_SCALE /
-		chip->cf.cf1;
-	chip->rcf.cf2 = APDS_PARAM_SCALE * APDS_PARAM_SCALE /
-		chip->cf.cf2;
-	chip->rcf.irf2 = chip->cf.irf2 * APDS_PARAM_SCALE /
-		chip->cf.cf2;
-
-	/* Set something to start with */
-	chip->lux_thres_hi = APDS_LUX_DEF_THRES_HI;
-	chip->lux_thres_lo = APDS_LUX_DEF_THRES_LO;
-	chip->lux_calib = APDS_LUX_NEUTRAL_CALIB_VALUE;
-
-	chip->prox_thres = APDS_PROX_DEF_THRES;
-	chip->pdrive = chip->pdata->pdrive;
-	chip->pdiode = APDS_PDIODE_IR;
-	chip->pgain = APDS_PGAIN_1X;
-	chip->prox_calib = APDS_PROX_NEUTRAL_CALIB_VALUE;
-	chip->prox_persistence = APDS_DEFAULT_PROX_PERS;
-	chip->prox_continuous_mode = false;
-
-	chip->regs[0].supply = reg_vcc;
-	chip->regs[1].supply = reg_vled;
-
-	err = regulator_bulk_get(&client->dev,
-				 ARRAY_SIZE(chip->regs), chip->regs);
-	if (err < 0) {
-		dev_err(&client->dev, "Cannot get regulators\n");
-		goto fail1;
-	}
-
-	err = regulator_bulk_enable(ARRAY_SIZE(chip->regs), chip->regs);
-	if (err < 0) {
-		dev_err(&client->dev, "Cannot enable regulators\n");
-		goto fail2;
-	}
-
-	usleep_range(APDS_STARTUP_DELAY, 2 * APDS_STARTUP_DELAY);
-
-	err = apds990x_detect(chip);
-	if (err < 0) {
-		dev_err(&client->dev, "APDS990X not found\n");
-		goto fail3;
-	}
-
-	pm_runtime_set_active(&client->dev);
-
-	apds990x_configure(chip);
-	apds990x_set_arate(chip, APDS_LUX_DEFAULT_RATE);
-	apds990x_mode_on(chip);
-
-	pm_runtime_enable(&client->dev);
-
-	if (chip->pdata->setup_resources) {
-		err = chip->pdata->setup_resources();
-		if (err) {
-			err = -EINVAL;
-			goto fail4;
-		}
-	}
-
-	err = sysfs_create_group(&chip->client->dev.kobj,
-				apds990x_attribute_group);
-	if (err < 0) {
-		dev_err(&chip->client->dev, "Sysfs registration failed\n");
-		goto fail5;
-	}
-
-	err = request_threaded_irq(client->irq, NULL,
-				apds990x_irq,
-				IRQF_TRIGGER_FALLING | IRQF_TRIGGER_LOW |
-				IRQF_ONESHOT,
-				"apds990x", chip);
-	if (err) {
-		dev_err(&client->dev, "could not get IRQ %d\n",
-			client->irq);
-		goto fail6;
-	}
-	return err;
-fail6:
-	sysfs_remove_group(&chip->client->dev.kobj,
-			&apds990x_attribute_group[0]);
-fail5:
-	if (chip->pdata && chip->pdata->release_resources)
-		chip->pdata->release_resources();
-fail4:
-	pm_runtime_disable(&client->dev);
-fail3:
-	regulator_bulk_disable(ARRAY_SIZE(chip->regs), chip->regs);
-fail2:
-	regulator_bulk_free(ARRAY_SIZE(chip->regs), chip->regs);
-fail1:
-	kfree(chip);
-	return err;
-}
-
-static void apds990x_remove(struct i2c_client *client)
-{
-	struct apds990x_chip *chip = i2c_get_clientdata(client);
-
-	free_irq(client->irq, chip);
-	sysfs_remove_group(&chip->client->dev.kobj,
-			apds990x_attribute_group);
-
-	if (chip->pdata && chip->pdata->release_resources)
-		chip->pdata->release_resources();
-
-	if (!pm_runtime_suspended(&client->dev))
-		apds990x_chip_off(chip);
-
-	pm_runtime_disable(&client->dev);
-	pm_runtime_set_suspended(&client->dev);
-
-	regulator_bulk_free(ARRAY_SIZE(chip->regs), chip->regs);
-
-	kfree(chip);
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int apds990x_suspend(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct apds990x_chip *chip = i2c_get_clientdata(client);
-
-	apds990x_chip_off(chip);
-	return 0;
-}
-
-static int apds990x_resume(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct apds990x_chip *chip = i2c_get_clientdata(client);
-
-	/*
-	 * If we were enabled at suspend time, it is expected
-	 * everything works nice and smoothly. Chip_on is enough
-	 */
-	apds990x_chip_on(chip);
-
-	return 0;
-}
-#endif
-
-#ifdef CONFIG_PM
-static int apds990x_runtime_suspend(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct apds990x_chip *chip = i2c_get_clientdata(client);
-
-	apds990x_chip_off(chip);
-	return 0;
-}
-
-static int apds990x_runtime_resume(struct device *dev)
-{
-	struct i2c_client *client = to_i2c_client(dev);
-	struct apds990x_chip *chip = i2c_get_clientdata(client);
-
-	apds990x_chip_on(chip);
-	return 0;
-}
-
-#endif
-
-static const struct i2c_device_id apds990x_id[] = {
-	{ "apds990x" },
-	{}
-};
-
-MODULE_DEVICE_TABLE(i2c, apds990x_id);
-
-static const struct dev_pm_ops apds990x_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(apds990x_suspend, apds990x_resume)
-	SET_RUNTIME_PM_OPS(apds990x_runtime_suspend,
-			apds990x_runtime_resume,
-			NULL)
-};
-
-static struct i2c_driver apds990x_driver = {
-	.driver	  = {
-		.name	= "apds990x",
-		.pm	= &apds990x_pm_ops,
-	},
-	.probe    = apds990x_probe,
-	.remove	  = apds990x_remove,
-	.id_table = apds990x_id,
-};
-
-module_i2c_driver(apds990x_driver);
-
-MODULE_DESCRIPTION("APDS990X combined ALS and proximity sensor");
-MODULE_AUTHOR("Samu Onkalo, Nokia Corporation");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c
index 0200288d3a7ab..5d5f357a19961 100644
--- a/drivers/misc/eeprom/at24.c
+++ b/drivers/misc/eeprom/at24.c
@@ -215,37 +215,37 @@ AT24_CHIP_DATA(at24_data_24c2048, 2097152 / 8, AT24_FLAG_ADDR16);
 AT24_CHIP_DATA(at24_data_INT3499, 8192 / 8, 0);
 
 static const struct i2c_device_id at24_ids[] = {
-	{ "24c00",	(kernel_ulong_t)&at24_data_24c00 },
-	{ "24c01",	(kernel_ulong_t)&at24_data_24c01 },
-	{ "24cs01",	(kernel_ulong_t)&at24_data_24cs01 },
-	{ "24c02",	(kernel_ulong_t)&at24_data_24c02 },
-	{ "24cs02",	(kernel_ulong_t)&at24_data_24cs02 },
-	{ "24mac402",	(kernel_ulong_t)&at24_data_24mac402 },
-	{ "24mac602",	(kernel_ulong_t)&at24_data_24mac602 },
-	{ "24aa025e48",	(kernel_ulong_t)&at24_data_24aa025e48 },
-	{ "24aa025e64",	(kernel_ulong_t)&at24_data_24aa025e64 },
-	{ "spd",	(kernel_ulong_t)&at24_data_spd },
-	{ "24c02-vaio",	(kernel_ulong_t)&at24_data_24c02_vaio },
-	{ "24c04",	(kernel_ulong_t)&at24_data_24c04 },
-	{ "24cs04",	(kernel_ulong_t)&at24_data_24cs04 },
-	{ "24c08",	(kernel_ulong_t)&at24_data_24c08 },
-	{ "24cs08",	(kernel_ulong_t)&at24_data_24cs08 },
-	{ "24c16",	(kernel_ulong_t)&at24_data_24c16 },
-	{ "24cs16",	(kernel_ulong_t)&at24_data_24cs16 },
-	{ "24c32",	(kernel_ulong_t)&at24_data_24c32 },
-	{ "24c32d-wl",	(kernel_ulong_t)&at24_data_24c32d_wlp },
-	{ "24cs32",	(kernel_ulong_t)&at24_data_24cs32 },
-	{ "24c64",	(kernel_ulong_t)&at24_data_24c64 },
-	{ "24c64-wl",	(kernel_ulong_t)&at24_data_24c64d_wlp },
-	{ "24cs64",	(kernel_ulong_t)&at24_data_24cs64 },
-	{ "24c128",	(kernel_ulong_t)&at24_data_24c128 },
-	{ "24c256",	(kernel_ulong_t)&at24_data_24c256 },
-	{ "24256e-wl",	(kernel_ulong_t)&at24_data_24256e_wlp },
-	{ "24c512",	(kernel_ulong_t)&at24_data_24c512 },
-	{ "24c1024",	(kernel_ulong_t)&at24_data_24c1024 },
-	{ "24c1025",	(kernel_ulong_t)&at24_data_24c1025 },
-	{ "24c2048",    (kernel_ulong_t)&at24_data_24c2048 },
-	{ "at24",	0 },
+	{ .name = "24c00",	.driver_data = (kernel_ulong_t)&at24_data_24c00 },
+	{ .name = "24c01",	.driver_data = (kernel_ulong_t)&at24_data_24c01 },
+	{ .name = "24cs01",	.driver_data = (kernel_ulong_t)&at24_data_24cs01 },
+	{ .name = "24c02",	.driver_data = (kernel_ulong_t)&at24_data_24c02 },
+	{ .name = "24cs02",	.driver_data = (kernel_ulong_t)&at24_data_24cs02 },
+	{ .name = "24mac402",	.driver_data = (kernel_ulong_t)&at24_data_24mac402 },
+	{ .name = "24mac602",	.driver_data = (kernel_ulong_t)&at24_data_24mac602 },
+	{ .name = "24aa025e48",	.driver_data = (kernel_ulong_t)&at24_data_24aa025e48 },
+	{ .name = "24aa025e64",	.driver_data = (kernel_ulong_t)&at24_data_24aa025e64 },
+	{ .name = "spd",	.driver_data = (kernel_ulong_t)&at24_data_spd },
+	{ .name = "24c02-vaio",	.driver_data = (kernel_ulong_t)&at24_data_24c02_vaio },
+	{ .name = "24c04",	.driver_data = (kernel_ulong_t)&at24_data_24c04 },
+	{ .name = "24cs04",	.driver_data = (kernel_ulong_t)&at24_data_24cs04 },
+	{ .name = "24c08",	.driver_data = (kernel_ulong_t)&at24_data_24c08 },
+	{ .name = "24cs08",	.driver_data = (kernel_ulong_t)&at24_data_24cs08 },
+	{ .name = "24c16",	.driver_data = (kernel_ulong_t)&at24_data_24c16 },
+	{ .name = "24cs16",	.driver_data = (kernel_ulong_t)&at24_data_24cs16 },
+	{ .name = "24c32",	.driver_data = (kernel_ulong_t)&at24_data_24c32 },
+	{ .name = "24c32d-wl",	.driver_data = (kernel_ulong_t)&at24_data_24c32d_wlp },
+	{ .name = "24cs32",	.driver_data = (kernel_ulong_t)&at24_data_24cs32 },
+	{ .name = "24c64",	.driver_data = (kernel_ulong_t)&at24_data_24c64 },
+	{ .name = "24c64-wl",	.driver_data = (kernel_ulong_t)&at24_data_24c64d_wlp },
+	{ .name = "24cs64",	.driver_data = (kernel_ulong_t)&at24_data_24cs64 },
+	{ .name = "24c128",	.driver_data = (kernel_ulong_t)&at24_data_24c128 },
+	{ .name = "24c256",	.driver_data = (kernel_ulong_t)&at24_data_24c256 },
+	{ .name = "24256e-wl",	.driver_data = (kernel_ulong_t)&at24_data_24256e_wlp },
+	{ .name = "24c512",	.driver_data = (kernel_ulong_t)&at24_data_24c512 },
+	{ .name = "24c1024",	.driver_data = (kernel_ulong_t)&at24_data_24c1024 },
+	{ .name = "24c1025",	.driver_data = (kernel_ulong_t)&at24_data_24c1025 },
+	{ .name = "24c2048",	.driver_data = (kernel_ulong_t)&at24_data_24c2048 },
+	{ .name = "at24",	.driver_data = 0 },
 	{ /* END OF LIST */ }
 };
 MODULE_DEVICE_TABLE(i2c, at24_ids);
diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c
index 1080f9acf70a3..5524354f3ccc8 100644
--- a/drivers/misc/fastrpc.c
+++ b/drivers/misc/fastrpc.c
@@ -310,6 +310,8 @@ struct fastrpc_user {
 	spinlock_t lock;
 	/* lock for allocations */
 	struct mutex mutex;
+	/* Reference count */
+	struct kref refcount;
 };
 
 /* Extract SMMU PA from consolidated IOVA */
@@ -497,15 +499,57 @@ static void fastrpc_channel_ctx_put(struct fastrpc_channel_ctx *cctx)
 	kref_put(&cctx->refcount, fastrpc_channel_ctx_free);
 }
 
+static void fastrpc_context_put(struct fastrpc_invoke_ctx *ctx);
+
+static void fastrpc_user_free(struct kref *ref)
+{
+	struct fastrpc_user *fl = container_of(ref, struct fastrpc_user, refcount);
+	struct fastrpc_invoke_ctx *ctx, *n;
+	struct fastrpc_map *map, *m;
+	struct fastrpc_buf *buf, *b;
+
+	if (fl->init_mem)
+		fastrpc_buf_free(fl->init_mem);
+
+	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
+		list_del(&ctx->node);
+		fastrpc_context_put(ctx);
+	}
+
+	list_for_each_entry_safe(map, m, &fl->maps, node)
+		fastrpc_map_put(map);
+
+	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
+		list_del(&buf->node);
+		fastrpc_buf_free(buf);
+	}
+
+	fastrpc_channel_ctx_put(fl->cctx);
+	mutex_destroy(&fl->mutex);
+	kfree(fl);
+}
+
+static void fastrpc_user_get(struct fastrpc_user *fl)
+{
+	kref_get(&fl->refcount);
+}
+
+static void fastrpc_user_put(struct fastrpc_user *fl)
+{
+	kref_put(&fl->refcount, fastrpc_user_free);
+}
+
 static void fastrpc_context_free(struct kref *ref)
 {
 	struct fastrpc_invoke_ctx *ctx;
 	struct fastrpc_channel_ctx *cctx;
+	struct fastrpc_user *fl;
 	unsigned long flags;
 	int i;
 
 	ctx = container_of(ref, struct fastrpc_invoke_ctx, refcount);
 	cctx = ctx->cctx;
+	fl = ctx->fl;
 
 	for (i = 0; i < ctx->nbufs; i++)
 		fastrpc_map_put(ctx->maps[i]);
@@ -521,6 +565,8 @@ static void fastrpc_context_free(struct kref *ref)
 	kfree(ctx->olaps);
 	kfree(ctx);
 
+	/* Release the reference taken in fastrpc_context_alloc() */
+	fastrpc_user_put(fl);
 	fastrpc_channel_ctx_put(cctx);
 }
 
@@ -628,6 +674,8 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
 
 	/* Released in fastrpc_context_put() */
 	fastrpc_channel_ctx_get(cctx);
+	/* Take a reference to user, released in fastrpc_context_free() */
+	fastrpc_user_get(user);
 
 	ctx->sc = sc;
 	ctx->retval = -1;
@@ -658,6 +706,7 @@ static struct fastrpc_invoke_ctx *fastrpc_context_alloc(
 	spin_lock(&user->lock);
 	list_del(&ctx->node);
 	spin_unlock(&user->lock);
+	fastrpc_user_put(user);
 	fastrpc_channel_ctx_put(cctx);
 	kfree(ctx->maps);
 	kfree(ctx->olaps);
@@ -1041,7 +1090,7 @@ static int fastrpc_get_args(u32 kernel, struct fastrpc_invoke_ctx *ctx)
 			pages[i].addr = ctx->maps[i]->dma_addr;
 
 			mmap_read_lock(current->mm);
-			vma = find_vma(current->mm, ctx->args[i].ptr);
+			vma = vma_lookup(current->mm, ctx->args[i].ptr);
 			if (vma)
 				pages[i].addr += (ctx->args[i].ptr & PAGE_MASK) -
 						 vma->vm_start;
@@ -1579,9 +1628,6 @@ static int fastrpc_device_release(struct inode *inode, struct file *file)
 {
 	struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data;
 	struct fastrpc_channel_ctx *cctx = fl->cctx;
-	struct fastrpc_invoke_ctx *ctx, *n;
-	struct fastrpc_map *map, *m;
-	struct fastrpc_buf *buf, *b;
 	unsigned long flags;
 
 	fastrpc_release_current_dsp_process(fl);
@@ -1590,28 +1636,10 @@ static int fastrpc_device_release(struct inode *inode, struct file *file)
 	list_del(&fl->user);
 	spin_unlock_irqrestore(&cctx->lock, flags);
 
-	if (fl->init_mem)
-		fastrpc_buf_free(fl->init_mem);
-
-	list_for_each_entry_safe(ctx, n, &fl->pending, node) {
-		list_del(&ctx->node);
-		fastrpc_context_put(ctx);
-	}
-
-	list_for_each_entry_safe(map, m, &fl->maps, node)
-		fastrpc_map_put(map);
-
-	list_for_each_entry_safe(buf, b, &fl->mmaps, node) {
-		list_del(&buf->node);
-		fastrpc_buf_free(buf);
-	}
-
 	fastrpc_session_free(cctx, fl->sctx);
-	fastrpc_channel_ctx_put(cctx);
-
-	mutex_destroy(&fl->mutex);
-	kfree(fl);
 	file->private_data = NULL;
+	/* Release the reference taken in fastrpc_device_open */
+	fastrpc_user_put(fl);
 
 	return 0;
 }
@@ -1655,6 +1683,7 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp)
 	spin_lock_irqsave(&cctx->lock, flags);
 	list_add_tail(&fl->user, &cctx->users);
 	spin_unlock_irqrestore(&cctx->lock, flags);
+	kref_init(&fl->refcount);
 
 	return 0;
 }
@@ -2197,19 +2226,22 @@ static int fastrpc_cb_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	int i, sessions = 0;
 	unsigned long flags;
-	int rc;
 	u32 dma_bits;
+	u32 sid = 0;
+	int rc;
 
 	cctx = dev_get_drvdata(dev->parent);
 	if (!cctx)
 		return -EINVAL;
 
 	of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions);
+	if (of_property_read_u32(dev->of_node, "reg", &sid))
+		dev_info(dev, "FastRPC Session ID not specified in DT\n");
 
 	spin_lock_irqsave(&cctx->lock, flags);
 	if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) {
-		dev_err(&pdev->dev, "too many sessions\n");
 		spin_unlock_irqrestore(&cctx->lock, flags);
+		dev_err(&pdev->dev, "too many sessions\n");
 		return -ENOSPC;
 	}
 	dma_bits = cctx->soc_data->dma_addr_bits_default;
@@ -2218,13 +2250,11 @@ static int fastrpc_cb_probe(struct platform_device *pdev)
 	sess->valid = true;
 	sess->dev = dev;
 	dev_set_drvdata(dev, sess);
+	sess->sid = sid;
 
 	if (cctx->domain_id == CDSP_DOMAIN_ID)
 		dma_bits = cctx->soc_data->dma_addr_bits_cdsp;
 
-	if (of_property_read_u32(dev->of_node, "reg", &sess->sid))
-		dev_info(dev, "FastRPC Session ID not specified in DT\n");
-
 	if (sessions > 0) {
 		struct fastrpc_session_ctx *dup_sess;
 
@@ -2431,7 +2461,6 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
 
 	kref_init(&data->refcount);
 
-	dev_set_drvdata(&rpdev->dev, data);
 	rdev->dma_mask = &data->dma_mask;
 	dma_set_mask_and_coherent(rdev, DMA_BIT_MASK(32));
 	INIT_LIST_HEAD(&data->users);
@@ -2440,6 +2469,7 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev)
 	idr_init(&data->ctx_idr);
 	data->domain_id = domain_id;
 	data->rpdev = rpdev;
+	dev_set_drvdata(&rpdev->dev, data);
 
 	err = of_platform_populate(rdev->of_node, NULL, NULL, rdev);
 	if (err)
@@ -2513,6 +2543,9 @@ static int fastrpc_rpmsg_callback(struct rpmsg_device *rpdev, void *data,
 	if (len < sizeof(*rsp))
 		return -EINVAL;
 
+	if (!cctx)
+		return -ENODEV;
+
 	ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4);
 
 	spin_lock_irqsave(&cctx->lock, flags);
diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
index dbd017cabbb92..3635741c3e7a7 100644
--- a/drivers/misc/pci_endpoint_test.c
+++ b/drivers/misc/pci_endpoint_test.c
@@ -1100,7 +1100,6 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test)
 
 	data = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_DATA);
 	addr = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_OFFSET);
-	bar = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_BAR);
 
 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
@@ -1108,6 +1107,11 @@ static int pci_endpoint_test_doorbell(struct pci_endpoint_test *test)
 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_STATUS, 0);
 
 	bar = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_DB_BAR);
+	if (bar < BAR_0 || bar >= PCI_STD_NUM_BARS) {
+		dev_err(dev, "BAR %d reported by endpoint out of range [0, %u]\n",
+			bar, PCI_STD_NUM_BARS - 1);
+		return -ERANGE;
+	}
 
 	writel(data, test->bar[bar] + addr);
 
diff --git a/drivers/misc/tifm_core.c b/drivers/misc/tifm_core.c
index da0827724a61c..aac8f0933e435 100644
--- a/drivers/misc/tifm_core.c
+++ b/drivers/misc/tifm_core.c
@@ -31,7 +31,7 @@ static const char *tifm_media_type_name(unsigned char type, unsigned char nt)
 	return card_type_name[nt][type - 1];
 }
 
-static int tifm_dev_match(struct tifm_dev *sock, struct tifm_device_id *id)
+static int tifm_dev_match(struct tifm_dev *sock, const struct tifm_device_id *id)
 {
 	if (sock->type == id->type)
 		return 1;
@@ -43,7 +43,7 @@ static int tifm_bus_match(struct device *dev, const struct device_driver *drv)
 	struct tifm_dev *sock = container_of(dev, struct tifm_dev, dev);
 	const struct tifm_driver *fm_drv = container_of_const(drv, struct tifm_driver,
 							      driver);
-	struct tifm_device_id *ids = fm_drv->id_table;
+	const struct tifm_device_id *ids = fm_drv->id_table;
 
 	if (ids) {
 		while (ids->type) {
diff --git a/drivers/mmc/core/mmc.c b/drivers/mmc/core/mmc.c
index 8846550a8892a..05444ecf3909f 100644
--- a/drivers/mmc/core/mmc.c
+++ b/drivers/mmc/core/mmc.c
@@ -1371,7 +1371,9 @@ static void mmc_select_driver_type(struct mmc_card *card)
 
 	card->drive_strength = drive_strength;
 
-	if (drv_type)
+	if (fixed_drv_type >= 0 && drive_strength)
+		mmc_set_driver_type(card->host, drive_strength);
+	else if (drv_type)
 		mmc_set_driver_type(card->host, drv_type);
 }
 
diff --git a/drivers/mmc/core/queue.c b/drivers/mmc/core/queue.c
index 39fcb662c43fc..c9028e4a7e560 100644
--- a/drivers/mmc/core/queue.c
+++ b/drivers/mmc/core/queue.c
@@ -214,8 +214,14 @@ static int mmc_mq_init_request(struct blk_mq_tag_set *set, struct request *req,
 	struct mmc_queue *mq = set->driver_data;
 	struct mmc_card *card = mq->card;
 	struct mmc_host *host = card->host;
+	u16 sg_len = mmc_get_max_segments(host);
 
-	mq_rq->sg = mmc_alloc_sg(mmc_get_max_segments(host), GFP_KERNEL);
+	if (!sg_len) {
+		dev_err(mmc_dev(host), "Wrong max_segs assigned\n");
+		return -EINVAL;
+	}
+
+	mq_rq->sg = mmc_alloc_sg(sg_len, GFP_KERNEL);
 	if (!mq_rq->sg)
 		return -ENOMEM;
 
diff --git a/drivers/mmc/host/cavium-thunderx.c b/drivers/mmc/host/cavium-thunderx.c
index 1373deb3f531e..84ff6d82ae3c0 100644
--- a/drivers/mmc/host/cavium-thunderx.c
+++ b/drivers/mmc/host/cavium-thunderx.c
@@ -188,6 +188,7 @@ static const struct pci_device_id thunder_mmc_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa010) },
 	{ 0, }  /* end of table */
 };
+MODULE_DEVICE_TABLE(pci, thunder_mmc_id_table);
 
 static struct pci_driver thunder_mmc_driver = {
 	.name = KBUILD_MODNAME,
@@ -201,4 +202,3 @@ module_pci_driver(thunder_mmc_driver);
 MODULE_AUTHOR("Cavium Inc.");
 MODULE_DESCRIPTION("Cavium ThunderX eMMC Driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(pci, thunder_mmc_id_table);
diff --git a/drivers/mmc/host/davinci_mmc.c b/drivers/mmc/host/davinci_mmc.c
index 42b0118a45a8e..42ad87aa48f59 100644
--- a/drivers/mmc/host/davinci_mmc.c
+++ b/drivers/mmc/host/davinci_mmc.c
@@ -928,7 +928,7 @@ static irqreturn_t mmc_davinci_irq(int irq, void *dev_id)
 		}
 	}
 
-	if (qstatus & MMCST0_TOUTRD) {
+	if (data && (qstatus & MMCST0_TOUTRD)) {
 		/* Read data timeout */
 		data->error = -ETIMEDOUT;
 		end_transfer = 1;
@@ -940,7 +940,7 @@ static irqreturn_t mmc_davinci_irq(int irq, void *dev_id)
 		davinci_abort_data(host, data);
 	}
 
-	if (qstatus & (MMCST0_CRCWR | MMCST0_CRCRD)) {
+	if (data && (qstatus & (MMCST0_CRCWR | MMCST0_CRCRD))) {
 		/* Data CRC error */
 		data->error = -EILSEQ;
 		end_transfer = 1;
diff --git a/drivers/mmc/host/dw_mmc-exynos.c b/drivers/mmc/host/dw_mmc-exynos.c
index 261344d3a8cfe..4b76b997ddc15 100644
--- a/drivers/mmc/host/dw_mmc-exynos.c
+++ b/drivers/mmc/host/dw_mmc-exynos.c
@@ -141,6 +141,7 @@ static int dw_mci_exynos_priv_init(struct dw_mci *host)
 		priv->ctrl_type == DW_MCI_TYPE_EXYNOS7870_SMU) {
 		/* Quirk needed for certain Exynos SoCs */
 		host->quirks |= DW_MMC_QUIRK_FIFO64_32;
+		host->dma_threshold = 512;
 	}
 
 	if (priv->ctrl_type == DW_MCI_TYPE_ARTPEC8) {
diff --git a/drivers/mmc/host/dw_mmc.c b/drivers/mmc/host/dw_mmc.c
index 20193ee7b73eb..3b4157f34d11f 100644
--- a/drivers/mmc/host/dw_mmc.c
+++ b/drivers/mmc/host/dw_mmc.c
@@ -40,7 +40,6 @@
 				 SDMMC_INT_RESP_ERR | SDMMC_INT_HLE)
 #define DW_MCI_ERROR_FLAGS	(DW_MCI_DATA_ERROR_FLAGS | \
 				 DW_MCI_CMD_ERROR_FLAGS)
-#define DW_MCI_DMA_THRESHOLD	16
 
 #define DW_MCI_FREQ_MAX	200000000	/* unit: HZ */
 #define DW_MCI_FREQ_MIN	100000		/* unit: HZ */
@@ -821,7 +820,7 @@ static int dw_mci_pre_dma_transfer(struct dw_mci *host,
 	 * non-word-aligned buffers or lengths. Also, we don't bother
 	 * with all the DMA setup overhead for short transfers.
 	 */
-	if (data->blocks * data->blksz < DW_MCI_DMA_THRESHOLD)
+	if (data->blocks * data->blksz < host->dma_threshold)
 		return -EINVAL;
 
 	if (data->blksz & 3)
@@ -3185,6 +3184,7 @@ struct dw_mci *dw_mci_alloc_host(struct device *dev)
 	host = mmc_priv(mmc);
 	host->mmc = mmc;
 	host->dev = dev;
+	host->dma_threshold = 16;
 
 	return host;
 }
diff --git a/drivers/mmc/host/dw_mmc.h b/drivers/mmc/host/dw_mmc.h
index 42e58be74ce09..2ce8585e2c1e5 100644
--- a/drivers/mmc/host/dw_mmc.h
+++ b/drivers/mmc/host/dw_mmc.h
@@ -78,8 +78,8 @@ struct dw_mci_dma_slave {
  * @sg_cpu: Virtual address of DMA buffer.
  * @dma_ops: Pointer to DMA callbacks.
  * @cmd_status: Snapshot of SR taken upon completion of the current
- * @ring_size: Buffer size for idma descriptors.
  *	command. Only valid when EVENT_CMD_COMPLETE is pending.
+ * @ring_size: Buffer size for idma descriptors.
  * @dms: structure of slave-dma private data.
  * @phy_regs: physical address of controller's register map
  * @data_status: Snapshot of SR taken upon completion of the current
@@ -107,6 +107,7 @@ struct dw_mci_dma_slave {
  * @ciu_clk: Pointer to card interface unit clock instance.
  * @fifo_depth: depth of FIFO.
  * @data_addr_override: override fifo reg offset with this value.
+ * @dma_threshold: data threshold value in bytes to carry out a DMA transfer.
  * @wm_aligned: force fifo watermark equal with data length in PIO mode.
  *	Set as true if alignment is needed.
  * @data_shift: log2 of FIFO item size.
@@ -163,6 +164,7 @@ struct dw_mci {
 	void __iomem		*regs;
 	void __iomem		*fifo_reg;
 	u32			data_addr_override;
+	u32			dma_threshold;
 	bool			wm_aligned;
 
 	struct scatterlist	*sg;
diff --git a/drivers/mmc/host/renesas_sdhi_internal_dmac.c b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
index f6ebb7bc7ede1..b716a518f2650 100644
--- a/drivers/mmc/host/renesas_sdhi_internal_dmac.c
+++ b/drivers/mmc/host/renesas_sdhi_internal_dmac.c
@@ -285,6 +285,7 @@ static const struct of_device_id renesas_sdhi_internal_dmac_of_match[] = {
 	{ .compatible = "renesas,sdhi-r8a77970", .data = &of_r8a77970_compatible, },
 	{ .compatible = "renesas,sdhi-r8a77990", .data = &of_r8a77990_compatible, },
 	{ .compatible = "renesas,sdhi-r8a77995", .data = &of_rcar_gen3_nohs400_compatible, },
+	{ .compatible = "renesas,sdhi-r8a779md", .data = &of_rcar_gen3_nohs400_compatible, },
 	{ .compatible = "renesas,sdhi-r9a09g011", .data = &of_rzg2l_compatible, },
 	{ .compatible = "renesas,sdhi-r9a09g057", .data = &of_rzg2l_compatible, },
 	{ .compatible = "renesas,rzg2l-sdhi", .data = &of_rzg2l_compatible, },
diff --git a/drivers/mmc/host/sdhci-msm.c b/drivers/mmc/host/sdhci-msm.c
index 633462c0be5f4..0882ce74e0c9b 100644
--- a/drivers/mmc/host/sdhci-msm.c
+++ b/drivers/mmc/host/sdhci-msm.c
@@ -1918,14 +1918,14 @@ static int sdhci_msm_ice_init(struct sdhci_msm_host *msm_host,
 		return 0;
 
 	ice = devm_of_qcom_ice_get(dev);
-	if (ice == ERR_PTR(-EOPNOTSUPP)) {
+	if (IS_ERR(ice)) {
+		if (ice != ERR_PTR(-EOPNOTSUPP))
+			return PTR_ERR(ice);
+
 		dev_warn(dev, "Disabling inline encryption support\n");
-		ice = NULL;
+		return 0;
 	}
 
-	if (IS_ERR_OR_NULL(ice))
-		return PTR_ERR_OR_ZERO(ice);
-
 	msm_host->ice = ice;
 
 	/* Initialize the blk_crypto_profile */
diff --git a/drivers/mmc/host/sdhci-of-k1.c b/drivers/mmc/host/sdhci-of-k1.c
index 455656f9842df..37b0911e7cf20 100644
--- a/drivers/mmc/host/sdhci-of-k1.c
+++ b/drivers/mmc/host/sdhci-of-k1.c
@@ -16,11 +16,19 @@
 #include <linux/of.h>
 #include <linux/of_device.h>
 #include <linux/reset.h>
+#include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
 
 #include "sdhci.h"
 #include "sdhci-pltfm.h"
 
+#define SPACEMIT_SDHC_OP_EXT_REG	0x108
+#define  SDHC_OVRRD_CLK_OEN		BIT(11)
+#define  SDHC_FORCE_CLK_ON		BIT(12)
+
+#define SPACEMIT_SDHC_LEGACY_CTRL_REG	0x10C
+#define  SDHC_GEN_PAD_CLK_ON		BIT(6)
+
 #define SPACEMIT_SDHC_MMC_CTRL_REG	0x114
 #define  SDHC_MISC_INT_EN		BIT(1)
 #define  SDHC_MISC_INT			BIT(2)
@@ -61,9 +69,34 @@
 #define  SDHC_PHY_DRIVE_SEL		GENMASK(2, 0)
 #define  SDHC_RX_BIAS_CTRL		BIT(5)
 
+#define SPACEMIT_SDHC_RX_CFG_REG        0x118
+#define  SDHC_RX_SDCLK_SEL0_MASK        GENMASK(1, 0)
+#define  SDHC_RX_SDCLK_SEL1_MASK        GENMASK(3, 2)
+#define  SDHC_RX_SDCLK_SEL1             FIELD_PREP(SDHC_RX_SDCLK_SEL1_MASK, 1)
+
+#define SPACEMIT_SDHC_DLINE_CTRL_REG    0x130
+#define  SDHC_DLINE_PU                  BIT(0)
+#define  SDHC_RX_DLINE_CODE_MASK        GENMASK(23, 16)
+#define  SDHC_TX_DLINE_CODE_MASK        GENMASK(31, 24)
+
+#define SPACEMIT_SDHC_DLINE_CFG_REG     0x134
+#define  SDHC_RX_DLINE_REG_MASK         GENMASK(7, 0)
+#define  SDHC_RX_DLINE_GAIN             BIT(8)
+#define  SDHC_TX_DLINE_REG_MASK         GENMASK(23, 16)
+
+#define SPACEMIT_RX_DLINE_REG		9
+#define SPACEMIT_RX_TUNE_DELAY_MIN	0x0
+#define SPACEMIT_RX_TUNE_DELAY_MAX	0xFF
+
+#define SPACEMIT_TX_TUNING_DLINE_REG	0x00
+#define SPACEMIT_TX_TUNING_DELAYCODE	127
+
 struct spacemit_sdhci_host {
 	struct clk *clk_core;
 	struct clk *clk_io;
+	struct pinctrl *pinctrl;
+	struct pinctrl_state *pinctrl_default;
+	struct pinctrl_state *pinctrl_uhs;
 };
 
 /* All helper functions will update clr/set while preserve rest bits */
@@ -85,6 +118,50 @@ static inline void spacemit_sdhci_clrsetbits(struct sdhci_host *host, u32 clr, u
 	sdhci_writel(host, val, reg);
 }
 
+static void spacemit_sdhci_set_rx_delay(struct sdhci_host *host, u8 delay)
+{
+	spacemit_sdhci_clrsetbits(host, SDHC_RX_DLINE_CODE_MASK,
+				  FIELD_PREP(SDHC_RX_DLINE_CODE_MASK, delay),
+				  SPACEMIT_SDHC_DLINE_CTRL_REG);
+}
+
+static void spacemit_sdhci_set_tx_delay(struct sdhci_host *host, u8 delay)
+{
+	spacemit_sdhci_clrsetbits(host, SDHC_TX_DLINE_CODE_MASK,
+				  FIELD_PREP(SDHC_TX_DLINE_CODE_MASK, delay),
+				  SPACEMIT_SDHC_DLINE_CTRL_REG);
+}
+
+static void spacemit_sdhci_set_tx_dline_reg(struct sdhci_host *host, u8 dline_reg)
+{
+	spacemit_sdhci_clrsetbits(host, SDHC_TX_DLINE_REG_MASK,
+				  FIELD_PREP(SDHC_TX_DLINE_REG_MASK, dline_reg),
+				  SPACEMIT_SDHC_DLINE_CFG_REG);
+}
+
+static void spacemit_sdhci_tx_tuning_prepare(struct sdhci_host *host)
+{
+	spacemit_sdhci_setbits(host, SDHC_TX_MUX_SEL, SPACEMIT_SDHC_TX_CFG_REG);
+	spacemit_sdhci_setbits(host, SDHC_DLINE_PU, SPACEMIT_SDHC_DLINE_CTRL_REG);
+	udelay(5);
+}
+
+static void spacemit_sdhci_prepare_tuning(struct sdhci_host *host)
+{
+	spacemit_sdhci_clrsetbits(host, SDHC_RX_DLINE_REG_MASK,
+				  FIELD_PREP(SDHC_RX_DLINE_REG_MASK, SPACEMIT_RX_DLINE_REG),
+				  SPACEMIT_SDHC_DLINE_CFG_REG);
+
+	spacemit_sdhci_setbits(host, SDHC_DLINE_PU, SPACEMIT_SDHC_DLINE_CTRL_REG);
+	udelay(5);
+
+	spacemit_sdhci_clrsetbits(host, SDHC_RX_SDCLK_SEL1_MASK, SDHC_RX_SDCLK_SEL1,
+				  SPACEMIT_SDHC_RX_CFG_REG);
+
+	if (host->mmc->ios.timing == MMC_TIMING_MMC_HS200)
+		spacemit_sdhci_setbits(host, SDHC_HS200_USE_RFIFO, SPACEMIT_SDHC_PHY_FUNC_REG);
+}
+
 static void spacemit_sdhci_reset(struct sdhci_host *host, u8 mask)
 {
 	sdhci_reset(host, mask);
@@ -101,6 +178,12 @@ static void spacemit_sdhci_reset(struct sdhci_host *host, u8 mask)
 
 	if (!(host->mmc->caps2 & MMC_CAP2_NO_MMC))
 		spacemit_sdhci_setbits(host, SDHC_MMC_CARD_MODE, SPACEMIT_SDHC_MMC_CTRL_REG);
+
+	spacemit_sdhci_setbits(host, SDHC_GEN_PAD_CLK_ON, SPACEMIT_SDHC_LEGACY_CTRL_REG);
+
+	if (host->mmc->caps2 & MMC_CAP2_NO_MMC)
+		spacemit_sdhci_setbits(host, SDHC_OVRRD_CLK_OEN | SDHC_FORCE_CLK_ON,
+				       SPACEMIT_SDHC_OP_EXT_REG);
 }
 
 static void spacemit_sdhci_set_uhs_signaling(struct sdhci_host *host, unsigned int timing)
@@ -174,6 +257,111 @@ static unsigned int spacemit_sdhci_clk_get_max_clock(struct sdhci_host *host)
 	return clk_get_rate(pltfm_host->clk);
 }
 
+static int spacemit_sdhci_execute_tuning(struct sdhci_host *host, u32 opcode)
+{
+	int current_len = 0, current_start = 0;
+	int max_pass_len = 0, max_pass_start = 0;
+	struct mmc_host *mmc = host->mmc;
+	struct mmc_ios ios = mmc->ios;
+	u8 final_delay;
+	int ret = 0;
+	int i;
+
+	/*
+	 * Tuning is required for SDR50/SDR104, HS200/HS400 cards and
+	 * if clock frequency is greater than 100MHz in these modes.
+	 */
+	if (host->clock < 100 * 1000 * 1000 ||
+	    !(ios.timing == MMC_TIMING_MMC_HS200 ||
+	      ios.timing == MMC_TIMING_UHS_SDR50 ||
+	      ios.timing == MMC_TIMING_UHS_SDR104))
+		return 0;
+
+	if (mmc->caps2 & MMC_CAP2_NO_MMC) {
+		spacemit_sdhci_set_tx_dline_reg(host, SPACEMIT_TX_TUNING_DLINE_REG);
+		spacemit_sdhci_set_tx_delay(host, SPACEMIT_TX_TUNING_DELAYCODE);
+		spacemit_sdhci_tx_tuning_prepare(host);
+
+		dev_dbg(mmc_dev(host->mmc), "TX tuning: dline_reg=%d, delaycode=%d\n",
+			SPACEMIT_TX_TUNING_DLINE_REG, SPACEMIT_TX_TUNING_DELAYCODE);
+	}
+
+	spacemit_sdhci_prepare_tuning(host);
+
+	for (i = SPACEMIT_RX_TUNE_DELAY_MIN; i <= SPACEMIT_RX_TUNE_DELAY_MAX; i++) {
+		spacemit_sdhci_set_rx_delay(host, i);
+		ret = mmc_send_tuning(host->mmc, opcode, NULL);
+
+		dev_dbg(mmc_dev(host->mmc), "RX delay %d: %s\n",
+			i, ret == 0 ? "pass" : "fail");
+
+		if (ret == 0) {
+			/* Test passed - extend current window */
+			if (current_len == 0)
+				current_start = i;
+			current_len++;
+		} else {
+			/* Test failed - check if current window is best so far */
+			if (current_len > max_pass_len) {
+				max_pass_len = current_len;
+				max_pass_start = current_start;
+			}
+			current_len = 0;
+		}
+	}
+
+	if (current_len > max_pass_len) {
+		max_pass_len = current_len;
+		max_pass_start = current_start;
+	}
+
+	if (max_pass_len < 3) {
+		dev_err(mmc_dev(host->mmc), "Tuning failed: no stable window found\n");
+		return -EIO;
+	}
+
+	final_delay = max_pass_start + max_pass_len / 2;
+	spacemit_sdhci_set_rx_delay(host, final_delay);
+	ret = mmc_send_tuning(host->mmc, opcode, NULL);
+	if (ret) {
+		u8 retry_delays[] = {
+			max_pass_start + max_pass_len / 4,
+			max_pass_start + (3 * max_pass_len) / 4,
+			max_pass_start,
+			max_pass_start + max_pass_len - 1
+		};
+		int retry_count = ARRAY_SIZE(retry_delays);
+
+		dev_warn(mmc_dev(mmc), "Primary delay %d failed, trying alternatives\n",
+			 final_delay);
+
+		for (i = 0; i < retry_count; i++) {
+			if (retry_delays[i] >= SPACEMIT_RX_TUNE_DELAY_MIN &&
+			    retry_delays[i] <= SPACEMIT_RX_TUNE_DELAY_MAX) {
+				spacemit_sdhci_set_rx_delay(host, retry_delays[i]);
+				ret = mmc_send_tuning(host->mmc, opcode, NULL);
+				if (!ret) {
+					final_delay = retry_delays[i];
+					dev_info(mmc_dev(mmc), "Retry successful with delay %d\n",
+						 final_delay);
+					break;
+				}
+			}
+		}
+
+		if (ret) {
+			dev_err(mmc_dev(mmc), "All retry attempts failed\n");
+			return -EIO;
+		}
+	}
+
+	dev_dbg(mmc_dev(host->mmc),
+		"Tuning successful: window %d-%d, using delay %d\n",
+		max_pass_start, max_pass_start + max_pass_len - 1, final_delay);
+
+	return 0;
+}
+
 static int spacemit_sdhci_pre_select_hs400(struct mmc_host *mmc)
 {
 	struct sdhci_host *host = mmc_priv(mmc);
@@ -206,6 +394,46 @@ static void spacemit_sdhci_pre_hs400_to_hs200(struct mmc_host *mmc)
 			       SPACEMIT_SDHC_PHY_CTRL_REG);
 }
 
+static int spacemit_sdhci_start_signal_voltage_switch(struct mmc_host *mmc,
+						      struct mmc_ios *ios)
+{
+	struct sdhci_host *host = mmc_priv(mmc);
+	struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host);
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(pltfm_host);
+	struct pinctrl_state *state;
+	int ret;
+
+	ret = sdhci_start_signal_voltage_switch(mmc, ios);
+	if (ret)
+		return ret;
+
+	if (!sdhst->pinctrl)
+		return 0;
+
+	/* Select appropriate pinctrl state based on signal voltage */
+	switch (ios->signal_voltage) {
+	case MMC_SIGNAL_VOLTAGE_330:
+		state = sdhst->pinctrl_default;
+		break;
+	case MMC_SIGNAL_VOLTAGE_180:
+		state = sdhst->pinctrl_uhs;
+		break;
+	default:
+		dev_warn(mmc_dev(mmc), "unsupported voltage %d\n", ios->signal_voltage);
+		return 0;
+	}
+
+	ret = pinctrl_select_state(sdhst->pinctrl, state);
+	if (ret) {
+		dev_warn(mmc_dev(mmc), "failed to select pinctrl state: %d\n", ret);
+		return 0;
+	}
+	dev_dbg(mmc_dev(mmc), "switched to %s pinctrl state\n",
+		ios->signal_voltage == MMC_SIGNAL_VOLTAGE_180 ? "UHS" : "default");
+
+	return 0;
+}
+
 static inline int spacemit_sdhci_get_clocks(struct device *dev,
 					    struct sdhci_pltfm_host *pltfm_host)
 {
@@ -239,12 +467,37 @@ static inline int spacemit_sdhci_get_resets(struct device *dev)
 	return 0;
 }
 
+static inline void spacemit_sdhci_get_pins(struct device *dev,
+					   struct sdhci_pltfm_host *pltfm_host)
+{
+	struct spacemit_sdhci_host *sdhst = sdhci_pltfm_priv(pltfm_host);
+
+	sdhst->pinctrl = devm_pinctrl_get(dev);
+	if (IS_ERR(sdhst->pinctrl)) {
+		sdhst->pinctrl = NULL;
+		dev_dbg(dev, "pinctrl not available, voltage switching will work without it\n");
+		return;
+	}
+
+	sdhst->pinctrl_default = pinctrl_lookup_state(sdhst->pinctrl, "default");
+	if (IS_ERR(sdhst->pinctrl_default))
+		sdhst->pinctrl_default = NULL;
+
+	sdhst->pinctrl_uhs = pinctrl_lookup_state(sdhst->pinctrl, "uhs");
+	if (IS_ERR(sdhst->pinctrl_uhs))
+		sdhst->pinctrl_uhs = NULL;
+
+	dev_dbg(dev, "pinctrl setup: default=%p, uhs=%p\n",
+		sdhst->pinctrl_default, sdhst->pinctrl_uhs);
+}
+
 static const struct sdhci_ops spacemit_sdhci_ops = {
 	.get_max_clock		= spacemit_sdhci_clk_get_max_clock,
 	.reset			= spacemit_sdhci_reset,
 	.set_bus_width		= sdhci_set_bus_width,
 	.set_clock		= spacemit_sdhci_set_clock,
 	.set_uhs_signaling	= spacemit_sdhci_set_uhs_signaling,
+	.platform_execute_tuning = spacemit_sdhci_execute_tuning,
 };
 
 static const struct sdhci_pltfm_data spacemit_sdhci_k1_pdata = {
@@ -311,6 +564,10 @@ static int spacemit_sdhci_probe(struct platform_device *pdev)
 
 	host->mmc->caps |= MMC_CAP_NEED_RSP_BUSY;
 
+	spacemit_sdhci_get_pins(dev, pltfm_host);
+
+	host->mmc_host_ops.start_signal_voltage_switch = spacemit_sdhci_start_signal_voltage_switch;
+
 	ret = spacemit_sdhci_get_clocks(dev, pltfm_host);
 	if (ret)
 		goto err_pltfm;
diff --git a/drivers/mmc/host/tifm_sd.c b/drivers/mmc/host/tifm_sd.c
index c1f7d5b37911d..28ab2526dab15 100644
--- a/drivers/mmc/host/tifm_sd.c
+++ b/drivers/mmc/host/tifm_sd.c
@@ -1041,9 +1041,10 @@ static int tifm_sd_resume(struct tifm_dev *sock)
 
 #endif /* CONFIG_PM */
 
-static struct tifm_device_id tifm_sd_id_tbl[] = {
+static const struct tifm_device_id tifm_sd_id_tbl[] = {
 	{ TIFM_TYPE_SD }, { }
 };
+MODULE_DEVICE_TABLE(tifm, tifm_sd_id_tbl);
 
 static struct tifm_driver tifm_sd_driver = {
 	.driver = {
@@ -1070,7 +1071,6 @@ static void __exit tifm_sd_exit(void)
 MODULE_AUTHOR("Alex Dubov");
 MODULE_DESCRIPTION("TI FlashMedia SD driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(tifm, tifm_sd_id_tbl);
 MODULE_VERSION(DRIVER_VERSION);
 
 module_init(tifm_sd_init);
diff --git a/drivers/mmc/host/via-sdmmc.c b/drivers/mmc/host/via-sdmmc.c
index c628b3bbfd7aa..8c049f8355cdf 100644
--- a/drivers/mmc/host/via-sdmmc.c
+++ b/drivers/mmc/host/via-sdmmc.c
@@ -323,9 +323,8 @@ struct via_crdr_mmc_host {
 #define VIA_CMD_TIMEOUT_MS		1000
 
 static const struct pci_device_id via_ids[] = {
-	{PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_9530,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0,},
-	{0,}
+	{ PCI_VDEVICE(VIA, PCI_DEVICE_ID_VIA_9530) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, via_ids);
diff --git a/drivers/mmc/host/wmt-sdmmc.c b/drivers/mmc/host/wmt-sdmmc.c
index 1b1d691e19fcc..489daee4f4fc9 100644
--- a/drivers/mmc/host/wmt-sdmmc.c
+++ b/drivers/mmc/host/wmt-sdmmc.c
@@ -744,6 +744,7 @@ static const struct of_device_id wmt_mci_dt_ids[] = {
 	{ .compatible = "wm,wm8505-sdhc", .data = &wm8505_caps },
 	{ /* Sentinel */ },
 };
+MODULE_DEVICE_TABLE(of, wmt_mci_dt_ids);
 
 static int wmt_mci_probe(struct platform_device *pdev)
 {
@@ -980,4 +981,3 @@ module_platform_driver(wmt_mci_driver);
 MODULE_DESCRIPTION("Wondermedia MMC/SD Driver");
 MODULE_AUTHOR("Tony Prisk");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, wmt_mci_dt_ids);
diff --git a/drivers/mtd/mtdoops.c b/drivers/mtd/mtdoops.c
index b88083751a0c7..39df7ce8f55f7 100644
--- a/drivers/mtd/mtdoops.c
+++ b/drivers/mtd/mtdoops.c
@@ -403,8 +403,7 @@ static struct mtd_notifier mtdoops_notifier = {
 static int __init mtdoops_init(void)
 {
 	struct mtdoops_context *cxt = &oops_cxt;
-	int mtd_index;
-	char *endp;
+	unsigned int mtd_index;
 
 	if (strlen(mtddev) == 0) {
 		pr_err("mtd device (mtddev=name/number) must be supplied\n");
@@ -421,9 +420,9 @@ static int __init mtdoops_init(void)
 
 	/* Setup the MTD device to use */
 	cxt->mtd_index = -1;
-	mtd_index = simple_strtoul(mtddev, &endp, 0);
-	if (*endp == '\0')
+	if (kstrtouint(mtddev, 0, &mtd_index) == 0) {
 		cxt->mtd_index = mtd_index;
+	}
 
 	cxt->oops_buf = vmalloc(record_size);
 	if (!cxt->oops_buf)
diff --git a/drivers/mtd/mtdsuper.c b/drivers/mtd/mtdsuper.c
index b7e3763c47f0c..c709ff7aa6b57 100644
--- a/drivers/mtd/mtdsuper.c
+++ b/drivers/mtd/mtdsuper.c
@@ -132,12 +132,12 @@ int get_tree_mtd(struct fs_context *fc,
 
 		} else if (isdigit(fc->source[3])) {
 			/* mount by MTD device number name */
-			char *endptr;
+			unsigned int mtdnr_val;
 
-			mtdnr = simple_strtoul(fc->source + 3, &endptr, 0);
-			if (!*endptr) {
+			if (kstrtouint(fc->source + 3, 0, &mtdnr_val) == 0) {
+				mtdnr = mtdnr_val;
 				/* It was a valid number */
-				pr_debug("MTDSB: mtd%%d, mtdnr %d\n", mtdnr);
+				pr_debug("MTDSB: mtd%%d, mtdnr %u\n", mtdnr_val);
 				return mtd_get_sb_by_nr(fc, mtdnr, fill_super);
 			}
 		}
diff --git a/drivers/mtd/nand/raw/Kconfig b/drivers/mtd/nand/raw/Kconfig
index 7408f34f0c68a..d488213b631f3 100644
--- a/drivers/mtd/nand/raw/Kconfig
+++ b/drivers/mtd/nand/raw/Kconfig
@@ -304,7 +304,7 @@ config MTD_NAND_HISI504
 	  Enables support for NAND controller on Hisilicon SoC Hip04.
 
 config MTD_NAND_QCOM
-	tristate "QCOM NAND controller"
+	tristate "Qualcomm NAND controller"
 	depends on ARCH_QCOM || COMPILE_TEST
 	depends on HAS_IOMEM
 	help
diff --git a/drivers/mtd/nand/spi/core.c b/drivers/mtd/nand/spi/core.c
index 0b076790bd9df..f1084d5e04b97 100644
--- a/drivers/mtd/nand/spi/core.c
+++ b/drivers/mtd/nand/spi/core.c
@@ -501,10 +501,18 @@ static int spinand_read_from_cache_op(struct spinand_device *spinand,
 		}
 	}
 
-	if (req->mode == MTD_OPS_RAW)
-		rdesc = spinand->dirmaps[req->pos.plane].rdesc;
+	rdesc = spinand->dirmaps[req->pos.plane].rdesc;
+
+	if (spinand->op_templates->cont_read_cache && req->continuous)
+		rdesc->info.op_tmpl = &rdesc->info.secondary_op_tmpl;
+	else
+		rdesc->info.op_tmpl = &rdesc->info.primary_op_tmpl;
+
+	if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED &&
+	    req->mode != MTD_OPS_RAW)
+		rdesc->info.op_tmpl->data.ecc = true;
 	else
-		rdesc = spinand->dirmaps[req->pos.plane].rdesc_ecc;
+		rdesc->info.op_tmpl->data.ecc = false;
 
 	if (spinand->flags & SPINAND_HAS_READ_PLANE_SELECT_BIT)
 		column |= req->pos.plane << fls(nanddev_page_size(nand));
@@ -593,10 +601,13 @@ static int spinand_write_to_cache_op(struct spinand_device *spinand,
 			       req->ooblen);
 	}
 
-	if (req->mode == MTD_OPS_RAW)
-		wdesc = spinand->dirmaps[req->pos.plane].wdesc;
+	wdesc = spinand->dirmaps[req->pos.plane].wdesc;
+
+	if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED &&
+	    req->mode != MTD_OPS_RAW)
+		wdesc->info.op_tmpl->data.ecc = true;
 	else
-		wdesc = spinand->dirmaps[req->pos.plane].wdesc_ecc;
+		wdesc->info.op_tmpl->data.ecc = false;
 
 	if (spinand->flags & SPINAND_HAS_PROG_PLANE_SELECT_BIT)
 		column |= req->pos.plane << fls(nanddev_page_size(nand));
@@ -892,6 +903,12 @@ static int spinand_mtd_continuous_page_read(struct mtd_info *mtd, loff_t from,
 	 * Each data read must be a multiple of 4-bytes and full pages should be read;
 	 * otherwise, the data output might get out of sequence from one read command
 	 * to another.
+	 *
+	 * Continuous reads never cross LUN boundaries. Some devices don't
+	 * support crossing planes boundaries. Some devices don't even support
+	 * crossing blocks boundaries. The common case being to read through UBI,
+	 * we will very rarely read two consequent blocks or more, so let's only enable
+	 * continuous reads when reading within the same erase block.
 	 */
 	nanddev_io_for_each_block(nand, NAND_PAGE_READ, from, ops, &iter) {
 		ret = spinand_select_target(spinand, iter.req.pos.target);
@@ -948,13 +965,10 @@ static void spinand_cont_read_init(struct spinand_device *spinand)
 {
 	struct nand_device *nand = spinand_to_nand(spinand);
 	enum nand_ecc_engine_type engine_type = nand->ecc.ctx.conf.engine_type;
+	struct spi_controller *ctlr = spinand->spimem->spi->controller;
 
 	/* OOBs cannot be retrieved so external/on-host ECC engine won't work */
-	if (spinand->set_cont_read &&
-	    (engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE ||
-	     engine_type == NAND_ECC_ENGINE_TYPE_NONE)) {
-		spinand->cont_read_possible = true;
-
+	if (spinand->set_cont_read) {
 		/*
 		 * Ensure continuous read is disabled on probe.
 		 * Some devices retain this state across soft reset,
@@ -962,6 +976,11 @@ static void spinand_cont_read_init(struct spinand_device *spinand)
 		 * in false positive returns from spinand_isbad().
 		 */
 		spinand_cont_read_enable(spinand, false);
+
+		if ((engine_type == NAND_ECC_ENGINE_TYPE_ON_DIE ||
+		     engine_type == NAND_ECC_ENGINE_TYPE_NONE) &&
+		    !spi_mem_controller_is_capable(ctlr, no_cs_assertion))
+			spinand->cont_read_possible = true;
 	}
 }
 
@@ -982,19 +1001,6 @@ static bool spinand_use_cont_read(struct mtd_info *mtd, loff_t from,
 	nanddev_offs_to_pos(nand, from, &start_pos);
 	nanddev_offs_to_pos(nand, from + ops->len - 1, &end_pos);
 
-	/*
-	 * Continuous reads never cross LUN boundaries. Some devices don't
-	 * support crossing planes boundaries. Some devices don't even support
-	 * crossing blocks boundaries. The common case being to read through UBI,
-	 * we will very rarely read two consequent blocks or more, so it is safer
-	 * and easier (can be improved) to only enable continuous reads when
-	 * reading within the same erase block.
-	 */
-	if (start_pos.target != end_pos.target ||
-	    start_pos.plane != end_pos.plane ||
-	    start_pos.eraseblock != end_pos.eraseblock)
-		return false;
-
 	return start_pos.page < end_pos.page;
 }
 
@@ -1236,6 +1242,7 @@ static struct spi_mem_dirmap_desc *spinand_create_rdesc(
 		 * its spi controller, use regular reading
 		 */
 		spinand->cont_read_possible = false;
+		memset(&info->secondary_op_tmpl, 0, sizeof(info->secondary_op_tmpl));
 
 		info->length = nanddev_page_size(nand) +
 			       nanddev_per_page_oobsize(nand);
@@ -1252,12 +1259,31 @@ static int spinand_create_dirmap(struct spinand_device *spinand,
 	struct nand_device *nand = spinand_to_nand(spinand);
 	struct spi_mem_dirmap_info info = { 0 };
 	struct spi_mem_dirmap_desc *desc;
+	bool enable_ecc = false, secondary_op = false;
+
+	if (nand->ecc.engine->integration == NAND_ECC_ENGINE_INTEGRATION_PIPELINED)
+		enable_ecc = true;
+
+	if (spinand->cont_read_possible && spinand->op_templates->cont_read_cache)
+		secondary_op = true;
+
+	/*
+	 * Continuous read implies that only the main data is retrieved, backed
+	 * by an on-die ECC engine. It is not possible to use a pipelind ECC
+	 * engine with continuous read.
+	 */
+	if (enable_ecc && secondary_op) {
+		secondary_op = false;
+		spinand->cont_read_possible = false;
+	}
 
 	/* The plane number is passed in MSB just above the column address */
 	info.offset = plane << fls(nand->memorg.pagesize);
 
+	/* Write descriptor */
 	info.length = nanddev_page_size(nand) + nanddev_per_page_oobsize(nand);
-	info.op_tmpl = *spinand->op_templates->update_cache;
+	info.primary_op_tmpl = *spinand->op_templates->update_cache;
+	info.primary_op_tmpl.data.ecc = enable_ecc;
 	desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev,
 					  spinand->spimem, &info);
 	if (IS_ERR(desc))
@@ -1265,37 +1291,18 @@ static int spinand_create_dirmap(struct spinand_device *spinand,
 
 	spinand->dirmaps[plane].wdesc = desc;
 
-	info.op_tmpl = *spinand->op_templates->read_cache;
-	desc = spinand_create_rdesc(spinand, &info);
-	if (IS_ERR(desc))
-		return PTR_ERR(desc);
-
-	spinand->dirmaps[plane].rdesc = desc;
-
-	if (nand->ecc.engine->integration != NAND_ECC_ENGINE_INTEGRATION_PIPELINED) {
-		spinand->dirmaps[plane].wdesc_ecc = spinand->dirmaps[plane].wdesc;
-		spinand->dirmaps[plane].rdesc_ecc = spinand->dirmaps[plane].rdesc;
-
-		return 0;
+	/* Read descriptor */
+	info.primary_op_tmpl = *spinand->op_templates->read_cache;
+	info.primary_op_tmpl.data.ecc = enable_ecc;
+	if (secondary_op) {
+		info.secondary_op_tmpl = *spinand->op_templates->cont_read_cache;
+		info.secondary_op_tmpl.data.ecc = enable_ecc;
 	}
-
-	info.length = nanddev_page_size(nand) + nanddev_per_page_oobsize(nand);
-	info.op_tmpl = *spinand->op_templates->update_cache;
-	info.op_tmpl.data.ecc = true;
-	desc = devm_spi_mem_dirmap_create(&spinand->spimem->spi->dev,
-					  spinand->spimem, &info);
-	if (IS_ERR(desc))
-		return PTR_ERR(desc);
-
-	spinand->dirmaps[plane].wdesc_ecc = desc;
-
-	info.op_tmpl = *spinand->op_templates->read_cache;
-	info.op_tmpl.data.ecc = true;
 	desc = spinand_create_rdesc(spinand, &info);
 	if (IS_ERR(desc))
 		return PTR_ERR(desc);
 
-	spinand->dirmaps[plane].rdesc_ecc = desc;
+	spinand->dirmaps[plane].rdesc = desc;
 
 	return 0;
 }
@@ -1421,7 +1428,7 @@ static void spinand_manufacturer_cleanup(struct spinand_device *spinand)
 		return spinand->manufacturer->ops->cleanup(spinand);
 }
 
-static bool spinand_op_is_odtr(const struct spi_mem_op *op)
+bool spinand_op_is_odtr(const struct spi_mem_op *op)
 {
 	return op->cmd.dtr && op->cmd.buswidth == 8;
 }
@@ -1640,6 +1647,33 @@ int spinand_match_and_init(struct spinand_device *spinand,
 		if (ret)
 			return ret;
 
+		if (info->op_variants.cont_read_cache) {
+			op = spinand_select_op_variant(spinand, SSDR,
+						       info->op_variants.cont_read_cache);
+			if (op) {
+				const struct spi_mem_op *read_op;
+
+				read_op = spinand->ssdr_op_templates.read_cache;
+
+				/*
+				 * Sometimes the fastest continuous read variant may not
+				 * be supported. In this case, prefer to use the fastest
+				 * read from cache variant and disable continuous reads.
+				 */
+				if (read_op->cmd.buswidth > op->cmd.buswidth ||
+				    (read_op->cmd.dtr && !op->cmd.dtr) ||
+				    read_op->addr.buswidth > op->addr.buswidth ||
+				    (read_op->addr.dtr && !op->addr.dtr) ||
+				    read_op->data.buswidth > op->data.buswidth ||
+				    (read_op->data.dtr && !op->data.dtr))
+					spinand->cont_read_possible = false;
+				else
+					spinand->ssdr_op_templates.cont_read_cache = op;
+			} else {
+				spinand->cont_read_possible = false;
+			}
+		}
+
 		/* I/O variants selection with octo-spi DDR commands (optional) */
 
 		ret = spinand_init_odtr_instruction_set(spinand);
@@ -1662,6 +1696,15 @@ int spinand_match_and_init(struct spinand_device *spinand,
 					       info->op_variants.update_cache);
 		spinand->odtr_op_templates.update_cache = op;
 
+		if (info->op_variants.cont_read_cache) {
+			op = spinand_select_op_variant(spinand, ODTR,
+						       info->op_variants.cont_read_cache);
+			if (op)
+				spinand->odtr_op_templates.cont_read_cache = op;
+			else
+				spinand->cont_read_possible = false;
+		}
+
 		return 0;
 	}
 
diff --git a/drivers/mtd/nand/spi/winbond.c b/drivers/mtd/nand/spi/winbond.c
index 7cc0f0091430c..9b78c1e6cbc9d 100644
--- a/drivers/mtd/nand/spi/winbond.c
+++ b/drivers/mtd/nand/spi/winbond.c
@@ -15,9 +15,11 @@
 
 #define SPINAND_MFR_WINBOND		0xEF
 
+#define WINBOND_CFG_HFREQ		BIT(0)
 #define WINBOND_CFG_BUF_READ		BIT(3)
 
 #define W25N04KV_STATUS_ECC_5_8_BITFLIPS	(3 << 4)
+#define W25W35NXXJW_STATUS_ECC_MULT_UNCOR	(3 << 4)
 
 #define W25N0XJW_SR4			0xD0
 #define W25N0XJW_SR4_HS			BIT(2)
@@ -29,6 +31,105 @@
 #define W35N01JW_VCR_IO_MODE_OCTAL_DDR		0xC7
 #define W35N01JW_VCR_DUMMY_CLOCK_REG	0x01
 
+/*
+ * Winbond chips ignore the address bytes during continuous reads, and
+ * because the dummy cycles are enough they indicate dropping the
+ * address cycles from the continuous read from cache variants. This is
+ * very poorly supported by SPI controller drivers which are "wired" to
+ * always at least provide the column. Keep using address cycles, but
+ * reduce the number of dummy cycles accordingly.
+ */
+#define WINBOND_CONT_READ_FROM_CACHE_FAST_1S_1S_1S_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0x0b, 1),				\
+		   SPI_MEM_OP_ADDR(1, 0, 1),				\
+		   SPI_MEM_OP_DUMMY(ndummy - 1, 1),			\
+		   SPI_MEM_OP_DATA_IN(len, buf, 1),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_1D_1D_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0x0d, 1),				\
+		   SPI_MEM_DTR_OP_ADDR(2, 0, 1),			\
+		   SPI_MEM_DTR_OP_DUMMY(ndummy, 1),			\
+		   SPI_MEM_DTR_OP_DATA_IN(len, buf, 1),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_1S_2S_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0x3b, 1),				\
+		   SPI_MEM_OP_ADDR(1, 0, 1),				\
+		   SPI_MEM_OP_DUMMY(ndummy - 1, 1),			\
+		   SPI_MEM_OP_DATA_IN(len, buf, 2),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_2S_2S_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0xbb, 1),				\
+		   SPI_MEM_OP_ADDR(1, 0, 2),				\
+		   SPI_MEM_OP_DUMMY(ndummy - 1, 2),			\
+		   SPI_MEM_OP_DATA_IN(len, buf, 2),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_2D_2D_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0xbd, 1),				\
+		   SPI_MEM_DTR_OP_ADDR(1, 0, 2),			\
+		   SPI_MEM_DTR_OP_DUMMY(ndummy - 1, 2),			\
+		   SPI_MEM_DTR_OP_DATA_IN(len, buf, 2),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_1S_4S_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0x6b, 1),				\
+		   SPI_MEM_OP_ADDR(1, 0, 1),				\
+		   SPI_MEM_OP_DUMMY(ndummy - 1, 1),			\
+		   SPI_MEM_OP_DATA_IN(len, buf, 4),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_1D_4D_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0x6d, 1),				\
+		   SPI_MEM_DTR_OP_ADDR(1, 0, 1),			\
+		   SPI_MEM_DTR_OP_DUMMY(ndummy - 1, 1),			\
+		   SPI_MEM_DTR_OP_DATA_IN(len, buf, 4),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_4S_4S_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0xeb, 1),				\
+		   SPI_MEM_OP_ADDR(1, 0, 4),				\
+		   SPI_MEM_OP_DUMMY(ndummy - 1, 4),			\
+		   SPI_MEM_OP_DATA_IN(len, buf, 4),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_4D_4D_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0xed, 1),				\
+		   SPI_MEM_DTR_OP_ADDR(1, 0, 4),			\
+		   SPI_MEM_DTR_OP_DUMMY(ndummy - 1, 4),			\
+		   SPI_MEM_DTR_OP_DATA_IN(len, buf, 4),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_1S_8S_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0x8b, 1),				\
+		   SPI_MEM_OP_ADDR(1, 0, 1),				\
+		   SPI_MEM_OP_DUMMY(ndummy - 1, 1),			\
+		   SPI_MEM_OP_DATA_IN(len, buf, 8),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_1D_8D_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0x9d, 1),				\
+		   SPI_MEM_DTR_OP_ADDR(1, 0, 1),			\
+		   SPI_MEM_DTR_OP_DUMMY(ndummy - 1, 1),			\
+		   SPI_MEM_DTR_OP_DATA_IN(len, buf, 8),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_1S_8S_8S_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_OP_CMD(0xcb, 1),				\
+		   SPI_MEM_OP_ADDR(1, 0, 8),				\
+		   SPI_MEM_OP_DUMMY(ndummy - 1, 8),			\
+		   SPI_MEM_OP_DATA_IN(len, buf, 8),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
+#define WINBOND_CONT_READ_FROM_CACHE_8D_8D_8D_OP(ndummy, buf, len, freq) \
+	SPI_MEM_OP(SPI_MEM_DTR_OP_RPT_CMD(0x9d, 8),			\
+		   SPI_MEM_DTR_OP_ADDR(2, 0, 8),			\
+		   SPI_MEM_DTR_OP_DUMMY(ndummy - 2, 8),			\
+		   SPI_MEM_DTR_OP_DATA_IN(len, buf, 8),			\
+		   SPI_MEM_OP_MAX_FREQ(freq))
+
 /*
  * "X2" in the core is equivalent to "dual output" in the datasheets,
  * "X4" in the core is equivalent to "quad output" in the datasheets.
@@ -49,6 +150,19 @@ static SPINAND_OP_VARIANTS(read_cache_octal_variants,
 		SPINAND_PAGE_READ_FROM_CACHE_FAST_1S_1S_1S_OP(0, 1, NULL, 0, 0),
 		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_1S_OP(0, 1, NULL, 0, 0));
 
+static SPINAND_OP_VARIANTS(cont_read_cache_octal_variants,
+		WINBOND_CONT_READ_FROM_CACHE_8D_8D_8D_OP(24, NULL, 0, 120 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_8D_8D_8D_OP(16, NULL, 0, 86 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_1D_8D_OP(3, NULL, 0, 120 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_1D_8D_OP(2, NULL, 0, 105 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_8S_8S_OP(20, NULL, 0, 0),
+		WINBOND_CONT_READ_FROM_CACHE_1S_8S_8S_OP(16, NULL, 0, 162 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_8S_8S_OP(12, NULL, 0, 124 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_8S_8S_OP(8, NULL, 0, 86 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_1S_8S_OP(2, NULL, 0, 0),
+		WINBOND_CONT_READ_FROM_CACHE_1S_1S_8S_OP(1, NULL, 0, 133 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_FAST_1S_1S_1S_OP(1, NULL, 0, 0));
+
 static SPINAND_OP_VARIANTS(write_cache_octal_variants,
 		SPINAND_PROG_LOAD_8D_8D_8D_OP(true, 0, NULL, 0),
 		SPINAND_PROG_LOAD_1S_8S_8S_OP(true, 0, NULL, 0),
@@ -75,6 +189,20 @@ static SPINAND_OP_VARIANTS(read_cache_dual_quad_dtr_variants,
 		SPINAND_PAGE_READ_FROM_CACHE_FAST_1S_1S_1S_OP(0, 1, NULL, 0, 0),
 		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_1S_OP(0, 1, NULL, 0, 54 * HZ_PER_MHZ));
 
+static SPINAND_OP_VARIANTS(cont_read_cache_dual_quad_dtr_variants,
+		WINBOND_CONT_READ_FROM_CACHE_1S_4D_4D_OP(11, NULL, 0, 80 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_1D_4D_OP(5, NULL, 0, 80 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_4S_4S_OP(7, NULL, 0, 0),
+		WINBOND_CONT_READ_FROM_CACHE_1S_4S_4S_OP(6, NULL, 0, 104 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_1S_4S_OP(4, NULL, 0, 0),
+		WINBOND_CONT_READ_FROM_CACHE_1S_2D_2D_OP(6, NULL, 0, 80 * HZ_PER_MHZ),
+		/* The 1S_1D_2D variant would require 4.5 dummy bytes, this is not possible */
+		WINBOND_CONT_READ_FROM_CACHE_1S_2S_2S_OP(5, NULL, 0, 0),
+		WINBOND_CONT_READ_FROM_CACHE_1S_2S_2S_OP(4, NULL, 0, 104 * HZ_PER_MHZ),
+		WINBOND_CONT_READ_FROM_CACHE_1S_1S_2S_OP(4, NULL, 0, 0),
+		/* The 1S_1D_1D variant would require 4.5 dummy bytes, this is not possible */
+		WINBOND_CONT_READ_FROM_CACHE_FAST_1S_1S_1S_OP(4, NULL, 0, 0));
+
 static SPINAND_OP_VARIANTS(read_cache_variants,
 		SPINAND_PAGE_READ_FROM_CACHE_1S_4S_4S_OP(0, 2, NULL, 0, 0),
 		SPINAND_PAGE_READ_FROM_CACHE_1S_1S_4S_OP(0, 1, NULL, 0, 0),
@@ -326,45 +454,84 @@ static int w25n02kv_ecc_get_status(struct spinand_device *spinand,
 	return -EINVAL;
 }
 
+static int w25w35nxxjw_ecc_get_status(struct spinand_device *spinand, u8 status)
+{
+	switch (status & STATUS_ECC_MASK) {
+	case STATUS_ECC_NO_BITFLIPS:
+		return 0;
+
+	case STATUS_ECC_HAS_BITFLIPS:
+		return 1;
+
+	case STATUS_ECC_UNCOR_ERROR:
+	case W25W35NXXJW_STATUS_ECC_MULT_UNCOR:
+		return -EBADMSG;
+
+	default:
+		break;
+	}
+
+	return -EINVAL;
+}
+
+static int w25n0xjw_set_sr4_hs(struct spinand_device *spinand, bool enable)
+{
+	int ret;
+	u8 sr4;
+
+	ret = spinand_read_reg_op(spinand, W25N0XJW_SR4, &sr4);
+	if (ret)
+		return ret;
+
+	if (enable)
+		sr4 |= W25N0XJW_SR4_HS;
+	else
+		sr4 &= ~W25N0XJW_SR4_HS;
+
+	return spinand_write_reg_op(spinand, W25N0XJW_SR4, sr4);
+}
+
+/*
+ * SDR dual and quad I/O operations over 104MHz require the HS bit to
+ * enable a few more dummy cycles.
+ */
+static bool w25n0xjw_op_needs_hs(const struct spi_mem_op *op)
+{
+	if (op->cmd.dtr || op->addr.dtr || op->dummy.dtr || op->data.dtr)
+		return false;
+	else if (op->cmd.buswidth != 1 || op->addr.buswidth == 1)
+		return false;
+	else if (op->max_freq && op->max_freq <= 104 * HZ_PER_MHZ)
+		return false;
+
+	return true;
+}
+
 static int w25n0xjw_hs_cfg(struct spinand_device *spinand,
 			   enum spinand_bus_interface iface)
 {
 	const struct spi_mem_op *op;
-	bool hs;
-	u8 sr4;
-	int ret;
 
 	if (iface != SSDR)
 		return -EOPNOTSUPP;
 
 	/*
-	 * SDR dual and quad I/O operations over 104MHz require the HS bit to
-	 * enable a few more dummy cycles.
+	 * At this stage, we do not yet know the continuous read template, nor
+	 * if there is going to be one. Let's assume the continuous read
+	 * template will be selected with the same heuristics as the buffered
+	 * read variant, as there cannot be a HS configuration mismatch between
+	 * them.
 	 */
 	op = spinand->op_templates->read_cache;
-	if (op->cmd.dtr || op->addr.dtr || op->dummy.dtr || op->data.dtr)
-		hs = false;
-	else if (op->cmd.buswidth != 1 || op->addr.buswidth == 1)
-		hs = false;
-	else if (op->max_freq && op->max_freq <= 104 * HZ_PER_MHZ)
-		hs = false;
-	else
-		hs = true;
-
-	ret = spinand_read_reg_op(spinand, W25N0XJW_SR4, &sr4);
-	if (ret)
-		return ret;
 
-	if (hs)
-		sr4 |= W25N0XJW_SR4_HS;
-	else
-		sr4 &= ~W25N0XJW_SR4_HS;
+	return w25n0xjw_set_sr4_hs(spinand, w25n0xjw_op_needs_hs(op));
+}
 
-	ret = spinand_write_reg_op(spinand, W25N0XJW_SR4, sr4);
-	if (ret)
-		return ret;
+static int w25n0xjw_set_cont_read(struct spinand_device *spinand, bool enable)
+{
+	u8 mask = enable ? 0 : WINBOND_CFG_BUF_READ;
 
-	return 0;
+	return spinand_upd_cfg(spinand, WINBOND_CFG_BUF_READ, mask);
 }
 
 static int w35n0xjw_write_vcr(struct spinand_device *spinand, u8 reg, u8 val)
@@ -451,6 +618,18 @@ static int w35n0xjw_vcr_cfg(struct spinand_device *spinand,
 	return 0;
 }
 
+static int w35n0xjw_set_cont_read(struct spinand_device *spinand, bool enable)
+{
+	const struct spi_mem_op *cont_op = spinand->op_templates->cont_read_cache;
+	u8 mask = enable ? 0 : WINBOND_CFG_BUF_READ;
+
+	if (cont_op && enable && spinand_op_is_odtr(cont_op) &&
+	    cont_op->max_freq >= 90 * HZ_PER_MHZ)
+		mask |= WINBOND_CFG_HFREQ;
+
+	return spinand_upd_cfg(spinand, WINBOND_CFG_BUF_READ | WINBOND_CFG_HFREQ, mask);
+}
+
 static const struct spinand_info winbond_spinand_table[] = {
 	/* 512M-bit densities */
 	SPINAND_INFO("W25N512GW", /* 1.8V */
@@ -485,12 +664,14 @@ static const struct spinand_info winbond_spinand_table[] = {
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xbc, 0x21),
 		     NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 1, 1),
 		     NAND_ECCREQ(1, 512),
-		     SPINAND_INFO_OP_VARIANTS(&read_cache_dual_quad_dtr_variants,
-					      &write_cache_variants,
-					      &update_cache_variants),
+		     SPINAND_INFO_OP_VARIANTS_WITH_CONT(&read_cache_dual_quad_dtr_variants,
+							&write_cache_variants,
+							&update_cache_variants,
+							&cont_read_cache_dual_quad_dtr_variants),
 		     SPINAND_HAS_QE_BIT,
-		     SPINAND_ECCINFO(&w25n01jw_ooblayout, NULL),
-		     SPINAND_CONFIGURE_CHIP(w25n0xjw_hs_cfg)),
+		     SPINAND_ECCINFO(&w25n01jw_ooblayout, w25w35nxxjw_ecc_get_status),
+		     SPINAND_CONFIGURE_CHIP(w25n0xjw_hs_cfg),
+		     SPINAND_CONT_READ(w25n0xjw_set_cont_read)),
 	SPINAND_INFO("W25N01KV", /* 3.3V */
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xae, 0x21),
 		     NAND_MEMORG(1, 2048, 96, 64, 1024, 20, 1, 1, 1),
@@ -504,35 +685,15 @@ static const struct spinand_info winbond_spinand_table[] = {
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xdc, 0x21),
 		     NAND_MEMORG(1, 4096, 128, 64, 512, 10, 1, 1, 1),
 		     NAND_ECCREQ(1, 512),
-		     SPINAND_INFO_OP_VARIANTS(&read_cache_octal_variants,
-					      &write_cache_octal_variants,
-					      &update_cache_octal_variants),
+		     SPINAND_INFO_OP_VARIANTS_WITH_CONT(&read_cache_octal_variants,
+							&write_cache_octal_variants,
+							&update_cache_octal_variants,
+							&cont_read_cache_octal_variants),
 		     0,
 		     SPINAND_INFO_VENDOR_OPS(&winbond_w35_ops),
-		     SPINAND_ECCINFO(&w35n01jw_ooblayout, NULL),
-		     SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg)),
-	SPINAND_INFO("W35N02JW", /* 1.8V */
-		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xdf, 0x22),
-		     NAND_MEMORG(1, 4096, 128, 64, 512, 10, 1, 2, 1),
-		     NAND_ECCREQ(1, 512),
-		     SPINAND_INFO_OP_VARIANTS(&read_cache_octal_variants,
-					      &write_cache_octal_variants,
-					      &update_cache_octal_variants),
-		     SPINAND_ODTR_PACKED_PAGE_READ,
-		     SPINAND_INFO_VENDOR_OPS(&winbond_w35_ops),
-		     SPINAND_ECCINFO(&w35n01jw_ooblayout, NULL),
-		     SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg)),
-	SPINAND_INFO("W35N04JW", /* 1.8V */
-		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xdf, 0x23),
-		     NAND_MEMORG(1, 4096, 128, 64, 512, 10, 1, 4, 1),
-		     NAND_ECCREQ(1, 512),
-		     SPINAND_INFO_OP_VARIANTS(&read_cache_octal_variants,
-					      &write_cache_octal_variants,
-					      &update_cache_octal_variants),
-		     SPINAND_ODTR_PACKED_PAGE_READ,
-		     SPINAND_INFO_VENDOR_OPS(&winbond_w35_ops),
-		     SPINAND_ECCINFO(&w35n01jw_ooblayout, NULL),
-		     SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg)),
+		     SPINAND_ECCINFO(&w35n01jw_ooblayout, w25w35nxxjw_ecc_get_status),
+		     SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg),
+		     SPINAND_CONT_READ(w35n0xjw_set_cont_read)),
 	/* 2G-bit densities */
 	SPINAND_INFO("W25M02GV", /* 2x1G-bit 3.3V */
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xab, 0x21),
@@ -549,12 +710,14 @@ static const struct spinand_info winbond_spinand_table[] = {
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xbf, 0x22),
 		     NAND_MEMORG(1, 2048, 64, 64, 1024, 20, 1, 2, 1),
 		     NAND_ECCREQ(1, 512),
-		     SPINAND_INFO_OP_VARIANTS(&read_cache_dual_quad_dtr_variants,
-					      &write_cache_variants,
-					      &update_cache_variants),
+		     SPINAND_INFO_OP_VARIANTS_WITH_CONT(&read_cache_dual_quad_dtr_variants,
+							&write_cache_variants,
+							&update_cache_variants,
+							&cont_read_cache_dual_quad_dtr_variants),
 		     SPINAND_HAS_QE_BIT,
-		     SPINAND_ECCINFO(&w25m02gv_ooblayout, NULL),
-		     SPINAND_CONFIGURE_CHIP(w25n0xjw_hs_cfg)),
+		     SPINAND_ECCINFO(&w25m02gv_ooblayout, w25w35nxxjw_ecc_get_status),
+		     SPINAND_CONFIGURE_CHIP(w25n0xjw_hs_cfg),
+		     SPINAND_CONT_READ(w25n0xjw_set_cont_read)),
 	SPINAND_INFO("W25N02KV", /* 3.3V */
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xaa, 0x22),
 		     NAND_MEMORG(1, 2048, 128, 64, 2048, 40, 1, 1, 1),
@@ -573,6 +736,19 @@ static const struct spinand_info winbond_spinand_table[] = {
 					      &update_cache_variants),
 		     0,
 		     SPINAND_ECCINFO(&w25n02kv_ooblayout, w25n02kv_ecc_get_status)),
+	SPINAND_INFO("W35N02JW", /* 1.8V */
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xdf, 0x22),
+		     NAND_MEMORG(1, 4096, 128, 64, 512, 10, 1, 2, 1),
+		     NAND_ECCREQ(1, 512),
+		     SPINAND_INFO_OP_VARIANTS_WITH_CONT(&read_cache_octal_variants,
+							&write_cache_octal_variants,
+							&update_cache_octal_variants,
+							&cont_read_cache_octal_variants),
+		     SPINAND_ODTR_PACKED_PAGE_READ,
+		     SPINAND_INFO_VENDOR_OPS(&winbond_w35_ops),
+		     SPINAND_ECCINFO(&w35n01jw_ooblayout, w25w35nxxjw_ecc_get_status),
+		     SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg),
+		     SPINAND_CONT_READ(w35n0xjw_set_cont_read)),
 	/* 4G-bit densities */
 	SPINAND_INFO("W25N04KV", /* 3.3V */
 		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xaa, 0x23),
@@ -592,6 +768,19 @@ static const struct spinand_info winbond_spinand_table[] = {
 					      &update_cache_variants),
 		     0,
 		     SPINAND_ECCINFO(&w25n02kv_ooblayout, w25n02kv_ecc_get_status)),
+	SPINAND_INFO("W35N04JW", /* 1.8V */
+		     SPINAND_ID(SPINAND_READID_METHOD_OPCODE_DUMMY, 0xdf, 0x23),
+		     NAND_MEMORG(1, 4096, 128, 64, 512, 10, 1, 4, 1),
+		     NAND_ECCREQ(1, 512),
+		     SPINAND_INFO_OP_VARIANTS_WITH_CONT(&read_cache_octal_variants,
+							&write_cache_octal_variants,
+							&update_cache_octal_variants,
+							&cont_read_cache_octal_variants),
+		     SPINAND_ODTR_PACKED_PAGE_READ,
+		     SPINAND_INFO_VENDOR_OPS(&winbond_w35_ops),
+		     SPINAND_ECCINFO(&w35n01jw_ooblayout, w25w35nxxjw_ecc_get_status),
+		     SPINAND_CONFIGURE_CHIP(w35n0xjw_vcr_cfg),
+		     SPINAND_CONT_READ(w35n0xjw_set_cont_read)),
 };
 
 static int winbond_spinand_init(struct spinand_device *spinand)
diff --git a/drivers/mtd/sm_ftl.c b/drivers/mtd/sm_ftl.c
index c8032755f9a46..fb0f97fb94f04 100644
--- a/drivers/mtd/sm_ftl.c
+++ b/drivers/mtd/sm_ftl.c
@@ -1133,7 +1133,7 @@ static void sm_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
 	struct sm_ftl *ftl;
 
 	/* Allocate & initialize our private structure */
-	ftl = kzalloc_obj(struct sm_ftl);
+	ftl = kzalloc_flex(*ftl, cis_buffer, SM_SECTOR_SIZE);
 	if (!ftl)
 		goto error1;
 
@@ -1145,25 +1145,20 @@ static void sm_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
 	/* Read media information */
 	if (sm_get_media_info(ftl, mtd)) {
 		dbg("found unsupported mtd device, aborting");
-		goto error2;
+		goto error1;
 	}
 
 
-	/* Allocate temporary CIS buffer for read retry support */
-	ftl->cis_buffer = kzalloc(SM_SECTOR_SIZE, GFP_KERNEL);
-	if (!ftl->cis_buffer)
-		goto error2;
-
 	/* Allocate zone array, it will be initialized on demand */
 	ftl->zones = kzalloc_objs(struct ftl_zone, ftl->zone_count);
 	if (!ftl->zones)
-		goto error3;
+		goto error2;
 
 	/* Allocate the cache*/
 	ftl->cache_data = kzalloc(ftl->block_size, GFP_KERNEL);
 
 	if (!ftl->cache_data)
-		goto error4;
+		goto error3;
 
 	sm_cache_init(ftl);
 
@@ -1171,7 +1166,7 @@ static void sm_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
 	/* Allocate upper layer structure and initialize it */
 	trans = kzalloc_obj(struct mtd_blktrans_dev);
 	if (!trans)
-		goto error5;
+		goto error4;
 
 	ftl->trans = trans;
 	trans->priv = ftl;
@@ -1184,12 +1179,12 @@ static void sm_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
 
 	if (sm_find_cis(ftl)) {
 		dbg("CIS not found on mtd device, aborting");
-		goto error6;
+		goto error5;
 	}
 
 	ftl->disk_attributes = sm_create_sysfs_attributes(ftl);
 	if (!ftl->disk_attributes)
-		goto error6;
+		goto error5;
 	trans->disk_attributes = ftl->disk_attributes;
 
 	sm_printk("Found %d MiB xD/SmartMedia FTL on mtd%d",
@@ -1206,17 +1201,15 @@ static void sm_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
 	/* Register device*/
 	if (add_mtd_blktrans_dev(trans)) {
 		dbg("error in mtdblktrans layer");
-		goto error6;
+		goto error5;
 	}
 	return;
-error6:
-	kfree(trans);
 error5:
-	kfree(ftl->cache_data);
+	kfree(trans);
 error4:
-	kfree(ftl->zones);
+	kfree(ftl->cache_data);
 error3:
-	kfree(ftl->cis_buffer);
+	kfree(ftl->zones);
 error2:
 	kfree(ftl);
 error1:
@@ -1242,7 +1235,6 @@ static void sm_remove_dev(struct mtd_blktrans_dev *dev)
 	}
 
 	sm_delete_sysfs_attributes(ftl);
-	kfree(ftl->cis_buffer);
 	kfree(ftl->zones);
 	kfree(ftl->cache_data);
 	kfree(ftl);
diff --git a/drivers/mtd/sm_ftl.h b/drivers/mtd/sm_ftl.h
index 6aed8b60de167..1a1a54ce836a6 100644
--- a/drivers/mtd/sm_ftl.h
+++ b/drivers/mtd/sm_ftl.h
@@ -39,7 +39,6 @@ struct sm_ftl {
 	int cis_block;			/* CIS block location */
 	int cis_boffset;		/* CIS offset in the block */
 	int cis_page_offset;		/* CIS offset in the page */
-	void *cis_buffer;		/* tmp buffer for cis reads */
 
 	/* Cache */
 	int cache_block;		/* block number of cached block */
@@ -56,6 +55,7 @@ struct sm_ftl {
 	int cylinders;
 
 	struct attribute_group *disk_attributes;
+	u8 cis_buffer[];		/* tmp buffer for cis reads */
 };
 
 struct chs_entry {
diff --git a/drivers/mtd/spi-nor/core.c b/drivers/mtd/spi-nor/core.c
index 5dd0b3cb52503..a7bc458edc5cd 100644
--- a/drivers/mtd/spi-nor/core.c
+++ b/drivers/mtd/spi-nor/core.c
@@ -3641,14 +3641,15 @@ EXPORT_SYMBOL_GPL(spi_nor_scan);
 static int spi_nor_create_read_dirmap(struct spi_nor *nor)
 {
 	struct spi_mem_dirmap_info info = {
-		.op_tmpl = SPI_MEM_OP(SPI_MEM_OP_CMD(nor->read_opcode, 0),
-				      SPI_MEM_OP_ADDR(nor->addr_nbytes, 0, 0),
-				      SPI_MEM_OP_DUMMY(nor->read_dummy, 0),
-				      SPI_MEM_OP_DATA_IN(0, NULL, 0)),
+		.op_tmpl = &info.primary_op_tmpl,
+		.primary_op_tmpl = SPI_MEM_OP(SPI_MEM_OP_CMD(nor->read_opcode, 0),
+					      SPI_MEM_OP_ADDR(nor->addr_nbytes, 0, 0),
+					      SPI_MEM_OP_DUMMY(nor->read_dummy, 0),
+					      SPI_MEM_OP_DATA_IN(0, NULL, 0)),
 		.offset = 0,
 		.length = nor->params->size,
 	};
-	struct spi_mem_op *op = &info.op_tmpl;
+	struct spi_mem_op *op = info.op_tmpl;
 
 	spi_nor_spimem_setup_op(nor, op, nor->read_proto);
 
@@ -3672,14 +3673,15 @@ static int spi_nor_create_read_dirmap(struct spi_nor *nor)
 static int spi_nor_create_write_dirmap(struct spi_nor *nor)
 {
 	struct spi_mem_dirmap_info info = {
-		.op_tmpl = SPI_MEM_OP(SPI_MEM_OP_CMD(nor->program_opcode, 0),
-				      SPI_MEM_OP_ADDR(nor->addr_nbytes, 0, 0),
-				      SPI_MEM_OP_NO_DUMMY,
-				      SPI_MEM_OP_DATA_OUT(0, NULL, 0)),
+		.op_tmpl = &info.primary_op_tmpl,
+		.primary_op_tmpl = SPI_MEM_OP(SPI_MEM_OP_CMD(nor->program_opcode, 0),
+					      SPI_MEM_OP_ADDR(nor->addr_nbytes, 0, 0),
+					      SPI_MEM_OP_NO_DUMMY,
+					      SPI_MEM_OP_DATA_OUT(0, NULL, 0)),
 		.offset = 0,
 		.length = nor->params->size,
 	};
-	struct spi_mem_op *op = &info.op_tmpl;
+	struct spi_mem_op *op = info.op_tmpl;
 
 	if (nor->program_opcode == SPINOR_OP_AAI_WP && nor->sst_write_second)
 		op->addr.nbytes = 0;
diff --git a/drivers/mtd/ubi/ubi.h b/drivers/mtd/ubi/ubi.h
index af466cd83ae07..95548d5aa9308 100644
--- a/drivers/mtd/ubi/ubi.h
+++ b/drivers/mtd/ubi/ubi.h
@@ -160,6 +160,7 @@ struct ubi_vid_io_buf {
  * struct ubi_wl_entry - wear-leveling entry.
  * @u.rb: link in the corresponding (free/used) RB-tree
  * @u.list: link in the protection queue
+ * @u: union of rb_node and list_head
  * @ec: erase counter
  * @pnum: physical eraseblock number
  *
@@ -282,6 +283,7 @@ struct ubi_eba_leb_desc {
  * @writers: number of users holding this volume in read-write mode
  * @exclusive: whether somebody holds this volume in exclusive mode
  * @metaonly: whether somebody is altering only meta data of this volume
+ * @is_dead: prevents taking a reference to the volume
  *
  * @reserved_pebs: how many physical eraseblocks are reserved for this volume
  * @vol_type: volume type (%UBI_DYNAMIC_VOLUME or %UBI_STATIC_VOLUME)
@@ -449,6 +451,7 @@ struct ubi_debug_info {
  *                @vol->eba_tbl.
  * @ref_count: count of references on the UBI device
  * @image_seq: image sequence number recorded on EC headers
+ * @is_dead: prevents taking a reference to the volume
  *
  * @rsvd_pebs: count of reserved physical eraseblocks
  * @avail_pebs: count of available physical eraseblocks
@@ -741,7 +744,7 @@ struct ubi_ainf_volume {
  * @highest_vol_id: highest volume ID
  * @is_empty: flag indicating whether the MTD device is empty or not
  * @force_full_scan: flag indicating whether we need to do a full scan and drop
-		     all existing Fastmap data structures
+ *		     all existing Fastmap data structures
  * @min_ec: lowest erase counter value
  * @max_ec: highest erase counter value
  * @max_sqnum: highest sequence number value
@@ -750,7 +753,7 @@ struct ubi_ainf_volume {
  * @ec_count: a temporary variable used when calculating @mean_ec
  * @aeb_slab_cache: slab cache for &struct ubi_ainf_peb objects
  * @ech: temporary EC header. Only available during scan
- * @vidh: temporary VID buffer. Only available during scan
+ * @vidb: temporary VID buffer. Only available during scan
  *
  * This data structure contains the result of attaching an MTD device and may
  * be used by other UBI sub-systems to build final UBI data structures, further
@@ -1090,7 +1093,7 @@ static inline void ubi_init_vid_buf(const struct ubi_device *ubi,
 }
 
 /**
- * ubi_init_vid_buf - Allocate a VID buffer
+ * ubi_alloc_vid_buf - Allocate a VID buffer
  * @ubi: the UBI device
  * @gfp_flags: GFP flags to use for the allocation
  */
diff --git a/drivers/mux/core.c b/drivers/mux/core.c
index 23538de2c91b4..5083e3d19606b 100644
--- a/drivers/mux/core.c
+++ b/drivers/mux/core.c
@@ -533,14 +533,16 @@ static struct mux_chip *of_find_mux_chip_by_node(struct device_node *np)
  * @state: Pointer to where the requested state is returned, or NULL when
  *         the required multiplexer states are handled by other means.
  * @optional: Whether to return NULL and silence errors when mux doesn't exist.
+ * @node: the device nodes, use dev->of_node if it is NULL.
  *
  * Return: Pointer to the mux-control on success, an ERR_PTR with a negative
  * errno on error, or NULL if optional is true and mux doesn't exist.
  */
 static struct mux_control *mux_get(struct device *dev, const char *mux_name,
-				   unsigned int *state, bool optional)
+				   unsigned int *state, bool optional,
+				   struct device_node *node)
 {
-	struct device_node *np = dev->of_node;
+	struct device_node *np = node ? node : dev->of_node;
 	struct of_phandle_args args;
 	struct mux_chip *mux_chip;
 	unsigned int controller;
@@ -635,7 +637,7 @@ static struct mux_control *mux_get(struct device *dev, const char *mux_name,
  */
 struct mux_control *mux_control_get(struct device *dev, const char *mux_name)
 {
-	struct mux_control *mux = mux_get(dev, mux_name, NULL, false);
+	struct mux_control *mux = mux_get(dev, mux_name, NULL, false, NULL);
 
 	if (!mux)
 		return ERR_PTR(-ENOENT);
@@ -654,7 +656,7 @@ EXPORT_SYMBOL_GPL(mux_control_get);
  */
 struct mux_control *mux_control_get_optional(struct device *dev, const char *mux_name)
 {
-	return mux_get(dev, mux_name, NULL, true);
+	return mux_get(dev, mux_name, NULL, true, NULL);
 }
 EXPORT_SYMBOL_GPL(mux_control_get_optional);
 
@@ -712,11 +714,14 @@ EXPORT_SYMBOL_GPL(devm_mux_control_get);
  * @dev: The device that needs a mux-state.
  * @mux_name: The name identifying the mux-state.
  * @optional: Whether to return NULL and silence errors when mux doesn't exist.
+ * @np: the device nodes, use dev->of_node if it is NULL.
  *
  * Return: Pointer to the mux-state on success, an ERR_PTR with a negative
  * errno on error, or NULL if optional is true and mux doesn't exist.
  */
-static struct mux_state *mux_state_get(struct device *dev, const char *mux_name, bool optional)
+static struct mux_state *
+mux_state_get(struct device *dev, const char *mux_name, bool optional,
+	      struct device_node *np)
 {
 	struct mux_state *mstate;
 
@@ -724,7 +729,7 @@ static struct mux_state *mux_state_get(struct device *dev, const char *mux_name,
 	if (!mstate)
 		return ERR_PTR(-ENOMEM);
 
-	mstate->mux = mux_get(dev, mux_name, &mstate->state, optional);
+	mstate->mux = mux_get(dev, mux_name, &mstate->state, optional, np);
 	if (IS_ERR(mstate->mux)) {
 		int err = PTR_ERR(mstate->mux);
 
@@ -766,6 +771,7 @@ static void devm_mux_state_release(struct device *dev, void *res)
  * @dev: The device that needs a mux-state.
  * @mux_name: The name identifying the mux-state.
  * @optional: Whether to return NULL and silence errors when mux doesn't exist.
+ * @np: The device nodes, use dev->of_node if it is NULL.
  * @init: Optional function pointer for mux-state object initialisation.
  * @exit: Optional function pointer for mux-state object cleanup on release.
  *
@@ -773,7 +779,7 @@ static void devm_mux_state_release(struct device *dev, void *res)
  * errno on error, or NULL if optional is true and mux doesn't exist.
  */
 static struct mux_state *__devm_mux_state_get(struct device *dev, const char *mux_name,
-					      bool optional,
+					      bool optional, struct device_node *np,
 					      int (*init)(struct mux_state *mstate),
 					      int (*exit)(struct mux_state *mstate))
 {
@@ -781,7 +787,7 @@ static struct mux_state *__devm_mux_state_get(struct device *dev, const char *mu
 	struct mux_state *mstate;
 	int ret;
 
-	mstate = mux_state_get(dev, mux_name, optional);
+	mstate = mux_state_get(dev, mux_name, optional, np);
 	if (IS_ERR(mstate))
 		return ERR_CAST(mstate);
 	else if (optional && !mstate)
@@ -815,20 +821,23 @@ static struct mux_state *__devm_mux_state_get(struct device *dev, const char *mu
 }
 
 /**
- * devm_mux_state_get() - Get the mux-state for a device, with resource
- *			  management.
+ * devm_mux_state_get_from_np() - Get the mux-state for a device, with resource
+ *				  management.
  * @dev: The device that needs a mux-control.
  * @mux_name: The name identifying the mux-control.
+ * @np: the device nodes, use dev->of_node if it is NULL.
  *
  * Return: Pointer to the mux-state, or an ERR_PTR with a negative errno.
  *
  * The mux-state will automatically be freed on release.
  */
-struct mux_state *devm_mux_state_get(struct device *dev, const char *mux_name)
+struct mux_state *
+devm_mux_state_get_from_np(struct device *dev, const char *mux_name,
+			   struct device_node *np)
 {
-	return __devm_mux_state_get(dev, mux_name, false, NULL, NULL);
+	return __devm_mux_state_get(dev, mux_name, false, np, NULL, NULL);
 }
-EXPORT_SYMBOL_GPL(devm_mux_state_get);
+EXPORT_SYMBOL_GPL(devm_mux_state_get_from_np);
 
 /**
  * devm_mux_state_get_optional() - Get the optional mux-state for a device,
@@ -843,7 +852,7 @@ EXPORT_SYMBOL_GPL(devm_mux_state_get);
  */
 struct mux_state *devm_mux_state_get_optional(struct device *dev, const char *mux_name)
 {
-	return __devm_mux_state_get(dev, mux_name, true, NULL, NULL);
+	return __devm_mux_state_get(dev, mux_name, true, NULL, NULL, NULL);
 }
 EXPORT_SYMBOL_GPL(devm_mux_state_get_optional);
 
@@ -861,7 +870,8 @@ EXPORT_SYMBOL_GPL(devm_mux_state_get_optional);
  */
 struct mux_state *devm_mux_state_get_selected(struct device *dev, const char *mux_name)
 {
-	return __devm_mux_state_get(dev, mux_name, false, mux_state_select, mux_state_deselect);
+	return __devm_mux_state_get(dev, mux_name, false, NULL,
+				    mux_state_select, mux_state_deselect);
 }
 EXPORT_SYMBOL_GPL(devm_mux_state_get_selected);
 
@@ -881,7 +891,8 @@ EXPORT_SYMBOL_GPL(devm_mux_state_get_selected);
 struct mux_state *devm_mux_state_get_optional_selected(struct device *dev,
 						       const char *mux_name)
 {
-	return __devm_mux_state_get(dev, mux_name, true, mux_state_select, mux_state_deselect);
+	return __devm_mux_state_get(dev, mux_name, true, NULL,
+				    mux_state_select, mux_state_deselect);
 }
 EXPORT_SYMBOL_GPL(devm_mux_state_get_optional_selected);
 
diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig
index 8ec98f6dfef93..ff79c466712df 100644
--- a/drivers/net/Kconfig
+++ b/drivers/net/Kconfig
@@ -630,11 +630,4 @@ config NET_FAILOVER
 	  a VM with direct attached VF by failing over to the paravirtual
 	  datapath when the VF is unplugged.
 
-config NETDEV_LEGACY_INIT
-	bool
-	depends on ISA
-	help
-	  Drivers that call netdev_boot_setup_check() should select this
-	  symbol, everything else no longer needs it.
-
 endif # NETDEVICES
diff --git a/drivers/net/Makefile b/drivers/net/Makefile
index b87a741fc9521..88e4c485d6b2f 100644
--- a/drivers/net/Makefile
+++ b/drivers/net/Makefile
@@ -21,7 +21,6 @@ obj-$(CONFIG_MACVTAP) += macvtap.o
 obj-$(CONFIG_MII) += mii.o
 obj-$(CONFIG_MDIO) += mdio.o
 obj-$(CONFIG_NET) += loopback.o
-obj-$(CONFIG_NETDEV_LEGACY_INIT) += Space.o
 obj-$(CONFIG_NETCONSOLE) += netconsole.o
 obj-$(CONFIG_NETKIT) += netkit.o
 obj-y += phy/
diff --git a/drivers/net/Space.c b/drivers/net/Space.c
deleted file mode 100644
index 305f0a712a646..0000000000000
--- a/drivers/net/Space.c
+++ /dev/null
@@ -1,237 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * INET		An implementation of the TCP/IP protocol suite for the LINUX
- *		operating system.  INET is implemented using the  BSD Socket
- *		interface as the means of communication with the user level.
- *
- *		Holds initial configuration information for devices.
- *
- * Version:	@(#)Space.c	1.0.7	08/12/93
- *
- * Authors:	Ross Biro
- *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
- *		Donald J. Becker, <becker@scyld.com>
- *
- * Changelog:
- *		Stephen Hemminger (09/2003)
- *		- get rid of pre-linked dev list, dynamic device allocation
- *		Paul Gortmaker (03/2002)
- *		- struct init cleanup, enable multiple ISA autoprobes.
- *		Arnaldo Carvalho de Melo <acme@conectiva.com.br> - 09/1999
- *		- fix sbni: s/device/net_device/
- *		Paul Gortmaker (06/98):
- *		 - sort probes in a sane way, make sure all (safe) probes
- *		   get run once & failed autoprobes don't autoprobe again.
- */
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/errno.h>
-#include <linux/init.h>
-#include <linux/netlink.h>
-#include <net/Space.h>
-
-/*
- * This structure holds boot-time configured netdevice settings. They
- * are then used in the device probing.
- */
-struct netdev_boot_setup {
-	char name[IFNAMSIZ];
-	struct ifmap map;
-};
-#define NETDEV_BOOT_SETUP_MAX 8
-
-
-/******************************************************************************
- *
- *		      Device Boot-time Settings Routines
- *
- ******************************************************************************/
-
-/* Boot time configuration table */
-static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
-
-/**
- *	netdev_boot_setup_add	- add new setup entry
- *	@name: name of the device
- *	@map: configured settings for the device
- *
- *	Adds new setup entry to the dev_boot_setup list.  The function
- *	returns 0 on error and 1 on success.  This is a generic routine to
- *	all netdevices.
- */
-static int netdev_boot_setup_add(char *name, struct ifmap *map)
-{
-	struct netdev_boot_setup *s;
-	int i;
-
-	s = dev_boot_setup;
-	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
-		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
-			strscpy_pad(s[i].name, name);
-			memcpy(&s[i].map, map, sizeof(s[i].map));
-			break;
-		}
-	}
-
-	return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
-}
-
-/**
- * netdev_boot_setup_check	- check boot time settings
- * @dev: the netdevice
- *
- * Check boot time settings for the device.
- * The found settings are set for the device to be used
- * later in the device probing.
- * Returns 0 if no settings found, 1 if they are.
- */
-int netdev_boot_setup_check(struct net_device *dev)
-{
-	struct netdev_boot_setup *s = dev_boot_setup;
-	int i;
-
-	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
-		if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
-		    !strcmp(dev->name, s[i].name)) {
-			dev->irq = s[i].map.irq;
-			dev->base_addr = s[i].map.base_addr;
-			dev->mem_start = s[i].map.mem_start;
-			dev->mem_end = s[i].map.mem_end;
-			return 1;
-		}
-	}
-	return 0;
-}
-EXPORT_SYMBOL(netdev_boot_setup_check);
-
-/**
- * netdev_boot_base	- get address from boot time settings
- * @prefix: prefix for network device
- * @unit: id for network device
- *
- * Check boot time settings for the base address of device.
- * The found settings are set for the device to be used
- * later in the device probing.
- * Returns 0 if no settings found.
- */
-static unsigned long netdev_boot_base(const char *prefix, int unit)
-{
-	const struct netdev_boot_setup *s = dev_boot_setup;
-	char name[IFNAMSIZ];
-	int i;
-
-	sprintf(name, "%s%d", prefix, unit);
-
-	/*
-	 * If device already registered then return base of 1
-	 * to indicate not to probe for this interface
-	 */
-	if (__dev_get_by_name(&init_net, name))
-		return 1;
-
-	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
-		if (!strcmp(name, s[i].name))
-			return s[i].map.base_addr;
-	return 0;
-}
-
-/*
- * Saves at boot time configured settings for any netdevice.
- */
-static int __init netdev_boot_setup(char *str)
-{
-	int ints[5];
-	struct ifmap map;
-
-	str = get_options(str, ARRAY_SIZE(ints), ints);
-	if (!str || !*str)
-		return 0;
-
-	/* Save settings */
-	memset(&map, 0, sizeof(map));
-	if (ints[0] > 0)
-		map.irq = ints[1];
-	if (ints[0] > 1)
-		map.base_addr = ints[2];
-	if (ints[0] > 2)
-		map.mem_start = ints[3];
-	if (ints[0] > 3)
-		map.mem_end = ints[4];
-
-	/* Add new entry to the list */
-	return netdev_boot_setup_add(str, &map);
-}
-
-__setup("netdev=", netdev_boot_setup);
-
-static int __init ether_boot_setup(char *str)
-{
-	return netdev_boot_setup(str);
-}
-__setup("ether=", ether_boot_setup);
-
-
-/* A unified ethernet device probe.  This is the easiest way to have every
- * ethernet adaptor have the name "eth[0123...]".
- */
-
-struct devprobe2 {
-	struct net_device *(*probe)(int unit);
-	int status;	/* non-zero if autoprobe has failed */
-};
-
-static int __init probe_list2(int unit, struct devprobe2 *p, int autoprobe)
-{
-	struct net_device *dev;
-
-	for (; p->probe; p++) {
-		if (autoprobe && p->status)
-			continue;
-		dev = p->probe(unit);
-		if (!IS_ERR(dev))
-			return 0;
-		if (autoprobe)
-			p->status = PTR_ERR(dev);
-	}
-	return -ENODEV;
-}
-
-/* ISA probes that touch addresses < 0x400 (including those that also
- * look for EISA/PCI cards in addition to ISA cards).
- */
-static struct devprobe2 isa_probes[] __initdata = {
-#if defined(CONFIG_NE2000) /* ISA (use ne2k-pci for PCI cards) */
-	{ne_probe, 0},
-#endif
-#ifdef CONFIG_CS89x0_ISA
-	{cs89x0_probe, 0},
-#endif
-	{NULL, 0},
-};
-
-/* Unified ethernet device probe, segmented per architecture and
- * per bus interface. This drives the legacy devices only for now.
- */
-
-static void __init ethif_probe2(int unit)
-{
-	unsigned long base_addr = netdev_boot_base("eth", unit);
-
-	if (base_addr == 1)
-		return;
-
-	probe_list2(unit, isa_probes, base_addr == 0);
-}
-
-/*  Statically configured drivers -- order matters here. */
-static int __init net_olddevs_init(void)
-{
-	int num;
-
-	for (num = 0; num < 8; ++num)
-		ethif_probe2(num);
-
-	return 0;
-}
-
-device_initcall(net_olddevs_init);
diff --git a/drivers/net/amt.c b/drivers/net/amt.c
index f2f3139e38a59..724a8163a5142 100644
--- a/drivers/net/amt.c
+++ b/drivers/net/amt.c
@@ -614,24 +614,24 @@ static void amt_send_discovery(struct amt_dev *amt)
 {
 	struct amt_header_discovery *amtd;
 	int hlen, tlen, offset;
-	struct socket *sock;
 	struct udphdr *udph;
 	struct sk_buff *skb;
 	struct iphdr *iph;
 	struct rtable *rt;
 	struct flowi4 fl4;
+	struct sock *sk;
 	u32 len;
 	int err;
 
 	rcu_read_lock();
-	sock = rcu_dereference(amt->sock);
-	if (!sock)
+	sk = rcu_dereference(amt->sk);
+	if (!sk)
 		goto out;
 
 	if (!netif_running(amt->stream_dev) || !netif_running(amt->dev))
 		goto out;
 
-	rt = ip_route_output_ports(amt->net, &fl4, sock->sk,
+	rt = ip_route_output_ports(amt->net, &fl4, sk,
 				   amt->discovery_ip, amt->local_ip,
 				   amt->gw_port, amt->relay_port,
 				   IPPROTO_UDP, 0,
@@ -690,7 +690,7 @@ static void amt_send_discovery(struct amt_dev *amt)
 	skb->ip_summed = CHECKSUM_NONE;
 	ip_select_ident(amt->net, skb, NULL);
 	ip_send_check(iph);
-	err = ip_local_out(amt->net, sock->sk, skb);
+	err = ip_local_out(amt->net, sk, skb);
 	if (unlikely(net_xmit_eval(err)))
 		amt->dev->stats.tx_errors++;
 
@@ -703,24 +703,24 @@ static void amt_send_request(struct amt_dev *amt, bool v6)
 {
 	struct amt_header_request *amtrh;
 	int hlen, tlen, offset;
-	struct socket *sock;
 	struct udphdr *udph;
 	struct sk_buff *skb;
 	struct iphdr *iph;
 	struct rtable *rt;
 	struct flowi4 fl4;
+	struct sock *sk;
 	u32 len;
 	int err;
 
 	rcu_read_lock();
-	sock = rcu_dereference(amt->sock);
-	if (!sock)
+	sk = rcu_dereference(amt->sk);
+	if (!sk)
 		goto out;
 
 	if (!netif_running(amt->stream_dev) || !netif_running(amt->dev))
 		goto out;
 
-	rt = ip_route_output_ports(amt->net, &fl4, sock->sk,
+	rt = ip_route_output_ports(amt->net, &fl4, sk,
 				   amt->remote_ip, amt->local_ip,
 				   amt->gw_port, amt->relay_port,
 				   IPPROTO_UDP, 0,
@@ -781,7 +781,7 @@ static void amt_send_request(struct amt_dev *amt, bool v6)
 	skb->ip_summed = CHECKSUM_NONE;
 	ip_select_ident(amt->net, skb, NULL);
 	ip_send_check(iph);
-	err = ip_local_out(amt->net, sock->sk, skb);
+	err = ip_local_out(amt->net, sk, skb);
 	if (unlikely(net_xmit_eval(err)))
 		amt->dev->stats.tx_errors++;
 
@@ -1000,14 +1000,14 @@ static bool amt_send_membership_update(struct amt_dev *amt,
 				       bool v6)
 {
 	struct amt_header_membership_update *amtmu;
-	struct socket *sock;
 	struct iphdr *iph;
 	struct flowi4 fl4;
 	struct rtable *rt;
+	struct sock *sk;
 	int err;
 
-	sock = rcu_dereference_bh(amt->sock);
-	if (!sock)
+	sk = rcu_dereference_bh(amt->sk);
+	if (!sk)
 		return true;
 
 	err = skb_cow_head(skb, LL_RESERVED_SPACE(amt->dev) + sizeof(*amtmu) +
@@ -1039,7 +1039,7 @@ static bool amt_send_membership_update(struct amt_dev *amt,
 		skb_set_inner_protocol(skb, htons(ETH_P_IP));
 	else
 		skb_set_inner_protocol(skb, htons(ETH_P_IPV6));
-	udp_tunnel_xmit_skb(rt, sock->sk, skb,
+	udp_tunnel_xmit_skb(rt, sk, skb,
 			    fl4.saddr,
 			    fl4.daddr,
 			    AMT_TOS,
@@ -1060,14 +1060,14 @@ static void amt_send_multicast_data(struct amt_dev *amt,
 				    bool v6)
 {
 	struct amt_header_mcast_data *amtmd;
-	struct socket *sock;
 	struct sk_buff *skb;
 	struct iphdr *iph;
 	struct flowi4 fl4;
 	struct rtable *rt;
+	struct sock *sk;
 
-	sock = rcu_dereference_bh(amt->sock);
-	if (!sock)
+	sk = rcu_dereference_bh(amt->sk);
+	if (!sk)
 		return;
 
 	skb = skb_copy_expand(oskb, sizeof(*amtmd) + sizeof(*iph) +
@@ -1097,7 +1097,7 @@ static void amt_send_multicast_data(struct amt_dev *amt,
 		skb_set_inner_protocol(skb, htons(ETH_P_IP));
 	else
 		skb_set_inner_protocol(skb, htons(ETH_P_IPV6));
-	udp_tunnel_xmit_skb(rt, sock->sk, skb,
+	udp_tunnel_xmit_skb(rt, sk, skb,
 			    fl4.saddr,
 			    fl4.daddr,
 			    AMT_TOS,
@@ -1116,13 +1116,13 @@ static bool amt_send_membership_query(struct amt_dev *amt,
 				      bool v6)
 {
 	struct amt_header_membership_query *amtmq;
-	struct socket *sock;
 	struct rtable *rt;
 	struct flowi4 fl4;
+	struct sock *sk;
 	int err;
 
-	sock = rcu_dereference_bh(amt->sock);
-	if (!sock)
+	sk = rcu_dereference_bh(amt->sk);
+	if (!sk)
 		return true;
 
 	err = skb_cow_head(skb, LL_RESERVED_SPACE(amt->dev) + sizeof(*amtmq) +
@@ -1156,7 +1156,7 @@ static bool amt_send_membership_query(struct amt_dev *amt,
 		skb_set_inner_protocol(skb, htons(ETH_P_IP));
 	else
 		skb_set_inner_protocol(skb, htons(ETH_P_IPV6));
-	udp_tunnel_xmit_skb(rt, sock->sk, skb,
+	udp_tunnel_xmit_skb(rt, sk, skb,
 			    fl4.saddr,
 			    fl4.daddr,
 			    AMT_TOS,
@@ -2554,24 +2554,24 @@ static void amt_send_advertisement(struct amt_dev *amt, __be32 nonce,
 {
 	struct amt_header_advertisement *amta;
 	int hlen, tlen, offset;
-	struct socket *sock;
 	struct udphdr *udph;
 	struct sk_buff *skb;
 	struct iphdr *iph;
 	struct rtable *rt;
 	struct flowi4 fl4;
+	struct sock *sk;
 	u32 len;
 	int err;
 
 	rcu_read_lock();
-	sock = rcu_dereference(amt->sock);
-	if (!sock)
+	sk = rcu_dereference(amt->sk);
+	if (!sk)
 		goto out;
 
 	if (!netif_running(amt->stream_dev) || !netif_running(amt->dev))
 		goto out;
 
-	rt = ip_route_output_ports(amt->net, &fl4, sock->sk,
+	rt = ip_route_output_ports(amt->net, &fl4, sk,
 				   daddr, amt->local_ip,
 				   dport, amt->relay_port,
 				   IPPROTO_UDP, 0,
@@ -2631,7 +2631,7 @@ static void amt_send_advertisement(struct amt_dev *amt, __be32 nonce,
 	skb->ip_summed = CHECKSUM_NONE;
 	ip_select_ident(amt->net, skb, NULL);
 	ip_send_check(iph);
-	err = ip_local_out(amt->net, sock->sk, skb);
+	err = ip_local_out(amt->net, sk, skb);
 	if (unlikely(net_xmit_eval(err)))
 		amt->dev->stats.tx_errors++;
 
@@ -2944,7 +2944,7 @@ static int amt_err_lookup(struct sock *sk, struct sk_buff *skb)
 	return 0;
 }
 
-static struct socket *amt_create_sock(struct net *net, __be16 port)
+static struct sock *amt_create_sock(struct net *net, __be16 port)
 {
 	struct udp_port_cfg udp_conf;
 	struct socket *sock;
@@ -2960,17 +2960,17 @@ static struct socket *amt_create_sock(struct net *net, __be16 port)
 	if (err < 0)
 		return ERR_PTR(err);
 
-	return sock;
+	return sock->sk;
 }
 
 static int amt_socket_create(struct amt_dev *amt)
 {
 	struct udp_tunnel_sock_cfg tunnel_cfg;
-	struct socket *sock;
+	struct sock *sk;
 
-	sock = amt_create_sock(amt->net, amt->relay_port);
-	if (IS_ERR(sock))
-		return PTR_ERR(sock);
+	sk = amt_create_sock(amt->net, amt->relay_port);
+	if (IS_ERR(sk))
+		return PTR_ERR(sk);
 
 	/* Mark socket as an encapsulation socket */
 	memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
@@ -2979,9 +2979,9 @@ static int amt_socket_create(struct amt_dev *amt)
 	tunnel_cfg.encap_rcv = amt_rcv;
 	tunnel_cfg.encap_err_lookup = amt_err_lookup;
 	tunnel_cfg.encap_destroy = NULL;
-	setup_udp_tunnel_sock(amt->net, sock, &tunnel_cfg);
+	setup_udp_tunnel_sock(amt->net, sk, &tunnel_cfg);
 
-	rcu_assign_pointer(amt->sock, sock);
+	rcu_assign_pointer(amt->sk, sk);
 	return 0;
 }
 
@@ -3019,8 +3019,8 @@ static int amt_dev_stop(struct net_device *dev)
 {
 	struct amt_dev *amt = netdev_priv(dev);
 	struct amt_tunnel_list *tunnel, *tmp;
-	struct socket *sock;
 	struct sk_buff *skb;
+	struct sock *sk;
 	int i;
 
 	cancel_delayed_work_sync(&amt->req_wq);
@@ -3028,11 +3028,11 @@ static int amt_dev_stop(struct net_device *dev)
 	cancel_delayed_work_sync(&amt->secret_wq);
 
 	/* shutdown */
-	sock = rtnl_dereference(amt->sock);
-	RCU_INIT_POINTER(amt->sock, NULL);
+	sk = rtnl_dereference(amt->sk);
+	RCU_INIT_POINTER(amt->sk, NULL);
 	synchronize_net();
-	if (sock)
-		udp_tunnel_sock_release(sock);
+	if (sk)
+		udp_tunnel_sock_release(sk);
 
 	cancel_work_sync(&amt->event_wq);
 	for (i = 0; i < AMT_MAX_EVENTS; i++) {
diff --git a/drivers/net/arcnet/com20020-pci.c b/drivers/net/arcnet/com20020-pci.c
index dbadda08dce23..6474c7be29925 100644
--- a/drivers/net/arcnet/com20020-pci.c
+++ b/drivers/net/arcnet/com20020-pci.c
@@ -459,168 +459,88 @@ static struct com20020_pci_card_info card_info_eae_fb2 = {
 
 static const struct pci_device_id com20020pci_id_table[] = {
 	{
-		0x1571, 0xa001,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		0,
-	},
-	{
-		0x1571, 0xa002,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		0,
-	},
-	{
-		0x1571, 0xa003,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		0
-	},
-	{
-		0x1571, 0xa004,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		0,
-	},
-	{
-		0x1571, 0xa005,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		0
-	},
-	{
-		0x1571, 0xa006,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		0
-	},
-	{
-		0x1571, 0xa007,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		0
-	},
-	{
-		0x1571, 0xa008,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		0
-	},
-	{
-		0x1571, 0xa009,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_5mbit
-	},
-	{
-		0x1571, 0xa00a,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_5mbit
-	},
-	{
-		0x1571, 0xa00b,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_5mbit
-	},
-	{
-		0x1571, 0xa00c,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_5mbit
-	},
-	{
-		0x1571, 0xa00d,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_5mbit
-	},
-	{
-		0x1571, 0xa00e,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_5mbit
-	},
-	{
-		0x1571, 0xa201,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_10mbit
-	},
-	{
-		0x1571, 0xa202,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_10mbit
-	},
-	{
-		0x1571, 0xa203,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_10mbit
-	},
-	{
-		0x1571, 0xa204,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_10mbit
-	},
-	{
-		0x1571, 0xa205,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_10mbit
-	},
-	{
-		0x1571, 0xa206,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_10mbit
-	},
-	{
-		0x10B5, 0x9030,
-		0x10B5, 0x2978,
-		0, 0,
-		(kernel_ulong_t)&card_info_sohard
-	},
-	{
-		0x10B5, 0x9050,
-		0x10B5, 0x2273,
-		0, 0,
-		(kernel_ulong_t)&card_info_sohard
-	},
-	{
-		0x10B5, 0x9050,
-		0x10B5, 0x3263,
-		0, 0,
-		(kernel_ulong_t)&card_info_eae_arc1
-	},
-	{
-		0x10B5, 0x9050,
-		0x10B5, 0x3292,
-		0, 0,
-		(kernel_ulong_t)&card_info_eae_ma1
-	},
-	{
-		0x10B5, 0x9050,
-		0x10B5, 0x3294,
-		0, 0,
-		(kernel_ulong_t)&card_info_eae_fb2
-	},
-	{
-		0x14BA, 0x6000,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_10mbit
-	},
-	{
-		0x10B5, 0x2200,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&card_info_10mbit
-	},
-	{ 0, }
+		PCI_DEVICE(0x1571, 0xa001),
+		.driver_data = 0,
+	}, {
+		PCI_DEVICE(0x1571, 0xa002),
+		.driver_data = 0,
+	}, {
+		PCI_DEVICE(0x1571, 0xa003),
+		.driver_data = 0,
+	}, {
+		PCI_DEVICE(0x1571, 0xa004),
+		.driver_data = 0,
+	}, {
+		PCI_DEVICE(0x1571, 0xa005),
+		.driver_data = 0,
+	}, {
+		PCI_DEVICE(0x1571, 0xa006),
+		.driver_data = 0,
+	}, {
+		PCI_DEVICE(0x1571, 0xa007),
+		.driver_data = 0,
+	}, {
+		PCI_DEVICE(0x1571, 0xa008),
+		.driver_data = 0,
+	}, {
+		PCI_DEVICE(0x1571, 0xa009),
+		.driver_data = (kernel_ulong_t)&card_info_5mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa00a),
+		.driver_data = (kernel_ulong_t)&card_info_5mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa00b),
+		.driver_data = (kernel_ulong_t)&card_info_5mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa00c),
+		.driver_data = (kernel_ulong_t)&card_info_5mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa00d),
+		.driver_data = (kernel_ulong_t)&card_info_5mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa00e),
+		.driver_data = (kernel_ulong_t)&card_info_5mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa201),
+		.driver_data = (kernel_ulong_t)&card_info_10mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa202),
+		.driver_data = (kernel_ulong_t)&card_info_10mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa203),
+		.driver_data = (kernel_ulong_t)&card_info_10mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa204),
+		.driver_data = (kernel_ulong_t)&card_info_10mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa205),
+		.driver_data = (kernel_ulong_t)&card_info_10mbit,
+	}, {
+		PCI_DEVICE(0x1571, 0xa206),
+		.driver_data = (kernel_ulong_t)&card_info_10mbit,
+	}, {
+		PCI_DEVICE_SUB(0x10B5, 0x9030, 0x10B5, 0x2978),
+		.driver_data = (kernel_ulong_t)&card_info_sohard,
+	}, {
+		PCI_DEVICE_SUB(0x10B5, 0x9050, 0x10B5, 0x2273),
+		.driver_data = (kernel_ulong_t)&card_info_sohard,
+	}, {
+		PCI_DEVICE_SUB(0x10B5, 0x9050, 0x10B5, 0x3263),
+		.driver_data = (kernel_ulong_t)&card_info_eae_arc1,
+	}, {
+		PCI_DEVICE_SUB(0x10B5, 0x9050, 0x10B5, 0x3292),
+		.driver_data = (kernel_ulong_t)&card_info_eae_ma1,
+	}, {
+		PCI_DEVICE_SUB(0x10B5, 0x9050, 0x10B5, 0x3294),
+		.driver_data = (kernel_ulong_t)&card_info_eae_fb2,
+	}, {
+		PCI_DEVICE(0x14BA, 0x6000),
+		.driver_data = (kernel_ulong_t)&card_info_10mbit,
+	}, {
+		PCI_DEVICE(0x10B5, 0x2200),
+		.driver_data = (kernel_ulong_t)&card_info_10mbit,
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, com20020pci_id_table);
diff --git a/drivers/net/bareudp.c b/drivers/net/bareudp.c
index da5866ba06999..5ef841c855269 100644
--- a/drivers/net/bareudp.c
+++ b/drivers/net/bareudp.c
@@ -53,7 +53,7 @@ struct bareudp_dev {
 	__be16             port;
 	u16	           sport_min;
 	bool               multi_proto_mode;
-	struct socket      __rcu *sock;
+	struct sock        __rcu *sk;
 	struct list_head   next;        /* bareudp node  on namespace list */
 	struct gro_cells   gro_cells;
 };
@@ -228,7 +228,7 @@ static void bareudp_uninit(struct net_device *dev)
 	gro_cells_destroy(&bareudp->gro_cells);
 }
 
-static struct socket *bareudp_create_sock(struct net *net, __be16 port)
+static struct sock *bareudp_create_sock(struct net *net, __be16 port)
 {
 	struct udp_port_cfg udp_conf;
 	struct socket *sock;
@@ -248,18 +248,18 @@ static struct socket *bareudp_create_sock(struct net *net, __be16 port)
 		return ERR_PTR(err);
 
 	udp_allow_gso(sock->sk);
-	return sock;
+	return sock->sk;
 }
 
 /* Create new listen socket if needed */
 static int bareudp_socket_create(struct bareudp_dev *bareudp, __be16 port)
 {
 	struct udp_tunnel_sock_cfg tunnel_cfg;
-	struct socket *sock;
+	struct sock *sk;
 
-	sock = bareudp_create_sock(bareudp->net, port);
-	if (IS_ERR(sock))
-		return PTR_ERR(sock);
+	sk = bareudp_create_sock(bareudp->net, port);
+	if (IS_ERR(sk))
+		return PTR_ERR(sk);
 
 	/* Mark socket as an encapsulation socket */
 	memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
@@ -268,29 +268,26 @@ static int bareudp_socket_create(struct bareudp_dev *bareudp, __be16 port)
 	tunnel_cfg.encap_rcv = bareudp_udp_encap_recv;
 	tunnel_cfg.encap_err_lookup = bareudp_err_lookup;
 	tunnel_cfg.encap_destroy = NULL;
-	setup_udp_tunnel_sock(bareudp->net, sock, &tunnel_cfg);
+	setup_udp_tunnel_sock(bareudp->net, sk, &tunnel_cfg);
 
-	rcu_assign_pointer(bareudp->sock, sock);
+	rcu_assign_pointer(bareudp->sk, sk);
 	return 0;
 }
 
 static int bareudp_open(struct net_device *dev)
 {
 	struct bareudp_dev *bareudp = netdev_priv(dev);
-	int ret = 0;
 
-	ret =  bareudp_socket_create(bareudp, bareudp->port);
-	return ret;
+	return bareudp_socket_create(bareudp, bareudp->port);
 }
 
 static void bareudp_sock_release(struct bareudp_dev *bareudp)
 {
-	struct socket *sock;
+	struct sock *sk;
 
-	sock = bareudp->sock;
-	rcu_assign_pointer(bareudp->sock, NULL);
-	synchronize_net();
-	udp_tunnel_sock_release(sock);
+	sk = rtnl_dereference(bareudp->sk);
+	rcu_assign_pointer(bareudp->sk, NULL);
+	udp_tunnel_sock_release(sk);
 }
 
 static int bareudp_stop(struct net_device *dev)
@@ -308,7 +305,7 @@ static int bareudp_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 	bool udp_sum = test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
 	bool xnet = !net_eq(bareudp->net, dev_net(bareudp->dev));
 	bool use_cache = ip_tunnel_dst_cache_usable(skb, info);
-	struct socket *sock = rcu_dereference(bareudp->sock);
+	struct sock *sk = rcu_dereference(bareudp->sk);
 	const struct ip_tunnel_key *key = &info->key;
 	struct rtable *rt;
 	__be16 sport, df;
@@ -320,7 +317,7 @@ static int bareudp_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 	if (skb_vlan_inet_prepare(skb, skb->protocol != htons(ETH_P_TEB)))
 		return -EINVAL;
 
-	if (!sock)
+	if (!sk)
 		return -ESHUTDOWN;
 
 	sport = udp_flow_src_port(bareudp->net, skb,
@@ -359,7 +356,7 @@ static int bareudp_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 		goto free_dst;
 
 	skb_set_inner_protocol(skb, bareudp->ethertype);
-	udp_tunnel_xmit_skb(rt, sock->sk, skb, saddr, info->key.u.ipv4.dst,
+	udp_tunnel_xmit_skb(rt, sk, skb, saddr, info->key.u.ipv4.dst,
 			    tos, ttl, df, sport, bareudp->port,
 			    !net_eq(bareudp->net, dev_net(bareudp->dev)),
 			    !test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags),
@@ -378,7 +375,7 @@ static int bareudp6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 	bool udp_sum = test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags);
 	bool xnet = !net_eq(bareudp->net, dev_net(bareudp->dev));
 	bool use_cache = ip_tunnel_dst_cache_usable(skb, info);
-	struct socket *sock  = rcu_dereference(bareudp->sock);
+	struct sock *sk = rcu_dereference(bareudp->sk);
 	const struct ip_tunnel_key *key = &info->key;
 	struct dst_entry *dst = NULL;
 	struct in6_addr saddr, daddr;
@@ -390,13 +387,13 @@ static int bareudp6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 	if (skb_vlan_inet_prepare(skb, skb->protocol != htons(ETH_P_TEB)))
 		return -EINVAL;
 
-	if (!sock)
+	if (!sk)
 		return -ESHUTDOWN;
 
 	sport = udp_flow_src_port(bareudp->net, skb,
 				  bareudp->sport_min, USHRT_MAX,
 				  true);
-	dst = udp_tunnel6_dst_lookup(skb, dev, bareudp->net, sock, 0, &saddr,
+	dst = udp_tunnel6_dst_lookup(skb, dev, bareudp->net, sk, 0, &saddr,
 				     key, sport, bareudp->port, key->tos,
 				     use_cache ?
 				     (struct dst_cache *) &info->dst_cache : NULL);
@@ -427,7 +424,7 @@ static int bareudp6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 		goto free_dst;
 
 	daddr = info->key.u.ipv6.dst;
-	udp_tunnel6_xmit_skb(dst, sock->sk, skb, dev,
+	udp_tunnel6_xmit_skb(dst, sk, skb, dev,
 			     &saddr, &daddr, prio, ttl,
 			     info->key.label, sport, bareudp->port,
 			     !test_bit(IP_TUNNEL_CSUM_BIT,
@@ -527,12 +524,13 @@ static int bareudp_fill_metadata_dst(struct net_device *dev,
 	} else if (ip_tunnel_info_af(info) == AF_INET6) {
 		struct dst_entry *dst;
 		struct in6_addr saddr;
-		struct socket *sock = rcu_dereference(bareudp->sock);
+		struct sock *sk;
 
-		if (!sock)
+		sk = rcu_dereference(bareudp->sk);
+		if (!sk)
 			return -ESHUTDOWN;
 
-		dst = udp_tunnel6_dst_lookup(skb, dev, bareudp->net, sock,
+		dst = udp_tunnel6_dst_lookup(skb, dev, bareudp->net, sk,
 					     0, &saddr, &info->key,
 					     sport, bareudp->port, info->key.tos,
 					     use_cache ? &info->dst_cache : NULL);
diff --git a/drivers/net/can/sja1000/plx_pci.c b/drivers/net/can/sja1000/plx_pci.c
index 08183833c9bc7..a03553b80a5dc 100644
--- a/drivers/net/can/sja1000/plx_pci.c
+++ b/drivers/net/can/sja1000/plx_pci.c
@@ -272,124 +272,89 @@ static struct plx_pci_card_info plx_pci_card_info_asem_dual_can = {
 static const struct pci_device_id plx_pci_tbl[] = {
 	{
 		/* Adlink PCI-7841/cPCI-7841 */
-		ADLINK_PCI_VENDOR_ID, ADLINK_PCI_DEVICE_ID,
-		PCI_ANY_ID, PCI_ANY_ID,
-		PCI_CLASS_NETWORK_OTHER << 8, ~0,
-		(kernel_ulong_t)&plx_pci_card_info_adlink
-	},
-	{
+		PCI_DEVICE(ADLINK_PCI_VENDOR_ID, ADLINK_PCI_DEVICE_ID),
+		.class = PCI_CLASS_NETWORK_OTHER << 8,
+		.class_mask = ~0,
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_adlink,
+	}, {
 		/* Adlink PCI-7841/cPCI-7841 SE */
-		ADLINK_PCI_VENDOR_ID, ADLINK_PCI_DEVICE_ID,
-		PCI_ANY_ID, PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_OTHER << 8, ~0,
-		(kernel_ulong_t)&plx_pci_card_info_adlink_se
-	},
-	{
+		PCI_DEVICE(ADLINK_PCI_VENDOR_ID, ADLINK_PCI_DEVICE_ID),
+		.class = PCI_CLASS_COMMUNICATION_OTHER << 8,
+		.class_mask = ~0,
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_adlink_se,
+	}, {
 		/* esd CAN-PCI/200 */
-		PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PCI200,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_esd200
-	},
-	{
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PCI200),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_esd200,
+	}, {
 		/* esd CAN-CPCI/200 */
-		PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9030,
-		PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_CPCI200,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_esd200
-	},
-	{
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9030,
+				PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_CPCI200),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_esd200,
+	}, {
 		/* esd CAN-PCI104/200 */
-		PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9030,
-		PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PCI104200,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_esd200
-	},
-	{
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9030,
+				PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PCI104200),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_esd200,
+	}, {
 		/* esd CAN-PCI/266 */
-		PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9056,
-		PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PCI266,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_esd266
-	},
-	{
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9056,
+				PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PCI266),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_esd266,
+	}, {
 		/* esd CAN-PMC/266 */
-		PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9056,
-		PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PMC266,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_esd266
-	},
-	{
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9056,
+				PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PMC266),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_esd266,
+	}, {
 		/* esd CAN-PCIE/2000 */
-		PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9056,
-		PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PCIE2000,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_esd2000
-	},
-	{
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9056,
+				PCI_VENDOR_ID_ESDGMBH, ESD_PCI_SUB_SYS_ID_PCIE2000),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_esd2000,
+	}, {
 		/* IXXAT PC-I 04/PCI card */
-		IXXAT_PCI_VENDOR_ID, IXXAT_PCI_DEVICE_ID,
-		PCI_ANY_ID, IXXAT_PCI_SUB_SYS_ID,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_ixxat
-	},
-	{
+		PCI_DEVICE_SUB(IXXAT_PCI_VENDOR_ID, IXXAT_PCI_DEVICE_ID,
+			       PCI_ANY_ID, IXXAT_PCI_SUB_SYS_ID),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_ixxat,
+	}, {
 		/* Marathon CAN-bus-PCI card */
-		PCI_VENDOR_ID_PLX, MARATHON_PCI_DEVICE_ID,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_marathon_pci
-	},
-	{
+		PCI_VDEVICE(PLX, MARATHON_PCI_DEVICE_ID),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_marathon_pci,
+	}, {
 		/* Marathon CAN-bus-PCIe card */
-		PCI_VENDOR_ID_PLX, MARATHON_PCIE_DEVICE_ID,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_marathon_pcie
-	},
-	{
+		PCI_VDEVICE(PLX, MARATHON_PCIE_DEVICE_ID),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_marathon_pcie,
+	}, {
 		/* TEWS TECHNOLOGIES TPMC810 card */
-		TEWS_PCI_VENDOR_ID, TEWS_PCI_DEVICE_ID_TMPC810,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_tews
-	},
-	{
+		PCI_DEVICE(TEWS_PCI_VENDOR_ID, TEWS_PCI_DEVICE_ID_TMPC810),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_tews,
+	}, {
 		/* Connect Tech Inc. CANpro/104-Plus Opto (CRG001) card */
-		PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9030,
-		PCI_SUBVENDOR_ID_CONNECT_TECH, CTI_PCI_DEVICE_ID_CRG001,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_cti
-	},
-	{
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9030,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, CTI_PCI_DEVICE_ID_CRG001),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_cti,
+	}, {
 		/* Elcus CAN-200-PCI */
-		CAN200PCI_VENDOR_ID, CAN200PCI_DEVICE_ID,
-		CAN200PCI_SUB_VENDOR_ID, CAN200PCI_SUB_DEVICE_ID,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_elcus
-	},
-	{
+		PCI_DEVICE_SUB(CAN200PCI_VENDOR_ID, CAN200PCI_DEVICE_ID,
+			       CAN200PCI_SUB_VENDOR_ID, CAN200PCI_SUB_DEVICE_ID),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_elcus,
+	}, {
 		/* moxa */
-		MOXA_PCI_VENDOR_ID, MOXA_PCI_DEVICE_ID,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_moxa
-	},
-	{
+		PCI_DEVICE(MOXA_PCI_VENDOR_ID, MOXA_PCI_DEVICE_ID),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_moxa,
+	}, {
 		/* ASEM Dual CAN raw */
-		ASEM_RAW_CAN_VENDOR_ID, ASEM_RAW_CAN_DEVICE_ID,
-		ASEM_RAW_CAN_SUB_VENDOR_ID, ASEM_RAW_CAN_SUB_DEVICE_ID,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_asem_dual_can
-	},
-	{
+		PCI_DEVICE_SUB(ASEM_RAW_CAN_VENDOR_ID, ASEM_RAW_CAN_DEVICE_ID,
+			       ASEM_RAW_CAN_SUB_VENDOR_ID, ASEM_RAW_CAN_SUB_DEVICE_ID),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_asem_dual_can,
+	}, {
 		/* ASEM Dual CAN raw -new model */
-		ASEM_RAW_CAN_VENDOR_ID, ASEM_RAW_CAN_DEVICE_ID,
-		ASEM_RAW_CAN_SUB_VENDOR_ID, ASEM_RAW_CAN_SUB_DEVICE_ID_BIS,
-		0, 0,
-		(kernel_ulong_t)&plx_pci_card_info_asem_dual_can
+		PCI_DEVICE_SUB(ASEM_RAW_CAN_VENDOR_ID, ASEM_RAW_CAN_DEVICE_ID,
+			       ASEM_RAW_CAN_SUB_VENDOR_ID, ASEM_RAW_CAN_SUB_DEVICE_ID_BIS),
+		.driver_data = (kernel_ulong_t)&plx_pci_card_info_asem_dual_can,
 	},
-	{ 0,}
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, plx_pci_tbl);
 
diff --git a/drivers/net/dsa/lan9303_i2c.c b/drivers/net/dsa/lan9303_i2c.c
index c62d27cdc117d..3a09f3907f441 100644
--- a/drivers/net/dsa/lan9303_i2c.c
+++ b/drivers/net/dsa/lan9303_i2c.c
@@ -89,7 +89,7 @@ static void lan9303_i2c_shutdown(struct i2c_client *client)
 /*-------------------------------------------------------------------------*/
 
 static const struct i2c_device_id lan9303_i2c_id[] = {
-	{ "lan9303" },
+	{ .name = "lan9303" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, lan9303_i2c_id);
diff --git a/drivers/net/dsa/microchip/ksz8.c b/drivers/net/dsa/microchip/ksz8.c
index c354abdafc1b5..825133e9ce6e6 100644
--- a/drivers/net/dsa/microchip/ksz8.c
+++ b/drivers/net/dsa/microchip/ksz8.c
@@ -16,6 +16,7 @@
 
 #include <linux/bitfield.h>
 #include <linux/delay.h>
+#include <linux/dsa/ksz_common.h>
 #include <linux/export.h>
 #include <linux/gpio.h>
 #include <linux/if_vlan.h>
@@ -31,6 +32,7 @@
 #include <linux/phylink.h>
 
 #include "ksz_common.h"
+#include "ksz_dcb.h"
 #include "ksz8_reg.h"
 #include "ksz8.h"
 
@@ -114,26 +116,26 @@ static int ksz8_ind_read8(struct ksz_device *dev, u8 table, u16 addr, u8 *val)
 	return ret;
 }
 
-int ksz8_pme_write8(struct ksz_device *dev, u32 reg, u8 value)
+static int ksz8_pme_write8(struct ksz_device *dev, u32 reg, u8 value)
 {
 	return ksz8_ind_write8(dev, (u8)(reg >> 8), (u8)(reg), value);
 }
 
-int ksz8_pme_pread8(struct ksz_device *dev, int port, int offset, u8 *data)
+static int ksz8_pme_pread8(struct ksz_device *dev, int port, int offset, u8 *data)
 {
 	u8 table = (u8)(offset >> 8 | (port + 1));
 
 	return ksz8_ind_read8(dev, table, (u8)(offset), data);
 }
 
-int ksz8_pme_pwrite8(struct ksz_device *dev, int port, int offset, u8 data)
+static int ksz8_pme_pwrite8(struct ksz_device *dev, int port, int offset, u8 data)
 {
 	u8 table = (u8)(offset >> 8 | (port + 1));
 
 	return ksz8_ind_write8(dev, table, (u8)(offset), data);
 }
 
-int ksz8_reset_switch(struct ksz_device *dev)
+static int ksz8_reset_switch(struct ksz_device *dev)
 {
 	if (ksz_is_ksz88x3(dev)) {
 		/* reset switch */
@@ -186,8 +188,9 @@ static int ksz8795_change_mtu(struct ksz_device *dev, int frame_size)
 	return ksz_rmw8(dev, REG_SW_CTRL_2, SW_LEGAL_PACKET_DISABLE, ctrl2);
 }
 
-int ksz8_change_mtu(struct ksz_device *dev, int port, int mtu)
+static int ksz8_change_mtu(struct dsa_switch *ds, int port, int mtu)
 {
+	struct ksz_device *dev = ds->priv;
 	u16 frame_size;
 
 	if (!dsa_is_cpu_port(dev->ds, port))
@@ -267,22 +270,7 @@ static int ksz8_port_queue_split(struct ksz_device *dev, int port, int queues)
 	return ksz_prmw8(dev, port, reg_2q, mask_2q, data_2q);
 }
 
-int ksz8_all_queues_split(struct ksz_device *dev, int queues)
-{
-	struct dsa_switch *ds = dev->ds;
-	const struct dsa_port *dp;
-
-	dsa_switch_for_each_port(dp, ds) {
-		int ret = ksz8_port_queue_split(dev, dp->index, queues);
-
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-
-void ksz8_r_mib_cnt(struct ksz_device *dev, int port, u16 addr, u64 *cnt)
+static void ksz8_r_mib_cnt(struct ksz_device *dev, int port, u16 addr, u64 *cnt)
 {
 	const u32 *masks;
 	const u16 *regs;
@@ -405,8 +393,8 @@ static void ksz8863_r_mib_pkt(struct ksz_device *dev, int port, u16 addr,
 	}
 }
 
-void ksz8_r_mib_pkt(struct ksz_device *dev, int port, u16 addr,
-		    u64 *dropped, u64 *cnt)
+static void ksz8_r_mib_pkt(struct ksz_device *dev, int port, u16 addr,
+			   u64 *dropped, u64 *cnt)
 {
 	if (is_ksz88xx(dev))
 		ksz8863_r_mib_pkt(dev, port, addr, dropped, cnt);
@@ -414,7 +402,7 @@ void ksz8_r_mib_pkt(struct ksz_device *dev, int port, u16 addr,
 		ksz8795_r_mib_pkt(dev, port, addr, dropped, cnt);
 }
 
-void ksz8_freeze_mib(struct ksz_device *dev, int port, bool freeze)
+static void ksz8_freeze_mib(struct ksz_device *dev, int port, bool freeze)
 {
 	if (is_ksz88xx(dev))
 		return;
@@ -429,7 +417,7 @@ void ksz8_freeze_mib(struct ksz_device *dev, int port, bool freeze)
 		ksz_cfg(dev, REG_SW_CTRL_6, BIT(port), false);
 }
 
-void ksz8_port_init_cnt(struct ksz_device *dev, int port)
+static void ksz8_port_init_cnt(struct ksz_device *dev, int port)
 {
 	struct ksz_port_mib *mib = &dev->ports[port].mib;
 	u64 *dropped;
@@ -954,7 +942,7 @@ static int ksz8_r_phy_bmcr(struct ksz_device *dev, u16 port, u16 *val)
 	return 0;
 }
 
-int ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
+static int ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 {
 	u8 ctrl, link, val1, val2;
 	int processed = true;
@@ -1218,7 +1206,7 @@ static int ksz8_w_phy_bmcr(struct ksz_device *dev, u16 port, u16 val)
 			 restart);
 }
 
-int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
+static int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 {
 	const u16 *regs;
 	u8 ctrl, data;
@@ -1278,7 +1266,7 @@ int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 	return 0;
 }
 
-void ksz8_cfg_port_member(struct ksz_device *dev, int port, u8 member)
+static void ksz8_cfg_port_member(struct ksz_device *dev, int port, u8 member)
 {
 	int offset = P_MIRROR_CTRL;
 	u8 data;
@@ -1291,8 +1279,9 @@ void ksz8_cfg_port_member(struct ksz_device *dev, int port, u8 member)
 	ksz_pwrite8(dev, port, offset, data);
 }
 
-void ksz8_flush_dyn_mac_table(struct ksz_device *dev, int port)
+static void ksz8_flush_dyn_mac_table(struct dsa_switch *ds, int port)
 {
+	struct ksz_device *dev = ds->priv;
 	u8 learn[DSA_MAX_PORTS];
 	int first, index, cnt;
 	const u16 *regs;
@@ -1326,9 +1315,10 @@ void ksz8_flush_dyn_mac_table(struct ksz_device *dev, int port)
 	}
 }
 
-int ksz8_fdb_dump(struct ksz_device *dev, int port,
-		  dsa_fdb_dump_cb_t *cb, void *data)
+static int ksz8_fdb_dump(struct dsa_switch *ds, int port,
+			 dsa_fdb_dump_cb_t *cb, void *data)
 {
+	struct ksz_device *dev = ds->priv;
 	u8 mac[ETH_ALEN];
 	u8 src_port, fid;
 	u16 entries = 0;
@@ -1431,33 +1421,37 @@ static int ksz8_del_sta_mac(struct ksz_device *dev, int port,
 	return ksz8_w_sta_mac_table(dev, index, &alu);
 }
 
-int ksz8_mdb_add(struct ksz_device *dev, int port,
-		 const struct switchdev_obj_port_mdb *mdb, struct dsa_db db)
+static int ksz8_mdb_add(struct dsa_switch *ds, int port,
+			const struct switchdev_obj_port_mdb *mdb,
+			struct dsa_db db)
 {
-	return ksz8_add_sta_mac(dev, port, mdb->addr, mdb->vid);
+	return ksz8_add_sta_mac(ds->priv, port, mdb->addr, mdb->vid);
 }
 
-int ksz8_mdb_del(struct ksz_device *dev, int port,
-		 const struct switchdev_obj_port_mdb *mdb, struct dsa_db db)
+static int ksz8_mdb_del(struct dsa_switch *ds, int port,
+			const struct switchdev_obj_port_mdb *mdb,
+			struct dsa_db db)
 {
-	return ksz8_del_sta_mac(dev, port, mdb->addr, mdb->vid);
+	return ksz8_del_sta_mac(ds->priv, port, mdb->addr, mdb->vid);
 }
 
-int ksz8_fdb_add(struct ksz_device *dev, int port, const unsigned char *addr,
-		 u16 vid, struct dsa_db db)
+static int ksz8_fdb_add(struct dsa_switch *ds, int port,
+			const unsigned char *addr, u16 vid, struct dsa_db db)
 {
-	return ksz8_add_sta_mac(dev, port, addr, vid);
+	return ksz8_add_sta_mac(ds->priv, port, addr, vid);
 }
 
-int ksz8_fdb_del(struct ksz_device *dev, int port, const unsigned char *addr,
-		 u16 vid, struct dsa_db db)
+static int ksz8_fdb_del(struct dsa_switch *ds, int port,
+			const unsigned char *addr, u16 vid, struct dsa_db db)
 {
-	return ksz8_del_sta_mac(dev, port, addr, vid);
+	return ksz8_del_sta_mac(ds->priv, port, addr, vid);
 }
 
-int ksz8_port_vlan_filtering(struct ksz_device *dev, int port, bool flag,
-			     struct netlink_ext_ack *extack)
+static int ksz8_port_vlan_filtering(struct dsa_switch *ds, int port, bool flag,
+				    struct netlink_ext_ack *extack)
 {
+	struct ksz_device *dev = ds->priv;
+
 	if (ksz_is_ksz88x3(dev) || ksz_is_ksz8463(dev))
 		return -ENOTSUPP;
 
@@ -1485,11 +1479,12 @@ static void ksz8_port_enable_pvid(struct ksz_device *dev, int port, bool state)
 	}
 }
 
-int ksz8_port_vlan_add(struct ksz_device *dev, int port,
-		       const struct switchdev_obj_port_vlan *vlan,
-		       struct netlink_ext_ack *extack)
+static int ksz8_port_vlan_add(struct dsa_switch *ds, int port,
+			      const struct switchdev_obj_port_vlan *vlan,
+			      struct netlink_ext_ack *extack)
 {
 	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
+	struct ksz_device *dev = ds->priv;
 	struct ksz_port *p = &dev->ports[port];
 	u16 data, new_pvid = 0;
 	u8 fid, member, valid;
@@ -1557,11 +1552,12 @@ int ksz8_port_vlan_add(struct ksz_device *dev, int port,
 	return 0;
 }
 
-int ksz8_port_vlan_del(struct ksz_device *dev, int port,
-		       const struct switchdev_obj_port_vlan *vlan)
+static int ksz8_port_vlan_del(struct dsa_switch *ds, int port,
+			      const struct switchdev_obj_port_vlan *vlan)
 {
-	u16 data, pvid;
+	struct ksz_device *dev = ds->priv;
 	u8 fid, member, valid;
+	u16 data, pvid;
 
 	if (ksz_is_ksz88x3(dev) || ksz_is_ksz8463(dev))
 		return -ENOTSUPP;
@@ -1589,10 +1585,11 @@ int ksz8_port_vlan_del(struct ksz_device *dev, int port,
 	return 0;
 }
 
-int ksz8_port_mirror_add(struct ksz_device *dev, int port,
-			 struct dsa_mall_mirror_tc_entry *mirror,
-			 bool ingress, struct netlink_ext_ack *extack)
+static int ksz8_port_mirror_add(struct dsa_switch *ds, int port,
+				struct dsa_mall_mirror_tc_entry *mirror,
+				bool ingress, struct netlink_ext_ack *extack)
 {
+	struct ksz_device *dev = ds->priv;
 	int offset = P_MIRROR_CTRL;
 
 	if (ksz_is_ksz8463(dev))
@@ -1615,9 +1612,10 @@ int ksz8_port_mirror_add(struct ksz_device *dev, int port,
 	return 0;
 }
 
-void ksz8_port_mirror_del(struct ksz_device *dev, int port,
-			  struct dsa_mall_mirror_tc_entry *mirror)
+static void ksz8_port_mirror_del(struct dsa_switch *ds, int port,
+				 struct dsa_mall_mirror_tc_entry *mirror)
 {
+	struct ksz_device *dev = ds->priv;
 	int offset = P_MIRROR_CTRL;
 	u8 data;
 
@@ -1654,7 +1652,7 @@ static void ksz8795_cpu_interface_select(struct ksz_device *dev, int port)
 	}
 }
 
-void ksz8_port_setup(struct ksz_device *dev, int port, bool cpu_port)
+static void ksz8_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 {
 	const u16 *regs = dev->info->regs;
 	struct dsa_switch *ds = dev->ds;
@@ -1709,7 +1707,7 @@ static void ksz88x3_config_rmii_clk(struct ksz_device *dev)
 		KSZ88X3_PORT3_RMII_CLK_INTERNAL, rmii_clk_internal);
 }
 
-void ksz8_config_cpu_port(struct dsa_switch *ds)
+static void ksz8_config_cpu_port(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 	struct ksz_port *p;
@@ -1883,10 +1881,12 @@ static void ksz8_cpu_port_link_up(struct ksz_device *dev, int speed, int duplex,
 		 SW_10_MBIT, ctrl);
 }
 
-void ksz8_phylink_mac_link_up(struct phylink_config *config,
-			      struct phy_device *phydev, unsigned int mode,
-			      phy_interface_t interface, int speed, int duplex,
-			      bool tx_pause, bool rx_pause)
+static void ksz8_phylink_mac_link_up(struct phylink_config *config,
+				     struct phy_device *phydev,
+				     unsigned int mode,
+				     phy_interface_t interface,
+				     int speed, int duplex,
+				     bool tx_pause, bool rx_pause)
 {
 	struct dsa_port *dp = dsa_phylink_to_port(config);
 	struct ksz_device *dev = dp->ds->priv;
@@ -1918,7 +1918,7 @@ static int ksz8_handle_global_errata(struct dsa_switch *ds)
 	return ret;
 }
 
-int ksz8_enable_stp_addr(struct ksz_device *dev)
+static int ksz8_enable_stp_addr(struct ksz_device *dev)
 {
 	struct alu_struct alu;
 
@@ -1932,7 +1932,7 @@ int ksz8_enable_stp_addr(struct ksz_device *dev)
 	return ksz8_w_sta_mac_table(dev, 0, &alu);
 }
 
-int ksz8_setup(struct dsa_switch *ds)
+static int ksz8_setup(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 	const u16 *regs = dev->info->regs;
@@ -1995,9 +1995,11 @@ int ksz8_setup(struct dsa_switch *ds)
 		return ret;
 }
 
-void ksz8_get_caps(struct ksz_device *dev, int port,
-		   struct phylink_config *config)
+static void ksz8_phylink_get_caps(struct dsa_switch *ds, int port,
+				  struct phylink_config *config)
 {
+	struct ksz_device *dev = ds->priv;
+
 	config->mac_capabilities = MAC_10 | MAC_100;
 
 	/* Silicon Errata Sheet (DS80000830A):
@@ -2011,14 +2013,16 @@ void ksz8_get_caps(struct ksz_device *dev, int port,
 	/* Asym pause is not supported on KSZ8863 and KSZ8873 */
 	if (!ksz_is_ksz88x3(dev))
 		config->mac_capabilities |= MAC_ASYM_PAUSE;
+
+	ksz_phylink_get_caps(ds, port, config);
 }
 
-u32 ksz8_get_port_addr(int port, int offset)
+static u32 ksz8_get_port_addr(int port, int offset)
 {
 	return PORT_CTRL_ADDR(port, offset);
 }
 
-u32 ksz8463_get_port_addr(int port, int offset)
+static u32 ksz8463_get_port_addr(int port, int offset)
 {
 	return offset + 0x18 * port;
 }
@@ -2028,7 +2032,7 @@ static u16 ksz8463_get_phy_addr(u16 phy, u16 reg, u16 offset)
 	return offset + reg * 2 + phy * (P2MBCR - P1MBCR);
 }
 
-int ksz8463_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
+static int ksz8463_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 {
 	u16 sw_reg = 0;
 	u16 data = 0;
@@ -2068,7 +2072,7 @@ int ksz8463_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val)
 	return 0;
 }
 
-int ksz8463_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
+static int ksz8463_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 {
 	u16 sw_reg = 0;
 	int ret;
@@ -2096,7 +2100,7 @@ int ksz8463_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val)
 	return 0;
 }
 
-int ksz8_switch_init(struct ksz_device *dev)
+static int ksz8_switch_init(struct ksz_device *dev)
 {
 	dev->cpu_port = fls(dev->info->cpu_ports) - 1;
 	dev->phy_port_cnt = dev->info->port_cnt - 1;
@@ -2105,11 +2109,350 @@ int ksz8_switch_init(struct ksz_device *dev)
 	return 0;
 }
 
-void ksz8_switch_exit(struct ksz_device *dev)
+static void ksz8_switch_exit(struct ksz_device *dev)
 {
 	ksz8_reset_switch(dev);
 }
 
+static enum dsa_tag_protocol ksz8463_get_tag_protocol(struct dsa_switch *ds,
+						      int port,
+						      enum dsa_tag_protocol mp)
+{
+	return DSA_TAG_PROTO_KSZ9893;
+}
+
+static int ksz8463_connect_tag_protocol(struct dsa_switch *ds,
+					enum dsa_tag_protocol proto)
+{
+	struct ksz_tagger_data *tagger_data;
+
+	if (proto != DSA_TAG_PROTO_KSZ9893)
+		return -EPROTONOSUPPORT;
+
+	tagger_data = ksz_tagger_data(ds);
+	tagger_data->xmit_work_fn = ksz_port_deferred_xmit;
+
+	return 0;
+}
+
+static enum dsa_tag_protocol ksz87xx_get_tag_protocol(struct dsa_switch *ds,
+						      int port,
+						      enum dsa_tag_protocol mp)
+{
+	return DSA_TAG_PROTO_KSZ8795;
+}
+
+static int ksz87xx_connect_tag_protocol(struct dsa_switch *ds,
+					enum dsa_tag_protocol proto)
+{
+	if (proto != DSA_TAG_PROTO_KSZ8795)
+		return -EPROTONOSUPPORT;
+
+	return 0;
+}
+
+static enum dsa_tag_protocol ksz88xx_get_tag_protocol(struct dsa_switch *ds,
+						      int port,
+						      enum dsa_tag_protocol mp)
+{
+	struct ksz_device *dev = ds->priv;
+
+	if (ksz_is_8895_family(dev)) /* KSZ8864, KSZ8895 */
+		return DSA_TAG_PROTO_KSZ8795;
+
+	return DSA_TAG_PROTO_KSZ9893;
+}
+
+static int ksz88xx_connect_tag_protocol(struct dsa_switch *ds,
+					enum dsa_tag_protocol proto)
+{
+	struct ksz_tagger_data *tagger_data;
+
+	if (ksz_is_8895_family(ds->priv)) { /* KSZ8864, KSZ8895 */
+		if (proto != DSA_TAG_PROTO_KSZ8795)
+			return -EPROTONOSUPPORT;
+
+		return 0;
+	}
+
+	if (proto != DSA_TAG_PROTO_KSZ9893)
+		return -EPROTONOSUPPORT;
+
+	tagger_data = ksz_tagger_data(ds);
+	tagger_data->xmit_work_fn = ksz_port_deferred_xmit;
+
+	return 0;
+}
+
+static void ksz88x3_phylink_mac_config(struct phylink_config *config,
+				       unsigned int mode,
+				       const struct phylink_link_state *state)
+{
+	struct dsa_port *dp = dsa_phylink_to_port(config);
+	struct ksz_device *dev = dp->ds->priv;
+
+	dev->ports[dp->index].manual_flow = !(state->pause & MLO_PAUSE_AN);
+}
+
+const struct phylink_mac_ops ksz88x3_phylink_mac_ops = {
+	.mac_config	= ksz88x3_phylink_mac_config,
+	.mac_link_down	= ksz_phylink_mac_link_down,
+	.mac_link_up	= ksz8_phylink_mac_link_up,
+	.mac_disable_tx_lpi = ksz_phylink_mac_disable_tx_lpi,
+	.mac_enable_tx_lpi = ksz_phylink_mac_enable_tx_lpi,
+};
+
+const struct phylink_mac_ops ksz8_phylink_mac_ops = {
+	.mac_config	= ksz_phylink_mac_config,
+	.mac_link_down	= ksz_phylink_mac_link_down,
+	.mac_link_up	= ksz8_phylink_mac_link_up,
+	.mac_disable_tx_lpi = ksz_phylink_mac_disable_tx_lpi,
+	.mac_enable_tx_lpi = ksz_phylink_mac_enable_tx_lpi,
+};
+
+const struct ksz_dev_ops ksz8463_dev_ops = {
+	.setup = ksz8_setup,
+	.get_port_addr = ksz8463_get_port_addr,
+	.cfg_port_member = ksz8_cfg_port_member,
+	.port_setup = ksz8_port_setup,
+	.r_phy = ksz8463_r_phy,
+	.w_phy = ksz8463_w_phy,
+	.r_mib_cnt = ksz8_r_mib_cnt,
+	.r_mib_pkt = ksz8_r_mib_pkt,
+	.r_mib_stat64 = ksz88xx_r_mib_stats64,
+	.freeze_mib = ksz8_freeze_mib,
+	.port_init_cnt = ksz8_port_init_cnt,
+	.config_cpu_port = ksz8_config_cpu_port,
+	.enable_stp_addr = ksz8_enable_stp_addr,
+	.reset = ksz8_reset_switch,
+	.init = ksz8_switch_init,
+	.exit = ksz8_switch_exit,
+};
+
+const struct ksz_dev_ops ksz87xx_dev_ops = {
+	.setup = ksz8_setup,
+	.get_port_addr = ksz8_get_port_addr,
+	.cfg_port_member = ksz8_cfg_port_member,
+	.port_setup = ksz8_port_setup,
+	.r_phy = ksz8_r_phy,
+	.w_phy = ksz8_w_phy,
+	.r_mib_cnt = ksz8_r_mib_cnt,
+	.r_mib_pkt = ksz8_r_mib_pkt,
+	.r_mib_stat64 = ksz_r_mib_stats64,
+	.freeze_mib = ksz8_freeze_mib,
+	.port_init_cnt = ksz8_port_init_cnt,
+	.config_cpu_port = ksz8_config_cpu_port,
+	.enable_stp_addr = ksz8_enable_stp_addr,
+	.reset = ksz8_reset_switch,
+	.init = ksz8_switch_init,
+	.exit = ksz8_switch_exit,
+	.pme_write8 = ksz8_pme_write8,
+	.pme_pread8 = ksz8_pme_pread8,
+	.pme_pwrite8 = ksz8_pme_pwrite8,
+};
+
+const struct ksz_dev_ops ksz88xx_dev_ops = {
+	.setup = ksz8_setup,
+	.get_port_addr = ksz8_get_port_addr,
+	.cfg_port_member = ksz8_cfg_port_member,
+	.port_setup = ksz8_port_setup,
+	.r_phy = ksz8_r_phy,
+	.w_phy = ksz8_w_phy,
+	.r_mib_cnt = ksz8_r_mib_cnt,
+	.r_mib_pkt = ksz8_r_mib_pkt,
+	.r_mib_stat64 = ksz88xx_r_mib_stats64,
+	.freeze_mib = ksz8_freeze_mib,
+	.port_init_cnt = ksz8_port_init_cnt,
+	.config_cpu_port = ksz8_config_cpu_port,
+	.enable_stp_addr = ksz8_enable_stp_addr,
+	.reset = ksz8_reset_switch,
+	.init = ksz8_switch_init,
+	.exit = ksz8_switch_exit,
+	.pme_write8 = ksz8_pme_write8,
+	.pme_pread8 = ksz8_pme_pread8,
+	.pme_pwrite8 = ksz8_pme_pwrite8,
+};
+
+const struct dsa_switch_ops ksz8463_switch_ops = {
+	.get_tag_protocol	= ksz8463_get_tag_protocol,
+	.connect_tag_protocol   = ksz8463_connect_tag_protocol,
+	.get_phy_flags		= ksz_get_phy_flags,
+	.setup			= ksz_setup,
+	.teardown		= ksz_teardown,
+	.phy_read		= ksz_phy_read16,
+	.phy_write		= ksz_phy_write16,
+	.phylink_get_caps	= ksz8_phylink_get_caps,
+	.port_setup		= ksz_port_setup,
+	.get_strings		= ksz_get_strings,
+	.get_ethtool_stats	= ksz_get_ethtool_stats,
+	.get_sset_count		= ksz_sset_count,
+	.port_bridge_join	= ksz_port_bridge_join,
+	.port_bridge_leave	= ksz_port_bridge_leave,
+	.port_hsr_join		= ksz_hsr_join,
+	.port_hsr_leave		= ksz_hsr_leave,
+	.port_set_mac_address	= ksz_port_set_mac_address,
+	.port_stp_state_set	= ksz_port_stp_state_set,
+	.port_teardown		= ksz_port_teardown,
+	.port_pre_bridge_flags	= ksz_port_pre_bridge_flags,
+	.port_bridge_flags	= ksz_port_bridge_flags,
+	.port_fast_age		= ksz8_flush_dyn_mac_table,
+	.port_vlan_filtering	= ksz8_port_vlan_filtering,
+	.port_vlan_add		= ksz8_port_vlan_add,
+	.port_vlan_del		= ksz8_port_vlan_del,
+	.port_fdb_dump		= ksz8_fdb_dump,
+	.port_fdb_add		= ksz8_fdb_add,
+	.port_fdb_del		= ksz8_fdb_del,
+	.port_mdb_add           = ksz8_mdb_add,
+	.port_mdb_del           = ksz8_mdb_del,
+	.port_mirror_add	= ksz8_port_mirror_add,
+	.port_mirror_del	= ksz8_port_mirror_del,
+	.get_stats64		= ksz_get_stats64,
+	.get_pause_stats	= ksz_get_pause_stats,
+	.port_change_mtu	= ksz8_change_mtu,
+	.port_max_mtu		= ksz_max_mtu,
+	.get_wol		= ksz_get_wol,
+	.set_wol		= ksz_set_wol,
+	.suspend		= ksz_suspend,
+	.resume			= ksz_resume,
+	.get_ts_info		= ksz_get_ts_info,
+	.port_hwtstamp_get	= ksz_hwtstamp_get,
+	.port_hwtstamp_set	= ksz_hwtstamp_set,
+	.port_txtstamp		= ksz_port_txtstamp,
+	.port_rxtstamp		= ksz_port_rxtstamp,
+	.cls_flower_add		= ksz_cls_flower_add,
+	.cls_flower_del		= ksz_cls_flower_del,
+	.port_setup_tc		= ksz_setup_tc,
+	.support_eee		= ksz_support_eee,
+	.set_mac_eee		= ksz_set_mac_eee,
+	.port_get_default_prio	= ksz_port_get_default_prio,
+	.port_set_default_prio	= ksz_port_set_default_prio,
+	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
+	.port_add_dscp_prio	= ksz_port_add_dscp_prio,
+	.port_del_dscp_prio	= ksz_port_del_dscp_prio,
+	.port_get_apptrust	= ksz_port_get_apptrust,
+	.port_set_apptrust	= ksz_port_set_apptrust,
+};
+
+const struct dsa_switch_ops ksz87xx_switch_ops = {
+	.get_tag_protocol	= ksz87xx_get_tag_protocol,
+	.connect_tag_protocol   = ksz87xx_connect_tag_protocol,
+	.get_phy_flags		= ksz_get_phy_flags,
+	.setup			= ksz_setup,
+	.teardown		= ksz_teardown,
+	.phy_read		= ksz_phy_read16,
+	.phy_write		= ksz_phy_write16,
+	.phylink_get_caps	= ksz8_phylink_get_caps,
+	.port_setup		= ksz_port_setup,
+	.get_strings		= ksz_get_strings,
+	.get_ethtool_stats	= ksz_get_ethtool_stats,
+	.get_sset_count		= ksz_sset_count,
+	.port_bridge_join	= ksz_port_bridge_join,
+	.port_bridge_leave	= ksz_port_bridge_leave,
+	.port_hsr_join		= ksz_hsr_join,
+	.port_hsr_leave		= ksz_hsr_leave,
+	.port_set_mac_address	= ksz_port_set_mac_address,
+	.port_stp_state_set	= ksz_port_stp_state_set,
+	.port_teardown		= ksz_port_teardown,
+	.port_pre_bridge_flags	= ksz_port_pre_bridge_flags,
+	.port_bridge_flags	= ksz_port_bridge_flags,
+	.port_fast_age		= ksz8_flush_dyn_mac_table,
+	.port_vlan_filtering	= ksz8_port_vlan_filtering,
+	.port_vlan_add		= ksz8_port_vlan_add,
+	.port_vlan_del		= ksz8_port_vlan_del,
+	.port_fdb_dump		= ksz8_fdb_dump,
+	.port_fdb_add		= ksz8_fdb_add,
+	.port_fdb_del		= ksz8_fdb_del,
+	.port_mdb_add           = ksz8_mdb_add,
+	.port_mdb_del           = ksz8_mdb_del,
+	.port_mirror_add	= ksz8_port_mirror_add,
+	.port_mirror_del	= ksz8_port_mirror_del,
+	.get_stats64		= ksz_get_stats64,
+	.get_pause_stats	= ksz_get_pause_stats,
+	.port_change_mtu	= ksz8_change_mtu,
+	.port_max_mtu		= ksz_max_mtu,
+	.get_wol		= ksz_get_wol,
+	.set_wol		= ksz_set_wol,
+	.suspend		= ksz_suspend,
+	.resume			= ksz_resume,
+	.get_ts_info		= ksz_get_ts_info,
+	.port_hwtstamp_get	= ksz_hwtstamp_get,
+	.port_hwtstamp_set	= ksz_hwtstamp_set,
+	.port_txtstamp		= ksz_port_txtstamp,
+	.port_rxtstamp		= ksz_port_rxtstamp,
+	.cls_flower_add		= ksz_cls_flower_add,
+	.cls_flower_del		= ksz_cls_flower_del,
+	.port_setup_tc		= ksz_setup_tc,
+	.support_eee		= ksz_support_eee,
+	.set_mac_eee		= ksz_set_mac_eee,
+	.port_get_default_prio	= ksz_port_get_default_prio,
+	.port_set_default_prio	= ksz_port_set_default_prio,
+	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
+	.port_add_dscp_prio	= ksz_port_add_dscp_prio,
+	.port_del_dscp_prio	= ksz_port_del_dscp_prio,
+	.port_get_apptrust	= ksz_port_get_apptrust,
+	.port_set_apptrust	= ksz_port_set_apptrust,
+};
+
+const struct dsa_switch_ops ksz88xx_switch_ops = {
+	.get_tag_protocol	= ksz88xx_get_tag_protocol,
+	.connect_tag_protocol   = ksz88xx_connect_tag_protocol,
+	.get_phy_flags		= ksz_get_phy_flags,
+	.setup			= ksz_setup,
+	.teardown		= ksz_teardown,
+	.phy_read		= ksz_phy_read16,
+	.phy_write		= ksz_phy_write16,
+	.phylink_get_caps	= ksz8_phylink_get_caps,
+	.port_setup		= ksz_port_setup,
+	.get_strings		= ksz_get_strings,
+	.get_ethtool_stats	= ksz_get_ethtool_stats,
+	.get_sset_count		= ksz_sset_count,
+	.port_bridge_join	= ksz_port_bridge_join,
+	.port_bridge_leave	= ksz_port_bridge_leave,
+	.port_hsr_join		= ksz_hsr_join,
+	.port_hsr_leave		= ksz_hsr_leave,
+	.port_set_mac_address	= ksz_port_set_mac_address,
+	.port_stp_state_set	= ksz_port_stp_state_set,
+	.port_teardown		= ksz_port_teardown,
+	.port_pre_bridge_flags	= ksz_port_pre_bridge_flags,
+	.port_bridge_flags	= ksz_port_bridge_flags,
+	.port_fast_age		= ksz8_flush_dyn_mac_table,
+	.port_vlan_filtering	= ksz8_port_vlan_filtering,
+	.port_vlan_add		= ksz8_port_vlan_add,
+	.port_vlan_del		= ksz8_port_vlan_del,
+	.port_fdb_dump		= ksz8_fdb_dump,
+	.port_fdb_add		= ksz8_fdb_add,
+	.port_fdb_del		= ksz8_fdb_del,
+	.port_mdb_add           = ksz8_mdb_add,
+	.port_mdb_del           = ksz8_mdb_del,
+	.port_mirror_add	= ksz8_port_mirror_add,
+	.port_mirror_del	= ksz8_port_mirror_del,
+	.get_stats64		= ksz_get_stats64,
+	.get_pause_stats	= ksz_get_pause_stats,
+	.port_change_mtu	= ksz8_change_mtu,
+	.port_max_mtu		= ksz_max_mtu,
+	.get_wol		= ksz_get_wol,
+	.set_wol		= ksz_set_wol,
+	.suspend		= ksz_suspend,
+	.resume			= ksz_resume,
+	.get_ts_info		= ksz_get_ts_info,
+	.port_hwtstamp_get	= ksz_hwtstamp_get,
+	.port_hwtstamp_set	= ksz_hwtstamp_set,
+	.port_txtstamp		= ksz_port_txtstamp,
+	.port_rxtstamp		= ksz_port_rxtstamp,
+	.cls_flower_add		= ksz_cls_flower_add,
+	.cls_flower_del		= ksz_cls_flower_del,
+	.port_setup_tc		= ksz_setup_tc,
+	.support_eee		= ksz_support_eee,
+	.set_mac_eee		= ksz_set_mac_eee,
+	.port_get_default_prio	= ksz_port_get_default_prio,
+	.port_set_default_prio	= ksz_port_set_default_prio,
+	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
+	.port_add_dscp_prio	= ksz_port_add_dscp_prio,
+	.port_del_dscp_prio	= ksz_port_del_dscp_prio,
+	.port_get_apptrust	= ksz_port_get_apptrust,
+	.port_set_apptrust	= ksz_port_set_apptrust,
+};
+
 MODULE_AUTHOR("Tristram Ha <Tristram.Ha@microchip.com>");
 MODULE_DESCRIPTION("Microchip KSZ8795 Series Switch DSA Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/net/dsa/microchip/ksz8.h b/drivers/net/dsa/microchip/ksz8.h
index 0f2cd1474b44f..bc371cc26c6f1 100644
--- a/drivers/net/dsa/microchip/ksz8.h
+++ b/drivers/net/dsa/microchip/ksz8.h
@@ -12,59 +12,13 @@
 #include <net/dsa.h>
 #include "ksz_common.h"
 
-int ksz8_setup(struct dsa_switch *ds);
-u32 ksz8_get_port_addr(int port, int offset);
-void ksz8_cfg_port_member(struct ksz_device *dev, int port, u8 member);
-void ksz8_flush_dyn_mac_table(struct ksz_device *dev, int port);
-void ksz8_port_setup(struct ksz_device *dev, int port, bool cpu_port);
-int ksz8_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val);
-int ksz8_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val);
-void ksz8_r_mib_cnt(struct ksz_device *dev, int port, u16 addr, u64 *cnt);
-void ksz8_r_mib_pkt(struct ksz_device *dev, int port, u16 addr,
-		    u64 *dropped, u64 *cnt);
-void ksz8_freeze_mib(struct ksz_device *dev, int port, bool freeze);
-void ksz8_port_init_cnt(struct ksz_device *dev, int port);
-int ksz8_fdb_dump(struct ksz_device *dev, int port,
-		  dsa_fdb_dump_cb_t *cb, void *data);
-int ksz8_fdb_add(struct ksz_device *dev, int port, const unsigned char *addr,
-		 u16 vid, struct dsa_db db);
-int ksz8_fdb_del(struct ksz_device *dev, int port, const unsigned char *addr,
-		 u16 vid, struct dsa_db db);
-int ksz8_mdb_add(struct ksz_device *dev, int port,
-		 const struct switchdev_obj_port_mdb *mdb, struct dsa_db db);
-int ksz8_mdb_del(struct ksz_device *dev, int port,
-		 const struct switchdev_obj_port_mdb *mdb, struct dsa_db db);
-int ksz8_port_vlan_filtering(struct ksz_device *dev, int port, bool flag,
-			     struct netlink_ext_ack *extack);
-int ksz8_port_vlan_add(struct ksz_device *dev, int port,
-		       const struct switchdev_obj_port_vlan *vlan,
-		       struct netlink_ext_ack *extack);
-int ksz8_port_vlan_del(struct ksz_device *dev, int port,
-		       const struct switchdev_obj_port_vlan *vlan);
-int ksz8_port_mirror_add(struct ksz_device *dev, int port,
-			 struct dsa_mall_mirror_tc_entry *mirror,
-			 bool ingress, struct netlink_ext_ack *extack);
-void ksz8_port_mirror_del(struct ksz_device *dev, int port,
-			  struct dsa_mall_mirror_tc_entry *mirror);
-void ksz8_get_caps(struct ksz_device *dev, int port,
-		   struct phylink_config *config);
-void ksz8_config_cpu_port(struct dsa_switch *ds);
-int ksz8_enable_stp_addr(struct ksz_device *dev);
-int ksz8_reset_switch(struct ksz_device *dev);
-int ksz8_switch_init(struct ksz_device *dev);
-void ksz8_switch_exit(struct ksz_device *dev);
-int ksz8_change_mtu(struct ksz_device *dev, int port, int mtu);
-int ksz8_pme_write8(struct ksz_device *dev, u32 reg, u8 value);
-int ksz8_pme_pread8(struct ksz_device *dev, int port, int offset, u8 *data);
-int ksz8_pme_pwrite8(struct ksz_device *dev, int port, int offset, u8 data);
-void ksz8_phylink_mac_link_up(struct phylink_config *config,
-			      struct phy_device *phydev, unsigned int mode,
-			      phy_interface_t interface, int speed, int duplex,
-			      bool tx_pause, bool rx_pause);
-int ksz8_all_queues_split(struct ksz_device *dev, int queues);
-
-u32 ksz8463_get_port_addr(int port, int offset);
-int ksz8463_r_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 *val);
-int ksz8463_w_phy(struct ksz_device *dev, u16 phy, u16 reg, u16 val);
+extern const struct ksz_dev_ops ksz8463_dev_ops;
+extern const struct ksz_dev_ops ksz87xx_dev_ops;
+extern const struct ksz_dev_ops ksz88xx_dev_ops;
+extern const struct phylink_mac_ops ksz88x3_phylink_mac_ops;
+extern const struct phylink_mac_ops ksz8_phylink_mac_ops;
+extern const struct dsa_switch_ops ksz8463_switch_ops;
+extern const struct dsa_switch_ops ksz87xx_switch_ops;
+extern const struct dsa_switch_ops ksz88xx_switch_ops;
 
 #endif
diff --git a/drivers/net/dsa/microchip/ksz8863_smi.c b/drivers/net/dsa/microchip/ksz8863_smi.c
index a8bfcd917bf70..ba08d2cf8e99f 100644
--- a/drivers/net/dsa/microchip/ksz8863_smi.c
+++ b/drivers/net/dsa/microchip/ksz8863_smi.c
@@ -140,14 +140,14 @@ static int ksz8863_smi_probe(struct mdio_device *mdiodev)
 	int ret;
 	int i;
 
-	dev = ksz_switch_alloc(&mdiodev->dev, mdiodev);
-	if (!dev)
-		return -ENOMEM;
-
 	chip = device_get_match_data(ddev);
 	if (!chip)
 		return -EINVAL;
 
+	dev = ksz_switch_alloc(&mdiodev->dev, chip, mdiodev);
+	if (!dev)
+		return -ENOMEM;
+
 	for (i = 0; i < __KSZ_NUM_REGMAPS; i++) {
 		rc = ksz8863_regmap_config[i];
 		rc.lock_arg = &dev->regmap_mutex;
diff --git a/drivers/net/dsa/microchip/ksz9477.c b/drivers/net/dsa/microchip/ksz9477.c
index d3c23dcaea8ca..65fd07ef73ad0 100644
--- a/drivers/net/dsa/microchip/ksz9477.c
+++ b/drivers/net/dsa/microchip/ksz9477.c
@@ -5,6 +5,7 @@
  * Copyright (C) 2017-2025 Microchip Technology Inc.
  */
 
+#include <linux/dsa/ksz_common.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/iopoll.h>
@@ -17,6 +18,7 @@
 
 #include "ksz9477_reg.h"
 #include "ksz_common.h"
+#include "ksz_dcb.h"
 #include "ksz9477.h"
 
 static void ksz_cfg(struct ksz_device *dev, u32 addr, u8 bits, bool set)
@@ -43,8 +45,9 @@ static void ksz9477_port_cfg32(struct ksz_device *dev, int port, int offset,
 			   bits, set ? bits : 0);
 }
 
-int ksz9477_change_mtu(struct ksz_device *dev, int port, int mtu)
+static int ksz9477_change_mtu(struct dsa_switch *ds, int port, int mtu)
 {
+	struct ksz_device *dev = ds->priv;
 	u16 frame_size;
 
 	if (!dsa_is_cpu_port(dev->ds, port))
@@ -308,7 +311,7 @@ static int ksz9477_pcs_write(struct mii_bus *bus, int phy, int mmd, int reg,
 	return 0;
 }
 
-int ksz9477_pcs_create(struct ksz_device *dev)
+static int ksz9477_pcs_create(struct ksz_device *dev)
 {
 	int port = ksz_get_sgmii_port(dev);
 	struct ksz_port *p = &dev->ports[port];
@@ -341,7 +344,7 @@ int ksz9477_pcs_create(struct ksz_device *dev)
 	return 0;
 }
 
-int ksz9477_reset_switch(struct ksz_device *dev)
+static int ksz9477_reset_switch(struct ksz_device *dev)
 {
 	u8 data8;
 	u32 data32;
@@ -516,7 +519,7 @@ static void ksz9477_r_phy_quirks(struct ksz_device *dev, u16 addr, u16 reg,
 		*data &= ~(BMSR_ESTATEN | BMSR_ERCAP);
 }
 
-int ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
+static int ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
 {
 	u16 val = 0xffff;
 	int ret;
@@ -572,7 +575,7 @@ int ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
 	return 0;
 }
 
-int ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val)
+static int ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val)
 {
 	u32 mask, val32;
 
@@ -602,8 +605,9 @@ void ksz9477_cfg_port_member(struct ksz_device *dev, int port, u8 member)
 	ksz_pwrite32(dev, port, REG_PORT_VLAN_MEMBERSHIP__4, member);
 }
 
-void ksz9477_flush_dyn_mac_table(struct ksz_device *dev, int port)
+void ksz9477_flush_dyn_mac_table(struct dsa_switch *ds, int port)
 {
+	struct ksz_device *dev = ds->priv;
 	const u16 *regs = dev->info->regs;
 	u8 data;
 
@@ -625,9 +629,11 @@ void ksz9477_flush_dyn_mac_table(struct ksz_device *dev, int port)
 	}
 }
 
-int ksz9477_port_vlan_filtering(struct ksz_device *dev, int port,
+int ksz9477_port_vlan_filtering(struct dsa_switch *ds, int port,
 				bool flag, struct netlink_ext_ack *extack)
 {
+	struct ksz_device *dev = ds->priv;
+
 	if (flag) {
 		ksz_port_cfg(dev, port, REG_PORT_LUE_CTRL,
 			     PORT_VLAN_LOOKUP_VID_0, true);
@@ -641,12 +647,13 @@ int ksz9477_port_vlan_filtering(struct ksz_device *dev, int port,
 	return 0;
 }
 
-int ksz9477_port_vlan_add(struct ksz_device *dev, int port,
+int ksz9477_port_vlan_add(struct dsa_switch *ds, int port,
 			  const struct switchdev_obj_port_vlan *vlan,
 			  struct netlink_ext_ack *extack)
 {
-	u32 vlan_table[3];
 	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
+	struct ksz_device *dev = ds->priv;
+	u32 vlan_table[3];
 	int err;
 
 	err = ksz9477_get_vlan_table(dev, vlan->vid, vlan_table);
@@ -677,10 +684,11 @@ int ksz9477_port_vlan_add(struct ksz_device *dev, int port,
 	return 0;
 }
 
-int ksz9477_port_vlan_del(struct ksz_device *dev, int port,
+int ksz9477_port_vlan_del(struct dsa_switch *ds, int port,
 			  const struct switchdev_obj_port_vlan *vlan)
 {
 	bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED;
+	struct ksz_device *dev = ds->priv;
 	u32 vlan_table[3];
 	u16 pvid;
 
@@ -710,9 +718,10 @@ int ksz9477_port_vlan_del(struct ksz_device *dev, int port,
 	return 0;
 }
 
-int ksz9477_fdb_add(struct ksz_device *dev, int port,
+int ksz9477_fdb_add(struct dsa_switch *ds, int port,
 		    const unsigned char *addr, u16 vid, struct dsa_db db)
 {
+	struct ksz_device *dev = ds->priv;
 	u32 alu_table[4];
 	u32 data;
 	int ret = 0;
@@ -766,9 +775,10 @@ int ksz9477_fdb_add(struct ksz_device *dev, int port,
 	return ret;
 }
 
-int ksz9477_fdb_del(struct ksz_device *dev, int port,
+int ksz9477_fdb_del(struct dsa_switch *ds, int port,
 		    const unsigned char *addr, u16 vid, struct dsa_db db)
 {
+	struct ksz_device *dev = ds->priv;
 	u32 alu_table[4];
 	u32 data;
 	int ret = 0;
@@ -855,13 +865,14 @@ static void ksz9477_convert_alu(struct alu_struct *alu, u32 *alu_table)
 	alu->mac[5] = alu_table[3] & 0xFF;
 }
 
-int ksz9477_fdb_dump(struct ksz_device *dev, int port,
+int ksz9477_fdb_dump(struct dsa_switch *ds, int port,
 		     dsa_fdb_dump_cb_t *cb, void *data)
 {
-	int ret = 0;
-	u32 ksz_data;
-	u32 alu_table[4];
+	struct ksz_device *dev = ds->priv;
 	struct alu_struct alu;
+	u32 alu_table[4];
+	u32 ksz_data;
+	int ret = 0;
 	int timeout;
 
 	mutex_lock(&dev->alu_mutex);
@@ -909,9 +920,10 @@ int ksz9477_fdb_dump(struct ksz_device *dev, int port,
 	return ret;
 }
 
-int ksz9477_mdb_add(struct ksz_device *dev, int port,
+int ksz9477_mdb_add(struct dsa_switch *ds, int port,
 		    const struct switchdev_obj_port_mdb *mdb, struct dsa_db db)
 {
+	struct ksz_device *dev = ds->priv;
 	u32 static_table[4];
 	const u8 *shifts;
 	const u32 *masks;
@@ -988,16 +1000,17 @@ int ksz9477_mdb_add(struct ksz_device *dev, int port,
 	return err;
 }
 
-int ksz9477_mdb_del(struct ksz_device *dev, int port,
+int ksz9477_mdb_del(struct dsa_switch *ds, int port,
 		    const struct switchdev_obj_port_mdb *mdb, struct dsa_db db)
 {
+	struct ksz_device *dev = ds->priv;
 	u32 static_table[4];
+	u32 mac_hi, mac_lo;
 	const u8 *shifts;
 	const u32 *masks;
-	u32 data;
-	int index;
 	int ret = 0;
-	u32 mac_hi, mac_lo;
+	int index;
+	u32 data;
 
 	shifts = dev->info->shifts;
 	masks = dev->info->masks;
@@ -1067,10 +1080,11 @@ int ksz9477_mdb_del(struct ksz_device *dev, int port,
 	return ret;
 }
 
-int ksz9477_port_mirror_add(struct ksz_device *dev, int port,
+int ksz9477_port_mirror_add(struct dsa_switch *ds, int port,
 			    struct dsa_mall_mirror_tc_entry *mirror,
 			    bool ingress, struct netlink_ext_ack *extack)
 {
+	struct ksz_device *dev = ds->priv;
 	u8 data;
 	int p;
 
@@ -1106,9 +1120,10 @@ int ksz9477_port_mirror_add(struct ksz_device *dev, int port,
 	return 0;
 }
 
-void ksz9477_port_mirror_del(struct ksz_device *dev, int port,
+void ksz9477_port_mirror_del(struct dsa_switch *ds, int port,
 			     struct dsa_mall_mirror_tc_entry *mirror)
 {
+	struct ksz_device *dev = ds->priv;
 	bool in_use = false;
 	u8 data;
 	int p;
@@ -1150,9 +1165,11 @@ static phy_interface_t ksz9477_get_interface(struct ksz_device *dev, int port)
 	return interface;
 }
 
-void ksz9477_get_caps(struct ksz_device *dev, int port,
-		      struct phylink_config *config)
+static void ksz9477_phylink_get_caps(struct dsa_switch *ds, int port,
+				     struct phylink_config *config)
 {
+	struct ksz_device *dev = ds->priv;
+
 	config->mac_capabilities = MAC_10 | MAC_100 | MAC_ASYM_PAUSE |
 				   MAC_SYM_PAUSE;
 
@@ -1166,10 +1183,13 @@ void ksz9477_get_caps(struct ksz_device *dev, int port,
 				 config->supported_interfaces,
 				 p->pcs->supported_interfaces);
 	}
+
+	ksz_phylink_get_caps(ds, port, config);
 }
 
-int ksz9477_set_ageing_time(struct ksz_device *dev, unsigned int msecs)
+static int ksz9477_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
 {
+	struct ksz_device *dev = ds->priv;
 	u32 secs = msecs / 1000;
 	u8 data, mult, value;
 	u32 max_val;
@@ -1234,7 +1254,7 @@ void ksz9477_port_queue_split(struct ksz_device *dev, int port)
 	ksz_prmw8(dev, port, REG_PORT_CTRL_0, PORT_QUEUE_SPLIT_MASK, data);
 }
 
-void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port)
+static void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 {
 	const u16 *regs = dev->info->regs;
 	struct dsa_switch *ds = dev->ds;
@@ -1289,7 +1309,7 @@ void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 	ksz_pwrite8(dev, port, regs[REG_PORT_PME_CTRL], 0);
 }
 
-void ksz9477_config_cpu_port(struct dsa_switch *ds)
+static void ksz9477_config_cpu_port(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 	struct ksz_port *p;
@@ -1454,7 +1474,7 @@ int ksz9477_enable_stp_addr(struct ksz_device *dev)
 	return 0;
 }
 
-int ksz9477_setup(struct dsa_switch *ds)
+static int ksz9477_setup(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 	const u16 *regs = dev->info->regs;
@@ -1500,7 +1520,7 @@ u32 ksz9477_get_port_addr(int port, int offset)
 	return PORT_CTRL_ADDR(port, offset);
 }
 
-int ksz9477_tc_cbs_set_cinc(struct ksz_device *dev, int port, u32 val)
+static int ksz9477_tc_cbs_set_cinc(struct ksz_device *dev, int port, u32 val)
 {
 	val = val >> 8;
 
@@ -1584,7 +1604,7 @@ void ksz9477_hsr_leave(struct dsa_switch *ds, int port, struct net_device *hsr)
 	ksz_port_cfg(dev, port, REG_PORT_LUE_CTRL, PORT_SRC_ADDR_FILTER, false);
 }
 
-int ksz9477_switch_init(struct ksz_device *dev)
+static int ksz9477_switch_init(struct ksz_device *dev)
 {
 	u8 data8;
 	int ret;
@@ -1604,11 +1624,250 @@ int ksz9477_switch_init(struct ksz_device *dev)
 	return 0;
 }
 
-void ksz9477_switch_exit(struct ksz_device *dev)
+static void ksz9477_switch_exit(struct ksz_device *dev)
 {
 	ksz9477_reset_switch(dev);
 }
 
+static enum dsa_tag_protocol ksz9477_get_tag_protocol(struct dsa_switch *ds,
+						      int port,
+						      enum dsa_tag_protocol mp)
+{
+	struct ksz_device *dev = ds->priv;
+
+	if (dev->chip_id == KSZ8563_CHIP_ID ||
+	    dev->chip_id == KSZ9893_CHIP_ID ||
+	    dev->chip_id == KSZ9563_CHIP_ID)
+		return DSA_TAG_PROTO_KSZ9893;
+
+	return DSA_TAG_PROTO_KSZ9477;
+}
+
+static int ksz9477_connect_tag_protocol(struct dsa_switch *ds,
+					enum dsa_tag_protocol proto)
+{
+	struct ksz_tagger_data *tagger_data;
+
+	if (proto != DSA_TAG_PROTO_KSZ9893 && proto != DSA_TAG_PROTO_KSZ9477)
+		return -EPROTONOSUPPORT;
+
+	tagger_data = ksz_tagger_data(ds);
+	tagger_data->xmit_work_fn = ksz_port_deferred_xmit;
+
+	return 0;
+}
+
+static void ksz9477_set_gbit(struct ksz_device *dev, int port, bool gbit)
+{
+	const u8 *bitval = dev->info->xmii_ctrl1;
+	const u16 *regs = dev->info->regs;
+	u8 data8;
+
+	ksz_pread8(dev, port, regs[P_XMII_CTRL_1], &data8);
+
+	data8 &= ~P_GMII_1GBIT_M;
+
+	if (gbit)
+		data8 |= FIELD_PREP(P_GMII_1GBIT_M, bitval[P_GMII_1GBIT]);
+	else
+		data8 |= FIELD_PREP(P_GMII_1GBIT_M, bitval[P_GMII_NOT_1GBIT]);
+
+	/* Write the updated value */
+	ksz_pwrite8(dev, port, regs[P_XMII_CTRL_1], data8);
+}
+
+static void ksz9477_set_100_10mbit(struct ksz_device *dev, int port, int speed)
+{
+	const u8 *bitval = dev->info->xmii_ctrl0;
+	const u16 *regs = dev->info->regs;
+	u8 data8;
+
+	ksz_pread8(dev, port, regs[P_XMII_CTRL_0], &data8);
+
+	data8 &= ~P_MII_100MBIT_M;
+
+	if (speed == SPEED_100)
+		data8 |= FIELD_PREP(P_MII_100MBIT_M, bitval[P_MII_100MBIT]);
+	else
+		data8 |= FIELD_PREP(P_MII_100MBIT_M, bitval[P_MII_10MBIT]);
+
+	/* Write the updated value */
+	ksz_pwrite8(dev, port, regs[P_XMII_CTRL_0], data8);
+}
+
+static void ksz9477_port_set_xmii_speed(struct ksz_device *dev, int port,
+					int speed)
+{
+	if (speed == SPEED_1000)
+		ksz9477_set_gbit(dev, port, true);
+	else
+		ksz9477_set_gbit(dev, port, false);
+
+	if (speed == SPEED_100 || speed == SPEED_10)
+		ksz9477_set_100_10mbit(dev, port, speed);
+}
+
+static void ksz9477_duplex_flowctrl(struct ksz_device *dev, int port, int duplex,
+				    bool tx_pause, bool rx_pause)
+{
+	const u8 *bitval = dev->info->xmii_ctrl0;
+	const u32 *masks = dev->info->masks;
+	const u16 *regs = dev->info->regs;
+	u8 mask;
+	u8 val;
+
+	mask = P_MII_DUPLEX_M | masks[P_MII_TX_FLOW_CTRL] |
+	       masks[P_MII_RX_FLOW_CTRL];
+
+	if (duplex == DUPLEX_FULL)
+		val = FIELD_PREP(P_MII_DUPLEX_M, bitval[P_MII_FULL_DUPLEX]);
+	else
+		val = FIELD_PREP(P_MII_DUPLEX_M, bitval[P_MII_HALF_DUPLEX]);
+
+	if (tx_pause)
+		val |= masks[P_MII_TX_FLOW_CTRL];
+
+	if (rx_pause)
+		val |= masks[P_MII_RX_FLOW_CTRL];
+
+	ksz_prmw8(dev, port, regs[P_XMII_CTRL_0], mask, val);
+}
+
+void ksz9477_phylink_mac_link_up(struct phylink_config *config,
+				 struct phy_device *phydev,
+				 unsigned int mode,
+				 phy_interface_t interface,
+				 int speed, int duplex, bool tx_pause,
+				 bool rx_pause)
+{
+	struct dsa_port *dp = dsa_phylink_to_port(config);
+	struct ksz_device *dev = dp->ds->priv;
+	int port = dp->index;
+	struct ksz_port *p;
+
+	p = &dev->ports[port];
+
+	/* Internal PHYs */
+	if (dev->info->internal_phy[port])
+		return;
+
+	p->speed = speed;
+
+	ksz9477_port_set_xmii_speed(dev, port, speed);
+
+	ksz9477_duplex_flowctrl(dev, port, duplex, tx_pause, rx_pause);
+}
+
+static struct phylink_pcs *
+ksz9477_phylink_mac_select_pcs(struct phylink_config *config,
+			       phy_interface_t interface)
+{
+	struct dsa_port *dp = dsa_phylink_to_port(config);
+	struct ksz_device *dev = dp->ds->priv;
+	struct ksz_port *p = &dev->ports[dp->index];
+
+	if (ksz_is_sgmii_port(dev, dp->index) &&
+	    (interface == PHY_INTERFACE_MODE_SGMII ||
+	    interface == PHY_INTERFACE_MODE_1000BASEX))
+		return p->pcs;
+
+	return NULL;
+}
+
+const struct phylink_mac_ops ksz9477_phylink_mac_ops = {
+	.mac_config	= ksz_phylink_mac_config,
+	.mac_link_down	= ksz_phylink_mac_link_down,
+	.mac_link_up	= ksz9477_phylink_mac_link_up,
+	.mac_disable_tx_lpi = ksz_phylink_mac_disable_tx_lpi,
+	.mac_enable_tx_lpi = ksz_phylink_mac_enable_tx_lpi,
+	.mac_select_pcs	= ksz9477_phylink_mac_select_pcs,
+};
+
+const struct ksz_dev_ops ksz9477_dev_ops = {
+	.setup = ksz9477_setup,
+	.get_port_addr = ksz9477_get_port_addr,
+	.cfg_port_member = ksz9477_cfg_port_member,
+	.port_setup = ksz9477_port_setup,
+	.r_phy = ksz9477_r_phy,
+	.w_phy = ksz9477_w_phy,
+	.r_mib_cnt = ksz9477_r_mib_cnt,
+	.r_mib_pkt = ksz9477_r_mib_pkt,
+	.r_mib_stat64 = ksz_r_mib_stats64,
+	.freeze_mib = ksz9477_freeze_mib,
+	.port_init_cnt = ksz9477_port_init_cnt,
+	.pme_write8 = ksz_write8,
+	.pme_pread8 = ksz_pread8,
+	.pme_pwrite8 = ksz_pwrite8,
+	.config_cpu_port = ksz9477_config_cpu_port,
+	.tc_cbs_set_cinc = ksz9477_tc_cbs_set_cinc,
+	.enable_stp_addr = ksz9477_enable_stp_addr,
+	.reset = ksz9477_reset_switch,
+	.init = ksz9477_switch_init,
+	.exit = ksz9477_switch_exit,
+	.pcs_create = ksz9477_pcs_create,
+};
+
+const struct dsa_switch_ops ksz9477_switch_ops = {
+	.get_tag_protocol	= ksz9477_get_tag_protocol,
+	.connect_tag_protocol   = ksz9477_connect_tag_protocol,
+	.get_phy_flags		= ksz_get_phy_flags,
+	.setup			= ksz_setup,
+	.teardown		= ksz_teardown,
+	.phy_read		= ksz_phy_read16,
+	.phy_write		= ksz_phy_write16,
+	.phylink_get_caps	= ksz9477_phylink_get_caps,
+	.port_setup		= ksz_port_setup,
+	.set_ageing_time	= ksz9477_set_ageing_time,
+	.get_strings		= ksz_get_strings,
+	.get_ethtool_stats	= ksz_get_ethtool_stats,
+	.get_sset_count		= ksz_sset_count,
+	.port_bridge_join	= ksz_port_bridge_join,
+	.port_bridge_leave	= ksz_port_bridge_leave,
+	.port_hsr_join		= ksz_hsr_join,
+	.port_hsr_leave		= ksz_hsr_leave,
+	.port_set_mac_address	= ksz_port_set_mac_address,
+	.port_stp_state_set	= ksz_port_stp_state_set,
+	.port_teardown		= ksz_port_teardown,
+	.port_pre_bridge_flags	= ksz_port_pre_bridge_flags,
+	.port_bridge_flags	= ksz_port_bridge_flags,
+	.port_fast_age		= ksz9477_flush_dyn_mac_table,
+	.port_vlan_filtering	= ksz9477_port_vlan_filtering,
+	.port_vlan_add		= ksz9477_port_vlan_add,
+	.port_vlan_del		= ksz9477_port_vlan_del,
+	.port_fdb_dump		= ksz9477_fdb_dump,
+	.port_fdb_add		= ksz9477_fdb_add,
+	.port_fdb_del		= ksz9477_fdb_del,
+	.port_mdb_add           = ksz9477_mdb_add,
+	.port_mdb_del           = ksz9477_mdb_del,
+	.port_mirror_add	= ksz9477_port_mirror_add,
+	.port_mirror_del	= ksz9477_port_mirror_del,
+	.get_stats64		= ksz_get_stats64,
+	.get_pause_stats	= ksz_get_pause_stats,
+	.port_change_mtu	= ksz9477_change_mtu,
+	.port_max_mtu		= ksz_max_mtu,
+	.get_wol		= ksz_get_wol,
+	.set_wol		= ksz_set_wol,
+	.suspend		= ksz_suspend,
+	.resume			= ksz_resume,
+	.get_ts_info		= ksz_get_ts_info,
+	.port_hwtstamp_get	= ksz_hwtstamp_get,
+	.port_hwtstamp_set	= ksz_hwtstamp_set,
+	.port_txtstamp		= ksz_port_txtstamp,
+	.port_rxtstamp		= ksz_port_rxtstamp,
+	.cls_flower_add		= ksz_cls_flower_add,
+	.cls_flower_del		= ksz_cls_flower_del,
+	.port_setup_tc		= ksz_setup_tc,
+	.support_eee		= ksz_support_eee,
+	.set_mac_eee		= ksz_set_mac_eee,
+	.port_get_default_prio	= ksz_port_get_default_prio,
+	.port_set_default_prio	= ksz_port_set_default_prio,
+	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
+	.port_add_dscp_prio	= ksz_port_add_dscp_prio,
+	.port_del_dscp_prio	= ksz_port_del_dscp_prio,
+	.port_get_apptrust	= ksz_port_get_apptrust,
+	.port_set_apptrust	= ksz_port_set_apptrust,
+};
+
 MODULE_AUTHOR("Woojung Huh <Woojung.Huh@microchip.com>");
 MODULE_DESCRIPTION("Microchip KSZ9477 Series Switch DSA Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/net/dsa/microchip/ksz9477.h b/drivers/net/dsa/microchip/ksz9477.h
index 0d1a6dfda23ea..599db0a6ba2e3 100644
--- a/drivers/net/dsa/microchip/ksz9477.h
+++ b/drivers/net/dsa/microchip/ksz9477.h
@@ -11,52 +11,39 @@
 #include <net/dsa.h>
 #include "ksz_common.h"
 
-int ksz9477_setup(struct dsa_switch *ds);
 u32 ksz9477_get_port_addr(int port, int offset);
 void ksz9477_cfg_port_member(struct ksz_device *dev, int port, u8 member);
-void ksz9477_flush_dyn_mac_table(struct ksz_device *dev, int port);
-void ksz9477_port_setup(struct ksz_device *dev, int port, bool cpu_port);
-int ksz9477_set_ageing_time(struct ksz_device *dev, unsigned int msecs);
-int ksz9477_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data);
-int ksz9477_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val);
+void ksz9477_flush_dyn_mac_table(struct dsa_switch *ds, int port);
 void ksz9477_r_mib_cnt(struct ksz_device *dev, int port, u16 addr, u64 *cnt);
 void ksz9477_r_mib_pkt(struct ksz_device *dev, int port, u16 addr,
 		       u64 *dropped, u64 *cnt);
 void ksz9477_freeze_mib(struct ksz_device *dev, int port, bool freeze);
 void ksz9477_port_init_cnt(struct ksz_device *dev, int port);
-int ksz9477_port_vlan_filtering(struct ksz_device *dev, int port,
+int ksz9477_port_vlan_filtering(struct dsa_switch *ds, int port,
 				bool flag, struct netlink_ext_ack *extack);
-int ksz9477_port_vlan_add(struct ksz_device *dev, int port,
+int ksz9477_port_vlan_add(struct dsa_switch *ds, int port,
 			  const struct switchdev_obj_port_vlan *vlan,
 			  struct netlink_ext_ack *extack);
-int ksz9477_port_vlan_del(struct ksz_device *dev, int port,
+int ksz9477_port_vlan_del(struct dsa_switch *ds, int port,
 			  const struct switchdev_obj_port_vlan *vlan);
-int ksz9477_port_mirror_add(struct ksz_device *dev, int port,
+int ksz9477_port_mirror_add(struct dsa_switch *ds, int port,
 			    struct dsa_mall_mirror_tc_entry *mirror,
 			    bool ingress, struct netlink_ext_ack *extack);
-void ksz9477_port_mirror_del(struct ksz_device *dev, int port,
+void ksz9477_port_mirror_del(struct dsa_switch *ds, int port,
 			     struct dsa_mall_mirror_tc_entry *mirror);
 int ksz9477_errata_monitor(struct ksz_device *dev, int port,
 			   u64 tx_late_col);
-void ksz9477_get_caps(struct ksz_device *dev, int port,
-		      struct phylink_config *config);
-int ksz9477_fdb_dump(struct ksz_device *dev, int port,
+int ksz9477_fdb_dump(struct dsa_switch *ds, int port,
 		     dsa_fdb_dump_cb_t *cb, void *data);
-int ksz9477_fdb_add(struct ksz_device *dev, int port,
+int ksz9477_fdb_add(struct dsa_switch *ds, int port,
 		    const unsigned char *addr, u16 vid, struct dsa_db db);
-int ksz9477_fdb_del(struct ksz_device *dev, int port,
+int ksz9477_fdb_del(struct dsa_switch *ds, int port,
 		    const unsigned char *addr, u16 vid, struct dsa_db db);
-int ksz9477_mdb_add(struct ksz_device *dev, int port,
+int ksz9477_mdb_add(struct dsa_switch *ds, int port,
 		    const struct switchdev_obj_port_mdb *mdb, struct dsa_db db);
-int ksz9477_mdb_del(struct ksz_device *dev, int port,
+int ksz9477_mdb_del(struct dsa_switch *ds, int port,
 		    const struct switchdev_obj_port_mdb *mdb, struct dsa_db db);
-int ksz9477_change_mtu(struct ksz_device *dev, int port, int mtu);
-void ksz9477_config_cpu_port(struct dsa_switch *ds);
-int ksz9477_tc_cbs_set_cinc(struct ksz_device *dev, int port, u32 val);
 int ksz9477_enable_stp_addr(struct ksz_device *dev);
-int ksz9477_reset_switch(struct ksz_device *dev);
-int ksz9477_switch_init(struct ksz_device *dev);
-void ksz9477_switch_exit(struct ksz_device *dev);
 void ksz9477_port_queue_split(struct ksz_device *dev, int port);
 void ksz9477_hsr_join(struct dsa_switch *ds, int port, struct net_device *hsr);
 void ksz9477_hsr_leave(struct dsa_switch *ds, int port, struct net_device *hsr);
@@ -97,6 +84,15 @@ void ksz9477_acl_match_process_l2(struct ksz_device *dev, int port,
 				  u16 ethtype, u8 *src_mac, u8 *dst_mac,
 				  unsigned long cookie, u32 prio);
 
-int ksz9477_pcs_create(struct ksz_device *dev);
+void ksz9477_phylink_mac_link_up(struct phylink_config *config,
+				 struct phy_device *phydev,
+				 unsigned int mode,
+				 phy_interface_t interface,
+				 int speed, int duplex, bool tx_pause,
+				 bool rx_pause);
+
+extern const struct ksz_dev_ops ksz9477_dev_ops;
+extern const struct phylink_mac_ops ksz9477_phylink_mac_ops;
+extern const struct dsa_switch_ops ksz9477_switch_ops;
 
 #endif
diff --git a/drivers/net/dsa/microchip/ksz9477_i2c.c b/drivers/net/dsa/microchip/ksz9477_i2c.c
index a2beb27459f18..ec0c9cab3defe 100644
--- a/drivers/net/dsa/microchip/ksz9477_i2c.c
+++ b/drivers/net/dsa/microchip/ksz9477_i2c.c
@@ -22,14 +22,14 @@ static int ksz9477_i2c_probe(struct i2c_client *i2c)
 	struct ksz_device *dev;
 	int i, ret;
 
-	dev = ksz_switch_alloc(&i2c->dev, i2c);
-	if (!dev)
-		return -ENOMEM;
-
 	chip = device_get_match_data(ddev);
 	if (!chip)
 		return -EINVAL;
 
+	dev = ksz_switch_alloc(&i2c->dev, chip, i2c);
+	if (!dev)
+		return -ENOMEM;
+
 	/* Save chip id to do special initialization when probing. */
 	dev->chip_id = chip->chip_id;
 	for (i = 0; i < __KSZ_NUM_REGMAPS; i++) {
@@ -80,8 +80,8 @@ static void ksz9477_i2c_shutdown(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id ksz9477_i2c_id[] = {
-	{ "ksz9477-switch" },
-	{}
+	{ .name = "ksz9477-switch" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, ksz9477_i2c_id);
diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c
index 144373e13bea3..4a7bcd1a93927 100644
--- a/drivers/net/dsa/microchip/ksz_common.c
+++ b/drivers/net/dsa/microchip/ksz_common.c
@@ -256,16 +256,6 @@ static const struct ksz_drive_strength ksz88x3_drive_strengths[] = {
 	{ KSZ8873_DRIVE_STRENGTH_16MA, 16000 },
 };
 
-static void ksz88x3_phylink_mac_config(struct phylink_config *config,
-				       unsigned int mode,
-				       const struct phylink_link_state *state);
-static void ksz_phylink_mac_config(struct phylink_config *config,
-				   unsigned int mode,
-				   const struct phylink_link_state *state);
-static void ksz_phylink_mac_link_down(struct phylink_config *config,
-				      unsigned int mode,
-				      phy_interface_t interface);
-
 /**
  * ksz_phylink_mac_disable_tx_lpi() - Callback to signal LPI support (Dummy)
  * @config: phylink config structure
@@ -273,7 +263,7 @@ static void ksz_phylink_mac_link_down(struct phylink_config *config,
  * This function is a dummy handler. See ksz_phylink_mac_enable_tx_lpi() for
  * a detailed explanation of EEE/LPI handling in KSZ switches.
  */
-static void ksz_phylink_mac_disable_tx_lpi(struct phylink_config *config)
+void ksz_phylink_mac_disable_tx_lpi(struct phylink_config *config)
 {
 }
 
@@ -310,246 +300,12 @@ static void ksz_phylink_mac_disable_tx_lpi(struct phylink_config *config)
  *
  * Returns: 0 (Always success)
  */
-static int ksz_phylink_mac_enable_tx_lpi(struct phylink_config *config,
-					 u32 timer, bool tx_clock_stop)
+int ksz_phylink_mac_enable_tx_lpi(struct phylink_config *config,
+				  u32 timer, bool tx_clock_stop)
 {
 	return 0;
 }
 
-static const struct phylink_mac_ops ksz88x3_phylink_mac_ops = {
-	.mac_config	= ksz88x3_phylink_mac_config,
-	.mac_link_down	= ksz_phylink_mac_link_down,
-	.mac_link_up	= ksz8_phylink_mac_link_up,
-	.mac_disable_tx_lpi = ksz_phylink_mac_disable_tx_lpi,
-	.mac_enable_tx_lpi = ksz_phylink_mac_enable_tx_lpi,
-};
-
-static const struct phylink_mac_ops ksz8_phylink_mac_ops = {
-	.mac_config	= ksz_phylink_mac_config,
-	.mac_link_down	= ksz_phylink_mac_link_down,
-	.mac_link_up	= ksz8_phylink_mac_link_up,
-	.mac_disable_tx_lpi = ksz_phylink_mac_disable_tx_lpi,
-	.mac_enable_tx_lpi = ksz_phylink_mac_enable_tx_lpi,
-};
-
-static const struct ksz_dev_ops ksz8463_dev_ops = {
-	.setup = ksz8_setup,
-	.get_port_addr = ksz8463_get_port_addr,
-	.cfg_port_member = ksz8_cfg_port_member,
-	.flush_dyn_mac_table = ksz8_flush_dyn_mac_table,
-	.port_setup = ksz8_port_setup,
-	.r_phy = ksz8463_r_phy,
-	.w_phy = ksz8463_w_phy,
-	.r_mib_cnt = ksz8_r_mib_cnt,
-	.r_mib_pkt = ksz8_r_mib_pkt,
-	.r_mib_stat64 = ksz88xx_r_mib_stats64,
-	.freeze_mib = ksz8_freeze_mib,
-	.port_init_cnt = ksz8_port_init_cnt,
-	.fdb_dump = ksz8_fdb_dump,
-	.fdb_add = ksz8_fdb_add,
-	.fdb_del = ksz8_fdb_del,
-	.mdb_add = ksz8_mdb_add,
-	.mdb_del = ksz8_mdb_del,
-	.vlan_filtering = ksz8_port_vlan_filtering,
-	.vlan_add = ksz8_port_vlan_add,
-	.vlan_del = ksz8_port_vlan_del,
-	.mirror_add = ksz8_port_mirror_add,
-	.mirror_del = ksz8_port_mirror_del,
-	.get_caps = ksz8_get_caps,
-	.config_cpu_port = ksz8_config_cpu_port,
-	.enable_stp_addr = ksz8_enable_stp_addr,
-	.reset = ksz8_reset_switch,
-	.init = ksz8_switch_init,
-	.exit = ksz8_switch_exit,
-	.change_mtu = ksz8_change_mtu,
-};
-
-static const struct ksz_dev_ops ksz88xx_dev_ops = {
-	.setup = ksz8_setup,
-	.get_port_addr = ksz8_get_port_addr,
-	.cfg_port_member = ksz8_cfg_port_member,
-	.flush_dyn_mac_table = ksz8_flush_dyn_mac_table,
-	.port_setup = ksz8_port_setup,
-	.r_phy = ksz8_r_phy,
-	.w_phy = ksz8_w_phy,
-	.r_mib_cnt = ksz8_r_mib_cnt,
-	.r_mib_pkt = ksz8_r_mib_pkt,
-	.r_mib_stat64 = ksz88xx_r_mib_stats64,
-	.freeze_mib = ksz8_freeze_mib,
-	.port_init_cnt = ksz8_port_init_cnt,
-	.fdb_dump = ksz8_fdb_dump,
-	.fdb_add = ksz8_fdb_add,
-	.fdb_del = ksz8_fdb_del,
-	.mdb_add = ksz8_mdb_add,
-	.mdb_del = ksz8_mdb_del,
-	.vlan_filtering = ksz8_port_vlan_filtering,
-	.vlan_add = ksz8_port_vlan_add,
-	.vlan_del = ksz8_port_vlan_del,
-	.mirror_add = ksz8_port_mirror_add,
-	.mirror_del = ksz8_port_mirror_del,
-	.get_caps = ksz8_get_caps,
-	.config_cpu_port = ksz8_config_cpu_port,
-	.enable_stp_addr = ksz8_enable_stp_addr,
-	.reset = ksz8_reset_switch,
-	.init = ksz8_switch_init,
-	.exit = ksz8_switch_exit,
-	.change_mtu = ksz8_change_mtu,
-	.pme_write8 = ksz8_pme_write8,
-	.pme_pread8 = ksz8_pme_pread8,
-	.pme_pwrite8 = ksz8_pme_pwrite8,
-};
-
-static const struct ksz_dev_ops ksz87xx_dev_ops = {
-	.setup = ksz8_setup,
-	.get_port_addr = ksz8_get_port_addr,
-	.cfg_port_member = ksz8_cfg_port_member,
-	.flush_dyn_mac_table = ksz8_flush_dyn_mac_table,
-	.port_setup = ksz8_port_setup,
-	.r_phy = ksz8_r_phy,
-	.w_phy = ksz8_w_phy,
-	.r_mib_cnt = ksz8_r_mib_cnt,
-	.r_mib_pkt = ksz8_r_mib_pkt,
-	.r_mib_stat64 = ksz_r_mib_stats64,
-	.freeze_mib = ksz8_freeze_mib,
-	.port_init_cnt = ksz8_port_init_cnt,
-	.fdb_dump = ksz8_fdb_dump,
-	.fdb_add = ksz8_fdb_add,
-	.fdb_del = ksz8_fdb_del,
-	.mdb_add = ksz8_mdb_add,
-	.mdb_del = ksz8_mdb_del,
-	.vlan_filtering = ksz8_port_vlan_filtering,
-	.vlan_add = ksz8_port_vlan_add,
-	.vlan_del = ksz8_port_vlan_del,
-	.mirror_add = ksz8_port_mirror_add,
-	.mirror_del = ksz8_port_mirror_del,
-	.get_caps = ksz8_get_caps,
-	.config_cpu_port = ksz8_config_cpu_port,
-	.enable_stp_addr = ksz8_enable_stp_addr,
-	.reset = ksz8_reset_switch,
-	.init = ksz8_switch_init,
-	.exit = ksz8_switch_exit,
-	.change_mtu = ksz8_change_mtu,
-	.pme_write8 = ksz8_pme_write8,
-	.pme_pread8 = ksz8_pme_pread8,
-	.pme_pwrite8 = ksz8_pme_pwrite8,
-};
-
-static void ksz9477_phylink_mac_link_up(struct phylink_config *config,
-					struct phy_device *phydev,
-					unsigned int mode,
-					phy_interface_t interface,
-					int speed, int duplex, bool tx_pause,
-					bool rx_pause);
-
-static struct phylink_pcs *
-ksz_phylink_mac_select_pcs(struct phylink_config *config,
-			   phy_interface_t interface)
-{
-	struct dsa_port *dp = dsa_phylink_to_port(config);
-	struct ksz_device *dev = dp->ds->priv;
-	struct ksz_port *p = &dev->ports[dp->index];
-
-	if (ksz_is_sgmii_port(dev, dp->index) &&
-	    (interface == PHY_INTERFACE_MODE_SGMII ||
-	    interface == PHY_INTERFACE_MODE_1000BASEX))
-		return p->pcs;
-
-	return NULL;
-}
-
-static const struct phylink_mac_ops ksz9477_phylink_mac_ops = {
-	.mac_config	= ksz_phylink_mac_config,
-	.mac_link_down	= ksz_phylink_mac_link_down,
-	.mac_link_up	= ksz9477_phylink_mac_link_up,
-	.mac_disable_tx_lpi = ksz_phylink_mac_disable_tx_lpi,
-	.mac_enable_tx_lpi = ksz_phylink_mac_enable_tx_lpi,
-	.mac_select_pcs	= ksz_phylink_mac_select_pcs,
-};
-
-static const struct ksz_dev_ops ksz9477_dev_ops = {
-	.setup = ksz9477_setup,
-	.get_port_addr = ksz9477_get_port_addr,
-	.cfg_port_member = ksz9477_cfg_port_member,
-	.flush_dyn_mac_table = ksz9477_flush_dyn_mac_table,
-	.port_setup = ksz9477_port_setup,
-	.set_ageing_time = ksz9477_set_ageing_time,
-	.r_phy = ksz9477_r_phy,
-	.w_phy = ksz9477_w_phy,
-	.r_mib_cnt = ksz9477_r_mib_cnt,
-	.r_mib_pkt = ksz9477_r_mib_pkt,
-	.r_mib_stat64 = ksz_r_mib_stats64,
-	.freeze_mib = ksz9477_freeze_mib,
-	.port_init_cnt = ksz9477_port_init_cnt,
-	.vlan_filtering = ksz9477_port_vlan_filtering,
-	.vlan_add = ksz9477_port_vlan_add,
-	.vlan_del = ksz9477_port_vlan_del,
-	.mirror_add = ksz9477_port_mirror_add,
-	.mirror_del = ksz9477_port_mirror_del,
-	.get_caps = ksz9477_get_caps,
-	.fdb_dump = ksz9477_fdb_dump,
-	.fdb_add = ksz9477_fdb_add,
-	.fdb_del = ksz9477_fdb_del,
-	.mdb_add = ksz9477_mdb_add,
-	.mdb_del = ksz9477_mdb_del,
-	.change_mtu = ksz9477_change_mtu,
-	.pme_write8 = ksz_write8,
-	.pme_pread8 = ksz_pread8,
-	.pme_pwrite8 = ksz_pwrite8,
-	.config_cpu_port = ksz9477_config_cpu_port,
-	.tc_cbs_set_cinc = ksz9477_tc_cbs_set_cinc,
-	.enable_stp_addr = ksz9477_enable_stp_addr,
-	.reset = ksz9477_reset_switch,
-	.init = ksz9477_switch_init,
-	.exit = ksz9477_switch_exit,
-	.pcs_create = ksz9477_pcs_create,
-};
-
-static const struct phylink_mac_ops lan937x_phylink_mac_ops = {
-	.mac_config	= ksz_phylink_mac_config,
-	.mac_link_down	= ksz_phylink_mac_link_down,
-	.mac_link_up	= ksz9477_phylink_mac_link_up,
-	.mac_disable_tx_lpi = ksz_phylink_mac_disable_tx_lpi,
-	.mac_enable_tx_lpi = ksz_phylink_mac_enable_tx_lpi,
-};
-
-static const struct ksz_dev_ops lan937x_dev_ops = {
-	.setup = lan937x_setup,
-	.teardown = lan937x_teardown,
-	.get_port_addr = ksz9477_get_port_addr,
-	.cfg_port_member = ksz9477_cfg_port_member,
-	.flush_dyn_mac_table = ksz9477_flush_dyn_mac_table,
-	.port_setup = lan937x_port_setup,
-	.set_ageing_time = lan937x_set_ageing_time,
-	.mdio_bus_preinit = lan937x_mdio_bus_preinit,
-	.create_phy_addr_map = lan937x_create_phy_addr_map,
-	.r_phy = lan937x_r_phy,
-	.w_phy = lan937x_w_phy,
-	.r_mib_cnt = ksz9477_r_mib_cnt,
-	.r_mib_pkt = ksz9477_r_mib_pkt,
-	.r_mib_stat64 = ksz_r_mib_stats64,
-	.freeze_mib = ksz9477_freeze_mib,
-	.port_init_cnt = ksz9477_port_init_cnt,
-	.vlan_filtering = ksz9477_port_vlan_filtering,
-	.vlan_add = ksz9477_port_vlan_add,
-	.vlan_del = ksz9477_port_vlan_del,
-	.mirror_add = ksz9477_port_mirror_add,
-	.mirror_del = ksz9477_port_mirror_del,
-	.get_caps = lan937x_phylink_get_caps,
-	.setup_rgmii_delay = lan937x_setup_rgmii_delay,
-	.fdb_dump = ksz9477_fdb_dump,
-	.fdb_add = ksz9477_fdb_add,
-	.fdb_del = ksz9477_fdb_del,
-	.mdb_add = ksz9477_mdb_add,
-	.mdb_del = ksz9477_mdb_del,
-	.change_mtu = lan937x_change_mtu,
-	.config_cpu_port = lan937x_config_cpu_port,
-	.tc_cbs_set_cinc = lan937x_tc_cbs_set_cinc,
-	.enable_stp_addr = ksz9477_enable_stp_addr,
-	.reset = lan937x_reset_switch,
-	.init = lan937x_switch_init,
-	.exit = lan937x_switch_exit,
-};
-
 static const u16 ksz8463_regs[] = {
 	[REG_SW_MAC_ADDR]		= 0x10,
 	[REG_IND_CTRL_0]		= 0x30,
@@ -1502,6 +1258,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 4,
 		.ops = &ksz8463_dev_ops,
+		.switch_ops = &ksz8463_switch_ops,
 		.phylink_mac_ops = &ksz88x3_phylink_mac_ops,
 		.mib_names = ksz88xx_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz88xx_mib_names),
@@ -1527,6 +1284,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_ipms = 8,
 		.tc_cbs_supported = true,
 		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
 		.phylink_mac_ops = &ksz9477_phylink_mac_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1557,6 +1315,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 4,
 		.ops = &ksz87xx_dev_ops,
+		.switch_ops = &ksz87xx_switch_ops,
 		.phylink_mac_ops = &ksz8_phylink_mac_ops,
 		.ksz87xx_eee_link_erratum = true,
 		.mib_names = ksz9477_mib_names,
@@ -1598,6 +1357,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 4,
 		.ops = &ksz87xx_dev_ops,
+		.switch_ops = &ksz87xx_switch_ops,
 		.phylink_mac_ops = &ksz8_phylink_mac_ops,
 		.ksz87xx_eee_link_erratum = true,
 		.mib_names = ksz9477_mib_names,
@@ -1625,6 +1385,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 4,
 		.ops = &ksz87xx_dev_ops,
+		.switch_ops = &ksz87xx_switch_ops,
 		.phylink_mac_ops = &ksz8_phylink_mac_ops,
 		.ksz87xx_eee_link_erratum = true,
 		.mib_names = ksz9477_mib_names,
@@ -1652,6 +1413,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 4,
 		.ops = &ksz88xx_dev_ops,
+		.switch_ops = &ksz88xx_switch_ops,
 		.phylink_mac_ops = &ksz88x3_phylink_mac_ops,
 		.mib_names = ksz88xx_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz88xx_mib_names),
@@ -1686,6 +1448,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 4,
 		.ops = &ksz88xx_dev_ops,
+		.switch_ops = &ksz88xx_switch_ops,
 		.phylink_mac_ops = &ksz88x3_phylink_mac_ops,
 		.mib_names = ksz88xx_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz88xx_mib_names),
@@ -1709,6 +1472,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 4,
 		.ops = &ksz88xx_dev_ops,
+		.switch_ops = &ksz88xx_switch_ops,
 		.phylink_mac_ops = &ksz88x3_phylink_mac_ops,
 		.mib_names = ksz88xx_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz88xx_mib_names),
@@ -1734,6 +1498,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_ipms = 8,
 		.tc_cbs_supported = true,
 		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
 		.phylink_mac_ops = &ksz9477_phylink_mac_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -1771,6 +1536,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 8,
 		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
 		.phylink_mac_ops = &ksz9477_phylink_mac_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -1806,6 +1572,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 8,
 		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
 		.phylink_mac_ops = &ksz9477_phylink_mac_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -1839,6 +1606,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 8,
 		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
 		.phylink_mac_ops = &ksz9477_phylink_mac_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1868,6 +1636,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_ipms = 8,
 		.tc_cbs_supported = true,
 		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
 		.phylink_mac_ops = &ksz9477_phylink_mac_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1898,6 +1667,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_ipms = 8,
 		.tc_cbs_supported = true,
 		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
 		.phylink_mac_ops = &ksz9477_phylink_mac_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -1934,6 +1704,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_ipms = 8,
 		.tc_cbs_supported = true,
 		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
 		.reg_mib_cnt = MIB_COUNTER_NUM,
@@ -1968,6 +1739,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.tc_cbs_supported = true,
 		.phy_side_mdio_supported = true,
 		.ops = &lan937x_dev_ops,
+		.switch_ops = &lan937x_switch_ops,
 		.phylink_mac_ops = &lan937x_phylink_mac_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -1998,6 +1770,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.tc_cbs_supported = true,
 		.phy_side_mdio_supported = true,
 		.ops = &lan937x_dev_ops,
+		.switch_ops = &lan937x_switch_ops,
 		.phylink_mac_ops = &lan937x_phylink_mac_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -2028,6 +1801,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.tc_cbs_supported = true,
 		.phy_side_mdio_supported = true,
 		.ops = &lan937x_dev_ops,
+		.switch_ops = &lan937x_switch_ops,
 		.phylink_mac_ops = &lan937x_phylink_mac_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -2062,6 +1836,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.tc_cbs_supported = true,
 		.phy_side_mdio_supported = true,
 		.ops = &lan937x_dev_ops,
+		.switch_ops = &lan937x_switch_ops,
 		.phylink_mac_ops = &lan937x_phylink_mac_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -2096,6 +1871,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.tc_cbs_supported = true,
 		.phy_side_mdio_supported = true,
 		.ops = &lan937x_dev_ops,
+		.switch_ops = &lan937x_switch_ops,
 		.phylink_mac_ops = &lan937x_phylink_mac_ops,
 		.mib_names = ksz9477_mib_names,
 		.mib_cnt = ARRAY_SIZE(ksz9477_mib_names),
@@ -2128,6 +1904,7 @@ const struct ksz_chip_data ksz_switch_chips[] = {
 		.num_tx_queues = 4,
 		.num_ipms = 8,
 		.ops = &ksz9477_dev_ops,
+		.switch_ops = &ksz9477_switch_ops,
 		.phylink_mac_ops = &ksz9477_phylink_mac_ops,
 		.phy_errata_9477 = true,
 		.mib_names = ksz9477_mib_names,
@@ -2193,8 +1970,8 @@ static int ksz_check_device_id(struct ksz_device *dev)
 	return 0;
 }
 
-static void ksz_phylink_get_caps(struct dsa_switch *ds, int port,
-				 struct phylink_config *config)
+void ksz_phylink_get_caps(struct dsa_switch *ds, int port,
+			  struct phylink_config *config)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -2218,9 +1995,6 @@ static void ksz_phylink_get_caps(struct dsa_switch *ds, int port,
 			  config->supported_interfaces);
 	}
 
-	if (dev->dev_ops->get_caps)
-		dev->dev_ops->get_caps(dev, port, config);
-
 	if (ds->ops->support_eee && ds->ops->support_eee(ds, port)) {
 		memcpy(config->lpi_interfaces, config->supported_interfaces,
 		       sizeof(config->lpi_interfaces));
@@ -2339,8 +2113,8 @@ void ksz88xx_r_mib_stats64(struct ksz_device *dev, int port)
 	spin_unlock(&mib->stats64_lock);
 }
 
-static void ksz_get_stats64(struct dsa_switch *ds, int port,
-			    struct rtnl_link_stats64 *s)
+void ksz_get_stats64(struct dsa_switch *ds, int port,
+		     struct rtnl_link_stats64 *s)
 {
 	struct ksz_device *dev = ds->priv;
 	struct ksz_port_mib *mib;
@@ -2352,8 +2126,8 @@ static void ksz_get_stats64(struct dsa_switch *ds, int port,
 	spin_unlock(&mib->stats64_lock);
 }
 
-static void ksz_get_pause_stats(struct dsa_switch *ds, int port,
-				struct ethtool_pause_stats *pause_stats)
+void ksz_get_pause_stats(struct dsa_switch *ds, int port,
+			 struct ethtool_pause_stats *pause_stats)
 {
 	struct ksz_device *dev = ds->priv;
 	struct ksz_port_mib *mib;
@@ -2365,8 +2139,8 @@ static void ksz_get_pause_stats(struct dsa_switch *ds, int port,
 	spin_unlock(&mib->stats64_lock);
 }
 
-static void ksz_get_strings(struct dsa_switch *ds, int port,
-			    u32 stringset, uint8_t *buf)
+void ksz_get_strings(struct dsa_switch *ds, int port,
+		     u32 stringset, uint8_t *buf)
 {
 	struct ksz_device *dev = ds->priv;
 	int i;
@@ -2973,7 +2747,7 @@ static int ksz_pirq_setup(struct ksz_device *dev, u8 p)
 
 static int ksz_parse_drive_strength(struct ksz_device *dev);
 
-static int ksz_setup(struct dsa_switch *ds)
+int ksz_setup(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 	u16 storm_mask, storm_rate;
@@ -3103,7 +2877,7 @@ static int ksz_setup(struct dsa_switch *ds)
 	return ret;
 }
 
-static void ksz_teardown(struct dsa_switch *ds)
+void ksz_teardown(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 	struct dsa_port *dp;
@@ -3203,7 +2977,7 @@ void ksz_init_mib_timer(struct ksz_device *dev)
 	}
 }
 
-static int ksz_phy_read16(struct dsa_switch *ds, int addr, int reg)
+int ksz_phy_read16(struct dsa_switch *ds, int addr, int reg)
 {
 	struct ksz_device *dev = ds->priv;
 	u16 val = 0xffff;
@@ -3216,7 +2990,7 @@ static int ksz_phy_read16(struct dsa_switch *ds, int addr, int reg)
 	return val;
 }
 
-static int ksz_phy_write16(struct dsa_switch *ds, int addr, int reg, u16 val)
+int ksz_phy_write16(struct dsa_switch *ds, int addr, int reg, u16 val)
 {
 	struct ksz_device *dev = ds->priv;
 	int ret;
@@ -3228,7 +3002,7 @@ static int ksz_phy_write16(struct dsa_switch *ds, int addr, int reg, u16 val)
 	return 0;
 }
 
-static u32 ksz_get_phy_flags(struct dsa_switch *ds, int port)
+u32 ksz_get_phy_flags(struct dsa_switch *ds, int port)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -3246,9 +3020,9 @@ static u32 ksz_get_phy_flags(struct dsa_switch *ds, int port)
 	return 0;
 }
 
-static void ksz_phylink_mac_link_down(struct phylink_config *config,
-				      unsigned int mode,
-				      phy_interface_t interface)
+void ksz_phylink_mac_link_down(struct phylink_config *config,
+			       unsigned int mode,
+			       phy_interface_t interface)
 {
 	struct dsa_port *dp = dsa_phylink_to_port(config);
 	struct ksz_device *dev = dp->ds->priv;
@@ -3260,7 +3034,7 @@ static void ksz_phylink_mac_link_down(struct phylink_config *config,
 		schedule_delayed_work(&dev->mib_read, 0);
 }
 
-static int ksz_sset_count(struct dsa_switch *ds, int port, int sset)
+int ksz_sset_count(struct dsa_switch *ds, int port, int sset)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -3270,8 +3044,8 @@ static int ksz_sset_count(struct dsa_switch *ds, int port, int sset)
 	return dev->info->mib_cnt;
 }
 
-static void ksz_get_ethtool_stats(struct dsa_switch *ds, int port,
-				  uint64_t *buf)
+void ksz_get_ethtool_stats(struct dsa_switch *ds, int port,
+			   uint64_t *buf)
 {
 	const struct dsa_port *dp = dsa_to_port(ds, port);
 	struct ksz_device *dev = ds->priv;
@@ -3288,10 +3062,10 @@ static void ksz_get_ethtool_stats(struct dsa_switch *ds, int port,
 	mutex_unlock(&mib->cnt_mutex);
 }
 
-static int ksz_port_bridge_join(struct dsa_switch *ds, int port,
-				struct dsa_bridge bridge,
-				bool *tx_fwd_offload,
-				struct netlink_ext_ack *extack)
+int ksz_port_bridge_join(struct dsa_switch *ds, int port,
+			 struct dsa_bridge bridge,
+			 bool *tx_fwd_offload,
+			 struct netlink_ext_ack *extack)
 {
 	/* port_stp_state_set() will be called after to put the port in
 	 * appropriate state so there is no need to do anything.
@@ -3300,90 +3074,14 @@ static int ksz_port_bridge_join(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static void ksz_port_bridge_leave(struct dsa_switch *ds, int port,
-				  struct dsa_bridge bridge)
+void ksz_port_bridge_leave(struct dsa_switch *ds, int port,
+			   struct dsa_bridge bridge)
 {
 	/* port_stp_state_set() will be called after to put the port in
 	 * forwarding state so there is no need to do anything.
 	 */
 }
 
-static void ksz_port_fast_age(struct dsa_switch *ds, int port)
-{
-	struct ksz_device *dev = ds->priv;
-
-	dev->dev_ops->flush_dyn_mac_table(dev, port);
-}
-
-static int ksz_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->set_ageing_time)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->set_ageing_time(dev, msecs);
-}
-
-static int ksz_port_fdb_add(struct dsa_switch *ds, int port,
-			    const unsigned char *addr, u16 vid,
-			    struct dsa_db db)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->fdb_add)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->fdb_add(dev, port, addr, vid, db);
-}
-
-static int ksz_port_fdb_del(struct dsa_switch *ds, int port,
-			    const unsigned char *addr,
-			    u16 vid, struct dsa_db db)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->fdb_del)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->fdb_del(dev, port, addr, vid, db);
-}
-
-static int ksz_port_fdb_dump(struct dsa_switch *ds, int port,
-			     dsa_fdb_dump_cb_t *cb, void *data)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->fdb_dump)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->fdb_dump(dev, port, cb, data);
-}
-
-static int ksz_port_mdb_add(struct dsa_switch *ds, int port,
-			    const struct switchdev_obj_port_mdb *mdb,
-			    struct dsa_db db)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->mdb_add)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->mdb_add(dev, port, mdb, db);
-}
-
-static int ksz_port_mdb_del(struct dsa_switch *ds, int port,
-			    const struct switchdev_obj_port_mdb *mdb,
-			    struct dsa_db db)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->mdb_del)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->mdb_del(dev, port, mdb, db);
-}
-
 static int ksz9477_set_default_prio_queue_mapping(struct ksz_device *dev,
 						  int port)
 {
@@ -3407,7 +3105,7 @@ static int ksz9477_set_default_prio_queue_mapping(struct ksz_device *dev,
 	return ksz_pwrite32(dev, port, KSZ9477_PORT_MRI_TC_MAP__4, queue_map);
 }
 
-static int ksz_port_setup(struct dsa_switch *ds, int port)
+int ksz_port_setup(struct dsa_switch *ds, int port)
 {
 	struct ksz_device *dev = ds->priv;
 	int ret;
@@ -3477,7 +3175,7 @@ void ksz_port_stp_state_set(struct dsa_switch *ds, int port, u8 state)
 	ksz_update_port_member(dev, port);
 }
 
-static void ksz_port_teardown(struct dsa_switch *ds, int port)
+void ksz_port_teardown(struct dsa_switch *ds, int port)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -3496,9 +3194,9 @@ static void ksz_port_teardown(struct dsa_switch *ds, int port)
 	}
 }
 
-static int ksz_port_pre_bridge_flags(struct dsa_switch *ds, int port,
-				     struct switchdev_brport_flags flags,
-				     struct netlink_ext_ack *extack)
+int ksz_port_pre_bridge_flags(struct dsa_switch *ds, int port,
+			      struct switchdev_brport_flags flags,
+			      struct netlink_ext_ack *extack)
 {
 	if (flags.mask & ~(BR_LEARNING | BR_ISOLATED))
 		return -EINVAL;
@@ -3506,9 +3204,9 @@ static int ksz_port_pre_bridge_flags(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static int ksz_port_bridge_flags(struct dsa_switch *ds, int port,
-				 struct switchdev_brport_flags flags,
-				 struct netlink_ext_ack *extack)
+int ksz_port_bridge_flags(struct dsa_switch *ds, int port,
+			  struct switchdev_brport_flags flags,
+			  struct netlink_ext_ack *extack)
 {
 	struct ksz_device *dev = ds->priv;
 	struct ksz_port *p = &dev->ports[port];
@@ -3527,122 +3225,7 @@ static int ksz_port_bridge_flags(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static enum dsa_tag_protocol ksz_get_tag_protocol(struct dsa_switch *ds,
-						  int port,
-						  enum dsa_tag_protocol mp)
-{
-	struct ksz_device *dev = ds->priv;
-	enum dsa_tag_protocol proto = DSA_TAG_PROTO_NONE;
-
-	if (ksz_is_ksz87xx(dev) || ksz_is_8895_family(dev))
-		proto = DSA_TAG_PROTO_KSZ8795;
-
-	if (dev->chip_id == KSZ88X3_CHIP_ID ||
-	    dev->chip_id == KSZ8463_CHIP_ID ||
-	    dev->chip_id == KSZ8563_CHIP_ID ||
-	    dev->chip_id == KSZ9893_CHIP_ID ||
-	    dev->chip_id == KSZ9563_CHIP_ID)
-		proto = DSA_TAG_PROTO_KSZ9893;
-
-	if (dev->chip_id == KSZ8567_CHIP_ID ||
-	    dev->chip_id == KSZ9477_CHIP_ID ||
-	    dev->chip_id == KSZ9896_CHIP_ID ||
-	    dev->chip_id == KSZ9897_CHIP_ID ||
-	    dev->chip_id == KSZ9567_CHIP_ID ||
-	    dev->chip_id == LAN9646_CHIP_ID)
-		proto = DSA_TAG_PROTO_KSZ9477;
-
-	if (is_lan937x(dev))
-		proto = DSA_TAG_PROTO_LAN937X;
-
-	return proto;
-}
-
-static int ksz_connect_tag_protocol(struct dsa_switch *ds,
-				    enum dsa_tag_protocol proto)
-{
-	struct ksz_tagger_data *tagger_data;
-
-	switch (proto) {
-	case DSA_TAG_PROTO_KSZ8795:
-		return 0;
-	case DSA_TAG_PROTO_KSZ9893:
-	case DSA_TAG_PROTO_KSZ9477:
-	case DSA_TAG_PROTO_LAN937X:
-		tagger_data = ksz_tagger_data(ds);
-		tagger_data->xmit_work_fn = ksz_port_deferred_xmit;
-		return 0;
-	default:
-		return -EPROTONOSUPPORT;
-	}
-}
-
-static int ksz_port_vlan_filtering(struct dsa_switch *ds, int port,
-				   bool flag, struct netlink_ext_ack *extack)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->vlan_filtering)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->vlan_filtering(dev, port, flag, extack);
-}
-
-static int ksz_port_vlan_add(struct dsa_switch *ds, int port,
-			     const struct switchdev_obj_port_vlan *vlan,
-			     struct netlink_ext_ack *extack)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->vlan_add)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->vlan_add(dev, port, vlan, extack);
-}
-
-static int ksz_port_vlan_del(struct dsa_switch *ds, int port,
-			     const struct switchdev_obj_port_vlan *vlan)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->vlan_del)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->vlan_del(dev, port, vlan);
-}
-
-static int ksz_port_mirror_add(struct dsa_switch *ds, int port,
-			       struct dsa_mall_mirror_tc_entry *mirror,
-			       bool ingress, struct netlink_ext_ack *extack)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->mirror_add)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->mirror_add(dev, port, mirror, ingress, extack);
-}
-
-static void ksz_port_mirror_del(struct dsa_switch *ds, int port,
-				struct dsa_mall_mirror_tc_entry *mirror)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (dev->dev_ops->mirror_del)
-		dev->dev_ops->mirror_del(dev, port, mirror);
-}
-
-static int ksz_change_mtu(struct dsa_switch *ds, int port, int mtu)
-{
-	struct ksz_device *dev = ds->priv;
-
-	if (!dev->dev_ops->change_mtu)
-		return -EOPNOTSUPP;
-
-	return dev->dev_ops->change_mtu(dev, port, mtu);
-}
-
-static int ksz_max_mtu(struct dsa_switch *ds, int port)
+int ksz_max_mtu(struct dsa_switch *ds, int port)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -3690,7 +3273,7 @@ static int ksz_max_mtu(struct dsa_switch *ds, int port)
  * Returns: true if the internal PHY on the given port supports fully
  * operational EEE, false otherwise.
  */
-static bool ksz_support_eee(struct dsa_switch *ds, int port)
+bool ksz_support_eee(struct dsa_switch *ds, int port)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -3733,8 +3316,8 @@ static bool ksz_support_eee(struct dsa_switch *ds, int port)
 	return false;
 }
 
-static int ksz_set_mac_eee(struct dsa_switch *ds, int port,
-			   struct ethtool_keee *e)
+int ksz_set_mac_eee(struct dsa_switch *ds, int port,
+		    struct ethtool_keee *e)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -3835,19 +3418,9 @@ phy_interface_t ksz_get_xmii(struct ksz_device *dev, int port, bool gbit)
 	return interface;
 }
 
-static void ksz88x3_phylink_mac_config(struct phylink_config *config,
-				       unsigned int mode,
-				       const struct phylink_link_state *state)
-{
-	struct dsa_port *dp = dsa_phylink_to_port(config);
-	struct ksz_device *dev = dp->ds->priv;
-
-	dev->ports[dp->index].manual_flow = !(state->pause & MLO_PAUSE_AN);
-}
-
-static void ksz_phylink_mac_config(struct phylink_config *config,
-				   unsigned int mode,
-				   const struct phylink_link_state *state)
+void ksz_phylink_mac_config(struct phylink_config *config,
+			    unsigned int mode,
+			    const struct phylink_link_state *state)
 {
 	struct dsa_port *dp = dsa_phylink_to_port(config);
 	struct ksz_device *dev = dp->ds->priv;
@@ -3890,106 +3463,6 @@ bool ksz_get_gbit(struct ksz_device *dev, int port)
 	return gbit;
 }
 
-static void ksz_set_gbit(struct ksz_device *dev, int port, bool gbit)
-{
-	const u8 *bitval = dev->info->xmii_ctrl1;
-	const u16 *regs = dev->info->regs;
-	u8 data8;
-
-	ksz_pread8(dev, port, regs[P_XMII_CTRL_1], &data8);
-
-	data8 &= ~P_GMII_1GBIT_M;
-
-	if (gbit)
-		data8 |= FIELD_PREP(P_GMII_1GBIT_M, bitval[P_GMII_1GBIT]);
-	else
-		data8 |= FIELD_PREP(P_GMII_1GBIT_M, bitval[P_GMII_NOT_1GBIT]);
-
-	/* Write the updated value */
-	ksz_pwrite8(dev, port, regs[P_XMII_CTRL_1], data8);
-}
-
-static void ksz_set_100_10mbit(struct ksz_device *dev, int port, int speed)
-{
-	const u8 *bitval = dev->info->xmii_ctrl0;
-	const u16 *regs = dev->info->regs;
-	u8 data8;
-
-	ksz_pread8(dev, port, regs[P_XMII_CTRL_0], &data8);
-
-	data8 &= ~P_MII_100MBIT_M;
-
-	if (speed == SPEED_100)
-		data8 |= FIELD_PREP(P_MII_100MBIT_M, bitval[P_MII_100MBIT]);
-	else
-		data8 |= FIELD_PREP(P_MII_100MBIT_M, bitval[P_MII_10MBIT]);
-
-	/* Write the updated value */
-	ksz_pwrite8(dev, port, regs[P_XMII_CTRL_0], data8);
-}
-
-static void ksz_port_set_xmii_speed(struct ksz_device *dev, int port, int speed)
-{
-	if (speed == SPEED_1000)
-		ksz_set_gbit(dev, port, true);
-	else
-		ksz_set_gbit(dev, port, false);
-
-	if (speed == SPEED_100 || speed == SPEED_10)
-		ksz_set_100_10mbit(dev, port, speed);
-}
-
-static void ksz_duplex_flowctrl(struct ksz_device *dev, int port, int duplex,
-				bool tx_pause, bool rx_pause)
-{
-	const u8 *bitval = dev->info->xmii_ctrl0;
-	const u32 *masks = dev->info->masks;
-	const u16 *regs = dev->info->regs;
-	u8 mask;
-	u8 val;
-
-	mask = P_MII_DUPLEX_M | masks[P_MII_TX_FLOW_CTRL] |
-	       masks[P_MII_RX_FLOW_CTRL];
-
-	if (duplex == DUPLEX_FULL)
-		val = FIELD_PREP(P_MII_DUPLEX_M, bitval[P_MII_FULL_DUPLEX]);
-	else
-		val = FIELD_PREP(P_MII_DUPLEX_M, bitval[P_MII_HALF_DUPLEX]);
-
-	if (tx_pause)
-		val |= masks[P_MII_TX_FLOW_CTRL];
-
-	if (rx_pause)
-		val |= masks[P_MII_RX_FLOW_CTRL];
-
-	ksz_prmw8(dev, port, regs[P_XMII_CTRL_0], mask, val);
-}
-
-static void ksz9477_phylink_mac_link_up(struct phylink_config *config,
-					struct phy_device *phydev,
-					unsigned int mode,
-					phy_interface_t interface,
-					int speed, int duplex, bool tx_pause,
-					bool rx_pause)
-{
-	struct dsa_port *dp = dsa_phylink_to_port(config);
-	struct ksz_device *dev = dp->ds->priv;
-	int port = dp->index;
-	struct ksz_port *p;
-
-	p = &dev->ports[port];
-
-	/* Internal PHYs */
-	if (dev->info->internal_phy[port])
-		return;
-
-	p->speed = speed;
-
-	ksz_port_set_xmii_speed(dev, port, speed);
-
-	ksz_duplex_flowctrl(dev, port, duplex, tx_pause, rx_pause);
-}
-
 static int ksz_switch_detect(struct ksz_device *dev)
 {
 	u8 id1, id2, id4;
@@ -4089,8 +3562,8 @@ static int ksz_switch_detect(struct ksz_device *dev)
 	return 0;
 }
 
-static int ksz_cls_flower_add(struct dsa_switch *ds, int port,
-			      struct flow_cls_offload *cls, bool ingress)
+int ksz_cls_flower_add(struct dsa_switch *ds, int port,
+		       struct flow_cls_offload *cls, bool ingress)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -4110,8 +3583,8 @@ static int ksz_cls_flower_add(struct dsa_switch *ds, int port,
 	return -EOPNOTSUPP;
 }
 
-static int ksz_cls_flower_del(struct dsa_switch *ds, int port,
-			      struct flow_cls_offload *cls, bool ingress)
+int ksz_cls_flower_del(struct dsa_switch *ds, int port,
+		       struct flow_cls_offload *cls, bool ingress)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -4514,8 +3987,8 @@ static int ksz_tc_setup_qdisc_ets(struct dsa_switch *ds, int port,
 	return -EOPNOTSUPP;
 }
 
-static int ksz_setup_tc(struct dsa_switch *ds, int port,
-			enum tc_setup_type type, void *type_data)
+int ksz_setup_tc(struct dsa_switch *ds, int port,
+		 enum tc_setup_type type, void *type_data)
 {
 	switch (type) {
 	case TC_SETUP_QDISC_CBS:
@@ -4574,8 +4047,8 @@ int ksz_handle_wake_reason(struct ksz_device *dev, int port)
  * If enabled and supported, it sets the supported and active WoL
  * flags.
  */
-static void ksz_get_wol(struct dsa_switch *ds, int port,
-			struct ethtool_wolinfo *wol)
+void ksz_get_wol(struct dsa_switch *ds, int port,
+		 struct ethtool_wolinfo *wol)
 {
 	struct ksz_device *dev = ds->priv;
 	const u16 *regs = dev->info->regs;
@@ -4622,8 +4095,8 @@ static void ksz_get_wol(struct dsa_switch *ds, int port,
  *
  * Return: 0 on success, or other error codes on failure.
  */
-static int ksz_set_wol(struct dsa_switch *ds, int port,
-		       struct ethtool_wolinfo *wol)
+int ksz_set_wol(struct dsa_switch *ds, int port,
+		struct ethtool_wolinfo *wol)
 {
 	u8 pme_ctrl = 0, pme_ctrl_old = 0;
 	struct ksz_device *dev = ds->priv;
@@ -4737,8 +4210,8 @@ static void ksz_wol_pre_shutdown(struct ksz_device *dev, bool *wol_enabled)
 	}
 }
 
-static int ksz_port_set_mac_address(struct dsa_switch *ds, int port,
-				    const unsigned char *addr)
+int ksz_port_set_mac_address(struct dsa_switch *ds, int port,
+			     const unsigned char *addr)
 {
 	struct dsa_port *dp = dsa_to_port(ds, port);
 	struct ethtool_wolinfo wol;
@@ -4890,8 +4363,8 @@ void ksz_switch_macaddr_put(struct dsa_switch *ds)
 	kfree(switch_macaddr);
 }
 
-static int ksz_hsr_join(struct dsa_switch *ds, int port, struct net_device *hsr,
-			struct netlink_ext_ack *extack)
+int ksz_hsr_join(struct dsa_switch *ds, int port, struct net_device *hsr,
+		 struct netlink_ext_ack *extack)
 {
 	struct ksz_device *dev = ds->priv;
 	enum hsr_version ver;
@@ -4939,8 +4412,8 @@ static int ksz_hsr_join(struct dsa_switch *ds, int port, struct net_device *hsr,
 	return 0;
 }
 
-static int ksz_hsr_leave(struct dsa_switch *ds, int port,
-			 struct net_device *hsr)
+int ksz_hsr_leave(struct dsa_switch *ds, int port,
+		  struct net_device *hsr)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -4956,7 +4429,7 @@ static int ksz_hsr_leave(struct dsa_switch *ds, int port,
 	return 0;
 }
 
-static int ksz_suspend(struct dsa_switch *ds)
+int ksz_suspend(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -4964,7 +4437,7 @@ static int ksz_suspend(struct dsa_switch *ds)
 	return 0;
 }
 
-static int ksz_resume(struct dsa_switch *ds)
+int ksz_resume(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 
@@ -4973,68 +4446,9 @@ static int ksz_resume(struct dsa_switch *ds)
 	return 0;
 }
 
-static const struct dsa_switch_ops ksz_switch_ops = {
-	.get_tag_protocol	= ksz_get_tag_protocol,
-	.connect_tag_protocol   = ksz_connect_tag_protocol,
-	.get_phy_flags		= ksz_get_phy_flags,
-	.setup			= ksz_setup,
-	.teardown		= ksz_teardown,
-	.phy_read		= ksz_phy_read16,
-	.phy_write		= ksz_phy_write16,
-	.phylink_get_caps	= ksz_phylink_get_caps,
-	.port_setup		= ksz_port_setup,
-	.set_ageing_time	= ksz_set_ageing_time,
-	.get_strings		= ksz_get_strings,
-	.get_ethtool_stats	= ksz_get_ethtool_stats,
-	.get_sset_count		= ksz_sset_count,
-	.port_bridge_join	= ksz_port_bridge_join,
-	.port_bridge_leave	= ksz_port_bridge_leave,
-	.port_hsr_join		= ksz_hsr_join,
-	.port_hsr_leave		= ksz_hsr_leave,
-	.port_set_mac_address	= ksz_port_set_mac_address,
-	.port_stp_state_set	= ksz_port_stp_state_set,
-	.port_teardown		= ksz_port_teardown,
-	.port_pre_bridge_flags	= ksz_port_pre_bridge_flags,
-	.port_bridge_flags	= ksz_port_bridge_flags,
-	.port_fast_age		= ksz_port_fast_age,
-	.port_vlan_filtering	= ksz_port_vlan_filtering,
-	.port_vlan_add		= ksz_port_vlan_add,
-	.port_vlan_del		= ksz_port_vlan_del,
-	.port_fdb_dump		= ksz_port_fdb_dump,
-	.port_fdb_add		= ksz_port_fdb_add,
-	.port_fdb_del		= ksz_port_fdb_del,
-	.port_mdb_add           = ksz_port_mdb_add,
-	.port_mdb_del           = ksz_port_mdb_del,
-	.port_mirror_add	= ksz_port_mirror_add,
-	.port_mirror_del	= ksz_port_mirror_del,
-	.get_stats64		= ksz_get_stats64,
-	.get_pause_stats	= ksz_get_pause_stats,
-	.port_change_mtu	= ksz_change_mtu,
-	.port_max_mtu		= ksz_max_mtu,
-	.get_wol		= ksz_get_wol,
-	.set_wol		= ksz_set_wol,
-	.suspend		= ksz_suspend,
-	.resume			= ksz_resume,
-	.get_ts_info		= ksz_get_ts_info,
-	.port_hwtstamp_get	= ksz_hwtstamp_get,
-	.port_hwtstamp_set	= ksz_hwtstamp_set,
-	.port_txtstamp		= ksz_port_txtstamp,
-	.port_rxtstamp		= ksz_port_rxtstamp,
-	.cls_flower_add		= ksz_cls_flower_add,
-	.cls_flower_del		= ksz_cls_flower_del,
-	.port_setup_tc		= ksz_setup_tc,
-	.support_eee		= ksz_support_eee,
-	.set_mac_eee		= ksz_set_mac_eee,
-	.port_get_default_prio	= ksz_port_get_default_prio,
-	.port_set_default_prio	= ksz_port_set_default_prio,
-	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
-	.port_add_dscp_prio	= ksz_port_add_dscp_prio,
-	.port_del_dscp_prio	= ksz_port_del_dscp_prio,
-	.port_get_apptrust	= ksz_port_get_apptrust,
-	.port_set_apptrust	= ksz_port_set_apptrust,
-};
-
-struct ksz_device *ksz_switch_alloc(struct device *base, void *priv)
+struct ksz_device *ksz_switch_alloc(struct device *base,
+				    const struct ksz_chip_data *chip,
+				    void *priv)
 {
 	struct dsa_switch *ds;
 	struct ksz_device *swdev;
@@ -5045,7 +4459,7 @@ struct ksz_device *ksz_switch_alloc(struct device *base, void *priv)
 
 	ds->dev = base;
 	ds->num_ports = DSA_MAX_PORTS;
-	ds->ops = &ksz_switch_ops;
+	ds->ops = chip->switch_ops;
 
 	swdev = devm_kzalloc(base, sizeof(*swdev), GFP_KERNEL);
 	if (!swdev)
diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h
index 18f13ee9c7b68..f6dad256a550f 100644
--- a/drivers/net/dsa/microchip/ksz_common.h
+++ b/drivers/net/dsa/microchip/ksz_common.h
@@ -73,6 +73,7 @@ struct ksz_chip_data {
 	 */
 	bool phy_side_mdio_supported;
 	const struct ksz_dev_ops *ops;
+	const struct dsa_switch_ops *switch_ops;
 	const struct phylink_mac_ops *phylink_mac_ops;
 	bool phy_errata_9477;
 	bool ksz87xx_eee_link_erratum;
@@ -362,10 +363,7 @@ struct ksz_dev_ops {
 	void (*teardown)(struct dsa_switch *ds);
 	u32 (*get_port_addr)(int port, int offset);
 	void (*cfg_port_member)(struct ksz_device *dev, int port, u8 member);
-	void (*flush_dyn_mac_table)(struct ksz_device *dev, int port);
-	void (*port_cleanup)(struct ksz_device *dev, int port);
 	void (*port_setup)(struct ksz_device *dev, int port, bool cpu_port);
-	int (*set_ageing_time)(struct ksz_device *dev, unsigned int msecs);
 
 	/**
 	 * @mdio_bus_preinit: Function pointer to pre-initialize the MDIO bus
@@ -410,33 +408,6 @@ struct ksz_dev_ops {
 	void (*r_mib_pkt)(struct ksz_device *dev, int port, u16 addr,
 			  u64 *dropped, u64 *cnt);
 	void (*r_mib_stat64)(struct ksz_device *dev, int port);
-	int  (*vlan_filtering)(struct ksz_device *dev, int port,
-			       bool flag, struct netlink_ext_ack *extack);
-	int  (*vlan_add)(struct ksz_device *dev, int port,
-			 const struct switchdev_obj_port_vlan *vlan,
-			 struct netlink_ext_ack *extack);
-	int  (*vlan_del)(struct ksz_device *dev, int port,
-			 const struct switchdev_obj_port_vlan *vlan);
-	int (*mirror_add)(struct ksz_device *dev, int port,
-			  struct dsa_mall_mirror_tc_entry *mirror,
-			  bool ingress, struct netlink_ext_ack *extack);
-	void (*mirror_del)(struct ksz_device *dev, int port,
-			   struct dsa_mall_mirror_tc_entry *mirror);
-	int (*fdb_add)(struct ksz_device *dev, int port,
-		       const unsigned char *addr, u16 vid, struct dsa_db db);
-	int (*fdb_del)(struct ksz_device *dev, int port,
-		       const unsigned char *addr, u16 vid, struct dsa_db db);
-	int (*fdb_dump)(struct ksz_device *dev, int port,
-			dsa_fdb_dump_cb_t *cb, void *data);
-	int (*mdb_add)(struct ksz_device *dev, int port,
-		       const struct switchdev_obj_port_mdb *mdb,
-		       struct dsa_db db);
-	int (*mdb_del)(struct ksz_device *dev, int port,
-		       const struct switchdev_obj_port_mdb *mdb,
-		       struct dsa_db db);
-	void (*get_caps)(struct ksz_device *dev, int port,
-			 struct phylink_config *config);
-	int (*change_mtu)(struct ksz_device *dev, int port, int mtu);
 	int (*pme_write8)(struct ksz_device *dev, u32 reg, u8 value);
 	int (*pme_pread8)(struct ksz_device *dev, int port, int offset,
 			  u8 *data);
@@ -444,11 +415,6 @@ struct ksz_dev_ops {
 			   u8 data);
 	void (*freeze_mib)(struct ksz_device *dev, int port, bool freeze);
 	void (*port_init_cnt)(struct ksz_device *dev, int port);
-	void (*phylink_mac_link_up)(struct ksz_device *dev, int port,
-				    unsigned int mode,
-				    phy_interface_t interface,
-				    struct phy_device *phydev, int speed,
-				    int duplex, bool tx_pause, bool rx_pause);
 	void (*setup_rgmii_delay)(struct ksz_device *dev, int port);
 	int (*tc_cbs_set_cinc)(struct ksz_device *dev, int port, u32 val);
 	void (*config_cpu_port)(struct dsa_switch *ds);
@@ -460,12 +426,19 @@ struct ksz_dev_ops {
 	int (*pcs_create)(struct ksz_device *dev);
 };
 
-struct ksz_device *ksz_switch_alloc(struct device *base, void *priv);
+struct ksz_device *ksz_switch_alloc(struct device *base,
+				    const struct ksz_chip_data *chip,
+				    void *priv);
 int ksz_switch_register(struct ksz_device *dev);
 void ksz_switch_remove(struct ksz_device *dev);
 int ksz_switch_suspend(struct device *dev);
 int ksz_switch_resume(struct device *dev);
 
+int ksz_setup(struct dsa_switch *ds);
+void ksz_teardown(struct dsa_switch *ds);
+int ksz_port_setup(struct dsa_switch *ds, int port);
+void ksz_port_teardown(struct dsa_switch *ds, int port);
+
 void ksz_init_mib_timer(struct ksz_device *dev);
 bool ksz_is_port_mac_global_usable(struct dsa_switch *ds, int port);
 void ksz_r_mib_stats64(struct ksz_device *dev, int port);
@@ -480,6 +453,72 @@ void ksz_switch_macaddr_put(struct dsa_switch *ds);
 void ksz_switch_shutdown(struct ksz_device *dev);
 int ksz_handle_wake_reason(struct ksz_device *dev, int port);
 
+int ksz_phy_read16(struct dsa_switch *ds, int addr, int reg);
+int ksz_phy_write16(struct dsa_switch *ds, int addr, int reg, u16 val);
+u32 ksz_get_phy_flags(struct dsa_switch *ds, int port);
+
+int ksz_sset_count(struct dsa_switch *ds, int port, int sset);
+void ksz_get_ethtool_stats(struct dsa_switch *ds, int port,
+			   uint64_t *buf);
+void ksz_get_stats64(struct dsa_switch *ds, int port,
+		     struct rtnl_link_stats64 *s);
+void ksz_get_pause_stats(struct dsa_switch *ds, int port,
+			 struct ethtool_pause_stats *pause_stats);
+void ksz_get_strings(struct dsa_switch *ds, int port,
+		     u32 stringset, uint8_t *buf);
+
+int ksz_port_bridge_join(struct dsa_switch *ds, int port,
+			 struct dsa_bridge bridge,
+			 bool *tx_fwd_offload,
+			 struct netlink_ext_ack *extack);
+void ksz_port_bridge_leave(struct dsa_switch *ds, int port,
+			   struct dsa_bridge bridge);
+int ksz_port_pre_bridge_flags(struct dsa_switch *ds, int port,
+			      struct switchdev_brport_flags flags,
+			      struct netlink_ext_ack *extack);
+int ksz_port_bridge_flags(struct dsa_switch *ds, int port,
+			  struct switchdev_brport_flags flags,
+			  struct netlink_ext_ack *extack);
+
+void ksz_phylink_get_caps(struct dsa_switch *ds, int port,
+			  struct phylink_config *config);
+void ksz_phylink_mac_disable_tx_lpi(struct phylink_config *config);
+int ksz_phylink_mac_enable_tx_lpi(struct phylink_config *config,
+				  u32 timer, bool tx_clock_stop);
+void ksz_phylink_mac_config(struct phylink_config *config,
+			    unsigned int mode,
+			    const struct phylink_link_state *state);
+void ksz_phylink_mac_link_down(struct phylink_config *config,
+			       unsigned int mode,
+			       phy_interface_t interface);
+
+int ksz_max_mtu(struct dsa_switch *ds, int port);
+
+bool ksz_support_eee(struct dsa_switch *ds, int port);
+int ksz_set_mac_eee(struct dsa_switch *ds, int port,
+		    struct ethtool_keee *e);
+
+int ksz_cls_flower_add(struct dsa_switch *ds, int port,
+		       struct flow_cls_offload *cls, bool ingress);
+int ksz_cls_flower_del(struct dsa_switch *ds, int port,
+		       struct flow_cls_offload *cls, bool ingress);
+int ksz_setup_tc(struct dsa_switch *ds, int port,
+		 enum tc_setup_type type, void *type_data);
+
+void ksz_get_wol(struct dsa_switch *ds, int port,
+		 struct ethtool_wolinfo *wol);
+int ksz_set_wol(struct dsa_switch *ds, int port,
+		struct ethtool_wolinfo *wol);
+int ksz_port_set_mac_address(struct dsa_switch *ds, int port,
+			     const unsigned char *addr);
+int ksz_hsr_join(struct dsa_switch *ds, int port, struct net_device *hsr,
+		 struct netlink_ext_ack *extack);
+int ksz_hsr_leave(struct dsa_switch *ds, int port,
+		  struct net_device *hsr);
+
+int ksz_suspend(struct dsa_switch *ds);
+int ksz_resume(struct dsa_switch *ds);
+
 /* Common register access functions */
 static inline struct regmap *ksz_regmap_8(struct ksz_device *dev)
 {
diff --git a/drivers/net/dsa/microchip/ksz_spi.c b/drivers/net/dsa/microchip/ksz_spi.c
index d8001734b0574..373e9054947cb 100644
--- a/drivers/net/dsa/microchip/ksz_spi.c
+++ b/drivers/net/dsa/microchip/ksz_spi.c
@@ -143,14 +143,14 @@ static int ksz_spi_probe(struct spi_device *spi)
 	struct ksz_device *dev;
 	int i, ret = 0;
 
-	dev = ksz_switch_alloc(&spi->dev, spi);
-	if (!dev)
-		return -ENOMEM;
-
 	chip = device_get_match_data(ddev);
 	if (!chip)
 		return -EINVAL;
 
+	dev = ksz_switch_alloc(&spi->dev, chip, spi);
+	if (!dev)
+		return -ENOMEM;
+
 	/* Save chip id to do special initialization when probing. */
 	dev->chip_id = chip->chip_id;
 	if (chip->chip_id == KSZ88X3_CHIP_ID)
diff --git a/drivers/net/dsa/microchip/lan937x.h b/drivers/net/dsa/microchip/lan937x.h
index df13ebbd356f9..d12483f9081d8 100644
--- a/drivers/net/dsa/microchip/lan937x.h
+++ b/drivers/net/dsa/microchip/lan937x.h
@@ -6,21 +6,8 @@
 #ifndef __LAN937X_CFG_H
 #define __LAN937X_CFG_H
 
-int lan937x_reset_switch(struct ksz_device *dev);
-int lan937x_setup(struct dsa_switch *ds);
-void lan937x_teardown(struct dsa_switch *ds);
-void lan937x_port_setup(struct ksz_device *dev, int port, bool cpu_port);
-void lan937x_config_cpu_port(struct dsa_switch *ds);
-int lan937x_switch_init(struct ksz_device *dev);
-void lan937x_switch_exit(struct ksz_device *dev);
-int lan937x_mdio_bus_preinit(struct ksz_device *dev, bool side_mdio);
-int lan937x_create_phy_addr_map(struct ksz_device *dev, bool side_mdio);
-int lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data);
-int lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val);
-int lan937x_change_mtu(struct ksz_device *dev, int port, int new_mtu);
-void lan937x_phylink_get_caps(struct ksz_device *dev, int port,
-			      struct phylink_config *config);
-void lan937x_setup_rgmii_delay(struct ksz_device *dev, int port);
-int lan937x_set_ageing_time(struct ksz_device *dev, unsigned int msecs);
-int lan937x_tc_cbs_set_cinc(struct ksz_device *dev, int port, u32 val);
+extern const struct ksz_dev_ops lan937x_dev_ops;
+extern const struct phylink_mac_ops lan937x_phylink_mac_ops;
+extern const struct dsa_switch_ops lan937x_switch_ops;
+
 #endif
diff --git a/drivers/net/dsa/microchip/lan937x_main.c b/drivers/net/dsa/microchip/lan937x_main.c
index 5a1496fff4452..f84548168b371 100644
--- a/drivers/net/dsa/microchip/lan937x_main.c
+++ b/drivers/net/dsa/microchip/lan937x_main.c
@@ -2,6 +2,7 @@
 /* Microchip LAN937X switch driver main logic
  * Copyright (C) 2019-2024 Microchip Technology Inc.
  */
+#include <linux/dsa/ksz_common.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/iopoll.h>
@@ -15,6 +16,7 @@
 
 #include "lan937x_reg.h"
 #include "ksz_common.h"
+#include "ksz_dcb.h"
 #include "ksz9477.h"
 #include "lan937x.h"
 
@@ -137,7 +139,7 @@ static int lan937x_port_cfg(struct ksz_device *dev, int port, int offset,
  *
  * Return: 0 on success, error code on failure.
  */
-int lan937x_create_phy_addr_map(struct ksz_device *dev, bool side_mdio)
+static int lan937x_create_phy_addr_map(struct ksz_device *dev, bool side_mdio)
 {
 	static const u8 *phy_addr_map;
 	u32 strap_val;
@@ -221,7 +223,7 @@ int lan937x_create_phy_addr_map(struct ksz_device *dev, bool side_mdio)
  *
  * Return: 0 on success, error code on failure.
  */
-int lan937x_mdio_bus_preinit(struct ksz_device *dev, bool side_mdio)
+static int lan937x_mdio_bus_preinit(struct ksz_device *dev, bool side_mdio)
 {
 	u16 data16;
 	int ret;
@@ -332,17 +334,17 @@ static int lan937x_internal_phy_read(struct ksz_device *dev, int addr, int reg,
 	return ksz_read16(dev, REG_VPHY_IND_DATA__2, val);
 }
 
-int lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
+static int lan937x_r_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 *data)
 {
 	return lan937x_internal_phy_read(dev, addr, reg, data);
 }
 
-int lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val)
+static int lan937x_w_phy(struct ksz_device *dev, u16 addr, u16 reg, u16 val)
 {
 	return lan937x_internal_phy_write(dev, addr, reg, val);
 }
 
-int lan937x_reset_switch(struct ksz_device *dev)
+static int lan937x_reset_switch(struct ksz_device *dev)
 {
 	u32 data32;
 	int ret;
@@ -373,7 +375,7 @@ int lan937x_reset_switch(struct ksz_device *dev)
 	return ksz_read32(dev, REG_SW_PORT_INT_STATUS__4, &data32);
 }
 
-void lan937x_port_setup(struct ksz_device *dev, int port, bool cpu_port)
+static void lan937x_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 {
 	const u32 *masks = dev->info->masks;
 	const u16 *regs = dev->info->regs;
@@ -409,7 +411,7 @@ void lan937x_port_setup(struct ksz_device *dev, int port, bool cpu_port)
 	dev->dev_ops->cfg_port_member(dev, port, member);
 }
 
-void lan937x_config_cpu_port(struct dsa_switch *ds)
+static void lan937x_config_cpu_port(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 	struct dsa_port *dp;
@@ -428,9 +430,9 @@ void lan937x_config_cpu_port(struct dsa_switch *ds)
 	}
 }
 
-int lan937x_change_mtu(struct ksz_device *dev, int port, int new_mtu)
+static int lan937x_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
 {
-	struct dsa_switch *ds = dev->ds;
+	struct ksz_device *dev = ds->priv;
 	int ret;
 
 	new_mtu += VLAN_ETH_HLEN + ETH_FCS_LEN;
@@ -459,8 +461,9 @@ int lan937x_change_mtu(struct ksz_device *dev, int port, int new_mtu)
 	return 0;
 }
 
-int lan937x_set_ageing_time(struct ksz_device *dev, unsigned int msecs)
+static int lan937x_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
 {
+	struct ksz_device *dev = ds->priv;
 	u8 data, mult, value8;
 	bool in_msec = false;
 	u32 max_val, value;
@@ -572,9 +575,11 @@ static void lan937x_set_rgmii_rx_delay(struct ksz_device *dev, int port)
 	lan937x_set_tune_adj(dev, port, REG_PORT_XMII_CTRL_4, val);
 }
 
-void lan937x_phylink_get_caps(struct ksz_device *dev, int port,
-			      struct phylink_config *config)
+static void lan937x_phylink_get_caps(struct dsa_switch *ds, int port,
+				     struct phylink_config *config)
 {
+	struct ksz_device *dev = ds->priv;
+
 	config->mac_capabilities = MAC_100FD;
 
 	if (dev->info->supports_rgmii[port]) {
@@ -585,9 +590,11 @@ void lan937x_phylink_get_caps(struct ksz_device *dev, int port,
 		config->mac_capabilities |= MAC_ASYM_PAUSE | MAC_SYM_PAUSE |
 					    MAC_100HD | MAC_10;
 	}
+
+	ksz_phylink_get_caps(ds, port, config);
 }
 
-void lan937x_setup_rgmii_delay(struct ksz_device *dev, int port)
+static void lan937x_setup_rgmii_delay(struct ksz_device *dev, int port)
 {
 	struct ksz_port *p = &dev->ports[port];
 
@@ -604,19 +611,19 @@ void lan937x_setup_rgmii_delay(struct ksz_device *dev, int port)
 	}
 }
 
-int lan937x_tc_cbs_set_cinc(struct ksz_device *dev, int port, u32 val)
+static int lan937x_tc_cbs_set_cinc(struct ksz_device *dev, int port, u32 val)
 {
 	return ksz_pwrite32(dev, port, REG_PORT_MTI_CREDIT_INCREMENT, val);
 }
 
-int lan937x_switch_init(struct ksz_device *dev)
+static int lan937x_switch_init(struct ksz_device *dev)
 {
 	dev->port_mask = (1 << dev->info->port_cnt) - 1;
 
 	return 0;
 }
 
-int lan937x_setup(struct dsa_switch *ds)
+static int lan937x_setup(struct dsa_switch *ds)
 {
 	struct ksz_device *dev = ds->priv;
 	int ret;
@@ -656,16 +663,130 @@ int lan937x_setup(struct dsa_switch *ds)
 			 SW_VPHY_DISABLE);
 }
 
-void lan937x_teardown(struct dsa_switch *ds)
+static void lan937x_teardown(struct dsa_switch *ds)
 {
 
 }
 
-void lan937x_switch_exit(struct ksz_device *dev)
+static void lan937x_switch_exit(struct ksz_device *dev)
 {
 	lan937x_reset_switch(dev);
 }
 
+static enum dsa_tag_protocol lan937x_get_tag_protocol(struct dsa_switch *ds,
+						      int port,
+						      enum dsa_tag_protocol mp)
+{
+	return DSA_TAG_PROTO_LAN937X;
+}
+
+static int lan937x_connect_tag_protocol(struct dsa_switch *ds,
+					enum dsa_tag_protocol proto)
+{
+	struct ksz_tagger_data *tagger_data;
+
+	if (proto != DSA_TAG_PROTO_LAN937X)
+		return -EPROTONOSUPPORT;
+
+	tagger_data = ksz_tagger_data(ds);
+	tagger_data->xmit_work_fn = ksz_port_deferred_xmit;
+
+	return 0;
+}
+
+const struct phylink_mac_ops lan937x_phylink_mac_ops = {
+	.mac_config	= ksz_phylink_mac_config,
+	.mac_link_down	= ksz_phylink_mac_link_down,
+	.mac_link_up	= ksz9477_phylink_mac_link_up,
+	.mac_disable_tx_lpi = ksz_phylink_mac_disable_tx_lpi,
+	.mac_enable_tx_lpi = ksz_phylink_mac_enable_tx_lpi,
+};
+
+const struct ksz_dev_ops lan937x_dev_ops = {
+	.setup = lan937x_setup,
+	.teardown = lan937x_teardown,
+	.get_port_addr = ksz9477_get_port_addr,
+	.cfg_port_member = ksz9477_cfg_port_member,
+	.port_setup = lan937x_port_setup,
+	.mdio_bus_preinit = lan937x_mdio_bus_preinit,
+	.create_phy_addr_map = lan937x_create_phy_addr_map,
+	.r_phy = lan937x_r_phy,
+	.w_phy = lan937x_w_phy,
+	.r_mib_cnt = ksz9477_r_mib_cnt,
+	.r_mib_pkt = ksz9477_r_mib_pkt,
+	.r_mib_stat64 = ksz_r_mib_stats64,
+	.freeze_mib = ksz9477_freeze_mib,
+	.port_init_cnt = ksz9477_port_init_cnt,
+	.setup_rgmii_delay = lan937x_setup_rgmii_delay,
+	.config_cpu_port = lan937x_config_cpu_port,
+	.tc_cbs_set_cinc = lan937x_tc_cbs_set_cinc,
+	.enable_stp_addr = ksz9477_enable_stp_addr,
+	.reset = lan937x_reset_switch,
+	.init = lan937x_switch_init,
+	.exit = lan937x_switch_exit,
+};
+
+const struct dsa_switch_ops lan937x_switch_ops = {
+	.get_tag_protocol	= lan937x_get_tag_protocol,
+	.connect_tag_protocol   = lan937x_connect_tag_protocol,
+	.get_phy_flags		= ksz_get_phy_flags,
+	.setup			= ksz_setup,
+	.teardown		= ksz_teardown,
+	.phy_read		= ksz_phy_read16,
+	.phy_write		= ksz_phy_write16,
+	.phylink_get_caps	= lan937x_phylink_get_caps,
+	.port_setup		= ksz_port_setup,
+	.set_ageing_time	= lan937x_set_ageing_time,
+	.get_strings		= ksz_get_strings,
+	.get_ethtool_stats	= ksz_get_ethtool_stats,
+	.get_sset_count		= ksz_sset_count,
+	.port_bridge_join	= ksz_port_bridge_join,
+	.port_bridge_leave	= ksz_port_bridge_leave,
+	.port_hsr_join		= ksz_hsr_join,
+	.port_hsr_leave		= ksz_hsr_leave,
+	.port_set_mac_address	= ksz_port_set_mac_address,
+	.port_stp_state_set	= ksz_port_stp_state_set,
+	.port_teardown		= ksz_port_teardown,
+	.port_pre_bridge_flags	= ksz_port_pre_bridge_flags,
+	.port_bridge_flags	= ksz_port_bridge_flags,
+	.port_fast_age		= ksz9477_flush_dyn_mac_table,
+	.port_vlan_filtering	= ksz9477_port_vlan_filtering,
+	.port_vlan_add		= ksz9477_port_vlan_add,
+	.port_vlan_del		= ksz9477_port_vlan_del,
+	.port_fdb_dump		= ksz9477_fdb_dump,
+	.port_fdb_add		= ksz9477_fdb_add,
+	.port_fdb_del		= ksz9477_fdb_del,
+	.port_mdb_add           = ksz9477_mdb_add,
+	.port_mdb_del           = ksz9477_mdb_del,
+	.port_mirror_add	= ksz9477_port_mirror_add,
+	.port_mirror_del	= ksz9477_port_mirror_del,
+	.get_stats64		= ksz_get_stats64,
+	.get_pause_stats	= ksz_get_pause_stats,
+	.port_change_mtu	= lan937x_change_mtu,
+	.port_max_mtu		= ksz_max_mtu,
+	.get_wol		= ksz_get_wol,
+	.set_wol		= ksz_set_wol,
+	.suspend		= ksz_suspend,
+	.resume			= ksz_resume,
+	.get_ts_info		= ksz_get_ts_info,
+	.port_hwtstamp_get	= ksz_hwtstamp_get,
+	.port_hwtstamp_set	= ksz_hwtstamp_set,
+	.port_txtstamp		= ksz_port_txtstamp,
+	.port_rxtstamp		= ksz_port_rxtstamp,
+	.cls_flower_add		= ksz_cls_flower_add,
+	.cls_flower_del		= ksz_cls_flower_del,
+	.port_setup_tc		= ksz_setup_tc,
+	.support_eee		= ksz_support_eee,
+	.set_mac_eee		= ksz_set_mac_eee,
+	.port_get_default_prio	= ksz_port_get_default_prio,
+	.port_set_default_prio	= ksz_port_set_default_prio,
+	.port_get_dscp_prio	= ksz_port_get_dscp_prio,
+	.port_add_dscp_prio	= ksz_port_add_dscp_prio,
+	.port_del_dscp_prio	= ksz_port_del_dscp_prio,
+	.port_get_apptrust	= ksz_port_get_apptrust,
+	.port_set_apptrust	= ksz_port_set_apptrust,
+};
+
 MODULE_AUTHOR("Arun Ramadoss <arun.ramadoss@microchip.com>");
 MODULE_DESCRIPTION("Microchip LAN937x Series Switch DSA Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c
index 44d670904ad89..3c2a3029b10cf 100644
--- a/drivers/net/dsa/mt7530.c
+++ b/drivers/net/dsa/mt7530.c
@@ -1023,12 +1023,16 @@ mt7530_set_ageing_time(struct dsa_switch *ds, unsigned int msecs)
 	unsigned int age_count;
 	unsigned int age_unit;
 
-	/* Applied timer is (AGE_CNT + 1) * (AGE_UNIT + 1) seconds */
-	if (secs < 1 || secs > (AGE_CNT_MAX + 1) * (AGE_UNIT_MAX + 1))
-		return -ERANGE;
-
-	/* iterate through all possible age_count to find the closest pair */
-	for (tmp_age_count = 0; tmp_age_count <= AGE_CNT_MAX; ++tmp_age_count) {
+	/* Applied timer is (AGE_CNT + 1) * (AGE_UNIT + 1) seconds.
+	 * The DSA core has already validated the range using
+	 * ds->ageing_time_min and ds->ageing_time_max.
+	 *
+	 * Iterate through all possible age_count values to find the closest
+	 * pair. Start from 1 because the per-entry aging counter is
+	 * initialized to AGE_CNT and a value of 0 means the entry will
+	 * never be aged out.
+	 */
+	for (tmp_age_count = 1; tmp_age_count <= AGE_CNT_MAX; ++tmp_age_count) {
 		unsigned int tmp_age_unit = secs / (tmp_age_count + 1) - 1;
 
 		if (tmp_age_unit <= AGE_UNIT_MAX) {
@@ -1296,37 +1300,40 @@ static void mt7530_setup_port5(struct dsa_switch *ds, phy_interface_t interface)
 static void
 mt753x_trap_frames(struct mt7530_priv *priv)
 {
-	/* Trap 802.1X PAE frames and BPDUs to the CPU port(s) and egress them
-	 * VLAN-untagged.
+	/* Trap 802.1X PAE frames and BPDUs to the CPU port(s) and egress
+	 * them with the EG_TAG attribute set to disabled (system default)
+	 * so that any VLAN tags in the frame are not modified by the
+	 * switch egress VLAN tag processing. This preserves VLAN tags
+	 * for reception on VLAN sub-interfaces.
 	 */
 	mt7530_rmw(priv, MT753X_BPC,
 		   PAE_BPDU_FR | PAE_EG_TAG_MASK | PAE_PORT_FW_MASK |
 			   BPDU_EG_TAG_MASK | BPDU_PORT_FW_MASK,
-		   PAE_BPDU_FR | PAE_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+		   PAE_BPDU_FR | PAE_EG_TAG(MT7530_VLAN_EG_DISABLED) |
 			   PAE_PORT_FW(TO_CPU_FW_CPU_ONLY) |
-			   BPDU_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+			   BPDU_EG_TAG(MT7530_VLAN_EG_DISABLED) |
 			   TO_CPU_FW_CPU_ONLY);
 
-	/* Trap frames with :01 and :02 MAC DAs to the CPU port(s) and egress
-	 * them VLAN-untagged.
+	/* Trap frames with :01 and :02 MAC DAs to the CPU port(s) and
+	 * egress them with EG_TAG disabled.
 	 */
 	mt7530_rmw(priv, MT753X_RGAC1,
 		   R02_BPDU_FR | R02_EG_TAG_MASK | R02_PORT_FW_MASK |
 			   R01_BPDU_FR | R01_EG_TAG_MASK | R01_PORT_FW_MASK,
-		   R02_BPDU_FR | R02_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+		   R02_BPDU_FR | R02_EG_TAG(MT7530_VLAN_EG_DISABLED) |
 			   R02_PORT_FW(TO_CPU_FW_CPU_ONLY) | R01_BPDU_FR |
-			   R01_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+			   R01_EG_TAG(MT7530_VLAN_EG_DISABLED) |
 			   TO_CPU_FW_CPU_ONLY);
 
-	/* Trap frames with :03 and :0E MAC DAs to the CPU port(s) and egress
-	 * them VLAN-untagged.
+	/* Trap frames with :03 and :0E MAC DAs to the CPU port(s) and
+	 * egress them with EG_TAG disabled.
 	 */
 	mt7530_rmw(priv, MT753X_RGAC2,
 		   R0E_BPDU_FR | R0E_EG_TAG_MASK | R0E_PORT_FW_MASK |
 			   R03_BPDU_FR | R03_EG_TAG_MASK | R03_PORT_FW_MASK,
-		   R0E_BPDU_FR | R0E_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+		   R0E_BPDU_FR | R0E_EG_TAG(MT7530_VLAN_EG_DISABLED) |
 			   R0E_PORT_FW(TO_CPU_FW_CPU_ONLY) | R03_BPDU_FR |
-			   R03_EG_TAG(MT7530_VLAN_EG_UNTAGGED) |
+			   R03_EG_TAG(MT7530_VLAN_EG_DISABLED) |
 			   TO_CPU_FW_CPU_ONLY);
 }
 
@@ -1616,6 +1623,49 @@ mt7530_port_bridge_join(struct dsa_switch *ds, int port,
 	return 0;
 }
 
+static int
+mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
+{
+	struct mt7530_dummy_poll p;
+	u32 val;
+	int ret;
+
+	val = VTCR_BUSY | VTCR_FUNC(cmd) | vid;
+	mt7530_write(priv, MT7530_VTCR, val);
+
+	INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_VTCR);
+	ret = readx_poll_timeout(_mt7530_read, &p, val,
+				 !(val & VTCR_BUSY), 20, 20000);
+	if (ret < 0) {
+		dev_err(priv->dev, "poll timeout\n");
+		return ret;
+	}
+
+	val = mt7530_read(priv, MT7530_VTCR);
+	if (val & VTCR_INVALID) {
+		dev_err(priv->dev, "read VTCR invalid\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int
+mt7530_setup_vlan0(struct mt7530_priv *priv)
+{
+	u32 val;
+
+	/* Validate the entry with independent learning, keep the original
+	 * ingress tag attribute.
+	 */
+	val = IVL_MAC | EG_CON | PORT_MEM(MT7530_ALL_MEMBERS) | FID(FID_BRIDGED) |
+	      VLAN_VALID;
+	mt7530_write(priv, MT7530_VAWD1, val);
+	mt7530_write(priv, MT7530_VAWD2, 0);
+
+	return mt7530_vlan_cmd(priv, MT7530_VTCR_WR_VID, 0);
+}
+
 static void
 mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port)
 {
@@ -1641,6 +1691,8 @@ mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port)
 		   G0_PORT_VID_DEF);
 
 	for (i = 0; i < priv->ds->num_ports; i++) {
+		if (i == port)
+			continue;
 		if (dsa_is_user_port(ds, i) &&
 		    dsa_port_is_vlan_filtering(dsa_to_port(ds, i))) {
 			all_user_ports_removed = false;
@@ -1652,13 +1704,9 @@ mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port)
 	 * the CPU port get out of VLAN filtering mode.
 	 */
 	if (all_user_ports_removed) {
-		struct dsa_port *dp = dsa_to_port(ds, port);
-		struct dsa_port *cpu_dp = dp->cpu_dp;
-
-		mt7530_write(priv, MT7530_PCR_P(cpu_dp->index),
-			     PCR_MATRIX(dsa_user_ports(priv->ds)));
-		mt7530_write(priv, MT7530_PVC_P(cpu_dp->index), PORT_SPEC_TAG
-			     | PVC_EG_TAG(MT7530_VLAN_EG_CONSISTENT));
+		mutex_lock(&priv->reg_mutex);
+		mt7530_setup_vlan0(priv);
+		mutex_unlock(&priv->reg_mutex);
 	}
 }
 
@@ -1846,33 +1894,6 @@ mt7530_port_mdb_del(struct dsa_switch *ds, int port,
 	return ret;
 }
 
-static int
-mt7530_vlan_cmd(struct mt7530_priv *priv, enum mt7530_vlan_cmd cmd, u16 vid)
-{
-	struct mt7530_dummy_poll p;
-	u32 val;
-	int ret;
-
-	val = VTCR_BUSY | VTCR_FUNC(cmd) | vid;
-	mt7530_write(priv, MT7530_VTCR, val);
-
-	INIT_MT7530_DUMMY_POLL(&p, priv, MT7530_VTCR);
-	ret = readx_poll_timeout(_mt7530_read, &p, val,
-				 !(val & VTCR_BUSY), 20, 20000);
-	if (ret < 0) {
-		dev_err(priv->dev, "poll timeout\n");
-		return ret;
-	}
-
-	val = mt7530_read(priv, MT7530_VTCR);
-	if (val & VTCR_INVALID) {
-		dev_err(priv->dev, "read VTCR invalid\n");
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
 static int
 mt7530_port_vlan_filtering(struct dsa_switch *ds, int port, bool vlan_filtering,
 			   struct netlink_ext_ack *extack)
@@ -1977,21 +1998,6 @@ mt7530_hw_vlan_update(struct mt7530_priv *priv, u16 vid,
 	mt7530_vlan_cmd(priv, MT7530_VTCR_WR_VID, vid);
 }
 
-static int
-mt7530_setup_vlan0(struct mt7530_priv *priv)
-{
-	u32 val;
-
-	/* Validate the entry with independent learning, keep the original
-	 * ingress tag attribute.
-	 */
-	val = IVL_MAC | EG_CON | PORT_MEM(MT7530_ALL_MEMBERS) | FID(FID_BRIDGED) |
-	      VLAN_VALID;
-	mt7530_write(priv, MT7530_VAWD1, val);
-
-	return mt7530_vlan_cmd(priv, MT7530_VTCR_WR_VID, 0);
-}
-
 static int
 mt7530_port_vlan_add(struct dsa_switch *ds, int port,
 		     const struct switchdev_obj_port_vlan *vlan,
@@ -2004,9 +2010,18 @@ mt7530_port_vlan_add(struct dsa_switch *ds, int port,
 
 	mutex_lock(&priv->reg_mutex);
 
+	/* VID 0 is managed exclusively by mt7530_setup_vlan0() for
+	 * VLAN-unaware bridge operation. Don't let the bridge overwrite
+	 * its EG_CON flag with VTAG_EN and corrupt PORT_MEM.
+	 */
+	if (vlan->vid == 0)
+		goto skip_vlan_table;
+
 	mt7530_hw_vlan_entry_init(&new_entry, port, untagged);
 	mt7530_hw_vlan_update(priv, vlan->vid, &new_entry, mt7530_hw_vlan_add);
 
+skip_vlan_table:
+
 	if (pvid) {
 		priv->ports[port].pvid = vlan->vid;
 
@@ -2046,10 +2061,15 @@ mt7530_port_vlan_del(struct dsa_switch *ds, int port,
 
 	mutex_lock(&priv->reg_mutex);
 
+	/* VID 0 is managed exclusively by mt7530_setup_vlan0(). */
+	if (vlan->vid == 0)
+		goto skip_vlan_table;
+
 	mt7530_hw_vlan_entry_init(&target_entry, port, 0);
 	mt7530_hw_vlan_update(priv, vlan->vid, &target_entry,
 			      mt7530_hw_vlan_del);
 
+skip_vlan_table:
 	/* PVID is being restored to the default whenever the PVID port
 	 * is being removed from the VLAN.
 	 */
@@ -2427,7 +2447,10 @@ mt7530_setup(struct dsa_switch *ds)
 	}
 
 	ds->assisted_learning_on_cpu_port = true;
+	ds->untag_vlan_aware_bridge_pvid = true;
 	ds->mtu_enforcement_ingress = true;
+	ds->ageing_time_min = 2 * 1000;
+	ds->ageing_time_max = (AGE_CNT_MAX + 1) * (AGE_UNIT_MAX + 1) * 1000;
 
 	if (priv->id == ID_MT7530) {
 		regulator_set_voltage(priv->core_pwr, 1000000, 1000000);
@@ -2616,7 +2639,10 @@ mt7531_setup_common(struct dsa_switch *ds)
 	int ret, i;
 
 	ds->assisted_learning_on_cpu_port = true;
+	ds->untag_vlan_aware_bridge_pvid = true;
 	ds->mtu_enforcement_ingress = true;
+	ds->ageing_time_min = 2 * 1000;
+	ds->ageing_time_max = (AGE_CNT_MAX + 1) * (AGE_UNIT_MAX + 1) * 1000;
 
 	mt753x_trap_frames(priv);
 
diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c
index 8ca5fd40df92e..4234755100ea6 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.c
+++ b/drivers/net/dsa/mv88e6xxx/chip.c
@@ -4434,7 +4434,6 @@ static const struct mv88e6xxx_ops mv88e6141_ops = {
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
 	.port_set_speed_duplex = mv88e6341_port_set_speed_duplex,
-	.port_max_speed_mode = mv88e6341_port_max_speed_mode,
 	.port_tag_remap = mv88e6095_port_tag_remap,
 	.port_set_policy = mv88e6352_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -4826,7 +4825,6 @@ static const struct mv88e6xxx_ops mv88e6190_ops = {
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
 	.port_set_speed_duplex = mv88e6390_port_set_speed_duplex,
-	.port_max_speed_mode = mv88e6390_port_max_speed_mode,
 	.port_tag_remap = mv88e6390_port_tag_remap,
 	.port_set_policy = mv88e6352_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -4886,7 +4884,6 @@ static const struct mv88e6xxx_ops mv88e6190x_ops = {
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
 	.port_set_speed_duplex = mv88e6390x_port_set_speed_duplex,
-	.port_max_speed_mode = mv88e6390x_port_max_speed_mode,
 	.port_tag_remap = mv88e6390_port_tag_remap,
 	.port_set_policy = mv88e6352_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -4946,7 +4943,6 @@ static const struct mv88e6xxx_ops mv88e6191_ops = {
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
 	.port_set_speed_duplex = mv88e6390_port_set_speed_duplex,
-	.port_max_speed_mode = mv88e6390_port_max_speed_mode,
 	.port_tag_remap = mv88e6390_port_tag_remap,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
 	.port_set_ucast_flood = mv88e6352_port_set_ucast_flood,
@@ -5110,7 +5106,6 @@ static const struct mv88e6xxx_ops mv88e6290_ops = {
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
 	.port_set_speed_duplex = mv88e6390_port_set_speed_duplex,
-	.port_max_speed_mode = mv88e6390_port_max_speed_mode,
 	.port_tag_remap = mv88e6390_port_tag_remap,
 	.port_set_policy = mv88e6352_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -5173,7 +5168,7 @@ static const struct mv88e6xxx_ops mv88e6320_ops = {
 	.port_set_link = mv88e6xxx_port_set_link,
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6320_port_set_rgmii_delay,
-	.port_set_speed_duplex = mv88e6185_port_set_speed_duplex,
+	.port_set_speed_duplex = mv88e6320_port_set_speed_duplex,
 	.port_tag_remap = mv88e6095_port_tag_remap,
 	.port_set_policy = mv88e6352_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -5200,6 +5195,9 @@ static const struct mv88e6xxx_ops mv88e6320_ops = {
 	.hardware_reset_pre = mv88e6xxx_g2_eeprom_wait,
 	.hardware_reset_post = mv88e6xxx_g2_eeprom_wait,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6352_g1_rmu_disable,
+	.atu_get_hash = mv88e6165_g1_atu_get_hash,
+	.atu_set_hash = mv88e6165_g1_atu_set_hash,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
 	.stu_getnext = mv88e6352_g1_stu_getnext,
@@ -5226,7 +5224,7 @@ static const struct mv88e6xxx_ops mv88e6321_ops = {
 	.port_set_link = mv88e6xxx_port_set_link,
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6320_port_set_rgmii_delay,
-	.port_set_speed_duplex = mv88e6185_port_set_speed_duplex,
+	.port_set_speed_duplex = mv88e6320_port_set_speed_duplex,
 	.port_tag_remap = mv88e6095_port_tag_remap,
 	.port_set_policy = mv88e6352_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -5249,9 +5247,13 @@ static const struct mv88e6xxx_ops mv88e6321_ops = {
 	.set_egress_port = mv88e6095_g1_set_egress_port,
 	.watchdog_ops = &mv88e6390_watchdog_ops,
 	.mgmt_rsvd2cpu = mv88e6352_g2_mgmt_rsvd2cpu,
+	.pot_clear = mv88e6xxx_g2_pot_clear,
 	.hardware_reset_pre = mv88e6xxx_g2_eeprom_wait,
 	.hardware_reset_post = mv88e6xxx_g2_eeprom_wait,
 	.reset = mv88e6352_g1_reset,
+	.rmu_disable = mv88e6352_g1_rmu_disable,
+	.atu_get_hash = mv88e6165_g1_atu_get_hash,
+	.atu_set_hash = mv88e6165_g1_atu_set_hash,
 	.vtu_getnext = mv88e6352_g1_vtu_getnext,
 	.vtu_loadpurge = mv88e6352_g1_vtu_loadpurge,
 	.stu_getnext = mv88e6352_g1_stu_getnext,
@@ -5278,7 +5280,6 @@ static const struct mv88e6xxx_ops mv88e6341_ops = {
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
 	.port_set_speed_duplex = mv88e6341_port_set_speed_duplex,
-	.port_max_speed_mode = mv88e6341_port_max_speed_mode,
 	.port_tag_remap = mv88e6095_port_tag_remap,
 	.port_set_policy = mv88e6352_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -5499,7 +5500,6 @@ static const struct mv88e6xxx_ops mv88e6390_ops = {
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
 	.port_set_speed_duplex = mv88e6390_port_set_speed_duplex,
-	.port_max_speed_mode = mv88e6390_port_max_speed_mode,
 	.port_tag_remap = mv88e6390_port_tag_remap,
 	.port_set_policy = mv88e6352_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -5564,7 +5564,6 @@ static const struct mv88e6xxx_ops mv88e6390x_ops = {
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
 	.port_set_speed_duplex = mv88e6390x_port_set_speed_duplex,
-	.port_max_speed_mode = mv88e6390x_port_max_speed_mode,
 	.port_tag_remap = mv88e6390_port_tag_remap,
 	.port_set_policy = mv88e6352_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -5627,7 +5626,6 @@ static const struct mv88e6xxx_ops mv88e6393x_ops = {
 	.port_sync_link = mv88e6xxx_port_sync_link,
 	.port_set_rgmii_delay = mv88e6390_port_set_rgmii_delay,
 	.port_set_speed_duplex = mv88e6393x_port_set_speed_duplex,
-	.port_max_speed_mode = mv88e6393x_port_max_speed_mode,
 	.port_tag_remap = mv88e6390_port_tag_remap,
 	.port_set_policy = mv88e6393x_port_set_policy,
 	.port_set_frame_mode = mv88e6351_port_set_frame_mode,
@@ -6289,7 +6287,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.global1_addr = 0x1b,
 		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
-		.g1_irqs = 8,
+		.g1_irqs = 9,
 		.g2_irqs = 10,
 		.stats_type = STATS_TYPE_BANK0 | STATS_TYPE_BANK1,
 		.atu_move_port_mask = 0xf,
@@ -6317,7 +6315,7 @@ static const struct mv88e6xxx_info mv88e6xxx_table[] = {
 		.global1_addr = 0x1b,
 		.global2_addr = 0x1c,
 		.age_time_coeff = 15000,
-		.g1_irqs = 8,
+		.g1_irqs = 9,
 		.g2_irqs = 10,
 		.stats_type = STATS_TYPE_BANK0 | STATS_TYPE_BANK1,
 		.atu_move_port_mask = 0xf,
diff --git a/drivers/net/dsa/mv88e6xxx/chip.h b/drivers/net/dsa/mv88e6xxx/chip.h
index 2b235ac2c5df5..4339f6810535e 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.h
+++ b/drivers/net/dsa/mv88e6xxx/chip.h
@@ -585,10 +585,6 @@ struct mv88e6xxx_ops {
 	int (*port_set_speed_duplex)(struct mv88e6xxx_chip *chip, int port,
 				     int speed, int duplex);
 
-	/* What interface mode should be used for maximum speed? */
-	phy_interface_t (*port_max_speed_mode)(struct mv88e6xxx_chip *chip,
-					       int port);
-
 	int (*port_tag_remap)(struct mv88e6xxx_chip *chip, int port);
 
 	int (*port_set_policy)(struct mv88e6xxx_chip *chip, int port,
diff --git a/drivers/net/dsa/mv88e6xxx/port.c b/drivers/net/dsa/mv88e6xxx/port.c
index 49cd82930b7a5..ea1fab71968a0 100644
--- a/drivers/net/dsa/mv88e6xxx/port.c
+++ b/drivers/net/dsa/mv88e6xxx/port.c
@@ -327,6 +327,21 @@ int mv88e6250_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 					       duplex);
 }
 
+int mv88e6320_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
+				    int speed, int duplex)
+{
+	bool has_200 = (port == 2 || port == 5 || port == 6);
+
+	if (speed > 1000)
+		return -EOPNOTSUPP;
+
+	if (speed == 200 && !has_200)
+		return -EOPNOTSUPP;
+
+	return mv88e6xxx_port_set_speed_duplex(chip, port, speed, has_200,
+					       false, duplex);
+}
+
 /* Support 10, 100, 200, 1000, 2500 Mbps (e.g. 88E6341) */
 int mv88e6341_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 				    int speed, int duplex)
@@ -344,15 +359,6 @@ int mv88e6341_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 					       duplex);
 }
 
-phy_interface_t mv88e6341_port_max_speed_mode(struct mv88e6xxx_chip *chip,
-					      int port)
-{
-	if (port == 5)
-		return PHY_INTERFACE_MODE_2500BASEX;
-
-	return PHY_INTERFACE_MODE_NA;
-}
-
 /* Support 10, 100, 200, 1000 Mbps (e.g. 88E6352 family) */
 int mv88e6352_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 				    int speed, int duplex)
@@ -384,15 +390,6 @@ int mv88e6390_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 					       duplex);
 }
 
-phy_interface_t mv88e6390_port_max_speed_mode(struct mv88e6xxx_chip *chip,
-					      int port)
-{
-	if (port == 9 || port == 10)
-		return PHY_INTERFACE_MODE_2500BASEX;
-
-	return PHY_INTERFACE_MODE_NA;
-}
-
 /* Support 10, 100, 200, 1000, 2500, 10000 Mbps (e.g. 88E6190X) */
 int mv88e6390x_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 				     int speed, int duplex)
@@ -407,15 +404,6 @@ int mv88e6390x_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 					       duplex);
 }
 
-phy_interface_t mv88e6390x_port_max_speed_mode(struct mv88e6xxx_chip *chip,
-					       int port)
-{
-	if (port == 9 || port == 10)
-		return PHY_INTERFACE_MODE_XAUI;
-
-	return PHY_INTERFACE_MODE_NA;
-}
-
 /* Support 10, 100, 200, 1000, 2500, 5000, 10000 Mbps (e.g. 88E6393X)
  * Function mv88e6xxx_port_set_speed_duplex() can't be used as the register
  * values for speeds 2500 & 5000 conflict.
@@ -509,19 +497,6 @@ int mv88e6393x_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 	return 0;
 }
 
-phy_interface_t mv88e6393x_port_max_speed_mode(struct mv88e6xxx_chip *chip,
-					       int port)
-{
-
-	if (port != 0 && port != 9 && port != 10)
-		return PHY_INTERFACE_MODE_NA;
-
-	if (chip->info->prod_num == MV88E6XXX_PORT_SWITCH_ID_PROD_6361)
-		return PHY_INTERFACE_MODE_2500BASEX;
-
-	return PHY_INTERFACE_MODE_10GBASER;
-}
-
 static int mv88e6xxx_port_set_cmode(struct mv88e6xxx_chip *chip, int port,
 				    phy_interface_t mode, bool force)
 {
diff --git a/drivers/net/dsa/mv88e6xxx/port.h b/drivers/net/dsa/mv88e6xxx/port.h
index a2492cf4d920f..5b6cde9f7406f 100644
--- a/drivers/net/dsa/mv88e6xxx/port.h
+++ b/drivers/net/dsa/mv88e6xxx/port.h
@@ -493,6 +493,8 @@ int mv88e6185_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 				    int speed, int duplex);
 int mv88e6250_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 				    int speed, int duplex);
+int mv88e6320_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
+				    int speed, int duplex);
 int mv88e6341_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 				    int speed, int duplex);
 int mv88e6352_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
@@ -504,15 +506,6 @@ int mv88e6390x_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 int mv88e6393x_port_set_speed_duplex(struct mv88e6xxx_chip *chip, int port,
 				     int speed, int duplex);
 
-phy_interface_t mv88e6341_port_max_speed_mode(struct mv88e6xxx_chip *chip,
-					      int port);
-phy_interface_t mv88e6390_port_max_speed_mode(struct mv88e6xxx_chip *chip,
-					      int port);
-phy_interface_t mv88e6390x_port_max_speed_mode(struct mv88e6xxx_chip *chip,
-					       int port);
-phy_interface_t mv88e6393x_port_max_speed_mode(struct mv88e6xxx_chip *chip,
-					       int port);
-
 int mv88e6xxx_port_set_state(struct mv88e6xxx_chip *chip, int port, u8 state);
 
 int mv88e6xxx_port_set_vlan_map(struct mv88e6xxx_chip *chip, int port, u16 map);
diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c
index 84cf8e7fb17a5..4272ea6e9ca84 100644
--- a/drivers/net/dsa/ocelot/felix.c
+++ b/drivers/net/dsa/ocelot/felix.c
@@ -2001,7 +2001,8 @@ static int felix_cls_flower_stats(struct dsa_switch *ds, int port,
 }
 
 static int felix_port_policer_add(struct dsa_switch *ds, int port,
-				  const struct flow_action_police *policer)
+				  const struct flow_action_police *policer,
+				  struct netlink_ext_ack *extack)
 {
 	struct ocelot *ocelot = ds->priv;
 	struct ocelot_policer pol = {
diff --git a/drivers/net/dsa/realtek/rtl8365mb.c b/drivers/net/dsa/realtek/rtl8365mb.c
index c35cef01ec265..0da048da533a6 100644
--- a/drivers/net/dsa/realtek/rtl8365mb.c
+++ b/drivers/net/dsa/realtek/rtl8365mb.c
@@ -544,6 +544,20 @@ static const struct rtl8365mb_chip_info rtl8365mb_chip_infos[] = {
 		.jam_table = rtl8365mb_init_jam_8365mb_vc,
 		.jam_size = ARRAY_SIZE(rtl8365mb_init_jam_8365mb_vc),
 	},
+	{
+		.name = "RTL8367SB",
+		.chip_id = 0x6367,
+		.chip_ver = 0x0010,
+		.extints = {
+			{ 6, 1, PHY_INTF(MII) | PHY_INTF(TMII) |
+				PHY_INTF(RMII) | PHY_INTF(RGMII) |
+				PHY_INTF(SGMII) | PHY_INTF(HSGMII) },
+			{ 7, 2, PHY_INTF(MII) | PHY_INTF(TMII) |
+				PHY_INTF(RMII) | PHY_INTF(RGMII) },
+		},
+		.jam_table = rtl8365mb_init_jam_8365mb_vc,
+		.jam_size = ARRAY_SIZE(rtl8365mb_init_jam_8365mb_vc),
+	},
 	{
 		.name = "RTL8367RB-VB",
 		.chip_id = 0x6367,
diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c
index c72c2bfdcffb2..dbfa45064747c 100644
--- a/drivers/net/dsa/sja1105/sja1105_main.c
+++ b/drivers/net/dsa/sja1105/sja1105_main.c
@@ -2847,7 +2847,8 @@ static void sja1105_mirror_del(struct dsa_switch *ds, int port,
 }
 
 static int sja1105_port_policer_add(struct dsa_switch *ds, int port,
-				    const struct flow_action_police *policer)
+				    const struct flow_action_police *policer,
+				    struct netlink_ext_ack *extack)
 {
 	struct sja1105_l2_policing_entry *policing;
 	struct sja1105_private *priv = ds->priv;
diff --git a/drivers/net/dsa/xrs700x/xrs700x_i2c.c b/drivers/net/dsa/xrs700x/xrs700x_i2c.c
index 9b731dea78c14..e6a7839c4a04d 100644
--- a/drivers/net/dsa/xrs700x/xrs700x_i2c.c
+++ b/drivers/net/dsa/xrs700x/xrs700x_i2c.c
@@ -127,8 +127,8 @@ static void xrs700x_i2c_shutdown(struct i2c_client *i2c)
 }
 
 static const struct i2c_device_id xrs700x_i2c_id[] = {
-	{ "xrs700x-switch" },
-	{}
+	{ .name = "xrs700x-switch" },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(i2c, xrs700x_i2c_id);
diff --git a/drivers/net/dsa/yt921x.c b/drivers/net/dsa/yt921x.c
index 5b66109ecc235..42dca9617fb17 100644
--- a/drivers/net/dsa/yt921x.c
+++ b/drivers/net/dsa/yt921x.c
@@ -255,63 +255,136 @@ yt921x_reg_toggle_bits(struct yt921x_priv *priv, u32 reg, u32 mask, bool set)
 	return yt921x_reg_update_bits(priv, reg, mask, !set ? 0 : mask);
 }
 
-/* Some registers, like VLANn_CTRL, should always be written in 64-bit, even if
- * you are to write only the lower / upper 32 bits.
+/* Some multi-word registers, like VLANn_CTRL, should be treated as a single
+ * long register. More specifically, writes to parts of its words won't become
+ * visible, until the last word is written.
  *
- * There is no such restriction for reading, but we still provide 64-bit read
- * wrappers so that we always handle u64 values.
+ * Here we require full read and write operations over these registers to
+ * eliminate potential issues, although partial reads/writes are also possible.
  */
 
-static int yt921x_reg64_read(struct yt921x_priv *priv, u32 reg, u64 *valp)
+static void update_ctrls_unaligned(u32 *lo, u32 *hi, u64 mask, u64 val)
+{
+	*lo &= ~lower_32_bits(mask);
+	*hi &= ~upper_32_bits(mask);
+	*lo |= lower_32_bits(val);
+	*hi |= upper_32_bits(val);
+}
+
+static int
+yt921x_regs_read(struct yt921x_priv *priv, u32 reg, u32 *vals,
+		 unsigned int num_regs)
 {
-	u32 lo;
-	u32 hi;
 	int res;
 
-	res = yt921x_reg_read(priv, reg, &lo);
-	if (res)
-		return res;
-	res = yt921x_reg_read(priv, reg + 4, &hi);
-	if (res)
-		return res;
+	for (unsigned int i = 0; i < num_regs; i++) {
+		res = yt921x_reg_read(priv, reg + 4 * i, &vals[i]);
+		if (res)
+			return res;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_regs_write(struct yt921x_priv *priv, u32 reg, const u32 *vals,
+		  unsigned int num_regs)
+{
+	int res;
+
+	for (unsigned int i = 0; i < num_regs; i++) {
+		res = yt921x_reg_write(priv, reg + 4 * i, vals[i]);
+		if (res)
+			return res;
+	}
 
-	*valp = ((u64)hi << 32) | lo;
 	return 0;
 }
 
-static int yt921x_reg64_write(struct yt921x_priv *priv, u32 reg, u64 val)
+static int
+yt921x_regs_update_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks,
+			const u32 *vals, unsigned int num_regs)
 {
+	bool changed = false;
+	u32 vs[4];
 	int res;
 
-	res = yt921x_reg_write(priv, reg, (u32)val);
+	BUILD_BUG_ON(num_regs > ARRAY_SIZE(vs));
+
+	res = yt921x_regs_read(priv, reg, vs, num_regs);
 	if (res)
 		return res;
-	return yt921x_reg_write(priv, reg + 4, (u32)(val >> 32));
+
+	for (unsigned int i = 0; i < num_regs; i++) {
+		u32 u = vs[i];
+
+		u &= ~masks[i];
+		u |= vals[i];
+		if (u != vs[i])
+			changed = true;
+
+		vs[i] = u;
+	}
+
+	if (!changed)
+		return 0;
+
+	return yt921x_regs_write(priv, reg, vs, num_regs);
 }
 
 static int
-yt921x_reg64_update_bits(struct yt921x_priv *priv, u32 reg, u64 mask, u64 val)
+yt921x_regs_clear_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks,
+		       unsigned int num_regs)
 {
+	bool changed = false;
+	u32 vs[4];
 	int res;
-	u64 v;
-	u64 u;
 
-	res = yt921x_reg64_read(priv, reg, &v);
+	BUILD_BUG_ON(num_regs > ARRAY_SIZE(vs));
+
+	res = yt921x_regs_read(priv, reg, vs, num_regs);
 	if (res)
 		return res;
 
-	u = v;
-	u &= ~mask;
-	u |= val;
-	if (u == v)
+	for (unsigned int i = 0; i < num_regs; i++) {
+		u32 u = vs[i];
+
+		u &= ~masks[i];
+		if (u != vs[i])
+			changed = true;
+
+		vs[i] = u;
+	}
+
+	if (!changed)
 		return 0;
 
-	return yt921x_reg64_write(priv, reg, u);
+	return yt921x_regs_write(priv, reg, vs, num_regs);
+}
+
+static int
+yt921x_reg64_write(struct yt921x_priv *priv, u32 reg, const u32 *vals)
+{
+	return yt921x_regs_write(priv, reg, vals, 2);
+}
+
+static int
+yt921x_reg64_update_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks,
+			 const u32 *vals)
+{
+	return yt921x_regs_update_bits(priv, reg, masks, vals, 2);
+}
+
+static int
+yt921x_reg64_clear_bits(struct yt921x_priv *priv, u32 reg, const u32 *masks)
+{
+	return yt921x_regs_clear_bits(priv, reg, masks, 2);
 }
 
-static int yt921x_reg64_clear_bits(struct yt921x_priv *priv, u32 reg, u64 mask)
+static int
+yt921x_reg96_write(struct yt921x_priv *priv, u32 reg, const u32 *vals)
 {
-	return yt921x_reg64_update_bits(priv, reg, mask, 0);
+	return yt921x_regs_write(priv, reg, vals, 3);
 }
 
 static int yt921x_reg_mdio_read(void *context, u32 reg, u32 *valp)
@@ -1007,6 +1080,13 @@ yt921x_dsa_set_mac_eee(struct dsa_switch *ds, int port, struct ethtool_keee *e)
 	return res;
 }
 
+static int yt921x_mtu_fetch(struct yt921x_priv *priv, int port)
+{
+	struct dsa_port *dp = dsa_to_port(&priv->ds, port);
+
+	return dp->user ? READ_ONCE(dp->user->mtu) : ETH_DATA_LEN;
+}
+
 static int
 yt921x_dsa_port_change_mtu(struct dsa_switch *ds, int port, int new_mtu)
 {
@@ -1038,6 +1118,266 @@ static int yt921x_dsa_port_max_mtu(struct dsa_switch *ds, int port)
 	return YT921X_FRAME_SIZE_MAX - ETH_HLEN - ETH_FCS_LEN - YT921X_TAG_LEN;
 }
 
+/* v * 2^e */
+static u64 ldexpu64(u64 v, int e)
+{
+	return e >= 0 ? v << e : v >> -e;
+}
+
+/* slot (ns) * rate (/s) / 10^9 (ns/s) = 2^C * token * 4^unit */
+static u32 rate2token(u64 rate, unsigned int slot_ns, int unit, int C)
+{
+	int e = 2 * unit + C + YT921X_TOKEN_RATE_C;
+
+	return div_u64(ldexpu64(slot_ns * rate, -e), 1000000000);
+}
+
+static u64 token2rate(u32 token, unsigned int slot_ns, int unit, int C)
+{
+	int e = 2 * unit + C + YT921X_TOKEN_RATE_C;
+
+	return div_u64(ldexpu64(mul_u32_u32(1000000000, token), e), slot_ns);
+}
+
+/* burst = 2^C * token * 4^unit */
+static u32 burst2token(u64 burst, int unit, int C)
+{
+	return ldexpu64(burst, -(2 * unit + C));
+}
+
+static u64 token2burst(u32 token, int unit, int C)
+{
+	return ldexpu64(token, 2 * unit + C);
+}
+
+struct yt921x_marker {
+	u32 cir;
+	u32 cbs;
+	u32 ebs;
+	int unit;
+	bool pkt_mode;
+};
+
+#define YT921X_MARKER_PKT_MODE		BIT(0)
+#define YT921X_MARKER_SINGLE_BUCKET	BIT(1)
+
+static int
+yt921x_marker_tfm(struct yt921x_marker *marker, u64 rate, u64 burst,
+		  unsigned int flags, unsigned int slot_ns, u32 cir_max,
+		  u32 cbs_max, int unit_max, struct yt921x_priv *priv, int port,
+		  struct netlink_ext_ack *extack)
+{
+	const int C = flags & YT921X_MARKER_PKT_MODE ? YT921X_TOKEN_PKT_C :
+		      YT921X_TOKEN_BYTE_C;
+	struct device *dev = to_device(priv);
+	struct yt921x_marker m;
+	u64 burst_est;
+	u64 burst_sug;
+	u64 burst_max;
+	u64 rate_max;
+
+	m.unit = unit_max;
+	rate_max = token2rate(cir_max, slot_ns, m.unit, C);
+	burst_max = token2burst(cbs_max, m.unit, C);
+
+	/* Check for unusual values */
+	if (rate > rate_max || burst > burst_max) {
+		NL_SET_ERR_MSG_MOD(extack, "Unexpected tremendous rate");
+		return -ERANGE;
+	}
+
+	/* Check for matching burst */
+	burst_est = div_u64(slot_ns * rate, 1000000000);
+	burst_sug = burst_est;
+	if (flags & YT921X_MARKER_PKT_MODE)
+		burst_sug++;
+	else
+		burst_sug += ETH_HLEN + yt921x_mtu_fetch(priv, port) +
+			     ETH_FCS_LEN;
+	if (burst_sug > burst)
+		NL_SET_ERR_MSG_FMT_MOD(extack,
+				       "Consider match rate %llu with burst at least %llu",
+				       rate, burst_sug);
+
+	/* Select unit */
+	for (; m.unit > 0; m.unit--) {
+		if (rate > (rate_max >> 2) || burst > (burst_max >> 2))
+			break;
+		rate_max >>= 2;
+		burst_max >>= 2;
+	}
+
+	/* Calculate information rate and bucket size */
+	m.cir = rate2token(rate, slot_ns, m.unit, C);
+	if (!m.cir)
+		m.cir = 1;
+	else if (WARN_ON(m.cir > cir_max))
+		m.cir = cir_max;
+	m.cbs = burst2token(burst, m.unit, C);
+	if (!m.cbs)
+		m.cbs = 1;
+	else if (WARN_ON(m.cbs > cbs_max))
+		m.cbs = cbs_max;
+
+	/* Cut EBS */
+	m.ebs = 0;
+	if (!(flags & YT921X_MARKER_SINGLE_BUCKET)) {
+		/* We don't have a chance to adjust rate when MTU is changed */
+		if (flags & YT921X_MARKER_PKT_MODE)
+			burst_est++;
+		else
+			burst_est += YT921X_FRAME_SIZE_MAX;
+
+		if (burst_est < burst) {
+			u32 pbs = m.cbs;
+
+			m.cbs = burst2token(burst_est, m.unit, C);
+			if (!m.cbs)
+				m.cbs = 1;
+			else if (WARN_ON(m.cbs > cbs_max))
+				m.cbs = cbs_max;
+
+			if (pbs > m.cbs)
+				m.ebs = pbs - m.cbs;
+		}
+	}
+
+	dev_dbg(dev,
+		"slot %u ns, rate %llu, burst %llu -> unit %d, cir %u, cbs %u, ebs %u\n",
+		slot_ns, rate, burst, m.unit, m.cir, m.cbs, m.ebs);
+
+	m.pkt_mode = flags & YT921X_MARKER_PKT_MODE;
+	*marker = m;
+	return 0;
+}
+
+static int
+yt921x_marker_tfm_police(struct yt921x_marker *marker,
+			 const struct flow_action_police *police,
+			 unsigned int flags, struct yt921x_priv *priv, int port,
+			 struct netlink_ext_ack *extack)
+{
+	bool pkt_mode = !!police->rate_pkt_ps;
+	u64 burst;
+	u64 rate;
+
+	rate = pkt_mode ? police->rate_pkt_ps : police->rate_bytes_ps;
+	burst = pkt_mode ? police->burst_pkt : police->burst;
+	if (pkt_mode)
+		flags |= YT921X_MARKER_PKT_MODE;
+
+	return yt921x_marker_tfm(marker, rate, burst, flags,
+				 priv->meter_slot_ns, YT921X_METER_CIR_MAX,
+				 YT921X_METER_CBS_MAX, YT921X_METER_UNIT_MAX,
+				 priv, port, extack);
+}
+
+static int
+yt921x_police_validate(const struct flow_action_police *police,
+		       const struct flow_action *action,
+		       const struct flow_action_entry *act,
+		       struct netlink_ext_ack *extack)
+{
+	if (police->exceed.act_id != FLOW_ACTION_DROP) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when exceed action is not drop");
+		return -EOPNOTSUPP;
+	}
+
+	if (police->notexceed.act_id != FLOW_ACTION_PIPE &&
+	    police->notexceed.act_id != FLOW_ACTION_ACCEPT) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when conform action is not pipe or ok");
+		return -EOPNOTSUPP;
+	}
+
+	if (police->notexceed.act_id == FLOW_ACTION_ACCEPT && action && act &&
+	    !flow_action_is_last_entry(action, act)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when conform action is ok, but action is not last");
+		return -EOPNOTSUPP;
+	}
+
+	/* mtu defaults to unlimited but we got 2040 here, don't know why */
+	if (police->peakrate_bytes_ps || police->avrate || police->overhead) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Offload not supported when peakrate/avrate/overhead is configured");
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static int
+yt921x_meter_config(struct yt921x_priv *priv, unsigned int id,
+		    const struct yt921x_marker *marker)
+{
+	u32 ctrls[3];
+
+	ctrls[0] = 0;
+	ctrls[1] = YT921X_METER_CTRLb_CIR(marker->cir);
+	ctrls[2] = YT921X_METER_CTRLc_UNIT(marker->unit) |
+		   YT921X_METER_CTRLc_DROP_R |
+		   YT921X_METER_CTRLc_TOKEN_OVERFLOW_EN |
+		   YT921X_METER_CTRLc_METER_EN;
+	if (marker->pkt_mode)
+		ctrls[2] |= YT921X_METER_CTRLc_PKT_MODE;
+	update_ctrls_unaligned(&ctrls[0], &ctrls[1],
+			       YT921X_METER_CTRLab_EBS_M,
+			       YT921X_METER_CTRLab_EBS(marker->ebs));
+	update_ctrls_unaligned(&ctrls[1], &ctrls[2],
+			       YT921X_METER_CTRLbc_CBS_M,
+			       YT921X_METER_CTRLbc_CBS(marker->cbs));
+
+	return yt921x_reg96_write(priv, YT921X_METERn_CTRL(id), ctrls);
+}
+
+static void yt921x_dsa_port_policer_del(struct dsa_switch *ds, int port)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct device *dev = to_device(priv);
+	int res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_reg_write(priv, YT921X_PORTn_METER(port), 0);
+	mutex_unlock(&priv->reg_lock);
+
+	if (res)
+		dev_err(dev, "Failed to %s port %d: %i\n", "delete policer on",
+			port, res);
+}
+
+static int
+yt921x_dsa_port_policer_add(struct dsa_switch *ds, int port,
+			    const struct flow_action_police *police,
+			    struct netlink_ext_ack *extack)
+{
+	struct yt921x_priv *priv = to_yt921x_priv(ds);
+	struct yt921x_marker marker;
+	u32 ctrl;
+	int res;
+
+	res = yt921x_police_validate(police, NULL, NULL, extack);
+	if (res)
+		return res;
+
+	res = yt921x_marker_tfm_police(&marker, police, 0, priv, port, extack);
+	if (res)
+		return res;
+
+	mutex_lock(&priv->reg_lock);
+	res = yt921x_meter_config(priv, port + YT921X_METER_NUM, &marker);
+	if (res)
+		goto end;
+
+	ctrl = YT921X_PORT_METER_ID(port) | YT921X_PORT_METER_EN;
+	res = yt921x_reg_write(priv, YT921X_PORTn_METER(port), ctrl);
+end:
+	mutex_unlock(&priv->reg_lock);
+
+	return res;
+}
+
 static int
 yt921x_mirror_del(struct yt921x_priv *priv, int port, bool ingress)
 {
@@ -1844,33 +2184,31 @@ yt921x_vlan_filtering(struct yt921x_priv *priv, int port, bool vlan_filtering)
 	return 0;
 }
 
-static int
-yt921x_vlan_del(struct yt921x_priv *priv, int port, u16 vid)
+static int yt921x_vlan_del(struct yt921x_priv *priv, int port, u16 vid)
 {
-	u64 mask64;
+	u32 masks[2];
 
-	mask64 = YT921X_VLAN_CTRL_PORTS(port) |
-		 YT921X_VLAN_CTRL_UNTAG_PORTn(port);
+	masks[0] = YT921X_VLAN_CTRLa_PORTn(port);
+	masks[1] = YT921X_VLAN_CTRLb_UNTAG_PORTn(port);
 
-	return yt921x_reg64_clear_bits(priv, YT921X_VLANn_CTRL(vid), mask64);
+	return yt921x_reg64_clear_bits(priv, YT921X_VLANn_CTRL(vid), masks);
 }
 
 static int
 yt921x_vlan_add(struct yt921x_priv *priv, int port, u16 vid, bool untagged)
 {
-	u64 mask64;
-	u64 ctrl64;
+	u32 masks[2];
+	u32 ctrls[2];
 
-	mask64 = YT921X_VLAN_CTRL_PORTn(port) |
-		 YT921X_VLAN_CTRL_PORTS(priv->cpu_ports_mask);
-	ctrl64 = mask64;
+	masks[0] = YT921X_VLAN_CTRLa_PORTn(port) |
+		   YT921X_VLAN_CTRLa_PORTS(priv->cpu_ports_mask);
+	ctrls[0] = masks[0];
 
-	mask64 |= YT921X_VLAN_CTRL_UNTAG_PORTn(port);
-	if (untagged)
-		ctrl64 |= YT921X_VLAN_CTRL_UNTAG_PORTn(port);
+	masks[1] = YT921X_VLAN_CTRLb_UNTAG_PORTn(port);
+	ctrls[1] = untagged ? masks[1] : 0;
 
 	return yt921x_reg64_update_bits(priv, YT921X_VLANn_CTRL(vid),
-					mask64, ctrl64);
+					masks, ctrls);
 }
 
 static int
@@ -2318,8 +2656,8 @@ yt921x_dsa_vlan_msti_set(struct dsa_switch *ds, struct dsa_bridge bridge,
 			 const struct switchdev_vlan_msti *msti)
 {
 	struct yt921x_priv *priv = to_yt921x_priv(ds);
-	u64 mask64;
-	u64 ctrl64;
+	u32 masks[2];
+	u32 ctrls[2];
 	int res;
 
 	if (!msti->vid)
@@ -2327,12 +2665,14 @@ yt921x_dsa_vlan_msti_set(struct dsa_switch *ds, struct dsa_bridge bridge,
 	if (!msti->msti || msti->msti >= YT921X_MSTI_NUM)
 		return -EINVAL;
 
-	mask64 = YT921X_VLAN_CTRL_STP_ID_M;
-	ctrl64 = YT921X_VLAN_CTRL_STP_ID(msti->msti);
+	masks[0] = 0;
+	ctrls[0] = 0;
+	masks[1] = YT921X_VLAN_CTRLb_STP_ID_M;
+	ctrls[1] = YT921X_VLAN_CTRLb_STP_ID(msti->msti);
 
 	mutex_lock(&priv->reg_lock);
 	res = yt921x_reg64_update_bits(priv, YT921X_VLANn_CTRL(msti->vid),
-				       mask64, ctrl64);
+				       masks, ctrls);
 	mutex_unlock(&priv->reg_lock);
 
 	return res;
@@ -2992,6 +3332,7 @@ static int yt921x_chip_detect(struct yt921x_priv *priv)
 	u32 chipid;
 	u32 major;
 	u32 mode;
+	u32 val;
 	int res;
 
 	res = yt921x_reg_read(priv, YT921X_CHIP_ID, &chipid);
@@ -3026,12 +3367,27 @@ static int yt921x_chip_detect(struct yt921x_priv *priv)
 		return -ENODEV;
 	}
 
+	res = yt921x_reg_read(priv, YT921X_SYS_CLK, &val);
+	if (res)
+		return res;
+	switch (FIELD_GET(YT921X_SYS_CLK_SEL_M, val)) {
+	case 0:
+		priv->cycle_ns = info->major == YT9215_MAJOR ? 8 : 6;
+		break;
+	case YT921X_SYS_CLK_143M:
+		priv->cycle_ns = 7;
+		break;
+	default:
+		priv->cycle_ns = 8;
+	}
+
 	/* Print chipid here since we are interested in lower 16 bits */
 	dev_info(dev,
 		 "Motorcomm %s ethernet switch, chipid: 0x%x, chipmode: 0x%x 0x%x\n",
 		 info->name, chipid, mode, extmode);
 
 	priv->info = info;
+
 	return 0;
 }
 
@@ -3083,7 +3439,7 @@ static int yt921x_chip_setup_dsa(struct yt921x_priv *priv)
 {
 	struct dsa_switch *ds = &priv->ds;
 	unsigned long cpu_ports_mask;
-	u64 ctrl64;
+	u32 ctrls[2];
 	u32 ctrl;
 	int port;
 	int res;
@@ -3144,10 +3500,28 @@ static int yt921x_chip_setup_dsa(struct yt921x_priv *priv)
 	/* Tagged VID 0 should be treated as untagged, which confuses the
 	 * hardware a lot
 	 */
-	ctrl64 = YT921X_VLAN_CTRL_LEARN_DIS | YT921X_VLAN_CTRL_PORTS_M;
-	res = yt921x_reg64_write(priv, YT921X_VLANn_CTRL(0), ctrl64);
+	ctrls[0] = YT921X_VLAN_CTRLa_LEARN_DIS | YT921X_VLAN_CTRLa_PORTS_M;
+	ctrls[1] = 0;
+	res = yt921x_reg64_write(priv, YT921X_VLANn_CTRL(0), ctrls);
+	if (res)
+		return res;
+
+	return 0;
+}
+
+static int yt921x_chip_setup_tc(struct yt921x_priv *priv)
+{
+	unsigned int op_ns;
+	u32 ctrl;
+	int res;
+
+	op_ns = 8 * priv->cycle_ns;
+
+	ctrl = max(priv->meter_slot_ns / op_ns, YT921X_METER_SLOT_MIN);
+	res = yt921x_reg_write(priv, YT921X_METER_SLOT, ctrl);
 	if (res)
 		return res;
+	priv->meter_slot_ns = ctrl * op_ns;
 
 	return 0;
 }
@@ -3198,7 +3572,7 @@ static int yt921x_chip_setup(struct yt921x_priv *priv)
 	u32 ctrl;
 	int res;
 
-	ctrl = YT921X_FUNC_MIB;
+	ctrl = YT921X_FUNC_MIB | YT921X_FUNC_METER;
 	res = yt921x_reg_set_bits(priv, YT921X_FUNC, ctrl);
 	if (res)
 		return res;
@@ -3207,6 +3581,10 @@ static int yt921x_chip_setup(struct yt921x_priv *priv)
 	if (res)
 		return res;
 
+	res = yt921x_chip_setup_tc(priv);
+	if (res)
+		return res;
+
 #if IS_ENABLED(CONFIG_DCB)
 	res = yt921x_chip_setup_qos(priv);
 	if (res)
@@ -3298,6 +3676,9 @@ static const struct dsa_switch_ops yt921x_dsa_switch_ops = {
 	/* mtu */
 	.port_change_mtu	= yt921x_dsa_port_change_mtu,
 	.port_max_mtu		= yt921x_dsa_port_max_mtu,
+	/* rate */
+	.port_policer_del	= yt921x_dsa_port_policer_del,
+	.port_policer_add	= yt921x_dsa_port_policer_add,
 	/* hsr */
 	.port_hsr_leave		= dsa_port_simple_hsr_leave,
 	.port_hsr_join		= dsa_port_simple_hsr_join,
diff --git a/drivers/net/dsa/yt921x.h b/drivers/net/dsa/yt921x.h
index 3f129b8d403fa..546b12a8994a4 100644
--- a/drivers/net/dsa/yt921x.h
+++ b/drivers/net/dsa/yt921x.h
@@ -23,6 +23,7 @@
 #define  YT921X_RST_HW				BIT(31)
 #define  YT921X_RST_SW				BIT(1)
 #define YT921X_FUNC			0x80004
+#define  YT921X_FUNC_METER			BIT(4)
 #define  YT921X_FUNC_MIB			BIT(1)
 #define YT921X_CHIP_ID			0x80008
 #define  YT921X_CHIP_ID_MAJOR			GENMASK(31, 16)
@@ -239,6 +240,11 @@
 #define  YT921X_EDATA_DATA_STATUS_M		GENMASK(3, 0)
 #define   YT921X_EDATA_DATA_STATUS(x)			FIELD_PREP(YT921X_EDATA_DATA_STATUS_M, (x))
 #define   YT921X_EDATA_DATA_IDLE			YT921X_EDATA_DATA_STATUS(3)
+#define YT921X_SYS_CLK			0xe0040
+#define  YT921X_SYS_CLK_SEL_M			GENMASK(1, 0)  /* unknown: 167M */
+#define   YT9215_SYS_CLK_125M				0
+#define   YT9218_SYS_CLK_167M				0
+#define   YT921X_SYS_CLK_143M				1
 
 #define YT921X_EXT_MBUS_OP		0x6a000
 #define YT921X_INT_MBUS_OP		0xf0000
@@ -429,24 +435,24 @@ enum yt921x_app_selector {
 #define  YT921X_FDB_HW_FLUSH_ON_LINKDOWN	BIT(0)
 
 #define YT921X_VLANn_CTRL(vlan)		(0x188000 + 8 * (vlan))
-#define  YT921X_VLAN_CTRL_UNTAG_PORTS_M		GENMASK_ULL(50, 40)
-#define   YT921X_VLAN_CTRL_UNTAG_PORTS(x)		FIELD_PREP(YT921X_VLAN_CTRL_UNTAG_PORTS_M, (x))
-#define  YT921X_VLAN_CTRL_UNTAG_PORTn(port)	BIT_ULL((port) + 40)
-#define  YT921X_VLAN_CTRL_STP_ID_M		GENMASK_ULL(39, 36)
-#define   YT921X_VLAN_CTRL_STP_ID(x)			FIELD_PREP(YT921X_VLAN_CTRL_STP_ID_M, (x))
-#define  YT921X_VLAN_CTRL_SVLAN_EN		BIT_ULL(35)
-#define  YT921X_VLAN_CTRL_FID_M			GENMASK_ULL(34, 23)
-#define   YT921X_VLAN_CTRL_FID(x)			FIELD_PREP(YT921X_VLAN_CTRL_FID_M, (x))
-#define  YT921X_VLAN_CTRL_LEARN_DIS		BIT_ULL(22)
-#define  YT921X_VLAN_CTRL_PRIO_EN		BIT_ULL(21)
-#define  YT921X_VLAN_CTRL_PRIO_M		GENMASK_ULL(20, 18)
-#define   YT921X_VLAN_CTRL_PRIO(x)			FIELD_PREP(YT921X_VLAN_CTRL_PRIO_M, (x))
-#define  YT921X_VLAN_CTRL_PORTS_M		GENMASK_ULL(17, 7)
-#define   YT921X_VLAN_CTRL_PORTS(x)			FIELD_PREP(YT921X_VLAN_CTRL_PORTS_M, (x))
-#define  YT921X_VLAN_CTRL_PORTn(port)		BIT_ULL((port) + 7)
-#define  YT921X_VLAN_CTRL_BYPASS_1X_AC		BIT_ULL(6)
-#define  YT921X_VLAN_CTRL_METER_EN		BIT_ULL(5)
-#define  YT921X_VLAN_CTRL_METER_ID_M		GENMASK_ULL(4, 0)
+#define  YT921X_VLAN_CTRLb_UNTAG_PORTS_M	GENMASK(18, 8)
+#define   YT921X_VLAN_CTRLb_UNTAG_PORTS(x)		FIELD_PREP(YT921X_VLAN_CTRLb_UNTAG_PORTS_M, (x))
+#define  YT921X_VLAN_CTRLb_UNTAG_PORTn(port)	BIT((port) + 8)
+#define  YT921X_VLAN_CTRLb_STP_ID_M		GENMASK(7, 4)
+#define   YT921X_VLAN_CTRLb_STP_ID(x)			FIELD_PREP(YT921X_VLAN_CTRLb_STP_ID_M, (x))
+#define  YT921X_VLAN_CTRLb_SVLAN_EN		BIT(3)
+#define  YT921X_VLAN_CTRLab_FID_M		GENMASK_ULL(34, 23)
+#define   YT921X_VLAN_CTRLab_FID(x)			FIELD_PREP(YT921X_VLAN_CTRLab_FID_M, (x))
+#define  YT921X_VLAN_CTRLa_LEARN_DIS		BIT(22)
+#define  YT921X_VLAN_CTRLa_PRIO_EN		BIT(21)
+#define  YT921X_VLAN_CTRLa_PRIO_M		GENMASK(20, 18)
+#define   YT921X_VLAN_CTRLa_PRIO(x)			FIELD_PREP(YT921X_VLAN_CTRLa_PRIO_M, (x))
+#define  YT921X_VLAN_CTRLa_PORTS_M		GENMASK(17, 7)
+#define   YT921X_VLAN_CTRLa_PORTS(x)			FIELD_PREP(YT921X_VLAN_CTRLa_PORTS_M, (x))
+#define  YT921X_VLAN_CTRLa_PORTn(port)		BIT((port) + 7)
+#define  YT921X_VLAN_CTRLa_BYPASS_1X_AC		BIT(6)
+#define  YT921X_VLAN_CTRLa_METER_EN		BIT(5)
+#define  YT921X_VLAN_CTRLa_METER_ID_M		GENMASK(4, 0)
 
 #define YT921X_TPID_IGRn(x)		(0x210000 + 4 * (x))	/* [0, 3] */
 #define  YT921X_TPID_IGR_TPID_M			GENMASK(15, 0)
@@ -465,6 +471,39 @@ enum yt921x_app_selector {
 #define  YT921X_LAG_HASH_MAC_DA			BIT(1)
 #define  YT921X_LAG_HASH_SRC_PORT		BIT(0)
 
+#define YT921X_PORTn_RATE(port)		(0x220000 + 4 * (port))
+#define  YT921X_PORT_RATE_GAP_VALUE		GENMASK(4, 0)	/* default 20 */
+#define YT921X_METER_SLOT		0x220104
+#define  YT921X_METER_SLOT_SLOT_M		GENMASK(11, 0)
+#define YT921X_PORTn_METER(port)	(0x220108 + 4 * (port))
+#define  YT921X_PORT_METER_EN			BIT(4)
+#define  YT921X_PORT_METER_ID_M			GENMASK(3, 0)
+#define   YT921X_PORT_METER_ID(x)			FIELD_PREP(YT921X_PORT_METER_ID_M, (x))
+#define YT921X_METERn_CTRL(x)		(0x220800 + 0x10 * (x))
+#define  YT921X_METER_CTRLc_METER_EN		BIT(14)
+#define  YT921X_METER_CTRLc_TOKEN_OVERFLOW_EN	BIT(13)	/* RFC4115: yellow use unused green bw */
+#define  YT921X_METER_CTRLc_DROP_M		GENMASK(12, 11)
+#define   YT921X_METER_CTRLc_DROP(x)			FIELD_PREP(YT921X_METER_CTRLc_DROP_M, (x))
+#define   YT921X_METER_CTRLc_DROP_GYR			YT921X_METER_CTRLc_DROP(0)
+#define   YT921X_METER_CTRLc_DROP_YR			YT921X_METER_CTRLc_DROP(1)
+#define   YT921X_METER_CTRLc_DROP_R			YT921X_METER_CTRLc_DROP(2)
+#define   YT921X_METER_CTRLc_DROP_NONE			YT921X_METER_CTRLc_DROP(3)
+#define  YT921X_METER_CTRLc_COLOR_BLIND		BIT(10)
+#define  YT921X_METER_CTRLc_UNIT_M		GENMASK(9, 7)
+#define   YT921X_METER_CTRLc_UNIT(x)			FIELD_PREP(YT921X_METER_CTRLc_UNIT_M, (x))
+#define  YT921X_METER_CTRLc_BYTE_MODE_INCLUDE_GAP	BIT(6)	/* +GAP_VALUE bytes each packet */
+#define  YT921X_METER_CTRLc_PKT_MODE		BIT(5)	/* 0: byte rate mode */
+#define  YT921X_METER_CTRLc_RFC2698		BIT(4)	/* 0: RFC4115 */
+#define  YT921X_METER_CTRLbc_CBS_M		GENMASK_ULL(35, 20)
+#define   YT921X_METER_CTRLbc_CBS(x)			FIELD_PREP(YT921X_METER_CTRLbc_CBS_M, (x))
+#define  YT921X_METER_CTRLb_CIR_M		GENMASK(19, 2)
+#define   YT921X_METER_CTRLb_CIR(x)			FIELD_PREP(YT921X_METER_CTRLb_CIR_M, (x))
+#define  YT921X_METER_CTRLab_EBS_M		GENMASK_ULL(33, 18)
+#define   YT921X_METER_CTRLab_EBS(x)			FIELD_PREP(YT921X_METER_CTRLab_EBS_M, (x))
+#define  YT921X_METER_CTRLa_EIR_M		GENMASK(17, 0)
+#define   YT921X_METER_CTRLa_EIR(x)			FIELD_PREP(YT921X_METER_CTRLa_EIR_M, (x))
+#define YT921X_METERn_STAT(x)		(0x221000 + 8 * (x))
+
 #define YT921X_PORTn_VLAN_CTRL(port)	(0x230010 + 4 * (port))
 #define  YT921X_PORT_VLAN_CTRL_SVLAN_PRIO_EN	BIT(31)
 #define  YT921X_PORT_VLAN_CTRL_CVLAN_PRIO_EN	BIT(30)
@@ -508,6 +547,16 @@ enum yt921x_fdb_entry_status {
 
 #define YT921X_MSTI_NUM		16
 
+#define YT921X_TOKEN_BYTE_C	1	/* 1 token = 2^1 byte */
+#define YT921X_TOKEN_PKT_C	-6	/* 1 token = 2^-6 packets */
+#define YT921X_TOKEN_RATE_C	-15
+/* Custom meters only, not including dedicated port meters (11) */
+#define YT921X_METER_NUM	64
+#define YT921X_METER_SLOT_MIN	80
+#define YT921X_METER_UNIT_MAX	((1 << 3) - 1)
+#define YT921X_METER_CIR_MAX	((1 << 18) - 1)
+#define YT921X_METER_CBS_MAX	((1 << 16) - 1)
+
 #define YT921X_LAG_NUM		2
 #define YT921X_LAG_PORT_NUM	4
 
@@ -602,8 +651,10 @@ struct yt921x_priv {
 	struct dsa_switch ds;
 
 	const struct yt921x_info *info;
+	unsigned int meter_slot_ns;
 	/* cache of dsa_cpu_ports(ds) */
 	u16 cpu_ports_mask;
+	unsigned char cycle_ns;
 
 	/* protect the access to the switch registers */
 	struct mutex reg_lock;
diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c
index f8a4eb365c3d0..f6732eab5923f 100644
--- a/drivers/net/dummy.c
+++ b/drivers/net/dummy.c
@@ -158,7 +158,7 @@ static int __init dummy_init_one(void)
 		return -ENOMEM;
 
 	dev_dummy->rtnl_link_ops = &dummy_link_ops;
-	err = register_netdevice(dev_dummy);
+	err = register_netdev(dev_dummy);
 	if (err < 0)
 		goto err;
 	return 0;
@@ -176,15 +176,11 @@ static int __init dummy_init_module(void)
 	if (err < 0)
 		return err;
 
-	rtnl_net_lock(&init_net);
-
 	for (i = 0; i < numdummies && !err; i++) {
 		err = dummy_init_one();
 		cond_resched();
 	}
 
-	rtnl_net_unlock(&init_net);
-
 	if (err < 0)
 		rtnl_link_unregister(&dummy_link_ops);
 
diff --git a/drivers/net/ethernet/3com/3c59x.c b/drivers/net/ethernet/3com/3c59x.c
index 4fe4efdb3737f..79beb86437df8 100644
--- a/drivers/net/ethernet/3com/3c59x.c
+++ b/drivers/net/ethernet/3com/3c59x.c
@@ -376,54 +376,54 @@ static struct vortex_chip_info {
 
 
 static const struct pci_device_id vortex_pci_tbl[] = {
-	{ 0x10B7, 0x5900, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C590 },
-	{ 0x10B7, 0x5920, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C592 },
-	{ 0x10B7, 0x5970, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C597 },
-	{ 0x10B7, 0x5950, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C595_1 },
-	{ 0x10B7, 0x5951, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C595_2 },
-
-	{ 0x10B7, 0x5952, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C595_3 },
-	{ 0x10B7, 0x9000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C900_1 },
-	{ 0x10B7, 0x9001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C900_2 },
-	{ 0x10B7, 0x9004, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C900_3 },
-	{ 0x10B7, 0x9005, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C900_4 },
-
-	{ 0x10B7, 0x9006, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C900_5 },
-	{ 0x10B7, 0x900A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C900B_FL },
-	{ 0x10B7, 0x9050, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C905_1 },
-	{ 0x10B7, 0x9051, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C905_2 },
-	{ 0x10B7, 0x9054, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C905B_TX },
-	{ 0x10B7, 0x9055, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C905B_1 },
-
-	{ 0x10B7, 0x9058, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C905B_2 },
-	{ 0x10B7, 0x905A, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C905B_FX },
-	{ 0x10B7, 0x9200, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C905C },
-	{ 0x10B7, 0x9202, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C9202 },
-	{ 0x10B7, 0x9800, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C980 },
-	{ 0x10B7, 0x9805, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C9805 },
-
-	{ 0x10B7, 0x7646, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3CSOHO100_TX },
-	{ 0x10B7, 0x5055, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C555 },
-	{ 0x10B7, 0x6055, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C556 },
-	{ 0x10B7, 0x6056, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C556B },
-	{ 0x10B7, 0x5b57, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C575 },
-
-	{ 0x10B7, 0x5057, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C575_1 },
-	{ 0x10B7, 0x5157, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3CCFE575 },
-	{ 0x10B7, 0x5257, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3CCFE575CT },
-	{ 0x10B7, 0x6560, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3CCFE656 },
-	{ 0x10B7, 0x6562, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3CCFEM656 },
-
-	{ 0x10B7, 0x6564, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3CCFEM656_1 },
-	{ 0x10B7, 0x4500, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C450 },
-	{ 0x10B7, 0x9201, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C920 },
-	{ 0x10B7, 0x1201, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C982A },
-	{ 0x10B7, 0x1202, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_3C982B },
-
-	{ 0x10B7, 0x9056, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_905BT4 },
-	{ 0x10B7, 0x9210, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_920B_EMB_WNM },
-
-	{0,}						/* 0 terminated list. */
+	{ PCI_DEVICE(0x10B7, 0x5900), .driver_data = CH_3C590 },
+	{ PCI_DEVICE(0x10B7, 0x5920), .driver_data = CH_3C592 },
+	{ PCI_DEVICE(0x10B7, 0x5970), .driver_data = CH_3C597 },
+	{ PCI_DEVICE(0x10B7, 0x5950), .driver_data = CH_3C595_1 },
+	{ PCI_DEVICE(0x10B7, 0x5951), .driver_data = CH_3C595_2 },
+
+	{ PCI_DEVICE(0x10B7, 0x5952), .driver_data = CH_3C595_3 },
+	{ PCI_DEVICE(0x10B7, 0x9000), .driver_data = CH_3C900_1 },
+	{ PCI_DEVICE(0x10B7, 0x9001), .driver_data = CH_3C900_2 },
+	{ PCI_DEVICE(0x10B7, 0x9004), .driver_data = CH_3C900_3 },
+	{ PCI_DEVICE(0x10B7, 0x9005), .driver_data = CH_3C900_4 },
+
+	{ PCI_DEVICE(0x10B7, 0x9006), .driver_data = CH_3C900_5 },
+	{ PCI_DEVICE(0x10B7, 0x900A), .driver_data = CH_3C900B_FL },
+	{ PCI_DEVICE(0x10B7, 0x9050), .driver_data = CH_3C905_1 },
+	{ PCI_DEVICE(0x10B7, 0x9051), .driver_data = CH_3C905_2 },
+	{ PCI_DEVICE(0x10B7, 0x9054), .driver_data = CH_3C905B_TX },
+	{ PCI_DEVICE(0x10B7, 0x9055), .driver_data = CH_3C905B_1 },
+
+	{ PCI_DEVICE(0x10B7, 0x9058), .driver_data = CH_3C905B_2 },
+	{ PCI_DEVICE(0x10B7, 0x905A), .driver_data = CH_3C905B_FX },
+	{ PCI_DEVICE(0x10B7, 0x9200), .driver_data = CH_3C905C },
+	{ PCI_DEVICE(0x10B7, 0x9202), .driver_data = CH_3C9202 },
+	{ PCI_DEVICE(0x10B7, 0x9800), .driver_data = CH_3C980 },
+	{ PCI_DEVICE(0x10B7, 0x9805), .driver_data = CH_3C9805 },
+
+	{ PCI_DEVICE(0x10B7, 0x7646), .driver_data = CH_3CSOHO100_TX },
+	{ PCI_DEVICE(0x10B7, 0x5055), .driver_data = CH_3C555 },
+	{ PCI_DEVICE(0x10B7, 0x6055), .driver_data = CH_3C556 },
+	{ PCI_DEVICE(0x10B7, 0x6056), .driver_data = CH_3C556B },
+	{ PCI_DEVICE(0x10B7, 0x5b57), .driver_data = CH_3C575 },
+
+	{ PCI_DEVICE(0x10B7, 0x5057), .driver_data = CH_3C575_1 },
+	{ PCI_DEVICE(0x10B7, 0x5157), .driver_data = CH_3CCFE575 },
+	{ PCI_DEVICE(0x10B7, 0x5257), .driver_data = CH_3CCFE575CT },
+	{ PCI_DEVICE(0x10B7, 0x6560), .driver_data = CH_3CCFE656 },
+	{ PCI_DEVICE(0x10B7, 0x6562), .driver_data = CH_3CCFEM656 },
+
+	{ PCI_DEVICE(0x10B7, 0x6564), .driver_data = CH_3CCFEM656_1 },
+	{ PCI_DEVICE(0x10B7, 0x4500), .driver_data = CH_3C450 },
+	{ PCI_DEVICE(0x10B7, 0x9201), .driver_data = CH_3C920 },
+	{ PCI_DEVICE(0x10B7, 0x1201), .driver_data = CH_3C982A },
+	{ PCI_DEVICE(0x10B7, 0x1202), .driver_data = CH_3C982B },
+
+	{ PCI_DEVICE(0x10B7, 0x9056), .driver_data = CH_905BT4 },
+	{ PCI_DEVICE(0x10B7, 0x9210), .driver_data = CH_920B_EMB_WNM },
+
+	{ }						/* 0 terminated list. */
 };
 MODULE_DEVICE_TABLE(pci, vortex_pci_tbl);
 
diff --git a/drivers/net/ethernet/3com/typhoon.c b/drivers/net/ethernet/3com/typhoon.c
index aaaff3ba43ef8..fadc86907ec2c 100644
--- a/drivers/net/ethernet/3com/typhoon.c
+++ b/drivers/net/ethernet/3com/typhoon.c
@@ -198,33 +198,54 @@ static struct typhoon_card_info typhoon_card_info[] = {
  * bits 12-16 indicate card type: (0) client and (1) server
  */
 static const struct pci_device_id typhoon_pci_tbl[] = {
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0,TYPHOON_TX },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990_TX_95,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, TYPHOON_TX95 },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990_TX_97,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, TYPHOON_TX97 },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990B,
-	  PCI_ANY_ID, 0x1000, 0, 0, TYPHOON_TXM },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990B,
-	  PCI_ANY_ID, 0x1102, 0, 0, TYPHOON_FXM },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990B,
-	  PCI_ANY_ID, 0x2000, 0, 0, TYPHOON_BSVR },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990_FX,
-	  PCI_ANY_ID, 0x1101, 0, 0, TYPHOON_FX95 },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990_FX,
-	  PCI_ANY_ID, 0x1102, 0, 0, TYPHOON_FX97 },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990_FX,
-	  PCI_ANY_ID, 0x2101, 0, 0, TYPHOON_FX95SVR },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990_FX,
-	  PCI_ANY_ID, 0x2102, 0, 0, TYPHOON_FX97SVR },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990SVR95,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, TYPHOON_SVR95 },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990SVR97,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, TYPHOON_SVR97 },
-	{ PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3CR990SVR,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, TYPHOON_SVR },
-	{ 0, }
+	{
+		PCI_VDEVICE(3COM, PCI_DEVICE_ID_3COM_3CR990),
+		.driver_data = TYPHOON_TX,
+	}, {
+		PCI_VDEVICE(3COM, PCI_DEVICE_ID_3COM_3CR990_TX_95),
+		.driver_data = TYPHOON_TX95,
+	}, {
+		PCI_VDEVICE(3COM, PCI_DEVICE_ID_3COM_3CR990_TX_97),
+		.driver_data = TYPHOON_TX97,
+	}, {
+		PCI_VDEVICE_SUB(3COM, PCI_DEVICE_ID_3COM_3CR990B,
+				PCI_ANY_ID, 0x1000),
+		.driver_data = TYPHOON_TXM,
+	}, {
+		PCI_VDEVICE_SUB(3COM, PCI_DEVICE_ID_3COM_3CR990B,
+				PCI_ANY_ID, 0x1102),
+		.driver_data = TYPHOON_FXM,
+	}, {
+		PCI_VDEVICE_SUB(3COM, PCI_DEVICE_ID_3COM_3CR990B,
+				PCI_ANY_ID, 0x2000),
+		.driver_data = TYPHOON_BSVR,
+	}, {
+		PCI_VDEVICE_SUB(3COM, PCI_DEVICE_ID_3COM_3CR990_FX,
+				PCI_ANY_ID, 0x1101),
+		.driver_data = TYPHOON_FX95,
+	}, {
+		PCI_VDEVICE_SUB(3COM, PCI_DEVICE_ID_3COM_3CR990_FX,
+				PCI_ANY_ID, 0x1102),
+		.driver_data = TYPHOON_FX97,
+	}, {
+		PCI_VDEVICE_SUB(3COM, PCI_DEVICE_ID_3COM_3CR990_FX,
+				PCI_ANY_ID, 0x2101),
+		.driver_data = TYPHOON_FX95SVR,
+	}, {
+		PCI_VDEVICE_SUB(3COM, PCI_DEVICE_ID_3COM_3CR990_FX,
+				PCI_ANY_ID, 0x2102),
+		.driver_data = TYPHOON_FX97SVR,
+	}, {
+		PCI_VDEVICE(3COM, PCI_DEVICE_ID_3COM_3CR990SVR95),
+		.driver_data = TYPHOON_SVR95,
+	}, {
+		PCI_VDEVICE(3COM, PCI_DEVICE_ID_3COM_3CR990SVR97),
+		.driver_data = TYPHOON_SVR97,
+	}, {
+		PCI_VDEVICE(3COM, PCI_DEVICE_ID_3COM_3CR990SVR),
+		.driver_data = TYPHOON_SVR,
+	},
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, typhoon_pci_tbl);
 
diff --git a/drivers/net/ethernet/8390/Kconfig b/drivers/net/ethernet/8390/Kconfig
index 5d12a595ab19b..57d73a7747ec4 100644
--- a/drivers/net/ethernet/8390/Kconfig
+++ b/drivers/net/ethernet/8390/Kconfig
@@ -90,7 +90,6 @@ config MCF8390
 config NE2000
 	tristate "NE2000/NE1000 support"
 	depends on (ISA || (Q40 && m) || MACH_TX49XX || ATARI_ETHERNEC)
-	select NETDEV_LEGACY_INIT if ISA
 	select CRC32
 	help
 	  If you have a network (Ethernet) card of this type, say Y here.
diff --git a/drivers/net/ethernet/8390/ne.c b/drivers/net/ethernet/8390/ne.c
index 961019c328421..1cbd5a4977ceb 100644
--- a/drivers/net/ethernet/8390/ne.c
+++ b/drivers/net/ethernet/8390/ne.c
@@ -50,7 +50,6 @@ static const char version2[] =
 #include <linux/etherdevice.h>
 #include <linux/jiffies.h>
 #include <linux/platform_device.h>
-#include <net/Space.h>
 
 #include <asm/io.h>
 
@@ -182,7 +181,6 @@ static void ne_block_input(struct net_device *dev, int count,
 static void ne_block_output(struct net_device *dev, const int count,
 		const unsigned char *buf, const int start_page);
 
-
 /*  Probe for various non-shared-memory ethercards.
 
    NEx000-clone boards have a Station Address PROM (SAPROM) in the packet
@@ -918,31 +916,47 @@ static void __init ne_add_devices(void)
 	}
 }
 
-static int __init ne_init(void)
+/*
+ * This structure holds boot-time configured netdevice settings. They
+ * are then used in the device probing.
+ */
+struct netdev_boot_setup {
+	char name[IFNAMSIZ];
+	struct ifmap map;
+};
+#define NETDEV_BOOT_SETUP_MAX 8
+
+/* Boot time configuration table */
+static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX];
+
+/**
+ * netdev_boot_setup_check	- check boot time settings
+ * @dev: the netdevice
+ *
+ * Check boot time settings for the device.
+ * The found settings are set for the device to be used
+ * later in the device probing.
+ * Returns 0 if no settings found, 1 if they are.
+ */
+static int netdev_boot_setup_check(struct net_device *dev)
 {
-	int retval;
-
-	if (IS_MODULE(CONFIG_NE2000))
-		ne_add_devices();
-
-	retval = platform_driver_probe(&ne_driver, ne_drv_probe);
+	struct netdev_boot_setup *s = dev_boot_setup;
+	int i;
 
-	if (IS_MODULE(CONFIG_NE2000) && retval) {
-		if (io[0] == 0)
-			pr_notice("ne.c: You must supply \"io=0xNNN\""
-			       " value(s) for ISA cards.\n");
-		ne_loop_rm_unreg(1);
-		return retval;
+	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
+		if (s[i].name[0] != '\0' && s[i].name[0] != ' ' &&
+		    !strcmp(dev->name, s[i].name)) {
+			dev->irq = s[i].map.irq;
+			dev->base_addr = s[i].map.base_addr;
+			dev->mem_start = s[i].map.mem_start;
+			dev->mem_end = s[i].map.mem_end;
+			return 1;
+		}
 	}
-
-	/* Unregister unused platform_devices. */
-	ne_loop_rm_unreg(0);
-	return retval;
+	return 0;
 }
-module_init(ne_init);
 
-#if !defined(MODULE) && defined(CONFIG_NETDEV_LEGACY_INIT)
-struct net_device * __init ne_probe(int unit)
+static struct net_device * __init ne_probe(int unit)
 {
 	int this_dev;
 	struct net_device *dev;
@@ -982,8 +996,187 @@ struct net_device * __init ne_probe(int unit)
 
 	return ERR_PTR(-ENODEV);
 }
+
+/******************************************************************************
+ *
+ *		      Device Boot-time Settings Routines
+ *
+ ******************************************************************************/
+
+/**
+ * netdev_boot_base	- get address from boot time settings
+ * @prefix: prefix for network device
+ * @unit: id for network device
+ *
+ * Check boot time settings for the base address of device.
+ * The found settings are set for the device to be used
+ * later in the device probing.
+ * Returns 0 if no settings found.
+ */
+static unsigned long netdev_boot_base(const char *prefix, int unit)
+{
+	const struct netdev_boot_setup *s = dev_boot_setup;
+	char name[IFNAMSIZ];
+	int i;
+
+	sprintf(name, "%s%d", prefix, unit);
+
+	/*
+	 * If device already registered then return base of 1
+	 * to indicate not to probe for this interface
+	 */
+	if (__dev_get_by_name(&init_net, name))
+		return 1;
+
+	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++)
+		if (!strcmp(name, s[i].name))
+			return s[i].map.base_addr;
+	return 0;
+}
+
+#if !defined(MODULE) && defined(CONFIG_ISA)
+/**
+ *	netdev_boot_setup_add	- add new setup entry
+ *	@name: name of the device
+ *	@map: configured settings for the device
+ *
+ *	Adds new setup entry to the dev_boot_setup list.  The function
+ *	returns 0 on error and 1 on success.  This is a generic routine to
+ *	all netdevices.
+ */
+static int netdev_boot_setup_add(char *name, struct ifmap *map)
+{
+	struct netdev_boot_setup *s;
+	int i;
+
+	s = dev_boot_setup;
+	for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) {
+		if (s[i].name[0] == '\0' || s[i].name[0] == ' ') {
+			strscpy_pad(s[i].name, name);
+			memcpy(&s[i].map, map, sizeof(s[i].map));
+			break;
+		}
+	}
+
+	return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1;
+}
+
+/*
+ * Saves at boot time configured settings for any netdevice.
+ */
+static int __init netdev_boot_setup(char *str)
+{
+	int ints[5];
+	struct ifmap map;
+
+	str = get_options(str, ARRAY_SIZE(ints), ints);
+	if (!str || !*str)
+		return 0;
+
+	/* Save settings */
+	memset(&map, 0, sizeof(map));
+	if (ints[0] > 0)
+		map.irq = ints[1];
+	if (ints[0] > 1)
+		map.base_addr = ints[2];
+	if (ints[0] > 2)
+		map.mem_start = ints[3];
+	if (ints[0] > 3)
+		map.mem_end = ints[4];
+
+	/* Add new entry to the list */
+	return netdev_boot_setup_add(str, &map);
+}
+
+__setup("netdev=", netdev_boot_setup);
+
+static int __init ether_boot_setup(char *str)
+{
+	return netdev_boot_setup(str);
+}
+__setup("ether=", ether_boot_setup);
 #endif
 
+/* A unified ethernet device probe.  This is the easiest way to have every
+ * ethernet adaptor have the name "eth[0123...]".
+ */
+
+struct devprobe2 {
+	struct net_device *(*probe)(int unit);
+	int status;	/* non-zero if autoprobe has failed */
+};
+
+static int __init probe_list2(int unit, struct devprobe2 *p, int autoprobe)
+{
+	struct net_device *dev;
+
+	for (; p->probe; p++) {
+		if (autoprobe && p->status)
+			continue;
+		dev = p->probe(unit);
+		if (!IS_ERR(dev))
+			return 0;
+		if (autoprobe)
+			p->status = PTR_ERR(dev);
+	}
+	return -ENODEV;
+}
+
+static struct devprobe2 isa_probes[] __initdata = {
+	{ne_probe, 0},
+	{NULL, 0},
+};
+
+/* Unified ethernet device probe, segmented per architecture and
+ * per bus interface. This drives the legacy devices only for now.
+ */
+
+static void __init ethif_probe2(int unit)
+{
+	unsigned long base_addr = netdev_boot_base("eth", unit);
+
+	if (base_addr == 1)
+		return;
+
+	probe_list2(unit, isa_probes, base_addr == 0);
+}
+
+/*  Statically configured drivers -- order matters here. */
+static int __init net_olddevs_init(void)
+{
+	int num;
+
+	for (num = 0; num < 8; ++num)
+		ethif_probe2(num);
+
+	return 0;
+}
+
+static int __init ne_init(void)
+{
+	int retval;
+
+	if (IS_MODULE(CONFIG_NE2000))
+		ne_add_devices();
+	else
+		net_olddevs_init();
+
+	retval = platform_driver_probe(&ne_driver, ne_drv_probe);
+
+	if (IS_MODULE(CONFIG_NE2000) && retval) {
+		if (io[0] == 0)
+			pr_notice("ne.c: You must supply \"io=0xNNN\""
+			       " value(s) for ISA cards.\n");
+		ne_loop_rm_unreg(1);
+		return retval;
+	}
+
+	/* Unregister unused platform_devices. */
+	ne_loop_rm_unreg(0);
+	return retval;
+}
+module_init(ne_init);
+
 static void __exit ne_exit(void)
 {
 	platform_driver_unregister(&ne_driver);
diff --git a/drivers/net/ethernet/8390/ne2k-pci.c b/drivers/net/ethernet/8390/ne2k-pci.c
index 1a34da07c0db2..b1a94c97c0b98 100644
--- a/drivers/net/ethernet/8390/ne2k-pci.c
+++ b/drivers/net/ethernet/8390/ne2k-pci.c
@@ -143,18 +143,18 @@ static struct {
 
 
 static const struct pci_device_id ne2k_pci_tbl[] = {
-	{ 0x10ec, 0x8029, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_RealTek_RTL_8029 },
-	{ 0x1050, 0x0940, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_Winbond_89C940 },
-	{ 0x11f6, 0x1401, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_Compex_RL2000 },
-	{ 0x8e2e, 0x3000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_KTI_ET32P2 },
-	{ 0x4a14, 0x5000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_NetVin_NV5000SC },
-	{ 0x1106, 0x0926, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_Via_86C926 },
-	{ 0x10bd, 0x0e34, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_SureCom_NE34 },
-	{ 0x1050, 0x5a5a, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_Winbond_W89C940F },
-	{ 0x12c3, 0x0058, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_Holtek_HT80232 },
-	{ 0x12c3, 0x5598, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_Holtek_HT80229 },
-	{ 0x8c4a, 0x1980, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CH_Winbond_89C940_8c4a },
-	{ 0, }
+	{ PCI_DEVICE(0x10ec, 0x8029), .driver_data = CH_RealTek_RTL_8029 },
+	{ PCI_DEVICE(0x1050, 0x0940), .driver_data = CH_Winbond_89C940 },
+	{ PCI_DEVICE(0x11f6, 0x1401), .driver_data = CH_Compex_RL2000 },
+	{ PCI_DEVICE(0x8e2e, 0x3000), .driver_data = CH_KTI_ET32P2 },
+	{ PCI_DEVICE(0x4a14, 0x5000), .driver_data = CH_NetVin_NV5000SC },
+	{ PCI_DEVICE(0x1106, 0x0926), .driver_data = CH_Via_86C926 },
+	{ PCI_DEVICE(0x10bd, 0x0e34), .driver_data = CH_SureCom_NE34 },
+	{ PCI_DEVICE(0x1050, 0x5a5a), .driver_data = CH_Winbond_W89C940F },
+	{ PCI_DEVICE(0x12c3, 0x0058), .driver_data = CH_Holtek_HT80232 },
+	{ PCI_DEVICE(0x12c3, 0x5598), .driver_data = CH_Holtek_HT80229 },
+	{ PCI_DEVICE(0x8c4a, 0x1980), .driver_data = CH_Winbond_89C940_8c4a },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, ne2k_pci_tbl);
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig
index b8f70e2a1763a..78c79ad7bba55 100644
--- a/drivers/net/ethernet/Kconfig
+++ b/drivers/net/ethernet/Kconfig
@@ -22,6 +22,7 @@ source "drivers/net/ethernet/aeroflex/Kconfig"
 source "drivers/net/ethernet/agere/Kconfig"
 source "drivers/net/ethernet/airoha/Kconfig"
 source "drivers/net/ethernet/alacritech/Kconfig"
+source "drivers/net/ethernet/alibaba/Kconfig"
 source "drivers/net/ethernet/allwinner/Kconfig"
 source "drivers/net/ethernet/altera/Kconfig"
 source "drivers/net/ethernet/amazon/Kconfig"
diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile
index 57344fec6ce04..bba55d9af3877 100644
--- a/drivers/net/ethernet/Makefile
+++ b/drivers/net/ethernet/Makefile
@@ -12,6 +12,7 @@ obj-$(CONFIG_NET_VENDOR_ADI) += adi/
 obj-$(CONFIG_NET_VENDOR_AGERE) += agere/
 obj-$(CONFIG_NET_VENDOR_AIROHA) += airoha/
 obj-$(CONFIG_NET_VENDOR_ALACRITECH) += alacritech/
+obj-$(CONFIG_NET_VENDOR_ALIBABA) += alibaba/
 obj-$(CONFIG_NET_VENDOR_ALLWINNER) += allwinner/
 obj-$(CONFIG_ALTERA_TSE) += altera/
 obj-$(CONFIG_NET_VENDOR_AMAZON) += amazon/
diff --git a/drivers/net/ethernet/adaptec/starfire.c b/drivers/net/ethernet/adaptec/starfire.c
index e1b8794b14c90..f1109d90e1fc1 100644
--- a/drivers/net/ethernet/adaptec/starfire.c
+++ b/drivers/net/ethernet/adaptec/starfire.c
@@ -278,8 +278,8 @@ enum chipset {
 };
 
 static const struct pci_device_id starfire_pci_tbl[] = {
-	{ PCI_VDEVICE(ADAPTEC, 0x6915), CH_6915 },
-	{ 0, }
+	{ PCI_VDEVICE(ADAPTEC, 0x6915), .driver_data = CH_6915 },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, starfire_pci_tbl);
 
diff --git a/drivers/net/ethernet/agere/et131x.c b/drivers/net/ethernet/agere/et131x.c
index 0f6e5373029ab..1b465a1676721 100644
--- a/drivers/net/ethernet/agere/et131x.c
+++ b/drivers/net/ethernet/agere/et131x.c
@@ -4084,9 +4084,9 @@ static int et131x_pci_setup(struct pci_dev *pdev,
 }
 
 static const struct pci_device_id et131x_pci_table[] = {
-	{ PCI_VDEVICE(ATT, ET131X_PCI_DEVICE_ID_GIG), 0UL},
-	{ PCI_VDEVICE(ATT, ET131X_PCI_DEVICE_ID_FAST), 0UL},
-	{ 0,}
+	{ PCI_VDEVICE(ATT, ET131X_PCI_DEVICE_ID_GIG) },
+	{ PCI_VDEVICE(ATT, ET131X_PCI_DEVICE_ID_FAST) },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, et131x_pci_table);
 
diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c
index d0c0c0ec8a80a..5a027cc7ffcb1 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.c
+++ b/drivers/net/ethernet/airoha/airoha_eth.c
@@ -201,15 +201,13 @@ static void airoha_fe_vip_setup(struct airoha_eth *eth)
 static u32 airoha_fe_get_pse_queue_rsv_pages(struct airoha_eth *eth,
 					     u32 port, u32 queue)
 {
-	u32 val;
-
 	airoha_fe_rmw(eth, REG_FE_PSE_QUEUE_CFG_WR,
 		      PSE_CFG_PORT_ID_MASK | PSE_CFG_QUEUE_ID_MASK,
 		      FIELD_PREP(PSE_CFG_PORT_ID_MASK, port) |
 		      FIELD_PREP(PSE_CFG_QUEUE_ID_MASK, queue));
-	val = airoha_fe_rr(eth, REG_FE_PSE_QUEUE_CFG_VAL);
 
-	return FIELD_GET(PSE_CFG_OQ_RSV_MASK, val);
+	return airoha_fe_get(eth, REG_FE_PSE_QUEUE_CFG_VAL,
+			     PSE_CFG_OQ_RSV_MASK);
 }
 
 static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth,
@@ -227,9 +225,7 @@ static void airoha_fe_set_pse_queue_rsv_pages(struct airoha_eth *eth,
 
 static u32 airoha_fe_get_pse_all_rsv(struct airoha_eth *eth)
 {
-	u32 val = airoha_fe_rr(eth, REG_FE_PSE_BUF_SET);
-
-	return FIELD_GET(PSE_ALLRSV_MASK, val);
+	return airoha_fe_get(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK);
 }
 
 static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth,
@@ -247,8 +243,7 @@ static int airoha_fe_set_pse_oq_rsv(struct airoha_eth *eth,
 		      FIELD_PREP(PSE_ALLRSV_MASK, all_rsv));
 
 	/* modify hthd */
-	tmp = airoha_fe_rr(eth, PSE_FQ_CFG);
-	fq_limit = FIELD_GET(PSE_FQ_LIMIT_MASK, tmp);
+	fq_limit = airoha_fe_get(eth, PSE_FQ_CFG, PSE_FQ_LIMIT_MASK);
 	tmp = fq_limit - all_rsv - 0x20;
 	airoha_fe_rmw(eth, REG_PSE_SHARE_USED_THD,
 		      PSE_SHARE_USED_HTHD_MASK,
@@ -548,9 +543,10 @@ static int airoha_qdma_fill_rx_queue(struct airoha_queue *q)
 		q->queued++;
 		nframes++;
 
+		offset += AIROHA_RX_HEADROOM;
 		e->buf = page_address(page) + offset;
 		e->dma_addr = page_pool_get_dma_addr(page) + offset;
-		e->dma_len = SKB_WITH_OVERHEAD(q->buf_size);
+		e->dma_len = SKB_WITH_OVERHEAD(AIROHA_RX_LEN(q->buf_size));
 
 		val = FIELD_PREP(QDMA_DESC_LEN_MASK, e->dma_len);
 		WRITE_ONCE(desc->ctrl, cpu_to_le32(val));
@@ -616,13 +612,12 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
 		q->tail = (q->tail + 1) % q->ndesc;
 		q->queued--;
 
-		dma_sync_single_for_cpu(eth->dev, e->dma_addr,
-					SKB_WITH_OVERHEAD(q->buf_size), dir);
+		dma_sync_single_for_cpu(eth->dev, e->dma_addr, e->dma_len,
+					dir);
 
 		page = virt_to_head_page(e->buf);
 		len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl);
-		data_len = q->skb ? q->buf_size
-				  : SKB_WITH_OVERHEAD(q->buf_size);
+		data_len = q->skb ? AIROHA_RX_LEN(q->buf_size) : e->dma_len;
 		if (!len || data_len < len)
 			goto free_frag;
 
@@ -632,10 +627,12 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget)
 
 		port = eth->ports[p];
 		if (!q->skb) { /* first buffer */
-			q->skb = napi_build_skb(e->buf, q->buf_size);
+			q->skb = napi_build_skb(e->buf - AIROHA_RX_HEADROOM,
+						q->buf_size);
 			if (!q->skb)
 				goto free_frag;
 
+			skb_reserve(q->skb, AIROHA_RX_HEADROOM);
 			__skb_put(q->skb, len);
 			skb_mark_for_recycle(q->skb);
 			q->skb->dev = port->dev;
@@ -1820,7 +1817,7 @@ static int airoha_set_gdm2_loopback(struct airoha_gdm_port *port)
 	airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX));
 	airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, BIT(AIROHA_GDM2_IDX));
 
-	src_port = eth->soc->ops.get_src_port_id(port, port->nbq);
+	src_port = eth->soc->ops.get_sport(port, port->nbq);
 	if (src_port < 0)
 		return src_port;
 
@@ -3194,7 +3191,7 @@ static const char * const en7581_xsi_rsts_names[] = {
 	"xfp-mac",
 };
 
-static int airoha_en7581_get_src_port_id(struct airoha_gdm_port *port, int nbq)
+static int airoha_en7581_get_sport(struct airoha_gdm_port *port, int nbq)
 {
 	switch (port->id) {
 	case AIROHA_GDM3_IDX:
@@ -3247,7 +3244,7 @@ static const char * const an7583_xsi_rsts_names[] = {
 	"xfp-mac",
 };
 
-static int airoha_an7583_get_src_port_id(struct airoha_gdm_port *port, int nbq)
+static int airoha_an7583_get_sport(struct airoha_gdm_port *port, int nbq)
 {
 	switch (port->id) {
 	case AIROHA_GDM3_IDX:
@@ -3295,7 +3292,7 @@ static const struct airoha_eth_soc_data en7581_soc_data = {
 	.num_xsi_rsts = ARRAY_SIZE(en7581_xsi_rsts_names),
 	.num_ppe = 2,
 	.ops = {
-		.get_src_port_id = airoha_en7581_get_src_port_id,
+		.get_sport = airoha_en7581_get_sport,
 		.get_vip_port = airoha_en7581_get_vip_port,
 	},
 };
@@ -3306,7 +3303,7 @@ static const struct airoha_eth_soc_data an7583_soc_data = {
 	.num_xsi_rsts = ARRAY_SIZE(an7583_xsi_rsts_names),
 	.num_ppe = 1,
 	.ops = {
-		.get_src_port_id = airoha_an7583_get_src_port_id,
+		.get_sport = airoha_an7583_get_sport,
 		.get_vip_port = airoha_an7583_get_vip_port,
 	},
 };
diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h
index 4fad3acc3ccf3..d3781103abb5d 100644
--- a/drivers/net/ethernet/airoha/airoha_eth.h
+++ b/drivers/net/ethernet/airoha/airoha_eth.h
@@ -32,6 +32,8 @@
 #define AIROHA_FE_MC_MAX_VLAN_TABLE	64
 #define AIROHA_FE_MC_MAX_VLAN_PORT	16
 #define AIROHA_NUM_TX_IRQ		2
+#define AIROHA_RX_HEADROOM		(NET_SKB_PAD + NET_IP_ALIGN)
+#define AIROHA_RX_LEN(_n)		((_n) - AIROHA_RX_HEADROOM)
 #define HW_DSCP_NUM			2048
 #define IRQ_QUEUE_LEN(_n)		((_n) ? 1024 : 2048)
 #define TX_DSCP_NUM			1024
@@ -578,7 +580,7 @@ struct airoha_eth_soc_data {
 	int num_xsi_rsts;
 	int num_ppe;
 	struct {
-		int (*get_src_port_id)(struct airoha_gdm_port *port, int nbq);
+		int (*get_sport)(struct airoha_gdm_port *port, int nbq);
 		u32 (*get_vip_port)(struct airoha_gdm_port *port, int nbq);
 	} ops;
 };
@@ -619,6 +621,8 @@ u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val);
 	airoha_rmw((eth)->fe_regs, (offset), 0, (val))
 #define airoha_fe_clear(eth, offset, val)			\
 	airoha_rmw((eth)->fe_regs, (offset), (val), 0)
+#define airoha_fe_get(eth, offset, mask)			\
+	FIELD_GET((mask), airoha_fe_rr((eth), (offset)))
 
 #define airoha_qdma_rr(qdma, offset)				\
 	airoha_rr((qdma)->regs, (offset))
@@ -630,6 +634,8 @@ u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val);
 	airoha_rmw((qdma)->regs, (offset), 0, (val))
 #define airoha_qdma_clear(qdma, offset, val)			\
 	airoha_rmw((qdma)->regs, (offset), (val), 0)
+#define airoha_qdma_get(qdma, offset, mask)			\
+	FIELD_GET((mask), airoha_qdma_rr((qdma), (offset)))
 
 static inline u16 airoha_qdma_get_txq(struct airoha_qdma *qdma, u16 qid)
 {
diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c
index 5c9dff6bccd1e..26da519236bfc 100644
--- a/drivers/net/ethernet/airoha/airoha_ppe.c
+++ b/drivers/net/ethernet/airoha/airoha_ppe.c
@@ -80,9 +80,8 @@ bool airoha_ppe_is_enabled(struct airoha_eth *eth, int index)
 
 static u32 airoha_ppe_get_timestamp(struct airoha_ppe *ppe)
 {
-	u16 timestamp = airoha_fe_rr(ppe->eth, REG_FE_FOE_TS);
-
-	return FIELD_GET(AIROHA_FOE_IB1_BIND_TIMESTAMP, timestamp);
+	return airoha_fe_get(ppe->eth, REG_FE_FOE_TS,
+			     AIROHA_FOE_IB1_BIND_TIMESTAMP);
 }
 
 void airoha_ppe_set_cpu_port(struct airoha_gdm_port *port, u8 ppe_id, u8 fport)
@@ -334,7 +333,7 @@ static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth,
 						info.wcid);
 		} else {
 			struct airoha_gdm_port *port = netdev_priv(dev);
-			u8 pse_port;
+			u8 pse_port, channel;
 
 			if (!airoha_is_valid_gdm_port(eth, port))
 				return -EINVAL;
@@ -347,6 +346,14 @@ static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth,
 					       * loopback
 					       */
 
+			/* For traffic forwarded to DSA devices select QoS
+			 * channel according to the DSA user port index, rely
+			 * on port id otherwise.
+			 */
+			channel = dsa_port >= 0 ? dsa_port : port->id;
+			channel = channel % AIROHA_NUM_QOS_CHANNELS;
+			qdata |= FIELD_PREP(AIROHA_FOE_CHANNEL, channel);
+
 			val |= FIELD_PREP(AIROHA_FOE_IB2_PSE_PORT, pse_port) |
 			       AIROHA_FOE_IB2_PSE_QOS;
 			/* For downlink traffic consume SRAM memory for hw
diff --git a/drivers/net/ethernet/alibaba/Kconfig b/drivers/net/ethernet/alibaba/Kconfig
new file mode 100644
index 0000000000000..b8fd3c92fa4cc
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/Kconfig
@@ -0,0 +1,28 @@
+#
+# Alibaba network device configuration
+#
+
+config NET_VENDOR_ALIBABA
+	bool "Alibaba Devices"
+	default y
+	help
+	  If you have a network (Ethernet) device belonging to this class, say Y.
+
+	  Note that the answer to this question doesn't directly affect the
+	  kernel: saying N will just cause the configurator to skip all
+	  the questions about Alibaba devices. If you say Y, you will be asked
+	  for your specific device in the following questions.
+
+if NET_VENDOR_ALIBABA
+
+config ALIBABA_EEA
+	tristate "Alibaba Elastic Ethernet Adapter support"
+	depends on PCI_MSI
+	depends on 64BIT
+	select PAGE_POOL
+	help
+	  This driver supports Alibaba Elastic Ethernet Adapter.
+
+	  To compile this driver as a module, choose M here.
+
+endif #NET_VENDOR_ALIBABA
diff --git a/drivers/net/ethernet/alibaba/Makefile b/drivers/net/ethernet/alibaba/Makefile
new file mode 100644
index 0000000000000..63a527e79ba76
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/Makefile
@@ -0,0 +1,5 @@
+#
+# Makefile for the Alibaba network device drivers.
+#
+
+obj-$(CONFIG_ALIBABA_EEA) += eea/
diff --git a/drivers/net/ethernet/alibaba/eea/Makefile b/drivers/net/ethernet/alibaba/eea/Makefile
new file mode 100644
index 0000000000000..c38db22cca343
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/Makefile
@@ -0,0 +1,9 @@
+
+obj-$(CONFIG_ALIBABA_EEA) += eea.o
+eea-y := eea_ring.o \
+	eea_net.o \
+	eea_pci.o \
+	eea_adminq.o \
+	eea_ethtool.o \
+	eea_tx.o \
+	eea_rx.o
diff --git a/drivers/net/ethernet/alibaba/eea/eea_adminq.c b/drivers/net/ethernet/alibaba/eea/eea_adminq.c
new file mode 100644
index 0000000000000..dfad1bdbc44d8
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_adminq.c
@@ -0,0 +1,542 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/iopoll.h>
+#include <linux/utsname.h>
+#include <linux/version.h>
+
+#include "eea_adminq.h"
+#include "eea_net.h"
+#include "eea_pci.h"
+#include "eea_ring.h"
+
+#define EEA_AQ_CMD_CFG_QUERY         ((0 << 8) | 0)
+
+#define EEA_AQ_CMD_QUEUE_CREATE      ((1 << 8) | 0)
+#define EEA_AQ_CMD_QUEUE_DESTROY_ALL ((1 << 8) | 1)
+
+#define EEA_AQ_CMD_HOST_INFO         ((2 << 8) | 0)
+
+#define EEA_AQ_CMD_DEV_STATUS        ((3 << 8) | 0)
+
+#define EEA_RING_DESC_F_AQ_PHASE     (BIT(15) | BIT(7))
+
+#define EEA_QUEUE_FLAGS_HW_SPLIT_HDR BIT(0)
+#define EEA_QUEUE_FLAGS_SQCQ         BIT(1)
+#define EEA_QUEUE_FLAGS_HWTS         BIT(2)
+
+struct eea_aq_create {
+	__le32 flags;
+	/* queue index.
+	 * rx: 0 == qidx % 2
+	 * tx: 1 == qidx % 2
+	 */
+	__le16 qidx;
+	/* the depth of the queue */
+	__le16 depth;
+	/*  0: without SPLIT HDR
+	 *  1: 128B
+	 *  2: 256B
+	 *  3: 512B
+	 */
+	u8 hdr_buf_size;
+	u8 sq_desc_size;
+	u8 cq_desc_size;
+	u8 reserve0;
+	/* The vector for the irq. rx,tx share the same vector */
+	__le16 msix_vector;
+	__le16 reserve;
+	/* sq ring cfg. */
+	__le32 sq_addr_low;
+	__le32 sq_addr_high;
+	/* cq ring cfg. Just valid when flags include EEA_QUEUE_FLAGS_SQCQ. */
+	__le32 cq_addr_low;
+	__le32 cq_addr_high;
+};
+
+struct eea_aq_queue_drv_status {
+	__le16 qidx;
+
+	__le16 sq_head;
+	__le16 cq_head;
+	__le16 reserved;
+};
+
+#define EEA_OS_DISTRO		0
+#define EEA_DRV_TYPE		0
+#define EEA_OS_LINUX		1
+#define EEA_SPEC_VER_MAJOR	1
+#define EEA_SPEC_VER_MINOR	0
+
+struct eea_aq_host_info_cfg {
+	__le16	os_type;
+	__le16	os_dist;
+	__le16	drv_type;
+
+	__le16	kern_ver_major;
+	__le16	kern_ver_minor;
+	__le16	kern_ver_sub_minor;
+
+	__le16	drv_ver_major;
+	__le16	drv_ver_minor;
+	__le16	drv_ver_sub_minor;
+
+	__le16	spec_ver_major;
+	__le16	spec_ver_minor;
+	__le16	pci_bdf;
+	__le32	pci_domain;
+
+	u8      os_ver_str[64];
+	u8      isa_str[64];
+};
+
+#define EEA_HINFO_MAX_REP_LEN	1024
+#define EEA_HINFO_REP_BAD	2
+
+struct eea_aq_host_info_rep {
+	u8	op_code;
+	u8	has_reply;
+	u8	reply_str[EEA_HINFO_MAX_REP_LEN];
+};
+
+static struct eea_ring *qid_to_ering(struct eea_net *enet, u32 qid)
+{
+	struct eea_ring *ering;
+
+	if (qid % 2 == 0)
+		ering = enet->rx[qid / 2]->ering;
+	else
+		ering = enet->tx[qid / 2].ering;
+
+	return ering;
+}
+
+#define EEA_AQ_TIMEOUT_US (60 * 1000 * 1000)
+
+static void eea_device_broken(struct eea_net *enet)
+{
+	if (enet->adminq.broken)
+		return;
+
+	eea_device_reset(enet->edev);
+	enet->adminq.broken = true;
+}
+
+static int eea_adminq_submit(struct eea_net *enet, u16 cmd,
+			     dma_addr_t req_addr, dma_addr_t res_addr,
+			     u32 req_size, u32 res_size, u32 *reply_len)
+{
+	struct eea_aq_cdesc *cdesc;
+	struct eea_aq_desc *desc;
+	int ret;
+
+	if (enet->adminq.broken)
+		return -EIO;
+
+	desc = eea_ering_aq_alloc_desc(enet->adminq.ring);
+
+	desc->classid = cmd >> 8;
+	desc->command = cmd & 0xff;
+
+	desc->data_addr = cpu_to_le64(req_addr);
+	desc->data_len = cpu_to_le32(req_size);
+
+	desc->reply_addr = cpu_to_le64(res_addr);
+	desc->reply_len = cpu_to_le32(res_size);
+
+	/* for update flags */
+	dma_wmb();
+
+	desc->flags = cpu_to_le16(enet->adminq.phase);
+
+	eea_ering_sq_commit_desc(enet->adminq.ring);
+
+	eea_ering_kick(enet->adminq.ring);
+
+	++enet->adminq.num;
+
+	if ((enet->adminq.num % enet->adminq.ring->num) == 0)
+		enet->adminq.phase ^= EEA_RING_DESC_F_AQ_PHASE;
+
+	ret = read_poll_timeout(eea_ering_cq_get_desc, cdesc, cdesc, 10,
+				EEA_AQ_TIMEOUT_US, false, enet->adminq.ring);
+	if (ret) {
+		netdev_err(enet->netdev,
+			   "adminq exec timeout. cmd: %d reset device.\n",
+			   cmd);
+		/* The device must be reset before unmapping buffers to avoid
+		 * potential DMA writes after the memory is freed.
+		 */
+		eea_device_broken(enet);
+		return ret;
+	}
+
+	/* Returns 0 on success, or a negative error code on failure. */
+	ret = le32_to_cpu(cdesc->status);
+
+	eea_ering_cq_ack_desc(enet->adminq.ring, 1);
+
+	if (ret)
+		netdev_err(enet->netdev,
+			   "adminq exec failed. cmd: %d ret %d\n", cmd, ret);
+	else
+		*reply_len = le32_to_cpu(cdesc->reply_len);
+
+	return ret;
+}
+
+static int eea_adminq_exec(struct eea_net *enet, u16 cmd,
+			   void *req, u32 req_size,
+			   void *res, u32 res_size,
+			   u32 *reply)
+{
+	dma_addr_t req_addr = 0, res_addr = 0;
+	struct device *dma;
+	u32 reply_len = 0;
+	int ret;
+
+	if (reply)
+		*reply = 0;
+
+	dma = enet->edev->dma_dev;
+
+	if (req) {
+		req_addr = dma_map_single(dma, req, req_size, DMA_TO_DEVICE);
+		if (unlikely(dma_mapping_error(dma, req_addr)))
+			return -ENOMEM;
+	}
+
+	if (res) {
+		res_addr = dma_map_single(dma, res, res_size, DMA_FROM_DEVICE);
+		if (unlikely(dma_mapping_error(dma, res_addr))) {
+			ret = -ENOMEM;
+			goto err_unmap_req;
+		}
+	}
+
+	mutex_lock(&enet->adminq.lock);
+	ret = eea_adminq_submit(enet, cmd, req_addr, res_addr,
+				req_size, res_size, &reply_len);
+	mutex_unlock(&enet->adminq.lock);
+	if (res) {
+		dma_unmap_single(dma, res_addr, res_size, DMA_FROM_DEVICE);
+
+		if (ret)
+			memset(res, 0, res_size);
+		else if (res_size > reply_len)
+			memset(res + reply_len, 0, res_size - reply_len);
+
+		if (reply)
+			*reply = reply_len;
+	}
+
+err_unmap_req:
+	if (req)
+		dma_unmap_single(dma, req_addr, req_size, DMA_TO_DEVICE);
+
+	return ret;
+}
+
+void eea_destroy_adminq(struct eea_net *enet)
+{
+	struct eea_aq *aq;
+
+	aq = &enet->adminq;
+
+	if (aq->ring) {
+		eea_ering_free(aq->ring);
+		aq->ring = NULL;
+		aq->phase = 0;
+	}
+
+	kfree(aq->q_req_buf);
+	kfree(aq->q_res_buf);
+
+	aq->q_req_buf = NULL;
+	aq->q_res_buf = NULL;
+}
+
+int eea_create_adminq(struct eea_net *enet, u32 qid)
+{
+	u32 db_size, q_size, num;
+	struct eea_ring *ering;
+	struct eea_aq *aq;
+	int err = -ENOMEM;
+
+	num = enet->edev->rx_num + enet->edev->tx_num;
+	aq = &enet->adminq;
+
+	ering = eea_ering_alloc(qid, 64, enet->edev, sizeof(struct eea_aq_desc),
+				sizeof(struct eea_aq_cdesc), "adminq");
+	if (!ering)
+		return -ENOMEM;
+
+	aq->ring = ering;
+
+	err = eea_pci_active_aq(ering, qid / 2 + 1);
+	if (err)
+		goto err;
+
+	aq->phase = BIT(7);
+	aq->num = 0;
+
+	q_size = sizeof(*aq->q_req_buf) * num;
+	db_size = sizeof(*aq->q_res_buf) * num;
+
+	aq->q_req_size = q_size;
+	aq->q_res_size = db_size;
+
+	err = -ENOMEM;
+
+	aq->q_req_buf = kzalloc(q_size, GFP_KERNEL);
+	if (!aq->q_req_buf)
+		goto err;
+
+	aq->q_res_buf = kzalloc(db_size, GFP_KERNEL);
+	if (!aq->q_res_buf)
+		goto err;
+
+	/* Before we set up the AQ, the device remains in an inactive state, so
+	 * there will be no DMA operations. If the 'set up AQ' process fails, we
+	 * can safely free the DMA-related memory.
+	 */
+	err = eea_pci_set_aq_up(enet->edev);
+	if (err)
+		goto err;
+
+	aq->broken = false;
+
+	mutex_init(&aq->lock);
+
+	return 0;
+
+err:
+	eea_destroy_adminq(enet);
+	return err;
+}
+
+int eea_adminq_query_cfg(struct eea_net *enet, struct eea_aq_cfg *cfg)
+{
+	return eea_adminq_exec(enet, EEA_AQ_CMD_CFG_QUERY, NULL, 0, cfg,
+			       sizeof(*cfg), NULL);
+}
+
+static void qcfg_fill(struct eea_aq_create *qcfg, struct eea_ring *ering,
+		      u32 flags)
+{
+	qcfg->flags = cpu_to_le32(flags);
+	qcfg->qidx = cpu_to_le16(ering->index);
+	qcfg->depth = cpu_to_le16(ering->num);
+
+	qcfg->hdr_buf_size = flags & EEA_QUEUE_FLAGS_HW_SPLIT_HDR ? 1 : 0;
+	qcfg->sq_desc_size = ering->sq.desc_size;
+	qcfg->cq_desc_size = ering->cq.desc_size;
+	qcfg->msix_vector = cpu_to_le16(ering->msix_vec);
+
+	qcfg->sq_addr_low = cpu_to_le32(lower_32_bits(ering->sq.dma_addr));
+	qcfg->sq_addr_high = cpu_to_le32(upper_32_bits(ering->sq.dma_addr));
+
+	qcfg->cq_addr_low = cpu_to_le32(lower_32_bits(ering->cq.dma_addr));
+	qcfg->cq_addr_high = cpu_to_le32(upper_32_bits(ering->cq.dma_addr));
+}
+
+int eea_adminq_create_q(struct eea_net *enet, u32 num, u32 flags)
+{
+	int i, db_size, q_size, err = -ENOMEM;
+	struct eea_net_cfg *cfg;
+	struct eea_ring *ering;
+	struct eea_aq *aq;
+	u32 reply_len;
+
+	cfg = &enet->cfg;
+	aq = &enet->adminq;
+
+	if (cfg->split_hdr)
+		flags |= EEA_QUEUE_FLAGS_HW_SPLIT_HDR;
+
+	flags |= EEA_QUEUE_FLAGS_SQCQ;
+	flags |= EEA_QUEUE_FLAGS_HWTS;
+
+	q_size = sizeof(*aq->q_req_buf) * num;
+	db_size = sizeof(*aq->q_res_buf) * num;
+
+	for (i = 0; i < num; i++) {
+		ering = qid_to_ering(enet, i);
+		qcfg_fill(aq->q_req_buf + i, ering, flags);
+	}
+
+	err = eea_adminq_exec(enet, EEA_AQ_CMD_QUEUE_CREATE,
+			      aq->q_req_buf, q_size,
+			      aq->q_res_buf, db_size,
+			      &reply_len);
+	if (err)
+		return err;
+
+	if (reply_len != db_size) {
+		eea_adminq_destroy_all_q(enet);
+		netdev_err(enet->netdev, "invalid reply len %u\n", reply_len);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < num; i++) {
+		ering = qid_to_ering(enet, i);
+		ering->db = eea_pci_db_addr(ering->edev,
+					    le32_to_cpu(aq->q_res_buf[i]));
+		if (!ering->db) {
+			netdev_err(enet->netdev, "invalid db off %u\n",
+				   le32_to_cpu(aq->q_res_buf[i]));
+			goto err;
+		}
+	}
+
+	return err;
+
+err:
+	eea_adminq_destroy_all_q(enet);
+	for (i = 0; i < num; i++) {
+		ering = qid_to_ering(enet, i);
+		ering->db = NULL;
+	}
+
+	return -EIO;
+}
+
+int eea_adminq_destroy_all_q(struct eea_net *enet)
+{
+	int err;
+
+	err = eea_adminq_exec(enet, EEA_AQ_CMD_QUEUE_DESTROY_ALL, NULL, 0,
+			      NULL, 0, NULL);
+	if (err) {
+		/* The device must be reset before unmapping buffers to avoid
+		 * potential DMA writes after the memory is freed.
+		 */
+		mutex_lock(&enet->adminq.lock);
+		eea_device_broken(enet);
+		mutex_unlock(&enet->adminq.lock);
+
+		netdev_err(enet->netdev, "QUEUE_DESTROY fail: reset device.\n");
+	}
+
+	return err;
+}
+
+/* The caller must ensure that both the 'rx' and 'tx' arrays are valid. */
+int eea_adminq_dev_status(struct eea_net *enet,
+			  struct eea_aq_dev_status *dstatus)
+{
+	struct eea_aq_queue_drv_status *drv_status;
+	struct __eea_aq_dev_status *dev_status;
+	int err, i, io_num, size, q_num;
+	struct eea_ring *ering;
+	void *rep, *req;
+
+	q_num = enet->cfg.rx_ring_num + enet->cfg.tx_ring_num + 1;
+	io_num = enet->cfg.rx_ring_num + enet->cfg.tx_ring_num;
+
+	req = kcalloc(q_num, sizeof(struct eea_aq_queue_drv_status),
+		      GFP_KERNEL);
+	if (!req)
+		return -ENOMEM;
+
+	size = struct_size(dev_status, q_status, q_num);
+
+	rep = kzalloc(size, GFP_KERNEL);
+	if (!rep) {
+		kfree(req);
+		return -ENOMEM;
+	}
+
+	drv_status = req;
+	for (i = 0; i < io_num; ++i, ++drv_status) {
+		ering = qid_to_ering(enet, i);
+		drv_status->qidx = cpu_to_le16(i);
+		drv_status->cq_head = cpu_to_le16(ering->cq.head);
+		drv_status->sq_head = cpu_to_le16(ering->sq.head);
+	}
+
+	drv_status->qidx = cpu_to_le16(i);
+	drv_status->cq_head = cpu_to_le16(enet->adminq.ring->cq.head);
+	drv_status->sq_head = cpu_to_le16(enet->adminq.ring->sq.head);
+
+	err = eea_adminq_exec(enet, EEA_AQ_CMD_DEV_STATUS, req,
+			      q_num * sizeof(struct eea_aq_queue_drv_status),
+			      rep, size, NULL);
+	kfree(req);
+	if (err) {
+		kfree(rep);
+		return err;
+	}
+
+	dstatus->num = q_num;
+	dstatus->status = rep;
+
+	return 0;
+}
+
+void eea_adminq_config_host_info(struct eea_net *enet)
+{
+	struct device *dev = enet->edev->dma_dev;
+	struct eea_aq_host_info_cfg *cfg;
+	struct eea_aq_host_info_rep *rep;
+	int rc = -ENOMEM;
+
+	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
+	if (!cfg)
+		return;
+
+	rep = kzalloc(sizeof(*rep), GFP_KERNEL);
+	if (!rep)
+		goto err_free_cfg;
+
+	cfg->os_type            = cpu_to_le16(EEA_OS_LINUX);
+	cfg->os_dist            = cpu_to_le16(EEA_OS_DISTRO);
+	cfg->drv_type           = cpu_to_le16(EEA_DRV_TYPE);
+
+	cfg->kern_ver_major     = cpu_to_le16(LINUX_VERSION_MAJOR);
+	cfg->kern_ver_minor     = cpu_to_le16(LINUX_VERSION_PATCHLEVEL);
+	cfg->kern_ver_sub_minor = cpu_to_le16(LINUX_VERSION_SUBLEVEL);
+
+	cfg->drv_ver_major      = cpu_to_le16(EEA_VER_MAJOR);
+	cfg->drv_ver_minor      = cpu_to_le16(EEA_VER_MINOR);
+	cfg->drv_ver_sub_minor  = cpu_to_le16(EEA_VER_SUB_MINOR);
+
+	cfg->spec_ver_major     = cpu_to_le16(EEA_SPEC_VER_MAJOR);
+	cfg->spec_ver_minor     = cpu_to_le16(EEA_SPEC_VER_MINOR);
+
+	cfg->pci_bdf            = cpu_to_le16(eea_pci_bdf(enet->edev));
+	cfg->pci_domain         = cpu_to_le32(eea_pci_domain_nr(enet->edev));
+
+	strscpy(cfg->os_ver_str, utsname()->release, sizeof(cfg->os_ver_str));
+	strscpy(cfg->isa_str, utsname()->machine, sizeof(cfg->isa_str));
+
+	rc = eea_adminq_exec(enet, EEA_AQ_CMD_HOST_INFO,
+			     cfg, sizeof(*cfg), rep, sizeof(*rep), NULL);
+
+	if (!rc) {
+		if (rep->op_code == EEA_HINFO_REP_BAD)
+			dev_warn(dev, "The hardware-driven state validation may be abnormal.\n");
+
+		if (rep->has_reply) {
+			char buf[EEA_HINFO_MAX_REP_LEN] = {0};
+
+			rep->reply_str[EEA_HINFO_MAX_REP_LEN - 1] = '\0';
+
+			string_escape_str(rep->reply_str, buf, sizeof(buf),
+					  ESCAPE_NP, NULL);
+
+			buf[EEA_HINFO_MAX_REP_LEN - 1] = '\0';
+
+			dev_warn(dev, "Device replied: %s\n", buf);
+		}
+	}
+
+	kfree(rep);
+err_free_cfg:
+	kfree(cfg);
+}
diff --git a/drivers/net/ethernet/alibaba/eea/eea_adminq.h b/drivers/net/ethernet/alibaba/eea/eea_adminq.h
new file mode 100644
index 0000000000000..0182f5641fcf2
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_adminq.h
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#ifndef __EEA_ADMINQ_H__
+#define __EEA_ADMINQ_H__
+
+struct eea_aq_cfg {
+	__le32 rx_depth_max;
+	__le32 rx_depth_def;
+
+	__le32 tx_depth_max;
+	__le32 tx_depth_def;
+
+	__le32 max_tso_size;
+	__le32 max_tso_segs;
+
+	u8 mac[ETH_ALEN];
+	__le16 status;
+
+	__le16 mtu;
+	__le16 reserved0;
+	__le16 reserved1;
+	u8 reserved2;
+	u8 reserved3;
+
+	__le16 reserved4;
+	__le16 reserved5;
+	__le16 reserved6;
+};
+
+struct eea_aq_queue_status {
+	__le16 qidx;
+#define EEA_QUEUE_STATUS_OK 0
+#define EEA_QUEUE_STATUS_NEED_RESET 1
+	__le16 status;
+};
+
+struct __eea_aq_dev_status {
+#define EEA_LINK_DOWN_STATUS  0
+#define EEA_LINK_UP_STATUS    1
+	__le16 link_status;
+	__le16 reserved;
+
+	struct eea_aq_queue_status q_status[];
+};
+
+struct eea_aq_dev_status {
+	u32 num;
+	struct __eea_aq_dev_status *status;
+};
+
+struct eea_aq {
+	struct eea_ring *ring;
+	u32 num;
+	bool broken;
+	u16 phase;
+
+	/* lock for adminq exec */
+	struct mutex lock;
+
+	u32 q_req_size;
+	u32 q_res_size;
+	struct eea_aq_create *q_req_buf;
+	__le32 *q_res_buf;
+};
+
+struct eea_net;
+
+int eea_create_adminq(struct eea_net *enet, u32 qid);
+void eea_destroy_adminq(struct eea_net *enet);
+
+int eea_adminq_query_cfg(struct eea_net *enet, struct eea_aq_cfg *cfg);
+
+int eea_adminq_create_q(struct eea_net *enet, u32 num, u32 flags);
+int eea_adminq_destroy_all_q(struct eea_net *enet);
+int eea_adminq_dev_status(struct eea_net *enet,
+			  struct eea_aq_dev_status *dstatus);
+void eea_adminq_config_host_info(struct eea_net *enet);
+#endif
diff --git a/drivers/net/ethernet/alibaba/eea/eea_desc.h b/drivers/net/ethernet/alibaba/eea/eea_desc.h
new file mode 100644
index 0000000000000..8d94a0f0f2379
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_desc.h
@@ -0,0 +1,138 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#ifndef __EEA_DESC_H__
+#define __EEA_DESC_H__
+
+#define EEA_DESC_TS_MASK GENMASK_ULL(47, 0)
+#define EEA_DESC_TS(desc) (le64_to_cpu((desc)->ts) & EEA_DESC_TS_MASK)
+
+struct eea_aq_desc {
+	__le16 flags;
+	__le16 id;
+	__le16 reserved;
+	u8 classid;
+	u8 command;
+	__le64 data_addr;
+	__le64 reply_addr;
+	__le32 data_len;
+	__le32 reply_len;
+};
+
+struct eea_aq_cdesc {
+	__le16 flags;
+	__le16 id;
+#define EEA_OK     0
+#define EEA_ERR    0xffffffff
+	__le32 status;
+	__le32 reply_len;
+	__le32 reserved1;
+
+	__le64 reserved2;
+	__le64 reserved3;
+};
+
+struct eea_rx_desc_no_hdr {
+	__le16 flags;
+	__le16 id;
+	__le16 len;
+	__le16 reserved1;
+
+	__le64 addr;
+};
+
+struct eea_rx_desc {
+	__le16 flags;
+	__le16 id;
+	__le16 len;
+	__le16 reserved1;
+
+	__le64 addr;
+
+	__le64 hdr_addr;
+	__le32 reserved2;
+	__le32 reserved3;
+};
+
+#define EEA_RX_CDESC_HDR_LEN_MASK GENMASK_ULL(9, 0)
+
+struct eea_rx_cdesc {
+#define EEA_DESC_F_DATA_VALID	BIT(6)
+#define EEA_DESC_F_SPLIT_HDR	BIT(5)
+	__le16 flags;
+	__le16 id;
+	__le16 len;
+#define EEA_NET_PT_NONE      0
+#define EEA_NET_PT_IPv4      1
+#define EEA_NET_PT_TCPv4     2
+#define EEA_NET_PT_UDPv4     3
+#define EEA_NET_PT_IPv6      4
+#define EEA_NET_PT_TCPv6     5
+#define EEA_NET_PT_UDPv6     6
+#define EEA_NET_PT_IPv6_EX   7
+#define EEA_NET_PT_TCPv6_EX  8
+#define EEA_NET_PT_UDPv6_EX  9
+	/* [9:0] is packet type. */
+	__le16 type;
+
+	/* hw timestamp [0:47]: ts */
+	__le64 ts;
+
+	__le32 hash;
+
+	/* 0-9: hdr_len  split header
+	 * 10-15: reserved1
+	 */
+	__le16 len_ex;
+	__le16 reserved2;
+
+	__le32 reserved3;
+	__le32 reserved4;
+};
+
+#define EEA_TX_GSO_NONE   0
+#define EEA_TX_GSO_TCPV4  1
+#define EEA_TX_GSO_TCPV6  4
+#define EEA_TX_GSO_UDP_L4 5
+#define EEA_TX_GSO_ECN    0x80
+
+struct eea_tx_desc {
+#define EEA_DESC_F_DO_CSUM	BIT(6)
+	__le16 flags;
+	__le16 id;
+	__le16 len;
+	__le16 reserved1;
+
+	__le64 addr;
+
+	__le16 csum_start;
+	__le16 csum_offset;
+	u8 gso_type;
+	u8 reserved2;
+	__le16 gso_size;
+	__le64 reserved3;
+};
+
+struct eea_tx_cdesc {
+	__le16 flags;
+	__le16 id;
+	__le16 len;
+	__le16 reserved1;
+
+	/* hw timestamp [0:47]: ts */
+	__le64 ts;
+};
+
+#define EEA_DB_FLAGS_OFF      0
+#define EEA_DB_IDX_OFF        (2 * 8)
+#define EEA_DB_TX_CQ_HEAD_OFF (4 * 8)
+#define EEA_DB_RX_CQ_HEAD_OFF (6 * 8)
+
+#define EEA_IDX_PRESENT   BIT(0)
+#define EEA_IRQ_MASK      BIT(1)
+#define EEA_IRQ_UNMASK    BIT(2)
+#endif
diff --git a/drivers/net/ethernet/alibaba/eea/eea_ethtool.c b/drivers/net/ethernet/alibaba/eea/eea_ethtool.c
new file mode 100644
index 0000000000000..479779aa2dd88
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_ethtool.c
@@ -0,0 +1,273 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#include <linux/ethtool.h>
+#include <linux/ethtool_netlink.h>
+#include <linux/rtnetlink.h>
+
+#include "eea_adminq.h"
+#include "eea_net.h"
+#include "eea_pci.h"
+
+struct eea_stat_desc {
+	char desc[ETH_GSTRING_LEN];
+	size_t offset;
+};
+
+#define EEA_TX_STAT(m)	{#m, offsetof(struct eea_tx_stats, m)}
+#define EEA_RX_STAT(m)	{#m, offsetof(struct eea_rx_stats, m)}
+
+static const struct eea_stat_desc eea_rx_stats_desc[] = {
+	EEA_RX_STAT(descs),
+	EEA_RX_STAT(kicks),
+};
+
+static const struct eea_stat_desc eea_tx_stats_desc[] = {
+	EEA_TX_STAT(descs),
+	EEA_TX_STAT(kicks),
+};
+
+#define EEA_TX_STATS_LEN	ARRAY_SIZE(eea_tx_stats_desc)
+#define EEA_RX_STATS_LEN	ARRAY_SIZE(eea_rx_stats_desc)
+
+static void eea_get_drvinfo(struct net_device *netdev,
+			    struct ethtool_drvinfo *info)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	struct eea_device *edev = enet->edev;
+
+	strscpy(info->driver,   KBUILD_MODNAME,     sizeof(info->driver));
+	strscpy(info->bus_info, eea_pci_name(edev), sizeof(info->bus_info));
+}
+
+static void eea_get_ringparam(struct net_device *netdev,
+			      struct ethtool_ringparam *ring,
+			      struct kernel_ethtool_ringparam *kernel_ring,
+			      struct netlink_ext_ack *extack)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+
+	ring->rx_max_pending = enet->cfg_hw.rx_ring_depth;
+	ring->tx_max_pending = enet->cfg_hw.tx_ring_depth;
+	ring->rx_pending = enet->cfg.rx_ring_depth;
+	ring->tx_pending = enet->cfg.tx_ring_depth;
+
+	kernel_ring->tcp_data_split = enet->cfg.split_hdr ?
+				      ETHTOOL_TCP_DATA_SPLIT_ENABLED :
+				      ETHTOOL_TCP_DATA_SPLIT_DISABLED;
+}
+
+static int eea_set_ringparam(struct net_device *netdev,
+			     struct ethtool_ringparam *ring,
+			     struct kernel_ethtool_ringparam *kernel_ring,
+			     struct netlink_ext_ack *extack)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	struct eea_net_init_ctx ctx;
+	bool need_update = false;
+	struct eea_net_cfg *cfg;
+	bool sh;
+
+	if (ring->rx_pending < EEA_NET_IO_RING_DEPTH_MIN ||
+	    ring->tx_pending < EEA_NET_IO_RING_DEPTH_MIN)
+		return -EINVAL;
+
+	if (!is_power_of_2(ring->rx_pending) ||
+	    !is_power_of_2(ring->tx_pending))
+		return -EINVAL;
+
+	eea_init_ctx(enet, &ctx);
+
+	cfg = &ctx.cfg;
+
+	if (ring->rx_pending != cfg->rx_ring_depth)
+		need_update = true;
+
+	if (ring->tx_pending != cfg->tx_ring_depth)
+		need_update = true;
+
+	sh = false;
+
+	switch (kernel_ring->tcp_data_split) {
+	case ETHTOOL_TCP_DATA_SPLIT_ENABLED:
+		sh = true;
+		break;
+
+	case ETHTOOL_TCP_DATA_SPLIT_DISABLED:
+		sh = false;
+		break;
+
+	case ETHTOOL_TCP_DATA_SPLIT_UNKNOWN:
+		sh = !!cfg->split_hdr;
+		break;
+	}
+
+	if (sh != !!(cfg->split_hdr))
+		need_update = true;
+
+	if (!need_update)
+		return 0;
+
+	cfg->rx_ring_depth = ring->rx_pending;
+	cfg->tx_ring_depth = ring->tx_pending;
+
+	/* By default, enet->cfg_hw.split_hdr is 128. */
+	cfg->split_hdr = sh ? enet->cfg_hw.split_hdr : 0;
+
+	return eea_reset_hw_resources(enet, &ctx);
+}
+
+static int eea_set_channels(struct net_device *netdev,
+			    struct ethtool_channels *channels)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	u16 queue_pairs = channels->combined_count;
+	struct eea_net_init_ctx ctx;
+	struct eea_net_cfg *cfg;
+
+	eea_init_ctx(enet, &ctx);
+
+	cfg = &ctx.cfg;
+
+	cfg->rx_ring_num = queue_pairs;
+	cfg->tx_ring_num = queue_pairs;
+
+	return eea_reset_hw_resources(enet, &ctx);
+}
+
+static void eea_get_channels(struct net_device *netdev,
+			     struct ethtool_channels *channels)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+
+	channels->combined_count = enet->cfg.rx_ring_num;
+	channels->max_combined   = enet->cfg_hw.rx_ring_num;
+}
+
+static void eea_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	u8 *p = data;
+	u32 i, j;
+
+	if (stringset != ETH_SS_STATS)
+		return;
+
+	for (i = 0; i < enet->cfg.rx_ring_num; i++) {
+		for (j = 0; j < EEA_RX_STATS_LEN; j++)
+			ethtool_sprintf(&p, "rx%u_%s", i,
+					eea_rx_stats_desc[j].desc);
+	}
+
+	for (i = 0; i < enet->cfg.tx_ring_num; i++) {
+		for (j = 0; j < EEA_TX_STATS_LEN; j++)
+			ethtool_sprintf(&p, "tx%u_%s", i,
+					eea_tx_stats_desc[j].desc);
+	}
+}
+
+static int eea_get_sset_count(struct net_device *netdev, int sset)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+
+	if (sset != ETH_SS_STATS)
+		return -EOPNOTSUPP;
+
+	return enet->cfg.rx_ring_num * EEA_RX_STATS_LEN +
+		enet->cfg.tx_ring_num * EEA_TX_STATS_LEN;
+}
+
+static void eea_stats_fill_for_q(struct u64_stats_sync *syncp, u32 num,
+				 const struct eea_stat_desc *desc,
+				 u64 *data, u32 idx)
+{
+	void *stats_base = syncp;
+	u32 start, i;
+
+	do {
+		start = u64_stats_fetch_begin(syncp);
+		for (i = 0; i < num; i++)
+			data[idx + i] =
+				u64_stats_read(stats_base + desc[i].offset);
+
+	} while (u64_stats_fetch_retry(syncp, start));
+
+	BUILD_BUG_ON(offsetof(struct eea_tx_stats, syncp));
+	BUILD_BUG_ON(offsetof(struct eea_rx_stats, syncp));
+}
+
+static void eea_get_ethtool_stats(struct net_device *netdev,
+				  struct ethtool_stats *stats, u64 *data)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	u32 i, idx = 0;
+
+	ASSERT_RTNL();
+
+	if (enet->rx) {
+		for (i = 0; i < enet->cfg.rx_ring_num; i++) {
+			struct eea_net_rx *rx = enet->rx[i];
+
+			eea_stats_fill_for_q(&rx->stats.syncp, EEA_RX_STATS_LEN,
+					     eea_rx_stats_desc, data, idx);
+
+			idx += EEA_RX_STATS_LEN;
+		}
+	}
+
+	if (enet->tx) {
+		for (i = 0; i < enet->cfg.tx_ring_num; i++) {
+			struct eea_net_tx *tx = &enet->tx[i];
+
+			eea_stats_fill_for_q(&tx->stats.syncp, EEA_TX_STATS_LEN,
+					     eea_tx_stats_desc, data, idx);
+
+			idx += EEA_TX_STATS_LEN;
+		}
+	}
+}
+
+void eea_update_rx_stats(struct eea_rx_stats *rx_stats,
+			 struct eea_rx_ctx_stats *stats)
+{
+	u64_stats_update_begin(&rx_stats->syncp);
+	u64_stats_add(&rx_stats->descs,             stats->descs);
+	u64_stats_add(&rx_stats->packets,           stats->packets);
+	u64_stats_add(&rx_stats->bytes,             stats->bytes);
+	u64_stats_add(&rx_stats->drops,             stats->drops);
+	u64_stats_add(&rx_stats->split_hdr_bytes,   stats->split_hdr_bytes);
+	u64_stats_add(&rx_stats->split_hdr_packets, stats->split_hdr_packets);
+	u64_stats_add(&rx_stats->length_errors,     stats->length_errors);
+	u64_stats_add(&rx_stats->kicks,             stats->kicks);
+	u64_stats_update_end(&rx_stats->syncp);
+}
+
+static int eea_get_link_ksettings(struct net_device *netdev,
+				  struct ethtool_link_ksettings *cmd)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+
+	cmd->base.speed  = enet->speed;
+	cmd->base.duplex = enet->duplex;
+	cmd->base.port   = PORT_OTHER;
+
+	return 0;
+}
+
+const struct ethtool_ops eea_ethtool_ops = {
+	.supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT,
+	.get_drvinfo        = eea_get_drvinfo,
+	.get_link           = ethtool_op_get_link,
+	.get_ringparam      = eea_get_ringparam,
+	.set_ringparam      = eea_set_ringparam,
+	.set_channels       = eea_set_channels,
+	.get_channels       = eea_get_channels,
+	.get_strings        = eea_get_strings,
+	.get_sset_count     = eea_get_sset_count,
+	.get_ethtool_stats  = eea_get_ethtool_stats,
+	.get_link_ksettings = eea_get_link_ksettings,
+};
diff --git a/drivers/net/ethernet/alibaba/eea/eea_ethtool.h b/drivers/net/ethernet/alibaba/eea/eea_ethtool.h
new file mode 100644
index 0000000000000..08c824a116dee
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_ethtool.h
@@ -0,0 +1,48 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#ifndef __EEA_ETHTOOL_H__
+#define __EEA_ETHTOOL_H__
+
+struct eea_tx_stats {
+	struct u64_stats_sync syncp;
+	u64_stats_t descs;
+	u64_stats_t packets;
+	u64_stats_t bytes;
+	u64_stats_t drops;
+	u64_stats_t kicks;
+};
+
+struct eea_rx_ctx_stats {
+	u64 descs;
+	u64 packets;
+	u64 bytes;
+	u64 drops;
+	u64 split_hdr_bytes;
+	u64 split_hdr_packets;
+	u64 kicks;
+	u64 length_errors;
+};
+
+struct eea_rx_stats {
+	struct u64_stats_sync syncp;
+	u64_stats_t descs;
+	u64_stats_t packets;
+	u64_stats_t bytes;
+	u64_stats_t drops;
+	u64_stats_t kicks;
+	u64_stats_t split_hdr_bytes;
+	u64_stats_t split_hdr_packets;
+
+	u64_stats_t length_errors;
+};
+
+void eea_update_rx_stats(struct eea_rx_stats *rx_stats,
+			 struct eea_rx_ctx_stats *stats);
+
+extern const struct ethtool_ops eea_ethtool_ops;
+#endif
diff --git a/drivers/net/ethernet/alibaba/eea/eea_net.c b/drivers/net/ethernet/alibaba/eea/eea_net.c
new file mode 100644
index 0000000000000..63e68580de940
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_net.c
@@ -0,0 +1,887 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#include <linux/etherdevice.h>
+#include <linux/module.h>
+#include <linux/netdevice.h>
+#include <linux/rtnetlink.h>
+#include <net/netdev_queues.h>
+
+#include "eea_adminq.h"
+#include "eea_net.h"
+#include "eea_pci.h"
+#include "eea_ring.h"
+
+#define EEA_SPLIT_HDR_SIZE ALIGN(128, L1_CACHE_BYTES)
+
+static irqreturn_t eea_irq_handler(int irq, void *data)
+{
+	struct eea_irq_blk *blk = data;
+
+	napi_schedule_irqoff(&blk->napi);
+
+	return IRQ_HANDLED;
+}
+
+static void eea_free_irq_blk(struct eea_net *enet)
+{
+	struct eea_irq_blk *blk;
+	u32 num;
+	int i;
+
+	if (!enet->irq_blks)
+		return;
+
+	num = enet->edev->rx_num;
+
+	for (i = 0; i < num; i++) {
+		blk = &enet->irq_blks[i];
+
+		if (blk->ready)
+			eea_pci_free_irq(blk);
+
+		blk->ready = false;
+	}
+
+	kvfree(enet->irq_blks);
+	enet->irq_blks = NULL;
+}
+
+/* The driver will always attempt to allocate IRQ blocks based on the maximum
+ * possible queue num.
+ */
+static int eea_alloc_irq_blks(struct eea_net *enet)
+{
+	struct eea_device *edev = enet->edev;
+	struct eea_irq_blk *blk, *irq_blks;
+	int i, err, num;
+
+	num = enet->edev->rx_num;
+
+	irq_blks = kvcalloc(num, sizeof(*blk), GFP_KERNEL);
+	if (!irq_blks)
+		return -ENOMEM;
+
+	enet->irq_blks = irq_blks;
+
+	for (i = 0; i < num; i++) {
+		blk = &irq_blks[i];
+		blk->idx = i;
+
+		/* vec 0 is for error notify. */
+		blk->msix_vec = i + 1;
+
+		err = eea_pci_request_irq(edev, blk, eea_irq_handler);
+		if (err)
+			goto err_free_irq_blk;
+
+		blk->ready = true;
+	}
+
+	return 0;
+
+err_free_irq_blk:
+	eea_free_irq_blk(enet);
+	return err;
+}
+
+static int eea_update_queues(struct eea_net *enet)
+{
+	return netif_set_real_num_queues(enet->netdev, enet->cfg.tx_ring_num,
+					 enet->cfg.rx_ring_num);
+}
+
+void eea_init_ctx(struct eea_net *enet, struct eea_net_init_ctx *ctx)
+{
+	memset(ctx, 0, sizeof(*ctx));
+
+	ctx->netdev = enet->netdev;
+	ctx->edev = enet->edev;
+	ctx->cfg = enet->cfg;
+}
+
+static void eea_bind_q_and_cfg(struct eea_net *enet,
+			       struct eea_net_init_ctx *ctx)
+{
+	struct eea_irq_blk *blk;
+	struct eea_net_rx *rx;
+	struct eea_net_tx *tx;
+	int i;
+
+	/* Since 'ndo_get_stats64' is not called in softirq context, there is no
+	 * need to use 'spin_lock_bh'.
+	 */
+	spin_lock(&enet->stats_lock);
+
+	enet->cfg = ctx->cfg;
+	enet->rx = ctx->rx;
+	enet->tx = ctx->tx;
+
+	for (i = 0; i < ctx->cfg.rx_ring_num; i++) {
+		blk = &enet->irq_blks[i];
+
+		rx = ctx->rx[i];
+		tx = &ctx->tx[i];
+
+		rx->enet = enet;
+		rx->napi = &blk->napi;
+		rx->ering->msix_vec = blk->msix_vec;
+
+		tx->enet = enet;
+		tx->ering->msix_vec = blk->msix_vec;
+
+		blk->rx = rx;
+	}
+
+	spin_unlock(&enet->stats_lock);
+}
+
+static void eea_unbind_q_and_cfg(struct eea_net *enet,
+				 struct eea_net_init_ctx *ctx)
+{
+	struct eea_irq_blk *blk;
+	struct eea_net_rx *rx;
+	int i;
+
+	spin_lock(&enet->stats_lock);
+
+	ctx->cfg = enet->cfg;
+	ctx->rx = enet->rx;
+	ctx->tx = enet->tx;
+
+	enet->rx = NULL;
+	enet->tx = NULL;
+
+	for (i = 0; i < ctx->cfg.rx_ring_num; i++) {
+		blk = &enet->irq_blks[i];
+
+		rx = ctx->rx[i];
+
+		rx->napi = NULL;
+
+		blk->rx = NULL;
+	}
+
+	spin_unlock(&enet->stats_lock);
+}
+
+static void eea_free_rxtx_q_mem(struct eea_net_init_ctx *ctx)
+{
+	struct eea_net_rx *rx;
+	struct eea_net_tx *tx;
+	int i;
+
+	for (i = 0; i < ctx->cfg.rx_ring_num; i++) {
+		rx = ctx->rx[i];
+		tx = &ctx->tx[i];
+
+		eea_free_rx(rx, &ctx->cfg);
+		eea_free_tx(tx, &ctx->cfg);
+	}
+
+	kvfree(ctx->rx);
+	kvfree(ctx->tx);
+}
+
+/* alloc tx/rx: struct, ring, meta, pp, napi */
+static int eea_alloc_rxtx_q_mem(struct eea_net_init_ctx *ctx)
+{
+	struct eea_net_rx *rx;
+	struct eea_net_tx *tx;
+	int err, i;
+
+	ctx->tx = kvcalloc(ctx->cfg.tx_ring_num, sizeof(*ctx->tx), GFP_KERNEL);
+	if (!ctx->tx)
+		return -ENOMEM;
+
+	ctx->rx = kvcalloc(ctx->cfg.rx_ring_num, sizeof(*ctx->rx), GFP_KERNEL);
+	if (!ctx->rx)
+		goto err_free_tx;
+
+	ctx->cfg.rx_sq_desc_size = sizeof(struct eea_rx_desc);
+	ctx->cfg.rx_cq_desc_size = sizeof(struct eea_rx_cdesc);
+	ctx->cfg.tx_sq_desc_size = sizeof(struct eea_tx_desc);
+	ctx->cfg.tx_cq_desc_size = sizeof(struct eea_tx_cdesc);
+
+	/* ethtool may config this. */
+	if (!ctx->cfg.split_hdr)
+		ctx->cfg.rx_sq_desc_size = sizeof(struct eea_rx_desc_no_hdr);
+
+	for (i = 0; i < ctx->cfg.rx_ring_num; i++) {
+		rx = eea_alloc_rx(ctx, i);
+		if (!rx)
+			goto err_free;
+
+		ctx->rx[i] = rx;
+
+		tx = ctx->tx + i;
+		err = eea_alloc_tx(ctx, tx, i);
+		if (err)
+			goto err_free;
+	}
+
+	return 0;
+
+err_free:
+	for (i = 0; i < ctx->cfg.rx_ring_num; i++) {
+		rx = ctx->rx[i];
+		tx = ctx->tx + i;
+
+		eea_free_rx(rx, &ctx->cfg);
+		eea_free_tx(tx, &ctx->cfg);
+	}
+
+	kvfree(ctx->rx);
+
+err_free_tx:
+	kvfree(ctx->tx);
+	return -ENOMEM;
+}
+
+static int eea_hw_active_ring(struct eea_net *enet)
+{
+	return eea_adminq_create_q(enet, enet->cfg.rx_ring_num
+				   + enet->cfg.tx_ring_num, 0);
+}
+
+static int eea_hw_unactive_ring(struct eea_net *enet)
+{
+	int err;
+
+	err = eea_adminq_destroy_all_q(enet);
+	if (err)
+		netdev_warn(enet->netdev, "unactive rxtx ring failed.\n");
+
+	return err;
+}
+
+/* stop rx napi, stop tx queue. */
+static void eea_stop_rxtx(struct net_device *netdev)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	int i;
+
+	netif_tx_disable(netdev);
+
+	for (i = 0; i < enet->cfg.rx_ring_num; i++)
+		enet_rx_stop(enet->rx[i]);
+
+	netif_carrier_off(netdev);
+}
+
+static void eea_start_rxtx(struct eea_net *enet)
+{
+	int i;
+
+	for (i = 0; i < enet->cfg.rx_ring_num; i++)
+		enet_rx_start(enet->rx[i]);
+
+	netif_tx_start_all_queues(enet->netdev);
+	netif_carrier_on(enet->netdev);
+
+	enet->started = true;
+}
+
+static int eea_netdev_stop(struct net_device *netdev)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	struct eea_net_init_ctx ctx;
+
+	/* This function can be called during device anomaly recovery. To
+	 * prevent duplicate stop operations, the `started` flag is introduced
+	 * for checking.
+	 */
+
+	if (!enet->started) {
+		netdev_warn(netdev, "eea netdev stop: but dev is not started.\n");
+		return 0;
+	}
+
+	eea_init_ctx(enet, &ctx);
+
+	eea_stop_rxtx(netdev);
+	eea_hw_unactive_ring(enet);
+	eea_unbind_q_and_cfg(enet, &ctx);
+	eea_free_rxtx_q_mem(&ctx);
+
+	enet->started = false;
+
+	return 0;
+}
+
+static int eea_netdev_open(struct net_device *netdev)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	struct eea_net_init_ctx ctx;
+	int err;
+
+	if (enet->link_err) {
+		netdev_err(netdev, "netdev open err, because link error: %d\n",
+			   enet->link_err);
+		return -EBUSY;
+	}
+
+	eea_init_ctx(enet, &ctx);
+
+	err = eea_alloc_rxtx_q_mem(&ctx);
+	if (err)
+		goto err_done;
+
+	eea_bind_q_and_cfg(enet, &ctx);
+
+	err = eea_update_queues(enet);
+	if (err)
+		goto err_free_q;
+
+	err = eea_hw_active_ring(enet);
+	if (err)
+		goto err_free_q;
+
+	eea_start_rxtx(enet);
+
+	return 0;
+
+err_free_q:
+	eea_unbind_q_and_cfg(enet, &ctx);
+	eea_free_rxtx_q_mem(&ctx);
+
+err_done:
+	return err;
+}
+
+/* Statistics may be reset to zero upon device reset. This is expected behavior
+ * for now and will be addressed in the future.
+ */
+static void eea_stats(struct net_device *netdev, struct rtnl_link_stats64 *tot)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	u64 packets, bytes, drop, lerr;
+	u32 start;
+	int i;
+
+	spin_lock(&enet->stats_lock);
+
+	if (enet->rx) {
+		for (i = 0; i < enet->cfg.rx_ring_num; i++) {
+			struct eea_net_rx *rx = enet->rx[i];
+
+			do {
+				start = u64_stats_fetch_begin(&rx->stats.syncp);
+				packets = u64_stats_read(&rx->stats.packets);
+				bytes = u64_stats_read(&rx->stats.bytes);
+				drop = u64_stats_read(&rx->stats.drops);
+				lerr = u64_stats_read(&rx->stats.length_errors);
+			} while (u64_stats_fetch_retry(&rx->stats.syncp,
+						       start));
+
+			tot->rx_packets       += packets;
+			tot->rx_bytes         += bytes;
+			tot->rx_dropped       += drop;
+			tot->rx_length_errors += lerr;
+			tot->rx_errors        += lerr;
+		}
+	}
+
+	if (enet->tx) {
+		for (i = 0; i < enet->cfg.tx_ring_num; i++) {
+			struct eea_net_tx *tx = &enet->tx[i];
+
+			do {
+				start = u64_stats_fetch_begin(&tx->stats.syncp);
+				packets = u64_stats_read(&tx->stats.packets);
+				bytes = u64_stats_read(&tx->stats.bytes);
+				drop = u64_stats_read(&tx->stats.drops);
+			} while (u64_stats_fetch_retry(&tx->stats.syncp,
+						       start));
+
+			tot->tx_packets += packets;
+			tot->tx_bytes   += bytes;
+			tot->tx_dropped += drop;
+		}
+	}
+
+	spin_unlock(&enet->stats_lock);
+}
+
+/* resources: ring, buffers, irq */
+int eea_reset_hw_resources(struct eea_net *enet, struct eea_net_init_ctx *ctx)
+{
+	struct eea_net_init_ctx ctx_old = {0};
+	int err, error;
+
+	if (!netif_running(enet->netdev) || !enet->started) {
+		spin_lock(&enet->stats_lock);
+		enet->cfg = ctx->cfg;
+		spin_unlock(&enet->stats_lock);
+		return 0;
+	}
+
+	err = eea_alloc_rxtx_q_mem(ctx);
+	if (err) {
+		netdev_warn(enet->netdev,
+			    "eea reset: alloc q failed. stop reset. err %d\n",
+			    err);
+		return err;
+	}
+
+	eea_stop_rxtx(enet->netdev);
+	eea_hw_unactive_ring(enet);
+
+	eea_unbind_q_and_cfg(enet, &ctx_old);
+	eea_bind_q_and_cfg(enet, ctx);
+
+	err = eea_update_queues(enet);
+	if (err) {
+		netdev_err(enet->netdev,
+			   "eea reset: set real num queues failed. err %d\n",
+			   err);
+		goto err_bind_old;
+	}
+
+	err = eea_hw_active_ring(enet);
+	if (err) {
+		netdev_err(enet->netdev, "eea reset: active new ring. err %d\n",
+			   err);
+		eea_unbind_q_and_cfg(enet, ctx);
+		goto err_free_q;
+	}
+
+	eea_start_rxtx(enet);
+	eea_free_rxtx_q_mem(&ctx_old);
+	return 0;
+
+err_bind_old:
+	eea_unbind_q_and_cfg(enet, ctx);
+	eea_bind_q_and_cfg(enet, &ctx_old);
+	error = eea_hw_active_ring(enet);
+	if (error) {
+		netdev_err(enet->netdev, "eea reset: active old ring. err %d\n",
+			   error);
+		eea_unbind_q_and_cfg(enet, &ctx_old);
+		err = error;
+		goto err_free_q;
+	}
+
+	eea_start_rxtx(enet);
+	eea_free_rxtx_q_mem(ctx);
+	return err;
+
+err_free_q:
+
+	/* An exception occurred at the hardware level, and there's not much we
+	 * can do about it -- we can only release the resources first.
+	 */
+	eea_free_rxtx_q_mem(ctx);
+	eea_free_rxtx_q_mem(&ctx_old);
+	enet->started = false;
+	return err;
+}
+
+int eea_queues_check_and_reset(struct eea_device *edev)
+{
+	struct eea_aq_dev_status dstatus = {0};
+	struct eea_aq_queue_status *qstatus;
+	struct eea_aq_queue_status *qs;
+	struct eea_net_init_ctx ctx;
+	bool need_reset = false;
+	int i, err = 0;
+
+	rtnl_lock();
+
+	if (!netif_running(edev->enet->netdev))
+		goto err_unlock;
+
+	/* Maybe stopped by ha. */
+	if (!edev->enet->started || edev->enet->link_err)
+		goto err_unlock;
+
+	err = eea_adminq_dev_status(edev->enet, &dstatus);
+	if (err) {
+		netdev_warn(edev->enet->netdev, "query queue status failed.\n");
+		goto err_unlock;
+	}
+
+	if (le16_to_cpu(dstatus.status->link_status) == EEA_LINK_DOWN_STATUS) {
+		/* The device is broken, can not be up. */
+		eea_netdev_stop(edev->enet->netdev);
+		edev->enet->link_err = EEA_LINK_ERR_LINK_DOWN;
+		netdev_warn(edev->enet->netdev, "device link is down. stop device.\n");
+		goto err_free;
+	}
+
+	qstatus = dstatus.status->q_status;
+
+	for (i = 0; i < dstatus.num; ++i) {
+		qs = &qstatus[i];
+
+		if (le16_to_cpu(qs->status) == EEA_QUEUE_STATUS_NEED_RESET) {
+			netdev_warn(edev->enet->netdev,
+				    "queue status: queue %u needs to reset\n",
+				    le16_to_cpu(qs->qidx));
+			need_reset = true;
+		}
+	}
+
+	if (need_reset) {
+		eea_init_ctx(edev->enet, &ctx);
+		err = eea_reset_hw_resources(edev->enet, &ctx);
+	}
+
+err_free:
+	kfree(dstatus.status);
+
+err_unlock:
+	rtnl_unlock();
+	return err;
+}
+
+static int eea_update_cfg(struct eea_net *enet,
+			  struct eea_device *edev,
+			  struct eea_aq_cfg *hwcfg)
+{
+	u32 rx_max = le32_to_cpu(hwcfg->rx_depth_max);
+	u32 tx_max = le32_to_cpu(hwcfg->tx_depth_max);
+	u32 rx_def = le32_to_cpu(hwcfg->rx_depth_def);
+	u32 tx_def = le32_to_cpu(hwcfg->tx_depth_def);
+
+	/* Now, we assert that the rx ring num is equal to the tx ring num. */
+	if (edev->rx_num != edev->tx_num) {
+		dev_err(edev->dma_dev, "Inconsistent ring num: RX %u, TX %u\n",
+			edev->rx_num, edev->tx_num);
+		return -EINVAL;
+	}
+
+	if (rx_max > EEA_NET_IO_HW_RING_DEPTH_MAX ||
+	    rx_max < EEA_NET_IO_HW_RING_DEPTH_MIN ||
+	    tx_max > EEA_NET_IO_HW_RING_DEPTH_MAX ||
+	    tx_max < EEA_NET_IO_HW_RING_DEPTH_MIN) {
+		dev_err(edev->dma_dev, "Invalid HW max depth: RX %u, TX %u\n",
+			rx_max, tx_max);
+		return -EINVAL;
+	}
+
+	if (rx_def > rx_max ||
+	    tx_def > tx_max ||
+	    rx_def < EEA_NET_IO_HW_RING_DEPTH_MIN ||
+	    tx_def < EEA_NET_IO_HW_RING_DEPTH_MIN) {
+		dev_err(edev->dma_dev, "Invalid default depth: RX %u (max %u), TX %u (max %u)\n",
+			rx_def, rx_max, tx_def, tx_max);
+		return -EINVAL;
+	}
+
+	if (!is_power_of_2(rx_max) || !is_power_of_2(tx_max) ||
+	    !is_power_of_2(rx_def) || !is_power_of_2(tx_def)) {
+		dev_err(edev->dma_dev, "Ring depth must be power of 2\n");
+		return -EINVAL;
+	}
+
+	enet->cfg_hw.rx_ring_depth = rx_max;
+	enet->cfg_hw.tx_ring_depth = tx_max;
+	enet->cfg_hw.rx_ring_num = edev->rx_num;
+	enet->cfg_hw.tx_ring_num = edev->tx_num;
+	enet->cfg_hw.split_hdr = EEA_SPLIT_HDR_SIZE;
+
+	enet->cfg.rx_ring_depth = rx_def;
+	enet->cfg.tx_ring_depth = tx_def;
+	enet->cfg.rx_ring_num = edev->rx_num;
+	enet->cfg.tx_ring_num = edev->tx_num;
+
+	return 0;
+}
+
+static int eea_netdev_init_features(struct net_device *netdev,
+				    struct eea_net *enet,
+				    struct eea_device *edev)
+{
+	struct eea_aq_cfg *cfg;
+	int err;
+	u32 mtu;
+
+	cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
+	if (!cfg)
+		return -ENOMEM;
+
+	err = eea_adminq_query_cfg(enet, cfg);
+	if (err)
+		goto err_free;
+
+	mtu = le16_to_cpu(cfg->mtu);
+	if (mtu < ETH_MIN_MTU) {
+		dev_err(edev->dma_dev, "The device gave us an invalid MTU. Here we can only exit the initialization. %u < %u\n",
+			mtu, ETH_MIN_MTU);
+		err = -EINVAL;
+		goto err_free;
+	}
+
+	err = eea_update_cfg(enet, edev, cfg);
+	if (err)
+		goto err_free;
+
+	netdev->priv_flags |= IFF_UNICAST_FLT;
+	netdev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
+
+	netdev->hw_features |= NETIF_F_HW_CSUM;
+	netdev->hw_features |= NETIF_F_GRO_HW;
+	netdev->hw_features |= NETIF_F_SG;
+	netdev->hw_features |= NETIF_F_TSO;
+	netdev->hw_features |= NETIF_F_TSO_ECN;
+	netdev->hw_features |= NETIF_F_TSO6;
+	netdev->hw_features |= NETIF_F_GSO_UDP_L4;
+
+	netdev->features |= NETIF_F_HIGHDMA;
+	netdev->features |= NETIF_F_HW_CSUM;
+	netdev->features |= NETIF_F_SG;
+	netdev->features |= NETIF_F_GSO_ROBUST;
+	netdev->features |= netdev->hw_features & NETIF_F_ALL_TSO;
+	netdev->features |= NETIF_F_RXCSUM;
+	netdev->features |= NETIF_F_GRO_HW;
+
+	netdev->vlan_features = netdev->features;
+
+	if (!is_valid_ether_addr(cfg->mac)) {
+		dev_err(edev->dma_dev, "The device gave invalid mac %pM\n",
+			cfg->mac);
+		err = -EINVAL;
+		goto err_free;
+	}
+
+	eth_hw_addr_set(netdev, cfg->mac);
+
+	enet->speed = SPEED_UNKNOWN;
+	enet->duplex = DUPLEX_UNKNOWN;
+
+	netdev->min_mtu = ETH_MIN_MTU;
+
+	netdev->mtu = mtu;
+
+	/* If jumbo frames are already enabled, then the returned MTU will be a
+	 * jumbo MTU, and the driver will automatically enable jumbo frame
+	 * support by default.
+	 */
+	netdev->max_mtu = mtu;
+
+err_free:
+	kfree(cfg);
+	return err;
+}
+
+static const struct net_device_ops eea_netdev = {
+	.ndo_open           = eea_netdev_open,
+	.ndo_stop           = eea_netdev_stop,
+	.ndo_start_xmit     = eea_tx_xmit,
+	.ndo_validate_addr  = eth_validate_addr,
+	.ndo_get_stats64    = eea_stats,
+	.ndo_features_check = passthru_features_check,
+};
+
+static struct eea_net *eea_netdev_alloc(struct eea_device *edev, u32 pairs)
+{
+	struct net_device *netdev;
+	struct eea_net *enet;
+	int err;
+
+	netdev = alloc_etherdev_mq(sizeof(struct eea_net), pairs);
+	if (!netdev) {
+		dev_err(edev->dma_dev,
+			"alloc_etherdev_mq failed with pairs %d\n", pairs);
+		return NULL;
+	}
+
+	netdev->netdev_ops = &eea_netdev;
+	netdev->ethtool_ops = &eea_ethtool_ops;
+	SET_NETDEV_DEV(netdev, edev->dma_dev);
+
+	enet = netdev_priv(netdev);
+	enet->netdev = netdev;
+	enet->edev = edev;
+	edev->enet = enet;
+
+	err = eea_alloc_irq_blks(enet);
+	if (err) {
+		dev_err(edev->dma_dev,
+			"eea_alloc_irq_blks failed with pairs %d\n", pairs);
+		free_netdev(netdev);
+		return NULL;
+	}
+
+	spin_lock_init(&enet->stats_lock);
+
+	return enet;
+}
+
+static void eea_update_ts_off(struct eea_device *edev, struct eea_net *enet)
+{
+	u64 ts;
+
+	ts = eea_pci_device_ts(edev);
+
+	enet->hw_ts_offset = ktime_get_real() - ts;
+}
+
+static int eea_net_reprobe(struct eea_device *edev)
+{
+	struct eea_net *enet = edev->enet;
+	int err = 0;
+
+	enet->edev = edev;
+
+	if (!enet->adminq.ring) {
+		err = eea_create_adminq(enet, edev->rx_num + edev->tx_num);
+		if (err)
+			return err;
+	}
+
+	err = eea_alloc_irq_blks(enet);
+	if (err)
+		goto err_destroy_aq;
+
+	eea_update_ts_off(edev, enet);
+
+	rtnl_lock();
+
+	enet->link_err = 0;
+	if (edev->ha_reset_netdev_running &&
+	    netif_running(edev->enet->netdev)) {
+		err = eea_netdev_open(enet->netdev);
+		if (err) {
+			enet->link_err = EEA_LINK_ERR_HA_RESET_DEV;
+			rtnl_unlock();
+			goto err_free_irq_blks;
+		}
+	}
+
+	rtnl_unlock();
+
+	enet->wait_pci_ready = false;
+	return 0;
+
+err_free_irq_blks:
+	eea_free_irq_blk(enet);
+
+err_destroy_aq:
+	eea_destroy_adminq(enet);
+
+	return err;
+}
+
+int eea_net_probe(struct eea_device *edev)
+{
+	struct eea_net *enet;
+	int err = -ENOMEM;
+
+	/* If edev->enet is not null, then this is called from ha reset worker.
+	 * Call eea_net_reprobe() directly.
+	 */
+	if (edev->enet)
+		return eea_net_reprobe(edev);
+
+	enet = eea_netdev_alloc(edev, edev->rx_num);
+	if (!enet)
+		return -ENOMEM;
+
+	err = eea_create_adminq(enet, edev->rx_num + edev->tx_num);
+	if (err)
+		goto err_free_netdev;
+
+	eea_adminq_config_host_info(enet);
+
+	err = eea_netdev_init_features(enet->netdev, enet, edev);
+	if (err)
+		goto err_reset_dev;
+
+	eea_update_ts_off(edev, enet);
+
+	netif_carrier_off(enet->netdev);
+
+	err = register_netdev(enet->netdev);
+	if (err)
+		goto err_reset_dev;
+
+	netdev_dbg(enet->netdev, "eea probe success.\n");
+
+	return 0;
+
+err_reset_dev:
+	eea_device_reset(edev);
+	eea_destroy_adminq(enet);
+
+err_free_netdev:
+	eea_free_irq_blk(enet);
+	free_netdev(enet->netdev);
+	return err;
+}
+
+static void eea_net_ha_reset_remove(struct eea_net *enet,
+				    struct eea_device *edev)
+{
+	rtnl_lock();
+	edev->ha_reset_netdev_running = false;
+	if (netif_running(enet->netdev)) {
+		eea_netdev_stop(enet->netdev);
+		edev->ha_reset_netdev_running = true;
+	}
+
+	/* Prevent that the user set up the net device. */
+	enet->link_err = EEA_LINK_ERR_HA_RESET_DEV;
+
+	rtnl_unlock();
+
+	eea_device_reset(edev);
+	eea_destroy_adminq(enet);
+	eea_free_irq_blk(enet);
+
+	enet->wait_pci_ready = true;
+}
+
+void eea_net_remove(struct eea_device *edev, bool ha)
+{
+	struct net_device *netdev;
+	struct eea_net *enet;
+
+	enet = edev->enet;
+	netdev = enet->netdev;
+
+	if (ha) {
+		if (enet->wait_pci_ready)
+			return;
+
+		eea_net_ha_reset_remove(enet, edev);
+		return;
+	}
+
+	unregister_netdev(netdev);
+
+	if (!enet->wait_pci_ready) {
+		eea_device_reset(edev);
+		eea_destroy_adminq(enet);
+		eea_free_irq_blk(enet);
+	}
+
+	free_netdev(netdev);
+}
+
+void eea_net_shutdown(struct eea_device *edev)
+{
+	struct net_device *netdev;
+	struct eea_net *enet;
+
+	enet = edev->enet;
+	netdev = enet->netdev;
+
+	rtnl_lock();
+
+	netif_device_detach(netdev);
+	dev_close(netdev);
+
+	if (!enet->wait_pci_ready) {
+		eea_device_reset(edev);
+		eea_destroy_adminq(enet);
+		eea_free_irq_blk(enet);
+	}
+
+	rtnl_unlock();
+}
diff --git a/drivers/net/ethernet/alibaba/eea/eea_net.h b/drivers/net/ethernet/alibaba/eea/eea_net.h
new file mode 100644
index 0000000000000..848bb90bccf8d
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_net.h
@@ -0,0 +1,198 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#ifndef __EEA_NET_H__
+#define __EEA_NET_H__
+
+#include <linux/ethtool.h>
+#include <linux/netdevice.h>
+
+#include "eea_adminq.h"
+#include "eea_ethtool.h"
+#include "eea_ring.h"
+
+#define EEA_VER_MAJOR		1
+#define EEA_VER_MINOR		0
+#define EEA_VER_SUB_MINOR	0
+
+struct eea_tx_meta;
+
+struct eea_net_tx {
+	struct eea_net *enet;
+
+	struct eea_ring *ering;
+
+	struct eea_tx_meta *meta;
+	struct eea_tx_meta *free;
+
+	struct device *dma_dev;
+
+	u32 index;
+
+	char name[16];
+
+	struct eea_tx_stats stats;
+};
+
+struct eea_rx_meta {
+	struct eea_rx_meta *next;
+
+	struct page *page;
+	dma_addr_t dma;
+	u32 offset;
+	u32 sync_for_cpu;
+	u32 frags;
+
+	struct page *hdr_page;
+	void *hdr_addr;
+	dma_addr_t hdr_dma;
+
+	u32 id;
+
+	u32 truesize;
+	u32 headroom;
+	u32 tailroom;
+
+	u32 len;
+
+	bool in_use;
+};
+
+struct eea_net_rx_pkt_ctx {
+	u16 idx;
+
+	bool data_valid;
+	bool do_drop;
+
+	u32 recv_len;
+	struct sk_buff *head_skb;
+};
+
+struct eea_net_rx {
+	struct eea_net *enet;
+
+	struct eea_ring *ering;
+
+	struct eea_rx_meta *meta;
+	struct eea_rx_meta *free;
+
+	struct device *dma_dev;
+
+	u32 index;
+
+	u32 flags;
+
+	u32 headroom;
+
+	struct napi_struct *napi;
+
+	struct eea_rx_stats stats;
+
+	char name[16];
+
+	struct eea_net_rx_pkt_ctx pkt;
+
+	struct page_pool *pp;
+};
+
+struct eea_net_cfg {
+	u32 rx_ring_depth;
+	u32 tx_ring_depth;
+	u32 rx_ring_num;
+	u32 tx_ring_num;
+
+	u8 rx_sq_desc_size;
+	u8 rx_cq_desc_size;
+	u8 tx_sq_desc_size;
+	u8 tx_cq_desc_size;
+
+	u32 split_hdr;
+
+	struct hwtstamp_config ts_cfg;
+};
+
+struct eea_net_init_ctx {
+	struct eea_net_cfg cfg;
+
+	struct eea_net_tx *tx;
+	struct eea_net_rx **rx;
+
+	struct net_device *netdev;
+	struct eea_device *edev;
+};
+
+enum {
+	EEA_LINK_ERR_NONE,
+	EEA_LINK_ERR_HA_RESET_DEV,
+	EEA_LINK_ERR_LINK_DOWN,
+};
+
+struct eea_irq_blk {
+	struct napi_struct napi;
+	u16 msix_vec;
+	bool ready;
+	struct eea_net_rx *rx;
+	char irq_name[32];
+	int irq;
+	int idx;
+
+};
+
+struct eea_net {
+	struct eea_device *edev;
+	struct net_device *netdev;
+
+	struct eea_aq adminq;
+
+	struct eea_net_tx *tx;
+	struct eea_net_rx **rx;
+
+	struct eea_net_cfg cfg;
+	struct eea_net_cfg cfg_hw;
+
+	struct eea_irq_blk *irq_blks;
+
+	u32 link_err;
+
+	bool started;
+	bool wait_pci_ready;
+
+	u8 duplex;
+	u32 speed;
+
+	u64 hw_ts_offset;
+
+	/* Protect the tx and rx of struct eea_net, when eea_stats accesses the
+	 * stats from rx and tx queues.
+	 */
+	spinlock_t stats_lock;
+};
+
+int eea_net_probe(struct eea_device *edev);
+void eea_net_remove(struct eea_device *edev, bool ha);
+void eea_net_shutdown(struct eea_device *edev);
+
+int eea_reset_hw_resources(struct eea_net *enet, struct eea_net_init_ctx *ctx);
+void eea_init_ctx(struct eea_net *enet, struct eea_net_init_ctx *ctx);
+int eea_queues_check_and_reset(struct eea_device *edev);
+
+/* rx apis */
+
+void enet_rx_stop(struct eea_net_rx *rx);
+void enet_rx_start(struct eea_net_rx *rx);
+
+void eea_free_rx(struct eea_net_rx *rx, struct eea_net_cfg *cfg);
+struct eea_net_rx *eea_alloc_rx(struct eea_net_init_ctx *ctx, u32 idx);
+
+/* tx apis */
+int eea_poll_tx(struct eea_net_tx *tx, int budget);
+netdev_tx_t eea_tx_xmit(struct sk_buff *skb, struct net_device *netdev);
+
+void eea_free_tx(struct eea_net_tx *tx, struct eea_net_cfg *cfg);
+int eea_alloc_tx(struct eea_net_init_ctx *ctx, struct eea_net_tx *tx, u32 idx);
+
+#endif
diff --git a/drivers/net/ethernet/alibaba/eea/eea_pci.c b/drivers/net/ethernet/alibaba/eea/eea_pci.c
new file mode 100644
index 0000000000000..9872e360dd5d0
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_pci.c
@@ -0,0 +1,744 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#include <linux/io-64-nonatomic-lo-hi.h>
+#include <linux/iopoll.h>
+
+#include "eea_net.h"
+#include "eea_pci.h"
+
+#define EEA_PCI_DB_OFFSET 4096
+#define EEA_PCI_DB_MIN_SIZE 8
+#define EEA_PCI_DB_MAX_SIZE 512
+#define EEA_PCI_Q_MAX_NUM 1000
+
+#define EEA_PCI_CAP_RESET_DEVICE 0xFA
+#define EEA_PCI_CAP_RESET_FLAG BIT(1)
+
+struct eea_pci_cfg {
+	__le32 reserve0;
+	__le32 reserve1;
+	__le32 drv_f_idx;
+	__le32 drv_f;
+
+#define EEA_S_INIT         (BIT(0) | BIT(1))
+#define EEA_S_OK           BIT(2)
+#define EEA_S_FEATURE_DONE BIT(3)
+#define EEA_S_FAILED       BIT(7)
+	u8   device_status;
+	u8   reserved[7];
+
+	__le32 rx_num_max;
+	__le32 tx_num_max;
+	__le32 db_blk_size;
+
+	/* admin queue cfg */
+	__le16 aq_size;
+	__le16 aq_msix_vector;
+	__le32 aq_db_off;
+
+	__le32 aq_sq_addr;
+	__le32 aq_sq_addr_hi;
+	__le32 aq_cq_addr;
+	__le32 aq_cq_addr_hi;
+
+	__le32 reserved1;
+	__le64 hw_ts;
+};
+
+struct eea_pci_device {
+	struct eea_device edev;
+	struct pci_dev *pci_dev;
+
+	u32 msix_vec_n;
+	u32 db_len;
+
+	void __iomem *reg;
+	void __iomem *db_base;
+	void __iomem *db_end;
+
+	int ha_irq;
+
+	struct work_struct ha_handle_work;
+	char ha_irq_name[32];
+	int reset_pos;
+	bool ha_ready;
+
+	bool shutdown;
+};
+
+#define cfg_pointer(reg, item) \
+	((void __iomem *)((reg) + offsetof(struct eea_pci_cfg, item)))
+
+#define cfg_write8(reg, item, val) iowrite8(val, cfg_pointer(reg, item))
+#define cfg_write16(reg, item, val) iowrite16(val, cfg_pointer(reg, item))
+#define cfg_write32(reg, item, val) iowrite32(val, cfg_pointer(reg, item))
+#define cfg_write64(reg, item, val) iowrite64_lo_hi(val, cfg_pointer(reg, item))
+
+#define cfg_read8(reg, item) ioread8(cfg_pointer(reg, item))
+#define cfg_read32(reg, item) ioread32(cfg_pointer(reg, item))
+#define cfg_read64(reg, item) ioread64(cfg_pointer(reg, item))
+
+/* Due to circular references, we have to add function definitions here. */
+static int __eea_pci_probe(struct pci_dev *pci_dev,
+			   struct eea_pci_device *ep_dev, bool pci_probe);
+static void __eea_pci_remove(struct pci_dev *pci_dev, bool pci_remove);
+
+const char *eea_pci_name(struct eea_device *edev)
+{
+	return pci_name(edev->ep_dev->pci_dev);
+}
+
+int eea_pci_domain_nr(struct eea_device *edev)
+{
+	return pci_domain_nr(edev->ep_dev->pci_dev->bus);
+}
+
+u16 eea_pci_bdf(struct eea_device *edev)
+{
+	return pci_dev_id(edev->ep_dev->pci_dev);
+}
+
+static void eea_pci_io_set_status(struct eea_device *edev, u8 status)
+{
+	struct eea_pci_device *ep_dev = edev->ep_dev;
+
+	cfg_write8(ep_dev->reg, device_status, status);
+}
+
+static u8 eea_pci_io_get_status(struct eea_device *edev)
+{
+	struct eea_pci_device *ep_dev = edev->ep_dev;
+
+	return cfg_read8(ep_dev->reg, device_status);
+}
+
+static void eea_add_status(struct eea_device *dev, u32 status)
+{
+	eea_pci_io_set_status(dev, eea_pci_io_get_status(dev) | status);
+}
+
+#define EEA_RESET_TIMEOUT_US (60 * 1000 * 1000)
+
+int eea_device_reset(struct eea_device *edev)
+{
+	struct eea_pci_device *ep_dev = edev->ep_dev;
+	int err;
+	u8 val;
+
+	eea_pci_io_set_status(edev, 0);
+
+	/* We are no longer waiting for device ack during the shutdown flow. */
+	if (ep_dev->shutdown)
+		return 0;
+
+	/* A longer timeout is set here to handle edge cases, though it should
+	 * return promptly in most scenarios.
+	 *
+	 * In our case, all replies are handled by the DPU software, so there is
+	 * no race condition between the hardware processes and the register.
+	 */
+	err = read_poll_timeout(cfg_read8, val, (!val || val == 0xFF), 20,
+				EEA_RESET_TIMEOUT_US,
+				false, ep_dev->reg, device_status);
+
+	/* Surprise PCIe Removal */
+	if (val == 0xFF)
+		return -EINVAL;
+
+	return err;
+}
+
+int eea_pci_set_aq_up(struct eea_device *edev)
+{
+	struct eea_pci_device *ep_dev = edev->ep_dev;
+	u8 status = eea_pci_io_get_status(edev);
+	int err;
+	u8 val;
+
+	eea_pci_io_set_status(edev, status | EEA_S_OK);
+
+	/* A longer timeout is set here to handle edge cases, though it should
+	 * return promptly in most scenarios.
+	 *
+	 * In our case, all replies are handled by the DPU software, so there is
+	 * no race condition between the hardware processes and the register.
+	 */
+	err = read_poll_timeout(cfg_read8, val,
+				val & (EEA_S_OK | EEA_S_FAILED),
+				20, EEA_RESET_TIMEOUT_US,
+				false, ep_dev->reg, device_status);
+
+	/* Surprise PCIe Removal */
+	if (val == 0xFF)
+		return -EINVAL;
+
+	/* device fail */
+	if (val & EEA_S_FAILED)
+		return -EINVAL;
+
+	return err;
+}
+
+static int eea_negotiate(struct eea_device *edev)
+{
+	struct eea_pci_device *ep_dev;
+	u32 status;
+
+	ep_dev = edev->ep_dev;
+
+	edev->features = 0;
+
+	cfg_write32(ep_dev->reg, drv_f_idx, 0);
+	cfg_write32(ep_dev->reg, drv_f, lower_32_bits(edev->features));
+	cfg_write32(ep_dev->reg, drv_f_idx, 1);
+	cfg_write32(ep_dev->reg, drv_f, upper_32_bits(edev->features));
+
+	eea_add_status(edev, EEA_S_FEATURE_DONE);
+	status = eea_pci_io_get_status(edev);
+
+	/* Surprise PCIe Removal */
+	if (status == 0xFF)
+		return -EINVAL;
+
+	if (!(status & EEA_S_FEATURE_DONE))
+		return -ENODEV;
+
+	return 0;
+}
+
+static void eea_pci_release_resource(struct eea_pci_device *ep_dev)
+{
+	struct pci_dev *pci_dev = ep_dev->pci_dev;
+	struct eea_device *edev;
+
+	edev = &ep_dev->edev;
+
+	if (edev->status < EEA_PCI_STATUS_READY)
+		return;
+
+	if (ep_dev->reg) {
+		pci_iounmap(pci_dev, ep_dev->reg);
+		ep_dev->reg = NULL;
+	}
+
+	if (ep_dev->msix_vec_n) {
+		ep_dev->msix_vec_n = 0;
+		pci_free_irq_vectors(ep_dev->pci_dev);
+	}
+
+	pci_clear_master(pci_dev);
+	pci_release_regions(pci_dev);
+	pci_disable_device(pci_dev);
+
+	edev->status = EEA_PCI_STATUS_NONE;
+}
+
+static int eea_pci_setup(struct pci_dev *pci_dev, struct eea_pci_device *ep_dev)
+{
+	int err, n, ret, len;
+
+	ep_dev->edev.status = EEA_PCI_STATUS_ERR;
+
+	ep_dev->pci_dev = pci_dev;
+
+	err = pci_enable_device(pci_dev);
+	if (err)
+		return err;
+
+	err = pci_request_regions(pci_dev, "EEA");
+	if (err)
+		goto err_disable_dev;
+
+	if (pci_resource_len(pci_dev, 0) < EEA_PCI_DB_OFFSET) {
+		dev_err(&pci_dev->dev, "Bar 0 is too small %llu\n",
+			(u64)pci_resource_len(pci_dev, 0));
+		err = -EINVAL;
+		goto err_release_regions;
+	}
+
+	ep_dev->reg = pci_iomap(pci_dev, 0, 0);
+	if (!ep_dev->reg) {
+		dev_err(&pci_dev->dev, "Failed to map pci bar!\n");
+		err = -ENOMEM;
+		goto err_release_regions;
+	}
+
+	err = eea_device_reset(&ep_dev->edev);
+	if (err) {
+		dev_err(&pci_dev->dev, "Failed to reset device for setup!\n");
+		goto err_unmap_reg;
+	}
+
+	err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64));
+	if (err) {
+		dev_warn(&pci_dev->dev, "Failed to enable 64-bit DMA.\n");
+		goto err_unmap_reg;
+	}
+
+	pci_set_master(pci_dev);
+
+	ep_dev->edev.rx_num = cfg_read32(ep_dev->reg, rx_num_max);
+	ep_dev->edev.tx_num = cfg_read32(ep_dev->reg, tx_num_max);
+
+	if (ep_dev->edev.rx_num > EEA_PCI_Q_MAX_NUM ||
+	    ep_dev->edev.tx_num > EEA_PCI_Q_MAX_NUM) {
+		dev_err(&pci_dev->dev, "Invalid queue num %u %u\n",
+			ep_dev->edev.rx_num,
+			ep_dev->edev.tx_num);
+		err = -EINVAL;
+		goto err_clear_master;
+	}
+
+	ep_dev->edev.db_blk_size = cfg_read32(ep_dev->reg, db_blk_size);
+	if (!IS_ALIGNED(ep_dev->edev.db_blk_size, 8) ||
+	    ep_dev->edev.db_blk_size > EEA_PCI_DB_MAX_SIZE ||
+	    ep_dev->edev.db_blk_size < EEA_PCI_DB_MIN_SIZE) {
+		dev_err(&pci_dev->dev, "Invalid db size %u\n",
+			ep_dev->edev.db_blk_size);
+		err = -EINVAL;
+		goto err_clear_master;
+	}
+
+	ep_dev->db_len = ep_dev->edev.db_blk_size * (ep_dev->edev.rx_num +
+						     ep_dev->edev.tx_num + 1);
+	ep_dev->db_base = ep_dev->reg + EEA_PCI_DB_OFFSET;
+	ep_dev->db_end = ep_dev->db_base + ep_dev->db_len;
+
+	len = ep_dev->db_end - ep_dev->reg;
+
+	if (pci_resource_len(pci_dev, 0) < len) {
+		dev_err(&pci_dev->dev, "Bar 0 is too small %llu\n",
+			(u64)pci_resource_len(pci_dev, 0));
+		err = -EINVAL;
+		goto err_clear_master;
+	}
+
+	/* In our design, the number of hardware interrupts matches the maximum
+	 * number of queues. If pci_alloc_irq_vectors failed, return directly.
+	 *
+	 * 2: adminq, error handle
+	 */
+	n = ep_dev->edev.rx_num + 2;
+	ret = pci_alloc_irq_vectors(ep_dev->pci_dev, n, n, PCI_IRQ_MSIX);
+	if (ret != n) {
+		err = ret;
+		goto err_clear_master;
+	}
+
+	ep_dev->msix_vec_n = ret;
+
+	ep_dev->edev.status = EEA_PCI_STATUS_READY;
+
+	return 0;
+
+err_clear_master:
+	pci_clear_master(pci_dev);
+
+err_unmap_reg:
+	pci_iounmap(pci_dev, ep_dev->reg);
+	ep_dev->reg = NULL;
+
+err_release_regions:
+	pci_release_regions(pci_dev);
+
+err_disable_dev:
+	pci_disable_device(pci_dev);
+
+	return err;
+}
+
+void __iomem *eea_pci_db_addr(struct eea_device *edev, u32 off)
+{
+	u32 max_off;
+
+	if (!IS_ALIGNED(off, 8))
+		return NULL;
+
+	max_off = edev->ep_dev->db_len - edev->db_blk_size;
+
+	if (off > max_off)
+		return NULL;
+
+	return edev->ep_dev->db_base + off;
+}
+
+int eea_pci_active_aq(struct eea_ring *ering, int msix_vec)
+{
+	struct eea_pci_device *ep_dev = ering->edev->ep_dev;
+
+	cfg_write16(ep_dev->reg, aq_size, ering->num);
+	cfg_write16(ep_dev->reg, aq_msix_vector, msix_vec);
+
+	cfg_write64(ep_dev->reg, aq_sq_addr, ering->sq.dma_addr);
+	cfg_write64(ep_dev->reg, aq_cq_addr, ering->cq.dma_addr);
+
+	ering->db = eea_pci_db_addr(ering->edev,
+				    cfg_read32(ep_dev->reg, aq_db_off));
+
+	if (!ering->db)
+		return -EIO;
+
+	return 0;
+}
+
+void eea_pci_free_irq(struct eea_irq_blk *blk)
+{
+	irq_update_affinity_hint(blk->irq, NULL);
+	free_irq(blk->irq, blk);
+}
+
+int eea_pci_request_irq(struct eea_device *edev, struct eea_irq_blk *blk,
+			irqreturn_t (*callback)(int irq, void *data))
+{
+	struct eea_pci_device *ep_dev = edev->ep_dev;
+	int irq;
+
+	snprintf(blk->irq_name, sizeof(blk->irq_name), "eea-q%d@%s", blk->idx,
+		 pci_name(ep_dev->pci_dev));
+
+	irq = pci_irq_vector(ep_dev->pci_dev, blk->msix_vec);
+
+	blk->irq = irq;
+
+	return request_irq(irq, callback, IRQF_NO_AUTOEN, blk->irq_name, blk);
+}
+
+static void eea_ha_handle_reset(struct eea_pci_device *ep_dev)
+{
+	struct eea_device *edev;
+	struct pci_dev *pci_dev;
+	u16 reset;
+	int err;
+
+	if (!ep_dev->reset_pos) {
+		eea_queues_check_and_reset(&ep_dev->edev);
+		return;
+	}
+
+	edev = &ep_dev->edev;
+
+	pci_read_config_word(ep_dev->pci_dev, ep_dev->reset_pos, &reset);
+
+	/* Clear bits using 0xFFFF and ignore all previous messages. */
+	pci_write_config_word(ep_dev->pci_dev, ep_dev->reset_pos, 0xFFFF);
+
+	if (reset & EEA_PCI_CAP_RESET_FLAG) {
+		dev_warn(&ep_dev->pci_dev->dev, "recv device reset request.\n");
+
+		pci_dev = ep_dev->pci_dev;
+
+		/* The pci remove callback may hold this lock. If the
+		 * pci remove callback is called, then we can ignore the
+		 * ha interrupt.
+		 */
+		if (mutex_trylock(&edev->ha_lock)) {
+			if (edev->status != EEA_PCI_STATUS_DONE) {
+				dev_err(&ep_dev->pci_dev->dev, "ha: reset device: pci status is %d. skip it.\n",
+					edev->status);
+
+				mutex_unlock(&edev->ha_lock);
+				return;
+			}
+
+			__eea_pci_remove(pci_dev, false);
+			err = __eea_pci_probe(pci_dev, ep_dev, false);
+			if (err)
+				/* Currently, for some reason, PCI
+				 * initialization or network device re-probing
+				 * has failed. Waiting for the PCI subsystem to
+				 * call the remove callback to release the
+				 * remaining resources.
+				 */
+				dev_err(&ep_dev->pci_dev->dev,
+					"ha: re-setup failed.\n");
+
+			mutex_unlock(&edev->ha_lock);
+		} else {
+			/* Device removal is in progress, so return directly. */
+			dev_warn(&ep_dev->pci_dev->dev,
+				 "ha device reset: trylock failed.\n");
+		}
+		return;
+	}
+
+	eea_queues_check_and_reset(&ep_dev->edev);
+}
+
+/* ha handle code */
+static void eea_ha_handle_work(struct work_struct *work)
+{
+	struct eea_pci_device *ep_dev;
+
+	ep_dev = container_of(work, struct eea_pci_device, ha_handle_work);
+
+	/* Ha interrupt is triggered, so there maybe some error, we may need to
+	 * reset the device or reset some queues.
+	 */
+	dev_warn(&ep_dev->pci_dev->dev, "recv ha interrupt.\n");
+
+	eea_ha_handle_reset(ep_dev);
+}
+
+static irqreturn_t eea_pci_ha_handle(int irq, void *data)
+{
+	struct eea_device *edev = data;
+
+	schedule_work(&edev->ep_dev->ha_handle_work);
+
+	return IRQ_HANDLED;
+}
+
+static void eea_pci_free_ha_irq(struct eea_device *edev)
+{
+	struct eea_pci_device *ep_dev = edev->ep_dev;
+	int irq;
+
+	if (ep_dev->ha_ready) {
+		irq = pci_irq_vector(ep_dev->pci_dev, 0);
+		free_irq(irq, edev);
+		ep_dev->ha_ready = false;
+	}
+}
+
+static int eea_pci_ha_init(struct eea_device *edev, struct pci_dev *pci_dev,
+			   bool pci_probe)
+{
+	int pos, cfg_type_off, cfg_drv_off, cfg_dev_off;
+	struct eea_pci_device *ep_dev = edev->ep_dev;
+	int irq, err;
+	u8 type;
+
+	snprintf(ep_dev->ha_irq_name, sizeof(ep_dev->ha_irq_name), "eea-ha@%s",
+		 pci_name(ep_dev->pci_dev));
+
+	irq = pci_irq_vector(ep_dev->pci_dev, 0);
+
+	if (pci_probe)
+		INIT_WORK(&ep_dev->ha_handle_work, eea_ha_handle_work);
+
+	/* This irq is not only work for ha, so request it always. */
+	err = request_irq(irq, eea_pci_ha_handle, IRQF_NO_AUTOEN,
+			  ep_dev->ha_irq_name, edev);
+	if (err)
+		return err;
+
+	ep_dev->ha_irq = irq;
+
+	ep_dev->ha_ready = true;
+	ep_dev->reset_pos = 0;
+
+	cfg_type_off = offsetof(struct eea_pci_cap, cfg_type);
+	cfg_drv_off = offsetof(struct eea_pci_reset_reg, driver);
+	cfg_dev_off = offsetof(struct eea_pci_reset_reg, device);
+
+	for (pos = pci_find_capability(pci_dev, PCI_CAP_ID_VNDR);
+	     pos > 0;
+	     pos = pci_find_next_capability(pci_dev, pos, PCI_CAP_ID_VNDR)) {
+		pci_read_config_byte(pci_dev, pos + cfg_type_off, &type);
+
+		if (type == EEA_PCI_CAP_RESET_DEVICE) {
+			/* notify device, driver support this feature. */
+			pci_write_config_word(pci_dev, pos + cfg_drv_off,
+					      EEA_PCI_CAP_RESET_FLAG);
+			pci_write_config_word(pci_dev, pos + cfg_dev_off,
+					      0xFFFF);
+
+			edev->ep_dev->reset_pos = pos + cfg_dev_off;
+			return 0;
+		}
+	}
+
+	/* irq just for event notify */
+	dev_warn(&edev->ep_dev->pci_dev->dev, "Not Found reset cap.\n");
+	return 0;
+}
+
+u64 eea_pci_device_ts(struct eea_device *edev)
+{
+	struct eea_pci_device *ep_dev = edev->ep_dev;
+
+	return cfg_read64(ep_dev->reg, hw_ts);
+}
+
+static int eea_init_device(struct eea_device *edev)
+{
+	int err;
+
+	err = eea_device_reset(edev);
+	if (err)
+		return err;
+
+	eea_pci_io_set_status(edev, EEA_S_INIT);
+
+	err = eea_negotiate(edev);
+	if (err)
+		goto err;
+
+	err = eea_net_probe(edev);
+	if (err)
+		goto err;
+
+	return 0;
+err:
+	eea_add_status(edev, EEA_S_FAILED);
+	return err;
+}
+
+static int __eea_pci_probe(struct pci_dev *pci_dev,
+			   struct eea_pci_device *ep_dev,
+			   bool pci_probe)
+{
+	struct eea_device *edev;
+	int err;
+
+	pci_set_drvdata(pci_dev, ep_dev);
+
+	edev = &ep_dev->edev;
+
+	err = eea_pci_setup(pci_dev, ep_dev);
+	if (err)
+		return err;
+
+	err = eea_init_device(&ep_dev->edev);
+	if (err)
+		goto err_pci_rel;
+
+	err = eea_pci_ha_init(edev, pci_dev, pci_probe);
+	if (err)
+		goto err_net_rm;
+
+	edev->status = EEA_PCI_STATUS_DONE;
+
+	enable_irq(ep_dev->ha_irq);
+
+	return 0;
+
+err_net_rm:
+	eea_net_remove(edev, !pci_probe);
+
+err_pci_rel:
+	eea_pci_release_resource(ep_dev);
+	return err;
+}
+
+static void __eea_pci_remove(struct pci_dev *pci_dev, bool pci_remove)
+{
+	struct eea_pci_device *ep_dev = pci_get_drvdata(pci_dev);
+	struct device *dev = get_device(&ep_dev->pci_dev->dev);
+	struct eea_device *edev = &ep_dev->edev;
+
+	eea_pci_free_ha_irq(edev);
+
+	if (pci_remove)
+		flush_work(&ep_dev->ha_handle_work);
+
+	eea_net_remove(edev, !pci_remove);
+
+	eea_pci_release_resource(ep_dev);
+
+	put_device(dev);
+}
+
+static int eea_pci_probe(struct pci_dev *pci_dev,
+			 const struct pci_device_id *id)
+{
+	struct eea_pci_device *ep_dev;
+	struct eea_device *edev;
+	int err;
+
+	ep_dev = kzalloc(sizeof(*ep_dev), GFP_KERNEL);
+	if (!ep_dev)
+		return -ENOMEM;
+
+	edev = &ep_dev->edev;
+
+	edev->ep_dev = ep_dev;
+	edev->dma_dev = &pci_dev->dev;
+
+	ep_dev->pci_dev = pci_dev;
+
+	mutex_init(&edev->ha_lock);
+
+	err = __eea_pci_probe(pci_dev, ep_dev, true);
+	if (err) {
+		mutex_destroy(&edev->ha_lock);
+		pci_set_drvdata(pci_dev, NULL);
+		kfree(ep_dev);
+	}
+
+	return err;
+}
+
+static void eea_pci_remove(struct pci_dev *pci_dev)
+{
+	struct eea_pci_device *ep_dev = pci_get_drvdata(pci_dev);
+	struct eea_device *edev;
+
+	edev = &ep_dev->edev;
+
+	mutex_lock(&edev->ha_lock);
+	__eea_pci_remove(pci_dev, true);
+	mutex_unlock(&edev->ha_lock);
+
+	pci_set_drvdata(pci_dev, NULL);
+
+	mutex_destroy(&edev->ha_lock);
+	kfree(ep_dev);
+}
+
+static void eea_pci_shutdown(struct pci_dev *pci_dev)
+{
+	struct eea_pci_device *ep_dev = pci_get_drvdata(pci_dev);
+	struct eea_device *edev;
+
+	edev = &ep_dev->edev;
+
+	ep_dev->shutdown = true;
+
+	mutex_lock(&edev->ha_lock);
+	eea_pci_free_ha_irq(edev);
+	flush_work(&ep_dev->ha_handle_work);
+	mutex_unlock(&edev->ha_lock);
+
+	eea_net_shutdown(edev);
+
+	pci_clear_master(pci_dev);
+}
+
+static const struct pci_device_id eea_pci_id_table[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x500B) },
+	{ 0 }
+};
+
+MODULE_DEVICE_TABLE(pci, eea_pci_id_table);
+
+static struct pci_driver eea_pci_driver = {
+	.name            = "alibaba_eea",
+	.id_table        = eea_pci_id_table,
+	.probe           = eea_pci_probe,
+	.remove          = eea_pci_remove,
+	.shutdown        = eea_pci_shutdown,
+	.sriov_configure = pci_sriov_configure_simple,
+};
+
+static __init int eea_pci_init(void)
+{
+	return pci_register_driver(&eea_pci_driver);
+}
+
+static __exit void eea_pci_exit(void)
+{
+	pci_unregister_driver(&eea_pci_driver);
+}
+
+module_init(eea_pci_init);
+module_exit(eea_pci_exit);
+
+MODULE_DESCRIPTION("Driver for Alibaba Elastic Ethernet Adapter");
+MODULE_AUTHOR("Xuan Zhuo <xuanzhuo@linux.alibaba.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/net/ethernet/alibaba/eea/eea_pci.h b/drivers/net/ethernet/alibaba/eea/eea_pci.h
new file mode 100644
index 0000000000000..6f6ba7d5514dd
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_pci.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#ifndef __EEA_PCI_H__
+#define __EEA_PCI_H__
+
+#include <linux/pci.h>
+
+#include "eea_net.h"
+#include "eea_ring.h"
+
+enum eea_pci_status {
+	EEA_PCI_STATUS_NONE,
+	EEA_PCI_STATUS_ERR,
+	EEA_PCI_STATUS_READY,
+	EEA_PCI_STATUS_DONE,
+};
+
+struct eea_irq_blk;
+
+struct eea_pci_cap {
+	__u8 cap_vndr;
+	__u8 cap_next;
+	__u8 cap_len;
+	__u8 cfg_type;
+};
+
+struct eea_pci_reset_reg {
+	struct eea_pci_cap cap;
+	__le16 driver;
+	__le16 device;
+};
+
+struct eea_pci_device;
+
+struct eea_device {
+	struct eea_pci_device *ep_dev;
+	struct device         *dma_dev;
+	struct eea_net        *enet;
+
+	u64 features;
+
+	enum eea_pci_status status;
+	bool ha_reset_netdev_running;
+
+	/* ha lock for the race between ha work and pci remove */
+	struct mutex ha_lock;
+
+	u32 rx_num;
+	u32 tx_num;
+	u32 db_blk_size;
+};
+
+const char *eea_pci_name(struct eea_device *edev);
+int eea_pci_domain_nr(struct eea_device *edev);
+u16 eea_pci_bdf(struct eea_device *edev);
+
+int eea_device_reset(struct eea_device *dev);
+int eea_pci_set_aq_up(struct eea_device *dev);
+int eea_pci_active_aq(struct eea_ring *ering, int msix_vec);
+
+int eea_pci_request_irq(struct eea_device *edev, struct eea_irq_blk *blk,
+			irqreturn_t (*callback)(int irq, void *data));
+void eea_pci_free_irq(struct eea_irq_blk *blk);
+
+u64 eea_pci_device_ts(struct eea_device *edev);
+
+void __iomem *eea_pci_db_addr(struct eea_device *edev, u32 off);
+#endif
diff --git a/drivers/net/ethernet/alibaba/eea/eea_ring.c b/drivers/net/ethernet/alibaba/eea/eea_ring.c
new file mode 100644
index 0000000000000..99dcabd094b89
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_ring.c
@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#include "eea_pci.h"
+#include "eea_ring.h"
+
+void eea_ering_irq_active(struct eea_ring *ering, struct eea_ring *tx_ering)
+{
+	u64 value = 0, rx_idx, tx_idx;
+
+	tx_idx = (u64)tx_ering->cq.hw_idx;
+	rx_idx = (u64)ering->cq.hw_idx;
+
+	value |= EEA_IRQ_UNMASK << EEA_DB_FLAGS_OFF;
+	value |= tx_idx << EEA_DB_TX_CQ_HEAD_OFF;
+	value |= rx_idx << EEA_DB_RX_CQ_HEAD_OFF;
+
+	writeq(value, ering->db);
+}
+
+void *eea_ering_cq_get_desc(const struct eea_ring *ering)
+{
+	u8 phase;
+	u8 *desc;
+
+	desc = ering->cq.desc + (ering->cq.head << ering->cq.desc_size_shift);
+
+	phase = READ_ONCE(*(u8 *)(desc + ering->cq.desc_size - 1));
+
+	if ((phase & EEA_RING_DESC_F_CQ_PHASE) == ering->cq.phase) {
+		dma_rmb();
+		return desc;
+	}
+
+	return NULL;
+}
+
+/* sq api */
+void *eea_ering_sq_alloc_desc(struct eea_ring *ering, u16 id, bool is_last,
+			      u16 flags)
+{
+	struct eea_ring_sq *sq = &ering->sq;
+	struct eea_common_desc *desc;
+
+	if (!sq->shadow_num) {
+		sq->shadow_idx = sq->head;
+		sq->shadow_id = cpu_to_le16(id);
+	}
+
+	if (!is_last)
+		flags |= EEA_RING_DESC_F_MORE;
+
+	desc = sq->desc + (sq->shadow_idx << sq->desc_size_shift);
+
+	desc->flags = cpu_to_le16(flags);
+	desc->id = sq->shadow_id;
+
+	if (unlikely(++sq->shadow_idx >= ering->num))
+		sq->shadow_idx = 0;
+
+	++sq->shadow_num;
+
+	return desc;
+}
+
+/* This is an allocation API for admin Q. For each call to admin Q, only one
+ * desc will be allocated.
+ */
+void *eea_ering_aq_alloc_desc(struct eea_ring *ering)
+{
+	struct eea_ring_sq *sq = &ering->sq;
+	struct eea_common_desc *desc;
+
+	if (!sq->shadow_num)
+		sq->shadow_idx = sq->head;
+
+	desc = sq->desc + (sq->shadow_idx << sq->desc_size_shift);
+
+	if (unlikely(++sq->shadow_idx >= ering->num))
+		sq->shadow_idx = 0;
+
+	++sq->shadow_num;
+
+	return desc;
+}
+
+void eea_ering_sq_commit_desc(struct eea_ring *ering)
+{
+	struct eea_ring_sq *sq = &ering->sq;
+	int num;
+
+	num = sq->shadow_num;
+
+	ering->num_free -= num;
+
+	sq->head       = sq->shadow_idx;
+	sq->hw_idx     += num;
+	sq->shadow_num = 0;
+}
+
+void eea_ering_sq_cancel(struct eea_ring *ering)
+{
+	ering->sq.shadow_num = 0;
+}
+
+/* cq api */
+void eea_ering_cq_ack_desc(struct eea_ring *ering, u32 num)
+{
+	struct eea_ring_cq *cq = &ering->cq;
+
+	cq->head += num;
+	cq->hw_idx += num;
+
+	if (unlikely(cq->head >= ering->num)) {
+		cq->head -= ering->num;
+		cq->phase ^= EEA_RING_DESC_F_CQ_PHASE;
+	}
+
+	ering->num_free += num;
+}
+
+/* notify */
+void eea_ering_kick(struct eea_ring *ering)
+{
+	u64 value = 0, idx;
+
+	idx = (u64)ering->sq.hw_idx;
+
+	value |= EEA_IDX_PRESENT << EEA_DB_FLAGS_OFF;
+	value |= idx << EEA_DB_IDX_OFF;
+
+	writeq(value, ering->db);
+}
+
+/* ering alloc/free */
+static void ering_free_queue(struct eea_device *edev, size_t size,
+			     void *queue, dma_addr_t dma_handle)
+{
+	dma_free_coherent(edev->dma_dev, size, queue, dma_handle);
+}
+
+static void *ering_alloc_queue(struct eea_device *edev, size_t size,
+			       dma_addr_t *dma_handle)
+{
+	gfp_t flags = GFP_KERNEL | __GFP_NOWARN;
+
+	return dma_alloc_coherent(edev->dma_dev, size, dma_handle, flags);
+}
+
+static int ering_alloc_queues(struct eea_ring *ering, struct eea_device *edev,
+			      size_t num, u8 sq_desc_size, u8 cq_desc_size)
+{
+	dma_addr_t addr;
+	size_t size;
+	void *ring;
+
+	size = num * sq_desc_size;
+
+	ring = ering_alloc_queue(edev, size, &addr);
+	if (!ring)
+		return -ENOMEM;
+
+	ering->sq.desc     = ring;
+	ering->sq.dma_addr = addr;
+	ering->sq.dma_size = size;
+	ering->sq.desc_size = sq_desc_size;
+	ering->sq.desc_size_shift = fls(sq_desc_size) - 1;
+
+	size = num * cq_desc_size;
+
+	ring = ering_alloc_queue(edev, size, &addr);
+	if (!ring)
+		goto err_free_sq;
+
+	ering->cq.desc     = ring;
+	ering->cq.dma_addr = addr;
+	ering->cq.dma_size = size;
+	ering->cq.desc_size = cq_desc_size;
+	ering->cq.desc_size_shift = fls(cq_desc_size) - 1;
+
+	ering->num = num;
+
+	return 0;
+
+err_free_sq:
+	ering_free_queue(ering->edev, ering->sq.dma_size,
+			 ering->sq.desc, ering->sq.dma_addr);
+	return -ENOMEM;
+}
+
+static void ering_init(struct eea_ring *ering)
+{
+	ering->cq.phase = EEA_RING_DESC_F_CQ_PHASE;
+	ering->num_free = ering->num;
+}
+
+struct eea_ring *eea_ering_alloc(u32 index, u32 num, struct eea_device *edev,
+				 u8 sq_desc_size, u8 cq_desc_size,
+				 const char *name)
+{
+	struct eea_ring *ering;
+
+	if (num > EEA_NET_IO_HW_RING_DEPTH_MAX ||
+	    num < EEA_NET_IO_RING_DEPTH_MIN)
+		return NULL;
+
+	if (!is_power_of_2(num))
+		return NULL;
+
+	if (!sq_desc_size || !is_power_of_2(sq_desc_size))
+		return NULL;
+
+	if (!cq_desc_size || !is_power_of_2(cq_desc_size))
+		return NULL;
+
+	ering = kzalloc(sizeof(*ering), GFP_KERNEL);
+	if (!ering)
+		return NULL;
+
+	ering->edev = edev;
+	ering->name = name;
+	ering->index = index;
+
+	if (ering_alloc_queues(ering, edev, num, sq_desc_size, cq_desc_size))
+		goto err_free;
+
+	ering_init(ering);
+
+	return ering;
+
+err_free:
+	kfree(ering);
+	return NULL;
+}
+
+void eea_ering_free(struct eea_ring *ering)
+{
+	ering_free_queue(ering->edev, ering->cq.dma_size,
+			 ering->cq.desc, ering->cq.dma_addr);
+
+	ering_free_queue(ering->edev, ering->sq.dma_size,
+			 ering->sq.desc, ering->sq.dma_addr);
+
+	kfree(ering);
+}
diff --git a/drivers/net/ethernet/alibaba/eea/eea_ring.h b/drivers/net/ethernet/alibaba/eea/eea_ring.h
new file mode 100644
index 0000000000000..a7ce465943a58
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_ring.h
@@ -0,0 +1,99 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#ifndef __EEA_RING_H__
+#define __EEA_RING_H__
+
+#include <linux/dma-mapping.h>
+#include "eea_desc.h"
+
+#define EEA_RING_DESC_F_MORE		BIT(0)
+#define EEA_RING_DESC_F_CQ_PHASE	BIT(7)
+
+/* These two values define the bounds for the queue depth returned by the
+ * hardware.
+ */
+#define EEA_NET_IO_HW_RING_DEPTH_MAX (32 * 1024)
+#define EEA_NET_IO_HW_RING_DEPTH_MIN 128
+
+/* This value constrains the minimum queue depth that the driver configures for
+ * the hardware, which typically applies to user-provided settings. Naturally,
+ * the configured depth must also not exceed the maximum capacity supported by
+ * the hardware.
+ */
+#define EEA_NET_IO_RING_DEPTH_MIN 64
+
+struct eea_common_desc {
+	__le16 flags;
+	__le16 id;
+};
+
+struct eea_device;
+
+struct eea_ring_sq {
+	void *desc;
+
+	u16 head;
+	u16 hw_idx;
+
+	u16 shadow_idx;
+	__le16 shadow_id;
+	u16 shadow_num;
+
+	u8 desc_size;
+	u8 desc_size_shift;
+
+	dma_addr_t dma_addr;
+	u32 dma_size;
+};
+
+struct eea_ring_cq {
+	void *desc;
+
+	u16 head;
+	u16 hw_idx;
+
+	u8 phase;
+	u8 desc_size_shift;
+	u8 desc_size;
+
+	dma_addr_t dma_addr;
+	u32 dma_size;
+};
+
+struct eea_ring {
+	const char *name;
+	struct eea_device *edev;
+	u32 index;
+	void __iomem *db;
+	u16 msix_vec;
+
+	u32 num;
+
+	u32 num_free;
+
+	struct eea_ring_sq sq;
+	struct eea_ring_cq cq;
+};
+
+struct eea_ring *eea_ering_alloc(u32 index, u32 num, struct eea_device *edev,
+				 u8 sq_desc_size, u8 cq_desc_size,
+				 const char *name);
+void eea_ering_free(struct eea_ring *ering);
+void eea_ering_kick(struct eea_ring *ering);
+
+void *eea_ering_sq_alloc_desc(struct eea_ring *ering, u16 id,
+			      bool is_last, u16 flags);
+void *eea_ering_aq_alloc_desc(struct eea_ring *ering);
+void eea_ering_sq_commit_desc(struct eea_ring *ering);
+void eea_ering_sq_cancel(struct eea_ring *ering);
+
+void eea_ering_cq_ack_desc(struct eea_ring *ering, u32 num);
+
+void eea_ering_irq_active(struct eea_ring *ering, struct eea_ring *tx_ering);
+void *eea_ering_cq_get_desc(const struct eea_ring *ering);
+#endif
diff --git a/drivers/net/ethernet/alibaba/eea/eea_rx.c b/drivers/net/ethernet/alibaba/eea/eea_rx.c
new file mode 100644
index 0000000000000..a3f0d2a79ad8b
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_rx.c
@@ -0,0 +1,814 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#include <net/netdev_rx_queue.h>
+#include <net/page_pool/helpers.h>
+
+#include "eea_adminq.h"
+#include "eea_net.h"
+#include "eea_pci.h"
+#include "eea_ring.h"
+
+#define EEA_ENABLE_F_NAPI        BIT(0)
+
+#define EEA_PAGE_FRAGS_NUM 1024
+
+#define EEA_RX_BUF_ALIGN 128
+
+#define EEA_RX_BUF_MAX_LEN (10 * 1024)
+
+struct eea_rx_ctx {
+	u32 len;
+	u32 hdr_len;
+
+	u16 flags;
+	bool more;
+
+	struct eea_rx_meta *meta;
+
+	struct eea_rx_ctx_stats stats;
+};
+
+static struct eea_rx_meta *eea_rx_meta_get(struct eea_net_rx *rx)
+{
+	struct eea_rx_meta *meta;
+
+	if (!rx->free)
+		return NULL;
+
+	meta = rx->free;
+	rx->free = meta->next;
+
+	return meta;
+}
+
+static void eea_rx_meta_put(struct eea_net_rx *rx, struct eea_rx_meta *meta)
+{
+	meta->next = rx->free;
+	rx->free = meta;
+}
+
+static void eea_free_rx_buffer(struct eea_net_rx *rx, struct eea_rx_meta *meta,
+			       bool allow_direct)
+{
+	u32 drain_count;
+
+	drain_count = EEA_PAGE_FRAGS_NUM - meta->frags;
+
+	if (page_pool_unref_page(meta->page, drain_count) == 0)
+		page_pool_put_unrefed_page(rx->pp, meta->page, -1,
+					   allow_direct);
+
+	meta->page = NULL;
+}
+
+static void eea_rx_meta_dma_sync_for_device(struct eea_net_rx *rx,
+					    struct eea_rx_meta *meta)
+{
+	u32 len;
+
+	if (meta->sync_for_cpu <= meta->offset + rx->headroom)
+		return;
+
+	len = meta->sync_for_cpu - meta->offset - rx->headroom;
+
+	dma_sync_single_for_device(rx->enet->edev->dma_dev,
+				   meta->dma + meta->offset + rx->headroom,
+				   len, DMA_FROM_DEVICE);
+	meta->sync_for_cpu = 0;
+}
+
+static void meta_align_offset(struct eea_net_rx *rx, struct eea_rx_meta *meta)
+{
+	int h, b;
+
+	h = rx->headroom;
+	b = meta->offset + h;
+
+	/* For better performance, we align the buffer address to
+	 * EEA_RX_BUF_ALIGN, as required by the device design.
+	 */
+	b = ALIGN(b, EEA_RX_BUF_ALIGN);
+
+	meta->offset = b - h;
+}
+
+static int eea_alloc_rx_buffer(struct eea_net_rx *rx, struct eea_rx_meta *meta)
+{
+	struct page *page;
+
+	if (meta->page) {
+		eea_rx_meta_dma_sync_for_device(rx, meta);
+		return 0;
+	}
+
+	page = page_pool_dev_alloc_pages(rx->pp);
+	if (!page)
+		return -ENOMEM;
+
+	page_pool_fragment_page(page, EEA_PAGE_FRAGS_NUM);
+
+	meta->page = page;
+	meta->dma = page_pool_get_dma_addr(page);
+	meta->offset = 0;
+	meta->frags = 0;
+	meta->sync_for_cpu = 0;
+
+	meta_align_offset(rx, meta);
+
+	return 0;
+}
+
+static u32 eea_consume_rx_buffer(struct eea_net_rx *rx,
+				 struct eea_rx_meta *meta,
+				 u32 consumed)
+{
+	u32 offset;
+	int min;
+
+	offset = meta->offset;
+
+	meta->offset += consumed;
+	++meta->frags;
+
+	min = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+	min += rx->headroom;
+	min += SKB_DATA_ALIGN(ETH_DATA_LEN);
+
+	meta_align_offset(rx, meta);
+
+	if (min + meta->offset > PAGE_SIZE) {
+		eea_free_rx_buffer(rx, meta, true);
+		return PAGE_SIZE - offset;
+	}
+
+	return meta->offset - offset;
+}
+
+static void eea_free_rx_hdr(struct eea_net_rx *rx, struct eea_net_cfg *cfg)
+{
+	struct eea_rx_meta *meta;
+	int i;
+
+	for (i = 0; i < cfg->rx_ring_depth; ++i) {
+		meta = &rx->meta[i];
+		meta->hdr_addr = NULL;
+
+		if (!meta->hdr_page)
+			continue;
+
+		dma_unmap_page(rx->dma_dev, meta->hdr_dma, PAGE_SIZE,
+			       DMA_FROM_DEVICE);
+		put_page(meta->hdr_page);
+
+		meta->hdr_page = NULL;
+	}
+}
+
+static int eea_alloc_rx_hdr(struct eea_net_init_ctx *ctx, struct eea_net_rx *rx)
+{
+	struct page *hdr_page = NULL;
+	struct eea_rx_meta *meta;
+	u32 offset = 0, hdrsize;
+	struct device *dmadev;
+	dma_addr_t dma;
+	int i;
+
+	dmadev = ctx->edev->dma_dev;
+	hdrsize = ctx->cfg.split_hdr;
+
+	for (i = 0; i < ctx->cfg.rx_ring_depth; ++i) {
+		meta = &rx->meta[i];
+		meta->hdr_page = NULL;
+
+		if (!hdr_page || offset + hdrsize > PAGE_SIZE) {
+			hdr_page = alloc_page(GFP_KERNEL);
+			if (!hdr_page)
+				goto err;
+
+			dma = dma_map_page(dmadev, hdr_page, 0, PAGE_SIZE,
+					   DMA_FROM_DEVICE);
+
+			if (unlikely(dma_mapping_error(dmadev, dma))) {
+				put_page(hdr_page);
+				goto err;
+			}
+
+			offset = 0;
+			meta->hdr_page = hdr_page;
+		}
+
+		meta->hdr_dma = dma + offset;
+		meta->hdr_addr = page_address(hdr_page) + offset;
+		offset += hdrsize;
+	}
+
+	return 0;
+
+err:
+	eea_free_rx_hdr(rx, &ctx->cfg);
+	return -ENOMEM;
+}
+
+static void eea_rx_meta_dma_sync_for_cpu(struct eea_net_rx *rx,
+					 struct eea_rx_meta *meta, u32 len)
+{
+	dma_sync_single_for_cpu(rx->enet->edev->dma_dev,
+				meta->dma + meta->offset + meta->headroom,
+				len, DMA_FROM_DEVICE);
+	meta->sync_for_cpu = meta->offset + meta->headroom + len;
+}
+
+static int eea_harden_check_overflow(struct eea_rx_ctx *ctx,
+				     struct eea_net *enet)
+{
+	u32 max_len;
+
+	max_len = ctx->meta->truesize - ctx->meta->headroom -
+		ctx->meta->tailroom;
+
+	if (unlikely(ctx->len > max_len)) {
+		pr_debug("%s: rx error: len %u exceeds truesize %u\n",
+			 enet->netdev->name, ctx->len, max_len);
+		++ctx->stats.length_errors;
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int eea_harden_check_size(struct eea_rx_ctx *ctx, struct eea_net *enet)
+{
+	int err;
+
+	err = eea_harden_check_overflow(ctx, enet);
+	if (err)
+		return err;
+
+	if (ctx->hdr_len) {
+		if (unlikely(ctx->hdr_len < ETH_HLEN)) {
+			pr_debug("%s: short hdr %u\n", enet->netdev->name,
+				 ctx->hdr_len);
+			++ctx->stats.length_errors;
+			return -EINVAL;
+		}
+
+		if (unlikely(ctx->hdr_len > enet->cfg.split_hdr)) {
+			pr_debug("%s: rx error: hdr len %u exceeds hdr buffer size %u\n",
+				 enet->netdev->name, ctx->hdr_len,
+				 enet->cfg.split_hdr);
+			++ctx->stats.length_errors;
+			return -EINVAL;
+		}
+
+		return 0;
+	}
+
+	if (unlikely(ctx->len < ETH_HLEN)) {
+		pr_debug("%s: short packet %u\n", enet->netdev->name, ctx->len);
+		++ctx->stats.length_errors;
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static struct sk_buff *eea_build_skb(void *buf, u32 buflen, u32 headroom,
+				     u32 len)
+{
+	struct sk_buff *skb;
+
+	skb = build_skb(buf, buflen);
+	if (unlikely(!skb))
+		return NULL;
+
+	skb_reserve(skb, headroom);
+	skb_put(skb, len);
+
+	return skb;
+}
+
+static struct sk_buff *eea_rx_build_split_hdr_skb(struct eea_net_rx *rx,
+						  struct eea_rx_ctx *ctx)
+{
+	struct eea_rx_meta *meta = ctx->meta;
+	u32 truesize, offset;
+	struct sk_buff *skb;
+	struct page *page;
+
+	dma_sync_single_for_cpu(rx->enet->edev->dma_dev, meta->hdr_dma,
+				ctx->hdr_len, DMA_FROM_DEVICE);
+
+	skb = napi_alloc_skb(rx->napi, ctx->hdr_len);
+	if (unlikely(!skb))
+		return NULL;
+
+	skb_put_data(skb, ctx->meta->hdr_addr, ctx->hdr_len);
+
+	if (ctx->len) {
+		page = meta->page;
+		offset = meta->offset + meta->headroom;
+
+		truesize = eea_consume_rx_buffer(rx, meta,
+						 meta->headroom + ctx->len);
+
+		skb_add_rx_frag(skb, 0, page, offset, ctx->len, truesize);
+	}
+
+	skb_mark_for_recycle(skb);
+
+	return skb;
+}
+
+static struct sk_buff *eea_rx_build_skb(struct eea_net_rx *rx,
+					struct eea_rx_ctx *ctx)
+{
+	struct eea_rx_meta *meta = ctx->meta;
+	u32 shinfo_size, bufsize, truesize;
+	struct sk_buff *skb;
+	struct page *page;
+	void *buf;
+
+	page = meta->page;
+
+	shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+
+	buf = page_address(page) + meta->offset;
+	bufsize = meta->headroom + SKB_DATA_ALIGN(ctx->len) + shinfo_size;
+
+	skb = eea_build_skb(buf, bufsize, meta->headroom, ctx->len);
+	if (unlikely(!skb))
+		return NULL;
+
+	truesize = eea_consume_rx_buffer(rx, meta, bufsize);
+	skb_mark_for_recycle(skb);
+
+	skb->truesize += truesize - bufsize;
+
+	return skb;
+}
+
+static void process_remain_buf(struct eea_net_rx *rx, struct eea_rx_ctx *ctx)
+{
+	struct eea_net *enet = rx->enet;
+	struct sk_buff *head_skb;
+	u32 offset, truesize, nr_frags;
+	struct page *page;
+
+	if (eea_harden_check_overflow(ctx, enet))
+		goto err;
+
+	head_skb = rx->pkt.head_skb;
+
+	nr_frags = skb_shinfo(head_skb)->nr_frags;
+	if (unlikely(nr_frags >= MAX_SKB_FRAGS))
+		goto err;
+
+	offset = ctx->meta->offset + ctx->meta->headroom;
+	page = ctx->meta->page;
+	truesize = eea_consume_rx_buffer(rx, ctx->meta,
+					 ctx->meta->headroom + ctx->len);
+
+	skb_add_rx_frag(head_skb, nr_frags, page, offset, ctx->len, truesize);
+
+	return;
+
+err:
+	dev_kfree_skb(rx->pkt.head_skb);
+	++ctx->stats.drops;
+	rx->pkt.do_drop = true;
+	rx->pkt.head_skb = NULL;
+}
+
+static void process_first_buf(struct eea_net_rx *rx, struct eea_rx_ctx *ctx)
+{
+	struct eea_net *enet = rx->enet;
+	struct sk_buff *skb = NULL;
+
+	if (eea_harden_check_size(ctx, enet))
+		goto err;
+
+	rx->pkt.data_valid = ctx->flags & EEA_DESC_F_DATA_VALID;
+
+	if (ctx->hdr_len)
+		skb = eea_rx_build_split_hdr_skb(rx, ctx);
+	else
+		skb = eea_rx_build_skb(rx, ctx);
+
+	if (unlikely(!skb))
+		goto err;
+
+	rx->pkt.head_skb = skb;
+
+	return;
+
+err:
+	++ctx->stats.drops;
+	rx->pkt.do_drop = true;
+}
+
+static void eea_submit_skb(struct eea_net_rx *rx, struct sk_buff *skb,
+			   struct eea_rx_cdesc *desc)
+{
+	struct eea_net *enet = rx->enet;
+
+	if (rx->pkt.data_valid)
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+
+	if (enet->cfg.ts_cfg.rx_filter == HWTSTAMP_FILTER_ALL)
+		skb_hwtstamps(skb)->hwtstamp = EEA_DESC_TS(desc) +
+			enet->hw_ts_offset;
+
+	skb_record_rx_queue(skb, rx->index);
+	skb->protocol = eth_type_trans(skb, enet->netdev);
+
+	napi_gro_receive(rx->napi, skb);
+}
+
+static int eea_rx_desc_to_ctx(struct eea_net_rx *rx,
+			      struct eea_rx_ctx *ctx,
+			      struct eea_rx_cdesc *desc)
+{
+	u16 id;
+
+	ctx->meta = NULL;
+
+	id = le16_to_cpu(desc->id);
+	if (unlikely(id >= rx->ering->num)) {
+		if (net_ratelimit())
+			netdev_err(rx->enet->netdev, "rx invalid id %d\n", id);
+		return -EINVAL;
+	}
+
+	ctx->meta = &rx->meta[id];
+	if (!ctx->meta->in_use) {
+		if (net_ratelimit())
+			netdev_err(rx->enet->netdev, "rx invalid id %d\n", id);
+		ctx->meta = NULL;
+		return -EINVAL;
+	}
+
+	ctx->meta->in_use = false;
+
+	ctx->len = le16_to_cpu(desc->len);
+	if (unlikely(ctx->len > ctx->meta->len)) {
+		if (net_ratelimit())
+			netdev_err(rx->enet->netdev, "rx invalid len(%d) id:%d\n",
+				   ctx->len, id);
+		return -EINVAL;
+	}
+
+	ctx->flags = le16_to_cpu(desc->flags);
+
+	ctx->hdr_len = 0;
+	if (ctx->flags & EEA_DESC_F_SPLIT_HDR) {
+		ctx->hdr_len = le16_to_cpu(desc->len_ex) &
+			EEA_RX_CDESC_HDR_LEN_MASK;
+		ctx->stats.split_hdr_bytes += ctx->hdr_len;
+		++ctx->stats.split_hdr_packets;
+	}
+
+	ctx->more = ctx->flags & EEA_RING_DESC_F_MORE;
+
+	return 0;
+}
+
+static int eea_cleanrx(struct eea_net_rx *rx, int budget,
+		       struct eea_rx_ctx *ctx)
+{
+	struct eea_rx_cdesc *desc;
+	struct eea_rx_meta *meta;
+	int recv, err;
+
+	for (recv = 0; recv < budget; ) {
+		desc = eea_ering_cq_get_desc(rx->ering);
+		if (!desc)
+			break;
+
+		err = eea_rx_desc_to_ctx(rx, ctx, desc);
+		if (unlikely(err)) {
+			if (ctx->meta)
+				eea_rx_meta_put(rx, ctx->meta);
+
+			if (rx->pkt.head_skb) {
+				dev_kfree_skb(rx->pkt.head_skb);
+				++ctx->stats.drops;
+			}
+
+			/* A hardware error occurred; we are attempting to
+			 * mitigate the impact. Subsequent packets may be
+			 * corrupted.
+			 */
+			ctx->more = false;
+			goto ack;
+		}
+
+		meta = ctx->meta;
+
+		if (unlikely(rx->pkt.do_drop))
+			goto skip;
+
+		eea_rx_meta_dma_sync_for_cpu(rx, meta, ctx->len);
+
+		rx->pkt.recv_len += ctx->len;
+		rx->pkt.recv_len += ctx->hdr_len;
+
+		if (!rx->pkt.idx)
+			process_first_buf(rx, ctx);
+		else
+			process_remain_buf(rx, ctx);
+
+		++rx->pkt.idx;
+
+		if (!ctx->more && rx->pkt.head_skb) {
+			eea_submit_skb(rx, rx->pkt.head_skb, desc);
+			ctx->stats.bytes += rx->pkt.recv_len;
+			++ctx->stats.packets;
+		}
+
+skip:
+		eea_rx_meta_put(rx, meta);
+ack:
+		eea_ering_cq_ack_desc(rx->ering, 1);
+		++ctx->stats.descs;
+
+		if (!ctx->more) {
+			memset(&rx->pkt, 0, sizeof(rx->pkt));
+			++recv;
+		}
+	}
+
+	return recv;
+}
+
+static void eea_rx_dma_sync_hdr(struct eea_net_rx *rx, dma_addr_t addr)
+{
+	dma_sync_single_for_device(rx->dma_dev, addr,
+				   rx->enet->cfg.split_hdr,
+				   DMA_FROM_DEVICE);
+}
+
+/* Only be called from napi. */
+static void eea_rx_post(struct eea_net_rx *rx, struct eea_rx_ctx *ctx)
+{
+	u32 tailroom, headroom, room, len;
+	struct eea_rx_meta *meta;
+	struct eea_rx_desc *desc;
+	int err = 0, num = 0;
+	dma_addr_t addr;
+
+	tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+	headroom = rx->headroom;
+	room = headroom + tailroom;
+
+	while (true) {
+		meta = eea_rx_meta_get(rx);
+		if (!meta)
+			break;
+
+		err = eea_alloc_rx_buffer(rx, meta);
+		if (err) {
+			eea_rx_meta_put(rx, meta);
+			break;
+		}
+
+		len = min_t(u32, PAGE_SIZE - meta->offset - room,
+			    EEA_RX_BUF_MAX_LEN);
+
+		len = ALIGN_DOWN(len, SMP_CACHE_BYTES);
+
+		addr = meta->dma + meta->offset + headroom;
+
+		desc = eea_ering_sq_alloc_desc(rx->ering, meta->id, true, 0);
+		desc->addr = cpu_to_le64(addr);
+		desc->len = cpu_to_le16(len);
+
+		if (meta->hdr_addr) {
+			eea_rx_dma_sync_hdr(rx, meta->hdr_dma);
+			desc->hdr_addr = cpu_to_le64(meta->hdr_dma);
+		}
+
+		eea_ering_sq_commit_desc(rx->ering);
+
+		meta->truesize = len + room;
+		meta->headroom = headroom;
+		meta->tailroom = tailroom;
+		meta->len = len;
+		meta->in_use = true;
+		++num;
+	}
+
+	if (num) {
+		eea_ering_kick(rx->ering);
+		++ctx->stats.kicks;
+	}
+}
+
+static int eea_poll(struct napi_struct *napi, int budget)
+{
+	struct eea_irq_blk *blk = container_of(napi, struct eea_irq_blk, napi);
+	struct eea_net_rx *rx = blk->rx;
+	struct eea_net_tx *tx = &rx->enet->tx[rx->index];
+	struct eea_rx_ctx ctx = {};
+	bool busy = false;
+	u32 received;
+
+	busy |= eea_poll_tx(tx, budget) >= budget;
+
+	received = eea_cleanrx(rx, budget, &ctx);
+
+	if (rx->ering->num_free > budget) {
+		/* Due to the hardware design, there is no notification when
+		 * buffers are exhausted. Therefore, we should proactively
+		 * pre-fill the buffers to avoid starvation.
+		 */
+		eea_rx_post(rx, &ctx);
+
+		if (rx->ering->num - rx->ering->num_free < budget)
+			busy = true;
+	}
+
+	eea_update_rx_stats(&rx->stats, &ctx.stats);
+
+	busy |= received >= budget;
+
+	if (busy)
+		return budget;
+
+	if (napi_complete_done(napi, received))
+		eea_ering_irq_active(rx->ering, tx->ering);
+
+	return received;
+}
+
+static void eea_free_rx_buffers(struct eea_net_rx *rx, struct eea_net_cfg *cfg)
+{
+	struct eea_rx_meta *meta;
+	u32 i;
+
+	if (rx->pkt.head_skb) {
+		dev_kfree_skb(rx->pkt.head_skb);
+		rx->pkt.head_skb = NULL;
+	}
+
+	for (i = 0; i < cfg->rx_ring_depth; ++i) {
+		meta = &rx->meta[i];
+		if (!meta->page)
+			continue;
+
+		eea_free_rx_buffer(rx, meta, false);
+	}
+}
+
+static struct page_pool *eea_create_pp(struct eea_net_init_ctx *ctx, u32 idx)
+{
+	struct page_pool_params pp_params = {0};
+
+	pp_params.order     = 0;
+	pp_params.flags     = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
+	pp_params.pool_size = ctx->cfg.rx_ring_depth;
+	pp_params.nid       = dev_to_node(ctx->edev->dma_dev);
+	pp_params.dev       = ctx->edev->dma_dev;
+	pp_params.netdev    = ctx->netdev;
+	pp_params.dma_dir   = DMA_FROM_DEVICE;
+	pp_params.max_len   = PAGE_SIZE;
+	pp_params.queue_idx = idx;
+
+	return page_pool_create(&pp_params);
+}
+
+static void eea_destroy_page_pool(struct eea_net_rx *rx)
+{
+	if (rx->pp)
+		page_pool_destroy(rx->pp);
+}
+
+void enet_rx_stop(struct eea_net_rx *rx)
+{
+	if (rx->flags & EEA_ENABLE_F_NAPI) {
+		rx->flags &= ~EEA_ENABLE_F_NAPI;
+
+		disable_irq(rx->enet->irq_blks[rx->index].irq);
+		napi_disable(rx->napi);
+
+		page_pool_disable_direct_recycling(rx->pp);
+		netif_napi_del(rx->napi);
+	}
+}
+
+void enet_rx_start(struct eea_net_rx *rx)
+{
+	netif_napi_add(rx->enet->netdev, rx->napi, eea_poll);
+
+	page_pool_enable_direct_recycling(rx->pp, rx->napi);
+
+	napi_enable(rx->napi);
+
+	rx->flags |= EEA_ENABLE_F_NAPI;
+
+	local_bh_disable();
+	napi_schedule(rx->napi);
+	local_bh_enable();
+
+	enable_irq(rx->enet->irq_blks[rx->index].irq);
+}
+
+/* Maybe called before eea_bind_q_and_cfg. So the cfg must be passed. */
+void eea_free_rx(struct eea_net_rx *rx, struct eea_net_cfg *cfg)
+{
+	if (!rx)
+		return;
+
+	if (rx->ering) {
+		eea_ering_free(rx->ering);
+		rx->ering = NULL;
+	}
+
+	if (rx->meta) {
+		eea_free_rx_buffers(rx, cfg);
+		eea_free_rx_hdr(rx, cfg);
+		kvfree(rx->meta);
+		rx->meta = NULL;
+	}
+
+	if (rx->pp) {
+		eea_destroy_page_pool(rx);
+		rx->pp = NULL;
+	}
+
+	kfree(rx);
+}
+
+static void eea_rx_meta_init(struct eea_net_rx *rx, u32 num)
+{
+	struct eea_rx_meta *meta;
+	int i;
+
+	rx->free = NULL;
+
+	for (i = 0; i < num; ++i) {
+		meta = &rx->meta[i];
+		meta->id = i;
+		meta->next = rx->free;
+		rx->free = meta;
+	}
+}
+
+struct eea_net_rx *eea_alloc_rx(struct eea_net_init_ctx *ctx, u32 idx)
+{
+	struct eea_ring *ering;
+	struct eea_net_rx *rx;
+	int err;
+
+	rx = kzalloc(sizeof(*rx), GFP_KERNEL);
+	if (!rx)
+		return rx;
+
+	rx->index = idx;
+	snprintf(rx->name, sizeof(rx->name), "rx.%u", idx);
+
+	u64_stats_init(&rx->stats.syncp);
+
+	/* ering */
+	ering = eea_ering_alloc(idx * 2, ctx->cfg.rx_ring_depth, ctx->edev,
+				ctx->cfg.rx_sq_desc_size,
+				ctx->cfg.rx_cq_desc_size,
+				rx->name);
+	if (!ering)
+		goto err_free_rx;
+
+	rx->ering = ering;
+
+	rx->dma_dev = ctx->edev->dma_dev;
+
+	/* meta */
+	rx->meta = kvcalloc(ctx->cfg.rx_ring_depth,
+			    sizeof(*rx->meta), GFP_KERNEL);
+	if (!rx->meta)
+		goto err_free_rx;
+
+	eea_rx_meta_init(rx, ctx->cfg.rx_ring_depth);
+
+	if (ctx->cfg.split_hdr) {
+		err = eea_alloc_rx_hdr(ctx, rx);
+		if (err)
+			goto err_free_rx;
+	}
+
+	rx->pp = eea_create_pp(ctx, idx);
+	if (IS_ERR(rx->pp)) {
+		err = PTR_ERR(rx->pp);
+		rx->pp = NULL;
+		goto err_free_rx;
+	}
+
+	return rx;
+
+err_free_rx:
+	eea_free_rx(rx, &ctx->cfg);
+	return NULL;
+}
diff --git a/drivers/net/ethernet/alibaba/eea/eea_tx.c b/drivers/net/ethernet/alibaba/eea/eea_tx.c
new file mode 100644
index 0000000000000..85fb0e9ca5ba6
--- /dev/null
+++ b/drivers/net/ethernet/alibaba/eea/eea_tx.c
@@ -0,0 +1,500 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Driver for Alibaba Elastic Ethernet Adapter.
+ *
+ * Copyright (C) 2025 Alibaba Inc.
+ */
+
+#include <net/netdev_queues.h>
+
+#include "eea_net.h"
+#include "eea_pci.h"
+#include "eea_ring.h"
+
+struct eea_sq_free_stats {
+	u64 packets;
+	u64 bytes;
+};
+
+struct eea_tx_meta {
+	struct eea_tx_meta *next;
+
+	u32 id;
+
+	union {
+		struct sk_buff *skb;
+		void *data;
+	};
+
+	u32 num;
+
+	dma_addr_t dma_addr;
+	struct eea_tx_desc *desc;
+	u32 dma_len;
+	bool unmap;
+	bool unmap_single;
+};
+
+static struct eea_tx_meta *eea_tx_meta_get(struct eea_net_tx *tx)
+{
+	struct eea_tx_meta *meta;
+
+	if (!tx->free)
+		return NULL;
+
+	meta = tx->free;
+	tx->free = meta->next;
+
+	return meta;
+}
+
+static void eea_tx_meta_put_and_unmap(struct eea_net_tx *tx,
+				      struct eea_tx_meta *meta)
+{
+	struct eea_tx_meta *head;
+
+	head = meta;
+
+	while (true) {
+		if (meta->unmap) {
+			if (meta->unmap_single)
+				dma_unmap_single(tx->dma_dev, meta->dma_addr,
+						 meta->dma_len, DMA_TO_DEVICE);
+			else
+				dma_unmap_page(tx->dma_dev, meta->dma_addr,
+					       meta->dma_len, DMA_TO_DEVICE);
+		}
+
+		if (meta->next) {
+			meta = meta->next;
+			continue;
+		}
+
+		break;
+	}
+
+	meta->next = tx->free;
+	tx->free = head;
+}
+
+static void eea_meta_free_xmit(struct eea_net_tx *tx,
+			       struct eea_tx_meta *meta,
+			       int budget,
+			       struct eea_tx_cdesc *desc,
+			       struct eea_sq_free_stats *stats)
+{
+	struct sk_buff *skb = meta->skb;
+
+	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && desc)) {
+		struct skb_shared_hwtstamps ts = {};
+
+		ts.hwtstamp = EEA_DESC_TS(desc) + tx->enet->hw_ts_offset;
+		skb_tstamp_tx(skb, &ts);
+	}
+
+	++stats->packets;
+	stats->bytes += meta->skb->len;
+	napi_consume_skb(meta->skb, budget);
+
+	meta->data = NULL;
+}
+
+static int eea_clean_tx(struct eea_net_tx *tx, int budget)
+{
+	struct eea_sq_free_stats stats = {0};
+	struct eea_tx_cdesc *desc;
+	struct eea_tx_meta *meta;
+	int desc_n;
+	u16 id;
+
+	while (stats.packets < budget) {
+		desc = eea_ering_cq_get_desc(tx->ering);
+		if (!desc)
+			break;
+
+		id = le16_to_cpu(desc->id);
+		if (unlikely(id >= tx->ering->num)) {
+			if (net_ratelimit())
+				netdev_err(tx->enet->netdev, "tx invalid id %d\n",
+					   id);
+			eea_ering_cq_ack_desc(tx->ering, 1);
+			continue;
+		}
+
+		meta = &tx->meta[id];
+
+		if (meta->data) {
+			eea_tx_meta_put_and_unmap(tx, meta);
+			eea_meta_free_xmit(tx, meta, budget, desc, &stats);
+			desc_n = meta->num;
+		} else {
+			if (net_ratelimit())
+				netdev_err(tx->enet->netdev,
+					   "tx meta->data is null. id %d num: %d\n",
+					   meta->id, meta->num);
+			desc_n = 1;
+		}
+
+		eea_ering_cq_ack_desc(tx->ering, desc_n);
+	}
+
+	if (stats.packets) {
+		u64_stats_update_begin(&tx->stats.syncp);
+		u64_stats_add(&tx->stats.bytes, stats.bytes);
+		u64_stats_add(&tx->stats.packets, stats.packets);
+		u64_stats_update_end(&tx->stats.syncp);
+	}
+
+	return stats.packets;
+}
+
+int eea_poll_tx(struct eea_net_tx *tx, int budget)
+{
+	struct eea_net *enet = tx->enet;
+	u32 index = tx - enet->tx;
+	struct netdev_queue *txq;
+	int num;
+
+	txq = netdev_get_tx_queue(enet->netdev, index);
+
+	__netif_tx_lock(txq, smp_processor_id());
+
+	num = eea_clean_tx(tx, budget);
+
+	if (netif_tx_queue_stopped(txq) &&
+	    tx->ering->num_free >= MAX_SKB_FRAGS + 2)
+		netif_tx_wake_queue(txq);
+
+	__netif_tx_unlock(txq);
+
+	return num;
+}
+
+static int eea_fill_desc_from_skb(const struct sk_buff *skb,
+				  struct eea_tx_desc *desc)
+{
+	if (skb_is_gso(skb)) {
+		struct skb_shared_info *sinfo = skb_shinfo(skb);
+
+		desc->gso_size = cpu_to_le16(sinfo->gso_size);
+		if (sinfo->gso_type & SKB_GSO_TCPV4)
+			desc->gso_type = EEA_TX_GSO_TCPV4;
+
+		else if (sinfo->gso_type & SKB_GSO_TCPV6)
+			desc->gso_type = EEA_TX_GSO_TCPV6;
+
+		else if (sinfo->gso_type & SKB_GSO_UDP_L4)
+			desc->gso_type = EEA_TX_GSO_UDP_L4;
+
+		else
+			return -EINVAL;
+
+		if (sinfo->gso_type & SKB_GSO_TCP_ECN)
+			desc->gso_type |= EEA_TX_GSO_ECN;
+	} else {
+		desc->gso_type = EEA_TX_GSO_NONE;
+	}
+
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
+		desc->csum_start = cpu_to_le16(skb_checksum_start_offset(skb));
+		desc->csum_offset = cpu_to_le16(skb->csum_offset);
+	}
+
+	return 0;
+}
+
+static struct eea_tx_meta *__eea_tx_desc_fill(struct eea_net_tx *tx,
+					      struct eea_tx_meta *head_meta,
+					      dma_addr_t addr, u32 data_len,
+					      u32 dma_len, bool last,
+					      void *data, u16 flags,
+					      bool unmap)
+{
+	struct eea_tx_meta *meta;
+	struct eea_tx_desc *desc;
+
+	meta = eea_tx_meta_get(tx);
+
+	desc = eea_ering_sq_alloc_desc(tx->ering, meta->id, last, flags);
+	desc->addr = cpu_to_le64(addr);
+	desc->len = cpu_to_le16(data_len);
+
+	meta->next     = NULL;
+	meta->dma_len  = dma_len;
+	meta->dma_addr = addr;
+	meta->data     = data;
+	meta->num      = 1;
+	meta->desc     = desc;
+	meta->unmap    = unmap;
+	meta->unmap_single = false;
+
+	if (head_meta) {
+		meta->next = head_meta->next;
+		head_meta->next = meta;
+		++head_meta->num;
+	}
+
+	return meta;
+}
+
+static struct eea_tx_meta *eea_tx_desc_fill(struct eea_net_tx *tx,
+					    struct eea_tx_meta *head_meta,
+					    dma_addr_t addr, u32 length,
+					    bool is_last, void *data, u16 flags)
+{
+	struct eea_tx_meta *meta;
+	u16 len, last;
+
+	WARN_ON_ONCE(length >= 2 * USHRT_MAX);
+
+	/* Since eea does not support BIG TCP, the maximum GSO size is capped at
+	 * 64KB. Consequently, a single skb buffer (head or fragment) will not
+	 * require more than two descriptors
+	 */
+	if (length > USHRT_MAX) {
+		len = USHRT_MAX;
+		last = false;
+	} else {
+		len = length;
+		last = is_last;
+	}
+
+	meta = __eea_tx_desc_fill(tx, head_meta, addr, len, length,
+				  last, data, flags, true);
+
+	if (length > USHRT_MAX) {
+		if (!head_meta)
+			head_meta = meta;
+
+		addr += USHRT_MAX;
+		len = length - USHRT_MAX;
+
+		__eea_tx_desc_fill(tx, head_meta, addr, len, 0, is_last,
+				   NULL, 0, false);
+	}
+
+	return meta;
+}
+
+static int eea_tx_add_skb_frag(struct eea_net_tx *tx,
+			       struct eea_tx_meta *head_meta,
+			       const skb_frag_t *frag, bool is_last)
+{
+	u32 len = skb_frag_size(frag);
+	dma_addr_t addr;
+
+	addr = skb_frag_dma_map(tx->dma_dev, frag, 0, len, DMA_TO_DEVICE);
+	if (unlikely(dma_mapping_error(tx->dma_dev, addr)))
+		return -ENOMEM;
+
+	eea_tx_desc_fill(tx, head_meta, addr, len, is_last, NULL, 0);
+
+	return 0;
+}
+
+static int eea_tx_post_skb(struct eea_net_tx *tx, struct sk_buff *skb)
+{
+	const struct skb_shared_info *shinfo = skb_shinfo(skb);
+	u32 hlen = skb_headlen(skb);
+	struct eea_tx_meta *meta;
+	const skb_frag_t *frag;
+	dma_addr_t addr;
+	u32 len = hlen;
+	int i, err;
+	u16 flags;
+	bool last;
+
+	if (len) {
+		addr = dma_map_single(tx->dma_dev, skb->data, len,
+				      DMA_TO_DEVICE);
+		if (unlikely(dma_mapping_error(tx->dma_dev, addr)))
+			return -ENOMEM;
+
+		last = !shinfo->nr_frags;
+		i = 0;
+	} else {
+		/* The net stack will never submit an skb with an skb->len of
+		 * 0. If the head len is 0, the number of frags must be greater
+		 * than 0.
+		 */
+		frag = &shinfo->frags[0];
+		len = skb_frag_size(frag);
+
+		addr = skb_frag_dma_map(tx->dma_dev, frag, 0, len,
+					DMA_TO_DEVICE);
+		if (unlikely(dma_mapping_error(tx->dma_dev, addr)))
+			return -ENOMEM;
+
+		last = shinfo->nr_frags == 1;
+		i = 1;
+	}
+
+	flags = skb->ip_summed == CHECKSUM_PARTIAL ? EEA_DESC_F_DO_CSUM : 0;
+
+	meta = eea_tx_desc_fill(tx, NULL, addr, len, last, skb, flags);
+	meta->unmap_single = !!hlen;
+
+	err = eea_fill_desc_from_skb(skb, meta->desc);
+	if (err)
+		goto err_cancel;
+
+	for (; i < shinfo->nr_frags; i++) {
+		frag = &shinfo->frags[i];
+		bool is_last = i == (shinfo->nr_frags - 1);
+
+		err = eea_tx_add_skb_frag(tx, meta, frag, is_last);
+		if (err)
+			goto err_cancel;
+	}
+
+	eea_ering_sq_commit_desc(tx->ering);
+
+	u64_stats_update_begin(&tx->stats.syncp);
+	u64_stats_add(&tx->stats.descs, meta->num);
+	u64_stats_update_end(&tx->stats.syncp);
+
+	return 0;
+
+err_cancel:
+	eea_ering_sq_cancel(tx->ering);
+	eea_tx_meta_put_and_unmap(tx, meta);
+	meta->data = NULL;
+	return err;
+}
+
+static void eea_tx_kick(struct eea_net_tx *tx)
+{
+	eea_ering_kick(tx->ering);
+
+	u64_stats_update_begin(&tx->stats.syncp);
+	u64_stats_inc(&tx->stats.kicks);
+	u64_stats_update_end(&tx->stats.syncp);
+}
+
+static int eea_tx_check_free_num(struct eea_net_tx *tx,
+				 struct netdev_queue *txq)
+{
+	int n;
+
+	/* MAX_SKB_FRAGS + 1: Covers the skb linear head and all paged fragments
+	 * 1: Extra slot for a head or fragment that exceeds 64KB.
+	 */
+	n = MAX_SKB_FRAGS + 2;
+	return netif_txq_maybe_stop(txq, tx->ering->num_free, n, n);
+}
+
+netdev_tx_t eea_tx_xmit(struct sk_buff *skb, struct net_device *netdev)
+{
+	struct eea_net *enet = netdev_priv(netdev);
+	int qnum = skb_get_queue_mapping(skb);
+	struct eea_net_tx *tx = &enet->tx[qnum];
+	struct netdev_queue *txq;
+	int err, enable;
+
+	txq = netdev_get_tx_queue(netdev, qnum);
+
+	enable = eea_tx_check_free_num(tx, txq);
+	if (!enable)
+		return NETDEV_TX_BUSY;
+
+	err = eea_tx_post_skb(tx, skb);
+	if (unlikely(err)) {
+		u64_stats_update_begin(&tx->stats.syncp);
+		u64_stats_inc(&tx->stats.drops);
+		u64_stats_update_end(&tx->stats.syncp);
+
+		dev_kfree_skb_any(skb);
+	} else {
+		if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))
+			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+		skb_tx_timestamp(skb);
+	}
+
+	/* NETDEV_TX_BUSY is expensive. So stop advancing the TX queue. */
+	eea_tx_check_free_num(tx, txq);
+
+	if (!netdev_xmit_more() || netif_xmit_stopped(txq))
+		eea_tx_kick(tx);
+
+	return NETDEV_TX_OK;
+}
+
+static void eea_free_meta(struct eea_net_tx *tx, struct eea_net_cfg *cfg)
+{
+	struct eea_sq_free_stats stats = {0};
+	struct eea_tx_meta *meta;
+	int i;
+
+	while ((meta = eea_tx_meta_get(tx)))
+		meta->skb = NULL;
+
+	for (i = 0; i < cfg->tx_ring_depth; i++) {
+		meta = &tx->meta[i];
+
+		if (!meta->skb)
+			continue;
+
+		eea_tx_meta_put_and_unmap(tx, meta);
+
+		eea_meta_free_xmit(tx, meta, 0, NULL, &stats);
+	}
+
+	kvfree(tx->meta);
+	tx->meta = NULL;
+}
+
+/* Maybe called before eea_bind_q_and_cfg. So the cfg must be passed. */
+void eea_free_tx(struct eea_net_tx *tx, struct eea_net_cfg *cfg)
+{
+	if (!tx)
+		return;
+
+	if (tx->ering) {
+		eea_ering_free(tx->ering);
+		tx->ering = NULL;
+	}
+
+	if (tx->meta)
+		eea_free_meta(tx, cfg);
+}
+
+int eea_alloc_tx(struct eea_net_init_ctx *ctx, struct eea_net_tx *tx, u32 idx)
+{
+	struct eea_tx_meta *meta;
+	struct eea_ring *ering;
+	u32 i;
+
+	u64_stats_init(&tx->stats.syncp);
+
+	snprintf(tx->name, sizeof(tx->name), "tx.%u", idx);
+
+	ering = eea_ering_alloc(idx * 2 + 1, ctx->cfg.tx_ring_depth, ctx->edev,
+				ctx->cfg.tx_sq_desc_size,
+				ctx->cfg.tx_cq_desc_size,
+				tx->name);
+	if (!ering)
+		goto err_free_tx;
+
+	tx->ering = ering;
+	tx->index = idx;
+	tx->dma_dev = ctx->edev->dma_dev;
+
+	/* meta */
+	tx->meta = kvcalloc(ctx->cfg.tx_ring_depth,
+			    sizeof(*tx->meta), GFP_KERNEL);
+	if (!tx->meta)
+		goto err_free_tx;
+
+	for (i = 0; i < ctx->cfg.tx_ring_depth; ++i) {
+		meta = &tx->meta[i];
+		meta->id = i;
+		meta->next = tx->free;
+		tx->free = meta;
+	}
+
+	return 0;
+
+err_free_tx:
+	eea_free_tx(tx, &ctx->cfg);
+	return -ENOMEM;
+}
diff --git a/drivers/net/ethernet/amd/pds_core/debugfs.c b/drivers/net/ethernet/amd/pds_core/debugfs.c
index 04c5e3abd8d70..810a0cd9bcac8 100644
--- a/drivers/net/ethernet/amd/pds_core/debugfs.c
+++ b/drivers/net/ethernet/amd/pds_core/debugfs.c
@@ -64,9 +64,14 @@ DEFINE_SHOW_ATTRIBUTE(identity);
 
 void pdsc_debugfs_add_ident(struct pdsc *pdsc)
 {
+	struct dentry *dentry;
+
 	/* This file will already exist in the reset flow */
-	if (debugfs_lookup("identity", pdsc->dentry))
+	dentry = debugfs_lookup("identity", pdsc->dentry);
+	if (!IS_ERR_OR_NULL(dentry)) {
+		dput(dentry);
 		return;
+	}
 
 	debugfs_create_file("identity", 0400, pdsc->dentry,
 			    pdsc, &identity_fops);
diff --git a/drivers/net/ethernet/amd/pds_core/dev.c b/drivers/net/ethernet/amd/pds_core/dev.c
index 2e1d0d01d03af..bded6b33289ce 100644
--- a/drivers/net/ethernet/amd/pds_core/dev.c
+++ b/drivers/net/ethernet/amd/pds_core/dev.c
@@ -162,12 +162,19 @@ static int pdsc_devcmd_wait(struct pdsc *pdsc, u8 opcode, int max_seconds)
 		dev_dbg(dev, "DEVCMD %d %s after %ld secs\n",
 			opcode, pdsc_devcmd_str(opcode), duration / HZ);
 
-	if ((!done || timeout) && running) {
+	if (!running) {
+		dev_err(dev, "DEVCMD %d %s fw not running\n",
+			opcode, pdsc_devcmd_str(opcode));
+		pdsc_devcmd_clean(pdsc);
+		return -ENXIO;
+	}
+
+	if (!done || timeout) {
 		dev_err(dev, "DEVCMD %d %s timeout, done %d timeout %d max_seconds=%d\n",
 			opcode, pdsc_devcmd_str(opcode), done, timeout,
 			max_seconds);
-		err = -ETIMEDOUT;
 		pdsc_devcmd_clean(pdsc);
+		return -ETIMEDOUT;
 	}
 
 	status = pdsc_devcmd_status(pdsc);
diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c
index a5ab994741790..4e4794c4dfdce 100644
--- a/drivers/net/ethernet/atheros/ag71xx.c
+++ b/drivers/net/ethernet/atheros/ag71xx.c
@@ -1856,6 +1856,9 @@ static int ag71xx_probe(struct platform_device *pdev)
 	ag71xx_int_disable(ag, AG71XX_INT_POLL);
 
 	ndev->irq = platform_get_irq(pdev, 0);
+	if (ndev->irq < 0)
+		return ndev->irq;
+
 	err = devm_request_irq(&pdev->dev, ndev->irq, ag71xx_interrupt,
 			       0x0, dev_name(&pdev->dev), ndev);
 	if (err) {
diff --git a/drivers/net/ethernet/atheros/atlx/atl2.c b/drivers/net/ethernet/atheros/atlx/atl2.c
index 280e2f5f4aa54..5560d5cc31692 100644
--- a/drivers/net/ethernet/atheros/atlx/atl2.c
+++ b/drivers/net/ethernet/atheros/atlx/atl2.c
@@ -2756,10 +2756,8 @@ static void atl2_force_ps(struct atl2_hw *hw)
 /* All parameters are treated the same, as an integer array of values.
  * This macro just reduces the need to repeat the same declaration code
  * over and over (plus this helps to avoid typo bugs).
- */
-#define ATL2_PARAM_INIT {[0 ... ATL2_MAX_NIC] = OPTION_UNSET}
-#ifndef module_param_array
-/* Module Parameters are always initialized to -1, so that the driver
+ *
+ * Module parameters are always initialized to -1, so that the driver
  * can tell the difference between no user specified value or the
  * user asking for the default value.
  * The true default values are loaded in when atl2_check_options is called.
@@ -2770,16 +2768,10 @@ static void atl2_force_ps(struct atl2_hw *hw)
  */
 
 #define ATL2_PARAM(X, desc) \
-    static const int X[ATL2_MAX_NIC + 1] = ATL2_PARAM_INIT; \
-    MODULE_PARM(X, "1-" __MODULE_STRING(ATL2_MAX_NIC) "i"); \
-    MODULE_PARM_DESC(X, desc);
-#else
-#define ATL2_PARAM(X, desc) \
-    static int X[ATL2_MAX_NIC+1] = ATL2_PARAM_INIT; \
+    static int X[ATL2_MAX_NIC+1] = {[0 ... ATL2_MAX_NIC] = OPTION_UNSET}; \
     static unsigned int num_##X; \
     module_param_array_named(X, X, int, &num_##X, 0); \
     MODULE_PARM_DESC(X, desc);
-#endif
 
 /*
  * Transmit Memory Size
@@ -2924,9 +2916,6 @@ static void atl2_check_options(struct atl2_adapter *adapter)
 		printk(KERN_NOTICE "Warning: no configuration for board #%i\n",
 			bd);
 		printk(KERN_NOTICE "Using defaults for all values\n");
-#ifndef module_param_array
-		bd = ATL2_MAX_NIC;
-#endif
 	}
 
 	/* Bytes of Transmit Memory */
@@ -2936,16 +2925,12 @@ static void atl2_check_options(struct atl2_adapter *adapter)
 	opt.def = ATL2_DEFAULT_TX_MEMSIZE;
 	opt.arg.r.min = ATL2_MIN_TX_MEMSIZE;
 	opt.arg.r.max = ATL2_MAX_TX_MEMSIZE;
-#ifdef module_param_array
 	if (num_TxMemSize > bd) {
-#endif
 		val = TxMemSize[bd];
 		atl2_validate_option(&val, &opt);
 		adapter->txd_ring_size = ((u32) val) * 1024;
-#ifdef module_param_array
 	} else
 		adapter->txd_ring_size = ((u32)opt.def) * 1024;
-#endif
 	/* txs ring size: */
 	adapter->txs_ring_size = adapter->txd_ring_size / 128;
 	if (adapter->txs_ring_size > 160)
@@ -2958,18 +2943,14 @@ static void atl2_check_options(struct atl2_adapter *adapter)
 	opt.def = ATL2_DEFAULT_RXD_COUNT;
 	opt.arg.r.min = ATL2_MIN_RXD_COUNT;
 	opt.arg.r.max = ATL2_MAX_RXD_COUNT;
-#ifdef module_param_array
 	if (num_RxMemBlock > bd) {
-#endif
 		val = RxMemBlock[bd];
 		atl2_validate_option(&val, &opt);
 		adapter->rxd_ring_size = (u32)val;
 		/* FIXME */
 		/* ((u16)val)&~1; */	/* even number */
-#ifdef module_param_array
 	} else
 		adapter->rxd_ring_size = (u32)opt.def;
-#endif
 	/* init RXD Flow control value */
 	adapter->hw.fc_rxd_hi = (adapter->rxd_ring_size / 8) * 7;
 	adapter->hw.fc_rxd_lo = (ATL2_MIN_RXD_COUNT / 8) >
@@ -2983,16 +2964,12 @@ static void atl2_check_options(struct atl2_adapter *adapter)
 	opt.def = INT_MOD_DEFAULT_CNT;
 	opt.arg.r.min = INT_MOD_MIN_CNT;
 	opt.arg.r.max = INT_MOD_MAX_CNT;
-#ifdef module_param_array
 	if (num_IntModTimer > bd) {
-#endif
 		val = IntModTimer[bd];
 		atl2_validate_option(&val, &opt);
 		adapter->imt = (u16) val;
-#ifdef module_param_array
 	} else
 		adapter->imt = (u16)(opt.def);
-#endif
 	/* Flash Vendor */
 	opt.type = range_option;
 	opt.name = "SPI Flash Vendor";
@@ -3000,16 +2977,12 @@ static void atl2_check_options(struct atl2_adapter *adapter)
 	opt.def = FLASH_VENDOR_DEFAULT;
 	opt.arg.r.min = FLASH_VENDOR_MIN;
 	opt.arg.r.max = FLASH_VENDOR_MAX;
-#ifdef module_param_array
 	if (num_FlashVendor > bd) {
-#endif
 		val = FlashVendor[bd];
 		atl2_validate_option(&val, &opt);
 		adapter->hw.flash_vendor = (u8) val;
-#ifdef module_param_array
 	} else
 		adapter->hw.flash_vendor = (u8)(opt.def);
-#endif
 	/* MediaType */
 	opt.type = range_option;
 	opt.name = "Speed/Duplex Selection";
@@ -3017,14 +2990,10 @@ static void atl2_check_options(struct atl2_adapter *adapter)
 	opt.def = MEDIA_TYPE_AUTO_SENSOR;
 	opt.arg.r.min = MEDIA_TYPE_AUTO_SENSOR;
 	opt.arg.r.max = MEDIA_TYPE_10M_HALF;
-#ifdef module_param_array
 	if (num_MediaType > bd) {
-#endif
 		val = MediaType[bd];
 		atl2_validate_option(&val, &opt);
 		adapter->hw.MediaType = (u16) val;
-#ifdef module_param_array
 	} else
 		adapter->hw.MediaType = (u16)(opt.def);
-#endif
 }
diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c
index ec63f50a849e2..ed0977832ce4f 100644
--- a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c
+++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c
@@ -344,40 +344,35 @@ static netdev_tx_t bcmasp_xmit(struct sk_buff *skb, struct net_device *dev)
 	return NETDEV_TX_OK;
 }
 
-static void bcmasp_netif_start(struct net_device *dev)
+static void umac_reset_and_init(struct bcmasp_intf *intf,
+				const unsigned char *addr)
 {
-	struct bcmasp_intf *intf = netdev_priv(dev);
-
-	bcmasp_set_rx_mode(dev);
-	napi_enable(&intf->tx_napi);
-	napi_enable(&intf->rx_napi);
-
-	bcmasp_enable_rx_irq(intf, 1);
-	bcmasp_enable_tx_irq(intf, 1);
-	bcmasp_enable_phy_irq(intf, 1);
+	struct phy_device *phydev = intf->ndev->phydev;
+	u32 mac0, mac1;
 
-	phy_start(dev->phydev);
-}
-
-static void umac_reset(struct bcmasp_intf *intf)
-{
 	umac_wl(intf, 0x0, UMC_CMD);
 	umac_wl(intf, UMC_CMD_SW_RESET, UMC_CMD);
 	usleep_range(10, 100);
 	/* We hold the umac in reset and bring it out of
 	 * reset when phy link is up.
 	 */
-}
 
-static void umac_set_hw_addr(struct bcmasp_intf *intf,
-			     const unsigned char *addr)
-{
-	u32 mac0 = (addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) |
-		    addr[3];
-	u32 mac1 = (addr[4] << 8) | addr[5];
+	umac_wl(intf, 0x800, UMC_FRM_LEN);
+	umac_wl(intf, 0xffff, UMC_PAUSE_CNTRL);
+	umac_wl(intf, 0x800, UMC_RX_MAX_PKT_SZ);
+
+	mac0 = (addr[0] << 24) | (addr[1] << 16) | (addr[2] << 8) |
+		addr[3];
+	mac1 = (addr[4] << 8) | addr[5];
 
 	umac_wl(intf, mac0, UMC_MAC0);
 	umac_wl(intf, mac1, UMC_MAC1);
+
+	/* Reset shadow values since we reset the umac */
+	intf->old_duplex = -1;
+	intf->old_link = -1;
+	intf->old_pause = -1;
+	phydev->eee_cfg.tx_lpi_timer = umac_rl(intf, UMC_EEE_LPI_TIMER);
 }
 
 static void umac_enable_set(struct bcmasp_intf *intf, u32 mask,
@@ -401,13 +396,6 @@ static void umac_enable_set(struct bcmasp_intf *intf, u32 mask,
 		usleep_range(1000, 2000);
 }
 
-static void umac_init(struct bcmasp_intf *intf)
-{
-	umac_wl(intf, 0x800, UMC_FRM_LEN);
-	umac_wl(intf, 0xffff, UMC_PAUSE_CNTRL);
-	umac_wl(intf, 0x800, UMC_RX_MAX_PKT_SZ);
-}
-
 static int bcmasp_tx_reclaim(struct bcmasp_intf *intf)
 {
 	struct bcmasp_intf_stats64 *stats = &intf->stats64;
@@ -927,7 +915,15 @@ static void bcmasp_rgmii_mode_en_set(struct bcmasp_intf *intf, bool enable)
 	rgmii_wl(intf, reg, RGMII_OOB_CNTRL);
 }
 
-static void bcmasp_netif_deinit(struct net_device *dev)
+static void bcmasp_phy_hw_unprepare(struct bcmasp_intf *intf)
+{
+	if (intf->internal_phy)
+		bcmasp_ephy_enable_set(intf, false);
+	else
+		bcmasp_rgmii_mode_en_set(intf, false);
+}
+
+static void bcmasp_netif_deinit(struct net_device *dev, bool stop_phy)
 {
 	struct bcmasp_intf *intf = netdev_priv(dev);
 	u32 reg, timeout = 1000;
@@ -950,7 +946,8 @@ static void bcmasp_netif_deinit(struct net_device *dev)
 
 	umac_enable_set(intf, UMC_CMD_TX_EN, 0);
 
-	phy_stop(dev->phydev);
+	if (stop_phy)
+		phy_stop(dev->phydev);
 
 	umac_enable_set(intf, UMC_CMD_RX_EN, 0);
 
@@ -978,17 +975,13 @@ static int bcmasp_stop(struct net_device *dev)
 	/* Stop tx from updating HW */
 	netif_tx_disable(dev);
 
-	bcmasp_netif_deinit(dev);
+	bcmasp_netif_deinit(dev, true);
 
 	bcmasp_reclaim_free_buffers(intf);
 
 	phy_disconnect(dev->phydev);
 
-	/* Disable internal EPHY or external PHY */
-	if (intf->internal_phy)
-		bcmasp_ephy_enable_set(intf, false);
-	else
-		bcmasp_rgmii_mode_en_set(intf, false);
+	bcmasp_phy_hw_unprepare(intf);
 
 	/* Disable the interface clocks */
 	bcmasp_core_clock_set_intf(intf, false);
@@ -998,10 +991,15 @@ static int bcmasp_stop(struct net_device *dev)
 	return 0;
 }
 
-static void bcmasp_configure_port(struct bcmasp_intf *intf)
+static void bcmasp_phy_hw_prepare(struct bcmasp_intf *intf)
 {
 	u32 reg, id_mode_dis = 0;
 
+	if (intf->internal_phy)
+		bcmasp_ephy_enable_set(intf, true);
+	else
+		bcmasp_rgmii_mode_en_set(intf, true);
+
 	reg = rgmii_rl(intf, RGMII_PORT_CNTRL);
 	reg &= ~RGMII_PORT_MODE_MASK;
 
@@ -1036,26 +1034,8 @@ static void bcmasp_configure_port(struct bcmasp_intf *intf)
 	rgmii_wl(intf, reg, RGMII_OOB_CNTRL);
 }
 
-static int bcmasp_netif_init(struct net_device *dev, bool phy_connect)
+static phy_interface_t bcmasp_phy_iface_for_connect(phy_interface_t mode)
 {
-	struct bcmasp_intf *intf = netdev_priv(dev);
-	phy_interface_t phy_iface = intf->phy_interface;
-	u32 phy_flags = PHY_BRCM_AUTO_PWRDWN_ENABLE |
-			PHY_BRCM_DIS_TXCRXC_NOENRGY |
-			PHY_BRCM_IDDQ_SUSPEND;
-	struct phy_device *phydev = NULL;
-	int ret;
-
-	/* Always enable interface clocks */
-	bcmasp_core_clock_set_intf(intf, true);
-
-	/* Enable internal PHY or external PHY before any MAC activity */
-	if (intf->internal_phy)
-		bcmasp_ephy_enable_set(intf, true);
-	else
-		bcmasp_rgmii_mode_en_set(intf, true);
-	bcmasp_configure_port(intf);
-
 	/* This is an ugly quirk but we have not been correctly
 	 * interpreting the phy_interface values and we have done that
 	 * across different drivers, so at least we are consistent in
@@ -1081,46 +1061,43 @@ static int bcmasp_netif_init(struct net_device *dev, bool phy_connect)
 	 * affected because they use different phy_interface_t values
 	 * or the Generic PHY driver.
 	 */
-	switch (phy_iface) {
+	switch (mode) {
 	case PHY_INTERFACE_MODE_RGMII:
-		phy_iface = PHY_INTERFACE_MODE_RGMII_ID;
-		break;
+		return PHY_INTERFACE_MODE_RGMII_ID;
 	case PHY_INTERFACE_MODE_RGMII_TXID:
-		phy_iface = PHY_INTERFACE_MODE_RGMII_RXID;
-		break;
+		return PHY_INTERFACE_MODE_RGMII_RXID;
 	default:
-		break;
+		return mode;
 	}
+}
 
-	if (phy_connect) {
-		phydev = of_phy_connect(dev, intf->phy_dn,
-					bcmasp_adj_link, phy_flags,
-					phy_iface);
-		if (!phydev) {
-			ret = -ENODEV;
-			netdev_err(dev, "could not attach to PHY\n");
-			goto err_phy_disable;
-		}
-
-		if (intf->internal_phy)
-			dev->phydev->irq = PHY_MAC_INTERRUPT;
-
-		/* Indicate that the MAC is responsible for PHY PM */
-		phydev->mac_managed_pm = true;
-
-		/* Set phylib's copy of the LPI timer */
-		phydev->eee_cfg.tx_lpi_timer = umac_rl(intf, UMC_EEE_LPI_TIMER);
+static int bcmasp_phy_attach(struct bcmasp_intf *intf)
+{
+	u32 phy_flags = PHY_BRCM_AUTO_PWRDWN_ENABLE |
+			PHY_BRCM_DIS_TXCRXC_NOENRGY |
+			PHY_BRCM_IDDQ_SUSPEND;
+	struct phy_device *phydev;
+	phy_interface_t phy_iface;
+
+	phy_iface = bcmasp_phy_iface_for_connect(intf->phy_interface);
+	phydev = of_phy_connect(intf->ndev, intf->phy_dn,
+				bcmasp_adj_link, phy_flags,
+				phy_iface);
+	if (!phydev) {
+		netdev_err(intf->ndev, "could not attach to PHY\n");
+		return -ENODEV;
 	}
+	if (intf->internal_phy)
+		intf->ndev->phydev->irq = PHY_MAC_INTERRUPT;
 
-	umac_reset(intf);
-
-	umac_init(intf);
+	phydev->mac_managed_pm = true;
 
-	umac_set_hw_addr(intf, dev->dev_addr);
+	return 0;
+}
 
-	intf->old_duplex = -1;
-	intf->old_link = -1;
-	intf->old_pause = -1;
+static void bcmasp_netif_init(struct net_device *dev)
+{
+	struct bcmasp_intf *intf = netdev_priv(dev);
 
 	bcmasp_init_tx(intf);
 	netif_napi_add_tx(intf->ndev, &intf->tx_napi, bcmasp_tx_poll);
@@ -1132,18 +1109,13 @@ static int bcmasp_netif_init(struct net_device *dev, bool phy_connect)
 
 	intf->crc_fwd = !!(umac_rl(intf, UMC_CMD) & UMC_CMD_CRC_FWD);
 
-	bcmasp_netif_start(dev);
-
-	netif_start_queue(dev);
-
-	return 0;
+	bcmasp_set_rx_mode(dev);
+	napi_enable(&intf->tx_napi);
+	napi_enable(&intf->rx_napi);
 
-err_phy_disable:
-	if (intf->internal_phy)
-		bcmasp_ephy_enable_set(intf, false);
-	else
-		bcmasp_rgmii_mode_en_set(intf, false);
-	return ret;
+	bcmasp_enable_rx_irq(intf, 1);
+	bcmasp_enable_tx_irq(intf, 1);
+	bcmasp_enable_phy_irq(intf, 1);
 }
 
 static int bcmasp_open(struct net_device *dev)
@@ -1161,14 +1133,30 @@ static int bcmasp_open(struct net_device *dev)
 	if (ret)
 		goto err_free_mem;
 
-	ret = bcmasp_netif_init(dev, true);
-	if (ret) {
-		clk_disable_unprepare(intf->parent->clk);
-		goto err_free_mem;
-	}
+	bcmasp_core_clock_set_intf(intf, true);
+
+	bcmasp_phy_hw_prepare(intf);
+
+	ret = bcmasp_phy_attach(intf);
+	if (ret)
+		goto err_phy_attach;
+
+	umac_reset_and_init(intf, dev->dev_addr);
+
+	dev->phydev->eee_cfg.tx_lpi_timer = umac_rl(intf, UMC_EEE_LPI_TIMER);
+
+	bcmasp_netif_init(dev);
+
+	phy_start(dev->phydev);
+
+	netif_start_queue(dev);
 
 	return ret;
 
+err_phy_attach:
+	bcmasp_phy_hw_unprepare(intf);
+	bcmasp_core_clock_set_intf(intf, false);
+	clk_disable_unprepare(intf->parent->clk);
 err_free_mem:
 	bcmasp_reclaim_free_buffers(intf);
 
@@ -1398,19 +1386,27 @@ int bcmasp_interface_suspend(struct bcmasp_intf *intf)
 {
 	struct device *kdev = &intf->parent->pdev->dev;
 	struct net_device *dev = intf->ndev;
+	bool wake;
 
 	if (!netif_running(dev))
 		return 0;
 
 	netif_device_detach(dev);
 
-	bcmasp_netif_deinit(dev);
+	wake = device_may_wakeup(kdev) && intf->wolopts;
 
-	if (!intf->wolopts) {
-		if (intf->internal_phy)
-			bcmasp_ephy_enable_set(intf, false);
-		else
-			bcmasp_rgmii_mode_en_set(intf, false);
+	bcmasp_netif_deinit(dev, !wake);
+
+	if (wake) {
+		/* Disable phy status updates while suspending */
+		mutex_lock(&dev->phydev->lock);
+		dev->phydev->state = PHY_READY;
+		mutex_unlock(&dev->phydev->lock);
+		cancel_delayed_work_sync(&dev->phydev->state_queue);
+
+		bcmasp_suspend_to_wol(intf);
+	} else {
+		bcmasp_phy_hw_unprepare(intf);
 
 		/* If Wake-on-LAN is disabled, we can safely
 		 * disable the network interface clocks.
@@ -1418,9 +1414,6 @@ int bcmasp_interface_suspend(struct bcmasp_intf *intf)
 		bcmasp_core_clock_set_intf(intf, false);
 	}
 
-	if (device_may_wakeup(kdev) && intf->wolopts)
-		bcmasp_suspend_to_wol(intf);
-
 	clk_disable_unprepare(intf->parent->clk);
 
 	return 0;
@@ -1444,8 +1437,11 @@ static void bcmasp_resume_from_wol(struct bcmasp_intf *intf)
 
 int bcmasp_interface_resume(struct bcmasp_intf *intf)
 {
+	struct device *kdev = &intf->parent->pdev->dev;
 	struct net_device *dev = intf->ndev;
+	bool wake;
 	int ret;
+	u32 reg;
 
 	if (!netif_running(dev))
 		return 0;
@@ -1454,17 +1450,41 @@ int bcmasp_interface_resume(struct bcmasp_intf *intf)
 	if (ret)
 		return ret;
 
-	ret = bcmasp_netif_init(dev, false);
-	if (ret)
-		goto out;
+	wake = device_may_wakeup(kdev) && intf->wolopts;
+
+	bcmasp_core_clock_set_intf(intf, true);
+
+	/* The interface might be HW reset in some suspend modes, so we may
+	 * need to restore the UNIMAC/PHY if that is the case.
+	 */
+	reg = umac_rl(intf, UMC_CMD);
+	if (wake && (reg & UMC_CMD_RX_EN)) {
+		umac_enable_set(intf, UMC_CMD_TX_EN, 1);
+		bcmasp_resume_from_wol(intf);
+	} else {
+		bcmasp_phy_hw_prepare(intf);
+		umac_reset_and_init(intf, dev->dev_addr);
+	}
+
+	bcmasp_netif_init(dev);
 
-	bcmasp_resume_from_wol(intf);
+	if (wake) {
+		/* If HW was reset, reprogram the unimac/PHY before resuming
+		 * link status tracking to avoid racing the state machine.
+		 */
+		if (!(reg & UMC_CMD_RX_EN))
+			bcmasp_adj_link(dev);
+
+		/* Resume link status tracking */
+		mutex_lock(&dev->phydev->lock);
+		dev->phydev->state = dev->phydev->link ? PHY_RUNNING : PHY_NOLINK;
+		mutex_unlock(&dev->phydev->lock);
+		phy_trigger_machine(dev->phydev);
+	} else {
+		phy_start(dev->phydev);
+	}
 
 	netif_device_attach(dev);
 
 	return 0;
-
-out:
-	clk_disable_unprepare(intf->parent->clk);
-	return ret;
 }
diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c
index f5722e9298338..3e2fd2939e878 100644
--- a/drivers/net/ethernet/broadcom/bnx2.c
+++ b/drivers/net/ethernet/broadcom/bnx2.c
@@ -115,29 +115,45 @@ static struct {
 	};
 
 static const struct pci_device_id bnx2_pci_tbl[] = {
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_5706,
-	  PCI_VENDOR_ID_HP, 0x3101, 0, 0, NC370T },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_5706,
-	  PCI_VENDOR_ID_HP, 0x3106, 0, 0, NC370I },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_5706,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM5706 },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_5708,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM5708 },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_5706S,
-	  PCI_VENDOR_ID_HP, 0x3102, 0, 0, NC370F },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_5706S,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM5706S },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_5708S,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM5708S },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_5709,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM5709 },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_NX2_5709S,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM5709S },
-	{ PCI_VENDOR_ID_BROADCOM, 0x163b,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM5716 },
-	{ PCI_VENDOR_ID_BROADCOM, 0x163c,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, BCM5716S },
-	{ 0, }
+	{
+		PCI_VDEVICE_SUB(BROADCOM, PCI_DEVICE_ID_NX2_5706,
+				PCI_VENDOR_ID_HP, 0x3101),
+		.driver_data = NC370T,
+	}, {
+		PCI_VDEVICE_SUB(BROADCOM, PCI_DEVICE_ID_NX2_5706,
+				PCI_VENDOR_ID_HP, 0x3106),
+		.driver_data = NC370I,
+	}, {
+		PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_5706),
+		.driver_data = BCM5706,
+	}, {
+		PCI_VDEVICE_SUB(BROADCOM, PCI_DEVICE_ID_NX2_5708,
+				PCI_ANY_ID, PCI_ANY_ID),
+		.driver_data = BCM5708,
+	}, {
+		PCI_VDEVICE_SUB(BROADCOM, PCI_DEVICE_ID_NX2_5706S,
+				PCI_VENDOR_ID_HP, 0x3102),
+		.driver_data = NC370F,
+	}, {
+		PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_5706S),
+		.driver_data = BCM5706S,
+	}, {
+		PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_5708S),
+		.driver_data = BCM5708S,
+	}, {
+		PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_5709),
+		.driver_data = BCM5709,
+	}, {
+		PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_5709S),
+		.driver_data = BCM5709S,
+	}, {
+		PCI_VDEVICE(BROADCOM, 0x163b),
+		.driver_data = BCM5716,
+	}, {
+		PCI_VDEVICE(BROADCOM, 0x163c),
+		.driver_data = BCM5716S,
+	},
+	{ }
 };
 
 static const struct flash_spec flash_table[] =
diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
index da0f8c353e6a7..6f96645ff23f4 100644
--- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
+++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c
@@ -259,31 +259,31 @@ static struct {
 #endif
 
 static const struct pci_device_id bnx2x_pci_tbl[] = {
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57710), BCM57710 },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57711), BCM57711 },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57711E), BCM57711E },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57712), BCM57712 },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57712_MF), BCM57712_MF },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57712_VF), BCM57712_VF },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57800), BCM57800 },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57800_MF), BCM57800_MF },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57800_VF), BCM57800_VF },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57810), BCM57810 },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57810_MF), BCM57810_MF },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_O), BCM57840_O },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_4_10), BCM57840_4_10 },
-	{ PCI_VDEVICE(QLOGIC,	PCI_DEVICE_ID_NX2_57840_4_10), BCM57840_4_10 },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_2_20), BCM57840_2_20 },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57810_VF), BCM57810_VF },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_MFO), BCM57840_MFO },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_MF), BCM57840_MF },
-	{ PCI_VDEVICE(QLOGIC,	PCI_DEVICE_ID_NX2_57840_MF), BCM57840_MF },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_VF), BCM57840_VF },
-	{ PCI_VDEVICE(QLOGIC,	PCI_DEVICE_ID_NX2_57840_VF), BCM57840_VF },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57811), BCM57811 },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57811_MF), BCM57811_MF },
-	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57811_VF), BCM57811_VF },
-	{ 0 }
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57710), .driver_data = BCM57710 },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57711), .driver_data = BCM57711 },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57711E), .driver_data = BCM57711E },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57712), .driver_data = BCM57712 },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57712_MF), .driver_data = BCM57712_MF },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57712_VF), .driver_data = BCM57712_VF },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57800), .driver_data = BCM57800 },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57800_MF), .driver_data = BCM57800_MF },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57800_VF), .driver_data = BCM57800_VF },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57810), .driver_data = BCM57810 },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57810_MF), .driver_data = BCM57810_MF },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_O), .driver_data = BCM57840_O },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_4_10), .driver_data = BCM57840_4_10 },
+	{ PCI_VDEVICE(QLOGIC,	PCI_DEVICE_ID_NX2_57840_4_10), .driver_data = BCM57840_4_10 },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_2_20), .driver_data = BCM57840_2_20 },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57810_VF), .driver_data = BCM57810_VF },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_MFO), .driver_data = BCM57840_MFO },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_MF), .driver_data = BCM57840_MF },
+	{ PCI_VDEVICE(QLOGIC,	PCI_DEVICE_ID_NX2_57840_MF), .driver_data = BCM57840_MF },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57840_VF), .driver_data = BCM57840_VF },
+	{ PCI_VDEVICE(QLOGIC,	PCI_DEVICE_ID_NX2_57840_VF), .driver_data = BCM57840_VF },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57811), .driver_data = BCM57811 },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57811_MF), .driver_data = BCM57811_MF },
+	{ PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_NX2_57811_VF), .driver_data = BCM57811_VF },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, bnx2x_pci_tbl);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index 008c34cff7b46..d4f93e62f5836 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -17123,7 +17123,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
 	dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops_unsupp;
 	if (BNXT_SUPPORTS_QUEUE_API(bp))
 		dev->queue_mgmt_ops = &bnxt_queue_mgmt_ops;
-	dev->netmem_tx = true;
+	dev->netmem_tx = NETMEM_TX_DMA;
 
 	rc = register_netdev(dev);
 	if (rc)
@@ -17394,7 +17394,6 @@ static pci_ers_result_t bnxt_io_slot_reset(struct pci_dev *pdev)
 				pci_write_config_dword(bp->pdev, off, 0);
 		}
 		pci_restore_state(pdev);
-		pci_save_state(pdev);
 
 		bnxt_inv_fw_health_reg(bp);
 		bnxt_try_map_fw_health_reg(bp);
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_main.c b/drivers/net/ethernet/cavium/liquidio/lio_main.c
index c1a3df2252549..0db08ac3d098f 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_main.c
@@ -404,17 +404,15 @@ static const struct pci_error_handlers liquidio_err_handler = {
 
 static const struct pci_device_id liquidio_pci_tbl[] = {
 	{       /* 68xx */
-		PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
+		PCI_VDEVICE(CAVIUM, 0x0091)
 	},
 	{       /* 66xx */
-		PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
+		PCI_VDEVICE(CAVIUM, 0x0092)
 	},
 	{       /* 23xx pf */
-		PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
+		PCI_VDEVICE(CAVIUM, 0x9702)
 	},
-	{
-		0, 0, 0, 0, 0, 0, 0
-	}
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
 
diff --git a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
index 43c595f3b84e4..36255352e7ea8 100644
--- a/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
+++ b/drivers/net/ethernet/cavium/liquidio/lio_vf_main.c
@@ -237,12 +237,9 @@ static const struct pci_error_handlers liquidio_vf_err_handler = {
 
 static const struct pci_device_id liquidio_vf_pci_tbl[] = {
 	{
-		PCI_VENDOR_ID_CAVIUM, OCTEON_CN23XX_VF_VID,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
+		PCI_VDEVICE(CAVIUM, OCTEON_CN23XX_VF_VID),
 	},
-	{
-		0, 0, 0, 0, 0, 0, 0
-	}
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, liquidio_vf_pci_tbl);
 
diff --git a/drivers/net/ethernet/chelsio/cxgb/common.h b/drivers/net/ethernet/chelsio/cxgb/common.h
index 304bb282ab03d..292f8ec248e29 100644
--- a/drivers/net/ethernet/chelsio/cxgb/common.h
+++ b/drivers/net/ethernet/chelsio/cxgb/common.h
@@ -48,7 +48,7 @@
 #define DRV_NAME "cxgb"
 
 #define CH_DEVICE(devid, ssid, idx) \
-	{ PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, ssid, 0, 0, idx }
+	{ PCI_VDEVICE_SUB(CHELSIO, devid, PCI_ANY_ID, ssid), .driver_data = idx }
 
 #define SUPPORTED_PAUSE       (1 << 13)
 #define SUPPORTED_LOOPBACK    (1 << 15)
diff --git a/drivers/net/ethernet/chelsio/cxgb/subr.c b/drivers/net/ethernet/chelsio/cxgb/subr.c
index 367a9e4581d58..d4488e400bc6e 100644
--- a/drivers/net/ethernet/chelsio/cxgb/subr.c
+++ b/drivers/net/ethernet/chelsio/cxgb/subr.c
@@ -533,7 +533,7 @@ const struct pci_device_id t1_pci_tbl[] = {
 	CH_DEVICE(11, 1, CH_BRD_T210_1F),
 	CH_DEVICE(14, 1, CH_BRD_T210_1CU),
 	CH_DEVICE(16, 1, CH_BRD_N204_4CU),
-	{ 0 }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, t1_pci_tbl);
diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
index 88b6d76817e9e..3bee82bc06806 100644
--- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c
@@ -83,7 +83,7 @@ enum {
 #define EEPROM_MAGIC 0x38E2F10C
 
 #define CH_DEVICE(devid, idx) \
-	{ PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
+	{ PCI_VDEVICE(CHELSIO, devid), .driver_data = idx }
 
 static const struct pci_device_id cxgb3_pci_tbl[] = {
 	CH_DEVICE(0x20, 0),	/* PE9000 */
@@ -99,7 +99,7 @@ static const struct pci_device_id cxgb3_pci_tbl[] = {
 	CH_DEVICE(0x35, 6),	/* T3C20-derived T3C10 */
 	CH_DEVICE(0x36, 3),	/* S320E-CR */
 	CH_DEVICE(0x37, 7),	/* N320E-G2 */
-	{0,}
+	{ }
 };
 
 MODULE_DESCRIPTION(DRV_DESC);
diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
index 10694d5924ee3..6df98fca932f7 100644
--- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c
@@ -112,10 +112,10 @@ char cxgb4_driver_name[] = KBUILD_MODNAME;
 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0
 
 #define CH_PCI_ID_TABLE_ENTRY(devid) \
-		{PCI_VDEVICE(CHELSIO, (devid)), CXGB4_UNIFIED_PF}
+		{ PCI_VDEVICE(CHELSIO, (devid)), .driver_data = CXGB4_UNIFIED_PF }
 
 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
-		{ 0, } \
+		{ } \
 	}
 
 #include "t4_pci_id_tbl.h"
diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
index 9eaa9aadee335..37b53cc64f392 100644
--- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
+++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c
@@ -3388,9 +3388,9 @@ static void cxgb4vf_pci_shutdown(struct pci_dev *pdev)
 #define CH_PCI_DEVICE_ID_FUNCTION	0x8
 
 #define CH_PCI_ID_TABLE_ENTRY(devid) \
-		{ PCI_VDEVICE(CHELSIO, (devid)), 0 }
+		{ PCI_VDEVICE(CHELSIO, (devid)) }
 
-#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END { 0, } }
+#define CH_PCI_DEVICE_ID_TABLE_DEFINE_END { } }
 
 #include "../cxgb4/t4_pci_id_tbl.h"
 
diff --git a/drivers/net/ethernet/cirrus/Kconfig b/drivers/net/ethernet/cirrus/Kconfig
index 5bdf731d95030..d76f9cd6bc2d1 100644
--- a/drivers/net/ethernet/cirrus/Kconfig
+++ b/drivers/net/ethernet/cirrus/Kconfig
@@ -6,7 +6,7 @@
 config NET_VENDOR_CIRRUS
 	bool "Cirrus devices"
 	default y
-	depends on ISA || EISA || ARM || MAC || COMPILE_TEST
+	depends on ARM || MAC || COMPILE_TEST
 	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
 
@@ -20,25 +20,9 @@ if NET_VENDOR_CIRRUS
 config CS89x0
 	tristate
 
-config CS89x0_ISA
-	tristate "CS89x0 ISA driver support"
-	depends on HAS_IOPORT_MAP
-	depends on ISA
-	depends on !PPC32
-	depends on CS89x0_PLATFORM=n
-	select NETDEV_LEGACY_INIT
-	select CS89x0
-	help
-	  Support for CS89x0 chipset based Ethernet cards. If you have a
-	  network (Ethernet) card of this type, say Y and read the file
-	  <file:Documentation/networking/device_drivers/ethernet/cirrus/cs89x0.rst>.
-
-	  To compile this driver as a module, choose M here. The module
-	  will be called cs89x0.
-
 config CS89x0_PLATFORM
 	tristate "CS89x0 platform driver support"
-	depends on ARM || (COMPILE_TEST && !PPC)
+	depends on ARM || COMPILE_TEST
 	select CS89x0
 	help
 	  Say Y to compile the cs89x0 platform driver. This makes this driver
diff --git a/drivers/net/ethernet/cirrus/cs89x0.c b/drivers/net/ethernet/cirrus/cs89x0.c
index b4bfd6c174e78..fc8b1cb950a16 100644
--- a/drivers/net/ethernet/cirrus/cs89x0.c
+++ b/drivers/net/ethernet/cirrus/cs89x0.c
@@ -22,19 +22,6 @@
  * Domenico Andreoli : cavokz@gmail.com
  */
 
-
-/*
- * Set this to zero to disable DMA code
- *
- * Note that even if DMA is turned off we still support the 'dma' and  'use_dma'
- * module options so we don't break any startup scripts.
- */
-#ifndef CONFIG_ISA_DMA_API
-#define ALLOW_DMA	0
-#else
-#define ALLOW_DMA	1
-#endif
-
 /*
  * Set this to zero to remove all the debug statements via
  * dead code elimination
@@ -71,13 +58,8 @@
 #include <linux/gfp.h>
 #include <linux/io.h>
 
-#include <net/Space.h>
-
 #include <asm/irq.h>
 #include <linux/atomic.h>
-#if ALLOW_DMA
-#include <asm/dma.h>
-#endif
 
 #include "cs89x0.h"
 
@@ -91,30 +73,6 @@ static char version[] __initdata =
 	"v2.4.3-pre1 Russell Nelson <nelson@crynwr.com>, Andrew Morton";
 
 #define DRV_NAME "cs89x0"
-
-/* First, a few definitions that the brave might change.
- * A zero-terminated list of I/O addresses to be probed. Some special flags..
- * Addr & 1 = Read back the address port, look for signature and reset
- * the page window before probing
- * Addr & 3 = Reset the page window and probe
- * The CLPS eval board has the Cirrus chip at 0x80090300, in ARM IO space,
- * but it is possible that a Cirrus board could be plugged into the ISA
- * slots.
- */
-/* The cs8900 has 4 IRQ pins, software selectable. cs8900_irq_map maps
- * them to system IRQ numbers. This mapping is card specific and is set to
- * the configuration of the Cirrus Eval board for this chip.
- */
-#if IS_ENABLED(CONFIG_CS89x0_ISA)
-static unsigned int netcard_portlist[] __used __initdata = {
-	0x300, 0x320, 0x340, 0x360, 0x200, 0x220, 0x240,
-	0x260, 0x280, 0x2a0, 0x2c0, 0x2e0, 0
-};
-static unsigned int cs8900_irq_map[] = {
-	10, 11, 12, 5
-};
-#endif
-
 #if DEBUGGING
 static unsigned int net_debug = DEBUGGING;
 #else
@@ -149,14 +107,6 @@ struct net_local {
 	int force;		/* force various values; see FORCE* above. */
 	spinlock_t lock;
 	void __iomem *virt_addr;/* CS89x0 virtual address. */
-#if ALLOW_DMA
-	int use_dma;		/* Flag: we're using dma */
-	int dma;		/* DMA channel */
-	int dmasize;		/* 16 or 64 */
-	unsigned char *dma_buff;	/* points to the beginning of the buffer */
-	unsigned char *end_dma_buff;	/* points to the end of the buffer */
-	unsigned char *rx_dma_ptr;	/* points to the next packet  */
-#endif
 };
 
 /* Example routines you must write ;->. */
@@ -166,18 +116,6 @@ struct net_local {
  * Permit 'cs89x0_dma=N' in the kernel boot environment
  */
 #if !defined(MODULE)
-#if ALLOW_DMA
-static int g_cs89x0_dma;
-
-static int __init dma_fn(char *str)
-{
-	g_cs89x0_dma = simple_strtol(str, NULL, 0);
-	return 1;
-}
-
-__setup("cs89x0_dma=", dma_fn);
-#endif	/* ALLOW_DMA */
-
 static int g_cs89x0_media__force;
 
 static int __init media_fn(char *str)
@@ -290,22 +228,9 @@ get_eeprom_cksum(int off, int len, int *buffer)
 static void
 write_irq(struct net_device *dev, int chip_type, int irq)
 {
-	int i;
-
 	if (chip_type == CS8900) {
-#if IS_ENABLED(CONFIG_CS89x0_ISA)
-		/* Search the mapping table for the corresponding IRQ pin. */
-		for (i = 0; i != ARRAY_SIZE(cs8900_irq_map); i++)
-			if (cs8900_irq_map[i] == irq)
-				break;
-		/* Not found */
-		if (i == ARRAY_SIZE(cs8900_irq_map))
-			i = 3;
-#else
 		/* INTRQ0 pin is used for interrupt generation. */
-		i = 0;
-#endif
-		writereg(dev, PP_CS8900_ISAINT, i);
+		writereg(dev, PP_CS8900_ISAINT, 0);
 	} else {
 		writereg(dev, PP_CS8920_ISAINT, irq);
 	}
@@ -326,163 +251,6 @@ count_rx_errors(int status, struct net_device *dev)
 		dev->stats.rx_frame_errors++;
 }
 
-/*********************************
- * This page contains DMA routines
- *********************************/
-
-#if ALLOW_DMA
-
-#define dma_page_eq(ptr1, ptr2) ((long)(ptr1) >> 17 == (long)(ptr2) >> 17)
-
-static void
-get_dma_channel(struct net_device *dev)
-{
-	struct net_local *lp = netdev_priv(dev);
-
-	if (lp->dma) {
-		dev->dma = lp->dma;
-		lp->isa_config |= ISA_RxDMA;
-	} else {
-		if ((lp->isa_config & ANY_ISA_DMA) == 0)
-			return;
-		dev->dma = lp->isa_config & DMA_NO_MASK;
-		if (lp->chip_type == CS8900)
-			dev->dma += 5;
-		if (dev->dma < 5 || dev->dma > 7) {
-			lp->isa_config &= ~ANY_ISA_DMA;
-			return;
-		}
-	}
-}
-
-static void
-write_dma(struct net_device *dev, int chip_type, int dma)
-{
-	struct net_local *lp = netdev_priv(dev);
-	if ((lp->isa_config & ANY_ISA_DMA) == 0)
-		return;
-	if (chip_type == CS8900)
-		writereg(dev, PP_CS8900_ISADMA, dma - 5);
-	else
-		writereg(dev, PP_CS8920_ISADMA, dma);
-}
-
-static void
-set_dma_cfg(struct net_device *dev)
-{
-	struct net_local *lp = netdev_priv(dev);
-
-	if (lp->use_dma) {
-		if ((lp->isa_config & ANY_ISA_DMA) == 0) {
-			cs89_dbg(3, err, "set_dma_cfg(): no DMA\n");
-			return;
-		}
-		if (lp->isa_config & ISA_RxDMA) {
-			lp->curr_rx_cfg |= RX_DMA_ONLY;
-			cs89_dbg(3, info, "set_dma_cfg(): RX_DMA_ONLY\n");
-		} else {
-			lp->curr_rx_cfg |= AUTO_RX_DMA;	/* not that we support it... */
-			cs89_dbg(3, info, "set_dma_cfg(): AUTO_RX_DMA\n");
-		}
-	}
-}
-
-static int
-dma_bufcfg(struct net_device *dev)
-{
-	struct net_local *lp = netdev_priv(dev);
-	if (lp->use_dma)
-		return (lp->isa_config & ANY_ISA_DMA) ? RX_DMA_ENBL : 0;
-	else
-		return 0;
-}
-
-static int
-dma_busctl(struct net_device *dev)
-{
-	int retval = 0;
-	struct net_local *lp = netdev_priv(dev);
-	if (lp->use_dma) {
-		if (lp->isa_config & ANY_ISA_DMA)
-			retval |= RESET_RX_DMA; /* Reset the DMA pointer */
-		if (lp->isa_config & DMA_BURST)
-			retval |= DMA_BURST_MODE; /* Does ISA config specify DMA burst ? */
-		if (lp->dmasize == 64)
-			retval |= RX_DMA_SIZE_64K; /* did they ask for 64K? */
-		retval |= MEMORY_ON;	/* we need memory enabled to use DMA. */
-	}
-	return retval;
-}
-
-static void
-dma_rx(struct net_device *dev)
-{
-	struct net_local *lp = netdev_priv(dev);
-	struct sk_buff *skb;
-	int status, length;
-	unsigned char *bp = lp->rx_dma_ptr;
-
-	status = bp[0] + (bp[1] << 8);
-	length = bp[2] + (bp[3] << 8);
-	bp += 4;
-
-	cs89_dbg(5, debug, "%s: receiving DMA packet at %lx, status %x, length %x\n",
-		 dev->name, (unsigned long)bp, status, length);
-
-	if ((status & RX_OK) == 0) {
-		count_rx_errors(status, dev);
-		goto skip_this_frame;
-	}
-
-	/* Malloc up new buffer. */
-	skb = netdev_alloc_skb(dev, length + 2);
-	if (skb == NULL) {
-		dev->stats.rx_dropped++;
-
-		/* AKPM: advance bp to the next frame */
-skip_this_frame:
-		bp += (length + 3) & ~3;
-		if (bp >= lp->end_dma_buff)
-			bp -= lp->dmasize * 1024;
-		lp->rx_dma_ptr = bp;
-		return;
-	}
-	skb_reserve(skb, 2);	/* longword align L3 header */
-
-	if (bp + length > lp->end_dma_buff) {
-		int semi_cnt = lp->end_dma_buff - bp;
-		skb_put_data(skb, bp, semi_cnt);
-		skb_put_data(skb, lp->dma_buff, length - semi_cnt);
-	} else {
-		skb_put_data(skb, bp, length);
-	}
-	bp += (length + 3) & ~3;
-	if (bp >= lp->end_dma_buff)
-		bp -= lp->dmasize*1024;
-	lp->rx_dma_ptr = bp;
-
-	cs89_dbg(3, info, "%s: received %d byte DMA packet of type %x\n",
-		 dev->name, length,
-		 ((skb->data[ETH_ALEN + ETH_ALEN] << 8) |
-		  skb->data[ETH_ALEN + ETH_ALEN + 1]));
-
-	skb->protocol = eth_type_trans(skb, dev);
-	netif_rx(skb);
-	dev->stats.rx_packets++;
-	dev->stats.rx_bytes += length;
-}
-
-static void release_dma_buff(struct net_local *lp)
-{
-	if (lp->dma_buff) {
-		free_pages((unsigned long)(lp->dma_buff),
-			   get_order(lp->dmasize * 1024));
-		lp->dma_buff = NULL;
-	}
-}
-
-#endif	/* ALLOW_DMA */
-
 static void
 control_dc_dc(struct net_device *dev, int on_not_off)
 {
@@ -781,27 +549,6 @@ static irqreturn_t net_interrupt(int irq, void *dev_id)
 				 */
 				netif_wake_queue(dev);	/* Inform upper layers. */
 			}
-#if ALLOW_DMA
-			if (lp->use_dma && (status & RX_DMA)) {
-				int count = readreg(dev, PP_DmaFrameCnt);
-				while (count) {
-					cs89_dbg(5, debug,
-						 "%s: receiving %d DMA frames\n",
-						 dev->name, count);
-					if (count > 1)
-						cs89_dbg(2, debug,
-							 "%s: receiving %d DMA frames\n",
-							 dev->name, count);
-					dma_rx(dev);
-					if (--count == 0)
-						count = readreg(dev, PP_DmaFrameCnt);
-					if (count > 0)
-						cs89_dbg(2, debug,
-							 "%s: continuing with %d DMA frames\n",
-							 dev->name, count);
-				}
-			}
-#endif
 			break;
 		case ISQ_RX_MISS_EVENT:
 			dev->stats.rx_missed_errors += (status >> 6);
@@ -860,14 +607,6 @@ net_open(struct net_device *dev)
 			goto bad_out;
 		}
 	} else {
-#if IS_ENABLED(CONFIG_CS89x0_ISA)
-		if (((1 << dev->irq) & lp->irq_map) == 0) {
-			pr_err("%s: IRQ %d is not in our map of allowable IRQs, which is %x\n",
-			       dev->name, dev->irq, lp->irq_map);
-			ret = -EAGAIN;
-			goto bad_out;
-		}
-#endif
 /* FIXME: Cirrus' release had this: */
 		writereg(dev, PP_BusCTL, readreg(dev, PP_BusCTL)|ENABLE_IRQ);
 /* And 2.3.47 had this: */
@@ -882,46 +621,6 @@ net_open(struct net_device *dev)
 		}
 	}
 
-#if ALLOW_DMA
-	if (lp->use_dma && (lp->isa_config & ANY_ISA_DMA)) {
-		unsigned long flags;
-		lp->dma_buff = (unsigned char *)__get_dma_pages(GFP_KERNEL,
-								get_order(lp->dmasize * 1024));
-		if (!lp->dma_buff) {
-			pr_err("%s: cannot get %dK memory for DMA\n",
-			       dev->name, lp->dmasize);
-			goto release_irq;
-		}
-		cs89_dbg(1, debug, "%s: dma %lx %lx\n",
-			 dev->name,
-			 (unsigned long)lp->dma_buff,
-			 (unsigned long)isa_virt_to_bus(lp->dma_buff));
-		if ((unsigned long)lp->dma_buff >= MAX_DMA_ADDRESS ||
-		    !dma_page_eq(lp->dma_buff,
-				 lp->dma_buff + lp->dmasize * 1024 - 1)) {
-			pr_err("%s: not usable as DMA buffer\n", dev->name);
-			goto release_irq;
-		}
-		memset(lp->dma_buff, 0, lp->dmasize * 1024);	/* Why? */
-		if (request_dma(dev->dma, dev->name)) {
-			pr_err("%s: cannot get dma channel %d\n",
-			       dev->name, dev->dma);
-			goto release_irq;
-		}
-		write_dma(dev, lp->chip_type, dev->dma);
-		lp->rx_dma_ptr = lp->dma_buff;
-		lp->end_dma_buff = lp->dma_buff + lp->dmasize * 1024;
-		spin_lock_irqsave(&lp->lock, flags);
-		disable_dma(dev->dma);
-		clear_dma_ff(dev->dma);
-		set_dma_mode(dev->dma, DMA_RX_MODE); /* auto_init as well */
-		set_dma_addr(dev->dma, isa_virt_to_bus(lp->dma_buff));
-		set_dma_count(dev->dma, lp->dmasize * 1024);
-		enable_dma(dev->dma);
-		spin_unlock_irqrestore(&lp->lock, flags);
-	}
-#endif	/* ALLOW_DMA */
-
 	/* set the Ethernet address */
 	for (i = 0; i < ETH_ALEN / 2; i++)
 		writereg(dev, PP_IA + i * 2,
@@ -958,11 +657,6 @@ net_open(struct net_device *dev)
 		pr_err("%s: EEPROM is configured for unavailable media\n",
 		       dev->name);
 release_dma:
-#if ALLOW_DMA
-		free_dma(dev->dma);
-release_irq:
-		release_dma_buff(lp);
-#endif
 		writereg(dev, PP_LineCTL,
 			 readreg(dev, PP_LineCTL) & ~(SERIAL_TX_ON | SERIAL_RX_ON));
 		free_irq(dev->irq, dev);
@@ -1048,9 +742,6 @@ net_open(struct net_device *dev)
 
 	if (lp->isa_config & STREAM_TRANSFER)
 		lp->curr_rx_cfg |= RX_STREAM_ENBL;
-#if ALLOW_DMA
-	set_dma_cfg(dev);
-#endif
 	writereg(dev, PP_RxCFG, lp->curr_rx_cfg);
 
 	writereg(dev, PP_TxCFG, (TX_LOST_CRS_ENBL |
@@ -1063,19 +754,12 @@ net_open(struct net_device *dev)
 
 	writereg(dev, PP_BufCFG, (READY_FOR_TX_ENBL |
 				  RX_MISS_COUNT_OVRFLOW_ENBL |
-#if ALLOW_DMA
-				  dma_bufcfg(dev) |
-#endif
 				  TX_COL_COUNT_OVRFLOW_ENBL |
 				  TX_UNDERRUN_ENBL));
 
 	/* now that we've got our act together, enable everything */
-	writereg(dev, PP_BusCTL, (ENABLE_IRQ
-				  | (dev->mem_start ? MEMORY_ON : 0) /* turn memory on */
-#if ALLOW_DMA
-				  | dma_busctl(dev)
-#endif
-			 ));
+	writereg(dev, PP_BusCTL,
+		 (ENABLE_IRQ | (dev->mem_start ? MEMORY_ON : 0))); /* turn memory on */
 	netif_start_queue(dev);
 	cs89_dbg(1, debug, "net_open() succeeded\n");
 	return 0;
@@ -1087,10 +771,6 @@ net_open(struct net_device *dev)
 static int
 net_close(struct net_device *dev)
 {
-#if ALLOW_DMA
-	struct net_local *lp = netdev_priv(dev);
-#endif
-
 	netif_stop_queue(dev);
 
 	writereg(dev, PP_RxCFG, 0);
@@ -1100,13 +780,6 @@ net_close(struct net_device *dev)
 
 	free_irq(dev->irq, dev);
 
-#if ALLOW_DMA
-	if (lp->use_dma && lp->dma) {
-		free_dma(dev->dma);
-		release_dma_buff(lp);
-	}
-#endif
-
 	/* Update the statistics here. */
 	return 0;
 }
@@ -1321,13 +994,6 @@ cs89x0_probe1(struct net_device *dev, void __iomem *ioaddr, int modular)
 		memset(lp, 0, sizeof(*lp));
 		spin_lock_init(&lp->lock);
 #ifndef MODULE
-#if ALLOW_DMA
-		if (g_cs89x0_dma) {
-			lp->use_dma = 1;
-			lp->dma = g_cs89x0_dma;
-			lp->dmasize = 16;	/* Could make this an option... */
-		}
-#endif
 		lp->force = g_cs89x0_media__force;
 #endif
 	}
@@ -1525,40 +1191,12 @@ cs89x0_probe1(struct net_device *dev, void __iomem *ioaddr, int modular)
 			dev->irq = i;
 	} else {
 		i = lp->isa_config & INT_NO_MASK;
-#if IS_ENABLED(CONFIG_CS89x0_ISA)
-		if (lp->chip_type == CS8900) {
-			/* Translate the IRQ using the IRQ mapping table. */
-			if (i >= ARRAY_SIZE(cs8900_irq_map))
-				pr_err("invalid ISA interrupt number %d\n", i);
-			else
-				i = cs8900_irq_map[i];
-
-			lp->irq_map = CS8900_IRQ_MAP; /* fixed IRQ map for CS8900 */
-		} else {
-			int irq_map_buff[IRQ_MAP_LEN/2];
-
-			if (get_eeprom_data(dev, IRQ_MAP_EEPROM_DATA,
-					    IRQ_MAP_LEN / 2,
-					    irq_map_buff) >= 0) {
-				if ((irq_map_buff[0] & 0xff) == PNP_IRQ_FRMT)
-					lp->irq_map = ((irq_map_buff[0] >> 8) |
-						       (irq_map_buff[1] << 8));
-			}
-		}
-#endif
 		if (!dev->irq)
 			dev->irq = i;
 	}
 
 	pr_cont(" IRQ %d", dev->irq);
-
-#if ALLOW_DMA
-	if (lp->use_dma) {
-		get_dma_channel(dev);
-		pr_cont(", DMA %d", dev->dma);
-	} else
-#endif
-		pr_cont(", programmed I/O");
+	pr_cont(", programmed I/O");
 
 	/* print the ethernet address. */
 	pr_cont(", MAC %pM\n", dev->dev_addr);
@@ -1578,270 +1216,6 @@ cs89x0_probe1(struct net_device *dev, void __iomem *ioaddr, int modular)
 	return retval;
 }
 
-#if IS_ENABLED(CONFIG_CS89x0_ISA)
-/*
- * This function converts the I/O port address used by the cs89x0_probe() and
- * init_module() functions to the I/O memory address used by the
- * cs89x0_probe1() function.
- */
-static int __init
-cs89x0_ioport_probe(struct net_device *dev, unsigned long ioport, int modular)
-{
-	struct net_local *lp = netdev_priv(dev);
-	int ret;
-	void __iomem *io_mem;
-
-	if (!lp)
-		return -ENOMEM;
-
-	dev->base_addr = ioport;
-
-	if (!request_region(ioport, NETCARD_IO_EXTENT, DRV_NAME)) {
-		ret = -EBUSY;
-		goto out;
-	}
-
-	io_mem = ioport_map(ioport & ~3, NETCARD_IO_EXTENT);
-	if (!io_mem) {
-		ret = -ENOMEM;
-		goto release;
-	}
-
-	/* if they give us an odd I/O address, then do ONE write to
-	 * the address port, to get it back to address zero, where we
-	 * expect to find the EISA signature word. An IO with a base of 0x3
-	 * will skip the test for the ADD_PORT.
-	 */
-	if (ioport & 1) {
-		cs89_dbg(1, info, "%s: odd ioaddr 0x%lx\n", dev->name, ioport);
-		if ((ioport & 2) != 2) {
-			if ((ioread16(io_mem + ADD_PORT) & ADD_MASK) !=
-			    ADD_SIG) {
-				pr_err("%s: bad signature 0x%x\n",
-				       dev->name, ioread16(io_mem + ADD_PORT));
-				ret = -ENODEV;
-				goto unmap;
-			}
-		}
-	}
-
-	ret = cs89x0_probe1(dev, io_mem, modular);
-	if (!ret)
-		goto out;
-unmap:
-	ioport_unmap(io_mem);
-release:
-	release_region(ioport, NETCARD_IO_EXTENT);
-out:
-	return ret;
-}
-
-#ifndef MODULE
-/* Check for a network adaptor of this type, and return '0' iff one exists.
- * If dev->base_addr == 0, probe all likely locations.
- * If dev->base_addr == 1, always return failure.
- * If dev->base_addr == 2, allocate space for the device and return success
- * (detachable devices only).
- * Return 0 on success.
- */
-
-struct net_device * __init cs89x0_probe(int unit)
-{
-	struct net_device *dev = alloc_etherdev(sizeof(struct net_local));
-	unsigned *port;
-	int err = 0;
-	int irq;
-	int io;
-
-	if (!dev)
-		return ERR_PTR(-ENODEV);
-
-	sprintf(dev->name, "eth%d", unit);
-	netdev_boot_setup_check(dev);
-	io = dev->base_addr;
-	irq = dev->irq;
-
-	cs89_dbg(0, info, "cs89x0_probe(0x%x)\n", io);
-
-	if (io > 0x1ff)	{	/* Check a single specified location. */
-		err = cs89x0_ioport_probe(dev, io, 0);
-	} else if (io != 0) {	/* Don't probe at all. */
-		err = -ENXIO;
-	} else {
-		for (port = netcard_portlist; *port; port++) {
-			if (cs89x0_ioport_probe(dev, *port, 0) == 0)
-				break;
-			dev->irq = irq;
-		}
-		if (!*port)
-			err = -ENODEV;
-	}
-	if (err)
-		goto out;
-	return dev;
-out:
-	free_netdev(dev);
-	pr_warn("no cs8900 or cs8920 detected.  Be sure to disable PnP with SETUP\n");
-	return ERR_PTR(err);
-}
-#else
-static struct net_device *dev_cs89x0;
-
-/* Support the 'debug' module parm even if we're compiled for non-debug to
- * avoid breaking someone's startup scripts
- */
-
-static int io;
-static int irq;
-static int debug;
-static char media[8];
-static int duplex = -1;
-
-static int use_dma;			/* These generate unused var warnings if ALLOW_DMA = 0 */
-static int dma;
-static int dmasize = 16;		/* or 64 */
-
-module_param_hw(io, int, ioport, 0);
-module_param_hw(irq, int, irq, 0);
-module_param(debug, int, 0);
-module_param_string(media, media, sizeof(media), 0);
-module_param(duplex, int, 0);
-module_param_hw(dma , int, dma, 0);
-module_param(dmasize , int, 0);
-module_param(use_dma , int, 0);
-MODULE_PARM_DESC(io, "cs89x0 I/O base address");
-MODULE_PARM_DESC(irq, "cs89x0 IRQ number");
-#if DEBUGGING
-MODULE_PARM_DESC(debug, "cs89x0 debug level (0-6)");
-#else
-MODULE_PARM_DESC(debug, "(ignored)");
-#endif
-MODULE_PARM_DESC(media, "Set cs89x0 adapter(s) media type(s) (rj45,bnc,aui)");
-/* No other value than -1 for duplex seems to be currently interpreted */
-MODULE_PARM_DESC(duplex, "(ignored)");
-#if ALLOW_DMA
-MODULE_PARM_DESC(dma , "cs89x0 ISA DMA channel; ignored if use_dma=0");
-MODULE_PARM_DESC(dmasize , "cs89x0 DMA size in kB (16,64); ignored if use_dma=0");
-MODULE_PARM_DESC(use_dma , "cs89x0 using DMA (0-1)");
-#else
-MODULE_PARM_DESC(dma , "(ignored)");
-MODULE_PARM_DESC(dmasize , "(ignored)");
-MODULE_PARM_DESC(use_dma , "(ignored)");
-#endif
-
-MODULE_AUTHOR("Mike Cruse, Russwll Nelson <nelson@crynwr.com>, Andrew Morton");
-MODULE_LICENSE("GPL");
-
-/*
- * media=t             - specify media type
- * or media=2
- * or media=aui
- * or medai=auto
- * duplex=0            - specify forced half/full/autonegotiate duplex
- * debug=#             - debug level
- *
- * Default Chip Configuration:
- * DMA Burst = enabled
- * IOCHRDY Enabled = enabled
- * UseSA = enabled
- * CS8900 defaults to half-duplex if not specified on command-line
- * CS8920 defaults to autoneg if not specified on command-line
- * Use reset defaults for other config parameters
- *
- * Assumptions:
- * media type specified is supported (circuitry is present)
- * if memory address is > 1MB, then required mem decode hw is present
- * if 10B-2, then agent other than driver will enable DC/DC converter
- * (hw or software util)
- */
-
-static int __init cs89x0_isa_init_module(void)
-{
-	struct net_device *dev;
-	struct net_local *lp;
-	int ret = 0;
-
-#if DEBUGGING
-	net_debug = debug;
-#else
-	debug = 0;
-#endif
-	dev = alloc_etherdev(sizeof(struct net_local));
-	if (!dev)
-		return -ENOMEM;
-
-	dev->irq = irq;
-	dev->base_addr = io;
-	lp = netdev_priv(dev);
-
-#if ALLOW_DMA
-	if (use_dma) {
-		lp->use_dma = use_dma;
-		lp->dma = dma;
-		lp->dmasize = dmasize;
-	}
-#endif
-
-	spin_lock_init(&lp->lock);
-
-	/* boy, they'd better get these right */
-	if (!strcmp(media, "rj45"))
-		lp->adapter_cnf = A_CNF_MEDIA_10B_T | A_CNF_10B_T;
-	else if (!strcmp(media, "aui"))
-		lp->adapter_cnf = A_CNF_MEDIA_AUI   | A_CNF_AUI;
-	else if (!strcmp(media, "bnc"))
-		lp->adapter_cnf = A_CNF_MEDIA_10B_2 | A_CNF_10B_2;
-	else
-		lp->adapter_cnf = A_CNF_MEDIA_10B_T | A_CNF_10B_T;
-
-	if (duplex == -1)
-		lp->auto_neg_cnf = AUTO_NEG_ENABLE;
-
-	if (io == 0) {
-		pr_err("Module autoprobing not allowed\n");
-		pr_err("Append io=0xNNN\n");
-		ret = -EPERM;
-		goto out;
-	} else if (io <= 0x1ff) {
-		ret = -ENXIO;
-		goto out;
-	}
-
-#if ALLOW_DMA
-	if (use_dma && dmasize != 16 && dmasize != 64) {
-		pr_err("dma size must be either 16K or 64K, not %dK\n",
-		       dmasize);
-		ret = -EPERM;
-		goto out;
-	}
-#endif
-	ret = cs89x0_ioport_probe(dev, io, 1);
-	if (ret)
-		goto out;
-
-	dev_cs89x0 = dev;
-	return 0;
-out:
-	free_netdev(dev);
-	return ret;
-}
-module_init(cs89x0_isa_init_module);
-
-static void __exit cs89x0_isa_cleanup_module(void)
-{
-	struct net_local *lp = netdev_priv(dev_cs89x0);
-
-	unregister_netdev(dev_cs89x0);
-	iowrite16(PP_ChipID, lp->virt_addr + ADD_PORT);
-	ioport_unmap(lp->virt_addr);
-	release_region(dev_cs89x0->base_addr, NETCARD_IO_EXTENT);
-	free_netdev(dev_cs89x0);
-}
-module_exit(cs89x0_isa_cleanup_module);
-#endif /* MODULE */
-#endif /* CONFIG_CS89x0_ISA */
-
-#if IS_ENABLED(CONFIG_CS89x0_PLATFORM)
 static int __init cs89x0_platform_probe(struct platform_device *pdev)
 {
 	struct net_device *dev = alloc_etherdev(sizeof(struct net_local));
@@ -1906,8 +1280,6 @@ static struct platform_driver cs89x0_driver = {
 
 module_platform_driver_probe(cs89x0_driver, cs89x0_platform_probe);
 
-#endif /* CONFIG_CS89x0_PLATFORM */
-
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Crystal Semiconductor (Now Cirrus Logic) CS89[02]0 network driver");
 MODULE_AUTHOR("Russell Nelson <nelson@crynwr.com>");
diff --git a/drivers/net/ethernet/dec/tulip/de2104x.c b/drivers/net/ethernet/dec/tulip/de2104x.c
index f9504f340c4ae..f38a10c79907f 100644
--- a/drivers/net/ethernet/dec/tulip/de2104x.c
+++ b/drivers/net/ethernet/dec/tulip/de2104x.c
@@ -333,10 +333,8 @@ static unsigned int de_ok_to_advertise (struct de_private *de, u32 new_media);
 
 
 static const struct pci_device_id de_pci_tbl[] = {
-	{ PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_TULIP,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_TULIP_PLUS,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 1 },
+	{ PCI_VDEVICE(DEC, PCI_DEVICE_ID_DEC_TULIP), .driver_data = 0 },
+	{ PCI_VDEVICE(DEC, PCI_DEVICE_ID_DEC_TULIP_PLUS), .driver_data = 1 },
 	{ },
 };
 MODULE_DEVICE_TABLE(pci, de_pci_tbl);
diff --git a/drivers/net/ethernet/dec/tulip/dmfe.c b/drivers/net/ethernet/dec/tulip/dmfe.c
index 2d3bd343b6e63..dabc091a0c66e 100644
--- a/drivers/net/ethernet/dec/tulip/dmfe.c
+++ b/drivers/net/ethernet/dec/tulip/dmfe.c
@@ -2076,11 +2076,11 @@ static void dmfe_HPNA_remote_cmd_chk(struct dmfe_board_info * db)
 
 
 static const struct pci_device_id dmfe_pci_tbl[] = {
-	{ 0x1282, 0x9132, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PCI_DM9132_ID },
-	{ 0x1282, 0x9102, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PCI_DM9102_ID },
-	{ 0x1282, 0x9100, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PCI_DM9100_ID },
-	{ 0x1282, 0x9009, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PCI_DM9009_ID },
-	{ 0, }
+	{ PCI_DEVICE(0x1282, 0x9132), .driver_data = PCI_DM9132_ID },
+	{ PCI_DEVICE(0x1282, 0x9102), .driver_data = PCI_DM9102_ID },
+	{ PCI_DEVICE(0x1282, 0x9100), .driver_data = PCI_DM9100_ID },
+	{ PCI_DEVICE(0x1282, 0x9009), .driver_data = PCI_DM9009_ID },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, dmfe_pci_tbl);
 
@@ -2101,7 +2101,7 @@ static int __maybe_unused dmfe_suspend(struct device *dev_d)
 	dw32(DCR7, 0);
 	dw32(DCR5, dr32(DCR5));
 
-	/* Fre RX buffers */
+	/* Free RX buffers */
 	dmfe_free_rxbuffer(db);
 
 	/* Enable WOL */
diff --git a/drivers/net/ethernet/dec/tulip/tulip_core.c b/drivers/net/ethernet/dec/tulip/tulip_core.c
index b608585f19549..a7200eb66594d 100644
--- a/drivers/net/ethernet/dec/tulip/tulip_core.c
+++ b/drivers/net/ethernet/dec/tulip/tulip_core.c
@@ -196,45 +196,45 @@ const struct tulip_chip_table tulip_tbl[] = {
 
 
 static const struct pci_device_id tulip_pci_tbl[] = {
-	{ 0x1011, 0x0009, PCI_ANY_ID, PCI_ANY_ID, 0, 0, DC21140 },
-	{ 0x1011, 0x0019, PCI_ANY_ID, PCI_ANY_ID, 0, 0, DC21143 },
-	{ 0x11AD, 0x0002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, LC82C168 },
-	{ 0x10d9, 0x0512, PCI_ANY_ID, PCI_ANY_ID, 0, 0, MX98713 },
-	{ 0x10d9, 0x0531, PCI_ANY_ID, PCI_ANY_ID, 0, 0, MX98715 },
-/*	{ 0x10d9, 0x0531, PCI_ANY_ID, PCI_ANY_ID, 0, 0, MX98725 },*/
-	{ 0x125B, 0x1400, PCI_ANY_ID, PCI_ANY_ID, 0, 0, AX88140 },
-	{ 0x11AD, 0xc115, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PNIC2 },
-	{ 0x1317, 0x0981, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1317, 0x0985, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1317, 0x1985, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1317, 0x9511, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x13D1, 0xAB02, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x13D1, 0xAB03, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x13D1, 0xAB08, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x104A, 0x0981, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x104A, 0x2774, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1259, 0xa120, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x11F6, 0x9881, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMPEX9881 },
-	{ 0x8086, 0x0039, PCI_ANY_ID, PCI_ANY_ID, 0, 0, I21145 },
+	{ PCI_DEVICE(0x1011, 0x0009), .driver_data = DC21140 },
+	{ PCI_DEVICE(0x1011, 0x0019), .driver_data = DC21143 },
+	{ PCI_DEVICE(0x11AD, 0x0002), .driver_data = LC82C168 },
+	{ PCI_DEVICE(0x10d9, 0x0512), .driver_data = MX98713 },
+	{ PCI_DEVICE(0x10d9, 0x0531), .driver_data = MX98715 },
+/*	{ PCI_DEVICE(0x10d9, 0x0531), .driver_data = MX98725 },*/
+	{ PCI_DEVICE(0x125B, 0x1400), .driver_data = AX88140 },
+	{ PCI_DEVICE(0x11AD, 0xc115), .driver_data = PNIC2 },
+	{ PCI_DEVICE(0x1317, 0x0981), .driver_data = COMET },
+	{ PCI_DEVICE(0x1317, 0x0985), .driver_data = COMET },
+	{ PCI_DEVICE(0x1317, 0x1985), .driver_data = COMET },
+	{ PCI_DEVICE(0x1317, 0x9511), .driver_data = COMET },
+	{ PCI_DEVICE(0x13D1, 0xAB02), .driver_data = COMET },
+	{ PCI_DEVICE(0x13D1, 0xAB03), .driver_data = COMET },
+	{ PCI_DEVICE(0x13D1, 0xAB08), .driver_data = COMET },
+	{ PCI_DEVICE(0x104A, 0x0981), .driver_data = COMET },
+	{ PCI_DEVICE(0x104A, 0x2774), .driver_data = COMET },
+	{ PCI_DEVICE(0x1259, 0xa120), .driver_data = COMET },
+	{ PCI_DEVICE(0x11F6, 0x9881), .driver_data = COMPEX9881 },
+	{ PCI_DEVICE(0x8086, 0x0039), .driver_data = I21145 },
 #ifdef CONFIG_TULIP_DM910X
-	{ 0x1282, 0x9100, PCI_ANY_ID, PCI_ANY_ID, 0, 0, DM910X },
-	{ 0x1282, 0x9102, PCI_ANY_ID, PCI_ANY_ID, 0, 0, DM910X },
+	{ PCI_DEVICE(0x1282, 0x9100), .driver_data = DM910X },
+	{ PCI_DEVICE(0x1282, 0x9102), .driver_data = DM910X },
 #endif
-	{ 0x1113, 0x1216, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1113, 0x1217, PCI_ANY_ID, PCI_ANY_ID, 0, 0, MX98715 },
-	{ 0x1113, 0x9511, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1186, 0x1541, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1186, 0x1561, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1186, 0x1591, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x14f1, 0x1803, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CONEXANT },
-	{ 0x1626, 0x8410, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1737, 0xAB09, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x1737, 0xAB08, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x17B3, 0xAB08, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
-	{ 0x10b7, 0x9300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET }, /* 3Com 3CSOHO100B-TX */
-	{ 0x14ea, 0xab08, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET }, /* Planex FNW-3602-TX */
-	{ 0x1414, 0x0001, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET }, /* Microsoft MN-120 */
-	{ 0x1414, 0x0002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, COMET },
+	{ PCI_DEVICE(0x1113, 0x1216), .driver_data = COMET },
+	{ PCI_DEVICE(0x1113, 0x1217), .driver_data = MX98715 },
+	{ PCI_DEVICE(0x1113, 0x9511), .driver_data = COMET },
+	{ PCI_DEVICE(0x1186, 0x1541), .driver_data = COMET },
+	{ PCI_DEVICE(0x1186, 0x1561), .driver_data = COMET },
+	{ PCI_DEVICE(0x1186, 0x1591), .driver_data = COMET },
+	{ PCI_DEVICE(0x14f1, 0x1803), .driver_data = CONEXANT },
+	{ PCI_DEVICE(0x1626, 0x8410), .driver_data = COMET },
+	{ PCI_DEVICE(0x1737, 0xAB09), .driver_data = COMET },
+	{ PCI_DEVICE(0x1737, 0xAB08), .driver_data = COMET },
+	{ PCI_DEVICE(0x17B3, 0xAB08), .driver_data = COMET },
+	{ PCI_DEVICE(0x10b7, 0x9300), .driver_data = COMET }, /* 3Com 3CSOHO100B-TX */
+	{ PCI_DEVICE(0x14ea, 0xab08), .driver_data = COMET }, /* Planex FNW-3602-TX */
+	{ PCI_DEVICE(0x1414, 0x0001), .driver_data = COMET }, /* Microsoft MN-120 */
+	{ PCI_DEVICE(0x1414, 0x0002), .driver_data = COMET },
 	{ } /* terminate list */
 };
 MODULE_DEVICE_TABLE(pci, tulip_pci_tbl);
@@ -1279,8 +1279,8 @@ static const struct net_device_ops tulip_netdev_ops = {
 };
 
 static const struct pci_device_id early_486_chipsets[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82424) },
-	{ PCI_DEVICE(PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_496) },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_82424) },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_496) },
 	{ },
 };
 
diff --git a/drivers/net/ethernet/dec/tulip/uli526x.c b/drivers/net/ethernet/dec/tulip/uli526x.c
index 6e4d8d31aba90..34b234b6b1590 100644
--- a/drivers/net/ethernet/dec/tulip/uli526x.c
+++ b/drivers/net/ethernet/dec/tulip/uli526x.c
@@ -1728,9 +1728,9 @@ static u16 phy_read_1bit(struct uli526x_board_info *db)
 
 
 static const struct pci_device_id uli526x_pci_tbl[] = {
-	{ 0x10B9, 0x5261, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PCI_ULI5261_ID },
-	{ 0x10B9, 0x5263, PCI_ANY_ID, PCI_ANY_ID, 0, 0, PCI_ULI5263_ID },
-	{ 0, }
+	{ PCI_DEVICE(0x10B9, 0x5261), .driver_data = PCI_ULI5261_ID },
+	{ PCI_DEVICE(0x10B9, 0x5263), .driver_data = PCI_ULI5263_ID },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, uli526x_pci_tbl);
 
diff --git a/drivers/net/ethernet/dec/tulip/winbond-840.c b/drivers/net/ethernet/dec/tulip/winbond-840.c
index a24a25a5f73d5..a5581f1d01417 100644
--- a/drivers/net/ethernet/dec/tulip/winbond-840.c
+++ b/drivers/net/ethernet/dec/tulip/winbond-840.c
@@ -210,9 +210,16 @@ enum chip_capability_flags {
 };
 
 static const struct pci_device_id w840_pci_tbl[] = {
-	{ 0x1050, 0x0840, PCI_ANY_ID, 0x8153,     0, 0, 0 },
-	{ 0x1050, 0x0840, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 1 },
-	{ 0x11f6, 0x2011, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 2 },
+	{
+		PCI_DEVICE_SUB(0x1050, 0x0840, PCI_ANY_ID, 0x8153),
+		.driver_data = 0,
+	}, {
+		PCI_DEVICE(0x1050, 0x0840),
+		.driver_data = 1,
+	}, {
+		PCI_DEVICE(0x11f6, 0x2011),
+		.driver_data = 2,
+	},
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, w840_pci_tbl);
diff --git a/drivers/net/ethernet/dlink/dl2k.h b/drivers/net/ethernet/dlink/dl2k.h
index 9ebf7a6db93e4..00ef7d03b4d69 100644
--- a/drivers/net/ethernet/dlink/dl2k.h
+++ b/drivers/net/ethernet/dlink/dl2k.h
@@ -418,12 +418,12 @@ struct netdev_private {
 #define CHIP_IP1000A	1
 
 static const struct pci_device_id rio_pci_tbl[] = {
-	{0x1186, 0x4000, PCI_ANY_ID, PCI_ANY_ID, },
-	{0x13f0, 0x1021, PCI_ANY_ID, PCI_ANY_ID, },
-	{ PCI_VDEVICE(SUNDANCE,	0x1023), CHIP_IP1000A },
-	{ PCI_VDEVICE(SUNDANCE,	0x2021), CHIP_IP1000A },
-	{ PCI_VDEVICE(DLINK,	0x9021), CHIP_IP1000A },
-	{ PCI_VDEVICE(DLINK,	0x4020), CHIP_IP1000A },
+	{ PCI_DEVICE(0x1186, 0x4000) },
+	{ PCI_DEVICE(0x13f0, 0x1021) },
+	{ PCI_VDEVICE(SUNDANCE,	0x1023), .driver_data = CHIP_IP1000A },
+	{ PCI_VDEVICE(SUNDANCE,	0x2021), .driver_data = CHIP_IP1000A },
+	{ PCI_VDEVICE(DLINK,	0x9021), .driver_data = CHIP_IP1000A },
+	{ PCI_VDEVICE(DLINK,	0x4020), .driver_data = CHIP_IP1000A },
 	{ }
 };
 MODULE_DEVICE_TABLE (pci, rio_pci_tbl);
diff --git a/drivers/net/ethernet/dlink/sundance.c b/drivers/net/ethernet/dlink/sundance.c
index 277c50ef773fb..58873fcd3eafd 100644
--- a/drivers/net/ethernet/dlink/sundance.c
+++ b/drivers/net/ethernet/dlink/sundance.c
@@ -192,13 +192,13 @@ IVc. Errata
 #endif
 
 static const struct pci_device_id sundance_pci_tbl[] = {
-	{ 0x1186, 0x1002, 0x1186, 0x1002, 0, 0, 0 },
-	{ 0x1186, 0x1002, 0x1186, 0x1003, 0, 0, 1 },
-	{ 0x1186, 0x1002, 0x1186, 0x1012, 0, 0, 2 },
-	{ 0x1186, 0x1002, 0x1186, 0x1040, 0, 0, 3 },
-	{ 0x1186, 0x1002, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 4 },
-	{ 0x13F0, 0x0201, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 5 },
-	{ 0x13F0, 0x0200, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 6 },
+	{ PCI_DEVICE_SUB(0x1186, 0x1002, 0x1186, 0x1002), .driver_data = 0 },
+	{ PCI_DEVICE_SUB(0x1186, 0x1002, 0x1186, 0x1003), .driver_data = 1 },
+	{ PCI_DEVICE_SUB(0x1186, 0x1002, 0x1186, 0x1012), .driver_data = 2 },
+	{ PCI_DEVICE_SUB(0x1186, 0x1002, 0x1186, 0x1040), .driver_data = 3 },
+	{ PCI_DEVICE(0x1186, 0x1002), .driver_data = 4 },
+	{ PCI_DEVICE(0x13F0, 0x0201), .driver_data = 5 },
+	{ PCI_DEVICE(0x13F0, 0x0200), .driver_data = 6 },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, sundance_pci_tbl);
diff --git a/drivers/net/ethernet/fealnx.c b/drivers/net/ethernet/fealnx.c
index 3c9961806f756..bdc38aac58509 100644
--- a/drivers/net/ethernet/fealnx.c
+++ b/drivers/net/ethernet/fealnx.c
@@ -1935,10 +1935,10 @@ static int netdev_close(struct net_device *dev)
 }
 
 static const struct pci_device_id fealnx_pci_tbl[] = {
-	{0x1516, 0x0800, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{0x1516, 0x0803, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 1},
-	{0x1516, 0x0891, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 2},
-	{} /* terminate list */
+	{ PCI_DEVICE(0x1516, 0x0800), .driver_data = 0 },
+	{ PCI_DEVICE(0x1516, 0x0803), .driver_data = 1 },
+	{ PCI_DEVICE(0x1516, 0x0891), .driver_data = 2 },
+	{ } /* terminate list */
 };
 MODULE_DEVICE_TABLE(pci, fealnx_pci_tbl);
 
diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c
index 424d973c97f24..e4d78ae52dafe 100644
--- a/drivers/net/ethernet/google/gve/gve_main.c
+++ b/drivers/net/ethernet/google/gve/gve_main.c
@@ -2894,7 +2894,7 @@ static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 		goto abort_with_wq;
 
 	if (!gve_is_gqi(priv) && !gve_is_qpl(priv))
-		dev->netmem_tx = true;
+		dev->netmem_tx = NETMEM_TX_DMA;
 
 	err = register_netdev(dev);
 	if (err)
@@ -2967,9 +2967,9 @@ static void gve_shutdown(struct pci_dev *pdev)
 	rtnl_unlock();
 }
 
-#ifdef CONFIG_PM
-static int gve_suspend(struct pci_dev *pdev, pm_message_t state)
+static int gve_suspend(struct device *dev)
 {
+	struct pci_dev *pdev = to_pci_dev(dev);
 	struct net_device *netdev = pci_get_drvdata(pdev);
 	struct gve_priv *priv = netdev_priv(netdev);
 	bool was_up = netif_running(priv->dev);
@@ -2990,8 +2990,9 @@ static int gve_suspend(struct pci_dev *pdev, pm_message_t state)
 	return 0;
 }
 
-static int gve_resume(struct pci_dev *pdev)
+static int gve_resume(struct device *dev)
 {
+	struct pci_dev *pdev = to_pci_dev(dev);
 	struct net_device *netdev = pci_get_drvdata(pdev);
 	struct gve_priv *priv = netdev_priv(netdev);
 	int err;
@@ -3004,7 +3005,8 @@ static int gve_resume(struct pci_dev *pdev)
 	rtnl_unlock();
 	return err;
 }
-#endif /* CONFIG_PM */
+
+static DEFINE_SIMPLE_DEV_PM_OPS(gve_pm_ops, gve_suspend, gve_resume);
 
 static const struct pci_device_id gve_id_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_GOOGLE, PCI_DEV_ID_GVNIC) },
@@ -3017,10 +3019,7 @@ static struct pci_driver gve_driver = {
 	.probe		= gve_probe,
 	.remove		= gve_remove,
 	.shutdown	= gve_shutdown,
-#ifdef CONFIG_PM
-	.suspend        = gve_suspend,
-	.resume         = gve_resume,
-#endif
+	.driver.pm	= pm_sleep_ptr(&gve_pm_ops),
 };
 
 module_pci_driver(gve_driver);
diff --git a/drivers/net/ethernet/hisilicon/hibmcge/hbg_main.c b/drivers/net/ethernet/hisilicon/hibmcge/hbg_main.c
index 068da2fd1fea8..b3e01b2f83199 100644
--- a/drivers/net/ethernet/hisilicon/hibmcge/hbg_main.c
+++ b/drivers/net/ethernet/hisilicon/hibmcge/hbg_main.c
@@ -489,7 +489,7 @@ static void hbg_shutdown(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id hbg_pci_tbl[] = {
-	{PCI_VDEVICE(HUAWEI, 0x3730), 0},
+	{ PCI_VDEVICE(HUAWEI, 0x3730) },
 	{ }
 };
 MODULE_DEVICE_TABLE(pci, hbg_pci_tbl);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
index a3206c97923e7..4c34a144d21c8 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c
@@ -86,25 +86,39 @@ module_param(page_pool_enabled, bool, 0400);
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id hns3_pci_tbl[] = {
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
-	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
-	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
-	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
-	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
-	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA),
-	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
-	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
+	{
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA),
+		.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC),
+		.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA),
+		.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC),
+		.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC),
+		.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA),
+		.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
+		.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
+	},
 	/* required last entry */
-	{0,}
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, hns3_pci_tbl);
 
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
index dd4045c773d43..e17b92a411a22 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c
@@ -72,16 +72,16 @@ static struct hnae3_ae_algo ae_algo;
 static struct workqueue_struct *hclge_wq;
 
 static const struct pci_device_id ae_algo_pci_tbl[] = {
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_GE) },
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE) },
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA) },
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_25GE_RDMA_MACSEC) },
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA) },
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_50GE_RDMA_MACSEC) },
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_RDMA_MACSEC) },
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA) },
 	/* required last entry */
-	{0, }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, ae_algo_pci_tbl);
diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
index 0cc911e6c732d..063a692c2b41e 100644
--- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
+++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c
@@ -26,11 +26,15 @@ static struct hnae3_ae_algo ae_algovf;
 static struct workqueue_struct *hclgevf_wq;
 
 static const struct pci_device_id ae_algovf_pci_tbl[] = {
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF), 0},
-	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
-	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
+	{
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_VF),
+		.driver_data = 0,
+	}, {
+		PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
+		.driver_data = HNAE3_DEV_SUPPORT_ROCE_DCB_BITS,
+	},
 	/* required last entry */
-	{0, }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, ae_algovf_pci_tbl);
diff --git a/drivers/net/ethernet/huawei/hinic/hinic_main.c b/drivers/net/ethernet/huawei/hinic/hinic_main.c
index ae1f523d6841b..42f4792d255ba 100644
--- a/drivers/net/ethernet/huawei/hinic/hinic_main.c
+++ b/drivers/net/ethernet/huawei/hinic/hinic_main.c
@@ -1467,12 +1467,12 @@ static void hinic_shutdown(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id hinic_pci_table[] = {
-	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_QUAD_PORT_25GE), 0},
-	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_DUAL_PORT_100GE), 0},
-	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_DUAL_PORT_100GE_MEZZ), 0},
-	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_QUAD_PORT_25GE_MEZZ), 0},
-	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_VF), 0},
-	{ 0, 0}
+	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_QUAD_PORT_25GE) },
+	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_DUAL_PORT_100GE) },
+	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_DUAL_PORT_100GE_MEZZ) },
+	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_QUAD_PORT_25GE_MEZZ) },
+	{ PCI_VDEVICE(HUAWEI, HINIC_DEV_ID_VF) },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, hinic_pci_table);
 
diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c b/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c
index f83d5bd1041e3..36df807cbe095 100644
--- a/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c
+++ b/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c
@@ -426,10 +426,9 @@ static void hinic3_remove(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id hinic3_pci_table[] = {
-	{PCI_VDEVICE(HUAWEI, PCI_DEV_ID_HINIC3_PF), 0},
-	{PCI_VDEVICE(HUAWEI, PCI_DEV_ID_HINIC3_VF), 0},
-	{0, 0}
-
+	{ PCI_VDEVICE(HUAWEI, PCI_DEV_ID_HINIC3_PF) },
+	{ PCI_VDEVICE(HUAWEI, PCI_DEV_ID_HINIC3_VF) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, hinic3_pci_table);
diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c
index 9074b558de353..29960762e64a8 100644
--- a/drivers/net/ethernet/intel/e100.c
+++ b/drivers/net/ethernet/intel/e100.c
@@ -176,9 +176,12 @@ MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 MODULE_PARM_DESC(eeprom_bad_csum_allow, "Allow bad eeprom checksums");
 MODULE_PARM_DESC(use_io, "Force use of i/o access mode");
 
-#define INTEL_8255X_ETHERNET_DEVICE(device_id, ich) {\
-	PCI_VENDOR_ID_INTEL, device_id, PCI_ANY_ID, PCI_ANY_ID, \
-	PCI_CLASS_NETWORK_ETHERNET << 8, 0xFFFF00, ich }
+#define INTEL_8255X_ETHERNET_DEVICE(device_id, ich) { \
+	PCI_DEVICE(PCI_VENDOR_ID_INTEL, (device_id)), \
+	.class = PCI_CLASS_NETWORK_ETHERNET << 8, \
+	.class_mask = 0xFFFF00, \
+	.driver_data = (ich) }
+
 static const struct pci_device_id e100_id_table[] = {
 	INTEL_8255X_ETHERNET_DEVICE(0x1029, 0),
 	INTEL_8255X_ETHERNET_DEVICE(0x1030, 0),
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c
index 7ce0cc8ab8f4c..5b7ba1594f0d5 100644
--- a/drivers/net/ethernet/intel/e1000e/netdev.c
+++ b/drivers/net/ethernet/intel/e1000e/netdev.c
@@ -7799,137 +7799,370 @@ static const struct pci_error_handlers e1000_err_handler = {
 };
 
 static const struct pci_device_id e1000_pci_tbl[] = {
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER), board_82571 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER), board_82571 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER), board_82571 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP),
-	  board_82571 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER), board_82571 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES), board_82571 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL), board_82571 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD), board_82571 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER), board_82571 },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI), board_82572 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER), board_82572 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER), board_82572 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES), board_82572 },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E), board_82573 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT), board_82573 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L), board_82573 },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L), board_82574 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA), board_82574 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V), board_82583 },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT),
-	  board_80003es2lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT),
-	  board_80003es2lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT),
-	  board_80003es2lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT),
-	  board_80003es2lan },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE), board_ich8lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G), board_ich8lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT), board_ich8lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT), board_ich8lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C), board_ich8lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M), board_ich8lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT), board_ich8lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3), board_ich8lan },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V), board_ich9lan },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF), board_ich9lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V), board_ich9lan },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM), board_ich10lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF), board_ich10lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V), board_ich10lan },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM), board_pchlan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC), board_pchlan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM), board_pchlan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC), board_pchlan },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM), board_pch2lan },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V), board_pch2lan },
-
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM), board_pch_lpt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V), board_pch_lpt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM), board_pch_lpt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V), board_pch_lpt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM2), board_pch_lpt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V2), board_pch_lpt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM3), board_pch_lpt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V3), board_pch_lpt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM2), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V2), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LBG_I219_LM3), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM4), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V4), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM5), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V5), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM6), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V6), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM7), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V7), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM8), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V8), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM9), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V9), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM10), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V10), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM11), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V11), board_pch_cnp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM12), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V12), board_pch_spt },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM13), board_pch_tgp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V13), board_pch_tgp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM14), board_pch_tgp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V14), board_pch_tgp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM15), board_pch_tgp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V15), board_pch_tgp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM23), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V23), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM16), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM22), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V22), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM19), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V19), board_pch_adp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18), board_pch_mtp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18), board_pch_mtp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM20), board_pch_mtp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V20), board_pch_mtp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM21), board_pch_mtp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V21), board_pch_mtp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ARL_I219_LM24), board_pch_mtp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ARL_I219_V24), board_pch_mtp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_LM25), board_pch_ptp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_V25), board_pch_ptp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_LM27), board_pch_ptp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_V27), board_pch_ptp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_NVL_I219_LM29), board_pch_ptp },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_NVL_I219_V29), board_pch_ptp },
-
-	{ 0, 0, 0, 0, 0, 0, 0 }	/* terminate list */
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_COPPER),
+		.driver_data = board_82571,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_FIBER),
+		.driver_data = board_82571,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER),
+		.driver_data = board_82571,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_COPPER_LP),
+		.driver_data = board_82571,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_QUAD_FIBER),
+		.driver_data = board_82571,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES),
+		.driver_data = board_82571,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_DUAL),
+		.driver_data = board_82571,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82571EB_SERDES_QUAD),
+		.driver_data = board_82571,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82571PT_QUAD_COPPER),
+		.driver_data = board_82571,
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI),
+		.driver_data = board_82572,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_COPPER),
+		.driver_data = board_82572,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_FIBER),
+		.driver_data = board_82572,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82572EI_SERDES),
+		.driver_data = board_82572,
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E),
+		.driver_data = board_82573,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82573E_IAMT),
+		.driver_data = board_82573,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82573L),
+		.driver_data = board_82573,
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82574L),
+		.driver_data = board_82574,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82574LA),
+		.driver_data = board_82574,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_82583V),
+		.driver_data = board_82583,
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_DPT),
+		.driver_data = board_80003es2lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_COPPER_SPT),
+		.driver_data = board_80003es2lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_DPT),
+		.driver_data = board_80003es2lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_80003ES2LAN_SERDES_SPT),
+		.driver_data = board_80003es2lan,
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE),
+		.driver_data = board_ich8lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_G),
+		.driver_data = board_ich8lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IFE_GT),
+		.driver_data = board_ich8lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_AMT),
+		.driver_data = board_ich8lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_C),
+		.driver_data = board_ich8lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M),
+		.driver_data = board_ich8lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_IGP_M_AMT),
+		.driver_data = board_ich8lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH8_82567V_3),
+		.driver_data = board_ich8lan,
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_G),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IFE_GT),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_AMT),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_C),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_BM),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_AMT),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH9_IGP_M_V),
+		.driver_data = board_ich9lan
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LM),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_LF),
+		.driver_data = board_ich9lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_R_BM_V),
+		.driver_data = board_ich9lan,
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LM),
+		.driver_data = board_ich10lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_LF),
+		.driver_data = board_ich10lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_ICH10_D_BM_V),
+		.driver_data = board_ich10lan,
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LM),
+		.driver_data = board_pchlan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_M_HV_LC),
+		.driver_data = board_pchlan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DM),
+		.driver_data = board_pchlan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_D_HV_DC),
+		.driver_data = board_pchlan
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_LM),
+		.driver_data = board_pch2lan,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH2_LV_V),
+		.driver_data = board_pch2lan
+	},
+
+	{
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_LM),
+		.driver_data = board_pch_lpt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPT_I217_V),
+		.driver_data = board_pch_lpt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_LM),
+		.driver_data = board_pch_lpt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LPTLP_I218_V),
+		.driver_data = board_pch_lpt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM2),
+		.driver_data = board_pch_lpt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V2),
+		.driver_data = board_pch_lpt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_LM3),
+		.driver_data = board_pch_lpt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_I218_V3),
+		.driver_data = board_pch_lpt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM2),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V2),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LBG_I219_LM3),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM4),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V4),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_LM5),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_SPT_I219_V5),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM6),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V6),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_LM7),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CNP_I219_V7),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM8),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V8),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_LM9),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ICP_I219_V9),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM10),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V10),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM11),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V11),
+		.driver_data = board_pch_cnp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_LM12),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_CMP_I219_V12),
+		.driver_data = board_pch_spt,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM13),
+		.driver_data = board_pch_tgp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V13),
+		.driver_data = board_pch_tgp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM14),
+		.driver_data = board_pch_tgp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V14),
+		.driver_data = board_pch_tgp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_LM15),
+		.driver_data = board_pch_tgp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_TGP_I219_V15),
+		.driver_data = board_pch_tgp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM23),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V23),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM16),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V16),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM17),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V17),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_LM22),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_RPL_I219_V22),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_LM19),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ADP_I219_V19),
+		.driver_data = board_pch_adp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_LM18),
+		.driver_data = board_pch_mtp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_MTP_I219_V18),
+		.driver_data = board_pch_mtp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM20),
+		.driver_data = board_pch_mtp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V20),
+		.driver_data = board_pch_mtp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_LM21),
+		.driver_data = board_pch_mtp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_LNP_I219_V21),
+		.driver_data = board_pch_mtp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ARL_I219_LM24),
+		.driver_data = board_pch_mtp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_ARL_I219_V24),
+		.driver_data = board_pch_mtp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_LM25),
+		.driver_data = board_pch_ptp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_V25),
+		.driver_data = board_pch_ptp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_LM27),
+		.driver_data = board_pch_ptp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_PTP_I219_V27),
+		.driver_data = board_pch_ptp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_NVL_I219_LM29),
+		.driver_data = board_pch_ptp,
+	}, {
+		PCI_VDEVICE(INTEL, E1000_DEV_ID_PCH_NVL_I219_V29),
+		.driver_data = board_pch_ptp
+	},
+
+	{ }	/* terminate list */
 };
 MODULE_DEVICE_TABLE(pci, e1000_pci_tbl);
 
diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
index d75b8a50413d2..f5b4d062709a5 100644
--- a/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
+++ b/drivers/net/ethernet/intel/fm10k/fm10k_pci.c
@@ -21,12 +21,12 @@ static const struct fm10k_info *fm10k_info_tbl[] = {
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id fm10k_pci_tbl[] = {
-	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_PF), fm10k_device_pf },
-	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_SDI_FM10420_QDA2), fm10k_device_pf },
-	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_SDI_FM10420_DA2), fm10k_device_pf },
-	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_VF), fm10k_device_vf },
+	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_PF), .driver_data = fm10k_device_pf },
+	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_SDI_FM10420_QDA2), .driver_data = fm10k_device_pf },
+	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_SDI_FM10420_DA2), .driver_data = fm10k_device_pf },
+	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_VF), .driver_data = fm10k_device_vf },
 	/* required last entry */
-	{ 0, }
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, fm10k_pci_tbl);
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c
index 6d4f9218dc684..a04683004a567 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_main.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_main.c
@@ -63,40 +63,43 @@ static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf);
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id i40e_pci_tbl[] = {
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP) },
 	/*
 	 * This ID conflicts with ipw2200, but the devices can be differentiated
 	 * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200
 	 * devices use PCI_CLASS_NETWORK_OTHER.
 	 */
-	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, I40E_DEV_ID_10G_B),
-		PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
-	{PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_INTEL, I40E_DEV_ID_10G_B),
+		.class = PCI_CLASS_NETWORK_ETHERNET << 8,
+		.class_mask = 0xffff00,
+	},
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B) },
+	{ PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28) },
 	/* required last entry */
-	{0, }
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
 
diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
index 7d07c389bb231..ff62b5f2c8150 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c
+++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c
@@ -24,9 +24,6 @@
 #define I40E_PTP_1GB_INCVAL_MULT	20
 #define I40E_ISGN			0x80000000
 
-#define I40E_PRTTSYN_CTL1_TSYNTYPE_V1  BIT(I40E_PRTTSYN_CTL1_TSYNTYPE_SHIFT)
-#define I40E_PRTTSYN_CTL1_TSYNTYPE_V2  (2 << \
-					I40E_PRTTSYN_CTL1_TSYNTYPE_SHIFT)
 #define I40E_SUBDEV_ID_25G_PTP_PIN	0xB
 
 enum i40e_ptp_pin {
@@ -1220,7 +1217,7 @@ static int i40e_ptp_set_timestamp_mode(struct i40e_pf *pf,
 		pf->ptp_rx = true;
 		tsyntype = I40E_PRTTSYN_CTL1_V1MESSTYPE0_MASK |
 			   I40E_PRTTSYN_CTL1_TSYNTYPE_V1 |
-			   I40E_PRTTSYN_CTL1_UDP_ENA_MASK;
+			   I40E_PRTTSYN_CTL1_UDP_ENA_319;
 		config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 		break;
 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
@@ -1237,9 +1234,9 @@ static int i40e_ptp_set_timestamp_mode(struct i40e_pf *pf,
 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
 		pf->ptp_rx = true;
 		tsyntype = I40E_PRTTSYN_CTL1_V2MESSTYPE0_MASK |
-			   I40E_PRTTSYN_CTL1_TSYNTYPE_V2;
+			   I40E_PRTTSYN_CTL1_TSYNTYPE_V2_EVENT;
 		if (test_bit(I40E_HW_CAP_PTP_L4, pf->hw.caps)) {
-			tsyntype |= I40E_PRTTSYN_CTL1_UDP_ENA_MASK;
+			tsyntype |= I40E_PRTTSYN_CTL1_UDP_ENA_319;
 			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 		} else {
 			config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT;
diff --git a/drivers/net/ethernet/intel/i40e/i40e_register.h b/drivers/net/ethernet/intel/i40e/i40e_register.h
index 432afbb642013..d426d83e02144 100644
--- a/drivers/net/ethernet/intel/i40e/i40e_register.h
+++ b/drivers/net/ethernet/intel/i40e/i40e_register.h
@@ -788,8 +788,18 @@
 #define I40E_PRTTSYN_CTL1_V2MESSTYPE0_SHIFT 16
 #define I40E_PRTTSYN_CTL1_V2MESSTYPE0_MASK I40E_MASK(0xF, I40E_PRTTSYN_CTL1_V2MESSTYPE0_SHIFT)
 #define I40E_PRTTSYN_CTL1_TSYNTYPE_SHIFT 24
+#define I40E_PRTTSYN_CTL1_TSYNTYPE_MASK I40E_MASK(0x3, I40E_PRTTSYN_CTL1_TSYNTYPE_SHIFT)
+/* Timestamp UDP v1 packets */
+#define I40E_PRTTSYN_CTL1_TSYNTYPE_V1 \
+	FIELD_PREP(I40E_PRTTSYN_CTL1_TSYNTYPE_MASK, 1)
+/* Timestamp L2 and UDP v2 packets with message type < 8 */
+#define I40E_PRTTSYN_CTL1_TSYNTYPE_V2_EVENT \
+	FIELD_PREP(I40E_PRTTSYN_CTL1_TSYNTYPE_MASK, 3)
 #define I40E_PRTTSYN_CTL1_UDP_ENA_SHIFT 26
 #define I40E_PRTTSYN_CTL1_UDP_ENA_MASK I40E_MASK(0x3, I40E_PRTTSYN_CTL1_UDP_ENA_SHIFT)
+/* Timestamp UDP packets on port 319 */
+#define I40E_PRTTSYN_CTL1_UDP_ENA_319 \
+	FIELD_PREP(I40E_PRTTSYN_CTL1_UDP_ENA_MASK, 1)
 #define I40E_PRTTSYN_CTL1_TSYNENA_SHIFT 31
 #define I40E_PRTTSYN_CTL1_TSYNENA_MASK I40E_MASK(0x1, I40E_PRTTSYN_CTL1_TSYNENA_SHIFT)
 #define I40E_PRTTSYN_INC_H 0x001E4060 /* Reset: GLOBR */
diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c
index d2914c511e1e0..8b53ffb756500 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_main.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
@@ -36,12 +36,12 @@ static const char iavf_copyright[] =
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id iavf_pci_tbl[] = {
-	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
-	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
-	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
-	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
+	{ PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF) },
+	{ PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV) },
+	{ PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF) },
+	{ PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF) },
 	/* required last entry */
-	{0, }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.c b/drivers/net/ethernet/intel/ice/ice_dpll.c
index 892bc7c2e28b4..b9c7df50123db 100644
--- a/drivers/net/ethernet/intel/ice/ice_dpll.c
+++ b/drivers/net/ethernet/intel/ice/ice_dpll.c
@@ -3637,7 +3637,7 @@ static int
 ice_dpll_init_fwnode_pins(struct ice_pf *pf, struct ice_dpll_pin *pins,
 			  int start_idx)
 {
-	char pin_name[8];
+	char pin_name[16];
 	int i, ret;
 
 	pf->dplls.wq = create_singlethread_workqueue("ice_dpll_wq");
diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c
index f28416a707d77..236d293aba984 100644
--- a/drivers/net/ethernet/intel/ice/ice_ethtool.c
+++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c
@@ -1251,7 +1251,7 @@ static int ice_lbtest_receive_frames(struct ice_rx_ring *rx_ring)
 		rx_buf = &rx_ring->rx_fqes[i];
 		page = __netmem_to_page(rx_buf->netmem);
 		received_buf = page_address(page) + rx_buf->offset +
-			       page->pp->p.offset;
+			       pp_page_to_nmdesc(page)->pp->p.offset;
 
 		if (ice_lbtest_check_frame(received_buf))
 			valid_frames++;
diff --git a/drivers/net/ethernet/intel/ice/ice_fw_update.c b/drivers/net/ethernet/intel/ice/ice_fw_update.c
index c143e546dd8c9..36314610927be 100644
--- a/drivers/net/ethernet/intel/ice/ice_fw_update.c
+++ b/drivers/net/ethernet/intel/ice/ice_fw_update.c
@@ -726,7 +726,7 @@ static int ice_finalize_update(struct pldmfw *context)
 	switch (priv->reset_level) {
 	case ICE_AQC_NVM_EMPR_FLAG:
 		devlink_flash_update_status_notify(devlink,
-						   "Activate new firmware by devlink reload",
+						   "Activate new firmware by devlink reload action fw_activate",
 						   NULL, 0, 0);
 		break;
 	case ICE_AQC_NVM_PERST_FLAG:
diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c
index c52c465280f74..e2fbe111f8496 100644
--- a/drivers/net/ethernet/intel/ice/ice_main.c
+++ b/drivers/net/ethernet/intel/ice/ice_main.c
@@ -3682,7 +3682,7 @@ int ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
 		ret = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx,
 					       ICE_MCAST_VLAN_PROMISC_BITS,
 					       vid);
-		if (ret)
+		if (ret && ret != -EEXIST)
 			goto finish;
 	}
 
@@ -4104,6 +4104,12 @@ int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx, bool locked)
 	}
 	ice_pf_dcb_recfg(pf, locked);
 	ice_vsi_open(vsi);
+	/* Rx rings are reallocated during VSI rebuild and lose their ptp_rx
+	 * flag. Restore timestamp mode so newly allocated rings are set up
+	 * for hardware Rx timestamping.
+	 */
+	if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags))
+		ice_ptp_restore_timestamp_mode(pf);
 	goto done;
 
 rebuild_err:
diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c
index 24fb7a3e14d63..2c18e16fe053e 100644
--- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.c
+++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.c
@@ -2141,16 +2141,23 @@ int ice_start_phy_timer_eth56g(struct ice_hw *hw, u8 port)
 	}
 	incval = (u64)hi << 32 | lo;
 
+	if (!ice_ptp_lock(hw)) {
+		dev_err(ice_hw_to_dev(hw), "Failed to acquire PTP semaphore\n");
+		return -EBUSY;
+	}
+
 	err = ice_write_40b_ptp_reg_eth56g(hw, port, PHY_REG_TIMETUS_L, incval);
 	if (err)
-		return err;
+		goto err_ptp_unlock;
 
 	err = ice_ptp_one_port_cmd(hw, port, ICE_PTP_INIT_INCVAL);
 	if (err)
-		return err;
+		goto err_ptp_unlock;
 
 	ice_ptp_exec_tmr_cmd(hw);
 
+	ice_ptp_unlock(hw);
+
 	err = ice_sync_phy_timer_eth56g(hw, port);
 	if (err)
 		return err;
@@ -2166,6 +2173,10 @@ int ice_start_phy_timer_eth56g(struct ice_hw *hw, u8 port)
 	ice_debug(hw, ICE_DBG_PTP, "Enabled clock on PHY port %u\n", port);
 
 	return 0;
+
+err_ptp_unlock:
+	ice_ptp_unlock(hw);
+	return err;
 }
 
 /**
@@ -4503,18 +4514,17 @@ static int
 ice_read_phy_tstamp_ll_e810(struct ice_hw *hw, u8 idx, u8 *hi, u32 *lo)
 {
 	struct ice_e810_params *params = &hw->ptp.phy.e810;
-	unsigned long flags;
 	u32 val;
 	int err;
 
-	spin_lock_irqsave(&params->atqbal_wq.lock, flags);
+	spin_lock_irq(&params->atqbal_wq.lock);
 
 	/* Wait for any pending in-progress low latency interrupt */
 	err = wait_event_interruptible_locked_irq(params->atqbal_wq,
 						  !(params->atqbal_flags &
 						    ATQBAL_FLAGS_INTR_IN_PROGRESS));
 	if (err) {
-		spin_unlock_irqrestore(&params->atqbal_wq.lock, flags);
+		spin_unlock_irq(&params->atqbal_wq.lock);
 		return err;
 	}
 
@@ -4529,7 +4539,7 @@ ice_read_phy_tstamp_ll_e810(struct ice_hw *hw, u8 idx, u8 *hi, u32 *lo)
 				       REG_LL_PROXY_H);
 	if (err) {
 		ice_debug(hw, ICE_DBG_PTP, "Failed to read PTP timestamp using low latency read\n");
-		spin_unlock_irqrestore(&params->atqbal_wq.lock, flags);
+		spin_unlock_irq(&params->atqbal_wq.lock);
 		return err;
 	}
 
@@ -4539,7 +4549,7 @@ ice_read_phy_tstamp_ll_e810(struct ice_hw *hw, u8 idx, u8 *hi, u32 *lo)
 	/* Read the low 32 bit value and set the TS valid bit */
 	*lo = rd32(hw, REG_LL_PROXY_L) | TS_VALID;
 
-	spin_unlock_irqrestore(&params->atqbal_wq.lock, flags);
+	spin_unlock_irq(&params->atqbal_wq.lock);
 
 	return 0;
 }
@@ -5254,9 +5264,13 @@ static void ice_ptp_init_phy_e830(struct ice_ptp_hw *ptp)
  */
 bool ice_ptp_lock(struct ice_hw *hw)
 {
+	struct ice_pf *pf = container_of(hw, struct ice_pf, hw);
 	u32 hw_lock;
 	int i;
 
+	if (!ice_is_primary(hw))
+		hw = ice_get_primary_hw(pf);
+
 #define MAX_TRIES 15
 
 	for (i = 0; i < MAX_TRIES; i++) {
@@ -5283,6 +5297,11 @@ bool ice_ptp_lock(struct ice_hw *hw)
  */
 void ice_ptp_unlock(struct ice_hw *hw)
 {
+	struct ice_pf *pf = container_of(hw, struct ice_pf, hw);
+
+	if (!ice_is_primary(hw))
+		hw = ice_get_primary_hw(pf);
+
 	wr32(hw, PFTSYN_SEM + (PFTSYN_SEM_BYTES * hw->pf_id), 0);
 }
 
diff --git a/drivers/net/ethernet/intel/ice/virt/queues.c b/drivers/net/ethernet/intel/ice/virt/queues.c
index f73d5a3e83d43..31be2f76181c0 100644
--- a/drivers/net/ethernet/intel/ice/virt/queues.c
+++ b/drivers/net/ethernet/intel/ice/virt/queues.c
@@ -840,7 +840,7 @@ int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg)
 
 			if (qpi->rxq.databuffer_size != 0 &&
 			    (qpi->rxq.databuffer_size > ((16 * 1024) - 128) ||
-			     qpi->rxq.databuffer_size < 1024))
+			     qpi->rxq.databuffer_size < 128))
 				goto error_param;
 
 			ring->rx_buf_len = qpi->rxq.databuffer_size;
diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c
index ce91dda00ec0e..49d46670695f3 100644
--- a/drivers/net/ethernet/intel/igb/igb_main.c
+++ b/drivers/net/ethernet/intel/igb/igb_main.c
@@ -63,40 +63,40 @@ static const struct pci_device_id igb_pci_tbl[] = {
 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), .driver_data = board_82575 },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), .driver_data = board_82575 },
 	/* required last entry */
-	{0, }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
diff --git a/drivers/net/ethernet/intel/igbvf/netdev.c b/drivers/net/ethernet/intel/igbvf/netdev.c
index 5d04541072b3f..0a3d0a1cba433 100644
--- a/drivers/net/ethernet/intel/igbvf/netdev.c
+++ b/drivers/net/ethernet/intel/igbvf/netdev.c
@@ -2937,8 +2937,8 @@ static const struct pci_error_handlers igbvf_err_handler = {
 };
 
 static const struct pci_device_id igbvf_pci_tbl[] = {
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_VF), board_vf },
-	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_VF), board_i350_vf },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_VF), .driver_data = board_vf },
+	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_VF), .driver_data = board_i350_vf },
 	{ } /* terminate list */
 };
 MODULE_DEVICE_TABLE(pci, igbvf_pci_tbl);
diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c
index 8ac16808023cc..89b9e1d00fc7f 100644
--- a/drivers/net/ethernet/intel/igc/igc_main.c
+++ b/drivers/net/ethernet/intel/igc/igc_main.c
@@ -47,24 +47,24 @@ static const struct igc_info *igc_info_tbl[] = {
 };
 
 static const struct pci_device_id igc_pci_tbl[] = {
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base },
-	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), .driver_data = board_base },
+	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), .driver_data = board_base },
 	/* required last entry */
-	{0, }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
diff --git a/drivers/net/ethernet/intel/igc/igc_tsn.c b/drivers/net/ethernet/intel/igc/igc_tsn.c
index 8a110145bfee7..52de2bcbadbec 100644
--- a/drivers/net/ethernet/intel/igc/igc_tsn.c
+++ b/drivers/net/ethernet/intel/igc/igc_tsn.c
@@ -34,6 +34,7 @@ static int igc_fpe_init_smd_frame(struct igc_ring *ring,
 		return -ENOMEM;
 	}
 
+	buffer->type = IGC_TX_BUFFER_TYPE_SKB;
 	buffer->skb = skb;
 	buffer->protocol = 0;
 	buffer->bytecount = skb->len;
@@ -109,10 +110,16 @@ static int igc_fpe_xmit_smd_frame(struct igc_adapter *adapter,
 	__netif_tx_lock(nq, cpu);
 
 	err = igc_fpe_init_tx_descriptor(ring, skb, type);
-	igc_flush_tx_descriptors(ring);
+	if (err)
+		goto err_free_skb_any;
 
+	igc_flush_tx_descriptors(ring);
 	__netif_tx_unlock(nq);
+	return 0;
 
+err_free_skb_any:
+	__netif_tx_unlock(nq);
+	dev_kfree_skb_any(skb);
 	return err;
 }
 
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.c
index 52f73142ff756..665a9813e251b 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.c
@@ -628,6 +628,9 @@ static bool ixgbe_parse_e610_caps(struct ixgbe_hw *hw,
 			(phys_id & IXGBE_EXT_TOPO_DEV_IMG_PROG_EN) != 0;
 		break;
 	}
+	case LIBIE_AQC_CAPS_EEE:
+		caps->eee_support = (u8)number;
+		break;
 	default:
 		/* Not one of the recognized common capabilities */
 		return false;
@@ -1073,6 +1076,7 @@ void ixgbe_copy_phy_caps_to_cfg(struct ixgbe_aci_cmd_get_phy_caps_data *caps,
 	cfg->link_fec_opt = caps->link_fec_options;
 	cfg->module_compliance_enforcement =
 		caps->module_compliance_enforcement;
+	cfg->eee_entry_delay = caps->eee_entry_delay;
 }
 
 /**
@@ -1093,11 +1097,16 @@ int ixgbe_aci_set_phy_cfg(struct ixgbe_hw *hw,
 {
 	struct ixgbe_aci_cmd_set_phy_cfg *cmd;
 	struct libie_aq_desc desc;
+	bool use_buff_eee_field;
+	u16 buf_size;
 	int err;
 
 	if (!cfg)
 		return -EINVAL;
 
+	/* If FW supports EEE, we have to use buffer with EEE field. */
+	use_buff_eee_field = hw->dev_caps.common_cap.eee_support;
+
 	cmd = libie_aq_raw(&desc);
 	/* Ensure that only valid bits of cfg->caps can be turned on. */
 	cfg->caps &= IXGBE_ACI_PHY_ENA_VALID_MASK;
@@ -1106,7 +1115,17 @@ int ixgbe_aci_set_phy_cfg(struct ixgbe_hw *hw,
 	cmd->lport_num = hw->bus.func;
 	desc.flags |= cpu_to_le16(LIBIE_AQ_FLAG_RD);
 
-	err = ixgbe_aci_send_cmd(hw, &desc, cfg, sizeof(*cfg));
+	if (use_buff_eee_field)
+		buf_size = sizeof(*cfg);
+	else
+		/* Buffer w/o eee_entry_delay field is 2B smaller. */
+		buf_size = sizeof(*cfg) - sizeof(u16);
+
+	err = ixgbe_aci_send_cmd(hw, &desc, cfg, buf_size);
+
+	/* 1.40 config format is compatible with pre-1.40, just extends
+	 * it at the end.
+	 */
 	if (!err)
 		hw->phy.curr_user_phy_cfg = *cfg;
 
@@ -1386,6 +1405,7 @@ int ixgbe_aci_get_link_info(struct ixgbe_hw *hw, bool ena_lse,
 	li->topo_media_conflict = link_data.topo_media_conflict;
 	li->pacing = link_data.cfg & (IXGBE_ACI_CFG_PACING_M |
 				      IXGBE_ACI_CFG_PACING_TYPE_M);
+	li->eee_status = link_data.eee_status;
 
 	/* Update fc info. */
 	tx_pause = !!(link_data.an_info & IXGBE_ACI_LINK_PAUSE_TX);
@@ -1980,14 +2000,58 @@ int ixgbe_identify_phy_e610(struct ixgbe_hw *hw)
 	/* Set PHY ID */
 	memcpy(&hw->phy.id, pcaps.phy_id_oui, sizeof(u32));
 
-	hw->phy.eee_speeds_supported = IXGBE_LINK_SPEED_10_FULL |
-				       IXGBE_LINK_SPEED_100_FULL |
-				       IXGBE_LINK_SPEED_1GB_FULL;
+	/* E610 supports EEE only for speeds above 1G */
+	if (hw->device_id == IXGBE_DEV_ID_E610_2_5G_T)
+		hw->phy.eee_speeds_supported = IXGBE_LINK_SPEED_2_5GB_FULL;
+	else
+		hw->phy.eee_speeds_supported = IXGBE_LINK_SPEED_2_5GB_FULL |
+					       IXGBE_LINK_SPEED_5GB_FULL |
+					       IXGBE_LINK_SPEED_10GB_FULL;
+
 	hw->phy.eee_speeds_advertised = hw->phy.eee_speeds_supported;
 
 	return 0;
 }
 
+/**
+ * ixgbe_setup_eee_e610 - Enable/disable EEE support
+ * @hw: pointer to the HW structure
+ * @enable_eee: boolean flag to enable EEE
+ *
+ * Enable/disable EEE based on @enable_eee.
+ *
+ * Return: the exit code of the operation.
+ */
+int ixgbe_setup_eee_e610(struct ixgbe_hw *hw, bool enable_eee)
+{
+	struct ixgbe_aci_cmd_get_phy_caps_data phy_caps = {};
+	struct ixgbe_aci_cmd_set_phy_cfg_data phy_cfg = {};
+	u16 eee_cap = 0;
+	int err;
+
+	err = ixgbe_aci_get_phy_caps(hw, false,
+				     IXGBE_ACI_REPORT_ACTIVE_CFG, &phy_caps);
+	if (err)
+		return err;
+
+	ixgbe_copy_phy_caps_to_cfg(&phy_caps, &phy_cfg);
+	phy_cfg.caps |= (IXGBE_ACI_PHY_ENA_LINK |
+			IXGBE_ACI_PHY_ENA_AUTO_LINK_UPDT);
+
+	if (enable_eee) {
+		if (hw->phy.eee_speeds_advertised & IXGBE_LINK_SPEED_2_5GB_FULL)
+			eee_cap |= IXGBE_ACI_PHY_EEE_EN_2_5GBASE_T;
+		if (hw->phy.eee_speeds_advertised & IXGBE_LINK_SPEED_5GB_FULL)
+			eee_cap |= IXGBE_ACI_PHY_EEE_EN_5GBASE_T;
+		if (hw->phy.eee_speeds_advertised & IXGBE_LINK_SPEED_10GB_FULL)
+			eee_cap |= IXGBE_ACI_PHY_EEE_EN_10GBASE_T;
+	}
+
+	phy_cfg.eee_cap = cpu_to_le16(eee_cap);
+
+	return ixgbe_aci_set_phy_cfg(hw, &phy_cfg);
+}
+
 /**
  * ixgbe_identify_module_e610 - Identify SFP module type
  * @hw: pointer to hardware structure
@@ -4001,6 +4065,7 @@ static const struct ixgbe_mac_operations mac_ops_e610 = {
 	.fw_rollback_mode		= ixgbe_fw_rollback_mode_e610,
 	.get_nvm_ver			= ixgbe_get_active_nvm_ver,
 	.get_link_capabilities		= ixgbe_get_link_capabilities_e610,
+	.setup_eee			= ixgbe_setup_eee_e610,
 	.get_bus_info			= ixgbe_get_bus_info_generic,
 	.acquire_swfw_sync		= ixgbe_acquire_swfw_sync_X540,
 	.release_swfw_sync		= ixgbe_release_swfw_sync_X540,
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.h
index 11916b979d282..2cb76a3d30ae6 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.h
@@ -55,6 +55,7 @@ int ixgbe_init_phy_ops_e610(struct ixgbe_hw *hw);
 int ixgbe_identify_phy_e610(struct ixgbe_hw *hw);
 int ixgbe_identify_module_e610(struct ixgbe_hw *hw);
 int ixgbe_setup_phy_link_e610(struct ixgbe_hw *hw);
+int ixgbe_setup_eee_e610(struct ixgbe_hw *hw, bool enable_eee);
 int ixgbe_set_phy_power_e610(struct ixgbe_hw *hw, bool on);
 int ixgbe_enter_lplu_e610(struct ixgbe_hw *hw);
 int ixgbe_init_eeprom_params_e610(struct ixgbe_hw *hw);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
index ba049b3a9609e..36e43b5e88d1f 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c
@@ -12,6 +12,7 @@
 #include <linux/ethtool.h>
 #include <linux/vmalloc.h>
 #include <linux/highmem.h>
+#include <linux/string_choices.h>
 #include <linux/uaccess.h>
 
 #include "ixgbe.h"
@@ -3540,7 +3541,8 @@ static const struct {
 	{ IXGBE_LINK_SPEED_10_FULL, ETHTOOL_LINK_MODE_10baseT_Full_BIT },
 	{ IXGBE_LINK_SPEED_100_FULL, ETHTOOL_LINK_MODE_100baseT_Full_BIT },
 	{ IXGBE_LINK_SPEED_1GB_FULL, ETHTOOL_LINK_MODE_1000baseT_Full_BIT },
-	{ IXGBE_LINK_SPEED_2_5GB_FULL, ETHTOOL_LINK_MODE_2500baseX_Full_BIT },
+	{ IXGBE_LINK_SPEED_2_5GB_FULL, ETHTOOL_LINK_MODE_2500baseT_Full_BIT },
+	{ IXGBE_LINK_SPEED_5GB_FULL, ETHTOOL_LINK_MODE_5000baseT_Full_BIT },
 	{ IXGBE_LINK_SPEED_10GB_FULL, ETHTOOL_LINK_MODE_10000baseT_Full_BIT },
 };
 
@@ -3556,6 +3558,165 @@ static const struct {
 	{ FW_PHY_ACT_UD_2_10G_KR_EEE, ETHTOOL_LINK_MODE_10000baseKR_Full_BIT},
 };
 
+static int ixgbe_validate_keee(struct net_device *netdev,
+			       struct ethtool_keee *keee_requested)
+{
+	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
+	struct ethtool_keee keee_stored = {};
+	int err;
+
+	if (!(adapter->flags2 & IXGBE_FLAG2_EEE_CAPABLE))
+		return -EOPNOTSUPP;
+
+	err = netdev->ethtool_ops->get_eee(netdev, &keee_stored);
+	if (err)
+		return err;
+
+	if (keee_stored.tx_lpi_enabled != keee_requested->tx_lpi_enabled) {
+		e_err(drv, "Setting EEE tx-lpi is not supported\n");
+		return -EINVAL;
+	}
+
+	if (keee_stored.tx_lpi_timer != keee_requested->tx_lpi_timer) {
+		e_err(drv,
+		      "Setting EEE Tx LPI timer is not supported\n");
+		return -EINVAL;
+	}
+
+	if (!linkmode_equal(keee_stored.advertised,
+			    keee_requested->advertised)) {
+		e_err(drv,
+		      "Setting EEE advertised speeds is not supported\n");
+		return -EINVAL;
+	}
+
+	/* -EALREADY here is for internal use only, must be converted into
+	 * early bail out with 0 by caller
+	 */
+	if (keee_stored.eee_enabled == keee_requested->eee_enabled)
+		return -EALREADY;
+
+	return 0;
+}
+
+/**
+ * ixgbe_is_eee_link_speed_supported_e610 - Check if EEE can be enabled
+ * @adapter: pointer to the adapter struct
+ *
+ * Check whether current link configuration is capable of enabling EEE feature.
+ *
+ * E610 specific function - for other adapters supporting EEE there might be
+ * no such limitation.
+ *
+ * Return: true if EEE can be enabled, false otherwise.
+ */
+static bool
+ixgbe_is_eee_link_speed_supported_e610(struct ixgbe_adapter *adapter)
+{
+	switch (adapter->link_speed) {
+	case IXGBE_LINK_SPEED_10GB_FULL:
+	case IXGBE_LINK_SPEED_2_5GB_FULL:
+	case IXGBE_LINK_SPEED_5GB_FULL:
+		return true;
+	case IXGBE_LINK_SPEED_100_FULL:
+	case IXGBE_LINK_SPEED_1GB_FULL:
+		e_dev_info("Energy Efficient Ethernet (EEE) feature is not supported on link speeds equal to or below 1Gbps. EEE is supported on speeds above 1Gbps.\n");
+		fallthrough;
+	default:
+		return false;
+	}
+}
+
+static int ixgbe_get_eee_e610(struct net_device *netdev,
+			      struct ethtool_keee *kedata)
+{
+	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
+	struct ixgbe_aci_cmd_get_phy_caps_data pcaps;
+	struct ixgbe_hw *hw = &adapter->hw;
+	struct ixgbe_link_status link;
+	int err;
+
+	linkmode_zero(kedata->lp_advertised);
+	linkmode_zero(kedata->supported);
+	linkmode_zero(kedata->advertised);
+
+	err = ixgbe_aci_get_link_info(hw, true, &link);
+	if (err)
+		return err;
+
+	err = ixgbe_aci_get_phy_caps(hw, false, IXGBE_ACI_REPORT_ACTIVE_CFG,
+				     &pcaps);
+	if (err)
+		return err;
+
+	kedata->eee_active =  link.eee_status & IXGBE_ACI_LINK_EEE_ACTIVE;
+	kedata->eee_enabled = link.eee_status & IXGBE_ACI_LINK_EEE_ENABLED;
+
+	/* for E610 devices EEE enablement implies TX LPI enablement */
+	kedata->tx_lpi_enabled = kedata->eee_enabled;
+
+	if (kedata->eee_enabled)
+		kedata->tx_lpi_timer = le16_to_cpu(pcaps.eee_entry_delay);
+
+	for (int i = 0; i < ARRAY_SIZE(ixgbe_ls_map); i++) {
+		if (hw->phy.eee_speeds_supported &
+		    ixgbe_ls_map[i].mac_speed)
+			linkmode_set_bit(ixgbe_ls_map[i].link_mode,
+					 kedata->supported);
+
+		if (hw->phy.eee_speeds_advertised &
+		    ixgbe_ls_map[i].mac_speed)
+			linkmode_set_bit(ixgbe_ls_map[i].link_mode,
+					 kedata->advertised);
+	}
+
+	return 0;
+}
+
+static int ixgbe_set_eee_e610(struct net_device *netdev,
+			      struct ethtool_keee *kedata)
+{
+	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
+	struct ixgbe_hw *hw = &adapter->hw;
+	int err;
+
+	err = ixgbe_validate_keee(netdev, kedata);
+
+	if (err == -EALREADY) {
+		return 0;
+	} else if (err) {
+		if (err == -EOPNOTSUPP)
+			e_dev_info("Energy Efficient Ethernet (EEE) feature is currently not supported on this device, please update the device NVM to the latest and try again\n");
+		return err;
+	}
+
+	if (!(ixgbe_is_eee_link_speed_supported_e610(adapter)) &&
+	    kedata->eee_enabled)
+		return -EOPNOTSUPP;
+
+	hw->phy.eee_speeds_advertised = kedata->eee_enabled ?
+					hw->phy.eee_speeds_supported : 0;
+
+	err = hw->mac.ops.setup_eee(hw, kedata->eee_enabled);
+	if (err) {
+		e_dev_err("Setting EEE %s failed.\n",
+			  str_on_off(kedata->eee_enabled));
+		return err;
+	}
+
+	if (kedata->eee_enabled)
+		adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED;
+	else
+		adapter->flags2 &= ~IXGBE_FLAG2_EEE_ENABLED;
+
+	if (netif_running(netdev))
+		ixgbe_reinit_locked(adapter);
+	else
+		ixgbe_reset(adapter);
+
+	return 0;
+}
+
 static int
 ixgbe_get_eee_fw(struct ixgbe_adapter *adapter, struct ethtool_keee *edata)
 {
@@ -3614,53 +3775,28 @@ static int ixgbe_set_eee(struct net_device *netdev, struct ethtool_keee *edata)
 {
 	struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev);
 	struct ixgbe_hw *hw = &adapter->hw;
-	struct ethtool_keee eee_data;
 	int ret_val;
 
-	if (!(adapter->flags2 & IXGBE_FLAG2_EEE_CAPABLE))
-		return -EOPNOTSUPP;
-
-	memset(&eee_data, 0, sizeof(struct ethtool_keee));
-
-	ret_val = ixgbe_get_eee(netdev, &eee_data);
-	if (ret_val)
+	ret_val = ixgbe_validate_keee(netdev, edata);
+	if (ret_val == -EALREADY)
+		return 0;
+	else if (ret_val)
 		return ret_val;
 
-	if (eee_data.eee_enabled && !edata->eee_enabled) {
-		if (eee_data.tx_lpi_enabled != edata->tx_lpi_enabled) {
-			e_err(drv, "Setting EEE tx-lpi is not supported\n");
-			return -EINVAL;
-		}
-
-		if (eee_data.tx_lpi_timer != edata->tx_lpi_timer) {
-			e_err(drv,
-			      "Setting EEE Tx LPI timer is not supported\n");
-			return -EINVAL;
-		}
-
-		if (!linkmode_equal(eee_data.advertised, edata->advertised)) {
-			e_err(drv,
-			      "Setting EEE advertised speeds is not supported\n");
-			return -EINVAL;
-		}
+	if (edata->eee_enabled) {
+		adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED;
+		hw->phy.eee_speeds_advertised =
+					   hw->phy.eee_speeds_supported;
+	} else {
+		adapter->flags2 &= ~IXGBE_FLAG2_EEE_ENABLED;
+		hw->phy.eee_speeds_advertised = 0;
 	}
 
-	if (eee_data.eee_enabled != edata->eee_enabled) {
-		if (edata->eee_enabled) {
-			adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED;
-			hw->phy.eee_speeds_advertised =
-						   hw->phy.eee_speeds_supported;
-		} else {
-			adapter->flags2 &= ~IXGBE_FLAG2_EEE_ENABLED;
-			hw->phy.eee_speeds_advertised = 0;
-		}
-
-		/* reset link */
-		if (netif_running(netdev))
-			ixgbe_reinit_locked(adapter);
-		else
-			ixgbe_reset(adapter);
-	}
+	/* reset link */
+	if (netif_running(netdev))
+		ixgbe_reinit_locked(adapter);
+	else
+		ixgbe_reset(adapter);
 
 	return 0;
 }
@@ -3807,8 +3943,8 @@ static const struct ethtool_ops ixgbe_ethtool_ops_e610 = {
 	.set_rxfh		= ixgbe_set_rxfh,
 	.get_rxfh_fields	= ixgbe_get_rxfh_fields,
 	.set_rxfh_fields	= ixgbe_set_rxfh_fields,
-	.get_eee		= ixgbe_get_eee,
-	.set_eee		= ixgbe_set_eee,
+	.get_eee		= ixgbe_get_eee_e610,
+	.set_eee		= ixgbe_set_eee_e610,
 	.get_channels		= ixgbe_get_channels,
 	.set_channels		= ixgbe_set_channels,
 	.get_priv_flags		= ixgbe_get_priv_flags,
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
index 2646ee6f295f0..bc16e4c93fd42 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c
@@ -88,60 +88,60 @@ static const struct ixgbe_info *ixgbe_info_tbl[] = {
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id ixgbe_pci_tbl[] = {
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), board_82599 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), board_X540 },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), board_X550},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), board_X550},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), board_X550EM_x},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_XFI), board_X550EM_x},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), board_X550EM_x},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), board_X550EM_x},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), board_X550EM_x},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_1G_T), board_x550em_x_fw},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), board_x550em_a },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), board_x550em_a },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), board_x550em_a },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), board_x550em_a },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), board_x550em_a },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), board_x550em_a},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), board_x550em_a },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), board_x550em_a_fw },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), board_x550em_a_fw },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_BACKPLANE), board_e610},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_SFP), board_e610},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_10G_T), board_e610},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_2_5G_T), board_e610},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_SGMII), board_e610},
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), .driver_data = board_82598 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), .driver_data = board_X540 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_QSFP_SF_QP), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599EN_SFP), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF_QP), .driver_data = board_82599 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T1), .driver_data = board_X540 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T), .driver_data = board_X550 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550T1), .driver_data = board_X550 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KX4), .driver_data = board_X550EM_x },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_XFI), .driver_data = board_X550EM_x },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_KR), .driver_data = board_X550EM_x },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_10G_T), .driver_data = board_X550EM_x },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_SFP), .driver_data = board_X550EM_x },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_1G_T), .driver_data = board_x550em_x_fw },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR), .driver_data = board_x550em_a },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_KR_L), .driver_data = board_x550em_a },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP_N), .driver_data = board_x550em_a },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII), .driver_data = board_x550em_a },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SGMII_L), .driver_data = board_x550em_a },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_10G_T), .driver_data = board_x550em_a },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_SFP), .driver_data = board_x550em_a },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T), .driver_data = board_x550em_a_fw },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_1G_T_L), .driver_data = board_x550em_a_fw },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_BACKPLANE), .driver_data = board_e610 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_SFP), .driver_data = board_e610 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_10G_T), .driver_data = board_e610 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_2_5G_T), .driver_data = board_e610 },
+	{ PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_SGMII), .driver_data = board_e610 },
 	/* required last entry */
-	{0, }
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
 
@@ -6758,6 +6758,7 @@ void ixgbe_down(struct ixgbe_adapter *adapter)
 
 /**
  * ixgbe_set_eee_capable - helper function to determine EEE support on X550
+ * and E610
  * @adapter: board private structure
  */
 static void ixgbe_set_eee_capable(struct ixgbe_adapter *adapter)
@@ -6774,6 +6775,20 @@ static void ixgbe_set_eee_capable(struct ixgbe_adapter *adapter)
 			break;
 		adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED;
 		break;
+	case IXGBE_DEV_ID_E610_BACKPLANE:
+	case IXGBE_DEV_ID_E610_SFP:
+	case IXGBE_DEV_ID_E610_10G_T:
+	case IXGBE_DEV_ID_E610_2_5G_T:
+		if (hw->dev_caps.common_cap.eee_support &&
+		    hw->phy.eee_speeds_supported) {
+			adapter->flags2 |= IXGBE_FLAG2_EEE_CAPABLE;
+			/* For E610 adapters EEE should be enabled by default
+			 * if the feature is supported by FW.
+			 */
+			adapter->flags2 |= IXGBE_FLAG2_EEE_ENABLED;
+			break;
+		}
+		fallthrough;
 	default:
 		adapter->flags2 &= ~IXGBE_FLAG2_EEE_CAPABLE;
 		adapter->flags2 &= ~IXGBE_FLAG2_EEE_ENABLED;
@@ -8082,6 +8097,7 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
 	struct net_device *netdev = adapter->netdev;
 	struct ixgbe_hw *hw = &adapter->hw;
 	u32 link_speed = adapter->link_speed;
+	struct ethtool_keee keee = {};
 	const char *speed_str;
 	bool flow_rx, flow_tx;
 
@@ -8122,6 +8138,8 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
 	if (test_bit(__IXGBE_PTP_RUNNING, &adapter->state))
 		ixgbe_ptp_start_cyclecounter(adapter);
 
+	netdev->ethtool_ops->get_eee(netdev, &keee);
+
 	switch (link_speed) {
 	case IXGBE_LINK_SPEED_10GB_FULL:
 		speed_str = "10 Gbps";
@@ -8145,10 +8163,11 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter)
 		speed_str = "unknown speed";
 		break;
 	}
-	e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str,
+	e_info(drv, "NIC Link is Up %s, Flow Control: %s, EEE: %s\n", speed_str,
 	       ((flow_rx && flow_tx) ? "RX/TX" :
 	       (flow_rx ? "RX" :
-	       (flow_tx ? "TX" : "None"))));
+	       (flow_tx ? "TX" : "None"))),
+	       str_on_off(keee.eee_enabled));
 
 	netif_carrier_on(netdev);
 	ixgbe_check_vf_rate_limit(adapter);
@@ -12003,6 +12022,13 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 	if (err)
 		goto err_netdev;
 
+	if (hw->mac.type == ixgbe_mac_e610 &&
+	    (adapter->flags2 & IXGBE_FLAG2_EEE_CAPABLE)) {
+		bool eee_enable = adapter->flags2 & IXGBE_FLAG2_EEE_ENABLED;
+
+		hw->mac.ops.setup_eee(hw, eee_enable);
+	}
+
 	ixgbe_devlink_init_regions(adapter);
 	devl_register(adapter->devlink);
 	devl_unlock(adapter->devlink);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
index 61f2ef67defdd..a461b6542f96b 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h
@@ -2798,6 +2798,7 @@ struct ixgbe_hic_hdr2_rsp {
 };
 
 union ixgbe_hic_hdr2 {
+	u32 buf[1];
 	struct ixgbe_hic_hdr2_req req;
 	struct ixgbe_hic_hdr2_rsp rsp;
 };
@@ -3524,6 +3525,7 @@ struct ixgbe_mac_operations {
 	int (*get_link_capabilities)(struct ixgbe_hw *, ixgbe_link_speed *,
 				     bool *);
 	void (*set_rate_select_speed)(struct ixgbe_hw *, ixgbe_link_speed);
+	int (*setup_eee)(struct ixgbe_hw *hw, bool enable_eee);
 
 	/* Packet Buffer Manipulation */
 	void (*set_rxpba)(struct ixgbe_hw *, int, u32, int);
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type_e610.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type_e610.h
index ff8d640a50b1c..959cacecae499 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type_e610.h
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type_e610.h
@@ -323,10 +323,8 @@ struct ixgbe_aci_cmd_get_phy_caps_data {
 #define IXGBE_ACI_PHY_EEE_EN_100BASE_TX			BIT(0)
 #define IXGBE_ACI_PHY_EEE_EN_1000BASE_T			BIT(1)
 #define IXGBE_ACI_PHY_EEE_EN_10GBASE_T			BIT(2)
-#define IXGBE_ACI_PHY_EEE_EN_1000BASE_KX		BIT(3)
-#define IXGBE_ACI_PHY_EEE_EN_10GBASE_KR			BIT(4)
-#define IXGBE_ACI_PHY_EEE_EN_25GBASE_KR			BIT(5)
-#define IXGBE_ACI_PHY_EEE_EN_10BASE_T			BIT(11)
+#define IXGBE_ACI_PHY_EEE_EN_5GBASE_T			BIT(11)
+#define IXGBE_ACI_PHY_EEE_EN_2_5GBASE_T			BIT(12)
 	__le16 eeer_value;
 	u8 phy_id_oui[4]; /* PHY/Module ID connected on the port */
 	u8 phy_fw_ver[8];
@@ -356,7 +354,9 @@ struct ixgbe_aci_cmd_get_phy_caps_data {
 #define IXGBE_ACI_MOD_TYPE_BYTE2_SFP_PLUS		0xA0
 #define IXGBE_ACI_MOD_TYPE_BYTE2_QSFP_PLUS		0x86
 	u8 qualified_module_count;
-	u8 rsvd2[7];	/* Bytes 47:41 reserved */
+	u8 rsvd2;
+	__le16 eee_entry_delay;
+	u8 rsvd3[4];
 #define IXGBE_ACI_QUAL_MOD_COUNT_MAX			16
 	struct {
 		u8 v_oui[3];
@@ -382,6 +382,15 @@ struct ixgbe_aci_cmd_set_phy_cfg_data {
 	__le64 phy_type_low; /* Use values from IXGBE_PHY_TYPE_LOW_* */
 	__le64 phy_type_high; /* Use values from IXGBE_PHY_TYPE_HIGH_* */
 	u8 caps;
+	u8 low_power_ctrl_an;
+	__le16 eee_cap; /* Value from ixgbe_aci_get_phy_caps */
+	__le16 eeer_value; /* Use defines from ixgbe_aci_get_phy_caps */
+	u8 link_fec_opt; /* Use defines from ixgbe_aci_get_phy_caps */
+	u8 module_compliance_enforcement;
+	__le16  eee_entry_delay;
+} __packed;
+
+/* Set PHY config capabilities (@caps) defines */
 #define IXGBE_ACI_PHY_ENA_VALID_MASK		0xef
 #define IXGBE_ACI_PHY_ENA_TX_PAUSE_ABILITY	BIT(0)
 #define IXGBE_ACI_PHY_ENA_RX_PAUSE_ABILITY	BIT(1)
@@ -390,12 +399,6 @@ struct ixgbe_aci_cmd_set_phy_cfg_data {
 #define IXGBE_ACI_PHY_ENA_AUTO_LINK_UPDT	BIT(5)
 #define IXGBE_ACI_PHY_ENA_LESM			BIT(6)
 #define IXGBE_ACI_PHY_ENA_AUTO_FEC		BIT(7)
-	u8 low_power_ctrl_an;
-	__le16 eee_cap; /* Value from ixgbe_aci_get_phy_caps */
-	__le16 eeer_value; /* Use defines from ixgbe_aci_get_phy_caps */
-	u8 link_fec_opt; /* Use defines from ixgbe_aci_get_phy_caps */
-	u8 module_compliance_enforcement;
-};
 
 /* Restart AN command data structure (direct 0x0605)
  * Also used for response, with only the lport_num field present.
@@ -509,8 +512,9 @@ struct ixgbe_aci_cmd_get_link_status_data {
 #define IXGBE_ACI_LINK_SPEED_200GB		BIT(11)
 #define IXGBE_ACI_LINK_SPEED_UNKNOWN		BIT(15)
 	__le16 reserved3;
-	u8 ext_fec_status;
-#define IXGBE_ACI_LINK_RS_272_FEC_EN	BIT(0) /* RS 272 FEC enabled */
+	u8 eee_status;
+#define IXGBE_ACI_LINK_EEE_ENABLED		BIT(2)
+#define IXGBE_ACI_LINK_EEE_ACTIVE		BIT(3)
 	u8 reserved4;
 	__le64 phy_type_low; /* Use values from ICE_PHY_TYPE_LOW_* */
 	__le64 phy_type_high; /* Use values from ICE_PHY_TYPE_HIGH_* */
@@ -812,6 +816,7 @@ struct ixgbe_link_status {
 	 * of ixgbe_aci_get_phy_caps structure
 	 */
 	u8 module_type[IXGBE_ACI_MODULE_TYPE_TOTAL_BYTE];
+	u8 eee_status;
 };
 
 /* Common HW capabilities for SW use */
@@ -891,6 +896,7 @@ struct ixgbe_hw_caps {
 	u8 apm_wol_support;
 	u8 acpi_prog_mthd;
 	u8 proxy_support;
+	u8 eee_support;
 	bool nvm_update_pending_nvm;
 	bool nvm_update_pending_orom;
 	bool nvm_update_pending_netlist;
diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
index 76d2fa3ef5182..4a0ccbf448a2a 100644
--- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
+++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c
@@ -1228,7 +1228,7 @@ static int ixgbe_update_flash_X550(struct ixgbe_hw *hw)
 	buffer.req.buf_lenl = FW_SHADOW_RAM_DUMP_LEN;
 	buffer.req.checksum = FW_DEFAULT_CHECKSUM;
 
-	status = ixgbe_host_interface_command(hw, &buffer, sizeof(buffer),
+	status = ixgbe_host_interface_command(hw, buffer.buf, sizeof(buffer),
 					      IXGBE_HI_COMMAND_TIMEOUT, false);
 	return status;
 }
diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
index 42f89a179a3fa..7ce46b4b48219 100644
--- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
+++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c
@@ -64,20 +64,43 @@ static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id ixgbevf_pci_tbl[] = {
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF_HV), board_82599_vf_hv },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF_HV), board_X540_vf_hv },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF_HV), board_X550_vf_hv },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf },
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV), board_X550EM_x_vf_hv},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_VF), board_x550em_a_vf },
-	{PCI_VDEVICE_SUB(INTEL, IXGBE_DEV_ID_E610_VF, PCI_ANY_ID,
-			 IXGBE_SUBDEV_ID_E610_VF_HV), board_e610_vf_hv},
-	{PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_VF), board_e610_vf},
+	{
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF),
+		.driver_data = board_82599_vf,
+	}, {
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF_HV),
+		.driver_data = board_82599_vf_hv,
+	}, {
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF),
+		.driver_data = board_X540_vf,
+	}, {
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF_HV),
+		.driver_data = board_X540_vf_hv,
+	}, {
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF),
+		.driver_data = board_X550_vf,
+	}, {
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF_HV),
+		.driver_data = board_X550_vf_hv,
+	}, {
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF),
+		.driver_data = board_X550EM_x_vf,
+	}, {
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF_HV),
+		.driver_data = board_X550EM_x_vf_hv
+	}, {
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_A_VF),
+		.driver_data = board_x550em_a_vf,
+	}, {
+		PCI_VDEVICE_SUB(INTEL, IXGBE_DEV_ID_E610_VF,
+				PCI_ANY_ID, IXGBE_SUBDEV_ID_E610_VF_HV),
+		.driver_data = board_e610_vf_hv,
+	}, {
+		PCI_VDEVICE(INTEL, IXGBE_DEV_ID_E610_VF),
+		.driver_data = board_e610_vf,
+	},
 	/* required last entry */
-	{0, }
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
 
@@ -1221,6 +1244,7 @@ static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 		    ether_addr_equal(rx_ring->netdev->dev_addr,
 				     eth_hdr(skb)->h_source)) {
 			dev_kfree_skb_irq(skb);
+			skb = NULL;
 			continue;
 		}
 
diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
index 4f33a816bc7a1..2e94d5105016b 100644
--- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
+++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c
@@ -1294,13 +1294,18 @@ static inline void link_status_user_format(u64 lstat,
 					   struct cgx_link_user_info *linfo,
 					   struct cgx *cgx, u8 lmac_id)
 {
+	unsigned int speed;
+
 	linfo->link_up = FIELD_GET(RESP_LINKSTAT_UP, lstat);
 	linfo->full_duplex = FIELD_GET(RESP_LINKSTAT_FDUPLEX, lstat);
-	linfo->speed = cgx_speed_mbps[FIELD_GET(RESP_LINKSTAT_SPEED, lstat)];
 	linfo->an = FIELD_GET(RESP_LINKSTAT_AN, lstat);
 	linfo->fec = FIELD_GET(RESP_LINKSTAT_FEC, lstat);
 	linfo->lmac_type_id = FIELD_GET(RESP_LINKSTAT_LMAC_TYPE, lstat);
 
+	speed = FIELD_GET(RESP_LINKSTAT_SPEED, lstat);
+	linfo->speed = speed < ARRAY_SIZE(cgx_speed_mbps) ?
+		       cgx_speed_mbps[speed] : 0;
+
 	if (linfo->lmac_type_id >= LMAC_MODE_MAX) {
 		dev_err(&cgx->pdev->dev, "Unknown lmac_type_id %d reported by firmware on cgx port%d:%d",
 			linfo->lmac_type_id, cgx->cgx_id, lmac_id);
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
index a5a8f45587174..dbf1731966082 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c
@@ -619,11 +619,13 @@ static int cn20k_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id,
 		err = otx2_sync_mbox_msg(&pfvf->mbox);
 		if (err) {
 			qmem_free(pfvf->dev, pool->stack);
+			pool->stack = NULL;
 			return err;
 		}
 		aq = otx2_mbox_alloc_msg_npa_cn20k_aq_enq(&pfvf->mbox);
 		if (!aq) {
 			qmem_free(pfvf->dev, pool->stack);
+			pool->stack = NULL;
 			return -ENOMEM;
 		}
 	}
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
index 971fcab1c2483..3d253132a17f0 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.c
@@ -1482,11 +1482,13 @@ int otx2_pool_aq_init(struct otx2_nic *pfvf, u16 pool_id,
 		err = otx2_sync_mbox_msg(&pfvf->mbox);
 		if (err) {
 			qmem_free(pfvf->dev, pool->stack);
+			pool->stack = NULL;
 			return err;
 		}
 		aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox);
 		if (!aq) {
 			qmem_free(pfvf->dev, pool->stack);
+			pool->stack = NULL;
 			return -ENOMEM;
 		}
 	}
diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/rep.c b/drivers/net/ethernet/marvell/octeontx2/nic/rep.c
index 94f155ffb17f0..0f5d5642d3f7a 100644
--- a/drivers/net/ethernet/marvell/octeontx2/nic/rep.c
+++ b/drivers/net/ethernet/marvell/octeontx2/nic/rep.c
@@ -609,7 +609,7 @@ static int rvu_rep_rsrc_init(struct otx2_nic *priv)
 
 	err = otx2_init_hw_resources(priv);
 	if (err)
-		goto err_free_rsrc;
+		goto err_free_mem;
 
 	/* Set maximum frame size allowed in HW */
 	err = otx2_hw_set_mtu(priv, priv->hw.max_mtu);
@@ -621,6 +621,7 @@ static int rvu_rep_rsrc_init(struct otx2_nic *priv)
 
 err_free_rsrc:
 	otx2_free_hw_resources(priv);
+err_free_mem:
 	otx2_free_queue_mem(qset);
 	return err;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index e6b7e75894ff1..4fe6dbf0942f1 100644
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -4274,9 +4274,9 @@ int mlx4_restart_one(struct pci_dev *pdev)
 	return mlx4_restart_one_up(pdev, false, NULL);
 }
 
-#define MLX_SP(id) { PCI_VDEVICE(MELLANOX, id), MLX4_PCI_DEV_FORCE_SENSE_PORT }
-#define MLX_VF(id) { PCI_VDEVICE(MELLANOX, id), MLX4_PCI_DEV_IS_VF }
-#define MLX_GN(id) { PCI_VDEVICE(MELLANOX, id), 0 }
+#define MLX_SP(id) { PCI_VDEVICE(MELLANOX, id), .driver_data = MLX4_PCI_DEV_FORCE_SENSE_PORT }
+#define MLX_VF(id) { PCI_VDEVICE(MELLANOX, id), .driver_data = MLX4_PCI_DEV_IS_VF }
+#define MLX_GN(id) { PCI_VDEVICE(MELLANOX, id), .driver_data = 0 }
 
 static const struct pci_device_id mlx4_pci_table[] = {
 #ifdef CONFIG_MLX4_CORE_GEN2
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
index 202feab1558a3..4fe9d7d4f1438 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/alloc.c
@@ -37,10 +37,17 @@
 #include <linux/bitmap.h>
 #include <linux/dma-mapping.h>
 #include <linux/vmalloc.h>
+#include <linux/nodemask.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
 #include <linux/mlx5/driver.h>
 
 #include "mlx5_core.h"
 
+#define MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT	MLX5_ADAPTER_PAGE_SHIFT
+#define MLX5_FRAG_BUF_POOLS_NUM \
+	(PAGE_SHIFT - MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT + 1)
+
 struct mlx5_db_pgdir {
 	struct list_head	list;
 	unsigned long	       *bitmap;
@@ -48,6 +55,34 @@ struct mlx5_db_pgdir {
 	dma_addr_t		db_dma;
 };
 
+struct mlx5_dma_pool {
+	/* Protects page_list and per-page allocation bitmaps. */
+	struct mutex lock;
+	struct list_head page_list;
+	struct mlx5_core_dev *dev;
+	int node;
+	u8 block_shift;
+};
+
+struct mlx5_dma_pool_page {
+	struct mlx5_dma_pool *pool;
+	struct list_head pool_link;
+	unsigned long *bitmap;
+	void *buf;
+	dma_addr_t dma;
+};
+
+struct mlx5_frag_buf_node_pools {
+	struct mlx5_dma_pool *pools[MLX5_FRAG_BUF_POOLS_NUM];
+};
+
+struct mlx5_dma_pool_stats {
+	int node;
+	size_t block_size;
+	size_t used_blocks;
+	size_t allocated_blocks;
+};
+
 /* Handling for queue buffers -- we allocate a bunch of memory and
  * register it in a memory region at HCA virtual address 0.
  */
@@ -71,59 +106,339 @@ static void *mlx5_dma_zalloc_coherent_node(struct mlx5_core_dev *dev,
 	return cpu_handle;
 }
 
+static void mlx5_dma_pool_destroy(struct mlx5_dma_pool *pool)
+{
+	mutex_destroy(&pool->lock);
+	kfree(pool);
+}
+
+static struct mlx5_dma_pool *mlx5_dma_pool_create(struct mlx5_core_dev *dev,
+						  int node, u8 block_shift)
+{
+	struct mlx5_dma_pool *pool;
+
+	pool = kzalloc_obj(*pool);
+	if (!pool)
+		return NULL;
+
+	INIT_LIST_HEAD(&pool->page_list);
+	mutex_init(&pool->lock);
+	pool->dev = dev;
+	pool->node = node;
+	pool->block_shift = block_shift;
+	return pool;
+}
+
+static struct mlx5_dma_pool_page *
+mlx5_dma_pool_page_alloc(struct mlx5_dma_pool *pool)
+{
+	int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
+	struct mlx5_dma_pool_page *page;
+
+	page = kzalloc_obj(*page);
+	if (!page)
+		goto err_out;
+
+	page->pool = pool;
+	page->bitmap = bitmap_zalloc(blocks_per_page, GFP_KERNEL);
+	if (!page->bitmap)
+		goto err_free_page;
+
+	bitmap_fill(page->bitmap, blocks_per_page);
+	page->buf = mlx5_dma_zalloc_coherent_node(pool->dev, PAGE_SIZE,
+						  &page->dma, pool->node);
+	if (!page->buf)
+		goto err_free_bitmap;
+
+	return page;
+
+err_free_bitmap:
+	bitmap_free(page->bitmap);
+err_free_page:
+	kfree(page);
+err_out:
+	return NULL;
+}
+
+static void mlx5_dma_pool_page_free(struct mlx5_core_dev *dev,
+				    struct mlx5_dma_pool_page *page)
+{
+	dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, page->buf,
+			  page->dma);
+	bitmap_free(page->bitmap);
+	kfree(page);
+}
+
+static int mlx5_dma_pool_alloc_from_page(struct mlx5_dma_pool *pool,
+					 struct mlx5_dma_pool_page *page,
+					 unsigned long *idx_out)
+{
+	int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
+
+	*idx_out = find_first_bit(page->bitmap, blocks_per_page);
+	if (*idx_out >= blocks_per_page)
+		return -ENOMEM;
+
+	__clear_bit(*idx_out, page->bitmap);
+
+	if (bitmap_empty(page->bitmap, blocks_per_page))
+		list_move_tail(&page->pool_link, &pool->page_list);
+
+	return 0;
+}
+
+static struct mlx5_dma_pool_page *
+mlx5_dma_pool_alloc(struct mlx5_dma_pool *pool, unsigned long *idx_out)
+{
+	struct mlx5_dma_pool_page *page;
+
+	mutex_lock(&pool->lock);
+
+	page = list_first_entry_or_null(&pool->page_list,
+					struct mlx5_dma_pool_page, pool_link);
+	if (page && !mlx5_dma_pool_alloc_from_page(pool, page, idx_out))
+		goto unlock; /* successfully allocated from existing page */
+
+	page = mlx5_dma_pool_page_alloc(pool);
+	if (!page)
+		goto unlock;
+
+	list_add(&page->pool_link, &pool->page_list);
+	mlx5_dma_pool_alloc_from_page(pool, page, idx_out);
+
+unlock:
+	mutex_unlock(&pool->lock);
+	return page;
+}
+
+static void mlx5_dma_pool_free(struct mlx5_dma_pool *pool,
+			       struct mlx5_dma_pool_page *page,
+			       unsigned long idx)
+{
+	int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
+	bool was_full;
+
+	mutex_lock(&pool->lock);
+	was_full = bitmap_empty(page->bitmap, blocks_per_page);
+	__set_bit(idx, page->bitmap);
+
+	if (bitmap_full(page->bitmap, blocks_per_page)) {
+		list_del(&page->pool_link);
+		mlx5_dma_pool_page_free(pool->dev, page);
+	} else {
+		memset((u8 *)page->buf + (idx << pool->block_shift), 0,
+		       BIT(pool->block_shift));
+		if (was_full)
+			list_move(&page->pool_link, &pool->page_list);
+	}
+	mutex_unlock(&pool->lock);
+}
+
+static void mlx5_dma_pool_debugfs_get_stats(struct mlx5_dma_pool *pool,
+					    struct mlx5_dma_pool_stats *stats)
+{
+	int blocks_per_page = BIT(PAGE_SHIFT - pool->block_shift);
+	struct mlx5_dma_pool_page *page;
+	size_t free_blocks = 0;
+	size_t pages = 0;
+
+	mutex_lock(&pool->lock);
+	list_for_each_entry(page, &pool->page_list, pool_link) {
+		pages++;
+		free_blocks += bitmap_weight(page->bitmap, blocks_per_page);
+	}
+	mutex_unlock(&pool->lock);
+
+	stats->node = pool->node;
+	stats->block_size = BIT(pool->block_shift);
+	stats->allocated_blocks = pages * blocks_per_page;
+	stats->used_blocks = stats->allocated_blocks - free_blocks;
+}
+
+static void mlx5_dma_pool_debugfs_stats_print(struct seq_file *file,
+					      struct mlx5_dma_pool *pool)
+{
+	struct mlx5_dma_pool_stats stats = {};
+
+	mlx5_dma_pool_debugfs_get_stats(pool, &stats);
+	seq_printf(file, "%4d       %5zu      %7zu           %7zu\n",
+		   stats.node, stats.block_size, stats.used_blocks,
+		   stats.allocated_blocks);
+}
+
+static void mlx5_dma_pools_debugfs_print_header(struct seq_file *file)
+{
+	seq_puts(file, "node  block_size  used_blocks  allocated_blocks\n");
+}
+
+static void
+mlx5_frag_buf_node_pools_destroy(struct mlx5_frag_buf_node_pools *node_pools)
+{
+	for (int i = 0; i < MLX5_FRAG_BUF_POOLS_NUM; i++)
+		if (node_pools->pools[i])
+			mlx5_dma_pool_destroy(node_pools->pools[i]);
+	kfree(node_pools);
+}
+
+static struct mlx5_frag_buf_node_pools *
+mlx5_frag_buf_node_pools_create(struct mlx5_core_dev *dev, int node)
+{
+	struct mlx5_frag_buf_node_pools *node_pools;
+
+	node_pools = kzalloc_obj(*node_pools);
+	if (!node_pools)
+		return NULL;
+
+	for (int i = 0; i < MLX5_FRAG_BUF_POOLS_NUM; i++) {
+		u8 block_shift = MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT + i;
+
+		node_pools->pools[i] = mlx5_dma_pool_create(dev, node,
+							    block_shift);
+		if (!node_pools->pools[i]) {
+			mlx5_frag_buf_node_pools_destroy(node_pools);
+			return NULL;
+		}
+	}
+
+	return node_pools;
+}
+
+static int
+mlx5_frag_buf_dma_pools_debugfs_show(struct seq_file *file, void *priv)
+{
+	struct mlx5_core_dev *dev = file->private;
+	int node;
+
+	mlx5_dma_pools_debugfs_print_header(file);
+
+	if (!dev->priv.frag_buf_node_pools)
+		return 0;
+
+	for_each_node_state(node, N_POSSIBLE) {
+		struct mlx5_frag_buf_node_pools *node_pools;
+
+		node_pools = dev->priv.frag_buf_node_pools[node];
+		if (!node_pools)
+			continue;
+
+		for (int i = 0; i < MLX5_FRAG_BUF_POOLS_NUM; i++) {
+			struct mlx5_dma_pool *pool = node_pools->pools[i];
+
+			if (!pool)
+				continue;
+
+			mlx5_dma_pool_debugfs_stats_print(file, pool);
+		}
+	}
+
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(mlx5_frag_buf_dma_pools_debugfs);
+
+void mlx5_frag_buf_pools_cleanup(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	int node;
+
+	debugfs_remove(priv->dbg.frag_buf_dma_pools_debugfs);
+	priv->dbg.frag_buf_dma_pools_debugfs = NULL;
+
+	for_each_node_state(node, N_POSSIBLE) {
+		struct mlx5_frag_buf_node_pools *node_pools;
+
+		node_pools = priv->frag_buf_node_pools[node];
+		if (!node_pools)
+			continue;
+		mlx5_frag_buf_node_pools_destroy(node_pools);
+	}
+
+	kfree(priv->frag_buf_node_pools);
+	priv->frag_buf_node_pools = NULL;
+}
+
+int mlx5_frag_buf_pools_init(struct mlx5_core_dev *dev)
+{
+	struct mlx5_priv *priv = &dev->priv;
+	int node;
+
+	priv->frag_buf_node_pools = kzalloc_objs(*priv->frag_buf_node_pools,
+						 nr_node_ids);
+	if (!priv->frag_buf_node_pools)
+		return -ENOMEM;
+
+	for_each_node_state(node, N_POSSIBLE) {
+		struct mlx5_frag_buf_node_pools *node_pools;
+
+		node_pools = mlx5_frag_buf_node_pools_create(dev, node);
+		if (!node_pools) {
+			mlx5_frag_buf_pools_cleanup(dev);
+			return -ENOMEM;
+		}
+		priv->frag_buf_node_pools[node] = node_pools;
+	}
+
+	priv->dbg.frag_buf_dma_pools_debugfs =
+		debugfs_create_file("frag_buf_dma_pools", 0444,
+				    priv->dbg.dbg_root, dev,
+				    &mlx5_frag_buf_dma_pools_debugfs_fops);
+
+	return 0;
+}
+
 int mlx5_frag_buf_alloc_node(struct mlx5_core_dev *dev, int size,
 			     struct mlx5_frag_buf *buf, int node)
 {
-	int i;
+	struct mlx5_dma_pool *pool;
+	int pool_idx;
+
+	node = node == NUMA_NO_NODE ? numa_mem_id() : node;
 
 	buf->size = size;
 	buf->npages = DIV_ROUND_UP(size, PAGE_SIZE);
-	buf->page_shift = PAGE_SHIFT;
-	buf->frags = kzalloc_objs(struct mlx5_buf_list, buf->npages);
+	buf->page_shift = clamp_t(int, order_base_2(size),
+				  MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT,
+				  PAGE_SHIFT);
+	buf->frags = kcalloc_node(buf->npages, sizeof(*buf->frags),
+				  GFP_KERNEL, node);
 	if (!buf->frags)
-		goto err_out;
+		return -ENOMEM;
 
-	for (i = 0; i < buf->npages; i++) {
+	pool_idx = buf->page_shift - MLX5_FRAG_BUF_POOL_MIN_BLOCK_SHIFT;
+	pool = dev->priv.frag_buf_node_pools[node]->pools[pool_idx];
+	for (int i = 0; i < buf->npages; i++) {
 		struct mlx5_buf_list *frag = &buf->frags[i];
-		int frag_sz = min_t(int, size, PAGE_SIZE);
+		struct mlx5_dma_pool_page *page;
+		unsigned long idx;
 
-		frag->buf = mlx5_dma_zalloc_coherent_node(dev, frag_sz,
-							  &frag->map, node);
-		if (!frag->buf)
-			goto err_free_buf;
-		if (frag->map & ((1 << buf->page_shift) - 1)) {
-			dma_free_coherent(mlx5_core_dma_dev(dev), frag_sz,
-					  buf->frags[i].buf, buf->frags[i].map);
-			mlx5_core_warn(dev, "unexpected map alignment: %pad, page_shift=%d\n",
-				       &frag->map, buf->page_shift);
-			goto err_free_buf;
+		page = mlx5_dma_pool_alloc(pool, &idx);
+		if (!page) {
+			mlx5_frag_buf_free(dev, buf);
+			return -ENOMEM;
 		}
-		size -= frag_sz;
+		frag->buf = (u8 *)page->buf + (idx << pool->block_shift);
+		frag->map = page->dma + (idx << pool->block_shift);
+		frag->frag_page = page;
 	}
 
 	return 0;
-
-err_free_buf:
-	while (i--)
-		dma_free_coherent(mlx5_core_dma_dev(dev), PAGE_SIZE, buf->frags[i].buf,
-				  buf->frags[i].map);
-	kfree(buf->frags);
-err_out:
-	return -ENOMEM;
 }
 EXPORT_SYMBOL_GPL(mlx5_frag_buf_alloc_node);
 
 void mlx5_frag_buf_free(struct mlx5_core_dev *dev, struct mlx5_frag_buf *buf)
 {
-	int size = buf->size;
-	int i;
+	for (int i = 0; i < buf->npages; i++) {
+		struct mlx5_buf_list *frag = &buf->frags[i];
+		struct mlx5_dma_pool_page *page;
+		struct mlx5_dma_pool *pool;
+		unsigned long idx;
 
-	for (i = 0; i < buf->npages; i++) {
-		int frag_sz = min_t(int, size, PAGE_SIZE);
+		if (!frag->buf)
+			continue;
 
-		dma_free_coherent(mlx5_core_dma_dev(dev), frag_sz, buf->frags[i].buf,
-				  buf->frags[i].map);
-		size -= frag_sz;
+		page = frag->frag_page;
+		pool = page->pool;
+		idx = (frag->map - page->dma) >> pool->block_shift;
+		mlx5_dma_pool_free(pool, page, idx);
 	}
 	kfree(buf->frags);
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
index c89417c1a1f96..b5c8fbfb0eedc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c
@@ -2306,6 +2306,7 @@ int mlx5_cmd_alias_obj_create(struct mlx5_core_dev *dev,
 
 	attr = MLX5_ADDR_OF(create_alias_obj_in, in, alias_ctx);
 	MLX5_SET(alias_context, attr, vhca_id_to_be_accessed, alias_attr->vhca_id);
+	MLX5_SET(alias_context, attr, vhca_id_type, alias_attr->vhca_id_type);
 	MLX5_SET(alias_context, attr, object_id_to_be_accessed, alias_attr->obj_id);
 
 	key = MLX5_ADDR_OF(alias_context, attr, access_key);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c
index 8fe263190d38a..6347957fefcbe 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c
@@ -285,10 +285,6 @@ void mlx5_pages_debugfs_init(struct mlx5_core_dev *dev)
 	pages = dev->priv.dbg.pages_debugfs;
 
 	debugfs_create_u32("fw_pages_total", 0400, pages, &dev->priv.fw_pages);
-	debugfs_create_u32("fw_pages_vfs", 0400, pages, &dev->priv.page_counters[MLX5_VF]);
-	debugfs_create_u32("fw_pages_ec_vfs", 0400, pages, &dev->priv.page_counters[MLX5_EC_VF]);
-	debugfs_create_u32("fw_pages_sfs", 0400, pages, &dev->priv.page_counters[MLX5_SF]);
-	debugfs_create_u32("fw_pages_host_pf", 0400, pages, &dev->priv.page_counters[MLX5_HOST_PF]);
 	debugfs_create_u32("fw_pages_alloc_failed", 0400, pages, &dev->priv.fw_pages_alloc_failed);
 	debugfs_create_u32("fw_pages_give_dropped", 0400, pages, &dev->priv.give_pages_dropped);
 	debugfs_create_u32("fw_pages_reclaim_discard", 0400, pages,
@@ -300,6 +296,41 @@ void mlx5_pages_debugfs_cleanup(struct mlx5_core_dev *dev)
 	debugfs_remove_recursive(dev->priv.dbg.pages_debugfs);
 }
 
+void mlx5_pages_by_func_type_debugfs_init(struct mlx5_core_dev *dev)
+{
+	struct dentry *pages = dev->priv.dbg.pages_debugfs;
+
+	if (!pages)
+		return;
+
+	if (!dev->priv.eswitch &&
+	    MLX5_CAP_GEN(dev, icm_mng_function_id_mode) ==
+	    MLX5_ID_MODE_FUNCTION_VHCA_ID)
+		return;
+
+	debugfs_create_u32("fw_pages_vfs", 0400, pages,
+			   &dev->priv.page_counters[MLX5_VF]);
+	debugfs_create_u32("fw_pages_ec_vfs", 0400, pages,
+			   &dev->priv.page_counters[MLX5_EC_VF]);
+	debugfs_create_u32("fw_pages_sfs", 0400, pages,
+			   &dev->priv.page_counters[MLX5_SF]);
+	debugfs_create_u32("fw_pages_host_pf", 0400, pages,
+			   &dev->priv.page_counters[MLX5_HOST_PF]);
+}
+
+void mlx5_pages_by_func_type_debugfs_cleanup(struct mlx5_core_dev *dev)
+{
+	struct dentry *pages = dev->priv.dbg.pages_debugfs;
+
+	if (!pages)
+		return;
+
+	debugfs_lookup_and_remove("fw_pages_vfs", pages);
+	debugfs_lookup_and_remove("fw_pages_ec_vfs", pages);
+	debugfs_lookup_and_remove("fw_pages_sfs", pages);
+	debugfs_lookup_and_remove("fw_pages_host_pf", pages);
+}
+
 static u64 qp_read_field(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
 			 int index, int *is_str)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c
index bce72e8d1bc31..7c69d9029bfa4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/dpll.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/dpll.c
@@ -300,7 +300,8 @@ static int mlx5_dpll_state_on_dpll_set(const struct dpll_pin *pin,
 
 static int mlx5_dpll_ffo_get(const struct dpll_pin *pin, void *pin_priv,
 			     const struct dpll_device *dpll, void *dpll_priv,
-			     s64 *ffo, struct netlink_ext_ack *extack)
+			     struct dpll_ffo_param *ffo,
+			     struct netlink_ext_ack *extack)
 {
 	struct mlx5_dpll_synce_status synce_status;
 	struct mlx5_dpll *mdpll = pin_priv;
@@ -309,10 +310,11 @@ static int mlx5_dpll_ffo_get(const struct dpll_pin *pin, void *pin_priv,
 	err = mlx5_dpll_synce_status_get(mdpll->mdev, &synce_status);
 	if (err)
 		return err;
-	return mlx5_dpll_pin_ffo_get(&synce_status, ffo);
+	return mlx5_dpll_pin_ffo_get(&synce_status, &ffo->ffo);
 }
 
 static const struct dpll_pin_ops mlx5_dpll_pins_ops = {
+	.supported_ffo = BIT(DPLL_FFO_PORT_RXTX_RATE),
 	.direction_get = mlx5_dpll_pin_direction_get,
 	.state_on_dpll_get = mlx5_dpll_state_on_dpll_get,
 	.state_on_dpll_set = mlx5_dpll_state_on_dpll_set,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
index afdeb1b3d4256..8409ae73768ff 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c
@@ -160,13 +160,13 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx)
 	 * channels are being closed for other reason and this work is not
 	 * relevant anymore.
 	 */
-	while (!netdev_trylock(sq->netdev)) {
+	while (!netdev_trylock(priv->netdev)) {
 		if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state))
 			return 0;
 		msleep(20);
 	}
 
-	err = mlx5e_health_channel_eq_recover(sq->netdev, eq, sq->cq.ch_stats);
+	err = mlx5e_health_channel_eq_recover(priv->netdev, eq, sq->cq.ch_stats);
 	if (!err) {
 		to_ctx->status = 0; /* this sq recovered */
 		goto out;
@@ -186,7 +186,7 @@ static int mlx5e_tx_reporter_timeout_recover(void *ctx)
 		   "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n",
 		   err);
 out:
-	netdev_unlock(sq->netdev);
+	netdev_unlock(priv->netdev);
 	return err;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c
index 8d9a3b5ec973b..a3382f6a6b74c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.c
@@ -168,8 +168,6 @@ int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
 	return err;
 }
 
-#define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 2
-
 u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels)
 {
 	u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE,
@@ -179,13 +177,6 @@ u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels)
 	return min_t(u32, rqt_size, max_cap_rqt_size);
 }
 
-#define MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH 256
-
-unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void)
-{
-	return MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH / MLX5E_UNIFORM_SPREAD_RQT_FACTOR;
-}
-
 void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt)
 {
 	mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h
index 2f9e04a8418f1..c6d0a92b132c2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rqt.h
@@ -7,6 +7,7 @@
 #include <linux/kernel.h>
 
 #define MLX5E_INDIR_MIN_RQT_SIZE (BIT(8))
+#define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 4
 
 struct mlx5_core_dev;
 
@@ -38,7 +39,6 @@ static inline u32 mlx5e_rqt_get_rqtn(struct mlx5e_rqt *rqt)
 }
 
 u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels);
-unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void);
 int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id);
 int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids,
 			     unsigned int num_rqns,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c
index a2ec67a122d9d..de435df7ca50b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
 // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES.
 
+#include <linux/ethtool.h>
 #include "rss.h"
 
 #define mlx5e_rss_warn(__dev, format, ...)			\
@@ -85,9 +86,33 @@ bool mlx5e_rss_get_inner_ft_support(struct mlx5e_rss *rss)
 	return rss->params.inner_ft_support;
 }
 
-void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels)
+u32 *mlx5e_rss_get_indir_table(struct mlx5e_rss *rss)
 {
-	rss->indir.actual_table_size = mlx5e_rqt_size(rss->mdev, num_channels);
+	return rss->indir.table;
+}
+
+void mlx5e_rss_set_indir_actual_size(struct mlx5e_rss *rss, u32 size)
+{
+	rss->indir.actual_table_size = size;
+}
+
+/* Handles non-default contexts, replicate existing pattern into new entries,
+ * matching what ethtool_rxfh_ctxs_resize() does.
+ */
+void mlx5e_rss_ctx_resize(struct mlx5e_rss *rss, u32 new_size)
+{
+	u32 old_size = rss->indir.actual_table_size;
+	u32 i;
+
+	for (i = old_size; i < new_size; i++)
+		rss->indir.table[i] = rss->indir.table[i % old_size];
+}
+
+void mlx5e_rss_indir_resize(struct mlx5e_rss *rss, struct net_device *netdev,
+			    u32 new_size)
+{
+	ethtool_rxfh_indir_resize(netdev, rss->indir.table,
+				  rss->indir.actual_table_size, new_size);
 }
 
 int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h
index 17664757a561c..1bb0434612a41 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rss.h
@@ -34,7 +34,9 @@ struct mlx5e_rss;
 int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir,
 				u32 actual_table_size, u32 max_table_size);
 void mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir *indir);
-void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels);
+void mlx5e_rss_ctx_resize(struct mlx5e_rss *rss, u32 new_size);
+void mlx5e_rss_indir_resize(struct mlx5e_rss *rss, struct net_device *netdev,
+			    u32 new_size);
 struct mlx5e_rss *
 mlx5e_rss_init(struct mlx5_core_dev *mdev,
 	       const struct mlx5e_rss_params *params,
@@ -46,6 +48,8 @@ void mlx5e_rss_refcnt_dec(struct mlx5e_rss *rss);
 unsigned int mlx5e_rss_refcnt_read(struct mlx5e_rss *rss);
 
 bool mlx5e_rss_get_inner_ft_support(struct mlx5e_rss *rss);
+u32 *mlx5e_rss_get_indir_table(struct mlx5e_rss *rss);
+void mlx5e_rss_set_indir_actual_size(struct mlx5e_rss *rss, u32 size);
 u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
 		       bool inner);
 bool mlx5e_rss_valid_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool inner);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
index 92974b11ec752..e940635f5dcbc 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.c
@@ -40,13 +40,38 @@ static u32 *get_vhca_ids(struct mlx5e_rx_res *res, int offset)
 	return multi_vhca ? res->rss_vhca_ids + offset : NULL;
 }
 
-void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch)
+/* Updates the indirection table SW shadow, does not update the HW resources yet
+ */
+void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch,
+					  struct net_device *netdev)
 {
+	u32 new_size = mlx5e_rqt_size(res->mdev, nch);
 	int i;
 
-	for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) {
-		if (res->rss[i])
-			mlx5e_rss_params_indir_modify_actual_size(res->rss[i], nch);
+	WARN_ON_ONCE(res->rss_active);
+
+	/* Default context: fold/unfold user-configured table, then update size
+	 * and reset to uniform when unconfigured.
+	 */
+	mlx5e_rss_indir_resize(res->rss[0], netdev, new_size);
+
+	/* mlx5e_rss_indir_resize() is a no-op when the table is not
+	 * user-configured. actual_table_size is updated after the resize
+	 * because ethtool_rxfh_indir_resize() uses it as the old size to
+	 * replicate the pattern; updating it first would make the grow a no-op.
+	 * It must be updated before mlx5e_rss_set_indir_uniform() so that
+	 * the uniform fill covers all new entries, not just the old ones.
+	 */
+	mlx5e_rss_set_indir_actual_size(res->rss[0], new_size);
+	if (!netif_is_rxfh_configured(netdev))
+		mlx5e_rss_set_indir_uniform(res->rss[0], nch);
+
+	/* Non-default contexts */
+	for (i = 1; i < MLX5E_MAX_NUM_RSS; i++) {
+		if (res->rss[i]) {
+			mlx5e_rss_ctx_resize(res->rss[i], new_size);
+			mlx5e_rss_set_indir_actual_size(res->rss[i], new_size);
+		}
 	}
 }
 
@@ -209,13 +234,6 @@ static void mlx5e_rx_res_rss_disable(struct mlx5e_rx_res *res)
 	}
 }
 
-/* Updates the indirection table SW shadow, does not update the HW resources yet */
-void mlx5e_rx_res_rss_set_indir_uniform(struct mlx5e_rx_res *res, unsigned int nch)
-{
-	WARN_ON_ONCE(res->rss_active);
-	mlx5e_rss_set_indir_uniform(res->rss[0], nch);
-}
-
 void mlx5e_rx_res_rss_get_rxfh(struct mlx5e_rx_res *res, u32 rss_idx,
 			       u32 *indir, u8 *key, u8 *hfunc, bool *symmetric)
 {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
index 675780120a20d..8fff18d649787 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rx_res.h
@@ -48,7 +48,6 @@ void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *ch
 			     unsigned int ix, bool xsk);
 
 /* Configuration API */
-void mlx5e_rx_res_rss_set_indir_uniform(struct mlx5e_rx_res *res, unsigned int nch);
 void mlx5e_rx_res_rss_get_rxfh(struct mlx5e_rx_res *res, u32 rss_idx,
 			       u32 *indir, u8 *key, u8 *hfunc,
 			       bool *symmetric);
@@ -68,7 +67,8 @@ int mlx5e_rx_res_rss_destroy(struct mlx5e_rx_res *res, u32 rss_idx);
 int mlx5e_rx_res_rss_cnt(struct mlx5e_rx_res *res);
 int mlx5e_rx_res_rss_index(struct mlx5e_rx_res *res, struct mlx5e_rss *rss);
 struct mlx5e_rss *mlx5e_rx_res_rss_get(struct mlx5e_rx_res *res, u32 rss_idx);
-void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch);
+void mlx5e_rx_res_rss_update_num_channels(struct mlx5e_rx_res *res, u32 nch,
+					  struct net_device *netdev);
 
 /* Workaround for hairpin */
 struct mlx5e_rss_params_hash mlx5e_rx_res_get_current_hash(struct mlx5e_rx_res *res);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index 190b8b66b3ce1..d3bab198c99c0 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -708,7 +708,7 @@ static void mlx5e_free_xdpsq_desc(struct mlx5e_xdpsq *sq,
 				xdpi = mlx5e_xdpi_fifo_pop(xdpi_fifo);
 				page = xdpi.page.page;
 
-				/* No need to check PageNetpp() as we
+				/* No need to check page_pool_page_is_pp() as we
 				 * know this is a page_pool page.
 				 */
 				page_pool_recycle_direct(pp_page_to_nmdesc(page)->pp,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
index a52e12c3c95a2..db260e3d1412f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c
@@ -792,8 +792,10 @@ static int mlx5e_xfrm_add_state(struct net_device *dev,
 	sa_entry->dev = dev;
 	sa_entry->ipsec = ipsec;
 	/* Check if this SA is originated from acquire flow temporary SA */
-	if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ)
-		goto out;
+	if (x->xso.flags & XFRM_DEV_OFFLOAD_FLAG_ACQ) {
+		x->xso.offload_handle = (unsigned long)sa_entry;
+		return 0;
+	}
 
 	err = mlx5e_xfrm_validate_state(priv->mdev, x, extack);
 	if (err)
@@ -870,7 +872,6 @@ static int mlx5e_xfrm_add_state(struct net_device *dev,
 		xa_unlock_bh(&ipsec->sadb);
 	}
 
-out:
 	x->xso.offload_handle = (unsigned long)sa_entry;
 	if (allow_tunnel_mode)
 		mlx5_eswitch_unblock_encap(priv->mdev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
index ac8168ebb38c5..bca45679e2016 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
@@ -744,17 +744,14 @@ void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx)
 	mlx5e_ktls_priv_rx_put(priv_rx);
 }
 
-bool mlx5e_ktls_rx_handle_resync_list(struct mlx5e_channel *c, int budget)
+bool mlx5e_ktls_rx_handle_resync_list(struct mlx5e_icosq *sq, int budget)
 {
 	struct mlx5e_ktls_offload_context_rx *priv_rx, *tmp;
 	struct mlx5e_ktls_resync_resp *ktls_resync;
 	struct mlx5_wqe_ctrl_seg *db_cseg;
-	struct mlx5e_icosq *sq;
 	LIST_HEAD(local_list);
 	int i, j;
 
-	sq = c->async_icosq;
-
 	if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
 		return false;
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h
index 4022c7e78a2e8..93bd383a23d8c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_txrx.h
@@ -45,13 +45,13 @@ mlx5e_ktls_tx_try_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
 	return false;
 }
 
-bool mlx5e_ktls_rx_handle_resync_list(struct mlx5e_channel *c, int budget);
+bool mlx5e_ktls_rx_handle_resync_list(struct mlx5e_icosq *sq, int budget);
 
 static inline bool
-mlx5e_ktls_rx_pending_resync_list(struct mlx5e_channel *c, int budget)
+mlx5e_ktls_rx_pending_resync_list(struct mlx5e_icosq *sq, int budget)
 {
-	return budget && test_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC,
-				  &c->async_icosq->state);
+	return test_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state) &&
+		budget;
 }
 
 static inline void
@@ -70,13 +70,13 @@ mlx5e_ktls_tx_try_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
 }
 
 static inline bool
-mlx5e_ktls_rx_handle_resync_list(struct mlx5e_channel *c, int budget)
+mlx5e_ktls_rx_handle_resync_list(struct mlx5e_icosq *sq, int budget)
 {
 	return false;
 }
 
 static inline bool
-mlx5e_ktls_rx_pending_resync_list(struct mlx5e_channel *c, int budget)
+mlx5e_ktls_rx_pending_resync_list(struct mlx5e_icosq *sq, int budget)
 {
 	return false;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
index bb61e21790788..61993485e4517 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c
@@ -45,6 +45,10 @@
 
 #define LANES_UNKNOWN		 0
 
+#define MLX5E_MAX_INDIR_RQT_SIZE \
+	roundup_pow_of_two(MLX5E_MAX_NUM_CHANNELS * \
+			   MLX5E_UNIFORM_SPREAD_RQT_FACTOR)
+
 void mlx5e_ethtool_get_drvinfo(struct mlx5e_priv *priv,
 			       struct ethtool_drvinfo *drvinfo)
 {
@@ -495,11 +499,16 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
 {
 	struct mlx5e_params *cur_params = &priv->channels.params;
 	unsigned int count = ch->combined_count;
+	int new_rqt_size, cur_rqt_size;
 	struct mlx5e_params new_params;
+	struct mlx5e_rss *rss0;
 	bool arfs_enabled;
+	bool has_rss_ctxs;
 	bool opened;
 	int err = 0;
 
+	ASSERT_RTNL();
+
 	if (!count) {
 		netdev_info(priv->netdev, "%s: combined_count=0 not supported\n",
 			    __func__);
@@ -509,34 +518,37 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
 	if (cur_params->num_channels == count)
 		return 0;
 
-	mutex_lock(&priv->state_lock);
+	new_rqt_size = mlx5e_rqt_size(priv->mdev, count);
+	/* Validate that all non-default RSS contexts can be resized before
+	 * committing to the channel count change.
+	 * ethtool_rxfh_ctxs_can_resize() acquires rss_lock internally and
+	 * cannot be called under state_lock (rss_lock -> state_lock ordering).
+	 */
+	has_rss_ctxs = priv->rx_res && mlx5e_rx_res_rss_cnt(priv->rx_res) > 1;
+	if (has_rss_ctxs) {
+		err = ethtool_rxfh_ctxs_can_resize(priv->netdev, new_rqt_size);
+		if (err)
+			return err;
+	}
 
-	if (mlx5e_rx_res_get_current_hash(priv->rx_res).hfunc == ETH_RSS_HASH_XOR) {
-		unsigned int xor8_max_channels = mlx5e_rqt_max_num_channels_allowed_for_xor8();
+	mutex_lock(&priv->state_lock);
 
-		if (count > xor8_max_channels) {
-			err = -EINVAL;
-			netdev_err(priv->netdev, "%s: Requested number of channels (%d) exceeds the maximum allowed by the XOR8 RSS hfunc (%d)\n",
-				   __func__, count, xor8_max_channels);
-			goto out;
-		}
+	if (!priv->rx_res) {
+		err = -EINVAL;
+		goto out;
 	}
 
-	/* If RXFH is configured, changing the channels number is allowed only if
-	 * it does not require resizing the RSS table. This is because the previous
-	 * configuration may no longer be compatible with the new RSS table.
-	 */
-	if (netif_is_rxfh_configured(priv->netdev)) {
-		int cur_rqt_size = mlx5e_rqt_size(priv->mdev, cur_params->num_channels);
-		int new_rqt_size = mlx5e_rqt_size(priv->mdev, count);
-
-		if (new_rqt_size != cur_rqt_size) {
-			err = -EINVAL;
-			netdev_err(priv->netdev,
-				   "%s: RXFH is configured, block changing channels number that affects RSS table size (new: %d, current: %d)\n",
-				   __func__, new_rqt_size, cur_rqt_size);
-			goto out;
-		}
+	cur_rqt_size = mlx5e_rqt_size(priv->mdev, cur_params->num_channels);
+	rss0 = mlx5e_rx_res_rss_get(priv->rx_res, 0);
+
+	if (!ethtool_rxfh_indir_can_resize(priv->netdev,
+					   mlx5e_rss_get_indir_table(rss0),
+					   cur_rqt_size, new_rqt_size)) {
+		netdev_err(priv->netdev,
+			   "%s: cannot resize RSS table (%u -> %u); reset indirection table to allow this change\n",
+			   __func__, cur_rqt_size, new_rqt_size);
+		err = -EINVAL;
+		goto out;
 	}
 
 	/* Don't allow changing the number of channels if HTB offload is active,
@@ -584,6 +596,14 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv,
 out:
 	mutex_unlock(&priv->state_lock);
 
+	/* After a successful channel count change that altered the RQT size,
+	 * fold or unfold the indirection tables of all non-default RSS
+	 * contexts. Must run after state_lock is released because
+	 * ethtool_rxfh_ctxs_resize() acquires rss_lock internally.
+	 */
+	if (!err && cur_rqt_size != new_rqt_size && has_rss_ctxs)
+		ethtool_rxfh_ctxs_resize(priv->netdev, new_rqt_size);
+
 	return err;
 }
 
@@ -1501,29 +1521,6 @@ static int mlx5e_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *
 	return 0;
 }
 
-static int mlx5e_rxfh_hfunc_check(struct mlx5e_priv *priv,
-				  const struct ethtool_rxfh_param *rxfh,
-				  struct netlink_ext_ack *extack)
-{
-	unsigned int count;
-
-	count = priv->channels.params.num_channels;
-
-	if (rxfh->hfunc == ETH_RSS_HASH_XOR) {
-		unsigned int xor8_max_channels = mlx5e_rqt_max_num_channels_allowed_for_xor8();
-
-		if (count > xor8_max_channels) {
-			NL_SET_ERR_MSG_FMT_MOD(
-				extack,
-				"Number of channels (%u) exceeds the max for XOR8 RSS (%u)",
-				count, xor8_max_channels);
-			return -EINVAL;
-		}
-	}
-
-	return 0;
-}
-
 static int mlx5e_set_rxfh(struct net_device *dev,
 			  struct ethtool_rxfh_param *rxfh,
 			  struct netlink_ext_ack *extack)
@@ -1535,16 +1532,11 @@ static int mlx5e_set_rxfh(struct net_device *dev,
 
 	mutex_lock(&priv->state_lock);
 
-	err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack);
-	if (err)
-		goto unlock;
-
 	err = mlx5e_rx_res_rss_set_rxfh(priv->rx_res, rxfh->rss_context,
 					rxfh->indir, rxfh->key,
 					hfunc == ETH_RSS_HASH_NO_CHANGE ? NULL : &hfunc,
 					rxfh->input_xfrm == RXH_XFRM_NO_CHANGE ? NULL : &symmetric);
 
-unlock:
 	mutex_unlock(&priv->state_lock);
 	return err;
 }
@@ -1561,10 +1553,6 @@ static int mlx5e_create_rxfh_context(struct net_device *dev,
 
 	mutex_lock(&priv->state_lock);
 
-	err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack);
-	if (err)
-		goto unlock;
-
 	err = mlx5e_rx_res_rss_init(priv->rx_res, rxfh->rss_context,
 				    priv->channels.params.num_channels);
 	if (err)
@@ -1574,8 +1562,11 @@ static int mlx5e_create_rxfh_context(struct net_device *dev,
 					rxfh->indir, rxfh->key,
 					hfunc == ETH_RSS_HASH_NO_CHANGE ? NULL : &hfunc,
 					rxfh->input_xfrm == RXH_XFRM_NO_CHANGE ? NULL : &symmetric);
-	if (err)
+	if (err) {
+		WARN_ON(mlx5e_rx_res_rss_destroy(priv->rx_res,
+						 rxfh->rss_context));
 		goto unlock;
+	}
 
 	mlx5e_rx_res_rss_get_rxfh(priv->rx_res, rxfh->rss_context,
 				  ethtool_rxfh_context_indir(ctx),
@@ -1601,16 +1592,11 @@ static int mlx5e_modify_rxfh_context(struct net_device *dev,
 
 	mutex_lock(&priv->state_lock);
 
-	err = mlx5e_rxfh_hfunc_check(priv, rxfh, extack);
-	if (err)
-		goto unlock;
-
 	err = mlx5e_rx_res_rss_set_rxfh(priv->rx_res, rxfh->rss_context,
 					rxfh->indir, rxfh->key,
 					hfunc == ETH_RSS_HASH_NO_CHANGE ? NULL : &hfunc,
 					rxfh->input_xfrm == RXH_XFRM_NO_CHANGE ? NULL : &symmetric);
 
-unlock:
 	mutex_unlock(&priv->state_lock);
 	return err;
 }
@@ -2740,6 +2726,7 @@ const struct ethtool_ops mlx5e_ethtool_ops = {
 	.supported_input_xfrm = RXH_XFRM_SYM_OR_XOR,
 	.supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT |
 				 ETHTOOL_RING_USE_HDS_THRS,
+	.rxfh_indir_space = MLX5E_MAX_INDIR_RQT_SIZE,
 	.get_drvinfo       = mlx5e_get_drvinfo,
 	.get_link          = ethtool_op_get_link,
 	.get_link_ext_state  = mlx5e_get_link_ext_state,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 8f2b3abe00921..c5d26c6829a0d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -3296,12 +3296,9 @@ static int mlx5e_num_channels_changed(struct mlx5e_priv *priv)
 	}
 
 	/* This function may be called on attach, before priv->rx_res is created. */
-	if (priv->rx_res) {
-		mlx5e_rx_res_rss_update_num_channels(priv->rx_res, count);
-
-		if (!netif_is_rxfh_configured(priv->netdev))
-			mlx5e_rx_res_rss_set_indir_uniform(priv->rx_res, count);
-	}
+	if (priv->rx_res)
+		mlx5e_rx_res_rss_update_num_channels(priv->rx_res, count,
+						     netdev);
 
 	return 0;
 }
@@ -5500,6 +5497,11 @@ static void mlx5e_get_queue_stats_rx(struct net_device *dev, int i,
 	stats->bytes = rq_stats->bytes + xskrq_stats->bytes;
 	stats->alloc_fail = rq_stats->buff_alloc_err +
 			    xskrq_stats->buff_alloc_err;
+
+	stats->hw_gro_packets = rq_stats->gro_skbs + xskrq_stats->gro_skbs;
+	stats->hw_gro_wire_packets =
+		rq_stats->gro_packets + xskrq_stats->gro_packets;
+	stats->hw_gro_wire_bytes = rq_stats->gro_bytes + xskrq_stats->gro_bytes;
 }
 
 static void mlx5e_get_queue_stats_tx(struct net_device *dev, int i,
@@ -5518,6 +5520,15 @@ static void mlx5e_get_queue_stats_tx(struct net_device *dev, int i,
 	sq_stats = priv->txq2sq_stats[i];
 	stats->packets = sq_stats->packets;
 	stats->bytes = sq_stats->bytes;
+
+	stats->hw_gso_packets =
+		sq_stats->tso_packets + sq_stats->tso_inner_packets;
+	stats->hw_gso_bytes = sq_stats->tso_bytes + sq_stats->tso_inner_bytes;
+
+	stats->csum_none = sq_stats->csum_none;
+
+	stats->stop = sq_stats->stopped;
+	stats->wake = sq_stats->wake;
 }
 
 static void mlx5e_get_base_stats(struct net_device *dev,
@@ -5532,6 +5543,9 @@ static void mlx5e_get_base_stats(struct net_device *dev,
 		rx->packets = 0;
 		rx->bytes = 0;
 		rx->alloc_fail = 0;
+		rx->hw_gro_packets = 0;
+		rx->hw_gro_wire_packets = 0;
+		rx->hw_gro_wire_bytes = 0;
 
 		for (i = priv->channels.params.num_channels; i < priv->stats_nch; i++) {
 			struct netdev_queue_stats_rx rx_i = {0};
@@ -5541,6 +5555,9 @@ static void mlx5e_get_base_stats(struct net_device *dev,
 			rx->packets += rx_i.packets;
 			rx->bytes += rx_i.bytes;
 			rx->alloc_fail += rx_i.alloc_fail;
+			rx->hw_gro_packets += rx_i.hw_gro_packets;
+			rx->hw_gro_wire_packets += rx_i.hw_gro_wire_packets;
+			rx->hw_gro_wire_bytes += rx_i.hw_gro_wire_bytes;
 		}
 
 		/* always report PTP RX stats from base as there is no
@@ -5552,11 +5569,19 @@ static void mlx5e_get_base_stats(struct net_device *dev,
 
 			rx->packets += rq_stats->packets;
 			rx->bytes += rq_stats->bytes;
+			rx->hw_gro_packets += rq_stats->gro_skbs;
+			rx->hw_gro_wire_packets += rq_stats->gro_packets;
+			rx->hw_gro_wire_bytes += rq_stats->gro_bytes;
 		}
 	}
 
 	tx->packets = 0;
 	tx->bytes = 0;
+	tx->hw_gso_packets = 0;
+	tx->hw_gso_bytes = 0;
+	tx->csum_none = 0;
+	tx->stop = 0;
+	tx->wake = 0;
 
 	for (i = 0; i < priv->stats_nch; i++) {
 		struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i];
@@ -5583,6 +5608,13 @@ static void mlx5e_get_base_stats(struct net_device *dev,
 
 			tx->packets += sq_stats->packets;
 			tx->bytes += sq_stats->bytes;
+			tx->hw_gso_packets += sq_stats->tso_packets +
+					      sq_stats->tso_inner_packets;
+			tx->hw_gso_bytes += sq_stats->tso_bytes +
+					    sq_stats->tso_inner_bytes;
+			tx->csum_none += sq_stats->csum_none;
+			tx->stop += sq_stats->stopped;
+			tx->wake += sq_stats->wake;
 		}
 	}
 
@@ -5601,6 +5633,13 @@ static void mlx5e_get_base_stats(struct net_device *dev,
 
 			tx->packets += sq_stats->packets;
 			tx->bytes   += sq_stats->bytes;
+			tx->hw_gso_packets += sq_stats->tso_packets +
+					      sq_stats->tso_inner_packets;
+			tx->hw_gso_bytes += sq_stats->tso_bytes +
+					    sq_stats->tso_inner_bytes;
+			tx->csum_none += sq_stats->csum_none;
+			tx->stop += sq_stats->stopped;
+			tx->wake += sq_stats->wake;
 		}
 	}
 }
@@ -5924,7 +5963,7 @@ static void mlx5e_build_nic_netdev(struct net_device *netdev)
 
 	netdev->priv_flags       |= IFF_UNICAST_FLT;
 
-	netdev->netmem_tx = true;
+	netdev->netmem_tx = NETMEM_TX_DMA;
 
 	netif_set_tso_max_size(netdev, GSO_MAX_SIZE);
 	mlx5e_set_xdp_feature(priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
index 9f0272649fa1e..0b5e600e4a6a9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c
@@ -164,14 +164,14 @@ mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb)
 		else
 			ihs = skb_inner_tcp_all_headers(skb);
 		stats->tso_inner_packets++;
-		stats->tso_inner_bytes += skb->len - ihs;
+		stats->tso_inner_bytes += skb->len;
 	} else {
 		if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
 			ihs = skb_transport_offset(skb) + sizeof(struct udphdr);
 		else
 			ihs = skb_tcp_all_headers(skb);
 		stats->tso_packets++;
-		stats->tso_bytes += skb->len - ihs;
+		stats->tso_bytes += skb->len;
 	}
 
 	return ihs;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
index b31f689fe271c..1851056064698 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
@@ -166,14 +166,13 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 	if (unlikely(!budget))
 		goto out;
 
-	if (c->xdpsq)
-		busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq->cq);
-
-	if (c->xdp)
+	if (c->xdp) {
+		if (c->xdpsq)
+			busy |= mlx5e_poll_xdpsq_cq(&c->xdpsq->cq);
 		busy |= mlx5e_poll_xdpsq_cq(&c->rq_xdpsq.cq);
-
-	if (xsk_open)
-		work_done = mlx5e_poll_rx_cq(&xskrq->cq, budget);
+		if (xsk_open)
+			work_done += mlx5e_poll_rx_cq(&xskrq->cq, budget);
+	}
 
 	if (likely(budget - work_done))
 		work_done += mlx5e_poll_rx_cq(&rq->cq, budget - work_done);
@@ -190,20 +189,22 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 				  &aicosq->state);
 
 		/* Keep after async ICOSQ CQ poll */
-		if (unlikely(mlx5e_ktls_rx_pending_resync_list(c, budget)))
-			busy |= mlx5e_ktls_rx_handle_resync_list(c, budget);
+		if (unlikely(mlx5e_ktls_rx_pending_resync_list(aicosq, budget)))
+			busy |= mlx5e_ktls_rx_handle_resync_list(aicosq,
+								 budget);
+
+		if (xsk_open) {
+			busy |= mlx5e_poll_xdpsq_cq(&xsksq->cq);
+			busy_xsk |= mlx5e_napi_xsk_post(xsksq, xskrq);
+
+			busy |= busy_xsk;
+		}
 	}
 
 	busy |= INDIRECT_CALL_2(rq->post_wqes,
 				mlx5e_post_rx_mpwqes,
 				mlx5e_post_rx_wqes,
 				rq);
-	if (xsk_open) {
-		busy |= mlx5e_poll_xdpsq_cq(&xsksq->cq);
-		busy_xsk |= mlx5e_napi_xsk_post(xsksq, xskrq);
-	}
-
-	busy |= busy_xsk;
 
 	if (busy) {
 		if (likely(mlx5e_channel_no_affinity_change(c))) {
@@ -247,12 +248,12 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
 			mlx5e_cq_arm(&xsksq->cq);
 			mlx5e_cq_arm(&xskrq->cq);
 		}
+		if (c->xdpsq)
+			mlx5e_cq_arm(&c->xdpsq->cq);
 	}
-	if (c->xdpsq)
-		mlx5e_cq_arm(&c->xdpsq->cq);
 
 	if (unlikely(aff_change && busy_xsk)) {
-		mlx5e_trigger_irq(&c->icosq);
+		mlx5e_trigger_napi_async_icosq(c);
 		ch_stats->force_irq++;
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c
index e1d11326af1bb..8a79764345e7e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c
@@ -13,7 +13,8 @@ mlx5_esw_get_port_parent_id(struct mlx5_core_dev *dev, struct netdev_phys_item_i
 
 static bool mlx5_esw_devlink_port_supported(struct mlx5_eswitch *esw, u16 vport_num)
 {
-	return (mlx5_core_is_ecpf(esw->dev) && vport_num == MLX5_VPORT_PF) ||
+	return (mlx5_core_is_ecpf(esw->dev) &&
+		vport_num == MLX5_VPORT_HOST_PF) ||
 	       mlx5_eswitch_is_vf_vport(esw, vport_num) ||
 	       mlx5_core_is_ec_vf_vport(esw->dev, vport_num);
 }
@@ -35,7 +36,7 @@ static void mlx5_esw_offloads_pf_vf_devlink_port_attrs_set(struct mlx5_eswitch *
 	if (external)
 		controller_num = dev->priv.eswitch->offloads.host_number + 1;
 
-	if (vport_num == MLX5_VPORT_PF) {
+	if (vport_num == MLX5_VPORT_HOST_PF) {
 		memcpy(dl_port->attrs.switch_id.id, ppid.id, ppid.id_len);
 		dl_port->attrs.switch_id.id_len = ppid.id_len;
 		devlink_port_attrs_pci_pf_set(dl_port, controller_num, pfnum, external);
@@ -216,7 +217,7 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, struct mlx
 	if (err)
 		goto rate_err;
 
-	if (vport_num == MLX5_VPORT_PF) {
+	if (vport_num == MLX5_VPORT_HOST_PF) {
 		err = mlx5_esw_devlink_port_res_register(esw,
 							 &dl_port->dl_port);
 		if (err)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec.c
index da10e04777cf7..4811b60ea4301 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec.c
@@ -12,7 +12,7 @@ static int esw_ipsec_vf_query_generic(struct mlx5_core_dev *dev, u16 vport_num,
 	void *hca_cap, *query_cap;
 	int err;
 
-	if (!MLX5_CAP_GEN(dev, vhca_resource_manager))
+	if (!MLX5_CAP_GEN(dev, vport_group_manager))
 		return -EOPNOTSUPP;
 
 	if (!mlx5_esw_ipsec_vf_offload_supported(dev)) {
@@ -209,7 +209,7 @@ static int esw_ipsec_vf_offload_set_bytype(struct mlx5_eswitch *esw, struct mlx5
 	struct mlx5_core_dev *dev = esw->dev;
 	int err;
 
-	if (vport->vport == MLX5_VPORT_PF)
+	if (vport->vport == MLX5_VPORT_HOST_PF)
 		return -EOPNOTSUPP;
 
 	if (type == MLX5_ESW_VPORT_IPSEC_CRYPTO_OFFLOAD) {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c
index 3cfe743610d3f..ab50d2c734ede 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/ipsec_fs.c
@@ -142,7 +142,8 @@ static int mlx5_esw_ipsec_modify_flow_dests(struct mlx5_eswitch *esw,
 
 	attr = flow->attr;
 	esw_attr = attr->esw_attr;
-	if (esw_attr->out_count - esw_attr->split_count > 1)
+	if (!esw_attr->out_count ||
+	    esw_attr->out_count - esw_attr->split_count > 1)
 		return 0;
 
 	err = mlx5_eswitch_restore_ipsec_rule(esw, flow->rule[0], esw_attr,
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
index 123c96716a544..2f1f496f1c59d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c
@@ -806,7 +806,7 @@ static int mlx5_esw_vport_caps_get(struct mlx5_eswitch *esw, struct mlx5_vport *
 	void *hca_caps;
 	int err;
 
-	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager))
+	if (!MLX5_CAP_GEN(esw->dev, vport_group_manager))
 		return 0;
 
 	query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
@@ -852,6 +852,38 @@ bool mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id)
 	return true;
 }
 
+static enum mlx5_func_type
+esw_vport_to_func_type(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
+{
+	u16 vport_num = vport->vport;
+
+	if (vport_num == MLX5_VPORT_HOST_PF)
+		return MLX5_HOST_PF;
+	if (xa_get_mark(&esw->vports, vport_num, MLX5_ESW_VPT_SF))
+		return MLX5_SF;
+	if (xa_get_mark(&esw->vports, vport_num, MLX5_ESW_VPT_VF))
+		return MLX5_VF;
+	return MLX5_EC_VF;
+}
+
+u16 mlx5_esw_vhca_id_to_func_type(struct mlx5_core_dev *dev, u16 vhca_id)
+{
+	struct mlx5_eswitch *esw = dev->priv.eswitch;
+	void *entry;
+
+	if (vhca_id == MLX5_CAP_GEN(dev, vhca_id))
+		return MLX5_SELF;
+
+	if (!esw)
+		return MLX5_FUNC_TYPE_NONE;
+
+	entry = xa_load(&esw->vhca_type_map, vhca_id);
+	if (entry)
+		return xa_to_value(entry);
+
+	return MLX5_FUNC_TYPE_NONE;
+}
+
 static int esw_vport_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
 {
 	bool vst_mode_steering = esw_vst_mode_is_steering(esw);
@@ -908,6 +940,24 @@ static void esw_vport_cleanup(struct mlx5_eswitch *esw, struct mlx5_vport *vport
 	esw_vport_cleanup_acl(esw, vport);
 }
 
+static void mlx5_esw_vport_set_max_tx_speed(struct mlx5_eswitch *esw,
+					    struct mlx5_vport *vport)
+{
+	int ret;
+
+	if (!MLX5_CAP_ESW(esw->dev, esw_vport_state_max_tx_speed))
+		return;
+
+	ret = mlx5_modify_vport_max_tx_speed(esw->dev,
+					     MLX5_VPORT_STATE_OP_MOD_ESW_VPORT,
+					     vport->vport, true,
+					     vport->agg_max_tx_speed);
+	if (ret)
+		mlx5_core_dbg(esw->dev,
+			      "Failed to set vport %d speed %d, err=%d\n",
+			      vport->vport, vport->agg_max_tx_speed, ret);
+}
+
 int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
 			  enum mlx5_eswitch_vport_event enabled_events)
 {
@@ -926,7 +976,7 @@ int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
 	/* Sync with current vport context */
 	vport->enabled_events = enabled_events;
 	vport->enabled = true;
-	if (vport->vport != MLX5_VPORT_PF &&
+	if (vport->vport != MLX5_VPORT_HOST_PF &&
 	    (vport->info.ipsec_crypto_enabled || vport->info.ipsec_packet_enabled))
 		esw->enabled_ipsec_vf_count++;
 
@@ -938,20 +988,30 @@ int mlx5_esw_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport,
 		vport->info.trusted = true;
 
 	if (!mlx5_esw_is_manager_vport(esw, vport_num) &&
-	    MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
+	    MLX5_CAP_GEN(esw->dev, vport_group_manager)) {
 		ret = mlx5_esw_vport_vhca_id_map(esw, vport);
 		if (ret)
 			goto err_vhca_mapping;
+		ret = xa_insert(&esw->vhca_type_map, vport->vhca_id,
+				xa_mk_value(esw_vport_to_func_type(esw, vport)),
+				GFP_KERNEL);
+		if (ret)
+			goto err_type_map;
 	}
 
 	esw_vport_change_handle_locked(vport);
 
 	esw->enabled_vports++;
 	esw_debug(esw->dev, "Enabled VPORT(%d)\n", vport_num);
+
+	if (vport->agg_max_tx_speed)
+		mlx5_esw_vport_set_max_tx_speed(esw, vport);
 done:
 	mutex_unlock(&esw->state_lock);
 	return ret;
 
+err_type_map:
+	mlx5_esw_vport_vhca_id_unmap(esw, vport);
 err_vhca_mapping:
 	esw_vport_cleanup(esw, vport);
 	mutex_unlock(&esw->state_lock);
@@ -976,10 +1036,12 @@ void mlx5_esw_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vport)
 		arm_vport_context_events_cmd(esw->dev, vport_num, 0);
 
 	if (!mlx5_esw_is_manager_vport(esw, vport_num) &&
-	    MLX5_CAP_GEN(esw->dev, vhca_resource_manager))
+	    MLX5_CAP_GEN(esw->dev, vport_group_manager)) {
+		xa_erase(&esw->vhca_type_map, vport->vhca_id);
 		mlx5_esw_vport_vhca_id_unmap(esw, vport);
+	}
 
-	if (vport->vport != MLX5_VPORT_PF &&
+	if (vport->vport != MLX5_VPORT_HOST_PF &&
 	    (vport->info.ipsec_crypto_enabled || vport->info.ipsec_packet_enabled))
 		esw->enabled_ipsec_vf_count--;
 
@@ -1045,6 +1107,7 @@ const u32 *mlx5_esw_query_functions(struct mlx5_core_dev *dev)
 static int mlx5_esw_host_functions_enabled_query(struct mlx5_eswitch *esw)
 {
 	const u32 *query_host_out;
+	void *host_params;
 
 	if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
 		return 0;
@@ -1053,9 +1116,11 @@ static int mlx5_esw_host_functions_enabled_query(struct mlx5_eswitch *esw)
 	if (IS_ERR(query_host_out))
 		return PTR_ERR(query_host_out);
 
+	host_params = MLX5_ADDR_OF(query_esw_functions_out,
+				   query_host_out, net_function_params);
 	esw->esw_funcs.host_funcs_disabled =
-		MLX5_GET(query_esw_functions_out, query_host_out,
-			 host_params_context.host_pf_not_exist);
+		MLX5_GET(host_params_context, host_params,
+			 host_pf_not_exist);
 
 	kvfree(query_host_out);
 	return 0;
@@ -1070,13 +1135,17 @@ static void mlx5_eswitch_event_handler_register(struct mlx5_eswitch *esw)
 	}
 }
 
+static void mlx5_eswitch_invalidate_wq(struct mlx5_eswitch *esw)
+{
+	atomic_inc(&esw->generation);
+	wake_up_all(&esw->work_queue_wait);
+}
+
 static void mlx5_eswitch_event_handler_unregister(struct mlx5_eswitch *esw)
 {
 	if (esw->mode == MLX5_ESWITCH_OFFLOADS &&
-	    mlx5_eswitch_is_funcs_handler(esw->dev)) {
+	    mlx5_eswitch_is_funcs_handler(esw->dev))
 		mlx5_eq_notifier_unregister(esw->dev, &esw->esw_funcs.nb);
-		atomic_inc(&esw->esw_funcs.generation);
-	}
 }
 
 static void mlx5_eswitch_clear_vf_vports_info(struct mlx5_eswitch *esw)
@@ -1314,7 +1383,7 @@ int mlx5_esw_host_pf_enable_hca(struct mlx5_core_dev *dev)
 	if (!mlx5_core_is_ecpf(dev) || !mlx5_esw_allowed(esw))
 		return 0;
 
-	vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
+	vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_HOST_PF);
 	if (IS_ERR(vport))
 		return PTR_ERR(vport);
 
@@ -1340,7 +1409,7 @@ int mlx5_esw_host_pf_disable_hca(struct mlx5_core_dev *dev)
 	if (!mlx5_core_is_ecpf(dev) || !mlx5_esw_allowed(esw))
 		return 0;
 
-	vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
+	vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_HOST_PF);
 	if (IS_ERR(vport))
 		return PTR_ERR(vport);
 
@@ -1368,7 +1437,7 @@ mlx5_eswitch_enable_pf_vf_vports(struct mlx5_eswitch *esw,
 
 	/* Enable PF vport */
 	if (pf_needed && mlx5_esw_host_functions_enabled(esw->dev)) {
-		ret = mlx5_eswitch_load_pf_vf_vport(esw, MLX5_VPORT_PF,
+		ret = mlx5_eswitch_load_pf_vf_vport(esw, MLX5_VPORT_HOST_PF,
 						    enabled_events);
 		if (ret)
 			return ret;
@@ -1423,7 +1492,7 @@ mlx5_eswitch_enable_pf_vf_vports(struct mlx5_eswitch *esw,
 		mlx5_esw_host_pf_disable_hca(esw->dev);
 pf_hca_err:
 	if (pf_needed && mlx5_esw_host_functions_enabled(esw->dev))
-		mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_PF);
+		mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_HOST_PF);
 	return ret;
 }
 
@@ -1450,7 +1519,7 @@ void mlx5_eswitch_disable_pf_vf_vports(struct mlx5_eswitch *esw)
 	if ((mlx5_core_is_ecpf_esw_manager(esw->dev) ||
 	     esw->mode == MLX5_ESWITCH_LEGACY) &&
 	    mlx5_esw_host_functions_enabled(esw->dev))
-		mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_PF);
+		mlx5_eswitch_unload_pf_vf_vport(esw, MLX5_VPORT_HOST_PF);
 }
 
 static void mlx5_eswitch_get_devlink_param(struct mlx5_eswitch *esw)
@@ -1475,6 +1544,7 @@ static void mlx5_eswitch_get_devlink_param(struct mlx5_eswitch *esw)
 static void
 mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, int num_vfs)
 {
+	void *host_params;
 	const u32 *out;
 
 	if (num_vfs < 0)
@@ -1489,8 +1559,10 @@ mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, int num_vfs)
 	if (IS_ERR(out))
 		return;
 
-	esw->esw_funcs.num_vfs = MLX5_GET(query_esw_functions_out, out,
-					  host_params_context.host_num_of_vfs);
+	host_params = MLX5_ADDR_OF(query_esw_functions_out, out,
+				   net_function_params);
+	esw->esw_funcs.num_vfs = MLX5_GET(host_params_context, host_params,
+					  host_num_of_vfs);
 	if (mlx5_core_ec_sriov_enabled(esw->dev))
 		esw->esw_funcs.num_ec_vfs = num_vfs;
 
@@ -1701,6 +1773,9 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs)
 	if (toggle_lag)
 		mlx5_lag_disable_change(esw->dev);
 
+	mlx5_eswitch_invalidate_wq(esw);
+	mlx5_esw_reps_block(esw);
+
 	if (!mlx5_esw_is_fdb_created(esw)) {
 		ret = mlx5_eswitch_enable_locked(esw, num_vfs);
 	} else {
@@ -1723,6 +1798,8 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs)
 		}
 	}
 
+	mlx5_esw_reps_unblock(esw);
+
 	if (toggle_lag)
 		mlx5_lag_enable_change(esw->dev);
 
@@ -1746,6 +1823,9 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf)
 		 esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",
 		 esw->esw_funcs.num_vfs, esw->esw_funcs.num_ec_vfs, esw->enabled_vports);
 
+	mlx5_eswitch_invalidate_wq(esw);
+	mlx5_esw_reps_block(esw);
+
 	if (!mlx5_core_is_ecpf(esw->dev)) {
 		mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
 		if (clear_vf)
@@ -1756,6 +1836,8 @@ void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf)
 			mlx5_eswitch_clear_ec_vf_vports_info(esw);
 	}
 
+	mlx5_esw_reps_unblock(esw);
+
 	if (esw->mode == MLX5_ESWITCH_OFFLOADS) {
 		struct devlink *devlink = priv_to_devlink(esw->dev);
 
@@ -1785,6 +1867,7 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw)
 
 	mlx5_eq_notifier_unregister(esw->dev, &esw->nb);
 	mlx5_eswitch_event_handler_unregister(esw);
+	mlx5_eswitch_invalidate_wq(esw);
 
 	esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), necvfs(%d), active vports(%d)\n",
 		 esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS",
@@ -1810,7 +1893,11 @@ void mlx5_eswitch_disable(struct mlx5_eswitch *esw)
 
 	devl_assert_locked(priv_to_devlink(esw->dev));
 	mlx5_lag_disable_change(esw->dev);
+
+	mlx5_esw_reps_block(esw);
 	mlx5_eswitch_disable_locked(esw);
+	mlx5_esw_reps_unblock(esw);
+
 	esw->mode = MLX5_ESWITCH_LEGACY;
 	mlx5_lag_enable_change(esw->dev);
 }
@@ -1822,7 +1909,7 @@ static int mlx5_query_hca_cap_host_pf(struct mlx5_core_dev *dev, void *out)
 
 	MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 	MLX5_SET(query_hca_cap_in, in, op_mod, opmod);
-	MLX5_SET(query_hca_cap_in, in, function_id, MLX5_VPORT_PF);
+	MLX5_SET(query_hca_cap_in, in, function_id, MLX5_VPORT_HOST_PF);
 	MLX5_SET(query_hca_cap_in, in, other_function, true);
 	return mlx5_cmd_exec_inout(dev, query_hca_cap, in, out);
 }
@@ -1914,10 +2001,10 @@ static int mlx5_esw_vports_init(struct mlx5_eswitch *esw)
 	xa_init(&esw->vports);
 
 	if (mlx5_esw_host_functions_enabled(dev)) {
-		err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_PF);
+		err = mlx5_esw_vport_alloc(esw, idx, MLX5_VPORT_HOST_PF);
 		if (err)
 			goto err;
-		if (esw->first_host_vport == MLX5_VPORT_PF)
+		if (esw->first_host_vport == MLX5_VPORT_HOST_PF)
 			xa_set_mark(&esw->vports, idx, MLX5_ESW_VPT_HOST_FN);
 		idx++;
 		for (i = 0; i < mlx5_core_max_vfs(dev); i++) {
@@ -2069,9 +2156,12 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev)
 	atomic64_set(&esw->offloads.num_flows, 0);
 	ida_init(&esw->offloads.vport_metadata_ida);
 	xa_init_flags(&esw->offloads.vhca_map, XA_FLAGS_ALLOC);
+	xa_init(&esw->vhca_type_map);
 	mutex_init(&esw->state_lock);
 	init_rwsem(&esw->mode_lock);
 	refcount_set(&esw->qos.refcnt, 0);
+	init_waitqueue_head(&esw->work_queue_wait);
+	atomic_set(&esw->generation, 0);
 
 	esw->enabled_vports = 0;
 	esw->offloads.inline_mode = MLX5_INLINE_MODE_NONE;
@@ -2109,12 +2199,14 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw)
 
 	esw_info(esw->dev, "cleanup\n");
 
-	mlx5_esw_qos_cleanup(esw);
+	mlx5_eswitch_invalidate_wq(esw);
 	destroy_workqueue(esw->work_queue);
+	mlx5_esw_qos_cleanup(esw);
 	WARN_ON(refcount_read(&esw->qos.refcnt));
 	mutex_destroy(&esw->state_lock);
 	WARN_ON(!xa_empty(&esw->offloads.vhca_map));
 	xa_destroy(&esw->offloads.vhca_map);
+	xa_destroy(&esw->vhca_type_map);
 	ida_destroy(&esw->offloads.vport_metadata_ida);
 	mlx5e_mod_hdr_tbl_destroy(&esw->offloads.mod_hdr);
 	mutex_destroy(&esw->offloads.encap_tbl_lock);
@@ -2195,7 +2287,7 @@ bool mlx5_eswitch_is_vf_vport(struct mlx5_eswitch *esw, u16 vport_num)
 
 bool mlx5_eswitch_is_pf_vf_vport(struct mlx5_eswitch *esw, u16 vport_num)
 {
-	return vport_num == MLX5_VPORT_PF ||
+	return vport_num == MLX5_VPORT_HOST_PF ||
 		mlx5_eswitch_is_vf_vport(esw, vport_num);
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
index 5128f5020daec..7ac8ae6a101d6 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h
@@ -36,6 +36,7 @@
 #include <linux/if_ether.h>
 #include <linux/if_link.h>
 #include <linux/atomic.h>
+#include <linux/wait.h>
 #include <linux/xarray.h>
 #include <net/devlink.h>
 #include <linux/mlx5/device.h>
@@ -247,6 +248,7 @@ struct mlx5_vport {
 	enum mlx5_eswitch_vport_event enabled_events;
 	int index;
 	struct mlx5_devlink_port *dl_port;
+	u32 agg_max_tx_speed;
 };
 
 struct mlx5_esw_indir_table;
@@ -315,6 +317,7 @@ struct mlx5_esw_offload {
 	DECLARE_HASHTABLE(termtbl_tbl, 8);
 	struct mutex termtbl_mutex; /* protects termtbl hash */
 	struct xarray vhca_map;
+	struct mutex reps_lock; /* protects representor load/unload/register */
 	const struct mlx5_eswitch_rep_ops *rep_ops[NUM_REP_TYPES];
 	u8 inline_mode;
 	atomic64_t num_flows;
@@ -336,11 +339,11 @@ struct mlx5_host_work {
 	struct work_struct	work;
 	struct mlx5_eswitch	*esw;
 	int			work_gen;
+	void (*func)(struct mlx5_eswitch *esw);
 };
 
 struct mlx5_esw_functions {
 	struct mlx5_nb		nb;
-	atomic_t		generation;
 	bool			host_funcs_disabled;
 	u16			num_vfs;
 	u16			num_ec_vfs;
@@ -372,6 +375,7 @@ struct mlx5_eswitch {
 	struct dentry *debugfs_root;
 	struct workqueue_struct *work_queue;
 	struct xarray vports;
+	struct xarray vhca_type_map;
 	u32 flags;
 	int                     total_vports;
 	int                     enabled_vports;
@@ -385,6 +389,7 @@ struct mlx5_eswitch {
 	 */
 	struct rw_semaphore mode_lock;
 	atomic64_t user_count;
+	wait_queue_head_t work_queue_wait;
 
 	/* Protected with the E-Switch qos domain lock. */
 	struct {
@@ -410,6 +415,7 @@ struct mlx5_eswitch {
 	struct mlx5_devcom_comp_dev *devcom;
 	u16 enabled_ipsec_vf_count;
 	bool eswitch_operation_in_progress;
+	atomic_t generation;
 };
 
 void esw_offloads_disable(struct mlx5_eswitch *esw);
@@ -684,7 +690,7 @@ static inline bool mlx5_esw_is_owner(struct mlx5_eswitch *esw, u16 vport_num,
 static inline u16 mlx5_eswitch_first_host_vport_num(struct mlx5_core_dev *dev)
 {
 	return mlx5_core_is_ecpf_esw_manager(dev) ?
-		MLX5_VPORT_PF : MLX5_VPORT_FIRST_VF;
+		MLX5_VPORT_HOST_PF : MLX5_VPORT_FIRST_HOST_VF;
 }
 
 static inline bool mlx5_eswitch_is_funcs_handler(const struct mlx5_core_dev *dev)
@@ -860,6 +866,7 @@ void mlx5_esw_vport_vhca_id_unmap(struct mlx5_eswitch *esw,
 				  struct mlx5_vport *vport);
 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num);
 bool mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id);
+u16 mlx5_esw_vhca_id_to_func_type(struct mlx5_core_dev *dev, u16 vhca_id);
 
 void mlx5_esw_offloads_rep_remove(struct mlx5_eswitch *esw,
 				  const struct mlx5_vport *vport);
@@ -948,6 +955,8 @@ mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw,
 struct mlx5_flow_handle *
 mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num,
 			       struct mlx5_flow_table *lag_ft);
+void mlx5_esw_reps_block(struct mlx5_eswitch *esw);
+void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw);
 #else  /* CONFIG_MLX5_ESWITCH */
 /* eswitch API stubs */
 static inline int  mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; }
@@ -1025,12 +1034,21 @@ mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev)
 	return true;
 }
 
+static inline void mlx5_esw_reps_block(struct mlx5_eswitch *esw) {}
+static inline void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw) {}
+
 static inline bool
 mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id)
 {
 	return false;
 }
 
+static inline u16
+mlx5_esw_vhca_id_to_func_type(struct mlx5_core_dev *dev, u16 vhca_id)
+{
+	return MLX5_FUNC_TYPE_NONE;
+}
+
 static inline void
 mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev) {}
 static inline struct mlx5_flow_handle *
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
index a078d06f45673..d95af87a4f5f2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c
@@ -36,6 +36,7 @@
 #include <linux/mlx5/mlx5_ifc.h>
 #include <linux/mlx5/vport.h>
 #include <linux/mlx5/fs.h>
+#include <linux/lockdep.h>
 #include "mlx5_core.h"
 #include "eswitch.h"
 #include "esw/indir_table.h"
@@ -1216,9 +1217,10 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
 
 	if (mlx5_core_is_ecpf_esw_manager(peer_dev) &&
 	    mlx5_esw_host_functions_enabled(peer_dev)) {
-		peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF);
+		peer_vport = mlx5_eswitch_get_vport(peer_esw,
+						    MLX5_VPORT_HOST_PF);
 		esw_set_peer_miss_rule_source_port(esw, peer_esw, spec,
-						   MLX5_VPORT_PF);
+						   MLX5_VPORT_HOST_PF);
 
 		flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw),
 					   spec, &flow_act, &dest, 1);
@@ -1300,7 +1302,8 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
 
 	if (mlx5_core_is_ecpf_esw_manager(peer_dev) &&
 	    mlx5_esw_host_functions_enabled(peer_dev)) {
-		peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF);
+		peer_vport = mlx5_eswitch_get_vport(peer_esw,
+						    MLX5_VPORT_HOST_PF);
 		mlx5_del_flow_rules(flows[peer_vport->index]);
 	}
 add_pf_flow_err:
@@ -1342,7 +1345,8 @@ static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
 
 	if (mlx5_core_is_ecpf_esw_manager(peer_dev) &&
 	    mlx5_esw_host_functions_enabled(peer_dev)) {
-		peer_vport = mlx5_eswitch_get_vport(peer_esw, MLX5_VPORT_PF);
+		peer_vport = mlx5_eswitch_get_vport(peer_esw,
+						    MLX5_VPORT_HOST_PF);
 		mlx5_del_flow_rules(flows[peer_vport->index]);
 	}
 
@@ -2410,23 +2414,44 @@ static int esw_create_restore_table(struct mlx5_eswitch *esw)
 	return err;
 }
 
+static void mlx5_esw_assert_reps_locked(struct mlx5_eswitch *esw)
+{
+	lockdep_assert_held(&esw->offloads.reps_lock);
+}
+
+void mlx5_esw_reps_block(struct mlx5_eswitch *esw)
+{
+	mutex_lock(&esw->offloads.reps_lock);
+}
+
+static void mlx5_esw_reps_block_nested(struct mlx5_eswitch *esw)
+{
+	mutex_lock_nested(&esw->offloads.reps_lock, SINGLE_DEPTH_NESTING);
+}
+
+void mlx5_esw_reps_unblock(struct mlx5_eswitch *esw)
+{
+	mutex_unlock(&esw->offloads.reps_lock);
+}
+
 static void esw_mode_change(struct mlx5_eswitch *esw, u16 mode)
 {
+	mlx5_esw_reps_unblock(esw);
 	mlx5_devcom_comp_lock(esw->dev->priv.hca_devcom_comp);
 	if (esw->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_IB_ADEV ||
 	    mlx5_core_mp_enabled(esw->dev)) {
 		esw->mode = mode;
-		mlx5_rescan_drivers_locked(esw->dev);
-		mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp);
-		return;
+		goto out;
 	}
 
 	esw->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
 	mlx5_rescan_drivers_locked(esw->dev);
 	esw->mode = mode;
 	esw->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
+out:
 	mlx5_rescan_drivers_locked(esw->dev);
 	mlx5_devcom_comp_unlock(esw->dev->priv.hca_devcom_comp);
+	mlx5_esw_reps_block(esw);
 }
 
 static void mlx5_esw_fdb_drop_destroy(struct mlx5_eswitch *esw)
@@ -2642,6 +2667,7 @@ static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
 	mlx5_esw_for_each_rep(esw, i, rep)
 		mlx5_esw_offloads_rep_cleanup(esw, rep);
 	xa_destroy(&esw->offloads.vport_reps);
+	mutex_destroy(&esw->offloads.reps_lock);
 }
 
 static int esw_offloads_init_reps(struct mlx5_eswitch *esw)
@@ -2651,6 +2677,7 @@ static int esw_offloads_init_reps(struct mlx5_eswitch *esw)
 	int err;
 
 	xa_init(&esw->offloads.vport_reps);
+	mutex_init(&esw->offloads.reps_lock);
 
 	mlx5_esw_for_each_vport(esw, i, vport) {
 		err = mlx5_esw_offloads_rep_add(esw, vport);
@@ -2759,11 +2786,28 @@ void esw_offloads_cleanup(struct mlx5_eswitch *esw)
 }
 
 static int __esw_offloads_load_rep(struct mlx5_eswitch *esw,
-				   struct mlx5_eswitch_rep *rep, u8 rep_type)
+				   struct mlx5_eswitch_rep *rep,
+				   u8 rep_type, bool *newly_loaded)
 {
+	int err;
+
+	mlx5_esw_assert_reps_locked(esw);
+
+	if (newly_loaded)
+		*newly_loaded = false;
+
 	if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
-			   REP_REGISTERED, REP_LOADED) == REP_REGISTERED)
-		return esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
+			   REP_REGISTERED, REP_LOADED) != REP_REGISTERED)
+		return 0;
+
+	err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
+	if (err) {
+		atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
+		return err;
+	}
+
+	if (newly_loaded)
+		*newly_loaded = true;
 
 	return 0;
 }
@@ -2771,6 +2815,8 @@ static int __esw_offloads_load_rep(struct mlx5_eswitch *esw,
 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
 				      struct mlx5_eswitch_rep *rep, u8 rep_type)
 {
+	mlx5_esw_assert_reps_locked(esw);
+
 	if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
 			   REP_LOADED, REP_REGISTERED) == REP_LOADED) {
 		if (rep_type == REP_ETH)
@@ -2791,22 +2837,27 @@ static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
 {
 	struct mlx5_eswitch_rep *rep;
+	unsigned long loaded = 0;
+	bool newly_loaded;
 	int rep_type;
 	int err;
 
 	rep = mlx5_eswitch_get_rep(esw, vport_num);
 	for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
-		err = __esw_offloads_load_rep(esw, rep, rep_type);
+		err = __esw_offloads_load_rep(esw, rep, rep_type,
+					      &newly_loaded);
 		if (err)
 			goto err_reps;
+		if (newly_loaded)
+			loaded |= BIT(rep_type);
 	}
 
 	return 0;
 
 err_reps:
-	atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
-	for (--rep_type; rep_type >= 0; rep_type--)
-		__esw_offloads_unload_rep(esw, rep, rep_type);
+	while (--rep_type >= 0)
+		if (test_bit(rep_type, &loaded))
+			__esw_offloads_unload_rep(esw, rep, rep_type);
 	return err;
 }
 
@@ -3560,13 +3611,13 @@ int mlx5_eswitch_reload_ib_reps(struct mlx5_eswitch *esw)
 	if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
 		return 0;
 
-	ret = __esw_offloads_load_rep(esw, rep, REP_IB);
+	ret = __esw_offloads_load_rep(esw, rep, REP_IB, NULL);
 	if (ret)
 		return ret;
 
 	mlx5_esw_for_each_rep(esw, i, rep) {
 		if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED)
-			__esw_offloads_load_rep(esw, rep, REP_IB);
+			__esw_offloads_load_rep(esw, rep, REP_IB, NULL);
 	}
 
 	return 0;
@@ -3655,29 +3706,28 @@ static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
 	mutex_destroy(&esw->fdb_table.offloads.vports.lock);
 }
 
-static void
-esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, int work_gen,
-			      const u32 *out)
+static void esw_vfs_changed_event_handler(struct mlx5_eswitch *esw)
 {
-	struct devlink *devlink;
 	bool host_pf_disabled;
+	void *host_params;
 	u16 new_num_vfs;
+	const u32 *out;
 
-	devlink = priv_to_devlink(esw->dev);
-	devl_lock(devlink);
-
-	/* Stale work from one or more mode changes ago. Bail out. */
-	if (work_gen != atomic_read(&esw->esw_funcs.generation))
-		goto unlock;
+	out = mlx5_esw_query_functions(esw->dev);
+	if (IS_ERR(out))
+		return;
 
-	new_num_vfs = MLX5_GET(query_esw_functions_out, out,
-			       host_params_context.host_num_of_vfs);
-	host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
-				    host_params_context.host_pf_disabled);
+	host_params = MLX5_ADDR_OF(query_esw_functions_out, out,
+				   net_function_params);
+	new_num_vfs = MLX5_GET(host_params_context, host_params,
+			       host_num_of_vfs);
+	host_pf_disabled = MLX5_GET(host_params_context, host_params,
+				    host_pf_disabled);
 
 	if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
-		goto unlock;
+		goto free;
 
+	mlx5_esw_reps_block(esw);
 	/* Number of VFs can only change from "0 to x" or "x to 0". */
 	if (esw->esw_funcs.num_vfs > 0) {
 		mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
@@ -3686,60 +3736,98 @@ esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, int work_gen,
 
 		err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
 						  MLX5_VPORT_UC_ADDR_CHANGE);
-		if (err) {
-			devl_unlock(devlink);
-			return;
-		}
+		if (err)
+			goto unblock;
 	}
 	esw->esw_funcs.num_vfs = new_num_vfs;
-unlock:
-	devl_unlock(devlink);
+unblock:
+	mlx5_esw_reps_unblock(esw);
+free:
+	kvfree(out);
 }
 
-static void esw_functions_changed_event_handler(struct work_struct *work)
+static void esw_wq_handler(struct work_struct *work)
 {
 	struct mlx5_host_work *host_work;
 	struct mlx5_eswitch *esw;
-	const u32 *out;
+	struct devlink *devlink;
+	int work_gen;
 
 	host_work = container_of(work, struct mlx5_host_work, work);
 	esw = host_work->esw;
+	work_gen = host_work->work_gen;
+	devlink = priv_to_devlink(esw->dev);
 
-	out = mlx5_esw_query_functions(esw->dev);
-	if (IS_ERR(out))
-		goto out;
+	/* Do not block on devlink lock until stale work is filtered out.
+	 * Teardown can invalidate the generation and then wait for this
+	 * workqueue while holding devlink lock.
+	 */
+	for (;;) {
+		if (work_gen != atomic_read(&esw->generation))
+			goto free;
 
-	esw_vfs_changed_event_handler(esw, host_work->work_gen, out);
-	kvfree(out);
-out:
+		if (devl_trylock(devlink))
+			break;
+
+		wait_event_timeout(esw->work_queue_wait,
+				   work_gen != atomic_read(&esw->generation),
+				   msecs_to_jiffies(60));
+	}
+
+	/* Stale work from one or more mode changes ago. Bail out. */
+	if (work_gen != atomic_read(&esw->generation))
+		goto unlock;
+
+	host_work->func(esw);
+
+unlock:
+	devl_unlock(devlink);
+free:
 	kfree(host_work);
 }
 
-int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
+static int mlx5_esw_add_work(struct mlx5_eswitch *esw,
+			     void (*func)(struct mlx5_eswitch *esw),
+			     gfp_t gfp)
 {
-	struct mlx5_esw_functions *esw_funcs;
 	struct mlx5_host_work *host_work;
-	struct mlx5_eswitch *esw;
 
-	host_work = kzalloc_obj(*host_work, GFP_ATOMIC);
+	host_work = kzalloc_obj(*host_work, gfp);
 	if (!host_work)
-		return NOTIFY_DONE;
-
-	esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
-	esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
+		return -ENOMEM;
 
 	host_work->esw = esw;
-	host_work->work_gen = atomic_read(&esw_funcs->generation);
+	host_work->work_gen = atomic_read(&esw->generation);
 
-	INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
+	host_work->func = func;
+	INIT_WORK(&host_work->work, esw_wq_handler);
 	queue_work(esw->work_queue, &host_work->work);
 
+	return 0;
+}
+
+int mlx5_esw_funcs_changed_handler(struct notifier_block *nb,
+				   unsigned long type, void *data)
+{
+	struct mlx5_esw_functions *esw_funcs;
+	struct mlx5_eswitch *esw;
+	int ret;
+
+	esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
+	esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
+
+	ret = mlx5_esw_add_work(esw, esw_vfs_changed_event_handler,
+				GFP_ATOMIC);
+	if (ret)
+		return NOTIFY_DONE;
+
 	return NOTIFY_OK;
 }
 
 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
 {
 	const u32 *query_host_out;
+	void *host_params;
 
 	if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
 		return 0;
@@ -3749,8 +3837,10 @@ static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
 		return PTR_ERR(query_host_out);
 
 	/* Mark non local controller with non zero controller number. */
-	esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
-					     host_params_context.host_number);
+	host_params = MLX5_ADDR_OF(query_esw_functions_out,
+				   query_host_out, net_function_params);
+	esw->offloads.host_number = MLX5_GET(host_params_context,
+					     host_params, host_number);
 	kvfree(query_host_out);
 	return 0;
 }
@@ -4139,9 +4229,14 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
 		goto unlock;
 	}
 
+	/* Keep mode_lock and reps_lock unnested. The operation flag excludes
+	 * mode users while mode_lock is dropped before taking reps_lock.
+	 */
 	esw->eswitch_operation_in_progress = true;
 	up_write(&esw->mode_lock);
 
+	mlx5_esw_reps_block(esw);
+
 	if (mlx5_mode == MLX5_ESWITCH_OFFLOADS &&
 	    !mlx5_devlink_netdev_netns_immutable_set(devlink, true)) {
 		NL_SET_ERR_MSG_MOD(extack,
@@ -4174,6 +4269,10 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
 skip:
 	if (mlx5_mode == MLX5_ESWITCH_OFFLOADS && err)
 		mlx5_devlink_netdev_netns_immutable_set(devlink, false);
+	/* Reconfiguration is done; drop reps_lock before taking mode_lock again
+	 * to clear the operation flag.
+	 */
+	mlx5_esw_reps_unblock(esw);
 	down_write(&esw->mode_lock);
 	esw->eswitch_operation_in_progress = false;
 unlock:
@@ -4435,7 +4534,7 @@ static bool
 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
 {
 	/* Currently, only ECPF based device has representor for host PF. */
-	if (vport_num == MLX5_VPORT_PF &&
+	if (vport_num == MLX5_VPORT_HOST_PF &&
 	    (!mlx5_core_is_ecpf_esw_manager(esw->dev) ||
 	     !mlx5_esw_host_functions_enabled(esw->dev)))
 		return false;
@@ -4447,9 +4546,10 @@ mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
 	return true;
 }
 
-void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
-				      const struct mlx5_eswitch_rep_ops *ops,
-				      u8 rep_type)
+static void
+mlx5_eswitch_register_vport_reps_blocked(struct mlx5_eswitch *esw,
+					 const struct mlx5_eswitch_rep_ops *ops,
+					 u8 rep_type)
 {
 	struct mlx5_eswitch_rep_data *rep_data;
 	struct mlx5_eswitch_rep *rep;
@@ -4464,21 +4564,111 @@ void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
 		}
 	}
 }
+
+static void mlx5_eswitch_reload_reps_blocked(struct mlx5_eswitch *esw)
+{
+	struct mlx5_vport *vport;
+	unsigned long i;
+
+	if (esw->mode != MLX5_ESWITCH_OFFLOADS)
+		return;
+
+	if (mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK))
+		return;
+
+	mlx5_esw_for_each_vport(esw, i, vport) {
+		if (!vport)
+			continue;
+		if (!vport->enabled)
+			continue;
+		if (vport->vport == MLX5_VPORT_UPLINK)
+			continue;
+		if (!mlx5_eswitch_vport_has_rep(esw, vport->vport))
+			continue;
+
+		mlx5_esw_offloads_rep_load(esw, vport->vport);
+	}
+}
+
+static void mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw)
+{
+	mlx5_esw_reps_block(esw);
+	mlx5_eswitch_reload_reps_blocked(esw);
+	mlx5_esw_reps_unblock(esw);
+}
+
+static void
+mlx5_eswitch_register_vport_reps_locked(struct mlx5_eswitch *esw,
+					const struct mlx5_eswitch_rep_ops *ops,
+					u8 rep_type, bool nested)
+{
+	if (nested)
+		mlx5_esw_reps_block_nested(esw);
+	else
+		mlx5_esw_reps_block(esw);
+	mlx5_eswitch_register_vport_reps_blocked(esw, ops, rep_type);
+	mlx5_esw_reps_unblock(esw);
+
+	mlx5_esw_add_work(esw, mlx5_eswitch_reload_reps, GFP_KERNEL);
+}
+
+void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
+				      const struct mlx5_eswitch_rep_ops *ops,
+				      u8 rep_type)
+{
+	mlx5_eswitch_register_vport_reps_locked(esw, ops, rep_type, false);
+}
 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
 
-void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
+void
+mlx5_eswitch_register_vport_reps_nested(struct mlx5_eswitch *esw,
+					const struct mlx5_eswitch_rep_ops *ops,
+					u8 rep_type)
+{
+	mlx5_eswitch_register_vport_reps_locked(esw, ops, rep_type, true);
+}
+EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps_nested);
+
+static void
+mlx5_eswitch_unregister_vport_reps_blocked(struct mlx5_eswitch *esw,
+					   u8 rep_type)
 {
 	struct mlx5_eswitch_rep *rep;
 	unsigned long i;
 
-	if (esw->mode == MLX5_ESWITCH_OFFLOADS)
-		__unload_reps_all_vport(esw, rep_type);
+	__unload_reps_all_vport(esw, rep_type);
 
 	mlx5_esw_for_each_rep(esw, i, rep)
 		atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
+
+	esw->offloads.rep_ops[rep_type] = NULL;
+}
+
+static void
+mlx5_eswitch_unregister_vport_reps_locked(struct mlx5_eswitch *esw,
+					  u8 rep_type, bool nested)
+{
+	if (nested)
+		mlx5_esw_reps_block_nested(esw);
+	else
+		mlx5_esw_reps_block(esw);
+	mlx5_eswitch_unregister_vport_reps_blocked(esw, rep_type);
+	mlx5_esw_reps_unblock(esw);
+}
+
+void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
+{
+	mlx5_eswitch_unregister_vport_reps_locked(esw, rep_type, false);
 }
 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
 
+void mlx5_eswitch_unregister_vport_reps_nested(struct mlx5_eswitch *esw,
+					       u8 rep_type)
+{
+	mlx5_eswitch_unregister_vport_reps_locked(esw, rep_type, true);
+}
+EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps_nested);
+
 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
 {
 	struct mlx5_eswitch_rep *rep;
@@ -4640,8 +4830,9 @@ int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enab
 		return -EOPNOTSUPP;
 	}
 
-	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
-		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
+	if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Device doesn't support vport group management");
 		return -EOPNOTSUPP;
 	}
 
@@ -4716,8 +4907,9 @@ int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled,
 	struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
 	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
 
-	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
-		NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management");
+	if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) {
+		NL_SET_ERR_MSG_MOD(extack,
+				   "Device doesn't support vport group management");
 		return -EOPNOTSUPP;
 	}
 
@@ -4789,9 +4981,10 @@ int mlx5_devlink_pf_port_fn_state_get(struct devlink_port *port,
 {
 	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
 	const u32 *query_out;
+	void *host_params;
 	bool pf_disabled;
 
-	if (vport->vport != MLX5_VPORT_PF) {
+	if (vport->vport != MLX5_VPORT_HOST_PF) {
 		NL_SET_ERR_MSG_MOD(extack, "State get is not supported for VF");
 		return -EOPNOTSUPP;
 	}
@@ -4803,8 +4996,10 @@ int mlx5_devlink_pf_port_fn_state_get(struct devlink_port *port,
 	if (IS_ERR(query_out))
 		return PTR_ERR(query_out);
 
-	pf_disabled = MLX5_GET(query_esw_functions_out, query_out,
-			       host_params_context.host_pf_disabled);
+	host_params = MLX5_ADDR_OF(query_esw_functions_out, query_out,
+				   net_function_params);
+	pf_disabled = MLX5_GET(host_params_context, host_params,
+			       host_pf_disabled);
 
 	*opstate = pf_disabled ? DEVLINK_PORT_FN_OPSTATE_DETACHED :
 				 DEVLINK_PORT_FN_OPSTATE_ATTACHED;
@@ -4820,7 +5015,7 @@ int mlx5_devlink_pf_port_fn_state_set(struct devlink_port *port,
 	struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port);
 	struct mlx5_core_dev *dev;
 
-	if (vport->vport != MLX5_VPORT_PF) {
+	if (vport->vport != MLX5_VPORT_HOST_PF) {
 		NL_SET_ERR_MSG_MOD(extack, "State set is not supported for VF");
 		return -EOPNOTSUPP;
 	}
@@ -5036,9 +5231,9 @@ mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs,
 	int err;
 
 	esw = mlx5_devlink_eswitch_nocheck_get(port->devlink);
-	if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) {
+	if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) {
 		NL_SET_ERR_MSG_MOD(extack,
-				   "Device doesn't support VHCA management");
+				   "Device doesn't support vport group management");
 		return -EOPNOTSUPP;
 	}
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
index 449e4bd86c064..22b7efea34b83 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c
@@ -1063,34 +1063,147 @@ bool mlx5_lag_check_prereq(struct mlx5_lag *ldev)
 	return true;
 }
 
-void mlx5_lag_add_devices(struct mlx5_lag *ldev)
+static void mlx5_lag_assert_locked_transition(struct mlx5_lag *ldev)
 {
+	struct mlx5_devcom_comp_dev *devcom = NULL;
 	struct lag_func *pf;
 	int i;
 
+	lockdep_assert_held(&ldev->lock);
+
+	i = mlx5_get_next_ldev_func(ldev, 0);
+	if (i < MLX5_MAX_PORTS) {
+		pf = mlx5_lag_pf(ldev, i);
+		devcom = pf->dev->priv.hca_devcom_comp;
+	}
+	mlx5_devcom_comp_assert_locked(devcom);
+}
+
+static void mlx5_lag_drop_lock_for_reps(struct mlx5_lag *ldev)
+{
+	mlx5_lag_assert_locked_transition(ldev);
+
+	/* Keep PF membership stable while ldev->lock is dropped. Device add
+	 * and remove paths observe mode_changes_in_progress and retry.
+	 */
+	ldev->mode_changes_in_progress++;
+	mutex_unlock(&ldev->lock);
+}
+
+static void mlx5_lag_retake_lock_after_reps(struct mlx5_lag *ldev)
+{
+	mutex_lock(&ldev->lock);
+	ldev->mode_changes_in_progress--;
+}
+
+void mlx5_lag_rescan_dev_locked(struct mlx5_lag *ldev,
+				struct mlx5_core_dev *dev,
+				bool enable)
+{
+	if (dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
+		return;
+
+	if (enable)
+		dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
+	else
+		dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
+
+	/* Auxiliary bus probe/remove can register or unregister representor
+	 * callbacks and take reps_lock. Drop ldev->lock so the only ordering
+	 * remains reps_lock -> ldev->lock from representor callbacks.
+	 */
+	mlx5_lag_drop_lock_for_reps(ldev);
+	mlx5_rescan_drivers_locked(dev);
+	mlx5_lag_retake_lock_after_reps(ldev);
+}
+
+static void mlx5_lag_rescan_devices_locked(struct mlx5_lag *ldev, bool enable)
+{
+	struct mlx5_core_dev *devs[MLX5_MAX_PORTS];
+	struct lag_func *pf;
+	int num_devs = 0;
+	int i;
+
+	mlx5_lag_assert_locked_transition(ldev);
+
 	mlx5_ldev_for_each(i, 0, ldev) {
 		pf = mlx5_lag_pf(ldev, i);
 		if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
 			continue;
 
-		pf->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-		mlx5_rescan_drivers_locked(pf->dev);
+		if (enable)
+			pf->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
+		else
+			pf->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
+		devs[num_devs++] = pf->dev;
 	}
+
+	mlx5_lag_drop_lock_for_reps(ldev);
+	for (i = 0; i < num_devs; i++)
+		mlx5_rescan_drivers_locked(devs[i]);
+	mlx5_lag_retake_lock_after_reps(ldev);
+}
+
+void mlx5_lag_add_devices(struct mlx5_lag *ldev)
+{
+	mlx5_lag_rescan_devices_locked(ldev, true);
 }
 
 void mlx5_lag_remove_devices(struct mlx5_lag *ldev)
+{
+	mlx5_lag_rescan_devices_locked(ldev, false);
+}
+
+static int mlx5_lag_reload_ib_reps_unlocked(struct mlx5_lag *ldev, u32 flags,
+					    bool cont_on_fail)
 {
 	struct lag_func *pf;
+	int ret;
 	int i;
 
 	mlx5_ldev_for_each(i, 0, ldev) {
 		pf = mlx5_lag_pf(ldev, i);
-		if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)
-			continue;
-
-		pf->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-		mlx5_rescan_drivers_locked(pf->dev);
+		if (!(pf->dev->priv.flags & flags)) {
+			struct mlx5_eswitch *esw;
+
+			esw = pf->dev->priv.eswitch;
+			mlx5_esw_reps_block(esw);
+			ret = mlx5_eswitch_reload_ib_reps(esw);
+			mlx5_esw_reps_unblock(esw);
+			if (ret && !cont_on_fail)
+				return ret;
+		}
 	}
+
+	return 0;
+}
+
+static int mlx5_lag_reload_ib_reps(struct mlx5_lag *ldev, u32 flags,
+				   bool cont_on_fail)
+{
+	int ret;
+
+	/* The HCA devcom component lock serializes LAG mode transitions while
+	 * ldev->lock is dropped here. Dropping ldev->lock is required because
+	 * the reload takes the per-E-Switch reps_lock, and representor
+	 * load/unload callbacks can re-enter LAG netdev add/remove and take
+	 * ldev->lock. Keep the ordering reps_lock -> ldev->lock.
+	 */
+	mlx5_lag_drop_lock_for_reps(ldev);
+	ret = mlx5_lag_reload_ib_reps_unlocked(ldev, flags, cont_on_fail);
+	mlx5_lag_retake_lock_after_reps(ldev);
+
+	return ret;
+}
+
+int mlx5_lag_reload_ib_reps_from_locked(struct mlx5_lag *ldev, u32 flags,
+					bool cont_on_fail)
+{
+	int ret;
+
+	ret = mlx5_lag_reload_ib_reps(ldev, flags, cont_on_fail);
+
+	return ret;
 }
 
 void mlx5_disable_lag(struct mlx5_lag *ldev)
@@ -1111,10 +1224,7 @@ void mlx5_disable_lag(struct mlx5_lag *ldev)
 	if (shared_fdb) {
 		mlx5_lag_remove_devices(ldev);
 	} else if (roce_lag) {
-		if (!(dev0->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) {
-			dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-			mlx5_rescan_drivers_locked(dev0);
-		}
+		mlx5_lag_rescan_dev_locked(ldev, dev0, false);
 		mlx5_ldev_for_each(i, 0, ldev) {
 			if (i == idx)
 				continue;
@@ -1130,9 +1240,9 @@ void mlx5_disable_lag(struct mlx5_lag *ldev)
 		mlx5_lag_add_devices(ldev);
 
 	if (shared_fdb)
-		mlx5_ldev_for_each(i, 0, ldev)
-			if (!(mlx5_lag_pf(ldev, i)->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV))
-				mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
+		mlx5_lag_reload_ib_reps_from_locked(ldev,
+						    MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV,
+						    true);
 }
 
 bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev)
@@ -1274,6 +1384,11 @@ static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev,
 		if (vport->vport == MLX5_VPORT_UPLINK)
 			continue;
 
+		vport->agg_max_tx_speed = speed;
+
+		if (!vport->enabled)
+			continue;
+
 		ret = mlx5_modify_vport_max_tx_speed(mdev, op_mod,
 						     vport->vport, true, speed);
 		if (ret)
@@ -1388,10 +1503,9 @@ static void mlx5_do_bond(struct mlx5_lag *ldev)
 		if (err) {
 			if (shared_fdb || roce_lag)
 				mlx5_lag_add_devices(ldev);
-			if (shared_fdb) {
-				mlx5_ldev_for_each(i, 0, ldev)
-					mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
-			}
+			if (shared_fdb)
+				mlx5_lag_reload_ib_reps_from_locked(ldev, 0,
+								    true);
 
 			return;
 		}
@@ -1399,8 +1513,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev)
 		if (roce_lag) {
 			struct mlx5_core_dev *dev;
 
-			dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-			mlx5_rescan_drivers_locked(dev0);
+			mlx5_lag_rescan_dev_locked(ldev, dev0, true);
 			mlx5_ldev_for_each(i, 0, ldev) {
 				if (i == idx)
 					continue;
@@ -1409,24 +1522,15 @@ static void mlx5_do_bond(struct mlx5_lag *ldev)
 					mlx5_nic_vport_enable_roce(dev);
 			}
 		} else if (shared_fdb) {
-			int i;
-
-			dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-			mlx5_rescan_drivers_locked(dev0);
-
-			mlx5_ldev_for_each(i, 0, ldev) {
-				err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
-				if (err)
-					break;
-			}
-
+			mlx5_lag_rescan_dev_locked(ldev, dev0, true);
+			err = mlx5_lag_reload_ib_reps_from_locked(ldev, 0,
+								  false);
 			if (err) {
-				dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-				mlx5_rescan_drivers_locked(dev0);
+				mlx5_lag_rescan_dev_locked(ldev, dev0, false);
 				mlx5_deactivate_lag(ldev);
 				mlx5_lag_add_devices(ldev);
-				mlx5_ldev_for_each(i, 0, ldev)
-					mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
+				mlx5_lag_reload_ib_reps_from_locked(ldev, 0,
+								    true);
 				mlx5_core_err(dev0, "Failed to enable lag\n");
 				return;
 			}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
index 6c911374f409b..6afe7707d076e 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h
@@ -164,6 +164,9 @@ void mlx5_disable_lag(struct mlx5_lag *ldev);
 void mlx5_lag_remove_devices(struct mlx5_lag *ldev);
 int mlx5_deactivate_lag(struct mlx5_lag *ldev);
 void mlx5_lag_add_devices(struct mlx5_lag *ldev);
+void mlx5_lag_rescan_dev_locked(struct mlx5_lag *ldev,
+				struct mlx5_core_dev *dev,
+				bool enable);
 struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev);
 
 #ifdef CONFIG_MLX5_ESWITCH
@@ -199,4 +202,6 @@ int mlx5_get_next_ldev_func(struct mlx5_lag *ldev, int start_idx);
 int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq);
 int mlx5_lag_num_devs(struct mlx5_lag *ldev);
 int mlx5_lag_num_netdevs(struct mlx5_lag *ldev);
+int mlx5_lag_reload_ib_reps_from_locked(struct mlx5_lag *ldev, u32 flags,
+					bool cont_on_fail);
 #endif /* __MLX5_LAG_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
index 5eea12a6887ac..8a349f8fd8233 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c
@@ -70,7 +70,6 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
 	int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1);
 	struct mlx5_core_dev *dev0;
 	int err;
-	int i;
 
 	if (ldev->mode == MLX5_LAG_MODE_MPESW)
 		return 0;
@@ -101,26 +100,21 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev)
 		goto err_add_devices;
 	}
 
-	dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-	mlx5_rescan_drivers_locked(dev0);
-	mlx5_ldev_for_each(i, 0, ldev) {
-		err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
-		if (err)
-			goto err_rescan_drivers;
-	}
+	mlx5_lag_rescan_dev_locked(ldev, dev0, true);
+	err = mlx5_lag_reload_ib_reps_from_locked(ldev, 0, false);
+	if (err)
+		goto err_rescan_drivers;
 
 	mlx5_lag_set_vports_agg_speed(ldev);
 
 	return 0;
 
 err_rescan_drivers:
-	dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV;
-	mlx5_rescan_drivers_locked(dev0);
+	mlx5_lag_rescan_dev_locked(ldev, dev0, false);
 	mlx5_deactivate_lag(ldev);
 err_add_devices:
 	mlx5_lag_add_devices(ldev);
-	mlx5_ldev_for_each(i, 0, ldev)
-		mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch);
+	mlx5_lag_reload_ib_reps_from_locked(ldev, 0, true);
 	mlx5_mpesw_metadata_cleanup(ldev);
 	return err;
 }
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
index 4b5ac2db55ce5..d40c53193ea80 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.c
@@ -3,6 +3,7 @@
 
 #include <linux/mlx5/vport.h>
 #include <linux/list.h>
+#include <linux/lockdep.h>
 #include "lib/devcom.h"
 #include "lib/mlx5.h"
 #include "mlx5_core.h"
@@ -438,3 +439,10 @@ int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom)
 		return 0;
 	return down_write_trylock(&devcom->comp->sem);
 }
+
+void mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev *devcom)
+{
+	if (!devcom)
+		return;
+	lockdep_assert_held_write(&devcom->comp->sem);
+}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
index 91e5ae529d5c6..316052a85ca5c 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
@@ -75,5 +75,6 @@ void *mlx5_devcom_get_next_peer_data_rcu(struct mlx5_devcom_comp_dev *devcom,
 void mlx5_devcom_comp_lock(struct mlx5_devcom_comp_dev *devcom);
 void mlx5_devcom_comp_unlock(struct mlx5_devcom_comp_dev *devcom);
 int mlx5_devcom_comp_trylock(struct mlx5_devcom_comp_dev *devcom);
+void mlx5_devcom_comp_assert_locked(struct mlx5_devcom_comp_dev *devcom);
 
 #endif /* __LIB_MLX5_DEVCOM_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c
index 28cb670ba33e1..9aadb20b8b8ea 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/ipsec_fs_roce.c
@@ -116,6 +116,16 @@ static int ipsec_fs_create_aliased_ft(struct mlx5_core_dev *ibv_owner,
 	memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN);
 	alias_attr.obj_id = aliased_object_id;
 	alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
+	if (MLX5_CAP_GEN_2(ibv_owner, sw_vhca_id_valid) &&
+	    MLX5_CAP_GEN(ibv_allowed, ft_alias_sw_vhca_id)) {
+		vhca_id_to_be_accessed = MLX5_CAP_GEN_2(ibv_owner, sw_vhca_id);
+		alias_attr.vhca_id_type = VHCA_ID_TYPE_SW;
+	} else {
+		vhca_id_to_be_accessed = MLX5_CAP_GEN(ibv_owner, vhca_id);
+		alias_attr.vhca_id_type = VHCA_ID_TYPE_HW;
+		if (MLX5_CAP_GEN_2(ibv_owner, sw_vhca_id_valid))
+			mlx5_core_warn(ibv_owner, "IPsec with migration isn't supported, if migration is required update FW.\n");
+	}
 	alias_attr.vhca_id = vhca_id_to_be_accessed;
 	ret = mlx5_cmd_alias_obj_create(ibv_allowed, &alias_attr, obj_id);
 	if (ret) {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 74827e8ca1255..0c6e4efe38c83 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -597,6 +597,9 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx)
 	if (MLX5_CAP_GEN_MAX(dev, release_all_pages))
 		MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1);
 
+	if (MLX5_CAP_GEN_MAX(dev, icm_mng_function_id_mode))
+		MLX5_SET(cmd_hca_cap, set_hca_cap, icm_mng_function_id_mode, 1);
+
 	if (MLX5_CAP_GEN_MAX(dev, mkey_by_name))
 		MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1);
 
@@ -987,11 +990,12 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)
 		mlx5_core_err(dev, "Failed to init eswitch %d\n", err);
 		goto err_sriov_cleanup;
 	}
+	mlx5_pages_by_func_type_debugfs_init(dev);
 
 	err = mlx5_fpga_init(dev);
 	if (err) {
 		mlx5_core_err(dev, "Failed to init fpga device %d\n", err);
-		goto err_eswitch_cleanup;
+		goto err_page_debugfs_cleanup;
 	}
 
 	err = mlx5_vhca_event_init(dev);
@@ -1034,7 +1038,8 @@ static int mlx5_init_once(struct mlx5_core_dev *dev)
 	mlx5_vhca_event_cleanup(dev);
 err_fpga_cleanup:
 	mlx5_fpga_cleanup(dev);
-err_eswitch_cleanup:
+err_page_debugfs_cleanup:
+	mlx5_pages_by_func_type_debugfs_cleanup(dev);
 	mlx5_eswitch_cleanup(dev->priv.eswitch);
 err_sriov_cleanup:
 	mlx5_sriov_cleanup(dev);
@@ -1072,6 +1077,7 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
 	mlx5_sf_hw_table_cleanup(dev);
 	mlx5_vhca_event_cleanup(dev);
 	mlx5_fpga_cleanup(dev);
+	mlx5_pages_by_func_type_debugfs_cleanup(dev);
 	mlx5_eswitch_cleanup(dev->priv.eswitch);
 	mlx5_sriov_cleanup(dev);
 	mlx5_mpfs_cleanup(dev);
@@ -1817,6 +1823,10 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)
 	priv->dbg.dbg_root = debugfs_create_dir(dev_name(dev->device),
 						mlx5_debugfs_root);
 
+	err = mlx5_frag_buf_pools_init(dev);
+	if (err)
+		goto err_frag_buf_pools_init;
+
 	INIT_LIST_HEAD(&priv->traps);
 
 	err = mlx5_cmd_init(dev);
@@ -1878,6 +1888,8 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)
 err_timeout_init:
 	mlx5_cmd_cleanup(dev);
 err_cmd_init:
+	mlx5_frag_buf_pools_cleanup(dev);
+err_frag_buf_pools_init:
 	debugfs_remove(dev->priv.dbg.dbg_root);
 	mutex_destroy(&priv->pgdir_mutex);
 	mutex_destroy(&priv->alloc_mutex);
@@ -1902,6 +1914,7 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev)
 	mlx5_health_cleanup(dev);
 	mlx5_tout_cleanup(dev);
 	mlx5_cmd_cleanup(dev);
+	mlx5_frag_buf_pools_cleanup(dev);
 	debugfs_remove_recursive(dev->priv.dbg.dbg_root);
 	mutex_destroy(&priv->pgdir_mutex);
 	mutex_destroy(&priv->alloc_mutex);
@@ -2203,19 +2216,26 @@ static int mlx5_resume(struct pci_dev *pdev)
 
 static const struct pci_device_id mlx5_core_pci_table[] = {
 	{ PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTIB) },
-	{ PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF},	/* Connect-IB VF */
+	{ PCI_VDEVICE(MELLANOX, 0x1012),
+	  .driver_data = MLX5_PCI_DEV_IS_VF },			/* Connect-IB VF */
 	{ PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4) },
-	{ PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF},	/* ConnectX-4 VF */
+	{ PCI_VDEVICE(MELLANOX, 0x1014),
+	  .driver_data = MLX5_PCI_DEV_IS_VF },			/* ConnectX-4 VF */
 	{ PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) },
-	{ PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF},	/* ConnectX-4LX VF */
+	{ PCI_VDEVICE(MELLANOX, 0x1016),
+	  .driver_data = MLX5_PCI_DEV_IS_VF },			/* ConnectX-4LX VF */
 	{ PCI_VDEVICE(MELLANOX, 0x1017) },			/* ConnectX-5, PCIe 3.0 */
-	{ PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF},	/* ConnectX-5 VF */
+	{ PCI_VDEVICE(MELLANOX, 0x1018),
+	  .driver_data = MLX5_PCI_DEV_IS_VF },			/* ConnectX-5 VF */
 	{ PCI_VDEVICE(MELLANOX, 0x1019) },			/* ConnectX-5 Ex */
-	{ PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF},	/* ConnectX-5 Ex VF */
+	{ PCI_VDEVICE(MELLANOX, 0x101a),
+	  .driver_data = MLX5_PCI_DEV_IS_VF },			/* ConnectX-5 Ex VF */
 	{ PCI_VDEVICE(MELLANOX, 0x101b) },			/* ConnectX-6 */
-	{ PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF},	/* ConnectX-6 VF */
+	{ PCI_VDEVICE(MELLANOX, 0x101c),
+	  .driver_data = MLX5_PCI_DEV_IS_VF },			/* ConnectX-6 VF */
 	{ PCI_VDEVICE(MELLANOX, 0x101d) },			/* ConnectX-6 Dx */
-	{ PCI_VDEVICE(MELLANOX, 0x101e), MLX5_PCI_DEV_IS_VF},	/* ConnectX Family mlx5Gen Virtual Function */
+	{ PCI_VDEVICE(MELLANOX, 0x101e),
+	  .driver_data = MLX5_PCI_DEV_IS_VF },			/* ConnectX Family mlx5Gen Virtual Function */
 	{ PCI_VDEVICE(MELLANOX, 0x101f) },			/* ConnectX-6 LX */
 	{ PCI_VDEVICE(MELLANOX, 0x1021) },			/* ConnectX-7 */
 	{ PCI_VDEVICE(MELLANOX, 0x1023) },			/* ConnectX-8 */
@@ -2223,11 +2243,12 @@ static const struct pci_device_id mlx5_core_pci_table[] = {
 	{ PCI_VDEVICE(MELLANOX, 0x1027) },			/* ConnectX-10 */
 	{ PCI_VDEVICE(MELLANOX, 0x2101) },			/* ConnectX-10 NVLink-C2C */
 	{ PCI_VDEVICE(MELLANOX, 0xa2d2) },			/* BlueField integrated ConnectX-5 network controller */
-	{ PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF},	/* BlueField integrated ConnectX-5 network controller VF */
+	{ PCI_VDEVICE(MELLANOX, 0xa2d3),
+	  .driver_data = MLX5_PCI_DEV_IS_VF },			/* BlueField integrated ConnectX-5 network controller VF */
 	{ PCI_VDEVICE(MELLANOX, 0xa2d6) },			/* BlueField-2 integrated ConnectX-6 Dx network controller */
 	{ PCI_VDEVICE(MELLANOX, 0xa2dc) },			/* BlueField-3 integrated ConnectX-7 network controller */
 	{ PCI_VDEVICE(MELLANOX, 0xa2df) },			/* BlueField-4 integrated ConnectX-8 network controller */
-	{ 0, }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index 1507e881d962a..d70907f499a91 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -110,6 +110,7 @@ struct mlx5_cmd_allow_other_vhca_access_attr {
 struct mlx5_cmd_alias_obj_create_attr {
 	u32 obj_id;
 	u16 vhca_id;
+	u8 vhca_id_type;
 	u16 obj_type;
 	u8 access_key[ACCESS_KEY_LEN];
 };
@@ -436,6 +437,8 @@ mlx5_sf_coredev_to_adev(struct mlx5_core_dev *mdev)
 
 int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx);
 void mlx5_mdev_uninit(struct mlx5_core_dev *dev);
+int mlx5_frag_buf_pools_init(struct mlx5_core_dev *dev);
+void mlx5_frag_buf_pools_cleanup(struct mlx5_core_dev *dev);
 int mlx5_init_one(struct mlx5_core_dev *dev);
 int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev);
 void mlx5_uninit_one(struct mlx5_core_dev *dev);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c
index 77ffa31cc5059..ce2f7fa9bd48b 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c
@@ -38,6 +38,7 @@
 #include "mlx5_core.h"
 #include "lib/eq.h"
 #include "lib/tout.h"
+#include "eswitch.h"
 
 enum {
 	MLX5_PAGES_CANT_GIVE	= 0,
@@ -59,6 +60,7 @@ struct fw_page {
 	u64			addr;
 	struct page	       *page;
 	u32			function;
+	u16			func_type;
 	unsigned long		bitmask;
 	struct list_head	list;
 	unsigned int free_count;
@@ -69,9 +71,24 @@ enum {
 	MLX5_NUM_4K_IN_PAGE		= PAGE_SIZE / MLX5_ADAPTER_PAGE_SIZE,
 };
 
-static u32 get_function(u16 func_id, bool ec_function)
+static bool mlx5_page_mgt_mode_is_vhca_id(const struct mlx5_core_dev *dev)
 {
-	return (u32)func_id | (ec_function << 16);
+	return dev->priv.page_mgt_mode == MLX5_PAGE_MGT_MODE_VHCA_ID;
+}
+
+static void mlx5_page_mgt_mode_set(struct mlx5_core_dev *dev,
+				   enum mlx5_page_mgt_mode mode)
+{
+	dev->priv.page_mgt_mode = mode;
+}
+
+static u32 get_function_key(struct mlx5_core_dev *dev, u16 func_vhca_id,
+			    bool ec_function)
+{
+	if (mlx5_page_mgt_mode_is_vhca_id(dev))
+		return (u32)func_vhca_id;
+
+	return (u32)func_vhca_id | (ec_function << 16);
 }
 
 static u16 func_id_to_type(struct mlx5_core_dev *dev, u16 func_id, bool ec_function)
@@ -89,12 +106,21 @@ static u16 func_id_to_type(struct mlx5_core_dev *dev, u16 func_id, bool ec_funct
 	return MLX5_SF;
 }
 
+static u16 func_vhca_id_to_type(struct mlx5_core_dev *dev, u16 func_vhca_id,
+				bool ec_function)
+{
+	if (mlx5_page_mgt_mode_is_vhca_id(dev))
+		return mlx5_esw_vhca_id_to_func_type(dev, func_vhca_id);
+
+	return func_id_to_type(dev, func_vhca_id, ec_function);
+}
+
 static u32 mlx5_get_ec_function(u32 function)
 {
 	return function >> 16;
 }
 
-static u32 mlx5_get_func_id(u32 function)
+static u32 mlx5_get_func_vhca_id(u32 function)
 {
 	return function & 0xffff;
 }
@@ -123,7 +149,8 @@ static struct rb_root *page_root_per_function(struct mlx5_core_dev *dev, u32 fun
 	return root;
 }
 
-static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u32 function)
+static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page,
+		       u32 function, u16 func_type)
 {
 	struct rb_node *parent = NULL;
 	struct rb_root *root;
@@ -156,6 +183,7 @@ static int insert_page(struct mlx5_core_dev *dev, u64 addr, struct page *page, u
 	nfp->addr = addr;
 	nfp->page = page;
 	nfp->function = function;
+	nfp->func_type = func_type;
 	nfp->free_count = MLX5_NUM_4K_IN_PAGE;
 	for (i = 0; i < MLX5_NUM_4K_IN_PAGE; i++)
 		set_bit(i, &nfp->bitmask);
@@ -196,7 +224,7 @@ static struct fw_page *find_fw_page(struct mlx5_core_dev *dev, u64 addr,
 	return result;
 }
 
-static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id,
+static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_vhca_id,
 				s32 *npages, int boot)
 {
 	u32 out[MLX5_ST_SZ_DW(query_pages_out)] = {};
@@ -207,14 +235,20 @@ static int mlx5_cmd_query_pages(struct mlx5_core_dev *dev, u16 *func_id,
 	MLX5_SET(query_pages_in, in, op_mod, boot ?
 		 MLX5_QUERY_PAGES_IN_OP_MOD_BOOT_PAGES :
 		 MLX5_QUERY_PAGES_IN_OP_MOD_INIT_PAGES);
-	MLX5_SET(query_pages_in, in, embedded_cpu_function, mlx5_core_is_ecpf(dev));
+
+	if (mlx5_page_mgt_mode_is_vhca_id(dev))
+		MLX5_SET(query_pages_in, in, function_id,
+			 MLX5_CAP_GEN(dev, vhca_id));
+	else
+		MLX5_SET(query_pages_in, in, embedded_cpu_function,
+			 mlx5_core_is_ecpf(dev));
 
 	err = mlx5_cmd_exec_inout(dev, query_pages, in, out);
 	if (err)
 		return err;
 
 	*npages = MLX5_GET(query_pages_out, out, num_pages);
-	*func_id = MLX5_GET(query_pages_out, out, function_id);
+	*func_vhca_id = MLX5_GET(query_pages_out, out, function_id);
 
 	return err;
 }
@@ -245,6 +279,10 @@ static int alloc_4k(struct mlx5_core_dev *dev, u64 *addr, u32 function)
 	if (!fp->free_count)
 		list_del(&fp->list);
 
+	if (fp->func_type != MLX5_FUNC_TYPE_NONE)
+		dev->priv.page_counters[fp->func_type]++;
+	dev->priv.fw_pages++;
+
 	*addr = fp->addr + n * MLX5_ADAPTER_PAGE_SIZE;
 
 	return 0;
@@ -280,6 +318,11 @@ static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 function)
 		mlx5_core_warn_rl(dev, "page not found\n");
 		return;
 	}
+
+	if (fwp->func_type != MLX5_FUNC_TYPE_NONE)
+		dev->priv.page_counters[fwp->func_type]--;
+	dev->priv.fw_pages--;
+
 	n = (addr & ~MLX5_U64_4K_PAGE_MASK) >> MLX5_ADAPTER_PAGE_SHIFT;
 	fwp->free_count++;
 	set_bit(n, &fwp->bitmask);
@@ -289,7 +332,8 @@ static void free_4k(struct mlx5_core_dev *dev, u64 addr, u32 function)
 		list_add(&fwp->list, &dev->priv.free_list);
 }
 
-static int alloc_system_page(struct mlx5_core_dev *dev, u32 function)
+static int alloc_system_page(struct mlx5_core_dev *dev, u32 function,
+			     u16 func_type)
 {
 	struct device *device = mlx5_core_dma_dev(dev);
 	int nid = dev->priv.numa_node;
@@ -317,7 +361,7 @@ static int alloc_system_page(struct mlx5_core_dev *dev, u32 function)
 		goto map;
 	}
 
-	err = insert_page(dev, addr, page, function);
+	err = insert_page(dev, addr, page, function, func_type);
 	if (err) {
 		mlx5_core_err(dev, "failed to track allocated page\n");
 		dma_unmap_page(device, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
@@ -334,7 +378,7 @@ static int alloc_system_page(struct mlx5_core_dev *dev, u32 function)
 	return err;
 }
 
-static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id,
+static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_vhca_id,
 			     bool ec_function)
 {
 	u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {};
@@ -342,19 +386,23 @@ static void page_notify_fail(struct mlx5_core_dev *dev, u16 func_id,
 
 	MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
 	MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_CANT_GIVE);
-	MLX5_SET(manage_pages_in, in, function_id, func_id);
-	MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function);
+	MLX5_SET(manage_pages_in, in, function_id, func_vhca_id);
+
+	if (!mlx5_page_mgt_mode_is_vhca_id(dev))
+		MLX5_SET(manage_pages_in, in, embedded_cpu_function,
+			 ec_function);
 
 	err = mlx5_cmd_exec_in(dev, manage_pages, in);
 	if (err)
-		mlx5_core_warn(dev, "page notify failed func_id(%d) err(%d)\n",
-			       func_id, err);
+		mlx5_core_warn(dev,
+			       "page notify failed func_vhca_id(%d) err(%d)\n",
+			       func_vhca_id, err);
 }
 
-static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
+static int give_pages(struct mlx5_core_dev *dev, u16 func_vhca_id, int npages,
 		      int event, bool ec_function)
 {
-	u32 function = get_function(func_id, ec_function);
+	u32 function = get_function_key(dev, func_vhca_id, ec_function);
 	u32 out[MLX5_ST_SZ_DW(manage_pages_out)] = {0};
 	int inlen = MLX5_ST_SZ_BYTES(manage_pages_in);
 	int notify_fail = event;
@@ -364,6 +412,8 @@ static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 	u32 *in;
 	int i;
 
+	func_type = func_vhca_id_to_type(dev, func_vhca_id, ec_function);
+
 	inlen += npages * MLX5_FLD_SZ_BYTES(manage_pages_in, pas[0]);
 	in = kvzalloc(inlen, GFP_KERNEL);
 	if (!in) {
@@ -377,7 +427,8 @@ static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 		err = alloc_4k(dev, &addr, function);
 		if (err) {
 			if (err == -ENOMEM)
-				err = alloc_system_page(dev, function);
+				err = alloc_system_page(dev, function,
+							func_type);
 			if (err) {
 				dev->priv.fw_pages_alloc_failed += (npages - i);
 				goto out_4k;
@@ -390,9 +441,12 @@ static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 
 	MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
 	MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_GIVE);
-	MLX5_SET(manage_pages_in, in, function_id, func_id);
+	MLX5_SET(manage_pages_in, in, function_id, func_vhca_id);
 	MLX5_SET(manage_pages_in, in, input_num_entries, npages);
-	MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function);
+
+	if (!mlx5_page_mgt_mode_is_vhca_id(dev))
+		MLX5_SET(manage_pages_in, in, embedded_cpu_function,
+			 ec_function);
 
 	err = mlx5_cmd_do(dev, in, inlen, out, sizeof(out));
 	if (err == -EREMOTEIO) {
@@ -405,17 +459,15 @@ static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 	}
 	err = mlx5_cmd_check(dev, err, in, out);
 	if (err) {
-		mlx5_core_warn(dev, "func_id 0x%x, npages %d, err %d\n",
-			       func_id, npages, err);
+		mlx5_core_warn(dev, "func_vhca_id 0x%x, npages %d, err %d\n",
+			       func_vhca_id, npages, err);
 		goto out_dropped;
 	}
 
-	func_type = func_id_to_type(dev, func_id, ec_function);
-	dev->priv.page_counters[func_type] += npages;
-	dev->priv.fw_pages += npages;
-
-	mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x, err %d\n",
-		      npages, ec_function, func_id, err);
+	mlx5_core_dbg(dev,
+		      "npages %d, ec_function %d, func 0x%x, mode %d, err %d\n",
+		      npages, ec_function, func_vhca_id,
+		      mlx5_page_mgt_mode_is_vhca_id(dev), err);
 
 	kvfree(in);
 	return 0;
@@ -428,18 +480,17 @@ static int give_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 out_free:
 	kvfree(in);
 	if (notify_fail)
-		page_notify_fail(dev, func_id, ec_function);
+		page_notify_fail(dev, func_vhca_id, ec_function);
 	return err;
 }
 
-static void release_all_pages(struct mlx5_core_dev *dev, u16 func_id,
+static void release_all_pages(struct mlx5_core_dev *dev, u16 func_vhca_id,
 			      bool ec_function)
 {
-	u32 function = get_function(func_id, ec_function);
+	u32 function = get_function_key(dev, func_vhca_id, ec_function);
 	struct rb_root *root;
 	struct rb_node *p;
 	int npages = 0;
-	u16 func_type;
 
 	root = xa_load(&dev->priv.page_root_xa, function);
 	if (WARN_ON_ONCE(!root))
@@ -448,18 +499,20 @@ static void release_all_pages(struct mlx5_core_dev *dev, u16 func_id,
 	p = rb_first(root);
 	while (p) {
 		struct fw_page *fwp = rb_entry(p, struct fw_page, rb_node);
+		int used = MLX5_NUM_4K_IN_PAGE - fwp->free_count;
 
 		p = rb_next(p);
-		npages += (MLX5_NUM_4K_IN_PAGE - fwp->free_count);
+		npages += used;
+		if (fwp->func_type != MLX5_FUNC_TYPE_NONE)
+			dev->priv.page_counters[fwp->func_type] -= used;
 		free_fwp(dev, fwp, fwp->free_count);
 	}
 
-	func_type = func_id_to_type(dev, func_id, ec_function);
-	dev->priv.page_counters[func_type] -= npages;
 	dev->priv.fw_pages -= npages;
 
-	mlx5_core_dbg(dev, "npages %d, ec_function %d, func_id 0x%x\n",
-		      npages, ec_function, func_id);
+	mlx5_core_dbg(dev, "npages %d, ec_function %d, func 0x%x, mode %d\n",
+		      npages, ec_function, func_vhca_id,
+		      mlx5_page_mgt_mode_is_vhca_id(dev));
 }
 
 static u32 fwp_fill_manage_pages_out(struct fw_page *fwp, u32 *out, u32 index,
@@ -487,7 +540,7 @@ static int reclaim_pages_cmd(struct mlx5_core_dev *dev,
 	struct fw_page *fwp;
 	struct rb_node *p;
 	bool ec_function;
-	u32 func_id;
+	u32 func_vhca_id;
 	u32 npages;
 	u32 i = 0;
 	int err;
@@ -499,10 +552,11 @@ static int reclaim_pages_cmd(struct mlx5_core_dev *dev,
 
 	/* No hard feelings, we want our pages back! */
 	npages = MLX5_GET(manage_pages_in, in, input_num_entries);
-	func_id = MLX5_GET(manage_pages_in, in, function_id);
+	func_vhca_id = MLX5_GET(manage_pages_in, in, function_id);
 	ec_function = MLX5_GET(manage_pages_in, in, embedded_cpu_function);
 
-	root = xa_load(&dev->priv.page_root_xa, get_function(func_id, ec_function));
+	root = xa_load(&dev->priv.page_root_xa,
+		       get_function_key(dev, func_vhca_id, ec_function));
 	if (WARN_ON_ONCE(!root))
 		return -EEXIST;
 
@@ -518,14 +572,14 @@ static int reclaim_pages_cmd(struct mlx5_core_dev *dev,
 	return 0;
 }
 
-static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
-			 int *nclaimed, bool event, bool ec_function)
+static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_vhca_id,
+			 int npages, int *nclaimed, bool event,
+			 bool ec_function)
 {
-	u32 function = get_function(func_id, ec_function);
+	u32 function = get_function_key(dev, func_vhca_id, ec_function);
 	int outlen = MLX5_ST_SZ_BYTES(manage_pages_out);
 	u32 in[MLX5_ST_SZ_DW(manage_pages_in)] = {};
 	int num_claimed;
-	u16 func_type;
 	u32 *out;
 	int err;
 	int i;
@@ -540,12 +594,16 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 
 	MLX5_SET(manage_pages_in, in, opcode, MLX5_CMD_OP_MANAGE_PAGES);
 	MLX5_SET(manage_pages_in, in, op_mod, MLX5_PAGES_TAKE);
-	MLX5_SET(manage_pages_in, in, function_id, func_id);
+	MLX5_SET(manage_pages_in, in, function_id, func_vhca_id);
 	MLX5_SET(manage_pages_in, in, input_num_entries, npages);
-	MLX5_SET(manage_pages_in, in, embedded_cpu_function, ec_function);
 
-	mlx5_core_dbg(dev, "func 0x%x, npages %d, outlen %d\n",
-		      func_id, npages, outlen);
+	if (!mlx5_page_mgt_mode_is_vhca_id(dev))
+		MLX5_SET(manage_pages_in, in, embedded_cpu_function,
+			 ec_function);
+
+	mlx5_core_dbg(dev, "func 0x%x, npages %d, outlen %d mode %d\n",
+		      func_vhca_id, npages, outlen,
+		      mlx5_page_mgt_mode_is_vhca_id(dev));
 	err = reclaim_pages_cmd(dev, in, sizeof(in), out, outlen);
 	if (err) {
 		npages = MLX5_GET(manage_pages_in, in, input_num_entries);
@@ -577,10 +635,6 @@ static int reclaim_pages(struct mlx5_core_dev *dev, u16 func_id, int npages,
 	if (nclaimed)
 		*nclaimed = num_claimed;
 
-	func_type = func_id_to_type(dev, func_id, ec_function);
-	dev->priv.page_counters[func_type] -= num_claimed;
-	dev->priv.fw_pages -= num_claimed;
-
 out_free:
 	kvfree(out);
 	return err;
@@ -658,30 +712,102 @@ static int req_pages_handler(struct notifier_block *nb,
 	 * req->npages (and not min ()).
 	 */
 	req->npages = max_t(s32, npages, MAX_RECLAIM_NPAGES);
-	req->ec_function = ec_function;
+	if (!mlx5_page_mgt_mode_is_vhca_id(dev))
+		req->ec_function = ec_function;
 	req->release_all = release_all;
 	INIT_WORK(&req->work, pages_work_handler);
 	queue_work(dev->priv.pg_wq, &req->work);
 	return NOTIFY_OK;
 }
 
+/*
+ * After set_hca_cap(), the second satisfy_startup_pages(dev, 0) may see
+ * VHCA_ID mode. If page_root_xa already has the PF entry from the first
+ * (boot) call under FUNC_ID keys 0 or (ec_function << 16), migrate that
+ * entry to the device vhca_id key so lookups use VHCA_ID semantics.
+ */
+static int mlx5_pagealloc_migrate_pf_to_vhca_id(struct mlx5_core_dev *dev)
+{
+	u32 vhca_id_key, old_key;
+	struct rb_root *root;
+	struct fw_page *fwp;
+	struct rb_node *p;
+	bool ec_function;
+	int err;
+
+	if (xa_empty(&dev->priv.page_root_xa))
+		return 0;
+
+	vhca_id_key = MLX5_CAP_GEN(dev, vhca_id);
+	ec_function = mlx5_core_is_ecpf(dev);
+
+	old_key = ec_function ? (1U << 16) : 0;
+	root = xa_load(&dev->priv.page_root_xa, old_key);
+	if (!root)
+		return 0;
+
+	if (old_key == vhca_id_key)
+		return 0;
+
+	err = xa_insert(&dev->priv.page_root_xa, vhca_id_key, root, GFP_KERNEL);
+	if (err) {
+		mlx5_core_warn(dev,
+			       "failed to migrate page root key 0x%x to vhca_id 0x%x\n",
+			       old_key, vhca_id_key);
+		return err;
+	}
+
+	for (p = rb_first(root); p; p = rb_next(p)) {
+		fwp = rb_entry(p, struct fw_page, rb_node);
+		fwp->function = vhca_id_key;
+	}
+
+	xa_erase(&dev->priv.page_root_xa, old_key);
+
+	return 0;
+}
+
 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot)
 {
-	u16 func_id;
+	bool ec_function = false;
+	u16 func_vhca_id;
 	s32 npages;
 	int err;
 
-	err = mlx5_cmd_query_pages(dev, &func_id, &npages, boot);
+	/* Boot pages are requested before set_hca_cap(), so the capability
+	 * is not negotiated yet; use FUNC_ID mode for backward compatibility.
+	 * Init pages are requested after set_hca_cap(), which unconditionally
+	 * enables CAP_GEN_MAX. Current caps are not re-queried at this point,
+	 * so check CAP_GEN_MAX directly.
+	 */
+	if (boot) {
+		mlx5_page_mgt_mode_set(dev, MLX5_PAGE_MGT_MODE_FUNC_ID);
+	} else {
+		if (MLX5_CAP_GEN_MAX(dev, icm_mng_function_id_mode) ==
+		    MLX5_ID_MODE_FUNCTION_VHCA_ID) {
+			err = mlx5_pagealloc_migrate_pf_to_vhca_id(dev);
+			if (err)
+				return err;
+			mlx5_page_mgt_mode_set(dev, MLX5_PAGE_MGT_MODE_VHCA_ID);
+		}
+	}
+
+	err = mlx5_cmd_query_pages(dev, &func_vhca_id, &npages, boot);
 	if (err)
 		return err;
 
-	mlx5_core_dbg(dev, "requested %d %s pages for func_id 0x%x\n",
-		      npages, boot ? "boot" : "init", func_id);
+	mlx5_core_dbg(dev,
+		      "requested %d %s pages for func_vhca_id 0x%x\n",
+		      npages, boot ? "boot" : "init", func_vhca_id);
 
 	if (!npages)
 		return 0;
 
-	return give_pages(dev, func_id, npages, 0, mlx5_core_is_ecpf(dev));
+	/* In VHCA_ID mode, ec_function remains false (not used). */
+	if (!mlx5_page_mgt_mode_is_vhca_id(dev))
+		ec_function = mlx5_core_is_ecpf(dev);
+
+	return give_pages(dev, func_vhca_id, npages, 0, ec_function);
 }
 
 enum {
@@ -709,15 +835,17 @@ static int mlx5_reclaim_root_pages(struct mlx5_core_dev *dev,
 
 	while (!RB_EMPTY_ROOT(root)) {
 		u32 ec_function = mlx5_get_ec_function(function);
-		u32 function_id = mlx5_get_func_id(function);
+		u32 func_vhca_id = mlx5_get_func_vhca_id(function);
 		int nclaimed;
 		int err;
 
-		err = reclaim_pages(dev, function_id, optimal_reclaimed_pages(),
+		err = reclaim_pages(dev, func_vhca_id,
+				    optimal_reclaimed_pages(),
 				    &nclaimed, false, ec_function);
 		if (err) {
-			mlx5_core_warn(dev, "reclaim_pages err (%d) func_id=0x%x ec_func=0x%x\n",
-				       err, function_id, ec_function);
+			mlx5_core_warn(dev,
+				       "reclaim_pages err (%d) func_vhca_id=0x%x ec_func=0x%x\n",
+				       err, func_vhca_id, ec_function);
 			return err;
 		}
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c
index 8503e532f423b..2fc69897e35b4 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c
@@ -245,8 +245,10 @@ static int mlx5_sf_add(struct mlx5_core_dev *dev, struct mlx5_sf_table *table,
 	if (IS_ERR(sf))
 		return PTR_ERR(sf);
 
+	mlx5_esw_reps_block(esw);
 	err = mlx5_eswitch_load_sf_vport(esw, sf->hw_fn_id, MLX5_VPORT_UC_ADDR_CHANGE,
 					 &sf->dl_port, new_attr->controller, new_attr->sfnum);
+	mlx5_esw_reps_unblock(esw);
 	if (err)
 		goto esw_err;
 	*dl_port = &sf->dl_port.dl_port;
@@ -367,7 +369,10 @@ int mlx5_devlink_sf_port_del(struct devlink *devlink,
 	struct mlx5_sf_table *table = dev->priv.sf_table;
 	struct mlx5_sf *sf = mlx5_sf_by_dl_port(dl_port);
 
+	mlx5_esw_reps_block(dev->priv.eswitch);
 	mlx5_sf_del(table, sf);
+	mlx5_esw_reps_unblock(dev->priv.eswitch);
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h b/drivers/net/ethernet/mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h
index 4fc870140d710..487c94b562030 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/mlx5_ifc_vhca_event.h
@@ -4,14 +4,6 @@
 #ifndef __MLX5_IFC_VHCA_EVENT_H__
 #define __MLX5_IFC_VHCA_EVENT_H__
 
-enum mlx5_ifc_vhca_state {
-	MLX5_VHCA_STATE_INVALID = 0x0,
-	MLX5_VHCA_STATE_ALLOCATED = 0x1,
-	MLX5_VHCA_STATE_ACTIVE = 0x2,
-	MLX5_VHCA_STATE_IN_USE = 0x3,
-	MLX5_VHCA_STATE_TEARDOWN_REQUEST = 0x4,
-};
-
 struct mlx5_ifc_vhca_state_context_bits {
 	u8         arm_change_event[0x1];
 	u8         reserved_at_1[0xb];
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
index bf6f631cf2ce0..6eb6026eadd62 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c
@@ -274,6 +274,7 @@ void mlx5_sriov_detach(struct mlx5_core_dev *dev)
 static u16 mlx5_get_max_vfs(struct mlx5_core_dev *dev)
 {
 	u16 host_total_vfs;
+	void *host_params;
 	const u32 *out;
 
 	if (mlx5_core_is_ecpf_esw_manager(dev)) {
@@ -284,8 +285,10 @@ static u16 mlx5_get_max_vfs(struct mlx5_core_dev *dev)
 		 */
 		if (IS_ERR(out))
 			goto done;
-		host_total_vfs = MLX5_GET(query_esw_functions_out, out,
-					  host_params_context.host_total_vfs);
+		host_params = MLX5_ADDR_OF(query_esw_functions_out, out,
+					   net_function_params);
+		host_total_vfs = MLX5_GET(host_params_context, host_params,
+					  host_total_vfs);
 		kvfree(out);
 		return host_total_vfs;
 	}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c
index 6dcd9c2a78aa8..eae02bc742218 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/bwc.c
@@ -422,6 +422,18 @@ int mlx5hws_bwc_queue_poll(struct mlx5hws_context *ctx,
 	if (!got_comp && !drain)
 		return 0;
 
+	if (unlikely(ctx->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)) {
+		/* If the device is down for any reason (e.g. FLR), the HW will
+		 * no longer generate completions.
+		 * Note that ETIMEDOUT is returned here because the BWC layer
+		 * already has a special handling for timeouts - it breaks the
+		 * rehash / resize / shrink loops to avoid chain of timeouts.
+		 */
+		mlx5_core_warn_once(ctx->mdev,
+				    "BWC poll: device is down, polling for completion aborted\n");
+		return -ETIMEDOUT;
+	}
+
 	queue_full = mlx5hws_send_engine_full(&ctx->send_queue[queue_id]);
 	while (queue_full || ((got_comp || drain) && *pending_rules)) {
 		ret = mlx5hws_send_queue_poll(ctx, queue_id, comp, burst_th);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/table.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/table.c
index bd292485a25ba..dd7927983ab2a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/table.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/table.c
@@ -282,6 +282,9 @@ int mlx5hws_table_destroy(struct mlx5hws_table *tbl)
 		goto unlock_err;
 	}
 
+	if (tbl->default_miss.miss_tbl)
+		list_del_init(&tbl->default_miss.next);
+
 	list_del_init(&tbl->tbl_list_node);
 	mutex_unlock(&ctx->ctrl_lock);
 
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/vport.c
index d8e382b9fa61e..6dc3b11b79269 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/vport.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/hws/vport.c
@@ -50,7 +50,7 @@ static int hws_vport_add_gvmi(struct mlx5hws_context *ctx, u16 vport)
 static bool hws_vport_is_esw_mgr_vport(struct mlx5hws_context *ctx, u16 vport)
 {
 	return ctx->caps->is_ecpf ? vport == MLX5_VPORT_ECPF :
-				    vport == MLX5_VPORT_PF;
+				    vport == MLX5_VPORT_HOST_PF;
 }
 
 int mlx5hws_vport_get_gvmi(struct mlx5hws_context *ctx, u16 vport, u16 *vport_gvmi)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_types.h
index cc328292bf84a..e0344707f522d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_types.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/sws/dr_types.h
@@ -986,7 +986,6 @@ struct mlx5dr_matcher_rx_tx {
 					       [DR_RULE_MAX_STES];
 	u8 num_of_builders;
 	u8 num_of_builders_arr[DR_RULE_IPV_MAX][DR_RULE_IPV_MAX];
-	u64 default_icm_addr;
 	struct mlx5dr_table_rx_tx *nic_tbl;
 	u32 prio;
 	struct list_head list_node;
diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c
index 512933dbab9a3..1fee57054b20f 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c
@@ -713,7 +713,7 @@ static struct mlxsw_driver mlxsw_m_driver = {
 };
 
 static const struct i2c_device_id mlxsw_m_i2c_id[] = {
-	{ "mlxsw_minimal" },
+	{ .name = "mlxsw_minimal" },
 	{ }
 };
 
diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h
index 3bb89045eaf5e..01fcbd1c8d015 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/reg.h
+++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h
@@ -5651,6 +5651,14 @@ MLXSW_ITEM64(reg, ppcnt, tx_pause_duration,
 MLXSW_ITEM64(reg, ppcnt, tx_pause_transition,
 	     MLXSW_REG_PPCNT_COUNTERS_OFFSET + 0x70, 0, 64);
 
+/* reg_ppcnt_rx_discards
+ * Per-PG (ingress priority group) drop count.
+ * index slot is the PG, switch priority.
+ * Access: RO
+ */
+MLXSW_ITEM64(reg, ppcnt, rx_discards,
+	     MLXSW_REG_PPCNT_COUNTERS_OFFSET + 0x78, 0, 64);
+
 /* Ethernet Per Traffic Class Counters */
 
 /* reg_ppcnt_tc_transmit_queue
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
index 3a65420fa1ad1..1aaa75b47fe31 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c
@@ -5277,8 +5277,8 @@ static int mlxsw_sp_netdevice_event(struct notifier_block *nb,
 }
 
 static const struct pci_device_id mlxsw_sp1_pci_id_table[] = {
-	{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SPECTRUM), 0},
-	{0, },
+	{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SPECTRUM) },
+	{ },
 };
 
 static struct pci_driver mlxsw_sp1_pci_driver = {
@@ -5287,8 +5287,8 @@ static struct pci_driver mlxsw_sp1_pci_driver = {
 };
 
 static const struct pci_device_id mlxsw_sp2_pci_id_table[] = {
-	{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SPECTRUM2), 0},
-	{0, },
+	{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SPECTRUM2) },
+	{ },
 };
 
 static struct pci_driver mlxsw_sp2_pci_driver = {
@@ -5297,8 +5297,8 @@ static struct pci_driver mlxsw_sp2_pci_driver = {
 };
 
 static const struct pci_device_id mlxsw_sp3_pci_id_table[] = {
-	{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SPECTRUM3), 0},
-	{0, },
+	{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SPECTRUM3) },
+	{ },
 };
 
 static struct pci_driver mlxsw_sp3_pci_driver = {
@@ -5307,8 +5307,8 @@ static struct pci_driver mlxsw_sp3_pci_driver = {
 };
 
 static const struct pci_device_id mlxsw_sp4_pci_id_table[] = {
-	{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SPECTRUM4), 0},
-	{0, },
+	{PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_SPECTRUM4) },
+	{ },
 };
 
 static struct pci_driver mlxsw_sp4_pci_driver = {
diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c
index 0a8fb9c842d3b..7f78b1ef61cc6 100644
--- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c
+++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c
@@ -542,6 +542,10 @@ static struct mlxsw_sp_port_hw_stats mlxsw_sp_port_hw_prio_stats[] = {
 		.str = "tx_pause_duration_prio",
 		.getter = mlxsw_reg_ppcnt_tx_pause_duration_get,
 	},
+	{
+		.str = "rx_discards_pg",
+		.getter = mlxsw_reg_ppcnt_rx_discards_get,
+	},
 };
 
 #define MLXSW_SP_PORT_HW_PRIO_STATS_LEN ARRAY_SIZE(mlxsw_sp_port_hw_prio_stats)
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h
index 81794bd326e16..64b958df77744 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h
@@ -805,6 +805,7 @@ enum {
 #define FBNIC_CSR_END_PCS		0x10668 /* CSR section delimiter */
 
 #define FBNIC_CSR_START_RSFEC		0x10800 /* CSR section delimiter */
+#define FBNIC_RSFEC_CONTROL(n)	(0x10800 + 8 * (n))	/* 0x42000 + 32*n */
 
 /* We have 4 RSFEC engines present in our part, however we are only using 1.
  * As such only CCW(0) and NCCW(0) will never be non-zero and the other
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mdio.c b/drivers/net/ethernet/meta/fbnic/fbnic_mdio.c
index 709041f7fc43d..fe3a4ce88413c 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_mdio.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_mdio.c
@@ -7,10 +7,49 @@
 #include "fbnic.h"
 #include "fbnic_netdev.h"
 
+/* fbnic MDIO Interface Layout
+ *
+ *        +-------------------+
+ *        |        MAC        |
+ *        +-------------------+
+ *            |   |   |   |  <-- 25GMII, 50GMII, or CGMII
+ *        +-------------------+
+ *  MMD 3 |        PCS        |
+ *        +-------------------+
+ *        |        FEC        |
+ *        +-------------------+
+ *  MMD 8 |  Separated PMA    |
+ *        +-------------------+
+ *              |       |     <-- PMD Service Interface
+ *        +-------------------+
+ *  MMD 1 |        PMD        |
+ *        +-------------------+
+ */
+
 #define DW_VENDOR		BIT(15)
 #define FBNIC_PCS_VENDOR	BIT(9)
 #define FBNIC_PCS_ZERO_MASK	(DW_VENDOR - FBNIC_PCS_VENDOR)
 
+static int
+fbnic_mdio_ids(int id, int regnum)
+{
+	/* return correct IDs */
+	switch (regnum) {
+	case MDIO_DEVID1:
+		return id >> 16;
+	case MDIO_DEVID2:
+		return id & 0xffff;
+	case MDIO_DEVS1:
+		return MDIO_DEVS_SEP_PMA1 | MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS;
+	case MDIO_DEVS2:
+		return 0;
+	case MDIO_STAT2:
+		return MDIO_STAT2_DEVPRST_VAL;
+	}
+
+	return 0;
+}
+
 static int
 fbnic_mdio_read_pmd(struct fbnic_dev *fbd, int addr, int regnum)
 {
@@ -29,18 +68,6 @@ fbnic_mdio_read_pmd(struct fbnic_dev *fbd, int addr, int regnum)
 	}
 
 	switch (regnum) {
-	case MDIO_DEVID1:
-		ret = MP_FBNIC_XPCS_PMA_100G_ID >> 16;
-		break;
-	case MDIO_DEVID2:
-		ret = MP_FBNIC_XPCS_PMA_100G_ID & 0xffff;
-		break;
-	case MDIO_DEVS1:
-		ret = MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS;
-		break;
-	case MDIO_STAT2:
-		ret = MDIO_STAT2_DEVPRST_VAL;
-		break;
 	case MDIO_PMA_RXDET:
 		/* If training isn't complete default to 0 */
 		if (fbd->pmd_state != FBNIC_PMD_SEND_DATA)
@@ -51,6 +78,7 @@ fbnic_mdio_read_pmd(struct fbnic_dev *fbd, int addr, int regnum)
 		       (MDIO_PMD_RXDET_1 / FBNIC_AUI_MODE_R2));
 		break;
 	default:
+		ret = fbnic_mdio_ids(MP_FBNIC_XPCS_PMA_100G_ID, regnum);
 		break;
 	}
 
@@ -64,7 +92,7 @@ fbnic_mdio_read_pmd(struct fbnic_dev *fbd, int addr, int regnum)
 static int
 fbnic_mdio_read_pcs(struct fbnic_dev *fbd, int addr, int regnum)
 {
-	int ret, offset = 0;
+	int ret, offset = 0, overrides = 0;
 
 	/* We will need access to both PCS instances to get config info */
 	if (addr >= 2)
@@ -75,18 +103,25 @@ fbnic_mdio_read_pcs(struct fbnic_dev *fbd, int addr, int regnum)
 		return 0;
 
 	/* Intercept and return correct ID for PCS */
-	if (regnum == MDIO_DEVID1)
-		return DW_XPCS_ID >> 16;
-	if (regnum == MDIO_DEVID2)
-		return DW_XPCS_ID & 0xffff;
-	if (regnum == MDIO_DEVS1)
-		return MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS;
-
-	/* Swap vendor page bit for FBNIC PCS vendor page bit */
-	if (regnum & DW_VENDOR)
-		offset ^= DW_VENDOR | FBNIC_PCS_VENDOR;
+	switch (regnum) {
+	case MDIO_DEVID1 ... MDIO_DEVID2:
+		ret = fbnic_mdio_ids(DW_XPCS_ID, regnum);
+		break;
+	case MDIO_DEVS1:
+		/* DW IP returns MDIO_DEVS_SEP_PMA1, MDIO_DEVS_PMAPMD,
+		 * and MDIO_DEVS_PCS as 0
+		 */
+		overrides = fbnic_mdio_ids(DW_XPCS_ID, regnum);
+		fallthrough;
+	default:
+		/* Swap vendor page bit for FBNIC PCS vendor page bit */
+		if (regnum & DW_VENDOR)
+			offset ^= DW_VENDOR | FBNIC_PCS_VENDOR;
 
-	ret = fbnic_rd32(fbd, FBNIC_PCS_PAGE(addr) + (regnum ^ offset));
+		ret = fbnic_rd32(fbd, FBNIC_PCS_PAGE(addr) + (regnum ^ offset));
+		ret |= overrides;
+		break;
+	}
 
 	dev_dbg(fbd->dev,
 		"SWMII PCS Rd: Addr: %d RegNum: %d Value: 0x%04x\n",
@@ -95,6 +130,32 @@ fbnic_mdio_read_pcs(struct fbnic_dev *fbd, int addr, int regnum)
 	return ret;
 }
 
+static int
+fbnic_mdio_read_pma(struct fbnic_dev *fbd, int addr, int regnum)
+{
+	int ret = 0;
+
+	/* We will need access to both PMA instances to get config info */
+	if (addr >= 2)
+		return 0;
+
+	switch (regnum) {
+	case MDIO_PMA_RSFEC_CTRL ... MDIO_PMA_RSFEC_LANE_MAP:
+		ret = fbnic_rd32(fbd, FBNIC_RSFEC_CONTROL(addr) +
+				 regnum - MDIO_PMA_RSFEC_CTRL);
+		break;
+	default:
+		ret = fbnic_mdio_ids(MP_FBNIC_XPCS_PMA_100G_ID, regnum);
+		break;
+	}
+
+	dev_dbg(fbd->dev,
+		"SWMII PMA Rd: Addr: %d RegNum: %d Value: 0x%04x\n",
+		addr, regnum, ret);
+
+	return ret;
+}
+
 static int
 fbnic_mdio_read_c45(struct mii_bus *bus, int addr, int devnum, int regnum)
 {
@@ -106,6 +167,9 @@ fbnic_mdio_read_c45(struct mii_bus *bus, int addr, int devnum, int regnum)
 	if (devnum == MDIO_MMD_PCS)
 		return fbnic_mdio_read_pcs(fbd, addr, regnum);
 
+	if (devnum == MDIO_MMD_SEP_PMA1)
+		return fbnic_mdio_read_pma(fbd, addr, regnum);
+
 	return 0;
 }
 
@@ -125,7 +189,7 @@ fbnic_mdio_write_pcs(struct fbnic_dev *fbd, int addr, int regnum, u16 val)
 		addr, regnum, val);
 
 	/* Allow access to both halves of PCS for 50R2 config */
-	if (addr > 2)
+	if (addr >= 2)
 		return;
 
 	/* Skip write for reserved registers */
@@ -139,6 +203,26 @@ fbnic_mdio_write_pcs(struct fbnic_dev *fbd, int addr, int regnum, u16 val)
 	fbnic_wr32(fbd, FBNIC_PCS_PAGE(addr) + regnum, val);
 }
 
+static void
+fbnic_mdio_write_pma(struct fbnic_dev *fbd, int addr, int regnum, u16 val)
+{
+	dev_dbg(fbd->dev,
+		"SWMII PMA Wr: Addr: %d RegNum: %d Value: 0x%04x\n",
+		addr, regnum, val);
+
+	if (addr >= 2)
+		return;
+
+	switch (regnum) {
+	case MDIO_PMA_RSFEC_CTRL ... MDIO_PMA_RSFEC_LANE_MAP:
+		fbnic_wr32(fbd, FBNIC_RSFEC_CONTROL(addr) +
+				regnum - MDIO_PMA_RSFEC_CTRL, val);
+		break;
+	default:
+		break;
+	}
+}
+
 static int
 fbnic_mdio_write_c45(struct mii_bus *bus, int addr, int devnum,
 		     int regnum, u16 val)
@@ -151,6 +235,9 @@ fbnic_mdio_write_c45(struct mii_bus *bus, int addr, int devnum,
 	if (devnum == MDIO_MMD_PCS)
 		fbnic_mdio_write_pcs(fbd, addr, regnum, val);
 
+	if (devnum == MDIO_MMD_SEP_PMA1)
+		fbnic_mdio_write_pma(fbd, addr, regnum, val);
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
index 4dea2bb58d2fc..f99ca551c1ceb 100644
--- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
+++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c
@@ -752,7 +752,7 @@ struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd)
 	netdev->netdev_ops = &fbnic_netdev_ops;
 	netdev->stat_ops = &fbnic_stat_ops;
 	netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops;
-	netdev->netmem_tx = true;
+	netdev->netmem_tx = NETMEM_TX_DMA;
 
 	fbnic_set_ethtool_ops(netdev);
 
diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c
index 26557cd32008a..773fdd215a157 100644
--- a/drivers/net/ethernet/micrel/ksz884x.c
+++ b/drivers/net/ethernet/micrel/ksz884x.c
@@ -6839,11 +6839,9 @@ static int __maybe_unused pcidev_suspend(struct device *dev_d)
 static char pcidev_name[] = "ksz884xp";
 
 static const struct pci_device_id pcidev_table[] = {
-	{ PCI_VENDOR_ID_MICREL_KS, 0x8841,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_MICREL_KS, 0x8842,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ 0 }
+	{ PCI_VDEVICE(MICREL_KS, 0x8841) },
+	{ PCI_VDEVICE(MICREL_KS, 0x8842) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, pcidev_table);
diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index d8e816882f02c..3bc3fff559991 100644
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -224,6 +224,9 @@ static int mana_gd_query_max_resources(struct pci_dev *pdev)
 	if (gc->max_num_queues > gc->num_msix_usable - 1)
 		gc->max_num_queues = gc->num_msix_usable - 1;
 
+	if (gc->max_num_queues == 0)
+		return -ENOSPC;
+
 	return 0;
 }
 
diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c
index dbbde0fa57e71..fd8b324d7fb68 100644
--- a/drivers/net/ethernet/microsoft/mana/hw_channel.c
+++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c
@@ -77,21 +77,19 @@ static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq,
 }
 
 static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,
-				 struct hwc_work_request *rx_req)
+				 struct hwc_work_request *rx_req, u16 msg_id)
 {
 	const struct gdma_resp_hdr *resp_msg = rx_req->buf_va;
 	struct hwc_caller_ctx *ctx;
 	int err;
 
-	if (!test_bit(resp_msg->response.hwc_msg_id,
-		      hwc->inflight_msg_res.map)) {
-		dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n",
-			resp_msg->response.hwc_msg_id);
+	if (!test_bit(msg_id, hwc->inflight_msg_res.map)) {
+		dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n", msg_id);
 		mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
 		return;
 	}
 
-	ctx = hwc->caller_ctx + resp_msg->response.hwc_msg_id;
+	ctx = hwc->caller_ctx + msg_id;
 	err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);
 	if (err)
 		goto out;
@@ -251,6 +249,7 @@ static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,
 	struct gdma_sge *sge;
 	u64 rq_base_addr;
 	u64 rx_req_idx;
+	u16 msg_id;
 	u8 *wqe;
 
 	if (WARN_ON_ONCE(hwc_rxq->gdma_wq->id != gdma_rxq_id))
@@ -269,13 +268,17 @@ static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,
 	rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx];
 	resp = (struct gdma_resp_hdr *)rx_req->buf_va;
 
-	if (resp->response.hwc_msg_id >= hwc->num_inflight_msg) {
-		dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n",
-			resp->response.hwc_msg_id);
+	/* Read msg_id once from DMA buffer to prevent TOCTOU:
+	 * DMA memory is shared/unencrypted in CVMs - host can
+	 * modify it between reads.
+	 */
+	msg_id = READ_ONCE(resp->response.hwc_msg_id);
+	if (msg_id >= hwc->num_inflight_msg) {
+		dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n", msg_id);
 		return;
 	}
 
-	mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, rx_req);
+	mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, rx_req, msg_id);
 
 	/* Can no longer use 'resp', because the buffer is posted to the HW
 	 * in mana_hwc_handle_resp() above.
diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
index 7697c9b52ed34..b5e9bb184a1dd 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_bpf.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c
@@ -68,7 +68,7 @@ int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames,
 		count++;
 	}
 
-	tx_stats = &apc->tx_qp[q_idx].txq.stats;
+	tx_stats = &apc->tx_qp[q_idx]->txq.stats;
 
 	u64_stats_update_begin(&tx_stats->syncp);
 	tx_stats->xdp_xmit += count;
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index 9afc786b297a8..b2faa7cf398f3 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -355,9 +355,9 @@ netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)
 	if (skb_cow_head(skb, MANA_HEADROOM))
 		goto tx_drop_count;
 
-	txq = &apc->tx_qp[txq_idx].txq;
+	txq = &apc->tx_qp[txq_idx]->txq;
 	gdma_sq = txq->gdma_sq;
-	cq = &apc->tx_qp[txq_idx].tx_cq;
+	cq = &apc->tx_qp[txq_idx]->tx_cq;
 	tx_stats = &txq->stats;
 
 	BUILD_BUG_ON(MAX_TX_WQE_SGL_ENTRIES != MANA_MAX_TX_WQE_SGL_ENTRIES);
@@ -614,7 +614,7 @@ static void mana_get_stats64(struct net_device *ndev,
 	}
 
 	for (q = 0; q < num_queues; q++) {
-		tx_stats = &apc->tx_qp[q].txq.stats;
+		tx_stats = &apc->tx_qp[q]->txq.stats;
 
 		do {
 			start = u64_stats_fetch_begin(&tx_stats->syncp);
@@ -685,11 +685,11 @@ void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc)
 		put_page(virt_to_head_page(mpc->rxbufs_pre[i]));
 	}
 
-	kfree(mpc->das_pre);
+	kvfree(mpc->das_pre);
 	mpc->das_pre = NULL;
 
 out2:
-	kfree(mpc->rxbufs_pre);
+	kvfree(mpc->rxbufs_pre);
 	mpc->rxbufs_pre = NULL;
 
 out1:
@@ -806,11 +806,11 @@ int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu, int num_qu
 	num_rxb = num_queues * mpc->rx_queue_size;
 
 	WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n");
-	mpc->rxbufs_pre = kmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL);
+	mpc->rxbufs_pre = kvmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL);
 	if (!mpc->rxbufs_pre)
 		goto error;
 
-	mpc->das_pre = kmalloc_objs(dma_addr_t, num_rxb);
+	mpc->das_pre = kvmalloc_objs(dma_addr_t, num_rxb);
 	if (!mpc->das_pre)
 		goto error;
 
@@ -1262,6 +1262,12 @@ static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index,
 
 	*max_sq = resp.max_num_sq;
 	*max_rq = resp.max_num_rq;
+
+	if (*max_sq == 0 || *max_rq == 0) {
+		netdev_err(apc->ndev, "Invalid max queues from vPort config\n");
+		return -EPROTO;
+	}
+
 	if (resp.num_indirection_ent > 0 &&
 	    resp.num_indirection_ent <= MANA_INDIRECT_TABLE_MAX_SIZE &&
 	    is_power_of_2(resp.num_indirection_ent)) {
@@ -2321,21 +2327,26 @@ static void mana_destroy_txq(struct mana_port_context *apc)
 		return;
 
 	for (i = 0; i < apc->num_queues; i++) {
-		debugfs_remove_recursive(apc->tx_qp[i].mana_tx_debugfs);
-		apc->tx_qp[i].mana_tx_debugfs = NULL;
+		if (!apc->tx_qp[i])
+			continue;
+
+		debugfs_remove_recursive(apc->tx_qp[i]->mana_tx_debugfs);
+		apc->tx_qp[i]->mana_tx_debugfs = NULL;
 
-		napi = &apc->tx_qp[i].tx_cq.napi;
-		if (apc->tx_qp[i].txq.napi_initialized) {
+		napi = &apc->tx_qp[i]->tx_cq.napi;
+		if (apc->tx_qp[i]->txq.napi_initialized) {
 			napi_synchronize(napi);
 			napi_disable_locked(napi);
 			netif_napi_del_locked(napi);
-			apc->tx_qp[i].txq.napi_initialized = false;
+			apc->tx_qp[i]->txq.napi_initialized = false;
 		}
-		mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object);
+		mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i]->tx_object);
 
-		mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq);
+		mana_deinit_cq(apc, &apc->tx_qp[i]->tx_cq);
 
-		mana_deinit_txq(apc, &apc->tx_qp[i].txq);
+		mana_deinit_txq(apc, &apc->tx_qp[i]->txq);
+
+		kvfree(apc->tx_qp[i]);
 	}
 
 	kfree(apc->tx_qp);
@@ -2344,7 +2355,7 @@ static void mana_destroy_txq(struct mana_port_context *apc)
 
 static void mana_create_txq_debugfs(struct mana_port_context *apc, int idx)
 {
-	struct mana_tx_qp *tx_qp = &apc->tx_qp[idx];
+	struct mana_tx_qp *tx_qp = apc->tx_qp[idx];
 	char qnum[32];
 
 	sprintf(qnum, "TX-%d", idx);
@@ -2383,7 +2394,7 @@ static int mana_create_txq(struct mana_port_context *apc,
 	int err;
 	int i;
 
-	apc->tx_qp = kzalloc_objs(struct mana_tx_qp, apc->num_queues);
+	apc->tx_qp = kzalloc_objs(struct mana_tx_qp *, apc->num_queues);
 	if (!apc->tx_qp)
 		return -ENOMEM;
 
@@ -2403,10 +2414,16 @@ static int mana_create_txq(struct mana_port_context *apc,
 	gc = gd->gdma_context;
 
 	for (i = 0; i < apc->num_queues; i++) {
-		apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE;
+		apc->tx_qp[i] = kvzalloc_obj(*apc->tx_qp[i]);
+		if (!apc->tx_qp[i]) {
+			err = -ENOMEM;
+			goto out;
+		}
+
+		apc->tx_qp[i]->tx_object = INVALID_MANA_HANDLE;
 
 		/* Create SQ */
-		txq = &apc->tx_qp[i].txq;
+		txq = &apc->tx_qp[i]->txq;
 
 		u64_stats_init(&txq->stats.syncp);
 		txq->ndev = net;
@@ -2424,7 +2441,7 @@ static int mana_create_txq(struct mana_port_context *apc,
 			goto out;
 
 		/* Create SQ's CQ */
-		cq = &apc->tx_qp[i].tx_cq;
+		cq = &apc->tx_qp[i]->tx_cq;
 		cq->type = MANA_CQ_TYPE_TX;
 
 		cq->txq = txq;
@@ -2453,7 +2470,7 @@ static int mana_create_txq(struct mana_port_context *apc,
 
 		err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ,
 					 &wq_spec, &cq_spec,
-					 &apc->tx_qp[i].tx_object);
+					 &apc->tx_qp[i]->tx_object);
 
 		if (err)
 			goto out;
@@ -2556,7 +2573,7 @@ static void mana_destroy_rxq(struct mana_port_context *apc,
 	if (rxq->gdma_rq)
 		mana_gd_destroy_queue(gc, rxq->gdma_rq);
 
-	kfree(rxq);
+	kvfree(rxq);
 }
 
 static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key,
@@ -2696,7 +2713,7 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
 
 	gc = gd->gdma_context;
 
-	rxq = kzalloc_flex(*rxq, rx_oobs, apc->rx_queue_size);
+	rxq = kvzalloc_flex(*rxq, rx_oobs, apc->rx_queue_size);
 	if (!rxq)
 		return NULL;
 
@@ -3288,7 +3305,7 @@ static int mana_dealloc_queues(struct net_device *ndev)
 	 */
 
 	for (i = 0; i < apc->num_queues; i++) {
-		txq = &apc->tx_qp[i].txq;
+		txq = &apc->tx_qp[i]->txq;
 		tsleep = 1000;
 		while (atomic_read(&txq->pending_sends) > 0 &&
 		       time_before(jiffies, timeout)) {
@@ -3307,7 +3324,7 @@ static int mana_dealloc_queues(struct net_device *ndev)
 	}
 
 	for (i = 0; i < apc->num_queues; i++) {
-		txq = &apc->tx_qp[i].txq;
+		txq = &apc->tx_qp[i]->txq;
 		while ((skb = skb_dequeue(&txq->pending_skbs))) {
 			mana_unmap_skb(skb, apc);
 			dev_kfree_skb_any(skb);
diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
index 6a4b42fe09445..04350973e19e3 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c
@@ -260,7 +260,7 @@ static void mana_get_ethtool_stats(struct net_device *ndev,
 	}
 
 	for (q = 0; q < num_queues; q++) {
-		tx_stats = &apc->tx_qp[q].txq.stats;
+		tx_stats = &apc->tx_qp[q]->txq.stats;
 
 		do {
 			start = u64_stats_fetch_begin(&tx_stats->syncp);
diff --git a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_main.c b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_main.c
index 316f941629d4a..70a2b0082ba86 100644
--- a/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_main.c
+++ b/drivers/net/ethernet/mucse/rnpgbe/rnpgbe_main.c
@@ -17,12 +17,12 @@ static const char rnpgbe_driver_name[] = "rnpgbe";
  *   private_data (used for different hw chip) }
  */
 static struct pci_device_id rnpgbe_pci_tbl[] = {
-	{ PCI_VDEVICE(MUCSE, RNPGBE_DEVICE_ID_N210), board_n210 },
-	{ PCI_VDEVICE(MUCSE, RNPGBE_DEVICE_ID_N210L), board_n210 },
-	{ PCI_VDEVICE(MUCSE, RNPGBE_DEVICE_ID_N500_DUAL_PORT), board_n500 },
-	{ PCI_VDEVICE(MUCSE, RNPGBE_DEVICE_ID_N500_QUAD_PORT), board_n500 },
+	{ PCI_VDEVICE(MUCSE, RNPGBE_DEVICE_ID_N210), .driver_data = board_n210 },
+	{ PCI_VDEVICE(MUCSE, RNPGBE_DEVICE_ID_N210L), .driver_data = board_n210 },
+	{ PCI_VDEVICE(MUCSE, RNPGBE_DEVICE_ID_N500_DUAL_PORT), .driver_data = board_n500 },
+	{ PCI_VDEVICE(MUCSE, RNPGBE_DEVICE_ID_N500_QUAD_PORT), .driver_data = board_n500 },
 	/* required last entry */
-	{0, },
+	{ },
 };
 
 /**
diff --git a/drivers/net/ethernet/natsemi/natsemi.c b/drivers/net/ethernet/natsemi/natsemi.c
index b253734dbc805..35a63b78880be 100644
--- a/drivers/net/ethernet/natsemi/natsemi.c
+++ b/drivers/net/ethernet/natsemi/natsemi.c
@@ -248,8 +248,8 @@ static struct {
 };
 
 static const struct pci_device_id natsemi_pci_tbl[] = {
-	{ PCI_VENDOR_ID_NS, 0x0020, 0x12d9,     0x000c,     0, 0, 0 },
-	{ PCI_VENDOR_ID_NS, 0x0020, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 1 },
+	{ PCI_VDEVICE_SUB(NS, 0x0020, 0x12d9, 0x000c), .driver_data = 0 },
+	{ PCI_VDEVICE(NS, 0x0020), .driver_data = 1 },
 	{ }	/* terminate list */
 };
 MODULE_DEVICE_TABLE(pci, natsemi_pci_tbl);
diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
index 70368fe7c5106..1caa87da72b51 100644
--- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
+++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c
@@ -561,10 +561,10 @@ nfp_bpf_check_alu(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
 	const struct bpf_reg_state *dreg =
 		cur_regs(env) + meta->insn.dst_reg;
 
-	meta->umin_src = min(meta->umin_src, sreg->umin_value);
-	meta->umax_src = max(meta->umax_src, sreg->umax_value);
-	meta->umin_dst = min(meta->umin_dst, dreg->umin_value);
-	meta->umax_dst = max(meta->umax_dst, dreg->umax_value);
+	meta->umin_src = min(meta->umin_src, reg_umin(sreg));
+	meta->umax_src = max(meta->umax_src, reg_umax(sreg));
+	meta->umin_dst = min(meta->umin_dst, reg_umin(dreg));
+	meta->umax_dst = max(meta->umax_dst, reg_umax(dreg));
 
 	/* NFP supports u16 and u32 multiplication.
 	 *
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.c b/drivers/net/ethernet/netronome/nfp/nfp_main.c
index 48390b2fd44db..5a9fb5bad405b 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_main.c
@@ -33,39 +33,40 @@
 static const char nfp_driver_name[] = "nfp";
 
 static const struct pci_device_id nfp_pci_device_ids[] = {
-	{ PCI_VENDOR_ID_NETRONOME, PCI_DEVICE_ID_NFP3800,
-	  PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP3800,
+	{
+		PCI_VDEVICE_SUB(NETRONOME, PCI_DEVICE_ID_NFP3800,
+				PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP3800,
+	}, {
+		PCI_VDEVICE_SUB(NETRONOME, PCI_DEVICE_ID_NFP4000,
+				PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP6000,
+	}, {
+		PCI_VDEVICE_SUB(NETRONOME, PCI_DEVICE_ID_NFP5000,
+				PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP6000,
+	}, {
+		PCI_VDEVICE_SUB(NETRONOME, PCI_DEVICE_ID_NFP6000,
+				PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP6000,
+	}, {
+		PCI_VDEVICE_SUB(CORIGINE, PCI_DEVICE_ID_NFP3800,
+				PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP3800,
+	}, {
+		PCI_VDEVICE_SUB(CORIGINE, PCI_DEVICE_ID_NFP4000,
+				PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP6000,
+	}, {
+		PCI_VDEVICE_SUB(CORIGINE, PCI_DEVICE_ID_NFP5000,
+				PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP6000,
+	}, {
+		PCI_VDEVICE_SUB(CORIGINE, PCI_DEVICE_ID_NFP6000,
+				PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP6000,
 	},
-	{ PCI_VENDOR_ID_NETRONOME, PCI_DEVICE_ID_NFP4000,
-	  PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP6000,
-	},
-	{ PCI_VENDOR_ID_NETRONOME, PCI_DEVICE_ID_NFP5000,
-	  PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP6000,
-	},
-	{ PCI_VENDOR_ID_NETRONOME, PCI_DEVICE_ID_NFP6000,
-	  PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP6000,
-	},
-	{ PCI_VENDOR_ID_CORIGINE, PCI_DEVICE_ID_NFP3800,
-	  PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP3800,
-	},
-	{ PCI_VENDOR_ID_CORIGINE, PCI_DEVICE_ID_NFP4000,
-	  PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP6000,
-	},
-	{ PCI_VENDOR_ID_CORIGINE, PCI_DEVICE_ID_NFP5000,
-	  PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP6000,
-	},
-	{ PCI_VENDOR_ID_CORIGINE, PCI_DEVICE_ID_NFP6000,
-	  PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP6000,
-	},
-	{ 0, } /* Required last entry. */
+	{ } /* Required last entry. */
 };
 MODULE_DEVICE_TABLE(pci, nfp_pci_device_ids);
 
diff --git a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
index b5646c8843710..6c8f3321048ab 100644
--- a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
+++ b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c
@@ -38,23 +38,24 @@ struct nfp_net_vf {
 static const char nfp_net_driver_name[] = "nfp_netvf";
 
 static const struct pci_device_id nfp_netvf_pci_device_ids[] = {
-	{ PCI_VENDOR_ID_NETRONOME, PCI_DEVICE_ID_NFP3800_VF,
-	  PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP3800_VF,
+	{
+		PCI_VDEVICE_SUB(NETRONOME, PCI_DEVICE_ID_NFP3800_VF,
+				PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP3800_VF,
+	}, {
+		PCI_VDEVICE_SUB(NETRONOME, PCI_DEVICE_ID_NFP6000_VF,
+				PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP6000_VF,
+	}, {
+		PCI_VDEVICE_SUB(CORIGINE, PCI_DEVICE_ID_NFP3800_VF,
+				PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP3800_VF,
+	}, {
+		PCI_VDEVICE_SUB(CORIGINE, PCI_DEVICE_ID_NFP6000_VF,
+				PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID),
+		.driver_data = NFP_DEV_NFP6000_VF,
 	},
-	{ PCI_VENDOR_ID_NETRONOME, PCI_DEVICE_ID_NFP6000_VF,
-	  PCI_VENDOR_ID_NETRONOME, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP6000_VF,
-	},
-	{ PCI_VENDOR_ID_CORIGINE, PCI_DEVICE_ID_NFP3800_VF,
-	  PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP3800_VF,
-	},
-	{ PCI_VENDOR_ID_CORIGINE, PCI_DEVICE_ID_NFP6000_VF,
-	  PCI_VENDOR_ID_CORIGINE, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, NFP_DEV_NFP6000_VF,
-	},
-	{ 0, } /* Required last entry. */
+	{ } /* Required last entry. */
 };
 MODULE_DEVICE_TABLE(pci, nfp_netvf_pci_device_ids);
 
diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c
index 39842eb73bc33..cb0ae0650905c 100644
--- a/drivers/net/ethernet/qlogic/qede/qede_main.c
+++ b/drivers/net/ethernet/qlogic/qede/qede_main.c
@@ -76,20 +76,20 @@ enum qede_pci_private {
 };
 
 static const struct pci_device_id qede_pci_tbl[] = {
-	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_40), QEDE_PRIVATE_PF},
-	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_10), QEDE_PRIVATE_PF},
-	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_MF), QEDE_PRIVATE_PF},
-	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_100), QEDE_PRIVATE_PF},
-	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_50), QEDE_PRIVATE_PF},
-	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_25), QEDE_PRIVATE_PF},
+	{ PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_40), .driver_data = QEDE_PRIVATE_PF },
+	{ PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_10), .driver_data = QEDE_PRIVATE_PF },
+	{ PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_MF), .driver_data = QEDE_PRIVATE_PF },
+	{ PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_100), .driver_data = QEDE_PRIVATE_PF },
+	{ PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_50), .driver_data = QEDE_PRIVATE_PF },
+	{ PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_25), .driver_data = QEDE_PRIVATE_PF },
 #ifdef CONFIG_QED_SRIOV
-	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_IOV), QEDE_PRIVATE_VF},
+	{ PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_57980S_IOV), .driver_data = QEDE_PRIVATE_VF },
 #endif
-	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH), QEDE_PRIVATE_PF},
+	{ PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH), .driver_data = QEDE_PRIVATE_PF },
 #ifdef CONFIG_QED_SRIOV
-	{PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH_IOV), QEDE_PRIVATE_VF},
+	{ PCI_VDEVICE(QLOGIC, PCI_DEVICE_ID_AH_IOV), .driver_data = QEDE_PRIVATE_VF },
 #endif
-	{ 0 }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, qede_pci_tbl);
diff --git a/drivers/net/ethernet/qualcomm/Kconfig b/drivers/net/ethernet/qualcomm/Kconfig
index ba7efb108637c..57e1984d20ee7 100644
--- a/drivers/net/ethernet/qualcomm/Kconfig
+++ b/drivers/net/ethernet/qualcomm/Kconfig
@@ -48,7 +48,7 @@ config QCA7000_UART
 	  will be called qcauart.
 
 config QCOM_EMAC
-	tristate "Qualcomm Technologies, Inc. EMAC Gigabit Ethernet support"
+	tristate "Qualcomm EMAC Gigabit Ethernet support"
 	depends on HAS_DMA && HAS_IOMEM
 	select CRC32
 	select PHYLIB
@@ -61,7 +61,7 @@ config QCOM_EMAC
 	  Precision Clock Synchronization Protocol.
 
 config QCOM_PPE
-	tristate "Qualcomm Technologies, Inc. PPE Ethernet support"
+	tristate "Qualcomm PPE Ethernet support"
 	depends on COMMON_CLK && HAS_IOMEM && OF
 	depends on ARCH_QCOM || COMPILE_TEST
 	select REGMAP_MMIO
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
index 269c0449760c0..78d4df55740a1 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.c
@@ -213,8 +213,8 @@ static void rmnet_dellink(struct net_device *dev, struct list_head *head)
 	ep = rmnet_get_endpoint(real_port, mux_id);
 	if (ep) {
 		hlist_del_init_rcu(&ep->hlnode);
-		rmnet_vnd_dellink(mux_id, real_port, ep);
-		kfree(ep);
+		real_port->nr_rmnet_devs--;
+		kfree_rcu(ep, rcu);
 	}
 
 	netdev_upper_dev_unlink(real_dev, dev);
@@ -238,9 +238,9 @@ static void rmnet_force_unassociate_device(struct net_device *real_dev)
 		hash_for_each_safe(port->muxed_ep, bkt_ep, tmp_ep, ep, hlnode) {
 			unregister_netdevice_queue(ep->egress_dev, &list);
 			netdev_upper_dev_unlink(real_dev, ep->egress_dev);
-			rmnet_vnd_dellink(ep->mux_id, port, ep);
 			hlist_del_init_rcu(&ep->hlnode);
-			kfree(ep);
+			port->nr_rmnet_devs--;
+			kfree_rcu(ep, rcu);
 		}
 		rmnet_unregister_real_device(real_dev);
 		unregister_netdevice_many(&list);
diff --git a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
index ed112d51ac5a7..f50fae1c6bdd5 100644
--- a/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
+++ b/drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h
@@ -18,6 +18,7 @@ struct rmnet_endpoint {
 	u8 mux_id;
 	struct net_device *egress_dev;
 	struct hlist_node hlnode;
+	struct rcu_head rcu;
 };
 
 struct rmnet_egress_agg_params {
diff --git a/drivers/net/ethernet/realtek/8139too.c b/drivers/net/ethernet/realtek/8139too.c
index a8532ebd42ecf..8241bcf766644 100644
--- a/drivers/net/ethernet/realtek/8139too.c
+++ b/drivers/net/ethernet/realtek/8139too.c
@@ -235,44 +235,44 @@ static const struct {
 
 
 static const struct pci_device_id rtl8139_pci_tbl[] = {
-	{0x10ec, 0x8139, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x10ec, 0x8138, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x1113, 0x1211, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x1500, 0x1360, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x4033, 0x1360, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x1186, 0x1300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x1186, 0x1340, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x13d1, 0xab06, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x1259, 0xa117, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x1259, 0xa11e, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x14ea, 0xab06, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x14ea, 0xab07, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x11db, 0x1234, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x1432, 0x9130, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x02ac, 0x1012, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x018a, 0x0106, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x126c, 0x1211, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x1743, 0x8139, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x021b, 0x8139, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
-	{0x16ec, 0xab06, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
+	{ PCI_DEVICE(0x10ec, 0x8139), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x10ec, 0x8138), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x1113, 0x1211), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x1500, 0x1360), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x4033, 0x1360), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x1186, 0x1300), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x1186, 0x1340), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x13d1, 0xab06), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x1259, 0xa117), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x1259, 0xa11e), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x14ea, 0xab06), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x14ea, 0xab07), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x11db, 0x1234), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x1432, 0x9130), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x02ac, 0x1012), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x018a, 0x0106), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x126c, 0x1211), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x1743, 0x8139), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x021b, 0x8139), .driver_data = RTL8139 },
+	{ PCI_DEVICE(0x16ec, 0xab06), .driver_data = RTL8139 },
 
 #ifdef CONFIG_SH_SECUREEDGE5410
 	/* Bogus 8139 silicon reports 8129 without external PROM :-( */
-	{0x10ec, 0x8129, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8139 },
+	{ PCI_DEVICE(0x10ec, 0x8129), .driver_data = RTL8139 },
 #endif
 #ifdef CONFIG_8139TOO_8129
-	{0x10ec, 0x8129, PCI_ANY_ID, PCI_ANY_ID, 0, 0, RTL8129 },
+	{ PCI_DEVICE(0x10ec, 0x8129), .driver_data = RTL8129 },
 #endif
 
 	/* some crazy cards report invalid vendor ids like
 	 * 0x0001 here.  The other ids are valid and constant,
 	 * so we simply don't match on the main vendor id.
 	 */
-	{PCI_ANY_ID, 0x8139, 0x10ec, 0x8139, 0, 0, RTL8139 },
-	{PCI_ANY_ID, 0x8139, 0x1186, 0x1300, 0, 0, RTL8139 },
-	{PCI_ANY_ID, 0x8139, 0x13d1, 0xab06, 0, 0, RTL8139 },
+	{ PCI_DEVICE_SUB(PCI_ANY_ID, 0x8139, 0x10ec, 0x8139), .driver_data = RTL8139 },
+	{ PCI_DEVICE_SUB(PCI_ANY_ID, 0x8139, 0x1186, 0x1300), .driver_data = RTL8139 },
+	{ PCI_DEVICE_SUB(PCI_ANY_ID, 0x8139, 0x13d1, 0xab06), .driver_data = RTL8139 },
 
-	{0,}
+	{ }
 };
 MODULE_DEVICE_TABLE (pci, rtl8139_pci_tbl);
 
diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c
index 791277e750bad..ec4fc21fa21fb 100644
--- a/drivers/net/ethernet/realtek/r8169_main.c
+++ b/drivers/net/ethernet/realtek/r8169_main.c
@@ -229,7 +229,7 @@ static const struct pci_device_id rtl8169_pci_tbl[] = {
 	{ PCI_VDEVICE(REALTEK,	0x2502) },
 	{ PCI_VDEVICE(REALTEK,	0x2600) },
 	{ PCI_VDEVICE(REALTEK,	0x8129) },
-	{ PCI_VDEVICE(REALTEK,	0x8136), RTL_CFG_NO_GBIT },
+	{ PCI_VDEVICE(REALTEK,	0x8136), .driver_data = RTL_CFG_NO_GBIT },
 	{ PCI_VDEVICE(REALTEK,	0x8161) },
 	{ PCI_VDEVICE(REALTEK,	0x8162) },
 	{ PCI_VDEVICE(REALTEK,	0x8167) },
@@ -240,15 +240,15 @@ static const struct pci_device_id rtl8169_pci_tbl[] = {
 	{ PCI_VDEVICE(DLINK,	0x4302) },
 	{ PCI_VDEVICE(AT,	0xc107) },
 	{ PCI_VDEVICE(USR,	0x0116) },
-	{ PCI_VENDOR_ID_LINKSYS, 0x1032, PCI_ANY_ID, 0x0024 },
-	{ 0x0001, 0x8168, PCI_ANY_ID, 0x2410 },
+	{ PCI_VDEVICE_SUB(LINKSYS, 0x1032, PCI_ANY_ID, 0x0024) },
+	{ PCI_DEVICE_SUB(0x0001, 0x8168, PCI_ANY_ID, 0x2410) },
 	{ PCI_VDEVICE(REALTEK,	0x8125) },
 	{ PCI_VDEVICE(REALTEK,	0x8126) },
 	{ PCI_VDEVICE(REALTEK,	0x8127) },
 	{ PCI_VDEVICE(REALTEK,	0x3000) },
 	{ PCI_VDEVICE(REALTEK,	0x5000) },
 	{ PCI_VDEVICE(REALTEK,	0x0e10) },
-	{}
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, rtl8169_pci_tbl);
diff --git a/drivers/net/ethernet/realtek/rtase/rtase.h b/drivers/net/ethernet/realtek/rtase/rtase.h
index b9209eb6ea735..9bd6872474c11 100644
--- a/drivers/net/ethernet/realtek/rtase/rtase.h
+++ b/drivers/net/ethernet/realtek/rtase/rtase.h
@@ -153,6 +153,10 @@ enum rtase_registers {
 #define RTASE_FORCE_TXFLOW_EN BIT(10)
 #define RTASE_RX_CHKSUM       BIT(5)
 
+	RTASE_GPHY_STD_00 = 0x6024,
+#define RTASE_RXFLOW_EN BIT(7)
+#define RTASE_TXFLOW_EN BIT(6)
+
 	RTASE_Q0_RX_DESC_ADDR0 = 0x00E4,
 	RTASE_Q0_RX_DESC_ADDR4 = 0x00E8,
 	RTASE_Q1_RX_DESC_ADDR0 = 0x4000,
diff --git a/drivers/net/ethernet/realtek/rtase/rtase_main.c b/drivers/net/ethernet/realtek/rtase/rtase_main.c
index ef13109c49cff..bde9bccfb5a9b 100644
--- a/drivers/net/ethernet/realtek/rtase/rtase_main.c
+++ b/drivers/net/ethernet/realtek/rtase/rtase_main.c
@@ -974,6 +974,9 @@ static void rtase_hw_config(struct net_device *dev)
 	rtase_hw_set_features(dev, dev->features);
 
 	/* enable flow control */
+	reg_data16 = rtase_r16(tp, RTASE_GPHY_STD_00);
+	reg_data16 &= ~(RTASE_TXFLOW_EN | RTASE_RXFLOW_EN);
+	rtase_w16(tp, RTASE_GPHY_STD_00, reg_data16);
 	reg_data16 = rtase_r16(tp, RTASE_CPLUS_CMD);
 	reg_data16 |= (RTASE_FORCE_TXFLOW_EN | RTASE_FORCE_RXFLOW_EN);
 	rtase_w16(tp, RTASE_CPLUS_CMD, reg_data16);
diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c
index ff6ca7b2cd4f4..84a55f2b48ffe 100644
--- a/drivers/net/ethernet/rocker/rocker_main.c
+++ b/drivers/net/ethernet/rocker/rocker_main.c
@@ -40,8 +40,8 @@
 static const char rocker_driver_name[] = "rocker";
 
 static const struct pci_device_id rocker_pci_id_table[] = {
-	{PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_ROCKER), 0},
-	{0, }
+	{ PCI_VDEVICE(REDHAT, PCI_DEVICE_ID_REDHAT_ROCKER) },
+	{ }
 };
 
 struct rocker_wait {
diff --git a/drivers/net/ethernet/sis/sis190.c b/drivers/net/ethernet/sis/sis190.c
index 12ddea5149103..ed5276d492b97 100644
--- a/drivers/net/ethernet/sis/sis190.c
+++ b/drivers/net/ethernet/sis/sis190.c
@@ -331,9 +331,9 @@ static const struct {
 };
 
 static const struct pci_device_id sis190_pci_tbl[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_SI, 0x0190), 0, 0, 0 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_SI, 0x0191), 0, 0, 1 },
-	{ 0, },
+	{ PCI_VDEVICE(SI, 0x0190), .driver_data = 0 },
+	{ PCI_VDEVICE(SI, 0x0191), .driver_data = 1 },
+	{ },
 };
 
 MODULE_DEVICE_TABLE(pci, sis190_pci_tbl);
diff --git a/drivers/net/ethernet/sis/sis900.c b/drivers/net/ethernet/sis/sis900.c
index 1349ef50cd26f..6e9afd9042f7c 100644
--- a/drivers/net/ethernet/sis/sis900.c
+++ b/drivers/net/ethernet/sis/sis900.c
@@ -104,11 +104,11 @@ static const char * card_names[] = {
 };
 
 static const struct pci_device_id sis900_pci_tbl[] = {
-	{PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_900,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, SIS_900},
-	{PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_7016,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, SIS_7016},
-	{0,}
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_900),
+	  .driver_data = SIS_900 },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_7016),
+	  .driver_data = SIS_7016 },
+	{ }
 };
 MODULE_DEVICE_TABLE (pci, sis900_pci_tbl);
 
diff --git a/drivers/net/ethernet/smsc/epic100.c b/drivers/net/ethernet/smsc/epic100.c
index 389659db06a84..60538b7fe684e 100644
--- a/drivers/net/ethernet/smsc/epic100.c
+++ b/drivers/net/ethernet/smsc/epic100.c
@@ -166,11 +166,19 @@ static const struct epic_chip_info pci_id_tbl[] = {
 
 
 static const struct pci_device_id epic_pci_tbl[] = {
-	{ 0x10B8, 0x0005, 0x1092, 0x0AB4, 0, 0, SMSC_83C170_0 },
-	{ 0x10B8, 0x0005, PCI_ANY_ID, PCI_ANY_ID, 0, 0, SMSC_83C170 },
-	{ 0x10B8, 0x0006, PCI_ANY_ID, PCI_ANY_ID,
-	  PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, SMSC_83C175 },
-	{ 0,}
+	{
+		PCI_DEVICE_SUB(0x10B8, 0x0005, 0x1092, 0x0AB4),
+		.driver_data = SMSC_83C170_0,
+	}, {
+		PCI_DEVICE(0x10B8, 0x0005),
+		.driver_data = SMSC_83C170,
+	}, {
+		PCI_DEVICE(0x10B8, 0x0006),
+		.class = PCI_CLASS_NETWORK_ETHERNET << 8,
+		.class_mask = 0xffff00,
+		.driver_data = SMSC_83C175,
+	},
+	{ }
 };
 MODULE_DEVICE_TABLE (pci, epic_pci_tbl);
 
diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
index 7898b5075a8b1..b8d467ba6d72d 100644
--- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
+++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c
@@ -6,6 +6,7 @@
 #include <linux/pci.h>
 #include <linux/dmi.h>
 #include <linux/platform_data/x86/intel_pmc_ipc.h>
+#include <asm/cpuid/api.h>
 #include "dwmac-intel.h"
 #include "dwmac4.h"
 #include "stmmac.h"
diff --git a/drivers/net/ethernet/sun/cassini.c b/drivers/net/ethernet/sun/cassini.c
index fe00e7dd3fe46..823870c999bff 100644
--- a/drivers/net/ethernet/sun/cassini.c
+++ b/drivers/net/ethernet/sun/cassini.c
@@ -218,11 +218,9 @@ static u16 link_modes[] = {
 };
 
 static const struct pci_device_id cas_pci_tbl[] = {
-	{ PCI_VENDOR_ID_SUN, PCI_DEVICE_ID_SUN_CASSINI,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_NS, PCI_DEVICE_ID_NS_SATURN,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ 0, }
+	{ PCI_VDEVICE(SUN, PCI_DEVICE_ID_SUN_CASSINI) },
+	{ PCI_VDEVICE(NS, PCI_DEVICE_ID_NS_SATURN) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, cas_pci_tbl);
diff --git a/drivers/net/ethernet/sun/sungem.c b/drivers/net/ethernet/sun/sungem.c
index 8e69d917d827c..234ffc3c6a61f 100644
--- a/drivers/net/ethernet/sun/sungem.c
+++ b/drivers/net/ethernet/sun/sungem.c
@@ -85,8 +85,7 @@ MODULE_LICENSE("GPL");
 #define GEM_MODULE_NAME	"gem"
 
 static const struct pci_device_id gem_pci_tbl[] = {
-	{ PCI_VENDOR_ID_SUN, PCI_DEVICE_ID_SUN_GEM,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
+	{ PCI_VDEVICE(SUN, PCI_DEVICE_ID_SUN_GEM) },
 
 	/* These models only differ from the original GEM in
 	 * that their tx/rx fifos are of a different size and
@@ -95,21 +94,14 @@ static const struct pci_device_id gem_pci_tbl[] = {
 	 * Apple's GMAC does support gigabit on machines with
 	 * the BCM54xx PHYs. -BenH
 	 */
-	{ PCI_VENDOR_ID_SUN, PCI_DEVICE_ID_SUN_RIO_GEM,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_UNI_N_GMAC,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_UNI_N_GMACP,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_UNI_N_GMAC2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_K2_GMAC,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_SH_SUNGEM,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_IPID2_GMAC,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{0, }
+	{ PCI_VDEVICE(SUN, PCI_DEVICE_ID_SUN_RIO_GEM) },
+	{ PCI_VDEVICE(APPLE, PCI_DEVICE_ID_APPLE_UNI_N_GMAC) },
+	{ PCI_VDEVICE(APPLE, PCI_DEVICE_ID_APPLE_UNI_N_GMACP) },
+	{ PCI_VDEVICE(APPLE, PCI_DEVICE_ID_APPLE_UNI_N_GMAC2) },
+	{ PCI_VDEVICE(APPLE, PCI_DEVICE_ID_APPLE_K2_GMAC) },
+	{ PCI_VDEVICE(APPLE, PCI_DEVICE_ID_APPLE_SH_SUNGEM) },
+	{ PCI_VDEVICE(APPLE, PCI_DEVICE_ID_APPLE_IPID2_GMAC) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, gem_pci_tbl);
diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
index 7ac75fc8cdcf4..434a31080855a 100644
--- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c
+++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c
@@ -302,7 +302,7 @@ static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev,
 {
 	struct am65_cpsw_common *common = am65_ndev_to_common(ndev);
 	struct am65_cpsw_port *port = am65_ndev_to_port(ndev);
-	u32 port_mask, unreg_mcast = 0;
+	u32 port_mask;
 	int ret;
 
 	if (!common->is_emac_mode)
@@ -316,11 +316,9 @@ static int am65_cpsw_nuss_ndo_slave_add_vid(struct net_device *ndev,
 		return ret;
 
 	port_mask = BIT(port->port_id) | ALE_PORT_HOST;
-	if (!vid)
-		unreg_mcast = port_mask;
 	dev_info(common->dev, "Adding vlan %d to vlan filter\n", vid);
 	ret = cpsw_ale_vlan_add_modify(common->ale, vid, port_mask,
-				       unreg_mcast, port_mask, 0);
+				       0, port_mask, 0);
 
 	pm_runtime_put(common->dev);
 	return ret;
diff --git a/drivers/net/ethernet/ti/tlan.c b/drivers/net/ethernet/ti/tlan.c
index a55b0f951181f..274a1837025c7 100644
--- a/drivers/net/ethernet/ti/tlan.c
+++ b/drivers/net/ethernet/ti/tlan.c
@@ -119,33 +119,20 @@ static struct board {
 };
 
 static const struct pci_device_id tlan_pci_tbl[] = {
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL10,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL100,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 1 },
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_NETFLEX3I,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 2 },
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_THUNDER,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 3 },
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_NETFLEX3B,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 4 },
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL100PI,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 5 },
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL100D,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 6 },
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL100I,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 7 },
-	{ PCI_VENDOR_ID_OLICOM, PCI_DEVICE_ID_OLICOM_OC2183,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 8 },
-	{ PCI_VENDOR_ID_OLICOM, PCI_DEVICE_ID_OLICOM_OC2325,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 9 },
-	{ PCI_VENDOR_ID_OLICOM, PCI_DEVICE_ID_OLICOM_OC2326,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 10 },
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_NETELLIGENT_10_100_WS_5100,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 11 },
-	{ PCI_VENDOR_ID_COMPAQ, PCI_DEVICE_ID_NETELLIGENT_10_T2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 12 },
-	{ 0,}
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL10), .driver_data = 0 },
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL100), .driver_data = 1 },
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_COMPAQ_NETFLEX3I), .driver_data = 2 },
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_COMPAQ_THUNDER), .driver_data = 3 },
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_COMPAQ_NETFLEX3B), .driver_data = 4 },
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL100PI), .driver_data = 5 },
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL100D), .driver_data = 6 },
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_COMPAQ_NETEL100I), .driver_data = 7 },
+	{ PCI_VDEVICE(OLICOM, PCI_DEVICE_ID_OLICOM_OC2183), .driver_data = 8 },
+	{ PCI_VDEVICE(OLICOM, PCI_DEVICE_ID_OLICOM_OC2325), .driver_data = 9 },
+	{ PCI_VDEVICE(OLICOM, PCI_DEVICE_ID_OLICOM_OC2326), .driver_data = 10 },
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_NETELLIGENT_10_100_WS_5100), .driver_data = 11 },
+	{ PCI_VDEVICE(COMPAQ, PCI_DEVICE_ID_NETELLIGENT_10_T2), .driver_data = 12 },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, tlan_pci_tbl);
 
diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
index d8e3827a8b1f2..d51d8db95a763 100644
--- a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
+++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c
@@ -30,20 +30,20 @@ char ngbe_driver_name[] = "ngbe";
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id ngbe_pci_tbl[] = {
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL_W), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A2), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A2S), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A4), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A4S), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL2), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL2S), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL4), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL4S), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860LC), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A1), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A1L), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL_W) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A2) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A2S) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A4) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A4S) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL2) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL2S) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL4) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860AL4S) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860LC) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A1) },
+	{ PCI_VDEVICE(WANGXUN, NGBE_DEV_ID_EM_WX1860A1L) },
 	/* required last entry */
-	{ .device = 0 }
+	{ }
 };
 
 /**
diff --git a/drivers/net/ethernet/wangxun/ngbevf/ngbevf_main.c b/drivers/net/ethernet/wangxun/ngbevf/ngbevf_main.c
index 6ef43adcc4256..d79cf8d8484f9 100644
--- a/drivers/net/ethernet/wangxun/ngbevf/ngbevf_main.c
+++ b/drivers/net/ethernet/wangxun/ngbevf/ngbevf_main.c
@@ -26,20 +26,20 @@
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id ngbevf_pci_tbl[] = {
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL_W), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A2), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A2S), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A4), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A4S), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL2), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL2S), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL4), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL4S), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860NCSI), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A1), 0},
-	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL1), 0},
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL_W) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A2) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A2S) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A4) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A4S) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL2) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL2S) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL4) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL4S) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860NCSI) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860A1) },
+	{ PCI_VDEVICE(WANGXUN, NGBEVF_DEV_ID_EM_WX1860AL1) },
 	/* required last entry */
-	{ .device = 0 }
+	{ }
 };
 
 static const struct net_device_ops ngbevf_netdev_ops = {
diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
index 8b7c3753bb6ac..4c549c2644ab3 100644
--- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
+++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c
@@ -37,16 +37,16 @@ char txgbe_driver_name[] = "txgbe";
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id txgbe_pci_tbl[] = {
-	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_SP1000), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_WX1820), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5010), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5110), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5025), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5125), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5040), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5140), 0},
+	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_SP1000) },
+	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_WX1820) },
+	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5010) },
+	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5110) },
+	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5025) },
+	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5125) },
+	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5040) },
+	{ PCI_VDEVICE(WANGXUN, TXGBE_DEV_ID_AML5140) },
 	/* required last entry */
-	{ .device = 0 }
+	{ }
 };
 
 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
diff --git a/drivers/net/ethernet/wangxun/txgbevf/txgbevf_main.c b/drivers/net/ethernet/wangxun/txgbevf/txgbevf_main.c
index 37e4ec487afdd..8b16b900820aa 100644
--- a/drivers/net/ethernet/wangxun/txgbevf/txgbevf_main.c
+++ b/drivers/net/ethernet/wangxun/txgbevf/txgbevf_main.c
@@ -26,16 +26,16 @@
  *   Class, Class Mask, private data (not used) }
  */
 static const struct pci_device_id txgbevf_pci_tbl[] = {
-	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_SP1000), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_WX1820), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML500F), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML510F), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML5024), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML5124), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML503F), 0},
-	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML513F), 0},
+	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_SP1000) },
+	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_WX1820) },
+	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML500F) },
+	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML510F) },
+	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML5024) },
+	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML5124) },
+	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML503F) },
+	{ PCI_VDEVICE(WANGXUN, TXGBEVF_DEV_ID_AML513F) },
 	/* required last entry */
-	{ .device = 0 }
+	{ }
 };
 
 static const struct net_device_ops txgbevf_netdev_ops = {
diff --git a/drivers/net/ethernet/wiznet/Kconfig b/drivers/net/ethernet/wiznet/Kconfig
index 4bac2ad2d6a1c..bc83f6d389b14 100644
--- a/drivers/net/ethernet/wiznet/Kconfig
+++ b/drivers/net/ethernet/wiznet/Kconfig
@@ -5,7 +5,6 @@
 
 config NET_VENDOR_WIZNET
 	bool "WIZnet devices"
-	depends on HAS_IOMEM
 	default y
 	help
 	  If you have a network (Ethernet) card belonging to this class, say Y.
@@ -18,8 +17,8 @@ config NET_VENDOR_WIZNET
 if NET_VENDOR_WIZNET
 
 config WIZNET_W5100
-	tristate "WIZnet W5100 Ethernet support"
-	depends on HAS_IOMEM
+	tristate "WIZnet W5100/W5200/W5500 Ethernet support for SPI mode"
+	depends on SPI
 	help
 	  Support for WIZnet W5100 chips.
 
@@ -30,58 +29,4 @@ config WIZNET_W5100
 	  To compile this driver as a module, choose M here: the module
 	  will be called w5100.
 
-config WIZNET_W5300
-	tristate "WIZnet W5300 Ethernet support"
-	depends on HAS_IOMEM
-	help
-	  Support for WIZnet W5300 chips.
-
-	  W5300 is a single chip with integrated 10/100 Ethernet MAC,
-	  PHY and hardware TCP/IP stack, but this driver is limited to
-	  the MAC and PHY functions only, onchip TCP/IP is unused.
-
-	  To compile this driver as a module, choose M here: the module
-	  will be called w5300.
-
-choice
-	prompt "WIZnet interface mode"
-	depends on WIZNET_W5100 || WIZNET_W5300
-	default WIZNET_BUS_ANY
-
-config WIZNET_BUS_DIRECT
-	bool "Direct address bus mode"
-	help
-	  In direct address mode host system can directly access all registers
-	  after mapping to Memory-Mapped I/O space.
-
-config WIZNET_BUS_INDIRECT
-	bool "Indirect address bus mode"
-	help
-	  In indirect address mode host system indirectly accesses registers
-	  using Indirect Mode Address Register and Indirect Mode Data Register,
-	  which are directly mapped to Memory-Mapped I/O space.
-
-config WIZNET_BUS_ANY
-	bool "Select interface mode in runtime"
-	help
-	  If interface mode is unknown in compile time, it can be selected
-	  in runtime from board/platform resources configuration.
-
-	  Performance may decrease compared to explicitly selected bus mode.
-endchoice
-
-config WIZNET_W5100_SPI
-	tristate "WIZnet W5100/W5200/W5500 Ethernet support for SPI mode"
-	depends on WIZNET_BUS_ANY && WIZNET_W5100
-	depends on SPI
-	help
-	  In SPI mode host system accesses registers using SPI protocol
-	  (mode 0) on the SPI bus.
-
-	  Performance decreases compared to other bus interface mode.
-	  In W5100 SPI mode, burst READ/WRITE processing are not provided.
-
-	  To compile this driver as a module, choose M here: the module
-	  will be called w5100-spi.
-
 endif # NET_VENDOR_WIZNET
diff --git a/drivers/net/ethernet/wiznet/Makefile b/drivers/net/ethernet/wiznet/Makefile
index 78104f0bf4153..502150d6ead2d 100644
--- a/drivers/net/ethernet/wiznet/Makefile
+++ b/drivers/net/ethernet/wiznet/Makefile
@@ -1,4 +1,2 @@
 # SPDX-License-Identifier: GPL-2.0-only
-obj-$(CONFIG_WIZNET_W5100) += w5100.o
-obj-$(CONFIG_WIZNET_W5100_SPI) += w5100-spi.o
-obj-$(CONFIG_WIZNET_W5300) += w5300.o
+obj-$(CONFIG_WIZNET_W5100) += w5100.o w5100-spi.o
diff --git a/drivers/net/ethernet/wiznet/w5100-spi.c b/drivers/net/ethernet/wiznet/w5100-spi.c
index 990a3cce8c0fc..d2c5e99ab9c79 100644
--- a/drivers/net/ethernet/wiznet/w5100-spi.c
+++ b/drivers/net/ethernet/wiznet/w5100-spi.c
@@ -450,7 +450,7 @@ static int w5100_spi_probe(struct spi_device *spi)
 		return -EINVAL;
 	}
 
-	return w5100_probe(&spi->dev, ops, priv_size, mac, spi->irq, -EINVAL);
+	return w5100_probe(&spi->dev, ops, priv_size, mac, spi->irq);
 }
 
 static void w5100_spi_remove(struct spi_device *spi)
diff --git a/drivers/net/ethernet/wiznet/w5100.c b/drivers/net/ethernet/wiznet/w5100.c
index c5424d8821359..53d8dc642fbd6 100644
--- a/drivers/net/ethernet/wiznet/w5100.c
+++ b/drivers/net/ethernet/wiznet/w5100.c
@@ -11,7 +11,6 @@
 #include <linux/netdevice.h>
 #include <linux/etherdevice.h>
 #include <linux/platform_device.h>
-#include <linux/platform_data/wiznet.h>
 #include <linux/ethtool.h>
 #include <linux/skbuff.h>
 #include <linux/types.h>
@@ -23,7 +22,6 @@
 #include <linux/ioport.h>
 #include <linux/interrupt.h>
 #include <linux/irq.h>
-#include <linux/gpio.h>
 
 #include "w5100.h"
 
@@ -156,8 +154,6 @@ struct w5100_priv {
 	u16 s0_rx_buf_size;
 
 	int irq;
-	int link_irq;
-	int link_gpio;
 
 	struct napi_struct napi;
 	struct net_device *ndev;
@@ -172,311 +168,6 @@ struct w5100_priv {
 	struct work_struct restart_work;
 };
 
-/************************************************************************
- *
- *  Lowlevel I/O functions
- *
- ***********************************************************************/
-
-struct w5100_mmio_priv {
-	void __iomem *base;
-	/* Serialize access in indirect address mode */
-	spinlock_t reg_lock;
-};
-
-static inline struct w5100_mmio_priv *w5100_mmio_priv(struct net_device *dev)
-{
-	return w5100_ops_priv(dev);
-}
-
-static inline void __iomem *w5100_mmio(struct net_device *ndev)
-{
-	struct w5100_mmio_priv *mmio_priv = w5100_mmio_priv(ndev);
-
-	return mmio_priv->base;
-}
-
-/*
- * In direct address mode host system can directly access W5100 registers
- * after mapping to Memory-Mapped I/O space.
- *
- * 0x8000 bytes are required for memory space.
- */
-static inline int w5100_read_direct(struct net_device *ndev, u32 addr)
-{
-	return ioread8(w5100_mmio(ndev) + (addr << CONFIG_WIZNET_BUS_SHIFT));
-}
-
-static inline int __w5100_write_direct(struct net_device *ndev, u32 addr,
-				       u8 data)
-{
-	iowrite8(data, w5100_mmio(ndev) + (addr << CONFIG_WIZNET_BUS_SHIFT));
-
-	return 0;
-}
-
-static inline int w5100_write_direct(struct net_device *ndev, u32 addr, u8 data)
-{
-	__w5100_write_direct(ndev, addr, data);
-
-	return 0;
-}
-
-static int w5100_read16_direct(struct net_device *ndev, u32 addr)
-{
-	u16 data;
-	data  = w5100_read_direct(ndev, addr) << 8;
-	data |= w5100_read_direct(ndev, addr + 1);
-	return data;
-}
-
-static int w5100_write16_direct(struct net_device *ndev, u32 addr, u16 data)
-{
-	__w5100_write_direct(ndev, addr, data >> 8);
-	__w5100_write_direct(ndev, addr + 1, data);
-
-	return 0;
-}
-
-static int w5100_readbulk_direct(struct net_device *ndev, u32 addr, u8 *buf,
-				 int len)
-{
-	int i;
-
-	for (i = 0; i < len; i++, addr++)
-		*buf++ = w5100_read_direct(ndev, addr);
-
-	return 0;
-}
-
-static int w5100_writebulk_direct(struct net_device *ndev, u32 addr,
-				  const u8 *buf, int len)
-{
-	int i;
-
-	for (i = 0; i < len; i++, addr++)
-		__w5100_write_direct(ndev, addr, *buf++);
-
-	return 0;
-}
-
-static int w5100_mmio_init(struct net_device *ndev)
-{
-	struct platform_device *pdev = to_platform_device(ndev->dev.parent);
-	struct w5100_mmio_priv *mmio_priv = w5100_mmio_priv(ndev);
-
-	spin_lock_init(&mmio_priv->reg_lock);
-
-	mmio_priv->base = devm_platform_get_and_ioremap_resource(pdev, 0, NULL);
-	if (IS_ERR(mmio_priv->base))
-		return PTR_ERR(mmio_priv->base);
-
-	return 0;
-}
-
-static const struct w5100_ops w5100_mmio_direct_ops = {
-	.chip_id = W5100,
-	.read = w5100_read_direct,
-	.write = w5100_write_direct,
-	.read16 = w5100_read16_direct,
-	.write16 = w5100_write16_direct,
-	.readbulk = w5100_readbulk_direct,
-	.writebulk = w5100_writebulk_direct,
-	.init = w5100_mmio_init,
-};
-
-/*
- * In indirect address mode host system indirectly accesses registers by
- * using Indirect Mode Address Register (IDM_AR) and Indirect Mode Data
- * Register (IDM_DR), which are directly mapped to Memory-Mapped I/O space.
- * Mode Register (MR) is directly accessible.
- *
- * Only 0x04 bytes are required for memory space.
- */
-#define W5100_IDM_AR		0x01   /* Indirect Mode Address Register */
-#define W5100_IDM_DR		0x03   /* Indirect Mode Data Register */
-
-static int w5100_read_indirect(struct net_device *ndev, u32 addr)
-{
-	struct w5100_mmio_priv *mmio_priv = w5100_mmio_priv(ndev);
-	unsigned long flags;
-	u8 data;
-
-	spin_lock_irqsave(&mmio_priv->reg_lock, flags);
-	w5100_write16_direct(ndev, W5100_IDM_AR, addr);
-	data = w5100_read_direct(ndev, W5100_IDM_DR);
-	spin_unlock_irqrestore(&mmio_priv->reg_lock, flags);
-
-	return data;
-}
-
-static int w5100_write_indirect(struct net_device *ndev, u32 addr, u8 data)
-{
-	struct w5100_mmio_priv *mmio_priv = w5100_mmio_priv(ndev);
-	unsigned long flags;
-
-	spin_lock_irqsave(&mmio_priv->reg_lock, flags);
-	w5100_write16_direct(ndev, W5100_IDM_AR, addr);
-	w5100_write_direct(ndev, W5100_IDM_DR, data);
-	spin_unlock_irqrestore(&mmio_priv->reg_lock, flags);
-
-	return 0;
-}
-
-static int w5100_read16_indirect(struct net_device *ndev, u32 addr)
-{
-	struct w5100_mmio_priv *mmio_priv = w5100_mmio_priv(ndev);
-	unsigned long flags;
-	u16 data;
-
-	spin_lock_irqsave(&mmio_priv->reg_lock, flags);
-	w5100_write16_direct(ndev, W5100_IDM_AR, addr);
-	data  = w5100_read_direct(ndev, W5100_IDM_DR) << 8;
-	data |= w5100_read_direct(ndev, W5100_IDM_DR);
-	spin_unlock_irqrestore(&mmio_priv->reg_lock, flags);
-
-	return data;
-}
-
-static int w5100_write16_indirect(struct net_device *ndev, u32 addr, u16 data)
-{
-	struct w5100_mmio_priv *mmio_priv = w5100_mmio_priv(ndev);
-	unsigned long flags;
-
-	spin_lock_irqsave(&mmio_priv->reg_lock, flags);
-	w5100_write16_direct(ndev, W5100_IDM_AR, addr);
-	__w5100_write_direct(ndev, W5100_IDM_DR, data >> 8);
-	w5100_write_direct(ndev, W5100_IDM_DR, data);
-	spin_unlock_irqrestore(&mmio_priv->reg_lock, flags);
-
-	return 0;
-}
-
-static int w5100_readbulk_indirect(struct net_device *ndev, u32 addr, u8 *buf,
-				   int len)
-{
-	struct w5100_mmio_priv *mmio_priv = w5100_mmio_priv(ndev);
-	unsigned long flags;
-	int i;
-
-	spin_lock_irqsave(&mmio_priv->reg_lock, flags);
-	w5100_write16_direct(ndev, W5100_IDM_AR, addr);
-
-	for (i = 0; i < len; i++)
-		*buf++ = w5100_read_direct(ndev, W5100_IDM_DR);
-
-	spin_unlock_irqrestore(&mmio_priv->reg_lock, flags);
-
-	return 0;
-}
-
-static int w5100_writebulk_indirect(struct net_device *ndev, u32 addr,
-				    const u8 *buf, int len)
-{
-	struct w5100_mmio_priv *mmio_priv = w5100_mmio_priv(ndev);
-	unsigned long flags;
-	int i;
-
-	spin_lock_irqsave(&mmio_priv->reg_lock, flags);
-	w5100_write16_direct(ndev, W5100_IDM_AR, addr);
-
-	for (i = 0; i < len; i++)
-		__w5100_write_direct(ndev, W5100_IDM_DR, *buf++);
-
-	spin_unlock_irqrestore(&mmio_priv->reg_lock, flags);
-
-	return 0;
-}
-
-static int w5100_reset_indirect(struct net_device *ndev)
-{
-	w5100_write_direct(ndev, W5100_MR, MR_RST);
-	mdelay(5);
-	w5100_write_direct(ndev, W5100_MR, MR_PB | MR_AI | MR_IND);
-
-	return 0;
-}
-
-static const struct w5100_ops w5100_mmio_indirect_ops = {
-	.chip_id = W5100,
-	.read = w5100_read_indirect,
-	.write = w5100_write_indirect,
-	.read16 = w5100_read16_indirect,
-	.write16 = w5100_write16_indirect,
-	.readbulk = w5100_readbulk_indirect,
-	.writebulk = w5100_writebulk_indirect,
-	.init = w5100_mmio_init,
-	.reset = w5100_reset_indirect,
-};
-
-#if defined(CONFIG_WIZNET_BUS_DIRECT)
-
-static int w5100_read(struct w5100_priv *priv, u32 addr)
-{
-	return w5100_read_direct(priv->ndev, addr);
-}
-
-static int w5100_write(struct w5100_priv *priv, u32 addr, u8 data)
-{
-	return w5100_write_direct(priv->ndev, addr, data);
-}
-
-static int w5100_read16(struct w5100_priv *priv, u32 addr)
-{
-	return w5100_read16_direct(priv->ndev, addr);
-}
-
-static int w5100_write16(struct w5100_priv *priv, u32 addr, u16 data)
-{
-	return w5100_write16_direct(priv->ndev, addr, data);
-}
-
-static int w5100_readbulk(struct w5100_priv *priv, u32 addr, u8 *buf, int len)
-{
-	return w5100_readbulk_direct(priv->ndev, addr, buf, len);
-}
-
-static int w5100_writebulk(struct w5100_priv *priv, u32 addr, const u8 *buf,
-			   int len)
-{
-	return w5100_writebulk_direct(priv->ndev, addr, buf, len);
-}
-
-#elif defined(CONFIG_WIZNET_BUS_INDIRECT)
-
-static int w5100_read(struct w5100_priv *priv, u32 addr)
-{
-	return w5100_read_indirect(priv->ndev, addr);
-}
-
-static int w5100_write(struct w5100_priv *priv, u32 addr, u8 data)
-{
-	return w5100_write_indirect(priv->ndev, addr, data);
-}
-
-static int w5100_read16(struct w5100_priv *priv, u32 addr)
-{
-	return w5100_read16_indirect(priv->ndev, addr);
-}
-
-static int w5100_write16(struct w5100_priv *priv, u32 addr, u16 data)
-{
-	return w5100_write16_indirect(priv->ndev, addr, data);
-}
-
-static int w5100_readbulk(struct w5100_priv *priv, u32 addr, u8 *buf, int len)
-{
-	return w5100_readbulk_indirect(priv->ndev, addr, buf, len);
-}
-
-static int w5100_writebulk(struct w5100_priv *priv, u32 addr, const u8 *buf,
-			   int len)
-{
-	return w5100_writebulk_indirect(priv->ndev, addr, buf, len);
-}
-
-#else /* CONFIG_WIZNET_BUS_ANY */
-
 static int w5100_read(struct w5100_priv *priv, u32 addr)
 {
 	return priv->ops->read(priv->ndev, addr);
@@ -508,8 +199,6 @@ static int w5100_writebulk(struct w5100_priv *priv, u32 addr, const u8 *buf,
 	return priv->ops->writebulk(priv->ndev, addr, buf, len);
 }
 
-#endif
-
 static int w5100_readbuf(struct w5100_priv *priv, u16 offset, u8 *buf, int len)
 {
 	u32 addr;
@@ -725,16 +414,6 @@ static void w5100_get_drvinfo(struct net_device *ndev,
 		sizeof(info->bus_info));
 }
 
-static u32 w5100_get_link(struct net_device *ndev)
-{
-	struct w5100_priv *priv = netdev_priv(ndev);
-
-	if (gpio_is_valid(priv->link_gpio))
-		return !!gpio_get_value(priv->link_gpio);
-
-	return 1;
-}
-
 static u32 w5100_get_msglevel(struct net_device *ndev)
 {
 	struct w5100_priv *priv = netdev_priv(ndev);
@@ -937,24 +616,6 @@ static irqreturn_t w5100_interrupt(int irq, void *ndev_instance)
 	return IRQ_HANDLED;
 }
 
-static irqreturn_t w5100_detect_link(int irq, void *ndev_instance)
-{
-	struct net_device *ndev = ndev_instance;
-	struct w5100_priv *priv = netdev_priv(ndev);
-
-	if (netif_running(ndev)) {
-		if (gpio_get_value(priv->link_gpio) != 0) {
-			netif_info(priv, link, ndev, "link is up\n");
-			netif_carrier_on(ndev);
-		} else {
-			netif_info(priv, link, ndev, "link is down\n");
-			netif_carrier_off(ndev);
-		}
-	}
-
-	return IRQ_HANDLED;
-}
-
 static void w5100_setrx_work(struct work_struct *work)
 {
 	struct w5100_priv *priv = container_of(work, struct w5100_priv,
@@ -998,9 +659,6 @@ static int w5100_open(struct net_device *ndev)
 	w5100_hw_start(priv);
 	napi_enable(&priv->napi);
 	netif_start_queue(ndev);
-	if (!gpio_is_valid(priv->link_gpio) ||
-	    gpio_get_value(priv->link_gpio) != 0)
-		netif_carrier_on(ndev);
 	return 0;
 }
 
@@ -1020,7 +678,6 @@ static const struct ethtool_ops w5100_ethtool_ops = {
 	.get_drvinfo		= w5100_get_drvinfo,
 	.get_msglevel		= w5100_get_msglevel,
 	.set_msglevel		= w5100_set_msglevel,
-	.get_link		= w5100_get_link,
 	.get_regs_len		= w5100_get_regs_len,
 	.get_regs		= w5100_get_regs,
 };
@@ -1035,38 +692,6 @@ static const struct net_device_ops w5100_netdev_ops = {
 	.ndo_validate_addr	= eth_validate_addr,
 };
 
-static int w5100_mmio_probe(struct platform_device *pdev)
-{
-	struct wiznet_platform_data *data = dev_get_platdata(&pdev->dev);
-	const void *mac_addr = NULL;
-	struct resource *mem;
-	const struct w5100_ops *ops;
-	int irq;
-
-	if (data && is_valid_ether_addr(data->mac_addr))
-		mac_addr = data->mac_addr;
-
-	mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-	if (!mem)
-		return -EINVAL;
-	if (resource_size(mem) < W5100_BUS_DIRECT_SIZE)
-		ops = &w5100_mmio_indirect_ops;
-	else
-		ops = &w5100_mmio_direct_ops;
-
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-
-	return w5100_probe(&pdev->dev, ops, sizeof(struct w5100_mmio_priv),
-			   mac_addr, irq, data ? data->link_gpio : -EINVAL);
-}
-
-static void w5100_mmio_remove(struct platform_device *pdev)
-{
-	w5100_remove(&pdev->dev);
-}
-
 void *w5100_ops_priv(const struct net_device *ndev)
 {
 	return netdev_priv(ndev) +
@@ -1075,8 +700,7 @@ void *w5100_ops_priv(const struct net_device *ndev)
 EXPORT_SYMBOL_GPL(w5100_ops_priv);
 
 int w5100_probe(struct device *dev, const struct w5100_ops *ops,
-		int sizeof_ops_priv, const void *mac_addr, int irq,
-		int link_gpio)
+		int sizeof_ops_priv, const void *mac_addr, int irq)
 {
 	struct w5100_priv *priv;
 	struct net_device *ndev;
@@ -1127,7 +751,6 @@ int w5100_probe(struct device *dev, const struct w5100_ops *ops,
 	priv->ndev = ndev;
 	priv->ops = ops;
 	priv->irq = irq;
-	priv->link_gpio = link_gpio;
 
 	ndev->netdev_ops = &w5100_netdev_ops;
 	ndev->ethtool_ops = &w5100_ethtool_ops;
@@ -1180,26 +803,8 @@ int w5100_probe(struct device *dev, const struct w5100_ops *ops,
 	if (err)
 		goto err_hw;
 
-	if (gpio_is_valid(priv->link_gpio)) {
-		char *link_name = devm_kzalloc(dev, 16, GFP_KERNEL);
-
-		if (!link_name) {
-			err = -ENOMEM;
-			goto err_gpio;
-		}
-		snprintf(link_name, 16, "%s-link", netdev_name(ndev));
-		priv->link_irq = gpio_to_irq(priv->link_gpio);
-		if (request_any_context_irq(priv->link_irq, w5100_detect_link,
-					    IRQF_TRIGGER_RISING |
-					    IRQF_TRIGGER_FALLING,
-					    link_name, priv->ndev) < 0)
-			priv->link_gpio = -EINVAL;
-	}
-
 	return 0;
 
-err_gpio:
-	free_irq(priv->irq, ndev);
 err_hw:
 	destroy_workqueue(priv->xfer_wq);
 err_wq:
@@ -1217,8 +822,6 @@ void w5100_remove(struct device *dev)
 
 	w5100_hw_reset(priv);
 	free_irq(priv->irq, ndev);
-	if (gpio_is_valid(priv->link_gpio))
-		free_irq(priv->link_irq, ndev);
 
 	flush_work(&priv->setrx_work);
 	flush_work(&priv->restart_work);
@@ -1254,9 +857,6 @@ static int w5100_resume(struct device *dev)
 		w5100_hw_start(priv);
 
 		netif_device_attach(ndev);
-		if (!gpio_is_valid(priv->link_gpio) ||
-		    gpio_get_value(priv->link_gpio) != 0)
-			netif_carrier_on(ndev);
 	}
 	return 0;
 }
@@ -1264,13 +864,3 @@ static int w5100_resume(struct device *dev)
 
 SIMPLE_DEV_PM_OPS(w5100_pm_ops, w5100_suspend, w5100_resume);
 EXPORT_SYMBOL_GPL(w5100_pm_ops);
-
-static struct platform_driver w5100_mmio_driver = {
-	.driver		= {
-		.name	= DRV_NAME,
-		.pm	= &w5100_pm_ops,
-	},
-	.probe		= w5100_mmio_probe,
-	.remove		= w5100_mmio_remove,
-};
-module_platform_driver(w5100_mmio_driver);
diff --git a/drivers/net/ethernet/wiznet/w5100.h b/drivers/net/ethernet/wiznet/w5100.h
index 481af3b6d9e8c..76e1b8149041f 100644
--- a/drivers/net/ethernet/wiznet/w5100.h
+++ b/drivers/net/ethernet/wiznet/w5100.h
@@ -29,8 +29,7 @@ struct w5100_ops {
 void *w5100_ops_priv(const struct net_device *ndev);
 
 int w5100_probe(struct device *dev, const struct w5100_ops *ops,
-		int sizeof_ops_priv, const void *mac_addr, int irq,
-		int link_gpio);
+		int sizeof_ops_priv, const void *mac_addr, int irq);
 void w5100_remove(struct device *dev);
 
 extern const struct dev_pm_ops w5100_pm_ops;
diff --git a/drivers/net/ethernet/wiznet/w5300.c b/drivers/net/ethernet/wiznet/w5300.c
deleted file mode 100644
index 3e711dea3b2c8..0000000000000
--- a/drivers/net/ethernet/wiznet/w5300.c
+++ /dev/null
@@ -1,687 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Ethernet driver for the WIZnet W5300 chip.
- *
- * Copyright (C) 2008-2009 WIZnet Co.,Ltd.
- * Copyright (C) 2011 Taehun Kim <kth3321 <at> gmail.com>
- * Copyright (C) 2012 Mike Sinkovsky <msink@permonline.ru>
- */
-
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/netdevice.h>
-#include <linux/etherdevice.h>
-#include <linux/platform_device.h>
-#include <linux/platform_data/wiznet.h>
-#include <linux/ethtool.h>
-#include <linux/skbuff.h>
-#include <linux/types.h>
-#include <linux/errno.h>
-#include <linux/delay.h>
-#include <linux/slab.h>
-#include <linux/spinlock.h>
-#include <linux/io.h>
-#include <linux/ioport.h>
-#include <linux/interrupt.h>
-#include <linux/irq.h>
-#include <linux/gpio.h>
-
-#define DRV_NAME	"w5300"
-#define DRV_VERSION	"2012-04-04"
-
-MODULE_DESCRIPTION("WIZnet W5300 Ethernet driver v"DRV_VERSION);
-MODULE_AUTHOR("Mike Sinkovsky <msink@permonline.ru>");
-MODULE_ALIAS("platform:"DRV_NAME);
-MODULE_LICENSE("GPL");
-
-/*
- * Registers
- */
-#define W5300_MR		0x0000	/* Mode Register */
-#define   MR_DBW		  (1 << 15) /* Data bus width */
-#define   MR_MPF		  (1 << 14) /* Mac layer pause frame */
-#define   MR_WDF(n)		  (((n)&7)<<11) /* Write data fetch time */
-#define   MR_RDH		  (1 << 10) /* Read data hold time */
-#define   MR_FS			  (1 << 8)  /* FIFO swap */
-#define   MR_RST		  (1 << 7)  /* S/W reset */
-#define   MR_PB			  (1 << 4)  /* Ping block */
-#define   MR_DBS		  (1 << 2)  /* Data bus swap */
-#define   MR_IND		  (1 << 0)  /* Indirect mode */
-#define W5300_IR		0x0002	/* Interrupt Register */
-#define W5300_IMR		0x0004	/* Interrupt Mask Register */
-#define   IR_S0			  0x0001  /* S0 interrupt */
-#define W5300_SHARL		0x0008	/* Source MAC address (0123) */
-#define W5300_SHARH		0x000c	/* Source MAC address (45) */
-#define W5300_TMSRL		0x0020	/* Transmit Memory Size (0123) */
-#define W5300_TMSRH		0x0024	/* Transmit Memory Size (4567) */
-#define W5300_RMSRL		0x0028	/* Receive Memory Size (0123) */
-#define W5300_RMSRH		0x002c	/* Receive Memory Size (4567) */
-#define W5300_MTYPE		0x0030	/* Memory Type */
-#define W5300_IDR		0x00fe	/* Chip ID register */
-#define   IDR_W5300		  0x5300  /* =0x5300 for WIZnet W5300 */
-#define W5300_S0_MR		0x0200	/* S0 Mode Register */
-#define   S0_MR_CLOSED		  0x0000  /* Close mode */
-#define   S0_MR_MACRAW		  0x0004  /* MAC RAW mode (promiscuous) */
-#define   S0_MR_MACRAW_MF	  0x0044  /* MAC RAW mode (filtered) */
-#define W5300_S0_CR		0x0202	/* S0 Command Register */
-#define   S0_CR_OPEN		  0x0001  /* OPEN command */
-#define   S0_CR_CLOSE		  0x0010  /* CLOSE command */
-#define   S0_CR_SEND		  0x0020  /* SEND command */
-#define   S0_CR_RECV		  0x0040  /* RECV command */
-#define W5300_S0_IMR		0x0204	/* S0 Interrupt Mask Register */
-#define W5300_S0_IR		0x0206	/* S0 Interrupt Register */
-#define   S0_IR_RECV		  0x0004  /* Receive interrupt */
-#define   S0_IR_SENDOK		  0x0010  /* Send OK interrupt */
-#define W5300_S0_SSR		0x0208	/* S0 Socket Status Register */
-#define W5300_S0_TX_WRSR	0x0220	/* S0 TX Write Size Register */
-#define W5300_S0_TX_FSR		0x0224	/* S0 TX Free Size Register */
-#define W5300_S0_RX_RSR		0x0228	/* S0 Received data Size */
-#define W5300_S0_TX_FIFO	0x022e	/* S0 Transmit FIFO */
-#define W5300_S0_RX_FIFO	0x0230	/* S0 Receive FIFO */
-#define W5300_REGS_LEN		0x0400
-
-/*
- * Device driver private data structure
- */
-struct w5300_priv {
-	void __iomem *base;
-	spinlock_t reg_lock;
-	bool indirect;
-	u16  (*read) (struct w5300_priv *priv, u16 addr);
-	void (*write)(struct w5300_priv *priv, u16 addr, u16 data);
-	int irq;
-	int link_irq;
-	int link_gpio;
-
-	struct napi_struct napi;
-	struct net_device *ndev;
-	bool promisc;
-	u32 msg_enable;
-};
-
-/************************************************************************
- *
- *  Lowlevel I/O functions
- *
- ***********************************************************************/
-
-/*
- * In direct address mode host system can directly access W5300 registers
- * after mapping to Memory-Mapped I/O space.
- *
- * 0x400 bytes are required for memory space.
- */
-static inline u16 w5300_read_direct(struct w5300_priv *priv, u16 addr)
-{
-	return ioread16(priv->base + (addr << CONFIG_WIZNET_BUS_SHIFT));
-}
-
-static inline void w5300_write_direct(struct w5300_priv *priv,
-				      u16 addr, u16 data)
-{
-	iowrite16(data, priv->base + (addr << CONFIG_WIZNET_BUS_SHIFT));
-}
-
-/*
- * In indirect address mode host system indirectly accesses registers by
- * using Indirect Mode Address Register (IDM_AR) and Indirect Mode Data
- * Register (IDM_DR), which are directly mapped to Memory-Mapped I/O space.
- * Mode Register (MR) is directly accessible.
- *
- * Only 0x06 bytes are required for memory space.
- */
-#define W5300_IDM_AR		0x0002	 /* Indirect Mode Address */
-#define W5300_IDM_DR		0x0004	 /* Indirect Mode Data */
-
-static u16 w5300_read_indirect(struct w5300_priv *priv, u16 addr)
-{
-	unsigned long flags;
-	u16 data;
-
-	spin_lock_irqsave(&priv->reg_lock, flags);
-	w5300_write_direct(priv, W5300_IDM_AR, addr);
-	data = w5300_read_direct(priv, W5300_IDM_DR);
-	spin_unlock_irqrestore(&priv->reg_lock, flags);
-
-	return data;
-}
-
-static void w5300_write_indirect(struct w5300_priv *priv, u16 addr, u16 data)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&priv->reg_lock, flags);
-	w5300_write_direct(priv, W5300_IDM_AR, addr);
-	w5300_write_direct(priv, W5300_IDM_DR, data);
-	spin_unlock_irqrestore(&priv->reg_lock, flags);
-}
-
-#if defined(CONFIG_WIZNET_BUS_DIRECT)
-#define w5300_read	w5300_read_direct
-#define w5300_write	w5300_write_direct
-
-#elif defined(CONFIG_WIZNET_BUS_INDIRECT)
-#define w5300_read	w5300_read_indirect
-#define w5300_write	w5300_write_indirect
-
-#else /* CONFIG_WIZNET_BUS_ANY */
-#define w5300_read	priv->read
-#define w5300_write	priv->write
-#endif
-
-static u32 w5300_read32(struct w5300_priv *priv, u16 addr)
-{
-	u32 data;
-	data  = w5300_read(priv, addr) << 16;
-	data |= w5300_read(priv, addr + 2);
-	return data;
-}
-
-static void w5300_write32(struct w5300_priv *priv, u16 addr, u32 data)
-{
-	w5300_write(priv, addr, data >> 16);
-	w5300_write(priv, addr + 2, data);
-}
-
-static int w5300_command(struct w5300_priv *priv, u16 cmd)
-{
-	unsigned long timeout = jiffies + msecs_to_jiffies(100);
-
-	w5300_write(priv, W5300_S0_CR, cmd);
-
-	while (w5300_read(priv, W5300_S0_CR) != 0) {
-		if (time_after(jiffies, timeout))
-			return -EIO;
-		cpu_relax();
-	}
-
-	return 0;
-}
-
-static void w5300_read_frame(struct w5300_priv *priv, u8 *buf, int len)
-{
-	u16 fifo;
-	int i;
-
-	for (i = 0; i < len; i += 2) {
-		fifo = w5300_read(priv, W5300_S0_RX_FIFO);
-		*buf++ = fifo >> 8;
-		*buf++ = fifo;
-	}
-	fifo = w5300_read(priv, W5300_S0_RX_FIFO);
-	fifo = w5300_read(priv, W5300_S0_RX_FIFO);
-}
-
-static void w5300_write_frame(struct w5300_priv *priv, u8 *buf, int len)
-{
-	u16 fifo;
-	int i;
-
-	for (i = 0; i < len; i += 2) {
-		fifo  = *buf++ << 8;
-		fifo |= *buf++;
-		w5300_write(priv, W5300_S0_TX_FIFO, fifo);
-	}
-	w5300_write32(priv, W5300_S0_TX_WRSR, len);
-}
-
-static void w5300_write_macaddr(struct w5300_priv *priv)
-{
-	struct net_device *ndev = priv->ndev;
-	w5300_write32(priv, W5300_SHARL,
-		      ndev->dev_addr[0] << 24 |
-		      ndev->dev_addr[1] << 16 |
-		      ndev->dev_addr[2] << 8 |
-		      ndev->dev_addr[3]);
-	w5300_write(priv, W5300_SHARH,
-		      ndev->dev_addr[4] << 8 |
-		      ndev->dev_addr[5]);
-}
-
-static void w5300_hw_reset(struct w5300_priv *priv)
-{
-	w5300_write_direct(priv, W5300_MR, MR_RST);
-	mdelay(5);
-	w5300_write_direct(priv, W5300_MR, priv->indirect ?
-				 MR_WDF(7) | MR_PB | MR_IND :
-				 MR_WDF(7) | MR_PB);
-	w5300_write(priv, W5300_IMR, 0);
-	w5300_write_macaddr(priv);
-
-	/* Configure 128K of internal memory
-	 * as 64K RX fifo and 64K TX fifo
-	 */
-	w5300_write32(priv, W5300_RMSRL, 64 << 24);
-	w5300_write32(priv, W5300_RMSRH, 0);
-	w5300_write32(priv, W5300_TMSRL, 64 << 24);
-	w5300_write32(priv, W5300_TMSRH, 0);
-	w5300_write(priv, W5300_MTYPE, 0x00ff);
-}
-
-static void w5300_hw_start(struct w5300_priv *priv)
-{
-	w5300_write(priv, W5300_S0_MR, priv->promisc ?
-			  S0_MR_MACRAW : S0_MR_MACRAW_MF);
-	w5300_command(priv, S0_CR_OPEN);
-	w5300_write(priv, W5300_S0_IMR, S0_IR_RECV | S0_IR_SENDOK);
-	w5300_write(priv, W5300_IMR, IR_S0);
-}
-
-static void w5300_hw_close(struct w5300_priv *priv)
-{
-	w5300_write(priv, W5300_IMR, 0);
-	w5300_command(priv, S0_CR_CLOSE);
-}
-
-/***********************************************************************
- *
- *   Device driver functions / callbacks
- *
- ***********************************************************************/
-
-static void w5300_get_drvinfo(struct net_device *ndev,
-			      struct ethtool_drvinfo *info)
-{
-	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
-	strscpy(info->version, DRV_VERSION, sizeof(info->version));
-	strscpy(info->bus_info, dev_name(ndev->dev.parent),
-		sizeof(info->bus_info));
-}
-
-static u32 w5300_get_link(struct net_device *ndev)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	if (gpio_is_valid(priv->link_gpio))
-		return !!gpio_get_value(priv->link_gpio);
-
-	return 1;
-}
-
-static u32 w5300_get_msglevel(struct net_device *ndev)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	return priv->msg_enable;
-}
-
-static void w5300_set_msglevel(struct net_device *ndev, u32 value)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	priv->msg_enable = value;
-}
-
-static int w5300_get_regs_len(struct net_device *ndev)
-{
-	return W5300_REGS_LEN;
-}
-
-static void w5300_get_regs(struct net_device *ndev,
-			   struct ethtool_regs *regs, void *_buf)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-	u8 *buf = _buf;
-	u16 addr;
-	u16 data;
-
-	regs->version = 1;
-	for (addr = 0; addr < W5300_REGS_LEN; addr += 2) {
-		switch (addr & 0x23f) {
-		case W5300_S0_TX_FIFO: /* cannot read TX_FIFO */
-		case W5300_S0_RX_FIFO: /* cannot read RX_FIFO */
-			data = 0xffff;
-			break;
-		default:
-			data = w5300_read(priv, addr);
-			break;
-		}
-		*buf++ = data >> 8;
-		*buf++ = data;
-	}
-}
-
-static void w5300_tx_timeout(struct net_device *ndev, unsigned int txqueue)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	netif_stop_queue(ndev);
-	w5300_hw_reset(priv);
-	w5300_hw_start(priv);
-	ndev->stats.tx_errors++;
-	netif_trans_update(ndev);
-	netif_wake_queue(ndev);
-}
-
-static netdev_tx_t w5300_start_tx(struct sk_buff *skb, struct net_device *ndev)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	netif_stop_queue(ndev);
-
-	w5300_write_frame(priv, skb->data, skb->len);
-	ndev->stats.tx_packets++;
-	ndev->stats.tx_bytes += skb->len;
-	dev_kfree_skb(skb);
-	netif_dbg(priv, tx_queued, ndev, "tx queued\n");
-
-	w5300_command(priv, S0_CR_SEND);
-
-	return NETDEV_TX_OK;
-}
-
-static int w5300_napi_poll(struct napi_struct *napi, int budget)
-{
-	struct w5300_priv *priv = container_of(napi, struct w5300_priv, napi);
-	struct net_device *ndev = priv->ndev;
-	struct sk_buff *skb;
-	int rx_count;
-	u16 rx_len;
-
-	for (rx_count = 0; rx_count < budget; rx_count++) {
-		u32 rx_fifo_len = w5300_read32(priv, W5300_S0_RX_RSR);
-		if (rx_fifo_len == 0)
-			break;
-
-		rx_len = w5300_read(priv, W5300_S0_RX_FIFO);
-
-		skb = netdev_alloc_skb_ip_align(ndev, roundup(rx_len, 2));
-		if (unlikely(!skb)) {
-			u32 i;
-			for (i = 0; i < rx_fifo_len; i += 2)
-				w5300_read(priv, W5300_S0_RX_FIFO);
-			ndev->stats.rx_dropped++;
-			return -ENOMEM;
-		}
-
-		skb_put(skb, rx_len);
-		w5300_read_frame(priv, skb->data, rx_len);
-		skb->protocol = eth_type_trans(skb, ndev);
-
-		netif_receive_skb(skb);
-		ndev->stats.rx_packets++;
-		ndev->stats.rx_bytes += rx_len;
-	}
-
-	if (rx_count < budget) {
-		napi_complete_done(napi, rx_count);
-		w5300_write(priv, W5300_IMR, IR_S0);
-	}
-
-	return rx_count;
-}
-
-static irqreturn_t w5300_interrupt(int irq, void *ndev_instance)
-{
-	struct net_device *ndev = ndev_instance;
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	int ir = w5300_read(priv, W5300_S0_IR);
-	if (!ir)
-		return IRQ_NONE;
-	w5300_write(priv, W5300_S0_IR, ir);
-
-	if (ir & S0_IR_SENDOK) {
-		netif_dbg(priv, tx_done, ndev, "tx done\n");
-		netif_wake_queue(ndev);
-	}
-
-	if (ir & S0_IR_RECV) {
-		if (napi_schedule_prep(&priv->napi)) {
-			w5300_write(priv, W5300_IMR, 0);
-			__napi_schedule(&priv->napi);
-		}
-	}
-
-	return IRQ_HANDLED;
-}
-
-static irqreturn_t w5300_detect_link(int irq, void *ndev_instance)
-{
-	struct net_device *ndev = ndev_instance;
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	if (netif_running(ndev)) {
-		if (gpio_get_value(priv->link_gpio) != 0) {
-			netif_info(priv, link, ndev, "link is up\n");
-			netif_carrier_on(ndev);
-		} else {
-			netif_info(priv, link, ndev, "link is down\n");
-			netif_carrier_off(ndev);
-		}
-	}
-
-	return IRQ_HANDLED;
-}
-
-static void w5300_set_rx_mode(struct net_device *ndev)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-	bool set_promisc = (ndev->flags & IFF_PROMISC) != 0;
-
-	if (priv->promisc != set_promisc) {
-		priv->promisc = set_promisc;
-		w5300_hw_start(priv);
-	}
-}
-
-static int w5300_set_macaddr(struct net_device *ndev, void *addr)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-	struct sockaddr *sock_addr = addr;
-
-	if (!is_valid_ether_addr(sock_addr->sa_data))
-		return -EADDRNOTAVAIL;
-	eth_hw_addr_set(ndev, sock_addr->sa_data);
-	w5300_write_macaddr(priv);
-	return 0;
-}
-
-static int w5300_open(struct net_device *ndev)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	netif_info(priv, ifup, ndev, "enabling\n");
-	w5300_hw_start(priv);
-	napi_enable(&priv->napi);
-	netif_start_queue(ndev);
-	if (!gpio_is_valid(priv->link_gpio) ||
-	    gpio_get_value(priv->link_gpio) != 0)
-		netif_carrier_on(ndev);
-	return 0;
-}
-
-static int w5300_stop(struct net_device *ndev)
-{
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	netif_info(priv, ifdown, ndev, "shutting down\n");
-	w5300_hw_close(priv);
-	netif_carrier_off(ndev);
-	netif_stop_queue(ndev);
-	napi_disable(&priv->napi);
-	return 0;
-}
-
-static const struct ethtool_ops w5300_ethtool_ops = {
-	.get_drvinfo		= w5300_get_drvinfo,
-	.get_msglevel		= w5300_get_msglevel,
-	.set_msglevel		= w5300_set_msglevel,
-	.get_link		= w5300_get_link,
-	.get_regs_len		= w5300_get_regs_len,
-	.get_regs		= w5300_get_regs,
-};
-
-static const struct net_device_ops w5300_netdev_ops = {
-	.ndo_open		= w5300_open,
-	.ndo_stop		= w5300_stop,
-	.ndo_start_xmit		= w5300_start_tx,
-	.ndo_tx_timeout		= w5300_tx_timeout,
-	.ndo_set_rx_mode	= w5300_set_rx_mode,
-	.ndo_set_mac_address	= w5300_set_macaddr,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-static int w5300_hw_probe(struct platform_device *pdev)
-{
-	struct wiznet_platform_data *data = dev_get_platdata(&pdev->dev);
-	struct net_device *ndev = platform_get_drvdata(pdev);
-	struct w5300_priv *priv = netdev_priv(ndev);
-	const char *name = netdev_name(ndev);
-	struct resource *mem;
-	int mem_size;
-	int irq;
-	int ret;
-
-	if (data && is_valid_ether_addr(data->mac_addr)) {
-		eth_hw_addr_set(ndev, data->mac_addr);
-	} else {
-		eth_hw_addr_random(ndev);
-	}
-
-	priv->base = devm_platform_get_and_ioremap_resource(pdev, 0, &mem);
-	if (IS_ERR(priv->base))
-		return PTR_ERR(priv->base);
-
-	mem_size = resource_size(mem);
-
-	spin_lock_init(&priv->reg_lock);
-	priv->indirect = mem_size < W5300_BUS_DIRECT_SIZE;
-	if (priv->indirect) {
-		priv->read  = w5300_read_indirect;
-		priv->write = w5300_write_indirect;
-	} else {
-		priv->read  = w5300_read_direct;
-		priv->write = w5300_write_direct;
-	}
-
-	w5300_hw_reset(priv);
-	if (w5300_read(priv, W5300_IDR) != IDR_W5300)
-		return -ENODEV;
-
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-	ret = request_irq(irq, w5300_interrupt,
-			  IRQ_TYPE_LEVEL_LOW, name, ndev);
-	if (ret < 0)
-		return ret;
-	priv->irq = irq;
-
-	priv->link_gpio = data ? data->link_gpio : -EINVAL;
-	if (gpio_is_valid(priv->link_gpio)) {
-		char *link_name = devm_kzalloc(&pdev->dev, 16, GFP_KERNEL);
-		if (!link_name)
-			return -ENOMEM;
-		snprintf(link_name, 16, "%s-link", name);
-		priv->link_irq = gpio_to_irq(priv->link_gpio);
-		if (request_any_context_irq(priv->link_irq, w5300_detect_link,
-				IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING,
-				link_name, priv->ndev) < 0)
-			priv->link_gpio = -EINVAL;
-	}
-
-	netdev_info(ndev, "at 0x%llx irq %d\n", (u64)mem->start, irq);
-	return 0;
-}
-
-static int w5300_probe(struct platform_device *pdev)
-{
-	struct w5300_priv *priv;
-	struct net_device *ndev;
-	int err;
-
-	ndev = alloc_etherdev(sizeof(*priv));
-	if (!ndev)
-		return -ENOMEM;
-	SET_NETDEV_DEV(ndev, &pdev->dev);
-	platform_set_drvdata(pdev, ndev);
-	priv = netdev_priv(ndev);
-	priv->ndev = ndev;
-
-	ndev->netdev_ops = &w5300_netdev_ops;
-	ndev->ethtool_ops = &w5300_ethtool_ops;
-	ndev->watchdog_timeo = HZ;
-	netif_napi_add_weight(ndev, &priv->napi, w5300_napi_poll, 16);
-
-	/* This chip doesn't support VLAN packets with normal MTU,
-	 * so disable VLAN for this device.
-	 */
-	ndev->features |= NETIF_F_VLAN_CHALLENGED;
-
-	err = register_netdev(ndev);
-	if (err < 0)
-		goto err_register;
-
-	err = w5300_hw_probe(pdev);
-	if (err < 0)
-		goto err_hw_probe;
-
-	return 0;
-
-err_hw_probe:
-	unregister_netdev(ndev);
-err_register:
-	free_netdev(ndev);
-	return err;
-}
-
-static void w5300_remove(struct platform_device *pdev)
-{
-	struct net_device *ndev = platform_get_drvdata(pdev);
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	w5300_hw_reset(priv);
-	free_irq(priv->irq, ndev);
-	if (gpio_is_valid(priv->link_gpio))
-		free_irq(priv->link_irq, ndev);
-
-	unregister_netdev(ndev);
-	free_netdev(ndev);
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int w5300_suspend(struct device *dev)
-{
-	struct net_device *ndev = dev_get_drvdata(dev);
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	if (netif_running(ndev)) {
-		netif_carrier_off(ndev);
-		netif_device_detach(ndev);
-
-		w5300_hw_close(priv);
-	}
-	return 0;
-}
-
-static int w5300_resume(struct device *dev)
-{
-	struct net_device *ndev = dev_get_drvdata(dev);
-	struct w5300_priv *priv = netdev_priv(ndev);
-
-	if (!netif_running(ndev)) {
-		w5300_hw_reset(priv);
-		w5300_hw_start(priv);
-
-		netif_device_attach(ndev);
-		if (!gpio_is_valid(priv->link_gpio) ||
-		    gpio_get_value(priv->link_gpio) != 0)
-			netif_carrier_on(ndev);
-	}
-	return 0;
-}
-#endif /* CONFIG_PM_SLEEP */
-
-static SIMPLE_DEV_PM_OPS(w5300_pm_ops, w5300_suspend, w5300_resume);
-
-static struct platform_driver w5300_driver = {
-	.driver		= {
-		.name	= DRV_NAME,
-		.pm	= &w5300_pm_ops,
-	},
-	.probe		= w5300_probe,
-	.remove		= w5300_remove,
-};
-
-module_platform_driver(w5300_driver);
diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c
index c6563367d3826..e8ff03ed87dc5 100644
--- a/drivers/net/geneve.c
+++ b/drivers/net/geneve.c
@@ -106,7 +106,7 @@ struct geneve_sock {
 	bool			collect_md;
 	bool			gro_hint;
 	struct list_head	list;
-	struct socket		*sock;
+	struct sock		*sk;
 	struct rcu_head		rcu;
 	int			refcnt;
 	struct hlist_head	vni_list[VNI_HASH_SIZE];
@@ -167,7 +167,7 @@ static bool eq_tun_id_and_vni(u8 *tun_id, u8 *vni)
 
 static sa_family_t geneve_get_sk_family(struct geneve_sock *gs)
 {
-	return gs->sock->sk->sk_family;
+	return gs->sk->sk_family;
 }
 
 static struct geneve_dev *geneve_lookup(struct geneve_sock *gs,
@@ -760,11 +760,11 @@ static int geneve_udp_encap_err_lookup(struct sock *sk, struct sk_buff *skb)
 	return -EPFNOSUPPORT;
 }
 
-static struct socket *geneve_create_sock(struct net *net, bool ipv6,
-					 __be16 port, bool ipv6_rx_csum)
+static struct sock *geneve_create_sock(struct net *net, bool ipv6,
+				       __be16 port, bool ipv6_rx_csum)
 {
-	struct socket *sock;
 	struct udp_port_cfg udp_conf;
+	struct socket *sock;
 	int err;
 
 	memset(&udp_conf, 0, sizeof(udp_conf));
@@ -786,7 +786,7 @@ static struct socket *geneve_create_sock(struct net *net, bool ipv6,
 		return ERR_PTR(err);
 
 	udp_allow_gso(sock->sk);
-	return sock;
+	return sock->sk;
 }
 
 static bool geneve_hdr_match(struct sk_buff *skb,
@@ -974,28 +974,28 @@ static struct geneve_sock *geneve_socket_create(struct net *net, __be16 port,
 						bool ipv6, bool ipv6_rx_csum)
 {
 	struct geneve_net *gn = net_generic(net, geneve_net_id);
-	struct geneve_sock *gs;
-	struct socket *sock;
 	struct udp_tunnel_sock_cfg tunnel_cfg;
+	struct geneve_sock *gs;
+	struct sock *sk;
 	int h;
 
 	gs = kzalloc_obj(*gs);
 	if (!gs)
 		return ERR_PTR(-ENOMEM);
 
-	sock = geneve_create_sock(net, ipv6, port, ipv6_rx_csum);
-	if (IS_ERR(sock)) {
+	sk = geneve_create_sock(net, ipv6, port, ipv6_rx_csum);
+	if (IS_ERR(sk)) {
 		kfree(gs);
-		return ERR_CAST(sock);
+		return ERR_CAST(sk);
 	}
 
-	gs->sock = sock;
+	gs->sk = sk;
 	gs->refcnt = 1;
 	for (h = 0; h < VNI_HASH_SIZE; ++h)
 		INIT_HLIST_HEAD(&gs->vni_list[h]);
 
 	/* Initialize the geneve udp offloads structure */
-	udp_tunnel_notify_add_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);
+	udp_tunnel_notify_add_rx_port(sk, UDP_TUNNEL_TYPE_GENEVE);
 
 	/* Mark socket as an encapsulation socket */
 	memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
@@ -1006,7 +1006,7 @@ static struct geneve_sock *geneve_socket_create(struct net *net, __be16 port,
 	tunnel_cfg.encap_rcv = geneve_udp_encap_recv;
 	tunnel_cfg.encap_err_lookup = geneve_udp_encap_err_lookup;
 	tunnel_cfg.encap_destroy = NULL;
-	setup_udp_tunnel_sock(net, sock, &tunnel_cfg);
+	setup_udp_tunnel_sock(net, sk, &tunnel_cfg);
 	list_add(&gs->list, &gn->sock_list);
 	return gs;
 }
@@ -1017,8 +1017,8 @@ static void __geneve_sock_release(struct geneve_sock *gs)
 		return;
 
 	list_del(&gs->list);
-	udp_tunnel_notify_del_rx_port(gs->sock, UDP_TUNNEL_TYPE_GENEVE);
-	udp_tunnel_sock_release(gs->sock);
+	udp_tunnel_notify_del_rx_port(gs->sk, UDP_TUNNEL_TYPE_GENEVE);
+	udp_tunnel_sock_release(gs->sk);
 	kfree_rcu(gs, rcu);
 }
 
@@ -1032,7 +1032,6 @@ static void geneve_sock_release(struct geneve_dev *geneve)
 #endif
 
 	rcu_assign_pointer(geneve->sock4, NULL);
-	synchronize_net();
 
 	__geneve_sock_release(gs4);
 #if IS_ENABLED(CONFIG_IPV6)
@@ -1048,7 +1047,7 @@ static struct geneve_sock *geneve_find_sock(struct geneve_net *gn,
 	struct geneve_sock *gs;
 
 	list_for_each_entry(gs, &gn->sock_list, list) {
-		if (inet_sk(gs->sock->sk)->inet_sport == dst_port &&
+		if (inet_sk(gs->sk)->inet_sport == dst_port &&
 		    geneve_get_sk_family(gs) == family &&
 		    gs->gro_hint == gro_hint) {
 			return gs;
@@ -1390,7 +1389,7 @@ static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 	if (unlikely(err))
 		return err;
 
-	udp_tunnel_xmit_skb(rt, gs4->sock->sk, skb, saddr, info->key.u.ipv4.dst,
+	udp_tunnel_xmit_skb(rt, gs4->sk, skb, saddr, info->key.u.ipv4.dst,
 			    tos, ttl, df, sport, geneve->cfg.info.key.tp_dst,
 			    !net_eq(geneve->net, dev_net(geneve->dev)),
 			    !test_bit(IP_TUNNEL_CSUM_BIT, info->key.tun_flags),
@@ -1424,7 +1423,7 @@ static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 				  geneve->cfg.port_min,
 				  geneve->cfg.port_max, true);
 
-	dst = udp_tunnel6_dst_lookup(skb, dev, geneve->net, gs6->sock, 0,
+	dst = udp_tunnel6_dst_lookup(skb, dev, geneve->net, gs6->sk, 0,
 				     &saddr, key, sport,
 				     geneve->cfg.info.key.tp_dst, prio,
 				     use_cache ?
@@ -1480,7 +1479,7 @@ static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
 	if (unlikely(err))
 		return err;
 
-	udp_tunnel6_xmit_skb(dst, gs6->sock->sk, skb, dev,
+	udp_tunnel6_xmit_skb(dst, gs6->sk, skb, dev,
 			     &saddr, &key->u.ipv6.dst, prio, ttl,
 			     info->key.label, sport, geneve->cfg.info.key.tp_dst,
 			     !test_bit(IP_TUNNEL_CSUM_BIT,
@@ -1592,7 +1591,7 @@ static int geneve_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 					  geneve->cfg.port_min,
 					  geneve->cfg.port_max, true);
 
-		dst = udp_tunnel6_dst_lookup(skb, dev, geneve->net, gs6->sock, 0,
+		dst = udp_tunnel6_dst_lookup(skb, dev, geneve->net, gs6->sk, 0,
 					     &saddr, &info->key, sport,
 					     geneve->cfg.info.key.tp_dst, prio,
 					     use_cache ? &info->dst_cache : NULL);
@@ -1654,10 +1653,10 @@ static void geneve_offload_rx_ports(struct net_device *dev, bool push)
 
 	list_for_each_entry(gs, &gn->sock_list, list) {
 		if (push) {
-			udp_tunnel_push_rx_port(dev, gs->sock,
+			udp_tunnel_push_rx_port(dev, gs->sk,
 						UDP_TUNNEL_TYPE_GENEVE);
 		} else {
-			udp_tunnel_drop_rx_port(dev, gs->sock,
+			udp_tunnel_drop_rx_port(dev, gs->sk,
 						UDP_TUNNEL_TYPE_GENEVE);
 		}
 	}
@@ -2183,12 +2182,12 @@ static void geneve_quiesce(struct geneve_dev *geneve, struct geneve_sock **gs4,
 	*gs4 = rtnl_dereference(geneve->sock4);
 	rcu_assign_pointer(geneve->sock4, NULL);
 	if (*gs4)
-		rcu_assign_sk_user_data((*gs4)->sock->sk, NULL);
+		rcu_assign_sk_user_data((*gs4)->sk, NULL);
 #if IS_ENABLED(CONFIG_IPV6)
 	*gs6 = rtnl_dereference(geneve->sock6);
 	rcu_assign_pointer(geneve->sock6, NULL);
 	if (*gs6)
-		rcu_assign_sk_user_data((*gs6)->sock->sk, NULL);
+		rcu_assign_sk_user_data((*gs6)->sk, NULL);
 #else
 	*gs6 = NULL;
 #endif
@@ -2201,13 +2200,12 @@ static void geneve_unquiesce(struct geneve_dev *geneve, struct geneve_sock *gs4,
 {
 	rcu_assign_pointer(geneve->sock4, gs4);
 	if (gs4)
-		rcu_assign_sk_user_data(gs4->sock->sk, gs4);
+		rcu_assign_sk_user_data(gs4->sk, gs4);
 #if IS_ENABLED(CONFIG_IPV6)
 	rcu_assign_pointer(geneve->sock6, gs6);
 	if (gs6)
-		rcu_assign_sk_user_data(gs6->sock->sk, gs6);
+		rcu_assign_sk_user_data(gs6->sk, gs6);
 #endif
-	synchronize_net();
 }
 
 static int geneve_changelink(struct net_device *dev, struct nlattr *tb[],
diff --git a/drivers/net/gtp.c b/drivers/net/gtp.c
index 5150f2e4f66be..a60ef32b35b82 100644
--- a/drivers/net/gtp.c
+++ b/drivers/net/gtp.c
@@ -885,8 +885,8 @@ static void gtp_encap_disable_sock(struct sock *sk)
 static void gtp_encap_disable(struct gtp_dev *gtp)
 {
 	if (gtp->sk_created) {
-		udp_tunnel_sock_release(gtp->sk0->sk_socket);
-		udp_tunnel_sock_release(gtp->sk1u->sk_socket);
+		udp_tunnel_sock_release(gtp->sk0);
+		udp_tunnel_sock_release(gtp->sk1u);
 		gtp->sk_created = false;
 		gtp->sk0 = NULL;
 		gtp->sk1u = NULL;
@@ -1434,7 +1434,7 @@ static struct sock *gtp_create_sock(int type, struct gtp_dev *gtp,
 	tuncfg.encap_rcv = gtp_encap_recv;
 	tuncfg.encap_destroy = NULL;
 
-	setup_udp_tunnel_sock(net, sock, &tuncfg);
+	setup_udp_tunnel_sock(net, sock->sk, &tuncfg);
 
 	return sock->sk;
 }
@@ -1451,7 +1451,7 @@ static int gtp_create_sockets(struct gtp_dev *gtp, const struct nlattr *nla,
 
 	sk1u = gtp_create_sock(UDP_ENCAP_GTP1U, gtp, nla, family);
 	if (IS_ERR(sk1u)) {
-		udp_tunnel_sock_release(sk0->sk_socket);
+		udp_tunnel_sock_release(sk0);
 		return PTR_ERR(sk1u);
 	}
 
@@ -1689,7 +1689,7 @@ static struct sock *gtp_encap_enable_socket(int fd, int type,
 	tuncfg.encap_rcv = gtp_encap_recv;
 	tuncfg.encap_destroy = gtp_encap_destroy;
 
-	setup_udp_tunnel_sock(sock_net(sock->sk), sock, &tuncfg);
+	setup_udp_tunnel_sock(sock_net(sock->sk), sk, &tuncfg);
 
 out_rel_sock:
 	release_sock(sock->sk);
diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 5407d2ed71b35..43aa1bfd41cff 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -211,12 +211,12 @@ static void ifb_get_strings(struct net_device *dev, u32 stringset, u8 *buf)
 
 	switch (stringset) {
 	case ETH_SS_STATS:
-		for (i = 0; i < dev->real_num_rx_queues; i++)
+		for (i = 0; i < dev->num_tx_queues; i++)
 			for (j = 0; j < IFB_Q_STATS_LEN; j++)
 				ethtool_sprintf(&p, "rx_queue_%u_%.18s",
 						i, ifb_q_stats_desc[j].desc);
 
-		for (i = 0; i < dev->real_num_tx_queues; i++)
+		for (i = 0; i < dev->num_tx_queues; i++)
 			for (j = 0; j < IFB_Q_STATS_LEN; j++)
 				ethtool_sprintf(&p, "tx_queue_%u_%.18s",
 						i, ifb_q_stats_desc[j].desc);
@@ -229,8 +229,7 @@ static int ifb_get_sset_count(struct net_device *dev, int sset)
 {
 	switch (sset) {
 	case ETH_SS_STATS:
-		return IFB_Q_STATS_LEN * (dev->real_num_rx_queues +
-					  dev->real_num_tx_queues);
+		return IFB_Q_STATS_LEN * dev->num_tx_queues * 2;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -262,12 +261,12 @@ static void ifb_get_ethtool_stats(struct net_device *dev,
 	struct ifb_q_private *txp;
 	int i;
 
-	for (i = 0; i < dev->real_num_rx_queues; i++) {
+	for (i = 0; i < dev->num_tx_queues; i++) {
 		txp = dp->tx_private + i;
 		ifb_fill_stats_data(&data, &txp->rx_stats);
 	}
 
-	for (i = 0; i < dev->real_num_tx_queues; i++) {
+	for (i = 0; i < dev->num_tx_queues; i++) {
 		txp = dp->tx_private + i;
 		ifb_fill_stats_data(&data, &txp->tx_stats);
 	}
diff --git a/drivers/net/ipvlan/ipvlan_core.c b/drivers/net/ipvlan/ipvlan_core.c
index 1be8620ad3971..7ad12dc7845c8 100644
--- a/drivers/net/ipvlan/ipvlan_core.c
+++ b/drivers/net/ipvlan/ipvlan_core.c
@@ -286,7 +286,7 @@ void ipvlan_process_multicast(struct work_struct *work)
 				if (tx_pkt)
 					ret = dev_forward_skb(ipvlan->dev, nskb);
 				else
-					ret = netif_rx(nskb);
+					ret = netif_receive_skb(nskb);
 			}
 			ipvlan_count_rx(ipvlan, len, ret == NET_RX_SUCCESS, true);
 			local_bh_enable();
diff --git a/drivers/net/mctp/mctp-i2c.c b/drivers/net/mctp/mctp-i2c.c
index ee2913758e54e..b02476d338bbf 100644
--- a/drivers/net/mctp/mctp-i2c.c
+++ b/drivers/net/mctp/mctp-i2c.c
@@ -1101,8 +1101,8 @@ static struct notifier_block mctp_i2c_notifier = {
 };
 
 static const struct i2c_device_id mctp_i2c_id[] = {
-	{ "mctp-i2c-interface" },
-	{}
+	{ .name = "mctp-i2c-interface" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, mctp_i2c_id);
 
diff --git a/drivers/net/mdio/Kconfig b/drivers/net/mdio/Kconfig
index c71132f33f847..c591eec8e97a5 100644
--- a/drivers/net/mdio/Kconfig
+++ b/drivers/net/mdio/Kconfig
@@ -254,7 +254,7 @@ config MDIO_BUS_MUX_BCM_IPROC
 
 config MDIO_BUS_MUX_GPIO
 	tristate "GPIO controlled MDIO bus multiplexers"
-	depends on OF_GPIO && OF_MDIO
+	depends on OF_MDIO
 	select MDIO_BUS_MUX
 	help
 	  This module provides a driver for MDIO bus multiplexers that
diff --git a/drivers/net/mdio/mdio-mscc-miim.c b/drivers/net/mdio/mdio-mscc-miim.c
index 944efd33da6dc..259944d37fbdf 100644
--- a/drivers/net/mdio/mdio-mscc-miim.c
+++ b/drivers/net/mdio/mdio-mscc-miim.c
@@ -330,7 +330,7 @@ static int mscc_miim_probe(struct platform_device *pdev)
 
 	ret = of_mdiobus_register(bus, np);
 	if (ret < 0) {
-		dev_err(dev, "Cannot register MDIO bus (%d)\n", ret);
+		dev_err_probe(dev, ret, "Cannot register MDIO bus\n");
 		goto out_disable_clk;
 	}
 
diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c
index 57dd6821a8aa9..d804d44af87c8 100644
--- a/drivers/net/netconsole.c
+++ b/drivers/net/netconsole.c
@@ -32,8 +32,13 @@
 #include <linux/moduleparam.h>
 #include <linux/kernel.h>
 #include <linux/string.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/udp.h>
 #include <linux/netpoll.h>
 #include <linux/inet.h>
+#include <linux/unaligned.h>
+#include <net/ip6_checksum.h>
 #include <linux/configfs.h>
 #include <linux/etherdevice.h>
 #include <linux/hex.h>
@@ -45,6 +50,7 @@
 MODULE_AUTHOR("Matt Mackall <mpm@selenic.com>");
 MODULE_DESCRIPTION("Console driver for network interfaces");
 MODULE_LICENSE("GPL");
+MODULE_IMPORT_NS("NETDEV_INTERNAL");
 
 #define MAX_PARAM_LENGTH		256
 #define MAX_EXTRADATA_ENTRY_LEN		256
@@ -1648,6 +1654,175 @@ static struct notifier_block netconsole_netdev_notifier = {
 	.notifier_call  = netconsole_netdev_event,
 };
 
+static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve)
+{
+	int count = 0;
+	struct sk_buff *skb;
+
+	netpoll_zap_completion_queue();
+repeat:
+
+	skb = alloc_skb(len, GFP_ATOMIC);
+	if (!skb) {
+		skb = skb_dequeue(&np->skb_pool);
+		schedule_work(&np->refill_wq);
+	}
+
+	if (!skb) {
+		if (++count < 10) {
+			netpoll_poll_dev(np->dev);
+			goto repeat;
+		}
+		return NULL;
+	}
+
+	refcount_set(&skb->users, 1);
+	skb_reserve(skb, reserve);
+	return skb;
+}
+
+static void netpoll_udp_checksum(struct netpoll *np, struct sk_buff *skb,
+				 int len)
+{
+	struct udphdr *udph;
+	int udp_len;
+
+	udp_len = len + sizeof(struct udphdr);
+	udph = udp_hdr(skb);
+
+	/* check needs to be set, since it will be consumed in csum_partial */
+	udph->check = 0;
+	if (np->ipv6)
+		udph->check = csum_ipv6_magic(&np->local_ip.in6,
+					      &np->remote_ip.in6,
+					      udp_len, IPPROTO_UDP,
+					      csum_partial(udph, udp_len, 0));
+	else
+		udph->check = csum_tcpudp_magic(np->local_ip.ip,
+						np->remote_ip.ip,
+						udp_len, IPPROTO_UDP,
+						csum_partial(udph, udp_len, 0));
+	if (udph->check == 0)
+		udph->check = CSUM_MANGLED_0;
+}
+
+static void push_udp(struct netpoll *np, struct sk_buff *skb, int len)
+{
+	struct udphdr *udph;
+	int udp_len;
+
+	udp_len = len + sizeof(struct udphdr);
+
+	skb_push(skb, sizeof(struct udphdr));
+	skb_reset_transport_header(skb);
+
+	udph = udp_hdr(skb);
+	udph->source = htons(np->local_port);
+	udph->dest = htons(np->remote_port);
+	udph->len = htons(udp_len);
+
+	netpoll_udp_checksum(np, skb, len);
+}
+
+static void push_eth(struct netpoll *np, struct sk_buff *skb)
+{
+	struct ethhdr *eth;
+
+	eth = skb_push(skb, ETH_HLEN);
+	skb_reset_mac_header(skb);
+	ether_addr_copy(eth->h_source, np->dev->dev_addr);
+	ether_addr_copy(eth->h_dest, np->remote_mac);
+	if (np->ipv6)
+		eth->h_proto = htons(ETH_P_IPV6);
+	else
+		eth->h_proto = htons(ETH_P_IP);
+}
+
+static void push_ipv4(struct netpoll *np, struct sk_buff *skb, int len)
+{
+	static atomic_t ip_ident;
+	struct iphdr *iph;
+	int ip_len;
+
+	ip_len = len + sizeof(struct udphdr) + sizeof(struct iphdr);
+
+	skb_push(skb, sizeof(struct iphdr));
+	skb_reset_network_header(skb);
+	iph = ip_hdr(skb);
+
+	/* iph->version = 4; iph->ihl = 5; */
+	*(unsigned char *)iph = 0x45;
+	iph->tos = 0;
+	put_unaligned(htons(ip_len), &iph->tot_len);
+	iph->id = htons(atomic_inc_return(&ip_ident));
+	iph->frag_off = 0;
+	iph->ttl = 64;
+	iph->protocol = IPPROTO_UDP;
+	iph->check = 0;
+	put_unaligned(np->local_ip.ip, &iph->saddr);
+	put_unaligned(np->remote_ip.ip, &iph->daddr);
+	iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
+	skb->protocol = htons(ETH_P_IP);
+}
+
+static void push_ipv6(struct netpoll *np, struct sk_buff *skb, int len)
+{
+	struct ipv6hdr *ip6h;
+
+	skb_push(skb, sizeof(struct ipv6hdr));
+	skb_reset_network_header(skb);
+	ip6h = ipv6_hdr(skb);
+
+	/* ip6h->version = 6; ip6h->priority = 0; */
+	*(unsigned char *)ip6h = 0x60;
+	ip6h->flow_lbl[0] = 0;
+	ip6h->flow_lbl[1] = 0;
+	ip6h->flow_lbl[2] = 0;
+
+	ip6h->payload_len = htons(sizeof(struct udphdr) + len);
+	ip6h->nexthdr = IPPROTO_UDP;
+	ip6h->hop_limit = 32;
+	ip6h->saddr = np->local_ip.in6;
+	ip6h->daddr = np->remote_ip.in6;
+
+	skb->protocol = htons(ETH_P_IPV6);
+}
+
+static int netpoll_send_udp(struct netpoll *np, const char *msg, int len)
+{
+	int total_len, ip_len, udp_len;
+	struct sk_buff *skb;
+
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+		WARN_ON_ONCE(!irqs_disabled());
+
+	udp_len = len + sizeof(struct udphdr);
+	if (np->ipv6)
+		ip_len = udp_len + sizeof(struct ipv6hdr);
+	else
+		ip_len = udp_len + sizeof(struct iphdr);
+
+	total_len = ip_len + LL_RESERVED_SPACE(np->dev);
+
+	skb = find_skb(np, total_len + np->dev->needed_tailroom,
+		       total_len - len);
+	if (!skb)
+		return -ENOMEM;
+
+	skb_copy_to_linear_data(skb, msg, len);
+	skb_put(skb, len);
+
+	push_udp(np, skb, len);
+	if (np->ipv6)
+		push_ipv6(np, skb, len);
+	else
+		push_ipv4(np, skb, len);
+	push_eth(np, skb);
+	skb->dev = np->dev;
+
+	return (int)netpoll_send_skb(np, skb);
+}
+
 /**
  * send_udp - Wrapper for netpoll_send_udp that counts errors
  * @nt: target to send message to
diff --git a/drivers/net/netdevsim/psp.c b/drivers/net/netdevsim/psp.c
index 6936ecb8173e2..c6000ebc6bc8f 100644
--- a/drivers/net/netdevsim/psp.c
+++ b/drivers/net/netdevsim/psp.c
@@ -132,14 +132,15 @@ nsim_rx_spi_alloc(struct psp_dev *psd, u32 version,
 		  struct netlink_ext_ack *extack)
 {
 	struct netdevsim *ns = psd->drv_priv;
-	unsigned int new;
 	int i;
 
-	new = ++ns->psp.spi & PSP_SPI_KEY_ID;
-	if (psd->generation & 1)
-		new |= PSP_SPI_KEY_PHASE;
+	/* Check if incrementing the spi would change the phase bit */
+	if ((ns->psp.spi & PSP_SPI_KEY_ID) == PSP_SPI_KEY_ID) {
+		NL_SET_ERR_MSG(extack, "SPI space exhausted");
+		return -ENOSPC;
+	}
 
-	assoc->spi = cpu_to_be32(new);
+	assoc->spi = cpu_to_be32(++ns->psp.spi);
 	assoc->key[0] = psd->generation;
 	for (i = 1; i < PSP_MAX_KEY; i++)
 		assoc->key[i] = ns->psp.spi + i;
@@ -162,6 +163,16 @@ static int nsim_assoc_add(struct psp_dev *psd, struct psp_assoc *pas,
 
 static int nsim_key_rotate(struct psp_dev *psd, struct netlink_ext_ack *extack)
 {
+	struct netdevsim *ns = psd->drv_priv;
+
+	/* Flip key phase and reset SPI to 0 within that space
+	 * (will be pre-incremented, as 0 is an invalid SPI).
+	 */
+	if (ns->psp.spi & PSP_SPI_KEY_PHASE)
+		ns->psp.spi = 0;
+	else
+		ns->psp.spi = PSP_SPI_KEY_PHASE;
+
 	return 0;
 }
 
diff --git a/drivers/net/netkit.c b/drivers/net/netkit.c
index 5e2eecc3165dc..0ad6a806d7d56 100644
--- a/drivers/net/netkit.c
+++ b/drivers/net/netkit.c
@@ -466,6 +466,7 @@ static void netkit_setup(struct net_device *dev)
 	dev->priv_flags |= IFF_NO_QUEUE;
 	dev->priv_flags |= IFF_DISABLE_NETPOLL;
 	dev->lltx = true;
+	dev->netmem_tx = NETMEM_TX_NO_DMA;
 
 	dev->netdev_ops     = &netkit_netdev_ops;
 	dev->ethtool_ops    = &netkit_ethtool_ops;
diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c
index 22c555dd962ec..a6b777a9c2d9a 100644
--- a/drivers/net/ovpn/io.c
+++ b/drivers/net/ovpn/io.c
@@ -201,7 +201,7 @@ void ovpn_decrypt_post(void *data, int ret)
 	skb = NULL;
 drop:
 	if (unlikely(skb))
-		dev_dstats_rx_dropped(peer->ovpn->dev);
+		ovpn_dev_dstats_rx_dropped(peer->ovpn->dev);
 	kfree_skb(skb);
 drop_nocount:
 	if (likely(peer))
@@ -225,7 +225,7 @@ void ovpn_recv(struct ovpn_peer *peer, struct sk_buff *skb)
 		net_info_ratelimited("%s: no available key for peer %u, key-id: %u\n",
 				     netdev_name(peer->ovpn->dev), peer->id,
 				     key_id);
-		dev_dstats_rx_dropped(peer->ovpn->dev);
+		ovpn_dev_dstats_rx_dropped(peer->ovpn->dev);
 		kfree_skb(skb);
 		ovpn_peer_put(peer);
 		return;
@@ -301,7 +301,7 @@ void ovpn_encrypt_post(void *data, int ret)
 	rcu_read_unlock();
 err:
 	if (unlikely(skb))
-		dev_dstats_tx_dropped(peer->ovpn->dev);
+		ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
 	if (likely(peer))
 		ovpn_peer_put(peer);
 	if (likely(ks))
@@ -343,7 +343,7 @@ static void ovpn_send(struct ovpn_priv *ovpn, struct sk_buff *skb,
 	 */
 	skb_list_walk_safe(skb, curr, next) {
 		if (unlikely(!ovpn_encrypt_one(peer, curr))) {
-			dev_dstats_tx_dropped(ovpn->dev);
+			ovpn_dev_dstats_tx_dropped(ovpn->dev);
 			kfree_skb(curr);
 		}
 	}
@@ -414,7 +414,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
 		if (unlikely(!curr)) {
 			net_err_ratelimited("%s: skb_share_check failed for payload packet\n",
 					    netdev_name(dev));
-			dev_dstats_tx_dropped(ovpn->dev);
+			ovpn_dev_dstats_tx_dropped(ovpn->dev);
 			continue;
 		}
 
@@ -440,7 +440,7 @@ netdev_tx_t ovpn_net_xmit(struct sk_buff *skb, struct net_device *dev)
 drop:
 	ovpn_peer_put(peer);
 drop_no_peer:
-	dev_dstats_tx_dropped(ovpn->dev);
+	ovpn_dev_dstats_tx_dropped(ovpn->dev);
 	skb_tx_error(skb);
 	kfree_skb_list(skb);
 	return NETDEV_TX_OK;
diff --git a/drivers/net/ovpn/main.c b/drivers/net/ovpn/main.c
index 2e0420febda05..9993c1dfe471d 100644
--- a/drivers/net/ovpn/main.c
+++ b/drivers/net/ovpn/main.c
@@ -92,6 +92,8 @@ static void ovpn_net_uninit(struct net_device *dev)
 {
 	struct ovpn_priv *ovpn = netdev_priv(dev);
 
+	disable_delayed_work_sync(&ovpn->keepalive_work);
+	ovpn_peers_free(ovpn, NULL, OVPN_DEL_PEER_REASON_TEARDOWN);
 	gro_cells_destroy(&ovpn->gro_cells);
 }
 
@@ -208,15 +210,6 @@ static int ovpn_newlink(struct net_device *dev,
 	return register_netdevice(dev);
 }
 
-static void ovpn_dellink(struct net_device *dev, struct list_head *head)
-{
-	struct ovpn_priv *ovpn = netdev_priv(dev);
-
-	cancel_delayed_work_sync(&ovpn->keepalive_work);
-	ovpn_peers_free(ovpn, NULL, OVPN_DEL_PEER_REASON_TEARDOWN);
-	unregister_netdevice_queue(dev, head);
-}
-
 static int ovpn_fill_info(struct sk_buff *skb, const struct net_device *dev)
 {
 	struct ovpn_priv *ovpn = netdev_priv(dev);
@@ -235,7 +228,6 @@ static struct rtnl_link_ops ovpn_link_ops = {
 	.policy = ovpn_policy,
 	.maxtype = IFLA_OVPN_MAX,
 	.newlink = ovpn_newlink,
-	.dellink = ovpn_dellink,
 	.fill_info = ovpn_fill_info,
 };
 
diff --git a/drivers/net/ovpn/netlink.c b/drivers/net/ovpn/netlink.c
index 291e2e5bb450d..4c66c1ec497ec 100644
--- a/drivers/net/ovpn/netlink.c
+++ b/drivers/net/ovpn/netlink.c
@@ -462,10 +462,12 @@ int ovpn_nl_peer_new_doit(struct sk_buff *skb, struct genl_info *info)
 sock_release:
 	ovpn_socket_release(peer);
 peer_release:
-	/* release right away because peer was not yet hashed, thus it is not
-	 * used in any context
+	/* For UDP, the peer is unreachable until added to the hashtables, so
+	 * dropping the initial reference is enough. For TCP, the peer may be
+	 * concurrently reachable via sk_user_data->peer until
+	 * ovpn_socket_release() detaches; rely on the refcount.
 	 */
-	ovpn_peer_release(peer);
+	ovpn_peer_put(peer);
 
 	return ret;
 }
diff --git a/drivers/net/ovpn/peer.c b/drivers/net/ovpn/peer.c
index c02dfab51a6e6..a09d61296425a 100644
--- a/drivers/net/ovpn/peer.c
+++ b/drivers/net/ovpn/peer.c
@@ -354,7 +354,7 @@ static void ovpn_peer_release_rcu(struct rcu_head *head)
  * ovpn_peer_release - release peer private members
  * @peer: the peer to release
  */
-void ovpn_peer_release(struct ovpn_peer *peer)
+static void ovpn_peer_release(struct ovpn_peer *peer)
 {
 	ovpn_crypto_state_release(&peer->crypto);
 	spin_lock_bh(&peer->lock);
@@ -1034,14 +1034,29 @@ static int ovpn_peer_add_p2p(struct ovpn_priv *ovpn, struct ovpn_peer *peer)
  */
 int ovpn_peer_add(struct ovpn_priv *ovpn, struct ovpn_peer *peer)
 {
+	int ret = -ENODEV;
+
+	/* Prevent adding new peers while destroying the ovpn interface.
+	 * Failing to do so would end up holding the device reference
+	 * endlessly hostage of the new peer object with no chance of
+	 * release..
+	 */
+	netdev_lock(ovpn->dev);
+	if (ovpn->dev->reg_state != NETREG_REGISTERED)
+		goto out;
+
 	switch (ovpn->mode) {
 	case OVPN_MODE_MP:
-		return ovpn_peer_add_mp(ovpn, peer);
+		ret = ovpn_peer_add_mp(ovpn, peer);
+		break;
 	case OVPN_MODE_P2P:
-		return ovpn_peer_add_p2p(ovpn, peer);
+		ret = ovpn_peer_add_p2p(ovpn, peer);
+		break;
 	}
+out:
+	netdev_unlock(ovpn->dev);
 
-	return -EOPNOTSUPP;
+	return ret;
 }
 
 /**
diff --git a/drivers/net/ovpn/peer.h b/drivers/net/ovpn/peer.h
index 328401570cba8..86c8cffada6d4 100644
--- a/drivers/net/ovpn/peer.h
+++ b/drivers/net/ovpn/peer.h
@@ -127,7 +127,6 @@ static inline bool ovpn_peer_hold(struct ovpn_peer *peer)
 	return kref_get_unless_zero(&peer->refcount);
 }
 
-void ovpn_peer_release(struct ovpn_peer *peer);
 void ovpn_peer_release_kref(struct kref *kref);
 
 /**
diff --git a/drivers/net/ovpn/stats.h b/drivers/net/ovpn/stats.h
index 53433d8b6c331..3a45b97c00568 100644
--- a/drivers/net/ovpn/stats.h
+++ b/drivers/net/ovpn/stats.h
@@ -11,6 +11,8 @@
 #ifndef _NET_OVPN_OVPNSTATS_H_
 #define _NET_OVPN_OVPNSTATS_H_
 
+#include <linux/netdevice.h>
+
 /* one stat */
 struct ovpn_peer_stat {
 	atomic64_t bytes;
@@ -44,4 +46,18 @@ static inline void ovpn_peer_stats_increment_tx(struct ovpn_peer_stats *stats,
 	ovpn_peer_stats_increment(&stats->tx, n);
 }
 
+static inline void ovpn_dev_dstats_tx_dropped(struct net_device *dev)
+{
+	local_bh_disable();
+	dev_dstats_tx_dropped(dev);
+	local_bh_enable();
+}
+
+static inline void ovpn_dev_dstats_rx_dropped(struct net_device *dev)
+{
+	local_bh_disable();
+	dev_dstats_rx_dropped(dev);
+	local_bh_enable();
+}
+
 #endif /* _NET_OVPN_OVPNSTATS_H_ */
diff --git a/drivers/net/ovpn/tcp.c b/drivers/net/ovpn/tcp.c
index 65054cc84be55..433bd07a4f1be 100644
--- a/drivers/net/ovpn/tcp.c
+++ b/drivers/net/ovpn/tcp.c
@@ -152,7 +152,7 @@ static void ovpn_tcp_rcv(struct strparser *strp, struct sk_buff *skb)
 	if (WARN_ON(!ovpn_peer_hold(peer)))
 		goto err_nopeer;
 	schedule_work(&peer->tcp.defer_del_work);
-	dev_dstats_rx_dropped(peer->ovpn->dev);
+	ovpn_dev_dstats_rx_dropped(peer->ovpn->dev);
 err_nopeer:
 	kfree_skb(skb);
 }
@@ -298,9 +298,9 @@ static void ovpn_tcp_send_sock(struct ovpn_peer *peer, struct sock *sk)
 	} while (peer->tcp.out_msg.len > 0);
 
 	if (!peer->tcp.out_msg.len) {
-		preempt_disable();
+		local_bh_disable();
 		dev_dstats_tx_add(peer->ovpn->dev, skb->len);
-		preempt_enable();
+		local_bh_enable();
 	}
 
 	kfree_skb(peer->tcp.out_msg.skb);
@@ -331,7 +331,7 @@ static void ovpn_tcp_send_sock_skb(struct ovpn_peer *peer, struct sock *sk,
 		ovpn_tcp_send_sock(peer, sk);
 
 	if (peer->tcp.out_msg.skb) {
-		dev_dstats_tx_dropped(peer->ovpn->dev);
+		ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
 		kfree_skb(skb);
 		return;
 	}
@@ -353,7 +353,7 @@ void ovpn_tcp_send_skb(struct ovpn_peer *peer, struct sock *sk,
 	if (sock_owned_by_user(sk)) {
 		if (skb_queue_len(&peer->tcp.out_queue) >=
 		    READ_ONCE(net_hotdata.max_backlog)) {
-			dev_dstats_tx_dropped(peer->ovpn->dev);
+			ovpn_dev_dstats_tx_dropped(peer->ovpn->dev);
 			kfree_skb(skb);
 			goto unlock;
 		}
@@ -581,14 +581,19 @@ static void ovpn_tcp_close(struct sock *sk, long timeout)
 
 	rcu_read_lock();
 	sock = rcu_dereference_sk_user_data(sk);
-	if (!sock || !sock->peer || !ovpn_peer_hold(sock->peer)) {
+	if (!sock) {
 		rcu_read_unlock();
 		return;
 	}
+
 	peer = sock->peer;
+	if (!peer || !ovpn_peer_hold(peer)) {
+		rcu_read_unlock();
+		return;
+	}
 	rcu_read_unlock();
 
-	ovpn_peer_del(sock->peer, OVPN_DEL_PEER_REASON_TRANSPORT_DISCONNECT);
+	ovpn_peer_del(peer, OVPN_DEL_PEER_REASON_TRANSPORT_DISCONNECT);
 	peer->tcp.sk_cb.prot->close(sk, timeout);
 	ovpn_peer_put(peer);
 }
diff --git a/drivers/net/ovpn/udp.c b/drivers/net/ovpn/udp.c
index 059e896b4a2fd..7f69e8890b5b5 100644
--- a/drivers/net/ovpn/udp.c
+++ b/drivers/net/ovpn/udp.c
@@ -125,7 +125,7 @@ static int ovpn_udp_encap_recv(struct sock *sk, struct sk_buff *skb)
 	return 0;
 
 drop:
-	dev_dstats_rx_dropped(ovpn->dev);
+	ovpn_dev_dstats_rx_dropped(ovpn->dev);
 drop_noovpn:
 	kfree_skb(skb);
 	return 0;
@@ -399,7 +399,7 @@ int ovpn_udp_socket_attach(struct ovpn_socket *ovpn_sock, struct socket *sock,
 	if (!old_data) {
 		/* socket is currently unused - we can take it */
 		rcu_read_unlock();
-		setup_udp_tunnel_sock(sock_net(ovpn_sock->sk), sock, &cfg);
+		setup_udp_tunnel_sock(sock_net(ovpn_sock->sk), sock->sk, &cfg);
 		return 0;
 	}
 
diff --git a/drivers/net/pcs/pcs-xpcs-plat.c b/drivers/net/pcs/pcs-xpcs-plat.c
index b8c48f9effbf7..f4b1b8246ce96 100644
--- a/drivers/net/pcs/pcs-xpcs-plat.c
+++ b/drivers/net/pcs/pcs-xpcs-plat.c
@@ -349,7 +349,7 @@ static int xpcs_plat_init_dev(struct dw_xpcs_plat *pxpcs)
 	 * up later. Make sure DD-core is aware of the OF-node being re-used.
 	 */
 	device_set_node(&mdiodev->dev, fwnode_handle_get(dev_fwnode(dev)));
-	mdiodev->dev.of_node_reused = true;
+	dev_set_of_node_reused(&mdiodev->dev);
 
 	/* Pass the data further so the DW XPCS driver core could use it */
 	mdiodev->dev.platform_data = (void *)device_get_match_data(dev);
diff --git a/drivers/net/pfcp.c b/drivers/net/pfcp.c
index 28e6bc4a1f14c..5f1c9d2c0b49a 100644
--- a/drivers/net/pfcp.c
+++ b/drivers/net/pfcp.c
@@ -18,7 +18,7 @@
 struct pfcp_dev {
 	struct list_head	list;
 
-	struct socket		*sock;
+	struct sock		*sk;
 	struct net_device	*dev;
 	struct net		*net;
 
@@ -104,8 +104,8 @@ static int pfcp_encap_recv(struct sock *sk, struct sk_buff *skb)
 
 static void pfcp_del_sock(struct pfcp_dev *pfcp)
 {
-	udp_tunnel_sock_release(pfcp->sock);
-	pfcp->sock = NULL;
+	udp_tunnel_sock_release(pfcp->sk);
+	pfcp->sk = NULL;
 }
 
 static void pfcp_dev_uninit(struct net_device *dev)
@@ -151,7 +151,7 @@ static void pfcp_link_setup(struct net_device *dev)
 	netif_keep_dst(dev);
 }
 
-static struct socket *pfcp_create_sock(struct pfcp_dev *pfcp)
+static struct sock *pfcp_create_sock(struct pfcp_dev *pfcp)
 {
 	struct udp_tunnel_sock_cfg tuncfg = {};
 	struct udp_port_cfg udp_conf = {
@@ -172,16 +172,16 @@ static struct socket *pfcp_create_sock(struct pfcp_dev *pfcp)
 	tuncfg.encap_rcv = pfcp_encap_recv;
 	tuncfg.encap_type = 1;
 
-	setup_udp_tunnel_sock(net, sock, &tuncfg);
+	setup_udp_tunnel_sock(net, sock->sk, &tuncfg);
 
-	return sock;
+	return sock->sk;
 }
 
 static int pfcp_add_sock(struct pfcp_dev *pfcp)
 {
-	pfcp->sock = pfcp_create_sock(pfcp);
+	pfcp->sk = pfcp_create_sock(pfcp);
 
-	return PTR_ERR_OR_ZERO(pfcp->sock);
+	return PTR_ERR_OR_ZERO(pfcp->sk);
 }
 
 static int pfcp_newlink(struct net_device *dev,
@@ -216,6 +216,7 @@ static int pfcp_newlink(struct net_device *dev,
 
 exit_del_pfcp_sock:
 	pfcp_del_sock(pfcp);
+	synchronize_rcu();
 exit_err:
 	pfcp->net = NULL;
 	return err;
diff --git a/drivers/net/phy/Kconfig b/drivers/net/phy/Kconfig
index b5ee338b620d5..8b51bdc2e945e 100644
--- a/drivers/net/phy/Kconfig
+++ b/drivers/net/phy/Kconfig
@@ -238,7 +238,7 @@ config LXT_PHY
 config INTEL_XWAY_PHY
 	tristate "Intel XWAY PHYs"
 	help
-	  Supports the Intel XWAY (former Lantiq) 11G and 22E PHYs.
+	  Supports the Intel XWAY (former Lantiq) 11G and 22F PHYs.
 	  These PHYs are marked as standalone chips under the names
 	  PEF 7061, PEF 7071 and PEF 7072 or integrated into the Intel
 	  SoCs xRX200, xRX300, xRX330, xRX350 and xRX550.
diff --git a/drivers/net/phy/aquantia/aquantia_main.c b/drivers/net/phy/aquantia/aquantia_main.c
index 41f3676c7f1ec..e95d391a6dc23 100644
--- a/drivers/net/phy/aquantia/aquantia_main.c
+++ b/drivers/net/phy/aquantia/aquantia_main.c
@@ -385,15 +385,15 @@ static int aqr105_setup_forced(struct phy_device *phydev)
 			vend |= MDIO_AN_VEND_PROV_1000BASET_HALF;
 		break;
 	case SPEED_2500:
-		adv |= (ADVERTISE_NPAGE | ADVERTISE_RESV);
+		adv |= (ADVERTISE_NPAGE | ADVERTISE_XNP);
 		vend |= MDIO_AN_VEND_PROV_2500BASET_FULL;
 		break;
 	case SPEED_5000:
-		adv |= (ADVERTISE_NPAGE | ADVERTISE_RESV);
+		adv |= (ADVERTISE_NPAGE | ADVERTISE_XNP);
 		vend |= MDIO_AN_VEND_PROV_5000BASET_FULL;
 		break;
 	case SPEED_10000:
-		adv |= (ADVERTISE_NPAGE | ADVERTISE_RESV);
+		adv |= (ADVERTISE_NPAGE | ADVERTISE_XNP);
 		ctrl10 |= MDIO_AN_10GBT_CTRL_ADV10G;
 		break;
 	default:
diff --git a/drivers/net/phy/dp83867.c b/drivers/net/phy/dp83867.c
index 3fb2293f568f0..88255e92b4cdb 100644
--- a/drivers/net/phy/dp83867.c
+++ b/drivers/net/phy/dp83867.c
@@ -103,6 +103,10 @@
 #define DP83867_PHYCR_RX_FIFO_DEPTH_MASK	GENMASK(13, 12)
 #define DP83867_PHYCR_SGMII_EN			BIT(11)
 #define DP83867_PHYCR_FORCE_LINK_GOOD		BIT(10)
+#define DP83867_PHYCR_MDIX_MASK			GENMASK(6, 5)
+#define DP83867_PHYCR_MDIX_MDI			(0x0 << 5)
+#define DP83867_PHYCR_MDIX_MDIX			(0x1 << 5)
+#define DP83867_PHYCR_MDIX_AUTO			(0x3 << 5)
 
 /* RGMIIDCTL bits */
 #define DP83867_RGMII_TX_CLK_DELAY_MAX		0xf
@@ -123,6 +127,10 @@
 #define DP83867_PHYSTS_100			BIT(14)
 #define DP83867_PHYSTS_DUPLEX			BIT(13)
 #define DP83867_PHYSTS_LINK			BIT(10)
+#define DP83867_PHYSTS_MDIX_CD			BIT(9)
+#define DP83867_PHYSTS_MDIX_AB			BIT(8)
+#define DP83867_PHYSTS_MDIX_MASK		(DP83867_PHYSTS_MDIX_AB | \
+						DP83867_PHYSTS_MDIX_CD)
 
 /* CFG2 bits */
 #define DP83867_DOWNSHIFT_EN		(BIT(8) | BIT(9))
@@ -391,6 +399,22 @@ static int dp83867_read_status(struct phy_device *phydev)
 	else
 		phydev->speed = SPEED_10;
 
+	if (!(status & DP83867_PHYSTS_LINK)) {
+		phydev->mdix = ETH_TP_MDI_INVALID;
+	} else {
+		switch (status & DP83867_PHYSTS_MDIX_MASK) {
+		case 0:
+			phydev->mdix = ETH_TP_MDI;
+			break;
+		case DP83867_PHYSTS_MDIX_MASK:
+			phydev->mdix = ETH_TP_MDI_X;
+			break;
+		default:
+			phydev->mdix = ETH_TP_MDI_INVALID;
+			break;
+		}
+	}
+
 	return 0;
 }
 
@@ -714,6 +738,8 @@ static int dp83867_config_init(struct phy_device *phydev)
 	struct dp83867_private *dp83867 = phydev->priv;
 	int ret, val, bs;
 
+	phydev->mdix_ctrl = ETH_TP_MDI_AUTO;
+
 	/* Force speed optimization for the PHY even if it strapped */
 	ret = phy_modify(phydev, DP83867_CFG2, DP83867_DOWNSHIFT_EN,
 			 DP83867_DOWNSHIFT_EN);
@@ -873,6 +899,39 @@ static int dp83867_config_init(struct phy_device *phydev)
 	return 0;
 }
 
+static int dp83867_config_mdix(struct phy_device *phydev, u8 ctrl)
+{
+	int val;
+
+	switch (ctrl) {
+	case ETH_TP_MDI:
+		val = DP83867_PHYCR_MDIX_MDI;
+		break;
+	case ETH_TP_MDI_X:
+		val = DP83867_PHYCR_MDIX_MDIX;
+		break;
+	case ETH_TP_MDI_AUTO:
+		val = DP83867_PHYCR_MDIX_AUTO;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return phy_modify(phydev, MII_DP83867_PHYCTRL,
+			  DP83867_PHYCR_MDIX_MASK, val);
+}
+
+static int dp83867_config_aneg(struct phy_device *phydev)
+{
+	int ret;
+
+	ret = dp83867_config_mdix(phydev, phydev->mdix_ctrl);
+	if (ret)
+		return ret;
+
+	return genphy_config_aneg(phydev);
+}
+
 static int dp83867_phy_reset(struct phy_device *phydev)
 {
 	int err;
@@ -1127,6 +1186,7 @@ static struct phy_driver dp83867_driver[] = {
 
 		.probe          = dp83867_probe,
 		.config_init	= dp83867_config_init,
+		.config_aneg	= dp83867_config_aneg,
 		.soft_reset	= dp83867_phy_reset,
 
 		.read_status	= dp83867_read_status,
diff --git a/drivers/net/phy/intel-xway.c b/drivers/net/phy/intel-xway.c
index 12ff4c1f285d2..afbcec7117443 100644
--- a/drivers/net/phy/intel-xway.c
+++ b/drivers/net/phy/intel-xway.c
@@ -10,11 +10,16 @@
 #include <linux/of.h>
 #include <linux/bitfield.h>
 
+#define XWAY_MDIO_ERRCNT		0x15	/* error counter */
 #define XWAY_MDIO_MIICTRL		0x17	/* mii control */
 #define XWAY_MDIO_IMASK			0x19	/* interrupt mask */
 #define XWAY_MDIO_ISTAT			0x1A	/* interrupt status */
 #define XWAY_MDIO_LED			0x1B	/* led control */
 
+#define XWAY_MDIO_ERRCNT_SEL		GENMASK(11, 8)
+#define XWAY_MDIO_ERRCNT_COUNT		GENMASK(7, 0)
+#define XWAY_MDIO_ERRCNT_SEL_RXERR	0
+
 #define XWAY_MDIO_MIICTRL_RXSKEW_MASK	GENMASK(14, 12)
 #define XWAY_MDIO_MIICTRL_TXSKEW_MASK	GENMASK(10, 8)
 
@@ -169,6 +174,10 @@
 #define PHY_ID_PHY11G_VR9_1_2		0xD565A409
 #define PHY_ID_PHY22F_VR9_1_2		0xD565A419
 
+struct xway_gphy_priv {
+	u64 rx_errors;
+};
+
 static const int xway_internal_delay[] = {0, 500, 1000, 1500, 2000, 2500,
 					 3000, 3500};
 
@@ -299,6 +308,21 @@ static int xway_gphy_config_init(struct phy_device *phydev)
 	if (err)
 		return err;
 
+	/* Count MDI RX errors (SymbolErrorDuringCarrier) */
+	return phy_write(phydev, XWAY_MDIO_ERRCNT,
+			 FIELD_PREP(XWAY_MDIO_ERRCNT_SEL, XWAY_MDIO_ERRCNT_SEL_RXERR));
+}
+
+static int xway_gphy_probe(struct phy_device *phydev)
+{
+	struct device *dev = &phydev->mdio.dev;
+	struct xway_gphy_priv *priv;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+	phydev->priv = priv;
+
 	return 0;
 }
 
@@ -532,6 +556,31 @@ static int xway_gphy_led_polarity_set(struct phy_device *phydev, int index,
 	return -EINVAL;
 }
 
+static int xway_gphy_update_stats(struct phy_device *phydev)
+{
+	struct xway_gphy_priv *priv = phydev->priv;
+	int ret;
+
+	/* XWAY_MDIO_ERRCNT: 8-bit read-clear counter, SEL set to RXERR */
+	ret = phy_read(phydev, XWAY_MDIO_ERRCNT);
+	if (ret < 0)
+		return ret;
+
+	priv->rx_errors += FIELD_GET(XWAY_MDIO_ERRCNT_COUNT, ret);
+
+	return 0;
+}
+
+static void xway_gphy_get_phy_stats(struct phy_device *phydev,
+				    struct ethtool_eth_phy_stats *eth_stats,
+				    struct ethtool_phy_stats *stats)
+{
+	struct xway_gphy_priv *priv = phydev->priv;
+
+	eth_stats->SymbolErrorDuringCarrier = priv->rx_errors;
+	stats->rx_errors = priv->rx_errors;
+}
+
 static struct phy_driver xway_gphy[] = {
 	{
 		.phy_id		= PHY_ID_PHY11G_1_3,
@@ -539,6 +588,7 @@ static struct phy_driver xway_gphy[] = {
 		.name		= "Intel XWAY PHY11G (PEF 7071/PEF 7072) v1.3",
 		/* PHY_GBIT_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.config_aneg	= xway_gphy14_config_aneg,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -549,12 +599,15 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	}, {
 		.phy_id		= PHY_ID_PHY22F_1_3,
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY22F (PEF 7061) v1.3",
 		/* PHY_BASIC_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.config_aneg	= xway_gphy14_config_aneg,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -565,12 +618,15 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	}, {
 		.phy_id		= PHY_ID_PHY11G_1_4,
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY11G (PEF 7071/PEF 7072) v1.4",
 		/* PHY_GBIT_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.config_aneg	= xway_gphy14_config_aneg,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -581,12 +637,15 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	}, {
 		.phy_id		= PHY_ID_PHY22F_1_4,
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY22F (PEF 7061) v1.4",
 		/* PHY_BASIC_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.config_aneg	= xway_gphy14_config_aneg,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
@@ -597,12 +656,15 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	}, {
 		.phy_id		= PHY_ID_PHY11G_1_5,
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY11G (PEF 7071/PEF 7072) v1.5 / v1.6",
 		/* PHY_GBIT_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
 		.suspend	= genphy_suspend,
@@ -612,12 +674,15 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	}, {
 		.phy_id		= PHY_ID_PHY22F_1_5,
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY22F (PEF 7061) v1.5 / v1.6",
 		/* PHY_BASIC_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
 		.suspend	= genphy_suspend,
@@ -627,12 +692,15 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	}, {
 		.phy_id		= PHY_ID_PHY11G_VR9_1_1,
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY11G (xRX v1.1 integrated)",
 		/* PHY_GBIT_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
 		.suspend	= genphy_suspend,
@@ -642,12 +710,15 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	}, {
 		.phy_id		= PHY_ID_PHY22F_VR9_1_1,
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY22F (xRX v1.1 integrated)",
 		/* PHY_BASIC_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
 		.suspend	= genphy_suspend,
@@ -657,12 +728,15 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	}, {
 		.phy_id		= PHY_ID_PHY11G_VR9_1_2,
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY11G (xRX v1.2 integrated)",
 		/* PHY_GBIT_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
 		.suspend	= genphy_suspend,
@@ -672,12 +746,15 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	}, {
 		.phy_id		= PHY_ID_PHY22F_VR9_1_2,
 		.phy_id_mask	= 0xffffffff,
 		.name		= "Intel XWAY PHY22F (xRX v1.2 integrated)",
 		/* PHY_BASIC_FEATURES */
 		.config_init	= xway_gphy_config_init,
+		.probe		= xway_gphy_probe,
 		.handle_interrupt = xway_gphy_handle_interrupt,
 		.config_intr	= xway_gphy_config_intr,
 		.suspend	= genphy_suspend,
@@ -687,6 +764,8 @@ static struct phy_driver xway_gphy[] = {
 		.led_hw_control_get = xway_gphy_led_hw_control_get,
 		.led_hw_control_set = xway_gphy_led_hw_control_set,
 		.led_polarity_set = xway_gphy_led_polarity_set,
+		.update_stats	= xway_gphy_update_stats,
+		.get_phy_stats	= xway_gphy_get_phy_stats,
 	},
 };
 module_phy_driver(xway_gphy);
diff --git a/drivers/net/phy/motorcomm.c b/drivers/net/phy/motorcomm.c
index 4d62f7b362128..708491bc198a7 100644
--- a/drivers/net/phy/motorcomm.c
+++ b/drivers/net/phy/motorcomm.c
@@ -10,7 +10,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/phy.h>
-#include <linux/of.h>
+#include <linux/property.h>
 
 #define PHY_ID_YT8511		0x0000010a
 #define PHY_ID_YT8521		0x0000011a
@@ -843,12 +843,12 @@ static u32 ytphy_get_delay_reg_value(struct phy_device *phydev,
 				     u16 *rxc_dly_en,
 				     u32 dflt)
 {
-	struct device_node *node = phydev->mdio.dev.of_node;
+	struct device *dev = &phydev->mdio.dev;
 	int tb_size_half = tb_size / 2;
 	u32 val;
 	int i;
 
-	if (of_property_read_u32(node, prop_name, &val))
+	if (device_property_read_u32(dev, prop_name, &val))
 		goto err_dts_val;
 
 	/* when rxc_dly_en is NULL, it is get the delay for tx, only half of
@@ -996,12 +996,12 @@ static int yt8531_get_ds_map(struct phy_device *phydev, u32 cur)
 
 static int yt8531_set_ds(struct phy_device *phydev)
 {
-	struct device_node *node = phydev->mdio.dev.of_node;
+	struct device *dev = &phydev->mdio.dev;
 	u32 ds_field_low, ds_field_hi, val;
 	int ret, ds;
 
 	/* set rgmii rx clk driver strength */
-	if (!of_property_read_u32(node, "motorcomm,rx-clk-drv-microamp", &val)) {
+	if (!device_property_read_u32(dev, "motorcomm,rx-clk-drv-microamp", &val)) {
 		ds = yt8531_get_ds_map(phydev, val);
 		if (ds < 0)
 			return dev_err_probe(&phydev->mdio.dev, ds,
@@ -1018,7 +1018,7 @@ static int yt8531_set_ds(struct phy_device *phydev)
 		return ret;
 
 	/* set rgmii rx data driver strength */
-	if (!of_property_read_u32(node, "motorcomm,rx-data-drv-microamp", &val)) {
+	if (!device_property_read_u32(dev, "motorcomm,rx-data-drv-microamp", &val)) {
 		ds = yt8531_get_ds_map(phydev, val);
 		if (ds < 0)
 			return dev_err_probe(&phydev->mdio.dev, ds,
@@ -1051,7 +1051,6 @@ static int yt8531_set_ds(struct phy_device *phydev)
  */
 static int yt8521_probe(struct phy_device *phydev)
 {
-	struct device_node *node = phydev->mdio.dev.of_node;
 	struct device *dev = &phydev->mdio.dev;
 	struct yt8521_priv *priv;
 	int chip_config;
@@ -1101,7 +1100,7 @@ static int yt8521_probe(struct phy_device *phydev)
 			return ret;
 	}
 
-	if (of_property_read_u32(node, "motorcomm,clk-out-frequency-hz", &freq))
+	if (device_property_read_u32(dev, "motorcomm,clk-out-frequency-hz", &freq))
 		freq = YTPHY_DTS_OUTPUT_CLK_DIS;
 
 	if (phydev->drv->phy_id == PHY_ID_YT8521) {
@@ -1169,11 +1168,11 @@ static int yt8521_probe(struct phy_device *phydev)
 
 static int yt8531_probe(struct phy_device *phydev)
 {
-	struct device_node *node = phydev->mdio.dev.of_node;
+	struct device *dev = &phydev->mdio.dev;
 	u16 mask, val;
 	u32 freq;
 
-	if (of_property_read_u32(node, "motorcomm,clk-out-frequency-hz", &freq))
+	if (device_property_read_u32(dev, "motorcomm,clk-out-frequency-hz", &freq))
 		freq = YTPHY_DTS_OUTPUT_CLK_DIS;
 
 	switch (freq) {
@@ -1665,7 +1664,7 @@ static int yt8521_resume(struct phy_device *phydev)
  */
 static int yt8521_config_init(struct phy_device *phydev)
 {
-	struct device_node *node = phydev->mdio.dev.of_node;
+	struct device *dev = &phydev->mdio.dev;
 	int old_page;
 	int ret = 0;
 
@@ -1680,7 +1679,7 @@ static int yt8521_config_init(struct phy_device *phydev)
 			goto err_restore_page;
 	}
 
-	if (of_property_read_bool(node, "motorcomm,auto-sleep-disabled")) {
+	if (device_property_read_bool(dev, "motorcomm,auto-sleep-disabled")) {
 		/* disable auto sleep */
 		ret = ytphy_modify_ext(phydev, YT8521_EXTREG_SLEEP_CONTROL1_REG,
 				       YT8521_ESC1R_SLEEP_SW, 0);
@@ -1688,7 +1687,7 @@ static int yt8521_config_init(struct phy_device *phydev)
 			goto err_restore_page;
 	}
 
-	if (of_property_read_bool(node, "motorcomm,keep-pll-enabled")) {
+	if (device_property_read_bool(dev, "motorcomm,keep-pll-enabled")) {
 		/* enable RXC clock when no wire plug */
 		ret = ytphy_modify_ext(phydev, YT8521_CLOCK_GATING_REG,
 				       YT8521_CGR_RX_CLK_EN, 0);
@@ -1801,14 +1800,14 @@ static int yt8521_led_hw_control_get(struct phy_device *phydev, u8 index,
 
 static int yt8531_config_init(struct phy_device *phydev)
 {
-	struct device_node *node = phydev->mdio.dev.of_node;
+	struct device *dev = &phydev->mdio.dev;
 	int ret;
 
 	ret = ytphy_rgmii_clk_delay_config_with_lock(phydev);
 	if (ret < 0)
 		return ret;
 
-	if (of_property_read_bool(node, "motorcomm,auto-sleep-disabled")) {
+	if (device_property_read_bool(dev, "motorcomm,auto-sleep-disabled")) {
 		/* disable auto sleep */
 		ret = ytphy_modify_ext_with_lock(phydev,
 						 YT8521_EXTREG_SLEEP_CONTROL1_REG,
@@ -1817,7 +1816,7 @@ static int yt8531_config_init(struct phy_device *phydev)
 			return ret;
 	}
 
-	if (of_property_read_bool(node, "motorcomm,keep-pll-enabled")) {
+	if (device_property_read_bool(dev, "motorcomm,keep-pll-enabled")) {
 		/* enable RXC clock when no wire plug */
 		ret = ytphy_modify_ext_with_lock(phydev,
 						 YT8521_CLOCK_GATING_REG,
@@ -1844,7 +1843,7 @@ static int yt8531_config_init(struct phy_device *phydev)
  */
 static void yt8531_link_change_notify(struct phy_device *phydev)
 {
-	struct device_node *node = phydev->mdio.dev.of_node;
+	struct device *dev = &phydev->mdio.dev;
 	bool tx_clk_1000_inverted = false;
 	bool tx_clk_100_inverted = false;
 	bool tx_clk_10_inverted = false;
@@ -1852,17 +1851,17 @@ static void yt8531_link_change_notify(struct phy_device *phydev)
 	u16 val = 0;
 	int ret;
 
-	if (of_property_read_bool(node, "motorcomm,tx-clk-adj-enabled"))
+	if (device_property_read_bool(dev, "motorcomm,tx-clk-adj-enabled"))
 		tx_clk_adj_enabled = true;
 
 	if (!tx_clk_adj_enabled)
 		return;
 
-	if (of_property_read_bool(node, "motorcomm,tx-clk-10-inverted"))
+	if (device_property_read_bool(dev, "motorcomm,tx-clk-10-inverted"))
 		tx_clk_10_inverted = true;
-	if (of_property_read_bool(node, "motorcomm,tx-clk-100-inverted"))
+	if (device_property_read_bool(dev, "motorcomm,tx-clk-100-inverted"))
 		tx_clk_100_inverted = true;
-	if (of_property_read_bool(node, "motorcomm,tx-clk-1000-inverted"))
+	if (device_property_read_bool(dev, "motorcomm,tx-clk-1000-inverted"))
 		tx_clk_1000_inverted = true;
 
 	if (phydev->speed < 0)
diff --git a/drivers/net/phy/phy-c45.c b/drivers/net/phy/phy-c45.c
index d48aa7231b370..1269517414288 100644
--- a/drivers/net/phy/phy-c45.c
+++ b/drivers/net/phy/phy-c45.c
@@ -940,6 +940,14 @@ EXPORT_SYMBOL_GPL(genphy_c45_read_eee_abilities);
  */
 int genphy_c45_an_config_eee_aneg(struct phy_device *phydev)
 {
+	/* Writing MMD AN advertisements while autoneg is disabled has no
+	 * effect on link-partner negotiation, but on some PHYs (e.g. the
+	 * Broadcom BCM54213PE) the write itself disturbs the receive
+	 * datapath. Skip it.
+	 */
+	if (phydev->autoneg == AUTONEG_DISABLE)
+		return 0;
+
 	if (!phydev->eee_cfg.eee_enabled) {
 		__ETHTOOL_DECLARE_LINK_MODE_MASK(adv) = {};
 
diff --git a/drivers/net/phy/phy_device.c b/drivers/net/phy/phy_device.c
index c2cdf1ae35424..3370eb822017b 100644
--- a/drivers/net/phy/phy_device.c
+++ b/drivers/net/phy/phy_device.c
@@ -2877,7 +2877,8 @@ EXPORT_SYMBOL(phy_advertise_supported);
  */
 void phy_advertise_eee_all(struct phy_device *phydev)
 {
-	linkmode_copy(phydev->advertising_eee, phydev->supported_eee);
+	linkmode_andnot(phydev->advertising_eee, phydev->supported_eee,
+			phydev->eee_disabled_modes);
 }
 EXPORT_SYMBOL_GPL(phy_advertise_eee_all);
 
@@ -2903,7 +2904,8 @@ EXPORT_SYMBOL_GPL(phy_advertise_eee_all);
  */
 void phy_support_eee(struct phy_device *phydev)
 {
-	linkmode_copy(phydev->advertising_eee, phydev->supported_eee);
+	linkmode_andnot(phydev->advertising_eee, phydev->supported_eee,
+			phydev->eee_disabled_modes);
 	phydev->eee_cfg.tx_lpi_enabled = true;
 	phydev->eee_cfg.eee_enabled = true;
 
diff --git a/drivers/net/phy/realtek/realtek_main.c b/drivers/net/phy/realtek/realtek_main.c
index 79c867ef64dad..27268811f564f 100644
--- a/drivers/net/phy/realtek/realtek_main.c
+++ b/drivers/net/phy/realtek/realtek_main.c
@@ -40,7 +40,12 @@
 						 RTL8201F_ISR_LINK)
 
 #define RTL821x_INER				0x12
-#define RTL8211B_INER_INIT			0x6400
+#define RTL8211B_INER_SPEED			BIT(14)
+#define RTL8211B_INER_DUPLEX			BIT(13)
+#define RTL8211B_INER_LINK_STATUS		BIT(10)
+#define RTL8211B_INER_INIT			(RTL8211B_INER_SPEED | \
+						 RTL8211B_INER_DUPLEX | \
+						 RTL8211B_INER_LINK_STATUS)
 #define RTL8211E_INER_LINK_STATUS		BIT(10)
 #define RTL8211F_INER_PME			BIT(7)
 #define RTL8211F_INER_LINK_STATUS		BIT(4)
@@ -165,6 +170,18 @@
 
 #define RTL8221B_VND2_INSR			0xa4d4
 
+#define RTL822X_VND2_LED(x)			(0xd032 + ((x) * 2))
+#define RTL822X_VND2_LCR_LINK_10		BIT(0)
+#define RTL822X_VND2_LCR_LINK_100		BIT(1)
+#define RTL822X_VND2_LCR_LINK_1000		BIT(2)
+#define RTL822X_VND2_LCR_LINK_2500		BIT(5)
+
+#define RTL822X_VND2_LCR6			0xd040
+#define RTL822X_VND2_LED_ACT(x)			BIT(x)
+
+#define RTL822X_VND2_LCR7			0xd044
+#define RTL822X_VND2_LED_POLAR(x)		BIT(x)
+
 #define RTL8224_MII_RTCT			0x11
 #define RTL8224_MII_RTCT_ENABLE			BIT(0)
 #define RTL8224_MII_RTCT_PAIR_A			BIT(4)
@@ -189,6 +206,21 @@
 #define RTL8224_VND1_MDI_PAIR_SWAP		0xa90
 #define RTL8224_VND1_MDI_POLARITY_SWAP		0xa94
 
+#define RTL8226_VND1_UNKNOWN_6A21		0x6a21
+#define  RTL8226_VND1_UNKNOWN_6A21_MDI_SWAP_EN	BIT(5)
+
+#define RTL8226_VND2_UNKNOWN_D068		0xd068
+#define  RTL8226_VND2_UNKNOWN_D068_MDI_SWAP_FLAG	BIT(1)
+#define  RTL8226_VND2_UNKNOWN_D068_PAIR_SEL	GENMASK(4, 3)
+#define RTL8226_VND2_ADCCAL_OFFSET		0xd06a
+
+#define RTL8226_VND2_RG_LPF_CAP_XG_P0_P1	0xbd5a
+#define RTL8226_VND2_RG_LPF_CAP_XG_P2_P3	0xbd5c
+#define RTL8226_VND2_RG_LPF_CAP_P0_P1		0xbc18
+#define RTL8226_VND2_RG_LPF_CAP_P2_P3		0xbc1a
+#define  RTL8226_RG_LPF_CAP_PAIR_A_MASK		GENMASK(4, 0)
+#define  RTL8226_RG_LPF_CAP_PAIR_B_MASK		GENMASK(12, 8)
+
 #define RTL8366RB_POWER_SAVE			0x15
 #define RTL8366RB_POWER_SAVE_ON			BIT(12)
 
@@ -1468,6 +1500,144 @@ static int rtl822xb_write_mmd(struct phy_device *phydev, int devnum, u16 reg,
 	return write_ret;
 }
 
+static int rtl8226_set_mdi_swap(struct phy_device *phydev, bool swap_enable)
+{
+	u16 val = swap_enable ? RTL8226_VND1_UNKNOWN_6A21_MDI_SWAP_EN : 0;
+
+	return phy_modify_mmd(phydev, MDIO_MMD_VEND1, RTL8226_VND1_UNKNOWN_6A21,
+			      RTL8226_VND1_UNKNOWN_6A21_MDI_SWAP_EN, val);
+}
+
+static int rtl8226_swap_rg_lpf_cap(struct phy_device *phydev, u32 reg_p0_p1, u32 reg_p2_p3)
+{
+	u16 val_p0, val_p1, val_p2, val_p3;
+	int ret;
+
+	ret = phy_read_mmd(phydev, MDIO_MMD_VEND2, reg_p0_p1);
+	if (ret < 0)
+		return ret;
+
+	val_p0 = FIELD_GET(RTL8226_RG_LPF_CAP_PAIR_A_MASK, ret);
+	val_p1 = FIELD_GET(RTL8226_RG_LPF_CAP_PAIR_B_MASK, ret);
+
+	ret = phy_read_mmd(phydev, MDIO_MMD_VEND2, reg_p2_p3);
+	if (ret < 0)
+		return ret;
+
+	val_p2 = FIELD_GET(RTL8226_RG_LPF_CAP_PAIR_A_MASK, ret);
+	val_p3 = FIELD_GET(RTL8226_RG_LPF_CAP_PAIR_B_MASK, ret);
+
+	ret = phy_modify_mmd(phydev, MDIO_MMD_VEND2, reg_p0_p1,
+			     RTL8226_RG_LPF_CAP_PAIR_A_MASK | RTL8226_RG_LPF_CAP_PAIR_B_MASK,
+			     FIELD_PREP(RTL8226_RG_LPF_CAP_PAIR_A_MASK, val_p3) |
+			     FIELD_PREP(RTL8226_RG_LPF_CAP_PAIR_B_MASK, val_p2));
+	if (ret < 0)
+		return ret;
+
+	return phy_modify_mmd(phydev, MDIO_MMD_VEND2, reg_p2_p3,
+			     RTL8226_RG_LPF_CAP_PAIR_A_MASK | RTL8226_RG_LPF_CAP_PAIR_B_MASK,
+			     FIELD_PREP(RTL8226_RG_LPF_CAP_PAIR_A_MASK, val_p1) |
+			     FIELD_PREP(RTL8226_RG_LPF_CAP_PAIR_B_MASK, val_p0));
+}
+
+static int rtl8226_patch_mdi_swap(struct phy_device *phydev, bool swap_enable)
+{
+	u16 adccal_offset[4];
+	bool is_patched;
+	int ret;
+
+	ret = phy_read_mmd(phydev, MDIO_MMD_VEND2, RTL8226_VND2_UNKNOWN_D068);
+	if (ret < 0)
+		return ret;
+
+	is_patched = !(ret & RTL8226_VND2_UNKNOWN_D068_MDI_SWAP_FLAG);
+
+	if (is_patched == swap_enable) {
+		/* Nothing to do */
+		return 0;
+	}
+
+	if (!swap_enable) {
+		/* Patching is only implemented one-way, see next comment. */
+		phydev_err(phydev, "MDI swapping disabled, but PHY is already patched.\n");
+		return -EINVAL;
+	}
+
+	/* The exact meaning of these bits is unknown. We only know that bit 1
+	 * is used as a flag that swapping is already done.
+	 */
+	ret = phy_modify_mmd(phydev, MDIO_MMD_VEND2, RTL8226_VND2_UNKNOWN_D068, 0x7, 0x1);
+	if (ret < 0)
+		return ret;
+
+	for (int i = 0; i < 4; i++) {
+		ret = phy_modify_mmd(phydev, MDIO_MMD_VEND2, RTL8226_VND2_UNKNOWN_D068,
+				     RTL8226_VND2_UNKNOWN_D068_PAIR_SEL,
+				     FIELD_PREP(RTL8226_VND2_UNKNOWN_D068_PAIR_SEL, i));
+		if (ret < 0)
+			return ret;
+
+		ret = phy_read_mmd(phydev, MDIO_MMD_VEND2, RTL8226_VND2_ADCCAL_OFFSET);
+		if (ret < 0)
+			return ret;
+
+		adccal_offset[i] = ret;
+	}
+
+	for (int i = 0; i < 4; i++) {
+		ret = phy_modify_mmd(phydev, MDIO_MMD_VEND2, RTL8226_VND2_UNKNOWN_D068,
+				     RTL8226_VND2_UNKNOWN_D068_PAIR_SEL,
+				     FIELD_PREP(RTL8226_VND2_UNKNOWN_D068_PAIR_SEL, i));
+		if (ret < 0)
+			return ret;
+
+		ret = phy_write_mmd(phydev, MDIO_MMD_VEND2, RTL8226_VND2_ADCCAL_OFFSET,
+				    adccal_offset[3 - i]);
+		if (ret < 0)
+			return ret;
+	}
+
+	ret = rtl8226_swap_rg_lpf_cap(phydev, RTL8226_VND2_RG_LPF_CAP_XG_P0_P1,
+				      RTL8226_VND2_RG_LPF_CAP_XG_P2_P3);
+	if (ret < 0)
+		return ret;
+
+	return rtl8226_swap_rg_lpf_cap(phydev, RTL8226_VND2_RG_LPF_CAP_P0_P1,
+				       RTL8226_VND2_RG_LPF_CAP_P2_P3);
+}
+
+static int rtl8226_config_mdi_order(struct phy_device *phydev)
+{
+	u32 order;
+	bool swap_enable;
+	int ret;
+
+	ret = of_property_read_u32(phydev->mdio.dev.of_node, "enet-phy-pair-order", &order);
+
+	/* Property not present, nothing to do */
+	if (ret == -EINVAL || ret == -ENOSYS)
+		return 0;
+
+	if (ret)
+		return ret;
+
+	if (order & ~1)
+		return -EINVAL;
+
+	swap_enable = !!(order & 1);
+
+	ret = rtl8226_set_mdi_swap(phydev, swap_enable);
+	if (ret)
+		return ret;
+
+	return rtl8226_patch_mdi_swap(phydev, swap_enable);
+}
+
+static int rtl8226_probe(struct phy_device *phydev)
+{
+	return rtl8226_config_mdi_order(phydev);
+}
+
 static int rtl822x_set_serdes_option_mode(struct phy_device *phydev, bool gen1)
 {
 	bool has_2500, has_sgmii;
@@ -1797,6 +1967,151 @@ static int rtl822xb_c45_read_status(struct phy_device *phydev)
 	return 0;
 }
 
+static int rtl822xb_led_brightness_set(struct phy_device *phydev, u8 index,
+				       enum led_brightness value)
+{
+	int ret;
+
+	if (index >= RTL8211x_LED_COUNT)
+		return -EINVAL;
+
+	/* clear HW LED setup */
+	ret = phy_write_mmd(phydev, MDIO_MMD_VEND2,
+			    RTL822X_VND2_LED(index), 0);
+	if (ret < 0)
+		return ret;
+
+	/* clear HW LED blink */
+	ret = phy_clear_bits_mmd(phydev, MDIO_MMD_VEND2, RTL822X_VND2_LCR6,
+				 RTL822X_VND2_LED_ACT(index));
+	if (ret < 0)
+		return ret;
+
+	if (value != LED_OFF)
+		return phy_set_bits_mmd(phydev, MDIO_MMD_VEND2,
+					RTL822X_VND2_LCR7,
+					RTL822X_VND2_LED_POLAR(index));
+	else
+		return phy_clear_bits_mmd(phydev, MDIO_MMD_VEND2,
+					  RTL822X_VND2_LCR7,
+					  RTL822X_VND2_LED_POLAR(index));
+}
+
+static int rtl822xb_led_hw_is_supported(struct phy_device *phydev, u8 index,
+					unsigned long rules)
+{
+	const unsigned long  act_mask = BIT(TRIGGER_NETDEV_RX) |
+					BIT(TRIGGER_NETDEV_TX);
+
+	const unsigned long link_mask = BIT(TRIGGER_NETDEV_LINK) |
+					BIT(TRIGGER_NETDEV_LINK_10) |
+					BIT(TRIGGER_NETDEV_LINK_100) |
+					BIT(TRIGGER_NETDEV_LINK_1000) |
+					BIT(TRIGGER_NETDEV_LINK_2500);
+
+	if (index >= RTL8211x_LED_COUNT)
+		return -EINVAL;
+
+	/* Filter out any other unsupported triggers. */
+	if (rules & ~(link_mask | act_mask))
+		return -EOPNOTSUPP;
+
+	/* RX and TX are not differentiated, they are not possible
+	 * without combination with a link trigger.
+	 */
+	if ((rules & act_mask) && !(rules & link_mask))
+		return -EOPNOTSUPP;
+
+	return 0;
+}
+
+static int rtl822xb_led_hw_control_get(struct phy_device *phydev, u8 index,
+				       unsigned long *rules)
+{
+	int val;
+
+	if (index >= RTL8211x_LED_COUNT)
+		return -EINVAL;
+
+	val = phy_read_mmd(phydev, MDIO_MMD_VEND2, RTL822X_VND2_LED(index));
+	if (val < 0)
+		return val;
+
+	if (val & RTL822X_VND2_LCR_LINK_10)
+		__set_bit(TRIGGER_NETDEV_LINK_10, rules);
+
+	if (val & RTL822X_VND2_LCR_LINK_100)
+		__set_bit(TRIGGER_NETDEV_LINK_100, rules);
+
+	if (val & RTL822X_VND2_LCR_LINK_1000)
+		__set_bit(TRIGGER_NETDEV_LINK_1000, rules);
+
+	if (val & RTL822X_VND2_LCR_LINK_2500)
+		__set_bit(TRIGGER_NETDEV_LINK_2500, rules);
+
+	if ((val & RTL822X_VND2_LCR_LINK_10) &&
+	    (val & RTL822X_VND2_LCR_LINK_100) &&
+	    (val & RTL822X_VND2_LCR_LINK_1000) &&
+	    (val & RTL822X_VND2_LCR_LINK_2500))
+		__set_bit(TRIGGER_NETDEV_LINK, rules);
+
+	val = phy_read_mmd(phydev, MDIO_MMD_VEND2, RTL822X_VND2_LCR6);
+	if (val < 0)
+		return val;
+
+	if (val & RTL822X_VND2_LED_ACT(index)) {
+		__set_bit(TRIGGER_NETDEV_RX, rules);
+		__set_bit(TRIGGER_NETDEV_TX, rules);
+	}
+
+	return 0;
+}
+
+static int rtl822xb_led_hw_control_set(struct phy_device *phydev, u8 index,
+				       unsigned long rules)
+{
+	u16 val = 0;
+	bool act;
+	int ret;
+
+	if (index >= RTL8211x_LED_COUNT)
+		return -EINVAL;
+
+	if (test_bit(TRIGGER_NETDEV_LINK, &rules) ||
+	    test_bit(TRIGGER_NETDEV_LINK_10, &rules))
+		val |= RTL822X_VND2_LCR_LINK_10;
+
+	if (test_bit(TRIGGER_NETDEV_LINK, &rules) ||
+	    test_bit(TRIGGER_NETDEV_LINK_100, &rules))
+		val |= RTL822X_VND2_LCR_LINK_100;
+
+	if (test_bit(TRIGGER_NETDEV_LINK, &rules) ||
+	    test_bit(TRIGGER_NETDEV_LINK_1000, &rules))
+		val |= RTL822X_VND2_LCR_LINK_1000;
+
+	if (test_bit(TRIGGER_NETDEV_LINK, &rules) ||
+	    test_bit(TRIGGER_NETDEV_LINK_2500, &rules))
+		val |= RTL822X_VND2_LCR_LINK_2500;
+
+	ret = phy_write_mmd(phydev, MDIO_MMD_VEND2,
+			    RTL822X_VND2_LED(index), val);
+	if (ret < 0)
+		return ret;
+
+	act = test_bit(TRIGGER_NETDEV_RX, &rules) ||
+	      test_bit(TRIGGER_NETDEV_TX, &rules);
+
+	ret = phy_modify_mmd(phydev, MDIO_MMD_VEND2, RTL822X_VND2_LCR6,
+			     RTL822X_VND2_LED_ACT(index), act ?
+			     RTL822X_VND2_LED_ACT(index) : 0);
+	if (ret < 0)
+		return ret;
+
+	/* Reset polarity to default */
+	return phy_clear_bits_mmd(phydev, MDIO_MMD_VEND2, RTL822X_VND2_LCR7,
+				  RTL822X_VND2_LED_POLAR(index));
+}
+
 static int rtl8224_cable_test_start(struct phy_device *phydev)
 {
 	u32 val;
@@ -2522,6 +2837,7 @@ static struct phy_driver realtek_drvs[] = {
 		.soft_reset	= rtl822x_c45_soft_reset,
 		.get_features	= rtl822x_c45_get_features,
 		.config_aneg	= rtl822x_c45_config_aneg,
+		.probe		= rtl8226_probe,
 		.config_init	= rtl822x_config_init,
 		.inband_caps	= rtl822x_inband_caps,
 		.config_inband	= rtl822x_config_inband,
@@ -2565,6 +2881,10 @@ static struct phy_driver realtek_drvs[] = {
 		.write_page	= rtl821x_write_page,
 		.read_mmd	= rtl822xb_read_mmd,
 		.write_mmd	= rtl822xb_write_mmd,
+		.led_brightness_set = rtl822xb_led_brightness_set,
+		.led_hw_is_supported = rtl822xb_led_hw_is_supported,
+		.led_hw_control_get = rtl822xb_led_hw_control_get,
+		.led_hw_control_set = rtl822xb_led_hw_control_set,
 	}, {
 		.match_phy_device = rtl8221b_vm_cg_match_phy_device,
 		.name		= "RTL8221B-VM-CG 2.5Gbps PHY",
@@ -2584,6 +2904,10 @@ static struct phy_driver realtek_drvs[] = {
 		.write_page	= rtl821x_write_page,
 		.read_mmd	= rtl822xb_read_mmd,
 		.write_mmd	= rtl822xb_write_mmd,
+		.led_brightness_set = rtl822xb_led_brightness_set,
+		.led_hw_is_supported = rtl822xb_led_hw_is_supported,
+		.led_hw_control_get = rtl822xb_led_hw_control_get,
+		.led_hw_control_set = rtl822xb_led_hw_control_set,
 	}, {
 		.match_phy_device = rtl8251b_c45_match_phy_device,
 		.name		= "RTL8251B 5Gbps PHY",
diff --git a/drivers/net/ppp/Kconfig b/drivers/net/ppp/Kconfig
index 753354b4e36cb..4047f13b83dc0 100644
--- a/drivers/net/ppp/Kconfig
+++ b/drivers/net/ppp/Kconfig
@@ -110,8 +110,12 @@ config PPPOATM
 	  which can lead to bad results if the ATM peer loses state and
 	  changes its encapsulation unilaterally.
 
+config PPPOX
+	tristate
+
 config PPPOE
 	tristate "PPP over Ethernet"
+	select PPPOX
 	help
 	  Support for PPP over Ethernet.
 
@@ -157,6 +161,7 @@ config PPPOE_HASH_BITS
 config PPTP
 	tristate "PPP over IPv4 (PPTP)"
 	depends on NET_IPGRE_DEMUX
+	select PPPOX
 	help
 	  Support for PPP over IPv4.(Point-to-Point Tunneling Protocol)
 
@@ -168,6 +173,7 @@ config PPTP
 config PPPOL2TP
 	tristate "PPP over L2TP"
 	depends on L2TP
+	select PPPOX
 	help
 	  Support for PPP-over-L2TP socket family. L2TP is a protocol
 	  used by ISPs and enterprises to tunnel PPP traffic over UDP
@@ -191,8 +197,8 @@ config PPP_SYNC_TTY
 	tristate "PPP support for sync tty ports"
 	help
 	  Say Y (or M) here if you want to be able to use PPP over synchronous
-	  (HDLC) tty devices, such as the SyncLink adapter. These devices
-	  are often used for high-speed leased lines like T1/E1.
+	  (HDLC) tty devices. These devices are often used for high-speed leased
+	  lines like T1/E1.
 
 	  To compile this driver as a module, choose M here.
 
diff --git a/drivers/net/ppp/Makefile b/drivers/net/ppp/Makefile
index 16c457d6b3244..a815e14971799 100644
--- a/drivers/net/ppp/Makefile
+++ b/drivers/net/ppp/Makefile
@@ -9,6 +9,6 @@ obj-$(CONFIG_PPP_BSDCOMP) += bsd_comp.o
 obj-$(CONFIG_PPP_DEFLATE) += ppp_deflate.o
 obj-$(CONFIG_PPP_MPPE) += ppp_mppe.o
 obj-$(CONFIG_PPP_SYNC_TTY) += ppp_synctty.o
-obj-$(CONFIG_PPPOE) += pppox.o pppoe.o
-obj-$(CONFIG_PPPOL2TP) += pppox.o
-obj-$(CONFIG_PPTP) += pppox.o pptp.o
+obj-$(CONFIG_PPPOX) += pppox.o
+obj-$(CONFIG_PPPOE) += pppoe.o
+obj-$(CONFIG_PPTP) += pptp.o
diff --git a/drivers/net/ppp/pppoe.c b/drivers/net/ppp/pppoe.c
index bdd61c504a1c4..4a018acb52628 100644
--- a/drivers/net/ppp/pppoe.c
+++ b/drivers/net/ppp/pppoe.c
@@ -77,6 +77,7 @@
 #include <net/net_namespace.h>
 #include <net/netns/generic.h>
 #include <net/sock.h>
+#include <net/gro.h>
 
 #include <linux/uaccess.h>
 
@@ -136,15 +137,15 @@ static inline int cmp_addr(struct pppoe_addr *a, __be16 sid, char *addr)
 #error 8 must be a multiple of PPPOE_HASH_BITS
 #endif
 
-static int hash_item(__be16 sid, unsigned char *addr)
+static u8 hash_item(__be16 sid, const u8 addr[ETH_ALEN])
 {
-	unsigned char hash = 0;
+	const u16 *addr16 = (const u16 *)addr;
 	unsigned int i;
+	u16 hash16;
+	u8 hash;
 
-	for (i = 0; i < ETH_ALEN; i++)
-		hash ^= addr[i];
-	for (i = 0; i < sizeof(sid_t) * 8; i += 8)
-		hash ^= (__force __u32)sid >> i;
+	hash16 = addr16[0] ^ addr16[1] ^ addr16[2] ^ (__force u16)sid;
+	hash = (hash16 >> 8) ^ hash16;
 	for (i = 8; (i >>= 1) >= PPPOE_HASH_BITS;)
 		hash ^= hash >> i;
 
@@ -409,7 +410,7 @@ static int pppoe_rcv(struct sk_buff *skb, struct net_device *dev,
 	if (ppp_skb_is_compressed_proto(skb))
 		goto drop;
 
-	if (pskb_trim_rcsum(skb, len))
+	if (!skb_is_gso(skb) && pskb_trim_rcsum(skb, len))
 		goto drop;
 
 	ph = pppoe_hdr(skb);
@@ -1103,6 +1104,160 @@ static struct pernet_operations pppoe_net_ops = {
 	.size = sizeof(struct pppoe_net),
 };
 
+static u16
+compare_pppoe_header(const struct pppoe_hdr *phdr,
+		     const struct pppoe_hdr *phdr2)
+{
+	__be16 proto = *(const __be16 *)(phdr + 1);
+	__be16 proto2 = *(const __be16 *)(phdr2 + 1);
+
+	return (__force u16)((phdr->sid ^ phdr2->sid) | (proto ^ proto2));
+}
+
+static __be16 pppoe_hdr_proto(const struct pppoe_hdr *phdr)
+{
+	__be16 proto = *(const __be16 *)(phdr + 1);
+
+	switch (proto) {
+	case cpu_to_be16(PPP_IP):
+		return cpu_to_be16(ETH_P_IP);
+#if IS_ENABLED(CONFIG_IPV6)
+	case cpu_to_be16(PPP_IPV6):
+		return cpu_to_be16(ETH_P_IPV6);
+#endif
+	default:
+		return 0;
+	}
+}
+
+static struct sk_buff *pppoe_gro_receive(struct list_head *head,
+					 struct sk_buff *skb)
+{
+	const struct packet_offload *ptype;
+	unsigned int hlen, off_pppoe;
+	const struct pppoe_hdr *phdr;
+	struct sk_buff *pp = NULL;
+	struct sk_buff *p;
+	int flush = 1;
+	__be16 type;
+
+	off_pppoe = skb_gro_offset(skb);
+	hlen = off_pppoe + PPPOE_SES_HLEN;
+	phdr = skb_gro_header(skb, hlen, off_pppoe);
+	if (unlikely(!phdr))
+		goto out;
+
+	/* filter for session packets (type:1, ver:1, code:0) */
+	if (*(const __be16 *)phdr != cpu_to_be16(0x1100))
+		goto out;
+
+	/* ignore packets with padding or invalid length */
+	if (skb_gro_len(skb) != be16_to_cpu(phdr->length) + sizeof(*phdr))
+		goto out;
+
+	type = pppoe_hdr_proto(phdr);
+	ptype = gro_find_receive_by_type(type);
+	if (!ptype)
+		goto out;
+
+	flush = 0;
+
+	list_for_each_entry(p, head, list) {
+		const struct pppoe_hdr *phdr2;
+
+		if (!NAPI_GRO_CB(p)->same_flow)
+			continue;
+
+		phdr2 = (const struct pppoe_hdr *)(p->data + off_pppoe);
+		if (compare_pppoe_header(phdr, phdr2))
+			NAPI_GRO_CB(p)->same_flow = 0;
+	}
+
+	skb_gro_pull(skb, PPPOE_SES_HLEN);
+	skb_gro_postpull_rcsum(skb, phdr, PPPOE_SES_HLEN);
+
+	pp = indirect_call_gro_receive_inet(ptype->callbacks.gro_receive,
+					    ipv6_gro_receive, inet_gro_receive,
+					    head, skb);
+
+out:
+	skb_gro_flush_final(skb, pp, flush);
+
+	return pp;
+}
+
+static int pppoe_gro_complete(struct sk_buff *skb, int nhoff)
+{
+	struct pppoe_hdr *phdr = (struct pppoe_hdr *)(skb->data + nhoff);
+	__be16 type = pppoe_hdr_proto(phdr);
+	struct packet_offload *ptype;
+	unsigned int len;
+
+	ptype = gro_find_complete_by_type(type);
+	if (!ptype)
+		return -ENOENT;
+
+	len = skb->len - (nhoff + sizeof(*phdr));
+	len = min(len, 0xFFFFU);
+	phdr->length = cpu_to_be16(len);
+
+	return INDIRECT_CALL_INET(ptype->callbacks.gro_complete,
+				  ipv6_gro_complete, inet_gro_complete,
+				  skb, nhoff + PPPOE_SES_HLEN);
+}
+
+static struct sk_buff *pppoe_gso_segment(struct sk_buff *skb,
+					 netdev_features_t features)
+{
+	struct sk_buff *segs = ERR_PTR(-EINVAL);
+	struct packet_offload *ptype;
+	struct pppoe_hdr *phdr;
+	__be16 orig_type, type;
+	int len, nhoff;
+
+	skb_reset_network_header(skb);
+	nhoff = skb_network_header(skb) - skb_mac_header(skb);
+
+	if (unlikely(!pskb_may_pull(skb, PPPOE_SES_HLEN)))
+		goto out;
+
+	phdr = (struct pppoe_hdr *)skb_network_header(skb);
+	type = pppoe_hdr_proto(phdr);
+	ptype = gro_find_complete_by_type(type);
+	if (!ptype)
+		goto out;
+
+	orig_type = skb->protocol;
+	__skb_pull(skb, PPPOE_SES_HLEN);
+	features &= ~NETIF_F_GSO_SOFTWARE;
+	segs = ptype->callbacks.gso_segment(skb, features);
+	if (IS_ERR_OR_NULL(segs))
+		goto out;
+
+	skb = segs;
+	do {
+		phdr = (struct pppoe_hdr *)(skb_mac_header(skb) + nhoff);
+		len = skb->len - (nhoff + sizeof(*phdr));
+		phdr->length = cpu_to_be16(len);
+		skb->network_header = (u8 *)phdr - skb->head;
+		skb->protocol = orig_type;
+		skb_reset_mac_len(skb);
+	} while ((skb = skb->next));
+
+out:
+	return segs;
+}
+
+static struct packet_offload pppoe_packet_offload __read_mostly = {
+	.type = cpu_to_be16(ETH_P_PPP_SES),
+	.priority = 20,
+	.callbacks = {
+		.gro_receive = pppoe_gro_receive,
+		.gro_complete = pppoe_gro_complete,
+		.gso_segment = pppoe_gso_segment,
+	},
+};
+
 static int __init pppoe_init(void)
 {
 	int err;
@@ -1119,6 +1274,8 @@ static int __init pppoe_init(void)
 	if (err)
 		goto out_unregister_pppoe_proto;
 
+	if (IS_ENABLED(CONFIG_INET))
+		dev_add_offload(&pppoe_packet_offload);
 	dev_add_pack(&pppoes_ptype);
 	dev_add_pack(&pppoed_ptype);
 	register_netdevice_notifier(&pppoe_notifier);
@@ -1138,6 +1295,8 @@ static void __exit pppoe_exit(void)
 	unregister_netdevice_notifier(&pppoe_notifier);
 	dev_remove_pack(&pppoed_ptype);
 	dev_remove_pack(&pppoes_ptype);
+	if (IS_ENABLED(CONFIG_INET))
+		dev_remove_offload(&pppoe_packet_offload);
 	unregister_pppox_proto(PX_PROTO_OE);
 	proto_unregister(&pppoe_sk_proto);
 	unregister_pernet_device(&pppoe_net_ops);
diff --git a/drivers/net/pse-pd/pd692x0.c b/drivers/net/pse-pd/pd692x0.c
index 4a3c852780f58..49b1527829ad9 100644
--- a/drivers/net/pse-pd/pd692x0.c
+++ b/drivers/net/pse-pd/pd692x0.c
@@ -1853,7 +1853,7 @@ static void pd692x0_i2c_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id pd692x0_id[] = {
-	{ PD692X0_PSE_NAME },
+	{ .name = PD692X0_PSE_NAME },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, pd692x0_id);
diff --git a/drivers/net/pse-pd/pse_core.c b/drivers/net/pse-pd/pse_core.c
index 87aa4f4e97249..69dbdbde9d71a 100644
--- a/drivers/net/pse-pd/pse_core.c
+++ b/drivers/net/pse-pd/pse_core.c
@@ -210,7 +210,7 @@ static int of_load_pse_pis(struct pse_controller_dev *pcdev)
 			ret = of_load_pse_pi_pairsets(node, &pi, ret);
 			if (ret)
 				goto out;
-		} else if (ret != ENOENT) {
+		} else if (ret != -ENOENT) {
 			dev_err(pcdev->dev,
 				"error: wrong number of pairsets. Should be 1 or 2, got %d (%pOF)\n",
 				ret, node);
diff --git a/drivers/net/pse-pd/si3474.c b/drivers/net/pse-pd/si3474.c
index aa07ffbce54d7..1845b9c51cf7a 100644
--- a/drivers/net/pse-pd/si3474.c
+++ b/drivers/net/pse-pd/si3474.c
@@ -550,8 +550,8 @@ static int si3474_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id si3474_id[] = {
-	{ "si3474" },
-	{}
+	{ .name = "si3474" },
+	{ }
 };
 MODULE_DEVICE_TABLE(i2c, si3474_id);
 
diff --git a/drivers/net/pse-pd/tps23881.c b/drivers/net/pse-pd/tps23881.c
index d40cb35a25478..49d6389da067c 100644
--- a/drivers/net/pse-pd/tps23881.c
+++ b/drivers/net/pse-pd/tps23881.c
@@ -1528,8 +1528,8 @@ static int tps23881_i2c_probe(struct i2c_client *client)
 }
 
 static const struct i2c_device_id tps23881_id[] = {
-	{ "tps23881", .driver_data = (kernel_ulong_t)&tps23881_info[TPS23881] },
-	{ "tps23881b", .driver_data = (kernel_ulong_t)&tps23881_info[TPS23881B] },
+	{ .name = "tps23881", .driver_data = (kernel_ulong_t)&tps23881_info[TPS23881] },
+	{ .name = "tps23881b", .driver_data = (kernel_ulong_t)&tps23881_info[TPS23881B] },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, tps23881_id);
diff --git a/drivers/net/thunderbolt/main.c b/drivers/net/thunderbolt/main.c
index 7aae5d915a1eb..f8f97e8e2226c 100644
--- a/drivers/net/thunderbolt/main.c
+++ b/drivers/net/thunderbolt/main.c
@@ -34,11 +34,12 @@
 #define TBNET_RING_SIZE		256
 #define TBNET_LOGIN_RETRIES	60
 #define TBNET_LOGOUT_RETRIES	10
+#define TBNET_THROTTLING	128000
 #define TBNET_E2E		BIT(0)
 #define TBNET_MATCH_FRAGS_ID	BIT(1)
 #define TBNET_64K_FRAMES	BIT(2)
 #define TBNET_MAX_MTU		SZ_64K
-#define TBNET_FRAME_SIZE	SZ_4K
+#define TBNET_FRAME_SIZE	TB_MAX_FRAME_SIZE
 #define TBNET_MAX_PAYLOAD_SIZE	\
 	(TBNET_FRAME_SIZE - sizeof(struct thunderbolt_ip_frame_header))
 /* Rx packets need to hold space for skb_shared_info */
@@ -327,11 +328,6 @@ static void stop_login(struct tbnet *net)
 	netdev_dbg(net->dev, "login stopped\n");
 }
 
-static inline unsigned int tbnet_frame_size(const struct tbnet_frame *tf)
-{
-	return tf->frame.size ? : TBNET_FRAME_SIZE;
-}
-
 static void tbnet_free_buffers(struct tbnet_ring *ring)
 {
 	unsigned int i;
@@ -562,7 +558,7 @@ static struct tbnet_frame *tbnet_get_tx_buffer(struct tbnet *net)
 	tf->frame.size = 0;
 
 	dma_sync_single_for_cpu(dma_dev, tf->frame.buffer_phy,
-				tbnet_frame_size(tf), DMA_TO_DEVICE);
+				tb_ring_frame_size(&tf->frame), DMA_TO_DEVICE);
 
 	return tf;
 }
@@ -744,7 +740,7 @@ static bool tbnet_check_frame(struct tbnet *net, const struct tbnet_frame *tf,
 	}
 
 	/* Should be greater than just header i.e. contains data */
-	size = tbnet_frame_size(tf);
+	size = tb_ring_frame_size(&tf->frame);
 	if (size <= sizeof(*hdr)) {
 		net->stats.rx_length_errors++;
 		return false;
@@ -961,6 +957,9 @@ static int tbnet_open(struct net_device *dev)
 	}
 	net->rx_ring.ring = ring;
 
+	tb_ring_throttling(net->tx_ring.ring, TBNET_THROTTLING);
+	tb_ring_throttling(net->rx_ring.ring, TBNET_THROTTLING);
+
 	napi_enable(&net->napi);
 	start_login(net);
 
@@ -1011,7 +1010,8 @@ static bool tbnet_xmit_csum_and_map(struct tbnet *net, struct sk_buff *skb,
 						hdr->frame_index, hdr->frame_count);
 			dma_sync_single_for_device(dma_dev,
 				frames[i]->frame.buffer_phy,
-				tbnet_frame_size(frames[i]), DMA_TO_DEVICE);
+				tb_ring_frame_size(&frames[i]->frame),
+						   DMA_TO_DEVICE);
 		}
 
 		return true;
@@ -1085,7 +1085,7 @@ static bool tbnet_xmit_csum_and_map(struct tbnet *net, struct sk_buff *skb,
 	 */
 	for (i = 0; i < frame_count; i++) {
 		dma_sync_single_for_device(dma_dev, frames[i]->frame.buffer_phy,
-			tbnet_frame_size(frames[i]), DMA_TO_DEVICE);
+			tb_ring_frame_size(&frames[i]->frame), DMA_TO_DEVICE);
 	}
 
 	return true;
diff --git a/drivers/net/tun.c b/drivers/net/tun.c
index b183189f18535..bfa49fa9e3a14 100644
--- a/drivers/net/tun.c
+++ b/drivers/net/tun.c
@@ -145,6 +145,8 @@ struct tun_file {
 	struct list_head next;
 	struct tun_struct *detached;
 	struct ptr_ring tx_ring;
+	/* Protected by tx_ring.consumer_lock */
+	int cons_cnt;
 	struct xdp_rxq_info xdp_rxq;
 };
 
@@ -588,8 +590,13 @@ static void __tun_detach(struct tun_file *tfile, bool clean)
 		rcu_assign_pointer(tun->tfiles[index],
 				   tun->tfiles[tun->numqueues - 1]);
 		ntfile = rtnl_dereference(tun->tfiles[index]);
+		spin_lock(&ntfile->tx_ring.consumer_lock);
 		ntfile->queue_index = index;
 		ntfile->xdp_rxq.queue_index = index;
+		ntfile->cons_cnt = 0;
+		if (__ptr_ring_empty(&ntfile->tx_ring))
+			netif_wake_subqueue(tun->dev, index);
+		spin_unlock(&ntfile->tx_ring.consumer_lock);
 		rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
 				   NULL);
 
@@ -730,6 +737,9 @@ static int tun_attach(struct tun_struct *tun, struct file *file,
 		goto out;
 	}
 
+	spin_lock(&tfile->tx_ring.consumer_lock);
+	tfile->cons_cnt = 0;
+	spin_unlock(&tfile->tx_ring.consumer_lock);
 	tfile->queue_index = tun->numqueues;
 	tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
 
@@ -1008,6 +1018,7 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
 	struct netdev_queue *queue;
 	struct tun_file *tfile;
 	int len = skb->len;
+	int ret;
 
 	rcu_read_lock();
 	tfile = rcu_dereference(tun->tfiles[txq]);
@@ -1062,13 +1073,33 @@ static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	nf_reset_ct(skb);
 
-	if (ptr_ring_produce(&tfile->tx_ring, skb)) {
+	queue = netdev_get_tx_queue(dev, txq);
+
+	spin_lock(&tfile->tx_ring.producer_lock);
+	ret = __ptr_ring_produce(&tfile->tx_ring, skb);
+	if (!qdisc_txq_has_no_queue(queue) &&
+	    __ptr_ring_check_produce(&tfile->tx_ring) == -ENOSPC) {
+		netif_tx_stop_queue(queue);
+		/* Paired with smp_mb() in __tun_wake_queue() */
+		smp_mb__after_atomic();
+		if (!__ptr_ring_check_produce(&tfile->tx_ring))
+			netif_tx_wake_queue(queue);
+	}
+	spin_unlock(&tfile->tx_ring.producer_lock);
+
+	if (ret) {
+		/* This should be a rare case if a qdisc is present, but
+		 * can happen due to lltx.
+		 * Since skb_tx_timestamp(), skb_orphan(),
+		 * run_ebpf_filter() and pskb_trim() could have tinkered
+		 * with the SKB, returning NETDEV_TX_BUSY is unsafe and
+		 * we must drop instead.
+		 */
 		drop_reason = SKB_DROP_REASON_FULL_RING;
 		goto drop;
 	}
 
 	/* dev->lltx requires to do our own update of trans_start */
-	queue = netdev_get_tx_queue(dev, txq);
 	txq_trans_cond_update(queue);
 
 	/* Notify and wake up reader process */
@@ -2115,13 +2146,46 @@ static ssize_t tun_put_user(struct tun_struct *tun,
 	return total;
 }
 
-static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
+/* Callers must hold ring.consumer_lock */
+static void __tun_wake_queue(struct tun_struct *tun,
+			     struct tun_file *tfile, int consumed)
+{
+	struct netdev_queue *txq = netdev_get_tx_queue(tun->dev,
+						tfile->queue_index);
+
+	/* Paired with smp_mb__after_atomic() in tun_net_xmit() */
+	smp_mb();
+	if (netif_tx_queue_stopped(txq)) {
+		tfile->cons_cnt += consumed;
+		if (tfile->cons_cnt >= tfile->tx_ring.size / 2 ||
+		    __ptr_ring_empty(&tfile->tx_ring)) {
+			netif_tx_wake_queue(txq);
+			tfile->cons_cnt = 0;
+		}
+	}
+}
+
+static void *tun_ring_consume(struct tun_struct *tun, struct tun_file *tfile)
+{
+	void *ptr;
+
+	spin_lock(&tfile->tx_ring.consumer_lock);
+	ptr = __ptr_ring_consume(&tfile->tx_ring);
+	if (ptr)
+		__tun_wake_queue(tun, tfile, 1);
+
+	spin_unlock(&tfile->tx_ring.consumer_lock);
+	return ptr;
+}
+
+static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile,
+			   int noblock, int *err)
 {
 	DECLARE_WAITQUEUE(wait, current);
 	void *ptr = NULL;
 	int error = 0;
 
-	ptr = ptr_ring_consume(&tfile->tx_ring);
+	ptr = tun_ring_consume(tun, tfile);
 	if (ptr)
 		goto out;
 	if (noblock) {
@@ -2133,7 +2197,7 @@ static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err)
 
 	while (1) {
 		set_current_state(TASK_INTERRUPTIBLE);
-		ptr = ptr_ring_consume(&tfile->tx_ring);
+		ptr = tun_ring_consume(tun, tfile);
 		if (ptr)
 			break;
 		if (signal_pending(current)) {
@@ -2170,7 +2234,7 @@ static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
 
 	if (!ptr) {
 		/* Read frames from ring */
-		ptr = tun_ring_recv(tfile, noblock, &err);
+		ptr = tun_ring_recv(tun, tfile, noblock, &err);
 		if (!ptr)
 			return err;
 	}
@@ -3622,6 +3686,16 @@ static int tun_queue_resize(struct tun_struct *tun)
 					  dev->tx_queue_len, GFP_KERNEL,
 					  tun_ptr_free);
 
+	if (!ret) {
+		for (i = 0; i < tun->numqueues; i++) {
+			tfile = rtnl_dereference(tun->tfiles[i]);
+			spin_lock(&tfile->tx_ring.consumer_lock);
+			netif_wake_subqueue(tun->dev, tfile->queue_index);
+			tfile->cons_cnt = 0;
+			spin_unlock(&tfile->tx_ring.consumer_lock);
+		}
+	}
+
 	kfree(rings);
 	return ret;
 }
@@ -3730,6 +3804,29 @@ struct ptr_ring *tun_get_tx_ring(struct file *file)
 }
 EXPORT_SYMBOL_GPL(tun_get_tx_ring);
 
+/* Callers must hold ring.consumer_lock */
+void tun_wake_queue(struct file *file, int consumed)
+{
+	struct tun_file *tfile;
+	struct tun_struct *tun;
+
+	if (file->f_op != &tun_fops)
+		return;
+
+	tfile = file->private_data;
+	if (!tfile)
+		return;
+
+	rcu_read_lock();
+
+	tun = rcu_dereference(tfile->tun);
+	if (tun)
+		__tun_wake_queue(tun, tfile, consumed);
+
+	rcu_read_unlock();
+}
+EXPORT_SYMBOL_GPL(tun_wake_queue);
+
 module_init(tun_init);
 module_exit(tun_cleanup);
 MODULE_DESCRIPTION(DRV_DESCRIPTION);
diff --git a/drivers/net/usb/pegasus.c b/drivers/net/usb/pegasus.c
index db85f40734d7b..8700eeb8e22d0 100644
--- a/drivers/net/usb/pegasus.c
+++ b/drivers/net/usb/pegasus.c
@@ -1328,14 +1328,18 @@ static void __init parse_id(char *id)
 	unsigned int vendor_id = 0, device_id = 0, flags = 0, i = 0;
 	char *token, *name = NULL;
 
-	if ((token = strsep(&id, ":")) != NULL)
+	token = strsep(&id, ":");
+	if (token)
 		name = token;
 	/* name now points to a null terminated string*/
-	if ((token = strsep(&id, ":")) != NULL)
-		vendor_id = simple_strtoul(token, NULL, 16);
-	if ((token = strsep(&id, ":")) != NULL)
-		device_id = simple_strtoul(token, NULL, 16);
-	flags = simple_strtoul(id, NULL, 16);
+	token = strsep(&id, ":");
+	if (token && kstrtouint(token, 16, &vendor_id))
+		return;
+	token = strsep(&id, ":");
+	if (token && kstrtouint(token, 16, &device_id))
+		return;
+	if (id && *id && kstrtouint(id, 16, &flags))
+		return;
 	pr_info("%s: new device %s, vendor ID 0x%04x, device ID 0x%04x, flags: 0x%x\n",
 		driver_name, name, vendor_id, device_id, flags);
 
diff --git a/drivers/net/usb/qmi_wwan.c b/drivers/net/usb/qmi_wwan.c
index 05acac10cd2ba..94cdb61dca83d 100644
--- a/drivers/net/usb/qmi_wwan.c
+++ b/drivers/net/usb/qmi_wwan.c
@@ -1452,6 +1452,8 @@ static const struct usb_device_id products[] = {
 	{QMI_QUIRK_SET_DTR(0x1546, 0x1342, 4)},	/* u-blox LARA-L6 */
 	{QMI_QUIRK_SET_DTR(0x33f8, 0x0104, 4)}, /* Rolling RW101 RMNET */
 	{QMI_FIXED_INTF(0x2dee, 0x4d22, 5)},    /* MeiG Smart SRM825L */
+	{QMI_QUIRK_SET_DTR(0x2dee, 0x4d63, 0)}, /* MeiG SRM813Q w/ Modem(PPP) */
+	{QMI_QUIRK_SET_DTR(0x2dee, 0x4d64, 0)}, /* MeiG SRM813Q w/ NMEA */
 
 	/* 4. Gobi 1000 devices */
 	{QMI_GOBI1K_DEVICE(0x05c6, 0x9212)},	/* Acer Gobi Modem Device */
diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c
index 1ace1d2398c9c..8b7ee190fd2a8 100644
--- a/drivers/net/usb/r8152.c
+++ b/drivers/net/usb/r8152.c
@@ -621,6 +621,7 @@ enum spd_duplex {
 	FORCE_1000M_FULL,
 	NWAY_2500M_FULL,
 	NWAY_5000M_FULL,
+	NWAY_10000M_FULL,
 };
 
 /* OCP_ALDPS_CONFIG */
@@ -742,6 +743,7 @@ enum spd_duplex {
 #define BP4_SUPER_ONLY		0x1578	/* RTL_VER_04 only */
 
 enum rtl_register_content {
+	_10000bps	= BIT(14),
 	_5000bps	= BIT(12),
 	_2500bps	= BIT(10),
 	_1250bps	= BIT(9),
@@ -757,6 +759,8 @@ enum rtl_register_content {
 
 #define is_speed_2500(_speed)	(((_speed) & (_2500bps | LINK_STATUS)) == (_2500bps | LINK_STATUS))
 #define is_speed_5000(_speed)	(((_speed) & (_5000bps | LINK_STATUS)) == (_5000bps | LINK_STATUS))
+#define is_speed_10000(_speed)	(((_speed) & (_10000bps | LINK_STATUS)) \
+				 == (_10000bps | LINK_STATUS))
 #define is_flow_control(_speed)	(((_speed) & (_tx_flow | _rx_flow)) == (_tx_flow | _rx_flow))
 
 #define RTL8152_MAX_TX		4
@@ -1008,6 +1012,7 @@ struct r8152 {
 
 	u32 support_2500full:1;
 	u32 support_5000full:1;
+	u32 support_10000full:1;
 	u32 lenovo_macpassthru:1;
 	u32 dell_tb_rx_agg_bug:1;
 	u16 ocp_base;
@@ -1242,6 +1247,7 @@ enum rtl_version {
 	RTL_VER_14,
 	RTL_VER_15,
 	RTL_VER_16,
+	RTL_VER_17,
 
 	RTL_VER_MAX
 };
@@ -1260,6 +1266,7 @@ enum tx_csum_stat {
 #define RTL_ADVERTISED_1000_FULL		BIT(5)
 #define RTL_ADVERTISED_2500_FULL		BIT(6)
 #define RTL_ADVERTISED_5000_FULL		BIT(7)
+#define RTL_ADVERTISED_10000_FULL		BIT(8)
 
 /* Maximum number of multicast addresses to filter (vs. Rx-all-multicast).
  * The RTL chips use a 64 element hash table based on the Ethernet CRC.
@@ -3426,6 +3433,7 @@ static void rtl8152_nic_reset(struct r8152 *tp)
 		break;
 
 	case RTL_VER_16:
+	case RTL_VER_17:
 		ocp_byte_clr_bits(tp, MCU_TYPE_PLA, PLA_CR, CR_RE | CR_TE);
 		break;
 
@@ -3465,6 +3473,9 @@ static void rtl_eee_plus_en(struct r8152 *tp, bool enable)
 
 static void rtl_set_eee_plus(struct r8152 *tp)
 {
+	if (tp->version == RTL_VER_17)
+		return rtl_eee_plus_en(tp, false);
+
 	if (rtl8152_get_speed(tp) & _10bps)
 		rtl_eee_plus_en(tp, true);
 	else
@@ -3650,6 +3661,7 @@ static void r8153_set_rx_early_timeout(struct r8152 *tp)
 	case RTL_VER_13:
 	case RTL_VER_15:
 	case RTL_VER_16:
+	case RTL_VER_17:
 		ocp_write_word(tp, MCU_TYPE_USB, USB_RX_EARLY_TIMEOUT,
 			       640 / 8);
 		ocp_write_word(tp, MCU_TYPE_USB, USB_RX_EXTRA_AGGR_TMR,
@@ -3694,6 +3706,7 @@ static void r8153_set_rx_early_size(struct r8152 *tp)
 			       ocp_data / 8);
 		break;
 	case RTL_VER_16:
+	case RTL_VER_17:
 		ocp_write_word(tp, MCU_TYPE_USB, USB_RX_EARLY_SIZE,
 			       ocp_data / 16);
 		break;
@@ -4542,6 +4555,7 @@ static void rtl_clear_bp(struct r8152 *tp, u16 type)
 		break;
 	case RTL_VER_14:
 	case RTL_VER_16:
+	case RTL_VER_17:
 	default:
 		ocp_write_word(tp, type, USB_BP2_EN, 0);
 		bp_num = 16;
@@ -4649,10 +4663,11 @@ static bool rtl8152_is_fw_phy_speed_up_ok(struct r8152 *tp, struct fw_phy_speed_
 	case RTL_VER_11:
 	case RTL_VER_12:
 	case RTL_VER_14:
-	case RTL_VER_16:
 		goto out;
 	case RTL_VER_13:
 	case RTL_VER_15:
+	case RTL_VER_16:
+	case RTL_VER_17:
 	default:
 		break;
 	}
@@ -5812,6 +5827,7 @@ static void rtl_eee_enable(struct r8152 *tp, bool enable)
 	case RTL_VER_13:
 	case RTL_VER_15:
 	case RTL_VER_16:
+	case RTL_VER_17:
 		if (enable) {
 			r8156_eee_en(tp, true);
 			ocp_reg_write(tp, OCP_EEE_ADV, tp->eee_adv);
@@ -6402,7 +6418,7 @@ static int rtl8156_enable(struct r8152 *tp)
 	set_tx_qlen(tp);
 	rtl_set_eee_plus(tp);
 
-	if (tp->version >= RTL_VER_12 && tp->version <= RTL_VER_16)
+	if (tp->version >= RTL_VER_12 && tp->version <= RTL_VER_17)
 		ocp_word_clr_bits(tp, MCU_TYPE_USB, USB_RX_AGGR_NUM, RX_AGGR_NUM_MASK);
 
 	r8153_set_rx_early_timeout(tp);
@@ -6513,6 +6529,9 @@ static int rtl8152_set_speed(struct r8152 *tp, u8 autoneg, u32 speed, u8 duplex,
 
 			if (tp->support_5000full)
 				support |= RTL_ADVERTISED_5000_FULL;
+
+			if (tp->support_10000full)
+				support |= RTL_ADVERTISED_10000_FULL;
 		}
 
 		advertising &= support;
@@ -6559,9 +6578,10 @@ static int rtl8152_set_speed(struct r8152 *tp, u8 autoneg, u32 speed, u8 duplex,
 				r8152_mdio_write(tp, MII_CTRL1000, new1);
 		}
 
-		if (tp->support_2500full || tp->support_5000full) {
+		if (tp->support_2500full || tp->support_5000full || tp->support_10000full) {
 			orig = ocp_reg_read(tp, OCP_10GBT_CTRL);
-			new1 = orig & ~(MDIO_AN_10GBT_CTRL_ADV2_5G | MDIO_AN_10GBT_CTRL_ADV5G);
+			new1 = orig & ~(MDIO_AN_10GBT_CTRL_ADV2_5G | MDIO_AN_10GBT_CTRL_ADV5G
+					| MDIO_AN_10GBT_CTRL_ADV10G);
 
 			if (advertising & RTL_ADVERTISED_2500_FULL) {
 				new1 |= MDIO_AN_10GBT_CTRL_ADV2_5G;
@@ -6573,6 +6593,11 @@ static int rtl8152_set_speed(struct r8152 *tp, u8 autoneg, u32 speed, u8 duplex,
 				tp->ups_info.speed_duplex = NWAY_5000M_FULL;
 			}
 
+			if (advertising & RTL_ADVERTISED_10000_FULL) {
+				new1 |= MDIO_AN_10GBT_CTRL_ADV10G;
+				tp->ups_info.speed_duplex = NWAY_10000M_FULL;
+			}
+
 			if (orig != new1)
 				ocp_reg_write(tp, OCP_10GBT_CTRL, new1);
 		}
@@ -6802,7 +6827,7 @@ static void rtl8156_up(struct r8152 *tp)
 		return;
 
 	r8153b_u1u2en(tp, false);
-	if (tp->version != RTL_VER_16)
+	if (tp->version < RTL_VER_16)
 		r8153_u2p3en(tp, false);
 	r8153_aldps_en(tp, false);
 
@@ -6816,7 +6841,7 @@ static void rtl8156_up(struct r8152 *tp)
 
 	ocp_byte_clr_bits(tp, MCU_TYPE_PLA, PLA_OOB_CTRL, NOW_IS_OOB);
 
-	if (tp->version == RTL_VER_16)
+	if (tp->version >= RTL_VER_16)
 		ocp_word_clr_bits(tp, MCU_TYPE_PLA, PLA_RCR1, BIT(3));
 
 	ocp_word_clr_bits(tp, MCU_TYPE_PLA, PLA_SFF_STS_7, MCU_BORW_EN);
@@ -6839,12 +6864,9 @@ static void rtl8156_up(struct r8152 *tp)
 	ocp_word_w0w1(tp, MCU_TYPE_PLA, PLA_RXFIFO_FULL, RXFIFO_FULL_MASK,
 		      0x08);
 
-	ocp_word_clr_bits(tp, MCU_TYPE_PLA, PLA_MAC_PWR_CTRL3,
-			  PLA_MCU_SPDWN_EN);
-
 	ocp_word_clr_bits(tp, MCU_TYPE_PLA, PLA_MAC_PWR_CTRL3, PLA_MCU_SPDWN_EN);
 
-	if (tp->version != RTL_VER_16)
+	if (tp->version < RTL_VER_16)
 		ocp_word_clr_bits(tp, MCU_TYPE_USB, USB_SPEED_OPTION,
 				  RG_PWRDN_EN | ALL_SPEED_OFF);
 
@@ -6856,10 +6878,10 @@ static void rtl8156_up(struct r8152 *tp)
 	}
 
 	r8153_aldps_en(tp, true);
-	if (tp->version != RTL_VER_16)
+	if (tp->version < RTL_VER_16)
 		r8153_u2p3en(tp, true);
 
-	if (tp->version != RTL_VER_16 && tp->udev->speed >= USB_SPEED_SUPER)
+	if (tp->version < RTL_VER_16 && tp->udev->speed >= USB_SPEED_SUPER)
 		r8153b_u1u2en(tp, true);
 }
 
@@ -6874,7 +6896,7 @@ static void rtl8156_down(struct r8152 *tp)
 			  PLA_MCU_SPDWN_EN);
 
 	r8153b_u1u2en(tp, false);
-	if (tp->version != RTL_VER_16) {
+	if (tp->version < RTL_VER_16) {
 		r8153_u2p3en(tp, false);
 		r8153b_power_cut_en(tp, false);
 	}
@@ -7952,43 +7974,33 @@ static void r8156b_hw_phy_cfg(struct r8152 *tp)
 
 static void r8157_hw_phy_cfg(struct r8152 *tp)
 {
-	u32 ocp_data;
 	u16 data;
-	int ret;
 
 	r8156b_wait_loading_flash(tp);
 
-	ocp_data = ocp_read_word(tp, MCU_TYPE_USB, USB_MISC_0);
-	if (ocp_data & PCUT_STATUS) {
-		ocp_data &= ~PCUT_STATUS;
-		ocp_write_word(tp, MCU_TYPE_USB, USB_MISC_0, ocp_data);
-	}
+	ocp_word_test_and_clr_bits(tp, MCU_TYPE_USB, USB_MISC_0, PCUT_STATUS);
 
 	data = r8153_phy_status(tp, 0);
 	switch (data) {
 	case PHY_STAT_EXT_INIT:
+		rtl8152_apply_firmware(tp, true);
 		ocp_reg_clr_bits(tp, 0xa466, BIT(0));
 		ocp_reg_clr_bits(tp, 0xa468, BIT(3) | BIT(1));
 		break;
 	case PHY_STAT_LAN_ON:
 	case PHY_STAT_PWRDN:
 	default:
+		rtl8152_apply_firmware(tp, false);
 		break;
 	}
 
-	data = r8152_mdio_read(tp, MII_BMCR);
-	if (data & BMCR_PDOWN) {
-		data &= ~BMCR_PDOWN;
-		r8152_mdio_write(tp, MII_BMCR, data);
-	}
+	r8152_mdio_test_and_clr_bit(tp, MII_BMCR, BMCR_PDOWN);
 
 	r8153_aldps_en(tp, false);
 	rtl_eee_enable(tp, false);
 
-	ret = r8153_phy_status(tp, PHY_STAT_LAN_ON);
-	if (ret < 0)
-		return;
-	WARN_ON_ONCE(ret != PHY_STAT_LAN_ON);
+	data = r8153_phy_status(tp, PHY_STAT_LAN_ON);
+	WARN_ON_ONCE(data != PHY_STAT_LAN_ON);
 
 	/* PFM mode */
 	ocp_word_clr_bits(tp, MCU_TYPE_PLA, PLA_PHY_PWR, PFM_PWM_SWITCH);
@@ -7996,7 +8008,7 @@ static void r8157_hw_phy_cfg(struct r8152 *tp)
 	/* Advanced Power Saving parameter */
 	ocp_reg_set_bits(tp, 0xa430, BIT(0) | BIT(1));
 
-	/* aldpsce force mode */
+	/* Disable ALDPS force mode */
 	ocp_reg_clr_bits(tp, 0xa44a, BIT(2));
 
 	switch (tp->version) {
@@ -8120,6 +8132,190 @@ static void r8157_hw_phy_cfg(struct r8152 *tp)
 		sram2_write_w0w1(tp, 0x807c, 0xff00, 0x5000);
 		sram2_write_w0w1(tp, 0x809d, 0xff00, 0x5000);
 		break;
+
+	case RTL_VER_17:
+		/* Disable bypass turn off clk in ALDPS */
+		ocp_byte_clr_bits(tp, MCU_TYPE_PLA, 0xd3c8, BIT(0));
+
+		/* Power level tuning
+		 * test mode power level
+		 */
+		sram_write_w0w1(tp, 0x8415, 0xff00, 0x9300);
+		/* normal link power level 10G, 5G, 2.5G */
+		sram_write_w0w1(tp, 0x81a3, 0xff00, 0x0f00);
+		sram_write_w0w1(tp, 0x81ae, 0xff00, 0x0f00);
+		sram_write_w0w1(tp, 0x81b9, 0xff00, 0xb900);
+		/* normal link TX filter */
+		sram2_write_w0w1(tp, 0x83b0, 0x0e00, 0);
+		sram2_write_w0w1(tp, 0x83c5, 0x0e00, 0);
+		sram2_write_w0w1(tp, 0x83da, 0x0e00, 0);
+		sram2_write_w0w1(tp, 0x83ef, 0x0e00, 0);
+
+		/* AFE power saving for 2.5G & 5G */
+		sram_write(tp, 0x8173, 0x8620);
+		sram_write(tp, 0x8175, 0x8671);
+
+		sram_write_w0w1(tp, 0x817c, 0, BIT(13));
+		sram_write_w0w1(tp, 0x8187, 0, BIT(13));
+		sram_write_w0w1(tp, 0x8192, 0, BIT(13));
+		sram_write_w0w1(tp, 0x819d, 0, BIT(13));
+		sram_write_w0w1(tp, 0x81a8, BIT(13), 0);
+		sram_write_w0w1(tp, 0x81b3, BIT(13), 0);
+		sram_write_w0w1(tp, 0x81be, 0, BIT(13));
+
+		sram_write_w0w1(tp, 0x817d, 0xff00, 0xa600);
+		sram_write_w0w1(tp, 0x8188, 0xff00, 0xa600);
+		sram_write_w0w1(tp, 0x8193, 0xff00, 0xa600);
+		sram_write_w0w1(tp, 0x819e, 0xff00, 0xa600);
+		sram_write_w0w1(tp, 0x81a9, 0xff00, 0x1400);
+		sram_write_w0w1(tp, 0x81b4, 0xff00, 0x1400);
+		sram_write_w0w1(tp, 0x81bf, 0xff00, 0xa600);
+
+		/* RFI parameter
+		 * disable preset FBE
+		 */
+		ocp_reg_clr_bits(tp, 0xaeaa, BIT(5) | BIT(3));
+		/* modify PGA for 5G&10G */
+		sram2_write(tp, 0x84f0, 0x201c);
+		sram2_write(tp, 0x84f2, 0x3117);
+		/* RFI parameter */
+		ocp_reg_write(tp, 0xaec6, 0x0000);
+		ocp_reg_write(tp, 0xae20, 0xffff);
+		ocp_reg_write(tp, 0xaece, 0xffff);
+		ocp_reg_write(tp, 0xaed2, 0xffff);
+		ocp_reg_write(tp, 0xaec8, 0x0000);
+		ocp_reg_clr_bits(tp, 0xaed0, BIT(0));
+		ocp_reg_write(tp, 0xadb8, 0x0150);
+		sram2_write_w0w1(tp, 0x8197, 0xff00, 0x5000);
+		sram2_write_w0w1(tp, 0x8231, 0xff00, 0x5000);
+		sram2_write_w0w1(tp, 0x82cb, 0xff00, 0x5000);
+		sram2_write_w0w1(tp, 0x82cd, 0xff00, 0x5700);
+		sram2_write_w0w1(tp, 0x8233, 0xff00, 0x5700);
+		sram2_write_w0w1(tp, 0x8199, 0xff00, 0x5700);
+
+		sram2_write(tp, 0x815a, 0x0150);
+		sram2_write(tp, 0x81f4, 0x0150);
+		sram2_write(tp, 0x828e, 0x0150);
+		sram2_write(tp, 0x81b1, 0x0000);
+		sram2_write(tp, 0x824b, 0x0000);
+		sram2_write(tp, 0x82e5, 0x0000);
+
+		sram2_write_w0w1(tp, 0x84f7, 0xff00, 0x2800);
+		ocp_reg_set_bits(tp, 0xaec2, BIT(12));
+		sram2_write_w0w1(tp, 0x81b3, 0xff00, 0xad00);
+		sram2_write_w0w1(tp, 0x824d, 0xff00, 0xad00);
+		sram2_write_w0w1(tp, 0x82e7, 0xff00, 0xad00);
+		ocp_reg_w0w1(tp, 0xae4e, 0x000f, 0x0001);
+		sram2_write_w0w1(tp, 0x82ce, 0xf000, 0x4000);
+
+		/* 5G shift sel, default = '04'
+		 * 10G shift sel, default = '03'
+		 */
+		sram2_write_w0w1(tp, 0x83a5, 0xff00, 0x0400);
+		sram2_write_w0w1(tp, 0x83a6, 0xff00, 0x0400);
+		sram2_write_w0w1(tp, 0x83a7, 0xff00, 0x0400);
+		sram2_write_w0w1(tp, 0x83a8, 0xff00, 0x0400);
+
+		/* XG INRX parameters
+		 * RC coefficients
+		 */
+		sram2_write(tp, 0x84ac, 0x0000);
+		sram2_write(tp, 0x84ae, 0x0000);
+		sram2_write(tp, 0x84b0, 0xf818);
+		sram2_write_w0w1(tp, 0x84b2, 0xff00, 0x6000);
+		/* Training AAGC PAR (with uc2 patch) */
+		sram2_write(tp, 0x8ffc, 0x6008);
+		sram2_write(tp, 0x8ffe, 0xf450);
+		/* DAC BGK */
+		sram2_write_w0w1(tp, 0x8015, 0, BIT(9));
+		sram2_write_w0w1(tp, 0x8016, 0, BIT(11));
+		sram2_write_w0w1(tp, 0x8fe6, 0xff00, 0x0800);
+		sram2_write(tp, 0x8fe4, 0x2114);
+		/* 10G PBO table */
+		sram2_write(tp, 0x8647, 0xa7b1);
+		sram2_write(tp, 0x8649, 0xbbca);
+		sram2_write_w0w1(tp, 0x864b, 0xff00, 0xdc00);
+		/* 2.5G ado power window size */
+		sram2_write_w0w1(tp, 0x8154, 0xc000, 0x4000);
+		sram2_write_w0w1(tp, 0x8158, 0xc000, 0);
+		/* 10G lock far */
+		sram2_write(tp, 0x826c, 0xffff);
+		sram2_write(tp, 0x826e, 0xffff);
+		/* XG INRX parameter */
+		sram2_write_w0w1(tp, 0x8872, 0xff00, 0x0e00);
+		sram_write_w0w1(tp, 0x8012, 0, BIT(11));
+		sram_write_w0w1(tp, 0x8012, 0, BIT(14));
+		ocp_reg_set_bits(tp, 0xb576, BIT(0));
+		sram_write_w0w1(tp, 0x834a, 0xff00, 0x0700);
+		sram2_write_w0w1(tp, 0x8217, 0x3f00, 0x2a00);
+		sram_write_w0w1(tp, 0x81b1, 0xff00, 0x0b00);
+		sram2_write_w0w1(tp, 0x8fed, 0xff00, 0x4e00);
+		/* Slave about EC mu of datamode AAGC  and DAC BG */
+		sram2_write_w0w1(tp, 0x88ac, 0xff00, 0x2300);
+		/* improve UBE */
+		ocp_reg_set_bits(tp, 0xbf0c, 0x7 << 11);
+		/* close Sparse NEC, improve connect 5EUU cable performance */
+		sram2_write_w0w1(tp, 0x88de, 0xff00, 0);
+		/* 5G slave compatibility issue */
+		sram2_write(tp, 0x80b4, 0x5195);
+
+		/* XG Test Mode
+		 * xgtstm_map_tbl for mdi_cap_sel
+		 */
+		sram_write(tp, 0x8370, 0x8671);
+		sram_write(tp, 0x8372, 0x86c8);
+		/* xgtstm_amp_map_tbl for  REG_IBX_UP_SHIFT_L */
+		sram_write(tp, 0x8401, 0x86c8);
+		sram_write(tp, 0x8403, 0x86da);
+		sram_write_w0w1(tp, 0x8406, 0x1800, 0x1000);
+		sram_write_w0w1(tp, 0x8408, 0x1800, 0x1000);
+		sram_write_w0w1(tp, 0x840a, 0x1800, 0x1000);
+		sram_write_w0w1(tp, 0x840c, 0x1800, 0x1000);
+		sram_write_w0w1(tp, 0x840e, 0x1800, 0x1000);
+		sram_write_w0w1(tp, 0x8410, 0x1800, 0x1000);
+		sram_write_w0w1(tp, 0x8412, 0x1800, 0x1000);
+		sram_write_w0w1(tp, 0x8414, 0x1800, 0x1000);
+		sram_write_w0w1(tp, 0x8416, 0x1800, 0x1000);
+
+		/* Cable Test Patch */
+		sram_write(tp, 0x82bd, 0x1f40);
+
+		/* Thermal sensor parameters */
+		ocp_reg_w0w1(tp, 0xbfb4, 0x07ff, 0x0328);
+		ocp_reg_write(tp, 0xbfb6, 0x3e14);
+
+		/* spdchg_gtx_shape_100M */
+		ocp_reg_write(tp, OCP_SRAM_ADDR, 0x81c4);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x003b);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x0086);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x00b7);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x00db);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x00fe);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x00fe);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x00fe);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x00fe);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x00c3);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x0078);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x0047);
+		ocp_reg_write(tp, OCP_SRAM_DATA, 0x0023);
+
+		/* lsbmsk_parameters
+		 * RL6961_lsbmsk_parameter_250207
+		 */
+		sram2_write(tp, 0x88d7, 0x01a0);
+		sram2_write(tp, 0x88d9, 0x01a0);
+		sram2_write(tp, 0x8ffa, 0x002a);
+
+		sram2_write(tp, 0x8fee, 0xffdf);
+		sram2_write(tp, 0x8ff0, 0xffff);
+		sram2_write(tp, 0x8ff2, 0x0a4a);
+		sram2_write(tp, 0x8ff4, 0xaa5a);
+		sram2_write(tp, 0x8ff6, 0x0a4a);
+		sram2_write(tp, 0x8ff8, 0xaa5a);
+
+		sram2_write_w0w1(tp, 0x88d5, 0xff00, 0x0200);
+		break;
+
 	default:
 		break;
 	}
@@ -8155,6 +8351,18 @@ static void r8157_hw_phy_cfg(struct r8152 *tp)
 	set_bit(PHY_RESET, &tp->flags);
 }
 
+static int r8159_wait_backup_restore(struct r8152 *tp)
+{
+	u32 ocp_data;
+
+	ocp_data = ocp_read_word(tp, MCU_TYPE_USB, USB_MISC_0);
+	if (!(ocp_data & PCUT_STATUS))
+		return 0;
+
+	return poll_timeout_us(ocp_data = ocp_read_word(tp, MCU_TYPE_USB, USB_GPHY_CTRL),
+			       ocp_data & BACKUP_RESTRORE, 200, 20000, false);
+}
+
 static void r8156_init(struct r8152 *tp)
 {
 	u32 ocp_data;
@@ -8164,14 +8372,14 @@ static void r8156_init(struct r8152 *tp)
 	if (test_bit(RTL8152_INACCESSIBLE, &tp->flags))
 		return;
 
-	if (tp->version == RTL_VER_16) {
+	if (tp->version == RTL_VER_16 || tp->version == RTL_VER_17) {
 		ocp_byte_set_bits(tp, MCU_TYPE_USB, 0xcffe, BIT(3));
 		ocp_byte_clr_bits(tp, MCU_TYPE_USB, 0xd3ca, BIT(0));
 	}
 
 	ocp_byte_clr_bits(tp, MCU_TYPE_USB, USB_ECM_OP, EN_ALL_SPEED);
 
-	if (tp->version != RTL_VER_16)
+	if (tp->version < RTL_VER_16)
 		ocp_write_word(tp, MCU_TYPE_USB, USB_SPEED_OPTION, 0);
 
 	ocp_word_set_bits(tp, MCU_TYPE_USB, USB_ECM_OPTION, BYPASS_MAC_RESET);
@@ -8185,6 +8393,7 @@ static void r8156_init(struct r8152 *tp)
 	case RTL_VER_13:
 	case RTL_VER_15:
 	case RTL_VER_16:
+	case RTL_VER_17:
 		r8156b_wait_loading_flash(tp);
 		break;
 	default:
@@ -8201,6 +8410,12 @@ static void r8156_init(struct r8152 *tp)
 			return;
 	}
 
+	if (tp->version == RTL_VER_17 && r8159_wait_backup_restore(tp)) {
+		rtl_set_inaccessible(tp);
+		dev_err(&tp->intf->dev, "init failed, backup-restore timed out\n");
+		return;
+	}
+
 	data = r8153_phy_status(tp, 0);
 	if (data == PHY_STAT_EXT_INIT) {
 		ocp_reg_clr_bits(tp, 0xa468, BIT(3) | BIT(1));
@@ -8216,7 +8431,7 @@ static void r8156_init(struct r8152 *tp)
 
 	data = r8153_phy_status(tp, PHY_STAT_LAN_ON);
 
-	if (tp->version == RTL_VER_16)
+	if (tp->version >= RTL_VER_16)
 		r8157_u2p3en(tp, false);
 	else
 		r8153_u2p3en(tp, false);
@@ -8227,7 +8442,7 @@ static void r8156_init(struct r8152 *tp)
 	/* U1/U2/L1 idle timer. 500 us */
 	ocp_write_word(tp, MCU_TYPE_USB, USB_U1U2_TIMER, 500);
 
-	if (tp->version == RTL_VER_16)
+	if (tp->version >= RTL_VER_16)
 		r8157_power_cut_en(tp, false);
 	else
 		r8153b_power_cut_en(tp, false);
@@ -8260,7 +8475,7 @@ static void r8156_init(struct r8152 *tp)
 
 	r8156_mac_clk_spd(tp, true);
 
-	if (tp->version != RTL_VER_16)
+	if (tp->version < RTL_VER_16)
 		ocp_word_clr_bits(tp, MCU_TYPE_PLA, PLA_MAC_PWR_CTRL3, PLA_MCU_SPDWN_EN);
 
 	ocp_data = ocp_read_word(tp, MCU_TYPE_PLA, PLA_EXTRA_STATUS);
@@ -8273,8 +8488,13 @@ static void r8156_init(struct r8152 *tp)
 
 	set_bit(GREEN_ETHERNET, &tp->flags);
 
-	/* rx aggregation / 16 bytes Rx descriptor */
-	if (tp->version == RTL_VER_16)
+	/* RX aggregation / 16 bytes RX descriptor
+	 * BIT(11) is specific to RTL8159, with unknown meaning
+	 */
+	if (tp->version == RTL_VER_17)
+		ocp_word_clr_bits(tp, MCU_TYPE_USB, USB_USB_CTRL,
+				  RX_AGG_DISABLE | RX_DESC_16B | BIT(11));
+	else if (tp->version == RTL_VER_16)
 		ocp_word_clr_bits(tp, MCU_TYPE_USB, USB_USB_CTRL, RX_AGG_DISABLE | RX_DESC_16B);
 	else
 		ocp_word_clr_bits(tp, MCU_TYPE_USB, USB_USB_CTRL, RX_AGG_DISABLE | RX_ZERO_EN);
@@ -8282,7 +8502,7 @@ static void r8156_init(struct r8152 *tp)
 	if (tp->version < RTL_VER_12)
 		ocp_byte_set_bits(tp, MCU_TYPE_USB, USB_BMU_CONFIG, ACT_ODMA);
 
-	if (tp->version == RTL_VER_16) {
+	if (tp->version >= RTL_VER_16) {
 		/* Disable Rx Zero Len */
 		rtl_bmu_clr_bits(tp, 0x2300, BIT(3));
 		/* TX descriptor Signature */
@@ -8723,7 +8943,10 @@ int rtl8152_get_link_ksettings(struct net_device *netdev,
 	linkmode_mod_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT,
 			 cmd->link_modes.supported, tp->support_5000full);
 
-	if (tp->support_2500full || tp->support_5000full) {
+	linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+			 cmd->link_modes.supported, tp->support_10000full);
+
+	if (tp->support_2500full || tp->support_5000full || tp->support_10000full) {
 		u16 ocp_10gbt_ctrl = ocp_reg_read(tp, OCP_10GBT_CTRL);
 		u16 ocp_10gbt_stat = ocp_reg_read(tp, OCP_10GBT_STAT);
 
@@ -8752,6 +8975,19 @@ int rtl8152_get_link_ksettings(struct net_device *netdev,
 			if (is_speed_5000(rtl8152_get_speed(tp)))
 				cmd->base.speed = SPEED_5000;
 		}
+
+		if (tp->support_10000full) {
+			linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+					 cmd->link_modes.advertising,
+					 ocp_10gbt_ctrl & MDIO_AN_10GBT_CTRL_ADV10G);
+
+			linkmode_mod_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+					 cmd->link_modes.lp_advertising,
+					 ocp_10gbt_stat & MDIO_AN_10GBT_STAT_LP10G);
+
+			if (is_speed_10000(rtl8152_get_speed(tp)))
+				cmd->base.speed = SPEED_10000;
+		}
 	}
 
 	mutex_unlock(&tp->control);
@@ -8805,6 +9041,10 @@ static int rtl8152_set_link_ksettings(struct net_device *dev,
 		     cmd->link_modes.advertising))
 		advertising |= RTL_ADVERTISED_5000_FULL;
 
+	if (test_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT,
+		     cmd->link_modes.advertising))
+		advertising |= RTL_ADVERTISED_10000_FULL;
+
 	mutex_lock(&tp->control);
 
 	ret = rtl8152_set_speed(tp, cmd->base.autoneg, cmd->base.speed,
@@ -8968,6 +9208,13 @@ static int r8153_get_eee(struct r8152 *tp, struct ethtool_keee *eee)
 			linkmode_set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, common);
 	}
 
+	if (tp->support_10000full) {
+		linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, eee->supported);
+
+		if (speed & _10000bps)
+			linkmode_set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, common);
+	}
+
 	eee->eee_enabled = tp->eee_en;
 
 	if (speed & _1000bps)
@@ -9643,6 +9890,29 @@ static int rtl_ops_init(struct r8152 *tp)
 		r8157_desc_init(tp);
 		break;
 
+	case RTL_VER_17:
+		tp->eee_en		= true;
+		tp->eee_adv		= MDIO_EEE_100TX | MDIO_EEE_1000T | MDIO_EEE_10GT;
+		tp->eee_adv2		= MDIO_EEE_2_5GT | MDIO_EEE_5GT;
+		ops->init		= r8156_init;
+		ops->enable		= rtl8156_enable;
+		ops->disable		= rtl8153_disable;
+		ops->up			= rtl8156_up;
+		ops->down		= rtl8156_down;
+		ops->unload		= rtl8153_unload;
+		ops->eee_get		= r8153_get_eee;
+		ops->eee_set		= r8152_set_eee;
+		ops->in_nway		= rtl8153_in_nway;
+		ops->hw_phy_cfg		= r8157_hw_phy_cfg;
+		ops->autosuspend_en	= rtl8157_runtime_enable;
+		ops->change_mtu		= rtl8156_change_mtu;
+		tp->rx_buf_sz		= 48 * 1024;
+		tp->support_2500full	= 1;
+		tp->support_5000full	= 1;
+		tp->support_10000full	= 1;
+		r8157_desc_init(tp);
+		break;
+
 	default:
 		ret = -ENODEV;
 		dev_err(&tp->intf->dev, "Unknown Device\n");
@@ -9659,6 +9929,8 @@ static int rtl_ops_init(struct r8152 *tp)
 #define FIRMWARE_8153C_1	"rtl_nic/rtl8153c-1.fw"
 #define FIRMWARE_8156A_2	"rtl_nic/rtl8156a-2.fw"
 #define FIRMWARE_8156B_2	"rtl_nic/rtl8156b-2.fw"
+#define FIRMWARE_8157_1		"rtl_nic/rtl8157-1.fw"
+#define FIRMWARE_8159_1		"rtl_nic/rtl8159-1.fw"
 
 MODULE_FIRMWARE(FIRMWARE_8153A_2);
 MODULE_FIRMWARE(FIRMWARE_8153A_3);
@@ -9667,6 +9939,8 @@ MODULE_FIRMWARE(FIRMWARE_8153B_2);
 MODULE_FIRMWARE(FIRMWARE_8153C_1);
 MODULE_FIRMWARE(FIRMWARE_8156A_2);
 MODULE_FIRMWARE(FIRMWARE_8156B_2);
+MODULE_FIRMWARE(FIRMWARE_8157_1);
+MODULE_FIRMWARE(FIRMWARE_8159_1);
 
 static int rtl_fw_init(struct r8152 *tp)
 {
@@ -9705,6 +9979,12 @@ static int rtl_fw_init(struct r8152 *tp)
 		rtl_fw->pre_fw		= r8153b_pre_firmware_1;
 		rtl_fw->post_fw		= r8153c_post_firmware_1;
 		break;
+	case RTL_VER_16:
+		rtl_fw->fw_name		= FIRMWARE_8157_1;
+		break;
+	case RTL_VER_17:
+		rtl_fw->fw_name		= FIRMWARE_8159_1;
+		break;
 	default:
 		break;
 	}
@@ -9796,6 +10076,9 @@ static u8 __rtl_get_hw_ver(struct usb_device *udev)
 	case 0x1030:
 		version = RTL_VER_16;
 		break;
+	case 0x2020:
+		version = RTL_VER_17;
+		break;
 	default:
 		version = RTL_VER_UNKNOWN;
 		dev_info(&udev->dev, "Unknown version 0x%04x\n", ocp_data);
@@ -9948,6 +10231,7 @@ static int rtl8152_probe_once(struct usb_interface *intf,
 	case RTL_VER_13:
 	case RTL_VER_15:
 	case RTL_VER_16:
+	case RTL_VER_17:
 		netdev->max_mtu = size_to_mtu(16 * 1024);
 		break;
 	case RTL_VER_01:
@@ -9982,6 +10266,11 @@ static int rtl8152_probe_once(struct usb_interface *intf,
 			tp->speed = SPEED_5000;
 			tp->advertising |= RTL_ADVERTISED_5000_FULL;
 		}
+		if (tp->support_10000full &&
+		    tp->udev->speed >= USB_SPEED_SUPER) {
+			tp->speed = SPEED_10000;
+			tp->advertising |= RTL_ADVERTISED_10000_FULL;
+		}
 		tp->advertising |= RTL_ADVERTISED_1000_FULL;
 	}
 	tp->duplex = DUPLEX_FULL;
@@ -10108,6 +10397,7 @@ static const struct usb_device_id rtl8152_table[] = {
 	{ USB_DEVICE(VENDOR_ID_REALTEK, 0x8155) },
 	{ USB_DEVICE(VENDOR_ID_REALTEK, 0x8156) },
 	{ USB_DEVICE(VENDOR_ID_REALTEK, 0x8157) },
+	{ USB_DEVICE(VENDOR_ID_REALTEK, 0x815a) },
 
 	/* Microsoft */
 	{ USB_DEVICE(VENDOR_ID_MICROSOFT, 0x07ab) },
diff --git a/drivers/net/usb/usbnet.c b/drivers/net/usb/usbnet.c
index b72ba0803392b..25518635b7b72 100644
--- a/drivers/net/usb/usbnet.c
+++ b/drivers/net/usb/usbnet.c
@@ -149,10 +149,8 @@ int usbnet_get_endpoints(struct usbnet *dev, struct usb_interface *intf)
 			return tmp;
 	}
 
-	dev->in = usb_rcvbulkpipe(dev->udev,
-				  in->desc.bEndpointAddress & USB_ENDPOINT_NUMBER_MASK);
-	dev->out = usb_sndbulkpipe(dev->udev,
-				   out->desc.bEndpointAddress & USB_ENDPOINT_NUMBER_MASK);
+	dev->in = usb_rcvbulkpipe(dev->udev, usb_endpoint_num(&in->desc));
+	dev->out = usb_sndbulkpipe(dev->udev, usb_endpoint_num(&out->desc));
 	dev->status = status;
 	return 0;
 }
@@ -232,9 +230,7 @@ static int init_status(struct usbnet *dev, struct usb_interface *intf)
 	if (!dev->driver_info->status)
 		return 0;
 
-	pipe = usb_rcvintpipe(dev->udev,
-			      dev->status->desc.bEndpointAddress
-			      & USB_ENDPOINT_NUMBER_MASK);
+	pipe = usb_rcvintpipe(dev->udev, usb_endpoint_num(&dev->status->desc));
 	maxp = usb_maxpacket(dev->udev, pipe);
 
 	/* avoid 1 msec chatter:  min 8 msec poll rate */
diff --git a/drivers/net/vxlan/vxlan_core.c b/drivers/net/vxlan/vxlan_core.c
index e88798497503b..8c3885665b582 100644
--- a/drivers/net/vxlan/vxlan_core.c
+++ b/drivers/net/vxlan/vxlan_core.c
@@ -88,10 +88,10 @@ static struct vxlan_sock *vxlan_find_sock(struct net *net, sa_family_t family,
 	flags &= VXLAN_F_RCV_FLAGS;
 
 	hlist_for_each_entry_rcu(vs, vs_head(net, port), hlist) {
-		if (inet_sk(vs->sock->sk)->inet_sport == port &&
+		if (inet_sk(vs->sk)->inet_sport == port &&
 		    vxlan_get_sk_family(vs) == family &&
 		    vs->flags == flags &&
-		    vs->sock->sk->sk_bound_dev_if == ifindex)
+		    vs->sk->sk_bound_dev_if == ifindex)
 			return vs;
 	}
 	return NULL;
@@ -657,14 +657,18 @@ static struct vxlanhdr *vxlan_gro_prepare_receive(struct sock *sk,
 						  struct sk_buff *skb,
 						  struct gro_remcsum *grc)
 {
-	struct sk_buff *p;
 	struct vxlanhdr *vh, *vh2;
 	unsigned int hlen, off_vx;
-	struct vxlan_sock *vs = rcu_dereference_sk_user_data(sk);
+	struct vxlan_sock *vs;
+	struct sk_buff *p;
 	__be32 flags;
 
 	skb_gro_remcsum_init(grc);
 
+	vs = rcu_dereference_sk_user_data(sk);
+	if (!vs)
+		return NULL;
+
 	off_vx = skb_gro_offset(skb);
 	hlen = off_vx + sizeof(*vh);
 	vh = skb_gro_header(skb, hlen, off_vx);
@@ -1493,7 +1497,7 @@ static bool __vxlan_sock_release_prep(struct vxlan_sock *vs)
 		return false;
 
 	hlist_del_rcu(&vs->hlist);
-	udp_tunnel_notify_del_rx_port(vs->sock,
+	udp_tunnel_notify_del_rx_port(vs->sk,
 				      (vs->flags & VXLAN_F_GPE) ?
 				      UDP_TUNNEL_TYPE_VXLAN_GPE :
 				      UDP_TUNNEL_TYPE_VXLAN);
@@ -1511,7 +1515,6 @@ static void vxlan_sock_release(struct vxlan_dev *vxlan)
 #endif
 
 	RCU_INIT_POINTER(vxlan->vn4_sock, NULL);
-	synchronize_net();
 
 	if (vxlan->cfg.flags & VXLAN_F_VNIFILTER)
 		vxlan_vs_del_vnigrp(vxlan);
@@ -1519,14 +1522,14 @@ static void vxlan_sock_release(struct vxlan_dev *vxlan)
 		vxlan_vs_del_dev(vxlan);
 
 	if (__vxlan_sock_release_prep(sock4)) {
-		udp_tunnel_sock_release(sock4->sock);
-		kfree(sock4);
+		udp_tunnel_sock_release(sock4->sk);
+		kfree_rcu(sock4, rcu);
 	}
 
 #if IS_ENABLED(CONFIG_IPV6)
 	if (__vxlan_sock_release_prep(sock6)) {
-		udp_tunnel_sock_release(sock6->sock);
-		kfree(sock6);
+		udp_tunnel_sock_release(sock6->sk);
+		kfree_rcu(sock6, rcu);
 	}
 #endif
 }
@@ -2473,7 +2476,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		}
 
 		if (!ifindex)
-			ifindex = sock4->sock->sk->sk_bound_dev_if;
+			ifindex = sock4->sk->sk_bound_dev_if;
 
 		rt = udp_tunnel_dst_lookup(skb, dev, vxlan->net, ifindex,
 					   &saddr, pkey, src_port, dst_port,
@@ -2540,7 +2543,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 			goto tx_error;
 		}
 
-		udp_tunnel_xmit_skb(rt, sock4->sock->sk, skb, saddr,
+		udp_tunnel_xmit_skb(rt, sock4->sk, skb, saddr,
 				    pkey->u.ipv4.dst, tos, ttl, df,
 				    src_port, dst_port, xnet, !udp_sum,
 				    ipcb_flags);
@@ -2557,9 +2560,9 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 		}
 
 		if (!ifindex)
-			ifindex = sock6->sock->sk->sk_bound_dev_if;
+			ifindex = sock6->sk->sk_bound_dev_if;
 
-		ndst = udp_tunnel6_dst_lookup(skb, dev, vxlan->net, sock6->sock,
+		ndst = udp_tunnel6_dst_lookup(skb, dev, vxlan->net, sock6->sk,
 					      ifindex, &saddr, pkey,
 					      src_port, dst_port, tos,
 					      use_cache ? dst_cache : NULL);
@@ -2615,7 +2618,7 @@ void vxlan_xmit_one(struct sk_buff *skb, struct net_device *dev,
 			goto tx_error;
 		}
 
-		udp_tunnel6_xmit_skb(ndst, sock6->sock->sk, skb, dev,
+		udp_tunnel6_xmit_skb(ndst, sock6->sk, skb, dev,
 				     &saddr, &pkey->u.ipv6.dst, tos, ttl,
 				     pkey->label, src_port, dst_port, !udp_sum,
 				     ip6cb_flags);
@@ -3254,7 +3257,7 @@ static int vxlan_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
 		if (!sock6)
 			return -EIO;
 
-		ndst = udp_tunnel6_dst_lookup(skb, dev, vxlan->net, sock6->sock,
+		ndst = udp_tunnel6_dst_lookup(skb, dev, vxlan->net, sock6->sk,
 					      0, &info->key.u.ipv6.src,
 					      &info->key,
 					      sport, dport, info->key.tos,
@@ -3332,9 +3335,9 @@ static void vxlan_offload_rx_ports(struct net_device *dev, bool push)
 				type = UDP_TUNNEL_TYPE_VXLAN;
 
 			if (push)
-				udp_tunnel_push_rx_port(dev, vs->sock, type);
+				udp_tunnel_push_rx_port(dev, vs->sk, type);
 			else
-				udp_tunnel_drop_rx_port(dev, vs->sock, type);
+				udp_tunnel_drop_rx_port(dev, vs->sk, type);
 		}
 	}
 }
@@ -3540,8 +3543,8 @@ static const struct ethtool_ops vxlan_ethtool_ops = {
 	.get_link_ksettings	= vxlan_get_link_ksettings,
 };
 
-static struct socket *vxlan_create_sock(struct net *net, bool ipv6,
-					__be16 port, u32 flags, int ifindex)
+static struct sock *vxlan_create_sock(struct net *net, bool ipv6,
+				      __be16 port, u32 flags, int ifindex)
 {
 	struct socket *sock;
 	struct udp_port_cfg udp_conf;
@@ -3567,7 +3570,7 @@ static struct socket *vxlan_create_sock(struct net *net, bool ipv6,
 		return ERR_PTR(err);
 
 	udp_allow_gso(sock->sk);
-	return sock;
+	return sock->sk;
 }
 
 /* Create new listen socket if needed */
@@ -3575,10 +3578,10 @@ static struct vxlan_sock *vxlan_socket_create(struct net *net, bool ipv6,
 					      __be16 port, u32 flags,
 					      int ifindex)
 {
+	struct udp_tunnel_sock_cfg tunnel_cfg;
 	struct vxlan_sock *vs;
-	struct socket *sock;
+	struct sock *sk;
 	unsigned int h;
-	struct udp_tunnel_sock_cfg tunnel_cfg;
 
 	ASSERT_RTNL();
 
@@ -3589,18 +3592,18 @@ static struct vxlan_sock *vxlan_socket_create(struct net *net, bool ipv6,
 	for (h = 0; h < VNI_HASH_SIZE; ++h)
 		INIT_HLIST_HEAD(&vs->vni_list[h]);
 
-	sock = vxlan_create_sock(net, ipv6, port, flags, ifindex);
-	if (IS_ERR(sock)) {
+	sk = vxlan_create_sock(net, ipv6, port, flags, ifindex);
+	if (IS_ERR(sk)) {
 		kfree(vs);
-		return ERR_CAST(sock);
+		return ERR_CAST(sk);
 	}
 
-	vs->sock = sock;
+	vs->sk = sk;
 	refcount_set(&vs->refcnt, 1);
 	vs->flags = (flags & VXLAN_F_RCV_FLAGS);
 
 	hlist_add_head_rcu(&vs->hlist, vs_head(net, port));
-	udp_tunnel_notify_add_rx_port(sock,
+	udp_tunnel_notify_add_rx_port(sk,
 				      (vs->flags & VXLAN_F_GPE) ?
 				      UDP_TUNNEL_TYPE_VXLAN_GPE :
 				      UDP_TUNNEL_TYPE_VXLAN);
@@ -3620,7 +3623,7 @@ static struct vxlan_sock *vxlan_socket_create(struct net *net, bool ipv6,
 		tunnel_cfg.gro_complete = vxlan_gro_complete;
 	}
 
-	setup_udp_tunnel_sock(net, sock, &tunnel_cfg);
+	setup_udp_tunnel_sock(net, sk, &tunnel_cfg);
 
 	return vs;
 }
diff --git a/drivers/net/vxlan/vxlan_multicast.c b/drivers/net/vxlan/vxlan_multicast.c
index b0e80bca855cc..3b75b48dc726d 100644
--- a/drivers/net/vxlan/vxlan_multicast.c
+++ b/drivers/net/vxlan/vxlan_multicast.c
@@ -29,7 +29,7 @@ int vxlan_igmp_join(struct vxlan_dev *vxlan, union vxlan_addr *rip,
 			.imr_ifindex		= ifindex,
 		};
 
-		sk = sock4->sock->sk;
+		sk = sock4->sk;
 		lock_sock(sk);
 		ret = ip_mc_join_group(sk, &mreq);
 		release_sock(sk);
@@ -37,7 +37,7 @@ int vxlan_igmp_join(struct vxlan_dev *vxlan, union vxlan_addr *rip,
 	} else {
 		struct vxlan_sock *sock6 = rtnl_dereference(vxlan->vn6_sock);
 
-		sk = sock6->sock->sk;
+		sk = sock6->sk;
 		lock_sock(sk);
 		ret = ipv6_sock_mc_join(sk, ifindex, &ip->sin6.sin6_addr);
 		release_sock(sk);
@@ -62,7 +62,7 @@ int vxlan_igmp_leave(struct vxlan_dev *vxlan, union vxlan_addr *rip,
 			.imr_ifindex		= ifindex,
 		};
 
-		sk = sock4->sock->sk;
+		sk = sock4->sk;
 		lock_sock(sk);
 		ret = ip_mc_leave_group(sk, &mreq);
 		release_sock(sk);
@@ -70,7 +70,7 @@ int vxlan_igmp_leave(struct vxlan_dev *vxlan, union vxlan_addr *rip,
 	} else {
 		struct vxlan_sock *sock6 = rtnl_dereference(vxlan->vn6_sock);
 
-		sk = sock6->sock->sk;
+		sk = sock6->sk;
 		lock_sock(sk);
 		ret = ipv6_sock_mc_drop(sk, ifindex, &ip->sin6.sin6_addr);
 		release_sock(sk);
diff --git a/drivers/net/wan/farsync.c b/drivers/net/wan/farsync.c
index 73f3939007106..6508c53bdc24f 100644
--- a/drivers/net/wan/farsync.c
+++ b/drivers/net/wan/farsync.c
@@ -524,27 +524,21 @@ do {								\
 /*      PCI ID lookup table
  */
 static const struct pci_device_id fst_pci_dev_id[] = {
-	{PCI_VENDOR_ID_FARSITE, PCI_DEVICE_ID_FARSITE_T2P, PCI_ANY_ID,
-	 PCI_ANY_ID, 0, 0, FST_TYPE_T2P},
+	{ PCI_VDEVICE(FARSITE, PCI_DEVICE_ID_FARSITE_T2P), .driver_data = FST_TYPE_T2P },
 
-	{PCI_VENDOR_ID_FARSITE, PCI_DEVICE_ID_FARSITE_T4P, PCI_ANY_ID,
-	 PCI_ANY_ID, 0, 0, FST_TYPE_T4P},
+	{ PCI_VDEVICE(FARSITE, PCI_DEVICE_ID_FARSITE_T4P), .driver_data = FST_TYPE_T4P },
 
-	{PCI_VENDOR_ID_FARSITE, PCI_DEVICE_ID_FARSITE_T1U, PCI_ANY_ID,
-	 PCI_ANY_ID, 0, 0, FST_TYPE_T1U},
+	{ PCI_VDEVICE(FARSITE, PCI_DEVICE_ID_FARSITE_T1U), .driver_data = FST_TYPE_T1U },
 
-	{PCI_VENDOR_ID_FARSITE, PCI_DEVICE_ID_FARSITE_T2U, PCI_ANY_ID,
-	 PCI_ANY_ID, 0, 0, FST_TYPE_T2U},
+	{ PCI_VDEVICE(FARSITE, PCI_DEVICE_ID_FARSITE_T2U), .driver_data = FST_TYPE_T2U },
 
-	{PCI_VENDOR_ID_FARSITE, PCI_DEVICE_ID_FARSITE_T4U, PCI_ANY_ID,
-	 PCI_ANY_ID, 0, 0, FST_TYPE_T4U},
+	{ PCI_VDEVICE(FARSITE, PCI_DEVICE_ID_FARSITE_T4U), .driver_data = FST_TYPE_T4U },
 
-	{PCI_VENDOR_ID_FARSITE, PCI_DEVICE_ID_FARSITE_TE1, PCI_ANY_ID,
-	 PCI_ANY_ID, 0, 0, FST_TYPE_TE1},
+	{ PCI_VDEVICE(FARSITE, PCI_DEVICE_ID_FARSITE_TE1), .driver_data = FST_TYPE_TE1 },
 
-	{PCI_VENDOR_ID_FARSITE, PCI_DEVICE_ID_FARSITE_TE1C, PCI_ANY_ID,
-	 PCI_ANY_ID, 0, 0, FST_TYPE_TE1},
-	{0,}			/* End */
+	{ PCI_VDEVICE(FARSITE, PCI_DEVICE_ID_FARSITE_TE1C), .driver_data = FST_TYPE_TE1 },
+
+	{ }			/* End */
 };
 
 MODULE_DEVICE_TABLE(pci, fst_pci_dev_id);
diff --git a/drivers/net/wan/pc300too.c b/drivers/net/wan/pc300too.c
index 7e57d289b62ca..a91564e53caa6 100644
--- a/drivers/net/wan/pc300too.c
+++ b/drivers/net/wan/pc300too.c
@@ -461,15 +461,11 @@ static int pc300_pci_init_one(struct pci_dev *pdev,
 }
 
 static const struct pci_device_id pc300_pci_tbl[] = {
-	{ PCI_VENDOR_ID_CYCLADES, PCI_DEVICE_ID_PC300_RX_1, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_CYCLADES, PCI_DEVICE_ID_PC300_RX_2, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_CYCLADES, PCI_DEVICE_ID_PC300_TE_1, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_CYCLADES, PCI_DEVICE_ID_PC300_TE_2, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, 0, 0 },
-	{ 0, }
+	{ PCI_VDEVICE(CYCLADES, PCI_DEVICE_ID_PC300_RX_1) },
+	{ PCI_VDEVICE(CYCLADES, PCI_DEVICE_ID_PC300_RX_2) },
+	{ PCI_VDEVICE(CYCLADES, PCI_DEVICE_ID_PC300_TE_1) },
+	{ PCI_VDEVICE(CYCLADES, PCI_DEVICE_ID_PC300_TE_2) },
+	{ }
 };
 
 static struct pci_driver pc300_pci_driver = {
diff --git a/drivers/net/wan/pci200syn.c b/drivers/net/wan/pci200syn.c
index 89f4b4584a33b..9503130fd9a87 100644
--- a/drivers/net/wan/pci200syn.c
+++ b/drivers/net/wan/pci200syn.c
@@ -397,9 +397,9 @@ static int pci200_pci_init_one(struct pci_dev *pdev,
 }
 
 static const struct pci_device_id pci200_pci_tbl[] = {
-	{ PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050, PCI_VENDOR_ID_PLX,
-	  PCI_DEVICE_ID_PLX_PCI200SYN, 0, 0, 0 },
-	{ 0, }
+	{ PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+			  PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_PCI200SYN) },
+	{ }
 };
 
 static struct pci_driver pci200_pci_driver = {
diff --git a/drivers/net/wan/wanxl.c b/drivers/net/wan/wanxl.c
index 3f77071184505..d4da88c771129 100644
--- a/drivers/net/wan/wanxl.c
+++ b/drivers/net/wan/wanxl.c
@@ -806,13 +806,10 @@ static int wanxl_pci_init_one(struct pci_dev *pdev,
 }
 
 static const struct pci_device_id wanxl_pci_tbl[] = {
-	{ PCI_VENDOR_ID_SBE, PCI_DEVICE_ID_SBE_WANXL100, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_SBE, PCI_DEVICE_ID_SBE_WANXL200, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_SBE, PCI_DEVICE_ID_SBE_WANXL400, PCI_ANY_ID,
-	  PCI_ANY_ID, 0, 0, 0 },
-	{ 0, }
+	{ PCI_VDEVICE(SBE, PCI_DEVICE_ID_SBE_WANXL100) },
+	{ PCI_VDEVICE(SBE, PCI_DEVICE_ID_SBE_WANXL200) },
+	{ PCI_VDEVICE(SBE, PCI_DEVICE_ID_SBE_WANXL400) },
+	{ }
 };
 
 static struct pci_driver wanxl_pci_driver = {
diff --git a/drivers/net/wireguard/socket.c b/drivers/net/wireguard/socket.c
index c362c78d908eb..0028ef17dc716 100644
--- a/drivers/net/wireguard/socket.c
+++ b/drivers/net/wireguard/socket.c
@@ -335,7 +335,7 @@ static void sock_free(struct sock *sock)
 	if (unlikely(!sock))
 		return;
 	sk_clear_memalloc(sock);
-	udp_tunnel_sock_release(sock->sk_socket);
+	udp_tunnel_sock_release(sock);
 }
 
 static void set_sock_opts(struct socket *sock)
@@ -389,14 +389,14 @@ int wg_socket_init(struct wg_device *wg, u16 port)
 		goto out;
 	}
 	set_sock_opts(new4);
-	setup_udp_tunnel_sock(net, new4, &cfg);
+	setup_udp_tunnel_sock(net, new4->sk, &cfg);
 
 #if IS_ENABLED(CONFIG_IPV6)
 	if (ipv6_mod_enabled()) {
 		port6.local_udp_port = inet_sk(new4->sk)->inet_sport;
 		ret = udp_sock_create(net, &port6, &new6);
 		if (ret < 0) {
-			udp_tunnel_sock_release(new4);
+			udp_tunnel_sock_release(new4->sk);
 			if (ret == -EADDRINUSE && !port && retries++ < 100)
 				goto retry;
 			pr_err("%s: Could not create IPv6 socket\n",
@@ -404,7 +404,7 @@ int wg_socket_init(struct wg_device *wg, u16 port)
 			goto out;
 		}
 		set_sock_opts(new6);
-		setup_udp_tunnel_sock(net, new6, &cfg);
+		setup_udp_tunnel_sock(net, new6->sk, &cfg);
 	}
 #endif
 
diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h
index 73a9db302245d..dfee432615ebf 100644
--- a/drivers/net/wireless/ath/ath10k/core.h
+++ b/drivers/net/wireless/ath/ath10k/core.h
@@ -1269,7 +1269,6 @@ struct ath10k {
 	} testmode;
 
 	struct {
-		struct gpio_led wifi_led;
 		struct led_classdev cdev;
 		char label[48];
 		u32 gpio_state_pin;
diff --git a/drivers/net/wireless/ath/ath10k/htt_tx.c b/drivers/net/wireless/ath/ath10k/htt_tx.c
index d6f1d85ba8713..29e99fbf36fde 100644
--- a/drivers/net/wireless/ath/ath10k/htt_tx.c
+++ b/drivers/net/wireless/ath/ath10k/htt_tx.c
@@ -1353,7 +1353,7 @@ static int ath10k_htt_tx_hl(struct ath10k_htt *htt, enum ath10k_hw_txrx_mode txm
 		msdu_id = res;
 	}
 
-	/* As msdu is freed by mac80211 (in ieee80211_tx_status()) and by
+	/* As msdu is freed by mac80211 (in ieee80211_tx_status_skb()) and by
 	 * ath10k (in ath10k_htt_htc_tx_complete()) we have to increase
 	 * reference by one to avoid a use-after-free case and a double
 	 * free.
diff --git a/drivers/net/wireless/ath/ath10k/leds.c b/drivers/net/wireless/ath/ath10k/leds.c
index 3a6c8111e7c63..a3961e7760a53 100644
--- a/drivers/net/wireless/ath/ath10k/leds.c
+++ b/drivers/net/wireless/ath/ath10k/leds.c
@@ -19,15 +19,13 @@ static int ath10k_leds_set_brightness_blocking(struct led_classdev *led_cdev,
 {
 	struct ath10k *ar = container_of(led_cdev, struct ath10k,
 					 leds.cdev);
-	struct gpio_led *led = &ar->leds.wifi_led;
 
 	mutex_lock(&ar->conf_mutex);
 
 	if (ar->state != ATH10K_STATE_ON)
 		goto out;
 
-	ar->leds.gpio_state_pin = (brightness != LED_OFF) ^ led->active_low;
-	ath10k_wmi_gpio_output(ar, ar->hw_params.led_pin, ar->leds.gpio_state_pin);
+	ath10k_wmi_gpio_output(ar, ar->hw_params.led_pin, brightness == LED_OFF);
 
 out:
 	mutex_unlock(&ar->conf_mutex);
@@ -63,13 +61,9 @@ int ath10k_leds_register(struct ath10k *ar)
 
 	snprintf(ar->leds.label, sizeof(ar->leds.label), "ath10k-%s",
 		 wiphy_name(ar->hw->wiphy));
-	ar->leds.wifi_led.active_low = 1;
-	ar->leds.wifi_led.name = ar->leds.label;
-	ar->leds.wifi_led.default_state = LEDS_GPIO_DEFSTATE_KEEP;
 
 	ar->leds.cdev.name = ar->leds.label;
 	ar->leds.cdev.brightness_set_blocking = ath10k_leds_set_brightness_blocking;
-	ar->leds.cdev.default_trigger = ar->leds.wifi_led.default_trigger;
 
 	ret = led_classdev_register(wiphy_dev(ar->hw->wiphy), &ar->leds.cdev);
 	if (ret)
diff --git a/drivers/net/wireless/ath/ath10k/qmi.c b/drivers/net/wireless/ath/ath10k/qmi.c
index e7f90fd9e9b83..0d74548a5f349 100644
--- a/drivers/net/wireless/ath/ath10k/qmi.c
+++ b/drivers/net/wireless/ath/ath10k/qmi.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: ISC
 /*
  * Copyright (c) 2018 The Linux Foundation. All rights reserved.
- * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <linux/completion.h>
@@ -1112,7 +1112,7 @@ int ath10k_qmi_init(struct ath10k *ar, u32 msa_size)
 	spin_lock_init(&qmi->event_lock);
 	INIT_WORK(&qmi->event_work, ath10k_qmi_driver_event_work);
 
-	ret = qmi_add_lookup(&qmi->qmi_hdl, WLFW_SERVICE_ID_V01,
+	ret = qmi_add_lookup(&qmi->qmi_hdl, QMI_SERVICE_ID_WLFW,
 			     WLFW_SERVICE_VERS_V01, 0);
 	if (ret)
 		goto err_qmi_lookup;
diff --git a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h
index 9f311f3bc9e7f..88d58f78989dd 100644
--- a/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h
+++ b/drivers/net/wireless/ath/ath10k/qmi_wlfw_v01.h
@@ -7,7 +7,6 @@
 #ifndef WCN3990_QMI_SVC_V01_H
 #define WCN3990_QMI_SVC_V01_H
 
-#define WLFW_SERVICE_ID_V01 0x45
 #define WLFW_SERVICE_VERS_V01 0x01
 
 #define QMI_WLFW_BDF_DOWNLOAD_REQ_V01 0x0025
diff --git a/drivers/net/wireless/ath/ath10k/wmi.c b/drivers/net/wireless/ath/ath10k/wmi.c
index 0bdb38edd9152..e57588c19c800 100644
--- a/drivers/net/wireless/ath/ath10k/wmi.c
+++ b/drivers/net/wireless/ath/ath10k/wmi.c
@@ -3,7 +3,6 @@
  * Copyright (c) 2005-2011 Atheros Communications Inc.
  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
  * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved.
  * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
@@ -1947,15 +1946,15 @@ int ath10k_wmi_cmd_send(struct ath10k *ar, struct sk_buff *skb, u32 cmd_id)
 			ret = -ESHUTDOWN;
 			ath10k_dbg(ar, ATH10K_DBG_WMI,
 				   "drop wmi command %d, hardware is wedged\n", cmd_id);
-		}
-		/* try to send pending beacons first. they take priority */
-		ath10k_wmi_tx_beacons_nowait(ar);
+		} else {
+			/* try to send pending beacons first. they take priority */
+			ath10k_wmi_tx_beacons_nowait(ar);
 
-		ret = ath10k_wmi_cmd_send_nowait(ar, skb, cmd_id);
-
-		if (ret && test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
-			ret = -ESHUTDOWN;
+			ret = ath10k_wmi_cmd_send_nowait(ar, skb, cmd_id);
 
+			if (ret && test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
+				ret = -ESHUTDOWN;
+		}
 		(ret != -EAGAIN);
 	}), 3 * HZ);
 
diff --git a/drivers/net/wireless/ath/ath11k/Kconfig b/drivers/net/wireless/ath/ath11k/Kconfig
index 385513cfdc30e..122726f84492a 100644
--- a/drivers/net/wireless/ath/ath11k/Kconfig
+++ b/drivers/net/wireless/ath/ath11k/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 config ATH11K
-	tristate "Qualcomm Technologies 802.11ax chipset support"
+	tristate "Qualcomm 802.11ax chipset support"
 	depends on MAC80211 && HAS_DMA
 	select ATH_COMMON
 	select QCOM_QMI_HELPERS
diff --git a/drivers/net/wireless/ath/ath11k/dp.c b/drivers/net/wireless/ath/ath11k/dp.c
index bbb86f1651419..5a50b623bd07b 100644
--- a/drivers/net/wireless/ath/ath11k/dp.c
+++ b/drivers/net/wireless/ath/ath11k/dp.c
@@ -1040,6 +1040,7 @@ void ath11k_dp_free(struct ath11k_base *ab)
 		idr_destroy(&dp->tx_ring[i].txbuf_idr);
 		spin_unlock_bh(&dp->tx_ring[i].tx_idr_lock);
 		kfree(dp->tx_ring[i].tx_status);
+		dp->tx_ring[i].tx_status = NULL;
 	}
 
 	/* Deinit any SOC level resource */
diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c
index fe79109adc705..2a413e3a07a78 100644
--- a/drivers/net/wireless/ath/ath11k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath11k/dp_rx.c
@@ -1761,6 +1761,7 @@ static int ath11k_dp_rx_msdu_coalesce(struct ath11k *ar,
 	int buf_first_hdr_len, buf_first_len;
 	struct hal_rx_desc *ldesc;
 	int space_extra, rem_len, buf_len;
+	bool is_continuation;
 	u32 hal_rx_desc_sz = ar->ab->hw_params.hal_desc_sz;
 
 	/* As the msdu is spread across multiple rx buffers,
@@ -1810,7 +1811,8 @@ static int ath11k_dp_rx_msdu_coalesce(struct ath11k *ar,
 	rem_len = msdu_len - buf_first_len;
 	while ((skb = __skb_dequeue(msdu_list)) != NULL && rem_len > 0) {
 		rxcb = ATH11K_SKB_RXCB(skb);
-		if (rxcb->is_continuation)
+		is_continuation = rxcb->is_continuation;
+		if (is_continuation)
 			buf_len = DP_RX_BUFFER_SIZE - hal_rx_desc_sz;
 		else
 			buf_len = rem_len;
@@ -1828,7 +1830,7 @@ static int ath11k_dp_rx_msdu_coalesce(struct ath11k *ar,
 		dev_kfree_skb_any(skb);
 
 		rem_len -= buf_len;
-		if (!rxcb->is_continuation)
+		if (!is_continuation)
 			break;
 	}
 
@@ -2214,8 +2216,7 @@ ath11k_dp_rx_h_find_peer(struct ath11k_base *ab, struct sk_buff *msdu)
 
 	lockdep_assert_held(&ab->base_lock);
 
-	if (rxcb->peer_id)
-		peer = ath11k_peer_find_by_id(ab, rxcb->peer_id);
+	peer = ath11k_peer_find_by_id(ab, rxcb->peer_id);
 
 	if (peer)
 		return peer;
diff --git a/drivers/net/wireless/ath/ath11k/hal.c b/drivers/net/wireless/ath/ath11k/hal.c
index e821e5a62c1c0..98bd9e3f0aae9 100644
--- a/drivers/net/wireless/ath/ath11k/hal.c
+++ b/drivers/net/wireless/ath/ath11k/hal.c
@@ -1387,14 +1387,22 @@ EXPORT_SYMBOL(ath11k_hal_srng_deinit);
 
 void ath11k_hal_srng_clear(struct ath11k_base *ab)
 {
-	/* No need to memset rdp and wrp memory since each individual
-	 * segment would get cleared in ath11k_hal_srng_src_hw_init()
-	 * and ath11k_hal_srng_dst_hw_init().
+	/*
+	 * Preserve the shared pointer buffers, but clear the previous
+	 * firmware instance's hp/tp state before handing them back to FW.
+	 * LMAC rings reuse this shared memory without going through the
+	 * normal SRNG hw-init path that zeros non-LMAC ring pointers.
 	 */
 	memset(ab->hal.srng_list, 0,
 	       sizeof(ab->hal.srng_list));
 	memset(ab->hal.shadow_reg_addr, 0,
 	       sizeof(ab->hal.shadow_reg_addr));
+	if (ab->hal.rdp.vaddr)
+		memset(ab->hal.rdp.vaddr, 0,
+		       sizeof(*ab->hal.rdp.vaddr) * HAL_SRNG_RING_ID_MAX);
+	if (ab->hal.wrp.vaddr)
+		memset(ab->hal.wrp.vaddr, 0,
+		       sizeof(*ab->hal.wrp.vaddr) * HAL_SRNG_NUM_LMAC_RINGS);
 	ab->hal.avail_blk_resource = 0;
 	ab->hal.current_blk_index = 0;
 	ab->hal.num_shadow_reg_configured = 0;
diff --git a/drivers/net/wireless/ath/ath11k/hal_rx.c b/drivers/net/wireless/ath/ath11k/hal_rx.c
index 753bd93f02123..51e0840bc0d1e 100644
--- a/drivers/net/wireless/ath/ath11k/hal_rx.c
+++ b/drivers/net/wireless/ath/ath11k/hal_rx.c
@@ -1467,11 +1467,8 @@ ath11k_hal_rx_parse_mon_status_tlv(struct ath11k_base *ab,
 	case HAL_RX_MPDU_START: {
 		struct hal_rx_mpdu_info *mpdu_info =
 				(struct hal_rx_mpdu_info *)tlv_data;
-		u16 peer_id;
 
-		peer_id = ath11k_hal_rx_mpduinfo_get_peerid(ab, mpdu_info);
-		if (peer_id)
-			ppdu_info->peer_id = peer_id;
+		ppdu_info->peer_id = ath11k_hal_rx_mpduinfo_get_peerid(ab, mpdu_info);
 		break;
 	}
 	case HAL_RXPCU_PPDU_END_INFO: {
diff --git a/drivers/net/wireless/ath/ath11k/mhi.c b/drivers/net/wireless/ath/ath11k/mhi.c
index f994233df2bb9..a6c9ff112c68f 100644
--- a/drivers/net/wireless/ath/ath11k/mhi.c
+++ b/drivers/net/wireless/ath/ath11k/mhi.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2020 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <linux/msi.h>
@@ -282,8 +282,10 @@ static void ath11k_mhi_op_status_cb(struct mhi_controller *mhi_cntrl,
 			break;
 		}
 
+		spin_lock_bh(&ab->base_lock);
 		if (!(test_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags)))
 			queue_work(ab->workqueue_aux, &ab->reset_work);
+		spin_unlock_bh(&ab->base_lock);
 
 		break;
 	default:
diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c
index 7114eca8810db..35bb9e7a63a20 100644
--- a/drivers/net/wireless/ath/ath11k/pci.c
+++ b/drivers/net/wireless/ath/ath11k/pci.c
@@ -1210,6 +1210,14 @@ static void ath11k_pci_shutdown(struct pci_dev *pdev)
 	struct ath11k_pci *ab_pci = ath11k_pci_priv(ab);
 
 	ath11k_pci_set_irq_affinity_hint(ab_pci, NULL);
+
+	spin_lock_bh(&ab->base_lock);
+	set_bit(ATH11K_FLAG_UNREGISTERING, &ab->dev_flags);
+	spin_unlock_bh(&ab->base_lock);
+
+	cancel_work_sync(&ab->reset_work);
+	cancel_work_sync(&ab->dump_work);
+
 	ath11k_pci_power_down(ab, false);
 }
 
diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c
index feebbc30f3df3..410a7ee076a02 100644
--- a/drivers/net/wireless/ath/ath11k/qmi.c
+++ b/drivers/net/wireless/ath/ath11k/qmi.c
@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: BSD-3-Clause-Clear
 /*
  * Copyright (c) 2018-2019 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2025 Qualcomm Innovation Center, Inc. All rights reserved.
  * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
@@ -3337,7 +3336,7 @@ int ath11k_qmi_init_service(struct ath11k_base *ab)
 	spin_lock_init(&ab->qmi.event_lock);
 	INIT_WORK(&ab->qmi.event_work, ath11k_qmi_driver_event_work);
 
-	ret = qmi_add_lookup(&ab->qmi.handle, ATH11K_QMI_WLFW_SERVICE_ID_V01,
+	ret = qmi_add_lookup(&ab->qmi.handle, QMI_SERVICE_ID_WLFW,
 			     ATH11K_QMI_WLFW_SERVICE_VERS_V01,
 			     ab->qmi.service_ins_id);
 	if (ret < 0) {
diff --git a/drivers/net/wireless/ath/ath11k/qmi.h b/drivers/net/wireless/ath/ath11k/qmi.h
index 7968ab122b65d..eae416db8b525 100644
--- a/drivers/net/wireless/ath/ath11k/qmi.h
+++ b/drivers/net/wireless/ath/ath11k/qmi.h
@@ -15,7 +15,6 @@
 #define ATH11K_QMI_MAX_BDF_FILE_NAME_SIZE	64
 #define ATH11K_QMI_CALDB_ADDRESS		0x4BA00000
 #define ATH11K_QMI_WLANFW_MAX_BUILD_ID_LEN_V01	128
-#define ATH11K_QMI_WLFW_SERVICE_ID_V01		0x45
 #define ATH11K_QMI_WLFW_SERVICE_VERS_V01	0x01
 #define ATH11K_QMI_WLFW_SERVICE_INS_ID_V01	0x02
 #define ATH11K_QMI_WLFW_SERVICE_INS_ID_V01_QCA6390	0x01
diff --git a/drivers/net/wireless/ath/ath11k/testmode.c b/drivers/net/wireless/ath/ath11k/testmode.c
index a9751ea2a0b73..c72eed358f6dd 100644
--- a/drivers/net/wireless/ath/ath11k/testmode.c
+++ b/drivers/net/wireless/ath/ath11k/testmode.c
@@ -457,6 +457,7 @@ static int ath11k_tm_cmd_wmi_ftm(struct ath11k *ar, struct nlattr *tb[])
 		ret = ath11k_wmi_cmd_send(wmi, skb, cmd_id);
 		if (ret) {
 			ath11k_warn(ar->ab, "failed to send wmi ftm command: %d\n", ret);
+			dev_kfree_skb(skb);
 			goto out;
 		}
 
diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c
index 40747fba3b0c0..dca6e011cc40e 100644
--- a/drivers/net/wireless/ath/ath11k/wmi.c
+++ b/drivers/net/wireless/ath/ath11k/wmi.c
@@ -9299,7 +9299,7 @@ int ath11k_wmi_hw_data_filter_cmd(struct ath11k *ar, u32 vdev_id,
 {
 	struct wmi_hw_data_filter_cmd *cmd;
 	struct sk_buff *skb;
-	int len;
+	int ret, len;
 
 	len = sizeof(*cmd);
 	skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9324,7 +9324,13 @@ int ath11k_wmi_hw_data_filter_cmd(struct ath11k *ar, u32 vdev_id,
 		   "hw data filter enable %d filter_bitmap 0x%x\n",
 		   enable, filter_bitmap);
 
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_HW_DATA_FILTER_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_HW_DATA_FILTER_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_HW_DATA_FILTER_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_wow_host_wakeup_ind(struct ath11k *ar)
@@ -9332,6 +9338,7 @@ int ath11k_wmi_wow_host_wakeup_ind(struct ath11k *ar)
 	struct wmi_wow_host_wakeup_ind *cmd;
 	struct sk_buff *skb;
 	size_t len;
+	int ret;
 
 	len = sizeof(*cmd);
 	skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9345,14 +9352,20 @@ int ath11k_wmi_wow_host_wakeup_ind(struct ath11k *ar)
 
 	ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow host wakeup ind\n");
 
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_wow_enable(struct ath11k *ar)
 {
 	struct wmi_wow_enable_cmd *cmd;
 	struct sk_buff *skb;
-	int len;
+	int ret, len;
 
 	len = sizeof(*cmd);
 	skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9367,7 +9380,13 @@ int ath11k_wmi_wow_enable(struct ath11k *ar)
 	cmd->pause_iface_config = WOW_IFACE_PAUSE_ENABLED;
 	ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow enable\n");
 
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_WOW_ENABLE_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_scan_prob_req_oui(struct ath11k *ar,
@@ -9376,7 +9395,7 @@ int ath11k_wmi_scan_prob_req_oui(struct ath11k *ar,
 	struct sk_buff *skb;
 	struct wmi_scan_prob_req_oui_cmd *cmd;
 	u32 prob_req_oui;
-	int len;
+	int ret, len;
 
 	prob_req_oui = (((u32)mac_addr[0]) << 16) |
 		       (((u32)mac_addr[1]) << 8) | mac_addr[2];
@@ -9395,7 +9414,13 @@ int ath11k_wmi_scan_prob_req_oui(struct ath11k *ar,
 	ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "scan prob req oui %d\n",
 		   prob_req_oui);
 
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_SCAN_PROB_REQ_OUI_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_SCAN_PROB_REQ_OUI_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_SCAN_PROB_REQ_OUI_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_wow_add_wakeup_event(struct ath11k *ar, u32 vdev_id,
@@ -9405,6 +9430,7 @@ int ath11k_wmi_wow_add_wakeup_event(struct ath11k *ar, u32 vdev_id,
 	struct wmi_wow_add_del_event_cmd *cmd;
 	struct sk_buff *skb;
 	size_t len;
+	int ret;
 
 	len = sizeof(*cmd);
 	skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9422,7 +9448,13 @@ int ath11k_wmi_wow_add_wakeup_event(struct ath11k *ar, u32 vdev_id,
 	ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow add wakeup event %s enable %d vdev_id %d\n",
 		   wow_wakeup_event(event), enable, vdev_id);
 
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_wow_add_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id,
@@ -9435,6 +9467,7 @@ int ath11k_wmi_wow_add_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id,
 	struct sk_buff *skb;
 	u8 *ptr;
 	size_t len;
+	int ret;
 
 	len = sizeof(*cmd) +
 	      sizeof(*tlv) +			/* array struct */
@@ -9527,7 +9560,13 @@ int ath11k_wmi_wow_add_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id,
 	ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow add pattern vdev_id %d pattern_id %d pattern_offset %d\n",
 		   vdev_id, pattern_id, pattern_offset);
 
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ADD_WAKE_PATTERN_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_ADD_WAKE_PATTERN_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_WOW_ADD_WAKE_PATTERN_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_wow_del_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id)
@@ -9535,6 +9574,7 @@ int ath11k_wmi_wow_del_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id)
 	struct wmi_wow_del_pattern_cmd *cmd;
 	struct sk_buff *skb;
 	size_t len;
+	int ret;
 
 	len = sizeof(*cmd);
 	skb = ath11k_wmi_alloc_skb(ar->wmi->wmi_ab, len);
@@ -9553,7 +9593,13 @@ int ath11k_wmi_wow_del_pattern(struct ath11k *ar, u32 vdev_id, u32 pattern_id)
 	ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "tlv wow del pattern vdev_id %d pattern_id %d\n",
 		   vdev_id, pattern_id);
 
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_DEL_WAKE_PATTERN_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_WOW_DEL_WAKE_PATTERN_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_WOW_DEL_WAKE_PATTERN_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 static struct sk_buff *
@@ -9697,6 +9743,7 @@ int ath11k_wmi_wow_config_pno(struct ath11k *ar, u32 vdev_id,
 			      struct wmi_pno_scan_req  *pno_scan)
 {
 	struct sk_buff *skb;
+	int ret;
 
 	if (pno_scan->enable)
 		skb = ath11k_wmi_op_gen_config_pno_start(ar, vdev_id, pno_scan);
@@ -9706,7 +9753,13 @@ int ath11k_wmi_wow_config_pno(struct ath11k *ar, u32 vdev_id,
 	if (IS_ERR_OR_NULL(skb))
 		return -ENOMEM;
 
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 static void ath11k_wmi_fill_ns_offload(struct ath11k *ar,
@@ -9824,6 +9877,7 @@ int ath11k_wmi_arp_ns_offload(struct ath11k *ar,
 	u8 *buf_ptr;
 	size_t len;
 	u8 ns_cnt, ns_ext_tuples = 0;
+	int ret;
 
 	offload = &arvif->arp_ns_offload;
 	ns_cnt = offload->ipv6_count;
@@ -9862,7 +9916,13 @@ int ath11k_wmi_arp_ns_offload(struct ath11k *ar,
 	if (ns_ext_tuples)
 		ath11k_wmi_fill_ns_offload(ar, offload, &buf_ptr, enable, 1);
 
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_SET_ARP_NS_OFFLOAD_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_SET_ARP_NS_OFFLOAD_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_SET_ARP_NS_OFFLOAD_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_gtk_rekey_offload(struct ath11k *ar,
@@ -9870,7 +9930,7 @@ int ath11k_wmi_gtk_rekey_offload(struct ath11k *ar,
 {
 	struct wmi_gtk_rekey_offload_cmd *cmd;
 	struct ath11k_rekey_data *rekey_data = &arvif->rekey_data;
-	int len;
+	int ret, len;
 	struct sk_buff *skb;
 	__le64 replay_ctr;
 
@@ -9904,14 +9964,20 @@ int ath11k_wmi_gtk_rekey_offload(struct ath11k *ar,
 
 	ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "offload gtk rekey vdev: %d %d\n",
 		   arvif->vdev_id, enable);
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_GTK_OFFLOAD_CMDID offload\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_gtk_rekey_getinfo(struct ath11k *ar,
 				 struct ath11k_vif *arvif)
 {
 	struct wmi_gtk_rekey_offload_cmd *cmd;
-	int len;
+	int ret, len;
 	struct sk_buff *skb;
 
 	len = sizeof(*cmd);
@@ -9928,7 +9994,13 @@ int ath11k_wmi_gtk_rekey_getinfo(struct ath11k *ar,
 
 	ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "get gtk rekey vdev_id: %d\n",
 		   arvif->vdev_id);
-	return ath11k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+	ret = ath11k_wmi_cmd_send(ar->wmi, skb, WMI_GTK_OFFLOAD_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_GTK_OFFLOAD_CMDID getinfo\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_pdev_set_bios_sar_table_param(struct ath11k *ar, const u8 *sar_val)
@@ -9938,6 +10010,7 @@ int ath11k_wmi_pdev_set_bios_sar_table_param(struct ath11k *ar, const u8 *sar_va
 	struct sk_buff *skb;
 	u8 *buf_ptr;
 	u32 len, sar_len_aligned, rsvd_len_aligned;
+	int ret;
 
 	sar_len_aligned = roundup(BIOS_SAR_TABLE_LEN, sizeof(u32));
 	rsvd_len_aligned = roundup(BIOS_SAR_RSVD1_LEN, sizeof(u32));
@@ -9968,7 +10041,13 @@ int ath11k_wmi_pdev_set_bios_sar_table_param(struct ath11k *ar, const u8 *sar_va
 	tlv->header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_ARRAY_BYTE) |
 		      FIELD_PREP(WMI_TLV_LEN, rsvd_len_aligned);
 
-	return ath11k_wmi_cmd_send(wmi, skb, WMI_PDEV_SET_BIOS_SAR_TABLE_CMDID);
+	ret = ath11k_wmi_cmd_send(wmi, skb, WMI_PDEV_SET_BIOS_SAR_TABLE_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_PDEV_SET_BIOS_SAR_TABLE_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_pdev_set_bios_geo_table_param(struct ath11k *ar)
@@ -9979,6 +10058,7 @@ int ath11k_wmi_pdev_set_bios_geo_table_param(struct ath11k *ar)
 	struct sk_buff *skb;
 	u8 *buf_ptr;
 	u32 len, rsvd_len_aligned;
+	int ret;
 
 	rsvd_len_aligned = roundup(BIOS_SAR_RSVD2_LEN, sizeof(u32));
 	len = sizeof(*cmd) + TLV_HDR_SIZE + rsvd_len_aligned;
@@ -9998,7 +10078,13 @@ int ath11k_wmi_pdev_set_bios_geo_table_param(struct ath11k *ar)
 	tlv->header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_ARRAY_BYTE) |
 		      FIELD_PREP(WMI_TLV_LEN, rsvd_len_aligned);
 
-	return ath11k_wmi_cmd_send(wmi, skb, WMI_PDEV_SET_BIOS_GEO_TABLE_CMDID);
+	ret = ath11k_wmi_cmd_send(wmi, skb, WMI_PDEV_SET_BIOS_GEO_TABLE_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_PDEV_SET_BIOS_GEO_TABLE_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 int ath11k_wmi_sta_keepalive(struct ath11k *ar,
@@ -10009,6 +10095,7 @@ int ath11k_wmi_sta_keepalive(struct ath11k *ar,
 	struct wmi_sta_keepalive_arp_resp *arp;
 	struct sk_buff *skb;
 	size_t len;
+	int ret;
 
 	len = sizeof(*cmd) + sizeof(*arp);
 	skb = ath11k_wmi_alloc_skb(wmi->wmi_ab, len);
@@ -10040,7 +10127,13 @@ int ath11k_wmi_sta_keepalive(struct ath11k *ar,
 		   "sta keepalive vdev %d enabled %d method %d interval %d\n",
 		   arg->vdev_id, arg->enabled, arg->method, arg->interval);
 
-	return ath11k_wmi_cmd_send(wmi, skb, WMI_STA_KEEPALIVE_CMDID);
+	ret = ath11k_wmi_cmd_send(wmi, skb, WMI_STA_KEEPALIVE_CMDID);
+	if (ret) {
+		ath11k_warn(ar->ab, "failed to send WMI_STA_KEEPALIVE_CMDID\n");
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
 }
 
 bool ath11k_wmi_supports_6ghz_cc_ext(struct ath11k *ar)
diff --git a/drivers/net/wireless/ath/ath12k/Kconfig b/drivers/net/wireless/ath/ath12k/Kconfig
index d39c075758bda..4a2b240f967af 100644
--- a/drivers/net/wireless/ath/ath12k/Kconfig
+++ b/drivers/net/wireless/ath/ath12k/Kconfig
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: BSD-3-Clause-Clear
 config ATH12K
-	tristate "Qualcomm Technologies Wi-Fi 7 support (ath12k)"
+	tristate "Qualcomm Wi-Fi 7 support (ath12k)"
 	depends on MAC80211 && HAS_DMA && PCI
 	select QCOM_QMI_HELPERS
 	select MHI_BUS
@@ -15,7 +15,7 @@ config ATH12K
 	  If you choose to build a module, it'll be called ath12k.
 
 config ATH12K_AHB
-	bool "QTI ath12k AHB support"
+	bool "Qualcomm ath12k AHB support"
 	depends on ATH12K && REMOTEPROC
 	select QCOM_MDT_LOADER
 	select QCOM_SCM
@@ -33,7 +33,7 @@ config ATH12K_DEBUG
 	  you want optimal performance choose N.
 
 config ATH12K_DEBUGFS
-	bool "QTI ath12k debugfs support"
+	bool "Qualcomm ath12k debugfs support"
 	depends on ATH12K && MAC80211_DEBUGFS
 	help
 	  Enable ath12k debugfs support
diff --git a/drivers/net/wireless/ath/ath12k/ahb.c b/drivers/net/wireless/ath/ath12k/ahb.c
index 2dcf0a52e4c15..30733a244454e 100644
--- a/drivers/net/wireless/ath/ath12k/ahb.c
+++ b/drivers/net/wireless/ath/ath12k/ahb.c
@@ -583,31 +583,36 @@ static int ath12k_ahb_config_ext_irq(struct ath12k_base *ab)
 		netif_napi_add(irq_grp->napi_ndev, &irq_grp->napi,
 			       ath12k_ahb_ext_grp_napi_poll);
 
-		for (j = 0; j < ATH12K_EXT_IRQ_NUM_MAX; j++) {
-			/* For TX ring, ensure that the ring mask and the
-			 * tcl_to_wbm_rbm_map point to the same ring number.
-			 */
+		for (j = 0; j < DP_TCL_NUM_RING_MAX; j++) {
 			if (ring_mask->tx[i] &
-			    BIT(ab->hal.tcl_to_wbm_rbm_map[j].wbm_ring_num)) {
+			    BIT(ab->hal.tcl_to_wbm_rbm_map[j].wbm_ring_num) &&
+			    num_irq < ATH12K_EXT_IRQ_NUM_MAX) {
 				irq_grp->irqs[num_irq++] =
 					wbm2host_tx_completions_ring1 - j;
 			}
+		}
 
-			if (ring_mask->rx[i] & BIT(j)) {
+		for (j = 0; j < ATH12K_EXT_IRQ_NUM_MAX; j++) {
+			if (ring_mask->rx[i] & BIT(j) &&
+			    num_irq < ATH12K_EXT_IRQ_NUM_MAX) {
 				irq_grp->irqs[num_irq++] =
 					reo2host_destination_ring1 - j;
 			}
 
-			if (ring_mask->rx_err[i] & BIT(j))
+			if (ring_mask->rx_err[i] & BIT(j) &&
+			    num_irq < ATH12K_EXT_IRQ_NUM_MAX)
 				irq_grp->irqs[num_irq++] = reo2host_exception;
 
-			if (ring_mask->rx_wbm_rel[i] & BIT(j))
+			if (ring_mask->rx_wbm_rel[i] & BIT(j) &&
+			    num_irq < ATH12K_EXT_IRQ_NUM_MAX)
 				irq_grp->irqs[num_irq++] = wbm2host_rx_release;
 
-			if (ring_mask->reo_status[i] & BIT(j))
+			if (ring_mask->reo_status[i] & BIT(j) &&
+			    num_irq < ATH12K_EXT_IRQ_NUM_MAX)
 				irq_grp->irqs[num_irq++] = reo2host_status;
 
-			if (ring_mask->rx_mon_dest[i] & BIT(j))
+			if (ring_mask->rx_mon_dest[i] & BIT(j) &&
+			    num_irq < ATH12K_EXT_IRQ_NUM_MAX)
 				irq_grp->irqs[num_irq++] =
 					rxdma2host_monitor_destination_mac1;
 		}
diff --git a/drivers/net/wireless/ath/ath12k/core.c b/drivers/net/wireless/ath/ath12k/core.c
index 980a12fb2c6e7..742d4fd1b598c 100644
--- a/drivers/net/wireless/ath/ath12k/core.c
+++ b/drivers/net/wireless/ath/ath12k/core.c
@@ -1006,6 +1006,27 @@ static void ath12k_core_device_cleanup(struct ath12k_base *ab)
 	mutex_unlock(&ab->core_lock);
 }
 
+static int ath12k_core_device_setup(struct ath12k_base *ab)
+{
+	int ret;
+
+	guard(mutex)(&ab->core_lock);
+
+	ret = ath12k_core_pdev_create(ab);
+	if (ret) {
+		ath12k_err(ab, "failed to create pdev core %d\n", ret);
+		return ret;
+	}
+
+	ath12k_hif_irq_enable(ab);
+
+	ret = ath12k_core_rfkill_config(ab);
+	if (ret && ret != -EOPNOTSUPP)
+		return ret;
+
+	return 0;
+}
+
 static void ath12k_core_hw_group_stop(struct ath12k_hw_group *ag)
 {
 	struct ath12k_base *ab;
@@ -1015,10 +1036,6 @@ static void ath12k_core_hw_group_stop(struct ath12k_hw_group *ag)
 
 	clear_bit(ATH12K_GROUP_FLAG_REGISTERED, &ag->flags);
 
-	ath12k_mac_unregister(ag);
-
-	ath12k_mac_mlo_teardown(ag);
-
 	for (i = ag->num_devices - 1; i >= 0; i--) {
 		ab = ag->ab[i];
 		if (!ab)
@@ -1029,6 +1046,12 @@ static void ath12k_core_hw_group_stop(struct ath12k_hw_group *ag)
 		ath12k_core_device_cleanup(ab);
 	}
 
+	/* Unregister MAC (drops wiphys) only after per-device cleanup */
+	ath12k_mac_unregister(ag);
+
+	/* Teardown MLO state after MAC unregister for symmetry */
+	ath12k_mac_mlo_teardown(ag);
+
 	ath12k_mac_destroy(ag);
 }
 
@@ -1165,26 +1188,11 @@ static int ath12k_core_hw_group_start(struct ath12k_hw_group *ag)
 		if (!ab)
 			continue;
 
-		mutex_lock(&ab->core_lock);
-
 		set_bit(ATH12K_FLAG_REGISTERED, &ab->dev_flags);
 
-		ret = ath12k_core_pdev_create(ab);
-		if (ret) {
-			ath12k_err(ab, "failed to create pdev core %d\n", ret);
-			mutex_unlock(&ab->core_lock);
-			goto err;
-		}
-
-		ath12k_hif_irq_enable(ab);
-
-		ret = ath12k_core_rfkill_config(ab);
-		if (ret && ret != -EOPNOTSUPP) {
-			mutex_unlock(&ab->core_lock);
+		ret = ath12k_core_device_setup(ab);
+		if (ret)
 			goto err;
-		}
-
-		mutex_unlock(&ab->core_lock);
 	}
 
 	return 0;
@@ -2275,6 +2283,7 @@ void ath12k_core_deinit(struct ath12k_base *ab)
 void ath12k_core_free(struct ath12k_base *ab)
 {
 	timer_delete_sync(&ab->rx_replenish_retry);
+	ath12k_wmi_free();
 	destroy_workqueue(ab->workqueue_aux);
 	destroy_workqueue(ab->workqueue);
 	kfree(ab);
@@ -2299,6 +2308,9 @@ struct ath12k_base *ath12k_core_alloc(struct device *dev, size_t priv_size,
 	if (!ab->workqueue_aux)
 		goto err_free_wq;
 
+	if (ath12k_wmi_alloc() < 0)
+		goto err_free_wq_aux;
+
 	mutex_init(&ab->core_lock);
 	spin_lock_init(&ab->base_lock);
 	init_completion(&ab->reset_complete);
@@ -2333,6 +2345,8 @@ struct ath12k_base *ath12k_core_alloc(struct device *dev, size_t priv_size,
 
 	return ab;
 
+err_free_wq_aux:
+	destroy_workqueue(ab->workqueue_aux);
 err_free_wq:
 	destroy_workqueue(ab->workqueue);
 err_sc_free:
diff --git a/drivers/net/wireless/ath/ath12k/core.h b/drivers/net/wireless/ath/ath12k/core.h
index 8be435535a4e8..f6d8ec9ef7b04 100644
--- a/drivers/net/wireless/ath/ath12k/core.h
+++ b/drivers/net/wireless/ath/ath12k/core.h
@@ -542,8 +542,8 @@ struct ath12k_sta {
 #define ATH12K_MAX_5GHZ_FREQ	(ATH12K_5GHZ_MAX_CENTER + ATH12K_HALF_20MHZ_BW)
 #define ATH12K_MIN_6GHZ_FREQ	(ATH12K_6GHZ_MIN_CENTER - ATH12K_HALF_20MHZ_BW)
 #define ATH12K_MAX_6GHZ_FREQ	(ATH12K_6GHZ_MAX_CENTER + ATH12K_HALF_20MHZ_BW)
-#define ATH12K_NUM_CHANS 101
-#define ATH12K_MAX_5GHZ_CHAN 173
+#define ATH12K_NUM_CHANS 102
+#define ATH12K_MAX_5GHZ_CHAN 177
 
 static inline bool ath12k_is_2ghz_channel_freq(u32 freq)
 {
@@ -763,6 +763,14 @@ struct ath12k {
 	struct ath12k_pdev_rssi_offsets rssi_info;
 
 	struct ath12k_thermal thermal;
+
+	/* Protected by ar->data_lock */
+	struct ath12k_incumbent_signal_interference {
+		u32 center_freq;
+		enum nl80211_chan_width width;
+		u32 chan_bw_interference_bitmap;
+		bool handling_in_progress;
+	} incumbent_signal_interference;
 };
 
 struct ath12k_hw {
diff --git a/drivers/net/wireless/ath/ath12k/debugfs.c b/drivers/net/wireless/ath/ath12k/debugfs.c
index 8c81a1c224497..d17d4a8f1cb7d 100644
--- a/drivers/net/wireless/ath/ath12k/debugfs.c
+++ b/drivers/net/wireless/ath/ath12k/debugfs.c
@@ -1450,6 +1450,44 @@ static const struct file_operations fops_pdev_stats = {
 	.llseek = default_llseek,
 };
 
+static ssize_t
+ath12k_write_simulate_incumbent_signal_interference(struct file *file,
+						    const char __user *user_buf,
+						    size_t count, loff_t *ppos)
+{
+	struct ath12k *ar = file->private_data;
+	struct ath12k_hw *ah = ath12k_ar_to_ah(ar);
+	struct wiphy *wiphy = ath12k_ar_to_hw(ar)->wiphy;
+	u32 chan_bw_interference_bitmap;
+	int ret;
+
+	if (ah->state != ATH12K_HW_STATE_ON)
+		return -ENETDOWN;
+
+	/*
+	 * Bitmap uses the firmware primary-based ordering documented in
+	 * ath12k_wmi_transform_interference_bitmap() & intf_map_80.
+	 */
+	if (kstrtou32_from_user(user_buf, count, 0, &chan_bw_interference_bitmap))
+		return -EINVAL;
+
+	wiphy_lock(wiphy);
+	ret = ath12k_wmi_simulate_incumbent_signal_interference(ar, chan_bw_interference_bitmap);
+	if (ret)
+		goto exit;
+
+	ret = count;
+
+exit:
+	wiphy_unlock(wiphy);
+	return ret;
+}
+
+static const struct file_operations fops_simulate_incumbent_signal_interference = {
+	.write = ath12k_write_simulate_incumbent_signal_interference,
+	.open = simple_open,
+};
+
 static
 void ath12k_debugfs_fw_stats_register(struct ath12k *ar)
 {
@@ -1515,6 +1553,14 @@ void ath12k_debugfs_register(struct ath12k *ar)
 			    ar, &fops_tpc_stats_type);
 	init_completion(&ar->debug.tpc_complete);
 
+	if (ar->mac.sbands[NL80211_BAND_6GHZ].channels &&
+	    test_bit(WMI_TLV_SERVICE_DCS_INCUMBENT_SIGNAL_INTERFERENCE_SUPPORT,
+		     ar->ab->wmi_ab.svc_map)) {
+		debugfs_create_file("simulate_incumbent_signal_interference", 0200,
+				    ar->debug.debugfs_pdev, ar,
+				    &fops_simulate_incumbent_signal_interference);
+	}
+
 	ath12k_debugfs_htt_stats_register(ar);
 	ath12k_debugfs_fw_stats_register(ar);
 
diff --git a/drivers/net/wireless/ath/ath12k/dp_htt.c b/drivers/net/wireless/ath/ath12k/dp_htt.c
index 9c19d9707abfb..52e10059c6d5a 100644
--- a/drivers/net/wireless/ath/ath12k/dp_htt.c
+++ b/drivers/net/wireless/ath/ath12k/dp_htt.c
@@ -146,7 +146,7 @@ static int ath12k_htt_tlv_ppdu_stats_parse(struct ath12k_base *ab,
 }
 
 int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len,
-			   int (*iter)(struct ath12k_base *ar, u16 tag, u16 len,
+			   int (*iter)(struct ath12k_base *ab, u16 tag, u16 len,
 				       const void *ptr, void *data),
 			   void *data)
 {
diff --git a/drivers/net/wireless/ath/ath12k/dp_htt.h b/drivers/net/wireless/ath/ath12k/dp_htt.h
index 6020e632f74ec..987689f11cda4 100644
--- a/drivers/net/wireless/ath/ath12k/dp_htt.h
+++ b/drivers/net/wireless/ath/ath12k/dp_htt.h
@@ -1523,7 +1523,7 @@ int ath12k_dp_tx_htt_srng_setup(struct ath12k_base *ab, u32 ring_id,
 void ath12k_dp_htt_htc_t2h_msg_handler(struct ath12k_base *ab,
 				       struct sk_buff *skb);
 int ath12k_dp_htt_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len,
-			   int (*iter)(struct ath12k_base *ar, u16 tag, u16 len,
+			   int (*iter)(struct ath12k_base *ab, u16 tag, u16 len,
 				       const void *ptr, void *data),
 				       void *data);
 int ath12k_dp_tx_htt_h2t_ver_req_msg(struct ath12k_base *ab);
diff --git a/drivers/net/wireless/ath/ath12k/dp_rx.c b/drivers/net/wireless/ath/ath12k/dp_rx.c
index b108ccd0f6370..a68b28aa1f4ba 100644
--- a/drivers/net/wireless/ath/ath12k/dp_rx.c
+++ b/drivers/net/wireless/ath/ath12k/dp_rx.c
@@ -17,6 +17,11 @@
 #include "dp_mon.h"
 #include "debugfs_htt_stats.h"
 
+#define ATH12K_2GHZ_MIN_CHAN_NUM 1
+#define ATH12K_2GHZ_MAX_CHAN_NUM 14
+#define ATH12K_5GHZ_MIN_CHAN_NUM 36
+#define ATH12K_5GHZ_MAX_CHAN_NUM 177
+
 static int ath12k_dp_rx_tid_delete_handler(struct ath12k_base *ab,
 					   struct ath12k_dp_rx_tid_rxq *rx_tid);
 
@@ -1095,7 +1100,8 @@ static void ath12k_get_dot11_hdr_from_rx_desc(struct ath12k_pdev_dp *dp_pdev,
 static void ath12k_dp_rx_h_undecap_eth(struct ath12k_pdev_dp *dp_pdev,
 				       struct sk_buff *msdu,
 				       enum hal_encrypt_type enctype,
-				       struct hal_rx_desc_data *rx_info)
+				       struct hal_rx_desc_data *rx_info,
+				       enum ath12k_dp_rx_decap_type decap_type)
 {
 	struct ieee80211_hdr *hdr;
 	struct ethhdr *eth;
@@ -1103,12 +1109,24 @@ static void ath12k_dp_rx_h_undecap_eth(struct ath12k_pdev_dp *dp_pdev,
 	u8 sa[ETH_ALEN];
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
 	struct ath12k_dp_rx_rfc1042_hdr rfc = {0xaa, 0xaa, 0x03, {0x00, 0x00, 0x00}};
+	struct ath12k_dp_rx_rfc1042_hdr *llc;
 
 	eth = (struct ethhdr *)msdu->data;
 	ether_addr_copy(da, eth->h_dest);
 	ether_addr_copy(sa, eth->h_source);
-	rfc.snap_type = eth->h_proto;
-	skb_pull(msdu, sizeof(*eth));
+	if (decap_type == DP_RX_DECAP_TYPE_8023) {
+		/*
+		 * For 802.3 frames, eth->h_proto carries a length field, not
+		 * an EtherType. The actual EtherType is in the LLC/SNAP header
+		 * that follows the Ethernet header.
+		 */
+		llc = (struct ath12k_dp_rx_rfc1042_hdr *)(msdu->data + sizeof(*eth));
+		rfc.snap_type = llc->snap_type;
+		skb_pull(msdu, sizeof(*eth) + sizeof(*llc));
+	} else {
+		rfc.snap_type = eth->h_proto;
+		skb_pull(msdu, sizeof(*eth));
+	}
 	memcpy(skb_push(msdu, sizeof(rfc)), &rfc,
 	       sizeof(rfc));
 	ath12k_get_dot11_hdr_from_rx_desc(dp_pdev, msdu, rxcb, enctype, rx_info);
@@ -1126,9 +1144,10 @@ void ath12k_dp_rx_h_undecap(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu
 			    bool decrypted,
 			    struct hal_rx_desc_data *rx_info)
 {
+	enum ath12k_dp_rx_decap_type decap_type = rx_info->decap_type;
 	struct ethhdr *ehdr;
 
-	switch (rx_info->decap_type) {
+	switch (decap_type) {
 	case DP_RX_DECAP_TYPE_NATIVE_WIFI:
 		ath12k_dp_rx_h_undecap_nwifi(dp_pdev, msdu, enctype, rx_info);
 		break;
@@ -1142,19 +1161,33 @@ void ath12k_dp_rx_h_undecap(struct ath12k_pdev_dp *dp_pdev, struct sk_buff *msdu
 		/* mac80211 allows fast path only for authorized STA */
 		if (ehdr->h_proto == cpu_to_be16(ETH_P_PAE)) {
 			ATH12K_SKB_RXCB(msdu)->is_eapol = true;
-			ath12k_dp_rx_h_undecap_eth(dp_pdev, msdu, enctype, rx_info);
+			ath12k_dp_rx_h_undecap_eth(dp_pdev, msdu, enctype, rx_info,
+						   decap_type);
 			break;
 		}
 
 		/* PN for mcast packets will be validated in mac80211;
 		 * remove eth header and add 802.11 header.
 		 */
-		if (ATH12K_SKB_RXCB(msdu)->is_mcbc && decrypted)
-			ath12k_dp_rx_h_undecap_eth(dp_pdev, msdu, enctype, rx_info);
+		if (ATH12K_SKB_RXCB(msdu)->is_mcbc && decrypted) {
+			ath12k_dp_rx_h_undecap_eth(dp_pdev, msdu, enctype, rx_info,
+						   decap_type);
+			break;
+		}
+
+		rx_info->rx_status->flag |= RX_FLAG_8023;
 		break;
 	case DP_RX_DECAP_TYPE_8023:
-		/* TODO: Handle undecap for these formats */
-		break;
+		/*
+		 * Note that ethernet decap format indicates that the decapped
+		 * packet is either Ethernet 2 (DIX)  or 802.3 (uses SNAP/LLC).
+		 */
+		if (ATH12K_SKB_RXCB(msdu)->is_mcbc && decrypted) {
+			ath12k_dp_rx_h_undecap_eth(dp_pdev, msdu, enctype, rx_info,
+						   decap_type);
+			break;
+		}
+		rx_info->rx_status->flag |= RX_FLAG_8023;
 	}
 }
 EXPORT_SYMBOL(ath12k_dp_rx_h_undecap);
@@ -1289,9 +1322,11 @@ void ath12k_dp_rx_h_ppdu(struct ath12k_pdev_dp *dp_pdev,
 	    center_freq <= ATH12K_MAX_6GHZ_FREQ) {
 		rx_status->band = NL80211_BAND_6GHZ;
 		rx_status->freq = center_freq;
-	} else if (channel_num >= 1 && channel_num <= 14) {
+	} else if (channel_num >= ATH12K_2GHZ_MIN_CHAN_NUM &&
+		   channel_num <= ATH12K_2GHZ_MAX_CHAN_NUM) {
 		rx_status->band = NL80211_BAND_2GHZ;
-	} else if (channel_num >= 36 && channel_num <= 173) {
+	} else if (channel_num >= ATH12K_5GHZ_MIN_CHAN_NUM &&
+		   channel_num <= ATH12K_5GHZ_MAX_CHAN_NUM) {
 		rx_status->band = NL80211_BAND_5GHZ;
 	}
 
@@ -1336,9 +1371,7 @@ void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struc
 	struct ath12k_dp_peer *peer;
 	struct ath12k_skb_rxcb *rxcb = ATH12K_SKB_RXCB(msdu);
 	struct ieee80211_rx_status *status = rx_info->rx_status;
-	u8 decap = rx_info->decap_type;
 	bool is_mcbc = rxcb->is_mcbc;
-	bool is_eapol = rxcb->is_eapol;
 
 	peer = ath12k_dp_peer_find_by_peerid(dp_pdev, rxcb->peer_id);
 
@@ -1383,15 +1416,6 @@ void ath12k_dp_rx_deliver_msdu(struct ath12k_pdev_dp *dp_pdev, struct napi_struc
 
 	/* TODO: trace rx packet */
 
-	/* PN for multicast packets are not validate in HW,
-	 * so skip 802.3 rx path
-	 * Also, fast_rx expects the STA to be authorized, hence
-	 * eapol packets are sent in slow path.
-	 */
-	if (decap == DP_RX_DECAP_TYPE_ETHERNET2_DIX && !is_eapol &&
-	    !(is_mcbc && rx_status->flag & RX_FLAG_DECRYPTED))
-		rx_status->flag |= RX_FLAG_8023;
-
 	ieee80211_rx_napi(ath12k_pdev_dp_to_hw(dp_pdev), pubsta, msdu, napi);
 }
 EXPORT_SYMBOL(ath12k_dp_rx_deliver_msdu);
diff --git a/drivers/net/wireless/ath/ath12k/htc.h b/drivers/net/wireless/ath/ath12k/htc.h
index 7e3dccc7cc148..05b3e593542d3 100644
--- a/drivers/net/wireless/ath/ath12k/htc.h
+++ b/drivers/net/wireless/ath/ath12k/htc.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: BSD-3-Clause-Clear */
 /*
  * Copyright (c) 2018-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2021-2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #ifndef ATH12K_HTC_H
@@ -301,7 +301,7 @@ struct ath12k_htc {
 	u8 wmi_ep_count;
 };
 
-int ath12k_htc_init(struct ath12k_base *ar);
+int ath12k_htc_init(struct ath12k_base *ab);
 int ath12k_htc_wait_target(struct ath12k_htc *htc);
 int ath12k_htc_start(struct ath12k_htc *htc);
 int ath12k_htc_connect_service(struct ath12k_htc *htc,
@@ -309,8 +309,8 @@ int ath12k_htc_connect_service(struct ath12k_htc *htc,
 			       struct ath12k_htc_svc_conn_resp *conn_resp);
 int ath12k_htc_send(struct ath12k_htc *htc, enum ath12k_htc_ep_id eid,
 		    struct sk_buff *packet);
-struct sk_buff *ath12k_htc_alloc_skb(struct ath12k_base *ar, int size);
-void ath12k_htc_rx_completion_handler(struct ath12k_base *ar,
+struct sk_buff *ath12k_htc_alloc_skb(struct ath12k_base *ab, int size);
+void ath12k_htc_rx_completion_handler(struct ath12k_base *ab,
 				      struct sk_buff *skb);
 
 #endif
diff --git a/drivers/net/wireless/ath/ath12k/mac.c b/drivers/net/wireless/ath/ath12k/mac.c
index df2334f3bad64..87b27f7cff5d6 100644
--- a/drivers/net/wireless/ath/ath12k/mac.c
+++ b/drivers/net/wireless/ath/ath12k/mac.c
@@ -51,6 +51,9 @@
 	.max_power              = 30, \
 }
 
+#define ATH12K_5_9_GHZ_MIN_FREQ 5845
+#define ATH12K_5_9_GHZ_MAX_FREQ 5885
+
 static const struct ieee80211_channel ath12k_2ghz_channels[] = {
 	CHAN2G(1, 2412, 0),
 	CHAN2G(2, 2417, 0),
@@ -96,6 +99,7 @@ static const struct ieee80211_channel ath12k_5ghz_channels[] = {
 	CHAN5G(165, 5825, 0),
 	CHAN5G(169, 5845, 0),
 	CHAN5G(173, 5865, 0),
+	CHAN5G(177, 5885, 0),
 };
 
 static const struct ieee80211_channel ath12k_6ghz_channels[] = {
@@ -3446,7 +3450,9 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar,
 		arg->peer_eht_mcs_count++;
 		fallthrough;
 	default:
-		if (!(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
+		if ((vif->type == NL80211_IFTYPE_AP ||
+		     vif->type == NL80211_IFTYPE_MESH_POINT) &&
+		    !(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
 		      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
 			bw_20 = &eht_cap->eht_mcs_nss_supp.only_20mhz;
 
@@ -3475,7 +3481,9 @@ static void ath12k_peer_assoc_h_eht(struct ath12k *ar,
 	arg->punct_bitmap = ~arvif->punct_bitmap;
 	arg->eht_disable_mcs15 = link_conf->eht_disable_mcs15;
 
-	if (!(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
+	if ((vif->type == NL80211_IFTYPE_AP ||
+	     vif->type == NL80211_IFTYPE_MESH_POINT) &&
+	    !(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
 	      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
 		if (bw_20->rx_tx_mcs13_max_nss)
 			max_nss = max(max_nss, u8_get_bits(bw_20->rx_tx_mcs13_max_nss,
@@ -5611,12 +5619,14 @@ static int ath12k_mac_initiate_hw_scan(struct ieee80211_hw *hw,
 	if (ret)
 		goto exit;
 
-	arg = kzalloc_obj(*arg);
+	arg = kzalloc_flex(*arg, chan_list, n_channels);
 	if (!arg) {
 		ret = -ENOMEM;
 		goto exit;
 	}
 
+	arg->num_chan = n_channels;
+
 	ath12k_wmi_start_scan_init(ar, arg);
 	arg->vdev_id = arvif->vdev_id;
 	arg->scan_id = ATH12K_SCAN_ID;
@@ -5638,18 +5648,8 @@ static int ath12k_mac_initiate_hw_scan(struct ieee80211_hw *hw,
 		arg->scan_f_passive = 1;
 	}
 
-	if (n_channels) {
-		arg->num_chan = n_channels;
-		arg->chan_list = kcalloc(arg->num_chan, sizeof(*arg->chan_list),
-					 GFP_KERNEL);
-		if (!arg->chan_list) {
-			ret = -ENOMEM;
-			goto exit;
-		}
-
-		for (i = 0; i < arg->num_chan; i++)
-			arg->chan_list[i] = chan_list[i]->center_freq;
-	}
+	for (i = 0; i < arg->num_chan; i++)
+		arg->chan_list[i] = chan_list[i]->center_freq;
 
 	ret = ath12k_start_scan(ar, arg);
 	if (ret) {
@@ -5674,7 +5674,6 @@ static int ath12k_mac_initiate_hw_scan(struct ieee80211_hw *hw,
 
 exit:
 	if (arg) {
-		kfree(arg->chan_list);
 		kfree(arg->extraie.ptr);
 		kfree(arg);
 	}
@@ -9646,6 +9645,10 @@ static int ath12k_mac_start(struct ath12k *ar)
 	ar->allocated_vdev_map = 0;
 	ar->chan_tx_pwr = ATH12K_PDEV_TX_POWER_INVALID;
 
+	spin_lock_bh(&ar->data_lock);
+	ar->incumbent_signal_interference.handling_in_progress = false;
+	spin_unlock_bh(&ar->data_lock);
+
 	/* Configure monitor status ring with default rx_filter to get rx status
 	 * such as rssi, rx_duration.
 	 */
@@ -9673,6 +9676,12 @@ static int ath12k_mac_start(struct ath12k *ar)
 		}
 	}
 
+	ret = ath12k_thermal_throttling_config_default(ar);
+	if (ret) {
+		ath12k_err(ab, "failed to set thermal throttle: %d\n", ret);
+		goto err;
+	}
+
 	rcu_assign_pointer(ab->pdevs_active[ar->pdev_idx],
 			   &ab->pdevs[ar->pdev_idx]);
 
@@ -9853,6 +9862,10 @@ static void ath12k_mac_stop(struct ath12k *ar)
 	synchronize_rcu();
 
 	atomic_set(&ar->num_pending_mgmt_tx, 0);
+
+	spin_lock_bh(&ar->data_lock);
+	ar->incumbent_signal_interference.handling_in_progress = false;
+	spin_unlock_bh(&ar->data_lock);
 }
 
 void ath12k_mac_op_stop(struct ieee80211_hw *hw, bool suspend)
@@ -11439,8 +11452,10 @@ ath12k_mac_update_vif_chan(struct ath12k *ar,
 			   struct ieee80211_vif_chanctx_switch *vifs,
 			   int n_vifs)
 {
+	struct ath12k_incumbent_signal_interference *incumbent;
 	struct ath12k_wmi_vdev_up_params params = {};
 	struct ieee80211_bss_conf *link_conf;
+	struct cfg80211_chan_def *chandef;
 	struct ath12k_base *ab = ar->ab;
 	struct ath12k_link_vif *arvif;
 	struct ieee80211_vif *vif;
@@ -11552,6 +11567,42 @@ ath12k_mac_update_vif_chan(struct ath12k *ar,
 		if (!ath12k_mac_monitor_stop(ar))
 			ath12k_mac_monitor_start(ar);
 	}
+
+	incumbent = &ar->incumbent_signal_interference;
+	spin_lock_bh(&ar->data_lock);
+	if (incumbent->handling_in_progress) {
+		chandef = &vifs[0].new_ctx->def;
+		if (incumbent->chan_bw_interference_bitmap &
+		    ATH12K_WMI_DCS_SEG_PRI20) {
+			if (incumbent->center_freq !=
+			    chandef->chan->center_freq) {
+				incumbent->chan_bw_interference_bitmap = 0;
+				incumbent->handling_in_progress = false;
+				ath12k_dbg(ab, ATH12K_DBG_MAC,
+					   "incumbent signal interference chan switch completed\n");
+			} else {
+				ath12k_warn(ab,
+					    "incumbent signal interference chan switch not done, freq %u\n",
+					    incumbent->center_freq);
+			}
+		} else {
+			if (incumbent->center_freq !=
+			    chandef->chan->center_freq ||
+			    incumbent->width != chandef->width) {
+				incumbent->chan_bw_interference_bitmap = 0;
+				incumbent->handling_in_progress = false;
+				ath12k_dbg(ab, ATH12K_DBG_MAC,
+					   "Bandwidth/channel change due to incumbent signal interference completed\n");
+			} else {
+				ath12k_warn(ab, "Bandwidth/channel change due to incumbent sig intf not done intf_freq %u chan_freq %u intf_width %u chan_width %u\n",
+					    incumbent->center_freq,
+					    chandef->chan->center_freq,
+					    incumbent->width,
+					    chandef->width);
+			}
+		}
+	}
+	spin_unlock_bh(&ar->data_lock);
 }
 
 static void
@@ -13735,19 +13786,13 @@ int ath12k_mac_op_remain_on_channel(struct ieee80211_hw *hw,
 	scan_time_msec = hw->wiphy->max_remain_on_channel_duration * 2;
 
 	struct ath12k_wmi_scan_req_arg *arg __free(kfree) =
-					kzalloc_obj(*arg);
+					kzalloc_flex(*arg, chan_list, 1);
 	if (!arg)
 		return -ENOMEM;
 
-	ath12k_wmi_start_scan_init(ar, arg);
 	arg->num_chan = 1;
+	ath12k_wmi_start_scan_init(ar, arg);
 
-	u32 *chan_list __free(kfree) = kcalloc(arg->num_chan, sizeof(*chan_list),
-					       GFP_KERNEL);
-	if (!chan_list)
-		return -ENOMEM;
-
-	arg->chan_list = chan_list;
 	arg->vdev_id = arvif->vdev_id;
 	arg->scan_id = ATH12K_SCAN_ID;
 	arg->chan_list[0] = chan->center_freq;
@@ -13900,6 +13945,26 @@ static int ath12k_mac_update_band(struct ath12k *ar,
 	return 0;
 }
 
+static void ath12k_mac_update_5_9_ghz_ch_list(struct ath12k *ar,
+					      struct ieee80211_supported_band *band)
+{
+	int i;
+
+	if (test_bit(WMI_TLV_SERVICE_5_9GHZ_SUPPORT,
+		     ar->ab->wmi_ab.svc_map))
+		return;
+
+	guard(spinlock_bh)(&ar->ab->base_lock);
+	if (ar->ab->dfs_region != ATH12K_DFS_REG_FCC)
+		return;
+
+	for (i = 0; i < band->n_channels; i++) {
+		if (band->channels[i].center_freq >= ATH12K_5_9_GHZ_MIN_FREQ &&
+		    band->channels[i].center_freq <= ATH12K_5_9_GHZ_MAX_FREQ)
+			band->channels[i].flags |= IEEE80211_CHAN_DISABLED;
+	}
+}
+
 static int ath12k_mac_setup_channels_rates(struct ath12k *ar,
 					   u32 supported_bands,
 					   struct ieee80211_supported_band *bands[])
@@ -14033,6 +14098,8 @@ static int ath12k_mac_setup_channels_rates(struct ath12k *ar,
 			band->n_bitrates = ath12k_a_rates_size;
 			band->bitrates = ath12k_a_rates;
 
+			ath12k_mac_update_5_9_ghz_ch_list(ar, band);
+
 			if (ab->hw_params->single_pdev_only) {
 				phy_id = ath12k_get_phy_id(ar, WMI_HOST_WLAN_5GHZ_CAP);
 				reg_cap = &ab->hal_reg_cap[phy_id];
@@ -14461,6 +14528,8 @@ static int ath12k_mac_setup_register(struct ath12k *ar,
 	ar->rssi_info.temp_offset = 0;
 	ar->rssi_info.noise_floor = ar->rssi_info.min_nf_dbm + ar->rssi_info.temp_offset;
 
+	ath12k_thermal_init_configs(ar);
+
 	return 0;
 }
 
@@ -14813,6 +14882,7 @@ static void ath12k_mac_setup(struct ath12k *ar)
 	init_completion(&ar->completed_11d_scan);
 	init_completion(&ar->regd_update_completed);
 	init_completion(&ar->thermal.wmi_sync);
+	mutex_init(&ar->thermal.lock);
 
 	ar->thermal.temperature = 0;
 	ar->thermal.hwmon_dev = NULL;
diff --git a/drivers/net/wireless/ath/ath12k/qmi.c b/drivers/net/wireless/ath/ath12k/qmi.c
index 8c5dacf227da5..fd762b5d7bb59 100644
--- a/drivers/net/wireless/ath/ath12k/qmi.c
+++ b/drivers/net/wireless/ath/ath12k/qmi.c
@@ -4061,7 +4061,7 @@ int ath12k_qmi_init_service(struct ath12k_base *ab)
 	spin_lock_init(&ab->qmi.event_lock);
 	INIT_WORK(&ab->qmi.event_work, ath12k_qmi_driver_event_work);
 
-	ret = qmi_add_lookup(&ab->qmi.handle, ATH12K_QMI_WLFW_SERVICE_ID_V01,
+	ret = qmi_add_lookup(&ab->qmi.handle, QMI_SERVICE_ID_WLFW,
 			     ATH12K_QMI_WLFW_SERVICE_VERS_V01,
 			     ab->qmi.service_ins_id);
 	if (ret < 0) {
diff --git a/drivers/net/wireless/ath/ath12k/qmi.h b/drivers/net/wireless/ath/ath12k/qmi.h
index b5a4a01391cba..2a63e214eb42c 100644
--- a/drivers/net/wireless/ath/ath12k/qmi.h
+++ b/drivers/net/wireless/ath/ath12k/qmi.h
@@ -15,7 +15,6 @@
 #define ATH12K_QMI_MAX_BDF_FILE_NAME_SIZE	64
 #define ATH12K_QMI_CALDB_ADDRESS		0x4BA00000
 #define ATH12K_QMI_WLANFW_MAX_BUILD_ID_LEN_V01	128
-#define ATH12K_QMI_WLFW_SERVICE_ID_V01		0x45
 #define ATH12K_QMI_WLFW_SERVICE_VERS_V01	0x01
 #define ATH12K_QMI_WLFW_SERVICE_INS_ID_V01	0x02
 #define ATH12K_QMI_WLFW_SERVICE_INS_ID_V01_WCN7850 0x1
diff --git a/drivers/net/wireless/ath/ath12k/thermal.c b/drivers/net/wireless/ath/ath12k/thermal.c
index a764d2112a3cf..97fc49c40ac15 100644
--- a/drivers/net/wireless/ath/ath12k/thermal.c
+++ b/drivers/net/wireless/ath/ath12k/thermal.c
@@ -12,6 +12,137 @@
 #include "core.h"
 #include "debug.h"
 
+static const struct ath12k_wmi_tt_level_config_param
+tt_level_configs[ATH12K_TT_CFG_IDX_MAX][ENHANCED_THERMAL_LEVELS] = {
+	[ATH12K_TT_CFG_IDX_IPA] = {
+		[0] = {	.tmplwm = -100, .tmphwm = 115, .dcoffpercent = 0,
+			.pout_reduction_db = 0 },
+		[1] = { .tmplwm = 110, .tmphwm = 120, .dcoffpercent = 0,
+			.pout_reduction_db = 12	},
+		[2] = { .tmplwm = 115, .tmphwm = 125, .dcoffpercent = 50,
+			.pout_reduction_db = 12	},
+		[3] = { .tmplwm = 120, .tmphwm = 130, .dcoffpercent = 90,
+			.pout_reduction_db = 12	},
+		[4] = { .tmplwm = 125, .tmphwm = 130, .dcoffpercent = 100,
+			.pout_reduction_db = 12	},
+	},
+	[ATH12K_TT_CFG_IDX_XFEM] = {
+		[0] = {	.tmplwm = -100,	.tmphwm = 105, .dcoffpercent = 0,
+			.pout_reduction_db = 0 },
+		[1] = { .tmplwm = 100, .tmphwm = 110, .dcoffpercent = 0,
+			.pout_reduction_db = 0 },
+		[2] = { .tmplwm = 105, .tmphwm = 115, .dcoffpercent = 50,
+			.pout_reduction_db = 0 },
+		[3] = {	.tmplwm = 110, .tmphwm = 120, .dcoffpercent = 90,
+			.pout_reduction_db = 0 },
+		[4] = { .tmplwm = 115, .tmphwm = 120, .dcoffpercent = 100,
+			.pout_reduction_db = 0 },
+	},
+};
+
+static enum ath12k_thermal_cfg_idx ath12k_thermal_cfg_index(struct ath12k *ar)
+{
+	if (test_bit(WMI_TLV_SERVICE_IS_TARGET_IPA, ar->ab->wmi_ab.svc_map))
+		return ATH12K_TT_CFG_IDX_IPA;
+
+	return ATH12K_TT_CFG_IDX_XFEM;
+}
+
+int ath12k_thermal_throttling_config_default(struct ath12k *ar)
+{
+	struct ath12k_wmi_thermal_mitigation_arg param = {};
+	int ret;
+
+	if (test_bit(WMI_TLV_SERVICE_THERM_THROT_5_LEVELS, ar->ab->wmi_ab.svc_map))
+		param.num_levels = ENHANCED_THERMAL_LEVELS;
+	else
+		param.num_levels = THERMAL_LEVELS;
+
+	param.levelconf = ar->thermal.tt_level_configs;
+
+	ret = ath12k_wmi_send_thermal_mitigation_cmd(ar, &param);
+	if (ret)
+		ath12k_warn(ar->ab,
+			    "failed to send thermal mitigation cmd for default config: %d\n",
+			    ret);
+	return ret;
+}
+
+void ath12k_thermal_init_configs(struct ath12k *ar)
+{
+	enum ath12k_thermal_cfg_idx cfg_idx;
+
+	cfg_idx = ath12k_thermal_cfg_index(ar);
+	ar->thermal.tt_level_configs = &tt_level_configs[cfg_idx][0];
+}
+
+static int
+ath12k_thermal_get_max_throttle_state(struct thermal_cooling_device *cdev,
+				      unsigned long *state)
+{
+	*state = ATH12K_THERMAL_THROTTLE_MAX;
+
+	return 0;
+}
+
+static int
+ath12k_thermal_get_cur_throttle_state(struct thermal_cooling_device *cdev,
+				      unsigned long *state)
+{
+	struct ath12k *ar = cdev->devdata;
+
+	mutex_lock(&ar->thermal.lock);
+	*state = ar->thermal.throttle_state;
+	mutex_unlock(&ar->thermal.lock);
+
+	return 0;
+}
+
+int ath12k_thermal_set_throttling(struct ath12k *ar, u32 throttle_state)
+{
+	struct ath12k_wmi_thermal_mitigation_arg param = {};
+	struct ath12k_wmi_tt_level_config_param cfg = {};
+	int ret;
+
+	param.num_levels = 1;
+	cfg.dcoffpercent = throttle_state;
+	param.levelconf = &cfg;
+
+	ret = ath12k_wmi_send_thermal_mitigation_cmd(ar, &param);
+	if (ret)
+		ath12k_warn(ar->ab, "failed to send thermal mitigation cmd: %d\n",
+			    ret);
+
+	return ret;
+}
+
+static int
+ath12k_thermal_set_cur_throttle_state(struct thermal_cooling_device *cdev,
+				      unsigned long throttle_state)
+{
+	struct ath12k *ar = cdev->devdata;
+
+	if (throttle_state > ATH12K_THERMAL_THROTTLE_MAX)
+		return -EINVAL;
+
+	scoped_guard(mutex, &ar->thermal.lock) {
+		if (ar->thermal.throttle_state == throttle_state)
+			return 0;
+		ar->thermal.throttle_state = throttle_state;
+	}
+
+	if (throttle_state == 0)
+		return ath12k_thermal_throttling_config_default(ar);
+
+	return ath12k_thermal_set_throttling(ar, throttle_state);
+}
+
+static const struct thermal_cooling_device_ops ath12k_thermal_ops = {
+	.get_max_state = ath12k_thermal_get_max_throttle_state,
+	.get_cur_state = ath12k_thermal_get_cur_throttle_state,
+	.set_cur_state = ath12k_thermal_set_cur_throttle_state,
+};
+
 static ssize_t ath12k_thermal_temp_show(struct device *dev,
 					struct device_attribute *attr,
 					char *buf)
@@ -66,59 +197,108 @@ static struct attribute *ath12k_hwmon_attrs[] = {
 };
 ATTRIBUTE_GROUPS(ath12k_hwmon);
 
-int ath12k_thermal_register(struct ath12k_base *ab)
+static int ath12k_thermal_setup_radio(struct ath12k_base *ab, int i)
 {
+	char pdev_name[20];
 	struct ath12k *ar;
-	int i, j, ret;
+	int ret;
+
+	ar = ab->pdevs[i].ar;
+	if (!ar)
+		return 0;
+
+	ar->thermal.cdev =
+		thermal_cooling_device_register("ath12k_thermal", ar,
+						&ath12k_thermal_ops);
+	if (IS_ERR(ar->thermal.cdev)) {
+		ret = PTR_ERR(ar->thermal.cdev);
+		ar->thermal.cdev = NULL;
+		ath12k_err(ar->ab, "failed to register cooling device: %d\n",
+			   ret);
+		return ret;
+	}
+
+	scnprintf(pdev_name, sizeof(pdev_name), "cooling_device%u",
+		  ar->hw_link_id);
+
+	ret = sysfs_create_link(&ar->ah->hw->wiphy->dev.kobj,
+				&ar->thermal.cdev->device.kobj, pdev_name);
+	if (ret) {
+		ath12k_err(ab, "failed to create cooling device symlink: %d\n",
+			   ret);
+		goto unregister_cdev;
+	}
+
+	ar->thermal.hwmon_dev =
+		hwmon_device_register_with_groups(&ar->ah->hw->wiphy->dev,
+						  "ath12k_hwmon", ar,
+						  ath12k_hwmon_groups);
+	if (IS_ERR(ar->thermal.hwmon_dev)) {
+		ret = PTR_ERR(ar->thermal.hwmon_dev);
+		ar->thermal.hwmon_dev = NULL;
+		ath12k_err(ar->ab, "failed to register hwmon device: %d\n",
+			   ret);
+		goto remove_sysfs;
+	}
+
+	return 0;
+
+remove_sysfs:
+	sysfs_remove_link(&ar->ah->hw->wiphy->dev.kobj, pdev_name);
+unregister_cdev:
+	thermal_cooling_device_unregister(ar->thermal.cdev);
+	ar->thermal.cdev = NULL;
+	return ret;
+}
+
+static void ath12k_thermal_cleanup_radio(struct ath12k_base *ab, int i)
+{
+	char pdev_name[20];
+	struct ath12k *ar;
+
+	ar = ab->pdevs[i].ar;
+	if (!ar)
+		return;
+
+	hwmon_device_unregister(ar->thermal.hwmon_dev);
+	ar->thermal.hwmon_dev = NULL;
+
+	scnprintf(pdev_name, sizeof(pdev_name), "cooling_device%u",
+		  ar->hw_link_id);
+	sysfs_remove_link(&ar->ah->hw->wiphy->dev.kobj, pdev_name);
+
+	thermal_cooling_device_unregister(ar->thermal.cdev);
+	ar->thermal.cdev = NULL;
+}
+
+int ath12k_thermal_register(struct ath12k_base *ab)
+{
+	int i, ret;
 
 	if (!IS_REACHABLE(CONFIG_HWMON))
 		return 0;
 
 	for (i = 0; i < ab->num_radios; i++) {
-		ar = ab->pdevs[i].ar;
-		if (!ar)
-			continue;
-
-		ar->thermal.hwmon_dev =
-			hwmon_device_register_with_groups(&ar->ah->hw->wiphy->dev,
-							  "ath12k_hwmon", ar,
-							  ath12k_hwmon_groups);
-		if (IS_ERR(ar->thermal.hwmon_dev)) {
-			ret = PTR_ERR(ar->thermal.hwmon_dev);
-			ar->thermal.hwmon_dev = NULL;
-			ath12k_err(ar->ab, "failed to register hwmon device: %d\n",
-				   ret);
-			for (j = i - 1; j >= 0; j--) {
-				ar = ab->pdevs[j].ar;
-				if (!ar)
-					continue;
-
-				hwmon_device_unregister(ar->thermal.hwmon_dev);
-				ar->thermal.hwmon_dev = NULL;
-			}
-			return ret;
-		}
+		ret = ath12k_thermal_setup_radio(ab, i);
+		if (ret)
+			goto out;
 	}
 
 	return 0;
+out:
+	for (i--; i >= 0; i--)
+		ath12k_thermal_cleanup_radio(ab, i);
+
+	return ret;
 }
 
 void ath12k_thermal_unregister(struct ath12k_base *ab)
 {
-	struct ath12k *ar;
 	int i;
 
 	if (!IS_REACHABLE(CONFIG_HWMON))
 		return;
 
-	for (i = 0; i < ab->num_radios; i++) {
-		ar = ab->pdevs[i].ar;
-		if (!ar)
-			continue;
-
-		if (ar->thermal.hwmon_dev) {
-			hwmon_device_unregister(ar->thermal.hwmon_dev);
-			ar->thermal.hwmon_dev = NULL;
-		}
-	}
+	for (i = 0; i < ab->num_radios; i++)
+		ath12k_thermal_cleanup_radio(ab, i);
 }
diff --git a/drivers/net/wireless/ath/ath12k/thermal.h b/drivers/net/wireless/ath/ath12k/thermal.h
index 9d84056188e1e..30e7b0880e056 100644
--- a/drivers/net/wireless/ath/ath12k/thermal.h
+++ b/drivers/net/wireless/ath/ath12k/thermal.h
@@ -7,20 +7,41 @@
 #ifndef _ATH12K_THERMAL_
 #define _ATH12K_THERMAL_
 
+#include <linux/mutex.h>
+
 #define ATH12K_THERMAL_SYNC_TIMEOUT_HZ (5 * HZ)
 
+#define ATH12K_THERMAL_DEFAULT_DUTY_CYCLE 100
+#define ATH12K_THERMAL_THROTTLE_MAX 100
+
+enum ath12k_thermal_cfg_idx {
+	/* Internal Power Amplifier Device */
+	ATH12K_TT_CFG_IDX_IPA,
+	/* External Power Amplifier Device or External Front End Module */
+	ATH12K_TT_CFG_IDX_XFEM,
+	ATH12K_TT_CFG_IDX_MAX,
+};
+
 struct ath12k_thermal {
 	struct completion wmi_sync;
 
 	/* temperature value in Celsius degree protected by data_lock. */
 	int temperature;
 	struct device *hwmon_dev;
+	const struct ath12k_wmi_tt_level_config_param *tt_level_configs;
+	struct thermal_cooling_device *cdev;
+	/* Serialize thermal operations and hwmon reads */
+	struct mutex lock;
+	u32 throttle_state;
 };
 
 #if IS_REACHABLE(CONFIG_THERMAL)
 int ath12k_thermal_register(struct ath12k_base *ab);
 void ath12k_thermal_unregister(struct ath12k_base *ab);
 void ath12k_thermal_event_temperature(struct ath12k *ar, int temperature);
+int ath12k_thermal_throttling_config_default(struct ath12k *ar);
+void ath12k_thermal_init_configs(struct ath12k *ar);
+int ath12k_thermal_set_throttling(struct ath12k *ar, u32 throttle_state);
 #else
 static inline int ath12k_thermal_register(struct ath12k_base *ab)
 {
@@ -36,5 +57,19 @@ static inline void ath12k_thermal_event_temperature(struct ath12k *ar,
 {
 }
 
+static inline int ath12k_thermal_throttling_config_default(struct ath12k *ar)
+{
+	return 0;
+}
+
+static inline void ath12k_thermal_init_configs(struct ath12k *ar)
+{
+}
+
+static inline int ath12k_thermal_set_throttling(struct ath12k *ar,
+						u32 throttle_state)
+{
+	return 0;
+}
 #endif
 #endif /* _ATH12K_THERMAL_ */
diff --git a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h
index ac2a8ac03288f..dd60626b3240a 100644
--- a/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h
+++ b/drivers/net/wireless/ath/ath12k/wifi7/hal_rx.h
@@ -559,7 +559,6 @@ struct hal_eht_sig_ofdma_cmn_eb1 {
 #define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_4		GENMASK_ULL(35, 27)
 #define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_5		GENMASK_ULL(44, 36)
 #define HAL_RX_EHT_SIG_OFDMA_EB2_RU_ALLOC_2_6		GENMASK_ULL(53, 45)
-#define HAL_RX_EHT_SIG_OFDMA_EB2_MCS			GNEMASK_ULL(57, 54)
 
 struct hal_eht_sig_ofdma_cmn_eb2 {
 	__le64 info0;
diff --git a/drivers/net/wireless/ath/ath12k/wmi.c b/drivers/net/wireless/ath/ath12k/wmi.c
index b5e904a55aeab..18f91051199c4 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.c
+++ b/drivers/net/wireless/ath/ath12k/wmi.c
@@ -15,6 +15,8 @@
 #include <linux/time.h>
 #include <linux/of.h>
 #include <linux/cleanup.h>
+#include <linux/percpu.h>
+#include <linux/refcount.h>
 #include "core.h"
 #include "debugfs.h"
 #include "debug.h"
@@ -134,6 +136,10 @@ struct wmi_pdev_set_obss_bitmap_arg {
 	const char *label;
 };
 
+static DEFINE_MUTEX(ath12k_wmi_mutex);
+static refcount_t ath12k_wmi_refcount;
+static void __percpu *ath12k_wmi_tb;
+
 static const struct ath12k_wmi_tlv_policy ath12k_wmi_tlv_policies[] = {
 	[WMI_TAG_ARRAY_BYTE] = { .min_len = 0 },
 	[WMI_TAG_ARRAY_UINT32] = { .min_len = 0 },
@@ -228,6 +234,68 @@ static const int ath12k_hw_mode_pri_map[] = {
 	PRIMAP(WMI_HOST_HW_MODE_MAX),
 };
 
+/*
+ * Interference bitmap transform maps used by
+ * ath12k_wmi_transform_interference_bitmap().
+ *
+ * Firmware reports bitmap bits in a primary-based order where:
+ * - bit 0 is always the primary 20 MHz segment,
+ * - bit 1 is the adjacent 20 MHz in the same 40 MHz block,
+ * - bit 2 is the lower 20 MHz segment of the adjacent 40 MHz segment
+ * - bit 3 is the higher 20 MHz segment of the adjacent 40 MHz segment
+ * - remaining bits continue outward in 80/160/320 MHz groups.
+ *
+ * cfg80211 userspace notification expects absolute frequency order where:
+ * - bit 0 is the lowest-frequency 20 MHz segment in the current chandef,
+ * - bit N increases monotonically toward higher frequency.
+ *
+ * For each bandwidth-specific map:
+ * - row index    = primary 20 MHz index in absolute (low->high) order,
+ * - column index = source bit position from firmware bitmap,
+ * - value        = destination bit position in absolute order bitmap.
+ *
+ * Example for 80 MHz: if primary index is 2 (third 20 MHz chunk from low
+ * frequency), row intf_map_80[2] = { 2, 3, 0, 1 } means firmware bits {0,1,2,3}
+ * are remapped to destination bits {2,3,0,1} before notifying cfg80211.
+ */
+
+static const int intf_map_80[4][4] = {
+	{ 0, 1, 2, 3 },
+	{ 1, 0, 2, 3 },
+	{ 2, 3, 0, 1 },
+	{ 3, 2, 0, 1 }
+};
+
+static const int intf_map_160[8][8] = {
+	{ 0, 1, 2, 3, 4, 5, 6, 7 },
+	{ 1, 0, 2, 3, 4, 5, 6, 7 },
+	{ 2, 3, 0, 1, 4, 5, 6, 7 },
+	{ 3, 2, 0, 1, 4, 5, 6, 7 },
+	{ 4, 5, 6, 7, 0, 1, 2, 3 },
+	{ 5, 4, 6, 7, 0, 1, 2, 3 },
+	{ 6, 7, 4, 5, 0, 1, 2, 3 },
+	{ 7, 6, 4, 5, 0, 1, 2, 3 }
+};
+
+static const int intf_map_320[16][16] = {
+	{ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10,  11,  12,  13,  14,  15 },
+	{ 1,  0,  2,  3,  4,  5,  6,  7,  8,  9,  10,  11,  12,  13,  14,  15 },
+	{ 2,  3,  0,  1,  4,  5,  6,  7,  8,  9,  10,  11,  12,  13,  14,  15 },
+	{ 3,  2,  0,  1,  4,  5,  6,  7,  8,  9,  10,  11,  12,  13,  14,  15 },
+	{ 4,  5,  6,  7,  0,  1,  2,  3,  8,  9,  10,  11,  12,  13,  14,  15 },
+	{ 5,  4,  6,  7,  0,  1,  2,  3,  8,  9,  10,  11,  12,  13,  14,  15 },
+	{ 6,  7,  4,  5,  0,  1,  2,  3,  8,  9,  10,  11,  12,  13,  14,  15 },
+	{ 7,  6,  4,  5,  0,  1,  2,  3,  8,  9,  10,  11,  12,  13,  14,  15 },
+	{ 8,  9,  10, 11, 12, 13, 14, 15, 0,  1,  2,   3,   4,   5,   6,   7  },
+	{ 9,  8,  10, 11, 12, 13, 14, 15, 0,  1,  2,   3,   4,   5,   6,   7  },
+	{ 10, 11, 8,  9,  12, 13, 14, 15, 0,  1,  2,   3,   4,   5,   6,   7  },
+	{ 11, 10, 8,  9,  12, 13, 14, 15, 0,  1,  2,   3,   4,   5,   6,   7  },
+	{ 12, 13, 14, 15, 8,  9,  10, 11, 0,  1,  2,   3,   4,   5,   6,   7  },
+	{ 13, 12, 14, 15, 8,  9,  10, 11, 0,  1,  2,   3,   4,   5,   6,   7  },
+	{ 14, 15, 12, 13, 8,  9,  10, 11, 0,  1,  2,   3,   4,   5,   6,   7  },
+	{ 15, 14, 12, 13, 8,  9,  10, 11, 0,  1,  2,   3,   4,   5,   6,   7  }
+};
+
 static int
 ath12k_wmi_tlv_iter(struct ath12k_base *ab, const void *ptr, size_t len,
 		    int (*iter)(struct ath12k_base *ab, u16 tag, u16 len,
@@ -289,29 +357,19 @@ static int ath12k_wmi_tlv_iter_parse(struct ath12k_base *ab, u16 tag, u16 len,
 	return 0;
 }
 
-static int ath12k_wmi_tlv_parse(struct ath12k_base *ar, const void **tb,
-				const void *ptr, size_t len)
-{
-	return ath12k_wmi_tlv_iter(ar, ptr, len, ath12k_wmi_tlv_iter_parse,
-				   (void *)tb);
-}
-
 static const void **
-ath12k_wmi_tlv_parse_alloc(struct ath12k_base *ab,
-			   struct sk_buff *skb, gfp_t gfp)
+ath12k_wmi_tlv_parse(struct ath12k_base *ab, struct sk_buff *skb)
 {
 	const void **tb;
 	int ret;
 
-	tb = kzalloc_objs(*tb, WMI_TAG_MAX, gfp);
-	if (!tb)
-		return ERR_PTR(-ENOMEM);
+	tb = this_cpu_ptr(ath12k_wmi_tb);
+	memset(tb, 0, WMI_TAG_MAX * sizeof(*tb));
 
-	ret = ath12k_wmi_tlv_parse(ab, tb, skb->data, skb->len);
-	if (ret) {
-		kfree(tb);
+	ret = ath12k_wmi_tlv_iter(ab, skb->data, skb->len,
+				  ath12k_wmi_tlv_iter_parse, (void *)tb);
+	if (ret)
 		return ERR_PTR(ret);
-	}
 
 	return tb;
 }
@@ -3384,6 +3442,75 @@ int ath12k_wmi_send_set_current_country_cmd(struct ath12k *ar,
 	return ret;
 }
 
+int
+ath12k_wmi_send_thermal_mitigation_cmd(struct ath12k *ar,
+				       struct ath12k_wmi_thermal_mitigation_arg *arg)
+{
+	struct ath12k_wmi_therm_throt_level_config_param *lvl_conf;
+	struct ath12k_wmi_therm_throt_config_request_cmd *cmd;
+	struct ath12k_wmi_pdev *wmi = ar->wmi;
+	struct wmi_tlv *tlv;
+	struct sk_buff *skb;
+	int i, ret, len;
+
+	len = sizeof(*cmd) + TLV_HDR_SIZE + (arg->num_levels * sizeof(*lvl_conf));
+
+	skb = ath12k_wmi_alloc_skb(wmi->wmi_ab, len);
+	if (!skb)
+		return -ENOMEM;
+
+	cmd = (struct ath12k_wmi_therm_throt_config_request_cmd *)skb->data;
+	cmd->tlv_header = ath12k_wmi_tlv_cmd_hdr(WMI_TAG_THERM_THROT_CONFIG_REQUEST,
+						 sizeof(*cmd));
+	cmd->pdev_id = cpu_to_le32(ar->pdev->pdev_id);
+	cmd->enable = cpu_to_le32(1);
+	cmd->dc = cpu_to_le32(100);
+	cmd->dc_per_event = cpu_to_le32(0xffffffff);
+	cmd->therm_throt_levels = cpu_to_le32(arg->num_levels);
+
+	tlv = (struct wmi_tlv *)(skb->data + sizeof(*cmd));
+	tlv->header = ath12k_wmi_tlv_hdr(WMI_TAG_ARRAY_STRUCT,
+					 arg->num_levels * sizeof(*lvl_conf));
+
+	lvl_conf = (struct ath12k_wmi_therm_throt_level_config_param *)tlv->value;
+
+	for (i = 0; i < arg->num_levels; i++) {
+		lvl_conf->tlv_header =
+			ath12k_wmi_tlv_cmd_hdr(WMI_TAG_THERM_THROT_LEVEL_CONFIG_INFO,
+					       sizeof(*lvl_conf));
+
+		lvl_conf->temp_lwm = a_cpu_to_sle32(arg->levelconf[i].tmplwm);
+		lvl_conf->temp_hwm = a_cpu_to_sle32(arg->levelconf[i].tmphwm);
+		lvl_conf->dc_off_percent = cpu_to_le32(arg->levelconf[i].dcoffpercent);
+
+		if (test_bit(WMI_TLV_SERVICE_THERM_THROT_POUT_REDUCTION,
+			     ar->ab->wmi_ab.svc_map))
+			lvl_conf->pout_reduction_25db =
+				cpu_to_le32(arg->levelconf[i].pout_reduction_db);
+
+		if (test_bit(WMI_TLV_SERVICE_THERM_THROT_TX_CHAIN_MASK,
+			     ar->ab->wmi_ab.svc_map))
+			lvl_conf->tx_chain_mask = cpu_to_le32(ar->cfg_tx_chainmask);
+
+		lvl_conf->duty_cycle = cpu_to_le32(ATH12K_THERMAL_DEFAULT_DUTY_CYCLE);
+		lvl_conf++;
+	}
+
+	ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
+		   "WMI vdev set thermal throt pdev_id %u enable dc 100 dc_per_event 0xffffffff levels %d\n",
+		   ar->pdev->pdev_id, arg->num_levels);
+
+	ret = ath12k_wmi_cmd_send(wmi, skb, WMI_THERM_THROT_SET_CONF_CMDID);
+	if (ret) {
+		ath12k_warn(ar->ab,
+			    "failed to send WMI_THERM_THROT_SET_CONF cmd: %d\n",
+			    ret);
+		dev_kfree_skb(skb);
+	}
+
+	return ret;
+}
+
 int ath12k_wmi_send_11d_scan_start_cmd(struct ath12k *ar,
 				       struct wmi_11d_scan_start_arg *arg)
 {
@@ -3911,9 +4038,10 @@ ath12k_wmi_obss_color_collision_event(struct ath12k_base *ab, struct sk_buff *sk
 	const struct wmi_obss_color_collision_event *ev;
 	struct ath12k_link_vif *arvif;
 	u32 vdev_id, evt_type;
+	const void **tb;
 	u64 bitmap;
 
-	const void **tb __free(kfree) = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ath12k_warn(ab, "failed to parse OBSS color collision tlv %ld\n",
 			    PTR_ERR(tb));
@@ -4202,12 +4330,9 @@ int ath12k_wmi_pdev_lro_cfg(struct ath12k *ar,
 	if (ret) {
 		ath12k_warn(ar->ab,
 			    "failed to send lro cfg req wmi cmd\n");
-		goto err;
+		dev_kfree_skb(skb);
 	}
 
-	return 0;
-err:
-	dev_kfree_skb(skb);
 	return ret;
 }
 
@@ -4338,12 +4463,9 @@ int ath12k_wmi_vdev_spectral_conf(struct ath12k *ar,
 	if (ret) {
 		ath12k_warn(ar->ab,
 			    "failed to send spectral scan config wmi cmd\n");
-		goto err;
+		dev_kfree_skb(skb);
 	}
 
-	return 0;
-err:
-	dev_kfree_skb(skb);
 	return ret;
 }
 
@@ -4375,12 +4497,9 @@ int ath12k_wmi_vdev_spectral_enable(struct ath12k *ar, u32 vdev_id,
 	if (ret) {
 		ath12k_warn(ar->ab,
 			    "failed to send spectral enable wmi cmd\n");
-		goto err;
+		dev_kfree_skb(skb);
 	}
 
-	return 0;
-err:
-	dev_kfree_skb(skb);
 	return ret;
 }
 
@@ -4421,12 +4540,9 @@ int ath12k_wmi_pdev_dma_ring_cfg(struct ath12k *ar,
 	if (ret) {
 		ath12k_warn(ar->ab,
 			    "failed to send dma ring cfg req wmi cmd\n");
-		goto err;
+		dev_kfree_skb(skb);
 	}
 
-	return 0;
-err:
-	dev_kfree_skb(skb);
 	return ret;
 }
 
@@ -5714,7 +5830,7 @@ static int ath12k_pull_vdev_start_resp_tlv(struct ath12k_base *ab, struct sk_buf
 	const struct wmi_vdev_start_resp_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -5724,13 +5840,11 @@ static int ath12k_pull_vdev_start_resp_tlv(struct ath12k_base *ab, struct sk_buf
 	ev = tb[WMI_TAG_VDEV_START_RESPONSE_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch vdev start resp ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
 	*vdev_rsp = *ev;
 
-	kfree(tb);
 	return 0;
 }
 
@@ -5809,7 +5923,7 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 
 	ath12k_dbg(ab, ATH12K_DBG_WMI, "processing regulatory ext channel list\n");
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -5819,7 +5933,6 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_REG_CHAN_LIST_CC_EXT_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch reg chan list ext update ev\n");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -5849,7 +5962,6 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 	if (num_2g_reg_rules > MAX_REG_RULES || num_5g_reg_rules > MAX_REG_RULES) {
 		ath12k_warn(ab, "Num reg rules for 2G/5G exceeds max limit (num_2g_reg_rules: %d num_5g_reg_rules: %d max_rules: %d)\n",
 			    num_2g_reg_rules, num_5g_reg_rules, MAX_REG_RULES);
-		kfree(tb);
 		return -EINVAL;
 	}
 
@@ -5859,7 +5971,6 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 		if (num_6g_reg_rules_ap[i] > MAX_6GHZ_REG_RULES) {
 			ath12k_warn(ab, "Num 6G reg rules for AP mode(%d) exceeds max limit (num_6g_reg_rules_ap: %d, max_rules: %d)\n",
 				    i, num_6g_reg_rules_ap[i], MAX_6GHZ_REG_RULES);
-			kfree(tb);
 			return -EINVAL;
 		}
 
@@ -5884,14 +5995,12 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 		    num_6g_reg_rules_cl[WMI_REG_VLP_AP][i] >  MAX_6GHZ_REG_RULES) {
 			ath12k_warn(ab, "Num 6g client reg rules exceeds max limit, for client(type: %d)\n",
 				    i);
-			kfree(tb);
 			return -EINVAL;
 		}
 	}
 
 	if (!total_reg_rules) {
 		ath12k_warn(ab, "No reg rules available\n");
-		kfree(tb);
 		return -EINVAL;
 	}
 
@@ -5993,7 +6102,6 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 						      ext_wmi_reg_rule);
 
 		if (!reg_info->reg_rules_2g_ptr) {
-			kfree(tb);
 			ath12k_warn(ab, "Unable to Allocate memory for 2g rules\n");
 			return -ENOMEM;
 		}
@@ -6027,7 +6135,6 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 						      ext_wmi_reg_rule);
 
 		if (!reg_info->reg_rules_5g_ptr) {
-			kfree(tb);
 			ath12k_warn(ab, "Unable to Allocate memory for 5g rules\n");
 			return -ENOMEM;
 		}
@@ -6046,7 +6153,6 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 						      ext_wmi_reg_rule);
 
 		if (!reg_info->reg_rules_6g_ap_ptr[i]) {
-			kfree(tb);
 			ath12k_warn(ab, "Unable to Allocate memory for 6g ap rules\n");
 			return -ENOMEM;
 		}
@@ -6061,7 +6167,6 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 							      ext_wmi_reg_rule);
 
 			if (!reg_info->reg_rules_6g_client_ptr[j][i]) {
-				kfree(tb);
 				ath12k_warn(ab, "Unable to Allocate memory for 6g client rules\n");
 				return -ENOMEM;
 			}
@@ -6096,7 +6201,6 @@ static int ath12k_pull_reg_chan_list_ext_update_ev(struct ath12k_base *ab,
 
 	ath12k_dbg(ab, ATH12K_DBG_WMI, "processed regulatory ext channel list\n");
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6107,7 +6211,7 @@ static int ath12k_pull_peer_del_resp_ev(struct ath12k_base *ab, struct sk_buff *
 	const struct wmi_peer_delete_resp_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6117,7 +6221,6 @@ static int ath12k_pull_peer_del_resp_ev(struct ath12k_base *ab, struct sk_buff *
 	ev = tb[WMI_TAG_PEER_DELETE_RESP_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch peer delete resp ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -6127,7 +6230,6 @@ static int ath12k_pull_peer_del_resp_ev(struct ath12k_base *ab, struct sk_buff *
 	ether_addr_copy(peer_del_resp->peer_macaddr.addr,
 			ev->peer_macaddr.addr);
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6139,7 +6241,7 @@ static int ath12k_pull_vdev_del_resp_ev(struct ath12k_base *ab,
 	const struct wmi_vdev_delete_resp_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6149,13 +6251,11 @@ static int ath12k_pull_vdev_del_resp_ev(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_VDEV_DELETE_RESP_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch vdev delete resp ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
 	*vdev_id = le32_to_cpu(ev->vdev_id);
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6167,7 +6267,7 @@ static int ath12k_pull_bcn_tx_status_ev(struct ath12k_base *ab,
 	const struct wmi_bcn_tx_status_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6177,14 +6277,12 @@ static int ath12k_pull_bcn_tx_status_ev(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_OFFLOAD_BCN_TX_STATUS_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch bcn tx status ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
 	*vdev_id = le32_to_cpu(ev->vdev_id);
 	*tx_status = le32_to_cpu(ev->tx_status);
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6195,7 +6293,7 @@ static int ath12k_pull_vdev_stopped_param_tlv(struct ath12k_base *ab, struct sk_
 	const struct wmi_vdev_stopped_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6205,13 +6303,11 @@ static int ath12k_pull_vdev_stopped_param_tlv(struct ath12k_base *ab, struct sk_
 	ev = tb[WMI_TAG_VDEV_STOPPED_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch vdev stop ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
 	*vdev_id = le32_to_cpu(ev->vdev_id);
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6350,7 +6446,7 @@ static int ath12k_pull_mgmt_tx_compl_param_tlv(struct ath12k_base *ab,
 	const struct wmi_mgmt_tx_compl_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6360,7 +6456,6 @@ static int ath12k_pull_mgmt_tx_compl_param_tlv(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_MGMT_TX_COMPL_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch mgmt tx compl ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -6370,7 +6465,6 @@ static int ath12k_pull_mgmt_tx_compl_param_tlv(struct ath12k_base *ab,
 	param->ppdu_id = ev->ppdu_id;
 	param->ack_rssi = ev->ack_rssi;
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6533,7 +6627,7 @@ static int ath12k_pull_scan_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	const struct wmi_scan_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6543,7 +6637,6 @@ static int ath12k_pull_scan_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	ev = tb[WMI_TAG_SCAN_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch scan ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -6555,7 +6648,6 @@ static int ath12k_pull_scan_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	scan_evt_param->vdev_id = ev->vdev_id;
 	scan_evt_param->tsf_timestamp = ev->tsf_timestamp;
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6566,7 +6658,7 @@ static int ath12k_pull_peer_sta_kickout_ev(struct ath12k_base *ab, struct sk_buf
 	const struct wmi_peer_sta_kickout_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6576,7 +6668,6 @@ static int ath12k_pull_peer_sta_kickout_ev(struct ath12k_base *ab, struct sk_buf
 	ev = tb[WMI_TAG_PEER_STA_KICKOUT_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch peer sta kickout ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -6584,7 +6675,6 @@ static int ath12k_pull_peer_sta_kickout_ev(struct ath12k_base *ab, struct sk_buf
 	arg->reason = le32_to_cpu(ev->reason);
 	arg->rssi = le32_to_cpu(ev->rssi);
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6595,7 +6685,7 @@ static int ath12k_pull_roam_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	const struct wmi_roam_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6605,7 +6695,6 @@ static int ath12k_pull_roam_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	ev = tb[WMI_TAG_ROAM_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch roam ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -6613,7 +6702,6 @@ static int ath12k_pull_roam_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	roam_ev->reason = ev->reason;
 	roam_ev->rssi = ev->rssi;
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6647,7 +6735,7 @@ static int ath12k_pull_chan_info_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	const struct wmi_chan_info_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6657,7 +6745,6 @@ static int ath12k_pull_chan_info_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	ev = tb[WMI_TAG_CHAN_INFO_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch chan info ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -6674,7 +6761,6 @@ static int ath12k_pull_chan_info_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	ch_info_ev->mac_clk_mhz = ev->mac_clk_mhz;
 	ch_info_ev->vdev_id = ev->vdev_id;
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6686,7 +6772,7 @@ ath12k_pull_pdev_bss_chan_info_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	const struct wmi_pdev_bss_chan_info_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6696,7 +6782,6 @@ ath12k_pull_pdev_bss_chan_info_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	ev = tb[WMI_TAG_PDEV_BSS_CHAN_INFO_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch pdev bss chan info ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -6714,7 +6799,6 @@ ath12k_pull_pdev_bss_chan_info_ev(struct ath12k_base *ab, struct sk_buff *skb,
 	bss_ch_info_ev->rx_bss_cycle_count_low = ev->rx_bss_cycle_count_low;
 	bss_ch_info_ev->rx_bss_cycle_count_high = ev->rx_bss_cycle_count_high;
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6726,7 +6810,7 @@ ath12k_pull_vdev_install_key_compl_ev(struct ath12k_base *ab, struct sk_buff *sk
 	const struct wmi_vdev_install_key_compl_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6736,7 +6820,6 @@ ath12k_pull_vdev_install_key_compl_ev(struct ath12k_base *ab, struct sk_buff *sk
 	ev = tb[WMI_TAG_VDEV_INSTALL_KEY_COMPLETE_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch vdev install key compl ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
@@ -6746,7 +6829,6 @@ ath12k_pull_vdev_install_key_compl_ev(struct ath12k_base *ab, struct sk_buff *sk
 	arg->key_flags = le32_to_cpu(ev->key_flags);
 	arg->status = le32_to_cpu(ev->status);
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6757,7 +6839,7 @@ static int ath12k_pull_peer_assoc_conf_ev(struct ath12k_base *ab, struct sk_buff
 	const struct wmi_peer_assoc_conf_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6767,14 +6849,12 @@ static int ath12k_pull_peer_assoc_conf_ev(struct ath12k_base *ab, struct sk_buff
 	ev = tb[WMI_TAG_PEER_ASSOC_CONF_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch peer assoc conf ev");
-		kfree(tb);
 		return -EPROTO;
 	}
 
 	peer_assoc_conf->vdev_id = le32_to_cpu(ev->vdev_id);
 	peer_assoc_conf->macaddr = ev->peer_macaddr.addr;
 
-	kfree(tb);
 	return 0;
 }
 
@@ -6792,7 +6872,7 @@ static int ath12k_reg_11d_new_cc_event(struct ath12k_base *ab, struct sk_buff *s
 	const void **tb;
 	int ret, i;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -6801,7 +6881,6 @@ static int ath12k_reg_11d_new_cc_event(struct ath12k_base *ab, struct sk_buff *s
 
 	ev = tb[WMI_TAG_11D_NEW_COUNTRY_EVENT];
 	if (!ev) {
-		kfree(tb);
 		ath12k_warn(ab, "failed to fetch 11d new cc ev");
 		return -EPROTO;
 	}
@@ -6814,8 +6893,6 @@ static int ath12k_reg_11d_new_cc_event(struct ath12k_base *ab, struct sk_buff *s
 		   ab->new_alpha2[0],
 		   ab->new_alpha2[1]);
 
-	kfree(tb);
-
 	for (i = 0; i < ab->num_radios; i++) {
 		pdev = &ab->pdevs[i];
 		ar = pdev->ar;
@@ -8557,7 +8634,7 @@ static void ath12k_pdev_ctl_failsafe_check_event(struct ath12k_base *ab,
 	const struct wmi_pdev_ctl_failsafe_chk_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -8567,7 +8644,6 @@ static void ath12k_pdev_ctl_failsafe_check_event(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_PDEV_CTL_FAILSAFE_CHECK_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch pdev ctl failsafe check ev");
-		kfree(tb);
 		return;
 	}
 
@@ -8581,8 +8657,330 @@ static void ath12k_pdev_ctl_failsafe_check_event(struct ath12k_base *ab,
 	if (ev->ctl_failsafe_status != 0)
 		ath12k_warn(ab, "pdev ctl failsafe failure status %d",
 			    ev->ctl_failsafe_status);
+}
+
+static int
+ath12k_wmi_incumbent_signal_interference_subtlv_parser(struct ath12k_base *ab,
+						       u16 tag, u16 len,
+						       const void *ptr,
+						       void *data)
+{
+	const struct ath12k_wmi_incumbent_signal_interference_params *info;
+	struct ath12k_wmi_incumbent_signal_interference_arg *arg = data;
+
+	switch (tag) {
+	case WMI_TAG_DCS_INCUMBENT_SIGNAL_INTERFERENCE_TYPE:
+		if (len < sizeof(*info)) {
+			ath12k_warn(ab,
+				    "DCS incumbent signal interference subtlv 0x%x invalid len %u\n",
+				    tag, len);
+			return -EINVAL;
+		}
+
+		info = ptr;
+
+		arg->chan_width = le32_to_cpu(info->chan_width);
+		arg->chan_freq = le32_to_cpu(info->chan_freq);
+		arg->center_freq0 = le32_to_cpu(info->center_freq0);
+		arg->center_freq1 = le32_to_cpu(info->center_freq1);
+		arg->chan_bw_interference_bitmap =
+			le32_to_cpu(info->chan_bw_interference_bitmap);
+
+		ath12k_dbg(ab, ATH12K_DBG_WMI,
+			   "incumbent signal interference chan width %u freq %u center_freq0 %u center_freq1 %u bitmap 0x%x\n",
+			   arg->chan_width, arg->chan_freq,
+			   arg->center_freq0, arg->center_freq1,
+			   arg->chan_bw_interference_bitmap);
+		break;
+	default:
+		ath12k_warn(ab, "Received invalid tag 0x%x for WMI DCS interference in subtlvs\n",
+			    tag);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int ath12k_wmi_dcs_interference_event_parser(struct ath12k_base *ab,
+						    u16 tag, u16 len,
+						    const void *ptr, void *data)
+{
+	int ret = 0;
+
+	switch (tag) {
+	case WMI_TAG_DCS_INTERFERENCE_EVENT:
+		/* Fixed param should already be processed */
+		break;
+	case WMI_TAG_ARRAY_STRUCT:
+		ret = ath12k_wmi_tlv_iter(ab, ptr, len,
+					  ath12k_wmi_incumbent_signal_interference_subtlv_parser,
+					  data);
+		break;
+	default:
+		ath12k_warn(ab, "Received invalid tag 0x%x for WMI DCS interference event\n",
+			    tag);
+		ret = -EINVAL;
+		break;
+	}
+
+	return ret;
+}
+
+static bool
+ath12k_wmi_validate_interference_info(struct ath12k *ar,
+				      struct ath12k_wmi_incumbent_signal_interference_arg *info)
+{
+	switch (info->chan_width) {
+	case WMI_CHAN_WIDTH_20:
+		if (info->chan_bw_interference_bitmap > ATH12K_WMI_DCS_SEG_PRI20) {
+			ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
+				   "DCS interference event received with wrong chan width bmap 0x%x for 20 MHz",
+				   info->chan_bw_interference_bitmap);
+			return false;
+		}
+		break;
+	case WMI_CHAN_WIDTH_40:
+		if (info->chan_bw_interference_bitmap > (ATH12K_WMI_DCS_SEG_PRI20 |
+							 ATH12K_WMI_DCS_SEG_SEC20)) {
+			ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
+				   "DCS interference event received with wrong chan width bmap 0x%x for 40 MHz",
+				   info->chan_bw_interference_bitmap);
+			return false;
+		}
+		break;
+	case WMI_CHAN_WIDTH_80:
+		if (info->chan_bw_interference_bitmap > (ATH12K_WMI_DCS_SEG_PRI20 |
+							 ATH12K_WMI_DCS_SEG_SEC20 |
+							 ATH12K_WMI_DCS_SEG_SEC40)) {
+			ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
+				   "DCS interference event received with wrong chan width bmap 0x%x for 80 MHz",
+				   info->chan_bw_interference_bitmap);
+			return false;
+		}
+		break;
+	case WMI_CHAN_WIDTH_160:
+		if (info->chan_bw_interference_bitmap > (ATH12K_WMI_DCS_SEG_PRI20 |
+							 ATH12K_WMI_DCS_SEG_SEC20 |
+							 ATH12K_WMI_DCS_SEG_SEC40 |
+							 ATH12K_WMI_DCS_SEG_SEC80)) {
+			ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
+				   "DCS interference event received with wrong chan width bmap 0x%x for 160 MHz",
+				   info->chan_bw_interference_bitmap);
+			return false;
+		}
+		break;
+	case WMI_CHAN_WIDTH_320:
+		if (info->chan_bw_interference_bitmap > (ATH12K_WMI_DCS_SEG_PRI20 |
+							 ATH12K_WMI_DCS_SEG_SEC20 |
+							 ATH12K_WMI_DCS_SEG_SEC40 |
+							 ATH12K_WMI_DCS_SEG_SEC80 |
+							 ATH12K_WMI_DCS_SEG_SEC160)) {
+			ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
+				   "DCS interference event received with wrong chan width bmap 0x%x for 320 MHz",
+				   info->chan_bw_interference_bitmap);
+			return false;
+		}
+		break;
+	default:
+		ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
+			   "DCS interference event received with unknown channel width %u",
+			   info->chan_width);
+		return false;
+	}
+	return true;
+}
+
+static u32
+ath12k_wmi_transform_interference_bitmap(int input_bitmap,
+					 struct cfg80211_chan_def *chandef)
+{
+	u16 output_bits[ATH12K_MAX_20MHZ_SEGMENTS] = {};
+	u16 input_bits[ATH12K_MAX_20MHZ_SEGMENTS] = {};
+	u32 start_freq, segment_freq;
+	int primary_index = -1;
+	u32 output_bitmap = 0;
+	u16 num_sub_chans;
+	int bandwidth;
+
+	bandwidth = nl80211_chan_width_to_mhz(chandef->width);
+	if (bandwidth < 0)
+		return 0;
 
-	kfree(tb);
+	/*
+	 * Firmware reports bit 0 as primary 20 MHz irrespective of absolute
+	 * frequency position. Convert to standardized lowest-to-highest 20 MHz
+	 * ordering expected by cfg80211/mac80211 userspace consumers.
+	 */
+	num_sub_chans = bandwidth / 20;
+	start_freq = (chandef->center_freq1 - bandwidth / 2) + 10;
+
+	for (int i = 0; i < ATH12K_MAX_20MHZ_SEGMENTS; i++) {
+		segment_freq = start_freq + (i * 20);
+		if (segment_freq == chandef->chan->center_freq) {
+			primary_index = i;
+			break;
+		}
+	}
+	if (primary_index == -1)
+		return 0;
+
+	for (int i = 0; i < ATH12K_MAX_20MHZ_SEGMENTS; ++i)
+		input_bits[i] = BIT(i) & input_bitmap;
+
+	for (int i = 0; i < num_sub_chans; ++i) {
+		int src = i, dst = i;
+
+		switch (bandwidth) {
+		case 40:
+			if (primary_index == 1)
+				dst = 1 - i;
+			break;
+		case 80:
+			dst = intf_map_80[primary_index][i];
+			break;
+		case 160:
+			dst = intf_map_160[primary_index][i];
+			break;
+		case 320:
+			dst = intf_map_320[primary_index][i];
+			break;
+		}
+		output_bits[dst] = input_bits[src];
+	}
+
+	for (int i = 0; i < ATH12K_MAX_20MHZ_SEGMENTS; ++i)
+		output_bitmap |= output_bits[i] ? BIT(i) : 0;
+
+	return output_bitmap;
+}
+
+static void
+ath12k_wmi_process_incumbent_signal_interference_evt(struct ath12k_base *ab,
+						     struct sk_buff *skb,
+						     const struct ath12k_wmi_intf_arg *intf_arg)
+{
+	struct ath12k_wmi_incumbent_signal_interference_arg info = {};
+	struct ath12k_incumbent_signal_interference *incumbent;
+	struct ath12k_mac_get_any_chanctx_conf_arg arg;
+	u32 transformed_intf_bitmap;
+	struct ieee80211_hw *hw;
+	struct ath12k *ar;
+	int ret;
+
+	guard(rcu)();
+
+	ar = ath12k_mac_get_ar_by_pdev_id(ab, intf_arg->pdev_id);
+	if (!ar) {
+		ath12k_warn(ab, "incumbent signal interference detected on invalid pdev %d\n",
+			    intf_arg->pdev_id);
+		return;
+	}
+	if (!ar->supports_6ghz) {
+		ath12k_warn(ab, "pdev does not support 6 GHz, dropping DCS interference event\n");
+		return;
+	}
+
+	incumbent = &ar->incumbent_signal_interference;
+	spin_lock_bh(&ar->data_lock);
+	if (incumbent->handling_in_progress) {
+		spin_unlock_bh(&ar->data_lock);
+		ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
+			   "incumbent signal interference handling ongoing, dropping DCS interference event");
+		return;
+	}
+	spin_unlock_bh(&ar->data_lock);
+
+	ret = ath12k_wmi_tlv_iter(ab, skb->data, skb->len,
+				  ath12k_wmi_dcs_interference_event_parser,
+				  &info);
+	if (ret) {
+		ath12k_warn(ab,
+			    "failed to parse incumbent signal interference TLV. Error %d\n",
+			    ret);
+		return;
+	}
+
+	if (!ath12k_wmi_validate_interference_info(ar, &info)) {
+		ath12k_warn(ab, "invalid DCS incumbent signal interference TLV - Skipping event");
+		return;
+	}
+
+	arg.ar = ar;
+	arg.chanctx_conf = NULL;
+	hw = ath12k_ar_to_hw(ar);
+	ieee80211_iter_chan_contexts_atomic(hw,
+					    ath12k_mac_get_any_chanctx_conf_iter,
+					    &arg);
+	if (!arg.chanctx_conf) {
+		ath12k_warn(ab, "failed to find valid chanctx_conf in incumbent signal intf detected event\n");
+		return;
+	}
+
+	if (info.chan_freq != arg.chanctx_conf->def.chan->center_freq) {
+		ath12k_dbg(ab, ATH12K_DBG_WMI,
+			   "dcs interference event received with wrong channel %d (ctx freq %d)",
+			   info.chan_freq, arg.chanctx_conf->def.chan->center_freq);
+		return;
+	}
+
+	spin_lock_bh(&ar->data_lock);
+	incumbent->center_freq = arg.chanctx_conf->def.chan->center_freq;
+	incumbent->width = arg.chanctx_conf->def.width;
+	incumbent->chan_bw_interference_bitmap = info.chan_bw_interference_bitmap;
+	incumbent->handling_in_progress = true;
+	spin_unlock_bh(&ar->data_lock);
+	transformed_intf_bitmap =
+		ath12k_wmi_transform_interference_bitmap(info.chan_bw_interference_bitmap,
+							 &arg.chanctx_conf->def);
+	ath12k_dbg(ab, ATH12K_DBG_WMI,
+		   "incumbent signal interference bitmap 0x%x (transformed 0x%x)\n",
+		   info.chan_bw_interference_bitmap, transformed_intf_bitmap);
+	cfg80211_incumbent_signal_notify(hw->wiphy,
+					 &arg.chanctx_conf->def,
+					 transformed_intf_bitmap,
+					 GFP_ATOMIC);
+}
+
+static void
+ath12k_wmi_dcs_interference_event(struct ath12k_base *ab,
+				  struct sk_buff *skb)
+{
+	const struct ath12k_wmi_dcs_interference_ev_fixed_params *dcs_intf_ev;
+	struct ath12k_wmi_intf_arg dcs_intf_arg;
+	const struct wmi_tlv *tlv;
+	u16 tlv_tag;
+	u8 *ptr;
+
+	if (skb->len < (sizeof(*dcs_intf_ev) + TLV_HDR_SIZE)) {
+		ath12k_warn(ab, "DCS interference event is of incorrect length\n");
+		return;
+	}
+
+	ptr = skb->data;
+	tlv = (struct wmi_tlv *)ptr;
+	tlv_tag = le32_get_bits(tlv->header, WMI_TLV_TAG);
+	ptr += sizeof(*tlv);
+
+	if (tlv_tag != WMI_TAG_DCS_INTERFERENCE_EVENT) {
+		ath12k_warn(ab, "DCS interference event received with wrong tag\n");
+		return;
+	}
+
+	dcs_intf_ev = (struct ath12k_wmi_dcs_interference_ev_fixed_params *)ptr;
+
+	dcs_intf_arg.interference_type =
+		le32_to_cpu(dcs_intf_ev->interference_type);
+	dcs_intf_arg.pdev_id = le32_to_cpu(dcs_intf_ev->pdev_id);
+
+	if (dcs_intf_arg.interference_type ==
+	    ATH12K_WMI_DCS_INCUMBENT_SIGNAL_INTERFERENCE) {
+		ath12k_dbg(ab, ATH12K_DBG_WMI,
+			   "incumbent signal interference (Type %u) detected on pdev %u.",
+			   dcs_intf_arg.interference_type,
+			   dcs_intf_arg.pdev_id);
+		ath12k_wmi_process_incumbent_signal_interference_evt(ab, skb,
+								     &dcs_intf_arg);
+	}
 }
 
 static void
@@ -8654,7 +9052,7 @@ ath12k_wmi_pdev_csa_switch_count_status_event(struct ath12k_base *ab,
 	const u32 *vdev_ids;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -8666,7 +9064,6 @@ ath12k_wmi_pdev_csa_switch_count_status_event(struct ath12k_base *ab,
 
 	if (!ev || !vdev_ids) {
 		ath12k_warn(ab, "failed to fetch pdev csa switch count ev");
-		kfree(tb);
 		return;
 	}
 
@@ -8676,8 +9073,6 @@ ath12k_wmi_pdev_csa_switch_count_status_event(struct ath12k_base *ab,
 		   ev->num_vdevs);
 
 	ath12k_wmi_process_csa_switch_count_event(ab, ev, vdev_ids);
-
-	kfree(tb);
 }
 
 static void
@@ -8689,7 +9084,7 @@ ath12k_wmi_pdev_dfs_radar_detected_event(struct ath12k_base *ab, struct sk_buff
 	struct ath12k *ar;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -8700,7 +9095,6 @@ ath12k_wmi_pdev_dfs_radar_detected_event(struct ath12k_base *ab, struct sk_buff
 
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch pdev dfs radar detected ev");
-		kfree(tb);
 		return;
 	}
 
@@ -8739,8 +9133,6 @@ ath12k_wmi_pdev_dfs_radar_detected_event(struct ath12k_base *ab, struct sk_buff
 
 exit:
 	rcu_read_unlock();
-
-	kfree(tb);
 }
 
 static void ath12k_tm_wmi_event_segmented(struct ath12k_base *ab, u32 cmd_id,
@@ -8751,7 +9143,7 @@ static void ath12k_tm_wmi_event_segmented(struct ath12k_base *ab, u32 cmd_id,
 	int ret;
 	u16 length;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
@@ -8762,14 +9154,11 @@ static void ath12k_tm_wmi_event_segmented(struct ath12k_base *ab, u32 cmd_id,
 	ev = tb[WMI_TAG_ARRAY_BYTE];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch ftm msg\n");
-		kfree(tb);
 		return;
 	}
 
 	length = skb->len - TLV_HDR_SIZE;
 	ath12k_tm_process_event(ab, cmd_id, ev, length);
-	kfree(tb);
-	tb = NULL;
 }
 
 static void
@@ -8782,7 +9171,7 @@ ath12k_wmi_pdev_temperature_event(struct ath12k_base *ab,
 	int temp;
 	u32 pdev_id;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ath12k_warn(ab, "failed to parse tlv: %ld\n", PTR_ERR(tb));
 		return;
@@ -8791,15 +9180,12 @@ ath12k_wmi_pdev_temperature_event(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_PDEV_TEMPERATURE_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch pdev temp ev\n");
-		kfree(tb);
 		return;
 	}
 
 	temp = a_sle32_to_cpu(ev->temp);
 	pdev_id = le32_to_cpu(ev->pdev_id);
 
-	kfree(tb);
-
 	ath12k_dbg(ab, ATH12K_DBG_WMI,
 		   "pdev temperature ev temp %d pdev_id %u\n",
 		   temp, pdev_id);
@@ -8819,6 +9205,42 @@ ath12k_wmi_pdev_temperature_event(struct ath12k_base *ab,
 	rcu_read_unlock();
 }
 
+static void ath12k_wmi_thermal_throt_stats_event(struct ath12k_base *ab,
+						 struct sk_buff *skb)
+{
+	const struct wmi_therm_throt_stats_event *ev;
+	struct ath12k *ar;
+	const void **tb;
+
+	tb = ath12k_wmi_tlv_parse(ab, skb);
+	if (IS_ERR(tb)) {
+		ath12k_err(ab, "failed to parse thermal throttling stats tlv: %ld\n",
+			   PTR_ERR(tb));
+		return;
+	}
+
+	ev = tb[WMI_TAG_THERM_THROT_STATS_EVENT];
+	if (!ev) {
+		ath12k_err(ab, "failed to fetch thermal throt stats ev\n");
+		return;
+	}
+
+	rcu_read_lock();
+	ar = ath12k_mac_get_ar_by_pdev_id(ab, le32_to_cpu(ev->pdev_id));
+	if (!ar) {
+		ath12k_warn(ab, "received thermal_throt_stats in invalid pdev %u\n",
+			    le32_to_cpu(ev->pdev_id));
+		rcu_read_unlock();
+		return;
+	}
+	rcu_read_unlock();
+
+	ath12k_dbg(ab, ATH12K_DBG_WMI,
+		   "thermal stats ev level %u pdev_id %u temp %u throt_levels %u\n",
+		   le32_to_cpu(ev->level), le32_to_cpu(ev->pdev_id),
+		   le32_to_cpu(ev->temp), le32_to_cpu(ev->therm_throt_levels));
+}
+
 static void ath12k_fils_discovery_event(struct ath12k_base *ab,
 					struct sk_buff *skb)
 {
@@ -8826,7 +9248,7 @@ static void ath12k_fils_discovery_event(struct ath12k_base *ab,
 	const struct wmi_fils_discovery_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab,
@@ -8838,15 +9260,12 @@ static void ath12k_fils_discovery_event(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_HOST_SWFDA_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch FILS discovery event\n");
-		kfree(tb);
 		return;
 	}
 
 	ath12k_warn(ab,
 		    "FILS discovery frame expected from host for vdev_id: %u, transmission scheduled at %u, next TBTT: %u\n",
 		    ev->vdev_id, ev->fils_tt, ev->tbtt);
-
-	kfree(tb);
 }
 
 static void ath12k_probe_resp_tx_status_event(struct ath12k_base *ab,
@@ -8856,7 +9275,7 @@ static void ath12k_probe_resp_tx_status_event(struct ath12k_base *ab,
 	const struct wmi_probe_resp_tx_status_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab,
@@ -8869,7 +9288,6 @@ static void ath12k_probe_resp_tx_status_event(struct ath12k_base *ab,
 	if (!ev) {
 		ath12k_warn(ab,
 			    "failed to fetch probe response transmission status event");
-		kfree(tb);
 		return;
 	}
 
@@ -8877,8 +9295,6 @@ static void ath12k_probe_resp_tx_status_event(struct ath12k_base *ab,
 		ath12k_warn(ab,
 			    "Probe response transmission failed for vdev_id %u, status %u\n",
 			    ev->vdev_id, ev->tx_status);
-
-	kfree(tb);
 }
 
 static int ath12k_wmi_p2p_noa_event(struct ath12k_base *ab,
@@ -8890,7 +9306,7 @@ static int ath12k_wmi_p2p_noa_event(struct ath12k_base *ab,
 	struct ath12k *ar;
 	int ret, vdev_id;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse P2P NoA TLV: %d\n", ret);
@@ -8900,10 +9316,8 @@ static int ath12k_wmi_p2p_noa_event(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_P2P_NOA_EVENT];
 	noa = tb[WMI_TAG_P2P_NOA_INFO];
 
-	if (!ev || !noa) {
-		ret = -EPROTO;
-		goto out;
-	}
+	if (!ev || !noa)
+		return -EPROTO;
 
 	vdev_id = __le32_to_cpu(ev->vdev_id);
 
@@ -8926,8 +9340,6 @@ static int ath12k_wmi_p2p_noa_event(struct ath12k_base *ab,
 
 unlock:
 	rcu_read_unlock();
-out:
-	kfree(tb);
 	return ret;
 }
 
@@ -8938,7 +9350,7 @@ static void ath12k_rfkill_state_change_event(struct ath12k_base *ab,
 	const void **tb;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -8946,10 +9358,8 @@ static void ath12k_rfkill_state_change_event(struct ath12k_base *ab,
 	}
 
 	ev = tb[WMI_TAG_RFKILL_EVENT];
-	if (!ev) {
-		kfree(tb);
+	if (!ev)
 		return;
-	}
 
 	ath12k_dbg(ab, ATH12K_DBG_MAC,
 		   "wmi tlv rfkill state change gpio %d type %d radio_state %d\n",
@@ -8962,7 +9372,6 @@ static void ath12k_rfkill_state_change_event(struct ath12k_base *ab,
 	spin_unlock_bh(&ab->base_lock);
 
 	queue_work(ab->workqueue, &ab->rfkill_work);
-	kfree(tb);
 }
 
 static void
@@ -8978,7 +9387,7 @@ static void ath12k_wmi_twt_enable_event(struct ath12k_base *ab,
 	const struct wmi_twt_enable_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse wmi twt enable status event tlv: %d\n",
@@ -8989,15 +9398,12 @@ static void ath12k_wmi_twt_enable_event(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_TWT_ENABLE_COMPLETE_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch twt enable wmi event\n");
-		goto exit;
+		return;
 	}
 
 	ath12k_dbg(ab, ATH12K_DBG_MAC, "wmi twt enable event pdev id %u status %u\n",
 		   le32_to_cpu(ev->pdev_id),
 		   le32_to_cpu(ev->status));
-
-exit:
-	kfree(tb);
 }
 
 static void ath12k_wmi_twt_disable_event(struct ath12k_base *ab,
@@ -9007,7 +9413,7 @@ static void ath12k_wmi_twt_disable_event(struct ath12k_base *ab,
 	const struct wmi_twt_disable_event *ev;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse wmi twt disable status event tlv: %d\n",
@@ -9018,15 +9424,12 @@ static void ath12k_wmi_twt_disable_event(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_TWT_DISABLE_COMPLETE_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch twt disable wmi event\n");
-		goto exit;
+		return;
 	}
 
 	ath12k_dbg(ab, ATH12K_DBG_MAC, "wmi twt disable event pdev id %d status %u\n",
 		   le32_to_cpu(ev->pdev_id),
 		   le32_to_cpu(ev->status));
-
-exit:
-	kfree(tb);
 }
 
 static int ath12k_wmi_wow_wakeup_host_parse(struct ath12k_base *ab,
@@ -9099,7 +9502,7 @@ static void ath12k_wmi_gtk_offload_status_event(struct ath12k_base *ab,
 	const void **tb;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse tlv: %d\n", ret);
@@ -9109,7 +9512,6 @@ static void ath12k_wmi_gtk_offload_status_event(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_GTK_OFFLOAD_STATUS_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch gtk offload status ev");
-		kfree(tb);
 		return;
 	}
 
@@ -9119,7 +9521,6 @@ static void ath12k_wmi_gtk_offload_status_event(struct ath12k_base *ab,
 		rcu_read_unlock();
 		ath12k_warn(ab, "failed to get arvif for vdev_id:%d\n",
 			    le32_to_cpu(ev->vdev_id));
-		kfree(tb);
 		return;
 	}
 
@@ -9135,8 +9536,6 @@ static void ath12k_wmi_gtk_offload_status_event(struct ath12k_base *ab,
 				   (void *)&replay_ctr_be, GFP_ATOMIC);
 
 	rcu_read_unlock();
-
-	kfree(tb);
 }
 
 static void ath12k_wmi_event_mlo_setup_complete(struct ath12k_base *ab,
@@ -9148,7 +9547,7 @@ static void ath12k_wmi_event_mlo_setup_complete(struct ath12k_base *ab,
 	const void **tb;
 	int ret, i;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse mlo setup complete event tlv: %d\n",
@@ -9159,7 +9558,6 @@ static void ath12k_wmi_event_mlo_setup_complete(struct ath12k_base *ab,
 	ev = tb[WMI_TAG_MLO_SETUP_COMPLETE_EVENT];
 	if (!ev) {
 		ath12k_warn(ab, "failed to fetch mlo setup complete event\n");
-		kfree(tb);
 		return;
 	}
 
@@ -9178,14 +9576,11 @@ static void ath12k_wmi_event_mlo_setup_complete(struct ath12k_base *ab,
 	if (!ar) {
 		ath12k_warn(ab, "invalid pdev_id %d status %u in setup complete event\n",
 			    ev->pdev_id, ev->status);
-		goto out;
+		return;
 	}
 
 	ar->mlo_setup_status = le32_to_cpu(ev->status);
 	complete(&ar->mlo_setup_done);
-
-out:
-	kfree(tb);
 }
 
 static void ath12k_wmi_event_teardown_complete(struct ath12k_base *ab,
@@ -9195,7 +9590,7 @@ static void ath12k_wmi_event_teardown_complete(struct ath12k_base *ab,
 	const void **tb;
 	int ret;
 
-	tb = ath12k_wmi_tlv_parse_alloc(ab, skb, GFP_ATOMIC);
+	tb = ath12k_wmi_tlv_parse(ab, skb);
 	if (IS_ERR(tb)) {
 		ret = PTR_ERR(tb);
 		ath12k_warn(ab, "failed to parse teardown complete event tlv: %d\n", ret);
@@ -9203,13 +9598,8 @@ static void ath12k_wmi_event_teardown_complete(struct ath12k_base *ab,
 	}
 
 	ev = tb[WMI_TAG_MLO_TEARDOWN_COMPLETE];
-	if (!ev) {
+	if (!ev)
 		ath12k_warn(ab, "failed to fetch teardown complete event\n");
-		kfree(tb);
-		return;
-	}
-
-	kfree(tb);
 }
 
 #ifdef CONFIG_ATH12K_DEBUGFS
@@ -9900,6 +10290,9 @@ static void ath12k_wmi_op_rx(struct ath12k_base *ab, struct sk_buff *skb)
 	case WMI_PDEV_TEMPERATURE_EVENTID:
 		ath12k_wmi_pdev_temperature_event(ab, skb);
 		break;
+	case WMI_THERM_THROT_STATS_EVENTID:
+		ath12k_wmi_thermal_throt_stats_event(ab, skb);
+		break;
 	case WMI_PDEV_DMA_RING_BUF_RELEASE_EVENTID:
 		ath12k_wmi_pdev_dma_ring_buf_release_event(ab, skb);
 		break;
@@ -9954,6 +10347,9 @@ static void ath12k_wmi_op_rx(struct ath12k_base *ab, struct sk_buff *skb)
 	case WMI_OBSS_COLOR_COLLISION_DETECTION_EVENTID:
 		ath12k_wmi_obss_color_collision_event(ab, skb);
 		break;
+	case WMI_DCS_INTERFERENCE_EVENTID:
+		ath12k_wmi_dcs_interference_event(ab, skb);
+		break;
 	/* add Unsupported events (rare) here */
 	case WMI_TBTTOFFSET_EXT_UPDATE_EVENTID:
 	case WMI_PEER_OPER_MODE_CHANGE_EVENTID:
@@ -10172,6 +10568,42 @@ int ath12k_wmi_send_tpc_stats_request(struct ath12k *ar,
 	return ret;
 }
 
+int ath12k_wmi_simulate_incumbent_signal_interference(struct ath12k *ar,
+						      u32 chan_bw_interference_bitmap)
+{
+	struct wmi_unit_test_arg wmi_ut = {};
+	struct ath12k_link_vif *arvif;
+	struct ath12k_vif *ahvif;
+	bool arvif_found = false;
+
+	list_for_each_entry(arvif, &ar->arvifs, list) {
+		ahvif = arvif->ahvif;
+		if (arvif->is_started && ahvif->vdev_type == WMI_VDEV_TYPE_AP) {
+			arvif_found = true;
+			break;
+		}
+	}
+
+	if (!arvif_found)
+		return -EINVAL;
+
+	wmi_ut.args[ATH12K_WMI_INCUMBENT_SIGNAL_TEST_INTF] =
+		ATH12K_WMI_UNIT_TEST_INCUMBENT_SIGNAL_INTF_TYPE;
+	wmi_ut.args[ATH12K_WMI_INCUMBENT_SIGNAL_TEST_BITMAP] =
+		chan_bw_interference_bitmap;
+
+	wmi_ut.vdev_id = arvif->vdev_id;
+	wmi_ut.module_id = ATH12K_WMI_INCUMBENT_SIGNAL_UNIT_TEST_MODULE;
+	wmi_ut.num_args = ATH12K_WMI_INCUMBENT_SIGNAL_MAX_TEST_ARGS;
+	wmi_ut.diag_token = ATH12K_WMI_INCUMBENT_SIGNAL_UNIT_TEST_TOKEN;
+
+	ath12k_dbg(ar->ab, ATH12K_DBG_WMI,
+		   "Triggering incumbent signal interference simulation, interference bitmap: 0x%x\n",
+		   chan_bw_interference_bitmap);
+
+	return ath12k_wmi_send_unit_test_cmd(ar, &wmi_ut);
+}
+
 int ath12k_wmi_connect(struct ath12k_base *ab)
 {
 	u32 i;
@@ -10963,10 +11395,9 @@ int ath12k_wmi_mlo_setup(struct ath12k *ar, struct wmi_mlo_setup_arg *mlo_params
 		ath12k_warn(ar->ab, "failed to submit WMI_MLO_SETUP_CMDID command: %d\n",
 			    ret);
 		dev_kfree_skb(skb);
-		return ret;
 	}
 
-	return 0;
+	return ret;
 }
 
 int ath12k_wmi_mlo_ready(struct ath12k *ar)
@@ -10991,10 +11422,9 @@ int ath12k_wmi_mlo_ready(struct ath12k *ar)
 		ath12k_warn(ar->ab, "failed to submit WMI_MLO_READY_CMDID command: %d\n",
 			    ret);
 		dev_kfree_skb(skb);
-		return ret;
 	}
 
-	return 0;
+	return ret;
 }
 
 int ath12k_wmi_mlo_teardown(struct ath12k *ar)
@@ -11020,10 +11450,9 @@ int ath12k_wmi_mlo_teardown(struct ath12k *ar)
 		ath12k_warn(ar->ab, "failed to submit WMI MLO teardown command: %d\n",
 			    ret);
 		dev_kfree_skb(skb);
-		return ret;
 	}
 
-	return 0;
+	return ret;
 }
 
 bool ath12k_wmi_supports_6ghz_cc_ext(struct ath12k *ar)
@@ -11086,10 +11515,9 @@ int ath12k_wmi_send_vdev_set_tpc_power(struct ath12k *ar,
 	if (ret) {
 		ath12k_warn(ar->ab, "failed to send WMI_VDEV_SET_TPC_POWER_CMDID\n");
 		dev_kfree_skb(skb);
-		return ret;
 	}
 
-	return 0;
+	return ret;
 }
 
 static int
@@ -11312,3 +11740,31 @@ int ath12k_wmi_send_mlo_link_set_active_cmd(struct ath12k_base *ab,
 	dev_kfree_skb(skb);
 	return ret;
 }
+
+int ath12k_wmi_alloc(void)
+{
+	guard(mutex)(&ath12k_wmi_mutex);
+
+	if (!ath12k_wmi_tb) {
+		ath12k_wmi_tb = __alloc_percpu(WMI_TAG_MAX * sizeof(void *),
+					       __alignof__(void *));
+		if (!ath12k_wmi_tb)
+			return -ENOMEM;
+
+		refcount_set(&ath12k_wmi_refcount, 1);
+	} else {
+		refcount_inc(&ath12k_wmi_refcount);
+	}
+
+	return 0;
+}
+
+void ath12k_wmi_free(void)
+{
+	guard(mutex)(&ath12k_wmi_mutex);
+
+	if (refcount_dec_and_test(&ath12k_wmi_refcount)) {
+		free_percpu(ath12k_wmi_tb);
+		ath12k_wmi_tb = NULL;
+	}
+}
diff --git a/drivers/net/wireless/ath/ath12k/wmi.h b/drivers/net/wireless/ath/ath12k/wmi.h
index 5ba9b7d3a8888..14b8dcdf881dd 100644
--- a/drivers/net/wireless/ath/ath12k/wmi.h
+++ b/drivers/net/wireless/ath/ath12k/wmi.h
@@ -870,6 +870,7 @@ enum wmi_tlv_event_id {
 	WMI_READ_DATA_FROM_FLASH_EVENTID,
 	WMI_REPORT_RX_AGGR_FAILURE_EVENTID,
 	WMI_PKGID_EVENTID,
+	WMI_THERM_THROT_STATS_EVENTID,
 	WMI_GPIO_INPUT_EVENTID = WMI_TLV_CMD(WMI_GRP_GPIO),
 	WMI_UPLOADH_EVENTID,
 	WMI_CAPTUREH_EVENTID,
@@ -2014,7 +2015,7 @@ enum wmi_tlv_tag {
 	WMI_TAG_VDEV_CH_POWER_INFO,
 	WMI_TAG_MLO_LINK_SET_ACTIVE_CMD = 0x3BE,
 	WMI_TAG_EHT_RATE_SET = 0x3C4,
-	WMI_TAG_DCS_AWGN_INT_TYPE = 0x3C5,
+	WMI_TAG_DCS_INCUMBENT_SIGNAL_INTERFERENCE_TYPE = 0x3C5,
 	WMI_TAG_MLO_TX_SEND_PARAMS,
 	WMI_TAG_MLO_PARTNER_LINK_PARAMS,
 	WMI_TAG_MLO_PARTNER_LINK_PARAMS_PEER_ASSOC,
@@ -2259,6 +2260,7 @@ enum wmi_tlv_service {
 	WMI_TLV_SERVICE_FREQINFO_IN_METADATA = 219,
 	WMI_TLV_SERVICE_EXT2_MSG = 220,
 	WMI_TLV_SERVICE_BEACON_PROTECTION_SUPPORT = 244,
+	WMI_TLV_SERVICE_5_9GHZ_SUPPORT = 247,
 	WMI_TLV_SERVICE_SRG_SRP_SPATIAL_REUSE_SUPPORT = 249,
 	WMI_TLV_SERVICE_MBSS_PARAM_IN_VDEV_START_SUPPORT = 253,
 
@@ -2268,11 +2270,17 @@ enum wmi_tlv_service {
 
 	WMI_TLV_SERVICE_REG_CC_EXT_EVENT_SUPPORT = 281,
 
+	WMI_TLV_SERVICE_DCS_INCUMBENT_SIGNAL_INTERFERENCE_SUPPORT = 286,
+
 	WMI_TLV_SERVICE_11BE = 289,
 
 	WMI_TLV_SERVICE_WMSK_COMPACTION_RX_TLVS = 361,
 
 	WMI_TLV_SERVICE_PEER_METADATA_V1A_V1B_SUPPORT = 365,
+	WMI_TLV_SERVICE_THERM_THROT_POUT_REDUCTION = 410,
+	WMI_TLV_SERVICE_IS_TARGET_IPA = 425,
+	WMI_TLV_SERVICE_THERM_THROT_TX_CHAIN_MASK = 426,
+	WMI_TLV_SERVICE_THERM_THROT_5_LEVELS = 429,
 	WMI_TLV_SERVICE_ETH_OFFLOAD = 461,
 
 	WMI_MAX_EXT2_SERVICE,
@@ -3586,7 +3594,6 @@ struct ath12k_wmi_scan_req_arg {
 	u32 num_bssid;
 	u32 num_ssids;
 	u32 n_probes;
-	u32 *chan_list;
 	u32 notify_scan_events;
 	struct cfg80211_ssid ssid[WLAN_SCAN_MAX_NUM_SSID];
 	struct ath12k_wmi_mac_addr_params bssid_list[WLAN_SCAN_MAX_NUM_BSSID];
@@ -3595,6 +3602,7 @@ struct ath12k_wmi_scan_req_arg {
 	u32 num_hint_bssid;
 	struct ath12k_wmi_hint_short_ssid_arg hint_s_ssid[WLAN_SCAN_MAX_HINT_S_SSID];
 	struct ath12k_wmi_hint_bssid_arg hint_bssid[WLAN_SCAN_MAX_HINT_BSSID];
+	u32 chan_list[] __counted_by(num_chan);
 };
 
 struct wmi_ssid_arg {
@@ -4120,6 +4128,49 @@ enum set_init_cc_flags {
 	ALPHA_IS_SET,
 };
 
+struct wmi_therm_throt_stats_event {
+	__le32 pdev_id;
+	__le32 temp;
+	__le32 level;
+	__le32 therm_throt_levels;
+} __packed;
+
+#define THERMAL_LEVELS  4
+#define ENHANCED_THERMAL_LEVELS 5
+
+struct ath12k_wmi_tt_level_config_param {
+	s32 tmplwm;
+	s32 tmphwm;
+	u32 dcoffpercent;
+	u32 pout_reduction_db;
+};
+
+struct ath12k_wmi_therm_throt_config_request_cmd {
+	__le32 tlv_header;
+	__le32 pdev_id;
+	__le32 enable;
+	__le32 dc;
+	/* After how many duty cycles the firmware sends stats to host */
+	__le32 dc_per_event;
+	__le32 therm_throt_levels;
+} __packed;
+
+struct ath12k_wmi_therm_throt_level_config_param {
+	__le32 tlv_header;
+	a_sle32 temp_lwm;
+	a_sle32 temp_hwm;
+	__le32 dc_off_percent;
+	__le32 prio;
+	__le32 pout_reduction_25db;
+	__le32 tx_chain_mask;
+	__le32 duty_cycle;
+} __packed;
+
+struct ath12k_wmi_thermal_mitigation_arg {
+	int num_levels;
+	const struct ath12k_wmi_tt_level_config_param *levelconf;
+};
+
 struct ath12k_wmi_init_country_arg {
 	union {
 		u16 country_code;
@@ -4203,6 +4254,16 @@ enum dfs_test_args_idx {
 	DFS_MAX_TEST_ARGS,
 };
 
+#define ATH12K_WMI_INCUMBENT_SIGNAL_UNIT_TEST_MODULE	0x18
+#define ATH12K_WMI_INCUMBENT_SIGNAL_UNIT_TEST_TOKEN	0
+#define ATH12K_WMI_UNIT_TEST_INCUMBENT_SIGNAL_INTF_TYPE	1
+
+enum ath12k_wmi_incumbent_signal_test_args_idx {
+	ATH12K_WMI_INCUMBENT_SIGNAL_TEST_INTF,
+	ATH12K_WMI_INCUMBENT_SIGNAL_TEST_BITMAP,
+	ATH12K_WMI_INCUMBENT_SIGNAL_MAX_TEST_ARGS,
+};
+
 /* update if another test command requires more */
 #define WMI_UNIT_TEST_ARGS_MAX DFS_MAX_TEST_ARGS
 
@@ -4487,6 +4548,62 @@ struct ath12k_wmi_pdev_radar_event {
 	a_sle32 sidx;
 } __packed;
 
+#define ATH12K_WMI_DCS_INCUMBENT_SIGNAL_INTERFERENCE	0x04
+
+struct ath12k_wmi_dcs_interference_ev_fixed_params {
+	__le32 interference_type;
+	__le32 pdev_id;
+} __packed;
+
+struct ath12k_wmi_incumbent_signal_interference_params {
+	__le32 chan_width;
+	__le32 chan_freq;
+	__le32 center_freq0;
+	__le32 center_freq1;
+	__le32 chan_bw_interference_bitmap;
+} __packed;
+
+struct ath12k_wmi_incumbent_signal_interference_arg {
+	u32 chan_width;
+	u32 chan_freq;
+	u32 center_freq0;
+	u32 center_freq1;
+	u32 chan_bw_interference_bitmap;
+};
+
+struct ath12k_wmi_intf_arg {
+	u32 interference_type;
+	u32 pdev_id;
+};
+
+enum ath12k_wmi_dcs_interference_chan_segment {
+	/*
+	 * Firmware reports interference bitmap in primary-based order.
+	 * Bit 0 is the primary 20 MHz, bit 1 is the adjacent 20 MHz within
+	 * the primary 40 MHz. Bits 2-3 cover the secondary 40 MHz, bits 4-7
+	 * cover the secondary 80 MHz, and bits 8-15 cover the secondary 160 MHz.
+	 */
+	ATH12K_WMI_DCS_SEG_PRI20                 = 0x1,
+	ATH12K_WMI_DCS_SEG_SEC20                 = 0x2,
+	ATH12K_WMI_DCS_SEG_SEC40_LOW             = 0x4,
+	ATH12K_WMI_DCS_SEG_SEC40_UP              = 0x8,
+	ATH12K_WMI_DCS_SEG_SEC40                 = 0xC,
+	ATH12K_WMI_DCS_SEG_SEC80_LOW             = 0x10,
+	ATH12K_WMI_DCS_SEG_SEC80_LOW_UP          = 0x20,
+	ATH12K_WMI_DCS_SEG_SEC80_UP_LOW          = 0x40,
+	ATH12K_WMI_DCS_SEG_SEC80_UP              = 0x80,
+	ATH12K_WMI_DCS_SEG_SEC80                 = 0xF0,
+	ATH12K_WMI_DCS_SEG_SEC160_LOW            = 0x0100,
+	ATH12K_WMI_DCS_SEG_SEC160_LOW_UP         = 0x0200,
+	ATH12K_WMI_DCS_SEG_SEC160_LOW_UP_UP      = 0x0400,
+	ATH12K_WMI_DCS_SEG_SEC160_LOW_UP_UP_UP   = 0x0800,
+	ATH12K_WMI_DCS_SEG_SEC160_UP_LOW_LOW_LOW = 0x1000,
+	ATH12K_WMI_DCS_SEG_SEC160_UP_LOW_LOW     = 0x2000,
+	ATH12K_WMI_DCS_SEG_SEC160_UP_LOW         = 0x4000,
+	ATH12K_WMI_DCS_SEG_SEC160_UP             = 0x8000,
+	ATH12K_WMI_DCS_SEG_SEC160                = 0xFF00,
+};
+
 struct wmi_pdev_temperature_event {
 	/* temperature value in Celsius degree */
 	a_sle32 temp;
@@ -6514,6 +6631,8 @@ __le32 ath12k_wmi_tlv_hdr(u32 cmd, u32 len);
 int ath12k_wmi_send_tpc_stats_request(struct ath12k *ar,
 				      enum wmi_halphy_ctrl_path_stats_id tpc_stats_type);
 void ath12k_wmi_free_tpc_stats_mem(struct ath12k *ar);
+int ath12k_wmi_send_thermal_mitigation_cmd(struct ath12k *ar,
+					   struct ath12k_wmi_thermal_mitigation_arg *arg);
 
 static inline u32
 ath12k_wmi_caps_ext_get_pdev_id(const struct ath12k_wmi_caps_ext_params *param)
@@ -6576,4 +6695,9 @@ int ath12k_wmi_send_vdev_set_tpc_power(struct ath12k *ar,
 				       struct ath12k_reg_tpc_power_info *param);
 int ath12k_wmi_send_mlo_link_set_active_cmd(struct ath12k_base *ab,
 					    struct wmi_mlo_link_set_active_arg *param);
+int ath12k_wmi_simulate_incumbent_signal_interference(struct ath12k *ar,
+						      u32 chan_bw_interference_bitmap);
+int ath12k_wmi_alloc(void);
+void ath12k_wmi_free(void);
+
 #endif
diff --git a/drivers/net/wireless/ath/ath6kl/cfg80211.c b/drivers/net/wireless/ath/ath6kl/cfg80211.c
index 739a24a6ad676..cc0f2c45fc3a3 100644
--- a/drivers/net/wireless/ath/ath6kl/cfg80211.c
+++ b/drivers/net/wireless/ath/ath6kl/cfg80211.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004-2011 Atheros Communications Inc.
  * Copyright (c) 2011-2012 Qualcomm Atheros, Inc.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -3033,7 +3034,7 @@ static int ath6kl_remain_on_channel(struct wiphy *wiphy,
 				    struct wireless_dev *wdev,
 				    struct ieee80211_channel *chan,
 				    unsigned int duration,
-				    u64 *cookie)
+				    u64 *cookie, const u8 *rx_addr)
 {
 	struct ath6kl_vif *vif = ath6kl_vif_from_wdev(wdev);
 	struct ath6kl *ar = ath6kl_priv(vif->ndev);
diff --git a/drivers/net/wireless/ath/ath9k/ar9002_hw.c b/drivers/net/wireless/ath/ath9k/ar9002_hw.c
index b262244800410..0f24539b75ecc 100644
--- a/drivers/net/wireless/ath/ath9k/ar9002_hw.c
+++ b/drivers/net/wireless/ath/ath9k/ar9002_hw.c
@@ -80,14 +80,14 @@ static int ar9002_hw_init_mode_regs(struct ath_hw *ah)
 	/* iniAddac needs to be modified for these chips */
 	if (AR_SREV_9160(ah) || !AR_SREV_5416_22_OR_LATER(ah)) {
 		struct ar5416IniArray *addac = &ah->iniAddac;
-		u32 size = sizeof(u32) * addac->ia_rows * addac->ia_columns;
+		u32 n = addac->ia_rows * addac->ia_columns;
 		u32 *data;
 
-		data = devm_kzalloc(ah->dev, size, GFP_KERNEL);
+		data = devm_kmemdup_array(ah->dev, addac->ia_array, n, sizeof(u32),
+			GFP_KERNEL);
 		if (!data)
 			return -ENOMEM;
 
-		memcpy(data, addac->ia_array, size);
 		addac->ia_array = data;
 
 		if (!AR_SREV_5416_22_OR_LATER(ah)) {
diff --git a/drivers/net/wireless/ath/ath9k/ath9k_pci_owl_loader.c b/drivers/net/wireless/ath/ath9k/ath9k_pci_owl_loader.c
index fe1013a3a5880..b9ef34709202b 100644
--- a/drivers/net/wireless/ath/ath9k/ath9k_pci_owl_loader.c
+++ b/drivers/net/wireless/ath/ath9k/ath9k_pci_owl_loader.c
@@ -137,24 +137,6 @@ static void owl_fw_cb(const struct firmware *fw, void *context)
 	release_firmware(fw);
 }
 
-static const char *owl_get_eeprom_name(struct pci_dev *pdev)
-{
-	struct device *dev = &pdev->dev;
-	char *eeprom_name;
-
-	dev_dbg(dev, "using auto-generated eeprom filename\n");
-
-	eeprom_name = devm_kzalloc(dev, EEPROM_FILENAME_LEN, GFP_KERNEL);
-	if (!eeprom_name)
-		return NULL;
-
-	/* this should match the pattern used in ath9k/init.c */
-	scnprintf(eeprom_name, EEPROM_FILENAME_LEN, "ath9k-eeprom-pci-%s.bin",
-		  dev_name(dev));
-
-	return eeprom_name;
-}
-
 static void owl_nvmem_work(struct work_struct *work)
 {
 	struct owl_ctx *ctx = container_of(work, struct owl_ctx, work);
@@ -195,8 +177,9 @@ static int owl_nvmem_probe(struct owl_ctx *ctx)
 static int owl_probe(struct pci_dev *pdev,
 		     const struct pci_device_id *id)
 {
+	char eeprom_name[EEPROM_FILENAME_LEN];
+	struct device *dev = &pdev->dev;
 	struct owl_ctx *ctx;
-	const char *eeprom_name;
 	int err = 0;
 
 	if (pci_enable_device(pdev))
@@ -215,11 +198,11 @@ static int owl_probe(struct pci_dev *pdev,
 	if (err <= 0)
 		return err;
 
-	eeprom_name = owl_get_eeprom_name(pdev);
-	if (!eeprom_name) {
-		dev_err(&pdev->dev, "no eeprom filename found.\n");
-		return -ENODEV;
-	}
+	dev_dbg(dev, "using auto-generated eeprom filename\n");
+
+	/* this should match the pattern used in ath9k/init.c */
+	scnprintf(eeprom_name, sizeof(eeprom_name), "ath9k-eeprom-pci-%s.bin",
+		  dev_name(dev));
 
 	err = request_firmware_nowait(THIS_MODULE, true, eeprom_name,
 				      &pdev->dev, GFP_KERNEL, ctx, owl_fw_cb);
diff --git a/drivers/net/wireless/ath/ath9k/common-init.c b/drivers/net/wireless/ath/ath9k/common-init.c
index da102c791712e..52e02e0752d8d 100644
--- a/drivers/net/wireless/ath/ath9k/common-init.c
+++ b/drivers/net/wireless/ath/ath9k/common-init.c
@@ -133,13 +133,11 @@ int ath9k_cmn_init_channels_rates(struct ath_common *common)
 		     ATH9K_NUM_CHANNELS);
 
 	if (ah->caps.hw_caps & ATH9K_HW_CAP_2GHZ) {
-		channels = devm_kzalloc(ah->dev,
+		channels = devm_kmemdup(ah->dev, ath9k_2ghz_chantable,
 			sizeof(ath9k_2ghz_chantable), GFP_KERNEL);
 		if (!channels)
 		    return -ENOMEM;
 
-		memcpy(channels, ath9k_2ghz_chantable,
-		       sizeof(ath9k_2ghz_chantable));
 		common->sbands[NL80211_BAND_2GHZ].channels = channels;
 		common->sbands[NL80211_BAND_2GHZ].band = NL80211_BAND_2GHZ;
 		common->sbands[NL80211_BAND_2GHZ].n_channels =
@@ -150,13 +148,11 @@ int ath9k_cmn_init_channels_rates(struct ath_common *common)
 	}
 
 	if (ah->caps.hw_caps & ATH9K_HW_CAP_5GHZ) {
-		channels = devm_kzalloc(ah->dev,
+		channels = devm_kmemdup(ah->dev, ath9k_5ghz_chantable,
 			sizeof(ath9k_5ghz_chantable), GFP_KERNEL);
 		if (!channels)
 			return -ENOMEM;
 
-		memcpy(channels, ath9k_5ghz_chantable,
-		       sizeof(ath9k_5ghz_chantable));
 		common->sbands[NL80211_BAND_5GHZ].channels = channels;
 		common->sbands[NL80211_BAND_5GHZ].band = NL80211_BAND_5GHZ;
 		common->sbands[NL80211_BAND_5GHZ].n_channels =
diff --git a/drivers/net/wireless/ath/ath9k/hw.c b/drivers/net/wireless/ath/ath9k/hw.c
index a45351afcf6ee..05c95e67a853b 100644
--- a/drivers/net/wireless/ath/ath9k/hw.c
+++ b/drivers/net/wireless/ath/ath9k/hw.c
@@ -21,7 +21,7 @@
 #include <linux/time.h>
 #include <linux/bitops.h>
 #include <linux/etherdevice.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/unaligned.h>
 
 #include "hw.h"
@@ -2719,19 +2719,28 @@ static void ath9k_hw_gpio_cfg_output_mux(struct ath_hw *ah, u32 gpio, u32 type)
 static void ath9k_hw_gpio_cfg_soc(struct ath_hw *ah, u32 gpio, bool out,
 				  const char *label)
 {
+	enum gpiod_flags flags = out ? GPIOD_OUT_LOW : GPIOD_IN;
+	struct gpio_desc *gpiod;
 	int err;
 
-	if (ah->caps.gpio_requested & BIT(gpio))
+	if (ah->gpiods[gpio])
 		return;
 
-	err = devm_gpio_request_one(ah->dev, gpio, out ? GPIOF_OUT_INIT_LOW : GPIOF_IN, label);
+	/*
+	 * Obtains a system specific GPIO descriptor from another GPIO controller.
+	 * Ideally this should come from the device tree, this is a legacy code
+	 * path.
+	 */
+	gpiod = gpiod_get_index(NULL, "ath9k", gpio, flags);
+	err = PTR_ERR_OR_ZERO(gpiod);
 	if (err) {
 		ath_err(ath9k_hw_common(ah), "request GPIO%d failed:%d\n",
 			gpio, err);
 		return;
 	}
 
-	ah->caps.gpio_requested |= BIT(gpio);
+	gpiod_set_consumer_name(gpiod, label);
+	ah->gpiods[gpio] = gpiod;
 }
 
 static void ath9k_hw_gpio_cfg_wmac(struct ath_hw *ah, u32 gpio, bool out,
@@ -2791,10 +2800,12 @@ void ath9k_hw_gpio_free(struct ath_hw *ah, u32 gpio)
 	if (!AR_SREV_SOC(ah))
 		return;
 
-	WARN_ON(gpio >= ah->caps.num_gpio_pins);
+	if (ah->gpiods[gpio]) {
+		gpiod_put(ah->gpiods[gpio]);
+		ah->gpiods[gpio] = NULL;
+	}
 
-	if (ah->caps.gpio_requested & BIT(gpio))
-		ah->caps.gpio_requested &= ~BIT(gpio);
+	WARN_ON(gpio >= ah->caps.num_gpio_pins);
 }
 EXPORT_SYMBOL(ath9k_hw_gpio_free);
 
@@ -2822,8 +2833,8 @@ u32 ath9k_hw_gpio_get(struct ath_hw *ah, u32 gpio)
 			val = REG_READ(ah, AR_GPIO_IN(ah)) & BIT(gpio);
 		else
 			val = MS_REG_READ(AR, gpio);
-	} else if (BIT(gpio) & ah->caps.gpio_requested) {
-		val = gpio_get_value(gpio) & BIT(gpio);
+	} else if (ah->gpiods[gpio]) {
+		val = gpiod_get_value(ah->gpiods[gpio]);
 	} else {
 		WARN_ON(1);
 	}
@@ -2846,8 +2857,8 @@ void ath9k_hw_set_gpio(struct ath_hw *ah, u32 gpio, u32 val)
 			AR7010_GPIO_OUT : AR_GPIO_IN_OUT(ah);
 
 		REG_RMW(ah, out_addr, val << gpio, BIT(gpio));
-	} else if (BIT(gpio) & ah->caps.gpio_requested) {
-		gpio_set_value(gpio, val);
+	} else if (ah->gpiods[gpio]) {
+		gpiod_set_value(ah->gpiods[gpio], val);
 	} else {
 		WARN_ON(1);
 	}
diff --git a/drivers/net/wireless/ath/ath9k/hw.h b/drivers/net/wireless/ath/ath9k/hw.h
index eaa07d6dbde00..d9d2f64c55700 100644
--- a/drivers/net/wireless/ath/ath9k/hw.h
+++ b/drivers/net/wireless/ath/ath9k/hw.h
@@ -19,6 +19,7 @@
 
 #include <linux/if_ether.h>
 #include <linux/delay.h>
+#include <linux/gpio/consumer.h>
 #include <linux/io.h>
 #include <linux/firmware.h>
 
@@ -302,7 +303,6 @@ struct ath9k_hw_capabilities {
 	u8 max_rxchains;
 	u8 num_gpio_pins;
 	u32 gpio_mask;
-	u32 gpio_requested;
 	u8 rx_hp_qdepth;
 	u8 rx_lp_qdepth;
 	u8 rx_status_len;
@@ -783,6 +783,7 @@ struct ath_hw {
 	struct ath9k_hw_capabilities caps;
 	struct ath9k_channel channels[ATH9K_NUM_CHANNELS];
 	struct ath9k_channel *curchan;
+	struct gpio_desc *gpiods[32];
 
 	union {
 		struct ar5416_eeprom_def def;
diff --git a/drivers/net/wireless/ath/ath9k/init.c b/drivers/net/wireless/ath/ath9k/init.c
index ee951493e9937..e1a67e8ed09fe 100644
--- a/drivers/net/wireless/ath/ath9k/init.c
+++ b/drivers/net/wireless/ath/ath9k/init.c
@@ -285,7 +285,7 @@ int ath_descdma_setup(struct ath_softc *sc, struct ath_descdma *dd,
 {
 	struct ath_common *common = ath9k_hw_common(sc->sc_ah);
 	u8 *ds;
-	int i, bsize, desc_len;
+	int i, desc_len;
 
 	ath_dbg(common, CONFIG, "%s DMA: %u buffers %u desc/buf\n",
 		name, nbuf, ndesc);
@@ -339,8 +339,7 @@ int ath_descdma_setup(struct ath_softc *sc, struct ath_descdma *dd,
 	if (is_tx) {
 		struct ath_buf *bf;
 
-		bsize = sizeof(struct ath_buf) * nbuf;
-		bf = devm_kzalloc(sc->dev, bsize, GFP_KERNEL);
+		bf = devm_kcalloc(sc->dev, sizeof(*bf), nbuf, GFP_KERNEL);
 		if (!bf)
 			return -ENOMEM;
 
@@ -370,8 +369,7 @@ int ath_descdma_setup(struct ath_softc *sc, struct ath_descdma *dd,
 	} else {
 		struct ath_rxbuf *bf;
 
-		bsize = sizeof(struct ath_rxbuf) * nbuf;
-		bf = devm_kzalloc(sc->dev, bsize, GFP_KERNEL);
+		bf = devm_kcalloc(sc->dev, sizeof(struct ath_rxbuf), nbuf, GFP_KERNEL);
 		if (!bf)
 			return -ENOMEM;
 
@@ -576,7 +574,7 @@ static int ath9k_nvmem_request_eeprom(struct ath_softc *sc)
 	size_t len;
 	int err;
 
-	cell = devm_nvmem_cell_get(sc->dev, "calibration");
+	cell = nvmem_cell_get(sc->dev, "calibration");
 	if (IS_ERR(cell)) {
 		err = PTR_ERR(cell);
 
@@ -593,6 +591,7 @@ static int ath9k_nvmem_request_eeprom(struct ath_softc *sc)
 	}
 
 	buf = nvmem_cell_read(cell, &len);
+	nvmem_cell_put(cell);
 	if (IS_ERR(buf))
 		return PTR_ERR(buf);
 
diff --git a/drivers/net/wireless/ath/ath9k/recv.c b/drivers/net/wireless/ath/ath9k/recv.c
index 34c74ed99b7b5..93b41a1bb2af0 100644
--- a/drivers/net/wireless/ath/ath9k/recv.c
+++ b/drivers/net/wireless/ath/ath9k/recv.c
@@ -202,7 +202,6 @@ static int ath_rx_edma_init(struct ath_softc *sc, int nbufs)
 	struct sk_buff *skb;
 	struct ath_rxbuf *bf;
 	int error = 0, i;
-	u32 size;
 
 	ath9k_hw_set_rx_bufsize(ah, common->rx_bufsize -
 				    ah->caps.rx_status_len);
@@ -212,8 +211,7 @@ static int ath_rx_edma_init(struct ath_softc *sc, int nbufs)
 	ath_rx_edma_init_queue(&sc->rx.rx_edma[ATH9K_RX_QUEUE_HP],
 			       ah->caps.rx_hp_qdepth);
 
-	size = sizeof(struct ath_rxbuf) * nbufs;
-	bf = devm_kzalloc(sc->dev, size, GFP_KERNEL);
+	bf = devm_kcalloc(sc->dev, sizeof(struct ath_rxbuf), nbufs, GFP_KERNEL);
 	if (!bf)
 		return -ENOMEM;
 
diff --git a/drivers/net/wireless/ath/ath9k/xmit.c b/drivers/net/wireless/ath/ath9k/xmit.c
index 0ac9212e42f75..89d8b31787846 100644
--- a/drivers/net/wireless/ath/ath9k/xmit.c
+++ b/drivers/net/wireless/ath/ath9k/xmit.c
@@ -1993,7 +1993,6 @@ void ath_txq_schedule(struct ath_softc *sc, struct ath_txq *txq)
 
 	ieee80211_txq_schedule_start(hw, txq->mac80211_qnum);
 	spin_lock_bh(&sc->chan_lock);
-	rcu_read_lock();
 
 	if (sc->cur_chan->stopped)
 		goto out;
@@ -2011,7 +2010,6 @@ void ath_txq_schedule(struct ath_softc *sc, struct ath_txq *txq)
 	}
 
 out:
-	rcu_read_unlock();
 	spin_unlock_bh(&sc->chan_lock);
 	ieee80211_txq_schedule_end(hw, txq->mac80211_qnum);
 }
@@ -2746,6 +2744,11 @@ void ath_tx_edma_tasklet(struct ath_softc *sc)
 			continue;
 		}
 
+		if (ts.qid >= ATH9K_NUM_TX_QUEUES) {
+			ath_dbg(common, XMIT, "invalid qid %d\n", ts.qid);
+			continue;
+		}
+
 		txq = &sc->tx.txq[ts.qid];
 
 		ath_txq_lock(sc, txq);
diff --git a/drivers/net/wireless/ath/wil6210/cfg80211.c b/drivers/net/wireless/ath/wil6210/cfg80211.c
index 3d6e5aad48b1a..d6ef92cfcbaff 100644
--- a/drivers/net/wireless/ath/wil6210/cfg80211.c
+++ b/drivers/net/wireless/ath/wil6210/cfg80211.c
@@ -2,6 +2,7 @@
 /*
  * Copyright (c) 2012-2017 Qualcomm Atheros, Inc.
  * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
  */
 
 #include <linux/etherdevice.h>
@@ -1734,7 +1735,7 @@ static int wil_remain_on_channel(struct wiphy *wiphy,
 				 struct wireless_dev *wdev,
 				 struct ieee80211_channel *chan,
 				 unsigned int duration,
-				 u64 *cookie)
+				 u64 *cookie, const u8 *rx_addr)
 {
 	struct wil6210_priv *wil = wiphy_to_wil(wiphy);
 	int rc;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
index e1752a513c733..92c16a3173288 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.c
@@ -970,10 +970,12 @@ brcmf_p2p_discover_listen(struct brcmf_p2p_info *p2p, u16 channel, u32 duration)
  * @channel: channel to stay on.
  * @duration: time in ms to remain on channel.
  * @cookie: cookie.
+ * @rx_addr: Address to match against the destination of received frames
  */
 int brcmf_p2p_remain_on_channel(struct wiphy *wiphy, struct wireless_dev *wdev,
 				struct ieee80211_channel *channel,
-				unsigned int duration, u64 *cookie)
+				unsigned int duration, u64 *cookie,
+				const u8 *rx_addr)
 {
 	struct brcmf_cfg80211_info *cfg = wiphy_to_cfg(wiphy);
 	struct brcmf_p2p_info *p2p = &cfg->p2p;
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h
index d3137ebd71582..9f3f01ade2b77 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/p2p.h
@@ -157,7 +157,8 @@ int brcmf_p2p_scan_prep(struct wiphy *wiphy,
 			struct brcmf_cfg80211_vif *vif);
 int brcmf_p2p_remain_on_channel(struct wiphy *wiphy, struct wireless_dev *wdev,
 				struct ieee80211_channel *channel,
-				unsigned int duration, u64 *cookie);
+				unsigned int duration, u64 *cookie,
+				const u8 *rx_addr);
 int brcmf_p2p_notify_listen_complete(struct brcmf_if *ifp,
 				     const struct brcmf_event_msg *e,
 				     void *data);
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
index 45b342ea06373..13662aa4b4ea6 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/pcie.c
@@ -2708,17 +2708,18 @@ static const struct dev_pm_ops brcmf_pciedrvr_pm = {
 
 #define BRCMF_PCIE_DEVICE(dev_id, fw_vend) \
 	{ \
-		BRCM_PCIE_VENDOR_ID_BROADCOM, (dev_id), \
-		PCI_ANY_ID, PCI_ANY_ID, \
-		PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, \
-		BRCMF_DRVDATA_ ## fw_vend \
+		PCI_DEVICE(BRCM_PCIE_VENDOR_ID_BROADCOM, (dev_id)), \
+		.class = PCI_CLASS_NETWORK_OTHER << 8, \
+		.class_mask = 0xffff00, \
+		.driver_data = BRCMF_DRVDATA_ ## fw_vend, \
 	}
 #define BRCMF_PCIE_DEVICE_SUB(dev_id, subvend, subdev, fw_vend) \
 	{ \
-		BRCM_PCIE_VENDOR_ID_BROADCOM, (dev_id), \
-		(subvend), (subdev), \
-		PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, \
-		BRCMF_DRVDATA_ ## fw_vend \
+		PCI_DEVICE_SUB(BRCM_PCIE_VENDOR_ID_BROADCOM, (dev_id), \
+			       (subvend), (subdev)), \
+		.class = PCI_CLASS_NETWORK_OTHER << 8, \
+		.class_mask = 0xffff00, \
+		.driver_data = BRCMF_DRVDATA_ ## fw_vend, \
 	}
 
 static const struct pci_device_id brcmf_pcie_devid_table[] = {
diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c
index 8cec5ad79fdaf..d65a68045ca82 100644
--- a/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c
+++ b/drivers/net/wireless/broadcom/brcm80211/brcmsmac/phy/phy_lcn.c
@@ -1607,7 +1607,6 @@ wlc_lcnphy_radio_2064_channel_tune_4313(struct brcms_phy *pi, u8 channel)
 {
 	uint i;
 	const struct chan_info_2064_lcnphy *ci;
-	u8 rfpll_doubler = 0;
 	u8 pll_pwrup, pll_pwrup_ovr;
 	s32 qFcal;
 	u8 d15, d16, f16, e44, e45;
@@ -1618,18 +1617,12 @@ wlc_lcnphy_radio_2064_channel_tune_4313(struct brcms_phy *pi, u8 channel)
 	u16 g30, d28;
 
 	ci = &chan_info_2064_lcnphy[0];
-	rfpll_doubler = 1;
 
 	mod_radio_reg(pi, RADIO_2064_REG09D, 0x4, 0x1 << 2);
 
 	write_radio_reg(pi, RADIO_2064_REG09E, 0xf);
-	if (!rfpll_doubler) {
-		loop_bw = PLL_2064_LOOP_BW;
-		d30 = PLL_2064_D30;
-	} else {
-		loop_bw = PLL_2064_LOOP_BW_DOUBLER;
-		d30 = PLL_2064_D30_DOUBLER;
-	}
+	loop_bw = PLL_2064_LOOP_BW_DOUBLER;
+	d30 = PLL_2064_D30_DOUBLER;
 
 	if (CHSPEC_IS2G(pi->radio_chanspec)) {
 		for (i = 0; i < ARRAY_SIZE(chan_info_2064_lcnphy); i++)
@@ -1669,7 +1662,7 @@ wlc_lcnphy_radio_2064_channel_tune_4313(struct brcms_phy *pi, u8 channel)
 	e44 = 0;
 	e45 = 0;
 
-	fpfd = rfpll_doubler ? (pi->xtalfreq << 1) : (pi->xtalfreq);
+	fpfd = pi->xtalfreq << 1;
 	if (pi->xtalfreq > 26000000)
 		e44 = 1;
 	if (pi->xtalfreq > 52000000)
diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
index dd64d21174e3e..4bc9bb406e8e8 100644
--- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c
+++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c
@@ -11366,37 +11366,41 @@ static int ipw_wdev_init(struct net_device *dev)
 
 /* PCI driver stuff */
 static const struct pci_device_id card_ids[] = {
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2701, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2702, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2711, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2712, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2721, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2722, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2731, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2732, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2741, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x103c, 0x2741, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2742, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2751, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2752, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2753, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2754, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2761, 0, 0, 0},
-	{PCI_VENDOR_ID_INTEL, 0x1043, 0x8086, 0x2762, 0, 0, 0},
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2701) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2702) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2711) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2712) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2721) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2722) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2731) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2732) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2741) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x103c, 0x2741) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2742) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2751) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2752) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2753) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2754) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2761) },
+	{ PCI_VDEVICE_SUB(INTEL, 0x1043, 0x8086, 0x2762) },
+
 	/*
 	 * This ID conflicts with i40e, but the devices can be differentiated
 	 * because i40e devices use PCI_CLASS_NETWORK_ETHERNET and ipw2200
 	 * devices use PCI_CLASS_NETWORK_OTHER.
 	 */
-	{PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x104f),
-		PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, 0},
-	{PCI_VDEVICE(INTEL, 0x4220), 0},	/* BG */
-	{PCI_VDEVICE(INTEL, 0x4221), 0},	/* BG */
-	{PCI_VDEVICE(INTEL, 0x4223), 0},	/* ABG */
-	{PCI_VDEVICE(INTEL, 0x4224), 0},	/* ABG */
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x104f),
+		.class = PCI_CLASS_NETWORK_OTHER << 8,
+		.class_mask = 0xffff00,
+	},
+	{ PCI_VDEVICE(INTEL, 0x4220) },	/* BG */
+	{ PCI_VDEVICE(INTEL, 0x4221) },	/* BG */
+	{ PCI_VDEVICE(INTEL, 0x4223) },	/* ABG */
+	{ PCI_VDEVICE(INTEL, 0x4224) },	/* ABG */
 
 	/* required last entry */
-	{0,}
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, card_ids);
diff --git a/drivers/net/wireless/intel/iwlwifi/Makefile b/drivers/net/wireless/intel/iwlwifi/Makefile
index 941257b811b48..2a74efe09cf01 100644
--- a/drivers/net/wireless/intel/iwlwifi/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/Makefile
@@ -32,7 +32,7 @@ iwlwifi-objs		+= iwl-dbg-tlv.o
 iwlwifi-objs		+= iwl-trans.o
 
 iwlwifi-objs		+= fw/img.o fw/notif-wait.o fw/rs.o
-iwlwifi-objs		+= fw/dbg.o fw/pnvm.o fw/dump.o
+iwlwifi-objs		+= fw/dbg.o fw/dbg-old.o fw/pnvm.o fw/dump.o
 iwlwifi-objs		+= fw/regulatory.o
 iwlwifi-$(CONFIG_IWLMVM) += fw/paging.o fw/smem.o fw/init.o
 iwlwifi-$(CONFIG_IWLMLD) += fw/smem.o fw/init.o
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
index f0453f3f6ba63..01ca65eb5acdf 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/22000.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -15,26 +15,26 @@
 /* Lowest firmware API version supported */
 #define IWL_22000_UCODE_API_MIN	77
 
-/* Memory offsets and lengths */
-#define IWL_22000_SMEM_OFFSET		0x400000
-#define IWL_22000_SMEM_LEN		0xD0000
-
 #define IWL_CC_A_FW_PRE			"iwlwifi-cc-a0"
+#define IWL_QU_B_HR_B_FW_PRE		"iwlwifi-Qu-b0-hr-b0"
+#define IWL_QU_C_HR_B_FW_PRE		"iwlwifi-Qu-c0-hr-b0"
+#define IWL_QUZ_A_HR_B_FW_PRE		"iwlwifi-QuZ-a0-hr-b0"
 
 #define IWL_CC_A_MODULE_FIRMWARE(api)			\
 	IWL_CC_A_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_QU_B_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_QU_B_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_QUZ_A_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_QUZ_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_QU_C_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_QU_C_HR_B_FW_PRE "-" __stringify(api) ".ucode"
 
 static const struct iwl_family_base_params iwl_22000_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 256,
-	.shadow_ram_support = true,
-	.led_compensation = 57,
 	.wd_timeout = IWL_LONG_WD_TIMEOUT,
-	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
-	.smem_offset = IWL_22000_SMEM_OFFSET,
-	.smem_len = IWL_22000_SMEM_LEN,
 	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
 	.apmg_not_supported = true,
 	.mac_addr_from_csr = 0x380,
@@ -113,4 +113,7 @@ const char iwl_ax201_killer_1650s_name[] =
 const char iwl_ax201_killer_1650i_name[] =
 	"Killer(R) Wi-Fi 6 AX1650i 160MHz Wireless Network Adapter (201NGW)";
 
+MODULE_FIRMWARE(IWL_QU_B_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_QU_C_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_QUZ_A_HR_B_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_CC_A_MODULE_FIRMWARE(IWL_22000_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/7000.c b/drivers/net/wireless/intel/iwlwifi/cfg/7000.c
index f987ad3192c1d..1be72d71fccf6 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/7000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/7000.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2020, 2023, 2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2020, 2023, 2025-2026 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015 Intel Deutschland GmbH
  */
@@ -53,10 +53,7 @@ static const struct iwl_family_base_params iwl7000_base = {
 	.eeprom_size = OTP_LOW_IMAGE_SIZE_16K,
 	.num_of_queues = 31,
 	.max_tfd_queue_size = 256,
-	.shadow_ram_support = true,
-	.led_compensation = 57,
 	.wd_timeout = IWL_LONG_WD_TIMEOUT,
-	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
 	.apmg_wake_up_wa = true,
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/8000.c b/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
index 3c844cd419e89..834aa520ab0cc 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/8000.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2014, 2018-2020, 2023, 2025 Intel Corporation
+ * Copyright (C) 2014, 2018-2020, 2023, 2025-2026 Intel Corporation
  * Copyright (C) 2014-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016 Intel Deutschland GmbH
  */
@@ -39,10 +39,7 @@ static const struct iwl_family_base_params iwl8000_base = {
 	.eeprom_size = OTP_LOW_IMAGE_SIZE_32K,
 	.num_of_queues = 31,
 	.max_tfd_queue_size = 256,
-	.shadow_ram_support = true,
-	.led_compensation = 57,
 	.wd_timeout = IWL_LONG_WD_TIMEOUT,
-	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
 	.nvm_hw_section_num = 10,
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/9000.c b/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
index 5872fc9b8caf4..2954434ce851e 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/9000.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2021, 2023, 2025 Intel Corporation
+ * Copyright (C) 2018-2021, 2023, 2025-2026 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -30,10 +30,7 @@ static const struct iwl_family_base_params iwl9000_base = {
 	.eeprom_size = OTP_LOW_IMAGE_SIZE_32K,
 	.num_of_queues = 31,
 	.max_tfd_queue_size = 256,
-	.shadow_ram_support = true,
-	.led_compensation = 57,
 	.wd_timeout = IWL_LONG_WD_TIMEOUT,
-	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
 	.smem_offset = IWL9000_SMEM_OFFSET,
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/ax210.c b/drivers/net/wireless/intel/iwlwifi/cfg/ax210.c
index 582f61661062e..2519f577669ef 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/ax210.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/ax210.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -15,21 +15,30 @@
 /* Lowest firmware API version supported */
 #define IWL_AX210_UCODE_API_MIN	89
 
-/* Memory offsets and lengths */
-#define IWL_AX210_SMEM_OFFSET		0x400000
-#define IWL_AX210_SMEM_LEN		0xD0000
+#define IWL_SO_A_GF_A_FW_PRE		"iwlwifi-so-a0-gf-a0"
+#define IWL_TY_A_GF_A_FW_PRE		"iwlwifi-ty-a0-gf-a0"
+#define IWL_MA_A_GF_A_FW_PRE		"iwlwifi-ma-a0-gf-a0"
+#define IWL_MA_B_GF_A_FW_PRE		"iwlwifi-ma-b0-gf-a0"
+#define IWL_SO_A_GF4_A_FW_PRE		"iwlwifi-so-a0-gf4-a0"
+#define IWL_MA_A_GF4_A_FW_PRE		"iwlwifi-ma-a0-gf4-a0"
+#define IWL_MA_B_GF4_A_FW_PRE		"iwlwifi-ma-b0-gf4-a0"
+#define IWL_SO_A_HR_B_FW_PRE		"iwlwifi-so-a0-hr-b0"
+#define IWL_MA_A_HR_B_FW_PRE		"iwlwifi-ma-a0-hr-b0"
+#define IWL_MA_B_HR_B_FW_PRE		"iwlwifi-ma-b0-hr-b0"
+
+#define IWL_SO_A_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_SO_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_MA_A_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_MA_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
+#define IWL_MA_B_HR_B_MODULE_FIRMWARE(api)	\
+	IWL_MA_B_HR_B_FW_PRE "-" __stringify(api) ".ucode"
 
 static const struct iwl_family_base_params iwl_ax210_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 65536,
-	.shadow_ram_support = true,
-	.led_compensation = 57,
 	.wd_timeout = IWL_LONG_WD_TIMEOUT,
-	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
-	.smem_offset = IWL_AX210_SMEM_OFFSET,
-	.smem_len = IWL_AX210_SMEM_LEN,
 	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
 	.apmg_not_supported = true,
 	.mac_addr_from_csr = 0x380,
@@ -121,3 +130,14 @@ const struct iwl_mac_cfg iwl_ma_mac_cfg = {
 	.integrated = true,
 	.umac_prph_offset = 0x300000
 };
+
+IWL_FW_AND_PNVM(IWL_SO_A_GF_A_FW_PRE, IWL_AX210_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_TY_A_GF_A_FW_PRE, IWL_AX210_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_MA_A_GF_A_FW_PRE, IWL_AX210_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_MA_B_GF_A_FW_PRE, IWL_AX210_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_SO_A_GF4_A_FW_PRE, IWL_AX210_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_MA_A_GF4_A_FW_PRE, IWL_AX210_UCODE_API_MAX);
+IWL_FW_AND_PNVM(IWL_MA_B_GF4_A_FW_PRE, IWL_AX210_UCODE_API_MAX);
+MODULE_FIRMWARE(IWL_SO_A_HR_B_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_MA_A_HR_B_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
+MODULE_FIRMWARE(IWL_MA_B_HR_B_MODULE_FIRMWARE(IWL_AX210_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/bz.c b/drivers/net/wireless/intel/iwlwifi/cfg/bz.c
index 3653ddbf3ce93..ecb4f81a99f56 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/bz.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/bz.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -10,26 +10,17 @@
 #include "fw/api/txq.h"
 
 /* Highest firmware core release supported */
-#define IWL_BZ_UCODE_CORE_MAX	102
+#define IWL_BZ_UCODE_CORE_MAX	105
 
-/* Lowest firmware API version supported */
-#define IWL_BZ_UCODE_API_MIN	100
-
-/* Memory offsets and lengths */
-#define IWL_BZ_SMEM_OFFSET		0x400000
-#define IWL_BZ_SMEM_LEN			0xD0000
+/* Lowest firmware core release supported */
+#define IWL_BZ_UCODE_CORE_MIN	101
 
 static const struct iwl_family_base_params iwl_bz_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 65536,
-	.shadow_ram_support = true,
-	.led_compensation = 57,
 	.wd_timeout = IWL_LONG_WD_TIMEOUT,
-	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
-	.smem_offset = IWL_BZ_SMEM_OFFSET,
-	.smem_len = IWL_BZ_SMEM_LEN,
 	.apmg_not_supported = true,
 	.mac_addr_from_csr = 0x30,
 	.d3_debug_data_base_addr = 0x401000,
@@ -69,7 +60,7 @@ static const struct iwl_family_base_params iwl_bz_base = {
 	},
 	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
 	.ucode_api_max = ENCODE_CORE_AS_API(IWL_BZ_UCODE_CORE_MAX),
-	.ucode_api_min = IWL_BZ_UCODE_API_MIN,
+	.ucode_api_min = ENCODE_CORE_AS_API(IWL_BZ_UCODE_CORE_MIN),
 };
 
 const struct iwl_mac_cfg iwl_bz_mac_cfg = {
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/dr.c b/drivers/net/wireless/intel/iwlwifi/cfg/dr.c
index 83d893b10f8ec..e8968b3051d3f 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/dr.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/dr.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -9,28 +9,19 @@
 #include "fw/api/txq.h"
 
 /* Highest firmware core release supported */
-#define IWL_DR_UCODE_CORE_MAX	102
+#define IWL_DR_UCODE_CORE_MAX	105
 
-/* Lowest firmware API version supported */
-#define IWL_DR_UCODE_API_MIN	100
-
-/* Memory offsets and lengths */
-#define IWL_DR_SMEM_OFFSET		0x400000
-#define IWL_DR_SMEM_LEN			0xD0000
+/* Lowest firmware core release supported */
+#define IWL_DR_UCODE_CORE_MIN	101
 
 #define IWL_DR_A_PE_A_FW_PRE		"iwlwifi-dr-a0-pe-a0"
 
 static const struct iwl_family_base_params iwl_dr_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 65536,
-	.shadow_ram_support = true,
-	.led_compensation = 57,
 	.wd_timeout = IWL_LONG_WD_TIMEOUT,
-	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
-	.smem_offset = IWL_DR_SMEM_OFFSET,
-	.smem_len = IWL_DR_SMEM_LEN,
 	.apmg_not_supported = true,
 	.mac_addr_from_csr = 0x30,
 	.d3_debug_data_base_addr = 0x401000,
@@ -70,7 +61,7 @@ static const struct iwl_family_base_params iwl_dr_base = {
 	},
 	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
 	.ucode_api_max = ENCODE_CORE_AS_API(IWL_DR_UCODE_CORE_MAX),
-	.ucode_api_min = IWL_DR_UCODE_API_MIN,
+	.ucode_api_min = ENCODE_CORE_AS_API(IWL_DR_UCODE_CORE_MIN),
 };
 
 const struct iwl_mac_cfg iwl_dr_mac_cfg = {
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c
index ad2536f530847..294cf25ae2a66 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-fm.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include "iwl-config.h"
 
@@ -11,9 +11,6 @@
 #define IWL_GL_B_FM_B_FW_PRE		"iwlwifi-gl-b0-fm-b0"
 #define IWL_GL_C_FM_C_FW_PRE		"iwlwifi-gl-c0-fm-c0"
 
-/* NVM versions */
-#define IWL_FM_NVM_VERSION		0x0a1d
-
 #define IWL_DEVICE_FM							\
 	.ht_params = {							\
 		.stbc = true,						\
@@ -27,7 +24,6 @@
 	.uhb_supported = true,						\
 	.eht_supported = true,						\
 	.num_rbds = IWL_NUM_RBDS_EHT,					\
-	.nvm_ver = IWL_FM_NVM_VERSION,					\
 	.nvm_type = IWL_NVM_EXT
 
 const struct iwl_rf_cfg iwl_rf_fm = {
@@ -51,6 +47,8 @@ const char iwl_killer_be1790s_name[] =
 	"Killer(R) Wi-Fi 7 BE1790s 320MHz Wireless Network Adapter (BE401D2W)";
 const char iwl_killer_be1790i_name[] =
 	"Killer(R) Wi-Fi 7 BE1790i 320MHz Wireless Network Adapter (BE401NGW)";
+const char iwl_killer_be1730x_name[] =
+	"Killer(TM) Wi-Fi 7 BE1730x 160MHz Wireless Network Adapter (BE202)";
 
 const char iwl_be201_name[] = "Intel(R) Wi-Fi 7 BE201 320MHz";
 const char iwl_be200_name[] = "Intel(R) Wi-Fi 7 BE200 320MHz";
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-gf.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-gf.c
index c16cda087a7c3..99a5110924cd5 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-gf.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-gf.c
@@ -11,13 +11,6 @@
 /* Lowest firmware API version supported */
 #define IWL_GF_UCODE_API_MIN	100
 
-#define IWL_SO_A_GF_A_FW_PRE		"iwlwifi-so-a0-gf-a0"
-#define IWL_TY_A_GF_A_FW_PRE		"iwlwifi-ty-a0-gf-a0"
-#define IWL_MA_A_GF_A_FW_PRE		"iwlwifi-ma-a0-gf-a0"
-#define IWL_MA_B_GF_A_FW_PRE		"iwlwifi-ma-b0-gf-a0"
-#define IWL_SO_A_GF4_A_FW_PRE		"iwlwifi-so-a0-gf4-a0"
-#define IWL_MA_A_GF4_A_FW_PRE		"iwlwifi-ma-a0-gf4-a0"
-#define IWL_MA_B_GF4_A_FW_PRE		"iwlwifi-ma-b0-gf4-a0"
 #define IWL_BZ_A_GF_A_FW_PRE		"iwlwifi-bz-a0-gf-a0"
 #define IWL_BZ_A_GF4_A_FW_PRE		"iwlwifi-bz-a0-gf4-a0"
 #define IWL_SC_A_GF_A_FW_PRE		"iwlwifi-sc-a0-gf-a0"
@@ -35,9 +28,6 @@
 #define IWL_SC_A_GF4_A_MODULE_FIRMWARE(api) \
 	IWL_SC_A_GF4_A_FW_PRE "-" __stringify(api) ".ucode"
 
-/* NVM versions */
-#define IWL_GF_NVM_VERSION		0x0a1d
-
 const struct iwl_rf_cfg iwl_rf_gf = {
 	.uhb_supported = true,
 	.led_mode = IWL_LED_RF_STATE,
@@ -49,7 +39,6 @@ const struct iwl_rf_cfg iwl_rf_gf = {
 		.ht40_bands = BIT(NL80211_BAND_2GHZ) |
 			      BIT(NL80211_BAND_5GHZ),
 	},
-	.nvm_ver = IWL_GF_NVM_VERSION,
 	.nvm_type = IWL_NVM_EXT,
 	.num_rbds = IWL_NUM_RBDS_HE,
 	.ucode_api_min = IWL_GF_UCODE_API_MIN,
@@ -73,12 +62,6 @@ const char iwl_ax210_name[] = "Intel(R) Wi-Fi 6E AX210 160MHz";
 const char iwl_ax211_name[] = "Intel(R) Wi-Fi 6E AX211 160MHz";
 const char iwl_ax411_name[] = "Intel(R) Wi-Fi 6E AX411 160MHz";
 
-IWL_FW_AND_PNVM(IWL_SO_A_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
-IWL_FW_AND_PNVM(IWL_TY_A_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
-IWL_FW_AND_PNVM(IWL_MA_A_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
-IWL_FW_AND_PNVM(IWL_MA_B_GF_A_FW_PRE, IWL_GF_UCODE_API_MAX);
-IWL_FW_AND_PNVM(IWL_MA_A_GF4_A_FW_PRE, IWL_GF_UCODE_API_MAX);
-IWL_FW_AND_PNVM(IWL_MA_B_GF4_A_FW_PRE, IWL_GF_UCODE_API_MAX);
 MODULE_FIRMWARE(IWL_BZ_A_GF_A_MODULE_FIRMWARE(IWL_GF_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BZ_A_GF4_A_MODULE_FIRMWARE(IWL_GF_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_SC_A_GF_A_MODULE_FIRMWARE(IWL_GF_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-hr.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-hr.c
index 6cf187d92dbf6..16b9075acdd81 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-hr.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-hr.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include "iwl-config.h"
 
@@ -11,28 +11,10 @@
 /* Lowest firmware API version supported */
 #define IWL_HR_UCODE_API_MIN	100
 
-#define IWL_QU_B_HR_B_FW_PRE		"iwlwifi-Qu-b0-hr-b0"
-#define IWL_QU_C_HR_B_FW_PRE		"iwlwifi-Qu-c0-hr-b0"
-#define IWL_QUZ_A_HR_B_FW_PRE		"iwlwifi-QuZ-a0-hr-b0"
-#define IWL_SO_A_HR_B_FW_PRE		"iwlwifi-so-a0-hr-b0"
-#define IWL_MA_A_HR_B_FW_PRE		"iwlwifi-ma-a0-hr-b0"
-#define IWL_MA_B_HR_B_FW_PRE		"iwlwifi-ma-b0-hr-b0"
 #define IWL_BZ_A_HR_B_FW_PRE		"iwlwifi-bz-a0-hr-b0"
 #define IWL_SC_A_HR_A_FW_PRE		"iwlwifi-sc-a0-hr-b0"
 #define IWL_SC_A_HR_B_FW_PRE		"iwlwifi-sc-a0-hr-b0"
 
-#define IWL_QU_B_HR_B_MODULE_FIRMWARE(api)	\
-	IWL_QU_B_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_QUZ_A_HR_B_MODULE_FIRMWARE(api)	\
-	IWL_QUZ_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_QU_C_HR_B_MODULE_FIRMWARE(api)	\
-	IWL_QU_C_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_SO_A_HR_B_MODULE_FIRMWARE(api)	\
-	IWL_SO_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_A_HR_B_FW_MODULE_FIRMWARE(api)	\
-	IWL_MA_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
-#define IWL_MA_B_HR_B_FW_MODULE_FIRMWARE(api)	\
-	IWL_MA_B_HR_B_FW_PRE "-" __stringify(api) ".ucode"
 #define IWL_BZ_A_HR_B_MODULE_FIRMWARE(api)	\
 	IWL_BZ_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
 #define IWL_SC_A_HR_A_FW_MODULE_FIRMWARE(api)	\
@@ -40,9 +22,6 @@
 #define IWL_SC_A_HR_B_FW_MODULE_FIRMWARE(api)	\
 	IWL_SC_A_HR_B_FW_PRE "-" __stringify(api) ".ucode"
 
-/* NVM versions */
-#define IWL_HR_NVM_VERSION		0x0a1d
-
 #define IWL_DEVICE_HR							\
 	.led_mode = IWL_LED_RF_STATE,					\
 	.non_shared_ant = ANT_B,					\
@@ -54,7 +33,6 @@
 			      BIT(NL80211_BAND_5GHZ),			\
 	},								\
 	.num_rbds = IWL_NUM_RBDS_HE,					\
-	.nvm_ver = IWL_HR_NVM_VERSION,					\
 	.nvm_type = IWL_NVM_EXT,					\
 	.ucode_api_min = IWL_HR_UCODE_API_MIN,				\
 	.ucode_api_max = IWL_HR_UCODE_API_MAX
@@ -78,12 +56,6 @@ const char iwl_ax200_name[] = "Intel(R) Wi-Fi 6 AX200 160MHz";
 const char iwl_ax201_name[] = "Intel(R) Wi-Fi 6 AX201 160MHz";
 const char iwl_ax203_name[] = "Intel(R) Wi-Fi 6 AX203";
 
-MODULE_FIRMWARE(IWL_QU_B_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_QU_C_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_QUZ_A_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_SO_A_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_A_HR_B_FW_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
-MODULE_FIRMWARE(IWL_MA_B_HR_B_FW_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_BZ_A_HR_B_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_SC_A_HR_A_FW_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
 MODULE_FIRMWARE(IWL_SC_A_HR_B_FW_MODULE_FIRMWARE(IWL_HR_UCODE_API_MAX));
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c
index 2c29054ce7b84..7a04cb120b1b2 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-pe.c
@@ -1,10 +1,28 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2025 Intel Corporation
+ * Copyright (C) 2025-2026 Intel Corporation
  */
 #include "iwl-config.h"
 
-/* currently iwl_rf_wh/iwl_rf_wh_160mhz are just defines for the FM ones */
+#define IWL_DEVICE_PE							\
+	.ht_params = {							\
+		.stbc = true,						\
+		.ldpc = true,						\
+		.ht40_bands = BIT(NL80211_BAND_2GHZ) |			\
+			      BIT(NL80211_BAND_5GHZ),			\
+	},								\
+	.led_mode = IWL_LED_RF_STATE,					\
+	.non_shared_ant = ANT_B,					\
+	.vht_mu_mimo_supported = true,					\
+	.uhb_supported = true,						\
+	.eht_supported = true,						\
+	.uhr_supported = true,						\
+	.num_rbds = IWL_NUM_RBDS_EHT,					\
+	.nvm_type = IWL_NVM_EXT
+
+const struct iwl_rf_cfg iwl_rf_pe = {
+	IWL_DEVICE_PE,
+};
 
 const char iwl_killer_bn1850w2_name[] =
 	"Killer(R) Wi-Fi 8 BN1850w2 320MHz Wireless Network Adapter (BN201.D2W)";
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/rf-wh.c b/drivers/net/wireless/intel/iwlwifi/cfg/rf-wh.c
index b5803ea1eb782..c432aa1a0af64 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/rf-wh.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/rf-wh.c
@@ -1,12 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2025 Intel Corporation
+ * Copyright (C) 2025-2026 Intel Corporation
  */
 #include "iwl-config.h"
 
-/* NVM versions */
-#define IWL_WH_NVM_VERSION		0x0a1d
-
 #define IWL_DEVICE_WH							\
 	.ht_params = {							\
 		.stbc = true,						\
@@ -19,7 +16,6 @@
 	.vht_mu_mimo_supported = true,					\
 	.uhb_supported = true,						\
 	.num_rbds = IWL_NUM_RBDS_EHT,					\
-	.nvm_ver = IWL_WH_NVM_VERSION,					\
 	.nvm_type = IWL_NVM_EXT
 
 /* currently iwl_rf_wh/iwl_rf_wh_160mhz are just defines for the FM ones */
@@ -33,6 +29,8 @@ const char iwl_killer_be1775s_name[] =
 	"Killer(R) Wi-Fi 7 BE1775s 320MHz Wireless Network Adapter (BE211D2W)";
 const char iwl_killer_be1775i_name[] =
 	"Killer(R) Wi-Fi 7 BE1775i 320MHz Wireless Network Adapter (BE211NGW)";
+const char iwl_killer_be1735x_name[] =
+	"Killer(TM) Wi-Fi 7 BE1735x 160MHz Wireless Network Adapter (BE213)";
 
 const char iwl_be211_name[] = "Intel(R) Wi-Fi 7 BE211 320MHz";
 const char iwl_be213_name[] = "Intel(R) Wi-Fi 7 BE213 160MHz";
diff --git a/drivers/net/wireless/intel/iwlwifi/cfg/sc.c b/drivers/net/wireless/intel/iwlwifi/cfg/sc.c
index 749d46dc0236a..6aaa49aeec992 100644
--- a/drivers/net/wireless/intel/iwlwifi/cfg/sc.c
+++ b/drivers/net/wireless/intel/iwlwifi/cfg/sc.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include <linux/module.h>
 #include <linux/stringify.h>
@@ -10,17 +10,10 @@
 #include "fw/api/txq.h"
 
 /* Highest firmware core release supported */
-#define IWL_SC_UCODE_CORE_MAX	102
+#define IWL_SC_UCODE_CORE_MAX	105
 
-/* Lowest firmware API version supported */
-#define IWL_SC_UCODE_API_MIN	100
-
-/* NVM versions */
-#define IWL_SC_NVM_VERSION		0x0a1d
-
-/* Memory offsets and lengths */
-#define IWL_SC_SMEM_OFFSET		0x400000
-#define IWL_SC_SMEM_LEN			0xD0000
+/* Lowest firmware core release supported */
+#define IWL_SC_UCODE_CORE_MIN	101
 
 #define IWL_SC_A_FM_B_FW_PRE		"iwlwifi-sc-a0-fm-b0"
 #define IWL_SC_A_FM_C_FW_PRE		"iwlwifi-sc-a0-fm-c0"
@@ -31,14 +24,9 @@
 static const struct iwl_family_base_params iwl_sc_base = {
 	.num_of_queues = 512,
 	.max_tfd_queue_size = 65536,
-	.shadow_ram_support = true,
-	.led_compensation = 57,
 	.wd_timeout = IWL_LONG_WD_TIMEOUT,
-	.max_event_log_size = 512,
 	.shadow_reg_enable = true,
 	.pcie_l1_allowed = true,
-	.smem_offset = IWL_SC_SMEM_OFFSET,
-	.smem_len = IWL_SC_SMEM_LEN,
 	.apmg_not_supported = true,
 	.mac_addr_from_csr = 0x30,
 	.d3_debug_data_base_addr = 0x401000,
@@ -78,7 +66,7 @@ static const struct iwl_family_base_params iwl_sc_base = {
 	},
 	.features = IWL_TX_CSUM_NETIF_FLAGS | NETIF_F_RXCSUM,
 	.ucode_api_max = ENCODE_CORE_AS_API(IWL_SC_UCODE_CORE_MAX),
-	.ucode_api_min = IWL_SC_UCODE_API_MIN,
+	.ucode_api_min = ENCODE_CORE_AS_API(IWL_SC_UCODE_CORE_MIN),
 };
 
 const struct iwl_mac_cfg iwl_sc_mac_cfg = {
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
index 36159a7699167..abd2593505895 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/commands.h
@@ -2,7 +2,7 @@
 /*
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2022, 2024-2025 Intel Corporation
+ * Copyright (C) 2018-2022, 2024-2026 Intel Corporation
  */
 #ifndef __iwl_fw_api_commands_h__
 #define __iwl_fw_api_commands_h__
@@ -22,7 +22,6 @@
  *	&enum iwl_data_path_subcmd_ids
  * @SCAN_GROUP: scan group, uses command IDs from
  *	&enum iwl_scan_subcmd_ids
- * @NAN_GROUP: NAN group, uses command IDs from &enum iwl_nan_subcmd_ids
  * @LOCATION_GROUP: location group, uses command IDs from
  *	&enum iwl_location_subcmd_ids
  * @BT_COEX_GROUP: bt coex group, uses command IDs from
@@ -43,7 +42,6 @@ enum iwl_mvm_command_groups {
 	PHY_OPS_GROUP = 0x4,
 	DATA_PATH_GROUP = 0x5,
 	SCAN_GROUP = 0x6,
-	NAN_GROUP = 0x7,
 	LOCATION_GROUP = 0x8,
 	BT_COEX_GROUP = 0x9,
 	PROT_OFFLOAD_GROUP = 0xb,
@@ -59,8 +57,7 @@ enum iwl_legacy_cmds {
 	/**
 	 * @UCODE_ALIVE_NTFY:
 	 * Alive data from the firmware, as described in
-	 * &struct iwl_alive_ntf_v3 or &struct iwl_alive_ntf_v4 or
-	 * &struct iwl_alive_ntf_v5 or &struct iwl_alive_ntf_v7.
+	 * &struct iwl_alive_ntf_v3 or &struct iwl_alive_ntf_v7.
 	 */
 	UCODE_ALIVE_NTFY = 0x1,
 
@@ -386,7 +383,7 @@ enum iwl_legacy_cmds {
 	 * @STATISTICS_NOTIFICATION:
 	 * one of &struct iwl_notif_statistics_v10,
 	 * &struct iwl_notif_statistics_v11,
-	 * &struct iwl_notif_statistic,
+	 * &struct iwl_notif_statistics,
 	 * &struct iwl_statistics_operational_ntfy_ver_14
 	 * &struct iwl_statistics_operational_ntfy
 	 */
@@ -560,7 +557,7 @@ enum iwl_legacy_cmds {
 	WOWLAN_CONFIGURATION = 0xe1,
 
 	/**
-	 * @WOWLAN_TSC_RSC_PARAM: &struct iwl_wowlan_rsc_tsc_params_cmd_v4,
+	 * @WOWLAN_TSC_RSC_PARAM: &struct iwl_wowlan_rsc_tsc_params_cmd_ver_2,
 	 *	&struct iwl_wowlan_rsc_tsc_params_cmd
 	 */
 	WOWLAN_TSC_RSC_PARAM = 0xe2,
@@ -653,7 +650,7 @@ enum iwl_system_subcmd_ids {
 enum iwl_statistics_subcmd_ids {
 	/**
 	 * @STATISTICS_OPER_NOTIF: Notification about operational
-	 *	statistics &struct iwl_system_statistics_notif_oper
+	 *	statistics &struct iwl_system_statistics_notif_oper_v3
 	 */
 	STATISTICS_OPER_NOTIF = 0x0,
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
index 06370c161fe4b..e494e5b18d225 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/datapath.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  * Copyright (C) 2012-2014, 2018-2022 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
@@ -91,6 +91,13 @@ enum iwl_data_path_subcmd_ids {
 	 */
 	SEC_KEY_CMD = 0x18,
 
+	/**
+	 * @RSC_NOTIF: notification to update each Rx queue with the RSC. This
+	 *	notification is sent after resume and uses
+	 *	&struct iwl_wowlan_all_rsc_tsc_v5.
+	 */
+	RSC_NOTIF = 0xF1,
+
 	/**
 	 * @ESR_MODE_NOTIF: notification to recommend/force a wanted esr mode,
 	 *	uses &struct iwl_esr_mode_notif or &struct iwl_esr_mode_notif_v1
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/location.h b/drivers/net/wireless/intel/iwlwifi/fw/api/location.h
index 2ee3a48aa5dfe..421ea94ace017 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/location.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/location.h
@@ -2,7 +2,7 @@
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  * Copyright (C) 2018-2022 Intel Corporation
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #ifndef __iwl_fw_api_location_h__
 #define __iwl_fw_api_location_h__
@@ -81,12 +81,53 @@ enum iwl_location_subcmd_ids {
 	 * @TOF_RANGE_RESPONSE_NOTIF: ranging response, using one of
 	 *	&struct iwl_tof_range_rsp_ntfy_v5,
 	 *	&struct iwl_tof_range_rsp_ntfy_v6,
-	 *	&struct iwl_tof_range_rsp_ntfy_v7 or
-	 *	&struct iwl_tof_range_rsp_ntfy_v8
+	 *	&struct iwl_tof_range_rsp_ntfy_v7,
+	 *	&struct iwl_tof_range_rsp_ntfy_v9 or
+	 *	&struct iwl_tof_range_rsp_ntfy
 	 */
 	TOF_RANGE_RESPONSE_NOTIF = 0xFF,
 };
 
+/**
+ * enum iwl_location_frame_format - location frame formats
+ * @IWL_LOCATION_FRAME_FORMAT_LEGACY: legacy
+ * @IWL_LOCATION_FRAME_FORMAT_HT: HT
+ * @IWL_LOCATION_FRAME_FORMAT_VHT: VHT
+ * @IWL_LOCATION_FRAME_FORMAT_HE: HE
+ */
+enum iwl_location_frame_format {
+	IWL_LOCATION_FRAME_FORMAT_LEGACY,
+	IWL_LOCATION_FRAME_FORMAT_HT,
+	IWL_LOCATION_FRAME_FORMAT_VHT,
+	IWL_LOCATION_FRAME_FORMAT_HE,
+};
+
+/**
+ * enum iwl_location_bw - location bandwidth selection
+ * @IWL_LOCATION_BW_20MHZ: 20 MHz
+ * @IWL_LOCATION_BW_40MHZ: 40 MHz
+ * @IWL_LOCATION_BW_80MHZ: 80 MHz
+ * @IWL_LOCATION_BW_160MHZ: 160 MHz
+ * @IWL_LOCATION_BW_320MHZ: 320 MHz
+ */
+enum iwl_location_bw {
+	IWL_LOCATION_BW_20MHZ,
+	IWL_LOCATION_BW_40MHZ,
+	IWL_LOCATION_BW_80MHZ,
+	IWL_LOCATION_BW_160MHZ,
+	IWL_LOCATION_BW_320MHZ,
+};
+
+/**
+ * enum iwl_location_format_bw - format/BW encoding
+ * @IWL_LOCATION_FMT_BW_FORMAT: &enum iwl_location_frame_format
+ * @IWL_LOCATION_FMT_BW_BANDWIDTH: &enum iwl_location_bw
+ */
+enum iwl_location_format_bw {
+	IWL_LOCATION_FMT_BW_FORMAT	= 0x0f,
+	IWL_LOCATION_FMT_BW_BANDWIDTH	= 0xf0,
+};
+
 /**
  * struct iwl_tof_config_cmd - ToF configuration
  * @tof_disabled: indicates if ToF is disabled (or not)
@@ -263,8 +304,7 @@ struct iwl_tof_responder_config_cmd_v6 {
  * struct iwl_tof_responder_config_cmd_v7 - ToF AP mode (for debug)
  * @cmd_valid_fields: &iwl_tof_responder_cmd_valid_field
  * @responder_cfg_flags: &iwl_tof_responder_cfg_flags
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @rate: current AP rate
  * @channel_num: current AP Channel
  * @ctrl_ch_position: coding of the control channel position relative to
@@ -302,8 +342,7 @@ struct iwl_tof_responder_config_cmd_v7 {
  * struct iwl_tof_responder_config_cmd_v8 - ToF AP mode (for debug)
  * @cmd_valid_fields: &iwl_tof_responder_cmd_valid_field
  * @responder_cfg_flags: &iwl_tof_responder_cfg_flags
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @rate: current AP rate
  * @channel_num: current AP Channel
  * @ctrl_ch_position: coding of the control channel position relative to
@@ -348,8 +387,7 @@ struct iwl_tof_responder_config_cmd_v8 {
  * struct iwl_tof_responder_config_cmd_v9 - ToF AP mode (for debug)
  * @cmd_valid_fields: &iwl_tof_responder_cmd_valid_field
  * @responder_cfg_flags: &iwl_tof_responder_cfg_flags
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @bss_color: current AP bss_color
  * @channel_num: current AP Channel
  * @ctrl_ch_position: coding of the control channel position relative to
@@ -400,8 +438,7 @@ struct iwl_tof_responder_config_cmd_v9 {
  * struct iwl_tof_responder_config_cmd - ToF AP mode
  * @cmd_valid_fields: &iwl_tof_responder_cmd_valid_field
  * @responder_cfg_flags: &iwl_tof_responder_cfg_flags
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @bss_color: current AP bss_color
  * @channel_num: current AP Channel
  * @ctrl_ch_position: coding of the control channel position relative to
@@ -676,44 +713,13 @@ struct iwl_tof_range_req_ap_entry_v3 {
 	__le32 tsf_delta;
 } __packed; /* LOCATION_RANGE_REQ_AP_ENTRY_CMD_API_S_VER_3 */
 
-/**
- * enum iwl_location_frame_format - location frame formats
- * @IWL_LOCATION_FRAME_FORMAT_LEGACY: legacy
- * @IWL_LOCATION_FRAME_FORMAT_HT: HT
- * @IWL_LOCATION_FRAME_FORMAT_VHT: VHT
- * @IWL_LOCATION_FRAME_FORMAT_HE: HE
- */
-enum iwl_location_frame_format {
-	IWL_LOCATION_FRAME_FORMAT_LEGACY,
-	IWL_LOCATION_FRAME_FORMAT_HT,
-	IWL_LOCATION_FRAME_FORMAT_VHT,
-	IWL_LOCATION_FRAME_FORMAT_HE,
-};
-
-/**
- * enum iwl_location_bw - location bandwidth selection
- * @IWL_LOCATION_BW_20MHZ: 20MHz
- * @IWL_LOCATION_BW_40MHZ: 40MHz
- * @IWL_LOCATION_BW_80MHZ: 80MHz
- * @IWL_LOCATION_BW_160MHZ: 160MHz
- */
-enum iwl_location_bw {
-	IWL_LOCATION_BW_20MHZ,
-	IWL_LOCATION_BW_40MHZ,
-	IWL_LOCATION_BW_80MHZ,
-	IWL_LOCATION_BW_160MHZ,
-};
-
 #define TK_11AZ_LEN	32
 
-#define LOCATION_BW_POS	4
-
 /**
  * struct iwl_tof_range_req_ap_entry_v4 - AP configuration parameters
  * @initiator_ap_flags: see &enum iwl_initiator_ap_flags.
  * @channel_num: AP Channel number
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @ctrl_ch_position: Coding of the control channel position relative to the
  *	center frequency, see iwl_mvm_get_ctrl_pos().
  * @ftmr_max_retries: Max number of retries to send the FTMR in case of no
@@ -763,8 +769,7 @@ enum iwl_location_cipher {
  * struct iwl_tof_range_req_ap_entry_v6 - AP configuration parameters
  * @initiator_ap_flags: see &enum iwl_initiator_ap_flags.
  * @channel_num: AP Channel number
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @ctrl_ch_position: Coding of the control channel position relative to the
  *	center frequency, see iwl_mvm_get_ctrl_pos().
  * @ftmr_max_retries: Max number of retries to send the FTMR in case of no
@@ -810,8 +815,7 @@ struct iwl_tof_range_req_ap_entry_v6 {
  * struct iwl_tof_range_req_ap_entry_v7 - AP configuration parameters
  * @initiator_ap_flags: see &enum iwl_initiator_ap_flags.
  * @channel_num: AP Channel number
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @ctrl_ch_position: Coding of the control channel position relative to the
  *	center frequency, see iwl_mvm_get_ctrl_pos().
  * @ftmr_max_retries: Max number of retries to send the FTMR in case of no
@@ -868,8 +872,7 @@ struct iwl_tof_range_req_ap_entry_v7 {
  * struct iwl_tof_range_req_ap_entry_v8 - AP configuration parameters
  * @initiator_ap_flags: see &enum iwl_initiator_ap_flags.
  * @channel_num: AP Channel number
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @ctrl_ch_position: Coding of the control channel position relative to the
  *	center frequency, see iwl_mvm_get_ctrl_pos().
  * @ftmr_max_retries: Max number of retries to send the FTMR in case of no
@@ -939,8 +942,7 @@ struct iwl_tof_range_req_ap_entry_v8 {
  * struct iwl_tof_range_req_ap_entry_v9 - AP configuration parameters
  * @initiator_ap_flags: see &enum iwl_initiator_ap_flags.
  * @channel_num: AP Channel number
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @ctrl_ch_position: Coding of the control channel position relative to the
  *	center frequency, see iwl_mvm_get_ctrl_pos().
  * @ftmr_max_retries: Max number of retries to send the FTMR in case of no
@@ -1024,8 +1026,7 @@ struct iwl_tof_range_req_ap_entry_v9 {
  * @initiator_ap_flags: see &enum iwl_initiator_ap_flags.
  * @band: 0 for 5.2 GHz, 1 for 2.4 GHz, 2 for 6GHz
  * @channel_num: AP Channel number
- * @format_bw: bits 0 - 3: &enum iwl_location_frame_format.
- *             bits 4 - 7: &enum iwl_location_bw.
+ * @format_bw: &enum iwl_location_format_bw
  * @ctrl_ch_position: Coding of the control channel position relative to the
  *	center frequency, see iwl_mvm_get_ctrl_pos().
  * @bssid: AP's BSSID
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
index b398c582b8670..09b2cddc4ad2d 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/mac-cfg.h
@@ -8,6 +8,7 @@
 #define __iwl_fw_api_mac_cfg_h__
 
 #include "mac.h"
+#include "phy-ctxt.h"
 
 /**
  * enum iwl_mac_conf_subcmd_ids - mac configuration command IDs
@@ -71,6 +72,23 @@ enum iwl_mac_conf_subcmd_ids {
 	 * @NAN_CFG_CMD: &struct iwl_nan_config_cmd
 	 */
 	NAN_CFG_CMD = 0x12,
+	/**
+	 * @NAN_SCHEDULE_CMD: &struct iwl_nan_schedule_cmd
+	 */
+	NAN_SCHEDULE_CMD = 0x13,
+	/**
+	 * @NAN_PEER_CMD: &struct iwl_nan_peer_cmd
+	 */
+	NAN_PEER_CMD = 0x14,
+	/**
+	 * @NAN_ULW_ATTR_NOTIF: &struct iwl_nan_ulw_attr_notif
+	 */
+	NAN_ULW_ATTR_NOTIF = 0xf2,
+	/**
+	 * @NAN_SCHED_UPDATE_COMPLETED_NOTIF:
+	 *	&struct iwl_nan_sched_update_completed_notif
+	 */
+	NAN_SCHED_UPDATE_COMPLETED_NOTIF = 0xf3,
 	/**
 	 * @NAN_DW_END_NOTIF: &struct iwl_nan_dw_end_notif
 	 */
@@ -536,12 +554,20 @@ enum iwl_link_ctx_protection_flags {
  *      radar pulses).
  * @LINK_FLG_NDP_FEEDBACK_ENABLED: mark support for NDP feedback and change
  *	of threshold
+ * @LINK_FLG_NPCA: NPCA enabled
+ * @LINK_FLG_DPS: AP is a DPS assisting AP
+ * @LINK_FLG_MLPM: AP supports UHR multi-link PM
+ * @LINK_FLG_DUO: AP supports UHR DUO
  */
 enum iwl_link_ctx_flags {
 	LINK_FLG_BSS_COLOR_DIS		= BIT(0),
 	LINK_FLG_MU_EDCA_CW		= BIT(1),
 	LINK_FLG_RU_2MHZ_BLOCK		= BIT(2),
 	LINK_FLG_NDP_FEEDBACK_ENABLED	= BIT(3),
+	LINK_FLG_NPCA			= BIT(4),
+	LINK_FLG_DPS			= BIT(6),
+	LINK_FLG_MLPM			= BIT(7),
+	LINK_FLG_DUO			= BIT(8),
 }; /* LINK_CONTEXT_FLAG_E_VER_1 */
 
 /**
@@ -578,7 +604,7 @@ enum iwl_npca_flags {
  * @initial_qsrc: Indicates the value that is used to initialize the
  *	EDCAF QSRC[AC] variables
  * @min_dur_threshold: minimum PPDU time to switch to the non-primary
- *	NPCA channel (usec)
+ *	NPCA channel (spec representation)
  * @flags: NPCA flags, see &enum iwl_npca_flags
  * @reserved: reserved for alignment purposes
  */
@@ -726,6 +752,12 @@ struct iwl_link_config_cmd {
  * @STATION_TYPE_NAN_PEER_NDI: NAN data peer station type. A station
  *	of this type can have any number of links (even none) set in the
  *	link_mask. (Supported since version 3.)
+ * @STATION_TYPE_NAN_BCAST: NAN station used for synchronization and
+ *	discovery. No queue is associated with this station.
+ * @STATION_TYPE_NAN_MGMT: NAN station used for NAN management frames, e.g.,
+ *	SDFs and NAFs.
+ * @STATION_TYPE_NAN_MCAST_DATA: NAN station used for multicast NAN data
+ *	frames.
  * @STATION_TYPE_MAX: maximum number of FW station types
  * @STATION_TYPE_AUX: aux sta. In the FW there is no need for a special type
  *	for the aux sta, so this type is only for driver - internal use.
@@ -736,6 +768,9 @@ enum iwl_fw_sta_type {
 	STATION_TYPE_MCAST,
 	STATION_TYPE_NAN_PEER_NMI,
 	STATION_TYPE_NAN_PEER_NDI,
+	STATION_TYPE_NAN_BCAST,
+	STATION_TYPE_NAN_MGMT,
+	STATION_TYPE_NAN_MCAST_DATA,
 	STATION_TYPE_MAX,
 	STATION_TYPE_AUX = STATION_TYPE_MAX /* this doesn't exist in FW */
 }; /* STATION_TYPE_E_VER_1, _VER_2 */
@@ -873,7 +908,9 @@ struct iwl_sta_cfg_cmd_v2 {
  * ( STA_CONFIG_CMD = 0xA )
  *
  * @sta_id: index of station in uCode's station table
- * @link_mask: bitmap of link FW IDs used with this STA
+ * @link_mask: bitmap of link FW IDs used with this STA. Should be set to 0
+ *	for STATION_TYPE_NAN_BCAST and STATION_TYPE_NAN_MGMT as they are not
+ *	associated with any link added by the driver.
  * @peer_mld_address: the peers mld address
  * @reserved_for_peer_mld_address: reserved
  * @peer_link_address: the address of the link that is used to communicate
@@ -908,7 +945,8 @@ struct iwl_sta_cfg_cmd_v2 {
  * @mic_prep_pad_delay: MIC prep time padding
  * @mic_compute_pad_delay: MIC compute time padding
  * @nmi_sta_id: for an NDI peer STA, the NMI peer STA ID it relates to
- * @ndi_local_addr: for an NDI peer STA, the local NDI interface MAC address
+ * @ndi_local_addr: for an NDI peer STA or NAN multicast data station,
+ *	the local NDI interface MAC address
  * @reserved: Reserved for alignment
  */
 struct iwl_sta_cfg_cmd {
@@ -1204,7 +1242,8 @@ enum iwl_nan_flags {
  * @discovery_beacon_interval: discovery beacon interval in TUs
  * @cluster_id: lower last two bytes of the cluster ID, in case the local
  *	device starts a cluster
- * @sta_id: station ID of the NAN station
+ * @sta_id: station ID of the NAN station. Used only in version 1, in version 2
+ *	it is reserved.
  * @hb_channel: channel for 5 GHz if the device supports operation on 5 GHz.
  *	Valid values are 44 and 149, which correspond to the 5 GHz channel, and
  *	0 which means that NAN operation on the 5 GHz band is disabled.
@@ -1242,7 +1281,102 @@ struct iwl_nan_config_cmd {
 	__le32 nan_attr_len;
 	__le32 nan_vendor_elems_len;
 	u8 beacon_data[];
-} __packed; /*  NAN_CONFIG_CMD_API_S_VER_1 */
+} __packed; /*  NAN_CONFIG_CMD_API_S_VER_1, NAN_CONFIG_CMD_API_S_VER_2 */
+
+/**
+ * struct iwl_nan_schedule_cmd_v1 - NAN schedule command
+ * @channels: per channel information
+ * @channels.availability_map: bitmap of slots this channel is advertising
+ *	availability on, will be ULW'ed out if no link/inactive link is
+ *	referenced by the link ID below
+ * @channels.channel_entry: NAN channel entry descriptor
+ * @channels.link_id: FW link ID, or %0xFF for unset
+ * @channels.reserved: (reserved)
+ */
+struct iwl_nan_schedule_cmd_v1 {
+	struct {
+		__le32 availability_map;
+		u8 channel_entry[6];
+		u8 link_id;
+		u8 reserved;
+	} __packed channels[NUM_PHY_CTX];
+} __packed; /* NAN_SCHEDULE_CMD_API_S_VER_1 */
+
+#define IWL_MAX_AVAILABILITY_ATTR_LEN 54
+
+/**
+ * struct iwl_nan_schedule_cmd - NAN schedule command
+ * @channels: per channel information
+ * @channels.availability_map: bitmap of slots this channel is advertising
+ *	availability on, will be ULW'ed out if no link/inactive link is
+ *	referenced by the link ID below
+ * @channels.channel_entry: NAN channel entry descriptor
+ * @channels.link_id: FW link ID, or %0xFF for unset
+ * @channels.reserved: (reserved)
+ * @avail_attr: NAN availability attribute information
+ * @avail_attr.attr_len: length of the availability attribute
+ * @avail_attr.reserved: reserved
+ * @avail_attr.attr: the availability attribute including the attribute header
+ * @deferred: true if the firmware should defer applying the schedule until
+ *	notifying all peers. For a deferred schedule update, the firmware should
+ *	send a notification to the driver after the new schedule is applied.
+ * @reserved: reserved
+ */
+struct iwl_nan_schedule_cmd {
+	struct {
+		__le32 availability_map;
+		u8 channel_entry[6];
+		u8 link_id;
+		u8 reserved;
+	} __packed channels[NUM_PHY_CTX];
+
+	struct {
+		u8 attr_len;
+		u8 reserved;
+		u8 attr[IWL_MAX_AVAILABILITY_ATTR_LEN];
+	} __packed avail_attr;
+
+	u8 deferred;
+	u8 reserved[3];
+} __packed; /* NAN_SCHEDULE_CMD_API_S_VER_2 */
+
+/**
+ * struct iwl_nan_peer_cmd - NAN peer command
+ * @nmi_sta_id: NAN management station ID
+ * @sequence_id: NAN Availability attribute sequence ID
+ * @committed_dw_info: committed DW info from the NAN Device
+ *	Capability attribute
+ * @max_channel_switch_time: maximum channel switch time
+ *	(in microseconds); 0 means unavailable
+ * @reserved: (reserved)
+ * @per_phy: per-PHY information for this peer, indexed by PHY ID
+ * @per_phy.availability_map: bitmap of which slots this peer
+ *	is available in on this PHY. 0 indicates the this per-PHY entry
+ *	is unused.
+ * @per_phy.channel_entry: the channel description the peer is using,
+ *	used for comparisons in ULW management
+ * @per_phy.link_id: FW link ID, should be a valid id.
+ * @per_phy.map_id: map ID from peer's NAN Availability attributec
+ * @initial_ulw_size: size of the initial ULW blob
+ * @initial_ulw: initial ULW data from the peer
+ */
+struct iwl_nan_peer_cmd {
+	u8 nmi_sta_id;
+	u8 sequence_id;
+	__le16 committed_dw_info;
+	__le16 max_channel_switch_time;
+	__le16 reserved;
+
+	struct {
+		__le32 availability_map;
+		u8 channel_entry[6];
+		u8 link_id;
+		u8 map_id;
+	} __packed per_phy[NUM_PHY_CTX];
+
+	__le32 initial_ulw_size;
+	u8 initial_ulw[];
+} __packed; /* NAN_PEER_SCHEDULE_CMD_API_S_VER_1 */
 
 /**
  * enum iwl_nan_cluster_notif_flags - flags for the cluster notification
@@ -1280,4 +1414,44 @@ struct iwl_nan_dw_end_notif {
 	u8 reserved[3];
 } __packed; /* NAN_DW_END_NTF_API_S_VER_1 */
 
+#define IWL_NAN_MAX_ENDLESS_ULW_ATTR_LEN	48
+
+/**
+ * struct iwl_nan_ulw_attr_notif - sent to notify the host of a change in the
+ *	ULW attribute
+ *
+ * @attr_len: length of the ULW attribute in bytes
+ * @reserved: reserved
+ * @attr: the ULW attribute including the attribute header
+ */
+struct iwl_nan_ulw_attr_notif {
+	u8 attr_len;
+	u8 reserved[3];
+	u8 attr[IWL_NAN_MAX_ENDLESS_ULW_ATTR_LEN];
+} __packed; /* NAN_ULW_ATTR_NOTIF_API_S_VER_1 */
+
+/**
+ * enum iwl_nan_sched_update_status - NAN schedule update status
+ *
+ * @IWL_NAN_SCHED_UPDATE_SUCCESS: schedule update completed successfully
+ * @IWL_NAN_SCHED_UPDATE_FAILURE: schedule update failed. Currently not expected
+ *	to happen, but reserved for future use.
+ */
+enum iwl_nan_sched_update_status {
+	IWL_NAN_SCHED_UPDATE_SUCCESS = 0,
+	IWL_NAN_SCHED_UPDATE_FAILURE = 1,
+};
+
+/**
+ * struct iwl_nan_sched_update_completed_notif - NAN schedule update completed
+ *
+ * @status: status of the schedule update operation. See
+ *	&enum iwl_nan_sched_update_status
+ * @reserved: reserved
+ */
+struct iwl_nan_sched_update_completed_notif {
+	u8 status;
+	u8 reserved[3];
+} __packed; /* NAN_SCHED_UPDATE_COMPLETED_NTF_API_S_VER_1 */
+
 #endif /* __iwl_fw_api_mac_cfg_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
index a3f916630df2b..115e65ba19f8c 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/power.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -85,12 +85,13 @@ struct iwl_ltr_config_cmd {
  *		'1' PM could sleep over DTIM till listen Interval.
  * @POWER_FLAGS_SNOOZE_ENA_MSK: Enable snoozing only if uAPSD is enabled and all
  *		access categories are both delivery and trigger enabled.
+ *		(Not supported since version 3)
  * @POWER_FLAGS_BT_SCO_ENA: Enable BT SCO coex only if uAPSD and
  *		PBW Snoozing enabled
  * @POWER_FLAGS_ADVANCE_PM_ENA_MSK: Advanced PM (uAPSD) enable mask
  * @POWER_FLAGS_LPRX_ENA_MSK: Low Power RX enable.
  * @POWER_FLAGS_UAPSD_MISBEHAVING_ENA_MSK: AP/GO's uAPSD misbehaving
- *		detection enablement
+ *		detection enablement (Not supported since version 3)
  * @POWER_FLAGS_ENABLE_SMPS_MSK: SMPS is allowed for this vif
 */
 enum iwl_power_flags {
@@ -175,9 +176,9 @@ struct iwl_device_power_cmd {
 } __packed;
 
 /**
- * struct iwl_mac_power_cmd - New power command containing uAPSD support
+ * struct iwl_mac_power_cmd_v2 - power command V2 containing uAPSD support
  * MAC_PM_POWER_TABLE = 0xA9 (command, has simple generic response)
- * @id_and_color:	MAC contex identifier, &enum iwl_ctxt_id_and_color
+ * @id_and_color:	MAC context identifier, &enum iwl_ctxt_id_and_color
  * @flags:		Power table command flags from POWER_FLAGS_*
  * @keep_alive_seconds:	Keep alive period in seconds. Default - 25 sec.
  *			Minimum allowed:- 3 * DTIM. Keep alive period must be
@@ -216,7 +217,7 @@ struct iwl_device_power_cmd {
  * @limited_ps_threshold: (unused)
  * @reserved: reserved (padding)
  */
-struct iwl_mac_power_cmd {
+struct iwl_mac_power_cmd_v2 {
 	/* CONTEXT_DESC_API_T_VER_1 */
 	__le32 id_and_color;
 
@@ -242,6 +243,43 @@ struct iwl_mac_power_cmd {
 	u8 reserved;
 } __packed; /* CLIENT_PM_POWER_TABLE_S_VER_1, VER_2 */
 
+/**
+ * struct iwl_mac_power_cmd - power command
+ * MAC_PM_POWER_TABLE = 0xA9 (command, has simple generic response)
+ * @id_and_color:	MAC context identifier, &enum iwl_ctxt_id_and_color
+ * @flags:		Power table command flags from POWER_FLAGS_*
+ * @keep_alive_seconds:	Keep alive period in seconds. Default - 25 sec.
+ *			Minimum allowed:- 3 * DTIM. Keep alive period must be
+ *			set regardless of power scheme or current power state.
+ *			FW use this value also when PM is disabled.
+ * @rx_data_timeout:    Minimum time (usec) from last Rx packet for AM to
+ *			PSM transition - legacy PM
+ * @tx_data_timeout:    Minimum time (usec) from last Tx packet for AM to
+ *			PSM transition - legacy PM
+ * @lprx_rssi_threshold: Signal strength up to which LP RX can be enabled.
+ *			Default: 80dbm
+ * @skip_dtim_periods:	Number of DTIM periods to skip if Skip over DTIM flag
+ *			is set. For example, if it is required to skip over
+ *			one DTIM, this value need to be set to 2 (DTIM periods).
+ * @qndp_tid:		TID client shall use for uAPSD QNDP triggers
+ * @uapsd_ac_flags:	Set trigger-enabled and delivery-enabled indication for
+ *			each corresponding AC.
+ *			Use IEEE80211_WMM_IE_STA_QOSINFO_AC* for correct values.
+ */
+struct iwl_mac_power_cmd {
+	/* CONTEXT_DESC_API_T_VER_1 */
+	__le32 id_and_color;
+
+	__le16 flags;
+	__le16 keep_alive_seconds;
+	__le32 rx_data_timeout;
+	__le32 tx_data_timeout;
+	u8 lprx_rssi_threshold;
+	u8 skip_dtim_periods;
+	u8 qndp_tid;
+	u8 uapsd_ac_flags;
+} __packed; /* CLIENT_PM_POWER_TABLE_S_VER_3 */
+
 /*
  * struct iwl_uapsd_misbehaving_ap_notif - FW sends this notification when
  * associated AP is identified as improperly implementing uAPSD protocol.
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
index ac6c1ef2cbcd1..699343cf02799 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/rx.h
@@ -263,6 +263,9 @@ enum iwl_rx_mpdu_reorder_data {
 
 enum iwl_rx_mpdu_phy_info {
 	IWL_RX_MPDU_PHY_EOF_INDICATION	= BIT(0),
+	IWL_RX_MPDU_PHY_UEQM		= BIT(1) | BIT(2),
+	IWL_RX_MPDU_PHY_2X_LDPC		= BIT(3),
+	IWL_RX_MPDU_PHY_NPCA		= BIT(4),
 	IWL_RX_MPDU_PHY_AMPDU		= BIT(5),
 	IWL_RX_MPDU_PHY_AMPDU_TOGGLE	= BIT(6),
 	IWL_RX_MPDU_PHY_SHORT_PREAMBLE	= BIT(7),
@@ -732,6 +735,8 @@ struct iwl_rx_mpdu_desc {
 	     * RX_MPDU_RES_START_API_S_VER_4,
 	     * RX_MPDU_RES_START_API_S_VER_5,
 	     * RX_MPDU_RES_START_API_S_VER_6
+	     * RX_MPDU_RES_START_API_S_VER_7
+	     * RX_MPDU_RES_START_API_S_VER_8
 	     */
 
 #define IWL_RX_DESC_SIZE_V1 offsetofend(struct iwl_rx_mpdu_desc, v1)
@@ -1169,8 +1174,14 @@ struct iwl_he_tb_sigs {
 #define OFDM_UCODE_TRIG_BASE_RX_BANDWIDTH		0x00007000
 #define OFDM_UCODE_TRIG_BASE_PS160			0x00008000
 #define OFDM_UCODE_EHT_TRIG_CONTROL_CHANNEL		0x000f0000
+/* for UHR, since PE RF: */
+#define OFDM_UCODE_TRIG_BASE_RX_MCS_UHR_AND_LATER	0x01f00000
+#define OFDM_UCODE_RX_IS_DRU				0x02000000
+#define OFDM_UCODE_RX_IS_2XLDPC_ENABLED			0x04000000
+#define OFDM_UCODE_RX_DRU_DBW				0x60000000
 	__le32 tb_rx0;
-#define OFDM_UCODE_TRIG_BASE_RX_MCS			0x0000000f
+/* prior to PE RF, cannot have 5 bits needed for UHR: */
+#define OFDM_UCODE_TRIG_BASE_RX_MCS_PRE_UHR		0x0000000f
 #define OFDM_UCODE_TRIG_BASE_RX_DCM			0x00000010
 #define OFDM_UCODE_TRIG_BASE_RX_GI_LTF_TYPE		0x00000060
 #define OFDM_UCODE_TRIG_BASE_RX_NSTS			0x00000380
@@ -1275,15 +1286,16 @@ struct iwl_uhr_sigs {
 	__le32 usig_a1;
 #define OFDM_RX_FRAME_UHR_BSS_COLOR2			0x0000003f
 	__le32 usig_a1_uhr;
-#define	OFDM_RX_FRAME_UHR_PPDU_TYPE			0x00000003
-#define	OFDM_RX_FRAME_UHR_COBF_CSR_DISABLE		0x00000004
-#define	OFDM_RX_FRAME_UHR_PUNC_CHANNEL			0x000000f8
-#define	OFDM_RX_FRAME_UHR_USIG2_VALIDATE_B8		0x00000100
-#define	OFDM_RX_FRAME_UHR_SIG_MCS			0x00000600
-#define	OFDM_RX_FRAME_UHR_SIG_SYM_NUM			0x0000f800
-#define	OFDM_RX_FRAME_UHR_TRIG_SPATIAL_REUSE_1		0x000f0000
-#define	OFDM_RX_FRAME_UHR_TRIG_SPATIAL_REUSE_2		0x00f00000
-#define	OFDM_RX_FRAME_UHR_TRIG_USIG2_DISREGARD		0x1f000000
+#define OFDM_RX_FRAME_UHR_PPDU_TYPE			0x00000003
+#define OFDM_RX_FRAME_UHR_COBF_CSR_DISABLE		0x00000004
+#define OFDM_RX_FRAME_UHR_PUNC_CHANNEL			0x000000f8
+#define OFDM_RX_FRAME_UHR_USIG2_VALIDATE_B8		0x00000100
+#define OFDM_RX_FRAME_UHR_SIG_MCS			0x00000600
+#define OFDM_RX_FRAME_UHR_SIG_SYM_NUM			0x0000f800
+#define OFDM_RX_FRAME_UHR_TRIG_SPATIAL_REUSE_1		0x000f0000
+#define OFDM_RX_FRAME_UHR_TRIG_SPATIAL_REUSE_2		0x00f00000
+#define OFDM_RX_FRAME_UHR_TRIG_USIG2_DISREGARD		0x1f000000
+#define OFDM_RX_FRAME_UHR_USIG_CRC_OK			0x40000000
 	__le32 usig_a2_uhr;
 #define OFDM_RX_FRAME_UHR_SPATIAL_REUSE			0x0000000f
 #define OFDM_RX_FRAME_UHR_GI_LTF_TYPE			0x00000030
@@ -1305,6 +1317,7 @@ struct iwl_uhr_sigs {
 #define OFDM_RX_FRAME_UHR_CODING			0x00000200
 #define OFDM_RX_FRAME_UHR_SPATIAL_CONFIG		0x00003c00
 #define OFDM_RX_FRAME_UHR_STA_RU			0x003fc000
+#define OFDM_RX_FRAME_UHR_STA_RU_P80			0x00004000
 #define OFDM_RX_FRAME_UHR_STA_RU_PS160			0x00400000
 #define OFDM_RX_FRAME_UHR_UEQM				0x00800000
 #define OFDM_RX_FRAME_UHR_2XLDPC			0x01000000
@@ -1330,7 +1343,12 @@ struct iwl_uhr_tb_sigs {
 
 struct iwl_uhr_elr_sigs {
 	/* same as UHR above */
-	__le32 usig_a1, usig_a2_uhr;
+	__le32 usig_a1;
+#define OFDM_RX_VECTOR_UHR_ELR_PPDU_TYPE		0x00000003
+#define OFDM_RX_VECTOR_UHR_ELR_USIG_STA_ID		0x00001ffc
+#define OFDM_RX_VECTOR_UHR_ELR_VALIDATE			0x0000e000
+#define OFDM_RX_VECTOR_UHR_ELR_CRC_OK			0x00010000
+	__le32 usig_a2_uhr_elr;
 #define OFDM_RX_VECTOR_UHR_ELR_VER_ID			0x00000007
 #define OFDM_RX_VECTOR_UHR_ELR_UPLINK_FLAG		0x00000008
 #define OFDM_RX_VECTOR_UHR_ELR_MCS			0x00000010
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h b/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h
index e6f9abdfa546d..3e62a458b131b 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/sta.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2021, 2023, 2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2021, 2023, 2025-2026 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -191,7 +191,6 @@ enum iwl_sta_sleep_flag {
 #define STA_KEY_IDX_INVALID (0xff)
 #define STA_KEY_MAX_DATA_KEY_NUM (4)
 #define IWL_MAX_GLOBAL_KEYS (4)
-#define IWL_MAX_NUM_IGTKS 2
 #define STA_KEY_LEN_WEP40 (5)
 #define STA_KEY_LEN_WEP104 (13)
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h b/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h
index 68983f6a00266..8d16782a129ce 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/api/stats.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018, 2020-2021, 2023-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018, 2020-2021, 2023-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -556,6 +556,65 @@ struct iwl_stats_ntfy_per_mac {
 } __packed; /* STATISTICS_NTFY_PER_MAC_API_S_VER_1 */
 
 #define IWL_STATS_MAX_BW_INDEX 5
+/**
+ * struct iwl_stats_ntfy_per_phy_v1 - per PHY statistics
+ * @channel_load: channel load
+ * @channel_load_by_us: device contribution to MCLM
+ * @channel_load_not_by_us: other devices' contribution to MCLM
+ * @clt: CLT HW timer (TIM_CH_LOAD2)
+ * @act: active accumulator SW
+ * @elp: elapsed time accumulator SW
+ * @rx_detected_per_ch_width: number of deferred TX per channel width,
+ *	0 - 20, 1/2/3 - 40/80/160
+ * @success_per_ch_width: number of frames that got ACK/BACK/CTS
+ *	per channel BW. note, BACK counted as 1
+ * @fail_per_ch_width: number of frames that didn't get ACK/BACK/CTS
+ *	per channel BW. note BACK counted as 1
+ * @last_tx_ch_width_indx: last txed frame channel width index
+ */
+struct iwl_stats_ntfy_per_phy_v1 {
+	__le32 channel_load;
+	__le32 channel_load_by_us;
+	__le32 channel_load_not_by_us;
+	__le32 clt;
+	__le32 act;
+	__le32 elp;
+	__le32 rx_detected_per_ch_width[IWL_STATS_MAX_BW_INDEX];
+	__le32 success_per_ch_width[IWL_STATS_MAX_BW_INDEX];
+	__le32 fail_per_ch_width[IWL_STATS_MAX_BW_INDEX];
+	__le32 last_tx_ch_width_indx;
+} __packed; /* STATISTICS_NTFY_PER_PHY_API_S_VER_1 */
+
+/**
+ * struct iwl_stats_ntfy_coex - coex statistics
+ *
+ * @wifi_kill_cnt: count of wifi frames killed by BT
+ * @wifi_tx_cts_kill_cnt: count of wifi Tx CTS frames killed by BT
+ * @ttc2_ppdu_error_count: Count PPDU errors on TTC2 - BT Tx indication rises
+ *	within wifi Tx packet on non-shared antenna and wifi is NOT killed by
+ *	PTA/TCL.
+ * @trc2_ppdu_error_count: count PPDU errors on TRC2 - BT Rx indication rises
+ *	within wifi Tx packet on non-shared antenna and wifi is  NOT killed by
+ *	PTA/TCL
+ * @rrc1_collision_count: count RRC1 - BT Rx indication rises within wifi Rx
+ *	packet on shared antenna
+ * @rrc2_collision_count: count RRC2 - BT Rx indication rises within wifi Rx
+ *	packet on non-shared antenna
+ * @rtc2_collision_count: count RTC2 - BT Tx indication rises within wifi Rx
+ *	packet on non-shared antenna
+ * @reserved: reserved
+ */
+struct iwl_stats_ntfy_coex {
+	__le16 wifi_kill_cnt;
+	__le16 wifi_tx_cts_kill_cnt;
+	__le16 ttc2_ppdu_error_count;
+	__le16 trc2_ppdu_error_count;
+	__le16 rrc1_collision_count;
+	__le16 rrc2_collision_count;
+	__le16 rtc2_collision_count;
+	__le16 reserved;
+} __packed; /* STATISTICS_FW_NTFY_COEX_TELEMETRY_API_S_VER_1 */
+
 /**
  * struct iwl_stats_ntfy_per_phy - per PHY statistics
  * @channel_load: channel load
@@ -571,6 +630,7 @@ struct iwl_stats_ntfy_per_mac {
  * @fail_per_ch_width: number of frames that didn't get ACK/BACK/CTS
  *	per channel BW. note BACK counted as 1
  * @last_tx_ch_width_indx: last txed frame channel width index
+ * @coex: coex related data
  */
 struct iwl_stats_ntfy_per_phy {
 	__le32 channel_load;
@@ -583,7 +643,8 @@ struct iwl_stats_ntfy_per_phy {
 	__le32 success_per_ch_width[IWL_STATS_MAX_BW_INDEX];
 	__le32 fail_per_ch_width[IWL_STATS_MAX_BW_INDEX];
 	__le32 last_tx_ch_width_indx;
-} __packed; /* STATISTICS_NTFY_PER_PHY_API_S_VER_1 */
+	struct iwl_stats_ntfy_coex coex;
+} __packed; /* STATISTICS_NTFY_PER_PHY_API_S_VER_2 */
 
 /* unknown channel load (due to not being active on channel) */
 #define IWL_STATS_UNKNOWN_CHANNEL_LOAD	0xffffffff
@@ -599,12 +660,27 @@ struct iwl_stats_ntfy_per_sta {
 
 #define IWL_STATS_MAX_PHY_OPERATIONAL 3
 
+/**
+ * struct iwl_system_statistics_notif_oper_v3 - statistics notification
+ *
+ * @time_stamp: time when the notification is sent from firmware
+ * @per_link: per link statistics, &struct iwl_stats_ntfy_per_link
+ * @per_phy: per phy statistics, &struct iwl_stats_ntfy_per_phy_v1
+ * @per_sta: per sta statistics, &struct iwl_stats_ntfy_per_sta
+ */
+struct iwl_system_statistics_notif_oper_v3 {
+	__le32 time_stamp;
+	struct iwl_stats_ntfy_per_link per_link[IWL_FW_MAX_LINKS];
+	struct iwl_stats_ntfy_per_phy_v1 per_phy[IWL_STATS_MAX_PHY_OPERATIONAL];
+	struct iwl_stats_ntfy_per_sta per_sta[IWL_STATION_COUNT_MAX];
+} __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_API_S_VER_3 */
+
 /**
  * struct iwl_system_statistics_notif_oper - statistics notification
  *
  * @time_stamp: time when the notification is sent from firmware
  * @per_link: per link statistics, &struct iwl_stats_ntfy_per_link
- * @per_phy: per phy statistics, &struct iwl_stats_ntfy_per_phy
+ * @per_phy: per phy statistics, &struct iwl_stats_ntfy_per_phy_v1
  * @per_sta: per sta statistics, &struct iwl_stats_ntfy_per_sta
  */
 struct iwl_system_statistics_notif_oper {
@@ -612,7 +688,7 @@ struct iwl_system_statistics_notif_oper {
 	struct iwl_stats_ntfy_per_link per_link[IWL_FW_MAX_LINKS];
 	struct iwl_stats_ntfy_per_phy per_phy[IWL_STATS_MAX_PHY_OPERATIONAL];
 	struct iwl_stats_ntfy_per_sta per_sta[IWL_STATION_COUNT_MAX];
-} __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_API_S_VER_3 */
+} __packed; /* STATISTICS_FW_NTFY_OPERATIONAL_API_S_VER_4 */
 
 /**
  * struct iwl_system_statistics_part1_notif_oper - part1 stats notification
@@ -642,7 +718,7 @@ struct iwl_system_statistics_end_notif {
  * @hdr: general statistics header
  * @flags: bitmap of possible notification structures
  * @per_mac: per mac statistics, &struct iwl_stats_ntfy_per_mac
- * @per_phy: per phy statistics, &struct iwl_stats_ntfy_per_phy
+ * @per_phy: per phy statistics, &struct iwl_stats_ntfy_per_phy_v1
  * @per_sta: per sta statistics, &struct iwl_stats_ntfy_per_sta
  * @rx_time: rx time
  * @tx_time: usec the radio is transmitting.
@@ -653,7 +729,7 @@ struct iwl_statistics_operational_ntfy {
 	struct iwl_statistics_ntfy_hdr hdr;
 	__le32 flags;
 	struct iwl_stats_ntfy_per_mac per_mac[MAC_INDEX_AUX];
-	struct iwl_stats_ntfy_per_phy per_phy[IWL_STATS_MAX_PHY_OPERATIONAL];
+	struct iwl_stats_ntfy_per_phy_v1 per_phy[IWL_STATS_MAX_PHY_OPERATIONAL];
 	struct iwl_stats_ntfy_per_sta per_sta[IWL_STATION_COUNT_MAX];
 	__le64 rx_time;
 	__le64 tx_time;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg-old.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg-old.c
new file mode 100644
index 0000000000000..19b1bfd0abee4
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg-old.c
@@ -0,0 +1,1022 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * Copyright (C) 2005-2014, 2018-2026 Intel Corporation
+ * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
+ * Copyright (C) 2015-2017 Intel Deutschland GmbH
+ */
+#include <linux/devcoredump.h>
+#include "iwl-drv.h"
+#include "runtime.h"
+#include "dbg.h"
+#include "debugfs.h"
+#include "iwl-io.h"
+#include "iwl-prph.h"
+#include "iwl-csr.h"
+#include "iwl-fh.h"
+
+/**
+ * struct iwl_fw_dump_ptrs - set of pointers needed for the fw-error-dump
+ *
+ * @fwrt_ptr: pointer to the buffer coming from fwrt
+ * @trans_ptr: pointer to struct %iwl_trans_dump_data which contains the
+ *	transport's data.
+ * @fwrt_len: length of the valid data in fwrt_ptr
+ */
+struct iwl_fw_dump_ptrs {
+	struct iwl_trans_dump_data *trans_ptr;
+	void *fwrt_ptr;
+	u32 fwrt_len;
+};
+
+#define RADIO_REG_MAX_READ 0x2ad
+static void iwl_read_radio_regs(struct iwl_fw_runtime *fwrt,
+				struct iwl_fw_error_dump_data **dump_data)
+{
+	u8 *pos = (void *)(*dump_data)->data;
+	int i;
+
+	IWL_DEBUG_INFO(fwrt, "WRT radio registers dump\n");
+
+	if (!iwl_trans_grab_nic_access(fwrt->trans))
+		return;
+
+	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RADIO_REG);
+	(*dump_data)->len = cpu_to_le32(RADIO_REG_MAX_READ);
+
+	for (i = 0; i < RADIO_REG_MAX_READ; i++) {
+		u32 rd_cmd = RADIO_RSP_RD_CMD;
+
+		rd_cmd |= i << RADIO_RSP_ADDR_POS;
+		iwl_trans_write_prph(fwrt->trans, RSP_RADIO_CMD, rd_cmd);
+		*pos = (u8)iwl_trans_read_prph(fwrt->trans, RSP_RADIO_RDDAT);
+
+		pos++;
+	}
+
+	*dump_data = iwl_fw_error_next_data(*dump_data);
+
+	iwl_trans_release_nic_access(fwrt->trans);
+}
+
+static void iwl_fwrt_dump_rxf(struct iwl_fw_runtime *fwrt,
+			      struct iwl_fw_error_dump_data **dump_data,
+			      int size, u32 offset, int fifo_num)
+{
+	struct iwl_fw_error_dump_fifo *fifo_hdr;
+	u32 *fifo_data;
+	u32 fifo_len;
+	int i;
+
+	fifo_hdr = (void *)(*dump_data)->data;
+	fifo_data = (void *)fifo_hdr->data;
+	fifo_len = size;
+
+	/* No need to try to read the data if the length is 0 */
+	if (fifo_len == 0)
+		return;
+
+	/* Add a TLV for the RXF */
+	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RXF);
+	(*dump_data)->len = cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
+
+	fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
+	fifo_hdr->available_bytes =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						RXF_RD_D_SPACE + offset));
+	fifo_hdr->wr_ptr =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						RXF_RD_WR_PTR + offset));
+	fifo_hdr->rd_ptr =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						RXF_RD_RD_PTR + offset));
+	fifo_hdr->fence_ptr =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						RXF_RD_FENCE_PTR + offset));
+	fifo_hdr->fence_mode =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						RXF_SET_FENCE_MODE + offset));
+
+	/* Lock fence */
+	iwl_trans_write_prph(fwrt->trans, RXF_SET_FENCE_MODE + offset, 0x1);
+	/* Set fence pointer to the same place like WR pointer */
+	iwl_trans_write_prph(fwrt->trans, RXF_LD_WR2FENCE + offset, 0x1);
+	/* Set fence offset */
+	iwl_trans_write_prph(fwrt->trans,
+			     RXF_LD_FENCE_OFFSET_ADDR + offset, 0x0);
+
+	/* Read FIFO */
+	fifo_len /= sizeof(u32); /* Size in DWORDS */
+	for (i = 0; i < fifo_len; i++)
+		fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
+						 RXF_FIFO_RD_FENCE_INC +
+						 offset);
+	*dump_data = iwl_fw_error_next_data(*dump_data);
+}
+
+static void iwl_fwrt_dump_txf(struct iwl_fw_runtime *fwrt,
+			      struct iwl_fw_error_dump_data **dump_data,
+			      int size, u32 offset, int fifo_num)
+{
+	struct iwl_fw_error_dump_fifo *fifo_hdr;
+	u32 *fifo_data;
+	u32 fifo_len;
+	int i;
+
+	fifo_hdr = (void *)(*dump_data)->data;
+	fifo_data = (void *)fifo_hdr->data;
+	fifo_len = size;
+
+	/* No need to try to read the data if the length is 0 */
+	if (fifo_len == 0)
+		return;
+
+	/* Add a TLV for the FIFO */
+	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXF);
+	(*dump_data)->len = cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
+
+	fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
+	fifo_hdr->available_bytes =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						TXF_FIFO_ITEM_CNT + offset));
+	fifo_hdr->wr_ptr =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						TXF_WR_PTR + offset));
+	fifo_hdr->rd_ptr =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						TXF_RD_PTR + offset));
+	fifo_hdr->fence_ptr =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						TXF_FENCE_PTR + offset));
+	fifo_hdr->fence_mode =
+		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+						TXF_LOCK_FENCE + offset));
+
+	/* Set the TXF_READ_MODIFY_ADDR to TXF_WR_PTR */
+	iwl_trans_write_prph(fwrt->trans, TXF_READ_MODIFY_ADDR + offset,
+			     TXF_WR_PTR + offset);
+
+	/* Dummy-read to advance the read pointer to the head */
+	iwl_trans_read_prph(fwrt->trans, TXF_READ_MODIFY_DATA + offset);
+
+	/* Read FIFO */
+	for (i = 0; i < fifo_len / sizeof(u32); i++)
+		fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
+						  TXF_READ_MODIFY_DATA +
+						  offset);
+
+	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_txf)
+		fwrt->sanitize_ops->frob_txf(fwrt->sanitize_ctx,
+					     fifo_data, fifo_len);
+
+	*dump_data = iwl_fw_error_next_data(*dump_data);
+}
+
+static void iwl_fw_dump_rxf(struct iwl_fw_runtime *fwrt,
+			    struct iwl_fw_error_dump_data **dump_data)
+{
+	struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
+
+	IWL_DEBUG_INFO(fwrt, "WRT RX FIFO dump\n");
+
+	if (!iwl_trans_grab_nic_access(fwrt->trans))
+		return;
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF)) {
+		/* Pull RXF1 */
+		iwl_fwrt_dump_rxf(fwrt, dump_data,
+				  cfg->lmac[0].rxfifo1_size, 0, 0);
+		/* Pull RXF2 */
+		iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->rxfifo2_size,
+				  RXF_DIFF_FROM_PREV +
+				  fwrt->trans->mac_cfg->umac_prph_offset, 1);
+		/* Pull LMAC2 RXF1 */
+		if (fwrt->smem_cfg.num_lmacs > 1)
+			iwl_fwrt_dump_rxf(fwrt, dump_data,
+					  cfg->lmac[1].rxfifo1_size,
+					  LMAC2_PRPH_OFFSET, 2);
+	}
+
+	iwl_trans_release_nic_access(fwrt->trans);
+}
+
+static void iwl_fw_dump_txf(struct iwl_fw_runtime *fwrt,
+			    struct iwl_fw_error_dump_data **dump_data)
+{
+	struct iwl_fw_error_dump_fifo *fifo_hdr;
+	struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
+	u32 *fifo_data;
+	u32 fifo_len;
+	int i, j;
+
+	IWL_DEBUG_INFO(fwrt, "WRT TX FIFO dump\n");
+
+	if (!iwl_trans_grab_nic_access(fwrt->trans))
+		return;
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF)) {
+		/* Pull TXF data from LMAC1 */
+		for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries; i++) {
+			/* Mark the number of TXF we're pulling now */
+			iwl_trans_write_prph(fwrt->trans, TXF_LARC_NUM, i);
+			iwl_fwrt_dump_txf(fwrt, dump_data,
+					  cfg->lmac[0].txfifo_size[i], 0, i);
+		}
+
+		/* Pull TXF data from LMAC2 */
+		if (fwrt->smem_cfg.num_lmacs > 1) {
+			for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries;
+			     i++) {
+				/* Mark the number of TXF we're pulling now */
+				iwl_trans_write_prph(fwrt->trans,
+						     TXF_LARC_NUM +
+						     LMAC2_PRPH_OFFSET, i);
+				iwl_fwrt_dump_txf(fwrt, dump_data,
+						  cfg->lmac[1].txfifo_size[i],
+						  LMAC2_PRPH_OFFSET,
+						  i + cfg->num_txfifo_entries);
+			}
+		}
+	}
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
+	    fw_has_capa(&fwrt->fw->ucode_capa,
+			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
+		/* Pull UMAC internal TXF data from all TXFs */
+		for (i = 0;
+		     i < ARRAY_SIZE(fwrt->smem_cfg.internal_txfifo_size);
+		     i++) {
+			fifo_hdr = (void *)(*dump_data)->data;
+			fifo_data = (void *)fifo_hdr->data;
+			fifo_len = fwrt->smem_cfg.internal_txfifo_size[i];
+
+			/* No need to try to read the data if the length is 0 */
+			if (fifo_len == 0)
+				continue;
+
+			/* Add a TLV for the internal FIFOs */
+			(*dump_data)->type =
+				cpu_to_le32(IWL_FW_ERROR_DUMP_INTERNAL_TXF);
+			(*dump_data)->len =
+				cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
+
+			fifo_hdr->fifo_num = cpu_to_le32(i);
+
+			/* Mark the number of TXF we're pulling now */
+			iwl_trans_write_prph(fwrt->trans, TXF_CPU2_NUM, i +
+				fwrt->smem_cfg.num_txfifo_entries);
+
+			fifo_hdr->available_bytes =
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+								TXF_CPU2_FIFO_ITEM_CNT));
+			fifo_hdr->wr_ptr =
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+								TXF_CPU2_WR_PTR));
+			fifo_hdr->rd_ptr =
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+								TXF_CPU2_RD_PTR));
+			fifo_hdr->fence_ptr =
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+								TXF_CPU2_FENCE_PTR));
+			fifo_hdr->fence_mode =
+				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+								TXF_CPU2_LOCK_FENCE));
+
+			/* Set TXF_CPU2_READ_MODIFY_ADDR to TXF_CPU2_WR_PTR */
+			iwl_trans_write_prph(fwrt->trans,
+					     TXF_CPU2_READ_MODIFY_ADDR,
+					     TXF_CPU2_WR_PTR);
+
+			/* Dummy-read to advance the read pointer to head */
+			iwl_trans_read_prph(fwrt->trans,
+					    TXF_CPU2_READ_MODIFY_DATA);
+
+			/* Read FIFO */
+			fifo_len /= sizeof(u32); /* Size in DWORDS */
+			for (j = 0; j < fifo_len; j++)
+				fifo_data[j] =
+					iwl_trans_read_prph(fwrt->trans,
+							    TXF_CPU2_READ_MODIFY_DATA);
+			*dump_data = iwl_fw_error_next_data(*dump_data);
+		}
+	}
+
+	iwl_trans_release_nic_access(fwrt->trans);
+}
+
+struct iwl_prph_range {
+	u32 start, end;
+};
+
+static const struct iwl_prph_range iwl_prph_dump_addr_comm[] = {
+	{ .start = 0x00a00000, .end = 0x00a00000 },
+	{ .start = 0x00a0000c, .end = 0x00a00024 },
+	{ .start = 0x00a0002c, .end = 0x00a0003c },
+	{ .start = 0x00a00410, .end = 0x00a00418 },
+	{ .start = 0x00a00420, .end = 0x00a00420 },
+	{ .start = 0x00a00428, .end = 0x00a00428 },
+	{ .start = 0x00a00430, .end = 0x00a0043c },
+	{ .start = 0x00a00444, .end = 0x00a00444 },
+	{ .start = 0x00a004c0, .end = 0x00a004cc },
+	{ .start = 0x00a004d8, .end = 0x00a004d8 },
+	{ .start = 0x00a004e0, .end = 0x00a004f0 },
+	{ .start = 0x00a00840, .end = 0x00a00840 },
+	{ .start = 0x00a00850, .end = 0x00a00858 },
+	{ .start = 0x00a01004, .end = 0x00a01008 },
+	{ .start = 0x00a01010, .end = 0x00a01010 },
+	{ .start = 0x00a01018, .end = 0x00a01018 },
+	{ .start = 0x00a01024, .end = 0x00a01024 },
+	{ .start = 0x00a0102c, .end = 0x00a01034 },
+	{ .start = 0x00a0103c, .end = 0x00a01040 },
+	{ .start = 0x00a01048, .end = 0x00a01094 },
+	{ .start = 0x00a01c00, .end = 0x00a01c20 },
+	{ .start = 0x00a01c58, .end = 0x00a01c58 },
+	{ .start = 0x00a01c7c, .end = 0x00a01c7c },
+	{ .start = 0x00a01c28, .end = 0x00a01c54 },
+	{ .start = 0x00a01c5c, .end = 0x00a01c5c },
+	{ .start = 0x00a01c60, .end = 0x00a01cdc },
+	{ .start = 0x00a01ce0, .end = 0x00a01d0c },
+	{ .start = 0x00a01d18, .end = 0x00a01d20 },
+	{ .start = 0x00a01d2c, .end = 0x00a01d30 },
+	{ .start = 0x00a01d40, .end = 0x00a01d5c },
+	{ .start = 0x00a01d80, .end = 0x00a01d80 },
+	{ .start = 0x00a01d98, .end = 0x00a01d9c },
+	{ .start = 0x00a01da8, .end = 0x00a01da8 },
+	{ .start = 0x00a01db8, .end = 0x00a01df4 },
+	{ .start = 0x00a01dc0, .end = 0x00a01dfc },
+	{ .start = 0x00a01e00, .end = 0x00a01e2c },
+	{ .start = 0x00a01e40, .end = 0x00a01e60 },
+	{ .start = 0x00a01e68, .end = 0x00a01e6c },
+	{ .start = 0x00a01e74, .end = 0x00a01e74 },
+	{ .start = 0x00a01e84, .end = 0x00a01e90 },
+	{ .start = 0x00a01e9c, .end = 0x00a01ec4 },
+	{ .start = 0x00a01ed0, .end = 0x00a01ee0 },
+	{ .start = 0x00a01f00, .end = 0x00a01f1c },
+	{ .start = 0x00a01f44, .end = 0x00a01ffc },
+	{ .start = 0x00a02000, .end = 0x00a02048 },
+	{ .start = 0x00a02068, .end = 0x00a020f0 },
+	{ .start = 0x00a02100, .end = 0x00a02118 },
+	{ .start = 0x00a02140, .end = 0x00a0214c },
+	{ .start = 0x00a02168, .end = 0x00a0218c },
+	{ .start = 0x00a021c0, .end = 0x00a021c0 },
+	{ .start = 0x00a02400, .end = 0x00a02410 },
+	{ .start = 0x00a02418, .end = 0x00a02420 },
+	{ .start = 0x00a02428, .end = 0x00a0242c },
+	{ .start = 0x00a02434, .end = 0x00a02434 },
+	{ .start = 0x00a02440, .end = 0x00a02460 },
+	{ .start = 0x00a02468, .end = 0x00a024b0 },
+	{ .start = 0x00a024c8, .end = 0x00a024cc },
+	{ .start = 0x00a02500, .end = 0x00a02504 },
+	{ .start = 0x00a0250c, .end = 0x00a02510 },
+	{ .start = 0x00a02540, .end = 0x00a02554 },
+	{ .start = 0x00a02580, .end = 0x00a025f4 },
+	{ .start = 0x00a02600, .end = 0x00a0260c },
+	{ .start = 0x00a02648, .end = 0x00a02650 },
+	{ .start = 0x00a02680, .end = 0x00a02680 },
+	{ .start = 0x00a026c0, .end = 0x00a026d0 },
+	{ .start = 0x00a02700, .end = 0x00a0270c },
+	{ .start = 0x00a02804, .end = 0x00a02804 },
+	{ .start = 0x00a02818, .end = 0x00a0281c },
+	{ .start = 0x00a02c00, .end = 0x00a02db4 },
+	{ .start = 0x00a02df4, .end = 0x00a02fb0 },
+	{ .start = 0x00a03000, .end = 0x00a03014 },
+	{ .start = 0x00a0301c, .end = 0x00a0302c },
+	{ .start = 0x00a03034, .end = 0x00a03038 },
+	{ .start = 0x00a03040, .end = 0x00a03048 },
+	{ .start = 0x00a03060, .end = 0x00a03068 },
+	{ .start = 0x00a03070, .end = 0x00a03074 },
+	{ .start = 0x00a0307c, .end = 0x00a0307c },
+	{ .start = 0x00a03080, .end = 0x00a03084 },
+	{ .start = 0x00a0308c, .end = 0x00a03090 },
+	{ .start = 0x00a03098, .end = 0x00a03098 },
+	{ .start = 0x00a030a0, .end = 0x00a030a0 },
+	{ .start = 0x00a030a8, .end = 0x00a030b4 },
+	{ .start = 0x00a030bc, .end = 0x00a030bc },
+	{ .start = 0x00a030c0, .end = 0x00a0312c },
+	{ .start = 0x00a03c00, .end = 0x00a03c5c },
+	{ .start = 0x00a04400, .end = 0x00a04454 },
+	{ .start = 0x00a04460, .end = 0x00a04474 },
+	{ .start = 0x00a044c0, .end = 0x00a044ec },
+	{ .start = 0x00a04500, .end = 0x00a04504 },
+	{ .start = 0x00a04510, .end = 0x00a04538 },
+	{ .start = 0x00a04540, .end = 0x00a04548 },
+	{ .start = 0x00a04560, .end = 0x00a0457c },
+	{ .start = 0x00a04590, .end = 0x00a04598 },
+	{ .start = 0x00a045c0, .end = 0x00a045f4 },
+};
+
+static const struct iwl_prph_range iwl_prph_dump_addr_9000[] = {
+	{ .start = 0x00a05c00, .end = 0x00a05c18 },
+	{ .start = 0x00a05400, .end = 0x00a056e8 },
+	{ .start = 0x00a08000, .end = 0x00a098bc },
+	{ .start = 0x00a02400, .end = 0x00a02758 },
+	{ .start = 0x00a04764, .end = 0x00a0476c },
+	{ .start = 0x00a04770, .end = 0x00a04774 },
+	{ .start = 0x00a04620, .end = 0x00a04624 },
+};
+
+static const struct iwl_prph_range iwl_prph_dump_addr_22000[] = {
+	{ .start = 0x00a00000, .end = 0x00a00000 },
+	{ .start = 0x00a0000c, .end = 0x00a00024 },
+	{ .start = 0x00a0002c, .end = 0x00a00034 },
+	{ .start = 0x00a0003c, .end = 0x00a0003c },
+	{ .start = 0x00a00410, .end = 0x00a00418 },
+	{ .start = 0x00a00420, .end = 0x00a00420 },
+	{ .start = 0x00a00428, .end = 0x00a00428 },
+	{ .start = 0x00a00430, .end = 0x00a0043c },
+	{ .start = 0x00a00444, .end = 0x00a00444 },
+	{ .start = 0x00a00840, .end = 0x00a00840 },
+	{ .start = 0x00a00850, .end = 0x00a00858 },
+	{ .start = 0x00a01004, .end = 0x00a01008 },
+	{ .start = 0x00a01010, .end = 0x00a01010 },
+	{ .start = 0x00a01018, .end = 0x00a01018 },
+	{ .start = 0x00a01024, .end = 0x00a01024 },
+	{ .start = 0x00a0102c, .end = 0x00a01034 },
+	{ .start = 0x00a0103c, .end = 0x00a01040 },
+	{ .start = 0x00a01048, .end = 0x00a01050 },
+	{ .start = 0x00a01058, .end = 0x00a01058 },
+	{ .start = 0x00a01060, .end = 0x00a01070 },
+	{ .start = 0x00a0108c, .end = 0x00a0108c },
+	{ .start = 0x00a01c20, .end = 0x00a01c28 },
+	{ .start = 0x00a01d10, .end = 0x00a01d10 },
+	{ .start = 0x00a01e28, .end = 0x00a01e2c },
+	{ .start = 0x00a01e60, .end = 0x00a01e60 },
+	{ .start = 0x00a01e80, .end = 0x00a01e80 },
+	{ .start = 0x00a01ea0, .end = 0x00a01ea0 },
+	{ .start = 0x00a02000, .end = 0x00a0201c },
+	{ .start = 0x00a02024, .end = 0x00a02024 },
+	{ .start = 0x00a02040, .end = 0x00a02048 },
+	{ .start = 0x00a020c0, .end = 0x00a020e0 },
+	{ .start = 0x00a02400, .end = 0x00a02404 },
+	{ .start = 0x00a0240c, .end = 0x00a02414 },
+	{ .start = 0x00a0241c, .end = 0x00a0243c },
+	{ .start = 0x00a02448, .end = 0x00a024bc },
+	{ .start = 0x00a024c4, .end = 0x00a024cc },
+	{ .start = 0x00a02508, .end = 0x00a02508 },
+	{ .start = 0x00a02510, .end = 0x00a02514 },
+	{ .start = 0x00a0251c, .end = 0x00a0251c },
+	{ .start = 0x00a0252c, .end = 0x00a0255c },
+	{ .start = 0x00a02564, .end = 0x00a025a0 },
+	{ .start = 0x00a025a8, .end = 0x00a025b4 },
+	{ .start = 0x00a025c0, .end = 0x00a025c0 },
+	{ .start = 0x00a025e8, .end = 0x00a025f4 },
+	{ .start = 0x00a02c08, .end = 0x00a02c18 },
+	{ .start = 0x00a02c2c, .end = 0x00a02c38 },
+	{ .start = 0x00a02c68, .end = 0x00a02c78 },
+	{ .start = 0x00a03000, .end = 0x00a03000 },
+	{ .start = 0x00a03010, .end = 0x00a03014 },
+	{ .start = 0x00a0301c, .end = 0x00a0302c },
+	{ .start = 0x00a03034, .end = 0x00a03038 },
+	{ .start = 0x00a03040, .end = 0x00a03044 },
+	{ .start = 0x00a03060, .end = 0x00a03068 },
+	{ .start = 0x00a03070, .end = 0x00a03070 },
+	{ .start = 0x00a0307c, .end = 0x00a03084 },
+	{ .start = 0x00a0308c, .end = 0x00a03090 },
+	{ .start = 0x00a03098, .end = 0x00a03098 },
+	{ .start = 0x00a030a0, .end = 0x00a030a0 },
+	{ .start = 0x00a030a8, .end = 0x00a030b4 },
+	{ .start = 0x00a030bc, .end = 0x00a030c0 },
+	{ .start = 0x00a030c8, .end = 0x00a030f4 },
+	{ .start = 0x00a03100, .end = 0x00a0312c },
+	{ .start = 0x00a03c00, .end = 0x00a03c5c },
+	{ .start = 0x00a04400, .end = 0x00a04454 },
+	{ .start = 0x00a04460, .end = 0x00a04474 },
+	{ .start = 0x00a044c0, .end = 0x00a044ec },
+	{ .start = 0x00a04500, .end = 0x00a04504 },
+	{ .start = 0x00a04510, .end = 0x00a04538 },
+	{ .start = 0x00a04540, .end = 0x00a04548 },
+	{ .start = 0x00a04560, .end = 0x00a04560 },
+	{ .start = 0x00a04570, .end = 0x00a0457c },
+	{ .start = 0x00a04590, .end = 0x00a04590 },
+	{ .start = 0x00a04598, .end = 0x00a04598 },
+	{ .start = 0x00a045c0, .end = 0x00a045f4 },
+	{ .start = 0x00a05c18, .end = 0x00a05c1c },
+	{ .start = 0x00a0c000, .end = 0x00a0c018 },
+	{ .start = 0x00a0c020, .end = 0x00a0c028 },
+	{ .start = 0x00a0c038, .end = 0x00a0c094 },
+	{ .start = 0x00a0c0c0, .end = 0x00a0c104 },
+	{ .start = 0x00a0c10c, .end = 0x00a0c118 },
+	{ .start = 0x00a0c150, .end = 0x00a0c174 },
+	{ .start = 0x00a0c17c, .end = 0x00a0c188 },
+	{ .start = 0x00a0c190, .end = 0x00a0c198 },
+	{ .start = 0x00a0c1a0, .end = 0x00a0c1a8 },
+	{ .start = 0x00a0c1b0, .end = 0x00a0c1b8 },
+};
+
+static const struct iwl_prph_range iwl_prph_dump_addr_ax210[] = {
+	{ .start = 0x00d03c00, .end = 0x00d03c64 },
+	{ .start = 0x00d05c18, .end = 0x00d05c1c },
+	{ .start = 0x00d0c000, .end = 0x00d0c174 },
+};
+
+static void iwl_read_prph_block(struct iwl_trans *trans, u32 start,
+				u32 len_bytes, __le32 *data)
+{
+	u32 i;
+
+	for (i = 0; i < len_bytes; i += 4)
+		*data++ = cpu_to_le32(iwl_trans_read_prph(trans, start + i));
+}
+
+static void iwl_dump_prph(struct iwl_fw_runtime *fwrt,
+			  const struct iwl_prph_range *iwl_prph_dump_addr,
+			  u32 range_len, void *ptr)
+{
+	struct iwl_fw_error_dump_prph *prph;
+	struct iwl_trans *trans = fwrt->trans;
+	struct iwl_fw_error_dump_data **data =
+		(struct iwl_fw_error_dump_data **)ptr;
+	u32 i;
+
+	if (!data)
+		return;
+
+	IWL_DEBUG_INFO(trans, "WRT PRPH dump\n");
+
+	if (!iwl_trans_grab_nic_access(trans))
+		return;
+
+	for (i = 0; i < range_len; i++) {
+		/* The range includes both boundaries */
+		int num_bytes_in_chunk = iwl_prph_dump_addr[i].end -
+			 iwl_prph_dump_addr[i].start + 4;
+
+		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PRPH);
+		(*data)->len = cpu_to_le32(sizeof(*prph) +
+					num_bytes_in_chunk);
+		prph = (void *)(*data)->data;
+		prph->prph_start = cpu_to_le32(iwl_prph_dump_addr[i].start);
+
+		iwl_read_prph_block(trans, iwl_prph_dump_addr[i].start,
+				    /* our range is inclusive, hence + 4 */
+				    iwl_prph_dump_addr[i].end -
+				    iwl_prph_dump_addr[i].start + 4,
+				    (void *)prph->data);
+
+		*data = iwl_fw_error_next_data(*data);
+	}
+
+	iwl_trans_release_nic_access(trans);
+}
+
+static void iwl_fw_get_prph_len(struct iwl_fw_runtime *fwrt,
+				const struct iwl_prph_range *iwl_prph_dump_addr,
+				u32 range_len, void *ptr)
+{
+	u32 *prph_len = (u32 *)ptr;
+	int i, num_bytes_in_chunk;
+
+	if (!prph_len)
+		return;
+
+	for (i = 0; i < range_len; i++) {
+		/* The range includes both boundaries */
+		num_bytes_in_chunk =
+			iwl_prph_dump_addr[i].end -
+			iwl_prph_dump_addr[i].start + 4;
+
+		*prph_len += sizeof(struct iwl_fw_error_dump_data) +
+			sizeof(struct iwl_fw_error_dump_prph) +
+			num_bytes_in_chunk;
+	}
+}
+
+static void iwl_fw_prph_handler(struct iwl_fw_runtime *fwrt, void *ptr,
+				void (*handler)(struct iwl_fw_runtime *,
+						const struct iwl_prph_range *,
+						u32, void *))
+{
+	u32 range_len;
+
+	if (fwrt->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
+		range_len = ARRAY_SIZE(iwl_prph_dump_addr_ax210);
+		handler(fwrt, iwl_prph_dump_addr_ax210, range_len, ptr);
+	} else if (fwrt->trans->mac_cfg->device_family >=
+		   IWL_DEVICE_FAMILY_22000) {
+		range_len = ARRAY_SIZE(iwl_prph_dump_addr_22000);
+		handler(fwrt, iwl_prph_dump_addr_22000, range_len, ptr);
+	} else {
+		range_len = ARRAY_SIZE(iwl_prph_dump_addr_comm);
+		handler(fwrt, iwl_prph_dump_addr_comm, range_len, ptr);
+
+		if (fwrt->trans->mac_cfg->mq_rx_supported) {
+			range_len = ARRAY_SIZE(iwl_prph_dump_addr_9000);
+			handler(fwrt, iwl_prph_dump_addr_9000, range_len, ptr);
+		}
+	}
+}
+
+static void iwl_fw_dump_mem(struct iwl_fw_runtime *fwrt,
+			    struct iwl_fw_error_dump_data **dump_data,
+			    u32 len, u32 ofs, u32 type)
+{
+	struct iwl_fw_error_dump_mem *dump_mem;
+
+	if (!len)
+		return;
+
+	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
+	(*dump_data)->len = cpu_to_le32(len + sizeof(*dump_mem));
+	dump_mem = (void *)(*dump_data)->data;
+	dump_mem->type = cpu_to_le32(type);
+	dump_mem->offset = cpu_to_le32(ofs);
+	iwl_trans_read_mem_bytes(fwrt->trans, ofs, dump_mem->data, len);
+	*dump_data = iwl_fw_error_next_data(*dump_data);
+
+	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
+		fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx, ofs,
+					     dump_mem->data, len);
+
+	IWL_DEBUG_INFO(fwrt, "WRT memory dump. Type=%u\n", dump_mem->type);
+}
+
+#define ADD_LEN(len, item_len, const_len) \
+	do {size_t item = item_len; len += (!!item) * const_len + item; } \
+	while (0)
+
+static int iwl_fw_rxf_len(struct iwl_fw_runtime *fwrt,
+			  struct iwl_fwrt_shared_mem_cfg *mem_cfg)
+{
+	size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
+			 sizeof(struct iwl_fw_error_dump_fifo);
+	u32 fifo_len = 0;
+	int i;
+
+	if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF))
+		return 0;
+
+	/* Count RXF2 size */
+	ADD_LEN(fifo_len, mem_cfg->rxfifo2_size, hdr_len);
+
+	/* Count RXF1 sizes */
+	if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
+		mem_cfg->num_lmacs = MAX_NUM_LMAC;
+
+	for (i = 0; i < mem_cfg->num_lmacs; i++)
+		ADD_LEN(fifo_len, mem_cfg->lmac[i].rxfifo1_size, hdr_len);
+
+	return fifo_len;
+}
+
+static int iwl_fw_txf_len(struct iwl_fw_runtime *fwrt,
+			  struct iwl_fwrt_shared_mem_cfg *mem_cfg)
+{
+	size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
+			 sizeof(struct iwl_fw_error_dump_fifo);
+	u32 fifo_len = 0;
+	int i;
+
+	if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF))
+		goto dump_internal_txf;
+
+	/* Count TXF sizes */
+	if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
+		mem_cfg->num_lmacs = MAX_NUM_LMAC;
+
+	for (i = 0; i < mem_cfg->num_lmacs; i++) {
+		int j;
+
+		for (j = 0; j < mem_cfg->num_txfifo_entries; j++)
+			ADD_LEN(fifo_len, mem_cfg->lmac[i].txfifo_size[j],
+				hdr_len);
+	}
+
+dump_internal_txf:
+	if (!(iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
+	      fw_has_capa(&fwrt->fw->ucode_capa,
+			  IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)))
+		goto out;
+
+	for (i = 0; i < ARRAY_SIZE(mem_cfg->internal_txfifo_size); i++)
+		ADD_LEN(fifo_len, mem_cfg->internal_txfifo_size[i], hdr_len);
+
+out:
+	return fifo_len;
+}
+
+static void iwl_dump_paging(struct iwl_fw_runtime *fwrt,
+			    struct iwl_fw_error_dump_data **data)
+{
+	int i;
+
+	IWL_DEBUG_INFO(fwrt, "WRT paging dump\n");
+	for (i = 1; i < fwrt->num_of_paging_blk + 1; i++) {
+		struct iwl_fw_error_dump_paging *paging;
+		struct page *pages =
+			fwrt->fw_paging_db[i].fw_paging_block;
+		dma_addr_t addr = fwrt->fw_paging_db[i].fw_paging_phys;
+
+		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
+		(*data)->len = cpu_to_le32(sizeof(*paging) +
+					     PAGING_BLOCK_SIZE);
+		paging =  (void *)(*data)->data;
+		paging->index = cpu_to_le32(i);
+		dma_sync_single_for_cpu(fwrt->trans->dev, addr,
+					PAGING_BLOCK_SIZE,
+					DMA_BIDIRECTIONAL);
+		memcpy(paging->data, page_address(pages),
+		       PAGING_BLOCK_SIZE);
+		dma_sync_single_for_device(fwrt->trans->dev, addr,
+					   PAGING_BLOCK_SIZE,
+					   DMA_BIDIRECTIONAL);
+		(*data) = iwl_fw_error_next_data(*data);
+
+		if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
+			fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx,
+						     fwrt->fw_paging_db[i].fw_offs,
+						     paging->data,
+						     PAGING_BLOCK_SIZE);
+	}
+}
+
+static struct iwl_fw_error_dump_file *
+iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
+		       struct iwl_fw_dump_ptrs *fw_error_dump,
+		       struct iwl_fwrt_dump_data *data)
+{
+	struct iwl_fw_error_dump_file *dump_file;
+	struct iwl_fw_error_dump_data *dump_data;
+	struct iwl_fw_error_dump_info *dump_info;
+	struct iwl_fw_error_dump_smem_cfg *dump_smem_cfg;
+	struct iwl_fw_error_dump_trigger_desc *dump_trig;
+	u32 sram_len, sram_ofs;
+	const struct iwl_fw_dbg_mem_seg_tlv *fw_mem = fwrt->fw->dbg.mem_tlv;
+	struct iwl_fwrt_shared_mem_cfg *mem_cfg = &fwrt->smem_cfg;
+	u32 file_len, fifo_len = 0, prph_len = 0, radio_len = 0;
+	u32 smem_len = fwrt->fw->dbg.n_mem_tlv ? 0 : fwrt->trans->mac_cfg->base->smem_len;
+	u32 sram2_len = fwrt->fw->dbg.n_mem_tlv ?
+				0 : fwrt->trans->cfg->dccm2_len;
+	int i;
+
+	/* SRAM - include stack CCM if driver knows the values for it */
+	if (!fwrt->trans->cfg->dccm_offset ||
+	    !fwrt->trans->cfg->dccm_len) {
+		const struct fw_img *img;
+
+		if (fwrt->cur_fw_img >= IWL_UCODE_TYPE_MAX)
+			return NULL;
+		img = &fwrt->fw->img[fwrt->cur_fw_img];
+		sram_ofs = img->sec[IWL_UCODE_SECTION_DATA].offset;
+		sram_len = img->sec[IWL_UCODE_SECTION_DATA].len;
+	} else {
+		sram_ofs = fwrt->trans->cfg->dccm_offset;
+		sram_len = fwrt->trans->cfg->dccm_len;
+	}
+
+	/* reading RXF/TXF sizes */
+	if (iwl_trans_is_fw_error(fwrt->trans)) {
+		fifo_len = iwl_fw_rxf_len(fwrt, mem_cfg);
+		fifo_len += iwl_fw_txf_len(fwrt, mem_cfg);
+
+		/* Make room for PRPH registers */
+		if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_PRPH))
+			iwl_fw_prph_handler(fwrt, &prph_len,
+					    iwl_fw_get_prph_len);
+
+		if (fwrt->trans->mac_cfg->device_family ==
+		    IWL_DEVICE_FAMILY_7000 &&
+		    iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RADIO_REG))
+			radio_len = sizeof(*dump_data) + RADIO_REG_MAX_READ;
+	}
+
+	file_len = sizeof(*dump_file) + fifo_len + prph_len + radio_len;
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO))
+		file_len += sizeof(*dump_data) + sizeof(*dump_info);
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG))
+		file_len += sizeof(*dump_data) + sizeof(*dump_smem_cfg);
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
+		size_t hdr_len = sizeof(*dump_data) +
+				 sizeof(struct iwl_fw_error_dump_mem);
+
+		/* Dump SRAM only if no mem_tlvs */
+		if (!fwrt->fw->dbg.n_mem_tlv)
+			ADD_LEN(file_len, sram_len, hdr_len);
+
+		/* Make room for all mem types that exist */
+		ADD_LEN(file_len, smem_len, hdr_len);
+		ADD_LEN(file_len, sram2_len, hdr_len);
+
+		for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++)
+			ADD_LEN(file_len, le32_to_cpu(fw_mem[i].len), hdr_len);
+	}
+
+	/* Make room for fw's virtual image pages, if it exists */
+	if (iwl_fw_dbg_is_paging_enabled(fwrt))
+		file_len += fwrt->num_of_paging_blk *
+			(sizeof(*dump_data) +
+			 sizeof(struct iwl_fw_error_dump_paging) +
+			 PAGING_BLOCK_SIZE);
+
+	if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
+		file_len += sizeof(*dump_data) +
+			fwrt->trans->mac_cfg->base->d3_debug_data_length * 2;
+	}
+
+	/* If we only want a monitor dump, reset the file length */
+	if (data->monitor_only) {
+		file_len = sizeof(*dump_file) + sizeof(*dump_data) * 2 +
+			   sizeof(*dump_info) + sizeof(*dump_smem_cfg);
+	}
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
+	    data->desc)
+		file_len += sizeof(*dump_data) + sizeof(*dump_trig) +
+			data->desc->len;
+
+	dump_file = vzalloc(file_len);
+	if (!dump_file)
+		return NULL;
+
+	fw_error_dump->fwrt_ptr = dump_file;
+
+	dump_file->barker = cpu_to_le32(IWL_FW_ERROR_DUMP_BARKER);
+	dump_data = (void *)dump_file->data;
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO)) {
+		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_DEV_FW_INFO);
+		dump_data->len = cpu_to_le32(sizeof(*dump_info));
+		dump_info = (void *)dump_data->data;
+		dump_info->hw_type =
+			cpu_to_le32(CSR_HW_REV_TYPE(fwrt->trans->info.hw_rev));
+		dump_info->hw_step =
+			cpu_to_le32(fwrt->trans->info.hw_rev_step);
+		memcpy(dump_info->fw_human_readable, fwrt->fw->human_readable,
+		       sizeof(dump_info->fw_human_readable));
+		strscpy_pad(dump_info->dev_human_readable,
+			    fwrt->trans->info.name,
+			    sizeof(dump_info->dev_human_readable));
+		strscpy_pad(dump_info->bus_human_readable, fwrt->dev->bus->name,
+			sizeof(dump_info->bus_human_readable));
+		dump_info->num_of_lmacs = fwrt->smem_cfg.num_lmacs;
+		dump_info->lmac_err_id[0] =
+			cpu_to_le32(fwrt->dump.lmac_err_id[0]);
+		if (fwrt->smem_cfg.num_lmacs > 1)
+			dump_info->lmac_err_id[1] =
+				cpu_to_le32(fwrt->dump.lmac_err_id[1]);
+		dump_info->umac_err_id = cpu_to_le32(fwrt->dump.umac_err_id);
+
+		dump_data = iwl_fw_error_next_data(dump_data);
+	}
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG)) {
+		/* Dump shared memory configuration */
+		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_CFG);
+		dump_data->len = cpu_to_le32(sizeof(*dump_smem_cfg));
+		dump_smem_cfg = (void *)dump_data->data;
+		dump_smem_cfg->num_lmacs = cpu_to_le32(mem_cfg->num_lmacs);
+		dump_smem_cfg->num_txfifo_entries =
+			cpu_to_le32(mem_cfg->num_txfifo_entries);
+		for (i = 0; i < MAX_NUM_LMAC; i++) {
+			int j;
+			u32 *txf_size = mem_cfg->lmac[i].txfifo_size;
+
+			for (j = 0; j < TX_FIFO_MAX_NUM; j++)
+				dump_smem_cfg->lmac[i].txfifo_size[j] =
+					cpu_to_le32(txf_size[j]);
+			dump_smem_cfg->lmac[i].rxfifo1_size =
+				cpu_to_le32(mem_cfg->lmac[i].rxfifo1_size);
+		}
+		dump_smem_cfg->rxfifo2_size =
+			cpu_to_le32(mem_cfg->rxfifo2_size);
+		dump_smem_cfg->internal_txfifo_addr =
+			cpu_to_le32(mem_cfg->internal_txfifo_addr);
+		for (i = 0; i < TX_FIFO_INTERNAL_MAX_NUM; i++) {
+			dump_smem_cfg->internal_txfifo_size[i] =
+				cpu_to_le32(mem_cfg->internal_txfifo_size[i]);
+		}
+
+		dump_data = iwl_fw_error_next_data(dump_data);
+	}
+
+	/* We only dump the FIFOs if the FW is in error state */
+	if (fifo_len) {
+		iwl_fw_dump_rxf(fwrt, &dump_data);
+		iwl_fw_dump_txf(fwrt, &dump_data);
+	}
+
+	if (radio_len)
+		iwl_read_radio_regs(fwrt, &dump_data);
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
+	    data->desc) {
+		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_ERROR_INFO);
+		dump_data->len = cpu_to_le32(sizeof(*dump_trig) +
+					     data->desc->len);
+		dump_trig = (void *)dump_data->data;
+		memcpy(dump_trig, &data->desc->trig_desc,
+		       sizeof(*dump_trig) + data->desc->len);
+
+		dump_data = iwl_fw_error_next_data(dump_data);
+	}
+
+	/* In case we only want monitor dump, skip to dump transport data */
+	if (data->monitor_only)
+		goto out;
+
+	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
+		const struct iwl_fw_dbg_mem_seg_tlv *fw_dbg_mem =
+			fwrt->fw->dbg.mem_tlv;
+
+		if (!fwrt->fw->dbg.n_mem_tlv)
+			iwl_fw_dump_mem(fwrt, &dump_data, sram_len, sram_ofs,
+					IWL_FW_ERROR_DUMP_MEM_SRAM);
+
+		for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++) {
+			u32 len = le32_to_cpu(fw_dbg_mem[i].len);
+			u32 ofs = le32_to_cpu(fw_dbg_mem[i].ofs);
+
+			iwl_fw_dump_mem(fwrt, &dump_data, len, ofs,
+					le32_to_cpu(fw_dbg_mem[i].data_type));
+		}
+
+		iwl_fw_dump_mem(fwrt, &dump_data, smem_len,
+				fwrt->trans->mac_cfg->base->smem_offset,
+				IWL_FW_ERROR_DUMP_MEM_SMEM);
+
+		iwl_fw_dump_mem(fwrt, &dump_data, sram2_len,
+				fwrt->trans->cfg->dccm2_offset,
+				IWL_FW_ERROR_DUMP_MEM_SRAM);
+	}
+
+	if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
+		u32 addr = fwrt->trans->mac_cfg->base->d3_debug_data_base_addr;
+		size_t data_size = fwrt->trans->mac_cfg->base->d3_debug_data_length;
+
+		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_D3_DEBUG_DATA);
+		dump_data->len = cpu_to_le32(data_size * 2);
+
+		memcpy(dump_data->data, fwrt->dump.d3_debug_data, data_size);
+
+		kfree(fwrt->dump.d3_debug_data);
+		fwrt->dump.d3_debug_data = NULL;
+
+		iwl_trans_read_mem_bytes(fwrt->trans, addr,
+					 dump_data->data + data_size,
+					 data_size);
+
+		if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
+			fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx, addr,
+						     dump_data->data + data_size,
+						     data_size);
+
+		dump_data = iwl_fw_error_next_data(dump_data);
+	}
+
+	/* Dump fw's virtual image */
+	if (iwl_fw_dbg_is_paging_enabled(fwrt))
+		iwl_dump_paging(fwrt, &dump_data);
+
+	if (prph_len)
+		iwl_fw_prph_handler(fwrt, &dump_data, iwl_dump_prph);
+
+out:
+	dump_file->file_len = cpu_to_le32(file_len);
+	return dump_file;
+}
+
+void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt,
+		       struct iwl_fwrt_dump_data *dump_data)
+{
+	struct iwl_fw_dump_ptrs fw_error_dump = {};
+	struct iwl_fw_error_dump_file *dump_file;
+	struct scatterlist *sg_dump_data;
+	u32 file_len;
+	u32 dump_mask = fwrt->fw->dbg.dump_mask;
+
+	dump_file = iwl_fw_error_dump_file(fwrt, &fw_error_dump, dump_data);
+	if (!dump_file)
+		return;
+
+	if (dump_data->monitor_only)
+		dump_mask &= BIT(IWL_FW_ERROR_DUMP_FW_MONITOR);
+
+	fw_error_dump.trans_ptr = iwl_trans_dump_data(fwrt->trans, dump_mask,
+						      fwrt->sanitize_ops,
+						      fwrt->sanitize_ctx);
+	file_len = le32_to_cpu(dump_file->file_len);
+	fw_error_dump.fwrt_len = file_len;
+
+	if (fw_error_dump.trans_ptr) {
+		file_len += fw_error_dump.trans_ptr->len;
+		dump_file->file_len = cpu_to_le32(file_len);
+	}
+
+	sg_dump_data = iwl_fw_dbg_alloc_sgtable(file_len);
+	if (sg_dump_data) {
+		sg_pcopy_from_buffer(sg_dump_data,
+				     sg_nents(sg_dump_data),
+				     fw_error_dump.fwrt_ptr,
+				     fw_error_dump.fwrt_len, 0);
+		if (fw_error_dump.trans_ptr)
+			sg_pcopy_from_buffer(sg_dump_data,
+					     sg_nents(sg_dump_data),
+					     fw_error_dump.trans_ptr->data,
+					     fw_error_dump.trans_ptr->len,
+					     fw_error_dump.fwrt_len);
+		dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len,
+			       GFP_KERNEL);
+	}
+	vfree(fw_error_dump.fwrt_ptr);
+	vfree(fw_error_dump.trans_ptr);
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
index 0cffa5493704d..069c3bad6f29f 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -13,556 +13,13 @@
 #include "iwl-prph.h"
 #include "iwl-csr.h"
 #include "iwl-fh.h"
-/**
- * struct iwl_fw_dump_ptrs - set of pointers needed for the fw-error-dump
- *
- * @fwrt_ptr: pointer to the buffer coming from fwrt
- * @trans_ptr: pointer to struct %iwl_trans_dump_data which contains the
- *	transport's data.
- * @fwrt_len: length of the valid data in fwrt_ptr
- */
-struct iwl_fw_dump_ptrs {
-	struct iwl_trans_dump_data *trans_ptr;
-	void *fwrt_ptr;
-	u32 fwrt_len;
-};
-
-#define RADIO_REG_MAX_READ 0x2ad
-static void iwl_read_radio_regs(struct iwl_fw_runtime *fwrt,
-				struct iwl_fw_error_dump_data **dump_data)
-{
-	u8 *pos = (void *)(*dump_data)->data;
-	int i;
-
-	IWL_DEBUG_INFO(fwrt, "WRT radio registers dump\n");
-
-	if (!iwl_trans_grab_nic_access(fwrt->trans))
-		return;
-
-	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RADIO_REG);
-	(*dump_data)->len = cpu_to_le32(RADIO_REG_MAX_READ);
-
-	for (i = 0; i < RADIO_REG_MAX_READ; i++) {
-		u32 rd_cmd = RADIO_RSP_RD_CMD;
-
-		rd_cmd |= i << RADIO_RSP_ADDR_POS;
-		iwl_write_prph_no_grab(fwrt->trans, RSP_RADIO_CMD, rd_cmd);
-		*pos = (u8)iwl_read_prph_no_grab(fwrt->trans, RSP_RADIO_RDDAT);
-
-		pos++;
-	}
-
-	*dump_data = iwl_fw_error_next_data(*dump_data);
-
-	iwl_trans_release_nic_access(fwrt->trans);
-}
-
-static void iwl_fwrt_dump_rxf(struct iwl_fw_runtime *fwrt,
-			      struct iwl_fw_error_dump_data **dump_data,
-			      int size, u32 offset, int fifo_num)
-{
-	struct iwl_fw_error_dump_fifo *fifo_hdr;
-	u32 *fifo_data;
-	u32 fifo_len;
-	int i;
-
-	fifo_hdr = (void *)(*dump_data)->data;
-	fifo_data = (void *)fifo_hdr->data;
-	fifo_len = size;
-
-	/* No need to try to read the data if the length is 0 */
-	if (fifo_len == 0)
-		return;
-
-	/* Add a TLV for the RXF */
-	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RXF);
-	(*dump_data)->len = cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
-
-	fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
-	fifo_hdr->available_bytes =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						RXF_RD_D_SPACE + offset));
-	fifo_hdr->wr_ptr =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						RXF_RD_WR_PTR + offset));
-	fifo_hdr->rd_ptr =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						RXF_RD_RD_PTR + offset));
-	fifo_hdr->fence_ptr =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						RXF_RD_FENCE_PTR + offset));
-	fifo_hdr->fence_mode =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						RXF_SET_FENCE_MODE + offset));
-
-	/* Lock fence */
-	iwl_trans_write_prph(fwrt->trans, RXF_SET_FENCE_MODE + offset, 0x1);
-	/* Set fence pointer to the same place like WR pointer */
-	iwl_trans_write_prph(fwrt->trans, RXF_LD_WR2FENCE + offset, 0x1);
-	/* Set fence offset */
-	iwl_trans_write_prph(fwrt->trans,
-			     RXF_LD_FENCE_OFFSET_ADDR + offset, 0x0);
-
-	/* Read FIFO */
-	fifo_len /= sizeof(u32); /* Size in DWORDS */
-	for (i = 0; i < fifo_len; i++)
-		fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
-						 RXF_FIFO_RD_FENCE_INC +
-						 offset);
-	*dump_data = iwl_fw_error_next_data(*dump_data);
-}
-
-static void iwl_fwrt_dump_txf(struct iwl_fw_runtime *fwrt,
-			      struct iwl_fw_error_dump_data **dump_data,
-			      int size, u32 offset, int fifo_num)
-{
-	struct iwl_fw_error_dump_fifo *fifo_hdr;
-	u32 *fifo_data;
-	u32 fifo_len;
-	int i;
-
-	fifo_hdr = (void *)(*dump_data)->data;
-	fifo_data = (void *)fifo_hdr->data;
-	fifo_len = size;
-
-	/* No need to try to read the data if the length is 0 */
-	if (fifo_len == 0)
-		return;
-
-	/* Add a TLV for the FIFO */
-	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXF);
-	(*dump_data)->len = cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
-
-	fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
-	fifo_hdr->available_bytes =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						TXF_FIFO_ITEM_CNT + offset));
-	fifo_hdr->wr_ptr =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						TXF_WR_PTR + offset));
-	fifo_hdr->rd_ptr =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						TXF_RD_PTR + offset));
-	fifo_hdr->fence_ptr =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						TXF_FENCE_PTR + offset));
-	fifo_hdr->fence_mode =
-		cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-						TXF_LOCK_FENCE + offset));
-
-	/* Set the TXF_READ_MODIFY_ADDR to TXF_WR_PTR */
-	iwl_trans_write_prph(fwrt->trans, TXF_READ_MODIFY_ADDR + offset,
-			     TXF_WR_PTR + offset);
-
-	/* Dummy-read to advance the read pointer to the head */
-	iwl_trans_read_prph(fwrt->trans, TXF_READ_MODIFY_DATA + offset);
-
-	/* Read FIFO */
-	for (i = 0; i < fifo_len / sizeof(u32); i++)
-		fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
-						  TXF_READ_MODIFY_DATA +
-						  offset);
-
-	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_txf)
-		fwrt->sanitize_ops->frob_txf(fwrt->sanitize_ctx,
-					     fifo_data, fifo_len);
-
-	*dump_data = iwl_fw_error_next_data(*dump_data);
-}
-
-static void iwl_fw_dump_rxf(struct iwl_fw_runtime *fwrt,
-			    struct iwl_fw_error_dump_data **dump_data)
-{
-	struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
-
-	IWL_DEBUG_INFO(fwrt, "WRT RX FIFO dump\n");
-
-	if (!iwl_trans_grab_nic_access(fwrt->trans))
-		return;
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF)) {
-		/* Pull RXF1 */
-		iwl_fwrt_dump_rxf(fwrt, dump_data,
-				  cfg->lmac[0].rxfifo1_size, 0, 0);
-		/* Pull RXF2 */
-		iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->rxfifo2_size,
-				  RXF_DIFF_FROM_PREV +
-				  fwrt->trans->mac_cfg->umac_prph_offset, 1);
-		/* Pull LMAC2 RXF1 */
-		if (fwrt->smem_cfg.num_lmacs > 1)
-			iwl_fwrt_dump_rxf(fwrt, dump_data,
-					  cfg->lmac[1].rxfifo1_size,
-					  LMAC2_PRPH_OFFSET, 2);
-	}
-
-	iwl_trans_release_nic_access(fwrt->trans);
-}
-
-static void iwl_fw_dump_txf(struct iwl_fw_runtime *fwrt,
-			    struct iwl_fw_error_dump_data **dump_data)
-{
-	struct iwl_fw_error_dump_fifo *fifo_hdr;
-	struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
-	u32 *fifo_data;
-	u32 fifo_len;
-	int i, j;
-
-	IWL_DEBUG_INFO(fwrt, "WRT TX FIFO dump\n");
-
-	if (!iwl_trans_grab_nic_access(fwrt->trans))
-		return;
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF)) {
-		/* Pull TXF data from LMAC1 */
-		for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries; i++) {
-			/* Mark the number of TXF we're pulling now */
-			iwl_trans_write_prph(fwrt->trans, TXF_LARC_NUM, i);
-			iwl_fwrt_dump_txf(fwrt, dump_data,
-					  cfg->lmac[0].txfifo_size[i], 0, i);
-		}
-
-		/* Pull TXF data from LMAC2 */
-		if (fwrt->smem_cfg.num_lmacs > 1) {
-			for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries;
-			     i++) {
-				/* Mark the number of TXF we're pulling now */
-				iwl_trans_write_prph(fwrt->trans,
-						     TXF_LARC_NUM +
-						     LMAC2_PRPH_OFFSET, i);
-				iwl_fwrt_dump_txf(fwrt, dump_data,
-						  cfg->lmac[1].txfifo_size[i],
-						  LMAC2_PRPH_OFFSET,
-						  i + cfg->num_txfifo_entries);
-			}
-		}
-	}
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
-	    fw_has_capa(&fwrt->fw->ucode_capa,
-			IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
-		/* Pull UMAC internal TXF data from all TXFs */
-		for (i = 0;
-		     i < ARRAY_SIZE(fwrt->smem_cfg.internal_txfifo_size);
-		     i++) {
-			fifo_hdr = (void *)(*dump_data)->data;
-			fifo_data = (void *)fifo_hdr->data;
-			fifo_len = fwrt->smem_cfg.internal_txfifo_size[i];
-
-			/* No need to try to read the data if the length is 0 */
-			if (fifo_len == 0)
-				continue;
-
-			/* Add a TLV for the internal FIFOs */
-			(*dump_data)->type =
-				cpu_to_le32(IWL_FW_ERROR_DUMP_INTERNAL_TXF);
-			(*dump_data)->len =
-				cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
-
-			fifo_hdr->fifo_num = cpu_to_le32(i);
-
-			/* Mark the number of TXF we're pulling now */
-			iwl_trans_write_prph(fwrt->trans, TXF_CPU2_NUM, i +
-				fwrt->smem_cfg.num_txfifo_entries);
-
-			fifo_hdr->available_bytes =
-				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-								TXF_CPU2_FIFO_ITEM_CNT));
-			fifo_hdr->wr_ptr =
-				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-								TXF_CPU2_WR_PTR));
-			fifo_hdr->rd_ptr =
-				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-								TXF_CPU2_RD_PTR));
-			fifo_hdr->fence_ptr =
-				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-								TXF_CPU2_FENCE_PTR));
-			fifo_hdr->fence_mode =
-				cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
-								TXF_CPU2_LOCK_FENCE));
-
-			/* Set TXF_CPU2_READ_MODIFY_ADDR to TXF_CPU2_WR_PTR */
-			iwl_trans_write_prph(fwrt->trans,
-					     TXF_CPU2_READ_MODIFY_ADDR,
-					     TXF_CPU2_WR_PTR);
-
-			/* Dummy-read to advance the read pointer to head */
-			iwl_trans_read_prph(fwrt->trans,
-					    TXF_CPU2_READ_MODIFY_DATA);
-
-			/* Read FIFO */
-			fifo_len /= sizeof(u32); /* Size in DWORDS */
-			for (j = 0; j < fifo_len; j++)
-				fifo_data[j] =
-					iwl_trans_read_prph(fwrt->trans,
-							    TXF_CPU2_READ_MODIFY_DATA);
-			*dump_data = iwl_fw_error_next_data(*dump_data);
-		}
-	}
-
-	iwl_trans_release_nic_access(fwrt->trans);
-}
-
-struct iwl_prph_range {
-	u32 start, end;
-};
-
-static const struct iwl_prph_range iwl_prph_dump_addr_comm[] = {
-	{ .start = 0x00a00000, .end = 0x00a00000 },
-	{ .start = 0x00a0000c, .end = 0x00a00024 },
-	{ .start = 0x00a0002c, .end = 0x00a0003c },
-	{ .start = 0x00a00410, .end = 0x00a00418 },
-	{ .start = 0x00a00420, .end = 0x00a00420 },
-	{ .start = 0x00a00428, .end = 0x00a00428 },
-	{ .start = 0x00a00430, .end = 0x00a0043c },
-	{ .start = 0x00a00444, .end = 0x00a00444 },
-	{ .start = 0x00a004c0, .end = 0x00a004cc },
-	{ .start = 0x00a004d8, .end = 0x00a004d8 },
-	{ .start = 0x00a004e0, .end = 0x00a004f0 },
-	{ .start = 0x00a00840, .end = 0x00a00840 },
-	{ .start = 0x00a00850, .end = 0x00a00858 },
-	{ .start = 0x00a01004, .end = 0x00a01008 },
-	{ .start = 0x00a01010, .end = 0x00a01010 },
-	{ .start = 0x00a01018, .end = 0x00a01018 },
-	{ .start = 0x00a01024, .end = 0x00a01024 },
-	{ .start = 0x00a0102c, .end = 0x00a01034 },
-	{ .start = 0x00a0103c, .end = 0x00a01040 },
-	{ .start = 0x00a01048, .end = 0x00a01094 },
-	{ .start = 0x00a01c00, .end = 0x00a01c20 },
-	{ .start = 0x00a01c58, .end = 0x00a01c58 },
-	{ .start = 0x00a01c7c, .end = 0x00a01c7c },
-	{ .start = 0x00a01c28, .end = 0x00a01c54 },
-	{ .start = 0x00a01c5c, .end = 0x00a01c5c },
-	{ .start = 0x00a01c60, .end = 0x00a01cdc },
-	{ .start = 0x00a01ce0, .end = 0x00a01d0c },
-	{ .start = 0x00a01d18, .end = 0x00a01d20 },
-	{ .start = 0x00a01d2c, .end = 0x00a01d30 },
-	{ .start = 0x00a01d40, .end = 0x00a01d5c },
-	{ .start = 0x00a01d80, .end = 0x00a01d80 },
-	{ .start = 0x00a01d98, .end = 0x00a01d9c },
-	{ .start = 0x00a01da8, .end = 0x00a01da8 },
-	{ .start = 0x00a01db8, .end = 0x00a01df4 },
-	{ .start = 0x00a01dc0, .end = 0x00a01dfc },
-	{ .start = 0x00a01e00, .end = 0x00a01e2c },
-	{ .start = 0x00a01e40, .end = 0x00a01e60 },
-	{ .start = 0x00a01e68, .end = 0x00a01e6c },
-	{ .start = 0x00a01e74, .end = 0x00a01e74 },
-	{ .start = 0x00a01e84, .end = 0x00a01e90 },
-	{ .start = 0x00a01e9c, .end = 0x00a01ec4 },
-	{ .start = 0x00a01ed0, .end = 0x00a01ee0 },
-	{ .start = 0x00a01f00, .end = 0x00a01f1c },
-	{ .start = 0x00a01f44, .end = 0x00a01ffc },
-	{ .start = 0x00a02000, .end = 0x00a02048 },
-	{ .start = 0x00a02068, .end = 0x00a020f0 },
-	{ .start = 0x00a02100, .end = 0x00a02118 },
-	{ .start = 0x00a02140, .end = 0x00a0214c },
-	{ .start = 0x00a02168, .end = 0x00a0218c },
-	{ .start = 0x00a021c0, .end = 0x00a021c0 },
-	{ .start = 0x00a02400, .end = 0x00a02410 },
-	{ .start = 0x00a02418, .end = 0x00a02420 },
-	{ .start = 0x00a02428, .end = 0x00a0242c },
-	{ .start = 0x00a02434, .end = 0x00a02434 },
-	{ .start = 0x00a02440, .end = 0x00a02460 },
-	{ .start = 0x00a02468, .end = 0x00a024b0 },
-	{ .start = 0x00a024c8, .end = 0x00a024cc },
-	{ .start = 0x00a02500, .end = 0x00a02504 },
-	{ .start = 0x00a0250c, .end = 0x00a02510 },
-	{ .start = 0x00a02540, .end = 0x00a02554 },
-	{ .start = 0x00a02580, .end = 0x00a025f4 },
-	{ .start = 0x00a02600, .end = 0x00a0260c },
-	{ .start = 0x00a02648, .end = 0x00a02650 },
-	{ .start = 0x00a02680, .end = 0x00a02680 },
-	{ .start = 0x00a026c0, .end = 0x00a026d0 },
-	{ .start = 0x00a02700, .end = 0x00a0270c },
-	{ .start = 0x00a02804, .end = 0x00a02804 },
-	{ .start = 0x00a02818, .end = 0x00a0281c },
-	{ .start = 0x00a02c00, .end = 0x00a02db4 },
-	{ .start = 0x00a02df4, .end = 0x00a02fb0 },
-	{ .start = 0x00a03000, .end = 0x00a03014 },
-	{ .start = 0x00a0301c, .end = 0x00a0302c },
-	{ .start = 0x00a03034, .end = 0x00a03038 },
-	{ .start = 0x00a03040, .end = 0x00a03048 },
-	{ .start = 0x00a03060, .end = 0x00a03068 },
-	{ .start = 0x00a03070, .end = 0x00a03074 },
-	{ .start = 0x00a0307c, .end = 0x00a0307c },
-	{ .start = 0x00a03080, .end = 0x00a03084 },
-	{ .start = 0x00a0308c, .end = 0x00a03090 },
-	{ .start = 0x00a03098, .end = 0x00a03098 },
-	{ .start = 0x00a030a0, .end = 0x00a030a0 },
-	{ .start = 0x00a030a8, .end = 0x00a030b4 },
-	{ .start = 0x00a030bc, .end = 0x00a030bc },
-	{ .start = 0x00a030c0, .end = 0x00a0312c },
-	{ .start = 0x00a03c00, .end = 0x00a03c5c },
-	{ .start = 0x00a04400, .end = 0x00a04454 },
-	{ .start = 0x00a04460, .end = 0x00a04474 },
-	{ .start = 0x00a044c0, .end = 0x00a044ec },
-	{ .start = 0x00a04500, .end = 0x00a04504 },
-	{ .start = 0x00a04510, .end = 0x00a04538 },
-	{ .start = 0x00a04540, .end = 0x00a04548 },
-	{ .start = 0x00a04560, .end = 0x00a0457c },
-	{ .start = 0x00a04590, .end = 0x00a04598 },
-	{ .start = 0x00a045c0, .end = 0x00a045f4 },
-};
-
-static const struct iwl_prph_range iwl_prph_dump_addr_9000[] = {
-	{ .start = 0x00a05c00, .end = 0x00a05c18 },
-	{ .start = 0x00a05400, .end = 0x00a056e8 },
-	{ .start = 0x00a08000, .end = 0x00a098bc },
-	{ .start = 0x00a02400, .end = 0x00a02758 },
-	{ .start = 0x00a04764, .end = 0x00a0476c },
-	{ .start = 0x00a04770, .end = 0x00a04774 },
-	{ .start = 0x00a04620, .end = 0x00a04624 },
-};
-
-static const struct iwl_prph_range iwl_prph_dump_addr_22000[] = {
-	{ .start = 0x00a00000, .end = 0x00a00000 },
-	{ .start = 0x00a0000c, .end = 0x00a00024 },
-	{ .start = 0x00a0002c, .end = 0x00a00034 },
-	{ .start = 0x00a0003c, .end = 0x00a0003c },
-	{ .start = 0x00a00410, .end = 0x00a00418 },
-	{ .start = 0x00a00420, .end = 0x00a00420 },
-	{ .start = 0x00a00428, .end = 0x00a00428 },
-	{ .start = 0x00a00430, .end = 0x00a0043c },
-	{ .start = 0x00a00444, .end = 0x00a00444 },
-	{ .start = 0x00a00840, .end = 0x00a00840 },
-	{ .start = 0x00a00850, .end = 0x00a00858 },
-	{ .start = 0x00a01004, .end = 0x00a01008 },
-	{ .start = 0x00a01010, .end = 0x00a01010 },
-	{ .start = 0x00a01018, .end = 0x00a01018 },
-	{ .start = 0x00a01024, .end = 0x00a01024 },
-	{ .start = 0x00a0102c, .end = 0x00a01034 },
-	{ .start = 0x00a0103c, .end = 0x00a01040 },
-	{ .start = 0x00a01048, .end = 0x00a01050 },
-	{ .start = 0x00a01058, .end = 0x00a01058 },
-	{ .start = 0x00a01060, .end = 0x00a01070 },
-	{ .start = 0x00a0108c, .end = 0x00a0108c },
-	{ .start = 0x00a01c20, .end = 0x00a01c28 },
-	{ .start = 0x00a01d10, .end = 0x00a01d10 },
-	{ .start = 0x00a01e28, .end = 0x00a01e2c },
-	{ .start = 0x00a01e60, .end = 0x00a01e60 },
-	{ .start = 0x00a01e80, .end = 0x00a01e80 },
-	{ .start = 0x00a01ea0, .end = 0x00a01ea0 },
-	{ .start = 0x00a02000, .end = 0x00a0201c },
-	{ .start = 0x00a02024, .end = 0x00a02024 },
-	{ .start = 0x00a02040, .end = 0x00a02048 },
-	{ .start = 0x00a020c0, .end = 0x00a020e0 },
-	{ .start = 0x00a02400, .end = 0x00a02404 },
-	{ .start = 0x00a0240c, .end = 0x00a02414 },
-	{ .start = 0x00a0241c, .end = 0x00a0243c },
-	{ .start = 0x00a02448, .end = 0x00a024bc },
-	{ .start = 0x00a024c4, .end = 0x00a024cc },
-	{ .start = 0x00a02508, .end = 0x00a02508 },
-	{ .start = 0x00a02510, .end = 0x00a02514 },
-	{ .start = 0x00a0251c, .end = 0x00a0251c },
-	{ .start = 0x00a0252c, .end = 0x00a0255c },
-	{ .start = 0x00a02564, .end = 0x00a025a0 },
-	{ .start = 0x00a025a8, .end = 0x00a025b4 },
-	{ .start = 0x00a025c0, .end = 0x00a025c0 },
-	{ .start = 0x00a025e8, .end = 0x00a025f4 },
-	{ .start = 0x00a02c08, .end = 0x00a02c18 },
-	{ .start = 0x00a02c2c, .end = 0x00a02c38 },
-	{ .start = 0x00a02c68, .end = 0x00a02c78 },
-	{ .start = 0x00a03000, .end = 0x00a03000 },
-	{ .start = 0x00a03010, .end = 0x00a03014 },
-	{ .start = 0x00a0301c, .end = 0x00a0302c },
-	{ .start = 0x00a03034, .end = 0x00a03038 },
-	{ .start = 0x00a03040, .end = 0x00a03044 },
-	{ .start = 0x00a03060, .end = 0x00a03068 },
-	{ .start = 0x00a03070, .end = 0x00a03070 },
-	{ .start = 0x00a0307c, .end = 0x00a03084 },
-	{ .start = 0x00a0308c, .end = 0x00a03090 },
-	{ .start = 0x00a03098, .end = 0x00a03098 },
-	{ .start = 0x00a030a0, .end = 0x00a030a0 },
-	{ .start = 0x00a030a8, .end = 0x00a030b4 },
-	{ .start = 0x00a030bc, .end = 0x00a030c0 },
-	{ .start = 0x00a030c8, .end = 0x00a030f4 },
-	{ .start = 0x00a03100, .end = 0x00a0312c },
-	{ .start = 0x00a03c00, .end = 0x00a03c5c },
-	{ .start = 0x00a04400, .end = 0x00a04454 },
-	{ .start = 0x00a04460, .end = 0x00a04474 },
-	{ .start = 0x00a044c0, .end = 0x00a044ec },
-	{ .start = 0x00a04500, .end = 0x00a04504 },
-	{ .start = 0x00a04510, .end = 0x00a04538 },
-	{ .start = 0x00a04540, .end = 0x00a04548 },
-	{ .start = 0x00a04560, .end = 0x00a04560 },
-	{ .start = 0x00a04570, .end = 0x00a0457c },
-	{ .start = 0x00a04590, .end = 0x00a04590 },
-	{ .start = 0x00a04598, .end = 0x00a04598 },
-	{ .start = 0x00a045c0, .end = 0x00a045f4 },
-	{ .start = 0x00a05c18, .end = 0x00a05c1c },
-	{ .start = 0x00a0c000, .end = 0x00a0c018 },
-	{ .start = 0x00a0c020, .end = 0x00a0c028 },
-	{ .start = 0x00a0c038, .end = 0x00a0c094 },
-	{ .start = 0x00a0c0c0, .end = 0x00a0c104 },
-	{ .start = 0x00a0c10c, .end = 0x00a0c118 },
-	{ .start = 0x00a0c150, .end = 0x00a0c174 },
-	{ .start = 0x00a0c17c, .end = 0x00a0c188 },
-	{ .start = 0x00a0c190, .end = 0x00a0c198 },
-	{ .start = 0x00a0c1a0, .end = 0x00a0c1a8 },
-	{ .start = 0x00a0c1b0, .end = 0x00a0c1b8 },
-};
-
-static const struct iwl_prph_range iwl_prph_dump_addr_ax210[] = {
-	{ .start = 0x00d03c00, .end = 0x00d03c64 },
-	{ .start = 0x00d05c18, .end = 0x00d05c1c },
-	{ .start = 0x00d0c000, .end = 0x00d0c174 },
-};
-
-static void iwl_read_prph_block(struct iwl_trans *trans, u32 start,
-				u32 len_bytes, __le32 *data)
-{
-	u32 i;
-
-	for (i = 0; i < len_bytes; i += 4)
-		*data++ = cpu_to_le32(iwl_read_prph_no_grab(trans, start + i));
-}
-
-static void iwl_dump_prph(struct iwl_fw_runtime *fwrt,
-			  const struct iwl_prph_range *iwl_prph_dump_addr,
-			  u32 range_len, void *ptr)
-{
-	struct iwl_fw_error_dump_prph *prph;
-	struct iwl_trans *trans = fwrt->trans;
-	struct iwl_fw_error_dump_data **data =
-		(struct iwl_fw_error_dump_data **)ptr;
-	u32 i;
-
-	if (!data)
-		return;
-
-	IWL_DEBUG_INFO(trans, "WRT PRPH dump\n");
-
-	if (!iwl_trans_grab_nic_access(trans))
-		return;
-
-	for (i = 0; i < range_len; i++) {
-		/* The range includes both boundaries */
-		int num_bytes_in_chunk = iwl_prph_dump_addr[i].end -
-			 iwl_prph_dump_addr[i].start + 4;
-
-		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PRPH);
-		(*data)->len = cpu_to_le32(sizeof(*prph) +
-					num_bytes_in_chunk);
-		prph = (void *)(*data)->data;
-		prph->prph_start = cpu_to_le32(iwl_prph_dump_addr[i].start);
-
-		iwl_read_prph_block(trans, iwl_prph_dump_addr[i].start,
-				    /* our range is inclusive, hence + 4 */
-				    iwl_prph_dump_addr[i].end -
-				    iwl_prph_dump_addr[i].start + 4,
-				    (void *)prph->data);
-
-		*data = iwl_fw_error_next_data(*data);
-	}
-
-	iwl_trans_release_nic_access(trans);
-}
 
 /*
- * alloc_sgtable - allocates (chained) scatterlist in the given size,
+ * iwl_fw_dbg_alloc_sgtable - allocates (chained) scatterlist in the given size,
  *	fills it with pages and returns it
  * @size: the size (in bytes) of the table
  */
-static struct scatterlist *alloc_sgtable(ssize_t size)
+struct scatterlist *iwl_fw_dbg_alloc_sgtable(ssize_t size)
 {
 	struct scatterlist *result = NULL, *prev;
 	int nents, i, n_prev;
@@ -625,423 +82,6 @@ static struct scatterlist *alloc_sgtable(ssize_t size)
 	return result;
 }
 
-static void iwl_fw_get_prph_len(struct iwl_fw_runtime *fwrt,
-				const struct iwl_prph_range *iwl_prph_dump_addr,
-				u32 range_len, void *ptr)
-{
-	u32 *prph_len = (u32 *)ptr;
-	int i, num_bytes_in_chunk;
-
-	if (!prph_len)
-		return;
-
-	for (i = 0; i < range_len; i++) {
-		/* The range includes both boundaries */
-		num_bytes_in_chunk =
-			iwl_prph_dump_addr[i].end -
-			iwl_prph_dump_addr[i].start + 4;
-
-		*prph_len += sizeof(struct iwl_fw_error_dump_data) +
-			sizeof(struct iwl_fw_error_dump_prph) +
-			num_bytes_in_chunk;
-	}
-}
-
-static void iwl_fw_prph_handler(struct iwl_fw_runtime *fwrt, void *ptr,
-				void (*handler)(struct iwl_fw_runtime *,
-						const struct iwl_prph_range *,
-						u32, void *))
-{
-	u32 range_len;
-
-	if (fwrt->trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
-		range_len = ARRAY_SIZE(iwl_prph_dump_addr_ax210);
-		handler(fwrt, iwl_prph_dump_addr_ax210, range_len, ptr);
-	} else if (fwrt->trans->mac_cfg->device_family >=
-		   IWL_DEVICE_FAMILY_22000) {
-		range_len = ARRAY_SIZE(iwl_prph_dump_addr_22000);
-		handler(fwrt, iwl_prph_dump_addr_22000, range_len, ptr);
-	} else {
-		range_len = ARRAY_SIZE(iwl_prph_dump_addr_comm);
-		handler(fwrt, iwl_prph_dump_addr_comm, range_len, ptr);
-
-		if (fwrt->trans->mac_cfg->mq_rx_supported) {
-			range_len = ARRAY_SIZE(iwl_prph_dump_addr_9000);
-			handler(fwrt, iwl_prph_dump_addr_9000, range_len, ptr);
-		}
-	}
-}
-
-static void iwl_fw_dump_mem(struct iwl_fw_runtime *fwrt,
-			    struct iwl_fw_error_dump_data **dump_data,
-			    u32 len, u32 ofs, u32 type)
-{
-	struct iwl_fw_error_dump_mem *dump_mem;
-
-	if (!len)
-		return;
-
-	(*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
-	(*dump_data)->len = cpu_to_le32(len + sizeof(*dump_mem));
-	dump_mem = (void *)(*dump_data)->data;
-	dump_mem->type = cpu_to_le32(type);
-	dump_mem->offset = cpu_to_le32(ofs);
-	iwl_trans_read_mem_bytes(fwrt->trans, ofs, dump_mem->data, len);
-	*dump_data = iwl_fw_error_next_data(*dump_data);
-
-	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
-		fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx, ofs,
-					     dump_mem->data, len);
-
-	IWL_DEBUG_INFO(fwrt, "WRT memory dump. Type=%u\n", dump_mem->type);
-}
-
-#define ADD_LEN(len, item_len, const_len) \
-	do {size_t item = item_len; len += (!!item) * const_len + item; } \
-	while (0)
-
-static int iwl_fw_rxf_len(struct iwl_fw_runtime *fwrt,
-			  struct iwl_fwrt_shared_mem_cfg *mem_cfg)
-{
-	size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
-			 sizeof(struct iwl_fw_error_dump_fifo);
-	u32 fifo_len = 0;
-	int i;
-
-	if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF))
-		return 0;
-
-	/* Count RXF2 size */
-	ADD_LEN(fifo_len, mem_cfg->rxfifo2_size, hdr_len);
-
-	/* Count RXF1 sizes */
-	if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
-		mem_cfg->num_lmacs = MAX_NUM_LMAC;
-
-	for (i = 0; i < mem_cfg->num_lmacs; i++)
-		ADD_LEN(fifo_len, mem_cfg->lmac[i].rxfifo1_size, hdr_len);
-
-	return fifo_len;
-}
-
-static int iwl_fw_txf_len(struct iwl_fw_runtime *fwrt,
-			  struct iwl_fwrt_shared_mem_cfg *mem_cfg)
-{
-	size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
-			 sizeof(struct iwl_fw_error_dump_fifo);
-	u32 fifo_len = 0;
-	int i;
-
-	if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF))
-		goto dump_internal_txf;
-
-	/* Count TXF sizes */
-	if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
-		mem_cfg->num_lmacs = MAX_NUM_LMAC;
-
-	for (i = 0; i < mem_cfg->num_lmacs; i++) {
-		int j;
-
-		for (j = 0; j < mem_cfg->num_txfifo_entries; j++)
-			ADD_LEN(fifo_len, mem_cfg->lmac[i].txfifo_size[j],
-				hdr_len);
-	}
-
-dump_internal_txf:
-	if (!(iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
-	      fw_has_capa(&fwrt->fw->ucode_capa,
-			  IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)))
-		goto out;
-
-	for (i = 0; i < ARRAY_SIZE(mem_cfg->internal_txfifo_size); i++)
-		ADD_LEN(fifo_len, mem_cfg->internal_txfifo_size[i], hdr_len);
-
-out:
-	return fifo_len;
-}
-
-static void iwl_dump_paging(struct iwl_fw_runtime *fwrt,
-			    struct iwl_fw_error_dump_data **data)
-{
-	int i;
-
-	IWL_DEBUG_INFO(fwrt, "WRT paging dump\n");
-	for (i = 1; i < fwrt->num_of_paging_blk + 1; i++) {
-		struct iwl_fw_error_dump_paging *paging;
-		struct page *pages =
-			fwrt->fw_paging_db[i].fw_paging_block;
-		dma_addr_t addr = fwrt->fw_paging_db[i].fw_paging_phys;
-
-		(*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
-		(*data)->len = cpu_to_le32(sizeof(*paging) +
-					     PAGING_BLOCK_SIZE);
-		paging =  (void *)(*data)->data;
-		paging->index = cpu_to_le32(i);
-		dma_sync_single_for_cpu(fwrt->trans->dev, addr,
-					PAGING_BLOCK_SIZE,
-					DMA_BIDIRECTIONAL);
-		memcpy(paging->data, page_address(pages),
-		       PAGING_BLOCK_SIZE);
-		dma_sync_single_for_device(fwrt->trans->dev, addr,
-					   PAGING_BLOCK_SIZE,
-					   DMA_BIDIRECTIONAL);
-		(*data) = iwl_fw_error_next_data(*data);
-
-		if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
-			fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx,
-						     fwrt->fw_paging_db[i].fw_offs,
-						     paging->data,
-						     PAGING_BLOCK_SIZE);
-	}
-}
-
-static struct iwl_fw_error_dump_file *
-iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
-		       struct iwl_fw_dump_ptrs *fw_error_dump,
-		       struct iwl_fwrt_dump_data *data)
-{
-	struct iwl_fw_error_dump_file *dump_file;
-	struct iwl_fw_error_dump_data *dump_data;
-	struct iwl_fw_error_dump_info *dump_info;
-	struct iwl_fw_error_dump_smem_cfg *dump_smem_cfg;
-	struct iwl_fw_error_dump_trigger_desc *dump_trig;
-	u32 sram_len, sram_ofs;
-	const struct iwl_fw_dbg_mem_seg_tlv *fw_mem = fwrt->fw->dbg.mem_tlv;
-	struct iwl_fwrt_shared_mem_cfg *mem_cfg = &fwrt->smem_cfg;
-	u32 file_len, fifo_len = 0, prph_len = 0, radio_len = 0;
-	u32 smem_len = fwrt->fw->dbg.n_mem_tlv ? 0 : fwrt->trans->mac_cfg->base->smem_len;
-	u32 sram2_len = fwrt->fw->dbg.n_mem_tlv ?
-				0 : fwrt->trans->cfg->dccm2_len;
-	int i;
-
-	/* SRAM - include stack CCM if driver knows the values for it */
-	if (!fwrt->trans->cfg->dccm_offset ||
-	    !fwrt->trans->cfg->dccm_len) {
-		const struct fw_img *img;
-
-		if (fwrt->cur_fw_img >= IWL_UCODE_TYPE_MAX)
-			return NULL;
-		img = &fwrt->fw->img[fwrt->cur_fw_img];
-		sram_ofs = img->sec[IWL_UCODE_SECTION_DATA].offset;
-		sram_len = img->sec[IWL_UCODE_SECTION_DATA].len;
-	} else {
-		sram_ofs = fwrt->trans->cfg->dccm_offset;
-		sram_len = fwrt->trans->cfg->dccm_len;
-	}
-
-	/* reading RXF/TXF sizes */
-	if (iwl_trans_is_fw_error(fwrt->trans)) {
-		fifo_len = iwl_fw_rxf_len(fwrt, mem_cfg);
-		fifo_len += iwl_fw_txf_len(fwrt, mem_cfg);
-
-		/* Make room for PRPH registers */
-		if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_PRPH))
-			iwl_fw_prph_handler(fwrt, &prph_len,
-					    iwl_fw_get_prph_len);
-
-		if (fwrt->trans->mac_cfg->device_family ==
-		    IWL_DEVICE_FAMILY_7000 &&
-		    iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RADIO_REG))
-			radio_len = sizeof(*dump_data) + RADIO_REG_MAX_READ;
-	}
-
-	file_len = sizeof(*dump_file) + fifo_len + prph_len + radio_len;
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO))
-		file_len += sizeof(*dump_data) + sizeof(*dump_info);
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG))
-		file_len += sizeof(*dump_data) + sizeof(*dump_smem_cfg);
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
-		size_t hdr_len = sizeof(*dump_data) +
-				 sizeof(struct iwl_fw_error_dump_mem);
-
-		/* Dump SRAM only if no mem_tlvs */
-		if (!fwrt->fw->dbg.n_mem_tlv)
-			ADD_LEN(file_len, sram_len, hdr_len);
-
-		/* Make room for all mem types that exist */
-		ADD_LEN(file_len, smem_len, hdr_len);
-		ADD_LEN(file_len, sram2_len, hdr_len);
-
-		for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++)
-			ADD_LEN(file_len, le32_to_cpu(fw_mem[i].len), hdr_len);
-	}
-
-	/* Make room for fw's virtual image pages, if it exists */
-	if (iwl_fw_dbg_is_paging_enabled(fwrt))
-		file_len += fwrt->num_of_paging_blk *
-			(sizeof(*dump_data) +
-			 sizeof(struct iwl_fw_error_dump_paging) +
-			 PAGING_BLOCK_SIZE);
-
-	if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
-		file_len += sizeof(*dump_data) +
-			fwrt->trans->mac_cfg->base->d3_debug_data_length * 2;
-	}
-
-	/* If we only want a monitor dump, reset the file length */
-	if (data->monitor_only) {
-		file_len = sizeof(*dump_file) + sizeof(*dump_data) * 2 +
-			   sizeof(*dump_info) + sizeof(*dump_smem_cfg);
-	}
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
-	    data->desc)
-		file_len += sizeof(*dump_data) + sizeof(*dump_trig) +
-			data->desc->len;
-
-	dump_file = vzalloc(file_len);
-	if (!dump_file)
-		return NULL;
-
-	fw_error_dump->fwrt_ptr = dump_file;
-
-	dump_file->barker = cpu_to_le32(IWL_FW_ERROR_DUMP_BARKER);
-	dump_data = (void *)dump_file->data;
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO)) {
-		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_DEV_FW_INFO);
-		dump_data->len = cpu_to_le32(sizeof(*dump_info));
-		dump_info = (void *)dump_data->data;
-		dump_info->hw_type =
-			cpu_to_le32(CSR_HW_REV_TYPE(fwrt->trans->info.hw_rev));
-		dump_info->hw_step =
-			cpu_to_le32(fwrt->trans->info.hw_rev_step);
-		memcpy(dump_info->fw_human_readable, fwrt->fw->human_readable,
-		       sizeof(dump_info->fw_human_readable));
-		strscpy_pad(dump_info->dev_human_readable,
-			    fwrt->trans->info.name,
-			    sizeof(dump_info->dev_human_readable));
-		strscpy_pad(dump_info->bus_human_readable, fwrt->dev->bus->name,
-			sizeof(dump_info->bus_human_readable));
-		dump_info->num_of_lmacs = fwrt->smem_cfg.num_lmacs;
-		dump_info->lmac_err_id[0] =
-			cpu_to_le32(fwrt->dump.lmac_err_id[0]);
-		if (fwrt->smem_cfg.num_lmacs > 1)
-			dump_info->lmac_err_id[1] =
-				cpu_to_le32(fwrt->dump.lmac_err_id[1]);
-		dump_info->umac_err_id = cpu_to_le32(fwrt->dump.umac_err_id);
-
-		dump_data = iwl_fw_error_next_data(dump_data);
-	}
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG)) {
-		/* Dump shared memory configuration */
-		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_CFG);
-		dump_data->len = cpu_to_le32(sizeof(*dump_smem_cfg));
-		dump_smem_cfg = (void *)dump_data->data;
-		dump_smem_cfg->num_lmacs = cpu_to_le32(mem_cfg->num_lmacs);
-		dump_smem_cfg->num_txfifo_entries =
-			cpu_to_le32(mem_cfg->num_txfifo_entries);
-		for (i = 0; i < MAX_NUM_LMAC; i++) {
-			int j;
-			u32 *txf_size = mem_cfg->lmac[i].txfifo_size;
-
-			for (j = 0; j < TX_FIFO_MAX_NUM; j++)
-				dump_smem_cfg->lmac[i].txfifo_size[j] =
-					cpu_to_le32(txf_size[j]);
-			dump_smem_cfg->lmac[i].rxfifo1_size =
-				cpu_to_le32(mem_cfg->lmac[i].rxfifo1_size);
-		}
-		dump_smem_cfg->rxfifo2_size =
-			cpu_to_le32(mem_cfg->rxfifo2_size);
-		dump_smem_cfg->internal_txfifo_addr =
-			cpu_to_le32(mem_cfg->internal_txfifo_addr);
-		for (i = 0; i < TX_FIFO_INTERNAL_MAX_NUM; i++) {
-			dump_smem_cfg->internal_txfifo_size[i] =
-				cpu_to_le32(mem_cfg->internal_txfifo_size[i]);
-		}
-
-		dump_data = iwl_fw_error_next_data(dump_data);
-	}
-
-	/* We only dump the FIFOs if the FW is in error state */
-	if (fifo_len) {
-		iwl_fw_dump_rxf(fwrt, &dump_data);
-		iwl_fw_dump_txf(fwrt, &dump_data);
-	}
-
-	if (radio_len)
-		iwl_read_radio_regs(fwrt, &dump_data);
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
-	    data->desc) {
-		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_ERROR_INFO);
-		dump_data->len = cpu_to_le32(sizeof(*dump_trig) +
-					     data->desc->len);
-		dump_trig = (void *)dump_data->data;
-		memcpy(dump_trig, &data->desc->trig_desc,
-		       sizeof(*dump_trig) + data->desc->len);
-
-		dump_data = iwl_fw_error_next_data(dump_data);
-	}
-
-	/* In case we only want monitor dump, skip to dump trasport data */
-	if (data->monitor_only)
-		goto out;
-
-	if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
-		const struct iwl_fw_dbg_mem_seg_tlv *fw_dbg_mem =
-			fwrt->fw->dbg.mem_tlv;
-
-		if (!fwrt->fw->dbg.n_mem_tlv)
-			iwl_fw_dump_mem(fwrt, &dump_data, sram_len, sram_ofs,
-					IWL_FW_ERROR_DUMP_MEM_SRAM);
-
-		for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++) {
-			u32 len = le32_to_cpu(fw_dbg_mem[i].len);
-			u32 ofs = le32_to_cpu(fw_dbg_mem[i].ofs);
-
-			iwl_fw_dump_mem(fwrt, &dump_data, len, ofs,
-					le32_to_cpu(fw_dbg_mem[i].data_type));
-		}
-
-		iwl_fw_dump_mem(fwrt, &dump_data, smem_len,
-				fwrt->trans->mac_cfg->base->smem_offset,
-				IWL_FW_ERROR_DUMP_MEM_SMEM);
-
-		iwl_fw_dump_mem(fwrt, &dump_data, sram2_len,
-				fwrt->trans->cfg->dccm2_offset,
-				IWL_FW_ERROR_DUMP_MEM_SRAM);
-	}
-
-	if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
-		u32 addr = fwrt->trans->mac_cfg->base->d3_debug_data_base_addr;
-		size_t data_size = fwrt->trans->mac_cfg->base->d3_debug_data_length;
-
-		dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_D3_DEBUG_DATA);
-		dump_data->len = cpu_to_le32(data_size * 2);
-
-		memcpy(dump_data->data, fwrt->dump.d3_debug_data, data_size);
-
-		kfree(fwrt->dump.d3_debug_data);
-		fwrt->dump.d3_debug_data = NULL;
-
-		iwl_trans_read_mem_bytes(fwrt->trans, addr,
-					 dump_data->data + data_size,
-					 data_size);
-
-		if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
-			fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx, addr,
-						     dump_data->data + data_size,
-						     data_size);
-
-		dump_data = iwl_fw_error_next_data(dump_data);
-	}
-
-	/* Dump fw's virtual image */
-	if (iwl_fw_dbg_is_paging_enabled(fwrt))
-		iwl_dump_paging(fwrt, &dump_data);
-
-	if (prph_len)
-		iwl_fw_prph_handler(fwrt, &dump_data, iwl_dump_prph);
-
-out:
-	dump_file->file_len = cpu_to_le32(file_len);
-	return dump_file;
-}
-
 /**
  * struct iwl_dump_ini_region_data - region data
  * @reg_tlv: region TLV
@@ -1071,7 +111,7 @@ static int iwl_dump_ini_prph_mac_iter_common(struct iwl_fw_runtime *fwrt,
 static int
 iwl_dump_ini_prph_mac_iter(struct iwl_fw_runtime *fwrt,
 			   struct iwl_dump_ini_region_data *reg_data,
-			   void *range_ptr, u32 range_len, int idx)
+			   void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	u32 addr = le32_to_cpu(reg->addrs[idx]) +
@@ -1084,7 +124,7 @@ iwl_dump_ini_prph_mac_iter(struct iwl_fw_runtime *fwrt,
 static int
 iwl_dump_ini_prph_mac_block_iter(struct iwl_fw_runtime *fwrt,
 				 struct iwl_dump_ini_region_data *reg_data,
-				 void *range_ptr, u32 range_len, int idx)
+				 void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_addr_size *pairs = (void *)reg->addrs;
@@ -1132,16 +172,16 @@ static int iwl_dump_ini_prph_phy_iter_common(struct iwl_fw_runtime *fwrt,
 			continue;
 		}
 
-		iwl_write_prph_no_grab(fwrt->trans, indirect_wr_addr,
-				       WMAL_INDRCT_CMD(addr + i));
+		iwl_trans_write_prph(fwrt->trans, indirect_wr_addr,
+				     WMAL_INDRCT_CMD(addr + i));
 
 		if (fwrt->trans->info.hw_rf_id != IWL_CFG_RF_TYPE_JF1 &&
 		    fwrt->trans->info.hw_rf_id != IWL_CFG_RF_TYPE_JF2 &&
 		    fwrt->trans->info.hw_rf_id != IWL_CFG_RF_TYPE_HR1 &&
 		    fwrt->trans->info.hw_rf_id != IWL_CFG_RF_TYPE_HR2) {
 			udelay(2);
-			prph_stts = iwl_read_prph_no_grab(fwrt->trans,
-							  WMAL_MRSPF_STTS);
+			prph_stts = iwl_trans_read_prph(fwrt->trans,
+							WMAL_MRSPF_STTS);
 
 			/* Abort dump if status is 0xA5A5A5A2 or FIFO1 empty */
 			if (prph_stts == WMAL_TIMEOUT_VAL ||
@@ -1149,8 +189,8 @@ static int iwl_dump_ini_prph_phy_iter_common(struct iwl_fw_runtime *fwrt,
 				break;
 		}
 
-		prph_val = iwl_read_prph_no_grab(fwrt->trans,
-						 indirect_rd_addr);
+		prph_val = iwl_trans_read_prph(fwrt->trans,
+					       indirect_rd_addr);
 		*val++ = cpu_to_le32(prph_val);
 	}
 
@@ -1161,7 +201,7 @@ static int iwl_dump_ini_prph_phy_iter_common(struct iwl_fw_runtime *fwrt,
 static int
 iwl_dump_ini_prph_phy_iter(struct iwl_fw_runtime *fwrt,
 			   struct iwl_dump_ini_region_data *reg_data,
-			   void *range_ptr, u32 range_len, int idx)
+			   void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	u32 addr = le32_to_cpu(reg->addrs[idx]);
@@ -1174,7 +214,7 @@ iwl_dump_ini_prph_phy_iter(struct iwl_fw_runtime *fwrt,
 static int
 iwl_dump_ini_prph_phy_block_iter(struct iwl_fw_runtime *fwrt,
 				 struct iwl_dump_ini_region_data *reg_data,
-				 void *range_ptr, u32 range_len, int idx)
+				 void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_addr_size *pairs = (void *)reg->addrs;
@@ -1187,7 +227,7 @@ iwl_dump_ini_prph_phy_block_iter(struct iwl_fw_runtime *fwrt,
 
 static int iwl_dump_ini_csr_iter(struct iwl_fw_runtime *fwrt,
 				 struct iwl_dump_ini_region_data *reg_data,
-				 void *range_ptr, u32 range_len, int idx)
+				 void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1206,7 +246,7 @@ static int iwl_dump_ini_csr_iter(struct iwl_fw_runtime *fwrt,
 
 static int iwl_dump_ini_config_iter(struct iwl_fw_runtime *fwrt,
 				    struct iwl_dump_ini_region_data *reg_data,
-				    void *range_ptr, u32 range_len, int idx)
+				    void *range_ptr, int idx)
 {
 	struct iwl_trans *trans = fwrt->trans;
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
@@ -1234,7 +274,7 @@ static int iwl_dump_ini_config_iter(struct iwl_fw_runtime *fwrt,
 
 static int iwl_dump_ini_dev_mem_iter(struct iwl_fw_runtime *fwrt,
 				     struct iwl_dump_ini_region_data *reg_data,
-				     void *range_ptr, u32 range_len, int idx)
+				     void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1256,7 +296,7 @@ static int iwl_dump_ini_dev_mem_iter(struct iwl_fw_runtime *fwrt,
 }
 
 static int _iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
-				     void *range_ptr, u32 range_len, int idx)
+				     void *range_ptr, int idx)
 {
 	struct page *page = fwrt->fw_paging_db[idx].fw_paging_block;
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1276,7 +316,7 @@ static int _iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
 
 static int iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
 				    struct iwl_dump_ini_region_data *reg_data,
-				    void *range_ptr, u32 range_len, int idx)
+				    void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_error_dump_range *range;
 	u32 page_size;
@@ -1285,7 +325,7 @@ static int iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
 	idx++;
 
 	if (!fwrt->trans->mac_cfg->gen2)
-		return _iwl_dump_ini_paging_iter(fwrt, range_ptr, range_len, idx);
+		return _iwl_dump_ini_paging_iter(fwrt, range_ptr, idx);
 
 	range = range_ptr;
 	page_size = fwrt->trans->init_dram.paging[idx].size;
@@ -1301,7 +341,7 @@ static int iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
 static int
 iwl_dump_ini_mon_dram_iter(struct iwl_fw_runtime *fwrt,
 			   struct iwl_dump_ini_region_data *reg_data,
-			   void *range_ptr, u32 range_len, int idx)
+			   void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1320,7 +360,7 @@ iwl_dump_ini_mon_dram_iter(struct iwl_fw_runtime *fwrt,
 
 static int iwl_dump_ini_mon_smem_iter(struct iwl_fw_runtime *fwrt,
 				      struct iwl_dump_ini_region_data *reg_data,
-				      void *range_ptr, u32 range_len, int idx)
+				      void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1388,7 +428,7 @@ static bool iwl_ini_txf_iter(struct iwl_fw_runtime *fwrt,
 
 static int iwl_dump_ini_txf_iter(struct iwl_fw_runtime *fwrt,
 				 struct iwl_dump_ini_region_data *reg_data,
-				 void *range_ptr, u32 range_len, int idx)
+				 void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1410,7 +450,7 @@ static int iwl_dump_ini_txf_iter(struct iwl_fw_runtime *fwrt,
 	range->fifo_hdr.num_of_registers = cpu_to_le32(registers_num);
 	range->range_data_size = cpu_to_le32(iter->fifo_size + registers_size);
 
-	iwl_write_prph_no_grab(fwrt->trans, TXF_LARC_NUM + offs, iter->fifo);
+	iwl_trans_write_prph(fwrt->trans, TXF_LARC_NUM + offs, iter->fifo);
 
 	/*
 	 * read txf registers. for each register, write to the dump the
@@ -1420,8 +460,8 @@ static int iwl_dump_ini_txf_iter(struct iwl_fw_runtime *fwrt,
 		addr = le32_to_cpu(reg->addrs[i]) + offs;
 
 		reg_dump->addr = cpu_to_le32(addr);
-		reg_dump->data = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans,
-								   addr));
+		reg_dump->data = cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+								 addr));
 
 		reg_dump++;
 	}
@@ -1432,17 +472,17 @@ static int iwl_dump_ini_txf_iter(struct iwl_fw_runtime *fwrt,
 	}
 
 	/* Set the TXF_READ_MODIFY_ADDR to TXF_WR_PTR */
-	iwl_write_prph_no_grab(fwrt->trans, TXF_READ_MODIFY_ADDR + offs,
-			       TXF_WR_PTR + offs);
+	iwl_trans_write_prph(fwrt->trans, TXF_READ_MODIFY_ADDR + offs,
+			     TXF_WR_PTR + offs);
 
 	/* Dummy-read to advance the read pointer to the head */
-	iwl_read_prph_no_grab(fwrt->trans, TXF_READ_MODIFY_DATA + offs);
+	iwl_trans_read_prph(fwrt->trans, TXF_READ_MODIFY_DATA + offs);
 
 	/* Read FIFO */
 	addr = TXF_READ_MODIFY_DATA + offs;
 	data = (void *)reg_dump;
 	for (i = 0; i < iter->fifo_size; i += sizeof(*data))
-		*data++ = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans, addr));
+		*data++ = cpu_to_le32(iwl_trans_read_prph(fwrt->trans, addr));
 
 	if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_txf)
 		fwrt->sanitize_ops->frob_txf(fwrt->sanitize_ctx,
@@ -1457,7 +497,7 @@ static int iwl_dump_ini_txf_iter(struct iwl_fw_runtime *fwrt,
 static int
 iwl_dump_ini_prph_snps_dphyip_iter(struct iwl_fw_runtime *fwrt,
 				   struct iwl_dump_ini_region_data *reg_data,
-				   void *range_ptr, u32 range_len, int idx)
+				   void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1487,12 +527,12 @@ iwl_dump_ini_prph_snps_dphyip_iter(struct iwl_fw_runtime *fwrt,
 			continue;
 		}
 
-		iwl_write_prph_no_grab(fwrt->trans, indirect_rd_wr_addr,
-				       addr + i);
+		iwl_trans_write_prph(fwrt->trans, indirect_rd_wr_addr,
+				     addr + i);
 		/* wait a bit for value to be ready in register */
 		udelay(1);
-		prph_val = iwl_read_prph_no_grab(fwrt->trans,
-						 indirect_rd_wr_addr);
+		prph_val = iwl_trans_read_prph(fwrt->trans,
+					       indirect_rd_wr_addr);
 		*val++ = cpu_to_le32((prph_val & DPHYIP_INDIRECT_RD_MSK) >>
 				     DPHYIP_INDIRECT_RD_SHIFT);
 	}
@@ -1570,7 +610,7 @@ static void iwl_ini_get_rxf_data(struct iwl_fw_runtime *fwrt,
 
 static int iwl_dump_ini_rxf_iter(struct iwl_fw_runtime *fwrt,
 				 struct iwl_dump_ini_region_data *reg_data,
-				 void *range_ptr, u32 range_len, int idx)
+				 void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1601,8 +641,8 @@ static int iwl_dump_ini_rxf_iter(struct iwl_fw_runtime *fwrt,
 		addr = le32_to_cpu(reg->addrs[i]) + offs;
 
 		reg_dump->addr = cpu_to_le32(addr);
-		reg_dump->data = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans,
-								   addr));
+		reg_dump->data = cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
+								 addr));
 
 		reg_dump++;
 	}
@@ -1615,18 +655,17 @@ static int iwl_dump_ini_rxf_iter(struct iwl_fw_runtime *fwrt,
 	offs = rxf_data.offset;
 
 	/* Lock fence */
-	iwl_write_prph_no_grab(fwrt->trans, RXF_SET_FENCE_MODE + offs, 0x1);
+	iwl_trans_write_prph(fwrt->trans, RXF_SET_FENCE_MODE + offs, 0x1);
 	/* Set fence pointer to the same place like WR pointer */
-	iwl_write_prph_no_grab(fwrt->trans, RXF_LD_WR2FENCE + offs, 0x1);
+	iwl_trans_write_prph(fwrt->trans, RXF_LD_WR2FENCE + offs, 0x1);
 	/* Set fence offset */
-	iwl_write_prph_no_grab(fwrt->trans, RXF_LD_FENCE_OFFSET_ADDR + offs,
-			       0x0);
+	iwl_trans_write_prph(fwrt->trans, RXF_LD_FENCE_OFFSET_ADDR + offs, 0x0);
 
 	/* Read FIFO */
 	addr =  RXF_FIFO_RD_FENCE_INC + offs;
 	data = (void *)reg_dump;
 	for (i = 0; i < rxf_data.size; i += sizeof(*data))
-		*data++ = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans, addr));
+		*data++ = cpu_to_le32(iwl_trans_read_prph(fwrt->trans, addr));
 
 out:
 	iwl_trans_release_nic_access(fwrt->trans);
@@ -1637,7 +676,7 @@ static int iwl_dump_ini_rxf_iter(struct iwl_fw_runtime *fwrt,
 static int
 iwl_dump_ini_err_table_iter(struct iwl_fw_runtime *fwrt,
 			    struct iwl_dump_ini_region_data *reg_data,
-			    void *range_ptr, u32 range_len, int idx)
+			    void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_region_err_table *err_table = &reg->err_table;
@@ -1656,7 +695,7 @@ iwl_dump_ini_err_table_iter(struct iwl_fw_runtime *fwrt,
 static int
 iwl_dump_ini_special_mem_iter(struct iwl_fw_runtime *fwrt,
 			      struct iwl_dump_ini_region_data *reg_data,
-			      void *range_ptr, u32 range_len, int idx)
+			      void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_region_special_device_memory *special_mem =
@@ -1677,7 +716,7 @@ iwl_dump_ini_special_mem_iter(struct iwl_fw_runtime *fwrt,
 static int
 iwl_dump_ini_dbgi_sram_iter(struct iwl_fw_runtime *fwrt,
 			    struct iwl_dump_ini_region_data *reg_data,
-			    void *range_ptr, u32 range_len, int idx)
+			    void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1690,9 +729,11 @@ iwl_dump_ini_dbgi_sram_iter(struct iwl_fw_runtime *fwrt,
 
 	range->range_data_size = reg->dev_addr.size;
 	for (i = 0; i < (le32_to_cpu(reg->dev_addr.size) / 4); i++) {
-		prph_data = iwl_read_prph_no_grab(fwrt->trans, (i % 2) ?
-					  DBGI_SRAM_TARGET_ACCESS_RDATA_MSB :
-					  DBGI_SRAM_TARGET_ACCESS_RDATA_LSB);
+		prph_data =
+			iwl_trans_read_prph(fwrt->trans,
+					    (i % 2) ?
+						DBGI_SRAM_TARGET_ACCESS_RDATA_MSB :
+						DBGI_SRAM_TARGET_ACCESS_RDATA_LSB);
 		if (iwl_trans_is_hw_error_value(prph_data)) {
 			iwl_trans_release_nic_access(fwrt->trans);
 			return -EBUSY;
@@ -1705,7 +746,7 @@ iwl_dump_ini_dbgi_sram_iter(struct iwl_fw_runtime *fwrt,
 
 static int iwl_dump_ini_fw_pkt_iter(struct iwl_fw_runtime *fwrt,
 				    struct iwl_dump_ini_region_data *reg_data,
-				    void *range_ptr, u32 range_len, int idx)
+				    void *range_ptr, int idx)
 {
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
 	struct iwl_rx_packet *pkt = reg_data->dump_data->fw_pkt;
@@ -1726,7 +767,7 @@ static int iwl_dump_ini_fw_pkt_iter(struct iwl_fw_runtime *fwrt,
 
 static int iwl_dump_ini_imr_iter(struct iwl_fw_runtime *fwrt,
 				 struct iwl_dump_ini_region_data *reg_data,
-				 void *range_ptr, u32 range_len, int idx)
+				 void *range_ptr, int idx)
 {
 	/* read the IMR memory and DMA it to SRAM */
 	struct iwl_fw_ini_error_dump_range *range = range_ptr;
@@ -1791,7 +832,7 @@ static __le32 iwl_get_mon_reg(struct iwl_fw_runtime *fwrt, u32 alloc_id,
 	if (!reg_info || !reg_info->addr || !reg_info->mask)
 		return 0;
 
-	val = iwl_read_prph_no_grab(fwrt->trans, reg_info->addr + offs);
+	val = iwl_trans_read_prph(fwrt->trans, reg_info->addr + offs);
 
 	return cpu_to_le32(mask_apply_and_normalize(val, reg_info->mask));
 }
@@ -2236,24 +1277,22 @@ struct iwl_dump_ini_mem_ops {
 			      void *data, u32 data_len);
 	int (*fill_range)(struct iwl_fw_runtime *fwrt,
 			  struct iwl_dump_ini_region_data *reg_data,
-			  void *range, u32 range_len, int idx);
+			  void *range, int idx);
 };
 
-/**
- * iwl_dump_ini_mem - dump memory region
- *
- * @fwrt: fw runtime struct
- * @list: list to add the dump tlv to
- * @reg_data: memory region
- * @ops: memory dump operations
- *
- * Creates a dump tlv and copy a memory region into it.
- *
- * Returns: the size of the current dump tlv or 0 if failed
- */
-static u32 iwl_dump_ini_mem(struct iwl_fw_runtime *fwrt, struct list_head *list,
-			    struct iwl_dump_ini_region_data *reg_data,
-			    const struct iwl_dump_ini_mem_ops *ops)
+struct iwl_fw_ini_dump_entry {
+	const struct iwl_dump_ini_mem_ops *ops;
+	struct iwl_dump_ini_region_data reg_data;
+	struct list_head list;
+	u32 region_dump_policy;
+	u32 size;
+	u8 data[];
+} __packed;
+
+static void iwl_dump_ini_mem_prep(struct iwl_fw_runtime *fwrt,
+				  struct list_head *list,
+				  struct iwl_dump_ini_region_data *reg_data,
+				  const struct iwl_dump_ini_mem_ops *ops)
 {
 	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
 	struct iwl_fw_ini_dump_entry *entry;
@@ -2261,58 +1300,59 @@ static u32 iwl_dump_ini_mem(struct iwl_fw_runtime *fwrt, struct list_head *list,
 	struct iwl_fw_ini_error_dump_header *header;
 	u32 type = reg->type;
 	u32 id = le32_get_bits(reg->id, IWL_FW_INI_REGION_ID_MASK);
-	u32 num_of_ranges, i, size;
-	u8 *range;
-	u32 free_size;
-	u64 header_size;
+	u32 num_of_ranges, size;
 	u32 dump_policy = IWL_FW_INI_DUMP_VERBOSE;
+	u32 dp;
 
 	IWL_DEBUG_FW(fwrt, "WRT: Collecting region: dump type=%d, id=%d, type=%d\n",
 		     dump_policy, id, type);
 
 	if (le32_to_cpu(reg->hdr.version) >= 2) {
-		u32 dp = le32_get_bits(reg->id,
-				       IWL_FW_INI_REGION_DUMP_POLICY_MASK);
+		dp = le32_get_bits(reg->id, IWL_FW_INI_REGION_DUMP_POLICY_MASK);
 
 		if (dump_policy == IWL_FW_INI_DUMP_VERBOSE &&
 		    !(dp & IWL_FW_INI_DEBUG_DUMP_POLICY_NO_LIMIT)) {
 			IWL_DEBUG_FW(fwrt,
 				     "WRT: no dump - type %d and policy mismatch=%d\n",
 				     dump_policy, dp);
-			return 0;
+			return;
 		} else if (dump_policy == IWL_FW_INI_DUMP_MEDIUM &&
 			   !(dp & IWL_FW_IWL_DEBUG_DUMP_POLICY_MAX_LIMIT_5MB)) {
 			IWL_DEBUG_FW(fwrt,
 				     "WRT: no dump - type %d and policy mismatch=%d\n",
 				     dump_policy, dp);
-			return 0;
+			return;
 		} else if (dump_policy == IWL_FW_INI_DUMP_BRIEF &&
 			   !(dp & IWL_FW_INI_DEBUG_DUMP_POLICY_MAX_LIMIT_600KB)) {
 			IWL_DEBUG_FW(fwrt,
 				     "WRT: no dump - type %d and policy mismatch=%d\n",
 				     dump_policy, dp);
-			return 0;
+			return;
 		}
+	} else {
+		dp = 0;
 	}
 
 	if (!ops->get_num_of_ranges || !ops->get_size || !ops->fill_mem_hdr ||
 	    !ops->fill_range) {
 		IWL_DEBUG_FW(fwrt, "WRT: no ops for collecting data\n");
-		return 0;
+		return;
 	}
 
 	size = ops->get_size(fwrt, reg_data);
 
 	if (size < sizeof(*header)) {
 		IWL_DEBUG_FW(fwrt, "WRT: size didn't include space for header\n");
-		return 0;
+		return;
 	}
 
 	entry = vzalloc(sizeof(*entry) + sizeof(*tlv) + size);
 	if (!entry)
-		return 0;
+		return;
 
 	entry->size = sizeof(*tlv) + size;
+	entry->reg_data = *reg_data;
+	entry->region_dump_policy = dp;
 
 	tlv = (void *)entry->data;
 	tlv->type = reg->type;
@@ -2329,7 +1369,29 @@ static u32 iwl_dump_ini_mem(struct iwl_fw_runtime *fwrt, struct list_head *list,
 	header->name_len = cpu_to_le32(IWL_FW_INI_MAX_NAME);
 	memcpy(header->name, reg->name, IWL_FW_INI_MAX_NAME);
 
-	free_size = size;
+	entry->ops = ops;
+	list_add_tail(&entry->list, list);
+}
+
+static u32 iwl_dump_ini_mem(struct iwl_fw_runtime *fwrt,
+			    struct iwl_fw_ini_dump_entry *entry)
+{
+	struct iwl_dump_ini_region_data *reg_data = &entry->reg_data;
+	struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
+	const struct iwl_dump_ini_mem_ops *ops = entry->ops;
+	struct iwl_fw_ini_error_dump_data *tlv;
+	struct iwl_fw_ini_error_dump_header *header;
+	u32 type = reg->type;
+	u32 id = le32_get_bits(reg->id, IWL_FW_INI_REGION_ID_MASK);
+	u32 i;
+	u8 *range;
+	u32 free_size;
+	u64 header_size;
+
+	tlv = (void *)entry->data;
+	header = (void *)tlv->data;
+
+	free_size = entry->size - sizeof(*tlv);
 	range = ops->fill_mem_hdr(fwrt, reg_data, header, free_size);
 	if (!range) {
 		IWL_ERR(fwrt,
@@ -2350,9 +1412,8 @@ static u32 iwl_dump_ini_mem(struct iwl_fw_runtime *fwrt, struct list_head *list,
 
 	free_size -= header_size;
 
-	for (i = 0; i < num_of_ranges; i++) {
-		int range_size = ops->fill_range(fwrt, reg_data, range,
-						 free_size, i);
+	for (i = 0; i < le32_to_cpu(header->num_of_ranges); i++) {
+		int range_size = ops->fill_range(fwrt, reg_data, range, i);
 
 		if (range_size < 0) {
 			IWL_ERR(fwrt,
@@ -2372,11 +1433,10 @@ static u32 iwl_dump_ini_mem(struct iwl_fw_runtime *fwrt, struct list_head *list,
 		range = range + range_size;
 	}
 
-	list_add_tail(&entry->list, list);
-
 	return entry->size;
 
 out_err:
+	list_del(&entry->list);
 	vfree(entry);
 
 	return 0;
@@ -2605,22 +1665,19 @@ static bool iwl_dump_due_to_error(enum iwl_fw_ini_time_point tp_id)
 	       tp_id == IWL_FW_INI_TIME_POINT_FW_HW_ERROR;
 }
 
-static u32
-iwl_dump_ini_dump_regions(struct iwl_fw_runtime *fwrt,
-			  struct iwl_fwrt_dump_data *dump_data,
-			  struct list_head *list,
-			  enum iwl_fw_ini_time_point tp_id,
-			  u64 regions_mask,
-			  struct iwl_dump_ini_region_data *imr_reg_data,
-			  enum iwl_dump_ini_region_selector which)
+static void
+iwl_dump_ini_dump_regions_prep(struct iwl_fw_runtime *fwrt,
+			       struct iwl_fwrt_dump_data *dump_data,
+			       struct list_head *list,
+			       enum iwl_fw_ini_time_point tp_id,
+			       u64 regions_mask,
+			       struct iwl_ucode_tlv **imr_tlv)
 {
-	u32 size = 0;
-
 	for (int i = 0; i < ARRAY_SIZE(fwrt->trans->dbg.active_regions); i++) {
 		struct iwl_dump_ini_region_data reg_data = {
 			.dump_data = dump_data,
 		};
-		u32 reg_type, dp;
+		u32 reg_type;
 		struct iwl_fw_ini_region_tlv *reg;
 
 		if (!(BIT_ULL(i) & regions_mask))
@@ -2638,8 +1695,6 @@ iwl_dump_ini_dump_regions(struct iwl_fw_runtime *fwrt,
 		if (reg_type >= ARRAY_SIZE(iwl_dump_ini_region_ops))
 			continue;
 
-		dp = le32_get_bits(reg->id, IWL_FW_INI_REGION_DUMP_POLICY_MASK);
-
 		if ((reg_type == IWL_FW_INI_REGION_PERIPHERY_PHY ||
 		     reg_type == IWL_FW_INI_REGION_PERIPHERY_PHY_RANGE ||
 		     reg_type == IWL_FW_INI_REGION_PERIPHERY_SNPS_DPHYIP) &&
@@ -2650,19 +1705,6 @@ iwl_dump_ini_dump_regions(struct iwl_fw_runtime *fwrt,
 			continue;
 		}
 
-		switch (which) {
-		case IWL_INI_DUMP_ALL_REGIONS:
-			break;
-		case IWL_INI_DUMP_EARLY_REGIONS:
-			if (!(dp & IWL_FW_IWL_DEBUG_DUMP_POLICY_BEFORE_RESET))
-				continue;
-			break;
-		case IWL_INI_DUMP_LATE_REGIONS:
-			if (dp & IWL_FW_IWL_DEBUG_DUMP_POLICY_BEFORE_RESET)
-				continue;
-			break;
-		}
-
 		/*
 		 * DRAM_IMR can be collected only for FW/HW error timepoint
 		 * when fw is not alive. In addition, it must be collected
@@ -2671,19 +1713,45 @@ iwl_dump_ini_dump_regions(struct iwl_fw_runtime *fwrt,
 		 */
 		if (reg_type == IWL_FW_INI_REGION_DRAM_IMR) {
 			if (iwl_dump_due_to_error(tp_id))
-				imr_reg_data->reg_tlv =
-					fwrt->trans->dbg.active_regions[i];
+				*imr_tlv = fwrt->trans->dbg.active_regions[i];
 			else
 				IWL_INFO(fwrt,
 					 "WRT: trying to collect DRAM_IMR at time point: %d, skipping\n",
 					 tp_id);
-		/* continue to next region */
+			/* continue to next region */
 			continue;
 		}
 
+		iwl_dump_ini_mem_prep(fwrt, list, &reg_data,
+				      &iwl_dump_ini_region_ops[reg_type]);
+	}
+}
+
+static u32
+iwl_dump_ini_dump_entries(struct iwl_fw_runtime *fwrt,
+			  struct list_head *list,
+			  enum iwl_dump_ini_region_selector which)
+{
+	struct iwl_fw_ini_dump_entry *entry, *tmp;
+	u32 size = 0;
+
+	list_for_each_entry_safe(entry, tmp, list, list) {
+		u32 dp = entry->region_dump_policy;
+
+		switch (which) {
+		case IWL_INI_DUMP_ALL_REGIONS:
+			break;
+		case IWL_INI_DUMP_EARLY_REGIONS:
+			if (!(dp & IWL_FW_IWL_DEBUG_DUMP_POLICY_BEFORE_RESET))
+				continue;
+			break;
+		case IWL_INI_DUMP_LATE_REGIONS:
+			if (dp & IWL_FW_IWL_DEBUG_DUMP_POLICY_BEFORE_RESET)
+				continue;
+			break;
+		}
 
-		size += iwl_dump_ini_mem(fwrt, list, &reg_data,
-					 &iwl_dump_ini_region_ops[reg_type]);
+		size += iwl_dump_ini_mem(fwrt, entry);
 	}
 
 	return size;
@@ -2706,32 +1774,32 @@ static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
 	BUILD_BUG_ON((sizeof(trigger->regions_mask) * BITS_PER_BYTE) <
 		     ARRAY_SIZE(fwrt->trans->dbg.active_regions));
 
+	iwl_dump_ini_dump_regions_prep(fwrt, dump_data, list, tp_id,
+				       regions_mask, &imr_reg_data.reg_tlv);
+
+	/* append DRAM_IMR region to be collected last */
+	if (imr_reg_data.reg_tlv)
+		iwl_dump_ini_mem_prep(fwrt, list, &imr_reg_data,
+				      &iwl_dump_ini_region_ops[IWL_FW_INI_REGION_DRAM_IMR]);
+
 	if (trigger->apply_policy &
 			cpu_to_le32(IWL_FW_INI_APPLY_POLICY_SPLIT_DUMP_RESET)) {
-		size += iwl_dump_ini_dump_regions(fwrt, dump_data, list, tp_id,
-						  regions_mask, &imr_reg_data,
+		size += iwl_dump_ini_dump_entries(fwrt, list,
 						  IWL_INI_DUMP_EARLY_REGIONS);
 		iwl_trans_pcie_fw_reset_handshake(fwrt->trans);
-		size += iwl_dump_ini_dump_regions(fwrt, dump_data, list, tp_id,
-						  regions_mask, &imr_reg_data,
+		size += iwl_dump_ini_dump_entries(fwrt, list,
 						  IWL_INI_DUMP_LATE_REGIONS);
 	} else {
 		if (fw_has_capa(&fwrt->fw->ucode_capa,
 				IWL_UCODE_TLV_CAPA_RESET_DURING_ASSERT) &&
 		    iwl_dump_due_to_error(tp_id))
 			iwl_trans_pcie_fw_reset_handshake(fwrt->trans);
-		size += iwl_dump_ini_dump_regions(fwrt, dump_data, list, tp_id,
-						  regions_mask, &imr_reg_data,
+		size += iwl_dump_ini_dump_entries(fwrt, list,
 						  IWL_INI_DUMP_ALL_REGIONS);
 	}
-	/* collect DRAM_IMR region in the last */
-	if (imr_reg_data.reg_tlv)
-		size += iwl_dump_ini_mem(fwrt, list, &imr_reg_data,
-					 &iwl_dump_ini_region_ops[IWL_FW_INI_REGION_DRAM_IMR]);
 
-	if (size) {
+	if (size)
 		size += iwl_dump_ini_info(fwrt, trigger, list);
-	}
 
 	return size;
 }
@@ -2797,52 +1865,6 @@ static inline void iwl_fw_free_dump_desc(struct iwl_fw_runtime *fwrt,
 	fwrt->dump.umac_err_id = 0;
 }
 
-static void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt,
-			      struct iwl_fwrt_dump_data *dump_data)
-{
-	struct iwl_fw_dump_ptrs fw_error_dump = {};
-	struct iwl_fw_error_dump_file *dump_file;
-	struct scatterlist *sg_dump_data;
-	u32 file_len;
-	u32 dump_mask = fwrt->fw->dbg.dump_mask;
-
-	dump_file = iwl_fw_error_dump_file(fwrt, &fw_error_dump, dump_data);
-	if (!dump_file)
-		return;
-
-	if (dump_data->monitor_only)
-		dump_mask &= BIT(IWL_FW_ERROR_DUMP_FW_MONITOR);
-
-	fw_error_dump.trans_ptr = iwl_trans_dump_data(fwrt->trans, dump_mask,
-						      fwrt->sanitize_ops,
-						      fwrt->sanitize_ctx);
-	file_len = le32_to_cpu(dump_file->file_len);
-	fw_error_dump.fwrt_len = file_len;
-
-	if (fw_error_dump.trans_ptr) {
-		file_len += fw_error_dump.trans_ptr->len;
-		dump_file->file_len = cpu_to_le32(file_len);
-	}
-
-	sg_dump_data = alloc_sgtable(file_len);
-	if (sg_dump_data) {
-		sg_pcopy_from_buffer(sg_dump_data,
-				     sg_nents(sg_dump_data),
-				     fw_error_dump.fwrt_ptr,
-				     fw_error_dump.fwrt_len, 0);
-		if (fw_error_dump.trans_ptr)
-			sg_pcopy_from_buffer(sg_dump_data,
-					     sg_nents(sg_dump_data),
-					     fw_error_dump.trans_ptr->data,
-					     fw_error_dump.trans_ptr->len,
-					     fw_error_dump.fwrt_len);
-		dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len,
-			       GFP_KERNEL);
-	}
-	vfree(fw_error_dump.fwrt_ptr);
-	vfree(fw_error_dump.trans_ptr);
-}
-
 static void iwl_dump_ini_list_free(struct list_head *list)
 {
 	while (!list_empty(list)) {
@@ -2871,7 +1893,7 @@ static void iwl_fw_error_ini_dump(struct iwl_fw_runtime *fwrt,
 	if (!file_len)
 		return;
 
-	sg_dump_data = alloc_sgtable(file_len);
+	sg_dump_data = iwl_fw_dbg_alloc_sgtable(file_len);
 	if (sg_dump_data) {
 		struct iwl_fw_ini_dump_entry *entry;
 		int sg_entries = sg_nents(sg_dump_data);
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/dbg.h b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
index 8034c9ecba693..fc962a3205833 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/dbg.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2019, 2021-2025 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2019, 2021-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -38,6 +38,11 @@ struct iwl_fw_dbg_params {
 	u32 out_ctrl;
 };
 
+/* old-style dump entry point */
+void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt,
+		       struct iwl_fwrt_dump_data *dump_data);
+struct scatterlist *iwl_fw_dbg_alloc_sgtable(ssize_t size);
+
 extern const struct iwl_fw_dump_desc iwl_dump_desc_assert;
 
 int iwl_fw_dbg_collect_desc(struct iwl_fw_runtime *fwrt,
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
index ddee7c2deb36d..f06978d5b5ee3 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/debugfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2024, 2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -275,16 +275,19 @@ static ssize_t iwl_dbgfs_send_hcmd_write(struct iwl_fw_runtime *fwrt, char *buf,
 		goto out;
 	}
 
+	/* ignore this flag, we cannot use the response */
+	hcmd.flags &= ~CMD_WANT_SKB;
+	/* reject flags other than async, they cannot be used this way */
+	if (hcmd.flags & ~CMD_ASYNC) {
+		ret = -EINVAL;
+		goto out;
+	}
+
 	if (fwrt->ops && fwrt->ops->send_hcmd)
 		ret = fwrt->ops->send_hcmd(fwrt->ops_ctx, &hcmd);
 	else
 		ret = -EPERM;
 
-	if (ret < 0)
-		goto out;
-
-	if (hcmd.flags & CMD_WANT_SKB)
-		iwl_free_resp(&hcmd);
 out:
 	kfree(data);
 	return ret ?: count;
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
index 525a82030daa4..07f1240df8667 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/error-dump.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2014, 2018-2026 Intel Corporation
  * Copyright (C) 2014-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -265,18 +265,6 @@ struct iwl_fw_ini_error_dump_data {
 	__u8 data[];
 } __packed;
 
-/**
- * struct iwl_fw_ini_dump_entry - dump entry descriptor
- * @list: list of dump entries
- * @size: size of the data
- * @data: entry data
- */
-struct iwl_fw_ini_dump_entry {
-	struct list_head list;
-	u32 size;
-	u8 data[];
-} __packed;
-
 /**
  * struct iwl_fw_ini_dump_file_hdr - header of dump file
  * @barker: must be %IWL_FW_INI_ERROR_DUMP_BARKER
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/file.h b/drivers/net/wireless/intel/iwlwifi/fw/file.h
index f7a6f21267e9d..197c88c25f72d 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/file.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/file.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2008-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2008-2014, 2018-2024, 2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -111,6 +111,7 @@ enum iwl_ucode_tlv_type {
 	IWL_UCODE_TLV_FW_NUM_STATIONS		= IWL_UCODE_TLV_CONST_BASE + 0,
 	IWL_UCODE_TLV_FW_NUM_LINKS		= IWL_UCODE_TLV_CONST_BASE + 1,
 	IWL_UCODE_TLV_FW_NUM_BEACONS		= IWL_UCODE_TLV_CONST_BASE + 2,
+	IWL_UCODE_TLV_FW_NUM_MCAST_KEY_ENTRIES	= IWL_UCODE_TLV_CONST_BASE + 3,
 
 	IWL_UCODE_TLV_TYPE_DEBUG_INFO		= IWL_UCODE_TLV_DEBUG_BASE + 0,
 	IWL_UCODE_TLV_TYPE_BUFFER_ALLOCATION	= IWL_UCODE_TLV_DEBUG_BASE + 1,
@@ -1063,10 +1064,15 @@ struct iwl_fw_dump_exclude {
 	__le32 addr, size;
 };
 
-struct iwl_fw_fseq_bin_version {
+struct iwl_fw_fseq_bin_version_v1 {
 	__le32 major, minor;
 }; /* FW_TLV_FSEQ_BIN_VERSION_S */
 
+struct iwl_fw_fseq_bin_version {
+	/* rf_id is currently unused and always zero */
+	__le32 mac_id, rf_id, major, minor;
+}; /* FW_TLV_FSEQ_BIN_VERSION_S */
+
 static inline size_t _iwl_tlv_array_len(const struct iwl_ucode_tlv *tlv,
 					size_t fixed_size, size_t var_size)
 {
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/img.h b/drivers/net/wireless/intel/iwlwifi/fw/img.h
index 94113d1db8e13..75b1344f6cbe0 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/img.h
+++ b/drivers/net/wireless/intel/iwlwifi/fw/img.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2024 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2024, 2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016 Intel Deutschland GmbH
  */
@@ -53,6 +53,7 @@ struct iwl_ucode_capabilities {
 	u32 num_stations;
 	u32 num_links;
 	u32 num_beacons;
+	u32 num_mcast_key_entries;
 	DECLARE_BITMAP(_api, NUM_IWL_UCODE_TLV_API);
 	DECLARE_BITMAP(_capa, NUM_IWL_UCODE_TLV_CAPA);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
index 55128caac7ed9..8d9ff36e30f5e 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/regulatory.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2023, 2025 Intel Corporation
+ * Copyright (C) 2023, 2025-2026 Intel Corporation
  */
 #include <linux/dmi.h>
 #include "iwl-drv.h"
@@ -112,6 +112,11 @@ static const struct dmi_system_id dmi_ppag_approved_list[] = {
 			DMI_MATCH(DMI_SYS_VENDOR, "WIKO"),
 		},
 	},
+	{ .ident = "XIAOMI",
+	  .matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "XIAOMI"),
+		},
+	},
 	{}
 };
 
diff --git a/drivers/net/wireless/intel/iwlwifi/fw/rs.c b/drivers/net/wireless/intel/iwlwifi/fw/rs.c
index 746f2acffb8f5..2aa300b261581 100644
--- a/drivers/net/wireless/intel/iwlwifi/fw/rs.c
+++ b/drivers/net/wireless/intel/iwlwifi/fw/rs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2021-2022, 2025 Intel Corporation
+ * Copyright (C) 2021-2022, 2025-2026 Intel Corporation
  */
 
 #include <net/mac80211.h>
@@ -124,6 +124,9 @@ int rs_pretty_print_rate(char *buf, int bufsz, const u32 rate)
 	case RATE_MCS_MOD_TYPE_EHT:
 		type = "EHT";
 		break;
+	case RATE_MCS_MOD_TYPE_UHR:
+		type = "UHR";
+		break;
 	default:
 		type = "Unknown"; /* shouldn't happen */
 	}
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-config.h b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
index 5f40cd15e27f7..30d5ec31b9c35 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-config.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-config.h
@@ -416,6 +416,7 @@ struct iwl_mac_cfg {
  * @nvm_type: see &enum iwl_nvm_type
  * @uhb_supported: ultra high band channels supported
  * @eht_supported: EHT supported
+ * @uhr_supported: UHR supported
  * @num_rbds: number of receive buffer descriptors to use
  *	(only used for multi-queue capable devices)
  *
@@ -449,7 +450,8 @@ struct iwl_rf_cfg {
 	    lp_xtal_workaround:1,
 	    vht_mu_mimo_supported:1,
 	    uhb_supported:1,
-	    eht_supported:1;
+	    eht_supported:1,
+	    uhr_supported:1;
 	u8 valid_tx_ant;
 	u8 valid_rx_ant;
 	u8 non_shared_ant;
@@ -674,6 +676,7 @@ extern const char iwl_killer_be1750w_name[];
 extern const char iwl_killer_be1750x_name[];
 extern const char iwl_killer_be1790s_name[];
 extern const char iwl_killer_be1790i_name[];
+extern const char iwl_killer_be1730x_name[];
 extern const char iwl_be201_name[];
 extern const char iwl_be200_name[];
 extern const char iwl_be202_name[];
@@ -681,6 +684,7 @@ extern const char iwl_be401_name[];
 extern const char iwl_be213_name[];
 extern const char iwl_killer_be1775s_name[];
 extern const char iwl_killer_be1775i_name[];
+extern const char iwl_killer_be1735x_name[];
 extern const char iwl_be211_name[];
 extern const char iwl_killer_bn1850w2_name[];
 extern const char iwl_killer_bn1850i_name[];
@@ -744,7 +748,8 @@ extern const struct iwl_rf_cfg iwl_rf_fm_160mhz;
 #define iwl_rf_wh iwl_rf_fm
 #define iwl_rf_wh_160mhz iwl_rf_fm_160mhz
 extern const struct iwl_rf_cfg iwl_rf_wh_non_eht;
-#define iwl_rf_pe iwl_rf_fm
+extern const struct iwl_rf_cfg iwl_rf_pe;
+#define iwl_rf_pe_no_uhr iwl_rf_fm
 #endif /* CONFIG_IWLMLD */
 
 #endif /* __IWL_CONFIG_H__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
index f3fa37fee2e49..d2fa80a3dd040 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-csr.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2016 Intel Deutschland GmbH
  */
@@ -366,6 +366,7 @@ enum {
 #define CSR_HW_RF_ID_TYPE_GF4		(0x0010E000)
 #define CSR_HW_RF_ID_TYPE_FM		(0x00112000)
 #define CSR_HW_RF_ID_TYPE_WP		(0x00113000)
+#define CSR_HW_RF_ID_TYPE_PE		(0x00114000)
 
 /* HW_RF CHIP STEP  */
 #define CSR_HW_RF_STEP(_val) (((_val) >> 8) & 0xF)
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
index d5ded4d3a30b7..4885245295380 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-drv.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -1294,8 +1294,9 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 
 			if (tlv_len != sizeof(*fseq_ver))
 				goto invalid_tlv_len;
-			IWL_DEBUG_INFO(drv, "TLV_FW_FSEQ_VERSION: %.32s\n",
-				       fseq_ver->version);
+			IWL_DEBUG_INFO(drv,
+				       "TLV_FW_FSEQ_VERSION: %.32s (sha1: %.20s)\n",
+				       fseq_ver->version, fseq_ver->sha1);
 			}
 			break;
 		case IWL_UCODE_TLV_FW_NUM_STATIONS:
@@ -1330,6 +1331,12 @@ static int iwl_parse_tlv_firmware(struct iwl_drv *drv,
 			capa->num_beacons =
 				le32_to_cpup((const __le32 *)tlv_data);
 			break;
+		case IWL_UCODE_TLV_FW_NUM_MCAST_KEY_ENTRIES:
+			if (tlv_len != sizeof(u32))
+				goto invalid_tlv_len;
+			capa->num_mcast_key_entries =
+				le32_to_cpup((const __le32 *)tlv_data);
+			break;
 		case IWL_UCODE_TLV_UMAC_DEBUG_ADDRS: {
 			const struct iwl_umac_debug_addrs *dbg_ptrs =
 				(const void *)tlv_data;
@@ -1640,6 +1647,7 @@ static void iwl_req_fw_callback(const struct firmware *ucode_raw, void *context)
 	fw->ucode_capa.n_scan_channels = IWL_DEFAULT_SCAN_CHANNELS;
 	fw->ucode_capa.num_stations = IWL_STATION_COUNT_MAX;
 	fw->ucode_capa.num_beacons = 1;
+	fw->ucode_capa.num_mcast_key_entries = 2;
 	/* dump all fw memory areas by default */
 	fw->dbg.dump_mask = 0xffffffff;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-io.c b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
index b1944584c6931..bb746112ddadf 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-io.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-io.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2003-2014, 2018-2022, 2024-2025 Intel Corporation
+ * Copyright (C) 2003-2014, 2018-2022, 2024-2026 Intel Corporation
  * Copyright (C) 2015-2016 Intel Deutschland GmbH
  */
 #include <linux/device.h>
@@ -12,7 +12,6 @@
 #include "iwl-debug.h"
 #include "iwl-prph.h"
 #include "iwl-fh.h"
-#include "pcie/gen1_2/internal.h"
 
 void iwl_write8(struct iwl_trans *trans, u32 ofs, u8 val)
 {
@@ -168,6 +167,22 @@ int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr,
 	return -ETIMEDOUT;
 }
 
+int iwl_poll_umac_prph_bits_no_grab(struct iwl_trans *trans, u32 addr,
+				    u32 bits, u32 mask, int timeout)
+{
+	int t = 0;
+
+	do {
+		if ((iwl_read_umac_prph_no_grab(trans, addr) & mask) ==
+		    (bits & mask))
+			return 0;
+		udelay(IWL_POLL_INTERVAL);
+		t += IWL_POLL_INTERVAL;
+	} while (t < timeout);
+
+	return -ETIMEDOUT;
+}
+
 void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask)
 {
 	if (iwl_trans_grab_nic_access(trans)) {
@@ -396,12 +411,6 @@ int iwl_dump_fh(struct iwl_trans *trans, char **buf)
 	return 0;
 }
 
-int iwl_trans_activate_nic(struct iwl_trans *trans)
-{
-	return iwl_pcie_gen1_2_activate_nic(trans);
-}
-IWL_EXPORT_SYMBOL(iwl_trans_activate_nic);
-
 void iwl_trans_sync_nmi_with_addr(struct iwl_trans *trans, u32 inta_addr,
 				  u32 sw_err_bit)
 {
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-io.h b/drivers/net/wireless/intel/iwlwifi/iwl-io.h
index 5bcec239ffc4a..6dce2e5267a64 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-io.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-io.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2018-2021, 2025 Intel Corporation
+ * Copyright (C) 2018-2021, 2025-2026 Intel Corporation
  */
 #ifndef __iwl_io_h__
 #define __iwl_io_h__
@@ -51,14 +51,14 @@ static inline void iwl_write_prph(struct iwl_trans *trans, u32 ofs, u32 val)
 
 int iwl_poll_prph_bit(struct iwl_trans *trans, u32 addr,
 		      u32 bits, u32 mask, int timeout);
+int iwl_poll_umac_prph_bits_no_grab(struct iwl_trans *trans, u32 addr,
+				    u32 bits, u32 mask, int timeout);
 void iwl_set_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask);
 void iwl_set_bits_mask_prph(struct iwl_trans *trans, u32 ofs,
 			    u32 bits, u32 mask);
 void iwl_clear_bits_prph(struct iwl_trans *trans, u32 ofs, u32 mask);
 void iwl_force_nmi(struct iwl_trans *trans);
 
-int iwl_trans_activate_nic(struct iwl_trans *trans);
-
 /* Error handling */
 int iwl_dump_fh(struct iwl_trans *trans, char **buf);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
index 8f3f651451bb9..7027bca249a03 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2005-2014, 2018-2023, 2025 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2023, 2025-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -18,7 +18,6 @@
 #include "iwl-prph.h"
 #include "iwl-io.h"
 #include "iwl-csr.h"
-#include "fw/acpi.h"
 #include "fw/api/nvm-reg.h"
 #include "fw/api/commands.h"
 #include "fw/api/cmdhdr.h"
@@ -206,28 +205,30 @@ enum iwl_reg_capa_flags_v2 {
 }; /* GEO_CHANNEL_CAPABILITIES_API_S_VER_2 */
 
 /**
- * enum iwl_reg_capa_flags_v4 - global flags applied for the whole regulatory
+ * enum iwl_reg_capa_flags_v5 - global flags applied for the whole regulatory
  * domain.
- * @REG_CAPA_V4_160MHZ_ALLOWED: 11ac channel with a width of 160Mhz is allowed
+ * @REG_CAPA_V5_160MHZ_ALLOWED: 11ac channel with a width of 160Mhz is allowed
  *	for this regulatory domain (valid only in 5Ghz).
- * @REG_CAPA_V4_80MHZ_ALLOWED: 11ac channel with a width of 80Mhz is allowed
+ * @REG_CAPA_V5_80MHZ_ALLOWED: 11ac channel with a width of 80Mhz is allowed
  *	for this regulatory domain (valid only in 5Ghz).
- * @REG_CAPA_V4_MCS_12_ALLOWED: 11ac with MCS 12 is allowed.
- * @REG_CAPA_V4_MCS_13_ALLOWED: 11ac with MCS 13 is allowed.
- * @REG_CAPA_V4_11BE_DISABLED: 11be is forbidden for this regulatory domain.
- * @REG_CAPA_V4_11AX_DISABLED: 11ax is forbidden for this regulatory domain.
- * @REG_CAPA_V4_320MHZ_ALLOWED: 11be channel with a width of 320Mhz is allowed
+ * @REG_CAPA_V5_MCS_12_ALLOWED: 11ac with MCS 12 is allowed.
+ * @REG_CAPA_V5_MCS_13_ALLOWED: 11ac with MCS 13 is allowed.
+ * @REG_CAPA_V5_11BE_DISABLED: 11be is forbidden for this regulatory domain.
+ * @REG_CAPA_V5_11AX_DISABLED: 11ax is forbidden for this regulatory domain.
+ * @REG_CAPA_V5_320MHZ_ALLOWED: 11be channel with a width of 320Mhz is allowed
  *	for this regulatory domain (valid only in 5GHz).
+ * @REG_CAPA_V5_11BN_DISABLED: UHR is not allowed for this regulatory domain
  */
-enum iwl_reg_capa_flags_v4 {
-	REG_CAPA_V4_160MHZ_ALLOWED		= BIT(3),
-	REG_CAPA_V4_80MHZ_ALLOWED		= BIT(4),
-	REG_CAPA_V4_MCS_12_ALLOWED		= BIT(5),
-	REG_CAPA_V4_MCS_13_ALLOWED		= BIT(6),
-	REG_CAPA_V4_11BE_DISABLED		= BIT(8),
-	REG_CAPA_V4_11AX_DISABLED		= BIT(13),
-	REG_CAPA_V4_320MHZ_ALLOWED		= BIT(16),
-}; /* GEO_CHANNEL_CAPABILITIES_API_S_VER_4 */
+enum iwl_reg_capa_flags_v5 {
+	REG_CAPA_V5_160MHZ_ALLOWED		= BIT(3),
+	REG_CAPA_V5_80MHZ_ALLOWED		= BIT(4),
+	REG_CAPA_V5_MCS_12_ALLOWED		= BIT(5),
+	REG_CAPA_V5_MCS_13_ALLOWED		= BIT(6),
+	REG_CAPA_V5_11BE_DISABLED		= BIT(8),
+	REG_CAPA_V5_11AX_DISABLED		= BIT(13),
+	REG_CAPA_V5_320MHZ_ALLOWED		= BIT(16),
+	REG_CAPA_V5_11BN_DISABLED		= BIT(17),
+}; /* GEO_CHANNEL_CAPABILITIES_API_S_VER_4, 5 */
 
 /*
 * API v2 for reg_capa_flags is relevant from version 6 and onwards of the
@@ -546,7 +547,7 @@ static const u8 iwl_vendor_caps[] = {
 	0x00
 };
 
-static const struct ieee80211_sband_iftype_data iwl_he_eht_capa[] = {
+static const struct ieee80211_sband_iftype_data iwl_iftype_cap[] = {
 	{
 		.types_mask = BIT(NL80211_IFTYPE_STATION) |
 			      BIT(NL80211_IFTYPE_P2P_CLIENT),
@@ -690,6 +691,16 @@ static const struct ieee80211_sband_iftype_data iwl_he_eht_capa[] = {
 			 */
 			.eht_ppe_thres = {0xc1, 0x0e, 0xe0 }
 		},
+		.uhr_cap = {
+			.has_uhr = true,
+			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
+				   IEEE80211_UHR_PHY_CAP_ELR_TX,
+			.mac.mac_cap = {
+				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP |
+				      IEEE80211_UHR_MAC_CAP0_DPS_SUPP,
+				[1] = IEEE80211_UHR_MAC_CAP1_DUO_SUPP,
+			},
+		},
 	},
 	{
 		.types_mask = BIT(NL80211_IFTYPE_AP) |
@@ -788,6 +799,11 @@ static const struct ieee80211_sband_iftype_data iwl_he_eht_capa[] = {
 			 */
 			.eht_ppe_thres = {0xc1, 0x0e, 0xe0 }
 		},
+		.uhr_cap = {
+			.has_uhr = true,
+			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
+				   IEEE80211_UHR_PHY_CAP_ELR_TX,
+		},
 	},
 };
 
@@ -855,6 +871,9 @@ iwl_nvm_fixup_sband_iftd(struct iwl_trans *trans,
 	    fips_enabled)
 		iftype_data->eht_cap.has_eht = false;
 
+	if (!data->sku_cap_11bn_enable || !iftype_data->eht_cap.has_eht)
+		iftype_data->uhr_cap.has_uhr = false;
+
 	/* Advertise an A-MPDU exponent extension based on
 	 * operating band
 	 */
@@ -1023,9 +1042,9 @@ static void iwl_init_he_hw_capab(struct iwl_trans *trans,
 	struct ieee80211_sband_iftype_data *iftype_data;
 	int i;
 
-	BUILD_BUG_ON(sizeof(data->iftd.low) != sizeof(iwl_he_eht_capa));
-	BUILD_BUG_ON(sizeof(data->iftd.high) != sizeof(iwl_he_eht_capa));
-	BUILD_BUG_ON(sizeof(data->iftd.uhb) != sizeof(iwl_he_eht_capa));
+	BUILD_BUG_ON(sizeof(data->iftd.low) != sizeof(iwl_iftype_cap));
+	BUILD_BUG_ON(sizeof(data->iftd.high) != sizeof(iwl_iftype_cap));
+	BUILD_BUG_ON(sizeof(data->iftd.uhb) != sizeof(iwl_iftype_cap));
 
 	switch (sband->band) {
 	case NL80211_BAND_2GHZ:
@@ -1042,10 +1061,10 @@ static void iwl_init_he_hw_capab(struct iwl_trans *trans,
 		return;
 	}
 
-	memcpy(iftype_data, iwl_he_eht_capa, sizeof(iwl_he_eht_capa));
+	memcpy(iftype_data, iwl_iftype_cap, sizeof(iwl_iftype_cap));
 
 	_ieee80211_set_sband_iftype_data(sband, iftype_data,
-					 ARRAY_SIZE(iwl_he_eht_capa));
+					 ARRAY_SIZE(iwl_iftype_cap));
 
 	for (i = 0; i < sband->n_iftype_data; i++)
 		iwl_nvm_fixup_sband_iftd(trans, data, sband, &iftype_data[i],
@@ -1054,6 +1073,46 @@ static void iwl_init_he_hw_capab(struct iwl_trans *trans,
 	iwl_init_he_6ghz_capa(trans, data, sband, tx_chains, rx_chains);
 }
 
+static void
+iwl_init_nan_phy_capa(const struct iwl_fw *fw, struct iwl_nvm_data *data)
+{
+	const struct ieee80211_sta_he_cap *he_cap;
+
+	if (!fw_has_capa(&fw->ucode_capa, IWL_UCODE_TLV_CAPA_NAN_SYNC_SUPPORT))
+		return;
+
+	data->nan_phy_capa.ht = data->bands[NL80211_BAND_2GHZ].ht_cap;
+	data->nan_phy_capa.vht = data->bands[NL80211_BAND_5GHZ].vht_cap;
+
+	he_cap = ieee80211_get_he_iftype_cap(&data->bands[NL80211_BAND_2GHZ],
+					     NL80211_IFTYPE_STATION);
+	if (he_cap) {
+		data->nan_phy_capa.he = *he_cap;
+		data->nan_phy_capa.he.he_cap_elem.phy_cap_info[0] |=
+			IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+			IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G;
+	}
+
+	/*
+	 * FIXME: we copied HE capabilities from the 2.4 GHz band,
+	 * but there are bits that are band-dependent:
+	 *
+	 * IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_1 - 2.4 GHz - set
+	 * IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3 - 5 GHz - not set
+	 * IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G - set
+	 * IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G - set
+	 * IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G - set
+	 *
+	 * We copied from STA iftype - so we have the following bits set:
+	 * IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS
+	 * IEEE80211_HE_PHY_CAP2_MIDAMBLE_RX_TX_MAX_NSTS
+	 * IEEE80211_HE_PHY_CAP7_MAX_NC_1
+	 * IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO
+	 *
+	 * Need to check which one should actually be set for NAN.
+	 */
+}
+
 void iwl_reinit_cab(struct iwl_trans *trans, struct iwl_nvm_data *data,
 		    u8 tx_chains, u8 rx_chains, const struct iwl_fw *fw)
 {
@@ -1082,6 +1141,8 @@ void iwl_reinit_cab(struct iwl_trans *trans, struct iwl_nvm_data *data,
 	if (data->sku_cap_11ax_enable && !iwlwifi_mod_params.disable_11ax)
 		iwl_init_he_hw_capab(trans, data, sband, tx_chains, rx_chains,
 				     fw);
+
+	iwl_init_nan_phy_capa(fw, data);
 }
 IWL_EXPORT_SYMBOL(iwl_reinit_cab);
 
@@ -1611,6 +1672,9 @@ u32 iwl_nvm_get_regdom_bw_flags(const u16 *nvm_chan,
 	if (reg_capa.disable_11be)
 		flags |= NL80211_RRF_NO_EHT;
 
+	if (reg_capa.disable_11bn)
+		flags |= NL80211_RRF_NO_UHR;
+
 	return flags;
 }
 EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_nvm_get_regdom_bw_flags);
@@ -1621,11 +1685,13 @@ static struct iwl_reg_capa iwl_get_reg_capa(u32 flags, u8 resp_ver)
 
 	if (resp_ver >= REG_CAPA_V4_RESP_VER) {
 		reg_capa.allow_40mhz = true;
-		reg_capa.allow_80mhz = flags & REG_CAPA_V4_80MHZ_ALLOWED;
-		reg_capa.allow_160mhz = flags & REG_CAPA_V4_160MHZ_ALLOWED;
-		reg_capa.allow_320mhz = flags & REG_CAPA_V4_320MHZ_ALLOWED;
-		reg_capa.disable_11ax = flags & REG_CAPA_V4_11AX_DISABLED;
-		reg_capa.disable_11be = flags & REG_CAPA_V4_11BE_DISABLED;
+		reg_capa.allow_80mhz = flags & REG_CAPA_V5_80MHZ_ALLOWED;
+		reg_capa.allow_160mhz = flags & REG_CAPA_V5_160MHZ_ALLOWED;
+		reg_capa.allow_320mhz = flags & REG_CAPA_V5_320MHZ_ALLOWED;
+		reg_capa.disable_11ax = flags & REG_CAPA_V5_11AX_DISABLED;
+		reg_capa.disable_11be = flags & REG_CAPA_V5_11BE_DISABLED;
+		/* can check: was reserved and irrelevant for pre-UHR devices */
+		reg_capa.disable_11bn = flags & REG_CAPA_V5_11BN_DISABLED;
 	} else if (resp_ver >= REG_CAPA_V2_RESP_VER) {
 		reg_capa.allow_40mhz = flags & REG_CAPA_V2_40MHZ_ALLOWED;
 		reg_capa.allow_80mhz = flags & REG_CAPA_V2_80MHZ_ALLOWED;
@@ -1683,8 +1749,16 @@ iwl_parse_nvm_mcc_info(struct iwl_trans *trans,
 	IWL_DEBUG_DEV(dev, IWL_DL_LAR, "building regdom for %d channels\n",
 		      num_of_ch);
 
-	/* build a regdomain rule for every valid channel */
-	regd = kzalloc_flex(*regd, reg_rules, num_of_ch);
+	/* build a regdomain rule for every valid channel.
+	 * Certain firmware versions might report no valid channels
+	 * if booted in RF-kill, i.e. not all calibrations etc. are
+	 * running. We'll get out of this situation later when the
+	 * rfkill is removed and we update the regdomain again, but
+	 * since cfg80211 doesn't accept an empty regdomain, we need
+	 * to allocate space for at least one rule to add a dummy
+	 * (unusable) rule in this case so we can init.
+	 */
+	regd = kzalloc_flex(*regd, reg_rules, num_of_ch ?: 1);
 	if (!regd)
 		return ERR_PTR(-ENOMEM);
 
@@ -1758,14 +1832,7 @@ iwl_parse_nvm_mcc_info(struct iwl_trans *trans,
 		reg_query_regdb_wmm(regd->alpha2, center_freq, rule);
 	}
 
-	/*
-	 * Certain firmware versions might report no valid channels
-	 * if booted in RF-kill, i.e. not all calibrations etc. are
-	 * running. We'll get out of this situation later when the
-	 * rfkill is removed and we update the regdomain again, but
-	 * since cfg80211 doesn't accept an empty regdomain, add a
-	 * dummy (unusable) rule here in this case so we can init.
-	 */
+	/* If no valid rules were found, add a dummy rule */
 	if (!valid_rules) {
 		valid_rules = 1;
 		rule = &regd->reg_rules[valid_rules - 1];
@@ -2079,6 +2146,7 @@ struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans,
 		!!(mac_flags & NVM_MAC_SKU_FLAGS_MIMO_DISABLED);
 	if (trans->cfg->eht_supported)
 		nvm->sku_cap_11be_enable = true;
+	nvm->sku_cap_11bn_enable = trans->cfg->uhr_supported;
 
 	/* Initialize PHY sku data */
 	nvm->valid_tx_ant = (u8)le32_to_cpu(rsp->phy_sku.tx_chains);
@@ -2106,6 +2174,8 @@ struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans,
 	iwl_init_sbands(trans, nvm, channel_profile, tx_ant, rx_ant,
 			sbands_flags, v4, fw);
 
+	iwl_init_nan_phy_capa(fw, nvm);
+
 	iwl_free_resp(&hcmd);
 	return nvm;
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
index 12f28bb0e859d..e676d7c2d6cc0 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h
@@ -35,6 +35,7 @@ enum iwl_nvm_sbands_flags {
  *	for this regulatory domain (valid only in 6 Ghz).
  * @disable_11ax: 11ax is forbidden for this regulatory domain.
  * @disable_11be: 11be is forbidden for this regulatory domain.
+ * @disable_11bn: UHR/11bn is not allowed for this regulatory domain
  */
 struct iwl_reg_capa {
 	bool allow_40mhz;
@@ -43,6 +44,7 @@ struct iwl_reg_capa {
 	bool allow_320mhz;
 	bool disable_11ax;
 	bool disable_11be;
+	bool disable_11bn;
 };
 
 /**
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-utils.h b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-utils.h
index ac0a29a1c31f5..52d35b73ed74f 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-utils.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-utils.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018, 2020-2023 Intel Corporation
+ * Copyright (C) 2005-2014, 2018, 2020-2023, 2026 Intel Corporation
  * Copyright (C) 2015 Intel Mobile Communications GmbH
  */
 #ifndef __iwl_eeprom_parse_h__
@@ -32,6 +32,7 @@ struct iwl_nvm_data {
 	bool sku_cap_ipan_enable;
 	bool sku_cap_mimo_disabled;
 	bool sku_cap_11be_enable;
+	bool sku_cap_11bn_enable;
 
 	u16 radio_cfg_type;
 	u8 radio_cfg_step;
@@ -55,6 +56,12 @@ struct iwl_nvm_data {
 		struct ieee80211_sband_iftype_data uhb[2];
 	} iftd;
 
+	struct {
+		struct ieee80211_sta_ht_cap ht;
+		struct ieee80211_sta_vht_cap vht;
+		struct ieee80211_sta_he_cap he;
+	} nan_phy_capa;
+
 	struct ieee80211_channel channels[];
 };
 
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
index a7214ddcfaf56..6ca1f51b69a1c 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-prph.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016 Intel Deutschland GmbH
  */
@@ -411,6 +411,11 @@ enum {
 #define HPM_SECONDARY_DEVICE_STATE      0xa03404
 #define WFPM_MAC_OTP_CFG7_ADDR		0xa03338
 #define WFPM_MAC_OTP_CFG7_DATA		0xa0333c
+#define WFPM_RSRCS_4PHS_REQ_STTS	0xa033f8
+#define WFPM_RSRCS_4PHS_ACK_STTS	0xa033fc
+
+#define RSRC_REQ_CNVR_TOP BIT(6)
+#define RSRC_ACK_CNVR_TOP BIT(6)
 
 
 /* For UMAG_GEN_HW_STATUS reg check */
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
index 16b2c313e72b2..0009488ca51b1 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c
@@ -2,16 +2,14 @@
 /*
  * Copyright (C) 2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
- * Copyright (C) 2019-2021, 2023-2025 Intel Corporation
+ * Copyright (C) 2019-2021, 2023-2026 Intel Corporation
  */
 #include <linux/kernel.h>
 #include <linux/bsearch.h>
 #include <linux/list.h>
 
-#include "fw/api/tx.h"
 #include "iwl-trans.h"
 #include "iwl-drv.h"
-#include "iwl-fh.h"
 #include <linux/dmapool.h>
 #include "fw/api/commands.h"
 #include "pcie/gen1_2/internal.h"
@@ -461,6 +459,12 @@ int iwl_trans_read_mem(struct iwl_trans *trans, u32 addr,
 }
 IWL_EXPORT_SYMBOL(iwl_trans_read_mem);
 
+int iwl_trans_read_mem_no_grab(struct iwl_trans *trans, u32 addr,
+			       void *buf, u32 dwords)
+{
+	return iwl_trans_pcie_read_mem_no_grab(trans, addr, buf, dwords);
+}
+
 int iwl_trans_write_mem(struct iwl_trans *trans, u32 addr,
 			const void *buf, int dwords)
 {
@@ -822,3 +826,10 @@ bool iwl_trans_is_ltr_enabled(struct iwl_trans *trans)
 	return iwl_pcie_gen1_2_is_ltr_enabled(trans);
 }
 IWL_EXPORT_SYMBOL(iwl_trans_is_ltr_enabled);
+
+int iwl_trans_activate_nic(struct iwl_trans *trans)
+{
+	return iwl_pcie_gen1_2_activate_nic(trans);
+}
+IWL_EXPORT_SYMBOL(iwl_trans_activate_nic);
+
diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
index 61e4f4776dcb3..3ae840e546e82 100644
--- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
+++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2005-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2005-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -700,106 +700,6 @@ struct iwl_cmd_meta {
 	u32 tbs;
 };
 
-/*
- * The FH will write back to the first TB only, so we need to copy some data
- * into the buffer regardless of whether it should be mapped or not.
- * This indicates how big the first TB must be to include the scratch buffer
- * and the assigned PN.
- * Since PN location is 8 bytes at offset 12, it's 20 now.
- * If we make it bigger then allocations will be bigger and copy slower, so
- * that's probably not useful.
- */
-#define IWL_FIRST_TB_SIZE	20
-#define IWL_FIRST_TB_SIZE_ALIGN ALIGN(IWL_FIRST_TB_SIZE, 64)
-
-struct iwl_pcie_txq_entry {
-	void *cmd;
-	struct sk_buff *skb;
-	/* buffer to free after command completes */
-	const void *free_buf;
-	struct iwl_cmd_meta meta;
-};
-
-struct iwl_pcie_first_tb_buf {
-	u8 buf[IWL_FIRST_TB_SIZE_ALIGN];
-};
-
-/**
- * struct iwl_txq - Tx Queue for DMA
- * @tfds: transmit frame descriptors (DMA memory)
- * @first_tb_bufs: start of command headers, including scratch buffers, for
- *	the writeback -- this is DMA memory and an array holding one buffer
- *	for each command on the queue
- * @first_tb_dma: DMA address for the first_tb_bufs start
- * @entries: transmit entries (driver state)
- * @lock: queue lock
- * @reclaim_lock: reclaim lock
- * @stuck_timer: timer that fires if queue gets stuck
- * @trans: pointer back to transport (for timer)
- * @need_update: indicates need to update read/write index
- * @ampdu: true if this queue is an ampdu queue for an specific RA/TID
- * @wd_timeout: queue watchdog timeout (jiffies) - per queue
- * @frozen: tx stuck queue timer is frozen
- * @frozen_expiry_remainder: remember how long until the timer fires
- * @block: queue is blocked
- * @bc_tbl: byte count table of the queue (relevant only for gen2 transport)
- * @write_ptr: 1-st empty entry (index) host_w
- * @read_ptr: last used entry (index) host_r
- * @dma_addr:  physical addr for BD's
- * @n_window: safe queue window
- * @id: queue id
- * @low_mark: low watermark, resume queue if free space more than this
- * @high_mark: high watermark, stop queue if free space less than this
- * @overflow_q: overflow queue for handling frames that didn't fit on HW queue
- * @overflow_tx: need to transmit from overflow
- *
- * A Tx queue consists of circular buffer of BDs (a.k.a. TFDs, transmit frame
- * descriptors) and required locking structures.
- *
- * Note the difference between TFD_QUEUE_SIZE_MAX and n_window: the hardware
- * always assumes 256 descriptors, so TFD_QUEUE_SIZE_MAX is always 256 (unless
- * there might be HW changes in the future). For the normal TX
- * queues, n_window, which is the size of the software queue data
- * is also 256; however, for the command queue, n_window is only
- * 32 since we don't need so many commands pending. Since the HW
- * still uses 256 BDs for DMA though, TFD_QUEUE_SIZE_MAX stays 256.
- * This means that we end up with the following:
- *  HW entries: | 0 | ... | N * 32 | ... | N * 32 + 31 | ... | 255 |
- *  SW entries:           | 0      | ... | 31          |
- * where N is a number between 0 and 7. This means that the SW
- * data is a window overlayed over the HW queue.
- */
-struct iwl_txq {
-	void *tfds;
-	struct iwl_pcie_first_tb_buf *first_tb_bufs;
-	dma_addr_t first_tb_dma;
-	struct iwl_pcie_txq_entry *entries;
-	/* lock for syncing changes on the queue */
-	spinlock_t lock;
-	/* lock to prevent concurrent reclaim */
-	spinlock_t reclaim_lock;
-	unsigned long frozen_expiry_remainder;
-	struct timer_list stuck_timer;
-	struct iwl_trans *trans;
-	bool need_update;
-	bool frozen;
-	bool ampdu;
-	int block;
-	unsigned long wd_timeout;
-	struct sk_buff_head overflow_q;
-	struct iwl_dma_ptr bc_tbl;
-
-	int write_ptr;
-	int read_ptr;
-	dma_addr_t dma_addr;
-	int n_window;
-	u32 id;
-	int low_mark;
-	int high_mark;
-
-	bool overflow_tx;
-};
-
 /**
  * struct iwl_trans_info - transport info for outside use
  * @name: the device name
@@ -1019,6 +919,14 @@ void iwl_trans_write_prph(struct iwl_trans *trans, u32 ofs, u32 val);
 int iwl_trans_read_mem(struct iwl_trans *trans, u32 addr,
 		       void *buf, int dwords);
 
+/*
+ * Note the special calling convention - it's allowed to drop the
+ * internal transport lock and re-enable BHs temporarily, but will
+ * not release NIC access.
+ */
+int iwl_trans_read_mem_no_grab(struct iwl_trans *trans, u32 addr,
+			       void *buf, u32 dwords);
+
 int iwl_trans_read_config32(struct iwl_trans *trans, u32 ofs,
 			    u32 *val);
 
@@ -1034,6 +942,14 @@ void iwl_trans_debugfs_cleanup(struct iwl_trans *trans);
 				   (bufsize) / sizeof(u32));	\
 	})
 
+static inline int
+iwl_trans_read_mem_bytes_no_grab(struct iwl_trans *trans,
+				 u32 addr, void *buf, u32 bufsize)
+{
+	return iwl_trans_read_mem_no_grab(trans, addr, buf,
+					  bufsize / sizeof(u32));
+}
+
 int iwl_trans_write_imr_mem(struct iwl_trans *trans, u32 dst_addr,
 			    u64 src_addr, u32 byte_cnt);
 
@@ -1153,6 +1069,8 @@ static inline bool iwl_trans_dbg_ini_valid(struct iwl_trans *trans)
 
 void iwl_trans_interrupts(struct iwl_trans *trans, bool enable);
 
+int iwl_trans_activate_nic(struct iwl_trans *trans);
+
 static inline void iwl_trans_finish_sw_reset(struct iwl_trans *trans)
 {
 	clear_bit(STATUS_IN_SW_RESET, &trans->status);
@@ -1236,9 +1154,6 @@ enum iwl_reset_mode {
 void iwl_trans_pcie_reset(struct iwl_trans *trans, enum iwl_reset_mode mode);
 void iwl_trans_pcie_fw_reset_handshake(struct iwl_trans *trans);
 
-int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans,
-			     struct iwl_host_cmd *cmd);
-
 /* Internal helper */
 static inline void iwl_trans_set_info(struct iwl_trans *trans,
 				      struct iwl_trans_info *info)
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/agg.c b/drivers/net/wireless/intel/iwlwifi/mld/agg.c
index 3bf36f8f68742..e3627ad0321c8 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/agg.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/agg.c
@@ -64,6 +64,9 @@ static void iwl_mld_release_frames_from_notif(struct iwl_mld *mld,
 	}
 
 	/* pick any STA ID to find the pointer */
+	if (WARN_ON_ONCE(!ba_data->sta_mask))
+		goto out_unlock;
+
 	sta_id = ffs(ba_data->sta_mask) - 1;
 	link_sta = rcu_dereference(mld->fw_id_to_link_sta[sta_id]);
 	if (WARN_ON_ONCE(IS_ERR_OR_NULL(link_sta) || !link_sta->sta))
@@ -166,6 +169,9 @@ void iwl_mld_del_ba(struct iwl_mld *mld, int queue,
 		goto out_unlock;
 
 	/* pick any STA ID to find the pointer */
+	if (WARN_ON_ONCE(!ba_data->sta_mask))
+		goto out_unlock;
+
 	sta_id = ffs(ba_data->sta_mask) - 1;
 	link_sta = rcu_dereference(mld->fw_id_to_link_sta[sta_id]);
 	if (WARN_ON_ONCE(IS_ERR_OR_NULL(link_sta) || !link_sta->sta))
@@ -347,6 +353,9 @@ static void iwl_mld_rx_agg_session_expired(struct timer_list *t)
 	}
 
 	/* timer expired, pick any STA ID to find the pointer */
+	if (WARN_ON_ONCE(!ba_data->sta_mask))
+		goto unlock;
+
 	sta_id = ffs(ba_data->sta_mask) - 1;
 	link_sta = rcu_dereference(ba_data->mld->fw_id_to_link_sta[sta_id]);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ap.c b/drivers/net/wireless/intel/iwlwifi/mld/ap.c
index 5c59acc8c4c5a..bc426b911ce5c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/ap.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/ap.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024 Intel Corporation
+ * Copyright (C) 2024, 2026 Intel Corporation
  */
 #include <linux/crc32.h>
 
@@ -239,6 +239,58 @@ int iwl_mld_store_ap_early_key(struct iwl_mld *mld,
 	return -ENOSPC;
 }
 
+static void iwl_mld_stop_beacon(struct iwl_mld *mld, struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *link)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
+	struct iwl_mac_beacon_cmd cmd = {};
+	int cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, BEACON_TEMPLATE_CMD, 14);
+
+	if (WARN_ON(!mld_link))
+		return;
+
+	if (cmd_ver < 15)
+		return;
+
+	/* leave byte_cnt 0 */
+	cmd.link_id = cpu_to_le32(mld_link->fw_id);
+
+	iwl_mld_send_cmd_pdu(mld, BEACON_TEMPLATE_CMD, &cmd);
+}
+
+void
+iwl_mld_link_info_changed_ap_ibss(struct iwl_mld *mld,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_bss_conf *link,
+				  u64 changes)
+{
+	u32 link_changes = 0;
+
+	if (changes & BSS_CHANGED_ERP_SLOT)
+		link_changes |= LINK_CONTEXT_MODIFY_RATES_INFO;
+
+	if (changes & (BSS_CHANGED_ERP_CTS_PROT | BSS_CHANGED_HT))
+		link_changes |= LINK_CONTEXT_MODIFY_PROTECT_FLAGS;
+
+	if (changes & (BSS_CHANGED_QOS | BSS_CHANGED_BANDWIDTH))
+		link_changes |= LINK_CONTEXT_MODIFY_QOS_PARAMS;
+
+	if (changes & BSS_CHANGED_HE_BSS_COLOR)
+		link_changes |= LINK_CONTEXT_MODIFY_HE_PARAMS;
+
+	if (link_changes)
+		iwl_mld_change_link_in_fw(mld, link, link_changes);
+
+	if (changes & BSS_CHANGED_BEACON) {
+		WARN_ON(!link->enable_beacon);
+		iwl_mld_update_beacon_template(mld, vif, link);
+	}
+
+	/* Enabling beacons was already covered above */
+	if ((changes & BSS_CHANGED_BEACON_ENABLED) && !link->enable_beacon)
+		iwl_mld_stop_beacon(mld, vif, link);
+}
+
 static int iwl_mld_send_ap_early_keys(struct iwl_mld *mld,
 				      struct ieee80211_vif *vif,
 				      struct ieee80211_bss_conf *link)
@@ -276,10 +328,6 @@ int iwl_mld_start_ap_ibss(struct ieee80211_hw *hw,
 	if (vif->type == NL80211_IFTYPE_AP)
 		iwl_mld_send_ap_tx_power_constraint_cmd(mld, vif, link);
 
-	ret = iwl_mld_update_beacon_template(mld, vif, link);
-	if (ret)
-		return ret;
-
 	/* the link should be already activated when assigning chan context,
 	 * and LINK_CONTEXT_MODIFY_EHT_PARAMS is deprecated
 	 */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ap.h b/drivers/net/wireless/intel/iwlwifi/mld/ap.h
index 4a6f52b9552d7..f10e9c9a38ff1 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/ap.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/ap.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024 Intel Corporation
+ * Copyright (C) 2024, 2026 Intel Corporation
  */
 #ifndef __iwl_ap_h__
 #define __iwl_ap_h__
@@ -14,6 +14,12 @@ int iwl_mld_update_beacon_template(struct iwl_mld *mld,
 				   struct ieee80211_vif *vif,
 				   struct ieee80211_bss_conf *link_conf);
 
+void
+iwl_mld_link_info_changed_ap_ibss(struct iwl_mld *mld,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_bss_conf *link,
+				  u64 changes);
+
 int iwl_mld_start_ap_ibss(struct ieee80211_hw *hw,
 			  struct ieee80211_vif *vif,
 			  struct ieee80211_bss_conf *link);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/constants.h b/drivers/net/wireless/intel/iwlwifi/mld/constants.h
index e2a5eecc18c34..890abcab38374 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/constants.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/constants.h
@@ -1,11 +1,11 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #ifndef __iwl_mld_constants_h__
 #define __iwl_mld_constants_h__
 
-#define IWL_MLD_MISSED_BEACONS_SINCE_RX_THOLD			4
+#define IWL_MLD_MISSED_BEACONS_SINCE_RX_THOLD			6
 #define IWL_MLD_MISSED_BEACONS_THRESHOLD			8
 #define IWL_MLD_MISSED_BEACONS_THRESHOLD_LONG			19
 #define IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS		5
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/d3.c b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
index ef98efc8fb1b2..fc0a5871df2f1 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/d3.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/d3.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include "mld.h"
 
@@ -43,6 +43,12 @@ struct iwl_mld_resume_key_iter_data {
 	struct iwl_mld_wowlan_status *wowlan_status;
 };
 
+struct iwl_mld_rsc_resume_iter_data {
+	struct iwl_mld *mld;
+	const struct iwl_wowlan_all_rsc_tsc_v5 *notif;
+	int queue;
+};
+
 struct iwl_mld_suspend_key_iter_data {
 	struct iwl_wowlan_rsc_tsc_params_cmd *rsc;
 	bool have_rsc;
@@ -282,7 +288,8 @@ static void
 iwl_mld_convert_gtk_resume_data(struct iwl_mld *mld,
 				struct iwl_mld_wowlan_status *wowlan_status,
 				const struct iwl_wowlan_gtk_status *gtk_data,
-				const struct iwl_wowlan_all_rsc_tsc_v5 *sc)
+				const struct iwl_wowlan_all_rsc_tsc_v5 *sc,
+				int rsc_notif_ver)
 {
 	int status_idx = 0;
 
@@ -305,14 +312,18 @@ iwl_mld_convert_gtk_resume_data(struct iwl_mld *mld,
 		wowlan_status->gtk[status_idx].id =
 			wowlan_status->gtk[status_idx].flags &
 			IWL_WOWLAN_GTK_IDX_MASK;
-		/* The rsc for both gtk keys are stored in gtk[0]->sc->mcast_rsc
-		 * The gtk ids can be any two numbers between 0 and 3,
-		 * the id_map maps between the key id and the index in sc->mcast
-		 */
-		rsc_idx =
-			sc->mcast_key_id_map[wowlan_status->gtk[status_idx].id];
-		iwl_mld_convert_gtk_resume_seq(&wowlan_status->gtk[status_idx],
-					       sc, rsc_idx);
+		/* If RSC_NOTIF is not supported */
+		if (rsc_notif_ver == IWL_FW_CMD_VER_UNKNOWN) {
+			/* The rsc for both gtk keys are stored in
+			 * gtk[0]->sc->mcast_rsc. The gtk ids can be any two
+			 * numbers between 0 and 3, the id_map maps between the
+			 * key id and the index in sc->mcast
+			 */
+			rsc_idx =
+				sc->mcast_key_id_map[wowlan_status->gtk[status_idx].id];
+			iwl_mld_convert_gtk_resume_seq(&wowlan_status->gtk[status_idx],
+						       sc, rsc_idx);
+		}
 
 		if (key_status == IWL_WOWLAN_STATUS_NEW_KEY) {
 			memcpy(wowlan_status->gtk[status_idx].key,
@@ -598,6 +609,10 @@ iwl_mld_handle_wowlan_info_notif(struct iwl_mld *mld,
 						      PROT_OFFLOAD_GROUP,
 						      WOWLAN_INFO_NOTIFICATION,
 						      IWL_FW_CMD_VER_UNKNOWN);
+	int rsc_notif_ver = iwl_fw_lookup_notif_ver(mld->fw,
+						    DATA_PATH_GROUP,
+						    RSC_NOTIF,
+						    IWL_FW_CMD_VER_UNKNOWN);
 
 	if (wowlan_info_ver == 5) {
 		/* v5 format - validate before conversion */
@@ -642,8 +657,10 @@ iwl_mld_handle_wowlan_info_notif(struct iwl_mld *mld,
 		return true;
 
 	iwl_mld_convert_gtk_resume_data(mld, wowlan_status, notif->gtk,
-					&notif->gtk[0].sc);
-	iwl_mld_convert_ptk_resume_seq(mld, wowlan_status, &notif->gtk[0].sc);
+					&notif->gtk[0].sc, rsc_notif_ver);
+	if (rsc_notif_ver == IWL_FW_CMD_VER_UNKNOWN)
+		iwl_mld_convert_ptk_resume_seq(mld, wowlan_status,
+					       &notif->gtk[0].sc);
 	/* only one igtk is passed by FW */
 	iwl_mld_convert_igtk_resume_data(wowlan_status, &notif->igtk[0]);
 	iwl_mld_convert_bigtk_resume_data(wowlan_status, notif->bigtk);
@@ -902,8 +919,14 @@ iwl_mld_resume_keys_iter(struct ieee80211_hw *hw,
 	struct iwl_mld_resume_key_iter_data *data = _data;
 	struct iwl_mld_wowlan_status *wowlan_status = data->wowlan_status;
 	u8 status_idx;
-
-	if (key->keyidx >= 0 && key->keyidx <= 3) {
+	int rsc_notif_ver = iwl_fw_lookup_notif_ver(data->mld->fw,
+						    DATA_PATH_GROUP,
+						    RSC_NOTIF,
+						    IWL_FW_CMD_VER_UNKNOWN);
+
+	/* If RSC_NOTIF is not supported */
+	if (rsc_notif_ver == IWL_FW_CMD_VER_UNKNOWN &&
+	    key->keyidx >= 0 && key->keyidx <= 3) {
 		/* PTK */
 		if (sta) {
 			iwl_mld_update_ptk_rx_seq(data->mld, wowlan_status,
@@ -932,6 +955,105 @@ iwl_mld_resume_keys_iter(struct ieee80211_hw *hw,
 	}
 }
 
+static void
+iwl_mld_rsc_update_key_iter(struct ieee80211_hw *hw,
+			    struct ieee80211_vif *vif,
+			    struct ieee80211_sta *sta,
+			    struct ieee80211_key_conf *key,
+			    void *_data)
+{
+	struct iwl_mld_rsc_resume_iter_data *data = _data;
+	struct ieee80211_key_seq seq;
+
+	if (key->keyidx > 3)
+		return;
+
+	if (sta) {
+		/* PTK */
+		BUILD_BUG_ON(ARRAY_SIZE(data->notif->ucast_rsc) !=
+			     IWL_MAX_TID_COUNT);
+
+		if (key->cipher == WLAN_CIPHER_SUITE_TKIP) {
+			/* TKIP: just update key sequences */
+			for (int tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
+				iwl_mld_le64_to_tkip_seq(data->notif->ucast_rsc[tid],
+							 &seq);
+				ieee80211_set_key_rx_seq(key, tid, &seq);
+			}
+		} else {
+			struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+			struct iwl_mld_ptk_pn *mld_ptk_pn =
+				rcu_dereference_wiphy(data->mld->wiphy,
+						      mld_sta->ptk_pn[key->keyidx]);
+
+			if (WARN_ON(!mld_ptk_pn))
+				return;
+
+			if (WARN_ON(data->queue >=
+				    data->mld->trans->info.num_rxqs))
+				return;
+
+			for (int tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
+				iwl_mld_le64_to_aes_seq(data->notif->ucast_rsc[tid],
+							&seq);
+				ieee80211_set_key_rx_seq(key, tid, &seq);
+				memcpy(mld_ptk_pn->q[data->queue].pn[tid],
+				       seq.ccmp.pn,
+				       IEEE80211_CCMP_PN_LEN);
+			}
+		}
+
+		IWL_DEBUG_WOWLAN(data->mld,
+				 "Updated PTK RSC for key %d on queue %d\n",
+				 key->keyidx, data->queue);
+	} else {
+		/* GTK */
+		int rsc_idx = data->notif->mcast_key_id_map[key->keyidx];
+
+		if (rsc_idx == IWL_MCAST_KEY_MAP_INVALID)
+			return;
+
+		if (IWL_FW_CHECK(data->mld,
+				 rsc_idx >= ARRAY_SIZE(data->notif->mcast_rsc),
+				 "Invalid mcast key mapping: %d for key %d\n",
+				 rsc_idx, key->keyidx))
+			return;
+
+		for (int tid = 0; tid < IWL_MAX_TID_COUNT; tid++) {
+			__le64 rsc =
+				data->notif->mcast_rsc[rsc_idx][tid];
+
+			if (key->cipher == WLAN_CIPHER_SUITE_TKIP)
+				iwl_mld_le64_to_tkip_seq(rsc, &seq);
+			else
+				iwl_mld_le64_to_aes_seq(rsc, &seq);
+			ieee80211_set_key_rx_seq(key, tid, &seq);
+		}
+
+		IWL_DEBUG_WOWLAN(data->mld,
+				 "Updated GTK %d RSC (rsc_idx %d) on queue %d\n",
+				 key->keyidx, rsc_idx, data->queue);
+	}
+}
+
+void
+iwl_mld_process_rsc_notification(struct iwl_mld *mld,
+				 struct ieee80211_vif *vif,
+				 const struct iwl_wowlan_all_rsc_tsc_v5 *notif,
+				 int queue)
+{
+	struct iwl_mld_rsc_resume_iter_data iter_data = {
+		.mld = mld,
+		.notif = notif,
+		.queue = queue,
+	};
+
+	/* Iterate through all active keys and update RSC */
+	ieee80211_iter_keys_rcu(mld->hw, vif,
+				iwl_mld_rsc_update_key_iter,
+				&iter_data);
+}
+
 static void
 iwl_mld_add_mcast_rekey(struct ieee80211_vif *vif,
 			struct iwl_mld *mld,
@@ -951,19 +1073,25 @@ iwl_mld_add_mcast_rekey(struct ieee80211_vif *vif,
 
 	iwl_mld_update_mcast_rx_seq(key_config, key_data);
 
-	/* The FW holds only one igtk so we keep track of the valid one */
+	/* The FW holds only one IGTK so we keep track of the valid one */
 	if (key_config->keyidx == 4 || key_config->keyidx == 5) {
-		struct iwl_mld_link *mld_link =
-			iwl_mld_link_from_mac80211(link_conf);
+		struct iwl_mld_link_sta *mld_ap_link_sta;
+
+		mld_ap_link_sta = iwl_mld_get_ap_link_sta(vif,
+							  link_conf->link_id);
+		if (WARN_ON(!mld_ap_link_sta))
+			return;
 
 		/* If we had more than one rekey, mac80211 will tell us to
 		 * remove the old and add the new so we will update the IGTK in
 		 * drv_set_key
 		 */
-		if (mld_link->igtk && mld_link->igtk != key_config) {
+		if (mld_ap_link_sta->rx_igtk &&
+		    mld_ap_link_sta->rx_igtk != key_config) {
 			/* mark the old IGTK as not in FW */
-			mld_link->igtk->hw_key_idx = STA_KEY_IDX_INVALID;
-			mld_link->igtk = key_config;
+			mld_ap_link_sta->rx_igtk->hw_key_idx =
+				STA_KEY_IDX_INVALID;
+			mld_ap_link_sta->rx_igtk = key_config;
 		}
 	}
 
@@ -1930,12 +2058,12 @@ int iwl_mld_wowlan_suspend(struct iwl_mld *mld, struct cfg80211_wowlan *wowlan)
 	if (WARN_ON(!wowlan))
 		return 1;
 
-	IWL_DEBUG_WOWLAN(mld, "Starting the wowlan suspend flow\n");
-
 	bss_vif = iwl_mld_get_bss_vif(mld);
-	if (WARN_ON(!bss_vif))
+	if (!bss_vif)
 		return 1;
 
+	IWL_DEBUG_WOWLAN(mld, "Starting the wowlan suspend flow\n");
+
 	if (!bss_vif->cfg.assoc) {
 		int ret;
 		/* If we're not associated, this must be netdetect */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/d3.h b/drivers/net/wireless/intel/iwlwifi/mld/d3.h
index 618d6fb3c7964..c2e8ba877042f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/d3.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/d3.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024 Intel Corporation
+ * Copyright (C) 2024, 2026 Intel Corporation
  */
 #ifndef __iwl_mld_d3_h__
 #define __iwl_mld_d3_h__
@@ -42,6 +42,10 @@ int iwl_mld_wowlan_resume(struct iwl_mld *mld);
 void iwl_mld_set_rekey_data(struct ieee80211_hw *hw,
 			    struct ieee80211_vif *vif,
 			    struct cfg80211_gtk_rekey_data *data);
+void iwl_mld_process_rsc_notification(struct iwl_mld *mld,
+				      struct ieee80211_vif *vif,
+				      const struct iwl_wowlan_all_rsc_tsc_v5 *notif,
+				      int queue);
 #if IS_ENABLED(CONFIG_IPV6)
 void iwl_mld_ipv6_addr_change(struct ieee80211_hw *hw,
 			      struct ieee80211_vif *vif,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c b/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c
index b05b58eb12813..351a4f177e920 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/debugfs.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #include "mld.h"
@@ -979,15 +979,39 @@ void iwl_mld_add_vif_debugfs(struct ieee80211_hw *hw,
 	VIF_DEBUGFS_ADD_FILE(twt_operation, mld_vif_dbgfs, 0200);
 	VIF_DEBUGFS_ADD_FILE(int_mlo_scan, mld_vif_dbgfs, 0200);
 }
-#define LINK_DEBUGFS_WRITE_FILE_OPS(name, bufsz)			\
+
+#define LINK_DEBUGFS_WIPHY_WRITE_FILE_OPS(name, bufsz)			\
 	WIPHY_DEBUGFS_WRITE_FILE_OPS(link_##name, bufsz, bss_conf)
 
+/*
+ * Note: no locking is provided, so the function must have its own,
+ * but it cannot acquire the wiphy mutex.
+ */
+#define LINK_DEBUGFS_READ_FILE_OPS(name, bufsz)			\
+	_MLD_DEBUGFS_READ_FILE_OPS(link_##name, bufsz, struct ieee80211_bss_conf)
+
 #define LINK_DEBUGFS_ADD_FILE_ALIAS(alias, name, parent, mode)		\
 	debugfs_create_file(alias, mode, parent, link_conf,		\
 			    &iwl_dbgfs_link_##name##_ops)
 #define LINK_DEBUGFS_ADD_FILE(name, parent, mode)			\
 	LINK_DEBUGFS_ADD_FILE_ALIAS(#name, name, parent, mode)
 
+static ssize_t iwl_dbgfs_link_fw_id_read(struct ieee80211_bss_conf *link_conf,
+					 size_t buflen, void *buf)
+{
+	struct iwl_mld_link *mld_link;
+
+	guard(rcu)();
+
+	mld_link = iwl_mld_link_from_mac80211(link_conf);
+	if (!mld_link)
+		return -EINVAL;
+
+	return scnprintf(buf, buflen, "%d\n", mld_link->fw_id);
+}
+
+LINK_DEBUGFS_READ_FILE_OPS(fw_id, 64);
+
 void iwl_mld_add_link_debugfs(struct ieee80211_hw *hw,
 			      struct ieee80211_vif *vif,
 			      struct ieee80211_bss_conf *link_conf,
@@ -1008,6 +1032,8 @@ void iwl_mld_add_link_debugfs(struct ieee80211_hw *hw,
 		/* Release the reference from debugfs_lookup */
 		dput(mld_link_dir);
 	}
+
+	LINK_DEBUGFS_ADD_FILE(fw_id, mld_link_dir, 0400);
 }
 
 static ssize_t _iwl_dbgfs_fixed_rate_write(struct iwl_mld *mld, char *buf,
@@ -1035,7 +1061,6 @@ static ssize_t _iwl_dbgfs_fixed_rate_write(struct iwl_mld *mld, char *buf,
 
 	/* input is in FW format (v2 or v3) so convert to v3 */
 	rate = iwl_v3_rate_from_v2_v3(cpu_to_le32(rate), v3);
-	rate = le32_to_cpu(iwl_v3_rate_to_v2_v3(rate, mld->fw_rates_ver_3));
 
 	ret = iwl_mld_send_tlc_dhc(mld, fw_sta_id,
 				   partial ? IWL_TLC_DEBUG_PARTIAL_FIXED_RATE :
@@ -1050,20 +1075,22 @@ static ssize_t _iwl_dbgfs_fixed_rate_write(struct iwl_mld *mld, char *buf,
 	return ret ? : count;
 }
 
-static ssize_t iwl_dbgfs_fixed_rate_write(struct iwl_mld *mld, char *buf,
-					  size_t count, void *data)
+static ssize_t
+iwl_dbgfs_link_sta_fixed_rate_write(struct iwl_mld *mld, char *buf,
+				    size_t count, void *data)
 {
 	return _iwl_dbgfs_fixed_rate_write(mld, buf, count, data, false);
 }
 
-static ssize_t iwl_dbgfs_fixed_rate_v3_write(struct iwl_mld *mld, char *buf,
-					     size_t count, void *data)
+static ssize_t
+iwl_dbgfs_link_sta_fixed_rate_v3_write(struct iwl_mld *mld, char *buf,
+				       size_t count, void *data)
 {
 	return _iwl_dbgfs_fixed_rate_write(mld, buf, count, data, true);
 }
 
-static ssize_t iwl_dbgfs_tlc_dhc_write(struct iwl_mld *mld, char *buf,
-				       size_t count, void *data)
+static ssize_t iwl_dbgfs_link_sta_tlc_dhc_write(struct iwl_mld *mld, char *buf,
+						size_t count, void *data)
 {
 	struct ieee80211_link_sta *link_sta = data;
 	struct iwl_mld_link_sta *mld_link_sta;
@@ -1090,18 +1117,42 @@ static ssize_t iwl_dbgfs_tlc_dhc_write(struct iwl_mld *mld, char *buf,
 	return ret ? : count;
 }
 
+static ssize_t
+iwl_dbgfs_link_sta_fw_id_read(struct ieee80211_link_sta *link_sta,
+			      size_t buflen, void *buf)
+{
+	struct iwl_mld_link_sta *mld_link_sta;
+
+	guard(rcu)();
+
+	mld_link_sta = iwl_mld_link_sta_from_mac80211(link_sta);
+	if (!mld_link_sta)
+		return -EINVAL;
+
+	return scnprintf(buf, buflen, "%u\n", mld_link_sta->fw_id);
+}
+
 #define LINK_STA_DEBUGFS_ADD_FILE_ALIAS(alias, name, parent, mode)	\
 	debugfs_create_file(alias, mode, parent, link_sta,		\
-			    &iwl_dbgfs_##name##_ops)
+			    &iwl_dbgfs_link_sta_##name##_ops)
 #define LINK_STA_DEBUGFS_ADD_FILE(name, parent, mode)			\
 	LINK_STA_DEBUGFS_ADD_FILE_ALIAS(#name, name, parent, mode)
 
 #define LINK_STA_WIPHY_DEBUGFS_WRITE_OPS(name, bufsz)			\
-	WIPHY_DEBUGFS_WRITE_FILE_OPS(name, bufsz, link_sta)
+	WIPHY_DEBUGFS_WRITE_FILE_OPS(link_sta_##name, bufsz, link_sta)
+
+/*
+ * Note: no locking is provided, so the function must have its own,
+ * but it cannot acquire the wiphy mutex.
+ */
+#define LINK_STA_DEBUGFS_READ_OPS(name, bufsz)				\
+	_MLD_DEBUGFS_READ_FILE_OPS(link_sta_##name, bufsz,		\
+				   struct ieee80211_link_sta)
 
 LINK_STA_WIPHY_DEBUGFS_WRITE_OPS(tlc_dhc, 64);
 LINK_STA_WIPHY_DEBUGFS_WRITE_OPS(fixed_rate, 64);
 LINK_STA_WIPHY_DEBUGFS_WRITE_OPS(fixed_rate_v3, 64);
+LINK_STA_DEBUGFS_READ_OPS(fw_id, 64);
 
 void iwl_mld_add_link_sta_debugfs(struct ieee80211_hw *hw,
 				  struct ieee80211_vif *vif,
@@ -1111,4 +1162,5 @@ void iwl_mld_add_link_sta_debugfs(struct ieee80211_hw *hw,
 	LINK_STA_DEBUGFS_ADD_FILE(fixed_rate, dir, 0200);
 	LINK_STA_DEBUGFS_ADD_FILE(fixed_rate_v3, dir, 0200);
 	LINK_STA_DEBUGFS_ADD_FILE(tlc_dhc, dir, 0200);
+	LINK_STA_DEBUGFS_ADD_FILE(fw_id, dir, 0400);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.c b/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.c
index 3464b32687124..81df3fdfcbf56 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/ftm-initiator.c
@@ -71,24 +71,34 @@ iwl_mld_ftm_set_target_chandef(struct iwl_mld *mld,
 
 	switch (peer->chandef.width) {
 	case NL80211_CHAN_WIDTH_20_NOHT:
-		target->format_bw = IWL_LOCATION_FRAME_FORMAT_LEGACY;
-		target->format_bw |= IWL_LOCATION_BW_20MHZ << LOCATION_BW_POS;
+		target->format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_LEGACY,
+						   IWL_LOCATION_FMT_BW_FORMAT);
+		target->format_bw |= u8_encode_bits(IWL_LOCATION_BW_20MHZ,
+						    IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_20:
-		target->format_bw = IWL_LOCATION_FRAME_FORMAT_HT;
-		target->format_bw |= IWL_LOCATION_BW_20MHZ << LOCATION_BW_POS;
+		target->format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_HT,
+						   IWL_LOCATION_FMT_BW_FORMAT);
+		target->format_bw |= u8_encode_bits(IWL_LOCATION_BW_20MHZ,
+						    IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_40:
-		target->format_bw = IWL_LOCATION_FRAME_FORMAT_HT;
-		target->format_bw |= IWL_LOCATION_BW_40MHZ << LOCATION_BW_POS;
+		target->format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_HT,
+						   IWL_LOCATION_FMT_BW_FORMAT);
+		target->format_bw |= u8_encode_bits(IWL_LOCATION_BW_40MHZ,
+						    IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_80:
-		target->format_bw = IWL_LOCATION_FRAME_FORMAT_VHT;
-		target->format_bw |= IWL_LOCATION_BW_80MHZ << LOCATION_BW_POS;
+		target->format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_VHT,
+						   IWL_LOCATION_FMT_BW_FORMAT);
+		target->format_bw |= u8_encode_bits(IWL_LOCATION_BW_80MHZ,
+						    IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_160:
-		target->format_bw = IWL_LOCATION_FRAME_FORMAT_HE;
-		target->format_bw |= IWL_LOCATION_BW_160MHZ << LOCATION_BW_POS;
+		target->format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_HE,
+						   IWL_LOCATION_FMT_BW_FORMAT);
+		target->format_bw |= u8_encode_bits(IWL_LOCATION_BW_160MHZ,
+						    IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	default:
 		IWL_ERR(mld, "Unsupported BW in FTM request (%d)\n",
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/iface.c b/drivers/net/wireless/intel/iwlwifi/mld/iface.c
index 52b2ae496811a..2b837c6fa5fee 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/iface.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/iface.c
@@ -1,10 +1,11 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include <net/cfg80211.h>
 
 #include "iface.h"
+#include "nan.h"
 #include "hcmd.h"
 #include "key.h"
 #include "mlo.h"
@@ -55,6 +56,24 @@ void iwl_mld_cleanup_vif(void *data, u8 *mac, struct ieee80211_vif *vif)
 
 	ieee80211_iter_keys(mld->hw, vif, iwl_mld_cleanup_keys_iter, NULL);
 
+	if (vif->type == NL80211_IFTYPE_NAN) {
+		mld_vif->nan.mac_added = false;
+		/* Clean up NAN links */
+		for (int i = 0; i < ARRAY_SIZE(mld_vif->nan.links); i++)
+			iwl_mld_cleanup_nan_link(&mld_vif->nan.links[i]);
+
+		if (mld_vif->nan.bcast_sta.sta_id != IWL_INVALID_STA)
+			iwl_mld_free_internal_sta(mld, &mld_vif->nan.bcast_sta);
+		if (mld_vif->nan.mgmt_sta.sta_id != IWL_INVALID_STA)
+			iwl_mld_free_internal_sta(mld, &mld_vif->nan.mgmt_sta);
+
+		mld_vif->nan.tx_igtk = NULL;
+	}
+
+	if (vif->type == NL80211_IFTYPE_NAN_DATA &&
+	    mld_vif->nan.mcast_data_sta.sta_id != IWL_INVALID_STA)
+		iwl_mld_free_internal_sta(mld, &mld_vif->nan.mcast_data_sta);
+
 	CLEANUP_STRUCT(mld_vif);
 }
 
@@ -94,6 +113,8 @@ static int iwl_mld_mac80211_iftype_to_fw(const struct ieee80211_vif *vif)
 		return FW_MAC_TYPE_P2P_DEVICE;
 	case NL80211_IFTYPE_ADHOC:
 		return FW_MAC_TYPE_IBSS;
+	case NL80211_IFTYPE_NAN:
+		return FW_MAC_TYPE_NAN;
 	default:
 		WARN_ON_ONCE(1);
 	}
@@ -257,7 +278,7 @@ static void iwl_mld_fill_mac_cmd_sta(struct iwl_mld *mld,
 				     IEEE80211_EML_CAP_TRANSITION_TIMEOUT);
 
 		cmd->client.esr_transition_timeout =
-			min_t(u16, IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU,
+			min_t(u16, IEEE80211_EML_CAP_TRANSITION_TIMEOUT_64TU,
 			      esr_transition_timeout);
 		cmd->client.medium_sync_delay =
 			cpu_to_le16(vif->cfg.eml_med_sync_delay);
@@ -362,6 +383,42 @@ static void iwl_mld_fill_mac_cmd_ibss(struct iwl_mld *mld,
 					 MAC_CFG_FILTER_ACCEPT_GRP);
 }
 
+static int iwl_mld_fill_mac_cmd_nan(struct iwl_mld *mld,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_vif *ndi_being_added,
+				    struct iwl_mac_config_cmd *cmd)
+{
+	struct ieee80211_vif *iter;
+	u32 idx = 0;
+
+	cmd->filter_flags = cpu_to_le32(MAC_CFG_FILTER_ACCEPT_CONTROL_AND_MGMT);
+
+	/*
+	 * A NAN_DATA vif might be in the process of being added - it won't
+	 * be found by the iteration below since it's not yet active/in-driver.
+	 * In hw restart, the iteration below will find the ndi_being_added.
+	 */
+	if (ndi_being_added && !mld->fw_status.in_hw_restart) {
+		memcpy(cmd->nan.ndi_addrs[idx].addr, ndi_being_added->addr, ETH_ALEN);
+		idx++;
+	}
+
+	for_each_active_interface(iter, mld->hw) {
+		if (iter->type != NL80211_IFTYPE_NAN_DATA)
+			continue;
+
+		if (WARN_ON_ONCE(idx >= ARRAY_SIZE(cmd->nan.ndi_addrs)))
+			return -EINVAL;
+
+		memcpy(cmd->nan.ndi_addrs[idx].addr, iter->addr, ETH_ALEN);
+		idx++;
+	}
+
+	cmd->nan.ndi_addrs_count = cpu_to_le32(idx);
+
+	return 0;
+}
+
 static int
 iwl_mld_rm_mac_from_fw(struct iwl_mld *mld, struct ieee80211_vif *vif)
 {
@@ -374,16 +431,23 @@ iwl_mld_rm_mac_from_fw(struct iwl_mld *mld, struct ieee80211_vif *vif)
 	return iwl_mld_send_mac_cmd(mld, &cmd);
 }
 
-int iwl_mld_mac_fw_action(struct iwl_mld *mld, struct ieee80211_vif *vif,
-			  u32 action)
+static int
+__iwl_mld_mac_fw_action(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			u32 action, struct ieee80211_vif *ndi_being_added)
 {
 	struct iwl_mac_config_cmd cmd = {};
+	int ret;
 
 	lockdep_assert_wiphy(mld->wiphy);
 
-	/* NAN interface type is not known to FW */
-	if (vif->type == NL80211_IFTYPE_NAN)
-		return 0;
+	/* NAN_DATA interface type is not known to FW */
+	if (WARN_ON(vif->type == NL80211_IFTYPE_NAN_DATA))
+		return -EINVAL;
+
+	/* ndi_being_added is only relevant for NAN and when adding a NAN_DATA interface */
+	if (WARN_ON(ndi_being_added &&
+		    (vif->type != NL80211_IFTYPE_NAN || action != FW_CTXT_ACTION_MODIFY)))
+		return -EINVAL;
 
 	if (action == FW_CTXT_ACTION_REMOVE)
 		return iwl_mld_rm_mac_from_fw(mld, vif);
@@ -411,6 +475,11 @@ int iwl_mld_mac_fw_action(struct iwl_mld *mld, struct ieee80211_vif *vif,
 	case NL80211_IFTYPE_ADHOC:
 		iwl_mld_fill_mac_cmd_ibss(mld, vif, &cmd);
 		break;
+	case NL80211_IFTYPE_NAN:
+		ret = iwl_mld_fill_mac_cmd_nan(mld, vif, ndi_being_added, &cmd);
+		if (ret)
+			return ret;
+		break;
 	default:
 		WARN(1, "not supported yet\n");
 		return -EOPNOTSUPP;
@@ -419,6 +488,12 @@ int iwl_mld_mac_fw_action(struct iwl_mld *mld, struct ieee80211_vif *vif,
 	return iwl_mld_send_mac_cmd(mld, &cmd);
 }
 
+int iwl_mld_mac_fw_action(struct iwl_mld *mld, struct ieee80211_vif *vif,
+			  u32 action)
+{
+	return __iwl_mld_mac_fw_action(mld, vif, action, NULL);
+}
+
 static void iwl_mld_mlo_scan_start_wk(struct wiphy *wiphy,
 				      struct wiphy_work *wk)
 {
@@ -430,7 +505,7 @@ static void iwl_mld_mlo_scan_start_wk(struct wiphy *wiphy,
 	iwl_mld_int_mlo_scan(mld, iwl_mld_vif_to_mac80211(mld_vif));
 }
 
-static IWL_MLD_ALLOC_FN(vif, vif)
+IWL_MLD_ALLOC_FN_STATIC(vif, vif)
 
 /* Constructor function for struct iwl_mld_vif */
 static void
@@ -456,9 +531,40 @@ iwl_mld_init_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
 		wiphy_delayed_work_init(&mld_vif->mlo_scan_start_wk,
 					iwl_mld_mlo_scan_start_wk);
 	}
+
+	if (vif->type == NL80211_IFTYPE_NAN) {
+		for (int i = 0; i < ARRAY_SIZE(mld_vif->nan.links); i++) {
+			memset(&mld_vif->nan.links[i], 0, sizeof(mld_vif->nan.links[i]));
+			mld_vif->nan.links[i].fw_id = FW_CTXT_ID_INVALID;
+		}
+
+		iwl_mld_init_internal_sta(&mld_vif->nan.bcast_sta);
+		iwl_mld_init_internal_sta(&mld_vif->nan.mgmt_sta);
+	} else if (vif->type == NL80211_IFTYPE_NAN_DATA) {
+		iwl_mld_init_internal_sta(&mld_vif->nan.mcast_data_sta);
+	}
+
 	iwl_mld_init_internal_sta(&mld_vif->aux_sta);
 }
 
+static int iwl_mld_update_nan_mac(struct iwl_mld *mld,
+				  struct ieee80211_vif *ndi_being_added)
+{
+	struct ieee80211_vif *vif = mld->nan_device_vif;
+	struct iwl_mld_vif *mld_vif;
+
+	if (WARN_ON_ONCE(!vif))
+		return -ENODEV;
+
+	mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	if (!iwl_mld_vif_fw_id_valid(mld_vif))
+		return 0;
+
+	return __iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_MODIFY,
+				       ndi_being_added);
+}
+
 int iwl_mld_add_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
 {
 	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
@@ -468,10 +574,14 @@ int iwl_mld_add_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
 
 	iwl_mld_init_vif(mld, vif);
 
-	/* NAN interface type is not known to FW */
+	/* NAN MACs are added to FW only when a schedule is set */
 	if (vif->type == NL80211_IFTYPE_NAN)
 		return 0;
 
+	/* NAN_DATA interface type is not known to FW, but we need to update NAN MAC */
+	if (vif->type == NL80211_IFTYPE_NAN_DATA)
+		return iwl_mld_update_nan_mac(mld, vif);
+
 	ret = iwl_mld_allocate_vif_fw_id(mld, &mld_vif->fw_id, vif);
 	if (ret)
 		return ret;
@@ -483,23 +593,52 @@ int iwl_mld_add_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
 	return ret;
 }
 
+int iwl_mld_add_nan_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_NAN))
+		return -EINVAL;
+
+	ret = iwl_mld_allocate_vif_fw_id(mld, &mld_vif->fw_id, vif);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_ADD);
+	if (ret) {
+		RCU_INIT_POINTER(mld->fw_id_to_vif[mld_vif->fw_id], NULL);
+		return ret;
+	}
+
+	mld_vif->nan.mac_added = true;
+
+	return 0;
+}
+
 void iwl_mld_rm_vif(struct iwl_mld *mld, struct ieee80211_vif *vif)
 {
 	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
 
 	lockdep_assert_wiphy(mld->wiphy);
 
-	/* NAN interface type is not known to FW */
-	if (vif->type == NL80211_IFTYPE_NAN)
+	if (vif->type == NL80211_IFTYPE_NAN_DATA) {
+		iwl_mld_update_nan_mac(mld, NULL);
 		return;
+	}
 
-	iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_REMOVE);
-
-	if (WARN_ON(mld_vif->fw_id >= ARRAY_SIZE(mld->fw_id_to_vif)))
+	if (!iwl_mld_vif_fw_id_valid(mld_vif))
 		return;
 
+	iwl_mld_mac_fw_action(mld, vif, FW_CTXT_ACTION_REMOVE);
+
 	RCU_INIT_POINTER(mld->fw_id_to_vif[mld_vif->fw_id], NULL);
 
+	if (vif->type == NL80211_IFTYPE_NAN)
+		mld_vif->nan.mac_added = false;
+
 	iwl_mld_cancel_notifications_of_object(mld, IWL_MLD_OBJECT_TYPE_VIF,
 					       mld_vif->fw_id);
 }
@@ -616,24 +755,6 @@ void iwl_mld_handle_probe_resp_data_notif(struct iwl_mld *mld,
 		kfree_rcu(old_data, rcu_head);
 }
 
-void iwl_mld_handle_uapsd_misbehaving_ap_notif(struct iwl_mld *mld,
-					       struct iwl_rx_packet *pkt)
-{
-	struct iwl_uapsd_misbehaving_ap_notif *notif = (void *)pkt->data;
-	struct ieee80211_vif *vif;
-
-	if (IWL_FW_CHECK(mld, notif->mac_id >= ARRAY_SIZE(mld->fw_id_to_vif),
-			 "mac id is invalid: %d\n", notif->mac_id))
-		return;
-
-	vif = wiphy_dereference(mld->wiphy, mld->fw_id_to_vif[notif->mac_id]);
-
-	if (WARN_ON(!vif) || ieee80211_vif_is_mld(vif))
-		return;
-
-	IWL_WARN(mld, "uapsd misbehaving AP: %pM\n", vif->bss_conf.bssid);
-}
-
 void iwl_mld_handle_datapath_monitor_notif(struct iwl_mld *mld,
 					   struct iwl_rx_packet *pkt)
 {
@@ -748,6 +869,6 @@ struct ieee80211_vif *iwl_mld_get_bss_vif(struct iwl_mld *mld)
 
 	fw_id = __ffs(fw_id_bitmap);
 
-	return wiphy_dereference(mld->wiphy,
-				 mld->fw_id_to_vif[fw_id]);
+	return rcu_dereference_wiphy(mld->wiphy,
+				     mld->fw_id_to_vif[fw_id]);
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/iface.h b/drivers/net/wireless/intel/iwlwifi/mld/iface.h
index 8dfc79fed2534..bc6f45ff76f88 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/iface.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/iface.h
@@ -8,6 +8,7 @@
 #include <net/mac80211.h>
 
 #include "link.h"
+#include "nan.h"
 #include "session-protect.h"
 #include "d3.h"
 #include "fw/api/time-event.h"
@@ -137,7 +138,6 @@ struct iwl_mld_emlsr {
  * @beacon_inject_active: indicates an active debugfs beacon ie injection
  * @low_latency_causes: bit flags, indicating the causes for low-latency,
  *	see @iwl_mld_low_latency_cause.
- * @ps_disabled: indicates that PS is disabled for this interface
  * @last_link_activation_time: last time a link was activated, for
  *	deferring MLO scans (to make them more reliable)
  * @mld: pointer to the mld structure.
@@ -152,6 +152,16 @@ struct iwl_mld_emlsr {
  *	p2p device only. Set to %ROC_NUM_ACTIVITIES when not in use.
  * @aux_sta: station used for remain on channel. Used in P2P device.
  * @mlo_scan_start_wk: worker to start a deferred MLO scan
+ * @nan: NAN parameters
+ * @nan.links: NAN links for FW (indexed by FW link ID)
+ * @nan.mac_added: track whether or not the MAC was added to FW
+ * @nan.bcast_sta: internal station used for NAN synchronization and discovery
+ *	activities. No queue is associated with it.
+ * @nan.mgmt_sta: internal station used for NAN management frames, e.g., SDFs
+ *	and NAFs.
+ * @nan.mcast_data_sta: internal station used for multicast NAN Data frames.
+ * @nan.tx_igtk: TX IGTK key for NAN, tracked separately since NAN does not
+ *	use the vif links.
  */
 struct iwl_mld_vif {
 	/* Add here fields that need clean up on restart */
@@ -167,7 +177,6 @@ struct iwl_mld_vif {
 		bool beacon_inject_active;
 #endif
 		u8 low_latency_causes;
-		bool ps_disabled;
 		time64_t last_link_activation_time;
 	);
 	/* And here fields that survive a fw restart */
@@ -175,6 +184,16 @@ struct iwl_mld_vif {
 	struct iwl_mld_link deflink;
 	struct iwl_mld_link __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS];
 
+	struct {
+		/* use only with wiphy protection */
+		struct iwl_mld_nan_link links[IWL_FW_MAX_LINKS];
+		bool mac_added;
+		struct iwl_mld_int_sta bcast_sta;
+		struct iwl_mld_int_sta mgmt_sta;
+		struct iwl_mld_int_sta mcast_data_sta;
+		struct ieee80211_key_conf *tx_igtk;
+	} nan;
+
 	struct iwl_mld_emlsr emlsr;
 
 #ifdef CONFIG_PM_SLEEP
@@ -206,6 +225,20 @@ iwl_mld_vif_to_mac80211(struct iwl_mld_vif *mld_vif)
 /* Call only for interfaces that were added to the driver! */
 static inline bool iwl_mld_vif_fw_id_valid(struct iwl_mld_vif *mld_vif)
 {
+	struct ieee80211_vif *vif = iwl_mld_vif_to_mac80211(mld_vif);
+
+	switch (vif->type) {
+	case NL80211_IFTYPE_NAN_DATA:
+		return false;
+	case NL80211_IFTYPE_NAN:
+		if (!mld_vif->nan.mac_added)
+			return false;
+		break;
+	default:
+		break;
+	}
+
+	/* Should be added to FW */
 	if (WARN_ON(mld_vif->fw_id >= ARRAY_SIZE(mld_vif->mld->fw_id_to_vif)))
 		return false;
 
@@ -221,6 +254,12 @@ static inline bool iwl_mld_vif_fw_id_valid(struct iwl_mld_vif *mld_vif)
 	     link_id++)							\
 		if ((mld_link = iwl_mld_link_dereference_check(mld_vif, link_id)))
 
+#define for_each_mld_nan_valid_link(mld_vif, nan_link)			\
+	for (nan_link = &(mld_vif)->nan.links[0];			\
+	     nan_link < &(mld_vif)->nan.links[ARRAY_SIZE((mld_vif)->nan.links)]; \
+	     nan_link++)						\
+		if (nan_link->fw_id != FW_CTXT_ID_INVALID)
+
 /* Retrieve pointer to mld link from mac80211 structures */
 static inline struct iwl_mld_link *
 iwl_mld_link_from_mac80211(struct ieee80211_bss_conf *bss_conf)
@@ -235,6 +274,7 @@ void iwl_mld_cleanup_vif(void *data, u8 *mac, struct ieee80211_vif *vif);
 int iwl_mld_mac_fw_action(struct iwl_mld *mld, struct ieee80211_vif *vif,
 			  u32 action);
 int iwl_mld_add_vif(struct iwl_mld *mld, struct ieee80211_vif *vif);
+int iwl_mld_add_nan_vif(struct iwl_mld *mld, struct ieee80211_vif *vif);
 void iwl_mld_rm_vif(struct iwl_mld *mld, struct ieee80211_vif *vif);
 void iwl_mld_set_vif_associated(struct iwl_mld *mld,
 				struct ieee80211_vif *vif);
@@ -245,9 +285,6 @@ void iwl_mld_handle_probe_resp_data_notif(struct iwl_mld *mld,
 void iwl_mld_handle_datapath_monitor_notif(struct iwl_mld *mld,
 					   struct iwl_rx_packet *pkt);
 
-void iwl_mld_handle_uapsd_misbehaving_ap_notif(struct iwl_mld *mld,
-					       struct iwl_rx_packet *pkt);
-
 void iwl_mld_reset_cca_40mhz_workaround(struct iwl_mld *mld,
 					struct ieee80211_vif *vif);
 
@@ -258,4 +295,19 @@ static inline bool iwl_mld_vif_low_latency(const struct iwl_mld_vif *mld_vif)
 
 struct ieee80211_vif *iwl_mld_get_bss_vif(struct iwl_mld *mld);
 
+static inline struct iwl_mld_link_sta *
+iwl_mld_get_ap_link_sta(struct ieee80211_vif *vif, int link_id)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct ieee80211_sta *ap_sta = mld_vif->ap_sta;
+	struct iwl_mld_sta *mld_ap_sta;
+
+	if (!ap_sta)
+		return NULL;
+
+	mld_ap_sta = iwl_mld_sta_from_mac80211(ap_sta);
+
+	return iwl_mld_link_sta_dereference_check(mld_ap_sta, link_id);
+}
+
 #endif /* __iwl_mld_iface_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/key.c b/drivers/net/wireless/intel/iwlwifi/mld/key.c
index 04192c5f07ffc..bf80b4770b5af 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/key.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/key.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024 Intel Corporation
+ * Copyright (C) 2024, 2026 Intel Corporation
  */
 #include "key.h"
 #include "iface.h"
@@ -12,7 +12,6 @@ static u32 iwl_mld_get_key_flags(struct iwl_mld *mld,
 				 struct ieee80211_sta *sta,
 				 struct ieee80211_key_conf *key)
 {
-	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
 	bool pairwise = key->flags & IEEE80211_KEY_FLAG_PAIRWISE;
 	bool igtk = key->keyidx == 4 || key->keyidx == 5;
 	u32 flags = 0;
@@ -38,9 +37,6 @@ static u32 iwl_mld_get_key_flags(struct iwl_mld *mld,
 		break;
 	}
 
-	if (!sta && vif->type == NL80211_IFTYPE_STATION)
-		sta = mld_vif->ap_sta;
-
 	/* If we are installing an iGTK (in AP or STA mode), we need to tell
 	 * the firmware this key will en/decrypt MGMT frames.
 	 * Same goes if we are installing a pairwise key for an MFP station.
@@ -53,6 +49,12 @@ static u32 iwl_mld_get_key_flags(struct iwl_mld *mld,
 	if (key->flags & IEEE80211_KEY_FLAG_SPP_AMSDU)
 		flags |= IWL_SEC_KEY_FLAG_SPP_AMSDU;
 
+	/* When a GTK is configured for a station, it can only be
+	 * used for Rx and never for Tx. Thus, set the NO TX flag.
+	 */
+	if (!pairwise && sta)
+		flags |= IWL_SEC_KEY_FLAG_NO_TX;
+
 	return flags;
 }
 
@@ -67,6 +69,40 @@ static u32 iwl_mld_get_key_sta_mask(struct iwl_mld *mld,
 
 	lockdep_assert_wiphy(mld->wiphy);
 
+	if (vif->type == NL80211_IFTYPE_NAN_DATA && !sta) {
+		/* Older firmware versions do not support transmission of
+		 * multicast data frames.
+		 */
+		if (!iwl_mld_nan_use_nan_stations(mld))
+			return 0;
+
+		if (WARN_ON(mld_vif->nan.mcast_data_sta.sta_id ==
+			    IWL_INVALID_STA))
+			return 0;
+
+		return BIT(mld_vif->nan.mcast_data_sta.sta_id);
+	}
+
+	if (vif->type == NL80211_IFTYPE_NAN && !sta) {
+		/* Older firmware versions do not support installation of
+		 * IGTK/BIGTK keys.
+		 */
+		if (!iwl_mld_nan_use_nan_stations(mld))
+			return 0;
+
+		if (WARN_ON(mld_vif->nan.bcast_sta.sta_id == IWL_INVALID_STA ||
+			    mld_vif->nan.mgmt_sta.sta_id == IWL_INVALID_STA))
+			return 0;
+
+		if (key->keyidx >= 4 && key->keyidx <= 5)
+			return BIT(mld_vif->nan.mgmt_sta.sta_id);
+
+		if (key->keyidx >= 6 && key->keyidx <= 7)
+			return BIT(mld_vif->nan.bcast_sta.sta_id);
+
+		return 0;
+	}
+
 	/* AP group keys are per link and should be on the mcast/bcast STA */
 	if (vif->type == NL80211_IFTYPE_AP &&
 	    !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE)) {
@@ -92,11 +128,7 @@ static u32 iwl_mld_get_key_sta_mask(struct iwl_mld *mld,
 		return BIT(link->mcast_sta.sta_id);
 	}
 
-	/* for client mode use the AP STA also for group keys */
-	if (!sta && vif->type == NL80211_IFTYPE_STATION)
-		sta = mld_vif->ap_sta;
-
-	/* STA should be non-NULL now */
+	/* STA should be non-NULL */
 	if (WARN_ON(!sta))
 		return 0;
 
@@ -178,34 +210,69 @@ static void iwl_mld_remove_key_from_fw(struct iwl_mld *mld, u32 sta_mask,
 	iwl_mld_send_cmd_pdu(mld, WIDE_ID(DATA_PATH_GROUP, SEC_KEY_CMD), &cmd);
 }
 
+static struct ieee80211_key_conf **
+iwl_mld_get_igtk_ptr(struct ieee80211_vif *vif,
+		     struct ieee80211_sta *sta,
+		     struct ieee80211_key_conf *key)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	/* key's link ID is set to -1 for non-MLO */
+	int link_id = key->link_id < 0 ? 0 : key->link_id;
+	struct iwl_mld_link_sta *mld_ap_link_sta;
+	struct iwl_mld_link *mld_link;
+	struct iwl_mld_sta *mld_sta;
+
+	if (key->keyidx != 4 && key->keyidx != 5)
+		return NULL;
+
+	switch (vif->type) {
+	case NL80211_IFTYPE_STATION:
+		if (WARN_ON(!sta))
+			return NULL;
+
+		mld_sta = iwl_mld_sta_from_mac80211(sta);
+		mld_ap_link_sta = iwl_mld_link_sta_dereference_check(mld_sta,
+								     link_id);
+		if (WARN_ON(!mld_ap_link_sta))
+			return NULL;
+
+		return &mld_ap_link_sta->rx_igtk;
+	case NL80211_IFTYPE_NAN:
+		if (sta) {
+			mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+			return &mld_sta->deflink.rx_igtk;
+		}
+
+		return &mld_vif->nan.tx_igtk;
+	case NL80211_IFTYPE_AP:
+		mld_link = iwl_mld_link_dereference_check(mld_vif, link_id);
+		if (WARN_ON(!mld_link))
+			return NULL;
+
+		return &mld_link->tx_igtk;
+	default:
+		WARN_ONCE(1, "invalid iftype %d for IGTK\n", vif->type);
+		return NULL;
+	}
+}
+
 void iwl_mld_remove_key(struct iwl_mld *mld, struct ieee80211_vif *vif,
 			struct ieee80211_sta *sta,
 			struct ieee80211_key_conf *key)
 {
 	u32 sta_mask = iwl_mld_get_key_sta_mask(mld, vif, sta, key);
 	u32 key_flags = iwl_mld_get_key_flags(mld, vif, sta, key);
-	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct ieee80211_key_conf **igtk_ptr;
 
 	lockdep_assert_wiphy(mld->wiphy);
 
 	if (!sta_mask)
 		return;
 
-	if (key->keyidx == 4 || key->keyidx == 5) {
-		struct iwl_mld_link *mld_link;
-		unsigned int link_id = 0;
-
-		/* set to -1 for non-MLO right now */
-		if (key->link_id >= 0)
-			link_id = key->link_id;
-
-		mld_link = iwl_mld_link_dereference_check(mld_vif, link_id);
-		if (WARN_ON(!mld_link))
-			return;
-
-		if (mld_link->igtk == key)
-			mld_link->igtk = NULL;
-
+	igtk_ptr = iwl_mld_get_igtk_ptr(vif, sta, key);
+	if (igtk_ptr && *igtk_ptr == key) {
+		*igtk_ptr = NULL;
 		mld->num_igtks--;
 	}
 
@@ -222,46 +289,43 @@ int iwl_mld_add_key(struct iwl_mld *mld,
 {
 	u32 sta_mask = iwl_mld_get_key_sta_mask(mld, vif, sta, key);
 	u32 key_flags = iwl_mld_get_key_flags(mld, vif, sta, key);
-	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
-	struct iwl_mld_link *mld_link = NULL;
-	bool igtk = key->keyidx == 4 || key->keyidx == 5;
+	struct ieee80211_key_conf **igtk_ptr;
 	int ret;
 
 	lockdep_assert_wiphy(mld->wiphy);
 
-	if (!sta_mask)
+	if (!sta_mask) {
+		/* for NAN (GTK) indicate SW-only, it's not used at all */
+		if (vif->type == NL80211_IFTYPE_NAN_DATA && !sta &&
+		    !iwl_mld_nan_use_nan_stations(mld))
+			return 1;
+
+		/* otherwise that's not valid */
+		IWL_WARN(mld, "empty STA mask for key %d\n", key->keyidx);
 		return -EINVAL;
+	}
 
-	if (igtk) {
-		if (mld->num_igtks == IWL_MAX_NUM_IGTKS)
+	igtk_ptr = iwl_mld_get_igtk_ptr(vif, sta, key);
+	if (igtk_ptr) {
+		if (mld->num_igtks == mld->fw->ucode_capa.num_mcast_key_entries)
 			return -EOPNOTSUPP;
 
-		u8 link_id = 0;
-
-		/* set to -1 for non-MLO right now */
-		if (key->link_id >= 0)
-			link_id = key->link_id;
-
-		mld_link = iwl_mld_link_dereference_check(mld_vif, link_id);
-
-		if (WARN_ON(!mld_link))
-			return -EINVAL;
-
-		if (mld_link->igtk) {
+		if (*igtk_ptr) {
 			IWL_DEBUG_MAC80211(mld, "remove old IGTK %d\n",
-					   mld_link->igtk->keyidx);
-			iwl_mld_remove_key(mld, vif, sta, mld_link->igtk);
+					   (*igtk_ptr)->keyidx);
+			iwl_mld_remove_key(mld, vif, sta, *igtk_ptr);
 		}
-
-		WARN_ON(mld_link->igtk);
 	}
 
 	ret = iwl_mld_add_key_to_fw(mld, sta_mask, key_flags, key);
-	if (ret)
+	if (ret) {
+		IWL_WARN(mld, "failed to add key to FW (%d)\n", ret);
 		return ret;
+	}
 
-	if (mld_link) {
-		mld_link->igtk = key;
+	if (igtk_ptr) {
+		WARN_ON(*igtk_ptr);
+		*igtk_ptr = key;
 		mld->num_igtks++;
 	}
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/link.c b/drivers/net/wireless/intel/iwlwifi/mld/link.c
index b66e84d2365fb..234821f6a4414 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/link.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/link.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #include "constants.h"
@@ -16,14 +16,34 @@
 #include "fw/api/context.h"
 #include "fw/dbg.h"
 
-static int iwl_mld_send_link_cmd(struct iwl_mld *mld,
-				 struct iwl_link_config_cmd *cmd,
-				 enum iwl_ctxt_action action)
+/**
+ * struct iwl_mld_link_chan_load_threshold - channel load thresholds
+ * @high_lim: level up transition thresholds, in percentage
+ * @low_lim: level down transition thresholds, in percentage
+ */
+struct iwl_mld_link_chan_load_threshold {
+	u8 high_lim;
+	u8 low_lim;
+};
+
+static const struct iwl_mld_link_chan_load_threshold
+link_chan_load_thresh_tbl[] = {
+	[LINK_CHAN_LOAD_LVL1] = { .high_lim = 45, .low_lim = 40 },
+	[LINK_CHAN_LOAD_LVL2] = { .high_lim = 70, .low_lim = 65 },
+	[LINK_CHAN_LOAD_LVL3] = { .high_lim = 85, .low_lim = 80 },
+};
+
+int iwl_mld_send_link_cmd(struct iwl_mld *mld,
+			  struct iwl_link_config_cmd *cmd,
+			  enum iwl_ctxt_action action)
 {
 	int ret;
 
 	lockdep_assert_wiphy(mld->wiphy);
 
+	if (WARN_ON_ONCE(cmd->link_id == cpu_to_le32(FW_CTXT_ID_INVALID)))
+		return -EINVAL;
+
 	cmd->action = cpu_to_le32(action);
 	ret = iwl_mld_send_cmd_pdu(mld,
 				   WIDE_ID(MAC_CONF_GROUP, LINK_CONFIG_CMD),
@@ -315,10 +335,28 @@ iwl_mld_change_link_in_fw(struct iwl_mld *mld, struct ieee80211_bss_conf *link,
 			link_sta_dereference_check(mld_vif->ap_sta,
 						   link->link_id);
 
-		if (!WARN_ON(!link_sta) && link_sta->he_cap.has_he &&
+		if (WARN_ON(!link_sta))
+			return -EINVAL;
+
+		if (link_sta->he_cap.has_he &&
 		    link_sta->he_cap.he_cap_elem.mac_cap_info[5] &
 		    IEEE80211_HE_MAC_CAP5_OM_CTRL_UL_MU_DATA_DIS_RX)
 			cmd.ul_mu_data_disable = 1;
+
+		if (link_sta->uhr_cap.has_uhr &&
+		    link_sta->uhr_cap.mac.mac_cap[0] &
+				IEEE80211_UHR_MAC_CAP0_DPS_ASSIST_SUPP)
+			flags |= LINK_FLG_DPS;
+
+		if (link_sta->uhr_cap.has_uhr &&
+		    link_sta->uhr_cap.mac.mac_cap[1] &
+				IEEE80211_UHR_MAC_CAP1_DUO_SUPP)
+			flags |= LINK_FLG_DUO;
+
+		if (link_sta->uhr_cap.has_uhr &&
+		    mld_vif->ap_sta->ext_mld_capa_ops &
+				IEEE80211_UHR_ML_EXT_MLD_CAPA_ML_PM)
+			flags |= LINK_FLG_MLPM;
 	}
 
 	cmd.htc_trig_based_pkt_ext = link->htc_trig_based_pkt_ext;
@@ -354,6 +392,21 @@ iwl_mld_change_link_in_fw(struct iwl_mld *mld, struct ieee80211_bss_conf *link,
 	if (WARN_ON(changes & LINK_CONTEXT_MODIFY_EHT_PARAMS))
 		changes &= ~LINK_CONTEXT_MODIFY_EHT_PARAMS;
 
+	if (link->uhr_support && link->npca.enabled) {
+		flags |= LINK_FLG_NPCA;
+		if (link->npca.moplen)
+			cmd.npca_params.flags |= IWL_NPCA_FLAG_MAC_HDR_BASED;
+		cmd.npca_params.dis_subch_bmap =
+			cpu_to_le16(link->chanreq.oper.npca_punctured);
+		cmd.npca_params.initial_qsrc = link->npca.init_qsrc;
+		cmd.npca_params.min_dur_threshold = link->npca.min_dur_thresh;
+		/* spec/mac80211 have these in units of 4 usec */
+		cmd.npca_params.switch_delay =
+			4 * link->npca.switch_delay;
+		cmd.npca_params.switch_back_delay =
+			4 * link->npca.switch_back_delay;
+	}
+
 send_cmd:
 	cmd.modify_mask = cpu_to_le32(changes);
 	cmd.flags = cpu_to_le32(flags);
@@ -437,7 +490,8 @@ iwl_mld_rm_link_from_fw(struct iwl_mld *mld, struct ieee80211_bss_conf *link)
 	iwl_mld_send_link_cmd(mld, &cmd, FW_CTXT_ACTION_REMOVE);
 }
 
-static IWL_MLD_ALLOC_FN(link, bss_conf)
+IWL_MLD_ALLOC_FN(link, bss_conf)
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_allocate_link_fw_id);
 
 /* Constructor function for struct iwl_mld_link */
 static int
@@ -504,7 +558,6 @@ void iwl_mld_remove_link(struct iwl_mld *mld,
 	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(bss_conf->vif);
 	struct iwl_mld_link *link = iwl_mld_link_from_mac80211(bss_conf);
 	bool is_deflink = link == &mld_vif->deflink;
-	u8 fw_id = link->fw_id;
 
 	if (WARN_ON(!link || link->active))
 		return;
@@ -512,15 +565,15 @@ void iwl_mld_remove_link(struct iwl_mld *mld,
 	iwl_mld_rm_link_from_fw(mld, bss_conf);
 	/* Continue cleanup on failure */
 
-	if (!is_deflink)
-		kfree_rcu(link, rcu_head);
-
 	RCU_INIT_POINTER(mld_vif->link[bss_conf->link_id], NULL);
 
-	if (WARN_ON(fw_id >= mld->fw->ucode_capa.num_links))
+	if (WARN_ON(link->fw_id >= mld->fw->ucode_capa.num_links))
 		return;
 
-	RCU_INIT_POINTER(mld->fw_id_to_bss_conf[fw_id], NULL);
+	RCU_INIT_POINTER(mld->fw_id_to_bss_conf[link->fw_id], NULL);
+
+	if (!is_deflink)
+		kfree_rcu(link, rcu_head);
 }
 
 void iwl_mld_handle_missed_beacon_notif(struct iwl_mld *mld,
@@ -609,15 +662,20 @@ void iwl_mld_handle_missed_beacon_notif(struct iwl_mld *mld,
 	 * OR more than IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED
 	 * on current link and the link's bss_param_ch_count has changed on
 	 * the other link's beacon.
+	 *
+	 * When both links lose beacons, keep the primary (symmetric failure).
+	 * When only the current link is sick, keep the other link.
 	 */
-	if ((missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS &&
-	     scnd_lnk_bcn_lost >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS) ||
-	    missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH ||
-	    (bss_param_ch_cnt_link_id != link_id &&
-	     missed_bcon >=
-	     IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED)) {
+	if (missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS &&
+	    scnd_lnk_bcn_lost >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_2_LINKS) {
 		iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_MISSED_BEACON,
 				   iwl_mld_get_primary_link(vif));
+	} else if (missed_bcon >= IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH ||
+		   (bss_param_ch_cnt_link_id != link_id &&
+		    missed_bcon >=
+		    IWL_MLD_BCN_LOSS_EXIT_ESR_THRESH_BSS_PARAM_CHANGED)) {
+		iwl_mld_exit_emlsr(mld, vif, IWL_MLD_EMLSR_EXIT_MISSED_BEACON,
+				   iwl_mld_get_other_link(vif, link_id));
 	}
 }
 EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_handle_missed_beacon_notif);
@@ -652,35 +710,131 @@ struct iwl_mld_rssi_to_grade {
 	u16 grade;
 };
 
-#define RSSI_TO_GRADE_LINE(_lb, _hb_uhb, _grade) \
+#define RSSI_TO_GRADE_LINE_WITH_LB(_lb, _hb_uhb, _grade) \
 	{ \
 		.rssi = {_lb, _hb_uhb}, \
 		.grade = _grade \
 	}
 
-/*
- * This array must be sorted by increasing RSSI for proper functionality.
- * The grades are actually estimated throughput, represented as fixed-point
- * with a scale factor of 1/10.
- */
-static const struct iwl_mld_rssi_to_grade rssi_to_grade_map[] = {
-	RSSI_TO_GRADE_LINE(-85, -89, 172),
-	RSSI_TO_GRADE_LINE(-83, -86, 344),
-	RSSI_TO_GRADE_LINE(-82, -85, 516),
-	RSSI_TO_GRADE_LINE(-80, -83, 688),
-	RSSI_TO_GRADE_LINE(-77, -79, 1032),
-	RSSI_TO_GRADE_LINE(-73, -76, 1376),
-	RSSI_TO_GRADE_LINE(-70, -74, 1548),
-	RSSI_TO_GRADE_LINE(-69, -72, 1720),
-	RSSI_TO_GRADE_LINE(-65, -68, 2064),
-	RSSI_TO_GRADE_LINE(-61, -66, 2294),
-	RSSI_TO_GRADE_LINE(-58, -61, 2580),
-	RSSI_TO_GRADE_LINE(-55, -58, 2868),
-	RSSI_TO_GRADE_LINE(-46, -55, 3098),
-	RSSI_TO_GRADE_LINE(-43, -54, 3442)
+#define RSSI_TO_GRADE_LINE(_rssi, _grade) \
+	RSSI_TO_GRADE_LINE_WITH_LB(_rssi, _rssi, _grade)
+
+/* Tables must be sorted by increasing RSSI */
+
+/* 20 MHz Operational BW Grading Table */
+static const struct iwl_mld_rssi_to_grade rssi_to_grade_20mhz[] = {
+	RSSI_TO_GRADE_LINE_WITH_LB(-94, -95, 9),
+	RSSI_TO_GRADE_LINE_WITH_LB(-92, -93, 17),
+	RSSI_TO_GRADE_LINE_WITH_LB(-90, -90, 34),
+	RSSI_TO_GRADE_LINE_WITH_LB(-87, -87, 52),
+	RSSI_TO_GRADE_LINE_WITH_LB(-83, -84, 69),
+	RSSI_TO_GRADE_LINE_WITH_LB(-79, -80, 103),
+	RSSI_TO_GRADE_LINE_WITH_LB(-75, -75, 137),
+	RSSI_TO_GRADE_LINE_WITH_LB(-72, -73, 155),
+	RSSI_TO_GRADE_LINE_WITH_LB(-70, -71, 172),
+	RSSI_TO_GRADE_LINE_WITH_LB(-67, -68, 206),
+	RSSI_TO_GRADE_LINE_WITH_LB(-64, -65, 230),
+	RSSI_TO_GRADE_LINE_WITH_LB(-59, -60, 258),
+	RSSI_TO_GRADE_LINE_WITH_LB(-57, -58, 287),
+	RSSI_TO_GRADE_LINE_WITH_LB(-52, -53, 310),
+	RSSI_TO_GRADE_LINE_WITH_LB(-50, -50, 345),
 };
 
-#define MAX_GRADE (rssi_to_grade_map[ARRAY_SIZE(rssi_to_grade_map) - 1].grade)
+/* 40 MHz Operational BW Grading Table */
+static const struct iwl_mld_rssi_to_grade rssi_to_grade_40mhz[] = {
+	RSSI_TO_GRADE_LINE_WITH_LB(-95, -95, 9),
+	RSSI_TO_GRADE_LINE_WITH_LB(-93, -93, 17),
+	RSSI_TO_GRADE_LINE_WITH_LB(-91, -92, 18),
+	RSSI_TO_GRADE_LINE_WITH_LB(-89, -90, 34),
+	RSSI_TO_GRADE_LINE_WITH_LB(-87, -87, 68),
+	RSSI_TO_GRADE_LINE_WITH_LB(-84, -84, 104),
+	RSSI_TO_GRADE_LINE_WITH_LB(-80, -81, 138),
+	RSSI_TO_GRADE_LINE_WITH_LB(-77, -77, 206),
+	RSSI_TO_GRADE_LINE_WITH_LB(-72, -72, 274),
+	RSSI_TO_GRADE_LINE_WITH_LB(-69, -70, 310),
+	RSSI_TO_GRADE_LINE_WITH_LB(-67, -68, 344),
+	RSSI_TO_GRADE_LINE_WITH_LB(-64, -65, 412),
+	RSSI_TO_GRADE_LINE_WITH_LB(-61, -62, 460),
+	RSSI_TO_GRADE_LINE_WITH_LB(-56, -57, 516),
+	RSSI_TO_GRADE_LINE_WITH_LB(-54, -55, 574),
+	RSSI_TO_GRADE_LINE_WITH_LB(-49, -50, 620),
+	RSSI_TO_GRADE_LINE_WITH_LB(-46, -47, 690),
+};
+
+/* 80 MHz Operational BW Grading Table */
+static const struct iwl_mld_rssi_to_grade rssi_to_grade_80mhz[] = {
+	RSSI_TO_GRADE_LINE(-95, 9),
+	RSSI_TO_GRADE_LINE(-93, 17),
+	RSSI_TO_GRADE_LINE(-92, 18),
+	RSSI_TO_GRADE_LINE(-90, 34),
+	RSSI_TO_GRADE_LINE(-89, 36),
+	RSSI_TO_GRADE_LINE(-87, 68),
+	RSSI_TO_GRADE_LINE(-83, 136),
+	RSSI_TO_GRADE_LINE(-80, 208),
+	RSSI_TO_GRADE_LINE(-77, 276),
+	RSSI_TO_GRADE_LINE(-74, 412),
+	RSSI_TO_GRADE_LINE(-69, 548),
+	RSSI_TO_GRADE_LINE(-67, 620),
+	RSSI_TO_GRADE_LINE(-66, 688),
+	RSSI_TO_GRADE_LINE(-61, 824),
+	RSSI_TO_GRADE_LINE(-59, 920),
+	RSSI_TO_GRADE_LINE(-54, 1032),
+	RSSI_TO_GRADE_LINE(-52, 1148),
+	RSSI_TO_GRADE_LINE(-47, 1240),
+	RSSI_TO_GRADE_LINE(-44, 1380),
+};
+
+/* 160 MHz Operational BW Grading Table */
+static const struct iwl_mld_rssi_to_grade rssi_to_grade_160mhz[] = {
+	RSSI_TO_GRADE_LINE(-95, 9),
+	RSSI_TO_GRADE_LINE(-93, 17),
+	RSSI_TO_GRADE_LINE(-92, 18),
+	RSSI_TO_GRADE_LINE(-90, 34),
+	RSSI_TO_GRADE_LINE(-89, 36),
+	RSSI_TO_GRADE_LINE(-87, 68),
+	RSSI_TO_GRADE_LINE(-86, 72),
+	RSSI_TO_GRADE_LINE(-84, 136),
+	RSSI_TO_GRADE_LINE(-81, 272),
+	RSSI_TO_GRADE_LINE(-78, 416),
+	RSSI_TO_GRADE_LINE(-75, 552),
+	RSSI_TO_GRADE_LINE(-71, 824),
+	RSSI_TO_GRADE_LINE(-67, 1096),
+	RSSI_TO_GRADE_LINE(-65, 1240),
+	RSSI_TO_GRADE_LINE(-63, 1376),
+	RSSI_TO_GRADE_LINE(-59, 1648),
+	RSSI_TO_GRADE_LINE(-57, 1840),
+	RSSI_TO_GRADE_LINE(-52, 2064),
+	RSSI_TO_GRADE_LINE(-50, 2296),
+	RSSI_TO_GRADE_LINE(-46, 2480),
+	RSSI_TO_GRADE_LINE(-42, 2760),
+};
+
+/* 320 MHz Operational BW Grading Table */
+static const struct iwl_mld_rssi_to_grade rssi_to_grade_320mhz[] = {
+	RSSI_TO_GRADE_LINE(-95, 9),
+	RSSI_TO_GRADE_LINE(-93, 17),
+	RSSI_TO_GRADE_LINE(-92, 18),
+	RSSI_TO_GRADE_LINE(-90, 34),
+	RSSI_TO_GRADE_LINE(-89, 36),
+	RSSI_TO_GRADE_LINE(-87, 68),
+	RSSI_TO_GRADE_LINE(-86, 72),
+	RSSI_TO_GRADE_LINE(-84, 136),
+	RSSI_TO_GRADE_LINE(-83, 144),
+	RSSI_TO_GRADE_LINE(-81, 272),
+	RSSI_TO_GRADE_LINE(-78, 544),
+	RSSI_TO_GRADE_LINE(-75, 832),
+	RSSI_TO_GRADE_LINE(-72, 1104),
+	RSSI_TO_GRADE_LINE(-69, 1648),
+	RSSI_TO_GRADE_LINE(-64, 2192),
+	RSSI_TO_GRADE_LINE(-62, 2480),
+	RSSI_TO_GRADE_LINE(-61, 2752),
+	RSSI_TO_GRADE_LINE(-57, 3296),
+	RSSI_TO_GRADE_LINE(-55, 3680),
+	RSSI_TO_GRADE_LINE(-50, 4128),
+	RSSI_TO_GRADE_LINE(-47, 4592),
+	RSSI_TO_GRADE_LINE(-43, 4960),
+	RSSI_TO_GRADE_LINE(-40, 5520),
+};
 
 #define DEFAULT_CHAN_LOAD_2GHZ	30
 #define DEFAULT_CHAN_LOAD_5GHZ	15
@@ -690,31 +844,27 @@ static const struct iwl_mld_rssi_to_grade rssi_to_grade_map[] = {
 #define SCALE_FACTOR 256
 #define MAX_CHAN_LOAD 256
 
-static unsigned int
-iwl_mld_get_n_subchannels(const struct ieee80211_bss_conf *link_conf)
+static void
+iwl_mld_apply_puncturing_penalty(const struct ieee80211_bss_conf *link_conf,
+				 unsigned int *grade, int bw_mhz)
 {
-	enum nl80211_chan_width chan_width =
-		link_conf->chanreq.oper.width;
-	int mhz = nl80211_chan_width_to_mhz(chan_width);
-	unsigned int n_subchannels;
+	unsigned int n_punctured, n_subchannels;
 
-	if (WARN_ONCE(mhz < 20 || mhz > 320,
-		      "Invalid channel width : (%d)\n", mhz))
-		return 1;
+	/* Puncturing only applicable for BW >= 80 MHz */
+	if (bw_mhz < 80)
+		return;
 
-	/* total number of subchannels */
-	n_subchannels = mhz / 20;
+	n_punctured = hweight16(link_conf->chanreq.oper.punctured);
+	if (n_punctured == 0)
+		return;
 
-	/* No puncturing if less than 80 MHz */
-	if (mhz >= 80)
-		n_subchannels -= hweight16(link_conf->chanreq.oper.punctured);
+	n_subchannels = bw_mhz / 20;
 
-	return n_subchannels;
+	*grade = *grade * (n_subchannels - n_punctured) / n_subchannels;
 }
 
-static int
-iwl_mld_get_chan_load_from_element(struct iwl_mld *mld,
-				   struct ieee80211_bss_conf *link_conf)
+int iwl_mld_get_chan_load_from_element(struct iwl_mld *mld,
+				       struct ieee80211_bss_conf *link_conf)
 {
 	const struct cfg80211_bss_ies *ies;
 	const struct element *bss_load_elem = NULL;
@@ -788,6 +938,51 @@ int iwl_mld_get_chan_load_by_others(struct iwl_mld *mld,
 	return chan_load;
 }
 
+/* Returns whether internal MLO Scan needs to be triggered */
+bool iwl_mld_chan_load_requires_scan(struct iwl_mld *mld,
+				     struct ieee80211_bss_conf *link_conf,
+				     u32 new_chan_load)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link_conf);
+	enum iwl_mld_link_chan_load_level new_lvl;
+	bool scan_trig = false;
+
+	if (WARN_ON(!mld_link))
+		return false;
+
+	/* For each Level,
+	 * First check if high limit threshold crosses
+	 * If not then, check if low limit threshold crosses
+	 * Set new level based on low limit thresh only if old level
+	 * is not lower than level threshold
+	 */
+	for (new_lvl = LINK_CHAN_LOAD_LVL_MAX;
+	     new_lvl > LINK_CHAN_LOAD_LVL_NONE; new_lvl--) {
+		if (new_chan_load >=
+		    link_chan_load_thresh_tbl[new_lvl].high_lim)
+			break;
+		if (new_chan_load >=
+		    link_chan_load_thresh_tbl[new_lvl].low_lim &&
+		    mld_link->chan_load_lvl >= new_lvl)
+			break;
+	}
+
+	/* Trigger scan only for Level Up Transition */
+	if (new_lvl > mld_link->chan_load_lvl)
+		scan_trig = true;
+
+	IWL_DEBUG_EHT(mld,
+		      "Link %d: chan_load=%d%%, old_lvl=%d, new_lvl=%d, scan_trig=%d\n",
+		      link_conf->link_id, new_chan_load,
+		      mld_link->chan_load_lvl, new_lvl, scan_trig);
+
+	/* Update computed new level */
+	mld_link->chan_load_lvl = new_lvl;
+
+	return scan_trig;
+}
+EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_chan_load_requires_scan);
+
 static unsigned int
 iwl_mld_get_default_chan_load(struct ieee80211_bss_conf *link_conf)
 {
@@ -829,16 +1024,214 @@ iwl_mld_get_avail_chan_load(struct iwl_mld *mld,
 	return MAX_CHAN_LOAD - iwl_mld_get_chan_load(mld, link_conf);
 }
 
+static s8
+iwl_mld_get_dup_beacon_rssi_adjust(struct iwl_mld *mld,
+				   struct ieee80211_bss_conf *link_conf)
+{
+	const struct ieee80211_he_6ghz_oper *he_6ghz_oper;
+	const struct cfg80211_bss_ies *beacon_ies;
+	const struct element *elem;
+
+	/* Duplicated beacon feature is only specific to 6 GHz */
+	if (WARN_ONCE(link_conf->chanreq.oper.chan->band != NL80211_BAND_6GHZ,
+		      "Unexpected band %d\n",
+		      link_conf->chanreq.oper.chan->band))
+		return 0;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	beacon_ies = wiphy_dereference(mld->wiphy, link_conf->bss->beacon_ies);
+	if (!beacon_ies)
+		return 0;
+
+	elem = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION,
+				      beacon_ies->data, beacon_ies->len);
+	if (!elem ||
+	    elem->datalen < sizeof(struct ieee80211_he_operation) + 1 ||
+	    elem->datalen < ieee80211_he_oper_size(&elem->data[1]))
+		return 0;
+
+	he_6ghz_oper = ieee80211_he_6ghz_oper((const void *)&elem->data[1]);
+	if (!he_6ghz_oper)
+		return 0;
+
+	if (!(he_6ghz_oper->control & IEEE80211_HE_6GHZ_OPER_CTRL_DUP_BEACON))
+		return 0;
+
+	/* Apply adjustment based on operational bandwidth */
+	switch (link_conf->chanreq.oper.width) {
+	case NL80211_CHAN_WIDTH_20:
+	case NL80211_CHAN_WIDTH_20_NOHT:
+		return 0;
+	case NL80211_CHAN_WIDTH_40:
+		return 3;
+	case NL80211_CHAN_WIDTH_80:
+		return 6;
+	case NL80211_CHAN_WIDTH_160:
+		return 9;
+	case NL80211_CHAN_WIDTH_320:
+		return 12;
+	default:
+		WARN_ONCE(1, "Unexpected channel width: %d\n",
+			  link_conf->chanreq.oper.width);
+		return 0;
+	}
+}
+
+static s8 iwl_mld_get_primary_psd(const struct ieee80211_parsed_tpe_psd *psd,
+				  const struct cfg80211_chan_def *chandef,
+				  int bw_mhz)
+{
+	int start_freq, primary_idx;
+
+	if (!psd->valid)
+		return S8_MAX;
+
+	start_freq = chandef->center_freq1 - (bw_mhz / 2);
+	primary_idx = (chandef->chan->center_freq - start_freq - 10) / 20;
+
+	if (primary_idx < 0 || primary_idx >= psd->count)
+		return S8_MAX;
+
+	/* TPE element stores PSD limit as value * 2 */
+	return psd->power[primary_idx] / 2;
+}
+
+static s8 iwl_mld_get_psd_eirp_rssi_adjust(struct ieee80211_bss_conf *link_conf)
+{
+	const struct ieee80211_parsed_tpe *tpe = &link_conf->tpe;
+	s8 psd_20mhz, psd_oper, psd_local, psd_reg, psd_boost;
+	s8 min_20mhz, min_oper, adjustment, ap_power_limit;
+	s8 psd_avg_local = S8_MAX, psd_avg_reg = S8_MAX;
+	s8 eirp_20mhz, eirp_oper, eirp_local, eirp_reg;
+	int bw_mhz, num_subchans;
+	u8 bw_index;
+
+	/* PSD/EIRP adjustment is only specific to 6 GHz */
+	if (WARN_ONCE(link_conf->chanreq.oper.chan->band != NL80211_BAND_6GHZ,
+		      "PSD/EIRP adjustment called for non-6 GHz band %d\n",
+		      link_conf->chanreq.oper.chan->band))
+		return 0;
+
+	bw_mhz = nl80211_chan_width_to_mhz(link_conf->chanreq.oper.width);
+
+	switch (bw_mhz) {
+	case 20:
+		bw_index = 0;
+		break;
+	case 40:
+		bw_index = 1;
+		break;
+	case 80:
+		bw_index = 2;
+		break;
+	case 160:
+		bw_index = 3;
+		break;
+	case 320:
+		bw_index = 4;
+		break;
+	default:
+		WARN_ONCE(1, "Unexpected bandwidth: %d MHz\n", bw_mhz);
+		return 0;
+	}
+
+	if (link_conf->power_type == IEEE80211_REG_VLP_AP)
+		ap_power_limit = 14;
+	else
+		ap_power_limit = 23;
+
+	/* Primary 20 MHz PSD */
+	psd_local = iwl_mld_get_primary_psd(&tpe->psd_local[0],
+					    &link_conf->chanreq.oper,
+					    bw_mhz);
+	psd_reg = iwl_mld_get_primary_psd(&tpe->psd_reg_client[0],
+					  &link_conf->chanreq.oper,
+					  bw_mhz);
+	psd_20mhz = min(psd_local, psd_reg);
+
+	/* TPE element stores EIRP limit as value * 2 */
+	eirp_local = (tpe->max_local[0].valid && tpe->max_local[0].count > 0) ?
+			tpe->max_local[0].power[0] / 2 : S8_MAX;
+	eirp_reg = (tpe->max_reg_client[0].valid &&
+		    tpe->max_reg_client[0].count > 0) ?
+		      tpe->max_reg_client[0].power[0] / 2 : S8_MAX;
+	eirp_20mhz = min(eirp_local, eirp_reg);
+
+	num_subchans = bw_mhz / 20;
+
+	if (tpe->psd_local[0].valid) {
+		int sum_local = 0, valid_local = 0;
+		int count_local = min(num_subchans, tpe->psd_local[0].count);
+
+		for (int i = 0; i < count_local; i++) {
+			if (tpe->psd_local[0].power[i] != S8_MIN) {
+				sum_local += tpe->psd_local[0].power[i];
+				valid_local++;
+			}
+		}
+		/* TPE element stores PSD limit as value * 2 */
+		if (valid_local > 0)
+			psd_avg_local = sum_local / valid_local / 2;
+	}
+
+	if (tpe->psd_reg_client[0].valid) {
+		int sum_reg = 0, valid_reg = 0;
+		int count_reg = min(num_subchans, tpe->psd_reg_client[0].count);
+
+		for (int i = 0; i < count_reg; i++) {
+			if (tpe->psd_reg_client[0].power[i] != S8_MIN) {
+				sum_reg +=
+					tpe->psd_reg_client[0].power[i];
+				valid_reg++;
+			}
+		}
+		/* TPE element stores PSD limit as value * 2 */
+		if (valid_reg > 0)
+			psd_avg_reg = sum_reg / valid_reg / 2;
+	}
+
+	psd_oper = min(psd_avg_local, psd_avg_reg);
+
+	/* TPE element stores EIRP limit as value * 2 */
+	eirp_local = (tpe->max_local[0].valid &&
+		      tpe->max_local[0].count > bw_index) ?
+			tpe->max_local[0].power[bw_index] / 2 : S8_MAX;
+	eirp_reg = (tpe->max_reg_client[0].valid &&
+		    tpe->max_reg_client[0].count > bw_index) ?
+		      tpe->max_reg_client[0].power[bw_index] / 2 : S8_MAX;
+	eirp_oper = min(eirp_local, eirp_reg);
+
+	min_20mhz = min(ap_power_limit, min(eirp_20mhz, psd_20mhz));
+
+	/* PSD boost: 10*log10(BW/20) approximated as 3*ilog2(BW/20) */
+	psd_boost = 3 * ilog2(bw_mhz / 20);
+
+	/* Use int for psd_oper + psd_boost to prevent s8 overflow */
+	min_oper = min(ap_power_limit,
+		       min(eirp_oper,
+			   (s8)min_t(int, psd_oper + psd_boost, S8_MAX)));
+
+	adjustment = max(min_oper - min_20mhz, 0);
+
+	return adjustment;
+}
+
 /* This function calculates the grade of a link. Returns 0 in error case */
 unsigned int iwl_mld_get_link_grade(struct iwl_mld *mld,
 				    struct ieee80211_bss_conf *link_conf)
 {
+	const struct iwl_mld_rssi_to_grade *grade_table;
 	enum nl80211_band band;
-	int rssi_idx;
+	int rssi_idx, table_size, bw_mhz;
 	s32 link_rssi;
-	unsigned int grade = MAX_GRADE;
+	unsigned int grade;
+
+	if (WARN_ON_ONCE(!link_conf || !link_conf->bss))
+		return 0;
 
-	if (WARN_ON_ONCE(!link_conf))
+	bw_mhz = nl80211_chan_width_to_mhz(link_conf->chanreq.oper.width);
+	if (bw_mhz < 0)
 		return 0;
 
 	band = link_conf->chanreq.oper.chan->band;
@@ -853,25 +1246,64 @@ unsigned int iwl_mld_get_link_grade(struct iwl_mld *mld,
 	 * For 6 GHz the RSSI of the beacons is lower than
 	 * the RSSI of the data.
 	 */
-	if (band == NL80211_BAND_6GHZ && link_rssi)
-		link_rssi += 4;
+	if (band == NL80211_BAND_6GHZ && link_rssi) {
+		s8 rssi_adj_6g =
+			iwl_mld_get_dup_beacon_rssi_adjust(mld, link_conf);
+
+		if (!rssi_adj_6g)
+			rssi_adj_6g =
+				iwl_mld_get_psd_eirp_rssi_adjust(link_conf);
+
+		if (!rssi_adj_6g)
+			rssi_adj_6g = 4;
+
+		link_rssi += rssi_adj_6g;
+	}
+
+	/* Select grading table based on operational bandwidth */
+	switch (bw_mhz) {
+	case 20:
+		grade_table = rssi_to_grade_20mhz;
+		table_size = ARRAY_SIZE(rssi_to_grade_20mhz);
+		break;
+	case 40:
+		grade_table = rssi_to_grade_40mhz;
+		table_size = ARRAY_SIZE(rssi_to_grade_40mhz);
+		break;
+	case 80:
+		grade_table = rssi_to_grade_80mhz;
+		table_size = ARRAY_SIZE(rssi_to_grade_80mhz);
+		break;
+	case 160:
+		grade_table = rssi_to_grade_160mhz;
+		table_size = ARRAY_SIZE(rssi_to_grade_160mhz);
+		break;
+	case 320:
+		grade_table = rssi_to_grade_320mhz;
+		table_size = ARRAY_SIZE(rssi_to_grade_320mhz);
+		break;
+	default:
+		WARN_ONCE(1, "Invalid bandwidth: %d MHz\n", bw_mhz);
+		return 0;
+	}
+
+	/* Initialize grade to maximum value from selected table */
+	grade = grade_table[table_size - 1].grade;
 
 	rssi_idx = band == NL80211_BAND_2GHZ ? 0 : 1;
 
-	/* No valid RSSI - take the lowest grade */
+	/* No valid RSSI - take the lowest grade from selected table */
 	if (!link_rssi)
-		link_rssi = rssi_to_grade_map[0].rssi[rssi_idx];
+		link_rssi = grade_table[0].rssi[rssi_idx];
 
 	IWL_DEBUG_EHT(mld,
-		      "Calculating grade of link %d: band = %d, bandwidth = %d, punctured subchannels =0x%x RSSI = %d\n",
-		      link_conf->link_id, band,
-		      link_conf->chanreq.oper.width,
+		      "Calculating grade of link %d: band = %d, BW = %d, punct subchannels = 0x%x RSSI = %d\n",
+		      link_conf->link_id, band, bw_mhz,
 		      link_conf->chanreq.oper.punctured, link_rssi);
 
-	/* Get grade based on RSSI */
-	for (int i = 0; i < ARRAY_SIZE(rssi_to_grade_map); i++) {
-		const struct iwl_mld_rssi_to_grade *line =
-			&rssi_to_grade_map[i];
+	/* Get grade based on RSSI from the bandwidth-specific table */
+	for (int i = 0; i < table_size; i++) {
+		const struct iwl_mld_rssi_to_grade *line = &grade_table[i];
 
 		if (link_rssi > line->rssi[rssi_idx])
 			continue;
@@ -880,8 +1312,10 @@ unsigned int iwl_mld_get_link_grade(struct iwl_mld *mld,
 	}
 
 	/* Apply the channel load and puncturing factors */
-	grade = grade * iwl_mld_get_avail_chan_load(mld, link_conf) / SCALE_FACTOR;
-	grade = grade * iwl_mld_get_n_subchannels(link_conf);
+	grade = grade * iwl_mld_get_avail_chan_load(mld, link_conf) /
+		SCALE_FACTOR;
+
+	iwl_mld_apply_puncturing_penalty(link_conf, &grade, bw_mhz);
 
 	IWL_DEBUG_EHT(mld, "Link %d's grade: %d\n", link_conf->link_id, grade);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/link.h b/drivers/net/wireless/intel/iwlwifi/mld/link.h
index ca691259fc5e6..f1997e2800584 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/link.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/link.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #ifndef __iwl_mld_link_h__
 #define __iwl_mld_link_h__
@@ -10,6 +10,14 @@
 #include "mld.h"
 #include "sta.h"
 
+enum iwl_mld_link_chan_load_level {
+	LINK_CHAN_LOAD_LVL_NONE,
+	LINK_CHAN_LOAD_LVL1,
+	LINK_CHAN_LOAD_LVL2,
+	LINK_CHAN_LOAD_LVL3,
+	LINK_CHAN_LOAD_LVL_MAX  = LINK_CHAN_LOAD_LVL3
+};
+
 /**
  * struct iwl_probe_resp_data - data for NoA/CSA updates
  * @rcu_head: used for freeing the data on update
@@ -28,14 +36,17 @@ struct iwl_probe_resp_data {
  * @rcu_head: RCU head for freeing this data.
  * @fw_id: the fw id of the link.
  * @active: if the link is active or not.
+ * @avg_signal: The current average signal of beacons [dBm] retrieved from
+ *	firmware per-link periodic stats (STATISTICS_OPER_NOTIF).
  * @queue_params: QoS data from mac80211. This is updated with a call to
  *	drv_conf_tx per each AC, and then notified once with BSS_CHANGED_QOS.
  *	So we store it here and then send one link cmd for all the ACs.
  * @chan_ctx: pointer to the channel context assigned to the link. If a link
  *	has an assigned channel context it means that it is active.
  * @he_ru_2mhz_block: 26-tone RU OFDMA transmissions should be blocked.
- * @igtk: fw can only have one IGTK at a time, whereas mac80211 can have two.
- *	This tracks the one IGTK that currently exists in FW.
+ * @tx_igtk: FW can only have one IGTK per MAC at a time, whereas mac80211 can
+ *	have two. This tracks the one IGTK that currently exists in FW, for TX
+ *	purposes. The RX IGTKs are tracked per station.
  * @bigtks: BIGTKs of the AP. Only valid for STA mode.
  * @bcast_sta: station used for broadcast packets. Used in AP, GO and IBSS.
  * @mcast_sta: station used for multicast packets. Used in AP, GO and IBSS.
@@ -49,6 +60,8 @@ struct iwl_probe_resp_data {
  * @silent_deactivation: next deactivation needs to be silent.
  * @probe_resp_data: data from FW notification to store NOA related data to be
  *	inserted into probe response.
+ * @chan_load_lvl: current channel load level for a link, computed based on
+ *	channel load by others on a link.
  */
 struct iwl_mld_link {
 	struct rcu_head rcu_head;
@@ -57,11 +70,13 @@ struct iwl_mld_link {
 	struct_group(zeroed_on_hw_restart,
 		u8 fw_id;
 		bool active;
+		s8 avg_signal;
 		struct ieee80211_tx_queue_params queue_params[IEEE80211_NUM_ACS];
 		struct ieee80211_chanctx_conf __rcu *chan_ctx;
 		bool he_ru_2mhz_block;
-		struct ieee80211_key_conf *igtk;
+		struct ieee80211_key_conf *tx_igtk;
 		struct ieee80211_key_conf __rcu *bigtks[2];
+		enum iwl_mld_link_chan_load_level chan_load_lvl;
 	);
 	/* And here fields that survive a fw restart */
 	struct iwl_mld_int_sta bcast_sta;
@@ -99,6 +114,13 @@ iwl_mld_cleanup_link(struct iwl_mld *mld, struct iwl_mld_link *link)
 /* Convert a percentage from [0,100] to [0,255] */
 #define NORMALIZE_PERCENT_TO_255(percentage) ((percentage) * 256 / 100)
 
+int iwl_mld_allocate_link_fw_id(struct iwl_mld *mld, u8 *fw_id,
+				struct ieee80211_bss_conf *mac80211_ptr);
+
+int iwl_mld_send_link_cmd(struct iwl_mld *mld,
+			  struct iwl_link_config_cmd *cmd,
+			  enum iwl_ctxt_action action);
+
 int iwl_mld_add_link(struct iwl_mld *mld,
 		     struct ieee80211_bss_conf *bss_conf);
 void iwl_mld_remove_link(struct iwl_mld *mld,
@@ -123,10 +145,17 @@ unsigned int iwl_mld_get_link_grade(struct iwl_mld *mld,
 unsigned int iwl_mld_get_chan_load(struct iwl_mld *mld,
 				   struct ieee80211_bss_conf *link_conf);
 
+int iwl_mld_get_chan_load_from_element(struct iwl_mld *mld,
+				       struct ieee80211_bss_conf *link_conf);
+
 int iwl_mld_get_chan_load_by_others(struct iwl_mld *mld,
 				    struct ieee80211_bss_conf *link_conf,
 				    bool expect_active_link);
 
+bool  iwl_mld_chan_load_requires_scan(struct iwl_mld *mld,
+				      struct ieee80211_bss_conf *link_conf,
+				      u32 new_chan_load);
+
 void iwl_mld_handle_beacon_filter_notif(struct iwl_mld *mld,
 					struct iwl_rx_packet *pkt);
 
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
index da6fd74715688..92858b8f7395c 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mac80211.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #include <net/mac80211.h>
@@ -62,17 +62,16 @@ static const struct ieee80211_iface_limit iwl_mld_limits_ap[] = {
 
 static const struct ieee80211_iface_limit iwl_mld_limits_nan[] = {
 	{
-		.max = 2,
+		.max = 1,
 		.types = BIT(NL80211_IFTYPE_STATION),
 	},
 	{
 		.max = 1,
 		.types = BIT(NL80211_IFTYPE_NAN),
 	},
-	/* Removed when two channels are permitted */
 	{
-		.max = 1,
-		.types = BIT(NL80211_IFTYPE_AP),
+		.max = 2,
+		.types = BIT(NL80211_IFTYPE_NAN_DATA),
 	},
 };
 
@@ -90,19 +89,13 @@ iwl_mld_iface_combinations[] = {
 		.limits = iwl_mld_limits_ap,
 		.n_limits = ARRAY_SIZE(iwl_mld_limits_ap),
 	},
-	/* NAN combinations follow, these exclude P2P */
-	{
-		.num_different_channels = 2,
-		.max_interfaces = 3,
-		.limits = iwl_mld_limits_nan,
-		.n_limits = ARRAY_SIZE(iwl_mld_limits_nan) - 1,
-	},
+	/* NAN combination follow, this excludes P2P and AP */
 	{
-		.num_different_channels = 1,
+		.num_different_channels = 3,
 		.max_interfaces = 4,
 		.limits = iwl_mld_limits_nan,
 		.n_limits = ARRAY_SIZE(iwl_mld_limits_nan),
-	}
+	},
 };
 
 static const u8 ext_capa_base[IWL_MLD_STA_EXT_CAPA_SIZE] = {
@@ -114,10 +107,10 @@ static const u8 ext_capa_base[IWL_MLD_STA_EXT_CAPA_SIZE] = {
 };
 
 #define IWL_MLD_EMLSR_CAPA	(IEEE80211_EML_CAP_EMLSR_SUPP | \
-				 IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_32US << \
-					__bf_shf(IEEE80211_EML_CAP_EMLSR_PADDING_DELAY) | \
+				 IEEE80211_EML_CAP_EML_PADDING_DELAY_32US << \
+					__bf_shf(IEEE80211_EML_CAP_EML_PADDING_DELAY) | \
 				 IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US << \
-					__bf_shf(IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY))
+					__bf_shf(IEEE80211_EML_CAP_EML_TRANSITION_DELAY))
 #define IWL_MLD_CAPA_OPS (FIELD_PREP_CONST( \
 			IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP, \
 			IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_SAME) | \
@@ -272,6 +265,40 @@ static void iwl_mac_hw_set_flags(struct iwl_mld *mld)
 	ieee80211_hw_set(hw, TDLS_WIDER_BW);
 }
 
+static void iwl_mld_hw_set_nan(struct iwl_mld *mld)
+{
+	struct ieee80211_hw *hw = mld->hw;
+
+	hw->wiphy->interface_modes |= BIT(NL80211_IFTYPE_NAN);
+	hw->wiphy->interface_modes |= BIT(NL80211_IFTYPE_NAN_DATA);
+
+	hw->wiphy->nan_supported_bands = BIT(NL80211_BAND_2GHZ);
+	if (mld->nvm_data->bands[NL80211_BAND_5GHZ].n_channels)
+		hw->wiphy->nan_supported_bands |=
+			BIT(NL80211_BAND_5GHZ);
+
+	hw->wiphy->nan_capa.flags = WIPHY_NAN_FLAGS_CONFIGURABLE_SYNC |
+				    WIPHY_NAN_FLAGS_USERSPACE_DE;
+
+	hw->wiphy->nan_capa.op_mode = NAN_OP_MODE_PHY_MODE_VHT |
+				      NAN_OP_MODE_PHY_MODE_HE |
+				      NAN_OP_MODE_160MHZ;
+
+	hw->wiphy->nan_capa.n_antennas =
+		(hweight32(hw->wiphy->available_antennas_tx) &
+		 NAN_DEV_CAPA_NUM_TX_ANT_MASK) |
+		((hweight32(hw->wiphy->available_antennas_rx) <<
+		  NAN_DEV_CAPA_NUM_RX_ANT_POS) &
+		 NAN_DEV_CAPA_NUM_RX_ANT_MASK);
+
+	/* Maximal channel switch time is 4 msec */
+	hw->wiphy->nan_capa.max_channel_switch_time = 4 * USEC_PER_MSEC;
+
+	hw->wiphy->nan_capa.phy.ht = mld->nvm_data->nan_phy_capa.ht;
+	hw->wiphy->nan_capa.phy.vht = mld->nvm_data->nan_phy_capa.vht;
+	hw->wiphy->nan_capa.phy.he = mld->nvm_data->nan_phy_capa.he;
+}
+
 static void iwl_mac_hw_set_wiphy(struct iwl_mld *mld)
 {
 	struct ieee80211_hw *hw = mld->hw;
@@ -334,37 +361,16 @@ static void iwl_mac_hw_set_wiphy(struct iwl_mld *mld)
 
 	wiphy->hw_timestamp_max_peers = 1;
 
+	wiphy->iface_combinations = iwl_mld_iface_combinations;
+
 	if (iwl_mld_nan_supported(mld)) {
-		hw->wiphy->interface_modes |= BIT(NL80211_IFTYPE_NAN);
-		hw->wiphy->iface_combinations = iwl_mld_iface_combinations;
-		hw->wiphy->n_iface_combinations =
+		wiphy->n_iface_combinations =
 			ARRAY_SIZE(iwl_mld_iface_combinations);
-
-		hw->wiphy->nan_supported_bands = BIT(NL80211_BAND_2GHZ);
-		if (mld->nvm_data->bands[NL80211_BAND_5GHZ].n_channels)
-			hw->wiphy->nan_supported_bands |=
-				BIT(NL80211_BAND_5GHZ);
-
-		hw->wiphy->nan_capa.flags = WIPHY_NAN_FLAGS_CONFIGURABLE_SYNC |
-					    WIPHY_NAN_FLAGS_USERSPACE_DE;
-
-		hw->wiphy->nan_capa.op_mode = NAN_OP_MODE_PHY_MODE_MASK |
-					      NAN_OP_MODE_80P80MHZ |
-					      NAN_OP_MODE_160MHZ;
-
-		/* Support 2 antenna's for Tx and Rx */
-		hw->wiphy->nan_capa.n_antennas = 0x22;
-
-		/* Maximal channel switch time is 4 msec */
-		hw->wiphy->nan_capa.max_channel_switch_time = 4;
-		hw->wiphy->nan_capa.dev_capabilities =
-			NAN_DEV_CAPA_EXT_KEY_ID_SUPPORTED |
-			NAN_DEV_CAPA_NDPE_SUPPORTED;
+		iwl_mld_hw_set_nan(mld);
 	} else {
-		wiphy->iface_combinations = iwl_mld_iface_combinations;
-		/* Do not include NAN combinations */
+		/* Do not include NAN combination */
 		wiphy->n_iface_combinations =
-			ARRAY_SIZE(iwl_mld_iface_combinations) - 2;
+			ARRAY_SIZE(iwl_mld_iface_combinations) - 1;
 	}
 
 	wiphy_ext_feature_set(wiphy, NL80211_EXT_FEATURE_VHT_IBSS);
@@ -414,6 +420,8 @@ static void iwl_mac_hw_set_wiphy(struct iwl_mld *mld)
 
 	mld->ext_capab[0].eml_capabilities = IWL_MLD_EMLSR_CAPA;
 	mld->ext_capab[0].mld_capa_and_ops = IWL_MLD_CAPA_OPS;
+	mld->ext_capab[0].ext_mld_capa_and_ops =
+		IEEE80211_UHR_ML_EXT_MLD_CAPA_ML_PM;
 
 }
 
@@ -449,22 +457,6 @@ static void iwl_mac_hw_set_misc(struct iwl_mld *mld)
 
 static int iwl_mld_hw_verify_preconditions(struct iwl_mld *mld)
 {
-	int ratecheck;
-
-	/* check for rates version 3 */
-	ratecheck =
-		(iwl_fw_lookup_cmd_ver(mld->fw, TX_CMD, 0) >= 11) +
-		(iwl_fw_lookup_notif_ver(mld->fw, DATA_PATH_GROUP,
-					 TLC_MNG_UPDATE_NOTIF, 0) >= 4) +
-		(iwl_fw_lookup_notif_ver(mld->fw, LEGACY_GROUP,
-					 REPLY_RX_MPDU_CMD, 0) >= 6) +
-		(iwl_fw_lookup_notif_ver(mld->fw, LONG_GROUP, TX_CMD, 0) >= 9);
-
-	if (ratecheck != 0 && ratecheck != 4) {
-		IWL_ERR(mld, "Firmware has inconsistent rates\n");
-		return -EINVAL;
-	}
-
 	/* 11ax is expected to be enabled for all supported devices */
 	if (WARN_ON(!mld->nvm_data->sku_cap_11ax_enable))
 		return -EINVAL;
@@ -673,6 +665,28 @@ int iwl_mld_mac80211_add_interface(struct ieee80211_hw *hw,
 	if (ret)
 		return ret;
 
+	if (vif->type == NL80211_IFTYPE_NAN_DATA) {
+		if (WARN_ON(!mld->nan_device_vif)) {
+			ret = -EINVAL;
+			goto err;
+		}
+
+		if (iwl_mld_nan_use_nan_stations(mld)) {
+			struct iwl_mld_vif *mld_vif =
+				iwl_mld_vif_from_mac80211(vif);
+			struct iwl_mld_int_sta *sta =
+				&mld_vif->nan.mcast_data_sta;
+
+			ret = iwl_mld_add_nan_mcast_data_sta(mld,
+							     vif->addr,
+							     sta);
+			if (ret)
+				goto err;
+		}
+
+		return 0;
+	}
+
 	/*
 	 * Add the default link, but not if this is an MLD vif as that implies
 	 * the HW is restarting and it will be configured by change_vif_links.
@@ -739,10 +753,17 @@ void iwl_mld_mac80211_remove_interface(struct ieee80211_hw *hw,
 	if (vif->type == NL80211_IFTYPE_P2P_DEVICE)
 		mld->p2p_device_vif = NULL;
 
-	if (vif->type == NL80211_IFTYPE_NAN)
+	if (vif->type == NL80211_IFTYPE_NAN) {
 		mld->nan_device_vif = NULL;
-	else
+	} else if (vif->type != NL80211_IFTYPE_NAN_DATA) {
 		iwl_mld_remove_link(mld, &vif->bss_conf);
+	} else if (iwl_mld_nan_use_nan_stations(mld)) {
+		struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+		struct iwl_mld_int_sta *sta = &mld_vif->nan.mcast_data_sta;
+
+		if (sta->sta_id != IWL_INVALID_STA)
+			iwl_mld_remove_nan_mcast_data_sta(mld, sta);
+	}
 
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	debugfs_remove(iwl_mld_vif_from_mac80211(vif)->dbgfs_slink);
@@ -988,6 +1009,30 @@ void iwl_mld_remove_chanctx(struct ieee80211_hw *hw,
 	mld->used_phy_ids &= ~BIT(phy->fw_id);
 }
 
+static void
+iwl_mld_update_link_npca_puncturing(struct ieee80211_hw *hw,
+				    struct ieee80211_chanctx_conf *ctx)
+{
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct ieee80211_vif *vif;
+
+	for_each_active_interface(vif, hw) {
+		struct ieee80211_bss_conf *link;
+		int link_id;
+
+		for_each_vif_active_link(vif, link, link_id) {
+			if (rcu_access_pointer(link->chanctx_conf) != ctx)
+				continue;
+
+			if (!link->npca.enabled)
+				continue;
+
+			iwl_mld_change_link_in_fw(mld, link,
+						  LINK_CONTEXT_MODIFY_UHR_PARAMS);
+		}
+	}
+}
+
 static
 void iwl_mld_change_chanctx(struct ieee80211_hw *hw,
 			    struct ieee80211_chanctx_conf *ctx, u32 changed)
@@ -1003,9 +1048,19 @@ void iwl_mld_change_chanctx(struct ieee80211_hw *hw,
 			  IEEE80211_CHANCTX_CHANGE_CHANNEL)))
 		return;
 
+	/* NPCA puncturing is in link API for FW */
+	if (changed & IEEE80211_CHANCTX_CHANGE_NPCA_PUNCT) {
+		iwl_mld_update_link_npca_puncturing(hw, ctx);
+		changed &= ~IEEE80211_CHANCTX_CHANGE_NPCA_PUNCT;
+	}
+
 	/* Check if a FW update is required */
 
-	if (changed & IEEE80211_CHANCTX_CHANGE_AP)
+	if (!changed)
+		return;
+
+	if (changed & IEEE80211_CHANCTX_CHANGE_AP ||
+	    changed & IEEE80211_CHANCTX_CHANGE_NPCA)
 		goto update;
 
 	if (chandef->chan == phy->chandef.chan &&
@@ -1231,33 +1286,6 @@ int iwl_mld_mac80211_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
 	return 0;
 }
 
-static void
-iwl_mld_link_info_changed_ap_ibss(struct iwl_mld *mld,
-				  struct ieee80211_vif *vif,
-				  struct ieee80211_bss_conf *link,
-				  u64 changes)
-{
-	u32 link_changes = 0;
-
-	if (changes & BSS_CHANGED_ERP_SLOT)
-		link_changes |= LINK_CONTEXT_MODIFY_RATES_INFO;
-
-	if (changes & (BSS_CHANGED_ERP_CTS_PROT | BSS_CHANGED_HT))
-		link_changes |= LINK_CONTEXT_MODIFY_PROTECT_FLAGS;
-
-	if (changes & (BSS_CHANGED_QOS | BSS_CHANGED_BANDWIDTH))
-		link_changes |= LINK_CONTEXT_MODIFY_QOS_PARAMS;
-
-	if (changes & BSS_CHANGED_HE_BSS_COLOR)
-		link_changes |= LINK_CONTEXT_MODIFY_HE_PARAMS;
-
-	if (link_changes)
-		iwl_mld_change_link_in_fw(mld, link, link_changes);
-
-	if (changes & BSS_CHANGED_BEACON)
-		iwl_mld_update_beacon_template(mld, vif, link);
-}
-
 static
 u32 iwl_mld_link_changed_mapping(struct iwl_mld *mld,
 				 struct ieee80211_vif *vif,
@@ -1289,6 +1317,9 @@ u32 iwl_mld_link_changed_mapping(struct iwl_mld *mld,
 		link_changes |= LINK_CONTEXT_MODIFY_HE_PARAMS;
 	}
 
+	if (changes & BSS_CHANGED_NPCA)
+		link_changes |= LINK_CONTEXT_MODIFY_UHR_PARAMS;
+
 	return link_changes;
 }
 
@@ -1367,6 +1398,10 @@ iwl_mld_mac80211_link_info_changed(struct ieee80211_hw *hw,
 		if (changes & BSS_CHANGED_MU_GROUPS)
 			iwl_mld_update_mu_groups(mld, link_conf);
 		break;
+	case NL80211_IFTYPE_NAN:
+	case NL80211_IFTYPE_NAN_DATA:
+		/* NAN has no links */
+		break;
 	default:
 		/* shouldn't happen */
 		WARN_ON_ONCE(1);
@@ -1382,28 +1417,6 @@ iwl_mld_mac80211_link_info_changed(struct ieee80211_hw *hw,
 		iwl_mld_set_tx_power(mld, link_conf, link_conf->txpower);
 }
 
-static void
-iwl_mld_smps_workaround(struct iwl_mld *mld, struct ieee80211_vif *vif, bool enable)
-{
-	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
-	bool workaround_required =
-		iwl_fw_lookup_cmd_ver(mld->fw, MAC_PM_POWER_TABLE, 0) < 2;
-
-	if (!workaround_required)
-		return;
-
-	/* Send the device-level power commands since the
-	 * firmware checks the POWER_TABLE_CMD's POWER_SAVE_EN bit to
-	 * determine SMPS mode.
-	 */
-	if (mld_vif->ps_disabled == !enable)
-		return;
-
-	mld_vif->ps_disabled = !enable;
-
-	iwl_mld_update_device_power(mld, false);
-}
-
 static
 void iwl_mld_mac80211_vif_cfg_changed(struct ieee80211_hw *hw,
 				      struct ieee80211_vif *vif,
@@ -1414,6 +1427,11 @@ void iwl_mld_mac80211_vif_cfg_changed(struct ieee80211_hw *hw,
 
 	lockdep_assert_wiphy(mld->wiphy);
 
+	if (vif->type == NL80211_IFTYPE_NAN) {
+		iwl_mld_nan_vif_cfg_changed(mld, vif, changes);
+		return;
+	}
+
 	if (vif->type != NL80211_IFTYPE_STATION)
 		return;
 
@@ -1436,10 +1454,8 @@ void iwl_mld_mac80211_vif_cfg_changed(struct ieee80211_hw *hw,
 		}
 	}
 
-	if (changes & BSS_CHANGED_PS) {
-		iwl_mld_smps_workaround(mld, vif, vif->cfg.ps);
+	if (changes & BSS_CHANGED_PS)
 		iwl_mld_update_mac_power(mld, vif, false);
-	}
 
 	/* TODO: task=MLO BSS_CHANGED_MLD_VALID_LINKS/CHANGED_MLD_TTLM */
 }
@@ -1609,7 +1625,7 @@ iwl_mld_mac80211_conf_tx(struct ieee80211_hw *hw,
 
 	lockdep_assert_wiphy(mld->wiphy);
 
-	if (vif->type == NL80211_IFTYPE_NAN)
+	if (vif->type == NL80211_IFTYPE_NAN || vif->type == NL80211_IFTYPE_NAN_DATA)
 		return 0;
 
 	link = iwl_mld_link_dereference_check(mld_vif, link_id);
@@ -1825,7 +1841,7 @@ static int iwl_mld_move_sta_state_up(struct iwl_mld *mld,
 		   new_state == IEEE80211_STA_AUTHORIZED) {
 		ret = 0;
 
-		if (!sta->tdls) {
+		if (vif->type == NL80211_IFTYPE_STATION && !sta->tdls) {
 			mld_vif->authorized = true;
 
 			/* Ensure any block due to a non-BSS link is synced */
@@ -1846,7 +1862,6 @@ static int iwl_mld_move_sta_state_up(struct iwl_mld *mld,
 						    FW_CTXT_ACTION_MODIFY);
 			if (ret)
 				return ret;
-			iwl_mld_smps_workaround(mld, vif, vif->cfg.ps);
 		}
 
 		/* MFP is set by default before the station is authorized.
@@ -1891,7 +1906,6 @@ static int iwl_mld_move_sta_state_down(struct iwl_mld *mld,
 						  &mld_vif->mlo_scan_start_wk);
 
 			iwl_mld_reset_cca_40mhz_workaround(mld, vif);
-			iwl_mld_smps_workaround(mld, vif, true);
 		}
 
 		/* once we move into assoc state, need to update the FW to
@@ -2236,7 +2250,6 @@ static int iwl_mld_set_key_add(struct iwl_mld *mld,
 
 	ret = iwl_mld_add_key(mld, vif, sta, key);
 	if (ret) {
-		IWL_WARN(mld, "set key failed (%d)\n", ret);
 		if (ptk_pn) {
 			RCU_INIT_POINTER(mld_sta->ptk_pn[keyidx], NULL);
 			kfree(ptk_pn);
@@ -2278,7 +2291,9 @@ static void iwl_mld_set_key_remove(struct iwl_mld *mld,
 	}
 
 	/* if this key was stored to be added later to the FW - free it here */
-	if (!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
+	if (!(key->flags & IEEE80211_KEY_FLAG_PAIRWISE) &&
+	    (vif->type == NL80211_IFTYPE_AP ||
+	     vif->type == NL80211_IFTYPE_ADHOC))
 		iwl_mld_free_ap_early_key(mld, key, mld_vif);
 
 	/* We already removed it */
@@ -2296,9 +2311,20 @@ static int iwl_mld_mac80211_set_key(struct ieee80211_hw *hw,
 				    struct ieee80211_sta *sta,
 				    struct ieee80211_key_conf *key)
 {
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
 	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
 	int ret;
 
+	/*
+	 * FW always needs the AP STA for client mode.
+	 * Note that during removal this could already
+	 * be NULL (mac80211 removes keys after STAs)
+	 * but then we'll already have removed the key
+	 * and set hw_key_idx = STA_KEY_IDX_INVALID.
+	 */
+	if (!sta && vif->type == NL80211_IFTYPE_STATION)
+		sta = mld_vif->ap_sta;
+
 	switch (cmd) {
 	case SET_KEY:
 		ret = iwl_mld_set_key_add(mld, vif, sta, key);
@@ -2559,15 +2585,108 @@ iwl_mld_mac80211_mgd_protect_tdls_discover(struct ieee80211_hw *hw,
 			ret);
 }
 
+static int iwl_mld_count_free_link_ids(struct iwl_mld *mld)
+{
+	int free_count = 0;
+
+	for (int i = 0; i < mld->fw->ucode_capa.num_links; i++) {
+		if (!rcu_access_pointer(mld->fw_id_to_bss_conf[i]))
+			free_count++;
+	}
+
+	return free_count;
+}
+
+static bool
+iwl_mld_chanctx_used_by_other_vif(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_chanctx_conf *chanctx_conf)
+{
+	struct ieee80211_bss_conf *iter_link;
+	struct ieee80211_vif *iter_vif;
+	int link_id;
+
+	for_each_active_interface(iter_vif, hw) {
+		if (vif == iter_vif)
+			continue;
+
+		/* NAN doesn't have active links, so we don't count NAN users */
+		for_each_vif_active_link(iter_vif, iter_link, link_id) {
+			if (rcu_access_pointer(iter_link->chanctx_conf) ==
+			    chanctx_conf)
+				return true;
+		}
+	}
+
+	return false;
+}
+
 static bool iwl_mld_can_activate_links(struct ieee80211_hw *hw,
 				       struct ieee80211_vif *vif,
 				       u16 desired_links)
 {
 	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
 	int n_links = hweight16(desired_links);
+	int n_add = hweight16(desired_links & ~vif->active_links);
+	unsigned long to_deactivate = vif->active_links & ~desired_links;
+	int free_link_ids;
+	int i;
 
 	/* Check if HW supports the wanted number of links */
-	return n_links <= iwl_mld_max_active_links(mld, vif);
+	if (n_links > iwl_mld_max_active_links(mld, vif))
+		return false;
+
+	/*
+	 * During link switch, mac80211 first adds the new links, then removes
+	 * the old ones. This means we temporarily need extra link objects
+	 * during the transition. Check if we have enough free link IDs.
+	 */
+
+	free_link_ids = iwl_mld_count_free_link_ids(mld);
+
+	if (free_link_ids >= n_add)
+		return true;
+
+	if (!mld->nan_device_vif)
+		return false;
+
+	/*
+	 * Not enough free link IDs. First try to evacuate NAN from the
+	 * channel context of a link that is going to be deactivated.
+	 */
+	for_each_set_bit(i, &to_deactivate, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_bss_conf *link_conf;
+		struct ieee80211_chanctx_conf *chanctx_conf;
+
+		link_conf = link_conf_dereference_protected(vif, i);
+		if (!link_conf)
+			continue;
+
+		chanctx_conf = wiphy_dereference(mld->wiphy, link_conf->chanctx_conf);
+		if (!chanctx_conf)
+			continue;
+
+		if (iwl_mld_chanctx_used_by_other_vif(hw, vif, chanctx_conf))
+			continue;
+
+		if (ieee80211_nan_try_evacuate(hw, chanctx_conf)) {
+			free_link_ids = iwl_mld_count_free_link_ids(mld);
+			/*
+			 * Evacuation of one channel should do the job. If not,
+			 * something bad is happening. Don't try to evacuate more
+			 */
+			return free_link_ids >= n_add;
+		}
+	}
+
+	/* Couldn't find/evacuate any channel going to go unused, try any */
+	if (ieee80211_nan_try_evacuate(hw, NULL)) {
+		free_link_ids = iwl_mld_count_free_link_ids(mld);
+		if (free_link_ids >= n_add)
+			return true;
+	}
+
+	return false;
 }
 
 static int
@@ -2847,4 +2966,5 @@ const struct ieee80211_ops iwl_mld_hw_ops = {
 	.start_nan = iwl_mld_start_nan,
 	.stop_nan = iwl_mld_stop_nan,
 	.nan_change_conf = iwl_mld_nan_change_config,
+	.nan_peer_sched_changed = iwl_mld_mac802111_nan_peer_sched_changed,
 };
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mcc.c b/drivers/net/wireless/intel/iwlwifi/mld/mcc.c
index 16bb1b4904f99..8502129abe493 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mcc.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mcc.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #include <net/cfg80211.h>
@@ -129,7 +129,7 @@ iwl_mld_get_regdomain(struct iwl_mld *mld,
 
 	mld->mcc_src = resp->source_id;
 
-	/* FM is the earliest supported and later always do puncturing */
+	/* FM follows BIOS/MCC policy, WH disallows puncturing only in US/CA. */
 	if (CSR_HW_RFID_TYPE(mld->trans->info.hw_rf_id) == IWL_CFG_RF_TYPE_FM) {
 		if (!iwl_puncturing_is_allowed_in_bios(mld->bios_enable_puncturing,
 						       le16_to_cpu(resp->mcc)))
@@ -137,6 +137,15 @@ iwl_mld_get_regdomain(struct iwl_mld *mld,
 		else
 			__clear_bit(IEEE80211_HW_DISALLOW_PUNCTURING,
 				    mld->hw->flags);
+	} else if (CSR_HW_RFID_TYPE(mld->trans->info.hw_rf_id) ==
+			IWL_CFG_RF_TYPE_WH) {
+		u16 mcc = le16_to_cpu(resp->mcc);
+
+		if (mcc == IWL_MCC_US || mcc == IWL_MCC_CANADA)
+			ieee80211_hw_set(mld->hw, DISALLOW_PUNCTURING);
+		else
+			__clear_bit(IEEE80211_HW_DISALLOW_PUNCTURING,
+				    mld->hw->flags);
 	}
 
 out:
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.c b/drivers/net/wireless/intel/iwlwifi/mld/mld.c
index 9af79297c3b6a..78c78cf891cda 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mld.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include <linux/rtnetlink.h>
 #include <net/mac80211.h>
@@ -162,7 +162,6 @@ static const struct iwl_hcmd_names iwl_mld_legacy_names[] = {
 	HCMD_NAME(PHY_CONFIGURATION_CMD),
 	HCMD_NAME(SCAN_OFFLOAD_UPDATE_PROFILES_CMD),
 	HCMD_NAME(POWER_TABLE_CMD),
-	HCMD_NAME(PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION),
 	HCMD_NAME(BEACON_NOTIFICATION),
 	HCMD_NAME(BEACON_TEMPLATE_CMD),
 	HCMD_NAME(TX_ANT_CONFIGURATION_CMD),
@@ -236,6 +235,10 @@ static const struct iwl_hcmd_names iwl_mld_mac_conf_names[] = {
 	HCMD_NAME(STA_REMOVE_CMD),
 	HCMD_NAME(ROC_CMD),
 	HCMD_NAME(NAN_CFG_CMD),
+	HCMD_NAME(NAN_SCHEDULE_CMD),
+	HCMD_NAME(NAN_PEER_CMD),
+	HCMD_NAME(NAN_ULW_ATTR_NOTIF),
+	HCMD_NAME(NAN_SCHED_UPDATE_COMPLETED_NOTIF),
 	HCMD_NAME(NAN_DW_END_NOTIF),
 	HCMD_NAME(NAN_JOINED_CLUSTER_NOTIF),
 	HCMD_NAME(MISSED_BEACONS_NOTIF),
@@ -259,6 +262,7 @@ static const struct iwl_hcmd_names iwl_mld_data_path_names[] = {
 	HCMD_NAME(RX_BAID_ALLOCATION_CONFIG_CMD),
 	HCMD_NAME(SCD_QUEUE_CONFIG_CMD),
 	HCMD_NAME(SEC_KEY_CMD),
+	HCMD_NAME(RSC_NOTIF),
 	HCMD_NAME(ESR_MODE_NOTIF),
 	HCMD_NAME(MONITOR_NOTIF),
 	HCMD_NAME(TLC_MNG_UPDATE_NOTIF),
@@ -411,9 +415,6 @@ iwl_op_mode_mld_start(struct iwl_trans *trans, const struct iwl_rf_cfg *cfg,
 
 	iwl_construct_mld(mld, trans, cfg, fw, hw, dbgfs_dir);
 
-	/* we'll verify later it matches between commands */
-	mld->fw_rates_ver_3 = iwl_fw_lookup_cmd_ver(mld->fw, TX_CMD, 0) >= 11;
-
 	iwl_mld_construct_fw_runtime(mld, trans, fw, dbgfs_dir);
 
 	iwl_mld_get_bios_tables(mld);
@@ -675,6 +676,15 @@ iwl_mld_nic_error(struct iwl_op_mode *op_mode,
 	if (type != IWL_ERR_TYPE_RESET_HS_TIMEOUT &&
 	    mld->fw_status.running)
 		mld->fw_status.in_hw_restart = true;
+
+	/* FW is dead. We don't want to process its notifications.
+	 * Right, we cancel them also in iwl_mld_stop_fw, but
+	 * iwl_mld_async_handlers_wk might be executed before
+	 * ieee80211_restart_work.
+	 * In addition, in case of an error during recovery,
+	 * iwl_mld_stop_fw might be too late.
+	 */
+	iwl_mld_cancel_async_notifications(mld);
 }
 
 static void iwl_mld_dump_error(struct iwl_op_mode *op_mode,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mld.h b/drivers/net/wireless/intel/iwlwifi/mld/mld.h
index 606cb64f8ea4f..922aa3dbff543 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mld.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mld.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #ifndef __iwl_mld_h__
 #define __iwl_mld_h__
@@ -189,7 +189,6 @@
  *	TX rate_n_flags for non-STA mgmt frames (toggles on every TX failure).
  * @set_tx_ant: stores the last TX antenna bitmask set by user space (if any)
  * @set_rx_ant: stores the last RX antenna bitmask set by user space (if any)
- * @fw_rates_ver_3: FW rates are in version 3
  * @low_latency: low-latency manager.
  * @tzone: thermal zone device's data
  * @cooling_dev: cooling device's related data
@@ -299,8 +298,6 @@ struct iwl_mld {
 	u8 set_tx_ant;
 	u8 set_rx_ant;
 
-	bool fw_rates_ver_3;
-
 	struct iwl_mld_low_latency low_latency;
 
 	bool ibss_manager;
@@ -555,15 +552,24 @@ iwl_mld_allocate_##_type##_fw_id(struct iwl_mld *mld,					\
 	return -ENOSPC;									\
 }
 
+#define IWL_MLD_ALLOC_FN_STATIC(_type, _mac80211_type) \
+static IWL_MLD_ALLOC_FN(_type, _mac80211_type)
+
 static inline struct ieee80211_bss_conf *
 iwl_mld_fw_id_to_link_conf(struct iwl_mld *mld, u8 fw_link_id)
 {
+	struct ieee80211_bss_conf *link;
+
 	if (IWL_FW_CHECK(mld, fw_link_id >= mld->fw->ucode_capa.num_links,
 			 "Invalid fw_link_id: %d\n", fw_link_id))
 		return NULL;
 
-	return wiphy_dereference(mld->wiphy,
+	link = wiphy_dereference(mld->wiphy,
 				 mld->fw_id_to_bss_conf[fw_link_id]);
+	if (IS_ERR(link))
+		return NULL;
+
+	return link;
 }
 
 #define MSEC_TO_TU(_msec)	((_msec) * 1000 / 1024)
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c
index 8227ccb31d604..a2f8a69575350 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/mlo.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/mlo.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include "mlo.h"
 #include "phy.h"
@@ -1081,8 +1081,13 @@ static void iwl_mld_chan_load_update_iter(void *_data, u8 *mac,
 		container_of((const void *)phy, struct ieee80211_chanctx_conf,
 			     drv_priv);
 	struct iwl_mld *mld = iwl_mld_vif_from_mac80211(vif)->mld;
-	struct ieee80211_bss_conf *prim_link;
+	u32 new_chan_load = phy->avg_channel_load_not_by_us;
+	struct ieee80211_bss_conf *prim_link, *link_conf;
 	unsigned int prim_link_id;
+	int link_id;
+
+	if (!ieee80211_vif_is_mld(vif) || hweight16(vif->valid_links) <= 1)
+		return;
 
 	prim_link_id = iwl_mld_get_primary_link(vif);
 	prim_link = link_conf_dereference_protected(vif, prim_link_id);
@@ -1090,6 +1095,32 @@ static void iwl_mld_chan_load_update_iter(void *_data, u8 *mac,
 	if (WARN_ON(!prim_link))
 		return;
 
+	/* Evaluate MLO Internal Scan for high chan load beyond thresholds */
+	for_each_vif_active_link(vif, link_conf, link_id) {
+		if (rcu_access_pointer(link_conf->chanctx_conf) != chanctx)
+			continue;
+
+		/* No QBSS IE - links will be selected based on default channel
+		 * load values, so the same link will be selected again.
+		 * No point in scan.
+		 */
+		if (iwl_mld_get_chan_load_from_element(mld, link_conf) < 0)
+			continue;
+
+		if (iwl_mld_chan_load_requires_scan(mld,
+						    link_conf,
+						    new_chan_load)) {
+			/* When EMLSR is active, only trigger scan based on
+			 * primary link
+			 */
+			if (iwl_mld_emlsr_active(vif) && link_conf != prim_link)
+				continue;
+
+			iwl_mld_int_mlo_scan(mld, vif);
+			return;
+		}
+	}
+
 	if (chanctx != rcu_access_pointer(prim_link->chanctx_conf))
 		return;
 
@@ -1107,7 +1138,6 @@ static void iwl_mld_chan_load_update_iter(void *_data, u8 *mac,
 					   prim_link_id);
 	} else {
 		u32 old_chan_load = data->prev_chan_load_not_by_us;
-		u32 new_chan_load = phy->avg_channel_load_not_by_us;
 		u32 min_thresh = iwl_mld_get_min_chan_load_thresh(chanctx);
 
 #define THRESHOLD_CROSSED(threshold) \
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/nan.c b/drivers/net/wireless/intel/iwlwifi/mld/nan.c
index 4d8e85f2bd7c2..d34a9a2cbeae3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/nan.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/nan.c
@@ -5,8 +5,12 @@
 
 #include "mld.h"
 #include "iface.h"
+#include "link.h"
 #include "mlo.h"
+#include "tlc.h"
 #include "fw/api/mac-cfg.h"
+#include "fw/api/mac.h"
+#include "fw/api/rs.h"
 
 #define IWL_NAN_DISOVERY_BEACON_INTERNVAL_TU 512
 #define IWL_NAN_RSSI_CLOSE 55
@@ -14,8 +18,16 @@
 
 bool iwl_mld_nan_supported(struct iwl_mld *mld)
 {
-	return fw_has_capa(&mld->fw->ucode_capa,
-			   IWL_UCODE_TLV_CAPA_NAN_SYNC_SUPPORT);
+	const struct iwl_fw *fw = mld->fw;
+
+	if (fw_has_capa(&fw->ucode_capa, IWL_UCODE_TLV_CAPA_NAN_SYNC_SUPPORT) &&
+	    iwl_fw_lookup_cmd_ver(fw, WIDE_ID(MAC_CONF_GROUP, NAN_SCHEDULE_CMD), 0) >= 1 &&
+	    iwl_fw_lookup_cmd_ver(fw, WIDE_ID(MAC_CONF_GROUP, NAN_PEER_CMD), 0) >= 1 &&
+	    iwl_fw_lookup_cmd_ver(fw, WIDE_ID(MAC_CONF_GROUP, STA_CONFIG_CMD), 0) >= 3 &&
+	    iwl_fw_lookup_cmd_ver(fw, WIDE_ID(MAC_CONF_GROUP, MAC_CONFIG_CMD), 0) >= 4 &&
+	    iwl_fw_lookup_cmd_ver(fw, WIDE_ID(DATA_PATH_GROUP, TLC_MNG_CONFIG_CMD), 0) >= 6)
+		return true;
+	return false;
 }
 
 static int iwl_mld_nan_send_config_cmd(struct iwl_mld *mld,
@@ -38,6 +50,98 @@ static int iwl_mld_nan_send_config_cmd(struct iwl_mld *mld,
 	return iwl_mld_send_cmd(mld, &hcmd);
 }
 
+bool iwl_mld_nan_use_nan_stations(struct iwl_mld *mld)
+{
+	/*
+	 * If the FW supports version 1 of the NAN config command, it means that
+	 * it needs to receive the station ID of the auxiliary station in the
+	 * NAN configuration command. Otherwise, use the NAN dedicated station
+	 * types.
+	 */
+	return iwl_fw_lookup_cmd_ver(mld->fw,
+				     WIDE_ID(MAC_CONF_GROUP,
+					     NAN_CFG_CMD), 1) != 1;
+}
+
+static const struct iwl_mld_int_sta *
+iwl_mld_nan_get_mgmt_sta(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	const struct iwl_mld_int_sta *sta;
+
+	if (iwl_mld_nan_use_nan_stations(mld))
+		sta = &mld_vif->nan.mgmt_sta;
+	else
+		sta = &mld_vif->aux_sta;
+
+	if (WARN_ON(sta->sta_id == IWL_INVALID_STA))
+		return NULL;
+
+	return sta;
+}
+
+int iwl_mld_nan_get_mgmt_queue(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	const struct iwl_mld_int_sta *sta = iwl_mld_nan_get_mgmt_sta(mld, vif);
+
+	if (!sta)
+		return IWL_MLD_INVALID_QUEUE;
+
+	return sta->queue_id;
+}
+
+static void iwl_mld_nan_flush(struct iwl_mld *mld, struct ieee80211_vif *vif)
+{
+	const struct iwl_mld_int_sta *sta = iwl_mld_nan_get_mgmt_sta(mld, vif);
+
+	if (!sta)
+		return;
+
+	if (WARN_ON(sta->queue_id == IWL_MLD_INVALID_QUEUE))
+		return;
+
+	IWL_DEBUG_INFO(mld, "NAN: flush queues for sta=%u\n",
+		       sta->sta_id);
+
+	iwl_mld_flush_link_sta_txqs(mld, sta->sta_id);
+}
+
+static void iwl_mld_nan_remove_stations(struct iwl_mld *mld,
+					struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	iwl_mld_nan_flush(mld, vif);
+
+	if (!iwl_mld_nan_use_nan_stations(mld)) {
+		iwl_mld_remove_aux_sta(mld, vif);
+		return;
+	}
+
+	iwl_mld_remove_nan_bcast_sta(mld, &mld_vif->nan.bcast_sta);
+	iwl_mld_remove_nan_mgmt_sta(mld, &mld_vif->nan.mgmt_sta);
+}
+
+static int iwl_mld_nan_add_stations(struct iwl_mld *mld,
+				    struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	int ret;
+
+	if (!iwl_mld_nan_use_nan_stations(mld))
+		return iwl_mld_add_aux_sta(mld, &mld_vif->aux_sta);
+
+	ret = iwl_mld_add_nan_bcast_sta(mld, &mld_vif->nan.bcast_sta);
+	if (ret)
+		return ret;
+
+	ret = iwl_mld_add_nan_mgmt_sta(mld, &mld_vif->nan.mgmt_sta);
+	if (ret)
+		iwl_mld_remove_nan_bcast_sta(mld, &mld_vif->nan.bcast_sta);
+
+	return ret;
+}
+
 static int iwl_mld_nan_config(struct iwl_mld *mld,
 			      struct ieee80211_vif *vif,
 			      struct cfg80211_nan_conf *conf,
@@ -114,7 +218,12 @@ static int iwl_mld_nan_config(struct iwl_mld *mld,
 			       conf->vendor_elems_len);
 	}
 
-	cmd.sta_id = mld_vif->aux_sta.sta_id;
+	/* FW needs to know about the station ID only with version 1 of the
+	 * NAN configuration command
+	 */
+	if (!iwl_mld_nan_use_nan_stations(mld))
+		cmd.sta_id = mld_vif->aux_sta.sta_id;
+
 	return iwl_mld_nan_send_config_cmd(mld, &cmd, data,
 					   conf->extra_nan_attrs_len +
 					   conf->vendor_elems_len);
@@ -124,8 +233,6 @@ int iwl_mld_start_nan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 		      struct cfg80211_nan_conf *conf)
 {
 	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
-	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
-	struct iwl_mld_int_sta *aux_sta = &mld_vif->aux_sta;
 	int ret;
 
 	IWL_DEBUG_MAC80211(mld, "NAN: start: bands=0x%x\n", conf->bands);
@@ -134,19 +241,20 @@ int iwl_mld_start_nan(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
 	if (ret)
 		return ret;
 
-	ret = iwl_mld_add_aux_sta(mld, aux_sta);
+	ret = iwl_mld_nan_add_stations(mld, vif);
 	if (ret)
 		goto unblock_emlsr;
 
 	ret = iwl_mld_nan_config(mld, vif, conf, FW_CTXT_ACTION_ADD);
 	if (ret) {
 		IWL_ERR(mld, "Failed to start NAN. ret=%d\n", ret);
-		goto remove_aux;
+		goto remove_stas;
 	}
+
 	return 0;
 
-remove_aux:
-	iwl_mld_remove_aux_sta(mld, vif);
+remove_stas:
+	iwl_mld_nan_remove_stations(mld, vif);
 unblock_emlsr:
 	iwl_mld_update_emlsr_block(mld, false, IWL_MLD_EMLSR_BLOCKED_NAN);
 
@@ -174,7 +282,6 @@ int iwl_mld_stop_nan(struct ieee80211_hw *hw,
 		     struct ieee80211_vif *vif)
 {
 	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
-	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
 	struct iwl_nan_config_cmd cmd = {
 		.action = cpu_to_le32(FW_CTXT_ACTION_REMOVE),
 	};
@@ -191,8 +298,7 @@ int iwl_mld_stop_nan(struct ieee80211_hw *hw,
 	/* assume that higher layer guarantees that no additional frames are
 	 * added before calling this callback
 	 */
-	iwl_mld_flush_link_sta_txqs(mld, mld_vif->aux_sta.sta_id);
-	iwl_mld_remove_aux_sta(mld, vif);
+	iwl_mld_nan_remove_stations(mld, vif);
 
 	/* cancel based on object type being NAN, as the NAN objects do
 	 * not have a unique identifier associated with them
@@ -247,6 +353,39 @@ bool iwl_mld_cancel_nan_dw_end_notif(struct iwl_mld *mld,
 	return true;
 }
 
+bool iwl_mld_cancel_nan_ulw_attr_notif(struct iwl_mld *mld,
+				       struct iwl_rx_packet *pkt,
+				       u32 obj_id)
+{
+	return true;
+}
+
+void iwl_mld_handle_nan_ulw_attr_notif(struct iwl_mld *mld,
+				       struct iwl_rx_packet *pkt)
+{
+	struct iwl_nan_ulw_attr_notif *notif = (void *)pkt->data;
+	struct wireless_dev *wdev;
+
+	IWL_DEBUG_INFO(mld, "NAN: ULW attr update: len=%u\n", notif->attr_len);
+
+	if (IWL_FW_CHECK(mld, !mld->nan_device_vif,
+			 "NAN: ULW attr update without NAN vif\n"))
+		return;
+
+	if (IWL_FW_CHECK(mld, !ieee80211_vif_nan_started(mld->nan_device_vif),
+			 "NAN: ULW attr update without NAN started\n"))
+		return;
+
+	if (IWL_FW_CHECK(mld,
+			 notif->attr_len > IWL_NAN_MAX_ENDLESS_ULW_ATTR_LEN,
+			 "NAN: ULW attr update invalid len %u\n",
+			 notif->attr_len))
+		return;
+
+	wdev = ieee80211_vif_to_wdev(mld->nan_device_vif);
+	cfg80211_nan_ulw_update(wdev, notif->attr, notif->attr_len, GFP_KERNEL);
+}
+
 void iwl_mld_handle_nan_dw_end_notif(struct iwl_mld *mld,
 				     struct iwl_rx_packet *pkt)
 {
@@ -257,7 +396,7 @@ void iwl_mld_handle_nan_dw_end_notif(struct iwl_mld *mld,
 	struct wireless_dev *wdev;
 	struct ieee80211_channel *chan;
 
-	IWL_INFO(mld, "NAN: DW end: band=%u\n", notif->band);
+	IWL_DEBUG_INFO(mld, "NAN: DW end: band=%u\n", notif->band);
 
 	if (IWL_FW_CHECK(mld, !mld_vif, "NAN: DW end without mld_vif\n"))
 		return;
@@ -266,13 +405,7 @@ void iwl_mld_handle_nan_dw_end_notif(struct iwl_mld *mld,
 			 "NAN: DW end without NAN started\n"))
 		return;
 
-	if (WARN_ON(mld_vif->aux_sta.sta_id == IWL_INVALID_STA))
-		return;
-
-	IWL_DEBUG_INFO(mld, "NAN: flush queues for aux sta=%u\n",
-		       mld_vif->aux_sta.sta_id);
-
-	iwl_mld_flush_link_sta_txqs(mld, mld_vif->aux_sta.sta_id);
+	iwl_mld_nan_flush(mld, mld->nan_device_vif);
 
 	/* TODO: currently the notification specified the band on which the DW
 	 * ended. Need to change that to the actual channel on which the next DW
@@ -293,6 +426,581 @@ void iwl_mld_handle_nan_dw_end_notif(struct iwl_mld *mld,
 		return;
 	}
 
+	if (WARN_ON_ONCE(!chan))
+		return;
+
 	wdev = ieee80211_vif_to_wdev(mld->nan_device_vif);
 	cfg80211_next_nan_dw_notif(wdev, chan, GFP_KERNEL);
 }
+
+static void iwl_mld_nan_fill_rates(struct iwl_link_config_cmd *cmd)
+{
+	u32 ofdm = 0;
+
+	/* All OFDM rates - NAN uses OFDM only, no CCK */
+	ofdm |= IWL_RATE_BIT_MSK(6) >> IWL_FIRST_OFDM_RATE;
+	ofdm |= IWL_RATE_BIT_MSK(9) >> IWL_FIRST_OFDM_RATE;
+	ofdm |= IWL_RATE_BIT_MSK(12) >> IWL_FIRST_OFDM_RATE;
+	ofdm |= IWL_RATE_BIT_MSK(18) >> IWL_FIRST_OFDM_RATE;
+	ofdm |= IWL_RATE_BIT_MSK(24) >> IWL_FIRST_OFDM_RATE;
+	ofdm |= IWL_RATE_BIT_MSK(36) >> IWL_FIRST_OFDM_RATE;
+	ofdm |= IWL_RATE_BIT_MSK(48) >> IWL_FIRST_OFDM_RATE;
+	ofdm |= IWL_RATE_BIT_MSK(54) >> IWL_FIRST_OFDM_RATE;
+
+	cmd->ofdm_rates = cpu_to_le32(ofdm);
+	cmd->short_slot = cpu_to_le32(1);
+}
+
+static void iwl_mld_nan_fill_qos(struct iwl_ac_qos *ac, __le32 *qos_flags)
+{
+	/* AC_BK: CWmin=15, CWmax=1023, AIFSN=7, TXOP=0 */
+	ac[AC_BK].cw_min = cpu_to_le16(15);
+	ac[AC_BK].cw_max = cpu_to_le16(1023);
+	ac[AC_BK].aifsn = 7;
+	ac[AC_BK].fifos_mask = BIT(IWL_BZ_EDCA_TX_FIFO_BK);
+	ac[AC_BK].edca_txop = 0;
+
+	/* AC_BE: CWmin=15, CWmax=1023, AIFSN=3, TXOP=0 */
+	ac[AC_BE].cw_min = cpu_to_le16(15);
+	ac[AC_BE].cw_max = cpu_to_le16(1023);
+	ac[AC_BE].aifsn = 3;
+	ac[AC_BE].fifos_mask = BIT(IWL_BZ_EDCA_TX_FIFO_BE);
+	ac[AC_BE].edca_txop = 0;
+
+	/* AC_VI: CWmin=7, CWmax=15, AIFSN=2, TXOP=3008us */
+	ac[AC_VI].cw_min = cpu_to_le16(7);
+	ac[AC_VI].cw_max = cpu_to_le16(15);
+	ac[AC_VI].aifsn = 2;
+	ac[AC_VI].fifos_mask = BIT(IWL_BZ_EDCA_TX_FIFO_VI);
+	ac[AC_VI].edca_txop = cpu_to_le16(3008);
+
+	/* AC_VO: CWmin=3, CWmax=7, AIFSN=2, TXOP=1504us */
+	ac[AC_VO].cw_min = cpu_to_le16(3);
+	ac[AC_VO].cw_max = cpu_to_le16(7);
+	ac[AC_VO].aifsn = 2;
+	ac[AC_VO].fifos_mask = BIT(IWL_BZ_EDCA_TX_FIFO_VO);
+	ac[AC_VO].edca_txop = cpu_to_le16(1504);
+
+	*qos_flags |= cpu_to_le32(MAC_QOS_FLG_UPDATE_EDCA);
+}
+
+static void
+iwl_mld_nan_link_prep_cmd(struct iwl_mld *mld,
+			  struct iwl_mld_nan_link *nan_link,
+			  struct iwl_link_config_cmd *cmd,
+			  u32 modify_flags)
+{
+	struct ieee80211_vif *vif = mld->nan_device_vif;
+	struct iwl_mld_vif *mld_vif;
+
+	if (WARN_ON_ONCE(!vif))
+		return;
+
+	mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	memset(cmd, 0, sizeof(*cmd));
+
+	if (!nan_link->chanctx) {
+		cmd->phy_id = cpu_to_le32(FW_CTXT_ID_INVALID);
+	} else {
+		struct iwl_mld_phy *mld_phy;
+
+		mld_phy = iwl_mld_phy_from_mac80211(nan_link->chanctx);
+		cmd->phy_id = cpu_to_le32(mld_phy->fw_id);
+	}
+
+	if (modify_flags & LINK_CONTEXT_MODIFY_RATES_INFO)
+		iwl_mld_nan_fill_rates(cmd);
+
+	if (modify_flags & LINK_CONTEXT_MODIFY_QOS_PARAMS)
+		iwl_mld_nan_fill_qos(cmd->ac, &cmd->qos_flags);
+
+	cmd->link_id = cpu_to_le32(nan_link->fw_id);
+	cmd->mac_id = cpu_to_le32(mld_vif->fw_id);
+	cmd->active = cpu_to_le32(nan_link->active);
+
+	ether_addr_copy(cmd->local_link_addr, vif->addr);
+
+	cmd->modify_mask = cpu_to_le32(modify_flags);
+}
+
+static struct iwl_mld_nan_link *
+iwl_mld_nan_link_add(struct iwl_mld *mld,
+		     struct iwl_mld_vif *mld_vif,
+		     struct ieee80211_chanctx_conf *chanctx)
+{
+	struct iwl_mld_nan_link *nan_link;
+	struct iwl_link_config_cmd cmd;
+	u8 fw_id;
+	int ret;
+
+	lockdep_assert_wiphy(mld->wiphy);
+
+	ret = iwl_mld_allocate_link_fw_id(mld, &fw_id, ERR_PTR(-ENODEV));
+	/*
+	 * We should always have enough links. The schedule contains up to 3,
+	 * and the BSS vif cannot do EMLSR - so can only have 1.
+	 */
+	if (WARN_ON(ret < 0))
+		return NULL;
+
+	nan_link = &mld_vif->nan.links[fw_id];
+
+	if (WARN_ON_ONCE(nan_link->fw_id != FW_CTXT_ID_INVALID))
+		goto err;
+
+	nan_link->fw_id = fw_id;
+	nan_link->chanctx = chanctx;
+
+	iwl_mld_nan_link_prep_cmd(mld, nan_link, &cmd,
+				  LINK_CONTEXT_MODIFY_RATES_INFO |
+				  LINK_CONTEXT_MODIFY_QOS_PARAMS);
+
+	ret = iwl_mld_send_link_cmd(mld, &cmd, FW_CTXT_ACTION_ADD);
+	if (ret) {
+		nan_link->fw_id = FW_CTXT_ID_INVALID;
+		nan_link->chanctx = NULL;
+		goto err;
+	}
+
+	return nan_link;
+err:
+	RCU_INIT_POINTER(mld->fw_id_to_bss_conf[fw_id], NULL);
+	return NULL;
+}
+
+static int iwl_mld_nan_link_set_active(struct iwl_mld *mld,
+				       struct ieee80211_vif *vif,
+				       struct iwl_mld_nan_link *nan_link,
+				       bool active)
+{
+	struct iwl_link_config_cmd cmd;
+	struct ieee80211_sta *sta;
+	int ret;
+
+	if (nan_link->active == active)
+		return 0;
+
+	if (active) {
+		for_each_station(sta, mld->hw) {
+			struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+			if (mld_sta->sta_type == STATION_TYPE_NAN_PEER_NDI)
+				iwl_mld_config_tlc(mld, mld_sta->vif, sta);
+		}
+	}
+
+	nan_link->active = active;
+
+	iwl_mld_nan_link_prep_cmd(mld, nan_link, &cmd,
+				  LINK_CONTEXT_MODIFY_ACTIVE);
+
+	ret = iwl_mld_send_link_cmd(mld, &cmd, FW_CTXT_ACTION_MODIFY);
+	if (ret) {
+		nan_link->active = !nan_link->active;
+		return ret;
+	}
+
+	if (!active) {
+		nan_link->chanctx = NULL;
+		/* TODO: when FW is ready, Update phy in TLC to invalid after */
+	}
+
+	return 0;
+}
+
+static void iwl_mld_nan_link_remove(struct iwl_mld *mld,
+				    struct iwl_mld_nan_link *nan_link,
+				    u32 link_id)
+{
+	struct iwl_link_config_cmd cmd = {
+		.link_id = cpu_to_le32(link_id),
+		.phy_id = cpu_to_le32(FW_CTXT_ID_INVALID),
+	};
+
+	iwl_mld_send_link_cmd(mld, &cmd, FW_CTXT_ACTION_REMOVE);
+
+	RCU_INIT_POINTER(mld->fw_id_to_bss_conf[link_id], NULL);
+	nan_link->fw_id = FW_CTXT_ID_INVALID;
+	nan_link->active = false;
+	nan_link->chanctx = NULL;
+}
+
+static bool iwl_mld_nan_have_links(struct iwl_mld_vif *mld_vif)
+{
+	struct iwl_mld_nan_link *nan_link;
+
+	for_each_mld_nan_valid_link(mld_vif, nan_link)
+		return true;
+
+	return false;
+}
+
+static struct iwl_mld_nan_link *
+iwl_mld_nan_find_link(struct iwl_mld_vif *mld_vif,
+		      struct ieee80211_chanctx_conf *chanctx)
+{
+	struct iwl_mld_nan_link *nan_link;
+
+	for_each_mld_nan_valid_link(mld_vif, nan_link) {
+		if (nan_link->chanctx == chanctx)
+			return nan_link;
+	}
+
+	return NULL;
+}
+
+static void iwl_mld_nan_set_mcast_data_links(struct ieee80211_vif *vif)
+{
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	if (vif->type != NL80211_IFTYPE_NAN_DATA)
+		return;
+
+	/* Note that all errors are handled internally so nothing to do
+	 * with the return value (used only to silence compilation warnings)
+	 */
+	iwl_mld_update_nan_mcast_data_sta(mld_vif->mld, vif->addr,
+					  &mld_vif->nan.mcast_data_sta);
+}
+
+void iwl_mld_nan_vif_cfg_changed(struct iwl_mld *mld,
+				 struct ieee80211_vif *vif,
+				 u64 changes)
+{
+	struct iwl_nan_schedule_cmd cmd = {};
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	bool previously_empty_schedule = !iwl_mld_nan_have_links(mld_vif);
+	struct ieee80211_nan_sched_cfg *sched_cfg = &vif->cfg.nan_sched;
+	struct iwl_mld_nan_link *links[ARRAY_SIZE(sched_cfg->channels)] = {};
+	struct ieee80211_nan_channel **slots = sched_cfg->schedule;
+	bool link_used[ARRAY_SIZE(mld_vif->nan.links)] = {};
+	struct iwl_mld_nan_link *nan_link;
+	unsigned long remove_link_ids = 0;
+	bool added_links = false;
+	bool empty_schedule = true;
+	int ret, i;
+	u16 cmd_size;
+	u32 cmd_id = WIDE_ID(MAC_CONF_GROUP, NAN_SCHEDULE_CMD);
+	u8 version = iwl_fw_lookup_cmd_ver(mld->fw, cmd_id, 0);
+
+	if (!(changes & BSS_CHANGED_NAN_LOCAL_SCHED))
+		return;
+
+	switch (version) {
+	case 1:
+		if (sched_cfg->deferred) {
+			IWL_ERR(mld,
+				"NAN: deferred schedule not supported by FW\n");
+			return;
+		}
+
+		cmd_size = sizeof(struct iwl_nan_schedule_cmd_v1);
+		break;
+	case 2:
+		cmd_size = sizeof(struct iwl_nan_schedule_cmd);
+
+		if (sched_cfg->deferred)
+			cmd.deferred = 1;
+
+		if (sched_cfg->avail_blob_len &&
+		    !WARN_ON(sched_cfg->avail_blob_len >
+			     sizeof(cmd.avail_attr.attr))) {
+			cmd.avail_attr.attr_len = sched_cfg->avail_blob_len;
+			memcpy(cmd.avail_attr.attr, sched_cfg->avail_blob,
+			       sched_cfg->avail_blob_len);
+		}
+		break;
+	default:
+		IWL_ERR(mld, "NAN: unsupported NAN schedule cmd version %d\n",
+			version);
+		return;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(sched_cfg->channels); i++) {
+		if (!sched_cfg->channels[i].chanreq.oper.chan)
+			continue;
+		empty_schedule = false;
+		break;
+	}
+
+	/* add the MAC if needed (before adding links) */
+	if (!empty_schedule && previously_empty_schedule) {
+		WARN_ON(mld_vif->nan.mac_added);
+		ret = iwl_mld_add_nan_vif(mld, vif);
+
+		if (ret) {
+			IWL_ERR(mld, "NAN: failed to add MAC (%d)\n", ret);
+			return;
+		}
+	}
+
+	if (!mld_vif->nan.mac_added) {
+		/* nothing to do */
+		return;
+	}
+
+	/* this currently just uses the same index */
+	BUILD_BUG_ON(ARRAY_SIZE(sched_cfg->channels) !=
+		     ARRAY_SIZE(cmd.channels));
+
+	/*
+	 * mac80211 removes unused channels before adding new ones, so it may
+	 * update an empty schedule with an availability attribute because it
+	 * is going to add channels later. Since the firmware does not expect
+	 * an availability attribute without channels, ignore it in that case.
+	 */
+	if (empty_schedule)
+		cmd.avail_attr.attr_len = 0;
+
+	/* find links we can keep (same chanctx/PHY) */
+	for (i = 0; i < ARRAY_SIZE(sched_cfg->channels); i++) {
+		struct ieee80211_chanctx_conf *chanctx;
+		struct iwl_mld_nan_link *link;
+
+		chanctx = sched_cfg->channels[i].chanctx_conf;
+		/* ULW */
+		if (!chanctx)
+			continue;
+
+		link = iwl_mld_nan_find_link(mld_vif, chanctx);
+		links[i] = link;
+		if (link)
+			link_used[link->fw_id] = true;
+	}
+
+	/* add/reassign links for new channels */
+	for (i = 0; i < ARRAY_SIZE(sched_cfg->channels); i++) {
+		struct ieee80211_chanctx_conf *chanctx;
+
+		/* already have an existing active link */
+		if (links[i])
+			continue;
+
+		chanctx = sched_cfg->channels[i].chanctx_conf;
+		/* ULW or unused slot */
+		if (!chanctx)
+			continue;
+
+		/*
+		 * if this fails we still update the schedule, but
+		 * without a valid link we'll always ULW it
+		 */
+		links[i] = iwl_mld_nan_link_add(mld, mld_vif, chanctx);
+
+		/* we have a link, activate it */
+		if (links[i]) {
+			added_links = true;
+			link_used[links[i]->fw_id] = true;
+			iwl_mld_nan_link_set_active(mld, vif, links[i], true);
+		}
+	}
+
+	/* fill the command */
+	for (i = 0; i < ARRAY_SIZE(sched_cfg->channels); i++) {
+		cmd.channels[i].link_id = FW_CTXT_ID_INVALID;
+
+		if (!sched_cfg->channels[i].chanreq.oper.chan)
+			continue;
+
+		memcpy(cmd.channels[i].channel_entry,
+		       sched_cfg->channels[i].channel_entry, 6);
+		cmd.channels[i].link_id =
+			links[i] ? links[i]->fw_id : FW_CTXT_ID_INVALID;
+	}
+
+	for (i = 0; i < CFG80211_NAN_SCHED_NUM_TIME_SLOTS; i++) {
+		int chan_idx;
+
+		if (!slots[i])
+			continue;
+
+		chan_idx = slots[i] - sched_cfg->channels;
+		if (WARN_ON_ONCE(chan_idx < 0 ||
+				 chan_idx >= ARRAY_SIZE(cmd.channels)))
+			continue;
+
+		cmd.channels[chan_idx].availability_map |= cpu_to_le32(BIT(i));
+	}
+
+	ret = iwl_mld_send_cmd_pdu(mld, cmd_id, &cmd, cmd_size);
+	if (ret)
+		IWL_ERR(mld, "NAN: failed to update schedule (%d)\n", ret);
+
+	/* prepare stations for links we'll remove */
+	for_each_mld_nan_valid_link(mld_vif, nan_link) {
+		if (!link_used[nan_link->fw_id]) {
+			iwl_mld_nan_link_set_active(mld, vif, nan_link, false);
+			remove_link_ids |= BIT(nan_link->fw_id);
+			/* mark unused for STA updates */
+			nan_link->fw_id = FW_CTXT_ID_INVALID;
+		}
+	}
+
+	if (added_links || remove_link_ids) {
+		struct ieee80211_sta *sta;
+
+		for_each_station(sta, mld->hw) {
+			struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+
+			if (mld_sta->sta_type == STATION_TYPE_NAN_PEER_NMI ||
+			    mld_sta->sta_type == STATION_TYPE_NAN_PEER_NDI)
+				iwl_mld_add_modify_sta_cmd(mld, &sta->deflink);
+		}
+
+		/*
+		 * Iterate over all the NAN Data interfaces and update the links
+		 * for the internal multicast data station.
+		 * In recovery - the station will be added later in
+		 * drv_add_interface
+		 */
+		if (iwl_mld_nan_use_nan_stations(mld) && !mld->fw_status.in_hw_restart) {
+			struct ieee80211_vif *iter;
+
+			for_each_active_interface(iter, mld->hw)
+				iwl_mld_nan_set_mcast_data_links(iter);
+		}
+	}
+
+	/* delete unused links */
+	for_each_set_bit(i, &remove_link_ids, ARRAY_SIZE(mld_vif->nan.links))
+		iwl_mld_nan_link_remove(mld, &mld_vif->nan.links[i], i);
+
+	/* remove MAC if needed */
+	if (!previously_empty_schedule && empty_schedule) {
+		/* must have been added */
+		WARN_ON_ONCE(!mld_vif->nan.mac_added);
+
+		/* mac80211 should reconfigure same state */
+		if (!WARN_ON_ONCE(mld->fw_status.in_hw_restart &&
+				  !iwl_mld_error_before_recovery(mld)))
+			iwl_mld_rm_vif(mld, vif);
+	}
+}
+
+bool iwl_mld_cancel_nan_sched_update_completed_notif(struct iwl_mld *mld,
+						     struct iwl_rx_packet *pkt,
+						     u32 obj_id)
+{
+	return true;
+}
+
+void iwl_mld_handle_nan_sched_update_completed_notif(struct iwl_mld *mld,
+						     struct iwl_rx_packet *pkt)
+{
+	struct iwl_nan_sched_update_completed_notif *notif = (void *)pkt->data;
+	struct ieee80211_vif *vif = mld->nan_device_vif;
+
+	if (IWL_FW_CHECK(mld, !vif,
+			 "NAN: schedule update completed without NAN vif\n"))
+		return;
+
+	if (IWL_FW_CHECK(mld, !ieee80211_vif_nan_started(vif),
+			 "NAN: schedule update completed without NAN started\n"))
+		return;
+
+	/*
+	 * Deferred schedule update should not fail in firmware since all
+	 * channels and links were added.
+	 */
+	IWL_FW_CHECK(mld, notif->status != IWL_NAN_SCHED_UPDATE_SUCCESS,
+		     "NAN: deferred schedule update failed\n");
+
+	if (WARN_ON(!vif->cfg.nan_sched.deferred))
+		return;
+
+	ieee80211_nan_sched_update_done(vif);
+}
+
+int iwl_mld_mac802111_nan_peer_sched_changed(struct ieee80211_hw *hw,
+					     struct ieee80211_sta *sta)
+{
+	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
+	struct ieee80211_nan_peer_sched *sched = sta->nan_sched;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(mld_sta->vif);
+	struct iwl_mld *mld = IWL_MAC80211_GET_MLD(hw);
+	struct iwl_mld_nan_link *nan_link;
+	struct iwl_nan_peer_cmd cmd = {
+		.nmi_sta_id = mld_sta->deflink.fw_id,
+		.sequence_id = sched->seq_id,
+		.committed_dw_info = cpu_to_le16(sched->committed_dw),
+		.max_channel_switch_time = cpu_to_le16(sched->max_chan_switch),
+		.initial_ulw_size = cpu_to_le32(sched->ulw_size),
+		.per_phy[0 ... NUM_PHY_CTX - 1] = {
+			/* unused by FW if availability_map == 0 */
+			.map_id = CFG80211_NAN_INVALID_MAP_ID,
+			.link_id = FW_CTXT_ID_INVALID,
+		},
+		/* .initial_ulw directly provided below by data[1]/len[1] */
+	};
+	struct iwl_host_cmd hcmd = {
+		.id = WIDE_ID(MAC_CONF_GROUP, NAN_PEER_CMD),
+		.data[0] = &cmd,
+		.len[0] = sizeof(cmd),
+		.data[1] = sched->init_ulw,
+		.len[1] = sched->ulw_size,
+		.dataflags[1] = IWL_HCMD_DFL_DUP,
+	};
+	struct ieee80211_sta *iter;
+
+	/* Update TLC in case peer channels were added/removed/updated */
+	for_each_station(iter, mld->hw) {
+		struct iwl_mld_sta *tmp = iwl_mld_sta_from_mac80211(iter);
+
+		if (tmp->sta_type == STATION_TYPE_NAN_PEER_NDI)
+			iwl_mld_config_tlc(mld, tmp->vif, iter);
+	}
+
+	for (int i = 0; i < ARRAY_SIZE(sched->maps); i++) {
+		if (sched->maps[i].map_id == CFG80211_NAN_INVALID_MAP_ID)
+			continue;
+
+		BUILD_BUG_ON(ARRAY_SIZE(sched->maps[i].slots) != 32);
+		for (int slot = 0;
+		     slot < ARRAY_SIZE(sched->maps[i].slots);
+		     slot++) {
+			struct ieee80211_chanctx_conf *ctx;
+			struct ieee80211_nan_channel *chan;
+			struct iwl_mld_phy *phy;
+
+			chan = sched->maps[i].slots[slot];
+			if (!chan)
+				continue;
+
+			ctx = chan->chanctx_conf;
+			if (!ctx)
+				continue;
+
+			phy = iwl_mld_phy_from_mac80211(ctx);
+
+			for_each_mld_nan_valid_link(mld_vif, nan_link) {
+				if (nan_link->chanctx == ctx) {
+					cmd.per_phy[phy->fw_id].link_id =
+						nan_link->fw_id;
+					break;
+				}
+			}
+
+			if (WARN_ON(cmd.per_phy[phy->fw_id].link_id ==
+				    FW_CTXT_ID_INVALID))
+				continue;
+
+			/*
+			 * each channel can only appear in one map,
+			 * upper layers enforce that
+			 */
+			if (WARN_ON(cmd.per_phy[phy->fw_id].map_id != CFG80211_NAN_INVALID_MAP_ID &&
+				    cmd.per_phy[phy->fw_id].map_id != sched->maps[i].map_id))
+				continue;
+
+			cmd.per_phy[phy->fw_id].map_id = sched->maps[i].map_id;
+			memcpy(cmd.per_phy[phy->fw_id].channel_entry,
+			       chan->channel_entry,
+			       sizeof(cmd.per_phy[phy->fw_id].channel_entry));
+			cmd.per_phy[phy->fw_id].availability_map |=
+				cpu_to_le32(BIT(slot));
+		}
+	}
+
+	return iwl_mld_send_cmd(mld, &hcmd);
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/nan.h b/drivers/net/wireless/intel/iwlwifi/mld/nan.h
index c04d77208971e..5411bca52cded 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/nan.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/nan.h
@@ -1,12 +1,31 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2025 Intel Corporation
+ * Copyright (C) 2025-2026 Intel Corporation
  */
 #ifndef __iwl_mld_nan_h__
 #define __iwl_mld_nan_h__
 #include <net/cfg80211.h>
 #include <linux/etherdevice.h>
 
+/**
+ * struct iwl_mld_nan_link - struct representing a NAN link
+ * @chanctx: the channel context
+ * @active: indicates the NAN link is currently active
+ * @fw_id: FW link ID
+ */
+struct iwl_mld_nan_link {
+	struct ieee80211_chanctx_conf *chanctx;
+	bool active;
+	u8 fw_id;
+};
+
+/* Cleanup function for struct iwl_mld_nan_link, will be called in restart */
+static inline void iwl_mld_cleanup_nan_link(struct iwl_mld_nan_link *nan_link)
+{
+	memset(nan_link, 0, sizeof(*nan_link));
+	nan_link->fw_id = FW_CTXT_ID_INVALID;
+}
+
 bool iwl_mld_nan_supported(struct iwl_mld *mld);
 int iwl_mld_start_nan(struct ieee80211_hw *hw,
 		      struct ieee80211_vif *vif,
@@ -19,13 +38,33 @@ int iwl_mld_stop_nan(struct ieee80211_hw *hw,
 		     struct ieee80211_vif *vif);
 void iwl_mld_handle_nan_cluster_notif(struct iwl_mld *mld,
 				      struct iwl_rx_packet *pkt);
+void iwl_mld_handle_nan_ulw_attr_notif(struct iwl_mld *mld,
+				       struct iwl_rx_packet *pkt);
 void iwl_mld_handle_nan_dw_end_notif(struct iwl_mld *mld,
 				     struct iwl_rx_packet *pkt);
+void iwl_mld_handle_nan_sched_update_completed_notif(struct iwl_mld *mld,
+						     struct iwl_rx_packet *pkt);
 bool iwl_mld_cancel_nan_cluster_notif(struct iwl_mld *mld,
 				      struct iwl_rx_packet *pkt,
 				      u32 obj_id);
+bool iwl_mld_cancel_nan_ulw_attr_notif(struct iwl_mld *mld,
+				       struct iwl_rx_packet *pkt,
+				       u32 obj_id);
 bool iwl_mld_cancel_nan_dw_end_notif(struct iwl_mld *mld,
 				     struct iwl_rx_packet *pkt,
 				     u32 obj_id);
+bool iwl_mld_cancel_nan_sched_update_completed_notif(struct iwl_mld *mld,
+						     struct iwl_rx_packet *pkt,
+						     u32 obj_id);
+void iwl_mld_nan_vif_cfg_changed(struct iwl_mld *mld,
+				 struct ieee80211_vif *vif,
+				 u64 changes);
+
+int iwl_mld_mac802111_nan_peer_sched_changed(struct ieee80211_hw *hw,
+					     struct ieee80211_sta *sta);
+
+int iwl_mld_nan_get_mgmt_queue(struct iwl_mld *mld, struct ieee80211_vif *vif);
+
+bool iwl_mld_nan_use_nan_stations(struct iwl_mld *mld);
 
 #endif /* __iwl_mld_nan_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/notif.c b/drivers/net/wireless/intel/iwlwifi/mld/notif.c
index 9c88a8579a759..7574689e40883 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/notif.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/notif.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #include "mld.h"
@@ -307,11 +307,8 @@ CMD_VERSIONS(tx_resp_notif,
 	     CMD_VER_ENTRY(8, iwl_tx_resp)
 	     CMD_VER_ENTRY(9, iwl_tx_resp))
 CMD_VERSIONS(compressed_ba_notif,
-	     CMD_VER_ENTRY(5, iwl_compressed_ba_notif)
-	     CMD_VER_ENTRY(6, iwl_compressed_ba_notif)
 	     CMD_VER_ENTRY(7, iwl_compressed_ba_notif))
 CMD_VERSIONS(tlc_notif,
-	     CMD_VER_ENTRY(3, iwl_tlc_update_notif)
 	     CMD_VER_ENTRY(4, iwl_tlc_update_notif))
 CMD_VERSIONS(mu_mimo_grp_notif,
 	     CMD_VER_ENTRY(1, iwl_mu_group_mgmt_notif))
@@ -330,7 +327,8 @@ CMD_VERSIONS(probe_resp_data_notif,
 CMD_VERSIONS(datapath_monitor_notif,
 	     CMD_VER_ENTRY(1, iwl_datapath_monitor_notif))
 CMD_VERSIONS(stats_oper_notif,
-	     CMD_VER_ENTRY(3, iwl_system_statistics_notif_oper))
+	     CMD_VER_ENTRY(3, iwl_system_statistics_notif_oper_v3)
+	     CMD_VER_ENTRY(4, iwl_system_statistics_notif_oper))
 CMD_VERSIONS(stats_oper_part1_notif,
 	     CMD_VER_ENTRY(4, iwl_system_statistics_part1_notif_oper))
 CMD_VERSIONS(bt_coex_notif,
@@ -341,8 +339,6 @@ CMD_VERSIONS(emlsr_mode_notif,
 	     CMD_VER_ENTRY(2, iwl_esr_mode_notif))
 CMD_VERSIONS(emlsr_trans_fail_notif,
 	     CMD_VER_ENTRY(1, iwl_esr_trans_fail_notif))
-CMD_VERSIONS(uapsd_misbehaving_ap_notif,
-	     CMD_VER_ENTRY(1, iwl_uapsd_misbehaving_ap_notif))
 CMD_VERSIONS(time_msmt_notif,
 	     CMD_VER_ENTRY(1, iwl_time_msmt_notify))
 CMD_VERSIONS(time_sync_confirm_notif,
@@ -350,7 +346,10 @@ CMD_VERSIONS(time_sync_confirm_notif,
 CMD_VERSIONS(ftm_resp_notif, CMD_VER_ENTRY(10, iwl_tof_range_rsp_ntfy))
 CMD_VERSIONS(beacon_filter_notif, CMD_VER_ENTRY(2, iwl_beacon_filter_notif))
 CMD_VERSIONS(nan_cluster_notif, CMD_VER_ENTRY(1, iwl_nan_cluster_notif))
+CMD_VERSIONS(nan_ulw_attr_notif, CMD_VER_ENTRY(1, iwl_nan_ulw_attr_notif))
 CMD_VERSIONS(nan_dw_end_notif, CMD_VER_ENTRY(1, iwl_nan_dw_end_notif))
+CMD_VERSIONS(nan_sched_update_completed_notif,
+	     CMD_VER_ENTRY(1, iwl_nan_sched_update_completed_notif))
 
 DEFINE_SIMPLE_CANCELLATION(session_prot, iwl_session_prot_notif, mac_link_id)
 DEFINE_SIMPLE_CANCELLATION(tlc, iwl_tlc_update_notif, sta_id)
@@ -365,8 +364,6 @@ DEFINE_SIMPLE_CANCELLATION(scan_complete, iwl_umac_scan_complete, uid)
 DEFINE_SIMPLE_CANCELLATION(scan_start, iwl_umac_scan_start, uid)
 DEFINE_SIMPLE_CANCELLATION(probe_resp_data, iwl_probe_resp_data_notif,
 			   mac_id)
-DEFINE_SIMPLE_CANCELLATION(uapsd_misbehaving_ap, iwl_uapsd_misbehaving_ap_notif,
-			   mac_id)
 DEFINE_SIMPLE_CANCELLATION(ftm_resp, iwl_tof_range_rsp_ntfy, request_id)
 DEFINE_SIMPLE_CANCELLATION(beacon_filter, iwl_beacon_filter_notif, link_id)
 
@@ -457,8 +454,6 @@ const struct iwl_rx_handler iwl_mld_rx_handlers[] = {
 			     emlsr_mode_notif, RX_HANDLER_ASYNC)
 	RX_HANDLER_NO_OBJECT(MAC_CONF_GROUP, EMLSR_TRANS_FAIL_NOTIF,
 			     emlsr_trans_fail_notif, RX_HANDLER_ASYNC)
-	RX_HANDLER_OF_VIF(LEGACY_GROUP, PSM_UAPSD_AP_MISBEHAVING_NOTIFICATION,
-			  uapsd_misbehaving_ap_notif)
 	RX_HANDLER_NO_OBJECT(LEGACY_GROUP,
 			     WNM_80211V_TIMING_MEASUREMENT_NOTIFICATION,
 			     time_msmt_notif, RX_HANDLER_SYNC)
@@ -471,8 +466,12 @@ const struct iwl_rx_handler iwl_mld_rx_handlers[] = {
 			      ftm_resp_notif)
 	RX_HANDLER_OF_NAN(MAC_CONF_GROUP, NAN_JOINED_CLUSTER_NOTIF,
 			  nan_cluster_notif)
+	RX_HANDLER_OF_NAN(MAC_CONF_GROUP, NAN_ULW_ATTR_NOTIF,
+			  nan_ulw_attr_notif)
 	RX_HANDLER_OF_NAN(MAC_CONF_GROUP, NAN_DW_END_NOTIF,
 			  nan_dw_end_notif)
+	RX_HANDLER_OF_NAN(MAC_CONF_GROUP, NAN_SCHED_UPDATE_COMPLETED_NOTIF,
+			  nan_sched_update_completed_notif)
 };
 EXPORT_SYMBOL_IF_IWLWIFI_KUNIT(iwl_mld_rx_handlers);
 
@@ -607,6 +606,11 @@ void iwl_mld_rx(struct iwl_op_mode *op_mode, struct napi_struct *napi,
 	else if (unlikely(cmd_id == WIDE_ID(DATA_PATH_GROUP,
 					    RX_QUEUES_NOTIFICATION)))
 		iwl_mld_handle_rx_queues_sync_notif(mld, napi, pkt, 0);
+#ifdef CONFIG_PM_SLEEP
+	else if (unlikely(cmd_id == WIDE_ID(DATA_PATH_GROUP,
+					    RSC_NOTIF)))
+		iwl_mld_handle_rsc_notif(mld, pkt, 0);
+#endif
 	else if (cmd_id == WIDE_ID(DATA_PATH_GROUP, PHY_AIR_SNIFFER_NOTIF))
 		iwl_mld_handle_phy_air_sniffer_notif(mld, napi, pkt);
 	else
@@ -630,6 +634,11 @@ void iwl_mld_rx_rss(struct iwl_op_mode *op_mode, struct napi_struct *napi,
 		iwl_mld_handle_rx_queues_sync_notif(mld, napi, pkt, queue);
 	else if (unlikely(cmd_id == WIDE_ID(LEGACY_GROUP, FRAME_RELEASE)))
 		iwl_mld_handle_frame_release_notif(mld, napi, pkt, queue);
+#ifdef CONFIG_PM_SLEEP
+	else if (unlikely(cmd_id == WIDE_ID(DATA_PATH_GROUP,
+					    RSC_NOTIF)))
+		iwl_mld_handle_rsc_notif(mld, pkt, queue);
+#endif
 }
 
 void iwl_mld_delete_handlers(struct iwl_mld *mld, const u16 *cmds, int n_cmds)
@@ -685,10 +694,6 @@ void iwl_mld_cancel_async_notifications(struct iwl_mld *mld)
 {
 	struct iwl_async_handler_entry *entry, *tmp;
 
-	lockdep_assert_wiphy(mld->wiphy);
-
-	wiphy_work_cancel(mld->wiphy, &mld->async_handlers_wk);
-
 	spin_lock_bh(&mld->async_handlers_lock);
 	list_for_each_entry_safe(entry, tmp, &mld->async_handlers_list, list) {
 		iwl_mld_log_async_handler_op(mld, "Purged", &entry->rxb);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/phy.c b/drivers/net/wireless/intel/iwlwifi/mld/phy.c
index 1d93fb9e4dbf3..59bf088ead840 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/phy.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/phy.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include <net/mac80211.h>
 
@@ -99,9 +99,9 @@ iwl_mld_nl80211_width_to_fw(enum nl80211_chan_width width)
 /* Maps the driver specific control channel position (relative to the center
  * freq) definitions to the fw values
  */
-u8 iwl_mld_get_fw_ctrl_pos(const struct cfg80211_chan_def *chandef)
+static u8 _iwl_mld_get_fw_ctrl_pos(u32 control, u32 cf1)
 {
-	int offs = chandef->chan->center_freq - chandef->center_freq1;
+	int offs = control - cf1;
 	int abs_offs = abs(offs);
 	u8 ret;
 
@@ -127,6 +127,12 @@ u8 iwl_mld_get_fw_ctrl_pos(const struct cfg80211_chan_def *chandef)
 	return ret;
 }
 
+u8 iwl_mld_get_fw_ctrl_pos(const struct cfg80211_chan_def *chandef)
+{
+	return _iwl_mld_get_fw_ctrl_pos(chandef->chan->center_freq,
+					chandef->center_freq1);
+}
+
 int iwl_mld_phy_fw_action(struct iwl_mld *mld,
 			  struct ieee80211_chanctx_conf *ctx, u32 action)
 {
@@ -150,6 +156,18 @@ int iwl_mld_phy_fw_action(struct iwl_mld *mld,
 		cmd.sbb_ctrl_channel_loc = iwl_mld_get_fw_ctrl_pos(&ctx->ap);
 	}
 
+	/*
+	 * Set NPCA channel if NPCA is used; if not used, just set it to an
+	 * arbitrary channel on the other side to help firmware.
+	 */
+	if (chandef->npca_chan)
+		cmd.secondary_ctrl_chnl_loc =
+			_iwl_mld_get_fw_ctrl_pos(chandef->npca_chan->center_freq,
+						 chandef->center_freq1);
+	else
+		cmd.secondary_ctrl_chnl_loc =
+			cmd.ci.ctrl_pos ^ IWL_PHY_CTRL_POS_ABOVE;
+
 	ret = iwl_mld_send_cmd_pdu(mld, PHY_CONTEXT_CMD, &cmd);
 	if (ret)
 		IWL_ERR(mld, "Failed to send PHY_CONTEXT_CMD ret = %d\n", ret);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/power.c b/drivers/net/wireless/intel/iwlwifi/mld/power.c
index 49b0d9f8f865d..da065a446f815 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/power.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/power.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include <net/mac80211.h>
 
@@ -11,36 +11,11 @@
 #include "link.h"
 #include "constants.h"
 
-static void iwl_mld_vif_ps_iterator(void *data, u8 *mac,
-				    struct ieee80211_vif *vif)
-{
-	bool *ps_enable = (bool *)data;
-	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
-
-	if (vif->type != NL80211_IFTYPE_STATION)
-		return;
-
-	*ps_enable &= !mld_vif->ps_disabled;
-}
-
 int iwl_mld_update_device_power(struct iwl_mld *mld, bool d3)
 {
 	struct iwl_device_power_cmd cmd = {};
-	bool enable_ps = false;
-
-	if (iwlmld_mod_params.power_scheme != IWL_POWER_SCHEME_CAM) {
-		enable_ps = true;
-
-		/* Disable power save if any STA interface has
-		 * power save turned off
-		 */
-		ieee80211_iterate_active_interfaces_mtx(mld->hw,
-							IEEE80211_IFACE_ITER_NORMAL,
-							iwl_mld_vif_ps_iterator,
-							&enable_ps);
-	}
 
-	if (enable_ps)
+	if (iwlmld_mod_params.power_scheme != IWL_POWER_SCHEME_CAM)
 		cmd.flags |=
 			cpu_to_le16(DEVICE_POWER_FLAGS_POWER_SAVE_ENA_MSK);
 
@@ -113,10 +88,10 @@ static bool iwl_mld_power_is_radar(struct iwl_mld *mld,
 	return chanctx_conf->def.chan->flags & IEEE80211_CHAN_RADAR;
 }
 
-static void iwl_mld_power_configure_uapsd(struct iwl_mld *mld,
-					  struct iwl_mld_link *link,
-					  struct iwl_mac_power_cmd *cmd,
-					  bool ps_poll)
+static void iwl_mld_power_configure_uapsd_v2(struct iwl_mld *mld,
+					     struct iwl_mld_link *link,
+					     struct iwl_mac_power_cmd_v2 *cmd,
+					     bool ps_poll)
 {
 	bool tid_found = false;
 
@@ -175,10 +150,54 @@ static void iwl_mld_power_configure_uapsd(struct iwl_mld *mld,
 	cmd->uapsd_max_sp = mld->hw->uapsd_max_sp_len;
 }
 
+static void iwl_mld_power_configure_uapsd(struct iwl_mld *mld,
+					  struct iwl_mld_link *link,
+					  struct iwl_mac_power_cmd *cmd,
+					  bool ps_poll)
+{
+	bool tid_found = false;
+
+	/* set advanced pm flag with no uapsd ACs to enable ps-poll */
+	if (ps_poll) {
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_ADVANCE_PM_ENA_MSK);
+		return;
+	}
+
+	for (enum ieee80211_ac_numbers ac = IEEE80211_AC_VO;
+	     ac <= IEEE80211_AC_BK;
+	     ac++) {
+		if (!link->queue_params[ac].uapsd)
+			continue;
+
+		cmd->flags |=
+			cpu_to_le16(POWER_FLAGS_ADVANCE_PM_ENA_MSK);
+		cmd->uapsd_ac_flags |= BIT(ac);
+
+		/* QNDP TID - the highest TID with no admission control */
+		if (!tid_found && !link->queue_params[ac].acm) {
+			tid_found = true;
+			switch (ac) {
+			case IEEE80211_AC_VO:
+				cmd->qndp_tid = 6;
+				break;
+			case IEEE80211_AC_VI:
+				cmd->qndp_tid = 5;
+				break;
+			case IEEE80211_AC_BE:
+				cmd->qndp_tid = 0;
+				break;
+			case IEEE80211_AC_BK:
+				cmd->qndp_tid = 1;
+				break;
+			}
+		}
+	}
+}
+
 static void
 iwl_mld_power_config_skip_dtim(struct iwl_mld *mld,
 			       const struct ieee80211_bss_conf *link_conf,
-			       struct iwl_mac_power_cmd *cmd)
+			       u8 *skip_dtim_periods, __le16 *flags)
 {
 	unsigned int dtimper_tu;
 	unsigned int dtimper;
@@ -196,15 +215,15 @@ iwl_mld_power_config_skip_dtim(struct iwl_mld *mld,
 	/* configure skip over dtim up to 900 TU DTIM interval */
 	skip = max_t(int, 1, 900 / dtimper_tu);
 
-	cmd->skip_dtim_periods = skip;
-	cmd->flags |= cpu_to_le16(POWER_FLAGS_SKIP_OVER_DTIM_MSK);
+	*skip_dtim_periods = skip;
+	*flags |= cpu_to_le16(POWER_FLAGS_SKIP_OVER_DTIM_MSK);
 }
 
 #define POWER_KEEP_ALIVE_PERIOD_SEC    25
-static void iwl_mld_power_build_cmd(struct iwl_mld *mld,
-				    struct ieee80211_vif *vif,
-				    struct iwl_mac_power_cmd *cmd,
-				    bool d3)
+static void iwl_mld_power_build_cmd_v2(struct iwl_mld *mld,
+				       struct ieee80211_vif *vif,
+				       struct iwl_mac_power_cmd_v2 *cmd,
+				       bool d3)
 {
 	int dtimper, bi;
 	int keep_alive;
@@ -252,9 +271,7 @@ static void iwl_mld_power_build_cmd(struct iwl_mld *mld,
 		return;
 
 	cmd->flags |= cpu_to_le16(POWER_FLAGS_POWER_MANAGEMENT_ENA_MSK);
-
-	if (iwl_fw_lookup_cmd_ver(mld->fw, MAC_PM_POWER_TABLE, 0) >= 2)
-		cmd->flags |= cpu_to_le16(POWER_FLAGS_ENABLE_SMPS_MSK);
+	cmd->flags |= cpu_to_le16(POWER_FLAGS_ENABLE_SMPS_MSK);
 
 	/* firmware supports LPRX for beacons at rate 1 Mbps or 6 Mbps only */
 	if (link_conf->beacon_rate &&
@@ -265,7 +282,9 @@ static void iwl_mld_power_build_cmd(struct iwl_mld *mld,
 	}
 
 	if (d3) {
-		iwl_mld_power_config_skip_dtim(mld, link_conf, cmd);
+		iwl_mld_power_config_skip_dtim(mld, link_conf,
+					       &cmd->skip_dtim_periods,
+					       &cmd->flags);
 		cmd->rx_data_timeout =
 			cpu_to_le32(IWL_MLD_WOWLAN_PS_RX_DATA_TIMEOUT);
 		cmd->tx_data_timeout =
@@ -286,6 +305,95 @@ static void iwl_mld_power_build_cmd(struct iwl_mld *mld,
 	 * mac80211 will allow uAPSD. Always call iwl_mld_power_configure_uapsd
 	 * which will look at what mac80211 is saying.
 	 */
+#ifdef CONFIG_IWLWIFI_DEBUGFS
+	ps_poll = mld_vif->use_ps_poll;
+#endif
+	iwl_mld_power_configure_uapsd_v2(mld, link, cmd, ps_poll);
+}
+
+static void iwl_mld_power_build_cmd(struct iwl_mld *mld,
+				    struct ieee80211_vif *vif,
+				    struct iwl_mac_power_cmd *cmd,
+				    bool d3)
+{
+	int dtimper, bi;
+	int keep_alive;
+	struct iwl_mld_vif *mld_vif = iwl_mld_vif_from_mac80211(vif);
+	struct ieee80211_bss_conf *link_conf = &vif->bss_conf;
+	struct iwl_mld_link *link = &mld_vif->deflink;
+	bool ps_poll = false;
+	__le32 fw_id = cpu_to_le32(mld_vif->fw_id);
+
+	if (ieee80211_vif_is_mld(vif)) {
+		int link_id;
+
+		if (WARN_ON(!vif->active_links))
+			return;
+
+		/* The firmware consumes one single configuration for the vif
+		 * and can't differentiate between links, just pick the lowest
+		 * link_id's configuration and use that.
+		 */
+		link_id = __ffs(vif->active_links);
+		link_conf = link_conf_dereference_check(vif, link_id);
+		link = iwl_mld_link_dereference_check(mld_vif, link_id);
+
+		if (WARN_ON(!link_conf || !link))
+			return;
+	}
+	dtimper = link_conf->dtim_period;
+	bi = link_conf->beacon_int;
+
+	/* Regardless of power management state the driver must set
+	 * keep alive period. FW will use it for sending keep alive NDPs
+	 * immediately after association. Check that keep alive period
+	 * is at least 3 * DTIM
+	 */
+	keep_alive = DIV_ROUND_UP(ieee80211_tu_to_usec(3 * dtimper * bi),
+				  USEC_PER_SEC);
+	keep_alive = max(keep_alive, POWER_KEEP_ALIVE_PERIOD_SEC);
+
+	cmd->id_and_color = fw_id;
+	cmd->keep_alive_seconds = cpu_to_le16(keep_alive);
+
+	if (iwlmld_mod_params.power_scheme != IWL_POWER_SCHEME_CAM)
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_POWER_SAVE_ENA_MSK);
+
+	if (vif->cfg.ps && iwl_mld_tdls_sta_count(mld) == 0) {
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_POWER_MANAGEMENT_ENA_MSK);
+		cmd->flags |= cpu_to_le16(POWER_FLAGS_ENABLE_SMPS_MSK);
+
+		/* firmware supports LPRX for beacons at rate 1 Mbps or
+		 * 6 Mbps only
+		 */
+		if (link_conf->beacon_rate &&
+		    (link_conf->beacon_rate->bitrate == 10 ||
+		     link_conf->beacon_rate->bitrate == 60)) {
+			cmd->flags |= cpu_to_le16(POWER_FLAGS_LPRX_ENA_MSK);
+			cmd->lprx_rssi_threshold = POWER_LPRX_RSSI_THRESHOLD;
+		}
+	}
+
+	if (d3) {
+		iwl_mld_power_config_skip_dtim(mld, link_conf,
+					       &cmd->skip_dtim_periods,
+					       &cmd->flags);
+		cmd->rx_data_timeout =
+			cpu_to_le32(IWL_MLD_WOWLAN_PS_RX_DATA_TIMEOUT);
+		cmd->tx_data_timeout =
+			cpu_to_le32(IWL_MLD_WOWLAN_PS_TX_DATA_TIMEOUT);
+	} else if (iwl_mld_vif_low_latency(mld_vif) && vif->p2p) {
+		cmd->tx_data_timeout =
+			cpu_to_le32(IWL_MLD_SHORT_PS_TX_DATA_TIMEOUT);
+		cmd->rx_data_timeout =
+			cpu_to_le32(IWL_MLD_SHORT_PS_RX_DATA_TIMEOUT);
+	} else {
+		cmd->rx_data_timeout =
+			cpu_to_le32(IWL_MLD_DEFAULT_PS_RX_DATA_TIMEOUT);
+		cmd->tx_data_timeout =
+			cpu_to_le32(IWL_MLD_DEFAULT_PS_TX_DATA_TIMEOUT);
+	}
+
 #ifdef CONFIG_IWLWIFI_DEBUGFS
 	ps_poll = mld_vif->use_ps_poll;
 #endif
@@ -295,11 +403,23 @@ static void iwl_mld_power_build_cmd(struct iwl_mld *mld,
 int iwl_mld_update_mac_power(struct iwl_mld *mld, struct ieee80211_vif *vif,
 			     bool d3)
 {
-	struct iwl_mac_power_cmd cmd = {};
+	int cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, MAC_PM_POWER_TABLE, 0);
+
+	if (cmd_ver >= 3) {
+		struct iwl_mac_power_cmd cmd = {};
 
-	iwl_mld_power_build_cmd(mld, vif, &cmd, d3);
+		iwl_mld_power_build_cmd(mld, vif, &cmd, d3);
+		return iwl_mld_send_cmd_with_flags_pdu(mld,
+						       MAC_PM_POWER_TABLE, 0,
+						       &cmd, sizeof(cmd));
+	} else {
+		struct iwl_mac_power_cmd_v2 cmd = {};
 
-	return iwl_mld_send_cmd_pdu(mld, MAC_PM_POWER_TABLE, &cmd);
+		iwl_mld_power_build_cmd_v2(mld, vif, &cmd, d3);
+		return iwl_mld_send_cmd_with_flags_pdu(mld,
+						       MAC_PM_POWER_TABLE, 0,
+						       &cmd, sizeof(cmd));
+	}
 }
 
 static void
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c
index c65f4b56a3270..20ae338e5696d 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/ptp.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/ptp.c
@@ -321,10 +321,10 @@ void iwl_mld_ptp_remove(struct iwl_mld *mld)
 			       mld->ptp_data.ptp_clock_info.name,
 			       ptp_clock_index(mld->ptp_data.ptp_clock));
 
+		cancel_delayed_work_sync(&mld->ptp_data.dwork);
 		ptp_clock_unregister(mld->ptp_data.ptp_clock);
 		mld->ptp_data.ptp_clock = NULL;
 		mld->ptp_data.last_gp2 = 0;
 		mld->ptp_data.wrap_counter = 0;
-		cancel_delayed_work_sync(&mld->ptp_data.dwork);
 	}
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/rx.c b/drivers/net/wireless/intel/iwlwifi/mld/rx.c
index a2e586c6ea673..269439d789f46 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/rx.c
@@ -39,8 +39,7 @@ iwl_mld_fill_phy_data_from_mpdu(struct iwl_mld *mld,
 	}
 
 	phy_data->phy_info = le16_to_cpu(desc->phy_info);
-	phy_data->rate_n_flags = iwl_v3_rate_from_v2_v3(desc->v3.rate_n_flags,
-							mld->fw_rates_ver_3);
+	phy_data->rate_n_flags = le32_to_cpu(desc->v3.rate_n_flags);
 	phy_data->gp2_on_air_rise = le32_to_cpu(desc->v3.gp2_on_air_rise);
 	phy_data->energy_a = desc->v3.energy_a;
 	phy_data->energy_b = desc->v3.energy_b;
@@ -158,7 +157,7 @@ static bool iwl_mld_used_average_energy(struct iwl_mld *mld, int link_id,
 	guard(rcu)();
 
 	link_conf = rcu_dereference(mld->fw_id_to_bss_conf[link_id]);
-	if (!link_conf)
+	if (IS_ERR_OR_NULL(link_conf))
 		return false;
 
 	mld_link = iwl_mld_link_from_mac80211(link_conf);
@@ -1206,6 +1205,13 @@ static void iwl_mld_decode_eht_non_tb(struct iwl_mld_rx_phy_data *phy_data,
 	iwl_mld_eht_set_ru_alloc(rx_status,
 				 le32_get_bits(phy_data->ntfy->sigs.eht.b2,
 					       OFDM_RX_FRAME_EHT_STA_RU));
+
+	if (phy_data->with_data)
+		eht->user_info[0] |=
+			cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN) |
+			LE32_DEC_ENC(phy_data->ntfy->sigs.eht.user_id,
+				     OFDM_RX_FRAME_EHT_USER_FIELD_ID,
+				     IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID);
 }
 
 static void iwl_mld_decode_eht_phy_data(struct iwl_mld_rx_phy_data *phy_data,
@@ -1314,14 +1320,6 @@ static void iwl_mld_rx_eht(struct iwl_mld *mld, struct sk_buff *skb,
 	if (likely(!phy_data->ntfy))
 		return;
 
-	if (phy_data->with_data) {
-		eht->user_info[0] |=
-			cpu_to_le32(IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID_KNOWN) |
-			LE32_DEC_ENC(phy_data->ntfy->sigs.eht.user_id,
-				     OFDM_RX_FRAME_EHT_USER_FIELD_ID,
-				     IEEE80211_RADIOTAP_EHT_USER_INFO_STA_ID);
-	}
-
 	iwl_mld_decode_eht_usig(phy_data, skb);
 	iwl_mld_decode_eht_phy_data(phy_data, rx_status, eht);
 }
@@ -2262,6 +2260,30 @@ void iwl_mld_handle_rx_queues_sync_notif(struct iwl_mld *mld,
 		wake_up(&mld->rxq_sync.waitq);
 }
 
+#ifdef CONFIG_PM_SLEEP
+void iwl_mld_handle_rsc_notif(struct iwl_mld *mld,
+			      struct iwl_rx_packet *pkt, int queue)
+{
+	const struct iwl_wowlan_all_rsc_tsc_v5 *notif = (void *)pkt->data;
+	u32 len = iwl_rx_packet_payload_len(pkt);
+	struct ieee80211_vif *bss_vif;
+
+	if (IWL_FW_CHECK(mld, len != sizeof(*notif),
+			 "invalid notification size %u (%zu)\n",
+			 len, sizeof(*notif)))
+		return;
+
+	/* for the bss lookup and updating the keys' pn */
+	guard(rcu)();
+
+	bss_vif = iwl_mld_get_bss_vif(mld);
+	if (WARN_ON(!bss_vif))
+		return;
+
+	iwl_mld_process_rsc_notification(mld, bss_vif, notif, queue);
+}
+#endif /* CONFIG_PM_SLEEP */
+
 static void iwl_mld_no_data_rx(struct iwl_mld *mld,
 			       struct napi_struct *napi,
 			       struct iwl_rx_phy_air_sniffer_ntfy *ntfy)
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/rx.h b/drivers/net/wireless/intel/iwlwifi/mld/rx.h
index 09dddbd40f553..573b89c3c9c61 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/rx.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/rx.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #ifndef __iwl_mld_rx_h__
 #define __iwl_mld_rx_h__
@@ -61,6 +61,11 @@ void iwl_mld_handle_rx_queues_sync_notif(struct iwl_mld *mld,
 					 struct napi_struct *napi,
 					 struct iwl_rx_packet *pkt, int queue);
 
+#ifdef CONFIG_PM_SLEEP
+void iwl_mld_handle_rsc_notif(struct iwl_mld *mld,
+			      struct iwl_rx_packet *pkt, int queue);
+#endif
+
 void iwl_mld_pass_packet_to_mac80211(struct iwl_mld *mld,
 				     struct napi_struct *napi,
 				     struct sk_buff *skb, int queue,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/sta.c b/drivers/net/wireless/intel/iwlwifi/mld/sta.c
index 4c97d12ce2d01..77eeeed66116b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/sta.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/sta.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #include <linux/ieee80211.h>
@@ -13,6 +13,7 @@
 #include "key.h"
 #include "agg.h"
 #include "tlc.h"
+#include "nan.h"
 #include "fw/api/sta.h"
 #include "fw/api/mac.h"
 #include "fw/api/rx.h"
@@ -43,13 +44,13 @@ int iwl_mld_fw_sta_id_from_link_sta(struct iwl_mld *mld,
 
 static void
 iwl_mld_fill_ampdu_size_and_dens(struct ieee80211_link_sta *link_sta,
-				 struct ieee80211_bss_conf *link,
+				 bool is_6ghz,
 				 __le32 *tx_ampdu_max_size,
 				 __le32 *tx_ampdu_spacing)
 {
 	u32 agg_size = 0, mpdu_dens = 0;
 
-	if (WARN_ON(!link_sta || !link))
+	if (WARN_ON(!link_sta))
 		return;
 
 	/* Note that we always use only legacy & highest supported PPDUs, so
@@ -63,7 +64,7 @@ iwl_mld_fill_ampdu_size_and_dens(struct ieee80211_link_sta *link_sta,
 		mpdu_dens = link_sta->ht_cap.ampdu_density;
 	}
 
-	if (link->chanreq.oper.chan->band == NL80211_BAND_6GHZ) {
+	if (is_6ghz) {
 		/* overwrite HT values on 6 GHz */
 		mpdu_dens =
 			le16_get_bits(link_sta->he_6ghz_capa.capa,
@@ -427,8 +428,14 @@ static int iwl_mld_send_sta_cmd(struct iwl_mld *mld,
 
 		len = sizeof(struct iwl_sta_cfg_cmd_v2);
 		cmd_v2->link_id = cpu_to_le32(__ffs(le32_to_cpu(cmd->link_mask)));
+	} else if (cmd->station_type ==
+		   cpu_to_le32(STATION_TYPE_NAN_MCAST_DATA)) {
+		if (WARN_ON(!hweight32(le32_to_cpu(cmd->link_mask))))
+			return -EINVAL;
 	} else if (WARN_ON(cmd->station_type != cpu_to_le32(STATION_TYPE_NAN_PEER_NMI) &&
 			   cmd->station_type != cpu_to_le32(STATION_TYPE_NAN_PEER_NDI) &&
+			   cmd->station_type != cpu_to_le32(STATION_TYPE_NAN_BCAST) &&
+			   cmd->station_type != cpu_to_le32(STATION_TYPE_NAN_MGMT) &&
 			   hweight32(le32_to_cpu(cmd->link_mask)) != 1)) {
 		return -EINVAL;
 	}
@@ -439,29 +446,61 @@ static int iwl_mld_send_sta_cmd(struct iwl_mld *mld,
 	return ret;
 }
 
-static int
-iwl_mld_add_modify_sta_cmd(struct iwl_mld *mld,
-			   struct ieee80211_link_sta *link_sta)
+static u32 iwl_mld_get_nan_link_mask(struct iwl_mld *mld)
+{
+	struct iwl_mld_vif *nan_dev =
+		iwl_mld_vif_from_mac80211(mld->nan_device_vif);
+	struct iwl_mld_nan_link *nan_link;
+	u32 link_mask = 0;
+
+	for_each_mld_nan_valid_link(nan_dev, nan_link)
+		link_mask |= BIT(nan_link->fw_id);
+
+	return link_mask;
+}
+
+int iwl_mld_add_modify_sta_cmd(struct iwl_mld *mld,
+			      struct ieee80211_link_sta *link_sta)
 {
 	struct ieee80211_sta *sta = link_sta->sta;
 	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
-	struct ieee80211_bss_conf *link;
-	struct iwl_mld_link *mld_link;
 	struct iwl_sta_cfg_cmd cmd = {};
 	int fw_id = iwl_mld_fw_sta_id_from_link_sta(mld, link_sta);
+	bool is_6ghz, uora_exists;
+	u32 link_mask;
 
 	lockdep_assert_wiphy(mld->wiphy);
 
-	link = link_conf_dereference_protected(mld_sta->vif,
-					       link_sta->link_id);
+	if (WARN_ON(fw_id < 0))
+		return -EINVAL;
 
-	mld_link = iwl_mld_link_from_mac80211(link);
+	if (mld_sta->sta_type == STATION_TYPE_NAN_PEER_NMI ||
+	    mld_sta->sta_type == STATION_TYPE_NAN_PEER_NDI) {
+		if (WARN_ON(!mld->nan_device_vif))
+			return -EINVAL;
 
-	if (WARN_ON(!link || !mld_link) || fw_id < 0)
-		return -EINVAL;
+		is_6ghz = false;
+		uora_exists = false;
+
+		link_mask = iwl_mld_get_nan_link_mask(mld);
+	} else {
+		struct ieee80211_bss_conf *link;
+		struct iwl_mld_link *mld_link;
+
+		link = link_conf_dereference_protected(mld_sta->vif,
+						       link_sta->link_id);
+		mld_link = iwl_mld_link_from_mac80211(link);
+
+		if (WARN_ON(!link || !mld_link))
+			return -EINVAL;
+
+		link_mask = BIT(mld_link->fw_id);
+		is_6ghz = link->chanreq.oper.chan->band == NL80211_BAND_6GHZ;
+		uora_exists = link->uora_exists;
+	}
 
 	cmd.sta_id = cpu_to_le32(fw_id);
-	cmd.link_mask = cpu_to_le32(BIT(mld_link->fw_id));
+	cmd.link_mask = cpu_to_le32(link_mask);
 	cmd.station_type = cpu_to_le32(mld_sta->sta_type);
 
 	memcpy(&cmd.peer_mld_address, sta->addr, ETH_ALEN);
@@ -499,7 +538,7 @@ iwl_mld_add_modify_sta_cmd(struct iwl_mld *mld,
 		break;
 	}
 
-	iwl_mld_fill_ampdu_size_and_dens(link_sta, link,
+	iwl_mld_fill_ampdu_size_and_dens(link_sta, is_6ghz,
 					 &cmd.tx_ampdu_max_size,
 					 &cmd.tx_ampdu_spacing);
 
@@ -511,7 +550,7 @@ iwl_mld_add_modify_sta_cmd(struct iwl_mld *mld,
 
 	if (link_sta->he_cap.has_he) {
 		cmd.trig_rnd_alloc =
-			cpu_to_le32(link->uora_exists ? 1 : 0);
+			cpu_to_le32(uora_exists ? 1 : 0);
 
 		/* PPE Thresholds */
 		iwl_mld_fill_pkt_ext(mld, link_sta, &cmd.pkt_ext);
@@ -525,10 +564,29 @@ iwl_mld_add_modify_sta_cmd(struct iwl_mld *mld,
 			cmd.ack_enabled = cpu_to_le32(1);
 	}
 
+	if (mld_sta->sta_type == STATION_TYPE_NAN_PEER_NDI) {
+		struct ieee80211_sta *nmi_sta =
+			wiphy_dereference(mld->wiphy, sta->nmi);
+		int nmi_fw_id;
+
+		/* copy the local NDI address */
+		ether_addr_copy(cmd.ndi_local_addr, mld_sta->vif->addr);
+
+		if (WARN_ON(!nmi_sta))
+			return -EINVAL;
+
+		nmi_fw_id = iwl_mld_fw_sta_id_from_link_sta(mld,
+					&nmi_sta->deflink);
+		if (nmi_fw_id < 0)
+			return -EINVAL;
+
+		cmd.nmi_sta_id = (u8) nmi_fw_id;
+	}
+
 	return iwl_mld_send_sta_cmd(mld, &cmd);
 }
 
-static IWL_MLD_ALLOC_FN(link_sta, link_sta)
+IWL_MLD_ALLOC_FN_STATIC(link_sta, link_sta)
 
 static int
 iwl_mld_add_link_sta(struct iwl_mld *mld, struct ieee80211_link_sta *link_sta)
@@ -759,10 +817,23 @@ int iwl_mld_add_sta(struct iwl_mld *mld, struct ieee80211_sta *sta,
 {
 	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(sta);
 	struct ieee80211_link_sta *link_sta;
+	enum iwl_fw_sta_type type;
 	int link_id;
 	int ret;
 
-	ret = iwl_mld_init_sta(mld, sta, vif, STATION_TYPE_PEER);
+	switch (vif->type) {
+	case NL80211_IFTYPE_NAN:
+		type = STATION_TYPE_NAN_PEER_NMI;
+		break;
+	case NL80211_IFTYPE_NAN_DATA:
+		type = STATION_TYPE_NAN_PEER_NDI;
+		break;
+	default:
+		type = STATION_TYPE_PEER;
+		break;
+	}
+
+	ret = iwl_mld_init_sta(mld, sta, vif, type);
 	if (ret)
 		return ret;
 
@@ -1001,10 +1072,9 @@ static int iwl_mld_send_aux_sta_cmd(struct iwl_mld *mld,
 }
 
 static int
-iwl_mld_add_internal_sta_to_fw(struct iwl_mld *mld,
+iwl_mld_set_internal_sta_to_fw(struct iwl_mld *mld,
 			       const struct iwl_mld_int_sta *internal_sta,
-			       u8 fw_link_id,
-			       const u8 *addr)
+			       u32 link_mask, const u8 *addr)
 {
 	struct iwl_sta_cfg_cmd cmd = {};
 
@@ -1012,20 +1082,30 @@ iwl_mld_add_internal_sta_to_fw(struct iwl_mld *mld,
 		return iwl_mld_send_aux_sta_cmd(mld, internal_sta);
 
 	cmd.sta_id = cpu_to_le32((u8)internal_sta->sta_id);
-	cmd.link_mask = cpu_to_le32(BIT(fw_link_id));
+	cmd.link_mask = cpu_to_le32(link_mask);
 	cmd.station_type = cpu_to_le32(internal_sta->sta_type);
 
 	/* FW doesn't allow to add a IGTK/BIGTK if the sta isn't marked as MFP.
 	 * On the other hand, FW will never check this flag during RX since
 	 * an AP/GO doesn't receive protected broadcast management frames.
 	 * So, we can set it unconditionally.
+	 *
+	 * For NAN stations associated with a NAN Device, the MFP bit must be
+	 * set to 1, as otherwise the FW will assert when a key associated with
+	 * these stations would be added.
 	 */
-	if (internal_sta->sta_type == STATION_TYPE_BCAST_MGMT)
+	if (internal_sta->sta_type == STATION_TYPE_BCAST_MGMT ||
+	    internal_sta->sta_type == STATION_TYPE_NAN_BCAST ||
+	    internal_sta->sta_type == STATION_TYPE_NAN_MGMT)
 		cmd.mfp = cpu_to_le32(1);
 
 	if (addr) {
-		memcpy(cmd.peer_mld_address, addr, ETH_ALEN);
-		memcpy(cmd.peer_link_address, addr, ETH_ALEN);
+		if (internal_sta->sta_type == STATION_TYPE_NAN_MCAST_DATA) {
+			ether_addr_copy(cmd.ndi_local_addr, addr);
+		} else {
+			memcpy(cmd.peer_mld_address, addr, ETH_ALEN);
+			memcpy(cmd.peer_link_address, addr, ETH_ALEN);
+		}
 	}
 
 	return iwl_mld_send_sta_cmd(mld, &cmd);
@@ -1034,7 +1114,8 @@ iwl_mld_add_internal_sta_to_fw(struct iwl_mld *mld,
 static int iwl_mld_add_internal_sta(struct iwl_mld *mld,
 				    struct iwl_mld_int_sta *internal_sta,
 				    enum iwl_fw_sta_type sta_type,
-				    u8 fw_link_id, const u8 *addr, u8 tid)
+				    u32 link_mask, const u8 *addr,
+				    u8 tid, bool add_txq)
 {
 	int ret, queue_id;
 
@@ -1046,11 +1127,14 @@ static int iwl_mld_add_internal_sta(struct iwl_mld *mld,
 
 	internal_sta->sta_type = sta_type;
 
-	ret = iwl_mld_add_internal_sta_to_fw(mld, internal_sta, fw_link_id,
+	ret = iwl_mld_set_internal_sta_to_fw(mld, internal_sta, link_mask,
 					     addr);
 	if (ret)
 		goto err;
 
+	if (!add_txq)
+		return 0;
+
 	queue_id = iwl_mld_allocate_internal_txq(mld, internal_sta, tid);
 	if (queue_id < 0) {
 		iwl_mld_rm_sta_from_fw(mld, internal_sta->sta_id);
@@ -1085,8 +1169,8 @@ int iwl_mld_add_bcast_sta(struct iwl_mld *mld,
 
 	return iwl_mld_add_internal_sta(mld, &mld_link->bcast_sta,
 					STATION_TYPE_BCAST_MGMT,
-					mld_link->fw_id, addr,
-					IWL_MGMT_TID);
+					BIT(mld_link->fw_id), addr,
+					IWL_MGMT_TID, true);
 }
 
 int iwl_mld_add_mcast_sta(struct iwl_mld *mld,
@@ -1105,14 +1189,16 @@ int iwl_mld_add_mcast_sta(struct iwl_mld *mld,
 
 	return iwl_mld_add_internal_sta(mld, &mld_link->mcast_sta,
 					STATION_TYPE_MCAST,
-					mld_link->fw_id, mcast_addr, 0);
+					BIT(mld_link->fw_id), mcast_addr,
+					0, true);
 }
 
 int iwl_mld_add_aux_sta(struct iwl_mld *mld,
 			struct iwl_mld_int_sta *internal_sta)
 {
 	return iwl_mld_add_internal_sta(mld, internal_sta, STATION_TYPE_AUX,
-					0, NULL, IWL_MAX_TID_COUNT);
+					0, NULL, IWL_MAX_TID_COUNT,
+					true);
 }
 
 int iwl_mld_add_mon_sta(struct iwl_mld *mld,
@@ -1129,23 +1215,25 @@ int iwl_mld_add_mon_sta(struct iwl_mld *mld,
 
 	return iwl_mld_add_internal_sta(mld, &mld_link->mon_sta,
 					STATION_TYPE_BCAST_MGMT,
-					mld_link->fw_id, NULL,
-					IWL_MAX_TID_COUNT);
+					BIT(mld_link->fw_id), NULL,
+					IWL_MAX_TID_COUNT,
+					true);
 }
 
 static void iwl_mld_remove_internal_sta(struct iwl_mld *mld,
 					struct iwl_mld_int_sta *internal_sta,
 					bool flush, u8 tid)
 {
-	if (WARN_ON_ONCE(internal_sta->sta_id == IWL_INVALID_STA ||
-			 internal_sta->queue_id == IWL_MLD_INVALID_QUEUE))
+	if (WARN_ON_ONCE(internal_sta->sta_id == IWL_INVALID_STA))
 		return;
 
-	if (flush)
+	if (flush && !WARN_ON_ONCE(internal_sta->queue_id ==
+				   IWL_MLD_INVALID_QUEUE))
 		iwl_mld_flush_link_sta_txqs(mld, internal_sta->sta_id);
 
-	iwl_mld_free_txq(mld, BIT(internal_sta->sta_id),
-			 tid, internal_sta->queue_id);
+	if (internal_sta->queue_id != IWL_MLD_INVALID_QUEUE)
+		iwl_mld_free_txq(mld, BIT(internal_sta->sta_id),
+				 tid, internal_sta->queue_id);
 
 	iwl_mld_rm_sta_from_fw(mld, internal_sta->sta_id);
 
@@ -1346,3 +1434,82 @@ int iwl_mld_update_link_stas(struct iwl_mld *mld,
 
 	return ret;
 }
+
+int iwl_mld_add_nan_bcast_sta(struct iwl_mld *mld,
+			      struct iwl_mld_int_sta *sta)
+{
+	const u8 bcast_addr[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
+
+	return iwl_mld_add_internal_sta(mld, sta, STATION_TYPE_NAN_BCAST,
+					0, bcast_addr, 0, false);
+}
+
+int iwl_mld_add_nan_mgmt_sta(struct iwl_mld *mld,
+			     struct iwl_mld_int_sta *sta)
+{
+	return iwl_mld_add_internal_sta(mld, sta, STATION_TYPE_NAN_MGMT,
+					0, NULL, IWL_MAX_TID_COUNT, true);
+}
+
+int iwl_mld_add_nan_mcast_data_sta(struct iwl_mld *mld,
+				   const u8 *ndi_addr,
+				   struct iwl_mld_int_sta *sta)
+{
+	u32 link_mask = iwl_mld_get_nan_link_mask(mld);
+
+	/* In case that there are no NAN links, nothing to do */
+	if (!link_mask)
+		return 0;
+
+	return iwl_mld_add_internal_sta(mld, sta,
+					STATION_TYPE_NAN_MCAST_DATA,
+					link_mask, ndi_addr,
+					0, true);
+}
+
+int iwl_mld_update_nan_mcast_data_sta(struct iwl_mld *mld,
+				      const u8 *ndi_addr,
+				      struct iwl_mld_int_sta *sta)
+{
+	u32 link_mask;
+
+	if (WARN_ON(!mld->nan_device_vif))
+		return -EINVAL;
+
+	/* If the sta doesn't exist, add it */
+	if (sta->sta_id == IWL_INVALID_STA)
+		return iwl_mld_add_nan_mcast_data_sta(mld, ndi_addr, sta);
+
+	/* The station was already added */
+	if (WARN_ON(sta->sta_type != STATION_TYPE_NAN_MCAST_DATA))
+		return -EINVAL;
+
+	link_mask = iwl_mld_get_nan_link_mask(mld);
+	if (link_mask)
+		return iwl_mld_set_internal_sta_to_fw(mld,
+						      sta, link_mask,
+						      ndi_addr);
+
+	/* If no links are associated with NAN, remove the station */
+	iwl_mld_remove_nan_mcast_data_sta(mld, sta);
+
+	return 0;
+}
+
+void iwl_mld_remove_nan_bcast_sta(struct iwl_mld *mld,
+				  struct iwl_mld_int_sta *sta)
+{
+	iwl_mld_remove_internal_sta(mld, sta, false, 0);
+}
+
+void iwl_mld_remove_nan_mgmt_sta(struct iwl_mld *mld,
+				 struct iwl_mld_int_sta *sta)
+{
+	iwl_mld_remove_internal_sta(mld, sta, true, IWL_MAX_TID_COUNT);
+}
+
+void iwl_mld_remove_nan_mcast_data_sta(struct iwl_mld *mld,
+				       struct iwl_mld_int_sta *sta)
+{
+	iwl_mld_remove_internal_sta(mld, sta, true, 0);
+}
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/sta.h b/drivers/net/wireless/intel/iwlwifi/mld/sta.h
index 36288c2fb38c5..98d693235c663 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/sta.h
+++ b/drivers/net/wireless/intel/iwlwifi/mld/sta.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #ifndef __iwl_mld_sta_h__
@@ -28,6 +28,9 @@ struct iwl_mld_rxq_dup_data {
  * This represents the link-level sta - the driver level equivalent to the
  * ieee80211_link_sta
  *
+ * @rx_igtk: FW can only have one IGTK for RX at a time, whereas mac80211 will
+ *	have two. This tracks the one IGTK that currently exists in FW, to
+ *	remove it there when a new one is installed.
  * @last_rate_n_flags: rate_n_flags from the last &iwl_tlc_update_notif
  * @signal_avg: the signal average coming from the firmware
  * @in_fw: whether the link STA is uploaded to the FW (false during restart)
@@ -37,6 +40,7 @@ struct iwl_mld_rxq_dup_data {
 struct iwl_mld_link_sta {
 	/* Add here fields that need clean up on restart */
 	struct_group(zeroed_on_hw_restart,
+		struct ieee80211_key_conf *rx_igtk;
 		u32 last_rate_n_flags;
 		bool in_fw;
 		s8 signal_avg;
@@ -195,6 +199,8 @@ void iwl_mld_remove_sta(struct iwl_mld *mld, struct ieee80211_sta *sta);
 int iwl_mld_fw_sta_id_from_link_sta(struct iwl_mld *mld,
 				    struct ieee80211_link_sta *link_sta);
 u32 iwl_mld_fw_sta_id_mask(struct iwl_mld *mld, struct ieee80211_sta *sta);
+int iwl_mld_add_modify_sta_cmd(struct iwl_mld *mld,
+			       struct ieee80211_link_sta *link_sta);
 int iwl_mld_update_all_link_stations(struct iwl_mld *mld,
 				     struct ieee80211_sta *sta);
 void iwl_mld_flush_sta_txqs(struct iwl_mld *mld, struct ieee80211_sta *sta);
@@ -270,4 +276,28 @@ int iwl_mld_update_link_stas(struct iwl_mld *mld,
 			     struct ieee80211_vif *vif,
 			     struct ieee80211_sta *sta,
 			     u16 old_links, u16 new_links);
+
+int iwl_mld_add_nan_bcast_sta(struct iwl_mld *mld,
+			      struct iwl_mld_int_sta *sta);
+
+int iwl_mld_add_nan_mgmt_sta(struct iwl_mld *mld,
+			     struct iwl_mld_int_sta *sta);
+
+int iwl_mld_add_nan_mcast_data_sta(struct iwl_mld *mld,
+				   const u8 *ndi_addr,
+				   struct iwl_mld_int_sta *sta);
+
+int iwl_mld_update_nan_mcast_data_sta(struct iwl_mld *mld,
+				      const u8 *ndi_addr,
+				      struct iwl_mld_int_sta *sta);
+
+void iwl_mld_remove_nan_bcast_sta(struct iwl_mld *mld,
+				  struct iwl_mld_int_sta *sta);
+
+void iwl_mld_remove_nan_mgmt_sta(struct iwl_mld *mld,
+				 struct iwl_mld_int_sta *sta);
+
+void iwl_mld_remove_nan_mcast_data_sta(struct iwl_mld *mld,
+				       struct iwl_mld_int_sta *sta);
+
 #endif /* __iwl_mld_sta_h__ */
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/stats.c b/drivers/net/wireless/intel/iwlwifi/mld/stats.c
index 54eb0ead78eee..e7b283cbe1991 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/stats.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/stats.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #include "mld.h"
@@ -40,13 +40,21 @@ iwl_mld_fill_stats_from_oper_notif(struct iwl_mld *mld,
 				   struct iwl_rx_packet *pkt,
 				   u8 fw_sta_id, struct station_info *sinfo)
 {
-	const struct iwl_system_statistics_notif_oper *notif =
-		(void *)&pkt->data;
-	const struct iwl_stats_ntfy_per_sta *per_sta =
-		&notif->per_sta[fw_sta_id];
+	const struct iwl_stats_ntfy_per_sta *per_sta;
 	struct ieee80211_link_sta *link_sta;
 	struct iwl_mld_link_sta *mld_link_sta;
 
+	if (iwl_fw_lookup_notif_ver(mld->fw, STATISTICS_GROUP,
+				    STATISTICS_OPER_NOTIF, 3) >= 4) {
+		const struct iwl_system_statistics_notif_oper *notif =
+			(void *)&pkt->data;
+		per_sta = &notif->per_sta[fw_sta_id];
+	} else {
+		const struct iwl_system_statistics_notif_oper_v3 *notif =
+			(void *)&pkt->data;
+		per_sta = &notif->per_sta[fw_sta_id];
+	}
+
 	/* 0 isn't a valid value, but FW might send 0.
 	 * In that case, set the latest non-zero value we stored
 	 */
@@ -303,6 +311,40 @@ static void iwl_mld_sta_stats_fill_txrate(struct iwl_mld_sta *mld_sta,
 	}
 }
 
+static void iwl_mld_sta_stats_fill_beacon_signal_avg(struct ieee80211_vif *vif,
+						     struct station_info *sinfo)
+{
+	struct ieee80211_bss_conf *link_conf;
+	struct iwl_mld_link *link;
+	u8 link_id;
+
+	if (iwl_mld_emlsr_active(vif))
+		return;
+
+	/* TODO: support statistics for NAN */
+	if (vif->type == NL80211_IFTYPE_NAN ||
+	    vif->type == NL80211_IFTYPE_NAN_DATA)
+		return;
+
+	link_id = iwl_mld_get_primary_link(vif);
+	link_conf = link_conf_dereference_protected(vif, link_id);
+
+	if (WARN_ONCE(!link_conf,
+		      "link_conf is NULL for link_id=%u\n", link_id))
+		return;
+
+	link = iwl_mld_link_from_mac80211(link_conf);
+	if (WARN_ONCE(!link,
+		      "iwl_mld_link is NULL for link_id=%u\n", link_id))
+		return;
+
+	if (!link->avg_signal)
+		return;
+
+	sinfo->rx_beacon_signal_avg = link->avg_signal;
+	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_BEACON_SIGNAL_AVG);
+}
+
 void iwl_mld_mac80211_sta_statistics(struct ieee80211_hw *hw,
 				     struct ieee80211_vif *vif,
 				     struct ieee80211_sta *sta,
@@ -321,9 +363,9 @@ void iwl_mld_mac80211_sta_statistics(struct ieee80211_hw *hw,
 
 	iwl_mld_sta_stats_fill_txrate(mld_sta, sinfo);
 
-	/* TODO: NL80211_STA_INFO_BEACON_RX */
+	iwl_mld_sta_stats_fill_beacon_signal_avg(vif, sinfo);
 
-	/* TODO: NL80211_STA_INFO_BEACON_SIGNAL_AVG */
+	/* TODO: NL80211_STA_INFO_BEACON_RX */
 }
 
 #define IWL_MLD_TRAFFIC_LOAD_MEDIUM_THRESH	10 /* percentage */
@@ -435,6 +477,8 @@ iwl_mld_process_per_link_stats(struct iwl_mld *mld,
 	     fw_id++) {
 		const struct iwl_stats_ntfy_per_link *link_stats;
 		struct ieee80211_bss_conf *bss_conf;
+		struct iwl_mld_link *link;
+		u32 avg_raw;
 		int sig;
 
 		bss_conf = iwl_mld_fw_id_to_link_conf(mld, fw_id);
@@ -448,6 +492,13 @@ iwl_mld_process_per_link_stats(struct iwl_mld *mld,
 		sig = -le32_to_cpu(link_stats->beacon_filter_average_energy);
 		iwl_mld_update_link_sig(bss_conf->vif, sig, bss_conf);
 
+		link = iwl_mld_link_from_mac80211(bss_conf);
+		if (WARN_ON_ONCE(!link))
+			continue;
+
+		avg_raw = le32_to_cpu(link_stats->beacon_average_energy);
+		link->avg_signal = clamp_t(int, -(int)avg_raw, S8_MIN, 0);
+
 		/* TODO: parse more fields here (task=statistics)*/
 	}
 
@@ -507,6 +558,10 @@ static void iwl_mld_fill_chanctx_stats(struct ieee80211_hw *hw,
 						  (old_load >> 1);
 	}
 
+	IWL_DEBUG_EHT(phy->mld,
+		      "PHY %d: load_by_us=%u%% load_not_by_us=%u%%\n",
+		      phy->fw_id, phy->channel_load_by_us, new_load);
+
 	iwl_mld_emlsr_check_chan_load(hw, phy, old_load);
 }
 
@@ -523,17 +578,42 @@ iwl_mld_process_per_phy_stats(struct iwl_mld *mld,
 void iwl_mld_handle_stats_oper_notif(struct iwl_mld *mld,
 				     struct iwl_rx_packet *pkt)
 {
-	const struct iwl_system_statistics_notif_oper *stats =
+	struct iwl_system_statistics_notif_oper *_notif __free(kfree) = NULL;
+	const struct iwl_system_statistics_notif_oper *notif =
 		(void *)&pkt->data;
-	u32 curr_ts_usec = le32_to_cpu(stats->time_stamp);
 
-	BUILD_BUG_ON(ARRAY_SIZE(stats->per_sta) != IWL_STATION_COUNT_MAX);
-	BUILD_BUG_ON(ARRAY_SIZE(stats->per_link) <
+	BUILD_BUG_ON(ARRAY_SIZE(notif->per_sta) != IWL_STATION_COUNT_MAX);
+	BUILD_BUG_ON(ARRAY_SIZE(notif->per_link) <
 		     ARRAY_SIZE(mld->fw_id_to_bss_conf));
 
-	iwl_mld_process_per_link_stats(mld, stats->per_link, curr_ts_usec);
-	iwl_mld_process_per_sta_stats(mld, stats->per_sta);
-	iwl_mld_process_per_phy_stats(mld, stats->per_phy);
+	if (iwl_fw_lookup_notif_ver(mld->fw, STATISTICS_GROUP,
+				    STATISTICS_OPER_NOTIF, 3) == 3) {
+		const struct iwl_system_statistics_notif_oper_v3 *stats =
+			(void *)&pkt->data;
+		_notif = kzalloc_obj(*_notif);
+
+		if (!_notif)
+			return;
+
+		_notif->time_stamp = stats->time_stamp;
+		for (int i = 0; i < ARRAY_SIZE(_notif->per_link); i++)
+			_notif->per_link[i] = stats->per_link[i];
+
+		BUILD_BUG_ON(sizeof(_notif->per_phy[0]) <
+			     sizeof(stats->per_phy[0]));
+		for (int i = 0; i < ARRAY_SIZE(_notif->per_phy); i++)
+			memcpy(&_notif->per_phy[i], &stats->per_phy[i],
+			       sizeof(stats->per_phy[i]));
+		for (int i = 0; i < ARRAY_SIZE(_notif->per_sta); i++)
+			_notif->per_sta[i] = stats->per_sta[i];
+
+		notif = _notif;
+	}
+
+	iwl_mld_process_per_link_stats(mld, notif->per_link,
+				       le32_to_cpu(notif->time_stamp));
+	iwl_mld_process_per_sta_stats(mld, notif->per_sta);
+	iwl_mld_process_per_phy_stats(mld, notif->per_phy);
 }
 
 void iwl_mld_handle_stats_oper_part1_notif(struct iwl_mld *mld,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tests/Makefile b/drivers/net/wireless/intel/iwlwifi/mld/tests/Makefile
index 36317feb923ba..efa61638b8eeb 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tests/Makefile
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tests/Makefile
@@ -1,5 +1,6 @@
 # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 iwlmld-tests-y += module.o hcmd.o utils.o link.o rx.o agg.o link-selection.o
+iwlmld-tests-y += chan_load_thresh.o
 
 ccflags-y += -I$(src)/../
 obj-$(CONFIG_IWLWIFI_KUNIT_TESTS) += iwlmld-tests.o
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tests/chan_load_thresh.c b/drivers/net/wireless/intel/iwlwifi/mld/tests/chan_load_thresh.c
new file mode 100644
index 0000000000000..87e29e09949b1
--- /dev/null
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tests/chan_load_thresh.c
@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ * KUnit tests for channel helper functions
+ *
+ * Copyright (C) 2026 Intel Corporation
+ */
+#include <kunit/static_stub.h>
+#include "mld.h"
+#include "link.h"
+#include "iface.h"
+#include "utils.h"
+
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+
+struct test_chan_load_case {
+	const char *desc;
+	u32 load;
+	enum iwl_mld_link_chan_load_level old_lvl;
+	enum iwl_mld_link_chan_load_level expected_lvl;
+	bool expected_scan_trig;
+};
+
+static const struct test_chan_load_case test_chan_load_thresh_cases[] = {
+	/* Level-up transitions */
+	{
+		.desc = "Transition NONE->NONE",
+		.load = 20,
+		.old_lvl = LINK_CHAN_LOAD_LVL_NONE,
+		.expected_lvl = LINK_CHAN_LOAD_LVL_NONE,
+		.expected_scan_trig = false,
+	},
+	{
+		.desc = "Transition NONE->LVL1",
+		.load = 50,
+		.old_lvl = LINK_CHAN_LOAD_LVL_NONE,
+		.expected_lvl = LINK_CHAN_LOAD_LVL1,
+		.expected_scan_trig = true,
+	},
+	{
+		.desc = "Transition LVL1->LVL2",
+		.load = 75,
+		.old_lvl = LINK_CHAN_LOAD_LVL1,
+		.expected_lvl = LINK_CHAN_LOAD_LVL2,
+		.expected_scan_trig = true,
+	},
+	{
+		.desc = "Transition LVL2->LVL3",
+		.load = 90,
+		.old_lvl = LINK_CHAN_LOAD_LVL2,
+		.expected_lvl = LINK_CHAN_LOAD_LVL3,
+		.expected_scan_trig = true,
+	},
+
+	/* Level-down transitions */
+	{
+		.desc = "Transition LVL1->NONE",
+		.load = 30,
+		.old_lvl = LINK_CHAN_LOAD_LVL1,
+		.expected_lvl = LINK_CHAN_LOAD_LVL_NONE,
+		.expected_scan_trig = false,
+	},
+	{
+		.desc = "Transition LVL2->LVL1",
+		.load = 60,
+		.old_lvl = LINK_CHAN_LOAD_LVL2,
+		.expected_lvl = LINK_CHAN_LOAD_LVL1,
+		.expected_scan_trig = false,
+	},
+	{
+		.desc = "Transition LVL3->LVL2",
+		.load = 70,
+		.old_lvl = LINK_CHAN_LOAD_LVL3,
+		.expected_lvl = LINK_CHAN_LOAD_LVL2,
+		.expected_scan_trig = false,
+	},
+
+	/* No change */
+	{
+		.desc = "Transition LVL2->LVL2",
+		.load = 72,
+		.old_lvl = LINK_CHAN_LOAD_LVL2,
+		.expected_lvl = LINK_CHAN_LOAD_LVL2,
+		.expected_scan_trig = false,
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(test_chan_load_thresh_cases,
+		       test_chan_load_thresh_cases, desc);
+
+static void test_chan_load_thresholds(struct kunit *test)
+{
+	const struct test_chan_load_case *tc = test->param_value;
+	struct iwl_mld *mld = test->priv;
+	struct ieee80211_vif *vif;
+	struct iwl_mld_vif *mld_vif;
+	struct ieee80211_bss_conf *link_conf;
+	struct iwl_mld_link *mld_link;
+	struct iwl_mld_kunit_link assoc_link = {
+		.id = 0,
+		.chandef = &chandef_6ghz_160mhz,
+	};
+	bool scan_trig;
+	u32 chan_load;
+
+	/* Setup associated non-MLO station */
+	vif = iwlmld_kunit_setup_non_mlo_assoc(&assoc_link);
+	mld_vif = iwl_mld_vif_from_mac80211(vif);
+
+	link_conf = &vif->bss_conf;
+	mld_link = &mld_vif->deflink;
+
+	chan_load = tc->load;
+	mld_link->chan_load_lvl = tc->old_lvl;
+
+	/* Execute function under test */
+	wiphy_lock(mld->wiphy);
+	scan_trig = iwl_mld_chan_load_requires_scan(mld, link_conf, chan_load);
+	wiphy_unlock(mld->wiphy);
+
+	/* Check return value */
+	KUNIT_EXPECT_EQ(test, tc->expected_scan_trig, scan_trig);
+
+	/* Check updated channel-load level */
+	KUNIT_EXPECT_EQ(test, tc->expected_lvl, mld_link->chan_load_lvl);
+}
+
+static struct kunit_case chan_load_thresh_test_cases[] = {
+	KUNIT_CASE_PARAM(test_chan_load_thresholds,
+			 test_chan_load_thresh_cases_gen_params),
+	{}
+};
+
+static struct kunit_suite chan_load_thresh_test_suite = {
+	.name = "iwl_mld_chan_load_threshold_tests",
+	.init = iwlmld_kunit_test_init,
+	.test_cases = chan_load_thresh_test_cases,
+};
+
+kunit_test_suite(chan_load_thresh_test_suite);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tests/link-selection.c b/drivers/net/wireless/intel/iwlwifi/mld/tests/link-selection.c
index 766c24db36137..69d222a8194c5 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tests/link-selection.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tests/link-selection.c
@@ -2,7 +2,7 @@
 /*
  * KUnit tests for link selection functions
  *
- * Copyright (C) 2025 Intel Corporation
+ * Copyright (C) 2025-2026 Intel Corporation
  */
 #include <kunit/static_stub.h>
 
@@ -34,6 +34,7 @@ static const struct link_grading_test_case {
 			.link_id = 0,
 			.chandef = &chandef_2ghz_20mhz,
 			.active = false,
+			.signal = -70,
 			.has_chan_util_elem = true,
 			.chan_util = 128,
 		},
@@ -45,6 +46,7 @@ static const struct link_grading_test_case {
 			.link_id = 0,
 			.chandef = &chandef_2ghz_20mhz,
 			.active = false,
+			.signal = -70,
 			.has_chan_util_elem = true,
 			.chan_util = 180,
 		},
@@ -55,6 +57,7 @@ static const struct link_grading_test_case {
 		.input.link = {
 			.link_id = 0,
 			.chandef = &chandef_2ghz_20mhz,
+			.signal = -70,
 			.has_chan_util_elem = true,
 			.chan_util = 180,
 			.active = true,
@@ -67,6 +70,7 @@ static const struct link_grading_test_case {
 		.input.link = {
 			.link_id = 0,
 			.chandef = &chandef_2ghz_20mhz,
+			.signal = -70,
 			.active = true,
 		},
 		.expected_grade = 120,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tests/utils.c b/drivers/net/wireless/intel/iwlwifi/mld/tests/utils.c
index dce7472701672..cb1968b074526 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tests/utils.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tests/utils.c
@@ -2,7 +2,7 @@
 /*
  * KUnit tests for channel helper functions
  *
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 #include <kunit/test.h>
 #include <kunit/test-bug.h>
@@ -68,8 +68,6 @@ int iwlmld_kunit_test_init(struct kunit *test)
 	return 0;
 }
 
-static IWL_MLD_ALLOC_FN(link, bss_conf)
-
 static void iwlmld_kunit_init_link(struct ieee80211_vif *vif,
 				   struct ieee80211_bss_conf *link,
 				   struct iwl_mld_link *mld_link, int link_id)
@@ -94,7 +92,7 @@ static void iwlmld_kunit_init_link(struct ieee80211_vif *vif,
 	rcu_assign_pointer(vif->link_conf[link_id], link);
 }
 
-static IWL_MLD_ALLOC_FN(vif, vif)
+IWL_MLD_ALLOC_FN_STATIC(vif, vif)
 
 /* Helper function to add and initialize a VIF for KUnit tests */
 struct ieee80211_vif *iwlmld_kunit_add_vif(bool mlo, enum nl80211_iftype type)
@@ -199,7 +197,7 @@ void iwlmld_kunit_assign_chanctx_to_link(struct ieee80211_vif *vif,
 		vif->active_links |= BIT(link->link_id);
 }
 
-static IWL_MLD_ALLOC_FN(link_sta, link_sta)
+IWL_MLD_ALLOC_FN_STATIC(link_sta, link_sta)
 
 static void iwlmld_kunit_add_link_sta(struct ieee80211_sta *sta,
 				      struct ieee80211_link_sta *link_sta,
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tlc.c b/drivers/net/wireless/intel/iwlwifi/mld/tlc.c
index 78d6162d9297e..a03834d3ac653 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tlc.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tlc.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024-2025 Intel Corporation
+ * Copyright (C) 2024-2026 Intel Corporation
  */
 
 #include <net/mac80211.h>
@@ -15,9 +15,9 @@
 #include "fw/api/context.h"
 #include "fw/api/dhc.h"
 
-static u8 iwl_mld_fw_bw_from_sta_bw(const struct ieee80211_link_sta *link_sta)
+static u8 iwl_mld_fw_bw_from_sta_bw(enum ieee80211_sta_rx_bandwidth bandwidth)
 {
-	switch (link_sta->bandwidth) {
+	switch (bandwidth) {
 	case IEEE80211_STA_RX_BW_320:
 		return IWL_TLC_MNG_CH_WIDTH_320MHZ;
 	case IEEE80211_STA_RX_BW_160:
@@ -32,40 +32,61 @@ static u8 iwl_mld_fw_bw_from_sta_bw(const struct ieee80211_link_sta *link_sta)
 	}
 }
 
+struct iwl_mld_tlc_sta_capa {
+	u16 non_ht_rates;
+	u16 max_amsdu_len;
+	u8 rx_nss;
+	enum ieee80211_sta_rx_bandwidth bandwidth;
+	enum ieee80211_smps_mode smps_mode;
+	const struct ieee80211_sta_ht_cap *ht_cap;
+	const struct ieee80211_sta_vht_cap *vht_cap;
+	const struct ieee80211_sta_he_cap *he_cap;
+	const struct ieee80211_sta_eht_cap *eht_cap;
+	const struct ieee80211_sta_uhr_cap *uhr_cap;
+
+	const struct ieee80211_sta_he_cap *own_he_cap;
+	const struct ieee80211_sta_eht_cap *own_eht_cap;
+	const struct ieee80211_sta_uhr_cap *own_uhr_cap;
+};
+
 static __le16
 iwl_mld_get_tlc_cmd_flags(struct iwl_mld *mld,
-			  struct ieee80211_vif *vif,
-			  struct ieee80211_link_sta *link_sta,
-			  const struct ieee80211_sta_he_cap *own_he_cap,
-			  const struct ieee80211_sta_eht_cap *own_eht_cap,
-			  const struct ieee80211_sta_uhr_cap *own_uhr_cap)
+			  struct iwl_mld_tlc_sta_capa *capa)
 {
-	struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
-	struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
-	struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
-	bool has_vht = vht_cap->vht_supported;
+	const struct ieee80211_sta_ht_cap *ht_cap = capa->ht_cap;
+	const struct ieee80211_sta_vht_cap *vht_cap = capa->vht_cap;
+	const struct ieee80211_sta_he_cap *he_cap = capa->he_cap;
+	const struct ieee80211_sta_eht_cap *eht_cap = capa->eht_cap;
+	const struct ieee80211_sta_uhr_cap *uhr_cap = capa->uhr_cap;
+	const struct ieee80211_sta_he_cap *own_he_cap = capa->own_he_cap;
+	const struct ieee80211_sta_eht_cap *own_eht_cap = capa->own_eht_cap;
+	const struct ieee80211_sta_uhr_cap *own_uhr_cap = capa->own_uhr_cap;
+	bool has_vht = vht_cap && vht_cap->vht_supported;
 	u16 flags = 0;
 
 	/* STBC flags */
 	if (mld->cfg->ht_params.stbc &&
 	    (hweight8(iwl_mld_get_valid_tx_ant(mld)) > 1)) {
-		if (he_cap->has_he && he_cap->he_cap_elem.phy_cap_info[2] &
-				      IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ)
+		if (he_cap && he_cap->has_he &&
+		    he_cap->he_cap_elem.phy_cap_info[2] &
+				IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ)
 			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
-		else if (vht_cap->cap & IEEE80211_VHT_CAP_RXSTBC_MASK)
+		else if (vht_cap &&
+			 vht_cap->cap & IEEE80211_VHT_CAP_RXSTBC_MASK)
 			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
-		else if (ht_cap->cap & IEEE80211_HT_CAP_RX_STBC)
+		else if (ht_cap && ht_cap->cap & IEEE80211_HT_CAP_RX_STBC)
 			flags |= IWL_TLC_MNG_CFG_FLAGS_STBC_MSK;
 	}
 
 	/* LDPC */
 	if (mld->cfg->ht_params.ldpc &&
-	    ((ht_cap->cap & IEEE80211_HT_CAP_LDPC_CODING) ||
+	    ((ht_cap && ht_cap->cap & IEEE80211_HT_CAP_LDPC_CODING) ||
 	     (has_vht && (vht_cap->cap & IEEE80211_VHT_CAP_RXLDPC))))
 		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
 
-	if (he_cap->has_he && (he_cap->he_cap_elem.phy_cap_info[1] &
-	    IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
+	if (he_cap && he_cap->has_he &&
+	    (he_cap->he_cap_elem.phy_cap_info[1] &
+			IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD))
 		flags |= IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
 
 	if (own_he_cap &&
@@ -74,7 +95,7 @@ iwl_mld_get_tlc_cmd_flags(struct iwl_mld *mld,
 		flags &= ~IWL_TLC_MNG_CFG_FLAGS_LDPC_MSK;
 
 	/* DCM */
-	if (he_cap->has_he &&
+	if (he_cap && he_cap->has_he &&
 	    (he_cap->he_cap_elem.phy_cap_info[3] &
 	     IEEE80211_HE_PHY_CAP3_DCM_MAX_CONST_RX_MASK &&
 	     own_he_cap &&
@@ -85,15 +106,15 @@ iwl_mld_get_tlc_cmd_flags(struct iwl_mld *mld,
 	/* Extra EHT LTF */
 	if (own_eht_cap &&
 	    own_eht_cap->eht_cap_elem.phy_cap_info[5] &
-		IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF &&
-	    link_sta->eht_cap.has_eht &&
-	    link_sta->eht_cap.eht_cap_elem.phy_cap_info[5] &
-	    IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF) {
+			IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF &&
+	    eht_cap && eht_cap->has_eht &&
+	    eht_cap->eht_cap_elem.phy_cap_info[5] &
+			IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF) {
 		flags |= IWL_TLC_MNG_CFG_FLAGS_EHT_EXTRA_LTF_MSK;
 	}
 
-	if (link_sta->uhr_cap.has_uhr && own_uhr_cap &&
-	    link_sta->uhr_cap.phy.cap & IEEE80211_UHR_PHY_CAP_ELR_RX &&
+	if (uhr_cap && uhr_cap->has_uhr && own_uhr_cap &&
+	    uhr_cap->phy.cap & IEEE80211_UHR_PHY_CAP_ELR_RX &&
 	    own_uhr_cap->phy.cap & IEEE80211_UHR_PHY_CAP_ELR_TX)
 		flags |= IWL_TLC_MNG_CFG_FLAGS_UHR_ELR_1_5_MBPS_MSK |
 			 IWL_TLC_MNG_CFG_FLAGS_UHR_ELR_3_MBPS_MSK;
@@ -114,27 +135,27 @@ static u8 iwl_mld_get_fw_chains(struct iwl_mld *mld)
 	return fw_chains;
 }
 
-static u8 iwl_mld_get_fw_sgi(struct ieee80211_link_sta *link_sta)
+static u8 iwl_mld_get_fw_sgi(struct iwl_mld_tlc_sta_capa *capa)
 {
-	struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
-	struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
-	struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
+	const struct ieee80211_sta_ht_cap *ht_cap = capa->ht_cap;
+	const struct ieee80211_sta_vht_cap *vht_cap = capa->vht_cap;
+	const struct ieee80211_sta_he_cap *he_cap = capa->he_cap;
 	u8 sgi_chwidths = 0;
 
 	/* If the association supports HE, HT/VHT rates will never be used for
 	 * Tx and therefor there's no need to set the
 	 * sgi-per-channel-width-support bits
 	 */
-	if (he_cap->has_he)
+	if (he_cap && he_cap->has_he)
 		return 0;
 
-	if (ht_cap->cap & IEEE80211_HT_CAP_SGI_20)
+	if (ht_cap && ht_cap->cap & IEEE80211_HT_CAP_SGI_20)
 		sgi_chwidths |= BIT(IWL_TLC_MNG_CH_WIDTH_20MHZ);
-	if (ht_cap->cap & IEEE80211_HT_CAP_SGI_40)
+	if (ht_cap && ht_cap->cap & IEEE80211_HT_CAP_SGI_40)
 		sgi_chwidths |= BIT(IWL_TLC_MNG_CH_WIDTH_40MHZ);
-	if (vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_80)
+	if (vht_cap && vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_80)
 		sgi_chwidths |= BIT(IWL_TLC_MNG_CH_WIDTH_80MHZ);
-	if (vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_160)
+	if (vht_cap && vht_cap->cap & IEEE80211_VHT_CAP_SHORT_GI_160)
 		sgi_chwidths |= BIT(IWL_TLC_MNG_CH_WIDTH_160MHZ);
 
 	return sgi_chwidths;
@@ -164,31 +185,30 @@ iwl_mld_get_highest_fw_mcs(const struct ieee80211_sta_vht_cap *vht_cap,
 }
 
 static void
-iwl_mld_fill_vht_rates(const struct ieee80211_link_sta *link_sta,
-		       const struct ieee80211_sta_vht_cap *vht_cap,
+iwl_mld_fill_vht_rates(struct iwl_mld_tlc_sta_capa *capa,
 		       struct iwl_tlc_config_cmd *cmd)
 {
 	u32 supp;
 	int i, highest_mcs;
-	u8 max_nss = link_sta->rx_nss;
+	u8 max_nss = capa->rx_nss;
 	struct ieee80211_vht_cap ieee_vht_cap = {
-		.vht_cap_info = cpu_to_le32(vht_cap->cap),
-		.supp_mcs = vht_cap->vht_mcs,
+		.vht_cap_info = cpu_to_le32(capa->vht_cap->cap),
+		.supp_mcs = capa->vht_cap->vht_mcs,
 	};
 
 	/* the station support only a single receive chain */
-	if (link_sta->smps_mode == IEEE80211_SMPS_STATIC)
+	if (capa->smps_mode == IEEE80211_SMPS_STATIC)
 		max_nss = 1;
 
 	for (i = 0; i < max_nss && i < IWL_TLC_NSS_MAX; i++) {
 		int nss = i + 1;
 
-		highest_mcs = iwl_mld_get_highest_fw_mcs(vht_cap, nss);
+		highest_mcs = iwl_mld_get_highest_fw_mcs(capa->vht_cap, nss);
 		if (!highest_mcs)
 			continue;
 
 		supp = BIT(highest_mcs + 1) - 1;
-		if (link_sta->bandwidth == IEEE80211_STA_RX_BW_20)
+		if (capa->bandwidth == IEEE80211_STA_RX_BW_20)
 			supp &= ~BIT(IWL_TLC_MNG_HT_RATE_MCS9);
 
 		cmd->ht_rates[i][IWL_TLC_MCS_PER_BW_80] = cpu_to_le32(supp);
@@ -196,7 +216,7 @@ iwl_mld_fill_vht_rates(const struct ieee80211_link_sta *link_sta,
 		 * configuration is supported - only for MCS 0 since we already
 		 * decoded the MCS bits anyway ourselves.
 		 */
-		if (link_sta->bandwidth == IEEE80211_STA_RX_BW_160 &&
+		if (capa->bandwidth == IEEE80211_STA_RX_BW_160 &&
 		    ieee80211_get_vht_max_nss(&ieee_vht_cap,
 					      IEEE80211_VHT_CHANWIDTH_160MHZ,
 					      0, true, nss) >= nss)
@@ -223,20 +243,19 @@ static u32 iwl_mld_he_mac80211_mcs_to_fw_mcs(u16 mcs)
 }
 
 static void
-iwl_mld_fill_he_rates(const struct ieee80211_link_sta *link_sta,
-		      const struct ieee80211_sta_he_cap *own_he_cap,
+iwl_mld_fill_he_rates(struct iwl_mld_tlc_sta_capa *capa,
 		      struct iwl_tlc_config_cmd *cmd)
 {
-	const struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
+	const struct ieee80211_sta_he_cap *he_cap = capa->he_cap;
 	u16 mcs_160 = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_160);
 	u16 mcs_80 = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_80);
-	u16 tx_mcs_80 = le16_to_cpu(own_he_cap->he_mcs_nss_supp.tx_mcs_80);
-	u16 tx_mcs_160 = le16_to_cpu(own_he_cap->he_mcs_nss_supp.tx_mcs_160);
+	u16 tx_mcs_80 = le16_to_cpu(capa->own_he_cap->he_mcs_nss_supp.tx_mcs_80);
+	u16 tx_mcs_160 = le16_to_cpu(capa->own_he_cap->he_mcs_nss_supp.tx_mcs_160);
 	int i;
-	u8 nss = link_sta->rx_nss;
+	u8 nss = capa->rx_nss;
 
 	/* the station support only a single receive chain */
-	if (link_sta->smps_mode == IEEE80211_SMPS_STATIC)
+	if (capa->smps_mode == IEEE80211_SMPS_STATIC)
 		nss = 1;
 
 	for (i = 0; i < nss && i < IWL_TLC_NSS_MAX; i++) {
@@ -313,17 +332,15 @@ static u8 iwl_mld_get_eht_max_nss(u8 rx_nss, u8 tx_nss)
 
 static void
 iwl_mld_fill_eht_rates(struct ieee80211_vif *vif,
-		       const struct ieee80211_link_sta *link_sta,
-		       const struct ieee80211_sta_he_cap *own_he_cap,
-		       const struct ieee80211_sta_eht_cap *own_eht_cap,
+		       struct iwl_mld_tlc_sta_capa *capa,
 		       struct iwl_tlc_config_cmd *cmd)
 {
 	/* peer RX mcs capa */
 	const struct ieee80211_eht_mcs_nss_supp *eht_rx_mcs =
-		&link_sta->eht_cap.eht_mcs_nss_supp;
+		&capa->eht_cap->eht_mcs_nss_supp;
 	/* our TX mcs capa */
 	const struct ieee80211_eht_mcs_nss_supp *eht_tx_mcs =
-		&own_eht_cap->eht_mcs_nss_supp;
+		&capa->own_eht_cap->eht_mcs_nss_supp;
 
 	enum IWL_TLC_MCS_PER_BW bw;
 	struct ieee80211_eht_mcs_nss_supp_20mhz_only mcs_rx_20;
@@ -331,7 +348,7 @@ iwl_mld_fill_eht_rates(struct ieee80211_vif *vif,
 
 	/* peer is 20 MHz only */
 	if (vif->type == NL80211_IFTYPE_AP &&
-	    !(link_sta->he_cap.he_cap_elem.phy_cap_info[0] &
+	    !(capa->he_cap->he_cap_elem.phy_cap_info[0] &
 	      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
 		mcs_rx_20 = eht_rx_mcs->only_20mhz;
 	} else {
@@ -346,7 +363,7 @@ iwl_mld_fill_eht_rates(struct ieee80211_vif *vif,
 	}
 
 	/* NIC is capable of 20 MHz only */
-	if (!(own_he_cap->he_cap_elem.phy_cap_info[0] &
+	if (!(capa->own_he_cap->he_cap_elem.phy_cap_info[0] &
 	      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_MASK_ALL)) {
 		mcs_tx_20 = eht_tx_mcs->only_20mhz;
 	} else {
@@ -402,67 +419,51 @@ iwl_mld_fill_eht_rates(struct ieee80211_vif *vif,
 	}
 
 	/* the station support only a single receive chain */
-	if (link_sta->smps_mode == IEEE80211_SMPS_STATIC ||
-	    link_sta->rx_nss < 2)
+	if (capa->smps_mode == IEEE80211_SMPS_STATIC ||
+	    capa->rx_nss < 2)
 		memset(cmd->ht_rates[IWL_TLC_NSS_2], 0,
 		       sizeof(cmd->ht_rates[IWL_TLC_NSS_2]));
 }
 
 static void
-iwl_mld_fill_supp_rates(struct iwl_mld *mld, struct ieee80211_vif *vif,
-			struct ieee80211_link_sta *link_sta,
-			struct ieee80211_supported_band *sband,
-			const struct ieee80211_sta_he_cap *own_he_cap,
-			const struct ieee80211_sta_eht_cap *own_eht_cap,
-			const struct ieee80211_sta_uhr_cap *own_uhr_cap,
+iwl_mld_fill_supp_rates(struct iwl_mld *mld,
+			struct ieee80211_vif *vif,
+			struct iwl_mld_tlc_sta_capa *capa,
 			struct iwl_tlc_config_cmd *cmd)
 {
-	int i;
-	u16 non_ht_rates = 0;
-	unsigned long rates_bitmap;
-	const struct ieee80211_sta_ht_cap *ht_cap = &link_sta->ht_cap;
-	const struct ieee80211_sta_vht_cap *vht_cap = &link_sta->vht_cap;
-	const struct ieee80211_sta_he_cap *he_cap = &link_sta->he_cap;
-
-	/* non HT rates */
-	rates_bitmap = link_sta->supp_rates[sband->band];
-	for_each_set_bit(i, &rates_bitmap, BITS_PER_LONG)
-		non_ht_rates |= BIT(sband->bitrates[i].hw_value);
-
-	cmd->non_ht_rates = cpu_to_le16(non_ht_rates);
+	cmd->non_ht_rates = cpu_to_le16(capa->non_ht_rates);
 	cmd->mode = IWL_TLC_MNG_MODE_NON_HT;
 
-	if (link_sta->uhr_cap.has_uhr && own_uhr_cap) {
+	if (capa->uhr_cap && capa->uhr_cap->has_uhr && capa->own_uhr_cap) {
 		cmd->mode = IWL_TLC_MNG_MODE_UHR;
 		/*
 		 * FIXME: spec currently inherits from EHT but has no
 		 * finer MCS bits. Once that's there, need to add them
 		 * to the bitmaps (and maybe copy this to UHR, or so.)
 		 */
-		iwl_mld_fill_eht_rates(vif, link_sta, own_he_cap,
-				       own_eht_cap, cmd);
-	} else if (link_sta->eht_cap.has_eht && own_he_cap && own_eht_cap) {
+		iwl_mld_fill_eht_rates(vif, capa, cmd);
+	} else if (capa->eht_cap && capa->eht_cap->has_eht &&
+		   capa->own_he_cap && capa->own_eht_cap) {
 		cmd->mode = IWL_TLC_MNG_MODE_EHT;
-		iwl_mld_fill_eht_rates(vif, link_sta, own_he_cap,
-				       own_eht_cap, cmd);
-	} else if (he_cap->has_he && own_he_cap) {
+		iwl_mld_fill_eht_rates(vif, capa, cmd);
+	} else if (capa->he_cap && capa->he_cap->has_he && capa->own_he_cap) {
 		cmd->mode = IWL_TLC_MNG_MODE_HE;
-		iwl_mld_fill_he_rates(link_sta, own_he_cap, cmd);
-	} else if (vht_cap->vht_supported) {
+		iwl_mld_fill_he_rates(capa, cmd);
+	} else if (capa->vht_cap && capa->vht_cap->vht_supported) {
 		cmd->mode = IWL_TLC_MNG_MODE_VHT;
-		iwl_mld_fill_vht_rates(link_sta, vht_cap, cmd);
-	} else if (ht_cap->ht_supported) {
+		iwl_mld_fill_vht_rates(capa, cmd);
+	} else if (capa->ht_cap && capa->ht_cap->ht_supported) {
 		cmd->mode = IWL_TLC_MNG_MODE_HT;
 		cmd->ht_rates[IWL_TLC_NSS_1][IWL_TLC_MCS_PER_BW_80] =
-			cpu_to_le32(ht_cap->mcs.rx_mask[0]);
+			cpu_to_le32(capa->ht_cap->mcs.rx_mask[0]);
 
 		/* the station support only a single receive chain */
-		if (link_sta->smps_mode == IEEE80211_SMPS_STATIC)
+		if (capa->smps_mode == IEEE80211_SMPS_STATIC)
 			cmd->ht_rates[IWL_TLC_NSS_2][IWL_TLC_MCS_PER_BW_80] =
 				0;
 		else
 			cmd->ht_rates[IWL_TLC_NSS_2][IWL_TLC_MCS_PER_BW_80] =
-				cpu_to_le32(ht_cap->mcs.rx_mask[1]);
+				cpu_to_le32(capa->ht_cap->mcs.rx_mask[1]);
 	}
 }
 
@@ -527,60 +528,32 @@ static int iwl_mld_convert_tlc_cmd_to_v4(struct iwl_tlc_config_cmd *cmd,
 
 static void iwl_mld_send_tlc_cmd(struct iwl_mld *mld,
 				 struct ieee80211_vif *vif,
-				 struct ieee80211_link_sta *link_sta,
-				 struct ieee80211_bss_conf *link)
+				 struct iwl_mld_sta *mld_sta,
+				 int fw_sta_mask, int phy_id,
+				 struct iwl_mld_tlc_sta_capa *capa)
 {
-	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
-	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link);
-	enum nl80211_band band = link->chanreq.oper.chan->band;
-	struct ieee80211_supported_band *sband = mld->hw->wiphy->bands[band];
-	const struct ieee80211_sta_he_cap *own_he_cap =
-		ieee80211_get_he_iftype_cap_vif(sband, vif);
-	const struct ieee80211_sta_eht_cap *own_eht_cap =
-		ieee80211_get_eht_iftype_cap_vif(sband, vif);
-	const struct ieee80211_sta_uhr_cap *own_uhr_cap =
-		ieee80211_get_uhr_iftype_cap_vif(sband, vif);
 	struct iwl_tlc_config_cmd cmd = {
 		/* For AP mode, use 20 MHz until the STA is authorized */
 		.max_ch_width = mld_sta->sta_state > IEEE80211_STA_ASSOC ?
-			iwl_mld_fw_bw_from_sta_bw(link_sta) :
+			iwl_mld_fw_bw_from_sta_bw(capa->bandwidth) :
 			IWL_TLC_MNG_CH_WIDTH_20MHZ,
-		.flags = iwl_mld_get_tlc_cmd_flags(mld, vif, link_sta,
-						   own_he_cap, own_eht_cap,
-						   own_uhr_cap),
+		.flags = iwl_mld_get_tlc_cmd_flags(mld, capa),
 		.chains = iwl_mld_get_fw_chains(mld),
-		.sgi_ch_width_supp = iwl_mld_get_fw_sgi(link_sta),
-		.max_mpdu_len = cpu_to_le16(link_sta->agg.max_amsdu_len),
+		.sgi_ch_width_supp = iwl_mld_get_fw_sgi(capa),
+		.max_mpdu_len = cpu_to_le16(capa->max_amsdu_len),
 	};
-	int fw_sta_id = iwl_mld_fw_sta_id_from_link_sta(mld, link_sta);
 	u32 cmd_id = WIDE_ID(DATA_PATH_GROUP, TLC_MNG_CONFIG_CMD);
 	u8 cmd_ver = iwl_fw_lookup_cmd_ver(mld->fw, cmd_id, 0);
-	struct ieee80211_chanctx_conf *chan_ctx;
 	struct iwl_tlc_config_cmd_v5 cmd_v5 = {};
 	struct iwl_tlc_config_cmd_v4 cmd_v4 = {};
 	void *cmd_ptr;
 	u8 cmd_size;
-	u32 phy_id;
 	int ret;
 
-	if (fw_sta_id < 0)
-		return;
-
-	cmd.sta_mask = cpu_to_le32(BIT(fw_sta_id));
-
-	if (WARN_ON_ONCE(!mld_link))
-		return;
-
-	chan_ctx = rcu_dereference_wiphy(mld->wiphy, mld_link->chan_ctx);
-	if (WARN_ON(!chan_ctx))
-		return;
-
-	phy_id = iwl_mld_phy_from_mac80211(chan_ctx)->fw_id;
+	cmd.sta_mask = cpu_to_le32(fw_sta_mask);
 	cmd.phy_id = cpu_to_le32(phy_id);
 
-	iwl_mld_fill_supp_rates(mld, vif, link_sta, sband,
-				own_he_cap, own_eht_cap,
-				own_uhr_cap, &cmd);
+	iwl_mld_fill_supp_rates(mld, vif, capa, &cmd);
 
 	if (cmd_ver == 6) {
 		cmd_ptr = &cmd;
@@ -641,16 +614,32 @@ int iwl_mld_send_tlc_dhc(struct iwl_mld *mld, u8 sta_id, u32 type, u32 data)
 	return ret;
 }
 
-void iwl_mld_config_tlc_link(struct iwl_mld *mld,
-			     struct ieee80211_vif *vif,
-			     struct ieee80211_bss_conf *link_conf,
-			     struct ieee80211_link_sta *link_sta)
+static void _iwl_mld_config_tlc_link(struct iwl_mld *mld,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_chanctx_conf *chan_ctx,
+				     u32 fw_sta_mask,
+				     struct ieee80211_link_sta *link_sta,
+				     struct iwl_mld_tlc_sta_capa *capa)
 {
 	struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(link_sta->sta);
+	struct ieee80211_supported_band *sband;
+	unsigned long rates_bitmap;
+	enum nl80211_band band;
+	int i;
 
-	if (WARN_ON_ONCE(!link_conf->chanreq.oper.chan))
+	/*
+	 * Note: Due to NAN, the chan_ctx here might not be the same as the
+	 * vif->links[link_sta->link_id] one (which is NULL in NAN), so some
+	 * care is needed in this function (and the 'capa' exists to make it
+	 * less error-prone, the other functions need not worry about it.)
+	 */
+
+	if (WARN_ON(!chan_ctx))
 		return;
 
+	band = chan_ctx->def.chan->band;
+	sband = mld->hw->wiphy->bands[band];
+
 	/* Before we have information about a station, configure the A-MSDU RC
 	 * limit such that iwlmd and mac80211 would not be allowed to build
 	 * A-MSDUs.
@@ -660,7 +649,21 @@ void iwl_mld_config_tlc_link(struct iwl_mld *mld,
 		ieee80211_sta_recalc_aggregates(link_sta->sta);
 	}
 
-	iwl_mld_send_tlc_cmd(mld, vif, link_sta, link_conf);
+	/* non HT rates */
+	rates_bitmap = link_sta->supp_rates[sband->band];
+	for_each_set_bit(i, &rates_bitmap, BITS_PER_LONG)
+		capa->non_ht_rates |= BIT(sband->bitrates[i].hw_value);
+
+	capa->max_amsdu_len = link_sta->agg.max_amsdu_len;
+	capa->ht_cap = &link_sta->ht_cap;
+	capa->vht_cap = &link_sta->vht_cap;
+	capa->he_cap = &link_sta->he_cap;
+	capa->eht_cap = &link_sta->eht_cap;
+	capa->uhr_cap = &link_sta->uhr_cap;
+
+	iwl_mld_send_tlc_cmd(mld, vif, mld_sta, fw_sta_mask,
+			     iwl_mld_phy_from_mac80211(chan_ctx)->fw_id,
+			     capa);
 }
 
 void iwl_mld_tlc_update_phy(struct iwl_mld *mld, struct ieee80211_vif *vif,
@@ -706,6 +709,136 @@ void iwl_mld_tlc_update_phy(struct iwl_mld *mld, struct ieee80211_vif *vif,
 	}
 }
 
+static void iwl_mld_config_tlc_nan(struct iwl_mld *mld,
+				   struct ieee80211_vif *vif,
+				   struct ieee80211_link_sta *link_sta,
+				   int fw_sta_id)
+{
+	struct ieee80211_nan_peer_sched *sched;
+	struct iwl_mld_nan_link *nan_link;
+	struct ieee80211_sta *nmi, *iter;
+	struct iwl_mld_vif *nan_mld_vif;
+	u32 fw_sta_mask;
+
+	nmi = wiphy_dereference(mld->wiphy, link_sta->sta->nmi);
+	if (WARN_ON(!nmi))
+		return;
+
+	sched = nmi->nan_sched;
+
+	/* This sta might not be in the list yet as it is just getting added */
+	fw_sta_mask = BIT(fw_sta_id);
+	for_each_station(iter, mld->hw) {
+		struct iwl_mld_sta *mld_sta = iwl_mld_sta_from_mac80211(iter);
+		struct ieee80211_sta *iter_nmi;
+
+		iter_nmi = wiphy_dereference(mld->wiphy, iter->nmi);
+
+		if (iter_nmi == nmi)
+			fw_sta_mask |= BIT(mld_sta->deflink.fw_id);
+	}
+
+	if (WARN_ON(!mld->nan_device_vif))
+		return;
+
+	nan_mld_vif = iwl_mld_vif_from_mac80211(mld->nan_device_vif);
+
+	for_each_mld_nan_valid_link(nan_mld_vif, nan_link) {
+		struct ieee80211_chanctx_conf *chan_ctx;
+		struct iwl_mld_tlc_sta_capa capa = {};
+		struct cfg80211_chan_def *chandef;
+		bool common_channel = false;
+
+		chan_ctx = nan_link->chanctx;
+		if (!chan_ctx)
+			continue;
+
+		chandef = iwl_mld_get_chandef_from_chanctx(mld, chan_ctx);
+
+		capa.smps_mode = IEEE80211_SMPS_OFF; /* always off */
+
+		capa.rx_nss = 2; /* maximum we support */
+		capa.bandwidth = ieee80211_chan_width_to_rx_bw(chandef->width);
+
+		for (int j = 0; j < (sched ? sched->n_channels : 0); j++) {
+			enum ieee80211_sta_rx_bandwidth rx_bw;
+			enum nl80211_chan_width width;
+			int chains;
+
+			if (sched->channels[j].chanctx_conf != chan_ctx)
+				continue;
+
+			common_channel = true;
+
+			width = sched->channels[j].chanreq.oper.width;
+			rx_bw = ieee80211_chan_width_to_rx_bw(width);
+			capa.bandwidth = min(capa.bandwidth, rx_bw);
+
+			chains = sched->channels[j].needed_rx_chains;
+			capa.rx_nss = min(capa.rx_nss, chains);
+		}
+
+		/* Apply minimum parameters if there is no common channel */
+		if (!common_channel) {
+			capa.bandwidth = IEEE80211_STA_RX_BW_20;
+			capa.rx_nss = 1;
+		}
+
+		capa.own_he_cap = &mld->wiphy->nan_capa.phy.he;
+		/* no EHT/UHR for NAN */
+
+		_iwl_mld_config_tlc_link(mld, vif, chan_ctx, fw_sta_mask,
+					 link_sta, &capa);
+	}
+}
+
+void iwl_mld_config_tlc_link(struct iwl_mld *mld,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_bss_conf *link_conf,
+			     struct ieee80211_link_sta *link_sta)
+{
+	struct iwl_mld_link *mld_link = iwl_mld_link_from_mac80211(link_conf);
+	int fw_sta_id = iwl_mld_fw_sta_id_from_link_sta(mld, link_sta);
+	struct ieee80211_chanctx_conf *chan_ctx;
+	struct ieee80211_supported_band *sband;
+	struct iwl_mld_tlc_sta_capa capa = {};
+	struct cfg80211_chan_def *chandef;
+	enum nl80211_band band;
+
+	if (fw_sta_id < 0)
+		return;
+
+	if (vif->type == NL80211_IFTYPE_NAN)
+		return;
+
+	if (vif->type == NL80211_IFTYPE_NAN_DATA) {
+		iwl_mld_config_tlc_nan(mld, vif, link_sta, fw_sta_id);
+		return;
+	}
+
+	if (WARN_ON_ONCE(!mld_link))
+		return;
+
+	chan_ctx = rcu_dereference_wiphy(mld->wiphy, mld_link->chan_ctx);
+	if (WARN_ON_ONCE(!chan_ctx))
+		return;
+
+	chandef = &iwl_mld_phy_from_mac80211(chan_ctx)->chandef;
+	band = chandef->chan->band;
+	sband = mld->hw->wiphy->bands[band];
+
+	capa.smps_mode = link_sta->smps_mode;
+	capa.rx_nss = link_sta->rx_nss;
+	capa.bandwidth = ieee80211_chan_width_to_rx_bw(chandef->width);
+	capa.bandwidth = min(capa.bandwidth, link_sta->bandwidth);
+	capa.own_he_cap = ieee80211_get_he_iftype_cap_vif(sband, vif);
+	capa.own_eht_cap = ieee80211_get_eht_iftype_cap_vif(sband, vif);
+	capa.own_uhr_cap = ieee80211_get_uhr_iftype_cap_vif(sband, vif);
+
+	_iwl_mld_config_tlc_link(mld, vif, chan_ctx, BIT(fw_sta_id),
+				 link_sta, &capa);
+}
+
 void iwl_mld_config_tlc(struct iwl_mld *mld, struct ieee80211_vif *vif,
 			struct ieee80211_sta *sta)
 {
@@ -714,6 +847,7 @@ void iwl_mld_config_tlc(struct iwl_mld *mld, struct ieee80211_vif *vif,
 
 	lockdep_assert_wiphy(mld->wiphy);
 
+	/* Note: for NAN this is only the mac80211-level deflink */
 	for_each_vif_active_link(vif, link, link_id) {
 		struct ieee80211_link_sta *link_sta =
 			link_sta_dereference_check(sta, link_id);
@@ -822,9 +956,7 @@ void iwl_mld_handle_tlc_notif(struct iwl_mld *mld,
 		if (WARN_ON(!mld_link_sta))
 			return;
 
-		mld_link_sta->last_rate_n_flags =
-			iwl_v3_rate_from_v2_v3(notif->rate,
-					       mld->fw_rates_ver_3);
+		mld_link_sta->last_rate_n_flags = le32_to_cpu(notif->rate);
 
 		rs_pretty_print_rate(pretty_rate, sizeof(pretty_rate),
 				     mld_link_sta->last_rate_n_flags);
diff --git a/drivers/net/wireless/intel/iwlwifi/mld/tx.c b/drivers/net/wireless/intel/iwlwifi/mld/tx.c
index 546d09a38dab4..1e8716cbb4ce3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mld/tx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mld/tx.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2024 - 2025 Intel Corporation
+ * Copyright (C) 2024 - 2026 Intel Corporation
  */
 #include <net/ip.h>
 
@@ -71,14 +71,19 @@ static int iwl_mld_allocate_txq(struct iwl_mld *mld, struct ieee80211_txq *txq)
 {
 	u8 tid = txq->tid == IEEE80211_NUM_TIDS ? IWL_MGMT_TID : txq->tid;
 	u32 fw_sta_mask = iwl_mld_fw_sta_id_mask(mld, txq->sta);
-	/* We can't know when the station is asleep or awake, so we
-	 * must disable the queue hang detection.
-	 */
-	unsigned int watchdog_timeout = txq->vif->type == NL80211_IFTYPE_AP ?
-				IWL_WATCHDOG_DISABLED :
-				mld->trans->mac_cfg->base->wd_timeout;
+	unsigned int watchdog_timeout;
 	int queue, size;
 
+	switch (txq->vif->type) {
+	case NL80211_IFTYPE_AP:		/* STA might go to PS */
+	case NL80211_IFTYPE_NAN_DATA:	/* peer might ULW/break schedule */
+		watchdog_timeout = IWL_WATCHDOG_DISABLED;
+		break;
+	default:
+		watchdog_timeout = mld->trans->mac_cfg->base->wd_timeout;
+		break;
+	}
+
 	lockdep_assert_wiphy(mld->wiphy);
 
 	if (tid == IWL_MGMT_TID)
@@ -346,7 +351,8 @@ u8 iwl_mld_get_lowest_rate(struct iwl_mld *mld,
 	iwl_mld_get_basic_rates_and_band(mld, vif, info, &basic_rates, &band);
 
 	if (band >= NUM_NL80211_BANDS) {
-		WARN_ON(vif->type != NL80211_IFTYPE_NAN);
+		WARN_ON(vif->type != NL80211_IFTYPE_NAN &&
+			vif->type != NL80211_IFTYPE_NAN_DATA);
 		return IWL_FIRST_OFDM_RATE;
 	}
 
@@ -489,7 +495,7 @@ static __le32 iwl_mld_get_tx_rate_n_flags(struct iwl_mld *mld,
 		rate = iwl_mld_mac80211_rate_idx_to_fw(mld, info, -1) |
 		       iwl_mld_get_tx_ant(mld, info, sta, fc);
 
-	return iwl_v3_rate_to_v2_v3(rate, mld->fw_rates_ver_3);
+	return cpu_to_le32(rate);
 }
 
 static void
@@ -552,10 +558,12 @@ iwl_mld_fill_tx_cmd(struct iwl_mld *mld, struct sk_buff *skb,
 		flags |= IWL_TX_FLAGS_ENCRYPT_DIS;
 
 	/* For data and mgmt packets rate info comes from the fw.
-	 * Only set rate/antenna for injected frames with fixed rate, or
-	 * when no sta is given.
+	 * Only set rate/antenna for:
+	 * - injected frames with fixed rate,
+	 * - when no sta is given.
+	 * - frames that are sent to an NMI sta, which is only used for management.
 	 */
-	if (unlikely(!sta ||
+	if (unlikely(!sta || mld_sta->vif->type == NL80211_IFTYPE_NAN ||
 		     info->control.flags & IEEE80211_TX_CTRL_RATE_INJECT)) {
 		flags |= IWL_TX_FLAGS_CMD_RATE;
 		rate_n_flags = iwl_mld_get_tx_rate_n_flags(mld, info, sta,
@@ -673,11 +681,16 @@ iwl_mld_get_tx_queue_id(struct iwl_mld *mld, struct ieee80211_txq *txq,
 		WARN_ON(!ieee80211_is_mgmt(fc));
 		return mld_vif->aux_sta.queue_id;
 	case NL80211_IFTYPE_NAN:
-		mld_vif = iwl_mld_vif_from_mac80211(info->control.vif);
-
 		WARN_ON(!ieee80211_is_mgmt(fc));
+		return iwl_mld_nan_get_mgmt_queue(mld, info->control.vif);
+	case NL80211_IFTYPE_NAN_DATA:
+		WARN_ON(!ieee80211_is_data(fc));
 
-		return mld_vif->aux_sta.queue_id;
+		if (!iwl_mld_nan_use_nan_stations(mld))
+			break;
+
+		mld_vif = iwl_mld_vif_from_mac80211(info->control.vif);
+		return mld_vif->nan.mcast_data_sta.queue_id;
 	default:
 		WARN_ONCE(1, "Unsupported vif type\n");
 		break;
@@ -834,7 +847,7 @@ static int iwl_mld_tx_tso_segment(struct iwl_mld *mld, struct sk_buff *skb,
 		return -EINVAL;
 
 	max_tid_amsdu_len = sta->cur->max_tid_amsdu_len[tid];
-	if (!max_tid_amsdu_len)
+	if (!max_tid_amsdu_len || max_tid_amsdu_len == 1)
 		return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skbs);
 
 	/* Sub frame header + SNAP + IP header + TCP header + MSS */
@@ -846,6 +859,9 @@ static int iwl_mld_tx_tso_segment(struct iwl_mld *mld, struct sk_buff *skb,
 	 */
 	num_subframes = (max_tid_amsdu_len + pad) / (subf_len + pad);
 
+	if (WARN_ON_ONCE(!num_subframes))
+		return iwl_tx_tso_segment(skb, 1, netdev_flags, mpdus_skbs);
+
 	if (sta->max_amsdu_subframes &&
 	    num_subframes > sta->max_amsdu_subframes)
 		num_subframes = sta->max_amsdu_subframes;
@@ -970,6 +986,16 @@ void iwl_mld_tx_from_txq(struct iwl_mld *mld, struct ieee80211_txq *txq)
 	struct sk_buff *skb = NULL;
 	u8 zero_addr[ETH_ALEN] = {};
 
+	/*
+	 * Don't transmit during firmware restart. The firmware is dead,
+	 * so iwl_trans_tx() would return -EIO for each frame. Avoid the
+	 * overhead of dequeuing from mac80211 only to immediately free
+	 * the skbs, and the potential memory pressure from rapid skb
+	 * allocation churn during high-throughput restart scenarios.
+	 */
+	if (unlikely(mld->fw_status.in_hw_restart))
+		return;
+
 	/*
 	 * No need for threads to be pending here, they can leave the first
 	 * taker all the work.
@@ -1005,14 +1031,11 @@ void iwl_mld_tx_from_txq(struct iwl_mld *mld, struct ieee80211_txq *txq)
 	rcu_read_unlock();
 }
 
-static void iwl_mld_hwrate_to_tx_rate(struct iwl_mld *mld,
-				      __le32 rate_n_flags_fw,
+static void iwl_mld_hwrate_to_tx_rate(u32 rate_n_flags,
 				      struct ieee80211_tx_info *info)
 {
 	enum nl80211_band band = info->band;
 	struct ieee80211_tx_rate *tx_rate = &info->status.rates[0];
-	u32 rate_n_flags = iwl_v3_rate_from_v2_v3(rate_n_flags_fw,
-						  mld->fw_rates_ver_3);
 	u32 sgi = rate_n_flags & RATE_MCS_SGI_MSK;
 	u32 chan_width = rate_n_flags & RATE_MCS_CHAN_WIDTH_MSK;
 	u32 format = rate_n_flags & RATE_MCS_MOD_TYPE_MSK;
@@ -1143,7 +1166,8 @@ void iwl_mld_handle_tx_resp_notif(struct iwl_mld *mld,
 			iwl_dbg_tlv_time_point(&mld->fwrt, tp, NULL);
 		}
 
-		iwl_mld_hwrate_to_tx_rate(mld, tx_resp->initial_rate, info);
+		iwl_mld_hwrate_to_tx_rate(le32_to_cpu(tx_resp->initial_rate),
+					  info);
 
 		if (likely(!iwl_mld_time_sync_frame(mld, skb, hdr->addr1)))
 			ieee80211_tx_status_skb(mld->hw, skb);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
index 1b67836b1fac5..3a14ca5e512ae 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-initiator.c
@@ -261,20 +261,28 @@ iwl_mvm_ftm_target_chandef_v2(struct iwl_mvm *mvm,
 
 	switch (peer->chandef.width) {
 	case NL80211_CHAN_WIDTH_20_NOHT:
-		*format_bw = IWL_LOCATION_FRAME_FORMAT_LEGACY;
-		*format_bw |= IWL_LOCATION_BW_20MHZ << LOCATION_BW_POS;
+		*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_LEGACY,
+					    IWL_LOCATION_FMT_BW_FORMAT);
+		*format_bw |= u8_encode_bits(IWL_LOCATION_BW_20MHZ,
+					     IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_20:
-		*format_bw = IWL_LOCATION_FRAME_FORMAT_HT;
-		*format_bw |= IWL_LOCATION_BW_20MHZ << LOCATION_BW_POS;
+		*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_HT,
+					    IWL_LOCATION_FMT_BW_FORMAT);
+		*format_bw |= u8_encode_bits(IWL_LOCATION_BW_20MHZ,
+					     IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_40:
-		*format_bw = IWL_LOCATION_FRAME_FORMAT_HT;
-		*format_bw |= IWL_LOCATION_BW_40MHZ << LOCATION_BW_POS;
+		*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_HT,
+					    IWL_LOCATION_FMT_BW_FORMAT);
+		*format_bw |= u8_encode_bits(IWL_LOCATION_BW_40MHZ,
+					     IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_80:
-		*format_bw = IWL_LOCATION_FRAME_FORMAT_VHT;
-		*format_bw |= IWL_LOCATION_BW_80MHZ << LOCATION_BW_POS;
+		*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_VHT,
+					    IWL_LOCATION_FMT_BW_FORMAT);
+		*format_bw |= u8_encode_bits(IWL_LOCATION_BW_80MHZ,
+					     IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_160:
 		cmd_ver = iwl_fw_lookup_cmd_ver(mvm->fw,
@@ -282,8 +290,10 @@ iwl_mvm_ftm_target_chandef_v2(struct iwl_mvm *mvm,
 						IWL_FW_CMD_VER_UNKNOWN);
 
 		if (cmd_ver >= 13) {
-			*format_bw = IWL_LOCATION_FRAME_FORMAT_HE;
-			*format_bw |= IWL_LOCATION_BW_160MHZ << LOCATION_BW_POS;
+			*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_HE,
+						    IWL_LOCATION_FMT_BW_FORMAT);
+			*format_bw |= u8_encode_bits(IWL_LOCATION_BW_160MHZ,
+						     IWL_LOCATION_FMT_BW_BANDWIDTH);
 			break;
 		}
 		fallthrough;
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-responder.c b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-responder.c
index 83f6e508a094c..ae7a163c81c93 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ftm-responder.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ftm-responder.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2024 Intel Corporation
+ * Copyright (C) 2018-2024, 2026 Intel Corporation
  */
 #include <net/cfg80211.h>
 #include <linux/etherdevice.h>
@@ -54,27 +54,37 @@ static int iwl_mvm_ftm_responder_set_bw_v2(struct cfg80211_chan_def *chandef,
 {
 	switch (chandef->width) {
 	case NL80211_CHAN_WIDTH_20_NOHT:
-		*format_bw = IWL_LOCATION_FRAME_FORMAT_LEGACY;
-		*format_bw |= IWL_LOCATION_BW_20MHZ << LOCATION_BW_POS;
+		*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_LEGACY,
+					    IWL_LOCATION_FMT_BW_FORMAT);
+		*format_bw |= u8_encode_bits(IWL_LOCATION_BW_20MHZ,
+					     IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_20:
-		*format_bw = IWL_LOCATION_FRAME_FORMAT_HT;
-		*format_bw |= IWL_LOCATION_BW_20MHZ << LOCATION_BW_POS;
+		*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_HT,
+					    IWL_LOCATION_FMT_BW_FORMAT);
+		*format_bw |= u8_encode_bits(IWL_LOCATION_BW_20MHZ,
+					     IWL_LOCATION_FMT_BW_BANDWIDTH);
 		break;
 	case NL80211_CHAN_WIDTH_40:
-		*format_bw = IWL_LOCATION_FRAME_FORMAT_HT;
-		*format_bw |= IWL_LOCATION_BW_40MHZ << LOCATION_BW_POS;
+		*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_HT,
+					    IWL_LOCATION_FMT_BW_FORMAT);
+		*format_bw |= u8_encode_bits(IWL_LOCATION_BW_40MHZ,
+					     IWL_LOCATION_FMT_BW_BANDWIDTH);
 		*ctrl_ch_position = iwl_mvm_get_ctrl_pos(chandef);
 		break;
 	case NL80211_CHAN_WIDTH_80:
-		*format_bw = IWL_LOCATION_FRAME_FORMAT_VHT;
-		*format_bw |= IWL_LOCATION_BW_80MHZ << LOCATION_BW_POS;
+		*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_VHT,
+					    IWL_LOCATION_FMT_BW_FORMAT);
+		*format_bw |= u8_encode_bits(IWL_LOCATION_BW_80MHZ,
+					     IWL_LOCATION_FMT_BW_BANDWIDTH);
 		*ctrl_ch_position = iwl_mvm_get_ctrl_pos(chandef);
 		break;
 	case NL80211_CHAN_WIDTH_160:
 		if (cmd_ver >= 9) {
-			*format_bw = IWL_LOCATION_FRAME_FORMAT_HE;
-			*format_bw |= IWL_LOCATION_BW_160MHZ << LOCATION_BW_POS;
+			*format_bw = u8_encode_bits(IWL_LOCATION_FRAME_FORMAT_HE,
+						    IWL_LOCATION_FMT_BW_FORMAT);
+			*format_bw |= u8_encode_bits(IWL_LOCATION_BW_160MHZ,
+						     IWL_LOCATION_FMT_BW_BANDWIDTH);
 			*ctrl_ch_position = iwl_mvm_get_ctrl_pos(chandef);
 			break;
 		}
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
index c523c5e82d4ae..8ffa72aca3cf7 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac-ctxt.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -927,13 +927,18 @@ u8 iwl_mvm_mac_ctxt_get_lowest_rate(struct iwl_mvm *mvm,
 
 u16 iwl_mvm_mac_ctxt_get_beacon_flags(const struct iwl_fw *fw, u8 rate_idx)
 {
-	u16 flags = iwl_mvm_mac80211_idx_to_hwrate(fw, rate_idx);
 	bool is_new_rate = iwl_fw_lookup_cmd_ver(fw, BEACON_TEMPLATE_CMD, 0) > 10;
+	u16 flags = 0;
 
 	if (rate_idx <= IWL_LAST_CCK_RATE)
 		flags |= is_new_rate ? IWL_MAC_BEACON_CCK
 			  : IWL_MAC_BEACON_CCK_V1;
 
+	if (iwl_fw_lookup_cmd_ver(fw, TX_CMD, 0) > 8)
+		flags |= iwl_mvm_mac80211_idx_to_hwrate(fw, rate_idx);
+	else
+		flags |= iwl_fw_rate_idx_to_plcp(rate_idx);
+
 	return flags;
 }
 
@@ -962,6 +967,7 @@ static void iwl_mvm_mac_ctxt_set_tx(struct iwl_mvm *mvm,
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	struct ieee80211_tx_info *info;
+	u32 rate_n_flags = 0;
 	u8 rate;
 	u32 tx_flags;
 
@@ -981,18 +987,21 @@ static void iwl_mvm_mac_ctxt_set_tx(struct iwl_mvm *mvm,
 			 IWL_UCODE_TLV_CAPA_BEACON_ANT_SELECTION)) {
 		iwl_mvm_toggle_tx_ant(mvm, &mvm->mgmt_last_antenna_idx);
 
-		tx_params->rate_n_flags =
-			cpu_to_le32(BIT(mvm->mgmt_last_antenna_idx) <<
-				    RATE_MCS_ANT_POS);
+		rate_n_flags |= BIT(mvm->mgmt_last_antenna_idx) <<
+					RATE_MCS_ANT_POS;
 	}
 
 	rate = iwl_mvm_mac_ctxt_get_beacon_rate(mvm, info, vif);
 
-	tx_params->rate_n_flags |=
-		cpu_to_le32(iwl_mvm_mac80211_idx_to_hwrate(mvm->fw, rate));
-	if (rate == IWL_FIRST_CCK_RATE)
-		tx_params->rate_n_flags |= cpu_to_le32(RATE_MCS_CCK_MSK_V1);
+	if (rate < IWL_FIRST_OFDM_RATE)
+		rate_n_flags |= RATE_MCS_MOD_TYPE_CCK;
+	else
+		rate_n_flags |= RATE_MCS_MOD_TYPE_LEGACY_OFDM;
+
+	rate_n_flags |= iwl_mvm_mac80211_idx_to_hwrate(mvm->fw, rate);
 
+	tx_params->rate_n_flags = iwl_mvm_v3_rate_to_fw(rate_n_flags,
+							mvm->fw_rates_ver);
 }
 
 int iwl_mvm_mac_ctxt_send_beacon_cmd(struct iwl_mvm *mvm,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
index 1ec9807e48272..c256cbc6602e2 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -263,10 +263,10 @@ static const u8 tm_if_types_ext_capa_sta[] = {
  */
 
 #define IWL_MVM_EMLSR_CAPA	(IEEE80211_EML_CAP_EMLSR_SUPP | \
-				 IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_32US << \
-					__bf_shf(IEEE80211_EML_CAP_EMLSR_PADDING_DELAY) | \
+				 IEEE80211_EML_CAP_EML_PADDING_DELAY_32US << \
+					__bf_shf(IEEE80211_EML_CAP_EML_PADDING_DELAY) | \
 				 IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US << \
-					__bf_shf(IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY))
+					__bf_shf(IEEE80211_EML_CAP_EML_TRANSITION_DELAY))
 #define IWL_MVM_MLD_CAPA_OPS (FIELD_PREP_CONST( \
 			IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP, \
 			IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_SAME) | \
@@ -3144,7 +3144,7 @@ iwl_mvm_bss_info_changed_ap_ibss(struct iwl_mvm *mvm,
 	    iwl_mvm_mac_ctxt_beacon_changed(mvm, vif, &vif->bss_conf))
 		IWL_WARN(mvm, "Failed updating beacon data\n");
 
-	if (changes & BSS_CHANGED_FTM_RESPONDER) {
+	if ((changes & BSS_CHANGED_FTM_RESPONDER) && bss_conf->ftm_responder) {
 		int ret = iwl_mvm_ftm_start_responder(mvm, vif, &vif->bss_conf);
 
 		if (ret)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c
index f1dbfeae20bc0..6ea9a7c8da0cb 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mld-mac80211.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2022-2025 Intel Corporation
+ * Copyright (C) 2022-2026 Intel Corporation
  */
 #include "mvm.h"
 
@@ -729,7 +729,7 @@ iwl_mvm_mld_link_info_changed_ap_ibss(struct iwl_mvm *mvm,
 		IWL_WARN(mvm, "Failed updating beacon data\n");
 
 	/* FIXME: need to decide if we need FTM responder per link */
-	if (changes & BSS_CHANGED_FTM_RESPONDER) {
+	if (changes & BSS_CHANGED_FTM_RESPONDER && link_conf->ftm_responder) {
 		int ret = iwl_mvm_ftm_start_responder(mvm, vif, link_conf);
 
 		if (ret)
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
index 402ba5dee8b2c..be89b84204fb3 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -469,7 +469,7 @@ struct iwl_mvm_vif {
 	struct dentry *dbgfs_slink;
 	struct iwl_dbgfs_pm dbgfs_pm;
 	struct iwl_dbgfs_bf dbgfs_bf;
-	struct iwl_mac_power_cmd mac_pwr_cmd;
+	struct iwl_mac_power_cmd_v2 mac_pwr_cmd;
 	int dbgfs_quota_min;
 	bool ftm_unprotected;
 #endif
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
index ae177477b2012..726477336bb4f 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -335,7 +335,7 @@ static const struct iwl_rx_handlers iwl_mvm_rx_handlers[] = {
 	RX_HANDLER_GRP(STATISTICS_GROUP, STATISTICS_OPER_NOTIF,
 		       iwl_mvm_handle_rx_system_oper_stats,
 		       RX_HANDLER_ASYNC_LOCKED_WIPHY,
-		       struct iwl_system_statistics_notif_oper),
+		       struct iwl_system_statistics_notif_oper_v3),
 	RX_HANDLER_GRP(STATISTICS_GROUP, STATISTICS_OPER_PART1_NOTIF,
 		       iwl_mvm_handle_rx_system_oper_part1_stats,
 		       RX_HANDLER_ASYNC_LOCKED,
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/power.c b/drivers/net/wireless/intel/iwlwifi/mvm/power.c
index 610de29b7be0d..46792c5087532 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/power.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/power.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2019, 2021-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2019, 2021-2026 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -83,7 +83,7 @@ void iwl_mvm_beacon_filter_set_cqm_params(struct iwl_mvm *mvm,
 }
 
 static void iwl_mvm_power_log(struct iwl_mvm *mvm,
-			      struct iwl_mac_power_cmd *cmd)
+			      struct iwl_mac_power_cmd_v2 *cmd)
 {
 	IWL_DEBUG_POWER(mvm,
 			"Sending power table command on mac id 0x%X for power level %d, flags = 0x%X\n",
@@ -121,7 +121,7 @@ static void iwl_mvm_power_log(struct iwl_mvm *mvm,
 
 static void iwl_mvm_power_configure_uapsd(struct iwl_mvm *mvm,
 					  struct ieee80211_vif *vif,
-					  struct iwl_mac_power_cmd *cmd)
+					  struct iwl_mac_power_cmd_v2 *cmd)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
 	enum ieee80211_ac_numbers ac;
@@ -296,7 +296,7 @@ static bool iwl_mvm_power_is_radar(struct ieee80211_bss_conf *link_conf)
 
 static void iwl_mvm_power_config_skip_dtim(struct iwl_mvm *mvm,
 					   struct ieee80211_vif *vif,
-					   struct iwl_mac_power_cmd *cmd)
+					   struct iwl_mac_power_cmd_v2 *cmd)
 {
 	struct ieee80211_bss_conf *link_conf;
 	unsigned int min_link_skip = ~0;
@@ -344,7 +344,7 @@ static void iwl_mvm_power_config_skip_dtim(struct iwl_mvm *mvm,
 
 static void iwl_mvm_power_build_cmd(struct iwl_mvm *mvm,
 				    struct ieee80211_vif *vif,
-				    struct iwl_mac_power_cmd *cmd)
+				    struct iwl_mac_power_cmd_v2 *cmd)
 {
 	int dtimper, bi;
 	int keep_alive;
@@ -466,7 +466,7 @@ static void iwl_mvm_power_build_cmd(struct iwl_mvm *mvm,
 static int iwl_mvm_power_send_cmd(struct iwl_mvm *mvm,
 					 struct ieee80211_vif *vif)
 {
-	struct iwl_mac_power_cmd cmd = {};
+	struct iwl_mac_power_cmd_v2 cmd = {};
 
 	iwl_mvm_power_build_cmd(mvm, vif, &cmd);
 	iwl_mvm_power_log(mvm, &cmd);
@@ -717,7 +717,7 @@ int iwl_mvm_power_mac_dbgfs_read(struct iwl_mvm *mvm,
 				 int bufsz)
 {
 	struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif);
-	struct iwl_mac_power_cmd cmd = {};
+	struct iwl_mac_power_cmd_v2 cmd = {};
 	int pos = 0;
 
 	mutex_lock(&mvm->mutex);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
index f7b620136c857..e05e531b09f0b 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/ptp.c
@@ -325,11 +325,11 @@ void iwl_mvm_ptp_remove(struct iwl_mvm *mvm)
 			       mvm->ptp_data.ptp_clock_info.name,
 			       ptp_clock_index(mvm->ptp_data.ptp_clock));
 
+		cancel_delayed_work_sync(&mvm->ptp_data.dwork);
 		ptp_clock_unregister(mvm->ptp_data.ptp_clock);
 		mvm->ptp_data.ptp_clock = NULL;
 		memset(&mvm->ptp_data.ptp_clock_info, 0,
 		       sizeof(mvm->ptp_data.ptp_clock_info));
 		mvm->ptp_data.last_gp2 = 0;
-		cancel_delayed_work_sync(&mvm->ptp_data.dwork);
 	}
 }
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
index d0c0faae01221..269c4b45de807 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/rx.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -757,8 +757,9 @@ iwl_mvm_update_tcm_from_stats(struct iwl_mvm *mvm, __le32 *air_time_le,
 	spin_unlock(&mvm->tcm.lock);
 }
 
-static void iwl_mvm_handle_per_phy_stats(struct iwl_mvm *mvm,
-					 struct iwl_stats_ntfy_per_phy *per_phy)
+static void
+iwl_mvm_handle_per_phy_stats(struct iwl_mvm *mvm,
+			     struct iwl_stats_ntfy_per_phy_v1 *per_phy)
 {
 	int i;
 
@@ -937,7 +938,7 @@ void iwl_mvm_handle_rx_system_oper_stats(struct iwl_mvm *mvm,
 {
 	u8 average_energy[IWL_STATION_COUNT_MAX];
 	struct iwl_rx_packet *pkt = rxb_addr(rxb);
-	struct iwl_system_statistics_notif_oper *stats;
+	struct iwl_system_statistics_notif_oper_v3 *stats;
 	int i;
 	u32 notif_ver = iwl_fw_lookup_notif_ver(mvm->fw, STATISTICS_GROUP,
 						STATISTICS_OPER_NOTIF, 0);
diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
index 4a33a032c2a77..f052537e9567e 100644
--- a/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
+++ b/drivers/net/wireless/intel/iwlwifi/mvm/utils.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2012-2014, 2018-2025 Intel Corporation
+ * Copyright (C) 2012-2014, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2014 Intel Mobile Communications GmbH
  * Copyright (C) 2015-2017 Intel Deutschland GmbH
  */
@@ -159,15 +159,9 @@ int iwl_mvm_legacy_rate_to_mac80211_idx(u32 rate_n_flags,
 
 u8 iwl_mvm_mac80211_idx_to_hwrate(const struct iwl_fw *fw, int rate_idx)
 {
-	if (iwl_fw_lookup_cmd_ver(fw, TX_CMD, 0) > 8)
-		/* In the new rate legacy rates are indexed:
-		 * 0 - 3 for CCK and 0 - 7 for OFDM.
-		 */
-		return (rate_idx >= IWL_FIRST_OFDM_RATE ?
-			rate_idx - IWL_FIRST_OFDM_RATE :
-			rate_idx);
-
-	return iwl_fw_rate_idx_to_plcp(rate_idx);
+	return rate_idx >= IWL_FIRST_OFDM_RATE ?
+		rate_idx - IWL_FIRST_OFDM_RATE :
+		rate_idx;
 }
 
 u8 iwl_mvm_mac80211_ac_to_ucode_ac(enum ieee80211_ac_numbers ac)
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
index dc99e7ac47261..608100bc6b119 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c
@@ -537,6 +537,8 @@ VISIBLE_IF_IWLWIFI_KUNIT const struct pci_device_id iwl_hw_card_ids[] = {
 	{IWL_PCI_DEVICE(0xA840, 0x4314, iwl_bz_mac_cfg)},
 	{IWL_PCI_DEVICE(0xA840, 0x1775, iwl_bz_mac_cfg)},
 	{IWL_PCI_DEVICE(0xA840, 0x1776, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1735, iwl_bz_mac_cfg)},
+	{IWL_PCI_DEVICE(0xA840, 0x1736, iwl_bz_mac_cfg)},
 	{IWL_PCI_DEVICE(0x7740, PCI_ANY_ID, iwl_bz_mac_cfg)},
 	{IWL_PCI_DEVICE(0x4D40, PCI_ANY_ID, iwl_bz_mac_cfg)},
 
@@ -1052,6 +1054,8 @@ VISIBLE_IF_IWLWIFI_KUNIT const struct iwl_dev_info iwl_dev_info_table[] = {
 	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1750i_name, SUBDEV(0x1772)),
 	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1790s_name, SUBDEV(0x1791)),
 	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1790i_name, SUBDEV(0x1792)),
+	IWL_DEV_INFO(iwl_rf_fm_160mhz, iwl_killer_be1730x_name, SUBDEV(0x1730)),
+	IWL_DEV_INFO(iwl_rf_fm_160mhz, iwl_killer_be1730x_name, SUBDEV(0x1731)),
 
 /* Killer discrete */
 	IWL_DEV_INFO(iwl_rf_fm, iwl_killer_be1750w_name,
@@ -1069,7 +1073,7 @@ VISIBLE_IF_IWLWIFI_KUNIT const struct iwl_dev_info iwl_dev_info_table[] = {
 
 /* PE RF */
 	IWL_DEV_INFO(iwl_rf_pe, iwl_bn201_name, RF_TYPE(PE)),
-	IWL_DEV_INFO(iwl_rf_pe, iwl_be223_name, RF_TYPE(PE),
+	IWL_DEV_INFO(iwl_rf_pe_no_uhr, iwl_be223_name, RF_TYPE(PE),
 		     SUBDEV_MASKED(0x0524, 0xFFF)),
 	IWL_DEV_INFO(iwl_rf_pe, iwl_bn203_name, RF_TYPE(PE),
 		     SUBDEV_MASKED(0x0324, 0xFFF)),
@@ -1077,6 +1081,8 @@ VISIBLE_IF_IWLWIFI_KUNIT const struct iwl_dev_info iwl_dev_info_table[] = {
 /* Killer */
 	IWL_DEV_INFO(iwl_rf_wh, iwl_killer_be1775s_name, SUBDEV(0x1776)),
 	IWL_DEV_INFO(iwl_rf_wh, iwl_killer_be1775i_name, SUBDEV(0x1775)),
+	IWL_DEV_INFO(iwl_rf_wh_160mhz, iwl_killer_be1735x_name, SUBDEV(0x1735)),
+	IWL_DEV_INFO(iwl_rf_wh_160mhz, iwl_killer_be1735x_name, SUBDEV(0x1736)),
 
 	IWL_DEV_INFO(iwl_rf_pe, iwl_killer_bn1850w2_name, SUBDEV(0x1851)),
 	IWL_DEV_INFO(iwl_rf_pe, iwl_killer_bn1850i_name, SUBDEV(0x1852)),
@@ -1239,8 +1245,12 @@ static int _iwl_pci_resume(struct device *device, bool restore)
 		u32 scratch = iwl_read32(trans, CSR_FUNC_SCRATCH);
 
 		if (!(scratch & CSR_FUNC_SCRATCH_POWER_OFF_MASK) ||
-		    scratch == ~0U)
+		    scratch == ~0U) {
 			device_was_powered_off = true;
+			IWL_DEBUG_WOWLAN(trans,
+					 "Scratch 0x%08x indicates device was powered off\n",
+					 scratch);
+		}
 	} else {
 		/*
 		 * bh are re-enabled by iwl_trans_pcie_release_nic_access,
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h
index 7b7b35e442f9c..abc0c831d1ca7 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */
 /*
- * Copyright (C) 2003-2015, 2018-2025 Intel Corporation
+ * Copyright (C) 2003-2015, 2018-2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -287,6 +287,106 @@ enum iwl_pcie_imr_status {
 	IMR_D2S_ERROR,
 };
 
+/*
+ * The FH will write back to the first TB only, so we need to copy some data
+ * into the buffer regardless of whether it should be mapped or not.
+ * This indicates how big the first TB must be to include the scratch buffer
+ * and the assigned PN.
+ * Since PN location is 8 bytes at offset 12, it's 20 now.
+ * If we make it bigger then allocations will be bigger and copy slower, so
+ * that's probably not useful.
+ */
+#define IWL_FIRST_TB_SIZE	20
+#define IWL_FIRST_TB_SIZE_ALIGN ALIGN(IWL_FIRST_TB_SIZE, 64)
+
+struct iwl_pcie_txq_entry {
+	void *cmd;
+	struct sk_buff *skb;
+	/* buffer to free after command completes */
+	const void *free_buf;
+	struct iwl_cmd_meta meta;
+};
+
+struct iwl_pcie_first_tb_buf {
+	u8 buf[IWL_FIRST_TB_SIZE_ALIGN];
+};
+
+/**
+ * struct iwl_txq - Tx Queue for DMA
+ * @tfds: transmit frame descriptors (DMA memory)
+ * @first_tb_bufs: start of command headers, including scratch buffers, for
+ *	the writeback -- this is DMA memory and an array holding one buffer
+ *	for each command on the queue
+ * @first_tb_dma: DMA address for the first_tb_bufs start
+ * @entries: transmit entries (driver state)
+ * @lock: queue lock
+ * @reclaim_lock: reclaim lock
+ * @stuck_timer: timer that fires if queue gets stuck
+ * @trans: pointer back to transport (for timer)
+ * @need_update: indicates need to update read/write index
+ * @ampdu: true if this queue is an ampdu queue for a specific RA/TID
+ * @wd_timeout: queue watchdog timeout (jiffies) - per queue
+ * @frozen: tx stuck queue timer is frozen
+ * @frozen_expiry_remainder: remember how long until the timer fires
+ * @block: queue is blocked
+ * @bc_tbl: byte count table of the queue (relevant only for gen2 transport)
+ * @write_ptr: 1-st empty entry (index) host_w
+ * @read_ptr: last used entry (index) host_r
+ * @dma_addr: physical addr for BDs
+ * @n_window: safe queue window
+ * @id: queue id
+ * @low_mark: low watermark, resume queue if free space more than this
+ * @high_mark: high watermark, stop queue if free space less than this
+ * @overflow_q: overflow queue for handling frames that didn't fit on HW queue
+ * @overflow_tx: need to transmit from overflow
+ *
+ * A Tx queue consists of circular buffer of BDs (a.k.a. TFDs, transmit frame
+ * descriptors) and required locking structures.
+ *
+ * Note the difference between TFD_QUEUE_SIZE_MAX and n_window: the hardware
+ * always assumes 256 descriptors, so TFD_QUEUE_SIZE_MAX is always 256 (unless
+ * there might be HW changes in the future). For the normal TX
+ * queues, n_window, which is the size of the software queue data
+ * is also 256; however, for the command queue, n_window is only
+ * 32 since we don't need so many commands pending. Since the HW
+ * still uses 256 BDs for DMA though, TFD_QUEUE_SIZE_MAX stays 256.
+ * This means that we end up with the following:
+ *  HW entries: | 0 | ... | N * 32 | ... | N * 32 + 31 | ... | 255 |
+ *  SW entries:           | 0      | ... | 31          |
+ * where N is a number between 0 and 7. This means that the SW
+ * data is a window overlaid over the HW queue.
+ */
+struct iwl_txq {
+	void *tfds;
+	struct iwl_pcie_first_tb_buf *first_tb_bufs;
+	dma_addr_t first_tb_dma;
+	struct iwl_pcie_txq_entry *entries;
+	/* lock for syncing changes on the queue */
+	spinlock_t lock;
+	/* lock to prevent concurrent reclaim */
+	spinlock_t reclaim_lock;
+	unsigned long frozen_expiry_remainder;
+	struct timer_list stuck_timer;
+	struct iwl_trans *trans;
+	bool need_update;
+	bool frozen;
+	bool ampdu;
+	int block;
+	unsigned long wd_timeout;
+	struct sk_buff_head overflow_q;
+	struct iwl_dma_ptr bc_tbl;
+
+	int write_ptr;
+	int read_ptr;
+	dma_addr_t dma_addr;
+	int n_window;
+	u32 id;
+	int low_mark;
+	int high_mark;
+
+	bool overflow_tx;
+};
+
 /**
  * struct iwl_pcie_txqs - TX queues data
  *
@@ -1086,6 +1186,8 @@ u32 iwl_trans_pcie_read_prph(struct iwl_trans *trans, u32 reg);
 void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr, u32 val);
 int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
 			    void *buf, int dwords);
+int iwl_trans_pcie_read_mem_no_grab(struct iwl_trans *trans, u32 addr,
+				    void *buf, u32 dwords);
 int iwl_trans_pcie_sw_reset(struct iwl_trans *trans, bool retake_ownership);
 struct iwl_trans_dump_data *
 iwl_trans_pcie_dump_data(struct iwl_trans *trans, u32 dump_mask,
@@ -1153,6 +1255,9 @@ int iwl_trans_pcie_copy_imr(struct iwl_trans *trans,
 int iwl_trans_pcie_rxq_dma_data(struct iwl_trans *trans, int queue,
 				struct iwl_trans_rxq_dma_data *data);
 
+int iwl_trans_pcie_send_hcmd(struct iwl_trans *trans,
+			     struct iwl_host_cmd *cmd);
+
 static inline bool iwl_pcie_gen1_is_pm_supported(struct iwl_trans *trans)
 {
 	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c
index a50e845cea421..0e324aeb90556 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans-gen2.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
  * Copyright (C) 2017 Intel Deutschland GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #include "iwl-trans.h"
 #include "iwl-prph.h"
@@ -327,6 +327,13 @@ static void iwl_pcie_get_rf_name(struct iwl_trans *trans)
 		else
 			pos = scnprintf(buf, buflen, "WH");
 		break;
+	case CSR_HW_RFID_TYPE(CSR_HW_RF_ID_TYPE_PE):
+		if (SILICON_Z_STEP ==
+		    CSR_HW_RFID_STEP(trans->info.hw_rf_id))
+			pos = scnprintf(buf, buflen, "PETC");
+		else
+			pos = scnprintf(buf, buflen, "PE");
+		break;
 	default:
 		return;
 	}
@@ -398,9 +405,9 @@ void iwl_trans_pcie_gen2_fw_alive(struct iwl_trans *trans)
 	mutex_unlock(&trans_pcie->mutex);
 
 	if (trans->mac_cfg->device_family >= IWL_DEVICE_FAMILY_BZ)
-		trans->step_urm = !!(iwl_read_umac_prph(trans,
-							CNVI_PMU_STEP_FLOW) &
-					CNVI_PMU_STEP_FLOW_FORCE_URM);
+		trans->step_urm = !!(iwl_read_prph(trans,
+						   CNVI_PMU_STEP_FLOW) &
+				     CNVI_PMU_STEP_FLOW_FORCE_URM);
 }
 
 static bool iwl_pcie_set_ltr(struct iwl_trans *trans)
diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c
index a05f60f9224b4..1c4ee76d83877 100644
--- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c
+++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
 /*
- * Copyright (C) 2007-2015, 2018-2024 Intel Corporation
+ * Copyright (C) 2007-2015, 2018-2024, 2026 Intel Corporation
  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
  * Copyright (C) 2016-2017 Intel Deutschland GmbH
  */
@@ -141,10 +141,10 @@ static void iwl_pcie_alloc_fw_monitor_block(struct iwl_trans *trans,
 		return;
 
 	if (power != max_power)
-		IWL_ERR(trans,
-			"Sorry - debug buffer is only %luK while you requested %luK\n",
-			(unsigned long)BIT(power - 10),
-			(unsigned long)BIT(max_power - 10));
+		IWL_INFO(trans,
+			 "Sorry - debug buffer is only %luK while you requested %luK\n",
+			 (unsigned long)BIT(power - 10),
+			 (unsigned long)BIT(max_power - 10));
 
 	fw_mon->block = block;
 	fw_mon->physical = physical;
@@ -2044,7 +2044,7 @@ iwl_trans_pcie_call_prod_reset_dsm(struct pci_dev *pdev, u16 cmd, u16 value)
 						 0xDD, 0x26, 0xB5, 0xFD);
 
 	if (!acpi_check_dsm(ACPI_HANDLE(&pdev->dev), &dsm_guid, ACPI_DSM_REV,
-			    DSM_INTERNAL_FUNC_PRODUCT_RESET))
+			    BIT(DSM_INTERNAL_FUNC_PRODUCT_RESET)))
 		return ERR_PTR(-ENODEV);
 
 	return iwl_acpi_get_dsm_object(&pdev->dev, ACPI_DSM_REV,
@@ -2424,6 +2424,15 @@ bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans)
 	return false;
 }
 
+static void iwl_trans_pcie_resched_with_nic_access(struct iwl_trans *trans)
+{
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+
+	spin_unlock_bh(&trans_pcie->reg_lock);
+	cond_resched();
+	spin_lock_bh(&trans_pcie->reg_lock);
+}
+
 void __releases(nic_access_nobh)
 iwl_trans_pcie_release_nic_access(struct iwl_trans *trans)
 {
@@ -2506,6 +2515,51 @@ int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
 	return 0;
 }
 
+int iwl_trans_pcie_read_mem_no_grab(struct iwl_trans *trans, u32 addr,
+				    void *buf, u32 dwords)
+{
+	struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
+#define IWL_MAX_HW_ERRS 5
+	unsigned int num_consec_hw_errors = 0;
+	u32 offs = 0;
+	u32 *vals = buf;
+
+	lockdep_assert_held(&trans_pcie->reg_lock);
+
+	while (offs < dwords) {
+		/* limit the time we spin here under lock to 1/2s */
+		unsigned long end = jiffies + HZ / 2;
+		bool resched = false;
+
+		iwl_write32(trans, HBUS_TARG_MEM_RADDR,
+			    addr + 4 * offs);
+
+		while (offs < dwords) {
+			vals[offs] = iwl_read32(trans, HBUS_TARG_MEM_RDAT);
+
+			if (iwl_trans_is_hw_error_value(vals[offs]))
+				num_consec_hw_errors++;
+			else
+				num_consec_hw_errors = 0;
+
+			if (num_consec_hw_errors >= IWL_MAX_HW_ERRS)
+				return -EIO;
+
+			offs++;
+
+			if (time_after(jiffies, end)) {
+				resched = true;
+				break;
+			}
+		}
+
+		if (resched)
+			iwl_trans_pcie_resched_with_nic_access(trans);
+	}
+
+	return 0;
+}
+
 int iwl_trans_pcie_read_config32(struct iwl_trans *trans, u32 ofs,
 				 u32 *val)
 {
@@ -3201,8 +3255,7 @@ static ssize_t iwl_dbgfs_reset_write(struct file *file,
 				return -EINVAL;
 			trans->request_top_reset = 1;
 		}
-		iwl_op_mode_nic_error(trans->op_mode, IWL_ERR_TYPE_DEBUGFS);
-		iwl_trans_schedule_reset(trans, IWL_ERR_TYPE_DEBUGFS);
+		iwl_trans_fw_error(trans, IWL_ERR_TYPE_DEBUGFS);
 		return count;
 	}
 
@@ -4000,6 +4053,30 @@ static void get_crf_id(struct iwl_trans *iwl_trans,
 	else
 		sd_reg_ver_addr = SD_REG_VER;
 
+	/* wait until the device is ready to access the prph registers */
+	if (iwl_trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_DR ||
+	    iwl_trans->mac_cfg->device_family == IWL_DEVICE_FAMILY_SC) {
+		u32 req = iwl_read_umac_prph_no_grab(iwl_trans,
+						     WFPM_RSRCS_4PHS_REQ_STTS);
+		int ret;
+
+		if (!(req & RSRC_REQ_CNVR_TOP)) {
+			IWL_ERR(iwl_trans,
+				"WFPM_RSRCS_4PHS_REQ_STTS bit 6 is clear 0x%x\n",
+				req);
+			return;
+		}
+
+		ret = iwl_poll_umac_prph_bits_no_grab(iwl_trans,
+						      WFPM_RSRCS_4PHS_ACK_STTS,
+						      RSRC_ACK_CNVR_TOP,
+						      RSRC_ACK_CNVR_TOP,
+						      50 * 1000);
+		if (ret < 0)
+			IWL_ERR(iwl_trans,
+				"WFPM_RSRCS_4PHS_ACK_STTS bit 6 is clear\n");
+	}
+
 	/* Enable access to peripheral registers */
 	val = iwl_read_umac_prph_no_grab(iwl_trans, WFPM_CTRL_REG);
 	val |= WFPM_AUX_CTL_AUX_IF_MAC_OWNER_MSK;
diff --git a/drivers/net/wireless/intersil/p54/p54spi.c b/drivers/net/wireless/intersil/p54/p54spi.c
index 9d66dcae54e02..d18be25450285 100644
--- a/drivers/net/wireless/intersil/p54/p54spi.c
+++ b/drivers/net/wireless/intersil/p54/p54spi.c
@@ -8,6 +8,7 @@
  */
 
 #include <linux/module.h>
+#include <linux/mod_devicetable.h>
 #include <linux/platform_device.h>
 #include <linux/interrupt.h>
 #include <linux/firmware.h>
@@ -15,7 +16,7 @@
 #include <linux/irq.h>
 #include <linux/spi/spi.h>
 #include <linux/etherdevice.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/slab.h>
 
 #include "p54spi.h"
@@ -30,19 +31,6 @@
 MODULE_FIRMWARE("3826.arm");
 MODULE_FIRMWARE("3826.eeprom");
 
-/* gpios should be handled in board files and provided via platform data,
- * but because it's currently impossible for p54spi to have a header file
- * in include/linux, let's use module parameters for now
- */
-
-static int p54spi_gpio_power = 97;
-module_param(p54spi_gpio_power, int, 0444);
-MODULE_PARM_DESC(p54spi_gpio_power, "gpio number for power line");
-
-static int p54spi_gpio_irq = 87;
-module_param(p54spi_gpio_irq, int, 0444);
-MODULE_PARM_DESC(p54spi_gpio_irq, "gpio number for irq line");
-
 static void p54spi_spi_read(struct p54s_priv *priv, u8 address,
 			      void *buf, size_t len)
 {
@@ -262,14 +250,14 @@ static int p54spi_upload_firmware(struct ieee80211_hw *dev)
 
 static void p54spi_power_off(struct p54s_priv *priv)
 {
-	disable_irq(gpio_to_irq(p54spi_gpio_irq));
-	gpio_set_value(p54spi_gpio_power, 0);
+	disable_irq(priv->irq);
+	gpiod_set_value(priv->gpio_powerdown, 1);
 }
 
 static void p54spi_power_on(struct p54s_priv *priv)
 {
-	gpio_set_value(p54spi_gpio_power, 1);
-	enable_irq(gpio_to_irq(p54spi_gpio_irq));
+	gpiod_set_value(priv->gpio_powerdown, 0);
+	enable_irq(priv->irq);
 
 	/* need to wait a while before device can be accessed, the length
 	 * is just a guess
@@ -608,31 +596,20 @@ static int p54spi_probe(struct spi_device *spi)
 		goto err_free;
 	}
 
-	ret = gpio_request(p54spi_gpio_power, "p54spi power");
-	if (ret < 0) {
-		dev_err(&priv->spi->dev, "power GPIO request failed: %d", ret);
+	priv->gpio_powerdown = gpiod_get_optional(&spi->dev, "powerdown", GPIOD_OUT_HIGH);
+	if (IS_ERR(priv->gpio_powerdown)) {
+		ret = dev_err_probe(&priv->spi->dev, PTR_ERR(priv->gpio_powerdown),
+				    "powerdown GPIO request failed\n");
 		goto err_free;
 	}
 
-	ret = gpio_request(p54spi_gpio_irq, "p54spi irq");
-	if (ret < 0) {
-		dev_err(&priv->spi->dev, "irq GPIO request failed: %d", ret);
-		goto err_free_gpio_power;
-	}
-
-	gpio_direction_output(p54spi_gpio_power, 0);
-	gpio_direction_input(p54spi_gpio_irq);
-
-	ret = request_irq(gpio_to_irq(p54spi_gpio_irq),
-			  p54spi_interrupt, IRQF_NO_AUTOEN, "p54spi",
-			  priv->spi);
+	ret = request_irq(spi->irq, p54spi_interrupt, IRQF_NO_AUTOEN, "p54spi", priv->spi);
 	if (ret < 0) {
 		dev_err(&priv->spi->dev, "request_irq() failed");
-		goto err_free_gpio_irq;
+		goto err_free_gpio_power;
 	}
 
-	irq_set_irq_type(gpio_to_irq(p54spi_gpio_irq), IRQ_TYPE_EDGE_RISING);
-
+	priv->irq = spi->irq;
 	INIT_WORK(&priv->work, p54spi_work);
 	init_completion(&priv->fw_comp);
 	INIT_LIST_HEAD(&priv->tx_pending);
@@ -659,11 +636,9 @@ static int p54spi_probe(struct spi_device *spi)
 
 err_free_common:
 	release_firmware(priv->firmware);
-	free_irq(gpio_to_irq(p54spi_gpio_irq), spi);
-err_free_gpio_irq:
-	gpio_free(p54spi_gpio_irq);
+	free_irq(priv->irq, spi);
 err_free_gpio_power:
-	gpio_free(p54spi_gpio_power);
+	gpiod_put(priv->gpio_powerdown);
 err_free:
 	p54_free_common(priv->hw);
 	return ret;
@@ -675,10 +650,8 @@ static void p54spi_remove(struct spi_device *spi)
 
 	p54_unregister_common(priv->hw);
 
-	free_irq(gpio_to_irq(p54spi_gpio_irq), spi);
-
-	gpio_free(p54spi_gpio_power);
-	gpio_free(p54spi_gpio_irq);
+	free_irq(priv->irq, spi);
+	gpiod_put(priv->gpio_powerdown);
 	release_firmware(priv->firmware);
 
 	mutex_destroy(&priv->mutex);
@@ -686,10 +659,16 @@ static void p54spi_remove(struct spi_device *spi)
 	p54_free_common(priv->hw);
 }
 
+static const struct of_device_id p54spi_of_ids[] = {
+	{ .compatible = "st,stlc4560", },
+	{ },
+};
+MODULE_DEVICE_TABLE(of, p54spi_of_ids);
 
 static struct spi_driver p54spi_driver = {
 	.driver = {
 		.name		= "p54spi",
+		.of_match_table = p54spi_of_ids,
 	},
 
 	.probe		= p54spi_probe,
diff --git a/drivers/net/wireless/intersil/p54/p54spi.h b/drivers/net/wireless/intersil/p54/p54spi.h
index e5619a13fd61d..118785cc635aa 100644
--- a/drivers/net/wireless/intersil/p54/p54spi.h
+++ b/drivers/net/wireless/intersil/p54/p54spi.h
@@ -107,6 +107,9 @@ struct p54s_priv {
 
 	enum fw_state fw_state;
 	const struct firmware *firmware;
+
+	struct gpio_desc *gpio_powerdown;
+	int irq;
 };
 
 #endif /* P54SPI_H */
diff --git a/drivers/net/wireless/marvell/mwifiex/cfg80211.c b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
index c9a651bdf8822..c9daf893472fe 100644
--- a/drivers/net/wireless/marvell/mwifiex/cfg80211.c
+++ b/drivers/net/wireless/marvell/mwifiex/cfg80211.c
@@ -304,7 +304,8 @@ static int
 mwifiex_cfg80211_remain_on_channel(struct wiphy *wiphy,
 				   struct wireless_dev *wdev,
 				   struct ieee80211_channel *chan,
-				   unsigned int duration, u64 *cookie)
+				   unsigned int duration, u64 *cookie,
+				   const u8 *rx_addr)
 {
 	struct mwifiex_private *priv = mwifiex_netdev_get_priv(wdev->netdev);
 	int ret;
diff --git a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c
index 37cdf3e8a0670..22bad3cba8dfe 100644
--- a/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c
+++ b/drivers/net/wireless/mediatek/mt76/mt7925/mcu.c
@@ -1880,8 +1880,8 @@ mt7925_mcu_sta_eht_mld_tlv(struct sk_buff *skb,
 
 	eml_cap = (vif->cfg.eml_cap & (IEEE80211_EML_CAP_EMLSR_SUPP |
 				       IEEE80211_EML_CAP_TRANSITION_TIMEOUT)) |
-		  (ext_capa->eml_capabilities & (IEEE80211_EML_CAP_EMLSR_PADDING_DELAY |
-						IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY));
+		  (ext_capa->eml_capabilities & (IEEE80211_EML_CAP_EML_PADDING_DELAY |
+						IEEE80211_EML_CAP_EML_TRANSITION_DELAY));
 
 	if (eml_cap & IEEE80211_EML_CAP_EMLSR_SUPP) {
 		eht_mld->eml_cap[0] = u16_get_bits(eml_cap, GENMASK(7, 0));
diff --git a/drivers/net/wireless/microchip/wilc1000/cfg80211.c b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
index 3a774cc44b263..6654fce4ded84 100644
--- a/drivers/net/wireless/microchip/wilc1000/cfg80211.c
+++ b/drivers/net/wireless/microchip/wilc1000/cfg80211.c
@@ -1100,7 +1100,8 @@ static void wilc_wfi_remain_on_channel_expired(struct wilc_vif *vif, u64 cookie)
 static int remain_on_channel(struct wiphy *wiphy,
 			     struct wireless_dev *wdev,
 			     struct ieee80211_channel *chan,
-			     unsigned int duration, u64 *cookie)
+			     unsigned int duration, u64 *cookie,
+			     const u8 *rx_addr)
 {
 	int ret = 0;
 	struct wilc_vif *vif = netdev_priv(wdev->netdev);
diff --git a/drivers/net/wireless/microchip/wilc1000/wlan.c b/drivers/net/wireless/microchip/wilc1000/wlan.c
index 3fa8592eb2503..4b116fe6f9ea9 100644
--- a/drivers/net/wireless/microchip/wilc1000/wlan.c
+++ b/drivers/net/wireless/microchip/wilc1000/wlan.c
@@ -1265,7 +1265,7 @@ int wilc_wlan_firmware_download(struct wilc *wilc, const u8 *buffer,
 
 	ret = acquire_bus(wilc, WILC_BUS_ACQUIRE_AND_WAKEUP);
 	if (ret)
-		return ret;
+		goto fail;
 
 	wilc->hif_func->hif_read_reg(wilc, WILC_GLB_RESET_0, &reg);
 	reg &= ~BIT(10);
diff --git a/drivers/net/wireless/purelifi/plfxlc/usb.c b/drivers/net/wireless/purelifi/plfxlc/usb.c
index 5724ec173e642..6d24086eb8b7a 100644
--- a/drivers/net/wireless/purelifi/plfxlc/usb.c
+++ b/drivers/net/wireless/purelifi/plfxlc/usb.c
@@ -851,7 +851,7 @@ static int resume(struct usb_interface *interface)
 
 #endif
 
-static struct usb_driver driver = {
+static struct usb_driver usbdriver = {
 	.name = KBUILD_MODNAME,
 	.id_table = usb_ids,
 	.probe = probe,
@@ -865,25 +865,7 @@ static struct usb_driver driver = {
 	.disable_hub_initiated_lpm = 1,
 };
 
-static int __init usb_init(void)
-{
-	int r;
-
-	r = usb_register(&driver);
-	if (r) {
-		pr_err("%s usb_register() failed %d\n", driver.name, r);
-		return r;
-	}
-
-	pr_debug("Driver initialized :%s\n", driver.name);
-	return 0;
-}
-
-static void __exit usb_exit(void)
-{
-	usb_deregister(&driver);
-	pr_debug("%s %s\n", driver.name, __func__);
-}
+module_usb_driver(usbdriver);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("USB driver for pureLiFi devices");
@@ -891,6 +873,3 @@ MODULE_AUTHOR("pureLiFi");
 MODULE_VERSION("1.0");
 MODULE_FIRMWARE("plfxlc/lifi-x.bin");
 MODULE_DEVICE_TABLE(usb, usb_ids);
-
-module_init(usb_init);
-module_exit(usb_exit);
diff --git a/drivers/net/wireless/quantenna/qtnfmac/commands.c b/drivers/net/wireless/quantenna/qtnfmac/commands.c
index d9c4dd6beaecf..77929ed9ef783 100644
--- a/drivers/net/wireless/quantenna/qtnfmac/commands.c
+++ b/drivers/net/wireless/quantenna/qtnfmac/commands.c
@@ -261,8 +261,8 @@ int qtnf_cmd_send_start_ap(struct qtnf_vif *vif,
 	cmd->p2p_ctwindow = s->p2p_ctwindow;
 	cmd->p2p_opp_ps = s->p2p_opp_ps;
 	cmd->pbss = s->pbss;
-	cmd->ht_required = s->ht_required;
-	cmd->vht_required = s->vht_required;
+	cmd->ht_required = s->beacon.ht_required;
+	cmd->vht_required = s->beacon.vht_required;
 	cmd->twt_responder = s->twt_responder;
 	if (s->he_obss_pd.enable) {
 		cmd->sr_params.sr_control |= QLINK_SR_SRG_INFORMATION_PRESENT;
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00.h b/drivers/net/wireless/ralink/rt2x00/rt2x00.h
index 665887e9b118b..7d313e86d3f20 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00.h
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00.h
@@ -1009,11 +1009,12 @@ struct rt2x00_dev {
 	/* Extra TX headroom required for alignment purposes. */
 	unsigned int extra_tx_headroom;
 
-	struct usb_anchor *anchor;
 	unsigned int num_proto_errs;
 
 	/* Clock for System On Chip devices. */
 	struct clk *clk;
+
+	struct usb_anchor anchor[];
 };
 
 struct rt2x00_bar_list_entry {
diff --git a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
index 174d89b0b1d77..47e427ea86221 100644
--- a/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
+++ b/drivers/net/wireless/ralink/rt2x00/rt2x00usb.c
@@ -804,7 +804,7 @@ int rt2x00usb_probe(struct usb_interface *usb_intf,
 
 	usb_reset_device(usb_dev);
 
-	hw = ieee80211_alloc_hw(sizeof(struct rt2x00_dev), ops->hw);
+	hw = ieee80211_alloc_hw(struct_size(rt2x00dev, anchor, 1), ops->hw);
 	if (!hw) {
 		rt2x00_probe_err("Failed to allocate hardware\n");
 		return -ENOMEM;
@@ -826,13 +826,6 @@ int rt2x00usb_probe(struct usb_interface *usb_intf,
 	if (retval)
 		goto exit_free_device;
 
-	rt2x00dev->anchor = devm_kmalloc(&usb_intf->dev,
-					sizeof(struct usb_anchor),
-					GFP_KERNEL);
-	if (!rt2x00dev->anchor) {
-		retval = -ENOMEM;
-		goto exit_free_reg;
-	}
 	init_usb_anchor(rt2x00dev->anchor);
 
 	retval = rt2x00lib_probe_dev(rt2x00dev);
@@ -843,8 +836,6 @@ int rt2x00usb_probe(struct usb_interface *usb_intf,
 
 exit_free_anchor:
 	usb_kill_anchored_urbs(rt2x00dev->anchor);
-
-exit_free_reg:
 	rt2x00usb_free_reg(rt2x00dev);
 
 exit_free_device:
diff --git a/drivers/net/wireless/realtek/rtw89/fw.c b/drivers/net/wireless/realtek/rtw89/fw.c
index 17704f054727a..ffa04315a62b6 100644
--- a/drivers/net/wireless/realtek/rtw89/fw.c
+++ b/drivers/net/wireless/realtek/rtw89/fw.c
@@ -5050,7 +5050,7 @@ int rtw89_fw_h2c_join_info(struct rtw89_dev *rtwdev, struct rtw89_vif_link *rtwv
 		     le32_encode_bits(0, RTW89_H2C_JOININFO_W1_EMLSR_CAB) |
 		     le32_encode_bits(0, RTW89_H2C_JOININFO_W1_NSTR_EN) |
 		     le32_encode_bits(init_ps, RTW89_H2C_JOININFO_W1_INIT_PWR_STATE) |
-		     le32_encode_bits(IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US,
+		     le32_encode_bits(IEEE80211_EML_CAP_EML_PADDING_DELAY_256US,
 				      RTW89_H2C_JOININFO_W1_EMLSR_PADDING) |
 		     le32_encode_bits(IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US,
 				      RTW89_H2C_JOININFO_W1_EMLSR_TRANS_DELAY) |
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c b/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c
index 4633994133185..8679b21fd3fd7 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852a_rfk.c
@@ -334,8 +334,8 @@ static void _check_addc(struct rtw89_dev *rtwdev, enum rtw89_rf_path path)
 
 	for (i = 0; i < ADDC_T_AVG; i++) {
 		tmp = rtw89_phy_read32_mask(rtwdev, R_DBG32_D, MASKDWORD);
-		dc_re += sign_extend32(FIELD_GET(0xfff000, tmp), 11);
-		dc_im += sign_extend32(FIELD_GET(0xfff, tmp), 11);
+		dc_re += FIELD_GET_SIGNED(0xfff000, tmp);
+		dc_im += FIELD_GET_SIGNED(0xfff, tmp);
 	}
 
 	dc_re /= ADDC_T_AVG;
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b_common.c b/drivers/net/wireless/realtek/rtw89/rtw8852b_common.c
index 65b839323e3e3..df5fbae50ff55 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852b_common.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852b_common.c
@@ -206,9 +206,9 @@ static void rtw8852bx_efuse_parsing_tssi(struct rtw89_dev *rtwdev,
 static bool _decode_efuse_gain(u8 data, s8 *high, s8 *low)
 {
 	if (high)
-		*high = sign_extend32(FIELD_GET(GENMASK(7,  4), data), 3);
+		*high = FIELD_GET_SIGNED(GENMASK(7,  4), data);
 	if (low)
-		*low = sign_extend32(FIELD_GET(GENMASK(3,  0), data), 3);
+		*low = FIELD_GET_SIGNED(GENMASK(3,  0), data);
 
 	return data != 0xff;
 }
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852b_rfk.c b/drivers/net/wireless/realtek/rtw89/rtw8852b_rfk.c
index 70b1515c00fa5..8db6ea4751289 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852b_rfk.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852b_rfk.c
@@ -497,8 +497,8 @@ static void _check_addc(struct rtw89_dev *rtwdev, enum rtw89_rf_path path)
 
 	for (i = 0; i < ADDC_T_AVG; i++) {
 		tmp = rtw89_phy_read32_mask(rtwdev, R_DBG32_D, MASKDWORD);
-		dc_re += sign_extend32(FIELD_GET(0xfff000, tmp), 11);
-		dc_im += sign_extend32(FIELD_GET(0xfff, tmp), 11);
+		dc_re += FIELD_GET_SIGNED(0xfff000, tmp);
+		dc_im += FIELD_GET_SIGNED(0xfff, tmp);
 	}
 
 	dc_re /= ADDC_T_AVG;
diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852c.c b/drivers/net/wireless/realtek/rtw89/rtw8852c.c
index 40db7e3c0d974..32eecd184b7c7 100644
--- a/drivers/net/wireless/realtek/rtw89/rtw8852c.c
+++ b/drivers/net/wireless/realtek/rtw89/rtw8852c.c
@@ -517,9 +517,9 @@ static void rtw8852c_efuse_parsing_tssi(struct rtw89_dev *rtwdev,
 static bool _decode_efuse_gain(u8 data, s8 *high, s8 *low)
 {
 	if (high)
-		*high = sign_extend32(FIELD_GET(GENMASK(7,  4), data), 3);
+		*high = FIELD_GET_SIGNED(GENMASK(7, 4), data);
 	if (low)
-		*low = sign_extend32(FIELD_GET(GENMASK(3,  0), data), 3);
+		*low = FIELD_GET_SIGNED(GENMASK(3, 0), data);
 
 	return data != 0xff;
 }
diff --git a/drivers/net/wireless/virtual/Makefile b/drivers/net/wireless/virtual/Makefile
index 5773cc6d643ec..6ad860dd76439 100644
--- a/drivers/net/wireless/virtual/Makefile
+++ b/drivers/net/wireless/virtual/Makefile
@@ -1,3 +1,5 @@
 obj-$(CONFIG_MAC80211_HWSIM)	+= mac80211_hwsim.o
+mac80211_hwsim-objs		+= mac80211_hwsim_main.o
+mac80211_hwsim-objs		+= mac80211_hwsim_nan.o
 
 obj-$(CONFIG_VIRT_WIFI)	+= virt_wifi.o
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim.c b/drivers/net/wireless/virtual/mac80211_hwsim.c
deleted file mode 100644
index 1fcf5d0d2e13f..0000000000000
--- a/drivers/net/wireless/virtual/mac80211_hwsim.c
+++ /dev/null
@@ -1,7454 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * mac80211_hwsim - software simulator of 802.11 radio(s) for mac80211
- * Copyright (c) 2008, Jouni Malinen <j@w1.fi>
- * Copyright (c) 2011, Javier Lopez <jlopex@gmail.com>
- * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
- * Copyright (C) 2018 - 2025 Intel Corporation
- */
-
-/*
- * TODO:
- * - Add TSF sync and fix IBSS beacon transmission by adding
- *   competition for "air time" at TBTT
- * - RX filtering based on filter configuration (data->rx_filter)
- */
-
-#include <linux/list.h>
-#include <linux/slab.h>
-#include <linux/spinlock.h>
-#include <net/dst.h>
-#include <net/xfrm.h>
-#include <net/mac80211.h>
-#include <net/ieee80211_radiotap.h>
-#include <linux/if_arp.h>
-#include <linux/rtnetlink.h>
-#include <linux/etherdevice.h>
-#include <linux/platform_device.h>
-#include <linux/debugfs.h>
-#include <linux/module.h>
-#include <linux/ktime.h>
-#include <net/genetlink.h>
-#include <net/net_namespace.h>
-#include <net/netns/generic.h>
-#include <linux/rhashtable.h>
-#include <linux/nospec.h>
-#include <linux/virtio.h>
-#include <linux/virtio_ids.h>
-#include <linux/virtio_config.h>
-#include <linux/uaccess.h>
-#include <linux/string.h>
-#include "mac80211_hwsim.h"
-
-#define WARN_QUEUE 100
-#define MAX_QUEUE 200
-
-MODULE_AUTHOR("Jouni Malinen");
-MODULE_DESCRIPTION("Software simulator of 802.11 radio(s) for mac80211");
-MODULE_LICENSE("GPL");
-
-static int radios = 2;
-module_param(radios, int, 0444);
-MODULE_PARM_DESC(radios, "Number of simulated radios");
-
-static int channels = 1;
-module_param(channels, int, 0444);
-MODULE_PARM_DESC(channels, "Number of concurrent channels");
-
-static bool paged_rx = false;
-module_param(paged_rx, bool, 0644);
-MODULE_PARM_DESC(paged_rx, "Use paged SKBs for RX instead of linear ones");
-
-static bool rctbl = false;
-module_param(rctbl, bool, 0444);
-MODULE_PARM_DESC(rctbl, "Handle rate control table");
-
-static bool support_p2p_device = true;
-module_param(support_p2p_device, bool, 0444);
-MODULE_PARM_DESC(support_p2p_device, "Support P2P-Device interface type");
-
-static bool mlo;
-module_param(mlo, bool, 0444);
-MODULE_PARM_DESC(mlo, "Support MLO");
-
-static bool multi_radio;
-module_param(multi_radio, bool, 0444);
-MODULE_PARM_DESC(multi_radio, "Support Multiple Radios per wiphy");
-
-/**
- * enum hwsim_regtest - the type of regulatory tests we offer
- *
- * @HWSIM_REGTEST_DISABLED: No regulatory tests are performed,
- * 	this is the default value.
- * @HWSIM_REGTEST_DRIVER_REG_FOLLOW: Used for testing the driver regulatory
- *	hint, only one driver regulatory hint will be sent as such the
- * 	secondary radios are expected to follow.
- * @HWSIM_REGTEST_DRIVER_REG_ALL: Used for testing the driver regulatory
- * 	request with all radios reporting the same regulatory domain.
- * @HWSIM_REGTEST_DIFF_COUNTRY: Used for testing the drivers calling
- * 	different regulatory domains requests. Expected behaviour is for
- * 	an intersection to occur but each device will still use their
- * 	respective regulatory requested domains. Subsequent radios will
- * 	use the resulting intersection.
- * @HWSIM_REGTEST_WORLD_ROAM: Used for testing the world roaming. We accomplish
- *	this by using a custom beacon-capable regulatory domain for the first
- *	radio. All other device world roam.
- * @HWSIM_REGTEST_CUSTOM_WORLD: Used for testing the custom world regulatory
- * 	domain requests. All radios will adhere to this custom world regulatory
- * 	domain.
- * @HWSIM_REGTEST_CUSTOM_WORLD_2: Used for testing 2 custom world regulatory
- * 	domain requests. The first radio will adhere to the first custom world
- * 	regulatory domain, the second one to the second custom world regulatory
- * 	domain. All other devices will world roam.
- * @HWSIM_REGTEST_STRICT_FOLLOW: Used for testing strict regulatory domain
- *	settings, only the first radio will send a regulatory domain request
- *	and use strict settings. The rest of the radios are expected to follow.
- * @HWSIM_REGTEST_STRICT_ALL: Used for testing strict regulatory domain
- *	settings. All radios will adhere to this.
- * @HWSIM_REGTEST_STRICT_AND_DRIVER_REG: Used for testing strict regulatory
- *	domain settings, combined with secondary driver regulatory domain
- *	settings. The first radio will get a strict regulatory domain setting
- *	using the first driver regulatory request and the second radio will use
- *	non-strict settings using the second driver regulatory request. All
- *	other devices should follow the intersection created between the
- *	first two.
- * @HWSIM_REGTEST_ALL: Used for testing every possible mix. You will need
- * 	at least 6 radios for a complete test. We will test in this order:
- * 	1 - driver custom world regulatory domain
- * 	2 - second custom world regulatory domain
- * 	3 - first driver regulatory domain request
- * 	4 - second driver regulatory domain request
- * 	5 - strict regulatory domain settings using the third driver regulatory
- * 	    domain request
- * 	6 and on - should follow the intersection of the 3rd, 4rth and 5th radio
- * 	           regulatory requests.
- *
- * These are the different values you can use for the regtest
- * module parameter. This is useful to help test world roaming
- * and the driver regulatory_hint() call and combinations of these.
- * If you want to do specific alpha2 regulatory domain tests simply
- * use the userspace regulatory request as that will be respected as
- * well without the need of this module parameter. This is designed
- * only for testing the driver regulatory request, world roaming
- * and all possible combinations.
- */
-enum hwsim_regtest {
-	HWSIM_REGTEST_DISABLED = 0,
-	HWSIM_REGTEST_DRIVER_REG_FOLLOW = 1,
-	HWSIM_REGTEST_DRIVER_REG_ALL = 2,
-	HWSIM_REGTEST_DIFF_COUNTRY = 3,
-	HWSIM_REGTEST_WORLD_ROAM = 4,
-	HWSIM_REGTEST_CUSTOM_WORLD = 5,
-	HWSIM_REGTEST_CUSTOM_WORLD_2 = 6,
-	HWSIM_REGTEST_STRICT_FOLLOW = 7,
-	HWSIM_REGTEST_STRICT_ALL = 8,
-	HWSIM_REGTEST_STRICT_AND_DRIVER_REG = 9,
-	HWSIM_REGTEST_ALL = 10,
-};
-
-/* Set to one of the HWSIM_REGTEST_* values above */
-static int regtest = HWSIM_REGTEST_DISABLED;
-module_param(regtest, int, 0444);
-MODULE_PARM_DESC(regtest, "The type of regulatory test we want to run");
-
-static const char *hwsim_alpha2s[] = {
-	"FI",
-	"AL",
-	"US",
-	"DE",
-	"JP",
-	"AL",
-};
-
-static const struct ieee80211_regdomain hwsim_world_regdom_custom_01 = {
-	.n_reg_rules = 5,
-	.alpha2 =  "99",
-	.reg_rules = {
-		REG_RULE(2412-10, 2462+10, 40, 0, 20, 0),
-		REG_RULE(2484-10, 2484+10, 40, 0, 20, 0),
-		REG_RULE(5150-10, 5240+10, 40, 0, 30, 0),
-		REG_RULE(5745-10, 5825+10, 40, 0, 30, 0),
-		REG_RULE(5855-10, 5925+10, 40, 0, 33, 0),
-	}
-};
-
-static const struct ieee80211_regdomain hwsim_world_regdom_custom_02 = {
-	.n_reg_rules = 3,
-	.alpha2 =  "99",
-	.reg_rules = {
-		REG_RULE(2412-10, 2462+10, 40, 0, 20, 0),
-		REG_RULE(5725-10, 5850+10, 40, 0, 30,
-			 NL80211_RRF_NO_IR),
-		REG_RULE(5855-10, 5925+10, 40, 0, 33, 0),
-	}
-};
-
-static const struct ieee80211_regdomain hwsim_world_regdom_custom_03 = {
-	.n_reg_rules = 6,
-	.alpha2 =  "99",
-	.reg_rules = {
-		REG_RULE(2412 - 10, 2462 + 10, 40, 0, 20, 0),
-		REG_RULE(2484 - 10, 2484 + 10, 40, 0, 20, 0),
-		REG_RULE(5150 - 10, 5240 + 10, 40, 0, 30, 0),
-		REG_RULE(5745 - 10, 5825 + 10, 40, 0, 30, 0),
-		REG_RULE(5855 - 10, 5925 + 10, 40, 0, 33, 0),
-		REG_RULE(5955 - 10, 7125 + 10, 320, 0, 33, 0),
-	}
-};
-
-static const struct ieee80211_regdomain hwsim_world_regdom_custom_04 = {
-	.n_reg_rules = 6,
-	.alpha2 =  "99",
-	.reg_rules = {
-		REG_RULE(2412 - 10, 2462 + 10, 40, 0, 20, 0),
-		REG_RULE(2484 - 10, 2484 + 10, 40, 0, 20, 0),
-		REG_RULE(5150 - 10, 5240 + 10, 80, 0, 30, NL80211_RRF_AUTO_BW),
-		REG_RULE(5260 - 10, 5320 + 10, 80, 0, 30,
-			 NL80211_RRF_DFS_CONCURRENT | NL80211_RRF_DFS |
-			 NL80211_RRF_AUTO_BW),
-		REG_RULE(5500 - 10, 5720 + 10, 160, 0, 30,
-			 NL80211_RRF_DFS_CONCURRENT | NL80211_RRF_DFS),
-		REG_RULE(5745 - 10, 5825 + 10, 80, 0, 30, 0),
-		REG_RULE(5855 - 10, 5925 + 10, 80, 0, 33, 0),
-	}
-};
-
-static const struct ieee80211_regdomain *hwsim_world_regdom_custom[] = {
-	&hwsim_world_regdom_custom_01,
-	&hwsim_world_regdom_custom_02,
-	&hwsim_world_regdom_custom_03,
-	&hwsim_world_regdom_custom_04,
-};
-
-struct hwsim_vif_priv {
-	u32 magic;
-	u32 skip_beacons[IEEE80211_MLD_MAX_NUM_LINKS];
-	u8 bssid[ETH_ALEN];
-	bool assoc;
-	bool bcn_en;
-	u16 aid;
-};
-
-#define HWSIM_VIF_MAGIC	0x69537748
-
-static inline void hwsim_check_magic(struct ieee80211_vif *vif)
-{
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-	WARN(vp->magic != HWSIM_VIF_MAGIC,
-	     "Invalid VIF (%p) magic %#x, %pM, %d/%d\n",
-	     vif, vp->magic, vif->addr, vif->type, vif->p2p);
-}
-
-static inline void hwsim_set_magic(struct ieee80211_vif *vif)
-{
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-	vp->magic = HWSIM_VIF_MAGIC;
-}
-
-static inline void hwsim_clear_magic(struct ieee80211_vif *vif)
-{
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-	vp->magic = 0;
-}
-
-struct hwsim_sta_priv {
-	u32 magic;
-	unsigned int last_link;
-	u16 active_links_rx;
-};
-
-#define HWSIM_STA_MAGIC	0x6d537749
-
-static inline void hwsim_check_sta_magic(struct ieee80211_sta *sta)
-{
-	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
-	WARN_ON(sp->magic != HWSIM_STA_MAGIC);
-}
-
-static inline void hwsim_set_sta_magic(struct ieee80211_sta *sta)
-{
-	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
-	sp->magic = HWSIM_STA_MAGIC;
-}
-
-static inline void hwsim_clear_sta_magic(struct ieee80211_sta *sta)
-{
-	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
-	sp->magic = 0;
-}
-
-struct hwsim_chanctx_priv {
-	u32 magic;
-};
-
-#define HWSIM_CHANCTX_MAGIC 0x6d53774a
-
-static inline void hwsim_check_chanctx_magic(struct ieee80211_chanctx_conf *c)
-{
-	struct hwsim_chanctx_priv *cp = (void *)c->drv_priv;
-	WARN_ON(cp->magic != HWSIM_CHANCTX_MAGIC);
-}
-
-static inline void hwsim_set_chanctx_magic(struct ieee80211_chanctx_conf *c)
-{
-	struct hwsim_chanctx_priv *cp = (void *)c->drv_priv;
-	cp->magic = HWSIM_CHANCTX_MAGIC;
-}
-
-static inline void hwsim_clear_chanctx_magic(struct ieee80211_chanctx_conf *c)
-{
-	struct hwsim_chanctx_priv *cp = (void *)c->drv_priv;
-	cp->magic = 0;
-}
-
-static unsigned int hwsim_net_id;
-
-static DEFINE_IDA(hwsim_netgroup_ida);
-
-struct hwsim_net {
-	int netgroup;
-	u32 wmediumd;
-};
-
-static inline int hwsim_net_get_netgroup(struct net *net)
-{
-	struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
-
-	return hwsim_net->netgroup;
-}
-
-static inline int hwsim_net_set_netgroup(struct net *net)
-{
-	struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
-
-	hwsim_net->netgroup = ida_alloc(&hwsim_netgroup_ida, GFP_KERNEL);
-	return hwsim_net->netgroup >= 0 ? 0 : -ENOMEM;
-}
-
-static inline u32 hwsim_net_get_wmediumd(struct net *net)
-{
-	struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
-
-	return hwsim_net->wmediumd;
-}
-
-static inline void hwsim_net_set_wmediumd(struct net *net, u32 portid)
-{
-	struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
-
-	hwsim_net->wmediumd = portid;
-}
-
-static const struct class hwsim_class = {
-	.name	= "mac80211_hwsim"
-};
-
-static struct net_device *hwsim_mon; /* global monitor netdev */
-
-#define CHAN2G(_freq)  { \
-	.band = NL80211_BAND_2GHZ, \
-	.center_freq = (_freq), \
-	.hw_value = (_freq), \
-}
-
-#define CHAN5G(_freq) { \
-	.band = NL80211_BAND_5GHZ, \
-	.center_freq = (_freq), \
-	.hw_value = (_freq), \
-}
-
-#define CHAN6G(_freq) { \
-	.band = NL80211_BAND_6GHZ, \
-	.center_freq = (_freq), \
-	.hw_value = (_freq), \
-}
-
-static const struct ieee80211_channel hwsim_channels_2ghz[] = {
-	CHAN2G(2412), /* Channel 1 */
-	CHAN2G(2417), /* Channel 2 */
-	CHAN2G(2422), /* Channel 3 */
-	CHAN2G(2427), /* Channel 4 */
-	CHAN2G(2432), /* Channel 5 */
-	CHAN2G(2437), /* Channel 6 */
-	CHAN2G(2442), /* Channel 7 */
-	CHAN2G(2447), /* Channel 8 */
-	CHAN2G(2452), /* Channel 9 */
-	CHAN2G(2457), /* Channel 10 */
-	CHAN2G(2462), /* Channel 11 */
-	CHAN2G(2467), /* Channel 12 */
-	CHAN2G(2472), /* Channel 13 */
-	CHAN2G(2484), /* Channel 14 */
-};
-
-static const struct ieee80211_channel hwsim_channels_5ghz[] = {
-	CHAN5G(5180), /* Channel 36 */
-	CHAN5G(5200), /* Channel 40 */
-	CHAN5G(5220), /* Channel 44 */
-	CHAN5G(5240), /* Channel 48 */
-
-	CHAN5G(5260), /* Channel 52 */
-	CHAN5G(5280), /* Channel 56 */
-	CHAN5G(5300), /* Channel 60 */
-	CHAN5G(5320), /* Channel 64 */
-
-	CHAN5G(5500), /* Channel 100 */
-	CHAN5G(5520), /* Channel 104 */
-	CHAN5G(5540), /* Channel 108 */
-	CHAN5G(5560), /* Channel 112 */
-	CHAN5G(5580), /* Channel 116 */
-	CHAN5G(5600), /* Channel 120 */
-	CHAN5G(5620), /* Channel 124 */
-	CHAN5G(5640), /* Channel 128 */
-	CHAN5G(5660), /* Channel 132 */
-	CHAN5G(5680), /* Channel 136 */
-	CHAN5G(5700), /* Channel 140 */
-
-	CHAN5G(5745), /* Channel 149 */
-	CHAN5G(5765), /* Channel 153 */
-	CHAN5G(5785), /* Channel 157 */
-	CHAN5G(5805), /* Channel 161 */
-	CHAN5G(5825), /* Channel 165 */
-	CHAN5G(5845), /* Channel 169 */
-
-	CHAN5G(5855), /* Channel 171 */
-	CHAN5G(5860), /* Channel 172 */
-	CHAN5G(5865), /* Channel 173 */
-	CHAN5G(5870), /* Channel 174 */
-
-	CHAN5G(5875), /* Channel 175 */
-	CHAN5G(5880), /* Channel 176 */
-	CHAN5G(5885), /* Channel 177 */
-	CHAN5G(5890), /* Channel 178 */
-	CHAN5G(5895), /* Channel 179 */
-	CHAN5G(5900), /* Channel 180 */
-	CHAN5G(5905), /* Channel 181 */
-
-	CHAN5G(5910), /* Channel 182 */
-	CHAN5G(5915), /* Channel 183 */
-	CHAN5G(5920), /* Channel 184 */
-	CHAN5G(5925), /* Channel 185 */
-};
-
-static const struct ieee80211_channel hwsim_channels_6ghz[] = {
-	CHAN6G(5955), /* Channel 1 */
-	CHAN6G(5975), /* Channel 5 */
-	CHAN6G(5995), /* Channel 9 */
-	CHAN6G(6015), /* Channel 13 */
-	CHAN6G(6035), /* Channel 17 */
-	CHAN6G(6055), /* Channel 21 */
-	CHAN6G(6075), /* Channel 25 */
-	CHAN6G(6095), /* Channel 29 */
-	CHAN6G(6115), /* Channel 33 */
-	CHAN6G(6135), /* Channel 37 */
-	CHAN6G(6155), /* Channel 41 */
-	CHAN6G(6175), /* Channel 45 */
-	CHAN6G(6195), /* Channel 49 */
-	CHAN6G(6215), /* Channel 53 */
-	CHAN6G(6235), /* Channel 57 */
-	CHAN6G(6255), /* Channel 61 */
-	CHAN6G(6275), /* Channel 65 */
-	CHAN6G(6295), /* Channel 69 */
-	CHAN6G(6315), /* Channel 73 */
-	CHAN6G(6335), /* Channel 77 */
-	CHAN6G(6355), /* Channel 81 */
-	CHAN6G(6375), /* Channel 85 */
-	CHAN6G(6395), /* Channel 89 */
-	CHAN6G(6415), /* Channel 93 */
-	CHAN6G(6435), /* Channel 97 */
-	CHAN6G(6455), /* Channel 181 */
-	CHAN6G(6475), /* Channel 105 */
-	CHAN6G(6495), /* Channel 109 */
-	CHAN6G(6515), /* Channel 113 */
-	CHAN6G(6535), /* Channel 117 */
-	CHAN6G(6555), /* Channel 121 */
-	CHAN6G(6575), /* Channel 125 */
-	CHAN6G(6595), /* Channel 129 */
-	CHAN6G(6615), /* Channel 133 */
-	CHAN6G(6635), /* Channel 137 */
-	CHAN6G(6655), /* Channel 141 */
-	CHAN6G(6675), /* Channel 145 */
-	CHAN6G(6695), /* Channel 149 */
-	CHAN6G(6715), /* Channel 153 */
-	CHAN6G(6735), /* Channel 157 */
-	CHAN6G(6755), /* Channel 161 */
-	CHAN6G(6775), /* Channel 165 */
-	CHAN6G(6795), /* Channel 169 */
-	CHAN6G(6815), /* Channel 173 */
-	CHAN6G(6835), /* Channel 177 */
-	CHAN6G(6855), /* Channel 181 */
-	CHAN6G(6875), /* Channel 185 */
-	CHAN6G(6895), /* Channel 189 */
-	CHAN6G(6915), /* Channel 193 */
-	CHAN6G(6935), /* Channel 197 */
-	CHAN6G(6955), /* Channel 201 */
-	CHAN6G(6975), /* Channel 205 */
-	CHAN6G(6995), /* Channel 209 */
-	CHAN6G(7015), /* Channel 213 */
-	CHAN6G(7035), /* Channel 217 */
-	CHAN6G(7055), /* Channel 221 */
-	CHAN6G(7075), /* Channel 225 */
-	CHAN6G(7095), /* Channel 229 */
-	CHAN6G(7115), /* Channel 233 */
-};
-
-#define NUM_S1G_CHANS_US 51
-static struct ieee80211_channel hwsim_channels_s1g[NUM_S1G_CHANS_US];
-
-static const struct ieee80211_sta_s1g_cap hwsim_s1g_cap = {
-	.s1g = true,
-	.cap = { S1G_CAP0_SGI_1MHZ | S1G_CAP0_SGI_2MHZ,
-		 0,
-		 0,
-		 S1G_CAP3_MAX_MPDU_LEN,
-		 0,
-		 S1G_CAP5_AMPDU,
-		 0,
-		 S1G_CAP7_DUP_1MHZ,
-		 S1G_CAP8_TWT_RESPOND | S1G_CAP8_TWT_REQUEST,
-		 0},
-	.nss_mcs = { 0xfc | 1, /* MCS 7 for 1 SS */
-	/* RX Highest Supported Long GI Data Rate 0:7 */
-		     0,
-	/* RX Highest Supported Long GI Data Rate 0:7 */
-	/* TX S1G MCS Map 0:6 */
-		     0xfa,
-	/* TX S1G MCS Map :7 */
-	/* TX Highest Supported Long GI Data Rate 0:6 */
-		     0x80,
-	/* TX Highest Supported Long GI Data Rate 7:8 */
-	/* Rx Single spatial stream and S1G-MCS Map for 1MHz */
-	/* Tx Single spatial stream and S1G-MCS Map for 1MHz */
-		     0 },
-};
-
-static void hwsim_init_s1g_channels(struct ieee80211_channel *chans)
-{
-	int ch, freq;
-
-	for (ch = 0; ch < NUM_S1G_CHANS_US; ch++) {
-		freq = 902000 + (ch + 1) * 500;
-		chans[ch].band = NL80211_BAND_S1GHZ;
-		chans[ch].center_freq = KHZ_TO_MHZ(freq);
-		chans[ch].freq_offset = freq % 1000;
-		chans[ch].hw_value = ch + 1;
-	}
-}
-
-static const struct ieee80211_rate hwsim_rates[] = {
-	{ .bitrate = 10 },
-	{ .bitrate = 20, .flags = IEEE80211_RATE_SHORT_PREAMBLE },
-	{ .bitrate = 55, .flags = IEEE80211_RATE_SHORT_PREAMBLE },
-	{ .bitrate = 110, .flags = IEEE80211_RATE_SHORT_PREAMBLE },
-	{ .bitrate = 60 },
-	{ .bitrate = 90 },
-	{ .bitrate = 120 },
-	{ .bitrate = 180 },
-	{ .bitrate = 240 },
-	{ .bitrate = 360 },
-	{ .bitrate = 480 },
-	{ .bitrate = 540 }
-};
-
-#define DEFAULT_RX_RSSI -50
-
-static const u32 hwsim_ciphers[] = {
-	WLAN_CIPHER_SUITE_WEP40,
-	WLAN_CIPHER_SUITE_WEP104,
-	WLAN_CIPHER_SUITE_TKIP,
-	WLAN_CIPHER_SUITE_CCMP,
-	WLAN_CIPHER_SUITE_CCMP_256,
-	WLAN_CIPHER_SUITE_GCMP,
-	WLAN_CIPHER_SUITE_GCMP_256,
-	WLAN_CIPHER_SUITE_AES_CMAC,
-	WLAN_CIPHER_SUITE_BIP_CMAC_256,
-	WLAN_CIPHER_SUITE_BIP_GMAC_128,
-	WLAN_CIPHER_SUITE_BIP_GMAC_256,
-};
-
-#define OUI_QCA 0x001374
-#define QCA_NL80211_SUBCMD_TEST 1
-enum qca_nl80211_vendor_subcmds {
-	QCA_WLAN_VENDOR_ATTR_TEST = 8,
-	QCA_WLAN_VENDOR_ATTR_MAX = QCA_WLAN_VENDOR_ATTR_TEST
-};
-
-static const struct nla_policy
-hwsim_vendor_test_policy[QCA_WLAN_VENDOR_ATTR_MAX + 1] = {
-	[QCA_WLAN_VENDOR_ATTR_MAX] = { .type = NLA_U32 },
-};
-
-static int mac80211_hwsim_vendor_cmd_test(struct wiphy *wiphy,
-					  struct wireless_dev *wdev,
-					  const void *data, int data_len)
-{
-	struct sk_buff *skb;
-	struct nlattr *tb[QCA_WLAN_VENDOR_ATTR_MAX + 1];
-	int err;
-	u32 val;
-
-	err = nla_parse_deprecated(tb, QCA_WLAN_VENDOR_ATTR_MAX, data,
-				   data_len, hwsim_vendor_test_policy, NULL);
-	if (err)
-		return err;
-	if (!tb[QCA_WLAN_VENDOR_ATTR_TEST])
-		return -EINVAL;
-	val = nla_get_u32(tb[QCA_WLAN_VENDOR_ATTR_TEST]);
-	wiphy_dbg(wiphy, "%s: test=%u\n", __func__, val);
-
-	/* Send a vendor event as a test. Note that this would not normally be
-	 * done within a command handler, but rather, based on some other
-	 * trigger. For simplicity, this command is used to trigger the event
-	 * here.
-	 *
-	 * event_idx = 0 (index in mac80211_hwsim_vendor_commands)
-	 */
-	skb = cfg80211_vendor_event_alloc(wiphy, wdev, 100, 0, GFP_KERNEL);
-	if (skb) {
-		/* skb_put() or nla_put() will fill up data within
-		 * NL80211_ATTR_VENDOR_DATA.
-		 */
-
-		/* Add vendor data */
-		nla_put_u32(skb, QCA_WLAN_VENDOR_ATTR_TEST, val + 1);
-
-		/* Send the event - this will call nla_nest_end() */
-		cfg80211_vendor_event(skb, GFP_KERNEL);
-	}
-
-	/* Send a response to the command */
-	skb = cfg80211_vendor_cmd_alloc_reply_skb(wiphy, 10);
-	if (!skb)
-		return -ENOMEM;
-
-	/* skb_put() or nla_put() will fill up data within
-	 * NL80211_ATTR_VENDOR_DATA
-	 */
-	nla_put_u32(skb, QCA_WLAN_VENDOR_ATTR_TEST, val + 2);
-
-	return cfg80211_vendor_cmd_reply(skb);
-}
-
-static struct wiphy_vendor_command mac80211_hwsim_vendor_commands[] = {
-	{
-		.info = { .vendor_id = OUI_QCA,
-			  .subcmd = QCA_NL80211_SUBCMD_TEST },
-		.flags = WIPHY_VENDOR_CMD_NEED_NETDEV,
-		.doit = mac80211_hwsim_vendor_cmd_test,
-		.policy = hwsim_vendor_test_policy,
-		.maxattr = QCA_WLAN_VENDOR_ATTR_MAX,
-	}
-};
-
-/* Advertise support vendor specific events */
-static const struct nl80211_vendor_cmd_info mac80211_hwsim_vendor_events[] = {
-	{ .vendor_id = OUI_QCA, .subcmd = 1 },
-};
-
-static DEFINE_SPINLOCK(hwsim_radio_lock);
-static LIST_HEAD(hwsim_radios);
-static struct rhashtable hwsim_radios_rht;
-static int hwsim_radio_idx;
-static int hwsim_radios_generation = 1;
-static u8 hwsim_nan_cluster_id[ETH_ALEN];
-
-static struct platform_driver mac80211_hwsim_driver = {
-	.driver = {
-		.name = "mac80211_hwsim",
-	},
-};
-
-struct mac80211_hwsim_link_data {
-	u32 link_id;
-	u64 beacon_int	/* beacon interval in us */;
-	struct hrtimer beacon_timer;
-};
-
-struct mac80211_hwsim_data {
-	struct list_head list;
-	struct rhash_head rht;
-	struct ieee80211_hw *hw;
-	struct device *dev;
-	struct ieee80211_supported_band bands[NUM_NL80211_BANDS];
-	struct ieee80211_channel channels_2ghz[ARRAY_SIZE(hwsim_channels_2ghz)];
-	struct ieee80211_channel channels_5ghz[ARRAY_SIZE(hwsim_channels_5ghz)];
-	struct ieee80211_channel channels_6ghz[ARRAY_SIZE(hwsim_channels_6ghz)];
-	struct ieee80211_channel channels_s1g[ARRAY_SIZE(hwsim_channels_s1g)];
-	struct ieee80211_rate rates[ARRAY_SIZE(hwsim_rates)];
-	struct ieee80211_iface_combination if_combination;
-	struct ieee80211_iface_limit if_limits[4];
-	int n_if_limits;
-
-	struct ieee80211_iface_combination if_combination_radio;
-	struct wiphy_radio_freq_range radio_range[NUM_NL80211_BANDS];
-	struct wiphy_radio radio[NUM_NL80211_BANDS];
-
-	u32 ciphers[ARRAY_SIZE(hwsim_ciphers)];
-
-	struct mac_address addresses[3];
-	int channels, idx;
-	bool use_chanctx;
-	bool destroy_on_close;
-	u32 portid;
-	char alpha2[2];
-	const struct ieee80211_regdomain *regd;
-
-	struct ieee80211_channel *tmp_chan;
-	struct ieee80211_channel *roc_chan;
-	u32 roc_duration;
-	struct delayed_work roc_start;
-	struct delayed_work roc_done;
-	struct delayed_work hw_scan;
-	struct cfg80211_scan_request *hw_scan_request;
-	struct ieee80211_vif *hw_scan_vif;
-	int scan_chan_idx;
-	u8 scan_addr[ETH_ALEN];
-	struct {
-		struct ieee80211_channel *channel;
-		unsigned long next_start, start, end;
-	} survey_data[ARRAY_SIZE(hwsim_channels_2ghz) +
-		      ARRAY_SIZE(hwsim_channels_5ghz) +
-		      ARRAY_SIZE(hwsim_channels_6ghz)];
-
-	struct ieee80211_channel *channel;
-	enum nl80211_chan_width bw;
-	unsigned int rx_filter;
-	bool started, idle, scanning;
-	struct mutex mutex;
-	enum ps_mode {
-		PS_DISABLED, PS_ENABLED, PS_AUTO_POLL, PS_MANUAL_POLL
-	} ps;
-	bool ps_poll_pending;
-	struct dentry *debugfs;
-	struct cfg80211_chan_def radar_background_chandef;
-
-	atomic_t pending_cookie;
-	struct sk_buff_head pending;	/* packets pending */
-	/*
-	 * Only radios in the same group can communicate together (the
-	 * channel has to match too). Each bit represents a group. A
-	 * radio can be in more than one group.
-	 */
-	u64 group;
-
-	/* group shared by radios created in the same netns */
-	int netgroup;
-	/* wmediumd portid responsible for netgroup of this radio */
-	u32 wmediumd;
-
-	/* difference between this hw's clock and the real clock, in usecs */
-	s64 tsf_offset;
-	s64 bcn_delta;
-	/* absolute beacon transmission time. Used to cover up "tx" delay. */
-	u64 abs_bcn_ts;
-
-	/* Stats */
-	u64 tx_pkts;
-	u64 rx_pkts;
-	u64 tx_bytes;
-	u64 rx_bytes;
-	u64 tx_dropped;
-	u64 tx_failed;
-
-	/* RSSI in rx status of the receiver */
-	int rx_rssi;
-
-	/* only used when pmsr capability is supplied */
-	struct cfg80211_pmsr_capabilities pmsr_capa;
-	struct cfg80211_pmsr_request *pmsr_request;
-	struct wireless_dev *pmsr_request_wdev;
-
-	struct mac80211_hwsim_link_data link_data[IEEE80211_MLD_MAX_NUM_LINKS];
-
-	struct ieee80211_vif *nan_device_vif;
-	u8 nan_bands;
-
-	enum nl80211_band nan_curr_dw_band;
-	struct hrtimer nan_timer;
-	bool notify_dw;
-	struct ieee80211_vif *nan_vif;
-};
-
-static const struct rhashtable_params hwsim_rht_params = {
-	.nelem_hint = 2,
-	.automatic_shrinking = true,
-	.key_len = ETH_ALEN,
-	.key_offset = offsetof(struct mac80211_hwsim_data, addresses[1]),
-	.head_offset = offsetof(struct mac80211_hwsim_data, rht),
-};
-
-struct hwsim_radiotap_hdr {
-	struct ieee80211_radiotap_header_fixed hdr;
-	__le64 rt_tsft;
-	u8 rt_flags;
-	u8 rt_rate;
-	__le16 rt_channel;
-	__le16 rt_chbitmask;
-} __packed;
-
-struct hwsim_radiotap_ack_hdr {
-	struct ieee80211_radiotap_header_fixed hdr;
-	u8 rt_flags;
-	u8 pad;
-	__le16 rt_channel;
-	__le16 rt_chbitmask;
-} __packed;
-
-static struct mac80211_hwsim_data *get_hwsim_data_ref_from_addr(const u8 *addr)
-{
-	return rhashtable_lookup_fast(&hwsim_radios_rht, addr, hwsim_rht_params);
-}
-
-/* MAC80211_HWSIM netlink family */
-static struct genl_family hwsim_genl_family;
-
-enum hwsim_multicast_groups {
-	HWSIM_MCGRP_CONFIG,
-};
-
-static const struct genl_multicast_group hwsim_mcgrps[] = {
-	[HWSIM_MCGRP_CONFIG] = { .name = "config", },
-};
-
-/* MAC80211_HWSIM netlink policy */
-
-static const struct nla_policy
-hwsim_rate_info_policy[HWSIM_RATE_INFO_ATTR_MAX + 1] = {
-	[HWSIM_RATE_INFO_ATTR_FLAGS] = { .type = NLA_U8 },
-	[HWSIM_RATE_INFO_ATTR_MCS] = { .type = NLA_U8 },
-	[HWSIM_RATE_INFO_ATTR_LEGACY] = { .type = NLA_U16 },
-	[HWSIM_RATE_INFO_ATTR_NSS] = { .type = NLA_U8 },
-	[HWSIM_RATE_INFO_ATTR_BW] = { .type = NLA_U8 },
-	[HWSIM_RATE_INFO_ATTR_HE_GI] = { .type = NLA_U8 },
-	[HWSIM_RATE_INFO_ATTR_HE_DCM] = { .type = NLA_U8 },
-	[HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC] = { .type = NLA_U8 },
-	[HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH] = { .type = NLA_U8 },
-	[HWSIM_RATE_INFO_ATTR_EHT_GI] = { .type = NLA_U8 },
-	[HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC] = { .type = NLA_U8 },
-};
-
-static const struct nla_policy
-hwsim_ftm_result_policy[NL80211_PMSR_FTM_RESP_ATTR_MAX + 1] = {
-	[NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON] = { .type = NLA_U32 },
-	[NL80211_PMSR_FTM_RESP_ATTR_BURST_INDEX] = { .type = NLA_U16 },
-	[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS] = { .type = NLA_U32 },
-	[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES] = { .type = NLA_U32 },
-	[NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME] = { .type = NLA_U8 },
-	[NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP] = { .type = NLA_U8 },
-	[NL80211_PMSR_FTM_RESP_ATTR_BURST_DURATION] = { .type = NLA_U8 },
-	[NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST] = { .type = NLA_U8 },
-	[NL80211_PMSR_FTM_RESP_ATTR_RSSI_AVG] = { .type = NLA_U32 },
-	[NL80211_PMSR_FTM_RESP_ATTR_RSSI_SPREAD] = { .type = NLA_U32 },
-	[NL80211_PMSR_FTM_RESP_ATTR_TX_RATE] = NLA_POLICY_NESTED(hwsim_rate_info_policy),
-	[NL80211_PMSR_FTM_RESP_ATTR_RX_RATE] = NLA_POLICY_NESTED(hwsim_rate_info_policy),
-	[NL80211_PMSR_FTM_RESP_ATTR_RTT_AVG] = { .type = NLA_U64 },
-	[NL80211_PMSR_FTM_RESP_ATTR_RTT_VARIANCE] = { .type = NLA_U64 },
-	[NL80211_PMSR_FTM_RESP_ATTR_RTT_SPREAD] = { .type = NLA_U64 },
-	[NL80211_PMSR_FTM_RESP_ATTR_DIST_AVG] = { .type = NLA_U64 },
-	[NL80211_PMSR_FTM_RESP_ATTR_DIST_VARIANCE] = { .type = NLA_U64 },
-	[NL80211_PMSR_FTM_RESP_ATTR_DIST_SPREAD] = { .type = NLA_U64 },
-	[NL80211_PMSR_FTM_RESP_ATTR_LCI] = { .type = NLA_STRING },
-	[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC] = { .type = NLA_STRING },
-};
-
-static const struct nla_policy
-hwsim_pmsr_resp_type_policy[NL80211_PMSR_TYPE_MAX + 1] = {
-	[NL80211_PMSR_TYPE_FTM] = NLA_POLICY_NESTED(hwsim_ftm_result_policy),
-};
-
-static const struct nla_policy
-hwsim_pmsr_resp_policy[NL80211_PMSR_RESP_ATTR_MAX + 1] = {
-	[NL80211_PMSR_RESP_ATTR_STATUS] = { .type = NLA_U32 },
-	[NL80211_PMSR_RESP_ATTR_HOST_TIME] = { .type = NLA_U64 },
-	[NL80211_PMSR_RESP_ATTR_AP_TSF] = { .type = NLA_U64 },
-	[NL80211_PMSR_RESP_ATTR_FINAL] = { .type = NLA_FLAG },
-	[NL80211_PMSR_RESP_ATTR_DATA] = NLA_POLICY_NESTED(hwsim_pmsr_resp_type_policy),
-};
-
-static const struct nla_policy
-hwsim_pmsr_peer_result_policy[NL80211_PMSR_PEER_ATTR_MAX + 1] = {
-	[NL80211_PMSR_PEER_ATTR_ADDR] = NLA_POLICY_ETH_ADDR_COMPAT,
-	[NL80211_PMSR_PEER_ATTR_CHAN] = { .type = NLA_REJECT },
-	[NL80211_PMSR_PEER_ATTR_REQ] = { .type = NLA_REJECT },
-	[NL80211_PMSR_PEER_ATTR_RESP] = NLA_POLICY_NESTED(hwsim_pmsr_resp_policy),
-};
-
-static const struct nla_policy
-hwsim_pmsr_peers_result_policy[NL80211_PMSR_ATTR_MAX + 1] = {
-	[NL80211_PMSR_ATTR_MAX_PEERS] = { .type = NLA_REJECT },
-	[NL80211_PMSR_ATTR_REPORT_AP_TSF] = { .type = NLA_REJECT },
-	[NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR] = { .type = NLA_REJECT },
-	[NL80211_PMSR_ATTR_TYPE_CAPA] = { .type = NLA_REJECT },
-	[NL80211_PMSR_ATTR_PEERS] = NLA_POLICY_NESTED_ARRAY(hwsim_pmsr_peer_result_policy),
-};
-
-static const struct nla_policy
-hwsim_ftm_capa_policy[NL80211_PMSR_FTM_CAPA_ATTR_MAX + 1] = {
-	[NL80211_PMSR_FTM_CAPA_ATTR_ASAP] = { .type = NLA_FLAG },
-	[NL80211_PMSR_FTM_CAPA_ATTR_NON_ASAP] = { .type = NLA_FLAG },
-	[NL80211_PMSR_FTM_CAPA_ATTR_REQ_LCI] = { .type = NLA_FLAG },
-	[NL80211_PMSR_FTM_CAPA_ATTR_REQ_CIVICLOC] = { .type = NLA_FLAG },
-	[NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES] = { .type = NLA_U32 },
-	[NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS] = { .type = NLA_U32 },
-	[NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT] = NLA_POLICY_MAX(NLA_U8, 15),
-	[NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST] = NLA_POLICY_MAX(NLA_U8, 31),
-	[NL80211_PMSR_FTM_CAPA_ATTR_TRIGGER_BASED] = { .type = NLA_FLAG },
-	[NL80211_PMSR_FTM_CAPA_ATTR_NON_TRIGGER_BASED] = { .type = NLA_FLAG },
-};
-
-static const struct nla_policy
-hwsim_pmsr_capa_type_policy[NL80211_PMSR_TYPE_MAX + 1] = {
-	[NL80211_PMSR_TYPE_FTM] = NLA_POLICY_NESTED(hwsim_ftm_capa_policy),
-};
-
-static const struct nla_policy
-hwsim_pmsr_capa_policy[NL80211_PMSR_ATTR_MAX + 1] = {
-	[NL80211_PMSR_ATTR_MAX_PEERS] = { .type = NLA_U32 },
-	[NL80211_PMSR_ATTR_REPORT_AP_TSF] = { .type = NLA_FLAG },
-	[NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR] = { .type = NLA_FLAG },
-	[NL80211_PMSR_ATTR_TYPE_CAPA] = NLA_POLICY_NESTED(hwsim_pmsr_capa_type_policy),
-	[NL80211_PMSR_ATTR_PEERS] = { .type = NLA_REJECT }, // only for request.
-};
-
-static const struct nla_policy hwsim_genl_policy[HWSIM_ATTR_MAX + 1] = {
-	[HWSIM_ATTR_ADDR_RECEIVER] = NLA_POLICY_ETH_ADDR_COMPAT,
-	[HWSIM_ATTR_ADDR_TRANSMITTER] = NLA_POLICY_ETH_ADDR_COMPAT,
-	[HWSIM_ATTR_FRAME] = { .type = NLA_BINARY,
-			       .len = IEEE80211_MAX_DATA_LEN },
-	[HWSIM_ATTR_FLAGS] = { .type = NLA_U32 },
-	[HWSIM_ATTR_RX_RATE] = { .type = NLA_U32 },
-	[HWSIM_ATTR_SIGNAL] = { .type = NLA_U32 },
-	[HWSIM_ATTR_TX_INFO] = { .type = NLA_BINARY,
-				 .len = IEEE80211_TX_MAX_RATES *
-					sizeof(struct hwsim_tx_rate)},
-	[HWSIM_ATTR_COOKIE] = { .type = NLA_U64 },
-	[HWSIM_ATTR_CHANNELS] = { .type = NLA_U32 },
-	[HWSIM_ATTR_RADIO_ID] = { .type = NLA_U32 },
-	[HWSIM_ATTR_REG_HINT_ALPHA2] = { .type = NLA_STRING, .len = 2 },
-	[HWSIM_ATTR_REG_CUSTOM_REG] = { .type = NLA_U32 },
-	[HWSIM_ATTR_REG_STRICT_REG] = { .type = NLA_FLAG },
-	[HWSIM_ATTR_SUPPORT_P2P_DEVICE] = { .type = NLA_FLAG },
-	[HWSIM_ATTR_USE_CHANCTX] = { .type = NLA_FLAG },
-	[HWSIM_ATTR_DESTROY_RADIO_ON_CLOSE] = { .type = NLA_FLAG },
-	[HWSIM_ATTR_RADIO_NAME] = { .type = NLA_STRING },
-	[HWSIM_ATTR_NO_VIF] = { .type = NLA_FLAG },
-	[HWSIM_ATTR_FREQ] = { .type = NLA_U32 },
-	[HWSIM_ATTR_TX_INFO_FLAGS] = { .type = NLA_BINARY },
-	[HWSIM_ATTR_PERM_ADDR] = NLA_POLICY_ETH_ADDR_COMPAT,
-	[HWSIM_ATTR_IFTYPE_SUPPORT] = { .type = NLA_U32 },
-	[HWSIM_ATTR_CIPHER_SUPPORT] = { .type = NLA_BINARY },
-	[HWSIM_ATTR_MLO_SUPPORT] = { .type = NLA_FLAG },
-	[HWSIM_ATTR_PMSR_SUPPORT] = NLA_POLICY_NESTED(hwsim_pmsr_capa_policy),
-	[HWSIM_ATTR_PMSR_RESULT] = NLA_POLICY_NESTED(hwsim_pmsr_peers_result_policy),
-	[HWSIM_ATTR_MULTI_RADIO] = { .type = NLA_FLAG },
-	[HWSIM_ATTR_SUPPORT_NAN_DEVICE] = { .type = NLA_FLAG },
-	[HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR] = { .type = NLA_FLAG },
-};
-
-#if IS_REACHABLE(CONFIG_VIRTIO)
-
-/* MAC80211_HWSIM virtio queues */
-static struct virtqueue *hwsim_vqs[HWSIM_NUM_VQS];
-static bool hwsim_virtio_enabled;
-static DEFINE_SPINLOCK(hwsim_virtio_lock);
-
-static void hwsim_virtio_rx_work(struct work_struct *work);
-static DECLARE_WORK(hwsim_virtio_rx, hwsim_virtio_rx_work);
-
-static int hwsim_tx_virtio(struct mac80211_hwsim_data *data,
-			   struct sk_buff *skb)
-{
-	struct scatterlist sg[1];
-	unsigned long flags;
-	int err;
-
-	spin_lock_irqsave(&hwsim_virtio_lock, flags);
-	if (!hwsim_virtio_enabled) {
-		err = -ENODEV;
-		goto out_free;
-	}
-
-	sg_init_one(sg, skb->head, skb_end_offset(skb));
-	err = virtqueue_add_outbuf(hwsim_vqs[HWSIM_VQ_TX], sg, 1, skb,
-				   GFP_ATOMIC);
-	if (err)
-		goto out_free;
-	virtqueue_kick(hwsim_vqs[HWSIM_VQ_TX]);
-	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
-	return 0;
-
-out_free:
-	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
-	nlmsg_free(skb);
-	return err;
-}
-#else
-/* cause a linker error if this ends up being needed */
-extern int hwsim_tx_virtio(struct mac80211_hwsim_data *data,
-			   struct sk_buff *skb);
-#define hwsim_virtio_enabled false
-#endif
-
-static int hwsim_get_chanwidth(enum nl80211_chan_width bw)
-{
-	switch (bw) {
-	case NL80211_CHAN_WIDTH_20_NOHT:
-	case NL80211_CHAN_WIDTH_20:
-		return 20;
-	case NL80211_CHAN_WIDTH_40:
-		return 40;
-	case NL80211_CHAN_WIDTH_80:
-		return 80;
-	case NL80211_CHAN_WIDTH_80P80:
-	case NL80211_CHAN_WIDTH_160:
-		return 160;
-	case NL80211_CHAN_WIDTH_320:
-		return 320;
-	case NL80211_CHAN_WIDTH_5:
-		return 5;
-	case NL80211_CHAN_WIDTH_10:
-		return 10;
-	case NL80211_CHAN_WIDTH_1:
-		return 1;
-	case NL80211_CHAN_WIDTH_2:
-		return 2;
-	case NL80211_CHAN_WIDTH_4:
-		return 4;
-	case NL80211_CHAN_WIDTH_8:
-		return 8;
-	case NL80211_CHAN_WIDTH_16:
-		return 16;
-	}
-
-	return INT_MAX;
-}
-
-static void mac80211_hwsim_tx_frame(struct ieee80211_hw *hw,
-				    struct sk_buff *skb,
-				    struct ieee80211_channel *chan);
-
-/* sysfs attributes */
-static void hwsim_send_ps_poll(void *dat, u8 *mac, struct ieee80211_vif *vif)
-{
-	struct mac80211_hwsim_data *data = dat;
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-	struct sk_buff *skb;
-	struct ieee80211_pspoll *pspoll;
-
-	if (!vp->assoc)
-		return;
-
-	wiphy_dbg(data->hw->wiphy,
-		  "%s: send PS-Poll to %pM for aid %d\n",
-		  __func__, vp->bssid, vp->aid);
-
-	skb = dev_alloc_skb(sizeof(*pspoll));
-	if (!skb)
-		return;
-	pspoll = skb_put(skb, sizeof(*pspoll));
-	pspoll->frame_control = cpu_to_le16(IEEE80211_FTYPE_CTL |
-					    IEEE80211_STYPE_PSPOLL |
-					    IEEE80211_FCTL_PM);
-	pspoll->aid = cpu_to_le16(0xc000 | vp->aid);
-	memcpy(pspoll->bssid, vp->bssid, ETH_ALEN);
-	memcpy(pspoll->ta, mac, ETH_ALEN);
-
-	rcu_read_lock();
-	mac80211_hwsim_tx_frame(data->hw, skb,
-				rcu_dereference(vif->bss_conf.chanctx_conf)->def.chan);
-	rcu_read_unlock();
-}
-
-static void hwsim_send_nullfunc(struct mac80211_hwsim_data *data, u8 *mac,
-				struct ieee80211_vif *vif, int ps)
-{
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-	struct sk_buff *skb;
-	struct ieee80211_hdr *hdr;
-	struct ieee80211_tx_info *cb;
-
-	if (!vp->assoc)
-		return;
-
-	wiphy_dbg(data->hw->wiphy,
-		  "%s: send data::nullfunc to %pM ps=%d\n",
-		  __func__, vp->bssid, ps);
-
-	skb = dev_alloc_skb(sizeof(*hdr));
-	if (!skb)
-		return;
-	hdr = skb_put(skb, sizeof(*hdr) - ETH_ALEN);
-	hdr->frame_control = cpu_to_le16(IEEE80211_FTYPE_DATA |
-					 IEEE80211_STYPE_NULLFUNC |
-					 IEEE80211_FCTL_TODS |
-					 (ps ? IEEE80211_FCTL_PM : 0));
-	hdr->duration_id = cpu_to_le16(0);
-	memcpy(hdr->addr1, vp->bssid, ETH_ALEN);
-	memcpy(hdr->addr2, mac, ETH_ALEN);
-	memcpy(hdr->addr3, vp->bssid, ETH_ALEN);
-
-	cb = IEEE80211_SKB_CB(skb);
-	cb->control.rates[0].count = 1;
-	cb->control.rates[1].idx = -1;
-
-	rcu_read_lock();
-	mac80211_hwsim_tx_frame(data->hw, skb,
-				rcu_dereference(vif->bss_conf.chanctx_conf)->def.chan);
-	rcu_read_unlock();
-}
-
-
-static void hwsim_send_nullfunc_ps(void *dat, u8 *mac,
-				   struct ieee80211_vif *vif)
-{
-	struct mac80211_hwsim_data *data = dat;
-	hwsim_send_nullfunc(data, mac, vif, 1);
-}
-
-static void hwsim_send_nullfunc_no_ps(void *dat, u8 *mac,
-				      struct ieee80211_vif *vif)
-{
-	struct mac80211_hwsim_data *data = dat;
-	hwsim_send_nullfunc(data, mac, vif, 0);
-}
-
-static int hwsim_fops_ps_read(void *dat, u64 *val)
-{
-	struct mac80211_hwsim_data *data = dat;
-	*val = data->ps;
-	return 0;
-}
-
-static int hwsim_fops_ps_write(void *dat, u64 val)
-{
-	struct mac80211_hwsim_data *data = dat;
-	enum ps_mode old_ps;
-
-	if (val != PS_DISABLED && val != PS_ENABLED && val != PS_AUTO_POLL &&
-	    val != PS_MANUAL_POLL)
-		return -EINVAL;
-
-	if (val == PS_MANUAL_POLL) {
-		if (data->ps != PS_ENABLED)
-			return -EINVAL;
-		local_bh_disable();
-		ieee80211_iterate_active_interfaces_atomic(
-			data->hw, IEEE80211_IFACE_ITER_NORMAL,
-			hwsim_send_ps_poll, data);
-		local_bh_enable();
-		return 0;
-	}
-	old_ps = data->ps;
-	data->ps = val;
-
-	local_bh_disable();
-	if (old_ps == PS_DISABLED && val != PS_DISABLED) {
-		ieee80211_iterate_active_interfaces_atomic(
-			data->hw, IEEE80211_IFACE_ITER_NORMAL,
-			hwsim_send_nullfunc_ps, data);
-	} else if (old_ps != PS_DISABLED && val == PS_DISABLED) {
-		ieee80211_iterate_active_interfaces_atomic(
-			data->hw, IEEE80211_IFACE_ITER_NORMAL,
-			hwsim_send_nullfunc_no_ps, data);
-	}
-	local_bh_enable();
-
-	return 0;
-}
-
-DEFINE_DEBUGFS_ATTRIBUTE(hwsim_fops_ps, hwsim_fops_ps_read, hwsim_fops_ps_write,
-			 "%llu\n");
-
-static int hwsim_write_simulate_radar(void *dat, u64 val)
-{
-	struct mac80211_hwsim_data *data = dat;
-
-	ieee80211_radar_detected(data->hw, NULL);
-
-	return 0;
-}
-
-DEFINE_DEBUGFS_ATTRIBUTE(hwsim_simulate_radar, NULL,
-			 hwsim_write_simulate_radar, "%llu\n");
-
-static ssize_t hwsim_background_cac_write(struct file *file,
-					  const char __user *user_buf,
-					  size_t count, loff_t *ppos)
-{
-	struct mac80211_hwsim_data *data = file->private_data;
-	char buf[8] = {};
-
-	if (count >= sizeof(buf))
-		return -EINVAL;
-
-	if (copy_from_user(buf, user_buf, count))
-		return -EFAULT;
-
-	/* Check if background radar channel is configured */
-	if (!data->radar_background_chandef.chan)
-		return -ENOENT;
-
-	if (sysfs_streq(buf, "radar"))
-		cfg80211_background_radar_event(data->hw->wiphy,
-						&data->radar_background_chandef,
-						GFP_KERNEL);
-	else if (sysfs_streq(buf, "cancel"))
-		cfg80211_background_cac_abort(data->hw->wiphy);
-	else
-		return -EINVAL;
-
-	return count;
-}
-
-static const struct file_operations hwsim_background_cac_ops = {
-	.write = hwsim_background_cac_write,
-	.open = simple_open,
-	.llseek = default_llseek,
-};
-
-struct hwsim_chanctx_iter_arg {
-	struct ieee80211_chanctx_conf *conf;
-	u32 freq_mhz;
-};
-
-static void hwsim_6ghz_chanctx_iter(struct ieee80211_hw *hw,
-				    struct ieee80211_chanctx_conf *conf,
-				    void *data)
-{
-	struct hwsim_chanctx_iter_arg *arg = data;
-
-	if (conf->def.chan &&
-	    conf->def.chan->band == NL80211_BAND_6GHZ &&
-	    conf->def.chan->center_freq == arg->freq_mhz)
-		arg->conf = conf;
-}
-
-static ssize_t hwsim_simulate_incumbent_signal_write(struct file *file,
-						     const char __user *ubuf,
-						     size_t len, loff_t *ppos)
-{
-	struct mac80211_hwsim_data *data = file->private_data;
-	struct hwsim_chanctx_iter_arg arg = {};
-	u32 bitmap;
-	char buf[64];
-
-	if (!len || len > sizeof(buf) - 1)
-		return -EINVAL;
-
-	if (copy_from_user(buf, ubuf, len))
-		return -EFAULT;
-	buf[len] = '\0';
-
-	if (sscanf(buf, "%u %i", &arg.freq_mhz, &bitmap) != 2)
-		return -EINVAL;
-
-	if (!arg.freq_mhz)
-		return -EINVAL;
-
-	ieee80211_iter_chan_contexts_atomic(data->hw,
-					    hwsim_6ghz_chanctx_iter,
-					    &arg);
-
-	if (!arg.conf)
-		return -EINVAL;
-
-	cfg80211_incumbent_signal_notify(data->hw->wiphy,
-					 &arg.conf->def,
-					 bitmap,
-					 GFP_KERNEL);
-
-	return len;
-}
-
-static const struct file_operations hwsim_simulate_incumbent_signal_fops = {
-	.open	= simple_open,
-	.write	= hwsim_simulate_incumbent_signal_write,
-};
-
-static int hwsim_fops_group_read(void *dat, u64 *val)
-{
-	struct mac80211_hwsim_data *data = dat;
-	*val = data->group;
-	return 0;
-}
-
-static int hwsim_fops_group_write(void *dat, u64 val)
-{
-	struct mac80211_hwsim_data *data = dat;
-	data->group = val;
-	return 0;
-}
-
-DEFINE_DEBUGFS_ATTRIBUTE(hwsim_fops_group,
-			 hwsim_fops_group_read, hwsim_fops_group_write,
-			 "%llx\n");
-
-static int hwsim_fops_rx_rssi_read(void *dat, u64 *val)
-{
-	struct mac80211_hwsim_data *data = dat;
-	*val = data->rx_rssi;
-	return 0;
-}
-
-static int hwsim_fops_rx_rssi_write(void *dat, u64 val)
-{
-	struct mac80211_hwsim_data *data = dat;
-	int rssi = (int)val;
-
-	if (rssi >= 0 || rssi < -100)
-		return -EINVAL;
-
-	data->rx_rssi = rssi;
-	return 0;
-}
-
-DEFINE_DEBUGFS_ATTRIBUTE(hwsim_fops_rx_rssi,
-			 hwsim_fops_rx_rssi_read, hwsim_fops_rx_rssi_write,
-			 "%lld\n");
-
-static netdev_tx_t hwsim_mon_xmit(struct sk_buff *skb,
-					struct net_device *dev)
-{
-	/* TODO: allow packet injection */
-	dev_kfree_skb(skb);
-	return NETDEV_TX_OK;
-}
-
-static inline u64 mac80211_hwsim_get_tsf_raw(void)
-{
-	return ktime_to_us(ktime_get_real());
-}
-
-static __le64 __mac80211_hwsim_get_tsf(struct mac80211_hwsim_data *data)
-{
-	u64 now = mac80211_hwsim_get_tsf_raw();
-	return cpu_to_le64(now + data->tsf_offset);
-}
-
-static u64 mac80211_hwsim_get_tsf(struct ieee80211_hw *hw,
-				  struct ieee80211_vif *vif)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	return le64_to_cpu(__mac80211_hwsim_get_tsf(data));
-}
-
-static void mac80211_hwsim_set_tsf(struct ieee80211_hw *hw,
-		struct ieee80211_vif *vif, u64 tsf)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	u64 now = mac80211_hwsim_get_tsf(hw, vif);
-	/* MLD not supported here */
-	u32 bcn_int = data->link_data[0].beacon_int;
-	u64 delta = abs(tsf - now);
-	struct ieee80211_bss_conf *conf;
-
-	conf = link_conf_dereference_protected(vif, data->link_data[0].link_id);
-	if (conf && !conf->enable_beacon)
-		return;
-
-	/* adjust after beaconing with new timestamp at old TBTT */
-	if (tsf > now) {
-		data->tsf_offset += delta;
-		data->bcn_delta = do_div(delta, bcn_int);
-	} else {
-		data->tsf_offset -= delta;
-		data->bcn_delta = -(s64)do_div(delta, bcn_int);
-	}
-}
-
-static void mac80211_hwsim_monitor_rx(struct ieee80211_hw *hw,
-				      struct sk_buff *tx_skb,
-				      struct ieee80211_channel *chan)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	struct sk_buff *skb;
-	struct hwsim_radiotap_hdr *hdr;
-	u16 flags, bitrate;
-	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_skb);
-	struct ieee80211_rate *txrate = ieee80211_get_tx_rate(hw, info);
-
-	if (!txrate)
-		bitrate = 0;
-	else
-		bitrate = txrate->bitrate;
-
-	if (!netif_running(hwsim_mon))
-		return;
-
-	skb = skb_copy_expand(tx_skb, sizeof(*hdr), 0, GFP_ATOMIC);
-	if (skb == NULL)
-		return;
-
-	hdr = skb_push(skb, sizeof(*hdr));
-	hdr->hdr.it_version = PKTHDR_RADIOTAP_VERSION;
-	hdr->hdr.it_pad = 0;
-	hdr->hdr.it_len = cpu_to_le16(sizeof(*hdr));
-	hdr->hdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) |
-					  (1 << IEEE80211_RADIOTAP_RATE) |
-					  (1 << IEEE80211_RADIOTAP_TSFT) |
-					  (1 << IEEE80211_RADIOTAP_CHANNEL));
-	hdr->rt_tsft = __mac80211_hwsim_get_tsf(data);
-	hdr->rt_flags = 0;
-	hdr->rt_rate = bitrate / 5;
-	hdr->rt_channel = cpu_to_le16(chan->center_freq);
-	flags = IEEE80211_CHAN_2GHZ;
-	if (txrate && txrate->flags & IEEE80211_RATE_ERP_G)
-		flags |= IEEE80211_CHAN_OFDM;
-	else
-		flags |= IEEE80211_CHAN_CCK;
-	hdr->rt_chbitmask = cpu_to_le16(flags);
-
-	skb->dev = hwsim_mon;
-	skb_reset_mac_header(skb);
-	skb->ip_summed = CHECKSUM_UNNECESSARY;
-	skb->pkt_type = PACKET_OTHERHOST;
-	skb->protocol = htons(ETH_P_802_2);
-	memset(skb->cb, 0, sizeof(skb->cb));
-	netif_rx(skb);
-}
-
-
-static void mac80211_hwsim_monitor_ack(struct ieee80211_channel *chan,
-				       const u8 *addr)
-{
-	struct sk_buff *skb;
-	struct hwsim_radiotap_ack_hdr *hdr;
-	u16 flags;
-	struct ieee80211_hdr *hdr11;
-
-	if (!netif_running(hwsim_mon))
-		return;
-
-	skb = dev_alloc_skb(100);
-	if (skb == NULL)
-		return;
-
-	hdr = skb_put(skb, sizeof(*hdr));
-	hdr->hdr.it_version = PKTHDR_RADIOTAP_VERSION;
-	hdr->hdr.it_pad = 0;
-	hdr->hdr.it_len = cpu_to_le16(sizeof(*hdr));
-	hdr->hdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) |
-					  (1 << IEEE80211_RADIOTAP_CHANNEL));
-	hdr->rt_flags = 0;
-	hdr->pad = 0;
-	hdr->rt_channel = cpu_to_le16(chan->center_freq);
-	flags = IEEE80211_CHAN_2GHZ;
-	hdr->rt_chbitmask = cpu_to_le16(flags);
-
-	hdr11 = skb_put(skb, 10);
-	hdr11->frame_control = cpu_to_le16(IEEE80211_FTYPE_CTL |
-					   IEEE80211_STYPE_ACK);
-	hdr11->duration_id = cpu_to_le16(0);
-	memcpy(hdr11->addr1, addr, ETH_ALEN);
-
-	skb->dev = hwsim_mon;
-	skb_reset_mac_header(skb);
-	skb->ip_summed = CHECKSUM_UNNECESSARY;
-	skb->pkt_type = PACKET_OTHERHOST;
-	skb->protocol = htons(ETH_P_802_2);
-	memset(skb->cb, 0, sizeof(skb->cb));
-	netif_rx(skb);
-}
-
-struct mac80211_hwsim_addr_match_data {
-	u8 addr[ETH_ALEN];
-	bool ret;
-};
-
-static void mac80211_hwsim_addr_iter(void *data, u8 *mac,
-				     struct ieee80211_vif *vif)
-{
-	int i;
-	struct mac80211_hwsim_addr_match_data *md = data;
-
-	if (memcmp(mac, md->addr, ETH_ALEN) == 0) {
-		md->ret = true;
-		return;
-	}
-
-	/* Match the link address */
-	for (i = 0; i < ARRAY_SIZE(vif->link_conf); i++) {
-		struct ieee80211_bss_conf *conf;
-
-		conf = rcu_dereference(vif->link_conf[i]);
-		if (!conf)
-			continue;
-
-		if (memcmp(conf->addr, md->addr, ETH_ALEN) == 0) {
-			md->ret = true;
-			return;
-		}
-	}
-}
-
-static bool mac80211_hwsim_addr_match(struct mac80211_hwsim_data *data,
-				      const u8 *addr)
-{
-	struct mac80211_hwsim_addr_match_data md = {
-		.ret = false,
-	};
-
-	if (data->scanning && memcmp(addr, data->scan_addr, ETH_ALEN) == 0)
-		return true;
-
-	memcpy(md.addr, addr, ETH_ALEN);
-
-	ieee80211_iterate_active_interfaces_atomic(data->hw,
-						   IEEE80211_IFACE_ITER_NORMAL,
-						   mac80211_hwsim_addr_iter,
-						   &md);
-
-	return md.ret;
-}
-
-static bool hwsim_ps_rx_ok(struct mac80211_hwsim_data *data,
-			   struct sk_buff *skb)
-{
-	switch (data->ps) {
-	case PS_DISABLED:
-		return true;
-	case PS_ENABLED:
-		return false;
-	case PS_AUTO_POLL:
-		/* TODO: accept (some) Beacons by default and other frames only
-		 * if pending PS-Poll has been sent */
-		return true;
-	case PS_MANUAL_POLL:
-		/* Allow unicast frames to own address if there is a pending
-		 * PS-Poll */
-		if (data->ps_poll_pending &&
-		    mac80211_hwsim_addr_match(data, skb->data + 4)) {
-			data->ps_poll_pending = false;
-			return true;
-		}
-		return false;
-	}
-
-	return true;
-}
-
-static int hwsim_unicast_netgroup(struct mac80211_hwsim_data *data,
-				  struct sk_buff *skb, int portid)
-{
-	struct net *net;
-	bool found = false;
-	int res = -ENOENT;
-
-	rcu_read_lock();
-	for_each_net_rcu(net) {
-		if (data->netgroup == hwsim_net_get_netgroup(net)) {
-			res = genlmsg_unicast(net, skb, portid);
-			found = true;
-			break;
-		}
-	}
-	rcu_read_unlock();
-
-	if (!found)
-		nlmsg_free(skb);
-
-	return res;
-}
-
-static void mac80211_hwsim_config_mac_nl(struct ieee80211_hw *hw,
-					 const u8 *addr, bool add)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	u32 _portid = READ_ONCE(data->wmediumd);
-	struct sk_buff *skb;
-	void *msg_head;
-
-	WARN_ON(!is_valid_ether_addr(addr));
-
-	if (!_portid && !hwsim_virtio_enabled)
-		return;
-
-	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_ATOMIC);
-	if (!skb)
-		return;
-
-	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0,
-			       add ? HWSIM_CMD_ADD_MAC_ADDR :
-				     HWSIM_CMD_DEL_MAC_ADDR);
-	if (!msg_head) {
-		pr_debug("mac80211_hwsim: problem with msg_head\n");
-		goto nla_put_failure;
-	}
-
-	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER,
-		    ETH_ALEN, data->addresses[1].addr))
-		goto nla_put_failure;
-
-	if (nla_put(skb, HWSIM_ATTR_ADDR_RECEIVER, ETH_ALEN, addr))
-		goto nla_put_failure;
-
-	genlmsg_end(skb, msg_head);
-
-	if (hwsim_virtio_enabled)
-		hwsim_tx_virtio(data, skb);
-	else
-		hwsim_unicast_netgroup(data, skb, _portid);
-	return;
-nla_put_failure:
-	nlmsg_free(skb);
-}
-
-static inline u16 trans_tx_rate_flags_ieee2hwsim(struct ieee80211_tx_rate *rate)
-{
-	u16 result = 0;
-
-	if (rate->flags & IEEE80211_TX_RC_USE_RTS_CTS)
-		result |= MAC80211_HWSIM_TX_RC_USE_RTS_CTS;
-	if (rate->flags & IEEE80211_TX_RC_USE_CTS_PROTECT)
-		result |= MAC80211_HWSIM_TX_RC_USE_CTS_PROTECT;
-	if (rate->flags & IEEE80211_TX_RC_USE_SHORT_PREAMBLE)
-		result |= MAC80211_HWSIM_TX_RC_USE_SHORT_PREAMBLE;
-	if (rate->flags & IEEE80211_TX_RC_MCS)
-		result |= MAC80211_HWSIM_TX_RC_MCS;
-	if (rate->flags & IEEE80211_TX_RC_GREEN_FIELD)
-		result |= MAC80211_HWSIM_TX_RC_GREEN_FIELD;
-	if (rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
-		result |= MAC80211_HWSIM_TX_RC_40_MHZ_WIDTH;
-	if (rate->flags & IEEE80211_TX_RC_DUP_DATA)
-		result |= MAC80211_HWSIM_TX_RC_DUP_DATA;
-	if (rate->flags & IEEE80211_TX_RC_SHORT_GI)
-		result |= MAC80211_HWSIM_TX_RC_SHORT_GI;
-	if (rate->flags & IEEE80211_TX_RC_VHT_MCS)
-		result |= MAC80211_HWSIM_TX_RC_VHT_MCS;
-	if (rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH)
-		result |= MAC80211_HWSIM_TX_RC_80_MHZ_WIDTH;
-	if (rate->flags & IEEE80211_TX_RC_160_MHZ_WIDTH)
-		result |= MAC80211_HWSIM_TX_RC_160_MHZ_WIDTH;
-
-	return result;
-}
-
-static void mac80211_hwsim_tx_frame_nl(struct ieee80211_hw *hw,
-				       struct sk_buff *my_skb,
-				       int dst_portid,
-				       struct ieee80211_channel *channel)
-{
-	struct sk_buff *skb;
-	struct mac80211_hwsim_data *data = hw->priv;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) my_skb->data;
-	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(my_skb);
-	void *msg_head;
-	unsigned int hwsim_flags = 0;
-	int i;
-	struct hwsim_tx_rate tx_attempts[IEEE80211_TX_MAX_RATES];
-	struct hwsim_tx_rate_flag tx_attempts_flags[IEEE80211_TX_MAX_RATES];
-	uintptr_t cookie;
-
-	if (data->ps != PS_DISABLED)
-		hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
-	/* If the queue contains MAX_QUEUE skb's drop some */
-	if (skb_queue_len(&data->pending) >= MAX_QUEUE) {
-		/* Dropping until WARN_QUEUE level */
-		while (skb_queue_len(&data->pending) >= WARN_QUEUE) {
-			ieee80211_free_txskb(hw, skb_dequeue(&data->pending));
-			data->tx_dropped++;
-		}
-	}
-
-	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_ATOMIC);
-	if (skb == NULL)
-		goto nla_put_failure;
-
-	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0,
-			       HWSIM_CMD_FRAME);
-	if (msg_head == NULL) {
-		pr_debug("mac80211_hwsim: problem with msg_head\n");
-		goto nla_put_failure;
-	}
-
-	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER,
-		    ETH_ALEN, data->addresses[1].addr))
-		goto nla_put_failure;
-
-	/* We get the skb->data */
-	if (nla_put(skb, HWSIM_ATTR_FRAME, my_skb->len, my_skb->data))
-		goto nla_put_failure;
-
-	/* We get the flags for this transmission, and we translate them to
-	   wmediumd flags  */
-
-	if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)
-		hwsim_flags |= HWSIM_TX_CTL_REQ_TX_STATUS;
-
-	if (info->flags & IEEE80211_TX_CTL_NO_ACK)
-		hwsim_flags |= HWSIM_TX_CTL_NO_ACK;
-
-	if (nla_put_u32(skb, HWSIM_ATTR_FLAGS, hwsim_flags))
-		goto nla_put_failure;
-
-	if (nla_put_u32(skb, HWSIM_ATTR_FREQ, channel->center_freq))
-		goto nla_put_failure;
-
-	/* We get the tx control (rate and retries) info*/
-
-	for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) {
-		tx_attempts[i].idx = info->status.rates[i].idx;
-		tx_attempts_flags[i].idx = info->status.rates[i].idx;
-		tx_attempts[i].count = info->status.rates[i].count;
-		tx_attempts_flags[i].flags =
-				trans_tx_rate_flags_ieee2hwsim(
-						&info->status.rates[i]);
-	}
-
-	if (nla_put(skb, HWSIM_ATTR_TX_INFO,
-		    sizeof(struct hwsim_tx_rate)*IEEE80211_TX_MAX_RATES,
-		    tx_attempts))
-		goto nla_put_failure;
-
-	if (nla_put(skb, HWSIM_ATTR_TX_INFO_FLAGS,
-		    sizeof(struct hwsim_tx_rate_flag) * IEEE80211_TX_MAX_RATES,
-		    tx_attempts_flags))
-		goto nla_put_failure;
-
-	/* We create a cookie to identify this skb */
-	cookie = atomic_inc_return(&data->pending_cookie);
-	info->rate_driver_data[0] = (void *)cookie;
-	if (nla_put_u64_64bit(skb, HWSIM_ATTR_COOKIE, cookie, HWSIM_ATTR_PAD))
-		goto nla_put_failure;
-
-	genlmsg_end(skb, msg_head);
-
-	if (hwsim_virtio_enabled) {
-		if (hwsim_tx_virtio(data, skb))
-			goto err_free_txskb;
-	} else {
-		if (hwsim_unicast_netgroup(data, skb, dst_portid))
-			goto err_free_txskb;
-	}
-
-	/* Enqueue the packet */
-	skb_queue_tail(&data->pending, my_skb);
-	data->tx_pkts++;
-	data->tx_bytes += my_skb->len;
-	return;
-
-nla_put_failure:
-	nlmsg_free(skb);
-err_free_txskb:
-	pr_debug("mac80211_hwsim: error occurred in %s\n", __func__);
-	ieee80211_free_txskb(hw, my_skb);
-	data->tx_failed++;
-}
-
-static bool hwsim_chans_compat(struct ieee80211_channel *c1,
-			       struct ieee80211_channel *c2)
-{
-	if (!c1 || !c2)
-		return false;
-
-	return c1->center_freq == c2->center_freq;
-}
-
-struct tx_iter_data {
-	struct ieee80211_channel *channel;
-	bool receive;
-};
-
-static void mac80211_hwsim_tx_iter(void *_data, u8 *addr,
-				   struct ieee80211_vif *vif)
-{
-	struct tx_iter_data *data = _data;
-	int i;
-
-	/* For NAN Device simulation purposes, assume that NAN is always
-	 * on channel 6 or channel 149.
-	 */
-	if (vif->type == NL80211_IFTYPE_NAN) {
-		data->receive = (data->channel &&
-				 (data->channel->center_freq == 2437 ||
-				  data->channel->center_freq == 5745));
-		return;
-	}
-
-	for (i = 0; i < ARRAY_SIZE(vif->link_conf); i++) {
-		struct ieee80211_bss_conf *conf;
-		struct ieee80211_chanctx_conf *chanctx;
-
-		conf = rcu_dereference(vif->link_conf[i]);
-		if (!conf)
-			continue;
-
-		chanctx = rcu_dereference(conf->chanctx_conf);
-		if (!chanctx)
-			continue;
-
-		if (!hwsim_chans_compat(data->channel, chanctx->def.chan))
-			continue;
-
-		data->receive = true;
-		return;
-	}
-}
-
-static void mac80211_hwsim_add_vendor_rtap(struct sk_buff *skb)
-{
-	/*
-	 * To enable this code, #define the HWSIM_RADIOTAP_OUI,
-	 * e.g. like this:
-	 * #define HWSIM_RADIOTAP_OUI "\x02\x00\x00"
-	 * (but you should use a valid OUI, not that)
-	 *
-	 * If anyone wants to 'donate' a radiotap OUI/subns code
-	 * please send a patch removing this #ifdef and changing
-	 * the values accordingly.
-	 */
-#ifdef HWSIM_RADIOTAP_OUI
-	struct ieee80211_radiotap_vendor_tlv *rtap;
-	static const char vendor_data[8] = "ABCDEFGH";
-
-	// Make sure no padding is needed
-	BUILD_BUG_ON(sizeof(vendor_data) % 4);
-	/* this is last radiotap info before the mac header, so
-	 * skb_reset_mac_header for mac8022 to know the end of
-	 * the radiotap TLV/beginning of the 802.11 header
-	 */
-	skb_reset_mac_header(skb);
-
-	/*
-	 * Note that this code requires the headroom in the SKB
-	 * that was allocated earlier.
-	 */
-	rtap = skb_push(skb, sizeof(*rtap) + sizeof(vendor_data));
-
-	rtap->len = cpu_to_le16(sizeof(*rtap) -
-				sizeof(struct ieee80211_radiotap_tlv) +
-				sizeof(vendor_data));
-	rtap->type = cpu_to_le16(IEEE80211_RADIOTAP_VENDOR_NAMESPACE);
-
-	rtap->content.oui[0] = HWSIM_RADIOTAP_OUI[0];
-	rtap->content.oui[1] = HWSIM_RADIOTAP_OUI[1];
-	rtap->content.oui[2] = HWSIM_RADIOTAP_OUI[2];
-	rtap->content.oui_subtype = 127;
-	/* clear reserved field */
-	rtap->content.reserved = 0;
-	rtap->content.vendor_type = 0;
-	memcpy(rtap->content.data, vendor_data, sizeof(vendor_data));
-
-	IEEE80211_SKB_RXCB(skb)->flag |= RX_FLAG_RADIOTAP_TLV_AT_END;
-#endif
-}
-
-static void mac80211_hwsim_rx(struct mac80211_hwsim_data *data,
-			      struct ieee80211_rx_status *rx_status,
-			      struct sk_buff *skb)
-{
-	struct ieee80211_hdr *hdr = (void *)skb->data;
-
-	if (!ieee80211_has_morefrags(hdr->frame_control) &&
-	    !is_multicast_ether_addr(hdr->addr1) &&
-	    (ieee80211_is_mgmt(hdr->frame_control) ||
-	     ieee80211_is_data(hdr->frame_control))) {
-		struct ieee80211_sta *sta;
-		unsigned int link_id;
-
-		rcu_read_lock();
-		sta = ieee80211_find_sta_by_link_addrs(data->hw, hdr->addr2,
-						       hdr->addr1, &link_id);
-		if (sta) {
-			struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
-
-			if (ieee80211_has_pm(hdr->frame_control))
-				sp->active_links_rx &= ~BIT(link_id);
-			else
-				sp->active_links_rx |= BIT(link_id);
-
-			rx_status->link_valid = true;
-			rx_status->link_id = link_id;
-		}
-		rcu_read_unlock();
-	}
-
-	memcpy(IEEE80211_SKB_RXCB(skb), rx_status, sizeof(*rx_status));
-
-	mac80211_hwsim_add_vendor_rtap(skb);
-
-	data->rx_pkts++;
-	data->rx_bytes += skb->len;
-	ieee80211_rx_irqsafe(data->hw, skb);
-}
-
-static bool mac80211_hwsim_tx_frame_no_nl(struct ieee80211_hw *hw,
-					  struct sk_buff *skb,
-					  struct ieee80211_channel *chan)
-{
-	struct mac80211_hwsim_data *data = hw->priv, *data2;
-	bool ack = false;
-	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
-	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
-	struct ieee80211_rx_status rx_status;
-	u64 now;
-
-	memset(&rx_status, 0, sizeof(rx_status));
-	rx_status.flag |= RX_FLAG_MACTIME_START;
-	rx_status.freq = chan->center_freq;
-	rx_status.freq_offset = chan->freq_offset ? 1 : 0;
-	rx_status.band = chan->band;
-	if (info->control.rates[0].flags & IEEE80211_TX_RC_VHT_MCS) {
-		rx_status.rate_idx =
-			ieee80211_rate_get_vht_mcs(&info->control.rates[0]);
-		rx_status.nss =
-			ieee80211_rate_get_vht_nss(&info->control.rates[0]);
-		rx_status.encoding = RX_ENC_VHT;
-	} else {
-		rx_status.rate_idx = info->control.rates[0].idx;
-		if (info->control.rates[0].flags & IEEE80211_TX_RC_MCS)
-			rx_status.encoding = RX_ENC_HT;
-	}
-	if (info->control.rates[0].flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
-		rx_status.bw = RATE_INFO_BW_40;
-	else if (info->control.rates[0].flags & IEEE80211_TX_RC_80_MHZ_WIDTH)
-		rx_status.bw = RATE_INFO_BW_80;
-	else if (info->control.rates[0].flags & IEEE80211_TX_RC_160_MHZ_WIDTH)
-		rx_status.bw = RATE_INFO_BW_160;
-	else
-		rx_status.bw = RATE_INFO_BW_20;
-	if (info->control.rates[0].flags & IEEE80211_TX_RC_SHORT_GI)
-		rx_status.enc_flags |= RX_ENC_FLAG_SHORT_GI;
-	/* TODO: simulate optional packet loss */
-	rx_status.signal = data->rx_rssi;
-	if (info->control.vif)
-		rx_status.signal += info->control.vif->bss_conf.txpower;
-
-	if (data->ps != PS_DISABLED)
-		hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
-
-	/* release the skb's source info */
-	skb_orphan(skb);
-	skb_dst_drop(skb);
-	skb->mark = 0;
-	skb_ext_reset(skb);
-	nf_reset_ct(skb);
-
-	/*
-	 * Get absolute mactime here so all HWs RX at the "same time", and
-	 * absolute TX time for beacon mactime so the timestamp matches.
-	 * Giving beacons a different mactime than non-beacons looks messy, but
-	 * it helps the Toffset be exact and a ~10us mactime discrepancy
-	 * probably doesn't really matter.
-	 */
-	if (ieee80211_is_beacon(hdr->frame_control) ||
-	    ieee80211_is_probe_resp(hdr->frame_control)) {
-		rx_status.boottime_ns = ktime_get_boottime_ns();
-		now = data->abs_bcn_ts;
-	} else {
-		now = mac80211_hwsim_get_tsf_raw();
-	}
-
-	/* Copy skb to all enabled radios that are on the current frequency */
-	spin_lock(&hwsim_radio_lock);
-	list_for_each_entry(data2, &hwsim_radios, list) {
-		struct sk_buff *nskb;
-		struct tx_iter_data tx_iter_data = {
-			.receive = false,
-			.channel = chan,
-		};
-
-		if (data == data2)
-			continue;
-
-		if (!data2->started || (data2->idle && !data2->tmp_chan) ||
-		    !hwsim_ps_rx_ok(data2, skb))
-			continue;
-
-		if (!(data->group & data2->group))
-			continue;
-
-		if (data->netgroup != data2->netgroup)
-			continue;
-
-		if (!hwsim_chans_compat(chan, data2->tmp_chan) &&
-		    !hwsim_chans_compat(chan, data2->channel)) {
-			ieee80211_iterate_active_interfaces_atomic(
-				data2->hw, IEEE80211_IFACE_ITER_NORMAL,
-				mac80211_hwsim_tx_iter, &tx_iter_data);
-			if (!tx_iter_data.receive)
-				continue;
-		}
-
-		/*
-		 * reserve some space for our vendor and the normal
-		 * radiotap header, since we're copying anyway
-		 */
-		if (skb->len < PAGE_SIZE && paged_rx) {
-			struct page *page = alloc_page(GFP_ATOMIC);
-
-			if (!page)
-				continue;
-
-			nskb = dev_alloc_skb(128);
-			if (!nskb) {
-				__free_page(page);
-				continue;
-			}
-
-			memcpy(page_address(page), skb->data, skb->len);
-			skb_add_rx_frag(nskb, 0, page, 0, skb->len, skb->len);
-		} else {
-			nskb = skb_copy(skb, GFP_ATOMIC);
-			if (!nskb)
-				continue;
-		}
-
-		if (mac80211_hwsim_addr_match(data2, hdr->addr1))
-			ack = true;
-
-		rx_status.mactime = now + data2->tsf_offset;
-
-		mac80211_hwsim_rx(data2, &rx_status, nskb);
-	}
-	spin_unlock(&hwsim_radio_lock);
-
-	return ack;
-}
-
-static struct ieee80211_bss_conf *
-mac80211_hwsim_select_tx_link(struct mac80211_hwsim_data *data,
-			      struct ieee80211_vif *vif,
-			      struct ieee80211_sta *sta,
-			      struct ieee80211_hdr *hdr,
-			      struct ieee80211_link_sta **link_sta)
-{
-	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
-	int i;
-
-	if (!ieee80211_vif_is_mld(vif))
-		return &vif->bss_conf;
-
-	WARN_ON(is_multicast_ether_addr(hdr->addr1));
-
-	if (WARN_ON_ONCE(!sta || !sta->valid_links))
-		return &vif->bss_conf;
-
-	for (i = 0; i < ARRAY_SIZE(vif->link_conf); i++) {
-		struct ieee80211_bss_conf *bss_conf;
-		unsigned int link_id;
-
-		/* round-robin the available link IDs */
-		link_id = (sp->last_link + i + 1) % ARRAY_SIZE(vif->link_conf);
-
-		if (!(vif->active_links & BIT(link_id)))
-			continue;
-
-		if (!(sp->active_links_rx & BIT(link_id)))
-			continue;
-
-		*link_sta = rcu_dereference(sta->link[link_id]);
-		if (!*link_sta)
-			continue;
-
-		bss_conf = rcu_dereference(vif->link_conf[link_id]);
-		if (WARN_ON_ONCE(!bss_conf))
-			continue;
-
-		/* can happen while switching links */
-		if (!rcu_access_pointer(bss_conf->chanctx_conf))
-			continue;
-
-		sp->last_link = link_id;
-		return bss_conf;
-	}
-
-	return NULL;
-}
-
-static int mac80211_hwsim_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
-				  struct ieee80211_vif *vif,
-				  struct ieee80211_sta *sta,
-				  struct ieee80211_key_conf *key)
-{
-	switch (key->cipher) {
-	case WLAN_CIPHER_SUITE_CCMP:
-	case WLAN_CIPHER_SUITE_CCMP_256:
-	case WLAN_CIPHER_SUITE_GCMP:
-	case WLAN_CIPHER_SUITE_GCMP_256:
-		break;
-	default:
-		return 1;
-	}
-
-	key->flags |= IEEE80211_KEY_FLAG_RESERVE_TAILROOM;
-	return 0;
-}
-
-static void mac80211_hwsim_tx(struct ieee80211_hw *hw,
-			      struct ieee80211_tx_control *control,
-			      struct sk_buff *skb)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	struct ieee80211_tx_info *txi = IEEE80211_SKB_CB(skb);
-	struct ieee80211_hdr *hdr = (void *)skb->data;
-	struct ieee80211_chanctx_conf *chanctx_conf;
-	struct ieee80211_channel *channel;
-	struct ieee80211_vif *vif = txi->control.vif;
-	bool ack, unicast_data;
-	enum nl80211_chan_width confbw = NL80211_CHAN_WIDTH_20_NOHT;
-	u32 _portid, i;
-
-	if (WARN_ON(skb->len < 10)) {
-		/* Should not happen; just a sanity check for addr1 use */
-		ieee80211_free_txskb(hw, skb);
-		return;
-	}
-
-	unicast_data = is_unicast_ether_addr(hdr->addr1) &&
-		       ieee80211_is_data(hdr->frame_control);
-
-	if (unicast_data && ieee80211_encrypt_tx_skb(skb) < 0) {
-		ieee80211_free_txskb(hw, skb);
-		return;
-	}
-	/* re-assign hdr since skb data may have shifted after encryption */
-	hdr = (void *)skb->data;
-
-	if (vif && vif->type == NL80211_IFTYPE_NAN && !data->tmp_chan) {
-		/* For NAN Device simulation purposes, assume that NAN is always
-		 * on channel 6 or channel 149, unless a ROC is in progress (for
-		 * USD use cases).
-		 */
-		if (data->nan_curr_dw_band == NL80211_BAND_2GHZ)
-			channel = ieee80211_get_channel(hw->wiphy, 2437);
-		else if (data->nan_curr_dw_band == NL80211_BAND_5GHZ)
-			channel = ieee80211_get_channel(hw->wiphy, 5745);
-		else
-			channel = NULL;
-
-		if (WARN_ON(!channel)) {
-			ieee80211_free_txskb(hw, skb);
-			return;
-		}
-	} else if (!data->use_chanctx) {
-		channel = data->channel;
-		confbw = data->bw;
-	} else if (txi->hw_queue == 4) {
-		channel = data->tmp_chan;
-	} else {
-		u8 link = u32_get_bits(IEEE80211_SKB_CB(skb)->control.flags,
-				       IEEE80211_TX_CTRL_MLO_LINK);
-		struct ieee80211_link_sta *link_sta = NULL;
-		struct ieee80211_sta *sta = control->sta;
-		struct ieee80211_bss_conf *bss_conf;
-
-		/* This can happen in case of monitor injection */
-		if (!vif) {
-			ieee80211_free_txskb(hw, skb);
-			return;
-		}
-
-		if (link != IEEE80211_LINK_UNSPECIFIED) {
-			bss_conf = rcu_dereference(vif->link_conf[link]);
-			if (sta)
-				link_sta = rcu_dereference(sta->link[link]);
-		} else {
-			bss_conf = mac80211_hwsim_select_tx_link(data, vif, sta,
-								 hdr, &link_sta);
-		}
-
-		if (unlikely(!bss_conf)) {
-			/* if it's an MLO STA, it might have deactivated all
-			 * links temporarily - but we don't handle real PS in
-			 * this code yet, so just drop the frame in that case
-			 */
-			WARN(link != IEEE80211_LINK_UNSPECIFIED || !sta || !sta->mlo,
-			     "link:%d, sta:%pM, sta->mlo:%d\n",
-			     link, sta ? sta->addr : NULL, sta ? sta->mlo : -1);
-			ieee80211_free_txskb(hw, skb);
-			return;
-		}
-
-		/* Do address translations only between shared links. It is
-		 * possible that while an non-AP MLD station and an AP MLD
-		 * station have shared links, the frame is intended to be sent
-		 * on a link which is not shared (for example when sending a
-		 * probe response).
-		 */
-		if (sta && sta->mlo && link_sta) {
-			/* address translation to link addresses on TX */
-			ether_addr_copy(hdr->addr1, link_sta->addr);
-			ether_addr_copy(hdr->addr2, bss_conf->addr);
-			/* translate A3 only if it's the BSSID */
-			if (!ieee80211_has_tods(hdr->frame_control) &&
-			    !ieee80211_has_fromds(hdr->frame_control)) {
-				if (ether_addr_equal(hdr->addr3, sta->addr))
-					ether_addr_copy(hdr->addr3, link_sta->addr);
-				else if (ether_addr_equal(hdr->addr3, vif->addr))
-					ether_addr_copy(hdr->addr3, bss_conf->addr);
-			}
-			/* no need to look at A4, if present it's SA */
-		}
-
-		chanctx_conf = rcu_dereference(bss_conf->chanctx_conf);
-		if (chanctx_conf) {
-			channel = chanctx_conf->def.chan;
-			confbw = chanctx_conf->def.width;
-		} else {
-			channel = NULL;
-		}
-	}
-
-	if (!unicast_data && ieee80211_encrypt_tx_skb(skb) < 0) {
-		ieee80211_free_txskb(hw, skb);
-		return;
-	}
-	/* re-assign hdr since skb data may have shifted after encryption */
-	hdr = (void *)skb->data;
-
-	if (WARN(!channel, "TX w/o channel - queue = %d\n", txi->hw_queue)) {
-		ieee80211_free_txskb(hw, skb);
-		return;
-	}
-
-	if (data->idle && !data->tmp_chan) {
-		wiphy_dbg(hw->wiphy, "Trying to TX when idle - reject\n");
-		ieee80211_free_txskb(hw, skb);
-		return;
-	}
-
-	if (vif)
-		hwsim_check_magic(vif);
-	if (control->sta)
-		hwsim_check_sta_magic(control->sta);
-
-	if (ieee80211_hw_check(hw, SUPPORTS_RC_TABLE))
-		ieee80211_get_tx_rates(vif, control->sta, skb,
-				       txi->control.rates,
-				       ARRAY_SIZE(txi->control.rates));
-
-	for (i = 0; i < ARRAY_SIZE(txi->control.rates); i++) {
-		u16 rflags = txi->control.rates[i].flags;
-		/* initialize to data->bw for 5/10 MHz handling */
-		enum nl80211_chan_width bw = data->bw;
-
-		if (txi->control.rates[i].idx == -1)
-			break;
-
-		if (rflags & IEEE80211_TX_RC_40_MHZ_WIDTH)
-			bw = NL80211_CHAN_WIDTH_40;
-		else if (rflags & IEEE80211_TX_RC_80_MHZ_WIDTH)
-			bw = NL80211_CHAN_WIDTH_80;
-		else if (rflags & IEEE80211_TX_RC_160_MHZ_WIDTH)
-			bw = NL80211_CHAN_WIDTH_160;
-
-		if (WARN_ON(hwsim_get_chanwidth(bw) > hwsim_get_chanwidth(confbw)))
-			return;
-	}
-
-	if (skb->len >= 24 + 8 &&
-	    ieee80211_is_probe_resp(hdr->frame_control)) {
-		/* fake header transmission time */
-		struct ieee80211_mgmt *mgmt;
-		struct ieee80211_rate *txrate;
-		/* TODO: get MCS */
-		int bitrate = 100;
-		u64 ts;
-
-		mgmt = (struct ieee80211_mgmt *)skb->data;
-		txrate = ieee80211_get_tx_rate(hw, txi);
-		if (txrate)
-			bitrate = txrate->bitrate;
-		ts = mac80211_hwsim_get_tsf_raw();
-		mgmt->u.probe_resp.timestamp =
-			cpu_to_le64(ts + data->tsf_offset +
-				    24 * 8 * 10 / bitrate);
-	}
-
-	mac80211_hwsim_monitor_rx(hw, skb, channel);
-
-	/* wmediumd mode check */
-	_portid = READ_ONCE(data->wmediumd);
-
-	if (_portid || hwsim_virtio_enabled)
-		return mac80211_hwsim_tx_frame_nl(hw, skb, _portid, channel);
-
-	/* NO wmediumd detected, perfect medium simulation */
-	data->tx_pkts++;
-	data->tx_bytes += skb->len;
-	ack = mac80211_hwsim_tx_frame_no_nl(hw, skb, channel);
-
-	if (ack && skb->len >= 16)
-		mac80211_hwsim_monitor_ack(channel, hdr->addr2);
-
-	ieee80211_tx_info_clear_status(txi);
-
-	/* frame was transmitted at most favorable rate at first attempt */
-	txi->control.rates[0].count = 1;
-	txi->control.rates[1].idx = -1;
-
-	if (!(txi->flags & IEEE80211_TX_CTL_NO_ACK) && ack)
-		txi->flags |= IEEE80211_TX_STAT_ACK;
-	ieee80211_tx_status_irqsafe(hw, skb);
-}
-
-
-static int mac80211_hwsim_start(struct ieee80211_hw *hw)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	wiphy_dbg(hw->wiphy, "%s\n", __func__);
-	data->started = true;
-	return 0;
-}
-
-
-static void mac80211_hwsim_stop(struct ieee80211_hw *hw, bool suspend)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	int i;
-
-	data->started = false;
-
-	for (i = 0; i < ARRAY_SIZE(data->link_data); i++)
-		hrtimer_cancel(&data->link_data[i].beacon_timer);
-
-	while (!skb_queue_empty(&data->pending))
-		ieee80211_free_txskb(hw, skb_dequeue(&data->pending));
-
-	wiphy_dbg(hw->wiphy, "%s\n", __func__);
-}
-
-
-static int mac80211_hwsim_add_interface(struct ieee80211_hw *hw,
-					struct ieee80211_vif *vif)
-{
-	wiphy_dbg(hw->wiphy, "%s (type=%d mac_addr=%pM)\n",
-		  __func__, ieee80211_vif_type_p2p(vif),
-		  vif->addr);
-	hwsim_set_magic(vif);
-
-	if (vif->type != NL80211_IFTYPE_MONITOR)
-		mac80211_hwsim_config_mac_nl(hw, vif->addr, true);
-
-	vif->cab_queue = 0;
-	vif->hw_queue[IEEE80211_AC_VO] = 0;
-	vif->hw_queue[IEEE80211_AC_VI] = 1;
-	vif->hw_queue[IEEE80211_AC_BE] = 2;
-	vif->hw_queue[IEEE80211_AC_BK] = 3;
-
-	return 0;
-}
-
-#ifdef CONFIG_MAC80211_DEBUGFS
-static void
-mac80211_hwsim_link_add_debugfs(struct ieee80211_hw *hw,
-				struct ieee80211_vif *vif,
-				struct ieee80211_bss_conf *link_conf,
-				struct dentry *dir)
-{
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-
-	debugfs_create_u32("skip_beacons", 0600, dir,
-			   &vp->skip_beacons[link_conf->link_id]);
-}
-#endif
-
-static int mac80211_hwsim_change_interface(struct ieee80211_hw *hw,
-					   struct ieee80211_vif *vif,
-					   enum nl80211_iftype newtype,
-					   bool newp2p)
-{
-	newtype = ieee80211_iftype_p2p(newtype, newp2p);
-	wiphy_dbg(hw->wiphy,
-		  "%s (old type=%d, new type=%d, mac_addr=%pM)\n",
-		  __func__, ieee80211_vif_type_p2p(vif),
-		    newtype, vif->addr);
-	hwsim_check_magic(vif);
-
-	/*
-	 * interface may change from non-AP to AP in
-	 * which case this needs to be set up again
-	 */
-	vif->cab_queue = 0;
-
-	return 0;
-}
-
-static void mac80211_hwsim_remove_interface(
-	struct ieee80211_hw *hw, struct ieee80211_vif *vif)
-{
-	wiphy_dbg(hw->wiphy, "%s (type=%d mac_addr=%pM)\n",
-		  __func__, ieee80211_vif_type_p2p(vif),
-		  vif->addr);
-	hwsim_check_magic(vif);
-	hwsim_clear_magic(vif);
-	if (vif->type != NL80211_IFTYPE_MONITOR)
-		mac80211_hwsim_config_mac_nl(hw, vif->addr, false);
-}
-
-static void mac80211_hwsim_tx_frame(struct ieee80211_hw *hw,
-				    struct sk_buff *skb,
-				    struct ieee80211_channel *chan)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	u32 _portid = READ_ONCE(data->wmediumd);
-
-	if (ieee80211_hw_check(hw, SUPPORTS_RC_TABLE)) {
-		struct ieee80211_tx_info *txi = IEEE80211_SKB_CB(skb);
-		ieee80211_get_tx_rates(txi->control.vif, NULL, skb,
-				       txi->control.rates,
-				       ARRAY_SIZE(txi->control.rates));
-	}
-
-	mac80211_hwsim_monitor_rx(hw, skb, chan);
-
-	if (_portid || hwsim_virtio_enabled)
-		return mac80211_hwsim_tx_frame_nl(hw, skb, _portid, chan);
-
-	data->tx_pkts++;
-	data->tx_bytes += skb->len;
-	mac80211_hwsim_tx_frame_no_nl(hw, skb, chan);
-	dev_kfree_skb(skb);
-}
-
-static void __mac80211_hwsim_beacon_tx(struct ieee80211_bss_conf *link_conf,
-				       struct mac80211_hwsim_data *data,
-				       struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif,
-				       struct sk_buff *skb)
-{
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-	struct ieee80211_tx_info *info;
-	struct ieee80211_rate *txrate;
-	struct ieee80211_mgmt *mgmt;
-	/* TODO: get MCS */
-	int bitrate = 100;
-
-	if (vp->skip_beacons[link_conf->link_id]) {
-		vp->skip_beacons[link_conf->link_id]--;
-		dev_kfree_skb(skb);
-		return;
-	}
-
-	info = IEEE80211_SKB_CB(skb);
-	if (ieee80211_hw_check(hw, SUPPORTS_RC_TABLE))
-		ieee80211_get_tx_rates(vif, NULL, skb,
-				       info->control.rates,
-				       ARRAY_SIZE(info->control.rates));
-
-	txrate = ieee80211_get_tx_rate(hw, info);
-	if (txrate)
-		bitrate = txrate->bitrate;
-
-	mgmt = (struct ieee80211_mgmt *) skb->data;
-	/* fake header transmission time */
-	data->abs_bcn_ts = mac80211_hwsim_get_tsf_raw();
-	if (ieee80211_is_s1g_beacon(mgmt->frame_control)) {
-		struct ieee80211_ext *ext = (void *) mgmt;
-
-		ext->u.s1g_beacon.timestamp = cpu_to_le32(data->abs_bcn_ts +
-							  data->tsf_offset +
-							  10 * 8 * 10 /
-							  bitrate);
-	} else {
-		mgmt->u.beacon.timestamp = cpu_to_le64(data->abs_bcn_ts +
-						       data->tsf_offset +
-						       24 * 8 * 10 /
-						       bitrate);
-	}
-
-	mac80211_hwsim_tx_frame(hw, skb,
-			rcu_dereference(link_conf->chanctx_conf)->def.chan);
-}
-
-static void mac80211_hwsim_beacon_tx(void *arg, u8 *mac,
-				     struct ieee80211_vif *vif)
-{
-	struct mac80211_hwsim_link_data *link_data = arg;
-	u32 link_id = link_data->link_id;
-	struct ieee80211_bss_conf *link_conf, *tx_bss_conf;
-	struct mac80211_hwsim_data *data =
-		container_of(link_data, struct mac80211_hwsim_data,
-			     link_data[link_id]);
-	struct ieee80211_hw *hw = data->hw;
-	struct sk_buff *skb;
-
-	hwsim_check_magic(vif);
-
-	link_conf = rcu_dereference(vif->link_conf[link_id]);
-	if (!link_conf)
-		return;
-
-	if (vif->type != NL80211_IFTYPE_AP &&
-	    vif->type != NL80211_IFTYPE_MESH_POINT &&
-	    vif->type != NL80211_IFTYPE_ADHOC &&
-	    vif->type != NL80211_IFTYPE_OCB)
-		return;
-
-	tx_bss_conf = rcu_access_pointer(link_conf->tx_bss_conf);
-	if (tx_bss_conf && tx_bss_conf != link_conf)
-		return;
-
-	if (link_conf->ema_ap) {
-		struct ieee80211_ema_beacons *ema;
-		u8 i = 0;
-
-		ema = ieee80211_beacon_get_template_ema_list(hw, vif, link_id);
-		if (!ema || !ema->cnt)
-			return;
-
-		for (i = 0; i < ema->cnt; i++) {
-			__mac80211_hwsim_beacon_tx(link_conf, data, hw, vif,
-						   ema->bcn[i].skb);
-			ema->bcn[i].skb = NULL; /* Already freed */
-		}
-		ieee80211_beacon_free_ema_list(ema);
-	} else {
-		skb = ieee80211_beacon_get(hw, vif, link_id);
-		if (!skb)
-			return;
-
-		__mac80211_hwsim_beacon_tx(link_conf, data, hw, vif, skb);
-	}
-
-	while ((skb = ieee80211_get_buffered_bc(hw, vif)) != NULL) {
-		mac80211_hwsim_tx_frame(hw, skb,
-			rcu_dereference(link_conf->chanctx_conf)->def.chan);
-	}
-
-	if (link_conf->csa_active && ieee80211_beacon_cntdwn_is_complete(vif, link_id))
-		ieee80211_csa_finish(vif, link_id);
-
-	if (link_conf->color_change_active &&
-	    ieee80211_beacon_cntdwn_is_complete(vif, link_id))
-		ieee80211_color_change_finish(vif, link_id);
-}
-
-static enum hrtimer_restart
-mac80211_hwsim_beacon(struct hrtimer *timer)
-{
-	struct mac80211_hwsim_link_data *link_data =
-		container_of(timer, struct mac80211_hwsim_link_data, beacon_timer);
-	struct mac80211_hwsim_data *data =
-		container_of(link_data, struct mac80211_hwsim_data,
-			     link_data[link_data->link_id]);
-	struct ieee80211_hw *hw = data->hw;
-	u64 bcn_int = link_data->beacon_int;
-
-	if (!data->started)
-		return HRTIMER_NORESTART;
-
-	ieee80211_iterate_active_interfaces_atomic(
-		hw, IEEE80211_IFACE_ITER_NORMAL,
-		mac80211_hwsim_beacon_tx, link_data);
-
-	/* beacon at new TBTT + beacon interval */
-	if (data->bcn_delta) {
-		bcn_int -= data->bcn_delta;
-		data->bcn_delta = 0;
-	}
-	hrtimer_forward_now(&link_data->beacon_timer,
-			    ns_to_ktime(bcn_int * NSEC_PER_USEC));
-	return HRTIMER_RESTART;
-}
-
-static const char * const hwsim_chanwidths[] = {
-	[NL80211_CHAN_WIDTH_5] = "ht5",
-	[NL80211_CHAN_WIDTH_10] = "ht10",
-	[NL80211_CHAN_WIDTH_20_NOHT] = "noht",
-	[NL80211_CHAN_WIDTH_20] = "ht20",
-	[NL80211_CHAN_WIDTH_40] = "ht40",
-	[NL80211_CHAN_WIDTH_80] = "vht80",
-	[NL80211_CHAN_WIDTH_80P80] = "vht80p80",
-	[NL80211_CHAN_WIDTH_160] = "vht160",
-	[NL80211_CHAN_WIDTH_1] = "1MHz",
-	[NL80211_CHAN_WIDTH_2] = "2MHz",
-	[NL80211_CHAN_WIDTH_4] = "4MHz",
-	[NL80211_CHAN_WIDTH_8] = "8MHz",
-	[NL80211_CHAN_WIDTH_16] = "16MHz",
-	[NL80211_CHAN_WIDTH_320] = "eht320",
-};
-
-static int mac80211_hwsim_config(struct ieee80211_hw *hw, int radio_idx,
-				 u32 changed)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	struct ieee80211_conf *conf = &hw->conf;
-	static const char *smps_modes[IEEE80211_SMPS_NUM_MODES] = {
-		[IEEE80211_SMPS_AUTOMATIC] = "auto",
-		[IEEE80211_SMPS_OFF] = "off",
-		[IEEE80211_SMPS_STATIC] = "static",
-		[IEEE80211_SMPS_DYNAMIC] = "dynamic",
-	};
-	int idx;
-
-	if (conf->chandef.chan)
-		wiphy_dbg(hw->wiphy,
-			  "%s (freq=%d(%d - %d)/%s idle=%d ps=%d smps=%s)\n",
-			  __func__,
-			  conf->chandef.chan->center_freq,
-			  conf->chandef.center_freq1,
-			  conf->chandef.center_freq2,
-			  hwsim_chanwidths[conf->chandef.width],
-			  !!(conf->flags & IEEE80211_CONF_IDLE),
-			  !!(conf->flags & IEEE80211_CONF_PS),
-			  smps_modes[conf->smps_mode]);
-	else
-		wiphy_dbg(hw->wiphy,
-			  "%s (freq=0 idle=%d ps=%d smps=%s)\n",
-			  __func__,
-			  !!(conf->flags & IEEE80211_CONF_IDLE),
-			  !!(conf->flags & IEEE80211_CONF_PS),
-			  smps_modes[conf->smps_mode]);
-
-	data->idle = !!(conf->flags & IEEE80211_CONF_IDLE);
-
-	WARN_ON(conf->chandef.chan && data->use_chanctx);
-
-	mutex_lock(&data->mutex);
-	if (data->scanning && conf->chandef.chan) {
-		for (idx = 0; idx < ARRAY_SIZE(data->survey_data); idx++) {
-			if (data->survey_data[idx].channel == data->channel) {
-				data->survey_data[idx].start =
-					data->survey_data[idx].next_start;
-				data->survey_data[idx].end = jiffies;
-				break;
-			}
-		}
-
-		data->channel = conf->chandef.chan;
-		data->bw = conf->chandef.width;
-
-		for (idx = 0; idx < ARRAY_SIZE(data->survey_data); idx++) {
-			if (data->survey_data[idx].channel &&
-			    data->survey_data[idx].channel != data->channel)
-				continue;
-			data->survey_data[idx].channel = data->channel;
-			data->survey_data[idx].next_start = jiffies;
-			break;
-		}
-	} else {
-		data->channel = conf->chandef.chan;
-		data->bw = conf->chandef.width;
-	}
-	mutex_unlock(&data->mutex);
-
-	for (idx = 0; idx < ARRAY_SIZE(data->link_data); idx++) {
-		struct mac80211_hwsim_link_data *link_data =
-			&data->link_data[idx];
-
-		if (!data->started || !link_data->beacon_int) {
-			hrtimer_cancel(&link_data->beacon_timer);
-		} else if (!hrtimer_active(&link_data->beacon_timer)) {
-			u64 tsf = mac80211_hwsim_get_tsf(hw, NULL);
-			u32 bcn_int = link_data->beacon_int;
-			u64 until_tbtt = bcn_int - do_div(tsf, bcn_int);
-
-			hrtimer_start(&link_data->beacon_timer,
-				      ns_to_ktime(until_tbtt * NSEC_PER_USEC),
-				      HRTIMER_MODE_REL_SOFT);
-		}
-	}
-
-	return 0;
-}
-
-
-static void mac80211_hwsim_configure_filter(struct ieee80211_hw *hw,
-					    unsigned int changed_flags,
-					    unsigned int *total_flags,u64 multicast)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-
-	wiphy_dbg(hw->wiphy, "%s\n", __func__);
-
-	data->rx_filter = 0;
-	if (*total_flags & FIF_ALLMULTI)
-		data->rx_filter |= FIF_ALLMULTI;
-	if (*total_flags & FIF_MCAST_ACTION)
-		data->rx_filter |= FIF_MCAST_ACTION;
-
-	*total_flags = data->rx_filter;
-}
-
-static void mac80211_hwsim_bcn_en_iter(void *data, u8 *mac,
-				       struct ieee80211_vif *vif)
-{
-	unsigned int *count = data;
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-
-	if (vp->bcn_en)
-		(*count)++;
-}
-
-static void mac80211_hwsim_vif_info_changed(struct ieee80211_hw *hw,
-					    struct ieee80211_vif *vif,
-					    u64 changed)
-{
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-
-	hwsim_check_magic(vif);
-
-	wiphy_dbg(hw->wiphy, "%s(changed=0x%llx vif->addr=%pM)\n",
-		  __func__, changed, vif->addr);
-
-	if (changed & BSS_CHANGED_ASSOC) {
-		wiphy_dbg(hw->wiphy, "  ASSOC: assoc=%d aid=%d\n",
-			  vif->cfg.assoc, vif->cfg.aid);
-		vp->assoc = vif->cfg.assoc;
-		vp->aid = vif->cfg.aid;
-	}
-
-	if (vif->type == NL80211_IFTYPE_STATION &&
-	    changed & (BSS_CHANGED_MLD_VALID_LINKS | BSS_CHANGED_MLD_TTLM)) {
-		u16 usable_links = ieee80211_vif_usable_links(vif);
-
-		if (vif->active_links != usable_links)
-			ieee80211_set_active_links_async(vif, usable_links);
-	}
-}
-
-static void mac80211_hwsim_link_info_changed(struct ieee80211_hw *hw,
-					     struct ieee80211_vif *vif,
-					     struct ieee80211_bss_conf *info,
-					     u64 changed)
-{
-	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
-	struct mac80211_hwsim_data *data = hw->priv;
-	unsigned int link_id = info->link_id;
-	struct mac80211_hwsim_link_data *link_data = &data->link_data[link_id];
-
-	hwsim_check_magic(vif);
-
-	wiphy_dbg(hw->wiphy, "%s(changed=0x%llx vif->addr=%pM, link id %u)\n",
-		  __func__, (unsigned long long)changed, vif->addr, link_id);
-
-	if (changed & BSS_CHANGED_BSSID) {
-		wiphy_dbg(hw->wiphy, "%s: BSSID changed: %pM\n",
-			  __func__, info->bssid);
-		memcpy(vp->bssid, info->bssid, ETH_ALEN);
-	}
-
-	if (changed & BSS_CHANGED_BEACON_ENABLED) {
-		wiphy_dbg(hw->wiphy, "  BCN EN: %d (BI=%u)\n",
-			  info->enable_beacon, info->beacon_int);
-		vp->bcn_en = info->enable_beacon;
-		if (data->started &&
-		    !hrtimer_active(&link_data->beacon_timer) &&
-		    info->enable_beacon) {
-			u64 tsf, until_tbtt;
-			u32 bcn_int;
-			link_data->beacon_int = info->beacon_int * 1024;
-			tsf = mac80211_hwsim_get_tsf(hw, vif);
-			bcn_int = link_data->beacon_int;
-			until_tbtt = bcn_int - do_div(tsf, bcn_int);
-
-			hrtimer_start(&link_data->beacon_timer,
-				      ns_to_ktime(until_tbtt * NSEC_PER_USEC),
-				      HRTIMER_MODE_REL_SOFT);
-		} else if (!info->enable_beacon) {
-			unsigned int count = 0;
-			ieee80211_iterate_active_interfaces_atomic(
-				data->hw, IEEE80211_IFACE_ITER_NORMAL,
-				mac80211_hwsim_bcn_en_iter, &count);
-			wiphy_dbg(hw->wiphy, "  beaconing vifs remaining: %u",
-				  count);
-			if (count == 0) {
-				hrtimer_cancel(&link_data->beacon_timer);
-				link_data->beacon_int = 0;
-			}
-		}
-	}
-
-	if (changed & BSS_CHANGED_ERP_CTS_PROT) {
-		wiphy_dbg(hw->wiphy, "  ERP_CTS_PROT: %d\n",
-			  info->use_cts_prot);
-	}
-
-	if (changed & BSS_CHANGED_ERP_PREAMBLE) {
-		wiphy_dbg(hw->wiphy, "  ERP_PREAMBLE: %d\n",
-			  info->use_short_preamble);
-	}
-
-	if (changed & BSS_CHANGED_ERP_SLOT) {
-		wiphy_dbg(hw->wiphy, "  ERP_SLOT: %d\n", info->use_short_slot);
-	}
-
-	if (changed & BSS_CHANGED_HT) {
-		wiphy_dbg(hw->wiphy, "  HT: op_mode=0x%x\n",
-			  info->ht_operation_mode);
-	}
-
-	if (changed & BSS_CHANGED_BASIC_RATES) {
-		wiphy_dbg(hw->wiphy, "  BASIC_RATES: 0x%llx\n",
-			  (unsigned long long) info->basic_rates);
-	}
-
-	if (changed & BSS_CHANGED_TXPOWER)
-		wiphy_dbg(hw->wiphy, "  TX Power: %d dBm\n", info->txpower);
-}
-
-static void
-mac80211_hwsim_sta_rc_update(struct ieee80211_hw *hw,
-			     struct ieee80211_vif *vif,
-			     struct ieee80211_link_sta *link_sta,
-			     u32 changed)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	struct ieee80211_sta *sta = link_sta->sta;
-	u32 bw = U32_MAX;
-	int link_id;
-
-	rcu_read_lock();
-	for (link_id = 0;
-	     link_id < ARRAY_SIZE(vif->link_conf);
-	     link_id++) {
-		enum nl80211_chan_width confbw = NL80211_CHAN_WIDTH_20_NOHT;
-		struct ieee80211_bss_conf *vif_conf;
-
-		link_sta = rcu_dereference(sta->link[link_id]);
-
-		if (!link_sta)
-			continue;
-
-		switch (link_sta->bandwidth) {
-#define C(_bw) case IEEE80211_STA_RX_BW_##_bw: bw = _bw; break
-		C(20);
-		C(40);
-		C(80);
-		C(160);
-		C(320);
-#undef C
-		}
-
-		if (!data->use_chanctx) {
-			confbw = data->bw;
-		} else {
-			struct ieee80211_chanctx_conf *chanctx_conf;
-
-			vif_conf = rcu_dereference(vif->link_conf[link_id]);
-			if (WARN_ON(!vif_conf))
-				continue;
-
-			chanctx_conf = rcu_dereference(vif_conf->chanctx_conf);
-
-			if (!WARN_ON(!chanctx_conf))
-				confbw = chanctx_conf->def.width;
-		}
-
-		WARN(bw > hwsim_get_chanwidth(confbw),
-		     "intf %pM [link=%d]: bad STA %pM bandwidth %d MHz (%d) > channel config %d MHz (%d)\n",
-		     vif->addr, link_id, sta->addr, bw, sta->deflink.bandwidth,
-		     hwsim_get_chanwidth(data->bw), data->bw);
-
-
-	}
-	rcu_read_unlock();
-
-
-}
-
-static int mac80211_hwsim_sta_add(struct ieee80211_hw *hw,
-				  struct ieee80211_vif *vif,
-				  struct ieee80211_sta *sta)
-{
-	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
-
-	hwsim_check_magic(vif);
-	hwsim_set_sta_magic(sta);
-	mac80211_hwsim_sta_rc_update(hw, vif, &sta->deflink, 0);
-
-	if (sta->valid_links) {
-		WARN(hweight16(sta->valid_links) > 1,
-		     "expect to add STA with single link, have 0x%x\n",
-		     sta->valid_links);
-		sp->active_links_rx = sta->valid_links;
-	}
-
-	return 0;
-}
-
-static int mac80211_hwsim_sta_remove(struct ieee80211_hw *hw,
-				     struct ieee80211_vif *vif,
-				     struct ieee80211_sta *sta)
-{
-	hwsim_check_magic(vif);
-	hwsim_clear_sta_magic(sta);
-
-	return 0;
-}
-
-static int mac80211_hwsim_sta_state(struct ieee80211_hw *hw,
-				    struct ieee80211_vif *vif,
-				    struct ieee80211_sta *sta,
-				    enum ieee80211_sta_state old_state,
-				    enum ieee80211_sta_state new_state)
-{
-	if (new_state == IEEE80211_STA_NOTEXIST)
-		return mac80211_hwsim_sta_remove(hw, vif, sta);
-
-	if (old_state == IEEE80211_STA_NOTEXIST)
-		return mac80211_hwsim_sta_add(hw, vif, sta);
-
-	/*
-	 * in an MLO connection, when client is authorized
-	 * (AP station marked as such), enable all links
-	 */
-	if (ieee80211_vif_is_mld(vif) &&
-	    vif->type == NL80211_IFTYPE_STATION &&
-	    new_state == IEEE80211_STA_AUTHORIZED && !sta->tdls)
-		ieee80211_set_active_links_async(vif,
-						 ieee80211_vif_usable_links(vif));
-
-	return 0;
-}
-
-static void mac80211_hwsim_sta_notify(struct ieee80211_hw *hw,
-				      struct ieee80211_vif *vif,
-				      enum sta_notify_cmd cmd,
-				      struct ieee80211_sta *sta)
-{
-	hwsim_check_magic(vif);
-
-	switch (cmd) {
-	case STA_NOTIFY_SLEEP:
-	case STA_NOTIFY_AWAKE:
-		/* TODO: make good use of these flags */
-		break;
-	default:
-		WARN(1, "Invalid sta notify: %d\n", cmd);
-		break;
-	}
-}
-
-static int mac80211_hwsim_set_tim(struct ieee80211_hw *hw,
-				  struct ieee80211_sta *sta,
-				  bool set)
-{
-	hwsim_check_sta_magic(sta);
-	return 0;
-}
-
-static int mac80211_hwsim_conf_tx(struct ieee80211_hw *hw,
-				  struct ieee80211_vif *vif,
-				  unsigned int link_id, u16 queue,
-				  const struct ieee80211_tx_queue_params *params)
-{
-	wiphy_dbg(hw->wiphy,
-		  "%s (queue=%d txop=%d cw_min=%d cw_max=%d aifs=%d)\n",
-		  __func__, queue,
-		  params->txop, params->cw_min,
-		  params->cw_max, params->aifs);
-	return 0;
-}
-
-static int mac80211_hwsim_get_survey(struct ieee80211_hw *hw, int idx,
-				     struct survey_info *survey)
-{
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-
-	if (idx < 0 || idx >= ARRAY_SIZE(hwsim->survey_data))
-		return -ENOENT;
-
-	mutex_lock(&hwsim->mutex);
-	survey->channel = hwsim->survey_data[idx].channel;
-	if (!survey->channel) {
-		mutex_unlock(&hwsim->mutex);
-		return -ENOENT;
-	}
-
-	/*
-	 * Magically conjured dummy values --- this is only ok for simulated hardware.
-	 *
-	 * A real driver which cannot determine real values noise MUST NOT
-	 * report any, especially not a magically conjured ones :-)
-	 */
-	survey->filled = SURVEY_INFO_NOISE_DBM |
-			 SURVEY_INFO_TIME |
-			 SURVEY_INFO_TIME_BUSY;
-	survey->noise = -92;
-	survey->time =
-		jiffies_to_msecs(hwsim->survey_data[idx].end -
-				 hwsim->survey_data[idx].start);
-	/* report 12.5% of channel time is used */
-	survey->time_busy = survey->time/8;
-	mutex_unlock(&hwsim->mutex);
-
-	return 0;
-}
-
-static enum ieee80211_neg_ttlm_res
-mac80211_hwsim_can_neg_ttlm(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
-			    struct ieee80211_neg_ttlm *neg_ttlm)
-{
-	u32 i;
-
-	/* For testing purposes, accept if all TIDs are mapped to the same links
-	 * set, otherwise reject.
-	 */
-	for (i = 0; i < IEEE80211_TTLM_NUM_TIDS; i++) {
-		if (neg_ttlm->downlink[i] != neg_ttlm->uplink[i] ||
-		    neg_ttlm->downlink[i] != neg_ttlm->downlink[0])
-			return NEG_TTLM_RES_REJECT;
-	}
-
-	return NEG_TTLM_RES_ACCEPT;
-}
-
-#ifdef CONFIG_NL80211_TESTMODE
-/*
- * This section contains example code for using netlink
- * attributes with the testmode command in nl80211.
- */
-
-/* These enums need to be kept in sync with userspace */
-enum hwsim_testmode_attr {
-	__HWSIM_TM_ATTR_INVALID	= 0,
-	HWSIM_TM_ATTR_CMD	= 1,
-	HWSIM_TM_ATTR_PS	= 2,
-
-	/* keep last */
-	__HWSIM_TM_ATTR_AFTER_LAST,
-	HWSIM_TM_ATTR_MAX	= __HWSIM_TM_ATTR_AFTER_LAST - 1
-};
-
-enum hwsim_testmode_cmd {
-	HWSIM_TM_CMD_SET_PS		= 0,
-	HWSIM_TM_CMD_GET_PS		= 1,
-	HWSIM_TM_CMD_STOP_QUEUES	= 2,
-	HWSIM_TM_CMD_WAKE_QUEUES	= 3,
-};
-
-static const struct nla_policy hwsim_testmode_policy[HWSIM_TM_ATTR_MAX + 1] = {
-	[HWSIM_TM_ATTR_CMD] = { .type = NLA_U32 },
-	[HWSIM_TM_ATTR_PS] = { .type = NLA_U32 },
-};
-
-static int mac80211_hwsim_testmode_cmd(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif,
-				       void *data, int len)
-{
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-	struct nlattr *tb[HWSIM_TM_ATTR_MAX + 1];
-	struct sk_buff *skb;
-	int err, ps;
-
-	err = nla_parse_deprecated(tb, HWSIM_TM_ATTR_MAX, data, len,
-				   hwsim_testmode_policy, NULL);
-	if (err)
-		return err;
-
-	if (!tb[HWSIM_TM_ATTR_CMD])
-		return -EINVAL;
-
-	switch (nla_get_u32(tb[HWSIM_TM_ATTR_CMD])) {
-	case HWSIM_TM_CMD_SET_PS:
-		if (!tb[HWSIM_TM_ATTR_PS])
-			return -EINVAL;
-		ps = nla_get_u32(tb[HWSIM_TM_ATTR_PS]);
-		return hwsim_fops_ps_write(hwsim, ps);
-	case HWSIM_TM_CMD_GET_PS:
-		skb = cfg80211_testmode_alloc_reply_skb(hw->wiphy,
-						nla_total_size(sizeof(u32)));
-		if (!skb)
-			return -ENOMEM;
-		if (nla_put_u32(skb, HWSIM_TM_ATTR_PS, hwsim->ps))
-			goto nla_put_failure;
-		return cfg80211_testmode_reply(skb);
-	case HWSIM_TM_CMD_STOP_QUEUES:
-		ieee80211_stop_queues(hw);
-		return 0;
-	case HWSIM_TM_CMD_WAKE_QUEUES:
-		ieee80211_wake_queues(hw);
-		return 0;
-	default:
-		return -EOPNOTSUPP;
-	}
-
- nla_put_failure:
-	kfree_skb(skb);
-	return -ENOBUFS;
-}
-#endif
-
-static int mac80211_hwsim_ampdu_action(struct ieee80211_hw *hw,
-				       struct ieee80211_vif *vif,
-				       struct ieee80211_ampdu_params *params)
-{
-	struct ieee80211_sta *sta = params->sta;
-	enum ieee80211_ampdu_mlme_action action = params->action;
-	u16 tid = params->tid;
-
-	switch (action) {
-	case IEEE80211_AMPDU_TX_START:
-		return IEEE80211_AMPDU_TX_START_IMMEDIATE;
-	case IEEE80211_AMPDU_TX_STOP_CONT:
-	case IEEE80211_AMPDU_TX_STOP_FLUSH:
-	case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
-		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
-		break;
-	case IEEE80211_AMPDU_TX_OPERATIONAL:
-		break;
-	case IEEE80211_AMPDU_RX_START:
-	case IEEE80211_AMPDU_RX_STOP:
-		break;
-	default:
-		return -EOPNOTSUPP;
-	}
-
-	return 0;
-}
-
-static void mac80211_hwsim_flush(struct ieee80211_hw *hw,
-				 struct ieee80211_vif *vif,
-				 u32 queues, bool drop)
-{
-	/* Not implemented, queues only on kernel side */
-}
-
-static void hw_scan_work(struct work_struct *work)
-{
-	struct mac80211_hwsim_data *hwsim =
-		container_of(work, struct mac80211_hwsim_data, hw_scan.work);
-	struct cfg80211_scan_request *req = hwsim->hw_scan_request;
-	int dwell, i;
-
-	mutex_lock(&hwsim->mutex);
-	if (hwsim->scan_chan_idx >= req->n_channels) {
-		struct cfg80211_scan_info info = {
-			.aborted = false,
-		};
-
-		wiphy_dbg(hwsim->hw->wiphy, "hw scan complete\n");
-		ieee80211_scan_completed(hwsim->hw, &info);
-		hwsim->hw_scan_request = NULL;
-		hwsim->hw_scan_vif = NULL;
-		hwsim->tmp_chan = NULL;
-		mutex_unlock(&hwsim->mutex);
-		mac80211_hwsim_config_mac_nl(hwsim->hw, hwsim->scan_addr,
-					     false);
-		return;
-	}
-
-	wiphy_dbg(hwsim->hw->wiphy, "hw scan %d MHz\n",
-		  req->channels[hwsim->scan_chan_idx]->center_freq);
-
-	hwsim->tmp_chan = req->channels[hwsim->scan_chan_idx];
-	if (hwsim->tmp_chan->flags & (IEEE80211_CHAN_NO_IR |
-				      IEEE80211_CHAN_RADAR) ||
-	    !req->n_ssids) {
-		dwell = 120;
-	} else {
-		dwell = 30;
-		/* send probes */
-		for (i = 0; i < req->n_ssids; i++) {
-			struct sk_buff *probe;
-			struct ieee80211_mgmt *mgmt;
-
-			probe = ieee80211_probereq_get(hwsim->hw,
-						       hwsim->scan_addr,
-						       req->ssids[i].ssid,
-						       req->ssids[i].ssid_len,
-						       req->ie_len);
-			if (!probe)
-				continue;
-
-			mgmt = (struct ieee80211_mgmt *) probe->data;
-			memcpy(mgmt->da, req->bssid, ETH_ALEN);
-			memcpy(mgmt->bssid, req->bssid, ETH_ALEN);
-
-			if (req->ie_len)
-				skb_put_data(probe, req->ie, req->ie_len);
-
-			rcu_read_lock();
-			if (!ieee80211_tx_prepare_skb(hwsim->hw,
-						      hwsim->hw_scan_vif,
-						      probe,
-						      hwsim->tmp_chan->band,
-						      NULL)) {
-				rcu_read_unlock();
-				continue;
-			}
-
-			local_bh_disable();
-			mac80211_hwsim_tx_frame(hwsim->hw, probe,
-						hwsim->tmp_chan);
-			rcu_read_unlock();
-			local_bh_enable();
-		}
-	}
-	ieee80211_queue_delayed_work(hwsim->hw, &hwsim->hw_scan,
-				     msecs_to_jiffies(dwell));
-	hwsim->survey_data[hwsim->scan_chan_idx].channel = hwsim->tmp_chan;
-	hwsim->survey_data[hwsim->scan_chan_idx].start = jiffies;
-	hwsim->survey_data[hwsim->scan_chan_idx].end =
-		jiffies + msecs_to_jiffies(dwell);
-	hwsim->scan_chan_idx++;
-	mutex_unlock(&hwsim->mutex);
-}
-
-static int mac80211_hwsim_hw_scan(struct ieee80211_hw *hw,
-				  struct ieee80211_vif *vif,
-				  struct ieee80211_scan_request *hw_req)
-{
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-	struct cfg80211_scan_request *req = &hw_req->req;
-
-	mutex_lock(&hwsim->mutex);
-	if (WARN_ON(hwsim->tmp_chan || hwsim->hw_scan_request)) {
-		mutex_unlock(&hwsim->mutex);
-		return -EBUSY;
-	}
-	hwsim->hw_scan_request = req;
-	hwsim->hw_scan_vif = vif;
-	hwsim->scan_chan_idx = 0;
-	if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR)
-		get_random_mask_addr(hwsim->scan_addr,
-				     hw_req->req.mac_addr,
-				     hw_req->req.mac_addr_mask);
-	else
-		memcpy(hwsim->scan_addr, vif->addr, ETH_ALEN);
-	memset(hwsim->survey_data, 0, sizeof(hwsim->survey_data));
-	mutex_unlock(&hwsim->mutex);
-
-	mac80211_hwsim_config_mac_nl(hw, hwsim->scan_addr, true);
-	wiphy_dbg(hw->wiphy, "hwsim hw_scan request\n");
-
-	ieee80211_queue_delayed_work(hwsim->hw, &hwsim->hw_scan, 0);
-
-	return 0;
-}
-
-static void mac80211_hwsim_cancel_hw_scan(struct ieee80211_hw *hw,
-					  struct ieee80211_vif *vif)
-{
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-	struct cfg80211_scan_info info = {
-		.aborted = true,
-	};
-
-	wiphy_dbg(hw->wiphy, "hwsim cancel_hw_scan\n");
-
-	cancel_delayed_work_sync(&hwsim->hw_scan);
-
-	mutex_lock(&hwsim->mutex);
-	ieee80211_scan_completed(hwsim->hw, &info);
-	hwsim->tmp_chan = NULL;
-	hwsim->hw_scan_request = NULL;
-	hwsim->hw_scan_vif = NULL;
-	mutex_unlock(&hwsim->mutex);
-}
-
-static void mac80211_hwsim_sw_scan(struct ieee80211_hw *hw,
-				   struct ieee80211_vif *vif,
-				   const u8 *mac_addr)
-{
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-
-	mutex_lock(&hwsim->mutex);
-
-	if (hwsim->scanning) {
-		pr_debug("two hwsim sw_scans detected!\n");
-		goto out;
-	}
-
-	pr_debug("hwsim sw_scan request, prepping stuff\n");
-
-	memcpy(hwsim->scan_addr, mac_addr, ETH_ALEN);
-	mac80211_hwsim_config_mac_nl(hw, hwsim->scan_addr, true);
-	hwsim->scanning = true;
-	memset(hwsim->survey_data, 0, sizeof(hwsim->survey_data));
-
-out:
-	mutex_unlock(&hwsim->mutex);
-}
-
-static void mac80211_hwsim_sw_scan_complete(struct ieee80211_hw *hw,
-					    struct ieee80211_vif *vif)
-{
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-
-	mutex_lock(&hwsim->mutex);
-
-	pr_debug("hwsim sw_scan_complete\n");
-	hwsim->scanning = false;
-	mac80211_hwsim_config_mac_nl(hw, hwsim->scan_addr, false);
-	eth_zero_addr(hwsim->scan_addr);
-
-	mutex_unlock(&hwsim->mutex);
-}
-
-static void hw_roc_start(struct work_struct *work)
-{
-	struct mac80211_hwsim_data *hwsim =
-		container_of(work, struct mac80211_hwsim_data, roc_start.work);
-
-	mutex_lock(&hwsim->mutex);
-
-	wiphy_dbg(hwsim->hw->wiphy, "hwsim ROC begins\n");
-	hwsim->tmp_chan = hwsim->roc_chan;
-	ieee80211_ready_on_channel(hwsim->hw);
-
-	ieee80211_queue_delayed_work(hwsim->hw, &hwsim->roc_done,
-				     msecs_to_jiffies(hwsim->roc_duration));
-
-	mutex_unlock(&hwsim->mutex);
-}
-
-static void hw_roc_done(struct work_struct *work)
-{
-	struct mac80211_hwsim_data *hwsim =
-		container_of(work, struct mac80211_hwsim_data, roc_done.work);
-
-	mutex_lock(&hwsim->mutex);
-	ieee80211_remain_on_channel_expired(hwsim->hw);
-	hwsim->tmp_chan = NULL;
-	mutex_unlock(&hwsim->mutex);
-
-	wiphy_dbg(hwsim->hw->wiphy, "hwsim ROC expired\n");
-}
-
-static int mac80211_hwsim_roc(struct ieee80211_hw *hw,
-			      struct ieee80211_vif *vif,
-			      struct ieee80211_channel *chan,
-			      int duration,
-			      enum ieee80211_roc_type type)
-{
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-
-	mutex_lock(&hwsim->mutex);
-	if (WARN_ON(hwsim->tmp_chan || hwsim->hw_scan_request)) {
-		mutex_unlock(&hwsim->mutex);
-		return -EBUSY;
-	}
-
-	hwsim->roc_chan = chan;
-	hwsim->roc_duration = duration;
-	mutex_unlock(&hwsim->mutex);
-
-	wiphy_dbg(hw->wiphy, "hwsim ROC (%d MHz, %d ms)\n",
-		  chan->center_freq, duration);
-	ieee80211_queue_delayed_work(hw, &hwsim->roc_start, HZ/50);
-
-	return 0;
-}
-
-static int mac80211_hwsim_croc(struct ieee80211_hw *hw,
-			       struct ieee80211_vif *vif)
-{
-	struct mac80211_hwsim_data *hwsim = hw->priv;
-
-	cancel_delayed_work_sync(&hwsim->roc_start);
-	cancel_delayed_work_sync(&hwsim->roc_done);
-
-	mutex_lock(&hwsim->mutex);
-	hwsim->tmp_chan = NULL;
-	mutex_unlock(&hwsim->mutex);
-
-	wiphy_dbg(hw->wiphy, "hwsim ROC canceled\n");
-
-	return 0;
-}
-
-static int mac80211_hwsim_add_chanctx(struct ieee80211_hw *hw,
-				      struct ieee80211_chanctx_conf *ctx)
-{
-	hwsim_set_chanctx_magic(ctx);
-	wiphy_dbg(hw->wiphy,
-		  "add channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n",
-		  ctx->def.chan->center_freq, ctx->def.width,
-		  ctx->def.center_freq1, ctx->def.center_freq2);
-	return 0;
-}
-
-static void mac80211_hwsim_remove_chanctx(struct ieee80211_hw *hw,
-					  struct ieee80211_chanctx_conf *ctx)
-{
-	wiphy_dbg(hw->wiphy,
-		  "remove channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n",
-		  ctx->def.chan->center_freq, ctx->def.width,
-		  ctx->def.center_freq1, ctx->def.center_freq2);
-	hwsim_check_chanctx_magic(ctx);
-	hwsim_clear_chanctx_magic(ctx);
-}
-
-static void mac80211_hwsim_change_chanctx(struct ieee80211_hw *hw,
-					  struct ieee80211_chanctx_conf *ctx,
-					  u32 changed)
-{
-	hwsim_check_chanctx_magic(ctx);
-	wiphy_dbg(hw->wiphy,
-		  "change channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n",
-		  ctx->def.chan->center_freq, ctx->def.width,
-		  ctx->def.center_freq1, ctx->def.center_freq2);
-}
-
-static int mac80211_hwsim_assign_vif_chanctx(struct ieee80211_hw *hw,
-					     struct ieee80211_vif *vif,
-					     struct ieee80211_bss_conf *link_conf,
-					     struct ieee80211_chanctx_conf *ctx)
-{
-	hwsim_check_magic(vif);
-	hwsim_check_chanctx_magic(ctx);
-
-	/* if we activate a link while already associated wake it up */
-	if (vif->type == NL80211_IFTYPE_STATION && vif->cfg.assoc) {
-		struct sk_buff *skb;
-
-		skb = ieee80211_nullfunc_get(hw, vif, link_conf->link_id, true);
-		if (skb) {
-			local_bh_disable();
-			mac80211_hwsim_tx_frame(hw, skb, ctx->def.chan);
-			local_bh_enable();
-		}
-	}
-
-	return 0;
-}
-
-static void mac80211_hwsim_unassign_vif_chanctx(struct ieee80211_hw *hw,
-						struct ieee80211_vif *vif,
-						struct ieee80211_bss_conf *link_conf,
-						struct ieee80211_chanctx_conf *ctx)
-{
-	hwsim_check_magic(vif);
-	hwsim_check_chanctx_magic(ctx);
-
-	/* if we deactivate a link while associated suspend it first */
-	if (vif->type == NL80211_IFTYPE_STATION && vif->cfg.assoc) {
-		struct sk_buff *skb;
-
-		skb = ieee80211_nullfunc_get(hw, vif, link_conf->link_id, true);
-		if (skb) {
-			struct ieee80211_hdr *hdr = (void *)skb->data;
-
-			hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
-
-			local_bh_disable();
-			mac80211_hwsim_tx_frame(hw, skb, ctx->def.chan);
-			local_bh_enable();
-		}
-	}
-}
-
-static int mac80211_hwsim_switch_vif_chanctx(struct ieee80211_hw *hw,
-					     struct ieee80211_vif_chanctx_switch *vifs,
-					     int n_vifs,
-					     enum ieee80211_chanctx_switch_mode mode)
-{
-	int i;
-
-	if (n_vifs <= 0)
-		return -EINVAL;
-
-	wiphy_dbg(hw->wiphy,
-		  "switch vif channel context mode: %u\n", mode);
-
-	for (i = 0; i < n_vifs; i++) {
-		hwsim_check_chanctx_magic(vifs[i].old_ctx);
-		wiphy_dbg(hw->wiphy,
-			  "switch vif channel context: %d MHz/width: %d/cfreqs:%d/%d MHz -> %d MHz/width: %d/cfreqs:%d/%d MHz\n",
-			  vifs[i].old_ctx->def.chan->center_freq,
-			  vifs[i].old_ctx->def.width,
-			  vifs[i].old_ctx->def.center_freq1,
-			  vifs[i].old_ctx->def.center_freq2,
-			  vifs[i].new_ctx->def.chan->center_freq,
-			  vifs[i].new_ctx->def.width,
-			  vifs[i].new_ctx->def.center_freq1,
-			  vifs[i].new_ctx->def.center_freq2);
-
-		switch (mode) {
-		case CHANCTX_SWMODE_REASSIGN_VIF:
-			hwsim_check_chanctx_magic(vifs[i].new_ctx);
-			break;
-		case CHANCTX_SWMODE_SWAP_CONTEXTS:
-			hwsim_set_chanctx_magic(vifs[i].new_ctx);
-			hwsim_clear_chanctx_magic(vifs[i].old_ctx);
-			break;
-		default:
-			WARN(1, "Invalid mode %d\n", mode);
-		}
-	}
-	return 0;
-}
-
-static const char mac80211_hwsim_gstrings_stats[][ETH_GSTRING_LEN] = {
-	"tx_pkts_nic",
-	"tx_bytes_nic",
-	"rx_pkts_nic",
-	"rx_bytes_nic",
-	"d_tx_dropped",
-	"d_tx_failed",
-	"d_ps_mode",
-	"d_group",
-};
-
-#define MAC80211_HWSIM_SSTATS_LEN ARRAY_SIZE(mac80211_hwsim_gstrings_stats)
-
-static void mac80211_hwsim_get_et_strings(struct ieee80211_hw *hw,
-					  struct ieee80211_vif *vif,
-					  u32 sset, u8 *data)
-{
-	if (sset == ETH_SS_STATS)
-		memcpy(data, mac80211_hwsim_gstrings_stats,
-		       sizeof(mac80211_hwsim_gstrings_stats));
-}
-
-static int mac80211_hwsim_get_et_sset_count(struct ieee80211_hw *hw,
-					    struct ieee80211_vif *vif, int sset)
-{
-	if (sset == ETH_SS_STATS)
-		return MAC80211_HWSIM_SSTATS_LEN;
-	return 0;
-}
-
-static void mac80211_hwsim_get_et_stats(struct ieee80211_hw *hw,
-					struct ieee80211_vif *vif,
-					struct ethtool_stats *stats, u64 *data)
-{
-	struct mac80211_hwsim_data *ar = hw->priv;
-	int i = 0;
-
-	data[i++] = ar->tx_pkts;
-	data[i++] = ar->tx_bytes;
-	data[i++] = ar->rx_pkts;
-	data[i++] = ar->rx_bytes;
-	data[i++] = ar->tx_dropped;
-	data[i++] = ar->tx_failed;
-	data[i++] = ar->ps;
-	data[i++] = ar->group;
-
-	WARN_ON(i != MAC80211_HWSIM_SSTATS_LEN);
-}
-
-static int mac80211_hwsim_tx_last_beacon(struct ieee80211_hw *hw)
-{
-	return 1;
-}
-
-static int mac80211_hwsim_set_rts_threshold(struct ieee80211_hw *hw,
-					    int radio_idx, u32 value)
-{
-	return -EOPNOTSUPP;
-}
-
-static int mac80211_hwsim_change_vif_links(struct ieee80211_hw *hw,
-					   struct ieee80211_vif *vif,
-					   u16 old_links, u16 new_links,
-					   struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
-{
-	unsigned long rem = old_links & ~new_links;
-	unsigned long add = new_links & ~old_links;
-	int i;
-
-	if (!old_links)
-		rem |= BIT(0);
-	if (!new_links)
-		add |= BIT(0);
-
-	for_each_set_bit(i, &rem, IEEE80211_MLD_MAX_NUM_LINKS)
-		mac80211_hwsim_config_mac_nl(hw, old[i]->addr, false);
-
-	for_each_set_bit(i, &add, IEEE80211_MLD_MAX_NUM_LINKS) {
-		struct ieee80211_bss_conf *link_conf;
-
-		link_conf = link_conf_dereference_protected(vif, i);
-		if (WARN_ON(!link_conf))
-			continue;
-
-		mac80211_hwsim_config_mac_nl(hw, link_conf->addr, true);
-	}
-
-	return 0;
-}
-
-static int mac80211_hwsim_change_sta_links(struct ieee80211_hw *hw,
-					   struct ieee80211_vif *vif,
-					   struct ieee80211_sta *sta,
-					   u16 old_links, u16 new_links)
-{
-	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
-
-	hwsim_check_sta_magic(sta);
-
-	if (vif->type == NL80211_IFTYPE_STATION)
-		sp->active_links_rx = new_links;
-
-	return 0;
-}
-
-static int mac80211_hwsim_send_pmsr_ftm_request_peer(struct sk_buff *msg,
-						     struct cfg80211_pmsr_ftm_request_peer *request)
-{
-	struct nlattr *ftm;
-
-	if (!request->requested)
-		return -EINVAL;
-
-	ftm = nla_nest_start(msg, NL80211_PMSR_TYPE_FTM);
-	if (!ftm)
-		return -ENOBUFS;
-
-	if (nla_put_u32(msg, NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE, request->preamble))
-		return -ENOBUFS;
-
-	if (nla_put_u16(msg, NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD, request->burst_period))
-		return -ENOBUFS;
-
-	if (request->asap && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_ASAP))
-		return -ENOBUFS;
-
-	if (request->request_lci && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_REQUEST_LCI))
-		return -ENOBUFS;
-
-	if (request->request_civicloc &&
-	    nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_REQUEST_CIVICLOC))
-		return -ENOBUFS;
-
-	if (request->trigger_based && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED))
-		return -ENOBUFS;
-
-	if (request->non_trigger_based &&
-	    nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED))
-		return -ENOBUFS;
-
-	if (request->lmr_feedback && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK))
-		return -ENOBUFS;
-
-	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_NUM_BURSTS_EXP, request->num_bursts_exp))
-		return -ENOBUFS;
-
-	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_BURST_DURATION, request->burst_duration))
-		return -ENOBUFS;
-
-	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST, request->ftms_per_burst))
-		return -ENOBUFS;
-
-	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_NUM_FTMR_RETRIES, request->ftmr_retries))
-		return -ENOBUFS;
-
-	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_BURST_DURATION, request->burst_duration))
-		return -ENOBUFS;
-
-	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR, request->bss_color))
-		return -ENOBUFS;
-
-	nla_nest_end(msg, ftm);
-
-	return 0;
-}
-
-static int mac80211_hwsim_send_pmsr_request_peer(struct sk_buff *msg,
-						 struct cfg80211_pmsr_request_peer *request)
-{
-	struct nlattr *peer, *chandef, *req, *data;
-	int err;
-
-	peer = nla_nest_start(msg, NL80211_PMSR_ATTR_PEERS);
-	if (!peer)
-		return -ENOBUFS;
-
-	if (nla_put(msg, NL80211_PMSR_PEER_ATTR_ADDR, ETH_ALEN,
-		    request->addr))
-		return -ENOBUFS;
-
-	chandef = nla_nest_start(msg, NL80211_PMSR_PEER_ATTR_CHAN);
-	if (!chandef)
-		return -ENOBUFS;
-
-	err = nl80211_send_chandef(msg, &request->chandef);
-	if (err)
-		return err;
-
-	nla_nest_end(msg, chandef);
-
-	req = nla_nest_start(msg, NL80211_PMSR_PEER_ATTR_REQ);
-	if (!req)
-		return -ENOBUFS;
-
-	if (request->report_ap_tsf && nla_put_flag(msg, NL80211_PMSR_REQ_ATTR_GET_AP_TSF))
-		return -ENOBUFS;
-
-	data = nla_nest_start(msg, NL80211_PMSR_REQ_ATTR_DATA);
-	if (!data)
-		return -ENOBUFS;
-
-	err = mac80211_hwsim_send_pmsr_ftm_request_peer(msg, &request->ftm);
-	if (err)
-		return err;
-
-	nla_nest_end(msg, data);
-	nla_nest_end(msg, req);
-	nla_nest_end(msg, peer);
-
-	return 0;
-}
-
-static int mac80211_hwsim_send_pmsr_request(struct sk_buff *msg,
-					    struct cfg80211_pmsr_request *request)
-{
-	struct nlattr *pmsr;
-	int err;
-
-	pmsr = nla_nest_start(msg, NL80211_ATTR_PEER_MEASUREMENTS);
-	if (!pmsr)
-		return -ENOBUFS;
-
-	if (nla_put_u32(msg, NL80211_ATTR_TIMEOUT, request->timeout))
-		return -ENOBUFS;
-
-	if (!is_zero_ether_addr(request->mac_addr)) {
-		if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, request->mac_addr))
-			return -ENOBUFS;
-		if (nla_put(msg, NL80211_ATTR_MAC_MASK, ETH_ALEN, request->mac_addr_mask))
-			return -ENOBUFS;
-	}
-
-	for (int i = 0; i < request->n_peers; i++) {
-		err = mac80211_hwsim_send_pmsr_request_peer(msg, &request->peers[i]);
-		if (err)
-			return err;
-	}
-
-	nla_nest_end(msg, pmsr);
-
-	return 0;
-}
-
-static int mac80211_hwsim_start_pmsr(struct ieee80211_hw *hw,
-				     struct ieee80211_vif *vif,
-				     struct cfg80211_pmsr_request *request)
-{
-	struct mac80211_hwsim_data *data;
-	struct sk_buff *skb = NULL;
-	struct nlattr *pmsr;
-	void *msg_head;
-	u32 _portid;
-	int err = 0;
-
-	data = hw->priv;
-	_portid = READ_ONCE(data->wmediumd);
-	if (!_portid && !hwsim_virtio_enabled)
-		return -EOPNOTSUPP;
-
-	mutex_lock(&data->mutex);
-
-	if (data->pmsr_request) {
-		err = -EBUSY;
-		goto out_free;
-	}
-
-	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
-
-	if (!skb) {
-		err = -ENOMEM;
-		goto out_free;
-	}
-
-	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0, HWSIM_CMD_START_PMSR);
-
-	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER,
-		    ETH_ALEN, data->addresses[1].addr)) {
-		err = -ENOMEM;
-		goto out_free;
-	}
-
-	pmsr = nla_nest_start(skb, HWSIM_ATTR_PMSR_REQUEST);
-	if (!pmsr) {
-		err = -ENOMEM;
-		goto out_free;
-	}
-
-	err = mac80211_hwsim_send_pmsr_request(skb, request);
-	if (err)
-		goto out_free;
-
-	nla_nest_end(skb, pmsr);
-
-	genlmsg_end(skb, msg_head);
-	if (hwsim_virtio_enabled)
-		hwsim_tx_virtio(data, skb);
-	else
-		hwsim_unicast_netgroup(data, skb, _portid);
-
-	data->pmsr_request = request;
-	data->pmsr_request_wdev = ieee80211_vif_to_wdev(vif);
-
-out_free:
-	if (err && skb)
-		nlmsg_free(skb);
-
-	mutex_unlock(&data->mutex);
-	return err;
-}
-
-static void mac80211_hwsim_abort_pmsr(struct ieee80211_hw *hw,
-				      struct ieee80211_vif *vif,
-				      struct cfg80211_pmsr_request *request)
-{
-	struct mac80211_hwsim_data *data;
-	struct sk_buff *skb = NULL;
-	struct nlattr *pmsr;
-	void *msg_head;
-	u32 _portid;
-	int err = 0;
-
-	data = hw->priv;
-	_portid = READ_ONCE(data->wmediumd);
-	if (!_portid && !hwsim_virtio_enabled)
-		return;
-
-	mutex_lock(&data->mutex);
-
-	if (data->pmsr_request != request) {
-		err = -EINVAL;
-		goto out;
-	}
-
-	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
-	if (!skb) {
-		err = -ENOMEM;
-		goto out;
-	}
-
-	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0, HWSIM_CMD_ABORT_PMSR);
-
-	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER, ETH_ALEN, data->addresses[1].addr))
-		goto out;
-
-	pmsr = nla_nest_start(skb, HWSIM_ATTR_PMSR_REQUEST);
-	if (!pmsr) {
-		err = -ENOMEM;
-		goto out;
-	}
-
-	err = mac80211_hwsim_send_pmsr_request(skb, request);
-	if (err)
-		goto out;
-
-	err = nla_nest_end(skb, pmsr);
-	if (err)
-		goto out;
-
-	genlmsg_end(skb, msg_head);
-	if (hwsim_virtio_enabled)
-		hwsim_tx_virtio(data, skb);
-	else
-		hwsim_unicast_netgroup(data, skb, _portid);
-
-out:
-	if (err && skb)
-		nlmsg_free(skb);
-
-	mutex_unlock(&data->mutex);
-}
-
-static int mac80211_hwsim_parse_rate_info(struct nlattr *rateattr,
-					  struct rate_info *rate_info,
-					  struct genl_info *info)
-{
-	struct nlattr *tb[HWSIM_RATE_INFO_ATTR_MAX + 1];
-	int ret;
-
-	ret = nla_parse_nested(tb, HWSIM_RATE_INFO_ATTR_MAX,
-			       rateattr, hwsim_rate_info_policy, info->extack);
-	if (ret)
-		return ret;
-
-	if (tb[HWSIM_RATE_INFO_ATTR_FLAGS])
-		rate_info->flags = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_FLAGS]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_MCS])
-		rate_info->mcs = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_MCS]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_LEGACY])
-		rate_info->legacy = nla_get_u16(tb[HWSIM_RATE_INFO_ATTR_LEGACY]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_NSS])
-		rate_info->nss = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_NSS]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_BW])
-		rate_info->bw = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_BW]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_HE_GI])
-		rate_info->he_gi = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_HE_GI]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_HE_DCM])
-		rate_info->he_dcm = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_HE_DCM]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC])
-		rate_info->he_ru_alloc =
-			nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH])
-		rate_info->n_bonded_ch = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_EHT_GI])
-		rate_info->eht_gi = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_EHT_GI]);
-
-	if (tb[HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC])
-		rate_info->eht_ru_alloc = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC]);
-
-	return 0;
-}
-
-static int mac80211_hwsim_parse_ftm_result(struct nlattr *ftm,
-					   struct cfg80211_pmsr_ftm_result *result,
-					   struct genl_info *info)
-{
-	struct nlattr *tb[NL80211_PMSR_FTM_RESP_ATTR_MAX + 1];
-	int ret;
-
-	ret = nla_parse_nested(tb, NL80211_PMSR_FTM_RESP_ATTR_MAX,
-			       ftm, hwsim_ftm_result_policy, info->extack);
-	if (ret)
-		return ret;
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON])
-		result->failure_reason = nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON]);
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_INDEX])
-		result->burst_index = nla_get_u16(tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_INDEX]);
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS]) {
-		result->num_ftmr_attempts_valid = 1;
-		result->num_ftmr_attempts =
-			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS]);
-	}
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES]) {
-		result->num_ftmr_successes_valid = 1;
-		result->num_ftmr_successes =
-			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES]);
-	}
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME])
-		result->busy_retry_time =
-			nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME]);
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP])
-		result->num_bursts_exp = nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP]);
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_DURATION])
-		result->burst_duration = nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_DURATION]);
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST])
-		result->ftms_per_burst = nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST]);
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_AVG]) {
-		result->rssi_avg_valid = 1;
-		result->rssi_avg = nla_get_s32(tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_AVG]);
-	}
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_SPREAD]) {
-		result->rssi_spread_valid = 1;
-		result->rssi_spread =
-			nla_get_s32(tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_SPREAD]);
-	}
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_TX_RATE]) {
-		result->tx_rate_valid = 1;
-		ret = mac80211_hwsim_parse_rate_info(tb[NL80211_PMSR_FTM_RESP_ATTR_TX_RATE],
-						     &result->tx_rate, info);
-		if (ret)
-			return ret;
-	}
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RX_RATE]) {
-		result->rx_rate_valid = 1;
-		ret = mac80211_hwsim_parse_rate_info(tb[NL80211_PMSR_FTM_RESP_ATTR_RX_RATE],
-						     &result->rx_rate, info);
-		if (ret)
-			return ret;
-	}
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_AVG]) {
-		result->rtt_avg_valid = 1;
-		result->rtt_avg =
-			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_AVG]);
-	}
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_VARIANCE]) {
-		result->rtt_variance_valid = 1;
-		result->rtt_variance =
-			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_VARIANCE]);
-	}
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_SPREAD]) {
-		result->rtt_spread_valid = 1;
-		result->rtt_spread =
-			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_SPREAD]);
-	}
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_AVG]) {
-		result->dist_avg_valid = 1;
-		result->dist_avg =
-			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_AVG]);
-	}
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_VARIANCE]) {
-		result->dist_variance_valid = 1;
-		result->dist_variance =
-			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_VARIANCE]);
-	}
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_SPREAD]) {
-		result->dist_spread_valid = 1;
-		result->dist_spread =
-			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_SPREAD]);
-	}
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_LCI]) {
-		result->lci = nla_data(tb[NL80211_PMSR_FTM_RESP_ATTR_LCI]);
-		result->lci_len = nla_len(tb[NL80211_PMSR_FTM_RESP_ATTR_LCI]);
-	}
-
-	if (tb[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC]) {
-		result->civicloc = nla_data(tb[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC]);
-		result->civicloc_len = nla_len(tb[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC]);
-	}
-
-	return 0;
-}
-
-static int mac80211_hwsim_parse_pmsr_resp(struct nlattr *resp,
-					  struct cfg80211_pmsr_result *result,
-					  struct genl_info *info)
-{
-	struct nlattr *tb[NL80211_PMSR_RESP_ATTR_MAX + 1];
-	struct nlattr *pmsr;
-	int rem;
-	int ret;
-
-	ret = nla_parse_nested(tb, NL80211_PMSR_RESP_ATTR_MAX, resp, hwsim_pmsr_resp_policy,
-			       info->extack);
-	if (ret)
-		return ret;
-
-	if (tb[NL80211_PMSR_RESP_ATTR_STATUS])
-		result->status = nla_get_u32(tb[NL80211_PMSR_RESP_ATTR_STATUS]);
-
-	if (tb[NL80211_PMSR_RESP_ATTR_HOST_TIME])
-		result->host_time = nla_get_u64(tb[NL80211_PMSR_RESP_ATTR_HOST_TIME]);
-
-	if (tb[NL80211_PMSR_RESP_ATTR_AP_TSF]) {
-		result->ap_tsf_valid = 1;
-		result->ap_tsf = nla_get_u64(tb[NL80211_PMSR_RESP_ATTR_AP_TSF]);
-	}
-
-	result->final = !!tb[NL80211_PMSR_RESP_ATTR_FINAL];
-
-	if (!tb[NL80211_PMSR_RESP_ATTR_DATA])
-		return 0;
-
-	nla_for_each_nested(pmsr, tb[NL80211_PMSR_RESP_ATTR_DATA], rem) {
-		switch (nla_type(pmsr)) {
-		case NL80211_PMSR_TYPE_FTM:
-			result->type = NL80211_PMSR_TYPE_FTM;
-			ret = mac80211_hwsim_parse_ftm_result(pmsr, &result->ftm, info);
-			if (ret)
-				return ret;
-			break;
-		default:
-			NL_SET_ERR_MSG_ATTR(info->extack, pmsr, "Unknown pmsr resp type");
-			return -EINVAL;
-		}
-	}
-
-	return 0;
-}
-
-static int mac80211_hwsim_parse_pmsr_result(struct nlattr *peer,
-					    struct cfg80211_pmsr_result *result,
-					    struct genl_info *info)
-{
-	struct nlattr *tb[NL80211_PMSR_PEER_ATTR_MAX + 1];
-	int ret;
-
-	if (!peer)
-		return -EINVAL;
-
-	ret = nla_parse_nested(tb, NL80211_PMSR_PEER_ATTR_MAX, peer,
-			       hwsim_pmsr_peer_result_policy, info->extack);
-	if (ret)
-		return ret;
-
-	if (tb[NL80211_PMSR_PEER_ATTR_ADDR])
-		memcpy(result->addr, nla_data(tb[NL80211_PMSR_PEER_ATTR_ADDR]),
-		       ETH_ALEN);
-
-	if (tb[NL80211_PMSR_PEER_ATTR_RESP]) {
-		ret = mac80211_hwsim_parse_pmsr_resp(tb[NL80211_PMSR_PEER_ATTR_RESP], result, info);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-};
-
-static int hwsim_pmsr_report_nl(struct sk_buff *msg, struct genl_info *info)
-{
-	struct mac80211_hwsim_data *data;
-	struct nlattr *peers, *peer;
-	struct nlattr *reqattr;
-	const u8 *src;
-	int err;
-	int rem;
-
-	if (!info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER])
-		return -EINVAL;
-
-	src = nla_data(info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER]);
-	data = get_hwsim_data_ref_from_addr(src);
-	if (!data)
-		return -EINVAL;
-
-	mutex_lock(&data->mutex);
-	if (!data->pmsr_request) {
-		err = -EINVAL;
-		goto out;
-	}
-
-	reqattr = info->attrs[HWSIM_ATTR_PMSR_RESULT];
-	if (!reqattr) {
-		err = -EINVAL;
-		goto out;
-	}
-
-	peers = nla_find_nested(reqattr, NL80211_PMSR_ATTR_PEERS);
-	if (!peers) {
-		err = -EINVAL;
-		goto out;
-	}
-
-	nla_for_each_nested(peer, peers, rem) {
-		struct cfg80211_pmsr_result result = {};
-
-		err = mac80211_hwsim_parse_pmsr_result(peer, &result, info);
-		if (err)
-			goto out;
-
-		cfg80211_pmsr_report(data->pmsr_request_wdev,
-				     data->pmsr_request, &result, GFP_KERNEL);
-	}
-
-	cfg80211_pmsr_complete(data->pmsr_request_wdev, data->pmsr_request, GFP_KERNEL);
-
-	err = 0;
-out:
-	data->pmsr_request = NULL;
-	data->pmsr_request_wdev = NULL;
-
-	mutex_unlock(&data->mutex);
-	return err;
-}
-
-static enum hrtimer_restart
-mac80211_hwsim_nan_dw_start(struct hrtimer *timer)
-{
-	struct mac80211_hwsim_data *data =
-		container_of(timer, struct mac80211_hwsim_data,
-			     nan_timer);
-	struct ieee80211_hw *hw = data->hw;
-	u64 orig_tsf = mac80211_hwsim_get_tsf(hw, NULL), tsf = orig_tsf;
-	u32 dw_int = 512 * 1024;
-	u64 until_dw;
-
-	if (!data->nan_device_vif)
-		return HRTIMER_NORESTART;
-
-	if (data->nan_bands & BIT(NL80211_BAND_5GHZ)) {
-		if (data->nan_curr_dw_band == NL80211_BAND_2GHZ) {
-			dw_int = 128 * 1024;
-			data->nan_curr_dw_band = NL80211_BAND_5GHZ;
-		} else if (data->nan_curr_dw_band == NL80211_BAND_5GHZ) {
-			data->nan_curr_dw_band = NL80211_BAND_2GHZ;
-		}
-	}
-
-	until_dw = dw_int - do_div(tsf, dw_int);
-
-	/* The timer might fire just before the actual DW, in which case
-	 * update the timeout to the actual next DW
-	 */
-	if (until_dw < dw_int / 2)
-		until_dw += dw_int;
-
-	/* The above do_div() call directly modifies the 'tsf' variable, thus,
-	 * use a copy so that the print below would show the original TSF.
-	 */
-	wiphy_debug(hw->wiphy,
-		    "%s: tsf=%llx, curr_dw_band=%u, next_dw=%llu\n",
-		    __func__, orig_tsf, data->nan_curr_dw_band,
-		    until_dw);
-
-	hrtimer_forward_now(&data->nan_timer,
-			    ns_to_ktime(until_dw * NSEC_PER_USEC));
-
-	if (data->notify_dw) {
-		struct ieee80211_channel *ch;
-		struct wireless_dev *wdev =
-			ieee80211_vif_to_wdev(data->nan_device_vif);
-
-		if (data->nan_curr_dw_band == NL80211_BAND_5GHZ)
-			ch = ieee80211_get_channel(hw->wiphy, 5745);
-		else
-			ch = ieee80211_get_channel(hw->wiphy, 2437);
-
-		cfg80211_next_nan_dw_notif(wdev, ch, GFP_ATOMIC);
-	}
-
-	return HRTIMER_RESTART;
-}
-
-static int mac80211_hwsim_start_nan(struct ieee80211_hw *hw,
-				    struct ieee80211_vif *vif,
-				    struct cfg80211_nan_conf *conf)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	u64 tsf = mac80211_hwsim_get_tsf(hw, NULL);
-	u32 dw_int = 512 * 1000;
-	u64 until_dw = dw_int - do_div(tsf, dw_int);
-	struct wireless_dev *wdev = ieee80211_vif_to_wdev(vif);
-
-	if (vif->type != NL80211_IFTYPE_NAN)
-		return -EINVAL;
-
-	if (data->nan_device_vif)
-		return -EALREADY;
-
-	/* set this before starting the timer, as preemption might occur */
-	data->nan_device_vif = vif;
-	data->nan_bands = conf->bands;
-	data->nan_curr_dw_band = NL80211_BAND_2GHZ;
-
-	wiphy_debug(hw->wiphy, "nan_started, next_dw=%llu\n",
-		    until_dw);
-
-	hrtimer_start(&data->nan_timer,
-		      ns_to_ktime(until_dw * NSEC_PER_USEC),
-		      HRTIMER_MODE_REL_SOFT);
-
-	if (!is_zero_ether_addr(conf->cluster_id) &&
-	    is_zero_ether_addr(hwsim_nan_cluster_id)) {
-		memcpy(hwsim_nan_cluster_id, conf->cluster_id, ETH_ALEN);
-	} else if (is_zero_ether_addr(hwsim_nan_cluster_id)) {
-		hwsim_nan_cluster_id[0] = 0x50;
-		hwsim_nan_cluster_id[1] = 0x6f;
-		hwsim_nan_cluster_id[2] = 0x9a;
-		hwsim_nan_cluster_id[3] = 0x01;
-		hwsim_nan_cluster_id[4] = get_random_u8();
-		hwsim_nan_cluster_id[5] = get_random_u8();
-	}
-
-	data->notify_dw = conf->enable_dw_notification;
-
-	cfg80211_nan_cluster_joined(wdev, hwsim_nan_cluster_id, true,
-				    GFP_KERNEL);
-
-	return 0;
-}
-
-static int mac80211_hwsim_stop_nan(struct ieee80211_hw *hw,
-				   struct ieee80211_vif *vif)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-	struct mac80211_hwsim_data *data2;
-	bool nan_cluster_running = false;
-
-	if (vif->type != NL80211_IFTYPE_NAN || !data->nan_device_vif ||
-	    data->nan_device_vif != vif)
-		return -EINVAL;
-
-	hrtimer_cancel(&data->nan_timer);
-	data->nan_device_vif = NULL;
-
-	spin_lock_bh(&hwsim_radio_lock);
-	list_for_each_entry(data2, &hwsim_radios, list) {
-		if (data2->nan_device_vif) {
-			nan_cluster_running = true;
-			break;
-		}
-	}
-	spin_unlock_bh(&hwsim_radio_lock);
-
-	if (!nan_cluster_running)
-		memset(hwsim_nan_cluster_id, 0, ETH_ALEN);
-
-	return 0;
-}
-
-static int mac80211_hwsim_change_nan_config(struct ieee80211_hw *hw,
-					    struct ieee80211_vif *vif,
-					    struct cfg80211_nan_conf *conf,
-					    u32 changes)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-
-	if (vif->type != NL80211_IFTYPE_NAN)
-		return -EINVAL;
-
-	if (!data->nan_device_vif)
-		return -EINVAL;
-
-	wiphy_debug(hw->wiphy, "nan_config_changed: changes=0x%x\n", changes);
-
-	/* Handle only the changes we care about for simulation purposes */
-	if (changes & CFG80211_NAN_CONF_CHANGED_BANDS) {
-		data->nan_bands = conf->bands;
-		data->nan_curr_dw_band = NL80211_BAND_2GHZ;
-	}
-
-	if (changes & CFG80211_NAN_CONF_CHANGED_CONFIG)
-		data->notify_dw = conf->enable_dw_notification;
-
-	return 0;
-}
-
-static int mac80211_hwsim_set_radar_background(struct ieee80211_hw *hw,
-					       struct cfg80211_chan_def *chan)
-{
-	struct mac80211_hwsim_data *data = hw->priv;
-
-	if (!wiphy_ext_feature_isset(hw->wiphy,
-				     NL80211_EXT_FEATURE_RADAR_BACKGROUND))
-		return -EOPNOTSUPP;
-
-	if (chan)
-		data->radar_background_chandef = *chan;
-	else
-		memset(&data->radar_background_chandef, 0,
-		       sizeof(data->radar_background_chandef));
-
-	return 0;
-}
-
-#ifdef CONFIG_MAC80211_DEBUGFS
-#define HWSIM_DEBUGFS_OPS					\
-	.link_add_debugfs = mac80211_hwsim_link_add_debugfs,
-#else
-#define HWSIM_DEBUGFS_OPS
-#endif
-
-#define HWSIM_COMMON_OPS					\
-	.tx = mac80211_hwsim_tx,				\
-	.wake_tx_queue = ieee80211_handle_wake_tx_queue,	\
-	.start = mac80211_hwsim_start,				\
-	.stop = mac80211_hwsim_stop,				\
-	.add_interface = mac80211_hwsim_add_interface,		\
-	.change_interface = mac80211_hwsim_change_interface,	\
-	.remove_interface = mac80211_hwsim_remove_interface,	\
-	.config = mac80211_hwsim_config,			\
-	.configure_filter = mac80211_hwsim_configure_filter,	\
-	.vif_cfg_changed = mac80211_hwsim_vif_info_changed,	\
-	.link_info_changed = mac80211_hwsim_link_info_changed,  \
-	.tx_last_beacon = mac80211_hwsim_tx_last_beacon,	\
-	.sta_notify = mac80211_hwsim_sta_notify,		\
-	.link_sta_rc_update = mac80211_hwsim_sta_rc_update,	\
-	.conf_tx = mac80211_hwsim_conf_tx,			\
-	.get_survey = mac80211_hwsim_get_survey,		\
-	CFG80211_TESTMODE_CMD(mac80211_hwsim_testmode_cmd)	\
-	.ampdu_action = mac80211_hwsim_ampdu_action,		\
-	.flush = mac80211_hwsim_flush,				\
-	.get_et_sset_count = mac80211_hwsim_get_et_sset_count,	\
-	.get_et_stats = mac80211_hwsim_get_et_stats,		\
-	.get_et_strings = mac80211_hwsim_get_et_strings,	\
-	.start_pmsr = mac80211_hwsim_start_pmsr,		\
-	.abort_pmsr = mac80211_hwsim_abort_pmsr,		\
-	.start_nan = mac80211_hwsim_start_nan,                  \
-	.stop_nan = mac80211_hwsim_stop_nan,                    \
-	.nan_change_conf = mac80211_hwsim_change_nan_config,    \
-	.set_radar_background = mac80211_hwsim_set_radar_background, \
-	.set_key = mac80211_hwsim_set_key,			\
-	HWSIM_DEBUGFS_OPS
-
-#define HWSIM_NON_MLO_OPS					\
-	.sta_add = mac80211_hwsim_sta_add,			\
-	.sta_remove = mac80211_hwsim_sta_remove,		\
-	.set_tim = mac80211_hwsim_set_tim,			\
-	.get_tsf = mac80211_hwsim_get_tsf,			\
-	.set_tsf = mac80211_hwsim_set_tsf,
-
-static const struct ieee80211_ops mac80211_hwsim_ops = {
-	HWSIM_COMMON_OPS
-	HWSIM_NON_MLO_OPS
-	.sw_scan_start = mac80211_hwsim_sw_scan,
-	.sw_scan_complete = mac80211_hwsim_sw_scan_complete,
-	.add_chanctx = ieee80211_emulate_add_chanctx,
-	.remove_chanctx = ieee80211_emulate_remove_chanctx,
-	.change_chanctx = ieee80211_emulate_change_chanctx,
-	.switch_vif_chanctx = ieee80211_emulate_switch_vif_chanctx,
-};
-
-#define HWSIM_CHANCTX_OPS					\
-	.hw_scan = mac80211_hwsim_hw_scan,			\
-	.cancel_hw_scan = mac80211_hwsim_cancel_hw_scan,	\
-	.remain_on_channel = mac80211_hwsim_roc,		\
-	.cancel_remain_on_channel = mac80211_hwsim_croc,	\
-	.add_chanctx = mac80211_hwsim_add_chanctx,		\
-	.remove_chanctx = mac80211_hwsim_remove_chanctx,	\
-	.change_chanctx = mac80211_hwsim_change_chanctx,	\
-	.assign_vif_chanctx = mac80211_hwsim_assign_vif_chanctx,\
-	.unassign_vif_chanctx = mac80211_hwsim_unassign_vif_chanctx, \
-	.switch_vif_chanctx = mac80211_hwsim_switch_vif_chanctx,
-
-static const struct ieee80211_ops mac80211_hwsim_mchan_ops = {
-	HWSIM_COMMON_OPS
-	HWSIM_NON_MLO_OPS
-	HWSIM_CHANCTX_OPS
-};
-
-static const struct ieee80211_ops mac80211_hwsim_mlo_ops = {
-	HWSIM_COMMON_OPS
-	HWSIM_CHANCTX_OPS
-	.set_rts_threshold = mac80211_hwsim_set_rts_threshold,
-	.change_vif_links = mac80211_hwsim_change_vif_links,
-	.change_sta_links = mac80211_hwsim_change_sta_links,
-	.sta_state = mac80211_hwsim_sta_state,
-	.can_neg_ttlm = mac80211_hwsim_can_neg_ttlm,
-};
-
-struct hwsim_new_radio_params {
-	unsigned int channels;
-	const char *reg_alpha2;
-	const struct ieee80211_regdomain *regd;
-	bool reg_strict;
-	bool p2p_device;
-	bool use_chanctx;
-	bool multi_radio;
-	bool destroy_on_close;
-	const char *hwname;
-	bool no_vif;
-	const u8 *perm_addr;
-	u32 iftypes;
-	u32 *ciphers;
-	u8 n_ciphers;
-	bool mlo;
-	const struct cfg80211_pmsr_capabilities *pmsr_capa;
-	bool nan_device;
-	bool background_radar;
-};
-
-static void hwsim_mcast_config_msg(struct sk_buff *mcast_skb,
-				   struct genl_info *info)
-{
-	if (info)
-		genl_notify(&hwsim_genl_family, mcast_skb, info,
-			    HWSIM_MCGRP_CONFIG, GFP_KERNEL);
-	else
-		genlmsg_multicast(&hwsim_genl_family, mcast_skb, 0,
-				  HWSIM_MCGRP_CONFIG, GFP_KERNEL);
-}
-
-static int append_radio_msg(struct sk_buff *skb, int id,
-			    struct hwsim_new_radio_params *param)
-{
-	int ret;
-
-	ret = nla_put_u32(skb, HWSIM_ATTR_RADIO_ID, id);
-	if (ret < 0)
-		return ret;
-
-	if (param->channels) {
-		ret = nla_put_u32(skb, HWSIM_ATTR_CHANNELS, param->channels);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (param->reg_alpha2) {
-		ret = nla_put(skb, HWSIM_ATTR_REG_HINT_ALPHA2, 2,
-			      param->reg_alpha2);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (param->regd) {
-		int i;
-
-		for (i = 0; i < ARRAY_SIZE(hwsim_world_regdom_custom); i++) {
-			if (hwsim_world_regdom_custom[i] != param->regd)
-				continue;
-
-			ret = nla_put_u32(skb, HWSIM_ATTR_REG_CUSTOM_REG, i);
-			if (ret < 0)
-				return ret;
-			break;
-		}
-	}
-
-	if (param->reg_strict) {
-		ret = nla_put_flag(skb, HWSIM_ATTR_REG_STRICT_REG);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (param->p2p_device) {
-		ret = nla_put_flag(skb, HWSIM_ATTR_SUPPORT_P2P_DEVICE);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (param->use_chanctx) {
-		ret = nla_put_flag(skb, HWSIM_ATTR_USE_CHANCTX);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (param->multi_radio) {
-		ret = nla_put_flag(skb, HWSIM_ATTR_MULTI_RADIO);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (param->hwname) {
-		ret = nla_put(skb, HWSIM_ATTR_RADIO_NAME,
-			      strlen(param->hwname), param->hwname);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (param->nan_device) {
-		ret = nla_put_flag(skb, HWSIM_ATTR_SUPPORT_NAN_DEVICE);
-		if (ret < 0)
-			return ret;
-	}
-
-	if (param->background_radar) {
-		ret = nla_put_flag(skb, HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR);
-		if (ret < 0)
-			return ret;
-	}
-	return 0;
-}
-
-static void hwsim_mcast_new_radio(int id, struct genl_info *info,
-				  struct hwsim_new_radio_params *param)
-{
-	struct sk_buff *mcast_skb;
-	void *data;
-
-	mcast_skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
-	if (!mcast_skb)
-		return;
-
-	data = genlmsg_put(mcast_skb, 0, 0, &hwsim_genl_family, 0,
-			   HWSIM_CMD_NEW_RADIO);
-	if (!data)
-		goto out_err;
-
-	if (append_radio_msg(mcast_skb, id, param) < 0)
-		goto out_err;
-
-	genlmsg_end(mcast_skb, data);
-
-	hwsim_mcast_config_msg(mcast_skb, info);
-	return;
-
-out_err:
-	nlmsg_free(mcast_skb);
-}
-
-static const struct ieee80211_sband_iftype_data sband_capa_2ghz[] = {
-	{
-		.types_mask = BIT(NL80211_IFTYPE_STATION) |
-			      BIT(NL80211_IFTYPE_P2P_CLIENT),
-		.he_cap = {
-			.has_he = true,
-			.he_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_HE_MAC_CAP0_HTC_HE,
-				.mac_cap_info[1] =
-					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
-					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
-				.mac_cap_info[2] =
-					IEEE80211_HE_MAC_CAP2_BSR |
-					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
-					IEEE80211_HE_MAC_CAP2_ACK_EN,
-				.mac_cap_info[3] =
-					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
-				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
-				.phy_cap_info[0] =
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G,
-				.phy_cap_info[1] =
-					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
-					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
-					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
-					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
-				.phy_cap_info[2] =
-					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
-					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
-					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
-
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			.he_mcs_nss_supp = {
-				.rx_mcs_80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80 = cpu_to_le16(0xfffa),
-				.rx_mcs_160 = cpu_to_le16(0xffff),
-				.tx_mcs_160 = cpu_to_le16(0xffff),
-				.rx_mcs_80p80 = cpu_to_le16(0xffff),
-				.tx_mcs_80p80 = cpu_to_le16(0xffff),
-			},
-		},
-		.eht_cap = {
-			.has_eht = true,
-			.eht_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
-					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
-				.phy_cap_info[0] =
-					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
-					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
-					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE,
-				.phy_cap_info[3] =
-					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
-				.phy_cap_info[4] =
-					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
-					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
-					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
-					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
-				.phy_cap_info[5] =
-					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
-					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
-					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
-				.phy_cap_info[6] =
-					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
-					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
-				.phy_cap_info[7] =
-					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW,
-			},
-
-			/* For all MCS and bandwidth, set 8 NSS for both Tx and
-			 * Rx
-			 */
-			.eht_mcs_nss_supp = {
-				/*
-				 * Since B0, B1, B2 and B3 are not set in
-				 * the supported channel width set field in the
-				 * HE PHY capabilities information field the
-				 * device is a 20MHz only device on 2.4GHz band.
-				 */
-				.only_20mhz = {
-					.rx_tx_mcs7_max_nss = 0x88,
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-			},
-			/* PPE threshold information is not supported */
-		},
-		.uhr_cap = {
-			.has_uhr = true,
-			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
-				   IEEE80211_UHR_PHY_CAP_ELR_TX,
-		},
-	},
-	{
-		.types_mask = BIT(NL80211_IFTYPE_AP) |
-			      BIT(NL80211_IFTYPE_P2P_GO),
-		.he_cap = {
-			.has_he = true,
-			.he_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_HE_MAC_CAP0_HTC_HE,
-				.mac_cap_info[1] =
-					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
-					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
-				.mac_cap_info[2] =
-					IEEE80211_HE_MAC_CAP2_BSR |
-					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
-					IEEE80211_HE_MAC_CAP2_ACK_EN,
-				.mac_cap_info[3] =
-					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
-				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
-				.phy_cap_info[0] =
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G,
-				.phy_cap_info[1] =
-					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
-					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
-					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
-					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
-				.phy_cap_info[2] =
-					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
-					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
-					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
-
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			.he_mcs_nss_supp = {
-				.rx_mcs_80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80 = cpu_to_le16(0xfffa),
-				.rx_mcs_160 = cpu_to_le16(0xffff),
-				.tx_mcs_160 = cpu_to_le16(0xffff),
-				.rx_mcs_80p80 = cpu_to_le16(0xffff),
-				.tx_mcs_80p80 = cpu_to_le16(0xffff),
-			},
-		},
-		.eht_cap = {
-			.has_eht = true,
-			.eht_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
-					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
-				.phy_cap_info[0] =
-					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
-					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
-					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE,
-				.phy_cap_info[3] =
-					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
-				.phy_cap_info[4] =
-					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
-					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
-					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
-					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
-				.phy_cap_info[5] =
-					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
-					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
-					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
-				.phy_cap_info[6] =
-					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
-					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
-				.phy_cap_info[7] =
-					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW,
-			},
-
-			/* For all MCS and bandwidth, set 8 NSS for both Tx and
-			 * Rx
-			 */
-			.eht_mcs_nss_supp = {
-				/*
-				 * Since B0, B1, B2 and B3 are not set in
-				 * the supported channel width set field in the
-				 * HE PHY capabilities information field the
-				 * device is a 20MHz only device on 2.4GHz band.
-				 */
-				.only_20mhz = {
-					.rx_tx_mcs7_max_nss = 0x88,
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-			},
-			/* PPE threshold information is not supported */
-		},
-		.uhr_cap = {
-			.has_uhr = true,
-			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
-				   IEEE80211_UHR_PHY_CAP_ELR_TX,
-		},
-	},
-#ifdef CONFIG_MAC80211_MESH
-	{
-		.types_mask = BIT(NL80211_IFTYPE_MESH_POINT),
-		.he_cap = {
-			.has_he = true,
-			.he_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_HE_MAC_CAP0_HTC_HE,
-				.mac_cap_info[1] =
-					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
-				.mac_cap_info[2] =
-					IEEE80211_HE_MAC_CAP2_ACK_EN,
-				.mac_cap_info[3] =
-					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
-				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
-				.phy_cap_info[0] =
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G,
-				.phy_cap_info[1] =
-					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
-					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
-					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
-					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
-				.phy_cap_info[2] = 0,
-
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			.he_mcs_nss_supp = {
-				.rx_mcs_80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80 = cpu_to_le16(0xfffa),
-				.rx_mcs_160 = cpu_to_le16(0xffff),
-				.tx_mcs_160 = cpu_to_le16(0xffff),
-				.rx_mcs_80p80 = cpu_to_le16(0xffff),
-				.tx_mcs_80p80 = cpu_to_le16(0xffff),
-			},
-		},
-	},
-#endif
-};
-
-static const struct ieee80211_sband_iftype_data sband_capa_5ghz[] = {
-	{
-		.types_mask = BIT(NL80211_IFTYPE_STATION) |
-			      BIT(NL80211_IFTYPE_P2P_CLIENT),
-		.he_cap = {
-			.has_he = true,
-			.he_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_HE_MAC_CAP0_HTC_HE,
-				.mac_cap_info[1] =
-					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
-					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
-				.mac_cap_info[2] =
-					IEEE80211_HE_MAC_CAP2_BSR |
-					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
-					IEEE80211_HE_MAC_CAP2_ACK_EN,
-				.mac_cap_info[3] =
-					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
-				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
-				.phy_cap_info[0] =
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
-				.phy_cap_info[1] =
-					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
-					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
-					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
-					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
-				.phy_cap_info[2] =
-					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
-					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
-					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
-
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			.he_mcs_nss_supp = {
-				.rx_mcs_80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80 = cpu_to_le16(0xfffa),
-				.rx_mcs_160 = cpu_to_le16(0xfffa),
-				.tx_mcs_160 = cpu_to_le16(0xfffa),
-				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
-			},
-		},
-		.eht_cap = {
-			.has_eht = true,
-			.eht_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
-					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
-				.phy_cap_info[0] =
-					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
-					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
-					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
-					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
-				.phy_cap_info[1] =
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK,
-				.phy_cap_info[2] =
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK,
-				.phy_cap_info[3] =
-					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
-				.phy_cap_info[4] =
-					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
-					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
-					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
-					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
-				.phy_cap_info[5] =
-					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
-					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
-					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
-				.phy_cap_info[6] =
-					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
-					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
-				.phy_cap_info[7] =
-					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ,
-			},
-
-			/* For all MCS and bandwidth, set 8 NSS for both Tx and
-			 * Rx
-			 */
-			.eht_mcs_nss_supp = {
-				/*
-				 * As B1 and B2 are set in the supported
-				 * channel width set field in the HE PHY
-				 * capabilities information field include all
-				 * the following MCS/NSS.
-				 */
-				.bw._80 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-				.bw._160 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-			},
-			/* PPE threshold information is not supported */
-		},
-		.uhr_cap = {
-			.has_uhr = true,
-			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
-				   IEEE80211_UHR_PHY_CAP_ELR_TX,
-		},
-	},
-	{
-		.types_mask = BIT(NL80211_IFTYPE_AP) |
-			      BIT(NL80211_IFTYPE_P2P_GO),
-		.he_cap = {
-			.has_he = true,
-			.he_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_HE_MAC_CAP0_HTC_HE,
-				.mac_cap_info[1] =
-					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
-					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
-				.mac_cap_info[2] =
-					IEEE80211_HE_MAC_CAP2_BSR |
-					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
-					IEEE80211_HE_MAC_CAP2_ACK_EN,
-				.mac_cap_info[3] =
-					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
-				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
-				.phy_cap_info[0] =
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
-				.phy_cap_info[1] =
-					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
-					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
-					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
-					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
-				.phy_cap_info[2] =
-					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
-					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
-					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
-
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			.he_mcs_nss_supp = {
-				.rx_mcs_80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80 = cpu_to_le16(0xfffa),
-				.rx_mcs_160 = cpu_to_le16(0xfffa),
-				.tx_mcs_160 = cpu_to_le16(0xfffa),
-				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
-			},
-		},
-		.eht_cap = {
-			.has_eht = true,
-			.eht_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
-					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
-				.phy_cap_info[0] =
-					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
-					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
-					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
-					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
-				.phy_cap_info[1] =
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK,
-				.phy_cap_info[2] =
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK,
-				.phy_cap_info[3] =
-					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
-				.phy_cap_info[4] =
-					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
-					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
-					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
-					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
-				.phy_cap_info[5] =
-					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
-					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
-					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
-				.phy_cap_info[6] =
-					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
-					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
-				.phy_cap_info[7] =
-					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ,
-			},
-
-			/* For all MCS and bandwidth, set 8 NSS for both Tx and
-			 * Rx
-			 */
-			.eht_mcs_nss_supp = {
-				/*
-				 * As B1 and B2 are set in the supported
-				 * channel width set field in the HE PHY
-				 * capabilities information field include all
-				 * the following MCS/NSS.
-				 */
-				.bw._80 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-				.bw._160 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-			},
-			/* PPE threshold information is not supported */
-		},
-		.uhr_cap = {
-			.has_uhr = true,
-			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
-				   IEEE80211_UHR_PHY_CAP_ELR_TX,
-		},
-	},
-#ifdef CONFIG_MAC80211_MESH
-	{
-		/* TODO: should we support other types, e.g., IBSS?*/
-		.types_mask = BIT(NL80211_IFTYPE_MESH_POINT),
-		.he_cap = {
-			.has_he = true,
-			.he_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_HE_MAC_CAP0_HTC_HE,
-				.mac_cap_info[1] =
-					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
-				.mac_cap_info[2] =
-					IEEE80211_HE_MAC_CAP2_ACK_EN,
-				.mac_cap_info[3] =
-					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
-				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
-				.phy_cap_info[0] =
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
-				.phy_cap_info[1] =
-					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
-					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
-					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
-					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
-				.phy_cap_info[2] = 0,
-
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			.he_mcs_nss_supp = {
-				.rx_mcs_80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80 = cpu_to_le16(0xfffa),
-				.rx_mcs_160 = cpu_to_le16(0xfffa),
-				.tx_mcs_160 = cpu_to_le16(0xfffa),
-				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
-			},
-		},
-	},
-#endif
-};
-
-static const struct ieee80211_sband_iftype_data sband_capa_6ghz[] = {
-	{
-		.types_mask = BIT(NL80211_IFTYPE_STATION) |
-			      BIT(NL80211_IFTYPE_P2P_CLIENT),
-		.he_6ghz_capa = {
-			.capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START |
-					    IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP |
-					    IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN |
-					    IEEE80211_HE_6GHZ_CAP_SM_PS |
-					    IEEE80211_HE_6GHZ_CAP_RD_RESPONDER |
-					    IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS |
-					    IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS),
-		},
-		.he_cap = {
-			.has_he = true,
-			.he_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_HE_MAC_CAP0_HTC_HE,
-				.mac_cap_info[1] =
-					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
-					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
-				.mac_cap_info[2] =
-					IEEE80211_HE_MAC_CAP2_BSR |
-					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
-					IEEE80211_HE_MAC_CAP2_ACK_EN,
-				.mac_cap_info[3] =
-					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
-				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
-				.phy_cap_info[0] =
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
-				.phy_cap_info[1] =
-					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
-					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
-					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
-					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
-				.phy_cap_info[2] =
-					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
-					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
-					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
-
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			.he_mcs_nss_supp = {
-				.rx_mcs_80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80 = cpu_to_le16(0xfffa),
-				.rx_mcs_160 = cpu_to_le16(0xfffa),
-				.tx_mcs_160 = cpu_to_le16(0xfffa),
-				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
-			},
-		},
-		.eht_cap = {
-			.has_eht = true,
-			.eht_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
-					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
-				.phy_cap_info[0] =
-					IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ |
-					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
-					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
-					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
-					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
-				.phy_cap_info[1] =
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK,
-				.phy_cap_info[2] =
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK,
-				.phy_cap_info[3] =
-					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
-				.phy_cap_info[4] =
-					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
-					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
-					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
-					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
-				.phy_cap_info[5] =
-					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
-					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
-					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
-				.phy_cap_info[6] =
-					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
-					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK |
-					IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP,
-				.phy_cap_info[7] =
-					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_320MHZ,
-			},
-
-			/* For all MCS and bandwidth, set 8 NSS for both Tx and
-			 * Rx
-			 */
-			.eht_mcs_nss_supp = {
-				/*
-				 * As B1 and B2 are set in the supported
-				 * channel width set field in the HE PHY
-				 * capabilities information field and 320MHz in
-				 * 6GHz is supported include all the following
-				 * MCS/NSS.
-				 */
-				.bw._80 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-				.bw._160 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-				.bw._320 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-			},
-			/* PPE threshold information is not supported */
-		},
-		.uhr_cap = {
-			.has_uhr = true,
-			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
-				   IEEE80211_UHR_PHY_CAP_ELR_TX,
-		},
-	},
-	{
-		.types_mask = BIT(NL80211_IFTYPE_AP) |
-			      BIT(NL80211_IFTYPE_P2P_GO),
-		.he_6ghz_capa = {
-			.capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START |
-					    IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP |
-					    IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN |
-					    IEEE80211_HE_6GHZ_CAP_SM_PS |
-					    IEEE80211_HE_6GHZ_CAP_RD_RESPONDER |
-					    IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS |
-					    IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS),
-		},
-		.he_cap = {
-			.has_he = true,
-			.he_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_HE_MAC_CAP0_HTC_HE,
-				.mac_cap_info[1] =
-					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
-					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
-				.mac_cap_info[2] =
-					IEEE80211_HE_MAC_CAP2_BSR |
-					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
-					IEEE80211_HE_MAC_CAP2_ACK_EN,
-				.mac_cap_info[3] =
-					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
-				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
-				.phy_cap_info[0] =
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
-				.phy_cap_info[1] =
-					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
-					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
-					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
-					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
-				.phy_cap_info[2] =
-					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
-					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
-					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
-					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
-
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			.he_mcs_nss_supp = {
-				.rx_mcs_80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80 = cpu_to_le16(0xfffa),
-				.rx_mcs_160 = cpu_to_le16(0xfffa),
-				.tx_mcs_160 = cpu_to_le16(0xfffa),
-				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
-			},
-		},
-		.eht_cap = {
-			.has_eht = true,
-			.eht_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
-					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
-					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
-				.phy_cap_info[0] =
-					IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ |
-					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
-					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
-					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
-					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
-					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
-				.phy_cap_info[1] =
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK,
-				.phy_cap_info[2] =
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK |
-					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK,
-				.phy_cap_info[3] =
-					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
-					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
-				.phy_cap_info[4] =
-					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
-					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
-					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
-					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
-					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
-				.phy_cap_info[5] =
-					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
-					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
-					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
-					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
-					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
-				.phy_cap_info[6] =
-					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
-					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK |
-					IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP,
-				.phy_cap_info[7] =
-					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
-					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ |
-					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_320MHZ,
-			},
-
-			/* For all MCS and bandwidth, set 8 NSS for both Tx and
-			 * Rx
-			 */
-			.eht_mcs_nss_supp = {
-				/*
-				 * As B1 and B2 are set in the supported
-				 * channel width set field in the HE PHY
-				 * capabilities information field and 320MHz in
-				 * 6GHz is supported include all the following
-				 * MCS/NSS.
-				 */
-				.bw._80 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-				.bw._160 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-				.bw._320 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-			},
-			/* PPE threshold information is not supported */
-		},
-		.uhr_cap = {
-			.has_uhr = true,
-			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
-				   IEEE80211_UHR_PHY_CAP_ELR_TX,
-		},
-	},
-#ifdef CONFIG_MAC80211_MESH
-	{
-		/* TODO: should we support other types, e.g., IBSS?*/
-		.types_mask = BIT(NL80211_IFTYPE_MESH_POINT),
-		.he_6ghz_capa = {
-			.capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START |
-					    IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP |
-					    IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN |
-					    IEEE80211_HE_6GHZ_CAP_SM_PS |
-					    IEEE80211_HE_6GHZ_CAP_RD_RESPONDER |
-					    IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS |
-					    IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS),
-		},
-		.he_cap = {
-			.has_he = true,
-			.he_cap_elem = {
-				.mac_cap_info[0] =
-					IEEE80211_HE_MAC_CAP0_HTC_HE,
-				.mac_cap_info[1] =
-					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
-				.mac_cap_info[2] =
-					IEEE80211_HE_MAC_CAP2_ACK_EN,
-				.mac_cap_info[3] =
-					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
-					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
-				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
-				.phy_cap_info[0] =
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
-					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
-				.phy_cap_info[1] =
-					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
-					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
-					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
-					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
-				.phy_cap_info[2] = 0,
-
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			.he_mcs_nss_supp = {
-				.rx_mcs_80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80 = cpu_to_le16(0xfffa),
-				.rx_mcs_160 = cpu_to_le16(0xfffa),
-				.tx_mcs_160 = cpu_to_le16(0xfffa),
-				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
-				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
-			},
-		},
-		.eht_cap = {
-			.has_eht = true,
-			.eht_cap_elem = {
-				.mac_cap_info[0] = IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
-						   IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
-				.phy_cap_info[0] = IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ,
-				/* Leave all the other PHY capability bytes
-				 * unset, as DCM, beam forming, RU and PPE
-				 * threshold information are not supported
-				 */
-			},
-			/* For all MCS and bandwidth, set 8 NSS for both Tx and
-			 * Rx
-			 */
-			.eht_mcs_nss_supp = {
-				/* As B1 and B2 are set in the supported
-				 * channel width set field in the HE PHY
-				 * capabilities information field and 320MHz in
-				 * 6GHz is supported include all the following
-				 * MCS/NSS.
-				 */
-				.bw._80 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-				.bw._160 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-				.bw._320 = {
-					.rx_tx_mcs9_max_nss = 0x88,
-					.rx_tx_mcs11_max_nss = 0x88,
-					.rx_tx_mcs13_max_nss = 0x88,
-				},
-			},
-			/* PPE threshold information is not supported */
-		},
-		.uhr_cap = {
-			.has_uhr = true,
-			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
-				   IEEE80211_UHR_PHY_CAP_ELR_TX,
-		},
-	},
-#endif
-};
-
-static void mac80211_hwsim_sband_capab(struct ieee80211_supported_band *sband)
-{
-	switch (sband->band) {
-	case NL80211_BAND_2GHZ:
-		ieee80211_set_sband_iftype_data(sband, sband_capa_2ghz);
-		break;
-	case NL80211_BAND_5GHZ:
-		ieee80211_set_sband_iftype_data(sband, sband_capa_5ghz);
-		break;
-	case NL80211_BAND_6GHZ:
-		ieee80211_set_sband_iftype_data(sband, sband_capa_6ghz);
-		break;
-	default:
-		break;
-	}
-}
-
-#ifdef CONFIG_MAC80211_MESH
-#define HWSIM_MESH_BIT BIT(NL80211_IFTYPE_MESH_POINT)
-#else
-#define HWSIM_MESH_BIT 0
-#endif
-
-#define HWSIM_DEFAULT_IF_LIMIT \
-	(BIT(NL80211_IFTYPE_STATION) | \
-	 BIT(NL80211_IFTYPE_P2P_CLIENT) | \
-	 BIT(NL80211_IFTYPE_AP) | \
-	 BIT(NL80211_IFTYPE_P2P_GO) | \
-	 HWSIM_MESH_BIT)
-
-#define HWSIM_IFTYPE_SUPPORT_MASK \
-	(BIT(NL80211_IFTYPE_STATION) | \
-	 BIT(NL80211_IFTYPE_AP) | \
-	 BIT(NL80211_IFTYPE_P2P_CLIENT) | \
-	 BIT(NL80211_IFTYPE_P2P_GO) | \
-	 BIT(NL80211_IFTYPE_ADHOC) | \
-	 BIT(NL80211_IFTYPE_MESH_POINT) | \
-	 BIT(NL80211_IFTYPE_OCB))
-
-static const u8 iftypes_ext_capa_ap[] = {
-	 [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING,
-	 [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT,
-	 [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF |
-	       WLAN_EXT_CAPA8_MAX_MSDU_IN_AMSDU_LSB,
-	 [8] = WLAN_EXT_CAPA9_MAX_MSDU_IN_AMSDU_MSB,
-	 [9] = WLAN_EXT_CAPA10_TWT_RESPONDER_SUPPORT,
-};
-
-#define MAC80211_HWSIM_MLD_CAPA_OPS				\
-	FIELD_PREP_CONST(IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP, \
-			 IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_SAME) | \
-	FIELD_PREP_CONST(IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS, \
-			 IEEE80211_MLD_MAX_NUM_LINKS - 1)
-
-static const struct wiphy_iftype_ext_capab mac80211_hwsim_iftypes_ext_capa[] = {
-	{
-		.iftype = NL80211_IFTYPE_AP,
-		.extended_capabilities = iftypes_ext_capa_ap,
-		.extended_capabilities_mask = iftypes_ext_capa_ap,
-		.extended_capabilities_len = sizeof(iftypes_ext_capa_ap),
-		.eml_capabilities = IEEE80211_EML_CAP_EMLSR_SUPP |
-				    IEEE80211_EML_CAP_EMLMR_SUPPORT,
-		.mld_capa_and_ops = MAC80211_HWSIM_MLD_CAPA_OPS,
-	},
-};
-
-static int mac80211_hwsim_new_radio(struct genl_info *info,
-				    struct hwsim_new_radio_params *param)
-{
-	int err;
-	u8 addr[ETH_ALEN];
-	struct mac80211_hwsim_data *data;
-	struct ieee80211_hw *hw;
-	enum nl80211_band band;
-	const struct ieee80211_ops *ops = &mac80211_hwsim_ops;
-	struct net *net;
-	int idx, i;
-	int n_limits = 0;
-	int n_bands = 0;
-
-	if (WARN_ON(param->channels > 1 && !param->use_chanctx))
-		return -EINVAL;
-
-	spin_lock_bh(&hwsim_radio_lock);
-	idx = hwsim_radio_idx++;
-	spin_unlock_bh(&hwsim_radio_lock);
-
-	if (param->mlo)
-		ops = &mac80211_hwsim_mlo_ops;
-	else if (param->use_chanctx)
-		ops = &mac80211_hwsim_mchan_ops;
-	hw = ieee80211_alloc_hw_nm(sizeof(*data), ops, param->hwname);
-	if (!hw) {
-		pr_debug("mac80211_hwsim: ieee80211_alloc_hw failed\n");
-		err = -ENOMEM;
-		goto failed;
-	}
-
-	/* ieee80211_alloc_hw_nm may have used a default name */
-	param->hwname = wiphy_name(hw->wiphy);
-
-	if (info)
-		net = genl_info_net(info);
-	else
-		net = &init_net;
-	wiphy_net_set(hw->wiphy, net);
-
-	data = hw->priv;
-	data->hw = hw;
-
-	data->dev = device_create(&hwsim_class, NULL, 0, hw, "hwsim%d", idx);
-	if (IS_ERR(data->dev)) {
-		printk(KERN_DEBUG
-		       "mac80211_hwsim: device_create failed (%ld)\n",
-		       PTR_ERR(data->dev));
-		err = -ENOMEM;
-		goto failed_drvdata;
-	}
-	data->dev->driver = &mac80211_hwsim_driver.driver;
-	err = device_bind_driver(data->dev);
-	if (err != 0) {
-		pr_debug("mac80211_hwsim: device_bind_driver failed (%d)\n",
-		       err);
-		goto failed_bind;
-	}
-
-	skb_queue_head_init(&data->pending);
-
-	SET_IEEE80211_DEV(hw, data->dev);
-	if (!param->perm_addr) {
-		eth_zero_addr(addr);
-		addr[0] = 0x02;
-		addr[3] = idx >> 8;
-		addr[4] = idx;
-		memcpy(data->addresses[0].addr, addr, ETH_ALEN);
-		/* Why need here second address ? */
-		memcpy(data->addresses[1].addr, addr, ETH_ALEN);
-		data->addresses[1].addr[0] |= 0x40;
-		memcpy(data->addresses[2].addr, addr, ETH_ALEN);
-		data->addresses[2].addr[0] |= 0x50;
-
-		hw->wiphy->n_addresses = 3;
-		hw->wiphy->addresses = data->addresses;
-		/* possible address clash is checked at hash table insertion */
-	} else {
-		memcpy(data->addresses[0].addr, param->perm_addr, ETH_ALEN);
-		/* compatibility with automatically generated mac addr */
-		memcpy(data->addresses[1].addr, param->perm_addr, ETH_ALEN);
-		memcpy(data->addresses[2].addr, param->perm_addr, ETH_ALEN);
-		hw->wiphy->n_addresses = 3;
-		hw->wiphy->addresses = data->addresses;
-	}
-
-	data->channels = param->channels;
-	data->use_chanctx = param->use_chanctx;
-	data->idx = idx;
-	data->destroy_on_close = param->destroy_on_close;
-	if (info)
-		data->portid = info->snd_portid;
-
-	/* setup interface limits, only on interface types we support */
-	if (param->iftypes & BIT(NL80211_IFTYPE_ADHOC)) {
-		data->if_limits[n_limits].max = 1;
-		data->if_limits[n_limits].types = BIT(NL80211_IFTYPE_ADHOC);
-		n_limits++;
-	}
-
-	if (param->iftypes & HWSIM_DEFAULT_IF_LIMIT) {
-		data->if_limits[n_limits].max = 2048;
-		/*
-		 * For this case, we may only support a subset of
-		 * HWSIM_DEFAULT_IF_LIMIT, therefore we only want to add the
-		 * bits that both param->iftype & HWSIM_DEFAULT_IF_LIMIT have.
-		 */
-		data->if_limits[n_limits].types =
-					HWSIM_DEFAULT_IF_LIMIT & param->iftypes;
-		n_limits++;
-	}
-
-	if (param->iftypes & BIT(NL80211_IFTYPE_P2P_DEVICE)) {
-		data->if_limits[n_limits].max = 1;
-		data->if_limits[n_limits].types =
-						BIT(NL80211_IFTYPE_P2P_DEVICE);
-		n_limits++;
-	}
-
-	if (param->iftypes & BIT(NL80211_IFTYPE_NAN)) {
-		data->if_limits[n_limits].max = 1;
-		data->if_limits[n_limits].types = BIT(NL80211_IFTYPE_NAN);
-		n_limits++;
-
-		hw->wiphy->nan_supported_bands = BIT(NL80211_BAND_2GHZ) |
-						 BIT(NL80211_BAND_5GHZ);
-
-		hw->wiphy->nan_capa.flags = WIPHY_NAN_FLAGS_CONFIGURABLE_SYNC |
-					    WIPHY_NAN_FLAGS_USERSPACE_DE;
-		hw->wiphy->nan_capa.op_mode = NAN_OP_MODE_PHY_MODE_MASK |
-					      NAN_OP_MODE_80P80MHZ |
-					      NAN_OP_MODE_160MHZ;
-
-		hw->wiphy->nan_capa.n_antennas = 0x22;
-		hw->wiphy->nan_capa.max_channel_switch_time = 0;
-		hw->wiphy->nan_capa.dev_capabilities =
-			NAN_DEV_CAPA_EXT_KEY_ID_SUPPORTED |
-			NAN_DEV_CAPA_NDPE_SUPPORTED;
-
-		hrtimer_setup(&data->nan_timer, mac80211_hwsim_nan_dw_start,
-			      CLOCK_MONOTONIC, HRTIMER_MODE_ABS_SOFT);
-	}
-
-	data->if_combination.radar_detect_widths =
-				BIT(NL80211_CHAN_WIDTH_5) |
-				BIT(NL80211_CHAN_WIDTH_10) |
-				BIT(NL80211_CHAN_WIDTH_20_NOHT) |
-				BIT(NL80211_CHAN_WIDTH_20) |
-				BIT(NL80211_CHAN_WIDTH_40) |
-				BIT(NL80211_CHAN_WIDTH_80) |
-				BIT(NL80211_CHAN_WIDTH_160);
-
-	if (data->use_chanctx) {
-		hw->wiphy->max_scan_ssids = 255;
-		hw->wiphy->max_scan_ie_len = IEEE80211_MAX_DATA_LEN;
-		hw->wiphy->max_remain_on_channel_duration = 1000;
-		data->if_combination.num_different_channels = data->channels;
-	} else {
-		data->if_combination.num_different_channels = 1;
-	}
-
-	if (!n_limits) {
-		err = -EINVAL;
-		goto failed_hw;
-	}
-
-	data->if_combination.max_interfaces = 0;
-	for (i = 0; i < n_limits; i++)
-		data->if_combination.max_interfaces +=
-			data->if_limits[i].max;
-
-	data->if_combination.n_limits = n_limits;
-	data->if_combination.limits = data->if_limits;
-
-	/*
-	 * If we actually were asked to support combinations,
-	 * advertise them - if there's only a single thing like
-	 * only IBSS then don't advertise it as combinations.
-	 */
-	if (data->if_combination.max_interfaces > 1) {
-		hw->wiphy->iface_combinations = &data->if_combination;
-		hw->wiphy->n_iface_combinations = 1;
-	}
-
-	if (param->ciphers) {
-		memcpy(data->ciphers, param->ciphers,
-		       param->n_ciphers * sizeof(u32));
-		hw->wiphy->cipher_suites = data->ciphers;
-		hw->wiphy->n_cipher_suites = param->n_ciphers;
-	}
-
-	hw->wiphy->mbssid_max_interfaces = 8;
-	hw->wiphy->ema_max_profile_periodicity = 3;
-
-	data->rx_rssi = DEFAULT_RX_RSSI;
-
-	INIT_DELAYED_WORK(&data->roc_start, hw_roc_start);
-	INIT_DELAYED_WORK(&data->roc_done, hw_roc_done);
-	INIT_DELAYED_WORK(&data->hw_scan, hw_scan_work);
-
-	hw->queues = 5;
-	hw->offchannel_tx_hw_queue = 4;
-
-	ieee80211_hw_set(hw, SUPPORT_FAST_XMIT);
-	ieee80211_hw_set(hw, CHANCTX_STA_CSA);
-	ieee80211_hw_set(hw, SUPPORTS_HT_CCK_RATES);
-	ieee80211_hw_set(hw, QUEUE_CONTROL);
-	ieee80211_hw_set(hw, WANT_MONITOR_VIF);
-	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
-	ieee80211_hw_set(hw, MFP_CAPABLE);
-	ieee80211_hw_set(hw, SIGNAL_DBM);
-	ieee80211_hw_set(hw, SUPPORTS_PS);
-	ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS);
-	ieee80211_hw_set(hw, TDLS_WIDER_BW);
-	ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID);
-	ieee80211_hw_set(hw, STRICT);
-
-	if (param->mlo) {
-		hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_MLO;
-		ieee80211_hw_set(hw, HAS_RATE_CONTROL);
-		ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS);
-		ieee80211_hw_set(hw, CONNECTION_MONITOR);
-		ieee80211_hw_set(hw, AP_LINK_PS);
-
-		hw->wiphy->iftype_ext_capab = mac80211_hwsim_iftypes_ext_capa;
-		hw->wiphy->num_iftype_ext_capab =
-			ARRAY_SIZE(mac80211_hwsim_iftypes_ext_capa);
-	} else {
-		ieee80211_hw_set(hw, HOST_BROADCAST_PS_BUFFERING);
-		ieee80211_hw_set(hw, PS_NULLFUNC_STACK);
-		if (rctbl)
-			ieee80211_hw_set(hw, SUPPORTS_RC_TABLE);
-	}
-
-	hw->wiphy->flags &= ~WIPHY_FLAG_PS_ON_BY_DEFAULT;
-	hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS |
-			    WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL |
-			    WIPHY_FLAG_AP_UAPSD |
-			    WIPHY_FLAG_SUPPORTS_5_10_MHZ |
-			    WIPHY_FLAG_HAS_CHANNEL_SWITCH;
-	hw->wiphy->flags |= WIPHY_FLAG_IBSS_RSN;
-	hw->wiphy->features |= NL80211_FEATURE_ACTIVE_MONITOR |
-			       NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE |
-			       NL80211_FEATURE_STATIC_SMPS |
-			       NL80211_FEATURE_DYNAMIC_SMPS |
-			       NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR |
-			       NL80211_FEATURE_AP_SCAN;
-	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_VHT_IBSS);
-	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_BEACON_PROTECTION);
-	wiphy_ext_feature_set(hw->wiphy,
-			      NL80211_EXT_FEATURE_MULTICAST_REGISTRATIONS);
-	wiphy_ext_feature_set(hw->wiphy,
-			      NL80211_EXT_FEATURE_BEACON_RATE_LEGACY);
-	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER);
-
-	wiphy_ext_feature_set(hw->wiphy,
-			      NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT);
-	wiphy_ext_feature_set(hw->wiphy,
-			      NL80211_EXT_FEATURE_BSS_COLOR);
-	wiphy_ext_feature_set(hw->wiphy,
-			      NL80211_EXT_FEATURE_SPP_AMSDU_SUPPORT);
-	wiphy_ext_feature_set(hw->wiphy,
-			      NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
-	wiphy_ext_feature_set(hw->wiphy,
-			      NL80211_EXT_FEATURE_EXT_KEY_ID);
-	wiphy_ext_feature_set(hw->wiphy,
-			      NL80211_EXT_FEATURE_ASSOC_FRAME_ENCRYPTION);
-
-	hw->wiphy->interface_modes = param->iftypes;
-
-	/* ask mac80211 to reserve space for magic */
-	hw->vif_data_size = sizeof(struct hwsim_vif_priv);
-	hw->sta_data_size = sizeof(struct hwsim_sta_priv);
-	hw->chanctx_data_size = sizeof(struct hwsim_chanctx_priv);
-
-	memcpy(data->channels_2ghz, hwsim_channels_2ghz,
-		sizeof(hwsim_channels_2ghz));
-	memcpy(data->channels_5ghz, hwsim_channels_5ghz,
-		sizeof(hwsim_channels_5ghz));
-	memcpy(data->channels_6ghz, hwsim_channels_6ghz,
-		sizeof(hwsim_channels_6ghz));
-	memcpy(data->channels_s1g, hwsim_channels_s1g,
-	       sizeof(hwsim_channels_s1g));
-	memcpy(data->rates, hwsim_rates, sizeof(hwsim_rates));
-
-	for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) {
-		struct ieee80211_supported_band *sband = &data->bands[band];
-		struct wiphy_radio_freq_range *radio_range;
-		const struct ieee80211_channel *c;
-		struct wiphy_radio *radio;
-
-		sband->band = band;
-
-		switch (band) {
-		case NL80211_BAND_2GHZ:
-			sband->channels = data->channels_2ghz;
-			sband->n_channels = ARRAY_SIZE(hwsim_channels_2ghz);
-			sband->bitrates = data->rates;
-			sband->n_bitrates = ARRAY_SIZE(hwsim_rates);
-			break;
-		case NL80211_BAND_5GHZ:
-			sband->channels = data->channels_5ghz;
-			sband->n_channels = ARRAY_SIZE(hwsim_channels_5ghz);
-			sband->bitrates = data->rates + 4;
-			sband->n_bitrates = ARRAY_SIZE(hwsim_rates) - 4;
-
-			sband->vht_cap.vht_supported = true;
-			sband->vht_cap.cap =
-				IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454 |
-				IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ |
-				IEEE80211_VHT_CAP_RXLDPC |
-				IEEE80211_VHT_CAP_SHORT_GI_80 |
-				IEEE80211_VHT_CAP_SHORT_GI_160 |
-				IEEE80211_VHT_CAP_TXSTBC |
-				IEEE80211_VHT_CAP_RXSTBC_4 |
-				IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK;
-			sband->vht_cap.vht_mcs.rx_mcs_map =
-				cpu_to_le16(IEEE80211_VHT_MCS_SUPPORT_0_9 << 0 |
-					    IEEE80211_VHT_MCS_SUPPORT_0_9 << 2 |
-					    IEEE80211_VHT_MCS_SUPPORT_0_9 << 4 |
-					    IEEE80211_VHT_MCS_SUPPORT_0_9 << 6 |
-					    IEEE80211_VHT_MCS_SUPPORT_0_9 << 8 |
-					    IEEE80211_VHT_MCS_SUPPORT_0_9 << 10 |
-					    IEEE80211_VHT_MCS_SUPPORT_0_9 << 12 |
-					    IEEE80211_VHT_MCS_SUPPORT_0_9 << 14);
-			sband->vht_cap.vht_mcs.tx_mcs_map =
-				sband->vht_cap.vht_mcs.rx_mcs_map;
-			break;
-		case NL80211_BAND_6GHZ:
-			sband->channels = data->channels_6ghz;
-			sband->n_channels = ARRAY_SIZE(hwsim_channels_6ghz);
-			sband->bitrates = data->rates + 4;
-			sband->n_bitrates = ARRAY_SIZE(hwsim_rates) - 4;
-			break;
-		case NL80211_BAND_S1GHZ:
-			memcpy(&sband->s1g_cap, &hwsim_s1g_cap,
-			       sizeof(sband->s1g_cap));
-			sband->channels = data->channels_s1g;
-			sband->n_channels = ARRAY_SIZE(hwsim_channels_s1g);
-			break;
-		default:
-			continue;
-		}
-
-		if (band != NL80211_BAND_6GHZ){
-			sband->ht_cap.ht_supported = true;
-			sband->ht_cap.cap = IEEE80211_HT_CAP_SUP_WIDTH_20_40 |
-					    IEEE80211_HT_CAP_GRN_FLD |
-					    IEEE80211_HT_CAP_SGI_20 |
-					    IEEE80211_HT_CAP_SGI_40 |
-					    IEEE80211_HT_CAP_DSSSCCK40;
-			sband->ht_cap.ampdu_factor = 0x3;
-			sband->ht_cap.ampdu_density = 0x6;
-			memset(&sband->ht_cap.mcs, 0,
-			       sizeof(sband->ht_cap.mcs));
-			sband->ht_cap.mcs.rx_mask[0] = 0xff;
-			sband->ht_cap.mcs.rx_mask[1] = 0xff;
-			sband->ht_cap.mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED;
-		}
-
-		mac80211_hwsim_sband_capab(sband);
-
-		hw->wiphy->bands[band] = sband;
-
-		if (!param->multi_radio)
-			continue;
-
-		c = sband->channels;
-		radio_range = &data->radio_range[n_bands];
-		radio_range->start_freq = ieee80211_channel_to_khz(c) - 10000;
-
-		c += sband->n_channels - 1;
-		radio_range->end_freq = ieee80211_channel_to_khz(c) + 10000;
-
-		radio = &data->radio[n_bands++];
-		radio->freq_range = radio_range;
-		radio->n_freq_range = 1;
-		radio->iface_combinations = &data->if_combination_radio;
-		radio->n_iface_combinations = 1;
-	}
-
-	if (param->multi_radio) {
-		hw->wiphy->radio = data->radio;
-		hw->wiphy->n_radio = n_bands;
-
-		memcpy(&data->if_combination_radio, &data->if_combination,
-		       sizeof(data->if_combination));
-		data->if_combination.num_different_channels *= n_bands;
-	}
-
-	if (data->use_chanctx)
-		data->if_combination.radar_detect_widths = 0;
-
-	/* By default all radios belong to the first group */
-	data->group = 1;
-	mutex_init(&data->mutex);
-
-	data->netgroup = hwsim_net_get_netgroup(net);
-	data->wmediumd = hwsim_net_get_wmediumd(net);
-
-	/* Enable frame retransmissions for lossy channels */
-	hw->max_rates = 4;
-	hw->max_rate_tries = 11;
-
-	hw->wiphy->vendor_commands = mac80211_hwsim_vendor_commands;
-	hw->wiphy->n_vendor_commands =
-		ARRAY_SIZE(mac80211_hwsim_vendor_commands);
-	hw->wiphy->vendor_events = mac80211_hwsim_vendor_events;
-	hw->wiphy->n_vendor_events = ARRAY_SIZE(mac80211_hwsim_vendor_events);
-
-	if (param->reg_strict)
-		hw->wiphy->regulatory_flags |= REGULATORY_STRICT_REG;
-	if (param->regd) {
-		data->regd = param->regd;
-		hw->wiphy->regulatory_flags |= REGULATORY_CUSTOM_REG;
-		wiphy_apply_custom_regulatory(hw->wiphy, param->regd);
-		/* give the regulatory workqueue a chance to run */
-		schedule_timeout_interruptible(1);
-	}
-
-	wiphy_ext_feature_set(hw->wiphy,
-			      NL80211_EXT_FEATURE_DFS_CONCURRENT);
-	if (param->background_radar)
-		wiphy_ext_feature_set(hw->wiphy,
-				      NL80211_EXT_FEATURE_RADAR_BACKGROUND);
-
-	if (param->no_vif)
-		ieee80211_hw_set(hw, NO_AUTO_VIF);
-
-	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST);
-	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_PUNCT);
-
-	for (i = 0; i < ARRAY_SIZE(data->link_data); i++) {
-		hrtimer_setup(&data->link_data[i].beacon_timer, mac80211_hwsim_beacon,
-			      CLOCK_MONOTONIC, HRTIMER_MODE_ABS_SOFT);
-		data->link_data[i].link_id = i;
-	}
-
-	err = ieee80211_register_hw(hw);
-	if (err < 0) {
-		pr_debug("mac80211_hwsim: ieee80211_register_hw failed (%d)\n",
-		       err);
-		goto failed_hw;
-	}
-
-	wiphy_dbg(hw->wiphy, "hwaddr %pM registered\n", hw->wiphy->perm_addr);
-
-	if (param->reg_alpha2) {
-		data->alpha2[0] = param->reg_alpha2[0];
-		data->alpha2[1] = param->reg_alpha2[1];
-		regulatory_hint(hw->wiphy, param->reg_alpha2);
-	}
-
-	data->debugfs = debugfs_create_dir("hwsim", hw->wiphy->debugfsdir);
-	debugfs_create_file("ps", 0666, data->debugfs, data, &hwsim_fops_ps);
-	debugfs_create_file("group", 0666, data->debugfs, data,
-			    &hwsim_fops_group);
-	debugfs_create_file("rx_rssi", 0666, data->debugfs, data,
-			    &hwsim_fops_rx_rssi);
-	if (!data->use_chanctx)
-		debugfs_create_file("dfs_simulate_radar", 0222,
-				    data->debugfs,
-				    data, &hwsim_simulate_radar);
-	if (param->background_radar)
-		debugfs_create_file("dfs_background_cac", 0200,
-				    data->debugfs,
-				    data, &hwsim_background_cac_ops);
-	debugfs_create_file("simulate_incumbent_signal_interference", 0200,
-			    data->debugfs,
-			    data, &hwsim_simulate_incumbent_signal_fops);
-
-	if (param->pmsr_capa) {
-		data->pmsr_capa = *param->pmsr_capa;
-		hw->wiphy->pmsr_capa = &data->pmsr_capa;
-	}
-
-	spin_lock_bh(&hwsim_radio_lock);
-	err = rhashtable_insert_fast(&hwsim_radios_rht, &data->rht,
-				     hwsim_rht_params);
-	if (err < 0) {
-		if (info) {
-			GENL_SET_ERR_MSG(info, "perm addr already present");
-			NL_SET_BAD_ATTR(info->extack,
-					info->attrs[HWSIM_ATTR_PERM_ADDR]);
-		}
-		spin_unlock_bh(&hwsim_radio_lock);
-		goto failed_final_insert;
-	}
-
-	list_add_tail(&data->list, &hwsim_radios);
-	hwsim_radios_generation++;
-	spin_unlock_bh(&hwsim_radio_lock);
-
-	hwsim_mcast_new_radio(idx, info, param);
-
-	return idx;
-
-failed_final_insert:
-	debugfs_remove_recursive(data->debugfs);
-	ieee80211_unregister_hw(data->hw);
-failed_hw:
-	device_release_driver(data->dev);
-failed_bind:
-	device_unregister(data->dev);
-failed_drvdata:
-	ieee80211_free_hw(hw);
-failed:
-	return err;
-}
-
-static void hwsim_mcast_del_radio(int id, const char *hwname,
-				  struct genl_info *info)
-{
-	struct sk_buff *skb;
-	void *data;
-	int ret;
-
-	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
-	if (!skb)
-		return;
-
-	data = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0,
-			   HWSIM_CMD_DEL_RADIO);
-	if (!data)
-		goto error;
-
-	ret = nla_put_u32(skb, HWSIM_ATTR_RADIO_ID, id);
-	if (ret < 0)
-		goto error;
-
-	ret = nla_put(skb, HWSIM_ATTR_RADIO_NAME, strlen(hwname),
-		      hwname);
-	if (ret < 0)
-		goto error;
-
-	genlmsg_end(skb, data);
-
-	hwsim_mcast_config_msg(skb, info);
-
-	return;
-
-error:
-	nlmsg_free(skb);
-}
-
-static void mac80211_hwsim_del_radio(struct mac80211_hwsim_data *data,
-				     const char *hwname,
-				     struct genl_info *info)
-{
-	hwsim_mcast_del_radio(data->idx, hwname, info);
-	debugfs_remove_recursive(data->debugfs);
-	ieee80211_unregister_hw(data->hw);
-	device_release_driver(data->dev);
-	device_unregister(data->dev);
-	ieee80211_free_hw(data->hw);
-}
-
-static int mac80211_hwsim_get_radio(struct sk_buff *skb,
-				    struct mac80211_hwsim_data *data,
-				    u32 portid, u32 seq,
-				    struct netlink_callback *cb, int flags)
-{
-	void *hdr;
-	struct hwsim_new_radio_params param = { };
-	int res = -EMSGSIZE;
-
-	hdr = genlmsg_put(skb, portid, seq, &hwsim_genl_family, flags,
-			  HWSIM_CMD_GET_RADIO);
-	if (!hdr)
-		return -EMSGSIZE;
-
-	if (cb)
-		genl_dump_check_consistent(cb, hdr);
-
-	if (data->alpha2[0] && data->alpha2[1])
-		param.reg_alpha2 = data->alpha2;
-
-	param.reg_strict = !!(data->hw->wiphy->regulatory_flags &
-					REGULATORY_STRICT_REG);
-	param.p2p_device = !!(data->hw->wiphy->interface_modes &
-					BIT(NL80211_IFTYPE_P2P_DEVICE));
-	param.nan_device = !!(data->hw->wiphy->interface_modes &
-					BIT(NL80211_IFTYPE_NAN));
-	param.use_chanctx = data->use_chanctx;
-	param.regd = data->regd;
-	param.channels = data->channels;
-	param.hwname = wiphy_name(data->hw->wiphy);
-	param.pmsr_capa = &data->pmsr_capa;
-	param.background_radar =
-		wiphy_ext_feature_isset(data->hw->wiphy,
-					NL80211_EXT_FEATURE_RADAR_BACKGROUND);
-
-	res = append_radio_msg(skb, data->idx, &param);
-	if (res < 0)
-		goto out_err;
-
-	genlmsg_end(skb, hdr);
-	return 0;
-
-out_err:
-	genlmsg_cancel(skb, hdr);
-	return res;
-}
-
-static void mac80211_hwsim_free(void)
-{
-	struct mac80211_hwsim_data *data;
-
-	spin_lock_bh(&hwsim_radio_lock);
-	while ((data = list_first_entry_or_null(&hwsim_radios,
-						struct mac80211_hwsim_data,
-						list))) {
-		list_del(&data->list);
-		spin_unlock_bh(&hwsim_radio_lock);
-		mac80211_hwsim_del_radio(data, wiphy_name(data->hw->wiphy),
-					 NULL);
-		spin_lock_bh(&hwsim_radio_lock);
-	}
-	spin_unlock_bh(&hwsim_radio_lock);
-	class_unregister(&hwsim_class);
-}
-
-static const struct net_device_ops hwsim_netdev_ops = {
-	.ndo_start_xmit 	= hwsim_mon_xmit,
-	.ndo_set_mac_address 	= eth_mac_addr,
-	.ndo_validate_addr	= eth_validate_addr,
-};
-
-static void hwsim_mon_setup(struct net_device *dev)
-{
-	u8 addr[ETH_ALEN];
-
-	dev->netdev_ops = &hwsim_netdev_ops;
-	dev->needs_free_netdev = true;
-	ether_setup(dev);
-	dev->priv_flags |= IFF_NO_QUEUE;
-	dev->type = ARPHRD_IEEE80211_RADIOTAP;
-	eth_zero_addr(addr);
-	addr[0] = 0x12;
-	eth_hw_addr_set(dev, addr);
-}
-
-static void hwsim_register_wmediumd(struct net *net, u32 portid)
-{
-	struct mac80211_hwsim_data *data;
-
-	hwsim_net_set_wmediumd(net, portid);
-
-	spin_lock_bh(&hwsim_radio_lock);
-	list_for_each_entry(data, &hwsim_radios, list) {
-		if (data->netgroup == hwsim_net_get_netgroup(net))
-			data->wmediumd = portid;
-	}
-	spin_unlock_bh(&hwsim_radio_lock);
-}
-
-static int hwsim_tx_info_frame_received_nl(struct sk_buff *skb_2,
-					   struct genl_info *info)
-{
-
-	struct ieee80211_hdr *hdr;
-	struct mac80211_hwsim_data *data2;
-	struct ieee80211_tx_info *txi;
-	struct hwsim_tx_rate *tx_attempts;
-	u64 ret_skb_cookie;
-	struct sk_buff *skb, *tmp;
-	const u8 *src;
-	unsigned int hwsim_flags;
-	int i;
-	unsigned long flags;
-	bool found = false;
-
-	if (!info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER] ||
-	    !info->attrs[HWSIM_ATTR_FLAGS] ||
-	    !info->attrs[HWSIM_ATTR_COOKIE] ||
-	    !info->attrs[HWSIM_ATTR_SIGNAL] ||
-	    !info->attrs[HWSIM_ATTR_TX_INFO])
-		goto out;
-
-	src = (void *)nla_data(info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER]);
-	hwsim_flags = nla_get_u32(info->attrs[HWSIM_ATTR_FLAGS]);
-	ret_skb_cookie = nla_get_u64(info->attrs[HWSIM_ATTR_COOKIE]);
-
-	data2 = get_hwsim_data_ref_from_addr(src);
-	if (!data2)
-		goto out;
-
-	if (!hwsim_virtio_enabled) {
-		if (hwsim_net_get_netgroup(genl_info_net(info)) !=
-		    data2->netgroup)
-			goto out;
-
-		if (info->snd_portid != data2->wmediumd)
-			goto out;
-	}
-
-	/* look for the skb matching the cookie passed back from user */
-	spin_lock_irqsave(&data2->pending.lock, flags);
-	skb_queue_walk_safe(&data2->pending, skb, tmp) {
-		uintptr_t skb_cookie;
-
-		txi = IEEE80211_SKB_CB(skb);
-		skb_cookie = (uintptr_t)txi->rate_driver_data[0];
-
-		if (skb_cookie == ret_skb_cookie) {
-			__skb_unlink(skb, &data2->pending);
-			found = true;
-			break;
-		}
-	}
-	spin_unlock_irqrestore(&data2->pending.lock, flags);
-
-	/* not found */
-	if (!found)
-		goto out;
-
-	/* Tx info received because the frame was broadcasted on user space,
-	 so we get all the necessary info: tx attempts and skb control buff */
-
-	tx_attempts = (struct hwsim_tx_rate *)nla_data(
-		       info->attrs[HWSIM_ATTR_TX_INFO]);
-
-	/* now send back TX status */
-	txi = IEEE80211_SKB_CB(skb);
-
-	ieee80211_tx_info_clear_status(txi);
-
-	for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) {
-		txi->status.rates[i].idx = tx_attempts[i].idx;
-		txi->status.rates[i].count = tx_attempts[i].count;
-	}
-
-	txi->status.ack_signal = nla_get_u32(info->attrs[HWSIM_ATTR_SIGNAL]);
-
-	if (!(hwsim_flags & HWSIM_TX_CTL_NO_ACK) &&
-	   (hwsim_flags & HWSIM_TX_STAT_ACK)) {
-		if (skb->len >= 16) {
-			hdr = (struct ieee80211_hdr *) skb->data;
-			mac80211_hwsim_monitor_ack(data2->channel,
-						   hdr->addr2);
-		}
-		txi->flags |= IEEE80211_TX_STAT_ACK;
-	}
-
-	if (hwsim_flags & HWSIM_TX_CTL_NO_ACK)
-		txi->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
-
-	ieee80211_tx_status_irqsafe(data2->hw, skb);
-	return 0;
-out:
-	return -EINVAL;
-
-}
-
-static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2,
-					  struct genl_info *info)
-{
-	struct mac80211_hwsim_data *data2;
-	struct ieee80211_rx_status rx_status;
-	struct ieee80211_hdr *hdr;
-	const u8 *dst;
-	int frame_data_len;
-	void *frame_data;
-	struct sk_buff *skb = NULL;
-	struct ieee80211_channel *channel = NULL;
-
-	if (!info->attrs[HWSIM_ATTR_ADDR_RECEIVER] ||
-	    !info->attrs[HWSIM_ATTR_FRAME] ||
-	    !info->attrs[HWSIM_ATTR_RX_RATE] ||
-	    !info->attrs[HWSIM_ATTR_SIGNAL])
-		goto out;
-
-	dst = (void *)nla_data(info->attrs[HWSIM_ATTR_ADDR_RECEIVER]);
-	frame_data_len = nla_len(info->attrs[HWSIM_ATTR_FRAME]);
-	frame_data = (void *)nla_data(info->attrs[HWSIM_ATTR_FRAME]);
-
-	if (frame_data_len < sizeof(struct ieee80211_hdr_3addr) ||
-	    frame_data_len > IEEE80211_MAX_DATA_LEN)
-		goto err;
-
-	/* Allocate new skb here */
-	skb = alloc_skb(frame_data_len, GFP_KERNEL);
-	if (skb == NULL)
-		goto err;
-
-	/* Copy the data */
-	skb_put_data(skb, frame_data, frame_data_len);
-
-	data2 = get_hwsim_data_ref_from_addr(dst);
-	if (!data2)
-		goto out;
-
-	if (data2->use_chanctx) {
-		if (data2->tmp_chan)
-			channel = data2->tmp_chan;
-	} else {
-		channel = data2->channel;
-	}
-
-	if (!hwsim_virtio_enabled) {
-		if (hwsim_net_get_netgroup(genl_info_net(info)) !=
-		    data2->netgroup)
-			goto out;
-
-		if (info->snd_portid != data2->wmediumd)
-			goto out;
-	}
-
-	/* check if radio is configured properly */
-
-	if ((data2->idle && !data2->tmp_chan) || !data2->started)
-		goto out;
-
-	/* A frame is received from user space */
-	memset(&rx_status, 0, sizeof(rx_status));
-	if (info->attrs[HWSIM_ATTR_FREQ]) {
-		struct tx_iter_data iter_data = {};
-
-		/* throw away off-channel packets, but allow both the temporary
-		 * ("hw" scan/remain-on-channel), regular channels and links,
-		 * since the internal datapath also allows this
-		 */
-		rx_status.freq = nla_get_u32(info->attrs[HWSIM_ATTR_FREQ]);
-
-		iter_data.channel = ieee80211_get_channel(data2->hw->wiphy,
-							  rx_status.freq);
-		if (!iter_data.channel)
-			goto out;
-		rx_status.band = iter_data.channel->band;
-
-		mutex_lock(&data2->mutex);
-		if (!hwsim_chans_compat(iter_data.channel, channel)) {
-			ieee80211_iterate_active_interfaces_atomic(
-				data2->hw, IEEE80211_IFACE_ITER_NORMAL,
-				mac80211_hwsim_tx_iter, &iter_data);
-			if (!iter_data.receive) {
-				mutex_unlock(&data2->mutex);
-				goto out;
-			}
-		}
-		mutex_unlock(&data2->mutex);
-	} else if (!channel) {
-		goto out;
-	} else {
-		rx_status.freq = channel->center_freq;
-		rx_status.band = channel->band;
-	}
-
-	rx_status.rate_idx = nla_get_u32(info->attrs[HWSIM_ATTR_RX_RATE]);
-	if (rx_status.rate_idx >= data2->hw->wiphy->bands[rx_status.band]->n_bitrates)
-		goto out;
-	rx_status.signal = nla_get_u32(info->attrs[HWSIM_ATTR_SIGNAL]);
-
-	hdr = (void *)skb->data;
-
-	if (ieee80211_is_beacon(hdr->frame_control) ||
-	    ieee80211_is_probe_resp(hdr->frame_control))
-		rx_status.boottime_ns = ktime_get_boottime_ns();
-
-	mac80211_hwsim_rx(data2, &rx_status, skb);
-
-	return 0;
-err:
-	pr_debug("mac80211_hwsim: error occurred in %s\n", __func__);
-out:
-	dev_kfree_skb(skb);
-	return -EINVAL;
-}
-
-static int hwsim_register_received_nl(struct sk_buff *skb_2,
-				      struct genl_info *info)
-{
-	struct net *net = genl_info_net(info);
-	struct mac80211_hwsim_data *data;
-	int chans = 1;
-
-	spin_lock_bh(&hwsim_radio_lock);
-	list_for_each_entry(data, &hwsim_radios, list)
-		chans = max(chans, data->channels);
-	spin_unlock_bh(&hwsim_radio_lock);
-
-	/* In the future we should revise the userspace API and allow it
-	 * to set a flag that it does support multi-channel, then we can
-	 * let this pass conditionally on the flag.
-	 * For current userspace, prohibit it since it won't work right.
-	 */
-	if (chans > 1)
-		return -EOPNOTSUPP;
-
-	if (hwsim_net_get_wmediumd(net))
-		return -EBUSY;
-
-	hwsim_register_wmediumd(net, info->snd_portid);
-
-	pr_debug("mac80211_hwsim: received a REGISTER, "
-	       "switching to wmediumd mode with pid %d\n", info->snd_portid);
-
-	return 0;
-}
-
-/* ensures ciphers only include ciphers listed in 'hwsim_ciphers' array */
-static bool hwsim_known_ciphers(const u32 *ciphers, int n_ciphers)
-{
-	int i;
-
-	for (i = 0; i < n_ciphers; i++) {
-		int j;
-		int found = 0;
-
-		for (j = 0; j < ARRAY_SIZE(hwsim_ciphers); j++) {
-			if (ciphers[i] == hwsim_ciphers[j]) {
-				found = 1;
-				break;
-			}
-		}
-
-		if (!found)
-			return false;
-	}
-
-	return true;
-}
-
-static int parse_ftm_capa(const struct nlattr *ftm_capa, struct cfg80211_pmsr_capabilities *out,
-			  struct genl_info *info)
-{
-	struct nlattr *tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX + 1];
-	int ret;
-
-	ret = nla_parse_nested(tb, NL80211_PMSR_FTM_CAPA_ATTR_MAX, ftm_capa, hwsim_ftm_capa_policy,
-			       NULL);
-	if (ret) {
-		NL_SET_ERR_MSG_ATTR(info->extack, ftm_capa, "malformed FTM capability");
-		return -EINVAL;
-	}
-
-	out->ftm.supported = 1;
-	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES])
-		out->ftm.preambles = nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES]);
-	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS])
-		out->ftm.bandwidths = nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS]);
-	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT])
-		out->ftm.max_bursts_exponent =
-			nla_get_u8(tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT]);
-	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST])
-		out->ftm.max_ftms_per_burst =
-			nla_get_u8(tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST]);
-	out->ftm.asap = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_ASAP];
-	out->ftm.non_asap = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_NON_ASAP];
-	out->ftm.request_lci = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_REQ_LCI];
-	out->ftm.request_civicloc = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_REQ_CIVICLOC];
-	out->ftm.trigger_based = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_TRIGGER_BASED];
-	out->ftm.non_trigger_based = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_NON_TRIGGER_BASED];
-
-	return 0;
-}
-
-static int parse_pmsr_capa(const struct nlattr *pmsr_capa, struct cfg80211_pmsr_capabilities *out,
-			   struct genl_info *info)
-{
-	struct nlattr *tb[NL80211_PMSR_ATTR_MAX + 1];
-	struct nlattr *nla;
-	int size;
-	int ret;
-
-	ret = nla_parse_nested(tb, NL80211_PMSR_ATTR_MAX, pmsr_capa, hwsim_pmsr_capa_policy, NULL);
-	if (ret) {
-		NL_SET_ERR_MSG_ATTR(info->extack, pmsr_capa, "malformed PMSR capability");
-		return -EINVAL;
-	}
-
-	if (tb[NL80211_PMSR_ATTR_MAX_PEERS])
-		out->max_peers = nla_get_u32(tb[NL80211_PMSR_ATTR_MAX_PEERS]);
-	out->report_ap_tsf = !!tb[NL80211_PMSR_ATTR_REPORT_AP_TSF];
-	out->randomize_mac_addr = !!tb[NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR];
-
-	if (!tb[NL80211_PMSR_ATTR_TYPE_CAPA]) {
-		NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_ATTR_TYPE_CAPA],
-				    "malformed PMSR type");
-		return -EINVAL;
-	}
-
-	nla_for_each_nested(nla, tb[NL80211_PMSR_ATTR_TYPE_CAPA], size) {
-		switch (nla_type(nla)) {
-		case NL80211_PMSR_TYPE_FTM:
-			parse_ftm_capa(nla, out, info);
-			break;
-		default:
-			NL_SET_ERR_MSG_ATTR(info->extack, nla, "unsupported measurement type");
-			return -EINVAL;
-		}
-	}
-
-	return 0;
-}
-
-static int hwsim_new_radio_nl(struct sk_buff *msg, struct genl_info *info)
-{
-	struct hwsim_new_radio_params param = { 0 };
-	const char *hwname = NULL;
-	int ret;
-
-	param.reg_strict = info->attrs[HWSIM_ATTR_REG_STRICT_REG];
-	param.p2p_device = info->attrs[HWSIM_ATTR_SUPPORT_P2P_DEVICE];
-	param.nan_device = info->attrs[HWSIM_ATTR_SUPPORT_NAN_DEVICE];
-	param.channels = channels;
-	param.destroy_on_close =
-		info->attrs[HWSIM_ATTR_DESTROY_RADIO_ON_CLOSE];
-
-	if (info->attrs[HWSIM_ATTR_CHANNELS])
-		param.channels = nla_get_u32(info->attrs[HWSIM_ATTR_CHANNELS]);
-
-	if (param.channels < 1) {
-		GENL_SET_ERR_MSG(info, "must have at least one channel");
-		return -EINVAL;
-	}
-
-	if (info->attrs[HWSIM_ATTR_NO_VIF])
-		param.no_vif = true;
-
-	if (info->attrs[HWSIM_ATTR_USE_CHANCTX])
-		param.use_chanctx = true;
-	else
-		param.use_chanctx = (param.channels > 1);
-
-	if (info->attrs[HWSIM_ATTR_MULTI_RADIO])
-		param.multi_radio = true;
-
-	if (info->attrs[HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR])
-		param.background_radar = true;
-
-	if (info->attrs[HWSIM_ATTR_REG_HINT_ALPHA2])
-		param.reg_alpha2 =
-			nla_data(info->attrs[HWSIM_ATTR_REG_HINT_ALPHA2]);
-
-	if (info->attrs[HWSIM_ATTR_REG_CUSTOM_REG]) {
-		u32 idx = nla_get_u32(info->attrs[HWSIM_ATTR_REG_CUSTOM_REG]);
-
-		if (idx >= ARRAY_SIZE(hwsim_world_regdom_custom))
-			return -EINVAL;
-
-		idx = array_index_nospec(idx,
-					 ARRAY_SIZE(hwsim_world_regdom_custom));
-		param.regd = hwsim_world_regdom_custom[idx];
-	}
-
-	if (info->attrs[HWSIM_ATTR_PERM_ADDR]) {
-		if (!is_valid_ether_addr(
-				nla_data(info->attrs[HWSIM_ATTR_PERM_ADDR]))) {
-			GENL_SET_ERR_MSG(info,"MAC is no valid source addr");
-			NL_SET_BAD_ATTR(info->extack,
-					info->attrs[HWSIM_ATTR_PERM_ADDR]);
-			return -EINVAL;
-		}
-
-		param.perm_addr = nla_data(info->attrs[HWSIM_ATTR_PERM_ADDR]);
-	}
-
-	if (info->attrs[HWSIM_ATTR_IFTYPE_SUPPORT]) {
-		param.iftypes =
-			nla_get_u32(info->attrs[HWSIM_ATTR_IFTYPE_SUPPORT]);
-
-		if (param.iftypes & ~HWSIM_IFTYPE_SUPPORT_MASK) {
-			NL_SET_ERR_MSG_ATTR(info->extack,
-					    info->attrs[HWSIM_ATTR_IFTYPE_SUPPORT],
-					    "cannot support more iftypes than kernel");
-			return -EINVAL;
-		}
-	} else {
-		param.iftypes = HWSIM_IFTYPE_SUPPORT_MASK;
-	}
-
-	/* ensure both flag and iftype support is honored */
-	if (param.p2p_device ||
-	    param.iftypes & BIT(NL80211_IFTYPE_P2P_DEVICE)) {
-		param.iftypes |= BIT(NL80211_IFTYPE_P2P_DEVICE);
-		param.p2p_device = true;
-	}
-
-	/* ensure both flag and iftype support is honored */
-	if (param.nan_device ||
-	    param.iftypes & BIT(NL80211_IFTYPE_NAN)) {
-		param.iftypes |= BIT(NL80211_IFTYPE_NAN);
-		param.nan_device = true;
-	}
-
-	if (info->attrs[HWSIM_ATTR_CIPHER_SUPPORT]) {
-		u32 len = nla_len(info->attrs[HWSIM_ATTR_CIPHER_SUPPORT]);
-
-		param.ciphers =
-			nla_data(info->attrs[HWSIM_ATTR_CIPHER_SUPPORT]);
-
-		if (len % sizeof(u32)) {
-			NL_SET_ERR_MSG_ATTR(info->extack,
-					    info->attrs[HWSIM_ATTR_CIPHER_SUPPORT],
-					    "bad cipher list length");
-			return -EINVAL;
-		}
-
-		param.n_ciphers = len / sizeof(u32);
-
-		if (param.n_ciphers > ARRAY_SIZE(hwsim_ciphers)) {
-			NL_SET_ERR_MSG_ATTR(info->extack,
-					    info->attrs[HWSIM_ATTR_CIPHER_SUPPORT],
-					    "too many ciphers specified");
-			return -EINVAL;
-		}
-
-		if (!hwsim_known_ciphers(param.ciphers, param.n_ciphers)) {
-			NL_SET_ERR_MSG_ATTR(info->extack,
-					    info->attrs[HWSIM_ATTR_CIPHER_SUPPORT],
-					    "unsupported ciphers specified");
-			return -EINVAL;
-		}
-	}
-
-	param.mlo = info->attrs[HWSIM_ATTR_MLO_SUPPORT];
-
-	if (param.mlo || param.multi_radio)
-		param.use_chanctx = true;
-
-	if (info->attrs[HWSIM_ATTR_RADIO_NAME]) {
-		hwname = kstrndup((char *)nla_data(info->attrs[HWSIM_ATTR_RADIO_NAME]),
-				  nla_len(info->attrs[HWSIM_ATTR_RADIO_NAME]),
-				  GFP_KERNEL);
-		if (!hwname)
-			return -ENOMEM;
-		param.hwname = hwname;
-	}
-
-	if (info->attrs[HWSIM_ATTR_PMSR_SUPPORT]) {
-		struct cfg80211_pmsr_capabilities *pmsr_capa;
-
-		pmsr_capa = kzalloc_obj(*pmsr_capa);
-		if (!pmsr_capa) {
-			ret = -ENOMEM;
-			goto out_free;
-		}
-		param.pmsr_capa = pmsr_capa;
-
-		ret = parse_pmsr_capa(info->attrs[HWSIM_ATTR_PMSR_SUPPORT], pmsr_capa, info);
-		if (ret)
-			goto out_free;
-	}
-
-	ret = mac80211_hwsim_new_radio(info, &param);
-
-out_free:
-	kfree(hwname);
-	kfree(param.pmsr_capa);
-	return ret;
-}
-
-static int hwsim_del_radio_nl(struct sk_buff *msg, struct genl_info *info)
-{
-	struct mac80211_hwsim_data *data;
-	s64 idx = -1;
-	const char *hwname = NULL;
-
-	if (info->attrs[HWSIM_ATTR_RADIO_ID]) {
-		idx = nla_get_u32(info->attrs[HWSIM_ATTR_RADIO_ID]);
-	} else if (info->attrs[HWSIM_ATTR_RADIO_NAME]) {
-		hwname = kstrndup((char *)nla_data(info->attrs[HWSIM_ATTR_RADIO_NAME]),
-				  nla_len(info->attrs[HWSIM_ATTR_RADIO_NAME]),
-				  GFP_KERNEL);
-		if (!hwname)
-			return -ENOMEM;
-	} else
-		return -EINVAL;
-
-	spin_lock_bh(&hwsim_radio_lock);
-	list_for_each_entry(data, &hwsim_radios, list) {
-		if (idx >= 0) {
-			if (data->idx != idx)
-				continue;
-		} else {
-			if (!hwname ||
-			    strcmp(hwname, wiphy_name(data->hw->wiphy)))
-				continue;
-		}
-
-		if (!net_eq(wiphy_net(data->hw->wiphy), genl_info_net(info)))
-			continue;
-
-		list_del(&data->list);
-		rhashtable_remove_fast(&hwsim_radios_rht, &data->rht,
-				       hwsim_rht_params);
-		hwsim_radios_generation++;
-		spin_unlock_bh(&hwsim_radio_lock);
-		mac80211_hwsim_del_radio(data, wiphy_name(data->hw->wiphy),
-					 info);
-		kfree(hwname);
-		return 0;
-	}
-	spin_unlock_bh(&hwsim_radio_lock);
-
-	kfree(hwname);
-	return -ENODEV;
-}
-
-static int hwsim_get_radio_nl(struct sk_buff *msg, struct genl_info *info)
-{
-	struct mac80211_hwsim_data *data;
-	struct sk_buff *skb;
-	int idx, res = -ENODEV;
-
-	if (!info->attrs[HWSIM_ATTR_RADIO_ID])
-		return -EINVAL;
-	idx = nla_get_u32(info->attrs[HWSIM_ATTR_RADIO_ID]);
-
-	spin_lock_bh(&hwsim_radio_lock);
-	list_for_each_entry(data, &hwsim_radios, list) {
-		if (data->idx != idx)
-			continue;
-
-		if (!net_eq(wiphy_net(data->hw->wiphy), genl_info_net(info)))
-			continue;
-
-		skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
-		if (!skb) {
-			res = -ENOMEM;
-			goto out_err;
-		}
-
-		res = mac80211_hwsim_get_radio(skb, data, info->snd_portid,
-					       info->snd_seq, NULL, 0);
-		if (res < 0) {
-			nlmsg_free(skb);
-			goto out_err;
-		}
-
-		res = genlmsg_reply(skb, info);
-		break;
-	}
-
-out_err:
-	spin_unlock_bh(&hwsim_radio_lock);
-
-	return res;
-}
-
-static int hwsim_dump_radio_nl(struct sk_buff *skb,
-			       struct netlink_callback *cb)
-{
-	int last_idx = cb->args[0] - 1;
-	struct mac80211_hwsim_data *data = NULL;
-	int res = 0;
-	void *hdr;
-
-	spin_lock_bh(&hwsim_radio_lock);
-	cb->seq = hwsim_radios_generation;
-
-	if (last_idx >= hwsim_radio_idx-1)
-		goto done;
-
-	list_for_each_entry(data, &hwsim_radios, list) {
-		if (data->idx <= last_idx)
-			continue;
-
-		if (!net_eq(wiphy_net(data->hw->wiphy), sock_net(skb->sk)))
-			continue;
-
-		res = mac80211_hwsim_get_radio(skb, data,
-					       NETLINK_CB(cb->skb).portid,
-					       cb->nlh->nlmsg_seq, cb,
-					       NLM_F_MULTI);
-		if (res < 0)
-			break;
-
-		last_idx = data->idx;
-	}
-
-	cb->args[0] = last_idx + 1;
-
-	/* list changed, but no new element sent, set interrupted flag */
-	if (skb->len == 0 && cb->prev_seq && cb->seq != cb->prev_seq) {
-		hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
-				  cb->nlh->nlmsg_seq, &hwsim_genl_family,
-				  NLM_F_MULTI, HWSIM_CMD_GET_RADIO);
-		if (hdr) {
-			genl_dump_check_consistent(cb, hdr);
-			genlmsg_end(skb, hdr);
-		} else {
-			res = -EMSGSIZE;
-		}
-	}
-
-done:
-	spin_unlock_bh(&hwsim_radio_lock);
-	return res ?: skb->len;
-}
-
-/* Generic Netlink operations array */
-static const struct genl_small_ops hwsim_ops[] = {
-	{
-		.cmd = HWSIM_CMD_REGISTER,
-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.doit = hwsim_register_received_nl,
-		.flags = GENL_UNS_ADMIN_PERM,
-	},
-	{
-		.cmd = HWSIM_CMD_FRAME,
-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.doit = hwsim_cloned_frame_received_nl,
-	},
-	{
-		.cmd = HWSIM_CMD_TX_INFO_FRAME,
-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.doit = hwsim_tx_info_frame_received_nl,
-	},
-	{
-		.cmd = HWSIM_CMD_NEW_RADIO,
-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.doit = hwsim_new_radio_nl,
-		.flags = GENL_UNS_ADMIN_PERM,
-	},
-	{
-		.cmd = HWSIM_CMD_DEL_RADIO,
-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.doit = hwsim_del_radio_nl,
-		.flags = GENL_UNS_ADMIN_PERM,
-	},
-	{
-		.cmd = HWSIM_CMD_GET_RADIO,
-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.doit = hwsim_get_radio_nl,
-		.dumpit = hwsim_dump_radio_nl,
-	},
-	{
-		.cmd = HWSIM_CMD_REPORT_PMSR,
-		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
-		.doit = hwsim_pmsr_report_nl,
-	},
-};
-
-static struct genl_family hwsim_genl_family __ro_after_init = {
-	.name = "MAC80211_HWSIM",
-	.version = 1,
-	.maxattr = HWSIM_ATTR_MAX,
-	.policy = hwsim_genl_policy,
-	.netnsok = true,
-	.module = THIS_MODULE,
-	.small_ops = hwsim_ops,
-	.n_small_ops = ARRAY_SIZE(hwsim_ops),
-	.resv_start_op = HWSIM_CMD_REPORT_PMSR + 1, // match with __HWSIM_CMD_MAX
-	.mcgrps = hwsim_mcgrps,
-	.n_mcgrps = ARRAY_SIZE(hwsim_mcgrps),
-};
-
-static void remove_user_radios(u32 portid, int netgroup)
-{
-	struct mac80211_hwsim_data *entry, *tmp;
-	LIST_HEAD(list);
-
-	spin_lock_bh(&hwsim_radio_lock);
-	list_for_each_entry_safe(entry, tmp, &hwsim_radios, list) {
-		if (entry->destroy_on_close && entry->portid == portid &&
-		    entry->netgroup == netgroup) {
-			list_move(&entry->list, &list);
-			rhashtable_remove_fast(&hwsim_radios_rht, &entry->rht,
-					       hwsim_rht_params);
-			hwsim_radios_generation++;
-		}
-	}
-	spin_unlock_bh(&hwsim_radio_lock);
-
-	list_for_each_entry_safe(entry, tmp, &list, list) {
-		list_del(&entry->list);
-		mac80211_hwsim_del_radio(entry, wiphy_name(entry->hw->wiphy),
-					 NULL);
-	}
-}
-
-static int mac80211_hwsim_netlink_notify(struct notifier_block *nb,
-					 unsigned long state,
-					 void *_notify)
-{
-	struct netlink_notify *notify = _notify;
-
-	if (state != NETLINK_URELEASE)
-		return NOTIFY_DONE;
-
-	remove_user_radios(notify->portid, hwsim_net_get_netgroup(notify->net));
-
-	if (notify->portid == hwsim_net_get_wmediumd(notify->net)) {
-		printk(KERN_INFO "mac80211_hwsim: wmediumd released netlink"
-		       " socket, switching to perfect channel medium\n");
-		hwsim_register_wmediumd(notify->net, 0);
-	}
-	return NOTIFY_DONE;
-
-}
-
-static struct notifier_block hwsim_netlink_notifier = {
-	.notifier_call = mac80211_hwsim_netlink_notify,
-};
-
-static int __init hwsim_init_netlink(void)
-{
-	int rc;
-
-	printk(KERN_INFO "mac80211_hwsim: initializing netlink\n");
-
-	rc = genl_register_family(&hwsim_genl_family);
-	if (rc)
-		goto failure;
-
-	rc = netlink_register_notifier(&hwsim_netlink_notifier);
-	if (rc) {
-		genl_unregister_family(&hwsim_genl_family);
-		goto failure;
-	}
-
-	return 0;
-
-failure:
-	pr_debug("mac80211_hwsim: error occurred in %s\n", __func__);
-	return -EINVAL;
-}
-
-static __net_init int hwsim_init_net(struct net *net)
-{
-	return hwsim_net_set_netgroup(net);
-}
-
-static void __net_exit hwsim_exit_net(struct net *net)
-{
-	struct mac80211_hwsim_data *data, *tmp;
-	LIST_HEAD(list);
-
-	spin_lock_bh(&hwsim_radio_lock);
-	list_for_each_entry_safe(data, tmp, &hwsim_radios, list) {
-		if (!net_eq(wiphy_net(data->hw->wiphy), net))
-			continue;
-
-		/* Radios created in init_net are returned to init_net. */
-		if (data->netgroup == hwsim_net_get_netgroup(&init_net))
-			continue;
-
-		list_move(&data->list, &list);
-		rhashtable_remove_fast(&hwsim_radios_rht, &data->rht,
-				       hwsim_rht_params);
-		hwsim_radios_generation++;
-	}
-	spin_unlock_bh(&hwsim_radio_lock);
-
-	list_for_each_entry_safe(data, tmp, &list, list) {
-		list_del(&data->list);
-		mac80211_hwsim_del_radio(data,
-					 wiphy_name(data->hw->wiphy),
-					 NULL);
-	}
-
-	ida_free(&hwsim_netgroup_ida, hwsim_net_get_netgroup(net));
-}
-
-static struct pernet_operations hwsim_net_ops = {
-	.init = hwsim_init_net,
-	.exit = hwsim_exit_net,
-	.id   = &hwsim_net_id,
-	.size = sizeof(struct hwsim_net),
-};
-
-static void hwsim_exit_netlink(void)
-{
-	/* unregister the notifier */
-	netlink_unregister_notifier(&hwsim_netlink_notifier);
-	/* unregister the family */
-	genl_unregister_family(&hwsim_genl_family);
-}
-
-#if IS_REACHABLE(CONFIG_VIRTIO)
-static void hwsim_virtio_tx_done(struct virtqueue *vq)
-{
-	unsigned int len;
-	struct sk_buff *skb;
-	unsigned long flags;
-
-	spin_lock_irqsave(&hwsim_virtio_lock, flags);
-	while ((skb = virtqueue_get_buf(vq, &len)))
-		dev_kfree_skb_irq(skb);
-	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
-}
-
-static int hwsim_virtio_handle_cmd(struct sk_buff *skb)
-{
-	struct nlmsghdr *nlh;
-	struct genlmsghdr *gnlh;
-	struct nlattr *tb[HWSIM_ATTR_MAX + 1];
-	struct genl_info info = {};
-	int err;
-
-	nlh = nlmsg_hdr(skb);
-	gnlh = nlmsg_data(nlh);
-
-	if (skb->len < nlh->nlmsg_len)
-		return -EINVAL;
-
-	err = genlmsg_parse(nlh, &hwsim_genl_family, tb, HWSIM_ATTR_MAX,
-			    hwsim_genl_policy, NULL);
-	if (err) {
-		pr_err_ratelimited("hwsim: genlmsg_parse returned %d\n", err);
-		return err;
-	}
-
-	info.attrs = tb;
-
-	switch (gnlh->cmd) {
-	case HWSIM_CMD_FRAME:
-		hwsim_cloned_frame_received_nl(skb, &info);
-		break;
-	case HWSIM_CMD_TX_INFO_FRAME:
-		hwsim_tx_info_frame_received_nl(skb, &info);
-		break;
-	case HWSIM_CMD_REPORT_PMSR:
-		hwsim_pmsr_report_nl(skb, &info);
-		break;
-	default:
-		pr_err_ratelimited("hwsim: invalid cmd: %d\n", gnlh->cmd);
-		return -EPROTO;
-	}
-	return 0;
-}
-
-static void hwsim_virtio_rx_work(struct work_struct *work)
-{
-	struct virtqueue *vq;
-	unsigned int len;
-	struct sk_buff *skb;
-	struct scatterlist sg[1];
-	int err;
-	unsigned long flags;
-
-	spin_lock_irqsave(&hwsim_virtio_lock, flags);
-	if (!hwsim_virtio_enabled)
-		goto out_unlock;
-
-	skb = virtqueue_get_buf(hwsim_vqs[HWSIM_VQ_RX], &len);
-	if (!skb)
-		goto out_unlock;
-	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
-
-	skb->data = skb->head;
-	skb_reset_tail_pointer(skb);
-	skb_put(skb, len);
-	hwsim_virtio_handle_cmd(skb);
-
-	spin_lock_irqsave(&hwsim_virtio_lock, flags);
-	if (!hwsim_virtio_enabled) {
-		dev_kfree_skb_irq(skb);
-		goto out_unlock;
-	}
-	vq = hwsim_vqs[HWSIM_VQ_RX];
-	sg_init_one(sg, skb->head, skb_end_offset(skb));
-	err = virtqueue_add_inbuf(vq, sg, 1, skb, GFP_ATOMIC);
-	if (WARN(err, "virtqueue_add_inbuf returned %d\n", err))
-		dev_kfree_skb_irq(skb);
-	else
-		virtqueue_kick(vq);
-	schedule_work(&hwsim_virtio_rx);
-
-out_unlock:
-	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
-}
-
-static void hwsim_virtio_rx_done(struct virtqueue *vq)
-{
-	schedule_work(&hwsim_virtio_rx);
-}
-
-static int init_vqs(struct virtio_device *vdev)
-{
-	struct virtqueue_info vqs_info[HWSIM_NUM_VQS] = {
-		[HWSIM_VQ_TX] = { "tx", hwsim_virtio_tx_done },
-		[HWSIM_VQ_RX] = { "rx", hwsim_virtio_rx_done },
-	};
-
-	return virtio_find_vqs(vdev, HWSIM_NUM_VQS,
-			       hwsim_vqs, vqs_info, NULL);
-}
-
-static int fill_vq(struct virtqueue *vq)
-{
-	int i, err;
-	struct sk_buff *skb;
-	struct scatterlist sg[1];
-
-	for (i = 0; i < virtqueue_get_vring_size(vq); i++) {
-		skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
-		if (!skb)
-			return -ENOMEM;
-
-		sg_init_one(sg, skb->head, skb_end_offset(skb));
-		err = virtqueue_add_inbuf(vq, sg, 1, skb, GFP_KERNEL);
-		if (err) {
-			nlmsg_free(skb);
-			return err;
-		}
-	}
-	virtqueue_kick(vq);
-	return 0;
-}
-
-static void remove_vqs(struct virtio_device *vdev)
-{
-	int i;
-
-	virtio_reset_device(vdev);
-
-	for (i = 0; i < ARRAY_SIZE(hwsim_vqs); i++) {
-		struct virtqueue *vq = hwsim_vqs[i];
-		struct sk_buff *skb;
-
-		while ((skb = virtqueue_detach_unused_buf(vq)))
-			nlmsg_free(skb);
-	}
-
-	vdev->config->del_vqs(vdev);
-}
-
-static int hwsim_virtio_probe(struct virtio_device *vdev)
-{
-	int err;
-	unsigned long flags;
-
-	spin_lock_irqsave(&hwsim_virtio_lock, flags);
-	if (hwsim_virtio_enabled) {
-		spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
-		return -EEXIST;
-	}
-	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
-
-	err = init_vqs(vdev);
-	if (err)
-		return err;
-
-	virtio_device_ready(vdev);
-
-	err = fill_vq(hwsim_vqs[HWSIM_VQ_RX]);
-	if (err)
-		goto out_remove;
-
-	spin_lock_irqsave(&hwsim_virtio_lock, flags);
-	hwsim_virtio_enabled = true;
-	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
-
-	schedule_work(&hwsim_virtio_rx);
-	return 0;
-
-out_remove:
-	remove_vqs(vdev);
-	return err;
-}
-
-static void hwsim_virtio_remove(struct virtio_device *vdev)
-{
-	hwsim_virtio_enabled = false;
-
-	cancel_work_sync(&hwsim_virtio_rx);
-
-	remove_vqs(vdev);
-}
-
-/* MAC80211_HWSIM virtio device id table */
-static const struct virtio_device_id id_table[] = {
-	{ VIRTIO_ID_MAC80211_HWSIM, VIRTIO_DEV_ANY_ID },
-	{ 0 }
-};
-MODULE_DEVICE_TABLE(virtio, id_table);
-
-static struct virtio_driver virtio_hwsim = {
-	.driver.name = KBUILD_MODNAME,
-	.id_table = id_table,
-	.probe = hwsim_virtio_probe,
-	.remove = hwsim_virtio_remove,
-};
-
-static int hwsim_register_virtio_driver(void)
-{
-	return register_virtio_driver(&virtio_hwsim);
-}
-
-static void hwsim_unregister_virtio_driver(void)
-{
-	unregister_virtio_driver(&virtio_hwsim);
-}
-#else
-static inline int hwsim_register_virtio_driver(void)
-{
-	return 0;
-}
-
-static inline void hwsim_unregister_virtio_driver(void)
-{
-}
-#endif
-
-static int __init init_mac80211_hwsim(void)
-{
-	int i, err;
-
-	if (radios < 0 || radios > 100)
-		return -EINVAL;
-
-	if (channels < 1)
-		return -EINVAL;
-
-	err = rhashtable_init(&hwsim_radios_rht, &hwsim_rht_params);
-	if (err)
-		return err;
-
-	err = register_pernet_device(&hwsim_net_ops);
-	if (err)
-		goto out_free_rht;
-
-	err = platform_driver_register(&mac80211_hwsim_driver);
-	if (err)
-		goto out_unregister_pernet;
-
-	err = hwsim_init_netlink();
-	if (err)
-		goto out_unregister_driver;
-
-	err = hwsim_register_virtio_driver();
-	if (err)
-		goto out_exit_netlink;
-
-	err = class_register(&hwsim_class);
-	if (err)
-		goto out_exit_virtio;
-
-	hwsim_init_s1g_channels(hwsim_channels_s1g);
-
-	for (i = 0; i < radios; i++) {
-		struct hwsim_new_radio_params param = { 0 };
-
-		param.channels = channels;
-
-		switch (regtest) {
-		case HWSIM_REGTEST_DIFF_COUNTRY:
-			if (i < ARRAY_SIZE(hwsim_alpha2s))
-				param.reg_alpha2 = hwsim_alpha2s[i];
-			break;
-		case HWSIM_REGTEST_DRIVER_REG_FOLLOW:
-			if (!i)
-				param.reg_alpha2 = hwsim_alpha2s[0];
-			break;
-		case HWSIM_REGTEST_STRICT_ALL:
-			param.reg_strict = true;
-			fallthrough;
-		case HWSIM_REGTEST_DRIVER_REG_ALL:
-			param.reg_alpha2 = hwsim_alpha2s[0];
-			break;
-		case HWSIM_REGTEST_WORLD_ROAM:
-			if (i == 0)
-				param.regd = &hwsim_world_regdom_custom_01;
-			break;
-		case HWSIM_REGTEST_CUSTOM_WORLD:
-			param.regd = &hwsim_world_regdom_custom_03;
-			break;
-		case HWSIM_REGTEST_CUSTOM_WORLD_2:
-			if (i == 0)
-				param.regd = &hwsim_world_regdom_custom_03;
-			else if (i == 1)
-				param.regd = &hwsim_world_regdom_custom_02;
-			break;
-		case HWSIM_REGTEST_STRICT_FOLLOW:
-			if (i == 0) {
-				param.reg_strict = true;
-				param.reg_alpha2 = hwsim_alpha2s[0];
-			}
-			break;
-		case HWSIM_REGTEST_STRICT_AND_DRIVER_REG:
-			if (i == 0) {
-				param.reg_strict = true;
-				param.reg_alpha2 = hwsim_alpha2s[0];
-			} else if (i == 1) {
-				param.reg_alpha2 = hwsim_alpha2s[1];
-			}
-			break;
-		case HWSIM_REGTEST_ALL:
-			switch (i) {
-			case 0:
-				param.regd = &hwsim_world_regdom_custom_01;
-				break;
-			case 1:
-				param.regd = &hwsim_world_regdom_custom_02;
-				break;
-			case 2:
-				param.reg_alpha2 = hwsim_alpha2s[0];
-				break;
-			case 3:
-				param.reg_alpha2 = hwsim_alpha2s[1];
-				break;
-			case 4:
-				param.reg_strict = true;
-				param.reg_alpha2 = hwsim_alpha2s[2];
-				break;
-			}
-			break;
-		default:
-			break;
-		}
-
-		param.p2p_device = support_p2p_device;
-		param.mlo = mlo;
-		param.multi_radio = multi_radio;
-		param.background_radar = true;
-		param.use_chanctx = channels > 1 || mlo || multi_radio;
-		param.iftypes = HWSIM_IFTYPE_SUPPORT_MASK;
-		if (param.p2p_device)
-			param.iftypes |= BIT(NL80211_IFTYPE_P2P_DEVICE);
-
-		err = mac80211_hwsim_new_radio(NULL, &param);
-		if (err < 0)
-			goto out_free_radios;
-	}
-
-	hwsim_mon = alloc_netdev(0, "hwsim%d", NET_NAME_UNKNOWN,
-				 hwsim_mon_setup);
-	if (hwsim_mon == NULL) {
-		err = -ENOMEM;
-		goto out_free_radios;
-	}
-
-	rtnl_lock();
-	err = dev_alloc_name(hwsim_mon, hwsim_mon->name);
-	if (err < 0) {
-		rtnl_unlock();
-		goto out_free_mon;
-	}
-
-	err = register_netdevice(hwsim_mon);
-	if (err < 0) {
-		rtnl_unlock();
-		goto out_free_mon;
-	}
-	rtnl_unlock();
-
-	return 0;
-
-out_free_mon:
-	free_netdev(hwsim_mon);
-out_free_radios:
-	mac80211_hwsim_free();
-out_exit_virtio:
-	hwsim_unregister_virtio_driver();
-out_exit_netlink:
-	hwsim_exit_netlink();
-out_unregister_driver:
-	platform_driver_unregister(&mac80211_hwsim_driver);
-out_unregister_pernet:
-	unregister_pernet_device(&hwsim_net_ops);
-out_free_rht:
-	rhashtable_destroy(&hwsim_radios_rht);
-	return err;
-}
-module_init(init_mac80211_hwsim);
-
-static void __exit exit_mac80211_hwsim(void)
-{
-	pr_debug("mac80211_hwsim: unregister radios\n");
-
-	hwsim_unregister_virtio_driver();
-	hwsim_exit_netlink();
-
-	mac80211_hwsim_free();
-
-	rhashtable_destroy(&hwsim_radios_rht);
-	unregister_netdev(hwsim_mon);
-	platform_driver_unregister(&mac80211_hwsim_driver);
-	unregister_pernet_device(&hwsim_net_ops);
-}
-module_exit(exit_mac80211_hwsim);
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_i.h b/drivers/net/wireless/virtual/mac80211_hwsim_i.h
new file mode 100644
index 0000000000000..0f0f2ac6d80eb
--- /dev/null
+++ b/drivers/net/wireless/virtual/mac80211_hwsim_i.h
@@ -0,0 +1,168 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * mac80211_hwsim - software simulator of 802.11 radio(s) for mac80211
+ * Copyright (c) 2008, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2011, Javier Lopez <jlopex@gmail.com>
+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright (C) 2018 - 2026 Intel Corporation
+ */
+
+#ifndef __MAC80211_HWSIM_I_H
+#define __MAC80211_HWSIM_I_H
+
+#include <net/mac80211.h>
+#include "mac80211_hwsim.h"
+#include "mac80211_hwsim_nan.h"
+
+struct hwsim_sta_nan_sched {
+	/* Later members are protected by this lock */
+	spinlock_t lock;
+	u16 committed_dw;
+	struct {
+		u8 map_id;
+		struct cfg80211_chan_def chans[CFG80211_NAN_SCHED_NUM_TIME_SLOTS];
+	} maps[CFG80211_NAN_MAX_PEER_MAPS];
+};
+
+struct hwsim_sta_priv {
+	u32 magic;
+	unsigned int last_link;
+	u16 active_links_rx;
+
+	/* NAN peer schedule - must be accessed under nan_sched.lock */
+	struct hwsim_sta_nan_sched nan_sched;
+};
+
+#define HWSIM_STA_MAGIC	0x6d537749
+
+struct mac80211_hwsim_link_data {
+	u32 link_id;
+	u64 beacon_int	/* beacon interval in us */;
+	struct hrtimer beacon_timer;
+};
+
+#define HWSIM_NUM_CHANNELS_2GHZ		14
+#define HWSIM_NUM_CHANNELS_5GHZ		40
+#define HWSIM_NUM_CHANNELS_6GHZ		59
+#define HWSIM_NUM_S1G_CHANNELS_US	51
+#define HWSIM_NUM_RATES			12
+#define HWSIM_NUM_CIPHERS		11
+
+struct mac80211_hwsim_data {
+	struct list_head list;
+	struct rhash_head rht;
+	struct ieee80211_hw *hw;
+	struct device *dev;
+	struct ieee80211_supported_band bands[NUM_NL80211_BANDS];
+	struct ieee80211_channel channels_2ghz[HWSIM_NUM_CHANNELS_2GHZ];
+	struct ieee80211_channel channels_5ghz[HWSIM_NUM_CHANNELS_5GHZ];
+	struct ieee80211_channel channels_6ghz[HWSIM_NUM_CHANNELS_6GHZ];
+	struct ieee80211_channel channels_s1g[HWSIM_NUM_S1G_CHANNELS_US];
+	struct ieee80211_rate rates[HWSIM_NUM_RATES];
+	struct ieee80211_iface_combination if_combination;
+	struct ieee80211_iface_limit if_limits[5];
+	int n_if_limits;
+	/* Storage space for channels, etc. */
+	struct mac80211_hwsim_phy_data *phy_data;
+
+	struct ieee80211_iface_combination if_combination_radio;
+	struct wiphy_radio_freq_range radio_range[NUM_NL80211_BANDS];
+	struct wiphy_radio radio[NUM_NL80211_BANDS];
+
+	u32 ciphers[HWSIM_NUM_CIPHERS];
+
+	struct mac_address addresses[3];
+	int channels, idx;
+	bool use_chanctx;
+	bool destroy_on_close;
+	u32 portid;
+	char alpha2[2];
+	const struct ieee80211_regdomain *regd;
+
+	struct ieee80211_channel *tmp_chan;
+	struct ieee80211_channel *roc_chan;
+	u32 roc_duration;
+	struct delayed_work roc_start;
+	struct delayed_work roc_done;
+	struct delayed_work hw_scan;
+	struct cfg80211_scan_request *hw_scan_request;
+	struct ieee80211_vif *hw_scan_vif;
+	int scan_chan_idx;
+	u8 scan_addr[ETH_ALEN];
+	struct {
+		struct ieee80211_channel *channel;
+		unsigned long next_start, start, end;
+	} survey_data[HWSIM_NUM_CHANNELS_2GHZ +
+		      HWSIM_NUM_CHANNELS_5GHZ +
+		      HWSIM_NUM_CHANNELS_6GHZ];
+
+	struct ieee80211_channel *channel;
+	enum nl80211_chan_width bw;
+	unsigned int rx_filter;
+	bool started, idle, scanning;
+	struct mutex mutex;
+	enum ps_mode {
+		PS_DISABLED, PS_ENABLED, PS_AUTO_POLL, PS_MANUAL_POLL
+	} ps;
+	bool ps_poll_pending;
+	struct dentry *debugfs;
+	struct cfg80211_chan_def radar_background_chandef;
+
+	atomic_t pending_cookie;
+	struct sk_buff_head pending;	/* packets pending */
+	/*
+	 * Only radios in the same group can communicate together (the
+	 * channel has to match too). Each bit represents a group. A
+	 * radio can be in more than one group.
+	 */
+	u64 group;
+
+	/* group shared by radios created in the same netns */
+	int netgroup;
+	/* wmediumd portid responsible for netgroup of this radio */
+	u32 wmediumd;
+
+	/* difference between this hw's clock and the real clock, in usecs */
+	spinlock_t tsf_offset_lock;
+	s64 tsf_offset;
+
+	/* Stats */
+	u64 tx_pkts;
+	u64 rx_pkts;
+	u64 tx_bytes;
+	u64 rx_bytes;
+	u64 tx_dropped;
+	u64 tx_failed;
+
+	/* RSSI in rx status of the receiver */
+	int rx_rssi;
+
+	/* only used when pmsr capability is supplied */
+	struct cfg80211_pmsr_capabilities pmsr_capa;
+	struct cfg80211_pmsr_request *pmsr_request;
+	struct wireless_dev *pmsr_request_wdev;
+
+	struct mac80211_hwsim_link_data link_data[IEEE80211_MLD_MAX_NUM_LINKS];
+
+	struct mac80211_hwsim_nan_data nan;
+};
+
+extern spinlock_t hwsim_radio_lock;
+extern struct list_head hwsim_radios;
+
+ktime_t mac80211_hwsim_tsf_to_boottime(struct mac80211_hwsim_data *data,
+				       u64 tsf);
+u64 mac80211_hwsim_boottime_to_tsf(struct mac80211_hwsim_data *data,
+				   ktime_t ts);
+
+u64 mac80211_hwsim_get_tsf(struct ieee80211_hw *hw,
+			   struct ieee80211_vif *vif);
+
+void mac80211_hwsim_tx_frame(struct ieee80211_hw *hw,
+			     struct sk_buff *skb,
+			     struct ieee80211_channel *chan);
+
+void ieee80211_hwsim_wake_tx_queue(struct ieee80211_hw *hw,
+				   struct ieee80211_txq *txq);
+
+#endif /* __MAC80211_HWSIM_I_H */
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_main.c b/drivers/net/wireless/virtual/mac80211_hwsim_main.c
new file mode 100644
index 0000000000000..67a33e5e7d541
--- /dev/null
+++ b/drivers/net/wireless/virtual/mac80211_hwsim_main.c
@@ -0,0 +1,7585 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * mac80211_hwsim - software simulator of 802.11 radio(s) for mac80211
+ * Copyright (c) 2008, Jouni Malinen <j@w1.fi>
+ * Copyright (c) 2011, Javier Lopez <jlopex@gmail.com>
+ * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
+ * Copyright (C) 2018 - 2026 Intel Corporation
+ */
+
+/*
+ * TODO:
+ * - Add TSF sync and fix IBSS beacon transmission by adding
+ *   competition for "air time" at TBTT
+ * - RX filtering based on filter configuration (data->rx_filter)
+ */
+
+#include <linux/list.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <net/dst.h>
+#include <net/xfrm.h>
+#include <net/mac80211.h>
+#include <net/ieee80211_radiotap.h>
+#include <linux/if_arp.h>
+#include <linux/rtnetlink.h>
+#include <linux/etherdevice.h>
+#include <linux/platform_device.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/ktime.h>
+#include <net/genetlink.h>
+#include <net/net_namespace.h>
+#include <net/netns/generic.h>
+#include <linux/rhashtable.h>
+#include <linux/nospec.h>
+#include <linux/virtio.h>
+#include <linux/virtio_ids.h>
+#include <linux/virtio_config.h>
+#include <linux/uaccess.h>
+#include <linux/string.h>
+#include "mac80211_hwsim.h"
+#include "mac80211_hwsim_i.h"
+
+#define WARN_QUEUE 100
+#define MAX_QUEUE 200
+
+MODULE_AUTHOR("Jouni Malinen");
+MODULE_DESCRIPTION("Software simulator of 802.11 radio(s) for mac80211");
+MODULE_LICENSE("GPL");
+
+static int radios = 2;
+module_param(radios, int, 0444);
+MODULE_PARM_DESC(radios, "Number of simulated radios");
+
+static int channels = 1;
+module_param(channels, int, 0444);
+MODULE_PARM_DESC(channels, "Number of concurrent channels");
+
+static bool paged_rx = false;
+module_param(paged_rx, bool, 0644);
+MODULE_PARM_DESC(paged_rx, "Use paged SKBs for RX instead of linear ones");
+
+static bool rctbl = false;
+module_param(rctbl, bool, 0444);
+MODULE_PARM_DESC(rctbl, "Handle rate control table");
+
+static bool support_p2p_device = true;
+module_param(support_p2p_device, bool, 0444);
+MODULE_PARM_DESC(support_p2p_device, "Support P2P-Device interface type");
+
+static bool mlo;
+module_param(mlo, bool, 0444);
+MODULE_PARM_DESC(mlo, "Support MLO");
+
+static bool multi_radio;
+module_param(multi_radio, bool, 0444);
+MODULE_PARM_DESC(multi_radio, "Support Multiple Radios per wiphy");
+
+/**
+ * enum hwsim_regtest - the type of regulatory tests we offer
+ *
+ * @HWSIM_REGTEST_DISABLED: No regulatory tests are performed,
+ * 	this is the default value.
+ * @HWSIM_REGTEST_DRIVER_REG_FOLLOW: Used for testing the driver regulatory
+ *	hint, only one driver regulatory hint will be sent as such the
+ * 	secondary radios are expected to follow.
+ * @HWSIM_REGTEST_DRIVER_REG_ALL: Used for testing the driver regulatory
+ * 	request with all radios reporting the same regulatory domain.
+ * @HWSIM_REGTEST_DIFF_COUNTRY: Used for testing the drivers calling
+ * 	different regulatory domains requests. Expected behaviour is for
+ * 	an intersection to occur but each device will still use their
+ * 	respective regulatory requested domains. Subsequent radios will
+ * 	use the resulting intersection.
+ * @HWSIM_REGTEST_WORLD_ROAM: Used for testing the world roaming. We accomplish
+ *	this by using a custom beacon-capable regulatory domain for the first
+ *	radio. All other device world roam.
+ * @HWSIM_REGTEST_CUSTOM_WORLD: Used for testing the custom world regulatory
+ * 	domain requests. All radios will adhere to this custom world regulatory
+ * 	domain.
+ * @HWSIM_REGTEST_CUSTOM_WORLD_2: Used for testing 2 custom world regulatory
+ * 	domain requests. The first radio will adhere to the first custom world
+ * 	regulatory domain, the second one to the second custom world regulatory
+ * 	domain. All other devices will world roam.
+ * @HWSIM_REGTEST_STRICT_FOLLOW: Used for testing strict regulatory domain
+ *	settings, only the first radio will send a regulatory domain request
+ *	and use strict settings. The rest of the radios are expected to follow.
+ * @HWSIM_REGTEST_STRICT_ALL: Used for testing strict regulatory domain
+ *	settings. All radios will adhere to this.
+ * @HWSIM_REGTEST_STRICT_AND_DRIVER_REG: Used for testing strict regulatory
+ *	domain settings, combined with secondary driver regulatory domain
+ *	settings. The first radio will get a strict regulatory domain setting
+ *	using the first driver regulatory request and the second radio will use
+ *	non-strict settings using the second driver regulatory request. All
+ *	other devices should follow the intersection created between the
+ *	first two.
+ * @HWSIM_REGTEST_ALL: Used for testing every possible mix. You will need
+ * 	at least 6 radios for a complete test. We will test in this order:
+ * 	1 - driver custom world regulatory domain
+ * 	2 - second custom world regulatory domain
+ * 	3 - first driver regulatory domain request
+ * 	4 - second driver regulatory domain request
+ * 	5 - strict regulatory domain settings using the third driver regulatory
+ * 	    domain request
+ * 	6 and on - should follow the intersection of the 3rd, 4rth and 5th radio
+ * 	           regulatory requests.
+ *
+ * These are the different values you can use for the regtest
+ * module parameter. This is useful to help test world roaming
+ * and the driver regulatory_hint() call and combinations of these.
+ * If you want to do specific alpha2 regulatory domain tests simply
+ * use the userspace regulatory request as that will be respected as
+ * well without the need of this module parameter. This is designed
+ * only for testing the driver regulatory request, world roaming
+ * and all possible combinations.
+ */
+enum hwsim_regtest {
+	HWSIM_REGTEST_DISABLED = 0,
+	HWSIM_REGTEST_DRIVER_REG_FOLLOW = 1,
+	HWSIM_REGTEST_DRIVER_REG_ALL = 2,
+	HWSIM_REGTEST_DIFF_COUNTRY = 3,
+	HWSIM_REGTEST_WORLD_ROAM = 4,
+	HWSIM_REGTEST_CUSTOM_WORLD = 5,
+	HWSIM_REGTEST_CUSTOM_WORLD_2 = 6,
+	HWSIM_REGTEST_STRICT_FOLLOW = 7,
+	HWSIM_REGTEST_STRICT_ALL = 8,
+	HWSIM_REGTEST_STRICT_AND_DRIVER_REG = 9,
+	HWSIM_REGTEST_ALL = 10,
+};
+
+/* Set to one of the HWSIM_REGTEST_* values above */
+static int regtest = HWSIM_REGTEST_DISABLED;
+module_param(regtest, int, 0444);
+MODULE_PARM_DESC(regtest, "The type of regulatory test we want to run");
+
+static const char *hwsim_alpha2s[] = {
+	"FI",
+	"AL",
+	"US",
+	"DE",
+	"JP",
+	"AL",
+};
+
+static const struct ieee80211_regdomain hwsim_world_regdom_custom_01 = {
+	.n_reg_rules = 5,
+	.alpha2 =  "99",
+	.reg_rules = {
+		REG_RULE(2412-10, 2462+10, 40, 0, 20, 0),
+		REG_RULE(2484-10, 2484+10, 40, 0, 20, 0),
+		REG_RULE(5150-10, 5240+10, 40, 0, 30, 0),
+		REG_RULE(5745-10, 5825+10, 40, 0, 30, 0),
+		REG_RULE(5855-10, 5925+10, 40, 0, 33, 0),
+	}
+};
+
+static const struct ieee80211_regdomain hwsim_world_regdom_custom_02 = {
+	.n_reg_rules = 3,
+	.alpha2 =  "99",
+	.reg_rules = {
+		REG_RULE(2412-10, 2462+10, 40, 0, 20, 0),
+		REG_RULE(5725-10, 5850+10, 40, 0, 30,
+			 NL80211_RRF_NO_IR),
+		REG_RULE(5855-10, 5925+10, 40, 0, 33, 0),
+	}
+};
+
+static const struct ieee80211_regdomain hwsim_world_regdom_custom_03 = {
+	.n_reg_rules = 6,
+	.alpha2 =  "99",
+	.reg_rules = {
+		REG_RULE(2412 - 10, 2462 + 10, 40, 0, 20, 0),
+		REG_RULE(2484 - 10, 2484 + 10, 40, 0, 20, 0),
+		REG_RULE(5150 - 10, 5240 + 10, 40, 0, 30, 0),
+		REG_RULE(5745 - 10, 5825 + 10, 40, 0, 30, 0),
+		REG_RULE(5855 - 10, 5925 + 10, 40, 0, 33, 0),
+		REG_RULE(5955 - 10, 7125 + 10, 320, 0, 33, 0),
+	}
+};
+
+static const struct ieee80211_regdomain hwsim_world_regdom_custom_04 = {
+	.n_reg_rules = 6,
+	.alpha2 =  "99",
+	.reg_rules = {
+		REG_RULE(2412 - 10, 2462 + 10, 40, 0, 20, 0),
+		REG_RULE(2484 - 10, 2484 + 10, 40, 0, 20, 0),
+		REG_RULE(5150 - 10, 5240 + 10, 80, 0, 30, NL80211_RRF_AUTO_BW),
+		REG_RULE(5260 - 10, 5320 + 10, 80, 0, 30,
+			 NL80211_RRF_DFS_CONCURRENT | NL80211_RRF_DFS |
+			 NL80211_RRF_AUTO_BW),
+		REG_RULE(5500 - 10, 5720 + 10, 160, 0, 30,
+			 NL80211_RRF_DFS_CONCURRENT | NL80211_RRF_DFS),
+		REG_RULE(5745 - 10, 5825 + 10, 80, 0, 30, 0),
+		REG_RULE(5855 - 10, 5925 + 10, 80, 0, 33, 0),
+	}
+};
+
+static const struct ieee80211_regdomain *hwsim_world_regdom_custom[] = {
+	&hwsim_world_regdom_custom_01,
+	&hwsim_world_regdom_custom_02,
+	&hwsim_world_regdom_custom_03,
+	&hwsim_world_regdom_custom_04,
+};
+
+struct hwsim_vif_priv {
+	u32 magic;
+	u32 skip_beacons[IEEE80211_MLD_MAX_NUM_LINKS];
+	u8 bssid[ETH_ALEN];
+	bool assoc;
+	bool bcn_en;
+	u16 aid;
+};
+
+#define HWSIM_VIF_MAGIC	0x69537748
+
+static inline void hwsim_check_magic(struct ieee80211_vif *vif)
+{
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+	WARN(vp->magic != HWSIM_VIF_MAGIC,
+	     "Invalid VIF (%p) magic %#x, %pM, %d/%d\n",
+	     vif, vp->magic, vif->addr, vif->type, vif->p2p);
+}
+
+static inline void hwsim_set_magic(struct ieee80211_vif *vif)
+{
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+	vp->magic = HWSIM_VIF_MAGIC;
+}
+
+static inline void hwsim_clear_magic(struct ieee80211_vif *vif)
+{
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+	vp->magic = 0;
+}
+
+static inline void hwsim_check_sta_magic(struct ieee80211_sta *sta)
+{
+	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+	WARN_ON(sp->magic != HWSIM_STA_MAGIC);
+}
+
+static inline void hwsim_set_sta_magic(struct ieee80211_sta *sta)
+{
+	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+	sp->magic = HWSIM_STA_MAGIC;
+}
+
+static inline void hwsim_clear_sta_magic(struct ieee80211_sta *sta)
+{
+	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+	sp->magic = 0;
+}
+
+struct hwsim_chanctx_priv {
+	u32 magic;
+};
+
+#define HWSIM_CHANCTX_MAGIC 0x6d53774a
+
+static inline void hwsim_check_chanctx_magic(struct ieee80211_chanctx_conf *c)
+{
+	struct hwsim_chanctx_priv *cp = (void *)c->drv_priv;
+	WARN_ON(cp->magic != HWSIM_CHANCTX_MAGIC);
+}
+
+static inline void hwsim_set_chanctx_magic(struct ieee80211_chanctx_conf *c)
+{
+	struct hwsim_chanctx_priv *cp = (void *)c->drv_priv;
+	cp->magic = HWSIM_CHANCTX_MAGIC;
+}
+
+static inline void hwsim_clear_chanctx_magic(struct ieee80211_chanctx_conf *c)
+{
+	struct hwsim_chanctx_priv *cp = (void *)c->drv_priv;
+	cp->magic = 0;
+}
+
+static unsigned int hwsim_net_id;
+
+static DEFINE_IDA(hwsim_netgroup_ida);
+
+struct hwsim_net {
+	int netgroup;
+	u32 wmediumd;
+};
+
+static inline int hwsim_net_get_netgroup(struct net *net)
+{
+	struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
+
+	return hwsim_net->netgroup;
+}
+
+static inline int hwsim_net_set_netgroup(struct net *net)
+{
+	struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
+
+	hwsim_net->netgroup = ida_alloc(&hwsim_netgroup_ida, GFP_KERNEL);
+	return hwsim_net->netgroup >= 0 ? 0 : -ENOMEM;
+}
+
+static inline u32 hwsim_net_get_wmediumd(struct net *net)
+{
+	struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
+
+	return hwsim_net->wmediumd;
+}
+
+static inline void hwsim_net_set_wmediumd(struct net *net, u32 portid)
+{
+	struct hwsim_net *hwsim_net = net_generic(net, hwsim_net_id);
+
+	hwsim_net->wmediumd = portid;
+}
+
+static const struct class hwsim_class = {
+	.name	= "mac80211_hwsim"
+};
+
+static struct net_device *hwsim_mon; /* global monitor netdev */
+
+#define CHAN2G(_freq)  { \
+	.band = NL80211_BAND_2GHZ, \
+	.center_freq = (_freq), \
+	.hw_value = (_freq), \
+}
+
+#define CHAN5G(_freq) { \
+	.band = NL80211_BAND_5GHZ, \
+	.center_freq = (_freq), \
+	.hw_value = (_freq), \
+}
+
+#define CHAN6G(_freq) { \
+	.band = NL80211_BAND_6GHZ, \
+	.center_freq = (_freq), \
+	.hw_value = (_freq), \
+}
+
+static const struct ieee80211_channel hwsim_channels_2ghz[] = {
+	CHAN2G(2412), /* Channel 1 */
+	CHAN2G(2417), /* Channel 2 */
+	CHAN2G(2422), /* Channel 3 */
+	CHAN2G(2427), /* Channel 4 */
+	CHAN2G(2432), /* Channel 5 */
+	CHAN2G(2437), /* Channel 6 */
+	CHAN2G(2442), /* Channel 7 */
+	CHAN2G(2447), /* Channel 8 */
+	CHAN2G(2452), /* Channel 9 */
+	CHAN2G(2457), /* Channel 10 */
+	CHAN2G(2462), /* Channel 11 */
+	CHAN2G(2467), /* Channel 12 */
+	CHAN2G(2472), /* Channel 13 */
+	CHAN2G(2484), /* Channel 14 */
+};
+static_assert(HWSIM_NUM_CHANNELS_2GHZ == ARRAY_SIZE(hwsim_channels_2ghz),
+	      "Inconsistent 2 GHz channel count");
+
+static const struct ieee80211_channel hwsim_channels_5ghz[] = {
+	CHAN5G(5180), /* Channel 36 */
+	CHAN5G(5200), /* Channel 40 */
+	CHAN5G(5220), /* Channel 44 */
+	CHAN5G(5240), /* Channel 48 */
+
+	CHAN5G(5260), /* Channel 52 */
+	CHAN5G(5280), /* Channel 56 */
+	CHAN5G(5300), /* Channel 60 */
+	CHAN5G(5320), /* Channel 64 */
+
+	CHAN5G(5500), /* Channel 100 */
+	CHAN5G(5520), /* Channel 104 */
+	CHAN5G(5540), /* Channel 108 */
+	CHAN5G(5560), /* Channel 112 */
+	CHAN5G(5580), /* Channel 116 */
+	CHAN5G(5600), /* Channel 120 */
+	CHAN5G(5620), /* Channel 124 */
+	CHAN5G(5640), /* Channel 128 */
+	CHAN5G(5660), /* Channel 132 */
+	CHAN5G(5680), /* Channel 136 */
+	CHAN5G(5700), /* Channel 140 */
+
+	CHAN5G(5745), /* Channel 149 */
+	CHAN5G(5765), /* Channel 153 */
+	CHAN5G(5785), /* Channel 157 */
+	CHAN5G(5805), /* Channel 161 */
+	CHAN5G(5825), /* Channel 165 */
+	CHAN5G(5845), /* Channel 169 */
+
+	CHAN5G(5855), /* Channel 171 */
+	CHAN5G(5860), /* Channel 172 */
+	CHAN5G(5865), /* Channel 173 */
+	CHAN5G(5870), /* Channel 174 */
+
+	CHAN5G(5875), /* Channel 175 */
+	CHAN5G(5880), /* Channel 176 */
+	CHAN5G(5885), /* Channel 177 */
+	CHAN5G(5890), /* Channel 178 */
+	CHAN5G(5895), /* Channel 179 */
+	CHAN5G(5900), /* Channel 180 */
+	CHAN5G(5905), /* Channel 181 */
+
+	CHAN5G(5910), /* Channel 182 */
+	CHAN5G(5915), /* Channel 183 */
+	CHAN5G(5920), /* Channel 184 */
+	CHAN5G(5925), /* Channel 185 */
+};
+static_assert(HWSIM_NUM_CHANNELS_5GHZ == ARRAY_SIZE(hwsim_channels_5ghz),
+	      "Inconsistent 5 GHz channel count");
+
+static const struct ieee80211_channel hwsim_channels_6ghz[] = {
+	CHAN6G(5955), /* Channel 1 */
+	CHAN6G(5975), /* Channel 5 */
+	CHAN6G(5995), /* Channel 9 */
+	CHAN6G(6015), /* Channel 13 */
+	CHAN6G(6035), /* Channel 17 */
+	CHAN6G(6055), /* Channel 21 */
+	CHAN6G(6075), /* Channel 25 */
+	CHAN6G(6095), /* Channel 29 */
+	CHAN6G(6115), /* Channel 33 */
+	CHAN6G(6135), /* Channel 37 */
+	CHAN6G(6155), /* Channel 41 */
+	CHAN6G(6175), /* Channel 45 */
+	CHAN6G(6195), /* Channel 49 */
+	CHAN6G(6215), /* Channel 53 */
+	CHAN6G(6235), /* Channel 57 */
+	CHAN6G(6255), /* Channel 61 */
+	CHAN6G(6275), /* Channel 65 */
+	CHAN6G(6295), /* Channel 69 */
+	CHAN6G(6315), /* Channel 73 */
+	CHAN6G(6335), /* Channel 77 */
+	CHAN6G(6355), /* Channel 81 */
+	CHAN6G(6375), /* Channel 85 */
+	CHAN6G(6395), /* Channel 89 */
+	CHAN6G(6415), /* Channel 93 */
+	CHAN6G(6435), /* Channel 97 */
+	CHAN6G(6455), /* Channel 181 */
+	CHAN6G(6475), /* Channel 105 */
+	CHAN6G(6495), /* Channel 109 */
+	CHAN6G(6515), /* Channel 113 */
+	CHAN6G(6535), /* Channel 117 */
+	CHAN6G(6555), /* Channel 121 */
+	CHAN6G(6575), /* Channel 125 */
+	CHAN6G(6595), /* Channel 129 */
+	CHAN6G(6615), /* Channel 133 */
+	CHAN6G(6635), /* Channel 137 */
+	CHAN6G(6655), /* Channel 141 */
+	CHAN6G(6675), /* Channel 145 */
+	CHAN6G(6695), /* Channel 149 */
+	CHAN6G(6715), /* Channel 153 */
+	CHAN6G(6735), /* Channel 157 */
+	CHAN6G(6755), /* Channel 161 */
+	CHAN6G(6775), /* Channel 165 */
+	CHAN6G(6795), /* Channel 169 */
+	CHAN6G(6815), /* Channel 173 */
+	CHAN6G(6835), /* Channel 177 */
+	CHAN6G(6855), /* Channel 181 */
+	CHAN6G(6875), /* Channel 185 */
+	CHAN6G(6895), /* Channel 189 */
+	CHAN6G(6915), /* Channel 193 */
+	CHAN6G(6935), /* Channel 197 */
+	CHAN6G(6955), /* Channel 201 */
+	CHAN6G(6975), /* Channel 205 */
+	CHAN6G(6995), /* Channel 209 */
+	CHAN6G(7015), /* Channel 213 */
+	CHAN6G(7035), /* Channel 217 */
+	CHAN6G(7055), /* Channel 221 */
+	CHAN6G(7075), /* Channel 225 */
+	CHAN6G(7095), /* Channel 229 */
+	CHAN6G(7115), /* Channel 233 */
+};
+static_assert(HWSIM_NUM_CHANNELS_6GHZ == ARRAY_SIZE(hwsim_channels_6ghz),
+	      "Inconsistent 6 GHz channel count");
+
+static struct ieee80211_channel hwsim_channels_s1g[HWSIM_NUM_S1G_CHANNELS_US];
+
+static const struct ieee80211_sta_s1g_cap hwsim_s1g_cap = {
+	.s1g = true,
+	.cap = { S1G_CAP0_SGI_1MHZ | S1G_CAP0_SGI_2MHZ,
+		 0,
+		 0,
+		 S1G_CAP3_MAX_MPDU_LEN,
+		 0,
+		 S1G_CAP5_AMPDU,
+		 0,
+		 S1G_CAP7_DUP_1MHZ,
+		 S1G_CAP8_TWT_RESPOND | S1G_CAP8_TWT_REQUEST,
+		 0},
+	.nss_mcs = { 0xfc | 1, /* MCS 7 for 1 SS */
+	/* RX Highest Supported Long GI Data Rate 0:7 */
+		     0,
+	/* RX Highest Supported Long GI Data Rate 0:7 */
+	/* TX S1G MCS Map 0:6 */
+		     0xfa,
+	/* TX S1G MCS Map :7 */
+	/* TX Highest Supported Long GI Data Rate 0:6 */
+		     0x80,
+	/* TX Highest Supported Long GI Data Rate 7:8 */
+	/* Rx Single spatial stream and S1G-MCS Map for 1MHz */
+	/* Tx Single spatial stream and S1G-MCS Map for 1MHz */
+		     0 },
+};
+
+static void hwsim_init_s1g_channels(struct ieee80211_channel *chans)
+{
+	int ch, freq;
+
+	for (ch = 0; ch < ARRAY_SIZE(hwsim_channels_s1g); ch++) {
+		freq = 902000 + (ch + 1) * 500;
+		chans[ch].band = NL80211_BAND_S1GHZ;
+		chans[ch].center_freq = KHZ_TO_MHZ(freq);
+		chans[ch].freq_offset = freq % 1000;
+		chans[ch].hw_value = ch + 1;
+	}
+}
+
+static const struct ieee80211_rate hwsim_rates[] = {
+	{ .bitrate = 10 },
+	{ .bitrate = 20, .flags = IEEE80211_RATE_SHORT_PREAMBLE },
+	{ .bitrate = 55, .flags = IEEE80211_RATE_SHORT_PREAMBLE },
+	{ .bitrate = 110, .flags = IEEE80211_RATE_SHORT_PREAMBLE },
+	{ .bitrate = 60 },
+	{ .bitrate = 90 },
+	{ .bitrate = 120 },
+	{ .bitrate = 180 },
+	{ .bitrate = 240 },
+	{ .bitrate = 360 },
+	{ .bitrate = 480 },
+	{ .bitrate = 540 }
+};
+static_assert(HWSIM_NUM_RATES == ARRAY_SIZE(hwsim_rates),
+	      "Inconsistent rates count");
+
+#define DEFAULT_RX_RSSI -50
+
+static const u32 hwsim_ciphers[] = {
+	WLAN_CIPHER_SUITE_WEP40,
+	WLAN_CIPHER_SUITE_WEP104,
+	WLAN_CIPHER_SUITE_TKIP,
+	WLAN_CIPHER_SUITE_CCMP,
+	WLAN_CIPHER_SUITE_CCMP_256,
+	WLAN_CIPHER_SUITE_GCMP,
+	WLAN_CIPHER_SUITE_GCMP_256,
+	WLAN_CIPHER_SUITE_AES_CMAC,
+	WLAN_CIPHER_SUITE_BIP_CMAC_256,
+	WLAN_CIPHER_SUITE_BIP_GMAC_128,
+	WLAN_CIPHER_SUITE_BIP_GMAC_256,
+};
+static_assert(HWSIM_NUM_CIPHERS == ARRAY_SIZE(hwsim_ciphers),
+	      "Inconsistent cipher count");
+
+#define OUI_QCA 0x001374
+#define QCA_NL80211_SUBCMD_TEST 1
+enum qca_nl80211_vendor_subcmds {
+	QCA_WLAN_VENDOR_ATTR_TEST = 8,
+	QCA_WLAN_VENDOR_ATTR_MAX = QCA_WLAN_VENDOR_ATTR_TEST
+};
+
+static const struct nla_policy
+hwsim_vendor_test_policy[QCA_WLAN_VENDOR_ATTR_MAX + 1] = {
+	[QCA_WLAN_VENDOR_ATTR_MAX] = { .type = NLA_U32 },
+};
+
+static int mac80211_hwsim_vendor_cmd_test(struct wiphy *wiphy,
+					  struct wireless_dev *wdev,
+					  const void *data, int data_len)
+{
+	struct sk_buff *skb;
+	struct nlattr *tb[QCA_WLAN_VENDOR_ATTR_MAX + 1];
+	int err;
+	u32 val;
+
+	err = nla_parse_deprecated(tb, QCA_WLAN_VENDOR_ATTR_MAX, data,
+				   data_len, hwsim_vendor_test_policy, NULL);
+	if (err)
+		return err;
+	if (!tb[QCA_WLAN_VENDOR_ATTR_TEST])
+		return -EINVAL;
+	val = nla_get_u32(tb[QCA_WLAN_VENDOR_ATTR_TEST]);
+	wiphy_dbg(wiphy, "%s: test=%u\n", __func__, val);
+
+	/* Send a vendor event as a test. Note that this would not normally be
+	 * done within a command handler, but rather, based on some other
+	 * trigger. For simplicity, this command is used to trigger the event
+	 * here.
+	 *
+	 * event_idx = 0 (index in mac80211_hwsim_vendor_commands)
+	 */
+	skb = cfg80211_vendor_event_alloc(wiphy, wdev, 100, 0, GFP_KERNEL);
+	if (skb) {
+		/* skb_put() or nla_put() will fill up data within
+		 * NL80211_ATTR_VENDOR_DATA.
+		 */
+
+		/* Add vendor data */
+		nla_put_u32(skb, QCA_WLAN_VENDOR_ATTR_TEST, val + 1);
+
+		/* Send the event - this will call nla_nest_end() */
+		cfg80211_vendor_event(skb, GFP_KERNEL);
+	}
+
+	/* Send a response to the command */
+	skb = cfg80211_vendor_cmd_alloc_reply_skb(wiphy, 10);
+	if (!skb)
+		return -ENOMEM;
+
+	/* skb_put() or nla_put() will fill up data within
+	 * NL80211_ATTR_VENDOR_DATA
+	 */
+	nla_put_u32(skb, QCA_WLAN_VENDOR_ATTR_TEST, val + 2);
+
+	return cfg80211_vendor_cmd_reply(skb);
+}
+
+static struct wiphy_vendor_command mac80211_hwsim_vendor_commands[] = {
+	{
+		.info = { .vendor_id = OUI_QCA,
+			  .subcmd = QCA_NL80211_SUBCMD_TEST },
+		.flags = WIPHY_VENDOR_CMD_NEED_NETDEV,
+		.doit = mac80211_hwsim_vendor_cmd_test,
+		.policy = hwsim_vendor_test_policy,
+		.maxattr = QCA_WLAN_VENDOR_ATTR_MAX,
+	}
+};
+
+/* Advertise support vendor specific events */
+static const struct nl80211_vendor_cmd_info mac80211_hwsim_vendor_events[] = {
+	{ .vendor_id = OUI_QCA, .subcmd = 1 },
+};
+
+DEFINE_SPINLOCK(hwsim_radio_lock);
+LIST_HEAD(hwsim_radios);
+static struct rhashtable hwsim_radios_rht;
+static int hwsim_radio_idx;
+static int hwsim_radios_generation = 1;
+
+static struct platform_driver mac80211_hwsim_driver = {
+	.driver = {
+		.name = "mac80211_hwsim",
+	},
+};
+
+static const struct rhashtable_params hwsim_rht_params = {
+	.nelem_hint = 2,
+	.automatic_shrinking = true,
+	.key_len = ETH_ALEN,
+	.key_offset = offsetof(struct mac80211_hwsim_data, addresses[1]),
+	.head_offset = offsetof(struct mac80211_hwsim_data, rht),
+};
+
+struct hwsim_radiotap_hdr {
+	struct ieee80211_radiotap_header_fixed hdr;
+	__le64 rt_tsft;
+	u8 rt_flags;
+	u8 rt_rate;
+	__le16 rt_channel;
+	__le16 rt_chbitmask;
+} __packed;
+
+struct hwsim_radiotap_ack_hdr {
+	struct ieee80211_radiotap_header_fixed hdr;
+	u8 rt_flags;
+	u8 pad;
+	__le16 rt_channel;
+	__le16 rt_chbitmask;
+} __packed;
+
+static struct mac80211_hwsim_data *get_hwsim_data_ref_from_addr(const u8 *addr)
+{
+	return rhashtable_lookup_fast(&hwsim_radios_rht, addr, hwsim_rht_params);
+}
+
+/* MAC80211_HWSIM netlink family */
+static struct genl_family hwsim_genl_family;
+
+enum hwsim_multicast_groups {
+	HWSIM_MCGRP_CONFIG,
+};
+
+static const struct genl_multicast_group hwsim_mcgrps[] = {
+	[HWSIM_MCGRP_CONFIG] = { .name = "config", },
+};
+
+/* MAC80211_HWSIM netlink policy */
+
+static const struct nla_policy
+hwsim_rate_info_policy[HWSIM_RATE_INFO_ATTR_MAX + 1] = {
+	[HWSIM_RATE_INFO_ATTR_FLAGS] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_MCS] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_LEGACY] = { .type = NLA_U16 },
+	[HWSIM_RATE_INFO_ATTR_NSS] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_BW] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_HE_GI] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_HE_DCM] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_EHT_GI] = { .type = NLA_U8 },
+	[HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC] = { .type = NLA_U8 },
+};
+
+static const struct nla_policy
+hwsim_ftm_result_policy[NL80211_PMSR_FTM_RESP_ATTR_MAX + 1] = {
+	[NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_BURST_INDEX] = { .type = NLA_U16 },
+	[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_BURST_DURATION] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_RSSI_AVG] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_RSSI_SPREAD] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_TX_RATE] = NLA_POLICY_NESTED(hwsim_rate_info_policy),
+	[NL80211_PMSR_FTM_RESP_ATTR_RX_RATE] = NLA_POLICY_NESTED(hwsim_rate_info_policy),
+	[NL80211_PMSR_FTM_RESP_ATTR_RTT_AVG] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_RTT_VARIANCE] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_RTT_SPREAD] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_DIST_AVG] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_DIST_VARIANCE] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_DIST_SPREAD] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_RESP_ATTR_LCI] = { .type = NLA_STRING },
+	[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC] = { .type = NLA_STRING },
+	[NL80211_PMSR_FTM_RESP_ATTR_TX_LTF_REPETITION_COUNT] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_RX_LTF_REPETITION_COUNT] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_NUM_TX_SPATIAL_STREAMS] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_NUM_RX_SPATIAL_STREAMS] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_RESP_ATTR_NOMINAL_TIME] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_AVAILABILITY_WINDOW] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_CHANNEL_WIDTH] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_PREAMBLE] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_RESP_ATTR_IS_DELAYED_LMR] = { .type = NLA_FLAG },
+};
+
+static const struct nla_policy
+hwsim_pmsr_resp_type_policy[NL80211_PMSR_TYPE_MAX + 1] = {
+	[NL80211_PMSR_TYPE_FTM] = NLA_POLICY_NESTED(hwsim_ftm_result_policy),
+};
+
+static const struct nla_policy
+hwsim_pmsr_resp_policy[NL80211_PMSR_RESP_ATTR_MAX + 1] = {
+	[NL80211_PMSR_RESP_ATTR_STATUS] = { .type = NLA_U32 },
+	[NL80211_PMSR_RESP_ATTR_HOST_TIME] = { .type = NLA_U64 },
+	[NL80211_PMSR_RESP_ATTR_AP_TSF] = { .type = NLA_U64 },
+	[NL80211_PMSR_RESP_ATTR_FINAL] = { .type = NLA_FLAG },
+	[NL80211_PMSR_RESP_ATTR_DATA] = NLA_POLICY_NESTED(hwsim_pmsr_resp_type_policy),
+};
+
+static const struct nla_policy
+hwsim_pmsr_peer_result_policy[NL80211_PMSR_PEER_ATTR_MAX + 1] = {
+	[NL80211_PMSR_PEER_ATTR_ADDR] = NLA_POLICY_ETH_ADDR_COMPAT,
+	[NL80211_PMSR_PEER_ATTR_CHAN] = { .type = NLA_REJECT },
+	[NL80211_PMSR_PEER_ATTR_REQ] = { .type = NLA_REJECT },
+	[NL80211_PMSR_PEER_ATTR_RESP] = NLA_POLICY_NESTED(hwsim_pmsr_resp_policy),
+};
+
+static const struct nla_policy
+hwsim_pmsr_peers_result_policy[NL80211_PMSR_ATTR_MAX + 1] = {
+	[NL80211_PMSR_ATTR_MAX_PEERS] = { .type = NLA_REJECT },
+	[NL80211_PMSR_ATTR_REPORT_AP_TSF] = { .type = NLA_REJECT },
+	[NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR] = { .type = NLA_REJECT },
+	[NL80211_PMSR_ATTR_TYPE_CAPA] = { .type = NLA_REJECT },
+	[NL80211_PMSR_ATTR_PEERS] = NLA_POLICY_NESTED_ARRAY(hwsim_pmsr_peer_result_policy),
+};
+
+static const struct nla_policy
+hwsim_ftm_role_capa_policy[NL80211_PMSR_FTM_CAPA_ATTR_MAX + 1] = {
+	[NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_NTB] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_TB] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_EDCA] = { .type = NLA_FLAG },
+};
+
+static const struct nla_policy
+hwsim_ftm_type_capa_policy[NL80211_PMSR_FTM_TYPE_CAPA_ATTR_MAX + 1] = {
+	[NL80211_PMSR_FTM_TYPE_CAPA_ATTR_INFRA_SUPPORT] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_TYPE_CAPA_ATTR_PD_SUPPORT] = { .type = NLA_FLAG },
+};
+
+static const struct nla_policy
+hwsim_ftm_capa_policy[NL80211_PMSR_FTM_CAPA_ATTR_MAX + 1] = {
+	[NL80211_PMSR_FTM_CAPA_ATTR_ASAP] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_NON_ASAP] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_REQ_LCI] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_REQ_CIVICLOC] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT] = NLA_POLICY_MAX(NLA_U8, 15),
+	[NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST] = NLA_POLICY_MAX(NLA_U8, 31),
+	[NL80211_PMSR_FTM_CAPA_ATTR_TRIGGER_BASED] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_NON_TRIGGER_BASED] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_TX_ANTENNAS] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_RX_ANTENNAS] = { .type = NLA_U8 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_EDCA] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_NTB] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_PD_PREAMBLES] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_PD_BANDWIDTHS] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS] =
+		NLA_POLICY_NESTED(hwsim_ftm_role_capa_policy),
+	[NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS] =
+		NLA_POLICY_NESTED(hwsim_ftm_role_capa_policy),
+	[NL80211_PMSR_FTM_CAPA_ATTR_TYPE_CAPS] =
+		NLA_POLICY_NESTED(hwsim_ftm_type_capa_policy),
+	[NL80211_PMSR_FTM_CAPA_ATTR_CONCURRENT_ISTA_RSTA_SUPPORT] = { .type = NLA_FLAG },
+};
+
+static const struct nla_policy
+hwsim_pmsr_capa_type_policy[NL80211_PMSR_TYPE_MAX + 1] = {
+	[NL80211_PMSR_TYPE_FTM] = NLA_POLICY_NESTED(hwsim_ftm_capa_policy),
+};
+
+static const struct nla_policy
+hwsim_pmsr_capa_policy[NL80211_PMSR_ATTR_MAX + 1] = {
+	[NL80211_PMSR_ATTR_MAX_PEERS] = { .type = NLA_U32 },
+	[NL80211_PMSR_ATTR_REPORT_AP_TSF] = { .type = NLA_FLAG },
+	[NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR] = { .type = NLA_FLAG },
+	[NL80211_PMSR_ATTR_TYPE_CAPA] = NLA_POLICY_NESTED(hwsim_pmsr_capa_type_policy),
+	[NL80211_PMSR_ATTR_PEERS] = { .type = NLA_REJECT }, // only for request.
+};
+
+static const struct nla_policy hwsim_genl_policy[HWSIM_ATTR_MAX + 1] = {
+	[HWSIM_ATTR_ADDR_RECEIVER] = NLA_POLICY_ETH_ADDR_COMPAT,
+	[HWSIM_ATTR_ADDR_TRANSMITTER] = NLA_POLICY_ETH_ADDR_COMPAT,
+	[HWSIM_ATTR_FRAME] = { .type = NLA_BINARY,
+			       .len = IEEE80211_MAX_DATA_LEN },
+	[HWSIM_ATTR_FLAGS] = { .type = NLA_U32 },
+	[HWSIM_ATTR_RX_RATE] = { .type = NLA_U32 },
+	[HWSIM_ATTR_SIGNAL] = { .type = NLA_U32 },
+	[HWSIM_ATTR_TX_INFO] = { .type = NLA_BINARY,
+				 .len = IEEE80211_TX_MAX_RATES *
+					sizeof(struct hwsim_tx_rate)},
+	[HWSIM_ATTR_COOKIE] = { .type = NLA_U64 },
+	[HWSIM_ATTR_CHANNELS] = { .type = NLA_U32 },
+	[HWSIM_ATTR_RADIO_ID] = { .type = NLA_U32 },
+	[HWSIM_ATTR_REG_HINT_ALPHA2] = { .type = NLA_STRING, .len = 2 },
+	[HWSIM_ATTR_REG_CUSTOM_REG] = { .type = NLA_U32 },
+	[HWSIM_ATTR_REG_STRICT_REG] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_SUPPORT_P2P_DEVICE] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_USE_CHANCTX] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_DESTROY_RADIO_ON_CLOSE] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_RADIO_NAME] = { .type = NLA_STRING },
+	[HWSIM_ATTR_NO_VIF] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_FREQ] = { .type = NLA_U32 },
+	[HWSIM_ATTR_TX_INFO_FLAGS] = { .type = NLA_BINARY },
+	[HWSIM_ATTR_PERM_ADDR] = NLA_POLICY_ETH_ADDR_COMPAT,
+	[HWSIM_ATTR_IFTYPE_SUPPORT] = { .type = NLA_U32 },
+	[HWSIM_ATTR_CIPHER_SUPPORT] = { .type = NLA_BINARY },
+	[HWSIM_ATTR_MLO_SUPPORT] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_PMSR_SUPPORT] = NLA_POLICY_NESTED(hwsim_pmsr_capa_policy),
+	[HWSIM_ATTR_PMSR_RESULT] = NLA_POLICY_NESTED(hwsim_pmsr_peers_result_policy),
+	[HWSIM_ATTR_MULTI_RADIO] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_SUPPORT_NAN_DEVICE] = { .type = NLA_FLAG },
+	[HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR] = { .type = NLA_FLAG },
+};
+
+#if IS_REACHABLE(CONFIG_VIRTIO)
+
+/* MAC80211_HWSIM virtio queues */
+static struct virtqueue *hwsim_vqs[HWSIM_NUM_VQS];
+static bool hwsim_virtio_enabled;
+static DEFINE_SPINLOCK(hwsim_virtio_lock);
+
+static void hwsim_virtio_rx_work(struct work_struct *work);
+static DECLARE_WORK(hwsim_virtio_rx, hwsim_virtio_rx_work);
+
+static int hwsim_tx_virtio(struct mac80211_hwsim_data *data,
+			   struct sk_buff *skb)
+{
+	struct scatterlist sg[1];
+	unsigned long flags;
+	int err;
+
+	spin_lock_irqsave(&hwsim_virtio_lock, flags);
+	if (!hwsim_virtio_enabled) {
+		err = -ENODEV;
+		goto out_free;
+	}
+
+	sg_init_one(sg, skb->head, skb_end_offset(skb));
+	err = virtqueue_add_outbuf(hwsim_vqs[HWSIM_VQ_TX], sg, 1, skb,
+				   GFP_ATOMIC);
+	if (err)
+		goto out_free;
+	virtqueue_kick(hwsim_vqs[HWSIM_VQ_TX]);
+	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
+	return 0;
+
+out_free:
+	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
+	nlmsg_free(skb);
+	return err;
+}
+#else
+/* cause a linker error if this ends up being needed */
+extern int hwsim_tx_virtio(struct mac80211_hwsim_data *data,
+			   struct sk_buff *skb);
+#define hwsim_virtio_enabled false
+#endif
+
+static int hwsim_get_chanwidth(enum nl80211_chan_width bw)
+{
+	switch (bw) {
+	case NL80211_CHAN_WIDTH_20_NOHT:
+	case NL80211_CHAN_WIDTH_20:
+		return 20;
+	case NL80211_CHAN_WIDTH_40:
+		return 40;
+	case NL80211_CHAN_WIDTH_80:
+		return 80;
+	case NL80211_CHAN_WIDTH_80P80:
+	case NL80211_CHAN_WIDTH_160:
+		return 160;
+	case NL80211_CHAN_WIDTH_320:
+		return 320;
+	case NL80211_CHAN_WIDTH_5:
+		return 5;
+	case NL80211_CHAN_WIDTH_10:
+		return 10;
+	case NL80211_CHAN_WIDTH_1:
+		return 1;
+	case NL80211_CHAN_WIDTH_2:
+		return 2;
+	case NL80211_CHAN_WIDTH_4:
+		return 4;
+	case NL80211_CHAN_WIDTH_8:
+		return 8;
+	case NL80211_CHAN_WIDTH_16:
+		return 16;
+	}
+
+	return INT_MAX;
+}
+
+/* sysfs attributes */
+static void hwsim_send_ps_poll(void *dat, u8 *mac, struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_data *data = dat;
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+	struct sk_buff *skb;
+	struct ieee80211_pspoll *pspoll;
+
+	if (!vp->assoc)
+		return;
+
+	wiphy_dbg(data->hw->wiphy,
+		  "%s: send PS-Poll to %pM for aid %d\n",
+		  __func__, vp->bssid, vp->aid);
+
+	skb = dev_alloc_skb(sizeof(*pspoll));
+	if (!skb)
+		return;
+	pspoll = skb_put(skb, sizeof(*pspoll));
+	pspoll->frame_control = cpu_to_le16(IEEE80211_FTYPE_CTL |
+					    IEEE80211_STYPE_PSPOLL |
+					    IEEE80211_FCTL_PM);
+	pspoll->aid = cpu_to_le16(0xc000 | vp->aid);
+	memcpy(pspoll->bssid, vp->bssid, ETH_ALEN);
+	memcpy(pspoll->ta, mac, ETH_ALEN);
+
+	rcu_read_lock();
+	mac80211_hwsim_tx_frame(data->hw, skb,
+				rcu_dereference(vif->bss_conf.chanctx_conf)->def.chan);
+	rcu_read_unlock();
+}
+
+static void hwsim_send_nullfunc(struct mac80211_hwsim_data *data, u8 *mac,
+				struct ieee80211_vif *vif, int ps)
+{
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+	struct sk_buff *skb;
+	struct ieee80211_hdr *hdr;
+	struct ieee80211_tx_info *cb;
+
+	if (!vp->assoc)
+		return;
+
+	wiphy_dbg(data->hw->wiphy,
+		  "%s: send data::nullfunc to %pM ps=%d\n",
+		  __func__, vp->bssid, ps);
+
+	skb = dev_alloc_skb(sizeof(*hdr));
+	if (!skb)
+		return;
+	hdr = skb_put(skb, sizeof(*hdr) - ETH_ALEN);
+	hdr->frame_control = cpu_to_le16(IEEE80211_FTYPE_DATA |
+					 IEEE80211_STYPE_NULLFUNC |
+					 IEEE80211_FCTL_TODS |
+					 (ps ? IEEE80211_FCTL_PM : 0));
+	hdr->duration_id = cpu_to_le16(0);
+	memcpy(hdr->addr1, vp->bssid, ETH_ALEN);
+	memcpy(hdr->addr2, mac, ETH_ALEN);
+	memcpy(hdr->addr3, vp->bssid, ETH_ALEN);
+
+	cb = IEEE80211_SKB_CB(skb);
+	cb->control.rates[0].count = 1;
+	cb->control.rates[1].idx = -1;
+
+	rcu_read_lock();
+	mac80211_hwsim_tx_frame(data->hw, skb,
+				rcu_dereference(vif->bss_conf.chanctx_conf)->def.chan);
+	rcu_read_unlock();
+}
+
+
+static void hwsim_send_nullfunc_ps(void *dat, u8 *mac,
+				   struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_data *data = dat;
+	hwsim_send_nullfunc(data, mac, vif, 1);
+}
+
+static void hwsim_send_nullfunc_no_ps(void *dat, u8 *mac,
+				      struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_data *data = dat;
+	hwsim_send_nullfunc(data, mac, vif, 0);
+}
+
+static int hwsim_fops_ps_read(void *dat, u64 *val)
+{
+	struct mac80211_hwsim_data *data = dat;
+	*val = data->ps;
+	return 0;
+}
+
+static int hwsim_fops_ps_write(void *dat, u64 val)
+{
+	struct mac80211_hwsim_data *data = dat;
+	enum ps_mode old_ps;
+
+	if (val != PS_DISABLED && val != PS_ENABLED && val != PS_AUTO_POLL &&
+	    val != PS_MANUAL_POLL)
+		return -EINVAL;
+
+	if (val == PS_MANUAL_POLL) {
+		if (data->ps != PS_ENABLED)
+			return -EINVAL;
+		local_bh_disable();
+		ieee80211_iterate_active_interfaces_atomic(
+			data->hw, IEEE80211_IFACE_ITER_NORMAL,
+			hwsim_send_ps_poll, data);
+		local_bh_enable();
+		return 0;
+	}
+	old_ps = data->ps;
+	data->ps = val;
+
+	local_bh_disable();
+	if (old_ps == PS_DISABLED && val != PS_DISABLED) {
+		ieee80211_iterate_active_interfaces_atomic(
+			data->hw, IEEE80211_IFACE_ITER_NORMAL,
+			hwsim_send_nullfunc_ps, data);
+	} else if (old_ps != PS_DISABLED && val == PS_DISABLED) {
+		ieee80211_iterate_active_interfaces_atomic(
+			data->hw, IEEE80211_IFACE_ITER_NORMAL,
+			hwsim_send_nullfunc_no_ps, data);
+	}
+	local_bh_enable();
+
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(hwsim_fops_ps, hwsim_fops_ps_read, hwsim_fops_ps_write,
+			 "%llu\n");
+
+static int hwsim_write_simulate_radar(void *dat, u64 val)
+{
+	struct mac80211_hwsim_data *data = dat;
+
+	ieee80211_radar_detected(data->hw, NULL);
+
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(hwsim_simulate_radar, NULL,
+			 hwsim_write_simulate_radar, "%llu\n");
+
+static ssize_t hwsim_background_cac_write(struct file *file,
+					  const char __user *user_buf,
+					  size_t count, loff_t *ppos)
+{
+	struct mac80211_hwsim_data *data = file->private_data;
+	char buf[8] = {};
+
+	if (count >= sizeof(buf))
+		return -EINVAL;
+
+	if (copy_from_user(buf, user_buf, count))
+		return -EFAULT;
+
+	/* Check if background radar channel is configured */
+	if (!data->radar_background_chandef.chan)
+		return -ENOENT;
+
+	if (sysfs_streq(buf, "radar"))
+		cfg80211_background_radar_event(data->hw->wiphy,
+						&data->radar_background_chandef,
+						GFP_KERNEL);
+	else if (sysfs_streq(buf, "cancel"))
+		cfg80211_background_cac_abort(data->hw->wiphy);
+	else
+		return -EINVAL;
+
+	return count;
+}
+
+static const struct file_operations hwsim_background_cac_ops = {
+	.write = hwsim_background_cac_write,
+	.open = simple_open,
+	.llseek = default_llseek,
+};
+
+struct hwsim_chanctx_iter_arg {
+	struct ieee80211_chanctx_conf *conf;
+	u32 freq_mhz;
+};
+
+static void hwsim_6ghz_chanctx_iter(struct ieee80211_hw *hw,
+				    struct ieee80211_chanctx_conf *conf,
+				    void *data)
+{
+	struct hwsim_chanctx_iter_arg *arg = data;
+
+	if (conf->def.chan &&
+	    conf->def.chan->band == NL80211_BAND_6GHZ &&
+	    conf->def.chan->center_freq == arg->freq_mhz)
+		arg->conf = conf;
+}
+
+static ssize_t hwsim_simulate_incumbent_signal_write(struct file *file,
+						     const char __user *ubuf,
+						     size_t len, loff_t *ppos)
+{
+	struct mac80211_hwsim_data *data = file->private_data;
+	struct hwsim_chanctx_iter_arg arg = {};
+	u32 bitmap;
+	char buf[64];
+
+	if (!len || len > sizeof(buf) - 1)
+		return -EINVAL;
+
+	if (copy_from_user(buf, ubuf, len))
+		return -EFAULT;
+	buf[len] = '\0';
+
+	if (sscanf(buf, "%u %i", &arg.freq_mhz, &bitmap) != 2)
+		return -EINVAL;
+
+	if (!arg.freq_mhz)
+		return -EINVAL;
+
+	ieee80211_iter_chan_contexts_atomic(data->hw,
+					    hwsim_6ghz_chanctx_iter,
+					    &arg);
+
+	if (!arg.conf)
+		return -EINVAL;
+
+	cfg80211_incumbent_signal_notify(data->hw->wiphy,
+					 &arg.conf->def,
+					 bitmap,
+					 GFP_KERNEL);
+
+	return len;
+}
+
+static const struct file_operations hwsim_simulate_incumbent_signal_fops = {
+	.open	= simple_open,
+	.write	= hwsim_simulate_incumbent_signal_write,
+};
+
+static int hwsim_fops_group_read(void *dat, u64 *val)
+{
+	struct mac80211_hwsim_data *data = dat;
+	*val = data->group;
+	return 0;
+}
+
+static int hwsim_fops_group_write(void *dat, u64 val)
+{
+	struct mac80211_hwsim_data *data = dat;
+	data->group = val;
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(hwsim_fops_group,
+			 hwsim_fops_group_read, hwsim_fops_group_write,
+			 "%llx\n");
+
+static int hwsim_fops_rx_rssi_read(void *dat, u64 *val)
+{
+	struct mac80211_hwsim_data *data = dat;
+	*val = data->rx_rssi;
+	return 0;
+}
+
+static int hwsim_fops_rx_rssi_write(void *dat, u64 val)
+{
+	struct mac80211_hwsim_data *data = dat;
+	int rssi = (int)val;
+
+	if (rssi >= 0 || rssi < -100)
+		return -EINVAL;
+
+	data->rx_rssi = rssi;
+	return 0;
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(hwsim_fops_rx_rssi,
+			 hwsim_fops_rx_rssi_read, hwsim_fops_rx_rssi_write,
+			 "%lld\n");
+
+static netdev_tx_t hwsim_mon_xmit(struct sk_buff *skb,
+					struct net_device *dev)
+{
+	/* TODO: allow packet injection */
+	dev_kfree_skb(skb);
+	return NETDEV_TX_OK;
+}
+
+static inline u64 mac80211_hwsim_get_sim_tsf(void)
+{
+	return ktime_to_us(ktime_get_boottime());
+}
+
+ktime_t mac80211_hwsim_tsf_to_boottime(struct mac80211_hwsim_data *data,
+				       u64 tsf)
+{
+	scoped_guard(spinlock_bh, &data->tsf_offset_lock) {
+		return us_to_ktime(tsf - data->tsf_offset);
+	}
+}
+
+u64 mac80211_hwsim_boottime_to_tsf(struct mac80211_hwsim_data *data,
+				   ktime_t ts)
+{
+	scoped_guard(spinlock_bh, &data->tsf_offset_lock) {
+		return ktime_to_us(ts) + data->tsf_offset;
+	}
+}
+
+u64 mac80211_hwsim_get_tsf(struct ieee80211_hw *hw,
+			   struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	u64 sim_time = mac80211_hwsim_get_sim_tsf();
+
+	scoped_guard(spinlock_bh, &data->tsf_offset_lock) {
+		return sim_time + data->tsf_offset;
+	}
+}
+
+static __le64 __mac80211_hwsim_get_tsf(struct mac80211_hwsim_data *data)
+{
+	u64 sim_time = mac80211_hwsim_get_sim_tsf();
+
+	scoped_guard(spinlock_bh, &data->tsf_offset_lock) {
+		return cpu_to_le64(sim_time + data->tsf_offset);
+	}
+}
+
+static void mac80211_hwsim_set_tsf(struct ieee80211_hw *hw,
+		struct ieee80211_vif *vif, u64 tsf)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	u64 now = mac80211_hwsim_get_tsf(hw, vif);
+	u64 delta = abs(tsf - now);
+	struct ieee80211_bss_conf *conf;
+
+	conf = link_conf_dereference_protected(vif, data->link_data[0].link_id);
+	if (conf && !conf->enable_beacon)
+		return;
+
+	scoped_guard(spinlock_bh, &data->tsf_offset_lock) {
+		/* adjust after beaconing with new timestamp at old TBTT */
+		if (tsf > now)
+			data->tsf_offset += delta;
+		else
+			data->tsf_offset -= delta;
+	}
+}
+
+static void mac80211_hwsim_monitor_rx(struct ieee80211_hw *hw,
+				      struct sk_buff *tx_skb,
+				      struct ieee80211_channel *chan)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	struct sk_buff *skb;
+	struct hwsim_radiotap_hdr *hdr;
+	u16 flags, bitrate;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(tx_skb);
+	struct ieee80211_rate *txrate = ieee80211_get_tx_rate(hw, info);
+
+	if (!txrate)
+		bitrate = 0;
+	else
+		bitrate = txrate->bitrate;
+
+	if (!netif_running(hwsim_mon))
+		return;
+
+	skb = skb_copy_expand(tx_skb, sizeof(*hdr), 0, GFP_ATOMIC);
+	if (skb == NULL)
+		return;
+
+	hdr = skb_push(skb, sizeof(*hdr));
+	hdr->hdr.it_version = PKTHDR_RADIOTAP_VERSION;
+	hdr->hdr.it_pad = 0;
+	hdr->hdr.it_len = cpu_to_le16(sizeof(*hdr));
+	hdr->hdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) |
+					  (1 << IEEE80211_RADIOTAP_RATE) |
+					  (1 << IEEE80211_RADIOTAP_TSFT) |
+					  (1 << IEEE80211_RADIOTAP_CHANNEL));
+	hdr->rt_tsft = __mac80211_hwsim_get_tsf(data);
+	hdr->rt_flags = 0;
+	hdr->rt_rate = bitrate / 5;
+	hdr->rt_channel = cpu_to_le16(chan->center_freq);
+	flags = IEEE80211_CHAN_2GHZ;
+	if (txrate && txrate->flags & IEEE80211_RATE_ERP_G)
+		flags |= IEEE80211_CHAN_OFDM;
+	else
+		flags |= IEEE80211_CHAN_CCK;
+	hdr->rt_chbitmask = cpu_to_le16(flags);
+
+	skb->dev = hwsim_mon;
+	skb_reset_mac_header(skb);
+	skb->ip_summed = CHECKSUM_UNNECESSARY;
+	skb->pkt_type = PACKET_OTHERHOST;
+	skb->protocol = htons(ETH_P_802_2);
+	memset(skb->cb, 0, sizeof(skb->cb));
+	netif_rx(skb);
+}
+
+
+static void mac80211_hwsim_monitor_ack(struct ieee80211_channel *chan,
+				       const u8 *addr)
+{
+	struct sk_buff *skb;
+	struct hwsim_radiotap_ack_hdr *hdr;
+	u16 flags;
+	struct ieee80211_hdr *hdr11;
+
+	if (!netif_running(hwsim_mon))
+		return;
+
+	skb = dev_alloc_skb(100);
+	if (skb == NULL)
+		return;
+
+	hdr = skb_put(skb, sizeof(*hdr));
+	hdr->hdr.it_version = PKTHDR_RADIOTAP_VERSION;
+	hdr->hdr.it_pad = 0;
+	hdr->hdr.it_len = cpu_to_le16(sizeof(*hdr));
+	hdr->hdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) |
+					  (1 << IEEE80211_RADIOTAP_CHANNEL));
+	hdr->rt_flags = 0;
+	hdr->pad = 0;
+	hdr->rt_channel = cpu_to_le16(chan->center_freq);
+	flags = IEEE80211_CHAN_2GHZ;
+	hdr->rt_chbitmask = cpu_to_le16(flags);
+
+	hdr11 = skb_put(skb, 10);
+	hdr11->frame_control = cpu_to_le16(IEEE80211_FTYPE_CTL |
+					   IEEE80211_STYPE_ACK);
+	hdr11->duration_id = cpu_to_le16(0);
+	memcpy(hdr11->addr1, addr, ETH_ALEN);
+
+	skb->dev = hwsim_mon;
+	skb_reset_mac_header(skb);
+	skb->ip_summed = CHECKSUM_UNNECESSARY;
+	skb->pkt_type = PACKET_OTHERHOST;
+	skb->protocol = htons(ETH_P_802_2);
+	memset(skb->cb, 0, sizeof(skb->cb));
+	netif_rx(skb);
+}
+
+struct mac80211_hwsim_addr_match_data {
+	u8 addr[ETH_ALEN];
+	bool ret;
+};
+
+static void mac80211_hwsim_addr_iter(void *data, u8 *mac,
+				     struct ieee80211_vif *vif)
+{
+	int i;
+	struct mac80211_hwsim_addr_match_data *md = data;
+
+	if (memcmp(mac, md->addr, ETH_ALEN) == 0) {
+		md->ret = true;
+		return;
+	}
+
+	/* Match the link address */
+	for (i = 0; i < ARRAY_SIZE(vif->link_conf); i++) {
+		struct ieee80211_bss_conf *conf;
+
+		conf = rcu_dereference(vif->link_conf[i]);
+		if (!conf)
+			continue;
+
+		if (memcmp(conf->addr, md->addr, ETH_ALEN) == 0) {
+			md->ret = true;
+			return;
+		}
+	}
+}
+
+static bool mac80211_hwsim_addr_match(struct mac80211_hwsim_data *data,
+				      const u8 *addr)
+{
+	struct mac80211_hwsim_addr_match_data md = {
+		.ret = false,
+	};
+
+	if (data->scanning && memcmp(addr, data->scan_addr, ETH_ALEN) == 0)
+		return true;
+
+	memcpy(md.addr, addr, ETH_ALEN);
+
+	ieee80211_iterate_active_interfaces_atomic(data->hw,
+						   IEEE80211_IFACE_ITER_NORMAL,
+						   mac80211_hwsim_addr_iter,
+						   &md);
+
+	return md.ret;
+}
+
+static bool hwsim_ps_rx_ok(struct mac80211_hwsim_data *data,
+			   struct sk_buff *skb)
+{
+	switch (data->ps) {
+	case PS_DISABLED:
+		return true;
+	case PS_ENABLED:
+		return false;
+	case PS_AUTO_POLL:
+		/* TODO: accept (some) Beacons by default and other frames only
+		 * if pending PS-Poll has been sent */
+		return true;
+	case PS_MANUAL_POLL:
+		/* Allow unicast frames to own address if there is a pending
+		 * PS-Poll */
+		if (data->ps_poll_pending &&
+		    mac80211_hwsim_addr_match(data, skb->data + 4)) {
+			data->ps_poll_pending = false;
+			return true;
+		}
+		return false;
+	}
+
+	return true;
+}
+
+static int hwsim_unicast_netgroup(struct mac80211_hwsim_data *data,
+				  struct sk_buff *skb, int portid)
+{
+	struct net *net;
+	bool found = false;
+	int res = -ENOENT;
+
+	rcu_read_lock();
+	for_each_net_rcu(net) {
+		if (data->netgroup == hwsim_net_get_netgroup(net)) {
+			res = genlmsg_unicast(net, skb, portid);
+			found = true;
+			break;
+		}
+	}
+	rcu_read_unlock();
+
+	if (!found)
+		nlmsg_free(skb);
+
+	return res;
+}
+
+static void mac80211_hwsim_config_mac_nl(struct ieee80211_hw *hw,
+					 const u8 *addr, bool add)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	u32 _portid = READ_ONCE(data->wmediumd);
+	struct sk_buff *skb;
+	void *msg_head;
+
+	WARN_ON(!is_valid_ether_addr(addr));
+
+	if (!_portid && !hwsim_virtio_enabled)
+		return;
+
+	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0,
+			       add ? HWSIM_CMD_ADD_MAC_ADDR :
+				     HWSIM_CMD_DEL_MAC_ADDR);
+	if (!msg_head) {
+		pr_debug("mac80211_hwsim: problem with msg_head\n");
+		goto nla_put_failure;
+	}
+
+	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER,
+		    ETH_ALEN, data->addresses[1].addr))
+		goto nla_put_failure;
+
+	if (nla_put(skb, HWSIM_ATTR_ADDR_RECEIVER, ETH_ALEN, addr))
+		goto nla_put_failure;
+
+	genlmsg_end(skb, msg_head);
+
+	if (hwsim_virtio_enabled)
+		hwsim_tx_virtio(data, skb);
+	else
+		hwsim_unicast_netgroup(data, skb, _portid);
+	return;
+nla_put_failure:
+	nlmsg_free(skb);
+}
+
+static inline u16 trans_tx_rate_flags_ieee2hwsim(struct ieee80211_tx_rate *rate)
+{
+	u16 result = 0;
+
+	if (rate->flags & IEEE80211_TX_RC_USE_RTS_CTS)
+		result |= MAC80211_HWSIM_TX_RC_USE_RTS_CTS;
+	if (rate->flags & IEEE80211_TX_RC_USE_CTS_PROTECT)
+		result |= MAC80211_HWSIM_TX_RC_USE_CTS_PROTECT;
+	if (rate->flags & IEEE80211_TX_RC_USE_SHORT_PREAMBLE)
+		result |= MAC80211_HWSIM_TX_RC_USE_SHORT_PREAMBLE;
+	if (rate->flags & IEEE80211_TX_RC_MCS)
+		result |= MAC80211_HWSIM_TX_RC_MCS;
+	if (rate->flags & IEEE80211_TX_RC_GREEN_FIELD)
+		result |= MAC80211_HWSIM_TX_RC_GREEN_FIELD;
+	if (rate->flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
+		result |= MAC80211_HWSIM_TX_RC_40_MHZ_WIDTH;
+	if (rate->flags & IEEE80211_TX_RC_DUP_DATA)
+		result |= MAC80211_HWSIM_TX_RC_DUP_DATA;
+	if (rate->flags & IEEE80211_TX_RC_SHORT_GI)
+		result |= MAC80211_HWSIM_TX_RC_SHORT_GI;
+	if (rate->flags & IEEE80211_TX_RC_VHT_MCS)
+		result |= MAC80211_HWSIM_TX_RC_VHT_MCS;
+	if (rate->flags & IEEE80211_TX_RC_80_MHZ_WIDTH)
+		result |= MAC80211_HWSIM_TX_RC_80_MHZ_WIDTH;
+	if (rate->flags & IEEE80211_TX_RC_160_MHZ_WIDTH)
+		result |= MAC80211_HWSIM_TX_RC_160_MHZ_WIDTH;
+
+	return result;
+}
+
+static void mac80211_hwsim_write_tsf(struct mac80211_hwsim_data *data,
+				     struct sk_buff *skb, u64 sim_time)
+{
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_mgmt *mgmt = (struct ieee80211_mgmt *)skb->data;
+	struct ieee80211_rate *txrate;
+	/* TODO: get MCS */
+	int bitrate = 100;
+
+	spin_lock_bh(&data->tsf_offset_lock);
+
+	txrate = ieee80211_get_tx_rate(data->hw, info);
+	if (txrate)
+		bitrate = txrate->bitrate;
+
+	if (skb->len >= offsetofend(typeof(*mgmt), u.probe_resp.timestamp) &&
+	    ieee80211_is_probe_resp(hdr->frame_control)) {
+		mgmt->u.probe_resp.timestamp =
+			cpu_to_le64(sim_time + data->tsf_offset +
+				    24 * 8 * 10 / bitrate);
+	} else if (skb->len >= offsetofend(typeof(*mgmt), u.beacon.timestamp) &&
+		   ieee80211_is_beacon(mgmt->frame_control)) {
+		mgmt->u.beacon.timestamp = cpu_to_le64(sim_time +
+						       data->tsf_offset +
+						       24 * 8 * 10 /
+						       bitrate);
+	} else if (skb->len >= offsetofend(struct ieee80211_ext,
+					   u.s1g_beacon.timestamp) &&
+		   ieee80211_is_s1g_beacon(mgmt->frame_control)) {
+		struct ieee80211_ext *ext = (void *)mgmt;
+
+		ext->u.s1g_beacon.timestamp = cpu_to_le32(sim_time +
+							  data->tsf_offset +
+							  10 * 8 * 10 /
+							  bitrate);
+	}
+
+	spin_unlock_bh(&data->tsf_offset_lock);
+}
+
+static void mac80211_hwsim_tx_frame_nl(struct ieee80211_hw *hw,
+				       struct sk_buff *my_skb,
+				       int dst_portid,
+				       struct ieee80211_channel *channel)
+{
+	struct sk_buff *skb;
+	struct mac80211_hwsim_data *data = hw->priv;
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) my_skb->data;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(my_skb);
+	void *msg_head;
+	unsigned int hwsim_flags = 0;
+	int i;
+	struct hwsim_tx_rate tx_attempts[IEEE80211_TX_MAX_RATES];
+	struct hwsim_tx_rate_flag tx_attempts_flags[IEEE80211_TX_MAX_RATES];
+	uintptr_t cookie;
+	u64 sim_tsf;
+
+	if (data->ps != PS_DISABLED)
+		hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
+	/* If the queue contains MAX_QUEUE skb's drop some */
+	if (skb_queue_len(&data->pending) >= MAX_QUEUE) {
+		/* Dropping until WARN_QUEUE level */
+		while (skb_queue_len(&data->pending) >= WARN_QUEUE) {
+			ieee80211_free_txskb(hw, skb_dequeue(&data->pending));
+			data->tx_dropped++;
+		}
+	}
+
+	sim_tsf = mac80211_hwsim_get_sim_tsf();
+	mac80211_hwsim_write_tsf(data, my_skb, sim_tsf);
+
+	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_ATOMIC);
+	if (skb == NULL)
+		goto nla_put_failure;
+
+	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0,
+			       HWSIM_CMD_FRAME);
+	if (msg_head == NULL) {
+		pr_debug("mac80211_hwsim: problem with msg_head\n");
+		goto nla_put_failure;
+	}
+
+	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER,
+		    ETH_ALEN, data->addresses[1].addr))
+		goto nla_put_failure;
+
+	/* We get the skb->data */
+	if (nla_put(skb, HWSIM_ATTR_FRAME, my_skb->len, my_skb->data))
+		goto nla_put_failure;
+
+	/* We get the flags for this transmission, and we translate them to
+	   wmediumd flags  */
+
+	if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)
+		hwsim_flags |= HWSIM_TX_CTL_REQ_TX_STATUS;
+
+	if (info->flags & IEEE80211_TX_CTL_NO_ACK)
+		hwsim_flags |= HWSIM_TX_CTL_NO_ACK;
+
+	if (nla_put_u32(skb, HWSIM_ATTR_FLAGS, hwsim_flags))
+		goto nla_put_failure;
+
+	if (nla_put_u32(skb, HWSIM_ATTR_FREQ, channel->center_freq))
+		goto nla_put_failure;
+
+	/* We get the tx control (rate and retries) info*/
+
+	for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) {
+		tx_attempts[i].idx = info->status.rates[i].idx;
+		tx_attempts_flags[i].idx = info->status.rates[i].idx;
+		tx_attempts[i].count = info->status.rates[i].count;
+		tx_attempts_flags[i].flags =
+				trans_tx_rate_flags_ieee2hwsim(
+						&info->status.rates[i]);
+	}
+
+	if (nla_put(skb, HWSIM_ATTR_TX_INFO,
+		    sizeof(struct hwsim_tx_rate)*IEEE80211_TX_MAX_RATES,
+		    tx_attempts))
+		goto nla_put_failure;
+
+	if (nla_put(skb, HWSIM_ATTR_TX_INFO_FLAGS,
+		    sizeof(struct hwsim_tx_rate_flag) * IEEE80211_TX_MAX_RATES,
+		    tx_attempts_flags))
+		goto nla_put_failure;
+
+	/* We create a cookie to identify this skb */
+	cookie = atomic_inc_return(&data->pending_cookie);
+	info->rate_driver_data[0] = (void *)cookie;
+	if (nla_put_u64_64bit(skb, HWSIM_ATTR_COOKIE, cookie, HWSIM_ATTR_PAD))
+		goto nla_put_failure;
+
+	genlmsg_end(skb, msg_head);
+
+	if (hwsim_virtio_enabled) {
+		if (hwsim_tx_virtio(data, skb))
+			goto err_free_txskb;
+	} else {
+		if (hwsim_unicast_netgroup(data, skb, dst_portid))
+			goto err_free_txskb;
+	}
+
+	/* Enqueue the packet */
+	skb_queue_tail(&data->pending, my_skb);
+	data->tx_pkts++;
+	data->tx_bytes += my_skb->len;
+	return;
+
+nla_put_failure:
+	nlmsg_free(skb);
+err_free_txskb:
+	pr_debug("mac80211_hwsim: error occurred in %s\n", __func__);
+	ieee80211_free_txskb(hw, my_skb);
+	data->tx_failed++;
+}
+
+static bool hwsim_chans_compat(struct ieee80211_channel *c1,
+			       struct ieee80211_channel *c2)
+{
+	if (!c1 || !c2)
+		return false;
+
+	return c1->center_freq == c2->center_freq;
+}
+
+struct tx_iter_data {
+	struct ieee80211_channel *channel;
+	struct ieee80211_rx_status *rx_status;
+	struct ieee80211_hw *hw;
+	bool receive;
+};
+
+static void mac80211_hwsim_tx_iter(void *_data, u8 *addr,
+				   struct ieee80211_vif *vif)
+{
+	struct tx_iter_data *data = _data;
+	int i;
+
+	if (vif->type == NL80211_IFTYPE_NAN ||
+	    vif->type == NL80211_IFTYPE_NAN_DATA) {
+		data->receive = mac80211_hwsim_nan_receive(data->hw,
+							   data->channel,
+							   data->rx_status);
+		return;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(vif->link_conf); i++) {
+		struct ieee80211_bss_conf *conf;
+		struct ieee80211_chanctx_conf *chanctx;
+
+		conf = rcu_dereference(vif->link_conf[i]);
+		if (!conf)
+			continue;
+
+		chanctx = rcu_dereference(conf->chanctx_conf);
+		if (!chanctx)
+			continue;
+
+		if (!hwsim_chans_compat(data->channel, chanctx->def.chan))
+			continue;
+
+		data->receive = true;
+		return;
+	}
+}
+
+static void mac80211_hwsim_add_vendor_rtap(struct sk_buff *skb)
+{
+	/*
+	 * To enable this code, #define the HWSIM_RADIOTAP_OUI,
+	 * e.g. like this:
+	 * #define HWSIM_RADIOTAP_OUI "\x02\x00\x00"
+	 * (but you should use a valid OUI, not that)
+	 *
+	 * If anyone wants to 'donate' a radiotap OUI/subns code
+	 * please send a patch removing this #ifdef and changing
+	 * the values accordingly.
+	 */
+#ifdef HWSIM_RADIOTAP_OUI
+	struct ieee80211_radiotap_vendor_tlv *rtap;
+	static const char vendor_data[8] = "ABCDEFGH";
+
+	// Make sure no padding is needed
+	BUILD_BUG_ON(sizeof(vendor_data) % 4);
+	/* this is last radiotap info before the mac header, so
+	 * skb_reset_mac_header for mac8022 to know the end of
+	 * the radiotap TLV/beginning of the 802.11 header
+	 */
+	skb_reset_mac_header(skb);
+
+	/*
+	 * Note that this code requires the headroom in the SKB
+	 * that was allocated earlier.
+	 */
+	rtap = skb_push(skb, sizeof(*rtap) + sizeof(vendor_data));
+
+	rtap->len = cpu_to_le16(sizeof(*rtap) -
+				sizeof(struct ieee80211_radiotap_tlv) +
+				sizeof(vendor_data));
+	rtap->type = cpu_to_le16(IEEE80211_RADIOTAP_VENDOR_NAMESPACE);
+
+	rtap->content.oui[0] = HWSIM_RADIOTAP_OUI[0];
+	rtap->content.oui[1] = HWSIM_RADIOTAP_OUI[1];
+	rtap->content.oui[2] = HWSIM_RADIOTAP_OUI[2];
+	rtap->content.oui_subtype = 127;
+	/* clear reserved field */
+	rtap->content.reserved = 0;
+	rtap->content.vendor_type = 0;
+	memcpy(rtap->content.data, vendor_data, sizeof(vendor_data));
+
+	IEEE80211_SKB_RXCB(skb)->flag |= RX_FLAG_RADIOTAP_TLV_AT_END;
+#endif
+}
+
+static void mac80211_hwsim_rx(struct mac80211_hwsim_data *data,
+			      struct ieee80211_rx_status *rx_status,
+			      struct sk_buff *skb)
+{
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+
+	if (!ieee80211_has_morefrags(hdr->frame_control) &&
+	    !is_multicast_ether_addr(hdr->addr1) &&
+	    (ieee80211_is_mgmt(hdr->frame_control) ||
+	     ieee80211_is_data(hdr->frame_control))) {
+		struct ieee80211_sta *sta;
+		unsigned int link_id;
+
+		rcu_read_lock();
+		sta = ieee80211_find_sta_by_link_addrs(data->hw, hdr->addr2,
+						       hdr->addr1, &link_id);
+		if (sta) {
+			struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+
+			if (ieee80211_has_pm(hdr->frame_control))
+				sp->active_links_rx &= ~BIT(link_id);
+			else
+				sp->active_links_rx |= BIT(link_id);
+
+			rx_status->link_valid = true;
+			rx_status->link_id = link_id;
+		}
+		rcu_read_unlock();
+	}
+
+	memcpy(IEEE80211_SKB_RXCB(skb), rx_status, sizeof(*rx_status));
+
+	mac80211_hwsim_add_vendor_rtap(skb);
+
+	if (data->nan.device_vif)
+		mac80211_hwsim_nan_rx(data->hw, skb);
+
+	data->rx_pkts++;
+	data->rx_bytes += skb->len;
+	ieee80211_rx_irqsafe(data->hw, skb);
+}
+
+static bool mac80211_hwsim_tx_frame_no_nl(struct ieee80211_hw *hw,
+					  struct sk_buff *skb,
+					  struct ieee80211_channel *chan)
+{
+	struct mac80211_hwsim_data *data = hw->priv, *data2;
+	bool ack = false;
+	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *) skb->data;
+	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
+	struct ieee80211_rx_status rx_status;
+	u64 sim_tsf = mac80211_hwsim_get_sim_tsf();
+
+	mac80211_hwsim_write_tsf(data, skb, sim_tsf);
+
+	mac80211_hwsim_monitor_rx(hw, skb, chan);
+
+	memset(&rx_status, 0, sizeof(rx_status));
+	rx_status.flag |= RX_FLAG_MACTIME_START;
+	rx_status.freq = chan->center_freq;
+	rx_status.freq_offset = chan->freq_offset ? 1 : 0;
+	rx_status.band = chan->band;
+	if (info->control.rates[0].flags & IEEE80211_TX_RC_VHT_MCS) {
+		rx_status.rate_idx =
+			ieee80211_rate_get_vht_mcs(&info->control.rates[0]);
+		rx_status.nss =
+			ieee80211_rate_get_vht_nss(&info->control.rates[0]);
+		rx_status.encoding = RX_ENC_VHT;
+	} else {
+		rx_status.rate_idx = info->control.rates[0].idx;
+		if (info->control.rates[0].flags & IEEE80211_TX_RC_MCS)
+			rx_status.encoding = RX_ENC_HT;
+	}
+	if (info->control.rates[0].flags & IEEE80211_TX_RC_40_MHZ_WIDTH)
+		rx_status.bw = RATE_INFO_BW_40;
+	else if (info->control.rates[0].flags & IEEE80211_TX_RC_80_MHZ_WIDTH)
+		rx_status.bw = RATE_INFO_BW_80;
+	else if (info->control.rates[0].flags & IEEE80211_TX_RC_160_MHZ_WIDTH)
+		rx_status.bw = RATE_INFO_BW_160;
+	else
+		rx_status.bw = RATE_INFO_BW_20;
+	if (info->control.rates[0].flags & IEEE80211_TX_RC_SHORT_GI)
+		rx_status.enc_flags |= RX_ENC_FLAG_SHORT_GI;
+	/* TODO: simulate optional packet loss */
+	rx_status.signal = data->rx_rssi;
+	if (info->control.vif)
+		rx_status.signal += info->control.vif->bss_conf.txpower;
+
+	if (data->ps != PS_DISABLED)
+		hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
+
+	/* release the skb's source info */
+	skb_orphan(skb);
+	skb_dst_drop(skb);
+	skb->mark = 0;
+	skb_ext_reset(skb);
+	nf_reset_ct(skb);
+
+	if (ieee80211_is_beacon(hdr->frame_control) ||
+	    ieee80211_is_probe_resp(hdr->frame_control))
+		rx_status.boottime_ns = ktime_get_boottime_ns();
+
+	/* Copy skb to all enabled radios that are on the current frequency */
+	spin_lock(&hwsim_radio_lock);
+	list_for_each_entry(data2, &hwsim_radios, list) {
+		struct sk_buff *nskb;
+		struct tx_iter_data tx_iter_data = {
+			.receive = false,
+			.hw = data2->hw,
+			.channel = chan,
+			.rx_status = &rx_status,
+		};
+
+		if (data == data2)
+			continue;
+
+		if (!data2->started || (data2->idle && !data2->tmp_chan) ||
+		    !hwsim_ps_rx_ok(data2, skb))
+			continue;
+
+		if (!(data->group & data2->group))
+			continue;
+
+		if (data->netgroup != data2->netgroup)
+			continue;
+
+		/*
+		 * Set mactime early since NAN RX filtering relies on it
+		 * for slot calculation
+		 */
+		rx_status.mactime = sim_tsf + data2->tsf_offset;
+
+		if (!hwsim_chans_compat(chan, data2->tmp_chan) &&
+		    !hwsim_chans_compat(chan, data2->channel)) {
+			ieee80211_iterate_active_interfaces_atomic(
+				data2->hw, IEEE80211_IFACE_ITER_NORMAL,
+				mac80211_hwsim_tx_iter, &tx_iter_data);
+			if (!tx_iter_data.receive)
+				continue;
+		}
+
+		/*
+		 * reserve some space for our vendor and the normal
+		 * radiotap header, since we're copying anyway
+		 */
+		if (skb->len < PAGE_SIZE && paged_rx) {
+			struct page *page = alloc_page(GFP_ATOMIC);
+
+			if (!page)
+				continue;
+
+			nskb = dev_alloc_skb(128);
+			if (!nskb) {
+				__free_page(page);
+				continue;
+			}
+
+			memcpy(page_address(page), skb->data, skb->len);
+			skb_add_rx_frag(nskb, 0, page, 0, skb->len, skb->len);
+		} else {
+			nskb = skb_copy(skb, GFP_ATOMIC);
+			if (!nskb)
+				continue;
+		}
+
+		if (mac80211_hwsim_addr_match(data2, hdr->addr1))
+			ack = true;
+
+		mac80211_hwsim_rx(data2, &rx_status, nskb);
+	}
+	spin_unlock(&hwsim_radio_lock);
+
+	return ack;
+}
+
+static struct ieee80211_bss_conf *
+mac80211_hwsim_select_tx_link(struct mac80211_hwsim_data *data,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_sta *sta,
+			      struct ieee80211_hdr *hdr,
+			      struct ieee80211_link_sta **link_sta)
+{
+	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+	int i;
+
+	if (!ieee80211_vif_is_mld(vif))
+		return &vif->bss_conf;
+
+	WARN_ON(is_multicast_ether_addr(hdr->addr1));
+
+	if (WARN_ON_ONCE(!sta || !sta->valid_links))
+		return &vif->bss_conf;
+
+	for (i = 0; i < ARRAY_SIZE(vif->link_conf); i++) {
+		struct ieee80211_bss_conf *bss_conf;
+		unsigned int link_id;
+
+		/* round-robin the available link IDs */
+		link_id = (sp->last_link + i + 1) % ARRAY_SIZE(vif->link_conf);
+
+		if (!(vif->active_links & BIT(link_id)))
+			continue;
+
+		if (!(sp->active_links_rx & BIT(link_id)))
+			continue;
+
+		*link_sta = rcu_dereference(sta->link[link_id]);
+		if (!*link_sta)
+			continue;
+
+		bss_conf = rcu_dereference(vif->link_conf[link_id]);
+		if (WARN_ON_ONCE(!bss_conf))
+			continue;
+
+		/* can happen while switching links */
+		if (!rcu_access_pointer(bss_conf->chanctx_conf))
+			continue;
+
+		sp->last_link = link_id;
+		return bss_conf;
+	}
+
+	return NULL;
+}
+
+static int mac80211_hwsim_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_sta *sta,
+				  struct ieee80211_key_conf *key)
+{
+	switch (key->cipher) {
+	case WLAN_CIPHER_SUITE_CCMP:
+	case WLAN_CIPHER_SUITE_CCMP_256:
+	case WLAN_CIPHER_SUITE_GCMP:
+	case WLAN_CIPHER_SUITE_GCMP_256:
+		break;
+	default:
+		return 1;
+	}
+
+	key->flags |= IEEE80211_KEY_FLAG_RESERVE_TAILROOM;
+	return 0;
+}
+
+static void mac80211_hwsim_tx(struct ieee80211_hw *hw,
+			      struct ieee80211_tx_control *control,
+			      struct sk_buff *skb)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	struct ieee80211_tx_info *txi = IEEE80211_SKB_CB(skb);
+	struct ieee80211_hdr *hdr = (void *)skb->data;
+	struct ieee80211_chanctx_conf *chanctx_conf;
+	struct ieee80211_channel *channel;
+	struct ieee80211_vif *vif = txi->control.vif;
+	bool ack, unicast_data;
+	enum nl80211_chan_width confbw = NL80211_CHAN_WIDTH_20_NOHT;
+	u32 _portid, i;
+
+	if (WARN_ON(skb->len < 10)) {
+		/* Should not happen; just a sanity check for addr1 use */
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+
+	unicast_data = is_unicast_ether_addr(hdr->addr1) &&
+		       ieee80211_is_data(hdr->frame_control);
+
+	if (unicast_data && ieee80211_encrypt_tx_skb(skb) < 0) {
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+	/* re-assign hdr since skb data may have shifted after encryption */
+	hdr = (void *)skb->data;
+
+	if (vif && !data->tmp_chan &&
+	    (vif->type == NL80211_IFTYPE_NAN ||
+	     vif->type == NL80211_IFTYPE_NAN_DATA)) {
+		struct cfg80211_chan_def chandef;
+
+		mac80211_hwsim_nan_get_tx_chandef(hw, &chandef);
+		if (WARN_ON(!chandef.chan)) {
+			/* No valid channel in current slot, drop frame */
+			ieee80211_free_txskb(hw, skb);
+			return;
+		}
+		channel = chandef.chan;
+		confbw = chandef.width;
+	} else if (!data->use_chanctx) {
+		channel = data->channel;
+		confbw = data->bw;
+	} else if (txi->hw_queue == 4) {
+		channel = data->tmp_chan;
+	} else {
+		u8 link = u32_get_bits(IEEE80211_SKB_CB(skb)->control.flags,
+				       IEEE80211_TX_CTRL_MLO_LINK);
+		struct ieee80211_link_sta *link_sta = NULL;
+		struct ieee80211_sta *sta = control->sta;
+		struct ieee80211_bss_conf *bss_conf;
+
+		/* This can happen in case of monitor injection */
+		if (!vif) {
+			ieee80211_free_txskb(hw, skb);
+			return;
+		}
+
+		if (link != IEEE80211_LINK_UNSPECIFIED) {
+			bss_conf = rcu_dereference(vif->link_conf[link]);
+			if (sta)
+				link_sta = rcu_dereference(sta->link[link]);
+		} else {
+			bss_conf = mac80211_hwsim_select_tx_link(data, vif, sta,
+								 hdr, &link_sta);
+		}
+
+		if (unlikely(!bss_conf)) {
+			/* if it's an MLO STA, it might have deactivated all
+			 * links temporarily - but we don't handle real PS in
+			 * this code yet, so just drop the frame in that case
+			 */
+			WARN(link != IEEE80211_LINK_UNSPECIFIED || !sta || !sta->mlo,
+			     "link:%d, sta:%pM, sta->mlo:%d\n",
+			     link, sta ? sta->addr : NULL, sta ? sta->mlo : -1);
+			ieee80211_free_txskb(hw, skb);
+			return;
+		}
+
+		/* Do address translations only between shared links. It is
+		 * possible that while an non-AP MLD station and an AP MLD
+		 * station have shared links, the frame is intended to be sent
+		 * on a link which is not shared (for example when sending a
+		 * probe response).
+		 */
+		if (sta && sta->mlo && link_sta) {
+			/* address translation to link addresses on TX */
+			ether_addr_copy(hdr->addr1, link_sta->addr);
+			ether_addr_copy(hdr->addr2, bss_conf->addr);
+			/* translate A3 only if it's the BSSID */
+			if (!ieee80211_has_tods(hdr->frame_control) &&
+			    !ieee80211_has_fromds(hdr->frame_control)) {
+				if (ether_addr_equal(hdr->addr3, sta->addr))
+					ether_addr_copy(hdr->addr3, link_sta->addr);
+				else if (ether_addr_equal(hdr->addr3, vif->addr))
+					ether_addr_copy(hdr->addr3, bss_conf->addr);
+			}
+			/* no need to look at A4, if present it's SA */
+		}
+
+		chanctx_conf = rcu_dereference(bss_conf->chanctx_conf);
+		if (chanctx_conf) {
+			channel = chanctx_conf->def.chan;
+			confbw = chanctx_conf->def.width;
+		} else {
+			channel = NULL;
+		}
+	}
+
+	if (!unicast_data && ieee80211_encrypt_tx_skb(skb) < 0) {
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+	/* re-assign hdr since skb data may have shifted after encryption */
+	hdr = (void *)skb->data;
+
+	if (WARN(!channel, "TX w/o channel - queue = %d\n", txi->hw_queue)) {
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+
+	if (data->idle && !data->tmp_chan) {
+		wiphy_dbg(hw->wiphy, "Trying to TX when idle - reject\n");
+		ieee80211_free_txskb(hw, skb);
+		return;
+	}
+
+	if (vif)
+		hwsim_check_magic(vif);
+	if (control->sta)
+		hwsim_check_sta_magic(control->sta);
+
+	if (ieee80211_hw_check(hw, SUPPORTS_RC_TABLE))
+		ieee80211_get_tx_rates(vif, control->sta, skb,
+				       txi->control.rates,
+				       ARRAY_SIZE(txi->control.rates));
+
+	for (i = 0; i < ARRAY_SIZE(txi->control.rates); i++) {
+		u16 rflags = txi->control.rates[i].flags;
+		/* initialize to data->bw for 5/10 MHz handling */
+		enum nl80211_chan_width bw = data->bw;
+
+		if (txi->control.rates[i].idx == -1)
+			break;
+
+		if (rflags & IEEE80211_TX_RC_40_MHZ_WIDTH)
+			bw = NL80211_CHAN_WIDTH_40;
+		else if (rflags & IEEE80211_TX_RC_80_MHZ_WIDTH)
+			bw = NL80211_CHAN_WIDTH_80;
+		else if (rflags & IEEE80211_TX_RC_160_MHZ_WIDTH)
+			bw = NL80211_CHAN_WIDTH_160;
+
+		if (WARN_ON(hwsim_get_chanwidth(bw) > hwsim_get_chanwidth(confbw)))
+			return;
+	}
+
+	/* wmediumd mode check */
+	_portid = READ_ONCE(data->wmediumd);
+
+	if (_portid || hwsim_virtio_enabled)
+		return mac80211_hwsim_tx_frame_nl(hw, skb, _portid, channel);
+
+	/* NO wmediumd detected, perfect medium simulation */
+	data->tx_pkts++;
+	data->tx_bytes += skb->len;
+	ack = mac80211_hwsim_tx_frame_no_nl(hw, skb, channel);
+
+	if (ack && skb->len >= 16)
+		mac80211_hwsim_monitor_ack(channel, hdr->addr2);
+
+	ieee80211_tx_info_clear_status(txi);
+
+	/* frame was transmitted at most favorable rate at first attempt */
+	txi->control.rates[0].count = 1;
+	txi->control.rates[1].idx = -1;
+
+	if (!(txi->flags & IEEE80211_TX_CTL_NO_ACK) && ack)
+		txi->flags |= IEEE80211_TX_STAT_ACK;
+	ieee80211_tx_status_irqsafe(hw, skb);
+}
+
+void ieee80211_hwsim_wake_tx_queue(struct ieee80211_hw *hw,
+				   struct ieee80211_txq *txq)
+{
+	struct ieee80211_tx_control control = {
+		.sta = txq->sta,
+	};
+	struct sk_buff *skb;
+
+	if ((txq->vif->type == NL80211_IFTYPE_NAN ||
+	     txq->vif->type == NL80211_IFTYPE_NAN_DATA) &&
+	    !mac80211_hwsim_nan_txq_transmitting(hw, txq))
+		return;
+
+	while ((skb = ieee80211_tx_dequeue(hw, txq)))
+		mac80211_hwsim_tx(hw, &control, skb);
+}
+
+static int mac80211_hwsim_start(struct ieee80211_hw *hw)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	wiphy_dbg(hw->wiphy, "%s\n", __func__);
+	data->started = true;
+	return 0;
+}
+
+
+static void mac80211_hwsim_stop(struct ieee80211_hw *hw, bool suspend)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	int i;
+
+	data->started = false;
+
+	for (i = 0; i < ARRAY_SIZE(data->link_data); i++)
+		hrtimer_cancel(&data->link_data[i].beacon_timer);
+
+	while (!skb_queue_empty(&data->pending))
+		ieee80211_free_txskb(hw, skb_dequeue(&data->pending));
+
+	wiphy_dbg(hw->wiphy, "%s\n", __func__);
+}
+
+
+static int mac80211_hwsim_add_interface(struct ieee80211_hw *hw,
+					struct ieee80211_vif *vif)
+{
+	wiphy_dbg(hw->wiphy, "%s (type=%d mac_addr=%pM)\n",
+		  __func__, ieee80211_vif_type_p2p(vif),
+		  vif->addr);
+	hwsim_set_magic(vif);
+
+	if (vif->type != NL80211_IFTYPE_MONITOR)
+		mac80211_hwsim_config_mac_nl(hw, vif->addr, true);
+
+	vif->cab_queue = 0;
+	vif->hw_queue[IEEE80211_AC_VO] = 0;
+	vif->hw_queue[IEEE80211_AC_VI] = 1;
+	vif->hw_queue[IEEE80211_AC_BE] = 2;
+	vif->hw_queue[IEEE80211_AC_BK] = 3;
+
+	return 0;
+}
+
+#ifdef CONFIG_MAC80211_DEBUGFS
+static void
+mac80211_hwsim_link_add_debugfs(struct ieee80211_hw *hw,
+				struct ieee80211_vif *vif,
+				struct ieee80211_bss_conf *link_conf,
+				struct dentry *dir)
+{
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+
+	debugfs_create_u32("skip_beacons", 0600, dir,
+			   &vp->skip_beacons[link_conf->link_id]);
+}
+#endif
+
+static int mac80211_hwsim_change_interface(struct ieee80211_hw *hw,
+					   struct ieee80211_vif *vif,
+					   enum nl80211_iftype newtype,
+					   bool newp2p)
+{
+	newtype = ieee80211_iftype_p2p(newtype, newp2p);
+	wiphy_dbg(hw->wiphy,
+		  "%s (old type=%d, new type=%d, mac_addr=%pM)\n",
+		  __func__, ieee80211_vif_type_p2p(vif),
+		    newtype, vif->addr);
+	hwsim_check_magic(vif);
+
+	/*
+	 * interface may change from non-AP to AP in
+	 * which case this needs to be set up again
+	 */
+	vif->cab_queue = 0;
+
+	return 0;
+}
+
+static void mac80211_hwsim_remove_interface(
+	struct ieee80211_hw *hw, struct ieee80211_vif *vif)
+{
+	wiphy_dbg(hw->wiphy, "%s (type=%d mac_addr=%pM)\n",
+		  __func__, ieee80211_vif_type_p2p(vif),
+		  vif->addr);
+	hwsim_check_magic(vif);
+	hwsim_clear_magic(vif);
+	if (vif->type != NL80211_IFTYPE_MONITOR)
+		mac80211_hwsim_config_mac_nl(hw, vif->addr, false);
+}
+
+void mac80211_hwsim_tx_frame(struct ieee80211_hw *hw,
+			     struct sk_buff *skb,
+			     struct ieee80211_channel *chan)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	u32 _portid = READ_ONCE(data->wmediumd);
+
+	if (ieee80211_hw_check(hw, SUPPORTS_RC_TABLE)) {
+		struct ieee80211_tx_info *txi = IEEE80211_SKB_CB(skb);
+		ieee80211_get_tx_rates(txi->control.vif, NULL, skb,
+				       txi->control.rates,
+				       ARRAY_SIZE(txi->control.rates));
+	}
+
+	if (_portid || hwsim_virtio_enabled)
+		return mac80211_hwsim_tx_frame_nl(hw, skb, _portid, chan);
+
+	data->tx_pkts++;
+	data->tx_bytes += skb->len;
+	mac80211_hwsim_tx_frame_no_nl(hw, skb, chan);
+	dev_kfree_skb(skb);
+}
+
+static void __mac80211_hwsim_beacon_tx(struct ieee80211_bss_conf *link_conf,
+				       struct mac80211_hwsim_data *data,
+				       struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct sk_buff *skb)
+{
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+	struct ieee80211_tx_info *info;
+
+	if (vp->skip_beacons[link_conf->link_id]) {
+		vp->skip_beacons[link_conf->link_id]--;
+		dev_kfree_skb(skb);
+		return;
+	}
+
+	info = IEEE80211_SKB_CB(skb);
+	if (ieee80211_hw_check(hw, SUPPORTS_RC_TABLE))
+		ieee80211_get_tx_rates(vif, NULL, skb,
+				       info->control.rates,
+				       ARRAY_SIZE(info->control.rates));
+
+	mac80211_hwsim_tx_frame(hw, skb,
+			rcu_dereference(link_conf->chanctx_conf)->def.chan);
+}
+
+static void mac80211_hwsim_beacon_tx(void *arg, u8 *mac,
+				     struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_link_data *link_data = arg;
+	u32 link_id = link_data->link_id;
+	struct ieee80211_bss_conf *link_conf, *tx_bss_conf;
+	struct mac80211_hwsim_data *data =
+		container_of(link_data, struct mac80211_hwsim_data,
+			     link_data[link_id]);
+	struct ieee80211_hw *hw = data->hw;
+	struct sk_buff *skb;
+
+	hwsim_check_magic(vif);
+
+	link_conf = rcu_dereference(vif->link_conf[link_id]);
+	if (!link_conf)
+		return;
+
+	if (vif->type != NL80211_IFTYPE_AP &&
+	    vif->type != NL80211_IFTYPE_MESH_POINT &&
+	    vif->type != NL80211_IFTYPE_ADHOC &&
+	    vif->type != NL80211_IFTYPE_OCB)
+		return;
+
+	tx_bss_conf = rcu_access_pointer(link_conf->tx_bss_conf);
+	if (tx_bss_conf && tx_bss_conf != link_conf)
+		return;
+
+	if (link_conf->ema_ap) {
+		struct ieee80211_ema_beacons *ema;
+		u8 i = 0;
+
+		ema = ieee80211_beacon_get_template_ema_list(hw, vif, link_id);
+		if (!ema || !ema->cnt)
+			return;
+
+		for (i = 0; i < ema->cnt; i++) {
+			__mac80211_hwsim_beacon_tx(link_conf, data, hw, vif,
+						   ema->bcn[i].skb);
+			ema->bcn[i].skb = NULL; /* Already freed */
+		}
+		ieee80211_beacon_free_ema_list(ema);
+	} else {
+		skb = ieee80211_beacon_get(hw, vif, link_id);
+		if (!skb)
+			return;
+
+		__mac80211_hwsim_beacon_tx(link_conf, data, hw, vif, skb);
+	}
+
+	while ((skb = ieee80211_get_buffered_bc(hw, vif)) != NULL) {
+		mac80211_hwsim_tx_frame(hw, skb,
+			rcu_dereference(link_conf->chanctx_conf)->def.chan);
+	}
+
+	if (link_conf->csa_active && ieee80211_beacon_cntdwn_is_complete(vif, link_id))
+		ieee80211_csa_finish(vif, link_id);
+
+	if (link_conf->color_change_active &&
+	    ieee80211_beacon_cntdwn_is_complete(vif, link_id))
+		ieee80211_color_change_finish(vif, link_id);
+}
+
+static enum hrtimer_restart
+mac80211_hwsim_beacon(struct hrtimer *timer)
+{
+	struct mac80211_hwsim_link_data *link_data =
+		container_of(timer, struct mac80211_hwsim_link_data, beacon_timer);
+	struct mac80211_hwsim_data *data =
+		container_of(link_data, struct mac80211_hwsim_data,
+			     link_data[link_data->link_id]);
+	struct ieee80211_hw *hw = data->hw;
+	u32 remainder;
+	u64 tsf_now;
+	u64 tbtt;
+
+	if (!data->started)
+		return HRTIMER_NORESTART;
+
+	ieee80211_iterate_active_interfaces_atomic(
+		hw, IEEE80211_IFACE_ITER_NORMAL,
+		mac80211_hwsim_beacon_tx, link_data);
+
+	/* TSF is the same for all VIFs, parameter is unused */
+	tsf_now = mac80211_hwsim_get_tsf(hw, NULL);
+
+	/* Wrap value to be after the next TBTT */
+	tbtt = tsf_now + link_data->beacon_int;
+
+	/* Round TBTT down to the correct time */
+	div_u64_rem(tbtt, link_data->beacon_int, &remainder);
+	tbtt = tbtt - remainder;
+
+	hrtimer_set_expires(&link_data->beacon_timer,
+			    mac80211_hwsim_tsf_to_boottime(data, tbtt));
+
+	return HRTIMER_RESTART;
+}
+
+static const char * const hwsim_chanwidths[] = {
+	[NL80211_CHAN_WIDTH_5] = "ht5",
+	[NL80211_CHAN_WIDTH_10] = "ht10",
+	[NL80211_CHAN_WIDTH_20_NOHT] = "noht",
+	[NL80211_CHAN_WIDTH_20] = "ht20",
+	[NL80211_CHAN_WIDTH_40] = "ht40",
+	[NL80211_CHAN_WIDTH_80] = "vht80",
+	[NL80211_CHAN_WIDTH_80P80] = "vht80p80",
+	[NL80211_CHAN_WIDTH_160] = "vht160",
+	[NL80211_CHAN_WIDTH_1] = "1MHz",
+	[NL80211_CHAN_WIDTH_2] = "2MHz",
+	[NL80211_CHAN_WIDTH_4] = "4MHz",
+	[NL80211_CHAN_WIDTH_8] = "8MHz",
+	[NL80211_CHAN_WIDTH_16] = "16MHz",
+	[NL80211_CHAN_WIDTH_320] = "eht320",
+};
+
+static int mac80211_hwsim_config(struct ieee80211_hw *hw, int radio_idx,
+				 u32 changed)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	struct ieee80211_conf *conf = &hw->conf;
+	static const char *smps_modes[IEEE80211_SMPS_NUM_MODES] = {
+		[IEEE80211_SMPS_AUTOMATIC] = "auto",
+		[IEEE80211_SMPS_OFF] = "off",
+		[IEEE80211_SMPS_STATIC] = "static",
+		[IEEE80211_SMPS_DYNAMIC] = "dynamic",
+	};
+	int idx;
+
+	if (conf->chandef.chan)
+		wiphy_dbg(hw->wiphy,
+			  "%s (freq=%d(%d - %d)/%s idle=%d ps=%d smps=%s)\n",
+			  __func__,
+			  conf->chandef.chan->center_freq,
+			  conf->chandef.center_freq1,
+			  conf->chandef.center_freq2,
+			  hwsim_chanwidths[conf->chandef.width],
+			  !!(conf->flags & IEEE80211_CONF_IDLE),
+			  !!(conf->flags & IEEE80211_CONF_PS),
+			  smps_modes[conf->smps_mode]);
+	else
+		wiphy_dbg(hw->wiphy,
+			  "%s (freq=0 idle=%d ps=%d smps=%s)\n",
+			  __func__,
+			  !!(conf->flags & IEEE80211_CONF_IDLE),
+			  !!(conf->flags & IEEE80211_CONF_PS),
+			  smps_modes[conf->smps_mode]);
+
+	data->idle = !!(conf->flags & IEEE80211_CONF_IDLE);
+
+	WARN_ON(conf->chandef.chan && data->use_chanctx);
+
+	mutex_lock(&data->mutex);
+	if (data->scanning && conf->chandef.chan) {
+		for (idx = 0; idx < ARRAY_SIZE(data->survey_data); idx++) {
+			if (data->survey_data[idx].channel == data->channel) {
+				data->survey_data[idx].start =
+					data->survey_data[idx].next_start;
+				data->survey_data[idx].end = jiffies;
+				break;
+			}
+		}
+
+		data->channel = conf->chandef.chan;
+		data->bw = conf->chandef.width;
+
+		for (idx = 0; idx < ARRAY_SIZE(data->survey_data); idx++) {
+			if (data->survey_data[idx].channel &&
+			    data->survey_data[idx].channel != data->channel)
+				continue;
+			data->survey_data[idx].channel = data->channel;
+			data->survey_data[idx].next_start = jiffies;
+			break;
+		}
+	} else {
+		data->channel = conf->chandef.chan;
+		data->bw = conf->chandef.width;
+	}
+	mutex_unlock(&data->mutex);
+
+	for (idx = 0; idx < ARRAY_SIZE(data->link_data); idx++) {
+		struct mac80211_hwsim_link_data *link_data =
+			&data->link_data[idx];
+
+		if (!data->started || !link_data->beacon_int) {
+			hrtimer_cancel(&link_data->beacon_timer);
+		} else if (!hrtimer_active(&link_data->beacon_timer)) {
+			u64 tsf = mac80211_hwsim_get_tsf(hw, NULL);
+			u32 bcn_int = link_data->beacon_int;
+			u64 until_tbtt = bcn_int - do_div(tsf, bcn_int);
+
+			hrtimer_start(&link_data->beacon_timer,
+				      ns_to_ktime(until_tbtt * NSEC_PER_USEC),
+				      HRTIMER_MODE_REL_SOFT);
+		}
+	}
+
+	return 0;
+}
+
+
+static void mac80211_hwsim_configure_filter(struct ieee80211_hw *hw,
+					    unsigned int changed_flags,
+					    unsigned int *total_flags,u64 multicast)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+
+	wiphy_dbg(hw->wiphy, "%s\n", __func__);
+
+	data->rx_filter = 0;
+	if (*total_flags & FIF_ALLMULTI)
+		data->rx_filter |= FIF_ALLMULTI;
+	if (*total_flags & FIF_MCAST_ACTION)
+		data->rx_filter |= FIF_MCAST_ACTION;
+
+	*total_flags = data->rx_filter;
+}
+
+static void mac80211_hwsim_bcn_en_iter(void *data, u8 *mac,
+				       struct ieee80211_vif *vif)
+{
+	unsigned int *count = data;
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+
+	if (vp->bcn_en)
+		(*count)++;
+}
+
+static void mac80211_hwsim_vif_info_changed(struct ieee80211_hw *hw,
+					    struct ieee80211_vif *vif,
+					    u64 changed)
+{
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+
+	hwsim_check_magic(vif);
+
+	wiphy_dbg(hw->wiphy, "%s(changed=0x%llx vif->addr=%pM)\n",
+		  __func__, changed, vif->addr);
+
+	if (changed & BSS_CHANGED_ASSOC) {
+		wiphy_dbg(hw->wiphy, "  ASSOC: assoc=%d aid=%d\n",
+			  vif->cfg.assoc, vif->cfg.aid);
+		vp->assoc = vif->cfg.assoc;
+		vp->aid = vif->cfg.aid;
+	}
+
+	if (changed & BSS_CHANGED_NAN_LOCAL_SCHED)
+		mac80211_hwsim_nan_local_sched_changed(hw, vif);
+
+	if (vif->type == NL80211_IFTYPE_STATION &&
+	    changed & (BSS_CHANGED_MLD_VALID_LINKS | BSS_CHANGED_MLD_TTLM)) {
+		u16 usable_links = ieee80211_vif_usable_links(vif);
+
+		if (vif->active_links != usable_links)
+			ieee80211_set_active_links_async(vif, usable_links);
+	}
+}
+
+static void mac80211_hwsim_link_info_changed(struct ieee80211_hw *hw,
+					     struct ieee80211_vif *vif,
+					     struct ieee80211_bss_conf *info,
+					     u64 changed)
+{
+	struct hwsim_vif_priv *vp = (void *)vif->drv_priv;
+	struct mac80211_hwsim_data *data = hw->priv;
+	unsigned int link_id = info->link_id;
+	struct mac80211_hwsim_link_data *link_data = &data->link_data[link_id];
+
+	hwsim_check_magic(vif);
+
+	wiphy_dbg(hw->wiphy, "%s(changed=0x%llx vif->addr=%pM, link id %u)\n",
+		  __func__, (unsigned long long)changed, vif->addr, link_id);
+
+	if (changed & BSS_CHANGED_BSSID) {
+		wiphy_dbg(hw->wiphy, "%s: BSSID changed: %pM\n",
+			  __func__, info->bssid);
+		memcpy(vp->bssid, info->bssid, ETH_ALEN);
+	}
+
+	if (changed & BSS_CHANGED_BEACON_ENABLED) {
+		wiphy_dbg(hw->wiphy, "  BCN EN: %d (BI=%u)\n",
+			  info->enable_beacon, info->beacon_int);
+		vp->bcn_en = info->enable_beacon;
+		if (data->started &&
+		    !hrtimer_active(&link_data->beacon_timer) &&
+		    info->enable_beacon) {
+			u64 tsf, until_tbtt;
+			u32 bcn_int;
+			link_data->beacon_int = info->beacon_int * 1024;
+			tsf = mac80211_hwsim_get_tsf(hw, vif);
+			bcn_int = link_data->beacon_int;
+			until_tbtt = bcn_int - do_div(tsf, bcn_int);
+
+			hrtimer_start(&link_data->beacon_timer,
+				      ns_to_ktime(until_tbtt * NSEC_PER_USEC),
+				      HRTIMER_MODE_REL_SOFT);
+		} else if (!info->enable_beacon) {
+			unsigned int count = 0;
+			ieee80211_iterate_active_interfaces_atomic(
+				data->hw, IEEE80211_IFACE_ITER_NORMAL,
+				mac80211_hwsim_bcn_en_iter, &count);
+			wiphy_dbg(hw->wiphy, "  beaconing vifs remaining: %u",
+				  count);
+			if (count == 0) {
+				hrtimer_cancel(&link_data->beacon_timer);
+				link_data->beacon_int = 0;
+			}
+		}
+	}
+
+	if (changed & BSS_CHANGED_ERP_CTS_PROT) {
+		wiphy_dbg(hw->wiphy, "  ERP_CTS_PROT: %d\n",
+			  info->use_cts_prot);
+	}
+
+	if (changed & BSS_CHANGED_ERP_PREAMBLE) {
+		wiphy_dbg(hw->wiphy, "  ERP_PREAMBLE: %d\n",
+			  info->use_short_preamble);
+	}
+
+	if (changed & BSS_CHANGED_ERP_SLOT) {
+		wiphy_dbg(hw->wiphy, "  ERP_SLOT: %d\n", info->use_short_slot);
+	}
+
+	if (changed & BSS_CHANGED_HT) {
+		wiphy_dbg(hw->wiphy, "  HT: op_mode=0x%x\n",
+			  info->ht_operation_mode);
+	}
+
+	if (changed & BSS_CHANGED_BASIC_RATES) {
+		wiphy_dbg(hw->wiphy, "  BASIC_RATES: 0x%llx\n",
+			  (unsigned long long) info->basic_rates);
+	}
+
+	if (changed & BSS_CHANGED_TXPOWER)
+		wiphy_dbg(hw->wiphy, "  TX Power: %d dBm\n", info->txpower);
+}
+
+static void
+mac80211_hwsim_sta_rc_update(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct ieee80211_link_sta *link_sta,
+			     u32 changed)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	struct ieee80211_sta *sta = link_sta->sta;
+	u32 bw = U32_MAX;
+	int link_id;
+
+	if (vif->type == NL80211_IFTYPE_NAN ||
+	    vif->type == NL80211_IFTYPE_NAN_DATA)
+		return;
+
+	rcu_read_lock();
+	for (link_id = 0;
+	     link_id < ARRAY_SIZE(vif->link_conf);
+	     link_id++) {
+		enum nl80211_chan_width confbw = NL80211_CHAN_WIDTH_20_NOHT;
+		struct ieee80211_bss_conf *vif_conf;
+
+		link_sta = rcu_dereference(sta->link[link_id]);
+
+		if (!link_sta)
+			continue;
+
+		switch (link_sta->bandwidth) {
+#define C(_bw) case IEEE80211_STA_RX_BW_##_bw: bw = _bw; break
+		C(20);
+		C(40);
+		C(80);
+		C(160);
+		C(320);
+#undef C
+		}
+
+		if (!data->use_chanctx) {
+			confbw = data->bw;
+		} else {
+			struct ieee80211_chanctx_conf *chanctx_conf;
+
+			vif_conf = rcu_dereference(vif->link_conf[link_id]);
+			if (WARN_ON(!vif_conf))
+				continue;
+
+			chanctx_conf = rcu_dereference(vif_conf->chanctx_conf);
+
+			if (!WARN_ON(!chanctx_conf))
+				confbw = chanctx_conf->def.width;
+		}
+
+		WARN(bw > hwsim_get_chanwidth(confbw),
+		     "intf %pM [link=%d]: bad STA %pM bandwidth %d MHz (%d) > channel config %d MHz (%d)\n",
+		     vif->addr, link_id, sta->addr, bw, sta->deflink.bandwidth,
+		     hwsim_get_chanwidth(data->bw), data->bw);
+
+
+	}
+	rcu_read_unlock();
+
+
+}
+
+static int mac80211_hwsim_sta_add(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_sta *sta)
+{
+	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+
+	hwsim_check_magic(vif);
+	hwsim_set_sta_magic(sta);
+	mac80211_hwsim_sta_rc_update(hw, vif, &sta->deflink, 0);
+
+	if (sta->valid_links) {
+		WARN(hweight16(sta->valid_links) > 1,
+		     "expect to add STA with single link, have 0x%x\n",
+		     sta->valid_links);
+		sp->active_links_rx = sta->valid_links;
+	}
+
+	spin_lock_init(&sp->nan_sched.lock);
+
+	return 0;
+}
+
+static int mac80211_hwsim_sta_remove(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct ieee80211_sta *sta)
+{
+	hwsim_check_magic(vif);
+	hwsim_clear_sta_magic(sta);
+
+	return 0;
+}
+
+static int mac80211_hwsim_sta_state(struct ieee80211_hw *hw,
+				    struct ieee80211_vif *vif,
+				    struct ieee80211_sta *sta,
+				    enum ieee80211_sta_state old_state,
+				    enum ieee80211_sta_state new_state)
+{
+	if (new_state == IEEE80211_STA_NOTEXIST)
+		return mac80211_hwsim_sta_remove(hw, vif, sta);
+
+	if (old_state == IEEE80211_STA_NOTEXIST)
+		return mac80211_hwsim_sta_add(hw, vif, sta);
+
+	/*
+	 * in an MLO connection, when client is authorized
+	 * (AP station marked as such), enable all links
+	 */
+	if (ieee80211_vif_is_mld(vif) &&
+	    vif->type == NL80211_IFTYPE_STATION &&
+	    new_state == IEEE80211_STA_AUTHORIZED && !sta->tdls)
+		ieee80211_set_active_links_async(vif,
+						 ieee80211_vif_usable_links(vif));
+
+	return 0;
+}
+
+static void mac80211_hwsim_sta_notify(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      enum sta_notify_cmd cmd,
+				      struct ieee80211_sta *sta)
+{
+	hwsim_check_magic(vif);
+
+	switch (cmd) {
+	case STA_NOTIFY_SLEEP:
+	case STA_NOTIFY_AWAKE:
+		/* TODO: make good use of these flags */
+		break;
+	default:
+		WARN(1, "Invalid sta notify: %d\n", cmd);
+		break;
+	}
+}
+
+static int mac80211_hwsim_set_tim(struct ieee80211_hw *hw,
+				  struct ieee80211_sta *sta,
+				  bool set)
+{
+	hwsim_check_sta_magic(sta);
+	return 0;
+}
+
+static int mac80211_hwsim_conf_tx(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  unsigned int link_id, u16 queue,
+				  const struct ieee80211_tx_queue_params *params)
+{
+	wiphy_dbg(hw->wiphy,
+		  "%s (queue=%d txop=%d cw_min=%d cw_max=%d aifs=%d)\n",
+		  __func__, queue,
+		  params->txop, params->cw_min,
+		  params->cw_max, params->aifs);
+	return 0;
+}
+
+static int mac80211_hwsim_get_survey(struct ieee80211_hw *hw, int idx,
+				     struct survey_info *survey)
+{
+	struct mac80211_hwsim_data *hwsim = hw->priv;
+
+	if (idx < 0 || idx >= ARRAY_SIZE(hwsim->survey_data))
+		return -ENOENT;
+
+	mutex_lock(&hwsim->mutex);
+	survey->channel = hwsim->survey_data[idx].channel;
+	if (!survey->channel) {
+		mutex_unlock(&hwsim->mutex);
+		return -ENOENT;
+	}
+
+	/*
+	 * Magically conjured dummy values --- this is only ok for simulated hardware.
+	 *
+	 * A real driver which cannot determine real values noise MUST NOT
+	 * report any, especially not a magically conjured ones :-)
+	 */
+	survey->filled = SURVEY_INFO_NOISE_DBM |
+			 SURVEY_INFO_TIME |
+			 SURVEY_INFO_TIME_BUSY;
+	survey->noise = -92;
+	survey->time =
+		jiffies_to_msecs(hwsim->survey_data[idx].end -
+				 hwsim->survey_data[idx].start);
+	/* report 12.5% of channel time is used */
+	survey->time_busy = survey->time/8;
+	mutex_unlock(&hwsim->mutex);
+
+	return 0;
+}
+
+static enum ieee80211_neg_ttlm_res
+mac80211_hwsim_can_neg_ttlm(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
+			    struct ieee80211_neg_ttlm *neg_ttlm)
+{
+	u32 i;
+
+	/* For testing purposes, accept if all TIDs are mapped to the same links
+	 * set, otherwise reject.
+	 */
+	for (i = 0; i < IEEE80211_TTLM_NUM_TIDS; i++) {
+		if (neg_ttlm->downlink[i] != neg_ttlm->uplink[i] ||
+		    neg_ttlm->downlink[i] != neg_ttlm->downlink[0])
+			return NEG_TTLM_RES_REJECT;
+	}
+
+	return NEG_TTLM_RES_ACCEPT;
+}
+
+#ifdef CONFIG_NL80211_TESTMODE
+/*
+ * This section contains example code for using netlink
+ * attributes with the testmode command in nl80211.
+ */
+
+/* These enums need to be kept in sync with userspace */
+enum hwsim_testmode_attr {
+	__HWSIM_TM_ATTR_INVALID	= 0,
+	HWSIM_TM_ATTR_CMD	= 1,
+	HWSIM_TM_ATTR_PS	= 2,
+
+	/* keep last */
+	__HWSIM_TM_ATTR_AFTER_LAST,
+	HWSIM_TM_ATTR_MAX	= __HWSIM_TM_ATTR_AFTER_LAST - 1
+};
+
+enum hwsim_testmode_cmd {
+	HWSIM_TM_CMD_SET_PS		= 0,
+	HWSIM_TM_CMD_GET_PS		= 1,
+	HWSIM_TM_CMD_STOP_QUEUES	= 2,
+	HWSIM_TM_CMD_WAKE_QUEUES	= 3,
+};
+
+static const struct nla_policy hwsim_testmode_policy[HWSIM_TM_ATTR_MAX + 1] = {
+	[HWSIM_TM_ATTR_CMD] = { .type = NLA_U32 },
+	[HWSIM_TM_ATTR_PS] = { .type = NLA_U32 },
+};
+
+static int mac80211_hwsim_testmode_cmd(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       void *data, int len)
+{
+	struct mac80211_hwsim_data *hwsim = hw->priv;
+	struct nlattr *tb[HWSIM_TM_ATTR_MAX + 1];
+	struct sk_buff *skb;
+	int err, ps;
+
+	err = nla_parse_deprecated(tb, HWSIM_TM_ATTR_MAX, data, len,
+				   hwsim_testmode_policy, NULL);
+	if (err)
+		return err;
+
+	if (!tb[HWSIM_TM_ATTR_CMD])
+		return -EINVAL;
+
+	switch (nla_get_u32(tb[HWSIM_TM_ATTR_CMD])) {
+	case HWSIM_TM_CMD_SET_PS:
+		if (!tb[HWSIM_TM_ATTR_PS])
+			return -EINVAL;
+		ps = nla_get_u32(tb[HWSIM_TM_ATTR_PS]);
+		return hwsim_fops_ps_write(hwsim, ps);
+	case HWSIM_TM_CMD_GET_PS:
+		skb = cfg80211_testmode_alloc_reply_skb(hw->wiphy,
+						nla_total_size(sizeof(u32)));
+		if (!skb)
+			return -ENOMEM;
+		if (nla_put_u32(skb, HWSIM_TM_ATTR_PS, hwsim->ps))
+			goto nla_put_failure;
+		return cfg80211_testmode_reply(skb);
+	case HWSIM_TM_CMD_STOP_QUEUES:
+	case HWSIM_TM_CMD_WAKE_QUEUES:
+	default:
+		return -EOPNOTSUPP;
+	}
+
+ nla_put_failure:
+	kfree_skb(skb);
+	return -ENOBUFS;
+}
+#endif
+
+static int mac80211_hwsim_ampdu_action(struct ieee80211_hw *hw,
+				       struct ieee80211_vif *vif,
+				       struct ieee80211_ampdu_params *params)
+{
+	struct ieee80211_sta *sta = params->sta;
+	enum ieee80211_ampdu_mlme_action action = params->action;
+	u16 tid = params->tid;
+
+	switch (action) {
+	case IEEE80211_AMPDU_TX_START:
+		return IEEE80211_AMPDU_TX_START_IMMEDIATE;
+	case IEEE80211_AMPDU_TX_STOP_CONT:
+	case IEEE80211_AMPDU_TX_STOP_FLUSH:
+	case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
+		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
+		break;
+	case IEEE80211_AMPDU_TX_OPERATIONAL:
+		break;
+	case IEEE80211_AMPDU_RX_START:
+	case IEEE80211_AMPDU_RX_STOP:
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	return 0;
+}
+
+static void mac80211_hwsim_flush(struct ieee80211_hw *hw,
+				 struct ieee80211_vif *vif,
+				 u32 queues, bool drop)
+{
+	/* Not implemented, queues only on kernel side */
+}
+
+static void hw_scan_work(struct work_struct *work)
+{
+	struct mac80211_hwsim_data *hwsim =
+		container_of(work, struct mac80211_hwsim_data, hw_scan.work);
+	struct cfg80211_scan_request *req = hwsim->hw_scan_request;
+	int dwell, i;
+
+	mutex_lock(&hwsim->mutex);
+	if (hwsim->scan_chan_idx >= req->n_channels) {
+		struct cfg80211_scan_info info = {
+			.aborted = false,
+		};
+
+		wiphy_dbg(hwsim->hw->wiphy, "hw scan complete\n");
+		ieee80211_scan_completed(hwsim->hw, &info);
+		hwsim->hw_scan_request = NULL;
+		hwsim->hw_scan_vif = NULL;
+		hwsim->tmp_chan = NULL;
+		mutex_unlock(&hwsim->mutex);
+		mac80211_hwsim_config_mac_nl(hwsim->hw, hwsim->scan_addr,
+					     false);
+		return;
+	}
+
+	wiphy_dbg(hwsim->hw->wiphy, "hw scan %d MHz\n",
+		  req->channels[hwsim->scan_chan_idx]->center_freq);
+
+	hwsim->tmp_chan = req->channels[hwsim->scan_chan_idx];
+	if (hwsim->tmp_chan->flags & (IEEE80211_CHAN_NO_IR |
+				      IEEE80211_CHAN_RADAR) ||
+	    !req->n_ssids) {
+		dwell = 120;
+	} else {
+		dwell = 30;
+		/* send probes */
+		for (i = 0; i < req->n_ssids; i++) {
+			struct sk_buff *probe;
+			struct ieee80211_mgmt *mgmt;
+
+			probe = ieee80211_probereq_get(hwsim->hw,
+						       hwsim->scan_addr,
+						       req->ssids[i].ssid,
+						       req->ssids[i].ssid_len,
+						       req->ie_len);
+			if (!probe)
+				continue;
+
+			mgmt = (struct ieee80211_mgmt *) probe->data;
+			memcpy(mgmt->da, req->bssid, ETH_ALEN);
+			memcpy(mgmt->bssid, req->bssid, ETH_ALEN);
+
+			if (req->ie_len)
+				skb_put_data(probe, req->ie, req->ie_len);
+
+			rcu_read_lock();
+			if (!ieee80211_tx_prepare_skb(hwsim->hw,
+						      hwsim->hw_scan_vif,
+						      probe,
+						      hwsim->tmp_chan->band,
+						      NULL)) {
+				rcu_read_unlock();
+				continue;
+			}
+
+			local_bh_disable();
+			mac80211_hwsim_tx_frame(hwsim->hw, probe,
+						hwsim->tmp_chan);
+			rcu_read_unlock();
+			local_bh_enable();
+		}
+	}
+	ieee80211_queue_delayed_work(hwsim->hw, &hwsim->hw_scan,
+				     msecs_to_jiffies(dwell));
+	hwsim->survey_data[hwsim->scan_chan_idx].channel = hwsim->tmp_chan;
+	hwsim->survey_data[hwsim->scan_chan_idx].start = jiffies;
+	hwsim->survey_data[hwsim->scan_chan_idx].end =
+		jiffies + msecs_to_jiffies(dwell);
+	hwsim->scan_chan_idx++;
+	mutex_unlock(&hwsim->mutex);
+}
+
+static int mac80211_hwsim_hw_scan(struct ieee80211_hw *hw,
+				  struct ieee80211_vif *vif,
+				  struct ieee80211_scan_request *hw_req)
+{
+	struct mac80211_hwsim_data *hwsim = hw->priv;
+	struct cfg80211_scan_request *req = &hw_req->req;
+
+	mutex_lock(&hwsim->mutex);
+	if (WARN_ON(hwsim->tmp_chan || hwsim->hw_scan_request)) {
+		mutex_unlock(&hwsim->mutex);
+		return -EBUSY;
+	}
+	hwsim->hw_scan_request = req;
+	hwsim->hw_scan_vif = vif;
+	hwsim->scan_chan_idx = 0;
+	if (req->flags & NL80211_SCAN_FLAG_RANDOM_ADDR)
+		get_random_mask_addr(hwsim->scan_addr,
+				     hw_req->req.mac_addr,
+				     hw_req->req.mac_addr_mask);
+	else
+		memcpy(hwsim->scan_addr, vif->addr, ETH_ALEN);
+	memset(hwsim->survey_data, 0, sizeof(hwsim->survey_data));
+	mutex_unlock(&hwsim->mutex);
+
+	mac80211_hwsim_config_mac_nl(hw, hwsim->scan_addr, true);
+	wiphy_dbg(hw->wiphy, "hwsim hw_scan request\n");
+
+	ieee80211_queue_delayed_work(hwsim->hw, &hwsim->hw_scan, 0);
+
+	return 0;
+}
+
+static void mac80211_hwsim_cancel_hw_scan(struct ieee80211_hw *hw,
+					  struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_data *hwsim = hw->priv;
+	struct cfg80211_scan_info info = {
+		.aborted = true,
+	};
+
+	wiphy_dbg(hw->wiphy, "hwsim cancel_hw_scan\n");
+
+	cancel_delayed_work_sync(&hwsim->hw_scan);
+
+	mutex_lock(&hwsim->mutex);
+	ieee80211_scan_completed(hwsim->hw, &info);
+	hwsim->tmp_chan = NULL;
+	hwsim->hw_scan_request = NULL;
+	hwsim->hw_scan_vif = NULL;
+	mutex_unlock(&hwsim->mutex);
+}
+
+static void mac80211_hwsim_sw_scan(struct ieee80211_hw *hw,
+				   struct ieee80211_vif *vif,
+				   const u8 *mac_addr)
+{
+	struct mac80211_hwsim_data *hwsim = hw->priv;
+
+	mutex_lock(&hwsim->mutex);
+
+	if (hwsim->scanning) {
+		pr_debug("two hwsim sw_scans detected!\n");
+		goto out;
+	}
+
+	pr_debug("hwsim sw_scan request, prepping stuff\n");
+
+	memcpy(hwsim->scan_addr, mac_addr, ETH_ALEN);
+	mac80211_hwsim_config_mac_nl(hw, hwsim->scan_addr, true);
+	hwsim->scanning = true;
+	memset(hwsim->survey_data, 0, sizeof(hwsim->survey_data));
+
+out:
+	mutex_unlock(&hwsim->mutex);
+}
+
+static void mac80211_hwsim_sw_scan_complete(struct ieee80211_hw *hw,
+					    struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_data *hwsim = hw->priv;
+
+	mutex_lock(&hwsim->mutex);
+
+	pr_debug("hwsim sw_scan_complete\n");
+	hwsim->scanning = false;
+	mac80211_hwsim_config_mac_nl(hw, hwsim->scan_addr, false);
+	eth_zero_addr(hwsim->scan_addr);
+
+	mutex_unlock(&hwsim->mutex);
+}
+
+static void hw_roc_start(struct work_struct *work)
+{
+	struct mac80211_hwsim_data *hwsim =
+		container_of(work, struct mac80211_hwsim_data, roc_start.work);
+
+	mutex_lock(&hwsim->mutex);
+
+	wiphy_dbg(hwsim->hw->wiphy, "hwsim ROC begins\n");
+	hwsim->tmp_chan = hwsim->roc_chan;
+	ieee80211_ready_on_channel(hwsim->hw);
+
+	ieee80211_queue_delayed_work(hwsim->hw, &hwsim->roc_done,
+				     msecs_to_jiffies(hwsim->roc_duration));
+
+	mutex_unlock(&hwsim->mutex);
+}
+
+static void hw_roc_done(struct work_struct *work)
+{
+	struct mac80211_hwsim_data *hwsim =
+		container_of(work, struct mac80211_hwsim_data, roc_done.work);
+
+	mutex_lock(&hwsim->mutex);
+	ieee80211_remain_on_channel_expired(hwsim->hw);
+	hwsim->tmp_chan = NULL;
+	mutex_unlock(&hwsim->mutex);
+
+	wiphy_dbg(hwsim->hw->wiphy, "hwsim ROC expired\n");
+}
+
+static int mac80211_hwsim_roc(struct ieee80211_hw *hw,
+			      struct ieee80211_vif *vif,
+			      struct ieee80211_channel *chan,
+			      int duration,
+			      enum ieee80211_roc_type type)
+{
+	struct mac80211_hwsim_data *hwsim = hw->priv;
+
+	mutex_lock(&hwsim->mutex);
+	if (WARN_ON(hwsim->tmp_chan || hwsim->hw_scan_request)) {
+		mutex_unlock(&hwsim->mutex);
+		return -EBUSY;
+	}
+
+	hwsim->roc_chan = chan;
+	hwsim->roc_duration = duration;
+	mutex_unlock(&hwsim->mutex);
+
+	wiphy_dbg(hw->wiphy, "hwsim ROC (%d MHz, %d ms)\n",
+		  chan->center_freq, duration);
+	ieee80211_queue_delayed_work(hw, &hwsim->roc_start, HZ/50);
+
+	return 0;
+}
+
+static int mac80211_hwsim_croc(struct ieee80211_hw *hw,
+			       struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_data *hwsim = hw->priv;
+
+	cancel_delayed_work_sync(&hwsim->roc_start);
+	cancel_delayed_work_sync(&hwsim->roc_done);
+
+	mutex_lock(&hwsim->mutex);
+	hwsim->tmp_chan = NULL;
+	mutex_unlock(&hwsim->mutex);
+
+	wiphy_dbg(hw->wiphy, "hwsim ROC canceled\n");
+
+	return 0;
+}
+
+static int mac80211_hwsim_add_chanctx(struct ieee80211_hw *hw,
+				      struct ieee80211_chanctx_conf *ctx)
+{
+	hwsim_set_chanctx_magic(ctx);
+	wiphy_dbg(hw->wiphy,
+		  "add channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n",
+		  ctx->def.chan->center_freq, ctx->def.width,
+		  ctx->def.center_freq1, ctx->def.center_freq2);
+	return 0;
+}
+
+static void mac80211_hwsim_remove_chanctx(struct ieee80211_hw *hw,
+					  struct ieee80211_chanctx_conf *ctx)
+{
+	wiphy_dbg(hw->wiphy,
+		  "remove channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n",
+		  ctx->def.chan->center_freq, ctx->def.width,
+		  ctx->def.center_freq1, ctx->def.center_freq2);
+	hwsim_check_chanctx_magic(ctx);
+	hwsim_clear_chanctx_magic(ctx);
+}
+
+static void mac80211_hwsim_change_chanctx(struct ieee80211_hw *hw,
+					  struct ieee80211_chanctx_conf *ctx,
+					  u32 changed)
+{
+	hwsim_check_chanctx_magic(ctx);
+	wiphy_dbg(hw->wiphy,
+		  "change channel context control: %d MHz/width: %d/cfreqs:%d/%d MHz\n",
+		  ctx->def.chan->center_freq, ctx->def.width,
+		  ctx->def.center_freq1, ctx->def.center_freq2);
+}
+
+static int mac80211_hwsim_assign_vif_chanctx(struct ieee80211_hw *hw,
+					     struct ieee80211_vif *vif,
+					     struct ieee80211_bss_conf *link_conf,
+					     struct ieee80211_chanctx_conf *ctx)
+{
+	hwsim_check_magic(vif);
+	hwsim_check_chanctx_magic(ctx);
+
+	/* if we activate a link while already associated wake it up */
+	if (vif->type == NL80211_IFTYPE_STATION && vif->cfg.assoc) {
+		struct sk_buff *skb;
+
+		skb = ieee80211_nullfunc_get(hw, vif, link_conf->link_id, true);
+		if (skb) {
+			local_bh_disable();
+			mac80211_hwsim_tx_frame(hw, skb, ctx->def.chan);
+			local_bh_enable();
+		}
+	}
+
+	return 0;
+}
+
+static void mac80211_hwsim_unassign_vif_chanctx(struct ieee80211_hw *hw,
+						struct ieee80211_vif *vif,
+						struct ieee80211_bss_conf *link_conf,
+						struct ieee80211_chanctx_conf *ctx)
+{
+	hwsim_check_magic(vif);
+	hwsim_check_chanctx_magic(ctx);
+
+	/* if we deactivate a link while associated suspend it first */
+	if (vif->type == NL80211_IFTYPE_STATION && vif->cfg.assoc) {
+		struct sk_buff *skb;
+
+		skb = ieee80211_nullfunc_get(hw, vif, link_conf->link_id, true);
+		if (skb) {
+			struct ieee80211_hdr *hdr = (void *)skb->data;
+
+			hdr->frame_control |= cpu_to_le16(IEEE80211_FCTL_PM);
+
+			local_bh_disable();
+			mac80211_hwsim_tx_frame(hw, skb, ctx->def.chan);
+			local_bh_enable();
+		}
+	}
+}
+
+static int mac80211_hwsim_switch_vif_chanctx(struct ieee80211_hw *hw,
+					     struct ieee80211_vif_chanctx_switch *vifs,
+					     int n_vifs,
+					     enum ieee80211_chanctx_switch_mode mode)
+{
+	int i;
+
+	if (n_vifs <= 0)
+		return -EINVAL;
+
+	wiphy_dbg(hw->wiphy,
+		  "switch vif channel context mode: %u\n", mode);
+
+	for (i = 0; i < n_vifs; i++) {
+		hwsim_check_chanctx_magic(vifs[i].old_ctx);
+		wiphy_dbg(hw->wiphy,
+			  "switch vif channel context: %d MHz/width: %d/cfreqs:%d/%d MHz -> %d MHz/width: %d/cfreqs:%d/%d MHz\n",
+			  vifs[i].old_ctx->def.chan->center_freq,
+			  vifs[i].old_ctx->def.width,
+			  vifs[i].old_ctx->def.center_freq1,
+			  vifs[i].old_ctx->def.center_freq2,
+			  vifs[i].new_ctx->def.chan->center_freq,
+			  vifs[i].new_ctx->def.width,
+			  vifs[i].new_ctx->def.center_freq1,
+			  vifs[i].new_ctx->def.center_freq2);
+
+		switch (mode) {
+		case CHANCTX_SWMODE_REASSIGN_VIF:
+			hwsim_check_chanctx_magic(vifs[i].new_ctx);
+			break;
+		case CHANCTX_SWMODE_SWAP_CONTEXTS:
+			hwsim_set_chanctx_magic(vifs[i].new_ctx);
+			hwsim_clear_chanctx_magic(vifs[i].old_ctx);
+			break;
+		default:
+			WARN(1, "Invalid mode %d\n", mode);
+		}
+	}
+	return 0;
+}
+
+static const char mac80211_hwsim_gstrings_stats[][ETH_GSTRING_LEN] = {
+	"tx_pkts_nic",
+	"tx_bytes_nic",
+	"rx_pkts_nic",
+	"rx_bytes_nic",
+	"d_tx_dropped",
+	"d_tx_failed",
+	"d_ps_mode",
+	"d_group",
+};
+
+#define MAC80211_HWSIM_SSTATS_LEN ARRAY_SIZE(mac80211_hwsim_gstrings_stats)
+
+static void mac80211_hwsim_get_et_strings(struct ieee80211_hw *hw,
+					  struct ieee80211_vif *vif,
+					  u32 sset, u8 *data)
+{
+	if (sset == ETH_SS_STATS)
+		memcpy(data, mac80211_hwsim_gstrings_stats,
+		       sizeof(mac80211_hwsim_gstrings_stats));
+}
+
+static int mac80211_hwsim_get_et_sset_count(struct ieee80211_hw *hw,
+					    struct ieee80211_vif *vif, int sset)
+{
+	if (sset == ETH_SS_STATS)
+		return MAC80211_HWSIM_SSTATS_LEN;
+	return 0;
+}
+
+static void mac80211_hwsim_get_et_stats(struct ieee80211_hw *hw,
+					struct ieee80211_vif *vif,
+					struct ethtool_stats *stats, u64 *data)
+{
+	struct mac80211_hwsim_data *ar = hw->priv;
+	int i = 0;
+
+	data[i++] = ar->tx_pkts;
+	data[i++] = ar->tx_bytes;
+	data[i++] = ar->rx_pkts;
+	data[i++] = ar->rx_bytes;
+	data[i++] = ar->tx_dropped;
+	data[i++] = ar->tx_failed;
+	data[i++] = ar->ps;
+	data[i++] = ar->group;
+
+	WARN_ON(i != MAC80211_HWSIM_SSTATS_LEN);
+}
+
+static int mac80211_hwsim_tx_last_beacon(struct ieee80211_hw *hw)
+{
+	return 1;
+}
+
+static int mac80211_hwsim_set_rts_threshold(struct ieee80211_hw *hw,
+					    int radio_idx, u32 value)
+{
+	/* hwsim ignores the use_rts instruction from mac80211 anyway */
+	return 0;
+}
+
+static int mac80211_hwsim_change_vif_links(struct ieee80211_hw *hw,
+					   struct ieee80211_vif *vif,
+					   u16 old_links, u16 new_links,
+					   struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
+{
+	unsigned long rem = old_links & ~new_links;
+	unsigned long add = new_links & ~old_links;
+	int i;
+
+	if (!old_links)
+		rem |= BIT(0);
+	if (!new_links)
+		add |= BIT(0);
+
+	for_each_set_bit(i, &rem, IEEE80211_MLD_MAX_NUM_LINKS)
+		mac80211_hwsim_config_mac_nl(hw, old[i]->addr, false);
+
+	for_each_set_bit(i, &add, IEEE80211_MLD_MAX_NUM_LINKS) {
+		struct ieee80211_bss_conf *link_conf;
+
+		link_conf = link_conf_dereference_protected(vif, i);
+		if (WARN_ON(!link_conf))
+			continue;
+
+		mac80211_hwsim_config_mac_nl(hw, link_conf->addr, true);
+	}
+
+	return 0;
+}
+
+static int mac80211_hwsim_change_sta_links(struct ieee80211_hw *hw,
+					   struct ieee80211_vif *vif,
+					   struct ieee80211_sta *sta,
+					   u16 old_links, u16 new_links)
+{
+	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+
+	hwsim_check_sta_magic(sta);
+
+	if (vif->type == NL80211_IFTYPE_STATION)
+		sp->active_links_rx = new_links;
+
+	return 0;
+}
+
+static int mac80211_hwsim_send_pmsr_ftm_request_peer(struct sk_buff *msg,
+						     struct cfg80211_pmsr_ftm_request_peer *request)
+{
+	struct nlattr *ftm;
+
+	if (!request->requested)
+		return -EINVAL;
+
+	ftm = nla_nest_start(msg, NL80211_PMSR_TYPE_FTM);
+	if (!ftm)
+		return -ENOBUFS;
+
+	if (nla_put_u32(msg, NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE, request->preamble))
+		return -ENOBUFS;
+
+	if (nla_put_u16(msg, NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD, request->burst_period))
+		return -ENOBUFS;
+
+	if (request->asap && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_ASAP))
+		return -ENOBUFS;
+
+	if (request->request_lci && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_REQUEST_LCI))
+		return -ENOBUFS;
+
+	if (request->request_civicloc &&
+	    nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_REQUEST_CIVICLOC))
+		return -ENOBUFS;
+
+	if (request->trigger_based && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED))
+		return -ENOBUFS;
+
+	if (request->non_trigger_based &&
+	    nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED))
+		return -ENOBUFS;
+
+	if (request->lmr_feedback && nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_NUM_BURSTS_EXP, request->num_bursts_exp))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_BURST_DURATION, request->burst_duration))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST, request->ftms_per_burst))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_NUM_FTMR_RETRIES, request->ftmr_retries))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_BURST_DURATION, request->burst_duration))
+		return -ENOBUFS;
+
+	if (nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR, request->bss_color))
+		return -ENOBUFS;
+
+	if (request->min_time_between_measurements &&
+	    nla_put_u32(msg, NL80211_PMSR_FTM_REQ_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS,
+			request->min_time_between_measurements))
+		return -ENOBUFS;
+
+	if (request->max_time_between_measurements &&
+	    nla_put_u32(msg, NL80211_PMSR_FTM_REQ_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS,
+			request->max_time_between_measurements))
+		return -ENOBUFS;
+
+	if (request->availability_window &&
+	    nla_put_u8(msg, NL80211_PMSR_FTM_REQ_ATTR_AW_DURATION,
+		       request->availability_window))
+		return -ENOBUFS;
+
+	if (request->nominal_time &&
+	    nla_put_u32(msg, NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME,
+			request->nominal_time))
+		return -ENOBUFS;
+
+	if (request->num_measurements &&
+	    nla_put_u32(msg, NL80211_PMSR_FTM_REQ_ATTR_NUM_MEASUREMENTS,
+			request->num_measurements))
+		return -ENOBUFS;
+
+	if (request->ingress_distance &&
+	    nla_put_u64_64bit(msg, NL80211_PMSR_FTM_REQ_ATTR_INGRESS,
+			      request->ingress_distance,
+			      NL80211_PMSR_FTM_REQ_ATTR_PAD))
+		return -ENOBUFS;
+
+	if (request->egress_distance &&
+	    nla_put_u64_64bit(msg, NL80211_PMSR_FTM_REQ_ATTR_EGRESS,
+			      request->egress_distance,
+			      NL80211_PMSR_FTM_REQ_ATTR_PAD))
+		return -ENOBUFS;
+
+	if (request->pd_suppress_range_results &&
+	    nla_put_flag(msg, NL80211_PMSR_FTM_REQ_ATTR_PD_SUPPRESS_RESULTS))
+		return -ENOBUFS;
+
+	nla_nest_end(msg, ftm);
+
+	return 0;
+}
+
+static int mac80211_hwsim_send_pmsr_request_peer(struct sk_buff *msg,
+						 struct cfg80211_pmsr_request_peer *request)
+{
+	struct nlattr *peer, *chandef, *req, *data;
+	int err;
+
+	peer = nla_nest_start(msg, NL80211_PMSR_ATTR_PEERS);
+	if (!peer)
+		return -ENOBUFS;
+
+	if (nla_put(msg, NL80211_PMSR_PEER_ATTR_ADDR, ETH_ALEN,
+		    request->addr))
+		return -ENOBUFS;
+
+	chandef = nla_nest_start(msg, NL80211_PMSR_PEER_ATTR_CHAN);
+	if (!chandef)
+		return -ENOBUFS;
+
+	err = nl80211_send_chandef(msg, &request->chandef);
+	if (err)
+		return err;
+
+	nla_nest_end(msg, chandef);
+
+	req = nla_nest_start(msg, NL80211_PMSR_PEER_ATTR_REQ);
+	if (!req)
+		return -ENOBUFS;
+
+	if (request->report_ap_tsf && nla_put_flag(msg, NL80211_PMSR_REQ_ATTR_GET_AP_TSF))
+		return -ENOBUFS;
+
+	data = nla_nest_start(msg, NL80211_PMSR_REQ_ATTR_DATA);
+	if (!data)
+		return -ENOBUFS;
+
+	err = mac80211_hwsim_send_pmsr_ftm_request_peer(msg, &request->ftm);
+	if (err)
+		return err;
+
+	nla_nest_end(msg, data);
+	nla_nest_end(msg, req);
+	nla_nest_end(msg, peer);
+
+	return 0;
+}
+
+static int mac80211_hwsim_send_pmsr_request(struct sk_buff *msg,
+					    struct cfg80211_pmsr_request *request)
+{
+	struct nlattr *pmsr;
+	int err;
+
+	pmsr = nla_nest_start(msg, NL80211_ATTR_PEER_MEASUREMENTS);
+	if (!pmsr)
+		return -ENOBUFS;
+
+	if (nla_put_u32(msg, NL80211_ATTR_TIMEOUT, request->timeout))
+		return -ENOBUFS;
+
+	if (!is_zero_ether_addr(request->mac_addr)) {
+		if (nla_put(msg, NL80211_ATTR_MAC, ETH_ALEN, request->mac_addr))
+			return -ENOBUFS;
+		if (nla_put(msg, NL80211_ATTR_MAC_MASK, ETH_ALEN, request->mac_addr_mask))
+			return -ENOBUFS;
+	}
+
+	for (int i = 0; i < request->n_peers; i++) {
+		err = mac80211_hwsim_send_pmsr_request_peer(msg, &request->peers[i]);
+		if (err)
+			return err;
+	}
+
+	nla_nest_end(msg, pmsr);
+
+	return 0;
+}
+
+static int mac80211_hwsim_start_pmsr(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct cfg80211_pmsr_request *request)
+{
+	struct mac80211_hwsim_data *data;
+	struct sk_buff *skb = NULL;
+	struct nlattr *pmsr;
+	void *msg_head;
+	u32 _portid;
+	int err = 0;
+
+	data = hw->priv;
+	_portid = READ_ONCE(data->wmediumd);
+	if (!_portid && !hwsim_virtio_enabled)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&data->mutex);
+
+	if (data->pmsr_request) {
+		err = -EBUSY;
+		goto out_free;
+	}
+
+	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+
+	if (!skb) {
+		err = -ENOMEM;
+		goto out_free;
+	}
+
+	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0, HWSIM_CMD_START_PMSR);
+
+	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER,
+		    ETH_ALEN, data->addresses[1].addr)) {
+		err = -ENOMEM;
+		goto out_free;
+	}
+
+	pmsr = nla_nest_start(skb, HWSIM_ATTR_PMSR_REQUEST);
+	if (!pmsr) {
+		err = -ENOMEM;
+		goto out_free;
+	}
+
+	err = mac80211_hwsim_send_pmsr_request(skb, request);
+	if (err)
+		goto out_free;
+
+	nla_nest_end(skb, pmsr);
+
+	genlmsg_end(skb, msg_head);
+	if (hwsim_virtio_enabled)
+		hwsim_tx_virtio(data, skb);
+	else
+		hwsim_unicast_netgroup(data, skb, _portid);
+
+	data->pmsr_request = request;
+	data->pmsr_request_wdev = ieee80211_vif_to_wdev(vif);
+
+out_free:
+	if (err && skb)
+		nlmsg_free(skb);
+
+	mutex_unlock(&data->mutex);
+	return err;
+}
+
+static void mac80211_hwsim_abort_pmsr(struct ieee80211_hw *hw,
+				      struct ieee80211_vif *vif,
+				      struct cfg80211_pmsr_request *request)
+{
+	struct mac80211_hwsim_data *data;
+	struct sk_buff *skb = NULL;
+	struct nlattr *pmsr;
+	void *msg_head;
+	u32 _portid;
+	int err = 0;
+
+	data = hw->priv;
+	_portid = READ_ONCE(data->wmediumd);
+	if (!_portid && !hwsim_virtio_enabled)
+		return;
+
+	mutex_lock(&data->mutex);
+
+	if (data->pmsr_request != request) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!skb) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	msg_head = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0, HWSIM_CMD_ABORT_PMSR);
+
+	if (nla_put(skb, HWSIM_ATTR_ADDR_TRANSMITTER, ETH_ALEN, data->addresses[1].addr))
+		goto out;
+
+	pmsr = nla_nest_start(skb, HWSIM_ATTR_PMSR_REQUEST);
+	if (!pmsr) {
+		err = -ENOMEM;
+		goto out;
+	}
+
+	err = mac80211_hwsim_send_pmsr_request(skb, request);
+	if (err)
+		goto out;
+
+	err = nla_nest_end(skb, pmsr);
+	if (err)
+		goto out;
+
+	genlmsg_end(skb, msg_head);
+	if (hwsim_virtio_enabled)
+		hwsim_tx_virtio(data, skb);
+	else
+		hwsim_unicast_netgroup(data, skb, _portid);
+
+out:
+	if (err && skb)
+		nlmsg_free(skb);
+
+	mutex_unlock(&data->mutex);
+}
+
+static int mac80211_hwsim_parse_rate_info(struct nlattr *rateattr,
+					  struct rate_info *rate_info,
+					  struct genl_info *info)
+{
+	struct nlattr *tb[HWSIM_RATE_INFO_ATTR_MAX + 1];
+	int ret;
+
+	ret = nla_parse_nested(tb, HWSIM_RATE_INFO_ATTR_MAX,
+			       rateattr, hwsim_rate_info_policy, info->extack);
+	if (ret)
+		return ret;
+
+	if (tb[HWSIM_RATE_INFO_ATTR_FLAGS])
+		rate_info->flags = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_FLAGS]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_MCS])
+		rate_info->mcs = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_MCS]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_LEGACY])
+		rate_info->legacy = nla_get_u16(tb[HWSIM_RATE_INFO_ATTR_LEGACY]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_NSS])
+		rate_info->nss = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_NSS]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_BW])
+		rate_info->bw = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_BW]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_HE_GI])
+		rate_info->he_gi = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_HE_GI]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_HE_DCM])
+		rate_info->he_dcm = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_HE_DCM]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC])
+		rate_info->he_ru_alloc =
+			nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_HE_RU_ALLOC]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH])
+		rate_info->n_bonded_ch = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_N_BOUNDED_CH]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_EHT_GI])
+		rate_info->eht_gi = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_EHT_GI]);
+
+	if (tb[HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC])
+		rate_info->eht_ru_alloc = nla_get_u8(tb[HWSIM_RATE_INFO_ATTR_EHT_RU_ALLOC]);
+
+	return 0;
+}
+
+static int mac80211_hwsim_parse_ftm_result(struct nlattr *ftm,
+					   struct cfg80211_pmsr_ftm_result *result,
+					   struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_FTM_RESP_ATTR_MAX + 1];
+	int ret;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_FTM_RESP_ATTR_MAX,
+			       ftm, hwsim_ftm_result_policy, info->extack);
+	if (ret)
+		return ret;
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON])
+		result->failure_reason = nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_FAIL_REASON]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_INDEX])
+		result->burst_index = nla_get_u16(tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_INDEX]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS]) {
+		result->num_ftmr_attempts_valid = 1;
+		result->num_ftmr_attempts =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_ATTEMPTS]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES]) {
+		result->num_ftmr_successes_valid = 1;
+		result->num_ftmr_successes =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_FTMR_SUCCESSES]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME])
+		result->busy_retry_time =
+			nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_BUSY_RETRY_TIME]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP])
+		result->num_bursts_exp = nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_BURSTS_EXP]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_DURATION])
+		result->burst_duration = nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_BURST_DURATION]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST])
+		result->ftms_per_burst = nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_FTMS_PER_BURST]);
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_AVG]) {
+		result->rssi_avg_valid = 1;
+		result->rssi_avg = nla_get_s32(tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_AVG]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_SPREAD]) {
+		result->rssi_spread_valid = 1;
+		result->rssi_spread =
+			nla_get_s32(tb[NL80211_PMSR_FTM_RESP_ATTR_RSSI_SPREAD]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_TX_RATE]) {
+		result->tx_rate_valid = 1;
+		ret = mac80211_hwsim_parse_rate_info(tb[NL80211_PMSR_FTM_RESP_ATTR_TX_RATE],
+						     &result->tx_rate, info);
+		if (ret)
+			return ret;
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RX_RATE]) {
+		result->rx_rate_valid = 1;
+		ret = mac80211_hwsim_parse_rate_info(tb[NL80211_PMSR_FTM_RESP_ATTR_RX_RATE],
+						     &result->rx_rate, info);
+		if (ret)
+			return ret;
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_AVG]) {
+		result->rtt_avg_valid = 1;
+		result->rtt_avg =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_AVG]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_VARIANCE]) {
+		result->rtt_variance_valid = 1;
+		result->rtt_variance =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_VARIANCE]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_SPREAD]) {
+		result->rtt_spread_valid = 1;
+		result->rtt_spread =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_RTT_SPREAD]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_AVG]) {
+		result->dist_avg_valid = 1;
+		result->dist_avg =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_AVG]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_VARIANCE]) {
+		result->dist_variance_valid = 1;
+		result->dist_variance =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_VARIANCE]);
+	}
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_SPREAD]) {
+		result->dist_spread_valid = 1;
+		result->dist_spread =
+			nla_get_u64(tb[NL80211_PMSR_FTM_RESP_ATTR_DIST_SPREAD]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_LCI]) {
+		result->lci = nla_data(tb[NL80211_PMSR_FTM_RESP_ATTR_LCI]);
+		result->lci_len = nla_len(tb[NL80211_PMSR_FTM_RESP_ATTR_LCI]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC]) {
+		result->civicloc = nla_data(tb[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC]);
+		result->civicloc_len = nla_len(tb[NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_TX_LTF_REPETITION_COUNT]) {
+		result->tx_ltf_repetition_count_valid = 1;
+		result->tx_ltf_repetition_count =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_TX_LTF_REPETITION_COUNT]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_RX_LTF_REPETITION_COUNT]) {
+		result->rx_ltf_repetition_count_valid = 1;
+		result->rx_ltf_repetition_count =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_RX_LTF_REPETITION_COUNT]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS]) {
+		result->max_time_between_measurements_valid = 1;
+		result->max_time_between_measurements =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS]) {
+		result->min_time_between_measurements_valid = 1;
+		result->min_time_between_measurements =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_TX_SPATIAL_STREAMS]) {
+		result->num_tx_spatial_streams_valid = 1;
+		result->num_tx_spatial_streams =
+			nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_TX_SPATIAL_STREAMS]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_RX_SPATIAL_STREAMS]) {
+		result->num_rx_spatial_streams_valid = 1;
+		result->num_rx_spatial_streams =
+			nla_get_u8(tb[NL80211_PMSR_FTM_RESP_ATTR_NUM_RX_SPATIAL_STREAMS]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_NOMINAL_TIME]) {
+		result->nominal_time_valid = 1;
+		result->nominal_time =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_NOMINAL_TIME]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_AVAILABILITY_WINDOW]) {
+		result->availability_window_valid = 1;
+		result->availability_window =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_AVAILABILITY_WINDOW]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_CHANNEL_WIDTH]) {
+		result->chan_width_valid = 1;
+		result->chan_width =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_CHANNEL_WIDTH]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_RESP_ATTR_PREAMBLE]) {
+		result->preamble_valid = 1;
+		result->preamble =
+			nla_get_u32(tb[NL80211_PMSR_FTM_RESP_ATTR_PREAMBLE]);
+	}
+
+	result->is_delayed_lmr =
+		nla_get_flag(tb[NL80211_PMSR_FTM_RESP_ATTR_IS_DELAYED_LMR]);
+
+	return 0;
+}
+
+static int mac80211_hwsim_parse_pmsr_resp(struct nlattr *resp,
+					  struct cfg80211_pmsr_result *result,
+					  struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_RESP_ATTR_MAX + 1];
+	struct nlattr *pmsr;
+	int rem;
+	int ret;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_RESP_ATTR_MAX, resp, hwsim_pmsr_resp_policy,
+			       info->extack);
+	if (ret)
+		return ret;
+
+	if (tb[NL80211_PMSR_RESP_ATTR_STATUS])
+		result->status = nla_get_u32(tb[NL80211_PMSR_RESP_ATTR_STATUS]);
+
+	if (tb[NL80211_PMSR_RESP_ATTR_HOST_TIME])
+		result->host_time = nla_get_u64(tb[NL80211_PMSR_RESP_ATTR_HOST_TIME]);
+
+	if (tb[NL80211_PMSR_RESP_ATTR_AP_TSF]) {
+		result->ap_tsf_valid = 1;
+		result->ap_tsf = nla_get_u64(tb[NL80211_PMSR_RESP_ATTR_AP_TSF]);
+	}
+
+	result->final = !!tb[NL80211_PMSR_RESP_ATTR_FINAL];
+
+	if (!tb[NL80211_PMSR_RESP_ATTR_DATA])
+		return 0;
+
+	nla_for_each_nested(pmsr, tb[NL80211_PMSR_RESP_ATTR_DATA], rem) {
+		switch (nla_type(pmsr)) {
+		case NL80211_PMSR_TYPE_FTM:
+			result->type = NL80211_PMSR_TYPE_FTM;
+			ret = mac80211_hwsim_parse_ftm_result(pmsr, &result->ftm, info);
+			if (ret)
+				return ret;
+			break;
+		default:
+			NL_SET_ERR_MSG_ATTR(info->extack, pmsr, "Unknown pmsr resp type");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int mac80211_hwsim_parse_pmsr_result(struct nlattr *peer,
+					    struct cfg80211_pmsr_result *result,
+					    struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_PEER_ATTR_MAX + 1];
+	int ret;
+
+	if (!peer)
+		return -EINVAL;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_PEER_ATTR_MAX, peer,
+			       hwsim_pmsr_peer_result_policy, info->extack);
+	if (ret)
+		return ret;
+
+	if (tb[NL80211_PMSR_PEER_ATTR_ADDR])
+		memcpy(result->addr, nla_data(tb[NL80211_PMSR_PEER_ATTR_ADDR]),
+		       ETH_ALEN);
+
+	if (tb[NL80211_PMSR_PEER_ATTR_RESP]) {
+		ret = mac80211_hwsim_parse_pmsr_resp(tb[NL80211_PMSR_PEER_ATTR_RESP], result, info);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+};
+
+static int hwsim_pmsr_report_nl(struct sk_buff *msg, struct genl_info *info)
+{
+	struct mac80211_hwsim_data *data;
+	struct nlattr *peers, *peer;
+	struct nlattr *reqattr;
+	const u8 *src;
+	int err;
+	int rem;
+
+	if (!info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER])
+		return -EINVAL;
+
+	src = nla_data(info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER]);
+	data = get_hwsim_data_ref_from_addr(src);
+	if (!data)
+		return -EINVAL;
+
+	mutex_lock(&data->mutex);
+	if (!data->pmsr_request) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	reqattr = info->attrs[HWSIM_ATTR_PMSR_RESULT];
+	if (!reqattr) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	peers = nla_find_nested(reqattr, NL80211_PMSR_ATTR_PEERS);
+	if (!peers) {
+		err = -EINVAL;
+		goto out;
+	}
+
+	nla_for_each_nested(peer, peers, rem) {
+		struct cfg80211_pmsr_result result = {};
+
+		err = mac80211_hwsim_parse_pmsr_result(peer, &result, info);
+		if (err)
+			goto out;
+
+		cfg80211_pmsr_report(data->pmsr_request_wdev,
+				     data->pmsr_request, &result, GFP_KERNEL);
+	}
+
+	cfg80211_pmsr_complete(data->pmsr_request_wdev, data->pmsr_request, GFP_KERNEL);
+
+	err = 0;
+out:
+	data->pmsr_request = NULL;
+	data->pmsr_request_wdev = NULL;
+
+	mutex_unlock(&data->mutex);
+	return err;
+}
+
+static int mac80211_hwsim_set_radar_background(struct ieee80211_hw *hw,
+					       struct cfg80211_chan_def *chan)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+
+	if (!wiphy_ext_feature_isset(hw->wiphy,
+				     NL80211_EXT_FEATURE_RADAR_BACKGROUND))
+		return -EOPNOTSUPP;
+
+	if (chan)
+		data->radar_background_chandef = *chan;
+	else
+		memset(&data->radar_background_chandef, 0,
+		       sizeof(data->radar_background_chandef));
+
+	return 0;
+}
+
+#ifdef CONFIG_MAC80211_DEBUGFS
+#define HWSIM_DEBUGFS_OPS					\
+	.link_add_debugfs = mac80211_hwsim_link_add_debugfs,
+#else
+#define HWSIM_DEBUGFS_OPS
+#endif
+
+#define HWSIM_COMMON_OPS					\
+	.tx = mac80211_hwsim_tx,				\
+	.wake_tx_queue = ieee80211_hwsim_wake_tx_queue,		\
+	.start = mac80211_hwsim_start,				\
+	.stop = mac80211_hwsim_stop,				\
+	.add_interface = mac80211_hwsim_add_interface,		\
+	.change_interface = mac80211_hwsim_change_interface,	\
+	.remove_interface = mac80211_hwsim_remove_interface,	\
+	.config = mac80211_hwsim_config,			\
+	.configure_filter = mac80211_hwsim_configure_filter,	\
+	.vif_cfg_changed = mac80211_hwsim_vif_info_changed,	\
+	.link_info_changed = mac80211_hwsim_link_info_changed,  \
+	.tx_last_beacon = mac80211_hwsim_tx_last_beacon,	\
+	.sta_notify = mac80211_hwsim_sta_notify,		\
+	.link_sta_rc_update = mac80211_hwsim_sta_rc_update,	\
+	.conf_tx = mac80211_hwsim_conf_tx,			\
+	.get_survey = mac80211_hwsim_get_survey,		\
+	CFG80211_TESTMODE_CMD(mac80211_hwsim_testmode_cmd)	\
+	.ampdu_action = mac80211_hwsim_ampdu_action,		\
+	.flush = mac80211_hwsim_flush,				\
+	.get_et_sset_count = mac80211_hwsim_get_et_sset_count,	\
+	.get_et_stats = mac80211_hwsim_get_et_stats,		\
+	.get_et_strings = mac80211_hwsim_get_et_strings,	\
+	.start_pmsr = mac80211_hwsim_start_pmsr,		\
+	.abort_pmsr = mac80211_hwsim_abort_pmsr,		\
+	.set_radar_background = mac80211_hwsim_set_radar_background, \
+	.set_key = mac80211_hwsim_set_key,			\
+	.set_rts_threshold = mac80211_hwsim_set_rts_threshold,	\
+	.start_nan = mac80211_hwsim_nan_start,			\
+	.stop_nan = mac80211_hwsim_nan_stop,			\
+	.nan_change_conf = mac80211_hwsim_nan_change_config,	\
+	.nan_peer_sched_changed = mac80211_hwsim_nan_peer_sched_changed, \
+	HWSIM_DEBUGFS_OPS
+
+#define HWSIM_NON_MLO_OPS					\
+	.sta_add = mac80211_hwsim_sta_add,			\
+	.sta_remove = mac80211_hwsim_sta_remove,		\
+	.set_tim = mac80211_hwsim_set_tim,			\
+	.get_tsf = mac80211_hwsim_get_tsf,			\
+	.set_tsf = mac80211_hwsim_set_tsf,
+
+static const struct ieee80211_ops mac80211_hwsim_ops = {
+	HWSIM_COMMON_OPS
+	HWSIM_NON_MLO_OPS
+	.sw_scan_start = mac80211_hwsim_sw_scan,
+	.sw_scan_complete = mac80211_hwsim_sw_scan_complete,
+	.add_chanctx = ieee80211_emulate_add_chanctx,
+	.remove_chanctx = ieee80211_emulate_remove_chanctx,
+	.change_chanctx = ieee80211_emulate_change_chanctx,
+	.switch_vif_chanctx = ieee80211_emulate_switch_vif_chanctx,
+};
+
+#define HWSIM_CHANCTX_OPS					\
+	.hw_scan = mac80211_hwsim_hw_scan,			\
+	.cancel_hw_scan = mac80211_hwsim_cancel_hw_scan,	\
+	.remain_on_channel = mac80211_hwsim_roc,		\
+	.cancel_remain_on_channel = mac80211_hwsim_croc,	\
+	.add_chanctx = mac80211_hwsim_add_chanctx,		\
+	.remove_chanctx = mac80211_hwsim_remove_chanctx,	\
+	.change_chanctx = mac80211_hwsim_change_chanctx,	\
+	.assign_vif_chanctx = mac80211_hwsim_assign_vif_chanctx,\
+	.unassign_vif_chanctx = mac80211_hwsim_unassign_vif_chanctx, \
+	.switch_vif_chanctx = mac80211_hwsim_switch_vif_chanctx,
+
+static const struct ieee80211_ops mac80211_hwsim_mchan_ops = {
+	HWSIM_COMMON_OPS
+	HWSIM_NON_MLO_OPS
+	HWSIM_CHANCTX_OPS
+};
+
+static const struct ieee80211_ops mac80211_hwsim_mlo_ops = {
+	HWSIM_COMMON_OPS
+	HWSIM_CHANCTX_OPS
+	.change_vif_links = mac80211_hwsim_change_vif_links,
+	.change_sta_links = mac80211_hwsim_change_sta_links,
+	.sta_state = mac80211_hwsim_sta_state,
+	.can_neg_ttlm = mac80211_hwsim_can_neg_ttlm,
+};
+
+struct hwsim_new_radio_params {
+	unsigned int channels;
+	const char *reg_alpha2;
+	const struct ieee80211_regdomain *regd;
+	bool reg_strict;
+	bool p2p_device;
+	bool use_chanctx;
+	bool multi_radio;
+	bool destroy_on_close;
+	const char *hwname;
+	bool no_vif;
+	const u8 *perm_addr;
+	u32 iftypes;
+	u32 *ciphers;
+	u8 n_ciphers;
+	bool mlo;
+	const struct cfg80211_pmsr_capabilities *pmsr_capa;
+	bool nan_device;
+	bool background_radar;
+};
+
+static void hwsim_mcast_config_msg(struct sk_buff *mcast_skb,
+				   struct genl_info *info)
+{
+	if (info)
+		genl_notify(&hwsim_genl_family, mcast_skb, info,
+			    HWSIM_MCGRP_CONFIG, GFP_KERNEL);
+	else
+		genlmsg_multicast(&hwsim_genl_family, mcast_skb, 0,
+				  HWSIM_MCGRP_CONFIG, GFP_KERNEL);
+}
+
+static int append_radio_msg(struct sk_buff *skb, int id,
+			    struct hwsim_new_radio_params *param)
+{
+	int ret;
+
+	ret = nla_put_u32(skb, HWSIM_ATTR_RADIO_ID, id);
+	if (ret < 0)
+		return ret;
+
+	if (param->channels) {
+		ret = nla_put_u32(skb, HWSIM_ATTR_CHANNELS, param->channels);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (param->reg_alpha2) {
+		ret = nla_put(skb, HWSIM_ATTR_REG_HINT_ALPHA2, 2,
+			      param->reg_alpha2);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (param->regd) {
+		int i;
+
+		for (i = 0; i < ARRAY_SIZE(hwsim_world_regdom_custom); i++) {
+			if (hwsim_world_regdom_custom[i] != param->regd)
+				continue;
+
+			ret = nla_put_u32(skb, HWSIM_ATTR_REG_CUSTOM_REG, i);
+			if (ret < 0)
+				return ret;
+			break;
+		}
+	}
+
+	if (param->reg_strict) {
+		ret = nla_put_flag(skb, HWSIM_ATTR_REG_STRICT_REG);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (param->p2p_device) {
+		ret = nla_put_flag(skb, HWSIM_ATTR_SUPPORT_P2P_DEVICE);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (param->use_chanctx) {
+		ret = nla_put_flag(skb, HWSIM_ATTR_USE_CHANCTX);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (param->multi_radio) {
+		ret = nla_put_flag(skb, HWSIM_ATTR_MULTI_RADIO);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (param->hwname) {
+		ret = nla_put(skb, HWSIM_ATTR_RADIO_NAME,
+			      strlen(param->hwname), param->hwname);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (param->nan_device) {
+		ret = nla_put_flag(skb, HWSIM_ATTR_SUPPORT_NAN_DEVICE);
+		if (ret < 0)
+			return ret;
+	}
+
+	if (param->background_radar) {
+		ret = nla_put_flag(skb, HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR);
+		if (ret < 0)
+			return ret;
+	}
+	return 0;
+}
+
+static void hwsim_mcast_new_radio(int id, struct genl_info *info,
+				  struct hwsim_new_radio_params *param)
+{
+	struct sk_buff *mcast_skb;
+	void *data;
+
+	mcast_skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!mcast_skb)
+		return;
+
+	data = genlmsg_put(mcast_skb, 0, 0, &hwsim_genl_family, 0,
+			   HWSIM_CMD_NEW_RADIO);
+	if (!data)
+		goto out_err;
+
+	if (append_radio_msg(mcast_skb, id, param) < 0)
+		goto out_err;
+
+	genlmsg_end(mcast_skb, data);
+
+	hwsim_mcast_config_msg(mcast_skb, info);
+	return;
+
+out_err:
+	nlmsg_free(mcast_skb);
+}
+
+static const struct ieee80211_sband_iftype_data sband_capa_2ghz[] = {
+	{
+		.types_mask = BIT(NL80211_IFTYPE_STATION) |
+			      BIT(NL80211_IFTYPE_P2P_CLIENT),
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_BSR |
+					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] =
+					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xffff),
+				.tx_mcs_160 = cpu_to_le16(0xffff),
+				.rx_mcs_80p80 = cpu_to_le16(0xffff),
+				.tx_mcs_80p80 = cpu_to_le16(0xffff),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] =
+					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
+					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
+					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE,
+				.phy_cap_info[3] =
+					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+				.phy_cap_info[4] =
+					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
+					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
+					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
+					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
+				.phy_cap_info[5] =
+					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
+					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
+					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
+				.phy_cap_info[6] =
+					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
+					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
+				.phy_cap_info[7] =
+					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW,
+			},
+
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/*
+				 * Since B0, B1, B2 and B3 are not set in
+				 * the supported channel width set field in the
+				 * HE PHY capabilities information field the
+				 * device is a 20MHz only device on 2.4GHz band.
+				 */
+				.only_20mhz = {
+					.rx_tx_mcs7_max_nss = 0x88,
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+		.uhr_cap = {
+			.has_uhr = true,
+			.mac.mac_cap = {
+				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP,
+			},
+			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
+				   IEEE80211_UHR_PHY_CAP_ELR_TX,
+		},
+	},
+	{
+		.types_mask = BIT(NL80211_IFTYPE_AP) |
+			      BIT(NL80211_IFTYPE_P2P_GO),
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_BSR |
+					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] =
+					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xffff),
+				.tx_mcs_160 = cpu_to_le16(0xffff),
+				.rx_mcs_80p80 = cpu_to_le16(0xffff),
+				.tx_mcs_80p80 = cpu_to_le16(0xffff),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] =
+					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
+					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
+					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE,
+				.phy_cap_info[3] =
+					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+				.phy_cap_info[4] =
+					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
+					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
+					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
+					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
+				.phy_cap_info[5] =
+					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
+					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
+					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
+				.phy_cap_info[6] =
+					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
+					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
+				.phy_cap_info[7] =
+					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW,
+			},
+
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/*
+				 * Since B0, B1, B2 and B3 are not set in
+				 * the supported channel width set field in the
+				 * HE PHY capabilities information field the
+				 * device is a 20MHz only device on 2.4GHz band.
+				 */
+				.only_20mhz = {
+					.rx_tx_mcs7_max_nss = 0x88,
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+		.uhr_cap = {
+			.has_uhr = true,
+			.mac.mac_cap = {
+				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP,
+			},
+			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
+				   IEEE80211_UHR_PHY_CAP_ELR_TX,
+		},
+	},
+#ifdef CONFIG_MAC80211_MESH
+	{
+		.types_mask = BIT(NL80211_IFTYPE_MESH_POINT),
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] = 0,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xffff),
+				.tx_mcs_160 = cpu_to_le16(0xffff),
+				.rx_mcs_80p80 = cpu_to_le16(0xffff),
+				.tx_mcs_80p80 = cpu_to_le16(0xffff),
+			},
+		},
+	},
+#endif
+};
+
+static const struct ieee80211_sband_iftype_data sband_capa_5ghz[] = {
+	{
+		.types_mask = BIT(NL80211_IFTYPE_STATION) |
+			      BIT(NL80211_IFTYPE_P2P_CLIENT),
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_BSR |
+					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] =
+					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xfffa),
+				.tx_mcs_160 = cpu_to_le16(0xfffa),
+				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] =
+					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
+					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
+					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
+					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
+				.phy_cap_info[1] =
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK,
+				.phy_cap_info[2] =
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK,
+				.phy_cap_info[3] =
+					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+				.phy_cap_info[4] =
+					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
+					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
+					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
+					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
+				.phy_cap_info[5] =
+					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
+					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
+					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
+				.phy_cap_info[6] =
+					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
+					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
+				.phy_cap_info[7] =
+					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ,
+			},
+
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/*
+				 * As B1 and B2 are set in the supported
+				 * channel width set field in the HE PHY
+				 * capabilities information field include all
+				 * the following MCS/NSS.
+				 */
+				.bw._80 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._160 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+		.uhr_cap = {
+			.has_uhr = true,
+			.mac.mac_cap = {
+				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP,
+			},
+			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
+				   IEEE80211_UHR_PHY_CAP_ELR_TX,
+		},
+	},
+	{
+		.types_mask = BIT(NL80211_IFTYPE_AP) |
+			      BIT(NL80211_IFTYPE_P2P_GO),
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_BSR |
+					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] =
+					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xfffa),
+				.tx_mcs_160 = cpu_to_le16(0xfffa),
+				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] =
+					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
+					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
+					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
+					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
+				.phy_cap_info[1] =
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK,
+				.phy_cap_info[2] =
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK,
+				.phy_cap_info[3] =
+					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+				.phy_cap_info[4] =
+					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
+					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
+					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
+					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
+				.phy_cap_info[5] =
+					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
+					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
+					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
+				.phy_cap_info[6] =
+					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
+					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK,
+				.phy_cap_info[7] =
+					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ,
+			},
+
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/*
+				 * As B1 and B2 are set in the supported
+				 * channel width set field in the HE PHY
+				 * capabilities information field include all
+				 * the following MCS/NSS.
+				 */
+				.bw._80 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._160 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+		.uhr_cap = {
+			.has_uhr = true,
+			.mac.mac_cap = {
+				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP,
+			},
+			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
+				   IEEE80211_UHR_PHY_CAP_ELR_TX,
+		},
+	},
+#ifdef CONFIG_MAC80211_MESH
+	{
+		/* TODO: should we support other types, e.g., IBSS?*/
+		.types_mask = BIT(NL80211_IFTYPE_MESH_POINT),
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] = 0,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xfffa),
+				.tx_mcs_160 = cpu_to_le16(0xfffa),
+				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
+			},
+		},
+	},
+#endif
+};
+
+static const struct ieee80211_sband_iftype_data sband_capa_6ghz[] = {
+	{
+		.types_mask = BIT(NL80211_IFTYPE_STATION) |
+			      BIT(NL80211_IFTYPE_P2P_CLIENT),
+		.he_6ghz_capa = {
+			.capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START |
+					    IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP |
+					    IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN |
+					    IEEE80211_HE_6GHZ_CAP_SM_PS |
+					    IEEE80211_HE_6GHZ_CAP_RD_RESPONDER |
+					    IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS |
+					    IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS),
+		},
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_BSR |
+					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] =
+					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xfffa),
+				.tx_mcs_160 = cpu_to_le16(0xfffa),
+				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] =
+					IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ |
+					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
+					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
+					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
+					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
+				.phy_cap_info[1] =
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK,
+				.phy_cap_info[2] =
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK,
+				.phy_cap_info[3] =
+					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+				.phy_cap_info[4] =
+					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
+					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
+					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
+					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
+				.phy_cap_info[5] =
+					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
+					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
+					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
+				.phy_cap_info[6] =
+					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
+					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK |
+					IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP,
+				.phy_cap_info[7] =
+					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_320MHZ,
+			},
+
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/*
+				 * As B1 and B2 are set in the supported
+				 * channel width set field in the HE PHY
+				 * capabilities information field and 320MHz in
+				 * 6GHz is supported include all the following
+				 * MCS/NSS.
+				 */
+				.bw._80 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._160 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._320 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+		.uhr_cap = {
+			.has_uhr = true,
+			.mac.mac_cap = {
+				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP,
+			},
+			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
+				   IEEE80211_UHR_PHY_CAP_ELR_TX,
+		},
+	},
+	{
+		.types_mask = BIT(NL80211_IFTYPE_AP) |
+			      BIT(NL80211_IFTYPE_P2P_GO),
+		.he_6ghz_capa = {
+			.capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START |
+					    IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP |
+					    IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN |
+					    IEEE80211_HE_6GHZ_CAP_SM_PS |
+					    IEEE80211_HE_6GHZ_CAP_RD_RESPONDER |
+					    IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS |
+					    IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS),
+		},
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_BSR |
+					IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] =
+					IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+					IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+					IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+					IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xfffa),
+				.tx_mcs_160 = cpu_to_le16(0xfffa),
+				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_EHT_MAC_CAP0_EPCS_PRIO_ACCESS |
+					IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+					IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] =
+					IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ |
+					IEEE80211_EHT_PHY_CAP0_242_TONE_RU_GT20MHZ |
+					IEEE80211_EHT_PHY_CAP0_NDP_4_EHT_LFT_32_GI |
+					IEEE80211_EHT_PHY_CAP0_PARTIAL_BW_UL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMER |
+					IEEE80211_EHT_PHY_CAP0_SU_BEAMFORMEE |
+					IEEE80211_EHT_PHY_CAP0_BEAMFORMEE_SS_80MHZ_MASK,
+				.phy_cap_info[1] =
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_160MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP1_BEAMFORMEE_SS_320MHZ_MASK,
+				.phy_cap_info[2] =
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_80MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_160MHZ_MASK |
+					IEEE80211_EHT_PHY_CAP2_SOUNDING_DIM_320MHZ_MASK,
+				.phy_cap_info[3] =
+					IEEE80211_EHT_PHY_CAP3_NG_16_SU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_NG_16_MU_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_4_2_SU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_CODEBOOK_7_5_MU_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_SU_BF_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_MU_BF_PART_BW_FDBK |
+					IEEE80211_EHT_PHY_CAP3_TRIG_CQI_FDBK,
+				.phy_cap_info[4] =
+					IEEE80211_EHT_PHY_CAP4_PART_BW_DL_MU_MIMO |
+					IEEE80211_EHT_PHY_CAP4_PSR_SR_SUPP |
+					IEEE80211_EHT_PHY_CAP4_POWER_BOOST_FACT_SUPP |
+					IEEE80211_EHT_PHY_CAP4_EHT_MU_PPDU_4_EHT_LTF_08_GI |
+					IEEE80211_EHT_PHY_CAP4_MAX_NC_MASK,
+				.phy_cap_info[5] =
+					IEEE80211_EHT_PHY_CAP5_NON_TRIG_CQI_FEEDBACK |
+					IEEE80211_EHT_PHY_CAP5_TX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_RX_LESS_242_TONE_RU_SUPP |
+					IEEE80211_EHT_PHY_CAP5_PPE_THRESHOLD_PRESENT |
+					IEEE80211_EHT_PHY_CAP5_COMMON_NOMINAL_PKT_PAD_MASK |
+					IEEE80211_EHT_PHY_CAP5_MAX_NUM_SUPP_EHT_LTF_MASK,
+				.phy_cap_info[6] =
+					IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK |
+					IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK |
+					IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP,
+				.phy_cap_info[7] =
+					IEEE80211_EHT_PHY_CAP7_20MHZ_STA_RX_NDP_WIDER_BW |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
+					IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_80MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_160MHZ |
+					IEEE80211_EHT_PHY_CAP7_MU_BEAMFORMER_320MHZ,
+			},
+
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/*
+				 * As B1 and B2 are set in the supported
+				 * channel width set field in the HE PHY
+				 * capabilities information field and 320MHz in
+				 * 6GHz is supported include all the following
+				 * MCS/NSS.
+				 */
+				.bw._80 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._160 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._320 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+		.uhr_cap = {
+			.has_uhr = true,
+			.mac.mac_cap = {
+				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP,
+			},
+			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
+				   IEEE80211_UHR_PHY_CAP_ELR_TX,
+		},
+	},
+#ifdef CONFIG_MAC80211_MESH
+	{
+		/* TODO: should we support other types, e.g., IBSS?*/
+		.types_mask = BIT(NL80211_IFTYPE_MESH_POINT),
+		.he_6ghz_capa = {
+			.capa = cpu_to_le16(IEEE80211_HE_6GHZ_CAP_MIN_MPDU_START |
+					    IEEE80211_HE_6GHZ_CAP_MAX_AMPDU_LEN_EXP |
+					    IEEE80211_HE_6GHZ_CAP_MAX_MPDU_LEN |
+					    IEEE80211_HE_6GHZ_CAP_SM_PS |
+					    IEEE80211_HE_6GHZ_CAP_RD_RESPONDER |
+					    IEEE80211_HE_6GHZ_CAP_TX_ANTPAT_CONS |
+					    IEEE80211_HE_6GHZ_CAP_RX_ANTPAT_CONS),
+		},
+		.he_cap = {
+			.has_he = true,
+			.he_cap_elem = {
+				.mac_cap_info[0] =
+					IEEE80211_HE_MAC_CAP0_HTC_HE,
+				.mac_cap_info[1] =
+					IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+				.mac_cap_info[2] =
+					IEEE80211_HE_MAC_CAP2_ACK_EN,
+				.mac_cap_info[3] =
+					IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+					IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+				.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+				.phy_cap_info[0] =
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
+					IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
+				.phy_cap_info[1] =
+					IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+					IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+					IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+					IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+				.phy_cap_info[2] = 0,
+
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			.he_mcs_nss_supp = {
+				.rx_mcs_80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80 = cpu_to_le16(0xfffa),
+				.rx_mcs_160 = cpu_to_le16(0xfffa),
+				.tx_mcs_160 = cpu_to_le16(0xfffa),
+				.rx_mcs_80p80 = cpu_to_le16(0xfffa),
+				.tx_mcs_80p80 = cpu_to_le16(0xfffa),
+			},
+		},
+		.eht_cap = {
+			.has_eht = true,
+			.eht_cap_elem = {
+				.mac_cap_info[0] = IEEE80211_EHT_MAC_CAP0_OM_CONTROL |
+						   IEEE80211_EHT_MAC_CAP0_TRIG_TXOP_SHARING_MODE1,
+				.phy_cap_info[0] = IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ,
+				/* Leave all the other PHY capability bytes
+				 * unset, as DCM, beam forming, RU and PPE
+				 * threshold information are not supported
+				 */
+			},
+			/* For all MCS and bandwidth, set 8 NSS for both Tx and
+			 * Rx
+			 */
+			.eht_mcs_nss_supp = {
+				/* As B1 and B2 are set in the supported
+				 * channel width set field in the HE PHY
+				 * capabilities information field and 320MHz in
+				 * 6GHz is supported include all the following
+				 * MCS/NSS.
+				 */
+				.bw._80 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._160 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+				.bw._320 = {
+					.rx_tx_mcs9_max_nss = 0x88,
+					.rx_tx_mcs11_max_nss = 0x88,
+					.rx_tx_mcs13_max_nss = 0x88,
+				},
+			},
+			/* PPE threshold information is not supported */
+		},
+		.uhr_cap = {
+			.has_uhr = true,
+			.mac.mac_cap = {
+				[0] = IEEE80211_UHR_MAC_CAP0_NPCA_SUPP,
+			},
+			.phy.cap = IEEE80211_UHR_PHY_CAP_ELR_RX |
+				   IEEE80211_UHR_PHY_CAP_ELR_TX,
+		},
+	},
+#endif
+};
+
+#define HWSIM_VHT_MCS_MAP				\
+	(IEEE80211_VHT_MCS_SUPPORT_0_9 << 0 |		\
+	 IEEE80211_VHT_MCS_SUPPORT_0_9 << 2 |		\
+	 IEEE80211_VHT_MCS_SUPPORT_0_9 << 4 |		\
+	 IEEE80211_VHT_MCS_SUPPORT_0_9 << 6 |		\
+	 IEEE80211_VHT_MCS_SUPPORT_0_9 << 8 |		\
+	 IEEE80211_VHT_MCS_SUPPORT_0_9 << 10 |		\
+	 IEEE80211_VHT_MCS_SUPPORT_0_9 << 12 |		\
+	 IEEE80211_VHT_MCS_SUPPORT_0_9 << 14)
+
+static const struct ieee80211_sta_ht_cap hwsim_nan_ht_cap = {
+	.ht_supported = true,
+	.cap = IEEE80211_HT_CAP_SUP_WIDTH_20_40 |
+	       IEEE80211_HT_CAP_GRN_FLD |
+	       IEEE80211_HT_CAP_SGI_20 |
+	       IEEE80211_HT_CAP_SGI_40 |
+	       IEEE80211_HT_CAP_DSSSCCK40,
+	.ampdu_factor = 0x3,
+	.ampdu_density = 0x6,
+	.mcs = {
+		.rx_mask = { 0xff, 0xff },
+		.tx_params = IEEE80211_HT_MCS_TX_DEFINED,
+	},
+};
+
+static const struct ieee80211_sta_vht_cap hwsim_nan_vht_cap = {
+	.vht_supported = true,
+	.cap = IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454 |
+	       IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ |
+	       IEEE80211_VHT_CAP_RXLDPC |
+	       IEEE80211_VHT_CAP_SHORT_GI_80 |
+	       IEEE80211_VHT_CAP_SHORT_GI_160 |
+	       IEEE80211_VHT_CAP_TXSTBC |
+	       IEEE80211_VHT_CAP_RXSTBC_4 |
+	       IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK,
+	.vht_mcs = {
+		.rx_mcs_map = cpu_to_le16(HWSIM_VHT_MCS_MAP),
+		.tx_mcs_map = cpu_to_le16(HWSIM_VHT_MCS_MAP),
+	},
+};
+
+static const struct ieee80211_sta_he_cap hwsim_nan_he_cap = {
+	.has_he = true,
+	.he_cap_elem = {
+		.mac_cap_info[0] =
+			IEEE80211_HE_MAC_CAP0_HTC_HE,
+		.mac_cap_info[1] =
+			IEEE80211_HE_MAC_CAP1_TF_MAC_PAD_DUR_16US |
+			IEEE80211_HE_MAC_CAP1_MULTI_TID_AGG_RX_QOS_8,
+		.mac_cap_info[2] =
+			IEEE80211_HE_MAC_CAP2_BSR |
+			IEEE80211_HE_MAC_CAP2_MU_CASCADING |
+			IEEE80211_HE_MAC_CAP2_ACK_EN,
+		.mac_cap_info[3] =
+			IEEE80211_HE_MAC_CAP3_OMI_CONTROL |
+			IEEE80211_HE_MAC_CAP3_MAX_AMPDU_LEN_EXP_EXT_3,
+		.mac_cap_info[4] = IEEE80211_HE_MAC_CAP4_AMSDU_IN_AMPDU,
+		.phy_cap_info[0] =
+			IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G |
+			IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G |
+			IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G,
+		.phy_cap_info[1] =
+			IEEE80211_HE_PHY_CAP1_PREAMBLE_PUNC_RX_MASK |
+			IEEE80211_HE_PHY_CAP1_DEVICE_CLASS_A |
+			IEEE80211_HE_PHY_CAP1_LDPC_CODING_IN_PAYLOAD |
+			IEEE80211_HE_PHY_CAP1_MIDAMBLE_RX_TX_MAX_NSTS,
+		.phy_cap_info[2] =
+			IEEE80211_HE_PHY_CAP2_NDP_4x_LTF_AND_3_2US |
+			IEEE80211_HE_PHY_CAP2_STBC_TX_UNDER_80MHZ |
+			IEEE80211_HE_PHY_CAP2_STBC_RX_UNDER_80MHZ |
+			IEEE80211_HE_PHY_CAP2_UL_MU_FULL_MU_MIMO |
+			IEEE80211_HE_PHY_CAP2_UL_MU_PARTIAL_MU_MIMO,
+
+		/*
+		 * Leave all the other PHY capability bytes
+		 * unset, as DCM, beam forming, RU and PPE
+		 * threshold information are not supported
+		 */
+	},
+	.he_mcs_nss_supp = {
+		.rx_mcs_80 = cpu_to_le16(0xfffa),
+		.tx_mcs_80 = cpu_to_le16(0xfffa),
+		.rx_mcs_160 = cpu_to_le16(0xfffa),
+		.tx_mcs_160 = cpu_to_le16(0xfffa),
+		.rx_mcs_80p80 = cpu_to_le16(0xfffa),
+		.tx_mcs_80p80 = cpu_to_le16(0xfffa),
+	},
+};
+
+static void mac80211_hwsim_sband_capab(struct ieee80211_supported_band *sband)
+{
+	switch (sband->band) {
+	case NL80211_BAND_2GHZ:
+		ieee80211_set_sband_iftype_data(sband, sband_capa_2ghz);
+		break;
+	case NL80211_BAND_5GHZ:
+		ieee80211_set_sband_iftype_data(sband, sband_capa_5ghz);
+		break;
+	case NL80211_BAND_6GHZ:
+		ieee80211_set_sband_iftype_data(sband, sband_capa_6ghz);
+		break;
+	default:
+		break;
+	}
+}
+
+#ifdef CONFIG_MAC80211_MESH
+#define HWSIM_MESH_BIT BIT(NL80211_IFTYPE_MESH_POINT)
+#else
+#define HWSIM_MESH_BIT 0
+#endif
+
+#define HWSIM_DEFAULT_IF_LIMIT \
+	(BIT(NL80211_IFTYPE_STATION) | \
+	 BIT(NL80211_IFTYPE_P2P_CLIENT) | \
+	 BIT(NL80211_IFTYPE_AP) | \
+	 BIT(NL80211_IFTYPE_P2P_GO) | \
+	 HWSIM_MESH_BIT)
+
+#define HWSIM_IFTYPE_SUPPORT_MASK \
+	(BIT(NL80211_IFTYPE_STATION) | \
+	 BIT(NL80211_IFTYPE_AP) | \
+	 BIT(NL80211_IFTYPE_P2P_CLIENT) | \
+	 BIT(NL80211_IFTYPE_P2P_GO) | \
+	 BIT(NL80211_IFTYPE_ADHOC) | \
+	 BIT(NL80211_IFTYPE_MESH_POINT) | \
+	 BIT(NL80211_IFTYPE_OCB))
+
+static const u8 iftypes_ext_capa_ap[] = {
+	 [0] = WLAN_EXT_CAPA1_EXT_CHANNEL_SWITCHING,
+	 [2] = WLAN_EXT_CAPA3_MULTI_BSSID_SUPPORT,
+	 [7] = WLAN_EXT_CAPA8_OPMODE_NOTIF |
+	       WLAN_EXT_CAPA8_MAX_MSDU_IN_AMSDU_LSB,
+	 [8] = WLAN_EXT_CAPA9_MAX_MSDU_IN_AMSDU_MSB,
+	 [9] = WLAN_EXT_CAPA10_TWT_RESPONDER_SUPPORT,
+};
+
+#define MAC80211_HWSIM_MLD_CAPA_OPS				\
+	FIELD_PREP_CONST(IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP, \
+			 IEEE80211_MLD_CAP_OP_TID_TO_LINK_MAP_NEG_SUPP_SAME) | \
+	FIELD_PREP_CONST(IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS, \
+			 IEEE80211_MLD_MAX_NUM_LINKS - 1)
+
+static const struct wiphy_iftype_ext_capab mac80211_hwsim_iftypes_ext_capa[] = {
+	{
+		.iftype = NL80211_IFTYPE_AP,
+		.extended_capabilities = iftypes_ext_capa_ap,
+		.extended_capabilities_mask = iftypes_ext_capa_ap,
+		.extended_capabilities_len = sizeof(iftypes_ext_capa_ap),
+		.eml_capabilities = IEEE80211_EML_CAP_EMLSR_SUPP |
+				    IEEE80211_EML_CAP_EMLMR_SUPPORT,
+		.mld_capa_and_ops = MAC80211_HWSIM_MLD_CAPA_OPS,
+	},
+};
+
+static int mac80211_hwsim_new_radio(struct genl_info *info,
+				    struct hwsim_new_radio_params *param)
+{
+	int err;
+	u8 addr[ETH_ALEN];
+	struct mac80211_hwsim_data *data;
+	struct ieee80211_hw *hw;
+	enum nl80211_band band;
+	const struct ieee80211_ops *ops = &mac80211_hwsim_ops;
+	struct net *net;
+	int idx, i;
+	int n_limits = 0;
+	int n_bands = 0;
+
+	if (WARN_ON(param->channels > 1 && !param->use_chanctx))
+		return -EINVAL;
+
+	spin_lock_bh(&hwsim_radio_lock);
+	idx = hwsim_radio_idx++;
+	spin_unlock_bh(&hwsim_radio_lock);
+
+	if (param->mlo)
+		ops = &mac80211_hwsim_mlo_ops;
+	else if (param->use_chanctx)
+		ops = &mac80211_hwsim_mchan_ops;
+	hw = ieee80211_alloc_hw_nm(sizeof(*data), ops, param->hwname);
+	if (!hw) {
+		pr_debug("mac80211_hwsim: ieee80211_alloc_hw failed\n");
+		err = -ENOMEM;
+		goto failed;
+	}
+
+	/* ieee80211_alloc_hw_nm may have used a default name */
+	param->hwname = wiphy_name(hw->wiphy);
+
+	if (info)
+		net = genl_info_net(info);
+	else
+		net = &init_net;
+	wiphy_net_set(hw->wiphy, net);
+
+	data = hw->priv;
+	data->hw = hw;
+
+	data->dev = device_create(&hwsim_class, NULL, 0, hw, "hwsim%d", idx);
+	if (IS_ERR(data->dev)) {
+		printk(KERN_DEBUG
+		       "mac80211_hwsim: device_create failed (%ld)\n",
+		       PTR_ERR(data->dev));
+		err = -ENOMEM;
+		goto failed_drvdata;
+	}
+	data->dev->driver = &mac80211_hwsim_driver.driver;
+	err = device_bind_driver(data->dev);
+	if (err != 0) {
+		pr_debug("mac80211_hwsim: device_bind_driver failed (%d)\n",
+		       err);
+		goto failed_bind;
+	}
+
+	skb_queue_head_init(&data->pending);
+
+	SET_IEEE80211_DEV(hw, data->dev);
+	if (!param->perm_addr) {
+		eth_zero_addr(addr);
+		addr[0] = 0x02;
+		addr[3] = idx >> 8;
+		addr[4] = idx;
+		memcpy(data->addresses[0].addr, addr, ETH_ALEN);
+		/* Why need here second address ? */
+		memcpy(data->addresses[1].addr, addr, ETH_ALEN);
+		data->addresses[1].addr[0] |= 0x40;
+		memcpy(data->addresses[2].addr, addr, ETH_ALEN);
+		data->addresses[2].addr[0] |= 0x50;
+
+		hw->wiphy->n_addresses = 3;
+		hw->wiphy->addresses = data->addresses;
+		/* possible address clash is checked at hash table insertion */
+	} else {
+		memcpy(data->addresses[0].addr, param->perm_addr, ETH_ALEN);
+		/* compatibility with automatically generated mac addr */
+		memcpy(data->addresses[1].addr, param->perm_addr, ETH_ALEN);
+		memcpy(data->addresses[2].addr, param->perm_addr, ETH_ALEN);
+		hw->wiphy->n_addresses = 3;
+		hw->wiphy->addresses = data->addresses;
+	}
+
+	data->channels = param->channels;
+	data->use_chanctx = param->use_chanctx;
+	data->idx = idx;
+	data->destroy_on_close = param->destroy_on_close;
+	if (info)
+		data->portid = info->snd_portid;
+
+	/* setup interface limits, only on interface types we support */
+	if (param->iftypes & BIT(NL80211_IFTYPE_ADHOC)) {
+		data->if_limits[n_limits].max = 1;
+		data->if_limits[n_limits].types = BIT(NL80211_IFTYPE_ADHOC);
+		n_limits++;
+	}
+
+	if (param->iftypes & HWSIM_DEFAULT_IF_LIMIT) {
+		data->if_limits[n_limits].max = 2048;
+		/*
+		 * For this case, we may only support a subset of
+		 * HWSIM_DEFAULT_IF_LIMIT, therefore we only want to add the
+		 * bits that both param->iftype & HWSIM_DEFAULT_IF_LIMIT have.
+		 */
+		data->if_limits[n_limits].types =
+					HWSIM_DEFAULT_IF_LIMIT & param->iftypes;
+		n_limits++;
+	}
+
+	if (param->iftypes & BIT(NL80211_IFTYPE_P2P_DEVICE)) {
+		data->if_limits[n_limits].max = 1;
+		data->if_limits[n_limits].types =
+						BIT(NL80211_IFTYPE_P2P_DEVICE);
+		n_limits++;
+	}
+
+	if (param->iftypes & BIT(NL80211_IFTYPE_NAN)) {
+		data->if_limits[n_limits].max = 1;
+		data->if_limits[n_limits].types = BIT(NL80211_IFTYPE_NAN);
+		n_limits++;
+
+		hw->wiphy->nan_supported_bands = BIT(NL80211_BAND_2GHZ) |
+						 BIT(NL80211_BAND_5GHZ);
+
+		hw->wiphy->nan_capa.flags = WIPHY_NAN_FLAGS_CONFIGURABLE_SYNC |
+					    WIPHY_NAN_FLAGS_USERSPACE_DE;
+		hw->wiphy->nan_capa.op_mode = NAN_OP_MODE_PHY_MODE_MASK |
+					      NAN_OP_MODE_80P80MHZ |
+					      NAN_OP_MODE_160MHZ;
+
+		hw->wiphy->nan_capa.n_antennas = 0x22;
+		hw->wiphy->nan_capa.max_channel_switch_time = 0;
+
+		wiphy_ext_feature_set(hw->wiphy,
+				      NL80211_EXT_FEATURE_SECURE_NAN);
+
+		hrtimer_setup(&data->nan.slot_timer,
+			      mac80211_hwsim_nan_slot_timer,
+			      CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT);
+		hrtimer_setup(&data->nan.resume_txqs_timer,
+			      mac80211_hwsim_nan_resume_txqs_timer,
+			      CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT);
+		hrtimer_setup(&data->nan.discovery_beacon_timer,
+			      mac80211_hwsim_nan_discovery_beacon_timer,
+			      CLOCK_BOOTTIME, HRTIMER_MODE_ABS_SOFT);
+
+		spin_lock_init(&data->nan.state_lock);
+	}
+
+	if (param->iftypes & BIT(NL80211_IFTYPE_NAN_DATA)) {
+		data->if_limits[n_limits].max = 2;
+		data->if_limits[n_limits].types = BIT(NL80211_IFTYPE_NAN_DATA);
+		n_limits++;
+
+		hw->wiphy->nan_capa.phy.ht = hwsim_nan_ht_cap;
+		hw->wiphy->nan_capa.phy.vht = hwsim_nan_vht_cap;
+		hw->wiphy->nan_capa.phy.he = hwsim_nan_he_cap;
+
+		/*
+		 * NAN switches between bands/channels per its schedule,
+		 * so mac80211 rate control can't work here.
+		 */
+		ieee80211_hw_set(hw, HAS_RATE_CONTROL);
+	}
+
+	data->if_combination.radar_detect_widths =
+				BIT(NL80211_CHAN_WIDTH_5) |
+				BIT(NL80211_CHAN_WIDTH_10) |
+				BIT(NL80211_CHAN_WIDTH_20_NOHT) |
+				BIT(NL80211_CHAN_WIDTH_20) |
+				BIT(NL80211_CHAN_WIDTH_40) |
+				BIT(NL80211_CHAN_WIDTH_80) |
+				BIT(NL80211_CHAN_WIDTH_160);
+
+	if (data->use_chanctx) {
+		hw->wiphy->max_scan_ssids = 255;
+		hw->wiphy->max_scan_ie_len = IEEE80211_MAX_DATA_LEN;
+		hw->wiphy->max_remain_on_channel_duration = 1000;
+		data->if_combination.num_different_channels = data->channels;
+	} else {
+		data->if_combination.num_different_channels = 1;
+	}
+
+	if (!n_limits) {
+		err = -EINVAL;
+		goto failed_hw;
+	}
+
+	data->if_combination.max_interfaces = 0;
+	for (i = 0; i < n_limits; i++)
+		data->if_combination.max_interfaces +=
+			data->if_limits[i].max;
+
+	data->if_combination.n_limits = n_limits;
+	data->if_combination.limits = data->if_limits;
+
+	/*
+	 * If we actually were asked to support combinations,
+	 * advertise them - if there's only a single thing like
+	 * only IBSS then don't advertise it as combinations.
+	 */
+	if (data->if_combination.max_interfaces > 1) {
+		hw->wiphy->iface_combinations = &data->if_combination;
+		hw->wiphy->n_iface_combinations = 1;
+	}
+
+	if (param->ciphers) {
+		memcpy(data->ciphers, param->ciphers,
+		       param->n_ciphers * sizeof(u32));
+		hw->wiphy->cipher_suites = data->ciphers;
+		hw->wiphy->n_cipher_suites = param->n_ciphers;
+	}
+
+	hw->wiphy->mbssid_max_interfaces = 8;
+	hw->wiphy->ema_max_profile_periodicity = 3;
+
+	spin_lock_init(&data->tsf_offset_lock);
+
+	data->rx_rssi = DEFAULT_RX_RSSI;
+
+	INIT_DELAYED_WORK(&data->roc_start, hw_roc_start);
+	INIT_DELAYED_WORK(&data->roc_done, hw_roc_done);
+	INIT_DELAYED_WORK(&data->hw_scan, hw_scan_work);
+
+	hw->queues = 5;
+	hw->offchannel_tx_hw_queue = 4;
+
+	ieee80211_hw_set(hw, SUPPORT_FAST_XMIT);
+	ieee80211_hw_set(hw, CHANCTX_STA_CSA);
+	ieee80211_hw_set(hw, SUPPORTS_HT_CCK_RATES);
+	ieee80211_hw_set(hw, QUEUE_CONTROL);
+	ieee80211_hw_set(hw, WANT_MONITOR_VIF);
+	ieee80211_hw_set(hw, AMPDU_AGGREGATION);
+	ieee80211_hw_set(hw, MFP_CAPABLE);
+	ieee80211_hw_set(hw, SIGNAL_DBM);
+	ieee80211_hw_set(hw, SUPPORTS_PS);
+	ieee80211_hw_set(hw, REPORTS_TX_ACK_STATUS);
+	ieee80211_hw_set(hw, TDLS_WIDER_BW);
+	ieee80211_hw_set(hw, SUPPORTS_MULTI_BSSID);
+	ieee80211_hw_set(hw, STRICT);
+	ieee80211_hw_set(hw, BUFF_MMPDU_TXQ);
+	ieee80211_hw_set(hw, STA_MMPDU_TXQ);
+
+	if (param->mlo) {
+		hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_MLO;
+		ieee80211_hw_set(hw, HAS_RATE_CONTROL);
+		ieee80211_hw_set(hw, SUPPORTS_DYNAMIC_PS);
+		ieee80211_hw_set(hw, CONNECTION_MONITOR);
+		ieee80211_hw_set(hw, AP_LINK_PS);
+
+		hw->wiphy->iftype_ext_capab = mac80211_hwsim_iftypes_ext_capa;
+		hw->wiphy->num_iftype_ext_capab =
+			ARRAY_SIZE(mac80211_hwsim_iftypes_ext_capa);
+	} else {
+		ieee80211_hw_set(hw, HOST_BROADCAST_PS_BUFFERING);
+		ieee80211_hw_set(hw, PS_NULLFUNC_STACK);
+		if (rctbl)
+			ieee80211_hw_set(hw, SUPPORTS_RC_TABLE);
+	}
+
+	hw->wiphy->flags &= ~WIPHY_FLAG_PS_ON_BY_DEFAULT;
+	hw->wiphy->flags |= WIPHY_FLAG_SUPPORTS_TDLS |
+			    WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL |
+			    WIPHY_FLAG_AP_UAPSD |
+			    WIPHY_FLAG_SUPPORTS_5_10_MHZ |
+			    WIPHY_FLAG_HAS_CHANNEL_SWITCH;
+	hw->wiphy->flags |= WIPHY_FLAG_IBSS_RSN;
+	hw->wiphy->features |= NL80211_FEATURE_ACTIVE_MONITOR |
+			       NL80211_FEATURE_AP_MODE_CHAN_WIDTH_CHANGE |
+			       NL80211_FEATURE_STATIC_SMPS |
+			       NL80211_FEATURE_DYNAMIC_SMPS |
+			       NL80211_FEATURE_SCAN_RANDOM_MAC_ADDR |
+			       NL80211_FEATURE_AP_SCAN;
+	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_VHT_IBSS);
+	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_BEACON_PROTECTION);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_MULTICAST_REGISTRATIONS);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_BEACON_RATE_LEGACY);
+	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER);
+
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_SCAN_MIN_PREQ_CONTENT);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_BSS_COLOR);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_SPP_AMSDU_SUPPORT);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_CAN_REPLACE_PTK0);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_EXT_KEY_ID);
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_ASSOC_FRAME_ENCRYPTION);
+
+	hw->wiphy->interface_modes = param->iftypes;
+
+	/* ask mac80211 to reserve space for magic */
+	hw->vif_data_size = sizeof(struct hwsim_vif_priv);
+	hw->sta_data_size = sizeof(struct hwsim_sta_priv);
+	hw->chanctx_data_size = sizeof(struct hwsim_chanctx_priv);
+	hw->txq_data_size = 0;
+
+	memcpy(data->channels_2ghz, hwsim_channels_2ghz,
+		sizeof(hwsim_channels_2ghz));
+	memcpy(data->channels_5ghz, hwsim_channels_5ghz,
+		sizeof(hwsim_channels_5ghz));
+	memcpy(data->channels_6ghz, hwsim_channels_6ghz,
+		sizeof(hwsim_channels_6ghz));
+	memcpy(data->channels_s1g, hwsim_channels_s1g,
+	       sizeof(hwsim_channels_s1g));
+	memcpy(data->rates, hwsim_rates, sizeof(hwsim_rates));
+
+	for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) {
+		struct ieee80211_supported_band *sband = &data->bands[band];
+		struct wiphy_radio_freq_range *radio_range;
+		const struct ieee80211_channel *c;
+		struct wiphy_radio *radio;
+
+		sband->band = band;
+
+		switch (band) {
+		case NL80211_BAND_2GHZ:
+			sband->channels = data->channels_2ghz;
+			sband->n_channels = ARRAY_SIZE(hwsim_channels_2ghz);
+			sband->bitrates = data->rates;
+			sband->n_bitrates = ARRAY_SIZE(hwsim_rates);
+			break;
+		case NL80211_BAND_5GHZ:
+			sband->channels = data->channels_5ghz;
+			sband->n_channels = ARRAY_SIZE(hwsim_channels_5ghz);
+			sband->bitrates = data->rates + 4;
+			sband->n_bitrates = ARRAY_SIZE(hwsim_rates) - 4;
+
+			sband->vht_cap.vht_supported = true;
+			sband->vht_cap.cap =
+				IEEE80211_VHT_CAP_MAX_MPDU_LENGTH_11454 |
+				IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ |
+				IEEE80211_VHT_CAP_RXLDPC |
+				IEEE80211_VHT_CAP_SHORT_GI_80 |
+				IEEE80211_VHT_CAP_SHORT_GI_160 |
+				IEEE80211_VHT_CAP_TXSTBC |
+				IEEE80211_VHT_CAP_RXSTBC_4 |
+				IEEE80211_VHT_CAP_MAX_A_MPDU_LENGTH_EXPONENT_MASK;
+			sband->vht_cap.vht_mcs.rx_mcs_map =
+				cpu_to_le16(HWSIM_VHT_MCS_MAP);
+			sband->vht_cap.vht_mcs.tx_mcs_map =
+				sband->vht_cap.vht_mcs.rx_mcs_map;
+			break;
+		case NL80211_BAND_6GHZ:
+			sband->channels = data->channels_6ghz;
+			sband->n_channels = ARRAY_SIZE(hwsim_channels_6ghz);
+			sband->bitrates = data->rates + 4;
+			sband->n_bitrates = ARRAY_SIZE(hwsim_rates) - 4;
+			break;
+		case NL80211_BAND_S1GHZ:
+			memcpy(&sband->s1g_cap, &hwsim_s1g_cap,
+			       sizeof(sband->s1g_cap));
+			sband->channels = data->channels_s1g;
+			sband->n_channels = ARRAY_SIZE(hwsim_channels_s1g);
+			break;
+		default:
+			continue;
+		}
+
+		if (band != NL80211_BAND_6GHZ){
+			sband->ht_cap.ht_supported = true;
+			sband->ht_cap.cap = IEEE80211_HT_CAP_SUP_WIDTH_20_40 |
+					    IEEE80211_HT_CAP_GRN_FLD |
+					    IEEE80211_HT_CAP_SGI_20 |
+					    IEEE80211_HT_CAP_SGI_40 |
+					    IEEE80211_HT_CAP_DSSSCCK40 |
+					    IEEE80211_HT_CAP_TX_STBC |
+					    IEEE80211_HT_CAP_RX_STBC;
+			sband->ht_cap.ampdu_factor = 0x3;
+			sband->ht_cap.ampdu_density = 0x6;
+			memset(&sband->ht_cap.mcs, 0,
+			       sizeof(sband->ht_cap.mcs));
+			sband->ht_cap.mcs.rx_mask[0] = 0xff;
+			sband->ht_cap.mcs.rx_mask[1] = 0xff;
+			sband->ht_cap.mcs.tx_params = IEEE80211_HT_MCS_TX_DEFINED;
+		}
+
+		mac80211_hwsim_sband_capab(sband);
+
+		hw->wiphy->bands[band] = sband;
+
+		if (!param->multi_radio)
+			continue;
+
+		c = sband->channels;
+		radio_range = &data->radio_range[n_bands];
+		radio_range->start_freq = ieee80211_channel_to_khz(c) - 10000;
+
+		c += sband->n_channels - 1;
+		radio_range->end_freq = ieee80211_channel_to_khz(c) + 10000;
+
+		radio = &data->radio[n_bands++];
+		radio->freq_range = radio_range;
+		radio->n_freq_range = 1;
+		radio->iface_combinations = &data->if_combination_radio;
+		radio->n_iface_combinations = 1;
+	}
+
+	if (param->multi_radio) {
+		hw->wiphy->radio = data->radio;
+		hw->wiphy->n_radio = n_bands;
+
+		memcpy(&data->if_combination_radio, &data->if_combination,
+		       sizeof(data->if_combination));
+		data->if_combination.num_different_channels *= n_bands;
+	}
+
+	if (data->use_chanctx)
+		data->if_combination.radar_detect_widths = 0;
+
+	/* By default all radios belong to the first group */
+	data->group = 1;
+	mutex_init(&data->mutex);
+
+	data->netgroup = hwsim_net_get_netgroup(net);
+	data->wmediumd = hwsim_net_get_wmediumd(net);
+
+	/* Enable frame retransmissions for lossy channels */
+	hw->max_rates = 4;
+	hw->max_rate_tries = 11;
+
+	hw->wiphy->vendor_commands = mac80211_hwsim_vendor_commands;
+	hw->wiphy->n_vendor_commands =
+		ARRAY_SIZE(mac80211_hwsim_vendor_commands);
+	hw->wiphy->vendor_events = mac80211_hwsim_vendor_events;
+	hw->wiphy->n_vendor_events = ARRAY_SIZE(mac80211_hwsim_vendor_events);
+
+	if (param->reg_strict)
+		hw->wiphy->regulatory_flags |= REGULATORY_STRICT_REG;
+	if (param->regd) {
+		data->regd = param->regd;
+		hw->wiphy->regulatory_flags |= REGULATORY_CUSTOM_REG;
+		wiphy_apply_custom_regulatory(hw->wiphy, param->regd);
+		/* give the regulatory workqueue a chance to run */
+		schedule_timeout_interruptible(1);
+	}
+
+	wiphy_ext_feature_set(hw->wiphy,
+			      NL80211_EXT_FEATURE_DFS_CONCURRENT);
+	if (param->background_radar)
+		wiphy_ext_feature_set(hw->wiphy,
+				      NL80211_EXT_FEATURE_RADAR_BACKGROUND);
+
+	if (param->no_vif)
+		ieee80211_hw_set(hw, NO_AUTO_VIF);
+
+	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST);
+	wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_PUNCT);
+
+	for (i = 0; i < ARRAY_SIZE(data->link_data); i++) {
+		hrtimer_setup(&data->link_data[i].beacon_timer, mac80211_hwsim_beacon,
+			      CLOCK_MONOTONIC, HRTIMER_MODE_ABS_SOFT);
+		data->link_data[i].link_id = i;
+	}
+
+	err = ieee80211_register_hw(hw);
+	if (err < 0) {
+		pr_debug("mac80211_hwsim: ieee80211_register_hw failed (%d)\n",
+		       err);
+		goto failed_hw;
+	}
+
+	wiphy_dbg(hw->wiphy, "hwaddr %pM registered\n", hw->wiphy->perm_addr);
+
+	if (param->reg_alpha2) {
+		data->alpha2[0] = param->reg_alpha2[0];
+		data->alpha2[1] = param->reg_alpha2[1];
+		regulatory_hint(hw->wiphy, param->reg_alpha2);
+	}
+
+	data->debugfs = debugfs_create_dir("hwsim", hw->wiphy->debugfsdir);
+	debugfs_create_file("ps", 0666, data->debugfs, data, &hwsim_fops_ps);
+	debugfs_create_file("group", 0666, data->debugfs, data,
+			    &hwsim_fops_group);
+	debugfs_create_file("rx_rssi", 0666, data->debugfs, data,
+			    &hwsim_fops_rx_rssi);
+	if (!data->use_chanctx)
+		debugfs_create_file("dfs_simulate_radar", 0222,
+				    data->debugfs,
+				    data, &hwsim_simulate_radar);
+	if (param->background_radar)
+		debugfs_create_file("dfs_background_cac", 0200,
+				    data->debugfs,
+				    data, &hwsim_background_cac_ops);
+	debugfs_create_file("simulate_incumbent_signal_interference", 0200,
+			    data->debugfs,
+			    data, &hwsim_simulate_incumbent_signal_fops);
+
+	if (param->pmsr_capa) {
+		data->pmsr_capa = *param->pmsr_capa;
+		hw->wiphy->pmsr_capa = &data->pmsr_capa;
+	}
+
+	spin_lock_bh(&hwsim_radio_lock);
+	err = rhashtable_insert_fast(&hwsim_radios_rht, &data->rht,
+				     hwsim_rht_params);
+	if (err < 0) {
+		if (info) {
+			GENL_SET_ERR_MSG(info, "perm addr already present");
+			NL_SET_BAD_ATTR(info->extack,
+					info->attrs[HWSIM_ATTR_PERM_ADDR]);
+		}
+		spin_unlock_bh(&hwsim_radio_lock);
+		goto failed_final_insert;
+	}
+
+	list_add_tail(&data->list, &hwsim_radios);
+	hwsim_radios_generation++;
+	spin_unlock_bh(&hwsim_radio_lock);
+
+	hwsim_mcast_new_radio(idx, info, param);
+
+	return idx;
+
+failed_final_insert:
+	debugfs_remove_recursive(data->debugfs);
+	ieee80211_unregister_hw(data->hw);
+failed_hw:
+	device_release_driver(data->dev);
+failed_bind:
+	device_unregister(data->dev);
+failed_drvdata:
+	ieee80211_free_hw(hw);
+failed:
+	return err;
+}
+
+static void hwsim_mcast_del_radio(int id, const char *hwname,
+				  struct genl_info *info)
+{
+	struct sk_buff *skb;
+	void *data;
+	int ret;
+
+	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+	if (!skb)
+		return;
+
+	data = genlmsg_put(skb, 0, 0, &hwsim_genl_family, 0,
+			   HWSIM_CMD_DEL_RADIO);
+	if (!data)
+		goto error;
+
+	ret = nla_put_u32(skb, HWSIM_ATTR_RADIO_ID, id);
+	if (ret < 0)
+		goto error;
+
+	ret = nla_put(skb, HWSIM_ATTR_RADIO_NAME, strlen(hwname),
+		      hwname);
+	if (ret < 0)
+		goto error;
+
+	genlmsg_end(skb, data);
+
+	hwsim_mcast_config_msg(skb, info);
+
+	return;
+
+error:
+	nlmsg_free(skb);
+}
+
+static void mac80211_hwsim_del_radio(struct mac80211_hwsim_data *data,
+				     const char *hwname,
+				     struct genl_info *info)
+{
+	hwsim_mcast_del_radio(data->idx, hwname, info);
+	debugfs_remove_recursive(data->debugfs);
+	ieee80211_unregister_hw(data->hw);
+	device_release_driver(data->dev);
+	device_unregister(data->dev);
+	ieee80211_free_hw(data->hw);
+}
+
+static int mac80211_hwsim_get_radio(struct sk_buff *skb,
+				    struct mac80211_hwsim_data *data,
+				    u32 portid, u32 seq,
+				    struct netlink_callback *cb, int flags)
+{
+	void *hdr;
+	struct hwsim_new_radio_params param = { };
+	int res = -EMSGSIZE;
+
+	hdr = genlmsg_put(skb, portid, seq, &hwsim_genl_family, flags,
+			  HWSIM_CMD_GET_RADIO);
+	if (!hdr)
+		return -EMSGSIZE;
+
+	if (cb)
+		genl_dump_check_consistent(cb, hdr);
+
+	if (data->alpha2[0] && data->alpha2[1])
+		param.reg_alpha2 = data->alpha2;
+
+	param.reg_strict = !!(data->hw->wiphy->regulatory_flags &
+					REGULATORY_STRICT_REG);
+	param.p2p_device = !!(data->hw->wiphy->interface_modes &
+					BIT(NL80211_IFTYPE_P2P_DEVICE));
+	param.nan_device = !!(data->hw->wiphy->interface_modes &
+					BIT(NL80211_IFTYPE_NAN));
+	param.use_chanctx = data->use_chanctx;
+	param.regd = data->regd;
+	param.channels = data->channels;
+	param.hwname = wiphy_name(data->hw->wiphy);
+	param.pmsr_capa = &data->pmsr_capa;
+	param.background_radar =
+		wiphy_ext_feature_isset(data->hw->wiphy,
+					NL80211_EXT_FEATURE_RADAR_BACKGROUND);
+
+	res = append_radio_msg(skb, data->idx, &param);
+	if (res < 0)
+		goto out_err;
+
+	genlmsg_end(skb, hdr);
+	return 0;
+
+out_err:
+	genlmsg_cancel(skb, hdr);
+	return res;
+}
+
+static void mac80211_hwsim_free(void)
+{
+	struct mac80211_hwsim_data *data;
+
+	spin_lock_bh(&hwsim_radio_lock);
+	while ((data = list_first_entry_or_null(&hwsim_radios,
+						struct mac80211_hwsim_data,
+						list))) {
+		list_del(&data->list);
+		spin_unlock_bh(&hwsim_radio_lock);
+		mac80211_hwsim_del_radio(data, wiphy_name(data->hw->wiphy),
+					 NULL);
+		spin_lock_bh(&hwsim_radio_lock);
+	}
+	spin_unlock_bh(&hwsim_radio_lock);
+	class_unregister(&hwsim_class);
+}
+
+static const struct net_device_ops hwsim_netdev_ops = {
+	.ndo_start_xmit 	= hwsim_mon_xmit,
+	.ndo_set_mac_address 	= eth_mac_addr,
+	.ndo_validate_addr	= eth_validate_addr,
+};
+
+static void hwsim_mon_setup(struct net_device *dev)
+{
+	u8 addr[ETH_ALEN];
+
+	dev->netdev_ops = &hwsim_netdev_ops;
+	dev->needs_free_netdev = true;
+	ether_setup(dev);
+	dev->priv_flags |= IFF_NO_QUEUE;
+	dev->type = ARPHRD_IEEE80211_RADIOTAP;
+	eth_zero_addr(addr);
+	addr[0] = 0x12;
+	eth_hw_addr_set(dev, addr);
+}
+
+static void hwsim_register_wmediumd(struct net *net, u32 portid)
+{
+	struct mac80211_hwsim_data *data;
+
+	hwsim_net_set_wmediumd(net, portid);
+
+	spin_lock_bh(&hwsim_radio_lock);
+	list_for_each_entry(data, &hwsim_radios, list) {
+		if (data->netgroup == hwsim_net_get_netgroup(net))
+			data->wmediumd = portid;
+	}
+	spin_unlock_bh(&hwsim_radio_lock);
+}
+
+static int hwsim_tx_info_frame_received_nl(struct sk_buff *skb_2,
+					   struct genl_info *info)
+{
+
+	struct ieee80211_hdr *hdr;
+	struct mac80211_hwsim_data *data2;
+	struct ieee80211_tx_info *txi;
+	struct hwsim_tx_rate *tx_attempts;
+	u64 ret_skb_cookie;
+	struct sk_buff *skb, *tmp;
+	const u8 *src;
+	unsigned int hwsim_flags;
+	int i;
+	unsigned long flags;
+	bool found = false;
+
+	if (!info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER] ||
+	    !info->attrs[HWSIM_ATTR_FLAGS] ||
+	    !info->attrs[HWSIM_ATTR_COOKIE] ||
+	    !info->attrs[HWSIM_ATTR_SIGNAL] ||
+	    !info->attrs[HWSIM_ATTR_TX_INFO])
+		goto out;
+
+	src = (void *)nla_data(info->attrs[HWSIM_ATTR_ADDR_TRANSMITTER]);
+	hwsim_flags = nla_get_u32(info->attrs[HWSIM_ATTR_FLAGS]);
+	ret_skb_cookie = nla_get_u64(info->attrs[HWSIM_ATTR_COOKIE]);
+
+	data2 = get_hwsim_data_ref_from_addr(src);
+	if (!data2)
+		goto out;
+
+	if (!hwsim_virtio_enabled) {
+		if (hwsim_net_get_netgroup(genl_info_net(info)) !=
+		    data2->netgroup)
+			goto out;
+
+		if (info->snd_portid != data2->wmediumd)
+			goto out;
+	}
+
+	/* look for the skb matching the cookie passed back from user */
+	spin_lock_irqsave(&data2->pending.lock, flags);
+	skb_queue_walk_safe(&data2->pending, skb, tmp) {
+		uintptr_t skb_cookie;
+
+		txi = IEEE80211_SKB_CB(skb);
+		skb_cookie = (uintptr_t)txi->rate_driver_data[0];
+
+		if (skb_cookie == ret_skb_cookie) {
+			__skb_unlink(skb, &data2->pending);
+			found = true;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&data2->pending.lock, flags);
+
+	/* not found */
+	if (!found)
+		goto out;
+
+	mac80211_hwsim_monitor_rx(data2->hw, skb, data2->channel);
+
+	/* Tx info received because the frame was broadcasted on user space,
+	 so we get all the necessary info: tx attempts and skb control buff */
+
+	tx_attempts = (struct hwsim_tx_rate *)nla_data(
+		       info->attrs[HWSIM_ATTR_TX_INFO]);
+
+	/* now send back TX status */
+	txi = IEEE80211_SKB_CB(skb);
+
+	ieee80211_tx_info_clear_status(txi);
+
+	for (i = 0; i < IEEE80211_TX_MAX_RATES; i++) {
+		txi->status.rates[i].idx = tx_attempts[i].idx;
+		txi->status.rates[i].count = tx_attempts[i].count;
+	}
+
+	txi->status.ack_signal = nla_get_u32(info->attrs[HWSIM_ATTR_SIGNAL]);
+
+	if (!(hwsim_flags & HWSIM_TX_CTL_NO_ACK) &&
+	   (hwsim_flags & HWSIM_TX_STAT_ACK)) {
+		if (skb->len >= 16) {
+			hdr = (struct ieee80211_hdr *) skb->data;
+			mac80211_hwsim_monitor_ack(data2->channel,
+						   hdr->addr2);
+		}
+		txi->flags |= IEEE80211_TX_STAT_ACK;
+	}
+
+	if (hwsim_flags & HWSIM_TX_CTL_NO_ACK)
+		txi->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
+
+	ieee80211_tx_status_irqsafe(data2->hw, skb);
+	return 0;
+out:
+	return -EINVAL;
+
+}
+
+static int hwsim_cloned_frame_received_nl(struct sk_buff *skb_2,
+					  struct genl_info *info)
+{
+	struct mac80211_hwsim_data *data2;
+	struct ieee80211_rx_status rx_status;
+	struct ieee80211_hdr *hdr;
+	const u8 *dst;
+	int frame_data_len;
+	void *frame_data;
+	struct sk_buff *skb = NULL;
+	struct ieee80211_channel *channel = NULL;
+
+	if (!info->attrs[HWSIM_ATTR_ADDR_RECEIVER] ||
+	    !info->attrs[HWSIM_ATTR_FRAME] ||
+	    !info->attrs[HWSIM_ATTR_RX_RATE] ||
+	    !info->attrs[HWSIM_ATTR_SIGNAL])
+		goto out;
+
+	dst = (void *)nla_data(info->attrs[HWSIM_ATTR_ADDR_RECEIVER]);
+	frame_data_len = nla_len(info->attrs[HWSIM_ATTR_FRAME]);
+	frame_data = (void *)nla_data(info->attrs[HWSIM_ATTR_FRAME]);
+
+	if (frame_data_len < sizeof(struct ieee80211_hdr_3addr) ||
+	    frame_data_len > IEEE80211_MAX_DATA_LEN)
+		goto err;
+
+	/* Allocate new skb here */
+	skb = alloc_skb(frame_data_len, GFP_KERNEL);
+	if (skb == NULL)
+		goto err;
+
+	/* Copy the data */
+	skb_put_data(skb, frame_data, frame_data_len);
+
+	data2 = get_hwsim_data_ref_from_addr(dst);
+	if (!data2)
+		goto out;
+
+	if (data2->use_chanctx) {
+		if (data2->tmp_chan)
+			channel = data2->tmp_chan;
+	} else {
+		channel = data2->channel;
+	}
+
+	if (!hwsim_virtio_enabled) {
+		if (hwsim_net_get_netgroup(genl_info_net(info)) !=
+		    data2->netgroup)
+			goto out;
+
+		if (info->snd_portid != data2->wmediumd)
+			goto out;
+	}
+
+	/* check if radio is configured properly */
+
+	if ((data2->idle && !data2->tmp_chan) || !data2->started)
+		goto out;
+
+	/* A frame is received from user space */
+	memset(&rx_status, 0, sizeof(rx_status));
+	if (info->attrs[HWSIM_ATTR_FREQ]) {
+		struct tx_iter_data iter_data = {
+			.hw = data2->hw,
+			.rx_status = &rx_status,
+		};
+
+		/* throw away off-channel packets, but allow both the temporary
+		 * ("hw" scan/remain-on-channel), regular channels and links,
+		 * since the internal datapath also allows this
+		 */
+		rx_status.freq = nla_get_u32(info->attrs[HWSIM_ATTR_FREQ]);
+
+		iter_data.channel = ieee80211_get_channel(data2->hw->wiphy,
+							  rx_status.freq);
+		if (!iter_data.channel)
+			goto out;
+		rx_status.band = iter_data.channel->band;
+
+		mutex_lock(&data2->mutex);
+		if (!hwsim_chans_compat(iter_data.channel, channel)) {
+			ieee80211_iterate_active_interfaces_atomic(
+				data2->hw, IEEE80211_IFACE_ITER_NORMAL,
+				mac80211_hwsim_tx_iter, &iter_data);
+			if (!iter_data.receive) {
+				mutex_unlock(&data2->mutex);
+				goto out;
+			}
+		}
+		mutex_unlock(&data2->mutex);
+	} else if (!channel) {
+		goto out;
+	} else {
+		rx_status.freq = channel->center_freq;
+		rx_status.band = channel->band;
+	}
+
+	rx_status.rate_idx = nla_get_u32(info->attrs[HWSIM_ATTR_RX_RATE]);
+	if (rx_status.rate_idx >= data2->hw->wiphy->bands[rx_status.band]->n_bitrates)
+		goto out;
+	rx_status.signal = nla_get_u32(info->attrs[HWSIM_ATTR_SIGNAL]);
+
+	hdr = (void *)skb->data;
+
+	if (ieee80211_is_beacon(hdr->frame_control) ||
+	    ieee80211_is_probe_resp(hdr->frame_control))
+		rx_status.boottime_ns = ktime_get_boottime_ns();
+
+	mac80211_hwsim_rx(data2, &rx_status, skb);
+
+	return 0;
+err:
+	pr_debug("mac80211_hwsim: error occurred in %s\n", __func__);
+out:
+	dev_kfree_skb(skb);
+	return -EINVAL;
+}
+
+static int hwsim_register_received_nl(struct sk_buff *skb_2,
+				      struct genl_info *info)
+{
+	struct net *net = genl_info_net(info);
+	struct mac80211_hwsim_data *data;
+	int chans = 1;
+
+	spin_lock_bh(&hwsim_radio_lock);
+	list_for_each_entry(data, &hwsim_radios, list)
+		chans = max(chans, data->channels);
+	spin_unlock_bh(&hwsim_radio_lock);
+
+	/* In the future we should revise the userspace API and allow it
+	 * to set a flag that it does support multi-channel, then we can
+	 * let this pass conditionally on the flag.
+	 * For current userspace, prohibit it since it won't work right.
+	 */
+	if (chans > 1)
+		return -EOPNOTSUPP;
+
+	if (hwsim_net_get_wmediumd(net))
+		return -EBUSY;
+
+	hwsim_register_wmediumd(net, info->snd_portid);
+
+	pr_debug("mac80211_hwsim: received a REGISTER, "
+	       "switching to wmediumd mode with pid %d\n", info->snd_portid);
+
+	return 0;
+}
+
+/* ensures ciphers only include ciphers listed in 'hwsim_ciphers' array */
+static bool hwsim_known_ciphers(const u32 *ciphers, int n_ciphers)
+{
+	int i;
+
+	for (i = 0; i < n_ciphers; i++) {
+		int j;
+		int found = 0;
+
+		for (j = 0; j < ARRAY_SIZE(hwsim_ciphers); j++) {
+			if (ciphers[i] == hwsim_ciphers[j]) {
+				found = 1;
+				break;
+			}
+		}
+
+		if (!found)
+			return false;
+	}
+
+	return true;
+}
+
+static int parse_ftm_capa(const struct nlattr *ftm_capa, struct cfg80211_pmsr_capabilities *out,
+			  struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX + 1];
+	int ret;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_FTM_CAPA_ATTR_MAX, ftm_capa, hwsim_ftm_capa_policy,
+			       NULL);
+	if (ret) {
+		NL_SET_ERR_MSG_ATTR(info->extack, ftm_capa, "malformed FTM capability");
+		return -EINVAL;
+	}
+
+	out->ftm.supported = 1;
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES])
+		out->ftm.preambles = nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_PREAMBLES]);
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS])
+		out->ftm.bandwidths = nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_BANDWIDTHS]);
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT])
+		out->ftm.max_bursts_exponent =
+			nla_get_u8(tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_BURSTS_EXPONENT]);
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST])
+		out->ftm.max_ftms_per_burst =
+			nla_get_u8(tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_FTMS_PER_BURST]);
+	out->ftm.asap = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_ASAP];
+	out->ftm.non_asap = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_NON_ASAP];
+	out->ftm.request_lci = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_REQ_LCI];
+	out->ftm.request_civicloc = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_REQ_CIVICLOC];
+	out->ftm.trigger_based = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_TRIGGER_BASED];
+	out->ftm.non_trigger_based = !!tb[NL80211_PMSR_FTM_CAPA_ATTR_NON_TRIGGER_BASED];
+
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_TX_ANTENNAS])
+		out->ftm.max_no_of_tx_antennas =
+			nla_get_u8(tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_TX_ANTENNAS]);
+
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_RX_ANTENNAS])
+		out->ftm.max_no_of_rx_antennas =
+			nla_get_u8(tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_RX_ANTENNAS]);
+
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_EDCA])
+		out->ftm.min_allowed_ranging_interval_edca =
+			nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_EDCA]);
+
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_NTB])
+		out->ftm.min_allowed_ranging_interval_ntb =
+			nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_NTB]);
+
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_PD_PREAMBLES])
+		out->ftm.pd_preambles =
+			nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_PD_PREAMBLES]);
+
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_PD_BANDWIDTHS])
+		out->ftm.pd_bandwidths =
+			nla_get_u32(tb[NL80211_PMSR_FTM_CAPA_ATTR_PD_BANDWIDTHS]);
+
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS]) {
+		struct nlattr *ista_tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX + 1];
+
+		if (!nla_parse_nested(ista_tb, NL80211_PMSR_FTM_CAPA_ATTR_MAX,
+				      tb[NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS],
+				      hwsim_ftm_role_capa_policy, NULL)) {
+			out->ftm.ista.support_ntb =
+				!!ista_tb[NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_NTB];
+			out->ftm.ista.support_tb =
+				!!ista_tb[NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_TB];
+			out->ftm.ista.support_edca =
+				!!ista_tb[NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_EDCA];
+			if (ista_tb[NL80211_PMSR_ATTR_MAX_PEER_ISTA_ROLE])
+				out->ftm.ista.max_peers =
+					nla_get_u32(ista_tb[NL80211_PMSR_ATTR_MAX_PEER_ISTA_ROLE]);
+		}
+	}
+
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS]) {
+		struct nlattr *rsta_tb[NL80211_PMSR_FTM_CAPA_ATTR_MAX + 1];
+
+		if (!nla_parse_nested(rsta_tb, NL80211_PMSR_FTM_CAPA_ATTR_MAX,
+				      tb[NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS],
+				      hwsim_ftm_role_capa_policy, NULL)) {
+			out->ftm.rsta.support_ntb =
+				!!rsta_tb[NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_NTB];
+			out->ftm.rsta.support_tb =
+				!!rsta_tb[NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_TB];
+			out->ftm.rsta.support_edca =
+				!!rsta_tb[NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_EDCA];
+			if (rsta_tb[NL80211_PMSR_ATTR_MAX_PEER_RSTA_ROLE])
+				out->ftm.rsta.max_peers =
+					nla_get_u32(rsta_tb[NL80211_PMSR_ATTR_MAX_PEER_RSTA_ROLE]);
+		}
+	}
+
+	if (tb[NL80211_PMSR_FTM_CAPA_ATTR_TYPE_CAPS]) {
+		struct nlattr *type_tb[NL80211_PMSR_FTM_TYPE_CAPA_ATTR_MAX + 1];
+
+		if (!nla_parse_nested(type_tb, NL80211_PMSR_FTM_TYPE_CAPA_ATTR_MAX,
+				      tb[NL80211_PMSR_FTM_CAPA_ATTR_TYPE_CAPS],
+				      hwsim_ftm_type_capa_policy, NULL)) {
+			out->ftm.type.infra_support =
+				!!type_tb[NL80211_PMSR_FTM_TYPE_CAPA_ATTR_INFRA_SUPPORT];
+			out->ftm.type.pd_support =
+				!!type_tb[NL80211_PMSR_FTM_TYPE_CAPA_ATTR_PD_SUPPORT];
+		}
+	}
+
+	out->ftm.concurrent_ista_rsta_support =
+		!!tb[NL80211_PMSR_FTM_CAPA_ATTR_CONCURRENT_ISTA_RSTA_SUPPORT];
+
+	return 0;
+}
+
+static int parse_pmsr_capa(const struct nlattr *pmsr_capa, struct cfg80211_pmsr_capabilities *out,
+			   struct genl_info *info)
+{
+	struct nlattr *tb[NL80211_PMSR_ATTR_MAX + 1];
+	struct nlattr *nla;
+	int size;
+	int ret;
+
+	ret = nla_parse_nested(tb, NL80211_PMSR_ATTR_MAX, pmsr_capa, hwsim_pmsr_capa_policy, NULL);
+	if (ret) {
+		NL_SET_ERR_MSG_ATTR(info->extack, pmsr_capa, "malformed PMSR capability");
+		return -EINVAL;
+	}
+
+	if (tb[NL80211_PMSR_ATTR_MAX_PEERS])
+		out->max_peers = nla_get_u32(tb[NL80211_PMSR_ATTR_MAX_PEERS]);
+	out->report_ap_tsf = !!tb[NL80211_PMSR_ATTR_REPORT_AP_TSF];
+	out->randomize_mac_addr = !!tb[NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR];
+
+	if (!tb[NL80211_PMSR_ATTR_TYPE_CAPA]) {
+		NL_SET_ERR_MSG_ATTR(info->extack, tb[NL80211_PMSR_ATTR_TYPE_CAPA],
+				    "malformed PMSR type");
+		return -EINVAL;
+	}
+
+	nla_for_each_nested(nla, tb[NL80211_PMSR_ATTR_TYPE_CAPA], size) {
+		switch (nla_type(nla)) {
+		case NL80211_PMSR_TYPE_FTM:
+			parse_ftm_capa(nla, out, info);
+			break;
+		default:
+			NL_SET_ERR_MSG_ATTR(info->extack, nla, "unsupported measurement type");
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
+static int hwsim_new_radio_nl(struct sk_buff *msg, struct genl_info *info)
+{
+	struct hwsim_new_radio_params param = { 0 };
+	const char *hwname = NULL;
+	int ret;
+
+	param.reg_strict = info->attrs[HWSIM_ATTR_REG_STRICT_REG];
+	param.p2p_device = info->attrs[HWSIM_ATTR_SUPPORT_P2P_DEVICE];
+	param.nan_device = info->attrs[HWSIM_ATTR_SUPPORT_NAN_DEVICE];
+	param.channels = channels;
+	param.destroy_on_close =
+		info->attrs[HWSIM_ATTR_DESTROY_RADIO_ON_CLOSE];
+
+	if (info->attrs[HWSIM_ATTR_CHANNELS])
+		param.channels = nla_get_u32(info->attrs[HWSIM_ATTR_CHANNELS]);
+
+	if (param.channels < 1) {
+		GENL_SET_ERR_MSG(info, "must have at least one channel");
+		return -EINVAL;
+	}
+
+	if (info->attrs[HWSIM_ATTR_NO_VIF])
+		param.no_vif = true;
+
+	if (info->attrs[HWSIM_ATTR_USE_CHANCTX])
+		param.use_chanctx = true;
+	else
+		param.use_chanctx = (param.channels > 1);
+
+	if (info->attrs[HWSIM_ATTR_MULTI_RADIO])
+		param.multi_radio = true;
+
+	if (info->attrs[HWSIM_ATTR_SUPPORT_BACKGROUND_RADAR])
+		param.background_radar = true;
+
+	if (info->attrs[HWSIM_ATTR_REG_HINT_ALPHA2])
+		param.reg_alpha2 =
+			nla_data(info->attrs[HWSIM_ATTR_REG_HINT_ALPHA2]);
+
+	if (info->attrs[HWSIM_ATTR_REG_CUSTOM_REG]) {
+		u32 idx = nla_get_u32(info->attrs[HWSIM_ATTR_REG_CUSTOM_REG]);
+
+		if (idx >= ARRAY_SIZE(hwsim_world_regdom_custom))
+			return -EINVAL;
+
+		idx = array_index_nospec(idx,
+					 ARRAY_SIZE(hwsim_world_regdom_custom));
+		param.regd = hwsim_world_regdom_custom[idx];
+	}
+
+	if (info->attrs[HWSIM_ATTR_PERM_ADDR]) {
+		if (!is_valid_ether_addr(
+				nla_data(info->attrs[HWSIM_ATTR_PERM_ADDR]))) {
+			GENL_SET_ERR_MSG(info,"MAC is no valid source addr");
+			NL_SET_BAD_ATTR(info->extack,
+					info->attrs[HWSIM_ATTR_PERM_ADDR]);
+			return -EINVAL;
+		}
+
+		param.perm_addr = nla_data(info->attrs[HWSIM_ATTR_PERM_ADDR]);
+	}
+
+	if (info->attrs[HWSIM_ATTR_IFTYPE_SUPPORT]) {
+		param.iftypes =
+			nla_get_u32(info->attrs[HWSIM_ATTR_IFTYPE_SUPPORT]);
+
+		if (param.iftypes & ~HWSIM_IFTYPE_SUPPORT_MASK) {
+			NL_SET_ERR_MSG_ATTR(info->extack,
+					    info->attrs[HWSIM_ATTR_IFTYPE_SUPPORT],
+					    "cannot support more iftypes than kernel");
+			return -EINVAL;
+		}
+	} else {
+		param.iftypes = HWSIM_IFTYPE_SUPPORT_MASK;
+	}
+
+	/* ensure both flag and iftype support is honored */
+	if (param.p2p_device ||
+	    param.iftypes & BIT(NL80211_IFTYPE_P2P_DEVICE)) {
+		param.iftypes |= BIT(NL80211_IFTYPE_P2P_DEVICE);
+		param.p2p_device = true;
+	}
+
+	if (param.nan_device) {
+		if (param.multi_radio) {
+			NL_SET_ERR_MSG(info->extack,
+				       "NAN is not supported on multi-radio wiphys");
+			return -EINVAL;
+		}
+		param.iftypes |= BIT(NL80211_IFTYPE_NAN) |
+				 BIT(NL80211_IFTYPE_NAN_DATA);
+	}
+
+	if (info->attrs[HWSIM_ATTR_CIPHER_SUPPORT]) {
+		u32 len = nla_len(info->attrs[HWSIM_ATTR_CIPHER_SUPPORT]);
+
+		param.ciphers =
+			nla_data(info->attrs[HWSIM_ATTR_CIPHER_SUPPORT]);
+
+		if (len % sizeof(u32)) {
+			NL_SET_ERR_MSG_ATTR(info->extack,
+					    info->attrs[HWSIM_ATTR_CIPHER_SUPPORT],
+					    "bad cipher list length");
+			return -EINVAL;
+		}
+
+		param.n_ciphers = len / sizeof(u32);
+
+		if (param.n_ciphers > ARRAY_SIZE(hwsim_ciphers)) {
+			NL_SET_ERR_MSG_ATTR(info->extack,
+					    info->attrs[HWSIM_ATTR_CIPHER_SUPPORT],
+					    "too many ciphers specified");
+			return -EINVAL;
+		}
+
+		if (!hwsim_known_ciphers(param.ciphers, param.n_ciphers)) {
+			NL_SET_ERR_MSG_ATTR(info->extack,
+					    info->attrs[HWSIM_ATTR_CIPHER_SUPPORT],
+					    "unsupported ciphers specified");
+			return -EINVAL;
+		}
+	}
+
+	param.mlo = info->attrs[HWSIM_ATTR_MLO_SUPPORT];
+
+	if (param.mlo || param.multi_radio)
+		param.use_chanctx = true;
+
+	if (info->attrs[HWSIM_ATTR_RADIO_NAME]) {
+		hwname = kstrndup((char *)nla_data(info->attrs[HWSIM_ATTR_RADIO_NAME]),
+				  nla_len(info->attrs[HWSIM_ATTR_RADIO_NAME]),
+				  GFP_KERNEL);
+		if (!hwname)
+			return -ENOMEM;
+		param.hwname = hwname;
+	}
+
+	if (info->attrs[HWSIM_ATTR_PMSR_SUPPORT]) {
+		struct cfg80211_pmsr_capabilities *pmsr_capa;
+
+		pmsr_capa = kzalloc_obj(*pmsr_capa);
+		if (!pmsr_capa) {
+			ret = -ENOMEM;
+			goto out_free;
+		}
+		param.pmsr_capa = pmsr_capa;
+
+		ret = parse_pmsr_capa(info->attrs[HWSIM_ATTR_PMSR_SUPPORT], pmsr_capa, info);
+		if (ret)
+			goto out_free;
+	}
+
+	ret = mac80211_hwsim_new_radio(info, &param);
+
+out_free:
+	kfree(hwname);
+	kfree(param.pmsr_capa);
+	return ret;
+}
+
+static int hwsim_del_radio_nl(struct sk_buff *msg, struct genl_info *info)
+{
+	struct mac80211_hwsim_data *data;
+	s64 idx = -1;
+	const char *hwname = NULL;
+
+	if (info->attrs[HWSIM_ATTR_RADIO_ID]) {
+		idx = nla_get_u32(info->attrs[HWSIM_ATTR_RADIO_ID]);
+	} else if (info->attrs[HWSIM_ATTR_RADIO_NAME]) {
+		hwname = kstrndup((char *)nla_data(info->attrs[HWSIM_ATTR_RADIO_NAME]),
+				  nla_len(info->attrs[HWSIM_ATTR_RADIO_NAME]),
+				  GFP_KERNEL);
+		if (!hwname)
+			return -ENOMEM;
+	} else
+		return -EINVAL;
+
+	spin_lock_bh(&hwsim_radio_lock);
+	list_for_each_entry(data, &hwsim_radios, list) {
+		if (idx >= 0) {
+			if (data->idx != idx)
+				continue;
+		} else {
+			if (!hwname ||
+			    strcmp(hwname, wiphy_name(data->hw->wiphy)))
+				continue;
+		}
+
+		if (!net_eq(wiphy_net(data->hw->wiphy), genl_info_net(info)))
+			continue;
+
+		list_del(&data->list);
+		rhashtable_remove_fast(&hwsim_radios_rht, &data->rht,
+				       hwsim_rht_params);
+		hwsim_radios_generation++;
+		spin_unlock_bh(&hwsim_radio_lock);
+		mac80211_hwsim_del_radio(data, wiphy_name(data->hw->wiphy),
+					 info);
+		kfree(hwname);
+		return 0;
+	}
+	spin_unlock_bh(&hwsim_radio_lock);
+
+	kfree(hwname);
+	return -ENODEV;
+}
+
+static int hwsim_get_radio_nl(struct sk_buff *msg, struct genl_info *info)
+{
+	struct mac80211_hwsim_data *data;
+	struct sk_buff *skb;
+	int idx, res = -ENODEV;
+
+	if (!info->attrs[HWSIM_ATTR_RADIO_ID])
+		return -EINVAL;
+	idx = nla_get_u32(info->attrs[HWSIM_ATTR_RADIO_ID]);
+
+	spin_lock_bh(&hwsim_radio_lock);
+	list_for_each_entry(data, &hwsim_radios, list) {
+		if (data->idx != idx)
+			continue;
+
+		if (!net_eq(wiphy_net(data->hw->wiphy), genl_info_net(info)))
+			continue;
+
+		skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
+		if (!skb) {
+			res = -ENOMEM;
+			goto out_err;
+		}
+
+		res = mac80211_hwsim_get_radio(skb, data, info->snd_portid,
+					       info->snd_seq, NULL, 0);
+		if (res < 0) {
+			nlmsg_free(skb);
+			goto out_err;
+		}
+
+		res = genlmsg_reply(skb, info);
+		break;
+	}
+
+out_err:
+	spin_unlock_bh(&hwsim_radio_lock);
+
+	return res;
+}
+
+static int hwsim_dump_radio_nl(struct sk_buff *skb,
+			       struct netlink_callback *cb)
+{
+	int last_idx = cb->args[0] - 1;
+	struct mac80211_hwsim_data *data = NULL;
+	int res = 0;
+	void *hdr;
+
+	spin_lock_bh(&hwsim_radio_lock);
+	cb->seq = hwsim_radios_generation;
+
+	if (last_idx >= hwsim_radio_idx-1)
+		goto done;
+
+	list_for_each_entry(data, &hwsim_radios, list) {
+		if (data->idx <= last_idx)
+			continue;
+
+		if (!net_eq(wiphy_net(data->hw->wiphy), sock_net(skb->sk)))
+			continue;
+
+		res = mac80211_hwsim_get_radio(skb, data,
+					       NETLINK_CB(cb->skb).portid,
+					       cb->nlh->nlmsg_seq, cb,
+					       NLM_F_MULTI);
+		if (res < 0)
+			break;
+
+		last_idx = data->idx;
+	}
+
+	cb->args[0] = last_idx + 1;
+
+	/* list changed, but no new element sent, set interrupted flag */
+	if (skb->len == 0 && cb->prev_seq && cb->seq != cb->prev_seq) {
+		hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+				  cb->nlh->nlmsg_seq, &hwsim_genl_family,
+				  NLM_F_MULTI, HWSIM_CMD_GET_RADIO);
+		if (hdr) {
+			genl_dump_check_consistent(cb, hdr);
+			genlmsg_end(skb, hdr);
+		} else {
+			res = -EMSGSIZE;
+		}
+	}
+
+done:
+	spin_unlock_bh(&hwsim_radio_lock);
+	return res ?: skb->len;
+}
+
+/* Generic Netlink operations array */
+static const struct genl_small_ops hwsim_ops[] = {
+	{
+		.cmd = HWSIM_CMD_REGISTER,
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+		.doit = hwsim_register_received_nl,
+		.flags = GENL_UNS_ADMIN_PERM,
+	},
+	{
+		.cmd = HWSIM_CMD_FRAME,
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+		.doit = hwsim_cloned_frame_received_nl,
+	},
+	{
+		.cmd = HWSIM_CMD_TX_INFO_FRAME,
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+		.doit = hwsim_tx_info_frame_received_nl,
+	},
+	{
+		.cmd = HWSIM_CMD_NEW_RADIO,
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+		.doit = hwsim_new_radio_nl,
+		.flags = GENL_UNS_ADMIN_PERM,
+	},
+	{
+		.cmd = HWSIM_CMD_DEL_RADIO,
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+		.doit = hwsim_del_radio_nl,
+		.flags = GENL_UNS_ADMIN_PERM,
+	},
+	{
+		.cmd = HWSIM_CMD_GET_RADIO,
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+		.doit = hwsim_get_radio_nl,
+		.dumpit = hwsim_dump_radio_nl,
+	},
+	{
+		.cmd = HWSIM_CMD_REPORT_PMSR,
+		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
+		.doit = hwsim_pmsr_report_nl,
+	},
+};
+
+static struct genl_family hwsim_genl_family __ro_after_init = {
+	.name = "MAC80211_HWSIM",
+	.version = 1,
+	.maxattr = HWSIM_ATTR_MAX,
+	.policy = hwsim_genl_policy,
+	.netnsok = true,
+	.module = THIS_MODULE,
+	.small_ops = hwsim_ops,
+	.n_small_ops = ARRAY_SIZE(hwsim_ops),
+	.resv_start_op = HWSIM_CMD_REPORT_PMSR + 1, // match with __HWSIM_CMD_MAX
+	.mcgrps = hwsim_mcgrps,
+	.n_mcgrps = ARRAY_SIZE(hwsim_mcgrps),
+};
+
+static void remove_user_radios(u32 portid, int netgroup)
+{
+	struct mac80211_hwsim_data *entry, *tmp;
+	LIST_HEAD(list);
+
+	spin_lock_bh(&hwsim_radio_lock);
+	list_for_each_entry_safe(entry, tmp, &hwsim_radios, list) {
+		if (entry->destroy_on_close && entry->portid == portid &&
+		    entry->netgroup == netgroup) {
+			list_move(&entry->list, &list);
+			rhashtable_remove_fast(&hwsim_radios_rht, &entry->rht,
+					       hwsim_rht_params);
+			hwsim_radios_generation++;
+		}
+	}
+	spin_unlock_bh(&hwsim_radio_lock);
+
+	list_for_each_entry_safe(entry, tmp, &list, list) {
+		list_del(&entry->list);
+		mac80211_hwsim_del_radio(entry, wiphy_name(entry->hw->wiphy),
+					 NULL);
+	}
+}
+
+static int mac80211_hwsim_netlink_notify(struct notifier_block *nb,
+					 unsigned long state,
+					 void *_notify)
+{
+	struct netlink_notify *notify = _notify;
+
+	if (state != NETLINK_URELEASE)
+		return NOTIFY_DONE;
+
+	remove_user_radios(notify->portid, hwsim_net_get_netgroup(notify->net));
+
+	if (notify->portid == hwsim_net_get_wmediumd(notify->net)) {
+		printk(KERN_INFO "mac80211_hwsim: wmediumd released netlink"
+		       " socket, switching to perfect channel medium\n");
+		hwsim_register_wmediumd(notify->net, 0);
+	}
+	return NOTIFY_DONE;
+
+}
+
+static struct notifier_block hwsim_netlink_notifier = {
+	.notifier_call = mac80211_hwsim_netlink_notify,
+};
+
+static int __init hwsim_init_netlink(void)
+{
+	int rc;
+
+	printk(KERN_INFO "mac80211_hwsim: initializing netlink\n");
+
+	rc = genl_register_family(&hwsim_genl_family);
+	if (rc)
+		goto failure;
+
+	rc = netlink_register_notifier(&hwsim_netlink_notifier);
+	if (rc) {
+		genl_unregister_family(&hwsim_genl_family);
+		goto failure;
+	}
+
+	return 0;
+
+failure:
+	pr_debug("mac80211_hwsim: error occurred in %s\n", __func__);
+	return -EINVAL;
+}
+
+static __net_init int hwsim_init_net(struct net *net)
+{
+	return hwsim_net_set_netgroup(net);
+}
+
+static void __net_exit hwsim_exit_net(struct net *net)
+{
+	struct mac80211_hwsim_data *data, *tmp;
+	LIST_HEAD(list);
+
+	spin_lock_bh(&hwsim_radio_lock);
+	list_for_each_entry_safe(data, tmp, &hwsim_radios, list) {
+		if (!net_eq(wiphy_net(data->hw->wiphy), net))
+			continue;
+
+		/* Radios created in init_net are returned to init_net. */
+		if (data->netgroup == hwsim_net_get_netgroup(&init_net))
+			continue;
+
+		list_move(&data->list, &list);
+		rhashtable_remove_fast(&hwsim_radios_rht, &data->rht,
+				       hwsim_rht_params);
+		hwsim_radios_generation++;
+	}
+	spin_unlock_bh(&hwsim_radio_lock);
+
+	list_for_each_entry_safe(data, tmp, &list, list) {
+		list_del(&data->list);
+		mac80211_hwsim_del_radio(data,
+					 wiphy_name(data->hw->wiphy),
+					 NULL);
+	}
+
+	ida_free(&hwsim_netgroup_ida, hwsim_net_get_netgroup(net));
+}
+
+static struct pernet_operations hwsim_net_ops = {
+	.init = hwsim_init_net,
+	.exit = hwsim_exit_net,
+	.id   = &hwsim_net_id,
+	.size = sizeof(struct hwsim_net),
+};
+
+static void hwsim_exit_netlink(void)
+{
+	/* unregister the notifier */
+	netlink_unregister_notifier(&hwsim_netlink_notifier);
+	/* unregister the family */
+	genl_unregister_family(&hwsim_genl_family);
+}
+
+#if IS_REACHABLE(CONFIG_VIRTIO)
+static void hwsim_virtio_tx_done(struct virtqueue *vq)
+{
+	unsigned int len;
+	struct sk_buff *skb;
+	unsigned long flags;
+
+	spin_lock_irqsave(&hwsim_virtio_lock, flags);
+	while ((skb = virtqueue_get_buf(vq, &len)))
+		dev_kfree_skb_irq(skb);
+	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
+}
+
+static int hwsim_virtio_handle_cmd(struct sk_buff *skb)
+{
+	struct nlmsghdr *nlh;
+	struct genlmsghdr *gnlh;
+	struct nlattr *tb[HWSIM_ATTR_MAX + 1];
+	struct genl_info info = {};
+	int err;
+
+	nlh = nlmsg_hdr(skb);
+	gnlh = nlmsg_data(nlh);
+
+	if (skb->len < nlh->nlmsg_len)
+		return -EINVAL;
+
+	err = genlmsg_parse(nlh, &hwsim_genl_family, tb, HWSIM_ATTR_MAX,
+			    hwsim_genl_policy, NULL);
+	if (err) {
+		pr_err_ratelimited("hwsim: genlmsg_parse returned %d\n", err);
+		return err;
+	}
+
+	info.attrs = tb;
+
+	switch (gnlh->cmd) {
+	case HWSIM_CMD_FRAME:
+		hwsim_cloned_frame_received_nl(skb, &info);
+		break;
+	case HWSIM_CMD_TX_INFO_FRAME:
+		hwsim_tx_info_frame_received_nl(skb, &info);
+		break;
+	case HWSIM_CMD_REPORT_PMSR:
+		hwsim_pmsr_report_nl(skb, &info);
+		break;
+	default:
+		pr_err_ratelimited("hwsim: invalid cmd: %d\n", gnlh->cmd);
+		return -EPROTO;
+	}
+	return 0;
+}
+
+static void hwsim_virtio_rx_work(struct work_struct *work)
+{
+	struct virtqueue *vq;
+	unsigned int len;
+	struct sk_buff *skb;
+	struct scatterlist sg[1];
+	int err;
+	unsigned long flags;
+
+	spin_lock_irqsave(&hwsim_virtio_lock, flags);
+	if (!hwsim_virtio_enabled)
+		goto out_unlock;
+
+	skb = virtqueue_get_buf(hwsim_vqs[HWSIM_VQ_RX], &len);
+	if (!skb)
+		goto out_unlock;
+	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
+
+	skb->data = skb->head;
+	skb_reset_tail_pointer(skb);
+	skb_put(skb, len);
+	hwsim_virtio_handle_cmd(skb);
+
+	spin_lock_irqsave(&hwsim_virtio_lock, flags);
+	if (!hwsim_virtio_enabled) {
+		dev_kfree_skb_irq(skb);
+		goto out_unlock;
+	}
+	vq = hwsim_vqs[HWSIM_VQ_RX];
+	sg_init_one(sg, skb->head, skb_end_offset(skb));
+	err = virtqueue_add_inbuf(vq, sg, 1, skb, GFP_ATOMIC);
+	if (WARN(err, "virtqueue_add_inbuf returned %d\n", err))
+		dev_kfree_skb_irq(skb);
+	else
+		virtqueue_kick(vq);
+	schedule_work(&hwsim_virtio_rx);
+
+out_unlock:
+	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
+}
+
+static void hwsim_virtio_rx_done(struct virtqueue *vq)
+{
+	schedule_work(&hwsim_virtio_rx);
+}
+
+static int init_vqs(struct virtio_device *vdev)
+{
+	struct virtqueue_info vqs_info[HWSIM_NUM_VQS] = {
+		[HWSIM_VQ_TX] = { "tx", hwsim_virtio_tx_done },
+		[HWSIM_VQ_RX] = { "rx", hwsim_virtio_rx_done },
+	};
+
+	return virtio_find_vqs(vdev, HWSIM_NUM_VQS,
+			       hwsim_vqs, vqs_info, NULL);
+}
+
+static int fill_vq(struct virtqueue *vq)
+{
+	int i, err;
+	struct sk_buff *skb;
+	struct scatterlist sg[1];
+
+	for (i = 0; i < virtqueue_get_vring_size(vq); i++) {
+		skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
+		if (!skb)
+			return -ENOMEM;
+
+		sg_init_one(sg, skb->head, skb_end_offset(skb));
+		err = virtqueue_add_inbuf(vq, sg, 1, skb, GFP_KERNEL);
+		if (err) {
+			nlmsg_free(skb);
+			return err;
+		}
+	}
+	virtqueue_kick(vq);
+	return 0;
+}
+
+static void remove_vqs(struct virtio_device *vdev)
+{
+	int i;
+
+	virtio_reset_device(vdev);
+
+	for (i = 0; i < ARRAY_SIZE(hwsim_vqs); i++) {
+		struct virtqueue *vq = hwsim_vqs[i];
+		struct sk_buff *skb;
+
+		while ((skb = virtqueue_detach_unused_buf(vq)))
+			nlmsg_free(skb);
+	}
+
+	vdev->config->del_vqs(vdev);
+}
+
+static int hwsim_virtio_probe(struct virtio_device *vdev)
+{
+	int err;
+	unsigned long flags;
+
+	spin_lock_irqsave(&hwsim_virtio_lock, flags);
+	if (hwsim_virtio_enabled) {
+		spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
+		return -EEXIST;
+	}
+	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
+
+	err = init_vqs(vdev);
+	if (err)
+		return err;
+
+	virtio_device_ready(vdev);
+
+	err = fill_vq(hwsim_vqs[HWSIM_VQ_RX]);
+	if (err)
+		goto out_remove;
+
+	spin_lock_irqsave(&hwsim_virtio_lock, flags);
+	hwsim_virtio_enabled = true;
+	spin_unlock_irqrestore(&hwsim_virtio_lock, flags);
+
+	schedule_work(&hwsim_virtio_rx);
+	return 0;
+
+out_remove:
+	remove_vqs(vdev);
+	return err;
+}
+
+static void hwsim_virtio_remove(struct virtio_device *vdev)
+{
+	hwsim_virtio_enabled = false;
+
+	cancel_work_sync(&hwsim_virtio_rx);
+
+	remove_vqs(vdev);
+}
+
+/* MAC80211_HWSIM virtio device id table */
+static const struct virtio_device_id id_table[] = {
+	{ VIRTIO_ID_MAC80211_HWSIM, VIRTIO_DEV_ANY_ID },
+	{ 0 }
+};
+MODULE_DEVICE_TABLE(virtio, id_table);
+
+static struct virtio_driver virtio_hwsim = {
+	.driver.name = KBUILD_MODNAME,
+	.id_table = id_table,
+	.probe = hwsim_virtio_probe,
+	.remove = hwsim_virtio_remove,
+};
+
+static int hwsim_register_virtio_driver(void)
+{
+	return register_virtio_driver(&virtio_hwsim);
+}
+
+static void hwsim_unregister_virtio_driver(void)
+{
+	unregister_virtio_driver(&virtio_hwsim);
+}
+#else
+static inline int hwsim_register_virtio_driver(void)
+{
+	return 0;
+}
+
+static inline void hwsim_unregister_virtio_driver(void)
+{
+}
+#endif
+
+static int __init init_mac80211_hwsim(void)
+{
+	int i, err;
+
+	if (radios < 0 || radios > 100)
+		return -EINVAL;
+
+	if (channels < 1)
+		return -EINVAL;
+
+	err = rhashtable_init(&hwsim_radios_rht, &hwsim_rht_params);
+	if (err)
+		return err;
+
+	err = register_pernet_device(&hwsim_net_ops);
+	if (err)
+		goto out_free_rht;
+
+	err = platform_driver_register(&mac80211_hwsim_driver);
+	if (err)
+		goto out_unregister_pernet;
+
+	err = hwsim_init_netlink();
+	if (err)
+		goto out_unregister_driver;
+
+	err = hwsim_register_virtio_driver();
+	if (err)
+		goto out_exit_netlink;
+
+	err = class_register(&hwsim_class);
+	if (err)
+		goto out_exit_virtio;
+
+	hwsim_init_s1g_channels(hwsim_channels_s1g);
+
+	for (i = 0; i < radios; i++) {
+		struct hwsim_new_radio_params param = { 0 };
+
+		param.channels = channels;
+
+		switch (regtest) {
+		case HWSIM_REGTEST_DIFF_COUNTRY:
+			if (i < ARRAY_SIZE(hwsim_alpha2s))
+				param.reg_alpha2 = hwsim_alpha2s[i];
+			break;
+		case HWSIM_REGTEST_DRIVER_REG_FOLLOW:
+			if (!i)
+				param.reg_alpha2 = hwsim_alpha2s[0];
+			break;
+		case HWSIM_REGTEST_STRICT_ALL:
+			param.reg_strict = true;
+			fallthrough;
+		case HWSIM_REGTEST_DRIVER_REG_ALL:
+			param.reg_alpha2 = hwsim_alpha2s[0];
+			break;
+		case HWSIM_REGTEST_WORLD_ROAM:
+			if (i == 0)
+				param.regd = &hwsim_world_regdom_custom_01;
+			break;
+		case HWSIM_REGTEST_CUSTOM_WORLD:
+			param.regd = &hwsim_world_regdom_custom_03;
+			break;
+		case HWSIM_REGTEST_CUSTOM_WORLD_2:
+			if (i == 0)
+				param.regd = &hwsim_world_regdom_custom_03;
+			else if (i == 1)
+				param.regd = &hwsim_world_regdom_custom_02;
+			break;
+		case HWSIM_REGTEST_STRICT_FOLLOW:
+			if (i == 0) {
+				param.reg_strict = true;
+				param.reg_alpha2 = hwsim_alpha2s[0];
+			}
+			break;
+		case HWSIM_REGTEST_STRICT_AND_DRIVER_REG:
+			if (i == 0) {
+				param.reg_strict = true;
+				param.reg_alpha2 = hwsim_alpha2s[0];
+			} else if (i == 1) {
+				param.reg_alpha2 = hwsim_alpha2s[1];
+			}
+			break;
+		case HWSIM_REGTEST_ALL:
+			switch (i) {
+			case 0:
+				param.regd = &hwsim_world_regdom_custom_01;
+				break;
+			case 1:
+				param.regd = &hwsim_world_regdom_custom_02;
+				break;
+			case 2:
+				param.reg_alpha2 = hwsim_alpha2s[0];
+				break;
+			case 3:
+				param.reg_alpha2 = hwsim_alpha2s[1];
+				break;
+			case 4:
+				param.reg_strict = true;
+				param.reg_alpha2 = hwsim_alpha2s[2];
+				break;
+			}
+			break;
+		default:
+			break;
+		}
+
+		param.p2p_device = support_p2p_device;
+		param.mlo = mlo;
+		param.multi_radio = multi_radio;
+		param.background_radar = true;
+		param.use_chanctx = channels > 1 || mlo || multi_radio;
+		param.iftypes = HWSIM_IFTYPE_SUPPORT_MASK;
+		if (param.p2p_device)
+			param.iftypes |= BIT(NL80211_IFTYPE_P2P_DEVICE);
+
+		err = mac80211_hwsim_new_radio(NULL, &param);
+		if (err < 0)
+			goto out_free_radios;
+	}
+
+	hwsim_mon = alloc_netdev(0, "hwsim%d", NET_NAME_UNKNOWN,
+				 hwsim_mon_setup);
+	if (hwsim_mon == NULL) {
+		err = -ENOMEM;
+		goto out_free_radios;
+	}
+
+	rtnl_lock();
+	err = dev_alloc_name(hwsim_mon, hwsim_mon->name);
+	if (err < 0) {
+		rtnl_unlock();
+		goto out_free_mon;
+	}
+
+	err = register_netdevice(hwsim_mon);
+	if (err < 0) {
+		rtnl_unlock();
+		goto out_free_mon;
+	}
+	rtnl_unlock();
+
+	return 0;
+
+out_free_mon:
+	free_netdev(hwsim_mon);
+out_free_radios:
+	mac80211_hwsim_free();
+out_exit_virtio:
+	hwsim_unregister_virtio_driver();
+out_exit_netlink:
+	hwsim_exit_netlink();
+out_unregister_driver:
+	platform_driver_unregister(&mac80211_hwsim_driver);
+out_unregister_pernet:
+	unregister_pernet_device(&hwsim_net_ops);
+out_free_rht:
+	rhashtable_destroy(&hwsim_radios_rht);
+	return err;
+}
+module_init(init_mac80211_hwsim);
+
+static void __exit exit_mac80211_hwsim(void)
+{
+	pr_debug("mac80211_hwsim: unregister radios\n");
+
+	hwsim_unregister_virtio_driver();
+	hwsim_exit_netlink();
+
+	mac80211_hwsim_free();
+
+	rhashtable_destroy(&hwsim_radios_rht);
+	unregister_netdev(hwsim_mon);
+	platform_driver_unregister(&mac80211_hwsim_driver);
+	unregister_pernet_device(&hwsim_net_ops);
+}
+module_exit(exit_mac80211_hwsim);
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_nan.c b/drivers/net/wireless/virtual/mac80211_hwsim_nan.c
new file mode 100644
index 0000000000000..7be64c45babf9
--- /dev/null
+++ b/drivers/net/wireless/virtual/mac80211_hwsim_nan.c
@@ -0,0 +1,1346 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * mac80211_hwsim_nan - NAN software simulation for mac80211_hwsim
+ * Copyright (C) 2025-2026 Intel Corporation
+ */
+
+#include <net/cfg80211.h>
+#include "mac80211_hwsim_i.h"
+
+/* Defined as the lower 23 bits being zero */
+#define DW0_TSF_MASK		GENMASK(22, 0)
+
+/* DWs are repeated every 512 TUs */
+#define DWST_TU			512
+#define DWST_TSF_MASK		(ieee80211_tu_to_usec(DWST_TU) - 1)
+
+#define SLOT_TU			16
+#define SLOT_TSF_MASK		(ieee80211_tu_to_usec(DWST_TU) - 1)
+
+/* The 2.4 GHz DW is at the start, the 5 GHz is in slot 8 (after 128 TUs) */
+#define DW_5G_OFFSET_TU		128
+
+#define SLOT_24GHZ_DW		0
+#define SLOT_5GHZ_DW		(DW_5G_OFFSET_TU / SLOT_TU)
+
+/* The special DW0 happens every 16 DWSTs (8192 TUs) */
+static_assert(16 * DWST_TU * 1024 == 8192 * 1024);
+static_assert(DW0_TSF_MASK + 1 == 8192 * 1024);
+
+/* warmup phase should be 120 seconds, which is approximately 225 DWSTs */
+#define NAN_WARMUP_DWST		225
+
+#define NAN_RSSI_CLOSE (-60)
+#define NAN_RSSI_MIDDLE (-75)
+
+/* Quiet time at the end of each slot where TX is suppressed */
+#define NAN_CHAN_SWITCH_TIME_US		256
+
+struct hwsim_nan_sta_iter_ctx {
+	struct ieee80211_hw *hw;
+	bool can_tx;
+};
+
+struct hwsim_nan_mcast_data_iter_ctx {
+	struct ieee80211_hw *hw;
+	struct ieee80211_vif *vif;
+	size_t n_vif_sta;
+	size_t n_sta_can_tx;
+};
+
+static void mac80211_hwsim_nan_resume_txqs(struct mac80211_hwsim_data *data);
+
+static u64 hwsim_nan_get_timer_tsf(struct mac80211_hwsim_data *data)
+{
+	ktime_t expires = hrtimer_get_expires(&data->nan.slot_timer);
+
+	return mac80211_hwsim_boottime_to_tsf(data, expires);
+}
+
+static u8 hwsim_nan_slot_from_tsf(u64 tsf)
+{
+	return (tsf & DWST_TSF_MASK) / ieee80211_tu_to_usec(SLOT_TU);
+}
+
+static u64 hwsim_nan_encode_master_rank(u8 master_pref, u8 random_factor,
+					const u8 *addr)
+{
+	return ((u64)master_pref << 56) +
+		((u64)random_factor << 48) +
+		((u64)addr[5] << 40) +
+		((u64)addr[4] << 32) +
+		((u64)addr[3] << 24) +
+		((u64)addr[2] << 16) +
+		((u64)addr[1] << 8) +
+		((u64)addr[0] << 0);
+}
+
+static u64 hwsim_nan_get_master_rank(struct mac80211_hwsim_data *data)
+{
+	u8 master_pref = 0;
+	u8 random_factor = 0;
+
+	if (data->nan.phase == MAC80211_HWSIM_NAN_PHASE_UP) {
+		master_pref = data->nan.master_pref;
+		random_factor = data->nan.random_factor;
+	}
+
+	return hwsim_nan_encode_master_rank(master_pref, random_factor,
+					    data->nan.device_vif->addr);
+}
+
+static bool mac80211_hwsim_nan_is_dw_slot(struct mac80211_hwsim_data *data,
+					  u8 slot)
+{
+	return slot == SLOT_24GHZ_DW ||
+		(slot == SLOT_5GHZ_DW &&
+		 (data->nan.bands & BIT(NL80211_BAND_5GHZ)));
+}
+
+static bool
+hwsim_nan_rx_chandef_compatible(struct mac80211_hwsim_data *data, u8 slot,
+				struct ieee80211_channel *rx_chan, u8 rx_bw)
+{
+	static const int bw_to_mhz[] = {
+		[RATE_INFO_BW_20] = 20, [RATE_INFO_BW_40] = 40,
+		[RATE_INFO_BW_80] = 80, [RATE_INFO_BW_160] = 160,
+	};
+	struct cfg80211_chan_def sched_chandef;
+	int rx_mhz, sched_mhz;
+
+	scoped_guard(spinlock_bh, &data->nan.state_lock)
+		sched_chandef = data->nan.local_sched[slot];
+
+	if (!sched_chandef.chan ||
+	    sched_chandef.chan->center_freq != rx_chan->center_freq)
+		return false;
+
+	if (rx_bw >= ARRAY_SIZE(bw_to_mhz) || !bw_to_mhz[rx_bw])
+		return false;
+
+	rx_mhz = bw_to_mhz[rx_bw];
+	sched_mhz = cfg80211_chandef_get_width(&sched_chandef);
+
+	/* Accept RX at narrower or equal bandwidth */
+	return rx_mhz <= sched_mhz;
+}
+
+static bool hwsim_nan_peer_present_in_dw(struct hwsim_sta_priv *sp, u64 tsf)
+{
+	u8 slot = hwsim_nan_slot_from_tsf(tsf);
+	u8 cdw = 0;
+	u8 dw_index, wake_interval;
+	u16 committed_dw;
+
+	scoped_guard(spinlock_bh, &sp->nan_sched.lock)
+		committed_dw = sp->nan_sched.committed_dw;
+
+	/* If peer doesn't advertise committed DW, assume presence in
+	 * all 2.4 GHz DW slots
+	 */
+	if (!committed_dw)
+		return slot == SLOT_24GHZ_DW;
+
+	/* Get DW index (0-15) within the 16-DWST DW0 cycle */
+	dw_index = (tsf / ieee80211_tu_to_usec(DWST_TU)) & 0xf;
+
+	/* Extract CDW for the appropriate band (spec Table 80) */
+	if (slot == SLOT_24GHZ_DW)
+		cdw = committed_dw & 0x7;
+	else if (slot == SLOT_5GHZ_DW)
+		cdw = (committed_dw >> 3) & 0x7;
+
+	if (cdw == 0)
+		return false;
+
+	/* Peer wakes every 2^(cdw-1) DWs: 1, 2, 4, 8, or 16 */
+	wake_interval = 1 << (cdw - 1);
+
+	return (dw_index % wake_interval) == 0;
+}
+
+static bool
+hwsim_nan_peer_present_in_faw(struct hwsim_sta_priv *sp,
+			      struct mac80211_hwsim_data *data, u8 slot)
+{
+	struct cfg80211_chan_def local_chandef;
+
+	scoped_guard(spinlock_bh, &data->nan.state_lock)
+		local_chandef = data->nan.local_sched[slot];
+
+	if (!local_chandef.chan)
+		return false;
+
+	scoped_guard(spinlock_bh, &sp->nan_sched.lock) {
+		for (int i = 0; i < CFG80211_NAN_MAX_PEER_MAPS; i++) {
+			struct cfg80211_chan_def *peer_chandef;
+
+			if (sp->nan_sched.maps[i].map_id ==
+			    CFG80211_NAN_INVALID_MAP_ID)
+				continue;
+
+			peer_chandef = &sp->nan_sched.maps[i].chans[slot];
+			if (!peer_chandef->chan)
+				continue;
+
+			if (cfg80211_chandef_compatible(&local_chandef,
+							peer_chandef))
+				return true;
+		}
+	}
+
+	return false;
+}
+
+static void
+mac80211_hwsim_nan_schedule_slot(struct mac80211_hwsim_data *data, u8 slot,
+				 bool discontinuity)
+{
+	u64 tsf;
+
+	if (!discontinuity)
+		tsf = hwsim_nan_get_timer_tsf(data);
+	else
+		tsf = mac80211_hwsim_get_tsf(data->hw, data->nan.device_vif);
+
+	/* Only called by mac80211_hwsim_nan_dw_timer from softirq context */
+	lockdep_assert_in_softirq();
+
+	tsf &= ~DWST_TSF_MASK;
+	tsf += ieee80211_tu_to_usec(slot * SLOT_TU);
+
+	hrtimer_set_expires(&data->nan.slot_timer,
+			    mac80211_hwsim_tsf_to_boottime(data, tsf));
+}
+
+void mac80211_hwsim_nan_rx(struct ieee80211_hw *hw,
+			   struct sk_buff *skb)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	const struct ieee80211_mgmt *mgmt = (void *)skb->data;
+	struct element *nan_elem = (void *)mgmt->u.beacon.variable;
+	struct ieee80211_nan_anchor_master_info *ami = NULL;
+	const struct ieee80211_nan_attr *nan_attr;
+	struct ieee80211_rx_status rx_status;
+	bool joined_cluster = false;
+	bool adopt_tsf = false;
+	bool is_sync_beacon;
+	bool is_same_cluster;
+	u64 master_rank = 0;
+	ssize_t data_len;
+	u8 slot;
+
+	/* Need a NAN vendor element at the start */
+	if (skb->len < (offsetofend(struct ieee80211_mgmt, u.beacon) + 6) ||
+	    !ieee80211_is_beacon(mgmt->frame_control))
+		return;
+
+	data_len = skb->len - offsetofend(struct ieee80211_mgmt, u.beacon);
+
+	/* Copy the RX status to add a MAC timestamp if needed */
+	memcpy(&rx_status, IEEE80211_SKB_RXCB(skb),
+	       sizeof(struct ieee80211_rx_status));
+
+	/* And deal with the lack of mac time stamp */
+	if ((rx_status.flag & RX_FLAG_MACTIME) != RX_FLAG_MACTIME_START) {
+		u64 tsf = mac80211_hwsim_get_tsf(hw, data->nan.device_vif);;
+
+		/* In that case there should be no timestamp */
+		WARN_ON_ONCE(rx_status.flag & RX_FLAG_MACTIME);
+
+		/* No mac timestamp, set current TSF for the frame end */
+		rx_status.flag |= RX_FLAG_MACTIME_END;
+		rx_status.mactime = tsf;
+
+		/* And translate to the start for the rest of the code */
+		rx_status.mactime =
+			ieee80211_calculate_rx_timestamp(hw, &rx_status,
+							 skb->len, 0);
+		rx_status.flag &= ~RX_FLAG_MACTIME;
+		rx_status.flag |= RX_FLAG_MACTIME_START;
+
+		/* Match mac80211_hwsim_nan_receive, see comment there */
+		slot = hwsim_nan_slot_from_tsf(tsf + 128);
+	} else {
+		slot = hwsim_nan_slot_from_tsf(rx_status.mactime);
+	}
+
+	/*
+	 * (overly) simplify things, only track 2.4 GHz here. Also, ignore
+	 * frames outside of the 2.4 GHz DW slot, unless in the initial SCAN
+	 * phase.
+	 */
+	if ((slot != SLOT_24GHZ_DW &&
+	     data->nan.phase != MAC80211_HWSIM_NAN_PHASE_SCAN) ||
+	    rx_status.freq != 2437)
+		return;
+
+	/* Just ignore low RSSI beacons that we cannot sync to */
+	if (rx_status.signal < NAN_RSSI_MIDDLE)
+		return;
+
+	/* Needs to be a valid NAN cluster ID in A3 */
+	if (get_unaligned_be32(mgmt->bssid) != ((WLAN_OUI_WFA << 8) | 0x01))
+		return;
+
+	/* We are only interested in NAN beacons */
+	if (nan_elem->id != WLAN_EID_VENDOR_SPECIFIC ||
+	    nan_elem->datalen < 4 ||
+	    get_unaligned_be32(nan_elem->data) !=
+	    (WLAN_OUI_WFA << 8 | WLAN_OUI_TYPE_WFA_NAN))
+		return;
+
+	u8 *nan_defragmented __free(kfree) = kzalloc(data_len, GFP_ATOMIC);
+	if (!nan_defragmented)
+		return;
+
+	data_len = cfg80211_defragment_element(nan_elem,
+					       mgmt->u.beacon.variable,
+					       data_len,
+					       nan_defragmented, data_len,
+					       WLAN_EID_FRAGMENT);
+
+	if (data_len < 0)
+		return;
+
+	/* Assume it is a synchronization beacon if beacon_int is 512 TUs */
+	is_sync_beacon = le16_to_cpu(mgmt->u.beacon.beacon_int) == DWST_TU;
+	is_same_cluster = ether_addr_equal(mgmt->bssid, data->nan.cluster_id);
+
+	for_each_nan_attr(nan_attr, nan_defragmented + 4, data_len - 4) {
+		if (nan_attr->attr == NAN_ATTR_MASTER_INDICATION &&
+		    le16_to_cpu(nan_attr->length) >=
+		    sizeof(struct ieee80211_nan_master_indication)) {
+			struct ieee80211_nan_master_indication *mi =
+				(void *)nan_attr->data;
+
+			master_rank =
+				hwsim_nan_encode_master_rank(mi->master_pref,
+							     mi->random_factor,
+							     mgmt->sa);
+		}
+
+		if (nan_attr->attr == NAN_ATTR_CLUSTER_INFO &&
+		    le16_to_cpu(nan_attr->length) >=
+		    sizeof(struct ieee80211_nan_anchor_master_info)) {
+			ami = (void *)nan_attr->data;
+
+			/*
+			 * The AMBTT should be set to the beacon timestamp when
+			 * the sender is the anchor master. We can simply
+			 * modify the structure because we created a copy when
+			 * defragmenting the NAN element.
+			 */
+			if (ami->hop_count == 0)
+				ami->ambtt = cpu_to_le32(
+					le64_to_cpu(mgmt->u.beacon.timestamp));
+		}
+	}
+
+	/* Do the rest of the processing under lock */
+	spin_lock_bh(&data->nan.state_lock);
+
+	/*
+	 * sync beacon should be discarded if the master rank is the same
+	 * and the AMBTT is older than 16 * 512 TUs compared to our own TSF.
+	 *
+	 * Subtract the AMBTT from the lowered TSF. If the AMBTT is older
+	 * (smaller) then the calculation will not underflow.
+	 */
+	if (is_sync_beacon && ami &&
+	    ami->master_rank == data->nan.current_ami.master_rank &&
+	    (((u32)rx_status.mactime -
+	      ieee80211_tu_to_usec(16 * 512)) -
+	     le32_to_cpu(ami->ambtt)) < 0x8000000) {
+		wiphy_dbg(hw->wiphy,
+			  "NAN: ignoring sync beacon with old AMBTT\n");
+		is_sync_beacon = false;
+	}
+
+	if (is_same_cluster && is_sync_beacon &&
+	    master_rank > hwsim_nan_get_master_rank(data)) {
+		if (rx_status.signal > NAN_RSSI_CLOSE)
+			data->nan.master_transition_score += 3;
+		else
+			data->nan.master_transition_score += 1;
+	}
+
+	if (is_same_cluster && is_sync_beacon && ami &&
+	    ((ami->master_rank == data->nan.current_ami.master_rank &&
+	      ami->hop_count < data->nan.current_ami.hop_count) ||
+	     (master_rank > hwsim_nan_get_master_rank(data) &&
+	      ami->hop_count == data->nan.current_ami.hop_count))) {
+		if (rx_status.signal > NAN_RSSI_CLOSE)
+			data->nan.sync_transition_score += 3;
+		else
+			data->nan.sync_transition_score += 1;
+	}
+
+	/*
+	 * Decide on TSF adjustments before updating any other state
+	 */
+	if (is_same_cluster && is_sync_beacon && ami &&
+	    data->nan.current_ami.hop_count != 0) {
+		if (le64_to_cpu(ami->master_rank) >
+		    le64_to_cpu(data->nan.current_ami.master_rank) &&
+		    ami->master_rank != data->nan.last_ami.master_rank)
+			adopt_tsf = true;
+
+		if (le64_to_cpu(ami->master_rank) >
+		    le64_to_cpu(data->nan.current_ami.master_rank) &&
+		    ami->master_rank == data->nan.last_ami.master_rank &&
+		    le32_to_cpu(ami->ambtt) >
+		    le32_to_cpu(data->nan.last_ami.ambtt))
+			adopt_tsf = true;
+
+		if (le64_to_cpu(ami->master_rank) <
+		    le64_to_cpu(data->nan.current_ami.master_rank) &&
+		    le64_to_cpu(ami->master_rank) >
+		    hwsim_nan_get_master_rank(data) &&
+		    ether_addr_equal(ami->master_addr,
+				     data->nan.current_ami.master_addr))
+			adopt_tsf = true;
+
+		if (ami->master_rank == data->nan.current_ami.master_rank &&
+		    le32_to_cpu(ami->ambtt) >
+		    le32_to_cpu(data->nan.current_ami.ambtt))
+			adopt_tsf = true;
+
+		/* Anchor Master case is handled below */
+	}
+
+	/*
+	 * NAN Cluster merging
+	 */
+	if (!is_same_cluster && ami) {
+		u64 curr_amr;
+		u64 own_cg;
+		u64 frame_amr;
+		u64 cg;
+
+		/* Shifted down by 19 bits compared to spec */
+		frame_amr = le64_to_cpu(ami->master_rank);
+		cg = (u64)ami->master_pref << (64 - 19);
+		cg += le64_to_cpu(mgmt->u.beacon.timestamp) >> 19;
+
+		curr_amr = le64_to_cpu(data->nan.current_ami.master_rank);
+		own_cg = (u64)data->nan.current_ami.master_pref << (64 - 19);
+		own_cg += rx_status.mactime >> 19;
+
+		/*
+		 * Check if the cluster shall be joined
+		 *
+		 * When in the "scan" phase, just join immediately.
+		 */
+		if (cg > own_cg ||
+		    (cg == own_cg && frame_amr > curr_amr) ||
+		    data->nan.phase == MAC80211_HWSIM_NAN_PHASE_SCAN) {
+			/* Avoid a state transition */
+			data->nan.master_transition_score = 0;
+			data->nan.sync_transition_score = 0;
+
+			/*
+			 * NOTE: The spec says we should TX sync beacons on the
+			 * old schedule after joining. We do not implement this.
+			 */
+
+			wiphy_dbg(hw->wiphy, "NAN: joining cluster %pM\n",
+				  mgmt->bssid);
+
+			joined_cluster = true;
+			adopt_tsf = true;
+
+			memcpy(&data->nan.last_ami, &data->nan.current_ami,
+			       sizeof(data->nan.last_ami));
+			memcpy(&data->nan.current_ami, ami,
+			       sizeof(data->nan.last_ami));
+			data->nan.current_ami.hop_count += 1;
+
+			memcpy(data->nan.cluster_id, mgmt->bssid, ETH_ALEN);
+
+			/*
+			 * Assume we are UP if we joined a cluster.
+			 *
+			 * If the other anchor master is still in the warmup
+			 * phase, then we may temporarily become the anchor
+			 * master until it sets its own master preference to
+			 * be non-zero.
+			 */
+			data->nan.phase = MAC80211_HWSIM_NAN_PHASE_UP;
+			data->nan.random_factor_valid_dwst = 0;
+		}
+	}
+
+	/*
+	 * Anchor master selection
+	 */
+	/* We are not anchor master */
+	if (is_same_cluster && is_sync_beacon && ami &&
+	    data->nan.current_ami.hop_count != 0) {
+		if (le64_to_cpu(data->nan.current_ami.master_rank) <
+		    le64_to_cpu(ami->master_rank)) {
+			if (ami->master_rank == data->nan.last_ami.master_rank &&
+			    le32_to_cpu(ami->ambtt) <=
+			    le32_to_cpu(data->nan.last_ami.ambtt)) {
+				/* disregard frame */
+			} else {
+				memcpy(&data->nan.last_ami,
+				       &data->nan.current_ami,
+				       sizeof(data->nan.last_ami));
+				memcpy(&data->nan.current_ami, ami,
+				       sizeof(data->nan.last_ami));
+				data->nan.current_ami.hop_count += 1;
+			}
+		}
+
+		if (le64_to_cpu(data->nan.current_ami.master_rank) >
+		    le64_to_cpu(ami->master_rank)) {
+			if (!ether_addr_equal(data->nan.current_ami.master_addr,
+					      ami->master_addr)) {
+				/* disregard frame */
+			} else {
+				u64 amr = hwsim_nan_get_master_rank(data);
+
+				if (amr > le64_to_cpu(ami->master_rank)) {
+					/* assume ourselves as anchor master */
+					wiphy_dbg(hw->wiphy,
+						  "NAN: assume anchor master role\n");
+					data->nan.current_ami.master_rank =
+						cpu_to_le64(amr);
+					data->nan.current_ami.hop_count = 0;
+					memset(&data->nan.last_ami, 0,
+					       sizeof(data->nan.last_ami));
+					data->nan.last_ami.ambtt =
+						data->nan.current_ami.ambtt;
+					data->nan.current_ami.ambtt = 0;
+				} else {
+					memcpy(&data->nan.last_ami,
+					       &data->nan.current_ami,
+					       sizeof(data->nan.last_ami));
+					memcpy(&data->nan.current_ami, ami,
+					       sizeof(data->nan.last_ami));
+					data->nan.current_ami.hop_count += 1;
+				}
+			}
+		}
+
+		if (data->nan.current_ami.master_rank == ami->master_rank) {
+			if (le32_to_cpu(data->nan.current_ami.ambtt) <
+			    le32_to_cpu(ami->ambtt)) {
+				data->nan.current_ami.ambtt = ami->ambtt;
+			}
+
+			if (data->nan.current_ami.hop_count >
+			    ami->hop_count + 1) {
+				data->nan.current_ami.hop_count =
+					ami->hop_count + 1;
+			}
+		}
+	}
+
+	/* We are anchor master */
+	if (is_same_cluster && is_sync_beacon && ami &&
+	    data->nan.current_ami.hop_count == 0) {
+		WARN_ON_ONCE(!ether_addr_equal(data->nan.current_ami.master_addr,
+					       data->nan.device_vif->addr));
+
+		if (le64_to_cpu(ami->master_rank) <
+		    le64_to_cpu(data->nan.current_ami.master_rank) ||
+		    ether_addr_equal(ami->master_addr,
+				     data->nan.current_ami.master_addr)) {
+			/* disregard */
+		} else {
+			wiphy_dbg(hw->wiphy, "NAN: lost anchor master role\n");
+			adopt_tsf = true;
+			memcpy(&data->nan.last_ami, &data->nan.current_ami,
+			       sizeof(data->nan.last_ami));
+			memcpy(&data->nan.current_ami, ami,
+			       sizeof(data->nan.last_ami));
+			data->nan.current_ami.hop_count += 1;
+		}
+	}
+
+	if (adopt_tsf && !data->nan.tsf_adjusted) {
+		int threshold = 5;
+		s64 adjustment;
+
+		/* Timestamp is likely inaccurate (and late) in this case */
+		if (!(IEEE80211_SKB_RXCB(skb)->flag & RX_FLAG_MACTIME))
+			threshold = 128;
+
+		adjustment =
+			le64_to_cpu(mgmt->u.beacon.timestamp) -
+			ieee80211_calculate_rx_timestamp(hw, &rx_status,
+							 skb->len, 24);
+
+		scoped_guard(spinlock_bh, &data->tsf_offset_lock) {
+			if (adjustment < -threshold || adjustment > threshold) {
+				if (adjustment < -(s64)ieee80211_tu_to_usec(4) ||
+				    adjustment > (s64)ieee80211_tu_to_usec(4))
+					data->nan.tsf_discontinuity = true;
+
+				wiphy_debug(hw->wiphy,
+					    "NAN: Adjusting TSF by +/- %d us or more: %lld us (discontinuity: %d, from: %pM, old offset: %lld)\n",
+					    threshold, adjustment,
+					    data->nan.tsf_discontinuity, mgmt->sa,
+					    data->tsf_offset);
+			} else {
+				/* smooth things out a little bit */
+				adjustment /= 2;
+			}
+
+			/*
+			 * Do the TSF adjustment
+			 * The flag prevents further adjustments until the next
+			 * 2.4 GHz DW starts to avoid race conditions for
+			 * in-flight packets.
+			 */
+			data->nan.tsf_adjusted = true;
+			data->tsf_offset += adjustment;
+		}
+	}
+
+	spin_unlock_bh(&data->nan.state_lock);
+
+	if (joined_cluster)
+		ieee80211_nan_cluster_joined(data->nan.device_vif,
+					     data->nan.cluster_id, false,
+					     GFP_ATOMIC);
+}
+
+static void
+mac80211_hwsim_nan_exec_state_transitions(struct mac80211_hwsim_data *data)
+{
+	bool notify_join = false;
+
+	/*
+	 * Handle NAN role and state transitions at the end of the DW period
+	 * in accordance to Wi-Fi Aware version 4.0 section 3.3.7 point 2, i.e.
+	 * end of 5 GHz DW if enabled else at the end of the 2.4 GHz DW.
+	 */
+
+	spin_lock(&data->nan.state_lock);
+
+	/* Handle role transitions, Wi-Fi Aware version 4.0 section 3.3.6  */
+	if (data->nan.master_transition_score < 3)
+		data->nan.role = MAC80211_HWSIM_NAN_ROLE_MASTER;
+	else if (data->nan.role == MAC80211_HWSIM_NAN_ROLE_MASTER &&
+		 data->nan.master_transition_score >= 3)
+		data->nan.role = MAC80211_HWSIM_NAN_ROLE_SYNC;
+	else if (data->nan.role == MAC80211_HWSIM_NAN_ROLE_SYNC &&
+		 data->nan.sync_transition_score >= 3)
+		data->nan.role = MAC80211_HWSIM_NAN_ROLE_NON_SYNC;
+	else if (data->nan.role == MAC80211_HWSIM_NAN_ROLE_NON_SYNC &&
+		 data->nan.sync_transition_score < 3)
+		data->nan.role = MAC80211_HWSIM_NAN_ROLE_SYNC;
+
+	/*
+	 * The discovery beacon timer will stop automatically. Make sure it is
+	 * running if we are master. Do not bother with a proper alignment it
+	 * will sync itself to the TSF after the first TX.
+	 */
+	if (data->nan.role == MAC80211_HWSIM_NAN_ROLE_MASTER &&
+	    !hrtimer_active(&data->nan.discovery_beacon_timer))
+		hrtimer_start(&data->nan.discovery_beacon_timer,
+			      ns_to_ktime(10 * NSEC_PER_USEC),
+			      HRTIMER_MODE_REL_SOFT);
+
+	data->nan.master_transition_score = 0;
+	data->nan.sync_transition_score = 0;
+
+	if (data->nan.random_factor_valid_dwst == 0) {
+		u64 amr;
+
+		if (data->nan.phase == MAC80211_HWSIM_NAN_PHASE_SCAN) {
+			data->nan.phase = MAC80211_HWSIM_NAN_PHASE_WARMUP;
+			data->nan.random_factor_valid_dwst = NAN_WARMUP_DWST;
+
+			notify_join = true;
+		} else {
+			data->nan.phase = MAC80211_HWSIM_NAN_PHASE_UP;
+			data->nan.random_factor_valid_dwst =
+				get_random_u32_inclusive(120, 240);
+			data->nan.random_factor = get_random_u8();
+		}
+
+		amr = hwsim_nan_get_master_rank(data);
+
+		if (data->nan.current_ami.hop_count == 0) {
+			/* Update if we are already anchor master */
+			data->nan.current_ami.master_rank = cpu_to_le64(amr);
+		} else if (le64_to_cpu(data->nan.current_ami.master_rank) < amr) {
+			/* assume role if we have a higher rank */
+			wiphy_dbg(data->hw->wiphy,
+				  "NAN: assume anchor master role\n");
+			data->nan.current_ami.master_rank = cpu_to_le64(amr);
+			data->nan.current_ami.hop_count = 0;
+			memset(&data->nan.last_ami, 0,
+			       sizeof(data->nan.last_ami));
+			data->nan.last_ami.ambtt = data->nan.current_ami.ambtt;
+			data->nan.current_ami.ambtt = 0;
+		}
+	} else {
+		data->nan.random_factor_valid_dwst--;
+	}
+
+	spin_unlock(&data->nan.state_lock);
+
+	if (notify_join)
+		ieee80211_nan_cluster_joined(data->nan.device_vif,
+					     data->nan.cluster_id, true,
+					     GFP_ATOMIC);
+}
+
+static void
+mac80211_hwsim_nan_tx_beacon(struct mac80211_hwsim_data *data,
+			     bool is_discovery,
+			     struct ieee80211_channel *channel)
+{
+	struct ieee80211_vendor_ie nan_ie = {
+		.element_id = WLAN_EID_VENDOR_SPECIFIC,
+		.len = 27 - 2,
+		.oui = { u32_get_bits(WLAN_OUI_WFA, 0xff0000),
+			 u32_get_bits(WLAN_OUI_WFA, 0xff00),
+			 u32_get_bits(WLAN_OUI_WFA, 0xff) },
+		.oui_type = WLAN_OUI_TYPE_WFA_NAN,
+	};
+	size_t alloc_size =
+		IEEE80211_TX_STATUS_HEADROOM +
+		offsetofend(struct ieee80211_mgmt, u.beacon) +
+		27 /* size of NAN vendor element */;
+	struct ieee80211_nan_master_indication master_indication;
+	struct ieee80211_nan_attr nan_attr;
+	struct ieee80211_mgmt *mgmt;
+	struct sk_buff *skb;
+
+	/*
+	 * TODO: Should the configured vendor elements or NAN attributes be
+	 * included in some of these beacons?
+	 */
+
+	skb = alloc_skb(alloc_size, GFP_ATOMIC);
+	if (!skb)
+		return;
+
+	spin_lock(&data->nan.state_lock);
+
+	skb_reserve(skb, IEEE80211_TX_STATUS_HEADROOM);
+	mgmt = skb_put(skb, offsetofend(struct ieee80211_mgmt, u.beacon));
+
+	memset(mgmt, 0, offsetofend(struct ieee80211_mgmt, u.beacon));
+	memcpy(mgmt->sa, data->nan.device_vif->addr, ETH_ALEN);
+	memset(mgmt->da, 0xff, ETH_ALEN);
+	memcpy(mgmt->bssid, data->nan.cluster_id, ETH_ALEN);
+
+	mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
+					  IEEE80211_STYPE_BEACON);
+	mgmt->u.beacon.beacon_int = cpu_to_le16(is_discovery ? 100 : DWST_TU);
+	mgmt->u.beacon.capab_info =
+		cpu_to_le16(WLAN_CAPABILITY_SHORT_SLOT_TIME |
+			    WLAN_CAPABILITY_SHORT_PREAMBLE);
+
+	/* FIXME: set these to saner values? */
+	mgmt->duration = 0;
+	mgmt->seq_ctrl = 0;
+
+	/* Put the NAN element */
+	skb_put_data(skb, &nan_ie, sizeof(nan_ie));
+
+	nan_attr.attr = NAN_ATTR_MASTER_INDICATION;
+	nan_attr.length = cpu_to_le16(sizeof(master_indication));
+	if (data->nan.phase == MAC80211_HWSIM_NAN_PHASE_UP) {
+		master_indication.master_pref = data->nan.master_pref;
+		master_indication.random_factor = data->nan.random_factor;
+	} else {
+		master_indication.master_pref = 0;
+		master_indication.random_factor = 0;
+	}
+
+	skb_put_data(skb, &nan_attr, sizeof(nan_attr));
+	skb_put_data(skb, &master_indication, sizeof(master_indication));
+
+	nan_attr.attr = NAN_ATTR_CLUSTER_INFO;
+	nan_attr.length = cpu_to_le16(sizeof(data->nan.current_ami));
+	skb_put_data(skb, &nan_attr, sizeof(nan_attr));
+	skb_put_data(skb, &data->nan.current_ami,
+		     sizeof(data->nan.current_ami));
+
+	spin_unlock(&data->nan.state_lock);
+
+	mac80211_hwsim_tx_frame(data->hw, skb, channel);
+}
+
+enum hrtimer_restart
+mac80211_hwsim_nan_slot_timer(struct hrtimer *timer)
+{
+	struct mac80211_hwsim_data *data =
+		container_of(timer, struct mac80211_hwsim_data,
+			     nan.slot_timer);
+	struct ieee80211_hw *hw = data->hw;
+	struct ieee80211_channel *notify_dw_chan = NULL;
+	struct ieee80211_channel *beacon_sync_chan = NULL;
+	u64 tsf = hwsim_nan_get_timer_tsf(data);
+	u8 slot = hwsim_nan_slot_from_tsf(tsf);
+	bool dwst_of_dw0 = false;
+	bool dw_end = false;
+	bool tx_sync_beacon;
+
+	if (!data->nan.device_vif)
+		return HRTIMER_NORESTART;
+
+	if ((tsf & DW0_TSF_MASK & ~DWST_TSF_MASK) == 0)
+		dwst_of_dw0 = true;
+
+
+	scoped_guard(spinlock, &data->nan.state_lock) {
+		if (data->nan.tsf_discontinuity) {
+			data->nan.tsf_discontinuity = false;
+
+			mac80211_hwsim_nan_schedule_slot(data, 32, true);
+
+			return HRTIMER_RESTART;
+		}
+
+		if (slot == SLOT_24GHZ_DW)
+			data->nan.tsf_adjusted = false;
+
+		tx_sync_beacon =
+			data->nan.phase != MAC80211_HWSIM_NAN_PHASE_SCAN &&
+			data->nan.role != MAC80211_HWSIM_NAN_ROLE_NON_SYNC;
+	}
+
+	switch (slot) {
+	case SLOT_24GHZ_DW:
+		wiphy_dbg(data->hw->wiphy, "Start of 2.4 GHz DW, is DW0=%d\n",
+			  dwst_of_dw0);
+		beacon_sync_chan = ieee80211_get_channel(hw->wiphy, 2437);
+		break;
+
+	case SLOT_24GHZ_DW + 1:
+		if (!(data->nan.bands & BIT(NL80211_BAND_5GHZ))) {
+			notify_dw_chan = ieee80211_get_channel(hw->wiphy, 2437);
+			dw_end = true;
+		} else {
+			notify_dw_chan = ieee80211_get_channel(hw->wiphy, 5745);
+		}
+		break;
+
+	case SLOT_5GHZ_DW:
+		if (data->nan.bands & BIT(NL80211_BAND_5GHZ)) {
+			wiphy_dbg(data->hw->wiphy, "Start of 5 GHz DW\n");
+			beacon_sync_chan =
+				ieee80211_get_channel(hw->wiphy, 5745);
+		}
+		break;
+
+	case SLOT_5GHZ_DW + 1:
+		if (data->nan.bands & BIT(NL80211_BAND_5GHZ)) {
+			notify_dw_chan =
+				ieee80211_get_channel(hw->wiphy, 2437);
+			dw_end = true;
+		}
+		break;
+	}
+
+	/* TODO: This does not implement DW contention mitigation */
+	if (beacon_sync_chan && tx_sync_beacon)
+		mac80211_hwsim_nan_tx_beacon(data, false, beacon_sync_chan);
+
+	if (dw_end)
+		mac80211_hwsim_nan_exec_state_transitions(data);
+
+	if (data->nan.notify_dw && notify_dw_chan) {
+		struct wireless_dev *wdev =
+			ieee80211_vif_to_wdev(data->nan.device_vif);
+
+		cfg80211_next_nan_dw_notif(wdev, notify_dw_chan, GFP_ATOMIC);
+	}
+
+	mac80211_hwsim_nan_resume_txqs(data);
+
+	mac80211_hwsim_nan_schedule_slot(data, slot + 1, false);
+
+	return HRTIMER_RESTART;
+}
+
+enum hrtimer_restart
+mac80211_hwsim_nan_discovery_beacon_timer(struct hrtimer *timer)
+{
+	struct mac80211_hwsim_data *data =
+		container_of(timer, struct mac80211_hwsim_data,
+			     nan.discovery_beacon_timer);
+	u32 remainder;
+	u64 tsf_now;
+	u64 tbtt;
+
+	if (!data->nan.device_vif)
+		return HRTIMER_NORESTART;
+
+	scoped_guard(spinlock, &data->nan.state_lock) {
+		if (data->nan.phase == MAC80211_HWSIM_NAN_PHASE_SCAN ||
+		    data->nan.role != MAC80211_HWSIM_NAN_ROLE_MASTER)
+			return HRTIMER_NORESTART;
+	}
+
+	mac80211_hwsim_nan_tx_beacon(
+		data, true, ieee80211_get_channel(data->hw->wiphy, 2437));
+
+	if (data->nan.bands & BIT(NL80211_BAND_5GHZ))
+		mac80211_hwsim_nan_tx_beacon(
+			data, true,
+			ieee80211_get_channel(data->hw->wiphy, 5745));
+
+	/* Read the TSF from the current time in case of adjustments */
+	tsf_now = mac80211_hwsim_get_tsf(data->hw, data->nan.device_vif);
+
+	/* Wrap value to be after the next TBTT */
+	tbtt = tsf_now + ieee80211_tu_to_usec(100);
+
+	/* Round TBTT down to the correct time */
+	div_u64_rem(tbtt, ieee80211_tu_to_usec(100), &remainder);
+	tbtt = tbtt - remainder;
+
+	hrtimer_set_expires(&data->nan.discovery_beacon_timer,
+			    mac80211_hwsim_tsf_to_boottime(data, tbtt));
+
+	return HRTIMER_RESTART;
+}
+
+int mac80211_hwsim_nan_start(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct cfg80211_nan_conf *conf)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+
+	if (vif->type != NL80211_IFTYPE_NAN)
+		return -EINVAL;
+
+	if (data->nan.device_vif)
+		return -EALREADY;
+
+	/* set this before starting the timer, as preemption might occur */
+	data->nan.device_vif = vif;
+	data->nan.bands = conf->bands;
+
+	scoped_guard(spinlock_bh, &data->nan.state_lock) {
+		/* Start in the "scan" phase and stay there for a little bit */
+		data->nan.phase = MAC80211_HWSIM_NAN_PHASE_SCAN;
+		data->nan.random_factor_valid_dwst = 1;
+		data->nan.random_factor = 0;
+		data->nan.master_pref = conf->master_pref;
+		data->nan.role = MAC80211_HWSIM_NAN_ROLE_MASTER;
+		memset(&data->nan.current_ami, 0,
+		       sizeof(data->nan.current_ami));
+		memset(&data->nan.last_ami, 0, sizeof(data->nan.last_ami));
+		data->nan.current_ami.master_rank =
+			cpu_to_le64(hwsim_nan_get_master_rank(data));
+	}
+
+	/* Just run this "soon" and start in a random schedule position */
+	hrtimer_start(&data->nan.slot_timer,
+		      ns_to_ktime(10 * NSEC_PER_USEC),
+		      HRTIMER_MODE_REL_SOFT);
+
+	ether_addr_copy(data->nan.cluster_id, conf->cluster_id);
+
+	data->nan.notify_dw = conf->enable_dw_notification;
+
+	return 0;
+}
+
+int mac80211_hwsim_nan_stop(struct ieee80211_hw *hw,
+			    struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+
+	if (vif->type != NL80211_IFTYPE_NAN || !data->nan.device_vif ||
+	    data->nan.device_vif != vif)
+		return -EINVAL;
+
+	hrtimer_cancel(&data->nan.slot_timer);
+	hrtimer_cancel(&data->nan.resume_txqs_timer);
+	hrtimer_cancel(&data->nan.discovery_beacon_timer);
+	data->nan.device_vif = NULL;
+
+	return 0;
+}
+
+int mac80211_hwsim_nan_change_config(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct cfg80211_nan_conf *conf,
+				     u32 changes)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+
+	if (vif->type != NL80211_IFTYPE_NAN)
+		return -EINVAL;
+
+	if (!data->nan.device_vif)
+		return -EINVAL;
+
+	wiphy_debug(hw->wiphy, "nan_config_changed: changes=0x%x\n", changes);
+
+	/* Handle only the changes we care about for simulation purposes */
+	if (changes & CFG80211_NAN_CONF_CHANGED_BANDS)
+		data->nan.bands = conf->bands;
+
+	if (changes & CFG80211_NAN_CONF_CHANGED_CONFIG)
+		data->nan.notify_dw = conf->enable_dw_notification;
+
+	if (changes & CFG80211_NAN_CONF_CHANGED_PREF) {
+		scoped_guard(spinlock_bh, &data->nan.state_lock)
+			data->nan.master_pref = conf->master_pref;
+	}
+
+	return 0;
+}
+
+static void hwsim_nan_can_sta_transmit(void *_ctx, struct ieee80211_sta *sta)
+{
+	struct hwsim_nan_sta_iter_ctx *ctx = _ctx;
+
+	if (ctx->can_tx)
+		return;
+
+	for (int i = 0; i < ARRAY_SIZE(sta->txq); i++) {
+		struct ieee80211_txq *txq = sta->txq[i];
+
+		if (!txq)
+			continue;
+
+		if (txq->vif->type != NL80211_IFTYPE_NAN &&
+		    txq->vif->type != NL80211_IFTYPE_NAN_DATA)
+			return;
+
+		if (mac80211_hwsim_nan_txq_transmitting(ctx->hw, txq)) {
+			ctx->can_tx = true;
+			return;
+		}
+	}
+}
+
+static void mac80211_hwsim_nan_resume_txqs(struct mac80211_hwsim_data *data)
+{
+	u64 tsf = mac80211_hwsim_get_tsf(data->hw, data->nan.device_vif);
+	u8 slot = hwsim_nan_slot_from_tsf(tsf);
+	bool is_dw_slot = mac80211_hwsim_nan_is_dw_slot(data, slot);
+	struct hwsim_nan_sta_iter_ctx ctx = {
+		.hw = data->hw,
+		.can_tx = false,
+	};
+	u32 timeout_ns;
+
+	/* Outside DW, require local FAW schedule to proceed */
+	if (!is_dw_slot) {
+		scoped_guard(spinlock_bh, &data->nan.state_lock) {
+			if (!data->nan.local_sched[slot].chan)
+				return;
+		}
+	}
+
+	guard(rcu)();
+
+	/* Check if management queue can transmit */
+	if (mac80211_hwsim_nan_txq_transmitting(data->hw,
+						data->nan.device_vif->txq_mgmt))
+		goto resume_txqs_timer;
+
+	/* Check if any STA queue can transmit */
+	ieee80211_iterate_stations_atomic(data->hw,
+					  hwsim_nan_can_sta_transmit,
+					  &ctx);
+
+	if (!ctx.can_tx)
+		return;
+
+resume_txqs_timer:
+	/*
+	 * Wait a bit and also randomize things so that not everyone is TXing
+	 * at the same time. Each slot is 16 TU long, this waits between 100 us
+	 * and 5 ms before starting to TX (unless a new frame arrives).
+	 */
+	timeout_ns = get_random_u32_inclusive(100 * NSEC_PER_USEC,
+					      5 * NSEC_PER_MSEC);
+
+	hrtimer_start(&data->nan.resume_txqs_timer,
+		      ns_to_ktime(timeout_ns),
+		      HRTIMER_MODE_REL_SOFT);
+}
+
+static void hwsim_nan_wake_sta_iter(void *_data, struct ieee80211_sta *sta)
+{
+	struct ieee80211_hw *hw = _data;
+
+	for (int i = 0; i < ARRAY_SIZE(sta->txq); i++) {
+		struct ieee80211_txq *txq = sta->txq[i];
+
+		if (!txq)
+			continue;
+
+		/* exit early if non-NAN */
+		if (txq->vif->type != NL80211_IFTYPE_NAN &&
+		    txq->vif->type != NL80211_IFTYPE_NAN_DATA)
+			return;
+
+		if (mac80211_hwsim_nan_txq_transmitting(hw, txq))
+			ieee80211_hwsim_wake_tx_queue(hw, txq);
+	}
+}
+
+enum hrtimer_restart
+mac80211_hwsim_nan_resume_txqs_timer(struct hrtimer *timer)
+{
+	struct mac80211_hwsim_data *data =
+		container_of(timer, struct mac80211_hwsim_data,
+			     nan.resume_txqs_timer);
+
+	guard(rcu)();
+
+	/* Wake TX queue for management frames on the NAN device interface */
+	if (mac80211_hwsim_nan_txq_transmitting(data->hw,
+						data->nan.device_vif->txq_mgmt))
+		ieee80211_hwsim_wake_tx_queue(data->hw,
+					      data->nan.device_vif->txq_mgmt);
+
+	/* Wake TX queues for all stations */
+	ieee80211_iterate_stations_atomic(data->hw,
+					  hwsim_nan_wake_sta_iter,
+					  data->hw);
+
+	return HRTIMER_NORESTART;
+}
+
+static void
+hwsim_nan_can_mcast_sta_transmit(void *_ctx, struct ieee80211_sta *sta)
+{
+	struct hwsim_nan_mcast_data_iter_ctx *ctx = _ctx;
+	struct ieee80211_txq *txq = sta->txq[0];
+
+	if (!txq || txq->vif != ctx->vif)
+		return;
+
+	ctx->n_vif_sta++;
+	if (mac80211_hwsim_nan_txq_transmitting(ctx->hw, txq))
+		ctx->n_sta_can_tx++;
+}
+
+static bool
+mac80211_hwsim_nan_mcast_data_transmitting(struct ieee80211_hw *hw,
+					   struct ieee80211_txq *txq)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	struct hwsim_nan_mcast_data_iter_ctx ctx = {
+		.hw = hw,
+		.vif = txq->vif,
+		.n_sta_can_tx = 0,
+		.n_vif_sta = 0,
+	};
+
+	/* Check if all the stations associated with the current
+	 * interface are available.
+	 */
+	ieee80211_iterate_stations_atomic(data->hw,
+					  hwsim_nan_can_mcast_sta_transmit,
+					  &ctx);
+
+	return ctx.n_vif_sta && ctx.n_sta_can_tx == ctx.n_vif_sta;
+}
+
+bool mac80211_hwsim_nan_txq_transmitting(struct ieee80211_hw *hw,
+					 struct ieee80211_txq *txq)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	struct ieee80211_sta *nmi_sta;
+	struct hwsim_sta_priv *sp;
+	bool is_dw_slot;
+	u64 tsf;
+	u8 slot;
+
+	if (WARN_ON_ONCE(!data->nan.device_vif))
+		return true;
+
+	tsf = mac80211_hwsim_get_tsf(hw, data->nan.device_vif);
+	slot = hwsim_nan_slot_from_tsf(tsf);
+
+	/* Enforce a maximum channel switch time and guard against TX delays */
+	if (slot != hwsim_nan_slot_from_tsf(tsf + NAN_CHAN_SWITCH_TIME_US))
+		return false;
+
+	is_dw_slot = mac80211_hwsim_nan_is_dw_slot(data, slot);
+
+	if (!txq->sta) {
+		/* Non-STA TXQ: allow management frames during DW */
+		if (txq->vif->type == NL80211_IFTYPE_NAN)
+			return is_dw_slot;
+
+		/* Allow multicast data when all the peers are available
+		 * on this slot
+		 */
+		return mac80211_hwsim_nan_mcast_data_transmitting(hw, txq);
+	}
+
+	/* STA TXQ: need peer schedule for availability check */
+	nmi_sta = rcu_dereference(txq->sta->nmi) ?: txq->sta;
+	sp = (void *)nmi_sta->drv_priv;
+
+	/* DW slot: NDI can TX only mgmt but not worth checking,
+	 * NMI checks peer's committed DW
+	 */
+	if (is_dw_slot) {
+		if (txq->vif->type == NL80211_IFTYPE_NAN_DATA)
+			return false;
+		return hwsim_nan_peer_present_in_dw(sp, tsf);
+	}
+
+	/* FAW slot: verify local schedule and peer availability */
+	return hwsim_nan_peer_present_in_faw(sp, data, slot);
+}
+
+void mac80211_hwsim_nan_get_tx_chandef(struct ieee80211_hw *hw,
+				       struct cfg80211_chan_def *chandef)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	u64 tsf = mac80211_hwsim_get_tsf(data->hw, data->nan.device_vif);
+	u8 slot = hwsim_nan_slot_from_tsf(tsf);
+
+	/* DW slots are always 20 MHz */
+	if (slot == SLOT_24GHZ_DW) {
+		cfg80211_chandef_create(chandef,
+					ieee80211_get_channel(hw->wiphy, 2437),
+					NL80211_CHAN_NO_HT);
+		return;
+	}
+
+	if (slot == SLOT_5GHZ_DW && data->nan.bands & BIT(NL80211_BAND_5GHZ)) {
+		cfg80211_chandef_create(chandef,
+					ieee80211_get_channel(hw->wiphy, 5745),
+					NL80211_CHAN_NO_HT);
+		return;
+	}
+
+	/* FAW slot: copy local schedule for this slot */
+	scoped_guard(spinlock_bh, &data->nan.state_lock)
+		*chandef = data->nan.local_sched[slot];
+}
+
+bool mac80211_hwsim_nan_receive(struct ieee80211_hw *hw,
+				struct ieee80211_channel *channel,
+				struct ieee80211_rx_status *rx_status)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	u8 slot;
+
+	if (WARN_ON_ONCE(!data->nan.device_vif))
+		return false;
+
+	if (data->nan.phase == MAC80211_HWSIM_NAN_PHASE_SCAN)
+		return channel->center_freq == 2437;
+
+	if (rx_status->flag & RX_FLAG_MACTIME) {
+		slot = hwsim_nan_slot_from_tsf(rx_status->mactime);
+	} else {
+		u64 tsf;
+
+		/*
+		 * This is not perfect, but that should be fine.
+		 *
+		 * Assume the frame might be a bit early in relation to our
+		 * own TSF. This is largely because the TSF sync is going to be
+		 * pretty bad when the frame was RXed via NL and the beacon as
+		 * well as RX timestamps are not accurate.
+		 */
+		tsf = mac80211_hwsim_get_tsf(data->hw, data->nan.device_vif);
+		slot = hwsim_nan_slot_from_tsf(tsf + 128);
+	}
+
+	if (slot == SLOT_24GHZ_DW && channel->center_freq == 2437)
+		return true;
+
+	if (slot == SLOT_5GHZ_DW && data->nan.bands & BIT(NL80211_BAND_5GHZ) &&
+	    channel->center_freq == 5745)
+		return true;
+
+	/* Accept frames during FAW slots if chandef is compatible */
+	return hwsim_nan_rx_chandef_compatible(data, slot, channel,
+					       rx_status->bw);
+}
+
+void mac80211_hwsim_nan_local_sched_changed(struct ieee80211_hw *hw,
+					    struct ieee80211_vif *vif)
+{
+	struct mac80211_hwsim_data *data = hw->priv;
+	struct ieee80211_nan_channel **slots = vif->cfg.nan_sched.schedule;
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_NAN))
+		return;
+
+	spin_lock_bh(&data->nan.state_lock);
+
+	for (int i = 0; i < ARRAY_SIZE(data->nan.local_sched); i++) {
+		struct ieee80211_chanctx_conf *chanctx;
+
+		if (!slots[i] || IS_ERR(slots[i])) {
+			memset(&data->nan.local_sched[i], 0,
+			       sizeof(data->nan.local_sched[i]));
+			continue;
+		}
+
+		chanctx = slots[i]->chanctx_conf;
+		if (!chanctx) {
+			memset(&data->nan.local_sched[i], 0,
+			       sizeof(data->nan.local_sched[i]));
+			continue;
+		}
+
+		data->nan.local_sched[i] = chanctx->def;
+	}
+
+	spin_unlock_bh(&data->nan.state_lock);
+}
+
+int mac80211_hwsim_nan_peer_sched_changed(struct ieee80211_hw *hw,
+					  struct ieee80211_sta *sta)
+{
+	struct hwsim_sta_priv *sp = (void *)sta->drv_priv;
+	struct ieee80211_nan_peer_sched *sched = sta->nan_sched;
+
+	spin_lock_bh(&sp->nan_sched.lock);
+
+	/* Clear existing schedule */
+	sp->nan_sched.committed_dw = 0;
+	for (int i = 0; i < CFG80211_NAN_MAX_PEER_MAPS; i++) {
+		sp->nan_sched.maps[i].map_id = CFG80211_NAN_INVALID_MAP_ID;
+		memset(sp->nan_sched.maps[i].chans, 0,
+		       sizeof(sp->nan_sched.maps[i].chans));
+	}
+
+	if (!sched)
+		goto out;
+
+	sp->nan_sched.committed_dw = sched->committed_dw;
+
+	for (int i = 0; i < CFG80211_NAN_MAX_PEER_MAPS; i++) {
+		struct ieee80211_nan_peer_map *map = &sched->maps[i];
+
+		if (map->map_id == CFG80211_NAN_INVALID_MAP_ID)
+			continue;
+
+		sp->nan_sched.maps[i].map_id = map->map_id;
+
+		for (int j = 0; j < CFG80211_NAN_SCHED_NUM_TIME_SLOTS; j++) {
+			struct ieee80211_nan_channel *peer_chan =
+				map->slots[j];
+
+			if (peer_chan && peer_chan->chanreq.oper.chan)
+				sp->nan_sched.maps[i].chans[j] =
+					peer_chan->chanreq.oper;
+			else
+				memset(&sp->nan_sched.maps[i].chans[j], 0,
+				       sizeof(sp->nan_sched.maps[i].chans[j]));
+		}
+	}
+
+out:
+	spin_unlock_bh(&sp->nan_sched.lock);
+	return 0;
+}
diff --git a/drivers/net/wireless/virtual/mac80211_hwsim_nan.h b/drivers/net/wireless/virtual/mac80211_hwsim_nan.h
new file mode 100644
index 0000000000000..81e105ac7b8ed
--- /dev/null
+++ b/drivers/net/wireless/virtual/mac80211_hwsim_nan.h
@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * mac80211_hwsim_nan - NAN software simulation for mac80211_hwsim
+ * Copyright (C) 2025-2026 Intel Corporation
+ */
+
+#ifndef __MAC80211_HWSIM_NAN_H
+#define __MAC80211_HWSIM_NAN_H
+
+enum mac80211_hwsim_nan_phase {
+	MAC80211_HWSIM_NAN_PHASE_SCAN,
+	MAC80211_HWSIM_NAN_PHASE_WARMUP,
+	MAC80211_HWSIM_NAN_PHASE_UP,
+};
+
+enum mac80211_hwsim_nan_role {
+	MAC80211_HWSIM_NAN_ROLE_MASTER,
+	MAC80211_HWSIM_NAN_ROLE_SYNC,
+	MAC80211_HWSIM_NAN_ROLE_NON_SYNC,
+};
+
+struct mac80211_hwsim_nan_data {
+	struct ieee80211_vif *device_vif;
+	u8 bands;
+
+	struct hrtimer slot_timer;
+	struct hrtimer resume_txqs_timer;
+	bool notify_dw;
+
+	struct hrtimer discovery_beacon_timer;
+
+	/* Later members are protected by this lock */
+	spinlock_t state_lock;
+
+	u8 master_pref;
+	u8 random_factor;
+
+	u8 random_factor_valid_dwst;
+
+	enum mac80211_hwsim_nan_phase phase;
+	enum mac80211_hwsim_nan_role role;
+
+	u8 cluster_id[ETH_ALEN];
+
+	struct ieee80211_nan_anchor_master_info current_ami;
+	struct ieee80211_nan_anchor_master_info last_ami;
+
+	/* Wi-Fi Aware version 4.0, section 3.3.6.1 and 3.3.6.2 */
+	int master_transition_score;
+	/* Wi-Fi Aware version 4.0, section 3.3.6.3 and 3.3.6.4 */
+	int sync_transition_score;
+
+	bool tsf_adjusted;
+	bool tsf_discontinuity;
+
+	/*
+	 * Local schedule - stores channel definition for each 16TU slot.
+	 * Derived from NMI vif->cfg.nan_schedule. chan == NULL means not
+	 * available in that slot (except DW which is implicit).
+	 */
+	struct cfg80211_chan_def local_sched[CFG80211_NAN_SCHED_NUM_TIME_SLOTS];
+};
+
+enum hrtimer_restart
+mac80211_hwsim_nan_slot_timer(struct hrtimer *timer);
+enum hrtimer_restart
+mac80211_hwsim_nan_resume_txqs_timer(struct hrtimer *timer);
+enum hrtimer_restart
+mac80211_hwsim_nan_discovery_beacon_timer(struct hrtimer *timer);
+
+int mac80211_hwsim_nan_start(struct ieee80211_hw *hw,
+			     struct ieee80211_vif *vif,
+			     struct cfg80211_nan_conf *conf);
+
+int mac80211_hwsim_nan_stop(struct ieee80211_hw *hw,
+			    struct ieee80211_vif *vif);
+
+int mac80211_hwsim_nan_change_config(struct ieee80211_hw *hw,
+				     struct ieee80211_vif *vif,
+				     struct cfg80211_nan_conf *conf,
+				     u32 changes);
+
+int mac80211_hwsim_nan_peer_sched_changed(struct ieee80211_hw *hw,
+					  struct ieee80211_sta *sta);
+
+bool mac80211_hwsim_nan_txq_transmitting(struct ieee80211_hw *hw,
+					 struct ieee80211_txq *txq);
+
+void mac80211_hwsim_nan_get_tx_chandef(struct ieee80211_hw *hw,
+				       struct cfg80211_chan_def *chandef);
+
+bool mac80211_hwsim_nan_receive(struct ieee80211_hw *hw,
+				struct ieee80211_channel *channel,
+				struct ieee80211_rx_status *rx_status);
+
+void mac80211_hwsim_nan_rx(struct ieee80211_hw *hw,
+			   struct sk_buff *skb);
+
+void mac80211_hwsim_nan_local_sched_changed(struct ieee80211_hw *hw,
+					    struct ieee80211_vif *vif);
+
+#endif /* __MAC80211_HWSIM_NAN_H */
diff --git a/drivers/net/wwan/Kconfig b/drivers/net/wwan/Kconfig
index 88df55d78d902..958dbc7347fa8 100644
--- a/drivers/net/wwan/Kconfig
+++ b/drivers/net/wwan/Kconfig
@@ -38,7 +38,7 @@ config WWAN_HWSIM
 	  called wwan_hwsim.  If unsure, say N.
 
 config MHI_WWAN_CTRL
-	tristate "MHI WWAN control driver for QCOM-based PCIe modems"
+	tristate "MHI WWAN control driver for Qualcomm-based PCIe modems"
 	depends on MHI_BUS
 	help
 	  MHI WWAN CTRL allows QCOM-based PCIe modems to expose different modem
@@ -51,7 +51,7 @@ config MHI_WWAN_CTRL
 	  called mhi_wwan_ctrl.
 
 config MHI_WWAN_MBIM
-        tristate "MHI WWAN MBIM network driver for QCOM-based PCIe modems"
+        tristate "MHI WWAN MBIM network driver for Qualcomm-based PCIe modems"
         depends on MHI_BUS
         help
           MHI WWAN MBIM is a WWAN network driver for QCOM-based PCIe modems.
diff --git a/drivers/nfc/nxp-nci/core.c b/drivers/nfc/nxp-nci/core.c
index 66b198663387a..448e5a4e7338f 100644
--- a/drivers/nfc/nxp-nci/core.c
+++ b/drivers/nfc/nxp-nci/core.c
@@ -25,6 +25,7 @@
 			       NFC_PROTO_FELICA_MASK | \
 			       NFC_PROTO_ISO14443_MASK | \
 			       NFC_PROTO_ISO14443_B_MASK | \
+			       NFC_PROTO_ISO15693_MASK | \
 			       NFC_PROTO_NFC_DEP_MASK)
 
 #define NXP_NCI_RF_PLL_UNLOCKED_NTF nci_opcode_pack(NCI_GID_RF_MGMT, 0x21)
diff --git a/drivers/nfc/nxp-nci/i2c.c b/drivers/nfc/nxp-nci/i2c.c
index b3d34433bd14a..a6c08175d9dd9 100644
--- a/drivers/nfc/nxp-nci/i2c.c
+++ b/drivers/nfc/nxp-nci/i2c.c
@@ -16,6 +16,7 @@
 #include <linux/delay.h>
 #include <linux/i2c.h>
 #include <linux/interrupt.h>
+#include <linux/irq.h>
 #include <linux/module.h>
 #include <linux/nfc.h>
 #include <linux/gpio/consumer.h>
@@ -267,6 +268,7 @@ static int nxp_nci_i2c_probe(struct i2c_client *client)
 {
 	struct device *dev = &client->dev;
 	struct nxp_nci_i2c_phy *phy;
+	unsigned long irqflags;
 	int r;
 
 	if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) {
@@ -303,9 +305,26 @@ static int nxp_nci_i2c_probe(struct i2c_client *client)
 	if (r < 0)
 		return r;
 
+	/*
+	 * ACPI platforms may report incorrect IRQ trigger types
+	 * (e.g. level-high), which can lead to interrupt storms.
+	 *
+	 * Use the historically stable rising-edge trigger for ACPI devices.
+	 *
+	 * On non-ACPI systems (e.g. Device Tree), prefer the firmware-
+	 * provided trigger type, falling back to rising-edge if not set.
+	 */
+	if (ACPI_COMPANION(dev)) {
+		irqflags = IRQF_TRIGGER_RISING;
+	} else {
+		irqflags = irq_get_trigger_type(client->irq);
+		if (!irqflags)
+			irqflags = IRQF_TRIGGER_RISING;
+	}
+
 	r = request_threaded_irq(client->irq, NULL,
 				 nxp_nci_i2c_irq_thread_fn,
-				 IRQF_ONESHOT,
+				 irqflags | IRQF_ONESHOT,
 				 NXP_NCI_I2C_DRIVER_NAME, phy);
 	if (r < 0)
 		nfc_err(&client->dev, "Unable to register IRQ handler\n");
diff --git a/drivers/nfc/trf7970a.c b/drivers/nfc/trf7970a.c
index 08c27bb438b59..f22e091019dee 100644
--- a/drivers/nfc/trf7970a.c
+++ b/drivers/nfc/trf7970a.c
@@ -61,15 +61,15 @@
  * support that.  So, if an abort is received before trf7970a_send_cmd()
  * has sent the command to the tag, it simply returns -ECANCELED.  If the
  * command has already been sent to the tag, then the driver continues
- * normally and recieves the response data (or error) but just before
+ * normally and receives the response data (or error) but just before
  * sending the data upstream, it frees the rx_skb and sends -ECANCELED
  * upstream instead.  If the command failed, that error will be sent
  * upstream.
  *
- * When recieving data from a tag and the interrupt status register has
+ * When receiving data from a tag and the interrupt status register has
  * only the SRX bit set, it means that all of the data has been received
  * (once what's in the fifo has been read).  However, depending on timing
- * an interrupt status with only the SRX bit set may not be recived.  In
+ * an interrupt status with only the SRX bit set may not be received.  In
  * those cases, the timeout mechanism is used to wait 20 ms in case more
  * data arrives.  After 20 ms, it is assumed that all of the data has been
  * received and the accumulated rx data is sent upstream.  The
diff --git a/drivers/ntb/hw/amd/ntb_hw_amd.c b/drivers/ntb/hw/amd/ntb_hw_amd.c
index 1a163596ddf54..df9150669ad3a 100644
--- a/drivers/ntb/hw/amd/ntb_hw_amd.c
+++ b/drivers/ntb/hw/amd/ntb_hw_amd.c
@@ -376,7 +376,7 @@ static int amd_ntb_link_disable(struct ntb_dev *ntb)
 
 	if (ndev->ntb.topo == NTB_TOPO_SEC)
 		return -EINVAL;
-	dev_dbg(&ntb->pdev->dev, "Enabling Link.\n");
+	dev_dbg(&ntb->pdev->dev, "Disabling Link.\n");
 
 	return 0;
 }
diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c
index 7cabc82305d6b..f59f926d4bfaa 100644
--- a/drivers/ntb/ntb_transport.c
+++ b/drivers/ntb/ntb_transport.c
@@ -223,6 +223,7 @@ struct ntb_transport_mw {
 	void *alloc_addr;
 	void *virt_addr;
 	dma_addr_t dma_addr;
+	dma_addr_t original_dma_addr;
 };
 
 struct ntb_transport_client_dev {
@@ -765,8 +766,8 @@ static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)
 		return;
 
 	ntb_mw_clear_trans(nt->ndev, PIDX, num_mw);
-	dma_free_coherent(dma_dev, mw->alloc_size,
-			  mw->alloc_addr, mw->dma_addr);
+	dma_free_attrs(dma_dev, mw->alloc_size, mw->alloc_addr,
+		       mw->original_dma_addr, DMA_ATTR_FORCE_CONTIGUOUS);
 	mw->xlat_size = 0;
 	mw->buff_size = 0;
 	mw->alloc_size = 0;
@@ -802,6 +803,7 @@ static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw,
 		return -ENOMEM;
 	}
 	virt_addr = alloc_addr;
+	mw->original_dma_addr = dma_addr;
 
 	/*
 	 * we must ensure that the memory address allocated is BAR size
@@ -826,8 +828,8 @@ static int ntb_alloc_mw_buffer(struct ntb_transport_mw *mw,
 	return 0;
 
 err:
-	dma_free_coherent(ntb_dev, mw->alloc_size, alloc_addr, dma_addr);
-
+	dma_free_attrs(ntb_dev, mw->alloc_size, alloc_addr, dma_addr,
+		       DMA_ATTR_FORCE_CONTIGUOUS);
 	return rc;
 }
 
diff --git a/drivers/nubus/nubus.c b/drivers/nubus/nubus.c
index 559dce302d063..40ce4991c3563 100644
--- a/drivers/nubus/nubus.c
+++ b/drivers/nubus/nubus.c
@@ -41,9 +41,7 @@ module_param_named(populate_procfs, nubus_populate_procfs, bool, 0);
 
 LIST_HEAD(nubus_func_rsrcs);
 
-static struct device nubus_parent = {
-	.init_name	= "nubus",
-};
+static struct device *nubus_parent;
 
 /* Meaning of "bytelanes":
 
@@ -833,7 +831,7 @@ static void __init nubus_add_board(int slot, int bytelanes)
 		list_add_tail(&fres->list, &nubus_func_rsrcs);
 	}
 
-	if (nubus_device_register(&nubus_parent, board))
+	if (nubus_device_register(nubus_parent, board))
 		put_device(&board->dev);
 }
 
@@ -880,18 +878,17 @@ static void __init nubus_scan_bus(void)
 
 static int __init nubus_init(void)
 {
-	int err;
-
 	if (!MACH_IS_MAC)
 		return 0;
 
 	nubus_proc_init();
-	err = device_register(&nubus_parent);
-	if (err) {
-		put_device(&nubus_parent);
-		return err;
-	}
+
+	nubus_parent = root_device_register("nubus");
+	if (IS_ERR(nubus_parent))
+		return PTR_ERR(nubus_parent);
+
 	nubus_scan_bus();
+
 	return 0;
 }
 
diff --git a/drivers/nvmem/Kconfig b/drivers/nvmem/Kconfig
index 74ddbd0f79b0e..e10f7ff725ff9 100644
--- a/drivers/nvmem/Kconfig
+++ b/drivers/nvmem/Kconfig
@@ -28,6 +28,19 @@ source "drivers/nvmem/layouts/Kconfig"
 
 # Devices
 
+config NVMEM_AIROHA_SMC_EFUSES
+	tristate "Airoha SMC eFuse support"
+	depends on ARCH_AIROHA || COMPILE_TEST
+	depends on HAVE_ARM_SMCCC
+	default ARCH_AIROHA
+	help
+	  Say y here to enable support for reading eFuses on Airoha AN7581
+	  SoCs. These are e.g. used to store factory programmed
+	  calibration data required for the PCIe or the USB-C PHY or Thermal.
+
+	  This driver can also be built as a module. If so, the module will
+	  be called nvmem-airoha-smc-efuses.
+
 config NVMEM_AN8855_EFUSE
 	tristate "Airoha AN8855 eFuse support"
 	depends on COMPILE_TEST
@@ -138,7 +151,7 @@ config NVMEM_JZ4780_EFUSE
 
 config NVMEM_LAN9662_OTPC
 	tristate "Microchip LAN9662 OTP controller support"
-	depends on SOC_LAN966 || COMPILE_TEST
+	depends on SOC_LAN966 || ARCH_LAN969X || COMPILE_TEST
 	depends on HAS_IOMEM
 	help
 	  This driver enables the OTP controller available on Microchip LAN9662
@@ -262,7 +275,7 @@ config NVMEM_S32G_OCOTP
 	  Programmable memory pages.
 
 config NVMEM_QCOM_QFPROM
-	tristate "QCOM QFPROM Support"
+	tristate "Qualcomm QFPROM Support"
 	depends on ARCH_QCOM || COMPILE_TEST
 	depends on HAS_IOMEM
 	help
@@ -273,7 +286,7 @@ config NVMEM_QCOM_QFPROM
 	  will be called nvmem_qfprom.
 
 config NVMEM_QCOM_SEC_QFPROM
-        tristate "QCOM SECURE QFPROM Support"
+        tristate "Qualcomm SECURE QFPROM Support"
         depends on ARCH_QCOM || COMPILE_TEST
         depends on HAS_IOMEM
         depends on OF
@@ -303,7 +316,6 @@ config NVMEM_RAVE_SP_EEPROM
 config NVMEM_RCAR_EFUSE
 	tristate "Renesas R-Car Gen4 E-FUSE support"
 	depends on (ARCH_RENESAS && ARM64) || COMPILE_TEST
-	depends on NVMEM
 	help
 	  Enable support for reading the fuses in the E-FUSE or OTP
 	  non-volatile memory block on Renesas R-Car Gen4 SoCs.
@@ -483,4 +495,4 @@ config NVMEM_QORIQ_EFUSE
 	  This driver can also be built as a module. If so, the module
 	  will be called nvmem_qoriq_efuse.
 
-endif
+endif # NVMEM
diff --git a/drivers/nvmem/Makefile b/drivers/nvmem/Makefile
index 7252b8ec88d46..f6f2bc51dee19 100644
--- a/drivers/nvmem/Makefile
+++ b/drivers/nvmem/Makefile
@@ -10,6 +10,8 @@ nvmem_layouts-y			:= layouts.o
 obj-y				+= layouts/
 
 # Devices
+obj-$(CONFIG_NVMEM_AIROHA_SMC_EFUSES)	+= nvmem-airoha-smc-efuses.o
+nvmem-airoha-smc-efuses-y 		:= airoha-smc-efuses.o
 obj-$(CONFIG_NVMEM_AN8855_EFUSE)	+= nvmem-an8855-efuse.o
 nvmem-an8855-efuse-y 			:= an8855-efuse.o
 obj-$(CONFIG_NVMEM_APPLE_EFUSES)	+= nvmem-apple-efuses.o
diff --git a/drivers/nvmem/airoha-smc-efuses.c b/drivers/nvmem/airoha-smc-efuses.c
new file mode 100644
index 0000000000000..e56a99f4aa1f9
--- /dev/null
+++ b/drivers/nvmem/airoha-smc-efuses.c
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Author: Christian Marangi <ansuelsmth@gmail.com>
+ */
+
+#include <linux/arm-smccc.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/nvmem-provider.h>
+#include <linux/platform_device.h>
+#include <linux/of.h>
+#include <linux/regmap.h>
+
+#define AIROHA_SMC_EFUSE_FID		0x82000001
+#define AIROHA_SMC_EFUSE_SUB_ID_READ	0x44414552
+
+#define AIROHA_EFUSE_CELLS		64
+
+struct airoha_efuse_bank_priv {
+	u32 bank_index;
+};
+
+static int airoha_efuse_read(void *context, unsigned int offset,
+			     void *val, size_t bytes)
+{
+	struct regmap *regmap = context;
+
+	return regmap_bulk_read(regmap, offset,
+				val, bytes / sizeof(u32));
+}
+
+static int airoha_efuse_reg_read(void *context, unsigned int offset,
+				 unsigned int *val)
+{
+	struct airoha_efuse_bank_priv *priv = context;
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_invoke(AIROHA_SMC_EFUSE_FID,
+			     AIROHA_SMC_EFUSE_SUB_ID_READ,
+			     priv->bank_index, offset, 0, 0, 0, 0, &res);
+
+	/* check if SMC reported an error */
+	if (res.a0)
+		return -EIO;
+
+	*val = res.a1;
+	return 0;
+}
+
+static int airoha_efuse_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	for_each_child_of_node_scoped(dev->of_node, child) {
+		struct nvmem_config nvmem_config = {
+			.size = AIROHA_EFUSE_CELLS * sizeof(u32),
+			.stride = sizeof(u32),
+			.word_size = sizeof(u32),
+			.reg_read = airoha_efuse_read,
+		};
+		struct regmap_config regmap_config = {
+			.reg_read = airoha_efuse_reg_read,
+			.reg_bits = 32,
+			.val_bits = 32,
+			.reg_stride = 4,
+		};
+		struct airoha_efuse_bank_priv *priv;
+		struct nvmem_device *nvmem;
+		struct regmap *regmap;
+		const char *name;
+		u32 bank;
+
+		ret = of_property_read_u32(child, "reg", &bank);
+		if (ret)
+			return ret;
+
+		priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+		if (!priv)
+			return -ENOMEM;
+
+		name = devm_kasprintf(dev, GFP_KERNEL, "airoha-efuse-%u",
+				      bank);
+		if (!name)
+			return -ENOMEM;
+
+		priv->bank_index = bank;
+
+		regmap_config.name = name;
+		regmap = devm_regmap_init(dev, NULL, priv,
+					  &regmap_config);
+		if (IS_ERR(regmap))
+			return PTR_ERR(regmap);
+
+		nvmem_config.name = name;
+		nvmem_config.priv = regmap;
+		nvmem_config.dev = dev;
+		nvmem_config.id = bank;
+		nvmem_config.of_node = child;
+		nvmem = devm_nvmem_register(dev, &nvmem_config);
+		if (IS_ERR(nvmem))
+			return PTR_ERR(nvmem);
+	}
+
+	return 0;
+}
+
+static const struct of_device_id airoha_efuse_of_match[] = {
+	{ .compatible = "airoha,an7581-efuses", },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, airoha_efuse_of_match);
+
+static struct platform_driver airoha_efuse_driver = {
+	.probe = airoha_efuse_probe,
+	.driver = {
+		.name = "airoha-efuse",
+		.of_match_table = airoha_efuse_of_match,
+	},
+};
+module_platform_driver(airoha_efuse_driver);
+
+MODULE_AUTHOR("Christian Marangi <ansuelsmth@gmail.com>");
+MODULE_DESCRIPTION("Driver for Airoha SMC eFUSEs");
+MODULE_LICENSE("GPL");
diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c
index 311cb2e5a5c02..be28a366f6031 100644
--- a/drivers/nvmem/core.c
+++ b/drivers/nvmem/core.c
@@ -1019,6 +1019,10 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config)
 	if (rval)
 		goto err_remove_dev;
 
+	/* If the device has WP GPIO, default to read-only */
+	if (nvmem->wp_gpio)
+		nvmem->read_only = true;
+
 #ifdef CONFIG_NVMEM_SYSFS
 	rval = nvmem_populate_sysfs_cells(nvmem);
 	if (rval)
diff --git a/drivers/nvmem/lan9662-otpc.c b/drivers/nvmem/lan9662-otpc.c
index 56fc19f092a7f..62d1d6381bf88 100644
--- a/drivers/nvmem/lan9662-otpc.c
+++ b/drivers/nvmem/lan9662-otpc.c
@@ -27,7 +27,6 @@
 #define OTP_OTP_STATUS_OTP_CPUMPEN		BIT(1)
 #define OTP_OTP_STATUS_OTP_BUSY			BIT(0)
 
-#define OTP_MEM_SIZE 8192
 #define OTP_SLEEP_US 10
 #define OTP_TIMEOUT_US 500000
 
@@ -176,7 +175,6 @@ static struct nvmem_config otp_config = {
 	.word_size = 1,
 	.reg_read = lan9662_otp_read,
 	.reg_write = lan9662_otp_write,
-	.size = OTP_MEM_SIZE,
 };
 
 static int lan9662_otp_probe(struct platform_device *pdev)
@@ -196,6 +194,7 @@ static int lan9662_otp_probe(struct platform_device *pdev)
 
 	otp_config.priv = otp;
 	otp_config.dev = dev;
+	otp_config.size = (uintptr_t) device_get_match_data(dev);
 
 	nvmem = devm_nvmem_register(dev, &otp_config);
 
@@ -203,7 +202,14 @@ static int lan9662_otp_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id lan9662_otp_match[] = {
-	{ .compatible = "microchip,lan9662-otpc", },
+	{
+		.compatible = "microchip,lan9662-otpc",
+		.data = (const void *) SZ_8K,
+	},
+	{
+		.compatible = "microchip,lan9691-otpc",
+		.data = (const void *) SZ_16K,
+	},
 	{ },
 };
 MODULE_DEVICE_TABLE(of, lan9662_otp_match);
diff --git a/drivers/nvmem/layouts/onie-tlv.c b/drivers/nvmem/layouts/onie-tlv.c
index 0967a32319a28..8b0f3c1b8a0e9 100644
--- a/drivers/nvmem/layouts/onie-tlv.c
+++ b/drivers/nvmem/layouts/onie-tlv.c
@@ -119,7 +119,7 @@ static int onie_tlv_add_cells(struct device *dev, struct nvmem_device *nvmem,
 
 		cell.name = onie_tlv_cell_name(tlv.type);
 		if (!cell.name)
-			continue;
+			goto next;
 
 		cell.offset = hdr_len + offset + sizeof(tlv.type) + sizeof(tlv.len);
 		cell.bytes = tlv.len;
@@ -132,6 +132,7 @@ static int onie_tlv_add_cells(struct device *dev, struct nvmem_device *nvmem,
 			return ret;
 		}
 
+next:
 		offset += sizeof(tlv) + tlv.len;
 	}
 
diff --git a/drivers/nvmem/layouts/u-boot-env.c b/drivers/nvmem/layouts/u-boot-env.c
index f27f387bb52af..33ec2350386f5 100644
--- a/drivers/nvmem/layouts/u-boot-env.c
+++ b/drivers/nvmem/layouts/u-boot-env.c
@@ -38,9 +38,6 @@ static int u_boot_env_read_post_process_ethaddr(void *context, const char *id, i
 {
 	u8 mac[ETH_ALEN];
 
-	if (bytes != MAC_ADDR_STR_LEN)
-		return -EINVAL;
-
 	if (!mac_pton(buf, mac))
 		return -EINVAL;
 
@@ -75,7 +72,7 @@ static int u_boot_env_parse_cells(struct device *dev, struct nvmem_device *nvmem
 		info.offset = data_offset + value - data;
 		info.bytes = strlen(value);
 		info.np = of_get_child_by_name(dev->of_node, info.name);
-		if (!strcmp(var, "ethaddr")) {
+		if (!strcmp(var, "ethaddr") && info.bytes == MAC_ADDR_STR_LEN) {
 			info.raw_len = strlen(value);
 			info.bytes = ETH_ALEN;
 			info.read_post_process = u_boot_env_read_post_process_ethaddr;
diff --git a/drivers/nvmem/nintendo-otp.c b/drivers/nvmem/nintendo-otp.c
index 355e7f1fc6d54..e45a8a3a9774a 100644
--- a/drivers/nvmem/nintendo-otp.c
+++ b/drivers/nvmem/nintendo-otp.c
@@ -17,7 +17,7 @@
 #include <linux/module.h>
 #include <linux/mod_devicetable.h>
 #include <linux/nvmem-provider.h>
-#include <linux/of_device.h>
+#include <linux/of.h>
 #include <linux/platform_device.h>
 
 #define HW_OTPCMD  0
@@ -74,8 +74,7 @@ MODULE_DEVICE_TABLE(of, nintendo_otp_of_table);
 static int nintendo_otp_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	const struct of_device_id *of_id =
-		of_match_device(nintendo_otp_of_table, dev);
+	const struct nintendo_otp_devtype_data *data;
 	struct nvmem_device *nvmem;
 	struct nintendo_otp_priv *priv;
 
@@ -95,8 +94,8 @@ static int nintendo_otp_probe(struct platform_device *pdev)
 	if (IS_ERR(priv->regs))
 		return PTR_ERR(priv->regs);
 
-	if (of_id->data) {
-		const struct nintendo_otp_devtype_data *data = of_id->data;
+	data = of_device_get_match_data(dev);
+	if (data) {
 		config.name = data->name;
 		config.size = data->num_banks * BANK_SIZE;
 	}
diff --git a/drivers/nvmem/rockchip-otp.c b/drivers/nvmem/rockchip-otp.c
index 0ec78b5e19e7d..2c0feb036f3fd 100644
--- a/drivers/nvmem/rockchip-otp.c
+++ b/drivers/nvmem/rockchip-otp.c
@@ -78,9 +78,9 @@ struct rockchip_data {
 struct rockchip_otp {
 	struct device *dev;
 	void __iomem *base;
-	struct clk_bulk_data *clks;
 	struct reset_control *rst;
 	const struct rockchip_data *data;
+	struct clk_bulk_data clks[];
 };
 
 static int rockchip_otp_reset(struct rockchip_otp *otp)
@@ -424,7 +424,7 @@ static int rockchip_otp_probe(struct platform_device *pdev)
 	if (!data)
 		return dev_err_probe(dev, -EINVAL, "failed to get match data\n");
 
-	otp = devm_kzalloc(&pdev->dev, sizeof(struct rockchip_otp),
+	otp = devm_kzalloc(&pdev->dev, struct_size(otp, clks, data->num_clks),
 			   GFP_KERNEL);
 	if (!otp)
 		return -ENOMEM;
@@ -436,11 +436,6 @@ static int rockchip_otp_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(otp->base),
 				     "failed to ioremap resource\n");
 
-	otp->clks = devm_kcalloc(dev, data->num_clks, sizeof(*otp->clks),
-				 GFP_KERNEL);
-	if (!otp->clks)
-		return -ENOMEM;
-
 	for (i = 0; i < data->num_clks; ++i)
 		otp->clks[i].id = data->clks[i];
 
diff --git a/drivers/of/base.c b/drivers/of/base.c
index a650c91897cc0..f493a7a99a52e 100644
--- a/drivers/of/base.c
+++ b/drivers/of/base.c
@@ -82,9 +82,17 @@ EXPORT_SYMBOL(of_node_name_prefix);
 
 static bool __of_node_is_type(const struct device_node *np, const char *type)
 {
-	const char *match = __of_get_property(np, "device_type", NULL);
+	const char *match;
+	int len;
+
+	if (!np || !type)
+		return false;
+
+	match = __of_get_property(np, "device_type", &len);
+	if (!match || len <= 0 || strnlen(match, len) >= len)
+		return false;
 
-	return np && match && type && !strcmp(match, type);
+	return !strcmp(match, type);
 }
 
 #define EXCLUDED_DEFAULT_CELLS_PLATFORMS ( \
@@ -511,22 +519,22 @@ static bool __of_device_is_status(const struct device_node *device,
 		return false;
 
 	status = __of_get_property(device, "status", &statlen);
-	if (status == NULL)
+	if (!status || statlen <= 0)
+		return false;
+	if (strnlen(status, statlen) >= statlen)
 		return false;
 
-	if (statlen > 0) {
-		while (*strings) {
-			unsigned int len = strlen(*strings);
+	while (*strings) {
+		unsigned int len = strlen(*strings);
 
-			if ((*strings)[len - 1] == '-') {
-				if (!strncmp(status, *strings, len))
-					return true;
-			} else {
-				if (!strcmp(status, *strings))
-					return true;
-			}
-			strings++;
+		if ((*strings)[len - 1] == '-') {
+			if (!strncmp(status, *strings, len))
+				return true;
+		} else {
+			if (!strcmp(status, *strings))
+				return true;
 		}
+		strings++;
 	}
 
 	return false;
@@ -1237,10 +1245,11 @@ EXPORT_SYMBOL(of_find_matching_node_and_match);
 int of_alias_from_compatible(const struct device_node *node, char *alias, int len)
 {
 	const char *compatible, *p;
-	int cplen;
+	int ret;
 
-	compatible = of_get_property(node, "compatible", &cplen);
-	if (!compatible || strlen(compatible) > cplen)
+	ret = of_property_read_string_index(node, "compatible", 0,
+					    &compatible);
+	if (ret)
 		return -ENODEV;
 	p = strchr(compatible, ',');
 	strscpy(alias, p ? p + 1 : compatible, len);
diff --git a/drivers/of/cpu.c b/drivers/of/cpu.c
index 5214dc3d05ae1..bd0e918d6f290 100644
--- a/drivers/of/cpu.c
+++ b/drivers/of/cpu.c
@@ -60,7 +60,7 @@ static bool __of_find_n_match_cpu_property(struct device_node *cpun,
 	cell = of_get_property(cpun, prop_name, &prop_len);
 	if (!cell && !ac && arch_match_cpu_phys_id(cpu, 0))
 		return true;
-	if (!cell || !ac)
+	if (!cell || !ac || ac > 2)
 		return false;
 	prop_len /= sizeof(*cell) * ac;
 	for (tid = 0; tid < prop_len; tid++) {
diff --git a/drivers/of/device.c b/drivers/of/device.c
index f7e75e5276677..be4e1584e0afb 100644
--- a/drivers/of/device.c
+++ b/drivers/of/device.c
@@ -26,7 +26,7 @@
 const struct of_device_id *of_match_device(const struct of_device_id *matches,
 					   const struct device *dev)
 {
-	if (!matches || !dev->of_node || dev->of_node_reused)
+	if (!matches || !dev->of_node || dev_of_node_reused(dev))
 		return NULL;
 	return of_match_node(matches, dev->of_node);
 }
@@ -192,7 +192,7 @@ ssize_t of_device_modalias(struct device *dev, char *str, ssize_t len)
 {
 	ssize_t sl;
 
-	if (!dev || !dev->of_node || dev->of_node_reused)
+	if (!dev || !dev->of_node || dev_of_node_reused(dev))
 		return -ENODEV;
 
 	sl = of_modalias(dev->of_node, str, len - 2);
@@ -254,7 +254,7 @@ int of_device_uevent_modalias(const struct device *dev, struct kobj_uevent_env *
 {
 	int sl;
 
-	if ((!dev) || (!dev->of_node) || dev->of_node_reused)
+	if ((!dev) || (!dev->of_node) || dev_of_node_reused(dev))
 		return -ENODEV;
 
 	/* Devicetree modalias is tricky, we add it in 2 steps */
diff --git a/drivers/of/dynamic.c b/drivers/of/dynamic.c
index ade288372101b..442590d6511a7 100644
--- a/drivers/of/dynamic.c
+++ b/drivers/of/dynamic.c
@@ -74,6 +74,20 @@ static const char *action_names[] = {
 	[OF_RECONFIG_UPDATE_PROPERTY] = "UPDATE_PROPERTY",
 };
 
+static bool of_property_status_ok(const struct property *prop)
+{
+	const char *status;
+
+	if (!prop || !prop->value || prop->length <= 0)
+		return false;
+
+	status = prop->value;
+	if (strnlen(status, prop->length) >= prop->length)
+		return false;
+
+	return !strcmp(status, "okay") || !strcmp(status, "ok");
+}
+
 #define _do_print(func, prefix, action, node, prop, ...) ({	\
 	func("changeset: " prefix "%-15s %pOF%s%s\n",		\
 	     ##__VA_ARGS__, action_names[action], node,		\
@@ -135,11 +149,9 @@ int of_reconfig_get_state_change(unsigned long action, struct of_reconfig_data *
 
 	if (prop && !strcmp(prop->name, "status")) {
 		is_status = 1;
-		status_state = !strcmp(prop->value, "okay") ||
-			       !strcmp(prop->value, "ok");
+		status_state = of_property_status_ok(prop);
 		if (old_prop)
-			old_status_state = !strcmp(old_prop->value, "okay") ||
-					   !strcmp(old_prop->value, "ok");
+			old_status_state = of_property_status_ok(old_prop);
 	}
 
 	switch (action) {
diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c
index 82f7327c59ea9..ba65e36e183cc 100644
--- a/drivers/of/fdt.c
+++ b/drivers/of/fdt.c
@@ -391,8 +391,6 @@ void *__unflatten_device_tree(const void *blob,
 	if (!mem)
 		return NULL;
 
-	memset(mem, 0, size);
-
 	*(__be32 *)(mem + size) = cpu_to_be32(0xdeadbeef);
 
 	pr_debug("  unflattening %p...\n", mem);
diff --git a/drivers/of/property.c b/drivers/of/property.c
index 136946f8b746f..b276d1de32222 100644
--- a/drivers/of/property.c
+++ b/drivers/of/property.c
@@ -648,16 +648,31 @@ EXPORT_SYMBOL_GPL(of_prop_next_u32);
 
 const char *of_prop_next_string(const struct property *prop, const char *cur)
 {
-	const void *curv = cur;
+	const char *curv;
+	const char *end;
+	size_t len;
 
-	if (!prop)
+	if (!prop || !prop->value || !prop->length)
 		return NULL;
 
-	if (!cur)
-		return prop->value;
+	curv = cur ? cur : prop->value;
+	end = prop->value + prop->length;
 
-	curv += strlen(cur) + 1;
-	if (curv >= prop->value + prop->length)
+	if (curv < (const char *)prop->value || curv >= end)
+		return NULL;
+
+	if (cur) {
+		len = strnlen(curv, end - curv);
+		if (len >= end - curv)
+			return NULL;
+
+		curv += len + 1;
+		if (curv >= end)
+			return NULL;
+	}
+
+	len = strnlen(curv, end - curv);
+	if (len >= end - curv)
 		return NULL;
 
 	return curv;
diff --git a/drivers/of/unittest.c b/drivers/of/unittest.c
index 4078569a0f967..e255f54f4d760 100644
--- a/drivers/of/unittest.c
+++ b/drivers/of/unittest.c
@@ -713,6 +713,7 @@ static void __init of_unittest_parse_phandle_with_args_map(void)
 static void __init of_unittest_property_string(void)
 {
 	const char *strings[4];
+	const struct property *prop;
 	struct device_node *np;
 	int rc;
 
@@ -789,6 +790,37 @@ static void __init of_unittest_property_string(void)
 	strings[1] = NULL;
 	rc = of_property_read_string_array(np, "phandle-list-names", strings, 1);
 	unittest(rc == 1 && strings[1] == NULL, "Overwrote end of string array; rc=%i, str='%s'\n", rc, strings[1]);
+
+	/* of_prop_next_string() tests */
+	prop = of_find_property(np, "phandle-list-names", NULL);
+	strings[0] = of_prop_next_string(prop, NULL);
+	unittest(strings[0] && !strcmp(strings[0], "first"),
+		 "of_prop_next_string() failure; got '%s'\n", strings[0]);
+	strings[0] = of_prop_next_string(prop, strings[0]);
+	unittest(strings[0] && !strcmp(strings[0], "second"),
+		 "of_prop_next_string() failure; got '%s'\n", strings[0]);
+	strings[0] = of_prop_next_string(prop, strings[0]);
+	unittest(strings[0] && !strcmp(strings[0], "third"),
+		 "of_prop_next_string() failure; got '%s'\n", strings[0]);
+	strings[0] = of_prop_next_string(prop, strings[0]);
+	unittest(!strings[0],
+		 "of_prop_next_string() should return NULL at end of list\n");
+
+	prop = of_find_property(np, "unterminated-string", NULL);
+	strings[0] = of_prop_next_string(prop, NULL);
+	unittest(!strings[0],
+		 "of_prop_next_string() should reject unterminated first string\n");
+
+	prop = of_find_property(np, "unterminated-string-list", NULL);
+	strings[0] = of_prop_next_string(prop, NULL);
+	unittest(strings[0] && !strcmp(strings[0], "first"),
+		 "of_prop_next_string() failure; got '%s'\n", strings[0]);
+	strings[0] = of_prop_next_string(prop, strings[0]);
+	unittest(strings[0] && !strcmp(strings[0], "second"),
+		 "of_prop_next_string() failure; got '%s'\n", strings[0]);
+	strings[0] = of_prop_next_string(prop, strings[0]);
+	unittest(!strings[0],
+		 "of_prop_next_string() should reject unterminated trailing string\n");
 }
 
 #define propcmp(p1, p2) (((p1)->length == (p2)->length) && \
diff --git a/drivers/opp/core.c b/drivers/opp/core.c
index da3f5eba43419..ab0b0a2f85a17 100644
--- a/drivers/opp/core.c
+++ b/drivers/opp/core.c
@@ -2088,11 +2088,10 @@ int _opp_add(struct device *dev, struct dev_pm_opp *new_opp,
 			return ret;
 
 		list_add(&new_opp->node, head);
+		new_opp->opp_table = opp_table;
+		kref_init(&new_opp->kref);
 	}
 
-	new_opp->opp_table = opp_table;
-	kref_init(&new_opp->kref);
-
 	opp_debug_create_one(new_opp, opp_table);
 
 	if (!_opp_supported_by_regulators(new_opp, opp_table)) {
diff --git a/drivers/opp/of.c b/drivers/opp/of.c
index f96adfd5b2197..c02e20632fa64 100644
--- a/drivers/opp/of.c
+++ b/drivers/opp/of.c
@@ -673,7 +673,7 @@ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev,
 	 */
 	if (unlikely(opp_table->regulator_count == -1)) {
 		opp_table->regulator_count = 0;
-		return 0;
+		goto free_microwatt;
 	}
 
 	for (i = 0, j = 0; i < opp_table->regulator_count; i++) {
@@ -696,6 +696,7 @@ static int opp_parse_supplies(struct dev_pm_opp *opp, struct device *dev,
 			opp->supplies[i].u_watt = microwatt[i];
 	}
 
+free_microwatt:
 	kfree(microwatt);
 free_microamp:
 	kfree(microamp);
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index 33c88432b7287..0c7408509ba24 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -251,63 +251,6 @@ config PCI_DYNAMIC_OF_NODES
 	  Once this option is selected, the device tree nodes will be generated
 	  for all PCI bridges.
 
-choice
-	prompt "PCI Express hierarchy optimization setting"
-	default PCIE_BUS_DEFAULT
-	depends on EXPERT
-	help
-	  MPS (Max Payload Size) and MRRS (Max Read Request Size) are PCIe
-	  device parameters that affect performance and the ability to
-	  support hotplug and peer-to-peer DMA.
-
-	  The following choices set the MPS and MRRS optimization strategy
-	  at compile-time.  The choices are the same as those offered for
-	  the kernel command-line parameter 'pci', i.e.,
-	  'pci=pcie_bus_tune_off', 'pci=pcie_bus_safe',
-	  'pci=pcie_bus_perf', and 'pci=pcie_bus_peer2peer'.
-
-	  This is a compile-time setting and can be overridden by the above
-	  command-line parameters.  If unsure, choose PCIE_BUS_DEFAULT.
-
-config PCIE_BUS_TUNE_OFF
-	bool "Tune Off"
-	help
-	  Use the BIOS defaults; don't touch MPS at all.  This is the same
-	  as booting with 'pci=pcie_bus_tune_off'.
-
-config PCIE_BUS_DEFAULT
-	bool "Default"
-	help
-	  Default choice; ensure that the MPS matches upstream bridge.
-
-config PCIE_BUS_SAFE
-	bool "Safe"
-	help
-	  Use largest MPS that boot-time devices support.  If you have a
-	  closed system with no possibility of adding new devices, this
-	  will use the largest MPS that's supported by all devices.  This
-	  is the same as booting with 'pci=pcie_bus_safe'.
-
-config PCIE_BUS_PERFORMANCE
-	bool "Performance"
-	help
-	  Use MPS and MRRS for best performance.  Ensure that a given
-	  device's MPS is no larger than its parent MPS, which allows us to
-	  keep all switches/bridges to the max MPS supported by their
-	  parent.  This is the same as booting with 'pci=pcie_bus_perf'.
-
-config PCIE_BUS_PEER2PEER
-	bool "Peer2peer"
-	help
-	  Set MPS = 128 for all devices.  MPS configuration effected by the
-	  other options could cause the MPS on one root port to be
-	  different than that of the MPS on another, which may cause
-	  hot-added devices or peer-to-peer DMA to fail.  Set MPS to the
-	  smallest possible value (128B) system-wide to avoid these issues.
-	  This is the same as booting with 'pci=pcie_bus_peer2peer'.
-
-endchoice
-
 config VGA_ARB
 	bool "VGA Arbitration" if EXPERT
 	default y
diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c
index 6c1ad1f542d9b..655ed53436d3e 100644
--- a/drivers/pci/bus.c
+++ b/drivers/pci/bus.c
@@ -354,7 +354,6 @@ void pci_bus_add_device(struct pci_dev *dev)
 	pci_fixup_device(pci_fixup_final, dev);
 	if (pci_is_bridge(dev))
 		of_pci_make_dev_node(dev);
-	pci_create_sysfs_dev_files(dev);
 	pci_proc_attach_device(dev);
 	pci_bridge_d3_update(dev);
 
diff --git a/drivers/pci/controller/dwc/Kconfig b/drivers/pci/controller/dwc/Kconfig
index f2fde13107f2e..216ede0a867e5 100644
--- a/drivers/pci/controller/dwc/Kconfig
+++ b/drivers/pci/controller/dwc/Kconfig
@@ -560,4 +560,17 @@ config PCIE_VISCONTI_HOST
 	  Say Y here if you want PCIe controller support on Toshiba Visconti SoC.
 	  This driver supports TMPV7708 SoC.
 
+config PCIE_ULTRARISC
+	tristate "UltraRISC PCIe host controller"
+	depends on ARCH_ULTRARISC || COMPILE_TEST
+	select PCIE_DW_HOST
+	select PCI_MSI
+	default y if ARCH_ULTRARISC
+	help
+	  Enables support for the PCIe controller in the UltraRISC SoC.
+	  This driver supports UR-DP1000 SoC.
+
+	  By default, this symbol is enabled when ARCH_ULTRARISC is active,
+	  requiring no further configuration on that platform.
+
 endmenu
diff --git a/drivers/pci/controller/dwc/Makefile b/drivers/pci/controller/dwc/Makefile
index 7177451db8aa1..d4f88cfc6229f 100644
--- a/drivers/pci/controller/dwc/Makefile
+++ b/drivers/pci/controller/dwc/Makefile
@@ -39,6 +39,7 @@ obj-$(CONFIG_PCIE_RCAR_GEN4) += pcie-rcar-gen4.o
 obj-$(CONFIG_PCIE_SPACEMIT_K1) += pcie-spacemit-k1.o
 obj-$(CONFIG_PCIE_STM32_HOST) += pcie-stm32.o
 obj-$(CONFIG_PCIE_STM32_EP) += pcie-stm32-ep.o
+obj-$(CONFIG_PCIE_ULTRARISC) += pcie-ultrarisc.o
 
 # The following drivers are for devices that use the generic ACPI
 # pci_root.c driver but don't support standard ECAM config access.
diff --git a/drivers/pci/controller/dwc/pci-dra7xx.c b/drivers/pci/controller/dwc/pci-dra7xx.c
index cd904659c3216..3fc889944f021 100644
--- a/drivers/pci/controller/dwc/pci-dra7xx.c
+++ b/drivers/pci/controller/dwc/pci-dra7xx.c
@@ -92,7 +92,6 @@ struct dra7xx_pcie {
 	struct phy		**phy;
 	struct irq_domain	*irq_domain;
 	struct clk              *clk;
-	enum dw_pcie_device_mode mode;
 };
 
 struct dra7xx_pcie_of_data {
@@ -328,7 +327,7 @@ static irqreturn_t dra7xx_pcie_irq_handler(int irq, void *arg)
 		dev_dbg(dev, "Link Request Reset\n");
 
 	if (reg & LINK_UP_EVT) {
-		if (dra7xx->mode == DW_PCIE_EP_TYPE)
+		if (dra7xx->pci->mode == DW_PCIE_EP_TYPE)
 			dw_pcie_ep_linkup(ep);
 		dev_dbg(dev, "Link-up state change\n");
 	}
@@ -828,7 +827,7 @@ static int dra7xx_pcie_probe(struct platform_device *pdev)
 	default:
 		dev_err(dev, "INVALID device type %d\n", mode);
 	}
-	dra7xx->mode = mode;
+	dra7xx->pci->mode = mode;
 
 	ret = devm_request_threaded_irq(dev, irq, NULL, dra7xx_pcie_irq_handler,
 					IRQF_SHARED | IRQF_ONESHOT,
@@ -841,7 +840,7 @@ static int dra7xx_pcie_probe(struct platform_device *pdev)
 	return 0;
 
 err_deinit:
-	if (dra7xx->mode == DW_PCIE_RC_TYPE)
+	if (dra7xx->pci->mode == DW_PCIE_RC_TYPE)
 		dw_pcie_host_deinit(&dra7xx->pci->pp);
 	else
 		dw_pcie_ep_deinit(&dra7xx->pci->ep);
@@ -865,7 +864,7 @@ static int dra7xx_pcie_suspend(struct device *dev)
 	struct dw_pcie *pci = dra7xx->pci;
 	u32 val;
 
-	if (dra7xx->mode != DW_PCIE_RC_TYPE)
+	if (pci->mode != DW_PCIE_RC_TYPE)
 		return 0;
 
 	/* clear MSE */
@@ -882,7 +881,7 @@ static int dra7xx_pcie_resume(struct device *dev)
 	struct dw_pcie *pci = dra7xx->pci;
 	u32 val;
 
-	if (dra7xx->mode != DW_PCIE_RC_TYPE)
+	if (pci->mode != DW_PCIE_RC_TYPE)
 		return 0;
 
 	/* set MSE */
diff --git a/drivers/pci/controller/dwc/pci-imx6.c b/drivers/pci/controller/dwc/pci-imx6.c
index e35044cc52185..773ab65b2afac 100644
--- a/drivers/pci/controller/dwc/pci-imx6.c
+++ b/drivers/pci/controller/dwc/pci-imx6.c
@@ -34,6 +34,7 @@
 #include <linux/pm_runtime.h>
 
 #include "../../pci.h"
+#include "../pci-host-common.h"
 #include "pcie-designware.h"
 
 #define IMX8MQ_GPR_PCIE_REF_USE_PAD		BIT(9)
@@ -152,7 +153,6 @@ struct imx_lut_data {
 
 struct imx_pcie {
 	struct dw_pcie		*pci;
-	struct gpio_desc	*reset_gpiod;
 	struct clk_bulk_data	*clks;
 	int			num_clks;
 	bool			supports_clkreq;
@@ -168,6 +168,8 @@ struct imx_pcie {
 	u32			tx_swing_full;
 	u32			tx_swing_low;
 	struct regulator	*vpcie;
+	struct regulator	*vpcie_aux;
+	bool			vpcie_aux_enabled;
 	struct regulator	*vph;
 	void __iomem		*phy_base;
 
@@ -665,14 +667,6 @@ static int imx_pcie_attach_pd(struct device *dev)
 	return 0;
 }
 
-static int imx6sx_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
-{
-	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
-			   IMX6SX_GPR12_PCIE_TEST_POWERDOWN,
-			   enable ? 0 : IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
-	return 0;
-}
-
 static int imx6q_pcie_enable_ref_clk(struct imx_pcie *imx_pcie, bool enable)
 {
 	if (enable) {
@@ -786,6 +780,9 @@ static int imx6sx_pcie_core_reset(struct imx_pcie *imx_pcie, bool assert)
 	if (assert)
 		regmap_set_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
 				IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
+	else
+		regmap_clear_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR12,
+				  IMX6SX_GPR12_PCIE_TEST_POWERDOWN);
 
 	/* Force PCIe PHY reset */
 	regmap_update_bits(imx_pcie->iomuxc_gpr, IOMUXC_GPR5, IMX6SX_GPR5_PCIE_BTNRST_RESET,
@@ -1222,16 +1219,70 @@ static void imx_pcie_disable_device(struct pci_host_bridge *bridge,
 	imx_pcie_remove_lut(imx_pcie, pci_dev_id(pdev));
 }
 
+static int imx_pcie_parse_legacy_binding(struct imx_pcie *pcie)
+{
+	struct device *dev = pcie->pci->dev;
+	struct pci_host_bridge *bridge = pcie->pci->pp.bridge;
+	struct pci_host_port *port;
+	struct pci_host_perst *perst;
+	struct gpio_desc *reset;
+
+	reset = devm_gpiod_get_optional(dev, "reset", GPIOD_ASIS);
+	if (IS_ERR(reset))
+		return PTR_ERR(reset);
+
+	if (!reset)
+		return 0;
+
+	port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
+	if (!port)
+		return -ENOMEM;
+
+	perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL);
+	if (!perst)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&port->perst);
+	perst->desc = reset;
+	INIT_LIST_HEAD(&perst->list);
+	list_add_tail(&perst->list, &port->perst);
+
+	INIT_LIST_HEAD(&port->list);
+	list_add_tail(&port->list, &bridge->ports);
+
+	return devm_add_action_or_reset(dev, pci_host_common_delete_ports,
+					&bridge->ports);
+}
+
+static void imx_pcie_vpcie_aux_disable(void *data)
+{
+	struct regulator *vpcie_aux = data;
+
+	regulator_disable(vpcie_aux);
+}
+
 static void imx_pcie_assert_perst(struct imx_pcie *imx_pcie, bool assert)
 {
+	struct dw_pcie *pci = imx_pcie->pci;
+	struct pci_host_bridge *bridge = pci->pp.bridge;
+	struct pci_host_perst *perst;
+	struct pci_host_port *port;
+
+	if (!bridge || list_empty(&bridge->ports))
+		return;
+
 	if (assert) {
-		gpiod_set_value_cansleep(imx_pcie->reset_gpiod, 1);
+		list_for_each_entry(port, &bridge->ports, list) {
+			list_for_each_entry(perst, &port->perst, list)
+				gpiod_direction_output(perst->desc, 1);
+		}
 	} else {
-		if (imx_pcie->reset_gpiod) {
-			msleep(PCIE_T_PVPERL_MS);
-			gpiod_set_value_cansleep(imx_pcie->reset_gpiod, 0);
-			msleep(PCIE_RESET_CONFIG_WAIT_MS);
+		mdelay(PCIE_T_PVPERL_MS);
+		list_for_each_entry(port, &bridge->ports, list) {
+			list_for_each_entry(perst, &port->perst, list)
+				gpiod_direction_output(perst->desc, 0);
 		}
+		mdelay(PCIE_RESET_CONFIG_WAIT_MS);
 	}
 }
 
@@ -1240,8 +1291,46 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp)
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
 	struct device *dev = pci->dev;
 	struct imx_pcie *imx_pcie = to_imx_pcie(pci);
+	struct pci_host_bridge *bridge = pp->bridge;
 	int ret;
 
+	if (bridge && list_empty(&bridge->ports)) {
+		/* Parse Root Port nodes if present */
+		ret = pci_host_common_parse_ports(dev, bridge);
+		if (ret) {
+			if (ret != -ENODEV) {
+				dev_err(dev, "Failed to parse Root Port nodes: %d\n", ret);
+				return ret;
+			}
+
+			/*
+			 * Fall back to legacy binding for DT backwards
+			 * compatibility
+			 */
+			ret = imx_pcie_parse_legacy_binding(imx_pcie);
+			if (ret)
+				return ret;
+		}
+	}
+
+	imx_pcie_assert_perst(imx_pcie, true);
+
+	/* Keep 3.3Vaux supply enabled for entire PCIe controller lifecycle */
+	if (imx_pcie->vpcie_aux && !imx_pcie->vpcie_aux_enabled) {
+		ret = regulator_enable(imx_pcie->vpcie_aux);
+		if (ret) {
+			dev_err(dev, "failed to enable vpcie_aux regulator: %d\n",
+				ret);
+			return ret;
+		}
+		imx_pcie->vpcie_aux_enabled = true;
+
+		ret = devm_add_action_or_reset(dev, imx_pcie_vpcie_aux_disable,
+					       imx_pcie->vpcie_aux);
+		if (ret)
+			return ret;
+	}
+
 	if (imx_pcie->vpcie) {
 		ret = regulator_enable(imx_pcie->vpcie);
 		if (ret) {
@@ -1251,25 +1340,24 @@ static int imx_pcie_host_init(struct dw_pcie_rp *pp)
 		}
 	}
 
+	ret = imx_pcie_clk_enable(imx_pcie);
+	if (ret) {
+		dev_err(dev, "unable to enable pcie clocks: %d\n", ret);
+		goto err_reg_disable;
+	}
+
 	if (pp->bridge && imx_check_flag(imx_pcie, IMX_PCIE_FLAG_HAS_LUT)) {
 		pp->bridge->enable_device = imx_pcie_enable_device;
 		pp->bridge->disable_device = imx_pcie_disable_device;
 	}
 
 	imx_pcie_assert_core_reset(imx_pcie);
-	imx_pcie_assert_perst(imx_pcie, true);
 
 	if (imx_pcie->drvdata->init_phy)
 		imx_pcie->drvdata->init_phy(imx_pcie);
 
 	imx_pcie_configure_type(imx_pcie);
 
-	ret = imx_pcie_clk_enable(imx_pcie);
-	if (ret) {
-		dev_err(dev, "unable to enable pcie clocks: %d\n", ret);
-		goto err_reg_disable;
-	}
-
 	if (imx_pcie->phy) {
 		ret = phy_init(imx_pcie->phy);
 		if (ret) {
@@ -1678,13 +1766,6 @@ static int imx_pcie_probe(struct platform_device *pdev)
 			return PTR_ERR(imx_pcie->phy_base);
 	}
 
-	/* Fetch GPIOs */
-	imx_pcie->reset_gpiod = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_HIGH);
-	if (IS_ERR(imx_pcie->reset_gpiod))
-		return dev_err_probe(dev, PTR_ERR(imx_pcie->reset_gpiod),
-				     "unable to get reset gpio\n");
-	gpiod_set_consumer_name(imx_pcie->reset_gpiod, "PCIe reset");
-
 	/* Fetch clocks */
 	imx_pcie->num_clks = devm_clk_bulk_get_all(dev, &imx_pcie->clks);
 	if (imx_pcie->num_clks < 0)
@@ -1782,9 +1863,13 @@ static int imx_pcie_probe(struct platform_device *pdev)
 	of_property_read_u32(node, "fsl,max-link-speed", &pci->max_link_speed);
 	imx_pcie->supports_clkreq = of_property_read_bool(node, "supports-clkreq");
 
-	ret = devm_regulator_get_enable_optional(&pdev->dev, "vpcie3v3aux");
-	if (ret < 0 && ret != -ENODEV)
-		return dev_err_probe(dev, ret, "failed to enable Vaux supply\n");
+	imx_pcie->vpcie_aux = devm_regulator_get_optional(&pdev->dev,
+							  "vpcie3v3aux");
+	if (IS_ERR(imx_pcie->vpcie_aux)) {
+		if (PTR_ERR(imx_pcie->vpcie_aux) != -ENODEV)
+			return PTR_ERR(imx_pcie->vpcie_aux);
+		imx_pcie->vpcie_aux = NULL;
+	}
 
 	imx_pcie->vpcie = devm_regulator_get_optional(&pdev->dev, "vpcie");
 	if (IS_ERR(imx_pcie->vpcie)) {
@@ -1877,7 +1962,6 @@ static const struct imx_pcie_drvdata drvdata[] = {
 		.mode_off[0] = IOMUXC_GPR12,
 		.mode_mask[0] = IMX6Q_GPR12_DEVICE_TYPE,
 		.init_phy = imx6sx_pcie_init_phy,
-		.enable_ref_clk = imx6sx_pcie_enable_ref_clk,
 		.core_reset = imx6sx_pcie_core_reset,
 		.ops = &imx_pcie_host_ops,
 	},
diff --git a/drivers/pci/controller/dwc/pcie-al.c b/drivers/pci/controller/dwc/pcie-al.c
index 345c281c74fef..ef8c8ce7c78c7 100644
--- a/drivers/pci/controller/dwc/pcie-al.c
+++ b/drivers/pci/controller/dwc/pcie-al.c
@@ -253,7 +253,6 @@ static int al_pcie_config_prepare(struct al_pcie *pcie)
 	u8 subordinate_bus;
 	u8 secondary_bus;
 	u32 cfg_control;
-	u32 reg;
 
 	ft = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
 	if (!ft)
@@ -285,14 +284,9 @@ static int al_pcie_config_prepare(struct al_pcie *pcie)
 			     CFG_CONTROL;
 
 	cfg_control = al_pcie_controller_readl(pcie, cfg_control_offset);
-
-	reg = cfg_control &
-	      ~(CFG_CONTROL_SEC_BUS_MASK | CFG_CONTROL_SUBBUS_MASK);
-
-	reg |= FIELD_PREP(CFG_CONTROL_SUBBUS_MASK, subordinate_bus) |
-	       FIELD_PREP(CFG_CONTROL_SEC_BUS_MASK, secondary_bus);
-
-	al_pcie_controller_writel(pcie, cfg_control_offset, reg);
+	FIELD_MODIFY(CFG_CONTROL_SUBBUS_MASK, &cfg_control, subordinate_bus);
+	FIELD_MODIFY(CFG_CONTROL_SEC_BUS_MASK, &cfg_control, secondary_bus);
+	al_pcie_controller_writel(pcie, cfg_control_offset, cfg_control);
 
 	return 0;
 }
diff --git a/drivers/pci/controller/dwc/pcie-amd-mdb.c b/drivers/pci/controller/dwc/pcie-amd-mdb.c
index 7e50e11fbffda..dee295f281a30 100644
--- a/drivers/pci/controller/dwc/pcie-amd-mdb.c
+++ b/drivers/pci/controller/dwc/pcie-amd-mdb.c
@@ -7,7 +7,7 @@
 
 #include <linux/clk.h>
 #include <linux/delay.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/interrupt.h>
 #include <linux/irqdomain.h>
 #include <linux/kernel.h>
@@ -507,6 +507,13 @@ static int amd_mdb_pcie_probe(struct platform_device *pdev)
 	return amd_mdb_add_pcie_port(pcie, pdev);
 }
 
+static void amd_mdb_pcie_shutdown(struct platform_device *pdev)
+{
+	struct amd_mdb_pcie *pcie = platform_get_drvdata(pdev);
+
+	gpiod_set_value_cansleep(pcie->perst_gpio, 1);
+}
+
 static const struct of_device_id amd_mdb_pcie_of_match[] = {
 	{
 		.compatible = "amd,versal2-mdb-host",
@@ -521,6 +528,7 @@ static struct platform_driver amd_mdb_pcie_driver = {
 		.suppress_bind_attrs = true,
 	},
 	.probe = amd_mdb_pcie_probe,
+	.shutdown = amd_mdb_pcie_shutdown,
 };
 
 builtin_platform_driver(amd_mdb_pcie_driver);
diff --git a/drivers/pci/controller/dwc/pcie-artpec6.c b/drivers/pci/controller/dwc/pcie-artpec6.c
index 55cb957ae1f3b..5cd227dda9a10 100644
--- a/drivers/pci/controller/dwc/pcie-artpec6.c
+++ b/drivers/pci/controller/dwc/pcie-artpec6.c
@@ -34,7 +34,6 @@ struct artpec6_pcie {
 	struct regmap		*regmap;	/* DT axis,syscon-pcie */
 	void __iomem		*phy_base;	/* DT phy */
 	enum artpec_pcie_variants variant;
-	enum dw_pcie_device_mode mode;
 };
 
 struct artpec_pcie_of_data {
@@ -100,7 +99,7 @@ static u64 artpec6_pcie_cpu_addr_fixup(struct dw_pcie *pci, u64 cpu_addr)
 	struct dw_pcie_rp *pp = &pci->pp;
 	struct dw_pcie_ep *ep = &pci->ep;
 
-	switch (artpec6_pcie->mode) {
+	switch (artpec6_pcie->pci->mode) {
 	case DW_PCIE_RC_TYPE:
 		return cpu_addr - pp->cfg0_base;
 	case DW_PCIE_EP_TYPE:
@@ -413,7 +412,7 @@ static int artpec6_pcie_probe(struct platform_device *pdev)
 
 	artpec6_pcie->pci = pci;
 	artpec6_pcie->variant = variant;
-	artpec6_pcie->mode = mode;
+	artpec6_pcie->pci->mode = mode;
 
 	artpec6_pcie->phy_base =
 		devm_platform_ioremap_resource_byname(pdev, "phy");
@@ -428,7 +427,7 @@ static int artpec6_pcie_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, artpec6_pcie);
 
-	switch (artpec6_pcie->mode) {
+	switch (artpec6_pcie->pci->mode) {
 	case DW_PCIE_RC_TYPE:
 		if (!IS_ENABLED(CONFIG_PCIE_ARTPEC6_HOST))
 			return -ENODEV;
@@ -464,7 +463,7 @@ static int artpec6_pcie_probe(struct platform_device *pdev)
 
 		break;
 	default:
-		dev_err(dev, "INVALID device type %d\n", artpec6_pcie->mode);
+		dev_err(dev, "INVALID device type %d\n", artpec6_pcie->pci->mode);
 	}
 
 	return 0;
diff --git a/drivers/pci/controller/dwc/pcie-designware-debugfs.c b/drivers/pci/controller/dwc/pcie-designware-debugfs.c
index d0884253be97e..33835dfdab70b 100644
--- a/drivers/pci/controller/dwc/pcie-designware-debugfs.c
+++ b/drivers/pci/controller/dwc/pcie-designware-debugfs.c
@@ -265,8 +265,7 @@ static ssize_t lane_detect_write(struct file *file, const char __user *buf,
 		return ret;
 
 	val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + SD_STATUS_L1LANE_REG);
-	val &= ~(LANE_SELECT);
-	val |= FIELD_PREP(LANE_SELECT, lane);
+	FIELD_MODIFY(LANE_SELECT, &val, lane);
 	dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + SD_STATUS_L1LANE_REG, val);
 
 	return count;
@@ -306,6 +305,7 @@ static ssize_t err_inj_write(struct file *file, const char __user *buf,
 	u32 val, counter, vc_num, err_group, type_mask;
 	int val_diff = 0;
 	char *kern_buf;
+	int ret;
 
 	err_group = err_inj_list[pdata->idx].err_inj_group;
 	type_mask = err_inj_type_mask[err_group];
@@ -327,10 +327,10 @@ static ssize_t err_inj_write(struct file *file, const char __user *buf,
 			return -EINVAL;
 		}
 	} else {
-		val = kstrtou32(kern_buf, 0, &counter);
-		if (val) {
+		ret = kstrtou32(kern_buf, 0, &counter);
+		if (ret) {
 			kfree(kern_buf);
-			return val;
+			return ret;
 		}
 	}
 
@@ -339,14 +339,10 @@ static ssize_t err_inj_write(struct file *file, const char __user *buf,
 	val |= ((err_inj_list[pdata->idx].err_inj_type << EINJ_TYPE_SHIFT) & type_mask);
 	val |= FIELD_PREP(EINJ_COUNT, counter);
 
-	if (err_group == 1 || err_group == 4) {
-		val &= ~(EINJ_VAL_DIFF);
-		val |= FIELD_PREP(EINJ_VAL_DIFF, val_diff);
-	}
-	if (err_group == 4) {
-		val &= ~(EINJ_VC_NUM);
-		val |= FIELD_PREP(EINJ_VC_NUM, vc_num);
-	}
+	if (err_group == 1 || err_group == 4)
+		FIELD_MODIFY(EINJ_VAL_DIFF, &val, val_diff);
+	if (err_group == 4)
+		FIELD_MODIFY(EINJ_VC_NUM, &val, vc_num);
 
 	dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + ERR_INJ0_OFF + (0x4 * err_group), val);
 	dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + ERR_INJ_ENABLE_REG, (0x1 << err_group));
@@ -362,9 +358,8 @@ static void set_event_number(struct dwc_pcie_rasdes_priv *pdata,
 
 	val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG);
 	val &= ~EVENT_COUNTER_ENABLE;
-	val &= ~(EVENT_COUNTER_GROUP_SELECT | EVENT_COUNTER_EVENT_SELECT);
-	val |= FIELD_PREP(EVENT_COUNTER_GROUP_SELECT, event_list[pdata->idx].group_no);
-	val |= FIELD_PREP(EVENT_COUNTER_EVENT_SELECT, event_list[pdata->idx].event_no);
+	FIELD_MODIFY(EVENT_COUNTER_GROUP_SELECT, &val, event_list[pdata->idx].group_no);
+	FIELD_MODIFY(EVENT_COUNTER_EVENT_SELECT, &val, event_list[pdata->idx].event_no);
 	dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG, val);
 }
 
@@ -469,8 +464,7 @@ static ssize_t counter_lane_write(struct file *file, const char __user *buf,
 	mutex_lock(&rinfo->reg_event_lock);
 	set_event_number(pdata, pci, rinfo);
 	val = dw_pcie_readl_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG);
-	val &= ~(EVENT_COUNTER_LANE_SELECT);
-	val |= FIELD_PREP(EVENT_COUNTER_LANE_SELECT, lane);
+	FIELD_MODIFY(EVENT_COUNTER_LANE_SELECT, &val, lane);
 	dw_pcie_writel_dbi(pci, rinfo->ras_cap_offset + RAS_DES_EVENT_COUNTER_CTRL_REG, val);
 	mutex_unlock(&rinfo->reg_event_lock);
 
@@ -507,11 +501,6 @@ static int ltssm_status_show(struct seq_file *s, void *v)
 	return 0;
 }
 
-static int ltssm_status_open(struct inode *inode, struct file *file)
-{
-	return single_open(file, ltssm_status_show, inode->i_private);
-}
-
 #define dwc_debugfs_create(name)			\
 debugfs_create_file(#name, 0644, rasdes_debug, pci,	\
 			&dbg_ ## name ## _fops)
@@ -548,10 +537,7 @@ static const struct file_operations dwc_pcie_counter_value_ops = {
 	.read = counter_value_read,
 };
 
-static const struct file_operations dwc_pcie_ltssm_status_ops = {
-	.open = ltssm_status_open,
-	.read = seq_read,
-};
+DEFINE_SHOW_ATTRIBUTE(ltssm_status);
 
 static void dwc_pcie_rasdes_debugfs_deinit(struct dw_pcie *pci)
 {
@@ -642,7 +628,7 @@ static int dwc_pcie_rasdes_debugfs_init(struct dw_pcie *pci, struct dentry *dir)
 static void dwc_pcie_ltssm_debugfs_init(struct dw_pcie *pci, struct dentry *dir)
 {
 	debugfs_create_file("ltssm_status", 0444, dir, pci,
-			    &dwc_pcie_ltssm_status_ops);
+			    &ltssm_status_fops);
 }
 
 static int dw_pcie_ptm_check_capability(void *drvdata)
diff --git a/drivers/pci/controller/dwc/pcie-designware-ep.c b/drivers/pci/controller/dwc/pcie-designware-ep.c
index d4dc3b24da607..a396b58d67462 100644
--- a/drivers/pci/controller/dwc/pcie-designware-ep.c
+++ b/drivers/pci/controller/dwc/pcie-designware-ep.c
@@ -9,6 +9,7 @@
 #include <linux/align.h>
 #include <linux/bitfield.h>
 #include <linux/of.h>
+#include <linux/overflow.h>
 #include <linux/platform_device.h>
 
 #include "pcie-designware.h"
@@ -707,8 +708,7 @@ static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
 
 	reg = ep_func->msi_cap + PCI_MSI_FLAGS;
 	val = dw_pcie_ep_readw_dbi(ep, func_no, reg);
-	val &= ~PCI_MSI_FLAGS_QMASK;
-	val |= FIELD_PREP(PCI_MSI_FLAGS_QMASK, mmc);
+	FIELD_MODIFY(PCI_MSI_FLAGS_QMASK, &val, mmc);
 	dw_pcie_dbi_ro_wr_en(pci);
 	dw_pcie_ep_writew_dbi(ep, func_no, reg, val);
 	dw_pcie_dbi_ro_wr_dis(pci);
@@ -817,6 +817,122 @@ dw_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
 	return ep->ops->get_features(ep);
 }
 
+static const struct pci_epc_bar_rsvd_region *
+dw_pcie_ep_find_bar_rsvd_region(struct dw_pcie_ep *ep,
+				enum pci_epc_bar_rsvd_region_type type,
+				enum pci_barno *bar,
+				resource_size_t *bar_offset)
+{
+	const struct pci_epc_features *features;
+	const struct pci_epc_bar_desc *bar_desc;
+	const struct pci_epc_bar_rsvd_region *r;
+	int i, j;
+
+	if (!ep->ops->get_features)
+		return NULL;
+
+	features = ep->ops->get_features(ep);
+	if (!features)
+		return NULL;
+
+	for (i = BAR_0; i <= BAR_5; i++) {
+		bar_desc = &features->bar[i];
+
+		if (!bar_desc->nr_rsvd_regions || !bar_desc->rsvd_regions)
+			continue;
+
+		for (j = 0; j < bar_desc->nr_rsvd_regions; j++) {
+			r = &bar_desc->rsvd_regions[j];
+
+			if (r->type != type)
+				continue;
+
+			if (bar)
+				*bar = i;
+			if (bar_offset)
+				*bar_offset = r->offset;
+			return r;
+		}
+	}
+
+	return NULL;
+}
+
+static int
+dw_pcie_ep_get_aux_resources_count(struct pci_epc *epc, u8 func_no,
+				   u8 vfunc_no)
+{
+	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	struct dw_edma_chip *edma = &pci->edma;
+
+	if (!pci->edma_reg_size)
+		return 0;
+
+	if (edma->db_offset == ~0)
+		return 0;
+
+	return 1;
+}
+
+static int
+dw_pcie_ep_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			     struct pci_epc_aux_resource *resources,
+			     int num_resources)
+{
+	struct dw_pcie_ep *ep = epc_get_drvdata(epc);
+	struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
+	const struct pci_epc_bar_rsvd_region *rsvd;
+	struct dw_edma_chip *edma = &pci->edma;
+	enum pci_barno dma_ctrl_bar = NO_BAR;
+	resource_size_t db_offset = edma->db_offset;
+	resource_size_t dma_ctrl_bar_offset = 0;
+	resource_size_t dma_reg_size;
+	int count;
+
+	count = dw_pcie_ep_get_aux_resources_count(epc, func_no, vfunc_no);
+	if (count < 0)
+		return count;
+
+	if (num_resources < count)
+		return -ENOSPC;
+
+	if (!count)
+		return 0;
+
+	dma_reg_size = pci->edma_reg_size;
+
+	rsvd = dw_pcie_ep_find_bar_rsvd_region(ep,
+					       PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO,
+					       &dma_ctrl_bar,
+					       &dma_ctrl_bar_offset);
+	if (rsvd && rsvd->size < dma_reg_size)
+		dma_reg_size = rsvd->size;
+
+	/*
+	 * For interrupt-emulation doorbells, report a standalone resource
+	 * instead of bundling it into the DMA controller MMIO resource.
+	 */
+	if (range_end_overflows_t(resource_size_t, db_offset,
+				  sizeof(u32), dma_reg_size))
+		return -EINVAL;
+
+	resources[0] = (struct pci_epc_aux_resource) {
+		.type = PCI_EPC_AUX_DOORBELL_MMIO,
+		.phys_addr = pci->edma_reg_phys + db_offset,
+		.size = sizeof(u32),
+		.bar = dma_ctrl_bar,
+		.bar_offset = dma_ctrl_bar != NO_BAR ?
+				dma_ctrl_bar_offset + db_offset : 0,
+		.u.db_mmio = {
+			.irq = edma->db_irq,
+			.data = 0, /* write 0 to assert */
+		},
+	};
+
+	return 0;
+}
+
 static const struct pci_epc_ops epc_ops = {
 	.write_header		= dw_pcie_ep_write_header,
 	.set_bar		= dw_pcie_ep_set_bar,
@@ -832,6 +948,8 @@ static const struct pci_epc_ops epc_ops = {
 	.start			= dw_pcie_ep_start,
 	.stop			= dw_pcie_ep_stop,
 	.get_features		= dw_pcie_ep_get_features,
+	.get_aux_resources_count	= dw_pcie_ep_get_aux_resources_count,
+	.get_aux_resources	= dw_pcie_ep_get_aux_resources,
 };
 
 /**
diff --git a/drivers/pci/controller/dwc/pcie-designware-host.c b/drivers/pci/controller/dwc/pcie-designware-host.c
index c9517a3488368..cffb34f6f3a9e 100644
--- a/drivers/pci/controller/dwc/pcie-designware-host.c
+++ b/drivers/pci/controller/dwc/pcie-designware-host.c
@@ -16,9 +16,11 @@
 #include <linux/msi.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
+#include <linux/pci.h>
 #include <linux/pci_regs.h>
 #include <linux/platform_device.h>
 
+#include "../pci-host-common.h"
 #include "../../pci.h"
 #include "pcie-designware.h"
 
@@ -1218,18 +1220,14 @@ static int dw_pcie_pme_turn_off(struct dw_pcie *pci)
 
 int dw_pcie_suspend_noirq(struct dw_pcie *pci)
 {
-	u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
+	bool pme_capable = false;
 	int ret = 0;
 	u32 val;
 
 	if (!dw_pcie_link_up(pci))
 		goto stop_link;
 
-	/*
-	 * If L1SS is supported, then do not put the link into L2 as some
-	 * devices such as NVMe expect low resume latency.
-	 */
-	if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1)
+	if (!pci_host_common_d3cold_possible(pci->pp.bridge, &pme_capable))
 		return 0;
 
 	if (pci->pp.ops->pme_turn_off) {
@@ -1273,6 +1271,15 @@ int dw_pcie_suspend_noirq(struct dw_pcie *pci)
 	udelay(1);
 
 stop_link:
+	/*
+	 * TODO: "pme_capable" means some downstream device is wakeup-
+	 * enabled and is capable of generating PME from D3cold, which
+	 * requires auxiliary power.  Instead of always skipping power off
+	 * if PME is supported from D3cold, query the pwrctrl core and skip
+	 * power off only if device supports PME from D3cold and Vaux is
+	 * not supported.
+	 */
+	pci->pp.skip_pwrctrl_off = pme_capable;
 	dw_pcie_stop_link(pci);
 	if (pci->pp.ops->deinit)
 		pci->pp.ops->deinit(&pci->pp);
@@ -1290,8 +1297,6 @@ int dw_pcie_resume_noirq(struct dw_pcie *pci)
 	if (!pci->suspended)
 		return 0;
 
-	pci->suspended = false;
-
 	if (pci->pp.ops->init) {
 		ret = pci->pp.ops->init(&pci->pp);
 		if (ret) {
@@ -1313,6 +1318,8 @@ int dw_pcie_resume_noirq(struct dw_pcie *pci)
 	if (pci->pp.ops->post_init)
 		pci->pp.ops->post_init(&pci->pp);
 
+	pci->suspended = false;
+
 	return 0;
 
 err_stop_link:
diff --git a/drivers/pci/controller/dwc/pcie-designware-plat.c b/drivers/pci/controller/dwc/pcie-designware-plat.c
index d103ab759c4ee..14d94e7612295 100644
--- a/drivers/pci/controller/dwc/pcie-designware-plat.c
+++ b/drivers/pci/controller/dwc/pcie-designware-plat.c
@@ -8,7 +8,6 @@
  */
 #include <linux/clk.h>
 #include <linux/delay.h>
-#include <linux/gpio.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
@@ -22,7 +21,6 @@
 
 struct dw_plat_pcie {
 	struct dw_pcie			*pci;
-	enum dw_pcie_device_mode	mode;
 };
 
 struct dw_plat_pcie_of_data {
@@ -118,11 +116,11 @@ static int dw_plat_pcie_probe(struct platform_device *pdev)
 	pci->dev = dev;
 
 	dw_plat_pcie->pci = pci;
-	dw_plat_pcie->mode = mode;
+	dw_plat_pcie->pci->mode = mode;
 
 	platform_set_drvdata(pdev, dw_plat_pcie);
 
-	switch (dw_plat_pcie->mode) {
+	switch (dw_plat_pcie->pci->mode) {
 	case DW_PCIE_RC_TYPE:
 		if (!IS_ENABLED(CONFIG_PCIE_DW_PLAT_HOST))
 			return -ENODEV;
@@ -148,7 +146,7 @@ static int dw_plat_pcie_probe(struct platform_device *pdev)
 
 		break;
 	default:
-		dev_err(dev, "INVALID device type %d\n", dw_plat_pcie->mode);
+		dev_err(dev, "INVALID device type %d\n", dw_plat_pcie->pci->mode);
 		ret = -EINVAL;
 		break;
 	}
diff --git a/drivers/pci/controller/dwc/pcie-designware.c b/drivers/pci/controller/dwc/pcie-designware.c
index c11cf61b8319e..37d2715e38eb4 100644
--- a/drivers/pci/controller/dwc/pcie-designware.c
+++ b/drivers/pci/controller/dwc/pcie-designware.c
@@ -162,8 +162,12 @@ int dw_pcie_get_resources(struct dw_pcie *pci)
 			pci->edma.reg_base = devm_ioremap_resource(pci->dev, res);
 			if (IS_ERR(pci->edma.reg_base))
 				return PTR_ERR(pci->edma.reg_base);
+			pci->edma_reg_phys = res->start;
+			pci->edma_reg_size = resource_size(res);
 		} else if (pci->atu_size >= 2 * DEFAULT_DBI_DMA_OFFSET) {
 			pci->edma.reg_base = pci->atu_base + DEFAULT_DBI_DMA_OFFSET;
+			pci->edma_reg_phys = pci->atu_phys_addr + DEFAULT_DBI_DMA_OFFSET;
+			pci->edma_reg_size = pci->atu_size - DEFAULT_DBI_DMA_OFFSET;
 		}
 	}
 
@@ -487,13 +491,13 @@ static inline void dw_pcie_writel_atu_ob(struct dw_pcie *pci, u32 index, u32 reg
 static inline u32 dw_pcie_enable_ecrc(u32 val)
 {
 	/*
-	 * DWC versions 0x3530302a and 0x3536322a have a design issue where
-	 * the 'TD' bit in the Control register-1 of the ATU outbound
-	 * region acts like an override for the ECRC setting, i.e., the
-	 * presence of TLP Digest (ECRC) in the outgoing TLPs is solely
-	 * determined by this bit. This is contrary to the PCIe spec which
-	 * says that the enablement of the ECRC is solely determined by the
-	 * AER registers.
+	 * DesignWare core versions prior to 5.10A have a design issue where the
+	 * 'TD' bit in the Control register-1 of the ATU outbound region acts
+	 * like an override for the ECRC setting, i.e., the presence of TLP
+	 * Digest (ECRC) in the outgoing TLPs is solely determined by this
+	 * bit. This is contrary to the PCIe spec which says that the
+	 * enablement of the ECRC is solely determined by the AER
+	 * registers.
 	 *
 	 * Because of this, even when the ECRC is enabled through AER
 	 * registers, the transactions going through ATU won't have TLP
@@ -563,7 +567,7 @@ int dw_pcie_prog_outbound_atu(struct dw_pcie *pci,
 	if (upper_32_bits(limit_addr) > upper_32_bits(parent_bus_addr) &&
 	    dw_pcie_ver_is_ge(pci, 460A))
 		val |= PCIE_ATU_INCREASE_REGION_SIZE;
-	if (dw_pcie_ver_is(pci, 490A) || dw_pcie_ver_is(pci, 500A))
+	if (!dw_pcie_ver_is_ge(pci, 510A))
 		val = dw_pcie_enable_ecrc(val);
 	dw_pcie_writel_atu_ob(pci, atu->index, PCIE_ATU_REGION_CTRL1, val);
 
@@ -938,8 +942,7 @@ static void dw_pcie_link_set_max_link_width(struct dw_pcie *pci, u32 num_lanes)
 
 	cap = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
 	lnkcap = dw_pcie_readl_dbi(pci, cap + PCI_EXP_LNKCAP);
-	lnkcap &= ~PCI_EXP_LNKCAP_MLW;
-	lnkcap |= FIELD_PREP(PCI_EXP_LNKCAP_MLW, num_lanes);
+	FIELD_MODIFY(PCI_EXP_LNKCAP_MLW, &lnkcap, num_lanes);
 	dw_pcie_writel_dbi(pci, cap + PCI_EXP_LNKCAP, lnkcap);
 }
 
@@ -1249,6 +1252,34 @@ void dw_pcie_hide_unsupported_l1ss(struct dw_pcie *pci)
 	dw_pcie_writel_dbi(pci, l1ss + PCI_L1SS_CAP, l1ss_cap);
 }
 
+/* TODO: Need to handle multi Root Ports */
+void dw_pcie_program_t_power_on(struct dw_pcie *pci, u32 t_power_on)
+{
+	u8 scale, value;
+	u16 offset;
+	u32 val;
+
+	if (!t_power_on)
+		return;
+
+	offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
+	if (!offset)
+		return;
+
+	pcie_encode_t_power_on(t_power_on, &scale, &value);
+
+	dw_pcie_dbi_ro_wr_en(pci);
+
+	val = dw_pcie_readl_dbi(pci, offset + PCI_L1SS_CAP);
+	val &= ~(PCI_L1SS_CAP_P_PWR_ON_SCALE | PCI_L1SS_CAP_P_PWR_ON_VALUE);
+	FIELD_MODIFY(PCI_L1SS_CAP_P_PWR_ON_SCALE, &val, scale);
+	FIELD_MODIFY(PCI_L1SS_CAP_P_PWR_ON_VALUE, &val, value);
+
+	dw_pcie_writel_dbi(pci, offset + PCI_L1SS_CAP, val);
+
+	dw_pcie_dbi_ro_wr_dis(pci);
+}
+
 void dw_pcie_setup(struct dw_pcie *pci)
 {
 	u32 val;
diff --git a/drivers/pci/controller/dwc/pcie-designware.h b/drivers/pci/controller/dwc/pcie-designware.h
index 3e69ef60165b0..7dac37521b8e7 100644
--- a/drivers/pci/controller/dwc/pcie-designware.h
+++ b/drivers/pci/controller/dwc/pcie-designware.h
@@ -35,6 +35,7 @@
 #define DW_PCIE_VER_480A		0x3438302a
 #define DW_PCIE_VER_490A		0x3439302a
 #define DW_PCIE_VER_500A		0x3530302a
+#define DW_PCIE_VER_510A		0x3531302a
 #define DW_PCIE_VER_520A		0x3532302a
 #define DW_PCIE_VER_540A		0x3534302a
 #define DW_PCIE_VER_562A		0x3536322a
@@ -71,6 +72,8 @@
 
 /* Synopsys-specific PCIe configuration registers */
 #define PCIE_PORT_FORCE			0x708
+/* Bit[7:0] LINK_NUM: Link Number. Not used for endpoint */
+#define PORT_LINK_NUM_MASK		GENMASK(7, 0)
 #define PORT_FORCE_DO_DESKEW_FOR_SRIS	BIT(23)
 
 #define PCIE_PORT_AFR			0x70C
@@ -98,6 +101,28 @@
 #define PCIE_PORT_LANE_SKEW		0x714
 #define PORT_LANE_SKEW_INSERT_MASK	GENMASK(23, 0)
 
+/*
+ * PCIE_TIMER_CTRL_MAX_FUNC_NUM: Timer Control and Max Function Number
+ * Register.
+ *
+ * This register holds the ack frequency, latency, replay, fast link
+ * scaling timers, and max function number values.
+ *
+ * Bit[30:29] FAST_LINK_SCALING_FACTOR: Fast Link Timer Scaling Factor.
+ *   0x0 (SF_1024): Scaling Factor is 1024 (1ms is 1us).
+ *     When the LTSSM is in Config or L12 Entry State, 1ms
+ *     timer is 2us, 2ms timer is 4us and 3ms timer is 6us.
+ *   0x1 (SF_256): Scaling Factor is 256 (1ms is 4us)
+ *   0x2 (SF_64): Scaling Factor is 64 (1ms is 16us)
+ *   0x3 (SF_16): Scaling Factor is 16 (1ms is 64us)
+ */
+#define PCIE_TIMER_CTRL_MAX_FUNC_NUM	0x718
+#define PORT_FLT_SF_MASK	GENMASK(30, 29)
+#define PORT_FLT_SF_VAL_1024	0x0
+#define PORT_FLT_SF_VAL_256	0x1
+#define PORT_FLT_SF_VAL_64	0x2
+#define PORT_FLT_SF_VAL_16	0x3
+
 #define PCIE_PORT_DEBUG0		0x728
 #define PORT_LOGIC_LTSSM_STATE_MASK	0x3f
 #define PORT_LOGIC_LTSSM_STATE_L0	0x11
@@ -450,6 +475,7 @@ struct dw_pcie_rp {
 	bool			ecam_enabled;
 	bool			native_ecam;
 	bool                    skip_l23_ready;
+	bool			skip_pwrctrl_off;
 };
 
 struct dw_pcie_ep_ops {
@@ -544,6 +570,8 @@ struct dw_pcie {
 	int			max_link_speed;
 	u8			n_fts[2];
 	struct dw_edma_chip	edma;
+	phys_addr_t		edma_reg_phys;
+	resource_size_t		edma_reg_size;
 	bool			l1ss_support;	/* L1 PM Substates support */
 	struct clk_bulk_data	app_clks[DW_PCIE_NUM_APP_CLKS];
 	struct clk_bulk_data	core_clks[DW_PCIE_NUM_CORE_CLKS];
@@ -605,6 +633,7 @@ int dw_pcie_prog_ep_inbound_atu(struct dw_pcie *pci, u8 func_no, int index,
 				u8 bar, size_t size);
 void dw_pcie_disable_atu(struct dw_pcie *pci, u32 dir, int index);
 void dw_pcie_hide_unsupported_l1ss(struct dw_pcie *pci);
+void dw_pcie_program_t_power_on(struct dw_pcie *pci, u32 t_power_on);
 void dw_pcie_setup(struct dw_pcie *pci);
 void dw_pcie_iatu_detect(struct dw_pcie *pci);
 int dw_pcie_edma_detect(struct dw_pcie *pci);
diff --git a/drivers/pci/controller/dwc/pcie-eswin.c b/drivers/pci/controller/dwc/pcie-eswin.c
index 2845832b38243..ce8d64f8a3957 100644
--- a/drivers/pci/controller/dwc/pcie-eswin.c
+++ b/drivers/pci/controller/dwc/pcie-eswin.c
@@ -211,8 +211,7 @@ static int eswin_pcie_host_init(struct dw_pcie_rp *pp)
 
 	/* Configure Root Port type */
 	val = readl_relaxed(pci->elbi_base + PCIEELBI_CTRL0_OFFSET);
-	val &= ~PCIEELBI_CTRL0_DEV_TYPE;
-	val |= FIELD_PREP(PCIEELBI_CTRL0_DEV_TYPE, PCI_EXP_TYPE_ROOT_PORT);
+	FIELD_MODIFY(PCIEELBI_CTRL0_DEV_TYPE, &val, PCI_EXP_TYPE_ROOT_PORT);
 	writel_relaxed(val, pci->elbi_base + PCIEELBI_CTRL0_OFFSET);
 
 	list_for_each_entry(port, &pcie->ports, list) {
diff --git a/drivers/pci/controller/dwc/pcie-fu740.c b/drivers/pci/controller/dwc/pcie-fu740.c
index 66367252032b8..d0a34f680397e 100644
--- a/drivers/pci/controller/dwc/pcie-fu740.c
+++ b/drivers/pci/controller/dwc/pcie-fu740.c
@@ -13,7 +13,6 @@
 
 #include <linux/clk.h>
 #include <linux/delay.h>
-#include <linux/gpio.h>
 #include <linux/gpio/consumer.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
diff --git a/drivers/pci/controller/dwc/pcie-intel-gw.c b/drivers/pci/controller/dwc/pcie-intel-gw.c
index c21906eced618..2674cd376f495 100644
--- a/drivers/pci/controller/dwc/pcie-intel-gw.c
+++ b/drivers/pci/controller/dwc/pcie-intel-gw.c
@@ -47,7 +47,6 @@
 #define PCIE_APP_IRN_INTD		BIT(16)
 #define PCIE_APP_IRN_MSG_LTR		BIT(18)
 #define PCIE_APP_IRN_SYS_ERR_RC		BIT(29)
-#define PCIE_APP_INTX_OFST		12
 
 #define PCIE_APP_IRN_INT \
 	(PCIE_APP_IRN_AER_REPORT | PCIE_APP_IRN_PME | \
@@ -196,6 +195,13 @@ static void intel_pcie_device_rst_deassert(struct intel_pcie *pcie)
 	gpiod_set_value_cansleep(pcie->reset_gpio, 0);
 }
 
+static void intel_pcie_core_irq_enable(struct intel_pcie *pcie)
+{
+	pcie_app_wr(pcie, PCIE_APP_IRNEN, 0);
+	pcie_app_wr(pcie, PCIE_APP_IRNCR, PCIE_APP_IRN_INT);
+	pcie_app_wr(pcie, PCIE_APP_IRNEN, PCIE_APP_IRN_INT);
+}
+
 static void intel_pcie_core_irq_disable(struct intel_pcie *pcie)
 {
 	pcie_app_wr(pcie, PCIE_APP_IRNEN, 0);
@@ -278,6 +284,16 @@ static void intel_pcie_turn_off(struct intel_pcie *pcie)
 	pcie_rc_cfg_wr_mask(pcie, PCI_COMMAND, PCI_COMMAND_MEMORY, 0);
 }
 
+static int intel_pcie_start_link(struct dw_pcie *pci)
+{
+	struct intel_pcie *pcie = dev_get_drvdata(pci->dev);
+
+	intel_pcie_device_rst_deassert(pcie);
+	intel_pcie_ltssm_enable(pcie);
+
+	return 0;
+}
+
 static int intel_pcie_host_setup(struct intel_pcie *pcie)
 {
 	int ret;
@@ -285,49 +301,33 @@ static int intel_pcie_host_setup(struct intel_pcie *pcie)
 
 	intel_pcie_core_rst_assert(pcie);
 	intel_pcie_device_rst_assert(pcie);
-
-	ret = phy_init(pcie->phy);
-	if (ret)
-		return ret;
-
 	intel_pcie_core_rst_deassert(pcie);
 
+	/* Controller clock must be provided earlier than PHY */
 	ret = clk_prepare_enable(pcie->core_clk);
 	if (ret) {
 		dev_err(pcie->pci.dev, "Core clock enable failed: %d\n", ret);
 		goto clk_err;
 	}
 
-	pci->atu_base = pci->dbi_base + 0xC0000;
+	ret = phy_init(pcie->phy);
+	if (ret)
+		goto phy_err;
 
 	intel_pcie_ltssm_disable(pcie);
 	intel_pcie_link_setup(pcie);
 	intel_pcie_init_n_fts(pci);
 
-	ret = dw_pcie_setup_rc(&pci->pp);
-	if (ret)
-		goto app_init_err;
-
 	dw_pcie_upconfig_setup(pci);
 
-	intel_pcie_device_rst_deassert(pcie);
-	intel_pcie_ltssm_enable(pcie);
-
-	ret = dw_pcie_wait_for_link(pci);
-	if (ret)
-		goto app_init_err;
-
-	/* Enable integrated interrupts */
-	pcie_app_wr_mask(pcie, PCIE_APP_IRNEN, PCIE_APP_IRN_INT,
-			 PCIE_APP_IRN_INT);
+	intel_pcie_core_irq_enable(pcie);
 
 	return 0;
 
-app_init_err:
+phy_err:
 	clk_disable_unprepare(pcie->core_clk);
 clk_err:
 	intel_pcie_core_rst_assert(pcie);
-	phy_exit(pcie->phy);
 
 	return ret;
 }
@@ -381,6 +381,7 @@ static int intel_pcie_rc_init(struct dw_pcie_rp *pp)
 }
 
 static const struct dw_pcie_ops intel_pcie_ops = {
+	.start_link = intel_pcie_start_link,
 };
 
 static const struct dw_pcie_host_ops intel_pcie_dw_ops = {
@@ -392,6 +393,7 @@ static int intel_pcie_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct intel_pcie *pcie;
 	struct dw_pcie_rp *pp;
+	struct resource *res;
 	struct dw_pcie *pci;
 	int ret;
 
@@ -416,6 +418,32 @@ static int intel_pcie_probe(struct platform_device *pdev)
 	pci->ops = &intel_pcie_ops;
 	pp->ops = &intel_pcie_dw_ops;
 
+	/*
+	 * If the 'atu' region is not available in the devicetree, use the
+	 * default offset from DBI region for backwards compatibility. The
+	 * 'atu' region should always be specified in the devicetree, as
+	 * this is a hardware-specific address that should not be defined
+	 * in the driver.
+	 */
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "atu");
+	if (!res) {
+		res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
+		pci->dbi_base = devm_pci_remap_cfg_resource(pci->dev, res);
+		if (IS_ERR(pci->dbi_base))
+			return PTR_ERR(pci->dbi_base);
+
+		pci->dbi_phys_addr = res->start;
+		pci->atu_base = devm_ioremap(dev, res->start + 0xC0000, SZ_4K);
+		if (!pci->atu_base) {
+			dev_err(dev, "failed to remap ATU space\n");
+			return -ENOMEM;
+		}
+
+		pci->atu_size = SZ_4K;
+		pci->atu_phys_addr = res->start + 0xC0000;
+		dev_warn(dev, "ATU region not specified in DT. Using default offset\n");
+	}
+
 	ret = dw_pcie_host_init(pp);
 	if (ret) {
 		dev_err(dev, "Cannot initialize host\n");
diff --git a/drivers/pci/controller/dwc/pcie-keembay.c b/drivers/pci/controller/dwc/pcie-keembay.c
index 7cf2c312ecec2..2459c4d66b889 100644
--- a/drivers/pci/controller/dwc/pcie-keembay.c
+++ b/drivers/pci/controller/dwc/pcie-keembay.c
@@ -58,7 +58,6 @@
 struct keembay_pcie {
 	struct dw_pcie		pci;
 	void __iomem		*apb_base;
-	enum dw_pcie_device_mode mode;
 
 	struct clk		*clk_master;
 	struct clk		*clk_aux;
@@ -117,7 +116,7 @@ static int keembay_pcie_start_link(struct dw_pcie *pci)
 	u32 val;
 	int ret;
 
-	if (pcie->mode == DW_PCIE_EP_TYPE)
+	if (pcie->pci.mode == DW_PCIE_EP_TYPE)
 		return 0;
 
 	keembay_pcie_ltssm_set(pcie, false);
@@ -409,7 +408,7 @@ static int keembay_pcie_probe(struct platform_device *pdev)
 	pci->dev = dev;
 	pci->ops = &keembay_pcie_ops;
 
-	pcie->mode = mode;
+	pcie->pci.mode = mode;
 
 	pcie->apb_base = devm_platform_ioremap_resource_byname(pdev, "apb");
 	if (IS_ERR(pcie->apb_base))
@@ -417,7 +416,7 @@ static int keembay_pcie_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, pcie);
 
-	switch (pcie->mode) {
+	switch (pcie->pci.mode) {
 	case DW_PCIE_RC_TYPE:
 		if (!IS_ENABLED(CONFIG_PCIE_KEEMBAY_HOST))
 			return -ENODEV;
@@ -443,7 +442,7 @@ static int keembay_pcie_probe(struct platform_device *pdev)
 
 		break;
 	default:
-		dev_err(dev, "Invalid device type %d\n", pcie->mode);
+		dev_err(dev, "Invalid device type %d\n", pcie->pci.mode);
 		return -ENODEV;
 	}
 
diff --git a/drivers/pci/controller/dwc/pcie-nxp-s32g.c b/drivers/pci/controller/dwc/pcie-nxp-s32g.c
index b3ec38099fa38..31e1169b8ab6b 100644
--- a/drivers/pci/controller/dwc/pcie-nxp-s32g.c
+++ b/drivers/pci/controller/dwc/pcie-nxp-s32g.c
@@ -139,8 +139,7 @@ static int s32g_init_pcie_controller(struct dw_pcie_rp *pp)
 
 	/* Set RP mode */
 	val = s32g_pcie_readl_ctrl(s32g_pp, PCIE_S32G_PE0_GEN_CTRL_1);
-	val &= ~DEVICE_TYPE_MASK;
-	val |= FIELD_PREP(DEVICE_TYPE_MASK, PCI_EXP_TYPE_ROOT_PORT);
+	FIELD_MODIFY(DEVICE_TYPE_MASK, &val, PCI_EXP_TYPE_ROOT_PORT);
 
 	/* Use default CRNS */
 	val &= ~SRIS_MODE;
diff --git a/drivers/pci/controller/dwc/pcie-qcom-common.c b/drivers/pci/controller/dwc/pcie-qcom-common.c
index 5aa73c6287373..0da73caf20110 100644
--- a/drivers/pci/controller/dwc/pcie-qcom-common.c
+++ b/drivers/pci/controller/dwc/pcie-qcom-common.c
@@ -30,20 +30,15 @@ void qcom_pcie_common_set_equalization(struct dw_pcie *pci)
 
 		reg = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
 		reg &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
-		reg &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
-		reg |= FIELD_PREP(GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK,
-			  speed - PCIE_SPEED_8_0GT);
+		FIELD_MODIFY(GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK, &reg,
+			     speed - PCIE_SPEED_8_0GT);
 		dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, reg);
 
 		reg = dw_pcie_readl_dbi(pci, GEN3_EQ_FB_MODE_DIR_CHANGE_OFF);
-		reg &= ~(GEN3_EQ_FMDC_T_MIN_PHASE23 |
-			GEN3_EQ_FMDC_N_EVALS |
-			GEN3_EQ_FMDC_MAX_PRE_CURSOR_DELTA |
-			GEN3_EQ_FMDC_MAX_POST_CURSOR_DELTA);
-		reg |= FIELD_PREP(GEN3_EQ_FMDC_T_MIN_PHASE23, 0x1) |
-			FIELD_PREP(GEN3_EQ_FMDC_N_EVALS, 0xd) |
-			FIELD_PREP(GEN3_EQ_FMDC_MAX_PRE_CURSOR_DELTA, 0x5) |
-			FIELD_PREP(GEN3_EQ_FMDC_MAX_POST_CURSOR_DELTA, 0x5);
+		FIELD_MODIFY(GEN3_EQ_FMDC_T_MIN_PHASE23, &reg, 0x1);
+		FIELD_MODIFY(GEN3_EQ_FMDC_N_EVALS, &reg, 0xd);
+		FIELD_MODIFY(GEN3_EQ_FMDC_MAX_PRE_CURSOR_DELTA, &reg, 0x5);
+		FIELD_MODIFY(GEN3_EQ_FMDC_MAX_POST_CURSOR_DELTA, &reg, 0x5);
 		dw_pcie_writel_dbi(pci, GEN3_EQ_FB_MODE_DIR_CHANGE_OFF, reg);
 
 		reg = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
@@ -61,14 +56,10 @@ void qcom_pcie_common_set_16gt_lane_margining(struct dw_pcie *pci)
 	u32 reg;
 
 	reg = dw_pcie_readl_dbi(pci, GEN4_LANE_MARGINING_1_OFF);
-	reg &= ~(MARGINING_MAX_VOLTAGE_OFFSET |
-		MARGINING_NUM_VOLTAGE_STEPS |
-		MARGINING_MAX_TIMING_OFFSET |
-		MARGINING_NUM_TIMING_STEPS);
-	reg |= FIELD_PREP(MARGINING_MAX_VOLTAGE_OFFSET, 0x24) |
-		FIELD_PREP(MARGINING_NUM_VOLTAGE_STEPS, 0x78) |
-		FIELD_PREP(MARGINING_MAX_TIMING_OFFSET, 0x32) |
-		FIELD_PREP(MARGINING_NUM_TIMING_STEPS, 0x10);
+	FIELD_MODIFY(MARGINING_MAX_VOLTAGE_OFFSET, &reg, 0x24);
+	FIELD_MODIFY(MARGINING_NUM_VOLTAGE_STEPS, &reg, 0x78);
+	FIELD_MODIFY(MARGINING_MAX_TIMING_OFFSET, &reg, 0x32);
+	FIELD_MODIFY(MARGINING_NUM_TIMING_STEPS, &reg, 0x10);
 	dw_pcie_writel_dbi(pci, GEN4_LANE_MARGINING_1_OFF, reg);
 
 	reg = dw_pcie_readl_dbi(pci, GEN4_LANE_MARGINING_2_OFF);
@@ -76,13 +67,10 @@ void qcom_pcie_common_set_16gt_lane_margining(struct dw_pcie *pci)
 		MARGINING_SAMPLE_REPORTING_METHOD |
 		MARGINING_IND_LEFT_RIGHT_TIMING |
 		MARGINING_VOLTAGE_SUPPORTED;
-	reg &= ~(MARGINING_IND_UP_DOWN_VOLTAGE |
-		MARGINING_MAXLANES |
-		MARGINING_SAMPLE_RATE_TIMING |
-		MARGINING_SAMPLE_RATE_VOLTAGE);
-	reg |= FIELD_PREP(MARGINING_MAXLANES, pci->num_lanes) |
-		FIELD_PREP(MARGINING_SAMPLE_RATE_TIMING, 0x3f) |
-		FIELD_PREP(MARGINING_SAMPLE_RATE_VOLTAGE, 0x3f);
+	reg &= ~MARGINING_IND_UP_DOWN_VOLTAGE;
+	FIELD_MODIFY(MARGINING_MAXLANES, &reg, pci->num_lanes);
+	FIELD_MODIFY(MARGINING_SAMPLE_RATE_TIMING, &reg, 0x3f);
+	FIELD_MODIFY(MARGINING_SAMPLE_RATE_VOLTAGE, &reg, 0x3f);
 	dw_pcie_writel_dbi(pci, GEN4_LANE_MARGINING_2_OFF, reg);
 }
 EXPORT_SYMBOL_GPL(qcom_pcie_common_set_16gt_lane_margining);
diff --git a/drivers/pci/controller/dwc/pcie-qcom-ep.c b/drivers/pci/controller/dwc/pcie-qcom-ep.c
index 257c2bcb5f76f..56184e6ca6e69 100644
--- a/drivers/pci/controller/dwc/pcie-qcom-ep.c
+++ b/drivers/pci/controller/dwc/pcie-qcom-ep.c
@@ -494,15 +494,13 @@ static int qcom_pcie_perst_deassert(struct dw_pcie *pci)
 	/* Set the L0s Exit Latency to 2us-4us = 0x6 */
 	offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
 	val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
-	val &= ~PCI_EXP_LNKCAP_L0SEL;
-	val |= FIELD_PREP(PCI_EXP_LNKCAP_L0SEL, 0x6);
+	FIELD_MODIFY(PCI_EXP_LNKCAP_L0SEL, &val, 0x6);
 	dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, val);
 
 	/* Set the L1 Exit Latency to be 32us-64 us = 0x6 */
 	offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
 	val = dw_pcie_readl_dbi(pci, offset + PCI_EXP_LNKCAP);
-	val &= ~PCI_EXP_LNKCAP_L1EL;
-	val |= FIELD_PREP(PCI_EXP_LNKCAP_L1EL, 0x6);
+	FIELD_MODIFY(PCI_EXP_LNKCAP_L1EL, &val, 0x6);
 	dw_pcie_writel_dbi(pci, offset + PCI_EXP_LNKCAP, val);
 
 	dw_pcie_dbi_ro_wr_dis(pci);
diff --git a/drivers/pci/controller/dwc/pcie-qcom.c b/drivers/pci/controller/dwc/pcie-qcom.c
index af6bf5cce65be..cb8134c5f0e1c 100644
--- a/drivers/pci/controller/dwc/pcie-qcom.c
+++ b/drivers/pci/controller/dwc/pcie-qcom.c
@@ -71,6 +71,7 @@
 
 /* ELBI registers */
 #define ELBI_SYS_CTRL				0x04
+#define ELBI_SYS_STTS				0x08
 
 /* DBI registers */
 #define AXI_MSTR_RESP_COMP_CTRL0		0x818
@@ -131,6 +132,7 @@
 
 /* PARF_LTSSM register fields */
 #define LTSSM_EN				BIT(8)
+#define PARF_LTSSM_STATE_MASK			GENMASK(5, 0)
 
 /* PARF_NO_SNOOP_OVERRIDE register fields */
 #define WR_NO_SNOOP_OVERRIDE_EN			BIT(1)
@@ -144,6 +146,10 @@
 
 /* ELBI_SYS_CTRL register fields */
 #define ELBI_SYS_CTRL_LT_ENABLE			BIT(0)
+#define ELBI_SYS_CTRL_PME_TURNOFF_MSG		BIT(4)
+
+/* ELBI_SYS_STTS register fields */
+#define ELBI_SYS_STTS_LTSSM_STATE_MASK		GENMASK(17, 12)
 
 /* AXI_MSTR_RESP_COMP_CTRL0 register fields */
 #define CFG_REMOTE_RD_REQ_BRIDGE_SIZE_2K	0x4
@@ -245,6 +251,7 @@ struct qcom_pcie_ops {
 	void (*deinit)(struct qcom_pcie *pcie);
 	void (*ltssm_enable)(struct qcom_pcie *pcie);
 	int (*config_sid)(struct qcom_pcie *pcie);
+	enum dw_pcie_ltssm (*get_ltssm)(struct qcom_pcie *pcie);
 };
 
  /**
@@ -269,6 +276,7 @@ struct qcom_pcie_perst {
 struct qcom_pcie_port {
 	struct list_head list;
 	struct phy *phy;
+	u32 l1ss_t_power_on;
 	struct list_head perst;
 };
 
@@ -282,7 +290,7 @@ struct qcom_pcie {
 	const struct qcom_pcie_cfg *cfg;
 	struct dentry *debugfs;
 	struct list_head ports;
-	bool suspended;
+	struct gpio_desc *reset;
 	bool use_pm_opp;
 };
 
@@ -428,6 +436,15 @@ static void qcom_pcie_2_1_0_ltssm_enable(struct qcom_pcie *pcie)
 	writel(val, pci->elbi_base + ELBI_SYS_CTRL);
 }
 
+static enum dw_pcie_ltssm qcom_pcie_2_1_0_get_ltssm(struct qcom_pcie *pcie)
+{
+	struct dw_pcie *pci = pcie->pci;
+	u32 val;
+
+	val = readl(pci->elbi_base + ELBI_SYS_STTS);
+	return (enum dw_pcie_ltssm)FIELD_GET(ELBI_SYS_STTS_LTSSM_STATE_MASK, val);
+}
+
 static int qcom_pcie_get_resources_2_1_0(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_1_0 *res = &pcie->res.v2_1_0;
@@ -517,7 +534,7 @@ static int qcom_pcie_post_init_2_1_0(struct qcom_pcie *pcie)
 	u32 val;
 	int ret;
 
-	/* enable PCIe clocks and resets */
+	/* Force PHY out of lowest power state */
 	val = readl(pcie->parf + PARF_PHY_CTRL);
 	val &= ~PHY_TEST_PWR_DOWN;
 	writel(val, pcie->parf + PARF_PHY_CTRL);
@@ -684,6 +701,12 @@ static int qcom_pcie_get_resources_2_3_2(struct qcom_pcie *pcie)
 static void qcom_pcie_deinit_2_3_2(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_3_2 *res = &pcie->res.v2_3_2;
+	u32 val;
+
+	/* Force PHY to lowest power state*/
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val |= PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
 	clk_bulk_disable_unprepare(res->num_clks, res->clks);
 	regulator_bulk_disable(ARRAY_SIZE(res->supplies), res->supplies);
@@ -716,7 +739,7 @@ static int qcom_pcie_post_init_2_3_2(struct qcom_pcie *pcie)
 {
 	u32 val;
 
-	/* enable PCIe clocks and resets */
+	/* Force PHY out of lowest power state */
 	val = readl(pcie->parf + PARF_PHY_CTRL);
 	val &= ~PHY_TEST_PWR_DOWN;
 	writel(val, pcie->parf + PARF_PHY_CTRL);
@@ -780,6 +803,12 @@ static int qcom_pcie_get_resources_2_4_0(struct qcom_pcie *pcie)
 static void qcom_pcie_deinit_2_4_0(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_4_0 *res = &pcie->res.v2_4_0;
+	u32 val;
+
+	/* Force PHY to lowest power state*/
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val |= PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
 	reset_control_bulk_assert(res->num_resets, res->resets);
 	clk_bulk_disable_unprepare(res->num_clks, res->clks);
@@ -848,6 +877,12 @@ static int qcom_pcie_get_resources_2_3_3(struct qcom_pcie *pcie)
 static void qcom_pcie_deinit_2_3_3(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_3_3 *res = &pcie->res.v2_3_3;
+	u32 val;
+
+	/* Force PHY to lowest power state */
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val |= PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
 	clk_bulk_disable_unprepare(res->num_clks, res->clks);
 }
@@ -903,6 +938,7 @@ static int qcom_pcie_post_init_2_3_3(struct qcom_pcie *pcie)
 	u16 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
 	u32 val;
 
+	/* Force PHY out of lowest power state */
 	val = readl(pcie->parf + PARF_PHY_CTRL);
 	val &= ~PHY_TEST_PWR_DOWN;
 	writel(val, pcie->parf + PARF_PHY_CTRL);
@@ -998,7 +1034,7 @@ static int qcom_pcie_init_2_7_0(struct qcom_pcie *pcie)
 	/* configure PCIe to RC mode */
 	writel(DEVICE_TYPE_RC, pcie->parf + PARF_DEVICE_TYPE);
 
-	/* enable PCIe clocks and resets */
+	/* Force PHY out of lowest power state */
 	val = readl(pcie->parf + PARF_PHY_CTRL);
 	val &= ~PHY_TEST_PWR_DOWN;
 	writel(val, pcie->parf + PARF_PHY_CTRL);
@@ -1069,6 +1105,12 @@ static void qcom_pcie_host_post_init_2_7_0(struct qcom_pcie *pcie)
 static void qcom_pcie_deinit_2_7_0(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_7_0 *res = &pcie->res.v2_7_0;
+	u32 val;
+
+	/* Force PHY to lowest power state */
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val |= PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
 	clk_bulk_disable_unprepare(res->num_clks, res->clks);
 
@@ -1173,6 +1215,12 @@ static int qcom_pcie_get_resources_2_9_0(struct qcom_pcie *pcie)
 static void qcom_pcie_deinit_2_9_0(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_resources_2_9_0 *res = &pcie->res.v2_9_0;
+	u32 val;
+
+	/* Force PHY to lowest power state */
+	val = readl(pcie->parf + PARF_PHY_CTRL);
+	val |= PHY_TEST_PWR_DOWN;
+	writel(val, pcie->parf + PARF_PHY_CTRL);
 
 	clk_bulk_disable_unprepare(res->num_clks, res->clks);
 }
@@ -1213,6 +1261,7 @@ static int qcom_pcie_post_init_2_9_0(struct qcom_pcie *pcie)
 	u32 val;
 	int i;
 
+	/* Force PHY out of lowest power state */
 	val = readl(pcie->parf + PARF_PHY_CTRL);
 	val &= ~PHY_TEST_PWR_DOWN;
 	writel(val, pcie->parf + PARF_PHY_CTRL);
@@ -1260,6 +1309,19 @@ static bool qcom_pcie_link_up(struct dw_pcie *pci)
 	return val & PCI_EXP_LNKSTA_DLLLA;
 }
 
+static enum dw_pcie_ltssm qcom_pcie_get_ltssm(struct dw_pcie *pci)
+{
+	struct qcom_pcie *pcie = to_qcom_pcie(pci);
+	u32 val;
+
+	if (pcie->cfg->ops->get_ltssm)
+		return pcie->cfg->ops->get_ltssm(pcie);
+
+	val = readl(pcie->parf + PARF_LTSSM);
+
+	return (enum dw_pcie_ltssm)FIELD_GET(PARF_LTSSM_STATE_MASK, val);
+}
+
 static void qcom_pcie_phy_power_off(struct qcom_pcie *pcie)
 {
 	struct qcom_pcie_port *port;
@@ -1288,6 +1350,14 @@ static int qcom_pcie_phy_power_on(struct qcom_pcie *pcie)
 	return 0;
 }
 
+static void qcom_pcie_configure_ports(struct qcom_pcie *pcie)
+{
+	struct qcom_pcie_port *port;
+
+	list_for_each_entry(port, &pcie->ports, list)
+		dw_pcie_program_t_power_on(pcie->pci, port->l1ss_t_power_on);
+}
+
 static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
 {
 	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
@@ -1304,13 +1374,17 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
 	if (ret)
 		goto err_deinit;
 
-	ret = pci_pwrctrl_create_devices(pci->dev);
-	if (ret)
-		goto err_disable_phy;
+	if (!pci->suspended) {
+		ret = pci_pwrctrl_create_devices(pci->dev);
+		if (ret)
+			goto err_disable_phy;
+	}
 
-	ret = pci_pwrctrl_power_on_devices(pci->dev);
-	if (ret)
-		goto err_pwrctrl_destroy;
+	if (!pp->skip_pwrctrl_off) {
+		ret = pci_pwrctrl_power_on_devices(pci->dev);
+		if (ret)
+			goto err_pwrctrl_destroy;
+	}
 
 	if (pcie->cfg->ops->post_init) {
 		ret = pcie->cfg->ops->post_init(pcie);
@@ -1322,6 +1396,8 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
 	dw_pcie_remove_capability(pcie->pci, PCI_CAP_ID_MSIX);
 	dw_pcie_remove_ext_capability(pcie->pci, PCI_EXT_CAP_ID_DPC);
 
+	qcom_pcie_configure_ports(pcie);
+
 	qcom_pcie_perst_deassert(pcie);
 
 	if (pcie->cfg->ops->config_sid) {
@@ -1335,9 +1411,10 @@ static int qcom_pcie_host_init(struct dw_pcie_rp *pp)
 err_assert_reset:
 	qcom_pcie_perst_assert(pcie);
 err_pwrctrl_power_off:
-	pci_pwrctrl_power_off_devices(pci->dev);
+	if (!pp->skip_pwrctrl_off)
+		pci_pwrctrl_power_off_devices(pci->dev);
 err_pwrctrl_destroy:
-	if (ret != -EPROBE_DEFER)
+	if (ret != -EPROBE_DEFER && !pci->suspended)
 		pci_pwrctrl_destroy_devices(pci->dev);
 err_disable_phy:
 	qcom_pcie_phy_power_off(pcie);
@@ -1354,11 +1431,14 @@ static void qcom_pcie_host_deinit(struct dw_pcie_rp *pp)
 
 	qcom_pcie_perst_assert(pcie);
 
-	/*
-	 * No need to destroy pwrctrl devices as this function only gets called
-	 * during system suspend as of now.
-	 */
-	pci_pwrctrl_power_off_devices(pci->dev);
+	if (!pci->pp.skip_pwrctrl_off) {
+		/*
+		 * No need to destroy pwrctrl devices as this function only
+		 * gets called during system suspend as of now.
+		 */
+		pci_pwrctrl_power_off_devices(pci->dev);
+	}
+
 	qcom_pcie_phy_power_off(pcie);
 	pcie->cfg->ops->deinit(pcie);
 }
@@ -1372,10 +1452,18 @@ static void qcom_pcie_host_post_init(struct dw_pcie_rp *pp)
 		pcie->cfg->ops->host_post_init(pcie);
 }
 
+static void qcom_pcie_host_pme_turn_off(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+
+	writel(ELBI_SYS_CTRL_PME_TURNOFF_MSG, pci->elbi_base + ELBI_SYS_CTRL);
+}
+
 static const struct dw_pcie_host_ops qcom_pcie_dw_ops = {
 	.init		= qcom_pcie_host_init,
 	.deinit		= qcom_pcie_host_deinit,
 	.post_init	= qcom_pcie_host_post_init,
+	.pme_turn_off	= qcom_pcie_host_pme_turn_off,
 };
 
 /* Qcom IP rev.: 2.1.0	Synopsys IP rev.: 4.01a */
@@ -1385,6 +1473,7 @@ static const struct qcom_pcie_ops ops_2_1_0 = {
 	.post_init = qcom_pcie_post_init_2_1_0,
 	.deinit = qcom_pcie_deinit_2_1_0,
 	.ltssm_enable = qcom_pcie_2_1_0_ltssm_enable,
+	.get_ltssm = qcom_pcie_2_1_0_get_ltssm,
 };
 
 /* Qcom IP rev.: 1.0.0	Synopsys IP rev.: 4.11a */
@@ -1394,6 +1483,7 @@ static const struct qcom_pcie_ops ops_1_0_0 = {
 	.post_init = qcom_pcie_post_init_1_0_0,
 	.deinit = qcom_pcie_deinit_1_0_0,
 	.ltssm_enable = qcom_pcie_2_1_0_ltssm_enable,
+	.get_ltssm = qcom_pcie_2_1_0_get_ltssm,
 };
 
 /* Qcom IP rev.: 2.3.2	Synopsys IP rev.: 4.21a */
@@ -1512,6 +1602,7 @@ static const struct qcom_pcie_cfg cfg_fw_managed = {
 static const struct dw_pcie_ops dw_pcie_ops = {
 	.link_up = qcom_pcie_link_up,
 	.start_link = qcom_pcie_start_link,
+	.get_ltssm = qcom_pcie_get_ltssm,
 };
 
 static int qcom_pcie_icc_init(struct qcom_pcie *pcie)
@@ -1613,6 +1704,22 @@ static void qcom_pcie_icc_opp_update(struct qcom_pcie *pcie)
 	}
 }
 
+static int qcom_pcie_set_max_opp(struct device *dev)
+{
+	unsigned long max_freq = ULONG_MAX;
+	struct dev_pm_opp *opp;
+	int ret;
+
+	opp = dev_pm_opp_find_freq_floor(dev, &max_freq);
+	if (IS_ERR(opp))
+		return PTR_ERR(opp);
+
+	ret = dev_pm_opp_set_opp(dev, opp);
+	dev_pm_opp_put(opp);
+
+	return ret;
+}
+
 static int qcom_pcie_link_transition_count(struct seq_file *s, void *data)
 {
 	struct qcom_pcie *pcie = (struct qcom_pcie *)dev_get_drvdata(s->private);
@@ -1703,6 +1810,11 @@ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie,
 	struct gpio_desc *reset;
 	int ret;
 
+	if (pcie->reset) {
+		reset = pcie->reset;
+		goto skip_perst_parsing;
+	}
+
 	if (!of_find_property(np, "reset-gpios", NULL))
 		goto parse_child_node;
 
@@ -1721,6 +1833,7 @@ static int qcom_pcie_parse_perst(struct qcom_pcie *pcie,
 		return PTR_ERR(reset);
 	}
 
+skip_perst_parsing:
 	perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL);
 	if (!perst)
 		return -ENOMEM;
@@ -1764,6 +1877,9 @@ static int qcom_pcie_parse_port(struct qcom_pcie *pcie, struct device_node *node
 	if (ret)
 		return ret;
 
+	/* TODO: Move to DWC core after multi Root Port support is added */
+	of_property_read_u32(node, "t-power-on-us", &port->l1ss_t_power_on);
+
 	port->phy = phy;
 	INIT_LIST_HEAD(&port->list);
 	list_add_tail(&port->list, &pcie->ports);
@@ -1778,6 +1894,15 @@ static int qcom_pcie_parse_ports(struct qcom_pcie *pcie)
 	struct device *dev = pcie->pci->dev;
 	int ret = -ENODEV;
 
+	if (of_find_property(dev->of_node, "perst-gpios", NULL)) {
+		pcie->reset = devm_gpiod_get_optional(dev, "perst",
+						      GPIOD_OUT_HIGH);
+		if (IS_ERR(pcie->reset))
+			return PTR_ERR(pcie->reset);
+
+		dev_warn(dev, "Reusing PERST# from Root Complex node. DT needs to be fixed!\n");
+	}
+
 	for_each_available_child_of_node_scoped(dev->of_node, of_port) {
 		if (!of_node_is_type(of_port, "pci"))
 			continue;
@@ -1845,9 +1970,7 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 	struct qcom_pcie_perst *perst, *tmp_perst;
 	struct qcom_pcie_port *port, *tmp_port;
 	const struct qcom_pcie_cfg *pcie_cfg;
-	unsigned long max_freq = ULONG_MAX;
 	struct device *dev = &pdev->dev;
-	struct dev_pm_opp *opp;
 	struct qcom_pcie *pcie;
 	struct dw_pcie_rp *pp;
 	struct resource *res;
@@ -1951,21 +2074,9 @@ static int qcom_pcie_probe(struct platform_device *pdev)
 	 * probe(), OPP will be updated using qcom_pcie_icc_opp_update().
 	 */
 	if (!ret) {
-		opp = dev_pm_opp_find_freq_floor(dev, &max_freq);
-		if (IS_ERR(opp)) {
-			ret = PTR_ERR(opp);
-			dev_err_probe(pci->dev, ret,
-				      "Unable to find max freq OPP\n");
-			goto err_pm_runtime_put;
-		} else {
-			ret = dev_pm_opp_set_opp(dev, opp);
-		}
-
-		dev_pm_opp_put(opp);
+		ret = qcom_pcie_set_max_opp(dev);
 		if (ret) {
-			dev_err_probe(pci->dev, ret,
-				      "Failed to set OPP for freq %lu\n",
-				      max_freq);
+			dev_err_probe(dev, ret, "Failed to set max OPP\n");
 			goto err_pm_runtime_put;
 		}
 
@@ -2039,53 +2150,51 @@ static int qcom_pcie_suspend_noirq(struct device *dev)
 	if (!pcie)
 		return 0;
 
-	/*
-	 * Set minimum bandwidth required to keep data path functional during
-	 * suspend.
-	 */
-	if (pcie->icc_mem) {
-		ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1));
-		if (ret) {
-			dev_err(dev,
-				"Failed to set bandwidth for PCIe-MEM interconnect path: %d\n",
-				ret);
-			return ret;
-		}
-	}
+	ret = dw_pcie_suspend_noirq(pcie->pci);
+	if (ret)
+		return ret;
 
-	/*
-	 * Turn OFF the resources only for controllers without active PCIe
-	 * devices. For controllers with active devices, the resources are kept
-	 * ON and the link is expected to be in L0/L1 (sub)states.
-	 *
-	 * Turning OFF the resources for controllers with active PCIe devices
-	 * will trigger access violation during the end of the suspend cycle,
-	 * as kernel tries to access the PCIe devices config space for masking
-	 * MSIs.
-	 *
-	 * Also, it is not desirable to put the link into L2/L3 state as that
-	 * implies VDD supply will be removed and the devices may go into
-	 * powerdown state. This will affect the lifetime of the storage devices
-	 * like NVMe.
-	 */
-	if (!dw_pcie_link_up(pcie->pci)) {
-		qcom_pcie_host_deinit(&pcie->pci->pp);
-		pcie->suspended = true;
-	}
+	if (pcie->pci->suspended) {
+		ret = icc_disable(pcie->icc_mem);
+		if (ret)
+			dev_err(dev, "Failed to disable PCIe-MEM interconnect path: %d\n", ret);
 
-	/*
-	 * Only disable CPU-PCIe interconnect path if the suspend is non-S2RAM.
-	 * Because on some platforms, DBI access can happen very late during the
-	 * S2RAM and a non-active CPU-PCIe interconnect path may lead to NoC
-	 * error.
-	 */
-	if (pm_suspend_target_state != PM_SUSPEND_MEM) {
 		ret = icc_disable(pcie->icc_cpu);
 		if (ret)
 			dev_err(dev, "Failed to disable CPU-PCIe interconnect path: %d\n", ret);
 
 		if (pcie->use_pm_opp)
 			dev_pm_opp_set_opp(pcie->pci->dev, NULL);
+	} else {
+		/*
+		 * Set minimum bandwidth required to keep data path
+		 * functional during suspend.
+		 */
+		if (pcie->icc_mem) {
+			ret = icc_set_bw(pcie->icc_mem, 0, kBps_to_icc(1));
+			if (ret) {
+				dev_err(dev,
+					"Failed to set bandwidth for PCIe-MEM interconnect path: %d\n",
+					ret);
+				return ret;
+			}
+		}
+
+		/*
+		 * Only disable CPU-PCIe interconnect path if the suspend
+		 * is non-S2RAM.  On some platforms, DBI access can happen
+		 * very late during S2RAM and a non-active CPU-PCIe
+		 * interconnect path may lead to NoC error.
+		 */
+		if (pm_suspend_target_state != PM_SUSPEND_MEM) {
+			ret = icc_disable(pcie->icc_cpu);
+			if (ret)
+				dev_err(dev, "Failed to disable CPU-PCIe interconnect path: %d\n",
+					ret);
+
+			if (pcie->use_pm_opp)
+				dev_pm_opp_set_opp(pcie->pci->dev, NULL);
+		}
 	}
 	return ret;
 }
@@ -2099,25 +2208,62 @@ static int qcom_pcie_resume_noirq(struct device *dev)
 	if (!pcie)
 		return 0;
 
-	if (pm_suspend_target_state != PM_SUSPEND_MEM) {
+	if (pcie->pci->suspended) {
+		if (pcie->use_pm_opp) {
+			ret = qcom_pcie_set_max_opp(dev);
+			if (ret) {
+				dev_err(dev, "Failed to set max OPP: %d\n", ret);
+				return ret;
+			}
+		}
+
 		ret = icc_enable(pcie->icc_cpu);
 		if (ret) {
 			dev_err(dev, "Failed to enable CPU-PCIe interconnect path: %d\n", ret);
 			return ret;
 		}
-	}
 
-	if (pcie->suspended) {
-		ret = qcom_pcie_host_init(&pcie->pci->pp);
-		if (ret)
-			return ret;
+		ret = icc_enable(pcie->icc_mem);
+		if (ret) {
+			dev_err(dev, "Failed to enable PCIe-MEM interconnect path: %d\n", ret);
+			goto disable_icc_cpu;
+		}
+
+		/*
+		 * Ignore -ENODEV & -EIO here since it is expected when no
+		 * endpoint is connected to the PCIe link.
+		 */
+		ret = dw_pcie_resume_noirq(pcie->pci);
+		if (ret && ret != -ENODEV && ret != -EIO)
+			goto disable_icc_mem;
+	} else {
+		if (pm_suspend_target_state != PM_SUSPEND_MEM) {
+			if (pcie->use_pm_opp) {
+				ret = qcom_pcie_set_max_opp(dev);
+				if (ret) {
+					dev_err(dev, "Failed to set max OPP: %d\n", ret);
+					return ret;
+				}
+			}
 
-		pcie->suspended = false;
+			ret = icc_enable(pcie->icc_cpu);
+			if (ret) {
+				dev_err(dev, "Failed to enable CPU-PCIe interconnect path: %d\n",
+					ret);
+				return ret;
+			}
+		}
 	}
 
 	qcom_pcie_icc_opp_update(pcie);
 
 	return 0;
+disable_icc_mem:
+	icc_disable(pcie->icc_mem);
+disable_icc_cpu:
+	icc_disable(pcie->icc_cpu);
+
+	return ret;
 }
 
 static const struct of_device_id qcom_pcie_match[] = {
diff --git a/drivers/pci/controller/dwc/pcie-tegra194.c b/drivers/pci/controller/dwc/pcie-tegra194.c
index 9dcfa194050e7..795cef5a915dc 100644
--- a/drivers/pci/controller/dwc/pcie-tegra194.c
+++ b/drivers/pci/controller/dwc/pcie-tegra194.c
@@ -272,6 +272,7 @@ struct tegra_pcie_dw {
 	u32 aspm_cmrt;
 	u32 aspm_pwr_on_t;
 	u32 aspm_l0s_enter_lat;
+	u32 aspm_l1_enter_lat;
 
 	struct regulator *pex_ctl_supply;
 	struct regulator *slot_ctl_3v3;
@@ -715,6 +716,8 @@ static void init_host_aspm(struct tegra_pcie_dw *pcie)
 	val = dw_pcie_readl_dbi(pci, PCIE_PORT_AFR);
 	val &= ~PORT_AFR_L0S_ENTRANCE_LAT_MASK;
 	val |= (pcie->aspm_l0s_enter_lat << PORT_AFR_L0S_ENTRANCE_LAT_SHIFT);
+	val &= ~PORT_AFR_L1_ENTRANCE_LAT_MASK;
+	val |= (pcie->aspm_l1_enter_lat << PORT_AFR_L1_ENTRANCE_LAT_SHIFT);
 	val |= PORT_AFR_ENTER_ASPM;
 	dw_pcie_writel_dbi(pci, PCIE_PORT_AFR, val);
 }
@@ -872,8 +875,7 @@ static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 
 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
-	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC;
-	val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, 0x3ff);
+	FIELD_MODIFY(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, &val, 0x3ff);
 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE;
 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
 
@@ -883,9 +885,8 @@ static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
 	dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
 
 	val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
-	val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC;
-	val |= FIELD_PREP(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC,
-			  pcie->of_data->gen4_preset_vec);
+	FIELD_MODIFY(GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC, &val,
+		     pcie->of_data->gen4_preset_vec);
 	val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE;
 	dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
 
@@ -1115,6 +1116,7 @@ static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
 {
 	struct platform_device *pdev = to_platform_device(pcie->dev);
 	struct device_node *np = pcie->dev->of_node;
+	u32 val;
 	int ret;
 
 	pcie->dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
@@ -1141,6 +1143,15 @@ static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
 		dev_info(pcie->dev,
 			 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
 
+	/* Default to max latency of 7. */
+	pcie->aspm_l1_enter_lat = 7;
+	ret = of_property_read_u32(np, "aspm-l1-entry-delay-ns", &val);
+	if (!ret) {
+		u32 us = DIV_ROUND_UP(val, 1000);
+
+		pcie->aspm_l1_enter_lat = min_t(u32, order_base_2(us), 7);
+	}
+
 	ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
 	if (ret < 0) {
 		dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
diff --git a/drivers/pci/controller/dwc/pcie-ultrarisc.c b/drivers/pci/controller/dwc/pcie-ultrarisc.c
new file mode 100644
index 0000000000000..6ee661ceff67a
--- /dev/null
+++ b/drivers/pci/controller/dwc/pcie-ultrarisc.c
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * DWC PCIe RC driver for UltraRISC SoCs
+ *
+ * Copyright (C) 2026 UltraRISC Technology (Shanghai) Co., Ltd.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/pci.h>
+#include <linux/platform_device.h>
+#include <linux/resource.h>
+
+#include "pcie-designware.h"
+
+#define PCIE_CUS_CORE          0x400000
+
+#define LTSSM_ENABLE           BIT(7)
+#define FAST_LINK_MODE         BIT(12)
+#define HOLD_PHY_RST           BIT(14)
+#define L1SUB_DISABLE          BIT(15)
+
+#define ULTRARISC_PCIE_COMP_TIMEOUT_65_210MS	0x6
+
+static struct pci_ops ultrarisc_pci_ops = {
+	.map_bus = dw_pcie_own_conf_map_bus,
+	.read = pci_generic_config_read32,
+	.write = pci_generic_config_write32,
+};
+
+static int ultrarisc_pcie_host_init(struct dw_pcie_rp *pp)
+{
+	struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
+	struct pci_host_bridge *bridge = pp->bridge;
+	u8 cap_exp;
+	u32 val;
+
+	bridge->ops = &ultrarisc_pci_ops;
+
+	if (dw_pcie_link_up(pci))
+		return 0;
+
+	val = dw_pcie_readl_dbi(pci, PCIE_CUS_CORE);
+	val &= ~FAST_LINK_MODE;
+	dw_pcie_writel_dbi(pci, PCIE_CUS_CORE, val);
+
+	val = dw_pcie_readl_dbi(pci, PCIE_TIMER_CTRL_MAX_FUNC_NUM);
+	FIELD_MODIFY(PORT_FLT_SF_MASK, &val, PORT_FLT_SF_VAL_64);
+	dw_pcie_writel_dbi(pci, PCIE_TIMER_CTRL_MAX_FUNC_NUM, val);
+
+	cap_exp = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
+	val = dw_pcie_readl_dbi(pci, cap_exp + PCI_EXP_LNKCTL2);
+	FIELD_MODIFY(PCI_EXP_LNKCTL2_TLS, &val, PCI_EXP_LNKCTL2_TLS_16_0GT);
+	dw_pcie_writel_dbi(pci, cap_exp + PCI_EXP_LNKCTL2, val);
+
+	val = dw_pcie_readl_dbi(pci, PCIE_PORT_FORCE);
+	FIELD_MODIFY(PORT_LINK_NUM_MASK, &val, 0);
+	dw_pcie_writel_dbi(pci, PCIE_PORT_FORCE, val);
+
+	val = dw_pcie_readl_dbi(pci, cap_exp + PCI_EXP_DEVCTL2);
+	FIELD_MODIFY(PCI_EXP_DEVCTL2_COMP_TIMEOUT, &val,
+		     ULTRARISC_PCIE_COMP_TIMEOUT_65_210MS);
+	dw_pcie_writel_dbi(pci, cap_exp + PCI_EXP_DEVCTL2, val);
+
+	val = dw_pcie_readl_dbi(pci, PCIE_CUS_CORE);
+	val &= ~(HOLD_PHY_RST | L1SUB_DISABLE);
+	dw_pcie_writel_dbi(pci, PCIE_CUS_CORE, val);
+
+	return 0;
+}
+
+static void ultrarisc_pcie_pme_turn_off(struct dw_pcie_rp *pp)
+{
+	/*
+	 * DP1000 does not support sending PME_Turn_Off from the RC.
+	 * Keep this callback empty to skip the generic MSG TLP path.
+	 */
+}
+
+static const struct dw_pcie_host_ops ultrarisc_pcie_host_ops = {
+	.init = ultrarisc_pcie_host_init,
+	.pme_turn_off = ultrarisc_pcie_pme_turn_off,
+};
+
+static int ultrarisc_pcie_start_link(struct dw_pcie *pci)
+{
+	u32 val;
+
+	val = dw_pcie_readl_dbi(pci, PCIE_CUS_CORE);
+	val |= LTSSM_ENABLE;
+	dw_pcie_writel_dbi(pci, PCIE_CUS_CORE, val);
+
+	return 0;
+}
+
+static const struct dw_pcie_ops dw_pcie_ops = {
+	.start_link = ultrarisc_pcie_start_link,
+};
+
+static int ultrarisc_pcie_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct dw_pcie_rp *pp;
+	struct dw_pcie *pci;
+	int ret;
+
+	pci = devm_kzalloc(dev, sizeof(*pci), GFP_KERNEL);
+	if (!pci)
+		return -ENOMEM;
+
+	pci->dev = dev;
+	pci->ops = &dw_pcie_ops;
+
+	/* Set a default value suitable for at most 16 in and 16 out windows */
+	pci->atu_size = SZ_8K;
+
+	pp = &pci->pp;
+
+	platform_set_drvdata(pdev, pci);
+
+	pp->num_vectors = MAX_MSI_IRQS;
+	/* No L2/L3 Ready indication is available on this platform */
+	pp->skip_l23_ready = true;
+	pp->ops = &ultrarisc_pcie_host_ops;
+
+	ret = dw_pcie_host_init(pp);
+	if (ret) {
+		dev_err(dev, "Failed to initialize host\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int ultrarisc_pcie_suspend_noirq(struct device *dev)
+{
+	struct dw_pcie *pci = dev_get_drvdata(dev);
+
+	return dw_pcie_suspend_noirq(pci);
+}
+
+static int ultrarisc_pcie_resume_noirq(struct device *dev)
+{
+	struct dw_pcie *pci = dev_get_drvdata(dev);
+
+	return dw_pcie_resume_noirq(pci);
+}
+
+static const struct dev_pm_ops ultrarisc_pcie_pm_ops = {
+	NOIRQ_SYSTEM_SLEEP_PM_OPS(ultrarisc_pcie_suspend_noirq,
+				  ultrarisc_pcie_resume_noirq)
+};
+
+static const struct of_device_id ultrarisc_pcie_of_match[] = {
+	{
+		.compatible = "ultrarisc,dp1000-pcie",
+	},
+	{},
+};
+MODULE_DEVICE_TABLE(of, ultrarisc_pcie_of_match);
+
+static struct platform_driver ultrarisc_pcie_driver = {
+	.driver = {
+		.name	= "ultrarisc-pcie",
+		.of_match_table = ultrarisc_pcie_of_match,
+		.suppress_bind_attrs = true,
+		.pm = &ultrarisc_pcie_pm_ops,
+	},
+	.probe = ultrarisc_pcie_probe,
+};
+builtin_platform_driver(ultrarisc_pcie_driver);
+
+MODULE_DESCRIPTION("UltraRISC DP1000 DWC PCIe host controller");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pci/controller/dwc/pcie-visconti.c b/drivers/pci/controller/dwc/pcie-visconti.c
index cdeac6177143c..f21775c3b3a1d 100644
--- a/drivers/pci/controller/dwc/pcie-visconti.c
+++ b/drivers/pci/controller/dwc/pcie-visconti.c
@@ -10,7 +10,6 @@
 
 #include <linux/clk.h>
 #include <linux/delay.h>
-#include <linux/gpio.h>
 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/iopoll.h>
diff --git a/drivers/pci/controller/pci-host-common.c b/drivers/pci/controller/pci-host-common.c
index d6258c1cffe5e..67455caaf9e68 100644
--- a/drivers/pci/controller/pci-host-common.c
+++ b/drivers/pci/controller/pci-host-common.c
@@ -9,6 +9,7 @@
 
 #include <linux/kernel.h>
 #include <linux/module.h>
+#include <linux/gpio/consumer.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/of_pci.h>
@@ -17,6 +18,178 @@
 
 #include "pci-host-common.h"
 
+/**
+ * pci_host_common_delete_ports - Cleanup function for port list
+ * @data: Pointer to the port list head
+ */
+void pci_host_common_delete_ports(void *data)
+{
+	struct list_head *ports = data;
+	struct pci_host_perst *perst, *tmp_perst;
+	struct pci_host_port *port, *tmp_port;
+
+	list_for_each_entry_safe(port, tmp_port, ports, list) {
+		list_for_each_entry_safe(perst, tmp_perst, &port->perst, list)
+			list_del(&perst->list);
+		list_del(&port->list);
+	}
+}
+EXPORT_SYMBOL_GPL(pci_host_common_delete_ports);
+
+/**
+ * pci_host_common_parse_perst - Parse PERST# from all nodes, depth first
+ * @dev: Device pointer
+ * @port: PCI host port
+ * @np: Device tree node to start parsing from
+ *
+ * Recursively parse PERST# GPIO from all PCIe bridge nodes starting from
+ * @np in a depth-first manner.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+static int pci_host_common_parse_perst(struct device *dev,
+				       struct pci_host_port *port,
+				       struct device_node *np)
+{
+	struct pci_host_perst *perst;
+	struct gpio_desc *reset;
+	int ret;
+
+	if (!of_property_present(np, "reset-gpios"))
+		goto parse_child_node;
+
+	reset = devm_fwnode_gpiod_get(dev, of_fwnode_handle(np), "reset",
+				      GPIOD_ASIS, "PERST#");
+	if (IS_ERR(reset)) {
+		/*
+		 * FIXME: GPIOLIB currently supports exclusive GPIO access only.
+		 * Non exclusive access is broken. But shared PERST# requires
+		 * non-exclusive access. So once GPIOLIB properly supports it,
+		 * implement it here.
+		 */
+		if (PTR_ERR(reset) == -EBUSY)
+			dev_err(dev, "Shared PERST# is not supported\n");
+
+		return PTR_ERR(reset);
+	}
+
+	perst = devm_kzalloc(dev, sizeof(*perst), GFP_KERNEL);
+	if (!perst)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&perst->list);
+	perst->desc = reset;
+	list_add_tail(&perst->list, &port->perst);
+
+parse_child_node:
+	for_each_available_child_of_node_scoped(np, child) {
+		ret = pci_host_common_parse_perst(dev, port, child);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+/**
+ * pci_host_common_parse_port - Parse a single Root Port node
+ * @dev: Device pointer
+ * @bridge: PCI host bridge
+ * @node: Device tree node of the Root Port
+ *
+ * Parse Root Port properties from the device tree.  Currently it only
+ * handles the PERST# GPIO (including PERST# GPIOs from all PCIe bridge
+ * nodes under this Root Port), which is optional.
+ *
+ * NOTE: This helper fetches resources (like PERST# GPIO) optionally.  If a
+ * controller driver has a hard dependency on certain resources (PHY,
+ * clocks, regulators, etc.), those resources MUST be modeled correctly in
+ * the DT binding and validated in DTS. This helper cannot enforce such
+ * dependencies and the driver may fail to operate if required resources
+ * are missing.
+ *
+ * Return: 0 on success, -ENODEV if PERST# found in RC node (legacy binding
+ * should be used), Other negative error codes on failure.
+ */
+static int pci_host_common_parse_port(struct device *dev,
+				      struct pci_host_bridge *bridge,
+				      struct device_node *node)
+{
+	struct pci_host_port *port;
+	int ret;
+
+	port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL);
+	if (!port)
+		return -ENOMEM;
+
+	INIT_LIST_HEAD(&port->perst);
+
+	ret = pci_host_common_parse_perst(dev, port, node);
+	if (ret)
+		return ret;
+
+	/*
+	 * 1. PERST# found in RP or its child nodes - list is not empty,
+	 *    continue
+	 *
+	 * 2. PERST# not found in RP/children, but found in RC node -
+	 *    return -ENODEV to fallback legacy binding
+	 *
+	 * 3. PERST# not found anywhere - list is empty, continue (optional
+	 *    PERST#)
+	 */
+	if (list_empty(&port->perst)) {
+		if (of_property_present(dev->of_node, "reset-gpios") ||
+		    of_property_present(dev->of_node, "reset-gpio"))
+			return -ENODEV;
+	}
+
+	INIT_LIST_HEAD(&port->list);
+	list_add_tail(&port->list, &bridge->ports);
+
+	return 0;
+}
+
+/**
+ * pci_host_common_parse_ports - Parse Root Port nodes from device tree
+ * @dev: Device pointer
+ * @bridge: PCI host bridge
+ *
+ * Iterate through child nodes of the host bridge and parse Root Port
+ * properties (currently only reset GPIOs).
+ *
+ * Return: 0 on success, -ENODEV if no ports found or PERST# found in RC
+ * node (legacy binding should be used), Other negative error codes on
+ * failure.
+ */
+int pci_host_common_parse_ports(struct device *dev, struct pci_host_bridge *bridge)
+{
+	int ret = -ENODEV;
+
+	for_each_available_child_of_node_scoped(dev->of_node, of_port) {
+		if (!of_node_is_type(of_port, "pci"))
+			continue;
+		ret = pci_host_common_parse_port(dev, bridge, of_port);
+		if (ret)
+			goto err_cleanup;
+	}
+
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, pci_host_common_delete_ports,
+					&bridge->ports);
+
+err_cleanup:
+	pci_host_common_delete_ports(&bridge->ports);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pci_host_common_parse_ports);
+
+#define PCI_HOST_D3COLD_ALLOWED        BIT(0)
+#define PCI_HOST_PME_D3COLD_CAPABLE    BIT(1)
+
+
 static void gen_pci_unmap_cfg(void *ptr)
 {
 	pci_ecam_free((struct pci_config_window *)ptr);
@@ -68,6 +241,10 @@ int pci_host_common_init(struct platform_device *pdev,
 	if (IS_ERR(cfg))
 		return PTR_ERR(cfg);
 
+	/* Do not reassign bus numbers if probe only */
+	if (!pci_has_flag(PCI_PROBE_ONLY))
+		pci_add_flags(PCI_REASSIGN_ALL_BUS);
+
 	bridge->sysdata = cfg;
 	bridge->ops = (struct pci_ops *)&ops->pci_ops;
 	bridge->enable_device = ops->enable_device;
@@ -106,5 +283,75 @@ void pci_host_common_remove(struct platform_device *pdev)
 }
 EXPORT_SYMBOL_GPL(pci_host_common_remove);
 
+static int __pci_host_common_d3cold_possible(struct pci_dev *pdev,
+					     void *userdata)
+{
+	u32 *flags = userdata;
+	int type;
+
+	/* Ignore conventional PCI devices */
+	if (!pci_is_pcie(pdev))
+		return 0;
+
+	type = pci_pcie_type(pdev);
+	if (type != PCI_EXP_TYPE_ENDPOINT &&
+	    type != PCI_EXP_TYPE_LEG_END &&
+	    type != PCI_EXP_TYPE_RC_END)
+		return 0;
+
+	if (!pdev->dev.driver && !pci_is_enabled(pdev))
+		return 0;
+
+	if (pdev->current_state != PCI_D3hot)
+		goto exit;
+
+	if (device_may_wakeup(&pdev->dev)) {
+		if (!pci_pme_capable(pdev, PCI_D3cold))
+			goto exit;
+		else
+			*flags |= PCI_HOST_PME_D3COLD_CAPABLE;
+	}
+
+	return 0;
+
+exit:
+	*flags &= ~PCI_HOST_D3COLD_ALLOWED;
+
+	return -EOPNOTSUPP;
+}
+
+/**
+ * pci_host_common_d3cold_possible - Determine whether the host bridge can
+ *				     transition the devices into D3cold.
+ *
+ * @bridge: PCI host bridge to check
+ * @pme_capable: Pointer to update if there is any device capable of generating
+ *		 PME from D3cold.
+ *
+ * Walk downstream PCIe endpoint devices and determine whether the host bridge
+ * is permitted to transition the devices into D3cold.
+ *
+ * Devices under host bridge can enter D3cold only if all active PCIe
+ * endpoints are in PCI_D3hot and any wakeup-enabled endpoint is capable of
+ * generating PME from D3cold.  Inactive endpoints are ignored.
+ *
+ * The @pme_capable output allows PCIe controller drivers to apply
+ * platform-specific handling to preserve wakeup functionality.
+ *
+ * Return: %true if the host bridge may enter D3cold, otherwise %false.
+ */
+bool pci_host_common_d3cold_possible(struct pci_host_bridge *bridge,
+				     bool *pme_capable)
+{
+	u32 flags = PCI_HOST_D3COLD_ALLOWED;
+
+	pci_walk_bus(bridge->bus, __pci_host_common_d3cold_possible, &flags);
+
+	*pme_capable = !!(flags & PCI_HOST_PME_D3COLD_CAPABLE);
+
+	return !!(flags & PCI_HOST_D3COLD_ALLOWED);
+}
+EXPORT_SYMBOL_GPL(pci_host_common_d3cold_possible);
+
 MODULE_DESCRIPTION("Common library for PCI host controller drivers");
 MODULE_LICENSE("GPL v2");
diff --git a/drivers/pci/controller/pci-host-common.h b/drivers/pci/controller/pci-host-common.h
index b5075d4bd7eb3..458678051d27f 100644
--- a/drivers/pci/controller/pci-host-common.h
+++ b/drivers/pci/controller/pci-host-common.h
@@ -12,6 +12,34 @@
 
 struct pci_ecam_ops;
 
+/**
+ * struct pci_host_perst - PERST# GPIO descriptor
+ * @list: List node for linking multiple PERST# GPIOs
+ * @desc: GPIO descriptor for PERST# signal
+ *
+ * This structure holds a single PERST# GPIO descriptor.
+ */
+struct pci_host_perst {
+	struct list_head	list;
+	struct gpio_desc	*desc;
+};
+
+/**
+ * struct pci_host_port - Generic Root Port properties
+ * @list: List node for linking multiple ports
+ * @perst: List of PERST# GPIO descriptors for this port and its children
+ *
+ * This structure contains common properties that can be parsed from
+ * Root Port device tree nodes.
+ */
+struct pci_host_port {
+	struct list_head	list;
+	struct list_head	perst;
+};
+
+void pci_host_common_delete_ports(void *data);
+int pci_host_common_parse_ports(struct device *dev,
+				struct pci_host_bridge *bridge);
 int pci_host_common_probe(struct platform_device *pdev);
 int pci_host_common_init(struct platform_device *pdev,
 			 struct pci_host_bridge *bridge,
@@ -20,4 +48,7 @@ void pci_host_common_remove(struct platform_device *pdev);
 
 struct pci_config_window *pci_host_common_ecam_create(struct device *dev,
 	struct pci_host_bridge *bridge, const struct pci_ecam_ops *ops);
+
+bool pci_host_common_d3cold_possible(struct pci_host_bridge *bridge,
+				     bool *pme_capable);
 #endif
diff --git a/drivers/pci/controller/pci-loongson.c b/drivers/pci/controller/pci-loongson.c
index bc630ab8a2831..d0c643996476f 100644
--- a/drivers/pci/controller/pci-loongson.c
+++ b/drivers/pci/controller/pci-loongson.c
@@ -80,6 +80,18 @@ DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON,
 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_LOONGSON,
 			DEV_LS7A_LPC, system_bus_quirk);
 
+static const struct pci_device_id loongson_internal_bridge_devids[] = {
+	{ PCI_VDEVICE(LOONGSON, DEV_LS2K_PCIE_PORT0) },
+	{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT0) },
+	{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT1) },
+	{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT2) },
+	{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT3) },
+	{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT4) },
+	{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT5) },
+	{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT6) },
+	{ 0, },
+};
+
 /*
  * Some Loongson PCIe ports have hardware limitations on their Maximum Read
  * Request Size. They can't handle anything larger than this.  Sane
@@ -92,24 +104,13 @@ static void loongson_set_min_mrrs_quirk(struct pci_dev *pdev)
 {
 	struct pci_bus *bus = pdev->bus;
 	struct pci_dev *bridge;
-	static const struct pci_device_id bridge_devids[] = {
-		{ PCI_VDEVICE(LOONGSON, DEV_LS2K_PCIE_PORT0) },
-		{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT0) },
-		{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT1) },
-		{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT2) },
-		{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT3) },
-		{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT4) },
-		{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT5) },
-		{ PCI_VDEVICE(LOONGSON, DEV_LS7A_PCIE_PORT6) },
-		{ 0, },
-	};
 
 	/* look for the matching bridge */
 	while (!pci_is_root_bus(bus)) {
 		bridge = bus->self;
 		bus = bus->parent;
 
-		if (pci_match_id(bridge_devids, bridge)) {
+		if (pci_match_id(loongson_internal_bridge_devids, bridge)) {
 			if (pcie_get_readrq(pdev) > 256) {
 				pci_info(pdev, "limiting MRRS to 256\n");
 				pcie_set_readrq(pdev, 256);
@@ -176,6 +177,42 @@ static void loongson_pci_msi_quirk(struct pci_dev *dev)
 }
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_LOONGSON, DEV_LS7A_PCIE_PORT5, loongson_pci_msi_quirk);
 
+/*
+ * Older steppings of the Loongson-3C6000 series incorrectly report the
+ * supported link speeds on their PCIe bridges (device IDs 0x3c19,
+ * 0x3c29) as only 2.5 GT/s, despite the upstream bus supporting speeds
+ * from 2.5 GT/s up to 16 GT/s.
+ */
+static void loongson_pci_bridge_speed_quirk(struct pci_dev *pdev)
+{
+	u8 old_supported_speeds = pdev->supported_speeds;
+
+	switch (pdev->bus->max_bus_speed) {
+	case PCIE_SPEED_16_0GT:
+		pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_16_0GB;
+		fallthrough;
+	case PCIE_SPEED_8_0GT:
+		pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_8_0GB;
+		fallthrough;
+	case PCIE_SPEED_5_0GT:
+		pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_5_0GB;
+		fallthrough;
+	case PCIE_SPEED_2_5GT:
+		pdev->supported_speeds |= PCI_EXP_LNKCAP2_SLS_2_5GB;
+		break;
+	default:
+		pci_warn(pdev, "unexpected max bus speed");
+
+		return;
+	}
+
+	if (pdev->supported_speeds != old_supported_speeds)
+		pci_info(pdev, "fixed up supported link speeds: 0x%x => 0x%x",
+			 old_supported_speeds, pdev->supported_speeds);
+}
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LOONGSON, 0x3c19, loongson_pci_bridge_speed_quirk);
+DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_LOONGSON, 0x3c29, loongson_pci_bridge_speed_quirk);
+
 static struct loongson_pci *pci_bus_to_loongson_pci(struct pci_bus *bus)
 {
 	struct pci_config_window *cfg;
@@ -230,11 +267,11 @@ static void __iomem *pci_loongson_map_bus(struct pci_bus *bus,
 	struct loongson_pci *priv = pci_bus_to_loongson_pci(bus);
 
 	/*
-	 * Do not read more than one device on the bus other than
-	 * the host bus.
+	 * Do not read more than one device on the internal bridges.
 	 */
 	if ((priv->data->flags & FLAG_DEV_FIX) && bus->self) {
-		if (!pci_is_root_bus(bus) && (device > 0))
+		if (!pci_is_root_bus(bus) && (device > 0) &&
+		    pci_match_id(loongson_internal_bridge_devids, bus->self))
 			return NULL;
 	}
 
diff --git a/drivers/pci/controller/pci-mvebu.c b/drivers/pci/controller/pci-mvebu.c
index a72aa57591c04..d6eb65b8cd7fd 100644
--- a/drivers/pci/controller/pci-mvebu.c
+++ b/drivers/pci/controller/pci-mvebu.c
@@ -263,8 +263,7 @@ static void mvebu_pcie_setup_hw(struct mvebu_pcie_port *port)
 	 * not set correctly then link with endpoint card is not established.
 	 */
 	lnkcap = mvebu_readl(port, PCIE_CAP_PCIEXP + PCI_EXP_LNKCAP);
-	lnkcap &= ~PCI_EXP_LNKCAP_MLW;
-	lnkcap |= FIELD_PREP(PCI_EXP_LNKCAP_MLW, port->is_x4 ? 4 : 1);
+	FIELD_MODIFY(PCI_EXP_LNKCAP_MLW, &lnkcap, port->is_x4 ? 4 : 1);
 	mvebu_writel(port, lnkcap, PCIE_CAP_PCIEXP + PCI_EXP_LNKCAP);
 
 	/* Disable Root Bridge I/O space, memory space and bus mastering. */
diff --git a/drivers/pci/controller/pcie-altera.c b/drivers/pci/controller/pcie-altera.c
index 3dbb7adc421c4..76f3823d96139 100644
--- a/drivers/pci/controller/pcie-altera.c
+++ b/drivers/pci/controller/pcie-altera.c
@@ -864,11 +864,25 @@ static int altera_pcie_init_irq_domain(struct altera_pcie *pcie)
 	return 0;
 }
 
+static void altera_pcie_disable_irq(struct altera_pcie *pcie)
+{
+	if (pcie->pcie_data->version == ALTERA_PCIE_V1 ||
+	    pcie->pcie_data->version == ALTERA_PCIE_V2) {
+		/* Disable all P2A interrupts */
+		cra_writel(pcie, 0, P2A_INT_ENABLE);
+	} else if (pcie->pcie_data->version == ALTERA_PCIE_V3) {
+		/* Disable port-level interrupts (CFG_AER, etc.) */
+		writel(0, pcie->hip_base +
+			  pcie->pcie_data->port_conf_offset +
+			  pcie->pcie_data->port_irq_enable_offset);
+	}
+}
+
 static void altera_pcie_irq_teardown(struct altera_pcie *pcie)
 {
+	altera_pcie_disable_irq(pcie);
 	irq_set_chained_handler_and_data(pcie->irq, NULL, NULL);
 	irq_domain_remove(pcie->irq_domain);
-	irq_dispose_mapping(pcie->irq);
 }
 
 static int altera_pcie_parse_dt(struct altera_pcie *pcie)
@@ -891,7 +905,6 @@ static int altera_pcie_parse_dt(struct altera_pcie *pcie)
 	if (pcie->irq < 0)
 		return pcie->irq;
 
-	irq_set_chained_handler_and_data(pcie->irq, pcie->pcie_data->ops->rp_isr, pcie);
 	return 0;
 }
 
@@ -1020,6 +1033,14 @@ static int altera_pcie_probe(struct platform_device *pdev)
 		return ret;
 	}
 
+	/*
+	 * The chained handler uses pcie->irq_domain, so set it only after the
+	 * INTx domain has been created.
+	 */
+	irq_set_chained_handler_and_data(pcie->irq,
+					 pcie->pcie_data->ops->rp_isr,
+					 pcie);
+
 	if (pcie->pcie_data->version == ALTERA_PCIE_V1 ||
 	    pcie->pcie_data->version == ALTERA_PCIE_V2) {
 		/* clear all interrupts */
@@ -1037,7 +1058,16 @@ static int altera_pcie_probe(struct platform_device *pdev)
 	bridge->busnr = pcie->root_bus_nr;
 	bridge->ops = &altera_pcie_ops;
 
-	return pci_host_probe(bridge);
+	ret = pci_host_probe(bridge);
+	if (ret)
+		goto err_teardown_irq;
+
+	return 0;
+
+err_teardown_irq:
+	altera_pcie_irq_teardown(pcie);
+
+	return ret;
 }
 
 static void altera_pcie_remove(struct platform_device *pdev)
diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c
index 714bcab97b603..08a0e7091cedc 100644
--- a/drivers/pci/controller/pcie-brcmstb.c
+++ b/drivers/pci/controller/pcie-brcmstb.c
@@ -2072,8 +2072,10 @@ static int brcm_pcie_probe(struct platform_device *pdev)
 		return PTR_ERR(pcie->clk);
 
 	ret = of_pci_get_max_link_speed(np);
-	if (pcie_get_link_speed(ret) == PCI_SPEED_UNKNOWN)
+	if (ret < 0 || ret > 3)
 		pcie->gen = 0;
+	else
+		pcie->gen = ret;
 
 	pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc");
 
diff --git a/drivers/pci/controller/pcie-iproc-bcma.c b/drivers/pci/controller/pcie-iproc-bcma.c
index 99a99900444de..593418c2bc3aa 100644
--- a/drivers/pci/controller/pcie-iproc-bcma.c
+++ b/drivers/pci/controller/pcie-iproc-bcma.c
@@ -64,7 +64,7 @@ static int iproc_bcma_pcie_probe(struct bcma_device *bdev)
 	if (ret)
 		return ret;
 
-	pcie->map_irq = iproc_bcma_pcie_map_irq;
+	bridge->map_irq = iproc_bcma_pcie_map_irq;
 
 	bcma_set_drvdata(bdev, pcie);
 
diff --git a/drivers/pci/controller/pcie-iproc-platform.c b/drivers/pci/controller/pcie-iproc-platform.c
index 0cb78c583c7ea..4c9a0c4bb923e 100644
--- a/drivers/pci/controller/pcie-iproc-platform.c
+++ b/drivers/pci/controller/pcie-iproc-platform.c
@@ -98,7 +98,7 @@ static int iproc_pltfm_pcie_probe(struct platform_device *pdev)
 	switch (pcie->type) {
 	case IPROC_PCIE_PAXC:
 	case IPROC_PCIE_PAXC_V2:
-		pcie->map_irq = NULL;
+		bridge->map_irq = NULL;
 		break;
 	default:
 		break;
diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c
index ccf71993ea359..c22d0aecaaac5 100644
--- a/drivers/pci/controller/pcie-iproc.c
+++ b/drivers/pci/controller/pcie-iproc.c
@@ -1502,7 +1502,6 @@ int iproc_pcie_setup(struct iproc_pcie *pcie, struct list_head *res)
 
 	host->ops = &iproc_pcie_ops;
 	host->sysdata = pcie;
-	host->map_irq = pcie->map_irq;
 
 	ret = pci_host_probe(host);
 	if (ret < 0) {
diff --git a/drivers/pci/controller/pcie-iproc.h b/drivers/pci/controller/pcie-iproc.h
index 969ded03b8c2d..c4443f236ca3b 100644
--- a/drivers/pci/controller/pcie-iproc.h
+++ b/drivers/pci/controller/pcie-iproc.h
@@ -61,7 +61,6 @@ struct iproc_msi;
  * @base_addr: PCIe host controller register base physical address
  * @mem: host bridge memory window resource
  * @phy: optional PHY device that controls the Serdes
- * @map_irq: function callback to map interrupts
  * @ep_is_internal: indicates an internal emulated endpoint device is connected
  * @iproc_cfg_read: indicates the iProc config read function should be used
  * @rej_unconfig_pf: indicates the root complex needs to detect and reject
@@ -91,7 +90,6 @@ struct iproc_pcie {
 	phys_addr_t base_addr;
 	struct resource mem;
 	struct phy *phy;
-	int (*map_irq)(const struct pci_dev *, u8, u8);
 	bool ep_is_internal;
 	bool iproc_cfg_read;
 	bool rej_unconfig_pf;
diff --git a/drivers/pci/controller/pcie-mediatek-gen3.c b/drivers/pci/controller/pcie-mediatek-gen3.c
index b0accd8285892..66c33a09dd95b 100644
--- a/drivers/pci/controller/pcie-mediatek-gen3.c
+++ b/drivers/pci/controller/pcie-mediatek-gen3.c
@@ -63,6 +63,12 @@
 #define PCIE_BRG_RSTB			BIT(2)
 #define PCIE_PE_RSTB			BIT(3)
 
+/*
+ * As described in the datasheet of MediaTek PCIe Gen3 controller, wait 10ms
+ * after setting PCIE_BRG_RSTB, and before accessing PCIe internal registers.
+ */
+#define PCIE_BRG_RST_RDY_MS		10
+
 #define PCIE_LTSSM_STATUS_REG		0x150
 #define PCIE_LTSSM_STATE_MASK		GENMASK(28, 24)
 #define PCIE_LTSSM_STATE(val)		((val & PCIE_LTSSM_STATE_MASK) >> 24)
@@ -430,6 +436,21 @@ static int mtk_pcie_devices_power_up(struct mtk_gen3_pcie *pcie)
 		return err;
 	}
 
+	/*
+	 * Some of MediaTek's chips won't output REFCLK when PCIE_PHY_RSTB is
+	 * asserted, we have to de-assert MAC & PHY & BRG reset signals first
+	 * to allow the REFCLK to be stable. While PCIE_BRG_RSTB is asserted,
+	 * there is a short period during which the PCIe internal register
+	 * cannot be accessed, so we need to wait 10ms here.
+	 */
+	msleep(PCIE_BRG_RST_RDY_MS);
+
+	if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) {
+		/* De-assert MAC, PHY and BRG reset signals */
+		val &= ~(PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB);
+		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
+	}
+
 	/*
 	 * Described in PCIe CEM specification revision 6.0.
 	 *
@@ -439,9 +460,8 @@ static int mtk_pcie_devices_power_up(struct mtk_gen3_pcie *pcie)
 	msleep(PCIE_T_PVPERL_MS);
 
 	if (!(pcie->soc->flags & SKIP_PCIE_RSTB)) {
-		/* De-assert reset signals */
-		val &= ~(PCIE_MAC_RSTB | PCIE_PHY_RSTB | PCIE_BRG_RSTB |
-			 PCIE_PE_RSTB);
+		/* De-assert PERST# signal */
+		val &= ~PCIE_PE_RSTB;
 		writel_relaxed(val, pcie->base + PCIE_RST_CTRL_REG);
 	}
 
@@ -494,8 +514,7 @@ static int mtk_pcie_startup_port(struct mtk_gen3_pcie *pcie)
 	/* Set Link Control 2 (LNKCTL2) speed restriction, if any */
 	if (pcie->max_link_speed) {
 		val = readl_relaxed(pcie->base + PCIE_CONF_LINK2_CTL_STS);
-		val &= ~PCIE_CONF_LINK2_LCR2_LINK_SPEED;
-		val |= FIELD_PREP(PCIE_CONF_LINK2_LCR2_LINK_SPEED, pcie->max_link_speed);
+		FIELD_MODIFY(PCIE_CONF_LINK2_LCR2_LINK_SPEED, &val, pcie->max_link_speed);
 		writel_relaxed(val, pcie->base + PCIE_CONF_LINK2_CTL_STS);
 	}
 
@@ -1222,8 +1241,8 @@ static int mtk_pcie_probe(struct platform_device *pdev)
 
 	err = pci_pwrctrl_create_devices(pcie->dev);
 	if (err) {
-		goto err_tear_down_irq;
 		dev_err_probe(dev, err, "failed to create pwrctrl devices\n");
+		goto err_tear_down_irq;
 	}
 
 	err = mtk_pcie_setup(pcie);
@@ -1260,12 +1279,20 @@ static void mtk_pcie_remove(struct platform_device *pdev)
 	pci_remove_root_bus(host->bus);
 	pci_unlock_rescan_remove();
 
-	pci_pwrctrl_power_off_devices(pcie->dev);
+	mtk_pcie_devices_power_down(pcie);
 	mtk_pcie_power_down(pcie);
 	pci_pwrctrl_destroy_devices(pcie->dev);
 	mtk_pcie_irq_teardown(pcie);
 }
 
+static void mtk_pcie_shutdown(struct platform_device *pdev)
+{
+	struct mtk_gen3_pcie *pcie = platform_get_drvdata(pdev);
+
+	mtk_pcie_devices_power_down(pcie);
+	mtk_pcie_power_down(pcie);
+}
+
 static void mtk_pcie_irq_save(struct mtk_gen3_pcie *pcie)
 {
 	int i;
@@ -1404,6 +1431,7 @@ MODULE_DEVICE_TABLE(of, mtk_pcie_of_match);
 static struct platform_driver mtk_pcie_driver = {
 	.probe = mtk_pcie_probe,
 	.remove = mtk_pcie_remove,
+	.shutdown = mtk_pcie_shutdown,
 	.driver = {
 		.name = "mtk-pcie-gen3",
 		.of_match_table = mtk_pcie_of_match,
diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c
index 591d301fa89d8..4802d4f80f780 100644
--- a/drivers/pci/endpoint/functions/pci-epf-test.c
+++ b/drivers/pci/endpoint/functions/pci-epf-test.c
@@ -94,6 +94,7 @@ struct pci_epf_test {
 	bool			dma_private;
 	const struct pci_epc_features *epc_features;
 	struct pci_epf_bar	db_bar;
+	bool			db_bar_programmed;
 	size_t			bar_size[PCI_STD_NUM_BARS];
 };
 
@@ -733,7 +734,9 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test,
 {
 	u32 status = le32_to_cpu(reg->status);
 	struct pci_epf *epf = epf_test->epf;
+	struct pci_epf_doorbell_msg *db;
 	struct pci_epc *epc = epf->epc;
+	unsigned long irq_flags;
 	struct msi_msg *msg;
 	enum pci_barno bar;
 	size_t offset;
@@ -743,13 +746,28 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test,
 	if (ret)
 		goto set_status_err;
 
-	msg = &epf->db_msg[0].msg;
-	bar = pci_epc_get_next_free_bar(epf_test->epc_features, epf_test->test_reg_bar + 1);
-	if (bar < BAR_0)
-		goto err_doorbell_cleanup;
+	db = &epf->db_msg[0];
+	msg = &db->msg;
+	epf_test->db_bar_programmed = false;
+
+	if (db->bar != NO_BAR) {
+		/*
+		 * The doorbell target is already exposed via a platform-owned
+		 * fixed BAR
+		 */
+		bar = db->bar;
+		offset = db->offset;
+	} else {
+		bar = pci_epc_get_next_free_bar(epf_test->epc_features,
+						epf_test->test_reg_bar + 1);
+		if (bar < BAR_0)
+			goto err_doorbell_cleanup;
+	}
+
+	irq_flags = epf->db_msg[0].irq_flags | IRQF_ONESHOT;
 
 	ret = request_threaded_irq(epf->db_msg[0].virq, NULL,
-				   pci_epf_test_doorbell_handler, IRQF_ONESHOT,
+				   pci_epf_test_doorbell_handler, irq_flags,
 				   "pci-ep-test-doorbell", epf_test);
 	if (ret) {
 		dev_err(&epf->dev,
@@ -761,22 +779,30 @@ static void pci_epf_test_enable_doorbell(struct pci_epf_test *epf_test,
 	reg->doorbell_data = cpu_to_le32(msg->data);
 	reg->doorbell_bar = cpu_to_le32(bar);
 
-	msg = &epf->db_msg[0].msg;
-	ret = pci_epf_align_inbound_addr(epf, bar, ((u64)msg->address_hi << 32) | msg->address_lo,
-					 &epf_test->db_bar.phys_addr, &offset);
+	if (db->bar == NO_BAR) {
+		ret = pci_epf_align_inbound_addr(epf, bar,
+						 ((u64)msg->address_hi << 32) |
+						 msg->address_lo,
+						 &epf_test->db_bar.phys_addr,
+						 &offset);
 
-	if (ret)
-		goto err_free_irq;
+		if (ret)
+			goto err_free_irq;
+	}
 
 	reg->doorbell_offset = cpu_to_le32(offset);
 
-	epf_test->db_bar.barno = bar;
-	epf_test->db_bar.size = epf->bar[bar].size;
-	epf_test->db_bar.flags = epf->bar[bar].flags;
+	if (db->bar == NO_BAR) {
+		epf_test->db_bar.barno = bar;
+		epf_test->db_bar.size = epf->bar[bar].size;
+		epf_test->db_bar.flags = epf->bar[bar].flags;
 
-	ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf_test->db_bar);
-	if (ret)
-		goto err_free_irq;
+		ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf_test->db_bar);
+		if (ret)
+			goto err_free_irq;
+
+		epf_test->db_bar_programmed = true;
+	}
 
 	status |= STATUS_DOORBELL_ENABLE_SUCCESS;
 	reg->status = cpu_to_le32(status);
@@ -806,17 +832,23 @@ static void pci_epf_test_disable_doorbell(struct pci_epf_test *epf_test,
 	free_irq(epf->db_msg[0].virq, epf_test);
 	pci_epf_test_doorbell_cleanup(epf_test);
 
-	/*
-	 * The doorbell feature temporarily overrides the inbound translation
-	 * to point to the address stored in epf_test->db_bar.phys_addr, i.e.,
-	 * it calls set_bar() twice without ever calling clear_bar(), as
-	 * calling clear_bar() would clear the BAR's PCI address assigned by
-	 * the host. Thus, when disabling the doorbell, restore the inbound
-	 * translation to point to the memory allocated for the BAR.
-	 */
-	ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf->bar[bar]);
-	if (ret)
-		goto set_status_err;
+	if (epf_test->db_bar_programmed) {
+		/*
+		 * The doorbell feature temporarily overrides the inbound
+		 * translation to point to the address stored in
+		 * epf_test->db_bar.phys_addr, i.e., it calls set_bar()
+		 * twice without ever calling clear_bar(), as calling
+		 * clear_bar() would clear the BAR's PCI address assigned
+		 * by the host. Thus, when disabling the doorbell, restore
+		 * the inbound translation to point to the memory allocated
+		 * for the BAR.
+		 */
+		ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no, &epf->bar[bar]);
+		if (ret)
+			goto set_status_err;
+
+		epf_test->db_bar_programmed = false;
+	}
 
 	status |= STATUS_DOORBELL_DISABLE_SUCCESS;
 	reg->status = cpu_to_le32(status);
diff --git a/drivers/pci/endpoint/functions/pci-epf-vntb.c b/drivers/pci/endpoint/functions/pci-epf-vntb.c
index 2256c3062b1ae..b493a300da4dc 100644
--- a/drivers/pci/endpoint/functions/pci-epf-vntb.c
+++ b/drivers/pci/endpoint/functions/pci-epf-vntb.c
@@ -134,6 +134,11 @@ struct epf_ntb {
 	u16 vntb_vid;
 
 	bool linkup;
+
+	/*
+	 * True when doorbells are interrupt-driven (MSI or embedded), false
+	 * when polled.
+	 */
 	bool msi_doorbell;
 	u32 spad_size;
 
@@ -517,6 +522,17 @@ static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
 	return 0;
 }
 
+static bool epf_ntb_db_irq_is_duplicated(const struct pci_epf *epf, unsigned int idx)
+{
+	unsigned int i;
+
+	for (i = 0; i < idx; i++)
+		if (epf->db_msg[i].virq == epf->db_msg[idx].virq)
+			return true;
+
+	return false;
+}
+
 static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
 					    struct pci_epf_bar *db_bar,
 					    const struct pci_epc_features *epc_features,
@@ -533,9 +549,24 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
 	if (ret)
 		return ret;
 
+	/*
+	 * The doorbell target may already be exposed by a platform-owned fixed
+	 * BAR. In that case, we must reuse it and the requested db_bar must
+	 * match.
+	 */
+	if (epf->db_msg[0].bar != NO_BAR && epf->db_msg[0].bar != barno) {
+		ret = -EINVAL;
+		goto err_free_doorbell;
+	}
+
 	for (req = 0; req < ntb->db_count; req++) {
+		/* Avoid requesting duplicate handlers */
+		if (epf_ntb_db_irq_is_duplicated(epf, req))
+			continue;
+
 		ret = request_irq(epf->db_msg[req].virq, epf_ntb_doorbell_handler,
-				  0, "pci_epf_vntb_db", ntb);
+				  epf->db_msg[req].irq_flags, "pci_epf_vntb_db",
+				  ntb);
 
 		if (ret) {
 			dev_err(&epf->dev,
@@ -545,6 +576,22 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
 		}
 	}
 
+	if (epf->db_msg[0].bar != NO_BAR) {
+		for (i = 0; i < ntb->db_count; i++) {
+			msg = &epf->db_msg[i].msg;
+
+			if (epf->db_msg[i].bar != barno) {
+				ret = -EINVAL;
+				goto err_free_irq;
+			}
+
+			ntb->reg->db_data[i] = msg->data;
+			ntb->reg->db_offset[i] = epf->db_msg[i].offset;
+		}
+		goto out;
+	}
+
+	/* Program inbound mapping for the doorbell */
 	msg = &epf->db_msg[0].msg;
 
 	high = 0;
@@ -591,6 +638,7 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
 		ntb->reg->db_offset[i] = offset;
 	}
 
+out:
 	ntb->reg->db_entry_size = 0;
 
 	ntb->msi_doorbell = true;
@@ -598,9 +646,13 @@ static int epf_ntb_db_bar_init_msi_doorbell(struct epf_ntb *ntb,
 	return 0;
 
 err_free_irq:
-	for (req--; req >= 0; req--)
+	for (req--; req >= 0; req--) {
+		if (epf_ntb_db_irq_is_duplicated(epf, req))
+			continue;
 		free_irq(epf->db_msg[req].virq, ntb);
+	}
 
+err_free_doorbell:
 	pci_epf_free_doorbell(ntb->epf);
 	return ret;
 }
@@ -666,8 +718,11 @@ static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
 	if (ntb->msi_doorbell) {
 		int i;
 
-		for (i = 0; i < ntb->db_count; i++)
+		for (i = 0; i < ntb->db_count; i++) {
+			if (epf_ntb_db_irq_is_duplicated(ntb->epf, i))
+				continue;
 			free_irq(ntb->epf->db_msg[i].virq, ntb);
+		}
 	}
 
 	if (ntb->epf->db_msg)
diff --git a/drivers/pci/endpoint/pci-ep-msi.c b/drivers/pci/endpoint/pci-ep-msi.c
index 1395919571f83..0855c7930abb4 100644
--- a/drivers/pci/endpoint/pci-ep-msi.c
+++ b/drivers/pci/endpoint/pci-ep-msi.c
@@ -6,8 +6,11 @@
  * Author: Frank Li <Frank.Li@nxp.com>
  */
 
+#include <linux/align.h>
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/export.h>
+#include <linux/interrupt.h>
 #include <linux/irqdomain.h>
 #include <linux/module.h>
 #include <linux/msi.h>
@@ -35,23 +38,116 @@ static void pci_epf_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
 	pci_epc_put(epc);
 }
 
-int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
+static int pci_epf_alloc_doorbell_embedded(struct pci_epf *epf, u16 num_db)
 {
+	const struct pci_epc_aux_resource *doorbell = NULL;
+	struct pci_epf_doorbell_msg *msg;
 	struct pci_epc *epc = epf->epc;
-	struct device *dev = &epf->dev;
-	struct irq_domain *domain;
-	void *msg;
-	int ret;
-	int i;
+	size_t map_size = 0, off = 0;
+	dma_addr_t iova_base = 0;
+	phys_addr_t phys_base;
+	int count, ret, i;
+	u64 addr;
+
+	count = pci_epc_get_aux_resources_count(epc, epf->func_no,
+						epf->vfunc_no);
+	if (count < 0)
+		return count;
+	if (!count)
+		return -ENODEV;
 
-	/* TODO: Multi-EPF support */
-	if (list_first_entry_or_null(&epc->pci_epf, struct pci_epf, list) != epf) {
-		dev_err(dev, "MSI doorbell doesn't support multiple EPF\n");
+	struct pci_epc_aux_resource *res __free(kfree) =
+				kcalloc(count, sizeof(*res), GFP_KERNEL);
+	if (!res)
+		return -ENOMEM;
+
+	ret = pci_epc_get_aux_resources(epc, epf->func_no, epf->vfunc_no,
+					res, count);
+	if (ret)
+		return ret;
+
+	/* TODO: Support multiple DOORBELL_MMIO resources per EPC. */
+	for (i = 0; i < count; i++) {
+		if (res[i].type != PCI_EPC_AUX_DOORBELL_MMIO)
+			continue;
+
+		doorbell = &res[i];
+		break;
+	}
+	if (!doorbell)
+		return -ENODEV;
+	addr = doorbell->phys_addr;
+	if (!IS_ALIGNED(addr, sizeof(u32)))
 		return -EINVAL;
+
+	/*
+	 * Reuse the pre-exposed BAR window if available. Otherwise map the MMIO
+	 * doorbell resource here. Any required IOMMU mapping is handled
+	 * internally, matching the MSI doorbell semantics.
+	 */
+	if (doorbell->bar == NO_BAR) {
+		phys_base = addr & PAGE_MASK;
+		off = addr - phys_base;
+		map_size = PAGE_ALIGN(off + sizeof(u32));
+
+		iova_base = dma_map_resource(epc->dev.parent, phys_base,
+					     map_size, DMA_FROM_DEVICE, 0);
+		if (dma_mapping_error(epc->dev.parent, iova_base))
+			return -EIO;
+
+		addr = iova_base + off;
 	}
 
-	if (epf->db_msg)
-		return -EBUSY;
+	msg = kcalloc(num_db, sizeof(*msg), GFP_KERNEL);
+	if (!msg) {
+		ret = -ENOMEM;
+		goto err_unmap;
+	}
+
+	/*
+	 * Embedded doorbell backends (e.g. DesignWare eDMA interrupt emulation)
+	 * typically provide a single IRQ and do not offer per-doorbell
+	 * distinguishable address/data pairs. The EPC aux resource therefore
+	 * exposes one DOORBELL_MMIO entry (u.db_mmio.irq).
+	 *
+	 * Still, pci_epf_alloc_doorbell() allows requesting multiple doorbells.
+	 * For such backends we replicate the same address/data for each entry
+	 * and mark the IRQ as shared (IRQF_SHARED). Consumers must treat them
+	 * as equivalent "kick" doorbells.
+	 */
+	for (i = 0; i < num_db; i++)
+		msg[i] = (struct pci_epf_doorbell_msg) {
+			.msg.address_lo = (u32)addr,
+			.msg.address_hi = (u32)(addr >> 32),
+			.msg.data = doorbell->u.db_mmio.data,
+			.virq = doorbell->u.db_mmio.irq,
+			.irq_flags = IRQF_SHARED,
+			.type = PCI_EPF_DOORBELL_EMBEDDED,
+			.bar = doorbell->bar,
+			.offset = (doorbell->bar == NO_BAR) ? 0 :
+				  doorbell->bar_offset,
+			.iova_base = iova_base,
+			.iova_size = map_size,
+		};
+
+	epf->num_db = num_db;
+	epf->db_msg = msg;
+	return 0;
+
+err_unmap:
+	if (map_size)
+		dma_unmap_resource(epc->dev.parent, iova_base, map_size,
+				   DMA_FROM_DEVICE, 0);
+	return ret;
+}
+
+static int pci_epf_alloc_doorbell_msi(struct pci_epf *epf, u16 num_db)
+{
+	struct pci_epf_doorbell_msg *msg;
+	struct device *dev = &epf->dev;
+	struct pci_epc *epc = epf->epc;
+	struct irq_domain *domain;
+	int ret, i;
 
 	domain = of_msi_map_get_device_domain(epc->dev.parent, 0,
 					      DOMAIN_BUS_PLATFORM_MSI);
@@ -74,6 +170,12 @@ int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
 	if (!msg)
 		return -ENOMEM;
 
+	for (i = 0; i < num_db; i++)
+		msg[i] = (struct pci_epf_doorbell_msg) {
+			.type = PCI_EPF_DOORBELL_MSI,
+			.bar = NO_BAR,
+		};
+
 	epf->num_db = num_db;
 	epf->db_msg = msg;
 
@@ -90,13 +192,60 @@ int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
 	for (i = 0; i < num_db; i++)
 		epf->db_msg[i].virq = msi_get_virq(epc->dev.parent, i);
 
-	return ret;
+	return 0;
+}
+
+int pci_epf_alloc_doorbell(struct pci_epf *epf, u16 num_db)
+{
+	struct pci_epc *epc = epf->epc;
+	struct device *dev = &epf->dev;
+	int ret;
+
+	/* TODO: Multi-EPF support */
+	if (list_first_entry_or_null(&epc->pci_epf, struct pci_epf, list) != epf) {
+		dev_err(dev, "Doorbell doesn't support multiple EPF\n");
+		return -EINVAL;
+	}
+
+	if (epf->db_msg)
+		return -EBUSY;
+
+	ret = pci_epf_alloc_doorbell_msi(epf, num_db);
+	if (!ret)
+		return 0;
+
+	/*
+	 * Fall back to embedded doorbell only when platform MSI is unavailable
+	 * for this EPC.
+	 */
+	if (ret != -ENODEV)
+		return ret;
+
+	ret = pci_epf_alloc_doorbell_embedded(epf, num_db);
+	if (ret) {
+		dev_err(dev, "Failed to allocate doorbell: %d\n", ret);
+		return ret;
+	}
+
+	dev_info(dev, "Using embedded (DMA) doorbell fallback\n");
+	return 0;
 }
 EXPORT_SYMBOL_GPL(pci_epf_alloc_doorbell);
 
 void pci_epf_free_doorbell(struct pci_epf *epf)
 {
-	platform_device_msi_free_irqs_all(epf->epc->dev.parent);
+	struct pci_epf_doorbell_msg *msg0;
+	struct pci_epc *epc = epf->epc;
+
+	if (!epf->db_msg)
+		return;
+
+	msg0 = &epf->db_msg[0];
+	if (msg0->type == PCI_EPF_DOORBELL_MSI)
+		platform_device_msi_free_irqs_all(epf->epc->dev.parent);
+	else if (msg0->type == PCI_EPF_DOORBELL_EMBEDDED && msg0->iova_size)
+		dma_unmap_resource(epc->dev.parent, msg0->iova_base,
+				   msg0->iova_size, DMA_FROM_DEVICE, 0);
 
 	kfree(epf->db_msg);
 	epf->db_msg = NULL;
diff --git a/drivers/pci/endpoint/pci-epc-core.c b/drivers/pci/endpoint/pci-epc-core.c
index 6c3c58185fc5d..831b40458dcd8 100644
--- a/drivers/pci/endpoint/pci-epc-core.c
+++ b/drivers/pci/endpoint/pci-epc-core.c
@@ -156,6 +156,86 @@ const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc,
 }
 EXPORT_SYMBOL_GPL(pci_epc_get_features);
 
+/**
+ * pci_epc_get_aux_resources_count() - get the number of EPC-provided auxiliary resources
+ * @epc: EPC device
+ * @func_no: function number
+ * @vfunc_no: virtual function number
+ *
+ * Some EPC backends integrate auxiliary blocks (e.g. DMA engines) whose control
+ * registers and/or descriptor memories can be exposed to the host by mapping
+ * them into BAR space. This helper queries how many such resources the backend
+ * provides.
+ *
+ * Return: the number of available resources on success, -EOPNOTSUPP if the
+ * backend does not support auxiliary resource queries, or another -errno on
+ * failure.
+ */
+int pci_epc_get_aux_resources_count(struct pci_epc *epc, u8 func_no,
+				    u8 vfunc_no)
+{
+	int count;
+
+	if (!epc || !epc->ops)
+		return -EINVAL;
+
+	if (!pci_epc_function_is_valid(epc, func_no, vfunc_no))
+		return -EINVAL;
+
+	if (!epc->ops->get_aux_resources_count)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&epc->lock);
+	count = epc->ops->get_aux_resources_count(epc, func_no,
+						  vfunc_no);
+	mutex_unlock(&epc->lock);
+
+	return count;
+}
+EXPORT_SYMBOL_GPL(pci_epc_get_aux_resources_count);
+
+/**
+ * pci_epc_get_aux_resources() - query EPC-provided auxiliary resources
+ * @epc: EPC device
+ * @func_no: function number
+ * @vfunc_no: virtual function number
+ * @resources: output array
+ * @num_resources: size of @resources array in entries
+ *
+ * Some EPC backends integrate auxiliary blocks (e.g. DMA engines) whose control
+ * registers and/or descriptor memories can be exposed to the host by mapping
+ * them into BAR space. This helper queries the backend for such resources.
+ *
+ * Return: 0 on success, -EOPNOTSUPP if the backend does not support auxiliary
+ * resource queries, or another -errno on failure.
+ */
+int pci_epc_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			      struct pci_epc_aux_resource *resources,
+			      int num_resources)
+{
+	int ret;
+
+	if (!resources || num_resources <= 0)
+		return -EINVAL;
+
+	if (!epc || !epc->ops)
+		return -EINVAL;
+
+	if (!pci_epc_function_is_valid(epc, func_no, vfunc_no))
+		return -EINVAL;
+
+	if (!epc->ops->get_aux_resources)
+		return -EOPNOTSUPP;
+
+	mutex_lock(&epc->lock);
+	ret = epc->ops->get_aux_resources(epc, func_no, vfunc_no, resources,
+					  num_resources);
+	mutex_unlock(&epc->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(pci_epc_get_aux_resources);
+
 /**
  * pci_epc_stop() - stop the PCI link
  * @epc: the link of the EPC device that has to be stopped
diff --git a/drivers/pci/ide.c b/drivers/pci/ide.c
index be74e8f0ae21a..beb67b8fb5c56 100644
--- a/drivers/pci/ide.c
+++ b/drivers/pci/ide.c
@@ -170,8 +170,7 @@ void pci_ide_init(struct pci_dev *pdev)
 		pci_read_config_dword(pdev, pos + PCI_IDE_SEL_CTL, &val);
 		if (val & PCI_IDE_SEL_CTL_EN)
 			continue;
-		val &= ~PCI_IDE_SEL_CTL_ID;
-		val |= FIELD_PREP(PCI_IDE_SEL_CTL_ID, PCI_IDE_RESERVED_STREAM_ID);
+		FIELD_MODIFY(PCI_IDE_SEL_CTL_ID, &val, PCI_IDE_RESERVED_STREAM_ID);
 		pci_write_config_dword(pdev, pos + PCI_IDE_SEL_CTL, val);
 	}
 
@@ -182,8 +181,7 @@ void pci_ide_init(struct pci_dev *pdev)
 		pci_read_config_dword(pdev, pos, &val);
 		if (val & PCI_IDE_LINK_CTL_EN)
 			continue;
-		val &= ~PCI_IDE_LINK_CTL_ID;
-		val |= FIELD_PREP(PCI_IDE_LINK_CTL_ID, PCI_IDE_RESERVED_STREAM_ID);
+		FIELD_MODIFY(PCI_IDE_LINK_CTL_ID, &val, PCI_IDE_RESERVED_STREAM_ID);
 		pci_write_config_dword(pdev, pos, val);
 	}
 
diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index 91ac4e37ecb9c..b0d24839c0840 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -150,7 +150,7 @@ static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus)
 		pci_remove_bus(virtbus);
 }
 
-resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
+resource_size_t pci_iov_resource_size(const struct pci_dev *dev, int resno)
 {
 	if (!dev->is_physfn)
 		return 0;
@@ -938,16 +938,21 @@ static void sriov_restore_vf_rebar_state(struct pci_dev *dev)
 		return;
 
 	pci_read_config_dword(dev, pos + PCI_VF_REBAR_CTRL, &ctrl);
+	if (PCI_POSSIBLE_ERROR(ctrl))
+		return;
+
 	nbars = FIELD_GET(PCI_VF_REBAR_CTRL_NBAR_MASK, ctrl);
 
 	for (i = 0; i < nbars; i++, pos += 8) {
 		int bar_idx, size;
 
 		pci_read_config_dword(dev, pos + PCI_VF_REBAR_CTRL, &ctrl);
+		if (PCI_POSSIBLE_ERROR(ctrl))
+			return;
+
 		bar_idx = FIELD_GET(PCI_VF_REBAR_CTRL_BAR_IDX, ctrl);
 		size = pci_rebar_bytes_to_size(dev->sriov->barsz[bar_idx]);
-		ctrl &= ~PCI_VF_REBAR_CTRL_BAR_SIZE;
-		ctrl |= FIELD_PREP(PCI_VF_REBAR_CTRL_BAR_SIZE, size);
+		FIELD_MODIFY(PCI_VF_REBAR_CTRL_BAR_SIZE, &ctrl, size);
 		pci_write_config_dword(dev, pos + PCI_VF_REBAR_CTRL, ctrl);
 	}
 }
@@ -1084,7 +1089,7 @@ void pci_iov_update_resource(struct pci_dev *dev, int resno)
 	}
 }
 
-resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev,
+resource_size_t __weak pcibios_iov_resource_alignment(const struct pci_dev *dev,
 						      int resno)
 {
 	return pci_iov_resource_size(dev, resno);
@@ -1100,7 +1105,8 @@ resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev,
  * the VF BAR size multiplied by the number of VFs.  The alignment
  * is just the VF BAR size.
  */
-resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno)
+resource_size_t pci_sriov_resource_alignment(const struct pci_dev *dev,
+					     int resno)
 {
 	return pcibios_iov_resource_alignment(dev, resno);
 }
diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c
index 81d24a270a795..33c8b5b98684f 100644
--- a/drivers/pci/msi/msi.c
+++ b/drivers/pci/msi/msi.c
@@ -201,8 +201,7 @@ static inline void pci_write_msg_msi(struct pci_dev *dev, struct msi_desc *desc,
 	u16 msgctl;
 
 	pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
-	msgctl &= ~PCI_MSI_FLAGS_QSIZE;
-	msgctl |= FIELD_PREP(PCI_MSI_FLAGS_QSIZE, desc->pci.msi_attrib.multiple);
+	FIELD_MODIFY(PCI_MSI_FLAGS_QSIZE, &msgctl, desc->pci.msi_attrib.multiple);
 	pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl);
 
 	pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, msg->address_lo);
@@ -532,9 +531,8 @@ void __pci_restore_msi_state(struct pci_dev *dev)
 
 	pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
 	pci_msi_update_mask(entry, 0, 0);
-	control &= ~PCI_MSI_FLAGS_QSIZE;
-	control |= PCI_MSI_FLAGS_ENABLE |
-		   FIELD_PREP(PCI_MSI_FLAGS_QSIZE, entry->pci.msi_attrib.multiple);
+	FIELD_MODIFY(PCI_MSI_FLAGS_QSIZE, &control, entry->pci.msi_attrib.multiple);
+	control |= PCI_MSI_FLAGS_ENABLE;
 	pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
 }
 
@@ -970,8 +968,7 @@ int pci_msix_write_tph_tag(struct pci_dev *pdev, unsigned int index, u16 tag)
 	if (!msi_desc || msi_desc->pci.msi_attrib.is_virtual)
 		return -ENXIO;
 
-	msi_desc->pci.msix_ctrl &= ~PCI_MSIX_ENTRY_CTRL_ST;
-	msi_desc->pci.msix_ctrl |= FIELD_PREP(PCI_MSIX_ENTRY_CTRL_ST, tag);
+	FIELD_MODIFY(PCI_MSIX_ENTRY_CTRL_ST, &msi_desc->pci.msix_ctrl, tag);
 	pci_msix_write_vector_ctrl(msi_desc, msi_desc->pci.msix_ctrl);
 	/* Flush the write */
 	readl(pci_msix_desc_addr(msi_desc));
diff --git a/drivers/pci/of.c b/drivers/pci/of.c
index 6da569fd3b8f6..8b18c4ba845cf 100644
--- a/drivers/pci/of.c
+++ b/drivers/pci/of.c
@@ -38,7 +38,7 @@ int pci_set_of_node(struct pci_dev *dev)
 	struct device *pdev __free(put_device) =
 		bus_find_device_by_of_node(&platform_bus_type, node);
 	if (pdev)
-		dev->bus->dev.of_node_reused = true;
+		dev_set_of_node_reused(&dev->bus->dev);
 
 	device_set_node(&dev->dev, of_fwnode_handle(no_free_ptr(node)));
 	return 0;
diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index 7c898542af8d5..adb17a4f69396 100644
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -262,6 +262,9 @@ int pcim_p2pdma_init(struct pci_dev *pdev)
 	struct pci_p2pdma *p2p;
 	int i, ret;
 
+	if (pdev->non_mappable_bars)
+		return -EOPNOTSUPP;
+
 	p2p = rcu_dereference_protected(pdev->p2pdma, 1);
 	if (p2p)
 		return 0;
@@ -318,7 +321,8 @@ struct p2pdma_provider *pcim_p2pdma_provider(struct pci_dev *pdev, int bar)
 {
 	struct pci_p2pdma *p2p;
 
-	if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
+	if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM) ||
+	    pdev->non_mappable_bars)
 		return NULL;
 
 	p2p = rcu_dereference_protected(pdev->p2pdma, 1);
diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c
index e3f59001785a1..f36778e62ac1a 100644
--- a/drivers/pci/pci-driver.c
+++ b/drivers/pci/pci-driver.c
@@ -519,13 +519,6 @@ static void pci_device_remove(struct device *dev)
 	/* Undo the runtime PM settings in local_pci_probe() */
 	pm_runtime_put_sync(dev);
 
-	/*
-	 * If the device is still on, set the power state as "unknown",
-	 * since it might change by the next time we load the driver.
-	 */
-	if (pci_dev->current_state == PCI_D0)
-		pci_dev->current_state = PCI_UNKNOWN;
-
 	/*
 	 * We would love to complain here if pci_dev->is_enabled is set, that
 	 * the driver should have called pci_disable_device(), but the
@@ -900,7 +893,7 @@ static int pci_pm_suspend_noirq(struct device *dev)
 
 	if (!pm) {
 		pci_save_state(pci_dev);
-		goto Fixup;
+		goto set_unknown;
 	}
 
 	if (pm->suspend_noirq) {
@@ -952,6 +945,7 @@ static int pci_pm_suspend_noirq(struct device *dev)
 		goto Fixup;
 	}
 
+set_unknown:
 	pci_pm_set_unknown_state(pci_dev);
 
 	/*
diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index d37860841260c..d4b5f385a6cd8 100644
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -37,8 +37,6 @@
 #define ARCH_PCI_DEV_GROUPS
 #endif
 
-static int sysfs_initialized;	/* = 0 */
-
 /* show configuration fields */
 #define pci_config_attr(field, format_string)				\
 static ssize_t								\
@@ -177,7 +175,7 @@ static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
 		max = PCI_BRIDGE_RESOURCES;
 
 	for (i = 0; i < max; i++) {
-		struct resource *res =  &pci_dev->resource[i];
+		struct resource *res = pci_resource_n(pci_dev, i);
 		struct resource zerores = {};
 
 		/* For backwards compatibility */
@@ -674,11 +672,6 @@ static const struct attribute_group pcibus_group = {
 	.attrs = pcibus_attrs,
 };
 
-const struct attribute_group *pcibus_groups[] = {
-	&pcibus_group,
-	NULL,
-};
-
 static ssize_t boot_vga_show(struct device *dev, struct device_attribute *attr,
 			     char *buf)
 {
@@ -689,7 +682,7 @@ static ssize_t boot_vga_show(struct device *dev, struct device_attribute *attr,
 		return sysfs_emit(buf, "%u\n", (pdev == vga_dev));
 
 	return sysfs_emit(buf, "%u\n",
-			  !!(pdev->resource[PCI_ROM_RESOURCE].flags &
+			  !!(pci_resource_flags(pdev, PCI_ROM_RESOURCE) &
 			     IORESOURCE_ROM_SHADOW));
 }
 static DEVICE_ATTR_RO(boot_vga);
@@ -877,19 +870,6 @@ static const struct attribute_group pci_dev_config_attr_group = {
 	.bin_size = pci_dev_config_attr_bin_size,
 };
 
-/*
- * llseek operation for mmappable PCI resources.
- * May be left unused if the arch doesn't provide them.
- */
-static __maybe_unused loff_t
-pci_llseek_resource(struct file *filep,
-		    struct kobject *kobj __always_unused,
-		    const struct bin_attribute *attr,
-		    loff_t offset, int whence)
-{
-	return fixed_size_llseek(filep, offset, whence, attr->size);
-}
-
 #ifdef HAVE_PCI_LEGACY
 /**
  * pci_read_legacy_io - read byte(s) from legacy I/O port space
@@ -981,91 +961,148 @@ static int pci_mmap_legacy_io(struct file *filp, struct kobject *kobj,
 	return pci_mmap_legacy_page_range(bus, vma, pci_mmap_io);
 }
 
-/**
- * pci_adjust_legacy_attr - adjustment of legacy file attributes
- * @b: bus to create files under
- * @mmap_type: I/O port or memory
- *
- * Stub implementation. Can be overridden by arch if necessary.
- */
-void __weak pci_adjust_legacy_attr(struct pci_bus *b,
-				   enum pci_mmap_state mmap_type)
+bool __weak pci_legacy_has_sparse(struct pci_bus *bus,
+				  enum pci_mmap_state type)
 {
+	return false;
 }
 
-/**
- * pci_create_legacy_files - create legacy I/O port and memory files
- * @b: bus to create files under
- *
- * Some platforms allow access to legacy I/O port and ISA memory space on
- * a per-bus basis.  This routine creates the files and ties them into
- * their associated read, write and mmap files from pci-sysfs.c
- *
- * On error unwind, but don't propagate the error to the caller
- * as it is ok to set up the PCI bus without these files.
- */
-void pci_create_legacy_files(struct pci_bus *b)
-{
-	int error;
-
-	if (!sysfs_initialized)
-		return;
-
-	b->legacy_io = kzalloc_objs(struct bin_attribute, 2, GFP_ATOMIC);
-	if (!b->legacy_io)
-		goto kzalloc_err;
-
-	sysfs_bin_attr_init(b->legacy_io);
-	b->legacy_io->attr.name = "legacy_io";
-	b->legacy_io->size = 0xffff;
-	b->legacy_io->attr.mode = 0600;
-	b->legacy_io->read = pci_read_legacy_io;
-	b->legacy_io->write = pci_write_legacy_io;
-	/* See pci_create_attr() for motivation */
-	b->legacy_io->llseek = pci_llseek_resource;
-	b->legacy_io->mmap = pci_mmap_legacy_io;
-	b->legacy_io->f_mapping = iomem_get_mapping;
-	pci_adjust_legacy_attr(b, pci_mmap_io);
-	error = device_create_bin_file(&b->dev, b->legacy_io);
-	if (error)
-		goto legacy_io_err;
-
-	/* Allocated above after the legacy_io struct */
-	b->legacy_mem = b->legacy_io + 1;
-	sysfs_bin_attr_init(b->legacy_mem);
-	b->legacy_mem->attr.name = "legacy_mem";
-	b->legacy_mem->size = 1024*1024;
-	b->legacy_mem->attr.mode = 0600;
-	b->legacy_mem->mmap = pci_mmap_legacy_mem;
-	/* See pci_create_attr() for motivation */
-	b->legacy_mem->llseek = pci_llseek_resource;
-	b->legacy_mem->f_mapping = iomem_get_mapping;
-	pci_adjust_legacy_attr(b, pci_mmap_mem);
-	error = device_create_bin_file(&b->dev, b->legacy_mem);
-	if (error)
-		goto legacy_mem_err;
-
-	return;
-
-legacy_mem_err:
-	device_remove_bin_file(&b->dev, b->legacy_io);
-legacy_io_err:
-	kfree(b->legacy_io);
-	b->legacy_io = NULL;
-kzalloc_err:
-	dev_warn(&b->dev, "could not create legacy I/O port and ISA memory resources in sysfs\n");
-}
-
-void pci_remove_legacy_files(struct pci_bus *b)
-{
-	if (b->legacy_io) {
-		device_remove_bin_file(&b->dev, b->legacy_io);
-		device_remove_bin_file(&b->dev, b->legacy_mem);
-		kfree(b->legacy_io); /* both are allocated here */
-	}
+static inline umode_t __pci_legacy_is_visible(struct kobject *kobj,
+					      const struct bin_attribute *a,
+					      enum pci_mmap_state type,
+					      bool sparse)
+{
+	struct pci_bus *bus = to_pci_bus(kobj_to_dev(kobj));
+
+	if (pci_legacy_has_sparse(bus, type) != sparse)
+		return 0;
+
+	return a->attr.mode;
+}
+
+static umode_t pci_legacy_io_is_visible(struct kobject *kobj,
+					const struct bin_attribute *a, int n)
+{
+	return __pci_legacy_is_visible(kobj, a, pci_mmap_io, false);
+}
+
+static umode_t pci_legacy_io_sparse_is_visible(struct kobject *kobj,
+					       const struct bin_attribute *a,
+					       int n)
+{
+	return __pci_legacy_is_visible(kobj, a, pci_mmap_io, true);
+}
+
+static umode_t pci_legacy_mem_is_visible(struct kobject *kobj,
+					 const struct bin_attribute *a, int n)
+{
+	return __pci_legacy_is_visible(kobj, a, pci_mmap_mem, false);
+}
+
+static umode_t pci_legacy_mem_sparse_is_visible(struct kobject *kobj,
+						const struct bin_attribute *a,
+						int n)
+{
+	return __pci_legacy_is_visible(kobj, a, pci_mmap_mem, true);
 }
+
+static loff_t pci_llseek_resource_legacy(struct file *filep,
+					 struct kobject *kobj __always_unused,
+					 const struct bin_attribute *attr,
+					 loff_t offset, int whence)
+{
+	return fixed_size_llseek(filep, offset, whence, attr->size);
+}
+
+static const struct bin_attribute pci_legacy_io_attr = {
+	.attr = { .name = "legacy_io", .mode = 0600 },
+	.size = PCI_LEGACY_IO_SIZE,
+	.read = pci_read_legacy_io,
+	.write = pci_write_legacy_io,
+	.mmap = pci_mmap_legacy_io,
+	.llseek = pci_llseek_resource_legacy,
+	.f_mapping = iomem_get_mapping,
+};
+
+static const struct bin_attribute pci_legacy_io_sparse_attr = {
+	.attr = { .name = "legacy_io_sparse", .mode = 0600 },
+	.size = PCI_LEGACY_IO_SIZE << 5,
+	.read = pci_read_legacy_io,
+	.write = pci_write_legacy_io,
+	.mmap = pci_mmap_legacy_io,
+	.llseek = pci_llseek_resource_legacy,
+	.f_mapping = iomem_get_mapping,
+};
+
+static const struct bin_attribute pci_legacy_mem_attr = {
+	.attr = { .name = "legacy_mem", .mode = 0600 },
+	.size = PCI_LEGACY_MEM_SIZE,
+	.mmap = pci_mmap_legacy_mem,
+	.llseek = pci_llseek_resource_legacy,
+	.f_mapping = iomem_get_mapping,
+};
+
+static const struct bin_attribute pci_legacy_mem_sparse_attr = {
+	.attr = { .name = "legacy_mem_sparse", .mode = 0600 },
+	.size = PCI_LEGACY_MEM_SIZE << 5,
+	.mmap = pci_mmap_legacy_mem,
+	.llseek = pci_llseek_resource_legacy,
+	.f_mapping = iomem_get_mapping,
+};
+
+static const struct bin_attribute *const pci_legacy_io_attrs[] = {
+	&pci_legacy_io_attr,
+	NULL,
+};
+
+static const struct bin_attribute *const pci_legacy_io_sparse_attrs[] = {
+	&pci_legacy_io_sparse_attr,
+	NULL,
+};
+
+static const struct bin_attribute *const pci_legacy_mem_attrs[] = {
+	&pci_legacy_mem_attr,
+	NULL,
+};
+
+static const struct bin_attribute *const pci_legacy_mem_sparse_attrs[] = {
+	&pci_legacy_mem_sparse_attr,
+	NULL,
+};
+
+static const struct attribute_group pci_legacy_io_group = {
+	.bin_attrs = pci_legacy_io_attrs,
+	.is_bin_visible = pci_legacy_io_is_visible,
+};
+
+static const struct attribute_group pci_legacy_io_sparse_group = {
+	.bin_attrs = pci_legacy_io_sparse_attrs,
+	.is_bin_visible = pci_legacy_io_sparse_is_visible,
+};
+
+static const struct attribute_group pci_legacy_mem_group = {
+	.bin_attrs = pci_legacy_mem_attrs,
+	.is_bin_visible = pci_legacy_mem_is_visible,
+};
+
+static const struct attribute_group pci_legacy_mem_sparse_group = {
+	.bin_attrs = pci_legacy_mem_sparse_attrs,
+	.is_bin_visible = pci_legacy_mem_sparse_is_visible,
+};
+
 #endif /* HAVE_PCI_LEGACY */
 
+const struct attribute_group *pcibus_groups[] = {
+	&pcibus_group,
+#ifdef HAVE_PCI_LEGACY
+	&pci_legacy_io_group,
+	&pci_legacy_io_sparse_group,
+	&pci_legacy_mem_group,
+	&pci_legacy_mem_sparse_group,
+#endif
+	NULL,
+};
+
 #if defined(HAVE_PCI_MMAP) || defined(ARCH_GENERIC_PCI_MMAP_RESOURCE)
 /**
  * pci_mmap_resource - map a PCI resource into user memory space
@@ -1082,20 +1119,24 @@ static int pci_mmap_resource(struct kobject *kobj, const struct bin_attribute *a
 	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
 	int bar = (unsigned long)attr->private;
 	enum pci_mmap_state mmap_type;
-	struct resource *res = &pdev->resource[bar];
 	int ret;
 
 	ret = security_locked_down(LOCKDOWN_PCI_ACCESS);
 	if (ret)
 		return ret;
 
-	if (res->flags & IORESOURCE_MEM && iomem_is_exclusive(res->start))
+	if (!pci_resource_is_mem(pdev, bar) &&
+	    !(pci_resource_is_io(pdev, bar) && arch_can_pci_mmap_io()))
+		return -EIO;
+
+	if (pci_resource_is_mem(pdev, bar) &&
+	    iomem_is_exclusive(pci_resource_start(pdev, bar)))
 		return -EINVAL;
 
 	if (!pci_mmap_fits(pdev, bar, vma, PCI_MMAP_SYSFS))
 		return -EINVAL;
 
-	mmap_type = res->flags & IORESOURCE_MEM ? pci_mmap_mem : pci_mmap_io;
+	mmap_type = pci_resource_is_mem(pdev, bar) ? pci_mmap_mem : pci_mmap_io;
 
 	return pci_mmap_resource_range(pdev, bar, vma, mmap_type, write_combine);
 }
@@ -1123,6 +1164,9 @@ static ssize_t pci_resource_io(struct file *filp, struct kobject *kobj,
 	int bar = (unsigned long)attr->private;
 	unsigned long port = off;
 
+	if (!pci_resource_is_io(pdev, bar))
+		return -EIO;
+
 	port += pci_resource_start(pdev, bar);
 
 	if (port > pci_resource_end(pdev, bar))
@@ -1157,14 +1201,14 @@ static ssize_t pci_resource_io(struct file *filp, struct kobject *kobj,
 #endif
 }
 
-static ssize_t pci_read_resource_io(struct file *filp, struct kobject *kobj,
+static ssize_t pci_read_resource(struct file *filp, struct kobject *kobj,
 				    const struct bin_attribute *attr, char *buf,
 				    loff_t off, size_t count)
 {
 	return pci_resource_io(filp, kobj, attr, buf, off, count, false);
 }
 
-static ssize_t pci_write_resource_io(struct file *filp, struct kobject *kobj,
+static ssize_t pci_write_resource(struct file *filp, struct kobject *kobj,
 				     const struct bin_attribute *attr, char *buf,
 				     loff_t off, size_t count)
 {
@@ -1177,127 +1221,192 @@ static ssize_t pci_write_resource_io(struct file *filp, struct kobject *kobj,
 	return pci_resource_io(filp, kobj, attr, buf, off, count, true);
 }
 
-/**
- * pci_remove_resource_files - cleanup resource files
- * @pdev: dev to cleanup
- *
- * If we created resource files for @pdev, remove them from sysfs and
- * free their resources.
- */
-static void pci_remove_resource_files(struct pci_dev *pdev)
+static loff_t pci_llseek_resource(struct file *filep,
+				  struct kobject *kobj,
+				  const struct bin_attribute *attr,
+				  loff_t offset, int whence)
 {
-	int i;
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
+	int bar = (unsigned long)attr->private;
+
+	return fixed_size_llseek(filep, offset, whence,
+				 pci_resource_len(pdev, bar));
+}
 
-	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
-		struct bin_attribute *res_attr;
+/*
+ * generic_file_llseek() consults f_mapping->host to determine
+ * the file size. As iomem_inode knows nothing about the
+ * attribute, it's not going to work, so override it as well.
+ */
+#if arch_can_pci_mmap_io()
+# define __PCI_RESOURCE_IO_MMAP_ATTRS		\
+	.f_mapping = iomem_get_mapping,		\
+	.llseek = pci_llseek_resource,		\
+	.mmap = pci_mmap_resource_uc,
+#else
+# define __PCI_RESOURCE_IO_MMAP_ATTRS
+#endif
 
-		res_attr = pdev->res_attr[i];
-		if (res_attr) {
-			sysfs_remove_bin_file(&pdev->dev.kobj, res_attr);
-			kfree(res_attr);
-		}
+#define pci_dev_resource_io_attr(_bar)					\
+static const struct bin_attribute dev_resource##_bar##_io_attr = {	\
+	.attr = { .name = "resource" __stringify(_bar), .mode = 0600 },	\
+	.private = (void *)(unsigned long)(_bar),			\
+	.read = pci_read_resource,					\
+	.write = pci_write_resource,					\
+	__PCI_RESOURCE_IO_MMAP_ATTRS					\
+}
 
-		res_attr = pdev->res_attr_wc[i];
-		if (res_attr) {
-			sysfs_remove_bin_file(&pdev->dev.kobj, res_attr);
-			kfree(res_attr);
-		}
-	}
+#define pci_dev_resource_uc_attr(_bar)					\
+static const struct bin_attribute dev_resource##_bar##_uc_attr = {	\
+	.attr = { .name = "resource" __stringify(_bar), .mode = 0600 },	\
+	.private = (void *)(unsigned long)(_bar),			\
+	.f_mapping = iomem_get_mapping,					\
+	.llseek = pci_llseek_resource,					\
+	.mmap = pci_mmap_resource_uc,					\
 }
 
-static int pci_create_attr(struct pci_dev *pdev, int num, int write_combine)
+#define pci_dev_resource_wc_attr(_bar)					      \
+static const struct bin_attribute dev_resource##_bar##_wc_attr = {	      \
+	.attr = { .name = "resource" __stringify(_bar) "_wc", .mode = 0600 }, \
+	.private = (void *)(unsigned long)(_bar),			      \
+	.f_mapping = iomem_get_mapping,					      \
+	.llseek = pci_llseek_resource,					      \
+	.mmap = pci_mmap_resource_wc,					      \
+}
+
+static inline umode_t
+__pci_resource_attr_is_visible(struct kobject *kobj,
+			       const struct bin_attribute *a,
+			       int bar, bool write_combine,
+			       unsigned long flags)
 {
-	/* allocate attribute structure, piggyback attribute name */
-	int name_len = write_combine ? 13 : 10;
-	struct bin_attribute *res_attr;
-	char *res_attr_name;
-	int retval;
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
 
-	res_attr = kzalloc(sizeof(*res_attr) + name_len, GFP_ATOMIC);
-	if (!res_attr)
-		return -ENOMEM;
+	if (pdev->non_mappable_bars)
+		return 0;
 
-	res_attr_name = (char *)(res_attr + 1);
+	if (!pci_resource_len(pdev, bar))
+		return 0;
 
-	sysfs_bin_attr_init(res_attr);
-	if (write_combine) {
-		sprintf(res_attr_name, "resource%d_wc", num);
-		res_attr->mmap = pci_mmap_resource_wc;
-	} else {
-		sprintf(res_attr_name, "resource%d", num);
-		if (pci_resource_flags(pdev, num) & IORESOURCE_IO) {
-			res_attr->read = pci_read_resource_io;
-			res_attr->write = pci_write_resource_io;
-			if (arch_can_pci_mmap_io())
-				res_attr->mmap = pci_mmap_resource_uc;
-		} else {
-			res_attr->mmap = pci_mmap_resource_uc;
-		}
-	}
-	if (res_attr->mmap) {
-		res_attr->f_mapping = iomem_get_mapping;
-		/*
-		 * generic_file_llseek() consults f_mapping->host to determine
-		 * the file size. As iomem_inode knows nothing about the
-		 * attribute, it's not going to work, so override it as well.
-		 */
-		res_attr->llseek = pci_llseek_resource;
-	}
-	res_attr->attr.name = res_attr_name;
-	res_attr->attr.mode = 0600;
-	res_attr->size = pci_resource_len(pdev, num);
-	res_attr->private = (void *)(unsigned long)num;
-	retval = sysfs_create_bin_file(&pdev->dev.kobj, res_attr);
-	if (retval) {
-		kfree(res_attr);
-		return retval;
-	}
+	if ((pci_resource_flags(pdev, bar) & flags) != flags)
+		return 0;
 
-	if (write_combine)
-		pdev->res_attr_wc[num] = res_attr;
-	else
-		pdev->res_attr[num] = res_attr;
+	if (write_combine && !arch_can_pci_mmap_wc())
+		return 0;
 
-	return 0;
+	return a->attr.mode;
 }
 
-/**
- * pci_create_resource_files - create resource files in sysfs for @dev
- * @pdev: dev in question
- *
- * Walk the resources in @pdev creating files for each resource available.
- */
-static int pci_create_resource_files(struct pci_dev *pdev)
+static umode_t pci_dev_resource_io_is_visible(struct kobject *kobj,
+					      const struct bin_attribute *a,
+					      int n)
 {
-	int i;
-	int retval;
+	return __pci_resource_attr_is_visible(kobj, a, n, false,
+					      IORESOURCE_IO);
+}
 
-	/* Skip devices with non-mappable BARs */
-	if (pdev->non_mappable_bars)
-		return 0;
+static umode_t pci_dev_resource_uc_is_visible(struct kobject *kobj,
+					      const struct bin_attribute *a,
+					      int n)
+{
+	return __pci_resource_attr_is_visible(kobj, a, n, false,
+					      IORESOURCE_MEM);
+}
 
-	/* Expose the PCI resources from this device as files */
-	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
+static umode_t pci_dev_resource_wc_is_visible(struct kobject *kobj,
+					      const struct bin_attribute *a,
+					      int n)
+{
+	return __pci_resource_attr_is_visible(kobj, a, n, true,
+					      IORESOURCE_MEM | IORESOURCE_PREFETCH);
+}
 
-		/* skip empty resources */
-		if (!pci_resource_len(pdev, i))
-			continue;
+static size_t pci_dev_resource_bin_size(struct kobject *kobj,
+					const struct bin_attribute *a,
+					int n)
+{
+	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
 
-		retval = pci_create_attr(pdev, i, 0);
-		/* for prefetchable resources, create a WC mappable file */
-		if (!retval && arch_can_pci_mmap_wc() &&
-		    pdev->resource[i].flags & IORESOURCE_PREFETCH)
-			retval = pci_create_attr(pdev, i, 1);
-		if (retval) {
-			pci_remove_resource_files(pdev);
-			return retval;
-		}
-	}
-	return 0;
-}
-#else /* !(defined(HAVE_PCI_MMAP) || defined(ARCH_GENERIC_PCI_MMAP_RESOURCE)) */
-int __weak pci_create_resource_files(struct pci_dev *dev) { return 0; }
-void __weak pci_remove_resource_files(struct pci_dev *dev) { return; }
+	return pci_resource_len(pdev, n);
+}
+
+pci_dev_resource_io_attr(0);
+pci_dev_resource_io_attr(1);
+pci_dev_resource_io_attr(2);
+pci_dev_resource_io_attr(3);
+pci_dev_resource_io_attr(4);
+pci_dev_resource_io_attr(5);
+
+pci_dev_resource_uc_attr(0);
+pci_dev_resource_uc_attr(1);
+pci_dev_resource_uc_attr(2);
+pci_dev_resource_uc_attr(3);
+pci_dev_resource_uc_attr(4);
+pci_dev_resource_uc_attr(5);
+
+pci_dev_resource_wc_attr(0);
+pci_dev_resource_wc_attr(1);
+pci_dev_resource_wc_attr(2);
+pci_dev_resource_wc_attr(3);
+pci_dev_resource_wc_attr(4);
+pci_dev_resource_wc_attr(5);
+
+static const struct bin_attribute *const pci_dev_resource_io_attrs[] = {
+	&dev_resource0_io_attr,
+	&dev_resource1_io_attr,
+	&dev_resource2_io_attr,
+	&dev_resource3_io_attr,
+	&dev_resource4_io_attr,
+	&dev_resource5_io_attr,
+	NULL,
+};
+
+static const struct bin_attribute *const pci_dev_resource_uc_attrs[] = {
+	&dev_resource0_uc_attr,
+	&dev_resource1_uc_attr,
+	&dev_resource2_uc_attr,
+	&dev_resource3_uc_attr,
+	&dev_resource4_uc_attr,
+	&dev_resource5_uc_attr,
+	NULL,
+};
+
+static const struct bin_attribute *const pci_dev_resource_wc_attrs[] = {
+	&dev_resource0_wc_attr,
+	&dev_resource1_wc_attr,
+	&dev_resource2_wc_attr,
+	&dev_resource3_wc_attr,
+	&dev_resource4_wc_attr,
+	&dev_resource5_wc_attr,
+	NULL,
+};
+
+static const struct attribute_group pci_dev_resource_io_attr_group = {
+	.bin_attrs = pci_dev_resource_io_attrs,
+	.is_bin_visible = pci_dev_resource_io_is_visible,
+	.bin_size = pci_dev_resource_bin_size,
+};
+
+static const struct attribute_group pci_dev_resource_uc_attr_group = {
+	.bin_attrs = pci_dev_resource_uc_attrs,
+	.is_bin_visible = pci_dev_resource_uc_is_visible,
+	.bin_size = pci_dev_resource_bin_size,
+};
+
+static const struct attribute_group pci_dev_resource_wc_attr_group = {
+	.bin_attrs = pci_dev_resource_wc_attrs,
+	.is_bin_visible = pci_dev_resource_wc_is_visible,
+	.bin_size = pci_dev_resource_bin_size,
+};
+
+static const struct attribute_group *pci_dev_resource_attr_groups[] = {
+	&pci_dev_resource_io_attr_group,
+	&pci_dev_resource_uc_attr_group,
+	&pci_dev_resource_wc_attr_group,
+	NULL,
+};
+#else
+#define pci_dev_resource_attr_groups NULL
 #endif
 
 /**
@@ -1552,6 +1661,7 @@ static const struct attribute_group pci_dev_reset_method_attr_group = {
 	.is_visible = pci_dev_reset_attr_is_visible,
 };
 
+#if defined(HAVE_PCI_MMAP) || defined(ARCH_GENERIC_PCI_MMAP_RESOURCE)
 static ssize_t __resource_resize_show(struct device *dev, int n, char *buf)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
@@ -1576,6 +1686,9 @@ static ssize_t __resource_resize_store(struct device *dev, int n,
 	int ret;
 	u16 cmd;
 
+	if (!capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
 	if (kstrtoul(buf, 0, &size) < 0)
 		return -EINVAL;
 
@@ -1598,14 +1711,17 @@ static ssize_t __resource_resize_store(struct device *dev, int n,
 	pci_write_config_word(pdev, PCI_COMMAND,
 			      cmd & ~PCI_COMMAND_MEMORY);
 
-	pci_remove_resource_files(pdev);
+	sysfs_remove_groups(&pdev->dev.kobj, pci_dev_resource_attr_groups);
 
 	ret = pci_resize_resource(pdev, n, size, 0);
+	if (ret)
+		pci_warn(pdev, "Failed to resize BAR %d: %pe\n",
+			 n, ERR_PTR(ret));
 
 	pci_assign_unassigned_bus_resources(bus);
 
-	if (pci_create_resource_files(pdev))
-		pci_warn(pdev, "Failed to recreate resource files after BAR resizing\n");
+	if (sysfs_create_groups(&pdev->dev.kobj, pci_dev_resource_attr_groups))
+		pci_warn(pdev, "Failed to recreate resource groups after BAR resizing\n");
 
 	pci_write_config_word(pdev, PCI_COMMAND, cmd);
 pm_put:
@@ -1648,7 +1764,7 @@ static struct attribute *resource_resize_attrs[] = {
 	NULL,
 };
 
-static umode_t resource_resize_is_visible(struct kobject *kobj,
+static umode_t resource_resize_attr_is_visible(struct kobject *kobj,
 					  struct attribute *a, int n)
 {
 	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
@@ -1656,54 +1772,11 @@ static umode_t resource_resize_is_visible(struct kobject *kobj,
 	return pci_rebar_get_current_size(pdev, n) < 0 ? 0 : a->mode;
 }
 
-static const struct attribute_group pci_dev_resource_resize_group = {
+static const struct attribute_group pci_dev_resource_resize_attr_group = {
 	.attrs = resource_resize_attrs,
-	.is_visible = resource_resize_is_visible,
+	.is_visible = resource_resize_attr_is_visible,
 };
-
-int __must_check pci_create_sysfs_dev_files(struct pci_dev *pdev)
-{
-	if (!sysfs_initialized)
-		return -EACCES;
-
-	return pci_create_resource_files(pdev);
-}
-
-/**
- * pci_remove_sysfs_dev_files - cleanup PCI specific sysfs files
- * @pdev: device whose entries we should free
- *
- * Cleanup when @pdev is removed from sysfs.
- */
-void pci_remove_sysfs_dev_files(struct pci_dev *pdev)
-{
-	if (!sysfs_initialized)
-		return;
-
-	pci_remove_resource_files(pdev);
-}
-
-static int __init pci_sysfs_init(void)
-{
-	struct pci_dev *pdev = NULL;
-	struct pci_bus *pbus = NULL;
-	int retval;
-
-	sysfs_initialized = 1;
-	for_each_pci_dev(pdev) {
-		retval = pci_create_sysfs_dev_files(pdev);
-		if (retval) {
-			pci_dev_put(pdev);
-			return retval;
-		}
-	}
-
-	while ((pbus = pci_find_next_bus(pbus)))
-		pci_create_legacy_files(pbus);
-
-	return 0;
-}
-late_initcall(pci_sysfs_init);
+#endif
 
 static struct attribute *pci_dev_dev_attrs[] = {
 	&dev_attr_boot_vga.attr,
@@ -1774,6 +1847,12 @@ static const struct attribute_group pci_dev_group = {
 
 const struct attribute_group *pci_dev_groups[] = {
 	&pci_dev_group,
+#if defined(HAVE_PCI_MMAP) || defined(ARCH_GENERIC_PCI_MMAP_RESOURCE)
+	&pci_dev_resource_io_attr_group,
+	&pci_dev_resource_uc_attr_group,
+	&pci_dev_resource_wc_attr_group,
+	&pci_dev_resource_resize_attr_group,
+#endif
 	&pci_dev_config_attr_group,
 	&pci_dev_rom_attr_group,
 	&pci_dev_reset_attr_group,
@@ -1785,7 +1864,6 @@ const struct attribute_group *pci_dev_groups[] = {
 #ifdef CONFIG_ACPI
 	&pci_dev_acpi_attr_group,
 #endif
-	&pci_dev_resource_resize_group,
 	ARCH_PCI_DEV_GROUPS
 	NULL,
 };
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index d34266651ad09..e305dbec39a50 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -120,17 +120,7 @@ unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
 
 
 /* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */
-#ifdef CONFIG_PCIE_BUS_TUNE_OFF
-enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF;
-#elif defined CONFIG_PCIE_BUS_SAFE
-enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE;
-#elif defined CONFIG_PCIE_BUS_PERFORMANCE
-enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE;
-#elif defined CONFIG_PCIE_BUS_PEER2PEER
-enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER;
-#else
 enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
-#endif
 
 /*
  * The default CLS is used if arch didn't set CLS explicitly and not
@@ -1253,8 +1243,8 @@ static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout)
 		}
 
 		if (delay > timeout) {
-			pci_warn(dev, "not ready %dms after %s; giving up\n",
-				 delay - 1, reset_type);
+			pci_err(dev, "not ready %dms after %s; giving up\n",
+				delay - 1, reset_type);
 			return -ENOTTY;
 		}
 
@@ -1300,7 +1290,18 @@ int pci_power_up(struct pci_dev *dev)
 	bool need_restore;
 	pci_power_t state;
 	u16 pmcsr;
+	int ret;
 
+	/*
+	 * When setting power state to D0, platform_pci_set_power_state()
+	 * ensures main power is on.  If it puts the device in D0, it also
+	 * completes any required delays after the transition; if it leaves
+	 * the device in D1, D2, or D3hot, we use the PM Capability to
+	 * transition to D0.
+	 *
+	 * In all cases, the device is either Configuration-Ready or
+	 * inaccessible upon return.
+	 */
 	platform_pci_set_power_state(dev, PCI_D0);
 
 	if (!dev->pm_cap) {
@@ -1341,10 +1342,19 @@ int pci_power_up(struct pci_dev *dev)
 	pci_write_config_word(dev, dev->pm_cap + PCI_PM_CTRL, 0);
 
 	/* Mandatory transition delays; see PCI PM 1.2. */
-	if (state == PCI_D3hot)
+	if (state == PCI_D3hot) {
 		pci_dev_d3_sleep(dev);
-	else if (state == PCI_D2)
+		if (!(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET)) {
+			ret = pci_dev_wait(dev, "power up D3hot->D0uninitialized",
+					   PCIE_RESET_READY_POLL_MS);
+			if (ret) {
+				dev->current_state = PCI_D3cold;
+				return -EIO;
+			}
+		}
+	} else if (state == PCI_D2) {
 		udelay(PCI_PM_D2_DELAY);
+	}
 
 end:
 	dev->current_state = PCI_D0;
@@ -1514,9 +1524,6 @@ static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool
 				     pci_power_name(dev->current_state),
 				     pci_power_name(state));
 
-	if (dev->bus->self)
-		pcie_aspm_pm_state_change(dev->bus->self, locked);
-
 	return 0;
 }
 
@@ -1764,7 +1771,7 @@ int pci_save_state(struct pci_dev *dev)
 EXPORT_SYMBOL(pci_save_state);
 
 static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
-				     u32 saved_val, int retry, bool force)
+				     u32 saved_val, bool force)
 {
 	u32 val;
 
@@ -1772,52 +1779,42 @@ static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
 	if (!force && val == saved_val)
 		return;
 
-	for (;;) {
-		pci_dbg(pdev, "restore config %#04x: %#010x -> %#010x\n",
-			offset, val, saved_val);
-		pci_write_config_dword(pdev, offset, saved_val);
-		if (retry-- <= 0)
-			return;
+	pci_dbg(pdev, "restore config %#04x: %#010x -> %#010x\n", offset, val,
+		saved_val);
 
-		pci_read_config_dword(pdev, offset, &val);
-		if (val == saved_val)
-			return;
-
-		mdelay(1);
-	}
+	pci_write_config_dword(pdev, offset, saved_val);
 }
 
 static void pci_restore_config_space_range(struct pci_dev *pdev,
-					   int start, int end, int retry,
-					   bool force)
+					   int start, int end, bool force)
 {
 	int index;
 
 	for (index = end; index >= start; index--)
 		pci_restore_config_dword(pdev, 4 * index,
 					 pdev->saved_config_space[index],
-					 retry, force);
+					 force);
 }
 
 static void pci_restore_config_space(struct pci_dev *pdev)
 {
 	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
-		pci_restore_config_space_range(pdev, 10, 15, 0, false);
+		pci_restore_config_space_range(pdev, 10, 15, false);
 		/* Restore BARs before the command register. */
-		pci_restore_config_space_range(pdev, 4, 9, 10, false);
-		pci_restore_config_space_range(pdev, 0, 3, 0, false);
+		pci_restore_config_space_range(pdev, 4, 9, false);
+		pci_restore_config_space_range(pdev, 0, 3, false);
 	} else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
-		pci_restore_config_space_range(pdev, 12, 15, 0, false);
+		pci_restore_config_space_range(pdev, 12, 15, false);
 
 		/*
 		 * Force rewriting of prefetch registers to avoid S3 resume
 		 * issues on Intel PCI bridges that occur when these
 		 * registers are not explicitly written.
 		 */
-		pci_restore_config_space_range(pdev, 9, 11, 0, true);
-		pci_restore_config_space_range(pdev, 0, 8, 0, false);
+		pci_restore_config_space_range(pdev, 9, 11, true);
+		pci_restore_config_space_range(pdev, 0, 8, false);
 	} else {
-		pci_restore_config_space_range(pdev, 0, 15, 0, false);
+		pci_restore_config_space_range(pdev, 0, 15, false);
 	}
 }
 
@@ -5770,8 +5767,7 @@ int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
 		if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
 			return -EIO;
 
-		cmd &= ~PCI_X_CMD_MAX_READ;
-		cmd |= FIELD_PREP(PCI_X_CMD_MAX_READ, v);
+		FIELD_MODIFY(PCI_X_CMD_MAX_READ, &cmd, v);
 		if (pci_write_config_word(dev, cap + PCI_X_CMD, cmd))
 			return -EIO;
 	}
diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h
index 4a14f88e543a2..b36667969ad5e 100644
--- a/drivers/pci/pci.h
+++ b/drivers/pci/pci.h
@@ -358,14 +358,6 @@ static inline int pci_proc_detach_bus(struct pci_bus *bus) { return 0; }
 int pci_hp_add_bridge(struct pci_dev *dev);
 bool pci_hp_spurious_link_change(struct pci_dev *pdev);
 
-#if defined(CONFIG_SYSFS) && defined(HAVE_PCI_LEGACY)
-void pci_create_legacy_files(struct pci_bus *bus);
-void pci_remove_legacy_files(struct pci_bus *bus);
-#else
-static inline void pci_create_legacy_files(struct pci_bus *bus) { }
-static inline void pci_remove_legacy_files(struct pci_bus *bus) { }
-#endif
-
 /* Lock for read/write access to pci device and bus lists */
 extern struct rw_semaphore pci_bus_sem;
 extern struct mutex pci_slot_mutex;
@@ -392,17 +384,17 @@ static inline int pci_no_d1d2(struct pci_dev *dev)
 
 }
 
+#ifdef HAVE_PCI_LEGACY
+bool pci_legacy_has_sparse(struct pci_bus *bus, enum pci_mmap_state type);
+#endif
+
 #ifdef CONFIG_SYSFS
-int pci_create_sysfs_dev_files(struct pci_dev *pdev);
-void pci_remove_sysfs_dev_files(struct pci_dev *pdev);
 extern const struct attribute_group *pci_dev_groups[];
 extern const struct attribute_group *pci_dev_attr_groups[];
 extern const struct attribute_group *pcibus_groups[];
 extern const struct attribute_group *pci_bus_groups[];
 extern const struct attribute_group pci_doe_sysfs_group;
 #else
-static inline int pci_create_sysfs_dev_files(struct pci_dev *pdev) { return 0; }
-static inline void pci_remove_sysfs_dev_files(struct pci_dev *pdev) { }
 #define pci_dev_groups NULL
 #define pci_dev_attr_groups NULL
 #define pcibus_groups NULL
@@ -419,7 +411,7 @@ static inline bool pci_is_cardbus_bridge(struct pci_dev *dev)
 	return dev->hdr_type == PCI_HEADER_TYPE_CARDBUS;
 }
 #ifdef CONFIG_CARDBUS
-unsigned long pci_cardbus_resource_alignment(struct resource *res);
+unsigned long pci_cardbus_resource_alignment(const struct resource *res);
 int pci_bus_size_cardbus_bridge(struct pci_bus *bus,
 				struct list_head *realloc_head);
 int pci_cardbus_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
@@ -428,7 +420,7 @@ int pci_cardbus_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
 int pci_setup_cardbus(char *str);
 
 #else
-static inline unsigned long pci_cardbus_resource_alignment(struct resource *res)
+static inline unsigned long pci_cardbus_resource_alignment(const struct resource *res)
 {
 	return 0;
 }
@@ -515,7 +507,7 @@ int pci_dev_res_add_to_list(struct list_head *head, struct pci_dev *dev,
 void __pci_bus_size_bridges(struct pci_bus *bus,
 			struct list_head *realloc_head);
 void __pci_bus_assign_resources(const struct pci_bus *bus,
-				struct list_head *realloc_head,
+				struct list_head *add_list,
 				struct list_head *fail_head);
 bool pci_bus_clip_resource(struct pci_dev *dev, int idx);
 void pci_walk_bus_locked(struct pci_bus *top,
@@ -947,7 +939,8 @@ int pci_iov_init(struct pci_dev *dev);
 void pci_iov_release(struct pci_dev *dev);
 void pci_iov_remove(struct pci_dev *dev);
 void pci_iov_update_resource(struct pci_dev *dev, int resno);
-resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno);
+resource_size_t pci_sriov_resource_alignment(const struct pci_dev *dev,
+					     int resno);
 void pci_restore_iov_state(struct pci_dev *dev);
 int pci_iov_bus_range(struct pci_bus *bus);
 void pci_iov_resource_set_size(struct pci_dev *dev, int resno, int size);
@@ -981,7 +974,7 @@ static inline int pci_iov_init(struct pci_dev *dev)
 static inline void pci_iov_release(struct pci_dev *dev) { }
 static inline void pci_iov_remove(struct pci_dev *dev) { }
 static inline void pci_iov_update_resource(struct pci_dev *dev, int resno) { }
-static inline resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev,
+static inline resource_size_t pci_sriov_resource_alignment(const struct pci_dev *dev,
 							   int resno)
 {
 	return 0;
@@ -1043,17 +1036,8 @@ static inline void pci_suspend_ptm(struct pci_dev *dev) { }
 static inline void pci_resume_ptm(struct pci_dev *dev) { }
 #endif
 
-static inline resource_size_t pci_resource_alignment(struct pci_dev *dev,
-						     struct resource *res)
-{
-	int resno = pci_resource_num(dev, res);
-
-	if (pci_resource_is_iov(resno))
-		return pci_sriov_resource_alignment(dev, resno);
-	if (dev->class >> 8 == PCI_CLASS_BRIDGE_CARDBUS)
-		return pci_cardbus_resource_alignment(res);
-	return resource_alignment(res);
-}
+resource_size_t pci_resource_alignment(const struct pci_dev *dev,
+				       const struct resource *res);
 
 resource_size_t pci_min_window_alignment(struct pci_bus *bus,
 					 unsigned long type);
@@ -1110,6 +1094,7 @@ void pcie_aspm_pm_state_change(struct pci_dev *pdev, bool locked);
 void pcie_aspm_powersave_config_link(struct pci_dev *pdev);
 void pci_configure_ltr(struct pci_dev *pdev);
 void pci_bridge_reconfigure_ltr(struct pci_dev *pdev);
+void pcie_encode_t_power_on(u32 t_power_on_us, u8 *scale, u8 *value);
 #else
 static inline void pcie_aspm_remove_cap(struct pci_dev *pdev, u32 lnkcap) { }
 static inline void pcie_aspm_init_link_state(struct pci_dev *pdev) { }
@@ -1118,6 +1103,11 @@ static inline void pcie_aspm_pm_state_change(struct pci_dev *pdev, bool locked)
 static inline void pcie_aspm_powersave_config_link(struct pci_dev *pdev) { }
 static inline void pci_configure_ltr(struct pci_dev *pdev) { }
 static inline void pci_bridge_reconfigure_ltr(struct pci_dev *pdev) { }
+static inline void pcie_encode_t_power_on(u32 t_power_on_us, u8 *scale, u8 *value)
+{
+	*scale = 0;
+	*value = 0;
+}
 #endif
 
 #ifdef CONFIG_PCIE_ECRC
diff --git a/drivers/pci/pcie/aspm.c b/drivers/pci/pcie/aspm.c
index 925373b98dff0..172783e7f519a 100644
--- a/drivers/pci/pcie/aspm.c
+++ b/drivers/pci/pcie/aspm.c
@@ -525,6 +525,46 @@ static u32 calc_l12_pwron(struct pci_dev *pdev, u32 scale, u32 val)
 	return 0;
 }
 
+/**
+ * pcie_encode_t_power_on - Encode T_POWER_ON into scale and value fields
+ * @t_power_on_us: T_POWER_ON time in microseconds
+ * @scale: Encoded T_POWER_ON Scale (0..2)
+ * @value: Encoded T_POWER_ON Value
+ *
+ * T_POWER_ON is encoded as:
+ *   T_POWER_ON(us) = scale_unit(us) * value
+ *
+ * where scale_unit is selected by @scale:
+ *   0: 2us
+ *   1: 10us
+ *   2: 100us
+ *
+ * If @t_power_on_us exceeds the maximum representable value, the result
+ * is clamped to the largest encodable T_POWER_ON.
+ *
+ * See PCIe r7.0, sec 7.8.3.2.
+ */
+void pcie_encode_t_power_on(u32 t_power_on_us, u8 *scale, u8 *value)
+{
+	u8 maxv = FIELD_MAX(PCI_L1SS_CAP_P_PWR_ON_VALUE);
+
+	/* T_POWER_ON_Value ("value") is a 5-bit field with max value of 31. */
+	if (t_power_on_us <= 2 * maxv) {
+		*scale = 0; /* Value times 2us */
+		*value = DIV_ROUND_UP(t_power_on_us, 2);
+	} else if (t_power_on_us <= 10 * maxv) {
+		*scale = 1; /* Value times 10us */
+		*value = DIV_ROUND_UP(t_power_on_us, 10);
+	} else if (t_power_on_us <= 100 * maxv) {
+		*scale = 2; /* value times 100us */
+		*value = DIV_ROUND_UP(t_power_on_us, 100);
+	} else {
+		*scale = 2;
+		*value = maxv;
+	}
+}
+EXPORT_SYMBOL(pcie_encode_t_power_on);
+
 /*
  * Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1
  * register.  Ports enter L1.2 when the most recent LTR value is greater
diff --git a/drivers/pci/pcie/ptm.c b/drivers/pci/pcie/ptm.c
index a41ffd1914de4..bd3bd39f63728 100644
--- a/drivers/pci/pcie/ptm.c
+++ b/drivers/pci/pcie/ptm.c
@@ -152,8 +152,7 @@ static int __pci_enable_ptm(struct pci_dev *dev)
 	pci_read_config_dword(dev, ptm + PCI_PTM_CTRL, &ctrl);
 
 	ctrl |= PCI_PTM_CTRL_ENABLE;
-	ctrl &= ~PCI_PTM_GRANULARITY_MASK;
-	ctrl |= FIELD_PREP(PCI_PTM_GRANULARITY_MASK, dev->ptm_granularity);
+	FIELD_MODIFY(PCI_PTM_GRANULARITY_MASK, &ctrl, dev->ptm_granularity);
 	if (dev->ptm_root)
 		ctrl |= PCI_PTM_CTRL_ROOT;
 
diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c
index b63cd0c310bc0..dd0abbc63e18d 100644
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -660,6 +660,7 @@ static void pci_init_host_bridge(struct pci_host_bridge *bridge)
 {
 	INIT_LIST_HEAD(&bridge->windows);
 	INIT_LIST_HEAD(&bridge->dma_ranges);
+	INIT_LIST_HEAD(&bridge->ports);
 
 	/*
 	 * We assume we can manage these PCIe features.  Some systems may
@@ -1073,9 +1074,6 @@ static int pci_register_host_bridge(struct pci_host_bridge *bridge)
 			dev_err(&bus->dev, "failed to add bus: %d\n", err);
 	}
 
-	/* Create legacy_io and legacy_mem files for this bus */
-	pci_create_legacy_files(bus);
-
 	if (parent)
 		dev_info(parent, "PCI host bridge to bus %s\n", name);
 	else
@@ -1281,9 +1279,6 @@ static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
 			dev_err(&child->dev, "failed to add bus: %d\n", ret);
 	}
 
-	/* Create legacy_io and legacy_mem files for this bus */
-	pci_create_legacy_files(child);
-
 	return child;
 }
 
diff --git a/drivers/pci/pwrctrl/core.c b/drivers/pci/pwrctrl/core.c
index 97cff5b8ca885..10d817aefb096 100644
--- a/drivers/pci/pwrctrl/core.c
+++ b/drivers/pci/pwrctrl/core.c
@@ -39,7 +39,7 @@ static int pci_pwrctrl_notify(struct notifier_block *nb, unsigned long action,
 		 * If we got here then the PCI device is the second after the
 		 * power control platform device. Mark its OF node as reused.
 		 */
-		dev->of_node_reused = true;
+		dev_set_of_node_reused(dev);
 		break;
 	}
 
@@ -139,6 +139,48 @@ int devm_pci_pwrctrl_device_set_ready(struct device *dev,
 }
 EXPORT_SYMBOL_GPL(devm_pci_pwrctrl_device_set_ready);
 
+/*
+ * Check whether the pwrctrl device really needs to be created or not. The
+ * pwrctrl device will only be created if the node satisfies below requirements:
+ *
+ * 1. Presence of compatible property with "pci" prefix to match against the
+ *    pwrctrl driver (AND)
+ * 2. At least one of the power supplies defined in the devicetree node of the
+ *    device (OR) in the remote endpoint parent node to indicate pwrctrl
+ *    requirement.
+ */
+static bool pci_pwrctrl_is_required(struct device_node *np)
+{
+	struct device_node *endpoint;
+	const char *compat;
+	int ret;
+
+	ret = of_property_read_string(np, "compatible", &compat);
+	if (ret < 0)
+		return false;
+
+	if (!strstarts(compat, "pci"))
+		return false;
+
+	if (of_pci_supply_present(np))
+		return true;
+
+	if (of_graph_is_present(np)) {
+		for_each_endpoint_of_node(np, endpoint) {
+			struct device_node *remote __free(device_node) =
+				of_graph_get_remote_port_parent(endpoint);
+			if (remote) {
+				if (of_pci_supply_present(remote)) {
+					of_node_put(endpoint);
+					return true;
+				}
+			}
+		}
+	}
+
+	return false;
+}
+
 static int __pci_pwrctrl_power_off_device(struct device *dev)
 {
 	struct pci_pwrctrl *pwrctrl = dev_get_drvdata(dev);
@@ -157,14 +199,19 @@ static void pci_pwrctrl_power_off_device(struct device_node *np)
 	for_each_available_child_of_node_scoped(np, child)
 		pci_pwrctrl_power_off_device(child);
 
+	if (!pci_pwrctrl_is_required(np))
+		return;
+
 	pdev = of_find_device_by_node(np);
 	if (!pdev)
 		return;
 
-	if (device_is_bound(&pdev->dev)) {
-		ret = __pci_pwrctrl_power_off_device(&pdev->dev);
-		if (ret)
-			dev_err(&pdev->dev, "Failed to power off device: %d", ret);
+	scoped_guard(device, &pdev->dev) {
+		if (device_is_bound(&pdev->dev)) {
+			ret = __pci_pwrctrl_power_off_device(&pdev->dev);
+			if (ret)
+				dev_err(&pdev->dev, "Failed to power off device: %d", ret);
+		}
 	}
 
 	platform_device_put(pdev);
@@ -205,7 +252,7 @@ static int __pci_pwrctrl_power_on_device(struct device *dev)
 static int pci_pwrctrl_power_on_device(struct device_node *np)
 {
 	struct platform_device *pdev;
-	int ret;
+	int ret = 0;
 
 	for_each_available_child_of_node_scoped(np, child) {
 		ret = pci_pwrctrl_power_on_device(child);
@@ -213,16 +260,21 @@ static int pci_pwrctrl_power_on_device(struct device_node *np)
 			return ret;
 	}
 
+	if (!pci_pwrctrl_is_required(np))
+		return 0;
+
 	pdev = of_find_device_by_node(np);
 	if (!pdev)
 		return 0;
 
-	if (device_is_bound(&pdev->dev)) {
-		ret = __pci_pwrctrl_power_on_device(&pdev->dev);
-	} else {
-		/* FIXME: Use blocking wait instead of probe deferral */
-		dev_dbg(&pdev->dev, "driver is not bound\n");
-		ret = -EPROBE_DEFER;
+	scoped_guard(device, &pdev->dev) {
+		if (device_is_bound(&pdev->dev)) {
+			ret = __pci_pwrctrl_power_on_device(&pdev->dev);
+		} else {
+			/* FIXME: Use blocking wait instead of probe deferral */
+			dev_dbg(&pdev->dev, "driver is not bound\n");
+			ret = -EPROBE_DEFER;
+		}
 	}
 
 	platform_device_put(pdev);
@@ -268,48 +320,6 @@ int pci_pwrctrl_power_on_devices(struct device *parent)
 }
 EXPORT_SYMBOL_GPL(pci_pwrctrl_power_on_devices);
 
-/*
- * Check whether the pwrctrl device really needs to be created or not. The
- * pwrctrl device will only be created if the node satisfies below requirements:
- *
- * 1. Presence of compatible property with "pci" prefix to match against the
- *    pwrctrl driver (AND)
- * 2. At least one of the power supplies defined in the devicetree node of the
- *    device (OR) in the remote endpoint parent node to indicate pwrctrl
- *    requirement.
- */
-static bool pci_pwrctrl_is_required(struct device_node *np)
-{
-	struct device_node *endpoint;
-	const char *compat;
-	int ret;
-
-	ret = of_property_read_string(np, "compatible", &compat);
-	if (ret < 0)
-		return false;
-
-	if (!strstarts(compat, "pci"))
-		return false;
-
-	if (of_pci_supply_present(np))
-		return true;
-
-	if (of_graph_is_present(np)) {
-		for_each_endpoint_of_node(np, endpoint) {
-			struct device_node *remote __free(device_node) =
-				of_graph_get_remote_port_parent(endpoint);
-			if (remote) {
-				if (of_pci_supply_present(remote)) {
-					of_node_put(endpoint);
-					return true;
-				}
-			}
-		}
-	}
-
-	return false;
-}
-
 static int pci_pwrctrl_create_device(struct device_node *np,
 				     struct device *parent)
 {
diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
index caaed1a01dc02..1b4ae046dd69b 100644
--- a/drivers/pci/quirks.c
+++ b/drivers/pci/quirks.c
@@ -80,11 +80,10 @@ static bool pcie_lbms_seen(struct pci_dev *dev, u16 lnksta)
  * Restrict the speed to 2.5GT/s then with the Target Link Speed field,
  * request a retrain and check the result.
  *
- * If this turns out successful and we know by the Vendor:Device ID it is
- * safe to do so, then lift the restriction, letting the devices negotiate
- * a higher speed.  Also check for a similar 2.5GT/s speed restriction the
- * firmware may have already arranged and lift it with ports that already
- * report their data link being up.
+ * If this turns out successful, or where a 2.5GT/s speed restriction has
+ * been previously arranged by the firmware and the port reports its link
+ * already being up, lift the restriction, in a hope it is safe to do so,
+ * letting the devices negotiate a higher speed.
  *
  * Otherwise revert the speed to the original setting and request a retrain
  * again to remove any residual state, ignoring the result as it's supposed
@@ -95,51 +94,39 @@ static bool pcie_lbms_seen(struct pci_dev *dev, u16 lnksta)
  */
 int pcie_failed_link_retrain(struct pci_dev *dev)
 {
-	static const struct pci_device_id ids[] = {
-		{ PCI_VDEVICE(ASMEDIA, 0x2824) }, /* ASMedia ASM2824 */
-		{}
-	};
-	u16 lnksta, lnkctl2;
+	u16 lnksta, lnkctl2, oldlnkctl2;
+	enum pci_bus_speed speed_cap;
 	int ret = -ENOTTY;
 
 	if (!pci_is_pcie(dev) || !pcie_downstream_port(dev) ||
 	    !pcie_cap_has_lnkctl2(dev) || !dev->link_active_reporting)
 		return ret;
 
+	speed_cap = pcie_get_speed_cap(dev);
+	if (speed_cap <= PCIE_SPEED_2_5GT)
+		return ret;
+
 	pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
+	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &oldlnkctl2);
 	if (!(lnksta & PCI_EXP_LNKSTA_DLLLA) && pcie_lbms_seen(dev, lnksta)) {
-		u16 oldlnkctl2;
-
 		pci_info(dev, "broken device, retraining non-functional downstream link at 2.5GT/s\n");
-
-		pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &oldlnkctl2);
 		ret = pcie_set_target_speed(dev, PCIE_SPEED_2_5GT, false);
-		if (ret) {
-			pci_info(dev, "retraining failed\n");
-			pcie_set_target_speed(dev, PCIE_LNKCTL2_TLS2SPEED(oldlnkctl2),
-					      true);
-			return ret;
-		}
-
-		pcie_capability_read_word(dev, PCI_EXP_LNKSTA, &lnksta);
+		if (ret)
+			goto err;
 	}
 
 	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, &lnkctl2);
-
-	if ((lnksta & PCI_EXP_LNKSTA_DLLLA) &&
-	    (lnkctl2 & PCI_EXP_LNKCTL2_TLS) == PCI_EXP_LNKCTL2_TLS_2_5GT &&
-	    pci_match_id(ids, dev)) {
-		u32 lnkcap;
-
+	if ((lnkctl2 & PCI_EXP_LNKCTL2_TLS) == PCI_EXP_LNKCTL2_TLS_2_5GT) {
 		pci_info(dev, "removing 2.5GT/s downstream link speed restriction\n");
-		pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, &lnkcap);
-		ret = pcie_set_target_speed(dev, PCIE_LNKCAP_SLS2SPEED(lnkcap), false);
-		if (ret) {
-			pci_info(dev, "retraining failed\n");
-			return ret;
-		}
+		ret = pcie_set_target_speed(dev, speed_cap, false);
+		if (ret)
+			goto err;
 	}
 
+	return ret;
+err:
+	pci_info(dev, "retraining failed\n");
+	pcie_set_target_speed(dev, PCIE_LNKCTL2_TLS2SPEED(oldlnkctl2), true);
 	return ret;
 }
 
diff --git a/drivers/pci/rebar.c b/drivers/pci/rebar.c
index 39f8cf3b70d57..5bbdc9470a31b 100644
--- a/drivers/pci/rebar.c
+++ b/drivers/pci/rebar.c
@@ -211,8 +211,7 @@ int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size)
 		return pos;
 
 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
-	ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
-	ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
+	FIELD_MODIFY(PCI_REBAR_CTRL_BAR_SIZE, &ctrl, size);
 	pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
 
 	if (pci_resource_is_iov(bar))
@@ -231,6 +230,9 @@ void pci_restore_rebar_state(struct pci_dev *pdev)
 		return;
 
 	pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+	if (PCI_POSSIBLE_ERROR(ctrl))
+		return;
+
 	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
 
 	for (i = 0; i < nbars; i++, pos += 8) {
@@ -238,11 +240,13 @@ void pci_restore_rebar_state(struct pci_dev *pdev)
 		int bar_idx, size;
 
 		pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+		if (PCI_POSSIBLE_ERROR(ctrl))
+			return;
+
 		bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
 		res = pci_resource_n(pdev, bar_idx);
 		size = pci_rebar_bytes_to_size(resource_size(res));
-		ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
-		ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
+		FIELD_MODIFY(PCI_REBAR_CTRL_BAR_SIZE, &ctrl, size);
 		pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
 	}
 }
diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c
index e9d519993853f..d8bffa21498a9 100644
--- a/drivers/pci/remove.c
+++ b/drivers/pci/remove.c
@@ -26,7 +26,6 @@ static void pci_stop_dev(struct pci_dev *dev)
 
 	device_release_driver(&dev->dev);
 	pci_proc_detach_device(dev);
-	pci_remove_sysfs_dev_files(dev);
 	of_pci_remove_node(dev);
 }
 
@@ -66,8 +65,6 @@ void pci_remove_bus(struct pci_bus *bus)
 	list_del(&bus->node);
 	pci_bus_release_busn_res(bus);
 	up_write(&pci_bus_sem);
-	pci_remove_legacy_files(bus);
-
 	if (bus->ops->remove_bus)
 		bus->ops->remove_bus(bus);
 
diff --git a/drivers/pci/rom.c b/drivers/pci/rom.c
index e18d3a4383ba6..f2105c6ceef52 100644
--- a/drivers/pci/rom.c
+++ b/drivers/pci/rom.c
@@ -5,13 +5,40 @@
  * (C) Copyright 2004 Jon Smirl <jonsmirl@yahoo.com>
  * (C) Copyright 2004 Silicon Graphics, Inc. Jesse Barnes <jbarnes@sgi.com>
  */
+
+#include <linux/align.h>
+#include <linux/bits.h>
 #include <linux/kernel.h>
 #include <linux/export.h>
+#include <linux/io.h>
+#include <linux/overflow.h>
 #include <linux/pci.h>
+#include <linux/sizes.h>
 #include <linux/slab.h>
 
 #include "pci.h"
 
+#define PCI_ROM_HEADER_SIZE			0x1A
+#define PCI_ROM_POINTER_TO_DATA_STRUCT		0x18
+#define PCI_ROM_LAST_IMAGE_INDICATOR		0x15
+#define PCI_ROM_LAST_IMAGE_INDICATOR_BIT	BIT(7)
+#define PCI_ROM_IMAGE_LEN			0x10
+#define PCI_ROM_IMAGE_SECTOR_SIZE		SZ_512
+#define PCI_ROM_IMAGE_SIGNATURE			0xAA55
+
+/* Data structure signature is "PCIR" in ASCII representation */
+#define PCI_ROM_DATA_STRUCT_SIGNATURE		0x52494350
+#define PCI_ROM_DATA_STRUCT_LEN			0x0A
+
+/*
+ * Per PCI Firmware r3.3, sec 5.1.3, a conformant PCI Data Structure is at
+ * least 24 bytes (0x18), large enough to cover every fixed field this
+ * driver reads (up to the Indicator byte at offset 0x15).  Reject smaller
+ * device-claimed lengths so the follow-up readers in pci_get_rom_size()
+ * cannot escape the mapped ROM window.
+ */
+#define PCI_ROM_DATA_STRUCT_MIN_LEN		0x18
+
 /**
  * pci_enable_rom - enable ROM decoding for a PCI device
  * @pdev: PCI device to enable
@@ -69,6 +96,91 @@ void pci_disable_rom(struct pci_dev *pdev)
 }
 EXPORT_SYMBOL_GPL(pci_disable_rom);
 
+static bool pci_rom_header_valid(struct pci_dev *pdev, void __iomem *image,
+				 void __iomem *rom, size_t size,
+				 bool expect_valid)
+{
+	unsigned long rom_end = (unsigned long)rom + size - 1;
+	unsigned long header_end;
+	u16 signature;
+
+	/*
+	 * Per PCI Firmware r3.3, sec 5.1, each image must start on a
+	 * 512-byte boundary and must contain the PCI Expansion ROM header.
+	 * Because @rom is page-aligned (returned by ioremap()), checking
+	 * 512-byte alignment of @image is equivalent to enforcing the
+	 * spec's sector-aligned layout within the ROM.  This also
+	 * satisfies the natural-alignment requirement of readw() on archs
+	 * such as arm64 that disallow unaligned IOMEM access.
+	 */
+	if (!IS_ALIGNED((unsigned long)image, PCI_ROM_IMAGE_SECTOR_SIZE))
+		return false;
+
+	if (check_add_overflow((unsigned long)image, PCI_ROM_HEADER_SIZE - 1,
+				&header_end))
+		return false;
+
+	if (image < rom || header_end > rom_end)
+		return false;
+
+	/* Standard PCI ROMs start out with these bytes 55 AA */
+	signature = readw(image);
+	if (signature != PCI_ROM_IMAGE_SIGNATURE) {
+		if (expect_valid) {
+			pci_info(pdev, "Invalid PCI ROM header signature: expecting %#06x, got %#06x\n",
+				 PCI_ROM_IMAGE_SIGNATURE, signature);
+		} else {
+			pci_info(pdev, "No more images in PCI ROM\n");
+		}
+		return false;
+	}
+
+	return true;
+}
+
+static bool pci_rom_data_struct_valid(struct pci_dev *pdev, void __iomem *pds,
+				      void __iomem *rom, size_t size)
+{
+	unsigned long rom_end = (unsigned long)rom + size - 1;
+	unsigned long end;
+	u32 signature;
+	u16 data_len;
+
+	/*
+	 * Some CPU architectures require IOMEM access addresses to be
+	 * aligned, for example arm64, so since we're about to call
+	 * readl(), check here for 4-byte alignment.
+	 */
+	if (!IS_ALIGNED((unsigned long)pds, 4))
+		return false;
+
+	if (check_add_overflow((unsigned long)pds, PCI_ROM_DATA_STRUCT_LEN + 1,
+				&end))
+		return false;
+
+	if (pds < rom || end > rom_end)
+		return false;
+
+	signature = readl(pds);
+	if (signature != PCI_ROM_DATA_STRUCT_SIGNATURE) {
+		pci_info(pdev, "Invalid PCI ROM data signature: expecting %#010x, got %#010x\n",
+			 PCI_ROM_DATA_STRUCT_SIGNATURE, signature);
+		return false;
+	}
+
+	data_len = readw(pds + PCI_ROM_DATA_STRUCT_LEN);
+	if (data_len < PCI_ROM_DATA_STRUCT_MIN_LEN || data_len == U16_MAX)
+		return false;
+
+	if (check_add_overflow((unsigned long)pds, data_len - 1, &end))
+		return false;
+
+	if (end > rom_end)
+		return false;
+
+	return true;
+}
+
 /**
  * pci_get_rom_size - obtain the actual size of the ROM image
  * @pdev: target PCI device
@@ -84,37 +196,28 @@ static size_t pci_get_rom_size(struct pci_dev *pdev, void __iomem *rom,
 			       size_t size)
 {
 	void __iomem *image;
-	int last_image;
 	unsigned int length;
+	bool last_image;
 
 	image = rom;
 	do {
 		void __iomem *pds;
-		/* Standard PCI ROMs start out with these bytes 55 AA */
-		if (readw(image) != 0xAA55) {
-			pci_info(pdev, "Invalid PCI ROM header signature: expecting 0xaa55, got %#06x\n",
-				 readw(image));
+		if (!pci_rom_header_valid(pdev, image, rom, size, true))
 			break;
-		}
-		/* get the PCI data structure and check its "PCIR" signature */
-		pds = image + readw(image + 24);
-		if (readl(pds) != 0x52494350) {
-			pci_info(pdev, "Invalid PCI ROM data signature: expecting 0x52494350, got %#010x\n",
-				 readl(pds));
+
+		/* Get the PCI data structure and check its "PCIR" signature */
+		pds = image + readw(image + PCI_ROM_POINTER_TO_DATA_STRUCT);
+		if (!pci_rom_data_struct_valid(pdev, pds, rom, size))
 			break;
-		}
-		last_image = readb(pds + 21) & 0x80;
-		length = readw(pds + 16);
-		image += length * 512;
-		/* Avoid iterating through memory outside the resource window */
-		if (image >= rom + size)
+
+		last_image = readb(pds + PCI_ROM_LAST_IMAGE_INDICATOR) &
+				   PCI_ROM_LAST_IMAGE_INDICATOR_BIT;
+		length = readw(pds + PCI_ROM_IMAGE_LEN);
+		image += length * PCI_ROM_IMAGE_SECTOR_SIZE;
+
+		if (!last_image &&
+		    !pci_rom_header_valid(pdev, image, rom, size, false))
 			break;
-		if (!last_image) {
-			if (readw(image) != 0xAA55) {
-				pci_info(pdev, "No more image in the PCI ROM\n");
-				break;
-			}
-		}
 	} while (length && !last_image);
 
 	/* never return a size larger than the PCI resource window */
diff --git a/drivers/pci/setup-bus.c b/drivers/pci/setup-bus.c
index 4cf120ebe5adf..c0a949f2c9956 100644
--- a/drivers/pci/setup-bus.c
+++ b/drivers/pci/setup-bus.c
@@ -756,13 +756,13 @@ static void __assign_resources_sorted(struct list_head *head,
 }
 
 static void pdev_assign_resources_sorted(struct pci_dev *dev,
-					 struct list_head *add_head,
+					 struct list_head *add_list,
 					 struct list_head *fail_head)
 {
 	LIST_HEAD(head);
 
 	pdev_sort_resources(dev, &head);
-	__assign_resources_sorted(&head, add_head, fail_head);
+	__assign_resources_sorted(&head, add_list, fail_head);
 
 }
 
@@ -1501,14 +1501,14 @@ static void pdev_assign_fixed_resources(struct pci_dev *dev)
 	}
 }
 
-void __pci_bus_assign_resources(const struct pci_bus *bus,
-				struct list_head *realloc_head,
-				struct list_head *fail_head)
+static void __pci_bus_assign_resources_one(const struct pci_bus *bus,
+					   struct list_head *add_list,
+					   struct list_head *fail_head)
 {
 	struct pci_bus *b;
 	struct pci_dev *dev;
 
-	pbus_assign_resources_sorted(bus, realloc_head, fail_head);
+	pbus_assign_resources_sorted(bus, add_list, fail_head);
 
 	list_for_each_entry(dev, &bus->devices, bus_list) {
 		pdev_assign_fixed_resources(dev);
@@ -1517,7 +1517,7 @@ void __pci_bus_assign_resources(const struct pci_bus *bus,
 		if (!b)
 			continue;
 
-		__pci_bus_assign_resources(b, realloc_head, fail_head);
+		__pci_bus_assign_resources_one(b, add_list, fail_head);
 
 		switch (dev->hdr_type) {
 		case PCI_HEADER_TYPE_BRIDGE:
@@ -1537,6 +1537,16 @@ void __pci_bus_assign_resources(const struct pci_bus *bus,
 	}
 }
 
+void __pci_bus_assign_resources(const struct pci_bus *bus,
+				struct list_head *add_list,
+				struct list_head *fail_head)
+{
+	__pci_bus_assign_resources_one(bus, add_list, fail_head);
+
+	if (WARN_ON_ONCE(add_list && !list_empty(add_list)))
+		pci_dev_res_free_list(add_list);
+}
+
 void pci_bus_assign_resources(const struct pci_bus *bus)
 {
 	__pci_bus_assign_resources(bus, NULL, NULL);
@@ -1612,20 +1622,19 @@ void pci_bus_claim_resources(struct pci_bus *b)
 }
 EXPORT_SYMBOL(pci_bus_claim_resources);
 
-static void __pci_bridge_assign_resources(const struct pci_dev *bridge,
-					  struct list_head *add_head,
+static void __pci_bridge_assign_resources(struct pci_dev *bridge,
+					  struct list_head *add_list,
 					  struct list_head *fail_head)
 {
 	struct pci_bus *b;
 
-	pdev_assign_resources_sorted((struct pci_dev *)bridge,
-					 add_head, fail_head);
+	pdev_assign_resources_sorted(bridge, add_list, fail_head);
 
 	b = bridge->subordinate;
 	if (!b)
 		return;
 
-	__pci_bus_assign_resources(b, add_head, fail_head);
+	__pci_bus_assign_resources(b, add_list, fail_head);
 
 	switch (bridge->class >> 8) {
 	case PCI_CLASS_BRIDGE_PCI:
@@ -1641,6 +1650,9 @@ static void __pci_bridge_assign_resources(const struct pci_dev *bridge,
 			 pci_domain_nr(b), b->number);
 		break;
 	}
+
+	if (WARN_ON_ONCE(add_list && !list_empty(add_list)))
+		pci_dev_res_free_list(add_list);
 }
 
 static void pci_bridge_release_resources(struct pci_bus *bus,
@@ -2205,8 +2217,6 @@ void pci_assign_unassigned_root_bus_resources(struct pci_bus *bus)
 
 		/* Depth last, allocate resources and update the hardware. */
 		__pci_bus_assign_resources(bus, add_list, &fail_head);
-		if (WARN_ON_ONCE(add_list && !list_empty(add_list)))
-			pci_dev_res_free_list(add_list);
 		tried_times++;
 
 		/* Any device complain? */
@@ -2268,8 +2278,6 @@ void pci_assign_unassigned_bridge_resources(struct pci_dev *bridge)
 		pci_bridge_distribute_available_resources(bridge, &add_list);
 
 		__pci_bridge_assign_resources(bridge, &add_list, &fail_head);
-		if (WARN_ON_ONCE(!list_empty(&add_list)))
-			pci_dev_res_free_list(&add_list);
 		tried_times++;
 
 		if (list_empty(&fail_head))
@@ -2303,7 +2311,7 @@ static int pbus_reassign_bridge_resources(struct pci_bus *bus, struct resource *
 	unsigned long type = res->flags;
 	struct pci_dev_resource *dev_res;
 	struct pci_dev *bridge = NULL;
-	LIST_HEAD(added);
+	LIST_HEAD(add_list);
 	LIST_HEAD(failed);
 	unsigned int i;
 	int ret = 0;
@@ -2337,10 +2345,8 @@ static int pbus_reassign_bridge_resources(struct pci_bus *bus, struct resource *
 	if (!bridge)
 		return -ENOENT;
 
-	__pci_bus_size_bridges(bridge->subordinate, &added);
-	__pci_bridge_assign_resources(bridge, &added, &failed);
-	if (WARN_ON_ONCE(!list_empty(&added)))
-		pci_dev_res_free_list(&added);
+	__pci_bus_size_bridges(bridge->subordinate, &add_list);
+	__pci_bridge_assign_resources(bridge, &add_list, &failed);
 
 	if (!list_empty(&failed)) {
 		if (pci_required_resource_failed(&failed, type))
@@ -2473,7 +2479,5 @@ void pci_assign_unassigned_bus_resources(struct pci_bus *bus)
 			__pci_bus_size_bridges(dev->subordinate, &add_list);
 	up_read(&pci_bus_sem);
 	__pci_bus_assign_resources(bus, &add_list, NULL);
-	if (WARN_ON_ONCE(!list_empty(&add_list)))
-		pci_dev_res_free_list(&add_list);
 }
 EXPORT_SYMBOL_GPL(pci_assign_unassigned_bus_resources);
diff --git a/drivers/pci/setup-cardbus.c b/drivers/pci/setup-cardbus.c
index 1ebd13a1f7302..dcb8e2b3f73ba 100644
--- a/drivers/pci/setup-cardbus.c
+++ b/drivers/pci/setup-cardbus.c
@@ -22,7 +22,7 @@
 static unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
 static unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
 
-unsigned long pci_cardbus_resource_alignment(struct resource *res)
+unsigned long pci_cardbus_resource_alignment(const struct resource *res)
 {
 	if (res->flags & IORESOURCE_IO)
 		return pci_cardbus_io_size;
@@ -253,8 +253,7 @@ int pci_cardbus_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
 	 * yenta.c forces a secondary latency timer of 176.
 	 * Copy that behaviour here.
 	 */
-	buses &= ~PCI_SEC_LATENCY_TIMER_MASK;
-	buses |= FIELD_PREP(PCI_SEC_LATENCY_TIMER_MASK, CARDBUS_LATENCY_TIMER);
+	FIELD_MODIFY(PCI_SEC_LATENCY_TIMER_MASK, &buses, CARDBUS_LATENCY_TIMER);
 
 	/* We need to blast all three values with a single write */
 	pci_write_config_dword(dev, PCI_PRIMARY_BUS, buses);
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
index 991d3ed543f58..b6a28990ee548 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -13,13 +13,17 @@
  *	     Resource sorting
  */
 
+#include <linux/bitops.h>
 #include <linux/kernel.h>
 #include <linux/export.h>
 #include <linux/pci.h>
 #include <linux/errno.h>
 #include <linux/ioport.h>
 #include <linux/cache.h>
+#include <linux/minmax.h>
 #include <linux/slab.h>
+#include <linux/types.h>
+
 #include "pci.h"
 
 static void pci_std_update_resource(struct pci_dev *dev, int resno)
@@ -169,6 +173,8 @@ int pci_claim_resource(struct pci_dev *dev, int resource)
 		return -EBUSY;
 	}
 
+	pci_dbg(dev, "%s %pR: claiming\n", res_name, res);
+
 	return 0;
 }
 EXPORT_SYMBOL(pci_claim_resource);
@@ -246,6 +252,37 @@ static int pci_revert_fw_address(struct resource *res, struct pci_dev *dev,
 	return 0;
 }
 
+resource_size_t pci_resource_alignment(const struct pci_dev *dev,
+				       const struct resource *res)
+{
+	int resno = pci_resource_num(dev, res);
+	resource_size_t min_align = 0;
+
+	if (pci_resource_is_iov(resno))
+		return pci_sriov_resource_alignment(dev, resno);
+
+	if (dev->class >> 8 == PCI_CLASS_BRIDGE_CARDBUS)
+		return pci_cardbus_resource_alignment(res);
+
+	if (pci_resource_is_bridge_win(resno) &&
+	    (res->flags & (IORESOURCE_IO|IORESOURCE_MEM)))
+		min_align = pci_min_window_alignment(dev->bus, res->flags);
+
+	if (resource_assigned(res)) {
+		resource_size_t start_align = 1, size_align;
+
+		size_align = roundup_pow_of_two(resource_size(res));
+		if (res->start)
+			start_align <<= __ffs(res->start);
+		else
+			start_align = size_align;
+
+		return max(min(start_align, size_align), min_align);
+	}
+
+	return max(resource_alignment(res), min_align);
+}
+
 /*
  * For mem bridge windows, try to relocate tail remainder space to space
  * before res->start if there's enough free space there. This enables
diff --git a/drivers/pci/switch/switchtec.c b/drivers/pci/switch/switchtec.c
index 93ebec94b7632..41fc4b5127081 100644
--- a/drivers/pci/switch/switchtec.c
+++ b/drivers/pci/switch/switchtec.c
@@ -1852,6 +1852,22 @@ static const struct pci_device_id switchtec_pci_tbl[] = {
 	SWITCHTEC_PCI_DEVICE(0x5552, SWITCHTEC_GEN5),  /* PAXA 52XG5 */
 	SWITCHTEC_PCI_DEVICE(0x5536, SWITCHTEC_GEN5),  /* PAXA 36XG5 */
 	SWITCHTEC_PCI_DEVICE(0x5528, SWITCHTEC_GEN5),  /* PAXA 28XG5 */
+	SWITCHTEC_PCI_DEVICE(0x6048, SWITCHTEC_GEN6),  /* PFXs 48XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6064, SWITCHTEC_GEN6),  /* PFXs 64XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6044, SWITCHTEC_GEN6),  /* PFXs 144XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6060, SWITCHTEC_GEN6),  /* PFXs 160XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6148, SWITCHTEC_GEN6),  /* PSXs 48XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6164, SWITCHTEC_GEN6),  /* PSXs 64XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6144, SWITCHTEC_GEN6),  /* PSXs 144XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6160, SWITCHTEC_GEN6),  /* PSXs 160XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6248, SWITCHTEC_GEN6),  /* PFX 48XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6264, SWITCHTEC_GEN6),  /* PFX 64XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6244, SWITCHTEC_GEN6),  /* PFX 144XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6260, SWITCHTEC_GEN6),  /* PFX 160XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6348, SWITCHTEC_GEN6),  /* PSX 48XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6364, SWITCHTEC_GEN6),  /* PSX 64XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6344, SWITCHTEC_GEN6),  /* PSX 144XG6 */
+	SWITCHTEC_PCI_DEVICE(0x6360, SWITCHTEC_GEN6),  /* PSX 160XG6 */
 	SWITCHTEC_PCI100X_DEVICE(0x1001, SWITCHTEC_GEN4),  /* PCI1001 16XG4 */
 	SWITCHTEC_PCI100X_DEVICE(0x1002, SWITCHTEC_GEN4),  /* PCI1002 12XG4 */
 	SWITCHTEC_PCI100X_DEVICE(0x1003, SWITCHTEC_GEN4),  /* PCI1003 16XG4 */
diff --git a/drivers/pci/tph.c b/drivers/pci/tph.c
index 91145e8d9d951..655ffd60e62f6 100644
--- a/drivers/pci/tph.c
+++ b/drivers/pci/tph.c
@@ -139,8 +139,7 @@ static void set_ctrl_reg_req_en(struct pci_dev *pdev, u8 req_type)
 
 	pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, &reg);
 
-	reg &= ~PCI_TPH_CTRL_REQ_EN_MASK;
-	reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, req_type);
+	FIELD_MODIFY(PCI_TPH_CTRL_REQ_EN_MASK, &reg, req_type);
 
 	pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg);
 }
@@ -427,11 +426,8 @@ int pcie_enable_tph(struct pci_dev *pdev, int mode)
 	/* Write them into TPH control register */
 	pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, &reg);
 
-	reg &= ~PCI_TPH_CTRL_MODE_SEL_MASK;
-	reg |= FIELD_PREP(PCI_TPH_CTRL_MODE_SEL_MASK, pdev->tph_mode);
-
-	reg &= ~PCI_TPH_CTRL_REQ_EN_MASK;
-	reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, pdev->tph_req_type);
+	FIELD_MODIFY(PCI_TPH_CTRL_MODE_SEL_MASK, &reg, pdev->tph_mode);
+	FIELD_MODIFY(PCI_TPH_CTRL_REQ_EN_MASK, &reg, pdev->tph_req_type);
 
 	pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg);
 
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index ab90932fc2d01..245e7bb763b99 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -188,7 +188,7 @@ config FUJITSU_UNCORE_PMU
 	 monitoring Uncore events.
 
 config QCOM_L2_PMU
-	bool "Qualcomm Technologies L2-cache PMU"
+	bool "Qualcomm L2-cache PMU"
 	depends on ARCH_QCOM && ARM64 && ACPI
 	select QCOM_KRYO_L2_ACCESSORS
 	  help
@@ -198,7 +198,7 @@ config QCOM_L2_PMU
 	  monitoring L2 cache events.
 
 config QCOM_L3_PMU
-	bool "Qualcomm Technologies L3-cache PMU"
+	bool "Qualcomm L3-cache PMU"
 	depends on ARCH_QCOM && ARM64 && ACPI
 	select QCOM_IRQ_COMBINER
 	help
diff --git a/drivers/phy/Kconfig b/drivers/phy/Kconfig
index 227b9a4c612e8..7ef342f4bbb6c 100644
--- a/drivers/phy/Kconfig
+++ b/drivers/phy/Kconfig
@@ -66,6 +66,31 @@ config PHY_CAN_TRANSCEIVER
 	  functional modes using gpios and sets the attribute max link
 	  rate, for CAN drivers.
 
+config PHY_EYEQ5_ETH
+	tristate "Ethernet PHY Driver on EyeQ5"
+	depends on OF
+	depends on MACH_EYEQ5 || COMPILE_TEST
+	select AUXILIARY_BUS
+	select GENERIC_PHY
+	default MACH_EYEQ5
+	help
+	  Enable this to support the Ethernet PHY integrated on EyeQ5.
+	  It supports both RGMII and SGMII. Registers are located in a
+	  shared register region called OLB. If M is selected, the
+	  module will be called phy-eyeq5-eth.
+
+config PHY_ECONET_PCIE
+	tristate "EcoNet PCIe-PHY Driver"
+	depends on ECONET || COMPILE_TEST
+	depends on OF
+	select GENERIC_PHY
+	select REGMAP_MMIO
+	help
+	  Say Y here to add support for EcoNet PCIe PHY driver.
+	  This driver create the basic PHY instance and provides initialize
+	  callback for PCIe GEN1 and GEN2 ports. This PHY is found on
+	  EcoNet SoCs including EN751221 and EN7528.
+
 config PHY_GOOGLE_USB
 	tristate "Google Tensor SoC USB PHY driver"
 	select GENERIC_PHY
@@ -136,6 +161,7 @@ config PHY_XGENE
 source "drivers/phy/allwinner/Kconfig"
 source "drivers/phy/amlogic/Kconfig"
 source "drivers/phy/apple/Kconfig"
+source "drivers/phy/axiado/Kconfig"
 source "drivers/phy/broadcom/Kconfig"
 source "drivers/phy/cadence/Kconfig"
 source "drivers/phy/canaan/Kconfig"
diff --git a/drivers/phy/Makefile b/drivers/phy/Makefile
index f49d83f00a3d8..1e69ed50fb17e 100644
--- a/drivers/phy/Makefile
+++ b/drivers/phy/Makefile
@@ -9,6 +9,8 @@ obj-$(CONFIG_GENERIC_PHY)		+= phy-core.o
 obj-$(CONFIG_GENERIC_PHY_MIPI_DPHY)	+= phy-core-mipi-dphy.o
 obj-$(CONFIG_PHY_AIROHA_PCIE)		+= phy-airoha-pcie.o
 obj-$(CONFIG_PHY_CAN_TRANSCEIVER)	+= phy-can-transceiver.o
+obj-$(CONFIG_PHY_ECONET_PCIE)		+= phy-econet-pcie.o
+obj-$(CONFIG_PHY_EYEQ5_ETH)		+= phy-eyeq5-eth.o
 obj-$(CONFIG_PHY_GOOGLE_USB)		+= phy-google-usb.o
 obj-$(CONFIG_USB_LGM_PHY)		+= phy-lgm-usb.o
 obj-$(CONFIG_PHY_LPC18XX_USB_OTG)	+= phy-lpc18xx-usb-otg.o
@@ -20,6 +22,7 @@ obj-$(CONFIG_PHY_XGENE)			+= phy-xgene.o
 obj-$(CONFIG_GENERIC_PHY)		+= allwinner/	\
 					   amlogic/	\
 					   apple/	\
+					   axiado/	\
 					   broadcom/	\
 					   cadence/	\
 					   canaan/	\
diff --git a/drivers/phy/apple/atc.c b/drivers/phy/apple/atc.c
index e9d106f135c51..4156fabad742c 100644
--- a/drivers/phy/apple/atc.c
+++ b/drivers/phy/apple/atc.c
@@ -628,9 +628,6 @@ struct apple_atcphy {
 
 	struct reset_controller_dev rcdev;
 
-	struct typec_switch *sw;
-	struct typec_mux *mux;
-
 	struct mutex lock;
 };
 
@@ -2066,15 +2063,25 @@ static int atcphy_sw_set(struct typec_switch_dev *sw, enum typec_orientation ori
 	return 0;
 }
 
+static void atcphy_typec_switch_unregister(void *data)
+{
+	typec_switch_unregister(data);
+}
+
 static int atcphy_probe_switch(struct apple_atcphy *atcphy)
 {
+	struct typec_switch_dev *sw;
 	struct typec_switch_desc sw_desc = {
 		.drvdata = atcphy,
 		.fwnode = atcphy->dev->fwnode,
 		.set = atcphy_sw_set,
 	};
 
-	return PTR_ERR_OR_ZERO(typec_switch_register(atcphy->dev, &sw_desc));
+	sw = typec_switch_register(atcphy->dev, &sw_desc);
+	if (IS_ERR(sw))
+		return PTR_ERR(sw);
+
+	return devm_add_action_or_reset(atcphy->dev, atcphy_typec_switch_unregister, sw);
 }
 
 static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *state)
@@ -2146,15 +2153,25 @@ static int atcphy_mux_set(struct typec_mux_dev *mux, struct typec_mux_state *sta
 	return atcphy_configure(atcphy, target_mode);
 }
 
+static void atcphy_typec_mux_unregister(void *data)
+{
+	typec_mux_unregister(data);
+}
+
 static int atcphy_probe_mux(struct apple_atcphy *atcphy)
 {
+	struct typec_mux_dev *mux;
 	struct typec_mux_desc mux_desc = {
 		.drvdata = atcphy,
 		.fwnode = atcphy->dev->fwnode,
 		.set = atcphy_mux_set,
 	};
 
-	return PTR_ERR_OR_ZERO(typec_mux_register(atcphy->dev, &mux_desc));
+	mux = typec_mux_register(atcphy->dev, &mux_desc);
+	if (IS_ERR(mux))
+		return PTR_ERR(mux);
+
+	return devm_add_action_or_reset(atcphy->dev, atcphy_typec_mux_unregister, mux);
 }
 
 static int atcphy_load_tunables(struct apple_atcphy *atcphy)
diff --git a/drivers/phy/axiado/Kconfig b/drivers/phy/axiado/Kconfig
new file mode 100644
index 0000000000000..d159e03453459
--- /dev/null
+++ b/drivers/phy/axiado/Kconfig
@@ -0,0 +1,11 @@
+#
+# PHY drivers for Axiado platforms
+#
+
+config PHY_AX3000_EMMC
+	tristate "Axiado eMMC PHY driver"
+	depends on OF && (ARCH_AXIADO || COMPILE_TEST)
+	select GENERIC_PHY
+	help
+	  Enables this to support for the AX3000 EMMC PHY driver.
+	  If unsure, say N.
diff --git a/drivers/phy/axiado/Makefile b/drivers/phy/axiado/Makefile
new file mode 100644
index 0000000000000..1e2b1ba016092
--- /dev/null
+++ b/drivers/phy/axiado/Makefile
@@ -0,0 +1 @@
+obj-$(CONFIG_PHY_AX3000_EMMC)		+= phy-axiado-emmc.o
diff --git a/drivers/phy/axiado/phy-axiado-emmc.c b/drivers/phy/axiado/phy-axiado-emmc.c
new file mode 100644
index 0000000000000..e0e2174776ad0
--- /dev/null
+++ b/drivers/phy/axiado/phy-axiado-emmc.c
@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Axiado eMMC PHY driver
+ *
+ * Copyright (C) 2017 Arasan Chip Systems Inc.
+ * Copyright (C) 2022-2026 Axiado Corporation (or its affiliates).
+ *
+ * Based on Arasan Driver (sdhci-pci-arasan.c)
+ * sdhci-pci-arasan.c - Driver for Arasan PCI Controller with integrated phy.
+ */
+#include <linux/bitfield.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+
+/* Arasan eMMC 5.1 - PHY configuration registers */
+#define CAP_REG_IN_S1_MSB		0x04
+#define PHY_CTRL_1			0x38
+#define PHY_CTRL_2			0x3c
+#define PHY_CTRL_3			0x40
+#define STATUS				0x50
+
+#define DLL_ENBL	BIT(26)
+#define RTRIM_EN	BIT(21)
+#define PDB_ENBL	BIT(23)
+#define RETB_ENBL	BIT(1)
+
+#define REN_STRB	BIT(27)
+#define REN_CMD_EN	GENMASK(20, 12)
+
+/* Pull-UP Enable on CMD Line */
+#define PU_CMD_EN	GENMASK(11, 3)
+
+/* Selection value for the optimum delay from 1-32 output tap lines */
+#define OTAP_DLY	0x02
+/* DLL charge pump current trim default [1000] */
+#define DLL_TRM_ICP	0x08
+/* Select the frequency range of DLL Operation */
+#define FRQ_SEL	0x01
+
+#define OTAP_SEL_MASK		GENMASK(10, 7)
+#define DLL_TRM_MASK		GENMASK(25, 22)
+#define DLL_FRQSEL_MASK		GENMASK(27, 25)
+
+#define OTAP_SEL(x)		(FIELD_PREP(OTAP_SEL_MASK, x) | OTAPDLY_EN)
+#define DLL_TRM(x)		(FIELD_PREP(DLL_TRM_MASK, x) | DLL_ENBL)
+#define DLL_FRQSEL(x)	FIELD_PREP(DLL_FRQSEL_MASK, x)
+
+#define OTAPDLY_EN	BIT(11)
+
+#define SEL_DLY_RXCLK	BIT(18)
+#define SEL_DLY_TXCLK	BIT(19)
+
+#define CALDONE_MASK	0x40
+#define DLL_RDY_MASK	0x1
+#define MAX_CLK_BUF0	BIT(20)
+#define MAX_CLK_BUF1	BIT(21)
+#define MAX_CLK_BUF2	BIT(22)
+
+#define CLK_MULTIPLIER	0xc008e
+#define POLL_TIMEOUT_MS	3000
+#define POLL_DELAY_US	100
+
+struct axiado_emmc_phy {
+	void __iomem *reg_base;
+	struct device *dev;
+};
+
+static int axiado_emmc_phy_init(struct phy *phy)
+{
+	struct axiado_emmc_phy *ax_phy = phy_get_drvdata(phy);
+	struct device *dev = ax_phy->dev;
+	u32 val;
+	int ret;
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_1);
+	writel(val | RETB_ENBL | RTRIM_EN, ax_phy->reg_base + PHY_CTRL_1);
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_3);
+	writel(val | PDB_ENBL, ax_phy->reg_base + PHY_CTRL_3);
+
+	ret = readl_poll_timeout(ax_phy->reg_base + STATUS, val,
+				 val & CALDONE_MASK, POLL_DELAY_US,
+				 POLL_TIMEOUT_MS * 1000);
+	if (ret) {
+		dev_err(dev, "PHY calibration timeout\n");
+		return ret;
+	}
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_1);
+	writel(val | REN_CMD_EN | PU_CMD_EN, ax_phy->reg_base + PHY_CTRL_1);
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_2);
+	writel(val | REN_STRB, ax_phy->reg_base + PHY_CTRL_2);
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_3);
+	writel(val | MAX_CLK_BUF0 | MAX_CLK_BUF1 | MAX_CLK_BUF2,
+	       ax_phy->reg_base + PHY_CTRL_3);
+
+	writel(CLK_MULTIPLIER, ax_phy->reg_base + CAP_REG_IN_S1_MSB);
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_3);
+	writel(val | SEL_DLY_RXCLK | SEL_DLY_TXCLK,
+	       ax_phy->reg_base + PHY_CTRL_3);
+
+	return 0;
+}
+
+static int axiado_emmc_phy_power_on(struct phy *phy)
+{
+	struct axiado_emmc_phy *ax_phy = phy_get_drvdata(phy);
+	struct device *dev = ax_phy->dev;
+	u32 val;
+	int ret;
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_1);
+	writel(val | RETB_ENBL, ax_phy->reg_base + PHY_CTRL_1);
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_3);
+	writel(val | PDB_ENBL, ax_phy->reg_base + PHY_CTRL_3);
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_2);
+	writel(val | OTAP_SEL(OTAP_DLY), ax_phy->reg_base + PHY_CTRL_2);
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_1);
+	writel(val | DLL_TRM(DLL_TRM_ICP), ax_phy->reg_base + PHY_CTRL_1);
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_3);
+	writel(val | DLL_FRQSEL(FRQ_SEL), ax_phy->reg_base + PHY_CTRL_3);
+
+	ret = read_poll_timeout(readl, val, val & DLL_RDY_MASK, POLL_DELAY_US,
+				POLL_TIMEOUT_MS * 1000, false,
+				ax_phy->reg_base + STATUS);
+	if (ret) {
+		dev_err(dev, "DLL ready timeout\n");
+		return ret;
+	}
+
+	return 0;
+}
+
+static int axiado_emmc_phy_power_off(struct phy *phy)
+{
+	struct axiado_emmc_phy *ax_phy = phy_get_drvdata(phy);
+	u32 val;
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_1);
+	val &= ~(DLL_TRM_MASK | DLL_ENBL);
+	writel(val, ax_phy->reg_base + PHY_CTRL_1);
+
+	val = readl(ax_phy->reg_base + PHY_CTRL_3);
+	val &= ~(DLL_FRQSEL_MASK | PDB_ENBL);
+	writel(val, ax_phy->reg_base + PHY_CTRL_3);
+
+	return 0;
+}
+
+static const struct phy_ops axiado_emmc_phy_ops = {
+	.init = axiado_emmc_phy_init,
+	.power_on = axiado_emmc_phy_power_on,
+	.power_off = axiado_emmc_phy_power_off,
+	.owner = THIS_MODULE,
+};
+
+static const struct of_device_id axiado_emmc_phy_of_match[] = {
+	{ .compatible = "axiado,ax3000-emmc-phy" },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, axiado_emmc_phy_of_match);
+
+static int axiado_emmc_phy_probe(struct platform_device *pdev)
+{
+	struct axiado_emmc_phy *ax_phy;
+	struct phy_provider *phy_provider;
+	struct device *dev = &pdev->dev;
+	struct phy *generic_phy;
+
+	if (!dev->of_node)
+		return -ENODEV;
+
+	ax_phy = devm_kzalloc(dev, sizeof(*ax_phy), GFP_KERNEL);
+	if (!ax_phy)
+		return -ENOMEM;
+
+	ax_phy->reg_base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(ax_phy->reg_base))
+		return PTR_ERR(ax_phy->reg_base);
+
+	ax_phy->dev = dev;
+
+	generic_phy = devm_phy_create(dev, dev->of_node, &axiado_emmc_phy_ops);
+	if (IS_ERR(generic_phy))
+		return dev_err_probe(dev, PTR_ERR(generic_phy),
+				     "failed to create PHY\n");
+
+	phy_set_drvdata(generic_phy, ax_phy);
+	phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+
+	return PTR_ERR_OR_ZERO(phy_provider);
+}
+
+static struct platform_driver axiado_emmc_phy_driver = {
+	.probe = axiado_emmc_phy_probe,
+	.driver = {
+		.name = "axiado-emmc-phy",
+		.of_match_table = axiado_emmc_phy_of_match,
+	},
+};
+module_platform_driver(axiado_emmc_phy_driver);
+
+MODULE_DESCRIPTION("AX3000 eMMC PHY Driver");
+MODULE_AUTHOR("Axiado Corporation");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/eswin/phy-eic7700-sata.c b/drivers/phy/eswin/phy-eic7700-sata.c
index c33653d48daaf..76774b9e391b1 100644
--- a/drivers/phy/eswin/phy-eic7700-sata.c
+++ b/drivers/phy/eswin/phy-eic7700-sata.c
@@ -216,8 +216,8 @@ static int eic7700_sata_phy_probe(struct platform_device *pdev)
 		return -ENOENT;
 
 	regs = devm_ioremap(dev, res->start, resource_size(res));
-	if (IS_ERR(regs))
-		return PTR_ERR(regs);
+	if (!regs)
+		return -ENOMEM;
 
 	sata_phy->regmap = devm_regmap_init_mmio
 			   (dev, regs, &eic7700_sata_phy_regmap_config);
diff --git a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c
index 279b8ac7822df..4ab45c9f53dff 100644
--- a/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c
+++ b/drivers/phy/freescale/phy-fsl-imx8qm-hsio.c
@@ -107,7 +107,22 @@ static const char * const lan2_pcieb_clks[] = {"apb_pclk2", "pclk2", "ctl1_crr",
 static const char * const lan2_sata_clks[] = {"pclk2", "epcs_tx", "epcs_rx",
 					      "phy1_crr", "misc_crr"};
 
-static const struct regmap_config regmap_config = {
+static const struct regmap_config regmap_phy_config = {
+	.name = "phy",
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+};
+
+static const struct regmap_config regmap_ctrl_config = {
+	.name = "ctrl",
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+};
+
+static const struct regmap_config regmap_misc_config = {
+	.name = "misc",
 	.reg_bits = 32,
 	.val_bits = 32,
 	.reg_stride = 4,
@@ -562,19 +577,19 @@ static int imx_hsio_probe(struct platform_device *pdev)
 		return PTR_ERR(priv->base);
 
 	off = devm_platform_ioremap_resource_byname(pdev, "phy");
-	priv->phy = devm_regmap_init_mmio(dev, off, &regmap_config);
+	priv->phy = devm_regmap_init_mmio(dev, off, &regmap_phy_config);
 	if (IS_ERR(priv->phy))
 		return dev_err_probe(dev, PTR_ERR(priv->phy),
 				     "unable to find phy csr registers\n");
 
 	off = devm_platform_ioremap_resource_byname(pdev, "ctrl");
-	priv->ctrl = devm_regmap_init_mmio(dev, off, &regmap_config);
+	priv->ctrl = devm_regmap_init_mmio(dev, off, &regmap_ctrl_config);
 	if (IS_ERR(priv->ctrl))
 		return dev_err_probe(dev, PTR_ERR(priv->ctrl),
 				     "unable to find ctrl csr registers\n");
 
 	off = devm_platform_ioremap_resource_byname(pdev, "misc");
-	priv->misc = devm_regmap_init_mmio(dev, off, &regmap_config);
+	priv->misc = devm_regmap_init_mmio(dev, off, &regmap_misc_config);
 	if (IS_ERR(priv->misc))
 		return dev_err_probe(dev, PTR_ERR(priv->misc),
 				     "unable to find misc csr registers\n");
diff --git a/drivers/phy/freescale/phy-fsl-lynx-28g.c b/drivers/phy/freescale/phy-fsl-lynx-28g.c
index 63427fc34e268..92bfc5f65e0b8 100644
--- a/drivers/phy/freescale/phy-fsl-lynx-28g.c
+++ b/drivers/phy/freescale/phy-fsl-lynx-28g.c
@@ -57,6 +57,7 @@
 #define PLLnCR1_FRATE_5G_10GVCO			0x0
 #define PLLnCR1_FRATE_5G_25GVCO			0x10
 #define PLLnCR1_FRATE_10G_20GVCO		0x6
+#define PLLnCR1_FRATE_12G_25GVCO		0x16
 
 /* Per SerDes lane registers */
 /* Lane a General Control Register */
@@ -64,15 +65,19 @@
 #define LNaGCR0_PROTO_SEL			GENMASK(7, 3)
 #define LNaGCR0_PROTO_SEL_SGMII			0x1
 #define LNaGCR0_PROTO_SEL_XFI			0xa
+#define LNaGCR0_PROTO_SEL_25G			0x1a
 #define LNaGCR0_IF_WIDTH			GENMASK(2, 0)
 #define LNaGCR0_IF_WIDTH_10_BIT			0x0
 #define LNaGCR0_IF_WIDTH_20_BIT			0x2
+#define LNaGCR0_IF_WIDTH_40_BIT			0x4
 
 /* Lane a Tx Reset Control Register */
 #define LNaTRSTCTL(lane)			(0x800 + (lane) * 0x100 + 0x20)
-#define LNaTRSTCTL_HLT_REQ			BIT(27)
-#define LNaTRSTCTL_RST_DONE			BIT(30)
 #define LNaTRSTCTL_RST_REQ			BIT(31)
+#define LNaTRSTCTL_RST_DONE			BIT(30)
+#define LNaTRSTCTL_HLT_REQ			BIT(27)
+#define LNaTRSTCTL_STP_REQ			BIT(26)
+#define LNaTRSTCTL_DIS				BIT(24)
 
 /* Lane a Tx General Control Register */
 #define LNaTGCR0(lane)				(0x800 + (lane) * 0x100 + 0x24)
@@ -83,6 +88,7 @@
 #define LNaTGCR0_N_RATE_FULL			0x0
 #define LNaTGCR0_N_RATE_HALF			0x1
 #define LNaTGCR0_N_RATE_QUARTER			0x2
+#define LNaTGCR0_N_RATE_DOUBLE			0x3
 
 #define LNaTECR0(lane)				(0x800 + (lane) * 0x100 + 0x30)
 #define LNaTECR0_EQ_TYPE			GENMASK(30, 28)
@@ -98,9 +104,11 @@
 
 /* Lane a Rx Reset Control Register */
 #define LNaRRSTCTL(lane)			(0x800 + (lane) * 0x100 + 0x40)
-#define LNaRRSTCTL_HLT_REQ			BIT(27)
-#define LNaRRSTCTL_RST_DONE			BIT(30)
 #define LNaRRSTCTL_RST_REQ			BIT(31)
+#define LNaRRSTCTL_RST_DONE			BIT(30)
+#define LNaRRSTCTL_HLT_REQ			BIT(27)
+#define LNaRRSTCTL_STP_REQ			BIT(26)
+#define LNaRRSTCTL_DIS				BIT(24)
 #define LNaRRSTCTL_CDR_LOCK			BIT(12)
 
 /* Lane a Rx General Control Register */
@@ -112,6 +120,7 @@
 #define LNaRGCR0_N_RATE_FULL			0x0
 #define LNaRGCR0_N_RATE_HALF			0x1
 #define LNaRGCR0_N_RATE_QUARTER			0x2
+#define LNaRGCR0_N_RATE_DOUBLE			0x3
 
 #define LNaRGCR1(lane)				(0x800 + (lane) * 0x100 + 0x48)
 #define LNaRGCR1_RX_ORD_ELECIDLE		BIT(31)
@@ -249,6 +258,15 @@
 
 #define CR(x)					((x) * 4)
 
+#define LYNX_28G_LANE_HALT_SLEEP_US		100
+#define LYNX_28G_LANE_HALT_TIMEOUT_US		1000000
+
+#define LYNX_28G_LANE_RESET_SLEEP_US		100
+#define LYNX_28G_LANE_RESET_TIMEOUT_US		1000000
+
+#define LYNX_28G_LANE_STOP_SLEEP_US		100
+#define LYNX_28G_LANE_STOP_TIMEOUT_US		1000000
+
 enum lynx_28g_eq_type {
 	EQ_TYPE_NO_EQ = 0,
 	EQ_TYPE_2TAP = 1,
@@ -269,6 +287,7 @@ enum lynx_lane_mode {
 	LANE_MODE_1000BASEX_SGMII,
 	LANE_MODE_10GBASER,
 	LANE_MODE_USXGMII,
+	LANE_MODE_25GBASER,
 	LANE_MODE_MAX,
 };
 
@@ -407,6 +426,41 @@ static const struct lynx_28g_proto_conf lynx_28g_proto_conf[LANE_MODE_MAX] = {
 		.ttlcr0 = LNaTTLCR0_TTL_SLO_PM_BYP |
 			  LNaTTLCR0_DATA_IN_SSC,
 	},
+	[LANE_MODE_25GBASER] = {
+		.proto_sel = LNaGCR0_PROTO_SEL_25G,
+		.if_width = LNaGCR0_IF_WIDTH_40_BIT,
+		.teq_type = EQ_TYPE_3TAP,
+		.sgn_preq = 1,
+		.ratio_preq = 2,
+		.sgn_post1q = 1,
+		.ratio_post1q = 7,
+		.amp_red = 0,
+		.adpt_eq = 48,
+		.enter_idle_flt_sel = 0,
+		.exit_idle_flt_sel = 0,
+		.data_lost_th_sel = 0,
+		.gk2ovd = 0,
+		.gk3ovd = 0,
+		.gk4ovd = 5,
+		.gk2ovd_en = 0,
+		.gk3ovd_en = 0,
+		.gk4ovd_en = 1,
+		.eq_offset_ovd = 0x1f,
+		.eq_offset_ovd_en = 0,
+		.eq_offset_rng_dbl = 1,
+		.eq_blw_sel = 1,
+		.eq_boost = 2,
+		.spare_in = 3,
+		.smp_autoz_d1r = 2,
+		.smp_autoz_eg1r = 2,
+		.rccr0 = LNaRCCR0_CAL_EN |
+			 LNaRCCR0_CAL_DC3_DIS |
+			 LNaRCCR0_CAL_DC2_DIS |
+			 LNaRCCR0_CAL_DC1_DIS |
+			 LNaRCCR0_CAL_DC0_DIS,
+		.ttlcr0 = LNaTTLCR0_DATA_IN_SSC |
+			  FIELD_PREP_CONST(LNaTTLCR0_CDR_MIN_SMP_ON, 1),
+	},
 };
 
 struct lynx_pccr {
@@ -433,9 +487,15 @@ struct lynx_28g_lane {
 	enum lynx_lane_mode mode;
 };
 
+struct lynx_info {
+	bool (*lane_supports_mode)(int lane, enum lynx_lane_mode mode);
+	int first_lane;
+};
+
 struct lynx_28g_priv {
 	void __iomem *base;
 	struct device *dev;
+	const struct lynx_info *info;
 	/* Serialize concurrent access to registers shared between lanes,
 	 * like PCCn
 	 */
@@ -480,6 +540,8 @@ static const char *lynx_lane_mode_str(enum lynx_lane_mode lane_mode)
 		return "10GBase-R";
 	case LANE_MODE_USXGMII:
 		return "USXGMII";
+	case LANE_MODE_25GBASER:
+		return "25GBase-R";
 	default:
 		return "unknown";
 	}
@@ -495,16 +557,25 @@ static enum lynx_lane_mode phy_interface_to_lane_mode(phy_interface_t intf)
 		return LANE_MODE_10GBASER;
 	case PHY_INTERFACE_MODE_USXGMII:
 		return LANE_MODE_USXGMII;
+	case PHY_INTERFACE_MODE_25GBASER:
+		return LANE_MODE_25GBASER;
 	default:
 		return LANE_MODE_UNKNOWN;
 	}
 }
 
-static bool lynx_28g_supports_lane_mode(struct lynx_28g_priv *priv,
+/* A lane mode is supported if we have a PLL that can provide its required
+ * clock net, and if there is a protocol converter for that mode on that lane.
+ */
+static bool lynx_28g_supports_lane_mode(struct lynx_28g_lane *lane,
 					enum lynx_lane_mode mode)
 {
+	struct lynx_28g_priv *priv = lane->priv;
 	int i;
 
+	if (!priv->info->lane_supports_mode(lane->id, mode))
+		return false;
+
 	for (i = 0; i < LYNX_28G_NUM_PLL; i++) {
 		if (PLLnRSTCTL_DIS(priv->pll[i].rstctl))
 			continue;
@@ -575,6 +646,20 @@ static void lynx_28g_lane_set_nrate(struct lynx_28g_lane *lane,
 			break;
 		}
 		break;
+	case PLLnCR1_FRATE_12G_25GVCO:
+		switch (lane_mode) {
+		case LANE_MODE_25GBASER:
+			lynx_28g_lane_rmw(lane, LNaTGCR0,
+					  FIELD_PREP(LNaTGCR0_N_RATE, LNaTGCR0_N_RATE_DOUBLE),
+					  LNaTGCR0_N_RATE);
+			lynx_28g_lane_rmw(lane, LNaRGCR0,
+					  FIELD_PREP(LNaRGCR0_N_RATE, LNaRGCR0_N_RATE_DOUBLE),
+					  LNaRGCR0_N_RATE);
+			break;
+		default:
+			break;
+		}
+		break;
 	default:
 		break;
 	}
@@ -600,13 +685,39 @@ static void lynx_28g_lane_set_pll(struct lynx_28g_lane *lane,
 	}
 }
 
-static int lynx_28g_power_off(struct phy *phy)
+static bool lynx_28g_lane_halt_done(struct lynx_28g_lane *lane)
 {
-	struct lynx_28g_lane *lane = phy_get_drvdata(phy);
-	u32 trstctl, rrstctl;
+	u32 trstctl = lynx_28g_lane_read(lane, LNaTRSTCTL);
+	u32 rrstctl = lynx_28g_lane_read(lane, LNaRRSTCTL);
 
-	if (!lane->powered_up)
-		return 0;
+	return !(trstctl & LNaTRSTCTL_HLT_REQ) &&
+	       !(rrstctl & LNaRRSTCTL_HLT_REQ);
+}
+
+static bool lynx_28g_lane_stop_done(struct lynx_28g_lane *lane)
+{
+	u32 trstctl = lynx_28g_lane_read(lane, LNaTRSTCTL);
+	u32 rrstctl = lynx_28g_lane_read(lane, LNaRRSTCTL);
+
+	return !(trstctl & LNaTRSTCTL_STP_REQ) &&
+	       !(rrstctl & LNaRRSTCTL_STP_REQ);
+}
+
+static bool lynx_28g_lane_reset_done(struct lynx_28g_lane *lane)
+{
+	u32 trstctl = lynx_28g_lane_read(lane, LNaTRSTCTL);
+	u32 rrstctl = lynx_28g_lane_read(lane, LNaRRSTCTL);
+
+	return (trstctl & LNaTRSTCTL_RST_DONE) &&
+	       (rrstctl & LNaRRSTCTL_RST_DONE);
+}
+
+/* Halting puts the lane in a mode in which it can be reconfigured */
+static int lynx_28g_lane_halt(struct phy *phy)
+{
+	struct lynx_28g_lane *lane = phy_get_drvdata(phy);
+	bool done;
+	int err;
 
 	/* Issue a halt request */
 	lynx_28g_lane_rmw(lane, LNaTRSTCTL, LNaTRSTCTL_HLT_REQ,
@@ -615,11 +726,73 @@ static int lynx_28g_power_off(struct phy *phy)
 			  LNaRRSTCTL_HLT_REQ);
 
 	/* Wait until the halting process is complete */
-	do {
-		trstctl = lynx_28g_lane_read(lane, LNaTRSTCTL);
-		rrstctl = lynx_28g_lane_read(lane, LNaRRSTCTL);
-	} while ((trstctl & LNaTRSTCTL_HLT_REQ) ||
-		 (rrstctl & LNaRRSTCTL_HLT_REQ));
+	err = read_poll_timeout(lynx_28g_lane_halt_done, done, done,
+				LYNX_28G_LANE_HALT_SLEEP_US,
+				LYNX_28G_LANE_HALT_TIMEOUT_US,
+				false, lane);
+	if (err) {
+		dev_err(&phy->dev, "Lane %c halt failed: %pe\n",
+			'A' + lane->id, ERR_PTR(err));
+	}
+
+	return err;
+}
+
+static int lynx_28g_lane_reset(struct phy *phy)
+{
+	struct lynx_28g_lane *lane = phy_get_drvdata(phy);
+	bool done;
+	int err;
+
+	/* Issue a reset request on the lane */
+	lynx_28g_lane_rmw(lane, LNaTRSTCTL, LNaTRSTCTL_RST_REQ,
+			  LNaTRSTCTL_RST_REQ);
+	lynx_28g_lane_rmw(lane, LNaRRSTCTL, LNaRRSTCTL_RST_REQ,
+			  LNaRRSTCTL_RST_REQ);
+
+	/* Wait until the reset sequence is completed */
+	err = read_poll_timeout(lynx_28g_lane_reset_done, done, done,
+				LYNX_28G_LANE_RESET_SLEEP_US,
+				LYNX_28G_LANE_RESET_TIMEOUT_US,
+				false, lane);
+	if (err) {
+		dev_err(&phy->dev, "Lane %c reset failed: %pe\n",
+			'A' + lane->id, ERR_PTR(err));
+	}
+
+	return err;
+}
+
+static int lynx_28g_power_off(struct phy *phy)
+{
+	struct lynx_28g_lane *lane = phy_get_drvdata(phy);
+	bool done;
+	int err;
+
+	if (!lane->powered_up)
+		return 0;
+
+	/* Issue a stop request */
+	lynx_28g_lane_rmw(lane, LNaTRSTCTL, LNaTRSTCTL_STP_REQ,
+			  LNaTRSTCTL_STP_REQ);
+	lynx_28g_lane_rmw(lane, LNaRRSTCTL, LNaRRSTCTL_STP_REQ,
+			  LNaRRSTCTL_STP_REQ);
+
+	/* Wait until the stop process is complete */
+	err = read_poll_timeout(lynx_28g_lane_stop_done, done, done,
+				LYNX_28G_LANE_STOP_SLEEP_US,
+				LYNX_28G_LANE_STOP_TIMEOUT_US,
+				false, lane);
+	if (err) {
+		dev_err(&phy->dev, "Lane %c stop failed: %pe\n",
+			'A' + lane->id, ERR_PTR(err));
+	}
+
+	/* Power down the RX and TX portions of the lane */
+	lynx_28g_lane_rmw(lane, LNaRRSTCTL, LNaRRSTCTL_DIS,
+			  LNaRRSTCTL_DIS);
+	lynx_28g_lane_rmw(lane, LNaTRSTCTL, LNaTRSTCTL_DIS,
+			  LNaTRSTCTL_DIS);
 
 	lane->powered_up = false;
 
@@ -629,29 +802,29 @@ static int lynx_28g_power_off(struct phy *phy)
 static int lynx_28g_power_on(struct phy *phy)
 {
 	struct lynx_28g_lane *lane = phy_get_drvdata(phy);
-	u32 trstctl, rrstctl;
+	int err;
 
 	if (lane->powered_up)
 		return 0;
 
-	/* Issue a reset request on the lane */
-	lynx_28g_lane_rmw(lane, LNaTRSTCTL, LNaTRSTCTL_RST_REQ,
-			  LNaTRSTCTL_RST_REQ);
-	lynx_28g_lane_rmw(lane, LNaRRSTCTL, LNaRRSTCTL_RST_REQ,
-			  LNaRRSTCTL_RST_REQ);
+	/* Power up the RX and TX portions of the lane */
+	lynx_28g_lane_rmw(lane, LNaRRSTCTL, 0, LNaRRSTCTL_DIS);
+	lynx_28g_lane_rmw(lane, LNaTRSTCTL, 0, LNaTRSTCTL_DIS);
 
-	/* Wait until the reset sequence is completed */
-	do {
-		trstctl = lynx_28g_lane_read(lane, LNaTRSTCTL);
-		rrstctl = lynx_28g_lane_read(lane, LNaRRSTCTL);
-	} while (!(trstctl & LNaTRSTCTL_RST_DONE) ||
-		 !(rrstctl & LNaRRSTCTL_RST_DONE));
+	err = lynx_28g_lane_reset(phy);
+	if (err)
+		return err;
 
 	lane->powered_up = true;
 
 	return 0;
 }
 
+static int lynx_28g_e25g_pcvt(int lane)
+{
+	return 7 - lane;
+}
+
 static int lynx_28g_get_pccr(enum lynx_lane_mode lane_mode, int lane,
 			     struct lynx_pccr *pccr)
 {
@@ -667,6 +840,11 @@ static int lynx_28g_get_pccr(enum lynx_lane_mode lane_mode, int lane,
 		pccr->width = 4;
 		pccr->shift = SXGMII_CFG(lane);
 		break;
+	case LANE_MODE_25GBASER:
+		pccr->offset = PCCD;
+		pccr->width = 4;
+		pccr->shift = E25G_CFG(lynx_28g_e25g_pcvt(lane));
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -682,11 +860,99 @@ static int lynx_28g_get_pcvt_offset(int lane, enum lynx_lane_mode lane_mode)
 	case LANE_MODE_USXGMII:
 	case LANE_MODE_10GBASER:
 		return SXGMIIaCR0(lane);
+	case LANE_MODE_25GBASER:
+		return E25GaCR0(lynx_28g_e25g_pcvt(lane));
 	default:
 		return -EOPNOTSUPP;
 	}
 }
 
+static bool lx2160a_serdes1_lane_supports_mode(int lane,
+					       enum lynx_lane_mode mode)
+{
+	switch (mode) {
+	case LANE_MODE_25GBASER:
+		return lane != 2 && lane != 3;
+	default:
+		return true;
+	}
+}
+
+static bool lx2160a_serdes2_lane_supports_mode(int lane,
+					       enum lynx_lane_mode mode)
+{
+	switch (mode) {
+	case LANE_MODE_1000BASEX_SGMII:
+		return true;
+	case LANE_MODE_USXGMII:
+	case LANE_MODE_10GBASER:
+		return lane == 6 || lane == 7;
+	default:
+		return false;
+	}
+}
+
+static bool lx2160a_serdes3_lane_supports_mode(int lane,
+					       enum lynx_lane_mode mode)
+{
+	/*
+	 * Non-networking SerDes, and this driver supports only
+	 * networking protocols
+	 */
+	return false;
+}
+
+static bool lx2162a_serdes1_lane_supports_mode(int lane,
+					       enum lynx_lane_mode mode)
+{
+	return true;
+}
+
+static bool lx2162a_serdes2_lane_supports_mode(int lane,
+					       enum lynx_lane_mode mode)
+{
+	return lx2160a_serdes2_lane_supports_mode(lane, mode);
+}
+
+/* Feature set is not expected to grow for the deprecated compatible string */
+static bool lynx_28g_compat_lane_supports_mode(int lane,
+					       enum lynx_lane_mode mode)
+{
+	switch (mode) {
+	case LANE_MODE_1000BASEX_SGMII:
+	case LANE_MODE_USXGMII:
+	case LANE_MODE_10GBASER:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct lynx_info lynx_info_compat = {
+	.lane_supports_mode = lynx_28g_compat_lane_supports_mode,
+};
+
+static const struct lynx_info lynx_info_lx2160a_serdes1 = {
+	.lane_supports_mode = lx2160a_serdes1_lane_supports_mode,
+};
+
+static const struct lynx_info lynx_info_lx2160a_serdes2 = {
+	.lane_supports_mode = lx2160a_serdes2_lane_supports_mode,
+};
+
+static const struct lynx_info lynx_info_lx2160a_serdes3 = {
+	.lane_supports_mode = lx2160a_serdes3_lane_supports_mode,
+};
+
+static const struct lynx_info lynx_info_lx2162a_serdes1 = {
+	.lane_supports_mode = lx2162a_serdes1_lane_supports_mode,
+	.first_lane = 4,
+};
+
+static const struct lynx_info lynx_info_lx2162a_serdes2 = {
+	.lane_supports_mode = lx2162a_serdes2_lane_supports_mode,
+};
+
 static int lynx_pccr_read(struct lynx_28g_lane *lane, enum lynx_lane_mode mode,
 			  u32 *val)
 {
@@ -925,6 +1191,9 @@ static int lynx_28g_lane_enable_pcvt(struct lynx_28g_lane *lane,
 	case LANE_MODE_USXGMII:
 		val |= PCCC_SXGMIIn_CFG;
 		break;
+	case LANE_MODE_25GBASER:
+		val |= PCCD_E25Gn_CFG;
+		break;
 	default:
 		break;
 	}
@@ -939,7 +1208,6 @@ static int lynx_28g_lane_enable_pcvt(struct lynx_28g_lane *lane,
 static int lynx_28g_set_mode(struct phy *phy, enum phy_mode mode, int submode)
 {
 	struct lynx_28g_lane *lane = phy_get_drvdata(phy);
-	struct lynx_28g_priv *priv = lane->priv;
 	int powered_up = lane->powered_up;
 	enum lynx_lane_mode lane_mode;
 	int err = 0;
@@ -951,7 +1219,7 @@ static int lynx_28g_set_mode(struct phy *phy, enum phy_mode mode, int submode)
 		return -EOPNOTSUPP;
 
 	lane_mode = phy_interface_to_lane_mode(submode);
-	if (!lynx_28g_supports_lane_mode(priv, lane_mode))
+	if (!lynx_28g_supports_lane_mode(lane, lane_mode))
 		return -EOPNOTSUPP;
 
 	if (lane_mode == lane->mode)
@@ -960,8 +1228,11 @@ static int lynx_28g_set_mode(struct phy *phy, enum phy_mode mode, int submode)
 	/* If the lane is powered up, put the lane into the halt state while
 	 * the reconfiguration is being done.
 	 */
-	if (powered_up)
-		lynx_28g_power_off(phy);
+	if (powered_up) {
+		err = lynx_28g_lane_halt(phy);
+		if (err)
+			goto out;
+	}
 
 	err = lynx_28g_lane_disable_pcvt(lane, lane->mode);
 	if (err)
@@ -975,7 +1246,7 @@ static int lynx_28g_set_mode(struct phy *phy, enum phy_mode mode, int submode)
 
 out:
 	if (powered_up)
-		lynx_28g_power_on(phy);
+		lynx_28g_lane_reset(phy);
 
 	return err;
 }
@@ -984,14 +1255,13 @@ static int lynx_28g_validate(struct phy *phy, enum phy_mode mode, int submode,
 			     union phy_configure_opts *opts __always_unused)
 {
 	struct lynx_28g_lane *lane = phy_get_drvdata(phy);
-	struct lynx_28g_priv *priv = lane->priv;
 	enum lynx_lane_mode lane_mode;
 
 	if (mode != PHY_MODE_ETHERNET)
 		return -EOPNOTSUPP;
 
 	lane_mode = phy_interface_to_lane_mode(submode);
-	if (!lynx_28g_supports_lane_mode(priv, lane_mode))
+	if (!lynx_28g_supports_lane_mode(lane, lane_mode))
 		return -EOPNOTSUPP;
 
 	return 0;
@@ -1014,8 +1284,25 @@ static int lynx_28g_init(struct phy *phy)
 	return 0;
 }
 
+static int lynx_28g_exit(struct phy *phy)
+{
+	struct lynx_28g_lane *lane = phy_get_drvdata(phy);
+
+	/* The lane returns to the state where it isn't managed by the
+	 * consumer, so we must treat is as if it isn't initialized, and
+	 * always powered on.
+	 */
+	lane->init = false;
+	lane->powered_up = false;
+
+	lynx_28g_power_on(phy);
+
+	return 0;
+}
+
 static const struct phy_ops lynx_28g_ops = {
 	.init		= lynx_28g_init,
+	.exit		= lynx_28g_exit,
 	.power_on	= lynx_28g_power_on,
 	.power_off	= lynx_28g_power_off,
 	.set_mode	= lynx_28g_set_mode,
@@ -1051,8 +1338,12 @@ static void lynx_28g_pll_read_configuration(struct lynx_28g_priv *priv)
 			__set_bit(LANE_MODE_10GBASER, pll->supported);
 			__set_bit(LANE_MODE_USXGMII, pll->supported);
 			break;
+		case PLLnCR1_FRATE_12G_25GVCO:
+			/* 12.890625GHz clock net */
+			__set_bit(LANE_MODE_25GBASER, pll->supported);
+			break;
 		default:
-			/* 6GHz, 12.890625GHz, 8GHz */
+			/* 6GHz, 8GHz */
 			break;
 		}
 	}
@@ -1065,9 +1356,9 @@ static void lynx_28g_cdr_lock_check(struct work_struct *work)
 	struct lynx_28g_priv *priv = work_to_lynx(work);
 	struct lynx_28g_lane *lane;
 	u32 rrstctl;
-	int i;
+	int err, i;
 
-	for (i = 0; i < LYNX_28G_NUM_LANE; i++) {
+	for (i = priv->info->first_lane; i < LYNX_28G_NUM_LANE; i++) {
 		lane = &priv->lane[i];
 		if (!lane->phy)
 			continue;
@@ -1083,9 +1374,17 @@ static void lynx_28g_cdr_lock_check(struct work_struct *work)
 		if (!(rrstctl & LNaRRSTCTL_CDR_LOCK)) {
 			lynx_28g_lane_rmw(lane, LNaRRSTCTL, LNaRRSTCTL_RST_REQ,
 					  LNaRRSTCTL_RST_REQ);
-			do {
-				rrstctl = lynx_28g_lane_read(lane, LNaRRSTCTL);
-			} while (!(rrstctl & LNaRRSTCTL_RST_DONE));
+
+			err = read_poll_timeout(lynx_28g_lane_read, rrstctl,
+						!!(rrstctl & LNaRRSTCTL_RST_DONE),
+						LYNX_28G_LANE_RESET_SLEEP_US,
+						LYNX_28G_LANE_RESET_TIMEOUT_US,
+						false, lane, LNaRRSTCTL);
+			if (err) {
+				dev_warn_once(&lane->phy->dev,
+					      "Lane %c receiver reset failed: %pe\n",
+					      'A' + lane->id, ERR_PTR(err));
+			}
 		}
 
 		mutex_unlock(&lane->phy->mutex);
@@ -1111,6 +1410,9 @@ static void lynx_28g_lane_read_configuration(struct lynx_28g_lane *lane)
 		else
 			lane->mode = LANE_MODE_USXGMII;
 		break;
+	case LNaPSS_TYPE_25G:
+		lane->mode = LANE_MODE_25GBASER;
+		break;
 	default:
 		lane->mode = LANE_MODE_UNKNOWN;
 	}
@@ -1129,7 +1431,8 @@ static struct phy *lynx_28g_xlate(struct device *dev,
 
 	idx = args->args[0];
 
-	if (WARN_ON(idx >= LYNX_28G_NUM_LANE))
+	if (WARN_ON(idx >= LYNX_28G_NUM_LANE ||
+		    idx < priv->info->first_lane))
 		return ERR_PTR(-EINVAL);
 
 	return priv->lane[idx].phy;
@@ -1162,22 +1465,35 @@ static int lynx_28g_probe(struct platform_device *pdev)
 	struct device_node *dn;
 	int err;
 
+	dn = dev_of_node(dev);
+	if (!dn) {
+		dev_err(dev, "Device requires an OF node\n");
+		return -EINVAL;
+	}
+
 	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
 	priv->dev = dev;
+	priv->info = of_device_get_match_data(dev);
 	dev_set_drvdata(dev, priv);
 	spin_lock_init(&priv->pcc_lock);
 	INIT_DELAYED_WORK(&priv->cdr_check, lynx_28g_cdr_lock_check);
 
+	/*
+	 * If we get here it means we probed on a device tree where
+	 * "fsl,lynx-28g" wasn't the fallback, but the sole compatible string.
+	 */
+	if (priv->info == &lynx_info_compat)
+		dev_warn(dev, "Please update device tree to use per-device compatible strings\n");
+
 	priv->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(priv->base))
 		return PTR_ERR(priv->base);
 
 	lynx_28g_pll_read_configuration(priv);
 
-	dn = dev_of_node(dev);
 	if (of_get_child_count(dn)) {
 		struct device_node *child;
 
@@ -1194,7 +1510,7 @@ static int lynx_28g_probe(struct platform_device *pdev)
 				return -EINVAL;
 			}
 
-			if (reg >= LYNX_28G_NUM_LANE) {
+			if (reg < priv->info->first_lane || reg >= LYNX_28G_NUM_LANE) {
 				dev_err(dev, "\"reg\" property out of range for %pOF\n", child);
 				of_node_put(child);
 				return -EINVAL;
@@ -1207,7 +1523,7 @@ static int lynx_28g_probe(struct platform_device *pdev)
 			}
 		}
 	} else {
-		for (int i = 0; i < LYNX_28G_NUM_LANE; i++) {
+		for (int i = priv->info->first_lane; i < LYNX_28G_NUM_LANE; i++) {
 			err = lynx_28g_probe_lane(priv, i, NULL);
 			if (err)
 				return err;
@@ -1233,7 +1549,12 @@ static void lynx_28g_remove(struct platform_device *pdev)
 }
 
 static const struct of_device_id lynx_28g_of_match_table[] = {
-	{ .compatible = "fsl,lynx-28g" },
+	{ .compatible = "fsl,lx2160a-serdes1", .data = &lynx_info_lx2160a_serdes1 },
+	{ .compatible = "fsl,lx2160a-serdes2", .data = &lynx_info_lx2160a_serdes2 },
+	{ .compatible = "fsl,lx2160a-serdes3", .data = &lynx_info_lx2160a_serdes3 },
+	{ .compatible = "fsl,lx2162a-serdes1", .data = &lynx_info_lx2162a_serdes1 },
+	{ .compatible = "fsl,lx2162a-serdes2", .data = &lynx_info_lx2162a_serdes2 },
+	{ .compatible = "fsl,lynx-28g", .data = &lynx_info_compat }, /* fallback, keep last */
 	{ },
 };
 MODULE_DEVICE_TABLE(of, lynx_28g_of_match_table);
diff --git a/drivers/phy/marvell/phy-mvebu-a3700-utmi.c b/drivers/phy/marvell/phy-mvebu-a3700-utmi.c
index 04f4fb4bed702..f882bc57649c7 100644
--- a/drivers/phy/marvell/phy-mvebu-a3700-utmi.c
+++ b/drivers/phy/marvell/phy-mvebu-a3700-utmi.c
@@ -168,9 +168,8 @@ static int mvebu_a3700_utmi_phy_power_off(struct phy *phy)
 	u32 reg;
 
 	/* Disable PHY pull-up and enable USB2 suspend */
-	reg = readl(utmi->regs + USB2_PHY_CTRL(usb32));
-	reg &= ~(RB_USB2PHY_PU | RB_USB2PHY_SUSPM(usb32));
-	writel(reg, utmi->regs + USB2_PHY_CTRL(usb32));
+	regmap_update_bits(utmi->usb_misc, USB2_PHY_CTRL(usb32),
+			   RB_USB2PHY_PU | RB_USB2PHY_SUSPM(usb32), 0);
 
 	/* Power down OTG module */
 	if (usb32) {
diff --git a/drivers/phy/mediatek/phy-mtk-xsphy.c b/drivers/phy/mediatek/phy-mtk-xsphy.c
index c0ddb9273cc3f..cc1d66954212a 100644
--- a/drivers/phy/mediatek/phy-mtk-xsphy.c
+++ b/drivers/phy/mediatek/phy-mtk-xsphy.c
@@ -112,10 +112,10 @@ struct xsphy_instance {
 struct mtk_xsphy {
 	struct device *dev;
 	void __iomem *glb_base;	/* only shared u3 sif */
-	struct xsphy_instance **phys;
-	int nphys;
 	int src_ref_clk; /* MHZ, reference clock for slew rate calibrate */
 	int src_coef;    /* coefficient for slew rate calibrate */
+	int nphys;
+	struct xsphy_instance *phys[] __counted_by(nphys);
 };
 
 static void u2_phy_slew_rate_calibrate(struct mtk_xsphy *xsphy,
@@ -515,18 +515,15 @@ static int mtk_xsphy_probe(struct platform_device *pdev)
 	struct resource *glb_res;
 	struct mtk_xsphy *xsphy;
 	struct resource res;
+	size_t nphys;
 	int port;
 
-	xsphy = devm_kzalloc(dev, sizeof(*xsphy), GFP_KERNEL);
+	nphys = of_get_child_count(np);
+	xsphy = devm_kzalloc(dev, struct_size(xsphy, phys, nphys), GFP_KERNEL);
 	if (!xsphy)
 		return -ENOMEM;
 
-	xsphy->nphys = of_get_child_count(np);
-	xsphy->phys = devm_kcalloc(dev, xsphy->nphys,
-				       sizeof(*xsphy->phys), GFP_KERNEL);
-	if (!xsphy->phys)
-		return -ENOMEM;
-
+	xsphy->nphys = nphys;
 	xsphy->dev = dev;
 	platform_set_drvdata(pdev, xsphy);
 
diff --git a/drivers/phy/phy-airoha-pcie-regs.h b/drivers/phy/phy-airoha-pcie-regs.h
index b938a7b468fee..58572c7937223 100644
--- a/drivers/phy/phy-airoha-pcie-regs.h
+++ b/drivers/phy/phy-airoha-pcie-regs.h
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0-only
+/* SPDX-License-Identifier: GPL-2.0-only */
 /*
  * Copyright (c) 2024 AIROHA Inc
  * Author: Lorenzo Bianconi <lorenzo@kernel.org>
diff --git a/drivers/phy/phy-can-transceiver.c b/drivers/phy/phy-can-transceiver.c
index 2b52e47f247a2..75dc49e75ca0e 100644
--- a/drivers/phy/phy-can-transceiver.c
+++ b/drivers/phy/phy-can-transceiver.c
@@ -5,12 +5,11 @@
  * Copyright (C) 2021 Texas Instruments Incorporated - https://www.ti.com
  *
  */
-#include <linux/of.h>
+#include <linux/gpio/consumer.h>
 #include <linux/phy/phy.h>
 #include <linux/platform_device.h>
+#include <linux/property.h>
 #include <linux/module.h>
-#include <linux/gpio.h>
-#include <linux/gpio/consumer.h>
 #include <linux/mux/consumer.h>
 
 struct can_transceiver_data {
@@ -97,35 +96,6 @@ static const struct can_transceiver_data tja1057_drvdata = {
 	.flags = CAN_TRANSCEIVER_SILENT_PRESENT,
 };
 
-static const struct of_device_id can_transceiver_phy_ids[] = {
-	{
-		.compatible = "ti,tcan1042",
-		.data = &tcan1042_drvdata
-	},
-	{
-		.compatible = "ti,tcan1043",
-		.data = &tcan1043_drvdata
-	},
-	{
-		.compatible = "nxp,tja1048",
-		.data = &tja1048_drvdata
-	},
-	{
-		.compatible = "nxp,tja1051",
-		.data = &tja1051_drvdata
-	},
-	{
-		.compatible = "nxp,tja1057",
-		.data = &tja1057_drvdata
-	},
-	{
-		.compatible = "nxp,tjr1443",
-		.data = &tcan1043_drvdata
-	},
-	{ }
-};
-MODULE_DEVICE_TABLE(of, can_transceiver_phy_ids);
-
 static struct phy *can_transceiver_phy_xlate(struct device *dev,
 					     const struct of_phandle_args *args)
 {
@@ -152,19 +122,23 @@ static int can_transceiver_phy_probe(struct platform_device *pdev)
 	struct can_transceiver_phy *can_transceiver_phy;
 	struct can_transceiver_priv *priv;
 	const struct can_transceiver_data *drvdata;
-	const struct of_device_id *match;
 	struct phy *phy;
 	struct gpio_desc *silent_gpio;
 	struct gpio_desc *standby_gpio;
 	struct gpio_desc *enable_gpio;
 	struct mux_state *mux_state;
-	u32 max_bitrate = 0;
-	int err, i, num_ch = 1;
+	const char *propname;
+	int err, i, num_ch;
+	u32 max_bitrate;
+
+	drvdata = device_get_match_data(dev);
+	if (!drvdata)
+		return -ENODEV;
 
-	match = of_match_node(can_transceiver_phy_ids, pdev->dev.of_node);
-	drvdata = match->data;
 	if (drvdata->flags & CAN_TRANSCEIVER_DUAL_CH)
 		num_ch = 2;
+	else
+		num_ch = 1;
 
 	priv = devm_kzalloc(dev, struct_size(priv, can_transceiver_phy, num_ch), GFP_KERNEL);
 	if (!priv)
@@ -179,15 +153,23 @@ static int can_transceiver_phy_probe(struct platform_device *pdev)
 
 	priv->mux_state = mux_state;
 
-	err = device_property_read_u32(dev, "max-bitrate", &max_bitrate);
-	if ((err != -EINVAL) && !max_bitrate)
-		dev_warn(dev, "Invalid value for transceiver max bitrate. Ignoring bitrate limit\n");
+	propname = "max-bitrate";
+	if (device_property_present(dev, propname)) {
+		err = device_property_read_u32(dev, propname, &max_bitrate);
+		if (err)
+			return dev_err_probe(dev, err, "failed to parse %s\n", propname);
+
+		if (max_bitrate == 0)
+			dev_warn(dev, "Invalid value for transceiver max bitrate. Ignoring bitrate limit\n");
+	} else {
+		max_bitrate = 0;
+	}
 
 	for (i = 0; i < num_ch; i++) {
 		can_transceiver_phy = &priv->can_transceiver_phy[i];
 		can_transceiver_phy->priv = priv;
 
-		phy = devm_phy_create(dev, dev->of_node, &can_transceiver_phy_ops);
+		phy = devm_phy_create(dev, NULL, &can_transceiver_phy_ops);
 		if (IS_ERR(phy)) {
 			dev_err(dev, "failed to create can transceiver phy\n");
 			return PTR_ERR(phy);
@@ -231,6 +213,35 @@ static int can_transceiver_phy_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(phy_provider);
 }
 
+static const struct of_device_id can_transceiver_phy_ids[] = {
+	{
+		.compatible = "ti,tcan1042",
+		.data = &tcan1042_drvdata
+	},
+	{
+		.compatible = "ti,tcan1043",
+		.data = &tcan1043_drvdata
+	},
+	{
+		.compatible = "nxp,tja1048",
+		.data = &tja1048_drvdata
+	},
+	{
+		.compatible = "nxp,tja1051",
+		.data = &tja1051_drvdata
+	},
+	{
+		.compatible = "nxp,tja1057",
+		.data = &tja1057_drvdata
+	},
+	{
+		.compatible = "nxp,tjr1443",
+		.data = &tcan1043_drvdata
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, can_transceiver_phy_ids);
+
 static struct platform_driver can_transceiver_phy_driver = {
 	.probe = can_transceiver_phy_probe,
 	.driver = {
diff --git a/drivers/phy/phy-econet-pcie.c b/drivers/phy/phy-econet-pcie.c
new file mode 100644
index 0000000000000..96be69d1ffeb9
--- /dev/null
+++ b/drivers/phy/phy-econet-pcie.c
@@ -0,0 +1,182 @@
+// SPDX-License-Identifier: GPL-2.0+
+/*
+ * Author: Caleb James DeLisle <cjd@cjdns.fr>
+ *	   Ahmed Naseef <naseefkm@gmail.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <linux/bits.h>
+#include <linux/delay.h>
+
+/* Rx detection timing for EN751221: 16*8 clock cycles  */
+#define EN751221_RXDET_VAL		16
+
+/* Rx detection timing when in power mode 3 */
+#define EN75_RXDET_P3_REG		0xa28
+#define EN75_RXDET_P3_MASK		GENMASK(17, 9)
+
+/* Rx detection timing when in power mode 2 */
+#define EN75_RXDET_P2_REG		0xa2c
+#define EN75_RXDET_P2_MASK		GENMASK(8, 0)
+
+/* Rx impedance */
+#define EN75_RX_IMPEDANCE_REG		0xb2c
+#define EN75_RX_IMPEDANCE_MASK		GENMASK(13, 12)
+enum en75_rx_impedance {
+	EN75_RX_IMPEDANCE_100_OHM	= 0,
+	EN75_RX_IMPEDANCE_95_OHM	= 1,
+	EN75_RX_IMPEDANCE_90_OHM	= 2,
+};
+
+/* PLL Invert clock */
+#define EN75_PLL_PH_INV_REG		0x4a0
+#define EN75_PLL_PH_INV_MASK		BIT(5)
+
+struct en75_phy_op {
+	u32 reg;
+	u32 mask;
+	u32 val;
+};
+
+struct en7528_pcie_phy {
+	struct regmap *regmap;
+	const struct en75_phy_op *data;
+};
+
+/* Port 0 PHY: set LCDDS_CLK_PH_INV for PLL operation */
+static const struct en75_phy_op en7528_phy_gen1[] = {
+	{
+		.reg = EN75_PLL_PH_INV_REG,
+		.mask = EN75_PLL_PH_INV_MASK,
+		.val = 1,
+	},
+	{ /* sentinel */ }
+};
+
+/* EN7528 Port 1 PHY: Rx impedance tuning, target R -5 Ohm */
+static const struct en75_phy_op en7528_phy_gen2[] = {
+	{
+		.reg = EN75_RX_IMPEDANCE_REG,
+		.mask = EN75_RX_IMPEDANCE_MASK,
+		.val = EN75_RX_IMPEDANCE_95_OHM,
+	},
+	{ /* sentinel */ }
+};
+
+/* EN751221 Port 1 PHY, set RX detect to 16*8 clock cycles */
+static const struct en75_phy_op en751221_phy_gen2[] = {
+	{
+		.reg = EN75_RXDET_P3_REG,
+		.mask = EN75_RXDET_P3_MASK,
+		.val = EN751221_RXDET_VAL,
+	},
+	{
+		.reg = EN75_RXDET_P2_REG,
+		.mask = EN75_RXDET_P2_MASK,
+		.val = EN751221_RXDET_VAL,
+	},
+	{ /* sentinel */ }
+};
+
+static int en75_pcie_phy_init(struct phy *phy)
+{
+	struct en7528_pcie_phy *ephy = phy_get_drvdata(phy);
+	const struct en75_phy_op *data = ephy->data;
+	int i, ret;
+	u32 val;
+
+	for (i = 0; data[i].mask || data[i].val; i++) {
+		if (i)
+			usleep_range(1000, 2000);
+
+		val = field_prep(data[i].mask, data[i].val);
+
+		ret = regmap_update_bits(ephy->regmap, data[i].reg,
+					 data[i].mask, val);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static const struct phy_ops en75_pcie_phy_ops = {
+	.init	= en75_pcie_phy_init,
+	.owner	= THIS_MODULE,
+};
+
+static int en75_pcie_phy_probe(struct platform_device *pdev)
+{
+	struct regmap_config regmap_config = {
+		.reg_bits = 32,
+		.val_bits = 32,
+		.reg_stride = 4,
+	};
+	struct device *dev = &pdev->dev;
+	const struct en75_phy_op *data;
+	struct phy_provider *provider;
+	struct en7528_pcie_phy *ephy;
+	void __iomem *base;
+	struct phy *phy;
+	int i;
+
+	data = of_device_get_match_data(dev);
+	if (!data)
+		return -EINVAL;
+
+	ephy = devm_kzalloc(dev, sizeof(*ephy), GFP_KERNEL);
+	if (!ephy)
+		return -ENOMEM;
+
+	ephy->data = data;
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return PTR_ERR(base);
+
+	/* Set max_register to highest used register */
+	for (i = 0; data[i].mask || data[i].val; i++)
+		if (data[i].reg > regmap_config.max_register)
+			regmap_config.max_register = data[i].reg;
+
+	ephy->regmap = devm_regmap_init_mmio(dev, base, &regmap_config);
+	if (IS_ERR(ephy->regmap))
+		return PTR_ERR(ephy->regmap);
+
+	phy = devm_phy_create(dev, dev->of_node, &en75_pcie_phy_ops);
+	if (IS_ERR(phy))
+		return PTR_ERR(phy);
+
+	phy_set_drvdata(phy, ephy);
+
+	provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+
+	return PTR_ERR_OR_ZERO(provider);
+}
+
+static const struct of_device_id en75_pcie_phy_ids[] = {
+	{ .compatible = "econet,en7528-pcie-gen1", .data = en7528_phy_gen1 },
+	{ .compatible = "econet,en7528-pcie-gen2", .data = en7528_phy_gen2 },
+	{ .compatible = "econet,en751221-pcie-gen1", .data = en7528_phy_gen1 },
+	{ .compatible = "econet,en751221-pcie-gen2", .data = en751221_phy_gen2 },
+	{ /* sentinel */ }
+};
+MODULE_DEVICE_TABLE(of, en75_pcie_phy_ids);
+
+static struct platform_driver en75_pcie_phy_driver = {
+	.probe = en75_pcie_phy_probe,
+	.driver = {
+		.name = "econet-pcie-phy",
+		.of_match_table = en75_pcie_phy_ids,
+	},
+};
+module_platform_driver(en75_pcie_phy_driver);
+
+MODULE_AUTHOR("Caleb James DeLisle <cjd@cjdns.fr>");
+MODULE_DESCRIPTION("EcoNet PCIe PHY driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/phy-eyeq5-eth.c b/drivers/phy/phy-eyeq5-eth.c
new file mode 100644
index 0000000000000..c03d77c360f74
--- /dev/null
+++ b/drivers/phy/phy-eyeq5-eth.c
@@ -0,0 +1,280 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/auxiliary_bus.h>
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/err.h>
+#include <linux/errno.h>
+#include <linux/gfp_types.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/phy.h>
+#include <linux/phy/phy.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+
+#define EQ5_PHY_COUNT	2
+
+#define EQ5_PHY0_GP	0x128
+#define EQ5_PHY1_GP	0x12c
+#define EQ5_PHY0_SGMII	0x134
+#define EQ5_PHY1_SGMII	0x138
+
+#define EQ5_GP_TX_SWRST_DIS	BIT(0)		// Tx SW reset
+#define EQ5_GP_TX_M_CLKE	BIT(1)		// Tx M clock enable
+#define EQ5_GP_SYS_SWRST_DIS	BIT(2)		// Sys SW reset
+#define EQ5_GP_SYS_M_CLKE	BIT(3)		// Sys clock enable
+#define EQ5_GP_SGMII_MODE	BIT(4)		// SGMII mode
+#define EQ5_GP_RGMII_DRV	GENMASK(8, 5)	// RGMII drive strength
+
+#define EQ5_SGMII_PWR_EN	BIT(0)
+#define EQ5_SGMII_RST_DIS	BIT(1)
+#define EQ5_SGMII_PLL_EN	BIT(2)
+#define EQ5_SGMII_SIG_DET_SW	BIT(3)
+#define EQ5_SGMII_PWR_STATE	BIT(4)
+#define EQ5_SGMII_PLL_ACK	BIT(18)
+#define EQ5_SGMII_PWR_STATE_ACK	GENMASK(24, 20)
+
+/*
+ * Instead of storing a phy_interface_t, we store this enum.
+ *
+ * We do not deal with RGMII timings in this generic PHY driver,
+ * it is all handled inside the net PHY.
+ */
+enum eq5_phy_submode {
+	EQ5_PHY_SUBMODE_SGMII,
+	EQ5_PHY_SUBMODE_RGMII,
+};
+
+struct eq5_phy_inst {
+	struct device		*dev;
+	struct phy		*phy;
+	void __iomem		*gp, *sgmii;
+	enum eq5_phy_submode	submode;
+	bool			sgmii_support;
+};
+
+struct eq5_phy_private {
+	struct eq5_phy_inst	phys[EQ5_PHY_COUNT];
+};
+
+static int eq5_phy_exit(struct phy *phy)
+{
+	struct eq5_phy_inst *inst = phy_get_drvdata(phy);
+
+	writel(0, inst->gp);
+	writel(0, inst->sgmii);
+	udelay(5); /* settling time */
+	return 0;
+}
+
+static int eq5_phy_init(struct phy *phy)
+{
+	struct eq5_phy_inst *inst = phy_get_drvdata(phy);
+	u32 reg;
+
+	/*
+	 * Hardware stops listening to our instructions once it is started.
+	 * It must be reset to reconfigure it.
+	 */
+	eq5_phy_exit(phy);
+
+	reg = EQ5_GP_TX_SWRST_DIS | EQ5_GP_TX_M_CLKE |
+	      EQ5_GP_SYS_SWRST_DIS | EQ5_GP_SYS_M_CLKE |
+	      FIELD_PREP(EQ5_GP_RGMII_DRV, 0x9);
+	writel(reg, inst->gp);
+
+	return 0;
+}
+
+static int eq5_phy_power_on(struct phy *phy)
+{
+	struct eq5_phy_inst *inst = phy_get_drvdata(phy);
+	u32 reg;
+
+	if (inst->submode == EQ5_PHY_SUBMODE_SGMII) {
+		writel(readl(inst->gp) | EQ5_GP_SGMII_MODE, inst->gp);
+
+		reg = EQ5_SGMII_PWR_EN | EQ5_SGMII_RST_DIS | EQ5_SGMII_PLL_EN;
+		writel(reg, inst->sgmii);
+
+		if (readl_poll_timeout(inst->sgmii, reg,
+				       reg & EQ5_SGMII_PLL_ACK, 1, 100)) {
+			dev_err(inst->dev, "PLL timeout\n");
+			return -ETIMEDOUT;
+		}
+
+		reg = readl(inst->sgmii);
+		reg |= EQ5_SGMII_PWR_STATE | EQ5_SGMII_SIG_DET_SW;
+		writel(reg, inst->sgmii);
+	} else {
+		writel(readl(inst->gp) & ~EQ5_GP_SGMII_MODE, inst->gp);
+		writel(0, inst->sgmii);
+	}
+
+	return 0;
+}
+
+static int eq5_phy_power_off(struct phy *phy)
+{
+	struct eq5_phy_inst *inst = phy_get_drvdata(phy);
+
+	writel(readl(inst->gp) & ~EQ5_GP_SGMII_MODE, inst->gp);
+	writel(0, inst->sgmii);
+
+	return 0;
+}
+
+static int eq5_phy_validate(struct phy *phy, enum phy_mode mode, int submode,
+			    union phy_configure_opts *opts)
+{
+	struct eq5_phy_inst *inst = phy_get_drvdata(phy);
+
+	if (mode != PHY_MODE_ETHERNET)
+		return -EINVAL;
+
+	if (phy_interface_mode_is_rgmii(submode))
+		return 0;
+
+	if (inst->sgmii_support && submode == PHY_INTERFACE_MODE_SGMII)
+		return 0;
+
+	return -EINVAL;
+}
+
+static int eq5_phy_set_mode(struct phy *phy, enum phy_mode mode, int submode)
+{
+	struct eq5_phy_inst *inst = phy_get_drvdata(phy);
+	enum eq5_phy_submode target_submode;
+	int ret;
+
+	ret = eq5_phy_validate(phy, mode, submode, NULL);
+	if (ret)
+		return ret;
+
+	if (submode == PHY_INTERFACE_MODE_SGMII)
+		target_submode = EQ5_PHY_SUBMODE_SGMII;
+	else
+		target_submode = EQ5_PHY_SUBMODE_RGMII;
+
+	if (target_submode == inst->submode)
+		return 0;
+
+	inst->submode = target_submode;
+
+	if (phy->power_count) {
+		eq5_phy_init(phy);
+		return eq5_phy_power_on(phy);
+	}
+
+	return 0;
+}
+
+static const struct phy_ops eq5_phy_ops = {
+	.init		= eq5_phy_init,
+	.exit		= eq5_phy_exit,
+	.power_on	= eq5_phy_power_on,
+	.power_off	= eq5_phy_power_off,
+	.set_mode	= eq5_phy_set_mode,
+	.validate	= eq5_phy_validate,
+};
+
+static struct phy *eq5_phy_xlate(struct device *dev,
+				 const struct of_phandle_args *args)
+{
+	struct eq5_phy_private *priv = dev_get_drvdata(dev);
+
+	if (args->args_count != 1 || args->args[0] >= EQ5_PHY_COUNT)
+		return ERR_PTR(-EINVAL);
+
+	return priv->phys[args->args[0]].phy;
+}
+
+static int eq5_phy_probe_phy(struct device *dev, struct eq5_phy_private *priv,
+			     unsigned int index, void __iomem *base,
+			     unsigned int gp, unsigned int sgmii,
+			     bool sgmii_support)
+{
+	struct eq5_phy_inst *inst = &priv->phys[index];
+	struct phy *phy;
+
+	phy = devm_phy_create(dev, dev->of_node, &eq5_phy_ops);
+	if (IS_ERR(phy))
+		return dev_err_probe(dev, PTR_ERR(phy),
+				     "failed to create PHY %u\n", index);
+
+	inst->dev = dev;
+	inst->phy = phy;
+	inst->gp = base + gp;
+	inst->sgmii = base + sgmii;
+	inst->sgmii_support = sgmii_support;
+	phy_set_drvdata(phy, inst);
+
+	/*
+	 * Init inst->submode based on probe hardware state, allowing
+	 * consumers to power us on without first setting the mode.
+	 */
+	if (sgmii_support && (readl(inst->gp) & EQ5_GP_SGMII_MODE))
+		inst->submode = EQ5_PHY_SUBMODE_SGMII;
+	else
+		inst->submode = EQ5_PHY_SUBMODE_RGMII;
+
+	return 0;
+}
+
+static int eq5_phy_probe(struct auxiliary_device *adev,
+			 const struct auxiliary_device_id *id)
+{
+	struct device *dev = &adev->dev;
+	struct phy_provider *provider;
+	struct eq5_phy_private *priv;
+	void __iomem *base;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, priv);
+
+	base = (void __iomem *)dev_get_platdata(dev);
+
+	ret = eq5_phy_probe_phy(dev, priv, 0, base, EQ5_PHY0_GP,
+				EQ5_PHY0_SGMII, true);
+	if (ret)
+		return ret;
+
+	ret = eq5_phy_probe_phy(dev, priv, 1, base, EQ5_PHY1_GP,
+				EQ5_PHY1_SGMII, false);
+	if (ret)
+		return ret;
+
+	provider = devm_of_phy_provider_register(dev, eq5_phy_xlate);
+	if (IS_ERR(provider))
+		return dev_err_probe(dev, PTR_ERR(provider),
+				     "registering provider failed\n");
+
+	return 0;
+}
+
+static const struct auxiliary_device_id eq5_phy_id_table[] = {
+	{ .name = "clk_eyeq.phy" },
+	{}
+};
+MODULE_DEVICE_TABLE(auxiliary, eq5_phy_id_table);
+
+static struct auxiliary_driver eq5_phy_driver = {
+	.probe = eq5_phy_probe,
+	.id_table = eq5_phy_id_table,
+};
+module_auxiliary_driver(eq5_phy_driver);
+
+MODULE_DESCRIPTION("EyeQ5 Ethernet PHY driver");
+MODULE_AUTHOR("Théo Lebrun <theo.lebrun@bootlin.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/qualcomm/phy-qcom-edp.c b/drivers/phy/qualcomm/phy-qcom-edp.c
index 7372de05a0b81..a3c893f72908f 100644
--- a/drivers/phy/qualcomm/phy-qcom-edp.c
+++ b/drivers/phy/qualcomm/phy-qcom-edp.c
@@ -81,13 +81,15 @@ struct phy_ver_ops {
 	int (*com_clk_fwd_cfg)(const struct qcom_edp *edp);
 	int (*com_configure_pll)(const struct qcom_edp *edp);
 	int (*com_configure_ssc)(const struct qcom_edp *edp);
+	int (*com_ldo_config)(const struct qcom_edp *edp);
 };
 
 struct qcom_edp_phy_cfg {
 	bool is_edp;
 	const u8 *aux_cfg;
 	const u8 *vco_div_cfg;
-	const struct qcom_edp_swing_pre_emph_cfg *swing_pre_emph_cfg;
+	const struct qcom_edp_swing_pre_emph_cfg *dp_swing_pre_emph_cfg;
+	const struct qcom_edp_swing_pre_emph_cfg *edp_swing_pre_emph_cfg;
 	const struct phy_ver_ops *ver_ops;
 };
 
@@ -116,17 +118,17 @@ struct qcom_edp {
 };
 
 static const u8 dp_swing_hbr_rbr[4][4] = {
-	{ 0x08, 0x0f, 0x16, 0x1f },
+	{ 0x07, 0x0f, 0x16, 0x1f },
 	{ 0x11, 0x1e, 0x1f, 0xff },
 	{ 0x16, 0x1f, 0xff, 0xff },
 	{ 0x1f, 0xff, 0xff, 0xff }
 };
 
 static const u8 dp_pre_emp_hbr_rbr[4][4] = {
-	{ 0x00, 0x0d, 0x14, 0x1a },
+	{ 0x00, 0x0e, 0x15, 0x1a },
 	{ 0x00, 0x0e, 0x15, 0xff },
 	{ 0x00, 0x0e, 0xff, 0xff },
-	{ 0x03, 0xff, 0xff, 0xff }
+	{ 0x04, 0xff, 0xff, 0xff }
 };
 
 static const u8 dp_swing_hbr2_hbr3[4][4] = {
@@ -150,6 +152,47 @@ static const struct qcom_edp_swing_pre_emph_cfg dp_phy_swing_pre_emph_cfg = {
 	.pre_emphasis_hbr3_hbr2 = &dp_pre_emp_hbr2_hbr3,
 };
 
+static const u8 dp_pre_emp_hbr_rbr_v8[4][4] = {
+	{ 0x00, 0x0e, 0x15, 0x1a },
+	{ 0x00, 0x0e, 0x15, 0xff },
+	{ 0x00, 0x0e, 0xff, 0xff },
+	{ 0x00, 0xff, 0xff, 0xff }
+};
+
+static const struct qcom_edp_swing_pre_emph_cfg dp_phy_swing_pre_emph_cfg_v8 = {
+	.swing_hbr_rbr = &dp_swing_hbr_rbr,
+	.swing_hbr3_hbr2 = &dp_swing_hbr2_hbr3,
+	.pre_emphasis_hbr_rbr = &dp_pre_emp_hbr_rbr_v8,
+	.pre_emphasis_hbr3_hbr2 = &dp_pre_emp_hbr2_hbr3,
+};
+
+static const u8 dp_swing_hbr2_hbr3_v2[4][4] = {
+	{ 0x27, 0x2f, 0x36, 0xff },
+	{ 0x31, 0x3e, 0x3f, 0xff },
+	{ 0x3a, 0x3f, 0xff, 0xff },
+	{ 0xff, 0xff, 0xff, 0xff }
+};
+
+static const u8 dp_pre_emp_hbr2_hbr3_v2[4][4] = {
+	{ 0x20, 0x2e, 0x35, 0xff },
+	{ 0x20, 0x2e, 0x35, 0xff },
+	{ 0x20, 0x2e, 0xff, 0xff },
+	{ 0xff, 0xff, 0xff, 0xff }
+};
+
+static const struct qcom_edp_swing_pre_emph_cfg dp_phy_swing_pre_emph_cfg_v2 = {
+	/*
+	 * NOTE: The HPG does not specify a separate swing_hbr_rbr table.
+	 * Reuse the HBR2/HBR3 table for now.
+	 *
+	 * TODO: Update this once the HPG explicitly defines RBR/HBR swing values.
+	 */
+	.swing_hbr_rbr = &dp_swing_hbr2_hbr3_v2,
+	.swing_hbr3_hbr2 = &dp_swing_hbr2_hbr3_v2,
+	.pre_emphasis_hbr_rbr = &dp_pre_emp_hbr2_hbr3_v2,
+	.pre_emphasis_hbr3_hbr2 = &dp_pre_emp_hbr2_hbr3_v2,
+};
+
 static const u8 edp_swing_hbr_rbr[4][4] = {
 	{ 0x07, 0x0f, 0x16, 0x1f },
 	{ 0x0d, 0x16, 0x1e, 0xff },
@@ -158,7 +201,7 @@ static const u8 edp_swing_hbr_rbr[4][4] = {
 };
 
 static const u8 edp_pre_emp_hbr_rbr[4][4] = {
-	{ 0x05, 0x12, 0x17, 0x1d },
+	{ 0x05, 0x11, 0x17, 0x1d },
 	{ 0x05, 0x11, 0x18, 0xff },
 	{ 0x06, 0x11, 0xff, 0xff },
 	{ 0x00, 0xff, 0xff, 0xff }
@@ -172,10 +215,10 @@ static const u8 edp_swing_hbr2_hbr3[4][4] = {
 };
 
 static const u8 edp_pre_emp_hbr2_hbr3[4][4] = {
-	{ 0x08, 0x11, 0x17, 0x1b },
-	{ 0x00, 0x0c, 0x13, 0xff },
-	{ 0x05, 0x10, 0xff, 0xff },
-	{ 0x00, 0xff, 0xff, 0xff }
+	{ 0x0c, 0x15, 0x19, 0x1e },
+	{ 0x0b, 0x15, 0x19, 0xff },
+	{ 0x0e, 0x14, 0xff, 0xff },
+	{ 0x0d, 0xff, 0xff, 0xff }
 };
 
 static const struct qcom_edp_swing_pre_emph_cfg edp_phy_swing_pre_emph_cfg = {
@@ -193,25 +236,46 @@ static const u8 edp_phy_vco_div_cfg_v4[4] = {
 	0x01, 0x01, 0x02, 0x00,
 };
 
-static const u8 edp_pre_emp_hbr_rbr_v5[4][4] = {
-	{ 0x05, 0x11, 0x17, 0x1d },
+static const u8 edp_pre_emp_hbr_rbr_v2[4][4] = {
+	{ 0x05, 0x12, 0x17, 0x1d },
 	{ 0x05, 0x11, 0x18, 0xff },
 	{ 0x06, 0x11, 0xff, 0xff },
 	{ 0x00, 0xff, 0xff, 0xff }
 };
 
-static const u8 edp_pre_emp_hbr2_hbr3_v5[4][4] = {
+static const u8 edp_pre_emp_hbr2_hbr3_v2[4][4] = {
 	{ 0x0c, 0x15, 0x19, 0x1e },
-	{ 0x0b, 0x15, 0x19, 0xff },
+	{ 0x08, 0x15, 0x19, 0xff },
 	{ 0x0e, 0x14, 0xff, 0xff },
 	{ 0x0d, 0xff, 0xff, 0xff }
 };
 
-static const struct qcom_edp_swing_pre_emph_cfg edp_phy_swing_pre_emph_cfg_v5 = {
+static const struct qcom_edp_swing_pre_emph_cfg edp_phy_swing_pre_emph_cfg_v2 = {
 	.swing_hbr_rbr = &edp_swing_hbr_rbr,
 	.swing_hbr3_hbr2 = &edp_swing_hbr2_hbr3,
-	.pre_emphasis_hbr_rbr = &edp_pre_emp_hbr_rbr_v5,
-	.pre_emphasis_hbr3_hbr2 = &edp_pre_emp_hbr2_hbr3_v5,
+	.pre_emphasis_hbr_rbr = &edp_pre_emp_hbr_rbr_v2,
+	.pre_emphasis_hbr3_hbr2 = &edp_pre_emp_hbr2_hbr3_v2,
+};
+
+static const u8 edp_swing_hbr2_hbr3_v3[4][4] = {
+	{ 0x06, 0x11, 0x16, 0x1b },
+	{ 0x0b, 0x19, 0x1f, 0xff },
+	{ 0x18, 0x1f, 0xff, 0xff },
+	{ 0x1f, 0xff, 0xff, 0xff }
+};
+
+static const u8 edp_pre_emp_hbr2_hbr3_v3[4][4] = {
+	{ 0x0c, 0x15, 0x19, 0x1e },
+	{ 0x09, 0x14, 0x19, 0xff },
+	{ 0x0f, 0x14, 0xff, 0xff },
+	{ 0x0d, 0xff, 0xff, 0xff }
+};
+
+static const struct qcom_edp_swing_pre_emph_cfg edp_phy_swing_pre_emph_cfg_v3 = {
+	.swing_hbr_rbr = &edp_swing_hbr_rbr,
+	.swing_hbr3_hbr2 = &edp_swing_hbr2_hbr3_v3,
+	.pre_emphasis_hbr_rbr = &edp_pre_emp_hbr_rbr,
+	.pre_emphasis_hbr3_hbr2 = &edp_pre_emp_hbr2_hbr3_v3,
 };
 
 static const u8 edp_phy_aux_cfg_v5[DP_AUX_CFG_SIZE] = {
@@ -262,12 +326,7 @@ static int qcom_edp_phy_init(struct phy *phy)
 	       DP_PHY_PD_CTL_PLL_PWRDN | DP_PHY_PD_CTL_DP_CLAMP_EN,
 	       edp->edp + DP_PHY_PD_CTL);
 
-	/*
-	 * TODO: Re-work the conditions around setting the cfg8 value
-	 * when more information becomes available about why this is
-	 * even needed.
-	 */
-	if (edp->cfg->swing_pre_emph_cfg && !edp->is_edp)
+	if (!edp->is_edp)
 		aux_cfg[8] = 0xb7;
 
 	writel(0xfc, edp->edp + DP_PHY_MODE);
@@ -291,19 +350,18 @@ static int qcom_edp_phy_init(struct phy *phy)
 
 static int qcom_edp_set_voltages(struct qcom_edp *edp, const struct phy_configure_opts_dp *dp_opts)
 {
-	const struct qcom_edp_swing_pre_emph_cfg *cfg = edp->cfg->swing_pre_emph_cfg;
+	const struct qcom_edp_swing_pre_emph_cfg *cfg;
 	unsigned int v_level = 0;
 	unsigned int p_level = 0;
-	u8 ldo_config;
+	int ret;
 	u8 swing;
 	u8 emph;
 	int i;
 
-	if (!cfg)
-		return 0;
-
 	if (edp->is_edp)
-		cfg = &edp_phy_swing_pre_emph_cfg;
+		cfg = edp->cfg->edp_swing_pre_emph_cfg;
+	else
+		cfg = edp->cfg->dp_swing_pre_emph_cfg;
 
 	for (i = 0; i < dp_opts->lanes; i++) {
 		v_level = max(v_level, dp_opts->voltage[i]);
@@ -321,13 +379,13 @@ static int qcom_edp_set_voltages(struct qcom_edp *edp, const struct phy_configur
 	if (swing == 0xff || emph == 0xff)
 		return -EINVAL;
 
-	ldo_config = edp->is_edp ? 0x0 : 0x1;
+	ret = edp->cfg->ver_ops->com_ldo_config(edp);
+	if (ret)
+		return ret;
 
-	writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
 	writel(swing, edp->tx0 + TXn_TX_DRV_LVL);
 	writel(emph, edp->tx0 + TXn_TX_EMP_POST1_LVL);
 
-	writel(ldo_config, edp->tx1 + TXn_LDO_CONFIG);
 	writel(swing, edp->tx1 + TXn_TX_DRV_LVL);
 	writel(emph, edp->tx1 + TXn_TX_EMP_POST1_LVL);
 
@@ -551,6 +609,52 @@ static int qcom_edp_com_configure_pll_v4(const struct qcom_edp *edp)
 	return 0;
 }
 
+static int qcom_edp_ldo_config_v3(const struct qcom_edp *edp)
+{
+	const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts;
+	u32 ldo_config;
+
+	if (!edp->is_edp)
+		ldo_config = 0x0;
+	else if (dp_opts->link_rate <= 2700)
+		ldo_config = 0x81;
+	else
+		ldo_config = 0x41;
+
+	writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
+	writel(dp_opts->lanes > 2 ? ldo_config : 0x00, edp->tx1 + TXn_LDO_CONFIG);
+
+	return 0;
+}
+
+static int qcom_edp_ldo_config_v4(const struct qcom_edp *edp)
+{
+	const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts;
+	u32 ldo_config;
+
+	if (!edp->is_edp)
+		ldo_config = 0x0;
+	else if (dp_opts->link_rate <= 2700)
+		ldo_config = 0xc1;
+	else
+		ldo_config = 0x81;
+
+	writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
+	writel(dp_opts->lanes > 2 ? ldo_config : 0x00, edp->tx1 + TXn_LDO_CONFIG);
+
+	return 0;
+}
+
+static const struct phy_ver_ops qcom_edp_phy_ops_v3 = {
+	.com_power_on		= qcom_edp_phy_power_on_v4,
+	.com_resetsm_cntrl	= qcom_edp_phy_com_resetsm_cntrl_v4,
+	.com_bias_en_clkbuflr	= qcom_edp_com_bias_en_clkbuflr_v4,
+	.com_clk_fwd_cfg	= qcom_edp_com_clk_fwd_cfg_v4,
+	.com_configure_pll	= qcom_edp_com_configure_pll_v4,
+	.com_configure_ssc	= qcom_edp_com_configure_ssc_v4,
+	.com_ldo_config		= qcom_edp_ldo_config_v3,
+};
+
 static const struct phy_ver_ops qcom_edp_phy_ops_v4 = {
 	.com_power_on		= qcom_edp_phy_power_on_v4,
 	.com_resetsm_cntrl	= qcom_edp_phy_com_resetsm_cntrl_v4,
@@ -558,26 +662,39 @@ static const struct phy_ver_ops qcom_edp_phy_ops_v4 = {
 	.com_clk_fwd_cfg	= qcom_edp_com_clk_fwd_cfg_v4,
 	.com_configure_pll	= qcom_edp_com_configure_pll_v4,
 	.com_configure_ssc	= qcom_edp_com_configure_ssc_v4,
+	.com_ldo_config		= qcom_edp_ldo_config_v4,
 };
 
 static const struct qcom_edp_phy_cfg sa8775p_dp_phy_cfg = {
 	.is_edp = false,
 	.aux_cfg = edp_phy_aux_cfg_v5,
 	.vco_div_cfg = edp_phy_vco_div_cfg_v4,
-	.swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg_v5,
+	.dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+	.edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
 	.ver_ops = &qcom_edp_phy_ops_v4,
 };
 
 static const struct qcom_edp_phy_cfg sc7280_dp_phy_cfg = {
 	.aux_cfg = edp_phy_aux_cfg_v4,
 	.vco_div_cfg = edp_phy_vco_div_cfg_v4,
-	.ver_ops = &qcom_edp_phy_ops_v4,
+	.dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+	.edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg_v3,
+	.ver_ops = &qcom_edp_phy_ops_v3,
+};
+
+static const struct qcom_edp_phy_cfg sc8180x_dp_phy_cfg = {
+	.aux_cfg = edp_phy_aux_cfg_v4,
+	.vco_div_cfg = edp_phy_vco_div_cfg_v4,
+	.dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg_v2,
+	.edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg_v2,
+	.ver_ops = &qcom_edp_phy_ops_v3,
 };
 
 static const struct qcom_edp_phy_cfg sc8280xp_dp_phy_cfg = {
 	.aux_cfg = edp_phy_aux_cfg_v4,
 	.vco_div_cfg = edp_phy_vco_div_cfg_v4,
-	.swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+	.dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+	.edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
 	.ver_ops = &qcom_edp_phy_ops_v4,
 };
 
@@ -585,7 +702,8 @@ static const struct qcom_edp_phy_cfg sc8280xp_edp_phy_cfg = {
 	.is_edp = true,
 	.aux_cfg = edp_phy_aux_cfg_v4,
 	.vco_div_cfg = edp_phy_vco_div_cfg_v4,
-	.swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
+	.dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+	.edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
 	.ver_ops = &qcom_edp_phy_ops_v4,
 };
 
@@ -754,6 +872,24 @@ static int qcom_edp_com_configure_pll_v6(const struct qcom_edp *edp)
 	return 0;
 }
 
+static int qcom_edp_ldo_config_v6(const struct qcom_edp *edp)
+{
+	const struct phy_configure_opts_dp *dp_opts = &edp->dp_opts;
+	u32 ldo_config;
+
+	if (!edp->is_edp)
+		ldo_config = 0x0;
+	else if (dp_opts->link_rate <= 2700)
+		ldo_config = 0x51;
+	else
+		ldo_config = 0x91;
+
+	writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
+	writel(dp_opts->lanes > 2 ? ldo_config : 0x00, edp->tx1 + TXn_LDO_CONFIG);
+
+	return 0;
+}
+
 static const struct phy_ver_ops qcom_edp_phy_ops_v6 = {
 	.com_power_on		= qcom_edp_phy_power_on_v6,
 	.com_resetsm_cntrl	= qcom_edp_phy_com_resetsm_cntrl_v6,
@@ -761,12 +897,14 @@ static const struct phy_ver_ops qcom_edp_phy_ops_v6 = {
 	.com_clk_fwd_cfg	= qcom_edp_com_clk_fwd_cfg_v4,
 	.com_configure_pll	= qcom_edp_com_configure_pll_v6,
 	.com_configure_ssc	= qcom_edp_com_configure_ssc_v6,
+	.com_ldo_config		= qcom_edp_ldo_config_v6,
 };
 
 static struct qcom_edp_phy_cfg x1e80100_phy_cfg = {
 	.aux_cfg = edp_phy_aux_cfg_v4,
 	.vco_div_cfg = edp_phy_vco_div_cfg_v4,
-	.swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+	.dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg,
+	.edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
 	.ver_ops = &qcom_edp_phy_ops_v6,
 };
 
@@ -940,12 +1078,14 @@ static const struct phy_ver_ops qcom_edp_phy_ops_v8 = {
 	.com_clk_fwd_cfg	= qcom_edp_com_clk_fwd_cfg_v8,
 	.com_configure_pll	= qcom_edp_com_configure_pll_v8,
 	.com_configure_ssc	= qcom_edp_com_configure_ssc_v8,
+	.com_ldo_config		= qcom_edp_ldo_config_v6,
 };
 
 static struct qcom_edp_phy_cfg glymur_phy_cfg = {
 	.aux_cfg = edp_phy_aux_cfg_v8,
 	.vco_div_cfg = edp_phy_vco_div_cfg_v8,
-	.swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg_v5,
+	.dp_swing_pre_emph_cfg = &dp_phy_swing_pre_emph_cfg_v8,
+	.edp_swing_pre_emph_cfg = &edp_phy_swing_pre_emph_cfg,
 	.ver_ops = &qcom_edp_phy_ops_v8,
 };
 
@@ -954,7 +1094,6 @@ static int qcom_edp_phy_power_on(struct phy *phy)
 	const struct qcom_edp *edp = phy_get_drvdata(phy);
 	u32 bias0_en, drvr0_en, bias1_en, drvr1_en;
 	unsigned long pixel_freq;
-	u8 ldo_config = 0x0;
 	int ret;
 	u32 val;
 	u8 cfg1;
@@ -963,11 +1102,10 @@ static int qcom_edp_phy_power_on(struct phy *phy)
 	if (ret)
 		return ret;
 
-	if (edp->cfg->swing_pre_emph_cfg && !edp->is_edp)
-		ldo_config = 0x1;
+	ret = edp->cfg->ver_ops->com_ldo_config(edp);
+	if (ret)
+		return ret;
 
-	writel(ldo_config, edp->tx0 + TXn_LDO_CONFIG);
-	writel(ldo_config, edp->tx1 + TXn_LDO_CONFIG);
 	writel(0x00, edp->tx0 + TXn_LANE_MODE_1);
 	writel(0x00, edp->tx1 + TXn_LANE_MODE_1);
 
@@ -1347,7 +1485,7 @@ static const struct of_device_id qcom_edp_phy_match_table[] = {
 	{ .compatible = "qcom,glymur-dp-phy", .data = &glymur_phy_cfg, },
 	{ .compatible = "qcom,sa8775p-edp-phy", .data = &sa8775p_dp_phy_cfg, },
 	{ .compatible = "qcom,sc7280-edp-phy", .data = &sc7280_dp_phy_cfg, },
-	{ .compatible = "qcom,sc8180x-edp-phy", .data = &sc7280_dp_phy_cfg, },
+	{ .compatible = "qcom,sc8180x-edp-phy", .data = &sc8180x_dp_phy_cfg, },
 	{ .compatible = "qcom,sc8280xp-dp-phy", .data = &sc8280xp_dp_phy_cfg, },
 	{ .compatible = "qcom,sc8280xp-edp-phy", .data = &sc8280xp_edp_phy_cfg, },
 	{ .compatible = "qcom,x1e80100-dp-phy", .data = &x1e80100_phy_cfg, },
diff --git a/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c b/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c
index 68f1ba8fec4ad..0bec8657149c9 100644
--- a/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c
+++ b/drivers/phy/qualcomm/phy-qcom-m31-eusb2.c
@@ -285,7 +285,7 @@ static int m31eusb2_phy_probe(struct platform_device *pdev)
 
 	phy_set_drvdata(phy->phy, phy);
 
-	phy->repeater = devm_of_phy_get_by_index(dev, dev->of_node, 0);
+	phy->repeater = devm_phy_optional_get(dev, NULL);
 	if (IS_ERR(phy->repeater))
 		return dev_err_probe(dev, PTR_ERR(phy->repeater),
 				     "failed to get repeater\n");
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c
index 93f1aa10d4008..cdcfad2e86b1d 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-combo.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-combo.c
@@ -3691,6 +3691,13 @@ static int qmp_combo_com_init(struct qmp_combo *qmp, bool force)
 	if (ret)
 		goto err_assert_reset;
 
+	/* In DP-only mode, the pipe clk is still required for USB2 */
+	ret = clk_prepare_enable(qmp->pipe_clk);
+	if (ret) {
+		dev_err(qmp->dev, "pipe_clk enable failed err=%d\n", ret);
+		goto err_disable_clocks;
+	}
+
 	qphy_setbits(com, QPHY_V3_DP_COM_POWER_DOWN_CTRL, SW_PWRDN);
 
 	/* override hardware control for reset of qmp phy */
@@ -3749,6 +3756,8 @@ static int qmp_combo_com_init(struct qmp_combo *qmp, bool force)
 
 	return 0;
 
+err_disable_clocks:
+	clk_bulk_disable_unprepare(qmp->num_clks, qmp->clks);
 err_assert_reset:
 	reset_control_bulk_assert(cfg->num_resets, qmp->resets);
 err_disable_regulators:
@@ -3768,6 +3777,7 @@ static int qmp_combo_com_exit(struct qmp_combo *qmp, bool force)
 
 	reset_control_bulk_assert(cfg->num_resets, qmp->resets);
 
+	clk_disable_unprepare(qmp->pipe_clk);
 	clk_bulk_disable_unprepare(qmp->num_clks, qmp->clks);
 
 	regulator_bulk_disable(cfg->num_vregs, qmp->vregs);
@@ -3871,12 +3881,6 @@ static int qmp_combo_usb_power_on(struct phy *phy)
 
 	qmp_configure(qmp->dev, serdes, cfg->serdes_tbl, cfg->serdes_tbl_num);
 
-	ret = clk_prepare_enable(qmp->pipe_clk);
-	if (ret) {
-		dev_err(qmp->dev, "pipe_clk enable failed err=%d\n", ret);
-		return ret;
-	}
-
 	/* Tx, Rx, and PCS configurations */
 	qmp_configure_lane(qmp->dev, tx, cfg->tx_tbl, cfg->tx_tbl_num, 1);
 	qmp_configure_lane(qmp->dev, tx2, cfg->tx_tbl, cfg->tx_tbl_num, 2);
@@ -3922,8 +3926,6 @@ static int qmp_combo_usb_power_off(struct phy *phy)
 	struct qmp_combo *qmp = phy_get_drvdata(phy);
 	const struct qmp_phy_cfg *cfg = qmp->cfg;
 
-	clk_disable_unprepare(qmp->pipe_clk);
-
 	/* PHY reset */
 	qphy_setbits(qmp->pcs, cfg->regs[QPHY_SW_RESET], SW_RESET);
 
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.c
index a7c65cfe31dfb..37e96493b722a 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie-msm8996.c
@@ -59,67 +59,67 @@ static const unsigned int pciephy_regs_layout[QPHY_LAYOUT_SIZE] = {
 };
 
 static const struct qmp_phy_init_tbl msm8996_pcie_serdes_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x1c),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_ENABLE1, 0x10),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x33),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_EN, 0x42),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER1, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER2, 0x1f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x09),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x82),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x03),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x1a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x33),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_BUF_ENABLE, 0x1f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x04),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_EN_CENTER, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER1, 0x31),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER2, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER1, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER2, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE1, 0x2f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE2, 0x19),
-	QMP_PHY_INIT_CFG(QSERDES_COM_RESCODE_DIV_NUM, 0x15),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TRIM, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_EP_DIV, 0x19),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_ENABLE1, 0x10),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_RESCODE_DIV_NUM, 0x40),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x1c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_ENABLE1, 0x10),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x33),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_EN, 0x42),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER1, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER2, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x09),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x1a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x33),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_BUF_ENABLE, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_EN_CENTER, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER2, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER1, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE1, 0x2f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE2, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_RESCODE_DIV_NUM, 0x15),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TRIM, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_EP_DIV, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_ENABLE1, 0x10),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_RESCODE_DIV_NUM, 0x40),
 };
 
 static const struct qmp_phy_init_tbl msm8996_pcie_tx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
-	QMP_PHY_INIT_CFG(QSERDES_TX_LANE_MODE, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_LANE_MODE, 0x06),
 };
 
 static const struct qmp_phy_init_tbl msm8996_pcie_rx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_ENABLES, 0x1c),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL3, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL4, 0xdb),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_BAND, 0x18),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN, 0x04),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN_HALF, 0x04),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x4b),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_DEGLITCH_CNTRL, 0x14),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_LVL, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_ENABLES, 0x1c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL4, 0xdb),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_BAND, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_GAIN, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_GAIN_HALF, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x4b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_LVL, 0x19),
 };
 
 static const struct qmp_phy_init_tbl msm8996_pcie_pcs_tbl[] = {
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c
index fed2fc9bb3110..75afbd15aaf45 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-pcie.c
@@ -309,65 +309,65 @@ static const struct qmp_phy_init_tbl ipq6018_pcie_pcs_misc_tbl[] = {
 };
 
 static const struct qmp_phy_init_tbl ipq8074_pcie_serdes_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x18),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_ENABLE1, 0x10),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TRIM, 0xf),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_EN, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER1, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER2, 0x1f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x6),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0xf),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x20),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV, 0xa),
-	QMP_PHY_INIT_CFG(QSERDES_COM_RESETSM_CNTRL, 0x20),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0xa),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0xa),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x82),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x3),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0xD),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0xD04),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x33),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x2),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_BUF_ENABLE, 0x1f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0xb),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE0, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CTRL_BY_PSM, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_EN_CENTER, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER1, 0x31),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER2, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER1, 0x2),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER2, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE1, 0x2f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE2, 0x19),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_EP_DIV, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_ENABLE1, 0x10),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TRIM, 0xf),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_EN, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER1, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER2, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x6),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0xf),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_RESETSM_CNTRL, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x3),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0xD),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0xD04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x33),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_BUF_ENABLE, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0xb),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE0, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CTRL_BY_PSM, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_EN_CENTER, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER2, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER1, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER2, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE1, 0x2f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE2, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_EP_DIV, 0x19),
 };
 
 static const struct qmp_phy_init_tbl ipq8074_pcie_tx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
-	QMP_PHY_INIT_CFG(QSERDES_TX_LANE_MODE, 0x6),
-	QMP_PHY_INIT_CFG(QSERDES_TX_RES_CODE_LANE_OFFSET, 0x2),
-	QMP_PHY_INIT_CFG(QSERDES_TX_RCV_DETECT_LVL_2, 0x12),
-	QMP_PHY_INIT_CFG(QSERDES_TX_TX_EMP_POST1_LVL, 0x36),
-	QMP_PHY_INIT_CFG(QSERDES_TX_SLEW_CNTL, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_LANE_MODE, 0x6),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_RES_CODE_LANE_OFFSET, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_RCV_DETECT_LVL_2, 0x12),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_TX_EMP_POST1_LVL, 0x36),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_SLEW_CNTL, 0x0a),
 };
 
 static const struct qmp_phy_init_tbl ipq8074_pcie_rx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_ENABLES, 0x1c),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_DEGLITCH_CNTRL, 0x14),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL3, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL4, 0xdb),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x4b),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN, 0x4),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_ENABLES, 0x1c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL4, 0xdb),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x4b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_GAIN, 0x4),
 };
 
 static const struct qmp_phy_init_tbl ipq8074_pcie_pcs_tbl[] = {
@@ -752,65 +752,65 @@ static const struct qmp_phy_init_tbl ipq9574_gen3x2_pcie_pcs_misc_tbl[] = {
 };
 
 static const struct qmp_phy_init_tbl qcs615_pcie_serdes_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x18),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_ENABLE1, 0x10),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TRIM, 0xf),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_EN, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER1, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER2, 0x1f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x6),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0xf),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x20),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV, 0xa),
-	QMP_PHY_INIT_CFG(QSERDES_COM_RESETSM_CNTRL, 0x20),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x9),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x4),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x82),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x3),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0xd),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0x04),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x35),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x2),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_BUF_ENABLE, 0x1f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x4),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x30),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE0, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CTRL_BY_PSM, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0xa),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_EN_CENTER, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER1, 0x31),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER2, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER1, 0x2),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER2, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE1, 0x2f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE2, 0x19),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_EP_DIV, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_ENABLE1, 0x10),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TRIM, 0xf),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_EN, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER1, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER2, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x6),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0xf),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_RESETSM_CNTRL, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x9),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0x4),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x3),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0xd),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x35),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_BUF_ENABLE, 0x1f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x4),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE0, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CTRL_BY_PSM, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0xa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_EN_CENTER, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER2, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER1, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER2, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE1, 0x2f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE2, 0x19),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_EP_DIV, 0x19),
 };
 
 static const struct qmp_phy_init_tbl qcs615_pcie_rx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_ENABLES, 0x1c),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_DEGLITCH_CNTRL, 0x14),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2, 0x1),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL3, 0x0),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL4, 0xdb),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x4b),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN, 0x4),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN_HALF, 0x4),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_ENABLES, 0x1c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2, 0x1),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL4, 0xdb),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x4b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_GAIN, 0x4),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_GAIN_HALF, 0x4),
 };
 
 static const struct qmp_phy_init_tbl qcs615_pcie_tx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
-	QMP_PHY_INIT_CFG(QSERDES_TX_LANE_MODE, 0x6),
-	QMP_PHY_INIT_CFG(QSERDES_TX_RES_CODE_LANE_OFFSET, 0x2),
-	QMP_PHY_INIT_CFG(QSERDES_TX_RCV_DETECT_LVL_2, 0x12),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_LANE_MODE, 0x6),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_RES_CODE_LANE_OFFSET, 0x2),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_RCV_DETECT_LVL_2, 0x12),
 };
 
 static const struct qmp_phy_init_tbl qcs615_pcie_pcs_tbl[] = {
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com-v2.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com-v2.h
index 3ea1884f35dd5..cb599c113189c 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com-v2.h
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com-v2.h
@@ -34,6 +34,7 @@
 #define QSERDES_V2_COM_LOCK_CMP3_MODE1			0x060
 #define QSERDES_V2_COM_EP_CLOCK_DETECT_CTR		0x068
 #define QSERDES_V2_COM_SYSCLK_DET_COMP_STATUS		0x06c
+#define QSERDES_V2_COM_BG_TRIM				0x070
 #define QSERDES_V2_COM_CLK_EP_DIV			0x074
 #define QSERDES_V2_COM_CP_CTRL_MODE0			0x078
 #define QSERDES_V2_COM_CP_CTRL_MODE1			0x07c
@@ -47,6 +48,7 @@
 #define QSERDES_V2_COM_CML_SYSCLK_SEL			0x0b0
 #define QSERDES_V2_COM_RESETSM_CNTRL			0x0b4
 #define QSERDES_V2_COM_RESETSM_CNTRL2			0x0b8
+#define QSERDES_V2_COM_RESCODE_DIV_NUM			0x0c4
 #define QSERDES_V2_COM_LOCK_CMP_EN			0x0c8
 #define QSERDES_V2_COM_LOCK_CMP_CFG			0x0cc
 #define QSERDES_V2_COM_DEC_START_MODE0			0x0d0
@@ -83,6 +85,7 @@
 #define QSERDES_V2_COM_RESTRIM_CODE_STATUS		0x164
 #define QSERDES_V2_COM_PLLCAL_CODE1_STATUS		0x168
 #define QSERDES_V2_COM_PLLCAL_CODE2_STATUS		0x16c
+#define QSERDES_V2_COM_BG_CTRL				0x170
 #define QSERDES_V2_COM_CLK_SELECT			0x174
 #define QSERDES_V2_COM_HSCLK_SEL			0x178
 #define QSERDES_V2_COM_INTEGLOOP_BINCODE_STATUS		0x17c
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com.h
deleted file mode 100644
index 7fa5363feeb9b..0000000000000
--- a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-com.h
+++ /dev/null
@@ -1,140 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2017, The Linux Foundation. All rights reserved.
- */
-
-#ifndef QCOM_PHY_QMP_QSERDES_COM_H_
-#define QCOM_PHY_QMP_QSERDES_COM_H_
-
-/* Only for QMP V2 PHY - QSERDES COM registers */
-#define QSERDES_COM_ATB_SEL1				0x000
-#define QSERDES_COM_ATB_SEL2				0x004
-#define QSERDES_COM_FREQ_UPDATE				0x008
-#define QSERDES_COM_BG_TIMER				0x00c
-#define QSERDES_COM_SSC_EN_CENTER			0x010
-#define QSERDES_COM_SSC_ADJ_PER1			0x014
-#define QSERDES_COM_SSC_ADJ_PER2			0x018
-#define QSERDES_COM_SSC_PER1				0x01c
-#define QSERDES_COM_SSC_PER2				0x020
-#define QSERDES_COM_SSC_STEP_SIZE1			0x024
-#define QSERDES_COM_SSC_STEP_SIZE2			0x028
-#define QSERDES_COM_POST_DIV				0x02c
-#define QSERDES_COM_POST_DIV_MUX			0x030
-#define QSERDES_COM_BIAS_EN_CLKBUFLR_EN			0x034
-#define QSERDES_COM_CLK_ENABLE1				0x038
-#define QSERDES_COM_SYS_CLK_CTRL			0x03c
-#define QSERDES_COM_SYSCLK_BUF_ENABLE			0x040
-#define QSERDES_COM_PLL_EN				0x044
-#define QSERDES_COM_PLL_IVCO				0x048
-#define QSERDES_COM_LOCK_CMP1_MODE0			0x04c
-#define QSERDES_COM_LOCK_CMP2_MODE0			0x050
-#define QSERDES_COM_LOCK_CMP3_MODE0			0x054
-#define QSERDES_COM_LOCK_CMP1_MODE1			0x058
-#define QSERDES_COM_LOCK_CMP2_MODE1			0x05c
-#define QSERDES_COM_LOCK_CMP3_MODE1			0x060
-#define QSERDES_COM_LOCK_CMP1_MODE2			0x064
-#define QSERDES_COM_CMN_RSVD0				0x064
-#define QSERDES_COM_LOCK_CMP2_MODE2			0x068
-#define QSERDES_COM_EP_CLOCK_DETECT_CTRL		0x068
-#define QSERDES_COM_LOCK_CMP3_MODE2			0x06c
-#define QSERDES_COM_SYSCLK_DET_COMP_STATUS		0x06c
-#define QSERDES_COM_BG_TRIM				0x070
-#define QSERDES_COM_CLK_EP_DIV				0x074
-#define QSERDES_COM_CP_CTRL_MODE0			0x078
-#define QSERDES_COM_CP_CTRL_MODE1			0x07c
-#define QSERDES_COM_CP_CTRL_MODE2			0x080
-#define QSERDES_COM_CMN_RSVD1				0x080
-#define QSERDES_COM_PLL_RCTRL_MODE0			0x084
-#define QSERDES_COM_PLL_RCTRL_MODE1			0x088
-#define QSERDES_COM_PLL_RCTRL_MODE2			0x08c
-#define QSERDES_COM_CMN_RSVD2				0x08c
-#define QSERDES_COM_PLL_CCTRL_MODE0			0x090
-#define QSERDES_COM_PLL_CCTRL_MODE1			0x094
-#define QSERDES_COM_PLL_CCTRL_MODE2			0x098
-#define QSERDES_COM_CMN_RSVD3				0x098
-#define QSERDES_COM_PLL_CNTRL				0x09c
-#define QSERDES_COM_PHASE_SEL_CTRL			0x0a0
-#define QSERDES_COM_PHASE_SEL_DC			0x0a4
-#define QSERDES_COM_CORE_CLK_IN_SYNC_SEL		0x0a8
-#define QSERDES_COM_BIAS_EN_CTRL_BY_PSM			0x0a8
-#define QSERDES_COM_SYSCLK_EN_SEL			0x0ac
-#define QSERDES_COM_CML_SYSCLK_SEL			0x0b0
-#define QSERDES_COM_RESETSM_CNTRL			0x0b4
-#define QSERDES_COM_RESETSM_CNTRL2			0x0b8
-#define QSERDES_COM_RESTRIM_CTRL			0x0bc
-#define QSERDES_COM_RESTRIM_CTRL2			0x0c0
-#define QSERDES_COM_RESCODE_DIV_NUM			0x0c4
-#define QSERDES_COM_LOCK_CMP_EN				0x0c8
-#define QSERDES_COM_LOCK_CMP_CFG			0x0cc
-#define QSERDES_COM_DEC_START_MODE0			0x0d0
-#define QSERDES_COM_DEC_START_MODE1			0x0d4
-#define QSERDES_COM_DEC_START_MODE2			0x0d8
-#define QSERDES_COM_VCOCAL_DEADMAN_CTRL			0x0d8
-#define QSERDES_COM_DIV_FRAC_START1_MODE0		0x0dc
-#define QSERDES_COM_DIV_FRAC_START2_MODE0		0x0e0
-#define QSERDES_COM_DIV_FRAC_START3_MODE0		0x0e4
-#define QSERDES_COM_DIV_FRAC_START1_MODE1		0x0e8
-#define QSERDES_COM_DIV_FRAC_START2_MODE1		0x0ec
-#define QSERDES_COM_DIV_FRAC_START3_MODE1		0x0f0
-#define QSERDES_COM_DIV_FRAC_START1_MODE2		0x0f4
-#define QSERDES_COM_VCO_TUNE_MINVAL1			0x0f4
-#define QSERDES_COM_DIV_FRAC_START2_MODE2		0x0f8
-#define QSERDES_COM_VCO_TUNE_MINVAL2			0x0f8
-#define QSERDES_COM_DIV_FRAC_START3_MODE2		0x0fc
-#define QSERDES_COM_CMN_RSVD4				0x0fc
-#define QSERDES_COM_INTEGLOOP_INITVAL			0x100
-#define QSERDES_COM_INTEGLOOP_EN			0x104
-#define QSERDES_COM_INTEGLOOP_GAIN0_MODE0		0x108
-#define QSERDES_COM_INTEGLOOP_GAIN1_MODE0		0x10c
-#define QSERDES_COM_INTEGLOOP_GAIN0_MODE1		0x110
-#define QSERDES_COM_INTEGLOOP_GAIN1_MODE1		0x114
-#define QSERDES_COM_INTEGLOOP_GAIN0_MODE2		0x118
-#define QSERDES_COM_VCO_TUNE_MAXVAL1			0x118
-#define QSERDES_COM_INTEGLOOP_GAIN1_MODE2		0x11c
-#define QSERDES_COM_VCO_TUNE_MAXVAL2			0x11c
-#define QSERDES_COM_RES_TRIM_CONTROL2			0x120
-#define QSERDES_COM_VCO_TUNE_CTRL			0x124
-#define QSERDES_COM_VCO_TUNE_MAP			0x128
-#define QSERDES_COM_VCO_TUNE1_MODE0			0x12c
-#define QSERDES_COM_VCO_TUNE2_MODE0			0x130
-#define QSERDES_COM_VCO_TUNE1_MODE1			0x134
-#define QSERDES_COM_VCO_TUNE2_MODE1			0x138
-#define QSERDES_COM_VCO_TUNE1_MODE2			0x13c
-#define QSERDES_COM_VCO_TUNE_INITVAL1			0x13c
-#define QSERDES_COM_VCO_TUNE2_MODE2			0x140
-#define QSERDES_COM_VCO_TUNE_INITVAL2			0x140
-#define QSERDES_COM_VCO_TUNE_TIMER1			0x144
-#define QSERDES_COM_VCO_TUNE_TIMER2			0x148
-#define QSERDES_COM_SAR					0x14c
-#define QSERDES_COM_SAR_CLK				0x150
-#define QSERDES_COM_SAR_CODE_OUT_STATUS			0x154
-#define QSERDES_COM_SAR_CODE_READY_STATUS		0x158
-#define QSERDES_COM_CMN_STATUS				0x15c
-#define QSERDES_COM_RESET_SM_STATUS			0x160
-#define QSERDES_COM_RESTRIM_CODE_STATUS			0x164
-#define QSERDES_COM_PLLCAL_CODE1_STATUS			0x168
-#define QSERDES_COM_PLLCAL_CODE2_STATUS			0x16c
-#define QSERDES_COM_BG_CTRL				0x170
-#define QSERDES_COM_CLK_SELECT				0x174
-#define QSERDES_COM_HSCLK_SEL				0x178
-#define QSERDES_COM_INTEGLOOP_BINCODE_STATUS		0x17c
-#define QSERDES_COM_PLL_ANALOG				0x180
-#define QSERDES_COM_CORECLK_DIV				0x184
-#define QSERDES_COM_SW_RESET				0x188
-#define QSERDES_COM_CORE_CLK_EN				0x18c
-#define QSERDES_COM_C_READY_STATUS			0x190
-#define QSERDES_COM_CMN_CONFIG				0x194
-#define QSERDES_COM_CMN_RATE_OVERRIDE			0x198
-#define QSERDES_COM_SVS_MODE_CLK_SEL			0x19c
-#define QSERDES_COM_DEBUG_BUS0				0x1a0
-#define QSERDES_COM_DEBUG_BUS1				0x1a4
-#define QSERDES_COM_DEBUG_BUS2				0x1a8
-#define QSERDES_COM_DEBUG_BUS3				0x1ac
-#define QSERDES_COM_DEBUG_BUS_SEL			0x1b0
-#define QSERDES_COM_CMN_MISC1				0x1b4
-#define QSERDES_COM_CMN_MISC2				0x1b8
-#define QSERDES_COM_CORECLK_DIV_MODE1			0x1bc
-#define QSERDES_COM_CORECLK_DIV_MODE2			0x1c0
-#define QSERDES_COM_CMN_RSVD5				0x1c4
-
-#endif
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v2.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v2.h
index 34919720b7bc4..9ae0cf95e317e 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v2.h
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx-v2.h
@@ -7,62 +7,199 @@
 #define QCOM_PHY_QMP_QSERDES_TXRX_V2_H_
 
 /* Only for QMP V2 PHY - TX registers */
-#define QSERDES_V2_TX_BIST_MODE_LANENO			0x000
-#define QSERDES_V2_TX_CLKBUF_ENABLE			0x008
-#define QSERDES_V2_TX_TX_EMP_POST1_LVL			0x00c
-#define QSERDES_V2_TX_TX_DRV_LVL			0x01c
-#define QSERDES_V2_TX_RESET_TSYNC_EN			0x024
-#define QSERDES_V2_TX_PRE_STALL_LDO_BOOST_EN		0x028
-#define QSERDES_V2_TX_TX_BAND				0x02c
-#define QSERDES_V2_TX_SLEW_CNTL				0x030
-#define QSERDES_V2_TX_INTERFACE_SELECT			0x034
-#define QSERDES_V2_TX_RES_CODE_LANE_TX			0x03c
-#define QSERDES_V2_TX_RES_CODE_LANE_RX			0x040
-#define QSERDES_V2_TX_RES_CODE_LANE_OFFSET_TX		0x044
-#define QSERDES_V2_TX_RES_CODE_LANE_OFFSET_RX		0x048
-#define QSERDES_V2_TX_DEBUG_BUS_SEL			0x058
-#define QSERDES_V2_TX_TRANSCEIVER_BIAS_EN		0x05c
-#define QSERDES_V2_TX_HIGHZ_DRVR_EN			0x060
-#define QSERDES_V2_TX_TX_POL_INV			0x064
-#define QSERDES_V2_TX_PARRATE_REC_DETECT_IDLE_EN	0x068
-#define QSERDES_V2_TX_LANE_MODE_1			0x08c
-#define QSERDES_V2_TX_LANE_MODE_2			0x090
-#define QSERDES_V2_TX_LANE_MODE_3			0x094
-#define QSERDES_V2_TX_RCV_DETECT_LVL_2			0x0a4
-#define QSERDES_V2_TX_TRAN_DRVR_EMP_EN			0x0c0
-#define QSERDES_V2_TX_TX_INTERFACE_MODE			0x0c4
-#define QSERDES_V2_TX_VMODE_CTRL1			0x0f0
+#define QSERDES_V2_TX_BIST_MODE_LANENO				0x000
+#define QSERDES_V2_TX_BIST_INVERT				0x004
+#define QSERDES_V2_TX_CLKBUF_ENABLE				0x008
+#define QSERDES_V2_TX_CMN_CONTROL_ONE				0x00c
+#define QSERDES_V2_TX_CMN_CONTROL_TWO				0x010
+#define QSERDES_V2_TX_CMN_CONTROL_THREE				0x014
+#define QSERDES_V2_TX_TX_EMP_POST1_LVL				0x018
+#define QSERDES_V2_TX_TX_POST2_EMPH				0x01c
+#define QSERDES_V2_TX_TX_BOOST_LVL_UP_DN			0x020
+#define QSERDES_V2_TX_HP_PD_ENABLES				0x024
+#define QSERDES_V2_TX_TX_IDLE_LVL_LARGE_AMP			0x028
+#define QSERDES_V2_TX_TX_DRV_LVL				0x02c
+#define QSERDES_V2_TX_TX_DRV_LVL_OFFSET				0x030
+#define QSERDES_V2_TX_RESET_TSYNC_EN				0x034
+#define QSERDES_V2_TX_PRE_STALL_LDO_BOOST_EN			0x038
+#define QSERDES_V2_TX_TX_BAND					0x03c
+#define QSERDES_V2_TX_SLEW_CNTL					0x040
+#define QSERDES_V2_TX_INTERFACE_SELECT				0x044
+#define QSERDES_V2_TX_LPB_EN					0x048
+#define QSERDES_V2_TX_RES_CODE_LANE_TX				0x04c
+#define QSERDES_V2_TX_RES_CODE_LANE_RX				0x050
+#define QSERDES_V2_TX_RES_CODE_LANE_OFFSET			0x054
+#define QSERDES_V2_TX_PERL_LENGTH1				0x058
+#define QSERDES_V2_TX_PERL_LENGTH2				0x05c
+#define QSERDES_V2_TX_SERDES_BYP_EN_OUT				0x060
+#define QSERDES_V2_TX_DEBUG_BUS_SEL				0x064
+#define QSERDES_V2_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN		0x068
+#define QSERDES_V2_TX_TX_POL_INV				0x06c
+#define QSERDES_V2_TX_PARRATE_REC_DETECT_IDLE_EN		0x070
+#define QSERDES_V2_TX_BIST_PATTERN1				0x074
+#define QSERDES_V2_TX_BIST_PATTERN2				0x078
+#define QSERDES_V2_TX_BIST_PATTERN3				0x07c
+#define QSERDES_V2_TX_BIST_PATTERN4				0x080
+#define QSERDES_V2_TX_BIST_PATTERN5				0x084
+#define QSERDES_V2_TX_BIST_PATTERN6				0x088
+#define QSERDES_V2_TX_BIST_PATTERN7				0x08c
+#define QSERDES_V2_TX_BIST_PATTERN8				0x090
+#define QSERDES_V2_TX_LANE_MODE					0x094
+#define QSERDES_V2_TX_IDAC_CAL_LANE_MODE			0x098
+#define QSERDES_V2_TX_IDAC_CAL_LANE_MODE_CONFIGURATION		0x09c
+#define QSERDES_V2_TX_ATB_SEL1					0x0a0
+#define QSERDES_V2_TX_ATB_SEL2					0x0a4
+#define QSERDES_V2_TX_RCV_DETECT_LVL				0x0a8
+#define QSERDES_V2_TX_RCV_DETECT_LVL_2				0x0ac
+#define QSERDES_V2_TX_PRBS_SEED1				0x0b0
+#define QSERDES_V2_TX_PRBS_SEED2				0x0b4
+#define QSERDES_V2_TX_PRBS_SEED3				0x0b8
+#define QSERDES_V2_TX_PRBS_SEED4				0x0bc
+#define QSERDES_V2_TX_RESET_GEN					0x0c0
+#define QSERDES_V2_TX_RESET_GEN_MUXES				0x0c4
+#define QSERDES_V2_TX_TRAN_DRVR_EMP_EN				0x0c8
+#define QSERDES_V2_TX_TX_INTERFACE_MODE				0x0cc
+#define QSERDES_V2_TX_PWM_CTRL					0x0d0
+#define QSERDES_V2_TX_PWM_ENCODED_OR_DATA			0x0d4
+#define QSERDES_V2_TX_PWM_GEAR_1_DIVIDER_BAND2			0x0d8
+#define QSERDES_V2_TX_PWM_GEAR_2_DIVIDER_BAND2			0x0dc
+#define QSERDES_V2_TX_PWM_GEAR_3_DIVIDER_BAND2			0x0e0
+#define QSERDES_V2_TX_PWM_GEAR_4_DIVIDER_BAND2			0x0e4
+#define QSERDES_V2_TX_PWM_GEAR_1_DIVIDER_BAND0_1		0x0e8
+#define QSERDES_V2_TX_PWM_GEAR_2_DIVIDER_BAND0_1		0x0ec
+#define QSERDES_V2_TX_PWM_GEAR_3_DIVIDER_BAND0_1		0x0f0
+#define QSERDES_V2_TX_PWM_GEAR_4_DIVIDER_BAND0_1		0x0f4
+#define QSERDES_V2_TX_VMODE_CTRL1				0x0f8
+#define QSERDES_V2_TX_VMODE_CTRL2				0x0fc
+#define QSERDES_V2_TX_TX_ALOG_INTF_OBSV_CNTL			0x100
+#define QSERDES_V2_TX_BIST_STATUS				0x104
+#define QSERDES_V2_TX_BIST_ERROR_COUNT1				0x108
+#define QSERDES_V2_TX_BIST_ERROR_COUNT2				0x10c
+#define QSERDES_V2_TX_TX_ALOG_INTF_OBSV				0x110
 
 /* Only for QMP V2 PHY - RX registers */
-#define QSERDES_V2_RX_UCDR_FO_GAIN			0x008
-#define QSERDES_V2_RX_UCDR_SO_GAIN_HALF			0x00c
-#define QSERDES_V2_RX_UCDR_SO_GAIN			0x014
-#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN_HALF		0x024
-#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN_QUARTER		0x028
-#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN			0x02c
-#define QSERDES_V2_RX_UCDR_FASTLOCK_FO_GAIN		0x030
-#define QSERDES_V2_RX_UCDR_SO_SATURATION_AND_ENABLE	0x034
-#define QSERDES_V2_RX_UCDR_FASTLOCK_COUNT_LOW		0x03c
-#define QSERDES_V2_RX_UCDR_FASTLOCK_COUNT_HIGH		0x040
-#define QSERDES_V2_RX_UCDR_PI_CONTROLS			0x044
-#define QSERDES_V2_RX_RX_TERM_BW			0x07c
-#define QSERDES_V2_RX_VGA_CAL_CNTRL1			0x0bc
-#define QSERDES_V2_RX_VGA_CAL_CNTRL2			0x0c0
-#define QSERDES_V2_RX_RX_EQ_GAIN2_LSB			0x0c8
-#define QSERDES_V2_RX_RX_EQ_GAIN2_MSB			0x0cc
-#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL1		0x0d0
-#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2		0x0d4
-#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3		0x0d8
-#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL4		0x0dc
-#define QSERDES_V2_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1	0x0f8
-#define QSERDES_V2_RX_RX_OFFSET_ADAPTOR_CNTRL2		0x0fc
-#define QSERDES_V2_RX_SIGDET_ENABLES			0x100
-#define QSERDES_V2_RX_SIGDET_CNTRL			0x104
-#define QSERDES_V2_RX_SIGDET_LVL			0x108
-#define QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL		0x10c
-#define QSERDES_V2_RX_RX_BAND				0x110
-#define QSERDES_V2_RX_RX_INTERFACE_MODE			0x11c
-#define QSERDES_V2_RX_RX_MODE_00			0x164
-#define QSERDES_V2_RX_RX_MODE_01			0x168
+#define QSERDES_V2_RX_UCDR_FO_GAIN_HALF				0x000
+#define QSERDES_V2_RX_UCDR_FO_GAIN_QUARTER			0x004
+#define QSERDES_V2_RX_UCDR_FO_GAIN_EIGHTH			0x008
+#define QSERDES_V2_RX_UCDR_FO_GAIN				0x00c
+#define QSERDES_V2_RX_UCDR_SO_GAIN_HALF				0x010
+#define QSERDES_V2_RX_UCDR_SO_GAIN_QUARTER			0x014
+#define QSERDES_V2_RX_UCDR_SO_GAIN_EIGHTH			0x018
+#define QSERDES_V2_RX_UCDR_SO_GAIN				0x01c
+#define QSERDES_V2_RX_UCDR_SVS_FO_GAIN_HALF			0x020
+#define QSERDES_V2_RX_UCDR_SVS_FO_GAIN_QUARTER			0x024
+#define QSERDES_V2_RX_UCDR_SVS_FO_GAIN_EIGHTH			0x028
+#define QSERDES_V2_RX_UCDR_SVS_FO_GAIN				0x02c
+#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN_HALF			0x030
+#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN_QUARTER			0x034
+#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN_EIGHTH			0x038
+#define QSERDES_V2_RX_UCDR_SVS_SO_GAIN				0x03c
+#define QSERDES_V2_RX_UCDR_FASTLOCK_FO_GAIN			0x040
+#define QSERDES_V2_RX_UCDR_FD_GAIN				0x044
+#define QSERDES_V2_RX_UCDR_SO_SATURATION_AND_ENABLE		0x048
+#define QSERDES_V2_RX_UCDR_FO_TO_SO_DELAY			0x04c
+#define QSERDES_V2_RX_UCDR_FASTLOCK_COUNT_LOW			0x050
+#define QSERDES_V2_RX_UCDR_FASTLOCK_COUNT_HIGH			0x054
+#define QSERDES_V2_RX_UCDR_MODULATE				0x058
+#define QSERDES_V2_RX_UCDR_PI_CONTROLS				0x05c
+#define QSERDES_V2_RX_RBIST_CONTROL				0x060
+#define QSERDES_V2_RX_AUX_CONTROL				0x064
+#define QSERDES_V2_RX_AUX_DATA_TCOARSE				0x068
+#define QSERDES_V2_RX_AUX_DATA_TFINE_LSB			0x06c
+#define QSERDES_V2_RX_AUX_DATA_TFINE_MSB			0x070
+#define QSERDES_V2_RX_RCLK_AUXDATA_SEL				0x074
+#define QSERDES_V2_RX_AC_JTAG_ENABLE				0x078
+#define QSERDES_V2_RX_AC_JTAG_INITP				0x07c
+#define QSERDES_V2_RX_AC_JTAG_INITN				0x080
+#define QSERDES_V2_RX_AC_JTAG_LVL				0x084
+#define QSERDES_V2_RX_AC_JTAG_MODE				0x088
+#define QSERDES_V2_RX_AC_JTAG_RESET				0x08c
+#define QSERDES_V2_RX_RX_TERM_BW				0x090
+#define QSERDES_V2_RX_RX_RCVR_IQ_EN				0x094
+#define QSERDES_V2_RX_RX_IDAC_I_DC_OFFSETS			0x098
+#define QSERDES_V2_RX_RX_IDAC_IBAR_DC_OFFSETS			0x09c
+#define QSERDES_V2_RX_RX_IDAC_Q_DC_OFFSETS			0x0a0
+#define QSERDES_V2_RX_RX_IDAC_QBAR_DC_OFFSETS			0x0a4
+#define QSERDES_V2_RX_RX_IDAC_A_DC_OFFSETS			0x0a8
+#define QSERDES_V2_RX_RX_IDAC_ABAR_DC_OFFSETS			0x0ac
+#define QSERDES_V2_RX_RX_IDAC_EN				0x0b0
+#define QSERDES_V2_RX_RX_IDAC_ENABLES				0x0b4
+#define QSERDES_V2_RX_RX_IDAC_SIGN				0x0b8
+#define QSERDES_V2_RX_RX_HIGHZ_HIGHRATE				0x0bc
+#define QSERDES_V2_RX_RX_TERM_AC_BYPASS_DC_COUPLE_OFFSET	0x0c0
+#define QSERDES_V2_RX_RX_EQ_GAIN1_LSB				0x0c4
+#define QSERDES_V2_RX_RX_EQ_GAIN1_MSB				0x0c8
+#define QSERDES_V2_RX_RX_EQ_GAIN2_LSB				0x0cc
+#define QSERDES_V2_RX_RX_EQ_GAIN2_MSB				0x0d0
+#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL1			0x0d4
+#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2			0x0d8
+#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3			0x0dc
+#define QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL4			0x0e0
+#define QSERDES_V2_RX_RX_IDAC_CAL_CONFIGURATION			0x0e4
+#define QSERDES_V2_RX_RX_IDAC_TSETTLE_LOW			0x0e8
+#define QSERDES_V2_RX_RX_IDAC_TSETTLE_HIGH			0x0ec
+#define QSERDES_V2_RX_RX_IDAC_ENDSAMP_LOW			0x0f0
+#define QSERDES_V2_RX_RX_IDAC_ENDSAMP_HIGH			0x0f4
+#define QSERDES_V2_RX_RX_IDAC_MIDPOINT_LOW			0x0f8
+#define QSERDES_V2_RX_RX_IDAC_MIDPOINT_HIGH			0x0fc
+#define QSERDES_V2_RX_RX_EQ_OFFSET_LSB				0x100
+#define QSERDES_V2_RX_RX_EQ_OFFSET_MSB				0x104
+#define QSERDES_V2_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1		0x108
+#define QSERDES_V2_RX_RX_OFFSET_ADAPTOR_CNTRL2			0x10c
+#define QSERDES_V2_RX_SIGDET_ENABLES				0x110
+#define QSERDES_V2_RX_SIGDET_CNTRL				0x114
+#define QSERDES_V2_RX_SIGDET_LVL				0x118
+#define QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL			0x11c
+#define QSERDES_V2_RX_RX_BAND					0x120
+#define QSERDES_V2_RX_CDR_FREEZE_UP_DN				0x124
+#define QSERDES_V2_RX_CDR_RESET_OVERRIDE			0x128
+#define QSERDES_V2_RX_RX_INTERFACE_MODE				0x12c
+#define QSERDES_V2_RX_JITTER_GEN_MODE				0x130
+#define QSERDES_V2_RX_BUJ_AMP					0x134
+#define QSERDES_V2_RX_SJ_AMP1					0x138
+#define QSERDES_V2_RX_SJ_AMP2					0x13c
+#define QSERDES_V2_RX_SJ_PER1					0x140
+#define QSERDES_V2_RX_SJ_PER2					0x144
+#define QSERDES_V2_RX_BUJ_STEP_FREQ1				0x148
+#define QSERDES_V2_RX_BUJ_STEP_FREQ2				0x14c
+#define QSERDES_V2_RX_PPM_OFFSET1				0x150
+#define QSERDES_V2_RX_PPM_OFFSET2				0x154
+#define QSERDES_V2_RX_SIGN_PPM_PERIOD1				0x158
+#define QSERDES_V2_RX_SIGN_PPM_PERIOD2				0x15c
+#define QSERDES_V2_RX_SSC_CTRL					0x160
+#define QSERDES_V2_RX_SSC_COUNT1				0x164
+#define QSERDES_V2_RX_SSC_COUNT2				0x168
+#define QSERDES_V2_RX_RX_ALOG_INTF_OBSV_CNTL			0x16c
+#define QSERDES_V2_RX_RX_PWM_ENABLE_AND_DATA			0x170
+#define QSERDES_V2_RX_RX_PWM_GEAR1_TIMEOUT_COUNT		0x174
+#define QSERDES_V2_RX_RX_PWM_GEAR2_TIMEOUT_COUNT		0x178
+#define QSERDES_V2_RX_RX_PWM_GEAR3_TIMEOUT_COUNT		0x17c
+#define QSERDES_V2_RX_RX_PWM_GEAR4_TIMEOUT_COUNT		0x180
+#define QSERDES_V2_RX_PI_CTRL1					0x184
+#define QSERDES_V2_RX_PI_CTRL2					0x188
+#define QSERDES_V2_RX_PI_QUAD					0x18c
+#define QSERDES_V2_RX_IDATA1					0x190
+#define QSERDES_V2_RX_IDATA2					0x194
+#define QSERDES_V2_RX_AUX_DATA1					0x198
+#define QSERDES_V2_RX_AUX_DATA2					0x19c
+#define QSERDES_V2_RX_AC_JTAG_OUTP				0x1a0
+#define QSERDES_V2_RX_AC_JTAG_OUTN				0x1a4
+#define QSERDES_V2_RX_RX_SIGDET					0x1a8
+#define QSERDES_V2_RX_RX_VDCOFF					0x1ac
+#define QSERDES_V2_RX_IDAC_CAL_ON				0x1b0
+#define QSERDES_V2_RX_IDAC_STATUS_I				0x1b4
+#define QSERDES_V2_RX_IDAC_STATUS_IBAR				0x1b8
+#define QSERDES_V2_RX_IDAC_STATUS_Q				0x1bc
+#define QSERDES_V2_RX_IDAC_STATUS_QBAR				0x1c0
+#define QSERDES_V2_RX_IDAC_STATUS_A				0x1c4
+#define QSERDES_V2_RX_IDAC_STATUS_ABAR				0x1c8
+#define QSERDES_V2_RX_CALST_STATUS_I				0x1cc
+#define QSERDES_V2_RX_CALST_STATUS_Q				0x1d0
+#define QSERDES_V2_RX_CALST_STATUS_A				0x1d4
+#define QSERDES_V2_RX_RX_ALOG_INTF_OBSV				0x1d8
+#define QSERDES_V2_RX_READ_EQCODE				0x1dc
+#define QSERDES_V2_RX_READ_OFFSETCODE				0x1e0
+#define QSERDES_V2_RX_IA_ERROR_COUNTER_LOW			0x1e4
+#define QSERDES_V2_RX_IA_ERROR_COUNTER_HIGH			0x1e8
 
 #endif
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx.h b/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx.h
deleted file mode 100644
index d20694513eb49..0000000000000
--- a/drivers/phy/qualcomm/phy-qcom-qmp-qserdes-txrx.h
+++ /dev/null
@@ -1,205 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright (c) 2017, The Linux Foundation. All rights reserved.
- */
-
-#ifndef QCOM_PHY_QMP_QSERDES_TXRX_H_
-#define QCOM_PHY_QMP_QSERDES_TXRX_H_
-
-/* Only for QMP V2 PHY - TX registers */
-#define QSERDES_TX_BIST_MODE_LANENO			0x000
-#define QSERDES_TX_BIST_INVERT				0x004
-#define QSERDES_TX_CLKBUF_ENABLE			0x008
-#define QSERDES_TX_CMN_CONTROL_ONE			0x00c
-#define QSERDES_TX_CMN_CONTROL_TWO			0x010
-#define QSERDES_TX_CMN_CONTROL_THREE			0x014
-#define QSERDES_TX_TX_EMP_POST1_LVL			0x018
-#define QSERDES_TX_TX_POST2_EMPH			0x01c
-#define QSERDES_TX_TX_BOOST_LVL_UP_DN			0x020
-#define QSERDES_TX_HP_PD_ENABLES			0x024
-#define QSERDES_TX_TX_IDLE_LVL_LARGE_AMP		0x028
-#define QSERDES_TX_TX_DRV_LVL				0x02c
-#define QSERDES_TX_TX_DRV_LVL_OFFSET			0x030
-#define QSERDES_TX_RESET_TSYNC_EN			0x034
-#define QSERDES_TX_PRE_STALL_LDO_BOOST_EN		0x038
-#define QSERDES_TX_TX_BAND				0x03c
-#define QSERDES_TX_SLEW_CNTL				0x040
-#define QSERDES_TX_INTERFACE_SELECT			0x044
-#define QSERDES_TX_LPB_EN				0x048
-#define QSERDES_TX_RES_CODE_LANE_TX			0x04c
-#define QSERDES_TX_RES_CODE_LANE_RX			0x050
-#define QSERDES_TX_RES_CODE_LANE_OFFSET			0x054
-#define QSERDES_TX_PERL_LENGTH1				0x058
-#define QSERDES_TX_PERL_LENGTH2				0x05c
-#define QSERDES_TX_SERDES_BYP_EN_OUT			0x060
-#define QSERDES_TX_DEBUG_BUS_SEL			0x064
-#define QSERDES_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN	0x068
-#define QSERDES_TX_TX_POL_INV				0x06c
-#define QSERDES_TX_PARRATE_REC_DETECT_IDLE_EN		0x070
-#define QSERDES_TX_BIST_PATTERN1			0x074
-#define QSERDES_TX_BIST_PATTERN2			0x078
-#define QSERDES_TX_BIST_PATTERN3			0x07c
-#define QSERDES_TX_BIST_PATTERN4			0x080
-#define QSERDES_TX_BIST_PATTERN5			0x084
-#define QSERDES_TX_BIST_PATTERN6			0x088
-#define QSERDES_TX_BIST_PATTERN7			0x08c
-#define QSERDES_TX_BIST_PATTERN8			0x090
-#define QSERDES_TX_LANE_MODE				0x094
-#define QSERDES_TX_IDAC_CAL_LANE_MODE			0x098
-#define QSERDES_TX_IDAC_CAL_LANE_MODE_CONFIGURATION	0x09c
-#define QSERDES_TX_ATB_SEL1				0x0a0
-#define QSERDES_TX_ATB_SEL2				0x0a4
-#define QSERDES_TX_RCV_DETECT_LVL			0x0a8
-#define QSERDES_TX_RCV_DETECT_LVL_2			0x0ac
-#define QSERDES_TX_PRBS_SEED1				0x0b0
-#define QSERDES_TX_PRBS_SEED2				0x0b4
-#define QSERDES_TX_PRBS_SEED3				0x0b8
-#define QSERDES_TX_PRBS_SEED4				0x0bc
-#define QSERDES_TX_RESET_GEN				0x0c0
-#define QSERDES_TX_RESET_GEN_MUXES			0x0c4
-#define QSERDES_TX_TRAN_DRVR_EMP_EN			0x0c8
-#define QSERDES_TX_TX_INTERFACE_MODE			0x0cc
-#define QSERDES_TX_PWM_CTRL				0x0d0
-#define QSERDES_TX_PWM_ENCODED_OR_DATA			0x0d4
-#define QSERDES_TX_PWM_GEAR_1_DIVIDER_BAND2		0x0d8
-#define QSERDES_TX_PWM_GEAR_2_DIVIDER_BAND2		0x0dc
-#define QSERDES_TX_PWM_GEAR_3_DIVIDER_BAND2		0x0e0
-#define QSERDES_TX_PWM_GEAR_4_DIVIDER_BAND2		0x0e4
-#define QSERDES_TX_PWM_GEAR_1_DIVIDER_BAND0_1		0x0e8
-#define QSERDES_TX_PWM_GEAR_2_DIVIDER_BAND0_1		0x0ec
-#define QSERDES_TX_PWM_GEAR_3_DIVIDER_BAND0_1		0x0f0
-#define QSERDES_TX_PWM_GEAR_4_DIVIDER_BAND0_1		0x0f4
-#define QSERDES_TX_VMODE_CTRL1				0x0f8
-#define QSERDES_TX_VMODE_CTRL2				0x0fc
-#define QSERDES_TX_TX_ALOG_INTF_OBSV_CNTL		0x100
-#define QSERDES_TX_BIST_STATUS				0x104
-#define QSERDES_TX_BIST_ERROR_COUNT1			0x108
-#define QSERDES_TX_BIST_ERROR_COUNT2			0x10c
-#define QSERDES_TX_TX_ALOG_INTF_OBSV			0x110
-
-/* Only for QMP V2 PHY - RX registers */
-#define QSERDES_RX_UCDR_FO_GAIN_HALF			0x000
-#define QSERDES_RX_UCDR_FO_GAIN_QUARTER			0x004
-#define QSERDES_RX_UCDR_FO_GAIN_EIGHTH			0x008
-#define QSERDES_RX_UCDR_FO_GAIN				0x00c
-#define QSERDES_RX_UCDR_SO_GAIN_HALF			0x010
-#define QSERDES_RX_UCDR_SO_GAIN_QUARTER			0x014
-#define QSERDES_RX_UCDR_SO_GAIN_EIGHTH			0x018
-#define QSERDES_RX_UCDR_SO_GAIN				0x01c
-#define QSERDES_RX_UCDR_SVS_FO_GAIN_HALF		0x020
-#define QSERDES_RX_UCDR_SVS_FO_GAIN_QUARTER		0x024
-#define QSERDES_RX_UCDR_SVS_FO_GAIN_EIGHTH		0x028
-#define QSERDES_RX_UCDR_SVS_FO_GAIN			0x02c
-#define QSERDES_RX_UCDR_SVS_SO_GAIN_HALF		0x030
-#define QSERDES_RX_UCDR_SVS_SO_GAIN_QUARTER		0x034
-#define QSERDES_RX_UCDR_SVS_SO_GAIN_EIGHTH		0x038
-#define QSERDES_RX_UCDR_SVS_SO_GAIN			0x03c
-#define QSERDES_RX_UCDR_FASTLOCK_FO_GAIN		0x040
-#define QSERDES_RX_UCDR_FD_GAIN				0x044
-#define QSERDES_RX_UCDR_SO_SATURATION_AND_ENABLE	0x048
-#define QSERDES_RX_UCDR_FO_TO_SO_DELAY			0x04c
-#define QSERDES_RX_UCDR_FASTLOCK_COUNT_LOW		0x050
-#define QSERDES_RX_UCDR_FASTLOCK_COUNT_HIGH		0x054
-#define QSERDES_RX_UCDR_MODULATE			0x058
-#define QSERDES_RX_UCDR_PI_CONTROLS			0x05c
-#define QSERDES_RX_RBIST_CONTROL			0x060
-#define QSERDES_RX_AUX_CONTROL				0x064
-#define QSERDES_RX_AUX_DATA_TCOARSE			0x068
-#define QSERDES_RX_AUX_DATA_TFINE_LSB			0x06c
-#define QSERDES_RX_AUX_DATA_TFINE_MSB			0x070
-#define QSERDES_RX_RCLK_AUXDATA_SEL			0x074
-#define QSERDES_RX_AC_JTAG_ENABLE			0x078
-#define QSERDES_RX_AC_JTAG_INITP			0x07c
-#define QSERDES_RX_AC_JTAG_INITN			0x080
-#define QSERDES_RX_AC_JTAG_LVL				0x084
-#define QSERDES_RX_AC_JTAG_MODE				0x088
-#define QSERDES_RX_AC_JTAG_RESET			0x08c
-#define QSERDES_RX_RX_TERM_BW				0x090
-#define QSERDES_RX_RX_RCVR_IQ_EN			0x094
-#define QSERDES_RX_RX_IDAC_I_DC_OFFSETS			0x098
-#define QSERDES_RX_RX_IDAC_IBAR_DC_OFFSETS		0x09c
-#define QSERDES_RX_RX_IDAC_Q_DC_OFFSETS			0x0a0
-#define QSERDES_RX_RX_IDAC_QBAR_DC_OFFSETS		0x0a4
-#define QSERDES_RX_RX_IDAC_A_DC_OFFSETS			0x0a8
-#define QSERDES_RX_RX_IDAC_ABAR_DC_OFFSETS		0x0ac
-#define QSERDES_RX_RX_IDAC_EN				0x0b0
-#define QSERDES_RX_RX_IDAC_ENABLES			0x0b4
-#define QSERDES_RX_RX_IDAC_SIGN				0x0b8
-#define QSERDES_RX_RX_HIGHZ_HIGHRATE			0x0bc
-#define QSERDES_RX_RX_TERM_AC_BYPASS_DC_COUPLE_OFFSET	0x0c0
-#define QSERDES_RX_RX_EQ_GAIN1_LSB			0x0c4
-#define QSERDES_RX_RX_EQ_GAIN1_MSB			0x0c8
-#define QSERDES_RX_RX_EQ_GAIN2_LSB			0x0cc
-#define QSERDES_RX_RX_EQ_GAIN2_MSB			0x0d0
-#define QSERDES_RX_RX_EQU_ADAPTOR_CNTRL1		0x0d4
-#define QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2		0x0d8
-#define QSERDES_RX_RX_EQU_ADAPTOR_CNTRL3		0x0dc
-#define QSERDES_RX_RX_EQU_ADAPTOR_CNTRL4		0x0e0
-#define QSERDES_RX_RX_IDAC_CAL_CONFIGURATION		0x0e4
-#define QSERDES_RX_RX_IDAC_TSETTLE_LOW			0x0e8
-#define QSERDES_RX_RX_IDAC_TSETTLE_HIGH			0x0ec
-#define QSERDES_RX_RX_IDAC_ENDSAMP_LOW			0x0f0
-#define QSERDES_RX_RX_IDAC_ENDSAMP_HIGH			0x0f4
-#define QSERDES_RX_RX_IDAC_MIDPOINT_LOW			0x0f8
-#define QSERDES_RX_RX_IDAC_MIDPOINT_HIGH		0x0fc
-#define QSERDES_RX_RX_EQ_OFFSET_LSB			0x100
-#define QSERDES_RX_RX_EQ_OFFSET_MSB			0x104
-#define QSERDES_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1		0x108
-#define QSERDES_RX_RX_OFFSET_ADAPTOR_CNTRL2		0x10c
-#define QSERDES_RX_SIGDET_ENABLES			0x110
-#define QSERDES_RX_SIGDET_CNTRL				0x114
-#define QSERDES_RX_SIGDET_LVL				0x118
-#define QSERDES_RX_SIGDET_DEGLITCH_CNTRL		0x11c
-#define QSERDES_RX_RX_BAND				0x120
-#define QSERDES_RX_CDR_FREEZE_UP_DN			0x124
-#define QSERDES_RX_CDR_RESET_OVERRIDE			0x128
-#define QSERDES_RX_RX_INTERFACE_MODE			0x12c
-#define QSERDES_RX_JITTER_GEN_MODE			0x130
-#define QSERDES_RX_BUJ_AMP				0x134
-#define QSERDES_RX_SJ_AMP1				0x138
-#define QSERDES_RX_SJ_AMP2				0x13c
-#define QSERDES_RX_SJ_PER1				0x140
-#define QSERDES_RX_SJ_PER2				0x144
-#define QSERDES_RX_BUJ_STEP_FREQ1			0x148
-#define QSERDES_RX_BUJ_STEP_FREQ2			0x14c
-#define QSERDES_RX_PPM_OFFSET1				0x150
-#define QSERDES_RX_PPM_OFFSET2				0x154
-#define QSERDES_RX_SIGN_PPM_PERIOD1			0x158
-#define QSERDES_RX_SIGN_PPM_PERIOD2			0x15c
-#define QSERDES_RX_SSC_CTRL				0x160
-#define QSERDES_RX_SSC_COUNT1				0x164
-#define QSERDES_RX_SSC_COUNT2				0x168
-#define QSERDES_RX_RX_ALOG_INTF_OBSV_CNTL		0x16c
-#define QSERDES_RX_RX_PWM_ENABLE_AND_DATA		0x170
-#define QSERDES_RX_RX_PWM_GEAR1_TIMEOUT_COUNT		0x174
-#define QSERDES_RX_RX_PWM_GEAR2_TIMEOUT_COUNT		0x178
-#define QSERDES_RX_RX_PWM_GEAR3_TIMEOUT_COUNT		0x17c
-#define QSERDES_RX_RX_PWM_GEAR4_TIMEOUT_COUNT		0x180
-#define QSERDES_RX_PI_CTRL1				0x184
-#define QSERDES_RX_PI_CTRL2				0x188
-#define QSERDES_RX_PI_QUAD				0x18c
-#define QSERDES_RX_IDATA1				0x190
-#define QSERDES_RX_IDATA2				0x194
-#define QSERDES_RX_AUX_DATA1				0x198
-#define QSERDES_RX_AUX_DATA2				0x19c
-#define QSERDES_RX_AC_JTAG_OUTP				0x1a0
-#define QSERDES_RX_AC_JTAG_OUTN				0x1a4
-#define QSERDES_RX_RX_SIGDET				0x1a8
-#define QSERDES_RX_RX_VDCOFF				0x1ac
-#define QSERDES_RX_IDAC_CAL_ON				0x1b0
-#define QSERDES_RX_IDAC_STATUS_I			0x1b4
-#define QSERDES_RX_IDAC_STATUS_IBAR			0x1b8
-#define QSERDES_RX_IDAC_STATUS_Q			0x1bc
-#define QSERDES_RX_IDAC_STATUS_QBAR			0x1c0
-#define QSERDES_RX_IDAC_STATUS_A			0x1c4
-#define QSERDES_RX_IDAC_STATUS_ABAR			0x1c8
-#define QSERDES_RX_CALST_STATUS_I			0x1cc
-#define QSERDES_RX_CALST_STATUS_Q			0x1d0
-#define QSERDES_RX_CALST_STATUS_A			0x1d4
-#define QSERDES_RX_RX_ALOG_INTF_OBSV			0x1d8
-#define QSERDES_RX_READ_EQCODE				0x1dc
-#define QSERDES_RX_READ_OFFSETCODE			0x1e0
-#define QSERDES_RX_IA_ERROR_COUNTER_LOW			0x1e4
-#define QSERDES_RX_IA_ERROR_COUNTER_HIGH		0x1e8
-
-#endif
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
index 771bc7c2ab505..0f4ad24aa4058 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-ufs.c
@@ -147,71 +147,71 @@ static const struct qmp_phy_init_tbl milos_ufsphy_pcs[] = {
 };
 
 static const struct qmp_phy_init_tbl msm8996_ufsphy_serdes[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x0e),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0xd7),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x30),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x05),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV_MODE1, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_EN, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_CTRL, 0x10),
-	QMP_PHY_INIT_CFG(QSERDES_COM_RESETSM_CNTRL, 0x20),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_CFG, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER1, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER2, 0x3f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x54),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x05),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x82),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE1_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE2_MODE0, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x0c),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE1, 0x98),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE1, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE1, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE1, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE1, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE1_MODE1, 0xd6),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE2_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE1, 0x32),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE1, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x0e),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0xd7),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV_MODE1, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_EN, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_CTRL, 0x10),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_RESETSM_CNTRL, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_CFG, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER1, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER2, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x54),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE1_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE2_MODE0, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x0c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE1, 0x98),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE1, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE1, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE1, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE1, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE1_MODE1, 0xd6),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE2_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE1, 0x32),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE1, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE1, 0x00),
 };
 
 static const struct qmp_phy_init_tbl msm8996_ufsphy_tx[] = {
-	QMP_PHY_INIT_CFG(QSERDES_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
-	QMP_PHY_INIT_CFG(QSERDES_TX_LANE_MODE, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_LANE_MODE, 0x02),
 };
 
 static const struct qmp_phy_init_tbl msm8996_ufsphy_rx[] = {
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_LVL, 0x24),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_CNTRL, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_INTERFACE_MODE, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_DEGLITCH_CNTRL, 0x18),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_FASTLOCK_FO_GAIN, 0x0B),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_TERM_BW, 0x5b),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_GAIN1_LSB, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_GAIN1_MSB, 0x3f),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_GAIN2_LSB, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_GAIN2_MSB, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0E),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_LVL, 0x24),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_CNTRL, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_INTERFACE_MODE, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_FASTLOCK_FO_GAIN, 0x0B),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_TERM_BW, 0x5b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_GAIN1_LSB, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_GAIN1_MSB, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_GAIN2_LSB, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_GAIN2_MSB, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0E),
 };
 
 static const struct qmp_phy_init_tbl sc7280_ufsphy_tx[] = {
@@ -320,83 +320,83 @@ static const struct qmp_phy_init_tbl sc7280_ufsphy_hs_g4_rx[] = {
 };
 
 static const struct qmp_phy_init_tbl sm6115_ufsphy_serdes[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x0e),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x14),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x30),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV_MODE1, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_EN, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_CTRL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_RESETSM_CNTRL, 0x20),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_CFG, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER1, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_TIMER2, 0x3f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x04),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x05),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x82),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE1_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE2_MODE0, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x0c),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE1, 0x98),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE1, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE1, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE1, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE1, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE1_MODE1, 0xd6),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE2_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE1, 0x32),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE1, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TRIM, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_INITVAL1, 0xff),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_INITVAL2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x0e),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV_MODE1, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_EN, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_CTRL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_RESETSM_CNTRL, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_CFG, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER1, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_TIMER2, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE1_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE2_MODE0, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x0c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE1, 0x98),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE1, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE1, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE1, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE1, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE1_MODE1, 0xd6),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE2_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE1, 0x32),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE1, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TRIM, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_INITVAL1, 0xff),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_INITVAL2, 0x00),
 };
 
 static const struct qmp_phy_init_tbl sm6115_ufsphy_hs_b_serdes[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x44),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x44),
 };
 
 static const struct qmp_phy_init_tbl sm6115_ufsphy_tx[] = {
-	QMP_PHY_INIT_CFG(QSERDES_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
-	QMP_PHY_INIT_CFG(QSERDES_TX_LANE_MODE, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_LANE_MODE, 0x06),
 };
 
 static const struct qmp_phy_init_tbl sm6115_ufsphy_rx[] = {
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_LVL, 0x24),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_CNTRL, 0x0F),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_INTERFACE_MODE, 0x40),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_DEGLITCH_CNTRL, 0x1E),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_FASTLOCK_FO_GAIN, 0x0B),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_TERM_BW, 0x5B),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_GAIN1_LSB, 0xFF),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_GAIN1_MSB, 0x3F),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_GAIN2_LSB, 0xFF),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_GAIN2_MSB, 0x3F),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0D),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SVS_SO_GAIN_HALF, 0x04),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SVS_SO_GAIN_QUARTER, 0x04),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SVS_SO_GAIN, 0x04),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x5B),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_LVL, 0x24),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_CNTRL, 0x0F),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_INTERFACE_MODE, 0x40),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL, 0x1E),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_FASTLOCK_FO_GAIN, 0x0B),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_TERM_BW, 0x5B),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_GAIN1_LSB, 0xFF),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_GAIN1_MSB, 0x3F),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_GAIN2_LSB, 0xFF),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_GAIN2_MSB, 0x3F),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2, 0x0D),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SVS_SO_GAIN_HALF, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SVS_SO_GAIN_QUARTER, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SVS_SO_GAIN, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_SATURATION_AND_ENABLE, 0x5B),
 };
 
 static const struct qmp_phy_init_tbl sm6115_ufsphy_pcs[] = {
@@ -1112,6 +1112,7 @@ static const struct qmp_phy_init_tbl sm8750_ufsphy_pcs[] = {
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_MULTI_LANE_CTRL1, 0x02),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_MID_TERM_CTRL1, 0x43),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PCS_CTRL1, 0x40),
+	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_PLL_CNTL, 0x33),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_LARGE_AMP_DRV_LVL, 0x0f),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_RX_SIGDET_CTRL2, 0x68),
 	QMP_PHY_INIT_CFG(QPHY_V6_PCS_UFS_TX_POST_EMP_LVL_S4, 0x0e),
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c
index b0ecd5ba24648..c5507168e1354 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-usb.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-usb.c
@@ -244,59 +244,59 @@ static const struct qmp_phy_init_tbl glymur_usb3_uniphy_pcs_usb_tbl[] = {
 };
 
 static const struct qmp_phy_init_tbl ipq9574_usb3_serdes_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x1a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x30),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TRIM, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_FASTLOCK_FO_GAIN, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0x1a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TRIM, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_FASTLOCK_FO_GAIN, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x06),
 	/* PLL and Loop filter settings */
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x68),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0xab),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0xaa),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x09),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0xa0),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0xaa),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x29),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_CFG, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x68),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0xab),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0xaa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x09),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0xa0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0xaa),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x29),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_CFG, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x0a),
 	/* SSC settings */
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_EN_CENTER, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER1, 0x7d),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER2, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER2, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE1, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE2, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_EN_CENTER, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER1, 0x7d),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER2, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE1, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE2, 0x05),
 };
 
 static const struct qmp_phy_init_tbl ipq9574_usb3_tx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
-	QMP_PHY_INIT_CFG(QSERDES_TX_RCV_DETECT_LVL_2, 0x12),
-	QMP_PHY_INIT_CFG(QSERDES_TX_LANE_MODE, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_RCV_DETECT_LVL_2, 0x12),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_LANE_MODE, 0x06),
 };
 
 static const struct qmp_phy_init_tbl ipq9574_usb3_rx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL3, 0x6c),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4c),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL4, 0xb8),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x77),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_CNTRL, 0x03),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_DEGLITCH_CNTRL, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_ENABLES, 0x0c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_GAIN, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3, 0x6c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL4, 0xb8),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x77),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_CNTRL, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_ENABLES, 0x0c),
 };
 
 static const struct qmp_phy_init_tbl ipq9574_usb3_pcs_tbl[] = {
@@ -326,52 +326,52 @@ static const struct qmp_phy_init_tbl ipq9574_usb3_pcs_tbl[] = {
 };
 
 static const struct qmp_phy_init_tbl ipq8074_usb3_serdes_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x1a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x30),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TRIM, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_FASTLOCK_FO_GAIN, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0x1a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TRIM, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_FASTLOCK_FO_GAIN, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x06),
 	/* PLL and Loop filter settings */
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x82),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x03),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0x15),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x34),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_CFG, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0x15),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x34),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_CFG, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x0a),
 	/* SSC settings */
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_EN_CENTER, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER1, 0x31),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER2, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER2, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE1, 0xde),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE2, 0x07),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_EN_CENTER, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER2, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE1, 0xde),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE2, 0x07),
 };
 
 static const struct qmp_phy_init_tbl ipq8074_usb3_rx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4c),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL4, 0xb8),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x77),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_CNTRL, 0x03),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_DEGLITCH_CNTRL, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_ENABLES, 0x0),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_GAIN, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL4, 0xb8),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x77),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_CNTRL, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_ENABLES, 0x0),
 };
 
 static const struct qmp_phy_init_tbl ipq8074_usb3_pcs_tbl[] = {
@@ -401,59 +401,59 @@ static const struct qmp_phy_init_tbl ipq8074_usb3_pcs_tbl[] = {
 };
 
 static const struct qmp_phy_init_tbl msm8996_usb3_serdes_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x14),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x30),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TRIM, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TRIM, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x04),
 	/* PLL and Loop filter settings */
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x82),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x03),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_CTRL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0x15),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x34),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_CFG, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_CTRL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0x15),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x34),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_CFG, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x0a),
 	/* SSC settings */
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_EN_CENTER, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER1, 0x31),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER2, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER2, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE1, 0xde),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE2, 0x07),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_EN_CENTER, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER2, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE1, 0xde),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE2, 0x07),
 };
 
 static const struct qmp_phy_init_tbl msm8996_usb3_tx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
-	QMP_PHY_INIT_CFG(QSERDES_TX_RCV_DETECT_LVL_2, 0x12),
-	QMP_PHY_INIT_CFG(QSERDES_TX_LANE_MODE, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_HIGHZ_TRANSCEIVEREN_BIAS_DRVR_EN, 0x45),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_RCV_DETECT_LVL_2, 0x12),
+	QMP_PHY_INIT_CFG(QSERDES_V2_TX_LANE_MODE, 0x06),
 };
 
 static const struct qmp_phy_init_tbl msm8996_usb3_rx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_FASTLOCK_FO_GAIN, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_RX_UCDR_SO_GAIN, 0x04),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL2, 0x02),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4c),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQU_ADAPTOR_CNTRL4, 0xbb),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x77),
-	QMP_PHY_INIT_CFG(QSERDES_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_CNTRL, 0x03),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_LVL, 0x18),
-	QMP_PHY_INIT_CFG(QSERDES_RX_SIGDET_DEGLITCH_CNTRL, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_FASTLOCK_FO_GAIN, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_UCDR_SO_GAIN, 0x04),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL2, 0x02),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL3, 0x4c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQU_ADAPTOR_CNTRL4, 0xbb),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_EQ_OFFSET_ADAPTOR_CNTRL1, 0x77),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_RX_OFFSET_ADAPTOR_CNTRL2, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_CNTRL, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_LVL, 0x18),
+	QMP_PHY_INIT_CFG(QSERDES_V2_RX_SIGDET_DEGLITCH_CNTRL, 0x16),
 };
 
 static const struct qmp_phy_init_tbl msm8996_usb3_pcs_tbl[] = {
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c b/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
index c342479a37980..ab3055bb5b0c1 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
+++ b/drivers/phy/qualcomm/phy-qcom-qmp-usbc.c
@@ -177,44 +177,44 @@ static const struct qmp_phy_init_tbl msm8998_usb3_pcs_tbl[] = {
 };
 
 static const struct qmp_phy_init_tbl qcm2290_usb3_serdes_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x14),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x30),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_COM_RESETSM_CNTRL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_RESETSM_CNTRL2, 0x08),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TRIM, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x82),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x55),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x03),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0x15),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x34),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_EN, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP_CFG, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_EN_CENTER, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER1, 0x31),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_PER2, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER1, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_ADJ_PER2, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE1, 0xde),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SSC_STEP_SIZE2, 0x07),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_INITVAL, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CTRL_BY_PSM, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0x14),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_RESETSM_CNTRL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_RESETSM_CNTRL2, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TRIM, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x82),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x55),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0x15),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x34),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP_CFG, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_EN_CENTER, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER1, 0x31),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_PER2, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER1, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_ADJ_PER2, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE1, 0xde),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SSC_STEP_SIZE2, 0x07),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_INITVAL, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CTRL_BY_PSM, 0x01),
 };
 
 static const struct qmp_phy_init_tbl qcm2290_usb3_tx_tbl[] = {
@@ -290,81 +290,81 @@ static const struct qmp_phy_init_tbl qcm2290_usb3_pcs_tbl[] = {
 	QMP_PHY_INIT_CFG(QPHY_V3_PCS_RX_SIGDET_LVL, 0x88),
 };
 
-static const struct qmp_phy_init_tbl qmp_v2_dp_serdes_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_SVS_MODE_CLK_SEL, 0x01),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_EN_SEL, 0x37),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYS_CLK_CTRL, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BIAS_EN_CLKBUFLR_EN, 0x3f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_ENABLE1, 0x0e),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_CTRL, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_SYSCLK_BUF_ENABLE, 0x06),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CLK_SELECT, 0x30),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_IVCO, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_CCTRL_MODE0, 0x28),
-	QMP_PHY_INIT_CFG(QSERDES_COM_PLL_RCTRL_MODE0, 0x16),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CP_CTRL_MODE0, 0x0b),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN0_MODE0, 0x40),
-	QMP_PHY_INIT_CFG(QSERDES_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_MAP, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_BG_TIMER, 0x08),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORECLK_DIV, 0x05),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_CTRL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE2_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_VCO_TUNE_CTRL, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CORE_CLK_EN, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_CMN_CONFIG, 0x02),
+static const struct qmp_phy_init_tbl qcs615_dp_serdes_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SVS_MODE_CLK_SEL, 0x01),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_EN_SEL, 0x37),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYS_CLK_CTRL, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BIAS_EN_CLKBUFLR_EN, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_ENABLE1, 0x0e),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_CTRL, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_SYSCLK_BUF_ENABLE, 0x06),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CLK_SELECT, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_IVCO, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_CCTRL_MODE0, 0x28),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_PLL_RCTRL_MODE0, 0x16),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CP_CTRL_MODE0, 0x0b),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN0_MODE0, 0x40),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_INTEGLOOP_GAIN1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_MAP, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_BG_TIMER, 0x08),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORECLK_DIV, 0x05),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_CTRL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE2_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_VCO_TUNE_CTRL, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CORE_CLK_EN, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_CMN_CONFIG, 0x02),
 };
 
-static const struct qmp_phy_init_tbl qmp_v2_dp_serdes_tbl_rbr[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x2c),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x69),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x07),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0xbf),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x21),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
+static const struct qmp_phy_init_tbl qcs615_dp_serdes_tbl_rbr[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x2c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x69),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x07),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0xbf),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x21),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
 };
 
-static const struct qmp_phy_init_tbl qmp_v2_dp_serdes_tbl_hbr[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x24),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x69),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x80),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x07),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0x3f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x38),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
+static const struct qmp_phy_init_tbl qcs615_dp_serdes_tbl_hbr[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x24),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x69),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x80),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x07),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0x3f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x38),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
 };
 
-static const struct qmp_phy_init_tbl qmp_v2_dp_serdes_tbl_hbr2[] = {
-	QMP_PHY_INIT_CFG(QSERDES_COM_HSCLK_SEL, 0x20),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DEC_START_MODE0, 0x8c),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START1_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START2_MODE0, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_COM_DIV_FRAC_START3_MODE0, 0x0a),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP1_MODE0, 0x7f),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP2_MODE0, 0x70),
-	QMP_PHY_INIT_CFG(QSERDES_COM_LOCK_CMP3_MODE0, 0x00),
+static const struct qmp_phy_init_tbl qcs615_dp_serdes_tbl_hbr2[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_HSCLK_SEL, 0x20),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DEC_START_MODE0, 0x8c),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START1_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START2_MODE0, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_DIV_FRAC_START3_MODE0, 0x0a),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP1_MODE0, 0x7f),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP2_MODE0, 0x70),
+	QMP_PHY_INIT_CFG(QSERDES_V2_COM_LOCK_CMP3_MODE0, 0x00),
 };
 
-static const struct qmp_phy_init_tbl qmp_v2_dp_tx_tbl[] = {
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_TRANSCEIVER_BIAS_EN, 0x1a),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_VMODE_CTRL1, 0x40),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_PRE_STALL_LDO_BOOST_EN, 0x30),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_INTERFACE_SELECT, 0x3d),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_CLKBUF_ENABLE, 0x0f),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_RESET_TSYNC_EN, 0x03),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_TRAN_DRVR_EMP_EN, 0x03),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_PARRATE_REC_DETECT_IDLE_EN, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_TX_INTERFACE_MODE, 0x00),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_TX_EMP_POST1_LVL, 0x2b),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_TX_DRV_LVL, 0x2f),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_TX_BAND, 0x4),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_RES_CODE_LANE_OFFSET_TX, 0x12),
-	QMP_PHY_INIT_CFG(QSERDES_V2_TX_RES_CODE_LANE_OFFSET_RX, 0x12),
+static const struct qmp_phy_init_tbl qcs615_dp_tx_tbl[] = {
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_TRANSCEIVER_BIAS_EN, 0x1a),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_VMODE_CTRL1, 0x40),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_PRE_STALL_LDO_BOOST_EN, 0x30),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_INTERFACE_SELECT, 0x3d),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_CLKBUF_ENABLE, 0x0f),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_RESET_TSYNC_EN, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_TRAN_DRVR_EMP_EN, 0x03),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_PARRATE_REC_DETECT_IDLE_EN, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_TX_INTERFACE_MODE, 0x00),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_TX_EMP_POST1_LVL, 0x2b),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_TX_DRV_LVL, 0x2f),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_TX_BAND, 0x4),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_RES_CODE_LANE_OFFSET_TX, 0x12),
+	QMP_PHY_INIT_CFG(QSERDES_V3_TX_RES_CODE_LANE_OFFSET_RX, 0x12),
 };
 
 struct qmp_usbc_offsets {
@@ -555,14 +555,14 @@ static const struct qmp_usbc_offsets qmp_usbc_usb3dp_offsets_qcs615 = {
 	.dp_dp_phy	= 0x1000,
 };
 
-static const u8 qmp_v2_dp_pre_emphasis_hbr2_rbr[4][4] = {
+static const u8 qcs615_dp_pre_emphasis_hbr2_rbr[4][4] = {
 	{0x00, 0x0b, 0x12, 0xff},
 	{0x00, 0x0a, 0x12, 0xff},
 	{0x00, 0x0c, 0xff, 0xff},
 	{0xff, 0xff, 0xff, 0xff}
 };
 
-static const u8 qmp_v2_dp_voltage_swing_hbr2_rbr[4][4] = {
+static const u8 qcs615_dp_voltage_swing_hbr2_rbr[4][4] = {
 	{0x07, 0x0f, 0x14, 0xff},
 	{0x11, 0x1d, 0x1f, 0xff},
 	{0x18, 0x1f, 0xff, 0xff},
@@ -641,10 +641,10 @@ static const struct qmp_phy_cfg qcs615_usb3phy_cfg = {
 	.regs			= qmp_v3_usb3phy_regs_layout_qcm2290,
 };
 
-static void qmp_v2_dp_aux_init(struct qmp_usbc *qmp);
-static void qmp_v2_configure_dp_tx(struct qmp_usbc *qmp);
-static int qmp_v2_configure_dp_phy(struct qmp_usbc *qmp);
-static int qmp_v2_calibrate_dp_phy(struct qmp_usbc *qmp);
+static void qcs615_qmp_dp_aux_init(struct qmp_usbc *qmp);
+static void qcs615_qmp_configure_dp_tx(struct qmp_usbc *qmp);
+static int qcs615_qmp_configure_dp_phy(struct qmp_usbc *qmp);
+static int qcs615_qmp_calibrate_dp_phy(struct qmp_usbc *qmp);
 
 static const struct qmp_phy_cfg qcs615_usb3dp_phy_cfg = {
 	.offsets		= &qmp_usbc_usb3dp_offsets_qcs615,
@@ -660,25 +660,25 @@ static const struct qmp_phy_cfg qcs615_usb3dp_phy_cfg = {
 
 	.regs			= qmp_v3_usb3phy_regs_layout_qcm2290,
 
-	.dp_serdes_tbl		= qmp_v2_dp_serdes_tbl,
-	.dp_serdes_tbl_num	= ARRAY_SIZE(qmp_v2_dp_serdes_tbl),
-	.dp_tx_tbl		= qmp_v2_dp_tx_tbl,
-	.dp_tx_tbl_num		= ARRAY_SIZE(qmp_v2_dp_tx_tbl),
+	.dp_serdes_tbl		= qcs615_dp_serdes_tbl,
+	.dp_serdes_tbl_num	= ARRAY_SIZE(qcs615_dp_serdes_tbl),
+	.dp_tx_tbl		= qcs615_dp_tx_tbl,
+	.dp_tx_tbl_num		= ARRAY_SIZE(qcs615_dp_tx_tbl),
 
-	.serdes_tbl_rbr		= qmp_v2_dp_serdes_tbl_rbr,
-	.serdes_tbl_rbr_num	= ARRAY_SIZE(qmp_v2_dp_serdes_tbl_rbr),
-	.serdes_tbl_hbr		= qmp_v2_dp_serdes_tbl_hbr,
-	.serdes_tbl_hbr_num	= ARRAY_SIZE(qmp_v2_dp_serdes_tbl_hbr),
-	.serdes_tbl_hbr2	= qmp_v2_dp_serdes_tbl_hbr2,
-	.serdes_tbl_hbr2_num	= ARRAY_SIZE(qmp_v2_dp_serdes_tbl_hbr2),
+	.serdes_tbl_rbr		= qcs615_dp_serdes_tbl_rbr,
+	.serdes_tbl_rbr_num	= ARRAY_SIZE(qcs615_dp_serdes_tbl_rbr),
+	.serdes_tbl_hbr		= qcs615_dp_serdes_tbl_hbr,
+	.serdes_tbl_hbr_num	= ARRAY_SIZE(qcs615_dp_serdes_tbl_hbr),
+	.serdes_tbl_hbr2	= qcs615_dp_serdes_tbl_hbr2,
+	.serdes_tbl_hbr2_num	= ARRAY_SIZE(qcs615_dp_serdes_tbl_hbr2),
 
-	.swing_tbl		= &qmp_v2_dp_voltage_swing_hbr2_rbr,
-	.pre_emphasis_tbl	= &qmp_v2_dp_pre_emphasis_hbr2_rbr,
+	.swing_tbl		= &qcs615_dp_voltage_swing_hbr2_rbr,
+	.pre_emphasis_tbl	= &qcs615_dp_pre_emphasis_hbr2_rbr,
 
-	.dp_aux_init		= qmp_v2_dp_aux_init,
-	.configure_dp_tx	= qmp_v2_configure_dp_tx,
-	.configure_dp_phy	= qmp_v2_configure_dp_phy,
-	.calibrate_dp_phy	= qmp_v2_calibrate_dp_phy,
+	.dp_aux_init		= qcs615_qmp_dp_aux_init,
+	.configure_dp_tx	= qcs615_qmp_configure_dp_tx,
+	.configure_dp_phy	= qcs615_qmp_configure_dp_phy,
+	.calibrate_dp_phy	= qcs615_qmp_calibrate_dp_phy,
 
 	.reset_list		= usb3dpphy_reset_l,
 	.num_resets		= ARRAY_SIZE(usb3dpphy_reset_l),
@@ -744,7 +744,7 @@ static int qmp_usbc_com_exit(struct phy *phy)
 	return 0;
 }
 
-static void qmp_v2_dp_aux_init(struct qmp_usbc *qmp)
+static void qcs615_qmp_dp_aux_init(struct qmp_usbc *qmp)
 {
 	writel(DP_PHY_PD_CTL_AUX_PWRDN |
 	       DP_PHY_PD_CTL_LANE_0_1_PWRDN | DP_PHY_PD_CTL_LANE_2_3_PWRDN |
@@ -774,7 +774,7 @@ static void qmp_v2_dp_aux_init(struct qmp_usbc *qmp)
 	       qmp->dp_dp_phy + QSERDES_V2_DP_PHY_AUX_INTERRUPT_MASK);
 }
 
-static int qmp_v2_configure_dp_swing(struct qmp_usbc *qmp)
+static int qcs615_qmp_configure_dp_swing(struct qmp_usbc *qmp)
 {
 	const struct qmp_phy_cfg *cfg = qmp->cfg;
 	const struct phy_configure_opts_dp *dp_opts = &qmp->dp_opts;
@@ -794,7 +794,7 @@ static int qmp_v2_configure_dp_swing(struct qmp_usbc *qmp)
 		p_level = max(p_level, dp_opts->pre[i]);
 	}
 
-	if (v_level > 4 || p_level > 4) {
+	if (v_level >= 4 || p_level >= 4) {
 		dev_err(qmp->dev, "Invalid v(%d) | p(%d) level)\n",
 			v_level, p_level);
 		return -EINVAL;
@@ -809,10 +809,10 @@ static int qmp_v2_configure_dp_swing(struct qmp_usbc *qmp)
 	if (voltage_swing_cfg == 0xff && pre_emphasis_cfg == 0xff)
 		return -EINVAL;
 
-	writel(voltage_swing_cfg, tx + QSERDES_V2_TX_TX_DRV_LVL);
-	writel(pre_emphasis_cfg, tx + QSERDES_V2_TX_TX_EMP_POST1_LVL);
-	writel(voltage_swing_cfg, tx2 + QSERDES_V2_TX_TX_DRV_LVL);
-	writel(pre_emphasis_cfg, tx2 + QSERDES_V2_TX_TX_EMP_POST1_LVL);
+	writel(voltage_swing_cfg, tx + QSERDES_V3_TX_TX_DRV_LVL);
+	writel(pre_emphasis_cfg, tx + QSERDES_V3_TX_TX_EMP_POST1_LVL);
+	writel(voltage_swing_cfg, tx2 + QSERDES_V3_TX_TX_DRV_LVL);
+	writel(pre_emphasis_cfg, tx2 + QSERDES_V3_TX_TX_EMP_POST1_LVL);
 
 	return 0;
 }
@@ -864,30 +864,30 @@ static int qmp_usbc_configure_dp_clocks(struct qmp_usbc *qmp)
 	return 0;
 }
 
-static void qmp_v2_configure_dp_tx(struct qmp_usbc *qmp)
+static void qcs615_qmp_configure_dp_tx(struct qmp_usbc *qmp)
 {
 	const struct phy_configure_opts_dp *dp_opts = &qmp->dp_opts;
 	void __iomem *tx = qmp->dp_tx;
 	void __iomem *tx2 = qmp->dp_tx2;
 
 	/* program default setting first */
-	writel(0x2a, tx + QSERDES_V2_TX_TX_DRV_LVL);
-	writel(0x20, tx + QSERDES_V2_TX_TX_EMP_POST1_LVL);
-	writel(0x2a, tx2 + QSERDES_V2_TX_TX_DRV_LVL);
-	writel(0x20, tx2 + QSERDES_V2_TX_TX_EMP_POST1_LVL);
+	writel(0x2a, tx + QSERDES_V3_TX_TX_DRV_LVL);
+	writel(0x20, tx + QSERDES_V3_TX_TX_EMP_POST1_LVL);
+	writel(0x2a, tx2 + QSERDES_V3_TX_TX_DRV_LVL);
+	writel(0x20, tx2 + QSERDES_V3_TX_TX_EMP_POST1_LVL);
 
 	if (dp_opts->link_rate >= 2700) {
-		writel(0xc4, tx + QSERDES_V2_TX_LANE_MODE_1);
-		writel(0xc4, tx2 + QSERDES_V2_TX_LANE_MODE_1);
+		writel(0xc4, tx + QSERDES_V3_TX_LANE_MODE_1);
+		writel(0xc4, tx2 + QSERDES_V3_TX_LANE_MODE_1);
 	} else {
-		writel(0xc6, tx + QSERDES_V2_TX_LANE_MODE_1);
-		writel(0xc6, tx2 + QSERDES_V2_TX_LANE_MODE_1);
+		writel(0xc6, tx + QSERDES_V3_TX_LANE_MODE_1);
+		writel(0xc6, tx2 + QSERDES_V3_TX_LANE_MODE_1);
 	}
 
-	qmp_v2_configure_dp_swing(qmp);
+	qcs615_qmp_configure_dp_swing(qmp);
 }
 
-static int qmp_v2_configure_dp_phy(struct qmp_usbc *qmp)
+static int qcs615_qmp_configure_dp_phy(struct qmp_usbc *qmp)
 {
 	u32 status;
 	int ret;
@@ -906,9 +906,9 @@ static int qmp_v2_configure_dp_phy(struct qmp_usbc *qmp)
 	writel(0x01, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG);
 	writel(0x09, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG);
 
-	writel(0x20, qmp->dp_serdes + QSERDES_COM_RESETSM_CNTRL);
+	writel(0x20, qmp->dp_serdes + QSERDES_V2_COM_RESETSM_CNTRL);
 
-	if (readl_poll_timeout(qmp->dp_serdes + QSERDES_COM_C_READY_STATUS,
+	if (readl_poll_timeout(qmp->dp_serdes + QSERDES_V2_COM_C_READY_STATUS,
 			       status,
 			       ((status & BIT(0)) > 0),
 			       500,
@@ -917,7 +917,7 @@ static int qmp_v2_configure_dp_phy(struct qmp_usbc *qmp)
 		return -ETIMEDOUT;
 	}
 
-	if (readl_poll_timeout(qmp->dp_serdes + QSERDES_COM_CMN_STATUS,
+	if (readl_poll_timeout(qmp->dp_serdes + QSERDES_V2_COM_CMN_STATUS,
 			       status,
 			       ((status & BIT(0)) > 0),
 			       500,
@@ -926,7 +926,7 @@ static int qmp_v2_configure_dp_phy(struct qmp_usbc *qmp)
 		return -ETIMEDOUT;
 	}
 
-	if (readl_poll_timeout(qmp->dp_serdes + QSERDES_COM_CMN_STATUS,
+	if (readl_poll_timeout(qmp->dp_serdes + QSERDES_V2_COM_CMN_STATUS,
 			       status,
 			       ((status & BIT(1)) > 0),
 			       500,
@@ -955,12 +955,12 @@ static int qmp_v2_configure_dp_phy(struct qmp_usbc *qmp)
 		return -ETIMEDOUT;
 	}
 
-	writel(0x3f, qmp->dp_tx + QSERDES_V2_TX_TRANSCEIVER_BIAS_EN);
-	writel(0x10, qmp->dp_tx + QSERDES_V2_TX_HIGHZ_DRVR_EN);
-	writel(0x0a, qmp->dp_tx + QSERDES_V2_TX_TX_POL_INV);
-	writel(0x3f, qmp->dp_tx2 + QSERDES_V2_TX_TRANSCEIVER_BIAS_EN);
-	writel(0x10, qmp->dp_tx2 + QSERDES_V2_TX_HIGHZ_DRVR_EN);
-	writel(0x0a, qmp->dp_tx2 + QSERDES_V2_TX_TX_POL_INV);
+	writel(0x3f, qmp->dp_tx + QSERDES_V3_TX_TRANSCEIVER_BIAS_EN);
+	writel(0x10, qmp->dp_tx + QSERDES_V3_TX_HIGHZ_DRVR_EN);
+	writel(0x0a, qmp->dp_tx + QSERDES_V3_TX_TX_POL_INV);
+	writel(0x3f, qmp->dp_tx2 + QSERDES_V3_TX_TRANSCEIVER_BIAS_EN);
+	writel(0x10, qmp->dp_tx2 + QSERDES_V3_TX_HIGHZ_DRVR_EN);
+	writel(0x0a, qmp->dp_tx2 + QSERDES_V3_TX_TX_POL_INV);
 
 	writel(0x18, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG);
 	writel(0x19, qmp->dp_dp_phy + QSERDES_DP_PHY_CFG);
@@ -977,7 +977,7 @@ static int qmp_v2_configure_dp_phy(struct qmp_usbc *qmp)
 	return 0;
 }
 
-static int qmp_v2_calibrate_dp_phy(struct qmp_usbc *qmp)
+static int qcs615_qmp_calibrate_dp_phy(struct qmp_usbc *qmp)
 {
 	static const u8 cfg1_settings[] = {0x13, 0x23, 0x1d};
 	u8 val;
diff --git a/drivers/phy/qualcomm/phy-qcom-qmp.h b/drivers/phy/qualcomm/phy-qcom-qmp.h
index a873bdd7bffe6..11b7e03b4fab7 100644
--- a/drivers/phy/qualcomm/phy-qcom-qmp.h
+++ b/drivers/phy/qualcomm/phy-qcom-qmp.h
@@ -6,9 +6,6 @@
 #ifndef QCOM_PHY_QMP_H_
 #define QCOM_PHY_QMP_H_
 
-#include "phy-qcom-qmp-qserdes-com.h"
-#include "phy-qcom-qmp-qserdes-txrx.h"
-
 #include "phy-qcom-qmp-qserdes-com-v2.h"
 #include "phy-qcom-qmp-qserdes-txrx-v2.h"
 
diff --git a/drivers/phy/qualcomm/phy-qcom-usb-hs.c b/drivers/phy/qualcomm/phy-qcom-usb-hs.c
index 98a18987f1bec..928a982a8a76c 100644
--- a/drivers/phy/qualcomm/phy-qcom-usb-hs.c
+++ b/drivers/phy/qualcomm/phy-qcom-usb-hs.c
@@ -34,9 +34,9 @@ struct qcom_usb_hs_phy {
 	struct regulator *v1p8;
 	struct regulator *v3p3;
 	struct reset_control *reset;
-	struct ulpi_seq *init_seq;
 	struct extcon_dev *vbus_edev;
 	struct notifier_block vbus_notify;
+	struct ulpi_seq init_seq[];
 };
 
 static int qcom_usb_hs_phy_set_mode(struct phy *phy,
@@ -209,19 +209,16 @@ static int qcom_usb_hs_phy_probe(struct ulpi *ulpi)
 	int size;
 	int ret;
 
-	uphy = devm_kzalloc(&ulpi->dev, sizeof(*uphy), GFP_KERNEL);
+	size = of_property_count_u8_elems(ulpi->dev.of_node, "qcom,init-seq");
+	if (size < 0)
+		size = 0;
+
+	uphy = devm_kzalloc(&ulpi->dev, struct_size(uphy, init_seq, (size / 2) + 1), GFP_KERNEL);
 	if (!uphy)
 		return -ENOMEM;
 	ulpi_set_drvdata(ulpi, uphy);
 	uphy->ulpi = ulpi;
 
-	size = of_property_count_u8_elems(ulpi->dev.of_node, "qcom,init-seq");
-	if (size < 0)
-		size = 0;
-	uphy->init_seq = devm_kmalloc_array(&ulpi->dev, (size / 2) + 1,
-					   sizeof(*uphy->init_seq), GFP_KERNEL);
-	if (!uphy->init_seq)
-		return -ENOMEM;
 	ret = of_property_read_u8_array(ulpi->dev.of_node, "qcom,init-seq",
 					(u8 *)uphy->init_seq, size);
 	if (ret && size)
diff --git a/drivers/phy/renesas/phy-rcar-gen3-usb2.c b/drivers/phy/renesas/phy-rcar-gen3-usb2.c
index 79e820e2fe55d..9a45d840efeb2 100644
--- a/drivers/phy/renesas/phy-rcar-gen3-usb2.c
+++ b/drivers/phy/renesas/phy-rcar-gen3-usb2.c
@@ -314,13 +314,11 @@ static void rcar_gen3_init_from_a_peri_to_a_host(struct rcar_gen3_chan *ch)
 static bool rcar_gen3_check_id(struct rcar_gen3_chan *ch)
 {
 	if (ch->phy_data->vblvl_ctrl) {
-		bool vbus_valid;
-		bool device;
+		u32 val = readl(ch->base + USB2_ADPCTRL);
+		bool vbus_valid = val & USB2_ADPCTRL_VBUSVALID;
+		bool device = val & USB2_ADPCTRL_IDDIG;
 
-		device = !!(readl(ch->base + USB2_ADPCTRL) & USB2_ADPCTRL_IDDIG);
-		vbus_valid = !!(readl(ch->base + USB2_ADPCTRL) & USB2_ADPCTRL_VBUSVALID);
-
-		return vbus_valid ? device : !device;
+		return device == vbus_valid;
 	}
 
 	if (!ch->uses_otg_pins)
diff --git a/drivers/phy/renesas/phy-rzg3e-usb3.c b/drivers/phy/renesas/phy-rzg3e-usb3.c
index 6b3453ea0004c..060309547ea52 100644
--- a/drivers/phy/renesas/phy-rzg3e-usb3.c
+++ b/drivers/phy/renesas/phy-rzg3e-usb3.c
@@ -78,13 +78,11 @@ static void rzg3e_phy_usb2test_phy_init(void __iomem *base)
 	writel(val, base + USB3_TEST_UTMICTRL2);
 
 	val = readl(base + USB3_TEST_PRMCTRL5_R);
-	val &= ~USB3_TEST_PRMCTRL5_R_TXPREEMPAMPTUNE0_MASK;
-	val |= FIELD_PREP(USB3_TEST_PRMCTRL5_R_TXPREEMPAMPTUNE0_MASK, 2);
+	FIELD_MODIFY(USB3_TEST_PRMCTRL5_R_TXPREEMPAMPTUNE0_MASK, &val, 2);
 	writel(val, base + USB3_TEST_PRMCTRL5_R);
 
 	val = readl(base + USB3_TEST_PRMCTRL6_R);
-	val &= ~USB3_TEST_PRMCTRL6_R_OTGTUNE0_MASK;
-	val |= FIELD_PREP(USB3_TEST_PRMCTRL6_R_OTGTUNE0_MASK, 7);
+	FIELD_MODIFY(USB3_TEST_PRMCTRL6_R_OTGTUNE0_MASK, &val, 7);
 	writel(val, base + USB3_TEST_PRMCTRL6_R);
 
 	val = readl(base + USB3_TEST_RESET);
@@ -256,4 +254,4 @@ module_platform_driver(rzg3e_phy_usb3_driver);
 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Renesas RZ/G3E USB3.0 PHY Driver");
-MODULE_AUTHOR("biju.das.jz@bp.renesas.com>");
+MODULE_AUTHOR("Biju Das <biju.das.jz@bp.renesas.com>");
diff --git a/drivers/phy/rockchip/phy-rockchip-inno-usb2.c b/drivers/phy/rockchip/phy-rockchip-inno-usb2.c
index 8f4c08e599aa2..133cfd6624e82 100644
--- a/drivers/phy/rockchip/phy-rockchip-inno-usb2.c
+++ b/drivers/phy/rockchip/phy-rockchip-inno-usb2.c
@@ -179,6 +179,7 @@ struct rockchip_usb2phy_cfg {
 	unsigned int	num_ports;
 	int (*phy_tuning)(struct rockchip_usb2phy *rphy);
 	struct usb2phy_reg	clkout_ctl;
+	struct usb2phy_reg	clkout_ctl_phy;
 	const struct rockchip_usb2phy_port_cfg	port_cfgs[USB2PHY_NUM_PORTS];
 	const struct rockchip_chg_det_reg	chg_det;
 };
@@ -228,7 +229,7 @@ struct rockchip_usb2phy_port {
  * struct rockchip_usb2phy - usb2.0 phy driver data.
  * @dev: pointer to device.
  * @grf: General Register Files regmap.
- * @usbgrf: USB General Register Files regmap.
+ * @phy_base: USB PHY regmap.
  * @clks: array of phy input clocks.
  * @clk480m: clock struct of phy output clk.
  * @clk480m_hw: clock struct of phy output clk management.
@@ -246,7 +247,7 @@ struct rockchip_usb2phy_port {
 struct rockchip_usb2phy {
 	struct device	*dev;
 	struct regmap	*grf;
-	struct regmap	*usbgrf;
+	struct regmap	*phy_base;
 	struct clk_bulk_data	*clks;
 	struct clk	*clk480m;
 	struct clk_hw	clk480m_hw;
@@ -261,11 +262,6 @@ struct rockchip_usb2phy {
 	struct rockchip_usb2phy_port	ports[USB2PHY_NUM_PORTS];
 };
 
-static inline struct regmap *get_reg_base(struct rockchip_usb2phy *rphy)
-{
-	return rphy->usbgrf == NULL ? rphy->grf : rphy->usbgrf;
-}
-
 static inline int property_enable(struct regmap *base,
 				  const struct usb2phy_reg *reg, bool en)
 {
@@ -319,16 +315,33 @@ static void rockchip_usb2phy_clk_bulk_disable(void *data)
 	clk_bulk_disable_unprepare(rphy->num_clks, rphy->clks);
 }
 
-static int rockchip_usb2phy_clk480m_prepare(struct clk_hw *hw)
+static void
+rockchip_usb2phy_clk480m_clkout_ctl(struct clk_hw *hw, struct regmap **base,
+				    const struct usb2phy_reg **clkout_ctl)
 {
 	struct rockchip_usb2phy *rphy =
 		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
-	struct regmap *base = get_reg_base(rphy);
+
+	if (rphy->phy_cfg->clkout_ctl_phy.enable) {
+		*base = rphy->phy_base;
+		*clkout_ctl = &rphy->phy_cfg->clkout_ctl_phy;
+	} else {
+		*base = rphy->grf;
+		*clkout_ctl = &rphy->phy_cfg->clkout_ctl;
+	}
+}
+
+static int rockchip_usb2phy_clk480m_prepare(struct clk_hw *hw)
+{
+	const struct usb2phy_reg *clkout_ctl;
+	struct regmap *base;
 	int ret;
 
+	rockchip_usb2phy_clk480m_clkout_ctl(hw, &base, &clkout_ctl);
+
 	/* turn on 480m clk output if it is off */
-	if (!property_enabled(base, &rphy->phy_cfg->clkout_ctl)) {
-		ret = property_enable(base, &rphy->phy_cfg->clkout_ctl, true);
+	if (!property_enabled(base, clkout_ctl)) {
+		ret = property_enable(base, clkout_ctl, true);
 		if (ret)
 			return ret;
 
@@ -341,21 +354,23 @@ static int rockchip_usb2phy_clk480m_prepare(struct clk_hw *hw)
 
 static void rockchip_usb2phy_clk480m_unprepare(struct clk_hw *hw)
 {
-	struct rockchip_usb2phy *rphy =
-		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
-	struct regmap *base = get_reg_base(rphy);
+	const struct usb2phy_reg *clkout_ctl;
+	struct regmap *base;
+
+	rockchip_usb2phy_clk480m_clkout_ctl(hw, &base, &clkout_ctl);
 
 	/* turn off 480m clk output */
-	property_enable(base, &rphy->phy_cfg->clkout_ctl, false);
+	property_enable(base, clkout_ctl, false);
 }
 
 static int rockchip_usb2phy_clk480m_prepared(struct clk_hw *hw)
 {
-	struct rockchip_usb2phy *rphy =
-		container_of(hw, struct rockchip_usb2phy, clk480m_hw);
-	struct regmap *base = get_reg_base(rphy);
+	const struct usb2phy_reg *clkout_ctl;
+	struct regmap *base;
 
-	return property_enabled(base, &rphy->phy_cfg->clkout_ctl);
+	rockchip_usb2phy_clk480m_clkout_ctl(hw, &base, &clkout_ctl);
+
+	return property_enabled(base, clkout_ctl);
 }
 
 static unsigned long
@@ -574,7 +589,6 @@ static int rockchip_usb2phy_power_on(struct phy *phy)
 {
 	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
 	struct rockchip_usb2phy *rphy = dev_get_drvdata(phy->dev.parent);
-	struct regmap *base = get_reg_base(rphy);
 	int ret;
 
 	dev_dbg(&rport->phy->dev, "port power on\n");
@@ -586,7 +600,7 @@ static int rockchip_usb2phy_power_on(struct phy *phy)
 	if (ret)
 		return ret;
 
-	ret = property_enable(base, &rport->port_cfg->phy_sus, false);
+	ret = property_enable(rphy->grf, &rport->port_cfg->phy_sus, false);
 	if (ret) {
 		clk_disable_unprepare(rphy->clk480m);
 		return ret;
@@ -615,7 +629,6 @@ static int rockchip_usb2phy_power_off(struct phy *phy)
 {
 	struct rockchip_usb2phy_port *rport = phy_get_drvdata(phy);
 	struct rockchip_usb2phy *rphy = dev_get_drvdata(phy->dev.parent);
-	struct regmap *base = get_reg_base(rphy);
 	int ret;
 
 	dev_dbg(&rport->phy->dev, "port power off\n");
@@ -623,7 +636,7 @@ static int rockchip_usb2phy_power_off(struct phy *phy)
 	if (rport->suspended)
 		return 0;
 
-	ret = property_enable(base, &rport->port_cfg->phy_sus, true);
+	ret = property_enable(rphy->grf, &rport->port_cfg->phy_sus, true);
 	if (ret)
 		return ret;
 
@@ -787,28 +800,22 @@ static const char *chg_to_string(enum power_supply_type chg_type)
 static void rockchip_chg_enable_dcd(struct rockchip_usb2phy *rphy,
 				    bool en)
 {
-	struct regmap *base = get_reg_base(rphy);
-
-	property_enable(base, &rphy->phy_cfg->chg_det.rdm_pdwn_en, en);
-	property_enable(base, &rphy->phy_cfg->chg_det.idp_src_en, en);
+	property_enable(rphy->grf, &rphy->phy_cfg->chg_det.rdm_pdwn_en, en);
+	property_enable(rphy->grf, &rphy->phy_cfg->chg_det.idp_src_en, en);
 }
 
 static void rockchip_chg_enable_primary_det(struct rockchip_usb2phy *rphy,
 					    bool en)
 {
-	struct regmap *base = get_reg_base(rphy);
-
-	property_enable(base, &rphy->phy_cfg->chg_det.vdp_src_en, en);
-	property_enable(base, &rphy->phy_cfg->chg_det.idm_sink_en, en);
+	property_enable(rphy->grf, &rphy->phy_cfg->chg_det.vdp_src_en, en);
+	property_enable(rphy->grf, &rphy->phy_cfg->chg_det.idm_sink_en, en);
 }
 
 static void rockchip_chg_enable_secondary_det(struct rockchip_usb2phy *rphy,
 					      bool en)
 {
-	struct regmap *base = get_reg_base(rphy);
-
-	property_enable(base, &rphy->phy_cfg->chg_det.vdm_src_en, en);
-	property_enable(base, &rphy->phy_cfg->chg_det.idp_sink_en, en);
+	property_enable(rphy->grf, &rphy->phy_cfg->chg_det.vdm_src_en, en);
+	property_enable(rphy->grf, &rphy->phy_cfg->chg_det.idp_sink_en, en);
 }
 
 #define CHG_DCD_POLL_TIME	(100 * HZ / 1000)
@@ -820,7 +827,6 @@ static void rockchip_chg_detect_work(struct work_struct *work)
 	struct rockchip_usb2phy_port *rport =
 		container_of(work, struct rockchip_usb2phy_port, chg_work.work);
 	struct rockchip_usb2phy *rphy = dev_get_drvdata(rport->phy->dev.parent);
-	struct regmap *base = get_reg_base(rphy);
 	bool is_dcd, tmout, vout, vbus_attach;
 	unsigned long delay;
 
@@ -834,7 +840,7 @@ static void rockchip_chg_detect_work(struct work_struct *work)
 			rockchip_usb2phy_power_off(rport->phy);
 		/* put the controller in non-driving mode */
 		if (!vbus_attach)
-			property_enable(base, &rphy->phy_cfg->chg_det.opmode, false);
+			property_enable(rphy->grf, &rphy->phy_cfg->chg_det.opmode, false);
 		/* Start DCD processing stage 1 */
 		rockchip_chg_enable_dcd(rphy, true);
 		rphy->chg_state = USB_CHG_STATE_WAIT_FOR_DCD;
@@ -898,7 +904,7 @@ static void rockchip_chg_detect_work(struct work_struct *work)
 	case USB_CHG_STATE_DETECTED:
 		/* put the controller in normal mode */
 		if (!vbus_attach)
-			property_enable(base, &rphy->phy_cfg->chg_det.opmode, true);
+			property_enable(rphy->grf, &rphy->phy_cfg->chg_det.opmode, true);
 		rockchip_usb2phy_otg_sm_work(&rport->otg_sm_work.work);
 		dev_dbg(&rport->phy->dev, "charger = %s\n",
 			 chg_to_string(rphy->chg_type));
@@ -1353,27 +1359,18 @@ static int rockchip_usb2phy_probe(struct platform_device *pdev)
 	if (!rphy)
 		return -ENOMEM;
 
-	if (!dev->parent || !dev->parent->of_node) {
+	if (!dev->parent || !dev->parent->of_node ||
+	    of_property_present(np, "rockchip,usbgrf")) {
+		rphy->phy_base = device_node_to_regmap(np);
+		if (IS_ERR(rphy->phy_base))
+			return PTR_ERR(rphy->phy_base);
 		rphy->grf = syscon_regmap_lookup_by_phandle(np, "rockchip,usbgrf");
-		if (IS_ERR(rphy->grf)) {
-			dev_err(dev, "failed to locate usbgrf\n");
-			return PTR_ERR(rphy->grf);
-		}
 	} else {
 		rphy->grf = syscon_node_to_regmap(dev->parent->of_node);
-		if (IS_ERR(rphy->grf))
-			return PTR_ERR(rphy->grf);
-	}
-
-	if (of_device_is_compatible(np, "rockchip,rv1108-usb2phy")) {
-		rphy->usbgrf =
-			syscon_regmap_lookup_by_phandle(dev->of_node,
-							"rockchip,usbgrf");
-		if (IS_ERR(rphy->usbgrf))
-			return PTR_ERR(rphy->usbgrf);
-	} else {
-		rphy->usbgrf = NULL;
+		rphy->phy_base = rphy->grf;
 	}
+	if (IS_ERR(rphy->grf))
+		return PTR_ERR(rphy->grf);
 
 	if (of_property_read_u32_index(np, "reg", 0, &reg)) {
 		dev_err(dev, "the reg property is not assigned in %pOFn node\n", np);
@@ -1514,6 +1511,38 @@ static int rk3128_usb2phy_tuning(struct rockchip_usb2phy *rphy)
 				BIT(2) << BIT_WRITEABLE_SHIFT | 0);
 }
 
+static int rk3528_usb2phy_tuning(struct rockchip_usb2phy *rphy)
+{
+	int ret;
+
+	/* Turn off otg port differential receiver in suspend mode */
+	ret = regmap_write(rphy->phy_base, 0x30, BIT(18) | 0x0000);
+	if (ret)
+		return ret;
+
+	/* Turn off host port differential receiver in suspend mode */
+	ret = regmap_write(rphy->phy_base, 0x430, BIT(18) | 0x0000);
+	if (ret)
+		return ret;
+
+	/* Set otg port HS eye height to 400mv (default is 450mv) */
+	ret = regmap_write(rphy->phy_base, 0x30, GENMASK(22, 20) | 0x0000);
+	if (ret)
+		return ret;
+
+	/* Set host port HS eye height to 400mv (default is 450mv) */
+	ret = regmap_write(rphy->phy_base, 0x430, GENMASK(22, 20) | 0x0000);
+	if (ret)
+		return ret;
+
+	/* Choose the Tx fs/ls data as linestate from TX driver for otg port */
+	ret = regmap_write(rphy->phy_base, 0x94, GENMASK(22, 19) | 0x0018);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
 static int rk3576_usb2phy_tuning(struct rockchip_usb2phy *rphy)
 {
 	int ret;
@@ -1927,6 +1956,57 @@ static const struct rockchip_usb2phy_cfg rk3399_phy_cfgs[] = {
 	{ /* sentinel */ }
 };
 
+static const struct rockchip_usb2phy_cfg rk3528_phy_cfgs[] = {
+	{
+		.reg = 0xffdf0000,
+		.num_ports	= 2,
+		.phy_tuning	= rk3528_usb2phy_tuning,
+		.clkout_ctl_phy	= { 0x041c, 7, 2, 0, 0x27 },
+		.port_cfgs	= {
+			[USB2PHY_PORT_OTG] = {
+				.phy_sus	= { 0x004c, 8, 0, 0, 0x1d1 },
+				.bvalid_det_en	= { 0x0074, 3, 2, 0, 3 },
+				.bvalid_det_st	= { 0x0078, 3, 2, 0, 3 },
+				.bvalid_det_clr	= { 0x007c, 3, 2, 0, 3 },
+				.idfall_det_en	= { 0x0074, 5, 5, 0, 1 },
+				.idfall_det_st	= { 0x0078, 5, 5, 0, 1 },
+				.idfall_det_clr	= { 0x007c, 5, 5, 0, 1 },
+				.idrise_det_en	= { 0x0074, 4, 4, 0, 1 },
+				.idrise_det_st	= { 0x0078, 4, 4, 0, 1 },
+				.idrise_det_clr	= { 0x007c, 4, 4, 0, 1 },
+				.ls_det_en	= { 0x0074, 0, 0, 0, 1 },
+				.ls_det_st	= { 0x0078, 0, 0, 0, 1 },
+				.ls_det_clr	= { 0x007c, 0, 0, 0, 1 },
+				.utmi_avalid	= { 0x006c, 1, 1, 0, 1 },
+				.utmi_bvalid	= { 0x006c, 0, 0, 0, 1 },
+				.utmi_id	= { 0x006c, 6, 6, 0, 1 },
+				.utmi_ls	= { 0x006c, 5, 4, 0, 1 },
+			},
+			[USB2PHY_PORT_HOST] = {
+				.phy_sus	= { 0x005c, 8, 0, 0x1d2, 0x1d1 },
+				.ls_det_en	= { 0x0090, 0, 0, 0, 1 },
+				.ls_det_st	= { 0x0094, 0, 0, 0, 1 },
+				.ls_det_clr	= { 0x0098, 0, 0, 0, 1 },
+				.utmi_ls	= { 0x006c, 13, 12, 0, 1 },
+				.utmi_hstdet	= { 0x006c, 15, 15, 0, 1 },
+			}
+		},
+		.chg_det = {
+			.opmode		= { 0x004c, 3, 0, 5, 1 },
+			.cp_det		= { 0x006c, 19, 19, 0, 1 },
+			.dcp_det	= { 0x006c, 18, 18, 0, 1 },
+			.dp_det		= { 0x006c, 20, 20, 0, 1 },
+			.idm_sink_en	= { 0x0058, 1, 1, 0, 1 },
+			.idp_sink_en	= { 0x0058, 0, 0, 0, 1 },
+			.idp_src_en	= { 0x0058, 2, 2, 0, 1 },
+			.rdm_pdwn_en	= { 0x0058, 3, 3, 0, 1 },
+			.vdm_src_en	= { 0x0058, 5, 5, 0, 1 },
+			.vdp_src_en	= { 0x0058, 4, 4, 0, 1 },
+		},
+	},
+	{ /* sentinel */ }
+};
+
 static const struct rockchip_usb2phy_cfg rk3562_phy_cfgs[] = {
 	{
 		.reg = 0xff740000,
@@ -2294,6 +2374,7 @@ static const struct of_device_id rockchip_usb2phy_dt_match[] = {
 	{ .compatible = "rockchip,rk3328-usb2phy", .data = &rk3328_phy_cfgs },
 	{ .compatible = "rockchip,rk3366-usb2phy", .data = &rk3366_phy_cfgs },
 	{ .compatible = "rockchip,rk3399-usb2phy", .data = &rk3399_phy_cfgs },
+	{ .compatible = "rockchip,rk3528-usb2phy", .data = &rk3528_phy_cfgs },
 	{ .compatible = "rockchip,rk3562-usb2phy", .data = &rk3562_phy_cfgs },
 	{ .compatible = "rockchip,rk3568-usb2phy", .data = &rk3568_phy_cfgs },
 	{ .compatible = "rockchip,rk3576-usb2phy", .data = &rk3576_phy_cfgs },
diff --git a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c
index b60d6bf3f33c5..2b0f152f54709 100644
--- a/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c
+++ b/drivers/phy/rockchip/phy-rockchip-naneng-combphy.c
@@ -121,6 +121,7 @@
 #define RK3568_PHYREG32_SSC_OFFSET_500PPM	1
 
 #define RK3568_PHYREG33				0x80
+#define RK3568_PHYREG33_PLL_SSC_CTRL		BIT(5)
 #define RK3568_PHYREG33_PLL_KVCO_MASK		GENMASK(4, 2)
 #define RK3568_PHYREG33_PLL_KVCO_SHIFT		2
 #define RK3568_PHYREG33_PLL_KVCO_VALUE		2
@@ -446,6 +447,74 @@ static int rockchip_combphy_probe(struct platform_device *pdev)
 	return PTR_ERR_OR_ZERO(phy_provider);
 }
 
+static void rk_combphy_common_cfg_ssc(struct rockchip_combphy_priv *priv, unsigned long rate)
+{
+	struct device_node *np = priv->dev->of_node;
+	u32 val;
+
+	if (!priv->enable_ssc)
+		return;
+
+	/* Set SSC downward spread spectrum for PCIe and USB3 */
+	if (priv->type == PHY_TYPE_PCIE || priv->type == PHY_TYPE_USB3) {
+		val = FIELD_PREP(RK3568_PHYREG32_SSC_MASK, RK3568_PHYREG32_SSC_DOWNWARD);
+		rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val, RK3568_PHYREG32);
+	}
+
+	/* Set SSC downward spread spectrum +500ppm for SATA in 100MHz */
+	if (priv->type == PHY_TYPE_SATA && rate == REF_CLOCK_100MHz) {
+		val = FIELD_PREP(RK3568_PHYREG32_SSC_DIR_MASK,
+				 RK3568_PHYREG32_SSC_DOWNWARD);
+		val |= FIELD_PREP(RK3568_PHYREG32_SSC_OFFSET_MASK,
+				  RK3568_PHYREG32_SSC_OFFSET_500PPM);
+		rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val,
+					 RK3568_PHYREG32);
+	}
+
+	/* Enable SSC */
+	val = readl(priv->mmio + RK3568_PHYREG8);
+	val |= RK3568_PHYREG8_SSC_EN;
+	writel(val, priv->mmio + RK3568_PHYREG8);
+
+	/* Some SoCs need tuning PCIe SSC instead of default configuration in 24MHz */
+	if (!of_device_is_compatible(np, "rockchip,rk3588-naneng-combphy") &&
+	    !of_device_is_compatible(np, "rockchip,rk3576-naneng-combphy"))
+		return;
+
+	/* PLL control SSC module period should be set if need tuning */
+	val = readl(priv->mmio + RK3568_PHYREG33);
+	val |= RK3568_PHYREG33_PLL_SSC_CTRL;
+	writel(val, priv->mmio + RK3568_PHYREG33);
+
+	if (priv->type == PHY_TYPE_PCIE && rate == REF_CLOCK_24MHz) {
+		/* Set PLL loop divider */
+		writel(0x00, priv->mmio + RK3576_PHYREG17);
+		writel(RK3568_PHYREG18_PLL_LOOP, priv->mmio + RK3568_PHYREG18);
+
+		/* Set up rx_pck invert and rx msb to disable */
+		writel(0x00, priv->mmio + RK3588_PHYREG27);
+
+		/*
+		 * Set up SU adjust signal:
+		 * su_trim[7:0],   PLL KVCO adjust bits[2:0] to min
+		 * su_trim[15:8],  PLL LPF R1 adujst bits[9:7]=3'b101
+		 * su_trim[23:16], CKRCV adjust
+		 * su_trim[31:24], CKDRV adjust
+		 */
+		writel(0x90, priv->mmio + RK3568_PHYREG11);
+		writel(0x02, priv->mmio + RK3568_PHYREG12);
+		writel(0x08, priv->mmio + RK3568_PHYREG13);
+		writel(0x57, priv->mmio + RK3568_PHYREG14);
+		writel(0x40, priv->mmio + RK3568_PHYREG15);
+
+		writel(RK3568_PHYREG16_SSC_CNT_VALUE, priv->mmio + RK3568_PHYREG16);
+
+		val = FIELD_PREP(RK3568_PHYREG33_PLL_KVCO_MASK,
+				 RK3576_PHYREG33_PLL_KVCO_VALUE);
+		writel(val, priv->mmio + RK3568_PHYREG33);
+	}
+}
+
 static int rk3528_combphy_cfg(struct rockchip_combphy_priv *priv)
 {
 	const struct rockchip_combphy_grfcfg *cfg = priv->cfg->grfcfg;
@@ -600,21 +669,12 @@ static int rk3562_combphy_cfg(struct rockchip_combphy_priv *priv)
 
 	switch (priv->type) {
 	case PHY_TYPE_PCIE:
-		/* Set SSC downward spread spectrum */
-		val = RK3568_PHYREG32_SSC_DOWNWARD << RK3568_PHYREG32_SSC_DIR_SHIFT;
-		rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val, RK3568_PHYREG32);
-
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con0_for_pcie, true);
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con1_for_pcie, true);
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con2_for_pcie, true);
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con3_for_pcie, true);
 		break;
 	case PHY_TYPE_USB3:
-		/* Set SSC downward spread spectrum */
-		val = RK3568_PHYREG32_SSC_DOWNWARD << RK3568_PHYREG32_SSC_DIR_SHIFT;
-		rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val,
-					 RK3568_PHYREG32);
-
 		/* Enable adaptive CTLE for USB3.0 Rx */
 		rockchip_combphy_updatel(priv, RK3568_PHYREG15_CTLE_EN,
 					 RK3568_PHYREG15_CTLE_EN, RK3568_PHYREG15);
@@ -706,11 +766,7 @@ static int rk3562_combphy_cfg(struct rockchip_combphy_priv *priv)
 		}
 	}
 
-	if (priv->enable_ssc) {
-		val = readl(priv->mmio + RK3568_PHYREG8);
-		val |= RK3568_PHYREG8_SSC_EN;
-		writel(val, priv->mmio + RK3568_PHYREG8);
-	}
+	rk_combphy_common_cfg_ssc(priv, rate);
 
 	return 0;
 }
@@ -755,11 +811,6 @@ static int rk3568_combphy_cfg(struct rockchip_combphy_priv *priv)
 
 	switch (priv->type) {
 	case PHY_TYPE_PCIE:
-		/* Set SSC downward spread spectrum. */
-		val = RK3568_PHYREG32_SSC_DOWNWARD << RK3568_PHYREG32_SSC_DIR_SHIFT;
-
-		rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val, RK3568_PHYREG32);
-
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con0_for_pcie, true);
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con1_for_pcie, true);
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con2_for_pcie, true);
@@ -767,10 +818,6 @@ static int rk3568_combphy_cfg(struct rockchip_combphy_priv *priv)
 		break;
 
 	case PHY_TYPE_USB3:
-		/* Set SSC downward spread spectrum. */
-		val = RK3568_PHYREG32_SSC_DOWNWARD << RK3568_PHYREG32_SSC_DIR_SHIFT,
-		rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val, RK3568_PHYREG32);
-
 		/* Enable adaptive CTLE for USB3.0 Rx. */
 		val = readl(priv->mmio + RK3568_PHYREG15);
 		val |= RK3568_PHYREG15_CTLE_EN;
@@ -880,13 +927,6 @@ static int rk3568_combphy_cfg(struct rockchip_combphy_priv *priv)
 
 			writel(RK3568_PHYREG18_PLL_LOOP, priv->mmio + RK3568_PHYREG18);
 			writel(RK3568_PHYREG11_SU_TRIM_0_7, priv->mmio + RK3568_PHYREG11);
-		} else if (priv->type == PHY_TYPE_SATA) {
-			/* downward spread spectrum +500ppm */
-			val = RK3568_PHYREG32_SSC_DOWNWARD << RK3568_PHYREG32_SSC_DIR_SHIFT;
-			val |= RK3568_PHYREG32_SSC_OFFSET_500PPM <<
-			       RK3568_PHYREG32_SSC_OFFSET_SHIFT;
-			rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val,
-						 RK3568_PHYREG32);
 		}
 		break;
 
@@ -909,11 +949,7 @@ static int rk3568_combphy_cfg(struct rockchip_combphy_priv *priv)
 		}
 	}
 
-	if (priv->enable_ssc) {
-		val = readl(priv->mmio + RK3568_PHYREG8);
-		val |= RK3568_PHYREG8_SSC_EN;
-		writel(val, priv->mmio + RK3568_PHYREG8);
-	}
+	rk_combphy_common_cfg_ssc(priv, rate);
 
 	return 0;
 }
@@ -972,10 +1008,6 @@ static int rk3576_combphy_cfg(struct rockchip_combphy_priv *priv)
 
 	switch (priv->type) {
 	case PHY_TYPE_PCIE:
-		/* Set SSC downward spread spectrum */
-		val = FIELD_PREP(RK3568_PHYREG32_SSC_MASK, RK3568_PHYREG32_SSC_DOWNWARD);
-		rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val, RK3568_PHYREG32);
-
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con0_for_pcie, true);
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con1_for_pcie, true);
 		rockchip_combphy_param_write(priv->phy_grf, &cfg->con2_for_pcie, true);
@@ -983,10 +1015,6 @@ static int rk3576_combphy_cfg(struct rockchip_combphy_priv *priv)
 		break;
 
 	case PHY_TYPE_USB3:
-		/* Set SSC downward spread spectrum */
-		val = FIELD_PREP(RK3568_PHYREG32_SSC_MASK, RK3568_PHYREG32_SSC_DOWNWARD);
-		rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val, RK3568_PHYREG32);
-
 		/* Enable adaptive CTLE for USB3.0 Rx */
 		val = readl(priv->mmio + RK3568_PHYREG15);
 		val |= RK3568_PHYREG15_CTLE_EN;
@@ -1110,14 +1138,6 @@ static int rk3576_combphy_cfg(struct rockchip_combphy_priv *priv)
 			writel(0x88, priv->mmio + RK3568_PHYREG13);
 			writel(0x56, priv->mmio + RK3568_PHYREG14);
 		} else if (priv->type == PHY_TYPE_SATA) {
-			/* downward spread spectrum +500ppm */
-			val = FIELD_PREP(RK3568_PHYREG32_SSC_DIR_MASK,
-					 RK3568_PHYREG32_SSC_DOWNWARD);
-			val |= FIELD_PREP(RK3568_PHYREG32_SSC_OFFSET_MASK,
-					  RK3568_PHYREG32_SSC_OFFSET_500PPM);
-			rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val,
-						 RK3568_PHYREG32);
-
 			/* ssc ppm adjust to 3500ppm */
 			rockchip_combphy_updatel(priv, RK3576_PHYREG10_SSC_PCM_MASK,
 						 RK3576_PHYREG10_SSC_PCM_3500PPM,
@@ -1156,39 +1176,7 @@ static int rk3576_combphy_cfg(struct rockchip_combphy_priv *priv)
 		}
 	}
 
-	if (priv->enable_ssc) {
-		val = readl(priv->mmio + RK3568_PHYREG8);
-		val |= RK3568_PHYREG8_SSC_EN;
-		writel(val, priv->mmio + RK3568_PHYREG8);
-
-		if (priv->type == PHY_TYPE_PCIE && rate == REF_CLOCK_24MHz) {
-			/* Set PLL loop divider */
-			writel(0x00, priv->mmio + RK3576_PHYREG17);
-			writel(RK3568_PHYREG18_PLL_LOOP, priv->mmio + RK3568_PHYREG18);
-
-			/* Set up rx_pck invert and rx msb to disable */
-			writel(0x00, priv->mmio + RK3588_PHYREG27);
-
-			/*
-			 * Set up SU adjust signal:
-			 * su_trim[7:0],   PLL KVCO adjust bits[2:0] to min
-			 * su_trim[15:8],  PLL LPF R1 adujst bits[9:7]=3'b101
-			 * su_trim[23:16], CKRCV adjust
-			 * su_trim[31:24], CKDRV adjust
-			 */
-			writel(0x90, priv->mmio + RK3568_PHYREG11);
-			writel(0x02, priv->mmio + RK3568_PHYREG12);
-			writel(0x08, priv->mmio + RK3568_PHYREG13);
-			writel(0x57, priv->mmio + RK3568_PHYREG14);
-			writel(0x40, priv->mmio + RK3568_PHYREG15);
-
-			writel(RK3568_PHYREG16_SSC_CNT_VALUE, priv->mmio + RK3568_PHYREG16);
-
-			val = FIELD_PREP(RK3568_PHYREG33_PLL_KVCO_MASK,
-					 RK3576_PHYREG33_PLL_KVCO_VALUE);
-			writel(val, priv->mmio + RK3568_PHYREG33);
-		}
-	}
+	rk_combphy_common_cfg_ssc(priv, rate);
 
 	return 0;
 }
@@ -1255,10 +1243,6 @@ static int rk3588_combphy_cfg(struct rockchip_combphy_priv *priv)
 		}
 		break;
 	case PHY_TYPE_USB3:
-		/* Set SSC downward spread spectrum */
-		val = RK3568_PHYREG32_SSC_DOWNWARD << RK3568_PHYREG32_SSC_DIR_SHIFT;
-		rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val, RK3568_PHYREG32);
-
 		/* Enable adaptive CTLE for USB3.0 Rx. */
 		val = readl(priv->mmio + RK3568_PHYREG15);
 		val |= RK3568_PHYREG15_CTLE_EN;
@@ -1343,13 +1327,6 @@ static int rk3588_combphy_cfg(struct rockchip_combphy_priv *priv)
 
 			/* Set up su_trim:  */
 			writel(RK3568_PHYREG11_SU_TRIM_0_7, priv->mmio + RK3568_PHYREG11);
-		} else if (priv->type == PHY_TYPE_SATA) {
-			/* downward spread spectrum +500ppm */
-			val = RK3568_PHYREG32_SSC_DOWNWARD << RK3568_PHYREG32_SSC_DIR_SHIFT;
-			val |= RK3568_PHYREG32_SSC_OFFSET_500PPM <<
-			       RK3568_PHYREG32_SSC_OFFSET_SHIFT;
-			rockchip_combphy_updatel(priv, RK3568_PHYREG32_SSC_MASK, val,
-						 RK3568_PHYREG32);
 		}
 		break;
 	default:
@@ -1371,11 +1348,7 @@ static int rk3588_combphy_cfg(struct rockchip_combphy_priv *priv)
 		}
 	}
 
-	if (priv->enable_ssc) {
-		val = readl(priv->mmio + RK3568_PHYREG8);
-		val |= RK3568_PHYREG8_SSC_EN;
-		writel(val, priv->mmio + RK3568_PHYREG8);
-	}
+	rk_combphy_common_cfg_ssc(priv, rate);
 
 	return 0;
 }
diff --git a/drivers/phy/samsung/phy-exynos5-usbdrd.c b/drivers/phy/samsung/phy-exynos5-usbdrd.c
index 5a181cb4597e9..8711a3b62c8e1 100644
--- a/drivers/phy/samsung/phy-exynos5-usbdrd.c
+++ b/drivers/phy/samsung/phy-exynos5-usbdrd.c
@@ -1958,13 +1958,14 @@ const struct exynos5_usbdrd_phy_tuning exynos7870_tunes_utmi_postinit[] = {
 			      PHYPARAM0_TXPREEMPAMPTUNE | PHYPARAM0_TXHSXVTUNE |
 			      PHYPARAM0_TXFSLSTUNE | PHYPARAM0_SQRXTUNE |
 			      PHYPARAM0_OTGTUNE | PHYPARAM0_COMPDISTUNE),
-			     (FIELD_PREP_CONST(PHYPARAM0_TXVREFTUNE, 14) |
+			     (FIELD_PREP_CONST(PHYPARAM0_TXVREFTUNE, 3) |
 			      FIELD_PREP_CONST(PHYPARAM0_TXRISETUNE, 1) |
-			      FIELD_PREP_CONST(PHYPARAM0_TXRESTUNE, 3) |
+			      FIELD_PREP_CONST(PHYPARAM0_TXRESTUNE, 2) |
+			      FIELD_PREP_CONST(PHYPARAM0_TXPREEMPPULSETUNE, 0) |
 			      FIELD_PREP_CONST(PHYPARAM0_TXPREEMPAMPTUNE, 0) |
 			      FIELD_PREP_CONST(PHYPARAM0_TXHSXVTUNE, 0) |
 			      FIELD_PREP_CONST(PHYPARAM0_TXFSLSTUNE, 3) |
-			      FIELD_PREP_CONST(PHYPARAM0_SQRXTUNE, 6) |
+			      FIELD_PREP_CONST(PHYPARAM0_SQRXTUNE, 5) |
 			      FIELD_PREP_CONST(PHYPARAM0_OTGTUNE, 2) |
 			      FIELD_PREP_CONST(PHYPARAM0_COMPDISTUNE, 3))),
 	PHY_TUNING_ENTRY_LAST
diff --git a/drivers/phy/spacemit/phy-k1-usb2.c b/drivers/phy/spacemit/phy-k1-usb2.c
index 9215d0b223b2d..8df12eeb20b18 100644
--- a/drivers/phy/spacemit/phy-k1-usb2.c
+++ b/drivers/phy/spacemit/phy-k1-usb2.c
@@ -51,6 +51,9 @@
 #define PHY_K1_HS_HOST_DISC		0x40
 #define  PHY_K1_HS_HOST_DISC_CLR		BIT(0)
 
+#define PHY_K3_HS_HOST_DISC		0x20
+#define  PHY_K3_HS_HOST_DISC_CLR		BIT(8)
+
 #define PHY_PLL_DIV_CFG			0x98
 #define  PHY_FDIV_FRACT_8_15		GENMASK(7, 0)
 #define  PHY_FDIV_FRACT_16_19		GENMASK(11, 8)
@@ -97,7 +100,6 @@ static int spacemit_usb2phy_init(struct phy *phy)
 	ret = clk_enable(sphy->clk);
 	if (ret) {
 		dev_err(&phy->dev, "failed to enable clock\n");
-		clk_disable(sphy->clk);
 		return ret;
 	}
 
@@ -145,7 +147,7 @@ static int spacemit_usb2phy_exit(struct phy *phy)
 	return 0;
 }
 
-static int spacemit_usb2phy_disconnect(struct phy *phy, int port)
+static int spacemit_k1_usb2phy_disconnect(struct phy *phy, int port)
 {
 	struct spacemit_usb2phy *sphy = phy_get_drvdata(phy);
 
@@ -155,10 +157,27 @@ static int spacemit_usb2phy_disconnect(struct phy *phy, int port)
 	return 0;
 }
 
-static const struct phy_ops spacemit_usb2phy_ops = {
+static int spacemit_k3_usb2phy_disconnect(struct phy *phy, int port)
+{
+	struct spacemit_usb2phy *sphy = phy_get_drvdata(phy);
+
+	regmap_update_bits(sphy->regmap_base, PHY_K3_HS_HOST_DISC,
+			   PHY_K3_HS_HOST_DISC_CLR, PHY_K3_HS_HOST_DISC_CLR);
+
+	return 0;
+}
+
+static const struct phy_ops spacemit_k1_usb2phy_ops = {
 	.init = spacemit_usb2phy_init,
 	.exit = spacemit_usb2phy_exit,
-	.disconnect = spacemit_usb2phy_disconnect,
+	.disconnect = spacemit_k1_usb2phy_disconnect,
+	.owner = THIS_MODULE,
+};
+
+static const struct phy_ops spacemit_k3_usb2phy_ops = {
+	.init = spacemit_usb2phy_init,
+	.exit = spacemit_usb2phy_exit,
+	.disconnect = spacemit_k3_usb2phy_disconnect,
 	.owner = THIS_MODULE,
 };
 
@@ -167,12 +186,15 @@ static int spacemit_usb2phy_probe(struct platform_device *pdev)
 	struct phy_provider *phy_provider;
 	struct device *dev = &pdev->dev;
 	struct spacemit_usb2phy *sphy;
+	const struct phy_ops *ops;
 	void __iomem *base;
 
 	sphy = devm_kzalloc(dev, sizeof(*sphy), GFP_KERNEL);
 	if (!sphy)
 		return -ENOMEM;
 
+	ops = device_get_match_data(dev);
+
 	sphy->clk = devm_clk_get_prepared(&pdev->dev, NULL);
 	if (IS_ERR(sphy->clk))
 		return dev_err_probe(dev, PTR_ERR(sphy->clk), "Failed to get clock\n");
@@ -185,7 +207,7 @@ static int spacemit_usb2phy_probe(struct platform_device *pdev)
 	if (IS_ERR(sphy->regmap_base))
 		return dev_err_probe(dev, PTR_ERR(sphy->regmap_base), "Failed to init regmap\n");
 
-	sphy->phy = devm_phy_create(dev, NULL, &spacemit_usb2phy_ops);
+	sphy->phy = devm_phy_create(dev, NULL, ops);
 	if (IS_ERR(sphy->phy))
 		return dev_err_probe(dev, PTR_ERR(sphy->phy), "Failed to create phy\n");
 
@@ -196,7 +218,8 @@ static int spacemit_usb2phy_probe(struct platform_device *pdev)
 }
 
 static const struct of_device_id spacemit_usb2phy_dt_match[] = {
-	{ .compatible = "spacemit,k1-usb2-phy", },
+	{ .compatible = "spacemit,k1-usb2-phy", .data = &spacemit_k1_usb2phy_ops },
+	{ .compatible = "spacemit,k3-usb2-phy", .data = &spacemit_k3_usb2phy_ops },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, spacemit_usb2phy_dt_match);
diff --git a/drivers/phy/st/Kconfig b/drivers/phy/st/Kconfig
index 304614b6dabfc..49206185e5633 100644
--- a/drivers/phy/st/Kconfig
+++ b/drivers/phy/st/Kconfig
@@ -4,7 +4,7 @@
 #
 config PHY_MIPHY28LP
 	tristate "STMicroelectronics MIPHY28LP PHY driver for STiH407"
-	depends on ARCH_STI
+	depends on ARCH_STI || COMPILE_TEST
 	select GENERIC_PHY
 	help
 	  Enable this to support the miphy transceiver (for SATA/PCIE/USB3)
diff --git a/drivers/phy/st/phy-miphy28lp.c b/drivers/phy/st/phy-miphy28lp.c
index 43cef89af55ec..c576fc5569fe9 100644
--- a/drivers/phy/st/phy-miphy28lp.c
+++ b/drivers/phy/st/phy-miphy28lp.c
@@ -224,8 +224,8 @@ struct miphy28lp_dev {
 	struct device *dev;
 	struct regmap *regmap;
 	struct mutex miphy_mutex;
-	struct miphy28lp_phy **phys;
 	int nphys;
+	struct miphy28lp_phy *phys[] __counted_by(nphys);
 };
 
 enum miphy_sata_gen { SATA_GEN1, SATA_GEN2, SATA_GEN3 };
@@ -1168,16 +1168,14 @@ static int miphy28lp_probe(struct platform_device *pdev)
 	struct phy_provider *provider;
 	struct phy *phy;
 	int ret, port = 0;
+	size_t nphys;
 
-	miphy_dev = devm_kzalloc(&pdev->dev, sizeof(*miphy_dev), GFP_KERNEL);
+	nphys = of_get_child_count(np);
+	miphy_dev = devm_kzalloc(&pdev->dev, struct_size(miphy_dev, phys, nphys), GFP_KERNEL);
 	if (!miphy_dev)
 		return -ENOMEM;
 
-	miphy_dev->nphys = of_get_child_count(np);
-	miphy_dev->phys = devm_kcalloc(&pdev->dev, miphy_dev->nphys,
-				       sizeof(*miphy_dev->phys), GFP_KERNEL);
-	if (!miphy_dev->phys)
-		return -ENOMEM;
+	miphy_dev->nphys = nphys;
 
 	miphy_dev->regmap = syscon_regmap_lookup_by_phandle(np, "st,syscfg");
 	if (IS_ERR(miphy_dev->regmap)) {
diff --git a/drivers/phy/tegra/Kconfig b/drivers/phy/tegra/Kconfig
index 342fb736da4b1..f0734415fc94c 100644
--- a/drivers/phy/tegra/Kconfig
+++ b/drivers/phy/tegra/Kconfig
@@ -3,7 +3,7 @@ config PHY_TEGRA_XUSB
 	tristate "NVIDIA Tegra XUSB pad controller driver"
 	depends on ARCH_TEGRA && USB_SUPPORT
 	select USB_COMMON
-	select USB_CONN_GPIO
+	select USB_CONN_GPIO if GPIOLIB
 	select USB_PHY
 	help
 	  Choose this option if you have an NVIDIA Tegra SoC.
diff --git a/drivers/phy/tegra/xusb-tegra186.c b/drivers/phy/tegra/xusb-tegra186.c
index 1ddf112659744..60156aea27071 100644
--- a/drivers/phy/tegra/xusb-tegra186.c
+++ b/drivers/phy/tegra/xusb-tegra186.c
@@ -20,8 +20,8 @@
 /* FUSE USB_CALIB registers */
 #define HS_CURR_LEVEL_PADX_SHIFT(x)	((x) ? (11 + (x - 1) * 6) : 0)
 #define HS_CURR_LEVEL_PAD_MASK		0x3f
-#define HS_TERM_RANGE_ADJ_SHIFT		7
-#define HS_TERM_RANGE_ADJ_MASK		0xf
+#define HS_TERM_RANGE_ADJ_PADX_SHIFT(x)	((x) ? (5 + (x - 1) * 4) : 7)
+#define HS_TERM_RANGE_ADJ_PAD_MASK	0xf
 #define HS_SQUELCH_SHIFT		29
 #define HS_SQUELCH_MASK			0x7
 
@@ -253,7 +253,7 @@
 struct tegra_xusb_fuse_calibration {
 	u32 *hs_curr_level;
 	u32 hs_squelch;
-	u32 hs_term_range_adj;
+	u32 *hs_term_range_adj;
 	u32 rpd_ctrl;
 };
 
@@ -930,7 +930,7 @@ static int tegra186_utmi_phy_power_on(struct phy *phy)
 
 	value = padctl_readl(padctl, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
 	value &= ~TERM_RANGE_ADJ(~0);
-	value |= TERM_RANGE_ADJ(priv->calib.hs_term_range_adj);
+	value |= TERM_RANGE_ADJ(priv->calib.hs_term_range_adj[index]);
 	value &= ~RPD_CTRL(~0);
 	value |= RPD_CTRL(priv->calib.rpd_ctrl);
 	padctl_writel(padctl, value, XUSB_PADCTL_USB2_OTG_PADX_CTL1(index));
@@ -1464,17 +1464,23 @@ static const char * const tegra186_usb3_functions[] = {
 static int
 tegra186_xusb_read_fuse_calibration(struct tegra186_xusb_padctl *padctl)
 {
+	const struct tegra_xusb_padctl_soc *soc = padctl->base.soc;
 	struct device *dev = padctl->base.dev;
 	unsigned int i, count;
 	u32 value, *level;
+	u32 *hs_term_range_adj;
 	int err;
 
-	count = padctl->base.soc->ports.usb2.count;
+	count = soc->ports.usb2.count;
 
 	level = devm_kcalloc(dev, count, sizeof(u32), GFP_KERNEL);
 	if (!level)
 		return -ENOMEM;
 
+	hs_term_range_adj = devm_kcalloc(dev, count, sizeof(u32), GFP_KERNEL);
+	if (!hs_term_range_adj)
+		return -ENOMEM;
+
 	err = tegra_fuse_readl(TEGRA_FUSE_SKU_CALIB_0, &value);
 	if (err)
 		return dev_err_probe(dev, err,
@@ -1490,8 +1496,8 @@ tegra186_xusb_read_fuse_calibration(struct tegra186_xusb_padctl *padctl)
 
 	padctl->calib.hs_squelch = (value >> HS_SQUELCH_SHIFT) &
 					HS_SQUELCH_MASK;
-	padctl->calib.hs_term_range_adj = (value >> HS_TERM_RANGE_ADJ_SHIFT) &
-						HS_TERM_RANGE_ADJ_MASK;
+	hs_term_range_adj[0] = (value >> HS_TERM_RANGE_ADJ_PADX_SHIFT(0)) &
+				HS_TERM_RANGE_ADJ_PAD_MASK;
 
 	err = tegra_fuse_readl(TEGRA_FUSE_USB_CALIB_EXT_0, &value);
 	if (err) {
@@ -1503,6 +1509,17 @@ tegra186_xusb_read_fuse_calibration(struct tegra186_xusb_padctl *padctl)
 
 	padctl->calib.rpd_ctrl = (value >> RPD_CTRL_SHIFT) & RPD_CTRL_MASK;
 
+	for (i = 1; i < count; i++) {
+		if (soc->has_per_pad_term)
+			hs_term_range_adj[i] =
+				(value >> HS_TERM_RANGE_ADJ_PADX_SHIFT(i)) &
+				HS_TERM_RANGE_ADJ_PAD_MASK;
+		else
+			hs_term_range_adj[i] = hs_term_range_adj[0];
+	}
+
+	padctl->calib.hs_term_range_adj = hs_term_range_adj;
+
 	return 0;
 }
 
@@ -1708,6 +1725,7 @@ const struct tegra_xusb_padctl_soc tegra194_xusb_padctl_soc = {
 	.num_supplies = ARRAY_SIZE(tegra194_xusb_padctl_supply_names),
 	.supports_gen2 = true,
 	.poll_trk_completed = true,
+	.has_per_pad_term = true,
 };
 EXPORT_SYMBOL_GPL(tegra194_xusb_padctl_soc);
 
@@ -1732,6 +1750,7 @@ const struct tegra_xusb_padctl_soc tegra234_xusb_padctl_soc = {
 	.trk_hw_mode = false,
 	.trk_update_on_idle = true,
 	.supports_lp_cfg_en = true,
+	.has_per_pad_term = true,
 };
 EXPORT_SYMBOL_GPL(tegra234_xusb_padctl_soc);
 #endif
diff --git a/drivers/phy/tegra/xusb.h b/drivers/phy/tegra/xusb.h
index cd277d0ed9e18..77609e54de66a 100644
--- a/drivers/phy/tegra/xusb.h
+++ b/drivers/phy/tegra/xusb.h
@@ -435,6 +435,7 @@ struct tegra_xusb_padctl_soc {
 	bool trk_hw_mode;
 	bool trk_update_on_idle;
 	bool supports_lp_cfg_en;
+	bool has_per_pad_term;
 };
 
 struct tegra_xusb_padctl {
diff --git a/drivers/phy/ti/Kconfig b/drivers/phy/ti/Kconfig
index b40f280191314..b52c7eabe24cb 100644
--- a/drivers/phy/ti/Kconfig
+++ b/drivers/phy/ti/Kconfig
@@ -2,24 +2,29 @@
 #
 # Phy drivers for TI platforms
 #
-config PHY_DA8XX_USB
-	tristate "TI DA8xx USB PHY Driver"
-	depends on ARCH_DAVINCI_DA8XX || COMPILE_TEST
-	select GENERIC_PHY
-	select MFD_SYSCON
+config OMAP_CONTROL_PHY
+	tristate "OMAP CONTROL PHY Driver"
+	depends on ARCH_OMAP2PLUS || COMPILE_TEST
 	help
-	  Enable this to support the USB PHY on DA8xx SoCs.
-
-	  This driver controls both the USB 1.1 PHY and the USB 2.0 PHY.
+	  Enable this to add support for the PHY part present in the control
+	  module. This driver has API to power on the USB2 PHY and to write to
+	  the mailbox. The mailbox is present only in omap4 and the register to
+	  power on the USB2 PHY is present in OMAP4 and OMAP5. OMAP5 has an
+	  additional register to power on USB3 PHY/SATA PHY/PCIE PHY
+	  (PIPE3 PHY).
 
-config PHY_DM816X_USB
-	tristate "TI dm816x USB PHY driver"
-	depends on ARCH_OMAP2PLUS || COMPILE_TEST
+config OMAP_USB2
+	tristate "OMAP USB2 PHY Driver"
+	depends on ARCH_OMAP2PLUS || ARCH_K3 || COMPILE_TEST
 	depends on USB_SUPPORT
 	select GENERIC_PHY
 	select USB_PHY
+	select OMAP_CONTROL_PHY if ARCH_OMAP2PLUS || COMPILE_TEST
 	help
-	  Enable this for dm816x USB to work.
+	  Enable this to support the transceiver that is part of SOC. This
+	  driver takes care of all the PHY functionality apart from comparator.
+	  The USB OTG controller communicates with the comparator using this
+	  driver.
 
 config PHY_AM654_SERDES
 	tristate "TI AM654 SERDES support"
@@ -33,6 +38,25 @@ config PHY_AM654_SERDES
 	  This option enables support for TI AM654 SerDes PHY used for
 	  PCIe.
 
+config PHY_DA8XX_USB
+	tristate "TI DA8xx USB PHY Driver"
+	depends on ARCH_DAVINCI_DA8XX || COMPILE_TEST
+	select GENERIC_PHY
+	select MFD_SYSCON
+	help
+	  Enable this to support the USB PHY on DA8xx SoCs.
+
+	  This driver controls both the USB 1.1 PHY and the USB 2.0 PHY.
+
+config PHY_DM816X_USB
+	tristate "TI dm816x USB PHY driver"
+	depends on ARCH_OMAP2PLUS || COMPILE_TEST
+	depends on USB_SUPPORT
+	select GENERIC_PHY
+	select USB_PHY
+	help
+	  Enable this for dm816x USB to work.
+
 config PHY_J721E_WIZ
 	tristate "TI J721E WIZ (SERDES Wrapper) support"
 	depends on OF && (ARCH_K3 || COMPILE_TEST)
@@ -49,29 +73,30 @@ config PHY_J721E_WIZ
 	  three clock selects (pll0, pll1, dig) and resets for each of the
 	  lanes.
 
-config OMAP_CONTROL_PHY
-	tristate "OMAP CONTROL PHY Driver"
-	depends on ARCH_OMAP2PLUS || COMPILE_TEST
+config PHY_TI_DS125DF111
+	tristate "TI DS125DF111 2-Channel Retimer Driver"
+	depends on OF && I2C
+	select GENERIC_PHY
 	help
-	  Enable this to add support for the PHY part present in the control
-	  module. This driver has API to power on the USB2 PHY and to write to
-	  the mailbox. The mailbox is present only in omap4 and the register to
-	  power on the USB2 PHY is present in OMAP4 and OMAP5. OMAP5 has an
-	  additional register to power on USB3 PHY/SATA PHY/PCIE PHY
-	  (PIPE3 PHY).
+	  Enable this to add support for configuration and runtime management
+	  of the TI DS125DF111 Multi-Protocol 2-Channel Retimer.
+	  The retimer is modeled as a Generic PHY and supports both 10G and 1G
+	  link speeds.
 
-config OMAP_USB2
-	tristate "OMAP USB2 PHY Driver"
-	depends on ARCH_OMAP2PLUS || ARCH_K3 || COMPILE_TEST
-	depends on USB_SUPPORT
+config PHY_TI_GMII_SEL
+	tristate
 	select GENERIC_PHY
-	select USB_PHY
-	select OMAP_CONTROL_PHY if ARCH_OMAP2PLUS || COMPILE_TEST
+	select REGMAP
 	help
-	  Enable this to support the transceiver that is part of SOC. This
-	  driver takes care of all the PHY functionality apart from comparator.
-	  The USB OTG controller communicates with the comparator using this
-	  driver.
+	  This driver supports configuring of the TI CPSW Port mode depending on
+	  the Ethernet PHY connected to the CPSW Port.
+
+config PHY_TUSB1210
+	tristate "TI TUSB1210 ULPI PHY module"
+	depends on USB_ULPI_BUS
+	select GENERIC_PHY
+	help
+	  Support for TI TUSB1210 USB ULPI PHY.
 
 config TI_PIPE3
 	tristate "TI PIPE3 PHY Driver"
@@ -84,13 +109,6 @@ config TI_PIPE3
 	  This driver interacts with the "OMAP Control PHY Driver" to power
 	  on/off the PHY.
 
-config PHY_TUSB1210
-	tristate "TI TUSB1210 ULPI PHY module"
-	depends on USB_ULPI_BUS
-	select GENERIC_PHY
-	help
-	  Support for TI TUSB1210 USB ULPI PHY.
-
 config TWL4030_USB
 	tristate "TWL4030 USB Transceiver Driver"
 	depends on TWL4030_CORE && REGULATOR_TWL4030 && USB_MUSB_OMAP2PLUS
@@ -103,11 +121,3 @@ config TWL4030_USB
 	  family chips (including the TWL5030 and TPS659x0 devices).
 	  This transceiver supports high and full speed devices plus,
 	  in host mode, low speed.
-
-config PHY_TI_GMII_SEL
-	tristate
-	select GENERIC_PHY
-	select REGMAP
-	help
-	  This driver supports configuring of the TI CPSW Port mode depending on
-	  the Ethernet PHY connected to the CPSW Port.
diff --git a/drivers/phy/ti/Makefile b/drivers/phy/ti/Makefile
index dcba2571c9bd8..a002ef8764a20 100644
--- a/drivers/phy/ti/Makefile
+++ b/drivers/phy/ti/Makefile
@@ -1,11 +1,12 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_PHY_DA8XX_USB)		+= phy-da8xx-usb.o
-obj-$(CONFIG_PHY_DM816X_USB)		+= phy-dm816x-usb.o
 obj-$(CONFIG_OMAP_CONTROL_PHY)		+= phy-omap-control.o
 obj-$(CONFIG_OMAP_USB2)			+= phy-omap-usb2.o
-obj-$(CONFIG_TI_PIPE3)			+= phy-ti-pipe3.o
-obj-$(CONFIG_PHY_TUSB1210)		+= phy-tusb1210.o
-obj-$(CONFIG_TWL4030_USB)		+= phy-twl4030-usb.o
 obj-$(CONFIG_PHY_AM654_SERDES)		+= phy-am654-serdes.o
-obj-$(CONFIG_PHY_TI_GMII_SEL)		+= phy-gmii-sel.o
+obj-$(CONFIG_PHY_DA8XX_USB)		+= phy-da8xx-usb.o
+obj-$(CONFIG_PHY_DM816X_USB)		+= phy-dm816x-usb.o
 obj-$(CONFIG_PHY_J721E_WIZ)		+= phy-j721e-wiz.o
+obj-$(CONFIG_PHY_TI_DS125DF111)		+= phy-ds125df111.o
+obj-$(CONFIG_PHY_TI_GMII_SEL)		+= phy-gmii-sel.o
+obj-$(CONFIG_PHY_TUSB1210)		+= phy-tusb1210.o
+obj-$(CONFIG_TI_PIPE3)			+= phy-ti-pipe3.o
+obj-$(CONFIG_TWL4030_USB)		+= phy-twl4030-usb.o
diff --git a/drivers/phy/ti/phy-ds125df111.c b/drivers/phy/ti/phy-ds125df111.c
new file mode 100644
index 0000000000000..84ff96d7d589a
--- /dev/null
+++ b/drivers/phy/ti/phy-ds125df111.c
@@ -0,0 +1,294 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright 2026 NXP */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/i2c.h>
+#include <linux/iopoll.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/phy.h>
+#include <linux/phy/phy.h>
+#include <linux/slab.h>
+
+#define DS125DF111_NUM_CH			2
+#define DS125DF111_NUM_VCO_GROUP_REG		5
+
+#define DS125DF111_CH_SELECT			0xff
+#define DS125DF111_CH_SELECT_TARGET_MASK	GENMASK(3, 0)
+#define DS125DF111_CH_SELECT_EN			BIT(2)
+
+#define DS125DF111_CH_CTRL			0x00
+#define DS125DF111_CH_CTRL_RESET		BIT(2) /* self clearing */
+
+#define DS125DF111_CH_RST_SLEEP_US		10
+#define DS125DF111_CH_RST_TIMEOUT_US		10000
+
+#define DS125DF111_VCO_GROUP_BASE		0x60
+
+#define DS125DF111_RATIOS			0x2f
+#define DS125DF111_RATIOS_RATE_MASK		GENMASK(7, 6)
+#define DS125DF111_RATIOS_SUBRATE_MASK		GENMASK(5, 4)
+#define DS125DF111_RATIOS_MASK			GENMASK(7, 4)
+
+struct ds125df111_ch {
+	struct phy *phy;
+	struct ds125df111_priv *priv;
+	int idx;
+};
+
+struct ds125df111_priv {
+	struct ds125df111_ch ch[DS125DF111_NUM_CH];
+	struct i2c_client *client;
+	struct mutex mutex; /* protects access to shared registers */
+};
+
+enum ds125df111_mode {
+	FREQ_1G,
+	FREQ_10G,
+};
+
+static const struct ds125df111_config {
+	u8 vco_group[DS125DF111_NUM_VCO_GROUP_REG];
+	u8 rate;
+	u8 subrate;
+} ds125df111_cfg[] = {
+	[FREQ_1G] = {
+		/* VCO group #0 = 10GHz, VCO group #1 = 10GHz */
+		.vco_group = {0x00, 0xB2, 0x00, 0xB2, 0xCC},
+		/* By using the following combination of rate and subrate we
+		 * select divide ratios of 1, 2, 4, 8 on both groups
+		 */
+		.rate = 0x1,
+		.subrate = 0x2,
+	},
+
+	[FREQ_10G] = {
+		/* VCO group #0 = 10.3125GHz, VCO group #1 = 10.3125GHz */
+		.vco_group = {0x90, 0xB3, 0x90, 0xB3, 0xCD},
+		/* By using the following combination of rate and subrate we
+		 * select divide ratios of 1 on both groups
+		 */
+		.rate = 0x1,
+		.subrate = 0x3,
+	},
+};
+
+static int ds125df111_rmw(struct ds125df111_priv *priv, u8 reg, u8 clr, u8 set)
+{
+	struct i2c_client *i2c = priv->client;
+	int err;
+	u8 val;
+
+	err = i2c_smbus_read_byte_data(i2c, reg);
+	if (err < 0)
+		return err;
+
+	val = (u8)err;
+	val &= ~clr;
+	val |= set;
+
+	err = i2c_smbus_write_byte_data(i2c, reg, val);
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+static int ds125df111_configure(struct phy *phy,
+				const struct ds125df111_config *cfg)
+{
+	struct ds125df111_ch *ch = phy_get_drvdata(phy);
+	struct ds125df111_priv *priv = ch->priv;
+	struct i2c_client *i2c = priv->client;
+	struct device *dev = &phy->dev;
+	u8 ratios_val;
+	int err, i;
+	int val;
+
+	mutex_lock(&priv->mutex);
+
+	/* Make sure that any subsequent read/write operation will be directed
+	 * only to the registers of the selected channel
+	 */
+	err = ds125df111_rmw(priv, DS125DF111_CH_SELECT,
+			     DS125DF111_CH_SELECT_TARGET_MASK,
+			     DS125DF111_CH_SELECT_EN | ch->idx);
+	if (err < 0) {
+		dev_err(dev, "Unable to select channel: %pe\n", ERR_PTR(err));
+		goto out;
+	}
+
+	/* Reset channel registers and wait until the bit was cleared */
+	err = ds125df111_rmw(priv, DS125DF111_CH_CTRL, 0,
+			     DS125DF111_CH_CTRL_RESET);
+	if (err < 0) {
+		dev_err(dev, "Error resetting channel configuration: %pe\n",
+			ERR_PTR(err));
+		goto out;
+	}
+
+	err = read_poll_timeout(i2c_smbus_read_byte_data, val,
+				val < 0 || !(val & DS125DF111_CH_CTRL_RESET),
+				DS125DF111_CH_RST_SLEEP_US,
+				DS125DF111_CH_RST_TIMEOUT_US, false, i2c,
+				DS125DF111_CH_CTRL);
+	if (err) {
+		dev_err(dev, "Timed out waiting for channel reset: %pe\n",
+			ERR_PTR(err));
+		goto out;
+	}
+
+	if (val < 0) {
+		dev_err(dev, "Error reading reset status: %pe\n", ERR_PTR(val));
+		err = val;
+		goto out;
+	}
+
+	/* Program the VCO group frequencies */
+	for (i = 0; i < DS125DF111_NUM_VCO_GROUP_REG; i++) {
+		err = i2c_smbus_write_byte_data(i2c,
+						DS125DF111_VCO_GROUP_BASE + i,
+						cfg->vco_group[i]);
+		if (err < 0) {
+			dev_err(dev, "Error programming VCO group: %pe\n",
+				ERR_PTR(err));
+			goto out;
+		}
+	}
+
+	/* Set the divide ratios for the VCO groups */
+	ratios_val = FIELD_PREP(DS125DF111_RATIOS_RATE_MASK, cfg->rate) |
+		FIELD_PREP(DS125DF111_RATIOS_SUBRATE_MASK, cfg->subrate);
+	err = ds125df111_rmw(priv, DS125DF111_RATIOS, DS125DF111_RATIOS_MASK,
+			     ratios_val);
+	if (err < 0) {
+		dev_err(dev, "Error programming the divide ratios: %pe\n",
+			ERR_PTR(err));
+		goto out;
+	}
+
+out:
+	mutex_unlock(&priv->mutex);
+
+	return err;
+}
+
+static int ds125df111_set_mode(struct phy *phy, enum phy_mode mode, int submode)
+{
+	const struct ds125df111_config *cfg;
+
+	if (mode != PHY_MODE_ETHERNET)
+		return -EINVAL;
+
+	switch (submode) {
+	case PHY_INTERFACE_MODE_10GBASER:
+		cfg = &ds125df111_cfg[FREQ_10G];
+		break;
+	case PHY_INTERFACE_MODE_1000BASEX:
+	case PHY_INTERFACE_MODE_SGMII:
+		cfg = &ds125df111_cfg[FREQ_1G];
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return ds125df111_configure(phy, cfg);
+}
+
+static int ds125df111_validate(struct phy *phy, enum phy_mode mode, int submode,
+			       union phy_configure_opts *opts __always_unused)
+{
+	if (mode != PHY_MODE_ETHERNET)
+		return -EINVAL;
+
+	switch (submode) {
+	case PHY_INTERFACE_MODE_10GBASER:
+	case PHY_INTERFACE_MODE_1000BASEX:
+	case PHY_INTERFACE_MODE_SGMII:
+		return 0;
+	default:
+		return -EINVAL;
+	}
+}
+
+static const struct phy_ops ds125df111_ops = {
+	.validate	= ds125df111_validate,
+	.set_mode	= ds125df111_set_mode,
+	.owner		= THIS_MODULE,
+};
+
+static struct phy *ds125df111_xlate(struct device *dev,
+				    const struct of_phandle_args *args)
+{
+	struct ds125df111_priv *priv = dev_get_drvdata(dev);
+	u32 idx;
+
+	if (args->args_count != 1)
+		return ERR_PTR(-EINVAL);
+
+	idx = args->args[0];
+	if (idx >= DS125DF111_NUM_CH) {
+		dev_err(dev, "Maximum number of channels is %d\n",
+			DS125DF111_NUM_CH);
+		return ERR_PTR(-EINVAL);
+	}
+
+	return priv->ch[idx].phy;
+}
+
+static int ds125df111_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct phy_provider *provider;
+	struct ds125df111_priv *priv;
+	int i, err;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+	priv->client = client;
+	err = devm_mutex_init(dev, &priv->mutex);
+	if (err)
+		return err;
+
+	i2c_set_clientdata(client, priv);
+
+	for (i = 0; i < DS125DF111_NUM_CH; i++) {
+		struct ds125df111_ch *ch = &priv->ch[i];
+		struct phy *phy;
+
+		phy = devm_phy_create(dev, NULL, &ds125df111_ops);
+		if (IS_ERR(phy))
+			return PTR_ERR(phy);
+
+		ch->idx = i;
+		ch->priv = priv;
+		ch->phy = phy;
+
+		phy_set_drvdata(phy, ch);
+	}
+
+	provider = devm_of_phy_provider_register(dev, ds125df111_xlate);
+
+	return PTR_ERR_OR_ZERO(provider);
+}
+
+static const struct of_device_id ds125df111_dt_ids[] = {
+	{ .compatible = "ti,ds125df111", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, ds125df111_dt_ids);
+
+static struct i2c_driver ds125df111_driver = {
+	.driver = {
+		.name = "ds125df111",
+		.of_match_table = ds125df111_dt_ids,
+	},
+	.probe = ds125df111_probe,
+};
+module_i2c_driver(ds125df111_driver);
+
+MODULE_AUTHOR("Ioana Ciornei <ioana.ciornei@nxp.com>");
+MODULE_DESCRIPTION("TI DS125DF111 Retimer driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/phy/ti/phy-gmii-sel.c b/drivers/phy/ti/phy-gmii-sel.c
index 6213c2b6005a5..c2865a6b1d7fb 100644
--- a/drivers/phy/ti/phy-gmii-sel.c
+++ b/drivers/phy/ti/phy-gmii-sel.c
@@ -251,6 +251,15 @@ struct phy_gmii_sel_soc_data phy_gmii_sel_soc_am654 = {
 	.regfields = phy_gmii_sel_fields_am654,
 };
 
+static const
+struct phy_gmii_sel_soc_data phy_gmii_sel_soc_j722s = {
+	.use_of_data = true,
+	.features = BIT(PHY_GMII_SEL_RGMII_ID_MODE) |
+		    BIT(PHY_GMII_SEL_FIXED_TX_DELAY),
+	.regfields = phy_gmii_sel_fields_am654,
+	.extra_modes = BIT(PHY_INTERFACE_MODE_SGMII),
+};
+
 static const
 struct phy_gmii_sel_soc_data phy_gmii_sel_cpsw5g_soc_j7200 = {
 	.use_of_data = true,
@@ -307,6 +316,10 @@ static const struct of_device_id phy_gmii_sel_id_table[] = {
 		.compatible	= "ti,am654-phy-gmii-sel",
 		.data		= &phy_gmii_sel_soc_am654,
 	},
+	{
+		.compatible	= "ti,j722s-phy-gmii-sel",
+		.data		= &phy_gmii_sel_soc_j722s,
+	},
 	{
 		.compatible	= "ti,j7200-cpsw5g-phy-gmii-sel",
 		.data		= &phy_gmii_sel_cpsw5g_soc_j7200,
diff --git a/drivers/phy/ti/phy-j721e-wiz.c b/drivers/phy/ti/phy-j721e-wiz.c
index 6b584706b913a..7531a8a049123 100644
--- a/drivers/phy/ti/phy-j721e-wiz.c
+++ b/drivers/phy/ti/phy-j721e-wiz.c
@@ -331,6 +331,7 @@ enum wiz_type {
 	J721E_WIZ_16G,
 	J721E_WIZ_10G,	/* Also for J7200 SR1.0 */
 	AM64_WIZ_10G,
+	J722S_WIZ_10G,
 	J7200_WIZ_10G,  /* J7200 SR2.0 */
 	J784S4_WIZ_10G,
 	J721S2_WIZ_10G,
@@ -1020,6 +1021,7 @@ static void wiz_clock_cleanup(struct wiz *wiz, struct device_node *node)
 
 	switch (wiz->type) {
 	case AM64_WIZ_10G:
+	case J722S_WIZ_10G:
 	case J7200_WIZ_10G:
 	case J784S4_WIZ_10G:
 	case J721S2_WIZ_10G:
@@ -1089,6 +1091,7 @@ static void wiz_clock_init(struct wiz *wiz)
 
 	switch (wiz->type) {
 	case AM64_WIZ_10G:
+	case J722S_WIZ_10G:
 	case J7200_WIZ_10G:
 		switch (rate) {
 		case REF_CLK_100MHZ:
@@ -1158,6 +1161,7 @@ static int wiz_clock_probe(struct wiz *wiz, struct device_node *node)
 
 	switch (wiz->type) {
 	case AM64_WIZ_10G:
+	case J722S_WIZ_10G:
 	case J7200_WIZ_10G:
 	case J784S4_WIZ_10G:
 	case J721S2_WIZ_10G:
@@ -1246,6 +1250,14 @@ static int wiz_phy_fullrt_div(struct wiz *wiz, int lane)
 		if (wiz->lane_phy_type[lane] == PHY_TYPE_SGMII)
 			return regmap_field_write(wiz->p0_fullrt_div[lane], 0x2);
 		break;
+
+	case J722S_WIZ_10G:
+		if (wiz->lane_phy_type[lane] == PHY_TYPE_PCIE)
+			return regmap_field_write(wiz->p0_fullrt_div[lane], 0x1);
+		if (wiz->lane_phy_type[lane] == PHY_TYPE_SGMII)
+			return regmap_field_write(wiz->p0_fullrt_div[lane], 0x2);
+		break;
+
 	default:
 		return 0;
 	}
@@ -1350,6 +1362,15 @@ static struct wiz_data am64_10g_data = {
 	.clk_div_sel_num = WIZ_DIV_NUM_CLOCKS_10G,
 };
 
+static struct wiz_data j722s_10g_data = {
+	.type = J722S_WIZ_10G,
+	.pll0_refclk_mux_sel = &pll0_refclk_mux_sel,
+	.pll1_refclk_mux_sel = &pll1_refclk_mux_sel,
+	.refclk_dig_sel = &refclk_dig_sel_10g,
+	.clk_mux_sel = clk_mux_sel_10g,
+	.clk_div_sel_num = WIZ_DIV_NUM_CLOCKS_10G,
+};
+
 static struct wiz_data j7200_pg2_10g_data = {
 	.type = J7200_WIZ_10G,
 	.pll0_refclk_mux_sel = &sup_pll0_refclk_mux_sel,
@@ -1389,6 +1410,9 @@ static const struct of_device_id wiz_id_table[] = {
 	{
 		.compatible = "ti,am64-wiz-10g", .data = &am64_10g_data,
 	},
+	{
+		.compatible = "ti,j722s-wiz-10g", .data = &j722s_10g_data,
+	},
 	{
 		.compatible = "ti,j7200-wiz-10g", .data = &j7200_pg2_10g_data,
 	},
diff --git a/drivers/pinctrl/Kconfig b/drivers/pinctrl/Kconfig
index 03f2e3ee065f4..31d698fbaa01d 100644
--- a/drivers/pinctrl/Kconfig
+++ b/drivers/pinctrl/Kconfig
@@ -272,6 +272,15 @@ config PINCTRL_GEMINI
 	select GENERIC_PINCONF
 	select MFD_SYSCON
 
+config PINCTRL_GENERIC_MUX
+	tristate "Generic Pinctrl driver by using multiplexer"
+	depends on MULTIPLEXER
+	select PINMUX
+	select GENERIC_PINCTRL
+	help
+          Generic pinctrl driver by MULTIPLEXER framework to control on
+          board pin selection.
+
 config PINCTRL_INGENIC
 	bool "Pinctrl driver for the Ingenic JZ47xx SoCs"
 	default MACH_INGENIC
diff --git a/drivers/pinctrl/Makefile b/drivers/pinctrl/Makefile
index f7d5d5f76d0c8..1438a246b93f6 100644
--- a/drivers/pinctrl/Makefile
+++ b/drivers/pinctrl/Makefile
@@ -29,6 +29,7 @@ obj-$(CONFIG_PINCTRL_EQUILIBRIUM)   += pinctrl-equilibrium.o
 obj-$(CONFIG_PINCTRL_EP93XX)	+= pinctrl-ep93xx.o
 obj-$(CONFIG_PINCTRL_EYEQ5)	+= pinctrl-eyeq5.o
 obj-$(CONFIG_PINCTRL_GEMINI)	+= pinctrl-gemini.o
+obj-$(CONFIG_PINCTRL_GENERIC_MUX) += pinctrl-generic-mux.o
 obj-$(CONFIG_PINCTRL_INGENIC)	+= pinctrl-ingenic.o
 obj-$(CONFIG_PINCTRL_K210)	+= pinctrl-k210.o
 obj-$(CONFIG_PINCTRL_K230)	+= pinctrl-k230.o
@@ -66,7 +67,7 @@ obj-$(CONFIG_PINCTRL_ZYNQMP)	+= pinctrl-zynqmp.o
 obj-$(CONFIG_PINCTRL_ZYNQ)	+= pinctrl-zynq.o
 
 obj-y				+= actions/
-obj-$(CONFIG_ARCH_ASPEED)	+= aspeed/
+obj-$(CONFIG_PINCTRL_ASPEED)	+= aspeed/
 obj-y				+= bcm/
 obj-$(CONFIG_PINCTRL_BERLIN)	+= berlin/
 obj-y				+= cirrus/
@@ -82,7 +83,7 @@ obj-y				+= nuvoton/
 obj-y				+= nxp/
 obj-$(CONFIG_PINCTRL_PXA)	+= pxa/
 obj-y				+= qcom/
-obj-$(CONFIG_ARCH_REALTEK)      += realtek/
+obj-$(CONFIG_PINCTRL_RTD)	+= realtek/
 obj-$(CONFIG_PINCTRL_RENESAS)	+= renesas/
 obj-$(CONFIG_PINCTRL_SAMSUNG)	+= samsung/
 obj-y				+= sophgo/
@@ -93,8 +94,8 @@ obj-y				+= starfive/
 obj-$(CONFIG_PINCTRL_STM32)	+= stm32/
 obj-y				+= sunplus/
 obj-$(CONFIG_PINCTRL_SUNXI)	+= sunxi/
-obj-$(CONFIG_ARCH_TEGRA)	+= tegra/
+obj-y				+= tegra/
 obj-y				+= ti/
 obj-$(CONFIG_PINCTRL_UNIPHIER)	+= uniphier/
 obj-$(CONFIG_PINCTRL_VISCONTI)	+= visconti/
-obj-$(CONFIG_ARCH_VT8500)	+= vt8500/
+obj-$(CONFIG_PINCTRL_WMT)	+= vt8500/
diff --git a/drivers/pinctrl/aspeed/Kconfig b/drivers/pinctrl/aspeed/Kconfig
index 1a4e5b9ed471e..f9672cca891eb 100644
--- a/drivers/pinctrl/aspeed/Kconfig
+++ b/drivers/pinctrl/aspeed/Kconfig
@@ -31,3 +31,12 @@ config PINCTRL_ASPEED_G6
 	help
 	  Say Y here to enable pin controller support for Aspeed's 6th
 	  generation SoCs. GPIO is provided by a separate GPIO driver.
+
+config PINCTRL_ASPEED_G7_SOC0
+	bool "Aspeed G7 SoC pin control"
+	depends on (ARCH_ASPEED || COMPILE_TEST) && OF
+	select PINCTRL_ASPEED
+	help
+	  Say Y here to enable pin controller support for the SoC0 instance
+	  of Aspeed's 7th generation SoCs. GPIO is provided by a separate
+	  GPIO driver.
diff --git a/drivers/pinctrl/aspeed/Makefile b/drivers/pinctrl/aspeed/Makefile
index db2a7600ae2bd..0de524ca2c725 100644
--- a/drivers/pinctrl/aspeed/Makefile
+++ b/drivers/pinctrl/aspeed/Makefile
@@ -6,3 +6,4 @@ obj-$(CONFIG_PINCTRL_ASPEED)	+= pinctrl-aspeed.o pinmux-aspeed.o
 obj-$(CONFIG_PINCTRL_ASPEED_G4)	+= pinctrl-aspeed-g4.o
 obj-$(CONFIG_PINCTRL_ASPEED_G5)	+= pinctrl-aspeed-g5.o
 obj-$(CONFIG_PINCTRL_ASPEED_G6)	+= pinctrl-aspeed-g6.o
+obj-$(CONFIG_PINCTRL_ASPEED_G7_SOC0) += pinctrl-aspeed-g7-soc0.o
diff --git a/drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc0.c b/drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc0.c
new file mode 100644
index 0000000000000..94f216f55bdd0
--- /dev/null
+++ b/drivers/pinctrl/aspeed/pinctrl-aspeed-g7-soc0.c
@@ -0,0 +1,749 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/bitops.h>
+#include <linux/bits.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/pinctrl/pinconf.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+
+#include "pinctrl-aspeed.h"
+#include "pinmux-aspeed.h"
+#include "../pinctrl-utils.h"
+
+#define SCU200 0x200 /* System Reset Control #1  */
+
+#define SCU010 0x010 /* Hardware Strap Register */
+#define SCU400 0x400 /* Multi-function Pin Control #1  */
+#define SCU404 0x404 /* Multi-function Pin Control #2  */
+#define SCU408 0x408 /* Multi-function Pin Control #3  */
+#define SCU40C 0x40C /* Multi-function Pin Control #3  */
+#define SCU410 0x410 /* USB Multi-function Control Register  */
+#define SCU414 0x414 /* VGA Function Control Register  */
+
+#define SCU480 0x480 /* GPIO18A0 IO Control Register */
+#define SCU484 0x484 /* GPIO18A1 IO Control Register */
+#define SCU488 0x488 /* GPIO18A2 IO Control Register */
+#define SCU48C 0x48c /* GPIO18A3 IO Control Register */
+#define SCU490 0x490 /* GPIO18A4 IO Control Register */
+#define SCU494 0x494 /* GPIO18A5 IO Control Register */
+#define SCU498 0x498 /* GPIO18A6 IO Control Register */
+#define SCU49C 0x49c /* GPIO18A7 IO Control Register */
+#define SCU4A0 0x4A0 /* GPIO18B0 IO Control Register */
+#define SCU4A4 0x4A4 /* GPIO18B1 IO Control Register */
+#define SCU4A8 0x4A8 /* GPIO18B2 IO Control Register */
+#define SCU4AC 0x4AC /* GPIO18B3 IO Control Register */
+
+enum {
+	AC14,
+	AE15,
+	AD14,
+	AE14,
+	AF14,
+	AB13,
+	AB14,
+	AF15,
+	AF13,
+	AC13,
+	AD13,
+	AE13,
+	JTAG_PORT,
+	PCIERC0_PERST,
+	PCIERC1_PERST,
+	PORTA_MODE,
+	PORTA_U2,
+	PORTB_MODE,
+	PORTB_U2,
+	PORTA_U2_PHY,
+	PORTB_U2_PHY,
+	PORTA_U3,
+	PORTB_U3,
+	PORTA_U3_PHY,
+	PORTB_U3_PHY,
+};
+
+SIG_EXPR_LIST_DECL_SEMG(AC14, EMMCCLK, EMMCG1, EMMC, SIG_DESC_SET(SCU400, 0));
+SIG_EXPR_LIST_DECL_SEMG(AC14, VB1CS, VB1, VB, SIG_DESC_SET(SCU404, 0));
+PIN_DECL_2(AC14, GPIO18A0, EMMCCLK, VB1CS);
+
+SIG_EXPR_LIST_DECL_SEMG(AE15, EMMCCMD, EMMCG1, EMMC, SIG_DESC_SET(SCU400, 1));
+SIG_EXPR_LIST_DECL_SEMG(AE15, VB1CK, VB1, VB, SIG_DESC_SET(SCU404, 1));
+PIN_DECL_2(AE15, GPIO18A1, EMMCCMD, VB1CK);
+
+SIG_EXPR_LIST_DECL_SEMG(AD14, EMMCDAT0, EMMCG1, EMMC, SIG_DESC_SET(SCU400, 2));
+SIG_EXPR_LIST_DECL_SEMG(AD14, VB1MOSI, VB1, VB, SIG_DESC_SET(SCU404, 2));
+PIN_DECL_2(AD14, GPIO18A2, EMMCDAT0, VB1MOSI);
+
+SIG_EXPR_LIST_DECL_SEMG(AE14, EMMCDAT1, EMMCG4, EMMC, SIG_DESC_SET(SCU400, 3));
+SIG_EXPR_LIST_DECL_SEMG(AE14, VB1MISO, VB1, VB, SIG_DESC_SET(SCU404, 3));
+PIN_DECL_2(AE14, GPIO18A3, EMMCDAT1, VB1MISO);
+
+SIG_EXPR_LIST_DECL_SEMG(AF14, EMMCDAT2, EMMCG4, EMMC, SIG_DESC_SET(SCU400, 4));
+PIN_DECL_1(AF14, GPIO18A4, EMMCDAT2);
+
+SIG_EXPR_LIST_DECL_SEMG(AB13, EMMCDAT3, EMMCG4, EMMC, SIG_DESC_SET(SCU400, 5));
+PIN_DECL_1(AB13, GPIO18A5, EMMCDAT3);
+
+SIG_EXPR_LIST_DECL_SEMG(AB14, EMMCCDN, EMMCG1, EMMC, SIG_DESC_SET(SCU400, 6));
+SIG_EXPR_LIST_DECL_SEMG(AB14, VB0CS, VB0, VB, SIG_DESC_SET(SCU010, 17));
+PIN_DECL_2(AB14, GPIO18A6, EMMCCDN, VB0CS);
+
+SIG_EXPR_LIST_DECL_SEMG(AF15, EMMCWPN, EMMCG1, EMMC, SIG_DESC_SET(SCU400, 7));
+SIG_EXPR_LIST_DECL_SEMG(AF15, VB0CK, VB0, VB, SIG_DESC_SET(SCU010, 17));
+PIN_DECL_2(AF15, GPIO18A7, EMMCWPN, VB0CK);
+
+SIG_EXPR_LIST_DECL_SESG(AF13, TSPRSTN, TSPRSTN, SIG_DESC_SET(SCU010, 9));
+SIG_EXPR_LIST_DECL_SEMG(AF13, EMMCDAT4, EMMCG8, EMMC, SIG_DESC_SET(SCU400, 8));
+SIG_EXPR_LIST_DECL_SEMG(AF13, VB0MOSI, VB0, VB, SIG_DESC_SET(SCU010, 17));
+PIN_DECL_3(AF13, GPIO18B0, TSPRSTN, EMMCDAT4, VB0MOSI);
+
+SIG_EXPR_LIST_DECL_SESG(AC13, UFSCLKI, UFSCLKI, SIG_DESC_SET(SCU010, 19));
+SIG_EXPR_LIST_DECL_SEMG(AC13, EMMCDAT5, EMMCG8, EMMC, SIG_DESC_SET(SCU400, 9));
+SIG_EXPR_LIST_DECL_SEMG(AC13, VB0MISO, VB0, VB, SIG_DESC_SET(SCU010, 17));
+PIN_DECL_3(AC13, GPIO18B1, UFSCLKI, EMMCDAT5, VB0MISO);
+
+SIG_EXPR_LIST_DECL_SEMG(AD13, EMMCDAT6, EMMCG8, EMMC, SIG_DESC_SET(SCU400, 10));
+SIG_EXPR_LIST_DECL_SESG(AD13, DDCCLK, VGADDC, SIG_DESC_SET(SCU404, 10));
+PIN_DECL_2(AD13, GPIO18B2, EMMCDAT6, DDCCLK);
+
+SIG_EXPR_LIST_DECL_SEMG(AE13, EMMCDAT7, EMMCG8, EMMC, SIG_DESC_SET(SCU400, 11));
+SIG_EXPR_LIST_DECL_SESG(AE13, DDCDAT, VGADDC, SIG_DESC_SET(SCU404, 11));
+PIN_DECL_2(AE13, GPIO18B3, EMMCDAT7, DDCDAT);
+
+GROUP_DECL(EMMCG1, AC14, AE15, AD14);
+GROUP_DECL(EMMCG4, AC14, AE15, AD14, AE14, AF14, AB13);
+GROUP_DECL(EMMCG8, AC14, AE15, AD14, AE14, AF14, AB13, AF13, AC13, AD13, AE13);
+GROUP_DECL(EMMCWPN, AF15);
+GROUP_DECL(EMMCCDN, AB14);
+FUNC_DECL_(EMMC, "EMMCG1", "EMMCG4", "EMMCG8", "EMMCWPN", "EMMCCDN");
+
+GROUP_DECL(VB1, AC14, AE15, AD14, AE14);
+GROUP_DECL(VB0, AF15, AB14, AF13, AC13);
+FUNC_DECL_2(VB, VB1, VB0);
+
+FUNC_GROUP_DECL(TSPRSTN, AF13);
+
+FUNC_GROUP_DECL(UFSCLKI, AC13);
+
+FUNC_GROUP_DECL(VGADDC, AD13, AE13);
+
+/* JTAG Port Selection */
+#define JTAG_PORT_PSP_DESC   { ASPEED_IP_SCU, SCU408, GENMASK(12, 5), 0x0, 0 }
+#define JTAG_PORT_SSP_DESC   { ASPEED_IP_SCU, SCU408, GENMASK(12, 5), 0x41, 0 }
+#define JTAG_PORT_TSP_DESC   { ASPEED_IP_SCU, SCU408, GENMASK(12, 5), 0x42, 0 }
+#define JTAG_PORT_DDR_DESC   { ASPEED_IP_SCU, SCU408, GENMASK(12, 5), 0x43, 0 }
+#define JTAG_PORT_USB3A_DESC { ASPEED_IP_SCU, SCU408, GENMASK(12, 5), 0x44, 0 }
+#define JTAG_PORT_USB3B_DESC { ASPEED_IP_SCU, SCU408, GENMASK(12, 5), 0x45, 0 }
+#define JTAG_PORT_PCIEA_DESC { ASPEED_IP_SCU, SCU408, GENMASK(12, 5), 0x46, 0 }
+#define JTAG_PORT_PCIEB_DESC { ASPEED_IP_SCU, SCU408, GENMASK(12, 5), 0x47, 0 }
+#define JTAG_PORT_JTAGM0_DESC { ASPEED_IP_SCU, SCU408, GENMASK(12, 5), 0x8, 0 }
+
+SIG_EXPR_LIST_DECL_SEMG(JTAG_PORT, JTAGPSP, JTAG0, JTAGPSP, JTAG_PORT_PSP_DESC);
+SIG_EXPR_LIST_DECL_SEMG(JTAG_PORT, JTAGSSP, JTAG0, JTAGSSP, JTAG_PORT_SSP_DESC);
+SIG_EXPR_LIST_DECL_SEMG(JTAG_PORT, JTAGTSP, JTAG0, JTAGTSP, JTAG_PORT_TSP_DESC);
+SIG_EXPR_LIST_DECL_SEMG(JTAG_PORT, JTAGDDR, JTAG0, JTAGDDR, JTAG_PORT_DDR_DESC);
+SIG_EXPR_LIST_DECL_SEMG(JTAG_PORT, JTAGUSB3A, JTAG0, JTAGUSB3A, JTAG_PORT_USB3A_DESC);
+SIG_EXPR_LIST_DECL_SEMG(JTAG_PORT, JTAGUSB3B, JTAG0, JTAGUSB3B, JTAG_PORT_USB3B_DESC);
+SIG_EXPR_LIST_DECL_SEMG(JTAG_PORT, JTAGPCIEA, JTAG0, JTAGPCIEA, JTAG_PORT_PCIEA_DESC);
+SIG_EXPR_LIST_DECL_SEMG(JTAG_PORT, JTAGPCIEB, JTAG0, JTAGPCIEB, JTAG_PORT_PCIEB_DESC);
+SIG_EXPR_LIST_DECL_SEMG(JTAG_PORT, JTAGM0, JTAG0, JTAGM0, JTAG_PORT_JTAGM0_DESC);
+PIN_DECL_(JTAG_PORT, SIG_EXPR_LIST_PTR(JTAG_PORT, JTAGPSP), SIG_EXPR_LIST_PTR(JTAG_PORT, JTAGSSP),
+	  SIG_EXPR_LIST_PTR(JTAG_PORT, JTAGTSP), SIG_EXPR_LIST_PTR(JTAG_PORT, JTAGDDR),
+	  SIG_EXPR_LIST_PTR(JTAG_PORT, JTAGUSB3A), SIG_EXPR_LIST_PTR(JTAG_PORT, JTAGUSB3B),
+	  SIG_EXPR_LIST_PTR(JTAG_PORT, JTAGPCIEA), SIG_EXPR_LIST_PTR(JTAG_PORT, JTAGPCIEB),
+	  SIG_EXPR_LIST_PTR(JTAG_PORT, JTAGM0));
+
+GROUP_DECL(JTAG0, JTAG_PORT);
+
+FUNC_DECL_1(JTAGPSP, JTAG0);
+FUNC_DECL_1(JTAGSSP, JTAG0);
+FUNC_DECL_1(JTAGTSP, JTAG0);
+FUNC_DECL_1(JTAGDDR, JTAG0);
+FUNC_DECL_1(JTAGUSB3A, JTAG0);
+FUNC_DECL_1(JTAGUSB3B, JTAG0);
+FUNC_DECL_1(JTAGPCIEA, JTAG0);
+FUNC_DECL_1(JTAGPCIEB, JTAG0);
+FUNC_DECL_1(JTAGM0, JTAG0);
+
+/* PCIe Reset Control */
+SIG_EXPR_LIST_DECL_SESG(PCIERC0_PERST, PCIERC0PERST, PCIERC0PERST, SIG_DESC_SET(SCU200, 21));
+PIN_DECL_(PCIERC0_PERST, SIG_EXPR_LIST_PTR(PCIERC0_PERST, PCIERC0PERST));
+FUNC_GROUP_DECL(PCIERC0PERST, PCIERC0_PERST);
+
+SIG_EXPR_LIST_DECL_SESG(PCIERC1_PERST, PCIERC1PERST, PCIERC1PERST, SIG_DESC_SET(SCU200, 19));
+PIN_DECL_(PCIERC1_PERST, SIG_EXPR_LIST_PTR(PCIERC1_PERST, PCIERC1PERST));
+FUNC_GROUP_DECL(PCIERC1PERST, PCIERC1_PERST);
+
+#define PORTA_MODE_HPD0_DESC { ASPEED_IP_SCU, SCU410, GENMASK(25, 24), 0, 0 }
+#define PORTA_MODE_D0_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(25, 24), 1, 0 }
+#define PORTA_MODE_H_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(25, 24), 2, 0 }
+#define PORTA_MODE_HP_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(25, 24), 3, 0 }
+
+SIG_EXPR_LIST_DECL_SEMG(PORTA_MODE, USB2AHPD0, USB2AH, USB2AHPD0, PORTA_MODE_HPD0_DESC);
+SIG_EXPR_LIST_DECL_SEMG(PORTA_MODE, USB2AH, USB2AHAP, USB2AH, PORTA_MODE_H_DESC);
+SIG_EXPR_LIST_DECL_SEMG(PORTA_MODE, USB2AHP, USB2AHAP, USB2AHP, PORTA_MODE_HP_DESC);
+SIG_EXPR_LIST_DECL_SEMG(PORTA_MODE, USB2AD0, USB2AHAP, USB2AD0, PORTA_MODE_D0_DESC);
+PIN_DECL_(PORTA_MODE, SIG_EXPR_LIST_PTR(PORTA_MODE, USB2AHPD0),
+	  SIG_EXPR_LIST_PTR(PORTA_MODE, USB2AH), SIG_EXPR_LIST_PTR(PORTA_MODE, USB2AHP),
+	  SIG_EXPR_LIST_PTR(PORTA_MODE, USB2AD0));
+
+#define PORTA_U2_XHD_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(3, 2), 0, 0 }
+#define PORTA_U2_D1_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(3, 2), 1, 0 }
+#define PORTA_U2_XH_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(3, 2), 2, 0 }
+#define PORTA_U2_XH2E_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(3, 2), 3, 0 }
+
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U2, USB2AXHD1, USB2A, USB2AXHD1, PORTA_U2_XHD_DESC,
+			SIG_DESC_SET(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U2, USB2AXHPD1, USB2A, USB2AXHPD1, PORTA_U2_XHD_DESC,
+			SIG_DESC_CLEAR(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U2, USB2AXH, USB2AAP, USB2AXH, PORTA_U2_XH_DESC,
+			SIG_DESC_SET(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U2, USB2AXHP, USB2AAP, USB2AXHP, PORTA_U2_XH_DESC,
+			SIG_DESC_CLEAR(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U2, USB2AXH2B, USB2ABP, USB2AXH2B, PORTA_U2_XH2E_DESC,
+			SIG_DESC_SET(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U2, USB2AXHP2B, USB2ABP, USB2AXHP2B, PORTA_U2_XH2E_DESC,
+			SIG_DESC_CLEAR(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U2, USB2AD1, USB2ADAP, USB2AD1, PORTA_U2_D1_DESC);
+PIN_DECL_(PORTA_U2, SIG_EXPR_LIST_PTR(PORTA_U2, USB2AXHD1), SIG_EXPR_LIST_PTR(PORTA_U2, USB2AXHPD1),
+	  SIG_EXPR_LIST_PTR(PORTA_U2, USB2AXH), SIG_EXPR_LIST_PTR(PORTA_U2, USB2AXHP),
+	  SIG_EXPR_LIST_PTR(PORTA_U2, USB2AXH2B), SIG_EXPR_LIST_PTR(PORTA_U2, USB2AXHP2B),
+	  SIG_EXPR_LIST_PTR(PORTA_U2, USB2AD1));
+
+#define PORTB_MODE_HPD0_DESC { ASPEED_IP_SCU, SCU410, GENMASK(29, 28), 0, 0 }
+#define PORTB_MODE_D0_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(29, 28), 1, 0 }
+#define PORTB_MODE_H_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(29, 28), 2, 0 }
+#define PORTB_MODE_HP_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(29, 28), 3, 0 }
+
+SIG_EXPR_LIST_DECL_SEMG(PORTB_MODE, USB2BHPD0, USB2BH, USB2BHPD0, PORTB_MODE_HPD0_DESC);
+SIG_EXPR_LIST_DECL_SEMG(PORTB_MODE, USB2BH, USB2BHBP, USB2BH, PORTB_MODE_H_DESC);
+SIG_EXPR_LIST_DECL_SEMG(PORTB_MODE, USB2BHP, USB2BHBP, USB2BHP, PORTB_MODE_HP_DESC);
+SIG_EXPR_LIST_DECL_SEMG(PORTB_MODE, USB2BD0, USB2BHBP, USB2BD0, PORTB_MODE_D0_DESC);
+PIN_DECL_(PORTB_MODE, SIG_EXPR_LIST_PTR(PORTB_MODE, USB2BHPD0),
+	  SIG_EXPR_LIST_PTR(PORTB_MODE, USB2BH), SIG_EXPR_LIST_PTR(PORTB_MODE, USB2BHP),
+	  SIG_EXPR_LIST_PTR(PORTB_MODE, USB2BD0));
+
+#define PORTB_U2_XHD_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(7, 6), 0, 0 }
+#define PORTB_U2_D1_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(7, 6), 1, 0 }
+#define PORTB_U2_XH_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(7, 6), 2, 0 }
+#define PORTB_U2_XH2E_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(7, 6), 3, 0 }
+
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U2, USB2BXHD1, USB2B, USB2BXHD1, PORTB_U2_XHD_DESC,
+			SIG_DESC_SET(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U2, USB2BXHPD1, USB2B, USB2BXHPD1, PORTB_U2_XHD_DESC,
+			SIG_DESC_CLEAR(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U2, USB2BXH, USB2BBP, USB2BXH, PORTB_U2_XH_DESC,
+			SIG_DESC_SET(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U2, USB2BXHP, USB2BBP, USB2BXHP, PORTB_U2_XH_DESC,
+			SIG_DESC_CLEAR(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U2, USB2BXH2A, USB2BAP, USB2BXH2A, PORTB_U2_XH2E_DESC,
+			SIG_DESC_SET(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U2, USB2BXHP2A, USB2BAP, USB2BXHP2A, PORTB_U2_XH2E_DESC,
+			SIG_DESC_CLEAR(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U2, USB2BD1, USB2BDBP, USB2BD1, PORTB_U2_D1_DESC);
+PIN_DECL_(PORTB_U2, SIG_EXPR_LIST_PTR(PORTB_U2, USB2BXHD1), SIG_EXPR_LIST_PTR(PORTB_U2, USB2BXHPD1),
+	  SIG_EXPR_LIST_PTR(PORTB_U2, USB2BXH), SIG_EXPR_LIST_PTR(PORTB_U2, USB2BXHP),
+	  SIG_EXPR_LIST_PTR(PORTB_U2, USB2BXH2A), SIG_EXPR_LIST_PTR(PORTB_U2, USB2BXHP2A),
+	  SIG_EXPR_LIST_PTR(PORTB_U2, USB2BD1));
+/*
+ * USB2 virtual PHY pins.
+ *
+ * PORTA_U2_PHY and PORTB_U2_PHY are logical endpoints, not package pins.
+ * They alias existing USB2 expressions so pin groups can model direct and
+ * cross-coupled routing for host and mode paths.
+ *
+ * - USB2AAP/USB2ADAP/USB2AHAP: use PORTA_U2_PHY
+ * - USB2ABP                  : use PORTB_U2_PHY
+ * - USB2BBP/USB2BDBP/USB2BHBP: use PORTB_U2_PHY
+ * - USB2BAP                  : use PORTA_U2_PHY
+ *
+ * They do not have any registers to configure this behaviour; the goal is
+ * simply for the driver to prevent conflicting selections. For example,
+ * selecting group USB2ABP and USB2BBP at the same time should not be
+ * allowed.
+ */
+SIG_EXPR_LIST_ALIAS(PORTA_U2_PHY, USB2AXH, USB2AAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U2_PHY, USB2AXHP, USB2AAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U2_PHY, USB2BXH2A, USB2BAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U2_PHY, USB2BXHP2A, USB2BAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U2_PHY, USB2AD1, USB2ADAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U2_PHY, USB2AH, USB2AHAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U2_PHY, USB2AHP, USB2AHAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U2_PHY, USB2AD0, USB2AHAP);
+PIN_DECL_(PORTA_U2_PHY, SIG_EXPR_LIST_PTR(PORTA_U2_PHY, USB2AXH),
+	  SIG_EXPR_LIST_PTR(PORTA_U2_PHY, USB2AXHP), SIG_EXPR_LIST_PTR(PORTA_U2_PHY, USB2BXH2A),
+	  SIG_EXPR_LIST_PTR(PORTA_U2_PHY, USB2BXHP2A), SIG_EXPR_LIST_PTR(PORTA_U2_PHY, USB2AD1),
+	  SIG_EXPR_LIST_PTR(PORTA_U2_PHY, USB2AH), SIG_EXPR_LIST_PTR(PORTA_U2_PHY, USB2AHP),
+	  SIG_EXPR_LIST_PTR(PORTA_U2_PHY, USB2AD0));
+
+SIG_EXPR_LIST_ALIAS(PORTB_U2_PHY, USB2AXH2B, USB2ABP);
+SIG_EXPR_LIST_ALIAS(PORTB_U2_PHY, USB2AXHP2B, USB2ABP);
+SIG_EXPR_LIST_ALIAS(PORTB_U2_PHY, USB2BXH, USB2BBP);
+SIG_EXPR_LIST_ALIAS(PORTB_U2_PHY, USB2BXHP, USB2BBP);
+SIG_EXPR_LIST_ALIAS(PORTB_U2_PHY, USB2BD1, USB2BDBP);
+SIG_EXPR_LIST_ALIAS(PORTB_U2_PHY, USB2BH, USB2BHBP);
+SIG_EXPR_LIST_ALIAS(PORTB_U2_PHY, USB2BHP, USB2BHBP);
+SIG_EXPR_LIST_ALIAS(PORTB_U2_PHY, USB2BD0, USB2BHBP);
+PIN_DECL_(PORTB_U2_PHY, SIG_EXPR_LIST_PTR(PORTB_U2_PHY, USB2AXH2B),
+	  SIG_EXPR_LIST_PTR(PORTB_U2_PHY, USB2AXHP2B), SIG_EXPR_LIST_PTR(PORTB_U2_PHY, USB2BXH),
+	  SIG_EXPR_LIST_PTR(PORTB_U2_PHY, USB2BXHP), SIG_EXPR_LIST_PTR(PORTB_U2_PHY, USB2BD1),
+	  SIG_EXPR_LIST_PTR(PORTB_U2_PHY, USB2BH), SIG_EXPR_LIST_PTR(PORTB_U2_PHY, USB2BHP),
+	  SIG_EXPR_LIST_PTR(PORTB_U2_PHY, USB2BD0));
+
+GROUP_DECL(USB2A, PORTA_U2);
+GROUP_DECL(USB2AAP, PORTA_U2, PORTA_U2_PHY);
+GROUP_DECL(USB2ABP, PORTA_U2, PORTB_U2_PHY);
+GROUP_DECL(USB2ADAP, PORTA_U2, PORTA_U2_PHY);
+GROUP_DECL(USB2AH, PORTA_MODE);
+GROUP_DECL(USB2AHAP, PORTA_MODE, PORTA_U2_PHY);
+
+FUNC_DECL_1(USB2AXHD1, USB2A);
+FUNC_DECL_1(USB2AXHPD1, USB2A);
+FUNC_DECL_1(USB2AXH, USB2AAP);
+FUNC_DECL_1(USB2AXHP, USB2AAP);
+FUNC_DECL_1(USB2AXH2B, USB2ABP);
+FUNC_DECL_1(USB2AXHP2B, USB2ABP);
+FUNC_DECL_1(USB2AD1, USB2ADAP);
+FUNC_DECL_1(USB2AHPD0, USB2AH);
+FUNC_DECL_1(USB2AH, USB2AHAP);
+FUNC_DECL_1(USB2AHP, USB2AHAP);
+FUNC_DECL_1(USB2AD0, USB2AHAP);
+
+GROUP_DECL(USB2B, PORTB_U2);
+GROUP_DECL(USB2BBP, PORTB_U2, PORTB_U2_PHY);
+GROUP_DECL(USB2BAP, PORTB_U2, PORTA_U2_PHY);
+GROUP_DECL(USB2BDBP, PORTB_U2, PORTB_U2_PHY);
+GROUP_DECL(USB2BH, PORTB_MODE);
+GROUP_DECL(USB2BHBP, PORTB_MODE, PORTB_U2_PHY);
+
+FUNC_DECL_1(USB2BXHD1, USB2B);
+FUNC_DECL_1(USB2BXHPD1, USB2B);
+FUNC_DECL_1(USB2BXH, USB2BBP);
+FUNC_DECL_1(USB2BXHP, USB2BBP);
+FUNC_DECL_1(USB2BXH2A, USB2BAP);
+FUNC_DECL_1(USB2BXHP2A, USB2BAP);
+FUNC_DECL_1(USB2BD1, USB2BDBP);
+FUNC_DECL_1(USB2BHPD0, USB2BH);
+FUNC_DECL_1(USB2BH, USB2BHBP);
+FUNC_DECL_1(USB2BHP, USB2BHBP);
+FUNC_DECL_1(USB2BD0, USB2BHBP);
+
+#define PORTA_U3_XHD_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(1, 0), 0, 0 }
+#define PORTA_U3_XH_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(1, 0), 2, 0 }
+#define PORTA_U3_XH2E_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(1, 0), 3, 0 }
+
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U3, USB3AXHD, USB3A, USB3AXHD, PORTA_U3_XHD_DESC,
+			SIG_DESC_SET(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U3, USB3AXHPD, USB3A, USB3AXHPD, PORTA_U3_XHD_DESC,
+			SIG_DESC_CLEAR(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U3, USB3AXH, USB3AAP, USB3AXH, PORTA_U3_XH_DESC,
+			SIG_DESC_SET(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U3, USB3AXHP, USB3AAP, USB3AXHP, PORTA_U3_XH_DESC,
+			SIG_DESC_CLEAR(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U3, USB3AXH2B, USB3ABP, USB3AXH2B, PORTA_U3_XH2E_DESC,
+			SIG_DESC_SET(SCU410, 9));
+SIG_EXPR_LIST_DECL_SEMG(PORTA_U3, USB3AXHP2B, USB3ABP, USB3AXHP2B, PORTA_U3_XH2E_DESC,
+			SIG_DESC_CLEAR(SCU410, 9));
+PIN_DECL_(PORTA_U3, SIG_EXPR_LIST_PTR(PORTA_U3, USB3AXHD), SIG_EXPR_LIST_PTR(PORTA_U3, USB3AXHPD),
+	  SIG_EXPR_LIST_PTR(PORTA_U3, USB3AXH), SIG_EXPR_LIST_PTR(PORTA_U3, USB3AXHP),
+	  SIG_EXPR_LIST_PTR(PORTA_U3, USB3AXH2B), SIG_EXPR_LIST_PTR(PORTA_U3, USB3AXHP2B));
+
+#define PORTB_U3_XHD_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(5, 4), 0, 0 }
+#define PORTB_U3_XH_DESC    { ASPEED_IP_SCU, SCU410, GENMASK(5, 4), 2, 0 }
+#define PORTB_U3_XH2E_DESC   { ASPEED_IP_SCU, SCU410, GENMASK(5, 4), 3, 0 }
+
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U3, USB3BXHD, USB3B, USB3BXHD, PORTB_U3_XHD_DESC,
+			SIG_DESC_SET(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U3, USB3BXHPD, USB3B, USB3BXHPD, PORTB_U3_XHD_DESC,
+			SIG_DESC_CLEAR(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U3, USB3BXH, USB3BBP, USB3BXH, PORTB_U3_XH_DESC,
+			SIG_DESC_SET(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U3, USB3BXHP, USB3BBP, USB3BXHP, PORTB_U3_XH_DESC,
+			SIG_DESC_CLEAR(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U3, USB3BXH2A, USB3BAP, USB3BXH2A, PORTB_U3_XH2E_DESC,
+			SIG_DESC_SET(SCU410, 10));
+SIG_EXPR_LIST_DECL_SEMG(PORTB_U3, USB3BXHP2A, USB3BAP, USB3BXHP2A, PORTB_U3_XH2E_DESC,
+			SIG_DESC_CLEAR(SCU410, 10));
+PIN_DECL_(PORTB_U3, SIG_EXPR_LIST_PTR(PORTB_U3, USB3BXHD), SIG_EXPR_LIST_PTR(PORTB_U3, USB3BXHPD),
+	  SIG_EXPR_LIST_PTR(PORTB_U3, USB3BXH), SIG_EXPR_LIST_PTR(PORTB_U3, USB3BXHP),
+	  SIG_EXPR_LIST_PTR(PORTB_U3, USB3BXH2A), SIG_EXPR_LIST_PTR(PORTB_U3, USB3BXHP2A));
+
+/*
+ * USB3 virtual PHY pins.
+ *
+ * PORTA_U3_PHY and PORTB_U3_PHY are logical endpoints, not package pins.
+ * They alias existing USB3 expressions so pin groups can model both direct and
+ * cross-coupled routing to PHY A/B.
+ *
+ * - USB3AAP: PORTA_U3 + PORTA_U3_PHY   (A -> PHY A)
+ * - USB3ABP: PORTA_U3 + PORTB_U3_PHY   (A -> PHY B)
+ * - USB3BBP: PORTB_U3 + PORTB_U3_PHY   (B -> PHY B)
+ * - USB3BAP: PORTB_U3 + PORTA_U3_PHY   (B -> PHY A)
+ *
+ * They do not have any registers to configure this behavior; the goal is
+ * simply for the driver to prevent conflicting selections. For example,
+ * selecting group USB3ABP and USB3BBP at the same time should not be
+ * allowed.
+ */
+SIG_EXPR_LIST_ALIAS(PORTA_U3_PHY, USB3AXH, USB3AAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U3_PHY, USB3AXHP, USB3AAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U3_PHY, USB3BXH2A, USB3BAP);
+SIG_EXPR_LIST_ALIAS(PORTA_U3_PHY, USB3BXHP2A, USB3BAP);
+PIN_DECL_(PORTA_U3_PHY, SIG_EXPR_LIST_PTR(PORTA_U3_PHY, USB3AXH),
+	  SIG_EXPR_LIST_PTR(PORTA_U3_PHY, USB3AXHP), SIG_EXPR_LIST_PTR(PORTA_U3_PHY, USB3BXH2A),
+	  SIG_EXPR_LIST_PTR(PORTA_U3_PHY, USB3BXHP2A));
+
+SIG_EXPR_LIST_ALIAS(PORTB_U3_PHY, USB3AXH2B, USB3ABP);
+SIG_EXPR_LIST_ALIAS(PORTB_U3_PHY, USB3AXHP2B, USB3ABP);
+SIG_EXPR_LIST_ALIAS(PORTB_U3_PHY, USB3BXH, USB3BBP);
+SIG_EXPR_LIST_ALIAS(PORTB_U3_PHY, USB3BXHP, USB3BBP);
+PIN_DECL_(PORTB_U3_PHY, SIG_EXPR_LIST_PTR(PORTB_U3_PHY, USB3AXH2B),
+	  SIG_EXPR_LIST_PTR(PORTB_U3_PHY, USB3AXHP2B), SIG_EXPR_LIST_PTR(PORTB_U3_PHY, USB3BXH),
+	  SIG_EXPR_LIST_PTR(PORTB_U3_PHY, USB3BXHP));
+
+/* USB3A xHCI to vHUB */
+GROUP_DECL(USB3A, PORTA_U3);
+/* USB3A xHCI to USB3A PHY */
+GROUP_DECL(USB3AAP, PORTA_U3, PORTA_U3_PHY);
+/* USB3A xHCI to USB3B PHY */
+GROUP_DECL(USB3ABP, PORTA_U3, PORTB_U3_PHY);
+
+FUNC_DECL_1(USB3AXHD, USB3A);
+FUNC_DECL_1(USB3AXHPD, USB3A);
+FUNC_DECL_1(USB3AXH, USB3AAP);
+FUNC_DECL_1(USB3AXHP, USB3AAP);
+FUNC_DECL_1(USB3AXH2B, USB3ABP);
+FUNC_DECL_1(USB3AXHP2B, USB3ABP);
+
+/* USB3B xHCI to vHUB */
+GROUP_DECL(USB3B, PORTB_U3);
+/* USB3B xHCI to USB3A PHY */
+GROUP_DECL(USB3BAP, PORTB_U3, PORTA_U3_PHY);
+/* USB3B xHCI to USB3B PHY */
+GROUP_DECL(USB3BBP, PORTB_U3, PORTB_U3_PHY);
+
+FUNC_DECL_1(USB3BXHD, USB3B);
+FUNC_DECL_1(USB3BXHPD, USB3B);
+FUNC_DECL_1(USB3BXH, USB3BBP);
+FUNC_DECL_1(USB3BXHP, USB3BBP);
+FUNC_DECL_1(USB3BXH2A, USB3BAP);
+FUNC_DECL_1(USB3BXHP2A, USB3BAP);
+
+static const struct pinctrl_pin_desc aspeed_g7_soc0_pins[] = {
+	ASPEED_PINCTRL_PIN(AC14),
+	ASPEED_PINCTRL_PIN(AE15),
+	ASPEED_PINCTRL_PIN(AD14),
+	ASPEED_PINCTRL_PIN(AE14),
+	ASPEED_PINCTRL_PIN(AF14),
+	ASPEED_PINCTRL_PIN(AB13),
+	ASPEED_PINCTRL_PIN(AB14),
+	ASPEED_PINCTRL_PIN(AF15),
+	ASPEED_PINCTRL_PIN(AF13),
+	ASPEED_PINCTRL_PIN(AC13),
+	ASPEED_PINCTRL_PIN(AD13),
+	ASPEED_PINCTRL_PIN(AE13),
+	ASPEED_PINCTRL_PIN(JTAG_PORT),
+	ASPEED_PINCTRL_PIN(PCIERC0_PERST),
+	ASPEED_PINCTRL_PIN(PCIERC1_PERST),
+	ASPEED_PINCTRL_PIN(PORTA_MODE),
+	ASPEED_PINCTRL_PIN(PORTA_U2),
+	ASPEED_PINCTRL_PIN(PORTA_U3),
+	ASPEED_PINCTRL_PIN(PORTA_U2_PHY),
+	ASPEED_PINCTRL_PIN(PORTA_U3_PHY),
+	ASPEED_PINCTRL_PIN(PORTB_MODE),
+	ASPEED_PINCTRL_PIN(PORTB_U2),
+	ASPEED_PINCTRL_PIN(PORTB_U3),
+	ASPEED_PINCTRL_PIN(PORTB_U2_PHY),
+	ASPEED_PINCTRL_PIN(PORTB_U3_PHY),
+};
+
+static const struct aspeed_pin_group aspeed_g7_soc0_groups[] = {
+	ASPEED_PINCTRL_GROUP(EMMCCDN),
+	ASPEED_PINCTRL_GROUP(EMMCG1),
+	ASPEED_PINCTRL_GROUP(EMMCG4),
+	ASPEED_PINCTRL_GROUP(EMMCG8),
+	ASPEED_PINCTRL_GROUP(EMMCWPN),
+	ASPEED_PINCTRL_GROUP(TSPRSTN),
+	ASPEED_PINCTRL_GROUP(UFSCLKI),
+	ASPEED_PINCTRL_GROUP(VB0),
+	ASPEED_PINCTRL_GROUP(VB1),
+	ASPEED_PINCTRL_GROUP(VGADDC),
+	/* JTAG groups */
+	ASPEED_PINCTRL_GROUP(JTAG0),
+	/* PCIE RC groups */
+	ASPEED_PINCTRL_GROUP(PCIERC0PERST),
+	ASPEED_PINCTRL_GROUP(PCIERC1PERST),
+	/* USB3A groups */
+	ASPEED_PINCTRL_GROUP(USB3A),
+	ASPEED_PINCTRL_GROUP(USB3AAP),
+	ASPEED_PINCTRL_GROUP(USB3ABP),
+	/* USB3B groups */
+	ASPEED_PINCTRL_GROUP(USB3B),
+	ASPEED_PINCTRL_GROUP(USB3BAP),
+	ASPEED_PINCTRL_GROUP(USB3BBP),
+	/* USB2A groups */
+	ASPEED_PINCTRL_GROUP(USB2A),
+	ASPEED_PINCTRL_GROUP(USB2AAP),
+	ASPEED_PINCTRL_GROUP(USB2ABP),
+	ASPEED_PINCTRL_GROUP(USB2ADAP),
+	ASPEED_PINCTRL_GROUP(USB2AH),
+	ASPEED_PINCTRL_GROUP(USB2AHAP),
+	/* USB2B groups */
+	ASPEED_PINCTRL_GROUP(USB2B),
+	ASPEED_PINCTRL_GROUP(USB2BAP),
+	ASPEED_PINCTRL_GROUP(USB2BBP),
+	ASPEED_PINCTRL_GROUP(USB2BDBP),
+	ASPEED_PINCTRL_GROUP(USB2BH),
+	ASPEED_PINCTRL_GROUP(USB2BHBP),
+};
+
+static const struct aspeed_pin_function aspeed_g7_soc0_functions[] = {
+	ASPEED_PINCTRL_FUNC(EMMC),
+	ASPEED_PINCTRL_FUNC(TSPRSTN),
+	ASPEED_PINCTRL_FUNC(UFSCLKI),
+	ASPEED_PINCTRL_FUNC(VB),
+	ASPEED_PINCTRL_FUNC(VGADDC),
+	/* JTAG functions */
+	ASPEED_PINCTRL_FUNC(JTAGDDR),
+	ASPEED_PINCTRL_FUNC(JTAGM0),
+	ASPEED_PINCTRL_FUNC(JTAGPCIEA),
+	ASPEED_PINCTRL_FUNC(JTAGPCIEB),
+	ASPEED_PINCTRL_FUNC(JTAGPSP),
+	ASPEED_PINCTRL_FUNC(JTAGSSP),
+	ASPEED_PINCTRL_FUNC(JTAGTSP),
+	ASPEED_PINCTRL_FUNC(JTAGUSB3A),
+	ASPEED_PINCTRL_FUNC(JTAGUSB3B),
+	/* PCIE RC functions */
+	ASPEED_PINCTRL_FUNC(PCIERC0PERST),
+	ASPEED_PINCTRL_FUNC(PCIERC1PERST),
+	/* USB3A functions */
+	ASPEED_PINCTRL_FUNC(USB3AXH),
+	ASPEED_PINCTRL_FUNC(USB3AXH2B),
+	ASPEED_PINCTRL_FUNC(USB3AXHD),
+	ASPEED_PINCTRL_FUNC(USB3AXHP),
+	ASPEED_PINCTRL_FUNC(USB3AXHP2B),
+	ASPEED_PINCTRL_FUNC(USB3AXHPD),
+	/* USB3B functions */
+	ASPEED_PINCTRL_FUNC(USB3BXH),
+	ASPEED_PINCTRL_FUNC(USB3BXH2A),
+	ASPEED_PINCTRL_FUNC(USB3BXHD),
+	ASPEED_PINCTRL_FUNC(USB3BXHP),
+	ASPEED_PINCTRL_FUNC(USB3BXHP2A),
+	ASPEED_PINCTRL_FUNC(USB3BXHPD),
+	/* USB2A functions */
+	ASPEED_PINCTRL_FUNC(USB2AD0),
+	ASPEED_PINCTRL_FUNC(USB2AD1),
+	ASPEED_PINCTRL_FUNC(USB2AH),
+	ASPEED_PINCTRL_FUNC(USB2AHP),
+	ASPEED_PINCTRL_FUNC(USB2AHPD0),
+	ASPEED_PINCTRL_FUNC(USB2AXH),
+	ASPEED_PINCTRL_FUNC(USB2AXH2B),
+	ASPEED_PINCTRL_FUNC(USB2AXHD1),
+	ASPEED_PINCTRL_FUNC(USB2AXHP),
+	ASPEED_PINCTRL_FUNC(USB2AXHP2B),
+	ASPEED_PINCTRL_FUNC(USB2AXHPD1),
+	/* USB2B functions */
+	ASPEED_PINCTRL_FUNC(USB2BD0),
+	ASPEED_PINCTRL_FUNC(USB2BD1),
+	ASPEED_PINCTRL_FUNC(USB2BH),
+	ASPEED_PINCTRL_FUNC(USB2BHP),
+	ASPEED_PINCTRL_FUNC(USB2BHPD0),
+	ASPEED_PINCTRL_FUNC(USB2BXH),
+	ASPEED_PINCTRL_FUNC(USB2BXH2A),
+	ASPEED_PINCTRL_FUNC(USB2BXHD1),
+	ASPEED_PINCTRL_FUNC(USB2BXHP),
+	ASPEED_PINCTRL_FUNC(USB2BXHP2A),
+	ASPEED_PINCTRL_FUNC(USB2BXHPD1),
+};
+
+static const struct pinmux_ops aspeed_g7_soc0_pinmux_ops = {
+	.get_functions_count = aspeed_pinmux_get_fn_count,
+	.get_function_name = aspeed_pinmux_get_fn_name,
+	.get_function_groups = aspeed_pinmux_get_fn_groups,
+	.set_mux = aspeed_pinmux_set_mux,
+	.gpio_request_enable = aspeed_gpio_request_enable,
+	.strict = true,
+};
+
+static const struct pinctrl_ops aspeed_g7_soc0_pinctrl_ops = {
+	.get_groups_count = aspeed_pinctrl_get_groups_count,
+	.get_group_name = aspeed_pinctrl_get_group_name,
+	.get_group_pins = aspeed_pinctrl_get_group_pins,
+	.pin_dbg_show = aspeed_pinctrl_pin_dbg_show,
+	.dt_node_to_map = pinconf_generic_dt_node_to_map_all,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+static const struct pinconf_ops aspeed_g7_soc0_pinconf_ops = {
+	.is_generic = true,
+	.pin_config_get = aspeed_pin_config_get,
+	.pin_config_set = aspeed_pin_config_set,
+	.pin_config_group_get = aspeed_pin_config_group_get,
+	.pin_config_group_set = aspeed_pin_config_group_set,
+};
+
+/* pinctrl_desc */
+static const struct pinctrl_desc aspeed_g7_soc0_pinctrl_desc = {
+	.name = "aspeed-g7-soc0-pinctrl",
+	.pins = aspeed_g7_soc0_pins,
+	.npins = ARRAY_SIZE(aspeed_g7_soc0_pins),
+	.pctlops = &aspeed_g7_soc0_pinctrl_ops,
+	.pmxops = &aspeed_g7_soc0_pinmux_ops,
+	.confops = &aspeed_g7_soc0_pinconf_ops,
+};
+
+static const struct aspeed_pin_config aspeed_g7_soc0_configs[] = {
+	/* GPIO18A */
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AC14, AC14 }, SCU480, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AC14, AC14 }, SCU480, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AC14, AC14 }, SCU480, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AC14, AC14 }, SCU480, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AE15, AE15 }, SCU484, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AE15, AE15 }, SCU484, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AE15, AE15 }, SCU484, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AE15, AE15 }, SCU484, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AD14, AD14 }, SCU488, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AD14, AD14 }, SCU488, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AD14, AD14 }, SCU488, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AD14, AD14 }, SCU488, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AE14, AE14 }, SCU48C, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AE14, AE14 }, SCU48C, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AE14, AE14 }, SCU48C, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AE14, AE14 }, SCU48C, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AF14, AF14 }, SCU490, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AF14, AF14 }, SCU490, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AF14, AF14 }, SCU490, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AF14, AF14 }, SCU490, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AB13, AB13 }, SCU494, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AB13, AB13 }, SCU494, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AB13, AB13 }, SCU494, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AB13, AB13 }, SCU494, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AB14, AB14 }, SCU498, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AB14, AB14 }, SCU498, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AB14, AB14 }, SCU498, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AB14, AB14 }, SCU498, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AF15, AF15 }, SCU49C, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AF15, AF15 }, SCU49C, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AF15, AF15 }, SCU49C, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AF15, AF15 }, SCU49C, BIT(5) },
+	/* GPIO18B */
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AF13, AF13 }, SCU4A0, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AF13, AF13 }, SCU4A0, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AF13, AF13 }, SCU4A0, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AF13, AF13 }, SCU4A0, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AC13, AC13 }, SCU4A4, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AC13, AC13 }, SCU4A4, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AC13, AC13 }, SCU4A4, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AC13, AC13 }, SCU4A4, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AD13, AD13 }, SCU4A8, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AD13, AD13 }, SCU4A8, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AD13, AD13 }, SCU4A8, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AD13, AD13 }, SCU4A8, BIT(5) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, { AE13, AE13 }, SCU4AC, GENMASK(3, 0) },
+	{ PIN_CONFIG_BIAS_PULL_DOWN, { AE13, AE13 }, SCU4AC, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_PULL_UP, { AE13, AE13 }, SCU4AC, GENMASK(5, 4) },
+	{ PIN_CONFIG_BIAS_DISABLE, { AE13, AE13 }, SCU4AC, BIT(5) },
+};
+
+static const struct aspeed_pin_config_map aspeed_g7_soc0_pin_config_map[] = {
+	{ PIN_CONFIG_BIAS_PULL_DOWN, -1, 2, GENMASK(1, 0) },
+	{ PIN_CONFIG_BIAS_PULL_UP, -1, 3, GENMASK(1, 0) },
+	{ PIN_CONFIG_BIAS_DISABLE, -1, 0, BIT_MASK(0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 3, 0, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 6, 1, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 8, 2, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 11, 3, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 16, 4, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 18, 5, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 20, 6, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 23, 7, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 30, 8, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 32, 9, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 33, 10, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 35, 11, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 37, 12, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 38, 13, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 39, 14, GENMASK(3, 0) },
+	{ PIN_CONFIG_DRIVE_STRENGTH, 41, 15, GENMASK(3, 0) },
+
+};
+
+static int aspeed_g7_soc0_sig_expr_set(struct aspeed_pinmux_data *ctx,
+				       const struct aspeed_sig_expr *expr, bool enable)
+{
+	int ret;
+	int i;
+
+	for (i = 0; i < expr->ndescs; i++) {
+		const struct aspeed_sig_desc *desc = &expr->descs[i];
+		u32 pattern = enable ? desc->enable : desc->disable;
+		u32 val = (pattern << __ffs(desc->mask));
+
+		if (!ctx->maps[desc->ip])
+			return -ENODEV;
+
+		WARN_ON_ONCE(desc->ip != ASPEED_IP_SCU);
+
+		ret = regmap_update_bits(ctx->maps[desc->ip], desc->reg,
+					 desc->mask, val);
+		if (ret)
+			return ret;
+	}
+
+	ret = aspeed_sig_expr_eval(ctx, expr, enable);
+	if (ret < 0)
+		return ret;
+
+	return ret ? 0 : -EPERM;
+}
+
+static const struct aspeed_pinmux_ops aspeed_g7_soc0_ops = {
+	.set = aspeed_g7_soc0_sig_expr_set,
+};
+
+static struct aspeed_pinctrl_data aspeed_g7_soc0_pinctrl_data = {
+	.pins = aspeed_g7_soc0_pins,
+	.npins = ARRAY_SIZE(aspeed_g7_soc0_pins),
+	.pinmux = {
+		.ops = &aspeed_g7_soc0_ops,
+		.groups = aspeed_g7_soc0_groups,
+		.ngroups = ARRAY_SIZE(aspeed_g7_soc0_groups),
+		.functions = aspeed_g7_soc0_functions,
+		.nfunctions = ARRAY_SIZE(aspeed_g7_soc0_functions),
+	},
+	.configs = aspeed_g7_soc0_configs,
+	.nconfigs = ARRAY_SIZE(aspeed_g7_soc0_configs),
+	.confmaps = aspeed_g7_soc0_pin_config_map,
+	.nconfmaps = ARRAY_SIZE(aspeed_g7_soc0_pin_config_map),
+};
+
+static int aspeed_g7_soc0_pinctrl_probe(struct platform_device *pdev)
+{
+	return aspeed_pinctrl_probe(pdev, &aspeed_g7_soc0_pinctrl_desc,
+				    &aspeed_g7_soc0_pinctrl_data);
+}
+
+static const struct of_device_id aspeed_g7_soc0_pinctrl_match[] = {
+	{ .compatible = "aspeed,ast2700-soc0-pinctrl" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, aspeed_g7_soc0_pinctrl_match);
+
+static struct platform_driver aspeed_g7_soc0_pinctrl_driver = {
+	.probe = aspeed_g7_soc0_pinctrl_probe,
+	.driver = {
+		.name = "aspeed-g7-soc0-pinctrl",
+		.of_match_table = aspeed_g7_soc0_pinctrl_match,
+		.suppress_bind_attrs = true,
+	},
+};
+
+static int __init aspeed_g7_soc0_pinctrl_init(void)
+{
+	return platform_driver_register(&aspeed_g7_soc0_pinctrl_driver);
+}
+arch_initcall(aspeed_g7_soc0_pinctrl_init);
diff --git a/drivers/pinctrl/bcm/pinctrl-bcm4908.c b/drivers/pinctrl/bcm/pinctrl-bcm4908.c
index 12f7a253ea4d5..57969cdbc6353 100644
--- a/drivers/pinctrl/bcm/pinctrl-bcm4908.c
+++ b/drivers/pinctrl/bcm/pinctrl-bcm4908.c
@@ -466,6 +466,7 @@ static const struct of_device_id bcm4908_pinctrl_of_match_table[] = {
 	{ .compatible = "brcm,bcm4908-pinctrl", },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, bcm4908_pinctrl_of_match_table);
 
 static int bcm4908_pinctrl_probe(struct platform_device *pdev)
 {
@@ -561,4 +562,3 @@ module_platform_driver(bcm4908_pinctrl_driver);
 MODULE_AUTHOR("Rafał Miłecki");
 MODULE_DESCRIPTION("Broadcom BCM4908 pinmux driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, bcm4908_pinctrl_of_match_table);
diff --git a/drivers/pinctrl/bcm/pinctrl-ns.c b/drivers/pinctrl/bcm/pinctrl-ns.c
index 03bd01b4a945a..e134c9c73450b 100644
--- a/drivers/pinctrl/bcm/pinctrl-ns.c
+++ b/drivers/pinctrl/bcm/pinctrl-ns.c
@@ -204,6 +204,7 @@ static const struct of_device_id ns_pinctrl_of_match_table[] = {
 	{ .compatible = "brcm,bcm53012-pinmux", .data = (void *)FLAG_BCM53012, },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, ns_pinctrl_of_match_table);
 
 static int ns_pinctrl_probe(struct platform_device *pdev)
 {
@@ -295,4 +296,3 @@ static struct platform_driver ns_pinctrl_driver = {
 module_platform_driver(ns_pinctrl_driver);
 
 MODULE_AUTHOR("Rafał Miłecki");
-MODULE_DEVICE_TABLE(of, ns_pinctrl_of_match_table);
diff --git a/drivers/pinctrl/freescale/pinctrl-imx1-core.c b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
index b36c8a1461b7c..b7bd4ef9c0db5 100644
--- a/drivers/pinctrl/freescale/pinctrl-imx1-core.c
+++ b/drivers/pinctrl/freescale/pinctrl-imx1-core.c
@@ -540,10 +540,34 @@ static int imx1_pinctrl_parse_functions(struct device_node *np,
 	return 0;
 }
 
+/*
+ * Check if the DT contains pins in the direct child nodes. This indicates the
+ * newer DT format to store pins. This function returns true if the first found
+ * fsl,pins property is in a child of np. Otherwise false is returned.
+ */
+static bool imx1_pinctrl_dt_is_flat_functions(struct device_node *np)
+{
+	struct device_node *function_np;
+	struct device_node *pinctrl_np;
+
+	for_each_child_of_node(np, function_np) {
+		if (of_property_present(function_np, "fsl,pins"))
+			return true;
+
+		for_each_child_of_node(function_np, pinctrl_np) {
+			if (of_property_present(pinctrl_np, "fsl,pins"))
+				return false;
+		}
+	}
+
+	return true;
+}
+
 static int imx1_pinctrl_parse_dt(struct platform_device *pdev,
 		struct imx1_pinctrl *pctl, struct imx1_pinctrl_soc_info *info)
 {
 	struct device_node *np = pdev->dev.of_node;
+	bool flat_funcs;
 	int ret;
 	u32 nfuncs = 0;
 	u32 ngroups = 0;
@@ -552,9 +576,15 @@ static int imx1_pinctrl_parse_dt(struct platform_device *pdev,
 	if (!np)
 		return -ENODEV;
 
-	for_each_child_of_node_scoped(np, child) {
-		++nfuncs;
-		ngroups += of_get_child_count(child);
+	flat_funcs = imx1_pinctrl_dt_is_flat_functions(np);
+	if (flat_funcs) {
+		nfuncs = 1;
+		ngroups = of_get_child_count(np);
+	} else {
+		for_each_child_of_node_scoped(np, child) {
+			++nfuncs;
+			ngroups += of_get_child_count(child);
+		}
 	}
 
 	if (!nfuncs) {
@@ -574,10 +604,14 @@ static int imx1_pinctrl_parse_dt(struct platform_device *pdev,
 	if (!info->functions || !info->groups)
 		return -ENOMEM;
 
-	for_each_child_of_node_scoped(np, child) {
-		ret = imx1_pinctrl_parse_functions(child, info, ifunc++);
-		if (ret == -ENOMEM)
-			return -ENOMEM;
+	if (flat_funcs) {
+		imx1_pinctrl_parse_functions(np, info, 0);
+	} else {
+		for_each_child_of_node_scoped(np, child) {
+			ret = imx1_pinctrl_parse_functions(child, info, ifunc++);
+			if (ret == -ENOMEM)
+				return -ENOMEM;
+		}
 	}
 
 	return 0;
diff --git a/drivers/pinctrl/intel/pinctrl-intel.c b/drivers/pinctrl/intel/pinctrl-intel.c
index 97bf5ec78db4e..2e2526e01d583 100644
--- a/drivers/pinctrl/intel/pinctrl-intel.c
+++ b/drivers/pinctrl/intel/pinctrl-intel.c
@@ -1556,13 +1556,13 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl,
 				   struct intel_community *community,
 				   unsigned short capability_offset)
 {
-	void __iomem *base = community->regs + capability_offset + 4;
 	static const struct pwm_lpss_boardinfo info = {
 		.clk_rate = 19200000,
 		.npwm = 1,
 		.base_unit_bits = 22,
 	};
 	struct pwm_chip *chip;
+	void __iomem *base;
 
 	if (!(community->features & PINCTRL_FEATURE_PWM))
 		return 0;
@@ -1570,6 +1570,7 @@ static int intel_pinctrl_probe_pwm(struct intel_pinctrl *pctrl,
 	if (!IS_REACHABLE(CONFIG_PWM_LPSS))
 		return 0;
 
+	base = community->regs + capability_offset + 4;
 	chip = devm_pwm_lpss_probe(pctrl->dev, base, &info);
 	return PTR_ERR_OR_ZERO(chip);
 }
diff --git a/drivers/pinctrl/mediatek/mtk-eint.c b/drivers/pinctrl/mediatek/mtk-eint.c
index 2a3c04eedc5f3..47ac92ea98c2c 100644
--- a/drivers/pinctrl/mediatek/mtk-eint.c
+++ b/drivers/pinctrl/mediatek/mtk-eint.c
@@ -246,7 +246,7 @@ static int mtk_eint_irq_set_wake(struct irq_data *d, unsigned int on)
 }
 
 static void mtk_eint_chip_write_mask(const struct mtk_eint *eint,
-				     void __iomem *base, unsigned int **buf)
+				     unsigned int **buf)
 {
 	int inst, port, port_num;
 	void __iomem *reg;
@@ -425,7 +425,7 @@ static void mtk_eint_irq_handler(struct irq_desc *desc)
 
 int mtk_eint_do_suspend(struct mtk_eint *eint)
 {
-	mtk_eint_chip_write_mask(eint, eint->base, eint->wake_mask);
+	mtk_eint_chip_write_mask(eint, eint->wake_mask);
 
 	return 0;
 }
@@ -433,7 +433,7 @@ EXPORT_SYMBOL_GPL(mtk_eint_do_suspend);
 
 int mtk_eint_do_resume(struct mtk_eint *eint)
 {
-	mtk_eint_chip_write_mask(eint, eint->base, eint->cur_mask);
+	mtk_eint_chip_write_mask(eint, eint->cur_mask);
 
 	return 0;
 }
diff --git a/drivers/pinctrl/mediatek/pinctrl-airoha.c b/drivers/pinctrl/mediatek/pinctrl-airoha.c
index 995ba6175c950..e5a6a60fb3c66 100644
--- a/drivers/pinctrl/mediatek/pinctrl-airoha.c
+++ b/drivers/pinctrl/mediatek/pinctrl-airoha.c
@@ -2811,7 +2811,7 @@ static int airoha_pinconf_group_get(struct pinctrl_dev *pctrl_dev,
 				    unsigned int group, unsigned long *config)
 {
 	struct airoha_pinctrl *pinctrl = pinctrl_dev_get_drvdata(pctrl_dev);
-	u32 cur_config = 0;
+	unsigned long cur_config = 0;
 	int i;
 
 	for (i = 0; i < pinctrl->grps[group].npins; i++) {
diff --git a/drivers/pinctrl/mediatek/pinctrl-moore.c b/drivers/pinctrl/mediatek/pinctrl-moore.c
index 70f608347a5f6..17e30f83dc197 100644
--- a/drivers/pinctrl/mediatek/pinctrl-moore.c
+++ b/drivers/pinctrl/mediatek/pinctrl-moore.c
@@ -402,7 +402,8 @@ static int mtk_pinconf_group_get(struct pinctrl_dev *pctldev,
 				 unsigned int group, unsigned long *config)
 {
 	const unsigned int *pins;
-	unsigned int i, npins, old = 0;
+	unsigned int i, npins;
+	unsigned long old = 0;
 	int ret;
 
 	ret = pinctrl_generic_get_group_pins(pctldev, group, &pins, &npins);
@@ -520,6 +521,23 @@ static int mtk_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio,
 	return pinctrl_gpio_direction_output(chip, gpio);
 }
 
+static int mtk_gpio_get_direction(struct gpio_chip *chip, unsigned int offset)
+{
+	struct mtk_pinctrl *hw = gpiochip_get_data(chip);
+	const struct mtk_pin_desc *desc;
+	int ret, dir;
+
+	desc = (const struct mtk_pin_desc *)&hw->soc->pins[offset];
+	if (!desc->name)
+		return -ENOTSUPP;
+
+	ret = mtk_hw_get_value(hw, desc, PINCTRL_PIN_REG_DIR, &dir);
+	if (ret)
+		return ret;
+
+	return dir ? GPIO_LINE_DIRECTION_OUT : GPIO_LINE_DIRECTION_IN;
+}
+
 static int mtk_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
 {
 	struct mtk_pinctrl *hw = gpiochip_get_data(chip);
@@ -566,6 +584,7 @@ static int mtk_build_gpiochip(struct mtk_pinctrl *hw)
 	chip->parent		= hw->dev;
 	chip->request		= gpiochip_generic_request;
 	chip->free		= gpiochip_generic_free;
+	chip->get_direction	= mtk_gpio_get_direction;
 	chip->direction_input	= pinctrl_gpio_direction_input;
 	chip->direction_output	= mtk_gpio_direction_output;
 	chip->get		= mtk_gpio_get;
diff --git a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
index 3f518dce6d23f..dd2c8aa039385 100644
--- a/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
+++ b/drivers/pinctrl/mediatek/pinctrl-mtk-common.c
@@ -802,20 +802,27 @@ static const struct pinmux_ops mtk_pmx_ops = {
 	.get_function_name	= mtk_pmx_get_func_name,
 	.get_function_groups	= mtk_pmx_get_func_groups,
 	.set_mux		= mtk_pmx_set_mux,
-	.gpio_set_direction	= mtk_pmx_gpio_set_direction,
 	.gpio_request_enable	= mtk_pmx_gpio_request_enable,
 };
 
+static int mtk_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
+{
+	struct mtk_pinctrl *pctl = gpiochip_get_data(chip);
+
+	return mtk_pmx_gpio_set_direction(pctl->pctl_dev, NULL, offset, true);
+}
+
 static int mtk_gpio_direction_output(struct gpio_chip *chip,
 					unsigned offset, int value)
 {
+	struct mtk_pinctrl *pctl = gpiochip_get_data(chip);
 	int ret;
 
 	ret = mtk_gpio_set(chip, offset, value);
 	if (ret)
 		return ret;
 
-	return pinctrl_gpio_direction_output(chip, offset);
+	return mtk_pmx_gpio_set_direction(pctl->pctl_dev, NULL, offset, false);
 }
 
 static int mtk_gpio_get_direction(struct gpio_chip *chip, unsigned offset)
@@ -895,7 +902,7 @@ static const struct gpio_chip mtk_gpio_chip = {
 	.request		= gpiochip_generic_request,
 	.free			= gpiochip_generic_free,
 	.get_direction		= mtk_gpio_get_direction,
-	.direction_input	= pinctrl_gpio_direction_input,
+	.direction_input	= mtk_gpio_direction_input,
 	.direction_output	= mtk_gpio_direction_output,
 	.get			= mtk_gpio_get,
 	.set			= mtk_gpio_set,
diff --git a/drivers/pinctrl/mediatek/pinctrl-paris.c b/drivers/pinctrl/mediatek/pinctrl-paris.c
index 6bf37d8085fae..23f04b24fd65e 100644
--- a/drivers/pinctrl/mediatek/pinctrl-paris.c
+++ b/drivers/pinctrl/mediatek/pinctrl-paris.c
@@ -771,7 +771,6 @@ static const struct pinmux_ops mtk_pmxops = {
 	.get_function_name	= mtk_pmx_get_func_name,
 	.get_function_groups	= mtk_pmx_get_func_groups,
 	.set_mux		= mtk_pmx_set_mux,
-	.gpio_set_direction	= mtk_pinmux_gpio_set_direction,
 	.gpio_request_enable	= mtk_pinmux_gpio_request_enable,
 };
 
@@ -886,19 +885,22 @@ static int mtk_gpio_set(struct gpio_chip *chip, unsigned int gpio, int value)
 
 static int mtk_gpio_direction_input(struct gpio_chip *chip, unsigned int gpio)
 {
-	return pinctrl_gpio_direction_input(chip, gpio);
+	struct mtk_pinctrl *hw = gpiochip_get_data(chip);
+
+	return mtk_pinmux_gpio_set_direction(hw->pctrl, NULL, gpio, true);
 }
 
 static int mtk_gpio_direction_output(struct gpio_chip *chip, unsigned int gpio,
 				     int value)
 {
+	struct mtk_pinctrl *hw = gpiochip_get_data(chip);
 	int ret;
 
 	ret = mtk_gpio_set(chip, gpio, value);
 	if (ret)
 		return ret;
 
-	return pinctrl_gpio_direction_output(chip, gpio);
+	return mtk_pinmux_gpio_set_direction(hw->pctrl, NULL, gpio, false);
 }
 
 static int mtk_gpio_to_irq(struct gpio_chip *chip, unsigned int offset)
diff --git a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c
index e2293a872dcb7..35d27626a336b 100644
--- a/drivers/pinctrl/meson/pinctrl-amlogic-a4.c
+++ b/drivers/pinctrl/meson/pinctrl-amlogic-a4.c
@@ -292,7 +292,7 @@ static int aml_calc_reg_and_bit(struct pinctrl_gpio_range *range,
 static int aml_pinconf_get_pull(struct aml_pinctrl *info, unsigned int pin)
 {
 	struct pinctrl_gpio_range *range =
-			 pinctrl_find_gpio_range_from_pin(info->pctl, pin);
+			 pinctrl_find_gpio_range_from_pin_nolock(info->pctl, pin);
 	struct aml_gpio_bank *bank = gpio_chip_to_bank(range->gc);
 	unsigned int reg, bit, val;
 	int ret, conf;
@@ -326,7 +326,7 @@ static int aml_pinconf_get_drive_strength(struct aml_pinctrl *info,
 					  u16 *drive_strength_ua)
 {
 	struct pinctrl_gpio_range *range =
-			 pinctrl_find_gpio_range_from_pin(info->pctl, pin);
+			 pinctrl_find_gpio_range_from_pin_nolock(info->pctl, pin);
 	struct aml_gpio_bank *bank = gpio_chip_to_bank(range->gc);
 	unsigned int reg, bit;
 	unsigned int val;
@@ -365,7 +365,7 @@ static int aml_pinconf_get_gpio_bit(struct aml_pinctrl *info,
 				    unsigned int reg_type)
 {
 	struct pinctrl_gpio_range *range =
-			 pinctrl_find_gpio_range_from_pin(info->pctl, pin);
+			 pinctrl_find_gpio_range_from_pin_nolock(info->pctl, pin);
 	struct aml_gpio_bank *bank = gpio_chip_to_bank(range->gc);
 	unsigned int reg, bit, val;
 	int ret;
diff --git a/drivers/pinctrl/pinconf-generic.c b/drivers/pinctrl/pinconf-generic.c
index 64ed283097887..95c803c40a7d4 100644
--- a/drivers/pinctrl/pinconf-generic.c
+++ b/drivers/pinctrl/pinconf-generic.c
@@ -225,10 +225,9 @@ static int parse_fw_cfg(struct fwnode_handle *fwnode,
 			unsigned int count, unsigned long *cfg,
 			unsigned int *ncfg)
 {
-	unsigned long *properties;
 	int i, test;
 
-	properties = bitmap_zalloc(count, GFP_KERNEL);
+	unsigned long *properties __free(bitmap) = bitmap_zalloc(count, GFP_KERNEL);
 
 	for (i = 0; i < count; i++) {
 		u32 val;
@@ -263,7 +262,6 @@ static int parse_fw_cfg(struct fwnode_handle *fwnode,
 			if (ret) {
 				pr_err("%pfw: conflicting setting detected for %s\n",
 				       fwnode, par->property);
-				bitmap_free(properties);
 				return -EINVAL;
 			}
 		}
@@ -295,7 +293,6 @@ static int parse_fw_cfg(struct fwnode_handle *fwnode,
 		pr_err("%pfw: cannot have multiple drive configurations\n",
 		       fwnode);
 
-	bitmap_free(properties);
 	return 0;
 }
 
@@ -419,7 +416,7 @@ int pinconf_generic_parse_dt_config(struct device_node *np,
 	 * Now limit the number of configs to the real number of
 	 * found properties.
 	 */
-	*configs = kmemdup(cfg, ncfg * sizeof(unsigned long), GFP_KERNEL);
+	*configs = kmemdup_array(cfg, ncfg, sizeof(unsigned long), GFP_KERNEL);
 	if (!*configs) {
 		ret = -ENOMEM;
 		goto out;
diff --git a/drivers/pinctrl/pinconf.h b/drivers/pinctrl/pinconf.h
index 659a781e20911..fa8fb0d290d1d 100644
--- a/drivers/pinctrl/pinconf.h
+++ b/drivers/pinctrl/pinconf.h
@@ -172,6 +172,13 @@ int pinctrl_generic_pins_function_dt_node_to_map(struct pinctrl_dev *pctldev,
 						 struct device_node *np,
 						 struct pinctrl_map **maps,
 						 unsigned int *num_maps);
+
+int pinctrl_generic_to_map(struct pinctrl_dev *pctldev, struct device_node *parent,
+			   struct device_node *np, struct pinctrl_map **maps,
+			   unsigned int *num_maps, unsigned int *num_reserved_maps,
+			   const char **group_name, unsigned int ngroups,
+			   const char **functions, unsigned int *pins,
+			   unsigned int npins);
 #else
 static inline int
 pinctrl_generic_pins_function_dt_node_to_map(struct pinctrl_dev *pctldev,
@@ -181,4 +188,15 @@ pinctrl_generic_pins_function_dt_node_to_map(struct pinctrl_dev *pctldev,
 {
 	return -ENOTSUPP;
 }
+
+static inline int
+pinctrl_generic_to_map(struct pinctrl_dev *pctldev, struct device_node *parent,
+		       struct device_node *np, struct pinctrl_map **maps,
+		       unsigned int *num_maps, unsigned int *num_reserved_maps,
+		       const char **group_name, unsigned int ngroups,
+		       const char **functions, unsigned int *pins,
+		       void *function_data)
+{
+	return -ENOTSUPP;
+}
 #endif
diff --git a/drivers/pinctrl/pinctrl-amd.c b/drivers/pinctrl/pinctrl-amd.c
index e3128b0045d22..64315b0edf2a6 100644
--- a/drivers/pinctrl/pinctrl-amd.c
+++ b/drivers/pinctrl/pinctrl-amd.c
@@ -26,6 +26,7 @@
 #include <linux/interrupt.h>
 #include <linux/bitops.h>
 #include <linux/pinctrl/pinconf.h>
+#include <linux/dmi.h>
 #include <linux/pinctrl/pinconf-generic.h>
 #include <linux/pinctrl/pinmux.h>
 #include <linux/string_choices.h>
@@ -39,6 +40,39 @@
 static struct amd_gpio *pinctrl_dev;
 #endif
 
+static const struct dmi_system_id amd_gpio_quirk_yoga7_14agp11[] = {
+	{
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "83TD"),
+			DMI_MATCH(DMI_BOARD_NAME, "LNVNB161216"),
+		},
+	},
+	{ }
+};
+
+static void amd_gpio_apply_quirks(struct amd_gpio *gpio_dev)
+{
+	const unsigned int pin = 157; /* WACF2200 GpioInt per ACPI _CRS */
+	unsigned long flags;
+	u32 reg;
+
+	if (!dmi_check_system(amd_gpio_quirk_yoga7_14agp11))
+		return;
+	if (pin >= gpio_dev->gc.ngpio)
+		return;
+
+	raw_spin_lock_irqsave(&gpio_dev->lock, flags);
+	reg = readl(gpio_dev->base + pin * 4);
+	reg |= BIT(INTERRUPT_ENABLE_OFF) | BIT(INTERRUPT_MASK_OFF);
+	writel(reg, gpio_dev->base + pin * 4);
+	raw_spin_unlock_irqrestore(&gpio_dev->lock, flags);
+
+	dev_info(&gpio_dev->pdev->dev,
+		 "Enabled IRQ for GPIO %u (Yoga 7 14AGP11 touchscreen)\n",
+		 pin);
+}
+
 static int amd_gpio_get_direction(struct gpio_chip *gc, unsigned offset)
 {
 	unsigned long flags;
@@ -1219,6 +1253,7 @@ static int amd_gpio_probe(struct platform_device *pdev)
 
 	/* Disable and mask interrupts */
 	amd_gpio_irq_init(gpio_dev);
+	amd_gpio_apply_quirks(gpio_dev);
 
 	girq = &gpio_dev->gc.irq;
 	gpio_irq_chip_set_chip(girq, &amd_gpio_irqchip);
diff --git a/drivers/pinctrl/pinctrl-equilibrium.c b/drivers/pinctrl/pinctrl-equilibrium.c
index ba1c867b7b891..349eb944b4ac8 100644
--- a/drivers/pinctrl/pinctrl-equilibrium.c
+++ b/drivers/pinctrl/pinctrl-equilibrium.c
@@ -532,8 +532,9 @@ static int eqbr_pinconf_set(struct pinctrl_dev *pctldev, unsigned int pin,
 static int eqbr_pinconf_group_get(struct pinctrl_dev *pctldev,
 				  unsigned int group, unsigned long *config)
 {
-	unsigned int i, npins, old = 0;
 	const unsigned int *pins;
+	unsigned int i, npins;
+	unsigned long old = 0;
 	int ret;
 
 	ret = pinctrl_generic_get_group_pins(pctldev, group, &pins, &npins);
diff --git a/drivers/pinctrl/pinctrl-generic-mux.c b/drivers/pinctrl/pinctrl-generic-mux.c
new file mode 100644
index 0000000000000..da5a5ec015833
--- /dev/null
+++ b/drivers/pinctrl/pinctrl-generic-mux.c
@@ -0,0 +1,184 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Generic Pin Control Driver for Board-Level Mux Chips
+ * Copyright 2026 NXP
+ */
+
+#include <linux/cleanup.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/mutex.h>
+#include <linux/mux/consumer.h>
+#include <linux/platform_device.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
+#include <linux/slab.h>
+
+#include "core.h"
+#include "pinconf.h"
+#include "pinmux.h"
+#include "pinctrl-utils.h"
+
+struct mux_pin_function {
+	struct mux_state *mux_state;
+};
+
+struct mux_pinctrl {
+	struct device *dev;
+	struct pinctrl_dev *pctl;
+
+	/* mutex protect [pinctrl|pinmux]_generic functions */
+	struct mutex lock;
+};
+
+static int
+mux_pinmux_dt_node_to_map(struct pinctrl_dev *pctldev,
+			  struct device_node *np_config,
+			  struct pinctrl_map **maps, unsigned int *num_maps)
+{
+	unsigned int num_reserved_maps = 0;
+	struct mux_pin_function *function;
+	const char **group_names;
+	int ret;
+
+	function = devm_kzalloc(pctldev->dev, sizeof(*function), GFP_KERNEL);
+	if (!function)
+		return -ENOMEM;
+
+	group_names = devm_kcalloc(pctldev->dev, 1, sizeof(*group_names), GFP_KERNEL);
+	if (!group_names)
+		return -ENOMEM;
+
+	function->mux_state = devm_mux_state_get_from_np(pctldev->dev, NULL, np_config);
+	if (IS_ERR(function->mux_state))
+		return PTR_ERR(function->mux_state);
+
+	ret = pinctrl_generic_to_map(pctldev, np_config, np_config, maps,
+				     num_maps, &num_reserved_maps, group_names,
+				     0, &np_config->name, NULL, 0);
+
+	if (ret)
+		return ret;
+
+	ret = pinmux_generic_add_function(pctldev, np_config->name, group_names,
+					  1, function);
+	if (ret < 0) {
+		pinctrl_utils_free_map(pctldev, *maps, *num_maps);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct pinctrl_ops mux_pinctrl_ops = {
+	.get_groups_count = pinctrl_generic_get_group_count,
+	.get_group_name = pinctrl_generic_get_group_name,
+	.get_group_pins = pinctrl_generic_get_group_pins,
+	.dt_node_to_map = mux_pinmux_dt_node_to_map,
+	.dt_free_map = pinctrl_utils_free_map,
+};
+
+static int mux_pinmux_set_mux(struct pinctrl_dev *pctldev,
+			      unsigned int func_selector,
+			      unsigned int group_selector)
+{
+	struct mux_pinctrl *mpctl = pinctrl_dev_get_drvdata(pctldev);
+	const struct function_desc *function;
+	struct mux_pin_function *func;
+	int ret;
+
+	guard(mutex)(&mpctl->lock);
+
+	function = pinmux_generic_get_function(pctldev, func_selector);
+	func = function->data;
+
+	ret = mux_state_select(func->mux_state);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void mux_pinmux_release_mux(struct pinctrl_dev *pctldev,
+				   unsigned int func_selector,
+				   unsigned int group_selector)
+{
+	struct mux_pinctrl *mpctl = pinctrl_dev_get_drvdata(pctldev);
+	const struct function_desc *function;
+	struct mux_pin_function *func;
+
+	guard(mutex)(&mpctl->lock);
+
+	function = pinmux_generic_get_function(pctldev, func_selector);
+	func = function->data;
+
+	mux_state_deselect(func->mux_state);
+}
+
+static const struct pinmux_ops mux_pinmux_ops = {
+	.get_functions_count = pinmux_generic_get_function_count,
+	.get_function_name = pinmux_generic_get_function_name,
+	.get_function_groups = pinmux_generic_get_function_groups,
+	.set_mux = mux_pinmux_set_mux,
+	.release_mux = mux_pinmux_release_mux,
+};
+
+static int mux_pinctrl_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mux_pinctrl *mpctl;
+	struct pinctrl_desc *pctl_desc;
+	int ret;
+
+	mpctl = devm_kzalloc(dev, sizeof(*mpctl), GFP_KERNEL);
+	if (!mpctl)
+		return -ENOMEM;
+
+	mpctl->dev = dev;
+
+	platform_set_drvdata(pdev, mpctl);
+
+	pctl_desc = devm_kzalloc(dev, sizeof(*pctl_desc), GFP_KERNEL);
+	if (!pctl_desc)
+		return -ENOMEM;
+
+	ret = devm_mutex_init(dev, &mpctl->lock);
+	if (ret)
+		return ret;
+
+	pctl_desc->name = dev_name(dev);
+	pctl_desc->owner = THIS_MODULE;
+	pctl_desc->pctlops = &mux_pinctrl_ops;
+	pctl_desc->pmxops = &mux_pinmux_ops;
+
+	ret = devm_pinctrl_register_and_init(dev, pctl_desc, mpctl,
+					     &mpctl->pctl);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to register pinctrl.\n");
+
+	ret = pinctrl_enable(mpctl->pctl);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to enable pinctrl.\n");
+
+	return 0;
+}
+
+static const struct of_device_id mux_pinctrl_of_match[] = {
+	{ .compatible = "pinctrl-multiplexer" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, mux_pinctrl_of_match);
+
+static struct platform_driver mux_pinctrl_driver = {
+	.driver = {
+		.name = "generic-pinctrl-mux",
+		.of_match_table = mux_pinctrl_of_match,
+	},
+	.probe = mux_pinctrl_probe,
+};
+module_platform_driver(mux_pinctrl_driver);
+
+MODULE_AUTHOR("Frank Li <Frank.Li@nxp.com>");
+MODULE_DESCRIPTION("Generic Pin Control Driver for Board-Level Mux Chips");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/pinctrl-generic.c b/drivers/pinctrl/pinctrl-generic.c
index efb39c6a67033..e4cd16ce2bda3 100644
--- a/drivers/pinctrl/pinctrl-generic.c
+++ b/drivers/pinctrl/pinctrl-generic.c
@@ -17,29 +17,18 @@
 #include "pinctrl-utils.h"
 #include "pinmux.h"
 
-static int pinctrl_generic_pins_function_dt_subnode_to_map(struct pinctrl_dev *pctldev,
-							   struct device_node *parent,
-							   struct device_node *np,
-							   struct pinctrl_map **maps,
-							   unsigned int *num_maps,
-							   unsigned int *num_reserved_maps,
-							   const char **group_names,
-							   unsigned int ngroups)
+int pinctrl_generic_to_map(struct pinctrl_dev *pctldev, struct device_node *parent,
+			   struct device_node *np, struct pinctrl_map **maps,
+			   unsigned int *num_maps, unsigned int *num_reserved_maps,
+			   const char **group_names, unsigned int ngroups,
+			   const char **functions, unsigned int *pins,
+			   unsigned int npins)
 {
 	struct device *dev = pctldev->dev;
-	const char **functions;
+	unsigned int num_configs;
 	const char *group_name;
 	unsigned long *configs;
-	unsigned int num_configs, pin, *pins;
-	int npins, ret, reserve = 1;
-
-	npins = of_property_count_u32_elems(np, "pins");
-
-	if (npins < 1) {
-		dev_err(dev, "invalid pinctrl group %pOFn.%pOFn %d\n",
-			parent, np, npins);
-		return npins;
-	}
+	int ret, reserve = 1;
 
 	group_name = devm_kasprintf(dev, GFP_KERNEL, "%pOFn.%pOFn", parent, np);
 	if (!group_name)
@@ -47,26 +36,6 @@ static int pinctrl_generic_pins_function_dt_subnode_to_map(struct pinctrl_dev *p
 
 	group_names[ngroups] = group_name;
 
-	pins = devm_kcalloc(dev, npins, sizeof(*pins), GFP_KERNEL);
-	if (!pins)
-		return -ENOMEM;
-
-	functions = devm_kcalloc(dev, npins, sizeof(*functions), GFP_KERNEL);
-	if (!functions)
-		return -ENOMEM;
-
-	for (int i = 0; i < npins; i++) {
-		ret = of_property_read_u32_index(np, "pins", i, &pin);
-		if (ret)
-			return ret;
-
-		pins[i] = pin;
-
-		ret = of_property_read_string(np, "function", &functions[i]);
-		if (ret)
-			return ret;
-	}
-
 	ret = pinctrl_utils_reserve_map(pctldev, maps, num_reserved_maps, num_maps, reserve);
 	if (ret)
 		return ret;
@@ -102,6 +71,54 @@ static int pinctrl_generic_pins_function_dt_subnode_to_map(struct pinctrl_dev *p
 
 	return 0;
 };
+EXPORT_SYMBOL_GPL(pinctrl_generic_to_map);
+
+static int pinctrl_generic_pins_function_dt_subnode_to_map(struct pinctrl_dev *pctldev,
+							   struct device_node *parent,
+							   struct device_node *np,
+							   struct pinctrl_map **maps,
+							   unsigned int *num_maps,
+							   unsigned int *num_reserved_maps,
+							   const char **group_names,
+							   unsigned int ngroups)
+{
+	struct device *dev = pctldev->dev;
+	unsigned int pin, *pins;
+	const char **functions;
+	int npins, ret;
+
+	npins = of_property_count_u32_elems(np, "pins");
+
+	if (npins < 1) {
+		dev_err(dev, "invalid pinctrl group %pOFn.%pOFn %d\n",
+			parent, np, npins);
+		return npins;
+	}
+
+	pins = devm_kcalloc(dev, npins, sizeof(*pins), GFP_KERNEL);
+	if (!pins)
+		return -ENOMEM;
+
+	functions = devm_kcalloc(dev, npins, sizeof(*functions), GFP_KERNEL);
+	if (!functions)
+		return -ENOMEM;
+
+	for (int i = 0; i < npins; i++) {
+		ret = of_property_read_u32_index(np, "pins", i, &pin);
+		if (ret)
+			return ret;
+
+		pins[i] = pin;
+
+		ret = of_property_read_string(np, "function", &functions[i]);
+		if (ret)
+			return ret;
+	}
+
+	return pinctrl_generic_to_map(pctldev, parent, np, maps, num_maps,
+				      num_reserved_maps, group_names, ngroups,
+				      functions, pins, npins);
+}
 
 /*
  * For platforms that do not define groups or functions in the driver, but
diff --git a/drivers/pinctrl/pinctrl-ingenic.c b/drivers/pinctrl/pinctrl-ingenic.c
index c7f14546de052..29d7f4e54bc7b 100644
--- a/drivers/pinctrl/pinctrl-ingenic.c
+++ b/drivers/pinctrl/pinctrl-ingenic.c
@@ -4334,7 +4334,8 @@ static int ingenic_pinconf_group_get(struct pinctrl_dev *pctldev,
 		unsigned int group, unsigned long *config)
 {
 	const unsigned int *pins;
-	unsigned int i, npins, old = 0;
+	unsigned int i, npins;
+	unsigned long old = 0;
 	int ret;
 
 	ret = pinctrl_generic_get_group_pins(pctldev, group, &pins, &npins);
diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c
index 2f14c7f9c95a2..7e0fcd45fd265 100644
--- a/drivers/pinctrl/pinctrl-rockchip.c
+++ b/drivers/pinctrl/pinctrl-rockchip.c
@@ -5303,6 +5303,7 @@ static const struct of_device_id rockchip_pinctrl_dt_match[] = {
 		.data = &rk3588_pin_ctrl },
 	{},
 };
+MODULE_DEVICE_TABLE(of, rockchip_pinctrl_dt_match);
 
 static struct platform_driver rockchip_pinctrl_driver = {
 	.probe		= rockchip_pinctrl_probe,
@@ -5329,4 +5330,3 @@ module_exit(rockchip_pinctrl_drv_unregister);
 MODULE_DESCRIPTION("ROCKCHIP Pin Controller Driver");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:pinctrl-rockchip");
-MODULE_DEVICE_TABLE(of, rockchip_pinctrl_dt_match);
diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c
index 7e022a46ee422..4d5f85b7e6bbe 100644
--- a/drivers/pinctrl/pinctrl-single.c
+++ b/drivers/pinctrl/pinctrl-single.c
@@ -619,8 +619,9 @@ static int pcs_pinconf_set(struct pinctrl_dev *pctldev,
 static int pcs_pinconf_group_get(struct pinctrl_dev *pctldev,
 				unsigned group, unsigned long *config)
 {
+	unsigned long old = 0;
 	const unsigned *pins;
-	unsigned npins, old = 0;
+	unsigned npins;
 	int i, ret;
 
 	ret = pinctrl_generic_get_group_pins(pctldev, group, &pins, &npins);
diff --git a/drivers/pinctrl/pinmux.c b/drivers/pinctrl/pinmux.c
index 3a8dd184ba3d6..c705bc182266c 100644
--- a/drivers/pinctrl/pinmux.c
+++ b/drivers/pinctrl/pinmux.c
@@ -517,6 +517,7 @@ void pinmux_disable_setting(const struct pinctrl_setting *setting)
 {
 	struct pinctrl_dev *pctldev = setting->pctldev;
 	const struct pinctrl_ops *pctlops = pctldev->desc->pctlops;
+	const struct pinmux_ops *ops = pctldev->desc->pmxops;
 	int ret = 0;
 	const unsigned int *pins = NULL;
 	unsigned int num_pins = 0;
@@ -563,6 +564,10 @@ void pinmux_disable_setting(const struct pinctrl_setting *setting)
 				 pins[i], desc->name, gname);
 		}
 	}
+
+	if (ops->release_mux)
+		ops->release_mux(pctldev, setting->data.mux.func,
+				 setting->data.mux.group);
 }
 
 #ifdef CONFIG_DEBUG_FS
diff --git a/drivers/pinctrl/qcom/Kconfig b/drivers/pinctrl/qcom/Kconfig
index 80af372a11471..a09e840a01c66 100644
--- a/drivers/pinctrl/qcom/Kconfig
+++ b/drivers/pinctrl/qcom/Kconfig
@@ -6,6 +6,7 @@ config PINCTRL_MSM
 	depends on GPIOLIB
 	# OF for pinconf_generic_dt_node_to_map_group() from GENERIC_PINCONF
 	depends on OF
+	default ARCH_QCOM
 	select QCOM_SCM
 	select PINMUX
 	select GENERIC_PINMUX_FUNCTIONS
@@ -49,7 +50,7 @@ config PINCTRL_QCOM_SSBI_PMIC
 	 devices are pm8058 and pm8921.
 
 config PINCTRL_LPASS_LPI
-	tristate "Qualcomm Technologies Inc LPASS LPI pin controller driver"
+	tristate "Qualcomm LPASS LPI pin controller driver"
 	select PINMUX
 	select PINCONF
 	select GENERIC_PINCONF
@@ -61,7 +62,7 @@ config PINCTRL_LPASS_LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SoCs.
 
 config PINCTRL_MILOS_LPASS_LPI
-	tristate "Qualcomm Technologies Inc Milos LPASS LPI pin controller driver"
+	tristate "Qualcomm Milos LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
 	help
@@ -71,7 +72,7 @@ config PINCTRL_MILOS_LPASS_LPI
 	  platform.
 
 config PINCTRL_SC7280_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SC7280 and SM8350 LPASS LPI pin controller driver"
+	tristate "Qualcomm SC7280 and SM8350 LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
 	help
@@ -81,7 +82,7 @@ config PINCTRL_SC7280_LPASS_LPI
 	  and SM8350 platforms.
 
 config PINCTRL_SDM660_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SDM660 LPASS LPI pin controller driver"
+	tristate "Qualcomm SDM660 LPASS LPI pin controller driver"
 	depends on GPIOLIB
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
@@ -91,7 +92,7 @@ config PINCTRL_SDM660_LPASS_LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SDM660 platform.
 
 config PINCTRL_SM4250_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SM4250 LPASS LPI pin controller driver"
+	tristate "Qualcomm SM4250 LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
 	help
@@ -100,7 +101,7 @@ config PINCTRL_SM4250_LPASS_LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SM4250 platform.
 
 config PINCTRL_SDM670_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SDM670 LPASS LPI pin controller driver"
+	tristate "Qualcomm SDM670 LPASS LPI pin controller driver"
 	depends on GPIOLIB
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
@@ -110,7 +111,7 @@ config PINCTRL_SDM670_LPASS_LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SDM670 platform.
 
 config PINCTRL_SM6115_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SM6115 LPASS LPI pin controller driver"
+	tristate "Qualcomm SM6115 LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
 	help
@@ -118,8 +119,17 @@ config PINCTRL_SM6115_LPASS_LPI
 	  Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SM6115 platform.
 
+config PINCTRL_SM6350_LPASS_LPI
+	tristate "Qualcomm SM6350 LPASS LPI pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	depends on PINCTRL_LPASS_LPI
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
+	  Qualcomm Technologies Inc LPASS (Low Power Audio SubSystem) LPI
+	  (Low Power Island) found on the Qualcomm Technologies Inc SM6350 platform.
+
 config PINCTRL_SM8250_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SM8250 LPASS LPI pin controller driver"
+	tristate "Qualcomm SM8250 LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
 	help
@@ -128,7 +138,7 @@ config PINCTRL_SM8250_LPASS_LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SM8250 platform.
 
 config PINCTRL_SM8450_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SM8450 LPASS LPI pin controller driver"
+	tristate "Qualcomm SM8450 LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
 	help
@@ -137,7 +147,7 @@ config PINCTRL_SM8450_LPASS_LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SM8450 platform.
 
 config PINCTRL_SC8280XP_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SC8280XP LPASS LPI pin controller driver"
+	tristate "Qualcomm SC8280XP LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
 	help
@@ -146,7 +156,7 @@ config PINCTRL_SC8280XP_LPASS_LPI
 	  (Low Power Island) found on the Qualcomm Technologies Inc SC8280XP platform.
 
 config PINCTRL_SM8550_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SM8550 LPASS LPI pin controller driver"
+	tristate "Qualcomm SM8550 LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
 	help
@@ -156,7 +166,7 @@ config PINCTRL_SM8550_LPASS_LPI
 	  platform.
 
 config PINCTRL_SM8650_LPASS_LPI
-	tristate "Qualcomm Technologies Inc SM8650 LPASS LPI pin controller driver"
+	tristate "Qualcomm SM8650 LPASS LPI pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	depends on PINCTRL_LPASS_LPI
 	help
diff --git a/drivers/pinctrl/qcom/Kconfig.msm b/drivers/pinctrl/qcom/Kconfig.msm
index 836cdeca1006f..65bc7424065ab 100644
--- a/drivers/pinctrl/qcom/Kconfig.msm
+++ b/drivers/pinctrl/qcom/Kconfig.msm
@@ -4,6 +4,7 @@ if PINCTRL_MSM
 config PINCTRL_APQ8064
 	tristate "Qualcomm APQ8064 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm APQ8064 platform.
@@ -11,13 +12,15 @@ config PINCTRL_APQ8064
 config PINCTRL_APQ8084
 	tristate "Qualcomm APQ8084 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm APQ8084 platform.
 
 config PINCTRL_ELIZA
-	tristate "Qualcomm Technologies Inc Eliza pin controller driver"
+	tristate "Qualcomm Eliza pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc Top Level Mode Multiplexer block (TLMM)
@@ -26,8 +29,9 @@ config PINCTRL_ELIZA
 	  If unsure, say N.
 
 config PINCTRL_GLYMUR
-	tristate "Qualcomm Technologies Inc Glymur pin controller driver"
+	tristate "Qualcomm Glymur pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc Top Level Mode Multiplexer block (TLMM)
@@ -36,8 +40,9 @@ config PINCTRL_GLYMUR
 	  If unsure, say N.
 
 config PINCTRL_HAWI
-	tristate "Qualcomm Technologies Inc Hawi pin controller driver"
+	tristate "Qualcomm Hawi pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc Top Level Mode Multiplexer block (TLMM)
@@ -48,13 +53,15 @@ config PINCTRL_HAWI
 config PINCTRL_IPQ4019
 	tristate "Qualcomm IPQ4019 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm IPQ4019 platform.
 
 config PINCTRL_IPQ5018
-	tristate "Qualcomm Technologies, Inc. IPQ5018 pin controller driver"
+	tristate "Qualcomm IPQ5018 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for
 	  the Qualcomm Technologies Inc. TLMM block found on the
@@ -64,29 +71,33 @@ config PINCTRL_IPQ5018
 config PINCTRL_IPQ8064
 	tristate "Qualcomm IPQ8064 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm IPQ8064 platform.
 
 config PINCTRL_IPQ5210
-	tristate "Qualcomm Technologies Inc IPQ5210 pin controller driver"
+	tristate "Qualcomm IPQ5210 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc IPQ5210 platform.
 
 config PINCTRL_IPQ5332
-	tristate "Qualcomm Technologies Inc IPQ5332 pin controller driver"
+	tristate "Qualcomm IPQ5332 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc IPQ5332 platform.
 
 config PINCTRL_IPQ5424
-	tristate "Qualcomm Technologies, Inc. IPQ5424 pin controller driver"
+	tristate "Qualcomm IPQ5424 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for
           the Qualcomm Technologies Inc. TLMM block found on the
@@ -94,8 +105,9 @@ config PINCTRL_IPQ5424
           IPQ5424.
 
 config PINCTRL_IPQ8074
-	tristate "Qualcomm Technologies, Inc. IPQ8074 pin controller driver"
+	tristate "Qualcomm IPQ8074 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for
 	  the Qualcomm Technologies Inc. TLMM block found on the
@@ -103,8 +115,9 @@ config PINCTRL_IPQ8074
 	  IPQ8074.
 
 config PINCTRL_IPQ6018
-	tristate "Qualcomm Technologies, Inc. IPQ6018 pin controller driver"
+	tristate "Qualcomm IPQ6018 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for
 	  the Qualcomm Technologies Inc. TLMM block found on the
@@ -112,17 +125,29 @@ config PINCTRL_IPQ6018
 	  IPQ6018.
 
 config PINCTRL_IPQ9574
-	tristate "Qualcomm Technologies, Inc. IPQ9574 pin controller driver"
+	tristate "Qualcomm IPQ9574 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for
           the Qualcomm Technologies Inc. TLMM block found on the
           Qualcomm Technologies Inc. IPQ9574 platform. Select this for
           IPQ9574.
 
+config PINCTRL_IPQ9650
+	tristate "Qualcomm IPQ9650 pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
+	help
+	  This is the pinctrl, pinmux, pinconf and gpiolib driver for
+          the Qualcomm Technologies Inc. TLMM block found on the
+          Qualcomm Technologies Inc. IPQ9650 platform. Select this for
+          IPQ9650.
+
 config PINCTRL_KAANAPALI
-	tristate "Qualcomm Technologies Inc Kaanapali pin controller driver"
+	tristate "Qualcomm Kaanapali pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
@@ -131,6 +156,7 @@ config PINCTRL_KAANAPALI
 config PINCTRL_MSM8226
 	tristate "Qualcomm 8226 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
@@ -139,6 +165,7 @@ config PINCTRL_MSM8226
 config PINCTRL_MSM8660
 	tristate "Qualcomm 8660 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm 8660 platform.
@@ -146,12 +173,14 @@ config PINCTRL_MSM8660
 config PINCTRL_MSM8960
 	tristate "Qualcomm 8960 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm 8960 platform.
 
 config PINCTRL_MDM9607
 	tristate "Qualcomm 9607 pin controller driver"
+	default ARCH_QCOM if ARM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm 9607 platform.
@@ -159,6 +188,7 @@ config PINCTRL_MDM9607
 config PINCTRL_MDM9615
 	tristate "Qualcomm 9615 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm 9615 platform.
@@ -166,6 +196,7 @@ config PINCTRL_MDM9615
 config PINCTRL_MSM8X74
 	tristate "Qualcomm 8x74 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm 8974 platform.
@@ -173,12 +204,14 @@ config PINCTRL_MSM8X74
 config PINCTRL_MSM8909
 	tristate "Qualcomm 8909 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found on the Qualcomm MSM8909 platform.
 
 config PINCTRL_MSM8916
 	tristate "Qualcomm 8916 pin controller driver"
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found on the Qualcomm 8916 platform.
@@ -192,6 +225,7 @@ config PINCTRL_MSM8917
 config PINCTRL_MSM8953
 	tristate "Qualcomm 8953 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found on the Qualcomm MSM8953 platform.
@@ -201,6 +235,7 @@ config PINCTRL_MSM8953
 config PINCTRL_MSM8976
 	tristate "Qualcomm 8976 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found on the Qualcomm MSM8976 platform.
@@ -210,6 +245,7 @@ config PINCTRL_MSM8976
 config PINCTRL_MSM8994
 	tristate "Qualcomm 8994 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm 8994 platform. The
@@ -218,6 +254,7 @@ config PINCTRL_MSM8994
 config PINCTRL_MSM8996
 	tristate "Qualcomm MSM8996 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm MSM8996 platform.
@@ -225,13 +262,23 @@ config PINCTRL_MSM8996
 config PINCTRL_MSM8998
 	tristate "Qualcomm MSM8998 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm TLMM block found in the Qualcomm MSM8998 platform.
 
+config PINCTRL_NORD
+	tristate "Qualcomm Nord pin controller driver"
+	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
+	help
+	  This is the pinctrl, pinmux and pinconf driver for the Qualcomm
+	  TLMM block found on the Qualcomm NORD platforms.
+
 config PINCTRL_QCM2290
 	tristate "Qualcomm QCM2290 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  TLMM block found in the Qualcomm QCM2290 platform.
@@ -239,48 +286,54 @@ config PINCTRL_QCM2290
 config PINCTRL_QCS404
 	tristate "Qualcomm QCS404 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  TLMM block found in the Qualcomm QCS404 platform.
 
 config PINCTRL_QCS615
-	tristate "Qualcomm Technologies QCS615 pin controller driver"
+	tristate "Qualcomm QCS615 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  TLMM block found on the Qualcomm QCS615 platform.
 
 config PINCTRL_QCS8300
-	tristate "Qualcomm Technologies QCS8300 pin controller driver"
+	tristate "Qualcomm QCS8300 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux and pinconf driver for the Qualcomm
 	  TLMM block found on the Qualcomm QCS8300 platform.
 
 config PINCTRL_QDF2XXX
-	tristate "Qualcomm Technologies QDF2xxx pin controller driver"
+	tristate "Qualcomm QDF2xxx pin controller driver"
 	depends on ACPI
+	default ARCH_QCOM if ARM64
 	help
 	  This is the GPIO driver for the TLMM block found on the
 	  Qualcomm Technologies QDF2xxx SOCs.
 
 config PINCTRL_QDU1000
-	tristate "Qualcomm Technologies Inc QDU1000/QRU1000 pin controller driver"
+	tristate "Qualcomm QDU1000/QRU1000 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf, and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc QDU1000 and QRU1000 platforms.
 
 config PINCTRL_SA8775P
-	tristate "Qualcomm Technologies Inc SA8775P pin controller driver"
+	tristate "Qualcomm SA8775P pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux and pinconf driver for the Qualcomm
 	  TLMM block found on the Qualcomm SA8775P platforms.
 
 config PINCTRL_SAR2130P
-	tristate "Qualcomm Technologies Inc SAR2130P pin controller driver"
+	tristate "Qualcomm SAR2130P pin controller driver"
 	depends on ARM64 || COMPILE_TEST
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
@@ -288,202 +341,227 @@ config PINCTRL_SAR2130P
 	  Technologies Inc SAR2130P platform.
 
 config PINCTRL_SC7180
-	tristate "Qualcomm Technologies Inc SC7180 pin controller driver"
+	tristate "Qualcomm SC7180 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SC7180 platform.
 
 config PINCTRL_SC7280
-	tristate "Qualcomm Technologies Inc SC7280 pin controller driver"
+	tristate "Qualcomm SC7280 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SC7280 platform.
 
 config PINCTRL_SC8180X
-	tristate "Qualcomm Technologies Inc SC8180x pin controller driver"
+	tristate "Qualcomm SC8180x pin controller driver"
 	depends on (OF || ACPI)
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SC8180x platform.
 
 config PINCTRL_SC8280XP
-	tristate "Qualcomm Technologies Inc SC8280xp pin controller driver"
+	tristate "Qualcomm SC8280xp pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SC8280xp platform.
 
 config PINCTRL_SDM660
-	tristate "Qualcomm Technologies Inc SDM660 pin controller driver"
+	tristate "Qualcomm SDM660 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SDM660 platform.
 
 config PINCTRL_SDM670
-	tristate "Qualcomm Technologies Inc SDM670 pin controller driver"
+	tristate "Qualcomm SDM670 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SDM670 platform.
 
 config PINCTRL_SDM845
-	tristate "Qualcomm Technologies Inc SDM845 pin controller driver"
+	tristate "Qualcomm SDM845 pin controller driver"
 	depends on (OF || ACPI)
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SDM845 platform.
 
 config PINCTRL_SDX55
-	tristate "Qualcomm Technologies Inc SDX55 pin controller driver"
+	tristate "Qualcomm SDX55 pin controller driver"
 	depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SDX55 platform.
 
 config PINCTRL_SDX65
-        tristate "Qualcomm Technologies Inc SDX65 pin controller driver"
+        tristate "Qualcomm SDX65 pin controller driver"
         depends on ARM || COMPILE_TEST
+	default ARCH_QCOM
         help
          This is the pinctrl, pinmux, pinconf and gpiolib driver for the
          Qualcomm Technologies Inc TLMM block found on the Qualcomm
          Technologies Inc SDX65 platform.
 
 config PINCTRL_SDX75
-        tristate "Qualcomm Technologies Inc SDX75 pin controller driver"
+        tristate "Qualcomm SDX75 pin controller driver"
         depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
         help
          This is the pinctrl, pinmux, pinconf and gpiolib driver for the
          Qualcomm Technologies Inc TLMM block found on the Qualcomm
          Technologies Inc SDX75 platform.
 
 config PINCTRL_SM4450
-	tristate "Qualcomm Technologies Inc SM4450 pin controller driver"
+	tristate "Qualcomm SM4450 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SM4450 platform.
 
 config PINCTRL_SM6115
-	tristate "Qualcomm Technologies Inc SM6115,SM4250 pin controller driver"
+	tristate "Qualcomm SM6115,SM4250 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SM6115 and SM4250 platforms.
 
 config PINCTRL_SM6125
-	tristate "Qualcomm Technologies Inc SM6125 pin controller driver"
+	tristate "Qualcomm SM6125 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SM6125 platform.
 
 config PINCTRL_SM6350
-	tristate "Qualcomm Technologies Inc SM6350 pin controller driver"
+	tristate "Qualcomm SM6350 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SM6350 platform.
 
 config PINCTRL_SM6375
-	tristate "Qualcomm Technologies Inc SM6375 pin controller driver"
+	tristate "Qualcomm SM6375 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SM6375 platform.
 
 config PINCTRL_SM7150
-	tristate "Qualcomm Technologies Inc SM7150 pin controller driver"
+	tristate "Qualcomm SM7150 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SM7150 platform.
 
 config PINCTRL_MILOS
-	tristate "Qualcomm Technologies Inc Milos pin controller driver"
+	tristate "Qualcomm Milos pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc Milos platform.
 
 config PINCTRL_SM8150
-	tristate "Qualcomm Technologies Inc SM8150 pin controller driver"
+	tristate "Qualcomm SM8150 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	 This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	 Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	 Technologies Inc SM8150 platform.
 
 config PINCTRL_SM8250
-	tristate "Qualcomm Technologies Inc SM8250 pin controller driver"
+	tristate "Qualcomm SM8250 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SM8250 platform.
 
 config PINCTRL_SM8350
-	tristate "Qualcomm Technologies Inc SM8350 pin controller driver"
+	tristate "Qualcomm SM8350 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SM8350 platform.
 
 config PINCTRL_SM8450
-	tristate "Qualcomm Technologies Inc SM8450 pin controller driver"
+	tristate "Qualcomm SM8450 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SM8450 platform.
 
 config PINCTRL_SM8550
-	tristate "Qualcomm Technologies Inc SM8550 pin controller driver"
+	tristate "Qualcomm SM8550 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SM8550 platform.
 
 config PINCTRL_SM8650
-	tristate "Qualcomm Technologies Inc SM8650 pin controller driver"
+	tristate "Qualcomm SM8650 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SM8650 platform.
 
 config PINCTRL_SM8750
-	tristate "Qualcomm Technologies Inc SM8750 pin controller driver"
+	tristate "Qualcomm SM8750 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc TLMM block found on the Qualcomm
 	  Technologies Inc SM8750 platform.
 
 config PINCTRL_X1E80100
-	tristate "Qualcomm Technologies Inc X1E80100 pin controller driver"
+	tristate "Qualcomm X1E80100 pin controller driver"
 	depends on ARM64 || COMPILE_TEST
+	default ARCH_QCOM
 	help
 	  This is the pinctrl, pinmux, pinconf and gpiolib driver for the
 	  Qualcomm Technologies Inc Top Level Mode Multiplexer block (TLMM)
diff --git a/drivers/pinctrl/qcom/Makefile b/drivers/pinctrl/qcom/Makefile
index 84bda3ada8749..a1e9e2e292a0f 100644
--- a/drivers/pinctrl/qcom/Makefile
+++ b/drivers/pinctrl/qcom/Makefile
@@ -15,6 +15,7 @@ obj-$(CONFIG_PINCTRL_IPQ5424)	+= pinctrl-ipq5424.o
 obj-$(CONFIG_PINCTRL_IPQ8074)	+= pinctrl-ipq8074.o
 obj-$(CONFIG_PINCTRL_IPQ6018)	+= pinctrl-ipq6018.o
 obj-$(CONFIG_PINCTRL_IPQ9574)	+= pinctrl-ipq9574.o
+obj-$(CONFIG_PINCTRL_IPQ9650)	+= pinctrl-ipq9650.o
 obj-$(CONFIG_PINCTRL_KAANAPALI) += pinctrl-kaanapali.o
 obj-$(CONFIG_PINCTRL_MSM8226)	+= pinctrl-msm8226.o
 obj-$(CONFIG_PINCTRL_MSM8660)	+= pinctrl-msm8660.o
@@ -37,6 +38,7 @@ obj-$(CONFIG_PINCTRL_MDM9607)	+= pinctrl-mdm9607.o
 obj-$(CONFIG_PINCTRL_MDM9615)	+= pinctrl-mdm9615.o
 obj-$(CONFIG_PINCTRL_MILOS) += pinctrl-milos.o
 obj-$(CONFIG_PINCTRL_MILOS_LPASS_LPI) += pinctrl-milos-lpass-lpi.o
+obj-$(CONFIG_PINCTRL_NORD)	+= pinctrl-nord.o
 obj-$(CONFIG_PINCTRL_QCOM_SPMI_PMIC) += pinctrl-spmi-gpio.o
 obj-$(CONFIG_PINCTRL_QCOM_SPMI_PMIC) += pinctrl-spmi-mpp.o
 obj-$(CONFIG_PINCTRL_QCOM_SSBI_PMIC) += pinctrl-ssbi-gpio.o
@@ -63,6 +65,7 @@ obj-$(CONFIG_PINCTRL_SM6115) += pinctrl-sm6115.o
 obj-$(CONFIG_PINCTRL_SM6115_LPASS_LPI) += pinctrl-sm6115-lpass-lpi.o
 obj-$(CONFIG_PINCTRL_SM6125) += pinctrl-sm6125.o
 obj-$(CONFIG_PINCTRL_SM6350) += pinctrl-sm6350.o
+obj-$(CONFIG_PINCTRL_SM6350_LPASS_LPI) += pinctrl-sm6350-lpass-lpi.o
 obj-$(CONFIG_PINCTRL_SM6375) += pinctrl-sm6375.o
 obj-$(CONFIG_PINCTRL_SM7150) += pinctrl-sm7150.o
 obj-$(CONFIG_PINCTRL_SM8150) += pinctrl-sm8150.o
diff --git a/drivers/pinctrl/qcom/pinctrl-apq8064.c b/drivers/pinctrl/qcom/pinctrl-apq8064.c
index 3654913f1ae58..a47f3d4f1ea50 100644
--- a/drivers/pinctrl/qcom/pinctrl-apq8064.c
+++ b/drivers/pinctrl/qcom/pinctrl-apq8064.c
@@ -622,6 +622,7 @@ static const struct of_device_id apq8064_pinctrl_of_match[] = {
 	{ .compatible = "qcom,apq8064-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, apq8064_pinctrl_of_match);
 
 static struct platform_driver apq8064_pinctrl_driver = {
 	.driver = {
@@ -646,4 +647,3 @@ module_exit(apq8064_pinctrl_exit);
 MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@sonymobile.com>");
 MODULE_DESCRIPTION("Qualcomm APQ8064 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, apq8064_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-apq8084.c b/drivers/pinctrl/qcom/pinctrl-apq8084.c
index 9fdbe6743512a..e3c1f86aba7d0 100644
--- a/drivers/pinctrl/qcom/pinctrl-apq8084.c
+++ b/drivers/pinctrl/qcom/pinctrl-apq8084.c
@@ -1198,6 +1198,7 @@ static const struct of_device_id apq8084_pinctrl_of_match[] = {
 	{ .compatible = "qcom,apq8084-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, apq8084_pinctrl_of_match);
 
 static struct platform_driver apq8084_pinctrl_driver = {
 	.driver = {
@@ -1221,4 +1222,3 @@ module_exit(apq8084_pinctrl_exit);
 
 MODULE_DESCRIPTION("Qualcomm APQ8084 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, apq8084_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-eliza.c b/drivers/pinctrl/qcom/pinctrl-eliza.c
index c1f756cbcdeba..636442d0afed0 100644
--- a/drivers/pinctrl/qcom/pinctrl-eliza.c
+++ b/drivers/pinctrl/qcom/pinctrl-eliza.c
@@ -54,30 +54,6 @@
 		.intr_detection_width = 2,	\
 	}
 
-#define SDC_QDSD_PINGROUP(pg_name, ctl, pull, drv)	\
-	{					        \
-		.grp = PINCTRL_PINGROUP(#pg_name,	\
-			pg_name##_pins,			\
-			ARRAY_SIZE(pg_name##_pins)),	\
-		.ctl_reg = ctl,				\
-		.io_reg = 0,				\
-		.intr_cfg_reg = 0,			\
-		.intr_status_reg = 0,			\
-		.mux_bit = -1,				\
-		.pull_bit = pull,			\
-		.drv_bit = drv,				\
-		.oe_bit = -1,				\
-		.in_bit = -1,				\
-		.out_bit = -1,				\
-		.intr_enable_bit = -1,			\
-		.intr_status_bit = -1,			\
-		.intr_target_bit = -1,			\
-		.intr_raw_status_bit = -1,		\
-		.intr_polarity_bit = -1,		\
-		.intr_detection_bit = -1,		\
-		.intr_detection_width = -1,		\
-	}
-
 #define UFS_RESET(pg_name, ctl, io)			\
 	{					        \
 		.grp = PINCTRL_PINGROUP(#pg_name,	\
@@ -563,15 +539,34 @@ enum eliza_functions {
 	msm_mux_qup1_se0,
 	msm_mux_qup1_se1,
 	msm_mux_qup1_se2,
+	msm_mux_qup1_se2_l2_mira,
+	msm_mux_qup1_se2_l2_mirb,
+	msm_mux_qup1_se2_l3_mira,
+	msm_mux_qup1_se2_l3_mirb,
 	msm_mux_qup1_se3,
-	msm_mux_qup1_se4,
+	msm_mux_qup1_se4_l0,
+	msm_mux_qup1_se4_l1,
+	msm_mux_qup1_se4_l2,
+	msm_mux_qup1_se4_l3,
 	msm_mux_qup1_se5,
 	msm_mux_qup1_se6,
+	msm_mux_qup1_se6_l1_mira,
+	msm_mux_qup1_se6_l1_mirb,
+	msm_mux_qup1_se6_l3_mira,
+	msm_mux_qup1_se6_l3_mirb,
 	msm_mux_qup1_se7,
+	msm_mux_qup1_se7_l0_mira,
+	msm_mux_qup1_se7_l0_mirb,
+	msm_mux_qup1_se7_l1_mira,
+	msm_mux_qup1_se7_l1_mirb,
 	msm_mux_qup2_se0,
 	msm_mux_qup2_se1,
 	msm_mux_qup2_se2,
 	msm_mux_qup2_se3,
+	msm_mux_qup2_se3_l0_mira,
+	msm_mux_qup2_se3_l0_mirb,
+	msm_mux_qup2_se3_l1_mira,
+	msm_mux_qup2_se3_l1_mirb,
 	msm_mux_qup2_se4,
 	msm_mux_qup2_se5,
 	msm_mux_qup2_se6,
@@ -978,15 +973,43 @@ static const char *const qup1_se1_groups[] = {
 };
 
 static const char *const qup1_se2_groups[] = {
-	"gpio52", "gpio53", "gpio54", "gpio52", "gpio55", "gpio53", "gpio40", "gpio42", "gpio30",
+	"gpio52", "gpio53", "gpio40", "gpio42", "gpio30",
+};
+
+static const char *const qup1_se2_l2_mira_groups[] = {
+	"gpio54",
+};
+
+static const char *const qup1_se2_l2_mirb_groups[] = {
+	"gpio52",
+};
+
+static const char *const qup1_se2_l3_mira_groups[] = {
+	"gpio55",
+};
+
+static const char *const qup1_se2_l3_mirb_groups[] = {
+	"gpio53",
 };
 
 static const char *const qup1_se3_groups[] = {
 	"gpio44", "gpio45", "gpio46", "gpio47",
 };
 
-static const char *const qup1_se4_groups[] = {
-	"gpio36", "gpio37", "gpio37", "gpio36",
+static const char *const qup1_se4_l0_groups[] = {
+	"gpio36",
+};
+
+static const char *const qup1_se4_l1_groups[] = {
+	"gpio37",
+};
+
+static const char *const qup1_se4_l2_groups[] = {
+	"gpio37",
+};
+
+static const char *const qup1_se4_l3_groups[] = {
+	"gpio36",
 };
 
 static const char *const qup1_se5_groups[] = {
@@ -994,11 +1017,43 @@ static const char *const qup1_se5_groups[] = {
 };
 
 static const char *const qup1_se6_groups[] = {
-	"gpio40", "gpio42", "gpio54", "gpio42", "gpio40", "gpio55",
+	"gpio40", "gpio42",
+};
+
+static const char *const qup1_se6_l1_mira_groups[] = {
+	"gpio42",
+};
+
+static const char *const qup1_se6_l1_mirb_groups[] = {
+	"gpio54",
+};
+
+static const char *const qup1_se6_l3_mira_groups[] = {
+	"gpio40",
+};
+
+static const char *const qup1_se6_l3_mirb_groups[] = {
+	"gpio55",
 };
 
 static const char *const qup1_se7_groups[] = {
-	"gpio81", "gpio78", "gpio80", "gpio114", "gpio114", "gpio78",
+	"gpio78", "gpio114",
+};
+
+static const char *const qup1_se7_l0_mira_groups[] = {
+	"gpio81",
+};
+
+static const char *const qup1_se7_l0_mirb_groups[] = {
+	"gpio78",
+};
+
+static const char *const qup1_se7_l1_mira_groups[] = {
+	"gpio80",
+};
+
+static const char *const qup1_se7_l1_mirb_groups[] = {
+	"gpio114",
 };
 
 static const char *const qup2_se0_groups[] = {
@@ -1014,7 +1069,23 @@ static const char *const qup2_se2_groups[] = {
 };
 
 static const char *const qup2_se3_groups[] = {
-	"gpio79", "gpio116", "gpio97", "gpio100", "gpio100", "gpio116",
+	"gpio100", "gpio116",
+};
+
+static const char *const qup2_se3_l0_mira_groups[] = {
+	"gpio79",
+};
+
+static const char *const qup2_se3_l0_mirb_groups[] = {
+	"gpio116",
+};
+
+static const char *const qup2_se3_l1_mira_groups[] = {
+	"gpio97",
+};
+
+static const char *const qup2_se3_l1_mirb_groups[] = {
+	"gpio100",
 };
 
 static const char *const qup2_se4_groups[] = {
@@ -1236,15 +1307,34 @@ static const struct pinfunction eliza_functions[] = {
 	MSM_PIN_FUNCTION(qup1_se0),
 	MSM_PIN_FUNCTION(qup1_se1),
 	MSM_PIN_FUNCTION(qup1_se2),
+	MSM_PIN_FUNCTION(qup1_se2_l2_mira),
+	MSM_PIN_FUNCTION(qup1_se2_l2_mirb),
+	MSM_PIN_FUNCTION(qup1_se2_l3_mira),
+	MSM_PIN_FUNCTION(qup1_se2_l3_mirb),
 	MSM_PIN_FUNCTION(qup1_se3),
-	MSM_PIN_FUNCTION(qup1_se4),
+	MSM_PIN_FUNCTION(qup1_se4_l0),
+	MSM_PIN_FUNCTION(qup1_se4_l1),
+	MSM_PIN_FUNCTION(qup1_se4_l2),
+	MSM_PIN_FUNCTION(qup1_se4_l3),
 	MSM_PIN_FUNCTION(qup1_se5),
 	MSM_PIN_FUNCTION(qup1_se6),
+	MSM_PIN_FUNCTION(qup1_se6_l1_mira),
+	MSM_PIN_FUNCTION(qup1_se6_l1_mirb),
+	MSM_PIN_FUNCTION(qup1_se6_l3_mira),
+	MSM_PIN_FUNCTION(qup1_se6_l3_mirb),
 	MSM_PIN_FUNCTION(qup1_se7),
+	MSM_PIN_FUNCTION(qup1_se7_l0_mira),
+	MSM_PIN_FUNCTION(qup1_se7_l0_mirb),
+	MSM_PIN_FUNCTION(qup1_se7_l1_mira),
+	MSM_PIN_FUNCTION(qup1_se7_l1_mirb),
 	MSM_PIN_FUNCTION(qup2_se0),
 	MSM_PIN_FUNCTION(qup2_se1),
 	MSM_PIN_FUNCTION(qup2_se2),
 	MSM_PIN_FUNCTION(qup2_se3),
+	MSM_PIN_FUNCTION(qup2_se3_l0_mira),
+	MSM_PIN_FUNCTION(qup2_se3_l0_mirb),
+	MSM_PIN_FUNCTION(qup2_se3_l1_mira),
+	MSM_PIN_FUNCTION(qup2_se3_l1_mirb),
 	MSM_PIN_FUNCTION(qup2_se4),
 	MSM_PIN_FUNCTION(qup2_se5),
 	MSM_PIN_FUNCTION(qup2_se6),
@@ -1322,13 +1412,13 @@ static const struct msm_pingroup eliza_groups[] = {
 	[33] = PINGROUP(33, qup1_se1, ibi_i3c, host2wlan_sol, gcc_gp3, _, _, _, _, _, _, _),
 	[34] = PINGROUP(34, qup1_se1, qup1_se5, tb_trig_sdc1, ddr_bist_start, qdss_gpio_tracedata, _, _, _, _, _, _),
 	[35] = PINGROUP(35, qup1_se1, qup1_se5, tb_trig_sdc2, gcc_gp2, qdss_gpio_tracedata, _, _, _, _, _, _),
-	[36] = PINGROUP(36, qup1_se4, qup1_se4, ibi_i3c, _, _, _, _, _, _, _, _),
-	[37] = PINGROUP(37, qup1_se4, qup1_se4, ibi_i3c, _, _, _, _, _, _, _, _),
+	[36] = PINGROUP(36, qup1_se4_l0, qup1_se4_l3, ibi_i3c, _, _, _, _, _, _, _, _),
+	[37] = PINGROUP(37, qup1_se4_l1, qup1_se4_l2, ibi_i3c, _, _, _, _, _, _, _, _),
 	[38] = PINGROUP(38, _, _, _, _, _, _, _, _, _, _, _),
 	[39] = PINGROUP(39, _, _, _, _, _, _, _, _, _, _, _),
-	[40] = PINGROUP(40, qup1_se6, qup1_se2, qup1_se6, _, qdss_gpio_tracedata, gnss_adc1, ddr_pxi1, _, _, _, _),
+	[40] = PINGROUP(40, qup1_se6, qup1_se2, qup1_se6_l3_mira, _, qdss_gpio_tracedata, gnss_adc1, ddr_pxi1, _, _, _, _),
 	[41] = PINGROUP(41, _, _, _, _, _, _, _, _, _, _, _),
-	[42] = PINGROUP(42, qup1_se6, qup1_se2, qup1_se6, qdss_gpio_tracedata, gnss_adc0, ddr_pxi1, _, _, _, _, _),
+	[42] = PINGROUP(42, qup1_se6, qup1_se2, qup1_se6_l1_mira, qdss_gpio_tracedata, gnss_adc0, ddr_pxi1, _, _, _, _, _),
 	[43] = PINGROUP(43, _, _, _, _, _, _, _, _, _, _, _),
 	[44] = PINGROUP(44, qup1_se3, _, _, _, _, _, _, _, _, _, _),
 	[45] = PINGROUP(45, qup1_se3, _, _, _, _, _, _, _, _, _, _),
@@ -1338,10 +1428,10 @@ static const struct msm_pingroup eliza_groups[] = {
 	[49] = PINGROUP(49, _, _, _, _, _, _, _, _, _, _, _),
 	[50] = PINGROUP(50, sdc2_fb_clk, _, _, _, _, _, _, _, _, _, _),
 	[51] = PINGROUP(51, _, _, _, _, _, _, _, _, _, _, _),
-	[52] = PINGROUP(52, qup1_se2, pcie1_clk_req_n, qup1_se2, ddr_bist_complete, qdss_gpio_tracedata, _, vsense_trigger_mirnat, _, _, _, _),
-	[53] = PINGROUP(53, qup1_se2, qup1_se2, gcc_gp1, ddr_bist_stop, _, qdss_gpio_tracedata, _, _, _, _, _),
-	[54] = PINGROUP(54, qup1_se2, qup1_se6, qdss_gpio_tracedata, gnss_adc1, atest_usb, ddr_pxi0, _, _, _, _, _),
-	[55] = PINGROUP(55, qup1_se2, dp0_hot, qup1_se6, _, gnss_adc0, atest_usb, ddr_pxi0, _, _, _, _),
+	[52] = PINGROUP(52, qup1_se2, pcie1_clk_req_n, qup1_se2_l2_mirb, ddr_bist_complete, qdss_gpio_tracedata, _, vsense_trigger_mirnat, _, _, _, _),
+	[53] = PINGROUP(53, qup1_se2, qup1_se2_l3_mirb, gcc_gp1, ddr_bist_stop, _, qdss_gpio_tracedata, _, _, _, _, _),
+	[54] = PINGROUP(54, qup1_se2_l2_mira, qup1_se6_l1_mirb, qdss_gpio_traceclk, gnss_adc1, atest_usb, ddr_pxi0, _, _, _, _, _),
+	[55] = PINGROUP(55, qup1_se2_l3_mira, dp0_hot, qup1_se6_l3_mirb, _, gnss_adc0, atest_usb, ddr_pxi0, _, _, _, _),
 	[56] = PINGROUP(56, usb0_hs, tsense_pwm1, tsense_pwm2, tsense_pwm3, tsense_pwm4, _, _, _, _, _, _),
 	[57] = PINGROUP(57, sd_write_protect, _, _, _, _, _, _, _, _, _, _),
 	[58] = PINGROUP(58, _, _, _, _, _, _, _, _, _, _, _),
@@ -1358,16 +1448,16 @@ static const struct msm_pingroup eliza_groups[] = {
 	[69] = PINGROUP(69, cam_mclk, audio_ext_mclk0, resout_gpio, prng_rosc1, _, _, _, _, _, _, _),
 	[70] = PINGROUP(70, cci_i2c_sda, tmess_prng2, _, phase_flag, atest_char, _, _, _, _, _, _),
 	[71] = PINGROUP(71, cci_i2c_scl, tmess_prng3, _, phase_flag, atest_char, _, _, _, _, _, _),
-	[72] = PINGROUP(72, cci_i2c_sda, tmess_prng1, qdss_gpio_tracedata, atest_char, _, _, _, _, _, _, _),
+	[72] = PINGROUP(72, cci_i2c_sda, tmess_prng1, qdss_gpio_tracectl, atest_char, _, _, _, _, _, _, _),
 	[73] = PINGROUP(73, cci_i2c_scl, tmess_prng0, qdss_cti, atest_char, _, _, _, _, _, _, _),
 	[74] = PINGROUP(74, cci_i2c_sda, prng_rosc3, qdss_cti, atest_char, _, _, _, _, _, _, _),
 	[75] = PINGROUP(75, cci_i2c_scl, _, phase_flag, _, _, _, _, _, _, _, _),
 	[76] = PINGROUP(76, cci_i2c_sda, cci_timer, prng_rosc2, _, phase_flag, _, _, _, _, _, _),
 	[77] = PINGROUP(77, cci_i2c_scl, jitter_bist, _, _, _, _, _, _, _, _, _),
-	[78] = PINGROUP(78, qup1_se7, qup1_se7, _, phase_flag, _, _, _, _, _, _, _),
-	[79] = PINGROUP(79, qspi0, mdp_vsync, qup2_se3, _, _, _, _, _, _, _, _),
-	[80] = PINGROUP(80, pcie0_clk_req_n, qup1_se7, _, phase_flag, _, _, _, _, _, _, _),
-	[81] = PINGROUP(81, wcn_sw_ctrl, qup1_se7, dbg_out_clk, _, _, _, _, _, _, _, _),
+	[78] = PINGROUP(78, qup1_se7, qup1_se7_l0_mirb, _, phase_flag, _, _, _, _, _, _, _),
+	[79] = PINGROUP(79, qspi0, mdp_vsync, qup2_se3_l0_mira, _, _, _, _, _, _, _, _),
+	[80] = PINGROUP(80, pcie0_clk_req_n, qup1_se7_l1_mira, _, phase_flag, _, _, _, _, _, _, _),
+	[81] = PINGROUP(81, wcn_sw_ctrl, qup1_se7_l0_mira, dbg_out_clk, _, _, _, _, _, _, _, _),
 	[82] = PINGROUP(82, _, _, _, _, _, _, _, _, _, _, _),
 	[83] = PINGROUP(83, _, _, _, _, _, _, _, _, _, _, _),
 	[84] = PINGROUP(84, uim0_data, _, _, _, _, _, _, _, _, _, _),
@@ -1383,10 +1473,10 @@ static const struct msm_pingroup eliza_groups[] = {
 	[94] = PINGROUP(94, qlink_wmss, _, _, _, _, _, _, _, _, _, _),
 	[95] = PINGROUP(95, qlink_big_request, _, _, _, _, _, _, _, _, _, _),
 	[96] = PINGROUP(96, qlink_big_enable, _, _, _, _, _, _, _, _, _, _),
-	[97] = PINGROUP(97, uim1_data, qspi0, qup2_se3, _, _, _, _, _, _, _, _),
+	[97] = PINGROUP(97, uim1_data, qspi0, qup2_se3_l1_mira, _, _, _, _, _, _, _, _),
 	[98] = PINGROUP(98, uim1_clk, qspi0, _, _, _, _, _, _, _, _, _),
 	[99] = PINGROUP(99, uim1_reset, qspi0, _, _, _, _, _, _, _, _, _),
-	[100] = PINGROUP(100, uim1_present, qspi0, qup2_se3, coex_uart2_tx, qup2_se3, mdp_vsync, _, _, _, _, _),
+	[100] = PINGROUP(100, uim1_present, qspi0, qup2_se3, coex_uart2_tx, qup2_se3_l1_mirb, mdp_vsync, _, _, _, _, _),
 	[101] = PINGROUP(101, _, _, _, _, _, _, _, _, _, _, _),
 	[102] = PINGROUP(102, _, _, _, _, _, _, _, _, _, _, _),
 	[103] = PINGROUP(103, _, _, _, _, _, _, _, _, _, _, _),
@@ -1400,9 +1490,9 @@ static const struct msm_pingroup eliza_groups[] = {
 	[111] = PINGROUP(111, coex_uart1_tx, _, _, _, _, _, _, _, _, _, _),
 	[112] = PINGROUP(112, coex_uart1_rx, _, _, _, _, _, _, _, _, _, _),
 	[113] = PINGROUP(113, _, nav_gpio3, _, _, _, _, _, _, _, _, _),
-	[114] = PINGROUP(114, qup1_se7, qup1_se7, _, qdss_gpio_tracedata, _, _, _, _, _, _, _),
+	[114] = PINGROUP(114, qup1_se7, qup1_se7_l1_mirb, _, qdss_gpio_tracedata, _, _, _, _, _, _, _),
 	[115] = PINGROUP(115, _, qspi0, cci_async_in, _, _, _, _, _, _, _, _),
-	[116] = PINGROUP(116, qspi0, coex_uart2_rx, qup2_se3, qup2_se3, _, _, _, _, _, _, _),
+	[116] = PINGROUP(116, qspi0, coex_uart2_rx, qup2_se3, qup2_se3_l0_mirb, _, _, _, _, _, _, _),
 	[117] = PINGROUP(117, nav_gpio1, _, vfr_1, _, _, _, _, _, _, _, _),
 	[118] = PINGROUP(118, nav_gpio2, _, _, _, _, _, _, _, _, _, _),
 	[119] = PINGROUP(119, nav_gpio0, _, _, _, _, _, _, _, _, _, _),
@@ -1430,10 +1520,10 @@ static const struct msm_pingroup eliza_groups[] = {
 	[141] = PINGROUP(141, _, _, _, _, _, _, _, _, _, _, egpio),
 	[142] = PINGROUP(142, _, _, _, _, _, _, _, _, _, _, egpio),
 	[143] = PINGROUP(143, _, _, _, _, _, _, _, _, _, _, egpio),
-	[144] = PINGROUP(144, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, egpio),
+	[144] = PINGROUP(144, _, qdss_gpio_tracectl, _, _, _, _, _, _, _, _, egpio),
 	[145] = PINGROUP(145, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, _, egpio),
 	[146] = PINGROUP(146, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, _, egpio),
-	[147] = PINGROUP(147, ddr_bist_fail, _, qdss_gpio_tracedata, _, _, _, _, _, _, _, egpio),
+	[147] = PINGROUP(147, ddr_bist_fail, _, qdss_gpio_traceclk, _, _, _, _, _, _, _, egpio),
 	[148] = PINGROUP(148, _, _, _, _, _, _, _, _, _, _, egpio),
 	[149] = PINGROUP(149, _, _, _, _, _, _, _, _, _, _, egpio),
 	[150] = PINGROUP(150, _, _, _, _, _, _, _, _, _, _, egpio),
@@ -1519,6 +1609,7 @@ static const struct of_device_id eliza_tlmm_of_match[] = {
 	{ .compatible = "qcom,eliza-tlmm", },
 	{},
 };
+MODULE_DEVICE_TABLE(of, eliza_tlmm_of_match);
 
 static struct platform_driver eliza_tlmm_driver = {
 	.driver = {
@@ -1542,4 +1633,3 @@ module_exit(eliza_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI Eliza TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, eliza_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-glymur.c b/drivers/pinctrl/qcom/pinctrl-glymur.c
index 9838c78399234..20f076e593756 100644
--- a/drivers/pinctrl/qcom/pinctrl-glymur.c
+++ b/drivers/pinctrl/qcom/pinctrl-glymur.c
@@ -1777,6 +1777,7 @@ static const struct of_device_id glymur_tlmm_of_match[] = {
 	{ .compatible = "qcom,mahua-tlmm", .data = &mahua_tlmm },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, glymur_tlmm_of_match);
 
 static int glymur_tlmm_probe(struct platform_device *pdev)
 {
@@ -1811,4 +1812,3 @@ module_exit(glymur_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI Glymur TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, glymur_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-hawi.c b/drivers/pinctrl/qcom/pinctrl-hawi.c
index 5c7894f3b9cbd..cdde69e954a76 100644
--- a/drivers/pinctrl/qcom/pinctrl-hawi.c
+++ b/drivers/pinctrl/qcom/pinctrl-hawi.c
@@ -1584,6 +1584,7 @@ static const struct of_device_id hawi_tlmm_of_match[] = {
 	{ .compatible = "qcom,hawi-tlmm", },
 	{},
 };
+MODULE_DEVICE_TABLE(of, hawi_tlmm_of_match);
 
 static struct platform_driver hawi_tlmm_driver = {
 	.driver = {
@@ -1607,4 +1608,3 @@ module_exit(hawi_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI Hawi TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, hawi_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq4019.c b/drivers/pinctrl/qcom/pinctrl-ipq4019.c
index c5f0decc3eb32..adf7c8631f721 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq4019.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq4019.c
@@ -479,7 +479,7 @@ static const struct pinfunction ipq4019_functions[] = {
 	QCA_PIN_FUNCTION(blsp_uart0),
 	QCA_PIN_FUNCTION(blsp_uart1),
 	QCA_PIN_FUNCTION(chip_rst),
-	QCA_PIN_FUNCTION(gpio),
+	QCA_GPIO_PIN_FUNCTION(gpio),
 	QCA_PIN_FUNCTION(i2s_rx),
 	QCA_PIN_FUNCTION(i2s_spdif_in),
 	QCA_PIN_FUNCTION(i2s_spdif_out),
@@ -702,6 +702,7 @@ static const struct of_device_id ipq4019_pinctrl_of_match[] = {
 	{ .compatible = "qcom,ipq4019-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, ipq4019_pinctrl_of_match);
 
 static struct platform_driver ipq4019_pinctrl_driver = {
 	.driver = {
@@ -725,4 +726,3 @@ module_exit(ipq4019_pinctrl_exit);
 
 MODULE_DESCRIPTION("Qualcomm ipq4019 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, ipq4019_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq6018.c b/drivers/pinctrl/qcom/pinctrl-ipq6018.c
index cc83f9362a85a..9a42e343ff17c 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq6018.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq6018.c
@@ -1072,6 +1072,7 @@ static const struct of_device_id ipq6018_pinctrl_of_match[] = {
 	{ .compatible = "qcom,ipq6018-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, ipq6018_pinctrl_of_match);
 
 static struct platform_driver ipq6018_pinctrl_driver = {
 	.driver = {
@@ -1095,4 +1096,3 @@ module_exit(ipq6018_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI ipq6018 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, ipq6018_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq8064.c b/drivers/pinctrl/qcom/pinctrl-ipq8064.c
index 0a9e357e64c60..78d320d56be6d 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq8064.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq8064.c
@@ -624,6 +624,7 @@ static const struct of_device_id ipq8064_pinctrl_of_match[] = {
 	{ .compatible = "qcom,ipq8064-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, ipq8064_pinctrl_of_match);
 
 static struct platform_driver ipq8064_pinctrl_driver = {
 	.driver = {
@@ -648,4 +649,3 @@ module_exit(ipq8064_pinctrl_exit);
 MODULE_AUTHOR("Andy Gross <agross@codeaurora.org>");
 MODULE_DESCRIPTION("Qualcomm IPQ8064 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, ipq8064_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq8074.c b/drivers/pinctrl/qcom/pinctrl-ipq8074.c
index 64ce8ea8f544b..0b95f52adcc38 100644
--- a/drivers/pinctrl/qcom/pinctrl-ipq8074.c
+++ b/drivers/pinctrl/qcom/pinctrl-ipq8074.c
@@ -1033,6 +1033,7 @@ static const struct of_device_id ipq8074_pinctrl_of_match[] = {
 	{ .compatible = "qcom,ipq8074-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, ipq8074_pinctrl_of_match);
 
 static struct platform_driver ipq8074_pinctrl_driver = {
 	.driver = {
@@ -1056,4 +1057,3 @@ module_exit(ipq8074_pinctrl_exit);
 
 MODULE_DESCRIPTION("Qualcomm ipq8074 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, ipq8074_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-ipq9650.c b/drivers/pinctrl/qcom/pinctrl-ipq9650.c
new file mode 100644
index 0000000000000..64e443aa31b2f
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-ipq9650.c
@@ -0,0 +1,762 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-msm.h"
+
+#define REG_SIZE 0x1000
+#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9)	      \
+	{                                                             \
+		.grp = PINCTRL_PINGROUP("gpio" #id,                   \
+					gpio##id##_pins,              \
+					ARRAY_SIZE(gpio##id##_pins)), \
+		.ctl_reg = REG_SIZE * id,                             \
+		.io_reg = 0x4 + REG_SIZE * id,                        \
+		.intr_cfg_reg = 0x8 + REG_SIZE * id,                  \
+		.intr_status_reg = 0xc + REG_SIZE * id,               \
+		.mux_bit = 2,                                         \
+		.pull_bit = 0,                                        \
+		.drv_bit = 6,                                         \
+		.oe_bit = 9,                                          \
+		.in_bit = 0,                                          \
+		.out_bit = 1,                                         \
+		.intr_enable_bit = 0,                                 \
+		.intr_status_bit = 0,                                 \
+		.intr_target_bit = 5,                                 \
+		.intr_target_kpss_val = 3,                            \
+		.intr_raw_status_bit = 4,                             \
+		.intr_polarity_bit = 1,                               \
+		.intr_detection_bit = 2,                              \
+		.intr_detection_width = 2,                            \
+		.funcs = (int[]){                                     \
+			msm_mux_gpio, /* gpio mode */                 \
+			msm_mux_##f1,                                 \
+			msm_mux_##f2,                                 \
+			msm_mux_##f3,                                 \
+			msm_mux_##f4,                                 \
+			msm_mux_##f5,                                 \
+			msm_mux_##f6,                                 \
+			msm_mux_##f7,                                 \
+			msm_mux_##f8,                                 \
+			msm_mux_##f9,                                 \
+		},                                                    \
+		.nfuncs = 10,                                         \
+	}
+
+static const struct pinctrl_pin_desc ipq9650_pins[] = {
+	PINCTRL_PIN(0, "GPIO_0"),
+	PINCTRL_PIN(1, "GPIO_1"),
+	PINCTRL_PIN(2, "GPIO_2"),
+	PINCTRL_PIN(3, "GPIO_3"),
+	PINCTRL_PIN(4, "GPIO_4"),
+	PINCTRL_PIN(5, "GPIO_5"),
+	PINCTRL_PIN(6, "GPIO_6"),
+	PINCTRL_PIN(7, "GPIO_7"),
+	PINCTRL_PIN(8, "GPIO_8"),
+	PINCTRL_PIN(9, "GPIO_9"),
+	PINCTRL_PIN(10, "GPIO_10"),
+	PINCTRL_PIN(11, "GPIO_11"),
+	PINCTRL_PIN(12, "GPIO_12"),
+	PINCTRL_PIN(13, "GPIO_13"),
+	PINCTRL_PIN(14, "GPIO_14"),
+	PINCTRL_PIN(15, "GPIO_15"),
+	PINCTRL_PIN(16, "GPIO_16"),
+	PINCTRL_PIN(17, "GPIO_17"),
+	PINCTRL_PIN(18, "GPIO_18"),
+	PINCTRL_PIN(19, "GPIO_19"),
+	PINCTRL_PIN(20, "GPIO_20"),
+	PINCTRL_PIN(21, "GPIO_21"),
+	PINCTRL_PIN(22, "GPIO_22"),
+	PINCTRL_PIN(23, "GPIO_23"),
+	PINCTRL_PIN(24, "GPIO_24"),
+	PINCTRL_PIN(25, "GPIO_25"),
+	PINCTRL_PIN(26, "GPIO_26"),
+	PINCTRL_PIN(27, "GPIO_27"),
+	PINCTRL_PIN(28, "GPIO_28"),
+	PINCTRL_PIN(29, "GPIO_29"),
+	PINCTRL_PIN(30, "GPIO_30"),
+	PINCTRL_PIN(31, "GPIO_31"),
+	PINCTRL_PIN(32, "GPIO_32"),
+	PINCTRL_PIN(33, "GPIO_33"),
+	PINCTRL_PIN(34, "GPIO_34"),
+	PINCTRL_PIN(35, "GPIO_35"),
+	PINCTRL_PIN(36, "GPIO_36"),
+	PINCTRL_PIN(37, "GPIO_37"),
+	PINCTRL_PIN(38, "GPIO_38"),
+	PINCTRL_PIN(39, "GPIO_39"),
+	PINCTRL_PIN(40, "GPIO_40"),
+	PINCTRL_PIN(41, "GPIO_41"),
+	PINCTRL_PIN(42, "GPIO_42"),
+	PINCTRL_PIN(43, "GPIO_43"),
+	PINCTRL_PIN(44, "GPIO_44"),
+	PINCTRL_PIN(45, "GPIO_45"),
+	PINCTRL_PIN(46, "GPIO_46"),
+	PINCTRL_PIN(47, "GPIO_47"),
+	PINCTRL_PIN(48, "GPIO_48"),
+	PINCTRL_PIN(49, "GPIO_49"),
+	PINCTRL_PIN(50, "GPIO_50"),
+	PINCTRL_PIN(51, "GPIO_51"),
+	PINCTRL_PIN(52, "GPIO_52"),
+	PINCTRL_PIN(53, "GPIO_53"),
+};
+
+#define DECLARE_MSM_GPIO_PINS(pin) \
+	static const unsigned int gpio##pin##_pins[] = { pin }
+DECLARE_MSM_GPIO_PINS(0);
+DECLARE_MSM_GPIO_PINS(1);
+DECLARE_MSM_GPIO_PINS(2);
+DECLARE_MSM_GPIO_PINS(3);
+DECLARE_MSM_GPIO_PINS(4);
+DECLARE_MSM_GPIO_PINS(5);
+DECLARE_MSM_GPIO_PINS(6);
+DECLARE_MSM_GPIO_PINS(7);
+DECLARE_MSM_GPIO_PINS(8);
+DECLARE_MSM_GPIO_PINS(9);
+DECLARE_MSM_GPIO_PINS(10);
+DECLARE_MSM_GPIO_PINS(11);
+DECLARE_MSM_GPIO_PINS(12);
+DECLARE_MSM_GPIO_PINS(13);
+DECLARE_MSM_GPIO_PINS(14);
+DECLARE_MSM_GPIO_PINS(15);
+DECLARE_MSM_GPIO_PINS(16);
+DECLARE_MSM_GPIO_PINS(17);
+DECLARE_MSM_GPIO_PINS(18);
+DECLARE_MSM_GPIO_PINS(19);
+DECLARE_MSM_GPIO_PINS(20);
+DECLARE_MSM_GPIO_PINS(21);
+DECLARE_MSM_GPIO_PINS(22);
+DECLARE_MSM_GPIO_PINS(23);
+DECLARE_MSM_GPIO_PINS(24);
+DECLARE_MSM_GPIO_PINS(25);
+DECLARE_MSM_GPIO_PINS(26);
+DECLARE_MSM_GPIO_PINS(27);
+DECLARE_MSM_GPIO_PINS(28);
+DECLARE_MSM_GPIO_PINS(29);
+DECLARE_MSM_GPIO_PINS(30);
+DECLARE_MSM_GPIO_PINS(31);
+DECLARE_MSM_GPIO_PINS(32);
+DECLARE_MSM_GPIO_PINS(33);
+DECLARE_MSM_GPIO_PINS(34);
+DECLARE_MSM_GPIO_PINS(35);
+DECLARE_MSM_GPIO_PINS(36);
+DECLARE_MSM_GPIO_PINS(37);
+DECLARE_MSM_GPIO_PINS(38);
+DECLARE_MSM_GPIO_PINS(39);
+DECLARE_MSM_GPIO_PINS(40);
+DECLARE_MSM_GPIO_PINS(41);
+DECLARE_MSM_GPIO_PINS(42);
+DECLARE_MSM_GPIO_PINS(43);
+DECLARE_MSM_GPIO_PINS(44);
+DECLARE_MSM_GPIO_PINS(45);
+DECLARE_MSM_GPIO_PINS(46);
+DECLARE_MSM_GPIO_PINS(47);
+DECLARE_MSM_GPIO_PINS(48);
+DECLARE_MSM_GPIO_PINS(49);
+DECLARE_MSM_GPIO_PINS(50);
+DECLARE_MSM_GPIO_PINS(51);
+DECLARE_MSM_GPIO_PINS(52);
+DECLARE_MSM_GPIO_PINS(53);
+
+enum ipq9650_functions {
+	msm_mux_atest_char_start,
+	msm_mux_atest_char_status0,
+	msm_mux_atest_char_status1,
+	msm_mux_atest_char_status2,
+	msm_mux_atest_char_status3,
+	msm_mux_atest_tic_en,
+	msm_mux_audio_pri_mclk_in0,
+	msm_mux_audio_pri_mclk_out0,
+	msm_mux_audio_pri_mclk_in1,
+	msm_mux_audio_pri_mclk_out1,
+	msm_mux_audio_pri,
+	msm_mux_audio_sec,
+	msm_mux_audio_sec_mclk_in0,
+	msm_mux_audio_sec_mclk_out0,
+	msm_mux_audio_sec_mclk_in1,
+	msm_mux_audio_sec_mclk_out1,
+	msm_mux_core_voltage_0,
+	msm_mux_core_voltage_1,
+	msm_mux_core_voltage_2,
+	msm_mux_core_voltage_3,
+	msm_mux_core_voltage_4,
+	msm_mux_cri_rng0,
+	msm_mux_cri_rng1,
+	msm_mux_cri_rng2,
+	msm_mux_dbg_out_clk,
+	msm_mux_gcc_plltest_bypassnl,
+	msm_mux_gcc_plltest_resetn,
+	msm_mux_gcc_tlmm,
+	msm_mux_gpio,
+	msm_mux_mdc_mst,
+	msm_mux_mdc_slv0,
+	msm_mux_mdc_slv1,
+	msm_mux_mdio_mst,
+	msm_mux_mdio_slv,
+	msm_mux_mdio_slv0,
+	msm_mux_mdio_slv1,
+	msm_mux_pcie0_clk_req_n,
+	msm_mux_pcie0_wake,
+	msm_mux_pcie1_clk_req_n,
+	msm_mux_pcie1_wake,
+	msm_mux_pcie2_clk_req_n,
+	msm_mux_pcie2_wake,
+	msm_mux_pcie3_clk_req_n,
+	msm_mux_pcie3_wake,
+	msm_mux_pcie4_clk_req_n,
+	msm_mux_pcie4_wake,
+	msm_mux_pll_bist_sync,
+	msm_mux_pll_test,
+	msm_mux_pwm,
+	msm_mux_qdss_cti_trig_in_a0,
+	msm_mux_qdss_cti_trig_in_a1,
+	msm_mux_qdss_cti_trig_in_b0,
+	msm_mux_qdss_cti_trig_in_b1,
+	msm_mux_qdss_cti_trig_out_a0,
+	msm_mux_qdss_cti_trig_out_a1,
+	msm_mux_qdss_cti_trig_out_b0,
+	msm_mux_qdss_cti_trig_out_b1,
+	msm_mux_qdss_traceclk_a,
+	msm_mux_qdss_tracectl_a,
+	msm_mux_qdss_tracedata_a,
+	msm_mux_qspi_data,
+	msm_mux_qspi_clk,
+	msm_mux_qspi_cs_n,
+	msm_mux_qup_se0,
+	msm_mux_qup_se1,
+	msm_mux_qup_se2,
+	msm_mux_qup_se3,
+	msm_mux_qup_se4,
+	msm_mux_qup_se5,
+	msm_mux_qup_se6,
+	msm_mux_qup_se7,
+	msm_mux_resout,
+	msm_mux_rx_los0,
+	msm_mux_rx_los1,
+	msm_mux_rx_los2,
+	msm_mux_sdc_clk,
+	msm_mux_sdc_cmd,
+	msm_mux_sdc_data,
+	msm_mux_tsens_max,
+	msm_mux_tsn,
+	msm_mux__,
+};
+
+static const char *const gpio_groups[] = {
+	"gpio0",  "gpio1",  "gpio2",  "gpio3",	"gpio4",  "gpio5",  "gpio6",
+	"gpio7",  "gpio8",  "gpio9",  "gpio10", "gpio11", "gpio12", "gpio13",
+	"gpio14", "gpio15", "gpio16", "gpio17", "gpio18", "gpio19", "gpio20",
+	"gpio21", "gpio22", "gpio23", "gpio24", "gpio25", "gpio26", "gpio27",
+	"gpio28", "gpio29", "gpio30", "gpio31", "gpio32", "gpio33", "gpio34",
+	"gpio35", "gpio36", "gpio37", "gpio38", "gpio39", "gpio40", "gpio41",
+	"gpio42", "gpio43", "gpio44", "gpio45", "gpio46", "gpio47", "gpio48",
+	"gpio49", "gpio50", "gpio51", "gpio52", "gpio53",
+};
+
+static const char *const atest_char_start_groups[] = {
+	"gpio21",
+};
+
+static const char *const atest_char_status0_groups[] = {
+	"gpio33",
+};
+
+static const char *const atest_char_status1_groups[] = {
+	"gpio35",
+};
+
+static const char *const atest_char_status2_groups[] = {
+	"gpio22",
+};
+
+static const char *const atest_char_status3_groups[] = {
+	"gpio23",
+};
+
+static const char *const atest_tic_en_groups[] = {
+	"gpio53",
+};
+
+static const char *const audio_pri_mclk_in0_groups[] = {
+	"gpio53",
+};
+
+static const char *const audio_pri_mclk_out0_groups[] = {
+	"gpio53",
+};
+
+static const char *const audio_pri_mclk_in1_groups[] = {
+	"gpio51",
+};
+
+static const char *const audio_pri_mclk_out1_groups[] = {
+	"gpio51",
+};
+
+static const char *const audio_pri_groups[] = {
+	"gpio36", "gpio37", "gpio38", "gpio39",
+};
+
+static const char *const audio_sec_mclk_in0_groups[] = {
+	"gpio37",
+};
+
+static const char *const audio_sec_mclk_out0_groups[] = {
+	"gpio37",
+};
+
+static const char *const audio_sec_mclk_in1_groups[] = {
+	"gpio37",
+};
+
+static const char *const audio_sec_mclk_out1_groups[] = {
+	"gpio37",
+};
+
+static const char *const audio_sec_groups[] = {
+	"gpio45", "gpio46", "gpio47", "gpio48",
+};
+
+static const char *const core_voltage_0_groups[] = {
+	"gpio16",
+};
+
+static const char *const core_voltage_1_groups[] = {
+	"gpio17",
+};
+
+static const char *const core_voltage_2_groups[] = {
+	"gpio33",
+};
+
+static const char *const core_voltage_3_groups[] = {
+	"gpio34",
+};
+
+static const char *const core_voltage_4_groups[] = {
+	"gpio35",
+};
+
+static const char *const cri_rng0_groups[] = {
+	"gpio6",
+};
+
+static const char *const cri_rng1_groups[] = {
+	"gpio7",
+};
+
+static const char *const cri_rng2_groups[] = {
+	"gpio8",
+};
+
+static const char *const dbg_out_clk_groups[] = {
+	"gpio46",
+};
+
+static const char *const gcc_plltest_bypassnl_groups[] = {
+	"gpio33",
+};
+
+static const char *const gcc_plltest_resetn_groups[] = {
+	"gpio35",
+};
+
+static const char *const gcc_tlmm_groups[] = {
+	"gpio34",
+};
+
+static const char *const mdc_mst_groups[] = {
+	"gpio22",
+};
+
+static const char *const mdc_slv0_groups[] = {
+	"gpio20",
+};
+
+static const char *const mdc_slv1_groups[] = {
+	"gpio14",
+};
+
+static const char *const mdio_mst_groups[] = {
+	"gpio23",
+};
+
+static const char *const mdio_slv_groups[] = {
+	"gpio46",
+	"gpio47",
+};
+
+static const char *const mdio_slv0_groups[] = {
+	"gpio21",
+};
+
+static const char *const mdio_slv1_groups[] = {
+	"gpio15",
+};
+
+static const char *const pcie0_clk_req_n_groups[] = {
+	"gpio24",
+};
+
+static const char *const pcie0_wake_groups[] = {
+	"gpio26",
+};
+
+static const char *const pcie1_clk_req_n_groups[] = {
+	"gpio27",
+};
+
+static const char *const pcie1_wake_groups[] = {
+	"gpio29",
+};
+
+static const char *const pcie2_clk_req_n_groups[] = {
+	"gpio51",
+};
+
+static const char *const pcie2_wake_groups[] = {
+	"gpio53",
+};
+
+static const char *const pcie3_clk_req_n_groups[] = {
+	"gpio40",
+};
+
+static const char *const pcie3_wake_groups[] = {
+	"gpio42",
+};
+
+static const char *const pcie4_clk_req_n_groups[] = {
+	"gpio30",
+};
+
+static const char *const pcie4_wake_groups[] = {
+	"gpio32",
+};
+
+static const char *const pll_bist_sync_groups[] = {
+	"gpio47",
+};
+
+static const char *const pll_test_groups[] = {
+	"gpio39",
+};
+
+static const char *const pwm_groups[] = {
+	"gpio6", "gpio7", "gpio8", "gpio9", "gpio10", "gpio11", "gpio16",
+	"gpio17", "gpio33", "gpio34", "gpio35", "gpio43", "gpio44", "gpio45",
+	"gpio46", "gpio47", "gpio48",
+};
+
+static const char *const qdss_cti_trig_in_a0_groups[] = {
+	"gpio53",
+};
+
+static const char *const qdss_cti_trig_in_a1_groups[] = {
+	"gpio29",
+};
+
+static const char *const qdss_cti_trig_in_b0_groups[] = {
+	"gpio42",
+};
+
+static const char *const qdss_cti_trig_in_b1_groups[] = {
+	"gpio43",
+};
+
+static const char *const qdss_cti_trig_out_a0_groups[] = {
+	"gpio51",
+};
+
+static const char *const qdss_cti_trig_out_a1_groups[] = {
+	"gpio27",
+};
+
+static const char *const qdss_cti_trig_out_b0_groups[] = {
+	"gpio40",
+};
+
+static const char *const qdss_cti_trig_out_b1_groups[] = {
+	"gpio44",
+};
+
+static const char *const qdss_traceclk_a_groups[] = {
+	"gpio45",
+};
+
+static const char *const qdss_tracectl_a_groups[] = {
+	"gpio46",
+};
+
+static const char *const qdss_tracedata_a_groups[] = {
+	"gpio6",  "gpio7",  "gpio8",  "gpio9",	"gpio10", "gpio11",
+	"gpio12", "gpio13", "gpio14", "gpio15", "gpio20", "gpio21",
+	"gpio36", "gpio37", "gpio38", "gpio39",
+};
+
+static const char *const qspi_data_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3",
+};
+
+static const char *const qspi_clk_groups[] = {
+	"gpio5",
+};
+
+static const char *const qspi_cs_n_groups[] = {
+	"gpio4",
+};
+
+static const char *const qup_se0_groups[] = {
+	"gpio6", "gpio7", "gpio8", "gpio9", "gpio51", "gpio53",
+};
+
+static const char *const qup_se1_groups[] = {
+	"gpio10", "gpio11", "gpio12", "gpio13", "gpio27", "gpio29",
+};
+
+static const char *const qup_se2_groups[] = {
+	"gpio27", "gpio29", "gpio33", "gpio34",
+};
+
+static const char *const qup_se3_groups[] = {
+	"gpio16", "gpio17", "gpio20", "gpio21",
+};
+
+static const char *const qup_se4_groups[] = {
+	"gpio14", "gpio15", "gpio40", "gpio42", "gpio43", "gpio44",
+};
+
+static const char *const qup_se5_groups[] = {
+	"gpio40", "gpio42", "gpio45", "gpio46", "gpio47", "gpio48",
+};
+
+static const char *const qup_se6_groups[] = {
+	"gpio43", "gpio44", "gpio51", "gpio53",
+};
+
+static const char *const qup_se7_groups[] = {
+	"gpio36", "gpio37", "gpio38", "gpio39",
+};
+
+static const char *const resout_groups[] = {
+	"gpio49",
+};
+
+static const char *const rx_los0_groups[] = {
+	"gpio39", "gpio47", "gpio50",
+};
+
+static const char *const rx_los1_groups[] = {
+	"gpio38", "gpio46",
+};
+
+static const char *const rx_los2_groups[] = {
+	"gpio37", "gpio45",
+};
+
+static const char *const sdc_clk_groups[] = {
+	"gpio5",
+};
+
+static const char *const sdc_cmd_groups[] = {
+	"gpio4",
+};
+
+static const char *const sdc_data_groups[] = {
+	"gpio0", "gpio1", "gpio2", "gpio3",
+};
+
+static const char *const tsens_max_groups[] = {
+	"gpio14",
+};
+
+static const char *const tsn_groups[] = {
+	"gpio50",
+};
+
+static const struct pinfunction ipq9650_functions[] = {
+	MSM_PIN_FUNCTION(atest_char_start),
+	MSM_PIN_FUNCTION(atest_char_status0),
+	MSM_PIN_FUNCTION(atest_char_status1),
+	MSM_PIN_FUNCTION(atest_char_status2),
+	MSM_PIN_FUNCTION(atest_char_status3),
+	MSM_PIN_FUNCTION(atest_tic_en),
+	MSM_PIN_FUNCTION(audio_pri_mclk_in0),
+	MSM_PIN_FUNCTION(audio_pri_mclk_out0),
+	MSM_PIN_FUNCTION(audio_pri_mclk_in1),
+	MSM_PIN_FUNCTION(audio_pri_mclk_out1),
+	MSM_PIN_FUNCTION(audio_pri),
+	MSM_PIN_FUNCTION(audio_sec),
+	MSM_PIN_FUNCTION(audio_sec_mclk_in0),
+	MSM_PIN_FUNCTION(audio_sec_mclk_out0),
+	MSM_PIN_FUNCTION(audio_sec_mclk_in1),
+	MSM_PIN_FUNCTION(audio_sec_mclk_out1),
+	MSM_PIN_FUNCTION(core_voltage_0),
+	MSM_PIN_FUNCTION(core_voltage_1),
+	MSM_PIN_FUNCTION(core_voltage_2),
+	MSM_PIN_FUNCTION(core_voltage_3),
+	MSM_PIN_FUNCTION(core_voltage_4),
+	MSM_PIN_FUNCTION(cri_rng0),
+	MSM_PIN_FUNCTION(cri_rng1),
+	MSM_PIN_FUNCTION(cri_rng2),
+	MSM_PIN_FUNCTION(dbg_out_clk),
+	MSM_PIN_FUNCTION(gcc_plltest_bypassnl),
+	MSM_PIN_FUNCTION(gcc_plltest_resetn),
+	MSM_PIN_FUNCTION(gcc_tlmm),
+	MSM_GPIO_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(mdc_mst),
+	MSM_PIN_FUNCTION(mdc_slv0),
+	MSM_PIN_FUNCTION(mdc_slv1),
+	MSM_PIN_FUNCTION(mdio_mst),
+	MSM_PIN_FUNCTION(mdio_slv),
+	MSM_PIN_FUNCTION(mdio_slv0),
+	MSM_PIN_FUNCTION(mdio_slv1),
+	MSM_PIN_FUNCTION(pcie0_clk_req_n),
+	MSM_PIN_FUNCTION(pcie0_wake),
+	MSM_PIN_FUNCTION(pcie1_clk_req_n),
+	MSM_PIN_FUNCTION(pcie1_wake),
+	MSM_PIN_FUNCTION(pcie2_clk_req_n),
+	MSM_PIN_FUNCTION(pcie2_wake),
+	MSM_PIN_FUNCTION(pcie3_clk_req_n),
+	MSM_PIN_FUNCTION(pcie3_wake),
+	MSM_PIN_FUNCTION(pcie4_clk_req_n),
+	MSM_PIN_FUNCTION(pcie4_wake),
+	MSM_PIN_FUNCTION(pll_bist_sync),
+	MSM_PIN_FUNCTION(pll_test),
+	MSM_PIN_FUNCTION(pwm),
+	MSM_PIN_FUNCTION(qdss_cti_trig_in_a0),
+	MSM_PIN_FUNCTION(qdss_cti_trig_in_a1),
+	MSM_PIN_FUNCTION(qdss_cti_trig_in_b0),
+	MSM_PIN_FUNCTION(qdss_cti_trig_in_b1),
+	MSM_PIN_FUNCTION(qdss_cti_trig_out_a0),
+	MSM_PIN_FUNCTION(qdss_cti_trig_out_a1),
+	MSM_PIN_FUNCTION(qdss_cti_trig_out_b0),
+	MSM_PIN_FUNCTION(qdss_cti_trig_out_b1),
+	MSM_PIN_FUNCTION(qdss_traceclk_a),
+	MSM_PIN_FUNCTION(qdss_tracectl_a),
+	MSM_PIN_FUNCTION(qdss_tracedata_a),
+	MSM_PIN_FUNCTION(qspi_data),
+	MSM_PIN_FUNCTION(qspi_clk),
+	MSM_PIN_FUNCTION(qspi_cs_n),
+	MSM_PIN_FUNCTION(qup_se0),
+	MSM_PIN_FUNCTION(qup_se1),
+	MSM_PIN_FUNCTION(qup_se2),
+	MSM_PIN_FUNCTION(qup_se3),
+	MSM_PIN_FUNCTION(qup_se4),
+	MSM_PIN_FUNCTION(qup_se5),
+	MSM_PIN_FUNCTION(qup_se6),
+	MSM_PIN_FUNCTION(qup_se7),
+	MSM_PIN_FUNCTION(resout),
+	MSM_PIN_FUNCTION(rx_los0),
+	MSM_PIN_FUNCTION(rx_los1),
+	MSM_PIN_FUNCTION(rx_los2),
+	MSM_PIN_FUNCTION(sdc_clk),
+	MSM_PIN_FUNCTION(sdc_cmd),
+	MSM_PIN_FUNCTION(sdc_data),
+	MSM_PIN_FUNCTION(tsens_max),
+	MSM_PIN_FUNCTION(tsn),
+};
+
+static const struct msm_pingroup ipq9650_groups[] = {
+	[0] = PINGROUP(0, sdc_data, qspi_data, _, _, _, _, _, _, _),
+	[1] = PINGROUP(1, sdc_data, qspi_data, _, _, _, _, _, _, _),
+	[2] = PINGROUP(2, sdc_data, qspi_data, _, _, _, _, _, _, _),
+	[3] = PINGROUP(3, sdc_data, qspi_data, _, _, _, _, _, _, _),
+	[4] = PINGROUP(4, sdc_cmd, qspi_cs_n, _, _, _, _, _, _, _),
+	[5] = PINGROUP(5, sdc_clk, qspi_clk, _, _, _, _, _, _, _),
+	[6] = PINGROUP(6, qup_se0, pwm, _, cri_rng0, qdss_tracedata_a, _, _, _, _),
+	[7] = PINGROUP(7, qup_se0, pwm, _, cri_rng1, qdss_tracedata_a, _, _, _, _),
+	[8] = PINGROUP(8, qup_se0, pwm, _, cri_rng2, qdss_tracedata_a, _, _, _, _),
+	[9] = PINGROUP(9, qup_se0, pwm, _, qdss_tracedata_a, _, _, _, _, _),
+	[10] = PINGROUP(10, qup_se1, pwm, _, _, qdss_tracedata_a, _, _, _, _),
+	[11] = PINGROUP(11, qup_se1, pwm, _, _, qdss_tracedata_a, _, _, _, _),
+	[12] = PINGROUP(12, qup_se1, _, qdss_tracedata_a, _, _, _, _, _, _),
+	[13] = PINGROUP(13, qup_se1, _, qdss_tracedata_a, _, _, _, _, _, _),
+	[14] = PINGROUP(14, qup_se4, mdc_slv1, tsens_max, _, qdss_tracedata_a, _, _, _, _),
+	[15] = PINGROUP(15, qup_se4, mdio_slv1, _, qdss_tracedata_a, _, _, _, _, _),
+	[16] = PINGROUP(16, core_voltage_0, qup_se3, pwm, _, _, _, _, _, _),
+	[17] = PINGROUP(17, core_voltage_1, qup_se3, pwm, _, _, _, _, _, _),
+	[18] = PINGROUP(18, _, _, _, _, _, _, _, _, _),
+	[19] = PINGROUP(19, _, _, _, _, _, _, _, _, _),
+	[20] = PINGROUP(20, mdc_slv0, qup_se3, _, qdss_tracedata_a, _, _, _, _, _),
+	[21] = PINGROUP(21, mdio_slv0, qup_se3, atest_char_start, _, qdss_tracedata_a, _, _, _, _),
+	[22] = PINGROUP(22, mdc_mst, atest_char_status2, _, _, _, _, _, _, _),
+	[23] = PINGROUP(23, mdio_mst, atest_char_status3, _, _, _, _, _, _, _),
+	[24] = PINGROUP(24, pcie0_clk_req_n, _, _, _, _, _, _, _, _),
+	[25] = PINGROUP(25, _, _, _, _, _, _, _, _, _),
+	[26] = PINGROUP(26, pcie0_wake, _, _, _, _, _, _, _, _),
+	[27] = PINGROUP(27, pcie1_clk_req_n, qup_se2, qup_se1, _, qdss_cti_trig_out_a1, _, _, _, _),
+	[28] = PINGROUP(28, _, _, _, _, _, _, _, _, _),
+	[29] = PINGROUP(29, pcie1_wake, qup_se2, qup_se1, _, qdss_cti_trig_in_a1, _, _, _, _),
+	[30] = PINGROUP(30, pcie4_clk_req_n, _, _, _, _, _, _, _, _),
+	[31] = PINGROUP(31, _, _, _, _, _, _, _, _, _),
+	[32] = PINGROUP(32, pcie4_wake, _, _, _, _, _, _, _, _),
+	[33] = PINGROUP(33, core_voltage_2, qup_se2, gcc_plltest_bypassnl, pwm, atest_char_status0, _, _, _, _),
+	[34] = PINGROUP(34, core_voltage_3, qup_se2, gcc_tlmm, pwm, _, _, _, _, _),
+	[35] = PINGROUP(35, core_voltage_4, gcc_plltest_resetn, pwm, atest_char_status1, _, _, _, _, _),
+	[36] = PINGROUP(36, audio_pri, qup_se7, qdss_tracedata_a, _, _, _, _, _, _),
+	[37] = PINGROUP(37, audio_pri, qup_se7, audio_sec_mclk_out0, audio_sec_mclk_in0, rx_los2, qdss_tracedata_a, _, _, _),
+	[38] = PINGROUP(38, audio_pri, qup_se7, rx_los1, qdss_tracedata_a, _, _, _, _, _),
+	[39] = PINGROUP(39, audio_pri, qup_se7, audio_sec_mclk_out1, audio_sec_mclk_in1, pll_test, rx_los0, _, qdss_tracedata_a, _),
+	[40] = PINGROUP(40, pcie3_clk_req_n, qup_se5, qup_se4, _, qdss_cti_trig_out_b0, _, _, _, _),
+	[41] = PINGROUP(41, _, _, _, _, _, _, _, _, _),
+	[42] = PINGROUP(42, pcie3_wake, qup_se5, qup_se4, _, qdss_cti_trig_in_b0, _, _, _, _),
+	[43] = PINGROUP(43, qup_se4, qup_se6, pwm, _, qdss_cti_trig_in_b1, _, _, _, _),
+	[44] = PINGROUP(44, qup_se4, qup_se6, pwm, _, qdss_cti_trig_out_b1, _, _, _, _),
+	[45] = PINGROUP(45, qup_se5, rx_los2, audio_sec, pwm, _, qdss_traceclk_a, _, _, _),
+	[46] = PINGROUP(46, qup_se5, rx_los1, audio_sec, mdio_slv, pwm, dbg_out_clk, qdss_tracectl_a, _, _),
+	[47] = PINGROUP(47, qup_se5, rx_los0, audio_sec, mdio_slv, pll_bist_sync, pwm, _, _, _),
+	[48] = PINGROUP(48, qup_se5, audio_sec, pwm, _, _, _, _, _, _),
+	[49] = PINGROUP(49, resout, _, _, _, _, _, _, _, _),
+	[50] = PINGROUP(50, tsn, rx_los0, _, _, _, _, _, _, _),
+	[51] = PINGROUP(51, pcie2_clk_req_n, qup_se6, qup_se0, audio_pri_mclk_out1, audio_pri_mclk_in1, qdss_cti_trig_out_a0, _, _, _),
+	[52] = PINGROUP(52, _, _, _, _, _, _, _, _, _),
+	[53] = PINGROUP(53, pcie2_wake, qup_se6, qup_se0, audio_pri_mclk_out0, audio_pri_mclk_in0, qdss_cti_trig_in_a0, _, atest_tic_en, _),
+};
+
+static const struct msm_pinctrl_soc_data ipq9650_tlmm = {
+	.pins = ipq9650_pins,
+	.npins = ARRAY_SIZE(ipq9650_pins),
+	.functions = ipq9650_functions,
+	.nfunctions = ARRAY_SIZE(ipq9650_functions),
+	.groups = ipq9650_groups,
+	.ngroups = ARRAY_SIZE(ipq9650_groups),
+	.ngpios = 54,
+};
+
+static const struct of_device_id ipq9650_tlmm_of_match[] = {
+	{ .compatible = "qcom,ipq9650-tlmm", },
+	{},
+};
+
+static int ipq9650_tlmm_probe(struct platform_device *pdev)
+{
+	return msm_pinctrl_probe(pdev, &ipq9650_tlmm);
+}
+
+static struct platform_driver ipq9650_tlmm_driver = {
+	.driver = {
+		.name = "ipq9650-tlmm",
+		.of_match_table = ipq9650_tlmm_of_match,
+	},
+	.probe = ipq9650_tlmm_probe,
+};
+
+static int __init ipq9650_tlmm_init(void)
+{
+	return platform_driver_register(&ipq9650_tlmm_driver);
+}
+arch_initcall(ipq9650_tlmm_init);
+
+static void __exit ipq9650_tlmm_exit(void)
+{
+	platform_driver_unregister(&ipq9650_tlmm_driver);
+}
+module_exit(ipq9650_tlmm_exit);
+
+MODULE_DESCRIPTION("QTI IPQ9650 TLMM driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/qcom/pinctrl-kaanapali.c b/drivers/pinctrl/qcom/pinctrl-kaanapali.c
index 5cc45b9c55ab2..e95ac064b84ce 100644
--- a/drivers/pinctrl/qcom/pinctrl-kaanapali.c
+++ b/drivers/pinctrl/qcom/pinctrl-kaanapali.c
@@ -1774,6 +1774,7 @@ static const struct of_device_id kaanapali_tlmm_of_match[] = {
 	{ .compatible = "qcom,kaanapali-tlmm",},
 	{},
 };
+MODULE_DEVICE_TABLE(of, kaanapali_tlmm_of_match);
 
 static struct platform_driver kaanapali_tlmm_driver = {
 	.driver = {
@@ -1797,4 +1798,3 @@ module_exit(kaanapali_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI Kaanapali TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, kaanapali_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c
index 76aed32962794..15ced5027579b 100644
--- a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c
+++ b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.c
@@ -220,6 +220,8 @@ static int lpi_config_set_slew_rate(struct lpi_pinctrl *pctrl,
 
 	if (pctrl->data->flags & LPI_FLAG_SLEW_RATE_SAME_REG)
 		reg = pctrl->tlmm_base + LPI_TLMM_REG_OFFSET * group + LPI_GPIO_CFG_REG;
+	else if (g->slew_base_spare_1)
+		reg = pctrl->slew_base + LPI_SPARE_1_REG;
 	else
 		reg = pctrl->slew_base + LPI_SLEW_RATE_CTL_REG;
 
diff --git a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h
index f483684928613..6ba0c4eba984c 100644
--- a/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h
+++ b/drivers/pinctrl/qcom/pinctrl-lpass-lpi.h
@@ -16,6 +16,7 @@ struct platform_device;
 struct pinctrl_pin_desc;
 
 #define LPI_SLEW_RATE_CTL_REG	0xa000
+#define LPI_SPARE_1_REG		0xc000
 #define LPI_TLMM_REG_OFFSET		0x1000
 #define LPI_SLEW_RATE_MAX		0x03
 #define LPI_SLEW_BITS_SIZE		0x02
@@ -47,6 +48,7 @@ struct pinctrl_pin_desc;
 	{						\
 		.pin = id,				\
 		.slew_offset = soff,			\
+		.slew_base_spare_1 = false,		\
 		.funcs = (int[]){			\
 			LPI_MUX_gpio,			\
 			LPI_MUX_##f1,			\
@@ -62,6 +64,7 @@ struct pinctrl_pin_desc;
 	{							\
 		.pin = id,					\
 		.slew_offset = soff,				\
+		.slew_base_spare_1 = false,			\
 		.funcs = (int[]){				\
 			LPI_MUX_gpio,				\
 			LPI_MUX_##f1,				\
@@ -73,6 +76,22 @@ struct pinctrl_pin_desc;
 		.pin_offset = poff,				\
 	}
 
+#define LPI_PINGROUP_SLEW_SPARE_1(id, soff, f1, f2, f3, f4)	\
+	{							\
+		.pin = id,					\
+		.slew_offset = soff,				\
+		.slew_base_spare_1 = true,			\
+		.funcs = (int[]){				\
+			LPI_MUX_gpio,				\
+			LPI_MUX_##f1,				\
+			LPI_MUX_##f2,				\
+			LPI_MUX_##f3,				\
+			LPI_MUX_##f4,				\
+		},						\
+		.nfuncs = 5,					\
+		.pin_offset = 0,				\
+	}
+
 /*
  * Slew rate control is done in the same register as rest of the
  * pin configuration.
@@ -87,6 +106,7 @@ struct lpi_pingroup {
 	unsigned int *funcs;
 	unsigned int nfuncs;
 	unsigned int pin_offset;
+	bool slew_base_spare_1;
 };
 
 struct lpi_function {
diff --git a/drivers/pinctrl/qcom/pinctrl-mdm9607.c b/drivers/pinctrl/qcom/pinctrl-mdm9607.c
index 5794b0a110106..ce61eb4157556 100644
--- a/drivers/pinctrl/qcom/pinctrl-mdm9607.c
+++ b/drivers/pinctrl/qcom/pinctrl-mdm9607.c
@@ -1050,6 +1050,7 @@ static const struct of_device_id mdm9607_pinctrl_of_match[] = {
 	{ .compatible = "qcom,mdm9607-tlmm", },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, mdm9607_pinctrl_of_match);
 
 static struct platform_driver mdm9607_pinctrl_driver = {
 	.driver = {
@@ -1073,4 +1074,3 @@ module_exit(mdm9607_pinctrl_exit);
 
 MODULE_DESCRIPTION("Qualcomm mdm9607 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, mdm9607_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-mdm9615.c b/drivers/pinctrl/qcom/pinctrl-mdm9615.c
index 729fe3d7e14ef..f87e4d9a8f4f6 100644
--- a/drivers/pinctrl/qcom/pinctrl-mdm9615.c
+++ b/drivers/pinctrl/qcom/pinctrl-mdm9615.c
@@ -439,6 +439,7 @@ static const struct of_device_id mdm9615_pinctrl_of_match[] = {
 	{ .compatible = "qcom,mdm9615-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, mdm9615_pinctrl_of_match);
 
 static struct platform_driver mdm9615_pinctrl_driver = {
 	.driver = {
@@ -463,4 +464,3 @@ module_exit(mdm9615_pinctrl_exit);
 MODULE_AUTHOR("Neil Armstrong <narmstrong@baylibre.com>");
 MODULE_DESCRIPTION("Qualcomm MDM9615 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, mdm9615_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-milos.c b/drivers/pinctrl/qcom/pinctrl-milos.c
index 74b5253257af1..f5079a0ce0a3e 100644
--- a/drivers/pinctrl/qcom/pinctrl-milos.c
+++ b/drivers/pinctrl/qcom/pinctrl-milos.c
@@ -1310,6 +1310,7 @@ static const struct of_device_id milos_tlmm_of_match[] = {
 	{ .compatible = "qcom,milos-tlmm" },
 	{ /* sentinel */ }
 };
+MODULE_DEVICE_TABLE(of, milos_tlmm_of_match);
 
 static struct platform_driver milos_tlmm_driver = {
 	.driver = {
@@ -1333,4 +1334,3 @@ module_exit(milos_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI Milos TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, milos_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm.h b/drivers/pinctrl/qcom/pinctrl-msm.h
index a4af279f748ab..4fbff61de6bb3 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm.h
+++ b/drivers/pinctrl/qcom/pinctrl-msm.h
@@ -39,6 +39,11 @@ struct pinctrl_pin_desc;
 					fname##_groups,		\
 					ARRAY_SIZE(fname##_groups))
 
+#define QCA_GPIO_PIN_FUNCTION(fname)				\
+	[qca_mux_##fname] = PINCTRL_GPIO_PINFUNCTION(#fname,	\
+					fname##_groups,		\
+					ARRAY_SIZE(fname##_groups))
+
 /**
  * struct msm_pingroup - Qualcomm pingroup definition
  * @grp:                  Generic data of the pin group (name and pins)
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8226.c b/drivers/pinctrl/qcom/pinctrl-msm8226.c
index d27b7599ea832..6f02e7c2499e6 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8226.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8226.c
@@ -645,6 +645,7 @@ static const struct of_device_id msm8226_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8226-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, msm8226_pinctrl_of_match);
 
 static struct platform_driver msm8226_pinctrl_driver = {
 	.driver = {
@@ -669,4 +670,3 @@ module_exit(msm8226_pinctrl_exit);
 MODULE_AUTHOR("Bartosz Dudziak <bartosz.dudziak@snejp.pl>");
 MODULE_DESCRIPTION("Qualcomm MSM8226 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8226_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8660.c b/drivers/pinctrl/qcom/pinctrl-msm8660.c
index 5ded00396cd94..5b28a1c21a886 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8660.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8660.c
@@ -974,6 +974,7 @@ static const struct of_device_id msm8660_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8660-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, msm8660_pinctrl_of_match);
 
 static struct platform_driver msm8660_pinctrl_driver = {
 	.driver = {
@@ -998,4 +999,3 @@ module_exit(msm8660_pinctrl_exit);
 MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@sonymobile.com>");
 MODULE_DESCRIPTION("Qualcomm MSM8660 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8660_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8916.c b/drivers/pinctrl/qcom/pinctrl-msm8916.c
index 709c5d1d4d0a3..d115035ff96af 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8916.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8916.c
@@ -960,6 +960,7 @@ static const struct of_device_id msm8916_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8916-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, msm8916_pinctrl_of_match);
 
 static struct platform_driver msm8916_pinctrl_driver = {
 	.driver = {
@@ -983,4 +984,3 @@ module_exit(msm8916_pinctrl_exit);
 
 MODULE_DESCRIPTION("Qualcomm msm8916 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8916_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8953.c b/drivers/pinctrl/qcom/pinctrl-msm8953.c
index 02ea89f5feaac..d537fdaae1480 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8953.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8953.c
@@ -1807,6 +1807,7 @@ static const struct of_device_id msm8953_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8953-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, msm8953_pinctrl_of_match);
 
 static struct platform_driver msm8953_pinctrl_driver = {
 	.driver = {
@@ -1830,4 +1831,3 @@ module_exit(msm8953_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI msm8953 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8953_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8960.c b/drivers/pinctrl/qcom/pinctrl-msm8960.c
index 2fb15208aba05..a373150468ca1 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8960.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8960.c
@@ -1239,6 +1239,7 @@ static const struct of_device_id msm8960_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8960-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, msm8960_pinctrl_of_match);
 
 static struct platform_driver msm8960_pinctrl_driver = {
 	.driver = {
@@ -1263,4 +1264,3 @@ module_exit(msm8960_pinctrl_exit);
 MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@sonymobile.com>");
 MODULE_DESCRIPTION("Qualcomm MSM8960 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8960_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8976.c b/drivers/pinctrl/qcom/pinctrl-msm8976.c
index 906a90778b977..bba3c87d8144b 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8976.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8976.c
@@ -1087,6 +1087,7 @@ static const struct of_device_id msm8976_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8976-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, msm8976_pinctrl_of_match);
 
 static struct platform_driver msm8976_pinctrl_driver = {
 	.driver = {
@@ -1110,4 +1111,3 @@ module_exit(msm8976_pinctrl_exit);
 
 MODULE_DESCRIPTION("Qualcomm msm8976 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8976_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8994.c b/drivers/pinctrl/qcom/pinctrl-msm8994.c
index ecbe6b91d1da9..fdaa67c5869f3 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8994.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8994.c
@@ -1334,6 +1334,7 @@ static const struct of_device_id msm8994_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8994-pinctrl", },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, msm8994_pinctrl_of_match);
 
 static struct platform_driver msm8994_pinctrl_driver = {
 	.driver = {
@@ -1357,4 +1358,3 @@ module_exit(msm8994_pinctrl_exit);
 
 MODULE_DESCRIPTION("Qualcomm MSM8994 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8994_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8996.c b/drivers/pinctrl/qcom/pinctrl-msm8996.c
index 73b07a10a9571..332b18a8fa9ce 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8996.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8996.c
@@ -1911,6 +1911,7 @@ static const struct of_device_id msm8996_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8996-pinctrl", },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, msm8996_pinctrl_of_match);
 
 static struct platform_driver msm8996_pinctrl_driver = {
 	.driver = {
@@ -1934,4 +1935,3 @@ module_exit(msm8996_pinctrl_exit);
 
 MODULE_DESCRIPTION("Qualcomm msm8996 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8996_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8998.c b/drivers/pinctrl/qcom/pinctrl-msm8998.c
index dcf11b79e5628..0552c8212b292 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8998.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8998.c
@@ -1525,6 +1525,7 @@ static const struct of_device_id msm8998_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8998-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, msm8998_pinctrl_of_match);
 
 static struct platform_driver msm8998_pinctrl_driver = {
 	.driver = {
@@ -1548,4 +1549,3 @@ module_exit(msm8998_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI msm8998 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8998_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-msm8x74.c b/drivers/pinctrl/qcom/pinctrl-msm8x74.c
index ff432ec5815a1..9422629ec6ca7 100644
--- a/drivers/pinctrl/qcom/pinctrl-msm8x74.c
+++ b/drivers/pinctrl/qcom/pinctrl-msm8x74.c
@@ -1073,6 +1073,7 @@ static const struct of_device_id msm8x74_pinctrl_of_match[] = {
 	{ .compatible = "qcom,msm8974-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, msm8x74_pinctrl_of_match);
 
 static struct platform_driver msm8x74_pinctrl_driver = {
 	.driver = {
@@ -1097,5 +1098,4 @@ module_exit(msm8x74_pinctrl_exit);
 MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@sonymobile.com>");
 MODULE_DESCRIPTION("Qualcomm MSM8x74 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, msm8x74_pinctrl_of_match);
 
diff --git a/drivers/pinctrl/qcom/pinctrl-nord.c b/drivers/pinctrl/qcom/pinctrl-nord.c
new file mode 100644
index 0000000000000..7c21306e77ff7
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-nord.c
@@ -0,0 +1,1770 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-msm.h"
+
+#define REG_SIZE 0x1000
+#define PINGROUP(id, f1, f2, f3, f4, f5, f6, f7, f8, f9, f10, f11)    \
+	{                                                             \
+		.grp = PINCTRL_PINGROUP("gpio" #id,                   \
+					gpio##id##_pins,              \
+					ARRAY_SIZE(gpio##id##_pins)), \
+		.ctl_reg = REG_SIZE * id,                             \
+		.io_reg = 0x4 + REG_SIZE * id,                        \
+		.intr_cfg_reg = 0x8 + REG_SIZE * id,                  \
+		.intr_status_reg = 0xc + REG_SIZE * id,               \
+		.mux_bit = 2,                                         \
+		.pull_bit = 0,                                        \
+		.drv_bit = 6,                                         \
+		.egpio_enable = 12,                                   \
+		.egpio_present = 11,                                  \
+		.oe_bit = 9,                                          \
+		.in_bit = 0,                                          \
+		.out_bit = 1,                                         \
+		.intr_enable_bit = 0,                                 \
+		.intr_status_bit = 0,                                 \
+		.intr_wakeup_present_bit = 6,                         \
+		.intr_wakeup_enable_bit = 7,                          \
+		.intr_target_bit = 8,                                 \
+		.intr_target_kpss_val = 3,                            \
+		.intr_raw_status_bit = 4,                             \
+		.intr_polarity_bit = 1,                               \
+		.intr_detection_bit = 2,                              \
+		.intr_detection_width = 2,                            \
+		.funcs = (int[]){                                     \
+			msm_mux_gpio, /* gpio mode */                 \
+			msm_mux_##f1,                                 \
+			msm_mux_##f2,                                 \
+			msm_mux_##f3,                                 \
+			msm_mux_##f4,                                 \
+			msm_mux_##f5,                                 \
+			msm_mux_##f6,                                 \
+			msm_mux_##f7,                                 \
+			msm_mux_##f8,                                 \
+			msm_mux_##f9,                                 \
+			msm_mux_##f10,                                \
+			msm_mux_##f11 /* egpio mode */                \
+		},                                                    \
+		.nfuncs = 12,                                         \
+	}
+
+#define UFS_RESET(pg_name, ctl, io)                                  \
+	{                                                            \
+		.grp = PINCTRL_PINGROUP(#pg_name,                    \
+					pg_name##_pins,              \
+					ARRAY_SIZE(pg_name##_pins)), \
+		.ctl_reg = ctl,                                      \
+		.io_reg = io,                                        \
+		.intr_cfg_reg = 0,                                   \
+		.intr_status_reg = 0,                                \
+		.mux_bit = -1,                                       \
+		.pull_bit = 3,                                       \
+		.drv_bit = 0,                                        \
+		.oe_bit = -1,                                        \
+		.in_bit = -1,                                        \
+		.out_bit = 0,                                        \
+		.intr_enable_bit = -1,                               \
+		.intr_status_bit = -1,                               \
+		.intr_target_bit = -1,                               \
+		.intr_raw_status_bit = -1,                           \
+		.intr_polarity_bit = -1,                             \
+		.intr_detection_bit = -1,                            \
+		.intr_detection_width = -1,                          \
+	}
+
+static const struct pinctrl_pin_desc nord_pins[] = {
+	PINCTRL_PIN(0, "GPIO_0"),
+	PINCTRL_PIN(1, "GPIO_1"),
+	PINCTRL_PIN(2, "GPIO_2"),
+	PINCTRL_PIN(3, "GPIO_3"),
+	PINCTRL_PIN(4, "GPIO_4"),
+	PINCTRL_PIN(5, "GPIO_5"),
+	PINCTRL_PIN(6, "GPIO_6"),
+	PINCTRL_PIN(7, "GPIO_7"),
+	PINCTRL_PIN(8, "GPIO_8"),
+	PINCTRL_PIN(9, "GPIO_9"),
+	PINCTRL_PIN(10, "GPIO_10"),
+	PINCTRL_PIN(11, "GPIO_11"),
+	PINCTRL_PIN(12, "GPIO_12"),
+	PINCTRL_PIN(13, "GPIO_13"),
+	PINCTRL_PIN(14, "GPIO_14"),
+	PINCTRL_PIN(15, "GPIO_15"),
+	PINCTRL_PIN(16, "GPIO_16"),
+	PINCTRL_PIN(17, "GPIO_17"),
+	PINCTRL_PIN(18, "GPIO_18"),
+	PINCTRL_PIN(19, "GPIO_19"),
+	PINCTRL_PIN(20, "GPIO_20"),
+	PINCTRL_PIN(21, "GPIO_21"),
+	PINCTRL_PIN(22, "GPIO_22"),
+	PINCTRL_PIN(23, "GPIO_23"),
+	PINCTRL_PIN(24, "GPIO_24"),
+	PINCTRL_PIN(25, "GPIO_25"),
+	PINCTRL_PIN(26, "GPIO_26"),
+	PINCTRL_PIN(27, "GPIO_27"),
+	PINCTRL_PIN(28, "GPIO_28"),
+	PINCTRL_PIN(29, "GPIO_29"),
+	PINCTRL_PIN(30, "GPIO_30"),
+	PINCTRL_PIN(31, "GPIO_31"),
+	PINCTRL_PIN(32, "GPIO_32"),
+	PINCTRL_PIN(33, "GPIO_33"),
+	PINCTRL_PIN(34, "GPIO_34"),
+	PINCTRL_PIN(35, "GPIO_35"),
+	PINCTRL_PIN(36, "GPIO_36"),
+	PINCTRL_PIN(37, "GPIO_37"),
+	PINCTRL_PIN(38, "GPIO_38"),
+	PINCTRL_PIN(39, "GPIO_39"),
+	PINCTRL_PIN(40, "GPIO_40"),
+	PINCTRL_PIN(41, "GPIO_41"),
+	PINCTRL_PIN(42, "GPIO_42"),
+	PINCTRL_PIN(43, "GPIO_43"),
+	PINCTRL_PIN(44, "GPIO_44"),
+	PINCTRL_PIN(45, "GPIO_45"),
+	PINCTRL_PIN(46, "GPIO_46"),
+	PINCTRL_PIN(47, "GPIO_47"),
+	PINCTRL_PIN(48, "GPIO_48"),
+	PINCTRL_PIN(49, "GPIO_49"),
+	PINCTRL_PIN(50, "GPIO_50"),
+	PINCTRL_PIN(51, "GPIO_51"),
+	PINCTRL_PIN(52, "GPIO_52"),
+	PINCTRL_PIN(53, "GPIO_53"),
+	PINCTRL_PIN(54, "GPIO_54"),
+	PINCTRL_PIN(55, "GPIO_55"),
+	PINCTRL_PIN(56, "GPIO_56"),
+	PINCTRL_PIN(57, "GPIO_57"),
+	PINCTRL_PIN(58, "GPIO_58"),
+	PINCTRL_PIN(59, "GPIO_59"),
+	PINCTRL_PIN(60, "GPIO_60"),
+	PINCTRL_PIN(61, "GPIO_61"),
+	PINCTRL_PIN(62, "GPIO_62"),
+	PINCTRL_PIN(63, "GPIO_63"),
+	PINCTRL_PIN(64, "GPIO_64"),
+	PINCTRL_PIN(65, "GPIO_65"),
+	PINCTRL_PIN(66, "GPIO_66"),
+	PINCTRL_PIN(67, "GPIO_67"),
+	PINCTRL_PIN(68, "GPIO_68"),
+	PINCTRL_PIN(69, "GPIO_69"),
+	PINCTRL_PIN(70, "GPIO_70"),
+	PINCTRL_PIN(71, "GPIO_71"),
+	PINCTRL_PIN(72, "GPIO_72"),
+	PINCTRL_PIN(73, "GPIO_73"),
+	PINCTRL_PIN(74, "GPIO_74"),
+	PINCTRL_PIN(75, "GPIO_75"),
+	PINCTRL_PIN(76, "GPIO_76"),
+	PINCTRL_PIN(77, "GPIO_77"),
+	PINCTRL_PIN(78, "GPIO_78"),
+	PINCTRL_PIN(79, "GPIO_79"),
+	PINCTRL_PIN(80, "GPIO_80"),
+	PINCTRL_PIN(81, "GPIO_81"),
+	PINCTRL_PIN(82, "GPIO_82"),
+	PINCTRL_PIN(83, "GPIO_83"),
+	PINCTRL_PIN(84, "GPIO_84"),
+	PINCTRL_PIN(85, "GPIO_85"),
+	PINCTRL_PIN(86, "GPIO_86"),
+	PINCTRL_PIN(87, "GPIO_87"),
+	PINCTRL_PIN(88, "GPIO_88"),
+	PINCTRL_PIN(89, "GPIO_89"),
+	PINCTRL_PIN(90, "GPIO_90"),
+	PINCTRL_PIN(91, "GPIO_91"),
+	PINCTRL_PIN(92, "GPIO_92"),
+	PINCTRL_PIN(93, "GPIO_93"),
+	PINCTRL_PIN(94, "GPIO_94"),
+	PINCTRL_PIN(95, "GPIO_95"),
+	PINCTRL_PIN(96, "GPIO_96"),
+	PINCTRL_PIN(97, "GPIO_97"),
+	PINCTRL_PIN(98, "GPIO_98"),
+	PINCTRL_PIN(99, "GPIO_99"),
+	PINCTRL_PIN(100, "GPIO_100"),
+	PINCTRL_PIN(101, "GPIO_101"),
+	PINCTRL_PIN(102, "GPIO_102"),
+	PINCTRL_PIN(103, "GPIO_103"),
+	PINCTRL_PIN(104, "GPIO_104"),
+	PINCTRL_PIN(105, "GPIO_105"),
+	PINCTRL_PIN(106, "GPIO_106"),
+	PINCTRL_PIN(107, "GPIO_107"),
+	PINCTRL_PIN(108, "GPIO_108"),
+	PINCTRL_PIN(109, "GPIO_109"),
+	PINCTRL_PIN(110, "GPIO_110"),
+	PINCTRL_PIN(111, "GPIO_111"),
+	PINCTRL_PIN(112, "GPIO_112"),
+	PINCTRL_PIN(113, "GPIO_113"),
+	PINCTRL_PIN(114, "GPIO_114"),
+	PINCTRL_PIN(115, "GPIO_115"),
+	PINCTRL_PIN(116, "GPIO_116"),
+	PINCTRL_PIN(117, "GPIO_117"),
+	PINCTRL_PIN(118, "GPIO_118"),
+	PINCTRL_PIN(119, "GPIO_119"),
+	PINCTRL_PIN(120, "GPIO_120"),
+	PINCTRL_PIN(121, "GPIO_121"),
+	PINCTRL_PIN(122, "GPIO_122"),
+	PINCTRL_PIN(123, "GPIO_123"),
+	PINCTRL_PIN(124, "GPIO_124"),
+	PINCTRL_PIN(125, "GPIO_125"),
+	PINCTRL_PIN(126, "GPIO_126"),
+	PINCTRL_PIN(127, "GPIO_127"),
+	PINCTRL_PIN(128, "GPIO_128"),
+	PINCTRL_PIN(129, "GPIO_129"),
+	PINCTRL_PIN(130, "GPIO_130"),
+	PINCTRL_PIN(131, "GPIO_131"),
+	PINCTRL_PIN(132, "GPIO_132"),
+	PINCTRL_PIN(133, "GPIO_133"),
+	PINCTRL_PIN(134, "GPIO_134"),
+	PINCTRL_PIN(135, "GPIO_135"),
+	PINCTRL_PIN(136, "GPIO_136"),
+	PINCTRL_PIN(137, "GPIO_137"),
+	PINCTRL_PIN(138, "GPIO_138"),
+	PINCTRL_PIN(139, "GPIO_139"),
+	PINCTRL_PIN(140, "GPIO_140"),
+	PINCTRL_PIN(141, "GPIO_141"),
+	PINCTRL_PIN(142, "GPIO_142"),
+	PINCTRL_PIN(143, "GPIO_143"),
+	PINCTRL_PIN(144, "GPIO_144"),
+	PINCTRL_PIN(145, "GPIO_145"),
+	PINCTRL_PIN(146, "GPIO_146"),
+	PINCTRL_PIN(147, "GPIO_147"),
+	PINCTRL_PIN(148, "GPIO_148"),
+	PINCTRL_PIN(149, "GPIO_149"),
+	PINCTRL_PIN(150, "GPIO_150"),
+	PINCTRL_PIN(151, "GPIO_151"),
+	PINCTRL_PIN(152, "GPIO_152"),
+	PINCTRL_PIN(153, "GPIO_153"),
+	PINCTRL_PIN(154, "GPIO_154"),
+	PINCTRL_PIN(155, "GPIO_155"),
+	PINCTRL_PIN(156, "GPIO_156"),
+	PINCTRL_PIN(157, "GPIO_157"),
+	PINCTRL_PIN(158, "GPIO_158"),
+	PINCTRL_PIN(159, "GPIO_159"),
+	PINCTRL_PIN(160, "GPIO_160"),
+	PINCTRL_PIN(161, "GPIO_161"),
+	PINCTRL_PIN(162, "GPIO_162"),
+	PINCTRL_PIN(163, "GPIO_163"),
+	PINCTRL_PIN(164, "GPIO_164"),
+	PINCTRL_PIN(165, "GPIO_165"),
+	PINCTRL_PIN(166, "GPIO_166"),
+	PINCTRL_PIN(167, "GPIO_167"),
+	PINCTRL_PIN(168, "GPIO_168"),
+	PINCTRL_PIN(169, "GPIO_169"),
+	PINCTRL_PIN(170, "GPIO_170"),
+	PINCTRL_PIN(171, "GPIO_171"),
+	PINCTRL_PIN(172, "GPIO_172"),
+	PINCTRL_PIN(173, "GPIO_173"),
+	PINCTRL_PIN(174, "GPIO_174"),
+	PINCTRL_PIN(175, "GPIO_175"),
+	PINCTRL_PIN(176, "GPIO_176"),
+	PINCTRL_PIN(177, "GPIO_177"),
+	PINCTRL_PIN(178, "GPIO_178"),
+	PINCTRL_PIN(179, "GPIO_179"),
+	PINCTRL_PIN(180, "GPIO_180"),
+	PINCTRL_PIN(181, "UFS_RESET"),
+};
+
+#define DECLARE_MSM_GPIO_PINS(pin) \
+	static const unsigned int gpio##pin##_pins[] = { pin }
+DECLARE_MSM_GPIO_PINS(0);
+DECLARE_MSM_GPIO_PINS(1);
+DECLARE_MSM_GPIO_PINS(2);
+DECLARE_MSM_GPIO_PINS(3);
+DECLARE_MSM_GPIO_PINS(4);
+DECLARE_MSM_GPIO_PINS(5);
+DECLARE_MSM_GPIO_PINS(6);
+DECLARE_MSM_GPIO_PINS(7);
+DECLARE_MSM_GPIO_PINS(8);
+DECLARE_MSM_GPIO_PINS(9);
+DECLARE_MSM_GPIO_PINS(10);
+DECLARE_MSM_GPIO_PINS(11);
+DECLARE_MSM_GPIO_PINS(12);
+DECLARE_MSM_GPIO_PINS(13);
+DECLARE_MSM_GPIO_PINS(14);
+DECLARE_MSM_GPIO_PINS(15);
+DECLARE_MSM_GPIO_PINS(16);
+DECLARE_MSM_GPIO_PINS(17);
+DECLARE_MSM_GPIO_PINS(18);
+DECLARE_MSM_GPIO_PINS(19);
+DECLARE_MSM_GPIO_PINS(20);
+DECLARE_MSM_GPIO_PINS(21);
+DECLARE_MSM_GPIO_PINS(22);
+DECLARE_MSM_GPIO_PINS(23);
+DECLARE_MSM_GPIO_PINS(24);
+DECLARE_MSM_GPIO_PINS(25);
+DECLARE_MSM_GPIO_PINS(26);
+DECLARE_MSM_GPIO_PINS(27);
+DECLARE_MSM_GPIO_PINS(28);
+DECLARE_MSM_GPIO_PINS(29);
+DECLARE_MSM_GPIO_PINS(30);
+DECLARE_MSM_GPIO_PINS(31);
+DECLARE_MSM_GPIO_PINS(32);
+DECLARE_MSM_GPIO_PINS(33);
+DECLARE_MSM_GPIO_PINS(34);
+DECLARE_MSM_GPIO_PINS(35);
+DECLARE_MSM_GPIO_PINS(36);
+DECLARE_MSM_GPIO_PINS(37);
+DECLARE_MSM_GPIO_PINS(38);
+DECLARE_MSM_GPIO_PINS(39);
+DECLARE_MSM_GPIO_PINS(40);
+DECLARE_MSM_GPIO_PINS(41);
+DECLARE_MSM_GPIO_PINS(42);
+DECLARE_MSM_GPIO_PINS(43);
+DECLARE_MSM_GPIO_PINS(44);
+DECLARE_MSM_GPIO_PINS(45);
+DECLARE_MSM_GPIO_PINS(46);
+DECLARE_MSM_GPIO_PINS(47);
+DECLARE_MSM_GPIO_PINS(48);
+DECLARE_MSM_GPIO_PINS(49);
+DECLARE_MSM_GPIO_PINS(50);
+DECLARE_MSM_GPIO_PINS(51);
+DECLARE_MSM_GPIO_PINS(52);
+DECLARE_MSM_GPIO_PINS(53);
+DECLARE_MSM_GPIO_PINS(54);
+DECLARE_MSM_GPIO_PINS(55);
+DECLARE_MSM_GPIO_PINS(56);
+DECLARE_MSM_GPIO_PINS(57);
+DECLARE_MSM_GPIO_PINS(58);
+DECLARE_MSM_GPIO_PINS(59);
+DECLARE_MSM_GPIO_PINS(60);
+DECLARE_MSM_GPIO_PINS(61);
+DECLARE_MSM_GPIO_PINS(62);
+DECLARE_MSM_GPIO_PINS(63);
+DECLARE_MSM_GPIO_PINS(64);
+DECLARE_MSM_GPIO_PINS(65);
+DECLARE_MSM_GPIO_PINS(66);
+DECLARE_MSM_GPIO_PINS(67);
+DECLARE_MSM_GPIO_PINS(68);
+DECLARE_MSM_GPIO_PINS(69);
+DECLARE_MSM_GPIO_PINS(70);
+DECLARE_MSM_GPIO_PINS(71);
+DECLARE_MSM_GPIO_PINS(72);
+DECLARE_MSM_GPIO_PINS(73);
+DECLARE_MSM_GPIO_PINS(74);
+DECLARE_MSM_GPIO_PINS(75);
+DECLARE_MSM_GPIO_PINS(76);
+DECLARE_MSM_GPIO_PINS(77);
+DECLARE_MSM_GPIO_PINS(78);
+DECLARE_MSM_GPIO_PINS(79);
+DECLARE_MSM_GPIO_PINS(80);
+DECLARE_MSM_GPIO_PINS(81);
+DECLARE_MSM_GPIO_PINS(82);
+DECLARE_MSM_GPIO_PINS(83);
+DECLARE_MSM_GPIO_PINS(84);
+DECLARE_MSM_GPIO_PINS(85);
+DECLARE_MSM_GPIO_PINS(86);
+DECLARE_MSM_GPIO_PINS(87);
+DECLARE_MSM_GPIO_PINS(88);
+DECLARE_MSM_GPIO_PINS(89);
+DECLARE_MSM_GPIO_PINS(90);
+DECLARE_MSM_GPIO_PINS(91);
+DECLARE_MSM_GPIO_PINS(92);
+DECLARE_MSM_GPIO_PINS(93);
+DECLARE_MSM_GPIO_PINS(94);
+DECLARE_MSM_GPIO_PINS(95);
+DECLARE_MSM_GPIO_PINS(96);
+DECLARE_MSM_GPIO_PINS(97);
+DECLARE_MSM_GPIO_PINS(98);
+DECLARE_MSM_GPIO_PINS(99);
+DECLARE_MSM_GPIO_PINS(100);
+DECLARE_MSM_GPIO_PINS(101);
+DECLARE_MSM_GPIO_PINS(102);
+DECLARE_MSM_GPIO_PINS(103);
+DECLARE_MSM_GPIO_PINS(104);
+DECLARE_MSM_GPIO_PINS(105);
+DECLARE_MSM_GPIO_PINS(106);
+DECLARE_MSM_GPIO_PINS(107);
+DECLARE_MSM_GPIO_PINS(108);
+DECLARE_MSM_GPIO_PINS(109);
+DECLARE_MSM_GPIO_PINS(110);
+DECLARE_MSM_GPIO_PINS(111);
+DECLARE_MSM_GPIO_PINS(112);
+DECLARE_MSM_GPIO_PINS(113);
+DECLARE_MSM_GPIO_PINS(114);
+DECLARE_MSM_GPIO_PINS(115);
+DECLARE_MSM_GPIO_PINS(116);
+DECLARE_MSM_GPIO_PINS(117);
+DECLARE_MSM_GPIO_PINS(118);
+DECLARE_MSM_GPIO_PINS(119);
+DECLARE_MSM_GPIO_PINS(120);
+DECLARE_MSM_GPIO_PINS(121);
+DECLARE_MSM_GPIO_PINS(122);
+DECLARE_MSM_GPIO_PINS(123);
+DECLARE_MSM_GPIO_PINS(124);
+DECLARE_MSM_GPIO_PINS(125);
+DECLARE_MSM_GPIO_PINS(126);
+DECLARE_MSM_GPIO_PINS(127);
+DECLARE_MSM_GPIO_PINS(128);
+DECLARE_MSM_GPIO_PINS(129);
+DECLARE_MSM_GPIO_PINS(130);
+DECLARE_MSM_GPIO_PINS(131);
+DECLARE_MSM_GPIO_PINS(132);
+DECLARE_MSM_GPIO_PINS(133);
+DECLARE_MSM_GPIO_PINS(134);
+DECLARE_MSM_GPIO_PINS(135);
+DECLARE_MSM_GPIO_PINS(136);
+DECLARE_MSM_GPIO_PINS(137);
+DECLARE_MSM_GPIO_PINS(138);
+DECLARE_MSM_GPIO_PINS(139);
+DECLARE_MSM_GPIO_PINS(140);
+DECLARE_MSM_GPIO_PINS(141);
+DECLARE_MSM_GPIO_PINS(142);
+DECLARE_MSM_GPIO_PINS(143);
+DECLARE_MSM_GPIO_PINS(144);
+DECLARE_MSM_GPIO_PINS(145);
+DECLARE_MSM_GPIO_PINS(146);
+DECLARE_MSM_GPIO_PINS(147);
+DECLARE_MSM_GPIO_PINS(148);
+DECLARE_MSM_GPIO_PINS(149);
+DECLARE_MSM_GPIO_PINS(150);
+DECLARE_MSM_GPIO_PINS(151);
+DECLARE_MSM_GPIO_PINS(152);
+DECLARE_MSM_GPIO_PINS(153);
+DECLARE_MSM_GPIO_PINS(154);
+DECLARE_MSM_GPIO_PINS(155);
+DECLARE_MSM_GPIO_PINS(156);
+DECLARE_MSM_GPIO_PINS(157);
+DECLARE_MSM_GPIO_PINS(158);
+DECLARE_MSM_GPIO_PINS(159);
+DECLARE_MSM_GPIO_PINS(160);
+DECLARE_MSM_GPIO_PINS(161);
+DECLARE_MSM_GPIO_PINS(162);
+DECLARE_MSM_GPIO_PINS(163);
+DECLARE_MSM_GPIO_PINS(164);
+DECLARE_MSM_GPIO_PINS(165);
+DECLARE_MSM_GPIO_PINS(166);
+DECLARE_MSM_GPIO_PINS(167);
+DECLARE_MSM_GPIO_PINS(168);
+DECLARE_MSM_GPIO_PINS(169);
+DECLARE_MSM_GPIO_PINS(170);
+DECLARE_MSM_GPIO_PINS(171);
+DECLARE_MSM_GPIO_PINS(172);
+DECLARE_MSM_GPIO_PINS(173);
+DECLARE_MSM_GPIO_PINS(174);
+DECLARE_MSM_GPIO_PINS(175);
+DECLARE_MSM_GPIO_PINS(176);
+DECLARE_MSM_GPIO_PINS(177);
+DECLARE_MSM_GPIO_PINS(178);
+DECLARE_MSM_GPIO_PINS(179);
+DECLARE_MSM_GPIO_PINS(180);
+
+static const unsigned int ufs_reset_pins[] = { 181 };
+
+enum nord_functions {
+	msm_mux_gpio,
+	msm_mux_aoss_cti,
+	msm_mux_atest_char,
+	msm_mux_atest_usb20,
+	msm_mux_atest_usb21,
+	msm_mux_aud_intfc0_clk,
+	msm_mux_aud_intfc0_data,
+	msm_mux_aud_intfc0_ws,
+	msm_mux_aud_intfc10_clk,
+	msm_mux_aud_intfc10_data,
+	msm_mux_aud_intfc10_ws,
+	msm_mux_aud_intfc1_clk,
+	msm_mux_aud_intfc1_data,
+	msm_mux_aud_intfc1_ws,
+	msm_mux_aud_intfc2_clk,
+	msm_mux_aud_intfc2_data,
+	msm_mux_aud_intfc2_ws,
+	msm_mux_aud_intfc3_clk,
+	msm_mux_aud_intfc3_data,
+	msm_mux_aud_intfc3_ws,
+	msm_mux_aud_intfc4_clk,
+	msm_mux_aud_intfc4_data,
+	msm_mux_aud_intfc4_ws,
+	msm_mux_aud_intfc5_clk,
+	msm_mux_aud_intfc5_data,
+	msm_mux_aud_intfc5_ws,
+	msm_mux_aud_intfc6_clk,
+	msm_mux_aud_intfc6_data,
+	msm_mux_aud_intfc6_ws,
+	msm_mux_aud_intfc7_clk,
+	msm_mux_aud_intfc7_data,
+	msm_mux_aud_intfc7_ws,
+	msm_mux_aud_intfc8_clk,
+	msm_mux_aud_intfc8_data,
+	msm_mux_aud_intfc8_ws,
+	msm_mux_aud_intfc9_clk,
+	msm_mux_aud_intfc9_data,
+	msm_mux_aud_intfc9_ws,
+	msm_mux_aud_mclk0_mira,
+	msm_mux_aud_mclk0_mirb,
+	msm_mux_aud_mclk1_mira,
+	msm_mux_aud_mclk1_mirb,
+	msm_mux_aud_mclk2_mira,
+	msm_mux_aud_mclk2_mirb,
+	msm_mux_aud_refclk0,
+	msm_mux_aud_refclk1,
+	msm_mux_bist_done,
+	msm_mux_ccu_async_in,
+	msm_mux_ccu_i2c_scl,
+	msm_mux_ccu_i2c_sda,
+	msm_mux_ccu_timer,
+	msm_mux_clink_debug,
+	msm_mux_dbg_out,
+	msm_mux_dbg_out_clk,
+	msm_mux_ddr_bist_complete,
+	msm_mux_ddr_bist_fail,
+	msm_mux_ddr_bist_start,
+	msm_mux_ddr_bist_stop,
+	msm_mux_ddr_pxi,
+	msm_mux_dp_rx0,
+	msm_mux_dp_rx00,
+	msm_mux_dp_rx01,
+	msm_mux_dp_rx0_mute,
+	msm_mux_dp_rx1,
+	msm_mux_dp_rx10,
+	msm_mux_dp_rx11,
+	msm_mux_dp_rx1_mute,
+	msm_mux_edp0_hot,
+	msm_mux_edp0_lcd,
+	msm_mux_edp1_hot,
+	msm_mux_edp1_lcd,
+	msm_mux_edp2_hot,
+	msm_mux_edp2_lcd,
+	msm_mux_edp3_hot,
+	msm_mux_edp3_lcd,
+	msm_mux_emac0_mcg,
+	msm_mux_emac0_mdc,
+	msm_mux_emac0_mdio,
+	msm_mux_emac0_ptp,
+	msm_mux_emac1_mcg,
+	msm_mux_emac1_mdc,
+	msm_mux_emac1_mdio,
+	msm_mux_emac1_ptp,
+	msm_mux_gcc_gp1_clk,
+	msm_mux_gcc_gp2_clk,
+	msm_mux_gcc_gp3_clk,
+	msm_mux_gcc_gp4_clk,
+	msm_mux_gcc_gp5_clk,
+	msm_mux_gcc_gp6_clk,
+	msm_mux_gcc_gp7_clk,
+	msm_mux_gcc_gp8_clk,
+	msm_mux_jitter_bist,
+	msm_mux_lbist_pass,
+	msm_mux_mbist_pass,
+	msm_mux_mdp0_vsync_out,
+	msm_mux_mdp1_vsync_out,
+	msm_mux_mdp_vsync_e,
+	msm_mux_mdp_vsync_p,
+	msm_mux_mdp_vsync_s,
+	msm_mux_pcie0_clk_req_n,
+	msm_mux_pcie1_clk_req_n,
+	msm_mux_pcie2_clk_req_n,
+	msm_mux_pcie3_clk_req_n,
+	msm_mux_phase_flag,
+	msm_mux_pll_bist_sync,
+	msm_mux_pll_clk_aux,
+	msm_mux_prng_rosc0,
+	msm_mux_prng_rosc1,
+	msm_mux_pwrbrk_i_n,
+	msm_mux_qdss,
+	msm_mux_qdss_cti,
+	msm_mux_qspi,
+	msm_mux_qup0_se0,
+	msm_mux_qup0_se1,
+	msm_mux_qup0_se2,
+	msm_mux_qup0_se3,
+	msm_mux_qup0_se4,
+	msm_mux_qup0_se5,
+	msm_mux_qup1_se0,
+	msm_mux_qup1_se1,
+	msm_mux_qup1_se2,
+	msm_mux_qup1_se3,
+	msm_mux_qup1_se4,
+	msm_mux_qup1_se5,
+	msm_mux_qup1_se6,
+	msm_mux_qup2_se0,
+	msm_mux_qup2_se1,
+	msm_mux_qup2_se2,
+	msm_mux_qup2_se3,
+	msm_mux_qup2_se4,
+	msm_mux_qup2_se5,
+	msm_mux_qup2_se6,
+	msm_mux_qup3_se0_mira,
+	msm_mux_qup3_se0_mirb,
+	msm_mux_sailss_ospi,
+	msm_mux_sdc4_clk,
+	msm_mux_sdc4_cmd,
+	msm_mux_sdc4_data,
+	msm_mux_smb_alert,
+	msm_mux_smb_alert_n,
+	msm_mux_smb_clk,
+	msm_mux_smb_dat,
+	msm_mux_tb_trig_sdc4,
+	msm_mux_tmess_prng0,
+	msm_mux_tmess_prng1,
+	msm_mux_tsc_timer,
+	msm_mux_tsense_pwm,
+	msm_mux_usb0_hs,
+	msm_mux_usb0_phy_ps,
+	msm_mux_usb1_hs,
+	msm_mux_usb1_phy_ps,
+	msm_mux_usb2_hs,
+	msm_mux_usxgmii0_phy,
+	msm_mux_usxgmii1_phy,
+	msm_mux_vsense_trigger_mirnat,
+	msm_mux_wcn_sw,
+	msm_mux_wcn_sw_ctrl,
+	msm_mux__,
+};
+
+static const char *const gpio_groups[] = {
+	"gpio0",   "gpio1",   "gpio2",	 "gpio3",   "gpio4",   "gpio5",
+	"gpio6",   "gpio7",   "gpio8",	 "gpio9",   "gpio10",  "gpio11",
+	"gpio12",  "gpio13",  "gpio14",	 "gpio15",  "gpio16",  "gpio17",
+	"gpio18",  "gpio19",  "gpio20",	 "gpio21",  "gpio22",  "gpio23",
+	"gpio24",  "gpio25",  "gpio26",	 "gpio27",  "gpio28",  "gpio29",
+	"gpio30",  "gpio31",  "gpio32",	 "gpio33",  "gpio34",  "gpio35",
+	"gpio36",  "gpio37",  "gpio38",	 "gpio39",  "gpio40",  "gpio41",
+	"gpio42",  "gpio43",  "gpio44",	 "gpio45",  "gpio46",  "gpio47",
+	"gpio48",  "gpio49",  "gpio50",	 "gpio51",  "gpio52",  "gpio53",
+	"gpio54",  "gpio55",  "gpio56",	 "gpio57",  "gpio58",  "gpio59",
+	"gpio60",  "gpio61",  "gpio62",	 "gpio63",  "gpio64",  "gpio65",
+	"gpio66",  "gpio67",  "gpio68",	 "gpio69",  "gpio70",  "gpio71",
+	"gpio72",  "gpio73",  "gpio74",	 "gpio75",  "gpio76",  "gpio77",
+	"gpio78",  "gpio79",  "gpio80",	 "gpio81",  "gpio82",  "gpio83",
+	"gpio84",  "gpio85",  "gpio86",	 "gpio87",  "gpio88",  "gpio89",
+	"gpio90",  "gpio91",  "gpio92",	 "gpio93",  "gpio94",  "gpio95",
+	"gpio96",  "gpio97",  "gpio98",	 "gpio99",  "gpio100", "gpio101",
+	"gpio102", "gpio103", "gpio104", "gpio105", "gpio106", "gpio107",
+	"gpio108", "gpio109", "gpio110", "gpio111", "gpio112", "gpio113",
+	"gpio114", "gpio115", "gpio116", "gpio117", "gpio118", "gpio119",
+	"gpio120", "gpio121", "gpio122", "gpio123", "gpio124", "gpio125",
+	"gpio126", "gpio127", "gpio128", "gpio129", "gpio130", "gpio131",
+	"gpio132", "gpio133", "gpio134", "gpio135", "gpio136", "gpio137",
+	"gpio138", "gpio139", "gpio140", "gpio141", "gpio142", "gpio143",
+	"gpio144", "gpio145", "gpio146", "gpio147", "gpio148", "gpio149",
+	"gpio150", "gpio151", "gpio152", "gpio153", "gpio154", "gpio155",
+	"gpio156", "gpio157", "gpio158", "gpio159", "gpio160", "gpio161",
+	"gpio162", "gpio163", "gpio164", "gpio165", "gpio166", "gpio167",
+	"gpio168", "gpio169", "gpio170", "gpio171", "gpio172", "gpio173",
+	"gpio174", "gpio175", "gpio176", "gpio177", "gpio178", "gpio179",
+	"gpio180",
+};
+
+static const char *const aoss_cti_groups[] = {
+	"gpio83",
+	"gpio84",
+	"gpio85",
+	"gpio86",
+};
+
+static const char *const atest_char_groups[] = {
+	"gpio176", "gpio177", "gpio178", "gpio179", "gpio180",
+};
+
+static const char *const atest_usb20_groups[] = {
+	"gpio126",
+	"gpio128",
+	"gpio130",
+};
+
+static const char *const atest_usb21_groups[] = {
+	"gpio127",
+	"gpio129",
+	"gpio131",
+};
+
+static const char *const aud_intfc0_clk_groups[] = {
+	"gpio57",
+};
+
+static const char *const aud_intfc0_data_groups[] = {
+	"gpio59", "gpio60", "gpio61", "gpio62", "gpio63", "gpio64", "gpio65", "gpio66",
+};
+
+static const char *const aud_intfc0_ws_groups[] = {
+	"gpio58",
+};
+
+static const char *const aud_intfc10_clk_groups[] = {
+	"gpio61",
+};
+
+static const char *const aud_intfc10_data_groups[] = {
+	"gpio81", "gpio82",
+};
+
+static const char *const aud_intfc10_ws_groups[] = {
+	"gpio62",
+};
+
+static const char *const aud_intfc1_clk_groups[] = {
+	"gpio67",
+};
+
+static const char *const aud_intfc1_data_groups[] = {
+	"gpio69", "gpio70", "gpio71", "gpio72", "gpio73", "gpio74", "gpio75", "gpio76",
+};
+
+static const char *const aud_intfc1_ws_groups[] = {
+	"gpio68",
+};
+
+static const char *const aud_intfc2_clk_groups[] = {
+	"gpio77",
+};
+
+static const char *const aud_intfc2_data_groups[] = {
+	"gpio79", "gpio80", "gpio81", "gpio82",
+};
+
+static const char *const aud_intfc2_ws_groups[] = {
+	"gpio78",
+};
+
+static const char *const aud_intfc3_clk_groups[] = {
+	"gpio83",
+};
+
+static const char *const aud_intfc3_data_groups[] = {
+	"gpio85", "gpio86",
+};
+
+static const char *const aud_intfc3_ws_groups[] = {
+	"gpio84",
+};
+
+static const char *const aud_intfc4_clk_groups[] = {
+	"gpio87",
+};
+
+static const char *const aud_intfc4_data_groups[] = {
+	"gpio89", "gpio90",
+};
+
+static const char *const aud_intfc4_ws_groups[] = {
+	"gpio88",
+};
+
+static const char *const aud_intfc5_clk_groups[] = {
+	"gpio91",
+};
+
+static const char *const aud_intfc5_data_groups[] = {
+	"gpio93", "gpio94",
+};
+
+static const char *const aud_intfc5_ws_groups[] = {
+	"gpio92",
+};
+
+static const char *const aud_intfc6_clk_groups[] = {
+	"gpio95",
+};
+
+static const char *const aud_intfc6_data_groups[] = {
+	"gpio97", "gpio98",
+};
+
+static const char *const aud_intfc6_ws_groups[] = {
+	"gpio96",
+};
+
+static const char *const aud_intfc7_clk_groups[] = {
+	"gpio63",
+};
+
+static const char *const aud_intfc7_data_groups[] = {
+	"gpio65", "gpio66",
+};
+
+static const char *const aud_intfc7_ws_groups[] = {
+	"gpio64",
+};
+
+static const char *const aud_intfc8_clk_groups[] = {
+	"gpio73",
+};
+
+static const char *const aud_intfc8_data_groups[] = {
+	"gpio75", "gpio76",
+};
+
+static const char *const aud_intfc8_ws_groups[] = {
+	"gpio74",
+};
+
+static const char *const aud_intfc9_clk_groups[] = {
+	"gpio70",
+};
+
+static const char *const aud_intfc9_data_groups[] = {
+	"gpio72",
+};
+
+static const char *const aud_intfc9_ws_groups[] = {
+	"gpio71",
+};
+
+static const char *const aud_mclk0_mira_groups[] = {
+	"gpio99",
+};
+
+static const char *const aud_mclk0_mirb_groups[] = {
+	"gpio86",
+};
+
+static const char *const aud_mclk1_mira_groups[] = {
+	"gpio100",
+};
+
+static const char *const aud_mclk1_mirb_groups[] = {
+	"gpio90",
+};
+
+static const char *const aud_mclk2_mira_groups[] = {
+	"gpio101",
+};
+
+static const char *const aud_mclk2_mirb_groups[] = {
+	"gpio94",
+};
+
+static const char *const aud_refclk0_groups[] = {
+	"gpio100",
+};
+
+static const char *const aud_refclk1_groups[] = {
+	"gpio101",
+};
+
+static const char *const bist_done_groups[] = {
+	"gpio168",
+};
+
+static const char *const ccu_async_in_groups[] = {
+	"gpio45", "gpio176", "gpio177", "gpio178", "gpio179", "gpio180",
+};
+
+static const char *const ccu_i2c_scl_groups[] = {
+	"gpio16", "gpio18", "gpio20", "gpio22", "gpio24",
+	"gpio114", "gpio116", "gpio126", "gpio130", "gpio132",
+};
+
+static const char *const ccu_i2c_sda_groups[] = {
+	"gpio15", "gpio17", "gpio19", "gpio21", "gpio23",
+	"gpio113", "gpio115", "gpio125", "gpio129", "gpio131",
+};
+
+static const char *const ccu_timer_groups[] = {
+	"gpio25", "gpio26", "gpio27", "gpio28", "gpio29", "gpio30",
+	"gpio31", "gpio32", "gpio33", "gpio34", "gpio143", "gpio144",
+	"gpio150", "gpio151", "gpio152", "gpio153",
+};
+
+static const char *const clink_debug_groups[] = {
+	"gpio12", "gpio13", "gpio14", "gpio51",
+	"gpio52", "gpio53", "gpio54", "gpio55",
+};
+
+static const char *const dbg_out_groups[] = {
+	"gpio113",
+};
+
+static const char *const dbg_out_clk_groups[] = {
+	"gpio165",
+};
+
+static const char *const ddr_bist_complete_groups[] = {
+	"gpio37",
+};
+
+static const char *const ddr_bist_fail_groups[] = {
+	"gpio39",
+};
+
+static const char *const ddr_bist_start_groups[] = {
+	"gpio36",
+};
+
+static const char *const ddr_bist_stop_groups[] = {
+	"gpio38",
+};
+
+static const char *const ddr_pxi_groups[] = {
+	"gpio99",  "gpio100", "gpio109", "gpio110", "gpio113", "gpio114",
+	"gpio115", "gpio116", "gpio117", "gpio118", "gpio119", "gpio120",
+	"gpio121", "gpio122", "gpio126", "gpio127", "gpio128", "gpio129",
+	"gpio130", "gpio131", "gpio132", "gpio133", "gpio134", "gpio135",
+	"gpio136", "gpio137", "gpio138", "gpio139", "gpio162", "gpio163",
+	"gpio164", "gpio165",
+};
+
+static const char *const dp_rx0_groups[] = {
+	"gpio55", "gpio83", "gpio84",  "gpio85",  "gpio86",
+	"gpio88", "gpio89", "gpio137", "gpio138",
+};
+
+static const char *const dp_rx00_groups[] = {
+	"gpio99",
+};
+
+static const char *const dp_rx01_groups[] = {
+	"gpio100",
+};
+
+static const char *const dp_rx0_mute_groups[] = {
+	"gpio35",
+};
+
+static const char *const dp_rx1_groups[] = {
+	"gpio56", "gpio92", "gpio93",  "gpio95",  "gpio96",
+	"gpio97", "gpio98", "gpio158", "gpio159",
+};
+
+static const char *const dp_rx10_groups[] = {
+	"gpio121",
+};
+
+static const char *const dp_rx11_groups[] = {
+	"gpio122",
+};
+
+static const char *const dp_rx1_mute_groups[] = {
+	"gpio36",
+};
+
+static const char *const edp0_hot_groups[] = {
+	"gpio51",
+};
+
+static const char *const edp0_lcd_groups[] = {
+	"gpio47",
+};
+
+static const char *const edp1_hot_groups[] = {
+	"gpio52",
+};
+
+static const char *const edp1_lcd_groups[] = {
+	"gpio48",
+};
+
+static const char *const edp2_hot_groups[] = {
+	"gpio53",
+};
+
+static const char *const edp2_lcd_groups[] = {
+	"gpio49",
+};
+
+static const char *const edp3_hot_groups[] = {
+	"gpio54",
+};
+
+static const char *const edp3_lcd_groups[] = {
+	"gpio50",
+};
+
+static const char *const emac0_mcg_groups[] = {
+	"gpio16", "gpio17", "gpio18", "gpio19",
+};
+
+static const char *const emac0_mdc_groups[] = {
+	"gpio47",
+};
+
+static const char *const emac0_mdio_groups[] = {
+	"gpio48",
+};
+
+static const char *const emac0_ptp_groups[] = {
+	"gpio133", "gpio134", "gpio135", "gpio136",
+	"gpio139", "gpio140", "gpio141", "gpio142",
+};
+
+static const char *const emac1_mcg_groups[] = {
+	"gpio20", "gpio21", "gpio22", "gpio23",
+};
+
+static const char *const emac1_mdc_groups[] = {
+	"gpio49",
+};
+
+static const char *const emac1_mdio_groups[] = {
+	"gpio50",
+};
+
+static const char *const emac1_ptp_groups[] = {
+	"gpio37", "gpio38", "gpio39", "gpio40",
+	"gpio41", "gpio42", "gpio43", "gpio44",
+};
+
+static const char *const gcc_gp1_clk_groups[] = {
+	"gpio51",
+};
+
+static const char *const gcc_gp2_clk_groups[] = {
+	"gpio52",
+};
+
+static const char *const gcc_gp3_clk_groups[] = {
+	"gpio42",
+};
+
+static const char *const gcc_gp4_clk_groups[] = {
+	"gpio43",
+};
+
+static const char *const gcc_gp5_clk_groups[] = {
+	"gpio105",
+};
+
+static const char *const gcc_gp6_clk_groups[] = {
+	"gpio106",
+};
+
+static const char *const gcc_gp7_clk_groups[] = {
+	"gpio13",
+};
+
+static const char *const gcc_gp8_clk_groups[] = {
+	"gpio14",
+};
+
+static const char *const jitter_bist_groups[] = {
+	"gpio123",
+	"gpio138",
+};
+
+static const char *const lbist_pass_groups[] = {
+	"gpio121",
+};
+
+static const char *const mbist_pass_groups[] = {
+	"gpio122",
+};
+
+static const char *const mdp0_vsync_out_groups[] = {
+	"gpio113", "gpio114", "gpio115", "gpio116", "gpio121", "gpio122",
+	"gpio139", "gpio140", "gpio141", "gpio142", "gpio143",
+};
+
+static const char *const mdp1_vsync_out_groups[] = {
+	"gpio123", "gpio124", "gpio125", "gpio126", "gpio129", "gpio130",
+	"gpio131", "gpio132", "gpio133", "gpio134", "gpio135",
+};
+
+static const char *const mdp_vsync_e_groups[] = {
+	"gpio109",
+};
+
+static const char *const mdp_vsync_p_groups[] = {
+	"gpio110",
+};
+
+static const char *const mdp_vsync_s_groups[] = {
+	"gpio144",
+};
+
+static const char *const pcie0_clk_req_n_groups[] = {
+	"gpio1",
+};
+
+static const char *const pcie1_clk_req_n_groups[] = {
+	"gpio4",
+};
+
+static const char *const pcie2_clk_req_n_groups[] = {
+	"gpio7",
+};
+
+static const char *const pcie3_clk_req_n_groups[] = {
+	"gpio10",
+};
+
+static const char *const phase_flag_groups[] = {
+	"gpio67", "gpio68", "gpio69", "gpio70", "gpio71", "gpio72", "gpio73", "gpio74",
+	"gpio75", "gpio76", "gpio77", "gpio78", "gpio79", "gpio80", "gpio81", "gpio82",
+	"gpio83", "gpio84", "gpio85", "gpio86", "gpio87", "gpio88", "gpio89", "gpio90",
+	"gpio91", "gpio92", "gpio93", "gpio94", "gpio95", "gpio96", "gpio98", "gpio101",
+};
+
+static const char *const pll_bist_sync_groups[] = {
+	"gpio176",
+};
+
+static const char *const pll_clk_aux_groups[] = {
+	"gpio100",
+};
+
+static const char *const prng_rosc0_groups[] = {
+	"gpio117",
+};
+
+static const char *const prng_rosc1_groups[] = {
+	"gpio118",
+};
+
+static const char *const pwrbrk_i_n_groups[] = {
+	"gpio167",
+};
+
+static const char *const qdss_cti_groups[] = {
+	"gpio41",  "gpio42",  "gpio110", "gpio138",
+	"gpio142", "gpio144", "gpio162", "gpio163",
+};
+
+static const char *const qdss_groups[] = {
+	"gpio67", "gpio68", "gpio69", "gpio70", "gpio71", "gpio72", "gpio73", "gpio74",
+	"gpio75", "gpio76", "gpio77", "gpio78", "gpio79", "gpio80", "gpio81", "gpio82",
+	"gpio83", "gpio84", "gpio85", "gpio86", "gpio87", "gpio88", "gpio89", "gpio90",
+	"gpio91", "gpio92", "gpio93", "gpio94", "gpio95", "gpio96", "gpio97", "gpio98",
+	"gpio99", "gpio100", "gpio101", "gpio108",
+};
+
+static const char *const qspi_groups[] = {
+	"gpio102", "gpio103", "gpio104", "gpio105", "gpio106", "gpio107", "gpio108",
+};
+
+static const char *const qup0_se0_groups[] = {
+	"gpio109", "gpio110", "gpio111", "gpio112",
+};
+
+static const char *const qup0_se1_groups[] = {
+	"gpio109", "gpio110", "gpio111", "gpio112",
+};
+
+static const char *const qup0_se2_groups[] = {
+	"gpio113", "gpio114", "gpio115", "gpio116",
+};
+
+static const char *const qup0_se3_groups[] = {
+	"gpio113", "gpio114", "gpio115", "gpio116",
+};
+
+static const char *const qup0_se4_groups[] = {
+	"gpio117", "gpio118", "gpio119", "gpio120",
+};
+
+static const char *const qup0_se5_groups[] = {
+	"gpio109", "gpio110", "gpio121", "gpio122",
+};
+
+static const char *const qup1_se0_groups[] = {
+	"gpio123", "gpio124", "gpio125", "gpio126",
+};
+
+static const char *const qup1_se1_groups[] = {
+	"gpio123", "gpio124", "gpio125", "gpio126",
+};
+
+static const char *const qup1_se2_groups[] = {
+	"gpio127", "gpio128", "gpio129", "gpio130",
+};
+
+static const char *const qup1_se3_groups[] = {
+	"gpio129", "gpio130",
+};
+
+static const char *const qup1_se4_groups[] = {
+	"gpio131", "gpio132", "gpio137", "gpio138",
+};
+
+static const char *const qup1_se5_groups[] = {
+	"gpio133", "gpio134", "gpio135", "gpio136",
+};
+
+static const char *const qup1_se6_groups[] = {
+	"gpio131", "gpio132", "gpio137", "gpio138",
+};
+
+static const char *const qup2_se0_groups[] = {
+	"gpio139", "gpio140", "gpio141", "gpio142",
+};
+
+static const char *const qup2_se1_groups[] = {
+	"gpio143", "gpio144", "gpio154", "gpio155",
+};
+
+static const char *const qup2_se2_groups[] = {
+	"gpio145", "gpio146", "gpio147", "gpio148", "gpio149",
+};
+
+static const char *const qup2_se3_groups[] = {
+	"gpio150", "gpio151", "gpio152", "gpio153",
+};
+
+static const char *const qup2_se4_groups[] = {
+	"gpio143", "gpio144", "gpio150", "gpio151",
+	"gpio152", "gpio154", "gpio155",
+};
+
+static const char *const qup2_se5_groups[] = {
+	"gpio156", "gpio157", "gpio158", "gpio159",
+};
+
+static const char *const qup2_se6_groups[] = {
+	"gpio156", "gpio157", "gpio158", "gpio159",
+};
+
+static const char *const qup3_se0_mira_groups[] = {
+	"gpio102", "gpio103", "gpio104", "gpio105",
+	"gpio106", "gpio107", "gpio108",
+};
+
+static const char *const qup3_se0_mirb_groups[] = {
+	"gpio102", "gpio103",
+};
+
+static const char *const sailss_ospi_groups[] = {
+	"gpio164",
+	"gpio165",
+};
+
+static const char *const sdc4_clk_groups[] = {
+	"gpio175",
+};
+
+static const char *const sdc4_cmd_groups[] = {
+	"gpio174",
+};
+
+static const char *const sdc4_data_groups[] = {
+	"gpio170",
+	"gpio171",
+	"gpio172",
+	"gpio173",
+};
+
+static const char *const smb_alert_groups[] = {
+	"gpio110",
+};
+
+static const char *const smb_alert_n_groups[] = {
+	"gpio109",
+};
+
+static const char *const smb_clk_groups[] = {
+	"gpio112",
+};
+
+static const char *const smb_dat_groups[] = {
+	"gpio111",
+};
+
+static const char *const tb_trig_sdc4_groups[] = {
+	"gpio169",
+};
+
+static const char *const tmess_prng0_groups[] = {
+	"gpio94",
+};
+
+static const char *const tmess_prng1_groups[] = {
+	"gpio95",
+};
+
+static const char *const tsc_timer_groups[] = {
+	"gpio25", "gpio26", "gpio27", "gpio28", "gpio29",
+	"gpio30", "gpio31", "gpio32", "gpio33", "gpio34",
+};
+
+static const char *const tsense_pwm_groups[] = {
+	"gpio43", "gpio44", "gpio45", "gpio46", "gpio47", "gpio48", "gpio49", "gpio50",
+};
+
+static const char *const usb0_hs_groups[] = {
+	"gpio12",
+};
+
+static const char *const usb0_phy_ps_groups[] = {
+	"gpio164",
+};
+
+static const char *const usb1_hs_groups[] = {
+	"gpio13",
+};
+
+static const char *const usb1_phy_ps_groups[] = {
+	"gpio165",
+};
+
+static const char *const usb2_hs_groups[] = {
+	"gpio14",
+};
+
+static const char *const usxgmii0_phy_groups[] = {
+	"gpio45",
+};
+
+static const char *const usxgmii1_phy_groups[] = {
+	"gpio46",
+};
+
+static const char *const vsense_trigger_mirnat_groups[] = {
+	"gpio132",
+};
+
+static const char *const wcn_sw_groups[] = {
+	"gpio161",
+};
+
+static const char *const wcn_sw_ctrl_groups[] = {
+	"gpio160",
+};
+
+static const struct pinfunction nord_functions[] = {
+	MSM_GPIO_PIN_FUNCTION(gpio),
+	MSM_PIN_FUNCTION(aoss_cti),
+	MSM_PIN_FUNCTION(atest_char),
+	MSM_PIN_FUNCTION(atest_usb20),
+	MSM_PIN_FUNCTION(atest_usb21),
+	MSM_PIN_FUNCTION(aud_intfc0_clk),
+	MSM_PIN_FUNCTION(aud_intfc0_data),
+	MSM_PIN_FUNCTION(aud_intfc0_ws),
+	MSM_PIN_FUNCTION(aud_intfc10_clk),
+	MSM_PIN_FUNCTION(aud_intfc10_data),
+	MSM_PIN_FUNCTION(aud_intfc10_ws),
+	MSM_PIN_FUNCTION(aud_intfc1_clk),
+	MSM_PIN_FUNCTION(aud_intfc1_data),
+	MSM_PIN_FUNCTION(aud_intfc1_ws),
+	MSM_PIN_FUNCTION(aud_intfc2_clk),
+	MSM_PIN_FUNCTION(aud_intfc2_data),
+	MSM_PIN_FUNCTION(aud_intfc2_ws),
+	MSM_PIN_FUNCTION(aud_intfc3_clk),
+	MSM_PIN_FUNCTION(aud_intfc3_data),
+	MSM_PIN_FUNCTION(aud_intfc3_ws),
+	MSM_PIN_FUNCTION(aud_intfc4_clk),
+	MSM_PIN_FUNCTION(aud_intfc4_data),
+	MSM_PIN_FUNCTION(aud_intfc4_ws),
+	MSM_PIN_FUNCTION(aud_intfc5_clk),
+	MSM_PIN_FUNCTION(aud_intfc5_data),
+	MSM_PIN_FUNCTION(aud_intfc5_ws),
+	MSM_PIN_FUNCTION(aud_intfc6_clk),
+	MSM_PIN_FUNCTION(aud_intfc6_data),
+	MSM_PIN_FUNCTION(aud_intfc6_ws),
+	MSM_PIN_FUNCTION(aud_intfc7_clk),
+	MSM_PIN_FUNCTION(aud_intfc7_data),
+	MSM_PIN_FUNCTION(aud_intfc7_ws),
+	MSM_PIN_FUNCTION(aud_intfc8_clk),
+	MSM_PIN_FUNCTION(aud_intfc8_data),
+	MSM_PIN_FUNCTION(aud_intfc8_ws),
+	MSM_PIN_FUNCTION(aud_intfc9_clk),
+	MSM_PIN_FUNCTION(aud_intfc9_data),
+	MSM_PIN_FUNCTION(aud_intfc9_ws),
+	MSM_PIN_FUNCTION(aud_mclk0_mira),
+	MSM_PIN_FUNCTION(aud_mclk0_mirb),
+	MSM_PIN_FUNCTION(aud_mclk1_mira),
+	MSM_PIN_FUNCTION(aud_mclk1_mirb),
+	MSM_PIN_FUNCTION(aud_mclk2_mira),
+	MSM_PIN_FUNCTION(aud_mclk2_mirb),
+	MSM_PIN_FUNCTION(aud_refclk0),
+	MSM_PIN_FUNCTION(aud_refclk1),
+	MSM_PIN_FUNCTION(bist_done),
+	MSM_PIN_FUNCTION(ccu_async_in),
+	MSM_PIN_FUNCTION(ccu_i2c_scl),
+	MSM_PIN_FUNCTION(ccu_i2c_sda),
+	MSM_PIN_FUNCTION(ccu_timer),
+	MSM_PIN_FUNCTION(clink_debug),
+	MSM_PIN_FUNCTION(dbg_out),
+	MSM_PIN_FUNCTION(dbg_out_clk),
+	MSM_PIN_FUNCTION(ddr_bist_complete),
+	MSM_PIN_FUNCTION(ddr_bist_fail),
+	MSM_PIN_FUNCTION(ddr_bist_start),
+	MSM_PIN_FUNCTION(ddr_bist_stop),
+	MSM_PIN_FUNCTION(ddr_pxi),
+	MSM_PIN_FUNCTION(dp_rx0),
+	MSM_PIN_FUNCTION(dp_rx00),
+	MSM_PIN_FUNCTION(dp_rx01),
+	MSM_PIN_FUNCTION(dp_rx0_mute),
+	MSM_PIN_FUNCTION(dp_rx1),
+	MSM_PIN_FUNCTION(dp_rx10),
+	MSM_PIN_FUNCTION(dp_rx11),
+	MSM_PIN_FUNCTION(dp_rx1_mute),
+	MSM_PIN_FUNCTION(edp0_hot),
+	MSM_PIN_FUNCTION(edp0_lcd),
+	MSM_PIN_FUNCTION(edp1_hot),
+	MSM_PIN_FUNCTION(edp1_lcd),
+	MSM_PIN_FUNCTION(edp2_hot),
+	MSM_PIN_FUNCTION(edp2_lcd),
+	MSM_PIN_FUNCTION(edp3_hot),
+	MSM_PIN_FUNCTION(edp3_lcd),
+	MSM_PIN_FUNCTION(emac0_mcg),
+	MSM_PIN_FUNCTION(emac0_mdc),
+	MSM_PIN_FUNCTION(emac0_mdio),
+	MSM_PIN_FUNCTION(emac0_ptp),
+	MSM_PIN_FUNCTION(emac1_mcg),
+	MSM_PIN_FUNCTION(emac1_mdc),
+	MSM_PIN_FUNCTION(emac1_mdio),
+	MSM_PIN_FUNCTION(emac1_ptp),
+	MSM_PIN_FUNCTION(gcc_gp1_clk),
+	MSM_PIN_FUNCTION(gcc_gp2_clk),
+	MSM_PIN_FUNCTION(gcc_gp3_clk),
+	MSM_PIN_FUNCTION(gcc_gp4_clk),
+	MSM_PIN_FUNCTION(gcc_gp5_clk),
+	MSM_PIN_FUNCTION(gcc_gp6_clk),
+	MSM_PIN_FUNCTION(gcc_gp7_clk),
+	MSM_PIN_FUNCTION(gcc_gp8_clk),
+	MSM_PIN_FUNCTION(jitter_bist),
+	MSM_PIN_FUNCTION(lbist_pass),
+	MSM_PIN_FUNCTION(mbist_pass),
+	MSM_PIN_FUNCTION(mdp0_vsync_out),
+	MSM_PIN_FUNCTION(mdp1_vsync_out),
+	MSM_PIN_FUNCTION(mdp_vsync_e),
+	MSM_PIN_FUNCTION(mdp_vsync_p),
+	MSM_PIN_FUNCTION(mdp_vsync_s),
+	MSM_PIN_FUNCTION(pcie0_clk_req_n),
+	MSM_PIN_FUNCTION(pcie1_clk_req_n),
+	MSM_PIN_FUNCTION(pcie2_clk_req_n),
+	MSM_PIN_FUNCTION(pcie3_clk_req_n),
+	MSM_PIN_FUNCTION(phase_flag),
+	MSM_PIN_FUNCTION(pll_bist_sync),
+	MSM_PIN_FUNCTION(pll_clk_aux),
+	MSM_PIN_FUNCTION(prng_rosc0),
+	MSM_PIN_FUNCTION(prng_rosc1),
+	MSM_PIN_FUNCTION(pwrbrk_i_n),
+	MSM_PIN_FUNCTION(qdss),
+	MSM_PIN_FUNCTION(qdss_cti),
+	MSM_PIN_FUNCTION(qspi),
+	MSM_PIN_FUNCTION(qup0_se0),
+	MSM_PIN_FUNCTION(qup0_se1),
+	MSM_PIN_FUNCTION(qup0_se2),
+	MSM_PIN_FUNCTION(qup0_se3),
+	MSM_PIN_FUNCTION(qup0_se4),
+	MSM_PIN_FUNCTION(qup0_se5),
+	MSM_PIN_FUNCTION(qup1_se0),
+	MSM_PIN_FUNCTION(qup1_se1),
+	MSM_PIN_FUNCTION(qup1_se2),
+	MSM_PIN_FUNCTION(qup1_se3),
+	MSM_PIN_FUNCTION(qup1_se4),
+	MSM_PIN_FUNCTION(qup1_se5),
+	MSM_PIN_FUNCTION(qup1_se6),
+	MSM_PIN_FUNCTION(qup2_se0),
+	MSM_PIN_FUNCTION(qup2_se1),
+	MSM_PIN_FUNCTION(qup2_se2),
+	MSM_PIN_FUNCTION(qup2_se3),
+	MSM_PIN_FUNCTION(qup2_se4),
+	MSM_PIN_FUNCTION(qup2_se5),
+	MSM_PIN_FUNCTION(qup2_se6),
+	MSM_PIN_FUNCTION(qup3_se0_mira),
+	MSM_PIN_FUNCTION(qup3_se0_mirb),
+	MSM_PIN_FUNCTION(sailss_ospi),
+	MSM_PIN_FUNCTION(sdc4_clk),
+	MSM_PIN_FUNCTION(sdc4_cmd),
+	MSM_PIN_FUNCTION(sdc4_data),
+	MSM_PIN_FUNCTION(smb_alert),
+	MSM_PIN_FUNCTION(smb_alert_n),
+	MSM_PIN_FUNCTION(smb_clk),
+	MSM_PIN_FUNCTION(smb_dat),
+	MSM_PIN_FUNCTION(tb_trig_sdc4),
+	MSM_PIN_FUNCTION(tmess_prng0),
+	MSM_PIN_FUNCTION(tmess_prng1),
+	MSM_PIN_FUNCTION(tsc_timer),
+	MSM_PIN_FUNCTION(tsense_pwm),
+	MSM_PIN_FUNCTION(usb0_hs),
+	MSM_PIN_FUNCTION(usb0_phy_ps),
+	MSM_PIN_FUNCTION(usb1_hs),
+	MSM_PIN_FUNCTION(usb1_phy_ps),
+	MSM_PIN_FUNCTION(usb2_hs),
+	MSM_PIN_FUNCTION(usxgmii0_phy),
+	MSM_PIN_FUNCTION(usxgmii1_phy),
+	MSM_PIN_FUNCTION(vsense_trigger_mirnat),
+	MSM_PIN_FUNCTION(wcn_sw),
+	MSM_PIN_FUNCTION(wcn_sw_ctrl),
+};
+
+/* Every pin is maintained as a single group, and missing or non-existing pin
+ * would be maintained as dummy group to synchronize pin group index with
+ * pin descriptor registered with pinctrl core.
+ * Clients would not be able to request these dummy pin groups.
+ */
+static const struct msm_pingroup nord_groups[] = {
+	[0] = PINGROUP(0, _, _, _, _, _, _, _, _, _, _, _),
+	[1] = PINGROUP(1, pcie0_clk_req_n, _, _, _, _, _, _, _, _, _, _),
+	[2] = PINGROUP(2, _, _, _, _, _, _, _, _, _, _, _),
+	[3] = PINGROUP(3, _, _, _, _, _, _, _, _, _, _, _),
+	[4] = PINGROUP(4, pcie1_clk_req_n, _, _, _, _, _, _, _, _, _, _),
+	[5] = PINGROUP(5, _, _, _, _, _, _, _, _, _, _, _),
+	[6] = PINGROUP(6, _, _, _, _, _, _, _, _, _, _, _),
+	[7] = PINGROUP(7, pcie2_clk_req_n, _, _, _, _, _, _, _, _, _, _),
+	[8] = PINGROUP(8, _, _, _, _, _, _, _, _, _, _, _),
+	[9] = PINGROUP(9, _, _, _, _, _, _, _, _, _, _, _),
+	[10] = PINGROUP(10, pcie3_clk_req_n, _, _, _, _, _, _, _, _, _, _),
+	[11] = PINGROUP(11, _, _, _, _, _, _, _, _, _, _, _),
+	[12] = PINGROUP(12, usb0_hs, clink_debug, _, _, _, _, _, _, _, _, _),
+	[13] = PINGROUP(13, usb1_hs, clink_debug, gcc_gp7_clk, _, _, _, _, _, _, _, _),
+	[14] = PINGROUP(14, usb2_hs, clink_debug, gcc_gp8_clk, _, _, _, _, _, _, _, _),
+	[15] = PINGROUP(15, ccu_i2c_sda, _, _, _, _, _, _, _, _, _, _),
+	[16] = PINGROUP(16, ccu_i2c_scl, emac0_mcg, _, _, _, _, _, _, _, _, _),
+	[17] = PINGROUP(17, ccu_i2c_sda, emac0_mcg, _, _, _, _, _, _, _, _, _),
+	[18] = PINGROUP(18, ccu_i2c_scl, emac0_mcg, _, _, _, _, _, _, _, _, _),
+	[19] = PINGROUP(19, ccu_i2c_sda, emac0_mcg, _, _, _, _, _, _, _, _, _),
+	[20] = PINGROUP(20, ccu_i2c_scl, emac1_mcg, _, _, _, _, _, _, _, _, _),
+	[21] = PINGROUP(21, ccu_i2c_sda, emac1_mcg, _, _, _, _, _, _, _, _, _),
+	[22] = PINGROUP(22, ccu_i2c_scl, emac1_mcg, _, _, _, _, _, _, _, _, _),
+	[23] = PINGROUP(23, ccu_i2c_sda, emac1_mcg, _, _, _, _, _, _, _, _, _),
+	[24] = PINGROUP(24, ccu_i2c_scl, _, _, _, _, _, _, _, _, _, _),
+	[25] = PINGROUP(25, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[26] = PINGROUP(26, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[27] = PINGROUP(27, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[28] = PINGROUP(28, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[29] = PINGROUP(29, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[30] = PINGROUP(30, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[31] = PINGROUP(31, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[32] = PINGROUP(32, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[33] = PINGROUP(33, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[34] = PINGROUP(34, ccu_timer, tsc_timer, _, _, _, _, _, _, _, _, _),
+	[35] = PINGROUP(35, dp_rx0_mute, _, _, _, _, _, _, _, _, _, _),
+	[36] = PINGROUP(36, dp_rx1_mute, ddr_bist_start, _, _, _, _, _, _, _, _, _),
+	[37] = PINGROUP(37, emac1_ptp, ddr_bist_complete, _, _, _, _, _, _, _, _, _),
+	[38] = PINGROUP(38, emac1_ptp, ddr_bist_stop, _, _, _, _, _, _, _, _, _),
+	[39] = PINGROUP(39, emac1_ptp, ddr_bist_fail, _, _, _, _, _, _, _, _, _),
+	[40] = PINGROUP(40, emac1_ptp, _, _, _, _, _, _, _, _, _, _),
+	[41] = PINGROUP(41, emac1_ptp, qdss_cti, _, _, _, _, _, _, _, _, _),
+	[42] = PINGROUP(42, emac1_ptp, qdss_cti, gcc_gp3_clk, _, _, _, _, _, _, _, _),
+	[43] = PINGROUP(43, emac1_ptp, gcc_gp4_clk, tsense_pwm, _, _, _, _, _, _, _, _),
+	[44] = PINGROUP(44, emac1_ptp, tsense_pwm, _, _, _, _, _, _, _, _, _),
+	[45] = PINGROUP(45, usxgmii0_phy, ccu_async_in, tsense_pwm, _, _, _, _, _, _, _, _),
+	[46] = PINGROUP(46, usxgmii1_phy, tsense_pwm, _, _, _, _, _, _, _, _, _),
+	[47] = PINGROUP(47, emac0_mdc, edp0_lcd, tsense_pwm, _, _, _, _, _, _, _, _),
+	[48] = PINGROUP(48, emac0_mdio, edp1_lcd, tsense_pwm, _, _, _, _, _, _, _, _),
+	[49] = PINGROUP(49, emac1_mdc, edp2_lcd, tsense_pwm, _, _, _, _, _, _, _, _),
+	[50] = PINGROUP(50, emac1_mdio, edp3_lcd, tsense_pwm, _, _, _, _, _, _, _, _),
+	[51] = PINGROUP(51, edp0_hot, clink_debug, gcc_gp1_clk, _, _, _, _, _, _, _, _),
+	[52] = PINGROUP(52, edp1_hot, clink_debug, gcc_gp2_clk, _, _, _, _, _, _, _, _),
+	[53] = PINGROUP(53, edp2_hot, clink_debug, _, _, _, _, _, _, _, _, _),
+	[54] = PINGROUP(54, edp3_hot, clink_debug, _, _, _, _, _, _, _, _, _),
+	[55] = PINGROUP(55, dp_rx0, clink_debug, _, _, _, _, _, _, _, _, _),
+	[56] = PINGROUP(56, dp_rx1, _, _, _, _, _, _, _, _, _, _),
+	[57] = PINGROUP(57, aud_intfc0_clk, _, _, _, _, _, _, _, _, _, _),
+	[58] = PINGROUP(58, aud_intfc0_ws, _, _, _, _, _, _, _, _, _, _),
+	[59] = PINGROUP(59, aud_intfc0_data, _, _, _, _, _, _, _, _, _, _),
+	[60] = PINGROUP(60, aud_intfc0_data, _, _, _, _, _, _, _, _, _, _),
+	[61] = PINGROUP(61, aud_intfc0_data, aud_intfc10_clk, _, _, _, _, _, _, _, _, _),
+	[62] = PINGROUP(62, aud_intfc0_data, aud_intfc10_ws, _, _, _, _, _, _, _, _, _),
+	[63] = PINGROUP(63, aud_intfc0_data, aud_intfc7_clk, _, _, _, _, _, _, _, _, _),
+	[64] = PINGROUP(64, aud_intfc0_data, aud_intfc7_ws, _, _, _, _, _, _, _, _, _),
+	[65] = PINGROUP(65, aud_intfc0_data, aud_intfc7_data, _, _, _, _, _, _, _, _, _),
+	[66] = PINGROUP(66, aud_intfc0_data, aud_intfc7_data, _, _, _, _, _, _, _, _, _),
+	[67] = PINGROUP(67, aud_intfc1_clk, phase_flag, _, qdss, _, _, _, _, _, _, _),
+	[68] = PINGROUP(68, aud_intfc1_ws, phase_flag, _, qdss, _, _, _, _, _, _, _),
+	[69] = PINGROUP(69, aud_intfc1_data, phase_flag, _, qdss, _, _, _, _, _, _, _),
+	[70] = PINGROUP(70, aud_intfc1_data, aud_intfc9_clk, phase_flag,
+			_, qdss, _, _, _, _, _, _),
+	[71] = PINGROUP(71, aud_intfc1_data, aud_intfc9_ws, phase_flag,
+			_, qdss, _, _, _, _, _, _),
+	[72] = PINGROUP(72, aud_intfc1_data, aud_intfc9_data, phase_flag,
+			_, qdss, _, _, _, _, _, _),
+	[73] = PINGROUP(73, aud_intfc1_data, aud_intfc8_clk, phase_flag,
+			_, qdss, _, _, _, _, _, _),
+	[74] = PINGROUP(74, aud_intfc1_data, aud_intfc8_ws, phase_flag,
+			_, qdss, _, _, _, _, _, _),
+	[75] = PINGROUP(75, aud_intfc1_data, aud_intfc8_data, phase_flag,
+			_, qdss, _, _, _, _, _, _),
+	[76] = PINGROUP(76, aud_intfc1_data, aud_intfc8_data, phase_flag,
+			_, qdss, _, _, _, _, _, _),
+	[77] = PINGROUP(77, aud_intfc2_clk, phase_flag, _, qdss, _, _, _, _, _, _, _),
+	[78] = PINGROUP(78, aud_intfc2_ws, phase_flag, _, qdss, _, _, _, _, _, _, _),
+	[79] = PINGROUP(79, aud_intfc2_data, phase_flag, _, qdss, _, _, _, _, _, _, _),
+	[80] = PINGROUP(80, aud_intfc2_data, phase_flag, _, _, qdss, _, _, _, _, _, _),
+	[81] = PINGROUP(81, aud_intfc2_data, aud_intfc10_data, phase_flag,
+			_, _, qdss, _, _, _, _, _),
+	[82] = PINGROUP(82, aud_intfc2_data, aud_intfc10_data, phase_flag,
+			_, qdss, _, _, _, _, _, _),
+	[83] = PINGROUP(83, aud_intfc3_clk, dp_rx0, aoss_cti, phase_flag, _, qdss,
+			_, _, _, _, _),
+	[84] = PINGROUP(84, aud_intfc3_ws, dp_rx0, aoss_cti, phase_flag, _, qdss,
+			_, _, _, _, _),
+	[85] = PINGROUP(85, aud_intfc3_data, dp_rx0, aoss_cti, phase_flag,
+			_, qdss, _, _, _, _, _),
+	[86] = PINGROUP(86, aud_intfc3_data, aud_mclk0_mirb, dp_rx0, aoss_cti, phase_flag,
+			_, qdss, _, _, _, _),
+	[87] = PINGROUP(87, aud_intfc4_clk, phase_flag, _, qdss, _, _, _, _, _, _, _),
+	[88] = PINGROUP(88, aud_intfc4_ws, dp_rx0, phase_flag, _, qdss, _, _, _, _, _, _),
+	[89] = PINGROUP(89, aud_intfc4_data, dp_rx0, phase_flag, _, qdss,
+			_, _, _, _, _, _),
+	[90] = PINGROUP(90, aud_intfc4_data, aud_mclk1_mirb, phase_flag,
+			_, qdss, _, _, _, _, _, _),
+	[91] = PINGROUP(91, aud_intfc5_clk, phase_flag, _, qdss, _, _, _, _, _, _, _),
+	[92] = PINGROUP(92, aud_intfc5_ws, dp_rx1, phase_flag, _, qdss, _, _, _, _, _, _),
+	[93] = PINGROUP(93, aud_intfc5_data, dp_rx1, phase_flag, _, qdss,
+			_, _, _, _, _, _),
+	[94] = PINGROUP(94, aud_intfc5_data, aud_mclk2_mirb, phase_flag, tmess_prng0,
+			_, qdss, _, _, _, _, _),
+	[95] = PINGROUP(95, aud_intfc6_clk, dp_rx1, phase_flag, tmess_prng1,
+			_, qdss, _, _, _, _, _),
+	[96] = PINGROUP(96, aud_intfc6_ws, dp_rx1, phase_flag, _, qdss,
+			_, _, _, _, _, _),
+	[97] = PINGROUP(97, aud_intfc6_data, dp_rx1, qdss, _, _, _, _, _, _, _, _),
+	[98] = PINGROUP(98, aud_intfc6_data, dp_rx1, phase_flag, _, qdss,
+			_, _, _, _, _, _),
+	[99] = PINGROUP(99, aud_mclk0_mira, qdss, dp_rx00, ddr_pxi, _, _, _, _, _, _, _),
+	[100] = PINGROUP(100, aud_mclk1_mira, aud_refclk0, pll_clk_aux,
+			 qdss, dp_rx01, ddr_pxi, _, _, _, _, _),
+	[101] = PINGROUP(101, aud_mclk2_mira, aud_refclk1, phase_flag, _, qdss,
+			 _, _, _, _, _, _),
+	[102] = PINGROUP(102, qspi, qup3_se0_mira, qup3_se0_mirb, _, _, _, _, _, _, _, _),
+	[103] = PINGROUP(103, qspi, qup3_se0_mira, qup3_se0_mirb, _, _, _, _, _, _, _, _),
+	[104] = PINGROUP(104, qspi, qup3_se0_mira, _, _, _, _, _, _, _, _, _),
+	[105] = PINGROUP(105, qspi, qup3_se0_mira, gcc_gp5_clk, _, _, _, _, _, _, _, _),
+	[106] = PINGROUP(106, qspi, qup3_se0_mira, gcc_gp6_clk, _, _, _, _, _, _, _, _),
+	[107] = PINGROUP(107, qspi, qup3_se0_mira, _, _, _, _, _, _, _, _, _),
+	[108] = PINGROUP(108, qspi, qup3_se0_mira, qdss, _, _, _, _, _, _, _, _),
+	[109] = PINGROUP(109, qup0_se0, qup0_se1, qup0_se5, mdp_vsync_e,
+			 smb_alert_n, _, ddr_pxi, _, _, _, _),
+	[110] = PINGROUP(110, qup0_se0, qup0_se1, qup0_se5, qdss_cti,
+			 mdp_vsync_p, smb_alert, _, ddr_pxi, _, _, _),
+	[111] = PINGROUP(111, qup0_se1, qup0_se0, smb_dat, _, _, _, _, _, _, _, _),
+	[112] = PINGROUP(112, qup0_se1, qup0_se0, smb_clk, _, _, _, _, _, _, _, _),
+	[113] = PINGROUP(113, qup0_se2, qup0_se3, ccu_i2c_sda, mdp0_vsync_out,
+			 dbg_out, ddr_pxi, _, _, _, _, _),
+	[114] = PINGROUP(114, qup0_se2, qup0_se3, ccu_i2c_scl, mdp0_vsync_out,
+			 _, ddr_pxi, _, _, _, _, _),
+	[115] = PINGROUP(115, qup0_se3, qup0_se2, ccu_i2c_sda, mdp0_vsync_out,
+			 _, ddr_pxi, _, _, _, _, _),
+	[116] = PINGROUP(116, qup0_se3, qup0_se2, ccu_i2c_scl, mdp0_vsync_out,
+			 _, ddr_pxi, _, _, _, _, _),
+	[117] = PINGROUP(117, qup0_se4, prng_rosc0, _, ddr_pxi, _, _, _, _, _, _, _),
+	[118] = PINGROUP(118, qup0_se4, prng_rosc1, _, ddr_pxi, _, _, _, _, _, _, _),
+	[119] = PINGROUP(119, qup0_se4, _, ddr_pxi, _, _, _, _, _, _, _, _),
+	[120] = PINGROUP(120, qup0_se4, _, ddr_pxi, _, _, _, _, _, _, _, _),
+	[121] = PINGROUP(121, qup0_se5, lbist_pass, mdp0_vsync_out, _, dp_rx10, ddr_pxi,
+			 _, _, _, _, _),
+	[122] = PINGROUP(122, qup0_se5, mbist_pass, mdp0_vsync_out, _, dp_rx11, ddr_pxi,
+			 _, _, _, _, _),
+	[123] = PINGROUP(123, qup1_se0, qup1_se1, mdp1_vsync_out, jitter_bist,
+			 _, _, _, _, _, _, _),
+	[124] = PINGROUP(124, qup1_se0, qup1_se1, mdp1_vsync_out, _, _, _, _, _, _, _, _),
+	[125] = PINGROUP(125, qup1_se1, qup1_se0, ccu_i2c_sda, mdp1_vsync_out,
+			 _, _, _, _, _, _, _),
+	[126] = PINGROUP(126, qup1_se1, qup1_se0, ccu_i2c_scl, mdp1_vsync_out,
+			 _, atest_usb20, ddr_pxi, _, _, _, _),
+	[127] = PINGROUP(127, qup1_se2, qup1_se2, _, atest_usb21, ddr_pxi,
+			 _, _, _, _, _, _),
+	[128] = PINGROUP(128, qup1_se2, qup1_se2, _, atest_usb20, ddr_pxi,
+			 _, _, _, _, _, _),
+	[129] = PINGROUP(129, qup1_se3, qup1_se3, ccu_i2c_sda, mdp1_vsync_out,
+			 _, atest_usb21, ddr_pxi, _, _, _, _),
+	[130] = PINGROUP(130, qup1_se3, qup1_se3, ccu_i2c_scl, mdp1_vsync_out,
+			 _, atest_usb20, ddr_pxi, _, _, _, _),
+	[131] = PINGROUP(131, qup1_se4, qup1_se6, ccu_i2c_sda, mdp1_vsync_out,
+			 _, atest_usb21, ddr_pxi, _, _, _, _),
+	[132] = PINGROUP(132, qup1_se4, qup1_se6, ccu_i2c_scl, mdp1_vsync_out,
+			 _, vsense_trigger_mirnat, ddr_pxi, _, _, _, _),
+	[133] = PINGROUP(133, qup1_se5, emac0_ptp, mdp1_vsync_out, _, ddr_pxi,
+			 _, _, _, _, _, _),
+	[134] = PINGROUP(134, qup1_se5, emac0_ptp, mdp1_vsync_out, _, ddr_pxi,
+			 _, _, _, _, _, _),
+	[135] = PINGROUP(135, qup1_se5, emac0_ptp, mdp1_vsync_out, _, ddr_pxi,
+			 _, _, _, _, _, _),
+	[136] = PINGROUP(136, qup1_se5, emac0_ptp, _, ddr_pxi, _, _, _, _, _, _, _),
+	[137] = PINGROUP(137, qup1_se6, qup1_se4, dp_rx0, _, ddr_pxi, _, _, _, _, _, _),
+	[138] = PINGROUP(138, qup1_se6, qup1_se4, dp_rx0, qdss_cti, jitter_bist, ddr_pxi,
+			 _, _, _, _, _),
+	[139] = PINGROUP(139, qup2_se0, emac0_ptp, mdp0_vsync_out, ddr_pxi,
+			 _, _, _, _, _, _, _),
+	[140] = PINGROUP(140, qup2_se0, emac0_ptp, mdp0_vsync_out, _, _, _, _, _, _, _, _),
+	[141] = PINGROUP(141, qup2_se0, emac0_ptp, mdp0_vsync_out, _, _, _, _, _, _, _, _),
+	[142] = PINGROUP(142, qup2_se0, emac0_ptp, qdss_cti, mdp0_vsync_out, _, _, _, _, _, _, _),
+	[143] = PINGROUP(143, qup2_se1, qup2_se4, ccu_timer, mdp0_vsync_out,
+			 _, _, _, _, _, _, _),
+	[144] = PINGROUP(144, qup2_se1, qup2_se4, ccu_timer, qdss_cti, mdp_vsync_s,
+			 _, _, _, _, _, _),
+	[145] = PINGROUP(145, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[146] = PINGROUP(146, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[147] = PINGROUP(147, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[148] = PINGROUP(148, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[149] = PINGROUP(149, qup2_se2, _, _, _, _, _, _, _, _, _, _),
+	[150] = PINGROUP(150, qup2_se3, qup2_se4, ccu_timer, _, _, _, _, _, _, _, _),
+	[151] = PINGROUP(151, qup2_se3, qup2_se4, ccu_timer, _, _, _, _, _, _, _, _),
+	[152] = PINGROUP(152, qup2_se3, qup2_se4, ccu_timer, _, _, _, _, _, _, _, _),
+	[153] = PINGROUP(153, qup2_se3, ccu_timer, _, _, _, _, _, _, _, _, _),
+	[154] = PINGROUP(154, qup2_se4, qup2_se1, _, _, _, _, _, _, _, _, _),
+	[155] = PINGROUP(155, qup2_se4, qup2_se1, _, _, _, _, _, _, _, _, _),
+	[156] = PINGROUP(156, qup2_se5, qup2_se6, _, _, _, _, _, _, _, _, _),
+	[157] = PINGROUP(157, qup2_se5, qup2_se6, _, _, _, _, _, _, _, _, _),
+	[158] = PINGROUP(158, qup2_se6, qup2_se5, dp_rx1, _, _, _, _, _, _, _, _),
+	[159] = PINGROUP(159, qup2_se6, qup2_se5, dp_rx1, _, _, _, _, _, _, _, _),
+	[160] = PINGROUP(160, wcn_sw_ctrl, _, _, _, _, _, _, _, _, _, _),
+	[161] = PINGROUP(161, wcn_sw, _, _, _, _, _, _, _, _, _, _),
+	[162] = PINGROUP(162, qdss_cti, _, ddr_pxi, _, _, _, _, _, _, _, _),
+	[163] = PINGROUP(163, qdss_cti, _, ddr_pxi, _, _, _, _, _, _, _, _),
+	[164] = PINGROUP(164, usb0_phy_ps, _, sailss_ospi, ddr_pxi, _, _, _, _, _, _, _),
+	[165] = PINGROUP(165, usb1_phy_ps, dbg_out_clk, sailss_ospi, ddr_pxi,
+			 _, _, _, _, _, _, _),
+	[166] = PINGROUP(166, _, _, _, _, _, _, _, _, _, _, _),
+	[167] = PINGROUP(167, pwrbrk_i_n, _, _, _, _, _, _, _, _, _, _),
+	[168] = PINGROUP(168, bist_done, _, _, _, _, _, _, _, _, _, _),
+	[169] = PINGROUP(169, tb_trig_sdc4, _, _, _, _, _, _, _, _, _, _),
+	[170] = PINGROUP(170, sdc4_data, _, _, _, _, _, _, _, _, _, _),
+	[171] = PINGROUP(171, sdc4_data, _, _, _, _, _, _, _, _, _, _),
+	[172] = PINGROUP(172, sdc4_data, _, _, _, _, _, _, _, _, _, _),
+	[173] = PINGROUP(173, sdc4_data, _, _, _, _, _, _, _, _, _, _),
+	[174] = PINGROUP(174, sdc4_cmd, _, _, _, _, _, _, _, _, _, _),
+	[175] = PINGROUP(175, sdc4_clk, _, _, _, _, _, _, _, _, _, _),
+	[176] = PINGROUP(176, ccu_async_in, pll_bist_sync, atest_char,
+			 _, _, _, _, _, _, _, _),
+	[177] = PINGROUP(177, ccu_async_in, atest_char, _, _, _, _, _, _, _, _, _),
+	[178] = PINGROUP(178, ccu_async_in, atest_char, _, _, _, _, _, _, _, _, _),
+	[179] = PINGROUP(179, ccu_async_in, atest_char, _, _, _, _, _, _, _, _, _),
+	[180] = PINGROUP(180, ccu_async_in, atest_char, _, _, _, _, _, _, _, _, _),
+	[181] = UFS_RESET(ufs_reset, 0xbd004, 0xbe000),
+};
+
+static const struct msm_gpio_wakeirq_map nord_pdc_map[] = {
+	{ 0, 67 },    { 1, 68 },    { 2, 82 },	  { 3, 69 },	{ 4, 70 },
+	{ 5, 83 },    { 6, 71 },    { 7, 72 },	  { 8, 84 },	{ 9, 73 },
+	{ 10, 119 },  { 11, 85 },   { 45, 107 },  { 46, 98 },	{ 102, 77 },
+	{ 108, 78 },  { 110, 120 }, { 114, 80 },  { 116, 81 },	{ 120, 117 },
+	{ 124, 108 }, { 126, 99 },  { 128, 100 }, { 132, 101 }, { 138, 87 },
+	{ 142, 88 },  { 144, 89 },  { 153, 90 },  { 157, 91 },	{ 159, 118 },
+	{ 160, 110 }, { 161, 79 },  { 166, 109 }, { 168, 111 },
+};
+
+static const struct msm_pinctrl_soc_data nord_tlmm = {
+	.pins = nord_pins,
+	.npins = ARRAY_SIZE(nord_pins),
+	.functions = nord_functions,
+	.nfunctions = ARRAY_SIZE(nord_functions),
+	.groups = nord_groups,
+	.ngroups = ARRAY_SIZE(nord_groups),
+	.ngpios = 182,
+	.wakeirq_map = nord_pdc_map,
+	.nwakeirq_map = ARRAY_SIZE(nord_pdc_map),
+	.egpio_func = 11,
+};
+
+static const struct of_device_id nord_tlmm_of_match[] = {
+	{ .compatible = "qcom,nord-tlmm", .data = &nord_tlmm },
+	{},
+};
+
+static int nord_tlmm_probe(struct platform_device *pdev)
+{
+	const struct msm_pinctrl_soc_data *pinctrl_data;
+	struct device *dev = &pdev->dev;
+
+	pinctrl_data = device_get_match_data(dev);
+	if (!pinctrl_data)
+		return -EINVAL;
+
+	return msm_pinctrl_probe(pdev, &nord_tlmm);
+}
+
+static struct platform_driver nord_tlmm_driver = {
+	.driver = {
+		.name = "nord-tlmm",
+		.of_match_table = nord_tlmm_of_match,
+	},
+	.probe = nord_tlmm_probe,
+};
+
+static int __init nord_tlmm_init(void)
+{
+	return platform_driver_register(&nord_tlmm_driver);
+}
+arch_initcall(nord_tlmm_init);
+
+static void __exit nord_tlmm_exit(void)
+{
+	platform_driver_unregister(&nord_tlmm_driver);
+}
+module_exit(nord_tlmm_exit);
+
+MODULE_DESCRIPTION("QTI Nord TLMM driver");
+MODULE_LICENSE("GPL");
+MODULE_DEVICE_TABLE(of, nord_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-qcm2290.c b/drivers/pinctrl/qcom/pinctrl-qcm2290.c
index 3b28ac4988859..ffec2fd6412c6 100644
--- a/drivers/pinctrl/qcom/pinctrl-qcm2290.c
+++ b/drivers/pinctrl/qcom/pinctrl-qcm2290.c
@@ -75,29 +75,6 @@
 		.intr_detection_width = -1,		\
 	}
 
-#define UFS_RESET(pg_name, offset)				\
-	{					        \
-		.grp = PINCTRL_PINGROUP(#pg_name, 	\
-			pg_name##_pins, 		\
-			ARRAY_SIZE(pg_name##_pins)),	\
-		.ctl_reg = offset,			\
-		.io_reg = offset + 0x4,			\
-		.intr_cfg_reg = 0,			\
-		.intr_status_reg = 0,			\
-		.mux_bit = -1,				\
-		.pull_bit = 3,				\
-		.drv_bit = 0,				\
-		.oe_bit = -1,				\
-		.in_bit = -1,				\
-		.out_bit = 0,				\
-		.intr_enable_bit = -1,			\
-		.intr_status_bit = -1,			\
-		.intr_target_bit = -1,			\
-		.intr_raw_status_bit = -1,		\
-		.intr_polarity_bit = -1,		\
-		.intr_detection_bit = -1,		\
-		.intr_detection_width = -1,		\
-	}
 static const struct pinctrl_pin_desc qcm2290_pins[] = {
 	PINCTRL_PIN(0, "GPIO_0"),
 	PINCTRL_PIN(1, "GPIO_1"),
@@ -1124,6 +1101,7 @@ static const struct of_device_id qcm2290_pinctrl_of_match[] = {
 	{ .compatible = "qcom,qcm2290-tlmm", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, qcm2290_pinctrl_of_match);
 
 static struct platform_driver qcm2290_pinctrl_driver = {
 	.driver = {
@@ -1147,4 +1125,3 @@ module_exit(qcm2290_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI QCM2290 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, qcm2290_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-qcs404.c b/drivers/pinctrl/qcom/pinctrl-qcs404.c
index 1048a7093b2e1..01c0f0b2839ae 100644
--- a/drivers/pinctrl/qcom/pinctrl-qcs404.c
+++ b/drivers/pinctrl/qcom/pinctrl-qcs404.c
@@ -1635,6 +1635,7 @@ static const struct of_device_id qcs404_pinctrl_of_match[] = {
 	{ .compatible = "qcom,qcs404-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, qcs404_pinctrl_of_match);
 
 static struct platform_driver qcs404_pinctrl_driver = {
 	.driver = {
@@ -1658,4 +1659,3 @@ module_exit(qcs404_pinctrl_exit);
 
 MODULE_DESCRIPTION("Qualcomm QCS404 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, qcs404_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-qcs615.c b/drivers/pinctrl/qcom/pinctrl-qcs615.c
index 0ed4332d989eb..0e7df184122b5 100644
--- a/drivers/pinctrl/qcom/pinctrl-qcs615.c
+++ b/drivers/pinctrl/qcom/pinctrl-qcs615.c
@@ -1040,11 +1040,11 @@ static const struct msm_pingroup qcs615_groups[] = {
 static const struct msm_gpio_wakeirq_map qcs615_pdc_map[] = {
 	{ 1, 45 },    { 3, 31 },    { 7, 55 },    { 9, 110 },   { 11, 34 },
 	{ 13, 33 },   { 14, 35 },   { 17, 46 },   { 19, 48 },   { 21, 83 },
-	{ 22, 36 },   { 26, 38 },   { 35, 37 },   { 39, 125 },  { 41, 47 },
-	{ 47, 49 },   { 48, 51 },   { 50, 52 },   { 51, 123 },  { 55, 56 },
+	{ 22, 36 },   { 26, 38 },   { 35, 37 },   { 39, 118 },  { 41, 47 },
+	{ 47, 49 },   { 48, 51 },   { 50, 52 },   { 51, 116 },  { 55, 56 },
 	{ 56, 57 },   { 57, 58 },   { 60, 60 },   { 71, 54 },   { 80, 73 },
 	{ 81, 64 },   { 82, 50 },   { 83, 65 },   { 84, 92 },   { 85, 99 },
-	{ 86, 67 },   { 87, 84 },   { 88, 124 },  { 89, 122 },  { 90, 69 },
+	{ 86, 67 },   { 87, 84 },   { 88, 117 },  { 89, 115 },  { 90, 69 },
 	{ 92, 88 },   { 93, 75 },   { 94, 91 },   { 95, 72 },   { 96, 82 },
 	{ 97, 74 },   { 98, 95 },   { 99, 94 },   { 100, 100 }, { 101, 40 },
 	{ 102, 93 },  { 103, 77 },  { 104, 78 },  { 105, 96 },  { 107, 97 },
@@ -1073,6 +1073,7 @@ static const struct of_device_id qcs615_tlmm_of_match[] = {
 	},
 	{},
 };
+MODULE_DEVICE_TABLE(of, qcs615_tlmm_of_match);
 
 static int qcs615_tlmm_probe(struct platform_device *pdev)
 {
@@ -1101,4 +1102,3 @@ module_exit(qcs615_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI QCS615 TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, qcs615_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-qdu1000.c b/drivers/pinctrl/qcom/pinctrl-qdu1000.c
index 5125df7eb1272..77da87aa72aa2 100644
--- a/drivers/pinctrl/qcom/pinctrl-qdu1000.c
+++ b/drivers/pinctrl/qcom/pinctrl-qdu1000.c
@@ -99,12 +99,6 @@
 		.intr_detection_width = -1,		\
 	}
 
-#define QUP_I3C(qup_mode, qup_offset)			\
-	{						\
-		.mode = qup_mode,			\
-		.offset = qup_offset,			\
-	}
-
 static const struct pinctrl_pin_desc qdu1000_pins[] = {
 	PINCTRL_PIN(0, "GPIO_0"),
 	PINCTRL_PIN(1, "GPIO_1"),
diff --git a/drivers/pinctrl/qcom/pinctrl-sc7180.c b/drivers/pinctrl/qcom/pinctrl-sc7180.c
index 01cfcb416f331..a02a9a78a557f 100644
--- a/drivers/pinctrl/qcom/pinctrl-sc7180.c
+++ b/drivers/pinctrl/qcom/pinctrl-sc7180.c
@@ -1148,6 +1148,7 @@ static const struct of_device_id sc7180_pinctrl_of_match[] = {
 	{ .compatible = "qcom,sc7180-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sc7180_pinctrl_of_match);
 
 static struct platform_driver sc7180_pinctrl_driver = {
 	.driver = {
@@ -1172,4 +1173,3 @@ module_exit(sc7180_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI sc7180 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sc7180_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sc7280.c b/drivers/pinctrl/qcom/pinctrl-sc7280.c
index f22fd56efd896..bb32a56649df6 100644
--- a/drivers/pinctrl/qcom/pinctrl-sc7280.c
+++ b/drivers/pinctrl/qcom/pinctrl-sc7280.c
@@ -1494,6 +1494,7 @@ static const struct of_device_id sc7280_pinctrl_of_match[] = {
 	{ .compatible = "qcom,sc7280-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sc7280_pinctrl_of_match);
 
 static struct platform_driver sc7280_pinctrl_driver = {
 	.driver = {
@@ -1518,4 +1519,3 @@ module_exit(sc7280_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI sc7280 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sc7280_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sdm660.c b/drivers/pinctrl/qcom/pinctrl-sdm660.c
index ab0368653d309..c4a1ec90a3417 100644
--- a/drivers/pinctrl/qcom/pinctrl-sdm660.c
+++ b/drivers/pinctrl/qcom/pinctrl-sdm660.c
@@ -1433,6 +1433,7 @@ static const struct of_device_id sdm660_pinctrl_of_match[] = {
 	{ .compatible = "qcom,sdm630-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sdm660_pinctrl_of_match);
 
 static struct platform_driver sdm660_pinctrl_driver = {
 	.driver = {
@@ -1456,4 +1457,3 @@ module_exit(sdm660_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI sdm660 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sdm660_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sdm845.c b/drivers/pinctrl/qcom/pinctrl-sdm845.c
index b5ed2311b70eb..48fbe3623e17e 100644
--- a/drivers/pinctrl/qcom/pinctrl-sdm845.c
+++ b/drivers/pinctrl/qcom/pinctrl-sdm845.c
@@ -1339,6 +1339,7 @@ static const struct of_device_id sdm845_pinctrl_of_match[] = {
 	{ .compatible = "qcom,sdm845-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sdm845_pinctrl_of_match);
 
 static struct platform_driver sdm845_pinctrl_driver = {
 	.driver = {
@@ -1364,4 +1365,3 @@ module_exit(sdm845_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI sdm845 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sdm845_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sdx55.c b/drivers/pinctrl/qcom/pinctrl-sdx55.c
index 3e87f59279245..656a0726db92d 100644
--- a/drivers/pinctrl/qcom/pinctrl-sdx55.c
+++ b/drivers/pinctrl/qcom/pinctrl-sdx55.c
@@ -981,6 +981,7 @@ static const struct of_device_id sdx55_pinctrl_of_match[] = {
 	{ .compatible = "qcom,sdx55-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sdx55_pinctrl_of_match);
 
 static struct platform_driver sdx55_pinctrl_driver = {
 	.driver = {
@@ -1004,4 +1005,3 @@ module_exit(sdx55_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI sdx55 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sdx55_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sdx65.c b/drivers/pinctrl/qcom/pinctrl-sdx65.c
index 4e787341b2a2e..dde8993ca9b15 100644
--- a/drivers/pinctrl/qcom/pinctrl-sdx65.c
+++ b/drivers/pinctrl/qcom/pinctrl-sdx65.c
@@ -929,6 +929,7 @@ static const struct of_device_id sdx65_pinctrl_of_match[] = {
 	{ .compatible = "qcom,sdx65-tlmm", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sdx65_pinctrl_of_match);
 
 static struct platform_driver sdx65_pinctrl_driver = {
 	.driver = {
@@ -952,4 +953,3 @@ module_exit(sdx65_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI sdx65 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sdx65_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm4450.c b/drivers/pinctrl/qcom/pinctrl-sm4450.c
index 83650f173b013..51a66a20dc663 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm4450.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm4450.c
@@ -99,13 +99,6 @@
 		.intr_detection_width = -1,		\
 	}
 
-#define QUP_I3C(qup_mode, qup_offset)			\
-	{						\
-		.mode = qup_mode,			\
-		.offset = qup_offset,			\
-	}
-
-
 static const struct pinctrl_pin_desc sm4450_pins[] = {
 	PINCTRL_PIN(0, "GPIO_0"),
 	PINCTRL_PIN(1, "GPIO_1"),
diff --git a/drivers/pinctrl/qcom/pinctrl-sm6115.c b/drivers/pinctrl/qcom/pinctrl-sm6115.c
index 234451fbf47bf..e36716514f1f5 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm6115.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm6115.c
@@ -897,6 +897,7 @@ static const struct of_device_id sm6115_tlmm_of_match[] = {
 	{ .compatible = "qcom,sm6115-tlmm", },
 	{ }
 };
+MODULE_DEVICE_TABLE(of, sm6115_tlmm_of_match);
 
 static struct platform_driver sm6115_tlmm_driver = {
 	.driver = {
@@ -920,4 +921,3 @@ module_exit(sm6115_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI sm6115 tlmm driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sm6115_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm6125.c b/drivers/pinctrl/qcom/pinctrl-sm6125.c
index 2cf9136860fc5..7447ef9a49317 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm6125.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm6125.c
@@ -1256,6 +1256,7 @@ static const struct of_device_id sm6125_tlmm_of_match[] = {
 	{ .compatible = "qcom,sm6125-tlmm", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sm6125_tlmm_of_match);
 
 static struct platform_driver sm6125_tlmm_driver = {
 	.driver = {
@@ -1279,4 +1280,3 @@ module_exit(sm6125_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI sm6125 TLMM driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sm6125_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm6350-lpass-lpi.c b/drivers/pinctrl/qcom/pinctrl-sm6350-lpass-lpi.c
new file mode 100644
index 0000000000000..4d06abcfedfd7
--- /dev/null
+++ b/drivers/pinctrl/qcom/pinctrl-sm6350-lpass-lpi.c
@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2026, Luca Weiss <luca.weiss@fairphone.com>
+ */
+
+#include <linux/gpio/driver.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+
+#include "pinctrl-lpass-lpi.h"
+
+enum lpass_lpi_functions {
+	LPI_MUX_dmic1_clk,
+	LPI_MUX_dmic1_data,
+	LPI_MUX_dmic2_clk,
+	LPI_MUX_dmic2_data,
+	LPI_MUX_dmic3_clk,
+	LPI_MUX_dmic3_data,
+	LPI_MUX_i2s1_clk,
+	LPI_MUX_i2s1_data,
+	LPI_MUX_i2s1_ws,
+	LPI_MUX_i2s2_clk,
+	LPI_MUX_i2s2_data,
+	LPI_MUX_i2s2_ws,
+	LPI_MUX_qua_mi2s_data,
+	LPI_MUX_qua_mi2s_sclk,
+	LPI_MUX_qua_mi2s_ws,
+	LPI_MUX_swr_rx_clk,
+	LPI_MUX_swr_rx_data,
+	LPI_MUX_swr_tx_clk,
+	LPI_MUX_swr_tx_data,
+	LPI_MUX_wsa_swr_clk,
+	LPI_MUX_wsa_swr_data,
+	LPI_MUX_gpio,
+	LPI_MUX__,
+};
+
+static const struct pinctrl_pin_desc sm6350_lpi_pins[] = {
+	PINCTRL_PIN(0, "gpio0"),
+	PINCTRL_PIN(1, "gpio1"),
+	PINCTRL_PIN(2, "gpio2"),
+	PINCTRL_PIN(3, "gpio3"),
+	PINCTRL_PIN(4, "gpio4"),
+	PINCTRL_PIN(5, "gpio5"),
+	PINCTRL_PIN(6, "gpio6"),
+	PINCTRL_PIN(7, "gpio7"),
+	PINCTRL_PIN(8, "gpio8"),
+	PINCTRL_PIN(9, "gpio9"),
+	PINCTRL_PIN(10, "gpio10"),
+	PINCTRL_PIN(11, "gpio11"),
+	PINCTRL_PIN(12, "gpio12"),
+	PINCTRL_PIN(13, "gpio13"),
+	PINCTRL_PIN(14, "gpio14"),
+};
+
+static const char * const swr_tx_clk_groups[] = { "gpio0" };
+static const char * const swr_tx_data_groups[] = { "gpio1", "gpio2", "gpio14" };
+static const char * const swr_rx_clk_groups[] = { "gpio3" };
+static const char * const swr_rx_data_groups[] = { "gpio4", "gpio5" };
+static const char * const dmic1_clk_groups[] = { "gpio6" };
+static const char * const dmic1_data_groups[] = { "gpio7" };
+static const char * const dmic2_clk_groups[] = { "gpio8" };
+static const char * const dmic2_data_groups[] = { "gpio9" };
+static const char * const i2s2_clk_groups[] = { "gpio10" };
+static const char * const i2s2_ws_groups[] = { "gpio11" };
+static const char * const dmic3_clk_groups[] = { "gpio12" };
+static const char * const dmic3_data_groups[] = { "gpio13" };
+static const char * const qua_mi2s_sclk_groups[] = { "gpio0" };
+static const char * const qua_mi2s_ws_groups[] = { "gpio1" };
+static const char * const qua_mi2s_data_groups[] = { "gpio2", "gpio3", "gpio4", "gpio5" };
+static const char * const i2s1_clk_groups[] = { "gpio6" };
+static const char * const i2s1_ws_groups[] = { "gpio7" };
+static const char * const i2s1_data_groups[] = { "gpio8", "gpio9" };
+static const char * const wsa_swr_clk_groups[] = { "gpio10" };
+static const char * const wsa_swr_data_groups[] = { "gpio11" };
+static const char * const i2s2_data_groups[] = { "gpio12", "gpio13" };
+
+static const struct lpi_pingroup sm6350_groups[] = {
+	LPI_PINGROUP(0, 0, swr_tx_clk, qua_mi2s_sclk, _, _),
+	LPI_PINGROUP(1, 2, swr_tx_data, qua_mi2s_ws, _, _),
+	LPI_PINGROUP(2, 4, swr_tx_data, qua_mi2s_data, _, _),
+	LPI_PINGROUP(3, 8, swr_rx_clk, qua_mi2s_data, _, _),
+	LPI_PINGROUP(4, 10, swr_rx_data, qua_mi2s_data, _, _),
+	LPI_PINGROUP(5, 12, swr_rx_data, _, qua_mi2s_data, _),
+	LPI_PINGROUP(6, LPI_NO_SLEW, dmic1_clk, i2s1_clk, _,  _),
+	LPI_PINGROUP(7, LPI_NO_SLEW, dmic1_data, i2s1_ws, _, _),
+	LPI_PINGROUP(8, LPI_NO_SLEW, dmic2_clk, i2s1_data, _, _),
+	LPI_PINGROUP(9, LPI_NO_SLEW, dmic2_data, i2s1_data, _, _),
+	LPI_PINGROUP(10, 16, i2s2_clk, wsa_swr_clk, _, _),
+	LPI_PINGROUP(11, 18, i2s2_ws, wsa_swr_data, _, _),
+	LPI_PINGROUP(12, LPI_NO_SLEW, dmic3_clk, i2s2_data, _, _),
+	LPI_PINGROUP(13, LPI_NO_SLEW, dmic3_data, i2s2_data, _, _),
+	LPI_PINGROUP_SLEW_SPARE_1(14, 0, swr_tx_data, _, _, _),
+};
+
+static const struct lpi_function sm6350_functions[] = {
+	LPI_FUNCTION(dmic1_clk),
+	LPI_FUNCTION(dmic1_data),
+	LPI_FUNCTION(dmic2_clk),
+	LPI_FUNCTION(dmic2_data),
+	LPI_FUNCTION(dmic3_clk),
+	LPI_FUNCTION(dmic3_data),
+	LPI_FUNCTION(i2s1_clk),
+	LPI_FUNCTION(i2s1_data),
+	LPI_FUNCTION(i2s1_ws),
+	LPI_FUNCTION(i2s2_clk),
+	LPI_FUNCTION(i2s2_data),
+	LPI_FUNCTION(i2s2_ws),
+	LPI_FUNCTION(qua_mi2s_data),
+	LPI_FUNCTION(qua_mi2s_sclk),
+	LPI_FUNCTION(qua_mi2s_ws),
+	LPI_FUNCTION(swr_rx_clk),
+	LPI_FUNCTION(swr_rx_data),
+	LPI_FUNCTION(swr_tx_clk),
+	LPI_FUNCTION(swr_tx_data),
+	LPI_FUNCTION(wsa_swr_clk),
+	LPI_FUNCTION(wsa_swr_data),
+};
+
+static const struct lpi_pinctrl_variant_data sm6350_lpi_data = {
+	.pins = sm6350_lpi_pins,
+	.npins = ARRAY_SIZE(sm6350_lpi_pins),
+	.groups = sm6350_groups,
+	.ngroups = ARRAY_SIZE(sm6350_groups),
+	.functions = sm6350_functions,
+	.nfunctions = ARRAY_SIZE(sm6350_functions),
+};
+
+static const struct of_device_id lpi_pinctrl_of_match[] = {
+	{
+	       .compatible = "qcom,sm6350-lpass-lpi-pinctrl",
+	       .data = &sm6350_lpi_data,
+	},
+	{ }
+};
+MODULE_DEVICE_TABLE(of, lpi_pinctrl_of_match);
+
+static struct platform_driver lpi_pinctrl_driver = {
+	.driver = {
+		   .name = "qcom-sm6350-lpass-lpi-pinctrl",
+		   .of_match_table = lpi_pinctrl_of_match,
+	},
+	.probe = lpi_pinctrl_probe,
+	.remove = lpi_pinctrl_remove,
+};
+
+module_platform_driver(lpi_pinctrl_driver);
+MODULE_DESCRIPTION("Qualcomm SM6350 LPI GPIO pin control driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pinctrl/qcom/pinctrl-sm6350.c b/drivers/pinctrl/qcom/pinctrl-sm6350.c
index eb8cd4aa8a97b..4089c96ff7365 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm6350.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm6350.c
@@ -1363,6 +1363,7 @@ static const struct of_device_id sm6350_tlmm_of_match[] = {
 	{ .compatible = "qcom,sm6350-tlmm" },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sm6350_tlmm_of_match);
 
 static struct platform_driver sm6350_tlmm_driver = {
 	.driver = {
@@ -1386,4 +1387,3 @@ module_exit(sm6350_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI SM6350 TLMM driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sm6350_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm6375.c b/drivers/pinctrl/qcom/pinctrl-sm6375.c
index d4547dd9f21f7..8da71d940b90a 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm6375.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm6375.c
@@ -1506,6 +1506,7 @@ static const struct of_device_id sm6375_tlmm_of_match[] = {
 	{ .compatible = "qcom,sm6375-tlmm", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sm6375_tlmm_of_match);
 
 static struct platform_driver sm6375_tlmm_driver = {
 	.driver = {
@@ -1529,4 +1530,3 @@ module_exit(sm6375_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI SM6375 TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, sm6375_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8150.c b/drivers/pinctrl/qcom/pinctrl-sm8150.c
index 0767261f5149e..526cc7bb3c6c6 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm8150.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm8150.c
@@ -1493,18 +1493,18 @@ static const struct msm_gpio_wakeirq_map sm8150_pdc_map[] = {
 	{ 3, 31 }, { 5, 32 }, { 8, 33 }, { 9, 34 }, { 10, 100 },
 	{ 12, 104 }, { 24, 37 }, { 26, 38 }, { 27, 41 }, { 28, 42 },
 	{ 30, 39 }, { 36, 43 }, { 37, 44 }, { 38, 30 }, { 39, 118 },
-	{ 39, 125 }, { 41, 47 }, { 42, 48 }, { 46, 50 }, { 47, 49 },
-	{ 48, 51 }, { 49, 53 }, { 50, 52 }, { 51, 116 }, { 51, 123 },
+	{ 41, 47 }, { 42, 48 }, { 46, 50 }, { 47, 49 },
+	{ 48, 51 }, { 49, 53 }, { 50, 52 }, { 51, 116 },
 	{ 53, 54 }, { 54, 55 }, { 55, 56 }, { 56, 57 }, { 58, 58 },
 	{ 60, 60 }, { 61, 61 }, { 68, 62 }, { 70, 63 }, { 76, 71 },
 	{ 77, 66 }, { 81, 64 }, { 83, 65 }, { 86, 67 }, { 87, 84 },
-	{ 88, 117 }, { 88, 124 }, { 90, 69 }, { 91, 70 }, { 93, 75 },
+	{ 88, 117 }, { 90, 69 }, { 91, 70 }, { 93, 75 },
 	{ 95, 72 }, { 96, 73 }, { 97, 74 }, { 101, 40 }, { 103, 77 },
 	{ 104, 78 }, { 108, 79 }, { 112, 80 }, { 113, 81 }, { 114, 82 },
 	{ 117, 85 }, { 118, 101 }, { 119, 87 }, { 120, 88 }, { 121, 89 },
 	{ 122, 90 }, { 123, 91 }, { 124, 92 }, { 125, 93 }, { 129, 94 },
 	{ 132, 105 }, { 133, 83 }, { 134, 36 }, { 136, 97 }, { 142, 103 },
-	{ 144, 115 }, { 144, 122 }, { 147, 102 }, { 150, 107 },
+	{ 144, 115 }, { 147, 102 }, { 150, 107 },
 	{ 152, 108 }, { 153, 109 }
 };
 
@@ -1532,6 +1532,7 @@ static const struct of_device_id sm8150_pinctrl_of_match[] = {
 	{ .compatible = "qcom,sm8150-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sm8150_pinctrl_of_match);
 
 static struct platform_driver sm8150_pinctrl_driver = {
 	.driver = {
@@ -1555,4 +1556,3 @@ module_exit(sm8150_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI sm8150 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sm8150_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8250.c b/drivers/pinctrl/qcom/pinctrl-sm8250.c
index f73f3b052de4d..abf5f68ef62f5 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm8250.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm8250.c
@@ -1354,6 +1354,7 @@ static const struct of_device_id sm8250_pinctrl_of_match[] = {
 	{ .compatible = "qcom,sm8250-pinctrl", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sm8250_pinctrl_of_match);
 
 static struct platform_driver sm8250_pinctrl_driver = {
 	.driver = {
@@ -1377,4 +1378,3 @@ module_exit(sm8250_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI sm8250 pinctrl driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sm8250_pinctrl_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8350.c b/drivers/pinctrl/qcom/pinctrl-sm8350.c
index 377ddfc77e4fa..8bd278c97171b 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm8350.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm8350.c
@@ -1632,6 +1632,7 @@ static const struct of_device_id sm8350_tlmm_of_match[] = {
 	{ .compatible = "qcom,sm8350-tlmm", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sm8350_tlmm_of_match);
 
 static struct platform_driver sm8350_tlmm_driver = {
 	.driver = {
@@ -1655,4 +1656,3 @@ module_exit(sm8350_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI SM8350 TLMM driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sm8350_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8450.c b/drivers/pinctrl/qcom/pinctrl-sm8450.c
index a1d84074ea498..8ca5b6e15e6ca 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm8450.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm8450.c
@@ -1667,6 +1667,7 @@ static const struct of_device_id sm8450_tlmm_of_match[] = {
 	{ .compatible = "qcom,sm8450-tlmm", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, sm8450_tlmm_of_match);
 
 static struct platform_driver sm8450_tlmm_driver = {
 	.driver = {
@@ -1690,4 +1691,3 @@ module_exit(sm8450_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI SM8450 TLMM driver");
 MODULE_LICENSE("GPL v2");
-MODULE_DEVICE_TABLE(of, sm8450_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8550.c b/drivers/pinctrl/qcom/pinctrl-sm8550.c
index cc8fbf4d5e84c..d12256d970a0c 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm8550.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm8550.c
@@ -1752,6 +1752,7 @@ static const struct of_device_id sm8550_tlmm_of_match[] = {
 	{ .compatible = "qcom,sm8550-tlmm", },
 	{},
 };
+MODULE_DEVICE_TABLE(of, sm8550_tlmm_of_match);
 
 static struct platform_driver sm8550_tlmm_driver = {
 	.driver = {
@@ -1775,4 +1776,3 @@ module_exit(sm8550_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI SM8550 TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, sm8550_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8650.c b/drivers/pinctrl/qcom/pinctrl-sm8650.c
index ab41292e3b4e5..cf57d226c47fc 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm8650.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm8650.c
@@ -1732,6 +1732,7 @@ static const struct of_device_id sm8650_tlmm_of_match[] = {
 	{ .compatible = "qcom,sm8650-tlmm", },
 	{},
 };
+MODULE_DEVICE_TABLE(of, sm8650_tlmm_of_match);
 
 static struct platform_driver sm8650_tlmm_driver = {
 	.driver = {
@@ -1755,4 +1756,3 @@ module_exit(sm8650_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI SM8650 TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, sm8650_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-sm8750.c b/drivers/pinctrl/qcom/pinctrl-sm8750.c
index 4cfe73f30fac2..f27e55c088d51 100644
--- a/drivers/pinctrl/qcom/pinctrl-sm8750.c
+++ b/drivers/pinctrl/qcom/pinctrl-sm8750.c
@@ -1701,6 +1701,7 @@ static const struct of_device_id sm8750_tlmm_of_match[] = {
 	{ .compatible = "qcom,sm8750-tlmm", },
 	{},
 };
+MODULE_DEVICE_TABLE(of, sm8750_tlmm_of_match);
 
 static struct platform_driver sm8750_tlmm_driver = {
 	.driver = {
@@ -1724,4 +1725,3 @@ module_exit(sm8750_tlmm_exit);
 
 MODULE_DESCRIPTION("QTI SM8750 TLMM driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, sm8750_tlmm_of_match);
diff --git a/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c b/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c
index 5c966d51eda7b..c42dd7c736fec 100644
--- a/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c
+++ b/drivers/pinctrl/qcom/pinctrl-ssbi-gpio.c
@@ -53,7 +53,7 @@
 #define PM8XXX_GPIO_PHYSICAL_OFFSET	1
 
 /* custom pinconf parameters */
-#define PM8XXX_QCOM_DRIVE_STRENGH      (PIN_CONFIG_END + 1)
+#define PM8XXX_QCOM_DRIVE_STRENGTH     (PIN_CONFIG_END + 1)
 #define PM8XXX_QCOM_PULL_UP_STRENGTH   (PIN_CONFIG_END + 2)
 
 /**
@@ -97,13 +97,13 @@ struct pm8xxx_gpio {
 };
 
 static const struct pinconf_generic_params pm8xxx_gpio_bindings[] = {
-	{"qcom,drive-strength",		PM8XXX_QCOM_DRIVE_STRENGH,	0},
+	{"qcom,drive-strength",		PM8XXX_QCOM_DRIVE_STRENGTH,	0},
 	{"qcom,pull-up-strength",	PM8XXX_QCOM_PULL_UP_STRENGTH,	0},
 };
 
 #ifdef CONFIG_DEBUG_FS
 static const struct pin_config_item pm8xxx_conf_items[ARRAY_SIZE(pm8xxx_gpio_bindings)] = {
-	PCONFDUMP(PM8XXX_QCOM_DRIVE_STRENGH, "drive-strength", NULL, true),
+	PCONFDUMP(PM8XXX_QCOM_DRIVE_STRENGTH, "drive-strength", NULL, true),
 	PCONFDUMP(PM8XXX_QCOM_PULL_UP_STRENGTH,  "pull up strength", NULL, true),
 };
 #endif
@@ -291,7 +291,7 @@ static int pm8xxx_pin_config_get(struct pinctrl_dev *pctldev,
 	case PIN_CONFIG_POWER_SOURCE:
 		arg = pin->power_source;
 		break;
-	case PM8XXX_QCOM_DRIVE_STRENGH:
+	case PM8XXX_QCOM_DRIVE_STRENGTH:
 		arg = pin->output_strength;
 		break;
 	case PIN_CONFIG_DRIVE_PUSH_PULL:
@@ -373,7 +373,7 @@ static int pm8xxx_pin_config_set(struct pinctrl_dev *pctldev,
 			pin->power_source = arg;
 			banks |= BIT(0);
 			break;
-		case PM8XXX_QCOM_DRIVE_STRENGH:
+		case PM8XXX_QCOM_DRIVE_STRENGTH:
 			if (arg > PMIC_GPIO_STRENGTH_LOW) {
 				dev_err(pctrl->dev, "invalid drive strength\n");
 				return -EINVAL;
diff --git a/drivers/pinctrl/qcom/pinctrl-x1e80100.c b/drivers/pinctrl/qcom/pinctrl-x1e80100.c
index a9fe75fc45e5c..8d2b8246170b4 100644
--- a/drivers/pinctrl/qcom/pinctrl-x1e80100.c
+++ b/drivers/pinctrl/qcom/pinctrl-x1e80100.c
@@ -1851,6 +1851,7 @@ static const struct of_device_id x1e80100_pinctrl_of_match[] = {
 	{ .compatible = "qcom,x1e80100-tlmm", },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, x1e80100_pinctrl_of_match);
 
 static struct platform_driver x1e80100_pinctrl_driver = {
 	.driver = {
@@ -1874,4 +1875,3 @@ module_exit(x1e80100_pinctrl_exit);
 
 MODULE_DESCRIPTION("QTI X1E80100 TLMM pinctrl driver");
 MODULE_LICENSE("GPL");
-MODULE_DEVICE_TABLE(of, x1e80100_pinctrl_of_match);
diff --git a/drivers/pinctrl/realtek/Kconfig b/drivers/pinctrl/realtek/Kconfig
index 054e85db99e7c..a156c4ef556ef 100644
--- a/drivers/pinctrl/realtek/Kconfig
+++ b/drivers/pinctrl/realtek/Kconfig
@@ -2,8 +2,8 @@
 
 config PINCTRL_RTD
 	tristate "Realtek DHC core pin controller driver"
-	depends on ARCH_REALTEK
-	default y
+	depends on ARCH_REALTEK || (COMPILE_TEST && OF)
+	default ARCH_REALTEK
 	select PINMUX
 	select GENERIC_PINCONF
 	select REGMAP_MMIO
@@ -11,22 +11,22 @@ config PINCTRL_RTD
 config PINCTRL_RTD1619B
 	tristate "Realtek DHC 1619B pin controller driver"
 	depends on PINCTRL_RTD
-	default y
+	default ARCH_REALTEK
 
 config PINCTRL_RTD1319D
 	tristate "Realtek DHC 1319D pin controller driver"
 	depends on PINCTRL_RTD
-	default y
+	default ARCH_REALTEK
 
 config PINCTRL_RTD1315E
 	tristate "Realtek DHC 1315E pin controller driver"
 	depends on PINCTRL_RTD
-	default y
+	default ARCH_REALTEK
 
 config PINCTRL_RTD1625
 	tristate "Realtek DHC 1625 pin controller driver"
 	depends on PINCTRL_RTD
-	default y
+	default ARCH_REALTEK
 	help
 	  This driver enables support for the pin controller on the Realtek
 	  RTD1625 SoCs.
diff --git a/drivers/pinctrl/realtek/pinctrl-rtd.c b/drivers/pinctrl/realtek/pinctrl-rtd.c
index a2c672508a4bd..60222339726b4 100644
--- a/drivers/pinctrl/realtek/pinctrl-rtd.c
+++ b/drivers/pinctrl/realtek/pinctrl-rtd.c
@@ -34,14 +34,14 @@ struct rtd_pinctrl {
 };
 
 /* custom pinconf parameters */
-#define RTD_DRIVE_STRENGH_P (PIN_CONFIG_END + 1)
-#define RTD_DRIVE_STRENGH_N (PIN_CONFIG_END + 2)
+#define RTD_DRIVE_STRENGTH_P (PIN_CONFIG_END + 1)
+#define RTD_DRIVE_STRENGTH_N (PIN_CONFIG_END + 2)
 #define RTD_DUTY_CYCLE (PIN_CONFIG_END + 3)
 #define RTD_HIGH_VIL (PIN_CONFIG_END + 4)
 
 static const struct pinconf_generic_params rtd_custom_bindings[] = {
-	{"realtek,drive-strength-p", RTD_DRIVE_STRENGH_P, 0},
-	{"realtek,drive-strength-n", RTD_DRIVE_STRENGH_N, 0},
+	{"realtek,drive-strength-p", RTD_DRIVE_STRENGTH_P, 0},
+	{"realtek,drive-strength-n", RTD_DRIVE_STRENGTH_N, 0},
 	{"realtek,duty-cycle", RTD_DUTY_CYCLE, 0},
 	{"realtek,high-vil-microvolt", RTD_HIGH_VIL, 0},
 };
@@ -473,7 +473,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 		val = 1;
 		break;
 
-	case RTD_DRIVE_STRENGH_P:
+	case RTD_DRIVE_STRENGTH_P:
 		sconfig_desc = rtd_pinctrl_find_sconfig(data, pinnr);
 		if (!sconfig_desc) {
 			dev_err(data->dev, "P driving unsupported for pin: %s\n", name);
@@ -490,7 +490,7 @@ static int rtd_pconf_parse_conf(struct rtd_pinctrl *data,
 		val = set_val << p_off;
 		break;
 
-	case RTD_DRIVE_STRENGH_N:
+	case RTD_DRIVE_STRENGTH_N:
 		sconfig_desc = rtd_pinctrl_find_sconfig(data, pinnr);
 		if (!sconfig_desc) {
 			dev_err(data->dev, "N driving unsupported for pin: %s\n", name);
diff --git a/drivers/pinctrl/renesas/pinctrl-rzg2l.c b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
index 561e6018fd899..a106e087c224a 100644
--- a/drivers/pinctrl/renesas/pinctrl-rzg2l.c
+++ b/drivers/pinctrl/renesas/pinctrl-rzg2l.c
@@ -11,12 +11,14 @@
 #include <linux/gpio/driver.h>
 #include <linux/interrupt.h>
 #include <linux/io.h>
+#include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/mutex.h>
 #include <linux/of.h>
 #include <linux/of_irq.h>
 #include <linux/platform_device.h>
 #include <linux/property.h>
+#include <linux/regmap.h>
 #include <linux/seq_file.h>
 #include <linux/spinlock.h>
 
@@ -26,6 +28,7 @@
 #include <linux/pinctrl/pinctrl.h>
 #include <linux/pinctrl/pinmux.h>
 
+#include <dt-bindings/pinctrl/renesas,r9a08g046-pinctrl.h>
 #include <dt-bindings/pinctrl/renesas,r9a09g047-pinctrl.h>
 #include <dt-bindings/pinctrl/renesas,r9a09g057-pinctrl.h>
 #include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
@@ -63,10 +66,18 @@
 #define PIN_CFG_SMT			BIT(16)	/* Schmitt-trigger input control */
 #define PIN_CFG_ELC			BIT(17)
 #define PIN_CFG_IOLH_RZV2H		BIT(18)
+#define PIN_CFG_PVDD1833_OTH_AWO_POC	BIT(19) /* known on RZ/G3L only */
+#define PIN_CFG_PVDD1833_OTH_ISO_POC	BIT(20) /* known on RZ/G3L only */
+#define PIN_CFG_WDTOVF_N_POC		BIT(21) /* known on RZ/G3L only */
 
 #define RZG2L_SINGLE_PIN		BIT_ULL(63)	/* Dedicated pin */
 #define RZG2L_VARIABLE_CFG		BIT_ULL(62)	/* Variable cfg for port pins */
 
+#define PIN_CFG_OTHER_POC_MASK	\
+					(PIN_CFG_PVDD1833_OTH_AWO_POC | \
+					 PIN_CFG_PVDD1833_OTH_ISO_POC | \
+					 PIN_CFG_WDTOVF_N_POC)
+
 #define RZG2L_MPXED_COMMON_PIN_FUNCS(group) \
 					(PIN_CFG_IOLH_##group | \
 					 PIN_CFG_PUPD | \
@@ -85,6 +96,18 @@
 
 #define RZG2L_MPXED_ETH_PIN_FUNCS(x)	((x) | PIN_CFG_NF)
 
+#define RZG3L_MPXED_ETH_PIN_FUNCS(ether) \
+					(PIN_CFG_IO_VMC_##ether | \
+					 PIN_CFG_IOLH_C | \
+					 PIN_CFG_PUPD | \
+					 PIN_CFG_NF)
+
+#define RZG3L_MPXED_PIN_FUNCS(group)	(RZG2L_MPXED_COMMON_PIN_FUNCS(group) | \
+					 PIN_CFG_SOFT_PS)
+
+#define RZG3L_MPXED_PIN_FUNCS_POC(grp, poc) (RZG2L_MPXED_COMMON_PIN_FUNCS(grp) | \
+					 PIN_CFG_PVDD1833_OTH_##poc##_POC)
+
 #define PIN_CFG_PIN_MAP_MASK		GENMASK_ULL(61, 54)
 #define PIN_CFG_PIN_REG_MASK		GENMASK_ULL(53, 46)
 #define PIN_CFG_MASK			GENMASK_ULL(31, 0)
@@ -131,6 +154,18 @@
 	 FIELD_PREP_CONST(VARIABLE_PIN_CFG_PORT_MASK, (port)) | \
 	 FIELD_PREP_CONST(PIN_CFG_MASK, (cfg)))
 
+#define RZG3L_CLONE_CHANNEL_PIN_MASK	GENMASK(31, 24)
+#define RZG3L_CLONE_CHANNEL_PORT_MASK	GENMASK(23, 19)
+#define RZG3L_CLONE_CHANNEL_BIT_MASK	GENMASK(12, 9)
+#define RZG3L_CLONE_CHANNEL_VAL_MASK	BIT(8)
+#define RZG3L_CLONE_CHANNEL_PFC_MASK	GENMASK(7, 0)
+#define RZG3L_CLONE_CHANNEL_DATA(port, pins, bit, val, pfc) \
+	(FIELD_PREP_CONST(RZG3L_CLONE_CHANNEL_PIN_MASK, (pins)) | \
+	 FIELD_PREP_CONST(RZG3L_CLONE_CHANNEL_PORT_MASK, (port)) | \
+	 FIELD_PREP_CONST(RZG3L_CLONE_CHANNEL_BIT_MASK, (bit)) | \
+	 FIELD_PREP_CONST(RZG3L_CLONE_CHANNEL_VAL_MASK, (val)) | \
+	 FIELD_PREP_CONST(RZG3L_CLONE_CHANNEL_PFC_MASK, (pfc)))
+
 #define P(off)			(0x0000 + (off))
 #define PM(off)			(0x0100 + (off) * 2)
 #define PMC(off)		(0x0200 + (off))
@@ -145,7 +180,8 @@
 #define SMT(off)		(0x3400 + (off) * 8)
 #define SD_CH(off, ch)		((off) + (ch) * 4)
 #define ETH_POC(off, ch)	((off) + (ch) * 4)
-#define QSPI			(0x3008)
+#define QSPI			(0x3008) /* known on RZ/{G2L,G2LC,G2UL,Five} only */
+#define OTHER_POC		(0x3028) /* known on RZ/G3L only */
 
 #define PVDD_2500		2	/* I/O domain voltage 2.5V */
 #define PVDD_1800		1	/* I/O domain voltage <= 1.8V */
@@ -220,12 +256,16 @@ static const struct pin_config_item renesas_rzv2h_conf_items[] = {
  * @sd_ch: SD_CH register offset
  * @eth_poc: ETH_POC register offset
  * @oen: OEN register offset
+ * @qspi: QSPI register offset
+ * @other_poc: OTHER_POC register offset
  */
 struct rzg2l_register_offsets {
 	u16 pwpr;
 	u16 sd_ch;
 	u16 eth_poc;
 	u16 oen;
+	u16 qspi;
+	u16 other_poc;
 };
 
 /**
@@ -287,6 +327,8 @@ struct rzg2l_pinctrl_data {
 	const struct rzg2l_dedicated_configs *dedicated_pins;
 	unsigned int n_port_pins;
 	unsigned int n_dedicated_pins;
+	const u32 *clone_channel_data;
+	unsigned int n_clone_channel_data;
 	const struct rzg2l_hwcfg *hwcfg;
 	const u64 *variable_pin_cfg;
 	unsigned int n_variable_pin_cfg;
@@ -322,8 +364,12 @@ struct rzg2l_pinctrl_pin_settings {
  * @pupd: PUPD registers cache
  * @ien: IEN registers cache
  * @smt: SMT registers cache
+ * @sr: SR registers cache
+ * @nod: NOD registers cache
+ * @clone: Clone register cache
  * @sd_ch: SD_CH registers cache
  * @eth_poc: ET_POC registers cache
+ * @other_poc: OTHER_POC register cache
  * @oen: Output Enable register cache
  * @qspi: QSPI registers cache
  */
@@ -335,10 +381,14 @@ struct rzg2l_pinctrl_reg_cache {
 	u32	*iolh[2];
 	u32	*ien[2];
 	u32	*pupd[2];
-	u32	*smt;
+	u32	*smt[2];
+	u32	*sr[2];
+	u32	*nod[2];
+	u32	clone;
 	u8	sd_ch[2];
 	u8	eth_poc[2];
 	u8	oen;
+	u8      other_poc;
 	u8	qspi;
 };
 
@@ -353,6 +403,8 @@ struct rzg2l_pinctrl {
 
 	struct clk			*clk;
 
+	struct regmap			*syscon;
+
 	struct gpio_chip		gpio_chip;
 	struct pinctrl_gpio_range	gpio_range;
 	DECLARE_BITMAP(tint_slot, RZG2L_TINT_MAX_INTERRUPT);
@@ -366,6 +418,7 @@ struct rzg2l_pinctrl {
 	struct rzg2l_pinctrl_reg_cache	*cache;
 	struct rzg2l_pinctrl_reg_cache	*dedicated_cache;
 	atomic_t			wakeup_path;
+	u32				clone_offset;
 };
 
 static const u16 available_ps[] = { 1800, 2500, 3300 };
@@ -388,6 +441,60 @@ static u64 rzg2l_pinctrl_get_variable_pin_cfg(struct rzg2l_pinctrl *pctrl,
 	return 0;
 }
 
+static const u64 r9a08g046_variable_pin_cfg[] = {
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PA, 0, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PA, 1, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PA, 2, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PA, 3, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PA, 4, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PA, 5, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PA, 6, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PA, 7, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PB, 0, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PB, 1, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0) | PIN_CFG_OEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PB, 2, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PB, 3, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PB, 4, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PB, 5, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PB, 6, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PB, 7, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PD, 0, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PD, 1, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PD, 2, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PD, 3, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PD, 4, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PD, 5, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PD, 6, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PD, 7, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 0, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 1, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1) | PIN_CFG_OEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 2, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 3, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 4, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 5, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 6, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PE, 7, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 0, RZG3L_MPXED_PIN_FUNCS(B)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 1, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 2, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 3, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 4, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 5, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 6, RZG3L_MPXED_PIN_FUNCS_POC(B, ISO)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PG, 7, RZG3L_MPXED_PIN_FUNCS_POC(B, ISO)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 0, RZG3L_MPXED_PIN_FUNCS(B)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 1, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 2, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 3, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 4, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PH, 5, RZG3L_MPXED_PIN_FUNCS(B) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PJ, 0, RZG3L_MPXED_PIN_FUNCS(A) | PIN_CFG_IEN),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PJ, 1, RZG3L_MPXED_PIN_FUNCS(A)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PJ, 2, RZG3L_MPXED_PIN_FUNCS(A)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PJ, 3, RZG3L_MPXED_PIN_FUNCS(A)),
+	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3L_PJ, 4, RZG3L_MPXED_PIN_FUNCS(A)),
+};
+
 static const u64 r9a09g047_variable_pin_cfg[] = {
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3E_PA, 0, RZV2H_MPXED_PIN_FUNCS | PIN_CFG_IEN),
 	RZG2L_VARIABLE_PIN_CFG_PACK(RZG3E_PA, 1, RZV2H_MPXED_PIN_FUNCS),
@@ -537,6 +644,45 @@ static int rzg2l_validate_pin(struct rzg2l_pinctrl *pctrl,
 	return 0;
 }
 
+static int rzg2l_pinctrl_set_clone_mode(struct rzg2l_pinctrl *pctrl,
+					u8 port, u8 pin, u8 func)
+{
+	unsigned int i;
+
+	if (!pctrl->data->clone_channel_data)
+		return 0;
+
+	switch (func) {
+	case 2:
+	case 4 ... 7:
+		break;
+	default:
+		return 0;
+	}
+
+	for (i = 0; i < pctrl->data->n_clone_channel_data; i++) {
+		u32 pin_data = pctrl->data->clone_channel_data[i];
+		unsigned int pin_func_mask = FIELD_GET(RZG3L_CLONE_CHANNEL_PFC_MASK, pin_data);
+		unsigned int pin_mask = FIELD_GET(RZG3L_CLONE_CHANNEL_PIN_MASK, pin_data);
+		u32 bit, val;
+
+		if (!(pin_func_mask & BIT(func)) ||
+		    FIELD_GET(RZG3L_CLONE_CHANNEL_PORT_MASK, pin_data) != port)
+			continue;
+
+		if (!(pin_mask & BIT(pin)))
+			continue;
+
+		bit = FIELD_GET(RZG3L_CLONE_CHANNEL_BIT_MASK, pin_data);
+		val = FIELD_GET(RZG3L_CLONE_CHANNEL_VAL_MASK, pin_data);
+
+		return regmap_update_bits(pctrl->syscon, pctrl->clone_offset,
+					  BIT(bit), val << bit);
+	}
+
+	return 0;
+}
+
 static void rzg2l_pinctrl_set_pfc_mode(struct rzg2l_pinctrl *pctrl,
 				       u8 pin, u8 off, u8 func)
 {
@@ -601,16 +747,22 @@ static int rzg2l_pinctrl_set_mux(struct pinctrl_dev *pctldev,
 	for (i = 0; i < group->grp.npins; i++) {
 		u64 *pin_data = pctrl->desc.pins[pins[i]].drv_data;
 		u32 off = RZG2L_PIN_CFG_TO_PORT_OFFSET(*pin_data);
+		u32 port = RZG2L_PIN_ID_TO_PORT(pins[i]);
 		u32 pin = RZG2L_PIN_ID_TO_PIN(pins[i]);
+		unsigned int func;
 
-		ret = rzg2l_validate_pin(pctrl, *pin_data, RZG2L_PIN_ID_TO_PORT(pins[i]), pin);
+		ret = rzg2l_validate_pin(pctrl, *pin_data, port, pin);
 		if (ret)
 			return ret;
 
-		dev_dbg(pctrl->dev, "port:%u pin: %u off:%x PSEL:%u\n",
-			RZG2L_PIN_ID_TO_PORT(pins[i]), pin, off, psel_val[i] - hwcfg->func_base);
+		func = psel_val[i] - hwcfg->func_base;
+		dev_dbg(pctrl->dev, "port:%u pin: %u off:%x PSEL:%u\n", port, pin, off, func);
+
+		ret = rzg2l_pinctrl_set_clone_mode(pctrl, port, pin, func);
+		if (ret)
+			return ret;
 
-		rzg2l_pinctrl_set_pfc_mode(pctrl, pin, off, psel_val[i] - hwcfg->func_base);
+		rzg2l_pinctrl_set_pfc_mode(pctrl, pin, off, func);
 	}
 
 	return 0;
@@ -894,7 +1046,8 @@ static void rzg2l_rmw_pin_config(struct rzg2l_pinctrl *pctrl, u32 offset,
 	raw_spin_unlock_irqrestore(&pctrl->lock, flags);
 }
 
-static int rzg2l_caps_to_pwr_reg(const struct rzg2l_register_offsets *regs, u32 caps)
+static int rzg2l_caps_to_pwr_reg(const struct rzg2l_register_offsets *regs,
+				 u32 caps, u8 *mask)
 {
 	if (caps & PIN_CFG_IO_VMC_SD0)
 		return SD_CH(regs->sd_ch, 0);
@@ -906,6 +1059,16 @@ static int rzg2l_caps_to_pwr_reg(const struct rzg2l_register_offsets *regs, u32
 		return ETH_POC(regs->eth_poc, 1);
 	if (caps & PIN_CFG_IO_VMC_QSPI)
 		return QSPI;
+	if (caps & PIN_CFG_OTHER_POC_MASK) {
+		if (caps & PIN_CFG_PVDD1833_OTH_AWO_POC)
+			*mask = BIT(0);
+		else if (caps & PIN_CFG_PVDD1833_OTH_ISO_POC)
+			*mask = BIT(1);
+		else
+			*mask = BIT(2);
+
+		return OTHER_POC;
+	}
 
 	return -EINVAL;
 }
@@ -914,17 +1077,20 @@ static int rzg2l_get_power_source(struct rzg2l_pinctrl *pctrl, u32 pin, u32 caps
 {
 	const struct rzg2l_hwcfg *hwcfg = pctrl->data->hwcfg;
 	const struct rzg2l_register_offsets *regs = &hwcfg->regs;
+	u8 val, mask;
 	int pwr_reg;
-	u8 val;
 
 	if (caps & PIN_CFG_SOFT_PS)
 		return pctrl->settings[pin].power_source;
 
-	pwr_reg = rzg2l_caps_to_pwr_reg(regs, caps);
+	pwr_reg = rzg2l_caps_to_pwr_reg(regs, caps, &mask);
 	if (pwr_reg < 0)
 		return pwr_reg;
 
 	val = readb(pctrl->base + pwr_reg);
+	if (pwr_reg == OTHER_POC)
+		val = field_get(mask, val);
+
 	switch (val) {
 	case PVDD_1800:
 		return 1800;
@@ -942,8 +1108,8 @@ static int rzg2l_set_power_source(struct rzg2l_pinctrl *pctrl, u32 pin, u32 caps
 {
 	const struct rzg2l_hwcfg *hwcfg = pctrl->data->hwcfg;
 	const struct rzg2l_register_offsets *regs = &hwcfg->regs;
+	u8 poc_val, val, mask;
 	int pwr_reg;
-	u8 val;
 
 	if (caps & PIN_CFG_SOFT_PS) {
 		pctrl->settings[pin].power_source = ps;
@@ -952,25 +1118,37 @@ static int rzg2l_set_power_source(struct rzg2l_pinctrl *pctrl, u32 pin, u32 caps
 
 	switch (ps) {
 	case 1800:
-		val = PVDD_1800;
+		poc_val = PVDD_1800;
 		break;
 	case 2500:
 		if (!(caps & (PIN_CFG_IO_VMC_ETH0 | PIN_CFG_IO_VMC_ETH1)))
 			return -EINVAL;
-		val = PVDD_2500;
+		poc_val = PVDD_2500;
 		break;
 	case 3300:
-		val = PVDD_3300;
+		poc_val = PVDD_3300;
 		break;
 	default:
 		return -EINVAL;
 	}
 
-	pwr_reg = rzg2l_caps_to_pwr_reg(regs, caps);
+	pwr_reg = rzg2l_caps_to_pwr_reg(regs, caps, &mask);
 	if (pwr_reg < 0)
 		return pwr_reg;
 
-	writeb(val, pctrl->base + pwr_reg);
+	if (pwr_reg == OTHER_POC) {
+		scoped_guard(raw_spinlock, &pctrl->lock) {
+			val = readb(pctrl->base + pwr_reg);
+			if (poc_val)
+				val |= mask;
+			else
+				val &= ~mask;
+			writeb(val, pctrl->base + pwr_reg);
+		}
+	} else {
+		writeb(poc_val, pctrl->base + pwr_reg);
+	}
+
 	pctrl->settings[pin].power_source = ps;
 
 	return 0;
@@ -1083,7 +1261,7 @@ static int rzg2l_pin_to_oen_bit(struct rzg2l_pinctrl *pctrl, unsigned int _pin)
 	u64 caps = FIELD_GET(PIN_CFG_MASK, *pin_data);
 	u8 pin = RZG2L_PIN_ID_TO_PIN(_pin);
 
-	if (pin > pctrl->data->hwcfg->oen_max_pin)
+	if (pin != pctrl->data->hwcfg->oen_max_pin)
 		return -EINVAL;
 
 	/*
@@ -1643,8 +1821,9 @@ static int rzg2l_pinctrl_pinconf_group_get(struct pinctrl_dev *pctldev,
 					   unsigned int group,
 					   unsigned long *config)
 {
+	unsigned long prev_config = 0;
 	const unsigned int *pins;
-	unsigned int i, npins, prev_config = 0;
+	unsigned int i, npins;
 	int ret;
 
 	ret = pinctrl_generic_get_group_pins(pctldev, group, &pins, &npins);
@@ -2089,6 +2268,70 @@ static const u64 r9a09g047_gpio_configs[] = {
 	RZG2L_GPIO_PORT_PACK(4, 0x3c, RZV2H_MPXED_PIN_FUNCS),	/* PS */
 };
 
+static const char * const rzg3l_gpio_names[] = {
+	"", "", "", "", "", "", "", "",
+	"", "", "", "", "", "", "", "",
+	"P20", "P21", "P22", "P23", "P24", "P25", "P26", "P27",
+	"P30", "P31", "P32", "P33", "P34", "P35", "P36", "P37",
+	"", "", "", "", "", "", "", "",
+	"P50", "P51", "P52", "P53", "P54", "P55", "P56", "P57",
+	"P60", "P61", "P62", "P63", "P64", "P65", "P66", "P67",
+	"P70", "P71", "P72", "P73", "P74", "P75", "P76", "P77",
+	"P80", "P81", "P82", "P83", "P84", "P85", "P86", "P87",
+	"", "", "", "", "", "", "", "",
+	"PA0", "PA1", "PA2", "PA3", "PA4", "PA5", "PA6", "PA7",
+	"PB0", "PB1", "PB2", "PB3", "PB4", "PB5", "PB6", "PB7",
+	"PC0", "PC1", "PC2", "PC3", "PC4", "PC5", "PC6", "PC7",
+	"PD0", "PD1", "PD2", "PD3", "PD4", "PD5", "PD6", "PD7",
+	"PE0", "PE1", "PE2", "PE3", "PE4", "PE5", "PE6", "PE7",
+	"PF0", "PF1", "PF2", "PF3", "PF4", "PF5", "PF6", "PF7",
+	"PG0", "PG1", "PG2", "PG3", "PG4", "PG5", "PG6", "PG7",
+	"PH0", "PH1", "PH2", "PH3", "PH4", "PH5", "PH6", "PH7",
+	"", "", "", "", "", "", "", "",
+	"PJ0", "PJ1", "PJ2", "PJ3", "PJ4", "PJ5", "PJ6", "PJ7",
+	"PK0", "PK1", "PK2", "PK3", "PK4", "PK5", "PK6", "PK7",
+	"PL0", "PL1", "PL2", "PL3", "PL4", "PL5", "PL6", "PL7",
+	"PM0", "PM1", "PM2", "PM3", "PM4", "PM5", "PM6", "PM7",
+	"", "", "", "", "", "", "", "",
+	"", "", "", "", "", "", "", "",
+	"", "", "", "", "", "", "", "",
+	"", "", "", "", "", "", "", "",
+	"", "", "", "", "", "", "", "",
+	"PS0", "PS1", "PS2", "PS3", "PS4", "PS5", "PS6", "PS7",
+};
+
+static const u64 r9a08g046_gpio_configs[] = {
+	0x0,
+	0x0,
+	RZG2L_GPIO_PORT_PACK(2, 0x22, PIN_CFG_NF | PIN_CFG_IEN),	/* P2 */
+	RZG2L_GPIO_PORT_PACK(7, 0x23, RZG3L_MPXED_PIN_FUNCS_POC(A, AWO)), /* P3 */
+	0x0,
+	RZG2L_GPIO_PORT_PACK(7, 0x25, RZG3L_MPXED_PIN_FUNCS_POC(A, ISO)), /* P5 */
+	RZG2L_GPIO_PORT_PACK(7, 0x26, RZG3L_MPXED_PIN_FUNCS_POC(A, ISO)), /* P6 */
+	RZG2L_GPIO_PORT_PACK(8, 0x27, RZG3L_MPXED_PIN_FUNCS_POC(A, ISO)), /* P7 */
+	RZG2L_GPIO_PORT_PACK(6, 0x28, RZG3L_MPXED_PIN_FUNCS_POC(A, ISO)), /* P8 */
+	0x0,
+	RZG2L_GPIO_PORT_PACK_VARIABLE(8, 0x2a),				/* PA */
+	RZG2L_GPIO_PORT_PACK_VARIABLE(8, 0x2b),				/* PB */
+	RZG2L_GPIO_PORT_PACK(3, 0x2c, RZG3L_MPXED_ETH_PIN_FUNCS(ETH0)),	/* PC */
+	RZG2L_GPIO_PORT_PACK_VARIABLE(8, 0x2d),				/* PD */
+	RZG2L_GPIO_PORT_PACK_VARIABLE(8, 0x2e),				/* PE */
+	RZG2L_GPIO_PORT_PACK(3, 0x2f, RZG3L_MPXED_ETH_PIN_FUNCS(ETH1)),	/* PF */
+	RZG2L_GPIO_PORT_PACK_VARIABLE(8, 0x30),				/* PG */
+	RZG2L_GPIO_PORT_PACK_VARIABLE(6, 0x31),				/* PH */
+	0x0,
+	RZG2L_GPIO_PORT_PACK_VARIABLE(5, 0x33),				/* PJ */
+	RZG2L_GPIO_PORT_PACK(4, 0x34, RZG3L_MPXED_PIN_FUNCS_POC(B, ISO)), /* PK */
+	RZG2L_GPIO_PORT_PACK(5, 0x35, RZG3L_MPXED_PIN_FUNCS(C)),	/* PL */
+	RZG2L_GPIO_PORT_PACK(8, 0x36, RZG3L_MPXED_PIN_FUNCS(C)),	/* PM */
+	0x0,
+	0x0,
+	0x0,
+	0x0,
+	0x0,
+	RZG2L_GPIO_PORT_PACK(2, 0x3c, RZG3L_MPXED_PIN_FUNCS(A)),	/* PS */
+};
+
 static const char * const rzv2h_gpio_names[] = {
 	"P00", "P01", "P02", "P03", "P04", "P05", "P06", "P07",
 	"P10", "P11", "P12", "P13", "P14", "P15", "P16", "P17",
@@ -2427,6 +2670,128 @@ static struct rzg2l_dedicated_configs rzg3e_dedicated_pins[] = {
 	 (PIN_CFG_IOLH_RZV2H | PIN_CFG_SR | PIN_CFG_IEN | PIN_CFG_PUPD)) },
 };
 
+static const struct rzg2l_dedicated_configs rzg3l_dedicated_pins[] = {
+	{ "WDTOVF_N", RZG2L_SINGLE_PIN_PACK(0x5, 0,
+	  (PIN_CFG_IOLH_A | PIN_CFG_WDTOVF_N_POC)) },
+	{ "SCIF0_RXD", RZG2L_SINGLE_PIN_PACK(0x6, 0,
+	  (PIN_CFG_IOLH_A | PIN_CFG_PUPD | PIN_CFG_PVDD1833_OTH_AWO_POC)) },
+	{ "SCIF0_TXD", RZG2L_SINGLE_PIN_PACK(0x6, 1,
+	  (PIN_CFG_IOLH_A | PIN_CFG_PUPD | PIN_CFG_PVDD1833_OTH_AWO_POC)) },
+	{ "SD0_CLK", RZG2L_SINGLE_PIN_PACK(0x9, 0, PIN_CFG_IOLH_B) },
+	{ "SD0_CMD", RZG2L_SINGLE_PIN_PACK(0x9, 1,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	{ "SD0_RST#", RZG2L_SINGLE_PIN_PACK(0x9, 2, PIN_CFG_IOLH_B) },
+	{ "SD0_DS", RZG2L_SINGLE_PIN_PACK(0x9, 5,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	{ "SD0_DAT0", RZG2L_SINGLE_PIN_PACK(0x0a, 0,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	{ "SD0_DAT1", RZG2L_SINGLE_PIN_PACK(0x0a, 1,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	{ "SD0_DAT2", RZG2L_SINGLE_PIN_PACK(0x0a, 2,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	{ "SD0_DAT3", RZG2L_SINGLE_PIN_PACK(0x0a, 3,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	{ "SD0_DAT4", RZG2L_SINGLE_PIN_PACK(0x0a, 4,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	{ "SD0_DAT5", RZG2L_SINGLE_PIN_PACK(0x0a, 5,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	{ "SD0_DAT6", RZG2L_SINGLE_PIN_PACK(0x0a, 6,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+	{ "SD0_DAT7", RZG2L_SINGLE_PIN_PACK(0x0a, 7,
+	  (PIN_CFG_IOLH_B | PIN_CFG_IEN | PIN_CFG_PUPD)) },
+};
+
+static const u32 r9a08g046_clone_channel_data[] = {
+	/* I2C ch2 Bit:0 Value:0 PFC:4 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PG, GENMASK(7, 6), 0, 0, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PH, GENMASK(3, 2), 0, 0, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PK, GENMASK(1, 0), 0, 0, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PA, GENMASK(5, 4) | GENMASK(1, 0), 0, 0, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PB, GENMASK(5, 4) | GENMASK(1, 0), 0, 0, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PC, GENMASK(1, 0), 0, 0, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PD, GENMASK(7, 6) | GENMASK(3, 2), 0, 0, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PE, GENMASK(7, 6) | GENMASK(3, 2), 0, 0, BIT(4)),
+	/* I2C ch2 Bit:0 Value:1 PFC:4 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P5, GENMASK(5, 4) | GENMASK(1, 0), 0, 1, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P6, GENMASK(6, 5) | GENMASK(2, 1), 0, 1, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P8, GENMASK(5, 4) | GENMASK(1, 0), 0, 1, BIT(4)),
+	/* I2C ch3 Bit:1 Value:0 PFC:4 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PF, GENMASK(1, 0), 1, 0, BIT(4)),
+	/* I2C ch3 Bit:1 Value:1 PFC:4 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P2, GENMASK(1, 0), 1, 1, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P5, BIT(6) | GENMASK(3, 2), 1, 1, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P6, GENMASK(4, 3) | BIT(0), 1, 1, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P7, GENMASK(7, 6), 1, 1, BIT(4)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P8, GENMASK(3, 2), 1, 1, BIT(4)),
+	/* SCIF ch3 Bit:4 Value:0 PFC:{6,7} */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PG, GENMASK(6, 4), 4, 0, BIT(6)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PH, GENMASK(5, 3), 4, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PA, GENMASK(4, 2), 4, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PB, GENMASK(5, 3), 4, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PD, GENMASK(2, 0), 4, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PE, GENMASK(3, 1), 4, 0, BIT(7)),
+	/* SCIF ch3 Bit:4 Value:1 PFC:7 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P5, GENMASK(2, 0), 4, 1, BIT(7)),
+	/* SCIF ch4 Bit:5 Value:0 PFC:7 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PK, GENMASK(2, 0), 5, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PA, GENMASK(7, 5), 5, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PB, GENMASK(7, 6), 5, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PC, BIT(0), 5, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PD, GENMASK(5, 3), 5, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PE, GENMASK(6, 4), 5, 0, BIT(7)),
+	/* SCIF ch4 Bit:5 Value:1 PFC:7 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P5, GENMASK(5, 3), 5, 1, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P6, GENMASK(4, 2), 5, 1, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P7, GENMASK(7, 5), 5, 1, BIT(7)),
+	/* SCIF ch5 Bit:6 Value:0 PFC:7 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PE, BIT(7), 6, 0, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PF, GENMASK(1, 0), 6, 0, BIT(7)),
+	/* SCIF ch5 Bit:6 Value:1 PFC:7 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P5, BIT(6), 6, 1, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P6, GENMASK(6, 5) | GENMASK(1, 0), 6, 1, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P7, GENMASK(4, 2) | BIT(0), 6, 1, BIT(7)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P8, GENMASK(2, 0), 6, 1, BIT(7)),
+	/* RSPI ch1 Bit:8 Value:0 PFC:2 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PH, GENMASK(5, 0), 8, 0, BIT(2)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PD, GENMASK(7, 5), 8, 0, BIT(2)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PE, GENMASK(3, 0), 8, 0, BIT(2)),
+	/* RSPI ch1 Bit:8 Value:1 PFC:2 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P5, GENMASK(6, 0), 8, 1, BIT(2)),
+	/* RSPI ch2 Bit:9 Value:0 PFC:2 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PE, GENMASK(7, 4), 9, 0, BIT(2)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PF, GENMASK(2, 0), 9, 0, BIT(2)),
+	/* RSPI ch2 Bit:9 Value:1 PFC:2 */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P6, GENMASK(6, 0), 9, 1, BIT(2)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P7, BIT(7), 9, 1, BIT(2)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P8, GENMASK(5, 0), 9, 1, BIT(2)),
+	/* RSCI ch1 Bit:12 Value:0 PFC:{5,6} shared pins based on RSCI mode */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PG, GENMASK(3, 0), 12, 0, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PA, GENMASK(3, 0), 12, 0, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PB, GENMASK(7, 6), 12, 0, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PC, GENMASK(1, 0), 12, 0, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PD, GENMASK(7, 4), 12, 0, GENMASK(6, 5)),
+	/* RSCI ch1 Bit:12 Value:1 PFC:{5,6} shared pins based on RSCI mode */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P5, GENMASK(3, 0), 12, 1, GENMASK(6, 5)),
+	/* RSCI ch2 Bit:13 Value:0 PFC:{5,6} shared pins based on RSCI mode */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PH, GENMASK(3, 0), 13, 0, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PK, GENMASK(3, 0), 13, 0, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PA, GENMASK(7, 4), 13, 0, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PD, GENMASK(3, 0), 13, 0, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PE, GENMASK(3, 0), 13, 0, GENMASK(6, 5)),
+	/* RSCI ch2 Bit:13 Value:1 PFC:{5,6} shared pins based on RSCI mode */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P5, GENMASK(6, 4), 13, 1, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P6, GENMASK(6, 5) | BIT(0), 13, 1, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P7, GENMASK(7, 6) | GENMASK(1, 0), 13, 1, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P8, GENMASK(1, 0), 13, 1, GENMASK(6, 5)),
+	/* RSCI ch3 Bit:14 Value:0 PFC:{5,6} shared pins based on RSCI mode */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PE, GENMASK(7, 6), 14, 0, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_PF, GENMASK(1, 0), 14, 0, GENMASK(6, 5)),
+	/* RSCI ch3 Bit:14 Value:1 PFC:{5,6} shared pins based on RSCI mode */
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P6, GENMASK(4, 1), 14, 1, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P7, GENMASK(5, 2), 14, 1, GENMASK(6, 5)),
+	RZG3L_CLONE_CHANNEL_DATA(RZG3L_P8, GENMASK(5, 2), 14, 1, GENMASK(6, 5)),
+};
+
 static int rzg2l_gpio_get_gpioint(unsigned int virq, struct rzg2l_pinctrl *pctrl)
 {
 	const struct pinctrl_pin_desc *pin_desc = &pctrl->desc.pins[virq];
@@ -2737,10 +3102,6 @@ static int rzg2l_pinctrl_reg_cache_alloc(struct rzg2l_pinctrl *pctrl)
 	if (!cache->pfc)
 		return -ENOMEM;
 
-	cache->smt = devm_kcalloc(pctrl->dev, nports, sizeof(*cache->smt), GFP_KERNEL);
-	if (!cache->smt)
-		return -ENOMEM;
-
 	for (u8 i = 0; i < 2; i++) {
 		u32 n_dedicated_pins = pctrl->data->n_dedicated_pins;
 
@@ -2759,6 +3120,21 @@ static int rzg2l_pinctrl_reg_cache_alloc(struct rzg2l_pinctrl *pctrl)
 		if (!cache->pupd[i])
 			return -ENOMEM;
 
+		cache->smt[i] = devm_kcalloc(pctrl->dev, nports, sizeof(*cache->smt[i]),
+					     GFP_KERNEL);
+		if (!cache->smt[i])
+			return -ENOMEM;
+
+		cache->sr[i] = devm_kcalloc(pctrl->dev, nports, sizeof(*cache->sr[i]),
+					    GFP_KERNEL);
+		if (!cache->sr[i])
+			return -ENOMEM;
+
+		cache->nod[i] = devm_kcalloc(pctrl->dev, nports, sizeof(*cache->nod[i]),
+					     GFP_KERNEL);
+		if (!cache->nod[i])
+			return -ENOMEM;
+
 		/* Allocate dedicated cache. */
 		dedicated_cache->iolh[i] = devm_kcalloc(pctrl->dev, n_dedicated_pins,
 							sizeof(*dedicated_cache->iolh[i]),
@@ -2771,6 +3147,24 @@ static int rzg2l_pinctrl_reg_cache_alloc(struct rzg2l_pinctrl *pctrl)
 						       GFP_KERNEL);
 		if (!dedicated_cache->ien[i])
 			return -ENOMEM;
+
+		dedicated_cache->sr[i] = devm_kcalloc(pctrl->dev, n_dedicated_pins,
+						      sizeof(*dedicated_cache->sr[i]),
+						      GFP_KERNEL);
+		if (!dedicated_cache->sr[i])
+			return -ENOMEM;
+
+		dedicated_cache->nod[i] = devm_kcalloc(pctrl->dev, n_dedicated_pins,
+						       sizeof(*dedicated_cache->nod[i]),
+						       GFP_KERNEL);
+		if (!dedicated_cache->nod[i])
+			return -ENOMEM;
+
+		dedicated_cache->pupd[i] = devm_kcalloc(pctrl->dev, n_dedicated_pins,
+							sizeof(*dedicated_cache->pupd[i]),
+							GFP_KERNEL);
+		if (!dedicated_cache->pupd[i])
+			return -ENOMEM;
 	}
 
 	pctrl->cache = cache;
@@ -2955,6 +3349,9 @@ static int rzg2l_pinctrl_probe(struct platform_device *pdev)
 	BUILD_BUG_ON(ARRAY_SIZE(r9a08g045_gpio_configs) * RZG2L_PINS_PER_PORT >
 		     ARRAY_SIZE(rzg2l_gpio_names));
 
+	BUILD_BUG_ON(ARRAY_SIZE(r9a08g046_gpio_configs) * RZG2L_PINS_PER_PORT >
+		     ARRAY_SIZE(rzg3l_gpio_names));
+
 	BUILD_BUG_ON(ARRAY_SIZE(r9a09g047_gpio_configs) * RZG2L_PINS_PER_PORT >
 		     ARRAY_SIZE(rzg3e_gpio_names));
 
@@ -2981,6 +3378,16 @@ static int rzg2l_pinctrl_probe(struct platform_device *pdev)
 				     "failed to enable GPIO clk\n");
 	}
 
+	if (pctrl->data->clone_channel_data) {
+		struct device_node *np = pctrl->dev->of_node;
+
+		pctrl->syscon = syscon_regmap_lookup_by_phandle_args(np, "renesas,clonech",
+								     1, &pctrl->clone_offset);
+		if (IS_ERR(pctrl->syscon))
+			return dev_err_probe(pctrl->dev, PTR_ERR(pctrl->syscon),
+					     "Failed to parse renesas,clonech\n");
+	}
+
 	raw_spin_lock_init(&pctrl->lock);
 	spin_lock_init(&pctrl->bitmap_lock);
 	mutex_init(&pctrl->mutex);
@@ -3002,7 +3409,7 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen
 	struct rzg2l_pinctrl_reg_cache *cache = pctrl->cache;
 
 	for (u32 port = 0; port < nports; port++) {
-		bool has_iolh, has_ien, has_pupd, has_smt;
+		bool has_iolh, has_ien, has_pupd, has_smt, has_sr, has_nod;
 		u32 off, caps;
 		u8 pincnt;
 		u64 cfg;
@@ -3023,6 +3430,8 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen
 		has_ien = !!(caps & PIN_CFG_IEN);
 		has_pupd = !!(caps & PIN_CFG_PUPD);
 		has_smt = !!(caps & PIN_CFG_SMT);
+		has_sr = !!(caps & PIN_CFG_SR);
+		has_nod = !!(caps & PIN_CFG_NOD);
 
 		if (suspend)
 			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + PFC(off), cache->pfc[port]);
@@ -3049,7 +3458,7 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen
 			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + PUPD(off),
 						 cache->pupd[0][port]);
 			if (pincnt >= 4) {
-				RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + PUPD(off),
+				RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + PUPD(off) + 4,
 							 cache->pupd[1][port]);
 			}
 		}
@@ -3066,8 +3475,32 @@ static void rzg2l_pinctrl_pm_setup_regs(struct rzg2l_pinctrl *pctrl, bool suspen
 			}
 		}
 
-		if (has_smt)
-			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SMT(off), cache->smt[port]);
+		if (has_smt) {
+			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SMT(off),
+						 cache->smt[0][port]);
+			if (pincnt >= 4) {
+				RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SMT(off) + 4,
+							 cache->smt[1][port]);
+			}
+		}
+
+		if (has_sr) {
+			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SR(off),
+						 cache->sr[0][port]);
+			if (pincnt >= 4) {
+				RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SR(off) + 4,
+							 cache->sr[1][port]);
+			}
+		}
+
+		if (has_nod) {
+			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + NOD(off),
+						 cache->nod[0][port]);
+			if (pincnt >= 4) {
+				RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + NOD(off) + 4,
+							 cache->nod[1][port]);
+			}
+		}
 	}
 }
 
@@ -3082,7 +3515,7 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b
 	 * port offset are close together.
 	 */
 	for (i = 0, caps = 0; i < pctrl->data->n_dedicated_pins; i++) {
-		bool has_iolh, has_ien;
+		bool has_iolh, has_ien, has_sr, has_nod, has_pupd;
 		u32 off, next_off = 0;
 		u64 cfg, next_cfg;
 		u8 pincnt;
@@ -3101,8 +3534,12 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b
 		}
 
 		/* And apply them in a single shot. */
-		has_iolh = !!(caps & (PIN_CFG_IOLH_A | PIN_CFG_IOLH_B | PIN_CFG_IOLH_C));
+		has_iolh = !!(caps & (PIN_CFG_IOLH_A | PIN_CFG_IOLH_B |
+				      PIN_CFG_IOLH_C | PIN_CFG_IOLH_RZV2H));
 		has_ien = !!(caps & PIN_CFG_IEN);
+		has_sr = !!(caps & PIN_CFG_SR);
+		has_nod = !!(caps & PIN_CFG_NOD);
+		has_pupd = !!(caps & PIN_CFG_PUPD);
 		pincnt = hweight8(FIELD_GET(RZG2L_SINGLE_PIN_BITS_MASK, cfg));
 
 		if (has_iolh) {
@@ -3113,6 +3550,18 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b
 			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + IEN(off),
 						 cache->ien[0][i]);
 		}
+		if (has_sr) {
+			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + SR(off),
+						 cache->sr[0][i]);
+		}
+		if (has_nod) {
+			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + NOD(off),
+						 cache->nod[0][i]);
+		}
+		if (has_pupd) {
+			RZG2L_PCTRL_REG_ACCESS32(suspend, pctrl->base + PUPD(off),
+						 cache->pupd[0][i]);
+		}
 
 		if (pincnt >= 4) {
 			if (has_iolh) {
@@ -3125,6 +3574,21 @@ static void rzg2l_pinctrl_pm_setup_dedicated_regs(struct rzg2l_pinctrl *pctrl, b
 							 pctrl->base + IEN(off) + 4,
 							 cache->ien[1][i]);
 			}
+			if (has_sr) {
+				RZG2L_PCTRL_REG_ACCESS32(suspend,
+							 pctrl->base + SR(off) + 4,
+							 cache->sr[1][i]);
+			}
+			if (has_nod) {
+				RZG2L_PCTRL_REG_ACCESS32(suspend,
+							 pctrl->base + NOD(off) + 4,
+							 cache->nod[1][i]);
+			}
+			if (has_pupd) {
+				RZG2L_PCTRL_REG_ACCESS32(suspend,
+							 pctrl->base + PUPD(off) + 4,
+							 cache->pupd[1][i]);
+			}
 		}
 		caps = 0;
 	}
@@ -3207,8 +3671,19 @@ static int rzg2l_pinctrl_suspend_noirq(struct device *dev)
 			cache->eth_poc[i] = readb(pctrl->base + ETH_POC(regs->eth_poc, i));
 	}
 
-	cache->qspi = readb(pctrl->base + QSPI);
+	if (regs->qspi)
+		cache->qspi = readb(pctrl->base + regs->qspi);
 	cache->oen = readb(pctrl->base + pctrl->data->hwcfg->regs.oen);
+	if (regs->other_poc)
+		cache->other_poc = readb(pctrl->base + regs->other_poc);
+
+	if (pctrl->syscon) {
+		int ret;
+
+		ret = regmap_read(pctrl->syscon, pctrl->clone_offset, &cache->clone);
+		if (ret)
+			return ret;
+	}
 
 	if (!atomic_read(&pctrl->wakeup_path))
 		clk_disable_unprepare(pctrl->clk);
@@ -3227,13 +3702,22 @@ static int rzg2l_pinctrl_resume_noirq(struct device *dev)
 	unsigned long flags;
 	int ret;
 
+	if (pctrl->syscon) {
+		ret = regmap_write(pctrl->syscon, pctrl->clone_offset, cache->clone);
+		if (ret)
+			return ret;
+	}
+
 	if (!atomic_read(&pctrl->wakeup_path)) {
 		ret = clk_prepare_enable(pctrl->clk);
 		if (ret)
 			return ret;
 	}
 
-	writeb(cache->qspi, pctrl->base + QSPI);
+	if (regs->qspi)
+		writeb(cache->qspi, pctrl->base + regs->qspi);
+	if (regs->other_poc)
+		writeb(cache->other_poc, pctrl->base + regs->other_poc);
 
 	raw_spin_lock_irqsave(&pctrl->lock, flags);
 	rzg2l_oen_write_with_pwpr(pctrl, cache->oen);
@@ -3291,6 +3775,7 @@ static const struct rzg2l_hwcfg rzg2l_hwcfg = {
 		.sd_ch = 0x3000,
 		.eth_poc = 0x300c,
 		.oen = 0x3018,
+		.qspi = QSPI,
 	},
 	.iolh_groupa_ua = {
 		/* 3v3 power source */
@@ -3301,6 +3786,40 @@ static const struct rzg2l_hwcfg rzg2l_hwcfg = {
 	.oen_max_pin = 0,
 };
 
+static const struct rzg2l_hwcfg rzg3l_hwcfg = {
+	.regs = {
+		.pwpr = 0x3000,
+		.sd_ch = 0x3004,
+		.eth_poc = 0x3010,
+		.oen = 0x3018,
+		.other_poc = OTHER_POC,
+	},
+	.iolh_groupa_ua = {
+		/* 1v8 power source */
+		[RZG2L_IOLH_IDX_1V8] = 2200, 4400, 9000, 10000,
+		/* 3v3 power source */
+		[RZG2L_IOLH_IDX_3V3] = 1900, 4000, 8000, 9000,
+	},
+	.iolh_groupb_ua = {
+		/* 1v8 power source */
+		[RZG2L_IOLH_IDX_1V8] = 7000, 8000, 9000, 10000,
+		/* 3v3 power source */
+		[RZG2L_IOLH_IDX_3V3] = 4000, 6000, 8000, 9000,
+	},
+	.iolh_groupc_ua = {
+		/* 1v8 power source */
+		[RZG2L_IOLH_IDX_1V8] = 5200, 6000, 6550, 6800,
+		/* 2v5 source */
+		[RZG2L_IOLH_IDX_2V5] = 4700, 5300, 5800, 6100,
+		/* 3v3 power source */
+		[RZG2L_IOLH_IDX_3V3] = 4500, 5200, 5700, 6050,
+	},
+	.tint_start_index = 17,
+	.drive_strength_ua = true,
+	.func_base = 0,
+	.oen_max_pin = 1, /* Pin 1 of P{B,E}1_ISO is the maximum OEN pin. */
+};
+
 static const struct rzg2l_hwcfg rzg3s_hwcfg = {
 	.regs = {
 		.pwpr = 0x3000,
@@ -3394,6 +3913,25 @@ static struct rzg2l_pinctrl_data r9a08g045_data = {
 	.bias_param_to_hw = &rzg2l_bias_param_to_hw,
 };
 
+static struct rzg2l_pinctrl_data r9a08g046_data = {
+	.port_pins = rzg3l_gpio_names,
+	.port_pin_configs = r9a08g046_gpio_configs,
+	.n_ports = ARRAY_SIZE(r9a08g046_gpio_configs),
+	.variable_pin_cfg = r9a08g046_variable_pin_cfg,
+	.n_variable_pin_cfg = ARRAY_SIZE(r9a08g046_variable_pin_cfg),
+	.dedicated_pins = rzg3l_dedicated_pins,
+	.n_port_pins = ARRAY_SIZE(r9a08g046_gpio_configs) * RZG2L_PINS_PER_PORT,
+	.n_dedicated_pins = ARRAY_SIZE(rzg3l_dedicated_pins),
+	.clone_channel_data = r9a08g046_clone_channel_data,
+	.n_clone_channel_data = ARRAY_SIZE(r9a08g046_clone_channel_data),
+	.hwcfg = &rzg3l_hwcfg,
+	.pwpr_pfc_lock_unlock = &rzg2l_pwpr_pfc_lock_unlock,
+	.pmc_writeb = &rzg2l_pmc_writeb,
+	.pin_to_oen_bit = &rzg2l_pin_to_oen_bit,
+	.hw_to_bias_param = &rzg2l_hw_to_bias_param,
+	.bias_param_to_hw = &rzg2l_bias_param_to_hw,
+};
+
 static struct rzg2l_pinctrl_data r9a09g047_data = {
 	.port_pins = rzg3e_gpio_names,
 	.port_pin_configs = r9a09g047_gpio_configs,
@@ -3474,6 +4012,10 @@ static const struct of_device_id rzg2l_pinctrl_of_table[] = {
 		.compatible = "renesas,r9a08g045-pinctrl",
 		.data = &r9a08g045_data,
 	},
+	{
+		.compatible = "renesas,r9a08g046-pinctrl",
+		.data = &r9a08g046_data,
+	},
 	{
 		.compatible = "renesas,r9a09g047-pinctrl",
 		.data = &r9a09g047_data,
diff --git a/drivers/pinctrl/renesas/pinctrl-rzv2m.c b/drivers/pinctrl/renesas/pinctrl-rzv2m.c
index 495e7f5d41280..827b3e91a6cc2 100644
--- a/drivers/pinctrl/renesas/pinctrl-rzv2m.c
+++ b/drivers/pinctrl/renesas/pinctrl-rzv2m.c
@@ -695,8 +695,9 @@ static int rzv2m_pinctrl_pinconf_group_get(struct pinctrl_dev *pctldev,
 					   unsigned int group,
 					   unsigned long *config)
 {
+	unsigned long prev_config = 0;
 	const unsigned int *pins;
-	unsigned int i, npins, prev_config = 0;
+	unsigned int i, npins;
 	int ret;
 
 	ret = pinctrl_generic_get_group_pins(pctldev, group, &pins, &npins);
diff --git a/drivers/pinctrl/renesas/pinctrl.c b/drivers/pinctrl/renesas/pinctrl.c
index 3a742f74ecd1d..8585ed4bcfe09 100644
--- a/drivers/pinctrl/renesas/pinctrl.c
+++ b/drivers/pinctrl/renesas/pinctrl.c
@@ -719,6 +719,30 @@ static int sh_pfc_pinconf_set(struct pinctrl_dev *pctldev, unsigned _pin,
 	return 0;
 }
 
+static int sh_pfc_pinconf_group_get(struct pinctrl_dev *pctldev,
+				    unsigned int group, unsigned long *config)
+{
+	struct sh_pfc_pinctrl *pmx = pinctrl_dev_get_drvdata(pctldev);
+	const unsigned int *pins = pmx->pfc->info->groups[group].pins;
+	unsigned int num_pins = pmx->pfc->info->groups[group].nr_pins;
+	unsigned long prev_config = 0;
+	int ret;
+
+	for (unsigned int i = 0; i < num_pins; ++i) {
+		ret = sh_pfc_pinconf_get(pctldev, pins[i], config);
+		if (ret)
+			return ret;
+
+		/* configs should match for all pins in the group */
+		if (i && prev_config != *config)
+			return -ENOTSUPP;
+
+		prev_config = *config;
+	}
+
+	return 0;
+}
+
 static int sh_pfc_pinconf_group_set(struct pinctrl_dev *pctldev, unsigned group,
 				    unsigned long *configs,
 				    unsigned num_configs)
@@ -745,6 +769,7 @@ static const struct pinconf_ops sh_pfc_pinconf_ops = {
 	.is_generic			= true,
 	.pin_config_get			= sh_pfc_pinconf_get,
 	.pin_config_set			= sh_pfc_pinconf_set,
+	.pin_config_group_get		= sh_pfc_pinconf_group_get,
 	.pin_config_group_set		= sh_pfc_pinconf_group_set,
 	.pin_config_config_dbg_show	= pinconf_generic_dump_config,
 };
diff --git a/drivers/pinctrl/sophgo/pinctrl-cv18xx.c b/drivers/pinctrl/sophgo/pinctrl-cv18xx.c
index c3a2dcf71f2a6..59318a42690fe 100644
--- a/drivers/pinctrl/sophgo/pinctrl-cv18xx.c
+++ b/drivers/pinctrl/sophgo/pinctrl-cv18xx.c
@@ -27,8 +27,9 @@
 #include "pinctrl-cv18xx.h"
 
 struct cv1800_priv {
-	u32					*power_cfg;
-	void __iomem				*regs[2];
+	void __iomem	*regs[2];
+	unsigned int	num_power_cfg;
+	u32		power_cfg[] __counted_by(num_power_cfg);
 };
 
 static unsigned int cv1800_dt_get_pin_mux(u32 value)
@@ -417,14 +418,11 @@ static int cv1800_pinctrl_init(struct platform_device *pdev,
 	const struct sophgo_pinctrl_data *pctrl_data = pctrl->data;
 	struct cv1800_priv *priv;
 
-	priv = devm_kzalloc(&pdev->dev, sizeof(struct cv1800_priv), GFP_KERNEL);
+	priv = devm_kzalloc(&pdev->dev, struct_size(priv, power_cfg, pctrl_data->npds),
+			GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
-
-	priv->power_cfg = devm_kcalloc(&pdev->dev, pctrl_data->npds,
-				       sizeof(u32), GFP_KERNEL);
-	if (!priv->power_cfg)
-		return -ENOMEM;
+	priv->num_power_cfg = pctrl_data->npds;
 
 	priv->regs[0] = devm_platform_ioremap_resource_byname(pdev, "sys");
 	if (IS_ERR(priv->regs[0]))
diff --git a/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.c b/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.c
index e44480e71ea8f..3572e8edd9f30 100644
--- a/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.c
+++ b/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.c
@@ -857,17 +857,15 @@ int jh7110_pinctrl_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
+#if IS_ENABLED(CONFIG_PM_SLEEP)
+	sfp = devm_kzalloc(dev, struct_size(sfp, saved_regs, info->nsaved_regs),
+			GFP_KERNEL);
+#else
 	sfp = devm_kzalloc(dev, sizeof(*sfp), GFP_KERNEL);
+#endif
 	if (!sfp)
 		return -ENOMEM;
 
-#if IS_ENABLED(CONFIG_PM_SLEEP)
-	sfp->saved_regs = devm_kcalloc(dev, info->nsaved_regs,
-				       sizeof(*sfp->saved_regs), GFP_KERNEL);
-	if (!sfp->saved_regs)
-		return -ENOMEM;
-#endif
-
 	sfp->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(sfp->base))
 		return PTR_ERR(sfp->base);
diff --git a/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.h b/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.h
index 2da2d6858008c..188fc9d96269d 100644
--- a/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.h
+++ b/drivers/pinctrl/starfive/pinctrl-starfive-jh7110.h
@@ -21,7 +21,7 @@ struct jh7110_pinctrl {
 	/* register read/write mutex */
 	struct mutex mutex;
 	const struct jh7110_pinctrl_soc_info *info;
-	u32 *saved_regs;
+	u32 saved_regs[];
 };
 
 struct jh7110_gpio_irq_reg {
diff --git a/drivers/pinctrl/sunxi/pinctrl-sunxi.c b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
index d3042e0c9712e..25489beeb3125 100644
--- a/drivers/pinctrl/sunxi/pinctrl-sunxi.c
+++ b/drivers/pinctrl/sunxi/pinctrl-sunxi.c
@@ -925,7 +925,7 @@ static int sunxi_pmx_request(struct pinctrl_dev *pctldev, unsigned offset)
 	return 0;
 
 out:
-	regulator_put(s_reg->regulator);
+	regulator_put(reg);
 
 	return ret;
 }
diff --git a/drivers/pinctrl/tegra/Kconfig b/drivers/pinctrl/tegra/Kconfig
index 660d101ea3679..3e8789871f0fb 100644
--- a/drivers/pinctrl/tegra/Kconfig
+++ b/drivers/pinctrl/tegra/Kconfig
@@ -1,43 +1,45 @@
 # SPDX-License-Identifier: GPL-2.0-only
 config PINCTRL_TEGRA
-	bool
+	bool "NVIDIA Tegra pin controllers common" if COMPILE_TEST && !ARCH_TEGRA
 	select PINMUX
 	select PINCONF
 
 config PINCTRL_TEGRA20
-	bool
+	bool "NVIDIA Tegra20 pin controller" if COMPILE_TEST && !ARCH_TEGRA
 	select PINCTRL_TEGRA
+	depends on COMMON_CLK
 
 config PINCTRL_TEGRA30
-	bool
+	bool "NVIDIA Tegra30 pin controller" if COMPILE_TEST && !ARCH_TEGRA
 	select PINCTRL_TEGRA
 
 config PINCTRL_TEGRA114
-	bool
+	bool "NVIDIA Tegra114 pin controller" if COMPILE_TEST && !ARCH_TEGRA
 	select PINCTRL_TEGRA
 
 config PINCTRL_TEGRA124
-	bool
+	bool "NVIDIA Tegra124 pin controller" if COMPILE_TEST && !ARCH_TEGRA
 	select PINCTRL_TEGRA
 
 config PINCTRL_TEGRA210
-	bool
+	bool "NVIDIA Tegra210 pin controller" if COMPILE_TEST && !ARCH_TEGRA
 	select PINCTRL_TEGRA
 
 config PINCTRL_TEGRA186
-	bool
+	bool "NVIDIA Tegra186 pin controller" if COMPILE_TEST && !ARCH_TEGRA
 	select PINCTRL_TEGRA
 
 config PINCTRL_TEGRA194
-	bool
+	bool "NVIDIA Tegra194 pin controller" if COMPILE_TEST && !ARCH_TEGRA
 	select PINCTRL_TEGRA
 
 config PINCTRL_TEGRA234
-	bool
+	bool "NVIDIA Tegra234 pin controller" if COMPILE_TEST && !ARCH_TEGRA
 	select PINCTRL_TEGRA
 
 config PINCTRL_TEGRA_XUSB
-	def_bool y if ARCH_TEGRA
+	bool "NVIDIA Tegra XUSB pin controller" if COMPILE_TEST && !ARCH_TEGRA
+	default y if ARCH_TEGRA
 	select GENERIC_PHY
 	select PINCONF
 	select PINMUX
diff --git a/drivers/pinctrl/vt8500/Kconfig b/drivers/pinctrl/vt8500/Kconfig
index 2ca00b54b7a81..1a40c153a82a2 100644
--- a/drivers/pinctrl/vt8500/Kconfig
+++ b/drivers/pinctrl/vt8500/Kconfig
@@ -3,16 +3,17 @@
 # VIA/Wondermedia PINCTRL drivers
 #
 
-if ARCH_VT8500
+if ARCH_VT8500 || COMPILE_TEST
 
 config PINCTRL_WMT
 	bool
 	select PINMUX
 	select GENERIC_PINCONF
+	select GPIOLIB
 
 config PINCTRL_VT8500
 	bool "VIA VT8500 pin controller driver"
-	depends on ARCH_WM8505
+	depends on ARCH_WM8505 || COMPILE_TEST
 	select PINCTRL_WMT
 	help
 	  Say yes here to support the gpio/pin control module on
@@ -20,7 +21,7 @@ config PINCTRL_VT8500
 
 config PINCTRL_WM8505
 	bool "Wondermedia WM8505 pin controller driver"
-	depends on ARCH_WM8505
+	depends on ARCH_WM8505 || COMPILE_TEST
 	select PINCTRL_WMT
 	help
 	  Say yes here to support the gpio/pin control module on
@@ -28,7 +29,7 @@ config PINCTRL_WM8505
 
 config PINCTRL_WM8650
 	bool "Wondermedia WM8650 pin controller driver"
-	depends on ARCH_WM8505
+	depends on ARCH_WM8505 || COMPILE_TEST
 	select PINCTRL_WMT
 	help
 	  Say yes here to support the gpio/pin control module on
@@ -36,7 +37,7 @@ config PINCTRL_WM8650
 
 config PINCTRL_WM8750
 	bool "Wondermedia WM8750 pin controller driver"
-	depends on ARCH_WM8750
+	depends on ARCH_WM8750 || COMPILE_TEST
 	select PINCTRL_WMT
 	help
 	  Say yes here to support the gpio/pin control module on
@@ -44,7 +45,7 @@ config PINCTRL_WM8750
 
 config PINCTRL_WM8850
 	bool "Wondermedia WM8850 pin controller driver"
-	depends on ARCH_WM8850
+	depends on ARCH_WM8850 || COMPILE_TEST
 	select PINCTRL_WMT
 	help
 	  Say yes here to support the gpio/pin control module on
diff --git a/drivers/platform/arm64/Kconfig b/drivers/platform/arm64/Kconfig
index 10f905d7d6bfa..e32e01b2a9bdd 100644
--- a/drivers/platform/arm64/Kconfig
+++ b/drivers/platform/arm64/Kconfig
@@ -90,4 +90,17 @@ config EC_LENOVO_THINKPAD_T14S
 
 	  Say M or Y here to include this support.
 
+config EC_QCOM_HAMOA
+	tristate "Embedded Controller driver for Qualcomm Hamoa/Glymur reference devices"
+	depends on ARCH_QCOM || COMPILE_TEST
+	depends on I2C
+	depends on THERMAL || THERMAL=n
+	help
+	  Say M or Y here to enable the Embedded Controller driver for Qualcomm
+	  Snapdragon-based Hamoa/Glymur reference devices. The driver handles fan
+	  control, temperature sensors, access to EC state changes and supports
+	  reporting suspend entry/exit to the EC.
+
+	  This driver currently supports Hamoa/Purwa/Glymur reference devices.
+
 endif # ARM64_PLATFORM_DEVICES
diff --git a/drivers/platform/arm64/Makefile b/drivers/platform/arm64/Makefile
index 60c131cff6a15..7681be4a46e94 100644
--- a/drivers/platform/arm64/Makefile
+++ b/drivers/platform/arm64/Makefile
@@ -9,3 +9,4 @@ obj-$(CONFIG_EC_ACER_ASPIRE1)	+= acer-aspire1-ec.o
 obj-$(CONFIG_EC_HUAWEI_GAOKUN)	+= huawei-gaokun-ec.o
 obj-$(CONFIG_EC_LENOVO_YOGA_C630) += lenovo-yoga-c630.o
 obj-$(CONFIG_EC_LENOVO_THINKPAD_T14S) += lenovo-thinkpad-t14s.o
+obj-$(CONFIG_EC_QCOM_HAMOA) += qcom-hamoa-ec.o
diff --git a/drivers/platform/arm64/qcom-hamoa-ec.c b/drivers/platform/arm64/qcom-hamoa-ec.c
new file mode 100644
index 0000000000000..a018f7bf35d2a
--- /dev/null
+++ b/drivers/platform/arm64/qcom-hamoa-ec.c
@@ -0,0 +1,451 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2024 Maya Matuszczyk <maccraft123mc@gmail.com>
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/device.h>
+#include <linux/dev_printk.h>
+#include <linux/err.h>
+#include <linux/i2c.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pm.h>
+#include <linux/slab.h>
+#include <linux/thermal.h>
+
+#define EC_SCI_EVT_READ_CMD	0x05
+#define EC_FW_VERSION_CMD	0x0e
+#define EC_MODERN_STANDBY_CMD	0x23
+#define EC_FAN_DBG_CONTROL_CMD	0x30
+#define EC_SCI_EVT_CONTROL_CMD	0x35
+#define EC_THERMAL_CAP_CMD	0x42
+
+#define EC_FW_VERSION_RESP_LEN	4
+#define EC_THERMAL_CAP_RESP_LEN	3
+#define EC_FAN_DEBUG_CMD_LEN	6
+#define EC_FAN_SPEED_DATA_SIZE	4
+
+#define EC_MODERN_STANDBY_ENTER	0x01
+#define EC_MODERN_STANDBY_EXIT	0x00
+
+#define EC_FAN_DEBUG_MODE_OFF   0
+#define EC_FAN_DEBUG_MODE_ON    BIT(0)
+#define EC_FAN_ON               BIT(1)
+#define EC_FAN_DEBUG_TYPE_PWM   BIT(2)
+#define EC_MAX_FAN_CNT		2
+#define EC_FAN_NAME_SIZE	20
+#define EC_FAN_MAX_PWM		255
+
+enum qcom_ec_sci_events {
+	EC_FAN1_STATUS_CHANGE_EVT = 0x30,
+	EC_FAN2_STATUS_CHANGE_EVT,
+	EC_FAN1_SPEED_CHANGE_EVT,
+	EC_FAN2_SPEED_CHANGE_EVT,
+	EC_NEW_LUT_SET_EVT,
+	EC_FAN_PROFILE_SWITCH_EVT,
+	EC_THERMISTOR_1_THRESHOLD_CROSS_EVT,
+	EC_THERMISTOR_2_THRESHOLD_CROSS_EVT,
+	EC_THERMISTOR_3_THRESHOLD_CROSS_EVT,
+	/* Reserved: 0x39 - 0x3c/0x3f */
+	EC_RECOVERED_FROM_RESET_EVT = 0x3d,
+};
+
+struct qcom_ec_version {
+	u8 main_version;
+	u8 sub_version;
+	u8 test_version;
+};
+
+struct qcom_ec_thermal_cap {
+#define EC_THERMAL_FAN_CNT(x)		(FIELD_GET(GENMASK(1, 0), (x)))
+#define EC_THERMAL_FAN_TYPE(x)		(FIELD_GET(GENMASK(4, 2), (x)))
+#define EC_THERMAL_THERMISTOR_MASK(x)	(FIELD_GET(GENMASK(7, 0), (x)))
+	u8 fan_cnt;
+	u8 fan_type;
+	u8 thermistor_mask;
+};
+
+struct qcom_ec_cooling_dev {
+	struct thermal_cooling_device *cdev;
+	struct device *parent_dev;
+	u8 fan_id;
+	u8 state;
+};
+
+struct qcom_ec {
+	struct qcom_ec_cooling_dev *ec_cdev;
+	struct qcom_ec_thermal_cap thermal_cap;
+	struct qcom_ec_version version;
+	struct i2c_client *client;
+};
+
+static int qcom_ec_read(struct qcom_ec *ec, u8 cmd, u8 resp_len, u8 *resp)
+{
+	int ret;
+
+	ret = i2c_smbus_read_i2c_block_data(ec->client, cmd, resp_len, resp);
+	if (ret < 0)
+		return ret;
+	else if (ret == 0 || ret == 0xff)
+		return -EOPNOTSUPP;
+
+	if (resp[0] >= resp_len)
+		return -EINVAL;
+
+	return 0;
+}
+
+/*
+ * EC Device Firmware Version:
+ *
+ * Read Response:
+ * ----------------------------------------------------------------------
+ * | Offset	| Name		| Description				|
+ * ----------------------------------------------------------------------
+ * | 0x00	| Byte count	| Number of bytes in response		|
+ * |		|		| (excluding byte count)		|
+ * ----------------------------------------------------------------------
+ * | 0x01	| Test-version	| Test-version of EC firmware		|
+ * ----------------------------------------------------------------------
+ * | 0x02	| Sub-version	| Sub-version of EC firmware		|
+ * ----------------------------------------------------------------------
+ * | 0x03	| Main-version	| Main-version of EC firmware		|
+ * ----------------------------------------------------------------------
+ *
+ */
+static int qcom_ec_read_fw_version(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct qcom_ec *ec = i2c_get_clientdata(client);
+	struct qcom_ec_version *version = &ec->version;
+	u8 resp[EC_FW_VERSION_RESP_LEN];
+	int ret;
+
+	ret = qcom_ec_read(ec, EC_FW_VERSION_CMD, EC_FW_VERSION_RESP_LEN, resp);
+	if (ret < 0)
+		return ret;
+
+	version->main_version = resp[3];
+	version->sub_version = resp[2];
+	version->test_version = resp[1];
+
+	dev_dbg(dev, "EC Version %d.%d.%d\n",
+		version->main_version, version->sub_version, version->test_version);
+
+	return 0;
+}
+
+/*
+ * EC Device Thermal Capabilities:
+ *
+ * Read Response:
+ * ------------------------------------------------------------------------------
+ * | Offset		| Name		| Description				|
+ * ------------------------------------------------------------------------------
+ * | 0x00		| Byte count	| Number of bytes in response		|
+ * |			|		| (excluding byte count)		|
+ * ------------------------------------------------------------------------------
+ * | 0x02 (LSB)	| EC Thermal	| Bit 0-1: Number of fans		|
+ * | 0x03		| Capabilities	| Bit 2-4: Type of fan			|
+ * |			|		| Bit 5-6: Reserved			|
+ * |			|		| Bit 7: Data Valid/Invalid		|
+ * |			|		|	 (Valid - 1, Invalid - 0)	|
+ * |			|		| Bit 8-15: Thermistor 0 - 7 presence	|
+ * |			|		|	    (1 present, 0 absent)	|
+ * ------------------------------------------------------------------------------
+ *
+ */
+static int qcom_ec_thermal_capabilities(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+	struct qcom_ec *ec = i2c_get_clientdata(client);
+	struct qcom_ec_thermal_cap *cap = &ec->thermal_cap;
+	u8 resp[EC_THERMAL_CAP_RESP_LEN];
+	int ret;
+
+	ret = qcom_ec_read(ec, EC_THERMAL_CAP_CMD, EC_THERMAL_CAP_RESP_LEN, resp);
+	if (ret < 0)
+		return ret;
+
+	cap->fan_cnt = min(EC_MAX_FAN_CNT, EC_THERMAL_FAN_CNT(resp[1]));
+	cap->fan_type = EC_THERMAL_FAN_TYPE(resp[1]);
+	cap->thermistor_mask = EC_THERMAL_THERMISTOR_MASK(resp[2]);
+
+	dev_dbg(dev, "Fan count: %d Fan Type: %d Thermistor Mask: %x\n",
+		cap->fan_cnt, cap->fan_type, cap->thermistor_mask);
+
+	return 0;
+}
+
+static irqreturn_t qcom_ec_irq(int irq, void *data)
+{
+	struct qcom_ec *ec = data;
+	struct device *dev = &ec->client->dev;
+	int val;
+
+	val = i2c_smbus_read_byte_data(ec->client, EC_SCI_EVT_READ_CMD);
+	if (val < 0) {
+		dev_err_ratelimited(dev, "Failed to read EC SCI Event: %d\n", val);
+		return IRQ_HANDLED;
+	}
+
+	switch (val) {
+	case EC_FAN1_STATUS_CHANGE_EVT:
+		dev_dbg_ratelimited(dev, "Fan1 status changed\n");
+		break;
+	case EC_FAN2_STATUS_CHANGE_EVT:
+		dev_dbg_ratelimited(dev, "Fan2 status changed\n");
+		break;
+	case EC_FAN1_SPEED_CHANGE_EVT:
+		dev_dbg_ratelimited(dev, "Fan1 speed crossed low/high trip point\n");
+		break;
+	case EC_FAN2_SPEED_CHANGE_EVT:
+		dev_dbg_ratelimited(dev, "Fan2 speed crossed low/high trip point\n");
+		break;
+	case EC_NEW_LUT_SET_EVT:
+		dev_dbg_ratelimited(dev, "New LUT set\n");
+		break;
+	case EC_FAN_PROFILE_SWITCH_EVT:
+		dev_dbg_ratelimited(dev, "FAN Profile switched\n");
+		break;
+	case EC_THERMISTOR_1_THRESHOLD_CROSS_EVT:
+		dev_dbg_ratelimited(dev, "Thermistor 1 threshold crossed\n");
+		break;
+	case EC_THERMISTOR_2_THRESHOLD_CROSS_EVT:
+		dev_dbg_ratelimited(dev, "Thermistor 2 threshold crossed\n");
+		break;
+	case EC_THERMISTOR_3_THRESHOLD_CROSS_EVT:
+		dev_dbg_ratelimited(dev, "Thermistor 3 threshold crossed\n");
+		break;
+	case EC_RECOVERED_FROM_RESET_EVT:
+		dev_dbg_ratelimited(dev, "EC recovered from reset\n");
+		break;
+	default:
+		dev_notice_ratelimited(dev, "Unknown EC event: %d\n", val);
+		break;
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int qcom_ec_sci_evt_control(struct device *dev, bool enable)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+
+	return i2c_smbus_write_byte_data(client, EC_SCI_EVT_CONTROL_CMD, enable ? 1 : 0);
+}
+
+static int qcom_ec_fan_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state)
+{
+	*state = EC_FAN_MAX_PWM;
+
+	return 0;
+}
+
+static int qcom_ec_fan_get_cur_state(struct thermal_cooling_device *cdev, unsigned long *state)
+{
+	struct qcom_ec_cooling_dev *ec_cdev = cdev->devdata;
+
+	*state = ec_cdev->state;
+
+	return 0;
+}
+
+/*
+ * Fan Debug control command:
+ *
+ * Command Payload:
+ * --------------------------------------------------------------------------------------
+ * | Offset		| Name		| Description					|
+ * --------------------------------------------------------------------------------------
+ * | 0x00		| Command	| Fan control command				|
+ * --------------------------------------------------------------------------------------
+ * | 0x01		| Fan ID	| 0x1 : Fan 1					|
+ * |			|		| 0x2 : Fan 2					|
+ * --------------------------------------------------------------------------------------
+ * | 0x02		| Byte count = 4| Size of data to set fan speed			|
+ * --------------------------------------------------------------------------------------
+ * | 0x03		| Mode		| Bit 0: Debug Mode On/Off (0 - OFF, 1 - ON )	|
+ * |			|		| Bit 1: Fan On/Off (0 - Off, 1 - ON)		|
+ * |			|		| Bit 2: Debug Type (0 - RPM, 1 - PWM)		|
+ * --------------------------------------------------------------------------------------
+ * | 0x04 (LSB)	| Speed in RPM	| RPM value, if mode selected is RPM		|
+ * | 0x05		|		|						|
+ * --------------------------------------------------------------------------------------
+ * | 0x06		| Speed in PWM	| PWM value, if mode selected is PWM (0 - 255)	|
+ * ______________________________________________________________________________________
+ *
+ */
+static int qcom_ec_fan_debug_mode_off(struct qcom_ec_cooling_dev *ec_cdev)
+{
+	struct device *dev = ec_cdev->parent_dev;
+	struct i2c_client *client = to_i2c_client(dev);
+	u8 request[6] = { ec_cdev->fan_id, EC_FAN_SPEED_DATA_SIZE,
+			  EC_FAN_DEBUG_MODE_OFF, 0, 0, 0 };
+	int ret;
+
+	ret = i2c_smbus_write_i2c_block_data(client, EC_FAN_DBG_CONTROL_CMD,
+					     sizeof(request), request);
+	if (ret) {
+		dev_err(dev, "Failed to turn off fan%d debug mode: %d\n",
+			ec_cdev->fan_id, ret);
+	}
+
+	return ret;
+}
+
+static int qcom_ec_fan_set_cur_state(struct thermal_cooling_device *cdev, unsigned long state)
+{
+	struct qcom_ec_cooling_dev *ec_cdev = cdev->devdata;
+	struct device *dev = ec_cdev->parent_dev;
+	struct i2c_client *client = to_i2c_client(dev);
+	u8 request[6] = { ec_cdev->fan_id, EC_FAN_SPEED_DATA_SIZE,
+			  EC_FAN_DEBUG_MODE_ON | EC_FAN_ON | EC_FAN_DEBUG_TYPE_PWM,
+			  0, 0, state };
+	int ret;
+
+	ret = i2c_smbus_write_i2c_block_data(client, EC_FAN_DBG_CONTROL_CMD,
+					     sizeof(request), request);
+	if (ret) {
+		dev_err(dev, "Failed to set fan pwm: %d\n", ret);
+		return ret;
+	}
+
+	ec_cdev->state = state;
+
+	return 0;
+}
+
+static const struct thermal_cooling_device_ops qcom_ec_thermal_ops = {
+	.get_max_state = qcom_ec_fan_get_max_state,
+	.get_cur_state = qcom_ec_fan_get_cur_state,
+	.set_cur_state = qcom_ec_fan_set_cur_state,
+};
+
+static int qcom_ec_resume(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+
+	return i2c_smbus_write_byte_data(client, EC_MODERN_STANDBY_CMD,
+					 EC_MODERN_STANDBY_EXIT);
+}
+
+static int qcom_ec_suspend(struct device *dev)
+{
+	struct i2c_client *client = to_i2c_client(dev);
+
+	return i2c_smbus_write_byte_data(client, EC_MODERN_STANDBY_CMD,
+					 EC_MODERN_STANDBY_ENTER);
+}
+
+static int qcom_ec_probe(struct i2c_client *client)
+{
+	struct device *dev = &client->dev;
+	struct qcom_ec *ec;
+	unsigned int i;
+	int ret;
+
+	ec = devm_kzalloc(dev, sizeof(*ec), GFP_KERNEL);
+	if (!ec)
+		return -ENOMEM;
+
+	ec->client = client;
+
+	ret = devm_request_threaded_irq(dev, client->irq, NULL, qcom_ec_irq,
+					IRQF_ONESHOT, "qcom_ec", ec);
+	if (ret < 0)
+		return ret;
+
+	i2c_set_clientdata(client, ec);
+
+	ret = qcom_ec_read_fw_version(dev);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to read EC firmware version\n");
+
+	ret = qcom_ec_sci_evt_control(dev, true);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to enable SCI events\n");
+
+	ret = qcom_ec_thermal_capabilities(dev);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to read thermal capabilities\n");
+
+	if (ec->thermal_cap.fan_cnt == 0) {
+		dev_warn(dev, FW_BUG "Failed to get fan count, firmware update required\n");
+		return 0;
+	}
+
+	ec->ec_cdev = devm_kcalloc(dev, ec->thermal_cap.fan_cnt, sizeof(*ec->ec_cdev), GFP_KERNEL);
+	if (!ec->ec_cdev)
+		return -ENOMEM;
+
+	for (i = 0; i < ec->thermal_cap.fan_cnt; i++) {
+		struct qcom_ec_cooling_dev *ec_cdev = &ec->ec_cdev[i];
+		char name[EC_FAN_NAME_SIZE];
+
+		scnprintf(name, sizeof(name), "qcom_ec_fan_%u", i);
+		ec_cdev->fan_id = i + 1;
+		ec_cdev->parent_dev = dev;
+
+		ec_cdev->cdev = devm_thermal_of_cooling_device_register(dev, NULL, name, ec_cdev,
+									&qcom_ec_thermal_ops);
+		if (IS_ERR(ec_cdev->cdev)) {
+			return dev_err_probe(dev, PTR_ERR(ec_cdev->cdev),
+					     "Failed to register fan%d cooling device\n", i);
+		}
+	}
+
+	return 0;
+}
+
+static void qcom_ec_remove(struct i2c_client *client)
+{
+	struct qcom_ec *ec = i2c_get_clientdata(client);
+	struct device *dev = &client->dev;
+	int ret;
+
+	ret = qcom_ec_sci_evt_control(dev, false);
+	if (ret < 0)
+		dev_err(dev, "Failed to disable SCI events: %d\n", ret);
+
+	for (int i = 0; i < ec->thermal_cap.fan_cnt; i++) {
+		struct qcom_ec_cooling_dev *ec_cdev = &ec->ec_cdev[i];
+
+		qcom_ec_fan_debug_mode_off(ec_cdev);
+	}
+}
+
+static const struct of_device_id qcom_ec_of_match[] = {
+	{ .compatible = "qcom,hamoa-crd-ec" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, qcom_ec_of_match);
+
+static const struct i2c_device_id qcom_ec_i2c_id_table[] = {
+	{ "qcom-hamoa-ec", },
+	{}
+};
+MODULE_DEVICE_TABLE(i2c, qcom_ec_i2c_id_table);
+
+static DEFINE_SIMPLE_DEV_PM_OPS(qcom_ec_pm_ops,
+		qcom_ec_suspend,
+		qcom_ec_resume);
+
+static struct i2c_driver qcom_ec_i2c_driver = {
+	.driver = {
+		.name = "qcom-hamoa-ec",
+		.of_match_table = qcom_ec_of_match,
+		.pm = &qcom_ec_pm_ops,
+	},
+	.probe = qcom_ec_probe,
+	.remove = qcom_ec_remove,
+	.id_table = qcom_ec_i2c_id_table,
+};
+module_i2c_driver(qcom_ec_i2c_driver);
+
+MODULE_DESCRIPTION("QCOM Hamoa Embedded Controller");
+MODULE_LICENSE("GPL");
diff --git a/drivers/platform/chrome/chromeos_privacy_screen.c b/drivers/platform/chrome/chromeos_privacy_screen.c
index abc5d189a3894..407b04207de2f 100644
--- a/drivers/platform/chrome/chromeos_privacy_screen.c
+++ b/drivers/platform/chrome/chromeos_privacy_screen.c
@@ -104,6 +104,9 @@ static const struct drm_privacy_screen_ops chromeos_privacy_screen_ops = {
 
 static int chromeos_privacy_screen_probe(struct platform_device *pdev)
 {
+	if (!ACPI_COMPANION(&pdev->dev))
+		return -ENODEV;
+
 	struct drm_privacy_screen *drm_privacy_screen =
 		drm_privacy_screen_register(&pdev->dev,
 					    &chromeos_privacy_screen_ops,
diff --git a/drivers/platform/chrome/chromeos_tbmc.c b/drivers/platform/chrome/chromeos_tbmc.c
index 5133806b2d958..fd756761a481f 100644
--- a/drivers/platform/chrome/chromeos_tbmc.c
+++ b/drivers/platform/chrome/chromeos_tbmc.c
@@ -69,9 +69,13 @@ static int chromeos_tbmc_probe(struct platform_device *pdev)
 {
 	struct input_dev *idev;
 	struct device *dev = &pdev->dev;
-	struct acpi_device *adev = ACPI_COMPANION(dev);
+	struct acpi_device *adev;
 	int ret;
 
+	adev = ACPI_COMPANION(dev);
+	if (!adev)
+		return -ENODEV;
+
 	idev = devm_input_allocate_device(dev);
 	if (!idev)
 		return -ENOMEM;
diff --git a/drivers/platform/chrome/cros_ec_i2c.c b/drivers/platform/chrome/cros_ec_i2c.c
index def1144a077ea..2f46be4a27569 100644
--- a/drivers/platform/chrome/cros_ec_i2c.c
+++ b/drivers/platform/chrome/cros_ec_i2c.c
@@ -348,7 +348,7 @@ MODULE_DEVICE_TABLE(of, cros_ec_i2c_of_match);
 #endif
 
 static const struct i2c_device_id cros_ec_i2c_id[] = {
-	{ "cros-ec-i2c" },
+	{ .name = "cros-ec-i2c" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, cros_ec_i2c_id);
diff --git a/drivers/platform/chrome/cros_ec_sensorhub.c b/drivers/platform/chrome/cros_ec_sensorhub.c
index 9bad8f72680ea..f938c3fc84e4f 100644
--- a/drivers/platform/chrome/cros_ec_sensorhub.c
+++ b/drivers/platform/chrome/cros_ec_sensorhub.c
@@ -122,8 +122,12 @@ static int cros_ec_sensorhub_register(struct device *dev,
 		sensor_type[sensorhub->resp->info.type]++;
 	}
 
-	if (sensor_type[MOTIONSENSE_TYPE_ACCEL] >= 2)
+	if (sensor_type[MOTIONSENSE_TYPE_ACCEL] >= 2) {
 		ec->has_kb_wake_angle = true;
+		if (ec->group && sysfs_update_group(&ec->class_dev.kobj,
+						    ec->group))
+			dev_warn(dev, "Unable to update sysfs");
+	}
 
 	if (cros_ec_check_features(ec,
 				   EC_FEATURE_REFINED_TABLET_MODE_HYSTERESIS)) {
diff --git a/drivers/platform/chrome/cros_ec_sysfs.c b/drivers/platform/chrome/cros_ec_sysfs.c
index f22e9523da3e8..9d3767ab15480 100644
--- a/drivers/platform/chrome/cros_ec_sysfs.c
+++ b/drivers/platform/chrome/cros_ec_sysfs.c
@@ -405,7 +405,8 @@ static int cros_ec_sysfs_probe(struct platform_device *pd)
 	struct device *dev = &pd->dev;
 	int ret;
 
-	ret = sysfs_create_group(&ec_dev->class_dev.kobj, &cros_ec_attr_group);
+	ec_dev->group = &cros_ec_attr_group;
+	ret = sysfs_create_group(&ec_dev->class_dev.kobj, ec_dev->group);
 	if (ret < 0)
 		dev_err(dev, "failed to create attributes. err=%d\n", ret);
 
@@ -416,7 +417,7 @@ static void cros_ec_sysfs_remove(struct platform_device *pd)
 {
 	struct cros_ec_dev *ec_dev = dev_get_drvdata(pd->dev.parent);
 
-	sysfs_remove_group(&ec_dev->class_dev.kobj, &cros_ec_attr_group);
+	sysfs_remove_group(&ec_dev->class_dev.kobj, ec_dev->group);
 }
 
 static const struct platform_device_id cros_ec_sysfs_id[] = {
diff --git a/drivers/platform/chrome/cros_hps_i2c.c b/drivers/platform/chrome/cros_hps_i2c.c
index ac6498c593e3f..3b94856278312 100644
--- a/drivers/platform/chrome/cros_hps_i2c.c
+++ b/drivers/platform/chrome/cros_hps_i2c.c
@@ -131,7 +131,7 @@ static int hps_resume(struct device *dev)
 static DEFINE_RUNTIME_DEV_PM_OPS(hps_pm_ops, hps_suspend, hps_resume, NULL);
 
 static const struct i2c_device_id hps_i2c_id[] = {
-	{ "cros-hps" },
+	{ .name = "cros-hps" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, hps_i2c_id);
diff --git a/drivers/platform/chrome/cros_kbd_led_backlight.c b/drivers/platform/chrome/cros_kbd_led_backlight.c
index f4c2282129f50..80dc52833dc99 100644
--- a/drivers/platform/chrome/cros_kbd_led_backlight.c
+++ b/drivers/platform/chrome/cros_kbd_led_backlight.c
@@ -32,12 +32,11 @@ struct keyboard_led {
  * @brightness_set_blocking:	Set LED brightness level.  It can block the
  *				caller for the time required for accessing a
  *				LED device register
- * @max_brightness:		Maximum brightness.
  *
  * See struct led_classdev in include/linux/leds.h for more details.
  */
 struct keyboard_led_drvdata {
-	int (*init)(struct platform_device *pdev);
+	int (*init)(struct platform_device *pdev, struct keyboard_led *keyboard_led);
 
 	enum led_brightness (*brightness_get)(struct led_classdev *led_cdev);
 
@@ -45,12 +44,8 @@ struct keyboard_led_drvdata {
 			       enum led_brightness brightness);
 	int (*brightness_set_blocking)(struct led_classdev *led_cdev,
 				       enum led_brightness brightness);
-
-	enum led_brightness max_brightness;
 };
 
-#define KEYBOARD_BACKLIGHT_MAX 100
-
 #ifdef CONFIG_ACPI
 
 /* Keyboard LED ACPI Device must be defined in firmware */
@@ -94,7 +89,8 @@ keyboard_led_get_brightness_acpi(struct led_classdev *cdev)
 	return brightness;
 }
 
-static int keyboard_led_init_acpi(struct platform_device *pdev)
+static int keyboard_led_init_acpi(struct platform_device *pdev,
+				  struct keyboard_led *keyboard_led)
 {
 	acpi_handle handle;
 	acpi_status status;
@@ -116,17 +112,15 @@ static const struct keyboard_led_drvdata keyboard_led_drvdata_acpi = {
 	.init = keyboard_led_init_acpi,
 	.brightness_set = keyboard_led_set_brightness_acpi,
 	.brightness_get = keyboard_led_get_brightness_acpi,
-	.max_brightness = KEYBOARD_BACKLIGHT_MAX,
 };
 
 #endif /* CONFIG_ACPI */
 
-#if IS_ENABLED(CONFIG_MFD_CROS_EC_DEV)
-static int keyboard_led_init_ec_pwm_mfd(struct platform_device *pdev)
+static int keyboard_led_init_ec_pwm_mfd(struct platform_device *pdev,
+					struct keyboard_led *keyboard_led)
 {
 	struct cros_ec_dev *ec_dev = dev_get_drvdata(pdev->dev.parent);
 	struct cros_ec_device *cros_ec = ec_dev->ec_dev;
-	struct keyboard_led *keyboard_led = platform_get_drvdata(pdev);
 
 	keyboard_led->ec = cros_ec;
 
@@ -175,15 +169,8 @@ static const struct keyboard_led_drvdata keyboard_led_drvdata_ec_pwm_mfd = {
 	.init = keyboard_led_init_ec_pwm_mfd,
 	.brightness_set_blocking = keyboard_led_set_brightness_ec_pwm,
 	.brightness_get = keyboard_led_get_brightness_ec_pwm,
-	.max_brightness = KEYBOARD_BACKLIGHT_MAX,
 };
 
-#else /* IS_ENABLED(CONFIG_MFD_CROS_EC_DEV) */
-
-static const struct keyboard_led_drvdata keyboard_led_drvdata_ec_pwm_mfd = {};
-
-#endif /* IS_ENABLED(CONFIG_MFD_CROS_EC_DEV) */
-
 static int keyboard_led_is_mfd_device(struct platform_device *pdev)
 {
 	return IS_ENABLED(CONFIG_MFD_CROS_EC_DEV) && mfd_get_cell(pdev);
@@ -205,17 +192,16 @@ static int keyboard_led_probe(struct platform_device *pdev)
 	keyboard_led = devm_kzalloc(&pdev->dev, sizeof(*keyboard_led), GFP_KERNEL);
 	if (!keyboard_led)
 		return -ENOMEM;
-	platform_set_drvdata(pdev, keyboard_led);
 
 	if (drvdata->init) {
-		err = drvdata->init(pdev);
+		err = drvdata->init(pdev, keyboard_led);
 		if (err)
 			return err;
 	}
 
 	keyboard_led->cdev.name = "chromeos::kbd_backlight";
 	keyboard_led->cdev.flags |= LED_CORE_SUSPENDRESUME | LED_REJECT_NAME_CONFLICT;
-	keyboard_led->cdev.max_brightness = drvdata->max_brightness;
+	keyboard_led->cdev.max_brightness = 100;
 	keyboard_led->cdev.brightness_set = drvdata->brightness_set;
 	keyboard_led->cdev.brightness_set_blocking = drvdata->brightness_set_blocking;
 	keyboard_led->cdev.brightness_get = drvdata->brightness_get;
diff --git a/drivers/platform/chrome/wilco_ec/event.c b/drivers/platform/chrome/wilco_ec/event.c
index b6e935badc0e9..1b5cb89839e08 100644
--- a/drivers/platform/chrome/wilco_ec/event.c
+++ b/drivers/platform/chrome/wilco_ec/event.c
@@ -452,8 +452,13 @@ static void hangup_device(struct event_device_data *dev_data)
 static int event_device_probe(struct platform_device *pdev)
 {
 	struct event_device_data *dev_data;
+	struct acpi_device *adev;
 	int error, minor;
 
+	adev = ACPI_COMPANION(&pdev->dev);
+	if (!adev)
+		return -ENODEV;
+
 	minor = ida_alloc_max(&event_ida, EVENT_MAX_DEV-1, GFP_KERNEL);
 	if (minor < 0) {
 		error = minor;
@@ -494,8 +499,7 @@ static int event_device_probe(struct platform_device *pdev)
 		goto free_dev_data;
 
 	/* Install an ACPI notify handler. */
-	error = acpi_dev_install_notify_handler(ACPI_COMPANION(&pdev->dev),
-						ACPI_DEVICE_NOTIFY,
+	error = acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
 						event_device_notify, &pdev->dev);
 	if (error)
 		goto free_cdev;
diff --git a/drivers/platform/cznic/turris-omnia-mcu-gpio.c b/drivers/platform/cznic/turris-omnia-mcu-gpio.c
index 7f0ada4fa6062..4e430d6c3fc4b 100644
--- a/drivers/platform/cznic/turris-omnia-mcu-gpio.c
+++ b/drivers/platform/cznic/turris-omnia-mcu-gpio.c
@@ -893,7 +893,7 @@ static bool omnia_irq_read_pending_old(struct omnia_mcu *mcu,
 
 	if (status & OMNIA_STS_BUTTON_PRESSED) {
 		mcu->button_pressed_emul = true;
-		mod_delayed_work(system_wq, &mcu->button_release_emul_work,
+		mod_delayed_work(system_percpu_wq, &mcu->button_release_emul_work,
 				 msecs_to_jiffies(FRONT_BUTTON_RELEASE_DELAY_MS));
 	} else if (mcu->button_pressed_emul) {
 		status |= OMNIA_STS_BUTTON_PRESSED;
diff --git a/drivers/platform/surface/surface_aggregator_registry.c b/drivers/platform/surface/surface_aggregator_registry.c
index 0599d5adf02e6..f0881edfb6161 100644
--- a/drivers/platform/surface/surface_aggregator_registry.c
+++ b/drivers/platform/surface/surface_aggregator_registry.c
@@ -295,8 +295,6 @@ static const struct software_node *ssam_node_group_sl6[] = {
 /* Devices for Surface Laptop 7. */
 static const struct software_node *ssam_node_group_sl7[] = {
 	&ssam_node_root,
-	&ssam_node_bat_ac,
-	&ssam_node_bat_main,
 	&ssam_node_tmp_perf_profile_with_fan,
 	&ssam_node_fan_speed,
 	&ssam_node_hid_sam_keyboard,
diff --git a/drivers/platform/surface/surfacepro3_button.c b/drivers/platform/surface/surfacepro3_button.c
index 0293bc517b544..388a3e1a488cb 100644
--- a/drivers/platform/surface/surfacepro3_button.c
+++ b/drivers/platform/surface/surfacepro3_button.c
@@ -185,12 +185,15 @@ static bool surface_button_check_MSHW0040(struct device *dev, acpi_handle handle
 
 static int surface_button_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct surface_button *button;
+	struct acpi_device *device;
 	struct input_dev *input;
-	const char *hid = acpi_device_hid(device);
 	int error;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	if (strncmp(acpi_device_bid(device), SURFACE_BUTTON_OBJ_NAME,
 	    strlen(SURFACE_BUTTON_OBJ_NAME)))
 		return -ENODEV;
@@ -210,7 +213,8 @@ static int surface_button_probe(struct platform_device *pdev)
 	}
 
 	strscpy(acpi_device_name(device), SURFACE_BUTTON_DEVICE_NAME);
-	snprintf(button->phys, sizeof(button->phys), "%s/buttons", hid);
+	snprintf(button->phys, sizeof(button->phys), "%s/buttons",
+		 acpi_device_hid(device));
 
 	input->name = acpi_device_name(device);
 	input->phys = button->phys;
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index 2ffa4ecf65b03..b54b5212b2047 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -118,6 +118,7 @@ config BITLAND_MIFS_WMI
 	depends on ACPI_WMI
 	depends on HWMON
 	depends on INPUT
+	depends on LEDS_CLASS
 	depends on POWER_SUPPLY
 	select ACPI_PLATFORM_PROFILE
 	select INPUT_SPARSEKMAP
@@ -345,6 +346,7 @@ config AYANEO_EC
 config MERAKI_MX100
 	tristate "Cisco Meraki MX100 Platform Driver"
 	depends on GPIOLIB
+	depends on LPC_ICH
 	depends on GPIO_ICH
 	depends on LEDS_CLASS
 	select LEDS_GPIO
diff --git a/drivers/platform/x86/acer-wireless.c b/drivers/platform/x86/acer-wireless.c
index f464b13a58aff..fae8e5ad0f97c 100644
--- a/drivers/platform/x86/acer-wireless.c
+++ b/drivers/platform/x86/acer-wireless.c
@@ -37,9 +37,14 @@ static void acer_wireless_notify(acpi_handle handle, u32 event, void *data)
 
 static int acer_wireless_probe(struct platform_device *pdev)
 {
+	struct acpi_device *adev;
 	struct input_dev *idev;
 	int ret;
 
+	adev = ACPI_COMPANION(&pdev->dev);
+	if (!adev)
+		return -ENODEV;
+
 	idev = devm_input_allocate_device(&pdev->dev);
 	if (!idev)
 		return -ENOMEM;
@@ -57,8 +62,7 @@ static int acer_wireless_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	return acpi_dev_install_notify_handler(ACPI_COMPANION(&pdev->dev),
-					       ACPI_DEVICE_NOTIFY,
+	return acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
 					       acer_wireless_notify,
 					       &pdev->dev);
 }
diff --git a/drivers/platform/x86/adv_swbutton.c b/drivers/platform/x86/adv_swbutton.c
index 6fa60f3fc53c0..8f7a26e6de81d 100644
--- a/drivers/platform/x86/adv_swbutton.c
+++ b/drivers/platform/x86/adv_swbutton.c
@@ -48,10 +48,14 @@ static int adv_swbutton_probe(struct platform_device *device)
 {
 	struct adv_swbutton *button;
 	struct input_dev *input;
-	acpi_handle handle = ACPI_HANDLE(&device->dev);
+	acpi_handle handle;
 	acpi_status status;
 	int error;
 
+	handle = ACPI_HANDLE(&device->dev);
+	if (!handle)
+		return -ENODEV;
+
 	button = devm_kzalloc(&device->dev, sizeof(*button), GFP_KERNEL);
 	if (!button)
 		return -ENOMEM;
diff --git a/drivers/platform/x86/amd/hfi/hfi.c b/drivers/platform/x86/amd/hfi/hfi.c
index 83863a5e0fbc6..e0ebcb0c4acde 100644
--- a/drivers/platform/x86/amd/hfi/hfi.c
+++ b/drivers/platform/x86/amd/hfi/hfi.c
@@ -33,7 +33,8 @@
 
 #define AMD_HFI_DRIVER		"amd_hfi"
 #define AMD_HFI_MAILBOX_COUNT		1
-#define AMD_HETERO_RANKING_TABLE_VER	2
+#define AMD_HETERO_RANKING_TABLE_MIN_VER	2
+#define AMD_HETERO_RANKING_TABLE_MAX_VER	3
 
 #define AMD_HETERO_CPUID_27	0x80000027
 
@@ -158,7 +159,8 @@ static int amd_hfi_fill_metadata(struct amd_hfi_data *amd_hfi_data)
 		dev_err(amd_hfi_data->dev, "invalid signature in shared memory\n");
 		return -EINVAL;
 	}
-	if (amd_hfi_data->shmem->version_number != AMD_HETERO_RANKING_TABLE_VER) {
+	if (amd_hfi_data->shmem->version_number < AMD_HETERO_RANKING_TABLE_MIN_VER ||
+	    amd_hfi_data->shmem->version_number > AMD_HETERO_RANKING_TABLE_MAX_VER) {
 		dev_err(amd_hfi_data->dev, "invalid version %d\n",
 			amd_hfi_data->shmem->version_number);
 		return -EINVAL;
diff --git a/drivers/platform/x86/asus-armoury.c b/drivers/platform/x86/asus-armoury.c
index 5b0987ccc2702..495dc1e31d40e 100644
--- a/drivers/platform/x86/asus-armoury.c
+++ b/drivers/platform/x86/asus-armoury.c
@@ -370,7 +370,7 @@ static ssize_t mini_led_mode_current_value_show(struct kobject *kobj,
 	if (err)
 		return err;
 
-	mode = FIELD_GET(ASUS_MINI_LED_MODE_MASK, 0);
+	mode = FIELD_GET(ASUS_MINI_LED_MODE_MASK, mode);
 
 	for (i = 0; i < mini_led_mode_map_size; i++)
 		if (mode == mini_led_mode_map[i])
@@ -386,6 +386,7 @@ static ssize_t mini_led_mode_current_value_store(struct kobject *kobj,
 {
 	u32 *mini_led_mode_map;
 	size_t mini_led_mode_map_size;
+	char mapped_value[12];
 	u32 mode;
 	int err;
 
@@ -414,9 +415,16 @@ static ssize_t mini_led_mode_current_value_store(struct kobject *kobj,
 		return -ENODEV;
 	}
 
-	return armoury_attr_uint_store(kobj, attr, buf, count,
-				       0, mini_led_mode_map[mode],
-				       NULL, asus_armoury.mini_led_dev_id);
+	/*
+	 * armoury_attr_uint_store() parses and sends the value from the
+	 * passed buffer; hand it the mapped firmware value so the device
+	 * receives the translated mode instead of the raw index.
+	 */
+	snprintf(mapped_value, sizeof(mapped_value), "%u", mini_led_mode_map[mode]);
+
+	return armoury_attr_uint_store(kobj, attr, mapped_value, count, 0,
+				       mini_led_mode_map[mode], NULL,
+				       asus_armoury.mini_led_dev_id);
 }
 
 static ssize_t mini_led_mode_possible_values_show(struct kobject *kobj,
diff --git a/drivers/platform/x86/asus-armoury.h b/drivers/platform/x86/asus-armoury.h
index c30d2b451e013..692978b61959f 100644
--- a/drivers/platform/x86/asus-armoury.h
+++ b/drivers/platform/x86/asus-armoury.h
@@ -346,6 +346,29 @@ struct power_data {
  * _def is not required and will be assumed to be default == max if missing.
  */
 static const struct dmi_system_id power_limits[] = {
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "FA401EA"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 15,
+				.ppt_pl1_spl_max = 95,
+				.ppt_pl2_sppt_min = 35,
+				.ppt_pl2_sppt_max = 100,
+				.ppt_pl3_fppt_min = 35,
+				.ppt_pl3_fppt_max = 115,
+			},
+			.dc_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 15,
+				.ppt_pl1_spl_max = 71,
+				.ppt_pl2_sppt_min = 35,
+				.ppt_pl2_sppt_max = 71,
+				.ppt_pl3_fppt_min = 35,
+				.ppt_pl3_fppt_max = 71,
+			},
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "FA401UM"),
@@ -886,6 +909,33 @@ static const struct dmi_system_id power_limits[] = {
 			.requires_fan_curve = true,
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "FX607VU"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 28,
+				.ppt_pl1_spl_def = 115,
+				.ppt_pl1_spl_max = 135,
+				.ppt_pl2_sppt_min = 28,
+				.ppt_pl2_sppt_max = 135,
+				.nv_dynamic_boost_min = 5,
+				.nv_dynamic_boost_max = 25,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+			},
+			.dc_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 25,
+				.ppt_pl1_spl_max = 45,
+				.ppt_pl2_sppt_min = 35,
+				.ppt_pl2_sppt_max = 60,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+			},
+			.requires_fan_curve = true,
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "GA401Q"),
@@ -1253,6 +1303,35 @@ static const struct dmi_system_id power_limits[] = {
 			},
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "GU605CP"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 45,
+				.ppt_pl1_spl_max = 75,
+				.ppt_pl2_sppt_min = 56,
+				.ppt_pl2_sppt_max = 95,
+				.nv_dynamic_boost_min = 5,
+				.nv_dynamic_boost_max = 15,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+				.nv_tgp_min = 55,
+				.nv_tgp_def = 75,
+				.nv_tgp_max = 95,
+			},
+			.dc_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 25,
+				.ppt_pl1_spl_max = 75,
+				.ppt_pl2_sppt_min = 32,
+				.ppt_pl2_sppt_max = 95,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+			},
+			.requires_fan_curve = true,
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "GU605CR"),
@@ -1759,6 +1838,40 @@ static const struct dmi_system_id power_limits[] = {
 			.requires_fan_curve = true,
 		},
 	},
+	{
+		.matches = {
+			DMI_MATCH(DMI_BOARD_NAME, "G614FR"),
+		},
+		.driver_data = &(struct power_data) {
+			.ac_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 30,
+				.ppt_pl1_spl_max = 120,
+				.ppt_pl2_sppt_min = 65,
+				.ppt_pl2_sppt_def = 140,
+				.ppt_pl2_sppt_max = 162,
+				.ppt_pl3_fppt_min = 65,
+				.ppt_pl3_fppt_def = 140,
+				.ppt_pl3_fppt_max = 162,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+				.nv_dynamic_boost_min = 5,
+				.nv_dynamic_boost_max = 25,
+				.nv_tgp_min = 65,
+				.nv_tgp_max = 115,
+			},
+			.dc_data = &(struct power_limits) {
+				.ppt_pl1_spl_min = 25,
+				.ppt_pl1_spl_max = 65,
+				.ppt_pl2_sppt_min = 25,
+				.ppt_pl2_sppt_max = 65,
+				.ppt_pl3_fppt_min = 35,
+				.ppt_pl3_fppt_max = 75,
+				.nv_temp_target_min = 75,
+				.nv_temp_target_max = 87,
+			},
+			.requires_fan_curve = true,
+		},
+	},
 	{
 		.matches = {
 			DMI_MATCH(DMI_BOARD_NAME, "G614J"),
diff --git a/drivers/platform/x86/asus-laptop.c b/drivers/platform/x86/asus-laptop.c
index dbbb6292cd11c..140ac8a105372 100644
--- a/drivers/platform/x86/asus-laptop.c
+++ b/drivers/platform/x86/asus-laptop.c
@@ -1826,10 +1826,14 @@ static bool asus_device_present;
 
 static int asus_acpi_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *device;
 	struct asus_laptop *asus;
 	int result;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	pr_notice("Asus Laptop Support version %s\n",
 		  ASUS_LAPTOP_VERSION);
 	asus = kzalloc_obj(struct asus_laptop);
diff --git a/drivers/platform/x86/barco-p50-gpio.c b/drivers/platform/x86/barco-p50-gpio.c
index 2a6d8607c4026..e1400382465dc 100644
--- a/drivers/platform/x86/barco-p50-gpio.c
+++ b/drivers/platform/x86/barco-p50-gpio.c
@@ -124,7 +124,6 @@ static const struct software_node vendor_key_node = {
 };
 
 static const struct software_node *p50_swnodes[] = {
-	&gpiochip_node,
 	&gpio_leds_node,
 	&identify_led_node,
 	&gpio_keys_node,
@@ -424,6 +423,13 @@ MODULE_DEVICE_TABLE(dmi, dmi_ids);
 static int __init p50_module_init(void)
 {
 	struct resource res = DEFINE_RES_IO(P50_GPIO_IO_PORT_BASE, P50_PORT_CMD + 1);
+	struct platform_device_info pdevinfo = {
+		.name = DRIVER_NAME,
+		.id = PLATFORM_DEVID_NONE,
+		.res = &res,
+		.num_res = 1,
+		.swnode = &gpiochip_node,
+	};
 	int ret;
 
 	if (!dmi_first_match(dmi_ids))
@@ -433,7 +439,7 @@ static int __init p50_module_init(void)
 	if (ret)
 		return ret;
 
-	gpio_pdev = platform_device_register_simple(DRIVER_NAME, PLATFORM_DEVID_NONE, &res, 1);
+	gpio_pdev = platform_device_register_full(&pdevinfo);
 	if (IS_ERR(gpio_pdev)) {
 		pr_err("failed registering %s: %ld\n", DRIVER_NAME, PTR_ERR(gpio_pdev));
 		platform_driver_unregister(&p50_gpio_driver);
diff --git a/drivers/platform/x86/classmate-laptop.c b/drivers/platform/x86/classmate-laptop.c
index e6eed3d65580a..05890815795d5 100644
--- a/drivers/platform/x86/classmate-laptop.c
+++ b/drivers/platform/x86/classmate-laptop.c
@@ -13,6 +13,7 @@
 #include <linux/input.h>
 #include <linux/rfkill.h>
 #include <linux/sysfs.h>
+#include <linux/platform_device.h>
 
 struct cmpc_accel {
 	int sensitivity;
@@ -38,8 +39,8 @@ struct cmpc_accel {
 
 typedef void (*input_device_init)(struct input_dev *dev);
 
-static int cmpc_add_acpi_notify_device(struct acpi_device *acpi, char *name,
-				       input_device_init idev_init)
+static int cmpc_add_notify_device(struct device *dev, char *name,
+				  input_device_init idev_init)
 {
 	struct input_dev *inputdev;
 	int error;
@@ -48,22 +49,20 @@ static int cmpc_add_acpi_notify_device(struct acpi_device *acpi, char *name,
 	if (!inputdev)
 		return -ENOMEM;
 	inputdev->name = name;
-	inputdev->dev.parent = &acpi->dev;
+	inputdev->dev.parent = dev;
 	idev_init(inputdev);
 	error = input_register_device(inputdev);
 	if (error) {
 		input_free_device(inputdev);
 		return error;
 	}
-	dev_set_drvdata(&acpi->dev, inputdev);
+	dev_set_drvdata(dev, inputdev);
 	return 0;
 }
 
-static int cmpc_remove_acpi_notify_device(struct acpi_device *acpi)
+static void cmpc_remove_notify_device(struct device *dev)
 {
-	struct input_dev *inputdev = dev_get_drvdata(&acpi->dev);
-	input_unregister_device(inputdev);
-	return 0;
+	input_unregister_device(dev_get_drvdata(dev));
 }
 
 /*
@@ -179,15 +178,17 @@ static acpi_status cmpc_get_accel_v4(acpi_handle handle,
 	return status;
 }
 
-static void cmpc_accel_handler_v4(struct acpi_device *dev, u32 event)
+static void cmpc_accel_handler_v4(acpi_handle handle, u32 event, void *data)
 {
+	struct device *dev = data;
+
 	if (event == 0x81) {
 		int16_t x, y, z;
 		acpi_status status;
 
-		status = cmpc_get_accel_v4(dev->handle, &x, &y, &z);
+		status = cmpc_get_accel_v4(ACPI_HANDLE(dev), &x, &y, &z);
 		if (ACPI_SUCCESS(status)) {
-			struct input_dev *inputdev = dev_get_drvdata(&dev->dev);
+			struct input_dev *inputdev = dev_get_drvdata(dev);
 
 			input_report_abs(inputdev, ABS_X, x);
 			input_report_abs(inputdev, ABS_Y, y);
@@ -317,18 +318,17 @@ static struct device_attribute cmpc_accel_g_select_attr_v4 = {
 
 static int cmpc_accel_open_v4(struct input_dev *input)
 {
-	struct acpi_device *acpi;
+	acpi_handle handle = ACPI_HANDLE(input->dev.parent);
 	struct cmpc_accel *accel;
 
-	acpi = to_acpi_device(input->dev.parent);
 	accel = dev_get_drvdata(&input->dev);
 	if (!accel)
 		return -ENXIO;
 
-	cmpc_accel_set_sensitivity_v4(acpi->handle, accel->sensitivity);
-	cmpc_accel_set_g_select_v4(acpi->handle, accel->g_select);
+	cmpc_accel_set_sensitivity_v4(handle, accel->sensitivity);
+	cmpc_accel_set_g_select_v4(handle, accel->g_select);
 
-	if (ACPI_SUCCESS(cmpc_start_accel_v4(acpi->handle))) {
+	if (ACPI_SUCCESS(cmpc_start_accel_v4(handle))) {
 		accel->inputdev_state = CMPC_ACCEL_DEV_STATE_OPEN;
 		return 0;
 	}
@@ -337,13 +337,11 @@ static int cmpc_accel_open_v4(struct input_dev *input)
 
 static void cmpc_accel_close_v4(struct input_dev *input)
 {
-	struct acpi_device *acpi;
 	struct cmpc_accel *accel;
 
-	acpi = to_acpi_device(input->dev.parent);
 	accel = dev_get_drvdata(&input->dev);
 
-	cmpc_stop_accel_v4(acpi->handle);
+	cmpc_stop_accel_v4(ACPI_HANDLE(input->dev.parent));
 	accel->inputdev_state = CMPC_ACCEL_DEV_STATE_CLOSED;
 }
 
@@ -367,7 +365,7 @@ static int cmpc_accel_suspend_v4(struct device *dev)
 	accel = dev_get_drvdata(&inputdev->dev);
 
 	if (accel->inputdev_state == CMPC_ACCEL_DEV_STATE_OPEN)
-		return cmpc_stop_accel_v4(to_acpi_device(dev)->handle);
+		return cmpc_stop_accel_v4(ACPI_HANDLE(dev));
 
 	return 0;
 }
@@ -381,12 +379,12 @@ static int cmpc_accel_resume_v4(struct device *dev)
 	accel = dev_get_drvdata(&inputdev->dev);
 
 	if (accel->inputdev_state == CMPC_ACCEL_DEV_STATE_OPEN) {
-		cmpc_accel_set_sensitivity_v4(to_acpi_device(dev)->handle,
-					      accel->sensitivity);
-		cmpc_accel_set_g_select_v4(to_acpi_device(dev)->handle,
-					   accel->g_select);
+		acpi_handle handle = ACPI_HANDLE(dev);
 
-		if (ACPI_FAILURE(cmpc_start_accel_v4(to_acpi_device(dev)->handle)))
+		cmpc_accel_set_sensitivity_v4(handle, accel->sensitivity);
+		cmpc_accel_set_g_select_v4(handle, accel->g_select);
+
+		if (ACPI_FAILURE(cmpc_start_accel_v4(handle)))
 			return -EIO;
 	}
 
@@ -394,18 +392,30 @@ static int cmpc_accel_resume_v4(struct device *dev)
 }
 #endif
 
-static int cmpc_accel_add_v4(struct acpi_device *acpi)
+static int cmpc_accel_probe_v4(struct platform_device *pdev)
 {
 	int error;
 	struct input_dev *inputdev;
 	struct cmpc_accel *accel;
+	struct acpi_device *acpi;
 
-	accel = kmalloc_obj(*accel);
+	acpi = ACPI_COMPANION(&pdev->dev);
+	if (!acpi)
+		return -ENODEV;
+
+	accel = devm_kzalloc(&pdev->dev, sizeof(*accel), GFP_KERNEL);
 	if (!accel)
 		return -ENOMEM;
 
 	accel->inputdev_state = CMPC_ACCEL_DEV_STATE_CLOSED;
 
+	error = cmpc_add_notify_device(&pdev->dev, "cmpc_accel_v4", cmpc_accel_idev_init_v4);
+	if (error)
+		return error;
+
+	inputdev = dev_get_drvdata(&pdev->dev);
+	dev_set_drvdata(&acpi->dev, inputdev);
+
 	accel->sensitivity = CMPC_ACCEL_SENSITIVITY_DEFAULT;
 	cmpc_accel_set_sensitivity_v4(acpi->handle, accel->sensitivity);
 
@@ -420,30 +430,35 @@ static int cmpc_accel_add_v4(struct acpi_device *acpi)
 	if (error)
 		goto failed_g_select;
 
-	error = cmpc_add_acpi_notify_device(acpi, "cmpc_accel_v4",
-					    cmpc_accel_idev_init_v4);
+	error = acpi_dev_install_notify_handler(acpi, ACPI_DEVICE_NOTIFY,
+						cmpc_accel_handler_v4, &pdev->dev);
 	if (error)
-		goto failed_input;
+		goto failed_notify_handler;
 
-	inputdev = dev_get_drvdata(&acpi->dev);
 	dev_set_drvdata(&inputdev->dev, accel);
 
 	return 0;
 
-failed_input:
+failed_notify_handler:
 	device_remove_file(&acpi->dev, &cmpc_accel_g_select_attr_v4);
 failed_g_select:
 	device_remove_file(&acpi->dev, &cmpc_accel_sensitivity_attr_v4);
 failed_sensitivity:
-	kfree(accel);
+	dev_set_drvdata(&acpi->dev, NULL);
+	cmpc_remove_notify_device(&pdev->dev);
 	return error;
 }
 
-static void cmpc_accel_remove_v4(struct acpi_device *acpi)
+static void cmpc_accel_remove_v4(struct platform_device *pdev)
 {
-	device_remove_file(&acpi->dev, &cmpc_accel_sensitivity_attr_v4);
+	struct acpi_device *acpi = ACPI_COMPANION(&pdev->dev);
+
+	acpi_dev_remove_notify_handler(acpi, ACPI_DEVICE_NOTIFY,
+				       cmpc_accel_handler_v4);
 	device_remove_file(&acpi->dev, &cmpc_accel_g_select_attr_v4);
-	cmpc_remove_acpi_notify_device(acpi);
+	device_remove_file(&acpi->dev, &cmpc_accel_sensitivity_attr_v4);
+	dev_set_drvdata(&acpi->dev, NULL);
+	cmpc_remove_notify_device(&pdev->dev);
 }
 
 static SIMPLE_DEV_PM_OPS(cmpc_accel_pm, cmpc_accel_suspend_v4,
@@ -454,16 +469,14 @@ static const struct acpi_device_id cmpc_accel_device_ids_v4[] = {
 	{"", 0}
 };
 
-static struct acpi_driver cmpc_accel_acpi_driver_v4 = {
-	.name = "cmpc_accel_v4",
-	.class = "cmpc_accel_v4",
-	.ids = cmpc_accel_device_ids_v4,
-	.ops = {
-		.add = cmpc_accel_add_v4,
-		.remove = cmpc_accel_remove_v4,
-		.notify = cmpc_accel_handler_v4,
+static struct platform_driver cmpc_accel_acpi_driver_v4 = {
+	.probe = cmpc_accel_probe_v4,
+	.remove = cmpc_accel_remove_v4,
+	.driver = {
+		.name = "cmpc_accel_v4",
+		.acpi_match_table = cmpc_accel_device_ids_v4,
+		.pm = &cmpc_accel_pm,
 	},
-	.drv.pm = &cmpc_accel_pm,
 };
 
 
@@ -543,15 +556,17 @@ static acpi_status cmpc_get_accel(acpi_handle handle,
 	return status;
 }
 
-static void cmpc_accel_handler(struct acpi_device *dev, u32 event)
+static void cmpc_accel_handler(acpi_handle handle, u32 event, void *data)
 {
+	struct device *dev = data;
+
 	if (event == 0x81) {
 		unsigned char x, y, z;
 		acpi_status status;
 
-		status = cmpc_get_accel(dev->handle, &x, &y, &z);
+		status = cmpc_get_accel(ACPI_HANDLE(dev), &x, &y, &z);
 		if (ACPI_SUCCESS(status)) {
-			struct input_dev *inputdev = dev_get_drvdata(&dev->dev);
+			struct input_dev *inputdev = dev_get_drvdata(dev);
 
 			input_report_abs(inputdev, ABS_X, x);
 			input_report_abs(inputdev, ABS_Y, y);
@@ -618,20 +633,14 @@ static struct device_attribute cmpc_accel_sensitivity_attr = {
 
 static int cmpc_accel_open(struct input_dev *input)
 {
-	struct acpi_device *acpi;
-
-	acpi = to_acpi_device(input->dev.parent);
-	if (ACPI_SUCCESS(cmpc_start_accel(acpi->handle)))
+	if (ACPI_SUCCESS(cmpc_start_accel(ACPI_HANDLE(input->dev.parent))))
 		return 0;
 	return -EIO;
 }
 
 static void cmpc_accel_close(struct input_dev *input)
 {
-	struct acpi_device *acpi;
-
-	acpi = to_acpi_device(input->dev.parent);
-	cmpc_stop_accel(acpi->handle);
+	cmpc_stop_accel(ACPI_HANDLE(input->dev.parent));
 }
 
 static void cmpc_accel_idev_init(struct input_dev *inputdev)
@@ -644,16 +653,28 @@ static void cmpc_accel_idev_init(struct input_dev *inputdev)
 	inputdev->close = cmpc_accel_close;
 }
 
-static int cmpc_accel_add(struct acpi_device *acpi)
+static int cmpc_accel_probe(struct platform_device *pdev)
 {
 	int error;
 	struct input_dev *inputdev;
 	struct cmpc_accel *accel;
+	struct acpi_device *acpi;
 
-	accel = kmalloc_obj(*accel);
+	acpi = ACPI_COMPANION(&pdev->dev);
+	if (!acpi)
+		return -ENODEV;
+
+	accel = devm_kzalloc(&pdev->dev, sizeof(*accel), GFP_KERNEL);
 	if (!accel)
 		return -ENOMEM;
 
+	error = cmpc_add_notify_device(&pdev->dev, "cmpc_accel", cmpc_accel_idev_init);
+	if (error)
+		return error;
+
+	inputdev = dev_get_drvdata(&pdev->dev);
+	dev_set_drvdata(&acpi->dev, inputdev);
+
 	accel->sensitivity = CMPC_ACCEL_SENSITIVITY_DEFAULT;
 	cmpc_accel_set_sensitivity(acpi->handle, accel->sensitivity);
 
@@ -661,27 +682,32 @@ static int cmpc_accel_add(struct acpi_device *acpi)
 	if (error)
 		goto failed_file;
 
-	error = cmpc_add_acpi_notify_device(acpi, "cmpc_accel",
-					    cmpc_accel_idev_init);
+	error = acpi_dev_install_notify_handler(acpi, ACPI_DEVICE_NOTIFY,
+						cmpc_accel_handler, &pdev->dev);
 	if (error)
-		goto failed_input;
+		goto failed_notify_handler;
 
-	inputdev = dev_get_drvdata(&acpi->dev);
 	dev_set_drvdata(&inputdev->dev, accel);
 
 	return 0;
 
-failed_input:
+failed_notify_handler:
 	device_remove_file(&acpi->dev, &cmpc_accel_sensitivity_attr);
 failed_file:
-	kfree(accel);
+	dev_set_drvdata(&acpi->dev, NULL);
+	cmpc_remove_notify_device(&pdev->dev);
 	return error;
 }
 
-static void cmpc_accel_remove(struct acpi_device *acpi)
+static void cmpc_accel_remove(struct platform_device *pdev)
 {
+	struct acpi_device *acpi = ACPI_COMPANION(&pdev->dev);
+
+	acpi_dev_remove_notify_handler(acpi, ACPI_DEVICE_NOTIFY,
+				       cmpc_accel_handler);
 	device_remove_file(&acpi->dev, &cmpc_accel_sensitivity_attr);
-	cmpc_remove_acpi_notify_device(acpi);
+	dev_set_drvdata(&acpi->dev, NULL);
+	cmpc_remove_notify_device(&pdev->dev);
 }
 
 static const struct acpi_device_id cmpc_accel_device_ids[] = {
@@ -689,15 +715,13 @@ static const struct acpi_device_id cmpc_accel_device_ids[] = {
 	{"", 0}
 };
 
-static struct acpi_driver cmpc_accel_acpi_driver = {
-	.name = "cmpc_accel",
-	.class = "cmpc_accel",
-	.ids = cmpc_accel_device_ids,
-	.ops = {
-		.add = cmpc_accel_add,
-		.remove = cmpc_accel_remove,
-		.notify = cmpc_accel_handler,
-	}
+static struct platform_driver cmpc_accel_acpi_driver = {
+	.probe = cmpc_accel_probe,
+	.remove = cmpc_accel_remove,
+	.driver = {
+		.name = "cmpc_accel",
+		.acpi_match_table = cmpc_accel_device_ids,
+	},
 };
 
 
@@ -722,13 +746,14 @@ static acpi_status cmpc_get_tablet(acpi_handle handle,
 	return status;
 }
 
-static void cmpc_tablet_handler(struct acpi_device *dev, u32 event)
+static void cmpc_tablet_handler(acpi_handle handle, u32 event, void *data)
 {
+	struct device *dev = data;
 	unsigned long long val = 0;
-	struct input_dev *inputdev = dev_get_drvdata(&dev->dev);
+	struct input_dev *inputdev = dev_get_drvdata(dev);
 
 	if (event == 0x81) {
-		if (ACPI_SUCCESS(cmpc_get_tablet(dev->handle, &val))) {
+		if (ACPI_SUCCESS(cmpc_get_tablet(ACPI_HANDLE(dev), &val))) {
 			input_report_switch(inputdev, SW_TABLET_MODE, !val);
 			input_sync(inputdev);
 		}
@@ -737,28 +762,44 @@ static void cmpc_tablet_handler(struct acpi_device *dev, u32 event)
 
 static void cmpc_tablet_idev_init(struct input_dev *inputdev)
 {
+	acpi_handle handle = ACPI_HANDLE(inputdev->dev.parent);
 	unsigned long long val = 0;
-	struct acpi_device *acpi;
 
 	set_bit(EV_SW, inputdev->evbit);
 	set_bit(SW_TABLET_MODE, inputdev->swbit);
 
-	acpi = to_acpi_device(inputdev->dev.parent);
-	if (ACPI_SUCCESS(cmpc_get_tablet(acpi->handle, &val))) {
+	if (ACPI_SUCCESS(cmpc_get_tablet(handle, &val))) {
 		input_report_switch(inputdev, SW_TABLET_MODE, !val);
 		input_sync(inputdev);
 	}
 }
 
-static int cmpc_tablet_add(struct acpi_device *acpi)
+static int cmpc_tablet_probe(struct platform_device *pdev)
 {
-	return cmpc_add_acpi_notify_device(acpi, "cmpc_tablet",
-					   cmpc_tablet_idev_init);
+	struct acpi_device *acpi;
+	int error;
+
+	acpi = ACPI_COMPANION(&pdev->dev);
+	if (!acpi)
+		return -ENODEV;
+
+	error = cmpc_add_notify_device(&pdev->dev, "cmpc_tablet", cmpc_tablet_idev_init);
+	if (error)
+		return error;
+
+	error = acpi_dev_install_notify_handler(acpi, ACPI_DEVICE_NOTIFY,
+						cmpc_tablet_handler, &pdev->dev);
+	if (error)
+		cmpc_remove_notify_device(&pdev->dev);
+
+	return error;
 }
 
-static void cmpc_tablet_remove(struct acpi_device *acpi)
+static void cmpc_tablet_remove(struct platform_device *pdev)
 {
-	cmpc_remove_acpi_notify_device(acpi);
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, cmpc_tablet_handler);
+	cmpc_remove_notify_device(&pdev->dev);
 }
 
 #ifdef CONFIG_PM_SLEEP
@@ -767,7 +808,7 @@ static int cmpc_tablet_resume(struct device *dev)
 	struct input_dev *inputdev = dev_get_drvdata(dev);
 
 	unsigned long long val = 0;
-	if (ACPI_SUCCESS(cmpc_get_tablet(to_acpi_device(dev)->handle, &val))) {
+	if (ACPI_SUCCESS(cmpc_get_tablet(ACPI_HANDLE(dev), &val))) {
 		input_report_switch(inputdev, SW_TABLET_MODE, !val);
 		input_sync(inputdev);
 	}
@@ -782,16 +823,14 @@ static const struct acpi_device_id cmpc_tablet_device_ids[] = {
 	{"", 0}
 };
 
-static struct acpi_driver cmpc_tablet_acpi_driver = {
-	.name = "cmpc_tablet",
-	.class = "cmpc_tablet",
-	.ids = cmpc_tablet_device_ids,
-	.ops = {
-		.add = cmpc_tablet_add,
-		.remove = cmpc_tablet_remove,
-		.notify = cmpc_tablet_handler,
+static struct platform_driver cmpc_tablet_acpi_driver = {
+	.probe = cmpc_tablet_probe,
+	.remove = cmpc_tablet_remove,
+	.driver = {
+		.name = "cmpc_tablet",
+		.acpi_match_table = cmpc_tablet_device_ids,
+		.pm = &cmpc_tablet_pm,
 	},
-	.drv.pm = &cmpc_tablet_pm,
 };
 
 
@@ -958,11 +997,16 @@ struct ipml200_dev {
 	struct rfkill *rf;
 };
 
-static int cmpc_ipml_add(struct acpi_device *acpi)
+static int cmpc_ipml_probe(struct platform_device *pdev)
 {
 	int retval;
 	struct ipml200_dev *ipml;
 	struct backlight_properties props;
+	acpi_handle handle;
+
+	handle = ACPI_HANDLE(&pdev->dev);
+	if (!handle)
+		return -ENODEV;
 
 	ipml = kmalloc_obj(*ipml);
 	if (ipml == NULL)
@@ -971,16 +1015,16 @@ static int cmpc_ipml_add(struct acpi_device *acpi)
 	memset(&props, 0, sizeof(struct backlight_properties));
 	props.type = BACKLIGHT_PLATFORM;
 	props.max_brightness = 7;
-	ipml->bd = backlight_device_register("cmpc_bl", &acpi->dev,
-					     acpi->handle, &cmpc_bl_ops,
+	ipml->bd = backlight_device_register("cmpc_bl", &pdev->dev,
+					     handle, &cmpc_bl_ops,
 					     &props);
 	if (IS_ERR(ipml->bd)) {
 		retval = PTR_ERR(ipml->bd);
 		goto out_bd;
 	}
 
-	ipml->rf = rfkill_alloc("cmpc_rfkill", &acpi->dev, RFKILL_TYPE_WLAN,
-				&cmpc_rfkill_ops, acpi->handle);
+	ipml->rf = rfkill_alloc("cmpc_rfkill", &pdev->dev, RFKILL_TYPE_WLAN,
+				&cmpc_rfkill_ops, handle);
 	/*
 	 * If RFKILL is disabled, rfkill_alloc will return ERR_PTR(-ENODEV).
 	 * This is OK, however, since all other uses of the device will not
@@ -994,7 +1038,7 @@ static int cmpc_ipml_add(struct acpi_device *acpi)
 		}
 	}
 
-	dev_set_drvdata(&acpi->dev, ipml);
+	platform_set_drvdata(pdev, ipml);
 	return 0;
 
 out_bd:
@@ -1002,11 +1046,11 @@ static int cmpc_ipml_add(struct acpi_device *acpi)
 	return retval;
 }
 
-static void cmpc_ipml_remove(struct acpi_device *acpi)
+static void cmpc_ipml_remove(struct platform_device *pdev)
 {
 	struct ipml200_dev *ipml;
 
-	ipml = dev_get_drvdata(&acpi->dev);
+	ipml = platform_get_drvdata(pdev);
 
 	backlight_device_unregister(ipml->bd);
 
@@ -1023,14 +1067,13 @@ static const struct acpi_device_id cmpc_ipml_device_ids[] = {
 	{"", 0}
 };
 
-static struct acpi_driver cmpc_ipml_acpi_driver = {
-	.name = "cmpc",
-	.class = "cmpc",
-	.ids = cmpc_ipml_device_ids,
-	.ops = {
-		.add = cmpc_ipml_add,
-		.remove = cmpc_ipml_remove
-	}
+static struct platform_driver cmpc_ipml_acpi_driver = {
+	.probe = cmpc_ipml_probe,
+	.remove = cmpc_ipml_remove,
+	.driver = {
+		.name = "cmpc",
+		.acpi_match_table = cmpc_ipml_device_ids,
+	},
 };
 
 
@@ -1053,14 +1096,15 @@ static int cmpc_keys_codes[] = {
 	KEY_MAX
 };
 
-static void cmpc_keys_handler(struct acpi_device *dev, u32 event)
+static void cmpc_keys_handler(acpi_handle handle, u32 event, void *data)
 {
+	struct device *dev = data;
 	struct input_dev *inputdev;
 	int code = KEY_MAX;
 
 	if ((event & 0x0F) < ARRAY_SIZE(cmpc_keys_codes))
 		code = cmpc_keys_codes[event & 0x0F];
-	inputdev = dev_get_drvdata(&dev->dev);
+	inputdev = dev_get_drvdata(dev);
 	input_report_key(inputdev, code, !(event & 0x10));
 	input_sync(inputdev);
 }
@@ -1074,15 +1118,32 @@ static void cmpc_keys_idev_init(struct input_dev *inputdev)
 		set_bit(cmpc_keys_codes[i], inputdev->keybit);
 }
 
-static int cmpc_keys_add(struct acpi_device *acpi)
+static int cmpc_keys_probe(struct platform_device *pdev)
 {
-	return cmpc_add_acpi_notify_device(acpi, "cmpc_keys",
-					   cmpc_keys_idev_init);
+	struct acpi_device *acpi;
+	int error;
+
+	acpi = ACPI_COMPANION(&pdev->dev);
+	if (!acpi)
+		return -ENODEV;
+
+	error = cmpc_add_notify_device(&pdev->dev, "cmpc_keys", cmpc_keys_idev_init);
+	if (error)
+		return error;
+
+	error = acpi_dev_install_notify_handler(acpi, ACPI_DEVICE_NOTIFY,
+						cmpc_keys_handler, &pdev->dev);
+	if (error)
+		cmpc_remove_notify_device(&pdev->dev);
+
+	return error;
 }
 
-static void cmpc_keys_remove(struct acpi_device *acpi)
+static void cmpc_keys_remove(struct platform_device *pdev)
 {
-	cmpc_remove_acpi_notify_device(acpi);
+	acpi_dev_remove_notify_handler(ACPI_COMPANION(&pdev->dev),
+				       ACPI_DEVICE_NOTIFY, cmpc_keys_handler);
+	cmpc_remove_notify_device(&pdev->dev);
 }
 
 static const struct acpi_device_id cmpc_keys_device_ids[] = {
@@ -1090,15 +1151,13 @@ static const struct acpi_device_id cmpc_keys_device_ids[] = {
 	{"", 0}
 };
 
-static struct acpi_driver cmpc_keys_acpi_driver = {
-	.name = "cmpc_keys",
-	.class = "cmpc_keys",
-	.ids = cmpc_keys_device_ids,
-	.ops = {
-		.add = cmpc_keys_add,
-		.remove = cmpc_keys_remove,
-		.notify = cmpc_keys_handler,
-	}
+static struct platform_driver cmpc_keys_acpi_driver = {
+	.probe = cmpc_keys_probe,
+	.remove = cmpc_keys_remove,
+	.driver = {
+		.name = "cmpc_keys",
+		.acpi_match_table = cmpc_keys_device_ids,
+	},
 };
 
 
@@ -1110,39 +1169,39 @@ static int cmpc_init(void)
 {
 	int r;
 
-	r = acpi_bus_register_driver(&cmpc_keys_acpi_driver);
+	r = platform_driver_register(&cmpc_keys_acpi_driver);
 	if (r)
 		goto failed_keys;
 
-	r = acpi_bus_register_driver(&cmpc_ipml_acpi_driver);
+	r = platform_driver_register(&cmpc_ipml_acpi_driver);
 	if (r)
 		goto failed_bl;
 
-	r = acpi_bus_register_driver(&cmpc_tablet_acpi_driver);
+	r = platform_driver_register(&cmpc_tablet_acpi_driver);
 	if (r)
 		goto failed_tablet;
 
-	r = acpi_bus_register_driver(&cmpc_accel_acpi_driver);
+	r = platform_driver_register(&cmpc_accel_acpi_driver);
 	if (r)
 		goto failed_accel;
 
-	r = acpi_bus_register_driver(&cmpc_accel_acpi_driver_v4);
+	r = platform_driver_register(&cmpc_accel_acpi_driver_v4);
 	if (r)
 		goto failed_accel_v4;
 
 	return r;
 
 failed_accel_v4:
-	acpi_bus_unregister_driver(&cmpc_accel_acpi_driver);
+	platform_driver_unregister(&cmpc_accel_acpi_driver);
 
 failed_accel:
-	acpi_bus_unregister_driver(&cmpc_tablet_acpi_driver);
+	platform_driver_unregister(&cmpc_tablet_acpi_driver);
 
 failed_tablet:
-	acpi_bus_unregister_driver(&cmpc_ipml_acpi_driver);
+	platform_driver_unregister(&cmpc_ipml_acpi_driver);
 
 failed_bl:
-	acpi_bus_unregister_driver(&cmpc_keys_acpi_driver);
+	platform_driver_unregister(&cmpc_keys_acpi_driver);
 
 failed_keys:
 	return r;
@@ -1150,11 +1209,11 @@ static int cmpc_init(void)
 
 static void cmpc_exit(void)
 {
-	acpi_bus_unregister_driver(&cmpc_accel_acpi_driver_v4);
-	acpi_bus_unregister_driver(&cmpc_accel_acpi_driver);
-	acpi_bus_unregister_driver(&cmpc_tablet_acpi_driver);
-	acpi_bus_unregister_driver(&cmpc_ipml_acpi_driver);
-	acpi_bus_unregister_driver(&cmpc_keys_acpi_driver);
+	platform_driver_unregister(&cmpc_accel_acpi_driver_v4);
+	platform_driver_unregister(&cmpc_accel_acpi_driver);
+	platform_driver_unregister(&cmpc_tablet_acpi_driver);
+	platform_driver_unregister(&cmpc_ipml_acpi_driver);
+	platform_driver_unregister(&cmpc_keys_acpi_driver);
 }
 
 module_init(cmpc_init);
diff --git a/drivers/platform/x86/dell/alienware-wmi-base.c b/drivers/platform/x86/dell/alienware-wmi-base.c
index 64562b92314fa..19cadf21203a6 100644
--- a/drivers/platform/x86/dell/alienware-wmi-base.c
+++ b/drivers/platform/x86/dell/alienware-wmi-base.c
@@ -12,8 +12,13 @@
 #include <linux/cleanup.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/slab.h>
+#include <linux/types.h>
 #include <linux/dmi.h>
 #include <linux/leds.h>
+
+#include <asm/byteorder.h>
+
 #include "alienware-wmi.h"
 
 MODULE_AUTHOR("Mario Limonciello <mario.limonciello@outlook.com>");
@@ -150,22 +155,22 @@ u8 alienware_interface;
 int alienware_wmi_command(struct wmi_device *wdev, u32 method_id,
 			  void *in_args, size_t in_size, u32 *out_data)
 {
-	struct acpi_buffer out = {ACPI_ALLOCATE_BUFFER, NULL};
-	struct acpi_buffer in = {in_size, in_args};
-	acpi_status ret;
+	struct wmi_buffer out, in = {
+		.data = in_args,
+		.length = in_size,
+	};
+	int ret;
 
-	ret = wmidev_evaluate_method(wdev, 0, method_id, &in, out_data ? &out : NULL);
-	if (ACPI_FAILURE(ret))
-		return -EIO;
+	if (!out_data)
+		return wmidev_invoke_procedure(wdev, 0, method_id, &in);
 
-	union acpi_object *obj __free(kfree) = out.pointer;
+	ret = wmidev_invoke_method(wdev, 0, method_id, &in, &out,
+				   sizeof(*out_data));
+	if (ret)
+		return ret;
 
-	if (out_data) {
-		if (obj && obj->type == ACPI_TYPE_INTEGER)
-			*out_data = (u32)obj->integer.value;
-		else
-			return -ENOMSG;
-	}
+	__le32 *data __free(kfree) = out.data;
+	*out_data = le32_to_cpu(*data);
 
 	return 0;
 }
diff --git a/drivers/platform/x86/dell/dell-rbtn.c b/drivers/platform/x86/dell/dell-rbtn.c
index 34af9f4ff7419..180b8c6720e65 100644
--- a/drivers/platform/x86/dell/dell-rbtn.c
+++ b/drivers/platform/x86/dell/dell-rbtn.c
@@ -396,11 +396,15 @@ static void rbtn_cleanup(struct device *dev)
 
 static int rbtn_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct rbtn_data *rbtn_data;
+	struct acpi_device *device;
 	enum rbtn_type type;
 	int ret = 0;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	type = rbtn_check(device);
 	if (type == RBTN_UNKNOWN) {
 		dev_info(&pdev->dev, "Unknown device type\n");
diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c b/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c
index c2dd2de6bc203..db278ff4cc4d4 100644
--- a/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c
+++ b/drivers/platform/x86/dell/dell-wmi-sysman/biosattr-interface.c
@@ -13,8 +13,8 @@
 #define SETBIOSDEFAULTS_METHOD_ID					0x03
 #define SETATTRIBUTE_METHOD_ID						0x04
 
-static int call_biosattributes_interface(struct wmi_device *wdev, char *in_args, size_t size,
-					int method_id)
+static int call_biosattributes_interface(struct wmi_device *wdev, u8 *in_args,
+					 size_t size, int method_id)
 {
 	struct acpi_buffer output = {ACPI_ALLOCATE_BUFFER, NULL};
 	struct acpi_buffer input;
@@ -51,7 +51,7 @@ int set_attribute(const char *a_name, const char *a_value)
 {
 	size_t security_area_size, buffer_size;
 	size_t a_name_size, a_value_size;
-	char *buffer = NULL, *start;
+	u8 *buffer = NULL, *start;
 	int ret;
 
 	mutex_lock(&wmi_priv.mutex);
@@ -109,7 +109,7 @@ int set_bios_defaults(u8 deftype)
 {
 	size_t security_area_size, buffer_size;
 	size_t integer_area_size = sizeof(u8);
-	char *buffer = NULL;
+	u8 *buffer = NULL;
 	u8 *defaultType;
 	int ret;
 
diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h
index 5278a93fdaf79..3bddedad5eba2 100644
--- a/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h
+++ b/drivers/platform/x86/dell/dell-wmi-sysman/dell-wmi-sysman.h
@@ -190,8 +190,8 @@ int init_bios_attr_set_interface(void);
 int map_wmi_error(int error_code);
 size_t calculate_string_buffer(const char *str);
 size_t calculate_security_buffer(const char *authentication);
-void populate_security_buffer(char *buffer, const char *authentication);
-ssize_t populate_string_buffer(char *buffer, size_t buffer_len, const char *str);
+void populate_security_buffer(u8 *buffer, const char *authentication);
+ssize_t populate_string_buffer(u8 *buffer, size_t buffer_len, const char *str);
 int set_new_password(const char *password_type, const char *new);
 int init_bios_attr_pass_interface(void);
 void exit_bios_attr_pass_interface(void);
diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c b/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c
index e586f7957946b..026c134b3f977 100644
--- a/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c
+++ b/drivers/platform/x86/dell/dell-wmi-sysman/passwordattr-interface.c
@@ -8,7 +8,7 @@
 #include <linux/wmi.h>
 #include "dell-wmi-sysman.h"
 
-static int call_password_interface(struct wmi_device *wdev, char *in_args, size_t size)
+static int call_password_interface(struct wmi_device *wdev, u8 *in_args, size_t size)
 {
 	struct acpi_buffer output = {ACPI_ALLOCATE_BUFFER, NULL};
 	struct acpi_buffer input;
@@ -42,7 +42,7 @@ int set_new_password(const char *password_type, const char *new)
 {
 	size_t password_type_size, current_password_size, new_size;
 	size_t security_area_size, buffer_size;
-	char *buffer = NULL, *start;
+	u8 *buffer = NULL, *start;
 	char *current_password;
 	int ret;
 
diff --git a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c
index 51d25fdc13897..ab46a023cc34d 100644
--- a/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c
+++ b/drivers/platform/x86/dell/dell-wmi-sysman/sysman.c
@@ -36,7 +36,7 @@ static int reset_option = -1;
  * @buffer_len: length of the destination buffer
  * @str: the string to insert into buffer
  */
-ssize_t populate_string_buffer(char *buffer, size_t buffer_len, const char *str)
+ssize_t populate_string_buffer(u8 *buffer, size_t buffer_len, const char *str)
 {
 	u16 *length = (u16 *)buffer;
 	u16 *target = length + 1;
@@ -87,10 +87,10 @@ size_t calculate_security_buffer(const char *authentication)
  *
  * Currently only supported type is PLAIN TEXT
  */
-void populate_security_buffer(char *buffer, const char *authentication)
+void populate_security_buffer(u8 *buffer, const char *authentication)
 {
 	size_t seclen = strlen(authentication);
-	char *auth = buffer + sizeof(u32) * 2;
+	u8 *auth = buffer + sizeof(u32) * 2;
 	u32 *sectype = (u32 *) buffer;
 	u32 *seclenp = sectype + 1;
 
diff --git a/drivers/platform/x86/dell/dell_rbu.c b/drivers/platform/x86/dell/dell_rbu.c
index 3fa9de9aa47b8..768b15f406d3b 100644
--- a/drivers/platform/x86/dell/dell_rbu.c
+++ b/drivers/platform/x86/dell/dell_rbu.c
@@ -562,9 +562,9 @@ static ssize_t image_type_write(struct file *filp, struct kobject *kobj,
 		buffer[count] = '\0';
 
 	if (strstr(buffer, "mono"))
-		strcpy(image_type, "mono");
+		strscpy(image_type, "mono");
 	else if (strstr(buffer, "packet"))
-		strcpy(image_type, "packet");
+		strscpy(image_type, "packet");
 	else if (strstr(buffer, "init")) {
 		/*
 		 * If due to the user error the driver gets in a bad
diff --git a/drivers/platform/x86/eeepc-laptop.c b/drivers/platform/x86/eeepc-laptop.c
index 02a71095920e5..d18a80907611e 100644
--- a/drivers/platform/x86/eeepc-laptop.c
+++ b/drivers/platform/x86/eeepc-laptop.c
@@ -1363,10 +1363,14 @@ static bool eeepc_device_present;
 
 static int eeepc_acpi_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *device;
 	struct eeepc_laptop *eeepc;
 	int result;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	pr_notice(EEEPC_LAPTOP_NAME "\n");
 	eeepc = kzalloc_obj(struct eeepc_laptop);
 	if (!eeepc)
diff --git a/drivers/platform/x86/fujitsu-laptop.c b/drivers/platform/x86/fujitsu-laptop.c
index 2e265be2267ed..54d0b9cec4d3c 100644
--- a/drivers/platform/x86/fujitsu-laptop.c
+++ b/drivers/platform/x86/fujitsu-laptop.c
@@ -530,10 +530,14 @@ static void acpi_fujitsu_bl_notify(acpi_handle handle, u32 event, void *data)
 
 static int acpi_fujitsu_bl_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *device;
 	struct fujitsu_bl *priv;
 	int ret;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	if (acpi_video_get_backlight_type() != acpi_backlight_vendor)
 		return -ENODEV;
 
@@ -993,10 +997,14 @@ static void acpi_fujitsu_laptop_notify(acpi_handle handle, u32 event, void *data
 
 static int acpi_fujitsu_laptop_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct fujitsu_laptop *priv;
+	struct acpi_device *device;
 	int ret, i = 0;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
diff --git a/drivers/platform/x86/fujitsu-tablet.c b/drivers/platform/x86/fujitsu-tablet.c
index 8319df28e9b8a..2f8c1b89cbca1 100644
--- a/drivers/platform/x86/fujitsu-tablet.c
+++ b/drivers/platform/x86/fujitsu-tablet.c
@@ -445,10 +445,14 @@ static acpi_status fujitsu_walk_resources(struct acpi_resource *res, void *data)
 
 static int acpi_fujitsu_probe(struct platform_device *pdev)
 {
-	struct acpi_device *adev = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *adev;
 	acpi_status status;
 	int error;
 
+	adev = ACPI_COMPANION(&pdev->dev);
+	if (!adev)
+		return -ENODEV;
+
 	status = acpi_walk_resources(adev->handle, METHOD_NAME__CRS,
 			fujitsu_walk_resources, NULL);
 	if (ACPI_FAILURE(status) || !fujitsu.irq || !fujitsu.io_base)
diff --git a/drivers/platform/x86/hp/hp-wmi.c b/drivers/platform/x86/hp/hp-wmi.c
index 6950bec2a9d80..f63bc00d9a9b3 100644
--- a/drivers/platform/x86/hp/hp-wmi.c
+++ b/drivers/platform/x86/hp/hp-wmi.c
@@ -189,6 +189,10 @@ static const char * const victus_thermal_profile_boards[] = {
 
 /* DMI Board names of Victus 16-r and Victus 16-s laptops */
 static const struct dmi_system_id victus_s_thermal_profile_boards[] __initconst = {
+	{
+		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8902") },
+		.driver_data = (void *)&omen_v1_legacy_thermal_params,
+	},
 	{
 		.matches = { DMI_MATCH(DMI_BOARD_NAME, "8A44") },
 		.driver_data = (void *)&omen_v1_legacy_thermal_params,
diff --git a/drivers/platform/x86/hp/hp_accel.c b/drivers/platform/x86/hp/hp_accel.c
index 10d5af18d6398..39b73dc473f1c 100644
--- a/drivers/platform/x86/hp/hp_accel.c
+++ b/drivers/platform/x86/hp/hp_accel.c
@@ -300,6 +300,9 @@ static int lis3lv02d_probe(struct platform_device *device)
 	int ret;
 
 	lis3_dev.bus_priv = ACPI_COMPANION(&device->dev);
+	if (!lis3_dev.bus_priv)
+		return -ENODEV;
+
 	lis3_dev.init = lis3lv02d_acpi_init;
 	lis3_dev.read = lis3lv02d_acpi_read;
 	lis3_dev.write = lis3lv02d_acpi_write;
diff --git a/drivers/platform/x86/intel/hid.c b/drivers/platform/x86/intel/hid.c
index 2ddd8af8c1ce9..085093506dda9 100644
--- a/drivers/platform/x86/intel/hid.c
+++ b/drivers/platform/x86/intel/hid.c
@@ -688,12 +688,16 @@ static bool button_array_present(struct platform_device *device)
 
 static int intel_hid_probe(struct platform_device *device)
 {
-	acpi_handle handle = ACPI_HANDLE(&device->dev);
 	unsigned long long mode, dummy;
 	struct intel_hid_priv *priv;
+	acpi_handle handle;
 	acpi_status status;
 	int err;
 
+	handle = ACPI_HANDLE(&device->dev);
+	if (!handle)
+		return -ENODEV;
+
 	intel_hid_init_dsm(handle);
 
 	if (!intel_hid_evaluate_method(handle, INTEL_HID_DSM_HDMM_FN, &mode)) {
diff --git a/drivers/platform/x86/intel/int1092/intel_sar.c b/drivers/platform/x86/intel/int1092/intel_sar.c
index 88822023a1490..849f7b415c1e5 100644
--- a/drivers/platform/x86/intel/int1092/intel_sar.c
+++ b/drivers/platform/x86/intel/int1092/intel_sar.c
@@ -245,15 +245,20 @@ static void sar_get_data(int reg, struct wwan_sar_context *context)
 static int sar_probe(struct platform_device *device)
 {
 	struct wwan_sar_context *context;
+	acpi_handle handle;
 	int reg;
 	int result;
 
+	handle = ACPI_HANDLE(&device->dev);
+	if (!handle)
+		return -ENODEV;
+
 	context = kzalloc_obj(*context);
 	if (!context)
 		return -ENOMEM;
 
 	context->sar_device = device;
-	context->handle = ACPI_HANDLE(&device->dev);
+	context->handle = handle;
 	dev_set_drvdata(&device->dev, context);
 
 	result = guid_parse(SAR_DSM_UUID, &context->guid);
diff --git a/drivers/platform/x86/intel/int3472/tps68470_board_data.c b/drivers/platform/x86/intel/int3472/tps68470_board_data.c
index c1ddbf9a82c0b..c53542465fb46 100644
--- a/drivers/platform/x86/intel/int3472/tps68470_board_data.c
+++ b/drivers/platform/x86/intel/int3472/tps68470_board_data.c
@@ -238,7 +238,7 @@ static struct regulator_consumer_supply ovti5675_dvdd_consumer_supplies[] = {
 	REGULATOR_SUPPLY("dvdd", "i2c-OVTI5675:00"),
 };
 
-static const struct regulator_init_data msi_p14_ai_evo_tps68470_core_reg_init_data = {
+static const struct regulator_init_data msi_prestige_ai_evo_tps68470_core_reg_init_data = {
 	.constraints = {
 		.min_uV = 1200000,
 		.max_uV = 1200000,
@@ -249,7 +249,7 @@ static const struct regulator_init_data msi_p14_ai_evo_tps68470_core_reg_init_da
 	.consumer_supplies = ovti5675_dvdd_consumer_supplies,
 };
 
-static const struct regulator_init_data msi_p14_ai_evo_tps68470_ana_reg_init_data = {
+static const struct regulator_init_data msi_prestige_ai_evo_tps68470_ana_reg_init_data = {
 	.constraints = {
 		.min_uV = 2815200,
 		.max_uV = 2815200,
@@ -260,7 +260,7 @@ static const struct regulator_init_data msi_p14_ai_evo_tps68470_ana_reg_init_dat
 	.consumer_supplies = ovti5675_avdd_consumer_supplies,
 };
 
-static const struct regulator_init_data msi_p14_ai_evo_tps68470_vio_reg_init_data = {
+static const struct regulator_init_data msi_prestige_ai_evo_tps68470_vio_reg_init_data = {
 	.constraints = {
 		.min_uV = 1800600,
 		.max_uV = 1800600,
@@ -269,7 +269,7 @@ static const struct regulator_init_data msi_p14_ai_evo_tps68470_vio_reg_init_dat
 	},
 };
 
-static const struct regulator_init_data msi_p14_ai_evo_tps68470_vsio_reg_init_data = {
+static const struct regulator_init_data msi_prestige_ai_evo_tps68470_vsio_reg_init_data = {
 	.constraints = {
 		.min_uV = 1800600,
 		.max_uV = 1800600,
@@ -280,12 +280,92 @@ static const struct regulator_init_data msi_p14_ai_evo_tps68470_vsio_reg_init_da
 	.consumer_supplies = ovti5675_dovdd_consumer_supplies,
 };
 
-static const struct tps68470_regulator_platform_data msi_p14_ai_evo_tps68470_pdata = {
+static const struct tps68470_regulator_platform_data msi_prestige_ai_evo_tps68470_pdata = {
 	.reg_init_data = {
-		[TPS68470_CORE] = &msi_p14_ai_evo_tps68470_core_reg_init_data,
-		[TPS68470_ANA]  = &msi_p14_ai_evo_tps68470_ana_reg_init_data,
-		[TPS68470_VIO]  = &msi_p14_ai_evo_tps68470_vio_reg_init_data,
-		[TPS68470_VSIO] = &msi_p14_ai_evo_tps68470_vsio_reg_init_data,
+		[TPS68470_CORE] = &msi_prestige_ai_evo_tps68470_core_reg_init_data,
+		[TPS68470_ANA]  = &msi_prestige_ai_evo_tps68470_ana_reg_init_data,
+		[TPS68470_VIO]  = &msi_prestige_ai_evo_tps68470_vio_reg_init_data,
+		[TPS68470_VSIO] = &msi_prestige_ai_evo_tps68470_vsio_reg_init_data,
+	},
+};
+
+/* Settings for Intel NVL platform */
+
+static struct regulator_consumer_supply ovti13b1_core_consumer_supplies[] = {
+	REGULATOR_SUPPLY("dvdd", "i2c-OVTI13B1:01"),
+};
+
+static struct regulator_consumer_supply ovti13b1_ana_consumer_supplies[] = {
+	REGULATOR_SUPPLY("avdd", "i2c-OVTI13B1:01"),
+};
+
+static struct regulator_consumer_supply ovti13b1_vcm_consumer_supplies[] = {
+	REGULATOR_SUPPLY("vcc", "i2c-OVTI13B1:01-VCM"),
+};
+
+static struct regulator_consumer_supply ovti13b1_vsio_consumer_supplies[] = {
+	REGULATOR_SUPPLY("dovdd", "i2c-OVTI13B1:01"),
+};
+
+static const struct regulator_init_data intel_nvl_tps68470_core_reg_init_data = {
+		.constraints = {
+		.min_uV = 1200000,
+		.max_uV = 1200000,
+		.apply_uV = true,
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
+	},
+	.num_consumer_supplies = ARRAY_SIZE(ovti13b1_core_consumer_supplies),
+	.consumer_supplies = ovti13b1_core_consumer_supplies,
+};
+
+static const struct regulator_init_data intel_nvl_tps68470_ana_reg_init_data = {
+	.constraints = {
+		.min_uV = 2815200,
+		.max_uV = 2815200,
+		.apply_uV = true,
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
+	},
+	.num_consumer_supplies = ARRAY_SIZE(ovti13b1_ana_consumer_supplies),
+	.consumer_supplies = ovti13b1_ana_consumer_supplies,
+};
+static const struct regulator_init_data intel_nvl_tps68470_vcm_reg_init_data = {
+	.constraints = {
+		.min_uV = 2815200,
+		.max_uV = 2815200,
+		.apply_uV = true,
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
+	},
+	.num_consumer_supplies = ARRAY_SIZE(ovti13b1_vcm_consumer_supplies),
+	.consumer_supplies = ovti13b1_vcm_consumer_supplies,
+};
+
+/* Ensure the always-on VIO regulator has the same voltage as VSIO */
+static const struct regulator_init_data intel_nvl_tps68470_vio_reg_init_data = {
+	.constraints = {
+		.min_uV = 1800600,
+		.max_uV = 1800600,
+		.apply_uV = true,
+		.always_on = true,
+	},
+};
+static const struct regulator_init_data intel_nvl_tps68470_vsio_reg_init_data = {
+	.constraints = {
+		.min_uV = 1800600,
+		.max_uV = 1800600,
+		.apply_uV = true,
+		.valid_ops_mask = REGULATOR_CHANGE_STATUS,
+	},
+	.num_consumer_supplies = ARRAY_SIZE(ovti13b1_vsio_consumer_supplies),
+	.consumer_supplies = ovti13b1_vsio_consumer_supplies,
+};
+
+static const struct tps68470_regulator_platform_data intel_nvl_tps68470_pdata = {
+	.reg_init_data = {
+		[TPS68470_CORE] = &intel_nvl_tps68470_core_reg_init_data,
+		[TPS68470_ANA]  = &intel_nvl_tps68470_ana_reg_init_data,
+		[TPS68470_VCM]  = &intel_nvl_tps68470_vcm_reg_init_data,
+		[TPS68470_VIO] = &intel_nvl_tps68470_vio_reg_init_data,
+		[TPS68470_VSIO] = &intel_nvl_tps68470_vsio_reg_init_data,
 	},
 };
 
@@ -315,7 +395,7 @@ static struct gpiod_lookup_table dell_7212_int3479_gpios = {
 	}
 };
 
-static struct gpiod_lookup_table msi_p14_ai_evo_ovti5675_gpios = {
+static struct gpiod_lookup_table msi_prestige_ai_evo_ovti5675_gpios = {
 	.dev_id = "i2c-OVTI5675:00",
 	.table = {
 		GPIO_LOOKUP("tps68470-gpio", 9, "reset", GPIO_ACTIVE_LOW),
@@ -323,13 +403,21 @@ static struct gpiod_lookup_table msi_p14_ai_evo_ovti5675_gpios = {
 	}
 };
 
-static const struct property_entry msi_p14_ai_evo_gpio_props[] = {
+static struct gpiod_lookup_table intel_nvl_tps68470_gpios = {
+	.dev_id = "i2c-OVTI13B1:01",
+	.table = {
+		GPIO_LOOKUP("tps68470-gpio", 9, "reset", GPIO_ACTIVE_LOW),
+		{ }
+	}
+};
+
+static const struct property_entry int3472_tps68470_daisy_chain_gpio_props[] = {
 	PROPERTY_ENTRY_BOOL("daisy-chain-enable"),
 	{ }
 };
 
-static const struct software_node msi_p14_ai_evo_tps68470_gpio_swnode = {
-	.properties = msi_p14_ai_evo_gpio_props,
+static const struct software_node int3472_tps68470_daisy_chain_gpio_swnode = {
+	.properties = int3472_tps68470_daisy_chain_gpio_props,
 };
 
 static const struct int3472_tps68470_board_data surface_go_tps68470_board_data = {
@@ -361,13 +449,23 @@ static const struct int3472_tps68470_board_data dell_7212_tps68470_board_data =
 	},
 };
 
-static const struct int3472_tps68470_board_data msi_p14_ai_evo_tps68470_board_data = {
+static const struct int3472_tps68470_board_data msi_prestige_ai_evo_tps68470_board_data = {
 	.dev_name = "i2c-INT3472:06",
-	.tps68470_regulator_pdata = &msi_p14_ai_evo_tps68470_pdata,
-	.tps68470_gpio_swnode = &msi_p14_ai_evo_tps68470_gpio_swnode,
+	.tps68470_regulator_pdata = &msi_prestige_ai_evo_tps68470_pdata,
+	.tps68470_gpio_swnode = &int3472_tps68470_daisy_chain_gpio_swnode,
 	.n_gpiod_lookups = 1,
 	.tps68470_gpio_lookup_tables = {
-		&msi_p14_ai_evo_ovti5675_gpios,
+		&msi_prestige_ai_evo_ovti5675_gpios,
+	},
+};
+
+static const struct int3472_tps68470_board_data intel_nvl_tps68470_board_data = {
+	.dev_name = "i2c-INT3472:04",
+	.tps68470_regulator_pdata = &intel_nvl_tps68470_pdata,
+	.tps68470_gpio_swnode = &int3472_tps68470_daisy_chain_gpio_swnode,
+	.n_gpiod_lookups = 1,
+	.tps68470_gpio_lookup_tables = {
+		&intel_nvl_tps68470_gpios,
 	},
 };
 
@@ -400,12 +498,36 @@ static const struct dmi_system_id int3472_tps68470_board_data_table[] = {
 		},
 		.driver_data = (void *)&dell_7212_tps68470_board_data,
 	},
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Micro-Star International Co., Ltd."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Prestige 13 AI+ Evo A2VMG"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "MS-13Q3"),
+		},
+		.driver_data = (void *)&msi_prestige_ai_evo_tps68470_board_data,
+	},
 	{
 		.matches = {
 			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Micro-Star International Co., Ltd."),
 			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Prestige 14 AI+ Evo C2VMG"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "MS-14N3"),
+		},
+		.driver_data = (void *)&msi_prestige_ai_evo_tps68470_board_data,
+	},
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Micro-Star International Co., Ltd."),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Prestige 16 AI+ Evo B2VMG"),
+			DMI_EXACT_MATCH(DMI_BOARD_NAME, "MS-15A3"),
+		},
+		.driver_data = (void *)&msi_prestige_ai_evo_tps68470_board_data,
+	},
+	{
+		.matches = {
+			DMI_EXACT_MATCH(DMI_SYS_VENDOR, "Intel Corporation"),
+			DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "Nova Lake Client Platform"),
 		},
-		.driver_data = (void *)&msi_p14_ai_evo_tps68470_board_data,
+		.driver_data = (void *)&intel_nvl_tps68470_board_data,
 	},
 	{ }
 };
diff --git a/drivers/platform/x86/intel/pmc/Makefile b/drivers/platform/x86/intel/pmc/Makefile
index bb960c8721d77..23853e867c912 100644
--- a/drivers/platform/x86/intel/pmc/Makefile
+++ b/drivers/platform/x86/intel/pmc/Makefile
@@ -4,7 +4,8 @@
 #
 
 intel_pmc_core-y			:= core.o spt.o cnp.o icl.o \
-					   tgl.o adl.o mtl.o arl.o lnl.o ptl.o wcl.o
+					   tgl.o adl.o mtl.o arl.o \
+					   lnl.o ptl.o wcl.o nvl.o
 obj-$(CONFIG_INTEL_PMC_CORE)		+= intel_pmc_core.o
 intel_pmc_core_pltdrv-y			:= pltdrv.o
 obj-$(CONFIG_INTEL_PMC_CORE)		+= intel_pmc_core_pltdrv.o
diff --git a/drivers/platform/x86/intel/pmc/arl.c b/drivers/platform/x86/intel/pmc/arl.c
index eb23bc68340ab..11609d593383b 100644
--- a/drivers/platform/x86/intel/pmc/arl.c
+++ b/drivers/platform/x86/intel/pmc/arl.c
@@ -672,6 +672,9 @@ static struct pmc_info arl_pmc_info_list[] = {
 	{}
 };
 
+static const u8 arl_pmc_list[] = {PMC_IDX_MAIN, PMC_IDX_IOE, PMC_IDX_PCH};
+static const u8 arl_h_pmc_list[] = {PMC_IDX_MAIN, PMC_IDX_IOE};
+
 #define ARL_NPU_PCI_DEV			0xad1d
 #define ARL_GNA_PCI_DEV			0xae4c
 #define ARL_H_NPU_PCI_DEV		0x7d1d
@@ -720,8 +723,9 @@ static int arl_h_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_
 
 static u32 ARL_PMT_DMU_GUIDS[] = {ARL_PMT_DMU_GUID, 0x0};
 struct pmc_dev_info arl_pmc_dev = {
-	.pci_func = 0,
 	.dmu_guids = ARL_PMT_DMU_GUIDS,
+	.num_pmcs = ARRAY_SIZE(arl_pmc_list),
+	.pmc_list = arl_pmc_list,
 	.regmap_list = arl_pmc_info_list,
 	.map = &arl_socs_reg_map,
 	.sub_req_show = &pmc_core_substate_req_regs_fops,
@@ -729,12 +733,15 @@ struct pmc_dev_info arl_pmc_dev = {
 	.resume = arl_resume,
 	.init = arl_core_init,
 	.sub_req = pmc_core_pmt_get_lpm_req,
+	.ssram_hidden = true,
+	.die_c6_offset = MTL_PMT_DMU_DIE_C6_OFFSET,
 };
 
 static u32 ARL_H_PMT_DMU_GUIDS[] = {ARL_PMT_DMU_GUID, ARL_H_PMT_DMU_GUID, 0x0};
 struct pmc_dev_info arl_h_pmc_dev = {
-	.pci_func = 2,
 	.dmu_guids = ARL_H_PMT_DMU_GUIDS,
+	.num_pmcs = ARRAY_SIZE(arl_h_pmc_list),
+	.pmc_list = arl_h_pmc_list,
 	.regmap_list = arl_pmc_info_list,
 	.map = &mtl_socm_reg_map,
 	.sub_req_show = &pmc_core_substate_req_regs_fops,
@@ -742,4 +749,6 @@ struct pmc_dev_info arl_h_pmc_dev = {
 	.resume = arl_h_resume,
 	.init = arl_h_core_init,
 	.sub_req = pmc_core_pmt_get_lpm_req,
+	.ssram_hidden = true,
+	.die_c6_offset = MTL_PMT_DMU_DIE_C6_OFFSET,
 };
diff --git a/drivers/platform/x86/intel/pmc/core.c b/drivers/platform/x86/intel/pmc/core.c
index d91e1ab842d65..3adae8fe8b306 100644
--- a/drivers/platform/x86/intel/pmc/core.c
+++ b/drivers/platform/x86/intel/pmc/core.c
@@ -1071,6 +1071,44 @@ static int pmc_core_die_c6_us_show(struct seq_file *s, void *unused)
 }
 DEFINE_SHOW_ATTRIBUTE(pmc_core_die_c6_us);
 
+static int pmc_core_pkgc_counters_show(struct seq_file *s,
+				       struct telem_endpoint *ep,
+				       u32 offset, const char **counters)
+{
+	unsigned int i;
+	u32 counter;
+	int ret;
+
+	for (i = 0; counters[i]; i++) {
+		ret = pmt_telem_read32(ep, offset + i, &counter, 1);
+		if (ret)
+			return ret;
+		seq_printf(s, "%-30s %-30u\n", counters[i], counter);
+	}
+
+	return 0;
+}
+
+static int pmc_core_pkgc_ltr_blocker_show(struct seq_file *s, void *unused)
+{
+	struct pmc_dev *pmcdev = s->private;
+
+	return pmc_core_pkgc_counters_show(s, pmcdev->pc_ep,
+					   pmcdev->pkgc_ltr_blocker_offset,
+					   pmcdev->pkgc_ltr_blocker_counters);
+}
+DEFINE_SHOW_ATTRIBUTE(pmc_core_pkgc_ltr_blocker);
+
+static int pmc_core_pkgc_blocker_residency_show(struct seq_file *s, void *unused)
+{
+	struct pmc_dev *pmcdev = s->private;
+
+	return pmc_core_pkgc_counters_show(s, pmcdev->pc_ep,
+					   pmcdev->pkgc_blocker_offset,
+					   pmcdev->pkgc_blocker_counters);
+}
+DEFINE_SHOW_ATTRIBUTE(pmc_core_pkgc_blocker_residency);
+
 static int pmc_core_lpm_latch_mode_show(struct seq_file *s, void *unused)
 {
 	struct pmc_dev *pmcdev = s->private;
@@ -1322,28 +1360,45 @@ static struct telem_endpoint *pmc_core_register_endpoint(struct pci_dev *pcidev,
 	return ERR_PTR(-ENODEV);
 }
 
-void pmc_core_punit_pmt_init(struct pmc_dev *pmcdev, u32 *guids)
+void pmc_core_punit_pmt_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info)
 {
 	struct telem_endpoint *ep;
-	struct pci_dev *pcidev;
 
-	pcidev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(10, 0));
+	struct pci_dev *pcidev __free(pci_dev_put) = pci_get_domain_bus_and_slot(0, 0,
+										 PCI_DEVFN(10, 0));
 	if (!pcidev) {
 		dev_err(&pmcdev->pdev->dev, "PUNIT PMT device not found.");
 		return;
 	}
 
-	ep = pmc_core_register_endpoint(pcidev, guids);
-	pci_dev_put(pcidev);
-	if (IS_ERR(ep)) {
-		dev_err(&pmcdev->pdev->dev,
-			"pmc_core: couldn't get DMU telem endpoint %ld",
-			PTR_ERR(ep));
-		return;
+	if (pmc_dev_info->dmu_guids) {
+		ep = pmc_core_register_endpoint(pcidev, pmc_dev_info->dmu_guids);
+		if (IS_ERR(ep)) {
+			dev_err(&pmcdev->pdev->dev,
+				"pmc_core: couldn't get DMU telem endpoint %ld",
+				PTR_ERR(ep));
+			return;
+		}
+
+		pmcdev->punit_ep = ep;
+		pmcdev->die_c6_offset = pmc_dev_info->die_c6_offset;
 	}
 
-	pmcdev->punit_ep = ep;
-	pmcdev->die_c6_offset = MTL_PMT_DMU_DIE_C6_OFFSET;
+	if (pmc_dev_info->pc_guid) {
+		ep = pmt_telem_find_and_register_endpoint(&pcidev->dev, pmc_dev_info->pc_guid, 0);
+		if (IS_ERR(ep)) {
+			dev_err(&pmcdev->pdev->dev,
+				"pmc_core: couldn't get Package C-state telem endpoint %ld",
+				PTR_ERR(ep));
+			return;
+		}
+
+		pmcdev->pc_ep = ep;
+		pmcdev->pkgc_ltr_blocker_counters = pmc_dev_info->pkgc_ltr_blocker_counters;
+		pmcdev->pkgc_ltr_blocker_offset = pmc_dev_info->pkgc_ltr_blocker_offset;
+		pmcdev->pkgc_blocker_counters = pmc_dev_info->pkgc_blocker_counters;
+		pmcdev->pkgc_blocker_offset = pmc_dev_info->pkgc_blocker_offset;
+	}
 }
 
 void pmc_core_set_device_d3(unsigned int device)
@@ -1467,6 +1522,16 @@ static void pmc_core_dbgfs_register(struct pmc_dev *pmcdev, struct pmc_dev_info
 				    pmcdev->dbgfs_dir, pmcdev,
 				    &pmc_core_die_c6_us_fops);
 	}
+
+	if (pmcdev->pc_ep) {
+		debugfs_create_file("pkgc_ltr_blocker_show", 0444,
+				    pmcdev->dbgfs_dir, pmcdev,
+				    &pmc_core_pkgc_ltr_blocker_fops);
+		debugfs_create_file("pkgc_blocker_residency_show", 0444,
+				    pmcdev->dbgfs_dir, pmcdev,
+				    &pmc_core_pkgc_blocker_residency_fops);
+	}
+
 }
 
 /*
@@ -1581,17 +1646,13 @@ int pmc_core_pmt_get_blk_sub_req(struct pmc_dev *pmcdev, struct pmc *pmc,
 
 static int pmc_core_get_telem_info(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info)
 {
-	struct pci_dev *pcidev __free(pci_dev_put) = NULL;
 	struct telem_endpoint *ep;
 	unsigned int pmc_idx;
 	int ret;
 
-	pcidev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(20, pmc_dev_info->pci_func));
-	if (!pcidev)
-		return -ENODEV;
-
 	for (pmc_idx = 0; pmc_idx < ARRAY_SIZE(pmcdev->pmcs); ++pmc_idx) {
 		struct pmc *pmc;
+		u16 devid;
 
 		pmc = pmcdev->pmcs[pmc_idx];
 		if (!pmc)
@@ -1600,6 +1661,16 @@ static int pmc_core_get_telem_info(struct pmc_dev *pmcdev, struct pmc_dev_info *
 		if (!pmc->map->lpm_req_guid)
 			return -ENXIO;
 
+		if (pmc_dev_info->ssram_hidden)
+			devid = pmcdev->pmcs[PMC_IDX_MAIN]->devid;
+		else
+			devid = pmc->devid;
+
+		struct pci_dev *pcidev __free(pci_dev_put) =
+			pci_get_device(PCI_VENDOR_ID_INTEL, devid, NULL);
+		if (!pcidev)
+			return -ENODEV;
+
 		ep = pmt_telem_find_and_register_endpoint(&pcidev->dev, pmc->map->lpm_req_guid, 0);
 		if (IS_ERR(ep)) {
 			dev_dbg(&pmcdev->pdev->dev, "couldn't get telem endpoint %pe", ep);
@@ -1650,6 +1721,7 @@ static int pmc_core_pmc_add(struct pmc_dev *pmcdev, unsigned int pmc_idx)
 
 	pmc->map = map;
 	pmc->base_addr = pmc_ssram_telemetry.base_addr;
+	pmc->devid = pmc_ssram_telemetry.devid;
 	pmc->regbase = ioremap(pmc->base_addr, pmc->map->regmap_length);
 
 	if (!pmc->regbase) {
@@ -1662,16 +1734,17 @@ static int pmc_core_pmc_add(struct pmc_dev *pmcdev, unsigned int pmc_idx)
 	return 0;
 }
 
-static int pmc_core_ssram_get_reg_base(struct pmc_dev *pmcdev)
+static int pmc_core_ssram_get_reg_base(struct pmc_dev *pmcdev, u8 num_pmcs, const u8 *pmc_list)
 {
+	unsigned int i;
 	int ret;
 
-	ret = pmc_core_pmc_add(pmcdev, PMC_IDX_MAIN);
-	if (ret)
-		return ret;
-
-	pmc_core_pmc_add(pmcdev, PMC_IDX_IOE);
-	pmc_core_pmc_add(pmcdev, PMC_IDX_PCH);
+	for (i = 0; i < num_pmcs; ++i) {
+		/* Non-MAIN PMCs are allowed to fail */
+		ret = pmc_core_pmc_add(pmcdev, pmc_list[i]);
+		if (ret && (pmc_list[i] == PMC_IDX_MAIN))
+			return ret;
+	}
 
 	return 0;
 }
@@ -1693,7 +1766,9 @@ int generic_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info)
 	ssram = pmc_dev_info->regmap_list != NULL;
 	if (ssram) {
 		pmcdev->regmap_list = pmc_dev_info->regmap_list;
-		ret = pmc_core_ssram_get_reg_base(pmcdev);
+		ret = pmc_core_ssram_get_reg_base(pmcdev,
+						  pmc_dev_info->num_pmcs,
+						  pmc_dev_info->pmc_list);
 		/*
 		 * EAGAIN error code indicates Intel PMC SSRAM Telemetry driver
 		 * has not finished probe and PMC info is not available yet. Try
@@ -1717,8 +1792,8 @@ int generic_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info)
 	}
 
 	pmc_core_get_low_power_modes(pmcdev);
-	if (pmc_dev_info->dmu_guids)
-		pmc_core_punit_pmt_init(pmcdev, pmc_dev_info->dmu_guids);
+	if (pmc_dev_info->dmu_guids || pmc_dev_info->pc_guid)
+		pmc_core_punit_pmt_init(pmcdev, pmc_dev_info);
 
 	if (ssram) {
 		ret = pmc_core_get_telem_info(pmcdev, pmc_dev_info);
@@ -1739,6 +1814,9 @@ int generic_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info)
 	if (pmcdev->punit_ep)
 		pmt_telem_unregister_endpoint(pmcdev->punit_ep);
 
+	if (pmcdev->pc_ep)
+		pmt_telem_unregister_endpoint(pmcdev->pc_ep);
+
 	return ret;
 }
 
@@ -1771,6 +1849,8 @@ static const struct x86_cpu_id intel_pmc_core_ids[] = {
 	X86_MATCH_VFM(INTEL_LUNARLAKE_M,	&lnl_pmc_dev),
 	X86_MATCH_VFM(INTEL_PANTHERLAKE_L,	&ptl_pmc_dev),
 	X86_MATCH_VFM(INTEL_WILDCATLAKE_L,	&wcl_pmc_dev),
+	X86_MATCH_VFM(INTEL_NOVALAKE,		&nvl_s_pmc_dev),
+	X86_MATCH_VFM(INTEL_NOVALAKE_L,		&nvl_h_pmc_dev),
 	{}
 };
 
@@ -1835,6 +1915,9 @@ static void pmc_core_clean_structure(struct platform_device *pdev)
 	if (pmcdev->punit_ep)
 		pmt_telem_unregister_endpoint(pmcdev->punit_ep);
 
+	if (pmcdev->pc_ep)
+		pmt_telem_unregister_endpoint(pmcdev->pc_ep);
+
 	platform_set_drvdata(pdev, NULL);
 }
 
diff --git a/drivers/platform/x86/intel/pmc/core.h b/drivers/platform/x86/intel/pmc/core.h
index 118c8740ad3aa..55cf567febe4f 100644
--- a/drivers/platform/x86/intel/pmc/core.h
+++ b/drivers/platform/x86/intel/pmc/core.h
@@ -46,7 +46,7 @@ struct telem_endpoint;
 #define SPT_PMC_SLP_S0_RES_COUNTER_STEP		0x68
 #define PMC_BASE_ADDR_MASK			~(SPT_PMC_MMIO_REG_LEN - 1)
 #define MTPMC_MASK				0xffff0000
-#define PPFEAR_MAX_NUM_ENTRIES			12
+#define PPFEAR_MAX_NUM_ENTRIES			13
 #define SPT_PPFEAR_NUM_ENTRIES			5
 #define SPT_PMC_READ_DISABLE_BIT		0x16
 #define SPT_PMC_MSG_FULL_STS_BIT		0x18
@@ -307,6 +307,29 @@ enum ppfear_regs {
 #define WCL_NUM_S0IX_BLOCKER			94
 #define WCL_BLK_REQ_OFFSET			50
 
+/* Nova Lake */
+#define NVL_PCDH_PPFEAR_NUM_ENTRIES		13
+#define NVL_PCDH_PMC_MMIO_REG_LEN		0x363c
+#define NVL_PCDS_PMC_MMIO_REG_LEN		0x3118
+#define NVL_PCHS_PMC_MMIO_REG_LEN		0x30d8
+#define NVL_LPM_PRI_OFFSET			0x17a4
+#define NVL_LPM_EN_OFFSET			0x17a0
+#define NVL_LPM_RESIDENCY_OFFSET		0x17a8
+#define NVL_LPM_LIVE_STATUS_OFFSET		0x1760
+#define NVL_LPM_NUM_MAPS			15
+#define NVL_PCDH_NUM_S0IX_BLOCKER		107
+#define NVL_PCDS_NUM_S0IX_BLOCKER		71
+#define NVL_PCHS_NUM_S0IX_BLOCKER		54
+#define NVL_PCDS_PMC_LTR_RESERVED		0x1bac
+#define NVL_PCDH_BLK_REQ_OFFSET			53
+#define NVL_PCDS_BLK_REQ_OFFSET			18
+#define NVL_PCHS_BLK_REQ_OFFSET			46
+#define NVL_PMT_PC_GUID				0x13000101
+#define NVL_PMT_DMU_GUID			0x1a000101
+#define NVL_LTR_BLK_OFFSET			64
+#define NVL_PKGC_BLK_OFFSET			4
+#define NVL_PMT_DMU_DIE_C6_OFFSET		25
+
 /* SSRAM PMC Device ID */
 /* LNL */
 #define PMC_DEVID_LNL_SOCM	0xa87f
@@ -329,6 +352,11 @@ enum ppfear_regs {
 #define PMC_DEVID_MTL_IOEP	0x7ecf
 #define PMC_DEVID_MTL_IOEM	0x7ebf
 
+/* NVL */
+#define PMC_DEVID_NVL_PCDH	0xd37e
+#define PMC_DEVID_NVL_PCDS	0xd47e
+#define PMC_DEVID_NVL_PCHS	0x6e27
+
 extern const char *pmc_lpm_modes[];
 
 struct pmc_bit_map {
@@ -425,6 +453,7 @@ struct pmc_info {
  * @ltr_ign:		Holds LTR ignore data while suspended
  * @num_lpm_modes:	Count of enabled modes
  * @lpm_en_modes:	Array of enabled modes from lowest to highest priority
+ * @devid:		Device ID of the SSRAM device
  *
  * pmc contains info about one power management controller device.
  */
@@ -436,6 +465,7 @@ struct pmc {
 	u32 ltr_ign;
 	u8 num_lpm_modes;
 	u8 lpm_en_modes[LPM_MAX_NUM_MODES];
+	u16 devid;
 };
 
 /**
@@ -453,6 +483,11 @@ struct pmc {
  * @suspend:		Function to perform platform specific suspend
  * @resume:		Function to perform platform specific resume
  *
+ * @pkgc_ltr_blocker_counters: Array of PKGC LTR blocker counters
+ * @pkgc_ltr_blocker_offset: Offset to PKGC LTR blockers in telemetry region
+ * @pkgc_blocker_counters: Array of PKGC blocker counters
+ * @pkgc_blocker_offset: Offset to PKGC blocker in telemetry region
+ *
  * pmc_dev contains info about power management controller device.
  */
 struct pmc_dev {
@@ -471,8 +506,14 @@ struct pmc_dev {
 	u8 num_of_pkgc;
 
 	u32 die_c6_offset;
+	struct telem_endpoint *pc_ep;
 	struct telem_endpoint *punit_ep;
 	struct pmc_info *regmap_list;
+
+	const char **pkgc_ltr_blocker_counters;
+	u32 pkgc_ltr_blocker_offset;
+	const char **pkgc_blocker_counters;
+	u32 pkgc_blocker_offset;
 };
 
 enum pmc_index {
@@ -484,29 +525,45 @@ enum pmc_index {
 
 /**
  * struct pmc_dev_info - Structure to keep PMC device info
- * @pci_func:		Function number of the primary PMC
  * @dmu_guids:		List of Die Management Unit GUID
+ * @pc_guid:		GUID for telemetry region to read PKGC blocker info
+ * @pkgc_ltr_blocker_offset: Offset to PKGC LTR blockers in telemetry region
+ * @pkgc_blocker_offset:Offset to PKGC blocker in telemetry region
+ * @num_pmcs:		Number of entries in @pmc_list
+ * @pmc_list:		Index list of available PMC
  * @regmap_list:	Pointer to a list of pmc_info structure that could be
  *			available for the platform. When set, this field implies
  *			SSRAM support.
  * @map:		Pointer to a pmc_reg_map struct that contains platform
  *			specific attributes of the primary PMC
  * @sub_req_show:	File operations to show substate requirements
+ * @pkgc_ltr_blocker_counters: Array of PKGC LTR blocker counters
+ * @pkgc_blocker_counters: Array of PKGC blocker counters
  * @suspend:		Function to perform platform specific suspend
  * @resume:		Function to perform platform specific resume
  * @init:		Function to perform platform specific init action
  * @sub_req:		Function to achieve low power mode substate requirements
+ * @ssram_hidden:	Some SSRAM devices are hidden on this platform
+ * @die_c6_offset:	Telemetry offset to read Die C6 residency
  */
 struct pmc_dev_info {
-	u8 pci_func;
 	u32 *dmu_guids;
+	u32 pc_guid;
+	u32 pkgc_ltr_blocker_offset;
+	u32 pkgc_blocker_offset;
+	u8 num_pmcs;
+	const u8 *pmc_list;
 	struct pmc_info *regmap_list;
 	const struct pmc_reg_map *map;
 	const struct file_operations *sub_req_show;
+	const char **pkgc_ltr_blocker_counters;
+	const char **pkgc_blocker_counters;
 	void (*suspend)(struct pmc_dev *pmcdev);
 	int (*resume)(struct pmc_dev *pmcdev);
 	int (*init)(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info);
 	int (*sub_req)(struct pmc_dev *pmcdev, struct pmc *pmc, struct telem_endpoint *ep);
+	bool ssram_hidden;
+	u32 die_c6_offset;
 };
 
 extern const struct pmc_bit_map msr_map[];
@@ -529,13 +586,14 @@ extern const struct pmc_reg_map mtl_ioep_reg_map;
 extern const struct pmc_bit_map ptl_pcdp_clocksource_status_map[];
 extern const struct pmc_bit_map ptl_pcdp_vnn_req_status_3_map[];
 extern const struct pmc_bit_map ptl_pcdp_signal_status_map[];
+extern const struct pmc_bit_map ptl_pcdp_ltr_show_map[];
 
 void pmc_core_get_tgl_lpm_reqs(struct platform_device *pdev);
 int pmc_core_send_ltr_ignore(struct pmc_dev *pmcdev, u32 value, int ignore);
 
 int pmc_core_resume_common(struct pmc_dev *pmcdev);
 int get_primary_reg_base(struct pmc *pmc);
-void pmc_core_punit_pmt_init(struct pmc_dev *pmcdev, u32 *guids);
+void pmc_core_punit_pmt_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info);
 void pmc_core_set_device_d3(unsigned int device);
 
 int generic_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info);
@@ -552,6 +610,8 @@ extern struct pmc_dev_info arl_h_pmc_dev;
 extern struct pmc_dev_info lnl_pmc_dev;
 extern struct pmc_dev_info ptl_pmc_dev;
 extern struct pmc_dev_info wcl_pmc_dev;
+extern struct pmc_dev_info nvl_s_pmc_dev;
+extern struct pmc_dev_info nvl_h_pmc_dev;
 
 void cnl_suspend(struct pmc_dev *pmcdev);
 int cnl_resume(struct pmc_dev *pmcdev);
diff --git a/drivers/platform/x86/intel/pmc/lnl.c b/drivers/platform/x86/intel/pmc/lnl.c
index 1cd81ee54dcf8..5ff4922825d91 100644
--- a/drivers/platform/x86/intel/pmc/lnl.c
+++ b/drivers/platform/x86/intel/pmc/lnl.c
@@ -544,6 +544,8 @@ static struct pmc_info lnl_pmc_info_list[] = {
 	{}
 };
 
+static const u8 lnl_pmc_list[] = {PMC_IDX_MAIN};
+
 #define LNL_NPU_PCI_DEV		0x643e
 #define LNL_IPU_PCI_DEV		0x645d
 
@@ -571,7 +573,8 @@ static int lnl_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_in
 }
 
 struct pmc_dev_info lnl_pmc_dev = {
-	.pci_func = 2,
+	.num_pmcs = ARRAY_SIZE(lnl_pmc_list),
+	.pmc_list = lnl_pmc_list,
 	.regmap_list = lnl_pmc_info_list,
 	.map = &lnl_socm_reg_map,
 	.sub_req_show = &pmc_core_substate_req_regs_fops,
@@ -579,4 +582,5 @@ struct pmc_dev_info lnl_pmc_dev = {
 	.resume = lnl_resume,
 	.init = lnl_core_init,
 	.sub_req = pmc_core_pmt_get_lpm_req,
+	.ssram_hidden = true,
 };
diff --git a/drivers/platform/x86/intel/pmc/mtl.c b/drivers/platform/x86/intel/pmc/mtl.c
index 57508cbf9cd42..9abeabd3dbe27 100644
--- a/drivers/platform/x86/intel/pmc/mtl.c
+++ b/drivers/platform/x86/intel/pmc/mtl.c
@@ -965,6 +965,8 @@ static struct pmc_info mtl_pmc_info_list[] = {
 	{}
 };
 
+static const u8 mtl_pmc_list[] = {PMC_IDX_MAIN, PMC_IDX_IOE};
+
 #define MTL_GNA_PCI_DEV	0x7e4c
 #define MTL_IPU_PCI_DEV	0x7d19
 #define MTL_VPU_PCI_DEV	0x7d1d
@@ -994,8 +996,9 @@ static int mtl_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_in
 
 static u32 MTL_PMT_DMU_GUIDS[] = {MTL_PMT_DMU_GUID, 0x0};
 struct pmc_dev_info mtl_pmc_dev = {
-	.pci_func = 2,
 	.dmu_guids = MTL_PMT_DMU_GUIDS,
+	.num_pmcs = ARRAY_SIZE(mtl_pmc_list),
+	.pmc_list = mtl_pmc_list,
 	.regmap_list = mtl_pmc_info_list,
 	.map = &mtl_socm_reg_map,
 	.sub_req_show = &pmc_core_substate_req_regs_fops,
@@ -1003,4 +1006,6 @@ struct pmc_dev_info mtl_pmc_dev = {
 	.resume = mtl_resume,
 	.init = mtl_core_init,
 	.sub_req = pmc_core_pmt_get_lpm_req,
+	.ssram_hidden = true,
+	.die_c6_offset = MTL_PMT_DMU_DIE_C6_OFFSET,
 };
diff --git a/drivers/platform/x86/intel/pmc/nvl.c b/drivers/platform/x86/intel/pmc/nvl.c
new file mode 100644
index 0000000000000..8dabf2511dd9d
--- /dev/null
+++ b/drivers/platform/x86/intel/pmc/nvl.c
@@ -0,0 +1,1539 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This file contains platform specific structure definitions
+ * and init function used by Nova Lake PCH.
+ *
+ * Copyright (c) 2026, Intel Corporation.
+ */
+
+#include <linux/bits.h>
+#include <linux/pci.h>
+
+#include "core.h"
+
+/* PMC SSRAM PMT Telemetry GUIDS */
+#define PCDH_LPM_REQ_GUID 0x01093101
+#define PCHS_LPM_REQ_GUID 0x01092101
+#define PCDS_LPM_REQ_GUID 0x01091102
+
+/*
+ * Die Mapping to Product.
+ * Product PCDDie PCHDie
+ * NVL-H   PCD-H  None
+ * NVL-S   PCD-S  PCH-S
+ */
+
+static const struct pmc_bit_map nvl_pcdh_pfear_map[] = {
+	{"PMC_PGD0",                 BIT(0)},
+	{"FUSE_OSSE_PGD0",           BIT(1)},
+	{"SPI_PGD0",                 BIT(2)},
+	{"XHCI_PGD0",                BIT(3)},
+	{"SPA_PGD0",                 BIT(4)},
+	{"SPB_PGD0",                 BIT(5)},
+	{"MPFPW2_PGD0",              BIT(6)},
+	{"GBE_PGD0",                 BIT(7)},
+
+	{"SBR16B20_PGD0",            BIT(0)},
+	{"DBG_SBR_PGD0",             BIT(1)},
+	{"SBR16B7_PGD0",             BIT(2)},
+	{"STRC_PGD0",                BIT(3)},
+	{"SBR16B8_PGD0",             BIT(4)},
+	{"D2D_DISP_PGD1",            BIT(5)},
+	{"LPSS_PGD0",                BIT(6)},
+	{"LPC_PGD0",                 BIT(7)},
+
+	{"SMB_PGD0",                 BIT(0)},
+	{"ISH_PGD0",                 BIT(1)},
+	{"SBR16B2_PGD0",             BIT(2)},
+	{"NPK_PGD0",                 BIT(3)},
+	{"D2D_NOC_PGD1",             BIT(4)},
+	{"DBG_SBR16B_PGD0",          BIT(5)},
+	{"FUSE_PGD0",                BIT(6)},
+	{"SBR16B0_PGD0",             BIT(7)},
+
+	{"P2SB0_PGD0",               BIT(0)},
+	{"OTG_PGD0",                 BIT(1)},
+	{"EXI_PGD0",                 BIT(2)},
+	{"CSE_PGD0",                 BIT(3)},
+	{"CSME_KVM_PGD0",            BIT(4)},
+	{"CSME_PMT_PGD0",            BIT(5)},
+	{"CSME_CLINK_PGD0",          BIT(6)},
+	{"SBR16B21_PGD0",            BIT(7)},
+
+	{"CSME_USBR_PGD0",           BIT(0)},
+	{"SBR16B22_PGD0",            BIT(1)},
+	{"CSME_SMT1_PGD0",           BIT(2)},
+	{"MPFPW1_PGD0",              BIT(3)},
+	{"CSME_SMS2_PGD0",           BIT(4)},
+	{"CSME_SMS_PGD0",            BIT(5)},
+	{"CSME_RTC_PGD0",            BIT(6)},
+	{"CSMEPSF_PGD0",             BIT(7)},
+
+	{"D2D_NOC_PGD0",             BIT(0)},
+	{"ESE_PGD0",                 BIT(1)},
+	{"SBR16B6_PGD0",             BIT(2)},
+	{"P2SB1_PGD0",               BIT(3)},
+	{"SBR16B3_PGD0",             BIT(4)},
+	{"OSSE_SMT1_PGD0",           BIT(5)},
+	{"D2D_DISP_PGD0",            BIT(6)},
+	{"SNPS_USB2_A_PGD0",         BIT(7)},
+
+	{"U3FPW1_PGD0",              BIT(0)},
+	{"FIA_X_PGD0",               BIT(1)},
+	{"PSF4_PGD0",                BIT(2)},
+	{"CNVI_PGD0",                BIT(3)},
+	{"UFSX2_PGD0",               BIT(4)},
+	{"ENDBG_PGD0",               BIT(5)},
+	{"DBC_PGD0",                 BIT(6)},
+	{"FIA_PG_PGD0",              BIT(7)},
+
+	{"D2D_IPU_PGD0",             BIT(0)},
+	{"NPK_PGD1",                 BIT(1)},
+	{"FIACPCB_X_PGD0",           BIT(2)},
+	{"SBR8B4_PGD0",              BIT(3)},
+	{"DBG_PSF_PGD0",             BIT(4)},
+	{"PSF6_PGD0",                BIT(5)},
+	{"UFSPW1_PGD0",              BIT(6)},
+	{"FIA_U_PGD0",               BIT(7)},
+
+	{"PSF8_PGD0",                BIT(0)},
+	{"SBR16B9_PGD0",             BIT(1)},
+	{"PSF0_PGD0",                BIT(2)},
+	{"FIACPCB_U_PGD0",           BIT(3)},
+	{"TAM_PGD0",                 BIT(4)},
+	{"D2D_NOC_PGD2",             BIT(5)},
+	{"SBR8B2_PGD0",              BIT(6)},
+	{"THC0_PGD0",                BIT(7)},
+
+	{"THC1_PGD0",                BIT(0)},
+	{"PMC_PGD1",                 BIT(1)},
+	{"DISP_PGA1_PGD0",           BIT(2)},
+	{"TCSS_PGD0",                BIT(3)},
+	{"DISP_PGA_PGD0",            BIT(4)},
+	{"SBR16B1_PGD0",             BIT(5)},
+	{"SBRG_PGD0",                BIT(6)},
+	{"PSF5_PGD0",                BIT(7)},
+
+	{"SBR8B3_PGD0",              BIT(0)},
+	{"ACE_PGD0",                 BIT(1)},
+	{"ACE_PGD1",                 BIT(2)},
+	{"ACE_PGD2",                 BIT(3)},
+	{"ACE_PGD3",                 BIT(4)},
+	{"ACE_PGD4",                 BIT(5)},
+	{"ACE_PGD5",                 BIT(6)},
+	{"ACE_PGD6",                 BIT(7)},
+
+	{"ACE_PGD7",                 BIT(0)},
+	{"ACE_PGD8",                 BIT(1)},
+	{"ACE_PGD9",                 BIT(2)},
+	{"ACE_PGD10",                BIT(3)},
+	{"FIACPCB_PG_PGD0",          BIT(4)},
+	{"SNPS_USB2_B_PGD0",         BIT(5)},
+	{"OSSE_PGD0",                BIT(6)},
+	{"SBR8B0_PGD0",              BIT(7)},
+
+	{"SBR16B4_PGD0",             BIT(0)},
+	{"CSME_PTIO_PGD0",           BIT(1)},
+	{}
+};
+
+static const struct pmc_bit_map *ext_nvl_pcdh_pfear_map[] = {
+	nvl_pcdh_pfear_map,
+	NULL
+};
+
+static const struct pmc_bit_map nvl_pcdh_clocksource_status_map[] = {
+	{"AON2_OFF_STS",                 BIT(0),	1},
+	{"AON3_OFF_STS",                 BIT(1),	0},
+	{"AON4_OFF_STS",                 BIT(2),	1},
+	{"AON5_OFF_STS",                 BIT(3),	1},
+	{"AON1_OFF_STS",                 BIT(4),	0},
+	{"XTAL_LVM_OFF_STS",             BIT(5),	0},
+	{"MPFPW1_0_PLL_OFF_STS",         BIT(6),	1},
+	{"D2D_PLL_OFF_STS",              BIT(7),	1},
+	{"USB3_PLL_OFF_STS",             BIT(8),	1},
+	{"AON3_SPL_OFF_STS",             BIT(9),	1},
+	{"MPFPW2_0_PLL_OFF_STS",         BIT(12),	1},
+	{"XTAL_AGGR_OFF_STS",            BIT(17),	1},
+	{"USB2_PLL_OFF_STS",             BIT(18),	0},
+	{"DDI2_PLL_OFF_STS",             BIT(19),	1},
+	{"SE_TCSS_PLL_OFF_STS",          BIT(20),	1},
+	{"DDI_PLL_OFF_STS",              BIT(21),	1},
+	{"FILTER_PLL_OFF_STS",           BIT(22),	1},
+	{"ACE_PLL_OFF_STS",              BIT(24),	0},
+	{"FABRIC_PLL_OFF_STS",           BIT(25),	1},
+	{"SOC_PLL_OFF_STS",              BIT(26),	1},
+	{"REF_PLL_OFF_STS",              BIT(28),	1},
+	{"IMG_PLL_OFF_STS",              BIT(29),	1},
+	{"GENLOCK_FILTER_PLL_OFF_STS",   BIT(30),	1},
+	{"RTC_PLL_OFF_STS",              BIT(31),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_power_gating_status_0_map[] = {
+	{"PMC_PGD0_PG_STS",              BIT(0),	0},
+	{"FUSE_OSSE_PGD0_PG_STS",	 BIT(1),	0},
+	{"ESPISPI_PGD0_PG_STS",          BIT(2),	0},
+	{"XHCI_PGD0_PG_STS",             BIT(3),	1},
+	{"SPA_PGD0_PG_STS",              BIT(4),	1},
+	{"SPB_PGD0_PG_STS",              BIT(5),	1},
+	{"MPFPW2_PGD0_PG_STS",           BIT(6),	0},
+	{"GBE_PGD0_PG_STS",              BIT(7),	1},
+	{"SBR16B20_PGD0_PG_STS",         BIT(8),	0},
+	{"DBG_PGD0_PG_STS",              BIT(9),	0},
+	{"SBR16B7_PGD0_PG_STS",          BIT(10),	0},
+	{"STRC_PGD0_PG_STS",             BIT(11),	0},
+	{"SBR16B8_PGD0_PG_STS",          BIT(12),	0},
+	{"D2D_DISP_PGD1_PG_STS",         BIT(13),	1},
+	{"LPSS_PGD0_PG_STS",             BIT(14),	1},
+	{"LPC_PGD0_PG_STS",              BIT(15),	0},
+	{"SMB_PGD0_PG_STS",              BIT(16),	0},
+	{"ISH_PGD0_PG_STS",              BIT(17),	0},
+	{"SBR16B2_PGD0_PG_STS",          BIT(18),	0},
+	{"NPK_PGD0_PG_STS",              BIT(19),	0},
+	{"D2D_NOC_PGD1_PG_STS",          BIT(20),	1},
+	{"DBG_SBR16B_PGD0_PG_STS",       BIT(21),	0},
+	{"FUSE_PGD0_PG_STS",             BIT(22),	0},
+	{"SBR16B0_PGD0_PG_STS",          BIT(23),	0},
+	{"P2SB0_PGD0_PG_STS",            BIT(24),	1},
+	{"XDCI_PGD0_PG_STS",             BIT(25),	1},
+	{"EXI_PGD0_PG_STS",              BIT(26),	0},
+	{"CSE_PGD0_PG_STS",              BIT(27),	1},
+	{"KVMCC_PGD0_PG_STS",            BIT(28),	1},
+	{"PMT_PGD0_PG_STS",              BIT(29),	1},
+	{"CLINK_PGD0_PG_STS",            BIT(30),	1},
+	{"SBR16B21_PGD0_PG_STS",         BIT(31),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_power_gating_status_1_map[] = {
+	{"USBR0_PGD0_PG_STS",            BIT(0),	1},
+	{"SBR16B22_PGD0_PG_STS",         BIT(1),	0},
+	{"SMT1_PGD0_PG_STS",             BIT(2),	1},
+	{"MPFPW1_PGD0_PG_STS",           BIT(3),	0},
+	{"SMS2_PGD0_PG_STS",             BIT(4),	1},
+	{"SMS1_PGD0_PG_STS",             BIT(5),	1},
+	{"CSMERTC_PGD0_PG_STS",          BIT(6),	0},
+	{"CSMEPSF_PGD0_PG_STS",          BIT(7),	0},
+	{"D2D_NOC_PGD0_PG_STS",          BIT(8),	0},
+	{"ESE_PGD0_PG_STS",              BIT(9),	1},
+	{"SBR16B6_PGD0_PG_STS",          BIT(10),	0},
+	{"P2SB1_PGD0_PG_STS",            BIT(11),	1},
+	{"SBR16B3_PGD0_PG_STS",          BIT(12),	0},
+	{"OSSE_SMT1_PGD0_PG_STS",        BIT(13),	1},
+	{"D2D_DISP_PGD0_PG_STS",         BIT(14),	1},
+	{"SNPA_USB2_A_PGD0_PG_STS",      BIT(15),	0},
+	{"U3FPW1_PGD0_PG_STS",           BIT(16),	0},
+	{"FIA_X_PGD0_PG_STS",            BIT(17),	0},
+	{"PSF4_PGD0_PG_STS",             BIT(18),	0},
+	{"CNVI_PGD0_PG_STS",             BIT(19),	0},
+	{"UFSX2_PGD0_PG_STS",            BIT(20),	1},
+	{"ENDBG_PGD0_PG_STS",            BIT(21),	0},
+	{"DBC_PGD0_PG_STS",              BIT(22),	0},
+	{"FIA_PG_PGD0_PG_STS",           BIT(23),	0},
+	{"D2D_IPU_PGD0_PG_STS",          BIT(24),	1},
+	{"NPK_PGD1_PG_STS",              BIT(25),	0},
+	{"FIACPCB_X_PGD0_PG_STS",        BIT(26),	0},
+	{"SBR8B4_PGD0_PG_STS",           BIT(27),	0},
+	{"DBG_PSF_PGD0_PG_STS",          BIT(28),	0},
+	{"PSF6_PGD0_PG_STS",             BIT(29),	0},
+	{"UFSPW1_PGD0_PG_STS",           BIT(30),	0},
+	{"FIA_U_PGD0_PG_STS",            BIT(31),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_power_gating_status_2_map[] = {
+	{"PSF8_PGD0_PG_STS",             BIT(0),	0},
+	{"SBR16B9_PGD0_PG_STS",          BIT(1),	0},
+	{"PSF0_PGD0_PG_STS",             BIT(2),	0},
+	{"FIACPCB_U_PGD0_PG_STS",        BIT(3),	0},
+	{"TAM_PGD0_PG_STS",              BIT(4),	1},
+	{"D2D_NOC_PGD2_PG_STS",          BIT(5),	1},
+	{"SBR8B2_PGD0_PG_STS",           BIT(6),	0},
+	{"THC0_PGD0_PG_STS",             BIT(7),	1},
+	{"THC1_PGD0_PG_STS",             BIT(8),	1},
+	{"PMC_PGD1_PG_STS",              BIT(9),	0},
+	{"DISP_PGA1_PGD0_PG_STS",        BIT(10),	0},
+	{"TCSS_PGD0_PG_STS",             BIT(11),	0},
+	{"DISP_PGA_PGD0_PG_STS",         BIT(12),	0},
+	{"SBR16B1_PGD0_PG_STS",          BIT(13),	0},
+	{"SBRG_PGD0_PG_STS",             BIT(14),	0},
+	{"PSF5_PGD0_PG_STS",             BIT(15),	0},
+	{"SBR8B3_PGD0_PG_STS",           BIT(16),	0},
+	{"ACE_PGD0_PG_STS",              BIT(17),	0},
+	{"ACE_PGD1_PG_STS",              BIT(18),	0},
+	{"ACE_PGD2_PG_STS",              BIT(19),	0},
+	{"ACE_PGD3_PG_STS",              BIT(20),	0},
+	{"ACE_PGD4_PG_STS",              BIT(21),	0},
+	{"ACE_PGD5_PG_STS",              BIT(22),	0},
+	{"ACE_PGD6_PG_STS",              BIT(23),	0},
+	{"ACE_PGD7_PG_STS",              BIT(24),	0},
+	{"ACE_PGD8_PG_STS",              BIT(25),	0},
+	{"ACE_PGD9_PG_STS",              BIT(26),	0},
+	{"ACE_PGD10_PG_STS",             BIT(27),	0},
+	{"FIACPCB_PG_PGD0_PG_STS",       BIT(28),	0},
+	{"SNPS_USB2_B_PGD0_PG_STS",      BIT(29),	0},
+	{"OSSE_PGD0_PG_STS",             BIT(30),	1},
+	{"SBR8B0_PGD0_PG_STS",           BIT(31),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_power_gating_status_3_map[] = {
+	{"SBR16B4_PGD0_PG_STS",          BIT(0),	0},
+	{"PTIO_PGD0_PG_STS",             BIT(1),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_d3_status_0_map[] = {
+	{"LPSS_D3_STS",                  BIT(3),	1},
+	{"XDCI_D3_STS",                  BIT(4),	1},
+	{"XHCI_D3_STS",                  BIT(5),	1},
+	{"OSSE_D3_STS",                  BIT(6),	0},
+	{"SPA_D3_STS",                   BIT(12),	0},
+	{"SPB_D3_STS",                   BIT(13),	0},
+	{"ESPISPI_D3_STS",               BIT(18),	0},
+	{"PSTH_D3_STS",                  BIT(21),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_d3_status_1_map[] = {
+	{"OSSE_SMT1_D3_STS",             BIT(0),	0},
+	{"GBE_D3_STS",                   BIT(19),	0},
+	{"ITSS_D3_STS",                  BIT(23),	0},
+	{"CNVI_D3_STS",                  BIT(27),	0},
+	{"UFSX2_D3_STS",                 BIT(28),	0},
+	{"ESE_D3_STS",                   BIT(29),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_d3_status_2_map[] = {
+	{"CSMERTC_D3_STS",               BIT(1),	0},
+	{"CSE_D3_STS",                   BIT(4),	0},
+	{"KVMCC_D3_STS",                 BIT(5),	0},
+	{"USBR0_D3_STS",                 BIT(6),	0},
+	{"ISH_D3_STS",                   BIT(7),	0},
+	{"SMT1_D3_STS",                  BIT(8),	0},
+	{"SMT2_D3_STS",                  BIT(9),	0},
+	{"SMT3_D3_STS",                  BIT(10),	0},
+	{"OSSE_SMT2_D3_STS",             BIT(11),	0},
+	{"CLINK_D3_STS",                 BIT(14),	0},
+	{"PTIO_D3_STS",                  BIT(16),	0},
+	{"PMT_D3_STS",                   BIT(17),	0},
+	{"SMS1_D3_STS",                  BIT(18),	0},
+	{"SMS2_D3_STS",                  BIT(19),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_d3_status_3_map[] = {
+	{"THC0_D3_STS",                  BIT(14),	1},
+	{"THC1_D3_STS",                  BIT(15),	1},
+	{"OSSE_SMT3_D3_STS",             BIT(16),	0},
+	{"ACE_D3_STS",                   BIT(23),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_vnn_req_status_0_map[] = {
+	{"LPSS_VNN_REQ_STS",             BIT(3),	1},
+	{"OSSE_VNN_REQ_STS",             BIT(6),	1},
+	{"ESPISPI_VNN_REQ_STS",          BIT(18),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_vnn_req_status_1_map[] = {
+	{"OSSE_SMT1_VNN_REQ_STS",        BIT(0),	1},
+	{"NPK_VNN_REQ_STS",              BIT(4),	1},
+	{"DFXAGG_VNN_REQ_STS",           BIT(8),	0},
+	{"EXI_VNN_REQ_STS",              BIT(9),	1},
+	{"P2D_VNN_REQ_STS",              BIT(18),	1},
+	{"GBE_VNN_REQ_STS",              BIT(19),	1},
+	{"SMB_VNN_REQ_STS",              BIT(25),	1},
+	{"LPC_VNN_REQ_STS",              BIT(26),	0},
+	{"ESE_VNN_REQ_STS",              BIT(29),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_vnn_req_status_2_map[] = {
+	{"CSMERTC_VNN_REQ_STS",          BIT(1),	1},
+	{"CSE_VNN_REQ_STS",              BIT(4),	1},
+	{"ISH_VNN_REQ_STS",              BIT(7),	1},
+	{"SMT1_VNN_REQ_STS",             BIT(8),	1},
+	{"CLINK_VNN_REQ_STS",            BIT(14),	1},
+	{"SMS1_VNN_REQ_STS",             BIT(18),	1},
+	{"SMS2_VNN_REQ_STS",             BIT(19),	1},
+	{"GPIOCOM4_VNN_REQ_STS",         BIT(20),	1},
+	{"GPIOCOM3_VNN_REQ_STS",         BIT(21),	1},
+	{"DISP_SHIM_VNN_REQ_STS",        BIT(22),	1},
+	{"GPIOCOM1_VNN_REQ_STS",         BIT(23),	1},
+	{"GPIOCOM0_VNN_REQ_STS",         BIT(24),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_vnn_req_status_3_map[] = {
+	{"DTS0_VNN_REQ_STS",             BIT(7),	0},
+	{"GPIOCOM5_VNN_REQ_STS",         BIT(11),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_vnn_misc_status_map[] = {
+	{"CPU_C10_REQ_STS",              BIT(0),	0},
+	{"TS_OFF_REQ_STS",               BIT(1),	0},
+	{"PNDE_MET_REQ_STS",             BIT(2),	1},
+	{"PG5_PMA0_REQ_STS",             BIT(3),	1},
+	{"FW_THROTTLE_ALLOWED_REQ_STS",  BIT(4),	0},
+	{"VNN_SOC_REQ_STS",              BIT(6),	1},
+	{"ISH_VNNAON_REQ_STS",           BIT(7),	0},
+	{"D2D_NOC_CFI_QACTIVE_REQ_STS",	 BIT(8),	1},
+	{"D2D_NOC_GPSB_QACTIVE_REQ_STS", BIT(9),	1},
+	{"D2D_IPU_QACTIVE_REQ_STS",      BIT(10),	1},
+	{"PLT_GREATER_REQ_STS",          BIT(11),	1},
+	{"ALL_SBR_IDLE_REQ_STS",         BIT(12),	0},
+	{"PMC_IDLE_FB_OCP_REQ_STS",      BIT(13),	0},
+	{"PM_SYNC_STATES_REQ_STS",       BIT(14),	0},
+	{"EA_REQ_STS",                   BIT(15),	0},
+	{"MPHY_CORE_OFF_REQ_STS",        BIT(16),	0},
+	{"BRK_EV_EN_REQ_STS",            BIT(17),	0},
+	{"AUTO_DEMO_EN_REQ_STS",         BIT(18),	0},
+	{"ITSS_CLK_SRC_REQ_STS",         BIT(19),	1},
+	{"ARC_IDLE_REQ_STS",             BIT(21),	0},
+	{"PG5_PMA1_REQ_STS",             BIT(22),	1},
+	{"FIA_DEEP_PM_REQ_STS",          BIT(23),	0},
+	{"XDCI_ATTACHED_REQ_STS",        BIT(24),	1},
+	{"ARC_INTERRUPT_WAKE_REQ_STS",   BIT(25),	0},
+	{"D2D_DISP_DDI_QACTIVE_REQ_STS", BIT(26),	1},
+	{"PRE_WAKE0_REQ_STS",            BIT(27),	1},
+	{"PRE_WAKE1_REQ_STS",            BIT(28),	1},
+	{"PRE_WAKE2_REQ_STS",            BIT(29),	1},
+	{"PG5_PMA2_GVNN",                BIT(30),	1},
+	{"D2D_DISP_EDP_QACTIVE_REQ_STS", BIT(31),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcdh_rsc_status_map[] = {
+	{"CORE",		0,		1},
+	{"Memory",		0,		1},
+	{"PRIM_D2D",		0,		1},
+	{"PSF0",		0,		1},
+	{"PSF4",		0,		1},
+	{"PSF6",		0,		1},
+	{"PSF8",		0,		1},
+	{"SB",			0,		1},
+	{}
+};
+
+static const struct pmc_bit_map *nvl_pcdh_lpm_maps[] = {
+	nvl_pcdh_clocksource_status_map,
+	nvl_pcdh_power_gating_status_0_map,
+	nvl_pcdh_power_gating_status_1_map,
+	nvl_pcdh_power_gating_status_2_map,
+	nvl_pcdh_power_gating_status_3_map,
+	nvl_pcdh_d3_status_0_map,
+	nvl_pcdh_d3_status_1_map,
+	nvl_pcdh_d3_status_2_map,
+	nvl_pcdh_d3_status_3_map,
+	nvl_pcdh_vnn_req_status_0_map,
+	nvl_pcdh_vnn_req_status_1_map,
+	nvl_pcdh_vnn_req_status_2_map,
+	nvl_pcdh_vnn_req_status_3_map,
+	nvl_pcdh_vnn_misc_status_map,
+	ptl_pcdp_signal_status_map,
+	NULL
+};
+
+static const struct pmc_bit_map *nvl_pcdh_blk_maps[] = {
+	nvl_pcdh_power_gating_status_0_map,
+	nvl_pcdh_power_gating_status_1_map,
+	nvl_pcdh_power_gating_status_2_map,
+	nvl_pcdh_power_gating_status_3_map,
+	nvl_pcdh_rsc_status_map,
+	nvl_pcdh_vnn_req_status_0_map,
+	nvl_pcdh_vnn_req_status_1_map,
+	nvl_pcdh_vnn_req_status_2_map,
+	nvl_pcdh_vnn_req_status_3_map,
+	nvl_pcdh_d3_status_0_map,
+	nvl_pcdh_d3_status_1_map,
+	nvl_pcdh_d3_status_2_map,
+	nvl_pcdh_d3_status_3_map,
+	nvl_pcdh_clocksource_status_map,
+	nvl_pcdh_vnn_misc_status_map,
+	ptl_pcdp_signal_status_map,
+	NULL
+};
+
+static const struct pmc_bit_map nvl_pcds_pfear_map[] = {
+	{"PMC_PGD0",                 BIT(0)},
+	{"FUSE_OSSE_PGD0",           BIT(1)},
+	{"SPI_PGD0",                 BIT(2)},
+	{"XHCI_PGD0",                BIT(3)},
+	{"SPA_PGD0",                 BIT(4)},
+	{"SPB_PGD0",                 BIT(5)},
+	{"RSVD6",                    BIT(6)},
+	{"GBE_PGD0",                 BIT(7)},
+
+	{"RSVD8",                    BIT(0)},
+	{"RSVD9",                    BIT(1)},
+	{"SBR16B7_PGD0",             BIT(2)},
+	{"SBR16B21_PGD0",            BIT(3)},
+	{"RSVD12",                   BIT(4)},
+	{"D2D_DISP_PGD1",            BIT(5)},
+	{"LPSS_PGD0",                BIT(6)},
+	{"LPC_PGD0",                 BIT(7)},
+
+	{"SMB_PGD0",                 BIT(0)},
+	{"ISH_PGD0",                 BIT(1)},
+	{"SBR16B1_PGD0",             BIT(2)},
+	{"NPK_PGD0",                 BIT(3)},
+	{"D2D_NOC_PGD1",             BIT(4)},
+	{"DBG_SBR16B_PGD0",          BIT(5)},
+	{"FUSE_PGD0",                BIT(6)},
+	{"RSVD23",                   BIT(7)},
+
+	{"P2SB0_PGD0",               BIT(0)},
+	{"OTG_PGD0",                 BIT(1)},
+	{"EXI_PGD0",                 BIT(2)},
+	{"CSE_PGD0",                 BIT(3)},
+	{"CSME_KVM_PGD0",            BIT(4)},
+	{"CSME_PMT_PGD0",            BIT(5)},
+	{"CSME_CLINK_PGD0",          BIT(6)},
+	{"CSME_PTIO_PGD0",           BIT(7)},
+
+	{"CSME_USBR_PGD0",           BIT(0)},
+	{"SBR16B22_PGD0",            BIT(1)},
+	{"CSME_SMT1_PGD0",           BIT(2)},
+	{"P2SB1_PGD0",               BIT(3)},
+	{"CSME_SMS2_PGD0",           BIT(4)},
+	{"CSME_SMS_PGD0",            BIT(5)},
+	{"CSME_RTC_PGD0",            BIT(6)},
+	{"CSMEPSF_PGD0",             BIT(7)},
+
+	{"D2D_NOC_PGD0",             BIT(0)},
+	{"RSVD41",                   BIT(1)},
+	{"RSVD42",                   BIT(2)},
+	{"RSVD43",                   BIT(3)},
+	{"SBR16B2_PGD0",             BIT(4)},
+	{"OSSE_SMT1_PGD0",           BIT(5)},
+	{"D2D_DISP_PGD0",            BIT(6)},
+	{"RSVD47_PGD0",              BIT(7)},
+
+	{"RSVD48",                   BIT(0)},
+	{"DBG_PSF_PGD0",             BIT(1)},
+	{"RSVD50",                   BIT(2)},
+	{"CNVI_PGD0",                BIT(3)},
+	{"UFSX2_PGD0",               BIT(4)},
+	{"ENDBG_PGD0",               BIT(5)},
+	{"DBC_PGD0",                 BIT(6)},
+	{"SBR16B4_PGD0",             BIT(7)},
+
+	{"RSVD56",                   BIT(0)},
+	{"NPK_PGD1",                 BIT(1)},
+	{"RSVD58",                   BIT(2)},
+	{"SBR16B20_PGD0",            BIT(3)},
+	{"RSVD60",                   BIT(4)},
+	{"SBR8B20_PGD0",             BIT(5)},
+	{"RSVD62",                   BIT(6)},
+	{"FIA_U_PGD0",               BIT(7)},
+
+	{"PSF8_PGD0",                BIT(0)},
+	{"RSVD65",                   BIT(1)},
+	{"RSVD66",                   BIT(2)},
+	{"FIACPCB_U_PGD0",           BIT(3)},
+	{"TAM_PGD0",                 BIT(4)},
+	{"D2D_NOC_PGD2",             BIT(5)},
+	{"SBR8B2_PGD0",              BIT(6)},
+	{"THC0_PGD0",                BIT(7)},
+
+	{"THC1_PGD0",                BIT(0)},
+	{"PMC_PGD1",                 BIT(1)},
+	{"SBR16B3_PGD0",             BIT(2)},
+	{"TCSS_PGD0",                BIT(3)},
+	{"DISP_PGA_PGD0",            BIT(4)},
+	{"RSVD77",                   BIT(5)},
+	{"RSVD78",                   BIT(6)},
+	{"RSVD79",                   BIT(7)},
+
+	{"SBRG_PGD0",                BIT(0)},
+	{"RSVD81",                   BIT(1)},
+	{"SBR16B0_PGD0",             BIT(2)},
+	{"SBR8B0_PGD0",              BIT(3)},
+	{"PSF7_PGD0",                BIT(4)},
+	{"RSVD85",                   BIT(5)},
+	{"RSVD86",                   BIT(6)},
+	{"RSVD87",                   BIT(7)},
+
+	{"SBR16B6_PGD0",             BIT(0)},
+	{"PSD0_PGD0",                BIT(1)},
+	{"STRC_PGD0",                BIT(2)},
+	{"RSVD91",                   BIT(3)},
+	{"DBG_SBR_PGD0",             BIT(4)},
+	{"RSVD93",                   BIT(5)},
+	{"OSSE_PGD0",                BIT(6)},
+	{"DISP_PGA1_PGD0",           BIT(7)},
+	{}
+};
+
+static const struct pmc_bit_map *ext_nvl_pcds_pfear_map[] = {
+	nvl_pcds_pfear_map,
+	NULL
+};
+
+static const struct pmc_bit_map nvl_pcds_ltr_show_map[] = {
+	{"SOUTHPORT_A",		CNP_PMC_LTR_SPA},
+	{"SOUTHPORT_B",		CNP_PMC_LTR_SPB},
+	{"SATA",		CNP_PMC_LTR_SATA},
+	{"GIGABIT_ETHERNET",	CNP_PMC_LTR_GBE},
+	{"XHCI",		CNP_PMC_LTR_XHCI},
+	{"SOUTHPORT_F",		ADL_PMC_LTR_SPF},
+	{"ME",			CNP_PMC_LTR_ME},
+	{"SATA1",		CNP_PMC_LTR_EVA},
+	{"SOUTHPORT_C",		CNP_PMC_LTR_SPC},
+	{"HD_AUDIO",		CNP_PMC_LTR_AZ},
+	{"CNV",			CNP_PMC_LTR_CNV},
+	{"LPSS",		CNP_PMC_LTR_LPSS},
+	{"SOUTHPORT_D",		CNP_PMC_LTR_SPD},
+	{"SOUTHPORT_E",		CNP_PMC_LTR_SPE},
+	{"SATA2",		PTL_PMC_LTR_SATA2},
+	{"ESPI",		CNP_PMC_LTR_ESPI},
+	{"SCC",			CNP_PMC_LTR_SCC},
+	{"ISH",			CNP_PMC_LTR_ISH},
+	{"UFSX2",		CNP_PMC_LTR_UFSX2},
+	{"EMMC",		CNP_PMC_LTR_EMMC},
+	{"WIGIG",		ICL_PMC_LTR_WIGIG},
+	{"THC0",		TGL_PMC_LTR_THC0},
+	{"THC1",		TGL_PMC_LTR_THC1},
+	{"SOUTHPORT_G",		MTL_PMC_LTR_SPG},
+	{"RSVD",		NVL_PCDS_PMC_LTR_RESERVED},
+	{"IOE_PMC",		MTL_PMC_LTR_IOE_PMC},
+	{"DMI3",		ARL_PMC_LTR_DMI3},
+	{"OSSE",		LNL_PMC_LTR_OSSE},
+
+	/* Below two cannot be used for LTR_IGNORE */
+	{"CURRENT_PLATFORM",	PTL_PMC_LTR_CUR_PLT},
+	{"AGGREGATED_SYSTEM",	PTL_PMC_LTR_CUR_ASLT},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_clocksource_status_map[] = {
+	{"AON2_OFF_STS",                 BIT(0),	1},
+	{"AON3_OFF_STS",                 BIT(1),	0},
+	{"AON4_OFF_STS",                 BIT(2),	1},
+	{"AON5_OFF_STS",                 BIT(3),	1},
+	{"AON1_OFF_STS",                 BIT(4),	0},
+	{"XTAL_LVM_OFF_STS",             BIT(5),	0},
+	{"D2D_OFF_STS",                  BIT(8),	1},
+	{"AON3_SPL_OFF_STS",             BIT(9),	1},
+	{"XTAL_AGGR_OFF_STS",            BIT(17),	1},
+	{"BCLK_EXT_INJ_OFF_STS",         BIT(18),	1},
+	{"DDI2_PLL_OFF_STS",             BIT(19),	1},
+	{"SE_TCSS_PLL_OFF_STS",          BIT(20),	1},
+	{"DDI_PLL_OFF_STS",              BIT(21),	1},
+	{"FILTER_PLL_OFF_STS",           BIT(22),	1},
+	{"PHY_OC_EXT_INJ_OFF_STS",       BIT(23),	1},
+	{"ACE_PLL_OFF_STS",              BIT(24),	0},
+	{"FABRIC_PLL_OFF_STS",           BIT(25),	1},
+	{"SOC_PLL_OFF_STS",              BIT(26),	1},
+	{"REF_PLL_OFF_STS",              BIT(28),	1},
+	{"GENLOCK_FILTER_PLL_OFF_STS",   BIT(30),	1},
+	{"RTC_PLL_OFF_STS",              BIT(31),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_power_gating_status_0_map[] = {
+	{"PMC_PGD0_PG_STS",              BIT(0),	0},
+	{"FUSE_OSSE_PGD0_PG_STS",	 BIT(1),	0},
+	{"ESPISPI_PGD0_PG_STS",          BIT(2),	0},
+	{"XHCI_PGD0_PG_STS",             BIT(3),	0},
+	{"SPA_PGD0_PG_STS",              BIT(4),	0},
+	{"SPB_PGD0_PG_STS",              BIT(5),	0},
+	{"RSVD_6",                       BIT(6),	0},
+	{"GBE_PGD0_PG_STS",              BIT(7),	0},
+	{"RSVD_8",                       BIT(8),	0},
+	{"RSVD_9",                       BIT(9),	0},
+	{"SBR16B7_PGD0_PG_STS",          BIT(10),	0},
+	{"SBR16B21_PGD0_PG_STS",         BIT(11),	0},
+	{"RSVD_12",                      BIT(12),	0},
+	{"D2D_DISP_PGD1_PG_STS",         BIT(13),	1},
+	{"LPSS_PGD0_PG_STS",             BIT(14),	0},
+	{"LPC_PGD0_PG_STS",              BIT(15),	0},
+	{"SMB_PGD0_PG_STS",              BIT(16),	0},
+	{"ISH_PGD0_PG_STS",              BIT(17),	0},
+	{"SBR16B1_PGD0_PG_STS",          BIT(18),	0},
+	{"NPK_PGD0_PG_STS",              BIT(19),	0},
+	{"D2D_NOC_PGD1_PG_STS",          BIT(20),	1},
+	{"DBG_SBR16B_PGD0_PG_STS",       BIT(21),	0},
+	{"FUSE_PGD0_PG_STS",             BIT(22),	0},
+	{"RSVD_23",                      BIT(23),	0},
+	{"P2SB0_PGD0_PG_STS",            BIT(24),	1},
+	{"XDCI_PGD0_PG_STS",             BIT(25),	0},
+	{"EXI_PGD0_PG_STS",              BIT(26),	0},
+	{"CSE_PGD0_PG_STS",              BIT(27),	1},
+	{"KVMCC_PGD0_PG_STS",            BIT(28),	0},
+	{"PMT_PGD0_PG_STS",              BIT(29),	0},
+	{"CLINK_PGD0_PG_STS",            BIT(30),	0},
+	{"PTIO_PGD0_PG_STS",             BIT(31),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_power_gating_status_1_map[] = {
+	{"USBR0_PGD0_PG_STS",            BIT(0),	0},
+	{"SBR16B22_PGD0_PG_STS",         BIT(1),	0},
+	{"SMT1_PGD0_PG_STS",             BIT(2),	0},
+	{"P2SB1_PGD0_PG_STS",            BIT(3),	1},
+	{"SMS2_PGD0_PG_STS",             BIT(4),	0},
+	{"SMS1_PGD0_PG_STS",             BIT(5),	0},
+	{"CSMERTC_PGD0_PG_STS",          BIT(6),	0},
+	{"CSMEPSF_PGD0_PG_STS",          BIT(7),	0},
+	{"D2D_NOC_PGD0_PG_STS",          BIT(8),	0},
+	{"RSVD_9",                       BIT(9),	0},
+	{"RSVD_10",                      BIT(10),	0},
+	{"RSVD_11",                      BIT(11),	0},
+	{"SBR16B2_PGD0_PG_STS",          BIT(12),	0},
+	{"OSSE_SMT1_PGD0_PG_STS",        BIT(13),	1},
+	{"D2D_DISP_PGD0_PG_STS",         BIT(14),	1},
+	{"RSVD_15",                      BIT(15),	0},
+	{"RSVD_16",                      BIT(16),	0},
+	{"DBG_PSF_PGD0_PG_STS",          BIT(17),	0},
+	{"RSVD_18",                      BIT(18),	0},
+	{"CNVI_PGD0_PG_STS",             BIT(19),	0},
+	{"UFSX2_PGD0_PG_STS",            BIT(20),	0},
+	{"ENDBG_PGD0_PG_STS",            BIT(21),	0},
+	{"DBC_PGD0_PG_STS",              BIT(22),	0},
+	{"SBR16B4_PGD0_PG_STS",          BIT(23),	0},
+	{"RSVD_24",                      BIT(24),	0},
+	{"NPK_PGD1_PG_STS",              BIT(25),	0},
+	{"RSVD_26",                      BIT(26),	0},
+	{"SBR16B20_PGD0_PG_STS",         BIT(27),	0},
+	{"RSVD_28",                      BIT(28),	0},
+	{"SBR8B20_PGD0_PG_STS",          BIT(29),	0},
+	{"RSVD_30",                      BIT(30),	0},
+	{"FIA_U_PGD0_PG_STS",            BIT(31),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_power_gating_status_2_map[] = {
+	{"PSF8_PGD0_PG_STS",             BIT(0),	0},
+	{"RSVD_1",                       BIT(1),	0},
+	{"RSVD_2",                       BIT(2),	0},
+	{"FIACPCB_U_PGD0_PG_STS",        BIT(3),	0},
+	{"TAM_PGD0_PG_STS",              BIT(4),	1},
+	{"D2D_NOC_PGD2_PG_STS",          BIT(5),	1},
+	{"SBR8B2_PGD0_PG_STS",           BIT(6),	0},
+	{"THC0_PGD0_PG_STS",             BIT(7),	0},
+	{"THC1_PGD0_PG_STS",             BIT(8),	0},
+	{"PMC_PGD1_PG_STS",              BIT(9),	0},
+	{"SBR16B3_PGD0_PG_STS",          BIT(10),	0},
+	{"TCSS_PGD0_PG_STS",             BIT(11),	0},
+	{"DISP_PGA_PGD0_PG_STS",         BIT(12),	0},
+	{"RSVD_13",                      BIT(13),	0},
+	{"RSVD_14",                      BIT(14),	0},
+	{"RSVD_15",                      BIT(15),	0},
+	{"SBRG_PGD0_PG_STS",             BIT(16),	0},
+	{"RSVD_17",                      BIT(17),	0},
+	{"SBR16B0_PGD0_PG_STS",          BIT(18),	0},
+	{"SBR8B0_PGD0_PG_STS",           BIT(19),	0},
+	{"PSF7_PGD0_PG_STS",             BIT(20),	0},
+	{"RSVD_21",                      BIT(21),	0},
+	{"RSVD_22",                      BIT(22),	0},
+	{"RSVD_23",                      BIT(23),	0},
+	{"SBR16B6_PGD0_PG_STS",          BIT(24),	0},
+	{"PSF0_PGD0_PG_STS",             BIT(25),	0},
+	{"STRC_PGD0_PG_STS",             BIT(26),	0},
+	{"RSVD_27",                      BIT(27),	0},
+	{"DBG_SBR_PGD0_PG_STS",          BIT(28),	0},
+	{"RSVD_29",                      BIT(29),	0},
+	{"OSSE_PGD0_PG_STS",             BIT(30),	1},
+	{"DISP_PGA1_PGD0_PG_STS",        BIT(31),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_d3_status_0_map[] = {
+	{"LPSS_D3_STS",                  BIT(3),	1},
+	{"XDCI_D3_STS",                  BIT(4),	1},
+	{"XHCI_D3_STS",                  BIT(5),	1},
+	{"SPA_D3_STS",                   BIT(12),	0},
+	{"SPB_D3_STS",                   BIT(13),	0},
+	{"ESPISPI_D3_STS",               BIT(18),	0},
+	{"PSTH_D3_STS",                  BIT(21),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_d3_status_1_map[] = {
+	{"OSSE_D3_STS",                  BIT(14),	0},
+	{"GBE_D3_STS",                   BIT(19),	0},
+	{"ITSS_D3_STS",                  BIT(23),	0},
+	{"CNVI_D3_STS",                  BIT(27),	0},
+	{"UFSX2_D3_STS",                 BIT(28),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_d3_status_2_map[] = {
+	{"CSMERTC_D3_STS",               BIT(1),	0},
+	{"CSE_D3_STS",                   BIT(4),	0},
+	{"KVMCC_D3_STS",                 BIT(5),	0},
+	{"USBR0_D3_STS",                 BIT(6),	0},
+	{"ISH_D3_STS",                   BIT(7),	0},
+	{"SMT1_D3_STS",                  BIT(8),	0},
+	{"SMT2_D3_STS",                  BIT(9),	0},
+	{"SMT3_D3_STS",                  BIT(10),	0},
+	{"OSSE_SMT1_D3_STS",             BIT(12),	0},
+	{"CLINK_D3_STS",                 BIT(14),	0},
+	{"PTIO_D3_STS",                  BIT(16),	0},
+	{"PMT_D3_STS",                   BIT(17),	0},
+	{"SMS1_D3_STS",                  BIT(18),	0},
+	{"SMS2_D3_STS",                  BIT(19),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_d3_status_3_map[] = {
+	{"OSSE_SMT2_D3_STS",             BIT(0),	0},
+	{"THC0_D3_STS",                  BIT(14),	1},
+	{"THC1_D3_STS",                  BIT(15),	1},
+	{"OSSE_SMT3_D3_STS",             BIT(19),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_vnn_req_status_0_map[] = {
+	{"LPSS_VNN_REQ_STS",             BIT(3),	0},
+	{"ESPISPI_VNN_REQ_STS",          BIT(18),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_vnn_req_status_1_map[] = {
+	{"NPK_VNN_REQ_STS",              BIT(4),	1},
+	{"DFXAGG_VNN_REQ_STS",           BIT(8),	0},
+	{"EXI_VNN_REQ_STS",              BIT(9),	1},
+	{"OSSE_VNN_REQ_STS",             BIT(14),	1},
+	{"P2D_VNN_REQ_STS",              BIT(18),	1},
+	{"GBE_VNN_REQ_STS",              BIT(19),	0},
+	{"SMB_VNN_REQ_STS",              BIT(25),	1},
+	{"LPC_VNN_REQ_STS",              BIT(26),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_vnn_req_status_2_map[] = {
+	{"CSMERTC_VNN_REQ_STS",          BIT(1),	0},
+	{"CSE_VNN_REQ_STS",              BIT(4),	1},
+	{"ISH_VNN_REQ_STS",              BIT(7),	0},
+	{"SMT1_VNN_REQ_STS",             BIT(8),	0},
+	{"OSSE_SMT1_VNN_REQ_STS",        BIT(12),	1},
+	{"CLINK_VNN_REQ_STS",            BIT(14),	0},
+	{"SMS1_VNN_REQ_STS",             BIT(18),	0},
+	{"SMS2_VNN_REQ_STS",             BIT(19),	0},
+	{"GPIOCOM4_VNN_REQ_STS",         BIT(20),	0},
+	{"GPIOCOM3_VNN_REQ_STS",         BIT(21),	1},
+	{"GPIOCOM1_VNN_REQ_STS",         BIT(23),	1},
+	{"GPIOCOM0_VNN_REQ_STS",         BIT(24),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_vnn_req_status_3_map[] = {
+	{"DISP_SHIM_VNN_REQ_STS",        BIT(4),	1},
+	{"DTS0_VNN_REQ_STS",             BIT(7),	0},
+	{"GPIOCOM5_VNN_REQ_STS",         BIT(11),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_vnn_misc_status_map[] = {
+	{"CPU_C10_REQ_STS",              BIT(0),	0},
+	{"TS_OFF_REQ_STS",               BIT(1),	0},
+	{"PNDE_MET_REQ_STS",             BIT(2),	1},
+	{"PG5_PMA0_REQ_STS",             BIT(3),	1},
+	{"FW_THROTTLE_ALLOWED_REQ_STS",  BIT(4),	0},
+	{"VNN_SOC_REQ_STS",              BIT(6),	1},
+	{"ISH_VNNAON_REQ_STS",           BIT(7),	0},
+	{"D2D_NOC_CFI_QACTIVE_REQ_STS",	 BIT(8),	1},
+	{"D2D_NOC_GPSB_QACTIVE_REQ_STS", BIT(9),	1},
+	{"PLT_GREATER_REQ_STS",          BIT(11),	1},
+	{"ALL_SBR_IDLE_REQ_STS",         BIT(12),	0},
+	{"PMC_IDLE_FB_OCP_REQ_STS",      BIT(13),	0},
+	{"PM_SYNC_STATES_REQ_STS",       BIT(14),	0},
+	{"EA_REQ_STS",                   BIT(15),	0},
+	{"MPHY_CORE_OFF_REQ_STS",        BIT(16),	0},
+	{"BRK_EV_EN_REQ_STS",            BIT(17),	0},
+	{"AUTO_DEMO_EN_REQ_STS",         BIT(18),	0},
+	{"ITSS_CLK_SRC_REQ_STS",         BIT(19),	1},
+	{"ARC_IDLE_REQ_STS",             BIT(21),	0},
+	{"PG5_PMA1_REQ_STS",             BIT(22),	1},
+	{"DG5_PMA0_REQ_STS",             BIT(23),	1},
+	{"ARC_INTERRUPT_WAKE_REQ_STS",   BIT(25),	0},
+	{"D2D_DISP_DDI_QACTIVE_REQ_STS", BIT(26),	1},
+	{"PRE_WAKE0_REQ_STS",            BIT(27),	1},
+	{"PRE_WAKE1_REQ_STS",            BIT(28),	1},
+	{"PRE_WAKE2_REQ_STS",            BIT(29),	1},
+	{"D2D_DISP_EDP_QACTIVE_REQ_STS", BIT(31),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_rsc_status_map[] = {
+	{"CORE",		0,		1},
+	{"Memory",		0,		1},
+	{"PRIM_D2D",		0,		1},
+	{"PSF0",		0,		1},
+	{"SB",			0,		1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pcds_signal_status_map[] = {
+	{"LSX_Wake0_STS",		 BIT(0),	0},
+	{"LSX_Wake1_STS",		 BIT(1),	0},
+	{"LSX_Wake2_STS",		 BIT(2),	0},
+	{"LSX_Wake3_STS",		 BIT(3),	0},
+	{"LSX_Wake4_STS",		 BIT(4),	0},
+	{"LSX_Wake5_STS",		 BIT(5),	0},
+	{"LSX_Wake6_STS",		 BIT(6),	0},
+	{"LSX_Wake7_STS",		 BIT(7),	0},
+	{"LPSS_Wake0_STS",		 BIT(8),	1},
+	{"LPSS_Wake1_STS",		 BIT(9),	1},
+	{"Int_Timer_SS_Wake0_STS",	 BIT(10),	1},
+	{"Int_Timer_SS_Wake1_STS",	 BIT(11),	1},
+	{"Int_Timer_SS_Wake2_STS",	 BIT(12),	1},
+	{"Int_Timer_SS_Wake3_STS",	 BIT(13),	1},
+	{"Int_Timer_SS_Wake4_STS",	 BIT(14),	1},
+	{"Int_Timer_SS_Wake5_STS",	 BIT(15),	1},
+	{}
+};
+
+static const struct pmc_bit_map *nvl_pcds_lpm_maps[] = {
+	nvl_pcds_clocksource_status_map,
+	nvl_pcds_power_gating_status_0_map,
+	nvl_pcds_power_gating_status_1_map,
+	nvl_pcds_power_gating_status_2_map,
+	nvl_pcds_d3_status_0_map,
+	nvl_pcds_d3_status_1_map,
+	nvl_pcds_d3_status_2_map,
+	nvl_pcds_d3_status_3_map,
+	nvl_pcds_vnn_req_status_0_map,
+	nvl_pcds_vnn_req_status_1_map,
+	nvl_pcds_vnn_req_status_2_map,
+	nvl_pcds_vnn_req_status_3_map,
+	nvl_pcds_vnn_misc_status_map,
+	nvl_pcds_signal_status_map,
+	NULL
+};
+
+static const struct pmc_bit_map *nvl_pcds_blk_maps[] = {
+	nvl_pcds_power_gating_status_0_map,
+	nvl_pcds_power_gating_status_1_map,
+	nvl_pcds_power_gating_status_2_map,
+	nvl_pcds_rsc_status_map,
+	nvl_pcds_vnn_req_status_0_map,
+	nvl_pcds_vnn_req_status_1_map,
+	nvl_pcds_vnn_req_status_2_map,
+	nvl_pcds_vnn_req_status_3_map,
+	nvl_pcds_d3_status_0_map,
+	nvl_pcds_d3_status_1_map,
+	nvl_pcds_d3_status_2_map,
+	nvl_pcds_d3_status_3_map,
+	nvl_pcds_clocksource_status_map,
+	nvl_pcds_vnn_misc_status_map,
+	nvl_pcds_signal_status_map,
+	NULL
+};
+
+static const struct pmc_bit_map nvl_pchs_pfear_map[] = {
+	{"PMC_PGD0",                 BIT(0)},
+	{"FIA_D_PGD0",               BIT(1)},
+	{"SPI_PGD0",                 BIT(2)},
+	{"XHCI_PGD0",                BIT(3)},
+	{"SPA_PGD0",                 BIT(4)},
+	{"SPB_PGD0",                 BIT(5)},
+	{"MPFPW2_PGD0",              BIT(6)},
+	{"GBE_PGD0",                 BIT(7)},
+
+	{"RSVD8",                    BIT(0)},
+	{"PSF3_PGD0",                BIT(1)},
+	{"SBR5_PGD0",                BIT(2)},
+	{"SBR0_PGD0",                BIT(3)},
+	{"RSVD12",                   BIT(4)},
+	{"D2D_DISP_PGD1",            BIT(5)},
+	{"LPSS_PGD0",                BIT(6)},
+	{"LPC_PGD0",                 BIT(7)},
+
+	{"SMB_PGD0",                 BIT(0)},
+	{"ISH_PGD0",                 BIT(1)},
+	{"P2SB_PGD0",                BIT(2)},
+	{"NPK_PGD0",                 BIT(3)},
+	{"D2D_NOC_PGD1",             BIT(4)},
+	{"EAH_PGD0",                 BIT(5)},
+	{"FUSE_PGD0",                BIT(6)},
+	{"SBR8_PGD0",                BIT(7)},
+
+	{"PSF7_PGD0",                BIT(0)},
+	{"OTG_PGD0",                 BIT(1)},
+	{"EXI_PGD0",                 BIT(2)},
+	{"CSE_PGD0",                 BIT(3)},
+	{"CSME_KVM_PGD0",            BIT(4)},
+	{"CSME_PMT_PGD0",            BIT(5)},
+	{"CSME_CLINK_PGD0",          BIT(6)},
+	{"CSME_PTIO_PGD0",           BIT(7)},
+
+	{"CSME_USBR_PGD0",           BIT(0)},
+	{"SBR1_PGD0",                BIT(1)},
+	{"CSME_SMT1_PGD0",           BIT(2)},
+	{"MPFPW1_PGD0",              BIT(3)},
+	{"CSME_SMS2_PGD0",           BIT(4)},
+	{"CSME_SMS_PGD0",            BIT(5)},
+	{"CSME_RTC_PGD0",            BIT(6)},
+	{"CSMEPSF_PGD0",             BIT(7)},
+
+	{"D2D_NOC_PGD0",             BIT(0)},
+	{"ESE_PGD0",                 BIT(1)},
+	{"SBR2_PGD0",                BIT(2)},
+	{"SBR3_PGD0",                BIT(3)},
+	{"SBR4_PGD0",                BIT(4)},
+	{"RSVD45",                   BIT(5)},
+	{"D2D_DISP_PGD0",            BIT(6)},
+	{"PSF1_PGD0",                BIT(7)},
+
+	{"U3FPW1_PGD0",              BIT(0)},
+	{"DMI3FPW_PGD0",             BIT(1)},
+	{"PSF4_PGD0",                BIT(2)},
+	{"CNVI_PGD0",                BIT(3)},
+	{"RSVD52",                   BIT(4)},
+	{"ENDBG_PGD0",               BIT(5)},
+	{"DBC_PGD0",                 BIT(6)},
+	{"SMT4_PGD0",                BIT(7)},
+
+	{"RSVD56",                   BIT(0)},
+	{"NPK_PGD1",                 BIT(1)},
+	{"RSVD58",                   BIT(2)},
+	{"DMI3_PGD0",                BIT(3)},
+	{"RSVD60",                   BIT(4)},
+	{"FIACPCB_D_PGD0",           BIT(5)},
+	{"RSVD62",                   BIT(6)},
+	{"FIA_U_PGD0",               BIT(7)},
+
+	{"FIACPCB_PGS_PGD0",         BIT(0)},
+	{"FIA_PGS_PGD0",             BIT(1)},
+	{"RSVD66",                   BIT(2)},
+	{"FIACPCB_U_PGD0",           BIT(3)},
+	{"TAM_PGD0",                 BIT(4)},
+	{"D2D_NOC_PGD2",             BIT(5)},
+	{"PSF2_PGD0",                BIT(6)},
+	{"THC0_PGD0",                BIT(7)},
+
+	{"THC1_PGD0",                BIT(0)},
+	{"PMC_PGD1",                 BIT(1)},
+	{"SBR9_PGD0",                BIT(2)},
+	{"U3FPW2_PGD0",              BIT(3)},
+	{"RSVD76",                   BIT(4)},
+	{"DBG_PSF_PGD0",             BIT(5)},
+	{"DBG_SBR_PGD0",             BIT(6)},
+	{"SBR6_PGD0",                BIT(7)},
+
+	{"SPC_PGD0",                 BIT(0)},
+	{"ACE_PGD0",                 BIT(1)},
+	{"ACE_PGD1",                 BIT(2)},
+	{"ACE_PGD2",                 BIT(3)},
+	{"ACE_PGD3",                 BIT(4)},
+	{"ACE_PGD4",                 BIT(5)},
+	{"ACE_PGD5",                 BIT(6)},
+	{"ACE_PGD6",                 BIT(7)},
+
+	{"ACE_PGD7",                 BIT(0)},
+	{"ACE_PGD8",                 BIT(1)},
+	{"ACE_PGD9",                 BIT(2)},
+	{"ACE_PGD10",                BIT(3)},
+	{"U3FPW3_PGD0",              BIT(4)},
+	{"SBR7_PGD0",                BIT(5)},
+	{"OSSE_PGD0",                BIT(6)},
+	{"ST_PGD0",                  BIT(7)},
+	{}
+};
+
+static const struct pmc_bit_map *ext_nvl_pchs_pfear_map[] = {
+	nvl_pchs_pfear_map,
+	NULL
+};
+
+static const struct pmc_bit_map nvl_pchs_clocksource_status_map[] = {
+	{"AON2_OFF_STS",                 BIT(0),	1},
+	{"AON3_OFF_STS",                 BIT(1),	0},
+	{"AON4_OFF_STS",                 BIT(2),	0},
+	{"AON2_SPL_OFF_STS",             BIT(3),	0},
+	{"AONL_OFF_STS",                 BIT(4),	0},
+	{"XTAL_LVM_OFF_STS",             BIT(5),	0},
+	{"AON5_OFF_STS",                 BIT(6),	0},
+	{"USB3_PLL_OFF_STS",             BIT(8),	1},
+	{"MAIN_CRO_OFF_STS",             BIT(11),	0},
+	{"MAIN_DIVIDER_OFF_STS",         BIT(12),	1},
+	{"REF_PLL_NON_OC_OFF_STS",       BIT(13),	1},
+	{"DMI_PLL_OFF_STS",              BIT(14),	1},
+	{"PHY_EXT_INJ_OFF_STS",          BIT(15),	1},
+	{"AON6_MCRO_OFF_STS",            BIT(16),	0},
+	{"XTAL_AGGR_OFF_STS",            BIT(17),	0},
+	{"USB2_PLL_OFF_STS",             BIT(18),	1},
+	{"GBE_PLL_OFF_STS",              BIT(21),	1},
+	{"SATA_PLL_OFF_STS",             BIT(22),	1},
+	{"PCIE0_PLL_OFF_STS",            BIT(23),	1},
+	{"PCIE1_PLL_OFF_STS",            BIT(24),	1},
+	{"FABRIC_PLL_OFF_STS",           BIT(25),	1},
+	{"PCIE2_PLL_OFF_STS",            BIT(26),	1},
+	{"REF_PLL_OFF_STS",              BIT(28),	1},
+	{"REF38P4_PLL_OFF_STS",          BIT(31),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_power_gating_status_0_map[] = {
+	{"PMC_PGD0_PG_STS",              BIT(0),	0},
+	{"FIA_D_PGD0_PG_STS",            BIT(1),	0},
+	{"ESPISPI_PGD0_PG_STS",          BIT(2),	0},
+	{"XHCI_PGD0_PG_STS",             BIT(3),	0},
+	{"SPA_PGD0_PG_STS",              BIT(4),	1},
+	{"SPB_PGD0_PG_STS",              BIT(5),	1},
+	{"MPFPW2_PGD0_PG_STS",           BIT(6),	0},
+	{"GBE_PGD0_PG_STS",              BIT(7),	1},
+	{"RSVD_8",                       BIT(8),	0},
+	{"PSF3_PGD0_PG_STS",             BIT(9),	0},
+	{"SBR5_PGD0_PG_STS",             BIT(10),	0},
+	{"SBR0_PGD0_PG_STS",             BIT(11),	0},
+	{"RSVD_12",                      BIT(12),	0},
+	{"D2D_DISP_PGD1_PG_STS",         BIT(13),	0},
+	{"LPSS_PGD0_PG_STS",             BIT(14),	1},
+	{"LPC_PGD0_PG_STS",              BIT(15),	0},
+	{"SMB_PGD0_PG_STS",              BIT(16),	0},
+	{"ISH_PGD0_PG_STS",              BIT(17),	0},
+	{"P2S_PGD0_PG_STS",              BIT(18),	0},
+	{"NPK_PGD0_PG_STS",              BIT(19),	0},
+	{"D2D_NOC_PGD1_PG_STS",          BIT(20),	0},
+	{"EAH_PGD0_PG_STS",              BIT(21),	0},
+	{"FUSE_PGD0_PG_STS",             BIT(22),	0},
+	{"SBR8_PGD0_PG_STS",             BIT(23),	0},
+	{"PSF7_PGD0_PG_STS",             BIT(24),	0},
+	{"XDCI_PGD0_PG_STS",             BIT(25),	1},
+	{"EXI_PGD0_PG_STS",              BIT(26),	0},
+	{"CSE_PGD0_PG_STS",              BIT(27),	1},
+	{"KVMCC_PGD0_PG_STS",            BIT(28),	1},
+	{"PMT_PGD0_PG_STS",              BIT(29),	1},
+	{"CLINK_PGD0_PG_STS",            BIT(30),	1},
+	{"PTIO_PGD0_PG_STS",             BIT(31),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_power_gating_status_1_map[] = {
+	{"USBR0_PGD0_PG_STS",            BIT(0),	1},
+	{"SBR1_PGD0_PG_STS",             BIT(1),	0},
+	{"SMT1_PGD0_PG_STS",             BIT(2),	1},
+	{"MPFPW1_PGD0_PG_STS",           BIT(3),	0},
+	{"SMS2_PGD0_PG_STS",             BIT(4),	1},
+	{"SMS1_PGD0_PG_STS",             BIT(5),	1},
+	{"CSMERTC_PGD0_PG_STS",          BIT(6),	0},
+	{"CSMEPSF_PGD0_PG_STS",          BIT(7),	0},
+	{"D2D_NOC_PGD0_PG_STS",          BIT(8),	0},
+	{"ESE_PGD0_PG_STS",              BIT(9),	1},
+	{"SBR2_PGD0_PG_STS",             BIT(10),	0},
+	{"SBR3_PGD0_PG_STS",             BIT(11),	0},
+	{"SBR4_PGD0_PG_STS",             BIT(12),	0},
+	{"RSVD_13",                      BIT(13),	0},
+	{"D2D_DISP_PGD0_PG_STS",         BIT(14),	0},
+	{"PSF1_PGD0_PG_STS",             BIT(15),	0},
+	{"U3FPW1_PGD0_PG_STS",           BIT(16),	0},
+	{"DMI3FPW_PGD0_PG_STS",          BIT(17),	0},
+	{"PSF4_PGD0_PG_STS",             BIT(18),	0},
+	{"CNVI_PGD0_PG_STS",             BIT(19),	0},
+	{"RSVD_20",                      BIT(20),	0},
+	{"ENDBG_PGD0_PG_STS",            BIT(21),	0},
+	{"DBC_PGD0_PG_STS",              BIT(22),	0},
+	{"SMT4_PGD0_PG_STS",             BIT(23),	1},
+	{"RSVD_24",                      BIT(24),	0},
+	{"NPK_PGD1_PG_STS",              BIT(25),	0},
+	{"RSVD_26",                      BIT(26),	0},
+	{"DMI3_PGD0_PG_STS",             BIT(27),	1},
+	{"RSVD_28",                      BIT(28),	0},
+	{"FIACPCB_D_PGD0_PG_STS",        BIT(29),	0},
+	{"RSVD_30",                      BIT(30),	0},
+	{"FIA_U_PGD0_PG_STS",            BIT(31),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_power_gating_status_2_map[] = {
+	{"FIACPCB_PGS_PGD0_PG_STS",      BIT(0),	0},
+	{"FIA_PGS_PGD0_PG_STS",          BIT(1),	0},
+	{"RSVD_2",                       BIT(2),	0},
+	{"FIACPCB_U_PGD0_PG_STS",        BIT(3),	0},
+	{"TAM_PGD0_PG_STS",              BIT(4),	0},
+	{"D2D_NOC_PGD2_PG_STS",          BIT(5),	0},
+	{"PSF2_PGD0_PG_STS",             BIT(6),	0},
+	{"THC0_PGD0_PG_STS",             BIT(7),	1},
+	{"THC1_PGD0_PG_STS",             BIT(8),	1},
+	{"PMC_PGD1_PG_STS",              BIT(9),	0},
+	{"SBR9_PGA0_PGD0_PG_STS",        BIT(10),	0},
+	{"U3FPW2_PGD0_PG_STS",           BIT(11),	0},
+	{"RSVD_12",                      BIT(12),	0},
+	{"DBG_PSF_PGD0_PG_STS",          BIT(13),	0},
+	{"DBG_SBR_PGD0_PG_STS",          BIT(14),	0},
+	{"SBR6_PGD0_PG_STS",             BIT(15),	0},
+	{"SPC_PGD0_PG_STS",              BIT(16),	1},
+	{"ACE_PGD0_PG_STS",              BIT(17),	0},
+	{"ACE_PGD1_PG_STS",              BIT(18),	0},
+	{"ACE_PGD2_PG_STS",              BIT(19),	0},
+	{"ACE_PGD3_PG_STS",              BIT(20),	0},
+	{"ACE_PGD4_PG_STS",              BIT(21),	0},
+	{"ACE_PGD5_PG_STS",              BIT(22),	0},
+	{"ACE_PGD6_PG_STS",              BIT(23),	0},
+	{"ACE_PGD7_PG_STS",              BIT(24),	0},
+	{"ACE_PGD8_PG_STS",              BIT(25),	0},
+	{"ACE_PGD9_PG_STS",              BIT(26),	0},
+	{"ACE_PGD10_PG_STS",             BIT(27),	0},
+	{"U3FPW3_PGD0_PG_STS",           BIT(28),	0},
+	{"SBR7_PGD0_PG_STS",             BIT(29),	0},
+	{"OSSE_PGD0_PG_STS",             BIT(30),	0},
+	{"SATA_PGD0_PG_STS",             BIT(31),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_d3_status_0_map[] = {
+	{"LPSS_D3_STS",                  BIT(3),	1},
+	{"XDCI_D3_STS",                  BIT(4),	1},
+	{"XHCI_D3_STS",                  BIT(5),	0},
+	{"SPA_D3_STS",                   BIT(12),	0},
+	{"SPB_D3_STS",                   BIT(13),	0},
+	{"SPC_D3_STS",                   BIT(14),	0},
+	{"ESPISPI_D3_STS",               BIT(18),	0},
+	{"SATA_D3_STS",                  BIT(20),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_d3_status_1_map[] = {
+	{"OSSE_D3_STS",                  BIT(6),	0},
+	{"GBE_D3_STS",                   BIT(19),	0},
+	{"ITSS_D3_STS",                  BIT(23),	0},
+	{"P2S_D3_STS",                   BIT(24),	0},
+	{"CNVI_D3_STS",                  BIT(27),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_d3_status_2_map[] = {
+	{"CSMERTC_D3_STS",               BIT(1),	0},
+	{"CSE_D3_STS",                   BIT(4),	0},
+	{"KVMCC_D3_STS",                 BIT(5),	0},
+	{"USBR0_D3_STS",                 BIT(6),	0},
+	{"ISH_D3_STS",                   BIT(7),	0},
+	{"SMT1_D3_STS",                  BIT(8),	0},
+	{"SMT2_D3_STS",                  BIT(9),	0},
+	{"SMT3_D3_STS",                  BIT(10),	0},
+	{"SMT4_D3_STS",                  BIT(11),	0},
+	{"SMT5_D3_STS",                  BIT(12),	0},
+	{"SMT6_D3_STS",                  BIT(13),	0},
+	{"CLINK_D3_STS",                 BIT(14),	0},
+	{"PTIO_D3_STS",                  BIT(16),	0},
+	{"PMT_D3_STS",                   BIT(17),	0},
+	{"SMS1_D3_STS",                  BIT(18),	0},
+	{"SMS2_D3_STS",                  BIT(19),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_d3_status_3_map[] = {
+	{"THC0_D3_STS",                  BIT(14),	0},
+	{"THC1_D3_STS",                  BIT(15),	0},
+	{"ACE_D3_STS",                   BIT(23),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_vnn_req_status_1_map[] = {
+	{"NPK_VNN_REQ_STS",              BIT(4),	0},
+	{"OSSE_VNN_REQ_STS",             BIT(6),	0},
+	{"DFXAGG_VNN_REQ_STS",           BIT(8),	0},
+	{"EXI_VNN_REQ_STS",              BIT(9),	0},
+	{"GBE_VNN_REQ_STS",              BIT(19),	0},
+	{"SMB_VNN_REQ_STS",              BIT(25),	0},
+	{"LPC_VNN_REQ_STS",              BIT(26),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_vnn_req_status_2_map[] = {
+	{"CSMERTC_VNN_REQ_STS",          BIT(1),	0},
+	{"CSE_VNN_REQ_STS",              BIT(4),	0},
+	{"ISH_VNN_REQ_STS",              BIT(7),	0},
+	{"SMT1_VNN_REQ_STS",             BIT(8),	0},
+	{"SMT4_VNN_REQ_STS",             BIT(11),	0},
+	{"CLINK_VNN_REQ_STS",            BIT(14),	0},
+	{"SMS1_VNN_REQ_STS",             BIT(18),	0},
+	{"SMS2_VNN_REQ_STS",             BIT(19),	0},
+	{"GPIOCOM4_VNN_REQ_STS",         BIT(20),	0},
+	{"GPIOCOM3_VNN_REQ_STS",         BIT(21),	0},
+	{"GPIOCOM2_VNN_REQ_STS",         BIT(22),	0},
+	{"GPIOCOM1_VNN_REQ_STS",         BIT(23),	0},
+	{"GPIOCOM0_VNN_REQ_STS",         BIT(24),	0},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_vnn_misc_status_map[] = {
+	{"CPU_C10_REQ_STS",              BIT(0),	0},
+	{"TS_OFF_REQ_STS",               BIT(1),	0},
+	{"PNDE_MET_REQ_STS",             BIT(2),	1},
+	{"PG5_PMA0_GVNN_REQ_STS",        BIT(3),	1},
+	{"FW_THROTTLE_ALLOWED_REQ_STS",  BIT(4),	0},
+	{"DMI_IN_L1_REQ_STS",            BIT(6),	0},
+	{"ISH_VNNAON_REQ_STS",           BIT(7),	0},
+	{"PLT_GREATER_REQ_STS",          BIT(11),	1},
+	{"ALL_SBR_IDLE_REQ_STS",         BIT(12),	0},
+	{"PMC_IDLE_FB_OCP_REQ_STS",      BIT(13),	0},
+	{"PM_SYNC_STATES_REQ_STS",       BIT(14),	0},
+	{"EA_REQ_STS",                   BIT(15),	0},
+	{"DMI_CLKREQ_B_REQ_STS",         BIT(16),	0},
+	{"BRK_EV_EN_REQ_STS",            BIT(17),	0},
+	{"AUTO_DEMO_EN_REQ_STS",         BIT(18),	0},
+	{"ITSS_CLK_SRC_REQ_STS",         BIT(19),	1},
+	{"ARC_IDLE_REQ_STS",             BIT(21),	0},
+	{"PG5_PMA1_GVNN_REQ_STS",        BIT(22),	1},
+	{"FIA_DEEP_PM_REQ_STS",          BIT(23),	0},
+	{"XDCI_ATTACHED_REQ_STS",        BIT(24),	0},
+	{"ARC_INTERRUPT_WAKE_REQ_STS",   BIT(25),	0},
+	{"PRE_WAKE0_REQ_STS",            BIT(27),	1},
+	{"PRE_WAKE1_REQ_STS",            BIT(28),	1},
+	{"PRE_WAKE2_EN_REQ_STS",         BIT(29),	0},
+	{"PG5_PMA2_GVNN_REQ_STS",        BIT(30),	1},
+	{}
+};
+
+static const struct pmc_bit_map nvl_pchs_rsc_status_map[] = {
+	{"Memory",		0,		1},
+	{"Memory_NS",		0,		1},
+	{"PSF1",		0,		1},
+	{"PSF2",		0,		1},
+	{"PSF3",		0,		1},
+	{"REF_PLL",		0,		1},
+	{"SB",			0,		1},
+	{}
+};
+
+static const struct pmc_bit_map *nvl_pchs_lpm_maps[] = {
+	nvl_pchs_clocksource_status_map,
+	nvl_pchs_power_gating_status_0_map,
+	nvl_pchs_power_gating_status_1_map,
+	nvl_pchs_power_gating_status_2_map,
+	nvl_pchs_d3_status_0_map,
+	nvl_pchs_d3_status_1_map,
+	nvl_pchs_d3_status_2_map,
+	nvl_pchs_d3_status_3_map,
+	nvl_pcds_vnn_req_status_0_map,
+	nvl_pchs_vnn_req_status_1_map,
+	nvl_pchs_vnn_req_status_2_map,
+	nvl_pcdh_vnn_req_status_3_map,
+	nvl_pchs_vnn_misc_status_map,
+	ptl_pcdp_signal_status_map,
+	NULL
+};
+
+static const struct pmc_bit_map *nvl_pchs_blk_maps[] = {
+	nvl_pchs_power_gating_status_0_map,
+	nvl_pchs_power_gating_status_1_map,
+	nvl_pchs_power_gating_status_2_map,
+	nvl_pchs_rsc_status_map,
+	nvl_pchs_d3_status_0_map,
+	nvl_pchs_clocksource_status_map,
+	nvl_pchs_vnn_misc_status_map,
+	NULL
+};
+
+static const struct pmc_reg_map nvl_pcdh_reg_map = {
+	.pfear_sts = ext_nvl_pcdh_pfear_map,
+	.slp_s0_offset = CNP_PMC_SLP_S0_RES_COUNTER_OFFSET,
+	.slp_s0_res_counter_step = TGL_PMC_SLP_S0_RES_COUNTER_STEP,
+	.ltr_show_sts = ptl_pcdp_ltr_show_map,
+	.msr_sts = msr_map,
+	.ltr_ignore_offset = CNP_PMC_LTR_IGNORE_OFFSET,
+	.regmap_length = NVL_PCDH_PMC_MMIO_REG_LEN,
+	.ppfear0_offset = CNP_PMC_HOST_PPFEAR0A,
+	.ppfear_buckets = NVL_PCDH_PPFEAR_NUM_ENTRIES,
+	.pm_cfg_offset = CNP_PMC_PM_CFG_OFFSET,
+	.pm_read_disable_bit = CNP_PMC_READ_DISABLE_BIT,
+	.lpm_num_maps = NVL_LPM_NUM_MAPS,
+	.ltr_ignore_max = LNL_NUM_IP_IGN_ALLOWED,
+	.lpm_res_counter_step_x2 = TGL_PMC_LPM_RES_COUNTER_STEP_X2,
+	.etr3_offset = ETR3_OFFSET,
+	.lpm_sts_latch_en_offset = MTL_LPM_STATUS_LATCH_EN_OFFSET,
+	.lpm_priority_offset = NVL_LPM_PRI_OFFSET,
+	.lpm_en_offset = NVL_LPM_EN_OFFSET,
+	.lpm_residency_offset = NVL_LPM_RESIDENCY_OFFSET,
+	.lpm_sts = nvl_pcdh_lpm_maps,
+	.lpm_status_offset = MTL_LPM_STATUS_OFFSET,
+	.lpm_live_status_offset = NVL_LPM_LIVE_STATUS_OFFSET,
+	.s0ix_blocker_maps = nvl_pcdh_blk_maps,
+	.s0ix_blocker_offset = LNL_S0IX_BLOCKER_OFFSET,
+	.num_s0ix_blocker = NVL_PCDH_NUM_S0IX_BLOCKER,
+	.blocker_req_offset = NVL_PCDH_BLK_REQ_OFFSET,
+	.lpm_req_guid = PCDH_LPM_REQ_GUID,
+};
+
+static const struct pmc_reg_map nvl_pcds_reg_map = {
+	.pfear_sts = ext_nvl_pcds_pfear_map,
+	.slp_s0_offset = CNP_PMC_SLP_S0_RES_COUNTER_OFFSET,
+	.slp_s0_res_counter_step = TGL_PMC_SLP_S0_RES_COUNTER_STEP,
+	.ltr_show_sts = nvl_pcds_ltr_show_map,
+	.msr_sts = msr_map,
+	.ltr_ignore_offset = CNP_PMC_LTR_IGNORE_OFFSET,
+	.regmap_length = NVL_PCDS_PMC_MMIO_REG_LEN,
+	.ppfear0_offset = CNP_PMC_HOST_PPFEAR0A,
+	.ppfear_buckets = LNL_PPFEAR_NUM_ENTRIES,
+	.pm_cfg_offset = CNP_PMC_PM_CFG_OFFSET,
+	.pm_read_disable_bit = CNP_PMC_READ_DISABLE_BIT,
+	.lpm_num_maps = PTL_LPM_NUM_MAPS,
+	.ltr_ignore_max = LNL_NUM_IP_IGN_ALLOWED,
+	.lpm_res_counter_step_x2 = TGL_PMC_LPM_RES_COUNTER_STEP_X2,
+	.etr3_offset = ETR3_OFFSET,
+	.lpm_sts_latch_en_offset = MTL_LPM_STATUS_LATCH_EN_OFFSET,
+	.lpm_priority_offset = MTL_LPM_PRI_OFFSET,
+	.lpm_en_offset = MTL_LPM_EN_OFFSET,
+	.lpm_residency_offset = MTL_LPM_RESIDENCY_OFFSET,
+	.lpm_sts = nvl_pcds_lpm_maps,
+	.lpm_status_offset = MTL_LPM_STATUS_OFFSET,
+	.lpm_live_status_offset = MTL_LPM_LIVE_STATUS_OFFSET,
+	.s0ix_blocker_maps = nvl_pcds_blk_maps,
+	.s0ix_blocker_offset = LNL_S0IX_BLOCKER_OFFSET,
+	.num_s0ix_blocker = NVL_PCDS_NUM_S0IX_BLOCKER,
+	.lpm_req_guid = PCDS_LPM_REQ_GUID,
+	.blocker_req_offset = NVL_PCDS_BLK_REQ_OFFSET,
+};
+
+static const struct pmc_reg_map nvl_pchs_reg_map = {
+	.pfear_sts = ext_nvl_pchs_pfear_map,
+	.slp_s0_offset = CNP_PMC_SLP_S0_RES_COUNTER_OFFSET,
+	.slp_s0_res_counter_step = TGL_PMC_SLP_S0_RES_COUNTER_STEP,
+	.ltr_show_sts = ptl_pcdp_ltr_show_map,
+	.msr_sts = msr_map,
+	.ltr_ignore_offset = CNP_PMC_LTR_IGNORE_OFFSET,
+	.regmap_length = NVL_PCHS_PMC_MMIO_REG_LEN,
+	.ppfear0_offset = CNP_PMC_HOST_PPFEAR0A,
+	.ppfear_buckets = LNL_PPFEAR_NUM_ENTRIES,
+	.pm_cfg_offset = CNP_PMC_PM_CFG_OFFSET,
+	.pm_read_disable_bit = CNP_PMC_READ_DISABLE_BIT,
+	.lpm_num_maps = PTL_LPM_NUM_MAPS,
+	.ltr_ignore_max = LNL_NUM_IP_IGN_ALLOWED,
+	.lpm_res_counter_step_x2 = TGL_PMC_LPM_RES_COUNTER_STEP_X2,
+	.etr3_offset = ETR3_OFFSET,
+	.lpm_sts_latch_en_offset = MTL_LPM_STATUS_LATCH_EN_OFFSET,
+	.lpm_priority_offset = MTL_LPM_PRI_OFFSET,
+	.lpm_en_offset = MTL_LPM_EN_OFFSET,
+	.lpm_residency_offset = MTL_LPM_RESIDENCY_OFFSET,
+	.lpm_sts = nvl_pchs_lpm_maps,
+	.lpm_status_offset = MTL_LPM_STATUS_OFFSET,
+	.lpm_live_status_offset = MTL_LPM_LIVE_STATUS_OFFSET,
+	.s0ix_blocker_maps = nvl_pchs_blk_maps,
+	.s0ix_blocker_offset = LNL_S0IX_BLOCKER_OFFSET,
+	.num_s0ix_blocker = NVL_PCHS_NUM_S0IX_BLOCKER,
+	.blocker_req_offset = NVL_PCHS_BLK_REQ_OFFSET,
+	.lpm_req_guid = PCHS_LPM_REQ_GUID,
+};
+
+static struct pmc_info nvl_pmc_info_list[] = {
+	{
+		.devid	= PMC_DEVID_NVL_PCDH,
+		.map	= &nvl_pcdh_reg_map,
+	},
+	{
+		.devid  = PMC_DEVID_NVL_PCDS,
+		.map    = &nvl_pcds_reg_map,
+	},
+	{
+		.devid  = PMC_DEVID_NVL_PCHS,
+		.map    = &nvl_pchs_reg_map,
+	},
+	{}
+};
+
+static const char *nvl_ltr_block_counter_arr[] = {
+	"PKGC_PREVENT_LTR_IADOMAIN",
+	"PKGC_PREVENT_LTR_GDIE",
+	"PKGC_PREVENT_LTR_PCH",
+	"PKGC_PREVENT_LTR_DISPLAY",
+	"PKGC_PREVENT_LTR_IPU",
+	NULL
+};
+
+static const char *nvl_pkgc_blocker_residency[] = {
+	"PKGC_BLOCK_RESIDENCY_INVALID",
+	"PKGC_BLOCK_RESIDENCY_MISC",
+	"PKGC_BLOCK_RESIDENCY_CDIE_MISC",
+	"PKGC_BLOCK_RESIDENCY_MEDIA_MISC",
+	"PKGC_BLOCK_RESIDENCY_GT_MISC",
+	"PKGC_BLOCK_RESIDENCY_HUBATOM_MISC",
+	"PKGC_BLOCK_RESIDENCY_IPU_BUSY",
+	"PKGC_BLOCK_RESIDENCY_IPU_LTR",
+	"PKGC_BLOCK_RESIDENCY_IPU_TIMER",
+	"PKGC_BLOCK_RESIDENCY_DISP_BUSY",
+	"PKGC_BLOCK_RESIDENCY_DISP_LTR",
+	"PKGC_BLOCK_RESIDENCY_DISP_TIMER",
+	"PKGC_BLOCK_RESIDENCY_VPU_BUSY",
+	"PKGC_BLOCK_RESIDENCY_VPU_TIMER",
+	"PKGC_BLOCK_RESIDENCY_PMC_BUSY",
+	"PKGC_BLOCK_RESIDENCY_PMC_LTR",
+	"PKGC_BLOCK_RESIDENCY_PMC_TIMER",
+	"PKGC_BLOCK_RESIDENCY_HUBATOM_ARAT",
+	"PKGC_BLOCK_RESIDENCY_CDIE0_ARAT",
+	"PKGC_BLOCK_RESIDENCY_CDIE1_ARAT",
+	"PKGC_BLOCK_RESIDENCY_GT_ARAT",
+	"PKGC_BLOCK_RESIDENCY_MEDIA_ARAT",
+	"PKGC_BLOCK_RESIDENCY_DEMOTION",
+	"PKGC_BLOCK_RESIDENCY_THERMALS",
+	"PKGC_BLOCK_RESIDENCY_SNCU",
+	"PKGC_BLOCK_RESIDENCY_SVTU",
+	"PKGC_BLOCK_RESIDENCY_IAA",
+	"PKGC_BLOCK_RESIDENCY_IOC",
+	NULL,
+};
+
+static const u8 nvl_pmc_list[] = {PMC_IDX_MAIN, PMC_IDX_PCH};
+
+#define NVL_NPU_PCI_DEV                0xd71d
+
+/*
+ * Set power state of select devices that do not have drivers to D3
+ * so that they do not block Package C entry.
+ */
+static void nvl_d3_fixup(void)
+{
+	pmc_core_set_device_d3(NVL_NPU_PCI_DEV);
+}
+
+static int nvl_resume(struct pmc_dev *pmcdev)
+{
+	nvl_d3_fixup();
+	return cnl_resume(pmcdev);
+}
+
+static int nvl_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_info)
+{
+	nvl_d3_fixup();
+	return generic_core_init(pmcdev, pmc_dev_info);
+}
+
+static u32 nvl_pmt_dmu_guids[] = {NVL_PMT_DMU_GUID, 0x0};
+struct pmc_dev_info nvl_s_pmc_dev = {
+	.num_pmcs = ARRAY_SIZE(nvl_pmc_list),
+	.pmc_list = nvl_pmc_list,
+	.regmap_list = nvl_pmc_info_list,
+	.map = &nvl_pcds_reg_map,
+	.sub_req_show = &pmc_core_substate_blk_req_fops,
+	.suspend = cnl_suspend,
+	.resume = nvl_resume,
+	.init = nvl_core_init,
+	.sub_req = pmc_core_pmt_get_blk_sub_req,
+	.dmu_guids = nvl_pmt_dmu_guids,
+	.pc_guid = NVL_PMT_PC_GUID,
+	.pkgc_ltr_blocker_offset = NVL_LTR_BLK_OFFSET,
+	.pkgc_ltr_blocker_counters = nvl_ltr_block_counter_arr,
+	.pkgc_blocker_offset = NVL_PKGC_BLK_OFFSET,
+	.pkgc_blocker_counters = nvl_pkgc_blocker_residency,
+	.ssram_hidden = false,
+	.die_c6_offset = NVL_PMT_DMU_DIE_C6_OFFSET,
+};
+
+struct pmc_dev_info nvl_h_pmc_dev = {
+	.num_pmcs = ARRAY_SIZE(nvl_pmc_list),
+	.pmc_list = nvl_pmc_list,
+	.regmap_list = nvl_pmc_info_list,
+	.map = &nvl_pcdh_reg_map,
+	.sub_req_show = &pmc_core_substate_blk_req_fops,
+	.suspend = cnl_suspend,
+	.resume = nvl_resume,
+	.init = nvl_core_init,
+	.sub_req = pmc_core_pmt_get_blk_sub_req,
+	.dmu_guids = nvl_pmt_dmu_guids,
+	.pc_guid = NVL_PMT_PC_GUID,
+	.pkgc_ltr_blocker_offset = NVL_LTR_BLK_OFFSET,
+	.pkgc_ltr_blocker_counters = nvl_ltr_block_counter_arr,
+	.pkgc_blocker_offset = NVL_PKGC_BLK_OFFSET,
+	.pkgc_blocker_counters = nvl_pkgc_blocker_residency,
+	.ssram_hidden = false,
+	.die_c6_offset = NVL_PMT_DMU_DIE_C6_OFFSET,
+};
diff --git a/drivers/platform/x86/intel/pmc/ptl.c b/drivers/platform/x86/intel/pmc/ptl.c
index 1f48e2bbc699f..a7d5d4f77934f 100644
--- a/drivers/platform/x86/intel/pmc/ptl.c
+++ b/drivers/platform/x86/intel/pmc/ptl.c
@@ -137,7 +137,7 @@ static const struct pmc_bit_map *ext_ptl_pcdp_pfear_map[] = {
 	NULL
 };
 
-static const struct pmc_bit_map ptl_pcdp_ltr_show_map[] = {
+const struct pmc_bit_map ptl_pcdp_ltr_show_map[] = {
 	{"SOUTHPORT_A",		CNP_PMC_LTR_SPA},
 	{"SOUTHPORT_B",		CNP_PMC_LTR_SPB},
 	{"SATA",		CNP_PMC_LTR_SATA},
@@ -543,6 +543,8 @@ static struct pmc_info ptl_pmc_info_list[] = {
 	{}
 };
 
+static const u8 ptl_pmc_list[] = {PMC_IDX_MAIN};
+
 #define PTL_NPU_PCI_DEV                0xb03e
 #define PTL_IPU_PCI_DEV                0xb05d
 
@@ -569,7 +571,8 @@ static int ptl_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_in
 }
 
 struct pmc_dev_info ptl_pmc_dev = {
-	.pci_func = 2,
+	.num_pmcs = ARRAY_SIZE(ptl_pmc_list),
+	.pmc_list = ptl_pmc_list,
 	.regmap_list = ptl_pmc_info_list,
 	.map = &ptl_pcdp_reg_map,
 	.sub_req_show = &pmc_core_substate_blk_req_fops,
@@ -577,4 +580,5 @@ struct pmc_dev_info ptl_pmc_dev = {
 	.resume = ptl_resume,
 	.init = ptl_core_init,
 	.sub_req = pmc_core_pmt_get_blk_sub_req,
+	.ssram_hidden = true,
 };
diff --git a/drivers/platform/x86/intel/pmc/wcl.c b/drivers/platform/x86/intel/pmc/wcl.c
index a45707e6364f2..d34ac916907dc 100644
--- a/drivers/platform/x86/intel/pmc/wcl.c
+++ b/drivers/platform/x86/intel/pmc/wcl.c
@@ -469,6 +469,8 @@ static struct pmc_info wcl_pmc_info_list[] = {
 	{}
 };
 
+static const u8 wcl_pmc_list[] = {PMC_IDX_MAIN};
+
 #define WCL_NPU_PCI_DEV                0xfd3e
 
 /*
@@ -493,12 +495,14 @@ static int wcl_core_init(struct pmc_dev *pmcdev, struct pmc_dev_info *pmc_dev_in
 }
 
 struct pmc_dev_info wcl_pmc_dev = {
-	.pci_func = 2,
 	.regmap_list = wcl_pmc_info_list,
+	.num_pmcs = ARRAY_SIZE(wcl_pmc_list),
+	.pmc_list = wcl_pmc_list,
 	.map = &wcl_pcdn_reg_map,
 	.sub_req_show = &pmc_core_substate_blk_req_fops,
 	.suspend = cnl_suspend,
 	.resume = wcl_resume,
 	.init = wcl_core_init,
 	.sub_req = pmc_core_pmt_get_blk_sub_req,
+	.ssram_hidden = true,
 };
diff --git a/drivers/platform/x86/intel/rst.c b/drivers/platform/x86/intel/rst.c
index 4bd10927aad99..bb19f0d893050 100644
--- a/drivers/platform/x86/intel/rst.c
+++ b/drivers/platform/x86/intel/rst.c
@@ -102,9 +102,13 @@ static struct device_attribute irst_timeout_attr = {
 
 static int irst_probe(struct platform_device *pdev)
 {
-	struct acpi_device *acpi = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *acpi;
 	int error;
 
+	acpi = ACPI_COMPANION(&pdev->dev);
+	if (!acpi)
+		return -ENODEV;
+
 	error = device_create_file(&acpi->dev, &irst_timeout_attr);
 	if (unlikely(error))
 		return error;
diff --git a/drivers/platform/x86/intel/smartconnect.c b/drivers/platform/x86/intel/smartconnect.c
index 4d866b6366d69..71e91ac60e5da 100644
--- a/drivers/platform/x86/intel/smartconnect.c
+++ b/drivers/platform/x86/intel/smartconnect.c
@@ -12,10 +12,14 @@ MODULE_LICENSE("GPL");
 
 static int smartconnect_acpi_probe(struct platform_device *pdev)
 {
-	acpi_handle handle = ACPI_HANDLE(&pdev->dev);
 	unsigned long long value;
+	acpi_handle handle;
 	acpi_status status;
 
+	handle = ACPI_HANDLE(&pdev->dev);
+	if (!handle)
+		return -ENODEV;
+
 	status = acpi_evaluate_integer(handle, "GAOS", NULL, &value);
 	if (ACPI_FAILURE(status))
 		return -EINVAL;
diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
index 7070c94324e0e..3b554418a7a3c 100644
--- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
+++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.c
@@ -29,6 +29,13 @@ static ssize_t show_domain_id(struct kobject *kobj, struct kobj_attribute *attr,
 	return sysfs_emit(buf, "%u\n", data->domain_id);
 }
 
+static ssize_t show_instance_id(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
+{
+	struct uncore_data *data = container_of(attr, struct uncore_data, instance_id_kobj_attr);
+
+	return sysfs_emit(buf, "%u\n", data->instance_id);
+}
+
 static ssize_t show_fabric_cluster_id(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
 {
 	struct uncore_data *data = container_of(attr, struct uncore_data, fabric_cluster_id_kobj_attr);
@@ -200,6 +207,9 @@ static int create_attr_group(struct uncore_data *data, char *name)
 	if (data->domain_id != UNCORE_DOMAIN_ID_INVALID) {
 		init_attribute_root_ro(domain_id);
 		data->uncore_attrs[index++] = &data->domain_id_kobj_attr.attr;
+		init_attribute_root_ro(instance_id);
+		data->uncore_attrs[index++] = &data->instance_id_kobj_attr.attr;
+
 		init_attribute_root_ro(fabric_cluster_id);
 		data->uncore_attrs[index++] = &data->fabric_cluster_id_kobj_attr.attr;
 		init_attribute_root_ro(package_id);
@@ -268,7 +278,7 @@ int uncore_freq_add_entry(struct uncore_data *data, int cpu)
 		if (ret < 0)
 			goto uncore_unlock;
 
-		data->instance_id = ret;
+		data->seqnum_id = ret;
 		scnprintf(data->name, sizeof(data->name), "uncore%02d", ret);
 	} else {
 		scnprintf(data->name, sizeof(data->name), "package_%02d_die_%02d",
@@ -281,7 +291,7 @@ int uncore_freq_add_entry(struct uncore_data *data, int cpu)
 	ret = create_attr_group(data, data->name);
 	if (ret) {
 		if (data->domain_id != UNCORE_DOMAIN_ID_INVALID)
-			ida_free(&intel_uncore_ida, data->instance_id);
+			ida_free(&intel_uncore_ida, data->seqnum_id);
 	} else {
 		data->control_cpu = cpu;
 		data->valid = true;
@@ -301,7 +311,7 @@ void uncore_freq_remove_die_entry(struct uncore_data *data)
 	data->control_cpu = -1;
 	data->valid = false;
 	if (data->domain_id != UNCORE_DOMAIN_ID_INVALID)
-		ida_free(&intel_uncore_ida, data->instance_id);
+		ida_free(&intel_uncore_ida, data->seqnum_id);
 
 	mutex_unlock(&uncore_lock);
 }
diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.h b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.h
index 0abe850ef54ea..e319448dc1a44 100644
--- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.h
+++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-common.h
@@ -35,7 +35,8 @@
  * @die_id:		Die id for this instance
  * @domain_id:		Power domain id for this instance
  * @cluster_id:		cluster id in a domain
- * @instance_id:	Unique instance id to append to directory name
+ * @seqnum_id:		Unique sequential id to append to directory name
+ * @instance_id:	Die indices or feature instances for a single TPMI device
  * @name:		Sysfs entry name for this instance
  * @agent_type_mask:	Bit mask of all hardware agents for this domain
  * @uncore_attr_group:	Attribute group storage
@@ -56,6 +57,7 @@
  * @elc_floor_freq_khz_kobj_attr: Storage for kobject attribute elc_floor_freq_khz
  * @agent_types_kobj_attr: Storage for kobject attribute agent_type
  * @die_id_kobj_attr:	Attribute storage for die_id information
+ * @instance_id_kobj_attr: Attribute storage for instance_id value
  * @uncore_attrs:	Attribute storage for group creation
  *
  * This structure is used to encapsulate all data related to uncore sysfs
@@ -71,6 +73,7 @@ struct uncore_data {
 	int die_id;
 	int domain_id;
 	int cluster_id;
+	int seqnum_id;
 	int instance_id;
 	char name[32];
 	u16  agent_type_mask;
@@ -90,7 +93,8 @@ struct uncore_data {
 	struct kobj_attribute elc_floor_freq_khz_kobj_attr;
 	struct kobj_attribute agent_types_kobj_attr;
 	struct kobj_attribute die_id_kobj_attr;
-	struct attribute *uncore_attrs[15];
+	struct kobj_attribute instance_id_kobj_attr;
+	struct attribute *uncore_attrs[16];
 };
 
 #define UNCORE_DOMAIN_ID_INVALID	-1
diff --git a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c
index 88015a2c6a0d1..44e2a90004df3 100644
--- a/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c
+++ b/drivers/platform/x86/intel/uncore-frequency/uncore-frequency-tpmi.c
@@ -385,7 +385,19 @@ static u8 io_die_index_next;
 /* Lock to protect io_die_start, io_die_index_next */
 static DEFINE_MUTEX(domain_lock);
 
-static void set_domain_id(int id,  int num_resources,
+static inline void set_instance_id(int id, struct tpmi_uncore_cluster_info *cluster_info)
+{
+	/*
+	 * On non-partitioned systems domain_id can be used for mapping both
+	 * CPUs to compute die IDs and physical die indexes to MMIO mapped
+	 * memory. However on partitioned systems domain_id loses the second
+	 * association. Therefore instance_id should be used for that instead,
+	 * while domain_id should still be used to match CPUs to compute dies.
+	 */
+	cluster_info->uncore_data.instance_id = id;
+}
+
+static void set_domain_id(int id, int num_resources,
 			  struct oobmsm_plat_info *plat_info,
 			  struct tpmi_uncore_cluster_info *cluster_info)
 {
@@ -690,6 +702,7 @@ static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_
 			set_cdie_id(i, cluster_info, plat_info);
 
 			set_domain_id(i, num_resources, plat_info, cluster_info);
+			set_instance_id(i, cluster_info);
 
 			cluster_info->uncore_root = tpmi_uncore;
 
diff --git a/drivers/platform/x86/intel/vbtn.c b/drivers/platform/x86/intel/vbtn.c
index 9ca87e7075822..874023c38fd15 100644
--- a/drivers/platform/x86/intel/vbtn.c
+++ b/drivers/platform/x86/intel/vbtn.c
@@ -275,12 +275,16 @@ static bool intel_vbtn_has_switches(acpi_handle handle, bool dual_accel)
 
 static int intel_vbtn_probe(struct platform_device *device)
 {
-	acpi_handle handle = ACPI_HANDLE(&device->dev);
 	bool dual_accel, has_buttons, has_switches;
 	struct intel_vbtn_priv *priv;
+	acpi_handle handle;
 	acpi_status status;
 	int err;
 
+	handle = ACPI_HANDLE(&device->dev);
+	if (!handle)
+		return -ENODEV;
+
 	dual_accel = dual_accel_detect();
 	has_buttons = acpi_has_method(handle, "VBDL");
 	has_switches = intel_vbtn_has_switches(handle, dual_accel);
diff --git a/drivers/platform/x86/intel/vsec.c b/drivers/platform/x86/intel/vsec.c
index 7d5dbc1c1d05a..1657834fd2759 100644
--- a/drivers/platform/x86/intel/vsec.c
+++ b/drivers/platform/x86/intel/vsec.c
@@ -112,7 +112,6 @@ static void intel_vsec_dev_release(struct device *dev)
 	ida_free(intel_vsec_dev->ida, intel_vsec_dev->auxdev.id);
 
 	kfree(intel_vsec_dev->acpi_disc);
-	kfree(intel_vsec_dev->resource);
 	kfree(intel_vsec_dev);
 }
 
@@ -225,7 +224,6 @@ int intel_vsec_add_aux(struct device *parent,
 	ret = xa_alloc(&auxdev_array, &intel_vsec_dev->id, intel_vsec_dev,
 		       PMT_XA_LIMIT, GFP_KERNEL);
 	if (ret < 0) {
-		kfree(intel_vsec_dev->resource);
 		kfree(intel_vsec_dev);
 		return ret;
 	}
@@ -233,7 +231,6 @@ int intel_vsec_add_aux(struct device *parent,
 	id = ida_alloc(intel_vsec_dev->ida, GFP_KERNEL);
 	if (id < 0) {
 		xa_erase(&auxdev_array, intel_vsec_dev->id);
-		kfree(intel_vsec_dev->resource);
 		kfree(intel_vsec_dev);
 		return id;
 	}
@@ -282,7 +279,7 @@ static int intel_vsec_add_dev(struct device *dev, struct intel_vsec_header *head
 			      unsigned long cap_id, u64 base_addr)
 {
 	struct intel_vsec_device __free(kfree) *intel_vsec_dev = NULL;
-	struct resource __free(kfree) *res = NULL;
+	struct resource *res;
 	struct resource *tmp;
 	struct device *parent;
 	unsigned long quirks = info->quirks;
@@ -306,13 +303,12 @@ static int intel_vsec_add_dev(struct device *dev, struct intel_vsec_header *head
 		return -EINVAL;
 	}
 
-	intel_vsec_dev = kzalloc_obj(*intel_vsec_dev);
+	intel_vsec_dev = kzalloc_flex(*intel_vsec_dev, resource, header->num_entries);
 	if (!intel_vsec_dev)
 		return -ENOMEM;
 
-	res = kzalloc_objs(*res, header->num_entries);
-	if (!res)
-		return -ENOMEM;
+	intel_vsec_dev->num_resources = header->num_entries;
+	res = intel_vsec_dev->resource;
 
 	if (quirks & VSEC_QUIRK_TABLE_SHIFT)
 		header->offset >>= TABLE_OFFSET_SHIFT;
@@ -342,8 +338,6 @@ static int intel_vsec_add_dev(struct device *dev, struct intel_vsec_header *head
 	}
 
 	intel_vsec_dev->dev = dev;
-	intel_vsec_dev->resource = no_free_ptr(res);
-	intel_vsec_dev->num_resources = header->num_entries;
 	intel_vsec_dev->quirks = info->quirks;
 	intel_vsec_dev->base_addr = info->base_addr;
 	intel_vsec_dev->priv_data = info->priv_data;
@@ -649,29 +643,13 @@ static void intel_vsec_skip_missing_dependencies(struct pci_dev *pdev)
 	}
 }
 
-static int intel_vsec_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+static int intel_vsec_pci_init(struct pci_dev *pdev)
 {
-	const struct intel_vsec_platform_info *info;
-	struct vsec_priv *priv;
-	int num_caps, ret;
+	struct vsec_priv *priv = pci_get_drvdata(pdev);
+	const struct intel_vsec_platform_info *info = priv->info;
 	int run_once = 0;
 	bool found_any = false;
-
-	ret = pcim_enable_device(pdev);
-	if (ret)
-		return ret;
-
-	pci_save_state(pdev);
-	info = (const struct intel_vsec_platform_info *)id->driver_data;
-	if (!info)
-		return -EINVAL;
-
-	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
-	if (!priv)
-		return -ENOMEM;
-
-	priv->info = info;
-	pci_set_drvdata(pdev, priv);
+	int num_caps;
 
 	num_caps = hweight_long(info->caps);
 	while (num_caps--) {
@@ -692,6 +670,31 @@ static int intel_vsec_pci_probe(struct pci_dev *pdev, const struct pci_device_id
 	return 0;
 }
 
+static int intel_vsec_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	const struct intel_vsec_platform_info *info;
+	struct vsec_priv *priv;
+	int ret;
+
+	ret = pcim_enable_device(pdev);
+	if (ret)
+		return ret;
+
+	pci_save_state(pdev);
+	info = (const struct intel_vsec_platform_info *)id->driver_data;
+	if (!info)
+		return -EINVAL;
+
+	priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->info = info;
+	pci_set_drvdata(pdev, priv);
+
+	return intel_vsec_pci_init(pdev);
+}
+
 int intel_vsec_set_mapping(struct oobmsm_plat_info *plat_info,
 			   struct intel_vsec_device *vsec_dev)
 {
@@ -832,7 +835,6 @@ static pci_ers_result_t intel_vsec_pci_slot_reset(struct pci_dev *pdev)
 {
 	struct intel_vsec_device *intel_vsec_dev;
 	pci_ers_result_t status = PCI_ERS_RESULT_DISCONNECT;
-	const struct pci_device_id *pci_dev_id;
 	unsigned long index;
 
 	dev_info(&pdev->dev, "Resetting PCI slot\n");
@@ -853,10 +855,8 @@ static pci_ers_result_t intel_vsec_pci_slot_reset(struct pci_dev *pdev)
 		devm_release_action(&pdev->dev, intel_vsec_remove_aux,
 				    &intel_vsec_dev->auxdev);
 	}
-	pci_disable_device(pdev);
 	pci_restore_state(pdev);
-	pci_dev_id = pci_match_id(intel_vsec_pci_ids, pdev);
-	intel_vsec_pci_probe(pdev, pci_dev_id);
+	intel_vsec_pci_init(pdev);
 
 out:
 	return status;
diff --git a/drivers/platform/x86/intel/vsec_tpmi.c b/drivers/platform/x86/intel/vsec_tpmi.c
index 16fd7aa41f20e..8c4e2474b30cd 100644
--- a/drivers/platform/x86/intel/vsec_tpmi.c
+++ b/drivers/platform/x86/intel/vsec_tpmi.c
@@ -641,15 +641,12 @@ static int tpmi_create_device(struct intel_tpmi_info *tpmi_info,
 	if (!name)
 		return -EOPNOTSUPP;
 
-	res = kzalloc_objs(*res, pfs->pfs_header.num_entries);
-	if (!res)
+	feature_vsec_dev = kzalloc_flex(*feature_vsec_dev, resource, pfs->pfs_header.num_entries);
+	if (!feature_vsec_dev)
 		return -ENOMEM;
 
-	feature_vsec_dev = kzalloc_obj(*feature_vsec_dev);
-	if (!feature_vsec_dev) {
-		kfree(res);
-		return -ENOMEM;
-	}
+	feature_vsec_dev->num_resources = pfs->pfs_header.num_entries;
+	res = feature_vsec_dev->resource;
 
 	snprintf(feature_id_name, sizeof(feature_id_name), "tpmi-%s", name);
 
@@ -662,8 +659,6 @@ static int tpmi_create_device(struct intel_tpmi_info *tpmi_info,
 	}
 
 	feature_vsec_dev->dev = vsec_dev->dev;
-	feature_vsec_dev->resource = res;
-	feature_vsec_dev->num_resources = pfs->pfs_header.num_entries;
 	feature_vsec_dev->priv_data = &tpmi_info->plat_info;
 	feature_vsec_dev->priv_data_size = sizeof(tpmi_info->plat_info);
 	feature_vsec_dev->ida = &intel_vsec_tpmi_ida;
diff --git a/drivers/platform/x86/lenovo/thinkpad_acpi.c b/drivers/platform/x86/lenovo/thinkpad_acpi.c
index e1cee42a1683e..445e1403308ee 100644
--- a/drivers/platform/x86/lenovo/thinkpad_acpi.c
+++ b/drivers/platform/x86/lenovo/thinkpad_acpi.c
@@ -38,6 +38,7 @@
 #include <linux/backlight.h>
 #include <linux/bitfield.h>
 #include <linux/bitops.h>
+#include <linux/debugfs.h>
 #include <linux/delay.h>
 #include <linux/dmi.h>
 #include <linux/freezer.h>
@@ -2469,7 +2470,7 @@ static int hotkey_kthread(void *data)
 	bool was_frozen;
 
 	if (tpacpi_lifecycle == TPACPI_LIFE_EXITING)
-		goto exit;
+		return 0;
 
 	set_freezable();
 
@@ -2526,7 +2527,6 @@ static int hotkey_kthread(void *data)
 		si ^= 1;
 	}
 
-exit:
 	return 0;
 }
 
@@ -11217,6 +11217,26 @@ static ssize_t hwdd_status_show(struct device *dev,
 
 	return sysfs_emit(buf, "0\n");
 }
+
+/* sysfs type-c damage detection raw data - accessed via debugfs*/
+static int hwdd_raw_show(struct seq_file *m, void *v)
+{
+	unsigned int damage_status;
+	int err;
+
+	if (!ucdd_supported)
+		return -ENODEV;
+
+	/* Get USB TYPE-C damage status */
+	err = hwdd_command(HWDD_GET_DMG_USBC, &damage_status);
+	if (err)
+		return err;
+
+	seq_printf(m, "HWDD: 0x%x\n", damage_status);
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(hwdd_raw);
+
 static DEVICE_ATTR_RO(hwdd_status);
 static DEVICE_ATTR_RO(hwdd_detail);
 
@@ -11504,6 +11524,20 @@ static const char * __init str_supported(int is_supported)
 }
 #endif /* CONFIG_THINKPAD_ACPI_DEBUG */
 
+static struct dentry *tpacpi_dbg;
+static void tpacpi_debugfs_init(void)
+{
+	tpacpi_dbg = debugfs_create_dir("tpacpi", NULL);
+
+	/* HWDD raw data */
+	debugfs_create_file("hwdd-raw", 0444, tpacpi_dbg, NULL, &hwdd_raw_fops);
+}
+
+static void tpacpi_debugfs_remove(void)
+{
+	debugfs_remove_recursive(tpacpi_dbg);
+}
+
 static void ibm_exit(struct ibm_struct *ibm)
 {
 	dbg_printk(TPACPI_DBG_EXIT, "removing %s\n", ibm->name);
@@ -12068,6 +12102,8 @@ static void thinkpad_acpi_module_exit(void)
 		remove_proc_entry(TPACPI_PROC_DIR, acpi_root_dir);
 	if (tpacpi_wq)
 		destroy_workqueue(tpacpi_wq);
+	if (tpacpi_dbg)
+		tpacpi_debugfs_remove();
 
 	kfree(thinkpad_id.bios_version_str);
 	kfree(thinkpad_id.ec_version_str);
@@ -12198,6 +12234,8 @@ static int __init thinkpad_acpi_module_init(void)
 		return -ENODEV;
 	}
 
+	tpacpi_debugfs_init();
+
 	dmi_id = dmi_first_match(fwbug_list);
 	if (dmi_id)
 		tp_features.quirks = dmi_id->driver_data;
diff --git a/drivers/platform/x86/lg-laptop.c b/drivers/platform/x86/lg-laptop.c
index 9681412d694be..a8f2f465ef3f9 100644
--- a/drivers/platform/x86/lg-laptop.c
+++ b/drivers/platform/x86/lg-laptop.c
@@ -761,12 +761,11 @@ static void lg_laptop_remove_address_space_handler(void *data)
 
 static int acpi_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct platform_device_info pdev_info = {
-		.fwnode = acpi_fwnode_handle(device),
 		.name = PLATFORM_NAME,
 		.id = PLATFORM_DEVID_NONE,
 	};
+	struct acpi_device *device;
 	acpi_status status;
 	int ret;
 	const char *product;
@@ -775,6 +774,12 @@ static int acpi_probe(struct platform_device *pdev)
 	if (pf_device)
 		return 0;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
+	pdev_info.fwnode = acpi_fwnode_handle(device),
+
 	status = acpi_install_address_space_handler(device->handle, LG_ADDRESS_SPACE_ID,
 						    &lg_laptop_address_space_handler,
 						    NULL, &pdev->dev);
diff --git a/drivers/platform/x86/meraki-mx100.c b/drivers/platform/x86/meraki-mx100.c
index 8c5276d985123..9f4caa1f3a923 100644
--- a/drivers/platform/x86/meraki-mx100.c
+++ b/drivers/platform/x86/meraki-mx100.c
@@ -20,16 +20,11 @@
 #include <linux/input-event-codes.h>
 #include <linux/io.h>
 #include <linux/kernel.h>
+#include <linux/mfd/lpc_ich.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/property.h>
 
-#define TINK_GPIO_DRIVER_NAME "gpio_ich"
-
-static const struct software_node gpio_ich_node = {
-	.name = TINK_GPIO_DRIVER_NAME,
-};
-
 /* LEDs */
 static const struct software_node tink_gpio_leds_node = {
 	.name = "meraki-mx100-leds",
@@ -38,7 +33,7 @@ static const struct software_node tink_gpio_leds_node = {
 static const struct property_entry tink_internet_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:internet"),
 	PROPERTY_ENTRY_STRING("linux,default-trigger", "default-on"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 11, GPIO_ACTIVE_LOW),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 11, GPIO_ACTIVE_LOW),
 	{ }
 };
 
@@ -50,7 +45,7 @@ static const struct software_node tink_internet_led_node = {
 
 static const struct property_entry tink_lan2_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan2"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 18, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 18, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -62,7 +57,7 @@ static const struct software_node tink_lan2_led_node = {
 
 static const struct property_entry tink_lan3_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan3"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 20, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 20, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -74,7 +69,7 @@ static const struct software_node tink_lan3_led_node = {
 
 static const struct property_entry tink_lan4_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan4"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 22, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 22, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -86,7 +81,7 @@ static const struct software_node tink_lan4_led_node = {
 
 static const struct property_entry tink_lan5_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan5"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 23, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 23, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -98,7 +93,7 @@ static const struct software_node tink_lan5_led_node = {
 
 static const struct property_entry tink_lan6_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan6"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 32, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 32, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -110,7 +105,7 @@ static const struct software_node tink_lan6_led_node = {
 
 static const struct property_entry tink_lan7_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan7"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 34, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 34, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -122,7 +117,7 @@ static const struct software_node tink_lan7_led_node = {
 
 static const struct property_entry tink_lan8_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan8"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 35, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 35, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -134,7 +129,7 @@ static const struct software_node tink_lan8_led_node = {
 
 static const struct property_entry tink_lan9_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan9"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 36, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 36, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -146,7 +141,7 @@ static const struct software_node tink_lan9_led_node = {
 
 static const struct property_entry tink_lan10_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan10"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 37, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 37, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -158,7 +153,7 @@ static const struct software_node tink_lan10_led_node = {
 
 static const struct property_entry tink_lan11_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:lan11"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 48, GPIO_ACTIVE_HIGH),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 48, GPIO_ACTIVE_HIGH),
 	{ }
 };
 
@@ -170,7 +165,7 @@ static const struct software_node tink_lan11_led_node = {
 
 static const struct property_entry tink_ha_green_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:ha"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 16, GPIO_ACTIVE_LOW),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 16, GPIO_ACTIVE_LOW),
 	{ }
 };
 
@@ -182,7 +177,7 @@ static const struct software_node tink_ha_green_led_node = {
 
 static const struct property_entry tink_ha_orange_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:orange:ha"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 7, GPIO_ACTIVE_LOW),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 7, GPIO_ACTIVE_LOW),
 	{ }
 };
 
@@ -194,7 +189,7 @@ static const struct software_node tink_ha_orange_led_node = {
 
 static const struct property_entry tink_usb_green_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:green:usb"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 21, GPIO_ACTIVE_LOW),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 21, GPIO_ACTIVE_LOW),
 	{ }
 };
 
@@ -206,7 +201,7 @@ static const struct software_node tink_usb_green_led_node = {
 
 static const struct property_entry tink_usb_orange_led_props[] = {
 	PROPERTY_ENTRY_STRING("label", "mx100:orange:usb"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 19, GPIO_ACTIVE_LOW),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 19, GPIO_ACTIVE_LOW),
 	{ }
 };
 
@@ -230,7 +225,7 @@ static const struct software_node tink_gpio_keys_node = {
 static const struct property_entry tink_reset_key_props[] = {
 	PROPERTY_ENTRY_U32("linux,code", KEY_RESTART),
 	PROPERTY_ENTRY_STRING("label", "Reset"),
-	PROPERTY_ENTRY_GPIO("gpios", &gpio_ich_node, 60, GPIO_ACTIVE_LOW),
+	PROPERTY_ENTRY_GPIO("gpios", &lpc_ich_gpio_swnode, 60, GPIO_ACTIVE_LOW),
 	PROPERTY_ENTRY_U32("linux,input-type", EV_KEY),
 	PROPERTY_ENTRY_U32("debounce-interval", 100),
 	{ }
@@ -243,7 +238,6 @@ static const struct software_node tink_reset_key_node = {
 };
 
 static const struct software_node *tink_swnodes[] = {
-	&gpio_ich_node,
 	/* LEDs nodes */
 	&tink_gpio_leds_node,
 	&tink_internet_led_node,
@@ -348,3 +342,4 @@ MODULE_AUTHOR("Chris Blake <chrisrblake93@gmail.com>");
 MODULE_DESCRIPTION("Cisco Meraki MX100 Platform Driver");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:meraki-mx100");
+MODULE_IMPORT_NS("LPC_ICH");
diff --git a/drivers/platform/x86/panasonic-laptop.c b/drivers/platform/x86/panasonic-laptop.c
index 1337f7c49805e..719add753cb3d 100644
--- a/drivers/platform/x86/panasonic-laptop.c
+++ b/drivers/platform/x86/panasonic-laptop.c
@@ -246,11 +246,11 @@ struct pcc_acpi {
 	int			ac_brightness;
 	int			dc_brightness;
 	int			current_brightness;
-	u32			*sinf;
 	struct acpi_device	*device;
 	struct input_dev	*input_dev;
 	struct backlight_device	*backlight;
 	struct platform_device	*platform;
+	u32			sinf[] __counted_by(num_sifr);
 };
 
 /*
@@ -981,11 +981,15 @@ static int acpi_pcc_hotkey_resume(struct device *dev)
 
 static int acpi_pcc_hotkey_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct backlight_properties props;
+	struct acpi_device *device;
 	struct pcc_acpi *pcc;
 	int num_sifr, result;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	num_sifr = acpi_pcc_get_sqty(device);
 
 	/*
@@ -1003,21 +1007,15 @@ static int acpi_pcc_hotkey_probe(struct platform_device *pdev)
 	 */
 	num_sifr++;
 
-	pcc = kzalloc_obj(struct pcc_acpi);
+	pcc = kzalloc_flex(*pcc, sinf, num_sifr);
 	if (!pcc) {
 		pr_err("Couldn't allocate mem for pcc");
 		return -ENOMEM;
 	}
 
-	pcc->sinf = kcalloc(num_sifr + 1, sizeof(u32), GFP_KERNEL);
-	if (!pcc->sinf) {
-		result = -ENOMEM;
-		goto out_hotkey;
-	}
-
+	pcc->num_sifr = num_sifr;
 	pcc->device = device;
 	pcc->handle = device->handle;
-	pcc->num_sifr = num_sifr;
 	device->driver_data = pcc;
 	strscpy(acpi_device_name(device), ACPI_PCC_DEVICE_NAME);
 	strscpy(acpi_device_class(device), ACPI_PCC_CLASS);
@@ -1025,7 +1023,7 @@ static int acpi_pcc_hotkey_probe(struct platform_device *pdev)
 	result = acpi_pcc_init_input(pcc);
 	if (result) {
 		pr_err("Error installing keyinput handler\n");
-		goto out_sinf;
+		goto out_hotkey;
 	}
 
 	if (!acpi_pcc_retrieve_biosdata(pcc)) {
@@ -1106,10 +1104,8 @@ static int acpi_pcc_hotkey_probe(struct platform_device *pdev)
 	backlight_device_unregister(pcc->backlight);
 out_input:
 	input_unregister_device(pcc->input_dev);
-out_sinf:
-	device->driver_data = NULL;
-	kfree(pcc->sinf);
 out_hotkey:
+	device->driver_data = NULL;
 	kfree(pcc);
 
 	return result;
@@ -1139,7 +1135,6 @@ static void acpi_pcc_hotkey_remove(struct platform_device *pdev)
 
 	device->driver_data = NULL;
 
-	kfree(pcc->sinf);
 	kfree(pcc);
 }
 
diff --git a/drivers/platform/x86/pcengines-apuv2.c b/drivers/platform/x86/pcengines-apuv2.c
index 3f19589d1ba03..8ac5f719c5a39 100644
--- a/drivers/platform/x86/pcengines-apuv2.c
+++ b/drivers/platform/x86/pcengines-apuv2.c
@@ -262,7 +262,7 @@ static struct platform_device * __init apu_create_pdev(const char *name,
 		.id = PLATFORM_DEVID_NONE,
 		.data = data,
 		.size_data = size,
-		.fwnode = software_node_fwnode(swnode),
+		.swnode = swnode,
 	};
 	struct platform_device *pdev;
 	int err;
diff --git a/drivers/platform/x86/pmc_atom.c b/drivers/platform/x86/pmc_atom.c
index 48c2a0e59d185..046933f6d1b6f 100644
--- a/drivers/platform/x86/pmc_atom.c
+++ b/drivers/platform/x86/pmc_atom.c
@@ -570,8 +570,8 @@ static int pmc_setup_dev(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 /* Data for PCI driver interface used by pci_match_id() call below */
 static const struct pci_device_id pmc_pci_ids[] = {
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_VLV_PMC), (kernel_ulong_t)&byt_data },
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_CHT_PMC), (kernel_ulong_t)&cht_data },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_VLV_PMC), .driver_data = (kernel_ulong_t)&byt_data },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_CHT_PMC), .driver_data = (kernel_ulong_t)&cht_data },
 	{}
 };
 
diff --git a/drivers/platform/x86/sel3350-platform.c b/drivers/platform/x86/sel3350-platform.c
index 02e2081e2333b..f3a3142356325 100644
--- a/drivers/platform/x86/sel3350-platform.c
+++ b/drivers/platform/x86/sel3350-platform.c
@@ -9,6 +9,8 @@
  */
 
 #include <linux/acpi.h>
+#include <linux/array_size.h>
+#include <linux/err.h>
 #include <linux/gpio/consumer.h>
 #include <linux/gpio/machine.h>
 #include <linux/leds.h>
@@ -30,19 +32,82 @@
 #define SEL_PS_B_DETECT "sel_ps_b_detect"
 #define SEL_PS_B_GOOD   "sel_ps_b_good"
 
+#define AUX_LED_GRN1        "sel_aux_led_grn1"
+#define AUX_LED_GRN2        "sel_aux_led_grn2"
+#define AUX_LED_GRN3        "sel_aux_led_grn3"
+#define AUX_LED_GRN4        "sel_aux_led_grn4"
+#define ALARM_STATE_USER    "sel_alarm_state_user"
+#define ENABLE_STATE_USER   "sel_enable_state_user"
+#define AUX_LED_RED1        "sel_aux_led_red1"
+#define AUX_LED_RED2        "sel_aux_led_red2"
+#define AUX_LED_RED3        "sel_aux_led_red3"
+#define AUX_LED_RED4        "sel_aux_led_red4"
+
+static const char *const sel3350_leds_gpio_names[] = {
+	AUX_LED_GRN1,
+	AUX_LED_GRN2,
+	AUX_LED_GRN3,
+	AUX_LED_GRN4,
+	ALARM_STATE_USER,
+	ENABLE_STATE_USER,
+	AUX_LED_RED1,
+	AUX_LED_RED2,
+	AUX_LED_RED3,
+	AUX_LED_RED4,
+};
+
 /* LEDs */
-static const struct gpio_led sel3350_leds[] = {
-	{ .name = "sel:green:aux1" },
-	{ .name = "sel:green:aux2" },
-	{ .name = "sel:green:aux3" },
-	{ .name = "sel:green:aux4" },
-	{ .name = "sel:red:alarm" },
+static struct gpio_led sel3350_leds[] = {
+	{ .name = "sel:green:aux1",
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
+	{ .name = "sel:green:aux2",
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
+	{ .name = "sel:green:aux3",
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
+	{ .name = "sel:green:aux4",
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
+	{ .name = "sel:red:alarm",
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
 	{ .name = "sel:green:enabled",
-	  .default_state = LEDS_GPIO_DEFSTATE_ON },
-	{ .name = "sel:red:aux1" },
-	{ .name = "sel:red:aux2" },
-	{ .name = "sel:red:aux3" },
-	{ .name = "sel:red:aux4" },
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
+	{ .name = "sel:red:aux1",
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
+	{ .name = "sel:red:aux2",
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
+	{ .name = "sel:red:aux3",
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
+	{ .name = "sel:red:aux4",
+	  .default_state = LEDS_GPIO_DEFSTATE_KEEP,
+	  .retain_state_suspended = 1,
+	  .retain_state_shutdown = 1,
+	},
 };
 
 static const struct gpio_led_platform_data sel3350_leds_pdata = {
@@ -50,25 +115,6 @@ static const struct gpio_led_platform_data sel3350_leds_pdata = {
 	.leds = sel3350_leds,
 };
 
-/* Map GPIOs to LEDs */
-static struct gpiod_lookup_table sel3350_leds_table = {
-	.dev_id = "leds-gpio",
-	.table = {
-		GPIO_LOOKUP_IDX(BXT_NW, 49, NULL, 0, GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP_IDX(BXT_NW, 50, NULL, 1, GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP_IDX(BXT_NW, 51, NULL, 2, GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP_IDX(BXT_NW, 52, NULL, 3, GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP_IDX(BXT_W,  20, NULL, 4, GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP_IDX(BXT_W,  21, NULL, 5, GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP_IDX(BXT_SW, 37, NULL, 6, GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP_IDX(BXT_SW, 38, NULL, 7, GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP_IDX(BXT_SW, 39, NULL, 8, GPIO_ACTIVE_HIGH),
-		GPIO_LOOKUP_IDX(BXT_SW, 40, NULL, 9, GPIO_ACTIVE_HIGH),
-		{},
-	}
-};
-
-/* Map GPIOs to power supplies */
 static struct gpiod_lookup_table sel3350_gpios_table = {
 	.dev_id = B2093_GPIO_ACPI_ID ":00",
 	.table = {
@@ -76,6 +122,16 @@ static struct gpiod_lookup_table sel3350_gpios_table = {
 		GPIO_LOOKUP(BXT_NW, 45, SEL_PS_A_GOOD,   GPIO_ACTIVE_LOW),
 		GPIO_LOOKUP(BXT_NW, 46, SEL_PS_B_DETECT, GPIO_ACTIVE_LOW),
 		GPIO_LOOKUP(BXT_NW, 47, SEL_PS_B_GOOD,   GPIO_ACTIVE_LOW),
+		GPIO_LOOKUP(BXT_NW, 49, AUX_LED_GRN1, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP(BXT_NW, 50, AUX_LED_GRN2, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP(BXT_NW, 51, AUX_LED_GRN3, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP(BXT_NW, 52, AUX_LED_GRN4, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP(BXT_W,  20, ALARM_STATE_USER, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP(BXT_W,  21, ENABLE_STATE_USER, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP(BXT_SW, 37, AUX_LED_RED1, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP(BXT_SW, 38, AUX_LED_RED2, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP(BXT_SW, 39, AUX_LED_RED3, GPIO_ACTIVE_HIGH),
+		GPIO_LOOKUP(BXT_SW, 40, AUX_LED_RED4, GPIO_ACTIVE_HIGH),
 		{},
 	}
 };
@@ -149,6 +205,7 @@ struct sel3350_data {
 static int sel3350_probe(struct platform_device *pdev)
 {
 	int rs;
+	int i;
 	struct sel3350_data *sel3350;
 	struct power_supply_config ps_cfg = {};
 
@@ -158,9 +215,19 @@ static int sel3350_probe(struct platform_device *pdev)
 
 	platform_set_drvdata(pdev, sel3350);
 
-	gpiod_add_lookup_table(&sel3350_leds_table);
 	gpiod_add_lookup_table(&sel3350_gpios_table);
 
+	for (i = 0; i < ARRAY_SIZE(sel3350_leds); ++i) {
+		sel3350_leds[i].gpiod = devm_gpiod_get(&pdev->dev,
+						       sel3350_leds_gpio_names[i],
+						       GPIOD_ASIS);
+		if (IS_ERR_OR_NULL(sel3350_leds[i].gpiod)) {
+			rs = -EPROBE_DEFER;
+			goto err_gpio_loop;
+		}
+		gpiod_set_consumer_name(sel3350_leds[i].gpiod, sel3350_leds[i].name);
+	}
+
 	sel3350->leds_pdev = platform_device_register_data(
 			NULL,
 			"leds-gpio",
@@ -209,11 +276,15 @@ static int sel3350_probe(struct platform_device *pdev)
 
 	return 0;
 
+err_gpio_loop:
+	while (i--)
+		devm_gpiod_put(&pdev->dev, sel3350_leds[i].gpiod);
+	goto err_platform;
+
 err_ps:
 	platform_device_unregister(sel3350->leds_pdev);
 err_platform:
 	gpiod_remove_lookup_table(&sel3350_gpios_table);
-	gpiod_remove_lookup_table(&sel3350_leds_table);
 
 	return rs;
 }
@@ -224,7 +295,6 @@ static void sel3350_remove(struct platform_device *pdev)
 
 	platform_device_unregister(sel3350->leds_pdev);
 	gpiod_remove_lookup_table(&sel3350_gpios_table);
-	gpiod_remove_lookup_table(&sel3350_leds_table);
 }
 
 static const struct acpi_device_id sel3350_device_ids[] = {
diff --git a/drivers/platform/x86/sony-laptop.c b/drivers/platform/x86/sony-laptop.c
index b18f00e9082fa..67370967df6f5 100644
--- a/drivers/platform/x86/sony-laptop.c
+++ b/drivers/platform/x86/sony-laptop.c
@@ -3147,11 +3147,15 @@ static void sony_nc_backlight_cleanup(void)
 
 static int sony_nc_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *device;
 	acpi_status status;
 	int result = 0;
 	struct sony_nc_value *item;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	sony_nc_acpi_device = device;
 	strscpy(acpi_device_class(device), "sony/hotkey");
 
@@ -4509,11 +4513,15 @@ static void sony_pic_remove(struct platform_device *pdev)
 
 static int sony_pic_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct sony_pic_ioport *io, *tmp_io;
 	struct sony_pic_irq *irq, *tmp_irq;
+	struct acpi_device *device;
 	int result;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	spic_dev.acpi_dev = device;
 	strscpy(acpi_device_class(device), "sony/hotkey");
 	sony_pic_detect_device_type(&spic_dev);
diff --git a/drivers/platform/x86/system76_acpi.c b/drivers/platform/x86/system76_acpi.c
index 693cbb4613822..dd7b1b07c3165 100644
--- a/drivers/platform/x86/system76_acpi.c
+++ b/drivers/platform/x86/system76_acpi.c
@@ -674,10 +674,14 @@ static void system76_notify(acpi_handle handle, u32 event, void *context)
 // Probe a System76 platform device
 static int system76_probe(struct platform_device *pdev)
 {
-	struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *acpi_dev;
 	struct system76_data *data;
 	int err;
 
+	acpi_dev = ACPI_COMPANION(&pdev->dev);
+	if (!acpi_dev)
+		return -ENODEV;
+
 	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;
diff --git a/drivers/platform/x86/toshiba_acpi.c b/drivers/platform/x86/toshiba_acpi.c
index 35d899c017409..7cecb3a70b9c3 100644
--- a/drivers/platform/x86/toshiba_acpi.c
+++ b/drivers/platform/x86/toshiba_acpi.c
@@ -3374,7 +3374,7 @@ static const struct dmi_system_id toshiba_dmi_quirks[] __initconst = {
 
 static int toshiba_acpi_probe(struct platform_device *pdev)
 {
-	struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev);
+	struct acpi_device *acpi_dev;
 	struct toshiba_acpi_dev *dev;
 	const char *hci_method;
 	u32 dummy;
@@ -3383,6 +3383,10 @@ static int toshiba_acpi_probe(struct platform_device *pdev)
 	if (toshiba_acpi)
 		return -EBUSY;
 
+	acpi_dev = ACPI_COMPANION(&pdev->dev);
+	if (!acpi_dev)
+		return -ENODEV;
+
 	pr_info("Toshiba Laptop ACPI Extras version %s\n",
 	       TOSHIBA_ACPI_VERSION);
 
diff --git a/drivers/platform/x86/toshiba_bluetooth.c b/drivers/platform/x86/toshiba_bluetooth.c
index e50d4fc1e603d..e00abba58c7c0 100644
--- a/drivers/platform/x86/toshiba_bluetooth.c
+++ b/drivers/platform/x86/toshiba_bluetooth.c
@@ -230,10 +230,14 @@ static int toshiba_bt_resume(struct device *dev)
 
 static int toshiba_bt_rfkill_probe(struct platform_device *pdev)
 {
-	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
 	struct toshiba_bluetooth_dev *bt_dev;
+	struct acpi_device *device;
 	int result;
 
+	device = ACPI_COMPANION(&pdev->dev);
+	if (!device)
+		return -ENODEV;
+
 	result = toshiba_bluetooth_present(device->handle);
 	if (result)
 		return result;
diff --git a/drivers/platform/x86/toshiba_haps.c b/drivers/platform/x86/toshiba_haps.c
index 1486252b5983b..8d12241924df3 100644
--- a/drivers/platform/x86/toshiba_haps.c
+++ b/drivers/platform/x86/toshiba_haps.c
@@ -182,13 +182,17 @@ static int toshiba_haps_available(acpi_handle handle)
 
 static int toshiba_haps_probe(struct platform_device *pdev)
 {
-	struct acpi_device *acpi_dev = ACPI_COMPANION(&pdev->dev);
 	struct toshiba_haps_dev *haps;
+	struct acpi_device *acpi_dev;
 	int ret;
 
 	if (toshiba_haps)
 		return -EBUSY;
 
+	acpi_dev = ACPI_COMPANION(&pdev->dev);
+	if (!acpi_dev)
+		return -ENODEV;
+
 	if (!toshiba_haps_available(acpi_dev->handle))
 		return -ENODEV;
 
diff --git a/drivers/platform/x86/uniwill/uniwill-acpi.c b/drivers/platform/x86/uniwill/uniwill-acpi.c
index 945df50926371..8cc01bec77b9c 100644
--- a/drivers/platform/x86/uniwill/uniwill-acpi.c
+++ b/drivers/platform/x86/uniwill/uniwill-acpi.c
@@ -1359,6 +1359,16 @@ static int uniwill_led_init(struct uniwill_data *data)
 							 &init_data);
 }
 
+static unsigned int uniwill_sanitize_battery_threshold(unsigned int value)
+{
+	/* 0 means "charging threshold not active" */
+	if (!value)
+		return 100;
+
+	/* Guard against invalid values */
+	return min(value, 100);
+}
+
 static int uniwill_get_property(struct power_supply *psy, const struct power_supply_ext *ext,
 				void *drvdata, enum power_supply_property psp,
 				union power_supply_propval *val)
@@ -1405,7 +1415,8 @@ static int uniwill_get_property(struct power_supply *psy, const struct power_sup
 		if (ret < 0)
 			return ret;
 
-		val->intval = clamp_val(FIELD_GET(CHARGE_CTRL_MASK, regval), 0, 100);
+		regval = FIELD_GET(CHARGE_CTRL_MASK, regval);
+		val->intval = uniwill_sanitize_battery_threshold(regval);
 		return 0;
 	default:
 		return -EINVAL;
@@ -1420,11 +1431,11 @@ static int uniwill_set_property(struct power_supply *psy, const struct power_sup
 
 	switch (psp) {
 	case POWER_SUPPLY_PROP_CHARGE_CONTROL_END_THRESHOLD:
-		if (val->intval < 1 || val->intval > 100)
+		if (val->intval < 0 || val->intval > 100)
 			return -EINVAL;
 
 		return regmap_update_bits(data->regmap, EC_ADDR_CHARGE_CTRL, CHARGE_CTRL_MASK,
-					  val->intval);
+					  max(val->intval, 1));
 	default:
 		return -EINVAL;
 	}
@@ -1500,11 +1511,33 @@ static int uniwill_remove_battery(struct power_supply *battery, struct acpi_batt
 
 static int uniwill_battery_init(struct uniwill_data *data)
 {
+	unsigned int value, threshold, sanitized;
 	int ret;
 
 	if (!uniwill_device_supports(data, UNIWILL_FEATURE_BATTERY))
 		return 0;
 
+	ret = regmap_read(data->regmap, EC_ADDR_CHARGE_CTRL, &value);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * The charge control threshold might be initialized with 0 by
+	 * the EC to signal that said threshold is uninitialized. We thus
+	 * need to replace this placeholder value with a valid one (100)
+	 * to signal that we want to take control of battery charging.
+	 * For the sake of completeness we also apply this to other
+	 * invalid threshold values.
+	 */
+	threshold = FIELD_GET(CHARGE_CTRL_MASK, value);
+	sanitized = uniwill_sanitize_battery_threshold(threshold);
+	if (threshold != sanitized) {
+		FIELD_MODIFY(CHARGE_CTRL_MASK, &value, sanitized);
+		ret = regmap_write(data->regmap, EC_ADDR_CHARGE_CTRL, value);
+		if (ret < 0)
+			return ret;
+	}
+
 	ret = devm_mutex_init(data->dev, &data->battery_lock);
 	if (ret < 0)
 		return ret;
@@ -2456,8 +2489,6 @@ static int __init uniwill_init(void)
 		if (!force)
 			return -ENODEV;
 
-		/* Assume that the device supports all features */
-		device_descriptor.features = UINT_MAX;
 		pr_warn("Loading on a potentially unsupported device\n");
 	} else {
 		/*
@@ -2475,6 +2506,12 @@ static int __init uniwill_init(void)
 		device_descriptor = *descriptor;
 	}
 
+	if (force) {
+		/* Assume that the device supports all features except the charge limit */
+		device_descriptor.features = UINT_MAX & ~UNIWILL_FEATURE_BATTERY;
+		pr_warn("Enabling potentially unsupported features\n");
+	}
+
 	ret = platform_driver_register(&uniwill_driver);
 	if (ret < 0)
 		return ret;
diff --git a/drivers/platform/x86/wireless-hotkey.c b/drivers/platform/x86/wireless-hotkey.c
index f680d8ff8e877..3151844d1699d 100644
--- a/drivers/platform/x86/wireless-hotkey.c
+++ b/drivers/platform/x86/wireless-hotkey.c
@@ -89,9 +89,14 @@ static void wl_notify(acpi_handle handle, u32 event, void *data)
 
 static int wl_probe(struct platform_device *pdev)
 {
+	struct acpi_device *adev;
 	struct wl_button *button;
 	int err;
 
+	adev = ACPI_COMPANION(&pdev->dev);
+	if (!adev)
+		return -ENODEV;
+
 	button = kzalloc_obj(struct wl_button);
 	if (!button)
 		return -ENOMEM;
@@ -104,8 +109,8 @@ static int wl_probe(struct platform_device *pdev)
 		kfree(button);
 		return err;
 	}
-	err = acpi_dev_install_notify_handler(ACPI_COMPANION(&pdev->dev),
-					      ACPI_DEVICE_NOTIFY, wl_notify, button);
+	err = acpi_dev_install_notify_handler(adev, ACPI_DEVICE_NOTIFY,
+					      wl_notify, button);
 	if (err) {
 		pr_err("Failed to install ACPI notify handler\n");
 		wireless_input_destroy(&pdev->dev);
diff --git a/drivers/platform/x86/x86-android-tablets/core.c b/drivers/platform/x86/x86-android-tablets/core.c
index 021009e9085be..5db794d65eb5f 100644
--- a/drivers/platform/x86/x86-android-tablets/core.c
+++ b/drivers/platform/x86/x86-android-tablets/core.c
@@ -11,8 +11,10 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <linux/acpi.h>
+#include <linux/bug.h>
 #include <linux/device.h>
 #include <linux/dmi.h>
+#include <linux/fwnode.h>
 #include <linux/gpio/consumer.h>
 #include <linux/gpio/machine.h>
 #include <linux/irq.h>
@@ -360,6 +362,61 @@ static const struct software_node *cherryview_gpiochip_node_group[] = {
 	NULL
 };
 
+static void gpio_secondary_unset(void *data)
+{
+	struct device *dev = data;
+
+	set_secondary_fwnode(dev, NULL);
+}
+
+static void gpio_secondary_unregister_node_group(void *data)
+{
+	const struct software_node **nodes = data;
+
+	software_node_unregister_node_group(nodes);
+}
+
+static int gpio_secondary_fwnode_init(struct device *parent)
+{
+	const struct software_node *const *swnode;
+	struct fwnode_handle *fwnode;
+	int ret;
+
+	if (!gpiochip_node_group)
+		return 0;
+
+	ret = software_node_register_node_group(gpiochip_node_group);
+	if (ret)
+		return ret;
+
+	ret = devm_add_action_or_reset(parent,
+				       gpio_secondary_unregister_node_group,
+				       gpiochip_node_group);
+	if (ret)
+		return ret;
+
+	for (swnode = gpiochip_node_group; *swnode; swnode++) {
+		struct device *dev __free(put_device) =
+				acpi_bus_find_device_by_name((*swnode)->name);
+		if (!dev)
+			return dev_err_probe(parent,
+					     -ENODEV, "Failed to find the required GPIO controller: %s\n",
+					     (*swnode)->name);
+
+		fwnode = software_node_fwnode(*swnode);
+		if (WARN_ON(!fwnode))
+			return -ENOENT;
+
+		set_secondary_fwnode(dev, fwnode);
+
+		ret = devm_add_action_or_reset(parent, gpio_secondary_unset, dev);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static void x86_android_tablet_remove(struct platform_device *pdev)
 {
 	int i;
@@ -391,7 +448,6 @@ static void x86_android_tablet_remove(struct platform_device *pdev)
 
 	software_node_unregister_node_group(gpio_button_swnodes);
 	software_node_unregister_node_group(swnode_group);
-	software_node_unregister_node_group(gpiochip_node_group);
 }
 
 static __init int x86_android_tablet_probe(struct platform_device *pdev)
@@ -427,9 +483,11 @@ static __init int x86_android_tablet_probe(struct platform_device *pdev)
 		break;
 	}
 
-	ret = software_node_register_node_group(gpiochip_node_group);
-	if (ret)
+	ret = gpio_secondary_fwnode_init(&pdev->dev);
+	if (ret) {
+		x86_android_tablet_remove(pdev);
 		return ret;
+	}
 
 	ret = software_node_register_node_group(dev_info->swnode_group);
 	if (ret) {
diff --git a/drivers/platform/x86/xo15-ebook.c b/drivers/platform/x86/xo15-ebook.c
index 4d1b1b310cc56..e40e385c52bd4 100644
--- a/drivers/platform/x86/xo15-ebook.c
+++ b/drivers/platform/x86/xo15-ebook.c
@@ -15,6 +15,7 @@
 #include <linux/types.h>
 #include <linux/input.h>
 #include <linux/acpi.h>
+#include <linux/platform_device.h>
 
 #define MODULE_NAME "xo15-ebook"
 
@@ -38,15 +39,16 @@ MODULE_DEVICE_TABLE(acpi, ebook_device_ids);
 struct ebook_switch {
 	struct input_dev *input;
 	char phys[32];			/* for input device */
+	bool gpe_enabled;
 };
 
-static int ebook_send_state(struct acpi_device *device)
+static int ebook_send_state(struct device *dev)
 {
-	struct ebook_switch *button = acpi_driver_data(device);
+	struct ebook_switch *button = dev_get_drvdata(dev);
 	unsigned long long state;
 	acpi_status status;
 
-	status = acpi_evaluate_integer(device->handle, "EBK", NULL, &state);
+	status = acpi_evaluate_integer(ACPI_HANDLE(dev), "EBK", NULL, &state);
 	if (ACPI_FAILURE(status))
 		return -EIO;
 
@@ -56,16 +58,15 @@ static int ebook_send_state(struct acpi_device *device)
 	return 0;
 }
 
-static void ebook_switch_notify(struct acpi_device *device, u32 event)
+static void ebook_switch_notify(acpi_handle handle, u32 event, void *data)
 {
 	switch (event) {
 	case ACPI_FIXED_HARDWARE_EVENT:
 	case XO15_EBOOK_NOTIFY_STATUS:
-		ebook_send_state(device);
+		ebook_send_state(data);
 		break;
 	default:
-		acpi_handle_debug(device->handle,
-				  "Unsupported event [0x%x]\n", event);
+		acpi_handle_debug(handle, "Unsupported event [0x%x]\n", event);
 		break;
 	}
 }
@@ -73,37 +74,36 @@ static void ebook_switch_notify(struct acpi_device *device, u32 event)
 #ifdef CONFIG_PM_SLEEP
 static int ebook_switch_resume(struct device *dev)
 {
-	return ebook_send_state(to_acpi_device(dev));
+	return ebook_send_state(dev);
 }
 #endif
 
 static SIMPLE_DEV_PM_OPS(ebook_switch_pm, NULL, ebook_switch_resume);
 
-static int ebook_switch_add(struct acpi_device *device)
+static int ebook_switch_probe(struct platform_device *pdev)
 {
+	struct device *dev = &pdev->dev;
+	struct acpi_device *device = ACPI_COMPANION(dev);
 	const struct acpi_device_id *id;
 	struct ebook_switch *button;
 	struct input_dev *input;
 	int error;
 
-	button = kzalloc_obj(struct ebook_switch);
+	button = devm_kzalloc(dev, sizeof(*button), GFP_KERNEL);
 	if (!button)
 		return -ENOMEM;
 
-	device->driver_data = button;
+	platform_set_drvdata(pdev, button);
 
-	button->input = input = input_allocate_device();
-	if (!input) {
-		error = -ENOMEM;
-		goto err_free_button;
-	}
+	input = devm_input_allocate_device(dev);
+	if (!input)
+		return -ENOMEM;
+
+	button->input = input;
 
 	id = acpi_match_acpi_device(ebook_device_ids, device);
-	if (!id) {
-		dev_err(&device->dev, "Unsupported hid\n");
-		error = -ENODEV;
-		goto err_free_input;
-	}
+	if (!id)
+		return dev_err_probe(dev, -ENODEV, "Unsupported hid\n");
 
 	strscpy(acpi_device_name(device), XO15_EBOOK_DEVICE_NAME);
 	strscpy(acpi_device_class(device), XO15_EBOOK_CLASS "/" XO15_EBOOK_SUBCLASS);
@@ -113,50 +113,51 @@ static int ebook_switch_add(struct acpi_device *device)
 	input->name = acpi_device_name(device);
 	input->phys = button->phys;
 	input->id.bustype = BUS_HOST;
-	input->dev.parent = &device->dev;
 
 	input->evbit[0] = BIT_MASK(EV_SW);
 	set_bit(SW_TABLET_MODE, input->swbit);
 
 	error = input_register_device(input);
 	if (error)
-		goto err_free_input;
+		return error;
 
-	ebook_send_state(device);
+	error = acpi_dev_install_notify_handler(device, ACPI_DEVICE_NOTIFY,
+						ebook_switch_notify, dev);
+	if (error)
+		return error;
+
+	ebook_send_state(dev);
 
 	if (device->wakeup.flags.valid) {
 		/* Button's GPE is run-wake GPE */
 		acpi_enable_gpe(device->wakeup.gpe_device,
 				device->wakeup.gpe_number);
-		device_set_wakeup_enable(&device->dev, true);
+		button->gpe_enabled = true;
 	}
 
 	return 0;
-
- err_free_input:
-	input_free_device(input);
- err_free_button:
-	kfree(button);
-	return error;
 }
 
-static void ebook_switch_remove(struct acpi_device *device)
+static void ebook_switch_remove(struct platform_device *pdev)
 {
-	struct ebook_switch *button = acpi_driver_data(device);
+	struct ebook_switch *button = platform_get_drvdata(pdev);
+	struct acpi_device *device = ACPI_COMPANION(&pdev->dev);
+
+	if (button->gpe_enabled)
+		acpi_disable_gpe(device->wakeup.gpe_device,
+				 device->wakeup.gpe_number);
 
-	input_unregister_device(button->input);
-	kfree(button);
+	acpi_dev_remove_notify_handler(device, ACPI_DEVICE_NOTIFY,
+				       ebook_switch_notify);
 }
 
-static struct acpi_driver xo15_ebook_driver = {
-	.name = MODULE_NAME,
-	.class = XO15_EBOOK_CLASS,
-	.ids = ebook_device_ids,
-	.ops = {
-		.add = ebook_switch_add,
-		.remove = ebook_switch_remove,
-		.notify = ebook_switch_notify,
+static struct platform_driver xo15_ebook_driver = {
+	.probe = ebook_switch_probe,
+	.remove = ebook_switch_remove,
+	.driver = {
+		.name = MODULE_NAME,
+		.acpi_match_table = ebook_device_ids,
+		.pm = &ebook_switch_pm,
 	},
-	.drv.pm = &ebook_switch_pm,
 };
-module_acpi_driver(xo15_ebook_driver);
+module_platform_driver(xo15_ebook_driver);
diff --git a/drivers/pmdomain/arm/scmi_pm_domain.c b/drivers/pmdomain/arm/scmi_pm_domain.c
index 5454faed7d5d4..3d73aef21d2f9 100644
--- a/drivers/pmdomain/arm/scmi_pm_domain.c
+++ b/drivers/pmdomain/arm/scmi_pm_domain.c
@@ -113,6 +113,15 @@ static int scmi_pm_domain_probe(struct scmi_device *sdev)
 		goto err_rm_genpds;
 
 	dev_set_drvdata(dev, scmi_pd_data);
+
+	/*
+	 * Parse (optional) power-domains-child-ids property to establish
+	 * parent-child relationships.
+	*/
+	ret = of_genpd_add_child_ids(np, scmi_pd_data);
+	if (ret < 0)
+		dev_err(dev, "Failed to add child domain hierarchy: %d\n", ret);
+
 	dev_info(dev, "Initialized %d power domains", num_domains);
 
 	return 0;
@@ -130,9 +139,13 @@ static void scmi_pm_domain_remove(struct scmi_device *sdev)
 	struct device *dev = &sdev->dev;
 	struct device_node *np = dev->of_node;
 
+	scmi_pd_data = dev_get_drvdata(dev);
+
+	/* Remove any parent-child relationships established at probe time */
+	of_genpd_remove_child_ids(np, scmi_pd_data);
+
 	of_genpd_del_provider(np);
 
-	scmi_pd_data = dev_get_drvdata(dev);
 	for (i = 0; i < scmi_pd_data->num_domains; i++) {
 		if (!scmi_pd_data->domains[i])
 			continue;
diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c
index 71e930e80178e..f7731270015d9 100644
--- a/drivers/pmdomain/core.c
+++ b/drivers/pmdomain/core.c
@@ -2924,6 +2924,173 @@ static struct generic_pm_domain *genpd_get_from_provider(
 	return genpd;
 }
 
+/**
+ * of_genpd_add_child_ids() - Parse power-domains-child-ids property
+ * @np: Device node pointer associated with the PM domain provider.
+ * @data: Pointer to the onecell data associated with the PM domain provider.
+ *
+ * Parse the power-domains and power-domains-child-ids properties to establish
+ * parent-child relationships for PM domains. The power-domains property lists
+ * parent domains, and power-domains-child-ids lists which child domain IDs
+ * should be associated with each parent.
+ *
+ * Uses "all or nothing" semantics: either all relationships are established
+ * successfully, or none are (any partially-added relationships are unwound
+ * on error).
+ *
+ * Returns the number of parent-child relationships established on success,
+ * 0 if the properties don't exist, or a negative error code on failure.
+ */
+int of_genpd_add_child_ids(struct device_node *np,
+			   struct genpd_onecell_data *data)
+{
+	struct of_phandle_args parent_args;
+	struct generic_pm_domain *parent_genpd, *child_genpd;
+	struct generic_pm_domain **pairs; /* pairs[2*i]=parent, pairs[2*i+1]=child */
+	u32 child_id;
+	int i, ret, count, child_count, added = 0;
+
+	/* Check if both properties exist */
+	count = of_count_phandle_with_args(np, "power-domains", "#power-domain-cells");
+	if (count <= 0)
+		return 0;
+
+	child_count = of_property_count_u32_elems(np, "power-domains-child-ids");
+	if (child_count < 0)
+		return 0;
+	if (child_count != count)
+		return -EINVAL;
+
+	/* Allocate tracking array for error unwind (parent/child pairs) */
+	pairs = kmalloc_array(count * 2, sizeof(*pairs), GFP_KERNEL);
+	if (!pairs)
+		return -ENOMEM;
+
+	for (i = 0; i < count; i++) {
+		ret = of_property_read_u32_index(np, "power-domains-child-ids",
+						 i, &child_id);
+		if (ret)
+			goto err_unwind;
+
+		/* Validate child ID is within bounds */
+		if (child_id >= data->num_domains) {
+			pr_err("Child ID %u out of bounds (max %u) for %pOF\n",
+			       child_id, data->num_domains - 1, np);
+			ret = -EINVAL;
+			goto err_unwind;
+		}
+
+		/* Get the child domain */
+		child_genpd = data->domains[child_id];
+		if (!child_genpd) {
+			pr_err("Child domain %u is NULL for %pOF\n", child_id, np);
+			ret = -EINVAL;
+			goto err_unwind;
+		}
+
+		ret = of_parse_phandle_with_args(np, "power-domains",
+						 "#power-domain-cells", i,
+						 &parent_args);
+		if (ret)
+			goto err_unwind;
+
+		/* Get the parent domain */
+		parent_genpd = genpd_get_from_provider(&parent_args);
+		of_node_put(parent_args.np);
+		if (IS_ERR(parent_genpd)) {
+			pr_err("Failed to get parent domain for %pOF: %ld\n",
+			       np, PTR_ERR(parent_genpd));
+			ret = PTR_ERR(parent_genpd);
+			goto err_unwind;
+		}
+
+		/* Establish parent-child relationship */
+		ret = pm_genpd_add_subdomain(parent_genpd, child_genpd);
+		if (ret) {
+			pr_err("Failed to add child domain %u to parent in %pOF: %d\n",
+			       child_id, np, ret);
+			goto err_unwind;
+		}
+
+		/* Track for potential unwind */
+		pairs[2 * added] = parent_genpd;
+		pairs[2 * added + 1] = child_genpd;
+		added++;
+
+		pr_debug("Added child domain %u (%s) to parent %s for %pOF\n",
+			 child_id, child_genpd->name, parent_genpd->name, np);
+	}
+
+	kfree(pairs);
+	return count;
+
+err_unwind:
+	/* Reverse all previously established relationships */
+	while (added-- > 0)
+		pm_genpd_remove_subdomain(pairs[2 * added], pairs[2 * added + 1]);
+	kfree(pairs);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(of_genpd_add_child_ids);
+
+/**
+ * of_genpd_remove_child_ids() - Remove parent-child PM domain relationships
+ * @np: Device node pointer associated with the PM domain provider.
+ * @data: Pointer to the onecell data associated with the PM domain provider.
+ *
+ * Reverses the effect of of_genpd_add_child_ids() by parsing the same
+ * power-domains and power-domains-child-ids properties and calling
+ * pm_genpd_remove_subdomain() for each established relationship.
+ *
+ * Returns 0 on success, -ENOENT if properties don't exist, or negative error
+ * code on failure.
+ */
+int of_genpd_remove_child_ids(struct device_node *np,
+			   struct genpd_onecell_data *data)
+{
+	struct of_phandle_args parent_args;
+	struct generic_pm_domain *parent_genpd, *child_genpd;
+	u32 child_id;
+	int i, ret, count, child_count;
+
+	/* Check if both properties exist */
+	count = of_count_phandle_with_args(np, "power-domains", "#power-domain-cells");
+	if (count <= 0)
+		return -ENOENT;
+
+	child_count = of_property_count_u32_elems(np, "power-domains-child-ids");
+	if (child_count < 0)
+		return -ENOENT;
+	if (child_count != count)
+		return -EINVAL;
+
+	for (i = 0; i < count; i++) {
+		if (of_property_read_u32_index(np, "power-domains-child-ids",
+					       i, &child_id))
+			continue;
+
+		if (child_id >= data->num_domains || !data->domains[child_id])
+			continue;
+
+		ret = of_parse_phandle_with_args(np, "power-domains",
+						 "#power-domain-cells", i,
+						 &parent_args);
+		if (ret)
+			continue;
+
+		parent_genpd = genpd_get_from_provider(&parent_args);
+		of_node_put(parent_args.np);
+		if (IS_ERR(parent_genpd))
+			continue;
+
+		child_genpd = data->domains[child_id];
+		pm_genpd_remove_subdomain(parent_genpd, child_genpd);
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(of_genpd_remove_child_ids);
+
 /**
  * of_genpd_add_device() - Add a device to an I/O PM domain
  * @genpdspec: OF phandle args to use for look-up PM domain
diff --git a/drivers/power/sequencing/core.c b/drivers/power/sequencing/core.c
index 4dff71be11b60..14335c4f813ed 100644
--- a/drivers/power/sequencing/core.c
+++ b/drivers/power/sequencing/core.c
@@ -1043,7 +1043,7 @@ static int pwrseq_debugfs_seq_show(struct seq_file *seq, void *data)
 	struct pwrseq_target *target;
 	struct pwrseq_unit *unit;
 
-	seq_printf(seq, "%s:\n", dev_name(dev));
+	seq_printf(seq, "%s (%s):\n", dev_name(dev), dev_name(dev->parent));
 
 	seq_puts(seq, "  targets:\n");
 	list_for_each_entry(target, &pwrseq->targets, list)
diff --git a/drivers/power/supply/charger-manager.c b/drivers/power/supply/charger-manager.c
index c49e0e4d02f77..1b0239c591144 100644
--- a/drivers/power/supply/charger-manager.c
+++ b/drivers/power/supply/charger-manager.c
@@ -881,26 +881,22 @@ static bool cm_setup_timer(void)
 	mutex_unlock(&cm_list_mtx);
 
 	if (timer_req && cm_timer) {
-		ktime_t now, add;
-
 		/*
 		 * Set alarm with the polling interval (wakeup_ms)
 		 * The alarm time should be NOW + CM_RTC_SMALL or later.
 		 */
-		if (wakeup_ms == UINT_MAX ||
-			wakeup_ms < CM_RTC_SMALL * MSEC_PER_SEC)
+		if (wakeup_ms == UINT_MAX || wakeup_ms < CM_RTC_SMALL * MSEC_PER_SEC)
 			wakeup_ms = 2 * CM_RTC_SMALL * MSEC_PER_SEC;
 
 		pr_info("Charger Manager wakeup timer: %u ms\n", wakeup_ms);
 
-		now = ktime_get_boottime();
-		add = ktime_set(wakeup_ms / MSEC_PER_SEC,
-				(wakeup_ms % MSEC_PER_SEC) * NSEC_PER_MSEC);
-		alarm_start(cm_timer, ktime_add(now, add));
-
 		cm_suspend_duration_ms = wakeup_ms;
 
-		return true;
+		/*
+		 * The timer should always be queued as the timeout is at least
+		 * two seconds out. Handle it correctly nevertheless.
+		 */
+		return alarm_start_timer(cm_timer, ktime_add_ms(0, wakeup_ms), true);
 	}
 	return false;
 }
diff --git a/drivers/ptp/ptp_fc3.c b/drivers/ptp/ptp_fc3.c
index 70002500170ea..f0e000428a3f4 100644
--- a/drivers/ptp/ptp_fc3.c
+++ b/drivers/ptp/ptp_fc3.c
@@ -55,8 +55,8 @@ static s64 tdc_meas2offset(struct idtfc3 *idtfc3, u64 meas_read)
 {
 	s64 coarse, fine;
 
-	fine = sign_extend64(FIELD_GET(FINE_MEAS_MASK, meas_read), 12);
-	coarse = sign_extend64(FIELD_GET(COARSE_MEAS_MASK, meas_read), (39 - 13));
+	fine = FIELD_GET_SIGNED(FINE_MEAS_MASK, meas_read);
+	coarse = FIELD_GET_SIGNED(COARSE_MEAS_MASK, meas_read);
 
 	fine = div64_s64(fine * NSEC_PER_SEC, idtfc3->tdc_apll_freq * 62LL);
 	coarse = div64_s64(coarse * NSEC_PER_SEC, idtfc3->time_ref_freq);
diff --git a/drivers/ptp/ptp_vmw.c b/drivers/ptp/ptp_vmw.c
index 8510121d79d10..14fc587d0a002 100644
--- a/drivers/ptp/ptp_vmw.c
+++ b/drivers/ptp/ptp_vmw.c
@@ -18,7 +18,6 @@
 #define VMWARE_CMD_PCLK(nr) ((nr << 16) | 97)
 #define VMWARE_CMD_PCLK_GETTIME VMWARE_CMD_PCLK(0)
 
-static struct acpi_device *ptp_vmw_acpi_device;
 static struct ptp_clock *ptp_vmw_clock;
 
 
@@ -91,8 +90,6 @@ static int ptp_vmw_acpi_probe(struct platform_device *pdev)
 		pr_err("failed to register ptp clock\n");
 		return PTR_ERR(ptp_vmw_clock);
 	}
-
-	ptp_vmw_acpi_device = ACPI_COMPANION(&pdev->dev);
 	return 0;
 }
 
diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig
index 6f3147518376a..e8886a9b64d96 100644
--- a/drivers/pwm/Kconfig
+++ b/drivers/pwm/Kconfig
@@ -347,6 +347,18 @@ config PWM_INTEL_LGM
 	  To compile this driver as a module, choose M here: the module
 	  will be called pwm-intel-lgm.
 
+config PWM_IPQ
+	tristate "IPQ PWM support"
+	depends on ARCH_QCOM || COMPILE_TEST
+	depends on HAVE_CLK && HAS_IOMEM
+	help
+	  Generic PWM framework driver for IPQ PWM block which supports
+	  4 pwm channels. Each of the these channels can be configured
+	  independent of each other.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called pwm-ipq.
+
 config PWM_IQS620A
 	tristate "Azoteq IQS620A PWM support"
 	depends on MFD_IQS62X || COMPILE_TEST
diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile
index 0dc0d2b69025d..5630a521a7cff 100644
--- a/drivers/pwm/Makefile
+++ b/drivers/pwm/Makefile
@@ -29,6 +29,7 @@ obj-$(CONFIG_PWM_IMX1)		+= pwm-imx1.o
 obj-$(CONFIG_PWM_IMX27)		+= pwm-imx27.o
 obj-$(CONFIG_PWM_IMX_TPM)	+= pwm-imx-tpm.o
 obj-$(CONFIG_PWM_INTEL_LGM)	+= pwm-intel-lgm.o
+obj-$(CONFIG_PWM_IPQ)		+= pwm-ipq.o
 obj-$(CONFIG_PWM_IQS620A)	+= pwm-iqs620a.o
 obj-$(CONFIG_PWM_JZ4740)	+= pwm-jz4740.o
 obj-$(CONFIG_PWM_KEEMBAY)	+= pwm-keembay.o
diff --git a/drivers/pwm/pwm-atmel-tcb.c b/drivers/pwm/pwm-atmel-tcb.c
index 3d30aeab507e0..a765ef279b512 100644
--- a/drivers/pwm/pwm-atmel-tcb.c
+++ b/drivers/pwm/pwm-atmel-tcb.c
@@ -443,7 +443,7 @@ static int atmel_tcb_pwm_probe(struct platform_device *pdev)
 
 	err = clk_prepare_enable(tcbpwmc->slow_clk);
 	if (err)
-		goto err_disable_clk;;
+		goto err_disable_clk;
 
 	err = clk_rate_exclusive_get(tcbpwmc->clk);
 	if (err)
diff --git a/drivers/pwm/pwm-dwc.c b/drivers/pwm/pwm-dwc.c
index 86b72db587410..49609d132fdac 100644
--- a/drivers/pwm/pwm-dwc.c
+++ b/drivers/pwm/pwm-dwc.c
@@ -147,7 +147,7 @@ static int dwc_pwm_resume(struct device *dev)
 static DEFINE_SIMPLE_DEV_PM_OPS(dwc_pwm_pm_ops, dwc_pwm_suspend, dwc_pwm_resume);
 
 static const struct pci_device_id dwc_pwm_id_table[] = {
-	{ PCI_VDEVICE(INTEL, 0x4bb7), (kernel_ulong_t)&ehl_pwm_info },
+	{ PCI_VDEVICE(INTEL, 0x4bb7), .driver_data = (kernel_ulong_t)&ehl_pwm_info },
 	{  }	/* Terminating Entry */
 };
 MODULE_DEVICE_TABLE(pci, dwc_pwm_id_table);
diff --git a/drivers/pwm/pwm-ipq.c b/drivers/pwm/pwm-ipq.c
new file mode 100644
index 0000000000000..c533739481361
--- /dev/null
+++ b/drivers/pwm/pwm-ipq.c
@@ -0,0 +1,264 @@
+// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
+/*
+ * Copyright (c) 2016-2017, 2020 The Linux Foundation. All rights reserved.
+ *
+ * Limitations:
+ * - The PWM controller has no publicly available datasheet.
+ * - Each of the four channels is programmed via two 32-bit registers
+ *   (REG0 and REG1 at 8-byte stride).
+ * - Period and duty-cycle reconfiguration is fully atomic: new divider,
+ *   pre-divider, and high-duration values are latched by setting the
+ *   UPDATE bit (bit 30 in REG1). The hardware applies the new settings
+ *   at the beginning of the next period without disabling the output,
+ *   so the currently running period is always completed.
+ * - On disable (clearing the ENABLE bit 31 in REG1), the hardware
+ *   finishes the current period before stopping the output. The pin
+ *   is then driven to the inactive (low) level.
+ * - Upon disabling, the hardware resets the pre-divider (PRE_DIV) and divider
+ *   fields (PWM_DIV) in REG0 and REG1 to 0x0000 and 0x0001 respectively.
+ * - Only normal polarity is supported.
+ */
+
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/pwm.h>
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/of.h>
+#include <linux/math64.h>
+#include <linux/of_device.h>
+#include <linux/bitfield.h>
+#include <linux/units.h>
+
+/* The frequency range supported is 1 Hz to 100 Mhz (clock rate) */
+#define IPQ_PWM_MAX_PERIOD_NS	((u64)NSEC_PER_SEC)
+#define IPQ_PWM_MIN_PERIOD_NS	10
+
+/*
+ * Two 32-bit registers for each PWM: REG0, and REG1.
+ * Base offset for PWM #i is at 8 * #i.
+ */
+#define IPQ_PWM_REG0			0
+#define IPQ_PWM_REG0_PWM_DIV		GENMASK(15, 0)
+#define IPQ_PWM_REG0_HI_DURATION	GENMASK(31, 16)
+
+#define IPQ_PWM_REG1			4
+#define IPQ_PWM_REG1_PRE_DIV		GENMASK(15, 0)
+/*
+ * Enable bit is set to enable output toggling in pwm device.
+ * Update bit is set to trigger the change and is unset automatically
+ * to reflect the changed divider and high duration values in register.
+ */
+#define IPQ_PWM_REG1_UPDATE		BIT(30)
+#define IPQ_PWM_REG1_ENABLE		BIT(31)
+
+/*
+ * The max value specified for each field is based on the number of bits
+ * in the pwm control register for that field (16-bit)
+ */
+#define IPQ_PWM_MAX_DIV			FIELD_MAX(IPQ_PWM_REG0_PWM_DIV)
+
+struct ipq_pwm_chip {
+	void __iomem *mem;
+	unsigned long clk_rate;
+};
+
+static struct ipq_pwm_chip *ipq_pwm_from_chip(struct pwm_chip *chip)
+{
+	return pwmchip_get_drvdata(chip);
+}
+
+static unsigned int ipq_pwm_reg_read(struct pwm_device *pwm, unsigned int reg)
+{
+	struct ipq_pwm_chip *ipq_chip = ipq_pwm_from_chip(pwm->chip);
+	unsigned int off = 8 * pwm->hwpwm + reg;
+
+	return readl(ipq_chip->mem + off);
+}
+
+static void ipq_pwm_reg_write(struct pwm_device *pwm, unsigned int reg,
+			      unsigned int val)
+{
+	struct ipq_pwm_chip *ipq_chip = ipq_pwm_from_chip(pwm->chip);
+	unsigned int off = 8 * pwm->hwpwm + reg;
+
+	writel(val, ipq_chip->mem + off);
+}
+
+static int ipq_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm,
+			 const struct pwm_state *state)
+{
+	struct ipq_pwm_chip *ipq_chip = ipq_pwm_from_chip(chip);
+	unsigned int pre_div, pwm_div;
+	u64 period_ns, duty_ns;
+	unsigned long val = 0;
+	unsigned long hi_dur;
+
+	if (!state->enabled) {
+		/* clear IPQ_PWM_REG1_ENABLE */
+		ipq_pwm_reg_write(pwm, IPQ_PWM_REG1, IPQ_PWM_REG1_UPDATE);
+		return 0;
+	}
+
+	if (state->polarity != PWM_POLARITY_NORMAL)
+		return -EINVAL;
+
+	/*
+	 * Check the upper and lower bounds for the period as per
+	 * hardware limits
+	 */
+	if (state->period < IPQ_PWM_MIN_PERIOD_NS)
+		return -ERANGE;
+	period_ns = min(state->period, IPQ_PWM_MAX_PERIOD_NS);
+	duty_ns = min(state->duty_cycle, period_ns);
+
+	/*
+	 * Pick the maximal value for PWM_DIV that still allows a
+	 * 100% relative duty cycle. This allows a fine grained
+	 * selection of duty cycles.
+	 */
+	pwm_div = IPQ_PWM_MAX_DIV - 1;
+
+	/*
+	 * although mul_u64_u64_div_u64 returns a u64, in practice it
+	 * won't overflow due to above constraints. Take the max period
+	 * of 10^9 (NSEC_PER_SEC) and the pwm_div + 1 (IPQ_PWM_MAX_DIV)
+	 *  10^9 * 10^8
+	 * ------------- => which fits well into a 32-bit unsigned int.
+	 * 10^9 * 65,535
+	 */
+	pre_div = mul_u64_u64_div_u64(period_ns, ipq_chip->clk_rate,
+				      (u64)NSEC_PER_SEC * (pwm_div + 1));
+
+	if (!pre_div)
+		return -ERANGE;
+
+	pre_div -= 1;
+
+	if (pre_div > IPQ_PWM_MAX_DIV)
+		pre_div = IPQ_PWM_MAX_DIV;
+
+	/* pwm duty = HI_DUR * (PRE_DIV + 1) / clk_rate */
+	hi_dur = mul_u64_u64_div_u64(duty_ns, ipq_chip->clk_rate,
+				     (u64)NSEC_PER_SEC * (pre_div + 1));
+
+	val = FIELD_PREP(IPQ_PWM_REG0_HI_DURATION, hi_dur) |
+		FIELD_PREP(IPQ_PWM_REG0_PWM_DIV, pwm_div);
+	ipq_pwm_reg_write(pwm, IPQ_PWM_REG0, val);
+
+	val = FIELD_PREP(IPQ_PWM_REG1_PRE_DIV, pre_div);
+	ipq_pwm_reg_write(pwm, IPQ_PWM_REG1, val);
+
+	/* PWM enable toggle needs a separate write to REG1 */
+	val |= IPQ_PWM_REG1_UPDATE | IPQ_PWM_REG1_ENABLE;
+	ipq_pwm_reg_write(pwm, IPQ_PWM_REG1, val);
+
+	return 0;
+}
+
+static int ipq_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm,
+			     struct pwm_state *state)
+{
+	struct ipq_pwm_chip *ipq_chip = ipq_pwm_from_chip(chip);
+	unsigned int pre_div, pwm_div, hi_dur;
+	u64 effective_div, hi_div;
+	u32 reg0, reg1;
+
+	reg1 = ipq_pwm_reg_read(pwm, IPQ_PWM_REG1);
+	state->enabled = reg1 & IPQ_PWM_REG1_ENABLE;
+
+	if (!state->enabled)
+		return 0;
+
+	reg0 = ipq_pwm_reg_read(pwm, IPQ_PWM_REG0);
+
+	state->polarity = PWM_POLARITY_NORMAL;
+
+	pwm_div = FIELD_GET(IPQ_PWM_REG0_PWM_DIV, reg0);
+	hi_dur = FIELD_GET(IPQ_PWM_REG0_HI_DURATION, reg0);
+	pre_div = FIELD_GET(IPQ_PWM_REG1_PRE_DIV, reg1);
+
+	effective_div = (u64)(pwm_div + 1) * (pre_div + 1);
+
+	/*
+	 * effective_div <= 0x100000000, so the multiplication doesn't overflow.
+	 */
+	state->period = DIV64_U64_ROUND_UP(effective_div * NSEC_PER_SEC,
+					   ipq_chip->clk_rate);
+
+	hi_div = hi_dur * (pre_div + 1);
+	state->duty_cycle = DIV64_U64_ROUND_UP(hi_div * NSEC_PER_SEC,
+					       ipq_chip->clk_rate);
+
+	/*
+	 * ensure a valid config is passed back to PWM core in case duty_cycle
+	 * is > period (>100%)
+	 */
+	state->duty_cycle = min(state->duty_cycle, state->period);
+
+	return 0;
+}
+
+static const struct pwm_ops ipq_pwm_ops = {
+	.apply = ipq_pwm_apply,
+	.get_state = ipq_pwm_get_state,
+};
+
+static int ipq_pwm_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct ipq_pwm_chip *pwm;
+	struct pwm_chip *chip;
+	struct clk *clk;
+	int ret;
+
+	chip = devm_pwmchip_alloc(dev, 4, sizeof(*pwm));
+	if (IS_ERR(chip))
+		return PTR_ERR(chip);
+	pwm = ipq_pwm_from_chip(chip);
+
+	pwm->mem = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(pwm->mem))
+		return dev_err_probe(dev, PTR_ERR(pwm->mem),
+				     "Failed to acquire resource\n");
+
+	clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(clk))
+		return dev_err_probe(dev, PTR_ERR(clk),
+				     "Failed to get clock\n");
+
+	ret = devm_clk_rate_exclusive_get(dev, clk);
+	if (ret)
+		return dev_err_probe(dev, ret, "Failed to lock clock rate\n");
+
+	pwm->clk_rate = clk_get_rate(clk);
+	if (!pwm->clk_rate)
+		return dev_err_probe(dev, -EINVAL, "Failed due to clock rate being zero\n");
+
+	chip->ops = &ipq_pwm_ops;
+
+	ret = devm_pwmchip_add(dev, chip);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "Failed to add pwm chip\n");
+
+	return 0;
+}
+
+static const struct of_device_id pwm_ipq_dt_match[] = {
+	{ .compatible = "qcom,ipq6018-pwm", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, pwm_ipq_dt_match);
+
+static struct platform_driver ipq_pwm_driver = {
+	.driver = {
+		.name = "ipq-pwm",
+		.of_match_table = pwm_ipq_dt_match,
+	},
+	.probe = ipq_pwm_probe,
+};
+
+module_platform_driver(ipq_pwm_driver);
+
+MODULE_DESCRIPTION("Qualcomm IPQ PWM driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/pwm/pwm-lpss-pci.c b/drivers/pwm/pwm-lpss-pci.c
index ae25d9321d75a..3a0fd65935201 100644
--- a/drivers/pwm/pwm-lpss-pci.c
+++ b/drivers/pwm/pwm-lpss-pci.c
@@ -48,15 +48,15 @@ static void pwm_lpss_remove_pci(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id pwm_lpss_pci_ids[] = {
-	{ PCI_VDEVICE(INTEL, 0x0ac8), (unsigned long)&pwm_lpss_bxt_info},
-	{ PCI_VDEVICE(INTEL, 0x0f08), (unsigned long)&pwm_lpss_byt_info},
-	{ PCI_VDEVICE(INTEL, 0x0f09), (unsigned long)&pwm_lpss_byt_info},
-	{ PCI_VDEVICE(INTEL, 0x11a5), (unsigned long)&pwm_lpss_tng_info},
-	{ PCI_VDEVICE(INTEL, 0x1ac8), (unsigned long)&pwm_lpss_bxt_info},
-	{ PCI_VDEVICE(INTEL, 0x2288), (unsigned long)&pwm_lpss_bsw_info},
-	{ PCI_VDEVICE(INTEL, 0x2289), (unsigned long)&pwm_lpss_bsw_info},
-	{ PCI_VDEVICE(INTEL, 0x31c8), (unsigned long)&pwm_lpss_bxt_info},
-	{ PCI_VDEVICE(INTEL, 0x5ac8), (unsigned long)&pwm_lpss_bxt_info},
+	{ PCI_VDEVICE(INTEL, 0x0ac8), .driver_data = (unsigned long)&pwm_lpss_bxt_info },
+	{ PCI_VDEVICE(INTEL, 0x0f08), .driver_data = (unsigned long)&pwm_lpss_byt_info },
+	{ PCI_VDEVICE(INTEL, 0x0f09), .driver_data = (unsigned long)&pwm_lpss_byt_info },
+	{ PCI_VDEVICE(INTEL, 0x11a5), .driver_data = (unsigned long)&pwm_lpss_tng_info },
+	{ PCI_VDEVICE(INTEL, 0x1ac8), .driver_data = (unsigned long)&pwm_lpss_bxt_info },
+	{ PCI_VDEVICE(INTEL, 0x2288), .driver_data = (unsigned long)&pwm_lpss_bsw_info },
+	{ PCI_VDEVICE(INTEL, 0x2289), .driver_data = (unsigned long)&pwm_lpss_bsw_info },
+	{ PCI_VDEVICE(INTEL, 0x31c8), .driver_data = (unsigned long)&pwm_lpss_bxt_info },
+	{ PCI_VDEVICE(INTEL, 0x5ac8), .driver_data = (unsigned long)&pwm_lpss_bxt_info },
 	{ },
 };
 MODULE_DEVICE_TABLE(pci, pwm_lpss_pci_ids);
diff --git a/drivers/pwm/pwm-pca9685.c b/drivers/pwm/pwm-pca9685.c
index 107bebec3546e..a02255a64ea81 100644
--- a/drivers/pwm/pwm-pca9685.c
+++ b/drivers/pwm/pwm-pca9685.c
@@ -538,7 +538,7 @@ static int __maybe_unused pca9685_pwm_runtime_resume(struct device *dev)
 }
 
 static const struct i2c_device_id pca9685_id[] = {
-	{ "pca9685" },
+	{ .name = "pca9685" },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(i2c, pca9685_id);
diff --git a/drivers/pwm/pwm-pxa.c b/drivers/pwm/pwm-pxa.c
index 0f5bdb0e395ea..80d2fa10919f8 100644
--- a/drivers/pwm/pwm-pxa.c
+++ b/drivers/pwm/pwm-pxa.c
@@ -161,6 +161,7 @@ static int pwm_probe(struct platform_device *pdev)
 	const struct platform_device_id *id = platform_get_device_id(pdev);
 	struct pwm_chip *chip;
 	struct pxa_pwm_chip *pc;
+	struct clk *bus_clk;
 	struct device *dev = &pdev->dev;
 	struct reset_control *rst;
 	int ret = 0;
@@ -177,7 +178,12 @@ static int pwm_probe(struct platform_device *pdev)
 		return PTR_ERR(chip);
 	pc = to_pxa_pwm_chip(chip);
 
-	pc->clk = devm_clk_get(dev, NULL);
+	bus_clk = devm_clk_get_optional_enabled(dev, "bus");
+	if (IS_ERR(bus_clk))
+		return dev_err_probe(dev, PTR_ERR(bus_clk), "Failed to get bus clock\n");
+
+	/* Get named func clk if bus clock is valid */
+	pc->clk = devm_clk_get(dev, bus_clk ? "func" : NULL);
 	if (IS_ERR(pc->clk))
 		return dev_err_probe(dev, PTR_ERR(pc->clk), "Failed to get clock\n");
 
diff --git a/drivers/pwm/pwm-stm32.c b/drivers/pwm/pwm-stm32.c
index 935257a890b06..c708e4a7ad709 100644
--- a/drivers/pwm/pwm-stm32.c
+++ b/drivers/pwm/pwm-stm32.c
@@ -193,22 +193,6 @@ static int stm32_pwm_round_waveform_tohw(struct pwm_chip *chip,
 	return ret;
 }
 
-/*
- * This should be moved to lib/math/div64.c. Currently there are some changes
- * pending to mul_u64_u64_div_u64. Uwe will care for that when the dust settles.
- */
-static u64 stm32_pwm_mul_u64_u64_div_u64_roundup(u64 a, u64 b, u64 c)
-{
-	u64 res = mul_u64_u64_div_u64(a, b, c);
-	/* Those multiplications might overflow but it doesn't matter */
-	u64 rem = a * b - c * res;
-
-	if (rem)
-		res += 1;
-
-	return res;
-}
-
 static int stm32_pwm_round_waveform_fromhw(struct pwm_chip *chip,
 					   struct pwm_device *pwm,
 					   const void *_wfhw,
@@ -223,16 +207,15 @@ static int stm32_pwm_round_waveform_fromhw(struct pwm_chip *chip,
 		u64 ccr_ns;
 
 		/* The result doesn't overflow for rate >= 15259 */
-		wf->period_length_ns = stm32_pwm_mul_u64_u64_div_u64_roundup(((u64)wfhw->psc + 1) * (wfhw->arr + 1),
-									     NSEC_PER_SEC, rate);
+		wf->period_length_ns = mul_u64_u64_div_u64_roundup(((u64)wfhw->psc + 1) * (wfhw->arr + 1),
+								   NSEC_PER_SEC, rate);
 
-		ccr_ns = stm32_pwm_mul_u64_u64_div_u64_roundup(((u64)wfhw->psc + 1) * wfhw->ccr,
-							       NSEC_PER_SEC, rate);
+		ccr_ns = mul_u64_u64_div_u64_roundup(((u64)wfhw->psc + 1) * wfhw->ccr, NSEC_PER_SEC, rate);
 
 		if (wfhw->ccer & TIM_CCER_CCxP(ch + 1)) {
 			wf->duty_length_ns =
-				stm32_pwm_mul_u64_u64_div_u64_roundup(((u64)wfhw->psc + 1) * (wfhw->arr + 1 - wfhw->ccr),
-								      NSEC_PER_SEC, rate);
+				mul_u64_u64_div_u64_roundup(((u64)wfhw->psc + 1) * (wfhw->arr + 1 - wfhw->ccr),
+							    NSEC_PER_SEC, rate);
 
 			wf->duty_offset_ns = ccr_ns;
 		} else {
diff --git a/drivers/rapidio/devices/tsi721.c b/drivers/rapidio/devices/tsi721.c
index 66331e67cf4ef..71b87bf8c31d8 100644
--- a/drivers/rapidio/devices/tsi721.c
+++ b/drivers/rapidio/devices/tsi721.c
@@ -394,7 +394,6 @@ static void tsi721_db_dpc(struct work_struct *work)
 						    idb_work);
 	struct rio_mport *mport;
 	struct rio_dbell *dbell;
-	int found = 0;
 	u32 wr_ptr, rd_ptr;
 	u64 *idb_entry;
 	u32 regval;
@@ -412,6 +411,8 @@ static void tsi721_db_dpc(struct work_struct *work)
 	rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE;
 
 	while (wr_ptr != rd_ptr) {
+		int found = 0;
+
 		idb_entry = (u64 *)(priv->idb_base +
 					(TSI721_IDB_ENTRY_SIZE * rd_ptr));
 		rd_ptr++;
diff --git a/drivers/rapidio/rio.c b/drivers/rapidio/rio.c
index 0c175e6d424fd..f45e58b0971f5 100644
--- a/drivers/rapidio/rio.c
+++ b/drivers/rapidio/rio.c
@@ -1994,7 +1994,7 @@ int rio_init_mports(void)
 	 * TODO: Implement restart of discovery process for all or
 	 * individual discovering mports.
 	 */
-	rio_wq = alloc_workqueue("riodisc", 0, 0);
+	rio_wq = alloc_workqueue("riodisc", WQ_PERCPU, 0);
 	if (!rio_wq) {
 		pr_err("RIO: unable allocate rio_wq\n");
 		goto no_disc;
diff --git a/drivers/ras/amd/atl/Kconfig b/drivers/ras/amd/atl/Kconfig
index 6e03942cd7dae..44c2fd7febc54 100644
--- a/drivers/ras/amd/atl/Kconfig
+++ b/drivers/ras/amd/atl/Kconfig
@@ -12,7 +12,6 @@ config AMD_ATL
 	depends on AMD_NB && X86_64 && RAS
 	depends on AMD_NODE
 	depends on MEMORY_FAILURE
-	default N
 	help
 	  This library includes support for implementation-specific
 	  address translation procedures needed for various error
diff --git a/drivers/ras/amd/fmpm.c b/drivers/ras/amd/fmpm.c
index 34ef75af31cbf..4ccaaf7b70bfd 100644
--- a/drivers/ras/amd/fmpm.c
+++ b/drivers/ras/amd/fmpm.c
@@ -52,6 +52,7 @@
 #include <acpi/apei.h>
 
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/mce.h>
 
 #include "../debugfs.h"
diff --git a/drivers/regulator/Kconfig b/drivers/regulator/Kconfig
index d71dac9436e3e..78076ac6eac4d 100644
--- a/drivers/regulator/Kconfig
+++ b/drivers/regulator/Kconfig
@@ -757,7 +757,7 @@ config REGULATOR_MAX20086
 	select REGMAP_I2C
 	help
 	  This driver controls a Maxim MAX20086-MAX20089 camera power
-	  protectorvia I2C bus. The regulator has 2 or 4 outputs depending on
+	  protector via I2C bus. The regulator has 2 or 4 outputs depending on
 	  the device model. This driver is only capable to turn on/off them.
 
 config REGULATOR_MAX20411
diff --git a/drivers/regulator/mt6359-regulator.c b/drivers/regulator/mt6359-regulator.c
index c8a7888588249..af0e0339fbdd6 100644
--- a/drivers/regulator/mt6359-regulator.c
+++ b/drivers/regulator/mt6359-regulator.c
@@ -38,7 +38,7 @@ struct mt6359_regulator_info {
 	u32 lp_mode_mask;
 };
 
-#define MT6359_BUCK(match, _name, min, max, step,		\
+#define MT6359_BUCK(match, _name, supply, min, max, step,	\
 	_enable_reg, _status_reg,				\
 	_vsel_reg, _vsel_mask,					\
 	_lp_mode_reg, _lp_mode_shift,				\
@@ -46,6 +46,7 @@ struct mt6359_regulator_info {
 [MT6359_ID_##_name] = {						\
 	.desc = {						\
 		.name = #_name,					\
+		.supply_name = supply,				\
 		.of_match = of_match_ptr(match),		\
 		.regulators_node = of_match_ptr("regulators"),	\
 		.ops = &mt6359_volt_linear_ops,			\
@@ -69,11 +70,12 @@ struct mt6359_regulator_info {
 	.modeset_mask = BIT(_modeset_shift),			\
 }
 
-#define MT6359_LDO_LINEAR(match, _name, min, max, step,		\
+#define MT6359_LDO_LINEAR(match, _name, supply, min, max, step,	\
 	_enable_reg, _status_reg, _vsel_reg, _vsel_mask)	\
 [MT6359_ID_##_name] = {						\
 	.desc = {						\
 		.name = #_name,					\
+		.supply_name = supply,				\
 		.of_match = of_match_ptr(match),		\
 		.regulators_node = of_match_ptr("regulators"),	\
 		.ops = &mt6359_volt_linear_ops,			\
@@ -92,12 +94,13 @@ struct mt6359_regulator_info {
 	.qi = BIT(0),						\
 }
 
-#define MT6359_LDO(match, _name, _volt_table,			\
+#define MT6359_LDO(match, _name, supply, _volt_table,		\
 	_enable_reg, _enable_mask, _status_reg,			\
 	_vsel_reg, _vsel_mask, _en_delay)			\
 [MT6359_ID_##_name] = {						\
 	.desc = {						\
 		.name = #_name,					\
+		.supply_name = supply,				\
 		.of_match = of_match_ptr(match),		\
 		.regulators_node = of_match_ptr("regulators"),	\
 		.ops = &mt6359_volt_table_ops,			\
@@ -116,11 +119,13 @@ struct mt6359_regulator_info {
 	.qi = BIT(0),						\
 }
 
-#define MT6359_REG_FIXED(match, _name, _enable_reg,	\
-	_status_reg, _fixed_volt)			\
+#define MT6359_REG_FIXED(match, _name, supply,		\
+			 _enable_reg, _status_reg,	\
+			 _fixed_volt)			\
 [MT6359_ID_##_name] = {					\
 	.desc = {					\
 		.name = #_name,				\
+		.supply_name = supply,			\
 		.of_match = of_match_ptr(match),	\
 		.regulators_node = of_match_ptr("regulators"),	\
 		.ops = &mt6359_volt_fixed_ops,		\
@@ -136,12 +141,14 @@ struct mt6359_regulator_info {
 	.qi = BIT(0),					\
 }
 
-#define MT6359P_LDO1(match, _name, _ops, _volt_table,	\
-	_enable_reg, _enable_mask, _status_reg,		\
-	_vsel_reg, _vsel_mask)				\
+#define MT6359P_LDO1(match, _name, supply, _ops,	\
+		     _volt_table, _enable_reg,		\
+		     _enable_mask, _status_reg,		\
+		     _vsel_reg, _vsel_mask)		\
 [MT6359_ID_##_name] = {					\
 	.desc = {					\
 		.name = #_name,				\
+		.supply_name = supply,			\
 		.of_match = of_match_ptr(match),	\
 		.regulators_node = of_match_ptr("regulators"),	\
 		.ops = &_ops,				\
@@ -159,6 +166,20 @@ struct mt6359_regulator_info {
 	.qi = BIT(0),					\
 }
 
+#define MT6359_LDO_NOOP(match, _name, supply)		\
+[MT6359_ID_##_name] = {					\
+	.desc = {					\
+		.name = #_name,				\
+		.supply_name = supply,			\
+		.of_match = of_match_ptr(match),	\
+		.regulators_node = of_match_ptr("regulators"),	\
+		.ops = &mt6359_noop_ops,		\
+		.type = REGULATOR_VOLTAGE,		\
+		.id = MT6359_ID_##_name,		\
+		.owner = THIS_MODULE,			\
+	},						\
+}
+
 static const unsigned int vsim1_voltages[] = {
 	0, 0, 0, 1700000, 1800000, 0, 0, 0, 2700000, 0, 0, 3000000, 3100000,
 };
@@ -251,7 +272,7 @@ static int mt6359_get_status(struct regulator_dev *rdev)
 {
 	int ret;
 	u32 regval;
-	struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
+	const struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
 
 	ret = regmap_read(rdev->regmap, info->status_reg, &regval);
 	if (ret != 0) {
@@ -267,7 +288,7 @@ static int mt6359_get_status(struct regulator_dev *rdev)
 
 static unsigned int mt6359_regulator_get_mode(struct regulator_dev *rdev)
 {
-	struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
+	const struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
 	int ret, regval;
 
 	ret = regmap_read(rdev->regmap, info->modeset_reg, &regval);
@@ -299,7 +320,7 @@ static unsigned int mt6359_regulator_get_mode(struct regulator_dev *rdev)
 static int mt6359_regulator_set_mode(struct regulator_dev *rdev,
 				     unsigned int mode)
 {
-	struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
+	const struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
 	int ret = 0, val;
 	int curr_mode;
 
@@ -354,7 +375,7 @@ static int mt6359_regulator_set_mode(struct regulator_dev *rdev,
 static int mt6359p_vemc_set_voltage_sel(struct regulator_dev *rdev,
 					u32 sel)
 {
-	struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
+	const struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
 	int ret;
 	u32 val = 0;
 
@@ -393,7 +414,7 @@ static int mt6359p_vemc_set_voltage_sel(struct regulator_dev *rdev,
 
 static int mt6359p_vemc_get_voltage_sel(struct regulator_dev *rdev)
 {
-	struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
+	const struct mt6359_regulator_info *info = rdev_get_drvdata(rdev);
 	int ret;
 	u32 val = 0;
 
@@ -468,16 +489,19 @@ static const struct regulator_ops mt6359p_vemc_ops = {
 	.get_status = mt6359_get_status,
 };
 
+/* Used for backward-compatible placeholder regulators */
+static const struct regulator_ops mt6359_noop_ops = {};
+
 /* The array is indexed by id(MT6359_ID_XXX) */
-static struct mt6359_regulator_info mt6359_regulators[] = {
-	MT6359_BUCK("buck_vs1", VS1, 800000, 2200000, 12500,
+static const struct mt6359_regulator_info mt6359_regulators[] = {
+	MT6359_BUCK("buck_vs1", VS1, "vsys-vs1", 800000, 2200000, 12500,
 		    MT6359_RG_BUCK_VS1_EN_ADDR,
 		    MT6359_DA_VS1_EN_ADDR, MT6359_RG_BUCK_VS1_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VS1_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VS1_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VS1_LP_ADDR, MT6359_RG_BUCK_VS1_LP_SHIFT,
 		    MT6359_RG_VS1_FPWM_ADDR, MT6359_RG_VS1_FPWM_SHIFT),
-	MT6359_BUCK("buck_vgpu11", VGPU11, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vgpu11", VGPU11, "vsys-vgpu11", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VGPU11_EN_ADDR,
 		    MT6359_DA_VGPU11_EN_ADDR, MT6359_RG_BUCK_VGPU11_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VGPU11_VOSEL_MASK <<
@@ -485,7 +509,7 @@ static struct mt6359_regulator_info mt6359_regulators[] = {
 		    MT6359_RG_BUCK_VGPU11_LP_ADDR,
 		    MT6359_RG_BUCK_VGPU11_LP_SHIFT,
 		    MT6359_RG_VGPU11_FCCM_ADDR, MT6359_RG_VGPU11_FCCM_SHIFT),
-	MT6359_BUCK("buck_vmodem", VMODEM, 400000, 1100000, 6250,
+	MT6359_BUCK("buck_vmodem", VMODEM, "vsys-vmodem", 400000, 1100000, 6250,
 		    MT6359_RG_BUCK_VMODEM_EN_ADDR,
 		    MT6359_DA_VMODEM_EN_ADDR, MT6359_RG_BUCK_VMODEM_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VMODEM_VOSEL_MASK <<
@@ -493,35 +517,35 @@ static struct mt6359_regulator_info mt6359_regulators[] = {
 		    MT6359_RG_BUCK_VMODEM_LP_ADDR,
 		    MT6359_RG_BUCK_VMODEM_LP_SHIFT,
 		    MT6359_RG_VMODEM_FCCM_ADDR, MT6359_RG_VMODEM_FCCM_SHIFT),
-	MT6359_BUCK("buck_vpu", VPU, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vpu", VPU, "vsys-vpu", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VPU_EN_ADDR,
 		    MT6359_DA_VPU_EN_ADDR, MT6359_RG_BUCK_VPU_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VPU_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VPU_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VPU_LP_ADDR, MT6359_RG_BUCK_VPU_LP_SHIFT,
 		    MT6359_RG_VPU_FCCM_ADDR, MT6359_RG_VPU_FCCM_SHIFT),
-	MT6359_BUCK("buck_vcore", VCORE, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vcore", VCORE, "vsys-vcore", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VCORE_EN_ADDR,
 		    MT6359_DA_VCORE_EN_ADDR, MT6359_RG_BUCK_VCORE_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VCORE_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VCORE_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VCORE_LP_ADDR, MT6359_RG_BUCK_VCORE_LP_SHIFT,
 		    MT6359_RG_VCORE_FCCM_ADDR, MT6359_RG_VCORE_FCCM_SHIFT),
-	MT6359_BUCK("buck_vs2", VS2, 800000, 1600000, 12500,
+	MT6359_BUCK("buck_vs2", VS2, "vsys-vs2", 800000, 1600000, 12500,
 		    MT6359_RG_BUCK_VS2_EN_ADDR,
 		    MT6359_DA_VS2_EN_ADDR, MT6359_RG_BUCK_VS2_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VS2_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VS2_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VS2_LP_ADDR, MT6359_RG_BUCK_VS2_LP_SHIFT,
 		    MT6359_RG_VS2_FPWM_ADDR, MT6359_RG_VS2_FPWM_SHIFT),
-	MT6359_BUCK("buck_vpa", VPA, 500000, 3650000, 50000,
+	MT6359_BUCK("buck_vpa", VPA, "vsys-vpa", 500000, 3650000, 50000,
 		    MT6359_RG_BUCK_VPA_EN_ADDR,
 		    MT6359_DA_VPA_EN_ADDR, MT6359_RG_BUCK_VPA_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VPA_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VPA_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VPA_LP_ADDR, MT6359_RG_BUCK_VPA_LP_SHIFT,
 		    MT6359_RG_VPA_MODESET_ADDR, MT6359_RG_VPA_MODESET_SHIFT),
-	MT6359_BUCK("buck_vproc2", VPROC2, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vproc2", VPROC2, "vsys-vproc2", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VPROC2_EN_ADDR,
 		    MT6359_DA_VPROC2_EN_ADDR, MT6359_RG_BUCK_VPROC2_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VPROC2_VOSEL_MASK <<
@@ -529,7 +553,7 @@ static struct mt6359_regulator_info mt6359_regulators[] = {
 		    MT6359_RG_BUCK_VPROC2_LP_ADDR,
 		    MT6359_RG_BUCK_VPROC2_LP_SHIFT,
 		    MT6359_RG_VPROC2_FCCM_ADDR, MT6359_RG_VPROC2_FCCM_SHIFT),
-	MT6359_BUCK("buck_vproc1", VPROC1, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vproc1", VPROC1, "vsys-vproc1", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VPROC1_EN_ADDR,
 		    MT6359_DA_VPROC1_EN_ADDR, MT6359_RG_BUCK_VPROC1_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VPROC1_VOSEL_MASK <<
@@ -537,7 +561,7 @@ static struct mt6359_regulator_info mt6359_regulators[] = {
 		    MT6359_RG_BUCK_VPROC1_LP_ADDR,
 		    MT6359_RG_BUCK_VPROC1_LP_SHIFT,
 		    MT6359_RG_VPROC1_FCCM_ADDR, MT6359_RG_VPROC1_FCCM_SHIFT),
-	MT6359_BUCK("buck_vcore_sshub", VCORE_SSHUB, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vcore_sshub", VCORE_SSHUB, "vsys-vcore", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VCORE_SSHUB_EN_ADDR,
 		    MT6359_DA_VCORE_EN_ADDR,
 		    MT6359_RG_BUCK_VCORE_SSHUB_VOSEL_ADDR,
@@ -545,175 +569,169 @@ static struct mt6359_regulator_info mt6359_regulators[] = {
 		    MT6359_RG_BUCK_VCORE_SSHUB_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VCORE_LP_ADDR, MT6359_RG_BUCK_VCORE_LP_SHIFT,
 		    MT6359_RG_VCORE_FCCM_ADDR, MT6359_RG_VCORE_FCCM_SHIFT),
-	MT6359_REG_FIXED("ldo_vaud18", VAUD18, MT6359_RG_LDO_VAUD18_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vaud18", VAUD18, "vs1-ldo1", MT6359_RG_LDO_VAUD18_EN_ADDR,
 			 MT6359_DA_VAUD18_B_EN_ADDR, 1800000),
-	MT6359_LDO("ldo_vsim1", VSIM1, vsim1_voltages,
+	MT6359_LDO("ldo_vsim1", VSIM1, "vsys-ldo2", vsim1_voltages,
 		   MT6359_RG_LDO_VSIM1_EN_ADDR, MT6359_RG_LDO_VSIM1_EN_SHIFT,
 		   MT6359_DA_VSIM1_B_EN_ADDR, MT6359_RG_VSIM1_VOSEL_ADDR,
 		   MT6359_RG_VSIM1_VOSEL_MASK << MT6359_RG_VSIM1_VOSEL_SHIFT,
 		   480),
-	MT6359_LDO("ldo_vibr", VIBR, vibr_voltages,
+	MT6359_LDO("ldo_vibr", VIBR, "vsys-ldo1", vibr_voltages,
 		   MT6359_RG_LDO_VIBR_EN_ADDR, MT6359_RG_LDO_VIBR_EN_SHIFT,
 		   MT6359_DA_VIBR_B_EN_ADDR, MT6359_RG_VIBR_VOSEL_ADDR,
 		   MT6359_RG_VIBR_VOSEL_MASK << MT6359_RG_VIBR_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_vrf12", VRF12, vrf12_voltages,
+	MT6359_LDO("ldo_vrf12", VRF12, "vs2-ldo2", vrf12_voltages,
 		   MT6359_RG_LDO_VRF12_EN_ADDR, MT6359_RG_LDO_VRF12_EN_SHIFT,
 		   MT6359_DA_VRF12_B_EN_ADDR, MT6359_RG_VRF12_VOSEL_ADDR,
 		   MT6359_RG_VRF12_VOSEL_MASK << MT6359_RG_VRF12_VOSEL_SHIFT,
 		   120),
-	MT6359_REG_FIXED("ldo_vusb", VUSB, MT6359_RG_LDO_VUSB_EN_0_ADDR,
+	MT6359_REG_FIXED("ldo_vusb", VUSB, "vsys-ldo2", MT6359_RG_LDO_VUSB_EN_0_ADDR,
 			 MT6359_DA_VUSB_B_EN_ADDR, 3000000),
-	MT6359_LDO_LINEAR("ldo_vsram_proc2", VSRAM_PROC2, 500000, 1293750, 6250,
+	MT6359_LDO_LINEAR("ldo_vsram_proc2", VSRAM_PROC2, "vs2-ldo1", 500000, 1293750, 6250,
 			  MT6359_RG_LDO_VSRAM_PROC2_EN_ADDR,
 			  MT6359_DA_VSRAM_PROC2_B_EN_ADDR,
 			  MT6359_RG_LDO_VSRAM_PROC2_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_PROC2_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_PROC2_VOSEL_SHIFT),
-	MT6359_LDO("ldo_vio18", VIO18, volt18_voltages,
+	MT6359_LDO("ldo_vio18", VIO18, "vs1-ldo2", volt18_voltages,
 		   MT6359_RG_LDO_VIO18_EN_ADDR, MT6359_RG_LDO_VIO18_EN_SHIFT,
 		   MT6359_DA_VIO18_B_EN_ADDR, MT6359_RG_VIO18_VOSEL_ADDR,
 		   MT6359_RG_VIO18_VOSEL_MASK << MT6359_RG_VIO18_VOSEL_SHIFT,
 		   960),
-	MT6359_LDO("ldo_vcamio", VCAMIO, volt18_voltages,
+	MT6359_LDO("ldo_vcamio", VCAMIO, "vs1-ldo1", volt18_voltages,
 		   MT6359_RG_LDO_VCAMIO_EN_ADDR, MT6359_RG_LDO_VCAMIO_EN_SHIFT,
 		   MT6359_DA_VCAMIO_B_EN_ADDR, MT6359_RG_VCAMIO_VOSEL_ADDR,
 		   MT6359_RG_VCAMIO_VOSEL_MASK << MT6359_RG_VCAMIO_VOSEL_SHIFT,
 		   1290),
-	MT6359_REG_FIXED("ldo_vcn18", VCN18, MT6359_RG_LDO_VCN18_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vcn18", VCN18, "vs1-ldo2", MT6359_RG_LDO_VCN18_EN_ADDR,
 			 MT6359_DA_VCN18_B_EN_ADDR, 1800000),
-	MT6359_REG_FIXED("ldo_vfe28", VFE28, MT6359_RG_LDO_VFE28_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vfe28", VFE28, "vsys-ldo1", MT6359_RG_LDO_VFE28_EN_ADDR,
 			 MT6359_DA_VFE28_B_EN_ADDR, 2800000),
-	MT6359_LDO("ldo_vcn13", VCN13, vcn13_voltages,
+	MT6359_LDO("ldo_vcn13", VCN13, "vs2-ldo2", vcn13_voltages,
 		   MT6359_RG_LDO_VCN13_EN_ADDR, MT6359_RG_LDO_VCN13_EN_SHIFT,
 		   MT6359_DA_VCN13_B_EN_ADDR, MT6359_RG_VCN13_VOSEL_ADDR,
 		   MT6359_RG_VCN13_VOSEL_MASK << MT6359_RG_VCN13_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_vcn33_1_bt", VCN33_1_BT, vcn33_voltages,
+	MT6359_LDO("ldo_vcn33_1", VCN33_1, "vsys-ldo1", vcn33_voltages,
 		   MT6359_RG_LDO_VCN33_1_EN_0_ADDR,
 		   MT6359_RG_LDO_VCN33_1_EN_0_SHIFT,
 		   MT6359_DA_VCN33_1_B_EN_ADDR, MT6359_RG_VCN33_1_VOSEL_ADDR,
 		   MT6359_RG_VCN33_1_VOSEL_MASK <<
 		   MT6359_RG_VCN33_1_VOSEL_SHIFT, 240),
-	MT6359_LDO("ldo_vcn33_1_wifi", VCN33_1_WIFI, vcn33_voltages,
-		   MT6359_RG_LDO_VCN33_1_EN_1_ADDR,
-		   MT6359_RG_LDO_VCN33_1_EN_1_SHIFT,
-		   MT6359_DA_VCN33_1_B_EN_ADDR, MT6359_RG_VCN33_1_VOSEL_ADDR,
-		   MT6359_RG_VCN33_1_VOSEL_MASK <<
-		   MT6359_RG_VCN33_1_VOSEL_SHIFT, 240),
-	MT6359_REG_FIXED("ldo_vaux18", VAUX18, MT6359_RG_LDO_VAUX18_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vaux18", VAUX18, "vsys-ldo2", MT6359_RG_LDO_VAUX18_EN_ADDR,
 			 MT6359_DA_VAUX18_B_EN_ADDR, 1800000),
-	MT6359_LDO_LINEAR("ldo_vsram_others", VSRAM_OTHERS, 500000, 1293750,
+	MT6359_LDO_LINEAR("ldo_vsram_others", VSRAM_OTHERS, "vs2-ldo1", 500000, 1293750,
 			  6250,
 			  MT6359_RG_LDO_VSRAM_OTHERS_EN_ADDR,
 			  MT6359_DA_VSRAM_OTHERS_B_EN_ADDR,
 			  MT6359_RG_LDO_VSRAM_OTHERS_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_OTHERS_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_OTHERS_VOSEL_SHIFT),
-	MT6359_LDO("ldo_vefuse", VEFUSE, vefuse_voltages,
+	MT6359_LDO("ldo_vefuse", VEFUSE, "vs1-ldo2", vefuse_voltages,
 		   MT6359_RG_LDO_VEFUSE_EN_ADDR, MT6359_RG_LDO_VEFUSE_EN_SHIFT,
 		   MT6359_DA_VEFUSE_B_EN_ADDR, MT6359_RG_VEFUSE_VOSEL_ADDR,
 		   MT6359_RG_VEFUSE_VOSEL_MASK << MT6359_RG_VEFUSE_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_vxo22", VXO22, vxo22_voltages,
+	MT6359_LDO("ldo_vxo22", VXO22, "vsys-ldo2", vxo22_voltages,
 		   MT6359_RG_LDO_VXO22_EN_ADDR, MT6359_RG_LDO_VXO22_EN_SHIFT,
 		   MT6359_DA_VXO22_B_EN_ADDR, MT6359_RG_VXO22_VOSEL_ADDR,
 		   MT6359_RG_VXO22_VOSEL_MASK << MT6359_RG_VXO22_VOSEL_SHIFT,
 		   120),
-	MT6359_LDO("ldo_vrfck", VRFCK, vrfck_voltages,
+	MT6359_LDO("ldo_vrfck", VRFCK, "vsys-ldo2", vrfck_voltages,
 		   MT6359_RG_LDO_VRFCK_EN_ADDR, MT6359_RG_LDO_VRFCK_EN_SHIFT,
 		   MT6359_DA_VRFCK_B_EN_ADDR, MT6359_RG_VRFCK_VOSEL_ADDR,
 		   MT6359_RG_VRFCK_VOSEL_MASK << MT6359_RG_VRFCK_VOSEL_SHIFT,
 		   480),
-	MT6359_REG_FIXED("ldo_vbif28", VBIF28, MT6359_RG_LDO_VBIF28_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vbif28", VBIF28, "vsys-ldo2", MT6359_RG_LDO_VBIF28_EN_ADDR,
 			 MT6359_DA_VBIF28_B_EN_ADDR, 2800000),
-	MT6359_LDO("ldo_vio28", VIO28, vio28_voltages,
+	MT6359_LDO("ldo_vio28", VIO28, "vsys-ldo2", vio28_voltages,
 		   MT6359_RG_LDO_VIO28_EN_ADDR, MT6359_RG_LDO_VIO28_EN_SHIFT,
 		   MT6359_DA_VIO28_B_EN_ADDR, MT6359_RG_VIO28_VOSEL_ADDR,
 		   MT6359_RG_VIO28_VOSEL_MASK << MT6359_RG_VIO28_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_vemc", VEMC, vemc_voltages,
+	MT6359_LDO("ldo_vemc", VEMC, "vsys-ldo2", vemc_voltages,
 		   MT6359_RG_LDO_VEMC_EN_ADDR, MT6359_RG_LDO_VEMC_EN_SHIFT,
 		   MT6359_DA_VEMC_B_EN_ADDR, MT6359_RG_VEMC_VOSEL_ADDR,
 		   MT6359_RG_VEMC_VOSEL_MASK << MT6359_RG_VEMC_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_vcn33_2_bt", VCN33_2_BT, vcn33_voltages,
+	MT6359_LDO("ldo_vcn33_2", VCN33_2, "vsys-ldo1", vcn33_voltages,
 		   MT6359_RG_LDO_VCN33_2_EN_0_ADDR,
 		   MT6359_RG_LDO_VCN33_2_EN_0_SHIFT,
 		   MT6359_DA_VCN33_2_B_EN_ADDR, MT6359_RG_VCN33_2_VOSEL_ADDR,
 		   MT6359_RG_VCN33_2_VOSEL_MASK <<
 		   MT6359_RG_VCN33_2_VOSEL_SHIFT, 240),
-	MT6359_LDO("ldo_vcn33_2_wifi", VCN33_2_WIFI, vcn33_voltages,
-		   MT6359_RG_LDO_VCN33_2_EN_1_ADDR,
-		   MT6359_RG_LDO_VCN33_2_EN_1_SHIFT,
-		   MT6359_DA_VCN33_2_B_EN_ADDR, MT6359_RG_VCN33_2_VOSEL_ADDR,
-		   MT6359_RG_VCN33_2_VOSEL_MASK <<
-		   MT6359_RG_VCN33_2_VOSEL_SHIFT, 240),
-	MT6359_LDO("ldo_va12", VA12, va12_voltages,
+	MT6359_LDO("ldo_va12", VA12, "vs2-ldo2", va12_voltages,
 		   MT6359_RG_LDO_VA12_EN_ADDR, MT6359_RG_LDO_VA12_EN_SHIFT,
 		   MT6359_DA_VA12_B_EN_ADDR, MT6359_RG_VA12_VOSEL_ADDR,
 		   MT6359_RG_VA12_VOSEL_MASK << MT6359_RG_VA12_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_va09", VA09, va09_voltages,
+	MT6359_LDO("ldo_va09", VA09, "vs2-ldo2", va09_voltages,
 		   MT6359_RG_LDO_VA09_EN_ADDR, MT6359_RG_LDO_VA09_EN_SHIFT,
 		   MT6359_DA_VA09_B_EN_ADDR, MT6359_RG_VA09_VOSEL_ADDR,
 		   MT6359_RG_VA09_VOSEL_MASK << MT6359_RG_VA09_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_vrf18", VRF18, vrf18_voltages,
+	MT6359_LDO("ldo_vrf18", VRF18, "vs1-ldo2", vrf18_voltages,
 		   MT6359_RG_LDO_VRF18_EN_ADDR, MT6359_RG_LDO_VRF18_EN_SHIFT,
 		   MT6359_DA_VRF18_B_EN_ADDR, MT6359_RG_VRF18_VOSEL_ADDR,
 		   MT6359_RG_VRF18_VOSEL_MASK << MT6359_RG_VRF18_VOSEL_SHIFT,
 		   120),
-	MT6359_LDO_LINEAR("ldo_vsram_md", VSRAM_MD, 500000, 1100000, 6250,
+	MT6359_LDO_LINEAR("ldo_vsram_md", VSRAM_MD, "vs2-ldo1", 500000, 1100000, 6250,
 			  MT6359_RG_LDO_VSRAM_MD_EN_ADDR,
 			  MT6359_DA_VSRAM_MD_B_EN_ADDR,
 			  MT6359_RG_LDO_VSRAM_MD_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_MD_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_MD_VOSEL_SHIFT),
-	MT6359_LDO("ldo_vufs", VUFS, volt18_voltages,
+	MT6359_LDO("ldo_vufs", VUFS, "vs1-ldo1", volt18_voltages,
 		   MT6359_RG_LDO_VUFS_EN_ADDR, MT6359_RG_LDO_VUFS_EN_SHIFT,
 		   MT6359_DA_VUFS_B_EN_ADDR, MT6359_RG_VUFS_VOSEL_ADDR,
 		   MT6359_RG_VUFS_VOSEL_MASK << MT6359_RG_VUFS_VOSEL_SHIFT,
 		   1920),
-	MT6359_LDO("ldo_vm18", VM18, volt18_voltages,
+	MT6359_LDO("ldo_vm18", VM18, "vs1-ldo1", volt18_voltages,
 		   MT6359_RG_LDO_VM18_EN_ADDR, MT6359_RG_LDO_VM18_EN_SHIFT,
 		   MT6359_DA_VM18_B_EN_ADDR, MT6359_RG_VM18_VOSEL_ADDR,
 		   MT6359_RG_VM18_VOSEL_MASK << MT6359_RG_VM18_VOSEL_SHIFT,
 		   1920),
-	MT6359_LDO("ldo_vbbck", VBBCK, vbbck_voltages,
+	/* vbbck is fed from vio18 internally. */
+	MT6359_LDO("ldo_vbbck", VBBCK, "VIO18", vbbck_voltages,
 		   MT6359_RG_LDO_VBBCK_EN_ADDR, MT6359_RG_LDO_VBBCK_EN_SHIFT,
 		   MT6359_DA_VBBCK_B_EN_ADDR, MT6359_RG_VBBCK_VOSEL_ADDR,
 		   MT6359_RG_VBBCK_VOSEL_MASK << MT6359_RG_VBBCK_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO_LINEAR("ldo_vsram_proc1", VSRAM_PROC1, 500000, 1293750, 6250,
+	MT6359_LDO_LINEAR("ldo_vsram_proc1", VSRAM_PROC1, "vs2-ldo1", 500000, 1293750, 6250,
 			  MT6359_RG_LDO_VSRAM_PROC1_EN_ADDR,
 			  MT6359_DA_VSRAM_PROC1_B_EN_ADDR,
 			  MT6359_RG_LDO_VSRAM_PROC1_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_PROC1_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_PROC1_VOSEL_SHIFT),
-	MT6359_LDO("ldo_vsim2", VSIM2, vsim2_voltages,
+	MT6359_LDO("ldo_vsim2", VSIM2, "vsys-ldo2", vsim2_voltages,
 		   MT6359_RG_LDO_VSIM2_EN_ADDR, MT6359_RG_LDO_VSIM2_EN_SHIFT,
 		   MT6359_DA_VSIM2_B_EN_ADDR, MT6359_RG_VSIM2_VOSEL_ADDR,
 		   MT6359_RG_VSIM2_VOSEL_MASK << MT6359_RG_VSIM2_VOSEL_SHIFT,
 		   480),
-	MT6359_LDO_LINEAR("ldo_vsram_others_sshub", VSRAM_OTHERS_SSHUB,
+	MT6359_LDO_LINEAR("ldo_vsram_others_sshub", VSRAM_OTHERS_SSHUB, "vs2-ldo1",
 			  500000, 1293750, 6250,
 			  MT6359_RG_LDO_VSRAM_OTHERS_SSHUB_EN_ADDR,
 			  MT6359_DA_VSRAM_OTHERS_B_EN_ADDR,
 			  MT6359_RG_LDO_VSRAM_OTHERS_SSHUB_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_OTHERS_SSHUB_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_OTHERS_SSHUB_VOSEL_SHIFT),
+	/* Placeholders for DT backward compatibility */
+	MT6359_LDO_NOOP("ldo_vcn33_1_bt",   VCN33_1_BT,   "LDO_VCN33_1"),
+	MT6359_LDO_NOOP("ldo_vcn33_1_wifi", VCN33_1_WIFI, "LDO_VCN33_1"),
+	MT6359_LDO_NOOP("ldo_vcn33_2_bt",   VCN33_2_BT,   "LDO_VCN33_2"),
+	MT6359_LDO_NOOP("ldo_vcn33_2_wifi", VCN33_2_WIFI, "LDO_VCN33_2"),
 };
 
-static struct mt6359_regulator_info mt6359p_regulators[] = {
-	MT6359_BUCK("buck_vs1", VS1, 800000, 2200000, 12500,
+static const struct mt6359_regulator_info mt6359p_regulators[] = {
+	MT6359_BUCK("buck_vs1", VS1, "vsys-vs1", 800000, 2200000, 12500,
 		    MT6359_RG_BUCK_VS1_EN_ADDR,
 		    MT6359_DA_VS1_EN_ADDR, MT6359_RG_BUCK_VS1_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VS1_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VS1_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VS1_LP_ADDR, MT6359_RG_BUCK_VS1_LP_SHIFT,
 		    MT6359_RG_VS1_FPWM_ADDR, MT6359_RG_VS1_FPWM_SHIFT),
-	MT6359_BUCK("buck_vgpu11", VGPU11, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vgpu11", VGPU11, "vsys-vgpu11", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VGPU11_EN_ADDR,
 		    MT6359_DA_VGPU11_EN_ADDR, MT6359P_RG_BUCK_VGPU11_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VGPU11_VOSEL_MASK <<
@@ -721,7 +739,7 @@ static struct mt6359_regulator_info mt6359p_regulators[] = {
 		    MT6359_RG_BUCK_VGPU11_LP_ADDR,
 		    MT6359_RG_BUCK_VGPU11_LP_SHIFT,
 		    MT6359_RG_VGPU11_FCCM_ADDR, MT6359_RG_VGPU11_FCCM_SHIFT),
-	MT6359_BUCK("buck_vmodem", VMODEM, 400000, 1100000, 6250,
+	MT6359_BUCK("buck_vmodem", VMODEM, "vsys-vmodem", 400000, 1100000, 6250,
 		    MT6359_RG_BUCK_VMODEM_EN_ADDR,
 		    MT6359_DA_VMODEM_EN_ADDR, MT6359_RG_BUCK_VMODEM_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VMODEM_VOSEL_MASK <<
@@ -729,35 +747,35 @@ static struct mt6359_regulator_info mt6359p_regulators[] = {
 		    MT6359_RG_BUCK_VMODEM_LP_ADDR,
 		    MT6359_RG_BUCK_VMODEM_LP_SHIFT,
 		    MT6359_RG_VMODEM_FCCM_ADDR, MT6359_RG_VMODEM_FCCM_SHIFT),
-	MT6359_BUCK("buck_vpu", VPU, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vpu", VPU, "vsys-vpu", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VPU_EN_ADDR,
 		    MT6359_DA_VPU_EN_ADDR, MT6359_RG_BUCK_VPU_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VPU_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VPU_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VPU_LP_ADDR, MT6359_RG_BUCK_VPU_LP_SHIFT,
 		    MT6359_RG_VPU_FCCM_ADDR, MT6359_RG_VPU_FCCM_SHIFT),
-	MT6359_BUCK("buck_vcore", VCORE, 506250, 1300000, 6250,
+	MT6359_BUCK("buck_vcore", VCORE, "vsys-vcore", 506250, 1300000, 6250,
 		    MT6359_RG_BUCK_VCORE_EN_ADDR,
 		    MT6359_DA_VCORE_EN_ADDR, MT6359P_RG_BUCK_VCORE_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VCORE_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VCORE_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VCORE_LP_ADDR, MT6359_RG_BUCK_VCORE_LP_SHIFT,
 		    MT6359_RG_VCORE_FCCM_ADDR, MT6359_RG_VCORE_FCCM_SHIFT),
-	MT6359_BUCK("buck_vs2", VS2, 800000, 1600000, 12500,
+	MT6359_BUCK("buck_vs2", VS2, "vsys-vs2", 800000, 1600000, 12500,
 		    MT6359_RG_BUCK_VS2_EN_ADDR,
 		    MT6359_DA_VS2_EN_ADDR, MT6359_RG_BUCK_VS2_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VS2_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VS2_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VS2_LP_ADDR, MT6359_RG_BUCK_VS2_LP_SHIFT,
 		    MT6359_RG_VS2_FPWM_ADDR, MT6359_RG_VS2_FPWM_SHIFT),
-	MT6359_BUCK("buck_vpa", VPA, 500000, 3650000, 50000,
+	MT6359_BUCK("buck_vpa", VPA, "vsys-vpa", 500000, 3650000, 50000,
 		    MT6359_RG_BUCK_VPA_EN_ADDR,
 		    MT6359_DA_VPA_EN_ADDR, MT6359_RG_BUCK_VPA_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VPA_VOSEL_MASK <<
 		    MT6359_RG_BUCK_VPA_VOSEL_SHIFT,
 		    MT6359_RG_BUCK_VPA_LP_ADDR, MT6359_RG_BUCK_VPA_LP_SHIFT,
 		    MT6359_RG_VPA_MODESET_ADDR, MT6359_RG_VPA_MODESET_SHIFT),
-	MT6359_BUCK("buck_vproc2", VPROC2, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vproc2", VPROC2, "vsys-vproc2", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VPROC2_EN_ADDR,
 		    MT6359_DA_VPROC2_EN_ADDR, MT6359_RG_BUCK_VPROC2_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VPROC2_VOSEL_MASK <<
@@ -765,7 +783,7 @@ static struct mt6359_regulator_info mt6359p_regulators[] = {
 		    MT6359_RG_BUCK_VPROC2_LP_ADDR,
 		    MT6359_RG_BUCK_VPROC2_LP_SHIFT,
 		    MT6359_RG_VPROC2_FCCM_ADDR, MT6359_RG_VPROC2_FCCM_SHIFT),
-	MT6359_BUCK("buck_vproc1", VPROC1, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vproc1", VPROC1, "vsys-vproc1", 400000, 1193750, 6250,
 		    MT6359_RG_BUCK_VPROC1_EN_ADDR,
 		    MT6359_DA_VPROC1_EN_ADDR, MT6359_RG_BUCK_VPROC1_VOSEL_ADDR,
 		    MT6359_RG_BUCK_VPROC1_VOSEL_MASK <<
@@ -773,7 +791,7 @@ static struct mt6359_regulator_info mt6359p_regulators[] = {
 		    MT6359_RG_BUCK_VPROC1_LP_ADDR,
 		    MT6359_RG_BUCK_VPROC1_LP_SHIFT,
 		    MT6359_RG_VPROC1_FCCM_ADDR, MT6359_RG_VPROC1_FCCM_SHIFT),
-	MT6359_BUCK("buck_vgpu11_sshub", VGPU11_SSHUB, 400000, 1193750, 6250,
+	MT6359_BUCK("buck_vgpu11_sshub", VGPU11_SSHUB, "vsys-vgpu11", 400000, 1193750, 6250,
 		    MT6359P_RG_BUCK_VGPU11_SSHUB_EN_ADDR,
 		    MT6359_DA_VGPU11_EN_ADDR,
 		    MT6359P_RG_BUCK_VGPU11_SSHUB_VOSEL_ADDR,
@@ -782,195 +800,330 @@ static struct mt6359_regulator_info mt6359p_regulators[] = {
 		    MT6359_RG_BUCK_VGPU11_LP_ADDR,
 		    MT6359_RG_BUCK_VGPU11_LP_SHIFT,
 		    MT6359_RG_VGPU11_FCCM_ADDR, MT6359_RG_VGPU11_FCCM_SHIFT),
-	MT6359_REG_FIXED("ldo_vaud18", VAUD18, MT6359P_RG_LDO_VAUD18_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vaud18", VAUD18, "vs1-ldo1", MT6359P_RG_LDO_VAUD18_EN_ADDR,
 			 MT6359P_DA_VAUD18_B_EN_ADDR, 1800000),
-	MT6359_LDO("ldo_vsim1", VSIM1, vsim1_voltages,
+	MT6359_LDO("ldo_vsim1", VSIM1, "vsys-ldo2", vsim1_voltages,
 		   MT6359P_RG_LDO_VSIM1_EN_ADDR, MT6359P_RG_LDO_VSIM1_EN_SHIFT,
 		   MT6359P_DA_VSIM1_B_EN_ADDR, MT6359P_RG_VSIM1_VOSEL_ADDR,
 		   MT6359_RG_VSIM1_VOSEL_MASK << MT6359_RG_VSIM1_VOSEL_SHIFT,
 		   480),
-	MT6359_LDO("ldo_vibr", VIBR, vibr_voltages,
+	MT6359_LDO("ldo_vibr", VIBR, "vsys-ldo1", vibr_voltages,
 		   MT6359P_RG_LDO_VIBR_EN_ADDR, MT6359P_RG_LDO_VIBR_EN_SHIFT,
 		   MT6359P_DA_VIBR_B_EN_ADDR, MT6359P_RG_VIBR_VOSEL_ADDR,
 		   MT6359_RG_VIBR_VOSEL_MASK << MT6359_RG_VIBR_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_vrf12", VRF12, vrf12_voltages,
+	MT6359_LDO("ldo_vrf12", VRF12, "vs2-ldo2", vrf12_voltages,
 		   MT6359P_RG_LDO_VRF12_EN_ADDR, MT6359P_RG_LDO_VRF12_EN_SHIFT,
 		   MT6359P_DA_VRF12_B_EN_ADDR, MT6359P_RG_VRF12_VOSEL_ADDR,
 		   MT6359_RG_VRF12_VOSEL_MASK << MT6359_RG_VRF12_VOSEL_SHIFT,
 		   480),
-	MT6359_REG_FIXED("ldo_vusb", VUSB, MT6359P_RG_LDO_VUSB_EN_0_ADDR,
+	MT6359_REG_FIXED("ldo_vusb", VUSB, "vsys-ldo2", MT6359P_RG_LDO_VUSB_EN_0_ADDR,
 			 MT6359P_DA_VUSB_B_EN_ADDR, 3000000),
-	MT6359_LDO_LINEAR("ldo_vsram_proc2", VSRAM_PROC2, 500000, 1293750, 6250,
+	MT6359_LDO_LINEAR("ldo_vsram_proc2", VSRAM_PROC2, "vs2-ldo1", 500000, 1293750, 6250,
 			  MT6359P_RG_LDO_VSRAM_PROC2_EN_ADDR,
 			  MT6359P_DA_VSRAM_PROC2_B_EN_ADDR,
 			  MT6359P_RG_LDO_VSRAM_PROC2_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_PROC2_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_PROC2_VOSEL_SHIFT),
-	MT6359_LDO("ldo_vio18", VIO18, volt18_voltages,
+	MT6359_LDO("ldo_vio18", VIO18, "vs1-ldo2", volt18_voltages,
 		   MT6359P_RG_LDO_VIO18_EN_ADDR, MT6359P_RG_LDO_VIO18_EN_SHIFT,
 		   MT6359P_DA_VIO18_B_EN_ADDR, MT6359P_RG_VIO18_VOSEL_ADDR,
 		   MT6359_RG_VIO18_VOSEL_MASK << MT6359_RG_VIO18_VOSEL_SHIFT,
 		   960),
-	MT6359_LDO("ldo_vcamio", VCAMIO, volt18_voltages,
+	MT6359_LDO("ldo_vcamio", VCAMIO, "vs1-ldo1", volt18_voltages,
 		   MT6359P_RG_LDO_VCAMIO_EN_ADDR,
 		   MT6359P_RG_LDO_VCAMIO_EN_SHIFT,
 		   MT6359P_DA_VCAMIO_B_EN_ADDR, MT6359P_RG_VCAMIO_VOSEL_ADDR,
 		   MT6359_RG_VCAMIO_VOSEL_MASK << MT6359_RG_VCAMIO_VOSEL_SHIFT,
 		   1290),
-	MT6359_REG_FIXED("ldo_vcn18", VCN18, MT6359P_RG_LDO_VCN18_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vcn18", VCN18, "vs1-ldo2", MT6359P_RG_LDO_VCN18_EN_ADDR,
 			 MT6359P_DA_VCN18_B_EN_ADDR, 1800000),
-	MT6359_REG_FIXED("ldo_vfe28", VFE28, MT6359P_RG_LDO_VFE28_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vfe28", VFE28, "vsys-ldo1", MT6359P_RG_LDO_VFE28_EN_ADDR,
 			 MT6359P_DA_VFE28_B_EN_ADDR, 2800000),
-	MT6359_LDO("ldo_vcn13", VCN13, vcn13_voltages,
+	MT6359_LDO("ldo_vcn13", VCN13, "vs2-ldo2", vcn13_voltages,
 		   MT6359P_RG_LDO_VCN13_EN_ADDR, MT6359P_RG_LDO_VCN13_EN_SHIFT,
 		   MT6359P_DA_VCN13_B_EN_ADDR, MT6359P_RG_VCN13_VOSEL_ADDR,
 		   MT6359_RG_VCN13_VOSEL_MASK << MT6359_RG_VCN13_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_vcn33_1_bt", VCN33_1_BT, vcn33_voltages,
+	MT6359_LDO("ldo_vcn33_1", VCN33_1, "vsys-ldo1", vcn33_voltages,
 		   MT6359P_RG_LDO_VCN33_1_EN_0_ADDR,
 		   MT6359_RG_LDO_VCN33_1_EN_0_SHIFT,
 		   MT6359P_DA_VCN33_1_B_EN_ADDR, MT6359P_RG_VCN33_1_VOSEL_ADDR,
 		   MT6359_RG_VCN33_1_VOSEL_MASK <<
 		   MT6359_RG_VCN33_1_VOSEL_SHIFT, 240),
-	MT6359_LDO("ldo_vcn33_1_wifi", VCN33_1_WIFI, vcn33_voltages,
-		   MT6359P_RG_LDO_VCN33_1_EN_1_ADDR,
-		   MT6359P_RG_LDO_VCN33_1_EN_1_SHIFT,
-		   MT6359P_DA_VCN33_1_B_EN_ADDR, MT6359P_RG_VCN33_1_VOSEL_ADDR,
-		   MT6359_RG_VCN33_1_VOSEL_MASK <<
-		   MT6359_RG_VCN33_1_VOSEL_SHIFT, 240),
-	MT6359_REG_FIXED("ldo_vaux18", VAUX18, MT6359P_RG_LDO_VAUX18_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vaux18", VAUX18, "vsys-ldo2", MT6359P_RG_LDO_VAUX18_EN_ADDR,
 			 MT6359P_DA_VAUX18_B_EN_ADDR, 1800000),
-	MT6359_LDO_LINEAR("ldo_vsram_others", VSRAM_OTHERS, 500000, 1293750,
+	MT6359_LDO_LINEAR("ldo_vsram_others", VSRAM_OTHERS, "vs2-ldo1", 500000, 1293750,
 			  6250,
 			  MT6359P_RG_LDO_VSRAM_OTHERS_EN_ADDR,
 			  MT6359P_DA_VSRAM_OTHERS_B_EN_ADDR,
 			  MT6359P_RG_LDO_VSRAM_OTHERS_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_OTHERS_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_OTHERS_VOSEL_SHIFT),
-	MT6359_LDO("ldo_vefuse", VEFUSE, vefuse_voltages,
+	MT6359_LDO("ldo_vefuse", VEFUSE, "vs1-ldo2", vefuse_voltages,
 		   MT6359P_RG_LDO_VEFUSE_EN_ADDR,
 		   MT6359P_RG_LDO_VEFUSE_EN_SHIFT,
 		   MT6359P_DA_VEFUSE_B_EN_ADDR, MT6359P_RG_VEFUSE_VOSEL_ADDR,
 		   MT6359_RG_VEFUSE_VOSEL_MASK << MT6359_RG_VEFUSE_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO("ldo_vxo22", VXO22, vxo22_voltages,
+	MT6359_LDO("ldo_vxo22", VXO22, "vsys-ldo2", vxo22_voltages,
 		   MT6359P_RG_LDO_VXO22_EN_ADDR, MT6359P_RG_LDO_VXO22_EN_SHIFT,
 		   MT6359P_DA_VXO22_B_EN_ADDR, MT6359P_RG_VXO22_VOSEL_ADDR,
 		   MT6359_RG_VXO22_VOSEL_MASK << MT6359_RG_VXO22_VOSEL_SHIFT,
 		   480),
-	MT6359_LDO("ldo_vrfck_1", VRFCK, vrfck_voltages_1,
+	MT6359_LDO("ldo_vrfck_1", VRFCK, "vsys-ldo2", vrfck_voltages_1,
 		   MT6359P_RG_LDO_VRFCK_EN_ADDR, MT6359P_RG_LDO_VRFCK_EN_SHIFT,
 		   MT6359P_DA_VRFCK_B_EN_ADDR, MT6359P_RG_VRFCK_VOSEL_ADDR,
 		   MT6359_RG_VRFCK_VOSEL_MASK << MT6359_RG_VRFCK_VOSEL_SHIFT,
 		   480),
-	MT6359_REG_FIXED("ldo_vbif28", VBIF28, MT6359P_RG_LDO_VBIF28_EN_ADDR,
+	MT6359_REG_FIXED("ldo_vbif28", VBIF28, "vsys-ldo2", MT6359P_RG_LDO_VBIF28_EN_ADDR,
 			 MT6359P_DA_VBIF28_B_EN_ADDR, 2800000),
-	MT6359_LDO("ldo_vio28", VIO28, vio28_voltages,
+	MT6359_LDO("ldo_vio28", VIO28, "vsys-ldo2", vio28_voltages,
 		   MT6359P_RG_LDO_VIO28_EN_ADDR, MT6359P_RG_LDO_VIO28_EN_SHIFT,
 		   MT6359P_DA_VIO28_B_EN_ADDR, MT6359P_RG_VIO28_VOSEL_ADDR,
 		   MT6359_RG_VIO28_VOSEL_MASK << MT6359_RG_VIO28_VOSEL_SHIFT,
 		   1920),
-	MT6359P_LDO1("ldo_vemc_1", VEMC, mt6359p_vemc_ops, vemc_voltages_1,
+	MT6359P_LDO1("ldo_vemc_1", VEMC, "vsys-ldo2", mt6359p_vemc_ops, vemc_voltages_1,
 		     MT6359P_RG_LDO_VEMC_EN_ADDR, MT6359P_RG_LDO_VEMC_EN_SHIFT,
 		     MT6359P_DA_VEMC_B_EN_ADDR,
 		     MT6359P_RG_LDO_VEMC_VOSEL_0_ADDR,
 		     MT6359P_RG_LDO_VEMC_VOSEL_0_MASK <<
 		     MT6359P_RG_LDO_VEMC_VOSEL_0_SHIFT),
-	MT6359_LDO("ldo_vcn33_2_bt", VCN33_2_BT, vcn33_voltages,
+	MT6359_LDO("ldo_vcn33_2", VCN33_2, "vsys-ldo1", vcn33_voltages,
 		   MT6359P_RG_LDO_VCN33_2_EN_0_ADDR,
 		   MT6359P_RG_LDO_VCN33_2_EN_0_SHIFT,
 		   MT6359P_DA_VCN33_2_B_EN_ADDR, MT6359P_RG_VCN33_2_VOSEL_ADDR,
 		   MT6359_RG_VCN33_2_VOSEL_MASK <<
 		   MT6359_RG_VCN33_2_VOSEL_SHIFT, 240),
-	MT6359_LDO("ldo_vcn33_2_wifi", VCN33_2_WIFI, vcn33_voltages,
-		   MT6359P_RG_LDO_VCN33_2_EN_1_ADDR,
-		   MT6359_RG_LDO_VCN33_2_EN_1_SHIFT,
-		   MT6359P_DA_VCN33_2_B_EN_ADDR, MT6359P_RG_VCN33_2_VOSEL_ADDR,
-		   MT6359_RG_VCN33_2_VOSEL_MASK <<
-		   MT6359_RG_VCN33_2_VOSEL_SHIFT, 240),
-	MT6359_LDO("ldo_va12", VA12, va12_voltages,
+	MT6359_LDO("ldo_va12", VA12, "vs2-ldo2", va12_voltages,
 		   MT6359P_RG_LDO_VA12_EN_ADDR, MT6359P_RG_LDO_VA12_EN_SHIFT,
 		   MT6359P_DA_VA12_B_EN_ADDR, MT6359P_RG_VA12_VOSEL_ADDR,
 		   MT6359_RG_VA12_VOSEL_MASK << MT6359_RG_VA12_VOSEL_SHIFT,
 		   960),
-	MT6359_LDO("ldo_va09", VA09, va09_voltages,
+	MT6359_LDO("ldo_va09", VA09, "vs2-ldo2", va09_voltages,
 		   MT6359P_RG_LDO_VA09_EN_ADDR, MT6359P_RG_LDO_VA09_EN_SHIFT,
 		   MT6359P_DA_VA09_B_EN_ADDR, MT6359P_RG_VA09_VOSEL_ADDR,
 		   MT6359_RG_VA09_VOSEL_MASK << MT6359_RG_VA09_VOSEL_SHIFT,
 		   960),
-	MT6359_LDO("ldo_vrf18", VRF18, vrf18_voltages,
+	MT6359_LDO("ldo_vrf18", VRF18, "vs1-ldo2", vrf18_voltages,
 		   MT6359P_RG_LDO_VRF18_EN_ADDR, MT6359P_RG_LDO_VRF18_EN_SHIFT,
 		   MT6359P_DA_VRF18_B_EN_ADDR, MT6359P_RG_VRF18_VOSEL_ADDR,
 		   MT6359_RG_VRF18_VOSEL_MASK << MT6359_RG_VRF18_VOSEL_SHIFT,
 		   240),
-	MT6359_LDO_LINEAR("ldo_vsram_md", VSRAM_MD, 500000, 1293750, 6250,
+	MT6359_LDO_LINEAR("ldo_vsram_md", VSRAM_MD, "vs2-ldo1", 500000, 1293750, 6250,
 			  MT6359P_RG_LDO_VSRAM_MD_EN_ADDR,
 			  MT6359P_DA_VSRAM_MD_B_EN_ADDR,
 			  MT6359P_RG_LDO_VSRAM_MD_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_MD_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_MD_VOSEL_SHIFT),
-	MT6359_LDO("ldo_vufs", VUFS, volt18_voltages,
+	MT6359_LDO("ldo_vufs", VUFS, "vs1-ldo1", volt18_voltages,
 		   MT6359P_RG_LDO_VUFS_EN_ADDR, MT6359P_RG_LDO_VUFS_EN_SHIFT,
 		   MT6359P_DA_VUFS_B_EN_ADDR, MT6359P_RG_VUFS_VOSEL_ADDR,
 		   MT6359_RG_VUFS_VOSEL_MASK << MT6359_RG_VUFS_VOSEL_SHIFT,
 		   1920),
-	MT6359_LDO("ldo_vm18", VM18, volt18_voltages,
+	MT6359_LDO("ldo_vm18", VM18, "vs1-ldo1", volt18_voltages,
 		   MT6359P_RG_LDO_VM18_EN_ADDR, MT6359P_RG_LDO_VM18_EN_SHIFT,
 		   MT6359P_DA_VM18_B_EN_ADDR, MT6359P_RG_VM18_VOSEL_ADDR,
 		   MT6359_RG_VM18_VOSEL_MASK << MT6359_RG_VM18_VOSEL_SHIFT,
 		   1920),
-	MT6359_LDO("ldo_vbbck", VBBCK, vbbck_voltages,
+	MT6359_LDO("ldo_vbbck", VBBCK, "LDO_VIO18", vbbck_voltages,
 		   MT6359P_RG_LDO_VBBCK_EN_ADDR, MT6359P_RG_LDO_VBBCK_EN_SHIFT,
 		   MT6359P_DA_VBBCK_B_EN_ADDR, MT6359P_RG_VBBCK_VOSEL_ADDR,
 		   MT6359P_RG_VBBCK_VOSEL_MASK << MT6359P_RG_VBBCK_VOSEL_SHIFT,
 		   480),
-	MT6359_LDO_LINEAR("ldo_vsram_proc1", VSRAM_PROC1, 500000, 1293750, 6250,
+	MT6359_LDO_LINEAR("ldo_vsram_proc1", VSRAM_PROC1, "vs2-ldo1", 500000, 1293750, 6250,
 			  MT6359P_RG_LDO_VSRAM_PROC1_EN_ADDR,
 			  MT6359P_DA_VSRAM_PROC1_B_EN_ADDR,
 			  MT6359P_RG_LDO_VSRAM_PROC1_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_PROC1_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_PROC1_VOSEL_SHIFT),
-	MT6359_LDO("ldo_vsim2", VSIM2, vsim2_voltages,
+	MT6359_LDO("ldo_vsim2", VSIM2, "vsys-ldo2", vsim2_voltages,
 		   MT6359P_RG_LDO_VSIM2_EN_ADDR, MT6359P_RG_LDO_VSIM2_EN_SHIFT,
 		   MT6359P_DA_VSIM2_B_EN_ADDR, MT6359P_RG_VSIM2_VOSEL_ADDR,
 		   MT6359_RG_VSIM2_VOSEL_MASK << MT6359_RG_VSIM2_VOSEL_SHIFT,
 		   480),
-	MT6359_LDO_LINEAR("ldo_vsram_others_sshub", VSRAM_OTHERS_SSHUB,
+	MT6359_LDO_LINEAR("ldo_vsram_others_sshub", VSRAM_OTHERS_SSHUB, "vs2-ldo1",
 			  500000, 1293750, 6250,
 			  MT6359P_RG_LDO_VSRAM_OTHERS_SSHUB_EN_ADDR,
 			  MT6359P_DA_VSRAM_OTHERS_B_EN_ADDR,
 			  MT6359P_RG_LDO_VSRAM_OTHERS_SSHUB_VOSEL_ADDR,
 			  MT6359_RG_LDO_VSRAM_OTHERS_SSHUB_VOSEL_MASK <<
 			  MT6359_RG_LDO_VSRAM_OTHERS_SSHUB_VOSEL_SHIFT),
+	/* Placeholders for DT backward compatibility */
+	MT6359_LDO_NOOP("ldo_vcn33_1_bt",   VCN33_1_BT,   "LDO_VCN33_1"),
+	MT6359_LDO_NOOP("ldo_vcn33_1_wifi", VCN33_1_WIFI, "LDO_VCN33_1"),
+	MT6359_LDO_NOOP("ldo_vcn33_2_bt",   VCN33_2_BT,   "LDO_VCN33_2"),
+	MT6359_LDO_NOOP("ldo_vcn33_2_wifi", VCN33_2_WIFI, "LDO_VCN33_2"),
+};
+
+struct mt6359_vcn33_regs {
+	u32 wifi_en_reg;
+	u32 wifi_en_mask;
+	u32 bt_en_reg;
+	u32 bt_en_mask;
+};
+
+static const struct mt6359_vcn33_regs vcn33_regs[][2] = {
+	{ /* MT6359 */
+		{
+			.wifi_en_reg = MT6359_RG_LDO_VCN33_1_EN_1_ADDR,
+			.wifi_en_mask = BIT(MT6359_RG_LDO_VCN33_1_EN_1_SHIFT),
+			.bt_en_reg = MT6359_RG_LDO_VCN33_1_EN_0_ADDR,
+			.bt_en_mask = BIT(MT6359_RG_LDO_VCN33_1_EN_0_SHIFT),
+		}, {
+			.wifi_en_reg = MT6359_RG_LDO_VCN33_2_EN_1_ADDR,
+			.wifi_en_mask = BIT(MT6359_RG_LDO_VCN33_2_EN_1_SHIFT),
+			.bt_en_reg = MT6359_RG_LDO_VCN33_2_EN_0_ADDR,
+			.bt_en_mask = BIT(MT6359_RG_LDO_VCN33_2_EN_0_SHIFT),
+		}
+	}, { /* MT6359P */
+		{
+			.wifi_en_reg = MT6359P_RG_LDO_VCN33_1_EN_1_ADDR,
+			.wifi_en_mask = BIT(MT6359P_RG_LDO_VCN33_1_EN_1_SHIFT),
+			.bt_en_reg = MT6359P_RG_LDO_VCN33_1_EN_0_ADDR,
+			.bt_en_mask = BIT(MT6359_RG_LDO_VCN33_1_EN_0_SHIFT),
+		}, {
+			.wifi_en_reg = MT6359P_RG_LDO_VCN33_2_EN_1_ADDR,
+			.wifi_en_mask = BIT(MT6359_RG_LDO_VCN33_2_EN_1_SHIFT),
+			.bt_en_reg = MT6359P_RG_LDO_VCN33_2_EN_0_ADDR,
+			.bt_en_mask = BIT(MT6359P_RG_LDO_VCN33_2_EN_0_SHIFT),
+		}
+	}
 };
 
+static int mt6359_sync_vcn33_setting(struct device *dev, unsigned int idx)
+{
+	struct mt6397_chip *mt6397 = dev_get_drvdata(dev->parent);
+	unsigned int val;
+	int ret;
+
+	/*
+	 * VCN33_[12]_WIFI and VCN33_[12]_BT are two separate enable bits for
+	 * the same regulator. They share the same voltage setting and output
+	 * pin. Instead of having two potentially conflicting regulators, just
+	 * have one regulator. Sync the two enable bits and only use one in
+	 * the regulator device.
+	 */
+	for (unsigned int i = 0; i < ARRAY_SIZE(vcn33_regs[0]); i++) {
+		u32 bt_en_mask = vcn33_regs[idx][i].bt_en_mask;
+		u32 wifi_en_mask = vcn33_regs[idx][i].wifi_en_mask;
+
+		ret = regmap_read(mt6397->regmap, vcn33_regs[idx][i].wifi_en_reg, &val);
+		if (ret)
+			return dev_err_probe(dev, ret, "Failed to read VCN33_%u_WIFI setting\n",
+					     i + 1);
+
+		if (!(val & wifi_en_mask))
+			continue;
+
+		/* Sync VCN33_[12]_WIFI enable status to VCN33_[12]_BT */
+		ret = regmap_update_bits(mt6397->regmap, vcn33_regs[idx][i].bt_en_reg,
+					 bt_en_mask, bt_en_mask);
+		if (ret)
+			return dev_err_probe(dev, ret,
+					     "Failed to sync VCN33_%u_WIFI setting to VCN33_%u_BT\n",
+					     i + 1, i + 1);
+
+		/* Disable VCN33_[12]_WIFI */
+		ret = regmap_update_bits(mt6397->regmap, vcn33_regs[idx][i].wifi_en_reg,
+					 wifi_en_mask, 0);
+		if (ret)
+			return dev_err_probe(dev, ret, "Failed to disable VCN33_%u_WIFI\n", i + 1);
+	}
+
+	return 0;
+}
+
 static int mt6359_regulator_probe(struct platform_device *pdev)
 {
 	struct mt6397_chip *mt6397 = dev_get_drvdata(pdev->dev.parent);
 	struct regulator_config config = {};
 	struct regulator_dev *rdev;
-	struct mt6359_regulator_info *mt6359_info;
+	const struct mt6359_regulator_info *mt6359_info;
+	const char *vio18_name, *vcn33_1_name, *vcn33_2_name;
 	int i, hw_ver, ret;
 
 	ret = regmap_read(mt6397->regmap, MT6359P_HWCID, &hw_ver);
 	if (ret)
 		return ret;
 
-	if (hw_ver >= MT6359P_CHIP_VER)
+	if (hw_ver >= MT6359P_CHIP_VER) {
 		mt6359_info = mt6359p_regulators;
-	else
+		ret = mt6359_sync_vcn33_setting(&pdev->dev, 1);
+		if (ret)
+			return ret;
+	} else {
 		mt6359_info = mt6359_regulators;
+		ret = mt6359_sync_vcn33_setting(&pdev->dev, 0);
+		if (ret)
+			return ret;
+	}
+
+	vio18_name = mt6359_info[MT6359_ID_VIO18].desc.name;
+	vcn33_1_name = mt6359_info[MT6359_ID_VCN33_1].desc.name;
+	vcn33_2_name = mt6359_info[MT6359_ID_VCN33_2].desc.name;
 
 	config.dev = mt6397->dev;
 	config.regmap = mt6397->regmap;
 	for (i = 0; i < MT6359_MAX_REGULATOR; i++, mt6359_info++) {
-		config.driver_data = mt6359_info;
-		rdev = devm_regulator_register(&pdev->dev, &mt6359_info->desc, &config);
+		const struct regulator_desc *desc = &mt6359_info->desc;
+		struct regulator_desc *_desc;
+
+		/* drop const here, but all uses in the driver are const */
+		config.driver_data = (void *)mt6359_info;
+
+		/* Use vio18's actual name as supply_name for vbbck */
+		if (i == MT6359_ID_VBBCK && strcmp(desc->supply_name, vio18_name) != 0) {
+			_desc = devm_kzalloc(&pdev->dev, sizeof(*_desc), GFP_KERNEL);
+			if (!_desc)
+				return -ENOMEM;
+
+			memcpy(_desc, desc, sizeof(*_desc));
+			_desc->supply_name = vio18_name;
+			desc = _desc;
+		}
+
+		/* Use vcn33_1's actual name as supply_name for vcn33_1_(bt|wifi) */
+		if ((i == MT6359_ID_VCN33_1_BT || i == MT6359_ID_VCN33_1_WIFI) &&
+		    strcmp(desc->supply_name, vcn33_1_name) != 0) {
+			_desc = devm_kzalloc(&pdev->dev, sizeof(*_desc), GFP_KERNEL);
+			if (!_desc)
+				return -ENOMEM;
+
+			memcpy(_desc, desc, sizeof(*_desc));
+			_desc->supply_name = vcn33_1_name;
+			desc = _desc;
+		}
+
+		/* Use vcn33_2's actual name as supply_name for vcn33_2_(bt|wifi) */
+		if ((i == MT6359_ID_VCN33_2_BT || i == MT6359_ID_VCN33_2_WIFI) &&
+		    strcmp(desc->supply_name, vcn33_2_name) != 0) {
+			_desc = devm_kzalloc(&pdev->dev, sizeof(*_desc), GFP_KERNEL);
+			if (!_desc)
+				return -ENOMEM;
+
+			memcpy(_desc, desc, sizeof(*_desc));
+			_desc->supply_name = vcn33_2_name;
+			desc = _desc;
+		}
+
+		rdev = devm_regulator_register(&pdev->dev, desc, &config);
 		if (IS_ERR(rdev)) {
 			dev_err(&pdev->dev, "failed to register %s\n", mt6359_info->desc.name);
 			return PTR_ERR(rdev);
 		}
+
+		/* Save vio18 name for vbbck */
+		if (i == MT6359_ID_VIO18)
+			vio18_name = rdev_get_name(rdev);
+
+		/* Save vcn33_1 name for vbbck */
+		if (i == MT6359_ID_VCN33_1)
+			vcn33_1_name = rdev_get_name(rdev);
+
+		/* Save vcn33_2 name for vbbck */
+		if (i == MT6359_ID_VCN33_2)
+			vcn33_2_name = rdev_get_name(rdev);
 	}
 
 	return 0;
diff --git a/drivers/regulator/palmas-regulator.c b/drivers/regulator/palmas-regulator.c
index 60656a815b9ec..f82618a701061 100644
--- a/drivers/regulator/palmas-regulator.c
+++ b/drivers/regulator/palmas-regulator.c
@@ -1590,6 +1590,7 @@ static const struct of_device_id of_palmas_match_tbl[] = {
 	},
 	{ /* end */ }
 };
+MODULE_DEVICE_TABLE(of, of_palmas_match_tbl);
 
 static int palmas_regulators_probe(struct platform_device *pdev)
 {
@@ -1684,4 +1685,3 @@ MODULE_AUTHOR("Graeme Gregory <gg@slimlogic.co.uk>");
 MODULE_DESCRIPTION("Palmas voltage regulator driver");
 MODULE_LICENSE("GPL");
 MODULE_ALIAS("platform:palmas-pmic");
-MODULE_DEVICE_TABLE(of, of_palmas_match_tbl);
diff --git a/drivers/regulator/qcom-rpmh-regulator.c b/drivers/regulator/qcom-rpmh-regulator.c
index 0dcb50bf5c35b..756a4201225e7 100644
--- a/drivers/regulator/qcom-rpmh-regulator.c
+++ b/drivers/regulator/qcom-rpmh-regulator.c
@@ -1100,6 +1100,21 @@ static const struct rpmh_vreg_init_data pm8998_vreg_data[] = {
 	{}
 };
 
+static const struct rpmh_vreg_init_data pmau0102_vreg_data[] = {
+	RPMH_VREG("smps1",  SMPS, 1,  &pmic5_ftsmps527,  "vdd-s1"),
+	RPMH_VREG("smps2",  SMPS, 2,  &pmic5_ftsmps527,  "vdd-s2"),
+	RPMH_VREG("smps3",  SMPS, 3,  &pmic5_ftsmps527,  "vdd-s3"),
+	RPMH_VREG("smps4",  SMPS, 4,  &pmic5_ftsmps527,  "vdd-s4"),
+	RPMH_VREG("smps5",  SMPS, 5,  &pmic5_ftsmps527,  "vdd-s5"),
+	RPMH_VREG("smps6",  SMPS, 6,  &pmic5_ftsmps527,  "vdd-s6"),
+	RPMH_VREG("smps7",  SMPS, 7,  &pmic5_ftsmps527,  "vdd-s7"),
+	RPMH_VREG("smps8",  SMPS, 8,  &pmic5_ftsmps527,  "vdd-s8"),
+	RPMH_VREG("ldo1",   LDO,  1,  &pmic5_nldo515,    "vdd-l1"),
+	RPMH_VREG("ldo2",   LDO,  2,  &pmic5_nldo515,    "vdd-l2"),
+	RPMH_VREG("ldo3",   LDO,  3,  &pmic5_pldo515_mv, "vdd-l3"),
+	{}
+};
+
 static const struct rpmh_vreg_init_data pmg1110_vreg_data[] = {
 	RPMH_VREG("smps1",  SMPS, 1, &pmic5_ftsmps510,  "vdd-s1"),
 	{}
@@ -1877,6 +1892,10 @@ static const struct of_device_id __maybe_unused rpmh_regulator_match_table[] = {
 		.compatible = "qcom,pm8998-rpmh-regulators",
 		.data = pm8998_vreg_data,
 	},
+	{
+		.compatible = "qcom,pmau0102-rpmh-regulators",
+		.data = pmau0102_vreg_data,
+	},
 	{
 		.compatible = "qcom,pmg1110-rpmh-regulators",
 		.data = pmg1110_vreg_data,
diff --git a/drivers/regulator/qcom_smd-regulator.c b/drivers/regulator/qcom_smd-regulator.c
index 25ed9f713974b..3ee7f5d0c694d 100644
--- a/drivers/regulator/qcom_smd-regulator.c
+++ b/drivers/regulator/qcom_smd-regulator.c
@@ -913,6 +913,38 @@ static const struct rpm_regulator_data rpm_pm660l_regulators[] = {
 	{ }
 };
 
+static const struct rpm_regulator_data rpm_pm8150_regulators[] = {
+	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pmic5_ftsmps520, "vdd-s1" },
+	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pmic5_ftsmps520, "vdd-s2" },
+	{ "s3", QCOM_SMD_RPM_SMPA, 3, &pmic5_ftsmps520, "vdd-s3" },
+	{ "s4", QCOM_SMD_RPM_SMPA, 4, &pm8998_hfsmps, "vdd-s4" },
+	{ "s5", QCOM_SMD_RPM_SMPA, 5, &pm8998_hfsmps, "vdd-s5" },
+	{ "s6", QCOM_SMD_RPM_SMPA, 6, &pmic5_ftsmps520, "vdd-s6" },
+	{ "s7", QCOM_SMD_RPM_SMPA, 7, &pmic5_ftsmps520, "vdd-s7" },
+	{ "s8", QCOM_SMD_RPM_SMPA, 8, &pmic5_ftsmps520, "vdd-s8" },
+	{ "s9", QCOM_SMD_RPM_SMPA, 9, &pmic5_ftsmps520, "vdd-s9" },
+	{ "s10", QCOM_SMD_RPM_SMPA, 10, &pmic5_ftsmps520, "vdd-s10" },
+	{ "l1", QCOM_SMD_RPM_LDOA, 1, &pm660_nldo660, "vdd-l1-l8-l11" },
+	{ "l2", QCOM_SMD_RPM_LDOA, 2, &pm660_pldo660, "vdd-l2-l10" },
+	{ "l3", QCOM_SMD_RPM_LDOA, 3, &pm660_nldo660, "vdd-l3-l4-l5-l18" },
+	{ "l4", QCOM_SMD_RPM_LDOA, 4, &pm660_nldo660, "vdd-l3-l4-l5-l18" },
+	{ "l5", QCOM_SMD_RPM_LDOA, 5, &pm660_nldo660, "vdd-l3-l4-l5-l18" },
+	{ "l6", QCOM_SMD_RPM_LDOA, 6, &pm660_nldo660, "vdd-l6-l9" },
+	{ "l7", QCOM_SMD_RPM_LDOA, 7, &pm660_pldo660, "vdd-l7-l12-l14-l15" },
+	{ "l8", QCOM_SMD_RPM_LDOA, 8, &pm660_nldo660, "vdd-l1-l8-l11" },
+	{ "l9", QCOM_SMD_RPM_LDOA, 9, &pm660_nldo660, "vdd-l6-l9" },
+	{ "l10", QCOM_SMD_RPM_LDOA, 10, &pm660_pldo660, "vdd-l2-l10" },
+	{ "l11", QCOM_SMD_RPM_LDOA, 11, &pm660_nldo660, "vdd-l1-l8-l11" },
+	{ "l12", QCOM_SMD_RPM_LDOA, 12, &pm660_ht_lvpldo, "vdd-l7-l12-l14-l15" },
+	{ "l13", QCOM_SMD_RPM_LDOA, 13, &pm660_pldo660, "vdd-l13-l16-l17" },
+	{ "l14", QCOM_SMD_RPM_LDOA, 14, &pm660_ht_lvpldo, "vdd-l7-l12-l14-l15" },
+	{ "l15", QCOM_SMD_RPM_LDOA, 15, &pm660_ht_lvpldo, "vdd-l7-l12-l14-l15" },
+	{ "l16", QCOM_SMD_RPM_LDOA, 16, &pm660_pldo660, "vdd-l13-l16-l17" },
+	{ "l17", QCOM_SMD_RPM_LDOA, 17, &pm660_pldo660, "vdd-l13-l16-l17" },
+	{ "l18", QCOM_SMD_RPM_LDOA, 18, &pm660_nldo660, "vdd-l3-l4-l5-l18" },
+	{ }
+};
+
 static const struct rpm_regulator_data rpm_pm8226_regulators[] = {
 	{ "s1", QCOM_SMD_RPM_SMPA, 1, &pm8226_hfsmps, "vdd_s1" },
 	{ "s2", QCOM_SMD_RPM_SMPA, 2, &pm8226_ftsmps, "vdd_s2" },
@@ -1358,6 +1390,7 @@ static const struct of_device_id rpm_of_match[] = {
 	{ .compatible = "qcom,rpm-pm6125-regulators", .data = &rpm_pm6125_regulators },
 	{ .compatible = "qcom,rpm-pm660-regulators", .data = &rpm_pm660_regulators },
 	{ .compatible = "qcom,rpm-pm660l-regulators", .data = &rpm_pm660l_regulators },
+	{ .compatible = "qcom,rpm-pm8150-regulators", .data = &rpm_pm8150_regulators },
 	{ .compatible = "qcom,rpm-pm8226-regulators", .data = &rpm_pm8226_regulators },
 	{ .compatible = "qcom,rpm-pm8841-regulators", .data = &rpm_pm8841_regulators },
 	{ .compatible = "qcom,rpm-pm8909-regulators", .data = &rpm_pm8909_regulators },
diff --git a/drivers/regulator/tps65219-regulator.c b/drivers/regulator/tps65219-regulator.c
index d77ca486879fd..324c3a33af8a4 100644
--- a/drivers/regulator/tps65219-regulator.c
+++ b/drivers/regulator/tps65219-regulator.c
@@ -346,8 +346,9 @@ static irqreturn_t tps65219_regulator_irq_handler(int irq, void *data)
 		return IRQ_HANDLED;
 	}
 
-	regulator_notifier_call_chain(irq_data->rdev,
-				      irq_data->type->event, NULL);
+	if (irq_data->rdev)
+		regulator_notifier_call_chain(irq_data->rdev,
+					      irq_data->type->event, NULL);
 
 	dev_err(irq_data->dev, "Error IRQ trap %s for %s\n",
 		irq_data->type->event_name, irq_data->type->regulator_name);
@@ -398,14 +399,65 @@ static struct tps65219_chip_data chip_info_table[] = {
 	},
 };
 
-static int tps65219_regulator_probe(struct platform_device *pdev)
+static bool tps65219_is_regulator_name(const struct tps65219_chip_data *pmic,
+				       const char *name)
+{
+	int i;
+
+	for (i = 0; i < pmic->common_rdesc_size; i++)
+		if (!strcmp(pmic->common_rdesc[i].name, name))
+			return true;
+	for (i = 0; i < pmic->rdesc_size; i++)
+		if (!strcmp(pmic->rdesc[i].name, name))
+			return true;
+	return false;
+}
+
+static int tps65219_register_irqs(struct platform_device *pdev,
+				  struct tps65219 *tps,
+				  struct regulator_dev *rdev,
+				  struct tps65219_regulator_irq_type *irq_types,
+				  int nirqs,
+				  const char *regulator_name)
 {
 	struct tps65219_regulator_irq_data *irq_data;
+	int i, irq, error;
+
+	for (i = 0; i < nirqs; i++) {
+		if (strcmp(irq_types[i].regulator_name, regulator_name))
+			continue;
+
+		irq = platform_get_irq_byname(pdev, irq_types[i].irq_name);
+		if (irq < 0)
+			return -EINVAL;
+
+		irq_data = devm_kmalloc(tps->dev, sizeof(*irq_data), GFP_KERNEL);
+		if (!irq_data)
+			return -ENOMEM;
+
+		irq_data->dev = tps->dev;
+		irq_data->type = &irq_types[i];
+		irq_data->rdev = rdev;
+
+		error = devm_request_threaded_irq(tps->dev, irq, NULL,
+						  tps65219_regulator_irq_handler,
+						  IRQF_ONESHOT,
+						  irq_types[i].irq_name,
+						  irq_data);
+		if (error)
+			return dev_err_probe(tps->dev, error,
+					     "Failed to request %s IRQ %d\n",
+					     irq_types[i].irq_name, irq);
+	}
+	return 0;
+}
+
+static int tps65219_regulator_probe(struct platform_device *pdev)
+{
 	struct tps65219_regulator_irq_type *irq_type;
 	struct tps65219_chip_data *pmic;
 	struct regulator_dev *rdev;
 	int error;
-	int irq;
 	int i;
 
 	struct tps65219 *tps = dev_get_drvdata(pdev->dev.parent);
@@ -425,6 +477,19 @@ static int tps65219_regulator_probe(struct platform_device *pdev)
 			return dev_err_probe(tps->dev, PTR_ERR(rdev),
 					      "Failed to register %s regulator\n",
 					      pmic->common_rdesc[i].name);
+
+		error = tps65219_register_irqs(pdev, tps, rdev,
+					       pmic->common_irq_types,
+					       pmic->common_irq_size,
+					       pmic->common_rdesc[i].name);
+		if (error)
+			return error;
+		error = tps65219_register_irqs(pdev, tps, rdev,
+					       pmic->irq_types,
+					       pmic->dev_irq_size,
+					       pmic->common_rdesc[i].name);
+		if (error)
+			return error;
 	}
 
 	for (i = 0; i <  pmic->rdesc_size; i++) {
@@ -434,52 +499,42 @@ static int tps65219_regulator_probe(struct platform_device *pdev)
 			return dev_err_probe(tps->dev, PTR_ERR(rdev),
 					     "Failed to register %s regulator\n",
 					     pmic->rdesc[i].name);
+
+		error = tps65219_register_irqs(pdev, tps, rdev,
+					       pmic->common_irq_types,
+					       pmic->common_irq_size,
+					       pmic->rdesc[i].name);
+		if (error)
+			return error;
+		error = tps65219_register_irqs(pdev, tps, rdev,
+					       pmic->irq_types,
+					       pmic->dev_irq_size,
+					       pmic->rdesc[i].name);
+		if (error)
+			return error;
 	}
 
+	/* Register non-regulator IRQs (TIMEOUT, SENSOR) with rdev=NULL */
 	for (i = 0; i < pmic->common_irq_size; ++i) {
 		irq_type = &pmic->common_irq_types[i];
-		irq = platform_get_irq_byname(pdev, irq_type->irq_name);
-		if (irq < 0)
-			return -EINVAL;
-
-		irq_data = devm_kmalloc(tps->dev, sizeof(*irq_data), GFP_KERNEL);
-		if (!irq_data)
-			return -ENOMEM;
-
-		irq_data->dev = tps->dev;
-		irq_data->type = irq_type;
-		error = devm_request_threaded_irq(tps->dev, irq, NULL,
-						  tps65219_regulator_irq_handler,
-						  IRQF_ONESHOT,
-						  irq_type->irq_name,
-						  irq_data);
+		if (tps65219_is_regulator_name(pmic, irq_type->regulator_name))
+			continue;
+		error = tps65219_register_irqs(pdev, tps, NULL,
+					       irq_type, 1,
+					       irq_type->regulator_name);
 		if (error)
-			return dev_err_probe(tps->dev, error,
-					     "Failed to request %s IRQ %d\n",
-					     irq_type->irq_name, irq);
+			return error;
 	}
 
 	for (i = 0; i < pmic->dev_irq_size; ++i) {
 		irq_type = &pmic->irq_types[i];
-		irq = platform_get_irq_byname(pdev, irq_type->irq_name);
-		if (irq < 0)
-			return -EINVAL;
-
-		irq_data = devm_kmalloc(tps->dev, sizeof(*irq_data), GFP_KERNEL);
-		if (!irq_data)
-			return -ENOMEM;
-
-		irq_data->dev = tps->dev;
-		irq_data->type = irq_type;
-		error = devm_request_threaded_irq(tps->dev, irq, NULL,
-						  tps65219_regulator_irq_handler,
-						  IRQF_ONESHOT,
-						  irq_type->irq_name,
-						  irq_data);
+		if (tps65219_is_regulator_name(pmic, irq_type->regulator_name))
+			continue;
+		error = tps65219_register_irqs(pdev, tps, NULL,
+					       irq_type, 1,
+					       irq_type->regulator_name);
 		if (error)
-			return dev_err_probe(tps->dev, error,
-					     "Failed to request %s IRQ %d\n",
-					     irq_type->irq_name, irq);
+			return error;
 	}
 
 	return 0;
diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig
index ee54436fea5ad..c78e431b7b2d8 100644
--- a/drivers/remoteproc/Kconfig
+++ b/drivers/remoteproc/Kconfig
@@ -316,7 +316,6 @@ config ST_SLIM_REMOTEPROC
 config STM32_RPROC
 	tristate "STM32 remoteproc support"
 	depends on ARCH_STM32 || COMPILE_TEST
-	depends on REMOTEPROC
 	select MAILBOX
 	help
 	  Say y here to support STM32 MCU processors via the
diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c
index 0dd80e688b0ea..7f54322244aca 100644
--- a/drivers/remoteproc/imx_rproc.c
+++ b/drivers/remoteproc/imx_rproc.c
@@ -145,6 +145,41 @@ static const struct imx_rproc_att imx_rproc_att_imx95_m7[] = {
 	{ 0x80000000, 0x80000000, 0x50000000, 0 },
 };
 
+static const struct imx_rproc_att imx_rproc_att_imx94_m70[] = {
+	/* dev addr , sys addr  , size	    , flags */
+	/* TCM CODE NON-SECURE */
+	{ 0x00000000, 0x203C0000, 0x00040000, ATT_OWN | ATT_IOMEM },
+	/* TCM SYS NON-SECURE*/
+	{ 0x20000000, 0x20400000, 0x00040000, ATT_OWN | ATT_IOMEM },
+
+	/* DDR */
+	{ 0x80000000, 0x80000000, 0x10000000, 0 },
+};
+
+static const struct imx_rproc_att imx_rproc_att_imx94_m71[] = {
+	/* dev addr , sys addr  , size	    , flags */
+	/* TCM CODE NON-SECURE */
+	{ 0x00000000, 0x202C0000, 0x00040000, ATT_OWN | ATT_IOMEM },
+	/* TCM SYS NON-SECURE*/
+	{ 0x20000000, 0x20300000, 0x00040000, ATT_OWN | ATT_IOMEM },
+
+	/* DDR */
+	{ 0x80000000, 0x80000000, 0x10000000, 0 },
+};
+
+static const struct imx_rproc_att imx_rproc_att_imx94_m33s[] = {
+	/* dev addr , sys addr  , size	    , flags */
+	/* TCM CODE NON-SECURE */
+	{ 0x0FFC0000, 0x209C0000, 0x00040000, ATT_OWN | ATT_IOMEM },
+	/* TCM SYS NON-SECURE */
+	{ 0x20000000, 0x20A00000, 0x00040000, ATT_OWN | ATT_IOMEM },
+	/* M33S OCRAM NON-SECURE */
+	{ 0x20800000, 0x20800000, 0x180000, ATT_OWN | ATT_IOMEM },
+
+	/* DDR */
+	{ 0x80000000, 0x80000000, 0x10000000, 0 },
+};
+
 static const struct imx_rproc_att imx_rproc_att_imx93[] = {
 	/* dev addr , sys addr  , size	    , flags */
 	/* TCM CODE NON-SECURE */
@@ -339,13 +374,32 @@ static int imx_rproc_scu_api_start(struct rproc *rproc)
 	return imx_sc_pm_cpu_start(priv->ipc_handle, priv->rsrc_id, true, priv->entry);
 }
 
+static u64 imx_rproc_sm_get_reset_vector(struct rproc *rproc)
+{
+	struct imx_rproc *priv = rproc->priv;
+	u32 reset_vector_mask = priv->dcfg->reset_vector_mask ?: GENMASK(31, 0);
+
+	/*
+	 * The hardware fetches the first two words from reset_vectors
+	 * (hardware reset address) and populates SP and PC using the first
+	 * two words. Execution proceeds from PC. The ELF entry point does
+	 * not always match the hardware reset address.
+	 * To derive the correct hardware reset address, the lower address
+	 * bits must be masked off before programming the reset vector.
+	 */
+	return rproc->bootaddr & reset_vector_mask;
+}
+
 static int imx_rproc_sm_cpu_start(struct rproc *rproc)
 {
 	struct imx_rproc *priv = rproc->priv;
 	const struct imx_rproc_dcfg *dcfg = priv->dcfg;
+	u64 reset_vector;
 	int ret;
 
-	ret = scmi_imx_cpu_reset_vector_set(dcfg->cpuid, 0, true, false, false);
+	reset_vector = imx_rproc_sm_get_reset_vector(rproc);
+
+	ret = scmi_imx_cpu_reset_vector_set(dcfg->cpuid, reset_vector, true, false, false);
 	if (ret) {
 		dev_err(priv->dev, "Failed to set reset vector cpuid(%u): %d\n", dcfg->cpuid, ret);
 		return ret;
@@ -359,13 +413,16 @@ static int imx_rproc_sm_lmm_start(struct rproc *rproc)
 	struct imx_rproc *priv = rproc->priv;
 	const struct imx_rproc_dcfg *dcfg = priv->dcfg;
 	struct device *dev = priv->dev;
+	u64 reset_vector;
 	int ret;
 
+	reset_vector = imx_rproc_sm_get_reset_vector(rproc);
+
 	/*
 	 * If the remoteproc core can't start the M7, it will already be
 	 * handled in imx_rproc_sm_lmm_prepare().
 	 */
-	ret = scmi_imx_lmm_reset_vector_set(dcfg->lmid, dcfg->cpuid, 0, 0);
+	ret = scmi_imx_lmm_reset_vector_set(dcfg->lmid, dcfg->cpuid, 0, reset_vector);
 	if (ret) {
 		dev_err(dev, "Failed to set reset vector lmid(%u), cpuid(%u): %d\n",
 			dcfg->lmid, dcfg->cpuid, ret);
@@ -1455,6 +1512,33 @@ static const struct imx_rproc_dcfg imx_rproc_cfg_imx93 = {
 	.flags		= IMX_RPROC_NEED_CLKS,
 };
 
+static const struct imx_rproc_dcfg imx_rproc_cfg_imx94_m70 = {
+	.att		= imx_rproc_att_imx94_m70,
+	.att_size	= ARRAY_SIZE(imx_rproc_att_imx94_m70),
+	.ops		= &imx_rproc_ops_sm_lmm,
+	.cpuid		= 1,
+	.lmid		= 2,
+	.reset_vector_mask = GENMASK_U32(31, 16),
+};
+
+static const struct imx_rproc_dcfg imx_rproc_cfg_imx94_m71 = {
+	.att		= imx_rproc_att_imx94_m71,
+	.att_size	= ARRAY_SIZE(imx_rproc_att_imx94_m71),
+	.ops		= &imx_rproc_ops_sm_lmm,
+	.cpuid		= 7,
+	.lmid		= 3,
+	.reset_vector_mask = GENMASK_U32(31, 16),
+};
+
+static const struct imx_rproc_dcfg imx_rproc_cfg_imx94_m33s = {
+	.att		= imx_rproc_att_imx94_m33s,
+	.att_size	= ARRAY_SIZE(imx_rproc_att_imx94_m33s),
+	.ops		= &imx_rproc_ops_sm_lmm,
+	.cpuid		= 8,
+	.lmid		= 1,
+	.reset_vector_mask = GENMASK_U32(31, 16),
+};
+
 static const struct imx_rproc_dcfg imx_rproc_cfg_imx95_m7 = {
 	.att		= imx_rproc_att_imx95_m7,
 	.att_size	= ARRAY_SIZE(imx_rproc_att_imx95_m7),
@@ -1462,6 +1546,7 @@ static const struct imx_rproc_dcfg imx_rproc_cfg_imx95_m7 = {
 	/* Must align with System Manager Firmware */
 	.cpuid		= 1, /* Use 1 as cpu id for M7 core */
 	.lmid		= 1, /* Use 1 as Logical Machine ID where M7 resides */
+	.reset_vector_mask = GENMASK_U32(31, 16),
 };
 
 static const struct of_device_id imx_rproc_of_match[] = {
@@ -1478,6 +1563,9 @@ static const struct of_device_id imx_rproc_of_match[] = {
 	{ .compatible = "fsl,imx8qm-cm4", .data = &imx_rproc_cfg_imx8qm },
 	{ .compatible = "fsl,imx8ulp-cm33", .data = &imx_rproc_cfg_imx8ulp },
 	{ .compatible = "fsl,imx93-cm33", .data = &imx_rproc_cfg_imx93 },
+	{ .compatible = "fsl,imx94-cm70", .data = &imx_rproc_cfg_imx94_m70 },
+	{ .compatible = "fsl,imx94-cm71", .data = &imx_rproc_cfg_imx94_m71 },
+	{ .compatible = "fsl,imx94-cm33s", .data = &imx_rproc_cfg_imx94_m33s },
 	{ .compatible = "fsl,imx95-cm7", .data = &imx_rproc_cfg_imx95_m7 },
 	{},
 };
diff --git a/drivers/remoteproc/imx_rproc.h b/drivers/remoteproc/imx_rproc.h
index d37e6f90548ce..0d7d48352a109 100644
--- a/drivers/remoteproc/imx_rproc.h
+++ b/drivers/remoteproc/imx_rproc.h
@@ -41,6 +41,8 @@ struct imx_rproc_dcfg {
 	/* For System Manager(SM) based SoCs */
 	u32				cpuid; /* ID of the remote core */
 	u32				lmid;  /* ID of the Logcial Machine */
+	/* reset_vector = elf_entry_addr & reset_vector_mask */
+	u32				reset_vector_mask;
 };
 
 #endif /* _IMX_RPROC_H */
diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c
index 50a9974f3202e..45a62cb980725 100644
--- a/drivers/remoteproc/xlnx_r5_remoteproc.c
+++ b/drivers/remoteproc/xlnx_r5_remoteproc.c
@@ -948,16 +948,6 @@ static struct zynqmp_r5_core *zynqmp_r5_add_rproc_core(struct device *cdev)
 		goto free_rproc;
 	}
 
-	/*
-	 * If firmware is already available in the memory then move rproc state
-	 * to DETACHED. Firmware can be preloaded via debugger or by any other
-	 * agent (processors) in the system.
-	 * If firmware isn't available in the memory and resource table isn't
-	 * found, then rproc state remains OFFLINE.
-	 */
-	if (!zynqmp_r5_get_rsc_table_va(r5_core))
-		r5_rproc->state = RPROC_DETACHED;
-
 	r5_core->rproc = r5_rproc;
 	return r5_core;
 
@@ -1210,6 +1200,7 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster,
 {
 	struct device *dev = cluster->dev;
 	struct zynqmp_r5_core *r5_core;
+	u32 req, usage, status;
 	int ret = -EINVAL, i;
 
 	r5_core = cluster->r5_cores[0];
@@ -1255,6 +1246,41 @@ static int zynqmp_r5_core_init(struct zynqmp_r5_cluster *cluster,
 		ret = zynqmp_r5_get_sram_banks(r5_core);
 		if (ret)
 			return ret;
+
+		/*
+		 * It is possible that firmware is loaded into the memory, but
+		 * RPU (remote) is not running. In such case, RPU state will be
+		 * moved to RPROC_DETACHED wrongfully. To avoid it first make
+		 * sure RPU is power-on and out of reset before parsing for the
+		 * resource table.
+		 */
+		ret = zynqmp_pm_get_rpu_node_status(r5_core->pm_domain_id,
+						    &status, &req, &usage);
+		if (ret) {
+			dev_warn(r5_core->dev,
+				 "failed to get rpu node status, err %d\n", ret);
+			continue;
+		}
+
+		/*
+		 * If RPU state is power on and out of reset i.e. running, then
+		 * assign RPROC_DETACHED state. If the RPU is not out of reset
+		 * then do not attempt to attach to the remote processor.
+		 */
+		if (status == PM_NODE_RUNNING) {
+			/*
+			 * Not all the firmware that is running on the remote
+			 * core is expected to have the resource table. The
+			 * firmware might not use RPMsg at all, and in that case
+			 * resource table becomes irrelevant. However, we still
+			 * need to make sure that running core is not reported
+			 * as offline. so do not decide remote core state based
+			 * on the resource table availability
+			 */
+			if (zynqmp_r5_get_rsc_table_va(r5_core))
+				dev_dbg(r5_core->dev, "rsc tbl not found\n");
+			r5_core->rproc->state = RPROC_DETACHED;
+		}
 	}
 
 	return 0;
diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c
index 988fc291241d2..b69f994881111 100644
--- a/drivers/resctrl/mpam_devices.c
+++ b/drivers/resctrl/mpam_devices.c
@@ -224,6 +224,24 @@ static inline void _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val
 
 #define mpam_write_monsel_reg(msc, reg, val)   _mpam_write_monsel_reg(msc, MSMON_##reg, val)
 
+static bool mpam_msc_check_aidr(struct mpam_msc *msc)
+{
+	u32 aidr = __mpam_read_reg(msc, MPAMF_AIDR);
+	u32 major = FIELD_GET(MPAMF_AIDR_ARCH_MAJOR_REV, aidr);
+	u32 minor = FIELD_GET(MPAMF_AIDR_ARCH_MINOR_REV, aidr);
+
+	/*
+	 * v0.0 and >v2.x aren't supported, but anything else should be backward
+	 * compatible to v0.1 or v1.0.
+	 */
+	if (!major && !minor)
+		return false;
+	if (major > 1)
+		return false;
+
+	return true;
+}
+
 static u64 mpam_msc_read_idr(struct mpam_msc *msc)
 {
 	u64 idr_high = 0, idr_low;
@@ -945,9 +963,8 @@ static int mpam_msc_hw_probe(struct mpam_msc *msc)
 
 	lockdep_assert_held(&msc->probe_lock);
 
-	idr = __mpam_read_reg(msc, MPAMF_AIDR);
-	if ((idr & MPAMF_AIDR_ARCH_MAJOR_REV) != MPAM_ARCHITECTURE_V1) {
-		dev_err_once(dev, "MSC does not match MPAM architecture v1.x\n");
+	if (!mpam_msc_check_aidr(msc)) {
+		dev_err_once(dev, "MSC does not match architecture v1.x\n");
 		return -EIO;
 	}
 
diff --git a/drivers/rtc/rtc-ab8500.c b/drivers/rtc/rtc-ab8500.c
index ed2b6b8bb3bf8..c6147837f9570 100644
--- a/drivers/rtc/rtc-ab8500.c
+++ b/drivers/rtc/rtc-ab8500.c
@@ -284,11 +284,10 @@ static ssize_t ab8500_sysfs_show_rtc_calibration(struct device *dev,
 	retval = ab8500_rtc_get_calibration(dev, &calibration);
 	if (retval < 0) {
 		dev_err(dev, "Failed to read RTC calibration attribute\n");
-		sprintf(buf, "0\n");
 		return retval;
 	}
 
-	return sprintf(buf, "%d\n", calibration);
+	return sysfs_emit(buf, "%d\n", calibration);
 }
 
 static DEVICE_ATTR(rtc_calibration, S_IRUGO | S_IWUSR,
diff --git a/drivers/rtc/rtc-rv3032.c b/drivers/rtc/rtc-rv3032.c
index 6c09da7738e1a..6bafdec637aea 100644
--- a/drivers/rtc/rtc-rv3032.c
+++ b/drivers/rtc/rtc-rv3032.c
@@ -376,7 +376,7 @@ static int rv3032_read_offset(struct device *dev, long *offset)
 	if (ret < 0)
 		return ret;
 
-	steps = sign_extend32(FIELD_GET(RV3032_OFFSET_MSK, value), 5);
+	steps = FIELD_GET_SIGNED(RV3032_OFFSET_MSK, value);
 
 	*offset = DIV_ROUND_CLOSEST(steps * OFFSET_STEP_PPT, 1000);
 
diff --git a/drivers/s390/char/sclp_mem.c b/drivers/s390/char/sclp_mem.c
index 78c054e26d177..6df1926d4c62a 100644
--- a/drivers/s390/char/sclp_mem.c
+++ b/drivers/s390/char/sclp_mem.c
@@ -204,7 +204,7 @@ static ssize_t sclp_config_mem_store(struct kobject *kobj, struct kobj_attribute
 	addr = sclp_mem->id * block_size;
 	/*
 	 * Hold device_hotplug_lock when adding/removing memory blocks.
-	 * Additionally, also protect calls to find_memory_block() and
+	 * Additionally, also protect calls to memory_block_get() and
 	 * sclp_attach_storage().
 	 */
 	rc = lock_device_hotplug_sysfs();
@@ -231,20 +231,19 @@ static ssize_t sclp_config_mem_store(struct kobject *kobj, struct kobj_attribute
 			sclp_mem_change_state(addr, block_size, 0);
 			goto out_unlock;
 		}
-		mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(addr)));
-		put_device(&mem->dev);
+		mem = memory_block_get(phys_to_block_id(addr));
+		memory_block_put(mem);
 		WRITE_ONCE(sclp_mem->config, 1);
 	} else {
 		if (!sclp_mem->config)
 			goto out_unlock;
-		mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(addr)));
+		mem = memory_block_get(phys_to_block_id(addr));
 		if (mem->state != MEM_OFFLINE) {
-			put_device(&mem->dev);
+			memory_block_put(mem);
 			rc = -EBUSY;
 			goto out_unlock;
 		}
-		/* drop the ref just got via find_memory_block() */
-		put_device(&mem->dev);
+		memory_block_put(mem);
 		sclp_mem_change_state(addr, block_size, 0);
 		__remove_memory(addr, block_size);
 #ifdef CONFIG_KASAN
@@ -294,11 +293,11 @@ static ssize_t sclp_memmap_on_memory_store(struct kobject *kobj, struct kobj_att
 		return rc;
 	block_size = memory_block_size_bytes();
 	sclp_mem = container_of(kobj, struct sclp_mem, kobj);
-	mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(sclp_mem->id * block_size)));
+	mem = memory_block_get(phys_to_block_id(sclp_mem->id * block_size));
 	if (!mem) {
 		WRITE_ONCE(sclp_mem->memmap_on_memory, value);
 	} else {
-		put_device(&mem->dev);
+		memory_block_put(mem);
 		rc = -EBUSY;
 	}
 	unlock_device_hotplug();
diff --git a/drivers/s390/char/sclp_pci.c b/drivers/s390/char/sclp_pci.c
index 899063e64aef0..d61a7fc0dd610 100644
--- a/drivers/s390/char/sclp_pci.c
+++ b/drivers/s390/char/sclp_pci.c
@@ -98,6 +98,7 @@ static int sclp_pci_check_report(struct zpci_report_error_header *report)
 	case SCLP_ERRNOTIFY_AQ_REPAIR:
 	case SCLP_ERRNOTIFY_AQ_INFO_LOG:
 	case SCLP_ERRNOTIFY_AQ_OPTICS_DATA:
+	case SCLP_ERRNOTIFY_AQ_NVME_SMART_LOG:
 		break;
 	default:
 		return -EINVAL;
diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c
index fbb58edd6274d..9689f722c863c 100644
--- a/drivers/s390/cio/chsc.c
+++ b/drivers/s390/cio/chsc.c
@@ -1142,8 +1142,8 @@ int __init chsc_init(void)
 {
 	int ret;
 
-	sei_page = (void *)get_zeroed_page(GFP_KERNEL);
-	chsc_page = (void *)get_zeroed_page(GFP_KERNEL);
+	sei_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
+	chsc_page = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!sei_page || !chsc_page) {
 		ret = -ENOMEM;
 		goto out_err;
diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c
index 73413417a2ce2..b6cb8bb8bcc4b 100644
--- a/drivers/s390/cio/chsc_sch.c
+++ b/drivers/s390/cio/chsc_sch.c
@@ -292,7 +292,7 @@ static int chsc_ioctl_start(void __user *user_area)
 	if (!css_general_characteristics.dynio)
 		/* It makes no sense to try. */
 		return -EOPNOTSUPP;
-	chsc_area = (void *)get_zeroed_page(GFP_KERNEL);
+	chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL);
 	if (!chsc_area)
 		return -ENOMEM;
 	request = kzalloc_obj(*request);
@@ -340,7 +340,7 @@ static int chsc_ioctl_on_close_set(void __user *user_area)
 		ret = -ENOMEM;
 		goto out_unlock;
 	}
-	on_close_chsc_area = (void *)get_zeroed_page(GFP_KERNEL);
+	on_close_chsc_area = (void *)get_zeroed_page(GFP_DMA | GFP_KERNEL);
 	if (!on_close_chsc_area) {
 		ret = -ENOMEM;
 		goto out_free_request;
@@ -392,7 +392,7 @@ static int chsc_ioctl_start_sync(void __user *user_area)
 	struct chsc_sync_area *chsc_area;
 	int ret, ccode;
 
-	chsc_area = (void *)get_zeroed_page(GFP_KERNEL);
+	chsc_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!chsc_area)
 		return -ENOMEM;
 	if (copy_from_user(chsc_area, user_area, PAGE_SIZE)) {
@@ -438,7 +438,7 @@ static int chsc_ioctl_info_channel_path(void __user *user_cd)
 		u8 data[PAGE_SIZE - 20];
 	} __attribute__ ((packed)) *scpcd_area;
 
-	scpcd_area = (void *)get_zeroed_page(GFP_KERNEL);
+	scpcd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!scpcd_area)
 		return -ENOMEM;
 	cd = kzalloc_obj(*cd);
@@ -500,7 +500,7 @@ static int chsc_ioctl_info_cu(void __user *user_cd)
 		u8 data[PAGE_SIZE - 20];
 	} __attribute__ ((packed)) *scucd_area;
 
-	scucd_area = (void *)get_zeroed_page(GFP_KERNEL);
+	scucd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!scucd_area)
 		return -ENOMEM;
 	cd = kzalloc_obj(*cd);
@@ -563,7 +563,7 @@ static int chsc_ioctl_info_sch_cu(void __user *user_cud)
 		u8 data[PAGE_SIZE - 20];
 	} __attribute__ ((packed)) *sscud_area;
 
-	sscud_area = (void *)get_zeroed_page(GFP_KERNEL);
+	sscud_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!sscud_area)
 		return -ENOMEM;
 	cud = kzalloc_obj(*cud);
@@ -625,7 +625,7 @@ static int chsc_ioctl_conf_info(void __user *user_ci)
 		u8 data[PAGE_SIZE - 20];
 	} __attribute__ ((packed)) *sci_area;
 
-	sci_area = (void *)get_zeroed_page(GFP_KERNEL);
+	sci_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!sci_area)
 		return -ENOMEM;
 	ci = kzalloc_obj(*ci);
@@ -696,7 +696,7 @@ static int chsc_ioctl_conf_comp_list(void __user *user_ccl)
 		u32 res;
 	} __attribute__ ((packed)) *cssids_parm;
 
-	sccl_area = (void *)get_zeroed_page(GFP_KERNEL);
+	sccl_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!sccl_area)
 		return -ENOMEM;
 	ccl = kzalloc_obj(*ccl);
@@ -756,7 +756,7 @@ static int chsc_ioctl_chpd(void __user *user_chpd)
 	int ret;
 
 	chpd = kzalloc_obj(*chpd);
-	scpd_area = (void *)get_zeroed_page(GFP_KERNEL);
+	scpd_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!scpd_area || !chpd) {
 		ret = -ENOMEM;
 		goto out_free;
@@ -796,7 +796,7 @@ static int chsc_ioctl_dcal(void __user *user_dcal)
 		u8 data[PAGE_SIZE - 36];
 	} __attribute__ ((packed)) *sdcal_area;
 
-	sdcal_area = (void *)get_zeroed_page(GFP_KERNEL);
+	sdcal_area = (void *)get_zeroed_page(GFP_KERNEL | GFP_DMA);
 	if (!sdcal_area)
 		return -ENOMEM;
 	dcal = kzalloc_obj(*dcal);
diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c
index 5bbb112b1619a..fb591118ecb2a 100644
--- a/drivers/s390/cio/device.c
+++ b/drivers/s390/cio/device.c
@@ -1328,7 +1328,7 @@ static int purge_fn(struct subchannel *sch, void *data)
 
 	cdev = sch_get_cdev(sch);
 	if (cdev) {
-		if (cdev->private->state != DEV_STATE_OFFLINE)
+		if (cdev->online)
 			goto unlock;
 
 		if (atomic_cmpxchg(&cdev->private->onoff, 0, 1) != 0)
diff --git a/drivers/s390/cio/scm.c b/drivers/s390/cio/scm.c
index d13ed1011c039..171212a6d2d9c 100644
--- a/drivers/s390/cio/scm.c
+++ b/drivers/s390/cio/scm.c
@@ -229,7 +229,7 @@ int scm_update_information(void)
 	size_t num;
 	int ret;
 
-	scm_info = (void *)__get_free_page(GFP_KERNEL);
+	scm_info = (void *)__get_free_page(GFP_KERNEL | GFP_DMA);
 	if (!scm_info)
 		return -ENOMEM;
 
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index f24e27add721d..6a7497db5fb95 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -744,6 +744,23 @@ void ap_send_online_uevent(struct ap_device *ap_dev, int online)
 }
 EXPORT_SYMBOL(ap_send_online_uevent);
 
+void ap_send_se_bind_uevent(struct ap_device *ap_dev)
+{
+	char *envp[] = { "SE_BIND=1", NULL };
+
+	kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp);
+}
+
+void ap_send_se_assoc_uevent(struct ap_device *ap_dev, unsigned int assoc_idx)
+{
+	char buf[32];
+	char *envp[] = { buf, NULL };
+
+	snprintf(buf, sizeof(buf), "SE_ASSOC=%u", assoc_idx);
+
+	kobject_uevent_env(&ap_dev->device.kobj, KOBJ_CHANGE, envp);
+}
+
 static void ap_send_mask_changed_uevent(unsigned long *newapm,
 					unsigned long *newaqm)
 {
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
index 04ea256ecf919..ca5e142c9b249 100644
--- a/drivers/s390/crypto/ap_bus.h
+++ b/drivers/s390/crypto/ap_bus.h
@@ -373,5 +373,7 @@ int ap_wait_apqn_bindings_complete(unsigned long timeout);
 
 void ap_send_config_uevent(struct ap_device *ap_dev, bool cfg);
 void ap_send_online_uevent(struct ap_device *ap_dev, int online);
+void ap_send_se_bind_uevent(struct ap_device *ap_dev);
+void ap_send_se_assoc_uevent(struct ap_device *ap_dev, unsigned int assoc_idx);
 
 #endif /* _AP_BUS_H_ */
diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c
index ca9819e6f7e76..232b786d81d14 100644
--- a/drivers/s390/crypto/ap_queue.c
+++ b/drivers/s390/crypto/ap_queue.c
@@ -478,6 +478,7 @@ static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq)
 		pr_debug("queue 0x%02x.%04x associated with %u\n",
 			 AP_QID_CARD(aq->qid),
 			 AP_QID_QUEUE(aq->qid), aq->assoc_idx);
+		ap_send_se_assoc_uevent(&aq->ap_dev, aq->assoc_idx);
 		return AP_SM_WAIT_NONE;
 	case AP_BS_Q_USABLE_NO_SECURE_KEY:
 		/* association still pending */
@@ -1023,6 +1024,7 @@ static ssize_t se_bind_store(struct device *dev,
 	/* SE bind was successful */
 	AP_DBF_INFO("%s bapq(0x%02x.%04x) success\n", __func__,
 		    AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
+	ap_send_se_bind_uevent(&aq->ap_dev);
 	rc = count;
 
 out:
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index fc8e8b0bfa391..c3042393af234 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -474,7 +474,6 @@ config SCSI_ADVANSYS
 	tristate "AdvanSys SCSI support"
 	depends on SCSI
 	depends on (ISA || EISA || PCI) && HAS_IOPORT
-	depends on ISA_DMA_API || !ISA
 	help
 	  This is a driver for all SCSI host adapters manufactured by
 	  AdvanSys. It is documented in the kernel source in
diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile
index 16de3e41f94c4..842c254bb2269 100644
--- a/drivers/scsi/Makefile
+++ b/drivers/scsi/Makefile
@@ -177,8 +177,8 @@ sd_mod-$(CONFIG_BLK_DEV_ZONED) += sd_zbc.o
 
 sr_mod-objs	:= sr.o sr_ioctl.o sr_vendor.o
 ncr53c8xx-flags-$(CONFIG_SCSI_ZALON) \
-		:= -DCONFIG_NCR53C8XX_PREFETCH -DSCSI_NCR_BIG_ENDIAN \
-			-DCONFIG_SCSI_NCR53C8XX_NO_WORD_TRANSFERS
+		:= -DNCR53C8XX_PREFETCH -DSCSI_NCR_BIG_ENDIAN \
+			-DSCSI_NCR53C8XX_NO_WORD_TRANSFERS
 CFLAGS_ncr53c8xx.o	:= $(ncr53c8xx-flags-y) $(ncr53c8xx-flags-m)
 zalon7xx-objs	:= zalon.o ncr53c8xx.o
 
diff --git a/drivers/scsi/advansys.c b/drivers/scsi/advansys.c
index fcf059bf41e85..5cdbf2bdb13df 100644
--- a/drivers/scsi/advansys.c
+++ b/drivers/scsi/advansys.c
@@ -36,7 +36,6 @@
 #include <linux/dmapool.h>
 
 #include <asm/io.h>
-#include <asm/dma.h>
 
 #include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_device.h>
diff --git a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
index 14d563e82d208..213d5b5dea94f 100644
--- a/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
+++ b/drivers/scsi/hisi_sas/hisi_sas_v3_hw.c
@@ -2977,7 +2977,7 @@ static int sdev_configure_v3_hw(struct scsi_device *sdev,
 		return 0;
 
 	if (!device_link_add(&sdev->sdev_gendev, dev,
-			     DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE)) {
+			     DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME | DL_FLAG_RPM_ACTIVE)) {
 		if (pm_runtime_enabled(dev)) {
 			dev_info(dev, "add device link failed, disable runtime PM for the host\n");
 			pm_runtime_disable(dev);
@@ -2987,6 +2987,15 @@ static int sdev_configure_v3_hw(struct scsi_device *sdev,
 	return 0;
 }
 
+static void hisi_sas_sdev_destroy(struct scsi_device *sdev)
+{
+	struct Scsi_Host *shost = dev_to_shost(&sdev->sdev_gendev);
+	struct hisi_hba *hisi_hba = shost_priv(shost);
+	struct device *dev = hisi_hba->dev;
+
+	device_link_remove(&sdev->sdev_gendev, dev);
+}
+
 static struct attribute *host_v3_hw_attrs[] = {
 	&dev_attr_phy_event_threshold.attr,
 	&dev_attr_intr_conv_v3_hw.attr,
@@ -3401,6 +3410,7 @@ static const struct scsi_host_template sht_v3_hw = {
 	.sg_tablesize		= HISI_SAS_SGE_PAGE_CNT,
 	.sg_prot_tablesize	= HISI_SAS_SGE_PAGE_CNT,
 	.sdev_init		= hisi_sas_sdev_init,
+	.sdev_destroy		= hisi_sas_sdev_destroy,
 	.shost_groups		= host_v3_hw_groups,
 	.sdev_groups		= sdev_groups_v3_hw,
 	.tag_alloc_policy_rr	= true,
diff --git a/drivers/scsi/isci/host.c b/drivers/scsi/isci/host.c
index 6d2f4c831df74..ff199bab5d1a8 100644
--- a/drivers/scsi/isci/host.c
+++ b/drivers/scsi/isci/host.c
@@ -1252,6 +1252,9 @@ void isci_host_deinit(struct isci_host *ihost)
 
 	wait_for_stop(ihost);
 
+	/* No further IRQ-driven scheduling can happen past wait_for_stop(). */
+	tasklet_kill(&ihost->completion_tasklet);
+
 	/* phy stop is after controller stop to allow port and device to
 	 * go idle before shutting down the phys, but the expectation is
 	 * that i/o has been shut off well before we reach this
diff --git a/drivers/scsi/isci/scu_task_context.h b/drivers/scsi/isci/scu_task_context.h
index 582d22d54689d..e56cd0fdc35f1 100644
--- a/drivers/scsi/isci/scu_task_context.h
+++ b/drivers/scsi/isci/scu_task_context.h
@@ -211,8 +211,6 @@ typedef enum {
 
 #define SCU_CONTEXT_COMMAND_LOGICAL_PORT_SHIFT           12
 #define SCU_CONTEXT_COMMAND_LOGICAL_PORT_MASK            0x00007000
-#define scu_get_command_reqeust_logical_port(x)	\
-	((x) & SCU_CONTEXT_COMMAND_LOGICAL_PORT_MASK)
 
 
 #define MAKE_SCU_CONTEXT_COMMAND_TYPE(type) \
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index 25857d6ed6e84..160f02f2f51d1 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -3012,7 +3012,7 @@ static void iscsi_host_dec_session_cnt(struct Scsi_Host *shost)
  * This can be used by software iscsi_transports that allocate
  * a session per scsi host.
  *
- * Callers should set cmds_max to the largest total numer (mgmt + scsi) of
+ * Callers should set cmds_max to the largest total number (mgmt + scsi) of
  * tasks they support. The iscsi layer reserves ISCSI_MGMT_CMDS_MAX tasks
  * for nop handling and login/logout requests.
  */
@@ -3307,7 +3307,7 @@ int iscsi_conn_start(struct iscsi_cls_conn *cls_conn)
 
 	if (conn->ping_timeout && !conn->recv_timeout) {
 		iscsi_conn_printk(KERN_ERR, conn, "invalid recv timeout of "
-				  "zero. Using 5 seconds\n.");
+				  "zero. Using 5 seconds.\n");
 		conn->recv_timeout = 5;
 	}
 
diff --git a/drivers/scsi/mvsas/mv_sas.h b/drivers/scsi/mvsas/mv_sas.h
index 09ce3f2241f24..7521f969aa87d 100644
--- a/drivers/scsi/mvsas/mv_sas.h
+++ b/drivers/scsi/mvsas/mv_sas.h
@@ -35,10 +35,10 @@
 #define MVS_ID_NOT_MAPPED	0x7f
 #define WIDE_PORT_MAX_PHY		4
 #define mv_printk(fmt, arg ...)	\
-	printk(KERN_DEBUG"%s %d:" fmt, __FILE__, __LINE__, ## arg)
+	printk(KERN_DEBUG "%s: " fmt, __func__, ## arg)
 #ifdef MV_DEBUG
-#define mv_dprintk(format, arg...)	\
-	printk(KERN_DEBUG"%s %d:" format, __FILE__, __LINE__, ## arg)
+#define mv_dprintk(fmt, arg...)	\
+	mv_printk(fmt, ## arg)
 #else
 #define mv_dprintk(format, arg...) no_printk(format, ## arg)
 #endif
diff --git a/drivers/scsi/ncr53c8xx.c b/drivers/scsi/ncr53c8xx.c
index 4a255aafed806..5369ca3fe4fd3 100644
--- a/drivers/scsi/ncr53c8xx.c
+++ b/drivers/scsi/ncr53c8xx.c
@@ -1776,7 +1776,7 @@ struct ncb {
 **	return from the subroutine.
 */
 
-#ifdef CONFIG_NCR53C8XX_PREFETCH
+#ifdef NCR53C8XX_PREFETCH
 #define PREFETCH_FLUSH_CNT	2
 #define PREFETCH_FLUSH		SCR_CALL, PADDRH (wait_dma),
 #else
diff --git a/drivers/scsi/ncr53c8xx.h b/drivers/scsi/ncr53c8xx.h
index be38c902859e0..2f6865ca1b87d 100644
--- a/drivers/scsi/ncr53c8xx.h
+++ b/drivers/scsi/ncr53c8xx.h
@@ -397,7 +397,7 @@
 
 #else
 
-#ifdef CONFIG_SCSI_NCR53C8XX_NO_WORD_TRANSFERS
+#ifdef SCSI_NCR53C8XX_NO_WORD_TRANSFERS
 /* Only 8 or 32 bit transfers allowed */
 #define INW_OFF(o)		(readb((char __iomem *)np->reg + ncr_offw(o)) << 8 | readb((char __iomem *)np->reg + ncr_offw(o) + 1))
 #else
@@ -405,7 +405,7 @@
 #endif
 #define INL_OFF(o)		readl_raw((char __iomem *)np->reg + (o))
 
-#ifdef CONFIG_SCSI_NCR53C8XX_NO_WORD_TRANSFERS
+#ifdef SCSI_NCR53C8XX_NO_WORD_TRANSFERS
 /* Only 8 or 32 bit transfers allowed */
 #define OUTW_OFF(o, val)	do { writeb((char)((val) >> 8), (char __iomem *)np->reg + ncr_offw(o)); writeb((char)(val), (char __iomem *)np->reg + ncr_offw(o) + 1); } while (0)
 #else
diff --git a/drivers/scsi/pm8001/pm8001_ctl.c b/drivers/scsi/pm8001/pm8001_ctl.c
index cbfda8c04e956..bb38b2d63acb8 100644
--- a/drivers/scsi/pm8001/pm8001_ctl.c
+++ b/drivers/scsi/pm8001/pm8001_ctl.c
@@ -826,6 +826,14 @@ static ssize_t pm8001_store_update_fw(struct device *cdev,
 		goto out;
 	}
 
+	if (pm8001_ha->controller_fatal_error) {
+		pm8001_dbg(pm8001_ha, FAIL,
+			   "controller in fatal error state, firmware update rejected\n");
+		pm8001_ha->fw_status = FAIL_PARAMETERS;
+		ret = -EINVAL;
+		goto out;
+	}
+
 	for (i = 0; flash_command_table[i].code != FLASH_CMD_NONE; i++) {
 		if (!memcmp(flash_command_table[i].command,
 				 cmd_ptr, strlen(cmd_ptr))) {
diff --git a/drivers/scsi/pm8001/pm80xx_hwi.c b/drivers/scsi/pm8001/pm80xx_hwi.c
index 954f307352e61..2c0fa7ab33d22 100644
--- a/drivers/scsi/pm8001/pm80xx_hwi.c
+++ b/drivers/scsi/pm8001/pm80xx_hwi.c
@@ -401,6 +401,13 @@ ssize_t pm80xx_get_non_fatal_dump(struct device *cdev,
 	char *buf_copy = buf;
 
 	temp = (u32 *)pm8001_ha->memoryMap.region[FORENSIC_MEM].virt_ptr;
+
+	if (pm8001_ha->controller_fatal_error) {
+		pm8001_dbg(pm8001_ha, FAIL,
+			   "non-fatal dump not available in fatal error state\n");
+		return -EINVAL;
+	}
+
 	if (++pm8001_ha->non_fatal_count == 1) {
 		if (pm8001_ha->chip_id == chip_8001) {
 			snprintf(pm8001_ha->forensic_info.data_buf.direct_data,
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index ef22a4228b855..a35a5f777d166 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -858,7 +858,7 @@ static int scsi_probe_lun(struct scsi_device *sdev, unsigned char *inq_result,
 }
 
 /**
- * scsi_add_lun - allocate and fully initialze a scsi_device
+ * scsi_add_lun - allocate and fully initialize a scsi_device
  * @sdev:	holds information to be stored in the new scsi_device
  * @inq_result:	holds the result of a previous INQUIRY to the LUN
  * @bflags:	black/white list flag
diff --git a/drivers/scsi/scsi_transport_srp.c b/drivers/scsi/scsi_transport_srp.c
index d71ab5fdb7588..a61cbb079ab4e 100644
--- a/drivers/scsi/scsi_transport_srp.c
+++ b/drivers/scsi/scsi_transport_srp.c
@@ -234,7 +234,7 @@ static ssize_t store_reconnect_delay(struct device *dev,
 
 	if (rport->reconnect_delay <= 0 && delay > 0 &&
 	    rport->state != SRP_RPORT_RUNNING) {
-		queue_delayed_work(system_long_wq, &rport->reconnect_work,
+		queue_delayed_work(system_dfl_long_wq, &rport->reconnect_work,
 				   delay * HZ);
 	} else if (delay <= 0) {
 		cancel_delayed_work(&rport->reconnect_work);
@@ -390,7 +390,7 @@ static void srp_reconnect_work(struct work_struct *work)
 		delay = rport->reconnect_delay *
 			clamp(rport->failed_reconnects - 10, 1, 100);
 		if (delay > 0)
-			queue_delayed_work(system_long_wq,
+			queue_delayed_work(system_dfl_long_wq,
 					   &rport->reconnect_work, delay * HZ);
 	}
 }
@@ -474,7 +474,7 @@ static void __srp_start_tl_fail_timers(struct srp_rport *rport)
 	if (rport->state == SRP_RPORT_LOST)
 		return;
 	if (delay > 0)
-		queue_delayed_work(system_long_wq, &rport->reconnect_work,
+		queue_delayed_work(system_dfl_long_wq, &rport->reconnect_work,
 				   1UL * delay * HZ);
 	if ((fast_io_fail_tmo >= 0 || dev_loss_tmo >= 0) &&
 	    srp_rport_set_state(rport, SRP_RPORT_BLOCKED) == 0) {
@@ -482,11 +482,11 @@ static void __srp_start_tl_fail_timers(struct srp_rport *rport)
 			 rport->state);
 		scsi_block_targets(shost, &shost->shost_gendev);
 		if (fast_io_fail_tmo >= 0)
-			queue_delayed_work(system_long_wq,
+			queue_delayed_work(system_dfl_long_wq,
 					   &rport->fast_io_fail_work,
 					   1UL * fast_io_fail_tmo * HZ);
 		if (dev_loss_tmo >= 0)
-			queue_delayed_work(system_long_wq,
+			queue_delayed_work(system_dfl_long_wq,
 					   &rport->dev_loss_work,
 					   1UL * dev_loss_tmo * HZ);
 	}
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index adc3fa55ca2c3..599e75f333343 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -2476,8 +2476,7 @@ sd_spinup_disk(struct scsi_disk *sdkp)
 {
 	static const u8 cmd[10] = { TEST_UNIT_READY };
 	unsigned long spintime_expire = 0;
-	int spintime, sense_valid = 0;
-	unsigned int the_result;
+	int the_result, spintime, sense_valid = 0;
 	struct scsi_sense_hdr sshdr;
 	struct scsi_failure failure_defs[] = {
 		/* Do not retry Medium Not Present */
diff --git a/drivers/scsi/snic/vnic_dev.c b/drivers/scsi/snic/vnic_dev.c
index ed7771e628541..22303f8275836 100644
--- a/drivers/scsi/snic/vnic_dev.c
+++ b/drivers/scsi/snic/vnic_dev.c
@@ -132,7 +132,6 @@ static int vnic_dev_discover_res(struct vnic_dev *vdev,
 		case RES_TYPE_INTR_PBA_LEGACY:
 		case RES_TYPE_DEVCMD:
 		case RES_TYPE_DEVCMD2:
-			len = count;
 			break;
 
 		default:
diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c
index 6977ca8a06582..571ea549152b1 100644
--- a/drivers/scsi/storvsc_drv.c
+++ b/drivers/scsi/storvsc_drv.c
@@ -156,7 +156,7 @@ static bool hv_dev_is_fc(struct hv_device *hv_dev);
 #define STORVSC_LOGGING_WARN	2
 
 static int logging_level = STORVSC_LOGGING_ERROR;
-module_param(logging_level, int, S_IRUGO|S_IWUSR);
+module_param(logging_level, int, 0644);
 MODULE_PARM_DESC(logging_level,
 	"Logging level, 0 - None, 1 - Error (default), 2 - Warning.");
 
@@ -345,17 +345,17 @@ static int storvsc_change_queue_depth(struct scsi_device *sdev, int queue_depth)
 static int storvsc_vcpus_per_sub_channel = 4;
 static unsigned int storvsc_max_hw_queues;
 
-module_param(storvsc_ringbuffer_size, int, S_IRUGO);
+module_param(storvsc_ringbuffer_size, int, 0444);
 MODULE_PARM_DESC(storvsc_ringbuffer_size, "Ring buffer size (bytes)");
 
 module_param(storvsc_max_hw_queues, uint, 0644);
 MODULE_PARM_DESC(storvsc_max_hw_queues, "Maximum number of hardware queues");
 
-module_param(storvsc_vcpus_per_sub_channel, int, S_IRUGO);
+module_param(storvsc_vcpus_per_sub_channel, int, 0444);
 MODULE_PARM_DESC(storvsc_vcpus_per_sub_channel, "Ratio of VCPUs to subchannels");
 
 static int ring_avail_percent_lowater = 10;
-module_param(ring_avail_percent_lowater, int, S_IRUGO);
+module_param(ring_avail_percent_lowater, int, 0444);
 MODULE_PARM_DESC(ring_avail_percent_lowater,
 		"Select a channel if available ring size > this in percent");
 
diff --git a/drivers/soc/aspeed/Kconfig b/drivers/soc/aspeed/Kconfig
index f579ee0b5afac..63a656449a1ae 100644
--- a/drivers/soc/aspeed/Kconfig
+++ b/drivers/soc/aspeed/Kconfig
@@ -46,7 +46,6 @@ config ASPEED_P2A_CTRL
 
 config ASPEED_SOCINFO
 	bool "ASPEED SoC Information driver"
-	default ARCH_ASPEED
 	select SOC_BUS
 	default ARCH_ASPEED
 	help
diff --git a/drivers/soc/aspeed/aspeed-lpc-ctrl.c b/drivers/soc/aspeed/aspeed-lpc-ctrl.c
index b7dbb12bd095b..4d8574cac703b 100644
--- a/drivers/soc/aspeed/aspeed-lpc-ctrl.c
+++ b/drivers/soc/aspeed/aspeed-lpc-ctrl.c
@@ -338,6 +338,7 @@ static const struct of_device_id aspeed_lpc_ctrl_match[] = {
 	{ .compatible = "aspeed,ast2600-lpc-ctrl" },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, aspeed_lpc_ctrl_match);
 
 static struct platform_driver aspeed_lpc_ctrl_driver = {
 	.driver = {
@@ -350,7 +351,6 @@ static struct platform_driver aspeed_lpc_ctrl_driver = {
 
 module_platform_driver(aspeed_lpc_ctrl_driver);
 
-MODULE_DEVICE_TABLE(of, aspeed_lpc_ctrl_match);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Cyril Bur <cyrilbur@gmail.com>");
 MODULE_DESCRIPTION("Control for ASPEED LPC HOST to BMC mappings");
diff --git a/drivers/soc/aspeed/aspeed-lpc-snoop.c b/drivers/soc/aspeed/aspeed-lpc-snoop.c
index b03310c0830d2..28e491fffc7d7 100644
--- a/drivers/soc/aspeed/aspeed-lpc-snoop.c
+++ b/drivers/soc/aspeed/aspeed-lpc-snoop.c
@@ -366,6 +366,7 @@ static const struct of_device_id aspeed_lpc_snoop_match[] = {
 	  .data = &ast2500_model_data },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, aspeed_lpc_snoop_match);
 
 static struct platform_driver aspeed_lpc_snoop_driver = {
 	.driver = {
@@ -378,7 +379,6 @@ static struct platform_driver aspeed_lpc_snoop_driver = {
 
 module_platform_driver(aspeed_lpc_snoop_driver);
 
-MODULE_DEVICE_TABLE(of, aspeed_lpc_snoop_match);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Robert Lippert <rlippert@google.com>");
 MODULE_DESCRIPTION("Linux driver to control Aspeed LPC snoop functionality");
diff --git a/drivers/soc/aspeed/aspeed-p2a-ctrl.c b/drivers/soc/aspeed/aspeed-p2a-ctrl.c
index d88abc5f195b5..b6b7b80f842a2 100644
--- a/drivers/soc/aspeed/aspeed-p2a-ctrl.c
+++ b/drivers/soc/aspeed/aspeed-p2a-ctrl.c
@@ -416,6 +416,7 @@ static const struct of_device_id aspeed_p2a_ctrl_match[] = {
 	  .data = &ast2500_model_data },
 	{ },
 };
+MODULE_DEVICE_TABLE(of, aspeed_p2a_ctrl_match);
 
 static struct platform_driver aspeed_p2a_ctrl_driver = {
 	.driver = {
@@ -428,7 +429,6 @@ static struct platform_driver aspeed_p2a_ctrl_driver = {
 
 module_platform_driver(aspeed_p2a_ctrl_driver);
 
-MODULE_DEVICE_TABLE(of, aspeed_p2a_ctrl_match);
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Patrick Venture <venture@google.com>");
 MODULE_DESCRIPTION("Control for aspeed 2400/2500 P2A VGA HOST to BMC mappings");
diff --git a/drivers/soc/bcm/brcmstb/common.c b/drivers/soc/bcm/brcmstb/common.c
index 2da79bd429305..7be0374f59430 100644
--- a/drivers/soc/bcm/brcmstb/common.c
+++ b/drivers/soc/bcm/brcmstb/common.c
@@ -11,18 +11,22 @@
 #include <linux/soc/brcmstb/brcmstb.h>
 #include <linux/sys_soc.h>
 
-static u32 family_id;
-static u32 product_id;
+struct brcmstb_soc_info {
+	u32 family_id;
+	u32 product_id;
+};
+
+static struct brcmstb_soc_info *soc_info;
 
 u32 brcmstb_get_family_id(void)
 {
-	return family_id;
+	return soc_info ? soc_info->family_id : 0;
 }
 EXPORT_SYMBOL(brcmstb_get_family_id);
 
 u32 brcmstb_get_product_id(void)
 {
-	return product_id;
+	return soc_info ? soc_info->product_id : 0;
 }
 EXPORT_SYMBOL(brcmstb_get_product_id);
 
@@ -40,10 +44,12 @@ static const struct of_device_id sun_top_ctrl_match[] = {
 	{ }
 };
 
-static int __init brcmstb_soc_device_early_init(void)
+static int __init brcmstb_soc_device_init(void)
 {
+	struct soc_device_attribute *soc_dev_attr;
 	struct device_node *sun_top_ctrl;
 	void __iomem *sun_top_ctrl_base;
+	struct soc_device *soc_dev;
 	int ret = 0;
 
 	/* We could be on a multi-platform kernel, don't make this fatal but
@@ -51,63 +57,61 @@ static int __init brcmstb_soc_device_early_init(void)
 	 */
 	sun_top_ctrl = of_find_matching_node(NULL, sun_top_ctrl_match);
 	if (!sun_top_ctrl)
-		return ret;
+		return 0;
 
 	sun_top_ctrl_base = of_iomap(sun_top_ctrl, 0);
 	if (!sun_top_ctrl_base) {
 		ret = -ENODEV;
-		goto out;
+		goto out_put_node;
 	}
 
-	family_id = readl(sun_top_ctrl_base);
-	product_id = readl(sun_top_ctrl_base + 0x4);
-	iounmap(sun_top_ctrl_base);
-out:
-	of_node_put(sun_top_ctrl);
-	return ret;
-}
-early_initcall(brcmstb_soc_device_early_init);
-
-static int __init brcmstb_soc_device_init(void)
-{
-	struct soc_device_attribute *soc_dev_attr;
-	struct device_node *sun_top_ctrl;
-	struct soc_device *soc_dev;
-	int ret = 0;
+	soc_info = kzalloc(sizeof(*soc_info), GFP_KERNEL);
+	if (!soc_info) {
+		ret = -ENOMEM;
+		goto out_unmap;
+	}
 
-	/* We could be on a multi-platform kernel, don't make this fatal but
-	 * bail out early
-	 */
-	sun_top_ctrl = of_find_matching_node(NULL, sun_top_ctrl_match);
-	if (!sun_top_ctrl)
-		return ret;
+	soc_info->family_id = readl(sun_top_ctrl_base);
+	soc_info->product_id = readl(sun_top_ctrl_base + 0x4);
 
 	soc_dev_attr = kzalloc_obj(*soc_dev_attr);
 	if (!soc_dev_attr) {
 		ret = -ENOMEM;
-		goto out;
+		goto out_free_info;
 	}
 
 	soc_dev_attr->family = kasprintf(GFP_KERNEL, "%x",
-					 family_id >> 28 ?
-					 family_id >> 16 : family_id >> 8);
+					 soc_info->family_id >> 28 ?
+					 soc_info->family_id >> 16 : soc_info->family_id >> 8);
 	soc_dev_attr->soc_id = kasprintf(GFP_KERNEL, "%x",
-					 product_id >> 28 ?
-					 product_id >> 16 : product_id >> 8);
+					 soc_info->product_id >> 28 ?
+					 soc_info->product_id >> 16 : soc_info->product_id >> 8);
 	soc_dev_attr->revision = kasprintf(GFP_KERNEL, "%c%d",
-					 ((product_id & 0xf0) >> 4) + 'A',
-					   product_id & 0xf);
+					 ((soc_info->product_id & 0xf0) >> 4) + 'A',
+					   soc_info->product_id & 0xf);
 
 	soc_dev = soc_device_register(soc_dev_attr);
 	if (IS_ERR(soc_dev)) {
-		kfree(soc_dev_attr->family);
-		kfree(soc_dev_attr->soc_id);
-		kfree(soc_dev_attr->revision);
-		kfree(soc_dev_attr);
-		ret = -ENOMEM;
+		ret = PTR_ERR(soc_dev);
+		goto out_free_attr;
 	}
-out:
+
+	iounmap(sun_top_ctrl_base);
+	of_node_put(sun_top_ctrl);
+	return 0;
+
+out_free_attr:
+	kfree(soc_dev_attr->revision);
+	kfree(soc_dev_attr->soc_id);
+	kfree(soc_dev_attr->family);
+	kfree(soc_dev_attr);
+out_free_info:
+	kfree(soc_info);
+	soc_info = NULL;
+out_unmap:
+	iounmap(sun_top_ctrl_base);
+out_put_node:
 	of_node_put(sun_top_ctrl);
 	return ret;
 }
-arch_initcall(brcmstb_soc_device_init);
+early_initcall(brcmstb_soc_device_init);
diff --git a/drivers/soc/fsl/qe/qe.c b/drivers/soc/fsl/qe/qe.c
index 70b6eddb867b9..3ecc4ce9cfa25 100644
--- a/drivers/soc/fsl/qe/qe.c
+++ b/drivers/soc/fsl/qe/qe.c
@@ -89,6 +89,9 @@ void qe_reset(void)
 	if (qe_immr == NULL)
 		qe_immr = ioremap(get_qe_base(), QE_IMMAP_SIZE);
 
+	if (!qe_immr)
+		panic("QE:ioremap failed!");
+
 	qe_snums_init();
 
 	qe_issue_cmd(QE_RESET, QE_CR_SUBBLOCK_INVALID,
diff --git a/drivers/soc/fsl/qe/qe_ports_ic.c b/drivers/soc/fsl/qe/qe_ports_ic.c
index 8e2107e2cde5b..9b0bba64e91e4 100644
--- a/drivers/soc/fsl/qe/qe_ports_ic.c
+++ b/drivers/soc/fsl/qe/qe_ports_ic.c
@@ -17,6 +17,7 @@
 struct qepic_data {
 	void __iomem *reg;
 	struct irq_domain *host;
+	int irq;
 };
 
 static void qepic_mask(struct irq_data *d)
@@ -92,11 +93,18 @@ static const struct irq_domain_ops qepic_host_ops = {
 	.map = qepic_host_map,
 };
 
+static void qepic_remove(void *res)
+{
+	struct qepic_data *data = res;
+
+	irq_set_chained_handler_and_data(data->irq, NULL, NULL);
+	irq_domain_remove(data->host);
+}
+
 static int qepic_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct qepic_data *data;
-	int irq;
 
 	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
 	if (!data)
@@ -106,17 +114,18 @@ static int qepic_probe(struct platform_device *pdev)
 	if (IS_ERR(data->reg))
 		return PTR_ERR(data->reg);
 
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
+	data->irq = platform_get_irq(pdev, 0);
+	if (data->irq < 0)
+		return data->irq;
 
-	data->host = irq_domain_add_linear(dev->of_node, 32, &qepic_host_ops, data);
+	data->host = irq_domain_create_linear(dev_fwnode(dev), 32, &qepic_host_ops, data);
 	if (!data->host)
 		return -ENODEV;
 
-	irq_set_chained_handler_and_data(irq, qepic_cascade, data);
+	irq_set_chained_handler_and_data(data->irq, qepic_cascade, data);
+
+	return devm_add_action_or_reset(dev, qepic_remove, data);
 
-	return 0;
 }
 
 static const struct of_device_id qepic_match[] = {
diff --git a/drivers/soc/imx/soc-imx8m.c b/drivers/soc/imx/soc-imx8m.c
index 77763a107edbd..fc080e56f50d4 100644
--- a/drivers/soc/imx/soc-imx8m.c
+++ b/drivers/soc/imx/soc-imx8m.c
@@ -247,7 +247,7 @@ static int imx8m_soc_probe(struct platform_device *pdev)
 	if (ret)
 		return ret;
 
-	data = device_get_match_data(dev);
+	data = of_machine_get_match_data(imx8_soc_match);
 	if (data) {
 		soc_dev_attr->soc_id = data->name;
 		ret = imx8m_soc_prepare(pdev, data->ocotp_compatible);
diff --git a/drivers/soc/mediatek/mt8167-mmsys.h b/drivers/soc/mediatek/mt8167-mmsys.h
index c468926561b47..eef14083c47b5 100644
--- a/drivers/soc/mediatek/mt8167-mmsys.h
+++ b/drivers/soc/mediatek/mt8167-mmsys.h
@@ -10,24 +10,29 @@
 #define MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN	0x06c
 
 #define MT8167_DITHER_MOUT_EN_RDMA0			0x1
+#define MT8167_DITHER_MOUT_EN_MASK			0x7
+
 #define MT8167_RDMA0_SOUT_DSI0				0x2
+#define MT8167_RDMA0_SOUT_MASK				0x3
+
 #define MT8167_DSI0_SEL_IN_RDMA0			0x1
+#define MT8167_DSI0_SEL_IN_MASK				0x3
 
 static const struct mtk_mmsys_routes mt8167_mmsys_routing_table[] = {
 	MMSYS_ROUTE(OVL0, COLOR0,
 		    MT8167_DISP_REG_CONFIG_DISP_OVL0_MOUT_EN, OVL0_MOUT_EN_COLOR0,
 		    OVL0_MOUT_EN_COLOR0),
 	MMSYS_ROUTE(DITHER0, RDMA0,
-		    MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN, MT8167_DITHER_MOUT_EN_RDMA0,
+		    MT8167_DISP_REG_CONFIG_DISP_DITHER_MOUT_EN, MT8167_DITHER_MOUT_EN_MASK,
 		    MT8167_DITHER_MOUT_EN_RDMA0),
 	MMSYS_ROUTE(OVL0, COLOR0,
 		    MT8167_DISP_REG_CONFIG_DISP_COLOR0_SEL_IN, COLOR0_SEL_IN_OVL0,
 		    COLOR0_SEL_IN_OVL0),
 	MMSYS_ROUTE(RDMA0, DSI0,
-		    MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN, MT8167_DSI0_SEL_IN_RDMA0,
+		    MT8167_DISP_REG_CONFIG_DISP_DSI0_SEL_IN, MT8167_DSI0_SEL_IN_MASK,
 		    MT8167_DSI0_SEL_IN_RDMA0),
 	MMSYS_ROUTE(RDMA0, DSI0,
-		    MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_DSI0,
+		    MT8167_DISP_REG_CONFIG_DISP_RDMA0_SOUT_SEL_IN, MT8167_RDMA0_SOUT_MASK,
 		    MT8167_RDMA0_SOUT_DSI0),
 };
 
diff --git a/drivers/soc/mediatek/mtk-mmsys.c b/drivers/soc/mediatek/mtk-mmsys.c
index bb4639ca0b8cd..2f3e0778bb175 100644
--- a/drivers/soc/mediatek/mtk-mmsys.c
+++ b/drivers/soc/mediatek/mtk-mmsys.c
@@ -167,9 +167,11 @@ static void mtk_mmsys_update_bits(struct mtk_mmsys *mmsys, u32 offset, u32 mask,
 	u32 tmp;
 
 	if (mmsys->cmdq_base.size && cmdq_pkt) {
-		ret = cmdq_pkt_write_mask(cmdq_pkt, mmsys->cmdq_base.subsys,
-					  mmsys->cmdq_base.offset + offset, val,
-					  mask);
+		ret = mmsys->cmdq_base.pkt_write_mask(cmdq_pkt,
+						      mmsys->cmdq_base.subsys,
+						      mmsys->cmdq_base.pa_base,
+						      mmsys->cmdq_base.offset + offset,
+						      val, mask);
 		if (ret)
 			pr_debug("CMDQ unavailable: using CPU write\n");
 		else
diff --git a/drivers/soc/mediatek/mtk-mutex.c b/drivers/soc/mediatek/mtk-mutex.c
index 38179e8cd98f8..eb5d381ff5aff 100644
--- a/drivers/soc/mediatek/mtk-mutex.c
+++ b/drivers/soc/mediatek/mtk-mutex.c
@@ -990,6 +990,7 @@ int mtk_mutex_enable_by_cmdq(struct mtk_mutex *mutex, void *pkt)
 	struct mtk_mutex_ctx *mtx = container_of(mutex, struct mtk_mutex_ctx,
 						 mutex[mutex->id]);
 	struct cmdq_pkt *cmdq_pkt = (struct cmdq_pkt *)pkt;
+	dma_addr_t en_addr = mtx->addr + DISP_REG_MUTEX_EN(mutex->id);
 
 	WARN_ON(&mtx->mutex[mutex->id] != mutex);
 
@@ -998,8 +999,8 @@ int mtk_mutex_enable_by_cmdq(struct mtk_mutex *mutex, void *pkt)
 		return -ENODEV;
 	}
 
-	cmdq_pkt_write(cmdq_pkt, mtx->cmdq_reg.subsys,
-		       mtx->addr + DISP_REG_MUTEX_EN(mutex->id), 1);
+	mtx->cmdq_reg.pkt_write(cmdq_pkt, mtx->cmdq_reg.subsys, en_addr, en_addr, 1);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(mtk_mutex_enable_by_cmdq);
diff --git a/drivers/soc/microchip/mpfs-sys-controller.c b/drivers/soc/microchip/mpfs-sys-controller.c
index 92d1142a59e68..0400a01b2338d 100644
--- a/drivers/soc/microchip/mpfs-sys-controller.c
+++ b/drivers/soc/microchip/mpfs-sys-controller.c
@@ -158,8 +158,8 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev)
 
 	of_data = (struct mpfs_syscon_config *) device_get_match_data(dev);
 	if (!of_data) {
-		dev_err(dev, "Error getting match data\n");
-		return -EINVAL;
+		ret = dev_err_probe(dev, -EINVAL, "Error getting match data\n");
+		goto out_free_channel;
 	}
 
 	for (i = 0; i < of_data->nb_subdevs; i++) {
@@ -173,6 +173,8 @@ static int mpfs_sys_controller_probe(struct platform_device *pdev)
 
 	return 0;
 
+out_free_channel:
+	mbox_free_channel(sys_controller->chan);
 out_free:
 	kfree(sys_controller);
 	return ret;
diff --git a/drivers/soc/qcom/ice.c b/drivers/soc/qcom/ice.c
index b203bc685cadd..e26d5a64763cb 100644
--- a/drivers/soc/qcom/ice.c
+++ b/drivers/soc/qcom/ice.c
@@ -16,6 +16,7 @@
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/xarray.h>
 
 #include <linux/firmware/qcom/qcom_scm.h>
 
@@ -108,11 +109,15 @@ struct qcom_ice {
 	void __iomem *base;
 
 	struct clk *core_clk;
+	struct clk *iface_clk;
 	bool use_hwkm;
 	bool hwkm_init_complete;
 	u8 hwkm_version;
 };
 
+static DEFINE_XARRAY(ice_handles);
+static DEFINE_MUTEX(ice_mutex);
+
 static bool qcom_ice_check_supported(struct qcom_ice *ice)
 {
 	u32 regval = qcom_ice_readl(ice, QCOM_ICE_REG_VERSION);
@@ -312,8 +317,13 @@ int qcom_ice_resume(struct qcom_ice *ice)
 
 	err = clk_prepare_enable(ice->core_clk);
 	if (err) {
-		dev_err(dev, "failed to enable core clock (%d)\n",
-			err);
+		dev_err(dev, "Failed to enable core clock: %d\n", err);
+		return err;
+	}
+
+	err = clk_prepare_enable(ice->iface_clk);
+	if (err) {
+		dev_err(dev, "Failed to enable iface clock: %d\n", err);
 		return err;
 	}
 	qcom_ice_hwkm_init(ice);
@@ -323,6 +333,7 @@ EXPORT_SYMBOL_GPL(qcom_ice_resume);
 
 int qcom_ice_suspend(struct qcom_ice *ice)
 {
+	clk_disable_unprepare(ice->iface_clk);
 	clk_disable_unprepare(ice->core_clk);
 	ice->hwkm_init_complete = false;
 
@@ -559,7 +570,7 @@ static struct qcom_ice *qcom_ice_create(struct device *dev,
 
 	if (!qcom_scm_ice_available()) {
 		dev_warn(dev, "ICE SCM interface not found\n");
-		return NULL;
+		return ERR_PTR(-EOPNOTSUPP);
 	}
 
 	engine = devm_kzalloc(dev, sizeof(*engine), GFP_KERNEL);
@@ -579,11 +590,17 @@ static struct qcom_ice *qcom_ice_create(struct device *dev,
 	engine->core_clk = devm_clk_get_optional_enabled(dev, "ice_core_clk");
 	if (!engine->core_clk)
 		engine->core_clk = devm_clk_get_optional_enabled(dev, "ice");
+	if (!engine->core_clk)
+		engine->core_clk = devm_clk_get_optional_enabled(dev, "core");
 	if (!engine->core_clk)
 		engine->core_clk = devm_clk_get_enabled(dev, NULL);
 	if (IS_ERR(engine->core_clk))
 		return ERR_CAST(engine->core_clk);
 
+	engine->iface_clk = devm_clk_get_optional_enabled(dev, "iface");
+	if (IS_ERR(engine->iface_clk))
+		return ERR_CAST(engine->iface_clk);
+
 	if (!qcom_ice_check_supported(engine))
 		return ERR_PTR(-EOPNOTSUPP);
 
@@ -631,6 +648,8 @@ static struct qcom_ice *of_qcom_ice_get(struct device *dev)
 		return qcom_ice_create(&pdev->dev, base);
 	}
 
+	guard(mutex)(&ice_mutex);
+
 	/*
 	 * If the consumer node does not provider an 'ice' reg range
 	 * (legacy DT binding), then it must at least provide a phandle
@@ -639,20 +658,21 @@ static struct qcom_ice *of_qcom_ice_get(struct device *dev)
 	struct device_node *node __free(device_node) = of_parse_phandle(dev->of_node,
 									"qcom,ice", 0);
 	if (!node)
-		return NULL;
+		return ERR_PTR(-ENODEV);
 
 	pdev = of_find_device_by_node(node);
 	if (!pdev) {
 		dev_err(dev, "Cannot find device node %s\n", node->name);
-		return ERR_PTR(-EPROBE_DEFER);
+		return ERR_PTR(-ENODEV);
 	}
 
-	ice = platform_get_drvdata(pdev);
-	if (!ice) {
-		dev_err(dev, "Cannot get ice instance from %s\n",
-			dev_name(&pdev->dev));
+	ice = xa_load(&ice_handles, pdev->dev.of_node->phandle);
+	if (IS_ERR_OR_NULL(ice)) {
 		platform_device_put(pdev);
-		return ERR_PTR(-EPROBE_DEFER);
+		if (!ice)
+			return ERR_PTR(-EPROBE_DEFER);
+		else
+			return ice;
 	}
 
 	link = device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER);
@@ -691,8 +711,7 @@ static void devm_of_qcom_ice_put(struct device *dev, void *res)
  * phandle via 'qcom,ice' property to an ICE DT, the ICE instance will already
  * be created and so this function will return that instead.
  *
- * Return: ICE pointer on success, NULL if there is no ICE data provided by the
- * consumer or ERR_PTR() on error.
+ * Return: ICE pointer on success, ERR_PTR() on error.
  */
 struct qcom_ice *devm_of_qcom_ice_get(struct device *dev)
 {
@@ -703,7 +722,7 @@ struct qcom_ice *devm_of_qcom_ice_get(struct device *dev)
 		return ERR_PTR(-ENOMEM);
 
 	ice = of_qcom_ice_get(dev);
-	if (!IS_ERR_OR_NULL(ice)) {
+	if (!IS_ERR(ice)) {
 		*dr = ice;
 		devres_add(dev, dr);
 	} else {
@@ -716,24 +735,40 @@ EXPORT_SYMBOL_GPL(devm_of_qcom_ice_get);
 
 static int qcom_ice_probe(struct platform_device *pdev)
 {
+	unsigned long phandle = pdev->dev.of_node->phandle;
 	struct qcom_ice *engine;
 	void __iomem *base;
 
+	guard(mutex)(&ice_mutex);
+
 	base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(base)) {
 		dev_warn(&pdev->dev, "ICE registers not found\n");
+		/* Store the error pointer for devm_of_qcom_ice_get() */
+		xa_store(&ice_handles, phandle, (__force void *)base, GFP_KERNEL);
 		return PTR_ERR(base);
 	}
 
 	engine = qcom_ice_create(&pdev->dev, base);
-	if (IS_ERR(engine))
+	if (IS_ERR(engine)) {
+		/* Store the error pointer for devm_of_qcom_ice_get() */
+		xa_store(&ice_handles, phandle, engine, GFP_KERNEL);
 		return PTR_ERR(engine);
+	}
 
-	platform_set_drvdata(pdev, engine);
+	xa_store(&ice_handles, phandle, engine, GFP_KERNEL);
 
 	return 0;
 }
 
+static void qcom_ice_remove(struct platform_device *pdev)
+{
+	unsigned long phandle = pdev->dev.of_node->phandle;
+
+	guard(mutex)(&ice_mutex);
+	xa_store(&ice_handles, phandle, NULL, GFP_KERNEL);
+}
+
 static const struct of_device_id qcom_ice_of_match_table[] = {
 	{ .compatible = "qcom,inline-crypto-engine" },
 	{ },
@@ -742,6 +777,7 @@ MODULE_DEVICE_TABLE(of, qcom_ice_of_match_table);
 
 static struct platform_driver qcom_ice_driver = {
 	.probe	= qcom_ice_probe,
+	.remove	= qcom_ice_remove,
 	.driver = {
 		.name = "qcom-ice",
 		.of_match_table = qcom_ice_of_match_table,
diff --git a/drivers/soc/qcom/llcc-qcom.c b/drivers/soc/qcom/llcc-qcom.c
index 0161ceec88421..dcc08f63e0201 100644
--- a/drivers/soc/qcom/llcc-qcom.c
+++ b/drivers/soc/qcom/llcc-qcom.c
@@ -5,7 +5,6 @@
  */
 
 #include <linux/bitfield.h>
-#include <linux/bitops.h>
 #include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/io.h>
@@ -14,6 +13,7 @@
 #include <linux/mutex.h>
 #include <linux/nvmem-consumer.h>
 #include <linux/of.h>
+#include <linux/of_reserved_mem.h>
 #include <linux/regmap.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
@@ -76,10 +76,16 @@
 #define LLCC_VERSION_4_1_0_0          0x04010000
 #define LLCC_VERSION_6_0_0_0          0X06000000
 
+#define SLC_SCT_MEM_LAYOUT_VERSION1   1 /* SCT Memory layout version */
+#define SLC_SCT_DONE                  0x00534354444f4e45 /* SCT programming OK */
+#define SLC_SCT_FAIL                  0x005343544641494c /* SCT programming failed */
+#define SLC_SCT_NAME_LEN              15
+#define SLC_SCT_SLICE_ACT_ON_BOOT     BIT(25)
+
 /**
- * struct llcc_slice_config - Data associated with the llcc slice
+ * struct llcc_slice_config - Data associated with the LLCC slice
  * @usecase_id: Unique id for the client's use case
- * @slice_id: llcc slice id for each client
+ * @slice_id: LLCC slice id for each client
  * @max_cap: The maximum capacity of the cache slice provided in KB
  * @priority: Priority of the client used to select victim line for replacement
  * @fixed_size: Boolean indicating if the slice has a fixed capacity
@@ -93,7 +99,7 @@
  *             slice: normal or TCM(Tightly Coupled Memory)
  * @probe_target_ways: Determines what ways to probe for access hit. When
  *                    configured to 1 only bonus and reserved ways are probed.
- *                    When configured to 0 all ways in llcc are probed.
+ *                    When configured to 0 all ways in LLCC are probed.
  * @dis_cap_alloc: Disable capacity based allocation for a client
  * @retain_on_pc: If this bit is set and client has maintained active vote
  *               then the ways assigned to this client are not flushed on power
@@ -143,6 +149,87 @@ struct llcc_slice_config {
 	u32 parent_slice_id;
 };
 
+/*
+ * struct slc_sct_error - Represents SCT error
+ * @code: FW code status
+ * @param: Holds the SCT programming error
+ */
+struct slc_sct_error {
+	__le64 code;
+	__le64 param;
+} __packed;
+
+/*
+ * struct slc_sct_status - SCT programming status
+ * @program_status: Indicates programming success or failure
+ * @version: SCT mem layout version
+ * @error: Error enum and its param
+ */
+struct slc_sct_status {
+	__le64 program_status;
+	/* Use the lower 8 bits */
+	__le64 version;
+	struct slc_sct_error error;
+} __packed;
+
+/*
+ * struct slc_sct_details - SCT details
+ * @revision:  revision of the SCT table
+ * @name: name of the SCT table
+ */
+struct slc_sct_details {
+	u8 revision;
+	char name[SLC_SCT_NAME_LEN];
+} __packed;
+
+/*
+ * struct tcm_mem_info - SC TCM Shared memory details
+ * @is_present: is TCM region present
+ * @offset: offset of TCM shared memory details
+ */
+struct slc_tcm_mem_info {
+	__le32 is_present;
+	__le32 offset;
+} __packed;
+
+/*
+ * struct slc_sct_slice_desc - Slice descriptor definition used in shmem
+ * @slice_id:  SCID of the slice
+ * @usecase_id: Usecase ID of the slice
+ * @slice_properties:
+ *	slice_size: Contains the slice descriptor size - 20 bit wide
+ *	rsvd: Reserved space - 4 bit wide
+ *	flags: Flags for descriptors - 3 bit wide
+ *		MPAM SCID: Bit 24
+ *		Activate on boot: Bit 25
+ *		Non-HLOS SCID: Bit 26
+ *	HWMutex: Ensures only one processor (CPU or MCU) at a time can
+ *	access the LLCC hardware resources - 5 bit wide
+ */
+struct slc_sct_slice_desc {
+	__le16 slice_id;
+	__le16 usecase_id;
+	__le32 slice_properties;
+} __packed;
+
+/*
+ * struct slc_sct_mem - Shared memory structure
+ * @sct_status: Status of SCT programming
+ * @sct_details: Sct revision and name details
+ * @tcm_mem_info: TCM shared memory presence & offset info
+ * @slice_descs_count: Number of slice desc present in SCT
+ * @scid_max: Maximum no. of SCIDs supported
+ * @slice_descs: Array of SCT slice desc
+ */
+struct slc_sct_mem {
+	struct slc_sct_status sct_status;
+	struct slc_sct_details sct_details;
+	struct slc_tcm_mem_info tcm_mem_info;
+	__le32 slice_descs_count;
+	__le32 scid_max;
+	struct slc_sct_slice_desc slice_descs[] __counted_by_le(slice_descs_count);
+} __packed;
+
 struct qcom_llcc_config {
 	const struct llcc_slice_config *sct_data;
 	const u32 *reg_offset;
@@ -4141,6 +4228,15 @@ static const u32 llcc_v6_reg_offset[] = {
 	[LLCC_TRP_WRS_CACHEABLE_EN]	= 0x00042088,
 };
 
+static const struct qcom_llcc_config hawi_sct_cfg[] = {
+	{
+		.sct_data	= NULL,
+		.size		= 0,
+		.reg_offset	= llcc_v6_reg_offset,
+		.edac_reg_offset = &llcc_v6_edac_reg_offset,
+	},
+};
+
 static const struct qcom_llcc_config kaanapali_cfg[] = {
 	{
 		.sct_data	= kaanapali_data,
@@ -4397,6 +4493,11 @@ static const struct qcom_llcc_config x1e80100_cfg[] = {
 	},
 };
 
+static const struct qcom_sct_config hawi_sct_cfgs = {
+	.llcc_config	= hawi_sct_cfg,
+	.num_config	= ARRAY_SIZE(hawi_sct_cfg),
+};
+
 static const struct qcom_sct_config kaanapali_cfgs = {
 	.llcc_config	= kaanapali_cfg,
 	.num_config	= ARRAY_SIZE(kaanapali_cfg),
@@ -4525,37 +4626,34 @@ static const struct qcom_sct_config x1e80100_cfgs = {
 static struct llcc_drv_data *drv_data = (void *) -EPROBE_DEFER;
 
 /**
- * llcc_slice_getd - get llcc slice descriptor
+ * llcc_slice_getd - get LLCC slice descriptor
  * @uid: usecase_id for the client
  *
- * A pointer to llcc slice descriptor will be returned on success
+ * A pointer to LLCC slice descriptor will be returned on success
  * and error pointer is returned on failure
  */
 struct llcc_slice_desc *llcc_slice_getd(u32 uid)
 {
-	const struct llcc_slice_config *cfg;
-	u32 sz, i;
-
 	if (IS_ERR(drv_data))
 		return ERR_CAST(drv_data);
 
-	cfg = drv_data->cfg;
-	sz = drv_data->cfg_size;
+	if (IS_ERR_OR_NULL(drv_data->desc))
+		return ERR_PTR(-ENODEV);
 
-	for (i = 0; cfg && i < sz; i++, cfg++)
-		if (cfg->usecase_id == uid)
-			break;
+	for (u32 i = 0; i < drv_data->cfg_size; i++) {
+		if (uid == drv_data->desc[i].uid)
+			return &drv_data->desc[i];
+	}
 
-	if (i == sz)
-		return ERR_PTR(-ENODEV);
+	dev_err(drv_data->dev, "Failed to get slice desc for uid: %u\n", uid);
 
-	return &drv_data->desc[i];
+	return ERR_PTR(-EINVAL);
 }
 EXPORT_SYMBOL_GPL(llcc_slice_getd);
 
 /**
- * llcc_slice_putd - llcc slice descriptor
- * @desc: Pointer to llcc slice descriptor
+ * llcc_slice_putd - LLCC slice descriptor
+ * @desc: Pointer to LLCC slice descriptor
  */
 void llcc_slice_putd(struct llcc_slice_desc *desc)
 {
@@ -4618,8 +4716,8 @@ static int llcc_update_act_ctrl(u32 sid,
 }
 
 /**
- * llcc_slice_activate - Activate the llcc slice
- * @desc: Pointer to llcc slice descriptor
+ * llcc_slice_activate - Activate the LLCC slice
+ * @desc: Pointer to LLCC slice descriptor
  *
  * A value of zero will be returned on success and a negative errno will
  * be returned in error cases
@@ -4654,8 +4752,8 @@ int llcc_slice_activate(struct llcc_slice_desc *desc)
 EXPORT_SYMBOL_GPL(llcc_slice_activate);
 
 /**
- * llcc_slice_deactivate - Deactivate the llcc slice
- * @desc: Pointer to llcc slice descriptor
+ * llcc_slice_deactivate - Deactivate the LLCC slice
+ * @desc: Pointer to LLCC slice descriptor
  *
  * A value of zero will be returned on success and a negative errno will
  * be returned in error cases
@@ -4691,7 +4789,7 @@ EXPORT_SYMBOL_GPL(llcc_slice_deactivate);
 
 /**
  * llcc_get_slice_id - return the slice id
- * @desc: Pointer to llcc slice descriptor
+ * @desc: Pointer to LLCC slice descriptor
  */
 int llcc_get_slice_id(struct llcc_slice_desc *desc)
 {
@@ -4704,7 +4802,7 @@ EXPORT_SYMBOL_GPL(llcc_get_slice_id);
 
 /**
  * llcc_get_slice_size - return the slice id
- * @desc: Pointer to llcc slice descriptor
+ * @desc: Pointer to LLCC slice descriptor
  */
 size_t llcc_get_slice_size(struct llcc_slice_desc *desc)
 {
@@ -4738,9 +4836,9 @@ static int _qcom_llcc_cfg_program(const struct llcc_slice_config *config,
 	/*
 	 * LLCC instances can vary for each target.
 	 * The SW writes to broadcast register which gets propagated
-	 * to each llcc instance (llcc0,.. llccN).
+	 * to each LLCC instance (llcc0,.. llccN).
 	 * Since the size of the memory is divided equally amongst the
-	 * llcc instances, we need to configure the max cap accordingly.
+	 * LLCC instances, we need to configure the max cap accordingly.
 	 */
 	max_cap_cacheline = max_cap_cacheline / drv_data->num_banks;
 	max_cap_cacheline >>= CACHE_LINE_SIZE_SHIFT;
@@ -5029,6 +5127,12 @@ static int qcom_llcc_cfg_program(struct platform_device *pdev,
 	sz = drv_data->cfg_size;
 	llcc_table = drv_data->cfg;
 
+	for (i = 0; i < sz; i++) {
+		drv_data->desc[i].uid = llcc_table[i].usecase_id;
+		drv_data->desc[i].slice_id = llcc_table[i].slice_id;
+		drv_data->desc[i].slice_size = llcc_table[i].max_cap;
+	}
+
 	if (drv_data->version >= LLCC_VERSION_6_0_0_0) {
 		for (i = 0; i < sz; i++) {
 			ret = _qcom_llcc_cfg_program_v6(&llcc_table[i], cfg);
@@ -5064,6 +5168,101 @@ static int qcom_llcc_get_cfg_index(struct platform_device *pdev, u8 *cfg_index,
 	return ret;
 }
 
+static int qcom_llcc_verify_fw_config(struct device *dev,
+				      const struct slc_sct_mem *slc_mem)
+{
+	u64 program_status;
+
+	program_status = le64_to_cpu(slc_mem->sct_status.program_status);
+
+	if (program_status == SLC_SCT_DONE) {
+		u32 desc_count = le32_to_cpu(slc_mem->slice_descs_count);
+		u32 scid_max = le32_to_cpu(slc_mem->scid_max);
+
+		if (desc_count > scid_max) {
+			dev_err(dev, "Descriptor count above max limit (%u > %u)\n",
+				desc_count, scid_max);
+			return -EINVAL;
+		}
+
+		u8 revision = slc_mem->sct_details.revision;
+		char name_buf[SLC_SCT_NAME_LEN];
+
+		memcpy(name_buf, slc_mem->sct_details.name,
+		       SLC_SCT_NAME_LEN - 1);
+		name_buf[SLC_SCT_NAME_LEN - 1] = '\0';
+
+		dev_dbg(dev, "SCT init: desc_count=%u, rev=%u, name=%s\n",
+			desc_count, revision, name_buf);
+
+		return 0;
+	} else if (program_status == SLC_SCT_FAIL) {
+		u8 version = (u8)(le64_to_cpu(slc_mem->sct_status.version));
+		u64 code = le64_to_cpu(slc_mem->sct_status.error.code);
+		u64 param = le64_to_cpu(slc_mem->sct_status.error.param);
+
+		if (version == SLC_SCT_MEM_LAYOUT_VERSION1) {
+			dev_err(dev, "SCT init failed: code = %llu, param = %llu, version = 0x%x\n",
+				code, param, version);
+		} else {
+			dev_err(dev, "Found unsupported version %u\n", version);
+		}
+	} else {
+		dev_err(dev, "Unknown SCT Initialization error\n");
+	}
+
+	return -EINVAL;
+}
+
+static int qcom_llcc_get_fw_config(struct platform_device *pdev)
+{
+	const struct slc_sct_mem *slc_mem = NULL;
+	const struct slc_sct_slice_desc *memslice;
+	struct device *dev = &pdev->dev;
+	u32 slice_properties;
+	struct resource res;
+	u32 i, sz;
+	int ret;
+
+	ret = of_reserved_mem_region_to_resource(dev->of_node, 0, &res);
+	if (ret) {
+		dev_err(dev, "Unable to locate DT /reserved-memory resource\n");
+		return ret;
+	}
+
+	slc_mem = devm_memremap(dev, res.start, resource_size(&res), MEMREMAP_WB);
+	if (!slc_mem) {
+		dev_err(dev, "Failed to memremap SLC shared memory\n");
+		return -ENOMEM;
+	}
+
+	ret = qcom_llcc_verify_fw_config(dev, slc_mem);
+	if (ret)
+		return ret;
+
+	sz = le32_to_cpu(slc_mem->slice_descs_count);
+
+	drv_data->desc = devm_kcalloc(dev, sz, sizeof(struct llcc_slice_desc),
+				      GFP_KERNEL);
+	if (!drv_data->desc)
+		return -ENOMEM;
+
+	for (i = 0; i < sz; i++) {
+		memslice = &slc_mem->slice_descs[i];
+		drv_data->desc[i].slice_id = le16_to_cpu(memslice->slice_id);
+		drv_data->desc[i].uid = le16_to_cpu(memslice->usecase_id);
+		slice_properties = le32_to_cpu(memslice->slice_properties);
+		/* Set refcount to 1 if FW already activated this descriptor */
+		if (FIELD_GET(SLC_SCT_SLICE_ACT_ON_BOOT, slice_properties))
+			refcount_set(&drv_data->desc[i].refcount, 1);
+	}
+
+	drv_data->cfg = NULL;
+	drv_data->cfg_size = sz;
+
+	return 0;
+}
+
 static void qcom_llcc_remove(struct platform_device *pdev)
 {
 	/* Set the global pointer to a error code to avoid referencing it */
@@ -5096,8 +5295,6 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 	struct platform_device *llcc_edac;
 	const struct qcom_sct_config *cfgs;
 	const struct qcom_llcc_config *cfg;
-	const struct llcc_slice_config *llcc_cfg;
-	u32 sz;
 	u8 cfg_index;
 	u32 version;
 	struct regmap *regmap;
@@ -5190,32 +5387,31 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 		}
 	}
 
-	llcc_cfg = cfg->sct_data;
-	sz = cfg->size;
-	drv_data->desc = devm_kcalloc(dev, sz, sizeof(struct llcc_slice_desc), GFP_KERNEL);
-	if (!drv_data->desc) {
-		ret = -ENOMEM;
-		goto err;
-	}
+	mutex_init(&drv_data->lock);
+	if (!cfg->size) {
+		ret = qcom_llcc_get_fw_config(pdev);
+		if (ret)
+			goto err;
+	} else {
+		drv_data->cfg = cfg->sct_data;
+		drv_data->cfg_size = cfg->size;
+		drv_data->desc = devm_kcalloc(dev, cfg->size,
+					      sizeof(struct llcc_slice_desc), GFP_KERNEL);
 
-	for (i = 0; i < sz; i++) {
-		drv_data->desc[i].slice_id = llcc_cfg[i].slice_id;
-		drv_data->desc[i].slice_size = llcc_cfg[i].max_cap;
-		refcount_set(&drv_data->desc[i].refcount, 0);
+		if (!drv_data->desc) {
+			ret = -ENOMEM;
+			goto err;
+		}
+
+		ret = qcom_llcc_cfg_program(pdev, cfg);
+		if (ret)
+			goto err;
 	}
 
-	drv_data->cfg = llcc_cfg;
-	drv_data->cfg_size = sz;
+	drv_data->ecc_irq = platform_get_irq_optional(pdev, 0);
 	drv_data->edac_reg_offset = cfg->edac_reg_offset;
 	drv_data->ecc_irq_configured = cfg->irq_configured;
-	mutex_init(&drv_data->lock);
-	platform_set_drvdata(pdev, drv_data);
-
-	ret = qcom_llcc_cfg_program(pdev, cfg);
-	if (ret)
-		goto err;
-
-	drv_data->ecc_irq = platform_get_irq_optional(pdev, 0);
+	drv_data->dev = dev;
 
 	/*
 	 * On some platforms, the access to EDAC registers will be locked by
@@ -5228,9 +5424,11 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 						"qcom_llcc_edac", -1, drv_data,
 						sizeof(*drv_data));
 		if (IS_ERR(llcc_edac))
-			dev_err(dev, "Failed to register llcc edac driver\n");
+			dev_err(dev, "Failed to register LLCC EDAC driver\n");
 	}
 
+	platform_set_drvdata(pdev, drv_data);
+
 	return 0;
 err:
 	drv_data = ERR_PTR(-ENODEV);
@@ -5239,6 +5437,7 @@ static int qcom_llcc_probe(struct platform_device *pdev)
 
 static const struct of_device_id qcom_llcc_of_match[] = {
 	{ .compatible = "qcom,glymur-llcc", .data = &glymur_cfgs },
+	{ .compatible = "qcom,hawi-llcc", .data = &hawi_sct_cfgs },
 	{ .compatible = "qcom,ipq5424-llcc", .data = &ipq5424_cfgs},
 	{ .compatible = "qcom,kaanapali-llcc", .data = &kaanapali_cfgs},
 	{ .compatible = "qcom,qcs615-llcc", .data = &qcs615_cfgs},
diff --git a/drivers/soc/qcom/qcom_pd_mapper.c b/drivers/soc/qcom/qcom_pd_mapper.c
index 7bb14c20ab5d6..b99718e25f2fc 100644
--- a/drivers/soc/qcom/qcom_pd_mapper.c
+++ b/drivers/soc/qcom/qcom_pd_mapper.c
@@ -266,6 +266,12 @@ static const struct qcom_pdm_domain_data adsp_charger_pd = {
 	.services = { NULL },
 };
 
+static const struct qcom_pdm_domain_data adsp_ois_pd = {
+	.domain = "msm/adsp/ois_pd",
+	.instance_id = 74,
+	.services = { NULL, },
+};
+
 static const struct qcom_pdm_domain_data adsp_root_pd = {
 	.domain = "msm/adsp/root_pd",
 	.instance_id = 74,
@@ -370,6 +376,7 @@ static const struct qcom_pdm_domain_data *glymur_domains[] = {
 
 static const struct qcom_pdm_domain_data *kaanapali_domains[] = {
 	&adsp_audio_pd,
+	&adsp_ois_pd,
 	&adsp_root_pd,
 	&adsp_sensor_pd,
 	&cdsp_root_pd,
@@ -581,6 +588,7 @@ static const struct of_device_id qcom_pdm_domains[] __maybe_unused = {
 	{ .compatible = "qcom,eliza", .data = sm8550_domains, },
 	{ .compatible = "qcom,apq8096", .data = msm8996_domains, },
 	{ .compatible = "qcom,glymur", .data = glymur_domains, },
+	{ .compatible = "qcom,hawi", .data = kaanapali_domains, },
 	{ .compatible = "qcom,kaanapali", .data = kaanapali_domains, },
 	{ .compatible = "qcom,mahua", .data = glymur_domains, },
 	{ .compatible = "qcom,milos", .data = sm8550_domains, },
diff --git a/drivers/soc/qcom/smem.c b/drivers/soc/qcom/smem.c
index d5c94b47f431f..afb21a778fe7b 100644
--- a/drivers/soc/qcom/smem.c
+++ b/drivers/soc/qcom/smem.c
@@ -85,9 +85,6 @@
 /* Processor/host identifier for the global partition */
 #define SMEM_GLOBAL_HOST	0xfffe
 
-/* Max number of processors/hosts in a system */
-#define SMEM_HOST_COUNT		25
-
 /**
   * struct smem_proc_comm - proc_comm communication struct (legacy)
   * @command:	current command to be executed
@@ -282,7 +279,7 @@ struct qcom_smem {
 	struct platform_device *socinfo;
 	struct smem_ptable *ptable;
 	struct smem_partition global_partition;
-	struct smem_partition partitions[SMEM_HOST_COUNT];
+	struct xarray partitions;
 
 	unsigned num_regions;
 	struct smem_region regions[] __counted_by(num_regions);
@@ -382,7 +379,7 @@ static struct qcom_smem *__smem = INIT_ERR_PTR(-EPROBE_DEFER);
 int qcom_smem_bust_hwspin_lock_by_host(unsigned int host)
 {
 	/* This function is for remote procs, so ignore SMEM_HOST_APPS */
-	if (host == SMEM_HOST_APPS || host >= SMEM_HOST_COUNT)
+	if (host == SMEM_HOST_APPS || !xa_load(&__smem->partitions, host))
 		return -EINVAL;
 
 	return hwspin_lock_bust(__smem->hwlock, SMEM_HOST_ID_TO_HWSPINLOCK_ID(host));
@@ -530,8 +527,8 @@ int qcom_smem_alloc(unsigned host, unsigned item, size_t size)
 	if (ret)
 		return ret;
 
-	if (host < SMEM_HOST_COUNT && __smem->partitions[host].virt_base) {
-		part = &__smem->partitions[host];
+	part = xa_load(&__smem->partitions, host);
+	if (part) {
 		ret = qcom_smem_alloc_private(__smem, part, item, size);
 	} else if (__smem->global_partition.virt_base) {
 		part = &__smem->global_partition;
@@ -697,8 +694,8 @@ void *qcom_smem_get(unsigned host, unsigned item, size_t *size)
 	if (item >= __smem->item_count)
 		return ERR_PTR(-EINVAL);
 
-	if (host < SMEM_HOST_COUNT && __smem->partitions[host].virt_base) {
-		part = &__smem->partitions[host];
+	part = xa_load(&__smem->partitions, host);
+	if (part) {
 		ptr = qcom_smem_get_private(__smem, part, item, size);
 	} else if (__smem->global_partition.virt_base) {
 		part = &__smem->global_partition;
@@ -730,8 +727,8 @@ int qcom_smem_get_free_space(unsigned host)
 	if (IS_ERR(__smem))
 		return PTR_ERR(__smem);
 
-	if (host < SMEM_HOST_COUNT && __smem->partitions[host].virt_base) {
-		part = &__smem->partitions[host];
+	part = xa_load(&__smem->partitions, host);
+	if (part) {
 		phdr = part->virt_base;
 		ret = le32_to_cpu(phdr->offset_free_cached) -
 		      le32_to_cpu(phdr->offset_free_uncached);
@@ -774,12 +771,11 @@ phys_addr_t qcom_smem_virt_to_phys(void *p)
 {
 	struct smem_partition *part;
 	struct smem_region *area;
+	unsigned long index;
 	u64 offset;
 	u32 i;
 
-	for (i = 0; i < SMEM_HOST_COUNT; i++) {
-		part = &__smem->partitions[i];
-
+	xa_for_each(&__smem->partitions, index, part) {
 		if (addr_in_range(part->virt_base, part->size, p)) {
 			offset = p - part->virt_base;
 
@@ -1016,16 +1012,20 @@ static int
 qcom_smem_enumerate_partitions(struct qcom_smem *smem, u16 local_host)
 {
 	struct smem_partition_header *header;
+	struct smem_partition *part;
 	struct smem_ptable_entry *entry;
 	struct smem_ptable *ptable;
 	u16 remote_host;
 	u16 host0, host1;
+	int ret;
 	int i;
 
 	ptable = qcom_smem_get_ptable(smem);
 	if (IS_ERR(ptable))
 		return PTR_ERR(ptable);
 
+	xa_init(&smem->partitions);
+
 	for (i = 0; i < le32_to_cpu(ptable->num_entries); i++) {
 		entry = &ptable->entry[i];
 		if (!le32_to_cpu(entry->offset))
@@ -1042,12 +1042,7 @@ qcom_smem_enumerate_partitions(struct qcom_smem *smem, u16 local_host)
 		else
 			continue;
 
-		if (remote_host >= SMEM_HOST_COUNT) {
-			dev_err(smem->dev, "bad host %u\n", remote_host);
-			return -EINVAL;
-		}
-
-		if (smem->partitions[remote_host].virt_base) {
+		if (xa_load(&smem->partitions, remote_host)) {
 			dev_err(smem->dev, "duplicate host %u\n", remote_host);
 			return -EINVAL;
 		}
@@ -1056,11 +1051,20 @@ qcom_smem_enumerate_partitions(struct qcom_smem *smem, u16 local_host)
 		if (!header)
 			return -EINVAL;
 
-		smem->partitions[remote_host].virt_base = (void __iomem *)header;
-		smem->partitions[remote_host].phys_base = smem->regions[0].aux_base +
-										le32_to_cpu(entry->offset);
-		smem->partitions[remote_host].size = le32_to_cpu(entry->size);
-		smem->partitions[remote_host].cacheline = le32_to_cpu(entry->cacheline);
+		part = devm_kzalloc(smem->dev, sizeof(struct smem_partition), GFP_KERNEL);
+		if (!part)
+			return -ENOMEM;
+
+		part->virt_base = (void __iomem *)header;
+		part->phys_base = smem->regions[0].aux_base + le32_to_cpu(entry->offset);
+		part->size = le32_to_cpu(entry->size);
+		part->cacheline = le32_to_cpu(entry->cacheline);
+
+		ret = xa_insert(&smem->partitions, remote_host, part, GFP_KERNEL);
+		if (ret) {
+			dev_err(smem->dev, "fail to insert host %u\n", remote_host);
+			return ret;
+		}
 	}
 
 	return 0;
@@ -1229,7 +1233,6 @@ static int qcom_smem_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	BUILD_BUG_ON(SMEM_HOST_APPS >= SMEM_HOST_COUNT);
 	ret = qcom_smem_enumerate_partitions(smem, SMEM_HOST_APPS);
 	if (ret < 0 && ret != -ENOENT)
 		return ret;
@@ -1249,6 +1252,7 @@ static void qcom_smem_remove(struct platform_device *pdev)
 {
 	platform_device_unregister(__smem->socinfo);
 
+	xa_destroy(&__smem->partitions);
 	/* Set to -EPROBE_DEFER to signal unprobed state */
 	__smem = ERR_PTR(-EPROBE_DEFER);
 }
diff --git a/drivers/soc/qcom/socinfo.c b/drivers/soc/qcom/socinfo.c
index 8ffd903ebddbb..af418adad7aaf 100644
--- a/drivers/soc/qcom/socinfo.c
+++ b/drivers/soc/qcom/socinfo.c
@@ -188,7 +188,19 @@ static const char *const pmic_models[] = {
 	[80] = "PM7550",
 	[82] = "PMC8380",
 	[83] = "SMB2360",
+	[86] = "PM8750B",
+	[87] = "PMD8028",
+	[88] = "PMC1020H",
+	[89] = "PMIV0104",
+	[90] = "PMIV0102",
 	[91] = "PMIV0108",
+	[92] = "PMK8850",
+	[93] = "PMH0101",
+	[94] = "PMAU0102",
+	[95] = "SMB2370",
+	[96] = "PMH0104",
+	[97] = "PMH0110",
+	[98] = "PMCX0102",
 };
 
 struct socinfo_params {
@@ -519,6 +531,7 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(IPQ5424) },
 	{ qcom_board_id(QCM6690) },
 	{ qcom_board_id(QCS6690) },
+	{ qcom_board_id(SM7750) },
 	{ qcom_board_id(SM8850) },
 	{ qcom_board_id(IPQ5404) },
 	{ qcom_board_id(QCS9100) },
@@ -526,13 +539,23 @@ static const struct soc_id soc_id[] = {
 	{ qcom_board_id(QCS8275) },
 	{ qcom_board_id(QCS9075) },
 	{ qcom_board_id(QCS615) },
+	{ qcom_board_id(SA8797P) },
 	{ qcom_board_id(CQ7790M) },
 	{ qcom_board_id(CQ7790S) },
+	{ qcom_board_id(CQ2390M) },
+	{ qcom_board_id(CQ2390S) },
+	{ qcom_board_id(IQ2390S) },
 	{ qcom_board_id(IPQ5200) },
 	{ qcom_board_id(IPQ5210) },
 	{ qcom_board_id(QCF2200) },
 	{ qcom_board_id(QCF3200) },
 	{ qcom_board_id(QCF3210) },
+	{ qcom_board_id(IPQ9620) },
+	{ qcom_board_id(IPQ9650) },
+	{ qcom_board_id(IPQ9610) },
+	{ qcom_board_id(IPQ9630) },
+	{ qcom_board_id(IPQ9640) },
+	{ qcom_board_id(IPQ9670) },
 };
 
 static const char *socinfo_machine(struct device *dev, unsigned int id)
diff --git a/drivers/soc/qcom/wcnss_ctrl.c b/drivers/soc/qcom/wcnss_ctrl.c
index ffb31a049d4a9..942e11feba65d 100644
--- a/drivers/soc/qcom/wcnss_ctrl.c
+++ b/drivers/soc/qcom/wcnss_ctrl.c
@@ -221,8 +221,10 @@ static int wcnss_download_nv(struct wcnss_ctrl *wcnss, bool *expect_cbc)
 	left = fw->size;
 
 	req = kzalloc_flex(*req, fragment, NV_FRAGMENT_SIZE);
-	if (!req)
-		return -ENOMEM;
+	if (!req) {
+		ret = -ENOMEM;
+		goto release_fw;
+	}
 
 	req->frag_size = NV_FRAGMENT_SIZE;
 	req->hdr.type = WCNSS_DOWNLOAD_NV_REQ;
@@ -243,7 +245,7 @@ static int wcnss_download_nv(struct wcnss_ctrl *wcnss, bool *expect_cbc)
 		ret = rpmsg_send(wcnss->channel, req, req->hdr.len);
 		if (ret < 0) {
 			dev_err(dev, "failed to send smd packet\n");
-			goto release_fw;
+			goto release_req;
 		}
 
 		/* Increment for next fragment */
@@ -262,9 +264,10 @@ static int wcnss_download_nv(struct wcnss_ctrl *wcnss, bool *expect_cbc)
 		ret = 0;
 	}
 
+release_req:
+	kfree(req);
 release_fw:
 	release_firmware(fw);
-	kfree(req);
 
 	return ret;
 }
diff --git a/drivers/soc/renesas/Kconfig b/drivers/soc/renesas/Kconfig
index 26bed0fdceb00..2ab150d04bb1f 100644
--- a/drivers/soc/renesas/Kconfig
+++ b/drivers/soc/renesas/Kconfig
@@ -465,6 +465,15 @@ config ARCH_R9A07G043
 
 endif # RISCV
 
+config RCAR_MFIS
+	tristate "Renesas R-Car MFIS driver"
+	depends on ARCH_RENESAS || COMPILE_TEST
+	depends on MAILBOX
+	help
+	  Select this option to enable the Renesas R-Car MFIS core driver for
+	  the MFIS device found on SoCs like R-Car. On families like Gen5, this
+	  is needed to communicate with the SCP.
+
 config PWC_RZV2M
 	bool "Renesas RZ/V2M PWC support" if COMPILE_TEST
 
diff --git a/drivers/soc/renesas/Makefile b/drivers/soc/renesas/Makefile
index 655dbcb087472..81bde85c2178a 100644
--- a/drivers/soc/renesas/Makefile
+++ b/drivers/soc/renesas/Makefile
@@ -14,6 +14,7 @@ obj-$(CONFIG_SYS_R9A09G057)	+= r9a09g057-sys.o
 
 # Family
 obj-$(CONFIG_PWC_RZV2M)		+= pwc-rzv2m.o
+obj-$(CONFIG_RCAR_MFIS)		+= rcar-mfis.o
 obj-$(CONFIG_RST_RCAR)		+= rcar-rst.o
 obj-$(CONFIG_RZN1_IRQMUX)	+= rzn1_irqmux.o
 obj-$(CONFIG_SYSC_RZ)		+= rz-sysc.o
diff --git a/drivers/soc/renesas/r9a08g045-sysc.c b/drivers/soc/renesas/r9a08g045-sysc.c
index 03d653d5cde55..63e4aa6a7cd01 100644
--- a/drivers/soc/renesas/r9a08g045-sysc.c
+++ b/drivers/soc/renesas/r9a08g045-sysc.c
@@ -37,18 +37,14 @@ static const struct rz_sysc_soc_id_init_data rzg3s_sysc_soc_id_init_data __initc
 	.specific_id_mask = GENMASK(27, 0),
 };
 
-static bool rzg3s_regmap_readable_reg(struct device *dev, unsigned int reg)
+static bool rzg3s_regmap_readable_writeable_reg(unsigned int reg)
 {
 	switch (reg) {
 	case SYS_XSPI_MAP_STAADD_CS0:
 	case SYS_XSPI_MAP_ENDADD_CS0:
 	case SYS_XSPI_MAP_STAADD_CS1:
 	case SYS_XSPI_MAP_ENDADD_CS1:
-	case SYS_GETH0_CFG:
-	case SYS_GETH1_CFG:
 	case SYS_PCIE_CFG:
-	case SYS_PCIE_MON:
-	case SYS_PCIE_ERR_MON:
 	case SYS_PCIE_PHY:
 	case SYS_I2C0_CFG:
 	case SYS_I2C1_CFG:
@@ -63,28 +59,27 @@ static bool rzg3s_regmap_readable_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-static bool rzg3s_regmap_writeable_reg(struct device *dev, unsigned int reg)
+static bool rzg3s_regmap_readable_reg(struct device *dev, unsigned int reg)
 {
+	if (rzg3s_regmap_readable_writeable_reg(reg))
+		return true;
+
 	switch (reg) {
-	case SYS_XSPI_MAP_STAADD_CS0:
-	case SYS_XSPI_MAP_ENDADD_CS0:
-	case SYS_XSPI_MAP_STAADD_CS1:
-	case SYS_XSPI_MAP_ENDADD_CS1:
-	case SYS_PCIE_CFG:
-	case SYS_PCIE_PHY:
-	case SYS_I2C0_CFG:
-	case SYS_I2C1_CFG:
-	case SYS_I2C2_CFG:
-	case SYS_I2C3_CFG:
-	case SYS_I3C_CFG:
-	case SYS_USB_PWRRDY:
-	case SYS_PCIE_RST_RSM_B:
+	case SYS_GETH0_CFG:
+	case SYS_GETH1_CFG:
+	case SYS_PCIE_MON:
+	case SYS_PCIE_ERR_MON:
 		return true;
 	default:
 		return false;
 	}
 }
 
+static bool rzg3s_regmap_writeable_reg(struct device *dev, unsigned int reg)
+{
+	return rzg3s_regmap_readable_writeable_reg(reg);
+}
+
 const struct rz_sysc_init_data rzg3s_sysc_init_data __initconst = {
 	.soc_id_init_data = &rzg3s_sysc_soc_id_init_data,
 	.readable_reg = rzg3s_regmap_readable_reg,
diff --git a/drivers/soc/renesas/r9a08g046-sysc.c b/drivers/soc/renesas/r9a08g046-sysc.c
index fd98df196d0a5..90db9d3835395 100644
--- a/drivers/soc/renesas/r9a08g046-sysc.c
+++ b/drivers/soc/renesas/r9a08g046-sysc.c
@@ -28,17 +28,14 @@
 #define SYS_PWRRDY_N			0xd70
 #define SYS_IPCONT_SEL_CLONECH		0xe2c
 
-static bool rzg3l_regmap_readable_reg(struct device *dev, unsigned int reg)
+static bool rzg3l_regmap_readable_writeable_reg(unsigned int reg)
 {
 	switch (reg) {
 	case SYS_XSPI_MAP_STAADD_CS0:
 	case SYS_XSPI_MAP_ENDADD_CS0:
 	case SYS_XSPI_MAP_STAADD_CS1:
 	case SYS_XSPI_MAP_ENDADD_CS1:
-	case SYS_GETH0_CFG:
-	case SYS_GETH1_CFG:
 	case SYS_PCIE_CFG:
-	case SYS_PCIE_MON:
 	case SYS_PCIE_PHY:
 	case SYS_I2C0_CFG:
 	case SYS_I2C1_CFG:
@@ -53,28 +50,26 @@ static bool rzg3l_regmap_readable_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-static bool rzg3l_regmap_writeable_reg(struct device *dev, unsigned int reg)
+static bool rzg3l_regmap_readable_reg(struct device *dev, unsigned int reg)
 {
+	if (rzg3l_regmap_readable_writeable_reg(reg))
+		return true;
+
 	switch (reg) {
-	case SYS_XSPI_MAP_STAADD_CS0:
-	case SYS_XSPI_MAP_ENDADD_CS0:
-	case SYS_XSPI_MAP_STAADD_CS1:
-	case SYS_XSPI_MAP_ENDADD_CS1:
-	case SYS_PCIE_CFG:
-	case SYS_PCIE_PHY:
-	case SYS_I2C0_CFG:
-	case SYS_I2C1_CFG:
-	case SYS_I2C2_CFG:
-	case SYS_I2C3_CFG:
-	case SYS_I3C_CFG:
-	case SYS_PWRRDY_N:
-	case SYS_IPCONT_SEL_CLONECH:
+	case SYS_GETH0_CFG:
+	case SYS_GETH1_CFG:
+	case SYS_PCIE_MON:
 		return true;
 	default:
 		return false;
 	}
 }
 
+static bool rzg3l_regmap_writeable_reg(struct device *dev, unsigned int reg)
+{
+	return rzg3l_regmap_readable_writeable_reg(reg);
+}
+
 static const struct rz_sysc_soc_id_init_data rzg3l_sysc_soc_id_init_data __initconst = {
 	.family = "RZ/G3L",
 	.id = 0x87d9447,
diff --git a/drivers/soc/renesas/r9a09g047-sys.c b/drivers/soc/renesas/r9a09g047-sys.c
index ea3ca10fcc33f..b617fb0bde7b8 100644
--- a/drivers/soc/renesas/r9a09g047-sys.c
+++ b/drivers/soc/renesas/r9a09g047-sys.c
@@ -83,11 +83,9 @@ static const struct rz_sysc_soc_id_init_data rzg3e_sys_soc_id_init_data __initco
 	.print_id = rzg3e_sys_print_id,
 };
 
-static bool rzg3e_regmap_readable_reg(struct device *dev, unsigned int reg)
+static bool rzg3e_regmap_readable_writeable_reg(unsigned int reg)
 {
 	switch (reg) {
-	case SYS_LSI_OTPTSU1TRMVAL0:
-	case SYS_LSI_OTPTSU1TRMVAL1:
 	case SYS_SPI_STAADDCS0:
 	case SYS_SPI_ENDADDCS0:
 	case SYS_SPI_STAADDCS1:
@@ -112,33 +110,25 @@ static bool rzg3e_regmap_readable_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-static bool rzg3e_regmap_writeable_reg(struct device *dev, unsigned int reg)
+static bool rzg3e_regmap_readable_reg(struct device *dev, unsigned int reg)
 {
+	if (rzg3e_regmap_readable_writeable_reg(reg))
+		return true;
+
 	switch (reg) {
-	case SYS_SPI_STAADDCS0:
-	case SYS_SPI_ENDADDCS0:
-	case SYS_SPI_STAADDCS1:
-	case SYS_SPI_ENDADDCS1:
-	case SYS_VSP_CLK:
-	case SYS_GBETH0_CFG:
-	case SYS_GBETH1_CFG:
-	case SYS_PCIE_INTX_CH0:
-	case SYS_PCIE_MSI1_CH0:
-	case SYS_PCIE_MSI2_CH0:
-	case SYS_PCIE_MSI3_CH0:
-	case SYS_PCIE_MSI4_CH0:
-	case SYS_PCIE_MSI5_CH0:
-	case SYS_PCIE_PME_CH0:
-	case SYS_PCIE_ACK_CH0:
-	case SYS_PCIE_MISC_CH0:
-	case SYS_PCIE_MODE_CH0:
-	case SYS_ADC_CFG:
+	case SYS_LSI_OTPTSU1TRMVAL0:
+	case SYS_LSI_OTPTSU1TRMVAL1:
 		return true;
 	default:
 		return false;
 	}
 }
 
+static bool rzg3e_regmap_writeable_reg(struct device *dev, unsigned int reg)
+{
+	return rzg3e_regmap_readable_writeable_reg(reg);
+}
+
 const struct rz_sysc_init_data rzg3e_sys_init_data __initconst = {
 	.soc_id_init_data = &rzg3e_sys_soc_id_init_data,
 	.readable_reg = rzg3e_regmap_readable_reg,
diff --git a/drivers/soc/renesas/r9a09g056-sys.c b/drivers/soc/renesas/r9a09g056-sys.c
index 2a8ebc2099610..e6a66688423ff 100644
--- a/drivers/soc/renesas/r9a09g056-sys.c
+++ b/drivers/soc/renesas/r9a09g056-sys.c
@@ -88,13 +88,9 @@ static const struct rz_sysc_soc_id_init_data rzv2n_sys_soc_id_init_data __initco
 	.print_id = rzv2n_sys_print_id,
 };
 
-static bool rzv2n_regmap_readable_reg(struct device *dev, unsigned int reg)
+static bool rzv2n_regmap_readable_writeable_reg(unsigned int reg)
 {
 	switch (reg) {
-	case SYS_LSI_OTPTSU0TRMVAL0:
-	case SYS_LSI_OTPTSU0TRMVAL1:
-	case SYS_LSI_OTPTSU1TRMVAL0:
-	case SYS_LSI_OTPTSU1TRMVAL1:
 	case SYS_GBETH0_CFG:
 	case SYS_GBETH1_CFG:
 	case SYS_PCIE_INTX_CH0:
@@ -114,28 +110,27 @@ static bool rzv2n_regmap_readable_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-static bool rzv2n_regmap_writeable_reg(struct device *dev, unsigned int reg)
+static bool rzv2n_regmap_readable_reg(struct device *dev, unsigned int reg)
 {
+	if (rzv2n_regmap_readable_writeable_reg(reg))
+		return true;
+
 	switch (reg) {
-	case SYS_GBETH0_CFG:
-	case SYS_GBETH1_CFG:
-	case SYS_PCIE_INTX_CH0:
-	case SYS_PCIE_MSI1_CH0:
-	case SYS_PCIE_MSI2_CH0:
-	case SYS_PCIE_MSI3_CH0:
-	case SYS_PCIE_MSI4_CH0:
-	case SYS_PCIE_MSI5_CH0:
-	case SYS_PCIE_PME_CH0:
-	case SYS_PCIE_ACK_CH0:
-	case SYS_PCIE_MISC_CH0:
-	case SYS_PCIE_MODE_CH0:
-	case SYS_ADC_CFG:
+	case SYS_LSI_OTPTSU0TRMVAL0:
+	case SYS_LSI_OTPTSU0TRMVAL1:
+	case SYS_LSI_OTPTSU1TRMVAL0:
+	case SYS_LSI_OTPTSU1TRMVAL1:
 		return true;
 	default:
 		return false;
 	}
 }
 
+static bool rzv2n_regmap_writeable_reg(struct device *dev, unsigned int reg)
+{
+	return rzv2n_regmap_readable_writeable_reg(reg);
+}
+
 const struct rz_sysc_init_data rzv2n_sys_init_data __initconst = {
 	.soc_id_init_data = &rzv2n_sys_soc_id_init_data,
 	.readable_reg = rzv2n_regmap_readable_reg,
diff --git a/drivers/soc/renesas/r9a09g057-sys.c b/drivers/soc/renesas/r9a09g057-sys.c
index f3e054206acbd..308492c31acbb 100644
--- a/drivers/soc/renesas/r9a09g057-sys.c
+++ b/drivers/soc/renesas/r9a09g057-sys.c
@@ -91,13 +91,9 @@ static const struct rz_sysc_soc_id_init_data rzv2h_sys_soc_id_init_data __initco
 	.print_id = rzv2h_sys_print_id,
 };
 
-static bool rzv2h_regmap_readable_reg(struct device *dev, unsigned int reg)
+static bool rzv2h_regmap_readable_writeable_reg(unsigned int reg)
 {
 	switch (reg) {
-	case SYS_LSI_OTPTSU0TRMVAL0:
-	case SYS_LSI_OTPTSU0TRMVAL1:
-	case SYS_LSI_OTPTSU1TRMVAL0:
-	case SYS_LSI_OTPTSU1TRMVAL1:
 	case SYS_GBETH0_CFG:
 	case SYS_GBETH1_CFG:
 	case SYS_PCIE_INTX_CH0:
@@ -128,39 +124,27 @@ static bool rzv2h_regmap_readable_reg(struct device *dev, unsigned int reg)
 	}
 }
 
-static bool rzv2h_regmap_writeable_reg(struct device *dev, unsigned int reg)
+static bool rzv2h_regmap_readable_reg(struct device *dev, unsigned int reg)
 {
+	if (rzv2h_regmap_readable_writeable_reg(reg))
+		return true;
+
 	switch (reg) {
-	case SYS_GBETH0_CFG:
-	case SYS_GBETH1_CFG:
-	case SYS_PCIE_INTX_CH0:
-	case SYS_PCIE_MSI1_CH0:
-	case SYS_PCIE_MSI2_CH0:
-	case SYS_PCIE_MSI3_CH0:
-	case SYS_PCIE_MSI4_CH0:
-	case SYS_PCIE_MSI5_CH0:
-	case SYS_PCIE_PME_CH0:
-	case SYS_PCIE_ACK_CH0:
-	case SYS_PCIE_MISC_CH0:
-	case SYS_PCIE_MODE_CH0:
-	case SYS_PCIE_INTX_CH1:
-	case SYS_PCIE_MSI1_CH1:
-	case SYS_PCIE_MSI2_CH1:
-	case SYS_PCIE_MSI3_CH1:
-	case SYS_PCIE_MSI4_CH1:
-	case SYS_PCIE_MSI5_CH1:
-	case SYS_PCIE_PME_CH1:
-	case SYS_PCIE_ACK_CH1:
-	case SYS_PCIE_MISC_CH1:
-	case SYS_PCIE_MODE_CH1:
-	case SYS_PCIE_MODE:
-	case SYS_ADC_CFG:
+	case SYS_LSI_OTPTSU0TRMVAL0:
+	case SYS_LSI_OTPTSU0TRMVAL1:
+	case SYS_LSI_OTPTSU1TRMVAL0:
+	case SYS_LSI_OTPTSU1TRMVAL1:
 		return true;
 	default:
 		return false;
 	}
 }
 
+static bool rzv2h_regmap_writeable_reg(struct device *dev, unsigned int reg)
+{
+	return rzv2h_regmap_readable_writeable_reg(reg);
+}
+
 const struct rz_sysc_init_data rzv2h_sys_init_data __initconst = {
 	.soc_id_init_data = &rzv2h_sys_soc_id_init_data,
 	.readable_reg = rzv2h_regmap_readable_reg,
diff --git a/drivers/soc/renesas/rcar-mfis.c b/drivers/soc/renesas/rcar-mfis.c
new file mode 100644
index 0000000000000..b334b35306ae1
--- /dev/null
+++ b/drivers/soc/renesas/rcar-mfis.c
@@ -0,0 +1,344 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Renesas R-Car MFIS (Multifunctional Interface) driver
+ *
+ * Copyright (C) Renesas Solutions Corp.
+ * Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>
+ * Wolfram Sang <wsa+renesas@sang-engineering.com>
+ */
+#include <dt-bindings/soc/renesas,r8a78000-mfis.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mailbox_controller.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/spinlock.h>
+
+#define MFISWPCNTR	0x0900
+#define MFISWACNTR	0x0904
+
+#define MFIS_X5H_IICR(i) ((i) * 0x1000 + 0x00)
+#define MFIS_X5H_EICR(i) ((i) * 0x1000 + 0x04)
+
+#define MFIS_UNPROTECT_KEY 0xACCE0000
+
+struct mfis_priv;
+
+struct mfis_reg {
+	void __iomem *base;
+	resource_size_t start;
+	struct mfis_priv *priv;
+};
+
+struct mfis_info {
+	u32 unprotect_mask;
+	unsigned int mb_num_channels;
+	unsigned int mb_reg_comes_from_dt:1;
+	unsigned int mb_tx_uses_eicr:1;
+	unsigned int mb_channels_are_unidir:1;
+};
+
+struct mfis_chan_priv {
+	u32 reg;
+	int irq;
+};
+
+struct mfis_priv {
+	spinlock_t unprotect_lock; /* guards access to the unprotection reg */
+	struct device *dev;
+	struct mfis_reg common_reg;
+	struct mfis_reg mbox_reg;
+	const struct mfis_info *info;
+
+	/* mailbox private data */
+	struct mbox_controller mbox;
+	struct mfis_chan_priv *chan_privs;
+};
+
+static u32 mfis_read(struct mfis_reg *mreg, unsigned int reg)
+{
+	return ioread32(mreg->base + reg);
+}
+
+static void mfis_write(struct mfis_reg *mreg, u32 reg, u32 val)
+{
+	struct mfis_priv *priv = mreg->priv;
+	u32 unprotect_mask = priv->info->unprotect_mask;
+	unsigned long flags;
+	u32 unprotect_code;
+
+	/*
+	 * [Gen4] key: 0xACCE0000, mask: 0x0000FFFF
+	 * [Gen5] key: 0xACC00000, mask: 0x000FFFFF
+	 */
+	unprotect_code = (MFIS_UNPROTECT_KEY & ~unprotect_mask) |
+			 ((mreg->start + reg) & unprotect_mask);
+
+	spin_lock_irqsave(&priv->unprotect_lock, flags);
+	iowrite32(unprotect_code, priv->common_reg.base + MFISWACNTR);
+	iowrite32(val, mreg->base + reg);
+	spin_unlock_irqrestore(&priv->unprotect_lock, flags);
+}
+
+/********************************************************
+ *			Mailbox				*
+ ********************************************************/
+
+#define mfis_mb_mbox_to_priv(_m) container_of((_m), struct mfis_priv, mbox)
+
+static irqreturn_t mfis_mb_iicr_interrupt(int irq, void *data)
+{
+	struct mbox_chan *chan = data;
+	struct mfis_priv *priv = mfis_mb_mbox_to_priv(chan->mbox);
+	struct mfis_chan_priv *chan_priv = chan->con_priv;
+
+	mbox_chan_received_data(chan, NULL);
+	/* Stop remote(!) doorbell */
+	mfis_write(&priv->mbox_reg, chan_priv->reg, 0);
+
+	return IRQ_HANDLED;
+}
+
+static int mfis_mb_startup(struct mbox_chan *chan)
+{
+	struct mfis_chan_priv *chan_priv = chan->con_priv;
+
+	if (!chan_priv->irq)
+		return 0;
+
+	return request_irq(chan_priv->irq, mfis_mb_iicr_interrupt, 0,
+			   dev_name(chan->mbox->dev), chan);
+}
+
+static void mfis_mb_shutdown(struct mbox_chan *chan)
+{
+	struct mfis_chan_priv *chan_priv = chan->con_priv;
+
+	if (chan_priv->irq)
+		free_irq(chan_priv->irq, chan);
+}
+
+static int mfis_mb_iicr_send_data(struct mbox_chan *chan, void *data)
+{
+	struct mfis_priv *priv = mfis_mb_mbox_to_priv(chan->mbox);
+	struct mfis_chan_priv *chan_priv = chan->con_priv;
+
+	/* Our doorbell still active? */
+	if (mfis_read(&priv->mbox_reg, chan_priv->reg) & BIT(0))
+		return -EBUSY;
+
+	/* Start our doorbell */
+	mfis_write(&priv->mbox_reg, chan_priv->reg, BIT(0));
+
+	return 0;
+}
+
+static bool mfis_mb_iicr_last_tx_done(struct mbox_chan *chan)
+{
+	struct mfis_priv *priv = mfis_mb_mbox_to_priv(chan->mbox);
+	struct mfis_chan_priv *chan_priv = chan->con_priv;
+
+	/* Our doorbell still active? */
+	return !(mfis_read(&priv->mbox_reg, chan_priv->reg) & BIT(0));
+}
+
+/* For MFIS variants using the IICR/EICR register pair */
+static const struct mbox_chan_ops mfis_iicr_ops = {
+	.startup = mfis_mb_startup,
+	.shutdown = mfis_mb_shutdown,
+	.send_data = mfis_mb_iicr_send_data,
+	.last_tx_done = mfis_mb_iicr_last_tx_done,
+};
+
+static struct mbox_chan *mfis_mb_of_xlate(struct mbox_controller *mbox,
+					  const struct of_phandle_args *sp)
+{
+	struct mfis_priv *priv = mfis_mb_mbox_to_priv(mbox);
+	struct mfis_chan_priv *chan_priv;
+	bool tx_uses_eicr, is_only_rx;
+	u32 chan_num, chan_flags;
+	struct mbox_chan *chan;
+
+	if (sp->args_count != 2)
+		return ERR_PTR(-EINVAL);
+
+	chan_num = sp->args[0];
+	chan_flags = sp->args[1];
+
+	if (chan_num >= priv->info->mb_num_channels)
+		return ERR_PTR(-EINVAL);
+
+	/* Channel layout is described in mfis_mb_probe() */
+	if (priv->info->mb_channels_are_unidir) {
+		is_only_rx = chan_flags & MFIS_CHANNEL_RX;
+		chan = mbox->chans + 2 * chan_num + is_only_rx;
+	} else {
+		is_only_rx = false;
+		chan = mbox->chans + chan_num;
+	}
+
+	if (priv->info->mb_reg_comes_from_dt) {
+		tx_uses_eicr = chan_flags & MFIS_CHANNEL_EICR;
+		if (tx_uses_eicr)
+			chan += mbox->num_chans / 2;
+	} else {
+		tx_uses_eicr = priv->info->mb_tx_uses_eicr;
+	}
+
+	chan_priv = chan->con_priv;
+	chan_priv->reg = (tx_uses_eicr ^ is_only_rx) ? MFIS_X5H_EICR(chan_num) :
+						       MFIS_X5H_IICR(chan_num);
+
+	if (!priv->info->mb_channels_are_unidir || is_only_rx) {
+		char irqname[8];
+		char suffix = tx_uses_eicr ? 'i' : 'e';
+
+		/* "ch0i" or "ch0e" */
+		scnprintf(irqname, sizeof(irqname), "ch%u%c", chan_num, suffix);
+
+		chan_priv->irq = of_irq_get_byname(mbox->dev->of_node, irqname);
+		if (chan_priv->irq < 0)
+			return ERR_PTR(chan_priv->irq);
+		if (chan_priv->irq == 0)
+			return ERR_PTR(-ENOENT);
+	}
+
+	return chan;
+}
+
+static int mfis_mb_probe(struct mfis_priv *priv)
+{
+	unsigned int num_chan = priv->info->mb_num_channels;
+	struct device *dev = priv->dev;
+	struct mbox_controller *mbox;
+	struct mbox_chan *chan;
+
+	if (priv->info->mb_channels_are_unidir) {
+		/* Channel layout: Ch0-TX, Ch0-RX, Ch1-TX... */
+		num_chan *= 2;
+	}
+
+	if (priv->info->mb_reg_comes_from_dt) {
+		/* Channel layout: <n> IICR channels, <n> EICR channels */
+		num_chan *= 2;
+	}
+
+	chan  = devm_kcalloc(dev, num_chan, sizeof(*chan), GFP_KERNEL);
+	if (!chan)
+		return -ENOMEM;
+
+	priv->chan_privs = devm_kcalloc(dev, num_chan, sizeof(*priv->chan_privs),
+					    GFP_KERNEL);
+	if (!priv->chan_privs)
+		return -ENOMEM;
+
+	mbox = &priv->mbox;
+
+	for (unsigned int i = 0; i < num_chan; i++)
+		chan[i].con_priv = &priv->chan_privs[i];
+
+	mbox->chans = chan;
+	mbox->num_chans = num_chan;
+	mbox->txdone_poll = true;
+	mbox->ops = &mfis_iicr_ops;
+	mbox->dev = dev;
+	mbox->of_xlate = mfis_mb_of_xlate;
+
+	return devm_mbox_controller_register(dev, mbox);
+}
+
+/********************************************************
+ *			Common				*
+ ********************************************************/
+static int mfis_reg_probe(struct platform_device *pdev, struct mfis_priv *priv,
+			  struct mfis_reg *mreg, const char *name, bool required)
+{
+	struct resource *res;
+	void __iomem *base;
+
+	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
+
+	/* If there is no mailbox resource, registers are in the common space */
+	if (!res && !required) {
+		*mreg = priv->common_reg;
+	} else {
+		base = devm_ioremap_resource(&pdev->dev, res);
+		if (IS_ERR(base))
+			return PTR_ERR(base);
+
+		mreg->base = base;
+		mreg->start = res->start;
+		mreg->priv = priv;
+	}
+
+	return 0;
+}
+
+static int mfis_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct mfis_priv *priv;
+	int ret;
+
+	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
+	if (!priv)
+		return -ENOMEM;
+
+	priv->dev = dev;
+	priv->info = of_device_get_match_data(dev);
+	if (!priv->info)
+		return -ENOENT;
+
+	spin_lock_init(&priv->unprotect_lock);
+
+	ret = mfis_reg_probe(pdev, priv, &priv->common_reg, "common", true);
+	if (ret)
+		return ret;
+
+	ret = mfis_reg_probe(pdev, priv, &priv->mbox_reg, "mboxes", false);
+	if (ret)
+		return ret;
+
+	return mfis_mb_probe(priv);
+}
+
+static const struct mfis_info mfis_info_r8a78000 = {
+	.unprotect_mask	= 0x000fffff,
+	.mb_num_channels = 64,
+	.mb_reg_comes_from_dt = true,
+	.mb_channels_are_unidir = true,
+};
+
+static const struct mfis_info mfis_info_r8a78000_scp = {
+	.unprotect_mask	= 0x000fffff,
+	.mb_num_channels = 32,
+	.mb_tx_uses_eicr = true,
+	.mb_channels_are_unidir = true,
+};
+
+static const struct of_device_id mfis_mfd_of_match[] = {
+	{ .compatible = "renesas,r8a78000-mfis", .data = &mfis_info_r8a78000, },
+	{ .compatible = "renesas,r8a78000-mfis-scp", .data = &mfis_info_r8a78000_scp, },
+	{}
+};
+MODULE_DEVICE_TABLE(of, mfis_mfd_of_match);
+
+static struct platform_driver mfis_driver = {
+	.driver = {
+		.name = "rcar-mfis",
+		.of_match_table = mfis_mfd_of_match,
+		.suppress_bind_attrs = true,
+	},
+	.probe	= mfis_probe,
+};
+module_platform_driver(mfis_driver);
+
+MODULE_AUTHOR("Kuninori Morimoto <kuninori.morimoto.gx@renesas.com>");
+MODULE_AUTHOR("Wolfram Sang <wsa+renesas@sang-engineering.com>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Renesas R-Car MFIS driver");
diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c
index 38ff0b823bdaf..dedb2a0be5864 100644
--- a/drivers/soc/renesas/renesas-soc.c
+++ b/drivers/soc/renesas/renesas-soc.c
@@ -442,8 +442,14 @@ static const struct renesas_id id_prr __initconst = {
 	.mask = 0xff00,
 };
 
+static const struct renesas_id id_mfis __initconst = {
+	.offset = 0x44,
+	.mask = 0xff00,
+};
+
 static const struct of_device_id renesas_ids[] __initconst = {
 	{ .compatible = "renesas,bsid",			.data = &id_bsid },
+	{ .compatible = "renesas,r8a78000-mfis",	.data = &id_mfis },
 	{ .compatible = "renesas,r9a07g043-sysc",	.data = &id_rzg2l },
 	{ .compatible = "renesas,r9a07g044-sysc",	.data = &id_rzg2l },
 	{ .compatible = "renesas,r9a07g054-sysc",	.data = &id_rzg2l },
@@ -468,7 +474,7 @@ static int __init renesas_soc_init(void)
 	const char *soc_id;
 	int ret;
 
-	match = of_match_node(renesas_socs, of_root);
+	match = of_machine_get_match(renesas_socs);
 	if (!match)
 		return -ENODEV;
 
@@ -501,7 +507,7 @@ static int __init renesas_soc_init(void)
 		product = readl(chipid + id->offset);
 		iounmap(chipid);
 
-		if (id == &id_prr) {
+		if (id == &id_prr || id == &id_mfis) {
 			/* R-Car M3-W ES1.1 incorrectly identifies as ES2.0 */
 			if ((product & 0x7fff) == 0x5210)
 				product ^= 0x11;
diff --git a/drivers/soc/sunxi/sunxi_sram.c b/drivers/soc/sunxi/sunxi_sram.c
index 446b9fc1f1759..2ccaeacf8c706 100644
--- a/drivers/soc/sunxi/sunxi_sram.c
+++ b/drivers/soc/sunxi/sunxi_sram.c
@@ -12,6 +12,7 @@
 
 #include <linux/debugfs.h>
 #include <linux/io.h>
+#include <linux/limits.h>
 #include <linux/mfd/syscon.h>
 #include <linux/module.h>
 #include <linux/of.h>
@@ -29,16 +30,16 @@ struct sunxi_sram_func {
 };
 
 struct sunxi_sram_data {
-	char			*name;
-	u8			reg;
-	u8			offset;
-	u8			width;
-	struct sunxi_sram_func	*func;
+	char				*name;
+	u8				reg;
+	u8				offset;
+	u8				width;
+	const struct sunxi_sram_func	*func;
 };
 
 struct sunxi_sram_desc {
 	struct sunxi_sram_data	data;
-	bool			claimed;
+	u8			claim_cnt;
 };
 
 #define SUNXI_SRAM_MAP(_reg_val, _val, _func)			\
@@ -54,7 +55,7 @@ struct sunxi_sram_desc {
 		.reg = _reg,					\
 		.offset = _off,					\
 		.width = _width,				\
-		.func = (struct sunxi_sram_func[]){		\
+		.func = (const struct sunxi_sram_func[]){	\
 			__VA_ARGS__, { } },			\
 	}
 
@@ -82,6 +83,12 @@ static struct sunxi_sram_desc sun50i_a64_sram_c = {
 				  SUNXI_SRAM_MAP(0, 1, "de2")),
 };
 
+static struct sunxi_sram_desc sun50i_h616_ve_sram = {
+	.data	= SUNXI_SRAM_DATA("VE", 0x0, 0, 1,
+				  SUNXI_SRAM_MAP(1, 0, "cpu"),
+				  SUNXI_SRAM_MAP(0, 1, "ve")),
+};
+
 static const struct of_device_id sunxi_sram_dt_ids[] = {
 	{
 		.compatible	= "allwinner,sun4i-a10-sram-a3-a4",
@@ -99,6 +106,10 @@ static const struct of_device_id sunxi_sram_dt_ids[] = {
 		.compatible	= "allwinner,sun50i-a64-sram-c",
 		.data		= &sun50i_a64_sram_c.data,
 	},
+	{
+		.compatible	= "allwinner,sun50i-h616-ve-sram",
+		.data		= &sun50i_h616_ve_sram.data,
+	},
 	{}
 };
 
@@ -111,7 +122,7 @@ static int sunxi_sram_show(struct seq_file *s, void *data)
 	struct device_node *sram_node, *section_node;
 	const struct sunxi_sram_data *sram_data;
 	const struct of_device_id *match;
-	struct sunxi_sram_func *func;
+	const struct sunxi_sram_func *func;
 	const __be32 *sram_addr_p, *section_addr_p;
 	u32 val;
 
@@ -164,69 +175,97 @@ static inline struct sunxi_sram_desc *to_sram_desc(const struct sunxi_sram_data
 	return container_of(data, struct sunxi_sram_desc, data);
 }
 
-static const struct sunxi_sram_data *sunxi_sram_of_parse(struct device_node *node,
-							 unsigned int *reg_value)
+static const struct sunxi_sram_data *sunxi_sram_get_match(struct device_node *np, u8 val,
+							  unsigned int *reg_value)
 {
 	const struct of_device_id *match;
 	const struct sunxi_sram_data *data;
-	struct sunxi_sram_func *func;
-	struct of_phandle_args args;
-	u8 val;
-	int ret;
-
-	ret = of_parse_phandle_with_fixed_args(node, "allwinner,sram", 1, 0,
-					       &args);
-	if (ret)
-		return ERR_PTR(ret);
+	const struct sunxi_sram_func *func;
 
-	if (!of_device_is_available(args.np)) {
-		ret = -EBUSY;
-		goto err;
-	}
+	if (!of_device_is_available(np))
+		return ERR_PTR(-ENODEV);
 
-	val = args.args[0];
-
-	match = of_match_node(sunxi_sram_dt_ids, args.np);
-	if (!match) {
-		ret = -EINVAL;
-		goto err;
-	}
+	match = of_match_node(sunxi_sram_dt_ids, np);
+	if (!match)
+		return ERR_PTR(-ENODEV);
 
 	data = match->data;
-	if (!data) {
-		ret = -EINVAL;
-		goto err;
-	}
-
-	for (func = data->func; func->func; func++) {
-		if (val == func->val) {
-			if (reg_value)
-				*reg_value = func->reg_val;
+	if (!data)
+		return ERR_PTR(-EINVAL);
 
+	for (func = data->func; func->func; func++)
+		if (val == func->val)
 			break;
-		}
-	}
 
-	if (!func->func) {
-		ret = -EINVAL;
-		goto err;
-	}
+	if (!func->func)
+		return ERR_PTR(-EINVAL);
 
-	of_node_put(args.np);
-	return match->data;
+	if (reg_value)
+		*reg_value = func->reg_val;
 
-err:
-	of_node_put(args.np);
-	return ERR_PTR(ret);
+	return data;
 }
 
-int sunxi_sram_claim(struct device *dev)
+#define SUNXI_SRAM_PROP "allwinner,sram"
+#define SUNXI_SRAM_CELLS 1
+
+static int sunxi_sram_claim_one(struct device_node *np, u8 arg)
 {
 	const struct sunxi_sram_data *sram_data;
 	struct sunxi_sram_desc *sram_desc;
 	unsigned int device;
 	u32 val, mask;
 
+	sram_data = sunxi_sram_get_match(np, arg, &device);
+	if (IS_ERR(sram_data))
+		return PTR_ERR(sram_data);
+
+	sram_desc = to_sram_desc(sram_data);
+
+	spin_lock(&sram_lock);
+
+	if (sram_desc->claim_cnt) {
+		if (!WARN_ON(sram_desc->claim_cnt == U8_MAX))
+			sram_desc->claim_cnt++;
+	} else {
+		mask = GENMASK(sram_data->offset + sram_data->width - 1,
+			       sram_data->offset);
+		val = readl(base + sram_data->reg);
+		val &= ~mask;
+		writel(val | ((device << sram_data->offset) & mask),
+		       base + sram_data->reg);
+
+		sram_desc->claim_cnt++;
+	}
+
+	spin_unlock(&sram_lock);
+
+	return 0;
+}
+
+static void sunxi_sram_release_one(struct device_node *np, u8 arg)
+{
+	const struct sunxi_sram_data *sram_data;
+	struct sunxi_sram_desc *sram_desc;
+
+	sram_data = sunxi_sram_get_match(np, arg, NULL);
+	if (IS_ERR(sram_data))
+		return;
+
+	sram_desc = to_sram_desc(sram_data);
+
+	spin_lock(&sram_lock);
+	if (!WARN_ON(sram_desc->claim_cnt == 0))
+		sram_desc->claim_cnt--;
+	spin_unlock(&sram_lock);
+}
+
+int sunxi_sram_claim(struct device *dev)
+{
+	struct of_phandle_iterator it;
+	int err;
+	int count = 0;
+
 	if (IS_ERR(base))
 		return PTR_ERR(base);
 
@@ -236,50 +275,53 @@ int sunxi_sram_claim(struct device *dev)
 	if (!dev || !dev->of_node)
 		return -EINVAL;
 
-	sram_data = sunxi_sram_of_parse(dev->of_node, &device);
-	if (IS_ERR(sram_data))
-		return PTR_ERR(sram_data);
+	of_for_each_phandle(&it, err, dev->of_node, SUNXI_SRAM_PROP,
+			    NULL, SUNXI_SRAM_CELLS) {
+		u32 args[SUNXI_SRAM_CELLS];
 
-	sram_desc = to_sram_desc(sram_data);
+		of_phandle_iterator_args(&it, args, SUNXI_SRAM_CELLS);
 
-	spin_lock(&sram_lock);
+		err = sunxi_sram_claim_one(it.node, args[0]);
+		if (err)
+			goto err;
 
-	if (sram_desc->claimed) {
-		spin_unlock(&sram_lock);
-		return -EBUSY;
+		count++;
 	}
 
-	mask = GENMASK(sram_data->offset + sram_data->width - 1,
-		       sram_data->offset);
-	val = readl(base + sram_data->reg);
-	val &= ~mask;
-	writel(val | ((device << sram_data->offset) & mask),
-	       base + sram_data->reg);
-
-	sram_desc->claimed = true;
-	spin_unlock(&sram_lock);
+	if (count == 0)
+		return -ENOENT;
 
 	return 0;
+
+err:
+	while (count--) {
+		struct of_phandle_args args;
+
+		of_parse_phandle_with_fixed_args(dev->of_node, SUNXI_SRAM_PROP,
+						 SUNXI_SRAM_CELLS, count, &args);
+		sunxi_sram_release_one(args.np, args.args[0]);
+	}
+
+	return err;
 }
 EXPORT_SYMBOL(sunxi_sram_claim);
 
 void sunxi_sram_release(struct device *dev)
 {
-	const struct sunxi_sram_data *sram_data;
-	struct sunxi_sram_desc *sram_desc;
+	struct of_phandle_iterator it;
+	int err;
 
 	if (!dev || !dev->of_node)
 		return;
 
-	sram_data = sunxi_sram_of_parse(dev->of_node, NULL);
-	if (IS_ERR(sram_data))
-		return;
+	of_for_each_phandle(&it, err, dev->of_node, SUNXI_SRAM_PROP,
+			    NULL, SUNXI_SRAM_CELLS) {
+		u32 args[SUNXI_SRAM_CELLS];
 
-	sram_desc = to_sram_desc(sram_data);
+		of_phandle_iterator_args(&it, args, SUNXI_SRAM_CELLS);
 
-	spin_lock(&sram_lock);
-	sram_desc->claimed = false;
-	spin_unlock(&sram_lock);
+		sunxi_sram_release_one(it.node, args[0]);
+	}
 }
 EXPORT_SYMBOL(sunxi_sram_release);
 
diff --git a/drivers/soc/ti/k3-ringacc.c b/drivers/soc/ti/k3-ringacc.c
index 7602b8a909b05..e2ca380812d29 100644
--- a/drivers/soc/ti/k3-ringacc.c
+++ b/drivers/soc/ti/k3-ringacc.c
@@ -1012,7 +1012,7 @@ static int k3_ringacc_ring_pop_head_proxy(struct k3_ring *ring, void *elem)
 static int k3_ringacc_ring_pop_tail_proxy(struct k3_ring *ring, void *elem)
 {
 	return k3_ringacc_ring_access_proxy(ring, elem,
-					    K3_RINGACC_ACCESS_MODE_POP_HEAD);
+					    K3_RINGACC_ACCESS_MODE_POP_TAIL);
 }
 
 static int k3_ringacc_ring_access_io(struct k3_ring *ring, void *elem,
@@ -1083,7 +1083,7 @@ static int k3_ringacc_ring_pop_io(struct k3_ring *ring, void *elem)
 static int k3_ringacc_ring_pop_tail_io(struct k3_ring *ring, void *elem)
 {
 	return k3_ringacc_ring_access_io(ring, elem,
-					 K3_RINGACC_ACCESS_MODE_POP_HEAD);
+					 K3_RINGACC_ACCESS_MODE_POP_TAIL);
 }
 
 /*
diff --git a/drivers/soc/ti/k3-socinfo.c b/drivers/soc/ti/k3-socinfo.c
index 676041879eca3..48dbef3acceb4 100644
--- a/drivers/soc/ti/k3-socinfo.c
+++ b/drivers/soc/ti/k3-socinfo.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/mfd/syscon.h>
+#include <linux/nvmem-consumer.h>
 #include <linux/of.h>
 #include <linux/of_address.h>
 #include <linux/regmap.h>
@@ -25,6 +26,8 @@
 #define CTRLMMR_WKUP_JTAGID_VARIANT_SHIFT	(28)
 #define CTRLMMR_WKUP_JTAGID_VARIANT_MASK	GENMASK(31, 28)
 
+#define GP_SW1_ADR_MASK			GENMASK(3, 0)
+
 #define CTRLMMR_WKUP_JTAGID_PARTNO_SHIFT	(12)
 #define CTRLMMR_WKUP_JTAGID_PARTNO_MASK		GENMASK(27, 12)
 
@@ -70,6 +73,23 @@ static const char * const am62lx_rev_string_map[] = {
 	"1.0", "1.1",
 };
 
+static const char * const am62p_gpsw_rev_string_map[] = {
+	"1.0", "1.1", "1.2",
+};
+
+static int
+k3_chipinfo_get_gpsw_variant(struct device *dev)
+{
+	u32 gpsw_val = 0;
+	int ret;
+
+	ret = nvmem_cell_read_u32(dev, "gpsw1", &gpsw_val);
+	if (ret)
+		return ret;
+
+	return gpsw_val & GP_SW1_ADR_MASK;
+}
+
 static int
 k3_chipinfo_partno_to_names(unsigned int partno,
 			    struct soc_device_attribute *soc_dev_attr)
@@ -86,9 +106,11 @@ k3_chipinfo_partno_to_names(unsigned int partno,
 }
 
 static int
-k3_chipinfo_variant_to_sr(unsigned int partno, unsigned int variant,
-			  struct soc_device_attribute *soc_dev_attr)
+k3_chipinfo_variant_to_sr(struct device *dev, unsigned int partno,
+			  unsigned int variant, struct soc_device_attribute *soc_dev_attr)
 {
+	int gpsw_variant = 0;
+
 	switch (partno) {
 	case JTAG_ID_PARTNO_J721E:
 		if (variant >= ARRAY_SIZE(j721e_rev_string_map))
@@ -102,6 +124,19 @@ k3_chipinfo_variant_to_sr(unsigned int partno, unsigned int variant,
 		soc_dev_attr->revision = kasprintf(GFP_KERNEL, "SR%s",
 						   am62lx_rev_string_map[variant]);
 		break;
+	case JTAG_ID_PARTNO_AM62PX:
+		/* Check GP_SW1 for silicon revision */
+		gpsw_variant = k3_chipinfo_get_gpsw_variant(dev);
+		if (gpsw_variant == -EPROBE_DEFER)
+			return gpsw_variant;
+		if (gpsw_variant < 0 || gpsw_variant >= ARRAY_SIZE(am62p_gpsw_rev_string_map)) {
+			dev_warn(dev, "Failed to get silicon variant (%d), set SR1.0\n",
+				 gpsw_variant);
+			gpsw_variant = 0;
+		}
+		soc_dev_attr->revision = kasprintf(GFP_KERNEL, "SR%s",
+						   am62p_gpsw_rev_string_map[gpsw_variant]);
+		break;
 	default:
 		variant++;
 		soc_dev_attr->revision = kasprintf(GFP_KERNEL, "SR%x.0",
@@ -173,7 +208,7 @@ static int k3_chipinfo_probe(struct platform_device *pdev)
 		goto err;
 	}
 
-	ret = k3_chipinfo_variant_to_sr(partno_id, variant, soc_dev_attr);
+	ret = k3_chipinfo_variant_to_sr(dev, partno_id, variant, soc_dev_attr);
 	if (ret) {
 		dev_err(dev, "Unknown SoC SR[0x%08X]: %d\n", jtag_id, ret);
 		goto err;
diff --git a/drivers/soc/ti/knav_qmss_queue.c b/drivers/soc/ti/knav_qmss_queue.c
index 86d7a9c9ae011..a0644578789f5 100644
--- a/drivers/soc/ti/knav_qmss_queue.c
+++ b/drivers/soc/ti/knav_qmss_queue.c
@@ -1873,7 +1873,12 @@ static int knav_queue_probe(struct platform_device *pdev)
 
 static void knav_queue_remove(struct platform_device *pdev)
 {
-	/* TODO: Free resources */
+	struct knav_device *kdev = platform_get_drvdata(pdev);
+
+	device_ready = false;
+	knav_queue_stop_pdsps(kdev);
+	knav_queue_free_regions(kdev);
+	knav_free_queue_ranges(kdev);
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 }
diff --git a/drivers/soc/xilinx/zynqmp_power.c b/drivers/soc/xilinx/zynqmp_power.c
index 9085db1b480aa..370e61ac47d88 100644
--- a/drivers/soc/xilinx/zynqmp_power.c
+++ b/drivers/soc/xilinx/zynqmp_power.c
@@ -303,18 +303,18 @@ static int zynqmp_pm_probe(struct platform_device *pdev)
 	 * is not available to use) or -ENODEV(Xilinx Event Manager not compiled),
 	 * then use ipi-mailbox or interrupt method.
 	 */
+	zynqmp_pm_init_suspend_work = devm_kzalloc(&pdev->dev,
+						   sizeof(struct zynqmp_pm_work_struct),
+						   GFP_KERNEL);
+	if (!zynqmp_pm_init_suspend_work)
+		return -ENOMEM;
+
+	INIT_WORK(&zynqmp_pm_init_suspend_work->callback_work,
+		  zynqmp_pm_init_suspend_work_fn);
+
 	ret = register_event(&pdev->dev, PM_INIT_SUSPEND_CB, 0, 0, false,
 			     suspend_event_callback);
 	if (!ret) {
-		zynqmp_pm_init_suspend_work = devm_kzalloc(&pdev->dev,
-							   sizeof(struct zynqmp_pm_work_struct),
-							   GFP_KERNEL);
-		if (!zynqmp_pm_init_suspend_work)
-			return -ENOMEM;
-
-		INIT_WORK(&zynqmp_pm_init_suspend_work->callback_work,
-			  zynqmp_pm_init_suspend_work_fn);
-
 		ret = zynqmp_pm_get_family_info(&pm_family_code);
 		if (ret < 0)
 			return ret;
@@ -326,14 +326,6 @@ static int zynqmp_pm_probe(struct platform_device *pdev)
 		else
 			return -ENODEV;
 
-		ret = register_event(&pdev->dev, PM_NOTIFY_CB, node_id, EVENT_SUBSYSTEM_RESTART,
-				     false, subsystem_restart_event_callback);
-		if (ret) {
-			dev_err(&pdev->dev, "Failed to Register with Xilinx Event manager %d\n",
-				ret);
-			return ret;
-		}
-
 		zynqmp_pm_init_restart_work = devm_kzalloc(&pdev->dev,
 							   sizeof(struct zynqmp_pm_work_struct),
 							   GFP_KERNEL);
@@ -342,19 +334,18 @@ static int zynqmp_pm_probe(struct platform_device *pdev)
 
 		INIT_WORK(&zynqmp_pm_init_restart_work->callback_work,
 			  zynqmp_pm_subsystem_restart_work_fn);
+
+		ret = register_event(&pdev->dev, PM_NOTIFY_CB, node_id, EVENT_SUBSYSTEM_RESTART,
+				     false, subsystem_restart_event_callback);
+		if (ret) {
+			dev_err(&pdev->dev, "Failed to Register with Xilinx Event manager %d\n",
+				ret);
+			return ret;
+		}
 	} else if (ret != -EACCES && ret != -ENODEV) {
 		dev_err(&pdev->dev, "Failed to Register with Xilinx Event manager %d\n", ret);
 		return ret;
 	} else if (of_property_present(pdev->dev.of_node, "mboxes")) {
-		zynqmp_pm_init_suspend_work =
-			devm_kzalloc(&pdev->dev,
-				     sizeof(struct zynqmp_pm_work_struct),
-				     GFP_KERNEL);
-		if (!zynqmp_pm_init_suspend_work)
-			return -ENOMEM;
-
-		INIT_WORK(&zynqmp_pm_init_suspend_work->callback_work,
-			  zynqmp_pm_init_suspend_work_fn);
 		client = devm_kzalloc(&pdev->dev, sizeof(*client), GFP_KERNEL);
 		if (!client)
 			return -ENOMEM;
@@ -398,8 +389,10 @@ static void zynqmp_pm_remove(struct platform_device *pdev)
 {
 	sysfs_remove_file(&pdev->dev.kobj, &dev_attr_suspend_mode.attr);
 
-	if (!rx_chan)
+	if (rx_chan) {
 		mbox_free_channel(rx_chan);
+		rx_chan = NULL;
+	}
 }
 
 static const struct of_device_id pm_of_match[] = {
diff --git a/drivers/soundwire/bus.c b/drivers/soundwire/bus.c
index fe5316d93fefe..ea3a24f805c00 100644
--- a/drivers/soundwire/bus.c
+++ b/drivers/soundwire/bus.c
@@ -1372,6 +1372,37 @@ int sdw_slave_get_current_bank(struct sdw_slave *slave)
 }
 EXPORT_SYMBOL_GPL(sdw_slave_get_current_bank);
 
+/**
+ * sdw_slave_wait_for_init - Wait for device initialisation
+ * @slave: Pointer to the SoundWire peripheral.
+ * @timeout_ms: Timeout in milliseconds.
+ *
+ * Wait for a peripheral device to enumerate and be initialised by the
+ * SoundWire core.
+ *
+ * Return: Zero on success, and a negative error code on failure.
+ */
+int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms)
+{
+	unsigned long time;
+
+	if (!slave->unattach_request)
+		return 0;
+
+	time = wait_for_completion_timeout(&slave->initialization_complete,
+					   msecs_to_jiffies(timeout_ms));
+	if (!time) {
+		dev_err(&slave->dev, "Initialization not complete\n");
+		sdw_show_ping_status(slave->bus, true);
+		return -ETIMEDOUT;
+	}
+
+	slave->unattach_request = 0;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(sdw_slave_wait_for_init);
+
 static int sdw_slave_set_frequency(struct sdw_slave *slave)
 {
 	int scale_index;
diff --git a/drivers/soundwire/generic_bandwidth_allocation.c b/drivers/soundwire/generic_bandwidth_allocation.c
index fb3970e12dac9..3575d69ce1c50 100644
--- a/drivers/soundwire/generic_bandwidth_allocation.c
+++ b/drivers/soundwire/generic_bandwidth_allocation.c
@@ -299,39 +299,35 @@ static int sdw_add_element_group_count(struct sdw_group *group,
 	int num = group->count;
 	int i;
 
-	for (i = 0; i <= num; i++) {
+	for (i = 0; i < num; i++) {
 		if (rate == group->rates[i] && lane == group->lanes[i])
-			break;
-
-		if (i != num)
-			continue;
+			return 0;
+	}
 
-		if (group->count >= group->max_size) {
-			unsigned int *rates;
-			unsigned int *lanes;
+	if (group->count >= group->max_size) {
+		unsigned int *rates;
+		unsigned int *lanes;
 
-			group->max_size += 1;
-			rates = krealloc(group->rates,
-					 (sizeof(int) * group->max_size),
-					 GFP_KERNEL);
-			if (!rates)
-				return -ENOMEM;
+		rates = krealloc_array(group->rates, group->max_size + 1,
+				       sizeof(*group->rates), GFP_KERNEL);
+		if (!rates)
+			return -ENOMEM;
 
-			group->rates = rates;
+		group->rates = rates;
 
-			lanes = krealloc(group->lanes,
-					 (sizeof(int) * group->max_size),
-					 GFP_KERNEL);
-			if (!lanes)
-				return -ENOMEM;
+		lanes = krealloc_array(group->lanes, group->max_size + 1,
+				       sizeof(*group->lanes), GFP_KERNEL);
+		if (!lanes)
+			return -ENOMEM;
 
-			group->lanes = lanes;
-		}
+		group->lanes = lanes;
 
-		group->rates[group->count] = rate;
-		group->lanes[group->count++] = lane;
+		group->max_size += 1;
 	}
 
+	group->rates[group->count] = rate;
+	group->lanes[group->count++] = lane;
+
 	return 0;
 }
 
diff --git a/drivers/soundwire/intel.c b/drivers/soundwire/intel.c
index dcd7440e78fa6..af65214836b4d 100644
--- a/drivers/soundwire/intel.c
+++ b/drivers/soundwire/intel.c
@@ -906,19 +906,6 @@ static int intel_trigger(struct snd_pcm_substream *substream, int cmd, struct sn
 	}
 
 	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_SUSPEND:
-
-		/*
-		 * The .prepare callback is used to deal with xruns and resume operations.
-		 * In the case of xruns, the DMAs and SHIM registers cannot be touched,
-		 * but for resume operations the DMAs and SHIM registers need to be initialized.
-		 * the .trigger callback is used to track the suspend case only.
-		 */
-
-		dai_runtime->suspended = true;
-
-		break;
-
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
 		dai_runtime->paused = true;
 		break;
@@ -955,10 +942,12 @@ static int intel_component_dais_suspend(struct snd_soc_component *component)
 	struct snd_soc_dai *dai;
 
 	/*
-	 * In the corner case where a SUSPEND happens during a PAUSE, the ALSA core
-	 * does not throw the TRIGGER_SUSPEND. This leaves the DAIs in an unbalanced state.
-	 * Since the component suspend is called last, we can trap this corner case
-	 * and force the DAIs to release their resources.
+	 * Mark all open streams as suspended.
+	 * Open streams at this point can be in SUSPENDED, PAUSED or STOPPED
+	 * state and during prepare the DMAs and SHIM registers need to be
+	 * initialized for them.
+	 * The STOPPED state is a special corner case which can happen if audio
+	 * experiences xrun at suspend time.
 	 */
 	for_each_component_dais(component, dai) {
 		struct sdw_cdns *cdns = snd_soc_dai_get_drvdata(dai);
@@ -966,13 +955,7 @@ static int intel_component_dais_suspend(struct snd_soc_component *component)
 
 		dai_runtime = cdns->dai_runtime_array[dai->id];
 
-		if (!dai_runtime)
-			continue;
-
-		if (dai_runtime->suspended)
-			continue;
-
-		if (dai_runtime->paused)
+		if (dai_runtime)
 			dai_runtime->suspended = true;
 	}
 
diff --git a/drivers/soundwire/intel_ace2x.c b/drivers/soundwire/intel_ace2x.c
index 20422534baf19..b37933efac5d2 100644
--- a/drivers/soundwire/intel_ace2x.c
+++ b/drivers/soundwire/intel_ace2x.c
@@ -317,6 +317,7 @@ static void intel_ace2x_bpt_close_stream(struct sdw_intel *sdw, struct sdw_slave
 		dev_err(cdns->dev, "%s: remove slave failed: %d\n",
 			__func__, ret);
 
+	sdw_release_stream(cdns->bus.bpt_stream);
 	cdns->bus.bpt_stream = NULL;
 }
 
@@ -894,19 +895,6 @@ static int intel_trigger(struct snd_pcm_substream *substream, int cmd, struct sn
 	}
 
 	switch (cmd) {
-	case SNDRV_PCM_TRIGGER_SUSPEND:
-
-		/*
-		 * The .prepare callback is used to deal with xruns and resume operations.
-		 * In the case of xruns, the DMAs and SHIM registers cannot be touched,
-		 * but for resume operations the DMAs and SHIM registers need to be initialized.
-		 * the .trigger callback is used to track the suspend case only.
-		 */
-
-		dai_runtime->suspended = true;
-
-		break;
-
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
 		dai_runtime->paused = true;
 		break;
@@ -930,8 +918,34 @@ static const struct snd_soc_dai_ops intel_pcm_dai_ops = {
 	.get_stream = intel_get_sdw_stream,
 };
 
+static int intel_component_dais_suspend(struct snd_soc_component *component)
+{
+	struct snd_soc_dai *dai;
+
+	/*
+	 * Mark all open streams as suspended.
+	 * Open streams at this point can be in SUSPENDED, PAUSED or STOPPED
+	 * state and during prepare the DMAs and SHIM registers need to be
+	 * initialized for them.
+	 * The STOPPED state is a special corner case which can happen if audio
+	 * experiences xrun at suspend time.
+	 */
+	for_each_component_dais(component, dai) {
+		struct sdw_cdns *cdns = snd_soc_dai_get_drvdata(dai);
+		struct sdw_cdns_dai_runtime *dai_runtime;
+
+		dai_runtime = cdns->dai_runtime_array[dai->id];
+
+		if (dai_runtime)
+			dai_runtime->suspended = true;
+	}
+
+	return 0;
+}
+
 static const struct snd_soc_component_driver dai_component = {
 	.name			= "soundwire",
+	.suspend		= intel_component_dais_suspend,
 };
 
 /*
diff --git a/drivers/soundwire/intel_auxdevice.c b/drivers/soundwire/intel_auxdevice.c
index 913e95207ee1a..0b8107bec9abe 100644
--- a/drivers/soundwire/intel_auxdevice.c
+++ b/drivers/soundwire/intel_auxdevice.c
@@ -51,6 +51,8 @@ struct wake_capable_part {
 };
 
 static struct wake_capable_part wake_capable_list[] = {
+	{0x01fa, 0x2A30},
+	{0x01fa, 0x2A3B},
 	{0x01fa, 0x4243},
 	{0x01fa, 0x4245},
 	{0x01fa, 0x4249},
@@ -70,6 +72,7 @@ static struct wake_capable_part wake_capable_list[] = {
 	{0x025d, 0x717},
 	{0x025d, 0x721},
 	{0x025d, 0x722},
+	{0x04b3, 0x9356},
 };
 
 static bool is_wake_capable(struct sdw_slave *slave)
diff --git a/drivers/soundwire/slave.c b/drivers/soundwire/slave.c
index ff763b692078d..e0c49cbcb1bab 100644
--- a/drivers/soundwire/slave.c
+++ b/drivers/soundwire/slave.c
@@ -244,8 +244,8 @@ int sdw_of_find_slaves(struct sdw_bus *bus)
 		struct sdw_slave_id id;
 		const __be32 *addr;
 
-		compat = of_get_property(node, "compatible", NULL);
-		if (!compat)
+		ret = of_property_read_string(node, "compatible", &compat);
+		if (ret)
 			continue;
 
 		ret = sscanf(compat, "sdw%01x%04hx%04hx%02hhx", &sdw_version,
diff --git a/drivers/soundwire/stream.c b/drivers/soundwire/stream.c
index 4ed8fb7663ad4..cdac009b1a756 100644
--- a/drivers/soundwire/stream.c
+++ b/drivers/soundwire/stream.c
@@ -697,6 +697,13 @@ static int sdw_program_params(struct sdw_bus *bus, bool prepare)
 		if (scale_index < 0)
 			return scale_index;
 
+		/* Skip the unattached Peripherals */
+		if (!completion_done(&slave->enumeration_complete)) {
+			dev_warn(&slave->dev,
+				 "Not enumerated, skip programming BUSCLOCK_SCALE\n");
+			continue;
+		}
+
 		ret = sdw_write_no_pm(slave, addr1, scale_index);
 		if (ret < 0) {
 			dev_err(&slave->dev, "SDW_SCP_BUSCLOCK_SCALE register write failed\n");
@@ -2229,11 +2236,15 @@ EXPORT_SYMBOL(sdw_stream_add_slave);
  * @slave: SDW Slave instance
  * @stream: SoundWire stream
  *
- * This removes and frees port_rt and slave_rt from a stream
+ * This removes and frees port_rt and slave_rt from a stream.
+ * If stream is NULL or an ERR_PTR, do nothing and return 0.
  */
 int sdw_stream_remove_slave(struct sdw_slave *slave,
 			    struct sdw_stream_runtime *stream)
 {
+	if (IS_ERR_OR_NULL(stream))
+		return 0;
+
 	mutex_lock(&slave->bus->bus_lock);
 
 	sdw_slave_port_free(slave, stream);
diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig
index b563f49e21977..957c3e065b839 100644
--- a/drivers/spi/Kconfig
+++ b/drivers/spi/Kconfig
@@ -321,7 +321,8 @@ config SPI_CADENCE_QUADSPI
 
 config SPI_CADENCE_XSPI
 	tristate "Cadence XSPI controller"
-	depends on OF && HAS_IOMEM && 64BIT
+	depends on HAS_IOMEM || COMPILE_TEST
+	depends on OF
 	depends on SPI_MEM
 	help
 	  Enable support for the Cadence XSPI Flash controller.
@@ -1085,6 +1086,15 @@ config SPI_SG2044_NOR
 	  also supporting 3Byte address devices and 4Byte address
 	  devices.
 
+config SPI_SPACEMIT_K1
+	tristate "K1 SPI Controller"
+	depends on ARCH_SPACEMIT || COMPILE_TEST
+	depends on OF
+	imply MMP_PDMA if ARCH_SPACEMIT
+	default m if ARCH_SPACEMIT
+	help
+	  Enable support for the SpacemiT K1 SPI controller.
+
 config SPI_SPRD
 	tristate "Spreadtrum SPI controller"
 	depends on ARCH_SPRD || COMPILE_TEST
diff --git a/drivers/spi/Makefile b/drivers/spi/Makefile
index 9d36190a98848..9fa12498ce8c0 100644
--- a/drivers/spi/Makefile
+++ b/drivers/spi/Makefile
@@ -143,6 +143,7 @@ obj-$(CONFIG_SPI_SIFIVE)		+= spi-sifive.o
 obj-$(CONFIG_SPI_SLAVE_MT27XX)          += spi-slave-mt27xx.o
 obj-$(CONFIG_SPI_SN_F_OSPI)		+= spi-sn-f-ospi.o
 obj-$(CONFIG_SPI_SG2044_NOR)	+= spi-sg2044-nor.o
+obj-$(CONFIG_SPI_SPACEMIT_K1)		+= spi-spacemit-k1.o
 obj-$(CONFIG_SPI_SPRD)			+= spi-sprd.o
 obj-$(CONFIG_SPI_SPRD_ADI)		+= spi-sprd-adi.o
 obj-$(CONFIG_SPI_STM32) 		+= spi-stm32.o
diff --git a/drivers/spi/spi-airoha-snfi.c b/drivers/spi/spi-airoha-snfi.c
index 7b6c09f91fef9..95bfde7c8e7f2 100644
--- a/drivers/spi/spi-airoha-snfi.c
+++ b/drivers/spi/spi-airoha-snfi.c
@@ -546,7 +546,7 @@ static int airoha_snand_dirmap_create(struct spi_mem_dirmap_desc *desc)
 	if (desc->info.length > SPI_NAND_CACHE_SIZE)
 		return -E2BIG;
 
-	if (!airoha_snand_supports_op(desc->mem, &desc->info.op_tmpl))
+	if (!airoha_snand_supports_op(desc->mem, desc->info.op_tmpl))
 		return -EOPNOTSUPP;
 
 	return 0;
@@ -572,7 +572,7 @@ static ssize_t airoha_snand_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	 * DUALIO and QUADIO opcodes are not supported by the spi controller,
 	 * replace them with supported opcodes.
 	 */
-	opcode = desc->info.op_tmpl.cmd.opcode;
+	opcode = desc->info.op_tmpl->cmd.opcode;
 	switch (opcode) {
 	case SPI_NAND_OP_READ_FROM_CACHE_SINGLE:
 	case SPI_NAND_OP_READ_FROM_CACHE_SINGLE_FAST:
@@ -761,7 +761,7 @@ static ssize_t airoha_snand_dirmap_write(struct spi_mem_dirmap_desc *desc,
 	/* minimum oob size is 64 */
 	bytes = round_up(offs + len, 64);
 
-	opcode = desc->info.op_tmpl.cmd.opcode;
+	opcode = desc->info.op_tmpl->cmd.opcode;
 	switch (opcode) {
 	case SPI_NAND_OP_PROGRAM_LOAD_SINGLE:
 	case SPI_NAND_OP_PROGRAM_LOAD_RAMDOM_SINGLE:
diff --git a/drivers/spi/spi-altera-platform.c b/drivers/spi/spi-altera-platform.c
index fc81de2610ef5..3ee5d3480bb46 100644
--- a/drivers/spi/spi-altera-platform.c
+++ b/drivers/spi/spi-altera-platform.c
@@ -39,12 +39,12 @@ static int altera_spi_probe(struct platform_device *pdev)
 	enum altera_spi_type type = ALTERA_SPI_TYPE_UNKNOWN;
 	struct altera_spi *hw;
 	struct spi_controller *host;
-	int err = -ENODEV;
+	int err;
 	u16 i;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(struct altera_spi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(struct altera_spi));
 	if (!host)
-		return err;
+		return -ENOMEM;
 
 	/* setup the host state. */
 	host->bus_num = -1;
@@ -54,8 +54,7 @@ static int altera_spi_probe(struct platform_device *pdev)
 			dev_err(&pdev->dev,
 				"Invalid number of chipselect: %u\n",
 				pdata->num_chipselect);
-			err = -EINVAL;
-			goto exit;
+			return -EINVAL;
 		}
 
 		host->num_chipselect = pdata->num_chipselect;
@@ -80,7 +79,7 @@ static int altera_spi_probe(struct platform_device *pdev)
 		hw->regmap = dev_get_regmap(pdev->dev.parent, NULL);
 		if (!hw->regmap) {
 			dev_err(&pdev->dev, "get regmap failed\n");
-			goto exit;
+			return -ENODEV;
 		}
 
 		regoff = platform_get_resource(pdev, IORESOURCE_REG, 0);
@@ -90,17 +89,14 @@ static int altera_spi_probe(struct platform_device *pdev)
 		void __iomem *res;
 
 		res = devm_platform_ioremap_resource(pdev, 0);
-		if (IS_ERR(res)) {
-			err = PTR_ERR(res);
-			goto exit;
-		}
+		if (IS_ERR(res))
+			return PTR_ERR(res);
 
 		hw->regmap = devm_regmap_init_mmio(&pdev->dev, res,
 						   &spi_altera_config);
 		if (IS_ERR(hw->regmap)) {
 			dev_err(&pdev->dev, "regmap mmio init failed\n");
-			err = PTR_ERR(hw->regmap);
-			goto exit;
+			return PTR_ERR(hw->regmap);
 		}
 	}
 
@@ -112,12 +108,12 @@ static int altera_spi_probe(struct platform_device *pdev)
 		err = devm_request_irq(&pdev->dev, hw->irq, altera_spi_irq, 0,
 				       pdev->name, host);
 		if (err)
-			goto exit;
+			return err;
 	}
 
 	err = devm_spi_register_controller(&pdev->dev, host);
 	if (err)
-		goto exit;
+		return err;
 
 	if (pdata) {
 		for (i = 0; i < pdata->num_devices; i++) {
@@ -131,9 +127,6 @@ static int altera_spi_probe(struct platform_device *pdev)
 	dev_info(&pdev->dev, "regoff %u, irq %d\n", hw->regoff, hw->irq);
 
 	return 0;
-exit:
-	spi_controller_put(host);
-	return err;
 }
 
 #ifdef CONFIG_OF
diff --git a/drivers/spi/spi-amd.c b/drivers/spi/spi-amd.c
index 4d1dce4f49740..71a6e5c475b03 100644
--- a/drivers/spi/spi-amd.c
+++ b/drivers/spi/spi-amd.c
@@ -868,7 +868,7 @@ static int amd_spi_probe(struct platform_device *pdev)
 	dev_dbg(dev, "io_remap_address: %p\n", amd_spi->io_remap_addr);
 
 	amd_spi->version = (uintptr_t)device_get_match_data(dev);
-	host->bus_num = 0;
+	host->bus_num = (amd_spi->version == AMD_HID2_SPI) ? 2 : 0;
 
 	return amd_spi_probe_common(dev, host);
 }
diff --git a/drivers/spi/spi-amlogic-spifc-a1.c b/drivers/spi/spi-amlogic-spifc-a1.c
index 7ee4c92e6e09a..77a2c11bec5ed 100644
--- a/drivers/spi/spi-amlogic-spifc-a1.c
+++ b/drivers/spi/spi-amlogic-spifc-a1.c
@@ -206,10 +206,9 @@ static int amlogic_spifc_a1_read(struct amlogic_spifc_a1 *spifc, void *buf,
 	u32 val = readl(spifc->base + SPIFC_A1_USER_CTRL3_REG);
 	int ret;
 
-	val &= ~(SPIFC_A1_USER_DIN_MODE | SPIFC_A1_USER_DIN_BYTES);
 	val |= SPIFC_A1_USER_DIN_ENABLE;
-	val |= FIELD_PREP(SPIFC_A1_USER_DIN_MODE, mode);
-	val |= FIELD_PREP(SPIFC_A1_USER_DIN_BYTES, size);
+	FIELD_MODIFY(SPIFC_A1_USER_DIN_MODE, &val, mode);
+	FIELD_MODIFY(SPIFC_A1_USER_DIN_BYTES, &val, size);
 	writel(val, spifc->base + SPIFC_A1_USER_CTRL3_REG);
 
 	ret = amlogic_spifc_a1_request(spifc, true);
diff --git a/drivers/spi/spi-amlogic-spisg.c b/drivers/spi/spi-amlogic-spisg.c
index f9de2d2c92132..afc8af04638d3 100644
--- a/drivers/spi/spi-amlogic-spisg.c
+++ b/drivers/spi/spi-amlogic-spisg.c
@@ -258,8 +258,7 @@ static int aml_spisg_setup_transfer(struct spisg_device *spisg,
 	int ret;
 
 	memset(desc, 0, sizeof(*desc));
-	if (exdesc)
-		memset(exdesc, 0, sizeof(*exdesc));
+	memset(exdesc, 0, sizeof(*exdesc));
 	aml_spisg_set_speed(spisg, xfer->speed_hz);
 	xfer->effective_speed_hz = spisg->effective_speed_hz;
 
@@ -601,14 +600,11 @@ static int aml_spisg_prepare_message(struct spi_controller *ctlr,
 
 	spisg->bytes_per_word = spi->bits_per_word >> 3;
 
-	spisg->cfg_spi &= ~CFG_SLAVE_SELECT;
-	spisg->cfg_spi |= FIELD_PREP(CFG_SLAVE_SELECT, spi_get_chipselect(spi, 0));
-
-	spisg->cfg_bus &= ~(CFG_CPOL | CFG_CPHA | CFG_B_L_ENDIAN | CFG_HALF_DUPLEX);
-	spisg->cfg_bus |= FIELD_PREP(CFG_CPOL, !!(spi->mode & SPI_CPOL)) |
-			  FIELD_PREP(CFG_CPHA, !!(spi->mode & SPI_CPHA)) |
-			  FIELD_PREP(CFG_B_L_ENDIAN, !!(spi->mode & SPI_LSB_FIRST)) |
-			  FIELD_PREP(CFG_HALF_DUPLEX, !!(spi->mode & SPI_3WIRE));
+	FIELD_MODIFY(CFG_SLAVE_SELECT, &spisg->cfg_spi, spi_get_chipselect(spi, 0));
+	FIELD_MODIFY(CFG_CPOL, &spisg->cfg_bus, !!(spi->mode & SPI_CPOL));
+	FIELD_MODIFY(CFG_CPHA, &spisg->cfg_bus, !!(spi->mode & SPI_CPHA));
+	FIELD_MODIFY(CFG_B_L_ENDIAN, &spisg->cfg_bus, !!(spi->mode & SPI_LSB_FIRST));
+	FIELD_MODIFY(CFG_HALF_DUPLEX, &spisg->cfg_bus, !!(spi->mode & SPI_3WIRE));
 
 	return 0;
 }
diff --git a/drivers/spi/spi-armada-3700.c b/drivers/spi/spi-armada-3700.c
index 78248729d3e9a..ca2faa265ca0c 100644
--- a/drivers/spi/spi-armada-3700.c
+++ b/drivers/spi/spi-armada-3700.c
@@ -818,17 +818,13 @@ static int a3700_spi_probe(struct platform_device *pdev)
 	u32 num_cs = 0;
 	int irq, ret = 0;
 
-	host = spi_alloc_host(dev, sizeof(*spi));
-	if (!host) {
-		dev_err(dev, "host allocation failed\n");
-		ret = -ENOMEM;
-		goto out;
-	}
+	host = devm_spi_alloc_host(dev, sizeof(*spi));
+	if (!host)
+		return -ENOMEM;
 
 	if (of_property_read_u32(dev->of_node, "num-cs", &num_cs)) {
 		dev_err(dev, "could not find num-cs\n");
-		ret = -ENXIO;
-		goto error;
+		return -ENXIO;
 	}
 
 	host->bus_num = pdev->id;
@@ -849,25 +845,20 @@ static int a3700_spi_probe(struct platform_device *pdev)
 	spi->host = host;
 
 	spi->base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(spi->base)) {
-		ret = PTR_ERR(spi->base);
-		goto error;
-	}
+	if (IS_ERR(spi->base))
+		return PTR_ERR(spi->base);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = -ENXIO;
-		goto error;
-	}
+	if (irq < 0)
+		return -ENXIO;
+
 	spi->irq = irq;
 
 	init_completion(&spi->done);
 
 	spi->clk = devm_clk_get_prepared(dev, NULL);
-	if (IS_ERR(spi->clk)) {
-		dev_err(dev, "could not find clk: %ld\n", PTR_ERR(spi->clk));
-		goto error;
-	}
+	if (IS_ERR(spi->clk))
+		return dev_err_probe(dev, PTR_ERR(spi->clk), "could not find clk\n");
 
 	host->max_speed_hz = min_t(unsigned long, A3700_SPI_MAX_SPEED_HZ,
 					clk_get_rate(spi->clk));
@@ -878,23 +869,16 @@ static int a3700_spi_probe(struct platform_device *pdev)
 
 	ret = devm_request_irq(dev, spi->irq, a3700_spi_interrupt, 0,
 			       dev_name(dev), host);
-	if (ret) {
-		dev_err(dev, "could not request IRQ: %d\n", ret);
-		goto error;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "could not request IRQ\n");
 
 	ret = devm_spi_register_controller(dev, host);
 	if (ret) {
 		dev_err(dev, "Failed to register host\n");
-		goto error;
+		return ret;
 	}
 
 	return 0;
-
-error:
-	spi_controller_put(host);
-out:
-	return ret;
 }
 
 static struct platform_driver a3700_spi_driver = {
diff --git a/drivers/spi/spi-aspeed-smc.c b/drivers/spi/spi-aspeed-smc.c
index c21323e07d3ca..c20a33734f5c9 100644
--- a/drivers/spi/spi-aspeed-smc.c
+++ b/drivers/spi/spi-aspeed-smc.c
@@ -697,7 +697,7 @@ static int aspeed_spi_dirmap_create(struct spi_mem_dirmap_desc *desc)
 {
 	struct aspeed_spi *aspi = spi_controller_get_devdata(desc->mem->spi->controller);
 	struct aspeed_spi_chip *chip = &aspi->chips[spi_get_chipselect(desc->mem->spi, 0)];
-	struct spi_mem_op *op = &desc->info.op_tmpl;
+	struct spi_mem_op *op = desc->info.op_tmpl;
 	u32 ctl_val;
 	int ret = 0;
 
@@ -769,7 +769,7 @@ static ssize_t aspeed_spi_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	if (chip->ahb_window_size < offset + len || chip->force_user_mode) {
 		int ret;
 
-		ret = aspeed_spi_read_user(chip, &desc->info.op_tmpl, offset, len, buf);
+		ret = aspeed_spi_read_user(chip, desc->info.op_tmpl, offset, len, buf);
 		if (ret < 0)
 			return ret;
 	} else {
diff --git a/drivers/spi/spi-at91-usart.c b/drivers/spi/spi-at91-usart.c
index 79edc1cd13c0f..77cad41182021 100644
--- a/drivers/spi/spi-at91-usart.c
+++ b/drivers/spi/spi-at91-usart.c
@@ -16,7 +16,6 @@
 #include <linux/gpio/consumer.h>
 #include <linux/pinctrl/consumer.h>
 #include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
 
 #include <linux/spi/spi.h>
 
@@ -496,14 +495,13 @@ static int at91_usart_spi_probe(struct platform_device *pdev)
 	if (IS_ERR(clk))
 		return PTR_ERR(clk);
 
-	ret = -ENOMEM;
-	controller = spi_alloc_host(&pdev->dev, sizeof(*aus));
+	controller = devm_spi_alloc_host(&pdev->dev, sizeof(*aus));
 	if (!controller)
-		goto at91_usart_spi_probe_fail;
+		return -ENOMEM;
 
 	ret = at91_usart_gpio_setup(pdev);
 	if (ret)
-		goto at91_usart_spi_probe_fail;
+		return ret;
 
 	controller->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP | SPI_CS_HIGH;
 	controller->dev.of_node = pdev->dev.parent->of_node;
@@ -525,10 +523,8 @@ static int at91_usart_spi_probe(struct platform_device *pdev)
 
 	aus->dev = &pdev->dev;
 	aus->regs = devm_ioremap_resource(&pdev->dev, regs);
-	if (IS_ERR(aus->regs)) {
-		ret = PTR_ERR(aus->regs);
-		goto at91_usart_spi_probe_fail;
-	}
+	if (IS_ERR(aus->regs))
+		return PTR_ERR(aus->regs);
 
 	aus->irq = irq;
 	aus->clk = clk;
@@ -536,11 +532,11 @@ static int at91_usart_spi_probe(struct platform_device *pdev)
 	ret = devm_request_irq(&pdev->dev, irq, at91_usart_spi_interrupt, 0,
 			       dev_name(&pdev->dev), controller);
 	if (ret)
-		goto at91_usart_spi_probe_fail;
+		return ret;
 
 	ret = clk_prepare_enable(clk);
 	if (ret)
-		goto at91_usart_spi_probe_fail;
+		return ret;
 
 	aus->spi_clk = clk_get_rate(clk);
 	at91_usart_spi_init(aus);
@@ -571,43 +567,22 @@ static int at91_usart_spi_probe(struct platform_device *pdev)
 	at91_usart_spi_release_dma(controller);
 at91_usart_fail_dma:
 	clk_disable_unprepare(clk);
-at91_usart_spi_probe_fail:
-	spi_controller_put(controller);
-	return ret;
-}
-
-__maybe_unused static int at91_usart_spi_runtime_suspend(struct device *dev)
-{
-	struct spi_controller *ctlr = dev_get_drvdata(dev);
-	struct at91_usart_spi *aus = spi_controller_get_devdata(ctlr);
-
-	clk_disable_unprepare(aus->clk);
-	pinctrl_pm_select_sleep_state(dev);
-
-	return 0;
-}
-
-__maybe_unused static int at91_usart_spi_runtime_resume(struct device *dev)
-{
-	struct spi_controller *ctrl = dev_get_drvdata(dev);
-	struct at91_usart_spi *aus = spi_controller_get_devdata(ctrl);
-
-	pinctrl_pm_select_default_state(dev);
 
-	return clk_prepare_enable(aus->clk);
+	return ret;
 }
 
 __maybe_unused static int at91_usart_spi_suspend(struct device *dev)
 {
 	struct spi_controller *ctrl = dev_get_drvdata(dev);
+	struct at91_usart_spi *aus = spi_controller_get_devdata(ctrl);
 	int ret;
 
 	ret = spi_controller_suspend(ctrl);
 	if (ret)
 		return ret;
 
-	if (!pm_runtime_suspended(dev))
-		at91_usart_spi_runtime_suspend(dev);
+	clk_disable_unprepare(aus->clk);
+	pinctrl_pm_select_sleep_state(dev);
 
 	return 0;
 }
@@ -618,11 +593,11 @@ __maybe_unused static int at91_usart_spi_resume(struct device *dev)
 	struct at91_usart_spi *aus = spi_controller_get_devdata(ctrl);
 	int ret;
 
-	if (!pm_runtime_suspended(dev)) {
-		ret = at91_usart_spi_runtime_resume(dev);
-		if (ret)
-			return ret;
-	}
+	pinctrl_pm_select_default_state(dev);
+
+	ret = clk_prepare_enable(aus->clk);
+	if (ret)
+		return ret;
 
 	at91_usart_spi_init(aus);
 
@@ -634,20 +609,14 @@ static void at91_usart_spi_remove(struct platform_device *pdev)
 	struct spi_controller *ctlr = platform_get_drvdata(pdev);
 	struct at91_usart_spi *aus = spi_controller_get_devdata(ctlr);
 
-	spi_controller_get(ctlr);
-
 	spi_unregister_controller(ctlr);
 
 	at91_usart_spi_release_dma(ctlr);
 	clk_disable_unprepare(aus->clk);
-
-	spi_controller_put(ctlr);
 }
 
 static const struct dev_pm_ops at91_usart_spi_pm_ops = {
 	SET_SYSTEM_SLEEP_PM_OPS(at91_usart_spi_suspend, at91_usart_spi_resume)
-	SET_RUNTIME_PM_OPS(at91_usart_spi_runtime_suspend,
-			   at91_usart_spi_runtime_resume, NULL)
 };
 
 static struct platform_driver at91_usart_spi_driver = {
diff --git a/drivers/spi/spi-atcspi200.c b/drivers/spi/spi-atcspi200.c
index 3832d9db3cbf8..6d4b6aeb3f5b1 100644
--- a/drivers/spi/spi-atcspi200.c
+++ b/drivers/spi/spi-atcspi200.c
@@ -550,7 +550,7 @@ static int atcspi_probe(struct platform_device *pdev)
 	struct resource *mem_res;
 	int ret;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*spi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*spi));
 	if (!host)
 		return -ENOMEM;
 
@@ -559,28 +559,24 @@ static int atcspi_probe(struct platform_device *pdev)
 	spi->dev = &pdev->dev;
 	dev_set_drvdata(&pdev->dev, host);
 
-	mutex_init(&spi->mutex_lock);
+	ret = devm_mutex_init(&pdev->dev, &spi->mutex_lock);
+	if (ret)
+		return ret;
 
 	ret = atcspi_init_resources(pdev, spi, &mem_res);
 	if (ret)
-		goto free_controller;
+		return ret;
 
 	ret = atcspi_enable_clk(spi);
 	if (ret)
-		goto free_controller;
+		return ret;
 
 	atcspi_init_controller(pdev, spi, host, mem_res);
 
 	ret = atcspi_setup(spi);
 	if (ret)
-		goto free_controller;
+		return ret;
 
-	ret = devm_spi_register_controller(&pdev->dev, host);
-	if (ret) {
-		dev_err_probe(spi->dev, ret,
-			      "Failed to register SPI controller\n");
-		goto free_controller;
-	}
 	spi->use_dma = false;
 	if (ATCSPI_DMA_SUPPORT) {
 		ret = atcspi_configure_dma(spi);
@@ -591,12 +587,12 @@ static int atcspi_probe(struct platform_device *pdev)
 			spi->use_dma = true;
 	}
 
-	return 0;
+	ret = devm_spi_register_controller(&pdev->dev, host);
+	if (ret)
+		return dev_err_probe(spi->dev, ret,
+				     "Failed to register SPI controller\n");
 
-free_controller:
-	mutex_destroy(&spi->mutex_lock);
-	spi_controller_put(host);
-	return ret;
+	return 0;
 }
 
 static int atcspi_suspend(struct device *dev)
diff --git a/drivers/spi/spi-atmel.c b/drivers/spi/spi-atmel.c
index 42db85d7ff8e1..25aa294631c83 100644
--- a/drivers/spi/spi-atmel.c
+++ b/drivers/spi/spi-atmel.c
@@ -1528,7 +1528,7 @@ static int atmel_spi_probe(struct platform_device *pdev)
 		return PTR_ERR(clk);
 
 	/* setup spi core then atmel-specific driver state */
-	host = spi_alloc_host(&pdev->dev, sizeof(*as));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*as));
 	if (!host)
 		return -ENOMEM;
 
@@ -1555,18 +1555,15 @@ static int atmel_spi_probe(struct platform_device *pdev)
 
 	as->pdev = pdev;
 	as->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &regs);
-	if (IS_ERR(as->regs)) {
-		ret = PTR_ERR(as->regs);
-		goto out_unmap_regs;
-	}
+	if (IS_ERR(as->regs))
+		return PTR_ERR(as->regs);
+
 	as->phybase = regs->start;
 	as->irq = irq;
 	as->clk = clk;
 	as->gclk = devm_clk_get_optional(&pdev->dev, "spi_gclk");
-	if (IS_ERR(as->gclk)) {
-		ret = PTR_ERR(as->gclk);
-		goto out_unmap_regs;
-	}
+	if (IS_ERR(as->gclk))
+		return PTR_ERR(as->gclk);
 
 	init_completion(&as->xfer_completion);
 
@@ -1576,11 +1573,10 @@ static int atmel_spi_probe(struct platform_device *pdev)
 	as->use_pdc = false;
 	if (as->caps.has_dma_support) {
 		ret = atmel_spi_configure_dma(host, as);
-		if (ret == 0) {
+		if (ret == 0)
 			as->use_dma = true;
-		} else if (ret == -EPROBE_DEFER) {
-			goto out_unmap_regs;
-		}
+		else if (ret == -EPROBE_DEFER)
+			return ret;
 	} else if (as->caps.has_pdc_support) {
 		as->use_pdc = true;
 	}
@@ -1620,12 +1616,12 @@ static int atmel_spi_probe(struct platform_device *pdev)
 					0, dev_name(&pdev->dev), host);
 	}
 	if (ret)
-		goto out_unmap_regs;
+		return ret;
 
 	/* Initialize the hardware */
 	ret = clk_prepare_enable(clk);
 	if (ret)
-		goto out_free_irq;
+		return ret;
 
 	/*
 	 * In cases where the peripheral clock is higher,the FLEX_SPI_CSRx.SCBR
@@ -1677,9 +1673,7 @@ static int atmel_spi_probe(struct platform_device *pdev)
 	clk_disable_unprepare(as->gclk);
 out_disable_clk:
 	clk_disable_unprepare(clk);
-out_free_irq:
-out_unmap_regs:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -1688,8 +1682,6 @@ static void atmel_spi_remove(struct platform_device *pdev)
 	struct spi_controller	*host = platform_get_drvdata(pdev);
 	struct atmel_spi	*as = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	pm_runtime_get_sync(&pdev->dev);
 
 	spi_unregister_controller(host);
@@ -1720,8 +1712,6 @@ static void atmel_spi_remove(struct platform_device *pdev)
 
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	spi_controller_put(host);
 }
 
 static int atmel_spi_runtime_suspend(struct device *dev)
diff --git a/drivers/spi/spi-bcm63xx-hsspi.c b/drivers/spi/spi-bcm63xx-hsspi.c
index e935e8ab9cfd6..58012e1b5ae74 100644
--- a/drivers/spi/spi-bcm63xx-hsspi.c
+++ b/drivers/spi/spi-bcm63xx-hsspi.c
@@ -783,7 +783,7 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev)
 					     "failed get pll clk rate\n");
 	}
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*bs));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*bs));
 	if (!host)
 		return dev_err_probe(dev, -ENOMEM, "alloc host no mem\n");
 
@@ -796,10 +796,8 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev)
 	bs->fifo = (u8 __iomem *)(bs->regs + HSSPI_FIFO_REG(0));
 	bs->wait_mode = HSSPI_WAIT_MODE_POLLING;
 	bs->prepend_buf = devm_kzalloc(dev, HSSPI_BUFFER_LEN, GFP_KERNEL);
-	if (!bs->prepend_buf) {
-		ret = -ENOMEM;
-		goto out_put_host;
-	}
+	if (!bs->prepend_buf)
+		return -ENOMEM;
 
 	mutex_init(&bs->bus_mutex);
 	mutex_init(&bs->msg_mutex);
@@ -845,7 +843,7 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev)
 				       pdev->name, bs);
 
 		if (ret)
-			goto out_put_host;
+			return ret;
 	}
 
 	pm_runtime_enable(&pdev->dev);
@@ -869,8 +867,7 @@ static int bcm63xx_hsspi_probe(struct platform_device *pdev)
 	sysfs_remove_group(&pdev->dev.kobj, &bcm63xx_hsspi_group);
 out_pm_disable:
 	pm_runtime_disable(&pdev->dev);
-out_put_host:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -880,15 +877,11 @@ static void bcm63xx_hsspi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct bcm63xx_hsspi *bs = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	/* reset the hardware and block queue progress */
 	__raw_writel(0, bs->regs + HSSPI_INT_MASK_REG);
 	sysfs_remove_group(&pdev->dev.kobj, &bcm63xx_hsspi_group);
-
-	spi_controller_put(host);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c
index 40cd7efc4b54e..f8cfe535b2a3c 100644
--- a/drivers/spi/spi-bcm63xx.c
+++ b/drivers/spi/spi-bcm63xx.c
@@ -541,11 +541,9 @@ static int bcm63xx_spi_probe(struct platform_device *pdev)
 	if (IS_ERR(reset))
 		return PTR_ERR(reset);
 
-	host = spi_alloc_host(dev, sizeof(*bs));
-	if (!host) {
-		dev_err(dev, "out of memory\n");
+	host = devm_spi_alloc_host(dev, sizeof(*bs));
+	if (!host)
 		return -ENOMEM;
-	}
 
 	bs = spi_controller_get_devdata(host);
 	init_completion(&bs->done);
@@ -554,10 +552,8 @@ static int bcm63xx_spi_probe(struct platform_device *pdev)
 	bs->pdev = pdev;
 
 	bs->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
-	if (IS_ERR(bs->regs)) {
-		ret = PTR_ERR(bs->regs);
-		goto out_err;
-	}
+	if (IS_ERR(bs->regs))
+		return PTR_ERR(bs->regs);
 
 	bs->irq = irq;
 	bs->clk = clk;
@@ -568,7 +564,7 @@ static int bcm63xx_spi_probe(struct platform_device *pdev)
 			       pdev->name, host);
 	if (ret) {
 		dev_err(dev, "unable to request irq\n");
-		goto out_err;
+		return ret;
 	}
 
 	host->bus_num = bus_num;
@@ -587,7 +583,7 @@ static int bcm63xx_spi_probe(struct platform_device *pdev)
 	/* Initialize hardware */
 	ret = clk_prepare_enable(bs->clk);
 	if (ret)
-		goto out_err;
+		return ret;
 
 	ret = reset_control_reset(reset);
 	if (ret) {
@@ -615,8 +611,7 @@ static int bcm63xx_spi_probe(struct platform_device *pdev)
 
 out_clk_disable:
 	clk_disable_unprepare(clk);
-out_err:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -625,8 +620,6 @@ static void bcm63xx_spi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct bcm63xx_spi *bs = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	/* reset spi block */
@@ -634,8 +627,6 @@ static void bcm63xx_spi_remove(struct platform_device *pdev)
 
 	/* HW shutdown */
 	clk_disable_unprepare(bs->clk);
-
-	spi_controller_put(host);
 }
 
 static int bcm63xx_spi_suspend(struct device *dev)
diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c
index 057381e56a7fd..aaba1a3ad5778 100644
--- a/drivers/spi/spi-cadence-quadspi.c
+++ b/drivers/spi/spi-cadence-quadspi.c
@@ -1478,7 +1478,6 @@ static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
 	int ret;
 	struct cqspi_st *cqspi = spi_controller_get_devdata(mem->spi->controller);
 	struct device *dev = &cqspi->pdev->dev;
-	const struct cqspi_driver_platdata *ddata = of_device_get_match_data(dev);
 
 	if (refcount_read(&cqspi->inflight_ops) == 0)
 		return -ENODEV;
@@ -1494,18 +1493,15 @@ static int cqspi_exec_mem_op(struct spi_mem *mem, const struct spi_mem_op *op)
 		return -EBUSY;
 	}
 
-	if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) {
-		ret = pm_runtime_resume_and_get(dev);
-		if (ret) {
-			dev_err(&mem->spi->dev, "resume failed with %d\n", ret);
-			goto dec_inflight_refcount;
-		}
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret) {
+		dev_err(&mem->spi->dev, "resume failed with %d\n", ret);
+		goto dec_inflight_refcount;
 	}
 
 	ret = cqspi_mem_process(mem, op);
 
-	if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM)))
-		pm_runtime_put_autosuspend(dev);
+	pm_runtime_put_autosuspend(dev);
 
 	if (ret)
 		dev_err(&mem->spi->dev, "operation failed with %d\n", ret);
@@ -1957,13 +1953,11 @@ static int cqspi_probe(struct platform_device *pdev)
 	cqspi->current_cs = -1;
 	cqspi->sclk = 0;
 
-	if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) {
-		pm_runtime_set_autosuspend_delay(dev, CQSPI_AUTOSUSPEND_TIMEOUT);
-		pm_runtime_use_autosuspend(dev);
-		pm_runtime_get_noresume(dev);
-		pm_runtime_set_active(dev);
-		pm_runtime_enable(dev);
-	}
+	pm_runtime_set_autosuspend_delay(dev, CQSPI_AUTOSUSPEND_TIMEOUT);
+	pm_runtime_use_autosuspend(dev);
+	pm_runtime_get_noresume(dev);
+	pm_runtime_set_active(dev);
+	pm_runtime_enable(dev);
 
 	host->num_chipselect = cqspi->num_chipselect;
 
@@ -1993,12 +1987,11 @@ static int cqspi_probe(struct platform_device *pdev)
 	if (cqspi->rx_chan)
 		dma_release_channel(cqspi->rx_chan);
 disable_rpm:
-	if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) {
-		pm_runtime_disable(dev);
-		pm_runtime_set_suspended(dev);
-		pm_runtime_put_noidle(dev);
-		pm_runtime_dont_use_autosuspend(dev);
-	}
+	pm_runtime_disable(dev);
+	pm_runtime_set_suspended(dev);
+	pm_runtime_put_noidle(dev);
+	pm_runtime_dont_use_autosuspend(dev);
+
 	cqspi_controller_enable(cqspi, 0);
 disable_clks:
 	clk_bulk_disable_unprepare(CLK_QSPI_NUM, cqspi->clks);
@@ -2008,13 +2001,10 @@ static int cqspi_probe(struct platform_device *pdev)
 
 static void cqspi_remove(struct platform_device *pdev)
 {
-	const struct cqspi_driver_platdata *ddata;
 	struct cqspi_st *cqspi = platform_get_drvdata(pdev);
-	struct device *dev = &pdev->dev;
+	const struct cqspi_driver_platdata *ddata = cqspi->ddata;
 	int ret = 0;
 
-	ddata = of_device_get_match_data(dev);
-
 	spi_unregister_controller(cqspi->host);
 
 	refcount_set(&cqspi->refcount, 0);
@@ -2033,12 +2023,10 @@ static void cqspi_remove(struct platform_device *pdev)
 		clk_bulk_disable_unprepare(CLK_QSPI_NUM, cqspi->clks);
 	}
 
-	if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) {
-		pm_runtime_disable(&pdev->dev);
-		pm_runtime_set_suspended(&pdev->dev);
-		pm_runtime_put_noidle(&pdev->dev);
-		pm_runtime_dont_use_autosuspend(&pdev->dev);
-	}
+	pm_runtime_disable(&pdev->dev);
+	pm_runtime_set_suspended(&pdev->dev);
+	pm_runtime_put_noidle(&pdev->dev);
+	pm_runtime_dont_use_autosuspend(&pdev->dev);
 }
 
 static int cqspi_runtime_suspend(struct device *dev)
diff --git a/drivers/spi/spi-cadence-xspi.c b/drivers/spi/spi-cadence-xspi.c
index 895b4b3276a56..c45b29c043bf3 100644
--- a/drivers/spi/spi-cadence-xspi.c
+++ b/drivers/spi/spi-cadence-xspi.c
@@ -22,6 +22,10 @@
 #include <linux/bitrev.h>
 #include <linux/util_macros.h>
 
+#ifndef CONFIG_64BIT
+#include <linux/io-64-nonatomic-lo-hi.h>
+#endif
+
 #define CDNS_XSPI_MAGIC_NUM_VALUE	0x6522
 #define CDNS_XSPI_MAX_BANKS		8
 #define CDNS_XSPI_NAME			"cadence-xspi"
@@ -453,8 +457,7 @@ static bool cdns_mrvl_xspi_setup_clock(struct cdns_xspi_dev *cdns_xspi,
 		writel(clk_reg,
 		       cdns_xspi->auxbase + MRVL_XSPI_CLK_CTRL_AUX_REG);
 		clk_reg = FIELD_PREP(MRVL_XSPI_CLK_DIV, i);
-		clk_reg &= ~MRVL_XSPI_CLK_DIV;
-		clk_reg |= FIELD_PREP(MRVL_XSPI_CLK_DIV, i);
+		FIELD_MODIFY(MRVL_XSPI_CLK_DIV, &clk_reg, i);
 		clk_reg |= MRVL_XSPI_CLK_ENABLE;
 		clk_reg |= MRVL_XSPI_IRQ_ENABLE;
 		update_clk = true;
diff --git a/drivers/spi/spi-cadence.c b/drivers/spi/spi-cadence.c
index 891e2ba369584..9b4e5b7013ae8 100644
--- a/drivers/spi/spi-cadence.c
+++ b/drivers/spi/spi-cadence.c
@@ -635,7 +635,7 @@ static int cdns_target_abort(struct spi_controller *ctlr)
  */
 static int cdns_spi_probe(struct platform_device *pdev)
 {
-	int ret = 0, irq;
+	int ret, irq;
 	struct spi_controller *ctlr;
 	struct cdns_spi *xspi;
 	u32 num_cs;
@@ -643,9 +643,9 @@ static int cdns_spi_probe(struct platform_device *pdev)
 
 	target = of_property_read_bool(pdev->dev.of_node, "spi-slave");
 	if (target)
-		ctlr = spi_alloc_target(&pdev->dev, sizeof(*xspi));
+		ctlr = devm_spi_alloc_target(&pdev->dev, sizeof(*xspi));
 	else
-		ctlr = spi_alloc_host(&pdev->dev, sizeof(*xspi));
+		ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(*xspi));
 
 	if (!ctlr)
 		return -ENOMEM;
@@ -654,23 +654,19 @@ static int cdns_spi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, ctlr);
 
 	xspi->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(xspi->regs)) {
-		ret = PTR_ERR(xspi->regs);
-		goto remove_ctlr;
-	}
+	if (IS_ERR(xspi->regs))
+		return PTR_ERR(xspi->regs);
 
 	xspi->pclk = devm_clk_get_enabled(&pdev->dev, "pclk");
 	if (IS_ERR(xspi->pclk)) {
 		dev_err(&pdev->dev, "pclk clock not found.\n");
-		ret = PTR_ERR(xspi->pclk);
-		goto remove_ctlr;
+		return PTR_ERR(xspi->pclk);
 	}
 
 	xspi->rstc = devm_reset_control_get_optional_exclusive(&pdev->dev, "spi");
 	if (IS_ERR(xspi->rstc)) {
-		ret = dev_err_probe(&pdev->dev, PTR_ERR(xspi->rstc),
-				    "Cannot get SPI reset.\n");
-		goto remove_ctlr;
+		return dev_err_probe(&pdev->dev, PTR_ERR(xspi->rstc),
+				     "Cannot get SPI reset.\n");
 	}
 
 	reset_control_assert(xspi->rstc);
@@ -679,8 +675,7 @@ static int cdns_spi_probe(struct platform_device *pdev)
 	xspi->ref_clk = devm_clk_get_enabled(&pdev->dev, "ref_clk");
 	if (IS_ERR(xspi->ref_clk)) {
 		dev_err(&pdev->dev, "ref_clk clock not found.\n");
-		ret = PTR_ERR(xspi->ref_clk);
-		goto remove_ctlr;
+		return PTR_ERR(xspi->ref_clk);
 	}
 
 	if (!spi_controller_is_target(ctlr)) {
@@ -710,7 +705,7 @@ static int cdns_spi_probe(struct platform_device *pdev)
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
 		ret = irq;
-		goto clk_dis_all;
+		goto err_disable_rpm;
 	}
 
 	ret = devm_request_irq(&pdev->dev, irq, cdns_spi_irq,
@@ -718,7 +713,7 @@ static int cdns_spi_probe(struct platform_device *pdev)
 	if (ret != 0) {
 		ret = -ENXIO;
 		dev_err(&pdev->dev, "request_irq failed\n");
-		goto clk_dis_all;
+		goto err_disable_rpm;
 	}
 
 	ctlr->use_gpio_descriptors = true;
@@ -748,23 +743,22 @@ static int cdns_spi_probe(struct platform_device *pdev)
 	ret = spi_register_controller(ctlr);
 	if (ret) {
 		dev_err(&pdev->dev, "spi_register_controller failed\n");
-		goto clk_dis_all;
+		goto err_disable_rpm;
 	}
 
 	if (!spi_controller_is_target(ctlr))
 		pm_runtime_put_autosuspend(&pdev->dev);
 
-	return ret;
+	return 0;
 
-clk_dis_all:
+err_disable_rpm:
 	if (!spi_controller_is_target(ctlr)) {
 		pm_runtime_disable(&pdev->dev);
 		pm_runtime_set_suspended(&pdev->dev);
 		pm_runtime_put_noidle(&pdev->dev);
 		pm_runtime_dont_use_autosuspend(&pdev->dev);
 	}
-remove_ctlr:
-	spi_controller_put(ctlr);
+
 	return ret;
 }
 
@@ -785,8 +779,6 @@ static void cdns_spi_remove(struct platform_device *pdev)
 	if (!spi_controller_is_target(ctlr))
 		ret = pm_runtime_get_sync(&pdev->dev);
 
-	spi_controller_get(ctlr);
-
 	spi_unregister_controller(ctlr);
 
 	if (ret >= 0)
@@ -798,8 +790,6 @@ static void cdns_spi_remove(struct platform_device *pdev)
 		pm_runtime_put_noidle(&pdev->dev);
 		pm_runtime_dont_use_autosuspend(&pdev->dev);
 	}
-
-	spi_controller_put(ctlr);
 }
 
 /**
diff --git a/drivers/spi/spi-cavium-octeon.c b/drivers/spi/spi-cavium-octeon.c
index b95bfa6a30132..28c922c720685 100644
--- a/drivers/spi/spi-cavium-octeon.c
+++ b/drivers/spi/spi-cavium-octeon.c
@@ -23,17 +23,15 @@ static int octeon_spi_probe(struct platform_device *pdev)
 	struct octeon_spi *p;
 	int err = -ENOENT;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(struct octeon_spi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(struct octeon_spi));
 	if (!host)
 		return -ENOMEM;
 	p = spi_controller_get_devdata(host);
 	platform_set_drvdata(pdev, host);
 
 	reg_base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(reg_base)) {
-		err = PTR_ERR(reg_base);
-		goto fail;
-	}
+	if (IS_ERR(reg_base))
+		return PTR_ERR(reg_base);
 
 	p->register_base = reg_base;
 	p->sys_freq = octeon_get_io_clock_rate();
@@ -57,15 +55,12 @@ static int octeon_spi_probe(struct platform_device *pdev)
 	err = spi_register_controller(host);
 	if (err) {
 		dev_err(&pdev->dev, "register host failed: %d\n", err);
-		goto fail;
+		return err;
 	}
 
 	dev_info(&pdev->dev, "OCTEON SPI bus driver\n");
 
 	return 0;
-fail:
-	spi_controller_put(host);
-	return err;
 }
 
 static void octeon_spi_remove(struct platform_device *pdev)
@@ -73,14 +68,10 @@ static void octeon_spi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct octeon_spi *p = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	/* Clear the CSENA* and put everything in a known state. */
 	writeq(0, p->register_base + OCTEON_SPI_CFG(p));
-
-	spi_controller_put(host);
 }
 
 static const struct of_device_id octeon_spi_match[] = {
diff --git a/drivers/spi/spi-cavium-thunderx.c b/drivers/spi/spi-cavium-thunderx.c
index f1a9aa696c874..529b4066c9220 100644
--- a/drivers/spi/spi-cavium-thunderx.c
+++ b/drivers/spi/spi-cavium-thunderx.c
@@ -24,7 +24,7 @@ static int thunderx_spi_probe(struct pci_dev *pdev,
 	struct octeon_spi *p;
 	int ret;
 
-	host = spi_alloc_host(dev, sizeof(struct octeon_spi));
+	host = devm_spi_alloc_host(dev, sizeof(struct octeon_spi));
 	if (!host)
 		return -ENOMEM;
 
@@ -32,17 +32,15 @@ static int thunderx_spi_probe(struct pci_dev *pdev,
 
 	ret = pcim_enable_device(pdev);
 	if (ret)
-		goto error;
+		return ret;
 
 	ret = pcim_request_all_regions(pdev, DRV_NAME);
 	if (ret)
-		goto error;
+		return ret;
 
 	p->register_base = pcim_iomap(pdev, 0, pci_resource_len(pdev, 0));
-	if (!p->register_base) {
-		ret = -EINVAL;
-		goto error;
-	}
+	if (!p->register_base)
+		return -EINVAL;
 
 	p->regs.config = 0x1000;
 	p->regs.status = 0x1008;
@@ -50,10 +48,8 @@ static int thunderx_spi_probe(struct pci_dev *pdev,
 	p->regs.data = 0x1080;
 
 	p->clk = devm_clk_get_enabled(dev, NULL);
-	if (IS_ERR(p->clk)) {
-		ret = PTR_ERR(p->clk);
-		goto error;
-	}
+	if (IS_ERR(p->clk))
+		return PTR_ERR(p->clk);
 
 	p->sys_freq = clk_get_rate(p->clk);
 	if (!p->sys_freq)
@@ -70,15 +66,7 @@ static int thunderx_spi_probe(struct pci_dev *pdev,
 
 	pci_set_drvdata(pdev, host);
 
-	ret = spi_register_controller(host);
-	if (ret)
-		goto error;
-
-	return 0;
-
-error:
-	spi_controller_put(host);
-	return ret;
+	return spi_register_controller(host);
 }
 
 static void thunderx_spi_remove(struct pci_dev *pdev)
@@ -90,14 +78,10 @@ static void thunderx_spi_remove(struct pci_dev *pdev)
 	if (!p)
 		return;
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	/* Put everything in a known state. */
 	writeq(0, p->register_base + OCTEON_SPI_CFG(p));
-
-	spi_controller_put(host);
 }
 
 static const struct pci_device_id thunderx_spi_pci_id_table[] = {
diff --git a/drivers/spi/spi-clps711x.c b/drivers/spi/spi-clps711x.c
index d6458e59d41b6..382c3c81c4b96 100644
--- a/drivers/spi/spi-clps711x.c
+++ b/drivers/spi/spi-clps711x.c
@@ -99,7 +99,7 @@ static int spi_clps711x_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*hw));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*hw));
 	if (!host)
 		return -ENOMEM;
 
@@ -113,22 +113,16 @@ static int spi_clps711x_probe(struct platform_device *pdev)
 	hw = spi_controller_get_devdata(host);
 
 	hw->spi_clk = devm_clk_get(&pdev->dev, NULL);
-	if (IS_ERR(hw->spi_clk)) {
-		ret = PTR_ERR(hw->spi_clk);
-		goto err_out;
-	}
+	if (IS_ERR(hw->spi_clk))
+		return PTR_ERR(hw->spi_clk);
 
 	hw->syscon = syscon_regmap_lookup_by_phandle(np, "syscon");
-	if (IS_ERR(hw->syscon)) {
-		ret = PTR_ERR(hw->syscon);
-		goto err_out;
-	}
+	if (IS_ERR(hw->syscon))
+		return PTR_ERR(hw->syscon);
 
 	hw->syncio = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(hw->syncio)) {
-		ret = PTR_ERR(hw->syncio);
-		goto err_out;
-	}
+	if (IS_ERR(hw->syncio))
+		return PTR_ERR(hw->syncio);
 
 	/* Disable extended mode due hardware problems */
 	regmap_update_bits(hw->syscon, SYSCON_OFFSET, SYSCON3_ADCCON, 0);
@@ -139,16 +133,9 @@ static int spi_clps711x_probe(struct platform_device *pdev)
 	ret = devm_request_irq(&pdev->dev, irq, spi_clps711x_isr, 0,
 			       dev_name(&pdev->dev), host);
 	if (ret)
-		goto err_out;
+		return ret;
 
-	ret = devm_spi_register_controller(&pdev->dev, host);
-	if (!ret)
-		return 0;
-
-err_out:
-	spi_controller_put(host);
-
-	return ret;
+	return devm_spi_register_controller(&pdev->dev, host);
 }
 
 static const struct of_device_id clps711x_spi_dt_ids[] = {
diff --git a/drivers/spi/spi-coldfire-qspi.c b/drivers/spi/spi-coldfire-qspi.c
index b45f44de85dcd..3b175c1da36b4 100644
--- a/drivers/spi/spi-coldfire-qspi.c
+++ b/drivers/spi/spi-coldfire-qspi.c
@@ -353,39 +353,33 @@ static int mcfqspi_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*mcfqspi));
-	if (host == NULL) {
-		dev_dbg(&pdev->dev, "spi_alloc_host failed\n");
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*mcfqspi));
+	if (host == NULL)
 		return -ENOMEM;
-	}
 
 	mcfqspi = spi_controller_get_devdata(host);
 
 	mcfqspi->iobase = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(mcfqspi->iobase)) {
-		status = PTR_ERR(mcfqspi->iobase);
-		goto fail0;
-	}
+	if (IS_ERR(mcfqspi->iobase))
+		return PTR_ERR(mcfqspi->iobase);
 
 	mcfqspi->irq = platform_get_irq(pdev, 0);
 	if (mcfqspi->irq < 0) {
 		dev_dbg(&pdev->dev, "platform_get_irq failed\n");
-		status = -ENXIO;
-		goto fail0;
+		return -ENXIO;
 	}
 
 	status = devm_request_irq(&pdev->dev, mcfqspi->irq, mcfqspi_irq_handler,
 				0, pdev->name, mcfqspi);
 	if (status) {
 		dev_dbg(&pdev->dev, "request_irq failed\n");
-		goto fail0;
+		return status;
 	}
 
 	mcfqspi->clk = devm_clk_get_enabled(&pdev->dev, "qspi_clk");
 	if (IS_ERR(mcfqspi->clk)) {
 		dev_dbg(&pdev->dev, "clk_get failed\n");
-		status = PTR_ERR(mcfqspi->clk);
-		goto fail0;
+		return PTR_ERR(mcfqspi->clk);
 	}
 
 	host->bus_num = pdata->bus_num;
@@ -395,7 +389,7 @@ static int mcfqspi_probe(struct platform_device *pdev)
 	status = mcfqspi_cs_setup(mcfqspi);
 	if (status) {
 		dev_dbg(&pdev->dev, "error initializing cs_control\n");
-		goto fail0;
+		return status;
 	}
 
 	init_waitqueue_head(&mcfqspi->waitq);
@@ -423,8 +417,6 @@ static int mcfqspi_probe(struct platform_device *pdev)
 fail1:
 	pm_runtime_disable(&pdev->dev);
 	mcfqspi_cs_teardown(mcfqspi);
-fail0:
-	spi_controller_put(host);
 
 	dev_dbg(&pdev->dev, "Coldfire QSPI probe failed\n");
 
@@ -436,8 +428,6 @@ static void mcfqspi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct mcfqspi *mcfqspi = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	pm_runtime_disable(&pdev->dev);
@@ -445,8 +435,6 @@ static void mcfqspi_remove(struct platform_device *pdev)
 	mcfqspi_wr_qmr(mcfqspi, MCFQSPI_QMR_MSTR);
 
 	mcfqspi_cs_teardown(mcfqspi);
-
-	spi_controller_put(host);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/spi/spi-dln2.c b/drivers/spi/spi-dln2.c
index 392f0d05f5082..8333dda7d1e8b 100644
--- a/drivers/spi/spi-dln2.c
+++ b/drivers/spi/spi-dln2.c
@@ -684,7 +684,7 @@ static int dln2_spi_probe(struct platform_device *pdev)
 	struct dln2_platform_data *pdata = dev_get_platdata(&pdev->dev);
 	int ret;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*dln2));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*dln2));
 	if (!host)
 		return -ENOMEM;
 
@@ -693,10 +693,8 @@ static int dln2_spi_probe(struct platform_device *pdev)
 	dln2 = spi_controller_get_devdata(host);
 
 	dln2->buf = devm_kmalloc(&pdev->dev, DLN2_SPI_BUF_SIZE, GFP_KERNEL);
-	if (!dln2->buf) {
-		ret = -ENOMEM;
-		goto exit_free_host;
-	}
+	if (!dln2->buf)
+		return -ENOMEM;
 
 	dln2->host = host;
 	dln2->pdev = pdev;
@@ -709,13 +707,13 @@ static int dln2_spi_probe(struct platform_device *pdev)
 	ret = dln2_spi_enable(dln2, false);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to disable SPI module\n");
-		goto exit_free_host;
+		return ret;
 	}
 
 	ret = dln2_spi_get_cs_num(dln2, &host->num_chipselect);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to get number of CS pins\n");
-		goto exit_free_host;
+		return ret;
 	}
 
 	ret = dln2_spi_get_speed_range(dln2,
@@ -723,20 +721,20 @@ static int dln2_spi_probe(struct platform_device *pdev)
 				       &host->max_speed_hz);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to read bus min/max freqs\n");
-		goto exit_free_host;
+		return ret;
 	}
 
 	ret = dln2_spi_get_supported_frame_sizes(dln2,
 						 &host->bits_per_word_mask);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to read supported frame sizes\n");
-		goto exit_free_host;
+		return ret;
 	}
 
 	ret = dln2_spi_cs_enable_all(dln2, true);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to enable CS pins\n");
-		goto exit_free_host;
+		return ret;
 	}
 
 	host->bus_num = -1;
@@ -749,7 +747,7 @@ static int dln2_spi_probe(struct platform_device *pdev)
 	ret = dln2_spi_enable(dln2, true);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to enable SPI module\n");
-		goto exit_free_host;
+		return ret;
 	}
 
 	pm_runtime_set_autosuspend_delay(&pdev->dev,
@@ -772,8 +770,6 @@ static int dln2_spi_probe(struct platform_device *pdev)
 
 	if (dln2_spi_enable(dln2, false) < 0)
 		dev_err(&pdev->dev, "Failed to disable SPI module\n");
-exit_free_host:
-	spi_controller_put(host);
 
 	return ret;
 }
@@ -783,16 +779,12 @@ static void dln2_spi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct dln2_spi *dln2 = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	pm_runtime_disable(&pdev->dev);
 
 	if (dln2_spi_enable(dln2, false) < 0)
 		dev_err(&pdev->dev, "Failed to disable SPI module\n");
-
-	spi_controller_put(host);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/spi/spi-dw-pci.c b/drivers/spi/spi-dw-pci.c
index 72d9f5bc87f75..7f002d5e5b881 100644
--- a/drivers/spi/spi-dw-pci.c
+++ b/drivers/spi/spi-dw-pci.c
@@ -185,15 +185,15 @@ static const struct pci_device_id dw_spi_pci_ids[] = {
 	 * exclusively used by SCU to communicate with MSIC.
 	 */
 	/* Intel MID platform SPI controller 1 */
-	{ PCI_VDEVICE(INTEL, 0x0800), (kernel_ulong_t)&dw_spi_pci_mid_desc_1},
+	{ PCI_VDEVICE(INTEL, 0x0800), .driver_data = (kernel_ulong_t)&dw_spi_pci_mid_desc_1 },
 	/* Intel MID platform SPI controller 2 */
-	{ PCI_VDEVICE(INTEL, 0x0812), (kernel_ulong_t)&dw_spi_pci_mid_desc_2},
+	{ PCI_VDEVICE(INTEL, 0x0812), .driver_data = (kernel_ulong_t)&dw_spi_pci_mid_desc_2 },
 	/* Intel Elkhart Lake PSE SPI controllers */
-	{ PCI_VDEVICE(INTEL, 0x4b84), (kernel_ulong_t)&dw_spi_pci_ehl_desc},
-	{ PCI_VDEVICE(INTEL, 0x4b85), (kernel_ulong_t)&dw_spi_pci_ehl_desc},
-	{ PCI_VDEVICE(INTEL, 0x4b86), (kernel_ulong_t)&dw_spi_pci_ehl_desc},
-	{ PCI_VDEVICE(INTEL, 0x4b87), (kernel_ulong_t)&dw_spi_pci_ehl_desc},
-	{},
+	{ PCI_VDEVICE(INTEL, 0x4b84), .driver_data = (kernel_ulong_t)&dw_spi_pci_ehl_desc },
+	{ PCI_VDEVICE(INTEL, 0x4b85), .driver_data = (kernel_ulong_t)&dw_spi_pci_ehl_desc },
+	{ PCI_VDEVICE(INTEL, 0x4b86), .driver_data = (kernel_ulong_t)&dw_spi_pci_ehl_desc },
+	{ PCI_VDEVICE(INTEL, 0x4b87), .driver_data = (kernel_ulong_t)&dw_spi_pci_ehl_desc },
+	{ },
 };
 MODULE_DEVICE_TABLE(pci, dw_spi_pci_ids);
 
diff --git a/drivers/spi/spi-ep93xx.c b/drivers/spi/spi-ep93xx.c
index db50018050e54..ea610b0537a93 100644
--- a/drivers/spi/spi-ep93xx.c
+++ b/drivers/spi/spi-ep93xx.c
@@ -582,12 +582,14 @@ static int ep93xx_spi_setup_dma(struct device *dev, struct ep93xx_spi *espi)
 	espi->dma_rx = dma_request_chan(dev, "rx");
 	if (IS_ERR(espi->dma_rx)) {
 		ret = dev_err_probe(dev, PTR_ERR(espi->dma_rx), "rx DMA setup failed");
+		espi->dma_rx = NULL;
 		goto fail_free_page;
 	}
 
 	espi->dma_tx = dma_request_chan(dev, "tx");
 	if (IS_ERR(espi->dma_tx)) {
 		ret = dev_err_probe(dev, PTR_ERR(espi->dma_tx), "tx DMA setup failed");
+		espi->dma_tx = NULL;
 		goto fail_release_rx;
 	}
 
@@ -629,7 +631,7 @@ static int ep93xx_spi_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*espi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*espi));
 	if (!host)
 		return -ENOMEM;
 
@@ -654,8 +656,7 @@ static int ep93xx_spi_probe(struct platform_device *pdev)
 	espi->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(espi->clk)) {
 		dev_err(&pdev->dev, "unable to get spi clock\n");
-		error = PTR_ERR(espi->clk);
-		goto fail_release_host;
+		return PTR_ERR(espi->clk);
 	}
 
 	/*
@@ -666,22 +667,21 @@ static int ep93xx_spi_probe(struct platform_device *pdev)
 	host->min_speed_hz = clk_get_rate(espi->clk) / (254 * 256);
 
 	espi->mmio = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
-	if (IS_ERR(espi->mmio)) {
-		error = PTR_ERR(espi->mmio);
-		goto fail_release_host;
-	}
+	if (IS_ERR(espi->mmio))
+		return PTR_ERR(espi->mmio);
+
 	espi->sspdr_phys = res->start + SSPDR;
 
 	error = devm_request_irq(&pdev->dev, irq, ep93xx_spi_interrupt,
 				0, "ep93xx-spi", host);
 	if (error) {
 		dev_err(&pdev->dev, "failed to request irq\n");
-		goto fail_release_host;
+		return error;
 	}
 
 	error = ep93xx_spi_setup_dma(&pdev->dev, espi);
 	if (error == -EPROBE_DEFER)
-		goto fail_release_host;
+		return error;
 
 	if (error)
 		dev_warn(&pdev->dev, "DMA setup failed. Falling back to PIO\n");
@@ -702,8 +702,6 @@ static int ep93xx_spi_probe(struct platform_device *pdev)
 
 fail_free_dma:
 	ep93xx_spi_release_dma(espi);
-fail_release_host:
-	spi_controller_put(host);
 
 	return error;
 }
@@ -713,13 +711,9 @@ static void ep93xx_spi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct ep93xx_spi *espi = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	ep93xx_spi_release_dma(espi);
-
-	spi_controller_put(host);
 }
 
 static const struct of_device_id ep93xx_spi_of_ids[] = {
diff --git a/drivers/spi/spi-falcon.c b/drivers/spi/spi-falcon.c
index cb15faabd88f2..e00e808eafeee 100644
--- a/drivers/spi/spi-falcon.c
+++ b/drivers/spi/spi-falcon.c
@@ -392,9 +392,8 @@ static int falcon_sflash_probe(struct platform_device *pdev)
 {
 	struct falcon_sflash *priv;
 	struct spi_controller *host;
-	int ret;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*priv));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*priv));
 	if (!host)
 		return -ENOMEM;
 
@@ -406,10 +405,7 @@ static int falcon_sflash_probe(struct platform_device *pdev)
 	host->setup = falcon_sflash_setup;
 	host->transfer_one_message = falcon_sflash_xfer_one;
 
-	ret = devm_spi_register_controller(&pdev->dev, host);
-	if (ret)
-		spi_controller_put(host);
-	return ret;
+	return devm_spi_register_controller(&pdev->dev, host);
 }
 
 static const struct of_device_id falcon_sflash_match[] = {
diff --git a/drivers/spi/spi-fsi.c b/drivers/spi/spi-fsi.c
index f6a75f0184c4c..451cb4cfdb9c9 100644
--- a/drivers/spi/spi-fsi.c
+++ b/drivers/spi/spi-fsi.c
@@ -554,7 +554,7 @@ static int fsi_spi_probe(struct fsi_device *fsi)
 		if (of_property_read_u32(np, "reg", &base))
 			continue;
 
-		ctlr = spi_alloc_host(dev, sizeof(*ctx));
+		ctlr = devm_spi_alloc_host(dev, sizeof(*ctx));
 		if (!ctlr)
 			break;
 
@@ -571,9 +571,9 @@ static int fsi_spi_probe(struct fsi_device *fsi)
 
 		rc = devm_spi_register_controller(dev, ctlr);
 		if (rc)
-			spi_controller_put(ctlr);
-		else
-			num_controllers_registered++;
+			continue;
+
+		num_controllers_registered++;
 	}
 
 	if (!num_controllers_registered)
diff --git a/drivers/spi/spi-fsl-espi.c b/drivers/spi/spi-fsl-espi.c
index 45b9974ae9111..f560bd537f7d7 100644
--- a/drivers/spi/spi-fsl-espi.c
+++ b/drivers/spi/spi-fsl-espi.c
@@ -667,7 +667,7 @@ static int fsl_espi_probe(struct device *dev, struct resource *mem,
 	struct fsl_espi *espi;
 	int ret;
 
-	host = spi_alloc_host(dev, sizeof(struct fsl_espi));
+	host = devm_spi_alloc_host(dev, sizeof(struct fsl_espi));
 	if (!host)
 		return -ENOMEM;
 
@@ -690,8 +690,7 @@ static int fsl_espi_probe(struct device *dev, struct resource *mem,
 	espi->spibrg = fsl_get_sys_freq();
 	if (espi->spibrg == -1) {
 		dev_err(dev, "Can't get sys frequency!\n");
-		ret = -EINVAL;
-		goto err_probe;
+		return -EINVAL;
 	}
 	/* determined by clock divider fields DIV16/PM in register SPMODEx */
 	host->min_speed_hz = DIV_ROUND_UP(espi->spibrg, 4 * 16 * 16);
@@ -700,15 +699,13 @@ static int fsl_espi_probe(struct device *dev, struct resource *mem,
 	init_completion(&espi->done);
 
 	espi->reg_base = devm_ioremap_resource(dev, mem);
-	if (IS_ERR(espi->reg_base)) {
-		ret = PTR_ERR(espi->reg_base);
-		goto err_probe;
-	}
+	if (IS_ERR(espi->reg_base))
+		return PTR_ERR(espi->reg_base);
 
 	/* Register for SPI Interrupt */
 	ret = devm_request_irq(dev, irq, fsl_espi_irq, 0, "fsl_espi", espi);
 	if (ret)
-		goto err_probe;
+		return ret;
 
 	fsl_espi_init_regs(dev, true);
 
@@ -732,8 +729,7 @@ static int fsl_espi_probe(struct device *dev, struct resource *mem,
 	pm_runtime_put_noidle(dev);
 	pm_runtime_disable(dev);
 	pm_runtime_set_suspended(dev);
-err_probe:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -784,13 +780,9 @@ static void of_fsl_espi_remove(struct platform_device *dev)
 {
 	struct spi_controller *host = platform_get_drvdata(dev);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	pm_runtime_disable(&dev->dev);
-
-	spi_controller_put(host);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/spi/spi-fsl-spi.c b/drivers/spi/spi-fsl-spi.c
index 1252c41c206f8..e45816ef7b656 100644
--- a/drivers/spi/spi-fsl-spi.c
+++ b/drivers/spi/spi-fsl-spi.c
@@ -535,7 +535,7 @@ static struct spi_controller *fsl_spi_probe(struct device *dev,
 	u32 regval;
 	int ret = 0;
 
-	host = spi_alloc_host(dev, sizeof(struct mpc8xxx_spi));
+	host = devm_spi_alloc_host(dev, sizeof(struct mpc8xxx_spi));
 	if (host == NULL) {
 		ret = -ENOMEM;
 		goto err;
@@ -559,7 +559,7 @@ static struct spi_controller *fsl_spi_probe(struct device *dev,
 
 	ret = fsl_spi_cpm_init(mpc8xxx_spi);
 	if (ret)
-		goto err_cpm_init;
+		goto err;
 
 	mpc8xxx_spi->reg_base = devm_ioremap_resource(dev, mem);
 	if (IS_ERR(mpc8xxx_spi->reg_base)) {
@@ -625,8 +625,6 @@ static struct spi_controller *fsl_spi_probe(struct device *dev,
 
 err_probe:
 	fsl_spi_cpm_free(mpc8xxx_spi);
-err_cpm_init:
-	spi_controller_put(host);
 err:
 	return ERR_PTR(ret);
 }
@@ -705,13 +703,9 @@ static void of_fsl_spi_remove(struct platform_device *ofdev)
 	struct spi_controller *host = platform_get_drvdata(ofdev);
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	fsl_spi_cpm_free(mpc8xxx_spi);
-
-	spi_controller_put(host);
 }
 
 static struct platform_driver of_fsl_spi_driver = {
@@ -757,13 +751,9 @@ static void plat_mpc8xxx_spi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct mpc8xxx_spi *mpc8xxx_spi = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	fsl_spi_cpm_free(mpc8xxx_spi);
-
-	spi_controller_put(host);
 }
 
 MODULE_ALIAS("platform:mpc8xxx_spi");
diff --git a/drivers/spi/spi-geni-qcom.c b/drivers/spi/spi-geni-qcom.c
index d5fb0edc8e0c8..a04cdc1e5ad4d 100644
--- a/drivers/spi/spi-geni-qcom.c
+++ b/drivers/spi/spi-geni-qcom.c
@@ -1,6 +1,9 @@
 // SPDX-License-Identifier: GPL-2.0
 // Copyright (c) 2017-2018, The Linux foundation. All rights reserved.
 
+#define CREATE_TRACE_POINTS
+#include <trace/events/qcom_geni_spi.h>
+
 #include <linux/clk.h>
 #include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
@@ -332,6 +335,9 @@ static int geni_spi_set_clock_and_bw(struct spi_geni_master *mas,
 	writel(clk_sel, se->base + SE_GENI_CLK_SEL);
 	writel(m_clk_cfg, se->base + GENI_SER_M_CLK_CFG);
 
+	trace_geni_spi_clk_cfg(mas->dev, clk_hz, mas->cur_sclk_hz, idx, div,
+			       mas->cur_bits_per_word);
+
 	/* Set BW quota for CPU as driver supports FIFO mode only. */
 	se->icc_paths[CPU_TO_GENI].avg_bw = Bps_to_icc(mas->cur_speed_hz);
 	ret = geni_icc_set_bw(se);
@@ -366,6 +372,9 @@ static int setup_fifo_params(struct spi_device *spi_slv,
 	if ((mode_changed & SPI_CS_HIGH) || (cs_changed && (spi_slv->mode & SPI_CS_HIGH)))
 		writel((spi_slv->mode & SPI_CS_HIGH) ? BIT(chipselect) : 0, se->base + SE_SPI_DEMUX_OUTPUT_INV);
 
+	trace_geni_spi_setup_params(mas->dev, chipselect, spi_slv->mode,
+				    mode_changed, cs_changed);
+
 	return 0;
 }
 
@@ -861,6 +870,8 @@ static int setup_se_xfer(struct spi_transfer *xfer,
 	spin_lock_irq(&mas->lock);
 	geni_se_setup_m_cmd(se, m_cmd, m_params);
 
+	trace_geni_spi_transfer(mas->dev, len, m_cmd);
+
 	if (mas->cur_xfer_mode == GENI_SE_DMA) {
 		if (m_cmd & SPI_RX_ONLY)
 			geni_se_rx_init_dma(se, sg_dma_address(xfer->rx_sg.sgl),
@@ -915,6 +926,8 @@ static irqreturn_t geni_spi_isr(int irq, void *data)
 	if (!m_irq && !dma_tx_status && !dma_rx_status)
 		return IRQ_NONE;
 
+	trace_geni_spi_irq(mas->dev, m_irq, dma_tx_status, dma_rx_status);
+
 	if (m_irq & (M_CMD_OVERRUN_EN | M_ILLEGAL_CMD_EN | M_CMD_FAILURE_EN |
 		     M_RX_FIFO_RD_ERR_EN | M_RX_FIFO_WR_ERR_EN |
 		     M_TX_FIFO_RD_ERR_EN | M_TX_FIFO_WR_ERR_EN))
diff --git a/drivers/spi/spi-hisi-kunpeng.c b/drivers/spi/spi-hisi-kunpeng.c
index 046bd894040b1..395214b81179e 100644
--- a/drivers/spi/spi-hisi-kunpeng.c
+++ b/drivers/spi/spi-hisi-kunpeng.c
@@ -520,10 +520,8 @@ static int hisi_spi_probe(struct platform_device *pdev)
 	}
 
 	ret = spi_register_controller(host);
-	if (ret) {
-		dev_err(dev, "failed to register spi host, ret=%d\n", ret);
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register spi host\n");
 
 	if (hisi_spi_debugfs_init(hs))
 		dev_info(dev, "failed to create debugfs dir\n");
diff --git a/drivers/spi/spi-hisi-sfc-v3xx.c b/drivers/spi/spi-hisi-sfc-v3xx.c
index b2af2eed197f2..eeeb863818624 100644
--- a/drivers/spi/spi-hisi-sfc-v3xx.c
+++ b/drivers/spi/spi-hisi-sfc-v3xx.c
@@ -436,7 +436,7 @@ static int hisi_sfc_v3xx_probe(struct platform_device *pdev)
 	u32 version, glb_config;
 	int ret;
 
-	ctlr = spi_alloc_host(&pdev->dev, sizeof(*host));
+	ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(*host));
 	if (!ctlr)
 		return -ENOMEM;
 
@@ -451,16 +451,12 @@ static int hisi_sfc_v3xx_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, host);
 
 	host->regbase = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(host->regbase)) {
-		ret = PTR_ERR(host->regbase);
-		goto err_put_host;
-	}
+	if (IS_ERR(host->regbase))
+		return PTR_ERR(host->regbase);
 
 	host->irq = platform_get_irq_optional(pdev, 0);
-	if (host->irq == -EPROBE_DEFER) {
-		ret = -EPROBE_DEFER;
-		goto err_put_host;
-	}
+	if (host->irq == -EPROBE_DEFER)
+		return -EPROBE_DEFER;
 
 	hisi_sfc_v3xx_disable_int(host);
 
@@ -501,16 +497,12 @@ static int hisi_sfc_v3xx_probe(struct platform_device *pdev)
 
 	ret = devm_spi_register_controller(dev, ctlr);
 	if (ret)
-		goto err_put_host;
+		return ret;
 
 	dev_info(&pdev->dev, "hw version 0x%x, %s mode.\n",
 		 version, host->irq ? "irq" : "polling");
 
 	return 0;
-
-err_put_host:
-	spi_controller_put(ctlr);
-	return ret;
 }
 
 static const struct acpi_device_id hisi_sfc_v3xx_acpi_ids[] = {
diff --git a/drivers/spi/spi-img-spfi.c b/drivers/spi/spi-img-spfi.c
index 57625a3ce2f2d..aec724e3f8246 100644
--- a/drivers/spi/spi-img-spfi.c
+++ b/drivers/spi/spi-img-spfi.c
@@ -530,7 +530,7 @@ static int img_spfi_probe(struct platform_device *pdev)
 	int ret;
 	u32 max_speed_hz;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*spfi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*spfi));
 	if (!host)
 		return -ENOMEM;
 	platform_set_drvdata(pdev, host);
@@ -541,36 +541,32 @@ static int img_spfi_probe(struct platform_device *pdev)
 	spin_lock_init(&spfi->lock);
 
 	spfi->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
-	if (IS_ERR(spfi->regs)) {
-		ret = PTR_ERR(spfi->regs);
-		goto put_spi;
-	}
+	if (IS_ERR(spfi->regs))
+		return PTR_ERR(spfi->regs);
+
 	spfi->phys = res->start;
 
 	spfi->irq = platform_get_irq(pdev, 0);
-	if (spfi->irq < 0) {
-		ret = spfi->irq;
-		goto put_spi;
-	}
+	if (spfi->irq < 0)
+		return spfi->irq;
+
 	ret = devm_request_irq(spfi->dev, spfi->irq, img_spfi_irq,
 			       IRQ_TYPE_LEVEL_HIGH, dev_name(spfi->dev), spfi);
 	if (ret)
-		goto put_spi;
+		return ret;
 
 	spfi->sys_clk = devm_clk_get(spfi->dev, "sys");
-	if (IS_ERR(spfi->sys_clk)) {
-		ret = PTR_ERR(spfi->sys_clk);
-		goto put_spi;
-	}
+	if (IS_ERR(spfi->sys_clk))
+		return PTR_ERR(spfi->sys_clk);
+
 	spfi->spfi_clk = devm_clk_get(spfi->dev, "spfi");
-	if (IS_ERR(spfi->spfi_clk)) {
-		ret = PTR_ERR(spfi->spfi_clk);
-		goto put_spi;
-	}
+	if (IS_ERR(spfi->spfi_clk))
+		return PTR_ERR(spfi->spfi_clk);
 
 	ret = clk_prepare_enable(spfi->sys_clk);
 	if (ret)
-		goto put_spi;
+		return ret;
+
 	ret = clk_prepare_enable(spfi->spfi_clk);
 	if (ret)
 		goto disable_pclk;
@@ -658,8 +654,6 @@ static int img_spfi_probe(struct platform_device *pdev)
 	clk_disable_unprepare(spfi->spfi_clk);
 disable_pclk:
 	clk_disable_unprepare(spfi->sys_clk);
-put_spi:
-	spi_controller_put(host);
 
 	return ret;
 }
@@ -669,8 +663,6 @@ static void img_spfi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct img_spfi *spfi = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	if (spfi->tx_ch)
@@ -683,8 +675,6 @@ static void img_spfi_remove(struct platform_device *pdev)
 		clk_disable_unprepare(spfi->spfi_clk);
 		clk_disable_unprepare(spfi->sys_clk);
 	}
-
-	spi_controller_put(host);
 }
 
 #ifdef CONFIG_PM
diff --git a/drivers/spi/spi-intel-pci.c b/drivers/spi/spi-intel-pci.c
index d8ef8f89330ac..8c429c832dddf 100644
--- a/drivers/spi/spi-intel-pci.c
+++ b/drivers/spi/spi-intel-pci.c
@@ -66,39 +66,39 @@ static int intel_spi_pci_probe(struct pci_dev *pdev,
 }
 
 static const struct pci_device_id intel_spi_pci_ids[] = {
-	{ PCI_VDEVICE(INTEL, 0x02a4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x06a4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x18e0), (unsigned long)&bxt_info },
-	{ PCI_VDEVICE(INTEL, 0x19e0), (unsigned long)&bxt_info },
-	{ PCI_VDEVICE(INTEL, 0x1bca), (unsigned long)&bxt_info },
-	{ PCI_VDEVICE(INTEL, 0x34a4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x38a4), (unsigned long)&bxt_info },
-	{ PCI_VDEVICE(INTEL, 0x43a4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x4b24), (unsigned long)&bxt_info },
-	{ PCI_VDEVICE(INTEL, 0x4d23), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x4da4), (unsigned long)&bxt_info },
-	{ PCI_VDEVICE(INTEL, 0x51a4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x54a4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x5794), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x5825), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x6e24), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x7723), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x7a24), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x7aa4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x7e23), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x7f24), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x9d24), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0x9da4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0xa0a4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0xa1a4), (unsigned long)&bxt_info },
-	{ PCI_VDEVICE(INTEL, 0xa224), (unsigned long)&bxt_info },
-	{ PCI_VDEVICE(INTEL, 0xa2a4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0xa324), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0xa3a4), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0xa823), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0xd323), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0xe323), (unsigned long)&cnl_info },
-	{ PCI_VDEVICE(INTEL, 0xe423), (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x02a4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x06a4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x18e0), .driver_data = (unsigned long)&bxt_info },
+	{ PCI_VDEVICE(INTEL, 0x19e0), .driver_data = (unsigned long)&bxt_info },
+	{ PCI_VDEVICE(INTEL, 0x1bca), .driver_data = (unsigned long)&bxt_info },
+	{ PCI_VDEVICE(INTEL, 0x34a4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x38a4), .driver_data = (unsigned long)&bxt_info },
+	{ PCI_VDEVICE(INTEL, 0x43a4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x4b24), .driver_data = (unsigned long)&bxt_info },
+	{ PCI_VDEVICE(INTEL, 0x4d23), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x4da4), .driver_data = (unsigned long)&bxt_info },
+	{ PCI_VDEVICE(INTEL, 0x51a4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x54a4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x5794), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x5825), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x6e24), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x7723), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x7a24), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x7aa4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x7e23), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x7f24), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x9d24), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0x9da4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0xa0a4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0xa1a4), .driver_data = (unsigned long)&bxt_info },
+	{ PCI_VDEVICE(INTEL, 0xa224), .driver_data = (unsigned long)&bxt_info },
+	{ PCI_VDEVICE(INTEL, 0xa2a4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0xa324), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0xa3a4), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0xa823), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0xd323), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0xe323), .driver_data = (unsigned long)&cnl_info },
+	{ PCI_VDEVICE(INTEL, 0xe423), .driver_data = (unsigned long)&cnl_info },
 	{ },
 };
 MODULE_DEVICE_TABLE(pci, intel_spi_pci_ids);
diff --git a/drivers/spi/spi-intel.c b/drivers/spi/spi-intel.c
index 1775ad39e6339..7494b921a743a 100644
--- a/drivers/spi/spi-intel.c
+++ b/drivers/spi/spi-intel.c
@@ -814,7 +814,7 @@ static int intel_spi_dirmap_create(struct spi_mem_dirmap_desc *desc)
 	struct intel_spi *ispi = spi_controller_get_devdata(desc->mem->spi->controller);
 	const struct intel_spi_mem_op *iop;
 
-	iop = intel_spi_match_mem_op(ispi, &desc->info.op_tmpl);
+	iop = intel_spi_match_mem_op(ispi, desc->info.op_tmpl);
 	if (!iop)
 		return -EOPNOTSUPP;
 
@@ -827,7 +827,7 @@ static ssize_t intel_spi_dirmap_read(struct spi_mem_dirmap_desc *desc, u64 offs,
 {
 	struct intel_spi *ispi = spi_controller_get_devdata(desc->mem->spi->controller);
 	const struct intel_spi_mem_op *iop = desc->priv;
-	struct spi_mem_op op = desc->info.op_tmpl;
+	struct spi_mem_op op = *desc->info.op_tmpl;
 	int ret;
 
 	/* Fill in the gaps */
@@ -844,7 +844,7 @@ static ssize_t intel_spi_dirmap_write(struct spi_mem_dirmap_desc *desc, u64 offs
 {
 	struct intel_spi *ispi = spi_controller_get_devdata(desc->mem->spi->controller);
 	const struct intel_spi_mem_op *iop = desc->priv;
-	struct spi_mem_op op = desc->info.op_tmpl;
+	struct spi_mem_op op = *desc->info.op_tmpl;
 	int ret;
 
 	op.addr.val = offs;
diff --git a/drivers/spi/spi-jcore.c b/drivers/spi/spi-jcore.c
index e37ca22e04ba6..a75cd61ec7a33 100644
--- a/drivers/spi/spi-jcore.c
+++ b/drivers/spi/spi-jcore.c
@@ -146,11 +146,10 @@ static int jcore_spi_probe(struct platform_device *pdev)
 	struct resource *res;
 	u32 clock_freq;
 	struct clk *clk;
-	int err = -ENODEV;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(struct jcore_spi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(struct jcore_spi));
 	if (!host)
-		return err;
+		return -ENOMEM;
 
 	/* Setup the host state. */
 	host->num_chipselect = 3;
@@ -167,14 +166,14 @@ static int jcore_spi_probe(struct platform_device *pdev)
 	/* Find and map our resources */
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res)
-		goto exit_busy;
+		return -EBUSY;
 	if (!devm_request_mem_region(&pdev->dev, res->start,
 				     resource_size(res), pdev->name))
-		goto exit_busy;
+		return -EBUSY;
 	hw->base = devm_ioremap(&pdev->dev, res->start,
 					resource_size(res));
 	if (!hw->base)
-		goto exit_busy;
+		return -EBUSY;
 
 	/*
 	 * The SPI clock rate controlled via a configurable clock divider
@@ -200,17 +199,7 @@ static int jcore_spi_probe(struct platform_device *pdev)
 	jcore_spi_baudrate(hw, 400000);
 
 	/* Register our spi controller */
-	err = devm_spi_register_controller(&pdev->dev, host);
-	if (err)
-		goto exit;
-
-	return 0;
-
-exit_busy:
-	err = -EBUSY;
-exit:
-	spi_controller_put(host);
-	return err;
+	return devm_spi_register_controller(&pdev->dev, host);
 }
 
 static const struct of_device_id jcore_spi_of_match[] = {
diff --git a/drivers/spi/spi-lantiq-ssc.c b/drivers/spi/spi-lantiq-ssc.c
index 75b9af8cb5db7..ecae50a50b142 100644
--- a/drivers/spi/spi-lantiq-ssc.c
+++ b/drivers/spi/spi-lantiq-ssc.c
@@ -913,7 +913,7 @@ static int lantiq_ssc_probe(struct platform_device *pdev)
 
 	hwcfg = of_device_get_match_data(dev);
 
-	host = spi_alloc_host(dev, sizeof(struct lantiq_ssc_spi));
+	host = devm_spi_alloc_host(dev, sizeof(struct lantiq_ssc_spi));
 	if (!host)
 		return -ENOMEM;
 
@@ -923,20 +923,16 @@ static int lantiq_ssc_probe(struct platform_device *pdev)
 	spi->hwcfg = hwcfg;
 	platform_set_drvdata(pdev, spi);
 	spi->regbase = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(spi->regbase)) {
-		err = PTR_ERR(spi->regbase);
-		goto err_host_put;
-	}
+	if (IS_ERR(spi->regbase))
+		return PTR_ERR(spi->regbase);
 
 	err = hwcfg->cfg_irq(pdev, spi);
 	if (err)
-		goto err_host_put;
+		return err;
 
 	spi->spi_clk = devm_clk_get_enabled(dev, "gate");
-	if (IS_ERR(spi->spi_clk)) {
-		err = PTR_ERR(spi->spi_clk);
-		goto err_host_put;
-	}
+	if (IS_ERR(spi->spi_clk))
+		return PTR_ERR(spi->spi_clk);
 
 	/*
 	 * Use the old clk_get_fpi() function on Lantiq platform, till it
@@ -947,10 +943,8 @@ static int lantiq_ssc_probe(struct platform_device *pdev)
 #else
 	spi->fpi_clk = clk_get(dev, "freq");
 #endif
-	if (IS_ERR(spi->fpi_clk)) {
-		err = PTR_ERR(spi->fpi_clk);
-		goto err_host_put;
-	}
+	if (IS_ERR(spi->fpi_clk))
+		return PTR_ERR(spi->fpi_clk);
 
 	num_cs = 8;
 	of_property_read_u32(pdev->dev.of_node, "num-cs", &num_cs);
@@ -1006,8 +1000,6 @@ static int lantiq_ssc_probe(struct platform_device *pdev)
 	destroy_workqueue(spi->wq);
 err_clk_put:
 	clk_put(spi->fpi_clk);
-err_host_put:
-	spi_controller_put(host);
 
 	return err;
 }
@@ -1016,8 +1008,6 @@ static void lantiq_ssc_remove(struct platform_device *pdev)
 {
 	struct lantiq_ssc_spi *spi = platform_get_drvdata(pdev);
 
-	spi_controller_get(spi->host);
-
 	spi_unregister_controller(spi->host);
 
 	lantiq_ssc_writel(spi, 0, LTQ_SPI_IRNEN);
@@ -1028,8 +1018,6 @@ static void lantiq_ssc_remove(struct platform_device *pdev)
 
 	destroy_workqueue(spi->wq);
 	clk_put(spi->fpi_clk);
-
-	spi_controller_put(spi->host);
 }
 
 static struct platform_driver lantiq_ssc_driver = {
diff --git a/drivers/spi/spi-lp8841-rtc.c b/drivers/spi/spi-lp8841-rtc.c
index e466866d5e80e..b2546a3a9eaa7 100644
--- a/drivers/spi/spi-lp8841-rtc.c
+++ b/drivers/spi/spi-lp8841-rtc.c
@@ -185,7 +185,7 @@ spi_lp8841_rtc_probe(struct platform_device *pdev)
 	struct spi_controller		*host;
 	struct spi_lp8841_rtc		*data;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*data));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*data));
 	if (!host)
 		return -ENOMEM;
 	platform_set_drvdata(pdev, host);
@@ -199,8 +199,6 @@ spi_lp8841_rtc_probe(struct platform_device *pdev)
 	host->set_cs = spi_lp8841_rtc_set_cs;
 	host->transfer_one = spi_lp8841_rtc_transfer_one;
 	host->bits_per_word_mask = SPI_BPW_MASK(8);
-#ifdef CONFIG_OF
-#endif
 
 	data = spi_controller_get_devdata(host);
 
@@ -208,23 +206,17 @@ spi_lp8841_rtc_probe(struct platform_device *pdev)
 	ret = PTR_ERR_OR_ZERO(data->iomem);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to get IO address\n");
-		goto err_put_host;
+		return ret;
 	}
 
 	/* register with the SPI framework */
 	ret = devm_spi_register_controller(&pdev->dev, host);
 	if (ret) {
 		dev_err(&pdev->dev, "cannot register spi host\n");
-		goto err_put_host;
+		return ret;
 	}
 
-	return ret;
-
-
-err_put_host:
-	spi_controller_put(host);
-
-	return ret;
+	return 0;
 }
 
 MODULE_ALIAS("platform:" DRIVER_NAME);
diff --git a/drivers/spi/spi-mem.c b/drivers/spi/spi-mem.c
index a09371a075d2e..f64eda9bbd9f0 100644
--- a/drivers/spi/spi-mem.c
+++ b/drivers/spi/spi-mem.c
@@ -647,7 +647,7 @@ EXPORT_SYMBOL_GPL(spi_mem_calc_op_duration);
 static ssize_t spi_mem_no_dirmap_read(struct spi_mem_dirmap_desc *desc,
 				      u64 offs, size_t len, void *buf)
 {
-	struct spi_mem_op op = desc->info.op_tmpl;
+	struct spi_mem_op op = *desc->info.op_tmpl;
 	int ret;
 
 	op.addr.val = desc->info.offset + offs;
@@ -667,7 +667,7 @@ static ssize_t spi_mem_no_dirmap_read(struct spi_mem_dirmap_desc *desc,
 static ssize_t spi_mem_no_dirmap_write(struct spi_mem_dirmap_desc *desc,
 				       u64 offs, size_t len, const void *buf)
 {
-	struct spi_mem_op op = desc->info.op_tmpl;
+	struct spi_mem_op op = *desc->info.op_tmpl;
 	int ret;
 
 	op.addr.val = desc->info.offset + offs;
@@ -706,19 +706,37 @@ spi_mem_dirmap_create(struct spi_mem *mem,
 	int ret = -ENOTSUPP;
 
 	/* Make sure the number of address cycles is between 1 and 8 bytes. */
-	if (!info->op_tmpl.addr.nbytes || info->op_tmpl.addr.nbytes > 8)
+	if (!info->primary_op_tmpl.addr.nbytes || info->primary_op_tmpl.addr.nbytes > 8)
 		return ERR_PTR(-EINVAL);
 
 	/* data.dir should either be SPI_MEM_DATA_IN or SPI_MEM_DATA_OUT. */
-	if (info->op_tmpl.data.dir == SPI_MEM_NO_DATA)
+	if (info->primary_op_tmpl.data.dir == SPI_MEM_NO_DATA)
 		return ERR_PTR(-EINVAL);
 
+	/* Apply similar constraints to the secondary template */
+	if (info->secondary_op_tmpl.cmd.opcode) {
+		if (!info->secondary_op_tmpl.addr.nbytes ||
+		    info->secondary_op_tmpl.addr.nbytes > 8)
+			return ERR_PTR(-EINVAL);
+
+		if (info->secondary_op_tmpl.data.dir == SPI_MEM_NO_DATA)
+			return ERR_PTR(-EINVAL);
+
+		if (!spi_mem_supports_op(mem, &info->secondary_op_tmpl))
+			return ERR_PTR(-EOPNOTSUPP);
+
+		if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create &&
+		    !spi_mem_controller_is_capable(ctlr, secondary_op_tmpl))
+			return ERR_PTR(-EOPNOTSUPP);
+	}
+
 	desc = kzalloc_obj(*desc);
 	if (!desc)
 		return ERR_PTR(-ENOMEM);
 
 	desc->mem = mem;
 	desc->info = *info;
+	desc->info.op_tmpl = &desc->info.primary_op_tmpl;
 	if (ctlr->mem_ops && ctlr->mem_ops->dirmap_create) {
 		ret = spi_mem_access_start(mem);
 		if (ret) {
@@ -733,7 +751,7 @@ spi_mem_dirmap_create(struct spi_mem *mem,
 
 	if (ret) {
 		desc->nodirmap = true;
-		if (!spi_mem_supports_op(desc->mem, &desc->info.op_tmpl))
+		if (!spi_mem_supports_op(desc->mem, &desc->info.primary_op_tmpl))
 			ret = -EOPNOTSUPP;
 		else
 			ret = 0;
@@ -857,7 +875,7 @@ ssize_t spi_mem_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	struct spi_controller *ctlr = desc->mem->spi->controller;
 	ssize_t ret;
 
-	if (desc->info.op_tmpl.data.dir != SPI_MEM_DATA_IN)
+	if (desc->info.op_tmpl->data.dir != SPI_MEM_DATA_IN)
 		return -EINVAL;
 
 	if (!len)
@@ -903,7 +921,7 @@ ssize_t spi_mem_dirmap_write(struct spi_mem_dirmap_desc *desc,
 	struct spi_controller *ctlr = desc->mem->spi->controller;
 	ssize_t ret;
 
-	if (desc->info.op_tmpl.data.dir != SPI_MEM_DATA_OUT)
+	if (desc->info.op_tmpl->data.dir != SPI_MEM_DATA_OUT)
 		return -EINVAL;
 
 	if (!len)
diff --git a/drivers/spi/spi-meson-spicc.c b/drivers/spi/spi-meson-spicc.c
index b80f9f457b667..fa2d2db2d6e7c 100644
--- a/drivers/spi/spi-meson-spicc.c
+++ b/drivers/spi/spi-meson-spicc.c
@@ -539,9 +539,8 @@ static void meson_spicc_setup_xfer(struct meson_spicc_device *spicc,
 	conf = conf_orig = readl_relaxed(spicc->base + SPICC_CONREG);
 
 	/* Setup word width */
-	conf &= ~SPICC_BITLENGTH_MASK;
-	conf |= FIELD_PREP(SPICC_BITLENGTH_MASK,
-			   (spicc->bytes_per_word << 3) - 1);
+	FIELD_MODIFY(SPICC_BITLENGTH_MASK, &conf,
+		     (spicc->bytes_per_word << 3) - 1);
 
 	/* Ignore if unchanged */
 	if (conf != conf_orig)
@@ -982,7 +981,7 @@ static int meson_spicc_probe(struct platform_device *pdev)
 	struct meson_spicc_device *spicc;
 	int ret, irq;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*spicc));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*spicc));
 	if (!host) {
 		dev_err(&pdev->dev, "host allocation failed\n");
 		return -ENOMEM;
@@ -993,8 +992,7 @@ static int meson_spicc_probe(struct platform_device *pdev)
 	spicc->data = of_device_get_match_data(&pdev->dev);
 	if (!spicc->data) {
 		dev_err(&pdev->dev, "failed to get match data\n");
-		ret = -EINVAL;
-		goto out_host;
+		return -EINVAL;
 	}
 
 	spicc->pdev = pdev;
@@ -1005,8 +1003,7 @@ static int meson_spicc_probe(struct platform_device *pdev)
 	spicc->base = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(spicc->base)) {
 		dev_err(&pdev->dev, "io resource mapping failed\n");
-		ret = PTR_ERR(spicc->base);
-		goto out_host;
+		return PTR_ERR(spicc->base);
 	}
 
 	/* Set master mode and enable controller */
@@ -1017,39 +1014,33 @@ static int meson_spicc_probe(struct platform_device *pdev)
 	writel_relaxed(0, spicc->base + SPICC_INTREG);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = irq;
-		goto out_host;
-	}
+	if (irq < 0)
+		return irq;
 
 	ret = devm_request_irq(&pdev->dev, irq, meson_spicc_irq,
 			       0, NULL, spicc);
 	if (ret) {
 		dev_err(&pdev->dev, "irq request failed\n");
-		goto out_host;
+		return ret;
 	}
 
 	spicc->core = devm_clk_get_enabled(&pdev->dev, "core");
 	if (IS_ERR(spicc->core)) {
 		dev_err(&pdev->dev, "core clock request failed\n");
-		ret = PTR_ERR(spicc->core);
-		goto out_host;
+		return PTR_ERR(spicc->core);
 	}
 
 	if (spicc->data->has_pclk) {
 		spicc->pclk = devm_clk_get_enabled(&pdev->dev, "pclk");
 		if (IS_ERR(spicc->pclk)) {
 			dev_err(&pdev->dev, "pclk clock request failed\n");
-			ret = PTR_ERR(spicc->pclk);
-			goto out_host;
+			return PTR_ERR(spicc->pclk);
 		}
 	}
 
 	spicc->pinctrl = devm_pinctrl_get(&pdev->dev);
-	if (IS_ERR(spicc->pinctrl)) {
-		ret = PTR_ERR(spicc->pinctrl);
-		goto out_host;
-	}
+	if (IS_ERR(spicc->pinctrl))
+		return PTR_ERR(spicc->pinctrl);
 
 	device_reset_optional(&pdev->dev);
 
@@ -1070,43 +1061,34 @@ static int meson_spicc_probe(struct platform_device *pdev)
 	ret = meson_spicc_pow2_clk_init(spicc);
 	if (ret) {
 		dev_err(&pdev->dev, "pow2 clock registration failed\n");
-		goto out_host;
+		return ret;
 	}
 
 	if (spicc->data->has_enhance_clk_div) {
 		ret = meson_spicc_enh_clk_init(spicc);
 		if (ret) {
 			dev_err(&pdev->dev, "clock registration failed\n");
-			goto out_host;
+			return ret;
 		}
 	}
 
 	ret = spi_register_controller(host);
 	if (ret) {
 		dev_err(&pdev->dev, "spi registration failed\n");
-		goto out_host;
+		return ret;
 	}
 
 	return 0;
-
-out_host:
-	spi_controller_put(host);
-
-	return ret;
 }
 
 static void meson_spicc_remove(struct platform_device *pdev)
 {
 	struct meson_spicc_device *spicc = platform_get_drvdata(pdev);
 
-	spi_controller_get(spicc->host);
-
 	spi_unregister_controller(spicc->host);
 
 	/* Disable SPI */
 	writel(0, spicc->base + SPICC_CONREG);
-
-	spi_controller_put(spicc->host);
 }
 
 static const struct meson_spicc_data meson_spicc_gx_data = {
diff --git a/drivers/spi/spi-meson-spifc.c b/drivers/spi/spi-meson-spifc.c
index b818950a8cb7d..d700fa3152231 100644
--- a/drivers/spi/spi-meson-spifc.c
+++ b/drivers/spi/spi-meson-spifc.c
@@ -288,9 +288,9 @@ static int meson_spifc_probe(struct platform_device *pdev)
 	struct meson_spifc *spifc;
 	void __iomem *base;
 	unsigned int rate;
-	int ret = 0;
+	int ret;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(struct meson_spifc));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(struct meson_spifc));
 	if (!host)
 		return -ENOMEM;
 
@@ -300,23 +300,18 @@ static int meson_spifc_probe(struct platform_device *pdev)
 	spifc->dev = &pdev->dev;
 
 	base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(base)) {
-		ret = PTR_ERR(base);
-		goto out_err;
-	}
+	if (IS_ERR(base))
+		return PTR_ERR(base);
 
 	spifc->regmap = devm_regmap_init_mmio(spifc->dev, base,
 					      &spifc_regmap_config);
-	if (IS_ERR(spifc->regmap)) {
-		ret = PTR_ERR(spifc->regmap);
-		goto out_err;
-	}
+	if (IS_ERR(spifc->regmap))
+		return PTR_ERR(spifc->regmap);
 
 	spifc->clk = devm_clk_get_enabled(spifc->dev, NULL);
 	if (IS_ERR(spifc->clk)) {
 		dev_err(spifc->dev, "missing clock\n");
-		ret = PTR_ERR(spifc->clk);
-		goto out_err;
+		return PTR_ERR(spifc->clk);
 	}
 
 	rate = clk_get_rate(spifc->clk);
@@ -342,8 +337,7 @@ static int meson_spifc_probe(struct platform_device *pdev)
 	return 0;
 out_pm:
 	pm_runtime_disable(spifc->dev);
-out_err:
-	spi_controller_put(host);
+
 	return ret;
 }
 
diff --git a/drivers/spi/spi-microchip-core-qspi.c b/drivers/spi/spi-microchip-core-qspi.c
index 4dee0fea1df8d..c0ea65014d2ea 100644
--- a/drivers/spi/spi-microchip-core-qspi.c
+++ b/drivers/spi/spi-microchip-core-qspi.c
@@ -99,7 +99,6 @@
 #define REG_IEN			(0x0c)
 #define REG_STATUS		(0x10)
 #define REG_DIRECT_ACCESS	(0x14)
-#define REG_UPPER_ACCESS	(0x18)
 #define REG_RX_DATA		(0x40)
 #define REG_TX_DATA		(0x44)
 #define REG_X4_RX_DATA		(0x48)
@@ -632,7 +631,7 @@ static int mchp_coreqspi_prepare_message(struct spi_controller *ctlr, struct spi
 	ret = mchp_coreqspi_wait_for_ready(qspi);
 	if (ret) {
 		mutex_unlock(&qspi->op_lock);
-		dev_err(&ctlr->dev, "Timeout waiting on QSPI ready.\n");
+		dev_err(&m->spi->dev, "Timeout waiting on QSPI ready.\n");
 		return ret;
 	}
 
diff --git a/drivers/spi/spi-mpc52xx.c b/drivers/spi/spi-mpc52xx.c
index 924d820448fb4..04c2270cd2cf6 100644
--- a/drivers/spi/spi-mpc52xx.c
+++ b/drivers/spi/spi-mpc52xx.c
@@ -472,13 +472,15 @@ static int mpc52xx_spi_probe(struct platform_device *op)
 	if (ms->irq0 && ms->irq1) {
 		rc = request_irq(ms->irq0, mpc52xx_spi_irq, 0,
 				  "mpc5200-spi-modf", ms);
-		rc |= request_irq(ms->irq1, mpc52xx_spi_irq, 0,
-				  "mpc5200-spi-spif", ms);
-		if (rc) {
-			free_irq(ms->irq0, ms);
-			free_irq(ms->irq1, ms);
-			ms->irq0 = ms->irq1 = 0;
+		if (rc == 0) {
+			rc = request_irq(ms->irq1, mpc52xx_spi_irq, 0,
+					 "mpc5200-spi-spif", ms);
+			if (rc)
+				free_irq(ms->irq0, ms);
 		}
+
+		if (rc)
+			ms->irq0 = ms->irq1 = 0;
 	} else {
 		/* operate in polled mode */
 		ms->irq0 = ms->irq1 = 0;
@@ -498,8 +500,10 @@ static int mpc52xx_spi_probe(struct platform_device *op)
 
  err_register:
 	dev_err(&ms->host->dev, "initialization failed\n");
-	free_irq(ms->irq0, ms);
-	free_irq(ms->irq1, ms);
+	if (ms->irq0) {
+		free_irq(ms->irq0, ms);
+		free_irq(ms->irq1, ms);
+	}
 	cancel_work_sync(&ms->work);
  err_gpio:
 	while (i-- > 0)
@@ -522,8 +526,10 @@ static void mpc52xx_spi_remove(struct platform_device *op)
 
 	spi_unregister_controller(host);
 
-	free_irq(ms->irq0, ms);
-	free_irq(ms->irq1, ms);
+	if (ms->irq0) {
+		free_irq(ms->irq0, ms);
+		free_irq(ms->irq1, ms);
+	}
 
 	cancel_work_sync(&ms->work);
 
diff --git a/drivers/spi/spi-mtk-snfi.c b/drivers/spi/spi-mtk-snfi.c
index e616e6800e92f..6e96e50fedad4 100644
--- a/drivers/spi/spi-mtk-snfi.c
+++ b/drivers/spi/spi-mtk-snfi.c
@@ -961,7 +961,7 @@ static int mtk_snand_read_page_cache(struct mtk_snand *snf,
 		    &snf->op_done, usecs_to_jiffies(SNFI_POLL_INTERVAL))) {
 		dev_err(snf->dev, "DMA timed out for reading from cache.\n");
 		ret = -ETIMEDOUT;
-		goto cleanup;
+		goto cleanup2;
 	}
 
 	// Wait for BUS_SEC_CNTR returning expected value
diff --git a/drivers/spi/spi-mux.c b/drivers/spi/spi-mux.c
index bd122de152c0f..08fe1fa32dea3 100644
--- a/drivers/spi/spi-mux.c
+++ b/drivers/spi/spi-mux.c
@@ -127,9 +127,8 @@ static int spi_mux_probe(struct spi_device *spi)
 {
 	struct spi_controller *ctlr;
 	struct spi_mux_priv *priv;
-	int ret;
 
-	ctlr = spi_alloc_host(&spi->dev, sizeof(*priv));
+	ctlr = devm_spi_alloc_host(&spi->dev, sizeof(*priv));
 	if (!ctlr)
 		return -ENOMEM;
 
@@ -146,9 +145,8 @@ static int spi_mux_probe(struct spi_device *spi)
 
 	priv->mux = devm_mux_control_get(&spi->dev, NULL);
 	if (IS_ERR(priv->mux)) {
-		ret = dev_err_probe(&spi->dev, PTR_ERR(priv->mux),
-				    "failed to get control-mux\n");
-		goto err_put_ctlr;
+		return dev_err_probe(&spi->dev, PTR_ERR(priv->mux),
+				     "failed to get control-mux\n");
 	}
 
 	priv->current_cs = SPI_MUX_NO_CS;
@@ -164,16 +162,7 @@ static int spi_mux_probe(struct spi_device *spi)
 	ctlr->must_async = true;
 	ctlr->defer_optimize_message = true;
 
-	ret = devm_spi_register_controller(&spi->dev, ctlr);
-	if (ret)
-		goto err_put_ctlr;
-
-	return 0;
-
-err_put_ctlr:
-	spi_controller_put(ctlr);
-
-	return ret;
+	return devm_spi_register_controller(&spi->dev, ctlr);
 }
 
 static const struct spi_device_id spi_mux_id[] = {
diff --git a/drivers/spi/spi-mxic.c b/drivers/spi/spi-mxic.c
index b0e7fc828a509..83b688e652841 100644
--- a/drivers/spi/spi-mxic.c
+++ b/drivers/spi/spi-mxic.c
@@ -403,20 +403,20 @@ static ssize_t mxic_spi_mem_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	if (WARN_ON(offs + desc->info.offset + len > U32_MAX))
 		return -EINVAL;
 
-	writel(mxic_spi_prep_hc_cfg(desc->mem->spi, 0, desc->info.op_tmpl.data.swap16),
+	writel(mxic_spi_prep_hc_cfg(desc->mem->spi, 0, desc->info.op_tmpl->data.swap16),
 	       mxic->regs + HC_CFG);
 
-	writel(mxic_spi_mem_prep_op_cfg(&desc->info.op_tmpl, len),
+	writel(mxic_spi_mem_prep_op_cfg(desc->info.op_tmpl, len),
 	       mxic->regs + LRD_CFG);
 	writel(desc->info.offset + offs, mxic->regs + LRD_ADDR);
 	len = min_t(size_t, len, mxic->linear.size);
 	writel(len, mxic->regs + LRD_RANGE);
-	writel(LMODE_CMD0(desc->info.op_tmpl.cmd.opcode) |
+	writel(LMODE_CMD0(desc->info.op_tmpl->cmd.opcode) |
 	       LMODE_SLV_ACT(spi_get_chipselect(desc->mem->spi, 0)) |
 	       LMODE_EN,
 	       mxic->regs + LRD_CTRL);
 
-	if (mxic->ecc.use_pipelined_conf && desc->info.op_tmpl.data.ecc) {
+	if (mxic->ecc.use_pipelined_conf && desc->info.op_tmpl->data.ecc) {
 		ret = mxic_ecc_process_data_pipelined(mxic->ecc.pipelined_engine,
 						      NAND_PAGE_READ,
 						      mxic->linear.dma + offs);
@@ -448,20 +448,20 @@ static ssize_t mxic_spi_mem_dirmap_write(struct spi_mem_dirmap_desc *desc,
 	if (WARN_ON(offs + desc->info.offset + len > U32_MAX))
 		return -EINVAL;
 
-	writel(mxic_spi_prep_hc_cfg(desc->mem->spi, 0, desc->info.op_tmpl.data.swap16),
+	writel(mxic_spi_prep_hc_cfg(desc->mem->spi, 0, desc->info.op_tmpl->data.swap16),
 	       mxic->regs + HC_CFG);
 
-	writel(mxic_spi_mem_prep_op_cfg(&desc->info.op_tmpl, len),
+	writel(mxic_spi_mem_prep_op_cfg(desc->info.op_tmpl, len),
 	       mxic->regs + LWR_CFG);
 	writel(desc->info.offset + offs, mxic->regs + LWR_ADDR);
 	len = min_t(size_t, len, mxic->linear.size);
 	writel(len, mxic->regs + LWR_RANGE);
-	writel(LMODE_CMD0(desc->info.op_tmpl.cmd.opcode) |
+	writel(LMODE_CMD0(desc->info.op_tmpl->cmd.opcode) |
 	       LMODE_SLV_ACT(spi_get_chipselect(desc->mem->spi, 0)) |
 	       LMODE_EN,
 	       mxic->regs + LWR_CTRL);
 
-	if (mxic->ecc.use_pipelined_conf && desc->info.op_tmpl.data.ecc) {
+	if (mxic->ecc.use_pipelined_conf && desc->info.op_tmpl->data.ecc) {
 		ret = mxic_ecc_process_data_pipelined(mxic->ecc.pipelined_engine,
 						      NAND_PAGE_WRITE,
 						      mxic->linear.dma + offs);
@@ -509,7 +509,7 @@ static int mxic_spi_mem_dirmap_create(struct spi_mem_dirmap_desc *desc)
 	if (desc->info.offset + desc->info.length > U32_MAX)
 		return -EINVAL;
 
-	if (!mxic_spi_mem_supports_op(desc->mem, &desc->info.op_tmpl))
+	if (!mxic_spi_mem_supports_op(desc->mem, desc->info.op_tmpl))
 		return -EOPNOTSUPP;
 
 	return 0;
diff --git a/drivers/spi/spi-mxs.c b/drivers/spi/spi-mxs.c
index 0164e04d59a10..fda1d260b2966 100644
--- a/drivers/spi/spi-mxs.c
+++ b/drivers/spi/spi-mxs.c
@@ -563,7 +563,7 @@ static int mxs_spi_probe(struct platform_device *pdev)
 	if (ret)
 		clk_freq = clk_freq_default;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*spi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*spi));
 	if (!host)
 		return -ENOMEM;
 
@@ -589,13 +589,12 @@ static int mxs_spi_probe(struct platform_device *pdev)
 	ret = devm_request_irq(&pdev->dev, irq_err, mxs_ssp_irq_handler, 0,
 			       dev_name(&pdev->dev), ssp);
 	if (ret)
-		goto out_host_free;
+		return ret;
 
 	ssp->dmach = dma_request_chan(&pdev->dev, "rx-tx");
 	if (IS_ERR(ssp->dmach)) {
 		dev_err(ssp->dev, "Failed to request DMA\n");
-		ret = PTR_ERR(ssp->dmach);
-		goto out_host_free;
+		return PTR_ERR(ssp->dmach);
 	}
 
 	pm_runtime_enable(ssp->dev);
@@ -635,8 +634,7 @@ static int mxs_spi_probe(struct platform_device *pdev)
 	pm_runtime_disable(ssp->dev);
 out_dma_release:
 	dma_release_channel(ssp->dmach);
-out_host_free:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -650,8 +648,6 @@ static void mxs_spi_remove(struct platform_device *pdev)
 	spi = spi_controller_get_devdata(host);
 	ssp = &spi->ssp;
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	pm_runtime_disable(&pdev->dev);
@@ -659,8 +655,6 @@ static void mxs_spi_remove(struct platform_device *pdev)
 		mxs_spi_runtime_suspend(&pdev->dev);
 
 	dma_release_channel(ssp->dmach);
-
-	spi_controller_put(host);
 }
 
 static struct platform_driver mxs_spi_driver = {
diff --git a/drivers/spi/spi-npcm-fiu.c b/drivers/spi/spi-npcm-fiu.c
index 6617751009c30..4b825044038b2 100644
--- a/drivers/spi/spi-npcm-fiu.c
+++ b/drivers/spi/spi-npcm-fiu.c
@@ -299,11 +299,11 @@ static ssize_t npcm_fiu_direct_read(struct spi_mem_dirmap_desc *desc,
 		for (i = 0 ; i < len ; i++)
 			*(buf_rx + i) = ioread8(src + i);
 	} else {
-		if (desc->info.op_tmpl.addr.buswidth != fiu->drd_op.addr.buswidth ||
-		    desc->info.op_tmpl.dummy.nbytes != fiu->drd_op.dummy.nbytes ||
-		    desc->info.op_tmpl.cmd.opcode != fiu->drd_op.cmd.opcode ||
-		    desc->info.op_tmpl.addr.nbytes != fiu->drd_op.addr.nbytes)
-			npcm_fiu_set_drd(fiu, &desc->info.op_tmpl);
+		if (desc->info.op_tmpl->addr.buswidth != fiu->drd_op.addr.buswidth ||
+		    desc->info.op_tmpl->dummy.nbytes != fiu->drd_op.dummy.nbytes ||
+		    desc->info.op_tmpl->cmd.opcode != fiu->drd_op.cmd.opcode ||
+		    desc->info.op_tmpl->addr.nbytes != fiu->drd_op.addr.nbytes)
+			npcm_fiu_set_drd(fiu, desc->info.op_tmpl);
 
 		memcpy_fromio(buf_rx, src, len);
 	}
@@ -609,7 +609,7 @@ static int npcm_fiu_dirmap_create(struct spi_mem_dirmap_desc *desc)
 	}
 
 	if (!fiu->spix_mode &&
-	    desc->info.op_tmpl.data.dir == SPI_MEM_DATA_OUT) {
+	    desc->info.op_tmpl->data.dir == SPI_MEM_DATA_OUT) {
 		desc->nodirmap = true;
 		return 0;
 	}
@@ -644,9 +644,9 @@ static int npcm_fiu_dirmap_create(struct spi_mem_dirmap_desc *desc)
 				   NPCM_FIU_CFG_FIU_FIX);
 	}
 
-	if (desc->info.op_tmpl.data.dir == SPI_MEM_DATA_IN) {
+	if (desc->info.op_tmpl->data.dir == SPI_MEM_DATA_IN) {
 		if (!fiu->spix_mode)
-			npcm_fiu_set_drd(fiu, &desc->info.op_tmpl);
+			npcm_fiu_set_drd(fiu, desc->info.op_tmpl);
 		else
 			npcm_fiux_set_direct_rd(fiu);
 
diff --git a/drivers/spi/spi-npcm-pspi.c b/drivers/spi/spi-npcm-pspi.c
index cffef0a5977d0..a437a30d636c2 100644
--- a/drivers/spi/spi-npcm-pspi.c
+++ b/drivers/spi/spi-npcm-pspi.c
@@ -345,7 +345,7 @@ static int npcm_pspi_probe(struct platform_device *pdev)
 	int irq;
 	int ret;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*priv));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*priv));
 	if (!host)
 		return -ENOMEM;
 
@@ -356,21 +356,18 @@ static int npcm_pspi_probe(struct platform_device *pdev)
 	priv->is_save_param = false;
 
 	priv->base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(priv->base)) {
-		ret = PTR_ERR(priv->base);
-		goto out_host_put;
-	}
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
 
 	priv->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(priv->clk)) {
 		dev_err(&pdev->dev, "failed to get clock\n");
-		ret = PTR_ERR(priv->clk);
-		goto out_host_put;
+		return PTR_ERR(priv->clk);
 	}
 
 	ret = clk_prepare_enable(priv->clk);
 	if (ret)
-		goto out_host_put;
+		return ret;
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0) {
@@ -424,8 +421,6 @@ static int npcm_pspi_probe(struct platform_device *pdev)
 out_disable_clk:
 	clk_disable_unprepare(priv->clk);
 
-out_host_put:
-	spi_controller_put(host);
 	return ret;
 }
 
@@ -434,14 +429,10 @@ static void npcm_pspi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct npcm_pspi *priv = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	npcm_pspi_reset_hw(priv);
 	clk_disable_unprepare(priv->clk);
-
-	spi_controller_put(host);
 }
 
 static const struct of_device_id npcm_pspi_match[] = {
diff --git a/drivers/spi/spi-nxp-xspi.c b/drivers/spi/spi-nxp-xspi.c
index 385302a6e62f2..037eac24e6fd2 100644
--- a/drivers/spi/spi-nxp-xspi.c
+++ b/drivers/spi/spi-nxp-xspi.c
@@ -493,9 +493,8 @@ static void nxp_xspi_disable_ddr(struct nxp_xspi *xspi)
 	writel(reg, base + XSPI_MCR);
 
 	reg &= ~XSPI_MCR_DDR_EN;
-	reg &= ~XSPI_MCR_DQS_FA_SEL_MASK;
 	/* Use dummy pad loopback mode to sample data */
-	reg |= FIELD_PREP(XSPI_MCR_DQS_FA_SEL_MASK, 0x01);
+	FIELD_MODIFY(XSPI_MCR_DQS_FA_SEL_MASK, &reg, 0x01);
 	writel(reg, base + XSPI_MCR);
 	xspi->support_max_rate = 133000000;
 
@@ -524,15 +523,13 @@ static void nxp_xspi_enable_ddr(struct nxp_xspi *xspi)
 	writel(reg, base + XSPI_MCR);
 
 	reg |= XSPI_MCR_DDR_EN;
-	reg &= ~XSPI_MCR_DQS_FA_SEL_MASK;
 	/* Use external dqs to sample data */
-	reg |= FIELD_PREP(XSPI_MCR_DQS_FA_SEL_MASK, 0x03);
+	FIELD_MODIFY(XSPI_MCR_DQS_FA_SEL_MASK, &reg, 0x03);
 	writel(reg, base + XSPI_MCR);
 	xspi->support_max_rate = 200000000;
 
 	reg = readl(base + XSPI_FLSHCR);
-	reg &= ~XSPI_FLSHCR_TDH_MASK;
-	reg |= FIELD_PREP(XSPI_FLSHCR_TDH_MASK, 0x01);
+	FIELD_MODIFY(XSPI_FLSHCR_TDH_MASK, &reg, 0x01);
 	writel(reg, base + XSPI_FLSHCR);
 
 	reg = FIELD_PREP(XSPI_SMPR_DLLFSMPFA_MASK, 0x04);
@@ -1096,8 +1093,7 @@ static int nxp_xspi_default_setup(struct nxp_xspi *xspi)
 
 	/* Give read/write access right to EENV0 */
 	reg = readl(base + XSPI_FRAD0_WORD2);
-	reg &= ~XSPI_FRAD0_WORD2_MD0ACP_MASK;
-	reg |= FIELD_PREP(XSPI_FRAD0_WORD2_MD0ACP_MASK, 0x03);
+	FIELD_MODIFY(XSPI_FRAD0_WORD2_MD0ACP_MASK, &reg, 0x03);
 	writel(reg, base + XSPI_FRAD0_WORD2);
 
 	/* Enable the FRAD check for EENV0 */
diff --git a/drivers/spi/spi-omap2-mcspi.c b/drivers/spi/spi-omap2-mcspi.c
index 56b30ff58771a..d53f98aa0aacb 100644
--- a/drivers/spi/spi-omap2-mcspi.c
+++ b/drivers/spi/spi-omap2-mcspi.c
@@ -1484,9 +1484,9 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 	const struct of_device_id *match;
 
 	if (of_property_read_bool(node, "spi-slave"))
-		ctlr = spi_alloc_target(&pdev->dev, sizeof(*mcspi));
+		ctlr = devm_spi_alloc_target(&pdev->dev, sizeof(*mcspi));
 	else
-		ctlr = spi_alloc_host(&pdev->dev, sizeof(*mcspi));
+		ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(*mcspi));
 	if (!ctlr)
 		return -ENOMEM;
 
@@ -1530,10 +1530,9 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 	}
 
 	mcspi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
-	if (IS_ERR(mcspi->base)) {
-		status = PTR_ERR(mcspi->base);
-		goto free_ctlr;
-	}
+	if (IS_ERR(mcspi->base))
+		return PTR_ERR(mcspi->base);
+
 	mcspi->phys = r->start + regs_offset;
 	mcspi->base += regs_offset;
 
@@ -1544,10 +1543,8 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 	mcspi->dma_channels = devm_kcalloc(&pdev->dev, ctlr->num_chipselect,
 					   sizeof(struct omap2_mcspi_dma),
 					   GFP_KERNEL);
-	if (mcspi->dma_channels == NULL) {
-		status = -ENOMEM;
-		goto free_ctlr;
-	}
+	if (mcspi->dma_channels == NULL)
+		return -ENOMEM;
 
 	for (i = 0; i < ctlr->num_chipselect; i++) {
 		sprintf(mcspi->dma_channels[i].dma_rx_ch_name, "rx%d", i);
@@ -1556,26 +1553,27 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 		status = omap2_mcspi_request_dma(mcspi,
 						 &mcspi->dma_channels[i]);
 		if (status == -EPROBE_DEFER)
-			goto free_ctlr;
+			goto err_release_dma;
 	}
 
 	status = platform_get_irq(pdev, 0);
 	if (status < 0)
-		goto free_ctlr;
+		goto err_release_dma;
+
 	init_completion(&mcspi->txdone);
 	status = devm_request_irq(&pdev->dev, status,
 				  omap2_mcspi_irq_handler, 0, pdev->name,
 				  mcspi);
 	if (status) {
 		dev_err(&pdev->dev, "Cannot request IRQ");
-		goto free_ctlr;
+		goto err_release_dma;
 	}
 
 	mcspi->ref_clk = devm_clk_get_optional_enabled(&pdev->dev, NULL);
 	if (IS_ERR(mcspi->ref_clk)) {
 		status = PTR_ERR(mcspi->ref_clk);
 		dev_err_probe(&pdev->dev, status, "Failed to get ref_clk");
-		goto free_ctlr;
+		goto err_release_dma;
 	}
 	if (mcspi->ref_clk)
 		mcspi->ref_clk_hz = clk_get_rate(mcspi->ref_clk);
@@ -1590,21 +1588,21 @@ static int omap2_mcspi_probe(struct platform_device *pdev)
 
 	status = omap2_mcspi_controller_setup(mcspi);
 	if (status < 0)
-		goto disable_pm;
+		goto err_disable_rpm;
 
 	status = spi_register_controller(ctlr);
 	if (status < 0)
-		goto disable_pm;
+		goto err_disable_rpm;
 
-	return status;
+	return 0;
 
-disable_pm:
+err_disable_rpm:
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-free_ctlr:
+err_release_dma:
 	omap2_mcspi_release_dma(ctlr);
-	spi_controller_put(ctlr);
+
 	return status;
 }
 
@@ -1613,8 +1611,6 @@ static void omap2_mcspi_remove(struct platform_device *pdev)
 	struct spi_controller *ctlr = platform_get_drvdata(pdev);
 	struct omap2_mcspi *mcspi = spi_controller_get_devdata(ctlr);
 
-	spi_controller_get(ctlr);
-
 	spi_unregister_controller(ctlr);
 
 	omap2_mcspi_release_dma(ctlr);
@@ -1622,8 +1618,6 @@ static void omap2_mcspi_remove(struct platform_device *pdev)
 	pm_runtime_dont_use_autosuspend(mcspi->dev);
 	pm_runtime_put_sync(mcspi->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	spi_controller_put(ctlr);
 }
 
 /* work with hotplug and coldplug */
diff --git a/drivers/spi/spi-orion.c b/drivers/spi/spi-orion.c
index 64bf215c18047..265708a949843 100644
--- a/drivers/spi/spi-orion.c
+++ b/drivers/spi/spi-orion.c
@@ -651,11 +651,9 @@ static int orion_spi_probe(struct platform_device *pdev)
 	struct device_node *np;
 	int status;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*spi));
-	if (host == NULL) {
-		dev_dbg(&pdev->dev, "host allocation failed\n");
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*spi));
+	if (host == NULL)
 		return -ENOMEM;
-	}
 
 	if (pdev->id != -1)
 		host->bus_num = pdev->id;
@@ -689,17 +687,14 @@ static int orion_spi_probe(struct platform_device *pdev)
 	spi->devdata = devdata;
 
 	spi->clk = devm_clk_get_enabled(&pdev->dev, NULL);
-	if (IS_ERR(spi->clk)) {
-		status = PTR_ERR(spi->clk);
-		goto out;
-	}
+	if (IS_ERR(spi->clk))
+		return PTR_ERR(spi->clk);
 
 	/* The following clock is only used by some SoCs */
 	spi->axi_clk = devm_clk_get(&pdev->dev, "axi");
-	if (PTR_ERR(spi->axi_clk) == -EPROBE_DEFER) {
-		status = -EPROBE_DEFER;
-		goto out;
-	}
+	if (PTR_ERR(spi->axi_clk) == -EPROBE_DEFER)
+		return -EPROBE_DEFER;
+
 	if (!IS_ERR(spi->axi_clk))
 		clk_prepare_enable(spi->axi_clk);
 
@@ -796,8 +791,7 @@ static int orion_spi_probe(struct platform_device *pdev)
 	pm_runtime_dont_use_autosuspend(&pdev->dev);
 out_rel_axi_clk:
 	clk_disable_unprepare(spi->axi_clk);
-out:
-	spi_controller_put(host);
+
 	return status;
 }
 
@@ -807,15 +801,11 @@ static void orion_spi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct orion_spi *spi = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	pm_runtime_get_sync(&pdev->dev);
 	clk_disable_unprepare(spi->axi_clk);
 
-	spi_controller_put(host);
-
 	pm_runtime_disable(&pdev->dev);
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_set_suspended(&pdev->dev);
diff --git a/drivers/spi/spi-pci1xxxx.c b/drivers/spi/spi-pci1xxxx.c
index 8577a19705de4..af6ed78493e3f 100644
--- a/drivers/spi/spi-pci1xxxx.c
+++ b/drivers/spi/spi-pci1xxxx.c
@@ -173,27 +173,27 @@ struct pci1xxxx_spi {
 };
 
 static const struct pci_device_id pci1xxxx_spi_pci_id_table[] = {
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa004, PCI_ANY_ID, 0x0001), 0, 0, 0x02},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa004, PCI_ANY_ID, 0x0002), 0, 0, 0x01},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa004, PCI_ANY_ID, 0x0003), 0, 0, 0x11},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa004, PCI_ANY_ID, PCI_ANY_ID), 0, 0, 0x01},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa014, PCI_ANY_ID, 0x0001), 0, 0, 0x02},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa014, PCI_ANY_ID, 0x0002), 0, 0, 0x01},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa014, PCI_ANY_ID, 0x0003), 0, 0, 0x11},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa014, PCI_ANY_ID, PCI_ANY_ID), 0, 0, 0x01},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa024, PCI_ANY_ID, 0x0001), 0, 0, 0x02},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa024, PCI_ANY_ID, 0x0002), 0, 0, 0x01},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa024, PCI_ANY_ID, 0x0003), 0, 0, 0x11},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa024, PCI_ANY_ID, PCI_ANY_ID), 0, 0, 0x01},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa034, PCI_ANY_ID, 0x0001), 0, 0, 0x02},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa034, PCI_ANY_ID, 0x0002), 0, 0, 0x01},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa034, PCI_ANY_ID, 0x0003), 0, 0, 0x11},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa034, PCI_ANY_ID, PCI_ANY_ID), 0, 0, 0x01},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa044, PCI_ANY_ID, 0x0001), 0, 0, 0x02},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa044, PCI_ANY_ID, 0x0002), 0, 0, 0x01},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa044, PCI_ANY_ID, 0x0003), 0, 0, 0x11},
-	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa044, PCI_ANY_ID, PCI_ANY_ID), 0, 0, 0x01},
-	{ 0, }
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa004, PCI_ANY_ID, 0x0001), .driver_data = 0x02 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa004, PCI_ANY_ID, 0x0002), .driver_data = 0x01 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa004, PCI_ANY_ID, 0x0003), .driver_data = 0x11 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa004, PCI_ANY_ID, PCI_ANY_ID), .driver_data = 0x01 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa014, PCI_ANY_ID, 0x0001), .driver_data = 0x02 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa014, PCI_ANY_ID, 0x0002), .driver_data = 0x01 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa014, PCI_ANY_ID, 0x0003), .driver_data = 0x11 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa014, PCI_ANY_ID, PCI_ANY_ID), .driver_data = 0x01 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa024, PCI_ANY_ID, 0x0001), .driver_data = 0x02 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa024, PCI_ANY_ID, 0x0002), .driver_data = 0x01 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa024, PCI_ANY_ID, 0x0003), .driver_data = 0x11 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa024, PCI_ANY_ID, PCI_ANY_ID), .driver_data = 0x01 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa034, PCI_ANY_ID, 0x0001), .driver_data = 0x02 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa034, PCI_ANY_ID, 0x0002), .driver_data = 0x01 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa034, PCI_ANY_ID, 0x0003), .driver_data = 0x11 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa034, PCI_ANY_ID, PCI_ANY_ID), .driver_data = 0x01 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa044, PCI_ANY_ID, 0x0001), .driver_data = 0x02 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa044, PCI_ANY_ID, 0x0002), .driver_data = 0x01 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa044, PCI_ANY_ID, 0x0003), .driver_data = 0x11 },
+	{ PCI_DEVICE_SUB(VENDOR_ID_MCHP, 0xa044, PCI_ANY_ID, PCI_ANY_ID), .driver_data = 0x01 },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, pci1xxxx_spi_pci_id_table);
diff --git a/drivers/spi/spi-pic32-sqi.c b/drivers/spi/spi-pic32-sqi.c
index 41662992dbe50..5d3921e29461c 100644
--- a/drivers/spi/spi-pic32-sqi.c
+++ b/drivers/spi/spi-pic32-sqi.c
@@ -572,7 +572,7 @@ static int pic32_sqi_probe(struct platform_device *pdev)
 	struct pic32_sqi *sqi;
 	int ret;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*sqi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*sqi));
 	if (!host)
 		return -ENOMEM;
 
@@ -580,31 +580,25 @@ static int pic32_sqi_probe(struct platform_device *pdev)
 	sqi->host = host;
 
 	sqi->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(sqi->regs)) {
-		ret = PTR_ERR(sqi->regs);
-		goto err_free_host;
-	}
+	if (IS_ERR(sqi->regs))
+		return PTR_ERR(sqi->regs);
 
 	/* irq */
 	sqi->irq = platform_get_irq(pdev, 0);
-	if (sqi->irq < 0) {
-		ret = sqi->irq;
-		goto err_free_host;
-	}
+	if (sqi->irq < 0)
+		return sqi->irq;
 
 	/* clocks */
 	sqi->sys_clk = devm_clk_get_enabled(&pdev->dev, "reg_ck");
 	if (IS_ERR(sqi->sys_clk)) {
-		ret = PTR_ERR(sqi->sys_clk);
 		dev_err(&pdev->dev, "no sys_clk ?\n");
-		goto err_free_host;
+		return PTR_ERR(sqi->sys_clk);
 	}
 
 	sqi->base_clk = devm_clk_get_enabled(&pdev->dev, "spi_ck");
 	if (IS_ERR(sqi->base_clk)) {
-		ret = PTR_ERR(sqi->base_clk);
 		dev_err(&pdev->dev, "no base clk ?\n");
-		goto err_free_host;
+		return PTR_ERR(sqi->base_clk);
 	}
 
 	init_completion(&sqi->xfer_done);
@@ -616,7 +610,7 @@ static int pic32_sqi_probe(struct platform_device *pdev)
 	ret = ring_desc_ring_alloc(sqi);
 	if (ret) {
 		dev_err(&pdev->dev, "ring alloc failed\n");
-		goto err_free_host;
+		return ret;
 	}
 
 	/* install irq handlers */
@@ -656,8 +650,6 @@ static int pic32_sqi_probe(struct platform_device *pdev)
 err_free_ring:
 	ring_desc_ring_free(sqi);
 
-err_free_host:
-	spi_controller_put(host);
 	return ret;
 }
 
@@ -665,15 +657,11 @@ static void pic32_sqi_remove(struct platform_device *pdev)
 {
 	struct pic32_sqi *sqi = platform_get_drvdata(pdev);
 
-	spi_controller_get(sqi->host);
-
 	spi_unregister_controller(sqi->host);
 
 	/* release resources */
 	free_irq(sqi->irq, sqi);
 	ring_desc_ring_free(sqi);
-
-	spi_controller_put(sqi->host);
 }
 
 static const struct of_device_id pic32_sqi_of_ids[] = {
diff --git a/drivers/spi/spi-pic32.c b/drivers/spi/spi-pic32.c
index 70427e5299456..972128271e4bf 100644
--- a/drivers/spi/spi-pic32.c
+++ b/drivers/spi/spi-pic32.c
@@ -752,7 +752,7 @@ static int pic32_spi_probe(struct platform_device *pdev)
 	struct pic32_spi *pic32s;
 	int ret;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*pic32s));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*pic32s));
 	if (!host)
 		return -ENOMEM;
 
@@ -761,7 +761,7 @@ static int pic32_spi_probe(struct platform_device *pdev)
 
 	ret = pic32_spi_hw_probe(pdev, pic32s);
 	if (ret)
-		goto err_host;
+		return ret;
 
 	host->dev.of_node	= pdev->dev.of_node;
 	host->mode_bits	= SPI_MODE_3 | SPI_MODE_0 | SPI_CS_HIGH;
@@ -833,8 +833,7 @@ static int pic32_spi_probe(struct platform_device *pdev)
 
 err_bailout:
 	pic32_spi_dma_unprep(pic32s);
-err_host:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -842,14 +841,10 @@ static void pic32_spi_remove(struct platform_device *pdev)
 {
 	struct pic32_spi *pic32s = platform_get_drvdata(pdev);
 
-	spi_controller_get(pic32s->host);
-
 	spi_unregister_controller(pic32s->host);
 
 	pic32_spi_disable(pic32s);
 	pic32_spi_dma_unprep(pic32s);
-
-	spi_controller_put(pic32s->host);
 }
 
 static const struct of_device_id pic32_spi_of_match[] = {
diff --git a/drivers/spi/spi-pl022.c b/drivers/spi/spi-pl022.c
index 9c0211f94fd07..95652df5fd092 100644
--- a/drivers/spi/spi-pl022.c
+++ b/drivers/spi/spi-pl022.c
@@ -1868,7 +1868,7 @@ static int pl022_probe(struct amba_device *adev, const struct amba_id *id)
 	}
 
 	/* Allocate host with space for data */
-	host = spi_alloc_host(dev, sizeof(struct pl022));
+	host = devm_spi_alloc_host(dev, sizeof(struct pl022));
 	if (host == NULL) {
 		dev_err(&adev->dev, "probe - cannot alloc SPI host\n");
 		return -ENOMEM;
@@ -1907,7 +1907,7 @@ static int pl022_probe(struct amba_device *adev, const struct amba_id *id)
 
 	status = amba_request_regions(adev, NULL);
 	if (status)
-		goto err_no_ioregion;
+		return status;
 
 	pl022->phybase = adev->res.start;
 	pl022->virtbase = devm_ioremap(dev, adev->res.start,
@@ -1984,8 +1984,7 @@ static int pl022_probe(struct amba_device *adev, const struct amba_id *id)
  err_no_clk:
  err_no_ioremap:
 	amba_release_regions(adev);
- err_no_ioregion:
-	spi_controller_put(host);
+
 	return status;
 }
 
@@ -1997,8 +1996,6 @@ pl022_remove(struct amba_device *adev)
 	if (!pl022)
 		return;
 
-	spi_controller_get(pl022->host);
-
 	spi_unregister_controller(pl022->host);
 
 	/*
@@ -2012,8 +2009,6 @@ pl022_remove(struct amba_device *adev)
 		pl022_dma_remove(pl022);
 
 	amba_release_regions(adev);
-
-	spi_controller_put(pl022->host);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/spi/spi-qcom-qspi.c b/drivers/spi/spi-qcom-qspi.c
index 7e39038160e00..caf55a6f70b31 100644
--- a/drivers/spi/spi-qcom-qspi.c
+++ b/drivers/spi/spi-qcom-qspi.c
@@ -174,6 +174,7 @@ struct qcom_qspi {
 	void *virt_cmd_desc[QSPI_MAX_SG];
 	unsigned int n_cmd_desc;
 	struct icc_path *icc_path_cpu_to_qspi;
+	struct icc_path *icc_path_mem;
 	unsigned long last_speed;
 	/* Lock to protect data accessed by IRQs */
 	spinlock_t lock;
@@ -272,7 +273,7 @@ static void qcom_qspi_handle_err(struct spi_controller *host,
 static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
 {
 	int ret;
-	unsigned int avg_bw_cpu;
+	unsigned int avg_bw_cpu, avg_bw_mem;
 
 	if (speed_hz == ctrl->last_speed)
 		return 0;
@@ -285,7 +286,7 @@ static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
 	}
 
 	/*
-	 * Set BW quota for CPU.
+	 * Set BW quota for CPU and memory paths.
 	 * We don't have explicit peak requirement so keep it equal to avg_bw.
 	 */
 	avg_bw_cpu = Bps_to_icc(speed_hz);
@@ -296,6 +297,13 @@ static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)
 		return ret;
 	}
 
+	avg_bw_mem = Bps_to_icc(speed_hz);
+	ret = icc_set_bw(ctrl->icc_path_mem, avg_bw_mem, avg_bw_mem);
+	if (ret) {
+		dev_err(ctrl->dev, "ICC BW voting failed for memory: %d\n", ret);
+		return ret;
+	}
+
 	ctrl->last_speed = speed_hz;
 
 	return 0;
@@ -729,6 +737,14 @@ static int qcom_qspi_probe(struct platform_device *pdev)
 		return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),
 				     "Failed to get cpu path\n");
 
+	ctrl->icc_path_mem = devm_of_icc_get(dev, "qspi-memory");
+	if (IS_ERR(ctrl->icc_path_mem)) {
+		if (PTR_ERR(ctrl->icc_path_mem) != -ENODATA)
+			return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_mem),
+					     "Failed to get memory path\n");
+		ctrl->icc_path_mem = NULL;
+	}
+
 	/* Set BW vote for register access */
 	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),
 				Bps_to_icc(1000));
@@ -818,20 +834,42 @@ static int __maybe_unused qcom_qspi_runtime_suspend(struct device *dev)
 	struct qcom_qspi *ctrl = spi_controller_get_devdata(host);
 	int ret;
 
-	/* Drop the performance state vote */
-	dev_pm_opp_set_rate(dev, 0);
 	clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);
 
 	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);
 	if (ret) {
 		dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",
 			__func__, ret);
-		return ret;
+		goto err_enable_clk;
 	}
 
-	pinctrl_pm_select_sleep_state(dev);
+	ret = icc_disable(ctrl->icc_path_mem);
+	if (ret) {
+		dev_err_ratelimited(ctrl->dev, "ICC disable failed for memory: %d\n", ret);
+		goto err_enable_icc_cpu;
+	}
+
+	ret = pinctrl_pm_select_sleep_state(dev);
+	if (ret)
+		goto err_enable_icc_mem;
+
+	/* Drop the performance state vote */
+	ret = dev_pm_opp_set_rate(dev, 0);
+	if (ret)
+		goto err_select_default_state;
 
 	return 0;
+
+err_select_default_state:
+	pinctrl_pm_select_default_state(dev);
+err_enable_icc_mem:
+	icc_enable(ctrl->icc_path_mem);
+err_enable_icc_cpu:
+	icc_enable(ctrl->icc_path_cpu_to_qspi);
+err_enable_clk:
+	if (clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks))
+		dev_err_ratelimited(ctrl->dev, "Failed to re-enable clocks\n");
+	return ret;
 }
 
 static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
@@ -840,20 +878,42 @@ static int __maybe_unused qcom_qspi_runtime_resume(struct device *dev)
 	struct qcom_qspi *ctrl = spi_controller_get_devdata(host);
 	int ret;
 
-	pinctrl_pm_select_default_state(dev);
+	ret = dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
+	if (ret)
+		return ret;
+
+	ret = pinctrl_pm_select_default_state(dev);
+	if (ret)
+		goto err_opp_set_rate_zero;
 
 	ret = icc_enable(ctrl->icc_path_cpu_to_qspi);
 	if (ret) {
 		dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",
 			__func__, ret);
-		return ret;
+		goto err_select_sleep_state;
+	}
+
+	ret = icc_enable(ctrl->icc_path_mem);
+	if (ret) {
+		dev_err_ratelimited(ctrl->dev, "ICC enable failed for memory: %d\n", ret);
+		goto err_disable_icc_cpu;
 	}
 
 	ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);
 	if (ret)
-		return ret;
+		goto err_disable_icc_mem;
+
+	return 0;
 
-	return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);
+err_disable_icc_mem:
+	icc_disable(ctrl->icc_path_mem);
+err_disable_icc_cpu:
+	icc_disable(ctrl->icc_path_cpu_to_qspi);
+err_select_sleep_state:
+	pinctrl_pm_select_sleep_state(dev);
+err_opp_set_rate_zero:
+	dev_pm_opp_set_rate(dev, 0);
+	return ret;
 }
 
 static int __maybe_unused qcom_qspi_suspend(struct device *dev)
diff --git a/drivers/spi/spi-qup.c b/drivers/spi/spi-qup.c
index 45d9b4cb75e4a..034546b3f0582 100644
--- a/drivers/spi/spi-qup.c
+++ b/drivers/spi/spi-qup.c
@@ -996,8 +996,11 @@ static int spi_qup_init_dma(struct spi_controller *host, resource_size_t base)
 
 err:
 	dma_release_channel(host->dma_tx);
+	host->dma_tx = NULL;
 err_tx:
 	dma_release_channel(host->dma_rx);
+	host->dma_rx = NULL;
+
 	return ret;
 }
 
@@ -1071,11 +1074,9 @@ static int spi_qup_probe(struct platform_device *pdev)
 	if (ret && ret != -ENODEV)
 		return dev_err_probe(dev, ret, "invalid OPP table\n");
 
-	host = spi_alloc_host(dev, sizeof(struct spi_qup));
-	if (!host) {
-		dev_err(dev, "cannot allocate host\n");
+	host = devm_spi_alloc_host(dev, sizeof(struct spi_qup));
+	if (!host)
 		return -ENOMEM;
-	}
 
 	/* use num-cs unless not present or out of range */
 	if (of_property_read_u32(dev->of_node, "num-cs", &num_cs) ||
@@ -1108,7 +1109,7 @@ static int spi_qup_probe(struct platform_device *pdev)
 
 	ret = spi_qup_init_dma(host, res->start);
 	if (ret == -EPROBE_DEFER)
-		goto error;
+		return ret;
 	else if (!ret)
 		host->can_dma = spi_qup_can_dma;
 
@@ -1206,8 +1207,7 @@ static int spi_qup_probe(struct platform_device *pdev)
 	clk_disable_unprepare(iclk);
 error_dma:
 	spi_qup_release_dma(host);
-error:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -1320,8 +1320,6 @@ static void spi_qup_remove(struct platform_device *pdev)
 	struct spi_qup *controller = spi_controller_get_devdata(host);
 	int ret;
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	ret = pm_runtime_get_sync(&pdev->dev);
@@ -1343,8 +1341,6 @@ static void spi_qup_remove(struct platform_device *pdev)
 
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	spi_controller_put(host);
 }
 
 static const struct of_device_id spi_qup_dt_match[] = {
diff --git a/drivers/spi/spi-rpc-if.c b/drivers/spi/spi-rpc-if.c
index 6edc0c4db854d..1ef7bd91b3b37 100644
--- a/drivers/spi/spi-rpc-if.c
+++ b/drivers/spi/spi-rpc-if.c
@@ -83,7 +83,7 @@ static ssize_t xspi_spi_mem_dirmap_write(struct spi_mem_dirmap_desc *desc,
 	if (offs + desc->info.offset + len > U32_MAX)
 		return -EINVAL;
 
-	rpcif_spi_mem_prepare(desc->mem->spi, &desc->info.op_tmpl, &offs, &len);
+	rpcif_spi_mem_prepare(desc->mem->spi, desc->info.op_tmpl, &offs, &len);
 
 	return xspi_dirmap_write(rpc->dev, offs, len, buf);
 }
@@ -97,7 +97,7 @@ static ssize_t rpcif_spi_mem_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	if (offs + desc->info.offset + len > U32_MAX)
 		return -EINVAL;
 
-	rpcif_spi_mem_prepare(desc->mem->spi, &desc->info.op_tmpl, &offs, &len);
+	rpcif_spi_mem_prepare(desc->mem->spi, desc->info.op_tmpl, &offs, &len);
 
 	return rpcif_dirmap_read(rpc->dev, offs, len, buf);
 }
@@ -110,13 +110,13 @@ static int rpcif_spi_mem_dirmap_create(struct spi_mem_dirmap_desc *desc)
 	if (desc->info.offset + desc->info.length > U32_MAX)
 		return -EINVAL;
 
-	if (!rpcif_spi_mem_supports_op(desc->mem, &desc->info.op_tmpl))
+	if (!rpcif_spi_mem_supports_op(desc->mem, desc->info.op_tmpl))
 		return -EOPNOTSUPP;
 
 	if (!rpc->dirmap)
 		return -EOPNOTSUPP;
 
-	if (!rpc->xspi && desc->info.op_tmpl.data.dir != SPI_MEM_DATA_IN)
+	if (!rpc->xspi && desc->info.op_tmpl->data.dir != SPI_MEM_DATA_IN)
 		return -EOPNOTSUPP;
 
 	return 0;
diff --git a/drivers/spi/spi-rspi.c b/drivers/spi/spi-rspi.c
index a8180dece716f..a0c77e02bc90d 100644
--- a/drivers/spi/spi-rspi.c
+++ b/drivers/spi/spi-rspi.c
@@ -1171,14 +1171,10 @@ static void rspi_remove(struct platform_device *pdev)
 {
 	struct rspi_data *rspi = platform_get_drvdata(pdev);
 
-	spi_controller_get(rspi->ctlr);
-
 	spi_unregister_controller(rspi->ctlr);
 
 	rspi_release_dma(rspi->ctlr);
 	pm_runtime_disable(&pdev->dev);
-
-	spi_controller_put(rspi->ctlr);
 }
 
 static const struct spi_ops rspi_ops = {
@@ -1226,11 +1222,6 @@ static const struct of_device_id rspi_of_match[] __maybe_unused = {
 MODULE_DEVICE_TABLE(of, rspi_of_match);
 
 #ifdef CONFIG_OF
-static void rspi_reset_control_assert(void *data)
-{
-	reset_control_assert(data);
-}
-
 static int rspi_parse_dt(struct device *dev, struct spi_controller *ctlr)
 {
 	struct reset_control *rstc;
@@ -1246,22 +1237,10 @@ static int rspi_parse_dt(struct device *dev, struct spi_controller *ctlr)
 
 	ctlr->num_chipselect = num_cs;
 
-	rstc = devm_reset_control_get_optional_exclusive(dev, NULL);
+	rstc = devm_reset_control_get_optional_exclusive_deasserted(dev, NULL);
 	if (IS_ERR(rstc))
 		return dev_err_probe(dev, PTR_ERR(rstc),
-					     "failed to get reset ctrl\n");
-
-	error = reset_control_deassert(rstc);
-	if (error) {
-		dev_err(dev, "failed to deassert reset %d\n", error);
-		return error;
-	}
-
-	error = devm_add_action_or_reset(dev, rspi_reset_control_assert, rstc);
-	if (error) {
-		dev_err(dev, "failed to register assert devm action, %d\n", error);
-		return error;
-	}
+				     "failed to get reset ctrl and deassert reset\n");
 
 	return 0;
 }
@@ -1294,7 +1273,7 @@ static int rspi_probe(struct platform_device *pdev)
 	const struct spi_ops *ops;
 	unsigned long clksrc;
 
-	ctlr = spi_alloc_host(&pdev->dev, sizeof(struct rspi_data));
+	ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(struct rspi_data));
 	if (ctlr == NULL)
 		return -ENOMEM;
 
@@ -1302,7 +1281,7 @@ static int rspi_probe(struct platform_device *pdev)
 	if (ops) {
 		ret = rspi_parse_dt(&pdev->dev, ctlr);
 		if (ret)
-			goto error1;
+			return ret;
 	} else {
 		ops = (struct spi_ops *)pdev->id_entry->driver_data;
 		ctlr->num_chipselect = 2; /* default */
@@ -1314,16 +1293,13 @@ static int rspi_probe(struct platform_device *pdev)
 	rspi->ctlr = ctlr;
 
 	rspi->addr = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
-	if (IS_ERR(rspi->addr)) {
-		ret = PTR_ERR(rspi->addr);
-		goto error1;
-	}
+	if (IS_ERR(rspi->addr))
+		return PTR_ERR(rspi->addr);
 
 	rspi->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(rspi->clk)) {
 		dev_err(&pdev->dev, "cannot get clock\n");
-		ret = PTR_ERR(rspi->clk);
-		goto error1;
+		return PTR_ERR(rspi->clk);
 	}
 
 	rspi->pdev = pdev;
@@ -1396,8 +1372,6 @@ static int rspi_probe(struct platform_device *pdev)
 	rspi_release_dma(ctlr);
 error2:
 	pm_runtime_disable(&pdev->dev);
-error1:
-	spi_controller_put(ctlr);
 
 	return ret;
 }
diff --git a/drivers/spi/spi-sh-hspi.c b/drivers/spi/spi-sh-hspi.c
index 1e3ca718ca736..f840467cfdb2c 100644
--- a/drivers/spi/spi-sh-hspi.c
+++ b/drivers/spi/spi-sh-hspi.c
@@ -224,15 +224,14 @@ static int hspi_probe(struct platform_device *pdev)
 		return -EINVAL;
 	}
 
-	ctlr = spi_alloc_host(&pdev->dev, sizeof(*hspi));
+	ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(*hspi));
 	if (!ctlr)
 		return -ENOMEM;
 
 	clk = clk_get(&pdev->dev, NULL);
 	if (IS_ERR(clk)) {
 		dev_err(&pdev->dev, "couldn't get clock\n");
-		ret = -EINVAL;
-		goto error0;
+		return PTR_ERR(clk);
 	}
 
 	hspi = spi_controller_get_devdata(ctlr);
@@ -269,8 +268,6 @@ static int hspi_probe(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
  error1:
 	clk_put(clk);
- error0:
-	spi_controller_put(ctlr);
 
 	return ret;
 }
@@ -279,15 +276,11 @@ static void hspi_remove(struct platform_device *pdev)
 {
 	struct hspi_priv *hspi = platform_get_drvdata(pdev);
 
-	spi_controller_get(hspi->ctlr);
-
 	spi_unregister_controller(hspi->ctlr);
 
 	pm_runtime_disable(&pdev->dev);
 
 	clk_put(hspi->clk);
-
-	spi_controller_put(hspi->ctlr);
 }
 
 static const struct of_device_id hspi_of_match[] = {
diff --git a/drivers/spi/spi-sh-msiof.c b/drivers/spi/spi-sh-msiof.c
index f114b6313f4f6..070e16bc764f6 100644
--- a/drivers/spi/spi-sh-msiof.c
+++ b/drivers/spi/spi-sh-msiof.c
@@ -1215,9 +1215,9 @@ static int sh_msiof_spi_probe(struct platform_device *pdev)
 		info->dtdl = 200;
 
 	if (info->mode == MSIOF_SPI_TARGET)
-		ctlr = spi_alloc_target(dev, sizeof(struct sh_msiof_spi_priv));
+		ctlr = devm_spi_alloc_target(dev, sizeof(struct sh_msiof_spi_priv));
 	else
-		ctlr = spi_alloc_host(dev, sizeof(struct sh_msiof_spi_priv));
+		ctlr = devm_spi_alloc_host(dev, sizeof(struct sh_msiof_spi_priv));
 	if (ctlr == NULL)
 		return -ENOMEM;
 
@@ -1234,26 +1234,21 @@ static int sh_msiof_spi_probe(struct platform_device *pdev)
 	p->clk = devm_clk_get(dev, NULL);
 	if (IS_ERR(p->clk)) {
 		dev_err(dev, "cannot get clock\n");
-		ret = PTR_ERR(p->clk);
-		goto err1;
+		return PTR_ERR(p->clk);
 	}
 
 	i = platform_get_irq(pdev, 0);
-	if (i < 0) {
-		ret = i;
-		goto err1;
-	}
+	if (i < 0)
+		return i;
 
 	p->mapbase = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(p->mapbase)) {
-		ret = PTR_ERR(p->mapbase);
-		goto err1;
-	}
+	if (IS_ERR(p->mapbase))
+		return PTR_ERR(p->mapbase);
 
 	ret = devm_request_irq(dev, i, sh_msiof_spi_irq, 0, dev_name(dev), p);
 	if (ret) {
 		dev_err(dev, "unable to request irq\n");
-		goto err1;
+		return ret;
 	}
 
 	p->pdev = pdev;
@@ -1300,8 +1295,7 @@ static int sh_msiof_spi_probe(struct platform_device *pdev)
  err2:
 	sh_msiof_release_dma(p);
 	pm_runtime_disable(dev);
- err1:
-	spi_controller_put(ctlr);
+
 	return ret;
 }
 
@@ -1309,14 +1303,10 @@ static void sh_msiof_spi_remove(struct platform_device *pdev)
 {
 	struct sh_msiof_spi_priv *p = platform_get_drvdata(pdev);
 
-	spi_controller_get(p->ctlr);
-
 	spi_unregister_controller(p->ctlr);
 
 	sh_msiof_release_dma(p);
 	pm_runtime_disable(&pdev->dev);
-
-	spi_controller_put(p->ctlr);
 }
 
 static const struct platform_device_id spi_driver_ids[] = {
diff --git a/drivers/spi/spi-sifive.c b/drivers/spi/spi-sifive.c
index 74a3e32fd2b5d..cee4d92e46f43 100644
--- a/drivers/spi/spi-sifive.c
+++ b/drivers/spi/spi-sifive.c
@@ -296,7 +296,7 @@ static int sifive_spi_probe(struct platform_device *pdev)
 	u32 cs_bits, max_bits_per_word;
 	struct spi_controller *host;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(struct sifive_spi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(struct sifive_spi));
 	if (!host) {
 		dev_err(&pdev->dev, "out of memory\n");
 		return -ENOMEM;
@@ -307,24 +307,19 @@ static int sifive_spi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, host);
 
 	spi->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(spi->regs)) {
-		ret = PTR_ERR(spi->regs);
-		goto put_host;
-	}
+	if (IS_ERR(spi->regs))
+		return PTR_ERR(spi->regs);
 
 	/* Spin up the bus clock before hitting registers */
 	spi->clk = devm_clk_get_enabled(&pdev->dev, NULL);
 	if (IS_ERR(spi->clk)) {
 		dev_err(&pdev->dev, "Unable to find bus clock\n");
-		ret = PTR_ERR(spi->clk);
-		goto put_host;
+		return PTR_ERR(spi->clk);
 	}
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = irq;
-		goto put_host;
-	}
+	if (irq < 0)
+		return irq;
 
 	/* Optional parameters */
 	ret =
@@ -339,8 +334,7 @@ static int sifive_spi_probe(struct platform_device *pdev)
 
 	if (!ret && max_bits_per_word < 8) {
 		dev_err(&pdev->dev, "Only 8bit SPI words supported by the driver\n");
-		ret = -EINVAL;
-		goto put_host;
+		return -EINVAL;
 	}
 
 	/* probe the number of CS lines */
@@ -350,15 +344,13 @@ static int sifive_spi_probe(struct platform_device *pdev)
 	sifive_spi_write(spi, SIFIVE_SPI_REG_CSDEF, spi->cs_inactive);
 	if (!cs_bits) {
 		dev_err(&pdev->dev, "Could not auto probe CS lines\n");
-		ret = -EINVAL;
-		goto put_host;
+		return -EINVAL;
 	}
 
 	num_cs = ilog2(cs_bits) + 1;
 	if (num_cs > SIFIVE_SPI_MAX_CS) {
 		dev_err(&pdev->dev, "Invalid number of spi targets\n");
-		ret = -EINVAL;
-		goto put_host;
+		return -EINVAL;
 	}
 
 	/* Define our host */
@@ -386,7 +378,7 @@ static int sifive_spi_probe(struct platform_device *pdev)
 			       dev_name(&pdev->dev), spi);
 	if (ret) {
 		dev_err(&pdev->dev, "Unable to bind to interrupt\n");
-		goto put_host;
+		return ret;
 	}
 
 	dev_info(&pdev->dev, "mapped; irq=%d, cs=%d\n",
@@ -395,15 +387,10 @@ static int sifive_spi_probe(struct platform_device *pdev)
 	ret = spi_register_controller(host);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "spi_register_host failed\n");
-		goto put_host;
+		return ret;
 	}
 
 	return 0;
-
-put_host:
-	spi_controller_put(host);
-
-	return ret;
 }
 
 static void sifive_spi_remove(struct platform_device *pdev)
@@ -411,14 +398,10 @@ static void sifive_spi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct sifive_spi *spi = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	/* Disable all the interrupts just in case */
 	sifive_spi_write(spi, SIFIVE_SPI_REG_IE, 0);
-
-	spi_controller_put(host);
 }
 
 static int sifive_spi_suspend(struct device *dev)
diff --git a/drivers/spi/spi-slave-mt27xx.c b/drivers/spi/spi-slave-mt27xx.c
index 7aedeaa5889db..e60ab4c18bed0 100644
--- a/drivers/spi/spi-slave-mt27xx.c
+++ b/drivers/spi/spi-slave-mt27xx.c
@@ -388,11 +388,9 @@ static int mtk_spi_slave_probe(struct platform_device *pdev)
 	int irq, ret;
 	const struct of_device_id *of_id;
 
-	ctlr = spi_alloc_target(&pdev->dev, sizeof(*mdata));
-	if (!ctlr) {
-		dev_err(&pdev->dev, "failed to alloc spi target\n");
+	ctlr = devm_spi_alloc_target(&pdev->dev, sizeof(*mdata));
+	if (!ctlr)
 		return -ENOMEM;
-	}
 
 	ctlr->auto_runtime_pm = true;
 	ctlr->mode_bits = SPI_CPOL | SPI_CPHA;
@@ -406,8 +404,7 @@ static int mtk_spi_slave_probe(struct platform_device *pdev)
 	of_id = of_match_node(mtk_spi_slave_of_match, pdev->dev.of_node);
 	if (!of_id) {
 		dev_err(&pdev->dev, "failed to probe of_node\n");
-		ret = -EINVAL;
-		goto err_put_ctlr;
+		return -EINVAL;
 	}
 	mdata = spi_controller_get_devdata(ctlr);
 	mdata->dev_comp = of_id->data;
@@ -420,35 +417,31 @@ static int mtk_spi_slave_probe(struct platform_device *pdev)
 	init_completion(&mdata->xfer_done);
 	mdata->dev = &pdev->dev;
 	mdata->base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(mdata->base)) {
-		ret = PTR_ERR(mdata->base);
-		goto err_put_ctlr;
-	}
+	if (IS_ERR(mdata->base))
+		return PTR_ERR(mdata->base);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = irq;
-		goto err_put_ctlr;
-	}
+	if (irq < 0)
+		return irq;
 
 	ret = devm_request_irq(&pdev->dev, irq, mtk_spi_slave_interrupt,
 			       IRQF_TRIGGER_NONE, dev_name(&pdev->dev), ctlr);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to register irq (%d)\n", ret);
-		goto err_put_ctlr;
+		return ret;
 	}
 
 	mdata->spi_clk = devm_clk_get(&pdev->dev, "spi");
 	if (IS_ERR(mdata->spi_clk)) {
 		ret = PTR_ERR(mdata->spi_clk);
 		dev_err(&pdev->dev, "failed to get spi-clk: %d\n", ret);
-		goto err_put_ctlr;
+		return ret;
 	}
 
 	ret = clk_prepare_enable(mdata->spi_clk);
 	if (ret < 0) {
 		dev_err(&pdev->dev, "failed to enable spi_clk (%d)\n", ret);
-		goto err_put_ctlr;
+		return ret;
 	}
 
 	pm_runtime_enable(&pdev->dev);
@@ -465,8 +458,6 @@ static int mtk_spi_slave_probe(struct platform_device *pdev)
 
 err_disable_runtime_pm:
 	pm_runtime_disable(&pdev->dev);
-err_put_ctlr:
-	spi_controller_put(ctlr);
 
 	return ret;
 }
@@ -475,13 +466,9 @@ static void mtk_spi_slave_remove(struct platform_device *pdev)
 {
 	struct spi_controller *ctlr = platform_get_drvdata(pdev);
 
-	spi_controller_get(ctlr);
-
 	spi_unregister_controller(ctlr);
 
 	pm_runtime_disable(&pdev->dev);
-
-	spi_controller_put(ctlr);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/spi/spi-sn-f-ospi.c b/drivers/spi/spi-sn-f-ospi.c
index b459d51cb3a87..f0320e96fe232 100644
--- a/drivers/spi/spi-sn-f-ospi.c
+++ b/drivers/spi/spi-sn-f-ospi.c
@@ -222,9 +222,8 @@ static void f_ospi_config_clk(struct f_ospi *ospi, u32 device_hz)
 	 */
 	val = readl(ospi->base + OSPI_CLK_CTL);
 
-	val &= ~(OSPI_CLK_CTL_PHA | OSPI_CLK_CTL_DIV);
-	val |= FIELD_PREP(OSPI_CLK_CTL_PHA, OSPI_CLK_CTL_PHA_180)
-	     | FIELD_PREP(OSPI_CLK_CTL_DIV, div_reg);
+	FIELD_MODIFY(OSPI_CLK_CTL_PHA, &val, OSPI_CLK_CTL_PHA_180);
+	FIELD_MODIFY(OSPI_CLK_CTL_DIV, &val, div_reg);
 
 	writel(val, ospi->base + OSPI_CLK_CTL);
 }
diff --git a/drivers/spi/spi-spacemit-k1.c b/drivers/spi/spi-spacemit-k1.c
new file mode 100644
index 0000000000000..215fe66d27b4d
--- /dev/null
+++ b/drivers/spi/spi-spacemit-k1.c
@@ -0,0 +1,789 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// SpacemiT K1 SPI controller driver
+//
+// Copyright (C) 2026, RISCstar Solutions Corporation
+// Copyright (C) 2023, SpacemiT Corporation
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/dmaengine.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/scatterlist.h>
+#include <linux/sizes.h>
+#include <linux/spi/spi.h>
+#include <linux/units.h>
+
+#include "internals.h"
+
+/* This is the range of transfer rates supported by the K1 SoC */
+#define K1_SPI_MIN_SPEED_HZ		6250
+#define K1_SPI_MAX_SPEED_HZ		51200000
+
+/* DMA constraints */
+#define K1_SPI_DMA_ALIGNMENT		64
+#define K1_SPI_MAX_DMA_LEN		SZ_512K
+
+/* SSP Top Control Register */
+#define SSP_TOP_CTRL		0x00
+#define TOP_SSE				BIT(0)		/* Enable port */
+#define TOP_FRF_MASK			GENMASK(2, 1)	/* Frame format */
+#define TOP_FRF_MOTOROLA			0	/* Motorola SPI */
+#define TOP_DSS_MASK			GENMASK(9, 5)	/* Data size (1-32) */
+#define TOP_SPO				BIT(10)		/* Polarity: 0=low */
+#define TOP_SPH				BIT(11)		/* Half-cycle phase */
+#define TOP_LBM				BIT(12)		/* Loopback mode */
+#define TOP_TRAIL			BIT(13)		/* Trailing bytes */
+#define TOP_HOLD_FRAME_LOW		BIT(14)		/* Chip select */
+
+/* SSP FIFO Control Register */
+#define SSP_FIFO_CTRL		0x04
+#define FIFO_TFT_MASK			GENMASK(4, 0)	/* TX FIFO threshold */
+#define FIFO_RFT_MASK			GENMASK(9, 5)	/* RX FIFO threshold */
+#define FIFO_TSRE			BIT(10)		/* TX service request */
+#define FIFO_RSRE			BIT(11)		/* RX service request */
+
+/* SSP Interrupt Enable Register */
+#define SSP_INT_EN		0x08
+#define SSP_INT_EN_TINTE		BIT(1)		/* RX timeout */
+#define SSP_INT_EN_RIE			BIT(2)		/* RX FIFO */
+#define SSP_INT_EN_TIE			BIT(3)		/* TX FIFO */
+#define SSP_INT_EN_RIM			BIT(4)		/* RX FIFO overrun */
+#define SSP_INT_EN_TIM			BIT(5)		/* TX FIFO underrun */
+#define SSP_INT_EN_EBCEI		BIT(6)		/* Bit count error */
+
+/* TX interrupts, RX interrupts, and error interrupts */
+#define SSP_INT_EN_TX		SSP_INT_EN_TIE
+#define SSP_INT_EN_RX \
+		(SSP_INT_EN_TINTE | SSP_INT_EN_RIE)
+#define SSP_INT_EN_ERROR \
+		(SSP_INT_EN_RIM | SSP_INT_EN_TIM | SSP_INT_EN_EBCEI)
+
+/* SSP Time Out Register */
+#define SSP_TIMEOUT		0x0c
+#define SSP_TIMEOUT_MASK		GENMASK(23, 0)
+
+/* SSP Data Register */
+#define SSP_DATAR		0x10
+
+/* SSP Status Register */
+#define SSP_STATUS		0x14
+#define SSP_STATUS_BSY			BIT(0)		/* SPI/I2S busy */
+#define SSP_STATUS_TNF			BIT(6)		/* TX FIFO not full */
+#define SSP_STATUS_TFL			GENMASK(11, 7)	/* TX FIFO level */
+#define SSP_STATUS_TUR			BIT(12)		/* TX FIFO underrun */
+#define SSP_STATUS_RNE			BIT(14)		/* RX FIFO not empty */
+#define SSP_STATUS_RFL			GENMASK(19, 15)	/* RX FIFO level */
+#define SSP_STATUS_ROR			BIT(20)		/* RX FIFO overrun */
+#define SSP_STATUS_BCE			BIT(21)		/* Bit count error */
+
+/* Error status mask */
+#define SSP_STATUS_ERROR \
+		(SSP_STATUS_TUR | SSP_STATUS_ROR | SSP_STATUS_BCE)
+
+/* The FIFO sizes and thresholds are the same for RX and TX */
+#define K1_SPI_FIFO_SIZE	32
+#define K1_SPI_THRESH		(K1_SPI_FIFO_SIZE / 2)
+
+struct k1_spi_driver_data {
+	struct spi_controller *host;
+	void __iomem *base;
+	phys_addr_t base_addr;
+	unsigned long bus_rate;
+	struct clk *clk;
+	unsigned long rate;
+	int irq;
+
+	/* Current transfer information; not valid if message is null */
+	u32 bytes;			/* Bytes used for bits_per_word */
+	unsigned int rx_resid;		/* RX bytes left in transfer */
+	unsigned int tx_resid;		/* TX bytes left in transfer */
+	struct spi_transfer *transfer;	/* Current transfer */
+
+	bool dma_enabled;
+};
+
+/* Set our registers to a known initial state */
+static void
+k1_spi_register_reset(struct k1_spi_driver_data *drv_data, bool initial)
+{
+	u32 val = 0;
+
+	writel(0, drv_data->base + SSP_TOP_CTRL);
+
+	if (initial) {
+		/*
+		 * The TX and RX FIFO thresholds are the same no matter
+		 * what the speed or bits per word, so we can just set
+		 * them once.  The thresholds are one more than the values
+		 * in the register.
+		 */
+		val = FIELD_PREP(FIFO_RFT_MASK, K1_SPI_THRESH - 1);
+		val |= FIELD_PREP(FIFO_TFT_MASK, K1_SPI_THRESH - 1);
+	}
+	writel(val, drv_data->base + SSP_FIFO_CTRL);
+
+	writel(0, drv_data->base + SSP_INT_EN);
+	writel(0, drv_data->base + SSP_TIMEOUT);
+
+	/* Clear any pending interrupt conditions */
+	writel(~0, drv_data->base + SSP_STATUS);
+}
+
+/*
+ * The client can call the setup function multiple times, and each call
+ * can specify a different SPI mode (and transfer speed).  Each transfer
+ * can specify its own speed though, and the core code ensures each
+ * transfer's speed is set to something nonzero and supported by both
+ * the controller and the device.  We just set the speed for each transfer.
+ */
+static int k1_spi_setup(struct spi_device *spi)
+{
+	struct k1_spi_driver_data *drv_data;
+	u32 val;
+
+	drv_data = spi_controller_get_devdata(spi->controller);
+
+	/*
+	 * Configure the message format for this device.  We only
+	 * support Motorola SPI format in master mode.
+	 */
+	val = FIELD_PREP(TOP_FRF_MASK, TOP_FRF_MOTOROLA);
+
+	/* Translate the mode into the value used to program the hardware. */
+	if (spi->mode & SPI_CPHA)
+		val |= TOP_SPH;		/* 1/2 cycle */
+	if (spi->mode & SPI_CPOL)
+		val |= TOP_SPO;		/* active low */
+	if (spi->mode & SPI_LOOP)
+		val |= TOP_LBM;		/* enable loopback */
+	writel(val, drv_data->base + SSP_TOP_CTRL);
+
+	return 0;
+}
+
+static void k1_spi_cleanup(struct spi_device *spi)
+{
+	struct k1_spi_driver_data *drv_data;
+
+	drv_data = spi_controller_get_devdata(spi->controller);
+	k1_spi_register_reset(drv_data, false);
+}
+
+static bool k1_spi_can_dma(struct spi_controller *host, struct spi_device *spi,
+			   struct spi_transfer *transfer)
+{
+	struct k1_spi_driver_data *drv_data = spi_controller_get_devdata(host);
+	u32 burst_size;
+
+	if (!drv_data->dma_enabled)
+		return false;
+
+	if (transfer->len > SZ_2K)
+		return false;
+
+	/* Don't bother with DMA if we can't do even a single burst */
+	burst_size = K1_SPI_THRESH * spi_bpw_to_bytes(transfer->bits_per_word);
+
+	return transfer->len >= burst_size;
+}
+
+static void k1_spi_dma_callback(void *param)
+{
+	struct k1_spi_driver_data *drv_data = param;
+	u32 val;
+
+	val = readl(drv_data->base + SSP_FIFO_CTRL);
+	val &= ~(FIFO_TSRE | FIFO_RSRE);
+	writel(val, drv_data->base + SSP_FIFO_CTRL);
+
+	val = readl(drv_data->base + SSP_TOP_CTRL);
+	val &= ~TOP_TRAIL;
+	writel(val, drv_data->base + SSP_TOP_CTRL);
+
+	/* Check for any error conditions */
+	val = readl(drv_data->base + SSP_STATUS);
+	if (val & SSP_STATUS_ERROR)
+		drv_data->transfer->error |= SPI_TRANS_FAIL_IO;
+
+	/* Disable the port */
+	val = readl(drv_data->base + SSP_TOP_CTRL);
+	val &= ~TOP_SSE;
+	writel(val, drv_data->base + SSP_TOP_CTRL);
+
+	drv_data->transfer = NULL;
+
+	spi_finalize_current_transfer(drv_data->host);
+}
+
+/* Prepare a descriptor for TX or RX DMA */
+static struct dma_async_tx_descriptor *
+k1_spi_dma_prep(struct k1_spi_driver_data *drv_data,
+		struct spi_transfer *transfer, bool tx)
+{
+	phys_addr_t addr = drv_data->base_addr + SSP_DATAR;
+	u32 burst_size = K1_SPI_THRESH * drv_data->bytes;
+	struct dma_slave_config cfg = { };
+	enum dma_transfer_direction dir;
+	enum dma_slave_buswidth width;
+	struct dma_chan *chan;
+	struct sg_table *sgt;
+
+	switch (drv_data->bytes) {
+	case 1:
+		width = DMA_SLAVE_BUSWIDTH_1_BYTE;
+		break;
+	case 2:
+		width = DMA_SLAVE_BUSWIDTH_2_BYTES;
+		break;
+	default: /* bytes == 4 */
+		width = DMA_SLAVE_BUSWIDTH_4_BYTES;
+		break;
+	}
+
+	if (tx) {
+		chan = drv_data->host->dma_tx;
+		sgt = &transfer->tx_sg;
+		dir = DMA_MEM_TO_DEV;
+
+		cfg.dst_addr = addr;
+		cfg.dst_addr_width = width;
+		cfg.dst_maxburst = burst_size;
+	} else {
+		chan = drv_data->host->dma_rx;
+		sgt = &transfer->rx_sg;
+		dir = DMA_DEV_TO_MEM;
+
+		cfg.src_addr = addr;
+		cfg.src_addr_width = width;
+		cfg.src_maxburst = burst_size;
+	}
+	cfg.direction = dir;
+
+	if (dmaengine_slave_config(chan, &cfg))
+		return NULL;
+
+	return dmaengine_prep_slave_sg(chan, sgt->sgl, sgt->nents, dir,
+				       DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
+
+}
+
+static int k1_spi_dma_one(struct spi_controller *host, struct spi_device *spi,
+			  struct spi_transfer *transfer)
+{
+	struct k1_spi_driver_data *drv_data = spi_controller_get_devdata(host);
+	struct dma_async_tx_descriptor *desc;
+	u32 val;
+
+	/* Prepare the TX descriptor and submit it */
+	desc = k1_spi_dma_prep(drv_data, transfer, true);
+	if (!desc)
+		goto fallback;
+	dmaengine_submit(desc);
+
+	/* Prepare the RX descriptor and submit it */
+	desc = k1_spi_dma_prep(drv_data, transfer, false);
+	if (!desc)
+		goto fallback;
+
+	/* When RX is complete we also know TX has completed */
+	desc->callback = k1_spi_dma_callback;
+	desc->callback_param = drv_data;
+
+	dmaengine_submit(desc);
+
+	val = readl(drv_data->base + SSP_TOP_CTRL);
+	val |= TOP_TRAIL;		/* Trailing bytes handled by DMA */
+	writel(val, drv_data->base + SSP_TOP_CTRL);
+
+	val = readl(drv_data->base + SSP_FIFO_CTRL);
+	val |= FIFO_TSRE | FIFO_RSRE;
+	writel(val, drv_data->base + SSP_FIFO_CTRL);
+
+	/* Start RX first so we're ready the instant we start transmitting */
+	dma_async_issue_pending(host->dma_rx);
+	dma_async_issue_pending(host->dma_tx);
+
+	return 1;
+fallback:
+	transfer->error |= SPI_TRANS_FAIL_NO_START;
+
+	return -EAGAIN;
+}
+
+/* Flush the RX FIFO of any leftover data before processing a message */
+static int k1_spi_prepare_message(struct spi_controller *host,
+				  struct spi_message *message)
+{
+	struct k1_spi_driver_data *drv_data = spi_controller_get_devdata(host);
+	u32 val = readl(drv_data->base + SSP_STATUS);
+	u32 count;
+
+	/* If there's nothing in the FIFO, we're done */
+	if (!(val & SSP_STATUS_RNE))
+		return 0;
+
+	/* Read and discard what's there (one more than what the field says) */
+	count = FIELD_GET(SSP_STATUS_RFL, val) + 1;
+	do
+		(void)readl(drv_data->base + SSP_DATAR);
+	while (--count);
+
+	return 0;
+}
+
+/* Set logic level of chip select line (high=true means CS deasserted) */
+static void k1_spi_set_cs(struct spi_device *spi, bool high)
+{
+	struct k1_spi_driver_data *drv_data;
+	u32 val;
+
+	drv_data = spi_controller_get_devdata(spi->controller);
+
+	val = readl(drv_data->base + SSP_TOP_CTRL);
+	if (high)
+		val &= ~TOP_HOLD_FRAME_LOW;
+	else
+		val |= TOP_HOLD_FRAME_LOW;
+	writel(val, drv_data->base + SSP_TOP_CTRL);
+}
+
+/* Set the transfer speed; the SPI core code ensures it is supported */
+static int k1_spi_set_speed(struct k1_spi_driver_data *drv_data,
+			    struct spi_transfer *transfer)
+{
+	struct clk *clk = drv_data->clk;
+	u64 nsec_per_word;
+	u64 bus_ticks;
+	u32 timeout;
+	u32 val;
+	int ret;
+
+	ret = clk_set_rate(clk, transfer->speed_hz);
+	if (ret)
+		return ret;
+
+	drv_data->rate = clk_get_rate(clk);
+
+	/* No need for RX FIFO timeout if we're not receiving anything */
+	if (!transfer->rx_buf)
+		return 0;
+
+	/*
+	 * Compute the RX FIFO inactivity timeout value that should be used.
+	 * The inactivity timer restarts with each word that lands in the
+	 * FIFO.  If several "word transfer times" pass without any new data
+	 * in the RX FIFO, we might as well read what's there.
+	 *
+	 * The rate at which words land in the FIFO is determined by the
+	 * word size and the transfer rate.  One bit is transferred per
+	 * clock tick, and 8 (or 16 or 32) bits are transferred per word.
+	 *
+	 * So we can get word transfer time (in nanoseconds) from:
+	 *   nsec_per_tick = NSEC_PER_SEC / drv_data->rate;
+	 *   ticks_per_word = BITS_PER_BYTE * drv_data->bytes;
+	 * We do the divide last for better accuracy.
+	 */
+	nsec_per_word = (u64)NSEC_PER_SEC * BITS_PER_BYTE * drv_data->bytes;
+	nsec_per_word = DIV_ROUND_UP_ULL(nsec_per_word, drv_data->rate);
+
+	/*
+	 * The timeout (which we'll set to three word transfer times) is
+	 * expressed as a number of APB clock ticks.
+	 *   bus_ticks = 3 * nsec * (drv_data->bus_rate / NSEC_PER_SEC)
+	 */
+	bus_ticks = 3 * nsec_per_word * drv_data->bus_rate;
+	timeout = DIV_ROUND_UP_ULL(bus_ticks, NSEC_PER_SEC);
+
+	/* Set the RX timeout period (required for both DMA and PIO) */
+	val = FIELD_PREP(SSP_TIMEOUT_MASK, timeout);
+	writel(val, drv_data->base + SSP_TIMEOUT);
+
+	return 0;
+}
+
+static int k1_spi_transfer_one(struct spi_controller *host,
+			       struct spi_device *spi,
+			       struct spi_transfer *transfer)
+{
+	struct k1_spi_driver_data *drv_data = spi_controller_get_devdata(host);
+	u32 ctrl;
+	u32 val;
+	int ret;
+
+	/* Bits per word can change on a per-transfer basis */
+	drv_data->bytes = spi_bpw_to_bytes(transfer->bits_per_word);
+
+	/* Each transfer can also specify a different rate */
+	ret = k1_spi_set_speed(drv_data, transfer);
+	if (ret) {
+		dev_err(&host->dev,
+			"failed to set transfer speed: %d\n", ret);
+		return ret;
+	}
+
+	drv_data->rx_resid = transfer->len;
+	drv_data->tx_resid = transfer->len;
+
+	drv_data->transfer = transfer;
+
+	/* Clear any existing interrupt conditions */
+	writel(~0, drv_data->base + SSP_STATUS);
+
+	/* Set the data (word) size, and enable the port */
+	ctrl = readl(drv_data->base + SSP_TOP_CTRL);
+	ctrl &= ~TOP_DSS_MASK;
+	ctrl |= FIELD_PREP(TOP_DSS_MASK, transfer->bits_per_word - 1);
+	ctrl |= TOP_SSE;
+	writel(ctrl, drv_data->base + SSP_TOP_CTRL);
+
+	if (spi_xfer_is_dma_mapped(host, spi, transfer))
+		return k1_spi_dma_one(host, spi, transfer);
+
+	/* An interrupt will initiate the transfer */
+	val = SSP_INT_EN_TX | SSP_INT_EN_RX | SSP_INT_EN_ERROR;
+	writel(val, drv_data->base + SSP_INT_EN);
+
+	return 1;	/* We will call spi_finalize_current_transfer() */
+}
+
+static void
+k1_spi_handle_err(struct spi_controller *host, struct spi_message *message)
+{
+	struct k1_spi_driver_data *drv_data = spi_controller_get_devdata(host);
+
+	if (drv_data->dma_enabled) {
+		dmaengine_terminate_sync(host->dma_rx);
+		dmaengine_terminate_sync(host->dma_tx);
+	}
+}
+
+static void k1_spi_write_word(struct k1_spi_driver_data *drv_data)
+{
+	struct spi_transfer *transfer = drv_data->transfer;
+	u32 bytes = drv_data->bytes;
+	u32 val;
+
+	if (transfer->tx_buf) {
+		const void *buf;
+
+		buf = transfer->tx_buf + (transfer->len - drv_data->tx_resid);
+		if (bytes == 1)
+			val = *(u8 *)buf;
+		else if (bytes == 2)
+			val = *(u16 *)buf;
+		else	/* bytes == 4 */
+			val = *(u32 *)buf;
+	} else {
+		val = 0;	/* Null writer; write 1, 2, or 4 zero bytes */
+	}
+	/* Fill the next TX FIFO entry */
+	writel(val, drv_data->base + SSP_DATAR);
+
+	drv_data->tx_resid -= bytes;
+}
+
+/* The last-read status value is provided; we know SSP_STATUS_TNF is set */
+static bool k1_spi_write(struct k1_spi_driver_data *drv_data, u32 val)
+{
+	unsigned int count;
+
+	/* Get the number of open slots in the FIFO; zero means all */
+	count = FIELD_GET(SSP_STATUS_TFL, val) ? : K1_SPI_FIFO_SIZE;
+
+	/*
+	 * Limit how much we try to send at a time, to reduce the
+	 * chance the other side can overrun our RX FIFO.
+	 */
+	count = min3(count, K1_SPI_THRESH, drv_data->tx_resid / drv_data->bytes);
+	do
+		k1_spi_write_word(drv_data);
+	while (--count);
+
+	return !drv_data->tx_resid;
+}
+
+static void k1_spi_read_word(struct k1_spi_driver_data *drv_data)
+{
+	struct spi_transfer *transfer = drv_data->transfer;
+	u32 bytes = drv_data->bytes;
+	u32 val;
+
+	/* Consume the next RX FIFO entry */
+	val = readl(drv_data->base + SSP_DATAR);
+	if (transfer->rx_buf) {
+		void *buf;
+
+		buf = transfer->rx_buf + (transfer->len - drv_data->rx_resid);
+
+		if (bytes == 1)
+			*(u8 *)buf = val;
+		else if (bytes == 2)
+			*(u16 *)buf = val;
+		else	/* bytes == 4 */
+			*(u32 *)buf = val;
+	}	/* Otherwise null reader: discard the data */
+
+	drv_data->rx_resid -= bytes;
+}
+
+/* The last-read status value is provided; we know SSP_STATUS_RNE is set */
+static bool k1_spi_read(struct k1_spi_driver_data *drv_data, u32 val)
+{
+	do {
+		unsigned int count = FIELD_GET(SSP_STATUS_RFL, val) + 1;
+
+		/* Only read what we need */
+		count = min(count, drv_data->rx_resid / drv_data->bytes);
+		do
+			k1_spi_read_word(drv_data);
+		while (--count);
+
+		/* If there's no more to read, we're done */
+		if (!drv_data->rx_resid)
+			return true;
+
+		/* Check again in case more became available to read */
+		val = readl(drv_data->base + SSP_STATUS);
+		if (val & SSP_STATUS_RNE)
+			writel(SSP_STATUS_RNE, drv_data->base + SSP_STATUS);
+		else
+			return false;
+	} while (true);
+}
+
+static irqreturn_t k1_spi_ssp_isr(int irq, void *dev_id)
+{
+	struct k1_spi_driver_data *drv_data = dev_id;
+	u32 status;
+	u32 top_ctrl;
+
+	/* Get status and clear pending interrupts */
+	status = readl(drv_data->base + SSP_STATUS);
+	writel(status, drv_data->base + SSP_STATUS);
+
+	/* If no actionable status bits are set, this is not our interrupt */
+	if (!(status & (SSP_STATUS_ERROR | SSP_STATUS_TNF | SSP_STATUS_RNE)))
+		return IRQ_NONE;
+
+	/* Check for any error conditions first */
+	if (status & SSP_STATUS_ERROR) {
+		if (drv_data->transfer)
+			drv_data->transfer->error |= SPI_TRANS_FAIL_IO;
+		goto done;
+	}
+
+	/*
+	 * For SPI, bytes are transferred in both directions equally, and
+	 * RX always follows TX.  Start by writing if there is anything to
+	 * write, then read.  Once there's no more to read, we're done.
+	 */
+	if (drv_data->tx_resid && (status & SSP_STATUS_TNF)) {
+		/* If we finish writing, disable TX interrupts */
+		if (k1_spi_write(drv_data, status))
+			writel(SSP_INT_EN_RX | SSP_INT_EN_ERROR,
+			       drv_data->base + SSP_INT_EN);
+	}
+
+	/* We're not done unless we've read all that was requested */
+	if (drv_data->rx_resid) {
+		/* Read more if the FIFO is not empty */
+		if (status & SSP_STATUS_RNE)
+			if (k1_spi_read(drv_data, status))
+				goto done;
+
+		return IRQ_HANDLED;
+	}
+done:
+	/* Disable the port */
+	top_ctrl = readl(drv_data->base + SSP_TOP_CTRL);
+	top_ctrl &= ~TOP_SSE;
+	writel(top_ctrl, drv_data->base + SSP_TOP_CTRL);
+
+	/* Disable all interrupts */
+	writel(0, drv_data->base + SSP_INT_EN);
+
+	if (drv_data->transfer) {
+		drv_data->transfer = NULL;
+		spi_finalize_current_transfer(drv_data->host);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int
+k1_spi_dma_setup(struct k1_spi_driver_data *drv_data, struct device *dev)
+{
+	struct spi_controller *host = drv_data->host;
+	struct dma_chan *chan;
+
+	chan = dma_request_chan(dev, "tx");
+	if (IS_ERR(chan))
+		return PTR_ERR(chan);
+	host->dma_tx = chan;
+
+	chan = dma_request_chan(dev, "rx");
+	if (IS_ERR(chan)) {
+		dma_release_channel(host->dma_tx);
+		host->dma_tx = NULL;
+		return PTR_ERR(chan);
+	}
+	host->dma_rx = chan;
+
+	drv_data->dma_enabled = true;
+
+	return 0;
+}
+
+static void k1_spi_dma_cleanup(struct device *dev, void *res)
+{
+	struct k1_spi_driver_data **ptr = res;
+	struct k1_spi_driver_data *drv_data = *ptr;
+	struct spi_controller *host = drv_data->host;
+
+	if (!drv_data->dma_enabled)
+		return;
+
+	drv_data->dma_enabled = false;
+
+	dma_release_channel(host->dma_rx);
+	host->dma_rx = NULL;
+	dma_release_channel(host->dma_tx);
+	host->dma_tx = NULL;
+}
+
+static int
+devm_k1_spi_dma_setup(struct k1_spi_driver_data *drv_data, struct device *dev)
+{
+	struct k1_spi_driver_data **ptr;
+	int ret;
+
+	if (!IS_ENABLED(CONFIG_MMP_PDMA)) {
+		dev_info(dev, "DMA not available; using PIO\n");
+		return 0;
+	}
+
+	ptr = devres_alloc(k1_spi_dma_cleanup, sizeof(*ptr), GFP_KERNEL);
+	if (!ptr)
+		return -ENOMEM;
+
+	ret = k1_spi_dma_setup(drv_data, dev);
+	if (ret) {
+		devres_free(ptr);
+		return ret;
+	}
+
+	*ptr = drv_data;
+	devres_add(dev, ptr);
+
+	return 0;
+}
+
+static int k1_spi_probe(struct platform_device *pdev)
+{
+	struct k1_spi_driver_data *drv_data;
+	struct device *dev = &pdev->dev;
+	struct reset_control *reset;
+	struct spi_controller *host;
+	struct resource *iores;
+	struct clk *clk_bus;
+	int ret;
+
+	host = devm_spi_alloc_host(dev, sizeof(*drv_data));
+	if (!host)
+		return -ENOMEM;
+	drv_data = spi_controller_get_devdata(host);
+	drv_data->host = host;
+	platform_set_drvdata(pdev, drv_data);
+
+	ret = devm_k1_spi_dma_setup(drv_data, dev);
+	if (ret == -EPROBE_DEFER)
+		return ret;
+	if (ret)
+		dev_warn(dev, "DMA setup failed (%d), falling back to PIO\n", ret);
+
+	drv_data->base = devm_platform_get_and_ioremap_resource(pdev, 0,
+								&iores);
+	if (IS_ERR(drv_data->base))
+		return dev_err_probe(dev, PTR_ERR(drv_data->base),
+				     "error mapping memory\n");
+	drv_data->base_addr = iores->start;
+
+	clk_bus = devm_clk_get_enabled(dev, "bus");
+	if (IS_ERR(clk_bus))
+		return dev_err_probe(dev, PTR_ERR(clk_bus),
+				     "error getting/enabling bus clock\n");
+	drv_data->bus_rate = clk_get_rate(clk_bus);
+
+	drv_data->clk = devm_clk_get_enabled(dev, "core");
+	if (IS_ERR(drv_data->clk))
+		return dev_err_probe(dev, PTR_ERR(drv_data->clk),
+				     "error getting/enabling core clock\n");
+
+	reset = devm_reset_control_get_exclusive_deasserted(dev, NULL);
+	if (IS_ERR(reset))
+		return dev_err_probe(dev, PTR_ERR(reset),
+				     "error getting/deasserting reset\n");
+
+	k1_spi_register_reset(drv_data, true);
+
+	drv_data->irq = platform_get_irq(pdev, 0);
+	if (drv_data->irq < 0)
+		return dev_err_probe(dev, drv_data->irq, "error getting IRQ\n");
+
+	ret = devm_request_irq(dev, drv_data->irq, k1_spi_ssp_isr,
+			       IRQF_SHARED, dev_name(dev), drv_data);
+	if (ret < 0)
+		return dev_err_probe(dev, ret, "error requesting IRQ\n");
+
+	/* Initialize the host structure, then register it */
+	host->dev.of_node = dev_of_node(dev);
+	host->dev.parent = dev;
+	host->num_chipselect = 1;
+	if (drv_data->dma_enabled)
+		host->dma_alignment = K1_SPI_DMA_ALIGNMENT;
+	host->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP;
+	host->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);
+	host->min_speed_hz = K1_SPI_MIN_SPEED_HZ;
+	host->max_speed_hz = K1_SPI_MAX_SPEED_HZ;
+	host->flags = SPI_CONTROLLER_MUST_RX | SPI_CONTROLLER_MUST_TX;
+	host->max_dma_len = K1_SPI_MAX_DMA_LEN;
+
+	host->setup = k1_spi_setup;
+	host->cleanup = k1_spi_cleanup;
+	host->can_dma = k1_spi_can_dma;
+	host->prepare_message = k1_spi_prepare_message;
+	host->set_cs = k1_spi_set_cs;
+	host->transfer_one = k1_spi_transfer_one;
+	host->handle_err = k1_spi_handle_err;
+
+	ret = devm_spi_register_controller(dev, host);
+	if (ret)
+		dev_err(dev, "error registering controller\n");
+
+	return ret;
+}
+
+static const struct of_device_id k1_spi_dt_ids[] = {
+	{ .compatible = "spacemit,k1-spi", },
+	{}
+};
+MODULE_DEVICE_TABLE(of, k1_spi_dt_ids);
+
+static struct platform_driver k1_spi_driver = {
+	.probe = k1_spi_probe,
+	.driver = {
+		.name		= "k1-spi",
+		.of_match_table	= k1_spi_dt_ids,
+	},
+};
+module_platform_driver(k1_spi_driver);
+
+MODULE_DESCRIPTION("SpacemiT K1 SPI controller driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/spi/spi-sprd.c b/drivers/spi/spi-sprd.c
index fd3fd0ce122c9..29380938ef65b 100644
--- a/drivers/spi/spi-sprd.c
+++ b/drivers/spi/spi-sprd.c
@@ -923,16 +923,14 @@ static int sprd_spi_probe(struct platform_device *pdev)
 	int ret;
 
 	pdev->id = of_alias_get_id(pdev->dev.of_node, "spi");
-	sctlr = spi_alloc_host(&pdev->dev, sizeof(*ss));
+	sctlr = devm_spi_alloc_host(&pdev->dev, sizeof(*ss));
 	if (!sctlr)
 		return -ENOMEM;
 
 	ss = spi_controller_get_devdata(sctlr);
 	ss->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
-	if (IS_ERR(ss->base)) {
-		ret = PTR_ERR(ss->base);
-		goto free_controller;
-	}
+	if (IS_ERR(ss->base))
+		return PTR_ERR(ss->base);
 
 	ss->phy_base = res->start;
 	ss->dev = &pdev->dev;
@@ -949,15 +947,15 @@ static int sprd_spi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, sctlr);
 	ret = sprd_spi_clk_init(pdev, ss);
 	if (ret)
-		goto free_controller;
+		return ret;
 
 	ret = sprd_spi_irq_init(pdev, ss);
 	if (ret)
-		goto free_controller;
+		return ret;
 
 	ret = sprd_spi_dma_init(pdev, ss);
 	if (ret)
-		goto free_controller;
+		return ret;
 
 	ret = clk_prepare_enable(ss->clk);
 	if (ret)
@@ -991,9 +989,8 @@ static int sprd_spi_probe(struct platform_device *pdev)
 disable_clk:
 	clk_disable_unprepare(ss->clk);
 release_dma:
-	sprd_spi_dma_release(ss);
-free_controller:
-	spi_controller_put(sctlr);
+	if (ss->dma.enable)
+		sprd_spi_dma_release(ss);
 
 	return ret;
 }
@@ -1008,8 +1005,6 @@ static void sprd_spi_remove(struct platform_device *pdev)
 	if (ret < 0)
 		dev_err(ss->dev, "failed to resume SPI controller\n");
 
-	spi_controller_get(sctlr);
-
 	spi_unregister_controller(sctlr);
 
 	if (ret >= 0) {
@@ -1019,8 +1014,6 @@ static void sprd_spi_remove(struct platform_device *pdev)
 	}
 	pm_runtime_put_noidle(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
-
-	spi_controller_put(sctlr);
 }
 
 static int __maybe_unused sprd_spi_runtime_suspend(struct device *dev)
diff --git a/drivers/spi/spi-st-ssc4.c b/drivers/spi/spi-st-ssc4.c
index 9c8099fe6e19f..df61094fc4446 100644
--- a/drivers/spi/spi-st-ssc4.c
+++ b/drivers/spi/spi-st-ssc4.c
@@ -279,7 +279,7 @@ static int spi_st_probe(struct platform_device *pdev)
 	int irq, ret = 0;
 	u32 var;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*spi_st));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*spi_st));
 	if (!host)
 		return -ENOMEM;
 
@@ -296,13 +296,12 @@ static int spi_st_probe(struct platform_device *pdev)
 	spi_st->clk = devm_clk_get(&pdev->dev, "ssc");
 	if (IS_ERR(spi_st->clk)) {
 		dev_err(&pdev->dev, "Unable to request clock\n");
-		ret = PTR_ERR(spi_st->clk);
-		goto put_host;
+		return PTR_ERR(spi_st->clk);
 	}
 
 	ret = clk_prepare_enable(spi_st->clk);
 	if (ret)
-		goto put_host;
+		return ret;
 
 	init_completion(&spi_st->done);
 
@@ -361,8 +360,7 @@ static int spi_st_probe(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 clk_disable:
 	clk_disable_unprepare(spi_st->clk);
-put_host:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -371,16 +369,12 @@ static void spi_st_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct spi_st *spi_st = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	pm_runtime_disable(&pdev->dev);
 
 	clk_disable_unprepare(spi_st->clk);
 
-	spi_controller_put(host);
-
 	pinctrl_pm_select_sleep_state(&pdev->dev);
 }
 
diff --git a/drivers/spi/spi-stm32-ospi.c b/drivers/spi/spi-stm32-ospi.c
index 4461c6e24b9ed..fa0e8e83cb1be 100644
--- a/drivers/spi/spi-stm32-ospi.c
+++ b/drivers/spi/spi-stm32-ospi.c
@@ -470,10 +470,9 @@ static int stm32_ospi_send(struct spi_device *spi, const struct spi_mem_op *op)
 	u8 cs = spi->chip_select[ffs(spi->cs_index_mask) - 1];
 
 	cr = readl_relaxed(ospi->regs_base + OSPI_CR);
-	cr &= ~CR_CSSEL;
-	cr |= FIELD_PREP(CR_CSSEL, cs);
-	cr &= ~CR_FMODE_MASK;
-	cr |= FIELD_PREP(CR_FMODE_MASK, ospi->fmode);
+	FIELD_MODIFY(CR_CSSEL, &cr, cs);
+
+	FIELD_MODIFY(CR_FMODE_MASK, &cr, ospi->fmode);
 	writel_relaxed(cr, regs_base + OSPI_CR);
 
 	if (op->data.nbytes)
@@ -602,11 +601,11 @@ static int stm32_ospi_dirmap_create(struct spi_mem_dirmap_desc *desc)
 {
 	struct stm32_ospi *ospi = spi_controller_get_devdata(desc->mem->spi->controller);
 
-	if (desc->info.op_tmpl.data.dir == SPI_MEM_DATA_OUT)
+	if (desc->info.op_tmpl->data.dir == SPI_MEM_DATA_OUT)
 		return -EOPNOTSUPP;
 
 	/* Should never happen, as mm_base == null is an error probe exit condition */
-	if (!ospi->mm_base && desc->info.op_tmpl.data.dir == SPI_MEM_DATA_IN)
+	if (!ospi->mm_base && desc->info.op_tmpl->data.dir == SPI_MEM_DATA_IN)
 		return -EOPNOTSUPP;
 
 	if (!ospi->mm_size)
@@ -633,7 +632,7 @@ static ssize_t stm32_ospi_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	 * spi_mem_op template with offs, len and *buf in  order to get
 	 * all needed transfer information into struct spi_mem_op
 	 */
-	memcpy(&op, &desc->info.op_tmpl, sizeof(struct spi_mem_op));
+	memcpy(&op, desc->info.op_tmpl, sizeof(struct spi_mem_op));
 	dev_dbg(ospi->dev, "%s len = 0x%zx offs = 0x%llx buf = 0x%p\n", __func__, len, offs, buf);
 
 	op.data.nbytes = len;
diff --git a/drivers/spi/spi-stm32-qspi.c b/drivers/spi/spi-stm32-qspi.c
index df1bbacec90a2..b77daba6db83a 100644
--- a/drivers/spi/spi-stm32-qspi.c
+++ b/drivers/spi/spi-stm32-qspi.c
@@ -374,9 +374,8 @@ static int stm32_qspi_send(struct spi_device *spi, const struct spi_mem_op *op)
 	int timeout, err = 0, err_poll_status = 0;
 
 	cr = readl_relaxed(qspi->io_base + QSPI_CR);
-	cr &= ~CR_PRESC_MASK & ~CR_FSEL;
-	cr |= FIELD_PREP(CR_PRESC_MASK, flash->presc);
-	cr |= FIELD_PREP(CR_FSEL, flash->cs);
+	FIELD_MODIFY(CR_PRESC_MASK, &cr, flash->presc);
+	FIELD_MODIFY(CR_FSEL, &cr, flash->cs);
 	writel_relaxed(cr, qspi->io_base + QSPI_CR);
 
 	if (op->data.nbytes)
@@ -506,11 +505,11 @@ static int stm32_qspi_dirmap_create(struct spi_mem_dirmap_desc *desc)
 {
 	struct stm32_qspi *qspi = spi_controller_get_devdata(desc->mem->spi->controller);
 
-	if (desc->info.op_tmpl.data.dir == SPI_MEM_DATA_OUT)
+	if (desc->info.op_tmpl->data.dir == SPI_MEM_DATA_OUT)
 		return -EOPNOTSUPP;
 
 	/* should never happen, as mm_base == null is an error probe exit condition */
-	if (!qspi->mm_base && desc->info.op_tmpl.data.dir == SPI_MEM_DATA_IN)
+	if (!qspi->mm_base && desc->info.op_tmpl->data.dir == SPI_MEM_DATA_IN)
 		return -EOPNOTSUPP;
 
 	if (!qspi->mm_size)
@@ -536,7 +535,7 @@ static ssize_t stm32_qspi_dirmap_read(struct spi_mem_dirmap_desc *desc,
 	 * spi_mem_op template with offs, len and *buf in  order to get
 	 * all needed transfer information into struct spi_mem_op
 	 */
-	memcpy(&op, &desc->info.op_tmpl, sizeof(struct spi_mem_op));
+	memcpy(&op, desc->info.op_tmpl, sizeof(struct spi_mem_op));
 	dev_dbg(qspi->dev, "%s len = 0x%zx offs = 0x%llx buf = 0x%p\n", __func__, len, offs, buf);
 
 	op.data.nbytes = len;
diff --git a/drivers/spi/spi-sun4i.c b/drivers/spi/spi-sun4i.c
index b7fbb5270edbc..d5c16392cd4d7 100644
--- a/drivers/spi/spi-sun4i.c
+++ b/drivers/spi/spi-sun4i.c
@@ -434,32 +434,26 @@ static int sun4i_spi_probe(struct platform_device *pdev)
 	struct sun4i_spi *sspi;
 	int ret = 0, irq;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(struct sun4i_spi));
-	if (!host) {
-		dev_err(&pdev->dev, "Unable to allocate SPI Host\n");
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(struct sun4i_spi));
+	if (!host)
 		return -ENOMEM;
-	}
 
 	platform_set_drvdata(pdev, host);
 	sspi = spi_controller_get_devdata(host);
 
 	sspi->base_addr = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(sspi->base_addr)) {
-		ret = PTR_ERR(sspi->base_addr);
-		goto err_free_host;
-	}
+	if (IS_ERR(sspi->base_addr))
+		return PTR_ERR(sspi->base_addr);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = -ENXIO;
-		goto err_free_host;
-	}
+	if (irq < 0)
+		return -ENXIO;
 
 	ret = devm_request_irq(&pdev->dev, irq, sun4i_spi_handler,
 			       0, "sun4i-spi", sspi);
 	if (ret) {
 		dev_err(&pdev->dev, "Cannot request IRQ\n");
-		goto err_free_host;
+		return ret;
 	}
 
 	sspi->host = host;
@@ -477,15 +471,13 @@ static int sun4i_spi_probe(struct platform_device *pdev)
 	sspi->hclk = devm_clk_get(&pdev->dev, "ahb");
 	if (IS_ERR(sspi->hclk)) {
 		dev_err(&pdev->dev, "Unable to acquire AHB clock\n");
-		ret = PTR_ERR(sspi->hclk);
-		goto err_free_host;
+		return PTR_ERR(sspi->hclk);
 	}
 
 	sspi->mclk = devm_clk_get(&pdev->dev, "mod");
 	if (IS_ERR(sspi->mclk)) {
 		dev_err(&pdev->dev, "Unable to acquire module clock\n");
-		ret = PTR_ERR(sspi->mclk);
-		goto err_free_host;
+		return PTR_ERR(sspi->mclk);
 	}
 
 	init_completion(&sspi->done);
@@ -497,7 +489,7 @@ static int sun4i_spi_probe(struct platform_device *pdev)
 	ret = sun4i_spi_runtime_resume(&pdev->dev);
 	if (ret) {
 		dev_err(&pdev->dev, "Couldn't resume the device\n");
-		goto err_free_host;
+		return ret;
 	}
 
 	pm_runtime_set_active(&pdev->dev);
@@ -515,8 +507,7 @@ static int sun4i_spi_probe(struct platform_device *pdev)
 err_pm_disable:
 	pm_runtime_disable(&pdev->dev);
 	sun4i_spi_runtime_suspend(&pdev->dev);
-err_free_host:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -524,13 +515,9 @@ static void sun4i_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *host = platform_get_drvdata(pdev);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	pm_runtime_force_suspend(&pdev->dev);
-
-	spi_controller_put(host);
 }
 
 static const struct of_device_id sun4i_spi_match[] = {
diff --git a/drivers/spi/spi-sun6i.c b/drivers/spi/spi-sun6i.c
index 5ac73d324d063..4631e9c7ca1dd 100644
--- a/drivers/spi/spi-sun6i.c
+++ b/drivers/spi/spi-sun6i.c
@@ -633,7 +633,7 @@ static int sun6i_spi_probe(struct platform_device *pdev)
 	struct resource *mem;
 	int ret = 0, irq;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(struct sun6i_spi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(struct sun6i_spi));
 	if (!host) {
 		dev_err(&pdev->dev, "Unable to allocate SPI Host\n");
 		return -ENOMEM;
@@ -643,22 +643,18 @@ static int sun6i_spi_probe(struct platform_device *pdev)
 	sspi = spi_controller_get_devdata(host);
 
 	sspi->base_addr = devm_platform_get_and_ioremap_resource(pdev, 0, &mem);
-	if (IS_ERR(sspi->base_addr)) {
-		ret = PTR_ERR(sspi->base_addr);
-		goto err_free_host;
-	}
+	if (IS_ERR(sspi->base_addr))
+		return PTR_ERR(sspi->base_addr);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = -ENXIO;
-		goto err_free_host;
-	}
+	if (irq < 0)
+		return -ENXIO;
 
 	ret = devm_request_irq(&pdev->dev, irq, sun6i_spi_handler,
 			       0, "sun6i-spi", sspi);
 	if (ret) {
 		dev_err(&pdev->dev, "Cannot request IRQ\n");
-		goto err_free_host;
+		return ret;
 	}
 
 	sspi->host = host;
@@ -679,15 +675,13 @@ static int sun6i_spi_probe(struct platform_device *pdev)
 	sspi->hclk = devm_clk_get(&pdev->dev, "ahb");
 	if (IS_ERR(sspi->hclk)) {
 		dev_err(&pdev->dev, "Unable to acquire AHB clock\n");
-		ret = PTR_ERR(sspi->hclk);
-		goto err_free_host;
+		return PTR_ERR(sspi->hclk);
 	}
 
 	sspi->mclk = devm_clk_get(&pdev->dev, "mod");
 	if (IS_ERR(sspi->mclk)) {
 		dev_err(&pdev->dev, "Unable to acquire module clock\n");
-		ret = PTR_ERR(sspi->mclk);
-		goto err_free_host;
+		return PTR_ERR(sspi->mclk);
 	}
 
 	init_completion(&sspi->done);
@@ -696,17 +690,14 @@ static int sun6i_spi_probe(struct platform_device *pdev)
 	sspi->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL);
 	if (IS_ERR(sspi->rstc)) {
 		dev_err(&pdev->dev, "Couldn't get reset controller\n");
-		ret = PTR_ERR(sspi->rstc);
-		goto err_free_host;
+		return PTR_ERR(sspi->rstc);
 	}
 
 	host->dma_tx = dma_request_chan(&pdev->dev, "tx");
 	if (IS_ERR(host->dma_tx)) {
 		/* Check tx to see if we need defer probing driver */
-		if (PTR_ERR(host->dma_tx) == -EPROBE_DEFER) {
-			ret = -EPROBE_DEFER;
-			goto err_free_host;
-		}
+		if (PTR_ERR(host->dma_tx) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
 		dev_warn(&pdev->dev, "Failed to request TX DMA channel\n");
 		host->dma_tx = NULL;
 	}
@@ -759,8 +750,7 @@ static int sun6i_spi_probe(struct platform_device *pdev)
 err_free_dma_tx:
 	if (host->dma_tx)
 		dma_release_channel(host->dma_tx);
-err_free_host:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -768,8 +758,6 @@ static void sun6i_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *host = platform_get_drvdata(pdev);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	pm_runtime_force_suspend(&pdev->dev);
@@ -778,8 +766,6 @@ static void sun6i_spi_remove(struct platform_device *pdev)
 		dma_release_channel(host->dma_tx);
 	if (host->dma_rx)
 		dma_release_channel(host->dma_rx);
-
-	spi_controller_put(host);
 }
 
 static const struct sun6i_spi_cfg sun6i_a31_spi_cfg = {
diff --git a/drivers/spi/spi-sunplus-sp7021.c b/drivers/spi/spi-sunplus-sp7021.c
index 35601212fb78f..c1870322d976b 100644
--- a/drivers/spi/spi-sunplus-sp7021.c
+++ b/drivers/spi/spi-sunplus-sp7021.c
@@ -290,8 +290,7 @@ static void sp7021_spi_setup_clk(struct spi_controller *ctlr, struct spi_transfe
 	div = max(2U, clk_rate / xfer->speed_hz);
 
 	clk_sel = (div / 2) - 1;
-	pspim->xfer_conf &= ~SP7021_CLK_MASK;
-	pspim->xfer_conf |= FIELD_PREP(SP7021_CLK_MASK, clk_sel);
+	FIELD_MODIFY(SP7021_CLK_MASK, &pspim->xfer_conf, clk_sel);
 	writel(pspim->xfer_conf, pspim->m_base + SP7021_SPI_CONFIG_REG);
 }
 
diff --git a/drivers/spi/spi-synquacer.c b/drivers/spi/spi-synquacer.c
index 290c439897c4f..c14225e39fd1a 100644
--- a/drivers/spi/spi-synquacer.c
+++ b/drivers/spi/spi-synquacer.c
@@ -605,7 +605,7 @@ static int synquacer_spi_probe(struct platform_device *pdev)
 	int ret;
 	int rx_irq, tx_irq;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*sspi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*sspi));
 	if (!host)
 		return -ENOMEM;
 
@@ -617,10 +617,8 @@ static int synquacer_spi_probe(struct platform_device *pdev)
 	init_completion(&sspi->transfer_done);
 
 	sspi->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(sspi->regs)) {
-		ret = PTR_ERR(sspi->regs);
-		goto put_spi;
-	}
+	if (IS_ERR(sspi->regs))
+		return PTR_ERR(sspi->regs);
 
 	sspi->clk_src_type = SYNQUACER_HSSPI_CLOCK_SRC_IHCLK; /* Default */
 	device_property_read_u32(&pdev->dev, "socionext,ihclk-rate",
@@ -637,21 +635,19 @@ static int synquacer_spi_probe(struct platform_device *pdev)
 			sspi->clk = devm_clk_get(sspi->dev, "iPCLK");
 		} else {
 			dev_err(&pdev->dev, "specified wrong clock source\n");
-			ret = -EINVAL;
-			goto put_spi;
+			return -EINVAL;
 		}
 
 		if (IS_ERR(sspi->clk)) {
-			ret = dev_err_probe(&pdev->dev, PTR_ERR(sspi->clk),
-					    "clock not found\n");
-			goto put_spi;
+			return dev_err_probe(&pdev->dev, PTR_ERR(sspi->clk),
+					     "clock not found\n");
 		}
 
 		ret = clk_prepare_enable(sspi->clk);
 		if (ret) {
 			dev_err(&pdev->dev, "failed to enable clock (%d)\n",
 				ret);
-			goto put_spi;
+			return ret;
 		}
 
 		host->max_speed_hz = clk_get_rate(sspi->clk);
@@ -726,8 +722,6 @@ static int synquacer_spi_probe(struct platform_device *pdev)
 	pm_runtime_disable(sspi->dev);
 disable_clk:
 	clk_disable_unprepare(sspi->clk);
-put_spi:
-	spi_controller_put(host);
 
 	return ret;
 }
@@ -737,15 +731,11 @@ static void synquacer_spi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct synquacer_spi *sspi = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	pm_runtime_disable(sspi->dev);
 
 	clk_disable_unprepare(sspi->clk);
-
-	spi_controller_put(host);
 }
 
 static int __maybe_unused synquacer_spi_suspend(struct device *dev)
diff --git a/drivers/spi/spi-tegra114.c b/drivers/spi/spi-tegra114.c
index b8b0ebe0fe931..aa44ffd09e61b 100644
--- a/drivers/spi/spi-tegra114.c
+++ b/drivers/spi/spi-tegra114.c
@@ -1302,7 +1302,7 @@ static int tegra_spi_probe(struct platform_device *pdev)
 	int ret, spi_irq;
 	int bus_num;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*tspi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*tspi));
 	if (!host) {
 		dev_err(&pdev->dev, "host allocation failed\n");
 		return -ENOMEM;
@@ -1336,36 +1336,31 @@ static int tegra_spi_probe(struct platform_device *pdev)
 	tspi->soc_data = of_device_get_match_data(&pdev->dev);
 	if (!tspi->soc_data) {
 		dev_err(&pdev->dev, "unsupported tegra\n");
-		ret = -ENODEV;
-		goto exit_free_host;
+		return -ENODEV;
 	}
 
 	tspi->base = devm_platform_get_and_ioremap_resource(pdev, 0, &r);
-	if (IS_ERR(tspi->base)) {
-		ret = PTR_ERR(tspi->base);
-		goto exit_free_host;
-	}
+	if (IS_ERR(tspi->base))
+		return PTR_ERR(tspi->base);
+
 	tspi->phys = r->start;
 
 	spi_irq = platform_get_irq(pdev, 0);
-	if (spi_irq < 0) {
-		ret = spi_irq;
-		goto exit_free_host;
-	}
+	if (spi_irq < 0)
+		return spi_irq;
+
 	tspi->irq = spi_irq;
 
 	tspi->clk = devm_clk_get(&pdev->dev, "spi");
 	if (IS_ERR(tspi->clk)) {
 		dev_err(&pdev->dev, "can not get clock\n");
-		ret = PTR_ERR(tspi->clk);
-		goto exit_free_host;
+		return PTR_ERR(tspi->clk);
 	}
 
 	tspi->rst = devm_reset_control_get_exclusive(&pdev->dev, "spi");
 	if (IS_ERR(tspi->rst)) {
 		dev_err(&pdev->dev, "can not get reset\n");
-		ret = PTR_ERR(tspi->rst);
-		goto exit_free_host;
+		return PTR_ERR(tspi->rst);
 	}
 
 	tspi->max_buf_size = SPI_FIFO_DEPTH << 2;
@@ -1373,7 +1368,7 @@ static int tegra_spi_probe(struct platform_device *pdev)
 
 	ret = tegra_spi_init_dma_param(tspi, true);
 	if (ret < 0)
-		goto exit_free_host;
+		return ret;
 	ret = tegra_spi_init_dma_param(tspi, false);
 	if (ret < 0)
 		goto exit_rx_dma_free;
@@ -1431,8 +1426,7 @@ static int tegra_spi_probe(struct platform_device *pdev)
 	tegra_spi_deinit_dma_param(tspi, false);
 exit_rx_dma_free:
 	tegra_spi_deinit_dma_param(tspi, true);
-exit_free_host:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -1441,8 +1435,6 @@ static void tegra_spi_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct tegra_spi_data	*tspi = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	free_irq(tspi->irq, tspi);
@@ -1456,8 +1448,6 @@ static void tegra_spi_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 	if (!pm_runtime_status_suspended(&pdev->dev))
 		tegra_spi_runtime_suspend(&pdev->dev);
-
-	spi_controller_put(host);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/spi/spi-tegra20-sflash.c b/drivers/spi/spi-tegra20-sflash.c
index 9256729f2d491..2caa33f0a52c5 100644
--- a/drivers/spi/spi-tegra20-sflash.c
+++ b/drivers/spi/spi-tegra20-sflash.c
@@ -427,11 +427,9 @@ static int tegra_sflash_probe(struct platform_device *pdev)
 		return -ENODEV;
 	}
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*tsd));
-	if (!host) {
-		dev_err(&pdev->dev, "host allocation failed\n");
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*tsd));
+	if (!host)
 		return -ENOMEM;
-	}
 
 	/* the spi->mode bits understood by this driver: */
 	host->mode_bits = SPI_CPOL | SPI_CPHA;
@@ -450,14 +448,13 @@ static int tegra_sflash_probe(struct platform_device *pdev)
 		host->max_speed_hz = 25000000; /* 25MHz */
 
 	tsd->base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(tsd->base)) {
-		ret = PTR_ERR(tsd->base);
-		goto exit_free_host;
-	}
+	if (IS_ERR(tsd->base))
+		return PTR_ERR(tsd->base);
 
 	ret = platform_get_irq(pdev, 0);
 	if (ret < 0)
-		goto exit_free_host;
+		return ret;
+
 	tsd->irq = ret;
 
 	ret = request_irq(tsd->irq, tegra_sflash_isr, 0,
@@ -465,7 +462,7 @@ static int tegra_sflash_probe(struct platform_device *pdev)
 	if (ret < 0) {
 		dev_err(&pdev->dev, "Failed to register ISR for IRQ %d\n",
 					tsd->irq);
-		goto exit_free_host;
+		return ret;
 	}
 
 	tsd->clk = devm_clk_get(&pdev->dev, NULL);
@@ -518,8 +515,7 @@ static int tegra_sflash_probe(struct platform_device *pdev)
 		tegra_sflash_runtime_suspend(&pdev->dev);
 exit_free_irq:
 	free_irq(tsd->irq, tsd);
-exit_free_host:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -528,8 +524,6 @@ static void tegra_sflash_remove(struct platform_device *pdev)
 	struct spi_controller *host = platform_get_drvdata(pdev);
 	struct tegra_sflash_data	*tsd = spi_controller_get_devdata(host);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	free_irq(tsd->irq, tsd);
@@ -537,8 +531,6 @@ static void tegra_sflash_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 	if (!pm_runtime_status_suspended(&pdev->dev))
 		tegra_sflash_runtime_suspend(&pdev->dev);
-
-	spi_controller_put(host);
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/spi/spi-ti-qspi.c b/drivers/spi/spi-ti-qspi.c
index 1fbd710d616f6..34b154922ff24 100644
--- a/drivers/spi/spi-ti-qspi.c
+++ b/drivers/spi/spi-ti-qspi.c
@@ -765,7 +765,7 @@ static int ti_qspi_probe(struct platform_device *pdev)
 	int ret = 0, num_cs, irq;
 	dma_cap_mask_t mask;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*qspi));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*qspi));
 	if (!host)
 		return -ENOMEM;
 
@@ -793,8 +793,7 @@ static int ti_qspi_probe(struct platform_device *pdev)
 		r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 		if (r == NULL) {
 			dev_err(&pdev->dev, "missing platform data\n");
-			ret = -ENODEV;
-			goto free_host;
+			return -ENODEV;
 		}
 	}
 
@@ -812,28 +811,22 @@ static int ti_qspi_probe(struct platform_device *pdev)
 		qspi->mmap_size = resource_size(res_mmap);
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = irq;
-		goto free_host;
-	}
+	if (irq < 0)
+		return irq;
 
 	mutex_init(&qspi->list_lock);
 
 	qspi->base = devm_ioremap_resource(&pdev->dev, r);
-	if (IS_ERR(qspi->base)) {
-		ret = PTR_ERR(qspi->base);
-		goto free_host;
-	}
+	if (IS_ERR(qspi->base))
+		return PTR_ERR(qspi->base);
 
 
 	if (of_property_present(np, "syscon-chipselects")) {
 		qspi->ctrl_base =
 			syscon_regmap_lookup_by_phandle_args(np, "syscon-chipselects",
 							     1, &qspi->ctrl_reg);
-		if (IS_ERR(qspi->ctrl_base)) {
-			ret = PTR_ERR(qspi->ctrl_base);
-			goto free_host;
-		}
+		if (IS_ERR(qspi->ctrl_base))
+			return PTR_ERR(qspi->ctrl_base);
 	}
 
 	qspi->fclk = devm_clk_get(&pdev->dev, "fck");
@@ -867,6 +860,7 @@ static int ti_qspi_probe(struct platform_device *pdev)
 		dev_err(qspi->dev,
 			"dma_alloc_coherent failed, using PIO mode\n");
 		dma_release_channel(qspi->rx_chan);
+		qspi->rx_chan = NULL;
 		goto no_dma;
 	}
 	host->dma_rx = qspi->rx_chan;
@@ -889,14 +883,16 @@ static int ti_qspi_probe(struct platform_device *pdev)
 	qspi->current_cs = -1;
 
 	ret = spi_register_controller(host);
-	if (!ret)
-		return 0;
+	if (ret)
+		goto err_free_dma;
 
+	return 0;
+
+err_free_dma:
 	ti_qspi_dma_cleanup(qspi);
 
 	pm_runtime_disable(&pdev->dev);
-free_host:
-	spi_controller_put(host);
+
 	return ret;
 }
 
@@ -904,16 +900,12 @@ static void ti_qspi_remove(struct platform_device *pdev)
 {
 	struct ti_qspi *qspi = platform_get_drvdata(pdev);
 
-	spi_controller_get(qspi->host);
-
 	spi_unregister_controller(qspi->host);
 
 	pm_runtime_put_sync(&pdev->dev);
 	pm_runtime_disable(&pdev->dev);
 
 	ti_qspi_dma_cleanup(qspi);
-
-	spi_controller_put(qspi->host);
 }
 
 static const struct dev_pm_ops ti_qspi_pm_ops = {
diff --git a/drivers/spi/spi-topcliff-pch.c b/drivers/spi/spi-topcliff-pch.c
index 14d11450e86dc..02ced638d8b47 100644
--- a/drivers/spi/spi-topcliff-pch.c
+++ b/drivers/spi/spi-topcliff-pch.c
@@ -207,10 +207,10 @@ struct pch_pd_dev_save {
 };
 
 static const struct pci_device_id pch_spi_pcidev_id[] = {
-	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_GE_SPI),    1, },
-	{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_SPI), 2, },
-	{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7223_SPI), 1, },
-	{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7831_SPI), 1, },
+	{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_GE_SPI),    .driver_data = 1 },
+	{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7213_SPI), .driver_data = 2 },
+	{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7223_SPI), .driver_data = 1 },
+	{ PCI_VDEVICE(ROHM, PCI_DEVICE_ID_ML7831_SPI), .driver_data = 1 },
 	{ }
 };
 
diff --git a/drivers/spi/spi-uniphier.c b/drivers/spi/spi-uniphier.c
index eac6c3e8908bd..cc20fd11f03fd 100644
--- a/drivers/spi/spi-uniphier.c
+++ b/drivers/spi/spi-uniphier.c
@@ -184,14 +184,12 @@ static void uniphier_spi_set_transfer_size(struct spi_device *spi, int size)
 	u32 val;
 
 	val = readl(priv->base + SSI_TXWDS);
-	val &= ~(SSI_TXWDS_WDLEN_MASK | SSI_TXWDS_DTLEN_MASK);
-	val |= FIELD_PREP(SSI_TXWDS_WDLEN_MASK, size);
-	val |= FIELD_PREP(SSI_TXWDS_DTLEN_MASK, size);
+	FIELD_MODIFY(SSI_TXWDS_WDLEN_MASK, &val, size);
+	FIELD_MODIFY(SSI_TXWDS_DTLEN_MASK, &val, size);
 	writel(val, priv->base + SSI_TXWDS);
 
 	val = readl(priv->base + SSI_RXWDS);
-	val &= ~SSI_RXWDS_DTLEN_MASK;
-	val |= FIELD_PREP(SSI_RXWDS_DTLEN_MASK, size);
+	FIELD_MODIFY(SSI_RXWDS_DTLEN_MASK, &val, size);
 	writel(val, priv->base + SSI_RXWDS);
 }
 
@@ -308,9 +306,8 @@ static void uniphier_spi_set_fifo_threshold(struct uniphier_spi_priv *priv,
 	u32 val;
 
 	val = readl(priv->base + SSI_FC);
-	val &= ~(SSI_FC_TXFTH_MASK | SSI_FC_RXFTH_MASK);
-	val |= FIELD_PREP(SSI_FC_TXFTH_MASK, SSI_FIFO_DEPTH - threshold);
-	val |= FIELD_PREP(SSI_FC_RXFTH_MASK, threshold);
+	FIELD_MODIFY(SSI_FC_TXFTH_MASK, &val, SSI_FIFO_DEPTH - threshold);
+	FIELD_MODIFY(SSI_FC_RXFTH_MASK, &val, threshold);
 	writel(val, priv->base + SSI_FC);
 }
 
@@ -649,7 +646,7 @@ static int uniphier_spi_probe(struct platform_device *pdev)
 	int irq;
 	int ret;
 
-	host = spi_alloc_host(&pdev->dev, sizeof(*priv));
+	host = devm_spi_alloc_host(&pdev->dev, sizeof(*priv));
 	if (!host)
 		return -ENOMEM;
 
@@ -660,30 +657,26 @@ static int uniphier_spi_probe(struct platform_device *pdev)
 	priv->is_save_param = false;
 
 	priv->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
-	if (IS_ERR(priv->base)) {
-		ret = PTR_ERR(priv->base);
-		goto out_host_put;
-	}
+	if (IS_ERR(priv->base))
+		return PTR_ERR(priv->base);
+
 	priv->base_dma_addr = res->start;
 
 	priv->clk = devm_clk_get_enabled(&pdev->dev, NULL);
 	if (IS_ERR(priv->clk)) {
 		dev_err(&pdev->dev, "failed to get clock\n");
-		ret = PTR_ERR(priv->clk);
-		goto out_host_put;
+		return PTR_ERR(priv->clk);
 	}
 
 	irq = platform_get_irq(pdev, 0);
-	if (irq < 0) {
-		ret = irq;
-		goto out_host_put;
-	}
+	if (irq < 0)
+		return irq;
 
 	ret = devm_request_irq(&pdev->dev, irq, uniphier_spi_handler,
 			       0, "uniphier-spi", priv);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to request IRQ\n");
-		goto out_host_put;
+		return ret;
 	}
 
 	init_completion(&priv->xfer_done);
@@ -710,10 +703,9 @@ static int uniphier_spi_probe(struct platform_device *pdev)
 
 	host->dma_tx = dma_request_chan(&pdev->dev, "tx");
 	if (IS_ERR_OR_NULL(host->dma_tx)) {
-		if (PTR_ERR(host->dma_tx) == -EPROBE_DEFER) {
-			ret = -EPROBE_DEFER;
-			goto out_host_put;
-		}
+		if (PTR_ERR(host->dma_tx) == -EPROBE_DEFER)
+			return -EPROBE_DEFER;
+
 		host->dma_tx = NULL;
 		dma_tx_burst = INT_MAX;
 	} else {
@@ -762,8 +754,6 @@ static int uniphier_spi_probe(struct platform_device *pdev)
 		host->dma_tx = NULL;
 	}
 
-out_host_put:
-	spi_controller_put(host);
 	return ret;
 }
 
@@ -771,16 +761,12 @@ static void uniphier_spi_remove(struct platform_device *pdev)
 {
 	struct spi_controller *host = platform_get_drvdata(pdev);
 
-	spi_controller_get(host);
-
 	spi_unregister_controller(host);
 
 	if (host->dma_tx)
 		dma_release_channel(host->dma_tx);
 	if (host->dma_rx)
 		dma_release_channel(host->dma_rx);
-
-	spi_controller_put(host);
 }
 
 static const struct of_device_id uniphier_spi_match[] = {
diff --git a/drivers/spi/spi-wpcm-fiu.c b/drivers/spi/spi-wpcm-fiu.c
index 0e26ff1785056..cd78e927953dc 100644
--- a/drivers/spi/spi-wpcm-fiu.c
+++ b/drivers/spi/spi-wpcm-fiu.c
@@ -377,7 +377,7 @@ static int wpcm_fiu_dirmap_create(struct spi_mem_dirmap_desc *desc)
 	struct wpcm_fiu_spi *fiu = spi_controller_get_devdata(desc->mem->spi->controller);
 	int cs = spi_get_chipselect(desc->mem->spi, 0);
 
-	if (desc->info.op_tmpl.data.dir != SPI_MEM_DATA_IN)
+	if (desc->info.op_tmpl->data.dir != SPI_MEM_DATA_IN)
 		return -EOPNOTSUPP;
 
 	/*
diff --git a/drivers/spi/spi-xlp.c b/drivers/spi/spi-xlp.c
index be8bbe1cbba31..d014955e6d4b3 100644
--- a/drivers/spi/spi-xlp.c
+++ b/drivers/spi/spi-xlp.c
@@ -398,7 +398,7 @@ static int xlp_spi_probe(struct platform_device *pdev)
 
 	xspi->spi_clk = clk_get_rate(clk);
 
-	host = spi_alloc_host(&pdev->dev, 0);
+	host = devm_spi_alloc_host(&pdev->dev, 0);
 	if (!host) {
 		dev_err(&pdev->dev, "could not alloc host\n");
 		return -ENOMEM;
@@ -418,7 +418,6 @@ static int xlp_spi_probe(struct platform_device *pdev)
 	err = devm_spi_register_controller(&pdev->dev, host);
 	if (err) {
 		dev_err(&pdev->dev, "spi register host failed!\n");
-		spi_controller_put(host);
 		return err;
 	}
 
diff --git a/drivers/spi/spi-zynq-qspi.c b/drivers/spi/spi-zynq-qspi.c
index 406fd9d5337ee..d762aaf452afd 100644
--- a/drivers/spi/spi-zynq-qspi.c
+++ b/drivers/spi/spi-zynq-qspi.c
@@ -637,7 +637,7 @@ static int zynq_qspi_probe(struct platform_device *pdev)
 	struct zynq_qspi *xqspi;
 	u32 num_cs;
 
-	ctlr = spi_alloc_host(&pdev->dev, sizeof(*xqspi));
+	ctlr = devm_spi_alloc_host(&pdev->dev, sizeof(*xqspi));
 	if (!ctlr)
 		return -ENOMEM;
 
@@ -645,16 +645,13 @@ static int zynq_qspi_probe(struct platform_device *pdev)
 	xqspi->dev = dev;
 	platform_set_drvdata(pdev, ctlr);
 	xqspi->regs = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(xqspi->regs)) {
-		ret = PTR_ERR(xqspi->regs);
-		goto remove_ctlr;
-	}
+	if (IS_ERR(xqspi->regs))
+		return PTR_ERR(xqspi->regs);
 
 	xqspi->pclk = devm_clk_get_enabled(&pdev->dev, "pclk");
 	if (IS_ERR(xqspi->pclk)) {
 		dev_err(&pdev->dev, "pclk clock not found.\n");
-		ret = PTR_ERR(xqspi->pclk);
-		goto remove_ctlr;
+		return PTR_ERR(xqspi->pclk);
 	}
 
 	init_completion(&xqspi->data_completion);
@@ -662,21 +659,18 @@ static int zynq_qspi_probe(struct platform_device *pdev)
 	xqspi->refclk = devm_clk_get_enabled(&pdev->dev, "ref_clk");
 	if (IS_ERR(xqspi->refclk)) {
 		dev_err(&pdev->dev, "ref_clk clock not found.\n");
-		ret = PTR_ERR(xqspi->refclk);
-		goto remove_ctlr;
+		return PTR_ERR(xqspi->refclk);
 	}
 
 	xqspi->irq = platform_get_irq(pdev, 0);
-	if (xqspi->irq < 0) {
-		ret = xqspi->irq;
-		goto remove_ctlr;
-	}
+	if (xqspi->irq < 0)
+		return xqspi->irq;
+
 	ret = devm_request_irq(&pdev->dev, xqspi->irq, zynq_qspi_irq,
 			       0, pdev->name, xqspi);
 	if (ret != 0) {
-		ret = -ENXIO;
 		dev_err(&pdev->dev, "request_irq failed\n");
-		goto remove_ctlr;
+		return -ENXIO;
 	}
 
 	ret = of_property_read_u32(np, "num-cs",
@@ -684,9 +678,8 @@ static int zynq_qspi_probe(struct platform_device *pdev)
 	if (ret < 0) {
 		ctlr->num_chipselect = 1;
 	} else if (num_cs > ZYNQ_QSPI_MAX_NUM_CS) {
-		ret = -EINVAL;
 		dev_err(&pdev->dev, "only 2 chip selects are available\n");
-		goto remove_ctlr;
+		return -EINVAL;
 	} else {
 		ctlr->num_chipselect = num_cs;
 	}
@@ -705,15 +698,10 @@ static int zynq_qspi_probe(struct platform_device *pdev)
 	ret = spi_register_controller(ctlr);
 	if (ret) {
 		dev_err(&pdev->dev, "failed to register controller\n");
-		goto remove_ctlr;
+		return ret;
 	}
 
-	return ret;
-
-remove_ctlr:
-	spi_controller_put(ctlr);
-
-	return ret;
+	return 0;
 }
 
 /**
@@ -731,13 +719,9 @@ static void zynq_qspi_remove(struct platform_device *pdev)
 	struct spi_controller *ctlr = platform_get_drvdata(pdev);
 	struct zynq_qspi *xqspi = spi_controller_get_devdata(ctlr);
 
-	spi_controller_get(ctlr);
-
 	spi_unregister_controller(ctlr);
 
 	zynq_qspi_write(xqspi, ZYNQ_QSPI_ENABLE_OFFSET, 0);
-
-	spi_controller_put(ctlr);
 }
 
 static const struct of_device_id zynq_qspi_of_match[] = {
diff --git a/drivers/spi/spi.c b/drivers/spi/spi.c
index 104279858f567..5f57de24b9f76 100644
--- a/drivers/spi/spi.c
+++ b/drivers/spi/spi.c
@@ -3552,7 +3552,8 @@ int spi_register_controller(struct spi_controller *ctlr)
 	/* Register devices from the device tree and ACPI */
 	of_register_spi_devices(ctlr);
 	acpi_register_spi_devices(ctlr);
-	return status;
+
+	return 0;
 
 del_ctrl:
 	device_del(&ctlr->dev);
@@ -3560,6 +3561,7 @@ int spi_register_controller(struct spi_controller *ctlr)
 	mutex_lock(&board_lock);
 	idr_remove(&spi_controller_idr, ctlr->bus_num);
 	mutex_unlock(&board_lock);
+
 	return status;
 }
 EXPORT_SYMBOL_GPL(spi_register_controller);
diff --git a/drivers/staging/fbtft/fb_ssd1351.c b/drivers/staging/fbtft/fb_ssd1351.c
index 6736b09b2f458..15524b80035a1 100644
--- a/drivers/staging/fbtft/fb_ssd1351.c
+++ b/drivers/staging/fbtft/fb_ssd1351.c
@@ -218,8 +218,7 @@ static void register_onboard_backlight(struct fbtft_par *par)
 				       &bl_props);
 	if (IS_ERR(bd)) {
 		dev_err(par->info->device,
-			"cannot register backlight device (%ld)\n",
-			PTR_ERR(bd));
+			"cannot register backlight device (%pe)\n", bd);
 		return;
 	}
 	par->info->bl_dev = bd;
diff --git a/drivers/staging/fbtft/fbtft-core.c b/drivers/staging/fbtft/fbtft-core.c
index 3da42c8ca6e33..ca0c38221c16b 100644
--- a/drivers/staging/fbtft/fbtft-core.c
+++ b/drivers/staging/fbtft/fbtft-core.c
@@ -187,8 +187,7 @@ void fbtft_register_backlight(struct fbtft_par *par)
 				       &fbtft_bl_ops, &bl_props);
 	if (IS_ERR(bd)) {
 		dev_err(par->info->device,
-			"cannot register backlight device (%ld)\n",
-			PTR_ERR(bd));
+			"cannot register backlight device (%pe)\n", bd);
 		return;
 	}
 	par->info->bl_dev = bd;
diff --git a/drivers/staging/fbtft/fbtft-io.c b/drivers/staging/fbtft/fbtft-io.c
index de1904a443c27..7d331eba947aa 100644
--- a/drivers/staging/fbtft/fbtft-io.c
+++ b/drivers/staging/fbtft/fbtft-io.c
@@ -227,10 +227,3 @@ int fbtft_write_gpio16_wr(struct fbtft_par *par, void *buf, size_t len)
 	return 0;
 }
 EXPORT_SYMBOL(fbtft_write_gpio16_wr);
-
-int fbtft_write_gpio16_wr_latched(struct fbtft_par *par, void *buf, size_t len)
-{
-	dev_err(par->info->device, "%s: function not implemented\n", __func__);
-	return -1;
-}
-EXPORT_SYMBOL(fbtft_write_gpio16_wr_latched);
diff --git a/drivers/staging/fbtft/fbtft.h b/drivers/staging/fbtft/fbtft.h
index 317be17b95c16..c7afb0fd39433 100644
--- a/drivers/staging/fbtft/fbtft.h
+++ b/drivers/staging/fbtft/fbtft.h
@@ -262,7 +262,6 @@ int fbtft_write_spi_emulate_9(struct fbtft_par *par, void *buf, size_t len);
 int fbtft_read_spi(struct fbtft_par *par, void *buf, size_t len);
 int fbtft_write_gpio8_wr(struct fbtft_par *par, void *buf, size_t len);
 int fbtft_write_gpio16_wr(struct fbtft_par *par, void *buf, size_t len);
-int fbtft_write_gpio16_wr_latched(struct fbtft_par *par, void *buf, size_t len);
 
 /* fbtft-bus.c */
 int fbtft_write_vmem8_bus8(struct fbtft_par *par, size_t offset, size_t len);
diff --git a/drivers/staging/greybus/Documentation/firmware/authenticate.c b/drivers/staging/greybus/Documentation/firmware/authenticate.c
index 0ef88b7d24de0..ba4b16b045571 100644
--- a/drivers/staging/greybus/Documentation/firmware/authenticate.c
+++ b/drivers/staging/greybus/Documentation/firmware/authenticate.c
@@ -85,7 +85,7 @@ int main(int argc, char *argv[])
 	}
 
 	printf("Authenticated, result (%02x), sig-size (%02x)\n",
-		authenticate.result_code, authenticate.signature_size);
+	       authenticate.result_code, authenticate.signature_size);
 
 close_fd:
 	close(fd);
diff --git a/drivers/staging/greybus/Documentation/sysfs-bus-greybus b/drivers/staging/greybus/Documentation/sysfs-bus-greybus
index 2e998966cbe1f..feeffe54a59b2 100644
--- a/drivers/staging/greybus/Documentation/sysfs-bus-greybus
+++ b/drivers/staging/greybus/Documentation/sysfs-bus-greybus
@@ -26,7 +26,7 @@ Date:		March 2016
 KernelVersion:	4.XX
 Contact:	Greg Kroah-Hartman <greg@kroah.com>
 Description:
-		Writing a non-zero argument to this attibute disables the
+		Writing a non-zero argument to this attribute disables the
 		module's interfaces before physically ejecting it.
 
 What:		/sys/bus/greybus/devices/N-M/module_id
diff --git a/drivers/staging/iio/addac/adt7316-spi.c b/drivers/staging/iio/addac/adt7316-spi.c
index f91325d11394d..1debcc36c1afb 100644
--- a/drivers/staging/iio/addac/adt7316-spi.c
+++ b/drivers/staging/iio/addac/adt7316-spi.c
@@ -106,7 +106,18 @@ static int adt7316_spi_probe(struct spi_device *spi_dev)
 		return -EINVAL;
 	}
 
-	/* switch from default I2C protocol to SPI protocol */
+	/*
+	 * The device powers up in I2C mode. Switching to SPI mode
+	 * requires sending a sequence of SPI writes as described in
+	 * the datasheet "ADT7316/ADT7317/ADT7318", Rev. B,
+	 * in the "Serial Interface Selection" section.
+	 *
+	 * During this sequence, the device may still be in I2C mode,
+	 * so SPI transactions may not be recognized and can fail.
+	 * Such errors are therefore ignored.
+	 *
+	 * TL;DR: Do not change this!
+	 */
 	adt7316_spi_write(spi_dev, 0, 0);
 	adt7316_spi_write(spi_dev, 0, 0);
 	adt7316_spi_write(spi_dev, 0, 0);
diff --git a/drivers/staging/iio/frequency/ad9832.c b/drivers/staging/iio/frequency/ad9832.c
index b87ea1781b270..659821a1e2cb3 100644
--- a/drivers/staging/iio/frequency/ad9832.c
+++ b/drivers/staging/iio/frequency/ad9832.c
@@ -5,21 +5,25 @@
  * Copyright 2011 Analog Devices Inc.
  */
 
-#include <asm/div64.h>
-
+#include <linux/array_size.h>
 #include <linux/bitfield.h>
-#include <linux/bits.h>
+#include <linux/bitops.h>
 #include <linux/clk.h>
-#include <linux/device.h>
+#include <linux/dev_printk.h>
 #include <linux/err.h>
-#include <linux/kernel.h>
+#include <linux/kstrtox.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/regulator/consumer.h>
-#include <linux/slab.h>
 #include <linux/spi/spi.h>
 #include <linux/sysfs.h>
+#include <linux/types.h>
 #include <linux/unaligned.h>
 
+#include <asm/byteorder.h>
+#include <asm/div64.h>
+
 #include <linux/iio/iio.h>
 #include <linux/iio/sysfs.h>
 
@@ -113,8 +117,8 @@ struct ad9832_state {
 
 static unsigned long ad9832_calc_freqreg(unsigned long mclk, unsigned long fout)
 {
-	unsigned long long freqreg = (u64)fout *
-				     (u64)((u64)1L << AD9832_FREQ_BITS);
+	u64 freqreg = (u64)fout << AD9832_FREQ_BITS;
+
 	do_div(freqreg, mclk);
 	return freqreg;
 }
diff --git a/drivers/staging/iio/frequency/ad9834.c b/drivers/staging/iio/frequency/ad9834.c
index bdb2580e29bfd..4359b358e0e5a 100644
--- a/drivers/staging/iio/frequency/ad9834.c
+++ b/drivers/staging/iio/frequency/ad9834.c
@@ -5,18 +5,21 @@
  * Copyright 2010-2011 Analog Devices Inc.
  */
 
+#include <linux/bits.h>
 #include <linux/clk.h>
-#include <linux/interrupt.h>
-#include <linux/workqueue.h>
-#include <linux/device.h>
-#include <linux/kernel.h>
-#include <linux/slab.h>
-#include <linux/sysfs.h>
-#include <linux/list.h>
-#include <linux/spi/spi.h>
-#include <linux/regulator/consumer.h>
+#include <linux/dev_printk.h>
 #include <linux/err.h>
+#include <linux/kstrtox.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/regulator/consumer.h>
+#include <linux/spi/spi.h>
+#include <linux/string.h>
+#include <linux/sysfs.h>
+#include <linux/types.h>
+
+#include <asm/byteorder.h>
 #include <asm/div64.h>
 
 #include <linux/iio/iio.h>
@@ -164,7 +167,7 @@ static ssize_t ad9834_write(struct device *dev,
 		break;
 	case AD9834_OPBITEN:
 		if (st->control & AD9834_MODE) {
-			ret = -EINVAL;  /* AD9843 reserved mode */
+			ret = -EINVAL;  /* AD9834 reserved mode */
 			break;
 		}
 
@@ -239,7 +242,7 @@ static ssize_t ad9834_store_wavetype(struct device *dev,
 				st->control &= ~AD9834_OPBITEN;
 				st->control |= AD9834_MODE;
 			} else if (st->control & AD9834_OPBITEN) {
-				ret = -EINVAL;	/* AD9843 reserved mode */
+				ret = -EINVAL;	/* AD9834 reserved mode */
 			} else {
 				st->control |= AD9834_MODE;
 			}
@@ -389,17 +392,14 @@ static int ad9834_probe(struct spi_device *spi)
 		return dev_err_probe(&spi->dev, ret, "Failed to enable specified AVDD supply\n");
 
 	indio_dev = devm_iio_device_alloc(&spi->dev, sizeof(*st));
-	if (!indio_dev) {
-		ret = -ENOMEM;
-		return ret;
-	}
+	if (!indio_dev)
+		return -ENOMEM;
 	st = iio_priv(indio_dev);
 	mutex_init(&st->lock);
 	st->mclk = devm_clk_get_enabled(&spi->dev, NULL);
-	if (IS_ERR(st->mclk)) {
-		dev_err(&spi->dev, "Failed to enable master clock\n");
-		return PTR_ERR(st->mclk);
-	}
+	if (IS_ERR(st->mclk))
+		return dev_err_probe(&spi->dev, PTR_ERR(st->mclk),
+				     "Failed to enable master clock\n");
 
 	st->spi = spi;
 	st->devid = spi_get_device_id(spi)->driver_data;
@@ -441,10 +441,9 @@ static int ad9834_probe(struct spi_device *spi)
 
 	st->data = cpu_to_be16(AD9834_REG_CMD | st->control);
 	ret = spi_sync(st->spi, &st->msg);
-	if (ret) {
-		dev_err(&spi->dev, "device init failed\n");
-		return ret;
-	}
+	if (ret)
+		return dev_err_probe(&spi->dev, ret,
+				     "device init failed\n");
 
 	ret = ad9834_write_frequency(st, AD9834_REG_FREQ0, 1000000);
 	if (ret)
diff --git a/drivers/staging/media/Kconfig b/drivers/staging/media/Kconfig
index 1aa31bddf9703..06cbc5c548ca0 100644
--- a/drivers/staging/media/Kconfig
+++ b/drivers/staging/media/Kconfig
@@ -50,8 +50,4 @@ menuconfig STAGING_MEDIA_DEPRECATED
 
 	  If in doubt, say N here.
 
-if STAGING_MEDIA_DEPRECATED
-source "drivers/staging/media/deprecated/atmel/Kconfig"
-endif
-
 endif
diff --git a/drivers/staging/media/Makefile b/drivers/staging/media/Makefile
index 6f78b0edde1e3..6fd7179733d80 100644
--- a/drivers/staging/media/Makefile
+++ b/drivers/staging/media/Makefile
@@ -1,5 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_VIDEO_ATMEL_ISC_BASE)	+= deprecated/atmel/
 obj-$(CONFIG_INTEL_ATOMISP)     += atomisp/
 obj-$(CONFIG_VIDEO_IMX_MEDIA)	+= imx/
 obj-$(CONFIG_VIDEO_MAX96712)	+= max96712/
diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
index d3414312e1de2..f9cc5f45fe000 100644
--- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
+++ b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c
@@ -433,7 +433,7 @@ static int power_up(struct v4l2_subdev *sd)
 			goto fail_power;
 	}
 
-	msleep(5);
+	fsleep(5000);
 	return 0;
 
 fail_clk:
@@ -809,7 +809,7 @@ static int gc2235_probe(struct i2c_client *client)
 
 	ret = gc2235_s_config(&dev->sd, client->irq, gcpdev);
 	if (ret)
-		goto out_free;
+		goto err_unregister_subdev;
 
 	dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE;
 	dev->pad.flags = MEDIA_PAD_FL_SOURCE;
@@ -818,18 +818,16 @@ static int gc2235_probe(struct i2c_client *client)
 	ret =
 	    v4l2_ctrl_handler_init(&dev->ctrl_handler,
 				   ARRAY_SIZE(gc2235_controls));
-	if (ret) {
-		gc2235_remove(client);
-		return ret;
-	}
+	if (ret)
+		goto err_csi_cfg;
 
 	for (i = 0; i < ARRAY_SIZE(gc2235_controls); i++)
 		v4l2_ctrl_new_custom(&dev->ctrl_handler, &gc2235_controls[i],
 				     NULL);
 
 	if (dev->ctrl_handler.error) {
-		gc2235_remove(client);
-		return dev->ctrl_handler.error;
+		ret = dev->ctrl_handler.error;
+		goto err_ctrl_handler;
 	}
 
 	/* Use same lock for controls as for everything else. */
@@ -838,14 +836,23 @@ static int gc2235_probe(struct i2c_client *client)
 
 	ret = media_entity_pads_init(&dev->sd.entity, 1, &dev->pad);
 	if (ret)
-		gc2235_remove(client);
+		goto err_ctrl_handler;
+
+	ret = atomisp_register_i2c_module(&dev->sd, gcpdev);
+	if (ret)
+		goto err_media_cleanup;
 
-	return atomisp_register_i2c_module(&dev->sd, gcpdev);
+	return 0;
 
-out_free:
+err_media_cleanup:
+	media_entity_cleanup(&dev->sd.entity);
+err_ctrl_handler:
+	v4l2_ctrl_handler_free(&dev->ctrl_handler);
+err_csi_cfg:
+	dev->platform_data->csi_cfg(&dev->sd, 0);
+err_unregister_subdev:
 	v4l2_device_unregister_subdev(&dev->sd);
 	kfree(dev);
-
 	return ret;
 }
 
diff --git a/drivers/staging/media/atomisp/i2c/ov2722.h b/drivers/staging/media/atomisp/i2c/ov2722.h
index 00317d1053057..7d5ea8801fe74 100644
--- a/drivers/staging/media/atomisp/i2c/ov2722.h
+++ b/drivers/staging/media/atomisp/i2c/ov2722.h
@@ -232,342 +232,9 @@ struct ov2722_write_ctrl {
 	struct ov2722_write_buffer buffer;
 };
 
-/*
- * Register settings for various resolution
- */
-#if 0
-static const struct ov2722_reg ov2722_QVGA_30fps[] = {
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x0c},
-	{OV2722_8BIT, 0x373a, 0x1c},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x0c},
-	{OV2722_8BIT, 0x3705, 0x06},
-	{OV2722_8BIT, 0x3730, 0x0e},
-	{OV2722_8BIT, 0x3704, 0x1c},
-	{OV2722_8BIT, 0x3f06, 0x00},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0x46},
-	{OV2722_8BIT, 0x371e, 0x00},
-	{OV2722_8BIT, 0x371f, 0x63},
-	{OV2722_8BIT, 0x3708, 0x61},
-	{OV2722_8BIT, 0x3709, 0x12},
-	{OV2722_8BIT, 0x3800, 0x01},
-	{OV2722_8BIT, 0x3801, 0x42}, /* H crop start: 322 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x20}, /* V crop start: 32 */
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0x95}, /* H crop end:  1685 */
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x27}, /* V crop end:  1063 */
-	{OV2722_8BIT, 0x3808, 0x01},
-	{OV2722_8BIT, 0x3809, 0x50}, /* H output size: 336 */
-	{OV2722_8BIT, 0x380a, 0x01},
-	{OV2722_8BIT, 0x380b, 0x00}, /* V output size: 256 */
-
-	/* H blank timing */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00}, /* H total size: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa0}, /* V total size: 1184 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x04}, /* H window offset: 5 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x01}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0xc0},
-	{OV2722_8BIT, 0x3821, 0x06}, /* flip isp*/
-	{OV2722_8BIT, 0x3814, 0x71},
-	{OV2722_8BIT, 0x3815, 0x71},
-	{OV2722_8BIT, 0x3612, 0x49},
-	{OV2722_8BIT, 0x3618, 0x00},
-	{OV2722_8BIT, 0x3a08, 0x01},
-	{OV2722_8BIT, 0x3a09, 0xc3},
-	{OV2722_8BIT, 0x3a0a, 0x01},
-	{OV2722_8BIT, 0x3a0b, 0x77},
-	{OV2722_8BIT, 0x3a0d, 0x00},
-	{OV2722_8BIT, 0x3a0e, 0x00},
-	{OV2722_8BIT, 0x4520, 0x09},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3000, 0xff},
-	{OV2722_8BIT, 0x3001, 0xff},
-	{OV2722_8BIT, 0x3002, 0xf0},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0x53}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x63},
-	{OV2722_8BIT, 0x3634, 0x24},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1}, /* v_en, h_en, blc_en */
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xff},
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0}, /* AWB red */
-	{OV2722_8BIT, 0x5184, 0xb0}, /* AWB green */
-	{OV2722_8BIT, 0x5185, 0xb0}, /* AWB blue */
-	{OV2722_8BIT, 0x5180, 0x03}, /* AWB manual mode */
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x4800, 0x24}, /* clk lane gate enable */
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26},
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-
-	/* Added for power optimization */
-	{OV2722_8BIT, 0x3000, 0x00},
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3011, 0x22},
-	{OV2722_8BIT, 0x3a00, 0x58},
-	{OV2722_8BIT, 0x3503, 0x17},
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x46},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x10},
-	{OV2722_TOK_TERM, 0, 0},
-
-};
-
-static const struct ov2722_reg ov2722_480P_30fps[] = {
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x18},
-	{OV2722_8BIT, 0x373a, 0x3c},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x1d},
-	{OV2722_8BIT, 0x3705, 0x12},
-	{OV2722_8BIT, 0x3730, 0x1f},
-	{OV2722_8BIT, 0x3704, 0x3f},
-	{OV2722_8BIT, 0x3f06, 0x1d},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0x83},
-	{OV2722_8BIT, 0x371e, 0x00},
-	{OV2722_8BIT, 0x371f, 0xbd},
-	{OV2722_8BIT, 0x3708, 0x63},
-	{OV2722_8BIT, 0x3709, 0x52},
-	{OV2722_8BIT, 0x3800, 0x00},
-	{OV2722_8BIT, 0x3801, 0xf2}, /* H crop start: 322 - 80 = 242*/
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x20}, /* V crop start:  32*/
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0xBB}, /* H crop end:   1643 + 80 = 1723*/
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x03}, /* V crop end:   1027*/
-	{OV2722_8BIT, 0x3808, 0x02},
-	{OV2722_8BIT, 0x3809, 0xE0}, /* H output size: 656 +80 = 736*/
-	{OV2722_8BIT, 0x380a, 0x01},
-	{OV2722_8BIT, 0x380b, 0xF0}, /* V output size: 496 */
-
-	/* H blank timing */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00}, /* H total size: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa0}, /* V total size: 1184 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x04}, /* H window offset: 5 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x01}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06}, /* flip isp*/
-	{OV2722_8BIT, 0x3814, 0x31},
-	{OV2722_8BIT, 0x3815, 0x31},
-	{OV2722_8BIT, 0x3612, 0x4b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x02},
-	{OV2722_8BIT, 0x3a09, 0x67},
-	{OV2722_8BIT, 0x3a0a, 0x02},
-	{OV2722_8BIT, 0x3a0b, 0x00},
-	{OV2722_8BIT, 0x3a0d, 0x00},
-	{OV2722_8BIT, 0x3a0e, 0x00},
-	{OV2722_8BIT, 0x4520, 0x0a},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3000, 0xff},
-	{OV2722_8BIT, 0x3001, 0xff},
-	{OV2722_8BIT, 0x3002, 0xf0},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0x53}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x63},
-	{OV2722_8BIT, 0x3634, 0x24},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1}, /* v_en, h_en, blc_en */
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xff},
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0}, /* AWB red */
-	{OV2722_8BIT, 0x5184, 0xb0}, /* AWB green */
-	{OV2722_8BIT, 0x5185, 0xb0}, /* AWB blue */
-	{OV2722_8BIT, 0x5180, 0x03}, /* AWB manual mode */
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x4800, 0x24}, /* clk lane gate enable */
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26},
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-
-	/* Added for power optimization */
-	{OV2722_8BIT, 0x3000, 0x00},
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3011, 0x22},
-	{OV2722_8BIT, 0x3a00, 0x58},
-	{OV2722_8BIT, 0x3503, 0x17},
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x46},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x10},
-	{OV2722_TOK_TERM, 0, 0},
-};
-
-static const struct ov2722_reg ov2722_VGA_30fps[] = {
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x18},
-	{OV2722_8BIT, 0x373a, 0x3c},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x1d},
-	{OV2722_8BIT, 0x3705, 0x12},
-	{OV2722_8BIT, 0x3730, 0x1f},
-	{OV2722_8BIT, 0x3704, 0x3f},
-	{OV2722_8BIT, 0x3f06, 0x1d},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0x83},
-	{OV2722_8BIT, 0x371e, 0x00},
-	{OV2722_8BIT, 0x371f, 0xbd},
-	{OV2722_8BIT, 0x3708, 0x63},
-	{OV2722_8BIT, 0x3709, 0x52},
-	{OV2722_8BIT, 0x3800, 0x01},
-	{OV2722_8BIT, 0x3801, 0x42}, /* H crop start: 322 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x20}, /* V crop start:  32*/
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0x6B}, /* H crop end:   1643*/
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x03}, /* V crop end:   1027*/
-	{OV2722_8BIT, 0x3808, 0x02},
-	{OV2722_8BIT, 0x3809, 0x90}, /* H output size: 656 */
-	{OV2722_8BIT, 0x380a, 0x01},
-	{OV2722_8BIT, 0x380b, 0xF0}, /* V output size: 496 */
-
-	/* H blank timing */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00}, /* H total size: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa0}, /* V total size: 1184 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x04}, /* H window offset: 5 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x01}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06}, /* flip isp*/
-	{OV2722_8BIT, 0x3814, 0x31},
-	{OV2722_8BIT, 0x3815, 0x31},
-	{OV2722_8BIT, 0x3612, 0x4b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x02},
-	{OV2722_8BIT, 0x3a09, 0x67},
-	{OV2722_8BIT, 0x3a0a, 0x02},
-	{OV2722_8BIT, 0x3a0b, 0x00},
-	{OV2722_8BIT, 0x3a0d, 0x00},
-	{OV2722_8BIT, 0x3a0e, 0x00},
-	{OV2722_8BIT, 0x4520, 0x0a},
-	{OV2722_8BIT, 0x4837, 0x29},
-	{OV2722_8BIT, 0x3000, 0xff},
-	{OV2722_8BIT, 0x3001, 0xff},
-	{OV2722_8BIT, 0x3002, 0xf0},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0x53}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x63},
-	{OV2722_8BIT, 0x3634, 0x24},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1}, /* v_en, h_en, blc_en */
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xff},
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0}, /* AWB red */
-	{OV2722_8BIT, 0x5184, 0xb0}, /* AWB green */
-	{OV2722_8BIT, 0x5185, 0xb0}, /* AWB blue */
-	{OV2722_8BIT, 0x5180, 0x03}, /* AWB manual mode */
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x4800, 0x24}, /* clk lane gate enable */
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26},
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-
-	/* Added for power optimization */
-	{OV2722_8BIT, 0x3000, 0x00},
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3011, 0x22},
-	{OV2722_8BIT, 0x3a00, 0x58},
-	{OV2722_8BIT, 0x3503, 0x17},
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x46},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x10},
-	{OV2722_TOK_TERM, 0, 0},
-};
-#endif
-
 static const struct ov2722_reg ov2722_1632_1092_30fps[] = {
-	{OV2722_8BIT, 0x3021, 0x03}, /* For stand wait for
-				a whole frame complete.(vblank) */
+	/* For stand wait for a whole frame complete.(vblank) */
+	{OV2722_8BIT, 0x3021, 0x03},
 	{OV2722_8BIT, 0x3718, 0x10},
 	{OV2722_8BIT, 0x3702, 0x24},
 	{OV2722_8BIT, 0x373a, 0x60},
@@ -668,8 +335,8 @@ static const struct ov2722_reg ov2722_1632_1092_30fps[] = {
 };
 
 static const struct ov2722_reg ov2722_1452_1092_30fps[] = {
-	{OV2722_8BIT, 0x3021, 0x03}, /* For stand wait for
-				a whole frame complete.(vblank) */
+	/* For stand wait for a whole frame complete.(vblank) */
+	{OV2722_8BIT, 0x3021, 0x03},
 	{OV2722_8BIT, 0x3718, 0x10},
 	{OV2722_8BIT, 0x3702, 0x24},
 	{OV2722_8BIT, 0x373a, 0x60},
@@ -768,118 +435,9 @@ static const struct ov2722_reg ov2722_1452_1092_30fps[] = {
 	{OV2722_TOK_TERM, 0, 0}
 };
 
-#if 0
-static const struct ov2722_reg ov2722_1M3_30fps[] = {
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x24},
-	{OV2722_8BIT, 0x373a, 0x60},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x2e},
-	{OV2722_8BIT, 0x3705, 0x10},
-	{OV2722_8BIT, 0x3730, 0x30},
-	{OV2722_8BIT, 0x3704, 0x62},
-	{OV2722_8BIT, 0x3f06, 0x3a},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0xc4},
-	{OV2722_8BIT, 0x371e, 0x01},
-	{OV2722_8BIT, 0x371f, 0x0d},
-	{OV2722_8BIT, 0x3708, 0x61},
-	{OV2722_8BIT, 0x3709, 0x12},
-	{OV2722_8BIT, 0x3800, 0x01},
-	{OV2722_8BIT, 0x3801, 0x4a},	/* H crop start: 330 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0x03},	/* V crop start: 3 */
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0xe1},	/* H crop end:  1761 */
-	{OV2722_8BIT, 0x3806, 0x04},
-	{OV2722_8BIT, 0x3807, 0x47},	/* V crop end:  1095 */
-	{OV2722_8BIT, 0x3808, 0x05},
-	{OV2722_8BIT, 0x3809, 0x88},	/* H output size: 1416 */
-	{OV2722_8BIT, 0x380a, 0x04},
-	{OV2722_8BIT, 0x380b, 0x0a},	/* V output size: 1034 */
-
-	/* H blank timing */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00},	/* H total size: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa0},	/* V total size: 1184 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x05},	/* H window offset: 5 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x02},	/* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06},	/* flip isp */
-	{OV2722_8BIT, 0x3814, 0x11},
-	{OV2722_8BIT, 0x3815, 0x11},
-	{OV2722_8BIT, 0x3612, 0x0b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x01},
-	{OV2722_8BIT, 0x3a09, 0x50},
-	{OV2722_8BIT, 0x3a0a, 0x01},
-	{OV2722_8BIT, 0x3a0b, 0x18},
-	{OV2722_8BIT, 0x3a0d, 0x03},
-	{OV2722_8BIT, 0x3a0e, 0x03},
-	{OV2722_8BIT, 0x4520, 0x00},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3000, 0xff},
-	{OV2722_8BIT, 0x3001, 0xff},
-	{OV2722_8BIT, 0x3002, 0xf0},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0xd2},	/* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x23},
-	{OV2722_8BIT, 0x3634, 0x54},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1},	/* v_en, h_en, blc_en */
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xcf},
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0},	/* AWB red */
-	{OV2722_8BIT, 0x5184, 0xb0},	/* AWB green */
-	{OV2722_8BIT, 0x5185, 0xb0},	/* AWB blue */
-	{OV2722_8BIT, 0x5180, 0x03},	/* AWB manual mode */
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x4800, 0x24},	/* clk lane gate enable */
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26},
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-
-	/* Added for power optimization */
-	{OV2722_8BIT, 0x3000, 0x00},
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3503, 0x17},
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x46},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x10},
-	{OV2722_TOK_TERM, 0, 0},
-};
-#endif
-
 static const struct ov2722_reg ov2722_1080p_30fps[] = {
-	{OV2722_8BIT, 0x3021, 0x03}, /* For stand wait for a whole
-					frame complete.(vblank) */
+	/* For stand wait for a whole frame complete.(vblank) */
+	{OV2722_8BIT, 0x3021, 0x03},
 	{OV2722_8BIT, 0x3718, 0x10},
 	{OV2722_8BIT, 0x3702, 0x24},
 	{OV2722_8BIT, 0x373a, 0x60},
@@ -982,108 +540,6 @@ static const struct ov2722_reg ov2722_1080p_30fps[] = {
 	{OV2722_TOK_TERM, 0, 0}
 };
 
-#if 0 /* Currently unused */
-static const struct ov2722_reg ov2722_720p_30fps[] = {
-	{OV2722_8BIT, 0x3021, 0x03},
-	{OV2722_8BIT, 0x3718, 0x10},
-	{OV2722_8BIT, 0x3702, 0x24},
-	{OV2722_8BIT, 0x373a, 0x60},
-	{OV2722_8BIT, 0x3715, 0x01},
-	{OV2722_8BIT, 0x3703, 0x2e},
-	{OV2722_8BIT, 0x3705, 0x10},
-	{OV2722_8BIT, 0x3730, 0x30},
-	{OV2722_8BIT, 0x3704, 0x62},
-	{OV2722_8BIT, 0x3f06, 0x3a},
-	{OV2722_8BIT, 0x371c, 0x00},
-	{OV2722_8BIT, 0x371d, 0xc4},
-	{OV2722_8BIT, 0x371e, 0x01},
-	{OV2722_8BIT, 0x371f, 0x0d},
-	{OV2722_8BIT, 0x3708, 0x61},
-	{OV2722_8BIT, 0x3709, 0x12},
-	{OV2722_8BIT, 0x3800, 0x01},
-	{OV2722_8BIT, 0x3801, 0x40}, /* H crop start: 320 */
-	{OV2722_8BIT, 0x3802, 0x00},
-	{OV2722_8BIT, 0x3803, 0xb1}, /* V crop start: 177 */
-	{OV2722_8BIT, 0x3804, 0x06},
-	{OV2722_8BIT, 0x3805, 0x55}, /* H crop end: 1621 */
-	{OV2722_8BIT, 0x3806, 0x03},
-	{OV2722_8BIT, 0x3807, 0x95}, /* V crop end: 918 */
-	{OV2722_8BIT, 0x3808, 0x05},
-	{OV2722_8BIT, 0x3809, 0x10}, /* H output size: 0x0788==1928 */
-	{OV2722_8BIT, 0x380a, 0x02},
-	{OV2722_8BIT, 0x380b, 0xe0}, /* output size: 0x02DE==734 */
-	{OV2722_8BIT, 0x380c, 0x08},
-	{OV2722_8BIT, 0x380d, 0x00}, /* H timing: 2048 */
-	{OV2722_8BIT, 0x380e, 0x04},
-	{OV2722_8BIT, 0x380f, 0xa3}, /* V timing: 1187 */
-	{OV2722_8BIT, 0x3810, 0x00},
-	{OV2722_8BIT, 0x3811, 0x03}, /* H window offset: 3 */
-	{OV2722_8BIT, 0x3812, 0x00},
-	{OV2722_8BIT, 0x3813, 0x02}, /* V window offset: 2 */
-	{OV2722_8BIT, 0x3820, 0x80},
-	{OV2722_8BIT, 0x3821, 0x06}, /* mirror */
-	{OV2722_8BIT, 0x3814, 0x11},
-	{OV2722_8BIT, 0x3815, 0x11},
-	{OV2722_8BIT, 0x3612, 0x0b},
-	{OV2722_8BIT, 0x3618, 0x04},
-	{OV2722_8BIT, 0x3a08, 0x01},
-	{OV2722_8BIT, 0x3a09, 0x50},
-	{OV2722_8BIT, 0x3a0a, 0x01},
-	{OV2722_8BIT, 0x3a0b, 0x18},
-	{OV2722_8BIT, 0x3a0d, 0x03},
-	{OV2722_8BIT, 0x3a0e, 0x03},
-	{OV2722_8BIT, 0x4520, 0x00},
-	{OV2722_8BIT, 0x4837, 0x1b},
-	{OV2722_8BIT, 0x3600, 0x08},
-	{OV2722_8BIT, 0x3621, 0xc0},
-	{OV2722_8BIT, 0x3632, 0xd2}, /* added for power opt */
-	{OV2722_8BIT, 0x3633, 0x23},
-	{OV2722_8BIT, 0x3634, 0x54},
-	{OV2722_8BIT, 0x3f01, 0x0c},
-	{OV2722_8BIT, 0x5001, 0xc1},
-	{OV2722_8BIT, 0x3614, 0xf0},
-	{OV2722_8BIT, 0x3630, 0x2d},
-	{OV2722_8BIT, 0x370b, 0x62},
-	{OV2722_8BIT, 0x3706, 0x61},
-	{OV2722_8BIT, 0x4000, 0x02},
-	{OV2722_8BIT, 0x4002, 0xc5},
-	{OV2722_8BIT, 0x4005, 0x08},
-	{OV2722_8BIT, 0x404f, 0x84},
-	{OV2722_8BIT, 0x4051, 0x00},
-	{OV2722_8BIT, 0x5000, 0xcf}, /* manual 3a */
-	{OV2722_8BIT, 0x301d, 0xf0}, /* enable group hold */
-	{OV2722_8BIT, 0x3a18, 0x00},
-	{OV2722_8BIT, 0x3a19, 0x80},
-	{OV2722_8BIT, 0x4521, 0x00},
-	{OV2722_8BIT, 0x5183, 0xb0},
-	{OV2722_8BIT, 0x5184, 0xb0},
-	{OV2722_8BIT, 0x5185, 0xb0},
-	{OV2722_8BIT, 0x370c, 0x0c},
-	{OV2722_8BIT, 0x3035, 0x00},
-	{OV2722_8BIT, 0x3036, 0x26}, /* {0x3036, 0x2c}, //422.4 MHz */
-	{OV2722_8BIT, 0x3037, 0xa1},
-	{OV2722_8BIT, 0x303e, 0x19},
-	{OV2722_8BIT, 0x3038, 0x06},
-	{OV2722_8BIT, 0x3018, 0x04},
-	{OV2722_8BIT, 0x3000, 0x00}, /* added for power optimization */
-	{OV2722_8BIT, 0x3001, 0x00},
-	{OV2722_8BIT, 0x3002, 0x00},
-	{OV2722_8BIT, 0x3a0f, 0x40},
-	{OV2722_8BIT, 0x3a10, 0x38},
-	{OV2722_8BIT, 0x3a1b, 0x48},
-	{OV2722_8BIT, 0x3a1e, 0x30},
-	{OV2722_8BIT, 0x3a11, 0x90},
-	{OV2722_8BIT, 0x3a1f, 0x10},
-	{OV2722_8BIT, 0x3503, 0x17}, /* manual 3a */
-	{OV2722_8BIT, 0x3500, 0x00},
-	{OV2722_8BIT, 0x3501, 0x3F},
-	{OV2722_8BIT, 0x3502, 0x00},
-	{OV2722_8BIT, 0x3508, 0x00},
-	{OV2722_8BIT, 0x3509, 0x00},
-	{OV2722_TOK_TERM, 0, 0},
-};
-#endif
-
 static struct ov2722_resolution ov2722_res_preview[] = {
 	{
 		.desc = "ov2722_1632_1092_30fps",
@@ -1128,99 +584,6 @@ static struct ov2722_resolution ov2722_res_preview[] = {
 
 #define N_RES_PREVIEW (ARRAY_SIZE(ov2722_res_preview))
 
-/*
- * Disable non-preview configurations until the configuration selection is
- * improved.
- */
-#if 0
-struct ov2722_resolution ov2722_res_still[] = {
-	{
-		.desc = "ov2722_480P_30fps",
-		.width = 1632,
-		.height = 1092,
-		.fps = 30,
-		.pix_clk_freq = 85,
-		.used = 0,
-		.pixels_per_line = 2260,
-		.lines_per_frame = 1244,
-		.skip_frames = 3,
-		.regs = ov2722_1632_1092_30fps,
-		.mipi_freq = 422400,
-	},
-	{
-		.desc = "ov2722_1452_1092_30fps",
-		.width = 1452,
-		.height = 1092,
-		.fps = 30,
-		.pix_clk_freq = 85,
-		.used = 0,
-		.pixels_per_line = 2260,
-		.lines_per_frame = 1244,
-		.skip_frames = 3,
-		.regs = ov2722_1452_1092_30fps,
-		.mipi_freq = 422400,
-	},
-	{
-		.desc = "ov2722_1080P_30fps",
-		.width = 1932,
-		.height = 1092,
-		.pix_clk_freq = 69,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2068,
-		.lines_per_frame = 1114,
-		.skip_frames = 3,
-		.regs = ov2722_1080p_30fps,
-		.mipi_freq = 345600,
-	},
-};
-
-#define N_RES_STILL (ARRAY_SIZE(ov2722_res_still))
-
-struct ov2722_resolution ov2722_res_video[] = {
-	{
-		.desc = "ov2722_QVGA_30fps",
-		.width = 336,
-		.height = 256,
-		.fps = 30,
-		.pix_clk_freq = 73,
-		.used = 0,
-		.pixels_per_line = 2048,
-		.lines_per_frame = 1184,
-		.skip_frames = 3,
-		.regs = ov2722_QVGA_30fps,
-		.mipi_freq = 364800,
-	},
-	{
-		.desc = "ov2722_480P_30fps",
-		.width = 736,
-		.height = 496,
-		.fps = 30,
-		.pix_clk_freq = 73,
-		.used = 0,
-		.pixels_per_line = 2048,
-		.lines_per_frame = 1184,
-		.skip_frames = 3,
-		.regs = ov2722_480P_30fps,
-	},
-	{
-		.desc = "ov2722_1080P_30fps",
-		.width = 1932,
-		.height = 1092,
-		.pix_clk_freq = 69,
-		.fps = 30,
-		.used = 0,
-		.pixels_per_line = 2068,
-		.lines_per_frame = 1114,
-		.skip_frames = 3,
-		.regs = ov2722_1080p_30fps,
-		.mipi_freq = 345600,
-	},
-};
-
-#define N_RES_VIDEO (ARRAY_SIZE(ov2722_res_video))
-#endif
-
 static struct ov2722_resolution *ov2722_res = ov2722_res_preview;
 static unsigned long N_RES = N_RES_PREVIEW;
 #endif
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp.h b/drivers/staging/media/atomisp/include/linux/atomisp.h
index 3c8fa3f5808d0..7d7f57c2d67c8 100644
--- a/drivers/staging/media/atomisp/include/linux/atomisp.h
+++ b/drivers/staging/media/atomisp/include/linux/atomisp.h
@@ -22,7 +22,7 @@
 #define ATOMISP_HW_STEPPING_A0		0x00
 #define ATOMISP_HW_STEPPING_B0		0x10
 
-/*ISP binary running mode*/
+/* ISP binary running mode */
 #define CI_MODE_PREVIEW		0x8000
 #define CI_MODE_VIDEO		0x4000
 #define CI_MODE_STILL_CAPTURE	0x2000
@@ -59,9 +59,9 @@
 
 /* Configuration used by Bayer noise reduction and YCC noise reduction */
 struct atomisp_nr_config {
-	/* [gain] Strength of noise reduction for Bayer NR (Used by Bayer NR) */
+	/* [gain] Strength of noise reduction for Bayer NR */
 	unsigned int bnr_gain;
-	/* [gain] Strength of noise reduction for YCC NR (Used by YCC NR) */
+	/* [gain] Strength of noise reduction for YCC NR */
 	unsigned int ynr_gain;
 	/* [intensity] Sensitivity of Edge (Used by Bayer NR) */
 	unsigned int direction;
@@ -78,7 +78,8 @@ struct atomisp_tnr_config {
 	unsigned int threshold_uv;/* [intensity] Motion sensitivity for U/V */
 };
 
-/* Histogram. This contains num_elements values of type unsigned int.
+/*
+ * Contains num_elements values of type unsigned int.
  * The data pointer is a DDR pointer (virtual address).
  */
 struct atomisp_histogram {
@@ -94,7 +95,7 @@ enum atomisp_ob_mode {
 
 /* Optical black level configuration */
 struct atomisp_ob_config {
-	/* Obtical black level mode (Fixed / Raster) */
+	/* Optical black level mode (Fixed / Raster) */
 	enum atomisp_ob_mode mode;
 	/* [intensity] optical black level for GR (relevant for fixed mode) */
 	unsigned int level_gr;
@@ -146,8 +147,7 @@ struct atomisp_3a_config {
 	unsigned int ae_y_coef_r;	/* [gain] Weight of R for Y */
 	unsigned int ae_y_coef_g;	/* [gain] Weight of G for Y */
 	unsigned int ae_y_coef_b;	/* [gain] Weight of B for Y */
-	unsigned int awb_lg_high_raw;	/* [intensity]
-					   AWB level gate high for raw */
+	unsigned int awb_lg_high_raw;	/* [intensity] AWB level gate high for raw */
 	unsigned int awb_lg_low;	/* [intensity] AWB level gate low */
 	unsigned int awb_lg_high;	/* [intensity] AWB level gate high */
 	int af_fir1_coef[7];	/* [factor] AF FIR coefficients of fir1 */
@@ -188,14 +188,15 @@ struct atomisp_dis_vector {
 	int y;
 };
 
-/* DVS 2.0 Coefficient types. This structure contains 4 pointers to
- *  arrays that contain the coefficients for each type.
+/*
+ * DVS 2.0 Coefficient types. This structure contains 4 pointers to
+ * arrays that contain the coefficients for each type.
  */
 struct atomisp_dvs2_coef_types {
-	short __user *odd_real; /** real part of the odd coefficients*/
-	short __user *odd_imag; /** imaginary part of the odd coefficients*/
-	short __user *even_real;/** real part of the even coefficients*/
-	short __user *even_imag;/** imaginary part of the even coefficients*/
+	short __user *odd_real; /* Real part of the odd coefficients */
+	short __user *odd_imag; /* Imaginary part of the odd coefficients */
+	short __user *even_real;/* Real part of the even coefficients */
+	short __user *even_imag;/* Imaginary part of the even coefficients */
 };
 
 /*
@@ -203,10 +204,10 @@ struct atomisp_dvs2_coef_types {
  * arrays that contain the statistics for each type.
  */
 struct atomisp_dvs2_stat_types {
-	int __user *odd_real; /** real part of the odd statistics*/
-	int __user *odd_imag; /** imaginary part of the odd statistics*/
-	int __user *even_real;/** real part of the even statistics*/
-	int __user *even_imag;/** imaginary part of the even statistics*/
+	int __user *odd_real; /* Real part of the odd statistics */
+	int __user *odd_imag; /* Imaginary part of the odd statistics */
+	int __user *even_real;/* Real part of the even statistics */
+	int __user *even_imag;/* Imaginary part of the even statistics */
 };
 
 struct atomisp_dis_coefficients {
@@ -234,12 +235,12 @@ struct atomisp_3a_rgby_output {
 };
 
 /*
- * Because we have 2 pipes at max to output metadata, therefore driver will use
- * ATOMISP_MAIN_METADATA to specify the metadata from the pipe which keeps
- * streaming always and use ATOMISP_SEC_METADATA to specify the metadata from
- * the pipe which is streaming by request like capture pipe of ZSL or SDV mode
- * as secondary metadata. And for the use case which has only one pipe
- * streaming like online capture, ATOMISP_MAIN_METADATA will be used.
+ * As the driver can output metadata on two pipes at max,
+ * ATOMISP_MAIN_METADATA is used for the pipe that streams continuously.
+ * ATOMISP_SEC_METADATA is used for the pipe that streams on demand, e.g.,
+ * the capture pipe in ZSL or SDV modes.
+ * In use cases with a single streaming pipe (like online capture),
+ * only ATOMISP_MAIN_METADATA is used.
  */
 enum atomisp_metadata_type {
 	ATOMISP_MAIN_METADATA = 0,
@@ -257,7 +258,7 @@ struct atomisp_3a_statistics {
 	struct atomisp_3a_output __user *data;
 	struct atomisp_3a_rgby_output __user *rgby_data;
 	u32 exp_id; /* exposure ID */
-	u32 isp_config_id; /* isp config ID */
+	u32 isp_config_id;
 };
 
 /* White Balance (Gain Adjust) */
@@ -272,8 +273,8 @@ struct atomisp_wb_config {
 /* Color Space Conversion settings */
 struct atomisp_cc_config {
 	unsigned int fraction_bits;
-	int matrix[3 * 3];	/* RGB2YUV Color matrix, signed
-				   <13-fraction_bits>.<fraction_bits> */
+	/* RGB2YUV Color matrix, signed <13-fraction_bits>.<fraction_bits> */
+	int matrix[3 * 3];
 };
 
 /* De pixel noise configuration */
@@ -291,13 +292,15 @@ struct atomisp_ce_config {
 
 /* Defect pixel correction configuration */
 struct atomisp_dp_config {
-	/* [intensity] The threshold of defect Pixel Correction, representing
+	/*
+	 * [intensity] The threshold of defect Pixel Correction, representing
 	 * the permissible difference of intensity between one pixel and its
 	 * surrounding pixels. Smaller values result in more frequent pixel
 	 * corrections. u0_16
 	 */
 	unsigned int threshold;
-	/* [gain] The sensitivity of mis-correction. ISP will miss a lot of
+	/*
+	 * [gain] The sensitivity of mis-correction. ISP will miss a lot of
 	 * defects if the value is set too large. u8_8
 	 */
 	unsigned int gain;
@@ -312,7 +315,7 @@ struct atomisp_xnr_config {
 	__u16 threshold;
 };
 
-/* metadata config */
+/* Metadata config */
 struct atomisp_metadata_config {
 	u32 metadata_height;
 	u32 metadata_stride;
@@ -322,31 +325,30 @@ struct atomisp_metadata_config {
  * Generic resolution structure.
  */
 struct atomisp_resolution {
-	u32 width;  /** Width */
-	u32 height; /** Height */
+	u32 width;
+	u32 height;
 };
 
 /*
- * This specifies the coordinates (x,y)
+ * Specifies the zoom point coordinates (x,y)
  */
 struct atomisp_zoom_point {
-	s32 x; /** x coordinate */
-	s32 y; /** y coordinate */
+	s32 x;
+	s32 y;
 };
 
 /*
- * This specifies the region
+ * Specifies the zoom region
  */
 struct atomisp_zoom_region {
-	struct atomisp_zoom_point
-		origin; /* Starting point coordinates for the region */
+	struct atomisp_zoom_point origin; /* Starting point coordinates for the region */
 	struct atomisp_resolution resolution; /* Region resolution */
 };
 
 struct atomisp_dz_config {
-	u32 dx; /** Horizontal zoom factor */
-	u32 dy; /** Vertical zoom factor */
-	struct atomisp_zoom_region zoom_region; /** region for zoom */
+	u32 dx; /* Horizontal zoom factor */
+	u32 dy; /* Vertical zoom factor */
+	struct atomisp_zoom_region zoom_region;
 };
 
 struct atomisp_parm {
@@ -367,8 +369,8 @@ struct atomisp_parm {
 };
 
 struct dvs2_bq_resolution {
-	int width_bq;         /* width [BQ] */
-	int height_bq;        /* height [BQ] */
+	int width_bq;
+	int height_bq;
 };
 
 struct atomisp_dvs2_bq_resolutions {
@@ -378,9 +380,9 @@ struct atomisp_dvs2_bq_resolutions {
 	struct dvs2_bq_resolution output_bq;
 	/* GDC effective envelope size [BQ] */
 	struct dvs2_bq_resolution envelope_bq;
-	/* isp pipe filter size [BQ] */
+	/* ISP pipe filter size [BQ] */
 	struct dvs2_bq_resolution ispfilter_bq;
-	/* GDC shit size [BQ] */
+	/* GDC shift size [BQ] */
 	struct dvs2_bq_resolution gdc_shift_bq;
 };
 
@@ -411,8 +413,8 @@ struct atomisp_parameters {
 	struct atomisp_cnr_config  *cnr_config; /* Chroma Noise Reduction */
 	struct atomisp_macc_config *macc_config;  /* MACC */
 	struct atomisp_ctc_config  *ctc_config; /* Chroma Tone Control */
-	struct atomisp_aa_config   *aa_config;  /* Anti-Aliasing */
-	struct atomisp_aa_config   *baa_config;  /* Anti-Aliasing */
+	struct atomisp_aa_config   *aa_config;	/* Anti-Aliasing */
+	struct atomisp_aa_config   *baa_config;	/* Anti-Aliasing */
 	struct atomisp_ce_config   *ce_config;
 	struct atomisp_dvs_6axis_config *dvs_6axis_config;
 	struct atomisp_ob_config   *ob_config;  /* Objective Black config */
@@ -425,10 +427,8 @@ struct atomisp_parameters {
 	struct atomisp_3a_config   *a3a_config; /* 3A Statistics config */
 	struct atomisp_xnr_config  *xnr_config; /* eXtra Noise Reduction */
 	struct atomisp_dz_config   *dz_config;  /* Digital Zoom */
-	struct atomisp_cc_config *yuv2rgb_cc_config; /* Color
-							Correction config */
-	struct atomisp_cc_config *rgb2yuv_cc_config; /* Color
-							Correction config */
+	struct atomisp_cc_config *yuv2rgb_cc_config; /* Color Correction config */
+	struct atomisp_cc_config *rgb2yuv_cc_config; /* Color Correction config */
 	struct atomisp_macc_table  *macc_table;
 	struct atomisp_gamma_table *gamma_table;
 	struct atomisp_ctc_table   *ctc_table;
@@ -466,8 +466,8 @@ struct atomisp_parameters {
 	void	*res_mgr_2500_config;
 
 	/*
-	 * Output frame pointer the config is to be applied to (optional),
-	 * set to NULL to make this config is applied as global.
+	 * Output frame pointer to which the config will be applied to (optional),
+	 * Set to NULL to make this config be applied as global.
 	 */
 	void	*output_frame;
 	/*
@@ -490,9 +490,9 @@ struct atomisp_gamma_table {
 	unsigned short data[ATOMISP_GAMMA_TABLE_SIZE];
 };
 
-/* Morphing table for advanced ISP.
- * Each line of width elements takes up COORD_TABLE_EXT_WIDTH elements
- * in memory.
+/*
+ * Morphing table for advanced ISP.
+ * Each line of width elements takes up COORD_TABLE_EXT_WIDTH elements in memory.
  */
 #define ATOMISP_MORPH_TABLE_NUM_PLANES  6
 struct atomisp_morph_table {
@@ -519,7 +519,7 @@ struct atomisp_shading_table {
 	__u16 *data[ATOMISP_NUM_SC_COLORS];
 };
 
-/* parameter for MACC */
+/* Parameter for MACC */
 #define ATOMISP_NUM_MACC_AXES           16
 struct atomisp_macc_table {
 	short data[4 * ATOMISP_NUM_MACC_AXES];
@@ -530,7 +530,7 @@ struct atomisp_macc_config {
 	struct atomisp_macc_table table;
 };
 
-/* Parameter for ctc parameter control */
+/* CTC parameter control */
 #define ATOMISP_CTC_TABLE_SIZE          1024
 struct atomisp_ctc_table {
 	unsigned short data[ATOMISP_CTC_TABLE_SIZE];
@@ -538,9 +538,10 @@ struct atomisp_ctc_table {
 
 /* Parameter for overlay image loading */
 struct atomisp_overlay {
-	/* the frame containing the overlay data The overlay frame width should
-	 * be the multiples of 2*ISP_VEC_NELEMS. The overlay frame height
-	 * should be the multiples of 2.
+	/*
+	 * The frame containing the overlay data. The overlay frame width should
+	 * be a multiple of 2 * ISP_VEC_NELEMS. The overlay frame height
+	 * should be a multiple of 2.
 	 */
 	struct v4l2_framebuffer *frame;
 	/* Y value of overlay background */
@@ -549,23 +550,27 @@ struct atomisp_overlay {
 	char bg_u;
 	/* V value of overlay background */
 	char bg_v;
-	/* the blending percent of input data for Y subpixels */
+	/* The blending percentage of input data for Y subpixels */
 	unsigned char blend_input_perc_y;
-	/* the blending percent of input data for U subpixels */
+	/* The blending percentage of input data for U subpixels */
 	unsigned char blend_input_perc_u;
-	/* the blending percent of input data for V subpixels */
+	/* The blending percentage of input data for V subpixels */
 	unsigned char blend_input_perc_v;
-	/* the blending percent of overlay data for Y subpixels */
+	/* The blending percentage of overlay data for Y subpixels */
 	unsigned char blend_overlay_perc_y;
-	/* the blending percent of overlay data for U subpixels */
+	/* The blending percentage of overlay data for U subpixels */
 	unsigned char blend_overlay_perc_u;
-	/* the blending percent of overlay data for V subpixels */
+	/* The blending percentage of overlay data for V subpixels */
 	unsigned char blend_overlay_perc_v;
-	/* the overlay start x pixel position on output frame It should be the
-	   multiples of 2*ISP_VEC_NELEMS. */
+	/*
+	 * The overlay "start x" pixel position on the output frame. It should be a
+	 * multiple of 2 * ISP_VEC_NELEMS.
+	 */
 	unsigned int overlay_start_x;
-	/* the overlay start y pixel position on output frame It should be the
-	   multiples of 2. */
+	/*
+	 * The overlay "start y" pixel position on the output frame. It should be a
+	 * multiple of 2.
+	 */
 	unsigned int overlay_start_y;
 };
 
@@ -659,7 +664,7 @@ enum atomisp_burst_capture_options {
 #define EXT_ISP_SHOT_MODE_ANIMATED_PHOTO	10
 #define EXT_ISP_SHOT_MODE_SPORTS	11
 
-/*Private IOCTLs for ISP */
+/* Private IOCTLs for ISP */
 #define ATOMISP_IOC_G_XNR \
 	_IOR('v', BASE_VIDIOC_PRIVATE + 0, int)
 #define ATOMISP_IOC_S_XNR \
@@ -684,7 +689,8 @@ enum atomisp_burst_capture_options {
 	_IOR('v', BASE_VIDIOC_PRIVATE + 5, struct atomisp_ee_config)
 #define ATOMISP_IOC_S_EE \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 5, struct atomisp_ee_config)
-/* Digital Image Stabilization:
+/*
+ * Digital Image Stabilization:
  * 1. get dis statistics: reads DIS statistics from ISP (every frame)
  * 2. set dis coefficients: set DIS filter coefficients (one time)
  * 3. set dis motion vector: set motion vector (result of DIS, every frame)
@@ -716,54 +722,54 @@ enum atomisp_burst_capture_options {
 #define ATOMISP_IOC_S_ISP_GDC_TAB \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 10, struct atomisp_morph_table)
 
-/* macc parameter control*/
+/* MACC parameter control */
 #define ATOMISP_IOC_G_ISP_MACC \
 	_IOR('v', BASE_VIDIOC_PRIVATE + 12, struct atomisp_macc_config)
 #define ATOMISP_IOC_S_ISP_MACC \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 12, struct atomisp_macc_config)
 
-/* Defect pixel detection & Correction */
+/* Defect pixel detection & correction */
 #define ATOMISP_IOC_G_ISP_BAD_PIXEL_DETECTION \
 	_IOR('v', BASE_VIDIOC_PRIVATE + 13, struct atomisp_dp_config)
 #define ATOMISP_IOC_S_ISP_BAD_PIXEL_DETECTION \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 13, struct atomisp_dp_config)
 
-/* False Color Correction */
+/* False color correction */
 #define ATOMISP_IOC_G_ISP_FALSE_COLOR_CORRECTION \
 	_IOR('v', BASE_VIDIOC_PRIVATE + 14, struct atomisp_de_config)
 #define ATOMISP_IOC_S_ISP_FALSE_COLOR_CORRECTION \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 14, struct atomisp_de_config)
 
-/* ctc parameter control */
+/* CTC parameter control */
 #define ATOMISP_IOC_G_ISP_CTC \
 	_IOR('v', BASE_VIDIOC_PRIVATE + 15, struct atomisp_ctc_table)
 #define ATOMISP_IOC_S_ISP_CTC \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 15, struct atomisp_ctc_table)
 
-/* white balance Correction */
+/* White balance correction */
 #define ATOMISP_IOC_G_ISP_WHITE_BALANCE \
 	_IOR('v', BASE_VIDIOC_PRIVATE + 16, struct atomisp_wb_config)
 #define ATOMISP_IOC_S_ISP_WHITE_BALANCE \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 16, struct atomisp_wb_config)
 
-/* fpn table loading */
+/* FPN table loading */
 #define ATOMISP_IOC_S_ISP_FPN_TABLE \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 17, struct v4l2_framebuffer)
 
-/* overlay image loading */
+/* Overlay image loading */
 #define ATOMISP_IOC_G_ISP_OVERLAY \
 	_IOWR('v', BASE_VIDIOC_PRIVATE + 18, struct atomisp_overlay)
 #define ATOMISP_IOC_S_ISP_OVERLAY \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 18, struct atomisp_overlay)
 
-/* bcd driver bridge */
+/* BCD driver bridge */
 #define ATOMISP_IOC_CAMERA_BRIDGE \
 	_IOWR('v', BASE_VIDIOC_PRIVATE + 19, struct atomisp_bc_video_package)
 
 #define ATOMISP_IOC_S_EXPOSURE \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 21, struct atomisp_exposure)
 
-/* white balance Correction */
+/* White balance correction */
 #define ATOMISP_IOC_G_3A_CONFIG \
 	_IOR('v', BASE_VIDIOC_PRIVATE + 23, struct atomisp_3a_config)
 #define ATOMISP_IOC_S_3A_CONFIG \
@@ -773,7 +779,7 @@ enum atomisp_burst_capture_options {
 #define ATOMISP_IOC_S_ISP_SHD_TAB \
 	_IOWR('v', BASE_VIDIOC_PRIVATE + 27, struct atomisp_shading_table)
 
-/* Gamma Correction */
+/* Gamma correction */
 #define ATOMISP_IOC_G_ISP_GAMMA_CORRECTION \
 	_IOR('v', BASE_VIDIOC_PRIVATE + 28, struct atomisp_gc_config)
 
@@ -804,7 +810,7 @@ enum atomisp_burst_capture_options {
 #define ATOMISP_IOC_S_ARRAY_RESOLUTION \
 	_IOW('v', BASE_VIDIOC_PRIVATE + 45, struct atomisp_resolution)
 
-/* for depth mode sensor frame sync compensation */
+/* For depth mode sensor frame sync compensation */
 #define ATOMISP_IOC_G_DEPTH_SYNC_COMP \
 	_IOR('v', BASE_VIDIOC_PRIVATE + 46, unsigned int)
 
@@ -822,7 +828,7 @@ enum atomisp_burst_capture_options {
  *	_IOW('v', BASE_VIDIOC_PRIVATE + 56, struct atomisp_sensor_regs)
  */
 
-/*  ISP Private control IDs */
+/* ISP Private control IDs */
 #define V4L2_CID_ATOMISP_BAD_PIXEL_DETECTION \
 	(V4L2_CID_PRIVATE_BASE + 0)
 #define V4L2_CID_ATOMISP_POSTPROCESS_GDC_CAC \
@@ -836,8 +842,10 @@ enum atomisp_burst_capture_options {
 #define V4L2_CID_ATOMISP_LOW_LIGHT \
 	(V4L2_CID_PRIVATE_BASE + 5)
 
-/* Camera class:
- * Exposure, Flash and privacy (indicator) light controls, to be upstreamed */
+/*
+ * Camera class:
+ * Exposure, Flash and privacy (indicator) light controls, to be upstreamed
+ */
 #define V4L2_CID_CAMERA_LASTP1             (V4L2_CID_CAMERA_CLASS_BASE + 1024)
 
 #define V4L2_CID_RUN_MODE			(V4L2_CID_CAMERA_LASTP1 + 20)
@@ -879,7 +887,7 @@ enum atomisp_burst_capture_options {
 #define V4L2_EVENT_ATOMISP_ACC_COMPLETE     (V4L2_EVENT_PRIVATE_START + 4)
 #define V4L2_EVENT_ATOMISP_PAUSE_BUFFER	    (V4L2_EVENT_PRIVATE_START + 5)
 #define V4L2_EVENT_ATOMISP_CSS_RESET	    (V4L2_EVENT_PRIVATE_START + 6)
-/* Nonstandard color effects for V4L2_CID_COLORFX */
+/* Non-standard color effects for V4L2_CID_COLORFX */
 enum {
 	V4L2_COLORFX_SKIN_WHITEN_LOW = 1001,
 	V4L2_COLORFX_SKIN_WHITEN_HIGH = 1002,
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
index 426c5ee4ec188..74092af1c6590 100644
--- a/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
+++ b/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h
@@ -15,8 +15,7 @@ int atomisp_gmin_remove_subdev(struct v4l2_subdev *sd);
 int gmin_get_var_int(struct device *dev, bool is_gmin,
 		     const char *var, int def);
 struct camera_sensor_platform_data *
-gmin_camera_platform_data(
-    struct v4l2_subdev *subdev,
-    enum atomisp_input_format csi_format,
-    enum atomisp_bayer_order csi_bayer);
+	gmin_camera_platform_data(struct v4l2_subdev *subdev,
+				  enum atomisp_input_format csi_format,
+				  enum atomisp_bayer_order csi_bayer);
 #endif
diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
index 6146555fe9cfd..0d6f2a4039f1c 100644
--- a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
+++ b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h
@@ -57,7 +57,8 @@ enum atomisp_input_format {
 	ATOMISP_INPUT_FORMAT_RAW_16,   /* RAW data, 16 bits per pixel */
 	ATOMISP_INPUT_FORMAT_BINARY_8, /* Binary byte stream. */
 
-	/* CSI2-MIPI specific format: Generic short packet data. It is used to
+	/*
+	 * CSI2-MIPI specific format: Generic short packet data. It is used to
 	 * keep the timing information for the opening/closing of shutters,
 	 * triggering of flashes and etc.
 	 */
@@ -70,18 +71,18 @@ enum atomisp_input_format {
 	ATOMISP_INPUT_FORMAT_GENERIC_SHORT7,  /* Generic Short Packet Code 7 */
 	ATOMISP_INPUT_FORMAT_GENERIC_SHORT8,  /* Generic Short Packet Code 8 */
 
-	/* CSI2-MIPI specific format: YUV data.
-	 */
-	ATOMISP_INPUT_FORMAT_YUV420_8_SHIFT,  /* YUV420 8-bit (Chroma Shifted
-						 Pixel Sampling) */
-	ATOMISP_INPUT_FORMAT_YUV420_10_SHIFT, /* YUV420 8-bit (Chroma Shifted
-						 Pixel Sampling) */
+	/* YUV data */
+	/* YUV420 8-bit (Chroma Shifted Pixel Sampling) */
+	ATOMISP_INPUT_FORMAT_YUV420_8_SHIFT,
+	/* YUV420 10-bit (Chroma Shifted Pixel Sampling) */
+	ATOMISP_INPUT_FORMAT_YUV420_10_SHIFT,
 
-	/* CSI2-MIPI specific format: Generic long packet data
-	 */
-	ATOMISP_INPUT_FORMAT_EMBEDDED, /* Embedded 8-bit non Image Data */
+	/* CSI2-MIPI specific format: Generic long packet data */
+	/* Embedded 8-bit non Image Data */
+	ATOMISP_INPUT_FORMAT_EMBEDDED,
 
-	/* CSI2-MIPI specific format: User defined byte-based data. For example,
+	/*
+	 * User defined byte-based data. For example,
 	 * the data transmitter (e.g. the SoC sensor) can keep the JPEG data as
 	 * the User Defined Data Type 4 and the MPEG data as the
 	 * User Defined Data Type 7.
@@ -105,9 +106,9 @@ struct intel_v4l2_subdev_table {
 };
 
 /*
- *  Sensor of external ISP can send multiple streams with different mipi data
+ * Sensor of external ISP can send multiple streams with different MIPI data
  * type in the same virtual channel. This information needs to come from the
- * sensor or external ISP
+ * sensor or external ISP.
  */
 struct atomisp_isys_config_info {
 	u8 input_format;
@@ -118,16 +119,17 @@ struct atomisp_isys_config_info {
 struct atomisp_input_stream_info {
 	enum atomisp_input_stream_id stream;
 	u8 enable;
-	/* Sensor driver fills ch_id with the id
-	   of the virtual channel. */
+	/* Sensor driver fills ch_id with the id of the virtual channel. */
 	u8 ch_id;
-	/* Tells how many streams in this virtual channel. If 0 ignore rest
-	 * and the input format will be from mipi_info */
+	/*
+	 * Tells the number of streams in this virtual channel. If 0, ignore the
+	 * rest and the input format will be from mipi_info.
+	 */
 	u8 isys_configs;
 	/*
-	 * if more isys_configs is more than 0, sensor needs to configure the
-	 * input format differently. width and height can be 0. If width and
-	 * height is not zero, then the corresponding data needs to be set
+	 * If isys_configs is greater than 0, sensor needs to configure the
+	 * input format differently. Width and height can be 0. If width and
+	 * height are not zero, then the corresponding data needs to be set.
 	 */
 	struct atomisp_isys_config_info isys_info[MAX_STREAMS_PER_CHANNEL];
 };
diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
index fec369575d883..6cd500d9fd26d 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_cmd.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.c
@@ -811,7 +811,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error,
 			/* New global dvs 6axis config should be blocked
 			 * here if there's a buffer with per-frame parameters
 			 * pending in CSS frame buffer queue.
-			 * This is to aviod zooming vibration since global
+			 * This is to avoid zooming vibration since global
 			 * parameters take effect immediately while
 			 * per-frame parameters are taken after previous
 			 * buffers in CSS got processed.
@@ -974,7 +974,7 @@ irqreturn_t atomisp_isr_thread(int irq, void *isp_ptr)
 	 * to a FIFO, then process the event in the FIFO.
 	 * This will not have issue in single stream mode, but it do have some
 	 * issue in multiple stream case. The issue is that
-	 * ia_css_pipe_dequeue_buffer() will not return the corrent buffer in
+	 * ia_css_pipe_dequeue_buffer() will not return the correct buffer in
 	 * a specific pipe.
 	 *
 	 * This is due to ia_css_pipe_dequeue_buffer() does not take the
@@ -1380,8 +1380,10 @@ static void atomisp_update_grid_info(struct atomisp_sub_device *asd,
 	if (atomisp_css_get_grid_info(asd, pipe_id))
 		return;
 
-	/* We must free all buffers because they no longer match
-	   the grid size. */
+	/*
+	 * We must free all buffers because they no longer match
+	 * the grid size.
+	 */
 	atomisp_css_free_stat_buffers(asd);
 
 	err = atomisp_alloc_css_stat_bufs(asd, ATOMISP_INPUT_STREAM_GENERAL);
@@ -1393,8 +1395,10 @@ static void atomisp_update_grid_info(struct atomisp_sub_device *asd,
 	if (atomisp_alloc_3a_output_buf(asd)) {
 		/* Failure for 3A buffers does not influence DIS buffers */
 		if (asd->params.s3a_output_bytes != 0) {
-			/* For SOC sensor happens s3a_output_bytes == 0,
-			 * using if condition to exclude false error log */
+			/*
+			 * For SOC sensor happens s3a_output_bytes == 0,
+			 * using if condition to exclude false error log
+			 */
 			dev_err(isp->dev, "Failed to allocate memory for 3A statistics\n");
 		}
 		goto err;
@@ -1575,7 +1579,7 @@ int atomisp_set_dis_vector(struct atomisp_sub_device *asd,
 }
 
 /*
- * Function to set/get image stablization statistics
+ * Function to set/get image stabilization statistics
  */
 int atomisp_get_dis_stat(struct atomisp_sub_device *asd,
 			 struct atomisp_dis_statistics *stats)
@@ -1686,9 +1690,11 @@ int atomisp_3a_stat(struct atomisp_sub_device *asd, int flag,
 		return -EINVAL;
 
 	if (atomisp_compare_grid(asd, &config->grid_info) != 0) {
-		/* If the grid info in the argument differs from the current
-		   grid info, we tell the caller to reset the grid size and
-		   try again. */
+		/*
+		 * If the grid info in the argument differs from the current
+		 * grid info, we tell the caller to reset the grid size and
+		 * try again.
+		 */
 		return -EAGAIN;
 	}
 
@@ -1762,15 +1768,13 @@ int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd,
 		return -EINVAL;
 	}
 
-	if (dz_config->zoom_region.resolution.width
-	    == asd->sensor_array_res.width
-	    || dz_config->zoom_region.resolution.height
-	    == asd->sensor_array_res.height) {
+	if (dz_config->zoom_region.width == asd->sensor_array_res.width ||
+	    dz_config->zoom_region.height == asd->sensor_array_res.height) {
 		/*no need crop region*/
-		dz_config->zoom_region.origin.x = 0;
-		dz_config->zoom_region.origin.y = 0;
-		dz_config->zoom_region.resolution.width = eff_res.width;
-		dz_config->zoom_region.resolution.height = eff_res.height;
+		dz_config->zoom_region.left = 0;
+		dz_config->zoom_region.top = 0;
+		dz_config->zoom_region.width = eff_res.width;
+		dz_config->zoom_region.height = eff_res.height;
 		return 0;
 	}
 
@@ -1781,18 +1785,14 @@ int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd,
 	 */
 
 	if (!IS_ISP2401) {
-		dz_config->zoom_region.origin.x = dz_config->zoom_region.origin.x
-						  * eff_res.width
-						  / asd->sensor_array_res.width;
-		dz_config->zoom_region.origin.y = dz_config->zoom_region.origin.y
-						  * eff_res.height
-						  / asd->sensor_array_res.height;
-		dz_config->zoom_region.resolution.width = dz_config->zoom_region.resolution.width
-							  * eff_res.width
-							  / asd->sensor_array_res.width;
-		dz_config->zoom_region.resolution.height = dz_config->zoom_region.resolution.height
-							  * eff_res.height
-							  / asd->sensor_array_res.height;
+		dz_config->zoom_region.left = dz_config->zoom_region.left *
+					      eff_res.width / asd->sensor_array_res.width;
+		dz_config->zoom_region.top = dz_config->zoom_region.top *
+					     eff_res.height / asd->sensor_array_res.height;
+		dz_config->zoom_region.width = dz_config->zoom_region.width *
+					       eff_res.width / asd->sensor_array_res.width;
+		dz_config->zoom_region.height = dz_config->zoom_region.height *
+						eff_res.height / asd->sensor_array_res.height;
 		/*
 		 * Set same ratio of crop region resolution and current pipe output
 		 * resolution
@@ -1819,62 +1819,62 @@ int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd,
 				   - asd->sensor_array_res.width
 				   * out_res.height / out_res.width;
 			h_offset = h_offset / 2;
-			if (dz_config->zoom_region.origin.y < h_offset)
-				dz_config->zoom_region.origin.y = 0;
+			if (dz_config->zoom_region.top < h_offset)
+				dz_config->zoom_region.top = 0;
 			else
-				dz_config->zoom_region.origin.y = dz_config->zoom_region.origin.y - h_offset;
+				dz_config->zoom_region.top =
+					dz_config->zoom_region.top - h_offset;
 			w_offset = 0;
 		} else {
 			w_offset = asd->sensor_array_res.width
 				   - asd->sensor_array_res.height
 				   * out_res.width / out_res.height;
 			w_offset = w_offset / 2;
-			if (dz_config->zoom_region.origin.x < w_offset)
-				dz_config->zoom_region.origin.x = 0;
+			if (dz_config->zoom_region.left < w_offset)
+				dz_config->zoom_region.left = 0;
 			else
-				dz_config->zoom_region.origin.x = dz_config->zoom_region.origin.x - w_offset;
+				dz_config->zoom_region.left =
+					dz_config->zoom_region.left - w_offset;
 			h_offset = 0;
 		}
-		dz_config->zoom_region.origin.x = dz_config->zoom_region.origin.x
-						  * eff_res.width
-						  / (asd->sensor_array_res.width - 2 * w_offset);
-		dz_config->zoom_region.origin.y = dz_config->zoom_region.origin.y
-						  * eff_res.height
-						  / (asd->sensor_array_res.height - 2 * h_offset);
-		dz_config->zoom_region.resolution.width = dz_config->zoom_region.resolution.width
-						  * eff_res.width
-						  / (asd->sensor_array_res.width - 2 * w_offset);
-		dz_config->zoom_region.resolution.height = dz_config->zoom_region.resolution.height
-						  * eff_res.height
-						  / (asd->sensor_array_res.height - 2 * h_offset);
-	}
-
-	if (out_res.width * dz_config->zoom_region.resolution.height
-	    > dz_config->zoom_region.resolution.width * out_res.height) {
-		dz_config->zoom_region.resolution.height =
-		    dz_config->zoom_region.resolution.width
-		    * out_res.height / out_res.width;
+		dz_config->zoom_region.left = dz_config->zoom_region.left *
+			eff_res.width /
+			(asd->sensor_array_res.width - 2 * w_offset);
+		dz_config->zoom_region.top = dz_config->zoom_region.top *
+			eff_res.height /
+			(asd->sensor_array_res.height - 2 * h_offset);
+		dz_config->zoom_region.width = dz_config->zoom_region.width *
+			eff_res.width /
+			(asd->sensor_array_res.width - 2 * w_offset);
+		dz_config->zoom_region.height = dz_config->zoom_region.height *
+			eff_res.height /
+			(asd->sensor_array_res.height - 2 * h_offset);
+	}
+
+	if (out_res.width * dz_config->zoom_region.height >
+	    dz_config->zoom_region.width * out_res.height) {
+		dz_config->zoom_region.height = dz_config->zoom_region.width *
+						out_res.height / out_res.width;
 	} else {
-		dz_config->zoom_region.resolution.width =
-		    dz_config->zoom_region.resolution.height
-		    * out_res.width / out_res.height;
+		dz_config->zoom_region.width = dz_config->zoom_region.height *
+					       out_res.width / out_res.height;
 	}
 	dev_dbg(asd->isp->dev,
 		"%s crop region:(%d,%d),(%d,%d) eff_res(%d, %d) array_size(%d,%d) out_res(%d, %d)\n",
-		__func__, dz_config->zoom_region.origin.x,
-		dz_config->zoom_region.origin.y,
-		dz_config->zoom_region.resolution.width,
-		dz_config->zoom_region.resolution.height,
+		__func__, dz_config->zoom_region.left,
+		dz_config->zoom_region.top,
+		dz_config->zoom_region.width,
+		dz_config->zoom_region.height,
 		eff_res.width, eff_res.height,
 		asd->sensor_array_res.width,
 		asd->sensor_array_res.height,
 		out_res.width, out_res.height);
 
-	if ((dz_config->zoom_region.origin.x +
-	     dz_config->zoom_region.resolution.width
+	if ((dz_config->zoom_region.left +
+	     dz_config->zoom_region.width
 	     > eff_res.width) ||
-	    (dz_config->zoom_region.origin.y +
-	     dz_config->zoom_region.resolution.height
+	    (dz_config->zoom_region.top +
+	     dz_config->zoom_region.height
 	     > eff_res.height))
 		return -EINVAL;
 
@@ -1899,10 +1899,10 @@ static bool atomisp_check_zoom_region(
 
 	config.width = asd->sensor_array_res.width;
 	config.height = asd->sensor_array_res.height;
-	w = dz_config->zoom_region.origin.x +
-	    dz_config->zoom_region.resolution.width;
-	h = dz_config->zoom_region.origin.y +
-	    dz_config->zoom_region.resolution.height;
+	w = dz_config->zoom_region.left +
+	    dz_config->zoom_region.width;
+	h = dz_config->zoom_region.top +
+	    dz_config->zoom_region.height;
 
 	if ((w <= config.width) && (h <= config.height) && w > 0 && h > 0)
 		flag = true;
@@ -1910,10 +1910,10 @@ static bool atomisp_check_zoom_region(
 		/* setting error zoom region */
 		dev_err(asd->isp->dev,
 			"%s zoom region ERROR:dz_config:(%d,%d),(%d,%d)array_res(%d, %d)\n",
-			__func__, dz_config->zoom_region.origin.x,
-			dz_config->zoom_region.origin.y,
-			dz_config->zoom_region.resolution.width,
-			dz_config->zoom_region.resolution.height,
+			__func__, dz_config->zoom_region.left,
+			dz_config->zoom_region.top,
+			dz_config->zoom_region.width,
+			dz_config->zoom_region.height,
 			config.width, config.height);
 
 	return flag;
@@ -2032,8 +2032,8 @@ static unsigned int long copy_from_compatible(void *to, const void *from,
 {
 	if (from_user)
 		return copy_from_user(to, (void __user *)from, n);
-	else
-		memcpy(to, from, n);
+
+	memcpy(to, from, n);
 	return 0;
 }
 
@@ -2462,9 +2462,11 @@ int atomisp_css_cp_dvs2_coefs(struct atomisp_sub_device *asd,
 		if (sizeof(*cur) != sizeof(coefs->grid) ||
 		    memcmp(&coefs->grid, cur, sizeof(coefs->grid))) {
 			dev_err(asd->isp->dev, "dvs grid mismatch!\n");
-			/* If the grid info in the argument differs from the current
-			grid info, we tell the caller to reset the grid size and
-			try again. */
+			/*
+			 * If the grid info in the argument differs from the current
+			 * grid info, we tell the caller to reset the grid size and
+			 * try again.
+			 */
 			return -EAGAIN;
 		}
 
@@ -3025,9 +3027,11 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag,
 
 		atomisp_curr_user_grid_info(asd, &config->info);
 
-		/* We always return the resolution and stride even if there is
+		/*
+		 * We always return the resolution and stride even if there is
 		 * no valid metadata. This allows the caller to get the
-		 * information needed to allocate user-space buffers. */
+		 * information needed to allocate user-space buffers.
+		 */
 		config->metadata_config.metadata_height = asd->
 			stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream_info.
 			metadata_info.resolution.height;
@@ -3232,7 +3236,7 @@ int atomisp_bad_pixel_param(struct atomisp_sub_device *asd, int flag,
 }
 
 /*
- * Function to enable/disable video image stablization
+ * Function to enable/disable video image stabilization
  */
 int atomisp_video_stable(struct atomisp_sub_device *asd, int flag,
 			 __s32 *value)
@@ -3277,8 +3281,10 @@ atomisp_bytesperline_to_padded_width(unsigned int bytesperline,
 		return bytesperline / 2;
 	case IA_CSS_FRAME_FORMAT_RGBA888:
 		return bytesperline / 4;
-	/* The following cases could be removed, but we leave them
-	   in to document the formats that are included. */
+	/*
+	 * The following cases could be removed, but we leave them
+	 * in to document the formats that are included.
+	 */
 	case IA_CSS_FRAME_FORMAT_NV11:
 	case IA_CSS_FRAME_FORMAT_NV12:
 	case IA_CSS_FRAME_FORMAT_NV16:
@@ -3314,9 +3320,11 @@ atomisp_v4l2_framebuffer_to_css_frame(const struct v4l2_framebuffer *arg,
 	padded_width = atomisp_bytesperline_to_padded_width(
 			   arg->fmt.bytesperline, sh_format);
 
-	/* Note: the padded width on an ia_css_frame is in elements, not in
-	   bytes. The RAW frame we use here should always be a 16bit RAW
-	   frame. This is why we bytesperline/2 is equal to the padded with */
+	/*
+	 * Note: the padded width on an ia_css_frame is in elements, not in
+	 * bytes. The RAW frame we use here should always be a 16bit RAW
+	 * frame. This is why we bytesperline/2 is equal to the padded with
+	 */
 	if (ia_css_frame_allocate(&res, arg->fmt.width, arg->fmt.height,
 				       sh_format, padded_width, 0)) {
 		ret = -ENOMEM;
@@ -3364,10 +3372,8 @@ int atomisp_fixed_pattern_table(struct atomisp_sub_device *asd,
 	if (ret)
 		return ret;
 
-	if (sh_css_set_black_frame(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
-				   raw_black_frame) != 0)
-		return -ENOMEM;
-
+	ret = sh_css_set_black_frame(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream,
+				     raw_black_frame);
 	ia_css_frame_free(raw_black_frame);
 	return ret;
 }
@@ -3925,8 +3931,10 @@ static inline int atomisp_set_sensor_mipi_to_isp(
 		    asd->stream_env[stream_id].isys_info[1].height);
 	}
 
-	/* Compatibility for sensors which provide no media bus code
-	 * in s_mbus_framefmt() nor support pad formats. */
+	/*
+	 * Compatibility for sensors which provide no media bus code
+	 * in s_mbus_framefmt() nor support pad formats.
+	 */
 	if (mipi_info && mipi_info->input_format != -1) {
 		bayer_order = mipi_info->raw_bayer_order;
 
@@ -4003,7 +4011,7 @@ static int css_input_resolution_changed(struct atomisp_sub_device *asd,
 	struct atomisp_metadata_buf *md_buf = NULL, *_md_buf;
 	unsigned int i;
 
-	dev_dbg(asd->isp->dev, "css_input_resolution_changed to %ux%u\n",
+	dev_dbg(asd->isp->dev, "%s: to %ux%u\n", __func__,
 		ffmt->width, ffmt->height);
 
 	if (IS_ISP2401)
@@ -4384,8 +4392,10 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f)
 			ATOMISP_SUBDEV_PAD_SINK,
 			V4L2_SEL_TGT_CROP);
 
-	/* Try to enable YUV downscaling if ISP input is 10 % (either
-	 * width or height) bigger than the desired result. */
+	/*
+	 * Try to enable YUV downscaling if ISP input is 10 % (either
+	 * width or height) bigger than the desired result.
+	 */
 	if (!IS_MOFD ||
 	    isp_sink_crop.width * 9 / 10 < f->fmt.pix.width ||
 	    isp_sink_crop.height * 9 / 10 < f->fmt.pix.height ||
diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.h b/drivers/staging/media/atomisp/pci/atomisp_cmd.h
index 82199dc9284e9..d3d1f2574e77b 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_cmd.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.h
@@ -153,7 +153,7 @@ int atomisp_bad_pixel(struct atomisp_sub_device *asd, int flag,
 int atomisp_bad_pixel_param(struct atomisp_sub_device *asd, int flag,
 			    struct atomisp_dp_config *config);
 
-/* Function to enable/disable video image stablization */
+/* Function to enable/disable video image stabilization */
 int atomisp_video_stable(struct atomisp_sub_device *asd, int flag,
 			 __s32 *value);
 
diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c
index be5f37f4a6fd0..0ee52637ea309 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c
@@ -27,6 +27,7 @@
 
 #include <linux/io.h>
 #include <linux/pm_runtime.h>
+#include <linux/string_choices.h>
 
 /* Assume max number of ACC stages */
 #define MAX_ACC_STAGES	20
@@ -841,7 +842,7 @@ int atomisp_css_irq_enable(struct atomisp_device *isp,
 {
 	dev_dbg(isp->dev, "%s: css irq info 0x%08x: %s (%d).\n",
 		__func__, info,
-		enable ? "enable" : "disable", enable);
+		str_enable_disable(enable), enable);
 	if (ia_css_irq_enable(info, enable)) {
 		dev_warn(isp->dev, "%s:Invalid irq info: 0x%08x when %s.\n",
 			 __func__, info,
@@ -1116,8 +1117,11 @@ int atomisp_css_allocate_stat_buffers(struct atomisp_sub_device   *asd,
 					dvs_grid_info);
 		if (!dis_buf->dis_data) {
 			dev_err(isp->dev, "dvs buf allocation failed.\n");
-			if (s3a_buf)
+			if (s3a_buf) {
+				hmm_vunmap(s3a_buf->s3a_data->data_ptr);
+				ia_css_isp_3a_statistics_map_free(s3a_buf->s3a_map);
 				ia_css_isp_3a_statistics_free(s3a_buf->s3a_data);
+			}
 			return -EINVAL;
 		}
 
@@ -1131,10 +1135,16 @@ int atomisp_css_allocate_stat_buffers(struct atomisp_sub_device   *asd,
 		md_buf->metadata = ia_css_metadata_allocate(
 				       &asd->stream_env[stream_id].stream_info.metadata_info);
 		if (!md_buf->metadata) {
-			if (s3a_buf)
+			if (s3a_buf) {
+				hmm_vunmap(s3a_buf->s3a_data->data_ptr);
+				ia_css_isp_3a_statistics_map_free(s3a_buf->s3a_map);
 				ia_css_isp_3a_statistics_free(s3a_buf->s3a_data);
-			if (dis_buf)
+			}
+			if (dis_buf) {
+				hmm_vunmap(dis_buf->dis_data->data_ptr);
+				ia_css_isp_dvs_statistics_map_free(dis_buf->dvs_map);
 				ia_css_isp_dvs2_statistics_free(dis_buf->dis_data);
+			}
 			dev_err(isp->dev, "metadata buf allocation failed.\n");
 			return -EINVAL;
 		}
@@ -1957,7 +1967,7 @@ static void __configure_capture_pp_input(struct atomisp_sub_device *asd,
 
 /*
  * For CSS2.1, preview pipe could support bayer downscaling, yuv decimation and
- * yuv downscaling, which needs addtional configurations.
+ * yuv downscaling, which needs additional configurations.
  */
 static void __configure_preview_pp_input(struct atomisp_sub_device *asd,
 	unsigned int width, unsigned int height,
@@ -2044,7 +2054,7 @@ static void __configure_preview_pp_input(struct atomisp_sub_device *asd,
 		}
 	}
 	/*
-	 * calculate YUV Decimation, YUV downscaling facor:
+	 * calculate YUV Decimation, YUV downscaling factor:
 	 * YUV Downscaling factor must not exceed 2.
 	 * YUV Decimation factor could be 2, 4.
 	 */
@@ -2085,7 +2095,7 @@ static void __configure_preview_pp_input(struct atomisp_sub_device *asd,
 
 /*
  * For CSS2.1, offline video pipe could support bayer decimation, and
- * yuv downscaling, which needs addtional configurations.
+ * yuv downscaling, which needs additional configurations.
  */
 static void __configure_video_pp_input(struct atomisp_sub_device *asd,
 				       unsigned int width, unsigned int height,
@@ -3002,7 +3012,7 @@ int atomisp_css_get_zoom_factor(struct atomisp_sub_device *asd,
 }
 
 /*
- * Function to set/get image stablization statistics
+ * Function to set/get image stabilization statistics
  */
 int atomisp_css_get_dis_stat(struct atomisp_sub_device *asd,
 			     struct atomisp_dis_statistics *stats)
diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h
index 75781807544aa..5188df4f469ca 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h
@@ -40,7 +40,7 @@ enum atomisp_css_stream_state {
 };
 
 /*
- *  Sensor of external ISP can send multiple steams with different mipi data
+ *  Sensor of external ISP can send multiple streams with different mipi data
  * type in the same virtual channel. This information needs to come from the
  * sensor or external ISP
  */
diff --git a/drivers/staging/media/atomisp/pci/atomisp_csi2_bridge.c b/drivers/staging/media/atomisp/pci/atomisp_csi2_bridge.c
index ba61cc28fac1a..cca91c6d71a59 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_csi2_bridge.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_csi2_bridge.c
@@ -113,7 +113,7 @@ static char *gmin_cfg_get_dsm(struct acpi_device *adev, const char *key)
 	if (!obj)
 		return NULL;
 
-	for (i = 0; i < obj->package.count - 1; i += 2) {
+	for (i = 0; i + 1 < obj->package.count; i += 2) {
 		key_el = &obj->package.elements[i + 0];
 		val_el = &obj->package.elements[i + 1];
 
diff --git a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
index 4026e98c58451..a1d3a16921eaa 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c
@@ -11,8 +11,8 @@
 #include <linux/mfd/intel_soc_pmic.h>
 #include <linux/regulator/consumer.h>
 #include <linux/gpio/consumer.h>
-#include <linux/gpio.h>
 #include <linux/platform_device.h>
+#include <linux/string_choices.h>
 #include "../../include/linux/atomisp_platform.h"
 #include "../../include/linux/atomisp_gmin_platform.h"
 
@@ -917,7 +917,7 @@ static int gmin_acpi_pm_ctrl(struct v4l2_subdev *subdev, int on)
 		return 0;
 
 	dev_dbg(subdev->dev, "Setting power state to %s\n",
-		on ? "on" : "off");
+		str_on_off(on));
 
 	if (on)
 		ret = acpi_device_set_power(adev,
@@ -930,7 +930,7 @@ static int gmin_acpi_pm_ctrl(struct v4l2_subdev *subdev, int on)
 		gs->clock_on = on;
 	else
 		dev_err(subdev->dev, "Couldn't set power state to %s\n",
-			on ? "on" : "off");
+			str_on_off(on));
 
 	return ret;
 }
diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.c b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
index 3d56ca83ecb7c..9de9cd884d99f 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_subdev.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.c
@@ -180,8 +180,8 @@ static int isp_subdev_enum_mbus_code(struct v4l2_subdev *sd,
 	return 0;
 }
 
-static int isp_subdev_validate_rect(struct v4l2_subdev *sd, uint32_t pad,
-				    uint32_t target)
+static int isp_subdev_validate_rect(struct v4l2_subdev *sd, u32 pad,
+				    u32 target)
 {
 	switch (pad) {
 	case ATOMISP_SUBDEV_PAD_SINK:
@@ -203,8 +203,8 @@ static int isp_subdev_validate_rect(struct v4l2_subdev *sd, uint32_t pad,
 
 struct v4l2_rect *atomisp_subdev_get_rect(struct v4l2_subdev *sd,
 	struct v4l2_subdev_state *sd_state,
-	u32 which, uint32_t pad,
-	uint32_t target)
+	u32 which, u32 pad,
+	u32 target)
 {
 	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
 
@@ -229,8 +229,8 @@ struct v4l2_rect *atomisp_subdev_get_rect(struct v4l2_subdev *sd,
 
 struct v4l2_mbus_framefmt
 *atomisp_subdev_get_ffmt(struct v4l2_subdev *sd,
-			 struct v4l2_subdev_state *sd_state, uint32_t which,
-			 uint32_t pad)
+			 struct v4l2_subdev_state *sd_state, u32 which,
+			 u32 pad)
 {
 	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
 
@@ -242,7 +242,7 @@ struct v4l2_mbus_framefmt
 
 static void isp_get_fmt_rect(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_state *sd_state,
-			     uint32_t which,
+			     u32 which,
 			     struct v4l2_mbus_framefmt **ffmt,
 			     struct v4l2_rect *crop[ATOMISP_SUBDEV_PADS_NUM],
 			     struct v4l2_rect *comp[ATOMISP_SUBDEV_PADS_NUM])
@@ -291,7 +291,7 @@ static const char *atomisp_pad_str(unsigned int pad)
 
 int atomisp_subdev_set_selection(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
-				 u32 which, uint32_t pad, uint32_t target,
+				 u32 which, u32 pad, u32 target,
 				 u32 flags, struct v4l2_rect *r)
 {
 	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
@@ -467,7 +467,7 @@ static int isp_subdev_set_selection(struct v4l2_subdev *sd,
 
 void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_state *sd_state,
-			     uint32_t which,
+			     u32 which,
 			     u32 pad, struct v4l2_mbus_framefmt *ffmt)
 {
 	struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd);
@@ -808,7 +808,7 @@ static int isp_subdev_init_entities(struct atomisp_sub_device *asd)
 	int ret;
 
 	v4l2_subdev_init(sd, &isp_subdev_v4l2_ops);
-	sprintf(sd->name, "Atom ISP");
+	strscpy(sd->name, "Atom ISP");
 	v4l2_set_subdevdata(sd, asd);
 	sd->flags |= V4L2_SUBDEV_FL_HAS_EVENTS | V4L2_SUBDEV_FL_HAS_DEVNODE;
 
diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.h b/drivers/staging/media/atomisp/pci/atomisp_subdev.h
index e1d0168cb91d3..b12bb65be3f2f 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_subdev.h
+++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.h
@@ -315,20 +315,20 @@ bool atomisp_subdev_format_conversion(struct atomisp_sub_device *asd);
 /* Get pointer to appropriate format */
 struct v4l2_mbus_framefmt
 *atomisp_subdev_get_ffmt(struct v4l2_subdev *sd,
-			 struct v4l2_subdev_state *sd_state, uint32_t which,
-			 uint32_t pad);
+			 struct v4l2_subdev_state *sd_state, u32 which,
+			 u32 pad);
 struct v4l2_rect *atomisp_subdev_get_rect(struct v4l2_subdev *sd,
 	struct v4l2_subdev_state *sd_state,
-	u32 which, uint32_t pad,
-	uint32_t target);
+	u32 which, u32 pad,
+	u32 target);
 int atomisp_subdev_set_selection(struct v4l2_subdev *sd,
 				 struct v4l2_subdev_state *sd_state,
-				 u32 which, uint32_t pad, uint32_t target,
+				 u32 which, u32 pad, u32 target,
 				 u32 flags, struct v4l2_rect *r);
 /* Actually set the format */
 void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd,
 			     struct v4l2_subdev_state *sd_state,
-			     uint32_t which,
+			     u32 which,
 			     u32 pad, struct v4l2_mbus_framefmt *ffmt);
 
 void atomisp_subdev_cleanup_pending_events(struct atomisp_sub_device *asd);
diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
index 900a67552d6a0..8122303974092 100644
--- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
+++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c
@@ -16,6 +16,7 @@
 #include <linux/dmi.h>
 #include <linux/interrupt.h>
 #include <linux/bits.h>
+#include <linux/string_choices.h>
 #include <media/v4l2-fwnode.h>
 
 #include <asm/iosf_mbi.h>
@@ -527,7 +528,7 @@ static int atomisp_mrfld_power(struct atomisp_device *isp, bool enable)
 	u32 val = enable ? MRFLD_ISPSSPM0_IUNIT_POWER_ON :
 			   MRFLD_ISPSSPM0_IUNIT_POWER_OFF;
 
-	dev_dbg(isp->dev, "IUNIT power-%s.\n", enable ? "on" : "off");
+	dev_dbg(isp->dev, "IUNIT power-%s.\n", str_on_off(enable));
 
 	/* WA for P-Unit, if DVFS enabled, ISP timeout observed */
 	if (IS_CHT && enable && !isp->pm_only) {
@@ -569,7 +570,7 @@ static int atomisp_mrfld_power(struct atomisp_device *isp, bool enable)
 		usleep_range(100, 150);
 	} while (1);
 
-	dev_err(isp->dev, "IUNIT power-%s timeout.\n", enable ? "on" : "off");
+	dev_err(isp->dev, "IUNIT power-%s timeout.\n", str_on_off(enable));
 	return -EBUSY;
 }
 
diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h
index 86300991d30eb..4732b45b25eeb 100644
--- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h
+++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h
@@ -139,21 +139,16 @@ static inline uint8_t ia_css_circbuf_get_pos_at_offset(
     u32 base,
     int offset)
 {
-	u8 dest;
-
 	OP___assert(cb);
 	OP___assert(cb->desc);
 	OP___assert(cb->desc->size > 0);
 
 	/* step 1: adjudst the offset  */
-	while (offset < 0) {
+	while (offset < 0)
 		offset += cb->desc->size;
-	}
 
 	/* step 2: shift and round by the upper limit */
-	dest = OP_std_modadd(base, offset, cb->desc->size);
-
-	return dest;
+	return (base + offset) % cb->desc->size;
 }
 
 /**
diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h
index 5645a7bf493cc..64f754f1d49b3 100644
--- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h
+++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h
@@ -47,7 +47,7 @@ static inline bool ia_css_circbuf_desc_is_full(
     ia_css_circbuf_desc_t *cb_desc)
 {
 	OP___assert(cb_desc);
-	return (OP_std_modadd(cb_desc->end, 1, cb_desc->size) == cb_desc->start);
+	return ((cb_desc->end + 1) % cb_desc->size) == cb_desc->start;
 }
 
 /**
@@ -78,20 +78,15 @@ static inline uint8_t ia_css_circbuf_desc_get_pos_at_offset(
     u32 base,
     int offset)
 {
-	u8 dest;
-
 	OP___assert(cb_desc);
 	OP___assert(cb_desc->size > 0);
 
 	/* step 1: adjust the offset  */
-	while (offset < 0) {
+	while (offset < 0)
 		offset += cb_desc->size;
-	}
 
 	/* step 2: shift and round by the upper limit */
-	dest = OP_std_modadd(base, offset, cb_desc->size);
-
-	return dest;
+	return (base + offset) % cb_desc->size;
 }
 
 /**
diff --git a/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c b/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c
index 58e4e3173b405..4a8675d0129ac 100644
--- a/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c
+++ b/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c
@@ -238,9 +238,8 @@ void ia_css_refcount_clear(s32 id, clear_func clear_func_ptr)
 				hmm_free(entry->data);
 			}
 
-			if (entry->count != 0) {
+			if (entry->count != 0)
 				IA_CSS_WARNING("Ref count for entry %x is not zero!", entry->id);
-			}
 
 			assert(entry->count == 0);
 
diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c
index a9f736398f504..fa2f8ed5b053a 100644
--- a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c
+++ b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c
@@ -32,9 +32,8 @@ void ia_css_pipe_get_generic_stage_desc(
 	stage_desc->max_input_width = 0;
 	stage_desc->mode = binary->info->sp.pipeline.mode;
 	stage_desc->in_frame = in_frame;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
+	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
 		stage_desc->out_frame[i] = out_frame[i];
-	}
 	stage_desc->vf_frame = vf_frame;
 ERR:
 	IA_CSS_LEAVE_PRIVATE("");
@@ -59,9 +58,8 @@ void ia_css_pipe_get_firmwares_stage_desc(
 	stage_desc->max_input_width = 0;
 	stage_desc->mode = mode;
 	stage_desc->in_frame = in_frame;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
+	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
 		stage_desc->out_frame[i] = out_frame[i];
-	}
 	stage_desc->vf_frame = vf_frame;
 }
 
@@ -82,8 +80,7 @@ void ia_css_pipe_get_sp_func_stage_desc(
 	stage_desc->mode = (unsigned int)-1;
 	stage_desc->in_frame = NULL;
 	stage_desc->out_frame[0] = out_frame;
-	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
+	for (i = 1; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
 		stage_desc->out_frame[i] = NULL;
-	}
 	stage_desc->vf_frame = NULL;
 }
diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c
index c7c42b472cc7c..6cb3ecbd72974 100644
--- a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c
+++ b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c
@@ -26,9 +26,8 @@ void ia_css_pipe_util_create_output_frames(
 	unsigned int i;
 
 	assert(frames);
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
+	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
 		frames[i] = NULL;
-	}
 }
 
 void ia_css_pipe_util_set_output_frames(
diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h
index 3210dd6bf9ca7..8e295cd781299 100644
--- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h
+++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h
@@ -41,9 +41,8 @@ STORAGE_CLASS_STREAM2MMIO_C void stream2mmio_get_state(
 	 * Get the values of the register-set per
 	 * stream2mmio-controller sids.
 	 */
-	for (i = STREAM2MMIO_SID0_ID; i < N_STREAM2MMIO_SID_PROCS[ID]; i++) {
+	for (i = STREAM2MMIO_SID0_ID; i < N_STREAM2MMIO_SID_PROCS[ID]; i++)
 		stream2mmio_get_sid_state(ID, i, &state->sid_state[i]);
-	}
 }
 
 /**
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h
index 439c69444942e..e77d7cf61356a 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h
@@ -26,9 +26,8 @@ STORAGE_CLASS_EVENT_C void event_wait_for(const event_ID_t ID)
 STORAGE_CLASS_EVENT_C void cnd_event_wait_for(const event_ID_t ID,
 	const bool cnd)
 {
-	if (cnd) {
+	if (cnd)
 		event_wait_for(ID);
-	}
 }
 
 STORAGE_CLASS_EVENT_C hrt_data event_receive_token(const event_ID_t ID)
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c
index 8bb78b4d7c677..b31e3809c0e4c 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c
@@ -26,9 +26,7 @@ static inline void gdc_reg_store(
 /*
  * Exported function implementations
  */
-void gdc_lut_store(
-    const gdc_ID_t		ID,
-    const int			data[4][HRT_GDC_N])
+void gdc_lut_store(const gdc_ID_t ID, const int data[4][HRT_GDC_N])
 {
 	unsigned int i, lut_offset = HRT_GDC_LUT_IDX;
 
@@ -36,20 +34,17 @@ void gdc_lut_store(
 	assert(HRT_GDC_LUT_COEFF_OFFSET <= (4 * sizeof(hrt_data)));
 
 	for (i = 0; i < HRT_GDC_N; i++) {
-		hrt_data	entry_0 = data[0][i] & HRT_GDC_BCI_COEF_MASK;
-		hrt_data	entry_1 = data[1][i] & HRT_GDC_BCI_COEF_MASK;
-		hrt_data	entry_2 = data[2][i] & HRT_GDC_BCI_COEF_MASK;
-		hrt_data	entry_3 = data[3][i] & HRT_GDC_BCI_COEF_MASK;
+		hrt_data entry_0 = data[0][i] & HRT_GDC_BCI_COEF_MASK;
+		hrt_data entry_1 = data[1][i] & HRT_GDC_BCI_COEF_MASK;
+		hrt_data entry_2 = data[2][i] & HRT_GDC_BCI_COEF_MASK;
+		hrt_data entry_3 = data[3][i] & HRT_GDC_BCI_COEF_MASK;
 
-		hrt_data	word_0 = entry_0 |
-				     (entry_1 << HRT_GDC_LUT_COEFF_OFFSET);
-		hrt_data	word_1 = entry_2 |
-				     (entry_3 << HRT_GDC_LUT_COEFF_OFFSET);
+		hrt_data word_0 = entry_0 | (entry_1 << HRT_GDC_LUT_COEFF_OFFSET);
+		hrt_data word_1 = entry_2 | (entry_3 << HRT_GDC_LUT_COEFF_OFFSET);
 
 		gdc_reg_store(ID, lut_offset++, word_0);
 		gdc_reg_store(ID, lut_offset++, word_1);
 	}
-	return;
 }
 
 /*
@@ -85,8 +80,7 @@ void gdc_lut_convert_to_isp_format(const int in_lut[4][HRT_GDC_N],
 	}
 }
 
-int gdc_get_unity(
-    const gdc_ID_t		ID)
+int gdc_get_unity(const gdc_ID_t ID)
 {
 	assert(ID < N_GDC_ID);
 	(void)ID;
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c
index 40b3f1e48c566..c353461dcdf26 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c
@@ -62,9 +62,8 @@ void input_formatter_rst(
 	 * WICH USES THE STREAM2MEMRY BLOCK.
 	 * MUST BE FIXED PROPERLY
 	 */
-	if (!HIVE_IF_BIN_COPY[ID]) {
+	if (!HIVE_IF_BIN_COPY[ID])
 		input_formatter_reg_store(ID, addr, rst);
-	}
 
 	return;
 }
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c
index 9f1199c4761c4..68b0ad27615df 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c
@@ -138,11 +138,10 @@ void receiver_port_enable(
 	hrt_data	reg = receiver_port_reg_load(ID, port_ID,
 			  _HRT_CSS_RECEIVER_DEVICE_READY_REG_IDX);
 
-	if (cnd) {
+	if (cnd)
 		reg |= 0x01;
-	} else {
+	else
 		reg &= ~0x01;
-	}
 
 	receiver_port_reg_store(ID, port_ID,
 				_HRT_CSS_RECEIVER_DEVICE_READY_REG_IDX, reg);
@@ -194,9 +193,8 @@ static void receiver_rst(
 	assert(ID < N_RX_ID);
 
 // Disable all ports.
-	for (port_id = MIPI_PORT0_ID; port_id < N_MIPI_PORT_ID; port_id++) {
+	for (port_id = MIPI_PORT0_ID; port_id < N_MIPI_PORT_ID; port_id++)
 		receiver_port_enable(ID, port_id, false);
-	}
 
 	// AM: Additional actions for stopping receiver?
 }
@@ -830,15 +828,13 @@ input_system_err_t input_system_configuration_commit(void)
 	// The last configuration step is to configure the input buffer.
 	input_system_err_t error = input_buffer_configuration();
 
-	if (error != INPUT_SYSTEM_ERR_NO_ERROR) {
+	if (error != INPUT_SYSTEM_ERR_NO_ERROR)
 		return error;
-	}
 
 	// Translate the whole configuration into registers.
 	error = configuration_to_registers();
-	if (error != INPUT_SYSTEM_ERR_NO_ERROR) {
+	if (error != INPUT_SYSTEM_ERR_NO_ERROR)
 		return error;
-	}
 
 	// Translate the whole configuration into ctrl commands etc.
 
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c
index b66560bca6259..61dea386361aa 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c
@@ -55,9 +55,8 @@ void irq_clear_all(
 	assert(ID < N_IRQ_ID);
 	assert(IRQ_N_CHANNEL[ID] <= HRT_DATA_WIDTH);
 
-	if (IRQ_N_CHANNEL[ID] < HRT_DATA_WIDTH) {
+	if (IRQ_N_CHANNEL[ID] < HRT_DATA_WIDTH)
 		mask = ~((~(hrt_data)0) >> IRQ_N_CHANNEL[ID]);
-	}
 
 	irq_reg_store(ID,
 		      _HRT_IRQ_CONTROLLER_CLEAR_REG_IDX, mask);
@@ -115,9 +114,9 @@ void irq_enable_pulse(
 {
 	unsigned int edge_out = 0x0;
 
-	if (pulse) {
+	if (pulse)
 		edge_out = 0xffffffff;
-	}
+
 	/* output is given as edge, not pulse */
 	irq_reg_store(ID,
 		      _HRT_IRQ_CONTROLLER_EDGE_NOT_PULSE_REG_IDX, edge_out);
@@ -259,9 +258,9 @@ void virq_clear_all(void)
 {
 	irq_ID_t	irq_id;
 
-	for (irq_id = (irq_ID_t)0; irq_id < N_IRQ_ID; irq_id++) {
+	for (irq_id = (irq_ID_t)0; irq_id < N_IRQ_ID; irq_id++)
 		irq_clear_all(irq_id);
-	}
+
 	return;
 }
 
@@ -301,9 +300,9 @@ void virq_clear_info(struct virq_info *irq_info)
 
 	assert(irq_info);
 
-	for (ID = (irq_ID_t)0 ; ID < N_IRQ_ID; ID++) {
+	for (ID = (irq_ID_t)0 ; ID < N_IRQ_ID; ID++)
 		irq_info->irq_status_reg[ID] = 0;
-	}
+
 	return;
 }
 
@@ -324,20 +323,17 @@ enum hrt_isp_css_irq_status virq_get_channel_id(
 			break;
 	}
 
-	if (idx == IRQ_N_CHANNEL[IRQ0_ID]) {
+	if (idx == IRQ_N_CHANNEL[IRQ0_ID])
 		return hrt_isp_css_irq_status_error;
-	}
 
 	/* Check whether there are more bits set on device 0 */
-	if (irq_status != (1U << idx)) {
+	if (irq_status != (1U << idx))
 		status = hrt_isp_css_irq_status_more_irqs;
-	}
 
 	/* Check whether we have an IRQ on one of the nested devices */
 	for (ID = N_IRQ_ID - 1 ; ID > (irq_ID_t)0; ID--) {
-		if (IRQ_NESTING_ID[ID] == (enum virq_id)idx) {
+		if (IRQ_NESTING_ID[ID] == (enum virq_id)idx)
 			break;
-		}
 	}
 
 	/* If we have a nested IRQ, load that state, discard the device 0 state */
@@ -350,9 +346,8 @@ enum hrt_isp_css_irq_status virq_get_channel_id(
 				break;
 		}
 
-		if (idx == IRQ_N_CHANNEL[ID]) {
+		if (idx == IRQ_N_CHANNEL[ID])
 			return hrt_isp_css_irq_status_error;
-		}
 
 		/* Alternatively check whether there are more bits set on this device */
 		if (irq_status != (1U << idx)) {
@@ -408,9 +403,8 @@ static inline irq_ID_t virq_get_irq_id(
 	assert(channel_ID);
 
 	for (ID = (irq_ID_t)0 ; ID < N_IRQ_ID; ID++) {
-		if (irq_ID < IRQ_N_ID_OFFSET[ID + 1]) {
+		if (irq_ID < IRQ_N_ID_OFFSET[ID + 1])
 			break;
-		}
 	}
 
 	*channel_ID = (unsigned int)irq_ID - IRQ_N_ID_OFFSET[ID];
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c
index 064e88a5e0644..70d118fe1e705 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c
@@ -32,7 +32,6 @@ void mmu_invalidate_cache_all(void)
 {
 	mmu_ID_t	mmu_id;
 
-	for (mmu_id = (mmu_ID_t)0; mmu_id < N_MMU_ID; mmu_id++) {
+	for (mmu_id = (mmu_ID_t)0; mmu_id < N_MMU_ID; mmu_id++)
 		mmu_invalidate_cache(mmu_id);
-	}
 }
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c
index 722b684fbc37b..1c5c50406633e 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c
@@ -168,9 +168,9 @@ static void store_vector(
 	//load_vector (&v[1][0], &to[ISP_NWAY]); /* Fetch the next vector, since it will be overwritten. */
 	hive_uedge *data = (hive_uedge *)v;
 
-	for (i = 0; i < ISP_NWAY; i++) {
+	for (i = 0; i < ISP_NWAY; i++)
 		hive_sim_wide_pack(data, (hive_wide)&from[i], ISP_VEC_ELEMBITS, i);
-	}
+
 	assert(ISP_BAMEM_BASE[ID] != (hrt_address) - 1);
 #if !defined(HRT_MEMORY_ACCESS)
 	ia_css_device_store(ISP_BAMEM_BASE[ID] + (unsigned long)to, &v, size);
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h
index 1a435a3483182..b321f41011939 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h
@@ -11,72 +11,61 @@
 #include "device_access.h"
 #include "assert_support.h"
 
-/*! Set the page table base index of MMU[ID]
-
- \param	ID[in]				MMU identifier
- \param	base_index[in]		page table base index
-
- \return none, MMU[ID].page_table_base_index = base_index
+/**
+ * mmu_set_page_table_base_index() - Set the page table base index of MMU[ID]
+ * @ID:				MMU identifier
+ * @base_index:			page table base index
+ *
+ * Return: none, MMU[ID].page_table_base_index = base_index
  */
-void mmu_set_page_table_base_index(
-    const mmu_ID_t		ID,
-    const hrt_data		base_index);
-
-/*! Get the page table base index of MMU[ID]
+void mmu_set_page_table_base_index(const mmu_ID_t ID, const hrt_data base_index);
 
- \param	ID[in]				MMU identifier
- \param	base_index[in]		page table base index
-
- \return MMU[ID].page_table_base_index
+/**
+ * mmu_get_page_table_base_index() - Get the page table base index of MMU[ID]
+ * @ID:				MMU identifier
+ *
+ * Return: MMU[ID].page_table_base_index
  */
-hrt_data mmu_get_page_table_base_index(
-    const mmu_ID_t		ID);
-
-/*! Invalidate the page table cache of MMU[ID]
-
- \param	ID[in]				MMU identifier
+hrt_data mmu_get_page_table_base_index(const mmu_ID_t ID);
 
- \return none
+/**
+ * mmu_invalidate_cache() - nvalidate the page table cache of MMU[ID]
+ * @ID:				MMU identifier
+ *
+ * Return: none
  */
-void mmu_invalidate_cache(
-    const mmu_ID_t		ID);
+void mmu_invalidate_cache(const mmu_ID_t ID);
 
-/*! Invalidate the page table cache of all MMUs
-
- \return none
+/**
+ * mmu_invalidate_cache_all() - Invalidate the page table cache of all MMUs
+ *
+ * Return: none
  */
 void mmu_invalidate_cache_all(void);
 
-/*! Write to a control register of MMU[ID]
-
- \param	ID[in]				MMU identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return none, MMU[ID].ctrl[reg] = value
+/**
+ * mmu_reg_store() - Write to a control register of MMU[ID]
+ * @ID:				MMU identifier
+ * @reg:			register index
+ * @value:			The data to be written
+ *
+ * Return: none, MMU[ID].ctrl[reg] = value
  */
-static inline void mmu_reg_store(
-    const mmu_ID_t		ID,
-    const unsigned int	reg,
-    const hrt_data		value)
+static inline void mmu_reg_store(const mmu_ID_t ID, const unsigned int reg, const hrt_data value)
 {
 	assert(ID < N_MMU_ID);
 	assert(MMU_BASE[ID] != (hrt_address) - 1);
 	ia_css_device_store_uint32(MMU_BASE[ID] + reg * sizeof(hrt_data), value);
-	return;
 }
 
-/*! Read from a control register of MMU[ID]
-
- \param	ID[in]				MMU identifier
- \param	reg[in]				register index
- \param value[in]			The data to be written
-
- \return MMU[ID].ctrl[reg]
+/**
+ * mmu_reg_load() - Read from a control register of MMU[ID]
+ * @ID:				MMU identifier
+ * @reg:			register index
+ *
+ * Return: MMU[ID].ctrl[reg]
  */
-static inline hrt_data mmu_reg_load(
-    const mmu_ID_t		ID,
-    const unsigned int	reg)
+static inline hrt_data mmu_reg_load(const mmu_ID_t ID, const unsigned int reg)
 {
 	assert(ID < N_MMU_ID);
 	assert(MMU_BASE[ID] != (hrt_address) - 1);
diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h
index 2cb5c986790a8..72a070d947360 100644
--- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h
+++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h
@@ -14,10 +14,4 @@
 #define CEIL_MUL(a, b)       (CEIL_DIV(a, b) * (b))
 #define CEIL_SHIFT(a, b)     (((a) + (1 << (b)) - 1) >> (b))
 
-/*
- * For SP and ISP, SDK provides the definition of OP_std_modadd.
- * We need it only for host
- */
-#define OP_std_modadd(base, offset, size) ((base + offset) % (size))
-
 #endif /* __MATH_SUPPORT_H */
diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c b/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c
index 856561e951a5e..0a8f401a1ca1e 100644
--- a/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c
+++ b/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c
@@ -620,14 +620,23 @@ static void free_private_bo_pages(struct hmm_buffer_object *bo)
 /*Allocate pages which will be used only by ISP*/
 static int alloc_private_pages(struct hmm_buffer_object *bo)
 {
-	const gfp_t gfp = __GFP_NOWARN | __GFP_RECLAIM | __GFP_FS;
+	unsigned int nr_allocated = 0;
+	struct page *page;
 	int ret;
 
-	ret = alloc_pages_bulk(gfp, bo->pgnr, bo->pages);
-	if (ret != bo->pgnr) {
-		free_pages_bulk_array(ret, bo->pages);
-		dev_err(atomisp_dev, "alloc_pages_bulk() failed\n");
-		return -ENOMEM;
+	nr_allocated = alloc_pages_bulk(GFP_KERNEL, bo->pgnr, bo->pages);
+	/*
+	 * alloc_pages_bulk() does not try very hard to get pages under memory
+	 * pressure. If necessary fall back to alloc_page().
+	 */
+	while (nr_allocated < bo->pgnr) {
+		page = alloc_pages(GFP_KERNEL, 0);
+		if (!page) {
+			free_pages_bulk_array(nr_allocated, bo->pages);
+			return -ENOMEM;
+		}
+		bo->pages[nr_allocated] = page;
+		nr_allocated++;
 	}
 
 	ret = set_pages_array_uc(bo->pages, bo->pgnr);
@@ -975,8 +984,7 @@ void hmm_bo_unref(struct hmm_buffer_object *bo)
 
 static void hmm_bo_vm_open(struct vm_area_struct *vma)
 {
-	struct hmm_buffer_object *bo =
-	    (struct hmm_buffer_object *)vma->vm_private_data;
+	struct hmm_buffer_object *bo = vma->vm_private_data;
 
 	check_bo_null_return_void(bo);
 
@@ -993,8 +1001,7 @@ static void hmm_bo_vm_open(struct vm_area_struct *vma)
 
 static void hmm_bo_vm_close(struct vm_area_struct *vma)
 {
-	struct hmm_buffer_object *bo =
-	    (struct hmm_buffer_object *)vma->vm_private_data;
+	struct hmm_buffer_object *bo = vma->vm_private_data;
 
 	check_bo_null_return_void(bo);
 
diff --git a/drivers/staging/media/atomisp/pci/ia_css_types.h b/drivers/staging/media/atomisp/pci/ia_css_types.h
index 676d7e20b282d..2b7db9cda23ab 100644
--- a/drivers/staging/media/atomisp/pci/ia_css_types.h
+++ b/drivers/staging/media/atomisp/pci/ia_css_types.h
@@ -15,6 +15,8 @@
  * directly but still need to forward parameters for it.
  */
 
+#include <linux/videodev2.h>
+
 #include <type_support.h>
 
 #include "ia_css_frac.h"
@@ -427,14 +429,6 @@ struct ia_css_point {
 	s32 y; /** y coordinate */
 };
 
-/**
- * This specifies the region
- */
-struct ia_css_region {
-	struct ia_css_point origin; /** Starting point coordinates for the region */
-	struct ia_css_resolution resolution; /** Region resolution */
-};
-
 /**
  * Digital zoom:
  * This feature is currently available only for video, but will become
@@ -442,7 +436,7 @@ struct ia_css_region {
  * Set the digital zoom factor, this is a logarithmic scale. The actual zoom
  * factor will be 64/x.
  * Setting dx or dy to 0 disables digital zoom for that direction.
- * New API change for Digital zoom:(added struct ia_css_region zoom_region)
+ *
  * zoom_region specifies the origin of the zoom region and width and
  * height of that region.
  * origin : This is the coordinate (x,y) within the effective input resolution
@@ -455,7 +449,7 @@ struct ia_css_region {
 struct ia_css_dz_config {
 	u32 dx; /** Horizontal zoom factor */
 	u32 dy; /** Vertical zoom factor */
-	struct ia_css_region zoom_region; /** region for zoom */
+	struct v4l2_rect zoom_region; /** region for zoom */
 };
 
 /* The still capture mode, this can be RAW (simply copy sensor input to DDR),
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h
index 2f568a7062da7..f825f537a5366 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h
@@ -28,11 +28,13 @@
  *     ISP block: BAA2
  *     ISP1: BAA2 is used.
  *     ISP2: BAA2 is used.
+ *
+ * @strength: Strength of the filter, in u0.13 fixed-point format.
+ *            Valid range: [0, 8191]. A value of 0 means the filter is
+ *            ineffective (default).
  */
 struct ia_css_aa_config {
-	u16 strength;	/** Strength of the filter.
-					u0.13, [0,8191],
-					default/ineffective 0 */
+	u16 strength;
 };
 
 #endif /* __IA_CSS_AA2_TYPES_H */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
index 899d566234b9d..f4dd3ca03d75c 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c
@@ -11,30 +11,26 @@
 #include "ia_css_anr.host.h"
 
 const struct ia_css_anr_config default_anr_config = {
-	10,
-	{
+	.threshold = 10,
+	.thresholds = {
 		0, 3, 1, 2, 3, 6, 4, 5, 1, 4, 2, 3, 2, 5, 3, 4,
 		0, 3, 1, 2, 3, 6, 4, 5, 1, 4, 2, 3, 2, 5, 3, 4,
 		0, 3, 1, 2, 3, 6, 4, 5, 1, 4, 2, 3, 2, 5, 3, 4,
 		0, 3, 1, 2, 3, 6, 4, 5, 1, 4, 2, 3, 2, 5, 3, 4
 	},
-	{10, 20, 30}
+	.factors = {10, 20, 30},
 };
 
-void
-ia_css_anr_encode(
-    struct sh_css_isp_anr_params *to,
-    const struct ia_css_anr_config *from,
-    unsigned int size)
+void ia_css_anr_encode(struct sh_css_isp_anr_params *to,
+		       const struct ia_css_anr_config *from,
+		       unsigned int size)
 {
 	(void)size;
 	to->threshold = from->threshold;
 }
 
-void
-ia_css_anr_dump(
-    const struct sh_css_isp_anr_params *anr,
-    unsigned int level)
+void ia_css_anr_dump(const struct sh_css_isp_anr_params *anr,
+		     unsigned int level)
 {
 	if (!anr) return;
 	ia_css_debug_dtrace(level, "Advance Noise Reduction:\n");
@@ -42,10 +38,8 @@ ia_css_anr_dump(
 			    "anr_threshold", anr->threshold);
 }
 
-void
-ia_css_anr_debug_dtrace(
-    const struct ia_css_anr_config *config,
-    unsigned int level)
+void ia_css_anr_debug_dtrace(const struct ia_css_anr_config *config,
+			     unsigned int level)
 {
 	ia_css_debug_dtrace(level,
 			    "config.threshold=%d\n",
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
index 4f77900871c8b..2f17d62b92087 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h
@@ -12,20 +12,14 @@
 
 extern const struct ia_css_anr_config default_anr_config;
 
-void
-ia_css_anr_encode(
-    struct sh_css_isp_anr_params *to,
-    const struct ia_css_anr_config *from,
-    unsigned int size);
+void ia_css_anr_encode(struct sh_css_isp_anr_params *to,
+		       const struct ia_css_anr_config *from,
+		       unsigned int size);
 
-void
-ia_css_anr_dump(
-    const struct sh_css_isp_anr_params *anr,
-    unsigned int level);
+void ia_css_anr_dump(const struct sh_css_isp_anr_params *anr,
+		     unsigned int level);
 
-void
-ia_css_anr_debug_dtrace(
-    const struct ia_css_anr_config *config, unsigned int level)
-;
+void ia_css_anr_debug_dtrace(const struct ia_css_anr_config *config,
+			     unsigned int level);
 
 #endif /* __IA_CSS_ANR_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c
index 09599884bdaef..6ba34dc48ad45 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c
@@ -10,11 +10,9 @@
 
 #include "ia_css_anr2.host.h"
 
-void
-ia_css_anr2_vmem_encode(
-    struct ia_css_isp_anr2_params *to,
-    const struct ia_css_anr_thres *from,
-    size_t size)
+void ia_css_anr2_vmem_encode(struct ia_css_isp_anr2_params *to,
+			     const struct ia_css_anr_thres *from,
+			     size_t size)
 {
 	unsigned int i;
 
@@ -22,16 +20,13 @@ ia_css_anr2_vmem_encode(
 	for (i = 0; i < ANR_PARAM_SIZE; i++) {
 		unsigned int j;
 
-		for (j = 0; j < ISP_VEC_NELEMS; j++) {
+		for (j = 0; j < ISP_VEC_NELEMS; j++)
 			to->data[i][j] = from->data[i * ISP_VEC_NELEMS + j];
-		}
 	}
 }
 
-void
-ia_css_anr2_debug_dtrace(
-    const struct ia_css_anr_thres *config,
-    unsigned int level)
+void ia_css_anr2_debug_dtrace(const struct ia_css_anr_thres *config,
+			      unsigned int level)
 {
 	(void)config;
 	(void)level;
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h
index 2b1105f21c1e2..36fb6c2596999 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h
@@ -13,15 +13,12 @@
 #include "ia_css_anr2_param.h"
 #include "ia_css_anr2_table.host.h"
 
-void
-ia_css_anr2_vmem_encode(
-    struct ia_css_isp_anr2_params *to,
-    const struct ia_css_anr_thres *from,
-    size_t size);
+void ia_css_anr2_vmem_encode(struct ia_css_isp_anr2_params *to,
+			     const struct ia_css_anr_thres *from,
+			     size_t size);
 
-void
-ia_css_anr2_debug_dtrace(
-    const struct ia_css_anr_thres *config, unsigned int level)
+void ia_css_anr2_debug_dtrace(const struct ia_css_anr_thres *config,
+			      unsigned int level)
 ;
 
 #endif /* __IA_CSS_ANR2_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c
index 69c87e53f3c22..b87eb1a21b216 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c
@@ -12,10 +12,8 @@
 
 #include "ia_css_bh.host.h"
 
-void
-ia_css_bh_hmem_decode(
-    struct ia_css_3a_rgby_output *out_ptr,
-    const struct ia_css_bh_table *hmem_buf)
+void ia_css_bh_hmem_decode(struct ia_css_3a_rgby_output *out_ptr,
+			   const struct ia_css_bh_table *hmem_buf)
 {
 	int i;
 
@@ -37,11 +35,9 @@ ia_css_bh_hmem_decode(
 	}
 }
 
-void
-ia_css_bh_encode(
-    struct sh_css_isp_bh_params *to,
-    const struct ia_css_3a_config *from,
-    unsigned int size)
+void ia_css_bh_encode(struct sh_css_isp_bh_params *to,
+		      const struct ia_css_3a_config *from,
+		      unsigned int size)
 {
 	(void)size;
 	/* coefficients to calculate Y */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h
index 36b360cfe62e6..964d658ceec31 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h
@@ -10,15 +10,11 @@
 #include "ia_css_bh_param.h"
 #include "s3a/s3a_1.0/ia_css_s3a_types.h"
 
-void
-ia_css_bh_hmem_decode(
-    struct ia_css_3a_rgby_output *out_ptr,
-    const struct ia_css_bh_table *hmem_buf);
+void ia_css_bh_hmem_decode(struct ia_css_3a_rgby_output *out_ptr,
+			   const struct ia_css_bh_table *hmem_buf);
 
-void
-ia_css_bh_encode(
-    struct sh_css_isp_bh_params *to,
-    const struct ia_css_3a_config *from,
-    unsigned int size);
+void ia_css_bh_encode(struct sh_css_isp_bh_params *to,
+		      const struct ia_css_3a_config *from,
+		      unsigned int size);
 
 #endif /* __IA_CSS_BH_HOST_H */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c
index cd867937ee13f..fcd7d7e2afe8b 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c
@@ -125,14 +125,12 @@ ia_css_bnlm_vmem_encode(
 	bnlm_lut_encode(&to->div_lut, div_lut_nearests, div_lut_slopes,
 			BNLM_DIV_LUT_SIZE);
 	memset(to->div_lut_intercepts, 0, sizeof(to->div_lut_intercepts));
-	for (i = 0; i < BNLM_DIV_LUT_SIZE; i++) {
+	for (i = 0; i < BNLM_DIV_LUT_SIZE; i++)
 		to->div_lut_intercepts[0][i] = div_lut_intercepts[i];
-	}
 
 	memset(to->power_of_2, 0, sizeof(to->power_of_2));
-	for (i = 0; i < (ISP_VEC_ELEMBITS - 1); i++) {
+	for (i = 0; i < (ISP_VEC_ELEMBITS - 1); i++)
 		to->power_of_2[0][i] = 1 << i;
-	}
 }
 
 /* - Encodes BNLM public parameters into DMEM parameters */
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
index 38751b8e9e6ac..177487b6b2378 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c
@@ -52,13 +52,12 @@ static int ctc2_slope(int y1, int y0, int x1, int x0)
 	/*the slope must lie within the range
 	  (-max_slope-1) >= (dydx) >= (max_slope)
 	*/
-	if (slope <= -max_slope - 1) {
+	if (slope <= -max_slope - 1)
 		dydx = -max_slope - 1;
-	} else if (slope >= max_slope) {
+	else if (slope >= max_slope)
 		dydx = max_slope;
-	} else {
+	else
 		dydx = slope;
-	}
 
 	return dydx;
 }
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
index e9d6dd0bbfe28..cf6c291557588 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c
@@ -258,12 +258,11 @@ convert_allocate_dvs_6axis_config(
 	return me;
 }
 
-int
-store_dvs_6axis_config(
-    const struct ia_css_dvs_6axis_config *dvs_6axis_config,
-    const struct ia_css_binary *binary,
-    const struct ia_css_frame_info *dvs_in_frame_info,
-    ia_css_ptr ddr_addr_y) {
+int store_dvs_6axis_config(const struct ia_css_dvs_6axis_config *dvs_6axis_config,
+			   const struct ia_css_binary *binary,
+			   const struct ia_css_frame_info *dvs_in_frame_info,
+			   ia_css_ptr ddr_addr_y)
+{
 	struct ia_css_host_data *me;
 
 	assert(dvs_6axis_config);
@@ -274,8 +273,7 @@ store_dvs_6axis_config(
 					       binary,
 					       dvs_in_frame_info);
 
-	if (!me)
-	{
+	if (!me) {
 		IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM);
 		return -ENOMEM;
 	}
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c
index 8e4451fcc8e33..76d76ca4401a9 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c
@@ -140,17 +140,14 @@ ia_css_eed1_8_vmem_encode(
 	assert(tcinv_x[0] == 0);
 	assert(fcinv_x[0] == 0);
 
-	for (j = 1; j < NUMBER_OF_CHGRINV_POINTS; j++) {
+	for (j = 1; j < NUMBER_OF_CHGRINV_POINTS; j++)
 		assert(chgrinv_x[j] > chgrinv_x[j - 1]);
-	}
 
-	for (j = 1; j < NUMBER_OF_TCINV_POINTS; j++) {
+	for (j = 1; j < NUMBER_OF_TCINV_POINTS; j++)
 		assert(tcinv_x[j] > tcinv_x[j - 1]);
-	}
 
-	for (j = 1; j < NUMBER_OF_FCINV_POINTS; j++) {
+	for (j = 1; j < NUMBER_OF_FCINV_POINTS; j++)
 		assert(fcinv_x[j] > fcinv_x[j - 1]);
-	}
 
 	/* The implementation of the calculating 1/x is based on the availability
 	 * of the OP_vec_shuffle16 operation.
@@ -260,9 +257,9 @@ ia_css_eed1_8_encode(
 	to->margin_neg_diff = (from->neg_margin1 - from->neg_margin0);
 
 	/* Encode DEWEnhance exp (e_dew_enh_asr) */
-	for (i = 0; i < (IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS - 1); i++) {
+	for (i = 0; i < (IA_CSS_NUMBER_OF_DEW_ENHANCE_SEGMENTS - 1); i++)
 		min_exp = max(min_exp, from->dew_enhance_seg_exp[i]);
-	}
+
 	to->e_dew_enh_asr = 13 - clamp(min_exp, 0, 13);
 
 	to->dedgew_max = from->dedgew_max;
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
index a00f8d049a336..fb0e2a88cadb3 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c
@@ -21,10 +21,9 @@ static const struct ia_css_raw_configuration default_config = {
 };
 
 /* MW: These areMIPI / ISYS properties, not camera function properties */
-static enum sh_stream_format
-css2isp_stream_format(enum atomisp_input_format from) {
-	switch (from)
-	{
+static enum sh_stream_format css2isp_stream_format(enum atomisp_input_format from)
+{
+	switch (from) {
 	case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY:
 				return sh_stream_format_yuv420_legacy;
 	case ATOMISP_INPUT_FORMAT_YUV420_8:
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
index 13678138c48c4..823a1b0c09911 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c
@@ -240,9 +240,8 @@ ia_css_s3a_hmem_decode(
 	/* Calculate sum of histogram of R,
 	   which should not be less than count_for_3a */
 	sum_r = 0;
-	for (i = 0; i < HMEM_UNIT_SIZE; i++) {
+	for (i = 0; i < HMEM_UNIT_SIZE; i++)
 		sum_r += out_ptr[i].r;
-	}
 	if (sum_r < count_for_3a) {
 		/* histogram is invalid */
 		return;
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
index bd2def6c341a9..0ca7ef5bb0644 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c
@@ -50,9 +50,8 @@ void ia_css_sdis_horicoef_vmem_encode(
 	assert(size % (IA_CSS_DVS_NUM_COEF_TYPES * ISP_VEC_NELEMS * sizeof(
 			   short)) == 0);
 
-	for (type = 0; type < IA_CSS_DVS_NUM_COEF_TYPES; type++) {
+	for (type = 0; type < IA_CSS_DVS_NUM_COEF_TYPES; type++)
 		fill_row(&private[type * stride], &public[type * width], width, padding);
-	}
 }
 
 void ia_css_sdis_vertcoef_vmem_encode(
@@ -77,9 +76,8 @@ void ia_css_sdis_vertcoef_vmem_encode(
 	assert(size % (IA_CSS_DVS_NUM_COEF_TYPES * ISP_VEC_NELEMS * sizeof(
 			   short)) == 0);
 
-	for (type = 0; type < IA_CSS_DVS_NUM_COEF_TYPES; type++) {
+	for (type = 0; type < IA_CSS_DVS_NUM_COEF_TYPES; type++)
 		fill_row(&private[type * stride], &public[type * height], height, padding);
-	}
 }
 
 void ia_css_sdis_horiproj_encode(
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
index 4a38d678f3348..67860a5e44416 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c
@@ -164,10 +164,9 @@ void ia_css_sdis2_clear_coefficients(
 	dvs2_coefs->ver_coefs.even_imag = NULL;
 }
 
-int
-ia_css_get_dvs2_statistics(
-    struct ia_css_dvs2_statistics          *host_stats,
-    const struct ia_css_isp_dvs_statistics *isp_stats) {
+int ia_css_get_dvs2_statistics(struct ia_css_dvs2_statistics *host_stats,
+			       const struct ia_css_isp_dvs_statistics *isp_stats)
+{
 	struct ia_css_isp_dvs_statistics_map *map;
 	int ret = 0;
 
@@ -177,13 +176,11 @@ ia_css_get_dvs2_statistics(
 	assert(isp_stats);
 
 	map = ia_css_isp_dvs_statistics_map_allocate(isp_stats, NULL);
-	if (map)
-	{
+	if (map) {
 		hmm_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size);
 		ia_css_translate_dvs2_statistics(host_stats, map);
 		ia_css_isp_dvs_statistics_map_free(map);
-	} else
-	{
+	} else {
 		IA_CSS_ERROR("out of memory");
 		ret = -ENOMEM;
 	}
diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
index 3c675063c4a7e..152faab2b169e 100644
--- a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
+++ b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c
@@ -45,11 +45,10 @@ int ia_css_vf_config(struct sh_css_isp_vf_isp_config      *to,
  * to the requested viewfinder resolution on the upper side. The output cannot
  * be smaller than the requested viewfinder resolution.
  */
-int
-sh_css_vf_downscale_log2(
-    const struct ia_css_frame_info *out_info,
-    const struct ia_css_frame_info *vf_info,
-    unsigned int *downscale_log2) {
+int sh_css_vf_downscale_log2(const struct ia_css_frame_info *out_info,
+			     const struct ia_css_frame_info *vf_info,
+			     unsigned int *downscale_log2)
+{
 	unsigned int ds_log2 = 0;
 	unsigned int out_width;
 
@@ -65,8 +64,7 @@ sh_css_vf_downscale_log2(
 	* test for the height since the vmem buffers only put restrictions on
 	* the width of a line, not on the number of lines in a frame.
 	*/
-	while (out_width >= vf_info->res.width)
-	{
+	while (out_width >= vf_info->res.width) {
 		ds_log2++;
 		out_width /= 2;
 	}
@@ -80,13 +78,12 @@ sh_css_vf_downscale_log2(
 	return 0;
 }
 
-static int
-configure_kernel(
-    const struct ia_css_binary_info *info,
-    const struct ia_css_frame_info *out_info,
-    const struct ia_css_frame_info *vf_info,
-    unsigned int *downscale_log2,
-    struct ia_css_vf_configuration *config) {
+static int configure_kernel(const struct ia_css_binary_info *info,
+			    const struct ia_css_frame_info *out_info,
+			    const struct ia_css_frame_info *vf_info,
+			    unsigned int *downscale_log2,
+			    struct ia_css_vf_configuration *config)
+{
 	int err;
 	unsigned int vf_log_ds = 0;
 
diff --git a/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c b/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c
index af93ca96747c4..e9016d7775dc3 100644
--- a/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c
+++ b/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c
@@ -5,6 +5,7 @@
  */
 
 #include <linux/math.h>
+#include <linux/string_choices.h>
 
 #include <math_support.h>
 #include <gdc_device.h>	/* HR_GDC_N */
@@ -347,10 +348,10 @@ ia_css_binary_dvs_stat_grid_info(
 	return;
 }
 
-int
-ia_css_binary_3a_grid_info(const struct ia_css_binary *binary,
-			   struct ia_css_grid_info *info,
-			   struct ia_css_pipe *pipe) {
+int ia_css_binary_3a_grid_info(const struct ia_css_binary *binary,
+			       struct ia_css_grid_info *info,
+			       struct ia_css_pipe *pipe)
+{
 	struct ia_css_3a_grid_info *s3a_info;
 	int err = 0;
 
@@ -439,9 +440,9 @@ supports_bds_factor(u32 supported_factors,
 	return ((supported_factors & PACK_BDS_FACTOR(bds_factor)) != 0);
 }
 
-static int
-binary_init_info(struct ia_css_binary_xinfo *info, unsigned int i,
-		 bool *binary_found) {
+static int binary_init_info(struct ia_css_binary_xinfo *info, unsigned int i,
+			    bool *binary_found)
+{
 	const unsigned char *blob = sh_css_blob_info[i].blob;
 	unsigned int size = sh_css_blob_info[i].header.blob.size;
 
@@ -464,8 +465,8 @@ binary_init_info(struct ia_css_binary_xinfo *info, unsigned int i,
 /* When binaries are put at the beginning, they will only
  * be selected if no other primary matches.
  */
-int
-ia_css_binary_init_infos(void) {
+int ia_css_binary_init_infos(void)
+{
 	unsigned int i;
 	unsigned int num_of_isp_binaries = sh_css_num_binaries - NUM_OF_SPS - NUM_OF_BLS;
 
@@ -477,8 +478,7 @@ ia_css_binary_init_infos(void) {
 	if (!all_binaries)
 		return -ENOMEM;
 
-	for (i = 0; i < num_of_isp_binaries; i++)
-	{
+	for (i = 0; i < num_of_isp_binaries; i++) {
 		int ret;
 		struct ia_css_binary_xinfo *binary = &all_binaries[i];
 		bool binary_found;
@@ -497,13 +497,12 @@ ia_css_binary_init_infos(void) {
 	return 0;
 }
 
-int
-ia_css_binary_uninit(void) {
+int ia_css_binary_uninit(void)
+{
 	unsigned int i;
 	struct ia_css_binary_xinfo *b;
 
-	for (i = 0; i < IA_CSS_BINARY_NUM_MODES; i++)
-	{
+	for (i = 0; i < IA_CSS_BINARY_NUM_MODES; i++) {
 		for (b = binary_infos[i]; b; b = b->next) {
 			if (b->xmem_addr)
 				hmm_free(b->xmem_addr);
@@ -625,19 +624,19 @@ binary_in_frame_padded_width(int in_frame_width,
 	return rval;
 }
 
-int
-ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
-			bool online,
-			bool two_ppc,
-			enum atomisp_input_format stream_format,
-			const struct ia_css_frame_info *in_info, /* can be NULL */
-			const struct ia_css_frame_info *bds_out_info, /* can be NULL */
-			const struct ia_css_frame_info *out_info[], /* can be NULL */
-			const struct ia_css_frame_info *vf_info, /* can be NULL */
-			struct ia_css_binary *binary,
-			struct ia_css_resolution *dvs_env,
-			int stream_config_left_padding,
-			bool accelerator) {
+int ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
+			    bool online,
+			    bool two_ppc,
+			    enum atomisp_input_format stream_format,
+			    const struct ia_css_frame_info *in_info, /* can be NULL */
+			    const struct ia_css_frame_info *bds_out_info, /* can be NULL */
+			    const struct ia_css_frame_info *out_info[], /* can be NULL */
+			    const struct ia_css_frame_info *vf_info, /* can be NULL */
+			    struct ia_css_binary *binary,
+			    struct ia_css_resolution *dvs_env,
+			    int stream_config_left_padding,
+			    bool accelerator)
+{
 	const struct ia_css_binary_info *info = &xinfo->sp;
 	unsigned int dvs_env_width = 0,
 	dvs_env_height = 0,
@@ -664,25 +663,21 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 	assert(binary);
 
 	binary->info = xinfo;
-	if (!accelerator)
-	{
+	if (!accelerator) {
 		/* binary->css_params has been filled by accelerator itself. */
 		err = ia_css_isp_param_allocate_isp_parameters(
 		    &binary->mem_params, &binary->css_params,
 		    &info->mem_initializers);
-		if (err) {
+		if (err)
 			return err;
-		}
 	}
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-	{
+	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
 		if (out_info[i] && (out_info[i]->res.width != 0)) {
 			bin_out_info = out_info[i];
 			break;
 		}
 	}
-	if (in_info && bin_out_info)
-	{
+	if (in_info && bin_out_info) {
 		need_scaling = (in_info->res.width != bin_out_info->res.width) ||
 			       (in_info->res.height != bin_out_info->res.height);
 	}
@@ -713,8 +708,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 	binary->internal_frame_info.res.height      = isp_internal_height;
 	binary->internal_frame_info.raw_bit_depth   = bits_per_pixel;
 
-	if (in_info)
-	{
+	if (in_info) {
 		binary->effective_in_frame_res.width = in_info->res.width;
 		binary->effective_in_frame_res.height = in_info->res.height;
 
@@ -742,15 +736,13 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 		binary->in_frame_info.crop_info = in_info->crop_info;
 	}
 
-	if (online)
-	{
+	if (online) {
 		bits_per_pixel = ia_css_util_input_format_bpp(
 				     stream_format, two_ppc);
 	}
 	binary->in_frame_info.raw_bit_depth = bits_per_pixel;
 
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
-	{
+	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
 		if (out_info[i]) {
 			binary->out_frame_info[i].res.width     = out_info[i]->res.width;
 			binary->out_frame_info[i].res.height    = out_info[i]->res.height;
@@ -769,8 +761,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 		}
 	}
 
-	if (vf_info && (vf_info->res.width != 0))
-	{
+	if (vf_info && (vf_info->res.width != 0)) {
 		err = ia_css_vf_configure(binary, bin_out_info,
 					  (struct ia_css_frame_info *)vf_info, &vf_log_ds);
 		if (err) {
@@ -788,8 +779,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 	binary->input_format      = stream_format;
 
 	/* viewfinder output info */
-	if ((vf_info) && (vf_info->res.width != 0))
-	{
+	if ((vf_info) && (vf_info->res.width != 0)) {
 		unsigned int vf_out_vecs, vf_out_width, vf_out_height;
 
 		binary->vf_frame_info.format = vf_info->format;
@@ -821,23 +811,20 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 			binary->vf_frame_info.padded_width = vf_out_width;
 			binary->vf_frame_info.res.height   = vf_out_height;
 		}
-	} else
-	{
+	} else {
 		binary->vf_frame_info.res.width    = 0;
 		binary->vf_frame_info.padded_width = 0;
 		binary->vf_frame_info.res.height   = 0;
 	}
 
-	if (info->enable.ca_gdc)
-	{
+	if (info->enable.ca_gdc) {
 		binary->morph_tbl_width =
 		    _ISP_MORPH_TABLE_WIDTH(isp_internal_width);
 		binary->morph_tbl_aligned_width  =
 		    _ISP_MORPH_TABLE_ALIGNED_WIDTH(isp_internal_width);
 		binary->morph_tbl_height =
 		    _ISP_MORPH_TABLE_HEIGHT(isp_internal_height);
-	} else
-	{
+	} else {
 		binary->morph_tbl_width  = 0;
 		binary->morph_tbl_aligned_width  = 0;
 		binary->morph_tbl_height = 0;
@@ -847,8 +834,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 	sc_3a_dis_padded_width = binary->in_frame_info.padded_width;
 	sc_3a_dis_height = binary->in_frame_info.res.height;
 	if (bds_out_info && in_info &&
-	    bds_out_info->res.width != in_info->res.width)
-	{
+	    bds_out_info->res.width != in_info->res.width) {
 		/* TODO: Next, "internal_frame_info" should be derived from
 		 * bds_out. So this part will change once it is in place! */
 		sc_3a_dis_width = bds_out_info->res.width + info->pipeline.left_cropping;
@@ -858,18 +844,15 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 
 	s3a_isp_width = _ISP_S3A_ELEMS_ISP_WIDTH(sc_3a_dis_padded_width,
 			info->pipeline.left_cropping);
-	if (info->s3a.fixed_s3a_deci_log)
-	{
+	if (info->s3a.fixed_s3a_deci_log) {
 		s3a_log_deci = info->s3a.fixed_s3a_deci_log;
-	} else
-	{
+	} else {
 		s3a_log_deci = binary_grid_deci_factor_log2(s3a_isp_width,
 			       sc_3a_dis_height);
 	}
 	binary->deci_factor_log2  = s3a_log_deci;
 
-	if (info->enable.s3a)
-	{
+	if (info->enable.s3a) {
 		binary->s3atbl_width  =
 		    _ISP_S3ATBL_WIDTH(sc_3a_dis_width,
 				      s3a_log_deci);
@@ -882,21 +865,18 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo,
 		binary->s3atbl_isp_height =
 		    _ISP_S3ATBL_ISP_HEIGHT(sc_3a_dis_height,
 					   s3a_log_deci);
-	} else
-	{
+	} else {
 		binary->s3atbl_width  = 0;
 		binary->s3atbl_height = 0;
 		binary->s3atbl_isp_width  = 0;
 		binary->s3atbl_isp_height = 0;
 	}
 
-	if (info->enable.sc)
-	{
+	if (info->enable.sc) {
 		binary->sctbl_width_per_color = _ISP_SCTBL_WIDTH_PER_COLOR(sc_3a_dis_padded_width, s3a_log_deci);
 		binary->sctbl_aligned_width_per_color = SH_CSS_MAX_SCTBL_ALIGNED_WIDTH_PER_COLOR;
 		binary->sctbl_height = _ISP_SCTBL_HEIGHT(sc_3a_dis_height, s3a_log_deci);
-	} else
-	{
+	} else {
 		binary->sctbl_width_per_color         = 0;
 		binary->sctbl_aligned_width_per_color = 0;
 		binary->sctbl_height                  = 0;
@@ -1241,7 +1221,7 @@ int ia_css_binary_find(struct ia_css_binary_descr *descr, struct ia_css_binary *
 		dev_dbg(atomisp_dev, "Using binary %s (id %d), type %d, mode %d, continuous %s\n",
 			xcandidate->blob->name, xcandidate->sp.id, xcandidate->type,
 			xcandidate->sp.pipeline.mode,
-			xcandidate->sp.enable.continuous ? "true" : "false");
+			str_true_false(xcandidate->sp.enable.continuous));
 
 	if (err)
 		dev_err(atomisp_dev, "Failed to find a firmware binary matching the pipeline parameters\n");
diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c b/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c
index b411ca2f415e0..5113aa5973f3a 100644
--- a/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c
+++ b/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c
@@ -1123,9 +1123,8 @@ ia_css_debug_pipe_graph_dump_prologue(void)
 
 void ia_css_debug_pipe_graph_dump_epilogue(void)
 {
-	if (strlen(ring_buffer) > 0) {
+	if (strlen(ring_buffer) > 0)
 		dtrace_dot(ring_buffer);
-	}
 
 	if (pg_inst.stream_format != N_ATOMISP_INPUT_FORMAT) {
 		/* An input stream format has been set so assume we have
@@ -1256,8 +1255,7 @@ ia_css_debug_pipe_graph_dump_stage(
 				while (ei[p] != ',')
 					p--;
 				/* Last comma found, copy till that comma */
-				strscpy(enable_info1, ei,
-                                        p > sizeof(enable_info1) ? sizeof(enable_info1) : p);
+				strscpy(enable_info1, ei, umin(p, sizeof(enable_info1)));
 
 				ei += p + 1;
 				l = strlen(ei);
@@ -1268,8 +1266,7 @@ ia_css_debug_pipe_graph_dump_stage(
 					 * it is not guaranteed dword aligned
 					 */
 
-					strscpy(enable_info2, ei,
-						l > sizeof(enable_info2) ? sizeof(enable_info2) : l);
+					strscpy(enable_info2, ei, umin(l, sizeof(enable_info2)));
 
 					snprintf(enable_info, sizeof(enable_info), "%s\\n%s",
 						 enable_info1, enable_info2);
@@ -1280,8 +1277,7 @@ ia_css_debug_pipe_graph_dump_stage(
 					while (ei[p] != ',')
 						p--;
 
-					strscpy(enable_info2, ei,
-						p > sizeof(enable_info2) ? sizeof(enable_info2) : p);
+					strscpy(enable_info2, ei, umin(p, sizeof(enable_info2)));
 
 					ei += p + 1;
 					l = strlen(ei);
@@ -1303,7 +1299,7 @@ ia_css_debug_pipe_graph_dump_stage(
 						while (ei[p] != ',')
 							p--;
 						strscpy(enable_info3, ei,
-							p > sizeof(enable_info3) ? sizeof(enable_info3) : p);
+							umin(p, sizeof(enable_info3)));
 						ei += p + 1;
 						strscpy(enable_info3, ei,
 							sizeof(enable_info3));
@@ -1778,9 +1774,8 @@ static void debug_dump_one_trace(enum TRACE_CORE_ID proc_id)
 				* When tid value is 111b, the data will be interpreted differently:
 				* tid val is ignored, major field contains 2 bits (msb) for format type
 				*/
-				if (tid_val == FIELD_TID_SEL_FORMAT_PAT) {
+				if (tid_val == FIELD_TID_SEL_FORMAT_PAT)
 					dump_format = FIELD_FORMAT_UNPACK(trace_read_buf[j]);
-				}
 			}
 			switch (dump_format) {
 			case TRACE_DUMP_FORMAT_POINT:
diff --git a/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c b/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c
index 251dd75a76135..1d20eb6507570 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c
@@ -93,17 +93,15 @@ ia_css_init_memory_interface(
 	}
 }
 
-int
-ia_css_isp_param_allocate_isp_parameters(
-    struct ia_css_isp_param_host_segments *mem_params,
-    struct ia_css_isp_param_css_segments *css_params,
-    const struct ia_css_isp_param_isp_segments *mem_initializers) {
+int ia_css_isp_param_allocate_isp_parameters(struct ia_css_isp_param_host_segments *mem_params,
+					     struct ia_css_isp_param_css_segments *css_params,
+					     const struct ia_css_isp_param_isp_segments *mem_initializers)
+{
 	int err = 0;
 	unsigned int mem, pclass;
 
 	pclass = IA_CSS_PARAM_CLASS_PARAM;
-	for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++)
-	{
+	for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++) {
 		for (pclass = 0; pclass < IA_CSS_NUM_PARAM_CLASSES; pclass++) {
 			u32 size = 0;
 
@@ -171,15 +169,13 @@ ia_css_isp_param_load_fw_params(
 	}
 }
 
-int
-ia_css_isp_param_copy_isp_mem_if_to_ddr(
-    struct ia_css_isp_param_css_segments *ddr,
-    const struct ia_css_isp_param_host_segments *host,
-    enum ia_css_param_class pclass) {
+int ia_css_isp_param_copy_isp_mem_if_to_ddr(struct ia_css_isp_param_css_segments *ddr,
+					    const struct ia_css_isp_param_host_segments *host,
+					    enum ia_css_param_class pclass)
+{
 	unsigned int mem;
 
-	for (mem = 0; mem < N_IA_CSS_ISP_MEMORIES; mem++)
-	{
+	for (mem = 0; mem < N_IA_CSS_ISP_MEMORIES; mem++) {
 		size_t       size	  = host->params[pclass][mem].size;
 		ia_css_ptr ddr_mem_ptr  = ddr->params[pclass][mem].address;
 		char	    *host_mem_ptr = host->params[pclass][mem].address;
diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c
index e6c11d5f77b41..8e8433f5b07ac 100644
--- a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c
+++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c
@@ -295,9 +295,8 @@ static bool create_input_system_channel(
 	if (!rc)
 		return false;
 
-	if (!acquire_sid(me->stream2mmio_id, &me->stream2mmio_sid_id)) {
+	if (!acquire_sid(me->stream2mmio_id, &me->stream2mmio_sid_id))
 		return false;
-	}
 
 	if (!acquire_ib_buffer(
 		metadata ? cfg->metadata.bits_per_pixel :
diff --git a/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c b/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c
index 0470871f8fff2..8d11466fda1bd 100644
--- a/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c
+++ b/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c
@@ -575,9 +575,8 @@ static int pipeline_stage_create(
 	binary = stage_desc->binary;
 	firmware = stage_desc->firmware;
 	vf_frame = stage_desc->vf_frame;
-	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) {
+	for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++)
 		out_frame[i] = stage_desc->out_frame[i];
-	}
 
 	stage = kvzalloc_obj(*stage);
 	if (!stage) {
diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c
index afe77d4373f8d..d27c6567daeb0 100644
--- a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c
+++ b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c
@@ -167,7 +167,7 @@ int ia_css_queue_dequeue(ia_css_queue_t *qhandle, uint32_t *item)
 
 		*item = cb_elem.val;
 
-		cb_desc.start = OP_std_modadd(cb_desc.start, 1, cb_desc.size);
+		cb_desc.start = (cb_desc.start + 1) % cb_desc.size;
 
 		/* c. Store the queue object */
 		/* Set only fields requiring update with
@@ -315,7 +315,7 @@ int ia_css_queue_peek(ia_css_queue_t *qhandle, u32 offset, uint32_t *element)
 		if (offset > num_elems)
 			return -EINVAL;
 
-		offset = OP_std_modadd(cb_desc.start, offset, cb_desc.size);
+		offset = (cb_desc.start + offset) % cb_desc.size;
 		error = ia_css_queue_item_load(qhandle, (uint8_t)offset, &cb_elem);
 		if (error != 0)
 			return error;
diff --git a/drivers/staging/media/atomisp/pci/sh_css.c b/drivers/staging/media/atomisp/pci/sh_css.c
index 6cda5925fa45e..00082276f1db4 100644
--- a/drivers/staging/media/atomisp/pci/sh_css.c
+++ b/drivers/staging/media/atomisp/pci/sh_css.c
@@ -7,6 +7,7 @@
 /*! \file */
 #include <linux/mm.h>
 #include <linux/slab.h>
+#include <linux/string_choices.h>
 #include <linux/vmalloc.h>
 
 #include "hmm.h"
@@ -1478,7 +1479,7 @@ map_sp_threads(struct ia_css_stream *stream, bool map)
 	enum ia_css_pipe_id pipe_id;
 
 	IA_CSS_ENTER_PRIVATE("stream = %p, map = %s",
-			     stream, map ? "true" : "false");
+			     stream, str_true_false(map));
 
 	if (!stream) {
 		IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL);
@@ -2256,8 +2257,7 @@ alloc_continuous_frames(struct ia_css_pipe *pipe, bool init_time)
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 				    "alloc_continuous_frames() IA_CSS_FRAME_FORMAT_RAW_PACKED\n");
 		ref_info.format = IA_CSS_FRAME_FORMAT_RAW_PACKED;
-	} else
-	{
+	} else {
 		ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE,
 				    "alloc_continuous_frames() IA_CSS_FRAME_FORMAT_RAW\n");
 		ref_info.format = IA_CSS_FRAME_FORMAT_RAW;
@@ -5020,7 +5020,6 @@ static int load_primary_binaries(
 	struct ia_css_capture_settings *mycs;
 	unsigned int i;
 	bool need_extra_yuv_scaler = false;
-	struct ia_css_binary_descr prim_descr[MAX_NUM_PRIMARY_STAGES];
 
 	IA_CSS_ENTER_PRIVATE("");
 	assert(pipe);
@@ -5189,15 +5188,16 @@ static int load_primary_binaries(
 
 	/* Primary */
 	for (i = 0; i < mycs->num_primary_stage; i++) {
+		struct ia_css_binary_descr prim_descr;
 		struct ia_css_frame_info *local_vf_info = NULL;
 
 		if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0] &&
 		    (i == mycs->num_primary_stage - 1))
 			local_vf_info = &vf_info;
-		ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr[i],
+		ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr,
 						   &prim_in_info, &prim_out_info,
 						   local_vf_info, i);
-		err = ia_css_binary_find(&prim_descr[i], &mycs->primary_binary[i]);
+		err = ia_css_binary_find(&prim_descr, &mycs->primary_binary[i]);
 		if (err) {
 			IA_CSS_LEAVE_ERR_PRIVATE(err);
 			return err;
@@ -5819,36 +5819,37 @@ static int ia_css_pipe_create_cas_scaler_desc_single_output(
 		i *= max_scale_factor_per_stage;
 	}
 
-	descr->in_info = kmalloc(descr->num_stage *
-				 sizeof(struct ia_css_frame_info),
-				 GFP_KERNEL);
+	descr->in_info = kmalloc_objs(*descr->in_info,
+				      descr->num_stage,
+				      GFP_KERNEL);
 	if (!descr->in_info) {
 		err = -ENOMEM;
 		goto ERR;
 	}
-	descr->internal_out_info = kmalloc(descr->num_stage *
-					   sizeof(struct ia_css_frame_info),
-					   GFP_KERNEL);
+	descr->internal_out_info = kmalloc_objs(*descr->internal_out_info,
+						descr->num_stage,
+						GFP_KERNEL);
 	if (!descr->internal_out_info) {
 		err = -ENOMEM;
 		goto ERR;
 	}
-	descr->out_info = kmalloc(descr->num_stage *
-				  sizeof(struct ia_css_frame_info),
-				  GFP_KERNEL);
+	descr->out_info = kmalloc_objs(*descr->out_info,
+				       descr->num_stage,
+				       GFP_KERNEL);
 	if (!descr->out_info) {
 		err = -ENOMEM;
 		goto ERR;
 	}
-	descr->vf_info = kmalloc(descr->num_stage *
-				 sizeof(struct ia_css_frame_info),
-				 GFP_KERNEL);
+	descr->vf_info = kmalloc_objs(*descr->vf_info,
+				      descr->num_stage,
+				      GFP_KERNEL);
 	if (!descr->vf_info) {
 		err = -ENOMEM;
 		goto ERR;
 	}
-	descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool),
-					 GFP_KERNEL);
+	descr->is_output_stage = kmalloc_objs(*descr->is_output_stage,
+					      descr->num_stage,
+					      GFP_KERNEL);
 	if (!descr->is_output_stage) {
 		err = -ENOMEM;
 		goto ERR;
@@ -5968,35 +5969,37 @@ ia_css_pipe_create_cas_scaler_desc(struct ia_css_pipe *pipe,
 
 	descr->num_stage = num_stages;
 
-	descr->in_info = kmalloc_objs(struct ia_css_frame_info,
-				      descr->num_stage);
+	descr->in_info = kmalloc_objs(*descr->in_info,
+				      descr->num_stage,
+				      GFP_KERNEL);
 	if (!descr->in_info) {
 		err = -ENOMEM;
 		goto ERR;
 	}
-	descr->internal_out_info = kmalloc(descr->num_stage *
-					   sizeof(struct ia_css_frame_info),
-					   GFP_KERNEL);
+	descr->internal_out_info = kmalloc_objs(*descr->internal_out_info,
+						descr->num_stage,
+						GFP_KERNEL);
 	if (!descr->internal_out_info) {
 		err = -ENOMEM;
 		goto ERR;
 	}
-	descr->out_info = kmalloc(descr->num_stage *
-				  sizeof(struct ia_css_frame_info),
-				  GFP_KERNEL);
+	descr->out_info = kmalloc_objs(*descr->out_info,
+				       descr->num_stage,
+				       GFP_KERNEL);
 	if (!descr->out_info) {
 		err = -ENOMEM;
 		goto ERR;
 	}
-	descr->vf_info = kmalloc(descr->num_stage *
-				 sizeof(struct ia_css_frame_info),
-				 GFP_KERNEL);
+	descr->vf_info = kmalloc_objs(*descr->vf_info,
+				      descr->num_stage,
+				      GFP_KERNEL);
 	if (!descr->vf_info) {
 		err = -ENOMEM;
 		goto ERR;
 	}
-	descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool),
-					 GFP_KERNEL);
+	descr->is_output_stage = kmalloc_objs(*descr->is_output_stage,
+					      descr->num_stage,
+					      GFP_KERNEL);
 	if (!descr->is_output_stage) {
 		err = -ENOMEM;
 		goto ERR;
@@ -7856,8 +7859,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 
 	/* check if mipi size specified */
 	if (stream_config->mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR)
-		if (!IS_ISP2401 || !stream_config->online)
-		{
+		if (!IS_ISP2401 || !stream_config->online) {
 			unsigned int port = (unsigned int)stream_config->source.port.port;
 
 			if (port >= N_MIPI_PORT_ID) {
@@ -8189,6 +8191,48 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config,
 	return err;
 }
 
+static void ia_css_stream_destroy_isp2401(struct ia_css_stream *stream)
+{
+	int i, j;
+
+	for (i = 0; i < stream->num_pipes; i++) {
+		struct ia_css_pipe *entry = stream->pipes[i];
+		unsigned int sp_thread_id;
+		struct sh_css_sp_pipeline_terminal *terminal;
+
+		if (!entry)
+			continue;
+
+		if (!ia_css_pipeline_get_sp_thread_id(
+				ia_css_pipe_get_pipe_num(entry), &sp_thread_id))
+			continue;
+
+		terminal = &sh_css_sp_group.pipe_io[sp_thread_id].input;
+
+		for (j = 0; j < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; j++) {
+			ia_css_isys_stream_h isys_stream =
+				&terminal->context.virtual_input_system_stream[j];
+			if (stream->config.isys_config[j].valid && isys_stream->valid)
+				ia_css_isys_stream_destroy(isys_stream);
+		}
+	}
+
+	if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
+		for (i = 0; i < stream->num_pipes; i++) {
+			/*
+			 * free any mipi frames that are remaining:
+			 * some test stream create-destroy cycles do
+			 * not generate output frames
+			 * and the mipi buffer is not freed in the
+			 * deque function
+			 */
+			if (stream->pipes[i])
+				free_mipi_frames(stream->pipes[i]);
+		}
+	}
+	stream_unregister_with_csi_rx(stream);
+}
+
 int
 ia_css_stream_destroy(struct ia_css_stream *stream)
 {
@@ -8206,48 +8250,8 @@ ia_css_stream_destroy(struct ia_css_stream *stream)
 
 	if ((stream->last_pipe) &&
 	    ia_css_pipeline_is_mapped(stream->last_pipe->pipe_num)) {
-		if (IS_ISP2401) {
-			for (i = 0; i < stream->num_pipes; i++) {
-				struct ia_css_pipe *entry = stream->pipes[i];
-				unsigned int sp_thread_id;
-				struct sh_css_sp_pipeline_terminal *sp_pipeline_input_terminal;
-
-				assert(entry);
-				if (entry) {
-					/* get the SP thread id */
-					if (!ia_css_pipeline_get_sp_thread_id(
-							ia_css_pipe_get_pipe_num(entry), &sp_thread_id))
-						return -EINVAL;
-
-					/* get the target input terminal */
-					sp_pipeline_input_terminal =
-						&sh_css_sp_group.pipe_io[sp_thread_id].input;
-
-					for (i = 0; i < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; i++) {
-						ia_css_isys_stream_h isys_stream =
-							&sp_pipeline_input_terminal->context.virtual_input_system_stream[i];
-						if (stream->config.isys_config[i].valid && isys_stream->valid)
-							ia_css_isys_stream_destroy(isys_stream);
-					}
-				}
-			}
-
-			if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) {
-				for (i = 0; i < stream->num_pipes; i++) {
-					struct ia_css_pipe *entry = stream->pipes[i];
-					/*
-					 * free any mipi frames that are remaining:
-					 * some test stream create-destroy cycles do
-					 * not generate output frames
-					 * and the mipi buffer is not freed in the
-					 * deque function
-					 */
-					if (entry)
-						free_mipi_frames(entry);
-				}
-			}
-			stream_unregister_with_csi_rx(stream);
-		}
+		if (IS_ISP2401)
+			ia_css_stream_destroy_isp2401(stream);
 
 		for (i = 0; i < stream->num_pipes; i++) {
 			struct ia_css_pipe *curr_pipe = stream->pipes[i];
diff --git a/drivers/staging/media/atomisp/pci/sh_css_firmware.c b/drivers/staging/media/atomisp/pci/sh_css_firmware.c
index 57ecf5549c23d..af12df2f9b090 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_firmware.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_firmware.c
@@ -253,9 +253,9 @@ sh_css_load_firmware(struct device *dev, const char *fw_data,
 	sh_css_num_binaries = file_header->binary_nr;
 	/* Only allocate memory for ISP blob info */
 	if (sh_css_num_binaries > NUM_OF_SPS) {
-		sh_css_blob_info = kmalloc(
-		    (sh_css_num_binaries - NUM_OF_SPS) *
-		    sizeof(*sh_css_blob_info), GFP_KERNEL);
+		sh_css_blob_info =
+			kmalloc_array(sh_css_num_binaries - NUM_OF_SPS,
+				      sizeof(*sh_css_blob_info), GFP_KERNEL);
 		if (!sh_css_blob_info)
 			return -ENOMEM;
 	} else {
diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
index 9ccdb66de2df5..ad2a9b84e2321 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2015, Intel Corporation.
  */
 
+#include <linux/overflow.h>
 #include "sh_css_param_dvs.h"
 #include <assert_support.h>
 #include <type_support.h>
@@ -48,7 +49,7 @@ alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res,
 		}
 
 		/* Generate Y buffers  */
-		dvs_config->xcoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+		dvs_config->xcoords_y = kvmalloc(array3_size(width_y, height_y, sizeof(uint32_t)),
 						 GFP_KERNEL);
 		if (!dvs_config->xcoords_y) {
 			IA_CSS_ERROR("out of memory");
@@ -56,7 +57,7 @@ alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res,
 			goto exit;
 		}
 
-		dvs_config->ycoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+		dvs_config->ycoords_y = kvmalloc(array3_size(width_y, height_y, sizeof(uint32_t)),
 						 GFP_KERNEL);
 		if (!dvs_config->ycoords_y) {
 			IA_CSS_ERROR("out of memory");
@@ -67,7 +68,8 @@ alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res,
 		/* Generate UV buffers  */
 		IA_CSS_LOG("UV W %d H %d", width_uv, height_uv);
 
-		dvs_config->xcoords_uv = kvmalloc(width_uv * height_uv * sizeof(uint32_t),
+		dvs_config->xcoords_uv = kvmalloc(array3_size(width_uv, height_uv,
+							      sizeof(uint32_t)),
 						  GFP_KERNEL);
 		if (!dvs_config->xcoords_uv) {
 			IA_CSS_ERROR("out of memory");
@@ -75,7 +77,8 @@ alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res,
 			goto exit;
 		}
 
-		dvs_config->ycoords_uv = kvmalloc(width_uv * height_uv * sizeof(uint32_t),
+		dvs_config->ycoords_uv = kvmalloc(array3_size(width_uv, height_uv,
+							      sizeof(uint32_t)),
 						  GFP_KERNEL);
 		if (!dvs_config->ycoords_uv) {
 			IA_CSS_ERROR("out of memory");
@@ -269,5 +272,4 @@ ia_css_dvs_statistics_get(enum dvs_statistics_type type,
 		ia_css_get_dvs2_statistics(host_stats->p_dvs2_statistics_host,
 					   isp_stats->p_dvs_statistics_isp);
 	}
-	return;
 }
diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_shading.c b/drivers/staging/media/atomisp/pci/sh_css_param_shading.c
index 9105334c71b1f..0ac85cdea0102 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_param_shading.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_param_shading.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2015, Intel Corporation.
  */
 
+#include <linux/overflow.h>
 #include <linux/math.h>
 #include <linux/slab.h>
 
@@ -20,7 +21,8 @@
 
 #include "platform_support.h"
 
-/* Bilinear interpolation on shading tables:
+/*
+ * Bilinear interpolation on shading tables:
  * For each target point T, we calculate the 4 surrounding source points:
  * ul (upper left), ur (upper right), ll (lower left) and lr (lower right).
  * We then calculate the distances from the T to the source points: x0, x1,
@@ -116,8 +118,10 @@ crop_and_interpolate(unsigned int cropped_width,
 		 */
 		ty = out_start_row + i * out_cell_size;
 
-		/* calculate closest source points in shading table and
-		   make sure they fall within the table */
+		/*
+		 * calculate closest source points in shading table and
+		 * make sure they fall within the table
+		 */
 		src_y0 = ty / (int)in_cell_size;
 		if (in_cell_size < out_cell_size)
 			src_y1 = (ty + out_cell_size) / in_cell_size;
@@ -173,7 +177,8 @@ crop_and_interpolate(unsigned int cropped_width,
 			dx0 = tx - sx0;
 			dx1 = sx1 - tx;
 			divx = sx1 - sx0;
-			/* if we're at the edge, we just use the closest
+			/*
+			 * if we're at the edge, we just use the closest
 			 * point still in the grid. We make up for the divider
 			 * in this case by setting the distance to
 			 * out_cell_size, since it's actually 0.
@@ -291,8 +296,10 @@ prepare_shading_table(const struct ia_css_shading_table *in_table,
 	input_width  = min(input_width,  in_table->sensor_width);
 	input_height = min(input_height, in_table->sensor_height);
 
-	/* This prepare_shading_table() function is called only in legacy API (not in new API).
-	   Then, the legacy shading table width and height should be used. */
+	/*
+	 * This prepare_shading_table() function is called only in legacy API (not in new API).
+	 * Then, the legacy shading table width and height should be used.
+	 */
 	table_width  = binary->sctbl_width_per_color;
 	table_height = binary->sctbl_height;
 
@@ -339,7 +346,8 @@ ia_css_shading_table_alloc(
 	me->fraction_bits = 0;
 	for (i = 0; i < IA_CSS_SC_NUM_COLORS; i++) {
 		me->data[i] =
-		    kvmalloc(width * height * sizeof(*me->data[0]),
+		    kvmalloc(array3_size(width, height,
+					 sizeof(*me->data[0])),
 			     GFP_KERNEL);
 		if (!me->data[i]) {
 			unsigned int j;
diff --git a/drivers/staging/media/atomisp/pci/sh_css_params.c b/drivers/staging/media/atomisp/pci/sh_css_params.c
index fcebace11daf6..8420a22fd8f0b 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_params.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_params.c
@@ -4,6 +4,7 @@
  * Copyright (c) 2015, Intel Corporation.
  */
 
+#include <linux/overflow.h>
 #include <linux/math.h>
 
 #include "gdc_device.h"		/* gdc_lut_store(), ... */
@@ -657,11 +658,7 @@ static const int zoom_table[4][HRT_GDC_N] = {
 static const struct ia_css_dz_config default_dz_config = {
 	HRT_GDC_N,
 	HRT_GDC_N,
-	{
-		\
-		{0, 0}, \
-		{0, 0}, \
-	}
+	{ 0, 0, 0, 0 }
 };
 
 static const struct ia_css_vector default_motion_config = {
@@ -875,7 +872,8 @@ ia_css_process_kernel(struct ia_css_stream *stream,
 
 		/* update the other buffers to the pipe specific copies */
 		for (stage = pipeline->stages; stage; stage = stage->next) {
-			if (!stage || !stage->binary) continue;
+			if (!stage || !stage->binary)
+				continue;
 			process(pipeline->pipe_id, stage, params);
 		}
 	}
@@ -1210,8 +1208,8 @@ ia_css_process_zoom_and_motion(
 		}
 
 		assert(stage->stage_num < SH_CSS_MAX_STAGES);
-		if (params->dz_config.zoom_region.resolution.width == 0 &&
-		    params->dz_config.zoom_region.resolution.height == 0) {
+		if (params->dz_config.zoom_region.width == 0 &&
+		    params->dz_config.zoom_region.height == 0) {
 			sh_css_update_uds_and_crop_info(
 			    &info->sp,
 			    &binary->in_frame_info,
@@ -1381,11 +1379,11 @@ struct ia_css_morph_table *ia_css_morph_table_allocate(
 	}
 
 	for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) {
-		me->coordinates_x[i] = kvmalloc(height * width *
-						sizeof(*me->coordinates_x[i]),
+		me->coordinates_x[i] = kvmalloc(array3_size(height, width,
+							    sizeof(*me->coordinates_x[i])),
 						GFP_KERNEL);
-		me->coordinates_y[i] = kvmalloc(height * width *
-						sizeof(*me->coordinates_y[i]),
+		me->coordinates_y[i] = kvmalloc(array3_size(height, width,
+							    sizeof(*me->coordinates_y[i])),
 						GFP_KERNEL);
 
 		if ((!me->coordinates_x[i]) ||
@@ -1928,9 +1926,8 @@ sh_css_set_per_frame_isp_config_on_pipe(
 	params = stream->per_frame_isp_params_configs;
 
 	/* update new ISP params object with the new config */
-	if (!sh_css_init_isp_params_from_global(stream, params, false, pipe)) {
+	if (!sh_css_init_isp_params_from_global(stream, params, false, pipe))
 		err1 = -EINVAL;
-	}
 
 	err2 = sh_css_init_isp_params_from_config(stream->pipes[0], params, config, pipe);
 
@@ -2004,9 +2001,8 @@ sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe,
 		 * user. */
 		/* we do not exit from this point immediately to allow internal
 		 * firmware feature testing. */
-		if (is_dp_10bpp) {
+		if (is_dp_10bpp)
 			err = -EINVAL;
-		}
 	} else {
 		err = -EINVAL;
 		goto exit;
@@ -3034,9 +3030,8 @@ process_kernel_parameters(unsigned int pipe_id,
 		ia_css_ob_configure(&params->stream_configs.ob,
 				    isp_pipe_version, raw_bit_depth);
 	}
-	if (params->config_changed[IA_CSS_S3A_ID]) {
+	if (params->config_changed[IA_CSS_S3A_ID])
 		ia_css_s3a_configure(raw_bit_depth);
-	}
 	/* Copy stage uds parameters to config, since they can differ per stage.
 	 */
 	params->crop_config.crop_pos = params->uds[stage->stage_num].crop_pos;
@@ -3045,7 +3040,8 @@ process_kernel_parameters(unsigned int pipe_id,
 	/* Call parameter process functions for all kernels */
 	/* Skip SC, since that is called on a temp sc table */
 	for (param_id = 0; param_id < IA_CSS_NUM_PARAMETER_IDS; param_id++) {
-		if (param_id == IA_CSS_SC_ID) continue;
+		if (param_id == IA_CSS_SC_ID)
+			continue;
 		if (params->config_changed[param_id])
 			ia_css_kernel_process_param[param_id](pipe_id, stage, params);
 	}
@@ -3600,7 +3596,8 @@ sh_css_params_write_to_ddr_internal(
 						    IA_CSS_PARAM_CLASS_PARAM, mem);
 		size_t size = isp_data->size;
 
-		if (!size) continue;
+		if (!size)
+			continue;
 		buff_realloced = reallocate_buffer(&ddr_map->isp_mem_param[stage_num][mem],
 						&ddr_map_size->isp_mem_param[stage_num][mem],
 						size,
@@ -3899,9 +3896,8 @@ sh_css_invalidate_params(struct ia_css_stream *stream)
 	params->isp_params_changed = true;
 	for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) {
 		for (j = 0; j < SH_CSS_MAX_STAGES; j++) {
-			for (mem = 0; mem < N_IA_CSS_MEMORIES; mem++) {
+			for (mem = 0; mem < N_IA_CSS_MEMORIES; mem++)
 				params->isp_mem_params_changed[i][j][mem] = true;
-			}
 		}
 	}
 
@@ -4096,10 +4092,10 @@ sh_css_update_uds_and_crop_info_based_on_zoom_region(
 	assert(motion_vector);
 	assert(uds);
 	assert(sp_out_crop_pos);
-	x0 = zoom->zoom_region.origin.x;
-	y0 = zoom->zoom_region.origin.y;
-	x1 = zoom->zoom_region.resolution.width + x0;
-	y1 = zoom->zoom_region.resolution.height + y0;
+	x0 = zoom->zoom_region.left;
+	y0 = zoom->zoom_region.top;
+	x1 = zoom->zoom_region.width + x0;
+	y1 = zoom->zoom_region.height + y0;
 
 	if ((x0 > x1) || (y0 > y1) || (x1 > pipe_in_res.width) || (y1 > pipe_in_res.height))
 		return -EINVAL;
@@ -4206,13 +4202,17 @@ ia_css_dvs_statistics_allocate(const struct ia_css_dvs_grid_info *grid)
 		goto err;
 
 	me->grid = *grid;
-	me->hor_proj = kvmalloc(grid->height * IA_CSS_DVS_NUM_COEF_TYPES *
-				sizeof(*me->hor_proj), GFP_KERNEL);
+	me->hor_proj = kvmalloc(array3_size(grid->height,
+					    IA_CSS_DVS_NUM_COEF_TYPES,
+						sizeof(*me->hor_proj)),
+				GFP_KERNEL);
 	if (!me->hor_proj)
 		goto err;
 
-	me->ver_proj = kvmalloc(grid->width * IA_CSS_DVS_NUM_COEF_TYPES *
-				sizeof(*me->ver_proj), GFP_KERNEL);
+	me->ver_proj = kvmalloc(array3_size(grid->width,
+					    IA_CSS_DVS_NUM_COEF_TYPES,
+						sizeof(*me->ver_proj)),
+				GFP_KERNEL);
 	if (!me->ver_proj)
 		goto err;
 
@@ -4478,24 +4478,26 @@ ia_css_dvs2_6axis_config_allocate(const struct ia_css_stream *stream)
 				    params->pipe_dvs_6axis_config[IA_CSS_PIPE_ID_VIDEO]->height_uv;
 	IA_CSS_LOG("table Y: W %d H %d", width_y, height_y);
 	IA_CSS_LOG("table UV: W %d H %d", width_uv, height_uv);
-	dvs_config->xcoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+	dvs_config->xcoords_y = kvmalloc(array3_size(width_y, height_y,
+						     sizeof(uint32_t)),
 					 GFP_KERNEL);
 	if (!dvs_config->xcoords_y)
 		goto err;
 
-	dvs_config->ycoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t),
+	dvs_config->ycoords_y = kvmalloc(array3_size(width_y, height_y,
+						     sizeof(uint32_t)),
 					 GFP_KERNEL);
 	if (!dvs_config->ycoords_y)
 		goto err;
 
-	dvs_config->xcoords_uv = kvmalloc(width_uv * height_uv *
-					  sizeof(uint32_t),
+	dvs_config->xcoords_uv = kvmalloc(array3_size(width_uv, height_uv,
+						      sizeof(uint32_t)),
 					  GFP_KERNEL);
 	if (!dvs_config->xcoords_uv)
 		goto err;
 
-	dvs_config->ycoords_uv = kvmalloc(width_uv * height_uv *
-					  sizeof(uint32_t),
+	dvs_config->ycoords_uv = kvmalloc(array3_size(width_uv, height_uv,
+						      sizeof(uint32_t)),
 					  GFP_KERNEL);
 	if (!dvs_config->ycoords_uv)
 		goto err;
diff --git a/drivers/staging/media/atomisp/pci/sh_css_sp.c b/drivers/staging/media/atomisp/pci/sh_css_sp.c
index 6da151e7a8732..2beb7168517fa 100644
--- a/drivers/staging/media/atomisp/pci/sh_css_sp.c
+++ b/drivers/staging/media/atomisp/pci/sh_css_sp.c
@@ -775,9 +775,13 @@ static int configure_isp_from_args(const struct sh_css_sp_pipeline *pipeline,
 	ret = ia_css_fpn_configure(binary,  &binary->in_frame_info);
 	if (ret)
 		return ret;
-	ret = ia_css_crop_configure(binary, ia_css_frame_get_info(args->delay_frames[0]));
-	if (ret)
-		return ret;
+
+	if (binary->info->sp.enable.ref_frame) {
+		ret = ia_css_crop_configure(binary, ia_css_frame_get_info(args->delay_frames[0]));
+		if (ret)
+			return ret;
+	}
+
 	ret = ia_css_qplane_configure(pipeline, binary, &binary->in_frame_info);
 	if (ret)
 		return ret;
@@ -788,9 +792,6 @@ static int configure_isp_from_args(const struct sh_css_sp_pipeline *pipeline,
 	if (ret)
 		return ret;
 	ret = ia_css_copy_output_configure(binary, args->copy_output);
-	if (ret)
-		return ret;
-	ret = ia_css_output0_configure(binary, ia_css_frame_get_info(args->out_frame[0]));
 	if (ret)
 		return ret;
 	ret = ia_css_iterator_configure(binary, ia_css_frame_get_info(args->in_frame));
@@ -807,22 +808,18 @@ static int configure_isp_from_args(const struct sh_css_sp_pipeline *pipeline,
 	if (ret)
 		return ret;
 
-	/*
-	 * FIXME: args->delay_frames can be NULL here
-	 *
-	 * Somehow, the driver at the Intel Atom Yocto tree doesn't seem to
-	 * suffer from the same issue.
-	 *
-	 * Anyway, the function below should now handle a NULL delay_frames
-	 * without crashing, but the pipeline should likely be built without
-	 * adding it at the first place (or there are a hidden bug somewhere)
-	 */
-	ret = ia_css_ref_configure(binary, args->delay_frames, pipeline->dvs_frame_delay);
-	if (ret)
-		return ret;
-	ret = ia_css_tnr_configure(binary, args->tnr_frames);
-	if (ret)
-		return ret;
+	if (binary->info->sp.enable.ref_frame) {
+		ret = ia_css_ref_configure(binary, args->delay_frames, pipeline->dvs_frame_delay);
+		if (ret)
+			return ret;
+	}
+
+	if (binary->info->sp.enable.tnr) {
+		ret = ia_css_tnr_configure(binary, args->tnr_frames);
+		if (ret)
+			return ret;
+	}
+
 	return ia_css_bayer_io_config(binary, args);
 }
 
diff --git a/drivers/staging/media/atomisp/pci/system_local.c b/drivers/staging/media/atomisp/pci/system_local.c
index a8a93760d5b10..69bcd557a8213 100644
--- a/drivers/staging/media/atomisp/pci/system_local.c
+++ b/drivers/staging/media/atomisp/pci/system_local.c
@@ -86,8 +86,7 @@ const hrt_address GP_DEVICE_BASE[N_GP_DEVICE_ID] = {
 /*GP TIMER , all timer registers are inter-twined,
  * so, having multiple base addresses for
  * different timers does not help*/
-const hrt_address GP_TIMER_BASE =
-    (hrt_address)0x0000000000000600ULL;
+const hrt_address GP_TIMER_BASE = (hrt_address)0x0000000000000600ULL;
 
 /* GPIO */
 const hrt_address GPIO_BASE[N_GPIO_ID] = {
diff --git a/drivers/staging/media/av7110/av7110.c b/drivers/staging/media/av7110/av7110.c
index 014d0c6f0a8ba..862aee993889c 100644
--- a/drivers/staging/media/av7110/av7110.c
+++ b/drivers/staging/media/av7110/av7110.c
@@ -121,19 +121,22 @@ static void init_av7110_av(struct av7110 *av7110)
 	if (ret < 0)
 		pr_err("cannot set internal volume to maximum:%d\n", ret);
 
-	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetMonitorType,
-			    1, (u16)av7110->display_ar);
+	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER,
+			    AV7110_SET_MONITOR_TYPE, 1, av7110->display_ar);
 	if (ret < 0)
 		pr_err("unable to set aspect ratio\n");
-	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetPanScanType,
-			    1, av7110->display_panscan);
+	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER,
+			    AV7110_SET_PANSCAN_TYPE, 1,
+			    av7110->display_panscan);
 	if (ret < 0)
 		pr_err("unable to set pan scan\n");
 
-	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetWSSConfig, 2, 2, wss_cfg_4_3);
+	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER,
+			    AV7110_SET_WSS_CONFIG, 2, 2, wss_cfg_4_3);
 	if (ret < 0)
 		pr_err("unable to configure 4:3 wss\n");
-	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetWSSConfig, 2, 3, wss_cfg_16_9);
+	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER,
+			    AV7110_SET_WSS_CONFIG, 2, 3, wss_cfg_16_9);
 	if (ret < 0)
 		pr_err("unable to configure 16:9 wss\n");
 
@@ -314,17 +317,6 @@ static int DvbDmxFilterCallback(u8 *buffer1, size_t buffer1_len,
 	}
 }
 
-//#define DEBUG_TIMING
-static inline void print_time(char *s)
-{
-#ifdef DEBUG_TIMING
-	struct timespec64 ts;
-
-	ktime_get_real_ts64(&ts);
-	pr_info("%s(): %ptSp\n", s, &ts);
-#endif
-}
-
 #define DEBI_READ 0
 #define DEBI_WRITE 1
 static inline void start_debi_dma(struct av7110 *av7110, int dir,
@@ -353,7 +345,6 @@ static void debiirq(struct tasklet_struct *t)
 	int handle = (type >> 8) & 0x1f;
 	unsigned int xfer = 0;
 
-	print_time("debi");
 	dprintk(4, "type 0x%04x\n", type);
 
 	if (type == -1) {
@@ -473,7 +464,6 @@ static void gpioirq(struct tasklet_struct *t)
 	txbuf = irdebi(av7110, DEBINOSWAP, TX_BUFF, 0, 2);
 	len = (av7110->debilen + 3) & ~3;
 
-	print_time("gpio");
 	dprintk(8, "GPIO0 irq 0x%04x %d\n", av7110->debitype, av7110->debilen);
 
 	switch (av7110->debitype & 0xff) {
@@ -717,7 +707,8 @@ static inline int SetPIDs(struct av7110 *av7110, u16 vpid, u16 apid, u16 ttpid,
 	if (av7110->audiostate.bypass_mode)
 		aflags |= 0x8000;
 
-	return av7110_fw_cmd(av7110, COMTYPE_PIDFILTER, MultiPID, 6,
+	return av7110_fw_cmd(av7110, COMTYPE_PIDFILTER,
+			     AV7110_MULTI_PID, 6,
 			     pcrpid, vpid, apid, ttpid, subpid, aflags);
 }
 
@@ -762,7 +753,6 @@ static int StartHWFilter(struct dvb_demux_filter *dvbdmxfilter)
 	u16 buf[20];
 	int ret, i;
 	u16 handle;
-//	u16 mode = 0x0320;
 	u16 mode = 0xb96a;
 
 	dprintk(4, "%p\n", av7110);
@@ -785,7 +775,7 @@ static int StartHWFilter(struct dvb_demux_filter *dvbdmxfilter)
 		av7110_p2t_init(&av7110->p2t_filter[dvbdmxfilter->index], dvbdmxfeed);
 	}
 
-	buf[0] = (COMTYPE_PID_FILTER << 8) + AddPIDFilter;
+	buf[0] = (COMTYPE_PID_FILTER << 8) + AV7110_ADD_PID_FILTER;
 	buf[1] = 16;
 	buf[2] = dvbdmxfeed->pid;
 	buf[3] = mode;
@@ -828,7 +818,7 @@ static int StopHWFilter(struct dvb_demux_filter *dvbdmxfilter)
 
 	av7110->handle2filter[handle] = NULL;
 
-	buf[0] = (COMTYPE_PID_FILTER << 8) + DelPIDFilter;
+	buf[0] = (COMTYPE_PID_FILTER << 8) + AV7110_DEL_PID_FILTER;
 	buf[1] = 1;
 	buf[2] = handle;
 	ret = av7110_fw_request(av7110, buf, 3, answ, 2);
@@ -873,7 +863,8 @@ static int dvb_feed_start_pid(struct dvb_demux_feed *dvbdmxfeed)
 
 	if (dvbdmxfeed->pes_type < 2 && npids[0])
 		if (av7110->fe_synced) {
-			ret = av7110_fw_cmd(av7110, COMTYPE_PIDFILTER, Scan, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_PIDFILTER,
+					    AV7110_SCAN, 0);
 			if (ret)
 				return ret;
 		}
@@ -1911,11 +1902,13 @@ static int av7110_fe_lock_fix(struct av7110 *av7110, enum fe_status status)
 			av7110->pids[DMX_PES_TELETEXT], 0,
 			av7110->pids[DMX_PES_PCR]);
 		if (!ret)
-			ret = av7110_fw_cmd(av7110, COMTYPE_PIDFILTER, Scan, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_PIDFILTER,
+					    AV7110_SCAN, 0);
 	} else {
 		ret = SetPIDs(av7110, 0, 0, 0, 0, 0);
 		if (!ret) {
-			ret = av7110_fw_cmd(av7110, COMTYPE_PID_FILTER, FlushTSQueue, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_PID_FILTER,
+					    AV7110_FLUSH_TS_QUEUE, 0);
 			if (!ret)
 				ret = av7110_wait_msgstate(av7110, GPMQBusy);
 		}
@@ -2272,7 +2265,7 @@ static int frontend_init(struct av7110 *av7110)
  * original Roberto Deza's hardware:
  *
  * rps1 code for budgetpatch will copy internal HS event to GPIO3 pin.
- * GPIO3 is in budget-patch hardware connectd to port B VSYNC
+ * GPIO3 is in budget-patch hardware connected to port B VSYNC
  * HS is an internal event of 7146, accessible with RPS
  * and temporarily raised high every n lines
  * (n in defined in the RPS_THRESH1 counter threshold)
@@ -2785,8 +2778,6 @@ static void av7110_irq(struct saa7146_dev *dev, u32 *isr)
 {
 	struct av7110 *av7110 = dev->ext_priv;
 
-	//print_time("av7110_irq");
-
 	/* Note: Don't try to handle the DEBI error irq (MASK_18), in
 	 * intel mode the timeout is asserted all the time...
 	 */
diff --git a/drivers/staging/media/av7110/av7110.h b/drivers/staging/media/av7110/av7110.h
index b584754f4be0b..797dda437cbe2 100644
--- a/drivers/staging/media/av7110/av7110.h
+++ b/drivers/staging/media/av7110/av7110.h
@@ -243,8 +243,8 @@ struct av7110 {
 	struct dvb_video_events  video_events;
 	video_size_t		 video_size;
 
-	u16			wssMode;
-	u16			wssData;
+	u16			wss_mode;
+	u16			wss_data;
 
 	struct infrared		ir;
 
diff --git a/drivers/staging/media/av7110/av7110_av.c b/drivers/staging/media/av7110/av7110_av.c
index 2993ac43c49c4..22f1335d371b2 100644
--- a/drivers/staging/media/av7110/av7110_av.c
+++ b/drivers/staging/media/av7110/av7110_av.c
@@ -111,7 +111,7 @@ int av7110_av_start_record(struct av7110 *av7110, int av,
 
 	if (av7110->playing || (av7110->rec_mode & av))
 		return -EBUSY;
-	av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Stop, 0);
+	av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, AV7110_REC_PLAY_STOP, 0);
 	dvbdmx->recording = 1;
 	av7110->rec_mode |= av;
 
@@ -121,7 +121,9 @@ int av7110_av_start_record(struct av7110 *av7110, int av,
 				       dvbdmx->pesfilter[0]->pid,
 				       dvb_filter_pes2ts_cb,
 				       (void *)dvbdmx->pesfilter[0]);
-		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, AudioPES, 0);
+		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+				    AV7110_REC_PLAY_RECORD, 2,
+				    AV7110_AUDIO_PES, 0);
 		break;
 
 	case RP_VIDEO:
@@ -129,7 +131,9 @@ int av7110_av_start_record(struct av7110 *av7110, int av,
 				       dvbdmx->pesfilter[1]->pid,
 				       dvb_filter_pes2ts_cb,
 				       (void *)dvbdmx->pesfilter[1]);
-		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, VideoPES, 0);
+		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+				    AV7110_REC_PLAY_RECORD, 2,
+				    AV7110_VIDEO_PES, 0);
 		break;
 
 	case RP_AV:
@@ -141,7 +145,9 @@ int av7110_av_start_record(struct av7110 *av7110, int av,
 				       dvbdmx->pesfilter[1]->pid,
 				       dvb_filter_pes2ts_cb,
 				       (void *)dvbdmx->pesfilter[1]);
-		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, AV_PES, 0);
+		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+				    AV7110_REC_PLAY_RECORD, 2,
+				    AV7110_AV_PES, 0);
 		break;
 	}
 	return ret;
@@ -158,7 +164,7 @@ int av7110_av_start_play(struct av7110 *av7110, int av)
 	if (av7110->playing & av)
 		return -EBUSY;
 
-	av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Stop, 0);
+	av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, AV7110_REC_PLAY_STOP, 0);
 
 	if (av7110->playing == RP_NONE) {
 		av7110_ipack_reset(&av7110->ipack[0]);
@@ -168,15 +174,21 @@ int av7110_av_start_play(struct av7110 *av7110, int av)
 	av7110->playing |= av;
 	switch (av7110->playing) {
 	case RP_AUDIO:
-		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, AudioPES, 0);
+		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+				    AV7110_REC_PLAY_PLAY, 2,
+				    AV7110_AUDIO_PES, 0);
 		break;
 	case RP_VIDEO:
-		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, VideoPES, 0);
+		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+				    AV7110_REC_PLAY_PLAY, 2,
+				    AV7110_VIDEO_PES, 0);
 		av7110->sinfo = 0;
 		break;
 	case RP_AV:
 		av7110->sinfo = 0;
-		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, AV_PES, 0);
+		ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+				    AV7110_REC_PLAY_PLAY, 2,
+				    AV7110_AV_PES, 0);
 		break;
 	}
 	return ret;
@@ -190,15 +202,19 @@ int av7110_av_stop(struct av7110 *av7110, int av)
 
 	if (!(av7110->playing & av) && !(av7110->rec_mode & av))
 		return 0;
-	av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Stop, 0);
+	av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, AV7110_REC_PLAY_STOP, 0);
 	if (av7110->playing) {
 		av7110->playing &= ~av;
 		switch (av7110->playing) {
 		case RP_AUDIO:
-			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, AudioPES, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+					    AV7110_REC_PLAY_PLAY, 2,
+					    AV7110_AUDIO_PES, 0);
 			break;
 		case RP_VIDEO:
-			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Play, 2, VideoPES, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+					    AV7110_REC_PLAY_PLAY, 2,
+					    AV7110_VIDEO_PES, 0);
 			break;
 		case RP_NONE:
 			ret = av7110_set_vidmode(av7110, av7110->vidmode);
@@ -208,10 +224,14 @@ int av7110_av_stop(struct av7110 *av7110, int av)
 		av7110->rec_mode &= ~av;
 		switch (av7110->rec_mode) {
 		case RP_AUDIO:
-			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, AudioPES, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+					    AV7110_REC_PLAY_RECORD, 2,
+					    AV7110_AUDIO_PES, 0);
 			break;
 		case RP_VIDEO:
-			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Record, 2, VideoPES, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+					    AV7110_REC_PLAY_RECORD, 2,
+					    AV7110_VIDEO_PES, 0);
 			break;
 		case RP_NONE:
 			break;
@@ -325,7 +345,8 @@ int av7110_set_vidmode(struct av7110 *av7110, enum av7110_video_mode mode)
 
 	dprintk(2, "av7110:%p\n", av7110);
 
-	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, LoadVidCode, 1, mode);
+	ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER,
+			    AV7110_LOAD_VID_CODE, 1, mode);
 
 	if (!ret && !av7110->playing) {
 		ret = ChangePIDs(av7110, av7110->pids[DMX_PES_VIDEO],
@@ -333,7 +354,8 @@ int av7110_set_vidmode(struct av7110 *av7110, enum av7110_video_mode mode)
 			   av7110->pids[DMX_PES_TELETEXT],
 			   0, av7110->pids[DMX_PES_PCR]);
 		if (!ret)
-			ret = av7110_fw_cmd(av7110, COMTYPE_PIDFILTER, Scan, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_PIDFILTER,
+					    AV7110_SCAN, 0);
 	}
 	return ret;
 }
@@ -1168,7 +1190,8 @@ static int dvb_video_ioctl(struct file *file,
 		}
 		if (av7110->videostate.stream_source == VIDEO_SOURCE_MEMORY) {
 			if (av7110->playing == RP_AV) {
-				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Stop, 0);
+				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+						    AV7110_REC_PLAY_STOP, 0);
 				if (ret)
 					break;
 				av7110->playing &= ~RP_VIDEO;
@@ -1184,7 +1207,8 @@ static int dvb_video_ioctl(struct file *file,
 	case VIDEO_FREEZE:
 		av7110->videostate.play_state = VIDEO_FREEZED;
 		if (av7110->playing & RP_VIDEO)
-			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Pause, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+					    AV7110_REC_PLAY_PAUSE, 0);
 		else
 			ret = vidcom(av7110, AV_VIDEO_CMD_FREEZE, 1);
 		if (!ret)
@@ -1193,7 +1217,8 @@ static int dvb_video_ioctl(struct file *file,
 
 	case VIDEO_CONTINUE:
 		if (av7110->playing & RP_VIDEO)
-			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Continue, 0);
+			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+					    AV7110_REC_PLAY_CONTINUE, 0);
 		if (!ret)
 			ret = vidcom(av7110, AV_VIDEO_CMD_PLAY, 0);
 		if (!ret) {
@@ -1248,8 +1273,9 @@ static int dvb_video_ioctl(struct file *file,
 		if (ret < 0)
 			break;
 		av7110->videostate.display_format = format;
-		ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetPanScanType,
-				    1, av7110->display_panscan);
+		ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER,
+				    AV7110_SET_PANSCAN_TYPE, 1,
+				    av7110->display_panscan);
 		break;
 	}
 
@@ -1259,8 +1285,8 @@ static int dvb_video_ioctl(struct file *file,
 			break;
 		}
 		av7110->display_ar = arg;
-		ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetMonitorType,
-				    1, (u16)arg);
+		ret = av7110_fw_cmd(av7110, COMTYPE_ENCODER,
+				    AV7110_SET_MONITOR_TYPE, 1, arg);
 		break;
 
 #ifdef CONFIG_COMPAT
@@ -1291,7 +1317,8 @@ static int dvb_video_ioctl(struct file *file,
 		//note: arg is ignored by firmware
 		if (av7110->playing & RP_VIDEO)
 			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
-					    __Scan_I, 2, AV_PES, 0);
+					    AV7110_REC_PLAY_SCAN_I, 2,
+					    AV7110_AV_PES, 0);
 		else
 			ret = vidcom(av7110, AV_VIDEO_CMD_FFWD, arg);
 		if (!ret) {
@@ -1303,7 +1330,9 @@ static int dvb_video_ioctl(struct file *file,
 	case VIDEO_SLOWMOTION:
 		if (av7110->playing & RP_VIDEO) {
 			if (av7110->trickmode != TRICK_SLOW)
-				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY, __Slow, 2, 0, 0);
+				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
+						    AV7110_REC_PLAY_SLOW, 2,
+						    0, 0);
 			if (!ret)
 				ret = vidcom(av7110, AV_VIDEO_CMD_SLOW, arg);
 		} else {
@@ -1329,15 +1358,18 @@ static int dvb_video_ioctl(struct file *file,
 		av7110_ipack_reset(&av7110->ipack[1]);
 		if (av7110->playing == RP_AV) {
 			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
-					    __Play, 2, AV_PES, 0);
+					    AV7110_REC_PLAY_PLAY, 2,
+					    AV7110_AV_PES, 0);
 			if (ret)
 				break;
 			if (av7110->trickmode == TRICK_FAST)
 				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
-						    __Scan_I, 2, AV_PES, 0);
+						    AV7110_REC_PLAY_SCAN_I, 2,
+						    AV7110_AV_PES, 0);
 			if (av7110->trickmode == TRICK_SLOW) {
 				ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
-						    __Slow, 2, 0, 0);
+						    AV7110_REC_PLAY_SLOW, 2,
+						    0, 0);
 				if (!ret)
 					ret = vidcom(av7110, AV_VIDEO_CMD_SLOW, arg);
 			}
@@ -1483,7 +1515,8 @@ static int dvb_audio_ioctl(struct file *file,
 		av7110_ipack_reset(&av7110->ipack[0]);
 		if (av7110->playing == RP_AV)
 			ret = av7110_fw_cmd(av7110, COMTYPE_REC_PLAY,
-					    __Play, 2, AV_PES, 0);
+					    AV7110_REC_PLAY_PLAY, 2,
+					    AV7110_AV_PES, 0);
 		break;
 
 	case AUDIO_SET_ID:
diff --git a/drivers/staging/media/av7110/av7110_ca.c b/drivers/staging/media/av7110/av7110_ca.c
index 63d9c97a5190c..24876c8fdc8ce 100644
--- a/drivers/staging/media/av7110/av7110_ca.c
+++ b/drivers/staging/media/av7110/av7110_ca.c
@@ -307,7 +307,8 @@ static int dvb_ca_ioctl(struct file *file, unsigned int cmd, void *parg)
 			mutex_unlock(&av7110->ioctl_mutex);
 			return -EINVAL;
 		}
-		av7110_fw_cmd(av7110, COMTYPE_PIDFILTER, SetDescr, 5,
+		av7110_fw_cmd(av7110, COMTYPE_PIDFILTER,
+			      AV7110_SET_DESCR, 5,
 			      (descr->index << 8) | descr->parity,
 			      (descr->cw[0] << 8) | descr->cw[1],
 			      (descr->cw[2] << 8) | descr->cw[3],
diff --git a/drivers/staging/media/av7110/av7110_hw.c b/drivers/staging/media/av7110/av7110_hw.c
index 49ce295771e41..b0bd7666d48be 100644
--- a/drivers/staging/media/av7110/av7110_hw.c
+++ b/drivers/staging/media/av7110/av7110_hw.c
@@ -95,29 +95,6 @@ u32 av7110_debiread(struct av7110 *av7110, u32 config, int addr, unsigned int co
 	return result;
 }
 
-/* av7110 ARM core boot stuff */
-#if 0
-void av7110_reset_arm(struct av7110 *av7110)
-{
-	saa7146_setgpio(av7110->dev, RESET_LINE, SAA7146_GPIO_OUTLO);
-
-	/* Disable DEBI and GPIO irq */
-	SAA7146_IER_DISABLE(av7110->dev, MASK_19 | MASK_03);
-	SAA7146_ISR_CLEAR(av7110->dev, MASK_19 | MASK_03);
-
-	saa7146_setgpio(av7110->dev, RESET_LINE, SAA7146_GPIO_OUTHI);
-	msleep(30);	/* the firmware needs some time to initialize */
-
-	ARM_ResetMailBox(av7110);
-
-	SAA7146_ISR_CLEAR(av7110->dev, MASK_19 | MASK_03);
-	SAA7146_IER_ENABLE(av7110->dev, MASK_03);
-
-	av7110->arm_ready = 1;
-	dprintk(1, "reset ARM\n");
-}
-#endif  /*  0  */
-
 static int waitdebi(struct av7110 *av7110, int adr, int state)
 {
 	int k;
@@ -498,29 +475,6 @@ int av7110_fw_cmd(struct av7110 *av7110, int type, int com, int num, ...)
 	return ret;
 }
 
-#if 0
-int av7110_send_ci_cmd(struct av7110 *av7110, u8 subcom, u8 *buf, u8 len)
-{
-	int i, ret;
-	u16 cmd[18] = { ((COMTYPE_COMMON_IF << 8) + subcom),
-		16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
-
-	dprintk(4, "%p\n", av7110);
-
-	for (i = 0; i < len && i < 32; i++) {
-		if (i % 2 == 0)
-			cmd[(i / 2) + 2] = (u16)(buf[i]) << 8;
-		else
-			cmd[(i / 2) + 2] |= buf[i];
-	}
-
-	ret = av7110_send_fw_cmd(av7110, cmd, 18);
-	if (ret && ret != -ERESTARTSYS)
-		pr_err("%s(): error %d\n", __func__, ret);
-	return ret;
-}
-#endif  /*  0  */
-
 int av7110_fw_request(struct av7110 *av7110, u16 *request_buf,
 		      int request_buf_len, u16 *reply_buf, int reply_buf_len)
 {
diff --git a/drivers/staging/media/av7110/av7110_hw.h b/drivers/staging/media/av7110/av7110_hw.h
index d4579f411c56d..3afe474da8710 100644
--- a/drivers/staging/media/av7110/av7110_hw.h
+++ b/drivers/staging/media/av7110/av7110_hw.h
@@ -21,12 +21,12 @@ enum av7110_bootstate {
 };
 
 enum av7110_type_rec_play_format {
-	RP_None,
-	AudioPES,
-	AudioMp2,
-	AudioPCM,
-	VideoPES,
-	AV_PES
+	AV7110_RP_NONE,
+	AV7110_AUDIO_PES,
+	AV7110_AUDIO_MP2,
+	AV7110_AUDIO_PCM,
+	AV7110_VIDEO_PES,
+	AV7110_AV_PES,
 };
 
 enum av7110_osd_palette_type {
@@ -112,19 +112,19 @@ enum av7110_osd_command {
 };
 
 enum av7110_pid_command {
-	MultiPID,
-	VideoPID,
-	AudioPID,
-	InitFilt,
-	FiltError,
-	NewVersion,
-	CacheError,
-	AddPIDFilter,
-	DelPIDFilter,
-	Scan,
-	SetDescr,
-	SetIR,
-	FlushTSQueue
+	AV7110_MULTI_PID,
+	AV7110_VIDEO_PID,
+	AV7110_AUDIO_PID,
+	AV7110_INIT_FILT,
+	AV7110_FILT_ERROR,
+	AV7110_NEW_VERSION,
+	AV7110_CACHE_ERROR,
+	AV7110_ADD_PID_FILTER,
+	AV7110_DEL_PID_FILTER,
+	AV7110_SCAN,
+	AV7110_SET_DESCR,
+	AV7110_SET_IR,
+	AV7110_FLUSH_TS_QUEUE,
 };
 
 enum av7110_mpeg_command {
@@ -158,24 +158,24 @@ enum av7110_request_command {
 };
 
 enum av7110_encoder_command {
-	SetVidMode,
-	SetTestMode,
-	LoadVidCode,
-	SetMonitorType,
-	SetPanScanType,
-	SetFreezeMode,
-	SetWSSConfig
+	AV7110_SET_VID_MODE,
+	AV7110_SET_TEST_MODE,
+	AV7110_LOAD_VID_CODE,
+	AV7110_SET_MONITOR_TYPE,
+	AV7110_SET_PANSCAN_TYPE,
+	AV7110_SET_FREEZE_MODE,
+	AV7110_SET_WSS_CONFIG,
 };
 
 enum av7110_rec_play_state {
-	__Record,
-	__Stop,
-	__Play,
-	__Pause,
-	__Slow,
-	__FF_IP,
-	__Scan_I,
-	__Continue
+	AV7110_REC_PLAY_RECORD,
+	AV7110_REC_PLAY_STOP,
+	AV7110_REC_PLAY_PLAY,
+	AV7110_REC_PLAY_PAUSE,
+	AV7110_REC_PLAY_SLOW,
+	AV7110_REC_PLAY_FF_IP,
+	AV7110_REC_PLAY_SCAN_I,
+	AV7110_REC_PLAY_CONTINUE,
 };
 
 enum av7110_fw_cmd_misc {
@@ -452,7 +452,8 @@ static inline int SendDAC(struct av7110 *av7110, u8 addr, u8 data)
 
 static inline int av7710_set_video_mode(struct av7110 *av7110, int mode)
 {
-	return av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetVidMode, 1, mode);
+	return av7110_fw_cmd(av7110, COMTYPE_ENCODER,
+			     AV7110_SET_VID_MODE, 1, mode);
 }
 
 static inline int vidcom(struct av7110 *av7110, u32 com, u32 arg)
diff --git a/drivers/staging/media/av7110/av7110_ir.c b/drivers/staging/media/av7110/av7110_ir.c
index fdae467fd7ab8..9b7bf1857868d 100644
--- a/drivers/staging/media/av7110/av7110_ir.c
+++ b/drivers/staging/media/av7110/av7110_ir.c
@@ -71,8 +71,8 @@ int av7110_set_ir_config(struct av7110 *av7110)
 {
 	dprintk(4, "ir config = %08x\n", av7110->ir.ir_config);
 
-	return av7110_fw_cmd(av7110, COMTYPE_PIDFILTER, SetIR, 1,
-			     av7110->ir.ir_config);
+	return av7110_fw_cmd(av7110, COMTYPE_PIDFILTER,
+			     AV7110_SET_IR, 1, av7110->ir.ir_config);
 }
 
 static int change_protocol(struct rc_dev *rcdev, u64 *rc_type)
diff --git a/drivers/staging/media/av7110/av7110_v4l.c b/drivers/staging/media/av7110/av7110_v4l.c
index 200a7a29ea317..aed7581fa8a5f 100644
--- a/drivers/staging/media/av7110/av7110_v4l.c
+++ b/drivers/staging/media/av7110/av7110_v4l.c
@@ -556,7 +556,7 @@ static int vidioc_g_fmt_sliced_vbi_out(struct file *file, void *fh,
 	if (FW_VERSION(av7110->arm_app) < 0x2623)
 		return -EINVAL;
 	memset(&f->fmt.sliced, 0, sizeof(f->fmt.sliced));
-	if (av7110->wssMode) {
+	if (av7110->wss_mode) {
 		f->fmt.sliced.service_set = V4L2_SLICED_WSS_625;
 		f->fmt.sliced.service_lines[0][23] = V4L2_SLICED_WSS_625;
 	}
@@ -596,14 +596,14 @@ static int vidioc_s_fmt_sliced_vbi_out(struct file *file, void *fh,
 		return -EINVAL;
 	if (f->fmt.sliced.service_set & V4L2_SLICED_WSS_625) {
 		/* WSS controlled by userspace */
-		av7110->wssMode = 1;
-		av7110->wssData = 0;
+		av7110->wss_mode = 1;
+		av7110->wss_data = 0;
 	} else {
 		/* WSS controlled by firmware */
-		av7110->wssMode = 0;
-		av7110->wssData = 0;
+		av7110->wss_mode = 0;
+		av7110->wss_data = 0;
 		return av7110_fw_cmd(av7110, COMTYPE_ENCODER,
-				     SetWSSConfig, 1, 0);
+				     AV7110_SET_WSS_CONFIG, 1, 0);
 	}
 	return 0;
 }
@@ -616,17 +616,19 @@ static ssize_t av7110_vbi_write(struct file *file, const char __user *data, size
 	int rc;
 
 	dprintk(2, "\n");
-	if (FW_VERSION(av7110->arm_app) < 0x2623 || !av7110->wssMode || count != sizeof(d))
+	if (FW_VERSION(av7110->arm_app) < 0x2623 ||
+	    !av7110->wss_mode || count != sizeof(d))
 		return -EINVAL;
 	if (copy_from_user(&d, data, count))
 		return -EFAULT;
 	if ((d.id != 0 && d.id != V4L2_SLICED_WSS_625) || d.field != 0 || d.line != 23)
 		return -EINVAL;
 	if (d.id)
-		av7110->wssData = ((d.data[1] << 8) & 0x3f00) | d.data[0];
+		av7110->wss_data = ((d.data[1] << 8) & 0x3f00) | d.data[0];
 	else
-		av7110->wssData = 0x8000;
-	rc = av7110_fw_cmd(av7110, COMTYPE_ENCODER, SetWSSConfig, 2, 1, av7110->wssData);
+		av7110->wss_data = 0x8000;
+	rc = av7110_fw_cmd(av7110, COMTYPE_ENCODER,
+			   AV7110_SET_WSS_CONFIG, 2, 1, av7110->wss_data);
 	return (rc < 0) ? rc : count;
 }
 
diff --git a/drivers/staging/media/deprecated/atmel/Kconfig b/drivers/staging/media/deprecated/atmel/Kconfig
deleted file mode 100644
index 418841ea5a0dd..0000000000000
--- a/drivers/staging/media/deprecated/atmel/Kconfig
+++ /dev/null
@@ -1,47 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-
-comment "Atmel media platform drivers"
-
-config VIDEO_ATMEL_ISC
-	tristate "ATMEL Image Sensor Controller (ISC) support (DEPRECATED)"
-	depends on V4L_PLATFORM_DRIVERS
-	depends on VIDEO_DEV && COMMON_CLK
-	depends on ARCH_AT91 || COMPILE_TEST
-	depends on !VIDEO_MICROCHIP_ISC_BASE || COMPILE_TEST
-	select MEDIA_CONTROLLER
-	select VIDEO_V4L2_SUBDEV_API
-	select VIDEOBUF2_DMA_CONTIG
-	select REGMAP_MMIO
-	select V4L2_FWNODE
-	select VIDEO_ATMEL_ISC_BASE
-	help
-	   This module makes the ATMEL Image Sensor Controller available
-	   as a v4l2 device.
-
-	   This driver is deprecated and is scheduled for removal by
-	   the beginning of 2026. See the TODO file for more information.
-
-config VIDEO_ATMEL_XISC
-	tristate "ATMEL eXtended Image Sensor Controller (XISC) support (DEPRECATED)"
-	depends on V4L_PLATFORM_DRIVERS
-	depends on VIDEO_DEV && COMMON_CLK
-	depends on ARCH_AT91 || COMPILE_TEST
-	depends on !VIDEO_MICROCHIP_ISC_BASE || COMPILE_TEST
-	select VIDEOBUF2_DMA_CONTIG
-	select REGMAP_MMIO
-	select V4L2_FWNODE
-	select VIDEO_ATMEL_ISC_BASE
-	select MEDIA_CONTROLLER
-	select VIDEO_V4L2_SUBDEV_API
-	help
-	   This module makes the ATMEL eXtended Image Sensor Controller
-	   available as a v4l2 device.
-
-	   This driver is deprecated and is scheduled for removal by
-	   the beginning of 2026. See the TODO file for more information.
-
-config VIDEO_ATMEL_ISC_BASE
-	tristate
-	default n
-	help
-	  ATMEL ISC and XISC common code base.
diff --git a/drivers/staging/media/deprecated/atmel/Makefile b/drivers/staging/media/deprecated/atmel/Makefile
deleted file mode 100644
index 34eaeeac5bba7..0000000000000
--- a/drivers/staging/media/deprecated/atmel/Makefile
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-atmel-isc-objs = atmel-sama5d2-isc.o
-atmel-xisc-objs = atmel-sama7g5-isc.o
-atmel-isc-common-objs = atmel-isc-base.o atmel-isc-clk.o
-
-obj-$(CONFIG_VIDEO_ATMEL_ISC_BASE) += atmel-isc-common.o
-obj-$(CONFIG_VIDEO_ATMEL_ISC) += atmel-isc.o
-obj-$(CONFIG_VIDEO_ATMEL_XISC) += atmel-xisc.o
diff --git a/drivers/staging/media/deprecated/atmel/TODO b/drivers/staging/media/deprecated/atmel/TODO
deleted file mode 100644
index 71691df07a80c..0000000000000
--- a/drivers/staging/media/deprecated/atmel/TODO
+++ /dev/null
@@ -1,34 +0,0 @@
-The Atmel ISC driver is not compliant with media controller specification.
-In order to evolve this driver, it has to move to media controller, to
-support enhanced features and future products which embed it.
-The move to media controller involves several changes which are
-not backwards compatible with the current usability of the driver.
-
-The best example is the way the format is propagated from the top video
-driver /dev/videoX down to the sensor.
-
-In a simple configuration sensor ==> isc , the isc just calls subdev s_fmt
-and controls the sensor directly. This is achieved by having a lot of code
-inside the driver that will query the subdev at probe time and make a list
-of formats which are usable.
-Basically the user has nothing to configure, as the isc will handle
-everything at the top level. This is an easy way to capture, but also comes
-with the drawback of lack of flexibility.
-In a more complicated pipeline
-sensor ==> controller 1 ==> controller 2 ==> isc
-this will not be achievable, as controller 1 and controller 2 might be
-media-controller configurable, and will not propagate the formats down to
-the sensor.
-
-After discussions with the media maintainers, the decision is to move
-Atmel ISC to staging as-is, to keep the Kconfig symbols and the users
-to the driver in staging. Thus, all the existing users of the non
-media-controller paradigm will continue to be happy and use the old config
-way.
-
-The new driver was added in the media subsystem with a different
-symbol, with the conversion to media controller done, and new users
-of the driver will be able to use all the new features.
-
-The replacement driver is named VIDEO_MICROCHIP_ISC or
-VIDEO_MICROCHIP_XISC depending on the product flavor.
diff --git a/drivers/staging/media/deprecated/atmel/atmel-isc-base.c b/drivers/staging/media/deprecated/atmel/atmel-isc-base.c
deleted file mode 100644
index fb9ee8547392b..0000000000000
--- a/drivers/staging/media/deprecated/atmel/atmel-isc-base.c
+++ /dev/null
@@ -1,2008 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Microchip Image Sensor Controller (ISC) common driver base
- *
- * Copyright (C) 2016-2019 Microchip Technology, Inc.
- *
- * Author: Songjun Wu
- * Author: Eugen Hristev <eugen.hristev@microchip.com>
- *
- */
-#include <linux/delay.h>
-#include <linux/interrupt.h>
-#include <linux/math64.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_graph.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/regmap.h>
-#include <linux/videodev2.h>
-#include <linux/atmel-isc-media.h>
-
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-event.h>
-#include <media/v4l2-image-sizes.h>
-#include <media/v4l2-ioctl.h>
-#include <media/v4l2-fwnode.h>
-#include <media/v4l2-subdev.h>
-#include <media/videobuf2-dma-contig.h>
-
-#include "atmel-isc-regs.h"
-#include "atmel-isc.h"
-
-static unsigned int debug;
-module_param(debug, int, 0644);
-MODULE_PARM_DESC(debug, "debug level (0-2)");
-
-static unsigned int sensor_preferred = 1;
-module_param(sensor_preferred, uint, 0644);
-MODULE_PARM_DESC(sensor_preferred,
-		 "Sensor is preferred to output the specified format (1-on 0-off), default 1");
-
-#define ISC_IS_FORMAT_RAW(mbus_code) \
-	(((mbus_code) & 0xf000) == 0x3000)
-
-#define ISC_IS_FORMAT_GREY(mbus_code) \
-	(((mbus_code) == MEDIA_BUS_FMT_Y10_1X10) | \
-	(((mbus_code) == MEDIA_BUS_FMT_Y8_1X8)))
-
-static inline void isc_update_v4l2_ctrls(struct isc_device *isc)
-{
-	struct isc_ctrls *ctrls = &isc->ctrls;
-
-	/* In here we set the v4l2 controls w.r.t. our pipeline config */
-	v4l2_ctrl_s_ctrl(isc->r_gain_ctrl, ctrls->gain[ISC_HIS_CFG_MODE_R]);
-	v4l2_ctrl_s_ctrl(isc->b_gain_ctrl, ctrls->gain[ISC_HIS_CFG_MODE_B]);
-	v4l2_ctrl_s_ctrl(isc->gr_gain_ctrl, ctrls->gain[ISC_HIS_CFG_MODE_GR]);
-	v4l2_ctrl_s_ctrl(isc->gb_gain_ctrl, ctrls->gain[ISC_HIS_CFG_MODE_GB]);
-
-	v4l2_ctrl_s_ctrl(isc->r_off_ctrl, ctrls->offset[ISC_HIS_CFG_MODE_R]);
-	v4l2_ctrl_s_ctrl(isc->b_off_ctrl, ctrls->offset[ISC_HIS_CFG_MODE_B]);
-	v4l2_ctrl_s_ctrl(isc->gr_off_ctrl, ctrls->offset[ISC_HIS_CFG_MODE_GR]);
-	v4l2_ctrl_s_ctrl(isc->gb_off_ctrl, ctrls->offset[ISC_HIS_CFG_MODE_GB]);
-}
-
-static inline void isc_update_awb_ctrls(struct isc_device *isc)
-{
-	struct isc_ctrls *ctrls = &isc->ctrls;
-
-	/* In here we set our actual hw pipeline config */
-
-	regmap_write(isc->regmap, ISC_WB_O_RGR,
-		     ((ctrls->offset[ISC_HIS_CFG_MODE_R])) |
-		     ((ctrls->offset[ISC_HIS_CFG_MODE_GR]) << 16));
-	regmap_write(isc->regmap, ISC_WB_O_BGB,
-		     ((ctrls->offset[ISC_HIS_CFG_MODE_B])) |
-		     ((ctrls->offset[ISC_HIS_CFG_MODE_GB]) << 16));
-	regmap_write(isc->regmap, ISC_WB_G_RGR,
-		     ctrls->gain[ISC_HIS_CFG_MODE_R] |
-		     (ctrls->gain[ISC_HIS_CFG_MODE_GR] << 16));
-	regmap_write(isc->regmap, ISC_WB_G_BGB,
-		     ctrls->gain[ISC_HIS_CFG_MODE_B] |
-		     (ctrls->gain[ISC_HIS_CFG_MODE_GB] << 16));
-}
-
-static inline void isc_reset_awb_ctrls(struct isc_device *isc)
-{
-	unsigned int c;
-
-	for (c = ISC_HIS_CFG_MODE_GR; c <= ISC_HIS_CFG_MODE_B; c++) {
-		/* gains have a fixed point at 9 decimals */
-		isc->ctrls.gain[c] = 1 << 9;
-		/* offsets are in 2's complements */
-		isc->ctrls.offset[c] = 0;
-	}
-}
-
-
-static int isc_queue_setup(struct vb2_queue *vq,
-			    unsigned int *nbuffers, unsigned int *nplanes,
-			    unsigned int sizes[], struct device *alloc_devs[])
-{
-	struct isc_device *isc = vb2_get_drv_priv(vq);
-	unsigned int size = isc->fmt.fmt.pix.sizeimage;
-
-	if (*nplanes)
-		return sizes[0] < size ? -EINVAL : 0;
-
-	*nplanes = 1;
-	sizes[0] = size;
-
-	return 0;
-}
-
-static int isc_buffer_prepare(struct vb2_buffer *vb)
-{
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct isc_device *isc = vb2_get_drv_priv(vb->vb2_queue);
-	unsigned long size = isc->fmt.fmt.pix.sizeimage;
-
-	if (vb2_plane_size(vb, 0) < size) {
-		v4l2_err(&isc->v4l2_dev, "buffer too small (%lu < %lu)\n",
-			 vb2_plane_size(vb, 0), size);
-		return -EINVAL;
-	}
-
-	vb2_set_plane_payload(vb, 0, size);
-
-	vbuf->field = isc->fmt.fmt.pix.field;
-
-	return 0;
-}
-
-static void isc_crop_pfe(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-	u32 h, w;
-
-	h = isc->fmt.fmt.pix.height;
-	w = isc->fmt.fmt.pix.width;
-
-	/*
-	 * In case the sensor is not RAW, it will output a pixel (12-16 bits)
-	 * with two samples on the ISC Data bus (which is 8-12)
-	 * ISC will count each sample, so, we need to multiply these values
-	 * by two, to get the real number of samples for the required pixels.
-	 */
-	if (!ISC_IS_FORMAT_RAW(isc->config.sd_format->mbus_code)) {
-		h <<= 1;
-		w <<= 1;
-	}
-
-	/*
-	 * We limit the column/row count that the ISC will output according
-	 * to the configured resolution that we want.
-	 * This will avoid the situation where the sensor is misconfigured,
-	 * sending more data, and the ISC will just take it and DMA to memory,
-	 * causing corruption.
-	 */
-	regmap_write(regmap, ISC_PFE_CFG1,
-		     (ISC_PFE_CFG1_COLMIN(0) & ISC_PFE_CFG1_COLMIN_MASK) |
-		     (ISC_PFE_CFG1_COLMAX(w - 1) & ISC_PFE_CFG1_COLMAX_MASK));
-
-	regmap_write(regmap, ISC_PFE_CFG2,
-		     (ISC_PFE_CFG2_ROWMIN(0) & ISC_PFE_CFG2_ROWMIN_MASK) |
-		     (ISC_PFE_CFG2_ROWMAX(h - 1) & ISC_PFE_CFG2_ROWMAX_MASK));
-
-	regmap_update_bits(regmap, ISC_PFE_CFG0,
-			   ISC_PFE_CFG0_COLEN | ISC_PFE_CFG0_ROWEN,
-			   ISC_PFE_CFG0_COLEN | ISC_PFE_CFG0_ROWEN);
-}
-
-static void isc_start_dma(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-	u32 sizeimage = isc->fmt.fmt.pix.sizeimage;
-	u32 dctrl_dview;
-	dma_addr_t addr0;
-
-	addr0 = vb2_dma_contig_plane_dma_addr(&isc->cur_frm->vb.vb2_buf, 0);
-	regmap_write(regmap, ISC_DAD0 + isc->offsets.dma, addr0);
-
-	switch (isc->config.fourcc) {
-	case V4L2_PIX_FMT_YUV420:
-		regmap_write(regmap, ISC_DAD1 + isc->offsets.dma,
-			     addr0 + (sizeimage * 2) / 3);
-		regmap_write(regmap, ISC_DAD2 + isc->offsets.dma,
-			     addr0 + (sizeimage * 5) / 6);
-		break;
-	case V4L2_PIX_FMT_YUV422P:
-		regmap_write(regmap, ISC_DAD1 + isc->offsets.dma,
-			     addr0 + sizeimage / 2);
-		regmap_write(regmap, ISC_DAD2 + isc->offsets.dma,
-			     addr0 + (sizeimage * 3) / 4);
-		break;
-	default:
-		break;
-	}
-
-	dctrl_dview = isc->config.dctrl_dview;
-
-	regmap_write(regmap, ISC_DCTRL + isc->offsets.dma,
-		     dctrl_dview | ISC_DCTRL_IE_IS);
-	spin_lock(&isc->awb_lock);
-	regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_CAPTURE);
-	spin_unlock(&isc->awb_lock);
-}
-
-static void isc_set_pipeline(struct isc_device *isc, u32 pipeline)
-{
-	struct regmap *regmap = isc->regmap;
-	struct isc_ctrls *ctrls = &isc->ctrls;
-	u32 val, bay_cfg;
-	const u32 *gamma;
-	unsigned int i;
-
-	/* WB-->CFA-->CC-->GAM-->CSC-->CBC-->SUB422-->SUB420 */
-	for (i = 0; i < ISC_PIPE_LINE_NODE_NUM; i++) {
-		val = pipeline & BIT(i) ? 1 : 0;
-		regmap_field_write(isc->pipeline[i], val);
-	}
-
-	if (!pipeline)
-		return;
-
-	bay_cfg = isc->config.sd_format->cfa_baycfg;
-
-	regmap_write(regmap, ISC_WB_CFG, bay_cfg);
-	isc_update_awb_ctrls(isc);
-	isc_update_v4l2_ctrls(isc);
-
-	regmap_write(regmap, ISC_CFA_CFG, bay_cfg | ISC_CFA_CFG_EITPOL);
-
-	gamma = &isc->gamma_table[ctrls->gamma_index][0];
-	regmap_bulk_write(regmap, ISC_GAM_BENTRY, gamma, GAMMA_ENTRIES);
-	regmap_bulk_write(regmap, ISC_GAM_GENTRY, gamma, GAMMA_ENTRIES);
-	regmap_bulk_write(regmap, ISC_GAM_RENTRY, gamma, GAMMA_ENTRIES);
-
-	isc->config_dpc(isc);
-	isc->config_csc(isc);
-	isc->config_cbc(isc);
-	isc->config_cc(isc);
-	isc->config_gam(isc);
-}
-
-static int isc_update_profile(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-	u32 sr;
-	int counter = 100;
-
-	regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_UPPRO);
-
-	regmap_read(regmap, ISC_CTRLSR, &sr);
-	while ((sr & ISC_CTRL_UPPRO) && counter--) {
-		usleep_range(1000, 2000);
-		regmap_read(regmap, ISC_CTRLSR, &sr);
-	}
-
-	if (counter < 0) {
-		v4l2_warn(&isc->v4l2_dev, "Time out to update profile\n");
-		return -ETIMEDOUT;
-	}
-
-	return 0;
-}
-
-static void isc_set_histogram(struct isc_device *isc, bool enable)
-{
-	struct regmap *regmap = isc->regmap;
-	struct isc_ctrls *ctrls = &isc->ctrls;
-
-	if (enable) {
-		regmap_write(regmap, ISC_HIS_CFG + isc->offsets.his,
-			     ISC_HIS_CFG_MODE_GR |
-			     (isc->config.sd_format->cfa_baycfg
-					<< ISC_HIS_CFG_BAYSEL_SHIFT) |
-					ISC_HIS_CFG_RAR);
-		regmap_write(regmap, ISC_HIS_CTRL + isc->offsets.his,
-			     ISC_HIS_CTRL_EN);
-		regmap_write(regmap, ISC_INTEN, ISC_INT_HISDONE);
-		ctrls->hist_id = ISC_HIS_CFG_MODE_GR;
-		isc_update_profile(isc);
-		regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_HISREQ);
-
-		ctrls->hist_stat = HIST_ENABLED;
-	} else {
-		regmap_write(regmap, ISC_INTDIS, ISC_INT_HISDONE);
-		regmap_write(regmap, ISC_HIS_CTRL + isc->offsets.his,
-			     ISC_HIS_CTRL_DIS);
-
-		ctrls->hist_stat = HIST_DISABLED;
-	}
-}
-
-static int isc_configure(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-	u32 pfe_cfg0, dcfg, mask, pipeline;
-	struct isc_subdev_entity *subdev = isc->current_subdev;
-
-	pfe_cfg0 = isc->config.sd_format->pfe_cfg0_bps;
-	pipeline = isc->config.bits_pipeline;
-
-	dcfg = isc->config.dcfg_imode | isc->dcfg;
-
-	pfe_cfg0  |= subdev->pfe_cfg0 | ISC_PFE_CFG0_MODE_PROGRESSIVE;
-	mask = ISC_PFE_CFG0_BPS_MASK | ISC_PFE_CFG0_HPOL_LOW |
-	       ISC_PFE_CFG0_VPOL_LOW | ISC_PFE_CFG0_PPOL_LOW |
-	       ISC_PFE_CFG0_MODE_MASK | ISC_PFE_CFG0_CCIR_CRC |
-	       ISC_PFE_CFG0_CCIR656 | ISC_PFE_CFG0_MIPI;
-
-	regmap_update_bits(regmap, ISC_PFE_CFG0, mask, pfe_cfg0);
-
-	isc->config_rlp(isc);
-
-	regmap_write(regmap, ISC_DCFG + isc->offsets.dma, dcfg);
-
-	/* Set the pipeline */
-	isc_set_pipeline(isc, pipeline);
-
-	/*
-	 * The current implemented histogram is available for RAW R, B, GB, GR
-	 * channels. We need to check if sensor is outputting RAW BAYER
-	 */
-	if (isc->ctrls.awb &&
-	    ISC_IS_FORMAT_RAW(isc->config.sd_format->mbus_code))
-		isc_set_histogram(isc, true);
-	else
-		isc_set_histogram(isc, false);
-
-	/* Update profile */
-	return isc_update_profile(isc);
-}
-
-static int isc_start_streaming(struct vb2_queue *vq, unsigned int count)
-{
-	struct isc_device *isc = vb2_get_drv_priv(vq);
-	struct regmap *regmap = isc->regmap;
-	struct isc_buffer *buf;
-	unsigned long flags;
-	int ret;
-
-	/* Enable stream on the sub device */
-	ret = v4l2_subdev_call(isc->current_subdev->sd, video, s_stream, 1);
-	if (ret && ret != -ENOIOCTLCMD) {
-		v4l2_err(&isc->v4l2_dev, "stream on failed in subdev %d\n",
-			 ret);
-		goto err_start_stream;
-	}
-
-	ret = pm_runtime_resume_and_get(isc->dev);
-	if (ret < 0) {
-		v4l2_err(&isc->v4l2_dev, "RPM resume failed in subdev %d\n",
-			 ret);
-		goto err_pm_get;
-	}
-
-	ret = isc_configure(isc);
-	if (unlikely(ret))
-		goto err_configure;
-
-	/* Enable DMA interrupt */
-	regmap_write(regmap, ISC_INTEN, ISC_INT_DDONE);
-
-	spin_lock_irqsave(&isc->dma_queue_lock, flags);
-
-	isc->sequence = 0;
-	isc->stop = false;
-	reinit_completion(&isc->comp);
-
-	isc->cur_frm = list_first_entry(&isc->dma_queue,
-					struct isc_buffer, list);
-	list_del(&isc->cur_frm->list);
-
-	isc_crop_pfe(isc);
-	isc_start_dma(isc);
-
-	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
-
-	/* if we streaming from RAW, we can do one-shot white balance adj */
-	if (ISC_IS_FORMAT_RAW(isc->config.sd_format->mbus_code))
-		v4l2_ctrl_activate(isc->do_wb_ctrl, true);
-
-	return 0;
-
-err_configure:
-	pm_runtime_put_sync(isc->dev);
-err_pm_get:
-	v4l2_subdev_call(isc->current_subdev->sd, video, s_stream, 0);
-
-err_start_stream:
-	spin_lock_irqsave(&isc->dma_queue_lock, flags);
-	list_for_each_entry(buf, &isc->dma_queue, list)
-		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_QUEUED);
-	INIT_LIST_HEAD(&isc->dma_queue);
-	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
-
-	return ret;
-}
-
-static void isc_stop_streaming(struct vb2_queue *vq)
-{
-	struct isc_device *isc = vb2_get_drv_priv(vq);
-	unsigned long flags;
-	struct isc_buffer *buf;
-	int ret;
-
-	mutex_lock(&isc->awb_mutex);
-	v4l2_ctrl_activate(isc->do_wb_ctrl, false);
-
-	isc->stop = true;
-
-	/* Wait until the end of the current frame */
-	if (isc->cur_frm && !wait_for_completion_timeout(&isc->comp, 5 * HZ))
-		v4l2_err(&isc->v4l2_dev,
-			 "Timeout waiting for end of the capture\n");
-
-	mutex_unlock(&isc->awb_mutex);
-
-	/* Disable DMA interrupt */
-	regmap_write(isc->regmap, ISC_INTDIS, ISC_INT_DDONE);
-
-	pm_runtime_put_sync(isc->dev);
-
-	/* Disable stream on the sub device */
-	ret = v4l2_subdev_call(isc->current_subdev->sd, video, s_stream, 0);
-	if (ret && ret != -ENOIOCTLCMD)
-		v4l2_err(&isc->v4l2_dev, "stream off failed in subdev\n");
-
-	/* Release all active buffers */
-	spin_lock_irqsave(&isc->dma_queue_lock, flags);
-	if (unlikely(isc->cur_frm)) {
-		vb2_buffer_done(&isc->cur_frm->vb.vb2_buf,
-				VB2_BUF_STATE_ERROR);
-		isc->cur_frm = NULL;
-	}
-	list_for_each_entry(buf, &isc->dma_queue, list)
-		vb2_buffer_done(&buf->vb.vb2_buf, VB2_BUF_STATE_ERROR);
-	INIT_LIST_HEAD(&isc->dma_queue);
-	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
-}
-
-static void isc_buffer_queue(struct vb2_buffer *vb)
-{
-	struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
-	struct isc_buffer *buf = container_of(vbuf, struct isc_buffer, vb);
-	struct isc_device *isc = vb2_get_drv_priv(vb->vb2_queue);
-	unsigned long flags;
-
-	spin_lock_irqsave(&isc->dma_queue_lock, flags);
-	if (!isc->cur_frm && list_empty(&isc->dma_queue) &&
-		vb2_start_streaming_called(vb->vb2_queue)) {
-		isc->cur_frm = buf;
-		isc_start_dma(isc);
-	} else
-		list_add_tail(&buf->list, &isc->dma_queue);
-	spin_unlock_irqrestore(&isc->dma_queue_lock, flags);
-}
-
-static struct isc_format *find_format_by_fourcc(struct isc_device *isc,
-						 unsigned int fourcc)
-{
-	unsigned int num_formats = isc->num_user_formats;
-	struct isc_format *fmt;
-	unsigned int i;
-
-	for (i = 0; i < num_formats; i++) {
-		fmt = isc->user_formats[i];
-		if (fmt->fourcc == fourcc)
-			return fmt;
-	}
-
-	return NULL;
-}
-
-static const struct vb2_ops isc_vb2_ops = {
-	.queue_setup		= isc_queue_setup,
-	.buf_prepare		= isc_buffer_prepare,
-	.start_streaming	= isc_start_streaming,
-	.stop_streaming		= isc_stop_streaming,
-	.buf_queue		= isc_buffer_queue,
-};
-
-static int isc_querycap(struct file *file, void *priv,
-			 struct v4l2_capability *cap)
-{
-	strscpy(cap->driver, "microchip-isc", sizeof(cap->driver));
-	strscpy(cap->card, "Atmel Image Sensor Controller", sizeof(cap->card));
-
-	return 0;
-}
-
-static int isc_enum_fmt_vid_cap(struct file *file, void *priv,
-				 struct v4l2_fmtdesc *f)
-{
-	struct isc_device *isc = video_drvdata(file);
-	u32 index = f->index;
-	u32 i, supported_index;
-
-	if (index < isc->controller_formats_size) {
-		f->pixelformat = isc->controller_formats[index].fourcc;
-		return 0;
-	}
-
-	index -= isc->controller_formats_size;
-
-	supported_index = 0;
-
-	for (i = 0; i < isc->formats_list_size; i++) {
-		if (!ISC_IS_FORMAT_RAW(isc->formats_list[i].mbus_code) ||
-		    !isc->formats_list[i].sd_support)
-			continue;
-		if (supported_index == index) {
-			f->pixelformat = isc->formats_list[i].fourcc;
-			return 0;
-		}
-		supported_index++;
-	}
-
-	return -EINVAL;
-}
-
-static int isc_g_fmt_vid_cap(struct file *file, void *priv,
-			      struct v4l2_format *fmt)
-{
-	struct isc_device *isc = video_drvdata(file);
-
-	*fmt = isc->fmt;
-
-	return 0;
-}
-
-/*
- * Checks the current configured format, if ISC can output it,
- * considering which type of format the ISC receives from the sensor
- */
-static int isc_try_validate_formats(struct isc_device *isc)
-{
-	int ret;
-	bool bayer = false, yuv = false, rgb = false, grey = false;
-
-	/* all formats supported by the RLP module are OK */
-	switch (isc->try_config.fourcc) {
-	case V4L2_PIX_FMT_SBGGR8:
-	case V4L2_PIX_FMT_SGBRG8:
-	case V4L2_PIX_FMT_SGRBG8:
-	case V4L2_PIX_FMT_SRGGB8:
-	case V4L2_PIX_FMT_SBGGR10:
-	case V4L2_PIX_FMT_SGBRG10:
-	case V4L2_PIX_FMT_SGRBG10:
-	case V4L2_PIX_FMT_SRGGB10:
-	case V4L2_PIX_FMT_SBGGR12:
-	case V4L2_PIX_FMT_SGBRG12:
-	case V4L2_PIX_FMT_SGRBG12:
-	case V4L2_PIX_FMT_SRGGB12:
-		ret = 0;
-		bayer = true;
-		break;
-
-	case V4L2_PIX_FMT_YUV420:
-	case V4L2_PIX_FMT_YUV422P:
-	case V4L2_PIX_FMT_YUYV:
-	case V4L2_PIX_FMT_UYVY:
-	case V4L2_PIX_FMT_VYUY:
-		ret = 0;
-		yuv = true;
-		break;
-
-	case V4L2_PIX_FMT_RGB565:
-	case V4L2_PIX_FMT_ABGR32:
-	case V4L2_PIX_FMT_XBGR32:
-	case V4L2_PIX_FMT_ARGB444:
-	case V4L2_PIX_FMT_ARGB555:
-		ret = 0;
-		rgb = true;
-		break;
-	case V4L2_PIX_FMT_GREY:
-	case V4L2_PIX_FMT_Y10:
-	case V4L2_PIX_FMT_Y16:
-		ret = 0;
-		grey = true;
-		break;
-	default:
-	/* any other different formats are not supported */
-		ret = -EINVAL;
-	}
-	v4l2_dbg(1, debug, &isc->v4l2_dev,
-		 "Format validation, requested rgb=%u, yuv=%u, grey=%u, bayer=%u\n",
-		 rgb, yuv, grey, bayer);
-
-	/* we cannot output RAW if we do not receive RAW */
-	if ((bayer) && !ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code))
-		return -EINVAL;
-
-	/* we cannot output GREY if we do not receive RAW/GREY */
-	if (grey && !ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code) &&
-	    !ISC_IS_FORMAT_GREY(isc->try_config.sd_format->mbus_code))
-		return -EINVAL;
-
-	return ret;
-}
-
-/*
- * Configures the RLP and DMA modules, depending on the output format
- * configured for the ISC.
- * If direct_dump == true, just dump raw data 8/16 bits depending on format.
- */
-static int isc_try_configure_rlp_dma(struct isc_device *isc, bool direct_dump)
-{
-	isc->try_config.rlp_cfg_mode = 0;
-
-	switch (isc->try_config.fourcc) {
-	case V4L2_PIX_FMT_SBGGR8:
-	case V4L2_PIX_FMT_SGBRG8:
-	case V4L2_PIX_FMT_SGRBG8:
-	case V4L2_PIX_FMT_SRGGB8:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT8;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED8;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 8;
-		isc->try_config.bpp_v4l2 = 8;
-		break;
-	case V4L2_PIX_FMT_SBGGR10:
-	case V4L2_PIX_FMT_SGBRG10:
-	case V4L2_PIX_FMT_SGRBG10:
-	case V4L2_PIX_FMT_SRGGB10:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT10;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 16;
-		break;
-	case V4L2_PIX_FMT_SBGGR12:
-	case V4L2_PIX_FMT_SGBRG12:
-	case V4L2_PIX_FMT_SGRBG12:
-	case V4L2_PIX_FMT_SRGGB12:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT12;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 16;
-		break;
-	case V4L2_PIX_FMT_RGB565:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_RGB565;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 16;
-		break;
-	case V4L2_PIX_FMT_ARGB444:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_ARGB444;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 16;
-		break;
-	case V4L2_PIX_FMT_ARGB555:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_ARGB555;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 16;
-		break;
-	case V4L2_PIX_FMT_ABGR32:
-	case V4L2_PIX_FMT_XBGR32:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_ARGB32;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED32;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 32;
-		isc->try_config.bpp_v4l2 = 32;
-		break;
-	case V4L2_PIX_FMT_YUV420:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YYCC;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_YC420P;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PLANAR;
-		isc->try_config.bpp = 12;
-		isc->try_config.bpp_v4l2 = 8; /* only first plane */
-		break;
-	case V4L2_PIX_FMT_YUV422P:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YYCC;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_YC422P;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PLANAR;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 8; /* only first plane */
-		break;
-	case V4L2_PIX_FMT_YUYV:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YCYC | ISC_RLP_CFG_YMODE_YUYV;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED32;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 16;
-		break;
-	case V4L2_PIX_FMT_UYVY:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YCYC | ISC_RLP_CFG_YMODE_UYVY;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED32;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 16;
-		break;
-	case V4L2_PIX_FMT_VYUY:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YCYC | ISC_RLP_CFG_YMODE_VYUY;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED32;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 16;
-		break;
-	case V4L2_PIX_FMT_GREY:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DATY8;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED8;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 8;
-		isc->try_config.bpp_v4l2 = 8;
-		break;
-	case V4L2_PIX_FMT_Y16:
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DATY10 | ISC_RLP_CFG_LSH;
-		fallthrough;
-	case V4L2_PIX_FMT_Y10:
-		isc->try_config.rlp_cfg_mode |= ISC_RLP_CFG_MODE_DATY10;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		isc->try_config.bpp = 16;
-		isc->try_config.bpp_v4l2 = 16;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	if (direct_dump) {
-		isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DAT8;
-		isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED8;
-		isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED;
-		return 0;
-	}
-
-	return 0;
-}
-
-/*
- * Configuring pipeline modules, depending on which format the ISC outputs
- * and considering which format it has as input from the sensor.
- */
-static int isc_try_configure_pipeline(struct isc_device *isc)
-{
-	switch (isc->try_config.fourcc) {
-	case V4L2_PIX_FMT_RGB565:
-	case V4L2_PIX_FMT_ARGB555:
-	case V4L2_PIX_FMT_ARGB444:
-	case V4L2_PIX_FMT_ABGR32:
-	case V4L2_PIX_FMT_XBGR32:
-		/* if sensor format is RAW, we convert inside ISC */
-		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
-			isc->try_config.bits_pipeline = CFA_ENABLE |
-				WB_ENABLE | GAM_ENABLES | DPC_BLCENABLE |
-				CC_ENABLE;
-		} else {
-			isc->try_config.bits_pipeline = 0x0;
-		}
-		break;
-	case V4L2_PIX_FMT_YUV420:
-		/* if sensor format is RAW, we convert inside ISC */
-		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
-			isc->try_config.bits_pipeline = CFA_ENABLE |
-				CSC_ENABLE | GAM_ENABLES | WB_ENABLE |
-				SUB420_ENABLE | SUB422_ENABLE | CBC_ENABLE |
-				DPC_BLCENABLE;
-		} else {
-			isc->try_config.bits_pipeline = 0x0;
-		}
-		break;
-	case V4L2_PIX_FMT_YUV422P:
-		/* if sensor format is RAW, we convert inside ISC */
-		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
-			isc->try_config.bits_pipeline = CFA_ENABLE |
-				CSC_ENABLE | WB_ENABLE | GAM_ENABLES |
-				SUB422_ENABLE | CBC_ENABLE | DPC_BLCENABLE;
-		} else {
-			isc->try_config.bits_pipeline = 0x0;
-		}
-		break;
-	case V4L2_PIX_FMT_YUYV:
-	case V4L2_PIX_FMT_UYVY:
-	case V4L2_PIX_FMT_VYUY:
-		/* if sensor format is RAW, we convert inside ISC */
-		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
-			isc->try_config.bits_pipeline = CFA_ENABLE |
-				CSC_ENABLE | WB_ENABLE | GAM_ENABLES |
-				SUB422_ENABLE | CBC_ENABLE | DPC_BLCENABLE;
-		} else {
-			isc->try_config.bits_pipeline = 0x0;
-		}
-		break;
-	case V4L2_PIX_FMT_GREY:
-	case V4L2_PIX_FMT_Y16:
-		/* if sensor format is RAW, we convert inside ISC */
-		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) {
-			isc->try_config.bits_pipeline = CFA_ENABLE |
-				CSC_ENABLE | WB_ENABLE | GAM_ENABLES |
-				CBC_ENABLE | DPC_BLCENABLE;
-		} else {
-			isc->try_config.bits_pipeline = 0x0;
-		}
-		break;
-	default:
-		if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code))
-			isc->try_config.bits_pipeline = WB_ENABLE | DPC_BLCENABLE;
-		else
-			isc->try_config.bits_pipeline = 0x0;
-	}
-
-	/* Tune the pipeline to product specific */
-	isc->adapt_pipeline(isc);
-
-	return 0;
-}
-
-static void isc_try_fse(struct isc_device *isc,
-			struct v4l2_subdev_state *sd_state)
-{
-	struct v4l2_rect *try_crop =
-		v4l2_subdev_state_get_crop(sd_state, 0);
-	struct v4l2_subdev_frame_size_enum fse = {
-		.which = V4L2_SUBDEV_FORMAT_TRY,
-	};
-	int ret;
-
-	/*
-	 * If we do not know yet which format the subdev is using, we cannot
-	 * do anything.
-	 */
-	if (!isc->try_config.sd_format)
-		return;
-
-	fse.code = isc->try_config.sd_format->mbus_code;
-
-	ret = v4l2_subdev_call(isc->current_subdev->sd, pad, enum_frame_size,
-			       sd_state, &fse);
-	/*
-	 * Attempt to obtain format size from subdev. If not available,
-	 * just use the maximum ISC can receive.
-	 */
-	if (ret) {
-		try_crop->width = isc->max_width;
-		try_crop->height = isc->max_height;
-	} else {
-		try_crop->width = fse.max_width;
-		try_crop->height = fse.max_height;
-	}
-}
-
-static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f,
-			u32 *code)
-{
-	int i;
-	struct isc_format *sd_fmt = NULL, *direct_fmt = NULL;
-	struct v4l2_pix_format *pixfmt = &f->fmt.pix;
-	struct v4l2_subdev_pad_config pad_cfg = {};
-	struct v4l2_subdev_state pad_state = {
-		.pads = &pad_cfg,
-	};
-	struct v4l2_subdev_format format = {
-		.which = V4L2_SUBDEV_FORMAT_TRY,
-	};
-	u32 mbus_code;
-	int ret;
-	bool rlp_dma_direct_dump = false;
-
-	if (f->type != V4L2_BUF_TYPE_VIDEO_CAPTURE)
-		return -EINVAL;
-
-	/* Step 1: find a RAW format that is supported */
-	for (i = 0; i < isc->num_user_formats; i++) {
-		if (ISC_IS_FORMAT_RAW(isc->user_formats[i]->mbus_code)) {
-			sd_fmt = isc->user_formats[i];
-			break;
-		}
-	}
-	/* Step 2: We can continue with this RAW format, or we can look
-	 * for better: maybe sensor supports directly what we need.
-	 */
-	direct_fmt = find_format_by_fourcc(isc, pixfmt->pixelformat);
-
-	/* Step 3: We have both. We decide given the module parameter which
-	 * one to use.
-	 */
-	if (direct_fmt && sd_fmt && sensor_preferred)
-		sd_fmt = direct_fmt;
-
-	/* Step 4: we do not have RAW but we have a direct format. Use it. */
-	if (direct_fmt && !sd_fmt)
-		sd_fmt = direct_fmt;
-
-	/* Step 5: if we are using a direct format, we need to package
-	 * everything as 8 bit data and just dump it
-	 */
-	if (sd_fmt == direct_fmt)
-		rlp_dma_direct_dump = true;
-
-	/* Step 6: We have no format. This can happen if the userspace
-	 * requests some weird/invalid format.
-	 * In this case, default to whatever we have
-	 */
-	if (!sd_fmt && !direct_fmt) {
-		sd_fmt = isc->user_formats[isc->num_user_formats - 1];
-		v4l2_dbg(1, debug, &isc->v4l2_dev,
-			 "Sensor not supporting %.4s, using %.4s\n",
-			 (char *)&pixfmt->pixelformat, (char *)&sd_fmt->fourcc);
-	}
-
-	if (!sd_fmt) {
-		ret = -EINVAL;
-		goto isc_try_fmt_err;
-	}
-
-	/* Step 7: Print out what we decided for debugging */
-	v4l2_dbg(1, debug, &isc->v4l2_dev,
-		 "Preferring to have sensor using format %.4s\n",
-		 (char *)&sd_fmt->fourcc);
-
-	/* Step 8: at this moment we decided which format the subdev will use */
-	isc->try_config.sd_format = sd_fmt;
-
-	/* Limit to Atmel ISC hardware capabilities */
-	if (pixfmt->width > isc->max_width)
-		pixfmt->width = isc->max_width;
-	if (pixfmt->height > isc->max_height)
-		pixfmt->height = isc->max_height;
-
-	/*
-	 * The mbus format is the one the subdev outputs.
-	 * The pixels will be transferred in this format Sensor -> ISC
-	 */
-	mbus_code = sd_fmt->mbus_code;
-
-	/*
-	 * Validate formats. If the required format is not OK, default to raw.
-	 */
-
-	isc->try_config.fourcc = pixfmt->pixelformat;
-
-	if (isc_try_validate_formats(isc)) {
-		pixfmt->pixelformat = isc->try_config.fourcc = sd_fmt->fourcc;
-		/* Re-try to validate the new format */
-		ret = isc_try_validate_formats(isc);
-		if (ret)
-			goto isc_try_fmt_err;
-	}
-
-	ret = isc_try_configure_rlp_dma(isc, rlp_dma_direct_dump);
-	if (ret)
-		goto isc_try_fmt_err;
-
-	ret = isc_try_configure_pipeline(isc);
-	if (ret)
-		goto isc_try_fmt_err;
-
-	/* Obtain frame sizes if possible to have crop requirements ready */
-	isc_try_fse(isc, &pad_state);
-
-	v4l2_fill_mbus_format(&format.format, pixfmt, mbus_code);
-	ret = v4l2_subdev_call(isc->current_subdev->sd, pad, set_fmt,
-			       &pad_state, &format);
-	if (ret < 0)
-		goto isc_try_fmt_subdev_err;
-
-	v4l2_fill_pix_format(pixfmt, &format.format);
-
-	/* Limit to Atmel ISC hardware capabilities */
-	if (pixfmt->width > isc->max_width)
-		pixfmt->width = isc->max_width;
-	if (pixfmt->height > isc->max_height)
-		pixfmt->height = isc->max_height;
-
-	pixfmt->field = V4L2_FIELD_NONE;
-	pixfmt->bytesperline = (pixfmt->width * isc->try_config.bpp_v4l2) >> 3;
-	pixfmt->sizeimage = ((pixfmt->width * isc->try_config.bpp) >> 3) *
-			     pixfmt->height;
-
-	if (code)
-		*code = mbus_code;
-
-	return 0;
-
-isc_try_fmt_err:
-	v4l2_err(&isc->v4l2_dev, "Could not find any possible format for a working pipeline\n");
-isc_try_fmt_subdev_err:
-	memset(&isc->try_config, 0, sizeof(isc->try_config));
-
-	return ret;
-}
-
-static int isc_set_fmt(struct isc_device *isc, struct v4l2_format *f)
-{
-	struct v4l2_subdev_format format = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-	u32 mbus_code = 0;
-	int ret;
-
-	ret = isc_try_fmt(isc, f, &mbus_code);
-	if (ret)
-		return ret;
-
-	v4l2_fill_mbus_format(&format.format, &f->fmt.pix, mbus_code);
-	ret = v4l2_subdev_call(isc->current_subdev->sd, pad,
-			       set_fmt, NULL, &format);
-	if (ret < 0)
-		return ret;
-
-	/* Limit to Atmel ISC hardware capabilities */
-	if (f->fmt.pix.width > isc->max_width)
-		f->fmt.pix.width = isc->max_width;
-	if (f->fmt.pix.height > isc->max_height)
-		f->fmt.pix.height = isc->max_height;
-
-	isc->fmt = *f;
-
-	if (isc->try_config.sd_format && isc->config.sd_format &&
-	    isc->try_config.sd_format != isc->config.sd_format) {
-		isc->ctrls.hist_stat = HIST_INIT;
-		isc_reset_awb_ctrls(isc);
-		isc_update_v4l2_ctrls(isc);
-	}
-	/* make the try configuration active */
-	isc->config = isc->try_config;
-
-	v4l2_dbg(1, debug, &isc->v4l2_dev, "New ISC configuration in place\n");
-
-	return 0;
-}
-
-static int isc_s_fmt_vid_cap(struct file *file, void *priv,
-			      struct v4l2_format *f)
-{
-	struct isc_device *isc = video_drvdata(file);
-
-	if (vb2_is_busy(&isc->vb2_vidq))
-		return -EBUSY;
-
-	return isc_set_fmt(isc, f);
-}
-
-static int isc_try_fmt_vid_cap(struct file *file, void *priv,
-				struct v4l2_format *f)
-{
-	struct isc_device *isc = video_drvdata(file);
-
-	return isc_try_fmt(isc, f, NULL);
-}
-
-static int isc_enum_input(struct file *file, void *priv,
-			   struct v4l2_input *inp)
-{
-	if (inp->index != 0)
-		return -EINVAL;
-
-	inp->type = V4L2_INPUT_TYPE_CAMERA;
-	inp->std = 0;
-	strscpy(inp->name, "Camera", sizeof(inp->name));
-
-	return 0;
-}
-
-static int isc_g_input(struct file *file, void *priv, unsigned int *i)
-{
-	*i = 0;
-
-	return 0;
-}
-
-static int isc_s_input(struct file *file, void *priv, unsigned int i)
-{
-	if (i > 0)
-		return -EINVAL;
-
-	return 0;
-}
-
-static int isc_g_parm(struct file *file, void *fh, struct v4l2_streamparm *a)
-{
-	struct isc_device *isc = video_drvdata(file);
-
-	return v4l2_g_parm_cap(video_devdata(file), isc->current_subdev->sd, a);
-}
-
-static int isc_s_parm(struct file *file, void *fh, struct v4l2_streamparm *a)
-{
-	struct isc_device *isc = video_drvdata(file);
-
-	return v4l2_s_parm_cap(video_devdata(file), isc->current_subdev->sd, a);
-}
-
-static int isc_enum_framesizes(struct file *file, void *fh,
-			       struct v4l2_frmsizeenum *fsize)
-{
-	struct isc_device *isc = video_drvdata(file);
-	int ret = -EINVAL;
-	int i;
-
-	if (fsize->index)
-		return -EINVAL;
-
-	for (i = 0; i < isc->num_user_formats; i++)
-		if (isc->user_formats[i]->fourcc == fsize->pixel_format)
-			ret = 0;
-
-	for (i = 0; i < isc->controller_formats_size; i++)
-		if (isc->controller_formats[i].fourcc == fsize->pixel_format)
-			ret = 0;
-
-	if (ret)
-		return ret;
-
-	fsize->type = V4L2_FRMSIZE_TYPE_CONTINUOUS;
-
-	fsize->stepwise.min_width = 16;
-	fsize->stepwise.max_width = isc->max_width;
-	fsize->stepwise.min_height = 16;
-	fsize->stepwise.max_height = isc->max_height;
-	fsize->stepwise.step_width = 1;
-	fsize->stepwise.step_height = 1;
-
-	return 0;
-}
-
-static const struct v4l2_ioctl_ops isc_ioctl_ops = {
-	.vidioc_querycap		= isc_querycap,
-	.vidioc_enum_fmt_vid_cap	= isc_enum_fmt_vid_cap,
-	.vidioc_g_fmt_vid_cap		= isc_g_fmt_vid_cap,
-	.vidioc_s_fmt_vid_cap		= isc_s_fmt_vid_cap,
-	.vidioc_try_fmt_vid_cap		= isc_try_fmt_vid_cap,
-
-	.vidioc_enum_input		= isc_enum_input,
-	.vidioc_g_input			= isc_g_input,
-	.vidioc_s_input			= isc_s_input,
-
-	.vidioc_reqbufs			= vb2_ioctl_reqbufs,
-	.vidioc_querybuf		= vb2_ioctl_querybuf,
-	.vidioc_qbuf			= vb2_ioctl_qbuf,
-	.vidioc_expbuf			= vb2_ioctl_expbuf,
-	.vidioc_dqbuf			= vb2_ioctl_dqbuf,
-	.vidioc_create_bufs		= vb2_ioctl_create_bufs,
-	.vidioc_prepare_buf		= vb2_ioctl_prepare_buf,
-	.vidioc_streamon		= vb2_ioctl_streamon,
-	.vidioc_streamoff		= vb2_ioctl_streamoff,
-
-	.vidioc_g_parm			= isc_g_parm,
-	.vidioc_s_parm			= isc_s_parm,
-	.vidioc_enum_framesizes		= isc_enum_framesizes,
-
-	.vidioc_log_status		= v4l2_ctrl_log_status,
-	.vidioc_subscribe_event		= v4l2_ctrl_subscribe_event,
-	.vidioc_unsubscribe_event	= v4l2_event_unsubscribe,
-};
-
-static int isc_open(struct file *file)
-{
-	struct isc_device *isc = video_drvdata(file);
-	struct v4l2_subdev *sd = isc->current_subdev->sd;
-	int ret;
-
-	if (mutex_lock_interruptible(&isc->lock))
-		return -ERESTARTSYS;
-
-	ret = v4l2_fh_open(file);
-	if (ret < 0)
-		goto unlock;
-
-	if (!v4l2_fh_is_singular_file(file))
-		goto unlock;
-
-	ret = v4l2_subdev_call(sd, core, s_power, 1);
-	if (ret < 0 && ret != -ENOIOCTLCMD) {
-		v4l2_fh_release(file);
-		goto unlock;
-	}
-
-	ret = isc_set_fmt(isc, &isc->fmt);
-	if (ret) {
-		v4l2_subdev_call(sd, core, s_power, 0);
-		v4l2_fh_release(file);
-	}
-
-unlock:
-	mutex_unlock(&isc->lock);
-	return ret;
-}
-
-static int isc_release(struct file *file)
-{
-	struct isc_device *isc = video_drvdata(file);
-	struct v4l2_subdev *sd = isc->current_subdev->sd;
-	bool fh_singular;
-	int ret;
-
-	mutex_lock(&isc->lock);
-
-	fh_singular = v4l2_fh_is_singular_file(file);
-
-	ret = _vb2_fop_release(file, NULL);
-
-	if (fh_singular)
-		v4l2_subdev_call(sd, core, s_power, 0);
-
-	mutex_unlock(&isc->lock);
-
-	return ret;
-}
-
-static const struct v4l2_file_operations isc_fops = {
-	.owner		= THIS_MODULE,
-	.open		= isc_open,
-	.release	= isc_release,
-	.unlocked_ioctl	= video_ioctl2,
-	.read		= vb2_fop_read,
-	.mmap		= vb2_fop_mmap,
-	.poll		= vb2_fop_poll,
-};
-
-irqreturn_t atmel_isc_interrupt(int irq, void *dev_id)
-{
-	struct isc_device *isc = (struct isc_device *)dev_id;
-	struct regmap *regmap = isc->regmap;
-	u32 isc_intsr, isc_intmask, pending;
-	irqreturn_t ret = IRQ_NONE;
-
-	regmap_read(regmap, ISC_INTSR, &isc_intsr);
-	regmap_read(regmap, ISC_INTMASK, &isc_intmask);
-
-	pending = isc_intsr & isc_intmask;
-
-	if (likely(pending & ISC_INT_DDONE)) {
-		spin_lock(&isc->dma_queue_lock);
-		if (isc->cur_frm) {
-			struct vb2_v4l2_buffer *vbuf = &isc->cur_frm->vb;
-			struct vb2_buffer *vb = &vbuf->vb2_buf;
-
-			vb->timestamp = ktime_get_ns();
-			vbuf->sequence = isc->sequence++;
-			vb2_buffer_done(vb, VB2_BUF_STATE_DONE);
-			isc->cur_frm = NULL;
-		}
-
-		if (!list_empty(&isc->dma_queue) && !isc->stop) {
-			isc->cur_frm = list_first_entry(&isc->dma_queue,
-						     struct isc_buffer, list);
-			list_del(&isc->cur_frm->list);
-
-			isc_start_dma(isc);
-		}
-
-		if (isc->stop)
-			complete(&isc->comp);
-
-		ret = IRQ_HANDLED;
-		spin_unlock(&isc->dma_queue_lock);
-	}
-
-	if (pending & ISC_INT_HISDONE) {
-		schedule_work(&isc->awb_work);
-		ret = IRQ_HANDLED;
-	}
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(atmel_isc_interrupt);
-
-static void isc_hist_count(struct isc_device *isc, u32 *min, u32 *max)
-{
-	struct regmap *regmap = isc->regmap;
-	struct isc_ctrls *ctrls = &isc->ctrls;
-	u32 *hist_count = &ctrls->hist_count[ctrls->hist_id];
-	u32 *hist_entry = &ctrls->hist_entry[0];
-	u32 i;
-
-	*min = 0;
-	*max = HIST_ENTRIES;
-
-	regmap_bulk_read(regmap, ISC_HIS_ENTRY + isc->offsets.his_entry,
-			 hist_entry, HIST_ENTRIES);
-
-	*hist_count = 0;
-	/*
-	 * we deliberately ignore the end of the histogram,
-	 * the most white pixels
-	 */
-	for (i = 1; i < HIST_ENTRIES; i++) {
-		if (*hist_entry && !*min)
-			*min = i;
-		if (*hist_entry)
-			*max = i;
-		*hist_count += i * (*hist_entry++);
-	}
-
-	if (!*min)
-		*min = 1;
-
-	v4l2_dbg(1, debug, &isc->v4l2_dev,
-		 "isc wb: hist_id %u, hist_count %u",
-		 ctrls->hist_id, *hist_count);
-}
-
-static void isc_wb_update(struct isc_ctrls *ctrls)
-{
-	struct isc_device *isc = container_of(ctrls, struct isc_device, ctrls);
-	u32 *hist_count = &ctrls->hist_count[0];
-	u32 c, offset[4];
-	u64 avg = 0;
-	/* We compute two gains, stretch gain and grey world gain */
-	u32 s_gain[4], gw_gain[4];
-
-	/*
-	 * According to Grey World, we need to set gains for R/B to normalize
-	 * them towards the green channel.
-	 * Thus we want to keep Green as fixed and adjust only Red/Blue
-	 * Compute the average of the both green channels first
-	 */
-	avg = (u64)hist_count[ISC_HIS_CFG_MODE_GR] +
-		(u64)hist_count[ISC_HIS_CFG_MODE_GB];
-	avg >>= 1;
-
-	v4l2_dbg(1, debug, &isc->v4l2_dev,
-		 "isc wb: green components average %llu\n", avg);
-
-	/* Green histogram is null, nothing to do */
-	if (!avg)
-		return;
-
-	for (c = ISC_HIS_CFG_MODE_GR; c <= ISC_HIS_CFG_MODE_B; c++) {
-		/*
-		 * the color offset is the minimum value of the histogram.
-		 * we stretch this color to the full range by substracting
-		 * this value from the color component.
-		 */
-		offset[c] = ctrls->hist_minmax[c][HIST_MIN_INDEX];
-		/*
-		 * The offset is always at least 1. If the offset is 1, we do
-		 * not need to adjust it, so our result must be zero.
-		 * the offset is computed in a histogram on 9 bits (0..512)
-		 * but the offset in register is based on
-		 * 12 bits pipeline (0..4096).
-		 * we need to shift with the 3 bits that the histogram is
-		 * ignoring
-		 */
-		ctrls->offset[c] = (offset[c] - 1) << 3;
-
-		/*
-		 * the offset is then taken and converted to 2's complements,
-		 * and must be negative, as we subtract this value from the
-		 * color components
-		 */
-		ctrls->offset[c] = -ctrls->offset[c];
-
-		/*
-		 * the stretch gain is the total number of histogram bins
-		 * divided by the actual range of color component (Max - Min)
-		 * If we compute gain like this, the actual color component
-		 * will be stretched to the full histogram.
-		 * We need to shift 9 bits for precision, we have 9 bits for
-		 * decimals
-		 */
-		s_gain[c] = (HIST_ENTRIES << 9) /
-			(ctrls->hist_minmax[c][HIST_MAX_INDEX] -
-			ctrls->hist_minmax[c][HIST_MIN_INDEX] + 1);
-
-		/*
-		 * Now we have to compute the gain w.r.t. the average.
-		 * Add/lose gain to the component towards the average.
-		 * If it happens that the component is zero, use the
-		 * fixed point value : 1.0 gain.
-		 */
-		if (hist_count[c])
-			gw_gain[c] = div_u64(avg << 9, hist_count[c]);
-		else
-			gw_gain[c] = 1 << 9;
-
-		v4l2_dbg(1, debug, &isc->v4l2_dev,
-			 "isc wb: component %d, s_gain %u, gw_gain %u\n",
-			 c, s_gain[c], gw_gain[c]);
-		/* multiply both gains and adjust for decimals */
-		ctrls->gain[c] = s_gain[c] * gw_gain[c];
-		ctrls->gain[c] >>= 9;
-
-		/* make sure we are not out of range */
-		ctrls->gain[c] = clamp_val(ctrls->gain[c], 0, GENMASK(12, 0));
-
-		v4l2_dbg(1, debug, &isc->v4l2_dev,
-			 "isc wb: component %d, final gain %u\n",
-			 c, ctrls->gain[c]);
-	}
-}
-
-static void isc_awb_work(struct work_struct *w)
-{
-	struct isc_device *isc =
-		container_of(w, struct isc_device, awb_work);
-	struct regmap *regmap = isc->regmap;
-	struct isc_ctrls *ctrls = &isc->ctrls;
-	u32 hist_id = ctrls->hist_id;
-	u32 baysel;
-	unsigned long flags;
-	u32 min, max;
-	int ret;
-
-	if (ctrls->hist_stat != HIST_ENABLED)
-		return;
-
-	isc_hist_count(isc, &min, &max);
-
-	v4l2_dbg(1, debug, &isc->v4l2_dev,
-		 "isc wb mode %d: hist min %u , max %u\n", hist_id, min, max);
-
-	ctrls->hist_minmax[hist_id][HIST_MIN_INDEX] = min;
-	ctrls->hist_minmax[hist_id][HIST_MAX_INDEX] = max;
-
-	if (hist_id != ISC_HIS_CFG_MODE_B) {
-		hist_id++;
-	} else {
-		isc_wb_update(ctrls);
-		hist_id = ISC_HIS_CFG_MODE_GR;
-	}
-
-	ctrls->hist_id = hist_id;
-	baysel = isc->config.sd_format->cfa_baycfg << ISC_HIS_CFG_BAYSEL_SHIFT;
-
-	ret = pm_runtime_resume_and_get(isc->dev);
-	if (ret < 0)
-		return;
-
-	/*
-	 * only update if we have all the required histograms and controls
-	 * if awb has been disabled, we need to reset registers as well.
-	 */
-	if (hist_id == ISC_HIS_CFG_MODE_GR || ctrls->awb == ISC_WB_NONE) {
-		/*
-		 * It may happen that DMA Done IRQ will trigger while we are
-		 * updating white balance registers here.
-		 * In that case, only parts of the controls have been updated.
-		 * We can avoid that by locking the section.
-		 */
-		spin_lock_irqsave(&isc->awb_lock, flags);
-		isc_update_awb_ctrls(isc);
-		spin_unlock_irqrestore(&isc->awb_lock, flags);
-
-		/*
-		 * if we are doing just the one time white balance adjustment,
-		 * we are basically done.
-		 */
-		if (ctrls->awb == ISC_WB_ONETIME) {
-			v4l2_info(&isc->v4l2_dev,
-				  "Completed one time white-balance adjustment.\n");
-			/* update the v4l2 controls values */
-			isc_update_v4l2_ctrls(isc);
-			ctrls->awb = ISC_WB_NONE;
-		}
-	}
-	regmap_write(regmap, ISC_HIS_CFG + isc->offsets.his,
-		     hist_id | baysel | ISC_HIS_CFG_RAR);
-
-	/*
-	 * We have to make sure the streaming has not stopped meanwhile.
-	 * ISC requires a frame to clock the internal profile update.
-	 * To avoid issues, lock the sequence with a mutex
-	 */
-	mutex_lock(&isc->awb_mutex);
-
-	/* streaming is not active anymore */
-	if (isc->stop) {
-		mutex_unlock(&isc->awb_mutex);
-		return;
-	}
-
-	isc_update_profile(isc);
-
-	mutex_unlock(&isc->awb_mutex);
-
-	/* if awb has been disabled, we don't need to start another histogram */
-	if (ctrls->awb)
-		regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_HISREQ);
-
-	pm_runtime_put_sync(isc->dev);
-}
-
-static int isc_s_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct isc_device *isc = container_of(ctrl->handler,
-					     struct isc_device, ctrls.handler);
-	struct isc_ctrls *ctrls = &isc->ctrls;
-
-	if (ctrl->flags & V4L2_CTRL_FLAG_INACTIVE)
-		return 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_BRIGHTNESS:
-		ctrls->brightness = ctrl->val & ISC_CBC_BRIGHT_MASK;
-		break;
-	case V4L2_CID_CONTRAST:
-		ctrls->contrast = ctrl->val & ISC_CBC_CONTRAST_MASK;
-		break;
-	case V4L2_CID_GAMMA:
-		ctrls->gamma_index = ctrl->val;
-		break;
-	default:
-		return -EINVAL;
-	}
-
-	return 0;
-}
-
-static const struct v4l2_ctrl_ops isc_ctrl_ops = {
-	.s_ctrl	= isc_s_ctrl,
-};
-
-static int isc_s_awb_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct isc_device *isc = container_of(ctrl->handler,
-					     struct isc_device, ctrls.handler);
-	struct isc_ctrls *ctrls = &isc->ctrls;
-
-	if (ctrl->flags & V4L2_CTRL_FLAG_INACTIVE)
-		return 0;
-
-	switch (ctrl->id) {
-	case V4L2_CID_AUTO_WHITE_BALANCE:
-		if (ctrl->val == 1)
-			ctrls->awb = ISC_WB_AUTO;
-		else
-			ctrls->awb = ISC_WB_NONE;
-
-		/* configure the controls with new values from v4l2 */
-		if (ctrl->cluster[ISC_CTRL_R_GAIN]->is_new)
-			ctrls->gain[ISC_HIS_CFG_MODE_R] = isc->r_gain_ctrl->val;
-		if (ctrl->cluster[ISC_CTRL_B_GAIN]->is_new)
-			ctrls->gain[ISC_HIS_CFG_MODE_B] = isc->b_gain_ctrl->val;
-		if (ctrl->cluster[ISC_CTRL_GR_GAIN]->is_new)
-			ctrls->gain[ISC_HIS_CFG_MODE_GR] = isc->gr_gain_ctrl->val;
-		if (ctrl->cluster[ISC_CTRL_GB_GAIN]->is_new)
-			ctrls->gain[ISC_HIS_CFG_MODE_GB] = isc->gb_gain_ctrl->val;
-
-		if (ctrl->cluster[ISC_CTRL_R_OFF]->is_new)
-			ctrls->offset[ISC_HIS_CFG_MODE_R] = isc->r_off_ctrl->val;
-		if (ctrl->cluster[ISC_CTRL_B_OFF]->is_new)
-			ctrls->offset[ISC_HIS_CFG_MODE_B] = isc->b_off_ctrl->val;
-		if (ctrl->cluster[ISC_CTRL_GR_OFF]->is_new)
-			ctrls->offset[ISC_HIS_CFG_MODE_GR] = isc->gr_off_ctrl->val;
-		if (ctrl->cluster[ISC_CTRL_GB_OFF]->is_new)
-			ctrls->offset[ISC_HIS_CFG_MODE_GB] = isc->gb_off_ctrl->val;
-
-		isc_update_awb_ctrls(isc);
-
-		mutex_lock(&isc->awb_mutex);
-		if (vb2_is_streaming(&isc->vb2_vidq)) {
-			/*
-			 * If we are streaming, we can update profile to
-			 * have the new settings in place.
-			 */
-			isc_update_profile(isc);
-		} else {
-			/*
-			 * The auto cluster will activate automatically this
-			 * control. This has to be deactivated when not
-			 * streaming.
-			 */
-			v4l2_ctrl_activate(isc->do_wb_ctrl, false);
-		}
-		mutex_unlock(&isc->awb_mutex);
-
-		/* if we have autowhitebalance on, start histogram procedure */
-		if (ctrls->awb == ISC_WB_AUTO &&
-		    vb2_is_streaming(&isc->vb2_vidq) &&
-		    ISC_IS_FORMAT_RAW(isc->config.sd_format->mbus_code))
-			isc_set_histogram(isc, true);
-
-		/*
-		 * for one time whitebalance adjustment, check the button,
-		 * if it's pressed, perform the one time operation.
-		 */
-		if (ctrls->awb == ISC_WB_NONE &&
-		    ctrl->cluster[ISC_CTRL_DO_WB]->is_new &&
-		    !(ctrl->cluster[ISC_CTRL_DO_WB]->flags &
-		    V4L2_CTRL_FLAG_INACTIVE)) {
-			ctrls->awb = ISC_WB_ONETIME;
-			isc_set_histogram(isc, true);
-			v4l2_dbg(1, debug, &isc->v4l2_dev,
-				 "One time white-balance started.\n");
-		}
-		return 0;
-	}
-	return 0;
-}
-
-static int isc_g_volatile_awb_ctrl(struct v4l2_ctrl *ctrl)
-{
-	struct isc_device *isc = container_of(ctrl->handler,
-					     struct isc_device, ctrls.handler);
-	struct isc_ctrls *ctrls = &isc->ctrls;
-
-	switch (ctrl->id) {
-	/* being a cluster, this id will be called for every control */
-	case V4L2_CID_AUTO_WHITE_BALANCE:
-		ctrl->cluster[ISC_CTRL_R_GAIN]->val =
-					ctrls->gain[ISC_HIS_CFG_MODE_R];
-		ctrl->cluster[ISC_CTRL_B_GAIN]->val =
-					ctrls->gain[ISC_HIS_CFG_MODE_B];
-		ctrl->cluster[ISC_CTRL_GR_GAIN]->val =
-					ctrls->gain[ISC_HIS_CFG_MODE_GR];
-		ctrl->cluster[ISC_CTRL_GB_GAIN]->val =
-					ctrls->gain[ISC_HIS_CFG_MODE_GB];
-
-		ctrl->cluster[ISC_CTRL_R_OFF]->val =
-			ctrls->offset[ISC_HIS_CFG_MODE_R];
-		ctrl->cluster[ISC_CTRL_B_OFF]->val =
-			ctrls->offset[ISC_HIS_CFG_MODE_B];
-		ctrl->cluster[ISC_CTRL_GR_OFF]->val =
-			ctrls->offset[ISC_HIS_CFG_MODE_GR];
-		ctrl->cluster[ISC_CTRL_GB_OFF]->val =
-			ctrls->offset[ISC_HIS_CFG_MODE_GB];
-		break;
-	}
-	return 0;
-}
-
-static const struct v4l2_ctrl_ops isc_awb_ops = {
-	.s_ctrl = isc_s_awb_ctrl,
-	.g_volatile_ctrl = isc_g_volatile_awb_ctrl,
-};
-
-#define ISC_CTRL_OFF(_name, _id, _name_str) \
-	static const struct v4l2_ctrl_config _name = { \
-		.ops = &isc_awb_ops, \
-		.id = _id, \
-		.name = _name_str, \
-		.type = V4L2_CTRL_TYPE_INTEGER, \
-		.flags = V4L2_CTRL_FLAG_SLIDER, \
-		.min = -4095, \
-		.max = 4095, \
-		.step = 1, \
-		.def = 0, \
-	}
-
-ISC_CTRL_OFF(isc_r_off_ctrl, ISC_CID_R_OFFSET, "Red Component Offset");
-ISC_CTRL_OFF(isc_b_off_ctrl, ISC_CID_B_OFFSET, "Blue Component Offset");
-ISC_CTRL_OFF(isc_gr_off_ctrl, ISC_CID_GR_OFFSET, "Green Red Component Offset");
-ISC_CTRL_OFF(isc_gb_off_ctrl, ISC_CID_GB_OFFSET, "Green Blue Component Offset");
-
-#define ISC_CTRL_GAIN(_name, _id, _name_str) \
-	static const struct v4l2_ctrl_config _name = { \
-		.ops = &isc_awb_ops, \
-		.id = _id, \
-		.name = _name_str, \
-		.type = V4L2_CTRL_TYPE_INTEGER, \
-		.flags = V4L2_CTRL_FLAG_SLIDER, \
-		.min = 0, \
-		.max = 8191, \
-		.step = 1, \
-		.def = 512, \
-	}
-
-ISC_CTRL_GAIN(isc_r_gain_ctrl, ISC_CID_R_GAIN, "Red Component Gain");
-ISC_CTRL_GAIN(isc_b_gain_ctrl, ISC_CID_B_GAIN, "Blue Component Gain");
-ISC_CTRL_GAIN(isc_gr_gain_ctrl, ISC_CID_GR_GAIN, "Green Red Component Gain");
-ISC_CTRL_GAIN(isc_gb_gain_ctrl, ISC_CID_GB_GAIN, "Green Blue Component Gain");
-
-static int isc_ctrl_init(struct isc_device *isc)
-{
-	const struct v4l2_ctrl_ops *ops = &isc_ctrl_ops;
-	struct isc_ctrls *ctrls = &isc->ctrls;
-	struct v4l2_ctrl_handler *hdl = &ctrls->handler;
-	int ret;
-
-	ctrls->hist_stat = HIST_INIT;
-	isc_reset_awb_ctrls(isc);
-
-	ret = v4l2_ctrl_handler_init(hdl, 13);
-	if (ret < 0)
-		return ret;
-
-	/* Initialize product specific controls. For example, contrast */
-	isc->config_ctrls(isc, ops);
-
-	ctrls->brightness = 0;
-
-	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_BRIGHTNESS, -1024, 1023, 1, 0);
-	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_GAMMA, 0, isc->gamma_max, 1,
-			  isc->gamma_max);
-	isc->awb_ctrl = v4l2_ctrl_new_std(hdl, &isc_awb_ops,
-					  V4L2_CID_AUTO_WHITE_BALANCE,
-					  0, 1, 1, 1);
-
-	/* do_white_balance is a button, so min,max,step,default are ignored */
-	isc->do_wb_ctrl = v4l2_ctrl_new_std(hdl, &isc_awb_ops,
-					    V4L2_CID_DO_WHITE_BALANCE,
-					    0, 0, 0, 0);
-
-	if (!isc->do_wb_ctrl) {
-		ret = hdl->error;
-		v4l2_ctrl_handler_free(hdl);
-		return ret;
-	}
-
-	v4l2_ctrl_activate(isc->do_wb_ctrl, false);
-
-	isc->r_gain_ctrl = v4l2_ctrl_new_custom(hdl, &isc_r_gain_ctrl, NULL);
-	isc->b_gain_ctrl = v4l2_ctrl_new_custom(hdl, &isc_b_gain_ctrl, NULL);
-	isc->gr_gain_ctrl = v4l2_ctrl_new_custom(hdl, &isc_gr_gain_ctrl, NULL);
-	isc->gb_gain_ctrl = v4l2_ctrl_new_custom(hdl, &isc_gb_gain_ctrl, NULL);
-	isc->r_off_ctrl = v4l2_ctrl_new_custom(hdl, &isc_r_off_ctrl, NULL);
-	isc->b_off_ctrl = v4l2_ctrl_new_custom(hdl, &isc_b_off_ctrl, NULL);
-	isc->gr_off_ctrl = v4l2_ctrl_new_custom(hdl, &isc_gr_off_ctrl, NULL);
-	isc->gb_off_ctrl = v4l2_ctrl_new_custom(hdl, &isc_gb_off_ctrl, NULL);
-
-	/*
-	 * The cluster is in auto mode with autowhitebalance enabled
-	 * and manual mode otherwise.
-	 */
-	v4l2_ctrl_auto_cluster(10, &isc->awb_ctrl, 0, true);
-
-	v4l2_ctrl_handler_setup(hdl);
-
-	return 0;
-}
-
-static int isc_async_bound(struct v4l2_async_notifier *notifier,
-			    struct v4l2_subdev *subdev,
-			    struct v4l2_async_connection *asd)
-{
-	struct isc_device *isc = container_of(notifier->v4l2_dev,
-					      struct isc_device, v4l2_dev);
-	struct isc_subdev_entity *subdev_entity =
-		container_of(notifier, struct isc_subdev_entity, notifier);
-
-	if (video_is_registered(&isc->video_dev)) {
-		v4l2_err(&isc->v4l2_dev, "only supports one sub-device.\n");
-		return -EBUSY;
-	}
-
-	subdev_entity->sd = subdev;
-
-	return 0;
-}
-
-static void isc_async_unbind(struct v4l2_async_notifier *notifier,
-			      struct v4l2_subdev *subdev,
-			      struct v4l2_async_connection *asd)
-{
-	struct isc_device *isc = container_of(notifier->v4l2_dev,
-					      struct isc_device, v4l2_dev);
-	mutex_destroy(&isc->awb_mutex);
-	cancel_work_sync(&isc->awb_work);
-	video_unregister_device(&isc->video_dev);
-	v4l2_ctrl_handler_free(&isc->ctrls.handler);
-}
-
-static struct isc_format *find_format_by_code(struct isc_device *isc,
-					      unsigned int code, int *index)
-{
-	struct isc_format *fmt = &isc->formats_list[0];
-	unsigned int i;
-
-	for (i = 0; i < isc->formats_list_size; i++) {
-		if (fmt->mbus_code == code) {
-			*index = i;
-			return fmt;
-		}
-
-		fmt++;
-	}
-
-	return NULL;
-}
-
-static int isc_formats_init(struct isc_device *isc)
-{
-	struct isc_format *fmt;
-	struct v4l2_subdev *subdev = isc->current_subdev->sd;
-	unsigned int num_fmts, i, j;
-	u32 list_size = isc->formats_list_size;
-	struct v4l2_subdev_mbus_code_enum mbus_code = {
-		.which = V4L2_SUBDEV_FORMAT_ACTIVE,
-	};
-
-	num_fmts = 0;
-	while (!v4l2_subdev_call(subdev, pad, enum_mbus_code,
-	       NULL, &mbus_code)) {
-		mbus_code.index++;
-
-		fmt = find_format_by_code(isc, mbus_code.code, &i);
-		if (!fmt) {
-			v4l2_warn(&isc->v4l2_dev, "Mbus code %x not supported\n",
-				  mbus_code.code);
-			continue;
-		}
-
-		fmt->sd_support = true;
-		num_fmts++;
-	}
-
-	if (!num_fmts)
-		return -ENXIO;
-
-	isc->num_user_formats = num_fmts;
-	isc->user_formats = devm_kcalloc(isc->dev,
-					 num_fmts, sizeof(*isc->user_formats),
-					 GFP_KERNEL);
-	if (!isc->user_formats)
-		return -ENOMEM;
-
-	fmt = &isc->formats_list[0];
-	for (i = 0, j = 0; i < list_size; i++) {
-		if (fmt->sd_support)
-			isc->user_formats[j++] = fmt;
-		fmt++;
-	}
-
-	return 0;
-}
-
-static int isc_set_default_fmt(struct isc_device *isc)
-{
-	struct v4l2_format f = {
-		.type = V4L2_BUF_TYPE_VIDEO_CAPTURE,
-		.fmt.pix = {
-			.width		= VGA_WIDTH,
-			.height		= VGA_HEIGHT,
-			.field		= V4L2_FIELD_NONE,
-			.pixelformat	= isc->user_formats[0]->fourcc,
-		},
-	};
-	int ret;
-
-	ret = isc_try_fmt(isc, &f, NULL);
-	if (ret)
-		return ret;
-
-	isc->fmt = f;
-	return 0;
-}
-
-static int isc_async_complete(struct v4l2_async_notifier *notifier)
-{
-	struct isc_device *isc = container_of(notifier->v4l2_dev,
-					      struct isc_device, v4l2_dev);
-	struct video_device *vdev = &isc->video_dev;
-	struct vb2_queue *q = &isc->vb2_vidq;
-	int ret = 0;
-
-	INIT_WORK(&isc->awb_work, isc_awb_work);
-
-	ret = v4l2_device_register_subdev_nodes(&isc->v4l2_dev);
-	if (ret < 0) {
-		v4l2_err(&isc->v4l2_dev, "Failed to register subdev nodes\n");
-		return ret;
-	}
-
-	isc->current_subdev = container_of(notifier,
-					   struct isc_subdev_entity, notifier);
-	mutex_init(&isc->lock);
-	mutex_init(&isc->awb_mutex);
-
-	init_completion(&isc->comp);
-
-	/* Initialize videobuf2 queue */
-	q->type			= V4L2_BUF_TYPE_VIDEO_CAPTURE;
-	q->io_modes		= VB2_MMAP | VB2_DMABUF | VB2_READ;
-	q->drv_priv		= isc;
-	q->buf_struct_size	= sizeof(struct isc_buffer);
-	q->ops			= &isc_vb2_ops;
-	q->mem_ops		= &vb2_dma_contig_memops;
-	q->timestamp_flags	= V4L2_BUF_FLAG_TIMESTAMP_MONOTONIC;
-	q->lock			= &isc->lock;
-	q->min_queued_buffers	= 1;
-	q->dev			= isc->dev;
-
-	ret = vb2_queue_init(q);
-	if (ret < 0) {
-		v4l2_err(&isc->v4l2_dev,
-			 "vb2_queue_init() failed: %d\n", ret);
-		goto isc_async_complete_err;
-	}
-
-	/* Init video dma queues */
-	INIT_LIST_HEAD(&isc->dma_queue);
-	spin_lock_init(&isc->dma_queue_lock);
-	spin_lock_init(&isc->awb_lock);
-
-	ret = isc_formats_init(isc);
-	if (ret < 0) {
-		v4l2_err(&isc->v4l2_dev,
-			 "Init format failed: %d\n", ret);
-		goto isc_async_complete_err;
-	}
-
-	ret = isc_set_default_fmt(isc);
-	if (ret) {
-		v4l2_err(&isc->v4l2_dev, "Could not set default format\n");
-		goto isc_async_complete_err;
-	}
-
-	ret = isc_ctrl_init(isc);
-	if (ret) {
-		v4l2_err(&isc->v4l2_dev, "Init isc ctrols failed: %d\n", ret);
-		goto isc_async_complete_err;
-	}
-
-	/* Register video device */
-	strscpy(vdev->name, KBUILD_MODNAME, sizeof(vdev->name));
-	vdev->release		= video_device_release_empty;
-	vdev->fops		= &isc_fops;
-	vdev->ioctl_ops		= &isc_ioctl_ops;
-	vdev->v4l2_dev		= &isc->v4l2_dev;
-	vdev->vfl_dir		= VFL_DIR_RX;
-	vdev->queue		= q;
-	vdev->lock		= &isc->lock;
-	vdev->ctrl_handler	= &isc->ctrls.handler;
-	vdev->device_caps	= V4L2_CAP_STREAMING | V4L2_CAP_VIDEO_CAPTURE;
-	video_set_drvdata(vdev, isc);
-
-	ret = video_register_device(vdev, VFL_TYPE_VIDEO, -1);
-	if (ret < 0) {
-		v4l2_err(&isc->v4l2_dev,
-			 "video_register_device failed: %d\n", ret);
-		goto isc_async_complete_err;
-	}
-
-	return 0;
-
-isc_async_complete_err:
-	mutex_destroy(&isc->awb_mutex);
-	mutex_destroy(&isc->lock);
-	return ret;
-}
-
-const struct v4l2_async_notifier_operations atmel_isc_async_ops = {
-	.bound = isc_async_bound,
-	.unbind = isc_async_unbind,
-	.complete = isc_async_complete,
-};
-EXPORT_SYMBOL_GPL(atmel_isc_async_ops);
-
-void atmel_isc_subdev_cleanup(struct isc_device *isc)
-{
-	struct isc_subdev_entity *subdev_entity;
-
-	list_for_each_entry(subdev_entity, &isc->subdev_entities, list) {
-		v4l2_async_nf_unregister(&subdev_entity->notifier);
-		v4l2_async_nf_cleanup(&subdev_entity->notifier);
-	}
-
-	INIT_LIST_HEAD(&isc->subdev_entities);
-}
-EXPORT_SYMBOL_GPL(atmel_isc_subdev_cleanup);
-
-int atmel_isc_pipeline_init(struct isc_device *isc)
-{
-	struct device *dev = isc->dev;
-	struct regmap *regmap = isc->regmap;
-	struct regmap_field *regs;
-	unsigned int i;
-
-	/*
-	 * DPCEN-->GDCEN-->BLCEN-->WB-->CFA-->CC-->
-	 * GAM-->VHXS-->CSC-->CBC-->SUB422-->SUB420
-	 */
-	const struct reg_field regfields[ISC_PIPE_LINE_NODE_NUM] = {
-		REG_FIELD(ISC_DPC_CTRL, 0, 0),
-		REG_FIELD(ISC_DPC_CTRL, 1, 1),
-		REG_FIELD(ISC_DPC_CTRL, 2, 2),
-		REG_FIELD(ISC_WB_CTRL, 0, 0),
-		REG_FIELD(ISC_CFA_CTRL, 0, 0),
-		REG_FIELD(ISC_CC_CTRL, 0, 0),
-		REG_FIELD(ISC_GAM_CTRL, 0, 0),
-		REG_FIELD(ISC_GAM_CTRL, 1, 1),
-		REG_FIELD(ISC_GAM_CTRL, 2, 2),
-		REG_FIELD(ISC_GAM_CTRL, 3, 3),
-		REG_FIELD(ISC_VHXS_CTRL, 0, 0),
-		REG_FIELD(ISC_CSC_CTRL + isc->offsets.csc, 0, 0),
-		REG_FIELD(ISC_CBC_CTRL + isc->offsets.cbc, 0, 0),
-		REG_FIELD(ISC_SUB422_CTRL + isc->offsets.sub422, 0, 0),
-		REG_FIELD(ISC_SUB420_CTRL + isc->offsets.sub420, 0, 0),
-	};
-
-	for (i = 0; i < ISC_PIPE_LINE_NODE_NUM; i++) {
-		regs = devm_regmap_field_alloc(dev, regmap, regfields[i]);
-		if (IS_ERR(regs))
-			return PTR_ERR(regs);
-
-		isc->pipeline[i] =  regs;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(atmel_isc_pipeline_init);
-
-/* regmap configuration */
-#define ATMEL_ISC_REG_MAX    0xd5c
-const struct regmap_config atmel_isc_regmap_config = {
-	.reg_bits       = 32,
-	.reg_stride     = 4,
-	.val_bits       = 32,
-	.max_register	= ATMEL_ISC_REG_MAX,
-};
-EXPORT_SYMBOL_GPL(atmel_isc_regmap_config);
-
-MODULE_AUTHOR("Songjun Wu");
-MODULE_AUTHOR("Eugen Hristev");
-MODULE_DESCRIPTION("Atmel ISC common code base");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/media/deprecated/atmel/atmel-isc-clk.c b/drivers/staging/media/deprecated/atmel/atmel-isc-clk.c
deleted file mode 100644
index d442b5f4c931c..0000000000000
--- a/drivers/staging/media/deprecated/atmel/atmel-isc-clk.c
+++ /dev/null
@@ -1,311 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Microchip Image Sensor Controller (ISC) common clock driver setup
- *
- * Copyright (C) 2016 Microchip Technology, Inc.
- *
- * Author: Songjun Wu
- * Author: Eugen Hristev <eugen.hristev@microchip.com>
- *
- */
-#include <linux/clk.h>
-#include <linux/clkdev.h>
-#include <linux/clk-provider.h>
-#include <linux/pm_runtime.h>
-#include <linux/regmap.h>
-
-#include "atmel-isc-regs.h"
-#include "atmel-isc.h"
-
-static int isc_wait_clk_stable(struct clk_hw *hw)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-	struct regmap *regmap = isc_clk->regmap;
-	unsigned long timeout = jiffies + usecs_to_jiffies(1000);
-	unsigned int status;
-
-	while (time_before(jiffies, timeout)) {
-		regmap_read(regmap, ISC_CLKSR, &status);
-		if (!(status & ISC_CLKSR_SIP))
-			return 0;
-
-		usleep_range(10, 250);
-	}
-
-	return -ETIMEDOUT;
-}
-
-static int isc_clk_prepare(struct clk_hw *hw)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-	int ret;
-
-	ret = pm_runtime_resume_and_get(isc_clk->dev);
-	if (ret < 0)
-		return ret;
-
-	return isc_wait_clk_stable(hw);
-}
-
-static void isc_clk_unprepare(struct clk_hw *hw)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-
-	isc_wait_clk_stable(hw);
-
-	pm_runtime_put_sync(isc_clk->dev);
-}
-
-static int isc_clk_enable(struct clk_hw *hw)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-	u32 id = isc_clk->id;
-	struct regmap *regmap = isc_clk->regmap;
-	unsigned long flags;
-	unsigned int status;
-
-	dev_dbg(isc_clk->dev, "ISC CLK: %s, id = %d, div = %d, parent id = %d\n",
-		__func__, id, isc_clk->div, isc_clk->parent_id);
-
-	spin_lock_irqsave(&isc_clk->lock, flags);
-	regmap_update_bits(regmap, ISC_CLKCFG,
-			   ISC_CLKCFG_DIV_MASK(id) | ISC_CLKCFG_SEL_MASK(id),
-			   (isc_clk->div << ISC_CLKCFG_DIV_SHIFT(id)) |
-			   (isc_clk->parent_id << ISC_CLKCFG_SEL_SHIFT(id)));
-
-	regmap_write(regmap, ISC_CLKEN, ISC_CLK(id));
-	spin_unlock_irqrestore(&isc_clk->lock, flags);
-
-	regmap_read(regmap, ISC_CLKSR, &status);
-	if (status & ISC_CLK(id))
-		return 0;
-	else
-		return -EINVAL;
-}
-
-static void isc_clk_disable(struct clk_hw *hw)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-	u32 id = isc_clk->id;
-	unsigned long flags;
-
-	spin_lock_irqsave(&isc_clk->lock, flags);
-	regmap_write(isc_clk->regmap, ISC_CLKDIS, ISC_CLK(id));
-	spin_unlock_irqrestore(&isc_clk->lock, flags);
-}
-
-static int isc_clk_is_enabled(struct clk_hw *hw)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-	u32 status;
-	int ret;
-
-	ret = pm_runtime_resume_and_get(isc_clk->dev);
-	if (ret < 0)
-		return 0;
-
-	regmap_read(isc_clk->regmap, ISC_CLKSR, &status);
-
-	pm_runtime_put_sync(isc_clk->dev);
-
-	return status & ISC_CLK(isc_clk->id) ? 1 : 0;
-}
-
-static unsigned long
-isc_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-
-	return DIV_ROUND_CLOSEST(parent_rate, isc_clk->div + 1);
-}
-
-static int isc_clk_determine_rate(struct clk_hw *hw,
-				  struct clk_rate_request *req)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-	long best_rate = -EINVAL;
-	int best_diff = -1;
-	unsigned int i, div;
-
-	for (i = 0; i < clk_hw_get_num_parents(hw); i++) {
-		struct clk_hw *parent;
-		unsigned long parent_rate;
-
-		parent = clk_hw_get_parent_by_index(hw, i);
-		if (!parent)
-			continue;
-
-		parent_rate = clk_hw_get_rate(parent);
-		if (!parent_rate)
-			continue;
-
-		for (div = 1; div < ISC_CLK_MAX_DIV + 2; div++) {
-			unsigned long rate;
-			int diff;
-
-			rate = DIV_ROUND_CLOSEST(parent_rate, div);
-			diff = abs(req->rate - rate);
-
-			if (best_diff < 0 || best_diff > diff) {
-				best_rate = rate;
-				best_diff = diff;
-				req->best_parent_rate = parent_rate;
-				req->best_parent_hw = parent;
-			}
-
-			if (!best_diff || rate < req->rate)
-				break;
-		}
-
-		if (!best_diff)
-			break;
-	}
-
-	dev_dbg(isc_clk->dev,
-		"ISC CLK: %s, best_rate = %ld, parent clk: %s @ %ld\n",
-		__func__, best_rate,
-		__clk_get_name((req->best_parent_hw)->clk),
-		req->best_parent_rate);
-
-	if (best_rate < 0)
-		return best_rate;
-
-	req->rate = best_rate;
-
-	return 0;
-}
-
-static int isc_clk_set_parent(struct clk_hw *hw, u8 index)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-
-	if (index >= clk_hw_get_num_parents(hw))
-		return -EINVAL;
-
-	isc_clk->parent_id = index;
-
-	return 0;
-}
-
-static u8 isc_clk_get_parent(struct clk_hw *hw)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-
-	return isc_clk->parent_id;
-}
-
-static int isc_clk_set_rate(struct clk_hw *hw,
-			    unsigned long rate,
-			    unsigned long parent_rate)
-{
-	struct isc_clk *isc_clk = to_isc_clk(hw);
-	u32 div;
-
-	if (!rate)
-		return -EINVAL;
-
-	div = DIV_ROUND_CLOSEST(parent_rate, rate);
-	if (div > (ISC_CLK_MAX_DIV + 1) || !div)
-		return -EINVAL;
-
-	isc_clk->div = div - 1;
-
-	return 0;
-}
-
-static const struct clk_ops isc_clk_ops = {
-	.prepare	= isc_clk_prepare,
-	.unprepare	= isc_clk_unprepare,
-	.enable		= isc_clk_enable,
-	.disable	= isc_clk_disable,
-	.is_enabled	= isc_clk_is_enabled,
-	.recalc_rate	= isc_clk_recalc_rate,
-	.determine_rate	= isc_clk_determine_rate,
-	.set_parent	= isc_clk_set_parent,
-	.get_parent	= isc_clk_get_parent,
-	.set_rate	= isc_clk_set_rate,
-};
-
-static int isc_clk_register(struct isc_device *isc, unsigned int id)
-{
-	struct regmap *regmap = isc->regmap;
-	struct device_node *np = isc->dev->of_node;
-	struct isc_clk *isc_clk;
-	struct clk_init_data init;
-	const char *clk_name = np->name;
-	const char *parent_names[3];
-	int num_parents;
-
-	if (id == ISC_ISPCK && !isc->ispck_required)
-		return 0;
-
-	num_parents = of_clk_get_parent_count(np);
-	if (num_parents < 1 || num_parents > 3)
-		return -EINVAL;
-
-	if (num_parents > 2 && id == ISC_ISPCK)
-		num_parents = 2;
-
-	of_clk_parent_fill(np, parent_names, num_parents);
-
-	if (id == ISC_MCK)
-		of_property_read_string(np, "clock-output-names", &clk_name);
-	else
-		clk_name = "isc-ispck";
-
-	init.parent_names	= parent_names;
-	init.num_parents	= num_parents;
-	init.name		= clk_name;
-	init.ops		= &isc_clk_ops;
-	init.flags		= CLK_SET_RATE_GATE | CLK_SET_PARENT_GATE;
-
-	isc_clk = &isc->isc_clks[id];
-	isc_clk->hw.init	= &init;
-	isc_clk->regmap		= regmap;
-	isc_clk->id		= id;
-	isc_clk->dev		= isc->dev;
-	spin_lock_init(&isc_clk->lock);
-
-	isc_clk->clk = clk_register(isc->dev, &isc_clk->hw);
-	if (IS_ERR(isc_clk->clk)) {
-		dev_err(isc->dev, "%s: clock register fail\n", clk_name);
-		return PTR_ERR(isc_clk->clk);
-	} else if (id == ISC_MCK) {
-		of_clk_add_provider(np, of_clk_src_simple_get, isc_clk->clk);
-	}
-
-	return 0;
-}
-
-int atmel_isc_clk_init(struct isc_device *isc)
-{
-	unsigned int i;
-	int ret;
-
-	for (i = 0; i < ARRAY_SIZE(isc->isc_clks); i++)
-		isc->isc_clks[i].clk = ERR_PTR(-EINVAL);
-
-	for (i = 0; i < ARRAY_SIZE(isc->isc_clks); i++) {
-		ret = isc_clk_register(isc, i);
-		if (ret)
-			return ret;
-	}
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(atmel_isc_clk_init);
-
-void atmel_isc_clk_cleanup(struct isc_device *isc)
-{
-	unsigned int i;
-
-	of_clk_del_provider(isc->dev->of_node);
-
-	for (i = 0; i < ARRAY_SIZE(isc->isc_clks); i++) {
-		struct isc_clk *isc_clk = &isc->isc_clks[i];
-
-		if (!IS_ERR(isc_clk->clk))
-			clk_unregister(isc_clk->clk);
-	}
-}
-EXPORT_SYMBOL_GPL(atmel_isc_clk_cleanup);
diff --git a/drivers/staging/media/deprecated/atmel/atmel-isc-regs.h b/drivers/staging/media/deprecated/atmel/atmel-isc-regs.h
deleted file mode 100644
index d06b72228d4fa..0000000000000
--- a/drivers/staging/media/deprecated/atmel/atmel-isc-regs.h
+++ /dev/null
@@ -1,413 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __ATMEL_ISC_REGS_H
-#define __ATMEL_ISC_REGS_H
-
-#include <linux/bitops.h>
-
-/* ISC Control Enable Register 0 */
-#define ISC_CTRLEN      0x00000000
-
-/* ISC Control Disable Register 0 */
-#define ISC_CTRLDIS     0x00000004
-
-/* ISC Control Status Register 0 */
-#define ISC_CTRLSR      0x00000008
-
-#define ISC_CTRL_CAPTURE	BIT(0)
-#define ISC_CTRL_UPPRO		BIT(1)
-#define ISC_CTRL_HISREQ		BIT(2)
-#define ISC_CTRL_HISCLR		BIT(3)
-
-/* ISC Parallel Front End Configuration 0 Register */
-#define ISC_PFE_CFG0    0x0000000c
-
-#define ISC_PFE_CFG0_HPOL_LOW   BIT(0)
-#define ISC_PFE_CFG0_VPOL_LOW   BIT(1)
-#define ISC_PFE_CFG0_PPOL_LOW   BIT(2)
-#define ISC_PFE_CFG0_CCIR656    BIT(9)
-#define ISC_PFE_CFG0_CCIR_CRC   BIT(10)
-#define ISC_PFE_CFG0_MIPI	BIT(14)
-
-#define ISC_PFE_CFG0_MODE_PROGRESSIVE   (0x0 << 4)
-#define ISC_PFE_CFG0_MODE_MASK          GENMASK(6, 4)
-
-#define ISC_PFE_CFG0_BPS_EIGHT  (0x4 << 28)
-#define ISC_PFG_CFG0_BPS_NINE   (0x3 << 28)
-#define ISC_PFG_CFG0_BPS_TEN    (0x2 << 28)
-#define ISC_PFG_CFG0_BPS_ELEVEN (0x1 << 28)
-#define ISC_PFG_CFG0_BPS_TWELVE (0x0 << 28)
-#define ISC_PFE_CFG0_BPS_MASK   GENMASK(30, 28)
-
-#define ISC_PFE_CFG0_COLEN	BIT(12)
-#define ISC_PFE_CFG0_ROWEN	BIT(13)
-
-/* ISC Parallel Front End Configuration 1 Register */
-#define ISC_PFE_CFG1    0x00000010
-
-#define ISC_PFE_CFG1_COLMIN(v)		((v))
-#define ISC_PFE_CFG1_COLMIN_MASK	GENMASK(15, 0)
-#define ISC_PFE_CFG1_COLMAX(v)		((v) << 16)
-#define ISC_PFE_CFG1_COLMAX_MASK	GENMASK(31, 16)
-
-/* ISC Parallel Front End Configuration 2 Register */
-#define ISC_PFE_CFG2    0x00000014
-
-#define ISC_PFE_CFG2_ROWMIN(v)		((v))
-#define ISC_PFE_CFG2_ROWMIN_MASK	GENMASK(15, 0)
-#define ISC_PFE_CFG2_ROWMAX(v)		((v) << 16)
-#define ISC_PFE_CFG2_ROWMAX_MASK	GENMASK(31, 16)
-
-/* ISC Clock Enable Register */
-#define ISC_CLKEN               0x00000018
-
-/* ISC Clock Disable Register */
-#define ISC_CLKDIS              0x0000001c
-
-/* ISC Clock Status Register */
-#define ISC_CLKSR               0x00000020
-#define ISC_CLKSR_SIP		BIT(31)
-
-#define ISC_CLK(n)		BIT(n)
-
-/* ISC Clock Configuration Register */
-#define ISC_CLKCFG              0x00000024
-#define ISC_CLKCFG_DIV_SHIFT(n) ((n)*16)
-#define ISC_CLKCFG_DIV_MASK(n)  GENMASK(((n)*16 + 7), (n)*16)
-#define ISC_CLKCFG_SEL_SHIFT(n) ((n)*16 + 8)
-#define ISC_CLKCFG_SEL_MASK(n)  GENMASK(((n)*17 + 8), ((n)*16 + 8))
-
-/* ISC Interrupt Enable Register */
-#define ISC_INTEN       0x00000028
-
-/* ISC Interrupt Disable Register */
-#define ISC_INTDIS      0x0000002c
-
-/* ISC Interrupt Mask Register */
-#define ISC_INTMASK     0x00000030
-
-/* ISC Interrupt Status Register */
-#define ISC_INTSR       0x00000034
-
-#define ISC_INT_DDONE		BIT(8)
-#define ISC_INT_HISDONE		BIT(12)
-
-/* ISC DPC Control Register */
-#define ISC_DPC_CTRL	0x40
-
-#define ISC_DPC_CTRL_DPCEN	BIT(0)
-#define ISC_DPC_CTRL_GDCEN	BIT(1)
-#define ISC_DPC_CTRL_BLCEN	BIT(2)
-
-/* ISC DPC Config Register */
-#define ISC_DPC_CFG	0x44
-
-#define ISC_DPC_CFG_BAYSEL_SHIFT	0
-
-#define ISC_DPC_CFG_EITPOL		BIT(4)
-
-#define ISC_DPC_CFG_TA_ENABLE		BIT(14)
-#define ISC_DPC_CFG_TC_ENABLE		BIT(13)
-#define ISC_DPC_CFG_TM_ENABLE		BIT(12)
-
-#define ISC_DPC_CFG_RE_MODE		BIT(17)
-
-#define ISC_DPC_CFG_GDCCLP_SHIFT	20
-#define ISC_DPC_CFG_GDCCLP_MASK		GENMASK(22, 20)
-
-#define ISC_DPC_CFG_BLOFF_SHIFT		24
-#define ISC_DPC_CFG_BLOFF_MASK		GENMASK(31, 24)
-
-#define ISC_DPC_CFG_BAYCFG_SHIFT	0
-#define ISC_DPC_CFG_BAYCFG_MASK		GENMASK(1, 0)
-/* ISC DPC Threshold Median Register */
-#define ISC_DPC_THRESHM	0x48
-
-/* ISC DPC Threshold Closest Register */
-#define ISC_DPC_THRESHC	0x4C
-
-/* ISC DPC Threshold Average Register */
-#define ISC_DPC_THRESHA	0x50
-
-/* ISC DPC STatus Register */
-#define ISC_DPC_SR	0x54
-
-/* ISC White Balance Control Register */
-#define ISC_WB_CTRL     0x00000058
-
-/* ISC White Balance Configuration Register */
-#define ISC_WB_CFG      0x0000005c
-
-/* ISC White Balance Offset for R, GR Register */
-#define ISC_WB_O_RGR	0x00000060
-
-/* ISC White Balance Offset for B, GB Register */
-#define ISC_WB_O_BGB	0x00000064
-
-/* ISC White Balance Gain for R, GR Register */
-#define ISC_WB_G_RGR	0x00000068
-
-/* ISC White Balance Gain for B, GB Register */
-#define ISC_WB_G_BGB	0x0000006c
-
-/* ISC Color Filter Array Control Register */
-#define ISC_CFA_CTRL    0x00000070
-
-/* ISC Color Filter Array Configuration Register */
-#define ISC_CFA_CFG     0x00000074
-#define ISC_CFA_CFG_EITPOL	BIT(4)
-
-#define ISC_BAY_CFG_GRGR	0x0
-#define ISC_BAY_CFG_RGRG	0x1
-#define ISC_BAY_CFG_GBGB	0x2
-#define ISC_BAY_CFG_BGBG	0x3
-
-/* ISC Color Correction Control Register */
-#define ISC_CC_CTRL     0x00000078
-
-/* ISC Color Correction RR RG Register */
-#define ISC_CC_RR_RG	0x0000007c
-
-/* ISC Color Correction RB OR Register */
-#define ISC_CC_RB_OR	0x00000080
-
-/* ISC Color Correction GR GG Register */
-#define ISC_CC_GR_GG	0x00000084
-
-/* ISC Color Correction GB OG Register */
-#define ISC_CC_GB_OG	0x00000088
-
-/* ISC Color Correction BR BG Register */
-#define ISC_CC_BR_BG	0x0000008c
-
-/* ISC Color Correction BB OB Register */
-#define ISC_CC_BB_OB	0x00000090
-
-/* ISC Gamma Correction Control Register */
-#define ISC_GAM_CTRL    0x00000094
-
-#define ISC_GAM_CTRL_BIPART	BIT(4)
-
-/* ISC_Gamma Correction Blue Entry Register */
-#define ISC_GAM_BENTRY	0x00000098
-
-/* ISC_Gamma Correction Green Entry Register */
-#define ISC_GAM_GENTRY	0x00000198
-
-/* ISC_Gamma Correction Green Entry Register */
-#define ISC_GAM_RENTRY	0x00000298
-
-/* ISC VHXS Control Register */
-#define ISC_VHXS_CTRL	0x398
-
-/* ISC VHXS Source Size Register */
-#define ISC_VHXS_SS	0x39C
-
-/* ISC VHXS Destination Size Register */
-#define ISC_VHXS_DS	0x3A0
-
-/* ISC Vertical Factor Register */
-#define ISC_VXS_FACT	0x3a4
-
-/* ISC Horizontal Factor Register */
-#define ISC_HXS_FACT	0x3a8
-
-/* ISC Vertical Config Register */
-#define ISC_VXS_CFG	0x3ac
-
-/* ISC Horizontal Config Register */
-#define ISC_HXS_CFG	0x3b0
-
-/* ISC Vertical Tap Register */
-#define ISC_VXS_TAP	0x3b4
-
-/* ISC Horizontal Tap Register */
-#define ISC_HXS_TAP	0x434
-
-/* Offset for CSC register specific to sama5d2 product */
-#define ISC_SAMA5D2_CSC_OFFSET	0
-/* Offset for CSC register specific to sama7g5 product */
-#define ISC_SAMA7G5_CSC_OFFSET	0x11c
-
-/* Color Space Conversion Control Register */
-#define ISC_CSC_CTRL    0x00000398
-
-/* Color Space Conversion YR YG Register */
-#define ISC_CSC_YR_YG	0x0000039c
-
-/* Color Space Conversion YB OY Register */
-#define ISC_CSC_YB_OY	0x000003a0
-
-/* Color Space Conversion CBR CBG Register */
-#define ISC_CSC_CBR_CBG	0x000003a4
-
-/* Color Space Conversion CBB OCB Register */
-#define ISC_CSC_CBB_OCB	0x000003a8
-
-/* Color Space Conversion CRR CRG Register */
-#define ISC_CSC_CRR_CRG	0x000003ac
-
-/* Color Space Conversion CRB OCR Register */
-#define ISC_CSC_CRB_OCR	0x000003b0
-
-/* Offset for CBC register specific to sama5d2 product */
-#define ISC_SAMA5D2_CBC_OFFSET	0
-/* Offset for CBC register specific to sama7g5 product */
-#define ISC_SAMA7G5_CBC_OFFSET	0x11c
-
-/* Contrast And Brightness Control Register */
-#define ISC_CBC_CTRL    0x000003b4
-
-/* Contrast And Brightness Configuration Register */
-#define ISC_CBC_CFG	0x000003b8
-
-/* Brightness Register */
-#define ISC_CBC_BRIGHT	0x000003bc
-#define ISC_CBC_BRIGHT_MASK	GENMASK(10, 0)
-
-/* Contrast Register */
-#define ISC_CBC_CONTRAST	0x000003c0
-#define ISC_CBC_CONTRAST_MASK	GENMASK(11, 0)
-
-/* Hue Register */
-#define ISC_CBCHS_HUE	0x4e0
-/* Saturation Register */
-#define ISC_CBCHS_SAT	0x4e4
-
-/* Offset for SUB422 register specific to sama5d2 product */
-#define ISC_SAMA5D2_SUB422_OFFSET	0
-/* Offset for SUB422 register specific to sama7g5 product */
-#define ISC_SAMA7G5_SUB422_OFFSET	0x124
-
-/* Subsampling 4:4:4 to 4:2:2 Control Register */
-#define ISC_SUB422_CTRL 0x000003c4
-
-/* Offset for SUB420 register specific to sama5d2 product */
-#define ISC_SAMA5D2_SUB420_OFFSET	0
-/* Offset for SUB420 register specific to sama7g5 product */
-#define ISC_SAMA7G5_SUB420_OFFSET	0x124
-/* Subsampling 4:2:2 to 4:2:0 Control Register */
-#define ISC_SUB420_CTRL 0x000003cc
-
-/* Offset for RLP register specific to sama5d2 product */
-#define ISC_SAMA5D2_RLP_OFFSET	0
-/* Offset for RLP register specific to sama7g5 product */
-#define ISC_SAMA7G5_RLP_OFFSET	0x124
-/* Rounding, Limiting and Packing Configuration Register */
-#define ISC_RLP_CFG     0x000003d0
-
-#define ISC_RLP_CFG_MODE_DAT8           0x0
-#define ISC_RLP_CFG_MODE_DAT9           0x1
-#define ISC_RLP_CFG_MODE_DAT10          0x2
-#define ISC_RLP_CFG_MODE_DAT11          0x3
-#define ISC_RLP_CFG_MODE_DAT12          0x4
-#define ISC_RLP_CFG_MODE_DATY8          0x5
-#define ISC_RLP_CFG_MODE_DATY10         0x6
-#define ISC_RLP_CFG_MODE_ARGB444        0x7
-#define ISC_RLP_CFG_MODE_ARGB555        0x8
-#define ISC_RLP_CFG_MODE_RGB565         0x9
-#define ISC_RLP_CFG_MODE_ARGB32         0xa
-#define ISC_RLP_CFG_MODE_YYCC           0xb
-#define ISC_RLP_CFG_MODE_YYCC_LIMITED   0xc
-#define ISC_RLP_CFG_MODE_YCYC           0xd
-#define ISC_RLP_CFG_MODE_MASK           GENMASK(3, 0)
-
-#define ISC_RLP_CFG_LSH			BIT(5)
-
-#define ISC_RLP_CFG_YMODE_YUYV		(3 << 6)
-#define ISC_RLP_CFG_YMODE_YVYU		(2 << 6)
-#define ISC_RLP_CFG_YMODE_VYUY		(0 << 6)
-#define ISC_RLP_CFG_YMODE_UYVY		(1 << 6)
-
-#define ISC_RLP_CFG_YMODE_MASK		GENMASK(7, 6)
-
-/* Offset for HIS register specific to sama5d2 product */
-#define ISC_SAMA5D2_HIS_OFFSET	0
-/* Offset for HIS register specific to sama7g5 product */
-#define ISC_SAMA7G5_HIS_OFFSET	0x124
-/* Histogram Control Register */
-#define ISC_HIS_CTRL	0x000003d4
-
-#define ISC_HIS_CTRL_EN			BIT(0)
-#define ISC_HIS_CTRL_DIS		0x0
-
-/* Histogram Configuration Register */
-#define ISC_HIS_CFG	0x000003d8
-
-#define ISC_HIS_CFG_MODE_GR		0x0
-#define ISC_HIS_CFG_MODE_R		0x1
-#define ISC_HIS_CFG_MODE_GB		0x2
-#define ISC_HIS_CFG_MODE_B		0x3
-#define ISC_HIS_CFG_MODE_Y		0x4
-#define ISC_HIS_CFG_MODE_RAW		0x5
-#define ISC_HIS_CFG_MODE_YCCIR656	0x6
-
-#define ISC_HIS_CFG_BAYSEL_SHIFT	4
-
-#define ISC_HIS_CFG_RAR			BIT(8)
-
-/* Offset for DMA register specific to sama5d2 product */
-#define ISC_SAMA5D2_DMA_OFFSET	0
-/* Offset for DMA register specific to sama7g5 product */
-#define ISC_SAMA7G5_DMA_OFFSET	0x13c
-
-/* DMA Configuration Register */
-#define ISC_DCFG        0x000003e0
-#define ISC_DCFG_IMODE_PACKED8          0x0
-#define ISC_DCFG_IMODE_PACKED16         0x1
-#define ISC_DCFG_IMODE_PACKED32         0x2
-#define ISC_DCFG_IMODE_YC422SP          0x3
-#define ISC_DCFG_IMODE_YC422P           0x4
-#define ISC_DCFG_IMODE_YC420SP          0x5
-#define ISC_DCFG_IMODE_YC420P           0x6
-#define ISC_DCFG_IMODE_MASK             GENMASK(2, 0)
-
-#define ISC_DCFG_YMBSIZE_SINGLE         (0x0 << 4)
-#define ISC_DCFG_YMBSIZE_BEATS4         (0x1 << 4)
-#define ISC_DCFG_YMBSIZE_BEATS8         (0x2 << 4)
-#define ISC_DCFG_YMBSIZE_BEATS16        (0x3 << 4)
-#define ISC_DCFG_YMBSIZE_BEATS32        (0x4 << 4)
-#define ISC_DCFG_YMBSIZE_MASK           GENMASK(6, 4)
-
-#define ISC_DCFG_CMBSIZE_SINGLE         (0x0 << 8)
-#define ISC_DCFG_CMBSIZE_BEATS4         (0x1 << 8)
-#define ISC_DCFG_CMBSIZE_BEATS8         (0x2 << 8)
-#define ISC_DCFG_CMBSIZE_BEATS16        (0x3 << 8)
-#define ISC_DCFG_CMBSIZE_BEATS32        (0x4 << 8)
-#define ISC_DCFG_CMBSIZE_MASK           GENMASK(10, 8)
-
-/* DMA Control Register */
-#define ISC_DCTRL       0x000003e4
-
-#define ISC_DCTRL_DVIEW_PACKED          (0x0 << 1)
-#define ISC_DCTRL_DVIEW_SEMIPLANAR      (0x1 << 1)
-#define ISC_DCTRL_DVIEW_PLANAR          (0x2 << 1)
-#define ISC_DCTRL_DVIEW_MASK            GENMASK(2, 1)
-
-#define ISC_DCTRL_IE_IS			(0x0 << 4)
-
-/* DMA Descriptor Address Register */
-#define ISC_DNDA        0x000003e8
-
-/* DMA Address 0 Register */
-#define ISC_DAD0        0x000003ec
-
-/* DMA Address 1 Register */
-#define ISC_DAD1        0x000003f4
-
-/* DMA Address 2 Register */
-#define ISC_DAD2        0x000003fc
-
-/* Offset for version register specific to sama5d2 product */
-#define ISC_SAMA5D2_VERSION_OFFSET	0
-#define ISC_SAMA7G5_VERSION_OFFSET	0x13c
-/* Version Register */
-#define ISC_VERSION	0x0000040c
-
-/* Offset for version register specific to sama5d2 product */
-#define ISC_SAMA5D2_HIS_ENTRY_OFFSET	0
-/* Offset for version register specific to sama7g5 product */
-#define ISC_SAMA7G5_HIS_ENTRY_OFFSET	0x14c
-/* Histogram Entry */
-#define ISC_HIS_ENTRY	0x00000410
-
-#endif
diff --git a/drivers/staging/media/deprecated/atmel/atmel-isc.h b/drivers/staging/media/deprecated/atmel/atmel-isc.h
deleted file mode 100644
index 31767ea74be62..0000000000000
--- a/drivers/staging/media/deprecated/atmel/atmel-isc.h
+++ /dev/null
@@ -1,362 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Microchip Image Sensor Controller (ISC) driver header file
- *
- * Copyright (C) 2016-2019 Microchip Technology, Inc.
- *
- * Author: Songjun Wu
- * Author: Eugen Hristev <eugen.hristev@microchip.com>
- *
- */
-#ifndef _ATMEL_ISC_H_
-
-#include <linux/clk-provider.h>
-#include <linux/platform_device.h>
-
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/videobuf2-dma-contig.h>
-
-#define ISC_CLK_MAX_DIV		255
-
-enum isc_clk_id {
-	ISC_ISPCK = 0,
-	ISC_MCK = 1,
-};
-
-struct isc_clk {
-	struct clk_hw   hw;
-	struct clk      *clk;
-	struct regmap   *regmap;
-	spinlock_t	lock;	/* serialize access to clock registers */
-	u8		id;
-	u8		parent_id;
-	u32		div;
-	struct device	*dev;
-};
-
-#define to_isc_clk(v) container_of(v, struct isc_clk, hw)
-
-struct isc_buffer {
-	struct vb2_v4l2_buffer  vb;
-	struct list_head	list;
-};
-
-struct isc_subdev_entity {
-	struct v4l2_subdev		*sd;
-	struct v4l2_async_connection	*asd;
-	struct device_node		*epn;
-	struct v4l2_async_notifier      notifier;
-
-	u32 pfe_cfg0;
-
-	struct list_head list;
-};
-
-/*
- * struct isc_format - ISC media bus format information
-			This structure represents the interface between the ISC
-			and the sensor. It's the input format received by
-			the ISC.
- * @fourcc:		Fourcc code for this format
- * @mbus_code:		V4L2 media bus format code.
- * @cfa_baycfg:		If this format is RAW BAYER, indicate the type of bayer.
-			this is either BGBG, RGRG, etc.
- * @pfe_cfg0_bps:	Number of hardware data lines connected to the ISC
- */
-
-struct isc_format {
-	u32	fourcc;
-	u32	mbus_code;
-	u32	cfa_baycfg;
-
-	bool	sd_support;
-	u32	pfe_cfg0_bps;
-};
-
-/* Pipeline bitmap */
-#define DPC_DPCENABLE	BIT(0)
-#define DPC_GDCENABLE	BIT(1)
-#define DPC_BLCENABLE	BIT(2)
-#define WB_ENABLE	BIT(3)
-#define CFA_ENABLE	BIT(4)
-#define CC_ENABLE	BIT(5)
-#define GAM_ENABLE	BIT(6)
-#define GAM_BENABLE	BIT(7)
-#define GAM_GENABLE	BIT(8)
-#define GAM_RENABLE	BIT(9)
-#define VHXS_ENABLE	BIT(10)
-#define CSC_ENABLE	BIT(11)
-#define CBC_ENABLE	BIT(12)
-#define SUB422_ENABLE	BIT(13)
-#define SUB420_ENABLE	BIT(14)
-
-#define GAM_ENABLES	(GAM_RENABLE | GAM_GENABLE | GAM_BENABLE | GAM_ENABLE)
-
-/*
- * struct fmt_config - ISC format configuration and internal pipeline
-			This structure represents the internal configuration
-			of the ISC.
-			It also holds the format that ISC will present to v4l2.
- * @sd_format:		Pointer to an isc_format struct that holds the sensor
-			configuration.
- * @fourcc:		Fourcc code for this format.
- * @bpp:		Bytes per pixel in the current format.
- * @bpp_v4l2:		Bytes per pixel in the current format, for v4l2.
-			This differs from 'bpp' in the sense that in planar
-			formats, it refers only to the first plane.
- * @rlp_cfg_mode:	Configuration of the RLP (rounding, limiting packaging)
- * @dcfg_imode:		Configuration of the input of the DMA module
- * @dctrl_dview:	Configuration of the output of the DMA module
- * @bits_pipeline:	Configuration of the pipeline, which modules are enabled
- */
-struct fmt_config {
-	struct isc_format	*sd_format;
-
-	u32			fourcc;
-	u8			bpp;
-	u8			bpp_v4l2;
-
-	u32			rlp_cfg_mode;
-	u32			dcfg_imode;
-	u32			dctrl_dview;
-
-	u32			bits_pipeline;
-};
-
-#define HIST_ENTRIES		512
-#define HIST_BAYER		(ISC_HIS_CFG_MODE_B + 1)
-
-enum{
-	HIST_INIT = 0,
-	HIST_ENABLED,
-	HIST_DISABLED,
-};
-
-struct isc_ctrls {
-	struct v4l2_ctrl_handler handler;
-
-	u32 brightness;
-	u32 contrast;
-	u8 gamma_index;
-#define ISC_WB_NONE	0
-#define ISC_WB_AUTO	1
-#define ISC_WB_ONETIME	2
-	u8 awb;
-
-	/* one for each component : GR, R, GB, B */
-	u32 gain[HIST_BAYER];
-	s32 offset[HIST_BAYER];
-
-	u32 hist_entry[HIST_ENTRIES];
-	u32 hist_count[HIST_BAYER];
-	u8 hist_id;
-	u8 hist_stat;
-#define HIST_MIN_INDEX		0
-#define HIST_MAX_INDEX		1
-	u32 hist_minmax[HIST_BAYER][2];
-};
-
-#define ISC_PIPE_LINE_NODE_NUM	15
-
-/*
- * struct isc_reg_offsets - ISC device register offsets
- * @csc:		Offset for the CSC register
- * @cbc:		Offset for the CBC register
- * @sub422:		Offset for the SUB422 register
- * @sub420:		Offset for the SUB420 register
- * @rlp:		Offset for the RLP register
- * @his:		Offset for the HIS related registers
- * @dma:		Offset for the DMA related registers
- * @version:		Offset for the version register
- * @his_entry:		Offset for the HIS entries registers
- */
-struct isc_reg_offsets {
-	u32 csc;
-	u32 cbc;
-	u32 sub422;
-	u32 sub420;
-	u32 rlp;
-	u32 his;
-	u32 dma;
-	u32 version;
-	u32 his_entry;
-};
-
-/*
- * struct isc_device - ISC device driver data/config struct
- * @regmap:		Register map
- * @hclock:		Hclock clock input (refer datasheet)
- * @ispck:		iscpck clock (refer datasheet)
- * @isc_clks:		ISC clocks
- * @ispck_required:	ISC requires ISP Clock initialization
- * @dcfg:		DMA master configuration, architecture dependent
- *
- * @dev:		Registered device driver
- * @v4l2_dev:		v4l2 registered device
- * @video_dev:		registered video device
- *
- * @vb2_vidq:		video buffer 2 video queue
- * @dma_queue_lock:	lock to serialize the dma buffer queue
- * @dma_queue:		the queue for dma buffers
- * @cur_frm:		current isc frame/buffer
- * @sequence:		current frame number
- * @stop:		true if isc is not streaming, false if streaming
- * @comp:		completion reference that signals frame completion
- *
- * @fmt:		current v42l format
- * @user_formats:	list of formats that are supported and agreed with sd
- * @num_user_formats:	how many formats are in user_formats
- *
- * @config:		current ISC format configuration
- * @try_config:		the current ISC try format , not yet activated
- *
- * @ctrls:		holds information about ISC controls
- * @do_wb_ctrl:		control regarding the DO_WHITE_BALANCE button
- * @awb_work:		workqueue reference for autowhitebalance histogram
- *			analysis
- *
- * @lock:		lock for serializing userspace file operations
- *			with ISC operations
- * @awb_mutex:		serialize access to streaming status from awb work queue
- * @awb_lock:		lock for serializing awb work queue operations
- *			with DMA/buffer operations
- *
- * @pipeline:		configuration of the ISC pipeline
- *
- * @current_subdev:	current subdevice: the sensor
- * @subdev_entities:	list of subdevice entitites
- *
- * @gamma_table:	pointer to the table with gamma values, has
- *			gamma_max sets of GAMMA_ENTRIES entries each
- * @gamma_max:		maximum number of sets of inside the gamma_table
- *
- * @max_width:		maximum frame width, dependent on the internal RAM
- * @max_height:		maximum frame height, dependent on the internal RAM
- *
- * @config_dpc:		pointer to a function that initializes product
- *			specific DPC module
- * @config_csc:		pointer to a function that initializes product
- *			specific CSC module
- * @config_cbc:		pointer to a function that initializes product
- *			specific CBC module
- * @config_cc:		pointer to a function that initializes product
- *			specific CC module
- * @config_gam:		pointer to a function that initializes product
- *			specific GAMMA module
- * @config_rlp:		pointer to a function that initializes product
- *			specific RLP module
- * @config_ctrls:	pointer to a functoin that initializes product
- *			specific v4l2 controls.
- *
- * @adapt_pipeline:	pointer to a function that adapts the pipeline bits
- *			to the product specific pipeline
- *
- * @offsets:		struct holding the product specific register offsets
- * @controller_formats:	pointer to the array of possible formats that the
- *			controller can output
- * @formats_list:	pointer to the array of possible formats that can
- *			be used as an input to the controller
- * @controller_formats_size:	size of controller_formats array
- * @formats_list_size:	size of formats_list array
- */
-struct isc_device {
-	struct regmap		*regmap;
-	struct clk		*hclock;
-	struct clk		*ispck;
-	struct isc_clk		isc_clks[2];
-	bool			ispck_required;
-	u32			dcfg;
-
-	struct device		*dev;
-	struct v4l2_device	v4l2_dev;
-	struct video_device	video_dev;
-
-	struct vb2_queue	vb2_vidq;
-	spinlock_t		dma_queue_lock;
-	struct list_head	dma_queue;
-	struct isc_buffer	*cur_frm;
-	unsigned int		sequence;
-	bool			stop;
-	struct completion	comp;
-
-	struct v4l2_format	fmt;
-	struct isc_format	**user_formats;
-	unsigned int		num_user_formats;
-
-	struct fmt_config	config;
-	struct fmt_config	try_config;
-
-	struct isc_ctrls	ctrls;
-	struct work_struct	awb_work;
-
-	struct mutex		lock;
-	struct mutex		awb_mutex;
-	spinlock_t		awb_lock;
-
-	struct regmap_field	*pipeline[ISC_PIPE_LINE_NODE_NUM];
-
-	struct isc_subdev_entity	*current_subdev;
-	struct list_head		subdev_entities;
-
-	struct {
-#define ISC_CTRL_DO_WB 1
-#define ISC_CTRL_R_GAIN 2
-#define ISC_CTRL_B_GAIN 3
-#define ISC_CTRL_GR_GAIN 4
-#define ISC_CTRL_GB_GAIN 5
-#define ISC_CTRL_R_OFF 6
-#define ISC_CTRL_B_OFF 7
-#define ISC_CTRL_GR_OFF 8
-#define ISC_CTRL_GB_OFF 9
-		struct v4l2_ctrl	*awb_ctrl;
-		struct v4l2_ctrl	*do_wb_ctrl;
-		struct v4l2_ctrl	*r_gain_ctrl;
-		struct v4l2_ctrl	*b_gain_ctrl;
-		struct v4l2_ctrl	*gr_gain_ctrl;
-		struct v4l2_ctrl	*gb_gain_ctrl;
-		struct v4l2_ctrl	*r_off_ctrl;
-		struct v4l2_ctrl	*b_off_ctrl;
-		struct v4l2_ctrl	*gr_off_ctrl;
-		struct v4l2_ctrl	*gb_off_ctrl;
-	};
-
-#define GAMMA_ENTRIES	64
-	/* pointer to the defined gamma table */
-	const u32	(*gamma_table)[GAMMA_ENTRIES];
-	u32		gamma_max;
-
-	u32		max_width;
-	u32		max_height;
-
-	struct {
-		void (*config_dpc)(struct isc_device *isc);
-		void (*config_csc)(struct isc_device *isc);
-		void (*config_cbc)(struct isc_device *isc);
-		void (*config_cc)(struct isc_device *isc);
-		void (*config_gam)(struct isc_device *isc);
-		void (*config_rlp)(struct isc_device *isc);
-
-		void (*config_ctrls)(struct isc_device *isc,
-				     const struct v4l2_ctrl_ops *ops);
-
-		void (*adapt_pipeline)(struct isc_device *isc);
-	};
-
-	struct isc_reg_offsets		offsets;
-	const struct isc_format		*controller_formats;
-	struct isc_format		*formats_list;
-	u32				controller_formats_size;
-	u32				formats_list_size;
-};
-
-extern const struct regmap_config atmel_isc_regmap_config;
-extern const struct v4l2_async_notifier_operations atmel_isc_async_ops;
-
-irqreturn_t atmel_isc_interrupt(int irq, void *dev_id);
-int atmel_isc_pipeline_init(struct isc_device *isc);
-int atmel_isc_clk_init(struct isc_device *isc);
-void atmel_isc_subdev_cleanup(struct isc_device *isc);
-void atmel_isc_clk_cleanup(struct isc_device *isc);
-
-#endif
diff --git a/drivers/staging/media/deprecated/atmel/atmel-sama5d2-isc.c b/drivers/staging/media/deprecated/atmel/atmel-sama5d2-isc.c
deleted file mode 100644
index 71e6e278a4b3c..0000000000000
--- a/drivers/staging/media/deprecated/atmel/atmel-sama5d2-isc.c
+++ /dev/null
@@ -1,644 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Microchip Image Sensor Controller (ISC) driver
- *
- * Copyright (C) 2016-2019 Microchip Technology, Inc.
- *
- * Author: Songjun Wu
- * Author: Eugen Hristev <eugen.hristev@microchip.com>
- *
- *
- * Sensor-->PFE-->WB-->CFA-->CC-->GAM-->CSC-->CBC-->SUB-->RLP-->DMA
- *
- * ISC video pipeline integrates the following submodules:
- * PFE: Parallel Front End to sample the camera sensor input stream
- *  WB: Programmable white balance in the Bayer domain
- * CFA: Color filter array interpolation module
- *  CC: Programmable color correction
- * GAM: Gamma correction
- * CSC: Programmable color space conversion
- * CBC: Contrast and Brightness control
- * SUB: This module performs YCbCr444 to YCbCr420 chrominance subsampling
- * RLP: This module performs rounding, range limiting
- *      and packing of the incoming data
- */
-
-#include <linux/clk.h>
-#include <linux/clkdev.h>
-#include <linux/clk-provider.h>
-#include <linux/delay.h>
-#include <linux/interrupt.h>
-#include <linux/math64.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_graph.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/regmap.h>
-#include <linux/videodev2.h>
-
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-event.h>
-#include <media/v4l2-image-sizes.h>
-#include <media/v4l2-ioctl.h>
-#include <media/v4l2-fwnode.h>
-#include <media/v4l2-subdev.h>
-#include <media/videobuf2-dma-contig.h>
-
-#include "atmel-isc-regs.h"
-#include "atmel-isc.h"
-
-#define ISC_SAMA5D2_MAX_SUPPORT_WIDTH   2592
-#define ISC_SAMA5D2_MAX_SUPPORT_HEIGHT  1944
-
-#define ISC_SAMA5D2_PIPELINE \
-	(WB_ENABLE | CFA_ENABLE | CC_ENABLE | GAM_ENABLES | CSC_ENABLE | \
-	CBC_ENABLE | SUB422_ENABLE | SUB420_ENABLE)
-
-/* This is a list of the formats that the ISC can *output* */
-static const struct isc_format sama5d2_controller_formats[] = {
-	{
-		.fourcc		= V4L2_PIX_FMT_ARGB444,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_ARGB555,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_RGB565,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_ABGR32,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_XBGR32,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_YUV420,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_YUYV,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_YUV422P,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_GREY,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_Y10,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SBGGR8,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SGBRG8,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SGRBG8,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SRGGB8,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SBGGR10,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SGBRG10,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SGRBG10,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SRGGB10,
-	},
-};
-
-/* This is a list of formats that the ISC can receive as *input* */
-static struct isc_format sama5d2_formats_list[] = {
-	{
-		.fourcc		= V4L2_PIX_FMT_SBGGR8,
-		.mbus_code	= MEDIA_BUS_FMT_SBGGR8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGBRG8,
-		.mbus_code	= MEDIA_BUS_FMT_SGBRG8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGRBG8,
-		.mbus_code	= MEDIA_BUS_FMT_SGRBG8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SRGGB8,
-		.mbus_code	= MEDIA_BUS_FMT_SRGGB8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SBGGR10,
-		.mbus_code	= MEDIA_BUS_FMT_SBGGR10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGBRG10,
-		.mbus_code	= MEDIA_BUS_FMT_SGBRG10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGRBG10,
-		.mbus_code	= MEDIA_BUS_FMT_SGRBG10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SRGGB10,
-		.mbus_code	= MEDIA_BUS_FMT_SRGGB10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SBGGR12,
-		.mbus_code	= MEDIA_BUS_FMT_SBGGR12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGBRG12,
-		.mbus_code	= MEDIA_BUS_FMT_SGBRG12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
-		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGRBG12,
-		.mbus_code	= MEDIA_BUS_FMT_SGRBG12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
-		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SRGGB12,
-		.mbus_code	= MEDIA_BUS_FMT_SRGGB12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_GREY,
-		.mbus_code	= MEDIA_BUS_FMT_Y8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_YUYV,
-		.mbus_code	= MEDIA_BUS_FMT_YUYV8_2X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_RGB565,
-		.mbus_code	= MEDIA_BUS_FMT_RGB565_2X8_LE,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_Y10,
-		.mbus_code	= MEDIA_BUS_FMT_Y10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-	},
-
-};
-
-static void isc_sama5d2_config_csc(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-
-	/* Convert RGB to YUV */
-	regmap_write(regmap, ISC_CSC_YR_YG + isc->offsets.csc,
-		     0x42 | (0x81 << 16));
-	regmap_write(regmap, ISC_CSC_YB_OY + isc->offsets.csc,
-		     0x19 | (0x10 << 16));
-	regmap_write(regmap, ISC_CSC_CBR_CBG + isc->offsets.csc,
-		     0xFDA | (0xFB6 << 16));
-	regmap_write(regmap, ISC_CSC_CBB_OCB + isc->offsets.csc,
-		     0x70 | (0x80 << 16));
-	regmap_write(regmap, ISC_CSC_CRR_CRG + isc->offsets.csc,
-		     0x70 | (0xFA2 << 16));
-	regmap_write(regmap, ISC_CSC_CRB_OCR + isc->offsets.csc,
-		     0xFEE | (0x80 << 16));
-}
-
-static void isc_sama5d2_config_cbc(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-
-	regmap_write(regmap, ISC_CBC_BRIGHT + isc->offsets.cbc,
-		     isc->ctrls.brightness);
-	regmap_write(regmap, ISC_CBC_CONTRAST + isc->offsets.cbc,
-		     isc->ctrls.contrast);
-}
-
-static void isc_sama5d2_config_cc(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-
-	/* Configure each register at the neutral fixed point 1.0 or 0.0 */
-	regmap_write(regmap, ISC_CC_RR_RG, (1 << 8));
-	regmap_write(regmap, ISC_CC_RB_OR, 0);
-	regmap_write(regmap, ISC_CC_GR_GG, (1 << 8) << 16);
-	regmap_write(regmap, ISC_CC_GB_OG, 0);
-	regmap_write(regmap, ISC_CC_BR_BG, 0);
-	regmap_write(regmap, ISC_CC_BB_OB, (1 << 8));
-}
-
-static void isc_sama5d2_config_ctrls(struct isc_device *isc,
-				     const struct v4l2_ctrl_ops *ops)
-{
-	struct isc_ctrls *ctrls = &isc->ctrls;
-	struct v4l2_ctrl_handler *hdl = &ctrls->handler;
-
-	ctrls->contrast = 256;
-
-	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_CONTRAST, -2048, 2047, 1, 256);
-}
-
-static void isc_sama5d2_config_dpc(struct isc_device *isc)
-{
-	/* This module is not present on sama5d2 pipeline */
-}
-
-static void isc_sama5d2_config_gam(struct isc_device *isc)
-{
-	/* No specific gamma configuration */
-}
-
-static void isc_sama5d2_config_rlp(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-	u32 rlp_mode = isc->config.rlp_cfg_mode;
-
-	/*
-	 * In sama5d2, the YUV planar modes and the YUYV modes are treated
-	 * in the same way in RLP register.
-	 * Normally, YYCC mode should be Luma(n) - Color B(n) - Color R (n)
-	 * and YCYC should be Luma(n + 1) - Color B (n) - Luma (n) - Color R (n)
-	 * but in sama5d2, the YCYC mode does not exist, and YYCC must be
-	 * selected for both planar and interleaved modes, as in fact
-	 * both modes are supported.
-	 *
-	 * Thus, if the YCYC mode is selected, replace it with the
-	 * sama5d2-compliant mode which is YYCC .
-	 */
-	if ((rlp_mode & ISC_RLP_CFG_MODE_MASK) == ISC_RLP_CFG_MODE_YCYC) {
-		rlp_mode &= ~ISC_RLP_CFG_MODE_MASK;
-		rlp_mode |= ISC_RLP_CFG_MODE_YYCC;
-	}
-
-	regmap_update_bits(regmap, ISC_RLP_CFG + isc->offsets.rlp,
-			   ISC_RLP_CFG_MODE_MASK, rlp_mode);
-}
-
-static void isc_sama5d2_adapt_pipeline(struct isc_device *isc)
-{
-	isc->try_config.bits_pipeline &= ISC_SAMA5D2_PIPELINE;
-}
-
-/* Gamma table with gamma 1/2.2 */
-static const u32 isc_sama5d2_gamma_table[][GAMMA_ENTRIES] = {
-	/* 0 --> gamma 1/1.8 */
-	{      0x65,  0x66002F,  0x950025,  0xBB0020,  0xDB001D,  0xF8001A,
-	  0x1130018, 0x12B0017, 0x1420016, 0x1580014, 0x16D0013, 0x1810012,
-	  0x1940012, 0x1A60012, 0x1B80011, 0x1C90010, 0x1DA0010, 0x1EA000F,
-	  0x1FA000F, 0x209000F, 0x218000F, 0x227000E, 0x235000E, 0x243000E,
-	  0x251000E, 0x25F000D, 0x26C000D, 0x279000D, 0x286000D, 0x293000C,
-	  0x2A0000C, 0x2AC000C, 0x2B8000C, 0x2C4000C, 0x2D0000B, 0x2DC000B,
-	  0x2E7000B, 0x2F3000B, 0x2FE000B, 0x309000B, 0x314000B, 0x31F000A,
-	  0x32A000A, 0x334000B, 0x33F000A, 0x349000A, 0x354000A, 0x35E000A,
-	  0x368000A, 0x372000A, 0x37C000A, 0x386000A, 0x3900009, 0x399000A,
-	  0x3A30009, 0x3AD0009, 0x3B60009, 0x3BF000A, 0x3C90009, 0x3D20009,
-	  0x3DB0009, 0x3E40009, 0x3ED0009, 0x3F60009 },
-
-	/* 1 --> gamma 1/2 */
-	{      0x7F,  0x800034,  0xB50028,  0xDE0021, 0x100001E, 0x11E001B,
-	  0x1390019, 0x1520017, 0x16A0015, 0x1800014, 0x1940014, 0x1A80013,
-	  0x1BB0012, 0x1CD0011, 0x1DF0010, 0x1EF0010, 0x200000F, 0x20F000F,
-	  0x21F000E, 0x22D000F, 0x23C000E, 0x24A000E, 0x258000D, 0x265000D,
-	  0x273000C, 0x27F000D, 0x28C000C, 0x299000C, 0x2A5000C, 0x2B1000B,
-	  0x2BC000C, 0x2C8000B, 0x2D3000C, 0x2DF000B, 0x2EA000A, 0x2F5000A,
-	  0x2FF000B, 0x30A000A, 0x314000B, 0x31F000A, 0x329000A, 0x333000A,
-	  0x33D0009, 0x3470009, 0x350000A, 0x35A0009, 0x363000A, 0x36D0009,
-	  0x3760009, 0x37F0009, 0x3880009, 0x3910009, 0x39A0009, 0x3A30009,
-	  0x3AC0008, 0x3B40009, 0x3BD0008, 0x3C60008, 0x3CE0008, 0x3D60009,
-	  0x3DF0008, 0x3E70008, 0x3EF0008, 0x3F70008 },
-
-	/* 2 --> gamma 1/2.2 */
-	{      0x99,  0x9B0038,  0xD4002A,  0xFF0023, 0x122001F, 0x141001B,
-	  0x15D0019, 0x1760017, 0x18E0015, 0x1A30015, 0x1B80013, 0x1CC0012,
-	  0x1DE0011, 0x1F00010, 0x2010010, 0x2110010, 0x221000F, 0x230000F,
-	  0x23F000E, 0x24D000E, 0x25B000D, 0x269000C, 0x276000C, 0x283000C,
-	  0x28F000C, 0x29B000C, 0x2A7000C, 0x2B3000B, 0x2BF000B, 0x2CA000B,
-	  0x2D5000B, 0x2E0000A, 0x2EB000A, 0x2F5000A, 0x2FF000A, 0x30A000A,
-	  0x3140009, 0x31E0009, 0x327000A, 0x3310009, 0x33A0009, 0x3440009,
-	  0x34D0009, 0x3560009, 0x35F0009, 0x3680008, 0x3710008, 0x3790009,
-	  0x3820008, 0x38A0008, 0x3930008, 0x39B0008, 0x3A30008, 0x3AB0008,
-	  0x3B30008, 0x3BB0008, 0x3C30008, 0x3CB0007, 0x3D20008, 0x3DA0007,
-	  0x3E20007, 0x3E90007, 0x3F00008, 0x3F80007 },
-};
-
-static int isc_parse_dt(struct device *dev, struct isc_device *isc)
-{
-	struct device_node *np = dev->of_node;
-	struct device_node *epn;
-	struct isc_subdev_entity *subdev_entity;
-	unsigned int flags;
-	int ret = -EINVAL;
-
-	INIT_LIST_HEAD(&isc->subdev_entities);
-
-	for_each_endpoint_of_node(np, epn) {
-		struct v4l2_fwnode_endpoint v4l2_epn = { .bus_type = 0 };
-
-		ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(epn),
-						 &v4l2_epn);
-		if (ret) {
-			ret = -EINVAL;
-			dev_err(dev, "Could not parse the endpoint\n");
-			break;
-		}
-
-		subdev_entity = devm_kzalloc(dev, sizeof(*subdev_entity),
-					     GFP_KERNEL);
-		if (!subdev_entity) {
-			ret = -ENOMEM;
-			break;
-		}
-		subdev_entity->epn = epn;
-
-		flags = v4l2_epn.bus.parallel.flags;
-
-		if (flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)
-			subdev_entity->pfe_cfg0 = ISC_PFE_CFG0_HPOL_LOW;
-
-		if (flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
-			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_VPOL_LOW;
-
-		if (flags & V4L2_MBUS_PCLK_SAMPLE_FALLING)
-			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_PPOL_LOW;
-
-		if (v4l2_epn.bus_type == V4L2_MBUS_BT656)
-			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_CCIR_CRC |
-					ISC_PFE_CFG0_CCIR656;
-
-		list_add_tail(&subdev_entity->list, &isc->subdev_entities);
-	}
-	of_node_put(epn);
-
-	return ret;
-}
-
-static int atmel_isc_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct isc_device *isc;
-	void __iomem *io_base;
-	struct isc_subdev_entity *subdev_entity;
-	int irq;
-	int ret;
-	u32 ver;
-
-	isc = devm_kzalloc(dev, sizeof(*isc), GFP_KERNEL);
-	if (!isc)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, isc);
-	isc->dev = dev;
-
-	io_base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(io_base))
-		return PTR_ERR(io_base);
-
-	isc->regmap = devm_regmap_init_mmio(dev, io_base, &atmel_isc_regmap_config);
-	if (IS_ERR(isc->regmap)) {
-		ret = PTR_ERR(isc->regmap);
-		dev_err(dev, "failed to init register map: %d\n", ret);
-		return ret;
-	}
-
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-
-	ret = devm_request_irq(dev, irq, atmel_isc_interrupt, 0,
-			       "atmel-sama5d2-isc", isc);
-	if (ret < 0) {
-		dev_err(dev, "can't register ISR for IRQ %u (ret=%i)\n",
-			irq, ret);
-		return ret;
-	}
-
-	isc->gamma_table = isc_sama5d2_gamma_table;
-	isc->gamma_max = 2;
-
-	isc->max_width = ISC_SAMA5D2_MAX_SUPPORT_WIDTH;
-	isc->max_height = ISC_SAMA5D2_MAX_SUPPORT_HEIGHT;
-
-	isc->config_dpc = isc_sama5d2_config_dpc;
-	isc->config_csc = isc_sama5d2_config_csc;
-	isc->config_cbc = isc_sama5d2_config_cbc;
-	isc->config_cc = isc_sama5d2_config_cc;
-	isc->config_gam = isc_sama5d2_config_gam;
-	isc->config_rlp = isc_sama5d2_config_rlp;
-	isc->config_ctrls = isc_sama5d2_config_ctrls;
-
-	isc->adapt_pipeline = isc_sama5d2_adapt_pipeline;
-
-	isc->offsets.csc = ISC_SAMA5D2_CSC_OFFSET;
-	isc->offsets.cbc = ISC_SAMA5D2_CBC_OFFSET;
-	isc->offsets.sub422 = ISC_SAMA5D2_SUB422_OFFSET;
-	isc->offsets.sub420 = ISC_SAMA5D2_SUB420_OFFSET;
-	isc->offsets.rlp = ISC_SAMA5D2_RLP_OFFSET;
-	isc->offsets.his = ISC_SAMA5D2_HIS_OFFSET;
-	isc->offsets.dma = ISC_SAMA5D2_DMA_OFFSET;
-	isc->offsets.version = ISC_SAMA5D2_VERSION_OFFSET;
-	isc->offsets.his_entry = ISC_SAMA5D2_HIS_ENTRY_OFFSET;
-
-	isc->controller_formats = sama5d2_controller_formats;
-	isc->controller_formats_size = ARRAY_SIZE(sama5d2_controller_formats);
-	isc->formats_list = sama5d2_formats_list;
-	isc->formats_list_size = ARRAY_SIZE(sama5d2_formats_list);
-
-	/* sama5d2-isc - 8 bits per beat */
-	isc->dcfg = ISC_DCFG_YMBSIZE_BEATS8 | ISC_DCFG_CMBSIZE_BEATS8;
-
-	/* sama5d2-isc : ISPCK is required and mandatory */
-	isc->ispck_required = true;
-
-	ret = atmel_isc_pipeline_init(isc);
-	if (ret)
-		return ret;
-
-	isc->hclock = devm_clk_get(dev, "hclock");
-	if (IS_ERR(isc->hclock)) {
-		ret = PTR_ERR(isc->hclock);
-		dev_err(dev, "failed to get hclock: %d\n", ret);
-		return ret;
-	}
-
-	ret = clk_prepare_enable(isc->hclock);
-	if (ret) {
-		dev_err(dev, "failed to enable hclock: %d\n", ret);
-		return ret;
-	}
-
-	ret = atmel_isc_clk_init(isc);
-	if (ret) {
-		dev_err(dev, "failed to init isc clock: %d\n", ret);
-		goto unprepare_hclk;
-	}
-	ret = v4l2_device_register(dev, &isc->v4l2_dev);
-	if (ret) {
-		dev_err(dev, "unable to register v4l2 device.\n");
-		goto unprepare_clk;
-	}
-
-	ret = isc_parse_dt(dev, isc);
-	if (ret) {
-		dev_err(dev, "fail to parse device tree\n");
-		goto unregister_v4l2_device;
-	}
-
-	if (list_empty(&isc->subdev_entities)) {
-		dev_err(dev, "no subdev found\n");
-		ret = -ENODEV;
-		goto unregister_v4l2_device;
-	}
-
-	list_for_each_entry(subdev_entity, &isc->subdev_entities, list) {
-		struct v4l2_async_connection *asd;
-		struct fwnode_handle *fwnode =
-			of_fwnode_handle(subdev_entity->epn);
-
-		v4l2_async_nf_init(&subdev_entity->notifier, &isc->v4l2_dev);
-
-		asd = v4l2_async_nf_add_fwnode_remote(&subdev_entity->notifier,
-						      fwnode,
-						      struct v4l2_async_connection);
-
-		of_node_put(subdev_entity->epn);
-		subdev_entity->epn = NULL;
-
-		if (IS_ERR(asd)) {
-			ret = PTR_ERR(asd);
-			goto cleanup_subdev;
-		}
-
-		subdev_entity->notifier.ops = &atmel_isc_async_ops;
-
-		ret = v4l2_async_nf_register(&subdev_entity->notifier);
-		if (ret) {
-			dev_err(dev, "fail to register async notifier\n");
-			goto cleanup_subdev;
-		}
-
-		if (video_is_registered(&isc->video_dev))
-			break;
-	}
-
-	pm_runtime_set_active(dev);
-	pm_runtime_enable(dev);
-	pm_request_idle(dev);
-
-	isc->ispck = isc->isc_clks[ISC_ISPCK].clk;
-
-	ret = clk_prepare_enable(isc->ispck);
-	if (ret) {
-		dev_err(dev, "failed to enable ispck: %d\n", ret);
-		goto disable_pm;
-	}
-
-	/* ispck should be greater or equal to hclock */
-	ret = clk_set_rate(isc->ispck, clk_get_rate(isc->hclock));
-	if (ret) {
-		dev_err(dev, "failed to set ispck rate: %d\n", ret);
-		goto unprepare_clk;
-	}
-
-	regmap_read(isc->regmap, ISC_VERSION + isc->offsets.version, &ver);
-	dev_info(dev, "Microchip ISC version %x\n", ver);
-
-	return 0;
-
-unprepare_clk:
-	clk_disable_unprepare(isc->ispck);
-
-disable_pm:
-	pm_runtime_disable(dev);
-
-cleanup_subdev:
-	atmel_isc_subdev_cleanup(isc);
-
-unregister_v4l2_device:
-	v4l2_device_unregister(&isc->v4l2_dev);
-
-unprepare_hclk:
-	clk_disable_unprepare(isc->hclock);
-
-	atmel_isc_clk_cleanup(isc);
-
-	return ret;
-}
-
-static void atmel_isc_remove(struct platform_device *pdev)
-{
-	struct isc_device *isc = platform_get_drvdata(pdev);
-
-	pm_runtime_disable(&pdev->dev);
-
-	atmel_isc_subdev_cleanup(isc);
-
-	v4l2_device_unregister(&isc->v4l2_dev);
-
-	clk_disable_unprepare(isc->ispck);
-	clk_disable_unprepare(isc->hclock);
-
-	atmel_isc_clk_cleanup(isc);
-}
-
-static int __maybe_unused isc_runtime_suspend(struct device *dev)
-{
-	struct isc_device *isc = dev_get_drvdata(dev);
-
-	clk_disable_unprepare(isc->ispck);
-	clk_disable_unprepare(isc->hclock);
-
-	return 0;
-}
-
-static int __maybe_unused isc_runtime_resume(struct device *dev)
-{
-	struct isc_device *isc = dev_get_drvdata(dev);
-	int ret;
-
-	ret = clk_prepare_enable(isc->hclock);
-	if (ret)
-		return ret;
-
-	ret = clk_prepare_enable(isc->ispck);
-	if (ret)
-		clk_disable_unprepare(isc->hclock);
-
-	return ret;
-}
-
-static const struct dev_pm_ops atmel_isc_dev_pm_ops = {
-	SET_RUNTIME_PM_OPS(isc_runtime_suspend, isc_runtime_resume, NULL)
-};
-
-#if IS_ENABLED(CONFIG_OF)
-static const struct of_device_id atmel_isc_of_match[] = {
-	{ .compatible = "atmel,sama5d2-isc" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, atmel_isc_of_match);
-#endif
-
-static struct platform_driver atmel_isc_driver = {
-	.probe	= atmel_isc_probe,
-	.remove = atmel_isc_remove,
-	.driver	= {
-		.name		= "atmel-sama5d2-isc",
-		.pm		= &atmel_isc_dev_pm_ops,
-		.of_match_table = of_match_ptr(atmel_isc_of_match),
-	},
-};
-
-module_platform_driver(atmel_isc_driver);
-
-MODULE_AUTHOR("Songjun Wu");
-MODULE_DESCRIPTION("The V4L2 driver for Atmel-ISC");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/media/deprecated/atmel/atmel-sama7g5-isc.c b/drivers/staging/media/deprecated/atmel/atmel-sama7g5-isc.c
deleted file mode 100644
index 1f74c2dd044cf..0000000000000
--- a/drivers/staging/media/deprecated/atmel/atmel-sama7g5-isc.c
+++ /dev/null
@@ -1,607 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Microchip eXtended Image Sensor Controller (XISC) driver
- *
- * Copyright (C) 2019-2021 Microchip Technology, Inc. and its subsidiaries
- *
- * Author: Eugen Hristev <eugen.hristev@microchip.com>
- *
- * Sensor-->PFE-->DPC-->WB-->CFA-->CC-->GAM-->VHXS-->CSC-->CBHS-->SUB-->RLP-->DMA-->HIS
- *
- * ISC video pipeline integrates the following submodules:
- * PFE: Parallel Front End to sample the camera sensor input stream
- * DPC: Defective Pixel Correction with black offset correction, green disparity
- *      correction and defective pixel correction (3 modules total)
- *  WB: Programmable white balance in the Bayer domain
- * CFA: Color filter array interpolation module
- *  CC: Programmable color correction
- * GAM: Gamma correction
- *VHXS: Vertical and Horizontal Scaler
- * CSC: Programmable color space conversion
- *CBHS: Contrast Brightness Hue and Saturation control
- * SUB: This module performs YCbCr444 to YCbCr420 chrominance subsampling
- * RLP: This module performs rounding, range limiting
- *      and packing of the incoming data
- * DMA: This module performs DMA master accesses to write frames to external RAM
- * HIS: Histogram module performs statistic counters on the frames
- */
-
-#include <linux/clk.h>
-#include <linux/clkdev.h>
-#include <linux/clk-provider.h>
-#include <linux/delay.h>
-#include <linux/interrupt.h>
-#include <linux/math64.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/of_graph.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-#include <linux/regmap.h>
-#include <linux/videodev2.h>
-
-#include <media/v4l2-ctrls.h>
-#include <media/v4l2-device.h>
-#include <media/v4l2-event.h>
-#include <media/v4l2-image-sizes.h>
-#include <media/v4l2-ioctl.h>
-#include <media/v4l2-fwnode.h>
-#include <media/v4l2-subdev.h>
-#include <media/videobuf2-dma-contig.h>
-
-#include "atmel-isc-regs.h"
-#include "atmel-isc.h"
-
-#define ISC_SAMA7G5_MAX_SUPPORT_WIDTH   3264
-#define ISC_SAMA7G5_MAX_SUPPORT_HEIGHT  2464
-
-#define ISC_SAMA7G5_PIPELINE \
-	(WB_ENABLE | CFA_ENABLE | CC_ENABLE | GAM_ENABLES | CSC_ENABLE | \
-	CBC_ENABLE | SUB422_ENABLE | SUB420_ENABLE)
-
-/* This is a list of the formats that the ISC can *output* */
-static const struct isc_format sama7g5_controller_formats[] = {
-	{
-		.fourcc		= V4L2_PIX_FMT_ARGB444,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_ARGB555,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_RGB565,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_ABGR32,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_XBGR32,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_YUV420,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_UYVY,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_VYUY,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_YUYV,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_YUV422P,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_GREY,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_Y10,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_Y16,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SBGGR8,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SGBRG8,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SGRBG8,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SRGGB8,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SBGGR10,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SGBRG10,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SGRBG10,
-	}, {
-		.fourcc		= V4L2_PIX_FMT_SRGGB10,
-	},
-};
-
-/* This is a list of formats that the ISC can receive as *input* */
-static struct isc_format sama7g5_formats_list[] = {
-	{
-		.fourcc		= V4L2_PIX_FMT_SBGGR8,
-		.mbus_code	= MEDIA_BUS_FMT_SBGGR8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGBRG8,
-		.mbus_code	= MEDIA_BUS_FMT_SGBRG8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGRBG8,
-		.mbus_code	= MEDIA_BUS_FMT_SGRBG8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SRGGB8,
-		.mbus_code	= MEDIA_BUS_FMT_SRGGB8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SBGGR10,
-		.mbus_code	= MEDIA_BUS_FMT_SBGGR10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGBRG10,
-		.mbus_code	= MEDIA_BUS_FMT_SGBRG10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGRBG10,
-		.mbus_code	= MEDIA_BUS_FMT_SGRBG10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SRGGB10,
-		.mbus_code	= MEDIA_BUS_FMT_SRGGB10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SBGGR12,
-		.mbus_code	= MEDIA_BUS_FMT_SBGGR12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
-		.cfa_baycfg	= ISC_BAY_CFG_BGBG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGBRG12,
-		.mbus_code	= MEDIA_BUS_FMT_SGBRG12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
-		.cfa_baycfg	= ISC_BAY_CFG_GBGB,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SGRBG12,
-		.mbus_code	= MEDIA_BUS_FMT_SGRBG12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
-		.cfa_baycfg	= ISC_BAY_CFG_GRGR,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_SRGGB12,
-		.mbus_code	= MEDIA_BUS_FMT_SRGGB12_1X12,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TWELVE,
-		.cfa_baycfg	= ISC_BAY_CFG_RGRG,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_GREY,
-		.mbus_code	= MEDIA_BUS_FMT_Y8_1X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_YUYV,
-		.mbus_code	= MEDIA_BUS_FMT_YUYV8_2X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_UYVY,
-		.mbus_code	= MEDIA_BUS_FMT_UYVY8_2X8,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_RGB565,
-		.mbus_code	= MEDIA_BUS_FMT_RGB565_2X8_LE,
-		.pfe_cfg0_bps	= ISC_PFE_CFG0_BPS_EIGHT,
-	},
-	{
-		.fourcc		= V4L2_PIX_FMT_Y10,
-		.mbus_code	= MEDIA_BUS_FMT_Y10_1X10,
-		.pfe_cfg0_bps	= ISC_PFG_CFG0_BPS_TEN,
-	},
-};
-
-static void isc_sama7g5_config_csc(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-
-	/* Convert RGB to YUV */
-	regmap_write(regmap, ISC_CSC_YR_YG + isc->offsets.csc,
-		     0x42 | (0x81 << 16));
-	regmap_write(regmap, ISC_CSC_YB_OY + isc->offsets.csc,
-		     0x19 | (0x10 << 16));
-	regmap_write(regmap, ISC_CSC_CBR_CBG + isc->offsets.csc,
-		     0xFDA | (0xFB6 << 16));
-	regmap_write(regmap, ISC_CSC_CBB_OCB + isc->offsets.csc,
-		     0x70 | (0x80 << 16));
-	regmap_write(regmap, ISC_CSC_CRR_CRG + isc->offsets.csc,
-		     0x70 | (0xFA2 << 16));
-	regmap_write(regmap, ISC_CSC_CRB_OCR + isc->offsets.csc,
-		     0xFEE | (0x80 << 16));
-}
-
-static void isc_sama7g5_config_cbc(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-
-	/* Configure what is set via v4l2 ctrls */
-	regmap_write(regmap, ISC_CBC_BRIGHT + isc->offsets.cbc, isc->ctrls.brightness);
-	regmap_write(regmap, ISC_CBC_CONTRAST + isc->offsets.cbc, isc->ctrls.contrast);
-	/* Configure Hue and Saturation as neutral midpoint */
-	regmap_write(regmap, ISC_CBCHS_HUE, 0);
-	regmap_write(regmap, ISC_CBCHS_SAT, (1 << 4));
-}
-
-static void isc_sama7g5_config_cc(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-
-	/* Configure each register at the neutral fixed point 1.0 or 0.0 */
-	regmap_write(regmap, ISC_CC_RR_RG, (1 << 8));
-	regmap_write(regmap, ISC_CC_RB_OR, 0);
-	regmap_write(regmap, ISC_CC_GR_GG, (1 << 8) << 16);
-	regmap_write(regmap, ISC_CC_GB_OG, 0);
-	regmap_write(regmap, ISC_CC_BR_BG, 0);
-	regmap_write(regmap, ISC_CC_BB_OB, (1 << 8));
-}
-
-static void isc_sama7g5_config_ctrls(struct isc_device *isc,
-				     const struct v4l2_ctrl_ops *ops)
-{
-	struct isc_ctrls *ctrls = &isc->ctrls;
-	struct v4l2_ctrl_handler *hdl = &ctrls->handler;
-
-	ctrls->contrast = 16;
-
-	v4l2_ctrl_new_std(hdl, ops, V4L2_CID_CONTRAST, -2048, 2047, 1, 16);
-}
-
-static void isc_sama7g5_config_dpc(struct isc_device *isc)
-{
-	u32 bay_cfg = isc->config.sd_format->cfa_baycfg;
-	struct regmap *regmap = isc->regmap;
-
-	regmap_update_bits(regmap, ISC_DPC_CFG, ISC_DPC_CFG_BLOFF_MASK,
-			   (64 << ISC_DPC_CFG_BLOFF_SHIFT));
-	regmap_update_bits(regmap, ISC_DPC_CFG, ISC_DPC_CFG_BAYCFG_MASK,
-			   (bay_cfg << ISC_DPC_CFG_BAYCFG_SHIFT));
-}
-
-static void isc_sama7g5_config_gam(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-
-	regmap_update_bits(regmap, ISC_GAM_CTRL, ISC_GAM_CTRL_BIPART,
-			   ISC_GAM_CTRL_BIPART);
-}
-
-static void isc_sama7g5_config_rlp(struct isc_device *isc)
-{
-	struct regmap *regmap = isc->regmap;
-	u32 rlp_mode = isc->config.rlp_cfg_mode;
-
-	regmap_update_bits(regmap, ISC_RLP_CFG + isc->offsets.rlp,
-			   ISC_RLP_CFG_MODE_MASK | ISC_RLP_CFG_LSH |
-			   ISC_RLP_CFG_YMODE_MASK, rlp_mode);
-}
-
-static void isc_sama7g5_adapt_pipeline(struct isc_device *isc)
-{
-	isc->try_config.bits_pipeline &= ISC_SAMA7G5_PIPELINE;
-}
-
-/* Gamma table with gamma 1/2.2 */
-static const u32 isc_sama7g5_gamma_table[][GAMMA_ENTRIES] = {
-	/* index 0 --> gamma bipartite */
-	{
-	      0x980,  0x4c0320,  0x650260,  0x7801e0,  0x8701a0,  0x940180,
-	   0xa00160,  0xab0120,  0xb40120,  0xbd0120,  0xc60100,  0xce0100,
-	   0xd600e0,  0xdd00e0,  0xe400e0,  0xeb00c0,  0xf100c0,  0xf700c0,
-	   0xfd00c0, 0x10300a0, 0x10800c0, 0x10e00a0, 0x11300a0, 0x11800a0,
-	  0x11d00a0, 0x12200a0, 0x12700a0, 0x12c0080, 0x13000a0, 0x1350080,
-	  0x13900a0, 0x13e0080, 0x1420076, 0x17d0062, 0x1ae0054, 0x1d8004a,
-	  0x1fd0044, 0x21f003e, 0x23e003a, 0x25b0036, 0x2760032, 0x28f0030,
-	  0x2a7002e, 0x2be002c, 0x2d4002c, 0x2ea0028, 0x2fe0028, 0x3120026,
-	  0x3250024, 0x3370024, 0x3490022, 0x35a0022, 0x36b0020, 0x37b0020,
-	  0x38b0020, 0x39b001e, 0x3aa001e, 0x3b9001c, 0x3c7001c, 0x3d5001c,
-	  0x3e3001c, 0x3f1001c, 0x3ff001a, 0x40c001a },
-};
-
-static int xisc_parse_dt(struct device *dev, struct isc_device *isc)
-{
-	struct device_node *np = dev->of_node;
-	struct device_node *epn;
-	struct isc_subdev_entity *subdev_entity;
-	unsigned int flags;
-	int ret = -EINVAL;
-	bool mipi_mode;
-
-	INIT_LIST_HEAD(&isc->subdev_entities);
-
-	mipi_mode = of_property_read_bool(np, "microchip,mipi-mode");
-
-	for_each_endpoint_of_node(np, epn) {
-		struct v4l2_fwnode_endpoint v4l2_epn = { .bus_type = 0 };
-
-		ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(epn),
-						 &v4l2_epn);
-		if (ret) {
-			ret = -EINVAL;
-			dev_err(dev, "Could not parse the endpoint\n");
-			break;
-		}
-
-		subdev_entity = devm_kzalloc(dev, sizeof(*subdev_entity),
-					     GFP_KERNEL);
-		if (!subdev_entity) {
-			ret = -ENOMEM;
-			break;
-		}
-		subdev_entity->epn = epn;
-
-		flags = v4l2_epn.bus.parallel.flags;
-
-		if (flags & V4L2_MBUS_HSYNC_ACTIVE_LOW)
-			subdev_entity->pfe_cfg0 = ISC_PFE_CFG0_HPOL_LOW;
-
-		if (flags & V4L2_MBUS_VSYNC_ACTIVE_LOW)
-			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_VPOL_LOW;
-
-		if (flags & V4L2_MBUS_PCLK_SAMPLE_FALLING)
-			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_PPOL_LOW;
-
-		if (v4l2_epn.bus_type == V4L2_MBUS_BT656)
-			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_CCIR_CRC |
-					ISC_PFE_CFG0_CCIR656;
-
-		if (mipi_mode)
-			subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_MIPI;
-
-		list_add_tail(&subdev_entity->list, &isc->subdev_entities);
-	}
-	of_node_put(epn);
-
-	return ret;
-}
-
-static int microchip_xisc_probe(struct platform_device *pdev)
-{
-	struct device *dev = &pdev->dev;
-	struct isc_device *isc;
-	void __iomem *io_base;
-	struct isc_subdev_entity *subdev_entity;
-	int irq;
-	int ret;
-	u32 ver;
-
-	isc = devm_kzalloc(dev, sizeof(*isc), GFP_KERNEL);
-	if (!isc)
-		return -ENOMEM;
-
-	platform_set_drvdata(pdev, isc);
-	isc->dev = dev;
-
-	io_base = devm_platform_ioremap_resource(pdev, 0);
-	if (IS_ERR(io_base))
-		return PTR_ERR(io_base);
-
-	isc->regmap = devm_regmap_init_mmio(dev, io_base, &atmel_isc_regmap_config);
-	if (IS_ERR(isc->regmap)) {
-		ret = PTR_ERR(isc->regmap);
-		dev_err(dev, "failed to init register map: %d\n", ret);
-		return ret;
-	}
-
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-
-	ret = devm_request_irq(dev, irq, atmel_isc_interrupt, 0,
-			       "microchip-sama7g5-xisc", isc);
-	if (ret < 0) {
-		dev_err(dev, "can't register ISR for IRQ %u (ret=%i)\n",
-			irq, ret);
-		return ret;
-	}
-
-	isc->gamma_table = isc_sama7g5_gamma_table;
-	isc->gamma_max = 0;
-
-	isc->max_width = ISC_SAMA7G5_MAX_SUPPORT_WIDTH;
-	isc->max_height = ISC_SAMA7G5_MAX_SUPPORT_HEIGHT;
-
-	isc->config_dpc = isc_sama7g5_config_dpc;
-	isc->config_csc = isc_sama7g5_config_csc;
-	isc->config_cbc = isc_sama7g5_config_cbc;
-	isc->config_cc = isc_sama7g5_config_cc;
-	isc->config_gam = isc_sama7g5_config_gam;
-	isc->config_rlp = isc_sama7g5_config_rlp;
-	isc->config_ctrls = isc_sama7g5_config_ctrls;
-
-	isc->adapt_pipeline = isc_sama7g5_adapt_pipeline;
-
-	isc->offsets.csc = ISC_SAMA7G5_CSC_OFFSET;
-	isc->offsets.cbc = ISC_SAMA7G5_CBC_OFFSET;
-	isc->offsets.sub422 = ISC_SAMA7G5_SUB422_OFFSET;
-	isc->offsets.sub420 = ISC_SAMA7G5_SUB420_OFFSET;
-	isc->offsets.rlp = ISC_SAMA7G5_RLP_OFFSET;
-	isc->offsets.his = ISC_SAMA7G5_HIS_OFFSET;
-	isc->offsets.dma = ISC_SAMA7G5_DMA_OFFSET;
-	isc->offsets.version = ISC_SAMA7G5_VERSION_OFFSET;
-	isc->offsets.his_entry = ISC_SAMA7G5_HIS_ENTRY_OFFSET;
-
-	isc->controller_formats = sama7g5_controller_formats;
-	isc->controller_formats_size = ARRAY_SIZE(sama7g5_controller_formats);
-	isc->formats_list = sama7g5_formats_list;
-	isc->formats_list_size = ARRAY_SIZE(sama7g5_formats_list);
-
-	/* sama7g5-isc RAM access port is full AXI4 - 32 bits per beat */
-	isc->dcfg = ISC_DCFG_YMBSIZE_BEATS32 | ISC_DCFG_CMBSIZE_BEATS32;
-
-	/* sama7g5-isc : ISPCK does not exist, ISC is clocked by MCK */
-	isc->ispck_required = false;
-
-	ret = atmel_isc_pipeline_init(isc);
-	if (ret)
-		return ret;
-
-	isc->hclock = devm_clk_get(dev, "hclock");
-	if (IS_ERR(isc->hclock)) {
-		ret = PTR_ERR(isc->hclock);
-		dev_err(dev, "failed to get hclock: %d\n", ret);
-		return ret;
-	}
-
-	ret = clk_prepare_enable(isc->hclock);
-	if (ret) {
-		dev_err(dev, "failed to enable hclock: %d\n", ret);
-		return ret;
-	}
-
-	ret = atmel_isc_clk_init(isc);
-	if (ret) {
-		dev_err(dev, "failed to init isc clock: %d\n", ret);
-		goto unprepare_hclk;
-	}
-
-	ret = v4l2_device_register(dev, &isc->v4l2_dev);
-	if (ret) {
-		dev_err(dev, "unable to register v4l2 device.\n");
-		goto unprepare_hclk;
-	}
-
-	ret = xisc_parse_dt(dev, isc);
-	if (ret) {
-		dev_err(dev, "fail to parse device tree\n");
-		goto unregister_v4l2_device;
-	}
-
-	if (list_empty(&isc->subdev_entities)) {
-		dev_err(dev, "no subdev found\n");
-		ret = -ENODEV;
-		goto unregister_v4l2_device;
-	}
-
-	list_for_each_entry(subdev_entity, &isc->subdev_entities, list) {
-		struct v4l2_async_connection *asd;
-		struct fwnode_handle *fwnode =
-			of_fwnode_handle(subdev_entity->epn);
-
-		v4l2_async_nf_init(&subdev_entity->notifier, &isc->v4l2_dev);
-
-		asd = v4l2_async_nf_add_fwnode_remote(&subdev_entity->notifier,
-						      fwnode,
-						      struct v4l2_async_connection);
-
-		of_node_put(subdev_entity->epn);
-		subdev_entity->epn = NULL;
-
-		if (IS_ERR(asd)) {
-			ret = PTR_ERR(asd);
-			goto cleanup_subdev;
-		}
-
-		subdev_entity->notifier.ops = &atmel_isc_async_ops;
-
-		ret = v4l2_async_nf_register(&subdev_entity->notifier);
-		if (ret) {
-			dev_err(dev, "fail to register async notifier\n");
-			goto cleanup_subdev;
-		}
-
-		if (video_is_registered(&isc->video_dev))
-			break;
-	}
-
-	pm_runtime_set_active(dev);
-	pm_runtime_enable(dev);
-	pm_request_idle(dev);
-
-	regmap_read(isc->regmap, ISC_VERSION + isc->offsets.version, &ver);
-	dev_info(dev, "Microchip XISC version %x\n", ver);
-
-	return 0;
-
-cleanup_subdev:
-	atmel_isc_subdev_cleanup(isc);
-
-unregister_v4l2_device:
-	v4l2_device_unregister(&isc->v4l2_dev);
-
-unprepare_hclk:
-	clk_disable_unprepare(isc->hclock);
-
-	atmel_isc_clk_cleanup(isc);
-
-	return ret;
-}
-
-static void microchip_xisc_remove(struct platform_device *pdev)
-{
-	struct isc_device *isc = platform_get_drvdata(pdev);
-
-	pm_runtime_disable(&pdev->dev);
-
-	atmel_isc_subdev_cleanup(isc);
-
-	v4l2_device_unregister(&isc->v4l2_dev);
-
-	clk_disable_unprepare(isc->hclock);
-
-	atmel_isc_clk_cleanup(isc);
-}
-
-static int __maybe_unused xisc_runtime_suspend(struct device *dev)
-{
-	struct isc_device *isc = dev_get_drvdata(dev);
-
-	clk_disable_unprepare(isc->hclock);
-
-	return 0;
-}
-
-static int __maybe_unused xisc_runtime_resume(struct device *dev)
-{
-	struct isc_device *isc = dev_get_drvdata(dev);
-	int ret;
-
-	ret = clk_prepare_enable(isc->hclock);
-	if (ret)
-		return ret;
-
-	return ret;
-}
-
-static const struct dev_pm_ops microchip_xisc_dev_pm_ops = {
-	SET_RUNTIME_PM_OPS(xisc_runtime_suspend, xisc_runtime_resume, NULL)
-};
-
-#if IS_ENABLED(CONFIG_OF)
-static const struct of_device_id microchip_xisc_of_match[] = {
-	{ .compatible = "microchip,sama7g5-isc" },
-	{ }
-};
-MODULE_DEVICE_TABLE(of, microchip_xisc_of_match);
-#endif
-
-static struct platform_driver microchip_xisc_driver = {
-	.probe	= microchip_xisc_probe,
-	.remove = microchip_xisc_remove,
-	.driver	= {
-		.name		= "microchip-sama7g5-xisc",
-		.pm		= &microchip_xisc_dev_pm_ops,
-		.of_match_table = of_match_ptr(microchip_xisc_of_match),
-	},
-};
-
-module_platform_driver(microchip_xisc_driver);
-
-MODULE_AUTHOR("Eugen Hristev <eugen.hristev@microchip.com>");
-MODULE_DESCRIPTION("The V4L2 driver for Microchip-XISC");
-MODULE_LICENSE("GPL v2");
diff --git a/drivers/staging/media/ipu3/ipu3-css-params.c b/drivers/staging/media/ipu3/ipu3-css-params.c
index 2c48d57a3180f..92cce31e35c5d 100644
--- a/drivers/staging/media/ipu3/ipu3-css-params.c
+++ b/drivers/staging/media/ipu3/ipu3-css-params.c
@@ -1770,6 +1770,8 @@ static int imgu_css_cfg_acc_stripe(struct imgu_css *css, unsigned int pipe,
 		acc->stripe.bds_out_stripes[0].width =
 			ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width, f);
 	} else {
+		u32 offset;
+
 		/* Image processing is divided into two stripes */
 		acc->stripe.bds_out_stripes[0].width =
 			acc->stripe.bds_out_stripes[1].width =
@@ -1788,8 +1790,10 @@ static int imgu_css_cfg_acc_stripe(struct imgu_css *css, unsigned int pipe,
 			acc->stripe.bds_out_stripes[1].width += f;
 		}
 		/* Overlap between stripes is IPU3_UAPI_ISP_VEC_ELEMS * 4 */
-		acc->stripe.bds_out_stripes[1].offset =
-			acc->stripe.bds_out_stripes[0].width - 2 * f;
+		offset = acc->stripe.bds_out_stripes[0].width - 2 * f;
+		if (offset > 65535)
+			return -EINVAL;
+		acc->stripe.bds_out_stripes[1].offset = offset;
 	}
 
 	acc->stripe.effective_stripes[0].height =
diff --git a/drivers/staging/media/ipu7/abi/ipu7_fw_boot_abi.h b/drivers/staging/media/ipu7/abi/ipu7_fw_boot_abi.h
index a1519c4fe6618..56b90aab83ea4 100644
--- a/drivers/staging/media/ipu7/abi/ipu7_fw_boot_abi.h
+++ b/drivers/staging/media/ipu7/abi/ipu7_fw_boot_abi.h
@@ -42,7 +42,7 @@ struct ia_gofo_logger_config {
 	((IA_GOFO_BUTTRESS_FW_BOOT_PARAMS_IS_OFFSET) + \
 	(u32)(IA_GOFO_BUTTRESS_FW_BOOT_PARAMS_MAX_REG_IDX_PER_APP))
 #define IA_GOFO_BUTTRESS_FW_BOOT_PARAMS_PRIMARY_OFFSET (0U)
-#define IA_GOFO_CCG_IPU_BUTTRESS_FW_BOOT_PARAMS_SECONDARY_OFFSET (0x3000U / 4U)
+#define IA_GOFO_CCG_IPU_BUTTRESS_FW_BOOT_PARAMS_SECONDARY_OFFSET (0x3000 / 4U)
 #define IA_GOFO_HKR_IPU_BUTTRESS_FW_BOOT_PARAMS_SECONDARY_OFFSET \
 	(IA_GOFO_BUTTRESS_FW_BOOT_PARAMS_MAX_REG_IDX_PER_APP * 2U)
 #define IA_GOFO_HKR_HIF_BUTTRESS_FW_BOOT_PARAMS_SECONDARY_OFFSET \
@@ -75,7 +75,7 @@ enum ia_gofo_boot_uc_tile_frequency_units {
 };
 
 #define IA_GOFO_FW_BOOT_STATE_IS_CRITICAL(boot_state) \
-	(0xdead0000U == ((boot_state) & 0xffff0000U))
+	(0xdead0000 == ((boot_state) & 0xffff0000))
 
 struct ia_gofo_boot_config {
 	u32 length;
@@ -104,35 +104,35 @@ struct ia_gofo_secondary_boot_config {
 
 #pragma pack(pop)
 
-#define IA_GOFO_WDT_TIMEOUT_ERR			0xdead0401U
-#define IA_GOFO_MEM_FATAL_DME_ERR		0xdead0801U
-#define IA_GOFO_MEM_UNCORRECTABLE_LOCAL_ERR	0xdead0802U
-#define IA_GOFO_MEM_UNCORRECTABLE_DIRTY_ERR	0xdead0803U
-#define IA_GOFO_MEM_UNCORRECTABLE_DTAG_ERR	0xdead0804U
-#define IA_GOFO_MEM_UNCORRECTABLE_CACHE_ERR	0xdead0805U
-#define IA_GOFO_DOUBLE_EXCEPTION_ERR		0xdead0806U
-#define IA_GOFO_BIST_DMEM_FAULT_DETECTION_ERR	0xdead1000U
-#define IA_GOFO_BIST_DATA_INTEGRITY_FAILURE	0xdead1010U
+#define IA_GOFO_WDT_TIMEOUT_ERR			0xdead0401
+#define IA_GOFO_MEM_FATAL_DME_ERR		0xdead0801
+#define IA_GOFO_MEM_UNCORRECTABLE_LOCAL_ERR	0xdead0802
+#define IA_GOFO_MEM_UNCORRECTABLE_DIRTY_ERR	0xdead0803
+#define IA_GOFO_MEM_UNCORRECTABLE_DTAG_ERR	0xdead0804
+#define IA_GOFO_MEM_UNCORRECTABLE_CACHE_ERR	0xdead0805
+#define IA_GOFO_DOUBLE_EXCEPTION_ERR		0xdead0806
+#define IA_GOFO_BIST_DMEM_FAULT_DETECTION_ERR	0xdead1000
+#define IA_GOFO_BIST_DATA_INTEGRITY_FAILURE	0xdead1010
 
 enum ia_gofo_boot_state {
-	IA_GOFO_FW_BOOT_STATE_SECONDARY_BOOT_CONFIG_READY = 0x57a7b000U,
-	IA_GOFO_FW_BOOT_STATE_UNINIT = 0x57a7e000U,
-	IA_GOFO_FW_BOOT_STATE_STARTING_0 = 0x57a7d000U,
-	IA_GOFO_FW_BOOT_STATE_CACHE_INIT_DONE = 0x57a7d010U,
-	IA_GOFO_FW_BOOT_STATE_MEM_INIT_DONE = 0x57a7d020U,
-	IA_GOFO_FW_BOOT_STATE_STACK_INIT_DONE = 0x57a7d030U,
-	IA_GOFO_FW_BOOT_STATE_EARLY_BOOT_DONE = 0x57a7d100U,
-	IA_GOFO_FW_BOOT_STATE_BOOT_CONFIG_START = 0x57a7d200U,
-	IA_GOFO_FW_BOOT_STATE_QUEUE_INIT_DONE = 0x57a7d300U,
-	IA_GOFO_FW_BOOT_STATE_READY = 0x57a7e100U,
-	IA_GOFO_FW_BOOT_STATE_CRIT_UNSPECIFIED = 0xdead0001U,
-	IA_GOFO_FW_BOOT_STATE_CRIT_CFG_PTR = 0xdead0101U,
-	IA_GOFO_FW_BOOT_STATE_CRIT_CFG_VERSION = 0xdead0201U,
-	IA_GOFO_FW_BOOT_STATE_CRIT_MSG_VERSION = 0xdead0301U,
+	IA_GOFO_FW_BOOT_STATE_SECONDARY_BOOT_CONFIG_READY = 0x57a7b000,
+	IA_GOFO_FW_BOOT_STATE_UNINIT = 0x57a7e000,
+	IA_GOFO_FW_BOOT_STATE_STARTING_0 = 0x57a7d000,
+	IA_GOFO_FW_BOOT_STATE_CACHE_INIT_DONE = 0x57a7d010,
+	IA_GOFO_FW_BOOT_STATE_MEM_INIT_DONE = 0x57a7d020,
+	IA_GOFO_FW_BOOT_STATE_STACK_INIT_DONE = 0x57a7d030,
+	IA_GOFO_FW_BOOT_STATE_EARLY_BOOT_DONE = 0x57a7d100,
+	IA_GOFO_FW_BOOT_STATE_BOOT_CONFIG_START = 0x57a7d200,
+	IA_GOFO_FW_BOOT_STATE_QUEUE_INIT_DONE = 0x57a7d300,
+	IA_GOFO_FW_BOOT_STATE_READY = 0x57a7e100,
+	IA_GOFO_FW_BOOT_STATE_CRIT_UNSPECIFIED = 0xdead0001,
+	IA_GOFO_FW_BOOT_STATE_CRIT_CFG_PTR = 0xdead0101,
+	IA_GOFO_FW_BOOT_STATE_CRIT_CFG_VERSION = 0xdead0201,
+	IA_GOFO_FW_BOOT_STATE_CRIT_MSG_VERSION = 0xdead0301,
 	IA_GOFO_FW_BOOT_STATE_CRIT_WDT_TIMEOUT = IA_GOFO_WDT_TIMEOUT_ERR,
-	IA_GOFO_FW_BOOT_STATE_WRONG_DATA_SECTION_UNPACKING = 0xdead0501U,
-	IA_GOFO_FW_BOOT_STATE_WRONG_RO_DATA_SECTION_UNPACKING = 0xdead0601U,
-	IA_GOFO_FW_BOOT_STATE_INVALID_UNTRUSTED_ADDR_MIN = 0xdead0701U,
+	IA_GOFO_FW_BOOT_STATE_WRONG_DATA_SECTION_UNPACKING = 0xdead0501,
+	IA_GOFO_FW_BOOT_STATE_WRONG_RO_DATA_SECTION_UNPACKING = 0xdead0601,
+	IA_GOFO_FW_BOOT_STATE_INVALID_UNTRUSTED_ADDR_MIN = 0xdead0701,
 	IA_GOFO_FW_BOOT_STATE_CRIT_MEM_FATAL_DME = IA_GOFO_MEM_FATAL_DME_ERR,
 	IA_GOFO_FW_BOOT_STATE_CRIT_MEM_UNCORRECTABLE_LOCAL =
 	IA_GOFO_MEM_UNCORRECTABLE_LOCAL_ERR,
@@ -146,18 +146,18 @@ enum ia_gofo_boot_state {
 	IA_GOFO_DOUBLE_EXCEPTION_ERR,
 	IA_GOFO_FW_BOOT_STATE_CRIT_BIST_DMEM_FAULT_DETECTION_ERR =
 	IA_GOFO_BIST_DMEM_FAULT_DETECTION_ERR,
-	IA_GOFO_FW_BOOT_STATE_CRIT_DATA_INTEGRITY_FAILURE = 0xdead1010U,
-	IA_GOFO_FW_BOOT_STATE_CRIT_STACK_CHK_FAILURE = 0xdead1011U,
+	IA_GOFO_FW_BOOT_STATE_CRIT_DATA_INTEGRITY_FAILURE = 0xdead1010,
+	IA_GOFO_FW_BOOT_STATE_CRIT_STACK_CHK_FAILURE = 0xdead1011,
 	IA_GOFO_FW_BOOT_STATE_CRIT_SYSCOM_CONTEXT_INTEGRITY_FAILURE =
-	0xdead1012U,
-	IA_GOFO_FW_BOOT_STATE_CRIT_MPU_CONFIG_FAILURE = 0xdead1013U,
-	IA_GOFO_FW_BOOT_STATE_CRIT_SHARED_BUFFER_FAILURE = 0xdead1014U,
-	IA_GOFO_FW_BOOT_STATE_CRIT_CMEM_FAILURE = 0xdead1015U,
-	IA_GOFO_FW_BOOT_STATE_SHUTDOWN_CMD = 0x57a7f001U,
-	IA_GOFO_FW_BOOT_STATE_SHUTDOWN_START = 0x57a7e200U,
-	IA_GOFO_FW_BOOT_STATE_INACTIVE = 0x57a7e300U,
-	IA_GOFO_FW_BOOT_HW_CMD_ACK_TIMEOUT = 0x57a7e400U,
-	IA_GOFO_FW_BOOT_SYSTEM_CYCLES_ERROR = 0x57a7e500U
+	0xdead1012,
+	IA_GOFO_FW_BOOT_STATE_CRIT_MPU_CONFIG_FAILURE = 0xdead1013,
+	IA_GOFO_FW_BOOT_STATE_CRIT_SHARED_BUFFER_FAILURE = 0xdead1014,
+	IA_GOFO_FW_BOOT_STATE_CRIT_CMEM_FAILURE = 0xdead1015,
+	IA_GOFO_FW_BOOT_STATE_SHUTDOWN_CMD = 0x57a7f001,
+	IA_GOFO_FW_BOOT_STATE_SHUTDOWN_START = 0x57a7e200,
+	IA_GOFO_FW_BOOT_STATE_INACTIVE = 0x57a7e300,
+	IA_GOFO_FW_BOOT_HW_CMD_ACK_TIMEOUT = 0x57a7e400,
+	IA_GOFO_FW_BOOT_SYSTEM_CYCLES_ERROR = 0x57a7e500
 };
 
 #endif
diff --git a/drivers/staging/media/ipu7/abi/ipu7_fw_common_abi.h b/drivers/staging/media/ipu7/abi/ipu7_fw_common_abi.h
index 7bb6fac585a32..398a133504802 100644
--- a/drivers/staging/media/ipu7/abi/ipu7_fw_common_abi.h
+++ b/drivers/staging/media/ipu7/abi/ipu7_fw_common_abi.h
@@ -60,7 +60,7 @@ struct ia_gofo_tlv_list {
 
 #define IA_GOFO_MSG_ERR_MAX_DETAILS		(4U)
 #define IA_GOFO_MSG_ERR_OK			(0U)
-#define IA_GOFO_MSG_ERR_UNSPECIFED		(0xffffffffU)
+#define IA_GOFO_MSG_ERR_UNSPECIFED		(0xffffffff)
 #define IA_GOFO_MSG_ERR_GROUP_UNSPECIFIED	(0U)
 #define IA_GOFO_MSG_ERR_IS_OK(err)	(IA_GOFO_MSG_ERR_OK == (err).err_code)
 
@@ -145,7 +145,7 @@ struct ia_gofo_msg_indirect {
 #define IA_GOFO_MSG_LOG_DOC_FMT_ID_MIN	(0U)
 
 #define IA_GOFO_MSG_LOG_DOC_FMT_ID_MAX	(4095U)
-#define IA_GOFO_MSG_LOG_FMT_ID_INVALID	(0xfffffffU)
+#define IA_GOFO_MSG_LOG_FMT_ID_INVALID	(0xfffffff)
 
 struct ia_gofo_msg_log_info {
 	u16 log_counter;
diff --git a/drivers/staging/media/ipu7/abi/ipu7_fw_msg_abi.h b/drivers/staging/media/ipu7/abi/ipu7_fw_msg_abi.h
index 8a78dd0936df3..3112483859936 100644
--- a/drivers/staging/media/ipu7/abi/ipu7_fw_msg_abi.h
+++ b/drivers/staging/media/ipu7/abi/ipu7_fw_msg_abi.h
@@ -69,11 +69,11 @@ struct ipu7_msg_cb_profile {
 
 #define IPU_MSG_NODE_MAX_PROFILES	(2U)
 #define IPU_MSG_NODE_DEF_PROFILE_IDX	(0U)
-#define IPU_MSG_NODE_RSRC_ID_EXT_IP	(0xffU)
+#define IPU_MSG_NODE_RSRC_ID_EXT_IP	(0xff)
 
-#define IPU_MSG_NODE_DONT_CARE_TEB_HI	(0xffffffffU)
-#define IPU_MSG_NODE_DONT_CARE_TEB_LO	(0xffffffffU)
-#define IPU_MSG_NODE_RSRC_ID_IS		(0xfeU)
+#define IPU_MSG_NODE_DONT_CARE_TEB_HI	(0xffffffff)
+#define IPU_MSG_NODE_DONT_CARE_TEB_LO	(0xffffffff)
+#define IPU_MSG_NODE_RSRC_ID_IS		(0xfe)
 
 struct ipu7_msg_node {
 	struct ia_gofo_tlv_header tlv_header;
@@ -160,7 +160,7 @@ struct ipu7_msg_link_ep_pair {
 #define IPU_MSG_LINK_FOREIGN_KEY_MAX		(64U)
 #define IPU_MSG_LINK_PBK_ID_DONT_CARE		(255U)
 #define IPU_MSG_LINK_PBK_SLOT_ID_DONT_CARE	(255U)
-#define IPU_MSG_LINK_TERM_ID_DONT_CARE		(0xffU)
+#define IPU_MSG_LINK_TERM_ID_DONT_CARE		(0xff)
 
 struct ipu7_msg_link {
 	struct ia_gofo_tlv_header tlv_header;
@@ -333,7 +333,7 @@ enum ipu7_msg_err_device {
 #pragma pack(pop)
 
 #pragma pack(push, 1)
-#define IPU_MSG_GRAPH_ID_UNKNOWN	(0xffU)
+#define IPU_MSG_GRAPH_ID_UNKNOWN	(0xff)
 #define IPU_MSG_GRAPH_SEND_MSG_ENABLED	1U
 #define IPU_MSG_GRAPH_SEND_MSG_DISABLED	0U
 
diff --git a/drivers/staging/media/ipu7/ipu7-buttress-regs.h b/drivers/staging/media/ipu7/ipu7-buttress-regs.h
index 3eafd6a3813d9..7b646aa538cfc 100644
--- a/drivers/staging/media/ipu7/ipu7-buttress-regs.h
+++ b/drivers/staging/media/ipu7/ipu7-buttress-regs.h
@@ -287,7 +287,7 @@
 
 #define BUTTRESS_TSC_CMD_START_TSC_SYNC		BIT(0)
 #define BUTTRESS_PWR_STATUS_HH_STATUS_SHIFT	11
-#define BUTTRESS_PWR_STATUS_HH_STATUS_MASK	(0x3U << 11)
+#define BUTTRESS_PWR_STATUS_HH_STATUS_MASK	(0x3 << 11)
 #define BUTTRESS_TSW_WA_SOFT_RESET		BIT(8)
 /* new for PTL */
 #define BUTTRESS_SEL_PB_TIMESTAMP		BIT(9)
@@ -326,8 +326,8 @@
 #define BUTTRESS_CSE2IUDATA0_IPC_NACK_MASK 0xffff
 
 /* IS/PS freq control */
-#define BUTTRESS_IS_FREQ_CTL_RATIO_MASK	0xffU
-#define BUTTRESS_PS_FREQ_CTL_RATIO_MASK	0xffU
+#define BUTTRESS_IS_FREQ_CTL_RATIO_MASK	0xff
+#define BUTTRESS_PS_FREQ_CTL_RATIO_MASK	0xff
 
 #define IPU7_IS_FREQ_MAX		450
 #define IPU7_IS_FREQ_MIN		50
@@ -350,11 +350,11 @@
 /* buttree power status */
 #define IPU_BUTTRESS_PWR_STATE_IS_PWR_SHIFT	0
 #define IPU_BUTTRESS_PWR_STATE_IS_PWR_MASK	\
-	(0x3U << IPU_BUTTRESS_PWR_STATE_IS_PWR_SHIFT)
+	(0x3 << IPU_BUTTRESS_PWR_STATE_IS_PWR_SHIFT)
 
 #define IPU_BUTTRESS_PWR_STATE_PS_PWR_SHIFT	4
 #define IPU_BUTTRESS_PWR_STATE_PS_PWR_MASK	\
-	(0x3U << IPU_BUTTRESS_PWR_STATE_PS_PWR_SHIFT)
+	(0x3 << IPU_BUTTRESS_PWR_STATE_PS_PWR_SHIFT)
 
 #define IPU_BUTTRESS_PWR_STATE_DN_DONE		0x0
 #define IPU_BUTTRESS_PWR_STATE_UP_PROCESS	0x1
diff --git a/drivers/staging/media/ipu7/ipu7.c b/drivers/staging/media/ipu7/ipu7.c
index c771e763f8c50..310e3f24e571f 100644
--- a/drivers/staging/media/ipu7/ipu7.c
+++ b/drivers/staging/media/ipu7/ipu7.c
@@ -2169,21 +2169,18 @@ ipu7_isys_init(struct pci_dev *pdev, struct device *parent,
 	isys_adev->mmu = ipu7_mmu_init(dev, base, ISYS_MMID,
 				       &ipdata->hw_variant);
 	if (IS_ERR(isys_adev->mmu)) {
-		dev_err_probe(dev, PTR_ERR(isys_adev->mmu),
-			      "ipu7_mmu_init(isys_adev->mmu) failed\n");
+		ret = dev_err_probe(dev, PTR_ERR(isys_adev->mmu),
+				    "ipu7_mmu_init(isys_adev->mmu) failed\n");
 		put_device(&isys_adev->auxdev.dev);
-		kfree(pdata);
-		return ERR_CAST(isys_adev->mmu);
+		return ERR_PTR(ret);
 	}
 
 	isys_adev->mmu->dev = &isys_adev->auxdev.dev;
 	isys_adev->subsys = IPU_IS;
 
 	ret = ipu7_bus_add_device(isys_adev);
-	if (ret) {
-		kfree(pdata);
+	if (ret)
 		return ERR_PTR(ret);
-	}
 
 	return isys_adev;
 }
@@ -2216,21 +2213,18 @@ ipu7_psys_init(struct pci_dev *pdev, struct device *parent,
 	psys_adev->mmu = ipu7_mmu_init(&pdev->dev, base, PSYS_MMID,
 				       &ipdata->hw_variant);
 	if (IS_ERR(psys_adev->mmu)) {
-		dev_err_probe(&pdev->dev, PTR_ERR(psys_adev->mmu),
-			      "ipu7_mmu_init(psys_adev->mmu) failed\n");
+		ret = dev_err_probe(&pdev->dev, PTR_ERR(psys_adev->mmu),
+				    "ipu7_mmu_init(psys_adev->mmu) failed\n");
 		put_device(&psys_adev->auxdev.dev);
-		kfree(pdata);
-		return ERR_CAST(psys_adev->mmu);
+		return ERR_PTR(ret);
 	}
 
 	psys_adev->mmu->dev = &psys_adev->auxdev.dev;
 	psys_adev->subsys = IPU_PS;
 
 	ret = ipu7_bus_add_device(psys_adev);
-	if (ret) {
-		kfree(pdata);
+	if (ret)
 		return ERR_PTR(ret);
-	}
 
 	return psys_adev;
 }
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c
index 6600245dff0e2..ee0e286add67d 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus.c
@@ -391,6 +391,7 @@ static int cedrus_open(struct file *file)
 err_m2m_release:
 	v4l2_m2m_ctx_release(ctx->fh.m2m_ctx);
 err_free:
+	v4l2_fh_exit(&ctx->fh);
 	kfree(ctx);
 	mutex_unlock(&dev->dev_mutex);
 
@@ -476,7 +477,7 @@ static int cedrus_probe(struct platform_device *pdev)
 	ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev);
 	if (ret) {
 		dev_err(&pdev->dev, "Failed to register V4L2 device\n");
-		return ret;
+		goto err_hw;
 	}
 
 	vfd = &dev->vfd;
@@ -537,6 +538,8 @@ static int cedrus_probe(struct platform_device *pdev)
 	v4l2_m2m_release(dev->m2m_dev);
 err_v4l2:
 	v4l2_device_unregister(&dev->v4l2_dev);
+err_hw:
+	cedrus_hw_remove(dev);
 
 	return ret;
 }
diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
index 3e2843ef6ccec..fc54d993b11f2 100644
--- a/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
+++ b/drivers/staging/media/sunxi/cedrus/cedrus_h264.c
@@ -210,6 +210,9 @@ static void _cedrus_write_ref_list(struct cedrus_ctx *ctx,
 		u8 dpb_idx;
 
 		dpb_idx = ref_list[i].index;
+		if (dpb_idx >= V4L2_H264_NUM_DPB_ENTRIES)
+			continue;
+
 		dpb = &decode->dpb[dpb_idx];
 
 		if (!(dpb->flags & V4L2_H264_DPB_ENTRY_FLAG_ACTIVE))
diff --git a/drivers/staging/media/tegra-video/tegra210.c b/drivers/staging/media/tegra-video/tegra210.c
index da99f19a39e7b..a6606768fa036 100644
--- a/drivers/staging/media/tegra-video/tegra210.c
+++ b/drivers/staging/media/tegra-video/tegra210.c
@@ -362,8 +362,7 @@ dequeue_buf_done(struct tegra_vi_channel *chan)
 
 	buf = list_first_entry(&chan->done,
 			       struct tegra_channel_buffer, queue);
-	if (buf)
-		list_del_init(&buf->queue);
+	list_del_init(&buf->queue);
 	spin_unlock(&chan->done_lock);
 
 	return buf;
diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c
index f14cdc7b52113..456134a9e8cf2 100644
--- a/drivers/staging/media/tegra-video/vi.c
+++ b/drivers/staging/media/tegra-video/vi.c
@@ -80,8 +80,8 @@ static int tegra_get_format_idx_by_code(struct tegra_vi *vi,
 static u32 tegra_get_format_fourcc_by_idx(struct tegra_vi *vi,
 					  unsigned int index)
 {
-	if (index >= vi->soc->nformats)
-		return -EINVAL;
+	if (WARN_ON_ONCE(index >= vi->soc->nformats))
+		return vi->soc->video_formats[0].fourcc;
 
 	return vi->soc->video_formats[index].fourcc;
 }
diff --git a/drivers/staging/most/dim2/dim2.c b/drivers/staging/most/dim2/dim2.c
index 7953d4626752f..ec0e568ec12d1 100644
--- a/drivers/staging/most/dim2/dim2.c
+++ b/drivers/staging/most/dim2/dim2.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * dim2.c - MediaLB DIM2 Hardware Dependent Module
+ * MediaLB DIM2 Hardware Dependent Module
  *
  * Copyright (C) 2015-2016, Microchip Technology Germany II GmbH & Co. KG
  */
@@ -165,7 +165,7 @@ static int try_start_dim_transfer(struct hdm_channel *hdm_ch)
 	unsigned long flags;
 	struct dim_ch_state st;
 
-	if (!hdm_ch || !hdm_ch->is_initialized)
+	if (!hdm_ch->is_initialized)
 		return -EINVAL;
 
 	spin_lock_irqsave(&dim_lock, flags);
@@ -271,7 +271,7 @@ static void service_done_flag(struct dim2_hdm *dev, int ch_idx)
 	unsigned long flags;
 	u8 *data;
 
-	if (!hdm_ch || !hdm_ch->is_initialized)
+	if (!hdm_ch->is_initialized)
 		return;
 
 	spin_lock_irqsave(&dim_lock, flags);
diff --git a/drivers/staging/most/dim2/errors.h b/drivers/staging/most/dim2/errors.h
index 268332e5735e7..d707daaac3b65 100644
--- a/drivers/staging/most/dim2/errors.h
+++ b/drivers/staging/most/dim2/errors.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * errors.h - Definitions of errors for DIM2 HAL API
+ * Definitions of errors for DIM2 HAL API
  * (MediaLB, Device Interface Macro IP, OS62420)
  *
  * Copyright (C) 2015, Microchip Technology Germany II GmbH & Co. KG
diff --git a/drivers/staging/most/dim2/hal.c b/drivers/staging/most/dim2/hal.c
index 38376d3e3f0d4..4f137ef846a32 100644
--- a/drivers/staging/most/dim2/hal.c
+++ b/drivers/staging/most/dim2/hal.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * hal.c - DIM2 HAL implementation
+ * DIM2 HAL implementation
  * (MediaLB, Device Interface Macro IP, OS62420)
  *
  * Copyright (C) 2015-2016, Microchip Technology Germany II GmbH & Co. KG
diff --git a/drivers/staging/most/dim2/hal.h b/drivers/staging/most/dim2/hal.h
index ef10a8741c10b..5d69a5e7d2999 100644
--- a/drivers/staging/most/dim2/hal.h
+++ b/drivers/staging/most/dim2/hal.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * hal.h - DIM2 HAL interface
+ * DIM2 HAL interface
  * (MediaLB, Device Interface Macro IP, OS62420)
  *
  * Copyright (C) 2015, Microchip Technology Germany II GmbH & Co. KG
diff --git a/drivers/staging/most/dim2/reg.h b/drivers/staging/most/dim2/reg.h
index b0f36c208a57c..51ae158802b67 100644
--- a/drivers/staging/most/dim2/reg.h
+++ b/drivers/staging/most/dim2/reg.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * reg.h - Definitions for registers of DIM2
+ * Definitions for registers of DIM2
  * (MediaLB, Device Interface Macro IP, OS62420)
  *
  * Copyright (C) 2015, Microchip Technology Germany II GmbH & Co. KG
diff --git a/drivers/staging/most/dim2/sysfs.h b/drivers/staging/most/dim2/sysfs.h
index 09115cf4ed00e..60405cd473e7b 100644
--- a/drivers/staging/most/dim2/sysfs.h
+++ b/drivers/staging/most/dim2/sysfs.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- * sysfs.h - MediaLB sysfs information
+ * MediaLB sysfs information
  *
  * Copyright (C) 2015, Microchip Technology Germany II GmbH & Co. KG
  */
diff --git a/drivers/staging/most/net/net.c b/drivers/staging/most/net/net.c
index fffdb60cd230b..e6e4498d33ab1 100644
--- a/drivers/staging/most/net/net.c
+++ b/drivers/staging/most/net/net.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * net.c - Networking component for Mostcore
+ * Networking component for Mostcore
  *
  * Copyright (C) 2015, Microchip Technology Germany II GmbH & Co. KG
  */
@@ -80,11 +80,6 @@ static int skb_to_mamac(const struct sk_buff *skb, struct mbo *mbo)
 	unsigned int payload_len = skb->len - ETH_HLEN;
 	unsigned int mdp_len = payload_len + MDP_HDR_LEN;
 
-	if (mdp_len < skb->len) {
-		pr_err("drop: too large packet! (%u)\n", skb->len);
-		return -EINVAL;
-	}
-
 	if (mbo->buffer_length < mdp_len) {
 		pr_err("drop: too small buffer! (%d for %d)\n",
 		       mbo->buffer_length, mdp_len);
@@ -132,11 +127,6 @@ static int skb_to_mep(const struct sk_buff *skb, struct mbo *mbo)
 	u8 *buff = mbo->virt_address;
 	unsigned int mep_len = skb->len + MEP_HDR_LEN;
 
-	if (mep_len < skb->len) {
-		pr_err("drop: too large packet! (%u)\n", skb->len);
-		return -EINVAL;
-	}
-
 	if (mbo->buffer_length < mep_len) {
 		pr_err("drop: too small buffer! (%d for %d)\n",
 		       mbo->buffer_length, mep_len);
@@ -339,13 +329,13 @@ static int comp_probe_channel(struct most_interface *iface, int channel_idx,
 	} else {
 		ch = ccfg->direction == MOST_CH_TX ? &nd->tx : &nd->rx;
 		if (ch->linked) {
-			pr_err("direction is allocated\n");
+			netdev_err(nd->dev, "direction is allocated\n");
 			ret = -EINVAL;
 			goto unlock;
 		}
 
 		if (register_netdev(nd->dev)) {
-			pr_err("register_netdev() failed\n");
+			netdev_err(nd->dev, "register_netdev() failed\n");
 			ret = -EINVAL;
 			goto unlock;
 		}
@@ -464,7 +454,7 @@ static int comp_rx_data(struct mbo *mbo)
 
 	if (!skb) {
 		dev->stats.rx_dropped++;
-		pr_err_once("drop packet: no memory for skb\n");
+		netdev_err_once(dev, "drop packet: no memory for skb\n");
 		goto out;
 	}
 
diff --git a/drivers/staging/most/video/video.c b/drivers/staging/most/video/video.c
index 04351f8ccccf1..f846bb343b2e4 100644
--- a/drivers/staging/most/video/video.c
+++ b/drivers/staging/most/video/video.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * video.c - V4L2 component for Mostcore
+ * V4L2 component for Mostcore
  *
  * Copyright (C) 2015, Microchip Technology Germany II GmbH & Co. KG
  */
@@ -555,29 +555,8 @@ static int __init comp_init(void)
 
 static void __exit comp_exit(void)
 {
-	struct most_video_dev *mdev, *tmp;
-	LIST_HEAD(free_list);
-
-	/*
-	 * As the mostcore currently doesn't call disconnect_channel()
-	 * for linked channels while we call most_deregister_component()
-	 * we simulate this call here.
-	 * This must be fixed in core.
-	 */
-	spin_lock_irq(&list_lock);
-	list_replace_init(&video_devices, &free_list);
-	spin_unlock_irq(&list_lock);
-
-	list_for_each_entry_safe(mdev, tmp, &free_list, list) {
-		list_del_init(&mdev->list);
-		comp_unregister_videodev(mdev);
-		v4l2_device_disconnect(&mdev->v4l2_dev);
-		v4l2_device_put(&mdev->v4l2_dev);
-	}
-
 	most_deregister_configfs_subsys(&comp);
 	most_deregister_component(&comp);
-	BUG_ON(!list_empty(&video_devices));
 }
 
 module_init(comp_init);
diff --git a/drivers/staging/nvec/nvec.c b/drivers/staging/nvec/nvec.c
index 952c5a849a563..88c416ee0381d 100644
--- a/drivers/staging/nvec/nvec.c
+++ b/drivers/staging/nvec/nvec.c
@@ -494,6 +494,8 @@ static void nvec_tx_completed(struct nvec_chip *nvec)
 static void nvec_rx_completed(struct nvec_chip *nvec)
 {
 	if (nvec->rx->pos != nvec_msg_size(nvec->rx)) {
+		unsigned char msg_type = nvec->rx->data[0];
+
 		dev_err(nvec->dev, "RX incomplete: Expected %u bytes, got %u\n",
 			(uint)nvec_msg_size(nvec->rx),
 			(uint)nvec->rx->pos);
@@ -502,7 +504,7 @@ static void nvec_rx_completed(struct nvec_chip *nvec)
 		nvec->state = 0;
 
 		/* Battery quirk - Often incomplete, and likes to crash */
-		if (nvec->rx->data[0] == NVEC_BAT)
+		if (msg_type == NVEC_BAT)
 			complete(&nvec->ec_transfer);
 
 		return;
@@ -904,8 +906,8 @@ static void tegra_nvec_remove(struct platform_device *pdev)
 	nvec_unregister_notifier(nvec, &nvec->nvec_status_notifier);
 	cancel_work_sync(&nvec->rx_work);
 	cancel_work_sync(&nvec->tx_work);
-	/* FIXME: needs check whether nvec is responsible for power off */
-	pm_power_off = NULL;
+	if (pm_power_off == nvec_power_off)
+		pm_power_off = NULL;
 }
 
 #ifdef CONFIG_PM_SLEEP
diff --git a/drivers/staging/octeon/ethernet.c b/drivers/staging/octeon/ethernet.c
index eadb74fc14c8d..d85a9991faf6d 100644
--- a/drivers/staging/octeon/ethernet.c
+++ b/drivers/staging/octeon/ethernet.c
@@ -201,8 +201,8 @@ EXPORT_SYMBOL(cvm_oct_free_work);
  */
 static struct net_device_stats *cvm_oct_common_get_stats(struct net_device *dev)
 {
-	cvmx_pip_port_status_t rx_status;
-	cvmx_pko_port_status_t tx_status;
+	struct cvmx_pip_port_status rx_status;
+	struct cvmx_pko_port_status tx_status;
 	struct octeon_ethernet *priv = netdev_priv(dev);
 
 	if (priv->port < CVMX_PIP_NUM_INPUT_PORTS) {
@@ -798,7 +798,7 @@ static int cvm_oct_probe(struct platform_device *pdev)
 
 	num_interfaces = cvmx_helper_get_number_of_interfaces();
 	for (interface = 0; interface < num_interfaces; interface++) {
-		cvmx_helper_interface_mode_t imode =
+		enum cvmx_helper_interface_mode imode =
 		    cvmx_helper_interface_get_mode(interface);
 		int num_ports = cvmx_helper_ports_on_interface(interface);
 		int port;
diff --git a/drivers/staging/octeon/octeon-stubs.h b/drivers/staging/octeon/octeon-stubs.h
index 291eaffd2543a..9c1968b7e2d10 100644
--- a/drivers/staging/octeon/octeon-stubs.h
+++ b/drivers/staging/octeon/octeon-stubs.h
@@ -215,14 +215,14 @@ enum cvmx_fau_op_size {
 	CVMX_FAU_OP_SIZE_64 = 3
 };
 
-typedef enum {
+enum cvmx_spi_mode {
 	CVMX_SPI_MODE_UNKNOWN = 0,
 	CVMX_SPI_MODE_TX_HALFPLEX = 1,
 	CVMX_SPI_MODE_RX_HALFPLEX = 2,
 	CVMX_SPI_MODE_DUPLEX = 3
-} cvmx_spi_mode_t;
+};
 
-typedef enum {
+enum cvmx_helper_interface_mode {
 	CVMX_HELPER_INTERFACE_MODE_DISABLED,
 	CVMX_HELPER_INTERFACE_MODE_RGMII,
 	CVMX_HELPER_INTERFACE_MODE_GMII,
@@ -233,20 +233,20 @@ typedef enum {
 	CVMX_HELPER_INTERFACE_MODE_PICMG,
 	CVMX_HELPER_INTERFACE_MODE_NPI,
 	CVMX_HELPER_INTERFACE_MODE_LOOP,
-} cvmx_helper_interface_mode_t;
+};
 
-typedef enum {
+enum cvmx_pow_wait {
 	CVMX_POW_WAIT = 1,
 	CVMX_POW_NO_WAIT = 0,
-} cvmx_pow_wait_t;
+};
 
-typedef enum {
+enum cvmx_pko_lock {
 	CVMX_PKO_LOCK_NONE = 0,
 	CVMX_PKO_LOCK_ATOMIC_TAG = 1,
 	CVMX_PKO_LOCK_CMD_QUEUE = 2,
-} cvmx_pko_lock_t;
+};
 
-typedef enum {
+enum cvmx_pko_status {
 	CVMX_PKO_SUCCESS,
 	CVMX_PKO_INVALID_PORT,
 	CVMX_PKO_INVALID_QUEUE,
@@ -254,7 +254,7 @@ typedef enum {
 	CVMX_PKO_NO_MEMORY,
 	CVMX_PKO_PORT_ALREADY_SETUP,
 	CVMX_PKO_CMD_QUEUE_INIT_ERROR
-} cvmx_pko_status_t;
+};
 
 enum cvmx_pow_tag_type {
 	CVMX_POW_TAG_TYPE_ORDERED   = 0L,
@@ -386,7 +386,7 @@ union cvmx_ipd_sub_port_qos_cnt {
 	} s;
 };
 
-typedef struct {
+struct cvmx_pip_port_status {
 	u32 dropped_octets;
 	u32 dropped_packets;
 	u32 pci_raw_packets;
@@ -409,13 +409,13 @@ typedef struct {
 	u32 inb_packets;
 	u64 inb_octets;
 	u16 inb_errors;
-} cvmx_pip_port_status_t;
+};
 
-typedef struct {
+struct cvmx_pko_port_status {
 	u32 packets;
 	u64 octets;
 	u64 doorbell;
-} cvmx_pko_port_status_t;
+};
 
 union cvmx_pip_frm_len_chkx {
 	u64 u64;
@@ -1260,14 +1260,14 @@ static inline int octeon_is_simulation(void)
 }
 
 static inline void cvmx_pip_get_port_status(u64 port_num, u64 clear,
-					    cvmx_pip_port_status_t *status)
+					    struct cvmx_pip_port_status *status)
 { }
 
 static inline void cvmx_pko_get_port_status(u64 port_num, u64 clear,
-					    cvmx_pko_port_status_t *status)
+					    struct cvmx_pko_port_status *status)
 { }
 
-static inline cvmx_helper_interface_mode_t cvmx_helper_interface_get_mode(int
+static inline enum cvmx_helper_interface_mode cvmx_helper_interface_get_mode(int
 								   interface)
 {
 	return 0;
@@ -1344,11 +1344,11 @@ static inline unsigned int cvmx_get_core_num(void)
 }
 
 static inline void cvmx_pow_work_request_async_nocheck(int scr_addr,
-						       cvmx_pow_wait_t wait)
+						       enum cvmx_pow_wait wait)
 { }
 
 static inline void cvmx_pow_work_request_async(int scr_addr,
-					       cvmx_pow_wait_t wait)
+					       enum cvmx_pow_wait wait)
 { }
 
 static inline struct cvmx_wqe *cvmx_pow_work_response_async(int scr_addr)
@@ -1358,13 +1358,13 @@ static inline struct cvmx_wqe *cvmx_pow_work_response_async(int scr_addr)
 	return wqe;
 }
 
-static inline struct cvmx_wqe *cvmx_pow_work_request_sync(cvmx_pow_wait_t wait)
+static inline struct cvmx_wqe *cvmx_pow_work_request_sync(enum cvmx_pow_wait wait)
 {
 	return (void *)(unsigned long)wait;
 }
 
 static inline int cvmx_spi_restart_interface(int interface,
-					     cvmx_spi_mode_t mode, int timeout)
+					     enum cvmx_spi_mode mode, int timeout)
 {
 	return 0;
 }
@@ -1383,12 +1383,12 @@ static inline union cvmx_gmxx_rxx_rx_inbnd cvmx_spi4000_check_speed(int interfac
 }
 
 static inline void cvmx_pko_send_packet_prepare(u64 port, u64 queue,
-						cvmx_pko_lock_t use_locking)
+						enum cvmx_pko_lock use_locking)
 { }
 
-static inline cvmx_pko_status_t cvmx_pko_send_packet_finish(u64 port,
+static inline enum cvmx_pko_status cvmx_pko_send_packet_finish(u64 port,
 		u64 queue, union cvmx_pko_command_word0 pko_command,
-		union cvmx_buf_ptr packet, cvmx_pko_lock_t use_locking)
+		union cvmx_buf_ptr packet, enum cvmx_pko_lock use_locking)
 {
 	return 0;
 }
diff --git a/drivers/staging/rtl8723bs/core/rtw_ap.c b/drivers/staging/rtl8723bs/core/rtw_ap.c
index 4b4012411011f..065850a9e8947 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ap.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ap.c
@@ -20,8 +20,6 @@ void init_mlme_ap_info(struct adapter *padapter)
 	INIT_LIST_HEAD(&pacl_list->acl_node_q.queue);
 	spin_lock_init(&pacl_list->acl_node_q.lock);
 
-	/* pmlmeext->bstart_bss = false; */
-
 	start_ap_mode(padapter);
 }
 
@@ -32,8 +30,6 @@ void free_mlme_ap_info(struct adapter *padapter)
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
-	/* stop_ap_mode(padapter); */
-
 	pmlmepriv->update_bcn = false;
 	pmlmeext->bstart_bss = false;
 
@@ -60,7 +56,7 @@ static void update_BCNTIM(struct adapter *padapter)
 	/* update TIM IE */
 	u8 *p, *dst_ie, *premainder_ie = NULL, *pbackup_remainder_ie = NULL;
 	__le16 tim_bitmap_le;
-	uint offset, tmp_len, tim_ielen, tim_ie_offset, remainder_ielen;
+	unsigned int offset, tmp_len, tim_ielen, tim_ie_offset, remainder_ielen;
 
 	tim_bitmap_le = cpu_to_le16(pstapriv->tim_bitmap);
 
@@ -156,7 +152,7 @@ static void update_BCNTIM(struct adapter *padapter)
 		kfree(pbackup_remainder_ie);
 	}
 
-	offset =  (uint)(dst_ie - pie);
+	offset =  (unsigned int)(dst_ie - pie);
 	pnetwork_mlmeext->ie_length = offset + remainder_ielen;
 }
 
@@ -301,7 +297,7 @@ void expire_timeout_chk(struct adapter *padapter)
 
 			psta->keep_alive_trycnt = 0;
 			spin_lock_bh(&pstapriv->asoc_list_lock);
-			if (list_empty(&psta->asoc_list) == false) {
+			if (!list_empty(&psta->asoc_list)) {
 				list_del_init(&psta->asoc_list);
 				pstapriv->asoc_list_cnt--;
 				updated = ap_free_sta(padapter, psta, false,
@@ -377,7 +373,6 @@ void update_bmc_sta(struct adapter *padapter)
 
 	if (psta) {
 		psta->aid = 0;/* default set to 0 */
-		/* psta->mac_id = psta->aid+4; */
 		psta->mac_id = psta->aid + 1;/* mac_id = 1 for bc/mc stainfo */
 
 		pmlmeinfo->FW_sta_info[psta->mac_id].psta = psta;
@@ -448,12 +443,6 @@ void update_sta_info_apmode(struct adapter *padapter, struct sta_info *psta)
 	struct ht_priv *phtpriv_ap = &pmlmepriv->htpriv;
 	struct ht_priv *phtpriv_sta = &psta->htpriv;
 	u8 cur_ldpc_cap = 0, cur_stbc_cap = 0, cur_beamform_cap = 0;
-	/* set intf_tag to if1 */
-	/* psta->intf_tag = 0; */
-
-	/* psta->mac_id = psta->aid+4; */
-	/* psta->mac_id = psta->aid+1;//alloc macid when call rtw_alloc_stainfo(), */
-	/* release macid when call rtw_free_stainfo() */
 
 	/* ap mode */
 	rtw_hal_set_odm_var(padapter, HAL_ODM_STA_INFO, psta, true);
@@ -544,9 +533,6 @@ void update_sta_info_apmode(struct adapter *padapter, struct sta_info *psta)
 
 	memset(&psta->sta_stats, 0, sizeof(struct stainfo_stats));
 
-	/* add ratid */
-	/* add_ratid(padapter, psta); move to ap_sta_info_defer_update() */
-
 	spin_lock_bh(&psta->lock);
 	psta->state |= _FW_LINKED;
 	spin_unlock_bh(&psta->lock);
@@ -567,9 +553,6 @@ static void update_ap_info(struct adapter *padapter, struct sta_info *psta)
 
 	/* HT related cap */
 	if (phtpriv_ap->ht_option) {
-		/* check if sta supports rx ampdu */
-		/* phtpriv_ap->ampdu_enable = phtpriv_ap->ampdu_enable; */
-
 		/* check if sta support s Short GI 20M */
 		if ((phtpriv_ap->ht_cap.cap_info) & cpu_to_le16(IEEE80211_HT_CAP_SGI_20))
 			phtpriv_ap->sgi_20m = true;
@@ -621,11 +604,6 @@ static void update_hw_ht_param(struct adapter *padapter)
 	pmlmeinfo->SM_PS =
 		(le16_to_cpu(pmlmeinfo->HT_caps.u.HT_cap_element.HT_caps_info) &
 		 0x0C) >> 2;
-
-	/*  */
-	/*  Config current HT Protection mode. */
-	/*  */
-	/* pmlmeinfo->HT_protection = pmlmeinfo->HT_info.infos[1] & 0x3; */
 }
 
 void start_bss_network(struct adapter *padapter)
@@ -662,16 +640,11 @@ void start_bss_network(struct adapter *padapter)
 		pmlmeext->bstart_bss = true;
 
 	/* todo: update wmm, ht cap */
-	/* pmlmeinfo->WMM_enable; */
-	/* pmlmeinfo->HT_enable; */
 	if (pmlmepriv->qospriv.qos_option)
 		pmlmeinfo->WMM_enable = true;
 	if (pmlmepriv->htpriv.ht_option) {
 		pmlmeinfo->WMM_enable = true;
 		pmlmeinfo->HT_enable = true;
-		/* pmlmeinfo->HT_info_enable = true; */
-		/* pmlmeinfo->HT_caps_enable = true; */
-
 		update_hw_ht_param(padapter);
 	}
 
@@ -694,7 +667,6 @@ void start_bss_network(struct adapter *padapter)
 	rtw_hal_set_hwreg(padapter, HW_VAR_AC_PARAM_VO, (u8 *)(&acparm));
 	acparm = 0x005E4317; /*  VI */
 	rtw_hal_set_hwreg(padapter, HW_VAR_AC_PARAM_VI, (u8 *)(&acparm));
-	/* acparm = 0x00105320; // BE */
 	acparm = 0x005ea42b;
 	rtw_hal_set_hwreg(padapter, HW_VAR_AC_PARAM_BE, (u8 *)(&acparm));
 	acparm = 0x0000A444; /*  BK */
@@ -711,18 +683,8 @@ void start_bss_network(struct adapter *padapter)
 	rtw_hal_set_hwreg(padapter, HW_VAR_DO_IQK, NULL);
 
 	if (!pmlmepriv->cur_network.join_res) { /* setting only at  first time */
-		/* u32 initialgain; */
-
-		/* initialgain = 0x1e; */
-
-		/* disable dynamic functions, such as high power, DIG */
-		/* Save_DM_Func_Flag(padapter); */
-		/* Switch_DM_Func(padapter, DYNAMIC_FUNC_DISABLE, false); */
-
 		/* turn on all dynamic functions */
 		Switch_DM_Func(padapter, DYNAMIC_ALL_FUNC_ENABLE, true);
-
-		/* rtw_hal_set_hwreg(padapter, HW_VAR_INITIAL_GAIN, (u8 *)(&initialgain)); */
 	}
 
 	/* set channel, bwmode */
@@ -739,21 +701,17 @@ void start_bss_network(struct adapter *padapter)
 
 		if ((cbw40_enable) &&	 (pht_info->infos[0] & BIT(2))) {
 			/* switch to the 40M Hz mode */
-			/* pmlmeext->cur_bwmode = CHANNEL_WIDTH_40; */
 			cur_bwmode = CHANNEL_WIDTH_40;
 			switch (pht_info->infos[0] & 0x3) {
 			case 1:
-				/* pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_LOWER; */
 				cur_ch_offset = HAL_PRIME_CHNL_OFFSET_LOWER;
 				break;
 
 			case 3:
-				/* pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_UPPER; */
 				cur_ch_offset = HAL_PRIME_CHNL_OFFSET_UPPER;
 				break;
 
 			default:
-				/* pmlmeext->cur_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE; */
 				cur_ch_offset = HAL_PRIME_CHNL_OFFSET_DONT_CARE;
 				break;
 			}
@@ -789,8 +747,6 @@ void start_bss_network(struct adapter *padapter)
 
 	/* update bc/mc sta_info */
 	update_bmc_sta(padapter);
-
-	/* pmlmeext->bstart_bss = true; */
 }
 
 int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
@@ -798,10 +754,10 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	int ret = _SUCCESS;
 	u8 *p;
 	u8 *ht_caps_ie = NULL;
-	u8 *pHT_info_ie = NULL;
+	u8 *ht_info_ie = NULL;
 	struct sta_info *psta = NULL;
 	u16 cap, ht_cap = false;
-	uint ie_len = 0;
+	unsigned int ie_len = 0;
 	int group_cipher, pairwise_cipher;
 	u8 channel, network_type, support_rate[NDIS_802_11_LENGTH_RATES_EX];
 	int support_rate_num = 0;
@@ -836,12 +792,9 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 	/* beacon interval */
 	/* ie + 8;	8: TimeStamp, 2: Beacon Interval 2:Capability */
 	p = rtw_get_beacon_interval_from_ie(ie);
-	/* pbss_network->configuration.beacon_period = le16_to_cpu(*(unsigned short*)p); */
 	pbss_network->configuration.beacon_period = get_unaligned_le16(p);
 
 	/* capability */
-	/* cap = *(unsigned short *)rtw_get_capability_from_ie(ie); */
-	/* cap = le16_to_cpu(cap); */
 	cap = get_unaligned_le16(ie);
 
 	/* SSID */
@@ -1054,7 +1007,7 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 		       &ie_len,
 		       (pbss_network->ie_length - _BEACON_IE_OFFSET_));
 	if (p && ie_len > 0)
-		pHT_info_ie = p;
+		ht_info_ie = p;
 
 	switch (network_type) {
 	case WIRELESS_11B:
@@ -1075,12 +1028,6 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 
 	pmlmepriv->htpriv.ht_option = false;
 
-	if ((psecuritypriv->wpa2_pairwise_cipher & WPA_CIPHER_TKIP) ||
-	    (psecuritypriv->wpa_pairwise_cipher & WPA_CIPHER_TKIP)) {
-		/* todo: */
-		/* ht_cap = false; */
-	}
-
 	/* ht_cap */
 	if (pregistrypriv->ht_enable && ht_cap) {
 		pmlmepriv->htpriv.ht_option = true;
@@ -1091,14 +1038,13 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 
 		HT_caps_handler(padapter, (struct ndis_80211_var_ie *)ht_caps_ie);
 
-		HT_info_handler(padapter, (struct ndis_80211_var_ie *)pHT_info_ie);
+		HT_info_handler(padapter, (struct ndis_80211_var_ie *)ht_info_ie);
 	}
 
 	pbss_network->length =
 		get_wlan_bssid_ex_sz((struct wlan_bssid_ex *)pbss_network);
 
 	/* issue beacon to start bss network */
-	/* start_bss_network(padapter, (u8 *)pbss_network); */
 	rtw_startbss_cmd(padapter, RTW_CMDF_WAIT_ACK);
 
 	/* alloc sta_info for ap itself */
@@ -1117,9 +1063,6 @@ int rtw_check_beacon_data(struct adapter *padapter, u8 *pbuf,  int len)
 
 	pmlmepriv->cur_network.join_res = true;/* for check if already set beacon */
 
-	/* update bc/mc sta_info */
-	/* update_bmc_sta(padapter); */
-
 	return ret;
 }
 
@@ -1408,7 +1351,7 @@ static void update_bcn_wps_ie(struct adapter *padapter)
 	u8 *premainder_ie;
 	u8 *pbackup_remainder_ie = NULL;
 
-	uint wps_ielen = 0, wps_offset, remainder_ielen;
+	unsigned int wps_ielen = 0, wps_offset, remainder_ielen;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
@@ -1428,7 +1371,7 @@ static void update_bcn_wps_ie(struct adapter *padapter)
 	if (!pwps_ie_src)
 		return;
 
-	wps_offset = (uint)(pwps_ie - ie);
+	wps_offset = (unsigned int)(pwps_ie - ie);
 
 	premainder_ie = pwps_ie + wps_ielen;
 
@@ -1437,7 +1380,7 @@ static void update_bcn_wps_ie(struct adapter *padapter)
 	if (remainder_ielen)
 		pbackup_remainder_ie = kmemdup(premainder_ie, remainder_ielen, GFP_ATOMIC);
 
-	wps_ielen = (uint)pwps_ie_src[1];/* to get ie data len */
+	wps_ielen = (unsigned int)pwps_ie_src[1];/* to get ie data len */
 	if ((wps_offset + wps_ielen + 2 + remainder_ielen) <= MAX_IE_SZ) {
 		memcpy(pwps_ie, pwps_ie_src, wps_ielen + 2);
 		pwps_ie += (wps_ielen + 2);
@@ -1475,14 +1418,12 @@ void update_beacon(struct adapter *padapter, u8 ie_id, u8 *oui, u8 tx)
 {
 	struct mlme_priv *pmlmepriv;
 	struct mlme_ext_priv *pmlmeext;
-	/* struct mlme_ext_info *pmlmeinfo; */
 
 	if (!padapter)
 		return;
 
 	pmlmepriv = &padapter->mlmepriv;
 	pmlmeext = &padapter->mlmeextpriv;
-	/* pmlmeinfo = &(pmlmeext->mlmext_info); */
 
 	if (!pmlmeext->bstart_bss)
 		return;
@@ -1540,10 +1481,8 @@ void update_beacon(struct adapter *padapter, u8 ie_id, u8 *oui, u8 tx)
 
 	spin_unlock_bh(&pmlmepriv->bcn_update_lock);
 
-	if (tx) {
-		/* send_beacon(padapter);//send_beacon must execute on TSR level */
+	if (tx)
 		set_tx_beacon_cmd(padapter);
-	}
 }
 
 /*
@@ -1847,8 +1786,6 @@ u8 ap_free_sta(struct adapter *padapter,
 	psta->htpriv.agg_enable_bitmap = 0x0;/* reset */
 	psta->htpriv.candidate_tid_bitmap = 0x0;/* reset */
 
-	/* report_del_sta_event(padapter, psta->hwaddr, reason); */
-
 	/* clear cam entry / key */
 	rtw_clearstakey_cmd(padapter, psta, true);
 
@@ -1888,9 +1825,7 @@ void rtw_sta_flush(struct adapter *padapter)
 		list_del_init(&psta->asoc_list);
 		pstapriv->asoc_list_cnt--;
 
-		/* spin_unlock_bh(&pstapriv->asoc_list_lock); */
 		ap_free_sta(padapter, psta, true, WLAN_REASON_DEAUTH_LEAVING);
-		/* spin_lock_bh(&pstapriv->asoc_list_lock); */
 	}
 	spin_unlock_bh(&pstapriv->asoc_list_lock);
 
@@ -2017,7 +1952,6 @@ void start_ap_mode(struct adapter *padapter)
 
 	pmlmepriv->update_bcn = false;
 
-	/* init_mlme_ap_info(padapter); */
 	pmlmeext->bstart_bss = false;
 
 	pmlmepriv->num_sta_non_erp = 0;
@@ -2105,5 +2039,5 @@ void stop_ap_mode(struct adapter *padapter)
 
 	rtw_free_mlme_priv_ie_data(pmlmepriv);
 
-	rtw_btcoex_MediaStatusNotify(padapter, 0); /* disconnect */
+	rtw_btcoex_media_status_notify(padapter, 0); /* disconnect */
 }
diff --git a/drivers/staging/rtl8723bs/core/rtw_btcoex.c b/drivers/staging/rtl8723bs/core/rtw_btcoex.c
index 0191a943f0a45..9e00424d00bdd 100644
--- a/drivers/staging/rtl8723bs/core/rtw_btcoex.c
+++ b/drivers/staging/rtl8723bs/core/rtw_btcoex.c
@@ -8,17 +8,17 @@
 #include <rtw_btcoex.h>
 #include <hal_btcoex.h>
 
-void rtw_btcoex_MediaStatusNotify(struct adapter *padapter, u8 media_status)
+void rtw_btcoex_media_status_notify(struct adapter *padapter, u8 media_status)
 {
-	if ((media_status == RT_MEDIA_CONNECT)
-		&& (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) == true)) {
+	if ((media_status == RT_MEDIA_CONNECT) &&
+	    check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE)) {
 		rtw_hal_set_hwreg(padapter, HW_VAR_DL_RSVD_PAGE, NULL);
 	}
 
 	hal_btcoex_MediaStatusNotify(padapter, media_status);
 }
 
-void rtw_btcoex_HaltNotify(struct adapter *padapter)
+void rtw_btcoex_halt_notify(struct adapter *padapter)
 {
 	if (!padapter->bup)
 		return;
@@ -32,7 +32,7 @@ void rtw_btcoex_HaltNotify(struct adapter *padapter)
 /*  ================================================== */
 /*  Below Functions are called by BT-Coex */
 /*  ================================================== */
-void rtw_btcoex_RejectApAggregatedPacket(struct adapter *padapter, u8 enable)
+void rtw_btcoex_reject_ap_aggregated_packet(struct adapter *padapter, u8 enable)
 {
 	struct mlme_ext_info *pmlmeinfo;
 	struct sta_info *psta;
@@ -49,7 +49,7 @@ void rtw_btcoex_RejectApAggregatedPacket(struct adapter *padapter, u8 enable)
 	}
 }
 
-void rtw_btcoex_LPS_Enter(struct adapter *padapter)
+void rtw_btcoex_lps_enter(struct adapter *padapter)
 {
 	struct pwrctrl_priv *pwrpriv;
 	u8 lps_val;
@@ -61,11 +61,10 @@ void rtw_btcoex_LPS_Enter(struct adapter *padapter)
 	rtw_set_ps_mode(padapter, PS_MODE_MIN, 0, lps_val, "BTCOEX");
 }
 
-void rtw_btcoex_LPS_Leave(struct adapter *padapter)
+void rtw_btcoex_lps_leave(struct adapter *padapter)
 {
 	struct pwrctrl_priv *pwrpriv;
 
-
 	pwrpriv = adapter_to_pwrctl(padapter);
 
 	if (pwrpriv->pwr_mode != PS_MODE_ACTIVE) {
diff --git a/drivers/staging/rtl8723bs/core/rtw_cmd.c b/drivers/staging/rtl8723bs/core/rtw_cmd.c
index c1185c25ed369..adc151326feb8 100644
--- a/drivers/staging/rtl8723bs/core/rtw_cmd.c
+++ b/drivers/staging/rtl8723bs/core/rtw_cmd.c
@@ -672,7 +672,7 @@ int rtw_startbss_cmd(struct adapter  *padapter, int flags)
 u8 rtw_joinbss_cmd(struct adapter  *padapter, struct wlan_network *pnetwork)
 {
 	u8 res = _SUCCESS;
-	uint	t_len = 0;
+	unsigned int t_len = 0;
 	struct wlan_bssid_ex *psecnetwork;
 	struct cmd_obj *pcmd;
 	struct cmd_priv *pcmdpriv = &padapter->cmdpriv;
@@ -696,7 +696,7 @@ u8 rtw_joinbss_cmd(struct adapter  *padapter, struct wlan_network *pnetwork)
 	t_len = sizeof(struct wlan_bssid_ex);
 
 	/* for hidden ap to set fw_state here */
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE | WIFI_ADHOC_STATE) != true) {
+	if (!check_fwstate(pmlmepriv, WIFI_STATION_STATE | WIFI_ADHOC_STATE)) {
 		switch (ndis_network_mode) {
 		case Ndis802_11IBSS:
 			set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
@@ -1291,11 +1291,11 @@ void lps_ctrl_wk_hdl(struct adapter *padapter, u8 lps_ctrl_type)
 		/*  Reset LPS Setting */
 		pwrpriv->LpsIdleCount = 0;
 		rtw_hal_set_hwreg(padapter, HW_VAR_H2C_FW_JOINBSSRPT, (u8 *)(&mstatus));
-		rtw_btcoex_MediaStatusNotify(padapter, mstatus);
+		rtw_btcoex_media_status_notify(padapter, mstatus);
 		break;
 	case LPS_CTRL_DISCONNECT:
 		mstatus = 0;/* disconnect */
-		rtw_btcoex_MediaStatusNotify(padapter, mstatus);
+		rtw_btcoex_media_status_notify(padapter, mstatus);
 		LPS_Leave(padapter, "LPS_CTRL_DISCONNECT");
 		rtw_hal_set_hwreg(padapter, HW_VAR_H2C_FW_JOINBSSRPT, (u8 *)(&mstatus));
 		break;
@@ -1603,7 +1603,7 @@ static void rtw_btinfo_hdl(struct adapter *adapter, u8 *buf, u16 buf_len)
 	cmd_idx = info->cid;
 
 	if (info->len > buf_len - 2) {
-		rtw_warn_on(1);
+		WARN_ON(1);
 		len = buf_len - 2;
 	} else {
 		len = info->len;
@@ -1650,7 +1650,7 @@ u8 rtw_c2h_packet_wk_cmd(struct adapter *padapter, u8 *pbuf, u16 length)
 	return res;
 }
 
-/* dont call R/W in this function, beucase SDIO interrupt have claim host */
+/* do not call R/W in this function, because SDIO interrupt have claim host */
 /* or deadlock will happen and cause special-systemserver-died in android */
 u8 rtw_c2h_wk_cmd(struct adapter *padapter, u8 *c2h_evt)
 {
diff --git a/drivers/staging/rtl8723bs/core/rtw_efuse.c b/drivers/staging/rtl8723bs/core/rtw_efuse.c
index 099320e4eb500..803a608b74fa5 100644
--- a/drivers/staging/rtl8723bs/core/rtw_efuse.c
+++ b/drivers/staging/rtl8723bs/core/rtw_efuse.c
@@ -10,7 +10,7 @@
 
 /*  11/16/2008 MH Add description. Get current efuse area enabled word!!. */
 u8
-Efuse_CalculateWordCnts(u8 word_en)
+rtw_efuse_calculate_word_counts(u8 word_en)
 {
 	u8 word_cnts = 0;
 
@@ -26,7 +26,7 @@ Efuse_CalculateWordCnts(u8 word_en)
 }
 
 /*-----------------------------------------------------------------------------
- * Function:	EFUSE_Read1Byte
+ * Function:	rtw_efuse_read_1_byte
  *
  * Overview:	Copy from WMAC fot EFUSE read 1 byte.
  *
@@ -42,7 +42,7 @@ Efuse_CalculateWordCnts(u8 word_en)
  *
  */
 u8
-EFUSE_Read1Byte(
+rtw_efuse_read_1_byte(
 struct adapter *Adapter,
 u16		Address)
 {
@@ -79,11 +79,11 @@ u16		Address)
 	} else
 		return 0xFF;
 
-} /* EFUSE_Read1Byte */
+} /* rtw_efuse_read_1_byte */
 
 /*  11/16/2008 MH Read one byte from real Efuse. */
 u8
-efuse_OneByteRead(
+rtw_efuse_one_byte_read(
 struct adapter *padapter,
 u16	addr,
 u8	*data)
@@ -95,7 +95,7 @@ u8	*data)
 	/*  <20130121, Kordan> For SMIC EFUSE specificatoin. */
 	/* 0x34[11]: SW force PGMEN input of efuse to high. (for the bank selected by 0x34[9:8]) */
 	/* PHY_SetMacReg(padapter, 0x34, BIT11, 0); */
-	rtw_write16(padapter, 0x34, rtw_read16(padapter, 0x34) & (~BIT11));
+	rtw_write16(padapter, 0x34, rtw_read16(padapter, 0x34) & (~BIT(11)));
 
 	/*  -----------------e-fuse reg ctrl --------------------------------- */
 	/* address */
@@ -201,7 +201,7 @@ static void efuse_ShadowRead4Byte(struct adapter *padapter, u16 Offset, u32 *Val
 }	/*  efuse_ShadowRead4Byte */
 
 /*-----------------------------------------------------------------------------
- * Function:	EFUSE_ShadowMapUpdate
+ * Function:	rtw_efuse_shadow_map_update
  *
  * Overview:	Transfer current EFUSE content to shadow init and modify map.
  *
@@ -216,7 +216,7 @@ static void efuse_ShadowRead4Byte(struct adapter *padapter, u16 Offset, u32 *Val
  * 11/13/2008	MHC		Create Version 0.
  *
  */
-void EFUSE_ShadowMapUpdate(struct adapter *padapter, u8 efuseType)
+void rtw_efuse_shadow_map_update(struct adapter *padapter, u8 efuseType)
 {
 	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
 	u16 mapLen = 0;
@@ -230,11 +230,10 @@ void EFUSE_ShadowMapUpdate(struct adapter *padapter, u8 efuseType)
 
 	/* PlatformMoveMemory((void *)&pHalData->EfuseMap[EFUSE_MODIFY_MAP][0], */
 	/* void *)&pHalData->EfuseMap[EFUSE_INIT_MAP][0], mapLen); */
-} /*  EFUSE_ShadowMapUpdate */
-
+} /*  rtw_efuse_shadow_map_update */
 
 /*-----------------------------------------------------------------------------
- * Function:	EFUSE_ShadowRead
+ * Function:	rtw_efuse_shadow_read
  *
  * Overview:	Read from efuse init map !!!!!
  *
@@ -249,7 +248,7 @@ void EFUSE_ShadowMapUpdate(struct adapter *padapter, u8 efuseType)
  * 11/12/2008	MHC		Create Version 0.
  *
  */
-void EFUSE_ShadowRead(struct adapter *padapter, u8 Type, u16 Offset, u32 *Value)
+void rtw_efuse_shadow_read(struct adapter *padapter, u8 Type, u16 Offset, u32 *Value)
 {
 	if (Type == 1)
 		efuse_ShadowRead1Byte(padapter, Offset, (u8 *)Value);
@@ -258,4 +257,4 @@ void EFUSE_ShadowRead(struct adapter *padapter, u8 Type, u16 Offset, u32 *Value)
 	else if (Type == 4)
 		efuse_ShadowRead4Byte(padapter, Offset, (u32 *)Value);
 
-}	/* EFUSE_ShadowRead*/
+} /* rtw_efuse_shadow_read*/
diff --git a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
index 72b7f731dd471..d0bbe1bb979c8 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ieee80211.c
@@ -347,7 +347,7 @@ int rtw_generate_ie(struct registry_priv *pregistrypriv)
 
 	/* HT Cap. */
 	if ((pregistrypriv->wireless_mode & WIRELESS_11_24N) &&
-	    (pregistrypriv->ht_enable == true)) {
+	    pregistrypriv->ht_enable) {
 		/* todo: */
 	}
 
@@ -593,7 +593,6 @@ int rtw_get_wapi_ie(u8 *in_ie, uint in_len, u8 *wapi_ie, u16 *wapi_len)
 
 			if (wapi_len)
 				*wapi_len = in_ie[cnt + 1] + 2;
-
 		}
 
 		cnt += in_ie[cnt + 1] + 2;   /* get next */
diff --git a/drivers/staging/rtl8723bs/core/rtw_io.c b/drivers/staging/rtl8723bs/core/rtw_io.c
index 53ba99e1de00a..c793ae7ea5c46 100644
--- a/drivers/staging/rtl8723bs/core/rtw_io.c
+++ b/drivers/staging/rtl8723bs/core/rtw_io.c
@@ -27,9 +27,8 @@
 
 u8 rtw_read8(struct adapter *adapter, u32 addr)
 {
-	/* struct	io_queue	*pio_queue = (struct io_queue *)adapter->pio_queue; */
 	struct io_priv *pio_priv = &adapter->iopriv;
-	struct	intf_hdl		*pintfhdl = &pio_priv->intf;
+	struct intf_hdl *pintfhdl = &pio_priv->intf;
 	u8 (*_read8)(struct intf_hdl *pintfhdl, u32 addr);
 
 	_read8 = pintfhdl->io_ops._read8;
@@ -39,9 +38,8 @@ u8 rtw_read8(struct adapter *adapter, u32 addr)
 
 u16 rtw_read16(struct adapter *adapter, u32 addr)
 {
-	/* struct	io_queue	*pio_queue = (struct io_queue *)adapter->pio_queue; */
 	struct io_priv *pio_priv = &adapter->iopriv;
-	struct	intf_hdl		*pintfhdl = &pio_priv->intf;
+	struct intf_hdl *pintfhdl = &pio_priv->intf;
 	u16 (*_read16)(struct intf_hdl *pintfhdl, u32 addr);
 
 	_read16 = pintfhdl->io_ops._read16;
@@ -51,9 +49,8 @@ u16 rtw_read16(struct adapter *adapter, u32 addr)
 
 u32 rtw_read32(struct adapter *adapter, u32 addr)
 {
-	/* struct	io_queue	*pio_queue = (struct io_queue *)adapter->pio_queue; */
 	struct io_priv *pio_priv = &adapter->iopriv;
-	struct	intf_hdl		*pintfhdl = &pio_priv->intf;
+	struct intf_hdl *pintfhdl = &pio_priv->intf;
 	u32 (*_read32)(struct intf_hdl *pintfhdl, u32 addr);
 
 	_read32 = pintfhdl->io_ops._read32;
@@ -63,9 +60,8 @@ u32 rtw_read32(struct adapter *adapter, u32 addr)
 
 int rtw_write8(struct adapter *adapter, u32 addr, u8 val)
 {
-	/* struct	io_queue	*pio_queue = (struct io_queue *)adapter->pio_queue; */
 	struct io_priv *pio_priv = &adapter->iopriv;
-	struct	intf_hdl		*pintfhdl = &pio_priv->intf;
+	struct intf_hdl *pintfhdl = &pio_priv->intf;
 	int (*_write8)(struct intf_hdl *pintfhdl, u32 addr, u8 val);
 	int ret;
 
@@ -78,9 +74,8 @@ int rtw_write8(struct adapter *adapter, u32 addr, u8 val)
 
 int rtw_write16(struct adapter *adapter, u32 addr, u16 val)
 {
-	/* struct	io_queue	*pio_queue = (struct io_queue *)adapter->pio_queue; */
 	struct io_priv *pio_priv = &adapter->iopriv;
-	struct	intf_hdl		*pintfhdl = &pio_priv->intf;
+	struct intf_hdl *pintfhdl = &pio_priv->intf;
 	int (*_write16)(struct intf_hdl *pintfhdl, u32 addr, u16 val);
 	int ret;
 
@@ -92,9 +87,8 @@ int rtw_write16(struct adapter *adapter, u32 addr, u16 val)
 
 int rtw_write32(struct adapter *adapter, u32 addr, u32 val)
 {
-	/* struct	io_queue	*pio_queue = (struct io_queue *)adapter->pio_queue; */
 	struct io_priv *pio_priv = &adapter->iopriv;
-	struct	intf_hdl		*pintfhdl = &pio_priv->intf;
+	struct intf_hdl *pintfhdl = &pio_priv->intf;
 	int (*_write32)(struct intf_hdl *pintfhdl, u32 addr, u32 val);
 	int ret;
 
@@ -109,7 +103,7 @@ u32 rtw_write_port(struct adapter *adapter, u32 addr, u32 cnt, u8 *pmem)
 {
 	u32 (*_write_port)(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *pmem);
 	struct io_priv *pio_priv = &adapter->iopriv;
-	struct	intf_hdl		*pintfhdl = &pio_priv->intf;
+	struct intf_hdl *pintfhdl = &pio_priv->intf;
 
 	_write_port = pintfhdl->io_ops._write_port;
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
index c70541f95a735..12bf7780bea59 100644
--- a/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
+++ b/drivers/staging/rtl8723bs/core/rtw_ioctl_set.c
@@ -80,7 +80,7 @@ u8 rtw_do_join(struct adapter *padapter)
 			pmlmepriv->to_join = false;
 			_set_timer(&pmlmepriv->assoc_timer, MAX_JOIN_TIMEOUT);
 		} else {
-			if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true) {
+			if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE)) {
 				/*  submit createbss_cmd to change to a ADHOC_MASTER */
 
 				/* pmlmepriv->lock has been acquired by caller... */
@@ -120,9 +120,7 @@ u8 rtw_do_join(struct adapter *padapter)
 					pmlmepriv->to_join = false;
 				}
 			}
-
 		}
-
 	}
 
 exit:
@@ -139,32 +137,32 @@ u8 rtw_set_802_11_ssid(struct adapter *padapter, struct ndis_802_11_ssid *ssid)
 	netdev_dbg(padapter->pnetdev, "set ssid [%s] fw_state = 0x%08x\n",
 		   ssid->ssid, get_fwstate(pmlmepriv));
 
-	if (padapter->hw_init_completed == false) {
+	if (!padapter->hw_init_completed) {
 		status = _FAIL;
 		goto exit;
 	}
 
 	spin_lock_bh(&pmlmepriv->lock);
 
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY) == true)
+	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
 		goto handle_tkip_countermeasure;
-	else if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING) == true)
+	else if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING))
 		goto release_mlme_lock;
 
-	if (check_fwstate(pmlmepriv, _FW_LINKED|WIFI_ADHOC_MASTER_STATE) == true) {
+	if (check_fwstate(pmlmepriv, _FW_LINKED | WIFI_ADHOC_MASTER_STATE)) {
 		if ((pmlmepriv->assoc_ssid.ssid_length == ssid->ssid_length) &&
 		    (!memcmp(&pmlmepriv->assoc_ssid.ssid, ssid->ssid, ssid->ssid_length))) {
-			if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) == false) {
-				if (rtw_is_same_ibss(padapter, pnetwork) == false) {
+			if (!check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
+				if (!rtw_is_same_ibss(padapter, pnetwork)) {
 					/* if in WIFI_ADHOC_MASTER_STATE | WIFI_ADHOC_STATE, create bss or rejoin again */
 					rtw_disassoc_cmd(padapter, 0, true);
 
-					if (check_fwstate(pmlmepriv, _FW_LINKED) == true)
+					if (check_fwstate(pmlmepriv, _FW_LINKED))
 						rtw_indicate_disconnect(padapter);
 
 					rtw_free_assoc_resources(padapter, 1);
 
-					if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true) {
+					if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
 						_clr_fwstate_(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
 						set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
 					}
@@ -177,12 +175,12 @@ u8 rtw_set_802_11_ssid(struct adapter *padapter, struct ndis_802_11_ssid *ssid)
 		} else {
 			rtw_disassoc_cmd(padapter, 0, true);
 
-			if (check_fwstate(pmlmepriv, _FW_LINKED) == true)
+			if (check_fwstate(pmlmepriv, _FW_LINKED))
 				rtw_indicate_disconnect(padapter);
 
 			rtw_free_assoc_resources(padapter, 1);
 
-			if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true) {
+			if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
 				_clr_fwstate_(pmlmepriv, WIFI_ADHOC_MASTER_STATE);
 				set_fwstate(pmlmepriv, WIFI_ADHOC_STATE);
 			}
@@ -195,7 +193,7 @@ u8 rtw_set_802_11_ssid(struct adapter *padapter, struct ndis_802_11_ssid *ssid)
 		goto release_mlme_lock;
 	}
 
-	if (rtw_validate_ssid(ssid) == false) {
+	if (!rtw_validate_ssid(ssid)) {
 		status = _FAIL;
 		goto release_mlme_lock;
 	}
@@ -203,7 +201,7 @@ u8 rtw_set_802_11_ssid(struct adapter *padapter, struct ndis_802_11_ssid *ssid)
 	memcpy(&pmlmepriv->assoc_ssid, ssid, sizeof(struct ndis_802_11_ssid));
 	pmlmepriv->assoc_by_bssid = false;
 
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY) == true)
+	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
 		pmlmepriv->to_join = true;
 	else
 		status = rtw_do_join(padapter);
@@ -223,10 +221,10 @@ u8 rtw_set_802_11_connect(struct adapter *padapter, u8 *bssid, struct ndis_802_1
 	bool ssid_valid = true;
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
-	if (!ssid || rtw_validate_ssid(ssid) == false)
+	if (!ssid || !rtw_validate_ssid(ssid))
 		ssid_valid = false;
 
-	if (!bssid || rtw_validate_bssid(bssid) == false)
+	if (!bssid || !rtw_validate_bssid(bssid))
 		bssid_valid = false;
 
 	if (!ssid_valid && !bssid_valid) {
@@ -234,7 +232,7 @@ u8 rtw_set_802_11_connect(struct adapter *padapter, u8 *bssid, struct ndis_802_1
 		goto exit;
 	}
 
-	if (padapter->hw_init_completed == false) {
+	if (!padapter->hw_init_completed) {
 		status = _FAIL;
 		goto exit;
 	}
@@ -244,9 +242,9 @@ u8 rtw_set_802_11_connect(struct adapter *padapter, u8 *bssid, struct ndis_802_1
 	netdev_dbg(padapter->pnetdev, FUNC_ADPT_FMT "  fw_state = 0x%08x\n",
 		   FUNC_ADPT_ARG(padapter), get_fwstate(pmlmepriv));
 
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY) == true)
+	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
 		goto handle_tkip_countermeasure;
-	else if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING) == true)
+	else if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING))
 		goto release_mlme_lock;
 
 handle_tkip_countermeasure:
@@ -267,7 +265,7 @@ u8 rtw_set_802_11_connect(struct adapter *padapter, u8 *bssid, struct ndis_802_1
 		pmlmepriv->assoc_by_bssid = false;
 	}
 
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY) == true)
+	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
 		pmlmepriv->to_join = true;
 	else
 		status = rtw_do_join(padapter);
@@ -296,15 +294,15 @@ u8 rtw_set_802_11_infrastructure_mode(struct adapter *padapter,
 
 		spin_lock_bh(&pmlmepriv->lock);
 
-		if ((check_fwstate(pmlmepriv, _FW_LINKED) == true) || (*pold_state == Ndis802_11IBSS))
+		if (check_fwstate(pmlmepriv, _FW_LINKED) || (*pold_state == Ndis802_11IBSS))
 			rtw_disassoc_cmd(padapter, 0, true);
 
-		if ((check_fwstate(pmlmepriv, _FW_LINKED) == true) ||
-			(check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true))
+		if (check_fwstate(pmlmepriv, _FW_LINKED) ||
+		    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE))
 			rtw_free_assoc_resources(padapter, 1);
 
 		if ((*pold_state == Ndis802_11Infrastructure) || (*pold_state == Ndis802_11IBSS)) {
-			if (check_fwstate(pmlmepriv, _FW_LINKED) == true)
+			if (check_fwstate(pmlmepriv, _FW_LINKED))
 				rtw_indicate_disconnect(padapter); /* will clr Linked_state; before this function, we must have checked whether issue dis-assoc_cmd or not */
 		}
 
@@ -340,14 +338,13 @@ u8 rtw_set_802_11_infrastructure_mode(struct adapter *padapter,
 	return true;
 }
 
-
 u8 rtw_set_802_11_disassociate(struct adapter *padapter)
 {
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 
 	spin_lock_bh(&pmlmepriv->lock);
 
-	if (check_fwstate(pmlmepriv, _FW_LINKED) == true) {
+	if (check_fwstate(pmlmepriv, _FW_LINKED)) {
 		rtw_disassoc_cmd(padapter, 0, true);
 		rtw_indicate_disconnect(padapter);
 		/* modify for CONFIG_IEEE80211W, none 11w can use it */
@@ -369,7 +366,7 @@ u8 rtw_set_802_11_bssid_list_scan(struct adapter *padapter, struct ndis_802_11_s
 		res = false;
 		goto exit;
 	}
-	if (padapter->hw_init_completed == false) {
+	if (!padapter->hw_init_completed) {
 		res = false;
 		goto exit;
 	}
@@ -417,7 +414,6 @@ u8 rtw_set_802_11_authentication_mode(struct adapter *padapter, enum ndis_802_11
 
 u8 rtw_set_802_11_add_wep(struct adapter *padapter, struct ndis_802_11_wep *wep)
 {
-
 	signed int		keyid, res;
 	struct security_priv *psecuritypriv = &(padapter->securitypriv);
 	u8 ret = _SUCCESS;
@@ -471,8 +467,8 @@ u16 rtw_get_cur_max_rate(struct adapter *adapter)
 	struct sta_info *psta = NULL;
 	u8 short_GI = 0;
 
-	if ((check_fwstate(pmlmepriv, _FW_LINKED) != true)
-		&& (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) != true))
+	if (!check_fwstate(pmlmepriv, _FW_LINKED) &&
+	    !check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE))
 		return 0;
 
 	psta = rtw_get_stainfo(&adapter->stapriv, get_bssid(pmlmepriv));
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme.c b/drivers/staging/rtl8723bs/core/rtw_mlme.c
index ddfc56f0253df..ff77681d5087f 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme.c
@@ -351,9 +351,6 @@ int is_same_network(struct wlan_bssid_ex *src, struct wlan_bssid_ex *dst, u8 fea
 	u16 s_cap, d_cap;
 	__le16 tmps, tmpd;
 
-	if (rtw_bug_check(dst, src, &s_cap, &d_cap) == false)
-		return false;
-
 	memcpy((u8 *)&tmps, rtw_get_capability_from_ie(src->ies), 2);
 	memcpy((u8 *)&tmpd, rtw_get_capability_from_ie(dst->ies), 2);
 
@@ -457,15 +454,13 @@ static void update_current_network(struct adapter *adapter, struct wlan_bssid_ex
 {
 	struct	mlme_priv *pmlmepriv = &adapter->mlmepriv;
 
-	rtw_bug_check(&pmlmepriv->cur_network.network,
-		&pmlmepriv->cur_network.network,
-		&pmlmepriv->cur_network.network,
-		&pmlmepriv->cur_network.network);
-
 	if (check_fwstate(pmlmepriv, _FW_LINKED) && (is_same_network(&pmlmepriv->cur_network.network, pnetwork, 0))) {
 		update_network(&pmlmepriv->cur_network.network, pnetwork, adapter, true);
+		if (pmlmepriv->cur_network.network.ie_length < sizeof(struct ndis_802_11_fix_ie))
+			return;
+
 		rtw_update_protection(adapter, (pmlmepriv->cur_network.network.ies) + sizeof(struct ndis_802_11_fix_ie),
-								pmlmepriv->cur_network.network.ie_length);
+								pmlmepriv->cur_network.network.ie_length - sizeof(struct ndis_802_11_fix_ie));
 	}
 }
 
@@ -486,8 +481,6 @@ void rtw_update_scanned_network(struct adapter *adapter, struct wlan_bssid_ex *t
 	list_for_each(plist, phead) {
 		pnetwork = list_entry(plist, struct wlan_network, list);
 
-		rtw_bug_check(pnetwork, pnetwork, pnetwork, pnetwork);
-
 		if (is_same_network(&pnetwork->network, target, feature)) {
 			target_find = 1;
 			break;
@@ -1072,8 +1065,11 @@ static void rtw_joinbss_update_network(struct adapter *padapter, struct wlan_net
 			break;
 	}
 
+	if (cur_network->network.ie_length < sizeof(struct ndis_802_11_fix_ie))
+		return;
+
 	rtw_update_protection(padapter, (cur_network->network.ies) + sizeof(struct ndis_802_11_fix_ie),
-									(cur_network->network.ie_length));
+									(cur_network->network.ie_length - sizeof(struct ndis_802_11_fix_ie)));
 
 	rtw_update_ht_cap(padapter, cur_network->network.ies, cur_network->network.ie_length, (u8) cur_network->network.configuration.ds_config);
 }
@@ -1282,7 +1278,7 @@ void rtw_stassoc_event_callback(struct adapter *adapter, u8 *pbuf)
 	struct wlan_network	*cur_network = &pmlmepriv->cur_network;
 	struct wlan_network	*ptarget_wlan = NULL;
 
-	if (rtw_access_ctrl(adapter, pstassoc->macaddr) == false)
+	if (!rtw_access_ctrl(adapter, pstassoc->macaddr))
 		return;
 
 	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
@@ -1388,7 +1384,7 @@ void rtw_stadel_event_callback(struct adapter *adapter, u8 *pbuf)
 		u16 media_status;
 
 		media_status = (mac_id << 8) | 0; /*   MACID|OPMODE:0 means disconnect */
-		/* for STA, AP, ADHOC mode, report disconnect stauts to FW */
+		/* for STA, AP, ADHOC mode, report disconnect status to FW */
 		rtw_hal_set_hwreg(adapter, HW_VAR_H2C_MEDIA_STATUS_RPT, (u8 *)&media_status);
 	}
 
@@ -1566,7 +1562,7 @@ void rtw_mlme_reset_auto_scan_int(struct adapter *adapter)
 
 	if (pmlmeinfo->VHT_enable) /* disable auto scan when connect to 11AC AP */
 		mlme->auto_scan_int_ms = 0;
-	else if (adapter->registrypriv.wifi_spec && is_client_associated_to_ap(adapter) == true)
+	else if (adapter->registrypriv.wifi_spec && is_client_associated_to_ap(adapter))
 		mlme->auto_scan_int_ms = 60 * 1000;
 	else if (rtw_chk_roam_flags(adapter, RTW_ROAM_ACTIVE)) {
 		if (check_fwstate(mlme, WIFI_STATION_STATE) && check_fwstate(mlme, _FW_LINKED))
@@ -1668,10 +1664,10 @@ static int rtw_check_roaming_candidate(struct mlme_priv *mlme
 	int updated = false;
 	struct adapter *adapter = container_of(mlme, struct adapter, mlmepriv);
 
-	if (is_same_ess(&competitor->network, &mlme->cur_network.network) == false)
+	if (!is_same_ess(&competitor->network, &mlme->cur_network.network))
 		goto exit;
 
-	if (rtw_is_desired_network(adapter, competitor) == false)
+	if (!rtw_is_desired_network(adapter, competitor))
 		goto exit;
 
 	/* got specific addr to roam */
@@ -1707,7 +1703,7 @@ int rtw_select_roaming_candidate(struct mlme_priv *mlme)
 	struct	wlan_network	*candidate = NULL;
 
 	if (!mlme->cur_network_scanned) {
-		rtw_warn_on(1);
+		WARN_ON(1);
 		return ret;
 	}
 
@@ -1763,13 +1759,13 @@ static int rtw_check_join_candidate(struct mlme_priv *mlme
 			goto exit;
 	}
 
-	if (rtw_is_desired_network(adapter, competitor)  == false)
+	if (!rtw_is_desired_network(adapter, competitor))
 		goto exit;
 
 	if (rtw_to_roam(adapter) > 0) {
-		if (jiffies_to_msecs(jiffies - competitor->last_scanned) >= mlme->roam_scanr_exp_ms
-			|| is_same_ess(&competitor->network, &mlme->cur_network.network) == false
-		)
+		if (jiffies_to_msecs(jiffies - competitor->last_scanned) >=
+		    mlme->roam_scanr_exp_ms ||
+		    !is_same_ess(&competitor->network, &mlme->cur_network.network))
 			goto exit;
 	}
 
@@ -2207,19 +2203,19 @@ void rtw_ht_use_default_setting(struct adapter *padapter)
 	else
 		phtpriv->bss_coexist = 0;
 
-	phtpriv->sgi_40m = TEST_FLAG(pregistrypriv->short_gi, BIT1) ? true : false;
-	phtpriv->sgi_20m = TEST_FLAG(pregistrypriv->short_gi, BIT0) ? true : false;
+	phtpriv->sgi_40m = TEST_FLAG(pregistrypriv->short_gi, BIT(1)) ? true : false;
+	phtpriv->sgi_20m = TEST_FLAG(pregistrypriv->short_gi, BIT(0)) ? true : false;
 
 	/*  LDPC support */
 	rtw_hal_get_def_var(padapter, HAL_DEF_RX_LDPC, (u8 *)&bHwLDPCSupport);
 	CLEAR_FLAGS(phtpriv->ldpc_cap);
 	if (bHwLDPCSupport) {
-		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT4))
+		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT(4)))
 			SET_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_RX);
 	}
 	rtw_hal_get_def_var(padapter, HAL_DEF_TX_LDPC, (u8 *)&bHwLDPCSupport);
 	if (bHwLDPCSupport) {
-		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT5))
+		if (TEST_FLAG(pregistrypriv->ldpc_cap, BIT(5)))
 			SET_FLAG(phtpriv->ldpc_cap, LDPC_HT_ENABLE_TX);
 	}
 
@@ -2227,12 +2223,12 @@ void rtw_ht_use_default_setting(struct adapter *padapter)
 	rtw_hal_get_def_var(padapter, HAL_DEF_TX_STBC, (u8 *)&bHwSTBCSupport);
 	CLEAR_FLAGS(phtpriv->stbc_cap);
 	if (bHwSTBCSupport) {
-		if (TEST_FLAG(pregistrypriv->stbc_cap, BIT5))
+		if (TEST_FLAG(pregistrypriv->stbc_cap, BIT(5)))
 			SET_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_TX);
 	}
 	rtw_hal_get_def_var(padapter, HAL_DEF_RX_STBC, (u8 *)&bHwSTBCSupport);
 	if (bHwSTBCSupport) {
-		if (TEST_FLAG(pregistrypriv->stbc_cap, BIT4))
+		if (TEST_FLAG(pregistrypriv->stbc_cap, BIT(4)))
 			SET_FLAG(phtpriv->stbc_cap, STBC_HT_ENABLE_RX);
 	}
 
@@ -2240,10 +2236,10 @@ void rtw_ht_use_default_setting(struct adapter *padapter)
 	rtw_hal_get_def_var(padapter, HAL_DEF_EXPLICIT_BEAMFORMER, (u8 *)&bHwSupportBeamformer);
 	rtw_hal_get_def_var(padapter, HAL_DEF_EXPLICIT_BEAMFORMEE, (u8 *)&bHwSupportBeamformee);
 	CLEAR_FLAGS(phtpriv->beamform_cap);
-	if (TEST_FLAG(pregistrypriv->beamform_cap, BIT4) && bHwSupportBeamformer)
+	if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(4)) && bHwSupportBeamformer)
 		SET_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMER_ENABLE);
 
-	if (TEST_FLAG(pregistrypriv->beamform_cap, BIT5) && bHwSupportBeamformee)
+	if (TEST_FLAG(pregistrypriv->beamform_cap, BIT(5)) && bHwSupportBeamformee)
 		SET_FLAG(phtpriv->beamform_cap, BEAMFORMING_HT_BEAMFORMEE_ENABLE);
 }
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
index 5f00fe282d1be..ba19a162896ed 100644
--- a/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
+++ b/drivers/staging/rtl8723bs/core/rtw_mlme_ext.c
@@ -398,10 +398,6 @@ void init_mlme_ext_priv(struct adapter *padapter)
 	pmlmeext->chan_scan_time = SURVEY_TO;
 	pmlmeext->mlmeext_init = true;
 	pmlmeext->active_keep_alive_check = true;
-
-#ifdef DBG_FIXED_CHAN
-	pmlmeext->fixed_chan = 0xFF;
-#endif
 }
 
 void free_mlme_ext_priv(struct mlme_ext_priv *pmlmeext)
@@ -512,15 +508,14 @@ unsigned int OnProbeReq(struct adapter *padapter, union recv_frame *precv_frame)
 	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE))
 		return _SUCCESS;
 
-	if (check_fwstate(pmlmepriv, _FW_LINKED) == false &&
-		check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE|WIFI_AP_STATE) == false) {
+	if (!check_fwstate(pmlmepriv, _FW_LINKED) &&
+	    !check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE | WIFI_AP_STATE)) {
 		return _SUCCESS;
 	}
 
 	p = rtw_get_ie(pframe + WLAN_HDR_A3_LEN + _PROBEREQ_IE_OFFSET_, WLAN_EID_SSID, (int *)&ielen,
 			len - WLAN_HDR_A3_LEN - _PROBEREQ_IE_OFFSET_);
 
-
 	/* check (wildcard) SSID */
 	if (p) {
 		if (is_valid_p2p_probereq)
@@ -725,7 +720,7 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 		goto auth_fail;
 	}
 
-	if (rtw_access_ctrl(padapter, sa) == false) {
+	if (!rtw_access_ctrl(padapter, sa)) {
 		status = WLAN_STATUS_AP_UNABLE_TO_HANDLE_NEW_STA;
 		goto auth_fail;
 	}
@@ -748,7 +743,7 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 	} else {
 
 		spin_lock_bh(&pstapriv->asoc_list_lock);
-		if (list_empty(&pstat->asoc_list) == false) {
+		if (!list_empty(&pstat->asoc_list)) {
 			list_del_init(&pstat->asoc_list);
 			pstapriv->asoc_list_cnt--;
 			if (pstat->expire_to > 0) {
@@ -773,7 +768,6 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 	if (pstat->auth_seq == 0)
 		pstat->expire_to = pstapriv->auth_to;
 
-
 	if ((pstat->auth_seq + 1) != seq) {
 		status = WLAN_STATUS_UNKNOWN_AUTH_TRANSACTION;
 		goto auth_fail;
@@ -822,7 +816,6 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 		}
 	}
 
-
 	/*  Now, we are going to issue_auth... */
 	pstat->auth_seq = seq + 1;
 
@@ -831,7 +824,6 @@ unsigned int OnAuth(struct adapter *padapter, union recv_frame *precv_frame)
 	if (pstat->state & WIFI_FW_AUTH_SUCCESS)
 		pstat->auth_seq = 0;
 
-
 	return _SUCCESS;
 
 auth_fail:
@@ -958,7 +950,6 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 	else /*  WIFI_REASSOCREQ */
 		ie_offset = _REASOCREQ_IE_OFFSET_;
 
-
 	if (pkt_len < sizeof(struct ieee80211_hdr_3addr) + ie_offset)
 		return _FAIL;
 
@@ -988,7 +979,6 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 		pstat->state |= WIFI_FW_ASSOC_STATE;
 	}
 
-
 	pstat->capability = capab_info;
 
 	/* now parse all ieee802_11 ie to point to elems */
@@ -1124,7 +1114,6 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 		else
 			pstat->flags |= WLAN_STA_MAYBE_WPS;
 
-
 		/*  AP support WPA/RSN, and sta is going to do WPS, but AP is not ready */
 		/*  that the selected registrar of AP is _FLASE */
 		if ((psecuritypriv->wpa_psk > 0)
@@ -1159,13 +1148,11 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 			copy_len = min(sizeof(pstat->wpa_ie), wpa_ie_len + 2u);
 		}
 
-
 		if (copy_len > 0)
 			memcpy(pstat->wpa_ie, wpa_ie-2, copy_len);
 
 	}
 
-
 	/*  check if there is WMM IE & support WWM-PS */
 	pstat->flags &= ~WLAN_STA_WME;
 	pstat->qos_option = 0;
@@ -1238,13 +1225,11 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 	} else
 		pstat->flags &= ~WLAN_STA_HT;
 
-
-	if ((pmlmepriv->htpriv.ht_option == false) && (pstat->flags&WLAN_STA_HT)) {
+	if (!pmlmepriv->htpriv.ht_option && (pstat->flags&WLAN_STA_HT)) {
 		status = WLAN_STATUS_CHALLENGE_FAIL;
 		goto OnAssocReqFail;
 	}
 
-
 	if ((pstat->flags & WLAN_STA_HT) &&
 		    ((pstat->wpa2_pairwise_cipher&WPA_CIPHER_TKIP) ||
 		      (pstat->wpa_pairwise_cipher&WPA_CIPHER_TKIP))) {
@@ -1285,13 +1270,11 @@ unsigned int OnAssocReq(struct adapter *padapter, union recv_frame *precv_frame)
 
 			goto OnAssocReqFail;
 
-
 		} else {
 			pstapriv->sta_aid[pstat->aid - 1] = pstat;
 		}
 	}
 
-
 	pstat->state &= (~WIFI_FW_ASSOC_STATE);
 	pstat->state |= WIFI_FW_ASSOC_SUCCESS;
 
@@ -1474,7 +1457,7 @@ unsigned int OnDeAuth(struct adapter *padapter, union recv_frame *precv_frame)
 			u8 updated = false;
 
 			spin_lock_bh(&pstapriv->asoc_list_lock);
-			if (list_empty(&psta->asoc_list) == false) {
+			if (!list_empty(&psta->asoc_list)) {
 				list_del_init(&psta->asoc_list);
 				pstapriv->asoc_list_cnt--;
 				updated = ap_free_sta(padapter, psta, false, reason);
@@ -1485,7 +1468,6 @@ unsigned int OnDeAuth(struct adapter *padapter, union recv_frame *precv_frame)
 			associated_clients_update(padapter, updated);
 		}
 
-
 		return _SUCCESS;
 	}
 
@@ -1546,7 +1528,7 @@ unsigned int OnDisassoc(struct adapter *padapter, union recv_frame *precv_frame)
 			u8 updated = false;
 
 			spin_lock_bh(&pstapriv->asoc_list_lock);
-			if (list_empty(&psta->asoc_list) == false) {
+			if (!list_empty(&psta->asoc_list)) {
 				list_del_init(&psta->asoc_list);
 				pstapriv->asoc_list_cnt--;
 				updated = ap_free_sta(padapter, psta, false, reason);
@@ -2115,7 +2097,6 @@ void issue_beacon(struct adapter *padapter, int timeout_ms)
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
 	pwlanhdr = (struct ieee80211_hdr *)pframe;
 
-
 	fctrl = &(pwlanhdr->frame_control);
 	*(fctrl) = 0;
 
@@ -2204,12 +2185,10 @@ void issue_beacon(struct adapter *padapter, int timeout_ms)
 		pframe = rtw_set_ie(pframe, WLAN_EID_ERP_INFO, 1, &erpinfo, &pattrib->pktlen);
 	}
 
-
 	/*  EXTERNDED SUPPORTED RATE */
 	if (rate_len > 8)
 		pframe = rtw_set_ie(pframe, WLAN_EID_EXT_SUPP_RATES, (rate_len - 8), (cur_network->supported_rates + 8), &pattrib->pktlen);
 
-
 	/* todo:HT for adhoc */
 
 _issue_bcn:
@@ -2281,7 +2260,6 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 	pattrib->pktlen = pattrib->hdrlen;
 	pframe += pattrib->hdrlen;
 
-
 	if (cur_network->ie_length > MAX_IE_SZ)
 		return;
 
@@ -2405,23 +2383,17 @@ void issue_probersp(struct adapter *padapter, unsigned char *da, u8 is_valid_p2p
 			pframe = rtw_set_ie(pframe, WLAN_EID_ERP_INFO, 1, &erpinfo, &pattrib->pktlen);
 		}
 
-
 		/*  EXTERNDED SUPPORTED RATE */
 		if (rate_len > 8)
 			pframe = rtw_set_ie(pframe, WLAN_EID_EXT_SUPP_RATES, (rate_len - 8), (cur_network->supported_rates + 8), &pattrib->pktlen);
 
-
 		/* todo:HT for adhoc */
 
 	}
 
 	pattrib->last_txcmdsz = pattrib->pktlen;
 
-
 	dump_mgntframe(padapter, pmgntframe);
-
-	return;
-
 }
 
 static int _issue_probereq(struct adapter *padapter,
@@ -2449,7 +2421,6 @@ static int _issue_probereq(struct adapter *padapter,
 	pattrib = &pmgntframe->attrib;
 	update_mgntframe_attrib(padapter, pattrib);
 
-
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
@@ -2593,7 +2564,6 @@ void issue_auth(struct adapter *padapter, struct sta_info *psta, unsigned short
 	pframe += sizeof(struct ieee80211_hdr_3addr);
 	pattrib->pktlen = sizeof(struct ieee80211_hdr_3addr);
 
-
 	if (psta) { /*  for AP mode */
 		memcpy(pwlanhdr->addr1, psta->hwaddr, ETH_ALEN);
 		memcpy(pwlanhdr->addr2, myid(&(padapter->eeprompriv)), ETH_ALEN);
@@ -2654,7 +2624,6 @@ void issue_auth(struct adapter *padapter, struct sta_info *psta, unsigned short
 		le_tmp = cpu_to_le16(pmlmeinfo->auth_seq);
 		pframe = rtw_set_fixed_ie(pframe, _AUTH_SEQ_NUM_, (unsigned char *)&le_tmp, &(pattrib->pktlen));
 
-
 		/*  setting status code... */
 		le_tmp = cpu_to_le16(status);
 		pframe = rtw_set_fixed_ie(pframe, _STATUS_CODE_, (unsigned char *)&le_tmp, &(pattrib->pktlen));
@@ -2683,7 +2652,6 @@ void issue_auth(struct adapter *padapter, struct sta_info *psta, unsigned short
 	dump_mgntframe(padapter, pmgntframe);
 }
 
-
 void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_info *pstat, int pkt_type)
 {
 	struct xmit_frame	*pmgntframe;
@@ -2708,7 +2676,6 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i
 	pattrib = &pmgntframe->attrib;
 	update_mgntframe_attrib(padapter, pattrib);
 
-
 	memset(pmgntframe->buf_addr, 0, WLANHDR_OFFSET + TXDESC_OFFSET);
 
 	pframe = (u8 *)(pmgntframe->buf_addr) + TXDESC_OFFSET;
@@ -2721,7 +2688,6 @@ void issue_asocrsp(struct adapter *padapter, unsigned short status, struct sta_i
 	memcpy(GetAddr2Ptr(pwlanhdr), myid(&(padapter->eeprompriv)), ETH_ALEN);
 	memcpy(GetAddr3Ptr(pwlanhdr), get_my_bssid(&(pmlmeinfo->network)), ETH_ALEN);
 
-
 	SetSeqNum(pwlanhdr, pmlmeext->mgnt_seq);
 	pmlmeext->mgnt_seq++;
 	if ((pkt_type == WIFI_ASSOCRSP) || (pkt_type == WIFI_REASSOCRSP))
@@ -2880,7 +2846,6 @@ void issue_assocreq(struct adapter *padapter)
 	if (pmlmeext->cur_channel == 14) /*  for JAPAN, channel 14 can only uses B Mode(CCK) */
 		sta_bssrate_len = 4;
 
-
 	/* for (i = 0; i < sta_bssrate_len; i++) { */
 	/*  */
 
@@ -2889,12 +2854,10 @@ void issue_assocreq(struct adapter *padapter)
 			break;
 	}
 
-
 	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
 		if (pmlmeinfo->network.supported_rates[i] == 0)
 			break;
 
-
 		/*  Check if the AP's supported rates are also supported by STA. */
 		for (j = 0; j < sta_bssrate_len; j++) {
 			 /*  Avoid the proprietary data rate (22Mbps) of Handlink WSG-4000 AP */
@@ -2916,7 +2879,6 @@ void issue_assocreq(struct adapter *padapter)
 		goto exit; /* don't connect to AP if no joint supported rate */
 	}
 
-
 	if (bssrate_len > 8) {
 		pframe = rtw_set_ie(pframe, WLAN_EID_SUPP_RATES, 8, bssrate, &(pattrib->pktlen));
 		pframe = rtw_set_ie(pframe, WLAN_EID_EXT_SUPP_RATES, (bssrate_len - 8), (bssrate + 8), &(pattrib->pktlen));
@@ -2973,7 +2935,6 @@ void issue_assocreq(struct adapter *padapter)
 	if (pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_REALTEK)
 		pframe = rtw_set_ie(pframe, WLAN_EID_VENDOR_SPECIFIC, 6, REALTEK_96B_IE, &(pattrib->pktlen));
 
-
 	pattrib->last_txcmdsz = pattrib->pktlen;
 	dump_mgntframe(padapter, pmgntframe);
 
@@ -3070,7 +3031,6 @@ int issue_nulldata(struct adapter *padapter, unsigned char *da, unsigned int pow
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 	struct sta_info *psta;
 
-
 	/* da == NULL, assume it's null data for sta to ap*/
 	if (!da)
 		da = get_my_bssid(&(pmlmeinfo->network));
@@ -3082,7 +3042,7 @@ int issue_nulldata(struct adapter *padapter, unsigned char *da, unsigned int pow
 		else
 			rtw_hal_macid_wakeup(padapter, psta->mac_id);
 	} else {
-		rtw_warn_on(1);
+		WARN_ON(1);
 	}
 
 	do {
@@ -3120,7 +3080,6 @@ s32 issue_nulldata_in_interrupt(struct adapter *padapter, u8 *da)
 	struct mlme_ext_priv *pmlmeext;
 	struct mlme_ext_info *pmlmeinfo;
 
-
 	pmlmeext = &padapter->mlmeextpriv;
 	pmlmeinfo = &pmlmeext->mlmext_info;
 
@@ -3289,7 +3248,6 @@ static int _issue_deauth(struct adapter *padapter, unsigned char *da,
 
 	pattrib->last_txcmdsz = pattrib->pktlen;
 
-
 	if (wait_ack) {
 		ret = dump_mgntframe_and_wait_ack(padapter, pmgntframe);
 	} else {
@@ -3611,7 +3569,6 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter)
 	pframe = rtw_set_fixed_ie(pframe, 1, &(category), &(pattrib->pktlen));
 	pframe = rtw_set_fixed_ie(pframe, 1, &(action), &(pattrib->pktlen));
 
-
 	/*  */
 	if (pmlmepriv->num_FortyMHzIntolerant > 0) {
 		u8 iedata = 0;
@@ -3622,7 +3579,6 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter)
 
 	}
 
-
 	/*  */
 	memset(ICS, 0, sizeof(ICS));
 	if (pmlmepriv->num_sta_no_ht > 0) {
@@ -3663,7 +3619,6 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter)
 
 		spin_unlock_bh(&(pmlmepriv->scanned_queue.lock));
 
-
 		for (i = 0; i < 8; i++) {
 			int j, k = 0;
 
@@ -3691,7 +3646,6 @@ static void issue_action_BSSCoexistPacket(struct adapter *padapter)
 
 	}
 
-
 	pattrib->last_txcmdsz = pattrib->pktlen;
 
 	dump_mgntframe(padapter, pmgntframe);
@@ -3787,25 +3741,11 @@ void site_survey(struct adapter *padapter)
 	}
 
 	if (survey_channel != 0) {
-		/* PAUSE 4-AC Queue when site_survey */
-		/* rtw_hal_get_hwreg(padapter, HW_VAR_TXPAUSE, (u8 *)(&val8)); */
-		/* val8 |= 0x0f; */
-		/* rtw_hal_set_hwreg(padapter, HW_VAR_TXPAUSE, (u8 *)(&val8)); */
-		if (pmlmeext->sitesurvey_res.channel_idx == 0) {
-#ifdef DBG_FIXED_CHAN
-			if (pmlmeext->fixed_chan != 0xff)
-				set_channel_bwmode(padapter, pmlmeext->fixed_chan, HAL_PRIME_CHNL_OFFSET_DONT_CARE, CHANNEL_WIDTH_20);
-			else
-#endif
-				set_channel_bwmode(padapter, survey_channel, HAL_PRIME_CHNL_OFFSET_DONT_CARE, CHANNEL_WIDTH_20);
-		} else {
-#ifdef DBG_FIXED_CHAN
-			if (pmlmeext->fixed_chan != 0xff)
-				r8723bs_select_channel(padapter, pmlmeext->fixed_chan);
-			else
-#endif
-				r8723bs_select_channel(padapter, survey_channel);
-		}
+		if (pmlmeext->sitesurvey_res.channel_idx == 0)
+			set_channel_bwmode(padapter, survey_channel,
+					   HAL_PRIME_CHNL_OFFSET_DONT_CARE, CHANNEL_WIDTH_20);
+		else
+			r8723bs_select_channel(padapter, survey_channel);
 
 		if (ScanType == SCAN_ACTIVE) { /* obey the channel plan setting... */
 			{
@@ -3879,9 +3819,6 @@ void site_survey(struct adapter *padapter)
 			issue_action_BSSCoexistPacket(padapter);
 		}
 	}
-
-	return;
-
 }
 
 /* collect bss info from Beacon and Probe request/response frames. */
@@ -4170,7 +4107,6 @@ void start_clnt_auth(struct adapter *padapter)
 	pmlmeinfo->link_count = 0;
 	pmlmeext->retry = 0;
 
-
 	netdev_dbg(padapter->pnetdev, "start auth\n");
 	issue_auth(padapter, NULL, 0);
 
@@ -4178,7 +4114,6 @@ void start_clnt_auth(struct adapter *padapter)
 
 }
 
-
 void start_clnt_assoc(struct adapter *padapter)
 {
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
@@ -4225,7 +4160,6 @@ static void process_80211d(struct adapter *padapter, struct wlan_bssid_ex *bssid
 	u8 channel;
 	u8 i;
 
-
 	pregistrypriv = &padapter->registrypriv;
 	pmlmeext = &padapter->mlmeextpriv;
 
@@ -4416,9 +4350,6 @@ void report_survey_event(struct adapter *padapter, union recv_frame *precv_frame
 	rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
 
 	pmlmeext->sitesurvey_res.bss_cnt++;
-
-	return;
-
 }
 
 void report_surveydone_event(struct adapter *padapter)
@@ -4460,9 +4391,6 @@ void report_surveydone_event(struct adapter *padapter)
 	psurveydone_evt->bss_cnt = pmlmeext->sitesurvey_res.bss_cnt;
 
 	rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
-
-	return;
-
 }
 
 void report_join_res(struct adapter *padapter, int res)
@@ -4505,14 +4433,9 @@ void report_join_res(struct adapter *padapter, int res)
 	memcpy((unsigned char *)(&(pjoinbss_evt->network.network)), &(pmlmeinfo->network), sizeof(struct wlan_bssid_ex));
 	pjoinbss_evt->network.join_res	= pjoinbss_evt->network.aid = res;
 
-
 	rtw_joinbss_event_prehandle(padapter, (u8 *)&pjoinbss_evt->network);
 
-
 	rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
-
-	return;
-
 }
 
 void report_wmm_edca_update(struct adapter *padapter)
@@ -4554,9 +4477,6 @@ void report_wmm_edca_update(struct adapter *padapter)
 	pwmm_event->wmm = 0;
 
 	rtw_enqueue_cmd(pcmdpriv, pcmd_obj);
-
-	return;
-
 }
 
 void report_del_sta_event(struct adapter *padapter, unsigned char *MacAddr, unsigned short reason)
@@ -4600,7 +4520,6 @@ void report_del_sta_event(struct adapter *padapter, unsigned char *MacAddr, unsi
 	memcpy((unsigned char *)(&(pdel_sta_evt->macaddr)), MacAddr, ETH_ALEN);
 	memcpy((unsigned char *)(pdel_sta_evt->rsvd), (unsigned char *)(&reason), 2);
 
-
 	psta = rtw_get_stainfo(&padapter->stapriv, MacAddr);
 	if (psta)
 		mac_id = (int)psta->mac_id;
@@ -4784,7 +4703,6 @@ void mlmeext_joinbss_event_callback(struct adapter *padapter, int join_res)
 		/* update bc/mc sta_info */
 		update_bmc_sta(padapter);
 
-
 	/* turn on dynamic functions */
 	Switch_DM_Func(padapter, DYNAMIC_ALL_FUNC_ENABLE, true);
 
@@ -4973,7 +4891,7 @@ void linked_status_chk(struct adapter *padapter)
 		/*  todo: To check why we under miracast session, rx_chk would be false */
 		psta = rtw_get_stainfo(pstapriv, pmlmeinfo->network.mac_address);
 		if (psta) {
-			if (chk_ap_is_alive(padapter, psta) == false)
+			if (!chk_ap_is_alive(padapter, psta))
 				rx_chk = _FAIL;
 
 			if (pxmitpriv->last_tx_pkts == pxmitpriv->tx_pkts)
@@ -5097,7 +5015,6 @@ void link_timer_hdl(struct timer_list *t)
 	struct mlme_ext_priv *pmlmeext = &padapter->mlmeextpriv;
 	struct mlme_ext_info *pmlmeinfo = &pmlmeext->mlmext_info;
 
-
 	if (pmlmeinfo->state & WIFI_FW_AUTH_NULL) {
 		pmlmeinfo->state = WIFI_FW_NULL_STATE;
 		report_join_res(padapter, -3);
@@ -5191,7 +5108,7 @@ u8 setopmode_hdl(struct adapter *padapter, u8 *pbuf)
 	if (psetop->mode == Ndis802_11APMode) {
 		/*  Do this after port switch to */
 		/*  prevent from downloading rsvd page to wrong port */
-		rtw_btcoex_MediaStatusNotify(padapter, 1); /* connect */
+		rtw_btcoex_media_status_notify(padapter, 1); /* connect */
 	}
 
 	return H2C_SUCCESS;
@@ -5283,7 +5200,6 @@ u8 join_cmd_hdl(struct adapter *padapter, u8 *pbuf)
 		/* set_msr(padapter, _HW_STATE_NOLINK_); */
 		set_msr(padapter, _HW_STATE_STATION_);
 
-
 		rtw_hal_set_hwreg(padapter, HW_VAR_MLME_DISCONNECT, NULL);
 	}
 
@@ -5592,7 +5508,7 @@ u8 setkey_hdl(struct adapter *padapter, u8 *pbuf)
 		else
 			addr = null_addr;
 
-		ctrl = BIT(15) | BIT6 | ((pparm->algorithm) << 2) | pparm->keyid;
+		ctrl = BIT(15) | BIT(6) | ((pparm->algorithm) << 2) | pparm->keyid;
 		write_cam(padapter, cam_id, ctrl, addr, pparm->key);
 		netdev_dbg(padapter->pnetdev,
 			   "set group key camid:%d, addr:%pM, kid:%d, type:%s\n",
@@ -5890,7 +5806,7 @@ int rtw_chk_start_clnt_join(struct adapter *padapter, u8 *ch, u8 *bw, u8 *offset
 	bool connect_allow = true;
 
 	if (!ch || !bw || !offset) {
-		rtw_warn_on(1);
+		WARN_ON(1);
 		connect_allow = false;
 	}
 
diff --git a/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c b/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
index f074ea6e0f38e..ca1cb58fc8012 100644
--- a/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
+++ b/drivers/staging/rtl8723bs/core/rtw_pwrctrl.c
@@ -576,7 +576,7 @@ void LPS_Leave_check(struct adapter *padapter)
  *
  * This will be called when CPWM interrupt is up.
  *
- * using to update cpwn of drv; and drv willl make a decision to up or down pwr level
+ * using to update cpwn of drv; and drv will make a decision to up or down pwr level
  */
 void cpwm_int_hdl(struct adapter *padapter, struct reportpwrstate_parm *preportpwrstate)
 {
diff --git a/drivers/staging/rtl8723bs/core/rtw_recv.c b/drivers/staging/rtl8723bs/core/rtw_recv.c
index f78194d508dfc..e9ec6def7216f 100644
--- a/drivers/staging/rtl8723bs/core/rtw_recv.c
+++ b/drivers/staging/rtl8723bs/core/rtw_recv.c
@@ -21,9 +21,6 @@ void _rtw_init_sta_recv_priv(struct sta_recv_priv *psta_recvpriv)
 
 	spin_lock_init(&psta_recvpriv->lock);
 
-	/* for (i = 0; i<MAX_RX_NUMBLKS; i++) */
-	/* _rtw_init_queue(&psta_recvpriv->blk_strms[i]); */
-
 	INIT_LIST_HEAD(&psta_recvpriv->defrag_q.queue);
 	spin_lock_init(&psta_recvpriv->defrag_q.lock);
 }
@@ -60,7 +57,6 @@ signed int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *pada
 
 	precvframe = (union recv_frame *) precvpriv->precv_frame_buf;
 
-
 	for (i = 0; i < NR_RECVFRAME; i++) {
 		INIT_LIST_HEAD(&(precvframe->u.list));
 
@@ -73,7 +69,6 @@ signed int _rtw_init_recv_priv(struct recv_priv *precvpriv, struct adapter *pada
 
 		precvframe->u.hdr.adapter = padapter;
 		precvframe++;
-
 	}
 
 	res = rtw_hal_init_recv_priv(padapter);
@@ -115,7 +110,6 @@ void _rtw_free_recv_priv(struct recv_priv *precvpriv)
 
 union recv_frame *_rtw_alloc_recvframe(struct __queue *pfree_recv_queue)
 {
-
 	union recv_frame  *precvframe;
 	struct list_head	*plist, *phead;
 	struct adapter *padapter;
@@ -180,19 +174,14 @@ int rtw_free_recvframe(union recv_frame *precvframe, struct __queue *pfree_recv_
 	return _SUCCESS;
 }
 
-
-
-
 signed int _rtw_enqueue_recvframe(union recv_frame *precvframe, struct __queue *queue)
 {
-
 	struct adapter *padapter = precvframe->u.hdr.adapter;
 	struct recv_priv *precvpriv = &padapter->recvpriv;
 
 	/* INIT_LIST_HEAD(&(precvframe->u.hdr.list)); */
 	list_del_init(&(precvframe->u.hdr.list));
 
-
 	list_add_tail(&(precvframe->u.hdr.list), get_list_head(queue));
 
 	if (padapter)
@@ -257,7 +246,6 @@ u32 rtw_free_uc_swdec_pending_queue(struct adapter *adapter)
 	return cnt;
 }
 
-
 signed int rtw_enqueue_recvbuf_to_head(struct recv_buf *precvbuf, struct __queue *queue)
 {
 	spin_lock_bh(&queue->lock);
@@ -279,7 +267,6 @@ signed int rtw_enqueue_recvbuf(struct recv_buf *precvbuf, struct __queue *queue)
 	list_add_tail(&precvbuf->list, get_list_head(queue));
 	spin_unlock_bh(&queue->lock);
 	return _SUCCESS;
-
 }
 
 struct recv_buf *rtw_dequeue_recvbuf(struct __queue *queue)
@@ -299,13 +286,11 @@ struct recv_buf *rtw_dequeue_recvbuf(struct __queue *queue)
 		precvbuf = container_of(plist, struct recv_buf, list);
 
 		list_del_init(&precvbuf->list);
-
 	}
 
 	spin_unlock_bh(&queue->lock);
 
 	return precvbuf;
-
 }
 
 static void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup)
@@ -354,7 +339,6 @@ static void rtw_handle_tkip_mic_err(struct adapter *padapter, u8 bgroup)
 
 static signed int recvframe_chkmic(struct adapter *adapter,  union recv_frame *precvframe)
 {
-
 	signed int	i, res = _SUCCESS;
 	u32 datalen;
 	u8 miccode[8];
@@ -382,7 +366,7 @@ static signed int recvframe_chkmic(struct adapter *adapter,  union recv_frame *p
 
 				/* psecuritypriv->dot118021XGrpKeyid, pmlmeinfo->key_index, rxdata_key_idx); */
 
-				if (psecuritypriv->binstallGrpkey == false) {
+				if (!psecuritypriv->binstallGrpkey) {
 					res = _FAIL;
 					goto exit;
 				}
@@ -405,11 +389,11 @@ static signed int recvframe_chkmic(struct adapter *adapter,  union recv_frame *p
 					bmic_err = true;
 			}
 
-
-			if (bmic_err == true) {
+			if (bmic_err) {
 				/*  double check key_index for some timing issue , */
 				/*  cannot compare with psecuritypriv->dot118021XGrpKeyid also cause timing issue */
-				if ((is_multicast_ether_addr(prxattrib->ra) == true)  && (prxattrib->key_index != pmlmeinfo->key_index))
+				if (is_multicast_ether_addr(prxattrib->ra) &&
+				    (prxattrib->key_index != pmlmeinfo->key_index))
 					brpt_micerror = false;
 
 				if (prxattrib->bdecrypted && brpt_micerror)
@@ -426,18 +410,15 @@ static signed int recvframe_chkmic(struct adapter *adapter,  union recv_frame *p
 		}
 
 		recvframe_pull_tail(precvframe, 8);
-
 	}
 
 exit:
 	return res;
-
 }
 
 /* decrypt and set the ivlen, icvlen of the recv_frame */
 static union recv_frame *decryptor(struct adapter *padapter, union recv_frame *precv_frame)
 {
-
 	struct rx_pkt_attrib *prxattrib = &precv_frame->u.hdr.attrib;
 	struct security_priv *psecuritypriv = &padapter->securitypriv;
 	union recv_frame *return_packet = precv_frame;
@@ -463,7 +444,7 @@ static union recv_frame *decryptor(struct adapter *padapter, union recv_frame *p
 		}
 	}
 
-	if ((prxattrib->encrypt > 0) && ((prxattrib->bdecrypted == 0) || (psecuritypriv->sw_decrypt == true))) {
+	if ((prxattrib->encrypt > 0) && ((prxattrib->bdecrypted == 0) || psecuritypriv->sw_decrypt)) {
 		psecuritypriv->hw_decrypted = false;
 
 		switch (prxattrib->encrypt) {
@@ -582,7 +563,6 @@ static signed int recv_decache(union recv_frame *precv_frame, u8 bretry, struct
 	prxcache->tid_rxseq[tid] = seq_ctrl;
 
 	return _SUCCESS;
-
 }
 
 static void process_pwrbit_data(struct adapter *padapter, union recv_frame *precv_frame)
@@ -604,7 +584,6 @@ static void process_pwrbit_data(struct adapter *padapter, union recv_frame *prec
 				/* pstapriv->sta_dz_bitmap |= BIT(psta->aid); */
 
 				stop_sta_xmit(padapter, psta);
-
 			}
 		} else {
 			if (psta->state & WIFI_SLEEP_STATE) {
@@ -614,7 +593,6 @@ static void process_pwrbit_data(struct adapter *padapter, union recv_frame *prec
 				wakeup_sta_to_xmit(padapter, psta);
 			}
 		}
-
 	}
 }
 
@@ -713,9 +691,8 @@ static signed int sta2sta_data_frame(struct adapter *adapter, union recv_frame *
 	u8 *sta_addr = NULL;
 	signed int bmcast = is_multicast_ether_addr(pattrib->dst);
 
-	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true) ||
-		(check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true)) {
-
+	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
+	    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
 		/*  filter packets that SA is myself or multicast or broadcast */
 		if (!memcmp(myhwaddr, pattrib->src, ETH_ALEN)) {
 			ret = _FAIL;
@@ -736,7 +713,7 @@ static signed int sta2sta_data_frame(struct adapter *adapter, union recv_frame *
 
 		sta_addr = pattrib->src;
 
-	} else if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) == true) {
+	} else if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
 		/*  For Station mode, sa and bssid should always be BSSID, and DA is my mac-address */
 		if (memcmp(pattrib->bssid, pattrib->src, ETH_ALEN)) {
 			ret = _FAIL;
@@ -744,7 +721,7 @@ static signed int sta2sta_data_frame(struct adapter *adapter, union recv_frame *
 		}
 
 		sta_addr = pattrib->bssid;
-	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
+	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 		if (bmcast) {
 			/*  For AP mode, if DA == MCAST, then BSSID should be also MCAST */
 			if (!is_multicast_ether_addr(pattrib->bssid)) {
@@ -761,7 +738,7 @@ static signed int sta2sta_data_frame(struct adapter *adapter, union recv_frame *
 			sta_addr = pattrib->src;
 		}
 
-	} else if (check_fwstate(pmlmepriv, WIFI_MP_STATE) == true) {
+	} else if (check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
 		memcpy(pattrib->dst, GetAddr1Ptr(ptr), ETH_ALEN);
 		memcpy(pattrib->src, GetAddr2Ptr(ptr), ETH_ALEN);
 		memcpy(pattrib->bssid, GetAddr3Ptr(ptr), ETH_ALEN);
@@ -772,8 +749,6 @@ static signed int sta2sta_data_frame(struct adapter *adapter, union recv_frame *
 	} else
 		ret  = _FAIL;
 
-
-
 	if (bmcast)
 		*psta = rtw_get_bcmc_stainfo(adapter);
 	else
@@ -800,11 +775,9 @@ static signed int ap2sta_data_frame(struct adapter *adapter, union recv_frame *p
 	u8 *myhwaddr = myid(&adapter->eeprompriv);
 	signed int bmcast = is_multicast_ether_addr(pattrib->dst);
 
-	if ((check_fwstate(pmlmepriv, WIFI_STATION_STATE) == true) &&
-	    (check_fwstate(pmlmepriv, _FW_LINKED) == true ||
-	     check_fwstate(pmlmepriv, _FW_UNDER_LINKING) == true)
-		) {
-
+	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) &&
+	    (check_fwstate(pmlmepriv, _FW_LINKED) ||
+	     check_fwstate(pmlmepriv, _FW_UNDER_LINKING))) {
 		/*  filter packets that SA is myself or multicast or broadcast */
 		if (!memcmp(myhwaddr, pattrib->src, ETH_ALEN)) {
 			ret = _FAIL;
@@ -817,12 +790,10 @@ static signed int ap2sta_data_frame(struct adapter *adapter, union recv_frame *p
 			goto exit;
 		}
 
-
 		/*  check BSSID */
 		if (is_zero_ether_addr(pattrib->bssid) ||
 		    is_zero_ether_addr(mybssid) ||
 		    (memcmp(pattrib->bssid, mybssid, ETH_ALEN))) {
-
 			if (!bmcast)
 				issue_deauth(adapter, pattrib->bssid, WLAN_REASON_CLASS3_FRAME_FROM_NONASSOC_STA);
 
@@ -847,8 +818,8 @@ static signed int ap2sta_data_frame(struct adapter *adapter, union recv_frame *p
 			goto exit;
 		}
 
-	} else if ((check_fwstate(pmlmepriv, WIFI_MP_STATE) == true) &&
-		     (check_fwstate(pmlmepriv, _FW_LINKED) == true)) {
+	} else if (check_fwstate(pmlmepriv, WIFI_MP_STATE) &&
+		   check_fwstate(pmlmepriv, _FW_LINKED)) {
 		memcpy(pattrib->dst, GetAddr1Ptr(ptr), ETH_ALEN);
 		memcpy(pattrib->src, GetAddr2Ptr(ptr), ETH_ALEN);
 		memcpy(pattrib->bssid, GetAddr3Ptr(ptr), ETH_ALEN);
@@ -858,15 +829,13 @@ static signed int ap2sta_data_frame(struct adapter *adapter, union recv_frame *p
 		/*  */
 		memcpy(pattrib->bssid,  mybssid, ETH_ALEN);
 
-
 		*psta = rtw_get_stainfo(pstapriv, pattrib->bssid); /*  get sta_info */
 		if (!*psta) {
 			ret = _FAIL;
 			goto exit;
 		}
 
-
-	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
+	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 		/* Special case */
 		ret = RTW_RX_HANDLED;
 		goto exit;
@@ -874,7 +843,6 @@ static signed int ap2sta_data_frame(struct adapter *adapter, union recv_frame *p
 		if (!memcmp(myhwaddr, pattrib->dst, ETH_ALEN) && (!bmcast)) {
 			*psta = rtw_get_stainfo(pstapriv, pattrib->bssid); /*  get sta_info */
 			if (!*psta) {
-
 				/* for AP multicast issue , modify by yiwei */
 				static unsigned long send_issue_deauth_time;
 
@@ -903,7 +871,7 @@ static signed int sta2ap_data_frame(struct adapter *adapter, union recv_frame *p
 	unsigned char *mybssid  = get_bssid(pmlmepriv);
 	signed int ret = _SUCCESS;
 
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 		/* For AP mode, RA =BSSID, TX =STA(SRC_ADDR), A3 =DST_ADDR */
 		if (memcmp(pattrib->bssid, mybssid, ETH_ALEN)) {
 			ret = _FAIL;
@@ -1067,7 +1035,6 @@ static signed int validate_recv_ctrl_frame(struct adapter *padapter, union recv_
 	}
 
 	return _FAIL;
-
 }
 
 /* perform defrag */
@@ -1109,7 +1076,6 @@ static union recv_frame *recvframe_defrag(struct adapter *adapter,
 		pnextrframe = (union recv_frame *)plist;
 		pnfhdr = &pnextrframe->u.hdr;
 
-
 		/* check the fragment sequence  (2nd ~n fragment frame) */
 
 		if (curfragnum != pnfhdr->attrib.frag_num) {
@@ -1139,7 +1105,6 @@ static union recv_frame *recvframe_defrag(struct adapter *adapter,
 
 		pfhdr->attrib.icv_len = pnfhdr->attrib.icv_len;
 		plist = get_next(plist);
-
 	}
 
 	/* free the defrag_q queue and return the prframe */
@@ -1197,7 +1162,6 @@ static union recv_frame *recvframe_chk_defrag(struct adapter *padapter, union re
 					/* free current defrag_q */
 					rtw_free_recvframe_queue(pdefrag_q, pfree_recv_queue);
 
-
 			/* Then enqueue the 0~(n-1) fragment into the defrag_q */
 
 			/* spin_lock(&pdefrag_q->lock); */
@@ -1212,7 +1176,6 @@ static union recv_frame *recvframe_chk_defrag(struct adapter *padapter, union re
 			rtw_free_recvframe(precv_frame, pfree_recv_queue);
 			prtnframe = NULL;
 		}
-
 	}
 
 	if ((ismfrag == 0) && (fragnum != 0)) {
@@ -1233,10 +1196,8 @@ static union recv_frame *recvframe_chk_defrag(struct adapter *padapter, union re
 			rtw_free_recvframe(precv_frame, pfree_recv_queue);
 			prtnframe = NULL;
 		}
-
 	}
 
-
 	if ((prtnframe) && (prtnframe->u.hdr.attrib.privacy)) {
 		/* after defrag we must check tkip mic code */
 		if (recvframe_chkmic(padapter,  prtnframe) == _FAIL) {
@@ -1287,7 +1248,6 @@ static signed int validate_recv_mgnt_frame(struct adapter *padapter, union recv_
 	mgt_dispatcher(padapter, precv_frame);
 
 	return _SUCCESS;
-
 }
 
 static signed int validate_recv_data_frame(struct adapter *adapter, union recv_frame *precv_frame)
@@ -1343,13 +1303,11 @@ static signed int validate_recv_data_frame(struct adapter *adapter, union recv_f
 	default:
 		ret = _FAIL;
 		break;
-
 	}
 
 	if (ret == _FAIL || ret == RTW_RX_HANDLED)
 		goto exit;
 
-
 	if (!psta) {
 		ret = _FAIL;
 		goto exit;
@@ -1359,7 +1317,6 @@ static signed int validate_recv_data_frame(struct adapter *adapter, union recv_f
 	/* psta->signal_quality = prxcmd->sq; */
 	precv_frame->u.hdr.psta = psta;
 
-
 	pattrib->amsdu = 0;
 	pattrib->ack_policy = 0;
 	/* parsing QC field */
@@ -1377,7 +1334,6 @@ static signed int validate_recv_data_frame(struct adapter *adapter, union recv_f
 		pattrib->hdrlen = pattrib->to_fr_ds == 3 ? 30 : 24;
 	}
 
-
 	if (pattrib->order)/* HT-CTRL 11n */
 		pattrib->hdrlen += 4;
 
@@ -1413,7 +1369,7 @@ static signed int validate_80211w_mgmt(struct adapter *adapter, union recv_frame
 
 	/* only support station mode */
 	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) && check_fwstate(pmlmepriv, _FW_LINKED) &&
-	    adapter->securitypriv.binstallBIPkey == true) {
+	    adapter->securitypriv.binstallBIPkey) {
 		/* unicast management frame decrypt */
 		if (pattrib->privacy && !(is_multicast_ether_addr(GetAddr1Ptr(ptr))) &&
 			(subtype == WIFI_DEAUTH || subtype == WIFI_DISASSOC || subtype == WIFI_ACTION)) {
@@ -1475,7 +1431,6 @@ static signed int validate_80211w_mgmt(struct adapter *adapter, union recv_frame
 
 validate_80211w_fail:
 	return _FAIL;
-
 }
 
 static signed int validate_recv_frame(struct adapter *adapter, union recv_frame *precv_frame)
@@ -1536,17 +1491,6 @@ static signed int validate_recv_frame(struct adapter *adapter, union recv_frame
 			struct recv_priv *precvpriv = &adapter->recvpriv;
 
 			precvpriv->rx_drop++;
-		} else if (retval == _SUCCESS) {
-#ifdef DBG_RX_DUMP_EAP
-			u8 bDumpRxPkt;
-			u16 eth_type;
-
-			/*  dump eapol */
-			rtw_hal_get_def_var(adapter, HAL_DEF_DBG_DUMP_RXPKT, &(bDumpRxPkt));
-			/*  get ether_type */
-			memcpy(&eth_type, ptr + pattrib->hdrlen + pattrib->iv_len + LLC_HEADER_LENGTH, 2);
-			eth_type = ntohs((unsigned short) eth_type);
-#endif
 		}
 		break;
 	default:
@@ -1597,7 +1541,7 @@ static signed int wlanhdr_to_ethhdr(union recv_frame *precvframe)
 	eth_type = ntohs(be_tmp); /* pattrib->ether_type */
 	pattrib->eth_type = eth_type;
 
-	if ((check_fwstate(pmlmepriv, WIFI_MP_STATE) == true)) {
+	if (check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
 		ptr += rmv_len;
 		*ptr = 0x87;
 		*(ptr + 1) = 0x12;
@@ -1672,7 +1616,7 @@ static void rtw_recv_indicate_pkt(struct adapter *padapter, struct sk_buff *pkt,
 
 	/* Indicate the packets to upper layer */
 	if (pkt) {
-		if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
+		if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 			struct sk_buff *pskb2 = NULL;
 			struct sta_info *psta = NULL;
 			struct sta_priv *pstapriv = &padapter->stapriv;
@@ -1735,7 +1679,6 @@ static int amsdu_to_msdu(struct adapter *padapter, union recv_frame *prframe)
 	pdata = prframe->u.hdr.rx_data;
 
 	while (a_len > ETH_HLEN) {
-
 		/* Offset 12 denote 2 mac address */
 		nSubframe_Length = get_unaligned_be16(pdata + 12);
 
@@ -1827,7 +1770,6 @@ static int enqueue_reorder_recvframe(struct recv_reorder_ctrl *preorder_ctrl, un
 	/* spin_lock_irqsave(&ppending_recvframe_queue->lock, irql); */
 	/* spin_lock(&ppending_recvframe_queue->lock); */
 
-
 	phead = get_list_head(ppending_recvframe_queue);
 	plist = get_next(phead);
 
@@ -1843,10 +1785,8 @@ static int enqueue_reorder_recvframe(struct recv_reorder_ctrl *preorder_ctrl, un
 			return false;
 		else
 			break;
-
 	}
 
-
 	/* spin_lock_irqsave(&ppending_recvframe_queue->lock, irql); */
 	/* spin_lock(&ppending_recvframe_queue->lock); */
 
@@ -1858,7 +1798,6 @@ static int enqueue_reorder_recvframe(struct recv_reorder_ctrl *preorder_ctrl, un
 	/* spin_unlock_irqrestore(&ppending_recvframe_queue->lock, irql); */
 
 	return true;
-
 }
 
 static int rtw_recv_indicatepkt(struct adapter *padapter, union recv_frame *precv_frame)
@@ -1915,7 +1854,7 @@ static int recv_indicatepkts_in_order(struct adapter *padapter, struct recv_reor
 	plist = get_next(phead);
 
 	/*  Handling some condition for forced indicate case. */
-	if (bforced == true) {
+	if (bforced) {
 		if (list_empty(phead)) {
 			/*  spin_unlock_irqrestore(&ppending_recvframe_queue->lock, irql); */
 			/* spin_unlock(&ppending_recvframe_queue->lock); */
@@ -1926,13 +1865,11 @@ static int recv_indicatepkts_in_order(struct adapter *padapter, struct recv_reor
 		pattrib = &prframe->u.hdr.attrib;
 
 		preorder_ctrl->indicate_seq = pattrib->seq_num;
-
 	}
 
 	/*  Prepare indication list and indication. */
 	/*  Check if there is any packet need indicate. */
 	while (!list_empty(phead)) {
-
 		prframe = (union recv_frame *)plist;
 		pattrib = &prframe->u.hdr.attrib;
 
@@ -1950,8 +1887,8 @@ static int recv_indicatepkts_in_order(struct adapter *padapter, struct recv_reor
 
 			/* indicate this recv_frame */
 			if (!pattrib->amsdu) {
-				if ((padapter->bDriverStopped == false) &&
-				    (padapter->bSurpriseRemoved == false))
+				if (!padapter->bDriverStopped &&
+				    !padapter->bSurpriseRemoved)
 					rtw_recv_indicatepkt(padapter, prframe);/* indicate this recv_frame */
 
 			} else if (pattrib->amsdu == 1) {
@@ -1962,7 +1899,6 @@ static int recv_indicatepkts_in_order(struct adapter *padapter, struct recv_reor
 				/* error condition; */
 			}
 
-
 			/* Update local variables. */
 			bPktInBuf = false;
 
@@ -1970,7 +1906,6 @@ static int recv_indicatepkts_in_order(struct adapter *padapter, struct recv_reor
 			bPktInBuf = true;
 			break;
 		}
-
 	}
 
 	/* spin_unlock(&ppending_recvframe_queue->lock); */
@@ -1991,18 +1926,16 @@ static int recv_indicatepkt_reorder(struct adapter *padapter, union recv_frame *
 		wlanhdr_to_ethhdr(prframe);
 
 		if (pattrib->qos != 1) {
-			if ((padapter->bDriverStopped == false) &&
-			    (padapter->bSurpriseRemoved == false)) {
+			if (!padapter->bDriverStopped &&
+			    !padapter->bSurpriseRemoved) {
 				rtw_recv_indicatepkt(padapter, prframe);
 				return _SUCCESS;
-
 			}
 
 			return _FAIL;
-
 		}
 
-		if (preorder_ctrl->enable == false) {
+		if (!preorder_ctrl->enable) {
 			/* indicate this recv_frame */
 			preorder_ctrl->indicate_seq = pattrib->seq_num;
 
@@ -2013,7 +1946,7 @@ static int recv_indicatepkt_reorder(struct adapter *padapter, union recv_frame *
 			return _SUCCESS;
 		}
 	} else if (pattrib->amsdu == 1) { /* temp filter -> means didn't support A-MSDUs in a A-MPDU */
-		if (preorder_ctrl->enable == false) {
+		if (!preorder_ctrl->enable) {
 			preorder_ctrl->indicate_seq = pattrib->seq_num;
 
 			retval = amsdu_to_msdu(padapter, prframe);
@@ -2037,7 +1970,6 @@ static int recv_indicatepkt_reorder(struct adapter *padapter, union recv_frame *
 		goto _err_exit;
 	}
 
-
 	/* s4. */
 	/*  Indication process. */
 	/*  After Packet dropping and Sliding Window shifting as above, we can now just indicate the packets */
@@ -2049,7 +1981,7 @@ static int recv_indicatepkt_reorder(struct adapter *padapter, union recv_frame *
 	/*  */
 
 	/* recv_indicatepkts_in_order(padapter, preorder_ctrl, true); */
-	if (recv_indicatepkts_in_order(padapter, preorder_ctrl, false) == true) {
+	if (recv_indicatepkts_in_order(padapter, preorder_ctrl, false)) {
 		_set_timer(&preorder_ctrl->reordering_ctrl_timer, REORDER_WAIT_TIME);
 		spin_unlock_bh(&ppending_recvframe_queue->lock);
 	} else {
@@ -2065,7 +1997,6 @@ static int recv_indicatepkt_reorder(struct adapter *padapter, union recv_frame *
 	return _FAIL;
 }
 
-
 void rtw_reordering_ctrl_timeout_handler(struct timer_list *t)
 {
 	struct recv_reorder_ctrl *preorder_ctrl =
@@ -2073,17 +2004,15 @@ void rtw_reordering_ctrl_timeout_handler(struct timer_list *t)
 	struct adapter *padapter = preorder_ctrl->padapter;
 	struct __queue *ppending_recvframe_queue = &preorder_ctrl->pending_recvframe_queue;
 
-
 	if (padapter->bDriverStopped || padapter->bSurpriseRemoved)
 		return;
 
 	spin_lock_bh(&ppending_recvframe_queue->lock);
 
-	if (recv_indicatepkts_in_order(padapter, preorder_ctrl, true) == true)
+	if (recv_indicatepkts_in_order(padapter, preorder_ctrl, true))
 		_set_timer(&preorder_ctrl->reordering_ctrl_timer, REORDER_WAIT_TIME);
 
 	spin_unlock_bh(&ppending_recvframe_queue->lock);
-
 }
 
 static int process_recv_indicatepkts(struct adapter *padapter, union recv_frame *prframe)
@@ -2094,13 +2023,13 @@ static int process_recv_indicatepkts(struct adapter *padapter, union recv_frame
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	struct ht_priv *phtpriv = &pmlmepriv->htpriv;
 
-	if (phtpriv->ht_option == true) { /* B/G/N Mode */
+	if (phtpriv->ht_option) { /* B/G/N Mode */
 		/* prframe->u.hdr.preorder_ctrl = &precvpriv->recvreorder_ctrl[pattrib->priority]; */
 
 		if (recv_indicatepkt_reorder(padapter, prframe) != _SUCCESS) { /*  including perform A-MPDU Rx Ordering Buffer Control */
 
-			if ((padapter->bDriverStopped == false) &&
-			    (padapter->bSurpriseRemoved == false)) {
+			if (!padapter->bDriverStopped &&
+			    !padapter->bSurpriseRemoved) {
 				retval = _FAIL;
 				return retval;
 			}
@@ -2110,18 +2039,16 @@ static int process_recv_indicatepkts(struct adapter *padapter, union recv_frame
 		if (retval != _SUCCESS)
 			return retval;
 
-		if ((padapter->bDriverStopped == false) && (padapter->bSurpriseRemoved == false)) {
+		if (!padapter->bDriverStopped && !padapter->bSurpriseRemoved) {
 			/* indicate this recv_frame */
 			rtw_recv_indicatepkt(padapter, prframe);
 		} else {
 			retval = _FAIL;
 			return retval;
 		}
-
 	}
 
 	return retval;
-
 }
 
 static int recv_func_prehandle(struct adapter *padapter, union recv_frame *rframe)
@@ -2195,13 +2122,13 @@ static int recv_func(struct adapter *padapter, union recv_frame *rframe)
 	ret = recv_func_prehandle(padapter, rframe);
 
 	if (ret == _SUCCESS) {
-
 		/* check if need to enqueue into uc_swdec_pending_queue*/
 		if (check_fwstate(mlmepriv, WIFI_STATION_STATE) &&
-			!is_multicast_ether_addr(prxattrib->ra) && prxattrib->encrypt > 0 &&
-			(prxattrib->bdecrypted == 0 || psecuritypriv->sw_decrypt == true) &&
-			psecuritypriv->ndisauthtype == Ndis802_11AuthModeWPAPSK &&
-			!psecuritypriv->busetkipkey) {
+		    !is_multicast_ether_addr(prxattrib->ra) &&
+		    prxattrib->encrypt > 0 &&
+		    (prxattrib->bdecrypted == 0 || psecuritypriv->sw_decrypt) &&
+		    psecuritypriv->ndisauthtype == Ndis802_11AuthModeWPAPSK &&
+		    !psecuritypriv->busetkipkey) {
 			rtw_enqueue_recvframe(rframe, &padapter->recvpriv.uc_swdec_pending_queue);
 
 			if (recvpriv->free_recvframe_cnt < NR_RECVFRAME / 4) {
@@ -2221,7 +2148,6 @@ static int recv_func(struct adapter *padapter, union recv_frame *rframe)
 	return ret;
 }
 
-
 s32 rtw_recv_entry(union recv_frame *precvframe)
 {
 	struct adapter *padapter;
@@ -2263,7 +2189,6 @@ static void rtw_signal_stat_timer_hdl(struct timer_list *t)
 		adapter->recvpriv.signal_strength = adapter->recvpriv.signal_strength_dbg;
 		adapter->recvpriv.rssi = (s8)translate_percentage_to_dbm((u8)adapter->recvpriv.signal_strength_dbg);
 	} else {
-
 		if (recvpriv->signal_strength_data.update_req == 0) {/*  update_req is clear, means we got rx */
 			avg_signal_strength = recvpriv->signal_strength_data.avg_val;
 			num_signal_strength = recvpriv->signal_strength_data.total_num;
@@ -2286,8 +2211,8 @@ static void rtw_signal_stat_timer_hdl(struct timer_list *t)
 			}
 		}
 
-		if (check_fwstate(&adapter->mlmepriv, _FW_UNDER_SURVEY) == true ||
-		    check_fwstate(&adapter->mlmepriv, _FW_LINKED) == false
+		if (check_fwstate(&adapter->mlmepriv, _FW_UNDER_SURVEY) ||
+		    !check_fwstate(&adapter->mlmepriv, _FW_LINKED)
 		) {
 			goto set_timer;
 		}
@@ -2316,5 +2241,4 @@ static void rtw_signal_stat_timer_hdl(struct timer_list *t)
 
 set_timer:
 	rtw_set_signal_stat_timer(recvpriv);
-
 }
diff --git a/drivers/staging/rtl8723bs/core/rtw_security.c b/drivers/staging/rtl8723bs/core/rtw_security.c
index a00504ff29109..50d8e30f7eda8 100644
--- a/drivers/staging/rtl8723bs/core/rtw_security.c
+++ b/drivers/staging/rtl8723bs/core/rtw_security.c
@@ -449,47 +449,45 @@ u32 rtw_tkip_encrypt(struct adapter *padapter, u8 *pxmitframe)
 	pframe = ((struct xmit_frame *)pxmitframe)->buf_addr + hw_hdr_offset;
 
 	/* 4 start to encrypt each fragment */
-	if (pattrib->encrypt == _TKIP_) {
+	if (pattrib->encrypt != _TKIP_)
+		return _SUCCESS;
 
-		{
-			if (is_multicast_ether_addr(pattrib->ra))
-				prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey;
-			else
-				prwskey = pattrib->dot118021x_UncstKey.skey;
+	if (is_multicast_ether_addr(pattrib->ra))
+		prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey;
+	else
+		prwskey = pattrib->dot118021x_UncstKey.skey;
 
-			for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
-				iv = pframe + pattrib->hdrlen;
-				payload = pframe + pattrib->iv_len + pattrib->hdrlen;
+	for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
+		iv = pframe + pattrib->hdrlen;
+		payload = pframe + pattrib->iv_len + pattrib->hdrlen;
 
-				GET_TKIP_PN(iv, dot11txpn);
+		GET_TKIP_PN(iv, dot11txpn);
 
-				pnl = (u16)(dot11txpn.val);
-				pnh = (u32)(dot11txpn.val >> 16);
+		pnl = (u16)(dot11txpn.val);
+		pnh = (u32)(dot11txpn.val >> 16);
 
-				phase1((u16 *)&ttkey[0], prwskey, &pattrib->ta[0], pnh);
+		phase1((u16 *)&ttkey[0], prwskey, &pattrib->ta[0], pnh);
 
-				phase2(&rc4key[0], prwskey, (u16 *)&ttkey[0], pnl);
+		phase2(&rc4key[0], prwskey, (u16 *)&ttkey[0], pnl);
 
-				if ((curfragnum + 1) == pattrib->nr_frags) {	/* 4 the last fragment */
-					length = pattrib->last_txcmdsz - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len;
-					crc.f0 = cpu_to_le32(~crc32_le(~0, payload, length));
+		if ((curfragnum + 1) == pattrib->nr_frags) {	/* 4 the last fragment */
+			length = pattrib->last_txcmdsz - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len;
+			crc.f0 = cpu_to_le32(~crc32_le(~0, payload, length));
 
-					arc4_setkey(ctx, rc4key, 16);
-					arc4_crypt(ctx, payload, payload, length);
-					arc4_crypt(ctx, payload + length, crc.f1, 4);
+			arc4_setkey(ctx, rc4key, 16);
+			arc4_crypt(ctx, payload, payload, length);
+			arc4_crypt(ctx, payload + length, crc.f1, 4);
 
-				} else {
-					length = pxmitpriv->frag_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len;
-					crc.f0 = cpu_to_le32(~crc32_le(~0, payload, length));
+		} else {
+			length = pxmitpriv->frag_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len;
+			crc.f0 = cpu_to_le32(~crc32_le(~0, payload, length));
 
-					arc4_setkey(ctx, rc4key, 16);
-					arc4_crypt(ctx, payload, payload, length);
-					arc4_crypt(ctx, payload + length, crc.f1, 4);
+			arc4_setkey(ctx, rc4key, 16);
+			arc4_crypt(ctx, payload, payload, length);
+			arc4_crypt(ctx, payload + length, crc.f1, 4);
 
-					pframe += pxmitpriv->frag_len;
-					pframe = (u8 *)round_up((SIZE_PTR)(pframe), 4);
-				}
-			}
+			pframe += pxmitpriv->frag_len;
+			pframe = (u8 *)round_up((SIZE_PTR)(pframe), 4);
 		}
 	}
 	return res;
@@ -517,82 +515,82 @@ u32 rtw_tkip_decrypt(struct adapter *padapter, u8 *precvframe)
 	pframe = (unsigned char *)((union recv_frame *)precvframe)->u.hdr.rx_data;
 
 	/* 4 start to decrypt recvframe */
-	if (prxattrib->encrypt == _TKIP_) {
-		stainfo = rtw_get_stainfo(&padapter->stapriv, &prxattrib->ta[0]);
-		if (stainfo) {
-			if (is_multicast_ether_addr(prxattrib->ra)) {
-				static unsigned long start;
-				static u32 no_gkey_bc_cnt;
-				static u32 no_gkey_mc_cnt;
-
-				if (!psecuritypriv->binstallGrpkey) {
-					res = _FAIL;
-
-					if (start == 0)
-						start = jiffies;
-
-					if (is_broadcast_ether_addr(prxattrib->ra))
-						no_gkey_bc_cnt++;
-					else
-						no_gkey_mc_cnt++;
-
-					if (jiffies_to_msecs(jiffies - start) > 1000) {
-						if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
-							netdev_dbg(padapter->pnetdev,
-								   FUNC_ADPT_FMT " no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
-								   FUNC_ADPT_ARG(padapter),
-								   no_gkey_bc_cnt,
-								   no_gkey_mc_cnt);
-						}
-						start = jiffies;
-						no_gkey_bc_cnt = 0;
-						no_gkey_mc_cnt = 0;
-					}
-					goto exit;
-				}
+	if (prxattrib->encrypt != _TKIP_)
+		return _SUCCESS;
+
+	stainfo = rtw_get_stainfo(&padapter->stapriv, &prxattrib->ta[0]);
+	if (!stainfo)
+		return _FAIL;
+
+	if (is_multicast_ether_addr(prxattrib->ra)) {
+		static unsigned long start;
+		static u32 no_gkey_bc_cnt;
+		static u32 no_gkey_mc_cnt;
+
+		if (!psecuritypriv->binstallGrpkey) {
+			res = _FAIL;
+
+			if (start == 0)
+				start = jiffies;
 
+			if (is_broadcast_ether_addr(prxattrib->ra))
+				no_gkey_bc_cnt++;
+			else
+				no_gkey_mc_cnt++;
+
+			if (jiffies_to_msecs(jiffies - start) > 1000) {
 				if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
 					netdev_dbg(padapter->pnetdev,
-						   FUNC_ADPT_FMT " gkey installed. no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+						   FUNC_ADPT_FMT " no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
 						   FUNC_ADPT_ARG(padapter),
 						   no_gkey_bc_cnt,
 						   no_gkey_mc_cnt);
 				}
-				start = 0;
+				start = jiffies;
 				no_gkey_bc_cnt = 0;
 				no_gkey_mc_cnt = 0;
-
-				prwskey = psecuritypriv->dot118021XGrpKey[prxattrib->key_index].skey;
-			} else {
-				prwskey = &stainfo->dot118021x_UncstKey.skey[0];
 			}
+			goto exit;
+		}
 
-			iv = pframe + prxattrib->hdrlen;
-			payload = pframe + prxattrib->iv_len + prxattrib->hdrlen;
-			length = ((union recv_frame *)precvframe)->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len;
+		if (no_gkey_bc_cnt || no_gkey_mc_cnt) {
+			netdev_dbg(padapter->pnetdev,
+				   FUNC_ADPT_FMT " gkey installed. no_gkey_bc_cnt:%u, no_gkey_mc_cnt:%u\n",
+				   FUNC_ADPT_ARG(padapter),
+				   no_gkey_bc_cnt,
+				   no_gkey_mc_cnt);
+		}
+		start = 0;
+		no_gkey_bc_cnt = 0;
+		no_gkey_mc_cnt = 0;
 
-			GET_TKIP_PN(iv, dot11txpn);
+		prwskey = psecuritypriv->dot118021XGrpKey[prxattrib->key_index].skey;
+	} else {
+		prwskey = &stainfo->dot118021x_UncstKey.skey[0];
+	}
 
-			pnl = (u16)(dot11txpn.val);
-			pnh = (u32)(dot11txpn.val >> 16);
+	iv = pframe + prxattrib->hdrlen;
+	payload = pframe + prxattrib->iv_len + prxattrib->hdrlen;
+	length = ((union recv_frame *)precvframe)->u.hdr.len - prxattrib->hdrlen - prxattrib->iv_len;
 
-			phase1((u16 *)&ttkey[0], prwskey, &prxattrib->ta[0], pnh);
-			phase2(&rc4key[0], prwskey, (unsigned short *)&ttkey[0], pnl);
+	GET_TKIP_PN(iv, dot11txpn);
 
-			/* 4 decrypt payload include icv */
+	pnl = (u16)(dot11txpn.val);
+	pnh = (u32)(dot11txpn.val >> 16);
 
-			arc4_setkey(ctx, rc4key, 16);
-			arc4_crypt(ctx, payload, payload, length);
+	phase1((u16 *)&ttkey[0], prwskey, &prxattrib->ta[0], pnh);
+	phase2(&rc4key[0], prwskey, (unsigned short *)&ttkey[0], pnl);
 
-			*((u32 *)crc) = ~crc32_le(~0, payload, length - 4);
+	/* 4 decrypt payload include icv */
 
-			if (crc[3] != payload[length - 1] || crc[2] != payload[length - 2] ||
-			    crc[1] != payload[length - 3] || crc[0] != payload[length - 4])
-				res = _FAIL;
-		} else {
-			res = _FAIL;
-		}
-	}
+	arc4_setkey(ctx, rc4key, 16);
+	arc4_crypt(ctx, payload, payload, length);
+
+	*((u32 *)crc) = ~crc32_le(~0, payload, length - 4);
+
+	if (crc[3] != payload[length - 1] || crc[2] != payload[length - 2] ||
+	    crc[1] != payload[length - 3] || crc[0] != payload[length - 4])
+		res = _FAIL;
 exit:
 	return res;
 }
@@ -651,13 +649,8 @@ static void construct_mic_iv(u8 *mic_iv,
 
 		for (i = 2; i < 8; i++)
 			mic_iv[i] = mpdu[i + 8];   /* mic_iv[2:7] = A2[0:5] = mpdu[10:15] */
-		#ifdef CONSISTENT_PN_ORDER
-		for (i = 8; i < 14; i++)
-			mic_iv[i] = pn_vector[i - 8];           /* mic_iv[8:13] = PN[0:5] */
-		#else
 		for (i = 8; i < 14; i++)
 			mic_iv[i] = pn_vector[13 - i];          /* mic_iv[8:13] = PN[5:0] */
-		#endif
 		mic_iv[14] = (unsigned char)(payload_length / 256);
 		mic_iv[15] = (unsigned char)(payload_length % 256);
 }
@@ -774,13 +767,8 @@ static void construct_ctr_preload(u8 *ctr_preload,
 
 	for (i = 2; i < 8; i++)
 		ctr_preload[i] = mpdu[i + 8];                       /* ctr_preload[2:7] = A2[0:5] = mpdu[10:15] */
-#ifdef CONSISTENT_PN_ORDER
-	for (i = 8; i < 14; i++)
-		ctr_preload[i] =    pn_vector[i - 8];           /* ctr_preload[8:13] = PN[0:5] */
-#else
 	for (i = 8; i < 14; i++)
 		ctr_preload[i] =    pn_vector[13 - i];          /* ctr_preload[8:13] = PN[5:0] */
-#endif
 	ctr_preload[14] = (unsigned char)(c / 256); /* Ctr */
 	ctr_preload[15] = (unsigned char)(c % 256);
 }
@@ -965,24 +953,25 @@ u32 rtw_aes_encrypt(struct adapter *padapter, u8 *pxmitframe)
 	pframe = ((struct xmit_frame *)pxmitframe)->buf_addr + hw_hdr_offset;
 
 	/* 4 start to encrypt each fragment */
-	if (pattrib->encrypt == _AES_) {
-		if (is_multicast_ether_addr(pattrib->ra))
-			prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey;
-		else
-			prwskey = pattrib->dot118021x_UncstKey.skey;
+	if (pattrib->encrypt != _AES_)
+		return _SUCCESS;
 
-		for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
-			if ((curfragnum + 1) == pattrib->nr_frags) {	/* 4 the last fragment */
-				length = pattrib->last_txcmdsz - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len;
+	if (is_multicast_ether_addr(pattrib->ra))
+		prwskey = psecuritypriv->dot118021XGrpKey[psecuritypriv->dot118021XGrpKeyid].skey;
+	else
+		prwskey = pattrib->dot118021x_UncstKey.skey;
 
-				aes_cipher(prwskey, pattrib->hdrlen, pframe, length);
-			} else {
-				length = pxmitpriv->frag_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len;
+	for (curfragnum = 0; curfragnum < pattrib->nr_frags; curfragnum++) {
+		if ((curfragnum + 1) == pattrib->nr_frags) {	/* 4 the last fragment */
+			length = pattrib->last_txcmdsz - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len;
 
-				aes_cipher(prwskey, pattrib->hdrlen, pframe, length);
-				pframe += pxmitpriv->frag_len;
-				pframe = (u8 *)round_up((SIZE_PTR)(pframe), 4);
-			}
+			aes_cipher(prwskey, pattrib->hdrlen, pframe, length);
+		} else {
+			length = pxmitpriv->frag_len - pattrib->hdrlen - pattrib->iv_len - pattrib->icv_len;
+
+			aes_cipher(prwskey, pattrib->hdrlen, pframe, length);
+			pframe += pxmitpriv->frag_len;
+			pframe = (u8 *)round_up((SIZE_PTR)(pframe), 4);
 		}
 	}
 	return res;
@@ -1212,7 +1201,7 @@ u32 rtw_aes_decrypt(struct adapter *padapter, u8 *precvframe)
 	if (prxattrib->encrypt != _AES_)
 		return _SUCCESS;
 	stainfo = rtw_get_stainfo(&padapter->stapriv, &prxattrib->ta[0]);
-	if (stainfo)
+	if (!stainfo)
 		return _FAIL;
 	if (is_multicast_ether_addr(prxattrib->ra)) {
 		static unsigned long start;
diff --git a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
index 07a6db1d2317a..a1b7fe843979e 100644
--- a/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
+++ b/drivers/staging/rtl8723bs/core/rtw_sta_mgt.c
@@ -67,7 +67,6 @@ u32 _rtw_init_sta_priv(struct	sta_priv *pstapriv)
 
 	spin_lock_init(&pstapriv->sta_hash_lock);
 
-	/* _rtw_init_queue(&pstapriv->asoc_q); */
 	pstapriv->asoc_sta_count = 0;
 	INIT_LIST_HEAD(&pstapriv->sleep_q.queue);
 	spin_lock_init(&pstapriv->sleep_q.lock);
@@ -531,7 +530,7 @@ u8 rtw_access_ctrl(struct adapter *padapter, u8 *mac_addr)
 		paclnode = list_entry(plist, struct rtw_wlan_acl_node, list);
 
 		if (!memcmp(paclnode->addr, mac_addr, ETH_ALEN))
-			if (paclnode->valid == true) {
+			if (paclnode->valid) {
 				match = true;
 				break;
 			}
diff --git a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
index 6a7c09db4cd93..1d37c2d5b10d9 100644
--- a/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
+++ b/drivers/staging/rtl8723bs/core/rtw_wlan_util.c
@@ -147,7 +147,7 @@ unsigned int ratetbl2rateset(struct adapter *padapter, unsigned char *rateset)
 		default:
 			rate = ratetbl_val_2wifirate(rate);
 
-			if (is_basicrate(padapter, rate) == true)
+			if (is_basicrate(padapter, rate))
 				rate |= IEEE80211_BASIC_RATE_MASK;
 
 			rateset[len] = rate;
@@ -236,7 +236,7 @@ void Restore_DM_Func_Flag(struct adapter *padapter)
 
 void Switch_DM_Func(struct adapter *padapter, u32 mode, u8 enable)
 {
-	if (enable == true)
+	if (enable)
 		rtw_hal_set_hwreg(padapter, HW_VAR_DM_FUNC_SET, (u8 *)(&mode));
 	else
 		rtw_hal_set_hwreg(padapter, HW_VAR_DM_FUNC_CLR, (u8 *)(&mode));
@@ -254,36 +254,10 @@ inline u8 rtw_get_oper_ch(struct adapter *adapter)
 
 inline void rtw_set_oper_ch(struct adapter *adapter, u8 ch)
 {
-#ifdef DBG_CH_SWITCH
-	const int len = 128;
-	char msg[128] = {0};
-	int cnt = 0;
-	int i = 0;
-#endif  /* DBG_CH_SWITCH */
 	struct dvobj_priv *dvobj = adapter_to_dvobj(adapter);
 
 	if (dvobj->oper_channel != ch) {
 		dvobj->on_oper_ch_time = jiffies;
-
-#ifdef DBG_CH_SWITCH
-		cnt += scnprintf(msg + cnt, len - cnt, "switch to ch %3u", ch);
-
-		for (i = 0; i < dvobj->iface_nums; i++) {
-			struct adapter *iface = dvobj->padapters[i];
-
-			cnt += scnprintf(msg + cnt, len - cnt, " [%s:", ADPT_ARG(iface));
-			if (iface->mlmeextpriv.cur_channel == ch)
-				cnt += scnprintf(msg + cnt, len - cnt, "C");
-			else
-				cnt += scnprintf(msg + cnt, len - cnt, "_");
-			if (iface->wdinfo.listen_channel == ch && !rtw_p2p_chk_state(&iface->wdinfo, P2P_STATE_NONE))
-				cnt += scnprintf(msg + cnt, len - cnt, "L");
-			else
-				cnt += scnprintf(msg + cnt, len - cnt, "_");
-			cnt += scnprintf(msg + cnt, len - cnt, "]");
-		}
-
-#endif /* DBG_CH_SWITCH */
 	}
 
 	dvobj->oper_channel = ch;
@@ -340,7 +314,6 @@ void set_channel_bwmode(struct adapter *padapter, unsigned char channel, unsigne
 
 	center_ch = rtw_get_center_ch(channel, bwmode, channel_offset);
 
-
 	/* set Channel */
 	if (mutex_lock_interruptible(&(adapter_to_dvobj(padapter)->setch_mutex)))
 		return;
@@ -521,7 +494,7 @@ static bool _rtw_camid_is_gk(struct adapter *adapter, u8 cam_id)
 	if (!(cam_ctl->bitmap & BIT(cam_id)))
 		goto exit;
 
-	ret = (dvobj->cam_cache[cam_id].ctrl & BIT6) ? true : false;
+	ret = (dvobj->cam_cache[cam_id].ctrl & BIT(6)) ? true : false;
 
 exit:
 	return ret;
@@ -577,7 +550,7 @@ s16 rtw_camid_alloc(struct adapter *adapter, struct sta_info *sta, u8 kid)
 			netdev_dbg(adapter->pnetdev,
 				   FUNC_ADPT_FMT " group key with invalid key id:%u\n",
 				   FUNC_ADPT_ARG(adapter), kid);
-			rtw_warn_on(1);
+			WARN_ON(1);
 			goto bitmap_handle;
 		}
 
@@ -623,7 +596,7 @@ s16 rtw_camid_alloc(struct adapter *adapter, struct sta_info *sta, u8 kid)
 				netdev_dbg(adapter->pnetdev,
 					   FUNC_ADPT_FMT " group key id:%u no room\n",
 					   FUNC_ADPT_ARG(adapter), kid);
-			rtw_warn_on(1);
+			WARN_ON(1);
 			goto bitmap_handle;
 		}
 
@@ -847,7 +820,7 @@ static void bwmode_update_check(struct adapter *padapter, struct ndis_80211_var_
 	if (!pIE)
 		return;
 
-	if (phtpriv->ht_option == false)
+	if (!phtpriv->ht_option)
 		return;
 
 	if (pIE->length > sizeof(struct HT_info_element))
@@ -931,7 +904,7 @@ void HT_caps_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE)
 	if (!pIE)
 		return;
 
-	if (phtpriv->ht_option == false)
+	if (!phtpriv->ht_option)
 		return;
 
 	pmlmeinfo->HT_caps_enable = 1;
@@ -994,7 +967,7 @@ void HT_info_handler(struct adapter *padapter, struct ndis_80211_var_ie *pIE)
 	if (!pIE)
 		return;
 
-	if (phtpriv->ht_option == false)
+	if (!phtpriv->ht_option)
 		return;
 
 	if (pIE->length > sizeof(struct HT_info_element))
@@ -1122,7 +1095,7 @@ int rtw_check_bcn_info(struct adapter *Adapter, u8 *pframe, u32 packet_len)
 	struct mlme_priv *pmlmepriv = &Adapter->mlmepriv;
 	int ssid_len;
 
-	if (is_client_associated_to_ap(Adapter) == false)
+	if (!is_client_associated_to_ap(Adapter))
 		return true;
 
 	len = packet_len - sizeof(struct ieee80211_hdr_3addr);
@@ -1204,8 +1177,8 @@ int rtw_check_bcn_info(struct adapter *Adapter, u8 *pframe, u32 packet_len)
 		ssid_len = *(p + 1);
 		if (ssid_len > NDIS_802_11_LENGTH_SSID)
 			ssid_len = 0;
+		memcpy(bssid->ssid.ssid, (p + 2), ssid_len);
 	}
-	memcpy(bssid->ssid.ssid, (p + 2), ssid_len);
 	bssid->ssid.ssid_length = ssid_len;
 
 	if (memcmp(bssid->ssid.ssid, cur_network->network.ssid.ssid, 32) ||
@@ -1477,13 +1450,13 @@ void update_IOT_info(struct adapter *padapter)
 		pmlmeinfo->turboMode_cts2self = 0;
 		pmlmeinfo->turboMode_rtsen = 1;
 		/* disable high power */
-		Switch_DM_Func(padapter, (~DYNAMIC_BB_DYNAMIC_TXPWR), false);
+		Switch_DM_Func(padapter, ((u32)(~DYNAMIC_BB_DYNAMIC_TXPWR)), false);
 		break;
 	case HT_IOT_PEER_REALTEK:
 		/* rtw_write16(padapter, 0x4cc, 0xffff); */
 		/* rtw_write16(padapter, 0x546, 0x01c0); */
 		/* disable high power */
-		Switch_DM_Func(padapter, (~DYNAMIC_BB_DYNAMIC_TXPWR), false);
+		Switch_DM_Func(padapter, ((u32)(~DYNAMIC_BB_DYNAMIC_TXPWR)), false);
 		break;
 	default:
 		pmlmeinfo->turboMode_cts2self = 0;
@@ -1689,7 +1662,7 @@ void adaptive_early_32k(struct mlme_ext_priv *pmlmeext, u8 *pframe, uint len)
 		pmlmeext->bcn_delay_cnt[delay_ms]++;
 		/* pmlmeext->bcn_delay_ratio[delay_ms] = (pmlmeext->bcn_delay_cnt[delay_ms] * 100) /pmlmeext->bcn_cnt; */
 	/* dump for  adaptive_early_32k */
-	if (pmlmeext->bcn_cnt > 100 && (pmlmeext->adaptive_tsf_done == true)) {
+	if (pmlmeext->bcn_cnt > 100 && pmlmeext->adaptive_tsf_done) {
 		u8 ratio_20_delay, ratio_80_delay;
 		u8 DrvBcnEarly, DrvBcnTimeOut;
 
@@ -1737,7 +1710,7 @@ void rtw_alloc_macid(struct adapter *padapter, struct sta_info *psta)
 
 	spin_lock_bh(&pdvobj->lock);
 	for (i = 0; i < NUM_STA; i++) {
-		if (pdvobj->macid[i] == false) {
+		if (!pdvobj->macid[i]) {
 			pdvobj->macid[i]  = true;
 			break;
 		}
@@ -1762,7 +1735,7 @@ void rtw_release_macid(struct adapter *padapter, struct sta_info *psta)
 
 	spin_lock_bh(&pdvobj->lock);
 	if (psta->mac_id < NUM_STA && psta->mac_id != 1) {
-		if (pdvobj->macid[psta->mac_id] == true) {
+		if (pdvobj->macid[psta->mac_id]) {
 			pdvobj->macid[psta->mac_id] = false;
 			psta->mac_id = NUM_STA;
 		}
diff --git a/drivers/staging/rtl8723bs/core/rtw_xmit.c b/drivers/staging/rtl8723bs/core/rtw_xmit.c
index 7bce0343d59f1..458e471535ad5 100644
--- a/drivers/staging/rtl8723bs/core/rtw_xmit.c
+++ b/drivers/staging/rtl8723bs/core/rtw_xmit.c
@@ -33,22 +33,71 @@ void _rtw_init_sta_xmit_priv(struct sta_xmit_priv *psta_xmitpriv)
 	INIT_LIST_HEAD(&psta_xmitpriv->apsd);
 }
 
+static s32 rtw_alloc_hwxmits(struct adapter *padapter)
+{
+	struct hw_xmit *hwxmits;
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+
+	pxmitpriv->hwxmit_entry = HWXMIT_ENTRY;
+
+	pxmitpriv->hwxmits = NULL;
+
+	pxmitpriv->hwxmits = kzalloc_objs(*hwxmits, pxmitpriv->hwxmit_entry,
+					  GFP_ATOMIC);
+	if (!pxmitpriv->hwxmits)
+		return -ENOMEM;
+
+	hwxmits = pxmitpriv->hwxmits;
+
+	if (pxmitpriv->hwxmit_entry == 5) {
+		hwxmits[0] .sta_queue = &pxmitpriv->bm_pending;
+
+		hwxmits[1] .sta_queue = &pxmitpriv->vo_pending;
+
+		hwxmits[2] .sta_queue = &pxmitpriv->vi_pending;
+
+		hwxmits[3] .sta_queue = &pxmitpriv->bk_pending;
+
+		hwxmits[4] .sta_queue = &pxmitpriv->be_pending;
+	} else if (pxmitpriv->hwxmit_entry == 4) {
+		hwxmits[0] .sta_queue = &pxmitpriv->vo_pending;
+
+		hwxmits[1] .sta_queue = &pxmitpriv->vi_pending;
+
+		hwxmits[2] .sta_queue = &pxmitpriv->be_pending;
+
+		hwxmits[3] .sta_queue = &pxmitpriv->bk_pending;
+	} else {
+	}
+
+	return 0;
+}
+
+static int rtw_os_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 alloc_sz, u8 flag)
+{
+	if (alloc_sz > 0) {
+		pxmitbuf->pallocated_buf = kzalloc(alloc_sz, GFP_KERNEL);
+		if (!pxmitbuf->pallocated_buf)
+			return -ENOMEM;
+
+		pxmitbuf->pbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitbuf->pallocated_buf), XMITBUF_ALIGN_SZ);
+	}
+
+	return 0;
+}
+
 s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 {
 	int i;
 	struct xmit_buf *pxmitbuf;
 	struct xmit_frame *pxframe;
-	signed int	res = _SUCCESS;
+	int res;
 
 	spin_lock_init(&pxmitpriv->lock);
 	spin_lock_init(&pxmitpriv->lock_sctx);
 	init_completion(&pxmitpriv->xmit_comp);
 	init_completion(&pxmitpriv->terminate_xmitthread_comp);
 
-	/*
-	 * Please insert all the queue initialization using _rtw_init_queue below
-	 */
-
 	pxmitpriv->adapter = padapter;
 
 	INIT_LIST_HEAD(&pxmitpriv->be_pending.queue);
@@ -75,8 +124,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 
 	if (!pxmitpriv->pallocated_frame_buf) {
 		pxmitpriv->pxmit_frame_buf = NULL;
-		res = _FAIL;
-		goto exit;
+		return -ENOMEM;
 	}
 	pxmitpriv->pxmit_frame_buf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_frame_buf), 4);
 
@@ -111,10 +159,8 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 
 	pxmitpriv->pallocated_xmitbuf = vzalloc(NR_XMITBUFF * sizeof(struct xmit_buf) + 4);
 
-	if (!pxmitpriv->pallocated_xmitbuf) {
-		res = _FAIL;
-		goto exit;
-	}
+	if (!pxmitpriv->pallocated_xmitbuf)
+		return -ENOMEM;
 
 	pxmitpriv->pxmitbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_xmitbuf), 4);
 
@@ -129,11 +175,11 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 
 		/* Tx buf allocation may fail sometimes, so sleep and retry. */
 		res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ), true);
-		if (res == _FAIL) {
+		if (res) {
 			fsleep(10 * USEC_PER_MSEC);
 			res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, (MAX_XMITBUF_SZ + XMITBUF_ALIGN_SZ), true);
-			if (res == _FAIL)
-				goto exit;
+			if (res)
+				return res;
 		}
 
 		pxmitbuf->phead = pxmitbuf->pbuf;
@@ -162,8 +208,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 
 	if (!pxmitpriv->xframe_ext_alloc_addr) {
 		pxmitpriv->xframe_ext = NULL;
-		res = _FAIL;
-		goto exit;
+		return -ENOMEM;
 	}
 	pxmitpriv->xframe_ext = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->xframe_ext_alloc_addr), 4);
 	pxframe = (struct xmit_frame *)pxmitpriv->xframe_ext;
@@ -194,10 +239,8 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 
 	pxmitpriv->pallocated_xmit_extbuf = vzalloc(NR_XMIT_EXTBUFF * sizeof(struct xmit_buf) + 4);
 
-	if (!pxmitpriv->pallocated_xmit_extbuf) {
-		res = _FAIL;
-		goto exit;
-	}
+	if (!pxmitpriv->pallocated_xmit_extbuf)
+		return -ENOMEM;
 
 	pxmitpriv->pxmit_extbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitpriv->pallocated_xmit_extbuf), 4);
 
@@ -211,10 +254,8 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 		pxmitbuf->buf_tag = XMITBUF_MGNT;
 
 		res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf, MAX_XMIT_EXTBUF_SZ + XMITBUF_ALIGN_SZ, true);
-		if (res == _FAIL) {
-			res = _FAIL;
-			goto exit;
-		}
+		if (res)
+			return res;
 
 		pxmitbuf->phead = pxmitbuf->pbuf;
 		pxmitbuf->pend = pxmitbuf->pbuf + MAX_XMIT_EXTBUF_SZ;
@@ -243,10 +284,8 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 			res = rtw_os_xmit_resource_alloc(padapter, pxmitbuf,
 							 MAX_CMDBUF_SZ + XMITBUF_ALIGN_SZ,
 							 true);
-			if (res == _FAIL) {
-				res = _FAIL;
-				goto exit;
-			}
+			if (res)
+				return res;
 
 			pxmitbuf->phead = pxmitbuf->pbuf;
 			pxmitbuf->pend = pxmitbuf->pbuf + MAX_CMDBUF_SZ;
@@ -257,8 +296,8 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 	}
 
 	res = rtw_alloc_hwxmits(padapter);
-	if (res == _FAIL)
-		goto exit;
+	if (res)
+		return res;
 	rtw_init_hwxmits(pxmitpriv->hwxmits, pxmitpriv->hwxmit_entry);
 
 	for (i = 0; i < 4; i++)
@@ -270,8 +309,7 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter)
 
 	rtw_hal_init_xmit_priv(padapter);
 
-exit:
-	return res;
+	return 0;
 }
 
 void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv)
@@ -386,7 +424,8 @@ static void update_attrib_vcs_info(struct adapter *padapter, struct xmit_frame *
 	} else {
 		while (true) {
 			/* IOT action */
-			if ((pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_ATHEROS) && (pattrib->ampdu_en == true) &&
+			if ((pmlmeinfo->assoc_AP_vendor == HT_IOT_PEER_ATHEROS) &&
+			    pattrib->ampdu_en &&
 			    (padapter->securitypriv.dot11PrivacyAlgrthm == _AES_)) {
 				pattrib->vcs_mode = CTS_TO_SELF;
 				break;
@@ -422,7 +461,7 @@ static void update_attrib_vcs_info(struct adapter *padapter, struct xmit_frame *
 			/* to do list: check MIMO power save condition. */
 
 			/* check AMPDU aggregation for TXOP */
-			if (pattrib->ampdu_en == true) {
+			if (pattrib->ampdu_en) {
 				pattrib->vcs_mode = RTS_CTS;
 				break;
 			}
@@ -484,10 +523,10 @@ static s32 update_attrib_sec_info(struct adapter *padapter, struct pkt_attrib *p
 	memset(pattrib->dot11tkiptxmickey.skey,  0, 16);
 	pattrib->mac_id = psta->mac_id;
 
-	if (psta->ieee8021x_blocked == true) {
+	if (psta->ieee8021x_blocked) {
 		pattrib->encrypt = 0;
 
-		if ((pattrib->ether_type != 0x888e) && (check_fwstate(pmlmepriv, WIFI_MP_STATE) == false)) {
+		if ((pattrib->ether_type != 0x888e) && !check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
 			res = _FAIL;
 			goto exit;
 		}
@@ -652,8 +691,8 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 	memcpy(pattrib->dst, &etherhdr.h_dest, ETH_ALEN);
 	memcpy(pattrib->src, &etherhdr.h_source, ETH_ALEN);
 
-	if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true) ||
-		(check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true)) {
+	if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
+	    check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
 		memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
 		memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
 	} else if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
@@ -688,7 +727,7 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 			}
 		}
 
-		/* for parsing ICMP pakcets */
+		/* for parsing ICMP packets */
 		{
 			struct iphdr *piphdr = (struct iphdr *)tmp;
 
@@ -719,7 +758,7 @@ static s32 update_attrib(struct adapter *padapter, struct sk_buff *pkt, struct p
 		if (!psta)	{ /*  if we cannot get psta => drop the pkt */
 			res = _FAIL;
 			goto exit;
-		} else if ((check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) && (!(psta->state & _FW_LINKED))) {
+		} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) && !(psta->state & _FW_LINKED)) {
 			res = _FAIL;
 			goto exit;
 		}
@@ -901,7 +940,7 @@ s32 rtw_make_wlanhdr(struct adapter *padapter, u8 *hdr, struct pkt_attrib *pattr
 	SetFrameSubType(fctrl, pattrib->subtype);
 
 	if (pattrib->subtype & WIFI_DATA_TYPE) {
-		if (check_fwstate(pmlmepriv,  WIFI_STATION_STATE) == true) {
+		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
 			/* to_ds = 1, fr_ds = 0; */
 
 			{
@@ -915,7 +954,7 @@ s32 rtw_make_wlanhdr(struct adapter *padapter, u8 *hdr, struct pkt_attrib *pattr
 
 			if (pqospriv->qos_option)
 				qos_option = true;
-		} else if (check_fwstate(pmlmepriv,  WIFI_AP_STATE) == true) {
+		} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 			/* to_ds = 0, fr_ds = 1; */
 			SetFrDs(fctrl);
 			memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
@@ -924,8 +963,8 @@ s32 rtw_make_wlanhdr(struct adapter *padapter, u8 *hdr, struct pkt_attrib *pattr
 
 			if (pattrib->qos_en)
 				qos_option = true;
-		} else if ((check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true) ||
-		(check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true)) {
+		} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
+			   check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
 			memcpy(pwlanhdr->addr1, pattrib->dst, ETH_ALEN);
 			memcpy(pwlanhdr->addr2, pattrib->src, ETH_ALEN);
 			memcpy(pwlanhdr->addr3, get_bssid(pmlmepriv), ETH_ALEN);
@@ -983,7 +1022,7 @@ s32 rtw_make_wlanhdr(struct adapter *padapter, u8 *hdr, struct pkt_attrib *pattr
 						pattrib->ampdu_en = true;
 
 				/* re-check if enable ampdu by BA_starting_seqctrl */
-				if (pattrib->ampdu_en == true) {
+				if (pattrib->ampdu_en) {
 					u16 tx_seq;
 
 					tx_seq = psta->BA_starting_seqctrl[pattrib->priority & 0x0f];
@@ -1140,7 +1179,7 @@ s32 rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct
 
 		frg_inx++;
 
-		if (bmcst || (rtw_endofpktfile(&pktfile) == true)) {
+		if (bmcst || rtw_endofpktfile(&pktfile)) {
 			pattrib->nr_frags = frg_inx;
 
 			pattrib->last_txcmdsz = pattrib->hdrlen + pattrib->iv_len +
@@ -1166,7 +1205,7 @@ s32 rtw_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, struct
 
 	xmitframe_swencrypt(padapter, pxmitframe);
 
-	if (bmcst == false)
+	if (!bmcst)
 		update_attrib_vcs_info(padapter, pxmitframe);
 	else
 		pattrib->vcs_mode = NONE_VCS;
@@ -1320,7 +1359,7 @@ s32 rtw_mgmt_xmitframe_coalesce(struct adapter *padapter, struct sk_buff *pkt, s
 			/* set final tx command size */
 			pattrib->last_txcmdsz = pattrib->pktlen;
 
-			/* set protected bit must be beofre SW encrypt */
+			/* set protected bit must be before SW encrypt */
 			SetPrivacy(mem_start);
 			/* software encrypt */
 			xmitframe_swencrypt(padapter, pxmitframe);
@@ -1863,46 +1902,6 @@ s32 rtw_xmit_classifier(struct adapter *padapter, struct xmit_frame *pxmitframe)
 	return res;
 }
 
-s32 rtw_alloc_hwxmits(struct adapter *padapter)
-{
-	struct hw_xmit *hwxmits;
-	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
-
-	pxmitpriv->hwxmit_entry = HWXMIT_ENTRY;
-
-	pxmitpriv->hwxmits = NULL;
-
-	pxmitpriv->hwxmits = kzalloc_objs(*hwxmits, pxmitpriv->hwxmit_entry,
-					  GFP_ATOMIC);
-	if (!pxmitpriv->hwxmits)
-		return _FAIL;
-
-	hwxmits = pxmitpriv->hwxmits;
-
-	if (pxmitpriv->hwxmit_entry == 5) {
-		hwxmits[0] .sta_queue = &pxmitpriv->bm_pending;
-
-		hwxmits[1] .sta_queue = &pxmitpriv->vo_pending;
-
-		hwxmits[2] .sta_queue = &pxmitpriv->vi_pending;
-
-		hwxmits[3] .sta_queue = &pxmitpriv->bk_pending;
-
-		hwxmits[4] .sta_queue = &pxmitpriv->be_pending;
-	} else if (pxmitpriv->hwxmit_entry == 4) {
-		hwxmits[0] .sta_queue = &pxmitpriv->vo_pending;
-
-		hwxmits[1] .sta_queue = &pxmitpriv->vi_pending;
-
-		hwxmits[2] .sta_queue = &pxmitpriv->be_pending;
-
-		hwxmits[3] .sta_queue = &pxmitpriv->bk_pending;
-	} else {
-	}
-
-	return _SUCCESS;
-}
-
 void rtw_free_hwxmits(struct adapter *padapter)
 {
 	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
@@ -2003,14 +2002,14 @@ s32 rtw_xmit(struct adapter *padapter, struct sk_buff **ppkt)
 	do_queue_select(padapter, &pxmitframe->attrib);
 
 	spin_lock_bh(&pxmitpriv->lock);
-	if (xmitframe_enqueue_for_sleeping_sta(padapter, pxmitframe) == true) {
+	if (xmitframe_enqueue_for_sleeping_sta(padapter, pxmitframe)) {
 		spin_unlock_bh(&pxmitpriv->lock);
 		return 1;
 	}
 	spin_unlock_bh(&pxmitpriv->lock);
 
 	/* pre_xmitframe */
-	if (rtw_hal_xmit(padapter, pxmitframe) == false)
+	if (!rtw_hal_xmit(padapter, pxmitframe))
 		return 1;
 
 	return 0;
@@ -2039,7 +2038,7 @@ inline bool xmitframe_hiq_filter(struct xmit_frame *xmitframe)
 		allow = true;
 	else if (registry->hiq_filter == RTW_HIQ_FILTER_DENY_ALL) {
 	} else
-		rtw_warn_on(1);
+		WARN_ON(1);
 
 	return allow;
 }
@@ -2054,7 +2053,7 @@ signed int xmitframe_enqueue_for_sleeping_sta(struct adapter *padapter, struct x
 	signed int bmcst = is_multicast_ether_addr(pattrib->ra);
 	bool update_tim = false;
 
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == false)
+	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		return ret;
 	psta = rtw_get_stainfo(&padapter->stapriv, pattrib->ra);
 	if (pattrib->psta != psta)
diff --git a/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c b/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c
index b3e34f97cfc6b..f855df776a07e 100644
--- a/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c
+++ b/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.c
@@ -245,7 +245,7 @@ static void halbtc8723b1ant_QueryBtInfo(struct btc_coexist *pBtCoexist)
 
 	pCoexSta->bC2hBtInfoReqSent = true;
 
-	H2C_Parameter[0] |= BIT0;	/*  trigger */
+	H2C_Parameter[0] |= BIT(0);	/*  trigger */
 
 	pBtCoexist->fBtcFillH2c(pBtCoexist, 0x61, 1, H2C_Parameter);
 }
@@ -602,7 +602,7 @@ static void halbtc8723b1ant_SetSwPenaltyTxRateAdaptive(
 	H2C_Parameter[0] = 0x6;	/*  opCode, 0x6 = Retry_Penalty */
 
 	if (bLowPenaltyRa) {
-		H2C_Parameter[1] |= BIT0;
+		H2C_Parameter[1] |= BIT(0);
 		H2C_Parameter[2] = 0x00;  /* normal rate except MCS7/6/5, OFDM54/48/36 */
 		H2C_Parameter[3] = 0xf7;  /* MCS7 or OFDM54 */
 		H2C_Parameter[4] = 0xf8;  /* MCS6 or OFDM48 */
@@ -709,7 +709,7 @@ static void halbtc8723b1ant_SetFwIgnoreWlanAct(
 	u8 H2C_Parameter[1] = {0};
 
 	if (bEnable)
-		H2C_Parameter[0] |= BIT0; /* function enable */
+		H2C_Parameter[0] |= BIT(0); /* function enable */
 
 	pBtCoexist->fBtcFillH2c(pBtCoexist, 0x63, 1, H2C_Parameter);
 }
@@ -793,8 +793,9 @@ static void halbtc8723b1ant_SetAntPath(
 			/* Use H2C to set GNT_BT to HIGH */
 			H2C_Parameter[0] = 1;
 			pBtCoexist->fBtcFillH2c(pBtCoexist, 0x6E, 1, H2C_Parameter);
-		} else /*  set grant_bt to high */
+		} else { /*  set grant_bt to high */
 			pBtCoexist->fBtcWrite1Byte(pBtCoexist, 0x765, 0x18);
+		}
 
 		/* set wlan_act control by PTA */
 		pBtCoexist->fBtcWrite1Byte(pBtCoexist, 0x76e, 0x4);
@@ -810,8 +811,9 @@ static void halbtc8723b1ant_SetAntPath(
 			/* Use H2C to set GNT_BT to HIGH */
 			H2C_Parameter[0] = 1;
 			pBtCoexist->fBtcFillH2c(pBtCoexist, 0x6E, 1, H2C_Parameter);
-		} else /*  set grant_bt to high */
+		} else { /*  set grant_bt to high */
 			pBtCoexist->fBtcWrite1Byte(pBtCoexist, 0x765, 0x18);
+		}
 
 		/* set wlan_act to always low */
 		pBtCoexist->fBtcWrite1Byte(pBtCoexist, 0x76e, 0x4);
@@ -824,8 +826,8 @@ static void halbtc8723b1ant_SetAntPath(
 
 		/*  0x4c[24:23]= 00, Set Antenna control by BT_RFE_CTRL	BT Vendor 0xac = 0xf002 */
 		u4Tmp = pBtCoexist->fBtcRead4Byte(pBtCoexist, 0x4c);
-		u4Tmp &= ~BIT23;
-		u4Tmp &= ~BIT24;
+		u4Tmp &= ~BIT(23);
+		u4Tmp &= ~BIT(24);
 		pBtCoexist->fBtcWrite4Byte(pBtCoexist, 0x4c, u4Tmp);
 	} else {
 		/* Use H2C to set GNT_BT to LOW */
@@ -840,7 +842,7 @@ static void halbtc8723b1ant_SetAntPath(
 				u1Tmp = pBtCoexist->fBtcRead1Byte(pBtCoexist, 0x49d);
 				cntBtCalChk++;
 
-				if (u1Tmp & BIT0)
+				if (u1Tmp & BIT(0))
 					mdelay(50);
 				else
 					break;
@@ -859,8 +861,8 @@ static void halbtc8723b1ant_SetAntPath(
 		if (bInitHwCfg) {
 			/*  0x4c[23]= 0, 0x4c[24]= 1  Antenna control by WL/BT */
 			u4Tmp = pBtCoexist->fBtcRead4Byte(pBtCoexist, 0x4c);
-			u4Tmp &= ~BIT23;
-			u4Tmp |= BIT24;
+			u4Tmp &= ~BIT(23);
+			u4Tmp |= BIT(24);
 			pBtCoexist->fBtcWrite4Byte(pBtCoexist, 0x4c, u4Tmp);
 
 			pBtCoexist->fBtcWrite4Byte(pBtCoexist, 0x948, 0x0); /*  fixed internal switch S1->WiFi, S0->BT */
@@ -906,8 +908,8 @@ static void halbtc8723b1ant_SetAntPath(
 		if (bInitHwCfg) {
 			/*  0x4c[23]= 1, 0x4c[24]= 0  Antenna control by 0x64 */
 			u4Tmp = pBtCoexist->fBtcRead4Byte(pBtCoexist, 0x4c);
-			u4Tmp |= BIT23;
-			u4Tmp &= ~BIT24;
+			u4Tmp |= BIT(23);
+			u4Tmp &= ~BIT(24);
 			pBtCoexist->fBtcWrite4Byte(pBtCoexist, 0x4c, u4Tmp);
 
 			/* Fix Ext switch Main->S1, Aux->S0 */
@@ -965,12 +967,12 @@ static void halbtc8723b1ant_SetFwPstdma(
 	pBtCoexist->fBtcGet(pBtCoexist, BTC_GET_BL_WIFI_AP_MODE_ENABLE, &bApEnable);
 
 	if (bApEnable) {
-		if (byte1 & BIT4 && !(byte1 & BIT5)) {
-			realByte1 &= ~BIT4;
-			realByte1 |= BIT5;
+		if (byte1 & BIT(4) && !(byte1 & BIT(5))) {
+			realByte1 &= ~BIT(4);
+			realByte1 |= BIT(5);
 
-			realByte5 |= BIT5;
-			realByte5 &= ~BIT6;
+			realByte5 |= BIT(5);
+			realByte5 &= ~BIT(6);
 		}
 	}
 
@@ -1485,12 +1487,13 @@ static void halbtc8723b1ant_PsTdmaCheckForPowerSaveState(
 	if (lpsMode) {	/*  already under LPS state */
 		if (bNewPsState) {
 			/*  keep state under LPS, do nothing. */
-		} else /*  will leave LPS state, turn off psTdma first */
+		} else { /*  will leave LPS state, turn off psTdma first */
 			halbtc8723b1ant_PsTdma(pBtCoexist, NORMAL_EXEC, false, 0);
+		}
 	} else {						/*  NO PS state */
-		if (bNewPsState) /*  will enter LPS state, turn off psTdma first */
+		if (bNewPsState) { /*  will enter LPS state, turn off psTdma first */
 			halbtc8723b1ant_PsTdma(pBtCoexist, NORMAL_EXEC, false, 0);
-		else {
+		} else {
 			/*  keep state under NO PS state, do nothing. */
 		}
 	}
@@ -1855,8 +1858,9 @@ static void halbtc8723b1ant_ActionWifiConnected(struct btc_coexist *pBtCoexist)
 			halbtc8723b1ant_PowerSaveState(pBtCoexist, BTC_PS_WIFI_NATIVE, 0x0, 0x0);
 		else
 			halbtc8723b1ant_PowerSaveState(pBtCoexist, BTC_PS_LPS_ON, 0x50, 0x4);
-	} else
+	} else {
 		halbtc8723b1ant_PowerSaveState(pBtCoexist, BTC_PS_WIFI_NATIVE, 0x0, 0x0);
+	}
 
 	/*  tdma and coex table */
 	if (!bWifiBusy) {
@@ -2052,8 +2056,9 @@ static void halbtc8723b1ant_RunCoexistMechanism(struct btc_coexist *pBtCoexist)
 				halbtc8723b1ant_ActionWifiNotConnectedAssoAuth(pBtCoexist);
 		} else
 			halbtc8723b1ant_ActionWifiNotConnected(pBtCoexist);
-	} else /*  wifi LPS/Busy */
+	} else { /*  wifi LPS/Busy */
 		halbtc8723b1ant_ActionWifiConnected(pBtCoexist);
+	}
 }
 
 static void halbtc8723b1ant_InitCoexDm(struct btc_coexist *pBtCoexist)
@@ -2088,8 +2093,9 @@ static void halbtc8723b1ant_InitHwConfig(
 	if (bWifiOnly) {
 		halbtc8723b1ant_SetAntPath(pBtCoexist, BTC_ANT_PATH_WIFI, true, false);
 		halbtc8723b1ant_PsTdma(pBtCoexist, FORCE_EXEC, false, 9);
-	} else
+	} else {
 		halbtc8723b1ant_SetAntPath(pBtCoexist, BTC_ANT_PATH_BT, true, false);
+	}
 
 	/*  PTA parameter */
 	halbtc8723b1ant_CoexTableWithType(pBtCoexist, FORCE_EXEC, 0);
@@ -2115,7 +2121,7 @@ void EXhalbtc8723b1ant_PowerOnSetting(struct btc_coexist *pBtCoexist)
 
 	/*  enable BB, REG_SYS_FUNC_EN such that we can write 0x948 correctly. */
 	u2Tmp = pBtCoexist->fBtcRead2Byte(pBtCoexist, 0x2);
-	pBtCoexist->fBtcWrite2Byte(pBtCoexist, 0x2, u2Tmp | BIT0 | BIT1);
+	pBtCoexist->fBtcWrite2Byte(pBtCoexist, 0x2, u2Tmp | BIT(0) | BIT(1));
 
 	/*  set GRAN_BT = 1 */
 	pBtCoexist->fBtcWrite1Byte(pBtCoexist, 0x765, 0x18);
@@ -2494,7 +2500,7 @@ void EXhalbtc8723b1ant_BtInfoNotify(
 
 		/*  Here we need to resend some wifi info to BT */
 		/*  because bt is reset and loss of the info. */
-		if (pCoexSta->btInfoExt & BIT1) {
+		if (pCoexSta->btInfoExt & BIT(1)) {
 			pBtCoexist->fBtcGet(pBtCoexist, BTC_GET_BL_WIFI_CONNECTED, &bWifiConnected);
 			if (bWifiConnected)
 				EXhalbtc8723b1ant_MediaStatusNotify(pBtCoexist, BTC_MEDIA_CONNECT);
@@ -2502,7 +2508,7 @@ void EXhalbtc8723b1ant_BtInfoNotify(
 				EXhalbtc8723b1ant_MediaStatusNotify(pBtCoexist, BTC_MEDIA_DISCONNECT);
 		}
 
-		if (pCoexSta->btInfoExt & BIT3) {
+		if (pCoexSta->btInfoExt & BIT(3)) {
 			if (!pBtCoexist->bManualControl && !pBtCoexist->bStopCoexDm)
 				halbtc8723b1ant_IgnoreWlanAct(pBtCoexist, FORCE_EXEC, false);
 		} else {
diff --git a/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.h b/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.h
index de471e27a1852..7a12d92da0a91 100644
--- a/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.h
+++ b/drivers/staging/rtl8723bs/hal/HalBtc8723b1Ant.h
@@ -5,17 +5,17 @@
  *
  ******************************************************************************/
 /*  The following is for 8723B 1ANT BT Co-exist definition */
-#define	BT_INFO_8723B_1ANT_B_FTP		BIT7
-#define	BT_INFO_8723B_1ANT_B_A2DP		BIT6
-#define	BT_INFO_8723B_1ANT_B_HID		BIT5
-#define	BT_INFO_8723B_1ANT_B_SCO_BUSY		BIT4
-#define	BT_INFO_8723B_1ANT_B_ACL_BUSY		BIT3
-#define	BT_INFO_8723B_1ANT_B_INQ_PAGE		BIT2
-#define	BT_INFO_8723B_1ANT_B_SCO_ESCO		BIT1
-#define	BT_INFO_8723B_1ANT_B_CONNECTION		BIT0
+#define	BT_INFO_8723B_1ANT_B_FTP		BIT(7)
+#define	BT_INFO_8723B_1ANT_B_A2DP		BIT(6)
+#define	BT_INFO_8723B_1ANT_B_HID		BIT(5)
+#define	BT_INFO_8723B_1ANT_B_SCO_BUSY		BIT(4)
+#define	BT_INFO_8723B_1ANT_B_ACL_BUSY		BIT(3)
+#define	BT_INFO_8723B_1ANT_B_INQ_PAGE		BIT(2)
+#define	BT_INFO_8723B_1ANT_B_SCO_ESCO		BIT(1)
+#define	BT_INFO_8723B_1ANT_B_CONNECTION		BIT(0)
 
 #define	BT_INFO_8723B_1ANT_A2DP_BASIC_RATE(_BT_INFO_EXT_)	\
-		(((_BT_INFO_EXT_ & BIT0)) ? true : false)
+		(((_BT_INFO_EXT_ & BIT(0))) ? true : false)
 
 #define	BTC_RSSI_COEX_THRESH_TOL_8723B_1ANT 2
 
diff --git a/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c b/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c
index d32dbf94858fa..e4b056873d734 100644
--- a/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c
+++ b/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.c
@@ -185,7 +185,7 @@ static void halbtc8723b2ant_QueryBtInfo(struct btc_coexist *pBtCoexist)
 
 	pCoexSta->bC2hBtInfoReqSent = true;
 
-	H2C_Parameter[0] |= BIT0;	/*  trigger */
+	H2C_Parameter[0] |= BIT(0);	/*  trigger */
 
 	pBtCoexist->fBtcFillH2c(pBtCoexist, 0x61, 1, H2C_Parameter);
 }
@@ -294,9 +294,8 @@ static u8 halbtc8723b2ant_ActionAlgorithm(struct btc_coexist *pBtCoexist)
 
 	pBtCoexist->fBtcGet(pBtCoexist, BTC_GET_BL_HS_OPERATION, &bBtHsOn);
 
-	if (!pBtLinkInfo->bBtLinkExist) {
+	if (!pBtLinkInfo->bBtLinkExist)
 		return algorithm;
-	}
 
 	if (pBtLinkInfo->bScoExist)
 		numOfDiffProfile++;
@@ -510,7 +509,7 @@ static void halbtc8723b2ant_SetSwPenaltyTxRateAdaptive(
 	H2C_Parameter[0] = 0x6;	/*  opCode, 0x6 = Retry_Penalty */
 
 	if (bLowPenaltyRa) {
-		H2C_Parameter[1] |= BIT0;
+		H2C_Parameter[1] |= BIT(0);
 		H2C_Parameter[2] = 0x00;  /* normal rate except MCS7/6/5, OFDM54/48/36 */
 		H2C_Parameter[3] = 0xf7;  /* MCS7 or OFDM54 */
 		H2C_Parameter[4] = 0xf8;  /* MCS6 or OFDM48 */
@@ -749,7 +748,7 @@ static void halbtc8723b2ant_SetFwIgnoreWlanAct(
 	u8 	H2C_Parameter[1] = {0};
 
 	if (bEnable)
-		H2C_Parameter[0] |= BIT0;		/*  function enable */
+		H2C_Parameter[0] |= BIT(0);		/*  function enable */
 
 	pBtCoexist->fBtcFillH2c(pBtCoexist, 0x63, 1, H2C_Parameter);
 }
@@ -878,8 +877,8 @@ static void halbtc8723b2ant_SetAntPath(
 		if (bInitHwCfg) {
 			/*  0x4c[23]= 0, 0x4c[24]= 1  Antenna control by WL/BT */
 			u4Tmp = pBtCoexist->fBtcRead4Byte(pBtCoexist, 0x4c);
-			u4Tmp &= ~BIT23;
-			u4Tmp |= BIT24;
+			u4Tmp &= ~BIT(23);
+			u4Tmp |= BIT(24);
 			pBtCoexist->fBtcWrite4Byte(pBtCoexist, 0x4c, u4Tmp);
 		}
 
@@ -896,8 +895,8 @@ static void halbtc8723b2ant_SetAntPath(
 		if (bInitHwCfg) {
 			/*  0x4c[23]= 0, 0x4c[24]= 1  Antenna control by WL/BT */
 			u4Tmp = pBtCoexist->fBtcRead4Byte(pBtCoexist, 0x4c);
-			u4Tmp |= BIT23;
-			u4Tmp &= ~BIT24;
+			u4Tmp |= BIT(23);
+			u4Tmp &= ~BIT(24);
 			pBtCoexist->fBtcWrite4Byte(pBtCoexist, 0x4c, u4Tmp);
 		}
 
@@ -2211,7 +2210,7 @@ static void halbtc8723b2ant_RunCoexistMechanism(struct btc_coexist *pBtCoexist)
 	}
 
 	algorithm = halbtc8723b2ant_ActionAlgorithm(pBtCoexist);
-	if (pCoexSta->bC2hBtInquiryPage && (BT_8723B_2ANT_COEX_ALGO_PANHS != algorithm)) {
+	if (pCoexSta->bC2hBtInquiryPage && (algorithm != BT_8723B_2ANT_COEX_ALGO_PANHS)) {
 		halbtc8723b2ant_ActionBtInquiry(pBtCoexist);
 		return;
 	} else {
@@ -2337,7 +2336,7 @@ void EXhalbtc8723b2ant_PowerOnSetting(struct btc_coexist *pBtCoexist)
 
 	/*  enable BB, REG_SYS_FUNC_EN such that we can write 0x948 correctly. */
 	u2Tmp = pBtCoexist->fBtcRead2Byte(pBtCoexist, 0x2);
-	pBtCoexist->fBtcWrite2Byte(pBtCoexist, 0x2, u2Tmp | BIT0 | BIT1);
+	pBtCoexist->fBtcWrite2Byte(pBtCoexist, 0x2, u2Tmp | BIT(0) | BIT(1));
 
 	/*  set GRAN_BT = 1 */
 	pBtCoexist->fBtcWrite1Byte(pBtCoexist, 0x765, 0x18);
@@ -2490,7 +2489,7 @@ void EXhalbtc8723b2ant_BtInfoNotify(
 		return;
 	}
 
-	if (BT_INFO_SRC_8723B_2ANT_WIFI_FW != rspSource) {
+	if (rspSource != BT_INFO_SRC_8723B_2ANT_WIFI_FW) {
 		pCoexSta->btRetryCnt = pCoexSta->btInfoC2h[rspSource][2] & 0xf; /* [3:0] */
 
 		pCoexSta->btRssi = pCoexSta->btInfoC2h[rspSource][3] * 2 + 10;
@@ -2506,7 +2505,7 @@ void EXhalbtc8723b2ant_BtInfoNotify(
 
 		/*  Here we need to resend some wifi info to BT */
 		/*  because bt is reset and loss of the info. */
-		if ((pCoexSta->btInfoExt & BIT1)) {
+		if ((pCoexSta->btInfoExt & BIT(1))) {
 			pBtCoexist->fBtcGet(pBtCoexist, BTC_GET_BL_WIFI_CONNECTED, &bWifiConnected);
 
 			if (bWifiConnected)
@@ -2515,7 +2514,7 @@ void EXhalbtc8723b2ant_BtInfoNotify(
 				EXhalbtc8723b2ant_MediaStatusNotify(pBtCoexist, BTC_MEDIA_DISCONNECT);
 		}
 
-		if ((pCoexSta->btInfoExt & BIT3)) {
+		if ((pCoexSta->btInfoExt & BIT(3))) {
 			halbtc8723b2ant_IgnoreWlanAct(pBtCoexist, FORCE_EXEC, false);
 		} else {
 			/*  BT already NOT ignore Wlan active, do nothing here. */
diff --git a/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.h b/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.h
index 1896ac54614c2..ebc47f65e8d20 100644
--- a/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.h
+++ b/drivers/staging/rtl8723bs/hal/HalBtc8723b2Ant.h
@@ -5,14 +5,14 @@
  *
  ******************************************************************************/
 /*  The following is for 8723B 2Ant BT Co-exist definition */
-#define	BT_INFO_8723B_2ANT_B_FTP		BIT7
-#define	BT_INFO_8723B_2ANT_B_A2DP		BIT6
-#define	BT_INFO_8723B_2ANT_B_HID		BIT5
-#define	BT_INFO_8723B_2ANT_B_SCO_BUSY		BIT4
-#define	BT_INFO_8723B_2ANT_B_ACL_BUSY		BIT3
-#define	BT_INFO_8723B_2ANT_B_INQ_PAGE		BIT2
-#define	BT_INFO_8723B_2ANT_B_SCO_ESCO		BIT1
-#define	BT_INFO_8723B_2ANT_B_CONNECTION		BIT0
+#define	BT_INFO_8723B_2ANT_B_FTP		BIT(7)
+#define	BT_INFO_8723B_2ANT_B_A2DP		BIT(6)
+#define	BT_INFO_8723B_2ANT_B_HID		BIT(5)
+#define	BT_INFO_8723B_2ANT_B_SCO_BUSY		BIT(4)
+#define	BT_INFO_8723B_2ANT_B_ACL_BUSY		BIT(3)
+#define	BT_INFO_8723B_2ANT_B_INQ_PAGE		BIT(2)
+#define	BT_INFO_8723B_2ANT_B_SCO_ESCO		BIT(1)
+#define	BT_INFO_8723B_2ANT_B_CONNECTION		BIT(0)
 
 #define		BTC_RSSI_COEX_THRESH_TOL_8723B_2ANT		2
 
diff --git a/drivers/staging/rtl8723bs/hal/HalBtcOutSrc.h b/drivers/staging/rtl8723bs/hal/HalBtcOutSrc.h
index 9091f2f75fe1e..36b0f429dd4c2 100644
--- a/drivers/staging/rtl8723bs/hal/HalBtcOutSrc.h
+++ b/drivers/staging/rtl8723bs/hal/HalBtcOutSrc.h
@@ -69,11 +69,11 @@ enum btc_chip_interface {
 };
 
 /*  following is for wifi link status */
-#define WIFI_STA_CONNECTED				BIT0
-#define WIFI_AP_CONNECTED				BIT1
-#define WIFI_HS_CONNECTED				BIT2
-#define WIFI_P2P_GO_CONNECTED			BIT3
-#define WIFI_P2P_GC_CONNECTED			BIT4
+#define WIFI_STA_CONNECTED				BIT(0)
+#define WIFI_AP_CONNECTED				BIT(1)
+#define WIFI_HS_CONNECTED				BIT(2)
+#define WIFI_P2P_GO_CONNECTED			BIT(3)
+#define WIFI_P2P_GC_CONNECTED			BIT(4)
 
 struct btc_board_info {
 	/*  The following is some board information */
diff --git a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_BB.c b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_BB.c
index 4666b2ff3157f..afdde3980ec3e 100644
--- a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_BB.c
+++ b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_BB.c
@@ -11,11 +11,11 @@
 static bool CheckPositive(struct dm_odm_t *pDM_Odm, const u32 Condition1, const u32 Condition2)
 {
 	u8 _BoardType =
-		((pDM_Odm->BoardType & BIT4) >> 4) << 0 | /* _GLNA */
-		((pDM_Odm->BoardType & BIT3) >> 3) << 1 | /* _GPA */
-		((pDM_Odm->BoardType & BIT7) >> 7) << 2 | /* _ALNA */
-		((pDM_Odm->BoardType & BIT6) >> 6) << 3 | /* _APA */
-		((pDM_Odm->BoardType & BIT2) >> 2) << 4;  /* _BT */
+		((pDM_Odm->BoardType & BIT(4)) >> 4) << 0 | /* _GLNA */
+		((pDM_Odm->BoardType & BIT(3)) >> 3) << 1 | /* _GPA */
+		((pDM_Odm->BoardType & BIT(7)) >> 7) << 2 | /* _ALNA */
+		((pDM_Odm->BoardType & BIT(6)) >> 6) << 3 | /* _APA */
+		((pDM_Odm->BoardType & BIT(2)) >> 2) << 4;  /* _BT */
 
 	u32 cond1 = Condition1, cond2 = Condition2;
 	u32 driver1 =
@@ -51,13 +51,13 @@ static bool CheckPositive(struct dm_odm_t *pDM_Odm, const u32 Condition1, const
 		if ((cond1 & 0x0F) == 0) /* BoardType is DONTCARE */
 			return true;
 
-		if ((cond1 & BIT0) != 0) /* GLNA */
+		if ((cond1 & BIT(0)) != 0) /* GLNA */
 			bitMask |= 0x000000FF;
-		if ((cond1 & BIT1) != 0) /* GPA */
+		if ((cond1 & BIT(1)) != 0) /* GPA */
 			bitMask |= 0x0000FF00;
-		if ((cond1 & BIT2) != 0) /* ALNA */
+		if ((cond1 & BIT(2)) != 0) /* ALNA */
 			bitMask |= 0x00FF0000;
-		if ((cond1 & BIT3) != 0) /* APA */
+		if ((cond1 & BIT(3)) != 0) /* APA */
 			bitMask |= 0xFF000000;
 
 		/* BoardType of each RF path is matched */
@@ -223,7 +223,7 @@ void ODM_ReadAndConfig_MP_8723B_AGC_TAB(struct dm_odm_t *pDM_Odm)
 		} else {
 			/* This line is the beginning of branch. */
 			bool bMatched = true;
-			u8  cCond  = (u8)((v1 & (BIT29 | BIT28)) >> 28);
+			u8  cCond  = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
 
 			if (cCond == COND_ELSE) { /* ELSE, ENDIF */
 				bMatched = true;
@@ -255,10 +255,10 @@ void ODM_ReadAndConfig_MP_8723B_AGC_TAB(struct dm_odm_t *pDM_Odm)
 				}
 
 				/* Keeps reading until ENDIF. */
-				cCond = (u8)((v1 & (BIT29 | BIT28)) >> 28);
+				cCond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
 				while (cCond != COND_ENDIF && i < ArrayLen - 2) {
 					READ_NEXT_PAIR(v1, v2, i);
-					cCond = (u8)((v1 & (BIT29 | BIT28)) >> 28);
+					cCond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
 				}
 			}
 		}
@@ -483,7 +483,7 @@ void ODM_ReadAndConfig_MP_8723B_PHY_REG(struct dm_odm_t *pDM_Odm)
 		} else {
 			/* This line is the beginning of branch. */
 			bool bMatched = true;
-			u8  cCond  = (u8)((v1 & (BIT29 | BIT28)) >> 28);
+			u8  cCond  = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
 
 			if (cCond == COND_ELSE) { /* ELSE, ENDIF */
 				bMatched = true;
@@ -514,10 +514,10 @@ void ODM_ReadAndConfig_MP_8723B_PHY_REG(struct dm_odm_t *pDM_Odm)
 				}
 
 				/* Keeps reading until ENDIF. */
-				cCond = (u8)((v1 & (BIT29 | BIT28)) >> 28);
+				cCond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
 				while (cCond != COND_ENDIF && i < ArrayLen - 2) {
 					READ_NEXT_PAIR(v1, v2, i);
-					cCond = (u8)((v1 & (BIT29 | BIT28)) >> 28);
+					cCond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
 				}
 			}
 		}
diff --git a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.c b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.c
index 9a3393f5122b4..0e098bd2d9f61 100644
--- a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.c
+++ b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.c
@@ -11,11 +11,11 @@
 static bool CheckPositive(struct dm_odm_t *pDM_Odm, const u32 Condition1, const u32 Condition2)
 {
 	u8 _BoardType =
-		((pDM_Odm->BoardType & BIT4) >> 4) << 0 | /* _GLNA */
-		((pDM_Odm->BoardType & BIT3) >> 3) << 1 | /* _GPA */
-		((pDM_Odm->BoardType & BIT7) >> 7) << 2 | /* _ALNA */
-		((pDM_Odm->BoardType & BIT6) >> 6) << 3 | /* _APA */
-		((pDM_Odm->BoardType & BIT2) >> 2) << 4;  /* _BT */
+		((pDM_Odm->BoardType & BIT(4)) >> 4) << 0 | /* _GLNA */
+		((pDM_Odm->BoardType & BIT(3)) >> 3) << 1 | /* _GPA */
+		((pDM_Odm->BoardType & BIT(7)) >> 7) << 2 | /* _ALNA */
+		((pDM_Odm->BoardType & BIT(6)) >> 6) << 3 | /* _APA */
+		((pDM_Odm->BoardType & BIT(2)) >> 2) << 4;  /* _BT */
 
 	u32   cond1   = Condition1, cond2 = Condition2;
 	u32    driver1 =
@@ -51,13 +51,13 @@ static bool CheckPositive(struct dm_odm_t *pDM_Odm, const u32 Condition1, const
 		if ((cond1 & 0x0F) == 0) /* BoardType is DONTCARE */
 			return true;
 
-		if ((cond1 & BIT0) != 0) /* GLNA */
+		if ((cond1 & BIT(0)) != 0) /* GLNA */
 			bitMask |= 0x000000FF;
-		if ((cond1 & BIT1) != 0) /* GPA */
+		if ((cond1 & BIT(1)) != 0) /* GPA */
 			bitMask |= 0x0000FF00;
-		if ((cond1 & BIT2) != 0) /* ALNA */
+		if ((cond1 & BIT(2)) != 0) /* ALNA */
 			bitMask |= 0x00FF0000;
-		if ((cond1 & BIT3) != 0) /* APA */
+		if ((cond1 & BIT(3)) != 0) /* APA */
 			bitMask |= 0xFF000000;
 
 		/* BoardType of each RF path is matched */
@@ -195,7 +195,7 @@ void ODM_ReadAndConfig_MP_8723B_MAC_REG(struct dm_odm_t *pDM_Odm)
 		} else {
 			/* This line is the beginning of branch. */
 			bool bMatched = true;
-			u8  cCond  = (u8)((v1 & (BIT29 | BIT28)) >> 28);
+			u8  cCond  = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
 
 			if (cCond == COND_ELSE) { /* ELSE, ENDIF */
 				bMatched = true;
@@ -226,10 +226,10 @@ void ODM_ReadAndConfig_MP_8723B_MAC_REG(struct dm_odm_t *pDM_Odm)
 				}
 
 				/* Keeps reading until ENDIF. */
-				cCond = (u8)((v1 & (BIT29 | BIT28)) >> 28);
+				cCond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
 				while (cCond != COND_ENDIF && i < ArrayLen - 2) {
 					READ_NEXT_PAIR(v1, v2, i);
-					cCond = (u8)((v1 & (BIT29 | BIT28)) >> 28);
+					cCond = (u8)((v1 & (BIT(29) | BIT(28))) >> 28);
 				}
 			}
 		}
diff --git a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.h b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.h
index 50429c159feeb..82de8aa0008b4 100644
--- a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.h
+++ b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_MAC.h
@@ -1,17 +1,17 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /******************************************************************************
-*
-* Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
-*
-******************************************************************************/
+ *
+ * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
+ *
+ ******************************************************************************/
 
 #ifndef __INC_MP_MAC_HW_IMG_8723B_H
 #define __INC_MP_MAC_HW_IMG_8723B_H
 
 
 /******************************************************************************
-*                           MAC_REG.TXT
-******************************************************************************/
+ *                           MAC_REG.TXT
+ ******************************************************************************/
 
 void
 ODM_ReadAndConfig_MP_8723B_MAC_REG(/*  TC: Test Chip, MP: MP Chip */
diff --git a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.c b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.c
index 0c7d0307b822d..bfdebc8407307 100644
--- a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.c
+++ b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.c
@@ -13,11 +13,11 @@ static bool CheckPositive(
 )
 {
 	u8 _BoardType =
-			((pDM_Odm->BoardType & BIT4) >> 4) << 0 | /*  _GLNA */
-			((pDM_Odm->BoardType & BIT3) >> 3) << 1 | /*  _GPA */
-			((pDM_Odm->BoardType & BIT7) >> 7) << 2 | /*  _ALNA */
-			((pDM_Odm->BoardType & BIT6) >> 6) << 3 | /*  _APA */
-			((pDM_Odm->BoardType & BIT2) >> 2) << 4;  /*  _BT */
+			((pDM_Odm->BoardType & BIT(4)) >> 4) << 0 | /*  _GLNA */
+			((pDM_Odm->BoardType & BIT(3)) >> 3) << 1 | /*  _GPA */
+			((pDM_Odm->BoardType & BIT(7)) >> 7) << 2 | /*  _ALNA */
+			((pDM_Odm->BoardType & BIT(6)) >> 6) << 3 | /*  _APA */
+			((pDM_Odm->BoardType & BIT(2)) >> 2) << 4;  /*  _BT */
 
 	u32 cond1 = Condition1, cond2 = Condition2;
 	u32 driver1 =
@@ -59,13 +59,13 @@ static bool CheckPositive(
 		if ((cond1 & 0x0F) == 0) /*  BoardType is DONTCARE */
 			return true;
 
-		if ((cond1 & BIT0) != 0) /* GLNA */
+		if ((cond1 & BIT(0)) != 0) /* GLNA */
 			bitMask |= 0x000000FF;
-		if ((cond1 & BIT1) != 0) /* GPA */
+		if ((cond1 & BIT(1)) != 0) /* GPA */
 			bitMask |= 0x0000FF00;
-		if ((cond1 & BIT2) != 0) /* ALNA */
+		if ((cond1 & BIT(2)) != 0) /* ALNA */
 			bitMask |= 0x00FF0000;
-		if ((cond1 & BIT3) != 0) /* APA */
+		if ((cond1 & BIT(3)) != 0) /* APA */
 			bitMask |= 0xFF000000;
 
 		/*  BoardType of each RF path is matched */
@@ -227,7 +227,7 @@ void ODM_ReadAndConfig_MP_8723B_RadioA(struct dm_odm_t *pDM_Odm)
 		} else {
 			/*  This line is the beginning of branch. */
 			bool bMatched = true;
-			u8  cCond  = (u8)((v1 & (BIT29|BIT28)) >> 28);
+			u8  cCond  = (u8)((v1 & (BIT(29)|BIT(28))) >> 28);
 
 			if (cCond == COND_ELSE) { /*  ELSE, ENDIF */
 				bMatched = true;
@@ -259,10 +259,10 @@ void ODM_ReadAndConfig_MP_8723B_RadioA(struct dm_odm_t *pDM_Odm)
 				}
 
 				/*  Keeps reading until ENDIF. */
-				cCond = (u8)((v1 & (BIT29|BIT28)) >> 28);
+				cCond = (u8)((v1 & (BIT(29)|BIT(28))) >> 28);
 				while (cCond != COND_ENDIF && i < ArrayLen-2) {
 					READ_NEXT_PAIR(v1, v2, i);
-					cCond = (u8)((v1 & (BIT29|BIT28)) >> 28);
+					cCond = (u8)((v1 & (BIT(29)|BIT(28))) >> 28);
 				}
 			}
 		}
diff --git a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.h b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.h
index acf5679d188cf..04441fdeaa264 100644
--- a/drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.h
+++ b/drivers/staging/rtl8723bs/hal/HalHWImg8723B_RF.h
@@ -1,17 +1,17 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /******************************************************************************
-*
-* Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
-*
-******************************************************************************/
+ *
+ * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
+ *
+ ******************************************************************************/
 
 #ifndef __INC_MP_RF_HW_IMG_8723B_H
 #define __INC_MP_RF_HW_IMG_8723B_H
 
 
 /******************************************************************************
-*                           RadioA.TXT
-******************************************************************************/
+ *                           RadioA.TXT
+ ******************************************************************************/
 
 void
 ODM_ReadAndConfig_MP_8723B_RadioA(/*  TC: Test Chip, MP: MP Chip */
@@ -19,8 +19,8 @@ ODM_ReadAndConfig_MP_8723B_RadioA(/*  TC: Test Chip, MP: MP Chip */
 );
 
 /******************************************************************************
-*                           TxPowerTrack_SDIO.TXT
-******************************************************************************/
+ *                           TxPowerTrack_SDIO.TXT
+ ******************************************************************************/
 
 void
 ODM_ReadAndConfig_MP_8723B_TxPowerTrack_SDIO(/*  TC: Test Chip, MP: MP Chip */
@@ -29,8 +29,8 @@ ODM_ReadAndConfig_MP_8723B_TxPowerTrack_SDIO(/*  TC: Test Chip, MP: MP Chip */
 u32 ODM_GetVersion_MP_8723B_TxPowerTrack_SDIO(void);
 
 /******************************************************************************
-*                           TXPWR_LMT.TXT
-******************************************************************************/
+ *                           TXPWR_LMT.TXT
+ ******************************************************************************/
 
 void
 ODM_ReadAndConfig_MP_8723B_TXPWR_LMT(/*  TC: Test Chip, MP: MP Chip */
diff --git a/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c b/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
index 8f6849f2277e1..de431bc75627f 100644
--- a/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
+++ b/drivers/staging/rtl8723bs/hal/HalPhyRf_8723B.c
@@ -79,7 +79,7 @@ static void setIqkMatrix_8723B(
 			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_XCTxAFE, bMaskH4Bits, value32);
 
 			value32 = ((IqkResult_X * ele_D)>>7)&0x01;
-			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_ECCAThreshold, BIT24, value32);
+			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_ECCAThreshold, BIT(24), value32);
 			break;
 		case RF_PATH_B:
 			/* write new elements A, C, D to regC88 and regC9C,
@@ -92,7 +92,7 @@ static void setIqkMatrix_8723B(
 			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_XDTxAFE, bMaskH4Bits, value32);
 
 			value32 = ((IqkResult_X * ele_D)>>7)&0x01;
-			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_ECCAThreshold, BIT28, value32);
+			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_ECCAThreshold, BIT(28), value32);
 
 			break;
 		default:
@@ -103,13 +103,13 @@ static void setIqkMatrix_8723B(
 		case RF_PATH_A:
 			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_XATxIQImbalance, bMaskDWord, OFDMSwingTable_New[OFDM_index]);
 			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_XCTxAFE, bMaskH4Bits, 0x00);
-			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_ECCAThreshold, BIT24, 0x00);
+			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_ECCAThreshold, BIT(24), 0x00);
 			break;
 
 		case RF_PATH_B:
 			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_XBTxIQImbalance, bMaskDWord, OFDMSwingTable_New[OFDM_index]);
 			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_XDTxAFE, bMaskH4Bits, 0x00);
-			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_ECCAThreshold, BIT28, 0x00);
+			PHY_SetBBReg(pDM_Odm->Adapter, rOFDM0_ECCAThreshold, BIT(28), 0x00);
 			break;
 
 		default:
@@ -148,7 +148,8 @@ static void setCCKFilterCoefficient(struct dm_odm_t *pDM_Odm, u8 CCKSwingIndex)
  *When		Who	Remark
  *04/23/2012	MHC	Create Version 0.
  *
- *---------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------
+ */
 void ODM_TxPwrTrackSetPwr_8723B(
 	struct dm_odm_t *pDM_Odm,
 	enum pwrtrack_method Method,
@@ -301,7 +302,7 @@ static void GetDeltaSwingTable_8723B(
 	u16 rate = *(pDM_Odm->pForcedDataRate);
 	u8 channel = pHalData->CurrentChannel;
 
-	if (1 <= channel && channel <= 14) {
+	if (channel >= 1 && channel <= 14) {
 		if (IS_CCK_RATE(rate)) {
 			*TemperatureUP_A   = pRFCalibrateInfo->DeltaSwingTableIdx_2GCCKA_P;
 			*TemperatureDOWN_A = pRFCalibrateInfo->DeltaSwingTableIdx_2GCCKA_N;
@@ -426,7 +427,7 @@ static u8 phy_PathA_IQK_8723B(
 		tmp = 0x400 - tmp;
 
 	if (
-		!(regEAC & BIT28) &&
+		!(regEAC & BIT(28)) &&
 		(((regE94 & 0x03FF0000)>>16) != 0x142) &&
 		(((regE9C & 0x03FF0000)>>16) != 0x42) &&
 		(((regE94 & 0x03FF0000)>>16) < 0x110) &&
@@ -525,7 +526,7 @@ static u8 phy_PathA_RxIQK8723B(
 		tmp = 0x400 - tmp;
 
 	if (
-		!(regEAC & BIT28) &&
+		!(regEAC & BIT(28)) &&
 		(((regE94 & 0x03FF0000)>>16) != 0x142) &&
 		(((regE9C & 0x03FF0000)>>16) != 0x42) &&
 		(((regE94 & 0x03FF0000)>>16) < 0x110) &&
@@ -616,7 +617,7 @@ static u8 phy_PathA_RxIQK8723B(
 		tmp = 0x400 - tmp;
 
 	if (
-		!(regEAC & BIT27) && /* if Tx is OK, check whether Rx is OK */
+		!(regEAC & BIT(27)) && /* if Tx is OK, check whether Rx is OK */
 		(((regEA4 & 0x03FF0000)>>16) != 0x132) &&
 		(((regEAC & 0x03FF0000)>>16) != 0x36) &&
 		(((regEA4 & 0x03FF0000)>>16) < 0x110) &&
@@ -709,7 +710,7 @@ static u8 phy_PathB_IQK_8723B(struct adapter *padapter)
 		tmp = 0x400 - tmp;
 
 	if (
-		!(regEAC & BIT28) &&
+		!(regEAC & BIT(28)) &&
 		(((regE94 & 0x03FF0000)>>16) != 0x142) &&
 		(((regE9C & 0x03FF0000)>>16) != 0x42) &&
 		(((regE94 & 0x03FF0000)>>16) < 0x110) &&
@@ -804,7 +805,7 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 		tmp = 0x400 - tmp;
 
 	if (
-		!(regEAC & BIT28) &&
+		!(regEAC & BIT(28)) &&
 		(((regE94 & 0x03FF0000)>>16) != 0x142) &&
 		(((regE9C & 0x03FF0000)>>16) != 0x42)  &&
 		(((regE94 & 0x03FF0000)>>16) < 0x110) &&
@@ -896,7 +897,7 @@ static u8 phy_PathB_RxIQK8723B(struct adapter *padapter, bool configPathB)
 		tmp = 0x400 - tmp;
 
 	if (
-		!(regEAC & BIT27) && /* if Tx is OK, check whether Rx is OK */
+		!(regEAC & BIT(27)) && /* if Tx is OK, check whether Rx is OK */
 		(((regEA4 & 0x03FF0000)>>16) != 0x132) &&
 		(((regEAC & 0x03FF0000)>>16) != 0x36) &&
 		(((regEA4 & 0x03FF0000)>>16) < 0x110) &&
@@ -1195,9 +1196,9 @@ static void _PHY_MACSettingCalibration8723B(
 	rtw_write8(pDM_Odm->Adapter, MACReg[i], 0x3F);
 
 	for (i = 1 ; i < (IQK_MAC_REG_NUM - 1); i++) {
-		rtw_write8(pDM_Odm->Adapter, MACReg[i], (u8)(MACBackup[i]&(~BIT3)));
+		rtw_write8(pDM_Odm->Adapter, MACReg[i], (u8)(MACBackup[i]&(~BIT(3))));
 	}
-	rtw_write8(pDM_Odm->Adapter, MACReg[i], (u8)(MACBackup[i]&(~BIT5)));
+	rtw_write8(pDM_Odm->Adapter, MACReg[i], (u8)(MACBackup[i]&(~BIT(5))));
 
 }
 
@@ -1415,9 +1416,6 @@ static void phy_IQCalibrate_8723B(
 		}
 	}
 
-	if (0x00 == PathAOK) {
-	}
-
 /* path B IQK */
 	if (is2T) {
 
diff --git a/drivers/staging/rtl8723bs/hal/hal_btcoex.c b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
index 2a2dd60be8bb9..e508bf8dd7ff1 100644
--- a/drivers/staging/rtl8723bs/hal/hal_btcoex.c
+++ b/drivers/staging/rtl8723bs/hal/hal_btcoex.c
@@ -36,7 +36,7 @@ static void halbtcoutsrc_LeaveLps(struct btc_coexist *pBtCoexist)
 	pBtCoexist->btInfo.bBtCtrlLps = true;
 	pBtCoexist->btInfo.bBtLpsOn = false;
 
-	rtw_btcoex_LPS_Leave(padapter);
+	rtw_btcoex_lps_leave(padapter);
 }
 
 static void halbtcoutsrc_EnterLps(struct btc_coexist *pBtCoexist)
@@ -49,7 +49,7 @@ static void halbtcoutsrc_EnterLps(struct btc_coexist *pBtCoexist)
 	pBtCoexist->btInfo.bBtCtrlLps = true;
 	pBtCoexist->btInfo.bBtLpsOn = true;
 
-	rtw_btcoex_LPS_Enter(padapter);
+	rtw_btcoex_lps_enter(padapter);
 }
 
 static void halbtcoutsrc_NormalLps(struct btc_coexist *pBtCoexist)
@@ -60,7 +60,7 @@ static void halbtcoutsrc_NormalLps(struct btc_coexist *pBtCoexist)
 
 	if (pBtCoexist->btInfo.bBtCtrlLps) {
 		pBtCoexist->btInfo.bBtLpsOn = false;
-		rtw_btcoex_LPS_Leave(padapter);
+		rtw_btcoex_lps_leave(padapter);
 		pBtCoexist->btInfo.bBtCtrlLps = false;
 
 		/*  recover the LPS state to the original */
@@ -134,7 +134,7 @@ static void halbtcoutsrc_AggregationCheck(struct btc_coexist *pBtCoexist)
 	bNeedToAct = false;
 
 	if (pBtCoexist->btInfo.bRejectAggPkt) {
-		rtw_btcoex_RejectApAggregatedPacket(padapter, true);
+		rtw_btcoex_reject_ap_aggregated_packet(padapter, true);
 	} else {
 		if (pBtCoexist->btInfo.bPreBtCtrlAggBufSize !=
 			pBtCoexist->btInfo.bBtCtrlAggBufSize) {
@@ -151,8 +151,8 @@ static void halbtcoutsrc_AggregationCheck(struct btc_coexist *pBtCoexist)
 		}
 
 		if (bNeedToAct) {
-			rtw_btcoex_RejectApAggregatedPacket(padapter, true);
-			rtw_btcoex_RejectApAggregatedPacket(padapter, false);
+			rtw_btcoex_reject_ap_aggregated_packet(padapter, true);
+			rtw_btcoex_reject_ap_aggregated_packet(padapter, false);
 		}
 	}
 }
@@ -164,8 +164,8 @@ static u8 halbtcoutsrc_IsWifiBusy(struct adapter *padapter)
 
 	pmlmepriv = &padapter->mlmepriv;
 
-	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == true) {
-		if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true)
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE)) {
+		if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
 			return true;
 		if (pmlmepriv->link_detect_info.busy_traffic)
 			return true;
@@ -185,8 +185,8 @@ static u32 _halbtcoutsrc_GetWifiLinkStatus(struct adapter *padapter)
 	bp2p = false;
 	portConnectedStatus = 0;
 
-	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE) == true) {
-		if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
+	if (check_fwstate(pmlmepriv, WIFI_ASOC_STATE)) {
+		if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 			if (bp2p)
 				portConnectedStatus |= WIFI_P2P_GO_CONNECTED;
 			else
@@ -402,9 +402,6 @@ static u8 halbtcoutsrc_Get(void *pBtcContext, u8 getType, void *pOutBuf)
 
 	case BTC_GET_U1_MAC_PHY_MODE:
 		*pu8 = BTC_SMSP;
-/* 			*pU1Tmp = BTC_DMSP; */
-/* 			*pU1Tmp = BTC_DMDP; */
-/* 			*pU1Tmp = BTC_MP_UNKNOWN; */
 		break;
 
 	case BTC_GET_U1_AP_NUM:
@@ -528,7 +525,7 @@ static u8 halbtcoutsrc_Set(void *pBtcContext, u8 setType, void *pInBuf)
 	case BTC_SET_ACT_UPDATE_RAMASK:
 		pBtCoexist->btInfo.raMask = *pU4Tmp;
 
-		if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE) == true) {
+		if (check_fwstate(&padapter->mlmepriv, WIFI_ASOC_STATE)) {
 			struct sta_info *psta;
 			struct wlan_bssid_ex *cur_network;
 
diff --git a/drivers/staging/rtl8723bs/hal/hal_com.c b/drivers/staging/rtl8723bs/hal/hal_com.c
index 728a2171fbcb0..792cd071df22f 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com.c
@@ -30,48 +30,6 @@ void rtw_hal_data_deinit(struct adapter *padapter)
 	}
 }
 
-
-void dump_chip_info(struct hal_version	chip_version)
-{
-	char buf[128];
-	size_t cnt = 0;
-
-	cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "Chip Version Info: CHIP_8723B_%s_",
-			IS_NORMAL_CHIP(chip_version) ? "Normal_Chip" : "Test_Chip");
-
-	if (IS_CHIP_VENDOR_TSMC(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "TSMC_");
-	else if (IS_CHIP_VENDOR_UMC(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "UMC_");
-	else if (IS_CHIP_VENDOR_SMIC(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "SMIC_");
-
-	if (IS_A_CUT(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "A_CUT_");
-	else if (IS_B_CUT(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "B_CUT_");
-	else if (IS_C_CUT(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "C_CUT_");
-	else if (IS_D_CUT(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "D_CUT_");
-	else if (IS_E_CUT(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "E_CUT_");
-	else if (IS_I_CUT(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "I_CUT_");
-	else if (IS_J_CUT(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "J_CUT_");
-	else if (IS_K_CUT(chip_version))
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "K_CUT_");
-	else
-		cnt += scnprintf(buf + cnt, sizeof(buf) - cnt,
-				"UNKNOWN_CUT(%d)_", chip_version.CUTVersion);
-
-	cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "1T1R_");
-
-	cnt += scnprintf(buf + cnt, sizeof(buf) - cnt, "RomVer(%d)\n", chip_version.ROMVer);
-}
-
-
 #define	EEPROM_CHANNEL_PLAN_BY_HW_MASK	0x80
 
 /*
@@ -107,7 +65,7 @@ u8 hal_com_config_channel_plan(
 	pHalData->bDisableSWChannelPlan = false;
 	chnlPlan = def_channel_plan;
 
-	if (0xFF == hw_channel_plan)
+	if (hw_channel_plan == 0xFF)
 		auto_load_fail = true;
 
 	if (!auto_load_fail) {
@@ -136,7 +94,7 @@ bool HAL_IsLegalChannel(struct adapter *adapter, u32 Channel)
 	bool bLegalChannel = true;
 
 	if ((Channel <= 14) && (Channel >= 1)) {
-		if (is_supported_24g(adapter->registrypriv.wireless_mode) == false)
+		if (!is_supported_24g(adapter->registrypriv.wireless_mode))
 			bLegalChannel = false;
 	} else {
 		bLegalChannel = false;
@@ -294,7 +252,6 @@ void HalSetBrateCfg(struct adapter *Adapter, u8 *mBratesOS, u16 *pBrateCfg)
 	u8 i, is_brate, brate;
 
 	for (i = 0; i < NDIS_802_11_LENGTH_RATES_EX; i++) {
-
 		is_brate = mBratesOS[i] & IEEE80211_BASIC_RATE_MASK;
 		brate = mBratesOS[i] & 0x7f;
 
@@ -478,10 +435,10 @@ void rtw_init_hal_com_default_value(struct adapter *Adapter)
 }
 
 /*
-* C2H event format:
-* Field	 TRIGGER		CONTENT	   CMD_SEQ	CMD_LEN		 CMD_ID
-* BITS	 [127:120]	[119:16]      [15:8]		  [7:4]		   [3:0]
-*/
+ * C2H event format:
+ * Field	 TRIGGER		CONTENT	   CMD_SEQ	CMD_LEN		 CMD_ID
+ * BITS	 [127:120]	[119:16]      [15:8]		  [7:4]		   [3:0]
+ */
 
 void c2h_evt_clear(struct adapter *adapter)
 {
@@ -489,10 +446,10 @@ void c2h_evt_clear(struct adapter *adapter)
 }
 
 /*
-* C2H event format:
-* Field    TRIGGER    CMD_LEN    CONTENT    CMD_SEQ    CMD_ID
-* BITS    [127:120]   [119:112]    [111:16]	     [15:8]         [7:0]
-*/
+ * C2H event format:
+ * Field    TRIGGER    CMD_LEN    CONTENT    CMD_SEQ    CMD_ID
+ * BITS    [127:120]   [119:112]    [111:16]	     [15:8]         [7:0]
+ */
 s32 c2h_evt_read_88xx(struct adapter *adapter, u8 *buf)
 {
 	s32 ret = _FAIL;
@@ -526,9 +483,9 @@ s32 c2h_evt_read_88xx(struct adapter *adapter, u8 *buf)
 
 clear_evt:
 	/*
-	* Clear event to notify FW we have read the command.
-	* If this field isn't clear, the FW won't update the next command message.
-	*/
+	 * Clear event to notify FW we have read the command.
+	 * If this field isn't clear, the FW won't update the next command message.
+	 */
 	c2h_evt_clear(adapter);
 exit:
 	return ret;
@@ -576,7 +533,7 @@ void SetHwReg(struct adapter *adapter, u8 variable, u8 *val)
 
 	switch (variable) {
 	case HW_VAR_INIT_RTS_RATE:
-		rtw_warn_on(1);
+		WARN_ON(1);
 		break;
 	case HW_VAR_SEC_CFG:
 	{
@@ -605,7 +562,7 @@ void SetHwReg(struct adapter *adapter, u8 variable, u8 *val)
 		odm->SupportAbility = *((u32 *)val);
 		break;
 	case HW_VAR_DM_FUNC_OP:
-		if (*((u8 *)val) == true) {
+		if (*((u8 *)val)) {
 			/* save dm flag */
 			odm->BK_SupportAbility = odm->SupportAbility;
 		} else {
@@ -625,9 +582,9 @@ void SetHwReg(struct adapter *adapter, u8 variable, u8 *val)
 		break;
 	case HW_VAR_DM_FUNC_CLR:
 		/*
-		* input is already a mask to clear function
-		* don't invert it again! George, Lucas@20130513
-		*/
+		 * input is already a mask to clear function
+		 * don't invert it again! George, Lucas@20130513
+		 */
 		odm->SupportAbility &= *((u32 *)val);
 		break;
 	case HW_VAR_AMPDU_MIN_SPACE:
@@ -799,7 +756,7 @@ void rtw_bb_rf_gain_offset(struct adapter *padapter)
 	u32 v1 = 0, v2 = 0, target = 0;
 	u32 i = 0;
 
-	if (value & BIT4) {
+	if (value & BIT(4)) {
 		if (padapter->eeprompriv.EEPROMRFGainVal != 0xff) {
 			rtw_hal_read_rfreg(padapter, RF_PATH_A, 0x7f, 0xffffffff);
 
@@ -811,7 +768,7 @@ void rtw_bb_rf_gain_offset(struct adapter *padapter)
 					break;
 				}
 			}
-			PHY_SetRFReg(padapter, RF_PATH_A, REG_RF_BB_GAIN_OFFSET, BIT18|BIT17|BIT16|BIT15, target);
+			PHY_SetRFReg(padapter, RF_PATH_A, REG_RF_BB_GAIN_OFFSET, BIT(18)|BIT(17)|BIT(16)|BIT(15), target);
 
 			rtw_hal_read_rfreg(padapter, RF_PATH_A, 0x7f, 0xffffffff);
 		}
diff --git a/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c b/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
index bdd595a99b980..ea79e300f83b6 100644
--- a/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
+++ b/drivers/staging/rtl8723bs/hal/hal_com_phycfg.c
@@ -407,9 +407,9 @@ struct adapter *padapter
 }
 
 /*
-  * This function must be called if the value in the PHY_REG_PG.txt(or header)
-  * is exact dBm values
-  */
+ * This function must be called if the value in the PHY_REG_PG.txt(or header)
+ * is exact dBm values
+ */
 void PHY_TxPowerByRateConfiguration(struct adapter *padapter)
 {
 	phy_StoreTxPowerByRateBase(padapter);
@@ -461,23 +461,22 @@ u8 PHY_GetTxPowerIndexBase(
 	u8 txPower = 0;
 	u8 chnlIdx = (Channel-1);
 
-	if (HAL_IsLegalChannel(padapter, Channel) == false)
+	if (!HAL_IsLegalChannel(padapter, Channel))
 		chnlIdx = 0;
 
 	if (IS_CCK_RATE(Rate))
 		txPower = pHalData->Index24G_CCK_Base[RFPath][chnlIdx];
-	else if (MGN_6M <= Rate)
+	else if (Rate >= MGN_6M)
 		txPower = pHalData->Index24G_BW40_Base[RFPath][chnlIdx];
 
 	/*  OFDM-1T */
-	if ((MGN_6M <= Rate && Rate <= MGN_54M) && !IS_CCK_RATE(Rate))
+	if ((Rate >= MGN_6M && Rate <= MGN_54M) && !IS_CCK_RATE(Rate))
 		txPower += pHalData->OFDM_24G_Diff[RFPath][TX_1S];
 
-	if (BandWidth == CHANNEL_WIDTH_20) { /*  BW20-1S, BW20-2S */
-		if (MGN_MCS0 <= Rate && Rate <= MGN_MCS7)
+	if (Rate >= MGN_MCS0 && Rate <= MGN_MCS7) {
+		if (BandWidth == CHANNEL_WIDTH_20) /*  BW20-1S, BW20-2S */
 			txPower += pHalData->BW20_24G_Diff[RFPath][TX_1S];
-	} else if (BandWidth == CHANNEL_WIDTH_40) { /*  BW40-1S, BW40-2S */
-		if (MGN_MCS0 <= Rate && Rate <= MGN_MCS7)
+		else if (BandWidth == CHANNEL_WIDTH_40) /*  BW40-1S, BW40-2S */
 			txPower += pHalData->BW40_24G_Diff[RFPath][TX_1S];
 	}
 
@@ -490,7 +489,7 @@ s8 PHY_GetTxPowerTrackingOffset(struct adapter *padapter, u8 RFPath, u8 Rate)
 	struct dm_odm_t *pDM_Odm = &pHalData->odmpriv;
 	s8 offset = 0;
 
-	if (pDM_Odm->RFCalibrateInfo.TxPowerTrackControl  == false)
+	if (!pDM_Odm->RFCalibrateInfo.TxPowerTrackControl)
 		return offset;
 
 	if ((Rate == MGN_1M) || (Rate == MGN_2M) || (Rate == MGN_5_5M) || (Rate == MGN_11M))
diff --git a/drivers/staging/rtl8723bs/hal/hal_intf.c b/drivers/staging/rtl8723bs/hal/hal_intf.c
index 27c0c0198714f..03f23b75d5329 100644
--- a/drivers/staging/rtl8723bs/hal/hal_intf.c
+++ b/drivers/staging/rtl8723bs/hal/hal_intf.c
@@ -7,37 +7,6 @@
 #include <drv_types.h>
 #include <hal_data.h>
 
-void rtw_hal_chip_configure(struct adapter *padapter)
-{
-	rtl8723bs_interface_configure(padapter);
-}
-
-void rtw_hal_read_chip_info(struct adapter *padapter)
-{
-	ReadAdapterInfo8723BS(padapter);
-}
-
-void rtw_hal_read_chip_version(struct adapter *padapter)
-{
-	rtl8723b_read_chip_version(padapter);
-}
-
-void rtw_hal_def_value_init(struct adapter *padapter)
-{
-	rtl8723bs_init_default_value(padapter);
-}
-
-void rtw_hal_free_data(struct adapter *padapter)
-{
-	/* free HAL Data */
-	rtw_hal_data_deinit(padapter);
-}
-
-void rtw_hal_dm_init(struct adapter *padapter)
-{
-	rtl8723b_init_dm_priv(padapter);
-}
-
 static void rtw_hal_init_opmode(struct adapter *padapter)
 {
 	enum ndis_802_11_network_infrastructure networkType = Ndis802_11InfrastructureMax;
@@ -125,16 +94,6 @@ void rtw_hal_set_odm_var(struct adapter *padapter, enum hal_odm_variable eVariab
 	SetHalODMVar(padapter, eVariable, pValue1, bSet);
 }
 
-void rtw_hal_enable_interrupt(struct adapter *padapter)
-{
-	EnableInterrupt8723BSdio(padapter);
-}
-
-void rtw_hal_disable_interrupt(struct adapter *padapter)
-{
-	DisableInterrupt8723BSdio(padapter);
-}
-
 u8 rtw_hal_check_ips_status(struct adapter *padapter)
 {
 	return CheckIPSStatus(padapter);
@@ -160,7 +119,7 @@ s32	rtw_hal_mgnt_xmit(struct adapter *padapter, struct xmit_frame *pmgntframe)
 	/* pwlanhdr = (struct rtw_ieee80211_hdr *)pframe; */
 	/* memcpy(pmgntframe->attrib.ra, pwlanhdr->addr1, ETH_ALEN); */
 
-	if (padapter->securitypriv.binstallBIPkey == true) {
+	if (padapter->securitypriv.binstallBIPkey) {
 		if (is_multicast_ether_addr(pmgntframe->attrib.ra)) {
 			pmgntframe->attrib.encrypt = _BIP_;
 			/* pmgntframe->attrib.bswenc = true; */
@@ -206,7 +165,7 @@ void rtw_hal_update_ra_mask(struct sta_info *psta, u8 rssi_level)
 
 	pmlmepriv = &(padapter->mlmepriv);
 
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true)
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		add_ratid(padapter, psta, rssi_level);
 	else {
 		UpdateHalRAMask8723B(padapter, psta->mac_id, rssi_level);
diff --git a/drivers/staging/rtl8723bs/hal/hal_pwr_seq.c b/drivers/staging/rtl8723bs/hal/hal_pwr_seq.c
index 2438931ca51b2..0b60eba128afa 100644
--- a/drivers/staging/rtl8723bs/hal/hal_pwr_seq.c
+++ b/drivers/staging/rtl8723bs/hal/hal_pwr_seq.c
@@ -6,16 +6,14 @@
  ******************************************************************************/
 
 /*
-*
-This file includes all kinds of Power Action event for RTL8723B
-and corresponding hardware configurations which are released from HW SD.
-
-Major Change History:
-	When       Who               What
-	---------- ---------------   -------------------------------
-	2011-08-08 Roger            Create.
-
-*/
+ * This file includes all kinds of Power Action event for RTL8723B
+ * and corresponding hardware configurations which are released from HW SD.
+ *
+ * Major Change History:
+ *	When       Who               What
+ *	---------- ---------------   -------------------------------
+ *	2011-08-08 Roger            Create.
+ */
 
 #include "hal_pwr_seq.h"
 
diff --git a/drivers/staging/rtl8723bs/hal/odm.c b/drivers/staging/rtl8723bs/hal/odm.c
index 3dffa76207682..df394323d9019 100644
--- a/drivers/staging/rtl8723bs/hal/odm.c
+++ b/drivers/staging/rtl8723bs/hal/odm.c
@@ -279,22 +279,22 @@ static void odm_RefreshRateAdaptiveMaskCE(struct dm_odm_t *pDM_Odm)
 }
 
 /*-----------------------------------------------------------------------------
-* Function:	odm_RefreshRateAdaptiveMask()
-*
-* Overview:	Update rate table mask according to rssi
-*
-* Input:		NONE
-*
-* Output:		NONE
-*
-* Return:		NONE
-*
-* Revised History:
-*When		Who		Remark
-*05/27/2009	hpfan	Create Version 0.
-*
-* --------------------------------------------------------------------------
-*/
+ * Function:	odm_RefreshRateAdaptiveMask()
+ *
+ * Overview:	Update rate table mask according to rssi
+ *
+ * Input:		NONE
+ *
+ * Output:		NONE
+ *
+ * Return:		NONE
+ *
+ * Revised History:
+ *When		Who		Remark
+ *05/27/2009	hpfan	Create Version 0.
+ *
+ * --------------------------------------------------------------------------
+ */
 static void odm_RefreshRateAdaptiveMask(struct dm_odm_t *pDM_Odm)
 {
 
@@ -380,12 +380,9 @@ static void FindMinimumRSSI(struct adapter *padapter)
 
 	/* 1 1.Determine the minimum RSSI */
 
-	if (
-		(pDM_Odm->bLinked != true) &&
-		(pdmpriv->EntryMinUndecoratedSmoothedPWDB == 0)
-	) {
+	if (!pDM_Odm->bLinked && (pdmpriv->EntryMinUndecoratedSmoothedPWDB == 0))
 		pdmpriv->MinUndecoratedPWDBForDM = 0;
-	} else
+	else
 		pdmpriv->MinUndecoratedPWDBForDM = pdmpriv->EntryMinUndecoratedSmoothedPWDB;
 }
 
@@ -400,7 +397,7 @@ static void odm_RSSIMonitorCheckCE(struct dm_odm_t *pDM_Odm)
 	u32 PWDB_rssi[NUM_STA] = {0};/* 0~15]:MACID, [16~31]:PWDB_rssi */
 	struct ra_t *pRA_Table = &pDM_Odm->DM_RA_Table;
 
-	if (pDM_Odm->bLinked != true)
+	if (!pDM_Odm->bLinked)
 		return;
 
 	pRA_Table->firstconnect = pDM_Odm->bLinked;
@@ -431,7 +428,7 @@ static void odm_RSSIMonitorCheckCE(struct dm_odm_t *pDM_Odm)
 
 		for (i = 0; i < sta_cnt; i++) {
 			if (PWDB_rssi[i] != (0)) {
-				if (pHalData->fw_ractrl == true)/*  Report every sta's RSSI to FW */
+				if (pHalData->fw_ractrl)/*  Report every sta's RSSI to FW */
 					rtl8723b_set_rssi_cmd(Adapter, (u8 *)(&PWDB_rssi[i]));
 			}
 		}
@@ -554,7 +551,7 @@ void ODM_TXPowerTrackingCheck(struct dm_odm_t *pDM_Odm)
 		return;
 
 	if (!pDM_Odm->RFCalibrateInfo.TM_Trigger) { /* at least delay 1 sec */
-		PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_T_METER_NEW, (BIT17 | BIT16), 0x03);
+		PHY_SetRFReg(pDM_Odm->Adapter, RF_PATH_A, RF_T_METER_NEW, (BIT(17) | BIT(16)), 0x03);
 
 		pDM_Odm->RFCalibrateInfo.TM_Trigger = 1;
 		return;
@@ -623,7 +620,7 @@ void ODM_DMWatchdog(struct dm_odm_t *pDM_Odm)
 	}
 	odm_CCKPacketDetectionThresh(pDM_Odm);
 
-	if (*(pDM_Odm->pbPowerSaving) == true)
+	if (*pDM_Odm->pbPowerSaving)
 		return;
 
 
@@ -955,31 +952,6 @@ void ODM_CmnInfoUpdate(struct dm_odm_t *pDM_Odm, u32 CmnInfo, u64 Value)
 		pDM_Odm->bBtDisableEdcaTurbo = (bool)Value;
 		break;
 
-/*
-	case	ODM_CMNINFO_OP_MODE:
-		pDM_Odm->OPMode = (u8)Value;
-		break;
-
-	case	ODM_CMNINFO_WM_MODE:
-		pDM_Odm->WirelessMode = (u8)Value;
-		break;
-
-	case	ODM_CMNINFO_SEC_CHNL_OFFSET:
-		pDM_Odm->SecChOffset = (u8)Value;
-		break;
-
-	case	ODM_CMNINFO_SEC_MODE:
-		pDM_Odm->Security = (u8)Value;
-		break;
-
-	case	ODM_CMNINFO_BW:
-		pDM_Odm->BandWidth = (u8)Value;
-		break;
-
-	case	ODM_CMNINFO_CHNL:
-		pDM_Odm->Channel = (u8)Value;
-		break;
-*/
 	default:
 		/* do nothing */
 		break;
diff --git a/drivers/staging/rtl8723bs/hal/odm.h b/drivers/staging/rtl8723bs/hal/odm.h
index aa4e36b6f2b4d..ef23a2088733e 100644
--- a/drivers/staging/rtl8723bs/hal/odm.h
+++ b/drivers/staging/rtl8723bs/hal/odm.h
@@ -355,30 +355,30 @@ enum { /* _ODM_Support_Ability_Definition */
 	/*  */
 	/*  BB ODM section BIT 0-15 */
 	/*  */
-	ODM_BB_DIG			= BIT0,
-	ODM_BB_RA_MASK			= BIT1,
-	ODM_BB_DYNAMIC_TXPWR		= BIT2,
-	ODM_BB_FA_CNT			= BIT3,
-	ODM_BB_RSSI_MONITOR		= BIT4,
-	ODM_BB_CCK_PD			= BIT5,
-	ODM_BB_ANT_DIV			= BIT6,
-	ODM_BB_PWR_SAVE			= BIT7,
-	ODM_BB_PWR_TRAIN		= BIT8,
-	ODM_BB_RATE_ADAPTIVE		= BIT9,
-	ODM_BB_PATH_DIV			= BIT10,
-	ODM_BB_PSD			= BIT11,
-	ODM_BB_RXHP			= BIT12,
-	ODM_BB_ADAPTIVITY		= BIT13,
-	ODM_BB_CFO_TRACKING		= BIT14,
+	ODM_BB_DIG			= BIT(0),
+	ODM_BB_RA_MASK			= BIT(1),
+	ODM_BB_DYNAMIC_TXPWR		= BIT(2),
+	ODM_BB_FA_CNT			= BIT(3),
+	ODM_BB_RSSI_MONITOR		= BIT(4),
+	ODM_BB_CCK_PD			= BIT(5),
+	ODM_BB_ANT_DIV			= BIT(6),
+	ODM_BB_PWR_SAVE			= BIT(7),
+	ODM_BB_PWR_TRAIN		= BIT(8),
+	ODM_BB_RATE_ADAPTIVE		= BIT(9),
+	ODM_BB_PATH_DIV			= BIT(10),
+	ODM_BB_PSD			= BIT(11),
+	ODM_BB_RXHP			= BIT(12),
+	ODM_BB_ADAPTIVITY		= BIT(13),
+	ODM_BB_CFO_TRACKING		= BIT(14),
 
 	/*  MAC DM section BIT 16-23 */
-	ODM_MAC_EDCA_TURBO		= BIT16,
-	ODM_MAC_EARLY_MODE		= BIT17,
+	ODM_MAC_EDCA_TURBO		= BIT(16),
+	ODM_MAC_EARLY_MODE		= BIT(17),
 
 	/*  RF ODM section BIT 24-31 */
-	ODM_RF_TX_PWR_TRACK		= BIT24,
-	ODM_RF_RX_GAIN_TRACK	= BIT25,
-	ODM_RF_CALIBRATION		= BIT26,
+	ODM_RF_TX_PWR_TRACK		= BIT(24),
+	ODM_RF_RX_GAIN_TRACK	= BIT(25),
+	ODM_RF_CALIBRATION		= BIT(26),
 };
 
 /* 	ODM_CMNINFO_INTERFACE */
@@ -389,7 +389,7 @@ enum { /* tag_ODM_Support_Interface_Definition */
 
 /*  ODM_CMNINFO_IC_TYPE */
 enum { /* tag_ODM_Support_IC_Type_Definition */
-	ODM_RTL8723B	=	BIT8,
+	ODM_RTL8723B	=	BIT(8),
 };
 
 /* ODM_CMNINFO_CUT_VER */
@@ -434,10 +434,10 @@ enum { /* tag_ODM_RF_Type_Definition */
 /*  ODM_CMNINFO_WM_MODE */
 enum { /* tag_Wireless_Mode_Definition */
 	ODM_WM_UNKNOWN    = 0x0,
-	ODM_WM_B          = BIT0,
-	ODM_WM_G          = BIT1,
-	ODM_WM_N24G       = BIT3,
-	ODM_WM_AUTO       = BIT5,
+	ODM_WM_B          = BIT(0),
+	ODM_WM_G          = BIT(1),
+	ODM_WM_N24G       = BIT(3),
+	ODM_WM_AUTO       = BIT(5),
 };
 
 /*  ODM_CMNINFO_BW */
@@ -975,10 +975,6 @@ enum ODM_FW_Config_Type {
 	CONFIG_FW_BT,
 };
 
-#ifdef REMOVE_PACK
-#pragma pack()
-#endif
-
 /* include "odm_function.h" */
 
 /* 3 =========================================================== */
diff --git a/drivers/staging/rtl8723bs/hal/odm_CfoTracking.c b/drivers/staging/rtl8723bs/hal/odm_CfoTracking.c
index 0eaedd8f64697..7e6b49399abbf 100644
--- a/drivers/staging/rtl8723bs/hal/odm_CfoTracking.c
+++ b/drivers/staging/rtl8723bs/hal/odm_CfoTracking.c
@@ -138,7 +138,7 @@ void ODM_CfoTracking(void *pDM_VOID)
 		pCfoTrack->CFO_ave_pre = CFO_ave;
 
 		/* 4 1.4 Dynamic Xtal threshold */
-		if (pCfoTrack->bAdjust == false) {
+		if (!pCfoTrack->bAdjust) {
 			if (CFO_ave > CFO_TH_XTAL_HIGH || CFO_ave < (-CFO_TH_XTAL_HIGH))
 				pCfoTrack->bAdjust = true;
 		} else {
diff --git a/drivers/staging/rtl8723bs/hal/odm_DIG.c b/drivers/staging/rtl8723bs/hal/odm_DIG.c
index a31c8368f9d99..ef3c2db8553ea 100644
--- a/drivers/staging/rtl8723bs/hal/odm_DIG.c
+++ b/drivers/staging/rtl8723bs/hal/odm_DIG.c
@@ -19,8 +19,8 @@ void odm_NHMCounterStatisticsInit(void *pDM_VOID)
 	rtw_write32(pDM_Odm->Adapter, ODM_REG_NHM_TH3_TO_TH0_11N, 0xffffff52);	/* 0x898 = 0xffffff52		th_3, th_2, th_1, th_0 */
 	rtw_write32(pDM_Odm->Adapter, ODM_REG_NHM_TH7_TO_TH4_11N, 0xffffffff);	/* 0x89c = 0xffffffff		th_7, th_6, th_5, th_4 */
 	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_FPGA0_IQK_11N, bMaskByte0, 0xff);		/* 0xe28[7:0]= 0xff		th_8 */
-	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_NHM_TH9_TH10_11N, BIT10|BIT9|BIT8, 0x7);	/* 0x890[9:8]=3			enable CCX */
-	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_OFDM_FA_RSTC_11N, BIT7, 0x1);		/* 0xc0c[7]= 1			max power among all RX ants */
+	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_NHM_TH9_TH10_11N, BIT(10)|BIT(9)|BIT(8), 0x7);	/* 0x890[9:8]=3			enable CCX */
+	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_OFDM_FA_RSTC_11N, BIT(7), 0x1);		/* 0xc0c[7]= 1			max power among all RX ants */
 }
 
 void odm_NHMCounterStatistics(void *pDM_VOID)
@@ -48,8 +48,8 @@ void odm_NHMCounterStatisticsReset(void *pDM_VOID)
 {
 	struct dm_odm_t *pDM_Odm = (struct dm_odm_t *)pDM_VOID;
 
-	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_NHM_TH9_TH10_11N, BIT1, 0);
-	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_NHM_TH9_TH10_11N, BIT1, 1);
+	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_NHM_TH9_TH10_11N, BIT(1), 0);
+	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_NHM_TH9_TH10_11N, BIT(1), 1);
 }
 
 void odm_NHMBBInit(void *pDM_VOID)
@@ -82,7 +82,7 @@ void odm_NHMBB(void *pDM_VOID)
 
 
 	if ((pDM_Odm->NHMCurTxOkcnt) + 1 > (u64)(pDM_Odm->NHMCurRxOkcnt<<2) + 1) { /* Tx > 4*Rx possible for adaptivity test */
-		if (pDM_Odm->NHM_cnt_0 >= 190 || pDM_Odm->adaptivity_flag == true) {
+		if (pDM_Odm->NHM_cnt_0 >= 190 || pDM_Odm->adaptivity_flag) {
 			/* Enable EDCCA since it is possible running Adaptivity testing */
 			/* test_status = 1; */
 			pDM_Odm->adaptivity_flag = true;
@@ -99,7 +99,7 @@ void odm_NHMBB(void *pDM_VOID)
 			}
 		}
 	} else { /*  TX<RX */
-		if (pDM_Odm->adaptivity_flag == true && pDM_Odm->NHM_cnt_0 <= 200) {
+		if (pDM_Odm->adaptivity_flag && pDM_Odm->NHM_cnt_0 <= 200) {
 			/* test_status = 2; */
 			pDM_Odm->tolerance_cnt = 0;
 		} else {
@@ -143,9 +143,9 @@ void odm_SearchPwdBLowerBound(void *pDM_VOID, u8 IGI_target)
 		for (cnt = 0; cnt < 20; cnt++) {
 			value32 = PHY_QueryBBReg(pDM_Odm->Adapter, ODM_REG_RPT_11N, bMaskDWord);
 
-			if (value32 & BIT30)
+			if (value32 & BIT(30))
 				pDM_Odm->txEdcca1 = pDM_Odm->txEdcca1 + 1;
-			else if (value32 & BIT29)
+			else if (value32 & BIT(29))
 				pDM_Odm->txEdcca1 = pDM_Odm->txEdcca1 + 1;
 			else
 				pDM_Odm->txEdcca0 = pDM_Odm->txEdcca0 + 1;
@@ -189,7 +189,7 @@ void odm_AdaptivityInit(void *pDM_VOID)
 {
 	struct dm_odm_t *pDM_Odm = (struct dm_odm_t *)pDM_VOID;
 
-	if (pDM_Odm->Carrier_Sense_enable == false)
+	if (!pDM_Odm->Carrier_Sense_enable)
 		pDM_Odm->TH_L2H_ini = 0xf7; /*  -7 */
 	else
 		pDM_Odm->TH_L2H_ini = 0xa;
@@ -209,7 +209,7 @@ void odm_AdaptivityInit(void *pDM_VOID)
 	pDM_Odm->Adaptivity_IGI_upper = 0;
 	odm_NHMBBInit(pDM_Odm);
 
-	PHY_SetBBReg(pDM_Odm->Adapter, REG_RD_CTRL, BIT11, 1); /*  stop counting if EDCCA is asserted */
+	PHY_SetBBReg(pDM_Odm->Adapter, REG_RD_CTRL, BIT(11), 1); /*  stop counting if EDCCA is asserted */
 }
 
 
@@ -232,7 +232,7 @@ void odm_Adaptivity(void *pDM_VOID, u8 IGI)
 	pDM_Odm->IGI_target = (u8) IGI_target;
 
 	/* Search pwdB lower bound */
-	if (pDM_Odm->TxHangFlg == true) {
+	if (pDM_Odm->TxHangFlg) {
 		PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_DBG_RPT_11N, bMaskDWord, 0x208);
 		odm_SearchPwdBLowerBound(pDM_Odm, pDM_Odm->IGI_target);
 	}
@@ -251,12 +251,10 @@ void odm_Adaptivity(void *pDM_VOID, u8 IGI)
 	} else
 		EDCCA_State = true;
 
-	if (
-		pDM_Odm->bLinked &&
-		pDM_Odm->Carrier_Sense_enable == false &&
-		pDM_Odm->NHM_disable == false &&
-		pDM_Odm->TxHangFlg == false
-	)
+	if (pDM_Odm->bLinked &&
+	    !pDM_Odm->Carrier_Sense_enable &&
+	    !pDM_Odm->NHM_disable &&
+	    !pDM_Odm->TxHangFlg)
 		odm_NHMBB(pDM_Odm);
 
 	if (EDCCA_State) {
@@ -323,7 +321,7 @@ bool odm_DigAbort(void *pDM_VOID)
 		return	true;
 
 	/* add by Neil Chen to avoid PSD is processing */
-	if (pDM_Odm->bDMInitialGainEnable == false)
+	if (!pDM_Odm->bDMInitialGainEnable)
 		return	true;
 
 	return	false;
@@ -387,14 +385,14 @@ void odm_DIG(void *pDM_VOID)
 	if (odm_DigAbort(pDM_Odm))
 		return;
 
-	if (pDM_Odm->adaptivity_flag == true)
+	if (pDM_Odm->adaptivity_flag)
 		Adap_IGI_Upper = pDM_Odm->Adaptivity_IGI_upper;
 
 
 	/* 1 Update status */
 	DIG_Dynamic_MIN = pDM_DigTable->DIG_Dynamic_MIN_0;
-	FirstConnect = (pDM_Odm->bLinked) && (pDM_DigTable->bMediaConnect_0 == false);
-	FirstDisConnect = (!pDM_Odm->bLinked) && (pDM_DigTable->bMediaConnect_0 == true);
+	FirstConnect = (pDM_Odm->bLinked) && !pDM_DigTable->bMediaConnect_0;
+	FirstDisConnect = (!pDM_Odm->bLinked) && pDM_DigTable->bMediaConnect_0;
 
 	/* 1 Boundary Decision */
 	/* 2 For WIN\CE */
@@ -527,10 +525,8 @@ void odm_DIG(void *pDM_VOID)
 		CurrentIGI = pDM_DigTable->rx_gain_range_max;
 
 	/* 1 Force upper bound and lower bound for adaptivity */
-	if (
-		pDM_Odm->SupportAbility & ODM_BB_ADAPTIVITY &&
-		pDM_Odm->adaptivity_flag == true
-	) {
+	if (pDM_Odm->SupportAbility & ODM_BB_ADAPTIVITY &&
+	    pDM_Odm->adaptivity_flag) {
 		if (CurrentIGI > Adap_IGI_Upper)
 			CurrentIGI = Adap_IGI_Upper;
 
@@ -616,9 +612,9 @@ void odm_FalseAlarmCounterStatistics(void *pDM_VOID)
 
 	/* hold ofdm counter */
 	/* hold page C counter */
-	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_OFDM_FA_HOLDC_11N, BIT31, 1);
+	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_OFDM_FA_HOLDC_11N, BIT(31), 1);
 	/* hold page D counter */
-	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_OFDM_FA_RSTD_11N, BIT31, 1);
+	PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_OFDM_FA_RSTD_11N, BIT(31), 1);
 
 	ret_value = PHY_QueryBBReg(
 		pDM_Odm->Adapter, ODM_REG_OFDM_FA_TYPE1_11N, bMaskDWord
@@ -653,8 +649,8 @@ void odm_FalseAlarmCounterStatistics(void *pDM_VOID)
 
 	{
 		/* hold cck counter */
-		PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_CCK_FA_RST_11N, BIT12, 1);
-		PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_CCK_FA_RST_11N, BIT14, 1);
+		PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_CCK_FA_RST_11N, BIT(12), 1);
+		PHY_SetBBReg(pDM_Odm->Adapter, ODM_REG_CCK_FA_RST_11N, BIT(14), 1);
 
 		ret_value = PHY_QueryBBReg(
 			pDM_Odm->Adapter, ODM_REG_CCK_FA_LSB_11N, bMaskByte0
diff --git a/drivers/staging/rtl8723bs/hal/odm_DIG.h b/drivers/staging/rtl8723bs/hal/odm_DIG.h
index a5b041101c895..032c4abae49be 100644
--- a/drivers/staging/rtl8723bs/hal/odm_DIG.h
+++ b/drivers/staging/rtl8723bs/hal/odm_DIG.h
@@ -78,8 +78,8 @@ struct  false_ALARM_STATISTICS {
 };
 
 enum ODM_Pause_DIG_TYPE {
-	ODM_PAUSE_DIG = BIT0,
-	ODM_RESUME_DIG = BIT1
+	ODM_PAUSE_DIG = BIT(0),
+	ODM_RESUME_DIG = BIT(1)
 };
 
 #define		DM_DIG_THRESH_HIGH			40
diff --git a/drivers/staging/rtl8723bs/hal/odm_DynamicBBPowerSaving.c b/drivers/staging/rtl8723bs/hal/odm_DynamicBBPowerSaving.c
index 57c5736527d2f..ba6357c6a7ede 100644
--- a/drivers/staging/rtl8723bs/hal/odm_DynamicBBPowerSaving.c
+++ b/drivers/staging/rtl8723bs/hal/odm_DynamicBBPowerSaving.c
@@ -35,7 +35,7 @@ void ODM_RF_Saving(void *pDM_VOID, u8 bForceInNormal)
 	if (pDM_PSTable->initialize == 0) {
 
 		pDM_PSTable->Reg874 = (PHY_QueryBBReg(pDM_Odm->Adapter, 0x874, bMaskDWord)&0x1CC000)>>14;
-		pDM_PSTable->RegC70 = (PHY_QueryBBReg(pDM_Odm->Adapter, 0xc70, bMaskDWord)&BIT3)>>3;
+		pDM_PSTable->RegC70 = (PHY_QueryBBReg(pDM_Odm->Adapter, 0xc70, bMaskDWord)&BIT(3))>>3;
 		pDM_PSTable->Reg85C = (PHY_QueryBBReg(pDM_Odm->Adapter, 0x85c, bMaskDWord)&0xFF000000)>>24;
 		pDM_PSTable->RegA74 = (PHY_QueryBBReg(pDM_Odm->Adapter, 0xa74, bMaskDWord)&0xF000)>>12;
 		/* Reg818 = PHY_QueryBBReg(padapter, 0x818, bMaskDWord); */
@@ -63,18 +63,18 @@ void ODM_RF_Saving(void *pDM_VOID, u8 bForceInNormal)
 	if (pDM_PSTable->PreRFState != pDM_PSTable->CurRFState) {
 		if (pDM_PSTable->CurRFState == RF_Save) {
 			PHY_SetBBReg(pDM_Odm->Adapter, 0x874, 0x1C0000, 0x2); /* Reg874[20:18]=3'b010 */
-			PHY_SetBBReg(pDM_Odm->Adapter, 0xc70, BIT3, 0); /* RegC70[3]= 1'b0 */
+			PHY_SetBBReg(pDM_Odm->Adapter, 0xc70, BIT(3), 0); /* RegC70[3]= 1'b0 */
 			PHY_SetBBReg(pDM_Odm->Adapter, 0x85c, 0xFF000000, 0x63); /* Reg85C[31:24]= 0x63 */
 			PHY_SetBBReg(pDM_Odm->Adapter, 0x874, 0xC000, 0x2); /* Reg874[15:14]=2'b10 */
 			PHY_SetBBReg(pDM_Odm->Adapter, 0xa74, 0xF000, 0x3); /* RegA75[7:4]= 0x3 */
-			PHY_SetBBReg(pDM_Odm->Adapter, 0x818, BIT28, 0x0); /* Reg818[28]= 1'b0 */
-			PHY_SetBBReg(pDM_Odm->Adapter, 0x818, BIT28, 0x1); /* Reg818[28]= 1'b1 */
+			PHY_SetBBReg(pDM_Odm->Adapter, 0x818, BIT(28), 0x0); /* Reg818[28]= 1'b0 */
+			PHY_SetBBReg(pDM_Odm->Adapter, 0x818, BIT(28), 0x1); /* Reg818[28]= 1'b1 */
 		} else {
 			PHY_SetBBReg(pDM_Odm->Adapter, 0x874, 0x1CC000, pDM_PSTable->Reg874);
-			PHY_SetBBReg(pDM_Odm->Adapter, 0xc70, BIT3, pDM_PSTable->RegC70);
+			PHY_SetBBReg(pDM_Odm->Adapter, 0xc70, BIT(3), pDM_PSTable->RegC70);
 			PHY_SetBBReg(pDM_Odm->Adapter, 0x85c, 0xFF000000, pDM_PSTable->Reg85C);
 			PHY_SetBBReg(pDM_Odm->Adapter, 0xa74, 0xF000, pDM_PSTable->RegA74);
-			PHY_SetBBReg(pDM_Odm->Adapter, 0x818, BIT28, 0x0);
+			PHY_SetBBReg(pDM_Odm->Adapter, 0x818, BIT(28), 0x0);
 		}
 		pDM_PSTable->PreRFState = pDM_PSTable->CurRFState;
 	}
diff --git a/drivers/staging/rtl8723bs/hal/odm_HWConfig.c b/drivers/staging/rtl8723bs/hal/odm_HWConfig.c
index 994b8c578e7ab..4784d33eab0e9 100644
--- a/drivers/staging/rtl8723bs/hal/odm_HWConfig.c
+++ b/drivers/staging/rtl8723bs/hal/odm_HWConfig.c
@@ -335,7 +335,7 @@ static void odm_Process_RSSIForDM(
 				}
 			}
 
-			pEntry->rssi_stat.PacketMap = (pEntry->rssi_stat.PacketMap<<1) | BIT0;
+			pEntry->rssi_stat.PacketMap = (pEntry->rssi_stat.PacketMap<<1) | BIT(0);
 
 		} else {
 			RSSI_Ave = pPhyInfo->rx_pwd_ba11;
@@ -369,7 +369,7 @@ static void odm_Process_RSSIForDM(
 				pEntry->rssi_stat.ValidBit++;
 
 			for (i = 0; i < pEntry->rssi_stat.ValidBit; i++)
-				OFDM_pkt += (u8)(pEntry->rssi_stat.PacketMap>>i)&BIT0;
+				OFDM_pkt += (u8)(pEntry->rssi_stat.PacketMap>>i)&BIT(0);
 
 			if (pEntry->rssi_stat.ValidBit == 64) {
 				Weighting = ((OFDM_pkt<<4) > 64)?64:(OFDM_pkt<<4);
diff --git a/drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h b/drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h
index 3c71938899e06..6d7062a1cbbe0 100644
--- a/drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h
+++ b/drivers/staging/rtl8723bs/hal/odm_RegDefine11N.h
@@ -155,8 +155,8 @@
 
 /* DIG Related */
 #define	ODM_BIT_IGI_11N					0x0000007F
-#define	ODM_BIT_CCK_RPT_FORMAT_11N		BIT9
+#define	ODM_BIT_CCK_RPT_FORMAT_11N		BIT(9)
 #define	ODM_BIT_BB_RX_PATH_11N			0xF
-#define	ODM_BIT_BB_ATC_11N				BIT11
+#define	ODM_BIT_BB_ATC_11N				BIT(11)
 
 #endif
diff --git a/drivers/staging/rtl8723bs/hal/odm_interface.h b/drivers/staging/rtl8723bs/hal/odm_interface.h
index 400473ca58ca3..c3b13f5f37db5 100644
--- a/drivers/staging/rtl8723bs/hal/odm_interface.h
+++ b/drivers/staging/rtl8723bs/hal/odm_interface.h
@@ -18,14 +18,6 @@
 #define _bit_all(_name)			BIT_##_name
 #define _bit_ic(_name, _ic)		BIT_##_name##_ic
 
-/*===================================
-
-#define ODM_REG_DIG_11N		0xC50
-#define ODM_REG_DIG_11AC	0xDDD
-
-ODM_REG(DIG)
-=====================================*/
-
 #define _reg_11N(_name)			ODM_REG_##_name##_11N
 #define _bit_11N(_name)			ODM_BIT_##_name##_11N
 
diff --git a/drivers/staging/rtl8723bs/hal/odm_types.h b/drivers/staging/rtl8723bs/hal/odm_types.h
index 0da662e1493a6..bc0c4d027492e 100644
--- a/drivers/staging/rtl8723bs/hal/odm_types.h
+++ b/drivers/staging/rtl8723bs/hal/odm_types.h
@@ -18,12 +18,6 @@
 enum hal_status {
 	HAL_STATUS_SUCCESS,
 	HAL_STATUS_FAILURE,
-	/*RT_STATUS_PENDING,
-	RT_STATUS_RESOURCE,
-	RT_STATUS_INVALID_CONTEXT,
-	RT_STATUS_INVALID_PARAMETER,
-	RT_STATUS_NOT_SUPPORT,
-	RT_STATUS_OS_API_FAILED,*/
 };
 
 
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c b/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c
index 12416e499ac35..2485c415f14b9 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_cmd.c
@@ -8,6 +8,7 @@
 #include <drv_types.h>
 #include <rtl8723b_hal.h>
 #include <linux/etherdevice.h>
+#include <linux/iopoll.h>
 #include "hal_com_h2c.h"
 
 #define MAX_H2C_BOX_NUMS	4
@@ -18,31 +19,26 @@
 
 static u8 _is_fw_read_cmd_down(struct adapter *padapter, u8 msgbox_num)
 {
-	u8 read_down = false;
-	int retry_cnts = 100;
-
 	u8 valid;
+	int ret;
 
-	do {
-		valid = rtw_read8(padapter, REG_HMETFR) & BIT(msgbox_num);
-		if (0 == valid) {
-			read_down = true;
-		}
-	} while ((!read_down) && (retry_cnts--));
-
-	return read_down;
+	ret = read_poll_timeout_atomic(rtw_read8,
+				       valid, !(valid & BIT(msgbox_num)),
+				       0, 500, false,
+				       padapter, REG_HMETFR);
 
+	return !ret;
 }
 
 
 /*****************************************
-* H2C Msg format :
-*| 31 - 8		|7-5	| 4 - 0	|
-*| h2c_msg	|Class	|CMD_ID	|
-*| 31-0						|
-*| Ext msg					|
-*
-******************************************/
+ * H2C Msg format :
+ *| 31 - 8		|7-5	| 4 - 0	|
+ *| h2c_msg	|Class	|CMD_ID	|
+ *| 31-0						|
+ *| Ext msg					|
+ *
+ ******************************************/
 s32 FillH2CCmd8723B(struct adapter *padapter, u8 ElementID, u32 CmdLen, u8 *pCmdBuffer)
 {
 	u8 h2c_box_num;
@@ -363,7 +359,7 @@ void rtl8723b_set_FwPwrMode_cmd(struct adapter *padapter, u8 psmode)
 	}
 
 	if (psmode > 0) {
-		if (hal_btcoex_IsBtControlLps(padapter) == true) {
+		if (hal_btcoex_IsBtControlLps(padapter)) {
 			PowerState = hal_btcoex_RpwmVal(padapter);
 			byte5 = hal_btcoex_LpsVal(padapter);
 
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_dm.c b/drivers/staging/rtl8723bs/hal/rtl8723b_dm.c
index 928226679ab45..6123510103599 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_dm.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_dm.c
@@ -131,21 +131,19 @@ void rtl8723b_HalDmWatchDog(struct adapter *Adapter)
 
 	hw_init_completed = Adapter->hw_init_completed;
 
-	if (hw_init_completed == false)
+	if (!hw_init_completed)
 		goto skip_dm;
 
 	fw_current_in_ps_mode = adapter_to_pwrctl(Adapter)->fw_current_in_ps_mode;
 	rtw_hal_get_hwreg(Adapter, HW_VAR_FWLPS_RF_ON, (u8 *)(&bFwPSAwake));
 
-	if (
-		(hw_init_completed == true) &&
-		((!fw_current_in_ps_mode) && bFwPSAwake)
-	) {
+	if (hw_init_completed &&
+	    (!fw_current_in_ps_mode && bFwPSAwake)) {
 		rtw_hal_check_rxfifo_full(Adapter);
 	}
 
 	/* ODM */
-	if (hw_init_completed == true) {
+	if (hw_init_completed) {
 		u8 bLinked = false;
 		u8 bsta_state = false;
 		bool bBtDisabled = true;
@@ -207,7 +205,7 @@ void rtl8723b_HalDmWatchDog_in_LPS(struct adapter *Adapter)
 	struct sta_priv *pstapriv = &Adapter->stapriv;
 	struct sta_info *psta = NULL;
 
-	if (Adapter->hw_init_completed == false)
+	if (!Adapter->hw_init_completed)
 		goto skip_lps_dm;
 
 
@@ -216,7 +214,7 @@ void rtl8723b_HalDmWatchDog_in_LPS(struct adapter *Adapter)
 
 	ODM_CmnInfoUpdate(&pHalData->odmpriv, ODM_CMNINFO_LINK, bLinked);
 
-	if (bLinked == false)
+	if (!bLinked)
 		goto skip_lps_dm;
 
 	if (!(pDM_Odm->SupportAbility & ODM_BB_RSSI_MONITOR))
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
index e794fe3caf9d6..2a748367fbbae 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_hal_init.c
@@ -68,8 +68,6 @@ static int _BlockWrite(struct adapter *padapter, void *buffer, u32 buffSize)
 
 	/* 3 Phase #2 */
 	if (remainSize_p1) {
-		offset = blockCount_p1 * blockSize_p1;
-
 		blockCount_p2 = remainSize_p1/blockSize_p2;
 		remainSize_p2 = remainSize_p1%blockSize_p2;
 	}
@@ -269,7 +267,7 @@ void rtl8723b_FirmwareSelfReset(struct adapter *padapter)
 		rtw_write8(padapter, REG_HMETFR+3, 0x20);
 
 		val = rtw_read8(padapter, REG_SYS_FUNC_EN + 1);
-		while (val & BIT2) {
+		while (val & BIT(2)) {
 			Delay--;
 			if (Delay == 0)
 				break;
@@ -280,7 +278,7 @@ void rtl8723b_FirmwareSelfReset(struct adapter *padapter)
 		if (Delay == 0) {
 			/* force firmware reset */
 			val = rtw_read8(padapter, REG_SYS_FUNC_EN + 1);
-			rtw_write8(padapter, REG_SYS_FUNC_EN + 1, val & (~BIT2));
+			rtw_write8(padapter, REG_SYS_FUNC_EN + 1, val & (~BIT(2)));
 		}
 	}
 }
@@ -405,11 +403,11 @@ s32 rtl8723b_FirmwareDownload(struct adapter *padapter, bool  bUsedWoWLANFw)
 			break;
 	}
 	_FWDownloadEnable(padapter, false);
-	if (_SUCCESS != rtStatus)
+	if (rtStatus != _SUCCESS)
 		goto fwdl_stat;
 
 	rtStatus = _FWFreeToGo(padapter, 10, 200);
-	if (_SUCCESS != rtStatus)
+	if (rtStatus != _SUCCESS)
 		goto fwdl_stat;
 
 fwdl_stat:
@@ -651,7 +649,7 @@ static void hal_ReadEFuse_WiFi(
 	hal_EfuseSwitchToBank(padapter, 0);
 
 	while (AVAILABLE_EFUSE_ADDR(eFuse_Addr)) {
-		efuse_OneByteRead(padapter, eFuse_Addr++, &efuseHeader);
+		rtw_efuse_one_byte_read(padapter, eFuse_Addr++, &efuseHeader);
 		if (efuseHeader == 0xFF)
 			break;
 
@@ -659,7 +657,7 @@ static void hal_ReadEFuse_WiFi(
 		if (EXT_HEADER(efuseHeader)) { /* extended header */
 			offset = GET_HDR_OFFSET_2_0(efuseHeader);
 
-			efuse_OneByteRead(padapter, eFuse_Addr++, &efuseExtHdr);
+			rtw_efuse_one_byte_read(padapter, eFuse_Addr++, &efuseExtHdr);
 			if (ALL_WORDS_DISABLED(efuseExtHdr))
 				continue;
 
@@ -678,16 +676,18 @@ static void hal_ReadEFuse_WiFi(
 			for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) {
 				/*  Check word enable condition in the section */
 				if (!(wden & (0x01<<i))) {
-					efuse_OneByteRead(padapter, eFuse_Addr++, &efuseData);
+					rtw_efuse_one_byte_read(padapter, eFuse_Addr++,
+								&efuseData);
 					efuseTbl[addr] = efuseData;
 
-					efuse_OneByteRead(padapter, eFuse_Addr++, &efuseData);
+					rtw_efuse_one_byte_read(padapter, eFuse_Addr++,
+								&efuseData);
 					efuseTbl[addr+1] = efuseData;
 				}
 				addr += 2;
 			}
 		} else {
-			eFuse_Addr += Efuse_CalculateWordCnts(wden)*2;
+			eFuse_Addr += rtw_efuse_calculate_word_counts(wden)*2;
 		}
 	}
 
@@ -738,13 +738,13 @@ static void hal_ReadEFuse_BT(
 	Hal_GetEfuseDefinition(padapter, EFUSE_BT, TYPE_AVAILABLE_EFUSE_BYTES_BANK, &total);
 
 	for (bank = 1; bank < 3; bank++) { /*  8723b Max bake 0~2 */
-		if (hal_EfuseSwitchToBank(padapter, bank) == false)
+		if (!hal_EfuseSwitchToBank(padapter, bank))
 			goto exit;
 
 		eFuse_Addr = 0;
 
 		while (AVAILABLE_EFUSE_ADDR(eFuse_Addr)) {
-			efuse_OneByteRead(padapter, eFuse_Addr++, &efuseHeader);
+			rtw_efuse_one_byte_read(padapter, eFuse_Addr++, &efuseHeader);
 			if (efuseHeader == 0xFF)
 				break;
 
@@ -752,7 +752,7 @@ static void hal_ReadEFuse_BT(
 			if (EXT_HEADER(efuseHeader)) { /* extended header */
 				offset = GET_HDR_OFFSET_2_0(efuseHeader);
 
-				efuse_OneByteRead(padapter, eFuse_Addr++, &efuseExtHdr);
+				rtw_efuse_one_byte_read(padapter, eFuse_Addr++, &efuseExtHdr);
 				if (ALL_WORDS_DISABLED(efuseExtHdr))
 					continue;
 
@@ -770,16 +770,18 @@ static void hal_ReadEFuse_BT(
 				for (i = 0; i < EFUSE_MAX_WORD_UNIT; i++) {
 					/*  Check word enable condition in the section */
 					if (!(wden & (0x01<<i))) {
-						efuse_OneByteRead(padapter, eFuse_Addr++, &efuseData);
+						rtw_efuse_one_byte_read(padapter, eFuse_Addr++,
+									&efuseData);
 						efuseTbl[addr] = efuseData;
 
-						efuse_OneByteRead(padapter, eFuse_Addr++, &efuseData);
+						rtw_efuse_one_byte_read(padapter, eFuse_Addr++,
+									&efuseData);
 						efuseTbl[addr+1] = efuseData;
 					}
 					addr += 2;
 				}
 			} else {
-				eFuse_Addr += Efuse_CalculateWordCnts(wden)*2;
+				eFuse_Addr += rtw_efuse_calculate_word_counts(wden)*2;
 			}
 		}
 
@@ -823,27 +825,20 @@ void Hal_ReadEFuse(
 		hal_ReadEFuse_BT(padapter, _offset, _size_byte, pbuf);
 }
 
-static struct hal_version ReadChipVersion8723B(struct adapter *padapter)
+static void ReadChipVersion8723B(struct adapter *padapter)
 {
 	u32 value32;
-	struct hal_version ChipVersion;
 	struct hal_com_data *pHalData;
 
 /* YJ, TODO, move read chip type here */
 	pHalData = GET_HAL_DATA(padapter);
 
 	value32 = rtw_read32(padapter, REG_SYS_CFG);
-	ChipVersion.ICType = CHIP_8723B;
-	ChipVersion.ChipType = ((value32 & RTL_ID) ? TEST_CHIP : NORMAL_CHIP);
-	ChipVersion.VendorType = ((value32 & VENDOR_ID) ? CHIP_VENDOR_UMC : CHIP_VENDOR_TSMC);
-	ChipVersion.CUTVersion = (value32 & CHIP_VER_RTL_MASK)>>CHIP_VER_RTL_SHIFT; /*  IC version (CUT) */
+	pHalData->chip_normal = ((value32 & RTL_ID) ? false : true);
 
 	/*  For regulator mode. by tynli. 2011.01.14 */
 	pHalData->RegulatorMode = ((value32 & SPS_SEL) ? RT_LDO_REGULATOR : RT_SWITCHING_REGULATOR);
 
-	value32 = rtw_read32(padapter, REG_GPIO_OUTSTS);
-	ChipVersion.ROMVer = ((value32 & RF_RL_ID) >> 20);	/*  ROM code version. */
-
 	/*  For multi-function consideration. Added by Roger, 2010.10.06. */
 	pHalData->MultiFunc = RT_MULTI_FUNC_NONE;
 	value32 = rtw_read32(padapter, REG_MULTI_FUNC_CTRL);
@@ -851,12 +846,6 @@ static struct hal_version ReadChipVersion8723B(struct adapter *padapter)
 	pHalData->MultiFunc |= ((value32 & BT_FUNC_EN) ? RT_MULTI_FUNC_BT : 0);
 	pHalData->MultiFunc |= ((value32 & GPS_FUNC_EN) ? RT_MULTI_FUNC_GPS : 0);
 	pHalData->PolarityCtl = ((value32 & WL_HWPDN_SL) ? RT_POLARITY_HIGH_ACT : RT_POLARITY_LOW_ACT);
-
-	dump_chip_info(ChipVersion);
-
-	pHalData->VersionID = ChipVersion;
-
-	return ChipVersion;
 }
 
 void rtl8723b_read_chip_version(struct adapter *padapter)
@@ -882,7 +871,7 @@ void rtl8723b_InitBeaconParameters(struct adapter *padapter)
 	rtw_write16(padapter, REG_TBTT_PROHIBIT, 0x6404);/*  ms */
 	/*  Firmware will control REG_DRVERLYINT when power saving is enable, */
 	/*  so don't set this register on STA mode. */
-	if (check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE) == false)
+	if (!check_fwstate(&padapter->mlmepriv, WIFI_STATION_STATE))
 		rtw_write8(padapter, REG_DRVERLYINT, DRIVER_EARLY_INT_TIME_8723B); /*  5ms */
 	rtw_write8(padapter, REG_BCNDMATIM, BCN_DMA_ATIME_INT_TIME_8723B); /*  2ms */
 
@@ -912,7 +901,7 @@ void _InitBurstPktLen_8723BS(struct adapter *Adapter)
 
 	/*  ARFB table 9 for 11ac 5G 2SS */
 	rtw_write32(Adapter, REG_ARFR0_8723B, 0x00000010);
-	if (IS_NORMAL_CHIP(pHalData->VersionID))
+	if (pHalData->chip_normal)
 		rtw_write32(Adapter, REG_ARFR0_8723B+4, 0xfffff000);
 	else
 		rtw_write32(Adapter, REG_ARFR0_8723B+4, 0x3e0ff000);
@@ -998,7 +987,8 @@ void rtl8723b_SetBeaconRelatedRegisters(struct adapter *padapter)
 	rtw_write32(padapter, REG_TCR, value32);
 
 	/*  NOTE: Fix test chip's bug (about contention windows's randomness) */
-	if (check_fwstate(&padapter->mlmepriv, WIFI_ADHOC_STATE|WIFI_ADHOC_MASTER_STATE|WIFI_AP_STATE) == true) {
+	if (check_fwstate(&padapter->mlmepriv, WIFI_ADHOC_STATE |
+					       WIFI_ADHOC_MASTER_STATE | WIFI_AP_STATE)) {
 		rtw_write8(padapter, REG_RXTSF_OFFSET_CCK, 0x50);
 		rtw_write8(padapter, REG_RXTSF_OFFSET_OFDM, 0x50);
 	}
@@ -1014,9 +1004,9 @@ void rtl8723b_SetBeaconRelatedRegisters(struct adapter *padapter)
 void hal_notch_filter_8723b(struct adapter *adapter, bool enable)
 {
 	if (enable)
-		rtw_write8(adapter, rOFDM0_RxDSP+1, rtw_read8(adapter, rOFDM0_RxDSP+1) | BIT1);
+		rtw_write8(adapter, rOFDM0_RxDSP+1, rtw_read8(adapter, rOFDM0_RxDSP+1) | BIT(1));
 	else
-		rtw_write8(adapter, rOFDM0_RxDSP+1, rtw_read8(adapter, rOFDM0_RxDSP+1) & ~BIT1);
+		rtw_write8(adapter, rOFDM0_RxDSP+1, rtw_read8(adapter, rOFDM0_RxDSP+1) & ~BIT(1));
 }
 
 void UpdateHalRAMask8723B(struct adapter *padapter, u32 mac_id, u8 rssi_level)
@@ -1165,15 +1155,15 @@ s32 rtl8723b_InitLLTTable(struct adapter *padapter)
 
 static void hal_get_chnl_group_8723b(u8 channel, u8 *group)
 {
-	if (1  <= channel && channel <= 2)
+	if (channel >= 1 && channel <= 2)
 		*group = 0;
-	else if (3  <= channel && channel <= 5)
+	else if (channel >= 3 && channel <= 5)
 		*group = 1;
-	else if (6  <= channel && channel <= 8)
+	else if (channel >= 6 && channel <= 8)
 		*group = 2;
-	else if (9  <= channel && channel <= 11)
+	else if (channel >= 9 && channel <= 11)
 		*group = 3;
-	else if (12 <= channel && channel <= 14)
+	else if (channel >= 12 && channel <= 14)
 		*group = 4;
 }
 
@@ -1184,12 +1174,12 @@ void Hal_InitPGData(struct adapter *padapter, u8 *PROMContent)
 	if (!pEEPROM->bautoload_fail_flag) { /*  autoload OK. */
 		if (!pEEPROM->EepromOrEfuse) {
 			/*  Read EFUSE real map to shadow. */
-			EFUSE_ShadowMapUpdate(padapter, EFUSE_WIFI);
+			rtw_efuse_shadow_map_update(padapter, EFUSE_WIFI);
 			memcpy(PROMContent, pEEPROM->efuse_eeprom_data, HWSET_MAX_SIZE_8723B);
 		}
 	} else {/* autoload fail */
 		if (!pEEPROM->EepromOrEfuse)
-			EFUSE_ShadowMapUpdate(padapter, EFUSE_WIFI);
+			rtw_efuse_shadow_map_update(padapter, EFUSE_WIFI);
 		memcpy(PROMContent, pEEPROM->efuse_eeprom_data, HWSET_MAX_SIZE_8723B);
 	}
 }
@@ -1221,7 +1211,7 @@ static void Hal_ReadPowerValueFromPROM_8723B(
 
 	memset(pwrInfo24G, 0, sizeof(struct TxPowerInfo24G));
 
-	if (0xFF == PROMContent[eeAddr+1])
+	if (PROMContent[eeAddr+1] == 0xFF)
 		AutoLoadFail = true;
 
 	if (AutoLoadFail) {
@@ -1271,7 +1261,7 @@ static void Hal_ReadPowerValueFromPROM_8723B(
 					pwrInfo24G->BW20_Diff[rfPath][TxCount] =	EEPROM_DEFAULT_24G_HT20_DIFF;
 				else {
 					pwrInfo24G->BW20_Diff[rfPath][TxCount] =	(PROMContent[eeAddr]&0xf0)>>4;
-					if (pwrInfo24G->BW20_Diff[rfPath][TxCount] & BIT3)		/* 4bit sign number to 8 bit sign number */
+					if (pwrInfo24G->BW20_Diff[rfPath][TxCount] & BIT(3))		/* 4bit sign number to 8 bit sign number */
 						pwrInfo24G->BW20_Diff[rfPath][TxCount] |= 0xF0;
 				}
 
@@ -1279,7 +1269,7 @@ static void Hal_ReadPowerValueFromPROM_8723B(
 					pwrInfo24G->OFDM_Diff[rfPath][TxCount] = EEPROM_DEFAULT_24G_OFDM_DIFF;
 				else {
 					pwrInfo24G->OFDM_Diff[rfPath][TxCount] = (PROMContent[eeAddr]&0x0f);
-					if (pwrInfo24G->OFDM_Diff[rfPath][TxCount] & BIT3)		/* 4bit sign number to 8 bit sign number */
+					if (pwrInfo24G->OFDM_Diff[rfPath][TxCount] & BIT(3))		/* 4bit sign number to 8 bit sign number */
 						pwrInfo24G->OFDM_Diff[rfPath][TxCount] |= 0xF0;
 				}
 				pwrInfo24G->CCK_Diff[rfPath][TxCount] = 0;
@@ -1289,7 +1279,7 @@ static void Hal_ReadPowerValueFromPROM_8723B(
 					pwrInfo24G->BW40_Diff[rfPath][TxCount] = EEPROM_DEFAULT_DIFF;
 				else {
 					pwrInfo24G->BW40_Diff[rfPath][TxCount] = (PROMContent[eeAddr]&0xf0)>>4;
-					if (pwrInfo24G->BW40_Diff[rfPath][TxCount] & BIT3)		/* 4bit sign number to 8 bit sign number */
+					if (pwrInfo24G->BW40_Diff[rfPath][TxCount] & BIT(3))		/* 4bit sign number to 8 bit sign number */
 						pwrInfo24G->BW40_Diff[rfPath][TxCount] |= 0xF0;
 				}
 
@@ -1297,7 +1287,7 @@ static void Hal_ReadPowerValueFromPROM_8723B(
 					pwrInfo24G->BW20_Diff[rfPath][TxCount] = EEPROM_DEFAULT_DIFF;
 				else {
 					pwrInfo24G->BW20_Diff[rfPath][TxCount] = (PROMContent[eeAddr]&0x0f);
-					if (pwrInfo24G->BW20_Diff[rfPath][TxCount] & BIT3)		/* 4bit sign number to 8 bit sign number */
+					if (pwrInfo24G->BW20_Diff[rfPath][TxCount] & BIT(3))		/* 4bit sign number to 8 bit sign number */
 						pwrInfo24G->BW20_Diff[rfPath][TxCount] |= 0xF0;
 				}
 				eeAddr++;
@@ -1306,7 +1296,7 @@ static void Hal_ReadPowerValueFromPROM_8723B(
 					pwrInfo24G->OFDM_Diff[rfPath][TxCount] = EEPROM_DEFAULT_DIFF;
 				else {
 					pwrInfo24G->OFDM_Diff[rfPath][TxCount] = (PROMContent[eeAddr]&0xf0)>>4;
-					if (pwrInfo24G->OFDM_Diff[rfPath][TxCount] & BIT3)		/* 4bit sign number to 8 bit sign number */
+					if (pwrInfo24G->OFDM_Diff[rfPath][TxCount] & BIT(3))		/* 4bit sign number to 8 bit sign number */
 						pwrInfo24G->OFDM_Diff[rfPath][TxCount] |= 0xF0;
 				}
 
@@ -1314,7 +1304,7 @@ static void Hal_ReadPowerValueFromPROM_8723B(
 					pwrInfo24G->CCK_Diff[rfPath][TxCount] = EEPROM_DEFAULT_DIFF;
 				else {
 					pwrInfo24G->CCK_Diff[rfPath][TxCount] = (PROMContent[eeAddr]&0x0f);
-					if (pwrInfo24G->CCK_Diff[rfPath][TxCount] & BIT3)		/* 4bit sign number to 8 bit sign number */
+					if (pwrInfo24G->CCK_Diff[rfPath][TxCount] & BIT(3))		/* 4bit sign number to 8 bit sign number */
 						pwrInfo24G->CCK_Diff[rfPath][TxCount] |= 0xF0;
 				}
 				eeAddr++;
@@ -1447,7 +1437,7 @@ void Hal_EfuseParsePackageType_8723B(
 	u8 efuseContent;
 
 	Hal_EfusePowerSwitch(padapter, true);
-	efuse_OneByteRead(padapter, 0x1FB, &efuseContent);
+	rtw_efuse_one_byte_read(padapter, 0x1FB, &efuseContent);
 	Hal_EfusePowerSwitch(padapter, false);
 
 	package = efuseContent & 0x7;
@@ -1555,7 +1545,8 @@ void Hal_ReadRFGainOffset(
 
 	if (!AutoloadFail) {
 		Adapter->eeprompriv.EEPROMRFGainOffset = PROMContent[EEPROM_RF_GAIN_OFFSET];
-		Adapter->eeprompriv.EEPROMRFGainVal = EFUSE_Read1Byte(Adapter, EEPROM_RF_GAIN_VAL);
+		Adapter->eeprompriv.EEPROMRFGainVal = rtw_efuse_read_1_byte(Adapter,
+									    EEPROM_RF_GAIN_VAL);
 	} else {
 		Adapter->eeprompriv.EEPROMRFGainOffset = 0;
 		Adapter->eeprompriv.EEPROMRFGainVal = 0xFF;
@@ -2037,7 +2028,7 @@ static void hw_var_set_bcn_func(struct adapter *padapter, u8 variable, u8 *val)
 		val8 &= ~(EN_BCN_FUNCTION | EN_TXBCN_RPT);
 
 		/*  Always enable port0 beacon function for PSTDMA */
-		if (REG_BCN_CTRL == bcn_ctrl_reg)
+		if (bcn_ctrl_reg == REG_BCN_CTRL)
 			val8 |= EN_BCN_FUNCTION;
 
 		rtw_write8(padapter, bcn_ctrl_reg, val8);
@@ -2118,7 +2109,7 @@ static void hw_var_set_mlme_sitesurvey(struct adapter *padapter, u8 variable, u8
 	/*  config RCR to receive different BSSID & not to receive data frame */
 	value_rxfltmap2 = 0;
 
-	if ((check_fwstate(pmlmepriv, WIFI_AP_STATE) == true))
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		rcr_clear_bit = RCR_CBSSID_BCN;
 
 	value_rcr = rtw_read32(padapter, REG_RCR);
@@ -2185,7 +2176,7 @@ static void hw_var_set_mlme_join(struct adapter *padapter, u8 variable, u8 *val)
 			val32 |= RCR_CBSSID_DATA|RCR_CBSSID_BCN;
 		rtw_write32(padapter, REG_RCR, val32);
 
-		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) == true)
+		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE))
 			RetryLimit = (pEEPROM->CustomerID == RT_CID_CCX) ? 7 : 48;
 		else /*  Ad-hoc Mode */
 			RetryLimit = 0x7;
@@ -2723,7 +2714,7 @@ void SetHwReg8723B(struct adapter *padapter, u8 variable, u8 *val)
 		break;
 
 	case HW_VAR_DL_RSVD_PAGE:
-		if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) == true)
+		if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE))
 			rtl8723b_download_BTCoex_AP_mode_rsvd_page(padapter);
 		else
 			rtl8723b_download_rsvd_page(padapter, RT_MEDIA_CONNECT);
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c b/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c
index 7fac1c2ba8e0f..d6369eb7b5705 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_phycfg.c
@@ -19,13 +19,12 @@ static	u32 phy_CalculateBitShift(u32 BitMask)
 	u32 i;
 
 	for (i = 0; i <= 31; i++) {
-		if (((BitMask>>i) &  0x1) == 1)
+		if (((BitMask >> i) &  0x1) == 1)
 			break;
 	}
 	return i;
 }
 
-
 /**
  * PHY_QueryBBReg_8723B - Read "specific bits" from BB register.
  * @Adapter:
@@ -46,10 +45,8 @@ u32 PHY_QueryBBReg_8723B(struct adapter *Adapter, u32 RegAddr, u32 BitMask)
 	BitShift = phy_CalculateBitShift(BitMask);
 
 	return (OriginalValue & BitMask) >> BitShift;
-
 }
 
-
 /**
  * PHY_SetBBReg_8723B - Write "Specific bits" to BB register (page 8~).
  * @Adapter:
@@ -80,10 +77,8 @@ void PHY_SetBBReg_8723B(
 	}
 
 	rtw_write32(Adapter, RegAddr, Data);
-
 }
 
-
 /*  */
 /*  2. RF register R/W API */
 /*  */
@@ -109,18 +104,18 @@ static u32 phy_RFSerialRead_8723B(
 	NewOffset = Offset;
 
 	if (eRFPath == RF_PATH_A) {
-		tmplong2 = PHY_QueryBBReg(Adapter, rFPGA0_XA_HSSIParameter2|MaskforPhySet, bMaskDWord);
-		tmplong2 = (tmplong2 & (~bLSSIReadAddress)) | (NewOffset<<23) | bLSSIReadEdge;	/* T65 RF */
-		PHY_SetBBReg(Adapter, rFPGA0_XA_HSSIParameter2|MaskforPhySet, bMaskDWord, tmplong2&(~bLSSIReadEdge));
+		tmplong2 = PHY_QueryBBReg(Adapter, rFPGA0_XA_HSSIParameter2 | MaskforPhySet, bMaskDWord);
+		tmplong2 = (tmplong2 & (~bLSSIReadAddress)) | (NewOffset << 23) | bLSSIReadEdge;	/* T65 RF */
+		PHY_SetBBReg(Adapter, rFPGA0_XA_HSSIParameter2 | MaskforPhySet, bMaskDWord, tmplong2 & (~bLSSIReadEdge));
 	} else {
-		tmplong2 = PHY_QueryBBReg(Adapter, rFPGA0_XB_HSSIParameter2|MaskforPhySet, bMaskDWord);
-		tmplong2 = (tmplong2 & (~bLSSIReadAddress)) | (NewOffset<<23) | bLSSIReadEdge;	/* T65 RF */
-		PHY_SetBBReg(Adapter, rFPGA0_XB_HSSIParameter2|MaskforPhySet, bMaskDWord, tmplong2&(~bLSSIReadEdge));
+		tmplong2 = PHY_QueryBBReg(Adapter, rFPGA0_XB_HSSIParameter2 | MaskforPhySet, bMaskDWord);
+		tmplong2 = (tmplong2 & (~bLSSIReadAddress)) | (NewOffset << 23) | bLSSIReadEdge;	/* T65 RF */
+		PHY_SetBBReg(Adapter, rFPGA0_XB_HSSIParameter2 | MaskforPhySet, bMaskDWord, tmplong2 & (~bLSSIReadEdge));
 	}
 
-	tmplong2 = PHY_QueryBBReg(Adapter, rFPGA0_XA_HSSIParameter2|MaskforPhySet, bMaskDWord);
-	PHY_SetBBReg(Adapter, rFPGA0_XA_HSSIParameter2|MaskforPhySet, bMaskDWord, tmplong2 & (~bLSSIReadEdge));
-	PHY_SetBBReg(Adapter, rFPGA0_XA_HSSIParameter2|MaskforPhySet, bMaskDWord, tmplong2 | bLSSIReadEdge);
+	tmplong2 = PHY_QueryBBReg(Adapter, rFPGA0_XA_HSSIParameter2 | MaskforPhySet, bMaskDWord);
+	PHY_SetBBReg(Adapter, rFPGA0_XA_HSSIParameter2 | MaskforPhySet, bMaskDWord, tmplong2 & (~bLSSIReadEdge));
+	PHY_SetBBReg(Adapter, rFPGA0_XA_HSSIParameter2 | MaskforPhySet, bMaskDWord, tmplong2 | bLSSIReadEdge);
 
 	udelay(10);
 
@@ -129,19 +124,18 @@ static u32 phy_RFSerialRead_8723B(
 	udelay(10);
 
 	if (eRFPath == RF_PATH_A)
-		RfPiEnable = (u8)PHY_QueryBBReg(Adapter, rFPGA0_XA_HSSIParameter1|MaskforPhySet, BIT8);
+		RfPiEnable = (u8)PHY_QueryBBReg(Adapter, rFPGA0_XA_HSSIParameter1 | MaskforPhySet, BIT(8));
 	else if (eRFPath == RF_PATH_B)
-		RfPiEnable = (u8)PHY_QueryBBReg(Adapter, rFPGA0_XB_HSSIParameter1|MaskforPhySet, BIT8);
+		RfPiEnable = (u8)PHY_QueryBBReg(Adapter, rFPGA0_XB_HSSIParameter1 | MaskforPhySet, BIT(8));
 
 	if (RfPiEnable) {
 		/*  Read from BBreg8b8, 12 bits for 8190, 20bits for T65 RF */
-		retValue = PHY_QueryBBReg(Adapter, pPhyReg->rfLSSIReadBackPi|MaskforPhySet, bLSSIReadBackData);
+		retValue = PHY_QueryBBReg(Adapter, pPhyReg->rfLSSIReadBackPi | MaskforPhySet, bLSSIReadBackData);
 	} else {
 		/* Read from BBreg8a0, 12 bits for 8190, 20 bits for T65 RF */
-		retValue = PHY_QueryBBReg(Adapter, pPhyReg->rfLSSIReadBack|MaskforPhySet, bLSSIReadBackData);
+		retValue = PHY_QueryBBReg(Adapter, pPhyReg->rfLSSIReadBack | MaskforPhySet, bLSSIReadBackData);
 	}
 	return retValue;
-
 }
 
 /**
@@ -203,14 +197,13 @@ static void phy_RFSerialWrite_8723B(
 	/*  */
 	/*  Put write addr in [5:0]  and write data in [31:16] */
 	/*  */
-	DataAndAddr = ((NewOffset<<20) | (Data&0x000fffff)) & 0x0fffffff;	/*  T65 RF */
+	DataAndAddr = ((NewOffset << 20) | (Data & 0x000fffff)) & 0x0fffffff;	/*  T65 RF */
 	/*  */
 	/*  Write Operation */
 	/*  */
 	PHY_SetBBReg(Adapter, pPhyReg->rf3wireOffset, bMaskDWord, DataAndAddr);
 }
 
-
 /**
  * PHY_QueryRFReg_8723B - Query "Specific bits" to RF register (page 8~).
  * @Adapter:
@@ -266,18 +259,16 @@ void PHY_SetRFReg_8723B(
 	if (BitMask != bRFRegOffsetMask) {
 		Original_Value = phy_RFSerialRead_8723B(Adapter, eRFPath, RegAddr);
 		BitShift =  phy_CalculateBitShift(BitMask);
-		Data = ((Original_Value & (~BitMask)) | (Data<<BitShift));
+		Data = ((Original_Value & (~BitMask)) | (Data << BitShift));
 	}
 
 	phy_RFSerialWrite_8723B(Adapter, eRFPath, RegAddr, Data);
 }
 
-
 /*  */
 /*  3. Initial MAC/BB/RF config by reading MAC/BB/RF txt. */
 /*  */
 
-
 /*-----------------------------------------------------------------------------
  * PHY_MACConfig8192C - Config MAC by header file or parameter file.
  *
@@ -329,7 +320,6 @@ static void phy_InitBBRFRegisterDefinition(struct adapter *Adapter)
 	pHalData->PHYRegDef[RF_PATH_B].rfLSSIReadBack = rFPGA0_XB_LSSIReadBack;
 	pHalData->PHYRegDef[RF_PATH_A].rfLSSIReadBackPi = TransceiverA_HSPI_Readback;
 	pHalData->PHYRegDef[RF_PATH_B].rfLSSIReadBackPi = TransceiverB_HSPI_Readback;
-
 }
 
 static int phy_BB8723b_Config_ParaFile(struct adapter *Adapter)
@@ -373,7 +363,6 @@ static int phy_BB8723b_Config_ParaFile(struct adapter *Adapter)
 	return _SUCCESS;
 }
 
-
 int PHY_BBConfig8723B(struct adapter *Adapter)
 {
 	int	rtStatus = _SUCCESS;
@@ -385,19 +374,20 @@ int PHY_BBConfig8723B(struct adapter *Adapter)
 
 	/*  Enable BB and RF */
 	RegVal = rtw_read16(Adapter, REG_SYS_FUNC_EN);
-	rtw_write16(Adapter, REG_SYS_FUNC_EN, (u16)(RegVal|BIT13|BIT0|BIT1));
+	rtw_write16(Adapter, REG_SYS_FUNC_EN, (u16)(RegVal | BIT(13) | BIT(0) | BIT(1)));
 
 	rtw_write32(Adapter, 0x948, 0x280);	/*  Others use Antenna S1 */
 
-	rtw_write8(Adapter, REG_RF_CTRL, RF_EN|RF_RSTB|RF_SDMRSTB);
+	rtw_write8(Adapter, REG_RF_CTRL, RF_EN | RF_RSTB | RF_SDMRSTB);
 
 	msleep(1);
 
 	PHY_SetRFReg(Adapter, RF_PATH_A, 0x1, 0xfffff, 0x780);
 
-	rtw_write8(Adapter, REG_SYS_FUNC_EN, FEN_PPLL|FEN_PCIEA|FEN_DIO_PCIE|FEN_BB_GLB_RSTn|FEN_BBRSTB);
+	rtw_write8(Adapter, REG_SYS_FUNC_EN,
+		   FEN_PPLL | FEN_PCIEA | FEN_DIO_PCIE | FEN_BB_GLB_RSTn | FEN_BBRSTB);
 
-	rtw_write8(Adapter, REG_AFE_XTAL_CTRL+1, 0x80);
+	rtw_write8(Adapter, REG_AFE_XTAL_CTRL + 1, 0x80);
 
 	/*  */
 	/*  Config BB and AGC */
@@ -531,7 +521,7 @@ u8 PHY_GetTxPowerIndex(
 	struct hal_com_data *pHalData = GET_HAL_DATA(padapter);
 	s8 txPower = 0, powerDiffByRate = 0, limit = 0;
 
-	txPower = (s8) PHY_GetTxPowerIndexBase(padapter, RFPath, Rate, BandWidth, Channel);
+	txPower = (s8)PHY_GetTxPowerIndexBase(padapter, RFPath, Rate, BandWidth, Channel);
 	powerDiffByRate = PHY_GetTxPowerByRate(padapter, RF_PATH_A, Rate);
 
 	limit = phy_get_tx_pwr_lmt(
@@ -551,7 +541,7 @@ u8 PHY_GetTxPowerIndex(
 	if (txPower > MAX_POWER_INDEX)
 		txPower = MAX_POWER_INDEX;
 
-	return (u8) txPower;
+	return (u8)txPower;
 }
 
 void PHY_SetTxPowerLevel8723B(struct adapter *Adapter, u8 Channel)
@@ -584,7 +574,7 @@ static void phy_SetRegBW_8723B(
 		break;
 
 	case CHANNEL_WIDTH_40:
-		u2tmp = RegRfMod_BW | BIT7;
+		u2tmp = RegRfMod_BW | BIT(7);
 		rtw_write16(Adapter, REG_TRXPTCL_CTL_8723B, (u2tmp & 0xFEFF)); /*  BIT 7 = 1, BIT 8 = 0 */
 		break;
 
@@ -613,7 +603,6 @@ static void phy_PostSetBwMode8723B(struct adapter *Adapter)
 	u8 SubChnlNum = 0;
 	struct hal_com_data *pHalData = GET_HAL_DATA(Adapter);
 
-
 	/* 3 Set Reg668 Reg440 BW */
 	phy_SetRegBW_8723B(Adapter, pHalData->CurrentChannelBW);
 
@@ -631,7 +620,7 @@ static void phy_PostSetBwMode8723B(struct adapter *Adapter)
 
 		PHY_SetBBReg(Adapter, rFPGA1_RFMOD, bRFMOD, 0x0);
 
-		PHY_SetBBReg(Adapter, rOFDM0_TxPseudoNoiseWgt, (BIT31|BIT30), 0x0);
+		PHY_SetBBReg(Adapter, rOFDM0_TxPseudoNoiseWgt, (BIT(31) | BIT(30)), 0x0);
 		break;
 
 	/* 40 MHz channel*/
@@ -641,11 +630,11 @@ static void phy_PostSetBwMode8723B(struct adapter *Adapter)
 		PHY_SetBBReg(Adapter, rFPGA1_RFMOD, bRFMOD, 0x1);
 
 		/*  Set Control channel to upper or lower. These settings are required only for 40MHz */
-		PHY_SetBBReg(Adapter, rCCK0_System, bCCKSideBand, (pHalData->nCur40MhzPrimeSC>>1));
+		PHY_SetBBReg(Adapter, rCCK0_System, bCCKSideBand, (pHalData->nCur40MhzPrimeSC >> 1));
 
 		PHY_SetBBReg(Adapter, rOFDM1_LSTF, 0xC00, pHalData->nCur40MhzPrimeSC);
 
-		PHY_SetBBReg(Adapter, 0x818, (BIT26|BIT27), (pHalData->nCur40MhzPrimeSC == HAL_PRIME_CHNL_OFFSET_LOWER) ? 2 : 1);
+		PHY_SetBBReg(Adapter, 0x818, (BIT(26) | BIT(27)), (pHalData->nCur40MhzPrimeSC == HAL_PRIME_CHNL_OFFSET_LOWER) ? 2 : 1);
 		break;
 	default:
 		break;
@@ -724,13 +713,11 @@ static void PHY_HandleSwChnlAndSetBW8723B(
 	if (!pHalData->bSetChnlBW && !pHalData->bSwChnl)
 		return;
 
-
 	if (pHalData->bSwChnl) {
 		pHalData->CurrentChannel = ChannelNum;
 		pHalData->CurrentCenterFrequencyIndex1 = ChannelNum;
 	}
 
-
 	if (pHalData->bSetChnlBW) {
 		pHalData->CurrentChannelBW = ChnlWidth;
 		pHalData->nCur40MhzPrimeSC = ExtChnlOffsetOf40MHz;
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c b/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c
index ffb35e1ace62a..5564f3f20ba52 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723b_rf6052.c
@@ -24,7 +24,7 @@
  * 11/05/2008	MHC		Add API for tw power setting.
  *
  *
-******************************************************************************/
+ ******************************************************************************/
 
 #include <rtl8723b_hal.h>
 
@@ -53,7 +53,8 @@
  * Return:      NONE
  *
  * Note:		For RF type 0222D
- *---------------------------------------------------------------------------*/
+ *---------------------------------------------------------------------------
+ */
 void PHY_RF6052SetBandwidth8723B(
 	struct adapter *Adapter, enum channel_width Bandwidth
 ) /* 20M or 40M */
@@ -62,13 +63,13 @@ void PHY_RF6052SetBandwidth8723B(
 
 	switch (Bandwidth) {
 	case CHANNEL_WIDTH_20:
-		pHalData->RfRegChnlVal[0] = ((pHalData->RfRegChnlVal[0] & 0xfffff3ff) | BIT10 | BIT11);
+		pHalData->RfRegChnlVal[0] = ((pHalData->RfRegChnlVal[0] & 0xfffff3ff) | BIT(10) | BIT(11));
 		PHY_SetRFReg(Adapter, RF_PATH_A, RF_CHNLBW, bRFRegOffsetMask, pHalData->RfRegChnlVal[0]);
 		PHY_SetRFReg(Adapter, RF_PATH_B, RF_CHNLBW, bRFRegOffsetMask, pHalData->RfRegChnlVal[0]);
 		break;
 
 	case CHANNEL_WIDTH_40:
-		pHalData->RfRegChnlVal[0] = ((pHalData->RfRegChnlVal[0] & 0xfffff3ff) | BIT10);
+		pHalData->RfRegChnlVal[0] = ((pHalData->RfRegChnlVal[0] & 0xfffff3ff) | BIT(10));
 		PHY_SetRFReg(Adapter, RF_PATH_A, RF_CHNLBW, bRFRegOffsetMask, pHalData->RfRegChnlVal[0]);
 		PHY_SetRFReg(Adapter, RF_PATH_B, RF_CHNLBW, bRFRegOffsetMask, pHalData->RfRegChnlVal[0]);
 		break;
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
index cab4707091e2b..888029e21dcb7 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_recv.c
@@ -135,7 +135,7 @@ static void update_recvframe_phyinfo(union recv_frame *precvframe,
 	precvframe->u.hdr.psta = NULL;
 	if (
 		pkt_info.bssid_match &&
-		(check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) == true)
+		(check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE))
 	) {
 		if (psta) {
 			precvframe->u.hdr.psta = psta;
diff --git a/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c b/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
index f50726d2ed0c9..a7f23335cdd92 100644
--- a/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
+++ b/drivers/staging/rtl8723bs/hal/rtl8723bs_xmit.c
@@ -93,7 +93,7 @@ static s32 rtl8723_dequeue_writeport(struct adapter *padapter)
 	)
 		goto free_xmitbuf;
 
-	if (rtw_sdio_wait_enough_TxOQT_space(padapter, pxmitbuf->agg_num) == false)
+	if (!rtw_sdio_wait_enough_TxOQT_space(padapter, pxmitbuf->agg_num))
 		goto free_xmitbuf;
 
 	traffic_check_for_leave_lps(padapter, true, pxmitbuf->agg_num);
@@ -225,7 +225,7 @@ static s32 xmit_xmitframes(struct adapter *padapter, struct xmit_priv *pxmitpriv
 
 			frame_phead = get_list_head(pframe_queue);
 
-			while (list_empty(frame_phead) == false) {
+			while (!list_empty(frame_phead)) {
 				frame_plist = get_next(frame_phead);
 				pxmitframe = container_of(frame_plist, struct xmit_frame, list);
 
@@ -269,7 +269,7 @@ static s32 xmit_xmitframes(struct adapter *padapter, struct xmit_priv *pxmitpriv
 				}
 
 				/*  ok to send, remove frame from queue */
-				if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE) == true)
+				if (check_fwstate(&padapter->mlmepriv, WIFI_AP_STATE))
 					if (
 						(pxmitframe->attrib.psta->state & WIFI_SLEEP_STATE) &&
 						(pxmitframe->attrib.triggered == 0)
@@ -395,9 +395,9 @@ static s32 rtl8723bs_xmit_handler(struct adapter *padapter)
 	spin_lock_bh(&pxmitpriv->lock);
 	ret = rtw_txframes_pending(padapter);
 	spin_unlock_bh(&pxmitpriv->lock);
-	if (ret == 1) {
+	if (ret == 1)
 		goto next;
-	}
+
 
 	return _SUCCESS;
 }
@@ -415,7 +415,7 @@ int rtl8723bs_xmit_thread(void *context)
 		if (signal_pending(current)) {
 			flush_signals(current);
 		}
-	} while (_SUCCESS == ret);
+	} while (ret == _SUCCESS);
 
 	complete(&pxmitpriv->SdioXmitTerminate);
 
@@ -566,7 +566,7 @@ void rtl8723bs_free_xmit_priv(struct adapter *padapter)
 	spin_unlock_bh(&pqueue->lock);
 
 	phead = &tmplist;
-	while (list_empty(phead) == false) {
+	while (!list_empty(phead)) {
 		plist = get_next(phead);
 		list_del_init(plist);
 
diff --git a/drivers/staging/rtl8723bs/hal/sdio_halinit.c b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
index f2f73c65a636a..1e0498268aeeb 100644
--- a/drivers/staging/rtl8723bs/hal/sdio_halinit.c
+++ b/drivers/staging/rtl8723bs/hal/sdio_halinit.c
@@ -46,8 +46,6 @@ u8 _InitPowerOn_8723BS(struct adapter *padapter)
 	u16 value16;
 	u32 value32;
 	u8 ret;
-/* 	u8 bMacPwrCtrlOn; */
-
 
 	/*  all of these MUST be configured before power on */
 
@@ -69,10 +67,6 @@ u8 _InitPowerOn_8723BS(struct adapter *padapter)
 	value16 |= EnPDN; /*  Enable HW power down and RF on */
 	rtw_write16(padapter, REG_APS_FSMCO, value16);
 
-	/*  Enable CMD53 R/W Operation */
-/* 	bMacPwrCtrlOn = true; */
-/* 	rtw_hal_set_hwreg(padapter, HW_VAR_APFM_ON_MAC, &bMacPwrCtrlOn); */
-
 	rtw_write8(padapter, REG_CR, 0x00);
 	/*  Enable MAC DMA/WMAC/SCHEDULE/SEC block */
 	value16 = rtw_read16(padapter, REG_CR);
@@ -359,7 +353,6 @@ static void _InitNetworkType(struct adapter *padapter)
 	value32 = rtw_read32(padapter, REG_CR);
 
 	/*  TODO: use the other function to set network type */
-/* 	value32 = (value32 & ~MASK_NETTYPE) | _NETTYPE(NT_LINK_AD_HOC); */
 	value32 = (value32 & ~MASK_NETTYPE) | _NETTYPE(NT_LINK_AP);
 
 	rtw_write32(padapter, REG_CR, value32);
@@ -410,9 +403,6 @@ static void _InitAdaptiveCtrl(struct adapter *padapter)
 	value32 |= RATE_RRSR_CCK_ONLY_1M;
 	rtw_write32(padapter, REG_RRSR, value32);
 
-	/*  CF-END Threshold */
-	/* m_spIoBase->rtw_write8(REG_CFEND_TH, 0x1); */
-
 	/*  SIFS (used in NAV) */
 	value16 = _SPEC_SIFS_CCK(0x10) | _SPEC_SIFS_OFDM(0x10);
 	rtw_write16(padapter, REG_SPEC_SIFS, value16);
@@ -486,9 +476,6 @@ static void _initSdioAggregationSetting(struct adapter *padapter)
 {
 	struct hal_com_data	*pHalData = GET_HAL_DATA(padapter);
 
-	/*  Tx aggregation setting */
-/* 	sdio_AggSettingTxUpdate(padapter); */
-
 	/*  Rx aggregation setting */
 	HalRxAggr8723BSdio(padapter);
 
@@ -574,10 +561,10 @@ static bool HalDetectPwrDownMode(struct adapter *Adapter)
 	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(Adapter);
 
 
-	EFUSE_ShadowRead(Adapter, 1, 0x7B/*EEPROM_RF_OPT3_92C*/, (u32 *)&tmpvalue);
+	rtw_efuse_shadow_read(Adapter, 1, 0x7B/*EEPROM_RF_OPT3_92C*/, (u32 *)&tmpvalue);
 
 	/*  2010/08/25 MH INF priority > PDN Efuse value. */
-	if (tmpvalue & BIT4 && pwrctrlpriv->reg_pdnmode)
+	if (tmpvalue & BIT(4) && pwrctrlpriv->reg_pdnmode)
 		pHalData->pwrdown = true;
 	else
 		pHalData->pwrdown = false;
@@ -596,10 +583,8 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 	pHalData = GET_HAL_DATA(padapter);
 	pwrctrlpriv = adapter_to_pwrctl(padapter);
 
-	if (
-		adapter_to_pwrctl(padapter)->bips_processing == true &&
-		adapter_to_pwrctl(padapter)->pre_ips_type == 0
-	) {
+	if (adapter_to_pwrctl(padapter)->bips_processing &&
+	    adapter_to_pwrctl(padapter)->pre_ips_type == 0) {
 		unsigned long start_time;
 		u8 cpwm_orig, cpwm_now;
 		u8 val8, bMacPwrCtrlOn = true;
@@ -640,9 +625,6 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 		return _SUCCESS;
 	}
 
-	/*  Disable Interrupt first. */
-/* 	rtw_hal_disable_interrupt(padapter); */
-
 	ret = _InitPowerOn_8723BS(padapter);
 	if (ret == _FAIL)
 		return _FAIL;
@@ -654,14 +636,12 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 		padapter->bFWReady = false;
 		pHalData->fw_ractrl = false;
 		return ret;
-	} else {
-		padapter->bFWReady = true;
-		pHalData->fw_ractrl = true;
 	}
 
-	rtl8723b_InitializeFirmwareVars(padapter);
+	padapter->bFWReady = true;
+	pHalData->fw_ractrl = true;
 
-/* 	SIC_Init(padapter); */
+	rtl8723b_InitializeFirmwareVars(padapter);
 
 	if (pwrctrlpriv->reg_rfoff)
 		pwrctrlpriv->rf_pwrstate = rf_off;
@@ -704,8 +684,6 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 	pHalData->RfRegChnlVal[1] =
 		PHY_QueryRFReg(padapter, (enum rf_path)1, RF_CHNLBW, bRFRegOffsetMask);
 
-
-	/* if (!pHalData->bMACFuncEnable) { */
 	_InitQueueReservedPage(padapter);
 	_InitTxBufferBoundary(padapter);
 
@@ -745,11 +723,6 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 	rtw_hal_set_chnl_bw(padapter, padapter->registrypriv.channel,
 		CHANNEL_WIDTH_20, HAL_PRIME_CHNL_OFFSET_DONT_CARE, HAL_PRIME_CHNL_OFFSET_DONT_CARE);
 
-	/*  Record original value for template. This is arough data, we can only use the data */
-	/*  for power adjust. The value can not be adjustde according to different power!!! */
-/* 	pHalData->OriginalCckTxPwrIdx = pHalData->CurrentCckTxPwrIdx; */
-/* 	pHalData->OriginalOfdm24GTxPwrIdx = pHalData->CurrentOfdm24GTxPwrIdx; */
-
 	rtl8723b_InitAntenna_Selection(padapter);
 
 	/*  */
@@ -791,8 +764,6 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 	/* ack for xmit mgmt frames. */
 	rtw_write32(padapter, REG_FWHW_TXQ_CTRL, rtw_read32(padapter, REG_FWHW_TXQ_CTRL) | BIT(12));
 
-/* 	pHalData->PreRpwmVal = SdioLocalCmd52Read1Byte(padapter, SDIO_REG_HRPWM1) & 0x80; */
-
 	{
 		pwrctrlpriv->rf_pwrstate = rf_on;
 
@@ -844,7 +815,7 @@ u32 rtl8723bs_hal_init(struct adapter *padapter)
 
 /*  */
 /*  Description: */
-/* 	RTL8723e card disable power sequence v003 which suggested by Scott. */
+/*	RTL8723e card disable power sequence v003 which suggested by Scott. */
 /*  */
 /*  First created by tynli. 2011.01.28. */
 /*  */
@@ -856,7 +827,7 @@ static void CardDisableRTL8723BSdio(struct adapter *padapter)
 	/*  Run LPS WL RFOFF flow */
 	HalPwrSeqCmdParsing(padapter, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, rtl8723B_enter_lps_flow);
 
-	/* 	==== Reset digital sequence   ====== */
+	/*	==== Reset digital sequence   ====== */
 
 	val = rtw_read8(padapter, REG_MCUFWDL);
 	if ((val & RAM_DL_SEL) && padapter->bFWReady) /* 8051 RAM code */
@@ -879,7 +850,7 @@ static void CardDisableRTL8723BSdio(struct adapter *padapter)
 	val |= BIT(0);
 	rtw_write8(padapter, REG_RSV_CTRL + 1, val);
 
-	/* 	==== Reset digital sequence end ====== */
+	/*	==== Reset digital sequence end ====== */
 
 	bMacPwrCtrlOn = false;	/*  Disable CMD53 R/W */
 	rtw_hal_set_hwreg(padapter, HW_VAR_APFM_ON_MAC, &bMacPwrCtrlOn);
@@ -981,13 +952,13 @@ void rtl8723bs_interface_configure(struct adapter *padapter)
 }
 
 /*  */
-/* 	Description: */
-/* 		We should set Efuse cell selection to WiFi cell in default. */
+/*	Description: */
+/*		We should set Efuse cell selection to WiFi cell in default. */
 /*  */
-/* 	Assumption: */
-/* 		PASSIVE_LEVEL */
+/*	Assumption: */
+/*		PASSIVE_LEVEL */
 /*  */
-/* 	Added by Roger, 2010.11.23. */
+/*	Added by Roger, 2010.11.23. */
 /*  */
 static void _EfuseCellSel(struct adapter *padapter)
 {
@@ -1073,24 +1044,22 @@ static void _ReadEfuseInfo8723BS(struct adapter *padapter)
 static void _ReadPROMContent(struct adapter *padapter)
 {
 	struct eeprom_priv *pEEPROM = GET_EEPROM_EFUSE_PRIV(padapter);
-	u8 	eeValue;
+	u8 eeValue;
 
 	eeValue = rtw_read8(padapter, REG_9346CR);
 	/*  To check system boot selection. */
 	pEEPROM->EepromOrEfuse = (eeValue & BOOT_FROM_EEPROM) ? true : false;
 	pEEPROM->bautoload_fail_flag = (eeValue & EEPROM_EN) ? false : true;
 
-/* 	pHalData->EEType = IS_BOOT_FROM_EEPROM(Adapter) ? EEPROM_93C46 : EEPROM_BOOT_EFUSE; */
-
 	_ReadEfuseInfo8723BS(padapter);
 }
 
 /*  */
-/* 	Description: */
-/* 		Read HW adapter information by E-Fuse or EEPROM according CR9346 reported. */
+/*	Description: */
+/*		Read HW adapter information by E-Fuse or EEPROM according CR9346 reported. */
 /*  */
-/* 	Assumption: */
-/* 		PASSIVE_LEVEL (SDIO interface) */
+/*	Assumption: */
+/*		PASSIVE_LEVEL (SDIO interface) */
 /*  */
 /*  */
 static s32 _ReadAdapterInfo8723BS(struct adapter *padapter)
@@ -1118,7 +1087,7 @@ static s32 _ReadAdapterInfo8723BS(struct adapter *padapter)
 	return _SUCCESS;
 }
 
-void ReadAdapterInfo8723BS(struct adapter *padapter)
+void rtw_read_adapter_info(struct adapter *padapter)
 {
 	/*  Read EEPROM size before call any EEPROM function */
 	padapter->EepromAddressSize = GetEEPROMSize8723B(padapter);
@@ -1203,14 +1172,14 @@ void SetHwRegWithBuf8723B(struct adapter *padapter, u8 variable, u8 *pbuf, int l
 }
 
 /*  */
-/* 	Description: */
-/* 		Query setting of specified variable. */
+/*	Description: */
+/*		Query setting of specified variable. */
 /*  */
 u8 GetHalDefVar8723BSDIO(
 	struct adapter *Adapter, enum hal_def_variable eVariable, void *pValue
 )
 {
-	u8 	bResult = _SUCCESS;
+	u8 bResult = _SUCCESS;
 
 	switch (eVariable) {
 	case HAL_DEF_IS_SUPPORT_ANT_DIV:
diff --git a/drivers/staging/rtl8723bs/hal/sdio_ops.c b/drivers/staging/rtl8723bs/hal/sdio_ops.c
index 8cece69dbf792..5d74913be573e 100644
--- a/drivers/staging/rtl8723bs/hal/sdio_ops.c
+++ b/drivers/staging/rtl8723bs/hal/sdio_ops.c
@@ -191,51 +191,6 @@ static u32 sdio_read32(struct intf_hdl *intfhdl, u32 addr)
 	return val;
 }
 
-static s32 sdio_readN(struct intf_hdl *intfhdl, u32 addr, u32 cnt, u8 *buf)
-{
-	struct adapter *adapter;
-	u8 mac_pwr_ctrl_on;
-	u8 device_id;
-	u16 offset;
-	u32 ftaddr;
-	u8 shift;
-	s32 err;
-
-	adapter = intfhdl->padapter;
-	err = 0;
-
-	ftaddr = _cvrt2ftaddr(addr, &device_id, &offset);
-
-	rtw_hal_get_hwreg(adapter, HW_VAR_APFM_ON_MAC, &mac_pwr_ctrl_on);
-	if (
-		((device_id == WLAN_IOREG_DEVICE_ID) && (offset < 0x100)) ||
-		(!mac_pwr_ctrl_on) ||
-		(adapter_to_pwrctl(adapter)->fw_current_in_ps_mode)
-	)
-		return sd_cmd52_read(intfhdl, ftaddr, cnt, buf);
-
-	/*  4 bytes alignment */
-	shift = ftaddr & 0x3;
-	if (shift == 0) {
-		err = sd_read(intfhdl, ftaddr, cnt, buf);
-	} else {
-		u8 *tmpbuf;
-		u32 n;
-
-		ftaddr &= ~(u16)0x3;
-		n = cnt + shift;
-		tmpbuf = kmalloc(n, GFP_ATOMIC);
-		if (!tmpbuf)
-			return -ENOMEM;
-
-		err = sd_read(intfhdl, ftaddr, n, tmpbuf);
-		if (!err)
-			memcpy(buf, tmpbuf + shift, cnt);
-		kfree(tmpbuf);
-	}
-	return err;
-}
-
 static s32 sdio_write8(struct intf_hdl *intfhdl, u32 addr, u8 val)
 {
 	u32 ftaddr;
@@ -295,73 +250,6 @@ static s32 sdio_write32(struct intf_hdl *intfhdl, u32 addr, u32 val)
 	return err;
 }
 
-static s32 sdio_writeN(struct intf_hdl *intfhdl, u32 addr, u32 cnt, u8 *buf)
-{
-	struct adapter *adapter;
-	u8 mac_pwr_ctrl_on;
-	u8 device_id;
-	u16 offset;
-	u32 ftaddr;
-	u8 shift;
-	s32 err;
-
-	adapter = intfhdl->padapter;
-	err = 0;
-
-	ftaddr = _cvrt2ftaddr(addr, &device_id, &offset);
-
-	rtw_hal_get_hwreg(adapter, HW_VAR_APFM_ON_MAC, &mac_pwr_ctrl_on);
-	if (
-		((device_id == WLAN_IOREG_DEVICE_ID) && (offset < 0x100)) ||
-		(!mac_pwr_ctrl_on) ||
-		(adapter_to_pwrctl(adapter)->fw_current_in_ps_mode)
-	)
-		return sd_cmd52_write(intfhdl, ftaddr, cnt, buf);
-
-	shift = ftaddr & 0x3;
-	if (shift == 0) {
-		err = sd_write(intfhdl, ftaddr, cnt, buf);
-	} else {
-		u8 *tmpbuf;
-		u32 n;
-
-		ftaddr &= ~(u16)0x3;
-		n = cnt + shift;
-		tmpbuf = kmalloc(n, GFP_ATOMIC);
-		if (!tmpbuf)
-			return -ENOMEM;
-		err = sd_read(intfhdl, ftaddr, 4, tmpbuf);
-		if (err) {
-			kfree(tmpbuf);
-			return err;
-		}
-		memcpy(tmpbuf + shift, buf, cnt);
-		err = sd_write(intfhdl, ftaddr, n, tmpbuf);
-		kfree(tmpbuf);
-	}
-	return err;
-}
-
-static void sdio_read_mem(
-	struct intf_hdl *intfhdl,
-	u32 addr,
-	u32 cnt,
-	u8 *rmem
-)
-{
-	sdio_readN(intfhdl, addr, cnt, rmem);
-}
-
-static void sdio_write_mem(
-	struct intf_hdl *intfhdl,
-	u32 addr,
-	u32 cnt,
-	u8 *wmem
-)
-{
-	sdio_writeN(intfhdl, addr, cnt, wmem);
-}
-
 /*
  * Description:
  *Read from RX FIFO
@@ -463,14 +351,10 @@ void sdio_set_intf_ops(struct adapter *adapter, struct _io_ops *ops)
 	ops->_read8 = &sdio_read8;
 	ops->_read16 = &sdio_read16;
 	ops->_read32 = &sdio_read32;
-	ops->_read_mem = &sdio_read_mem;
-	ops->_read_port = &sdio_read_port;
 
 	ops->_write8 = &sdio_write8;
 	ops->_write16 = &sdio_write16;
 	ops->_write32 = &sdio_write32;
-	ops->_writeN = &sdio_writeN;
-	ops->_write_mem = &sdio_write_mem;
 	ops->_write_port = &sdio_write_port;
 }
 
@@ -712,17 +596,14 @@ void InitSysInterrupt8723BSdio(struct adapter *adapter)
 	haldata->SysIntrMask = (0);
 }
 
-/*  */
-/*	Description: */
-/*		Enalbe SDIO Host Interrupt Mask configuration on SDIO local domain. */
-/*  */
-/*	Assumption: */
-/*		1. Using SDIO Local register ONLY for configuration. */
-/*		2. PASSIVE LEVEL */
-/*  */
-/*	Created by Roger, 2011.02.11. */
-/*  */
-void EnableInterrupt8723BSdio(struct adapter *adapter)
+/*
+ * Enable SDIO Host Interrupt Mask configuration on SDIO local domain.
+ *
+ * Assumption:
+ *	1. Using SDIO Local register ONLY for configuration.
+ *	2. PASSIVE LEVEL
+ */
+void rtw_sdio_enable_interrupt(struct adapter *adapter)
 {
 	struct hal_com_data *haldata;
 	__le32 himr;
@@ -745,16 +626,13 @@ void EnableInterrupt8723BSdio(struct adapter *adapter)
 	rtw_write8(adapter, REG_C2HEVT_CLEAR, C2H_EVT_HOST_CLOSE);
 }
 
-/*  */
-/*	Description: */
-/*		Disable SDIO Host IMR configuration to mask unnecessary interrupt service. */
-/*  */
-/*	Assumption: */
-/*		Using SDIO Local register ONLY for configuration. */
-/*  */
-/*	Created by Roger, 2011.02.11. */
-/*  */
-void DisableInterrupt8723BSdio(struct adapter *adapter)
+/*
+ * Disable SDIO Host IMR configuration to mask unnecessary interrupt service.
+ *
+ * Assumption:
+ *	Using SDIO Local register ONLY for configuration.
+ */
+void rtw_sdio_disable_interrupt(struct adapter *adapter)
 {
 	__le32 himr;
 
diff --git a/drivers/staging/rtl8723bs/include/HalVerDef.h b/drivers/staging/rtl8723bs/include/HalVerDef.h
deleted file mode 100644
index d0ce21ccc1cc6..0000000000000
--- a/drivers/staging/rtl8723bs/include/HalVerDef.h
+++ /dev/null
@@ -1,85 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/******************************************************************************
- *
- * Copyright(c) 2007 - 2011 Realtek Corporation. All rights reserved.
- *
- ******************************************************************************/
-#ifndef __HAL_VERSION_DEF_H__
-#define __HAL_VERSION_DEF_H__
-
-/*  hal_ic_type_e */
-enum hal_ic_type_e { /* tag_HAL_IC_Type_Definition */
-	CHIP_8723B	=	8,
-};
-
-/* hal_chip_type_e */
-enum hal_chip_type_e { /* tag_HAL_CHIP_Type_Definition */
-	TEST_CHIP		=	0,
-	NORMAL_CHIP	=	1,
-	FPGA			=	2,
-};
-
-/* hal_cut_version_e */
-enum hal_cut_version_e { /* tag_HAL_Cut_Version_Definition */
-	A_CUT_VERSION		=	0,
-	B_CUT_VERSION		=	1,
-	C_CUT_VERSION		=	2,
-	D_CUT_VERSION		=	3,
-	E_CUT_VERSION		=	4,
-	F_CUT_VERSION		=	5,
-	G_CUT_VERSION		=	6,
-	H_CUT_VERSION		=	7,
-	I_CUT_VERSION		=	8,
-	J_CUT_VERSION		=	9,
-	K_CUT_VERSION		=	10,
-};
-
-/*  HAL_Manufacturer */
-enum hal_vendor_e { /* tag_HAL_Manufacturer_Version_Definition */
-	CHIP_VENDOR_TSMC	=	0,
-	CHIP_VENDOR_UMC		=	1,
-	CHIP_VENDOR_SMIC	=	2,
-};
-
-struct hal_version { /* tag_HAL_VERSION */
-	enum hal_ic_type_e		ICType;
-	enum hal_chip_type_e		ChipType;
-	enum hal_cut_version_e	CUTVersion;
-	enum hal_vendor_e		VendorType;
-	u8 			ROMVer;
-};
-
-/* hal_version			VersionID; */
-
-/*  Get element */
-#define GET_CVID_IC_TYPE(version)			((enum hal_ic_type_e)((version).ICType))
-#define GET_CVID_CHIP_TYPE(version)			((enum hal_chip_type_e)((version).ChipType))
-#define GET_CVID_MANUFACTUER(version)		((enum hal_vendor_e)((version).VendorType))
-#define GET_CVID_CUT_VERSION(version)		((enum hal_cut_version_e)((version).CUTVersion))
-#define GET_CVID_ROM_VERSION(version)		(((version).ROMVer) & ROM_VERSION_MASK)
-
-/*  */
-/* Common Macro. -- */
-/*  */
-/* hal_version VersionID */
-
-/* hal_chip_type_e */
-#define IS_TEST_CHIP(version)			((GET_CVID_CHIP_TYPE(version) == TEST_CHIP) ? true : false)
-#define IS_NORMAL_CHIP(version)			((GET_CVID_CHIP_TYPE(version) == NORMAL_CHIP) ? true : false)
-
-/* hal_cut_version_e */
-#define IS_A_CUT(version)				((GET_CVID_CUT_VERSION(version) == A_CUT_VERSION) ? true : false)
-#define IS_B_CUT(version)				((GET_CVID_CUT_VERSION(version) == B_CUT_VERSION) ? true : false)
-#define IS_C_CUT(version)				((GET_CVID_CUT_VERSION(version) == C_CUT_VERSION) ? true : false)
-#define IS_D_CUT(version)				((GET_CVID_CUT_VERSION(version) == D_CUT_VERSION) ? true : false)
-#define IS_E_CUT(version)				((GET_CVID_CUT_VERSION(version) == E_CUT_VERSION) ? true : false)
-#define IS_I_CUT(version)				((GET_CVID_CUT_VERSION(version) == I_CUT_VERSION) ? true : false)
-#define IS_J_CUT(version)				((GET_CVID_CUT_VERSION(version) == J_CUT_VERSION) ? true : false)
-#define IS_K_CUT(version)				((GET_CVID_CUT_VERSION(version) == K_CUT_VERSION) ? true : false)
-
-/* hal_vendor_e */
-#define IS_CHIP_VENDOR_TSMC(version)	((GET_CVID_MANUFACTUER(version) == CHIP_VENDOR_TSMC) ? true : false)
-#define IS_CHIP_VENDOR_UMC(version)	((GET_CVID_MANUFACTUER(version) == CHIP_VENDOR_UMC) ? true : false)
-#define IS_CHIP_VENDOR_SMIC(version)	((GET_CVID_MANUFACTUER(version) == CHIP_VENDOR_SMIC) ? true : false)
-
-#endif
diff --git a/drivers/staging/rtl8723bs/include/cmd_osdep.h b/drivers/staging/rtl8723bs/include/cmd_osdep.h
index 5506f513dc019..41daa3c339a87 100644
--- a/drivers/staging/rtl8723bs/include/cmd_osdep.h
+++ b/drivers/staging/rtl8723bs/include/cmd_osdep.h
@@ -7,7 +7,6 @@
 #ifndef __CMD_OSDEP_H_
 #define __CMD_OSDEP_H_
 
-
 int rtw_init_cmd_priv(struct	cmd_priv *pcmdpriv);
 int rtw_init_evt_priv(struct evt_priv *pevtpriv);
 extern void _rtw_free_evt_priv(struct	evt_priv *pevtpriv);
diff --git a/drivers/staging/rtl8723bs/include/drv_types.h b/drivers/staging/rtl8723bs/include/drv_types.h
index 7ed375ba18d8c..552d0c5fa47ff 100644
--- a/drivers/staging/rtl8723bs/include/drv_types.h
+++ b/drivers/staging/rtl8723bs/include/drv_types.h
@@ -258,9 +258,6 @@ static inline struct dvobj_priv *pwrctl_to_dvobj(struct pwrctrl_priv *pwrctl_pri
 static inline struct device *dvobj_to_dev(struct dvobj_priv *dvobj)
 {
 	/* todo: get interface type from dvobj and the return the dev accordingly */
-#ifdef RTW_DVOBJ_CHIP_HW_TYPE
-#endif
-
 	return &dvobj->intf_data.func->dev;
 }
 
@@ -383,9 +380,9 @@ struct adapter {
 /*  */
 /*  Function disabled. */
 /*  */
-#define DF_TX_BIT		BIT0
-#define DF_RX_BIT		BIT1
-#define DF_IO_BIT		BIT2
+#define DF_TX_BIT		BIT(0)
+#define DF_RX_BIT		BIT(1)
+#define DF_IO_BIT		BIT(2)
 
 /* define RTW_ENABLE_FUNC(padapter, func) (atomic_sub(&adapter_to_dvobj(padapter)->disable_func, (func))) */
 
diff --git a/drivers/staging/rtl8723bs/include/hal_com.h b/drivers/staging/rtl8723bs/include/hal_com.h
index 483f0390addca..7bbf62cfd8257 100644
--- a/drivers/staging/rtl8723bs/include/hal_com.h
+++ b/drivers/staging/rtl8723bs/include/hal_com.h
@@ -7,7 +7,6 @@
 #ifndef __HAL_COMMON_H__
 #define __HAL_COMMON_H__
 
-#include "HalVerDef.h"
 #include "hal_pg.h"
 #include "hal_phy.h"
 #include "hal_phy_reg.h"
@@ -92,8 +91,6 @@ enum rt_media_status {
 u8 rtw_hal_data_init(struct adapter *padapter);
 void rtw_hal_data_deinit(struct adapter *padapter);
 
-void dump_chip_info(struct hal_version	ChipVersion);
-
 u8 /* return the final channel plan decision */
 hal_com_config_channel_plan(
 struct adapter *padapter,
diff --git a/drivers/staging/rtl8723bs/include/hal_com_reg.h b/drivers/staging/rtl8723bs/include/hal_com_reg.h
index cf5c15dc2bfd6..479748d856261 100644
--- a/drivers/staging/rtl8723bs/include/hal_com_reg.h
+++ b/drivers/staging/rtl8723bs/include/hal_com_reg.h
@@ -204,22 +204,22 @@
 /*  */
 /*        8723/8188E Host System Interrupt Status Register (offset 0x5C, 32 byte) */
 /*  */
-#define HSISR_GPIO12_0_INT				BIT0
-#define HSISR_SPS_OCP_INT				BIT5
-#define HSISR_RON_INT					BIT6
-#define HSISR_PDNINT					BIT7
-#define HSISR_GPIO9_INT					BIT25
+#define HSISR_GPIO12_0_INT				BIT(0)
+#define HSISR_SPS_OCP_INT				BIT(5)
+#define HSISR_RON_INT					BIT(6)
+#define HSISR_PDNINT					BIT(7)
+#define HSISR_GPIO9_INT					BIT(25)
 
 /*  */
 /*        Response Rate Set Register	(offset 0x440, 24bits) */
 /*  */
-#define RRSR_1M					BIT0
-#define RRSR_2M					BIT1
-#define RRSR_5_5M				BIT2
-#define RRSR_11M				BIT3
-#define RRSR_6M					BIT4
-#define RRSR_12M				BIT6
-#define RRSR_24M				BIT8
+#define RRSR_1M					BIT(0)
+#define RRSR_2M					BIT(1)
+#define RRSR_5_5M				BIT(2)
+#define RRSR_11M				BIT(3)
+#define RRSR_6M					BIT(4)
+#define RRSR_12M				BIT(6)
+#define RRSR_24M				BIT(8)
 
 #define RRSR_CCK_RATES (RRSR_11M|RRSR_5_5M|RRSR_2M|RRSR_1M)
 
@@ -250,12 +250,12 @@
 /*  */
 /*        BW_OPMODE bits				(Offset 0x603, 8bit) */
 /*  */
-#define BW_OPMODE_20MHZ			BIT2
+#define BW_OPMODE_20MHZ			BIT(2)
 
 /*  */
 /*        CAM Config Setting (offset 0x680, 1 byte) */
 /*  */
-#define CAM_VALID				BIT15
+#define CAM_VALID				BIT(15)
 
 #define CAM_CONTENT_COUNT	8
 
@@ -263,8 +263,8 @@
 
 #define TOTAL_CAM_ENTRY		32
 
-#define CAM_WRITE				BIT16
-#define CAM_POLLINIG			BIT31
+#define CAM_WRITE				BIT(16)
+#define CAM_POLLINIG			BIT(31)
 
 /*  */
 /*  12. Host Interrupt Status Registers */
@@ -273,20 +273,20 @@
 /*  */
 /*        8192C (RCR) Receive Configuration Register	(Offset 0x608, 32 bits) */
 /*  */
-#define RCR_APPFCS				BIT31	/*  WMAC append FCS after pauload */
-#define RCR_APP_MIC				BIT30	/*  MACRX will retain the MIC at the bottom of the packet. */
-#define RCR_APP_ICV				BIT29	/*  MACRX will retain the ICV at the bottom of the packet. */
-#define RCR_APP_PHYST_RXFF		BIT28	/*  PHY Status is appended before RX packet in RXFF */
-#define RCR_APP_BA_SSN			BIT27	/*  SSN of previous TXBA is appended as after original RXDESC as the 4-th DW of RXDESC. */
-#define RCR_HTC_LOC_CTRL		BIT14	/*  MFC<--HTC = 1 MFC-->HTC = 0 */
-#define RCR_AMF					BIT13	/*  Accept management type frame */
-#define RCR_ADF					BIT11	/*  Accept data type frame. This bit also regulates BA, BAR, and PS-Poll (AP mode only). */
-#define RCR_ACRC32				BIT8		/*  Accept CRC32 error packet */
-#define RCR_CBSSID_BCN			BIT7		/*  Accept BSSID match packet (Rx beacon, probe rsp) */
-#define RCR_CBSSID_DATA		BIT6		/*  Accept BSSID match packet (Data) */
-#define RCR_AB					BIT3		/*  Accept broadcast packet */
-#define RCR_AM					BIT2		/*  Accept multicast packet */
-#define RCR_APM					BIT1		/*  Accept physical match packet */
+#define RCR_APPFCS				BIT(31)	/*  WMAC append FCS after pauload */
+#define RCR_APP_MIC				BIT(30)	/*  MACRX will retain the MIC at the bottom of the packet. */
+#define RCR_APP_ICV				BIT(29)	/*  MACRX will retain the ICV at the bottom of the packet. */
+#define RCR_APP_PHYST_RXFF		BIT(28)	/*  PHY Status is appended before RX packet in RXFF */
+#define RCR_APP_BA_SSN			BIT(27)	/*  SSN of previous TXBA is appended as after original RXDESC as the 4-th DW of RXDESC. */
+#define RCR_HTC_LOC_CTRL		BIT(14)	/*  MFC<--HTC = 1 MFC-->HTC = 0 */
+#define RCR_AMF					BIT(13)	/*  Accept management type frame */
+#define RCR_ADF					BIT(11)	/*  Accept data type frame. This bit also regulates BA, BAR, and PS-Poll (AP mode only). */
+#define RCR_ACRC32				BIT(8)		/*  Accept CRC32 error packet */
+#define RCR_CBSSID_BCN			BIT(7)		/*  Accept BSSID match packet (Rx beacon, probe rsp) */
+#define RCR_CBSSID_DATA		BIT(6)		/*  Accept BSSID match packet (Data) */
+#define RCR_AB					BIT(3)		/*  Accept broadcast packet */
+#define RCR_AM					BIT(2)		/*  Accept multicast packet */
+#define RCR_APM					BIT(1)		/*  Accept physical match packet */
 
 
 /*  */
@@ -540,26 +540,26 @@
 #define SDIO_HIMR_DISABLED			0
 
 /*  RTL8723/RTL8188E SDIO Host Interrupt Mask Register */
-#define SDIO_HIMR_RX_REQUEST_MSK		BIT0
-#define SDIO_HIMR_AVAL_MSK			BIT1
+#define SDIO_HIMR_RX_REQUEST_MSK		BIT(0)
+#define SDIO_HIMR_AVAL_MSK			BIT(1)
 
 /*  SDIO Host Interrupt Service Routine */
-#define SDIO_HISR_RX_REQUEST			BIT0
-#define SDIO_HISR_AVAL					BIT1
-#define SDIO_HISR_TXERR					BIT2
-#define SDIO_HISR_RXERR					BIT3
-#define SDIO_HISR_TXFOVW				BIT4
-#define SDIO_HISR_RXFOVW				BIT5
-#define SDIO_HISR_TXBCNOK				BIT6
-#define SDIO_HISR_TXBCNERR				BIT7
-#define SDIO_HISR_C2HCMD				BIT17
-#define SDIO_HISR_CPWM1				BIT18
-#define SDIO_HISR_CPWM2				BIT19
-#define SDIO_HISR_HSISR_IND			BIT20
-#define SDIO_HISR_GTINT3_IND			BIT21
-#define SDIO_HISR_GTINT4_IND			BIT22
-#define SDIO_HISR_PSTIMEOUT			BIT23
-#define SDIO_HISR_OCPINT				BIT24
+#define SDIO_HISR_RX_REQUEST			BIT(0)
+#define SDIO_HISR_AVAL					BIT(1)
+#define SDIO_HISR_TXERR					BIT(2)
+#define SDIO_HISR_RXERR					BIT(3)
+#define SDIO_HISR_TXFOVW				BIT(4)
+#define SDIO_HISR_RXFOVW				BIT(5)
+#define SDIO_HISR_TXBCNOK				BIT(6)
+#define SDIO_HISR_TXBCNERR				BIT(7)
+#define SDIO_HISR_C2HCMD				BIT(17)
+#define SDIO_HISR_CPWM1				BIT(18)
+#define SDIO_HISR_CPWM2				BIT(19)
+#define SDIO_HISR_HSISR_IND			BIT(20)
+#define SDIO_HISR_GTINT3_IND			BIT(21)
+#define SDIO_HISR_GTINT4_IND			BIT(22)
+#define SDIO_HISR_PSTIMEOUT			BIT(23)
+#define SDIO_HISR_OCPINT				BIT(24)
 
 #define MASK_SDIO_HISR_CLEAR		(SDIO_HISR_TXERR |\
 									SDIO_HISR_RXERR |\
@@ -590,9 +590,9 @@
 #define C2H_EVT_FW_CLOSE		0xFF	/*  Set by FW indicating that FW had set the C2H command message and it's not yet read by driver. */
 
 /* 2REG_MULTI_FUNC_CTRL(For RTL8723 Only) */
-#define WL_HWPDN_SL			BIT1	/*  WiFi HW PDn polarity control */
-#define WL_FUNC_EN				BIT2	/*  WiFi function enable */
-#define BT_FUNC_EN				BIT18	/*  BT function enable */
-#define GPS_FUNC_EN			BIT22	/*  GPS function enable */
+#define WL_HWPDN_SL			BIT(1)	/*  WiFi HW PDn polarity control */
+#define WL_FUNC_EN				BIT(2)	/*  WiFi function enable */
+#define BT_FUNC_EN				BIT(18)	/*  BT function enable */
+#define GPS_FUNC_EN			BIT(22)	/*  GPS function enable */
 
 #endif /* __HAL_COMMON_H__ */
diff --git a/drivers/staging/rtl8723bs/include/hal_data.h b/drivers/staging/rtl8723bs/include/hal_data.h
index ff4383e30322f..bc9d8c945f38f 100644
--- a/drivers/staging/rtl8723bs/include/hal_data.h
+++ b/drivers/staging/rtl8723bs/include/hal_data.h
@@ -77,7 +77,7 @@ enum {
 struct dm_priv {
 	u8 DM_Type;
 
-#define DYNAMIC_FUNC_BT BIT0
+#define DYNAMIC_FUNC_BT BIT(0)
 
 	u8 DMFlag;
 	u8 InitDMFlag;
@@ -162,7 +162,6 @@ struct dm_priv {
 
 
 struct hal_com_data {
-	struct hal_version VersionID;
 	enum rt_multi_func MultiFunc; /*  For multi-function consideration. */
 	enum rt_polarity_ctl PolarityCtl; /*  For Wifi PDn Polarity control. */
 	enum rt_regulator_mode	RegulatorMode; /*  switching regulator or LDO */
@@ -391,6 +390,9 @@ struct hal_com_data {
 	/*  Interrupt related register information. */
 	u32 		SysIntrStatus;
 	u32 		SysIntrMask;
+
+	/* Chip version information */
+	bool chip_normal;	/* true - normal chip, false - test chip */
 };
 
 #define GET_HAL_DATA(__padapter)	((struct hal_com_data *)((__padapter)->HalData))
diff --git a/drivers/staging/rtl8723bs/include/hal_intf.h b/drivers/staging/rtl8723bs/include/hal_intf.h
index b193854bfe6e9..9b000681ff80f 100644
--- a/drivers/staging/rtl8723bs/include/hal_intf.h
+++ b/drivers/staging/rtl8723bs/include/hal_intf.h
@@ -9,10 +9,10 @@
 
 
 enum {
-	RTW_PCIE	= BIT0,
-	RTW_USB		= BIT1,
-	RTW_SDIO	= BIT2,
-	RTW_GSPI	= BIT3,
+	RTW_PCIE	= BIT(0),
+	RTW_USB		= BIT(1),
+	RTW_SDIO	= BIT(2),
+	RTW_GSPI	= BIT(3),
 };
 
 enum {
@@ -161,10 +161,10 @@ enum hal_intf_ps_func {
 typedef s32 (*c2h_id_filter)(u8 *c2h_evt);
 
 #define RF_CHANGE_BY_INIT	0
-#define RF_CHANGE_BY_IPS	BIT28
-#define RF_CHANGE_BY_PS		BIT29
-#define RF_CHANGE_BY_HW		BIT30
-#define RF_CHANGE_BY_SW		BIT31
+#define RF_CHANGE_BY_IPS	BIT(28)
+#define RF_CHANGE_BY_PS		BIT(29)
+#define RF_CHANGE_BY_HW		BIT(30)
+#define RF_CHANGE_BY_SW		BIT(31)
 
 #define GET_EEPROM_EFUSE_PRIV(adapter) (&adapter->eeprompriv)
 #define is_boot_from_eeprom(adapter) (adapter->eeprompriv.EepromOrEfuse)
@@ -181,12 +181,6 @@ typedef s32 (*c2h_id_filter)(u8 *c2h_evt);
 #define	RX_PNOWakeUp			0x55
 #define	AP_WakeUp			0x66
 
-void rtw_hal_def_value_init(struct adapter *padapter);
-
-void rtw_hal_free_data(struct adapter *padapter);
-
-void rtw_hal_dm_init(struct adapter *padapter);
-
 uint rtw_hal_init(struct adapter *padapter);
 uint rtw_hal_deinit(struct adapter *padapter);
 void rtw_hal_stop(struct adapter *padapter);
@@ -195,17 +189,10 @@ void rtw_hal_get_hwreg(struct adapter *padapter, u8 variable, u8 *val);
 
 void rtw_hal_set_hwreg_with_buf(struct adapter *padapter, u8 variable, u8 *pbuf, int len);
 
-void rtw_hal_chip_configure(struct adapter *padapter);
-void rtw_hal_read_chip_info(struct adapter *padapter);
-void rtw_hal_read_chip_version(struct adapter *padapter);
-
 u8 rtw_hal_get_def_var(struct adapter *padapter, enum hal_def_variable eVariable, void *pValue);
 
 void rtw_hal_set_odm_var(struct adapter *padapter, enum hal_odm_variable eVariable, void *pValue1, bool bSet);
 
-void rtw_hal_enable_interrupt(struct adapter *padapter);
-void rtw_hal_disable_interrupt(struct adapter *padapter);
-
 u8 rtw_hal_check_ips_status(struct adapter *padapter);
 
 s32	rtw_hal_xmitframe_enqueue(struct adapter *padapter, struct xmit_frame *pxmitframe);
diff --git a/drivers/staging/rtl8723bs/include/hal_phy.h b/drivers/staging/rtl8723bs/include/hal_phy.h
index 3d71a4f415923..abc0f27fdaa46 100644
--- a/drivers/staging/rtl8723bs/include/hal_phy.h
+++ b/drivers/staging/rtl8723bs/include/hal_phy.h
@@ -10,8 +10,8 @@
 /*  Antenna detection method, i.e., using single tone detection or RSSI reported from each antenna detected. */
 /*  Added by Roger, 2013.05.22. */
 /*  */
-#define ANT_DETECT_BY_SINGLE_TONE	BIT0
-#define ANT_DETECT_BY_RSSI				BIT1
+#define ANT_DETECT_BY_SINGLE_TONE	BIT(0)
+#define ANT_DETECT_BY_RSSI				BIT(1)
 #define IS_ANT_DETECT_SUPPORT_SINGLE_TONE(__Adapter)		((GET_HAL_DATA(__Adapter)->AntDetection) & ANT_DETECT_BY_SINGLE_TONE)
 #define IS_ANT_DETECT_SUPPORT_RSSI(__Adapter)		((GET_HAL_DATA(__Adapter)->AntDetection) & ANT_DETECT_BY_RSSI)
 
diff --git a/drivers/staging/rtl8723bs/include/hal_pwr_seq.h b/drivers/staging/rtl8723bs/include/hal_pwr_seq.h
index 48bf7f66a06eb..9639eef8a51d2 100644
--- a/drivers/staging/rtl8723bs/include/hal_pwr_seq.h
+++ b/drivers/staging/rtl8723bs/include/hal_pwr_seq.h
@@ -39,92 +39,92 @@
 #define RTL8723B_TRANS_CARDEMU_TO_ACT														\
 	/* format */																\
 	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value },  comments here*/								\
-	{0x0020, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0}, /*0x20[0] = 1b'1 enable LDOA12 MACRO block for all interface*/   \
-	{0x0067, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4, 0}, /*0x67[0] = 0 to disable BT_GPS_SEL pins*/	\
+	{0x0020, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)}, /*0x20[0] = 1b'1 enable LDOA12 MACRO block for all interface*/   \
+	{0x0067, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), 0}, /*0x67[0] = 0 to disable BT_GPS_SEL pins*/	\
 	{0x0001, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_DELAY, 1, PWRSEQ_DELAY_MS},/*Delay 1ms*/   \
-	{0x0000, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT5, 0}, /*0x00[5] = 1b'0 release analog Ips to digital , 1:isolation*/   \
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, (BIT4|BIT3|BIT2), 0},/* disable SW LPS 0x04[10]= 0 and WLSUS_EN 0x04[11]= 0*/	\
-	{0x0075, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/* Disable USB suspend */	\
-	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT1, BIT1},/* wait till 0x04[17] = 1    power ready*/	\
-	{0x0075, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0},/* Enable USB suspend */	\
-	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/* release WLON reset  0x04[16]= 1*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT7, 0},/* disable HWPDN 0x04[15]= 0*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, (BIT4|BIT3), 0},/* disable WL suspend*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/* polling until return 0*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT0, 0},/**/	\
-	{0x0010, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT6, BIT6},/* Enable WL control XTAL setting*/	\
-	{0x0049, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, BIT1},/*Enable falling edge triggering interrupt*/\
-	{0x0063, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, BIT1},/*Enable GPIO9 interrupt mode*/\
-	{0x0062, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, 0},/*Enable GPIO9 input mode*/\
-	{0x0058, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/*Enable HSISR GPIO[C:0] interrupt*/\
-	{0x005A, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, BIT1},/*Enable HSISR GPIO9 interrupt*/\
-	{0x0068, PWR_CUT_TESTCHIP_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT3, BIT3},/*For GPIO9 internal pull high setting by test chip*/\
-	{0x0069, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT6, BIT6},/*For GPIO9 internal pull high setting*/\
+	{0x0000, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(5), 0}, /*0x00[5] = 1b'0 release analog Ips to digital , 1:isolation*/   \
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, (BIT(4)|BIT(3)|BIT(2)), 0},/* disable SW LPS 0x04[10]= 0 and WLSUS_EN 0x04[11]= 0*/	\
+	{0x0075, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/* Disable USB suspend */	\
+	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(1), BIT(1)},/* wait till 0x04[17] = 1    power ready*/	\
+	{0x0075, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), (0)},/* Enable USB suspend */	\
+	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/* release WLON reset  0x04[16]= 1*/	\
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(7), 0},/* disable HWPDN 0x04[15]= 0*/	\
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, (BIT(4)|BIT(3)), 0},/* disable WL suspend*/	\
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/* polling until return 0*/	\
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(0), 0},/**/	\
+	{0x0010, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(6), BIT(6)},/* Enable WL control XTAL setting*/	\
+	{0x0049, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), BIT(1)},/*Enable falling edge triggering interrupt*/\
+	{0x0063, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), BIT(1)},/*Enable GPIO9 interrupt mode*/\
+	{0x0062, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), 0},/*Enable GPIO9 input mode*/\
+	{0x0058, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/*Enable HSISR GPIO[C:0] interrupt*/\
+	{0x005A, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), BIT(1)},/*Enable HSISR GPIO9 interrupt*/\
+	{0x0068, PWR_CUT_TESTCHIP_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3), BIT(3)},/*For GPIO9 internal pull high setting by test chip*/\
+	{0x0069, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(6), BIT(6)},/*For GPIO9 internal pull high setting*/\
 
 
 #define RTL8723B_TRANS_ACT_TO_CARDEMU													\
 	/* format */																\
 	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value },  comments here*/								\
 	{0x001F, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0},/*0x1F[7:0] = 0 turn off RF*/	\
-	{0x0049, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, 0},/*Enable rising edge triggering interrupt*/ \
-	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/* release WLON reset  0x04[16]= 1*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, BIT1}, /*0x04[9] = 1 turn off MAC by HW state machine*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT1, 0}, /*wait till 0x04[9] = 0 polling until return 0 to disable*/	\
-	{0x0010, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT6, 0},/* Enable BT control XTAL setting*/\
-	{0x0000, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT5, BIT5}, /*0x00[5] = 1b'1 analog Ips to digital , 1:isolation*/   \
-	{0x0020, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0}, /*0x20[0] = 1b'0 disable LDOA12 MACRO block*/\
+	{0x0049, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), 0},/*Enable rising edge triggering interrupt*/ \
+	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/* release WLON reset  0x04[16]= 1*/	\
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), BIT(1)}, /*0x04[9] = 1 turn off MAC by HW state machine*/	\
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(1), 0}, /*wait till 0x04[9] = 0 polling until return 0 to disable*/	\
+	{0x0010, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(6), 0},/* Enable BT control XTAL setting*/\
+	{0x0000, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(5), BIT(5)}, /*0x00[5] = 1b'1 analog Ips to digital , 1:isolation*/   \
+	{0x0020, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0}, /*0x20[0] = 1b'0 disable LDOA12 MACRO block*/\
 
 
 #define RTL8723B_TRANS_CARDEMU_TO_SUS													\
 	/* format */																\
 	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value },  comments here*/								\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4|BIT3, (BIT4|BIT3)}, /*0x04[12:11] = 2b'11 enable WL suspend for PCIe*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT3|BIT4, BIT3}, /*0x04[12:11] = 2b'01 enable WL suspend*/	\
-	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4, BIT4}, /*0x23[4] = 1b'1 12H LDO enter sleep mode*/   \
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4)|BIT(3), (BIT(4)|BIT(3))}, /*0x04[12:11] = 2b'11 enable WL suspend for PCIe*/	\
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3)|BIT(4), BIT(3)}, /*0x04[12:11] = 2b'01 enable WL suspend*/	\
+	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), BIT(4)}, /*0x23[4] = 1b'1 12H LDO enter sleep mode*/   \
 	{0x0007, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x20}, /*0x07[7:0] = 0x20 SDIO SOP option to disable BG/MB/ACK/SWR*/   \
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT3|BIT4, BIT3|BIT4}, /*0x04[12:11] = 2b'11 enable WL suspend for PCIe*/	\
-	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_WRITE, BIT0, BIT0}, /*Set SDIO suspend local register*/	\
-	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_POLLING, BIT1, 0}, /*wait power state to suspend*/
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3)|BIT(4), BIT(3)|BIT(4)}, /*0x04[12:11] = 2b'11 enable WL suspend for PCIe*/	\
+	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_WRITE, BIT(0), BIT(0)}, /*Set SDIO suspend local register*/	\
+	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_POLLING, BIT(1), 0}, /*wait power state to suspend*/
 
 #define RTL8723B_TRANS_SUS_TO_CARDEMU													\
 	/* format */																\
 	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value },  comments here*/								\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT3 | BIT7, 0}, /*clear suspend enable and power down enable*/	\
-	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_WRITE, BIT0, 0}, /*Set SDIO suspend local register*/	\
-	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_POLLING, BIT1, BIT1}, /*wait power state to suspend*/\
-	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4, 0}, /*0x23[4] = 1b'0 12H LDO enter normal mode*/   \
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT3|BIT4, 0}, /*0x04[12:11] = 2b'01enable WL suspend*/
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3) | BIT(7), 0}, /*clear suspend enable and power down enable*/	\
+	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_WRITE, BIT(0), 0}, /*Set SDIO suspend local register*/	\
+	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_POLLING, BIT(1), BIT(1)}, /*wait power state to suspend*/\
+	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), 0}, /*0x23[4] = 1b'0 12H LDO enter normal mode*/   \
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3)|BIT(4), 0}, /*0x04[12:11] = 2b'01enable WL suspend*/
 
 #define RTL8723B_TRANS_CARDEMU_TO_CARDDIS													\
 	/* format */																\
 	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value }, omments here*/								\
 	{0x0007, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x20}, /*0x07 = 0x20 , SOP option to disable BG/MB*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT3|BIT4, BIT3}, /*0x04[12:11] = 2b'01 enable WL suspend*/	\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT2, BIT2}, /*0x04[10] = 1, enable SW LPS*/	\
-	{0x004A, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 1}, /*0x48[16] = 1 to enable GPIO9 as EXT WAKEUP*/   \
-	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4, BIT4}, /*0x23[4] = 1b'1 12H LDO enter sleep mode*/   \
-	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_WRITE, BIT0, BIT0}, /*Set SDIO suspend local register*/	\
-	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_POLLING, BIT1, 0}, /*wait power state to suspend*/
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK|PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3)|BIT(4), BIT(3)}, /*0x04[12:11] = 2b'01 enable WL suspend*/	\
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(2), BIT(2)}, /*0x04[10] = 1, enable SW LPS*/	\
+	{0x004A, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 1}, /*0x48[16] = 1 to enable GPIO9 as EXT WAKEUP*/   \
+	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), BIT(4)}, /*0x23[4] = 1b'1 12H LDO enter sleep mode*/   \
+	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_WRITE, BIT(0), BIT(0)}, /*Set SDIO suspend local register*/	\
+	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_POLLING, BIT(1), 0}, /*wait power state to suspend*/
 
 #define RTL8723B_TRANS_CARDDIS_TO_CARDEMU													\
 	/* format */																\
 	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value }, comments here*/								\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT3 | BIT7, 0}, /*clear suspend enable and power down enable*/	\
-	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_WRITE, BIT0, 0}, /*Set SDIO suspend local register*/	\
-	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_POLLING, BIT1, BIT1}, /*wait power state to suspend*/\
-        {0x004A, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0}, /*0x48[16] = 0 to disable GPIO9 as EXT WAKEUP*/   \
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT3|BIT4, 0}, /*0x04[12:11] = 2b'01enable WL suspend*/\
-	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4, 0}, /*0x23[4] = 1b'0 12H LDO enter normal mode*/   \
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3) | BIT(7), 0}, /*clear suspend enable and power down enable*/	\
+	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_WRITE, BIT(0), 0}, /*Set SDIO suspend local register*/	\
+	{0x0086, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_SDIO, PWR_CMD_POLLING, BIT(1), BIT(1)}, /*wait power state to suspend*/\
+	    {0x004A, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0}, /*0x48[16] = 0 to disable GPIO9 as EXT WAKEUP*/   \
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(3)|BIT(4), 0}, /*0x04[12:11] = 2b'01enable WL suspend*/\
+	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), 0}, /*0x23[4] = 1b'0 12H LDO enter normal mode*/   \
 	{0x0301, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0},/*PCIe DMA start*/
 
 
 #define RTL8723B_TRANS_CARDEMU_TO_PDN												\
 	/* format */																\
 	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value }, comments here*/								\
-	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4, BIT4}, /*0x23[4] = 1b'1 12H LDO enter sleep mode*/   \
+	{0x0023, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), BIT(4)}, /*0x23[4] = 1b'1 12H LDO enter sleep mode*/   \
 	{0x0007, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK|PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x20}, /*0x07[7:0] = 0x20 SOP option to disable BG/MB/ACK/SWR*/   \
-	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0},/* 0x04[16] = 0*/\
-	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT7, BIT7},/* 0x04[15] = 1*/
+	{0x0006, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0},/* 0x04[16] = 0*/\
+	{0x0005, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(7), BIT(7)},/* 0x04[15] = 1*/
 
 #define RTL8723B_TRANS_ACT_TO_LPS														\
 	/* format */																\
@@ -135,13 +135,13 @@
 	{0x05F9, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, 0xFF, 0},/*Should be zero if no packet is transmitting*/	\
 	{0x05FA, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, 0xFF, 0},/*Should be zero if no packet is transmitting*/	\
 	{0x05FB, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, 0xFF, 0},/*Should be zero if no packet is transmitting*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0},/*CCK and OFDM are disabled, and clock are gated*/	\
+	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0},/*CCK and OFDM are disabled, and clock are gated*/	\
 	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_DELAY, 0, PWRSEQ_DELAY_US},/*Delay 1us*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, 0},/*Whole BB is reset*/	\
+	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), 0},/*Whole BB is reset*/	\
 	{0x0100, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x03},/*Reset MAC TRX*/	\
-	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, 0},/*check if removed later*/	\
+	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), 0},/*check if removed later*/	\
 	{0x0093, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x00},/*When driver enter Sus/ Disable, enable LOP for BT*/	\
-	{0x0553, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT5, BIT5},/*Respond TxOK to scheduler*/	\
+	{0x0553, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(5), BIT(5)},/*Respond TxOK to scheduler*/	\
 
 
 #define RTL8723B_TRANS_LPS_TO_ACT															\
@@ -151,59 +151,59 @@
 	{0xFE58, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_USB_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x84}, /*USB RPWM*/\
 	{0x0361, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_PCI_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x84}, /*PCIe RPWM*/\
 	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_DELAY, 0, PWRSEQ_DELAY_MS}, /*Delay*/\
-	{0x0008, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4, 0}, /*.	0x08[4] = 0		 switch TSF to 40M*/\
-	{0x0109, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT7, 0}, /*Polling 0x109[7]= 0  TSF in 40M*/\
-	{0x0029, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT6|BIT7, 0}, /*.	0x29[7:6] = 2b'00	 enable BB clock*/\
-	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, BIT1}, /*.	0x101[1] = 1*/\
+	{0x0008, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), 0}, /*.	0x08[4] = 0		 switch TSF to 40M*/\
+	{0x0109, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(7), 0}, /*Polling 0x109[7]= 0  TSF in 40M*/\
+	{0x0029, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(6)|BIT(7), 0}, /*.	0x29[7:6] = 2b'00	 enable BB clock*/\
+	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), BIT(1)}, /*.	0x101[1] = 1*/\
 	{0x0100, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0xFF}, /*.	0x100[7:0] = 0xFF	 enable WMAC TRX*/\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1|BIT0, BIT1|BIT0}, /*.	0x02[1:0] = 2b'11	 enable BB macro*/\
+	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1)|BIT(0), BIT(1)|BIT(0)}, /*.	0x02[1:0] = 2b'11	 enable BB macro*/\
 	{0x0522, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_ALL_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0}, /*.	0x522 = 0*/
 
 
  #define RTL8723B_TRANS_ACT_TO_SWLPS														\
 	/* format */																\
 	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value }, comments here*/								\
-	{0x0194, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/*enable 32 K source*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0},/*CCK and OFDM are disabled, and clock are gated*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 1},/*CCK and OFDM are enable*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0},/*CCK and OFDM are disabled, and clock are gated*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 1},/*CCK and OFDM are enable*/	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0},/*CCK and OFDM are disabled, and clock are gated*/	\
+	{0x0194, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/*enable 32 K source*/	\
+	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0},/*CCK and OFDM are disabled, and clock are gated*/	\
+	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 1},/*CCK and OFDM are enable*/	\
+	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0},/*CCK and OFDM are disabled, and clock are gated*/	\
+	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 1},/*CCK and OFDM are enable*/	\
+	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0},/*CCK and OFDM are disabled, and clock are gated*/	\
 	{0x0100, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x3F},/*Reset MAC TRX*/	\
-	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, 0},/*disable security engine*/	\
+	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), 0},/*disable security engine*/	\
 	{0x0093, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x40},/*When driver enter Sus/ Disable, enable LOP for BT*/	\
-	{0x0553, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT5, BIT5},/*reset dual TSF*/	\
-	{0x0003, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT2, 0},/*Reset CPU*/	\
+	{0x0553, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(5), BIT(5)},/*reset dual TSF*/	\
+	{0x0003, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(2), 0},/*Reset CPU*/	\
 	{0x0080, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0},/*Reset MCUFWDL register*/	\
-	{0x001D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0},/*Reset CPU IO Wrapper*/	\
-	{0x001D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 1},/*Reset CPU IO Wrapper*/	\
+	{0x001D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0},/*Reset CPU IO Wrapper*/	\
+	{0x001D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 1},/*Reset CPU IO Wrapper*/	\
 	{0x0287, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, 0xFF, 0},/*polling RXFF packet number = 0 */	\
-	{0x0286, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT1, BIT1},/*polling RXDMA idle */	\
-	{0x013D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/*Clear FW RPWM interrupt */\
-	{0x0139, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/*Set FW RPWM interrupt source*/\
-	{0x0008, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4, BIT4},/*switch TSF to 32K*/\
-	{0x0109, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT7, BIT7},/*polling TSF stable*/\
-	{0x0090, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/*Set FW LPS*/	\
-	{0x0090, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT0, 0},/*polling FW LPS ready */
+	{0x0286, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(1), BIT(1)},/*polling RXDMA idle */	\
+	{0x013D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/*Clear FW RPWM interrupt */\
+	{0x0139, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/*Set FW RPWM interrupt source*/\
+	{0x0008, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), BIT(4)},/*switch TSF to 32K*/\
+	{0x0109, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(7), BIT(7)},/*polling TSF stable*/\
+	{0x0090, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/*Set FW LPS*/	\
+	{0x0090, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(0), 0},/*polling FW LPS ready */
 
 
 #define RTL8723B_TRANS_SWLPS_TO_ACT															\
 	/* format */																\
 	/* { offset, cut_msk, fab_msk|interface_msk, base|cmd, msk, value }, comments here*/								\
-	{0x0008, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT4, 0},/*switch TSF to 32K*/\
-	{0x0109, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT7, 0},/*polling TSF stable*/\
-	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, BIT1}, /*.	0x101[1] = 1, enable security engine*/\
+	{0x0008, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(4), 0},/*switch TSF to 32K*/\
+	{0x0109, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(7), 0},/*polling TSF stable*/\
+	{0x0101, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), BIT(1)}, /*.	0x101[1] = 1, enable security engine*/\
 	{0x0100, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0xFF}, /*.	0x100[7:0] = 0xFF	 enable WMAC TRX*/\
 	{0x06B7, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x09}, /*.	reset MAC rx state machine*/\
 	{0x06B4, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0x86}, /*.	reset MAC rx state machine*/\
-	{0x0080, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT1, BIT1},/* set CPU RAM code ready*/	\
-	{0x001D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, 0},/*Reset CPU IO Wrapper*/	\
-	{0x0003, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT2, 0},/* Enable CPU*/	\
-	{0x001D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0},/*enable CPU IO Wrapper*/	\
-	{0x0003, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT2, BIT2},/* Enable CPU*/	\
-	{0x0080, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT7, BIT7},/*polling FW init ready */	\
-	{0x0080, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT6, BIT6},/*polling FW init ready */	\
-	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT0, BIT0}, /*.	0x02[1:0] = 2b'11	 enable BB macro*/\
+	{0x0080, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(1), BIT(1)},/* set CPU RAM code ready*/	\
+	{0x001D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), 0},/*Reset CPU IO Wrapper*/	\
+	{0x0003, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(2), 0},/* Enable CPU*/	\
+	{0x001D, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)},/*enable CPU IO Wrapper*/	\
+	{0x0003, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(2), BIT(2)},/* Enable CPU*/	\
+	{0x0080, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(7), BIT(7)},/*polling FW init ready */	\
+	{0x0080, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_POLLING, BIT(6), BIT(6)},/*polling FW init ready */	\
+	{0x0002, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, BIT(0), BIT(0)}, /*.	0x02[1:0] = 2b'11	 enable BB macro*/\
 	{0x0522, PWR_CUT_ALL_MSK, PWR_FAB_ALL_MSK, PWR_INTF_SDIO_MSK, PWR_BASEADDR_MAC, PWR_CMD_WRITE, 0xFF, 0}, /*.	0x522 = 0*/
 
 #define RTL8723B_TRANS_END															\
diff --git a/drivers/staging/rtl8723bs/include/ieee80211.h b/drivers/staging/rtl8723bs/include/ieee80211.h
index fbb12fe31a6c2..39ee139f1da96 100644
--- a/drivers/staging/rtl8723bs/include/ieee80211.h
+++ b/drivers/staging/rtl8723bs/include/ieee80211.h
@@ -231,7 +231,7 @@ struct eapol {
 	u8 version;
 	u8 type;
 	u16 length;
-} __attribute__ ((packed));
+} __packed;
 
 #define IEEE80211_FCS_LEN    4
 
@@ -274,7 +274,7 @@ struct ieee80211_snap_hdr {
 	u8    ssap;   /* always 0xAA */
 	u8    ctrl;   /* always 0x03 */
 	u8    oui[P80211_OUI_LEN];    /* organizational universal id */
-} __attribute__ ((packed));
+} __packed;
 
 #define SNAP_SIZE sizeof(struct ieee80211_snap_hdr)
 
@@ -394,18 +394,20 @@ enum {
 };
 
 #define IS_HT_RATE(_rate)				(_rate >= MGN_MCS0 && _rate <= MGN_MCS31)
-#define IS_CCK_RATE(_rate)				(MGN_1M == _rate || _rate == MGN_2M || _rate == MGN_5_5M || _rate == MGN_11M)
-#define IS_OFDM_RATE(_rate)				(MGN_6M <= _rate && _rate <= MGN_54M  && _rate != MGN_11M)
+#define IS_CCK_RATE(_rate)				(_rate == MGN_1M || _rate == MGN_2M || _rate == MGN_5_5M || _rate == MGN_11M)
+#define IS_OFDM_RATE(_rate)				(_rate >= MGN_6M && _rate <= MGN_54M  && _rate != MGN_11M)
 
 
 /* NOTE: This data is for statistical purposes; not all hardware provides this
  *       information for frames received.  Not setting these will not cause
- *       any adverse affects. */
+ *       any adverse affects.
+ */
 
 /* IEEE 802.11 requires that STA supports concurrent reception of at least
  * three fragmented frames. This define can be increased to support more
  * concurrent frames, but it should be noted that each entry can consume about
- * 2 kB of RAM and increasing cache size will slow down frame reassembly. */
+ * 2 kB of RAM and increasing cache size will slow down frame reassembly.
+ */
 #define IEEE80211_FRAG_CACHE_LEN 4
 
 #define SEC_KEY_1         (1<<0)
@@ -431,19 +433,17 @@ enum {
 #define BIP_AAD_SIZE  20
 
 /*
-
- 802.11 data frame from AP
-
-      ,-------------------------------------------------------------------.
-Bytes |  2   |  2   |    6    |    6    |    6    |  2   | 0..2312 |   4  |
-      |------|------|---------|---------|---------|------|---------|------|
-Desc. | ctrl | dura |  DA/RA  |   TA    |    SA   | Sequ |  frame  |  fcs |
-      |      | tion | (BSSID) |         |         | ence |  data   |      |
-      `-------------------------------------------------------------------'
-
-Total: 28-2340 bytes
-
-*/
+ * 802.11 data frame from AP
+ *
+ *       ,-------------------------------------------------------------------.
+ * Bytes |  2   |  2   |    6    |    6    |    6    |  2   | 0..2312 |   4  |
+ *       |------|------|---------|---------|---------|------|---------|------|
+ * Desc. | ctrl | dura |  DA/RA  |   TA    |    SA   | Sequ |  frame  |  fcs |
+ *       |      | tion | (BSSID) |         |         | ence |  data   |      |
+ *       `-------------------------------------------------------------------'
+ *
+ * Total: 28-2340 bytes
+ */
 
 #define BEACON_PROBE_SSID_ID_POSITION 12
 
@@ -467,7 +467,8 @@ Total: 28-2340 bytes
 /* MAX_RATES_LENGTH needs to be 12.  The spec says 8, and many APs
  * only use 8, and then use extended rates for the remaining supported
  * rates.  Other APs, however, stick all of their supported rates on the
- * main rates information element... */
+ * main rates information element...
+ */
 #define MAX_RATES_LENGTH                  ((u8)12)
 #define MAX_RATES_EX_LENGTH               ((u8)16)
 #define MAX_NETWORK_COUNT                  128
@@ -497,11 +498,11 @@ Total: 28-2340 bytes
 #define IEEE80211_PS_MBCAST IEEE80211_DTIM_MBCAST
 #define IW_ESSID_MAX_SIZE 32
 /*
-join_res:
--1: authentication fail
--2: association fail
-> 0: TID
-*/
+ * join_res:
+ * -1: authentication fail
+ * -2: association fail
+ * > 0: TID
+ */
 
 #define DEFAULT_MAX_SCAN_AGE (15 * HZ)
 #define DEFAULT_FTS 2346
@@ -555,8 +556,7 @@ enum {
 	ACT_PUBLIC_MAX
 };
 
-#define OUI_MICROSOFT 0x0050f2 /* Microsoft (also used in Wi-Fi specs)
-				* 00:50:F2 */
+#define OUI_MICROSOFT 0x0050f2 /* Microsoft OUI (also used in Wi-Fi specs) */
 #define WME_OUI_TYPE 2
 #define WME_OUI_SUBTYPE_INFORMATION_ELEMENT 0
 #define WME_OUI_SUBTYPE_PARAMETER_ELEMENT 1
diff --git a/drivers/staging/rtl8723bs/include/osdep_service.h b/drivers/staging/rtl8723bs/include/osdep_service.h
index 955e8678dc265..2f5011a8210cd 100644
--- a/drivers/staging/rtl8723bs/include/osdep_service.h
+++ b/drivers/staging/rtl8723bs/include/osdep_service.h
@@ -14,70 +14,19 @@
 
 #include <osdep_service_linux.h>
 
-#define BIT0	0x00000001
-#define BIT1	0x00000002
-#define BIT2	0x00000004
-#define BIT3	0x00000008
-#define BIT4	0x00000010
-#define BIT5	0x00000020
-#define BIT6	0x00000040
-#define BIT7	0x00000080
-#define BIT8	0x00000100
-#define BIT9	0x00000200
-#define BIT10	0x00000400
-#define BIT11	0x00000800
-#define BIT12	0x00001000
-#define BIT13	0x00002000
-#define BIT14	0x00004000
-#define BIT15	0x00008000
-#define BIT16	0x00010000
-#define BIT17	0x00020000
-#define BIT18	0x00040000
-#define BIT19	0x00080000
-#define BIT20	0x00100000
-#define BIT21	0x00200000
-#define BIT22	0x00400000
-#define BIT23	0x00800000
-#define BIT24	0x01000000
-#define BIT25	0x02000000
-#define BIT26	0x04000000
-#define BIT27	0x08000000
-#define BIT28	0x10000000
-#define BIT29	0x20000000
-#define BIT30	0x40000000
-#define BIT31	0x80000000
-#define BIT32	0x0100000000
-#define BIT33	0x0200000000
-#define BIT34	0x0400000000
-#define BIT35	0x0800000000
-#define BIT36	0x1000000000
 
 extern int RTW_STATUS_CODE(int error_code);
 
-void _kfree(u8 *pbuf, u32 sz);
-
 int _rtw_netif_rx(struct net_device *ndev, struct sk_buff *skb);
 
 #define rtw_netif_rx(ndev, skb) _rtw_netif_rx(ndev, skb)
 
-extern void _rtw_init_queue(struct __queue	*pqueue);
-
 static inline void flush_signals_thread(void)
 {
 	if (signal_pending(current))
 		flush_signals(current);
 }
 
-#define rtw_warn_on(condition) WARN_ON(condition)
-
-static inline int rtw_bug_check(void *parg1, void *parg2, void *parg3, void *parg4)
-{
-	int ret = true;
-
-	return ret;
-
-}
-
 #define _RND(sz, r) ((((sz)+((r)-1))/(r))*(r))
 
 extern void rtw_free_netdev(struct net_device *netdev);
@@ -100,10 +49,5 @@ bool rtw_cbuf_push(struct rtw_cbuf *cbuf, void *buf);
 void *rtw_cbuf_pop(struct rtw_cbuf *cbuf);
 struct rtw_cbuf *rtw_cbuf_alloc(u32 size);
 
-/*  String handler */
-/*
- * Write formatted output to sized buffer
- */
-#define rtw_sprintf(buf, size, format, arg...)	snprintf(buf, size, format, ##arg)
 
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtl8192c_recv.h b/drivers/staging/rtl8723bs/include/rtl8192c_recv.h
index e2e9aa03ffdf2..1f86654f04033 100644
--- a/drivers/staging/rtl8723bs/include/rtl8192c_recv.h
+++ b/drivers/staging/rtl8723bs/include/rtl8192c_recv.h
@@ -30,7 +30,6 @@ struct phy_stat {
 /*  Rx smooth factor */
 #define	Rx_Smooth_Factor (20)
 
-
 void rtl8192c_translate_rx_signal_stuff(union recv_frame *precvframe, struct phy_stat *pphy_status);
 void rtl8192c_query_rx_desc_status(union recv_frame *precvframe, struct recv_stat *pdesc);
 
diff --git a/drivers/staging/rtl8723bs/include/rtl8723b_dm.h b/drivers/staging/rtl8723bs/include/rtl8723b_dm.h
index 1d2da5286e7cc..50ab7aed079e6 100644
--- a/drivers/staging/rtl8723bs/include/rtl8723b_dm.h
+++ b/drivers/staging/rtl8723bs/include/rtl8723b_dm.h
@@ -29,5 +29,4 @@ void rtl8723b_HalDmWatchDog(struct adapter *padapter);
 void rtl8723b_HalDmWatchDog_in_LPS(struct adapter *padapter);
 void rtl8723b_hal_dm_in_lps(struct adapter *padapter);
 
-
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtl8723b_hal.h b/drivers/staging/rtl8723bs/include/rtl8723b_hal.h
index ffd03927841cf..b6006110ad6b0 100644
--- a/drivers/staging/rtl8723bs/include/rtl8723b_hal.h
+++ b/drivers/staging/rtl8723bs/include/rtl8723b_hal.h
@@ -118,7 +118,6 @@ struct rt_firmware_hdr {
 #define WMM_NORMAL_PAGE_NUM_LPQ_8723B 0x20
 #define WMM_NORMAL_PAGE_NUM_NPQ_8723B 0x20
 
-#include "HalVerDef.h"
 #include "hal_com.h"
 
 #define EFUSE_OOB_PROTECT_BYTES 15
@@ -171,7 +170,7 @@ struct c2h_evt_hdr_t {
 	u8 CmdID;
 	u8 CmdLen;
 	u8 CmdSeq;
-} __attribute__((__packed__));
+} __packed;
 
 enum { /* tag_Package_Definition */
 	PACKAGE_DEFAULT,
diff --git a/drivers/staging/rtl8723bs/include/rtl8723b_recv.h b/drivers/staging/rtl8723bs/include/rtl8723b_recv.h
index 783f64de0aec2..e9ebecb224a6f 100644
--- a/drivers/staging/rtl8723bs/include/rtl8723b_recv.h
+++ b/drivers/staging/rtl8723bs/include/rtl8723b_recv.h
@@ -90,6 +90,6 @@ void rtl8723b_process_phy_info(struct adapter *padapter, void *prframe);
 void rtl8723b_read_chip_version(struct adapter *padapter);
 void rtl8723bs_init_default_value(struct adapter *padapter);
 void rtl8723bs_interface_configure(struct adapter *padapter);
-void ReadAdapterInfo8723BS(struct adapter *padapter);
+void rtw_read_adapter_info(struct adapter *padapter);
 
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtl8723b_spec.h b/drivers/staging/rtl8723bs/include/rtl8723b_spec.h
index 6816040a6affb..7f749e32a5b97 100644
--- a/drivers/staging/rtl8723bs/include/rtl8723b_spec.h
+++ b/drivers/staging/rtl8723bs/include/rtl8723b_spec.h
@@ -167,7 +167,7 @@
 /*  */
 /*        8723B REG_CCK_CHECK						(offset 0x454) */
 /*  */
-#define BIT_BCN_PORT_SEL		BIT5
+#define BIT_BCN_PORT_SEL		BIT(5)
 
 /*  */
 /*  */
@@ -188,50 +188,50 @@
 /*  */
 #define	IMR_DISABLED_8723B					0
 /*  IMR DW0(0x00B0-00B3) Bit 0-31 */
-#define	IMR_TIMER2_8723B					BIT31		/*  Timeout interrupt 2 */
-#define	IMR_TIMER1_8723B					BIT30		/*  Timeout interrupt 1 */
-#define	IMR_PSTIMEOUT_8723B				BIT29		/*  Power Save Time Out Interrupt */
-#define	IMR_GTINT4_8723B					BIT28		/*  When GTIMER4 expires, this bit is set to 1 */
-#define	IMR_GTINT3_8723B					BIT27		/*  When GTIMER3 expires, this bit is set to 1 */
-#define	IMR_TXBCN0ERR_8723B				BIT26		/*  Transmit Beacon0 Error */
-#define	IMR_TXBCN0OK_8723B				BIT25		/*  Transmit Beacon0 OK */
-#define	IMR_TSF_BIT32_TOGGLE_8723B		BIT24		/*  TSF Timer BIT32 toggle indication interrupt */
-#define	IMR_BCNDMAINT0_8723B				BIT20		/*  Beacon DMA Interrupt 0 */
-#define	IMR_BCNDERR0_8723B				BIT16		/*  Beacon Queue DMA OK0 */
-#define	IMR_HSISR_IND_ON_INT_8723B		BIT15		/*  HSISR Indicator (HSIMR & HSISR is true, this bit is set to 1) */
-#define	IMR_BCNDMAINT_E_8723B			BIT14		/*  Beacon DMA Interrupt Extension for Win7 */
-#define	IMR_ATIMEND_8723B				BIT12		/*  CTWidnow End or ATIM Window End */
-#define	IMR_C2HCMD_8723B					BIT10		/*  CPU to Host Command INT Status, Write 1 clear */
-#define	IMR_CPWM2_8723B					BIT9			/*  CPU power Mode exchange INT Status, Write 1 clear */
-#define	IMR_CPWM_8723B					BIT8			/*  CPU power Mode exchange INT Status, Write 1 clear */
-#define	IMR_HIGHDOK_8723B				BIT7			/*  High Queue DMA OK */
-#define	IMR_MGNTDOK_8723B				BIT6			/*  Management Queue DMA OK */
-#define	IMR_BKDOK_8723B					BIT5			/*  AC_BK DMA OK */
-#define	IMR_BEDOK_8723B					BIT4			/*  AC_BE DMA OK */
-#define	IMR_VIDOK_8723B					BIT3			/*  AC_VI DMA OK */
-#define	IMR_VODOK_8723B					BIT2			/*  AC_VO DMA OK */
-#define	IMR_RDU_8723B					BIT1			/*  Rx Descriptor Unavailable */
-#define	IMR_ROK_8723B					BIT0			/*  Receive DMA OK */
+#define	IMR_TIMER2_8723B					BIT(31)		/*  Timeout interrupt 2 */
+#define	IMR_TIMER1_8723B					BIT(30)		/*  Timeout interrupt 1 */
+#define	IMR_PSTIMEOUT_8723B				BIT(29)		/*  Power Save Time Out Interrupt */
+#define	IMR_GTINT4_8723B					BIT(28)		/*  When GTIMER4 expires, this bit is set to 1 */
+#define	IMR_GTINT3_8723B					BIT(27)		/*  When GTIMER3 expires, this bit is set to 1 */
+#define	IMR_TXBCN0ERR_8723B				BIT(26)		/*  Transmit Beacon0 Error */
+#define	IMR_TXBCN0OK_8723B				BIT(25)		/*  Transmit Beacon0 OK */
+#define	IMR_TSF_BIT32_TOGGLE_8723B		BIT(24)		/*  TSF Timer BIT32 toggle indication interrupt */
+#define	IMR_BCNDMAINT0_8723B				BIT(20)		/*  Beacon DMA Interrupt 0 */
+#define	IMR_BCNDERR0_8723B				BIT(16)		/*  Beacon Queue DMA OK0 */
+#define	IMR_HSISR_IND_ON_INT_8723B		BIT(15)		/*  HSISR Indicator (HSIMR & HSISR is true, this bit is set to 1) */
+#define	IMR_BCNDMAINT_E_8723B			BIT(14)		/*  Beacon DMA Interrupt Extension for Win7 */
+#define	IMR_ATIMEND_8723B				BIT(12)		/*  CTWidnow End or ATIM Window End */
+#define	IMR_C2HCMD_8723B					BIT(10)		/*  CPU to Host Command INT Status, Write 1 clear */
+#define	IMR_CPWM2_8723B					BIT(9)			/*  CPU power Mode exchange INT Status, Write 1 clear */
+#define	IMR_CPWM_8723B					BIT(8)			/*  CPU power Mode exchange INT Status, Write 1 clear */
+#define	IMR_HIGHDOK_8723B				BIT(7)			/*  High Queue DMA OK */
+#define	IMR_MGNTDOK_8723B				BIT(6)			/*  Management Queue DMA OK */
+#define	IMR_BKDOK_8723B					BIT(5)			/*  AC_BK DMA OK */
+#define	IMR_BEDOK_8723B					BIT(4)			/*  AC_BE DMA OK */
+#define	IMR_VIDOK_8723B					BIT(3)			/*  AC_VI DMA OK */
+#define	IMR_VODOK_8723B					BIT(2)			/*  AC_VO DMA OK */
+#define	IMR_RDU_8723B					BIT(1)			/*  Rx Descriptor Unavailable */
+#define	IMR_ROK_8723B					BIT(0)			/*  Receive DMA OK */
 
 /*  IMR DW1(0x00B4-00B7) Bit 0-31 */
-#define	IMR_BCNDMAINT7_8723B				BIT27		/*  Beacon DMA Interrupt 7 */
-#define	IMR_BCNDMAINT6_8723B				BIT26		/*  Beacon DMA Interrupt 6 */
-#define	IMR_BCNDMAINT5_8723B				BIT25		/*  Beacon DMA Interrupt 5 */
-#define	IMR_BCNDMAINT4_8723B				BIT24		/*  Beacon DMA Interrupt 4 */
-#define	IMR_BCNDMAINT3_8723B				BIT23		/*  Beacon DMA Interrupt 3 */
-#define	IMR_BCNDMAINT2_8723B				BIT22		/*  Beacon DMA Interrupt 2 */
-#define	IMR_BCNDMAINT1_8723B				BIT21		/*  Beacon DMA Interrupt 1 */
-#define	IMR_BCNDOK7_8723B					BIT20		/*  Beacon Queue DMA OK Interrupt 7 */
-#define	IMR_BCNDOK6_8723B					BIT19		/*  Beacon Queue DMA OK Interrupt 6 */
-#define	IMR_BCNDOK5_8723B					BIT18		/*  Beacon Queue DMA OK Interrupt 5 */
-#define	IMR_BCNDOK4_8723B					BIT17		/*  Beacon Queue DMA OK Interrupt 4 */
-#define	IMR_BCNDOK3_8723B					BIT16		/*  Beacon Queue DMA OK Interrupt 3 */
-#define	IMR_BCNDOK2_8723B					BIT15		/*  Beacon Queue DMA OK Interrupt 2 */
-#define	IMR_BCNDOK1_8723B					BIT14		/*  Beacon Queue DMA OK Interrupt 1 */
-#define	IMR_ATIMEND_E_8723B				BIT13		/*  ATIM Window End Extension for Win7 */
-#define	IMR_TXERR_8723B					BIT11		/*  Tx Error Flag Interrupt Status, write 1 clear. */
-#define	IMR_RXERR_8723B					BIT10		/*  Rx Error Flag INT Status, Write 1 clear */
-#define	IMR_TXFOVW_8723B					BIT9			/*  Transmit FIFO Overflow */
-#define	IMR_RXFOVW_8723B					BIT8			/*  Receive FIFO Overflow */
+#define	IMR_BCNDMAINT7_8723B				BIT(27)		/*  Beacon DMA Interrupt 7 */
+#define	IMR_BCNDMAINT6_8723B				BIT(26)		/*  Beacon DMA Interrupt 6 */
+#define	IMR_BCNDMAINT5_8723B				BIT(25)		/*  Beacon DMA Interrupt 5 */
+#define	IMR_BCNDMAINT4_8723B				BIT(24)		/*  Beacon DMA Interrupt 4 */
+#define	IMR_BCNDMAINT3_8723B				BIT(23)		/*  Beacon DMA Interrupt 3 */
+#define	IMR_BCNDMAINT2_8723B				BIT(22)		/*  Beacon DMA Interrupt 2 */
+#define	IMR_BCNDMAINT1_8723B				BIT(21)		/*  Beacon DMA Interrupt 1 */
+#define	IMR_BCNDOK7_8723B					BIT(20)		/*  Beacon Queue DMA OK Interrupt 7 */
+#define	IMR_BCNDOK6_8723B					BIT(19)		/*  Beacon Queue DMA OK Interrupt 6 */
+#define	IMR_BCNDOK5_8723B					BIT(18)		/*  Beacon Queue DMA OK Interrupt 5 */
+#define	IMR_BCNDOK4_8723B					BIT(17)		/*  Beacon Queue DMA OK Interrupt 4 */
+#define	IMR_BCNDOK3_8723B					BIT(16)		/*  Beacon Queue DMA OK Interrupt 3 */
+#define	IMR_BCNDOK2_8723B					BIT(15)		/*  Beacon Queue DMA OK Interrupt 2 */
+#define	IMR_BCNDOK1_8723B					BIT(14)		/*  Beacon Queue DMA OK Interrupt 1 */
+#define	IMR_ATIMEND_E_8723B				BIT(13)		/*  ATIM Window End Extension for Win7 */
+#define	IMR_TXERR_8723B					BIT(11)		/*  Tx Error Flag Interrupt Status, write 1 clear. */
+#define	IMR_RXERR_8723B					BIT(10)		/*  Rx Error Flag INT Status, Write 1 clear */
+#define	IMR_TXFOVW_8723B					BIT(9)			/*  Transmit FIFO Overflow */
+#define	IMR_RXFOVW_8723B					BIT(8)			/*  Receive FIFO Overflow */
 
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtw_btcoex.h b/drivers/staging/rtl8723bs/include/rtw_btcoex.h
index 19764c80b8ba8..7651034987898 100644
--- a/drivers/staging/rtl8723bs/include/rtw_btcoex.h
+++ b/drivers/staging/rtl8723bs/include/rtw_btcoex.h
@@ -9,20 +9,19 @@
 
 #include <drv_types.h>
 
-
 #define	PACKET_NORMAL			0
 #define	PACKET_DHCP				1
 #define	PACKET_ARP				2
 #define	PACKET_EAPOL			3
 
-void rtw_btcoex_MediaStatusNotify(struct adapter *, u8 mediaStatus);
-void rtw_btcoex_HaltNotify(struct adapter *);
+void rtw_btcoex_media_status_notify(struct adapter *padapter, u8 media_status);
+void rtw_btcoex_halt_notify(struct adapter *padapter);
 
 /*  ================================================== */
 /*  Below Functions are called by BT-Coex */
 /*  ================================================== */
-void rtw_btcoex_RejectApAggregatedPacket(struct adapter *, u8 enable);
-void rtw_btcoex_LPS_Enter(struct adapter *);
-void rtw_btcoex_LPS_Leave(struct adapter *);
+void rtw_btcoex_reject_ap_aggregated_packet(struct adapter *padapter, u8 enable);
+void rtw_btcoex_lps_enter(struct adapter *padapter);
+void rtw_btcoex_lps_leave(struct adapter *padapter);
 
 #endif /*  __RTW_BTCOEX_H__ */
diff --git a/drivers/staging/rtl8723bs/include/rtw_cmd.h b/drivers/staging/rtl8723bs/include/rtw_cmd.h
index c4c3edee809d4..f78aa0d7abc5a 100644
--- a/drivers/staging/rtl8723bs/include/rtw_cmd.h
+++ b/drivers/staging/rtl8723bs/include/rtw_cmd.h
@@ -33,8 +33,8 @@
 
 	/* cmd flags */
 	enum {
-		RTW_CMDF_DIRECTLY = BIT0,
-		RTW_CMDF_WAIT_ACK = BIT1,
+		RTW_CMDF_DIRECTLY = BIT(0),
+		RTW_CMDF_WAIT_ACK = BIT(1),
 	};
 
 	struct cmd_priv {
@@ -712,4 +712,4 @@ enum {
 #define _GetRFReg_CMD_		_Read_RFREG_CMD_
 #define _SetRFReg_CMD_		_Write_RFREG_CMD_
 
-#endif /*  _CMD_H_ */
+#endif /* __RTW_CMD_H_ */
diff --git a/drivers/staging/rtl8723bs/include/rtw_efuse.h b/drivers/staging/rtl8723bs/include/rtw_efuse.h
index 191ffdf593d40..808ba94a59987 100644
--- a/drivers/staging/rtl8723bs/include/rtw_efuse.h
+++ b/drivers/staging/rtl8723bs/include/rtw_efuse.h
@@ -31,7 +31,7 @@ enum {
 #define		EFUSE_REPEAT_THRESHOLD_			3
 
 /*  */
-/* 	The following is for BT Efuse definition */
+/*	The following is for BT Efuse definition */
 /*  */
 #define		EFUSE_BT_MAX_MAP_LEN		1024
 #define		EFUSE_MAX_BANK			4
@@ -68,13 +68,11 @@ struct efuse_hal {
 	u8 fakeBTEfuseModifiedMap[EFUSE_BT_MAX_MAP_LEN];
 };
 
-u8 Efuse_CalculateWordCnts(u8 word_en);
-u8 efuse_OneByteRead(struct adapter *padapter, u16 addr, u8 *data);
+u8 rtw_efuse_calculate_word_counts(u8 word_en);
+u8 rtw_efuse_one_byte_read(struct adapter *padapter, u16 addr, u8 *data);
 
-u8 EFUSE_Read1Byte(struct adapter *padapter, u16 Address);
-void EFUSE_ShadowMapUpdate(struct adapter *padapter, u8 efuseType);
-void EFUSE_ShadowRead(struct adapter *padapter, u8 Type, u16 Offset, u32 *Value);
-void Rtw_Hal_ReadMACAddrFromFile(struct adapter *padapter);
-u32 Rtw_Hal_readPGDataFromConfigFile(struct adapter *padapter);
+u8 rtw_efuse_read_1_byte(struct adapter *padapter, u16 Address);
+void rtw_efuse_shadow_map_update(struct adapter *padapter, u8 efuseType);
+void rtw_efuse_shadow_read(struct adapter *padapter, u8 Type, u16 Offset, u32 *Value);
 
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtw_ht.h b/drivers/staging/rtl8723bs/include/rtw_ht.h
index 54fadb92334cf..da3efba7112ae 100644
--- a/drivers/staging/rtl8723bs/include/rtw_ht.h
+++ b/drivers/staging/rtl8723bs/include/rtw_ht.h
@@ -61,16 +61,16 @@ enum {
 	RT_HT_CAP_USE_JAGUAR_CCUT = 0x04,
 };
 
-#define	LDPC_HT_ENABLE_RX			BIT0
-#define	LDPC_HT_ENABLE_TX			BIT1
-#define	LDPC_HT_CAP_TX				BIT3
+#define	LDPC_HT_ENABLE_RX			BIT(0)
+#define	LDPC_HT_ENABLE_TX			BIT(1)
+#define	LDPC_HT_CAP_TX				BIT(3)
 
-#define	STBC_HT_ENABLE_RX			BIT0
-#define	STBC_HT_ENABLE_TX			BIT1
-#define	STBC_HT_CAP_TX				BIT3
+#define	STBC_HT_ENABLE_RX			BIT(0)
+#define	STBC_HT_ENABLE_TX			BIT(1)
+#define	STBC_HT_CAP_TX				BIT(3)
 
-#define	BEAMFORMING_HT_BEAMFORMER_ENABLE	BIT0	/*  Declare our NIC supports beamformer */
-#define	BEAMFORMING_HT_BEAMFORMEE_ENABLE	BIT1	/*  Declare our NIC supports beamformee */
+#define	BEAMFORMING_HT_BEAMFORMER_ENABLE	BIT(0)	/*  Declare our NIC supports beamformer */
+#define	BEAMFORMING_HT_BEAMFORMEE_ENABLE	BIT(1)	/*  Declare our NIC supports beamformee */
 
 /*  20/40 BSS Coexist */
 #define SET_EXT_CAPABILITY_ELE_BSS_COEXIST(_pEleStart, _val)			SET_BITS_TO_LE_1BYTE((_pEleStart), 0, 1, _val)
diff --git a/drivers/staging/rtl8723bs/include/rtw_io.h b/drivers/staging/rtl8723bs/include/rtw_io.h
index adf1de4d7924c..3c99a2bc19bd0 100644
--- a/drivers/staging/rtl8723bs/include/rtw_io.h
+++ b/drivers/staging/rtl8723bs/include/rtw_io.h
@@ -18,24 +18,8 @@ struct _io_ops {
 		int (*_write8)(struct intf_hdl *pintfhdl, u32 addr, u8 val);
 		int (*_write16)(struct intf_hdl *pintfhdl, u32 addr, u16 val);
 		int (*_write32)(struct intf_hdl *pintfhdl, u32 addr, u32 val);
-		int (*_writeN)(struct intf_hdl *pintfhdl, u32 addr, u32 length, u8 *pdata);
 
-		int (*_write8_async)(struct intf_hdl *pintfhdl, u32 addr, u8 val);
-		int (*_write16_async)(struct intf_hdl *pintfhdl, u32 addr, u16 val);
-		int (*_write32_async)(struct intf_hdl *pintfhdl, u32 addr, u32 val);
-
-		void (*_read_mem)(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *pmem);
-		void (*_write_mem)(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *pmem);
-
-		u32 (*_read_interrupt)(struct intf_hdl *pintfhdl, u32 addr);
-
-		u32 (*_read_port)(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *pmem);
 		u32 (*_write_port)(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, u8 *pmem);
-
-		u32 (*_write_scsi)(struct intf_hdl *pintfhdl, u32 cnt, u8 *pmem);
-
-		void (*_read_port_cancel)(struct intf_hdl *pintfhdl);
-		void (*_write_port_cancel)(struct intf_hdl *pintfhdl);
 };
 
 struct	intf_hdl {
diff --git a/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h b/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h
index ab349de733c8d..c3a1d1a84f789 100644
--- a/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h
+++ b/drivers/staging/rtl8723bs/include/rtw_ioctl_set.h
@@ -7,7 +7,6 @@
 #ifndef __RTW_IOCTL_SET_H_
 #define __RTW_IOCTL_SET_H_
 
-
 typedef u8 NDIS_802_11_PMKID_VALUE[16];
 
 u8 rtw_set_802_11_authentication_mode(struct adapter *pdapter, enum ndis_802_11_authentication_mode authmode);
diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme.h b/drivers/staging/rtl8723bs/include/rtw_mlme.h
index ac3ba746b64c9..8cc96164e1cf2 100644
--- a/drivers/staging/rtl8723bs/include/rtw_mlme.h
+++ b/drivers/staging/rtl8723bs/include/rtw_mlme.h
@@ -111,9 +111,9 @@ struct rt_link_detect_t {
 
 /* used for mlme_priv.roam_flags */
 enum {
-	RTW_ROAM_ON_EXPIRED = BIT0,
-	RTW_ROAM_ON_RESUME = BIT1,
-	RTW_ROAM_ACTIVE = BIT2,
+	RTW_ROAM_ON_EXPIRED = BIT(0),
+	RTW_ROAM_ON_RESUME = BIT(1),
+	RTW_ROAM_ACTIVE = BIT(2),
 };
 
 struct mlme_priv {
diff --git a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
index 95769f90d1961..cb23c6939bfc5 100644
--- a/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
+++ b/drivers/staging/rtl8723bs/include/rtw_mlme_ext.h
@@ -27,12 +27,12 @@
 
 /*  ====== ODM_ABILITY_E ======== */
 /*  BB ODM section BIT 0-15 */
-#define DYNAMIC_BB_DIG				BIT0 /* ODM_BB_DIG */
-#define DYNAMIC_BB_DYNAMIC_TXPWR	BIT2 /* ODM_BB_DYNAMIC_TXPWR */
-#define DYNAMIC_BB_ANT_DIV			BIT6 /* ODM_BB_ANT_DIV */
+#define DYNAMIC_BB_DIG				BIT(0) /* ODM_BB_DIG */
+#define DYNAMIC_BB_DYNAMIC_TXPWR	BIT(2) /* ODM_BB_DYNAMIC_TXPWR */
+#define DYNAMIC_BB_ANT_DIV			BIT(6) /* ODM_BB_ANT_DIV */
 
 /*  RF ODM section BIT 24-31 */
-#define DYNAMIC_RF_CALIBRATION		BIT26/* ODM_RF_CALIBRATION */
+#define DYNAMIC_RF_CALIBRATION		BIT(26)/* ODM_RF_CALIBRATION */
 
 #define DYNAMIC_ALL_FUNC_ENABLE		0xFFFFFFF
 
@@ -416,10 +416,6 @@ struct mlme_ext_priv {
 	u16  action_public_rxseq;
 
 	u8 active_keep_alive_check;
-#ifdef DBG_FIXED_CHAN
-	u8 fixed_chan;
-#endif
-
 };
 
 void init_mlme_default_rate_set(struct adapter *padapter);
@@ -709,9 +705,4 @@ enum {
 	MAX_C2HEVT
 };
 
-
-#ifdef _RTW_MLME_EXT_C_
-
-#endif/* _RTL8192C_CMD_C_ */
-
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h b/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h
index c27d07861b8c1..67dc74512e8f3 100644
--- a/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h
+++ b/drivers/staging/rtl8723bs/include/rtw_pwrctrl.h
@@ -86,8 +86,8 @@ enum rt_rf_power_state {
 
 /*  ASPM OSC Control bit, added by Roger, 2013.03.29. */
 #define	RT_PCI_ASPM_OSC_IGNORE		0	 /*  PCI ASPM ignore OSC control in default */
-#define	RT_PCI_ASPM_OSC_ENABLE		BIT0 /*  PCI ASPM controlled by OS according to ACPI Spec 5.0 */
-#define	RT_PCI_ASPM_OSC_DISABLE		BIT1 /*  PCI ASPM controlled by driver or BIOS, i.e., force enable ASPM */
+#define	RT_PCI_ASPM_OSC_ENABLE		BIT(0) /*  PCI ASPM controlled by OS according to ACPI Spec 5.0 */
+#define	RT_PCI_ASPM_OSC_DISABLE		BIT(1) /*  PCI ASPM controlled by driver or BIOS, i.e., force enable ASPM */
 
 enum {
 	PSBBREG_RF0 = 0,
diff --git a/drivers/staging/rtl8723bs/include/rtw_recv.h b/drivers/staging/rtl8723bs/include/rtw_recv.h
index 8e45871f07f08..9c32d480a20a1 100644
--- a/drivers/staging/rtl8723bs/include/rtw_recv.h
+++ b/drivers/staging/rtl8723bs/include/rtw_recv.h
@@ -349,7 +349,7 @@ s32  rtw_recv_entry(union recv_frame *precv_frame);
 static inline u8 *get_rxmem(union recv_frame *precvframe)
 {
 	/* always return rx_head... */
-	if (precvframe == NULL)
+	if (!precvframe)
 		return NULL;
 
 	return precvframe->u.hdr.rx_head;
@@ -362,7 +362,7 @@ static inline u8 *recvframe_pull(union recv_frame *precvframe, signed int sz)
 	/* used for extract sz bytes from rx_data, update rx_data and return the updated rx_data to the caller */
 
 
-	if (precvframe == NULL)
+	if (!precvframe)
 		return NULL;
 
 
@@ -387,7 +387,7 @@ static inline u8 *recvframe_put(union recv_frame *precvframe, signed int sz)
 	/* after putting, rx_tail must be still larger than rx_end. */
 	unsigned char *prev_rx_tail;
 
-	if (precvframe == NULL)
+	if (!precvframe)
 		return NULL;
 
 	prev_rx_tail = precvframe->u.hdr.rx_tail;
@@ -414,7 +414,7 @@ static inline u8 *recvframe_pull_tail(union recv_frame *precvframe, signed int s
 	/* used for extract sz bytes from rx_end, update rx_end and return the updated rx_end to the caller */
 	/* after pulling, rx_end must be still larger than rx_data. */
 
-	if (precvframe == NULL)
+	if (!precvframe)
 		return NULL;
 
 	precvframe->u.hdr.rx_tail -= sz;
diff --git a/drivers/staging/rtl8723bs/include/rtw_wifi_regd.h b/drivers/staging/rtl8723bs/include/rtw_wifi_regd.h
index e611651cb40b5..e4d8fe9bf68d9 100644
--- a/drivers/staging/rtl8723bs/include/rtw_wifi_regd.h
+++ b/drivers/staging/rtl8723bs/include/rtw_wifi_regd.h
@@ -13,5 +13,4 @@ void rtw_regd_init(struct wiphy *wiphy,
 					struct regulatory_request *request));
 void rtw_reg_notifier(struct wiphy *wiphy, struct regulatory_request *request);
 
-
 #endif
diff --git a/drivers/staging/rtl8723bs/include/rtw_xmit.h b/drivers/staging/rtl8723bs/include/rtw_xmit.h
index 544468f576924..b0189a703d286 100644
--- a/drivers/staging/rtl8723bs/include/rtw_xmit.h
+++ b/drivers/staging/rtl8723bs/include/rtw_xmit.h
@@ -457,7 +457,6 @@ s32 _rtw_init_xmit_priv(struct xmit_priv *pxmitpriv, struct adapter *padapter);
 void _rtw_free_xmit_priv(struct xmit_priv *pxmitpriv);
 
 
-s32 rtw_alloc_hwxmits(struct adapter *padapter);
 void rtw_free_hwxmits(struct adapter *padapter);
 
 
diff --git a/drivers/staging/rtl8723bs/include/sdio_ops.h b/drivers/staging/rtl8723bs/include/sdio_ops.h
index c7559a884608f..13f13076bc163 100644
--- a/drivers/staging/rtl8723bs/include/sdio_ops.h
+++ b/drivers/staging/rtl8723bs/include/sdio_ops.h
@@ -27,8 +27,8 @@ extern u8 CheckIPSStatus(struct adapter *padapter);
 
 extern void InitInterrupt8723BSdio(struct adapter *padapter);
 extern void InitSysInterrupt8723BSdio(struct adapter *padapter);
-extern void EnableInterrupt8723BSdio(struct adapter *padapter);
-extern void DisableInterrupt8723BSdio(struct adapter *padapter);
+void rtw_sdio_enable_interrupt(struct adapter *padapter);
+void rtw_sdio_disable_interrupt(struct adapter *padapter);
 extern u8 HalQueryTxBufferStatus8723BSdio(struct adapter *padapter);
 extern void HalQueryTxOQTBufferStatus8723BSdio(struct adapter *padapter);
 #endif /*  !__SDIO_OPS_H__ */
diff --git a/drivers/staging/rtl8723bs/include/sdio_ops_linux.h b/drivers/staging/rtl8723bs/include/sdio_ops_linux.h
index 18830dd183724..cd67a7201065f 100644
--- a/drivers/staging/rtl8723bs/include/sdio_ops_linux.h
+++ b/drivers/staging/rtl8723bs/include/sdio_ops_linux.h
@@ -25,6 +25,5 @@ void sd_write32(struct intf_hdl *pintfhdl, u32 addr, u32 v, s32 *err);
 s32 _sd_write(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, void *pdata);
 s32 sd_write(struct intf_hdl *pintfhdl, u32 addr, u32 cnt, void *pdata);
 
-
 void rtw_sdio_set_irq_thd(struct dvobj_priv *dvobj, void *thd_hdl);
 #endif
diff --git a/drivers/staging/rtl8723bs/include/wifi.h b/drivers/staging/rtl8723bs/include/wifi.h
index 230b2c4ffd3bf..c8844e2bf7b45 100644
--- a/drivers/staging/rtl8723bs/include/wifi.h
+++ b/drivers/staging/rtl8723bs/include/wifi.h
@@ -275,10 +275,7 @@ static inline unsigned char *get_hdr_bssid(unsigned char *pframe)
 
 static inline int IsFrameTypeCtrl(unsigned char *pframe)
 {
-	if (WIFI_CTRL_TYPE == GetFrameType(pframe))
-		return true;
-	else
-		return false;
+	return GetFrameType(pframe) == WIFI_CTRL_TYPE;
 }
 /*-----------------------------------------------------------------------------
 			Below is for the security related definition
@@ -336,7 +333,7 @@ struct ieee80211_ht_addt_info {
 	__le16	operation_mode;
 	__le16	stbc_param;
 	unsigned char 	basic_set[16];
-} __attribute__ ((packed));
+} __packed;
 
 
 struct HT_caps_element {
@@ -351,32 +348,32 @@ struct HT_caps_element {
 		} HT_cap_element;
 		unsigned char HT_cap[26];
 	} u;
-} __attribute__ ((packed));
+} __packed;
 
 struct HT_info_element {
 	unsigned char primary_channel;
 	unsigned char infos[5];
 	unsigned char MCS_rate[16];
-}  __attribute__ ((packed));
+} __packed;
 
 struct AC_param {
 	unsigned char 	ACI_AIFSN;
 	unsigned char 	CW;
 	__le16	TXOP_limit;
-}  __attribute__ ((packed));
+} __packed;
 
 struct WMM_para_element {
 	unsigned char 	QoS_info;
 	unsigned char 	reserved;
 	struct AC_param	ac_param[4];
-}  __attribute__ ((packed));
+} __packed;
 
 struct ADDBA_request {
 	unsigned char 	dialog_token;
 	__le16	BA_para_set;
 	__le16	BA_timeout_value;
 	__le16	BA_starting_seqctrl;
-}  __attribute__ ((packed));
+} __packed;
 
 /* 802.11n HT capabilities masks */
 #define IEEE80211_HT_CAP_LDPC_CODING		0x0001
@@ -450,15 +447,4 @@ struct regd_pair_mapping {
 	u16 reg_2ghz_ctl;
 };
 
-struct rtw_regulatory {
-	char alpha2[2];
-	u16 country_code;
-	u16 max_power_level;
-	u32 tp_scale;
-	u16 current_rd;
-	u16 current_rd_ext;
-	int16_t power_limit;
-	struct regd_pair_mapping *regpair;
-};
-
 #endif /*  _WIFI_H_ */
diff --git a/drivers/staging/rtl8723bs/include/xmit_osdep.h b/drivers/staging/rtl8723bs/include/xmit_osdep.h
index 880344bffe2f0..5b351652e31bd 100644
--- a/drivers/staging/rtl8723bs/include/xmit_osdep.h
+++ b/drivers/staging/rtl8723bs/include/xmit_osdep.h
@@ -30,7 +30,6 @@ extern netdev_tx_t rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetde
 
 void rtw_os_xmit_schedule(struct adapter *padapter);
 
-int rtw_os_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 alloc_sz, u8 flag);
 void rtw_os_xmit_resource_free(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 free_sz, u8 flag);
 
 extern uint rtw_remainder_len(struct pkt_file *pfile);
diff --git a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
index fd3bae31b0edb..1484336d7551a 100644
--- a/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
+++ b/drivers/staging/rtl8723bs/os_dep/ioctl_cfg80211.c
@@ -275,7 +275,7 @@ struct cfg80211_bss *rtw_cfg80211_inform_bss(struct adapter *padapter, struct wl
 	notify_timestamp = ktime_to_us(ktime_get_boottime());
 
 	/* We've set wiphy's signal_type as CFG80211_SIGNAL_TYPE_MBM: signal strength in mBm (100*dBm) */
-	if (check_fwstate(pmlmepriv, _FW_LINKED) == true &&
+	if (check_fwstate(pmlmepriv, _FW_LINKED) &&
 	    is_same_network(&pmlmepriv->cur_network.network, &pnetwork->network, 0)) {
 		notify_signal = 100 * translate_percentage_to_dbm(padapter->recvpriv.signal_strength);/* dbm */
 	} else {
@@ -372,12 +372,12 @@ void rtw_cfg80211_ibss_indicate_connect(struct adapter *padapter)
 		struct wlan_bssid_ex  *pnetwork = &(padapter->mlmeextpriv.mlmext_info.network);
 		struct wlan_network *scanned = pmlmepriv->cur_network_scanned;
 
-		if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true) {
+		if (check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
 			memcpy(&cur_network->network, pnetwork, sizeof(struct wlan_bssid_ex));
 			rtw_cfg80211_inform_bss(padapter, cur_network);
 		} else {
 			if (!scanned) {
-				rtw_warn_on(1);
+				WARN_ON(1);
 				return;
 			}
 			if (!memcmp(&scanned->network.ssid, &pnetwork->ssid,
@@ -385,7 +385,7 @@ void rtw_cfg80211_ibss_indicate_connect(struct adapter *padapter)
 			    !memcmp(scanned->network.mac_address, pnetwork->mac_address, ETH_ALEN))
 				rtw_cfg80211_inform_bss(padapter, scanned);
 			else
-				rtw_warn_on(1);
+				WARN_ON(1);
 		}
 
 		if (!rtw_cfg80211_check_bss(padapter))
@@ -410,7 +410,7 @@ void rtw_cfg80211_indicate_connect(struct adapter *padapter)
 		return;
 	}
 
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true)
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		return;
 
 	{
@@ -418,7 +418,7 @@ void rtw_cfg80211_indicate_connect(struct adapter *padapter)
 		struct wlan_network *scanned = pmlmepriv->cur_network_scanned;
 
 		if (!scanned) {
-			rtw_warn_on(1);
+			WARN_ON(1);
 			goto check_bss;
 		}
 
@@ -427,7 +427,7 @@ void rtw_cfg80211_indicate_connect(struct adapter *padapter)
 			    sizeof(struct ndis_802_11_ssid)))
 			rtw_cfg80211_inform_bss(padapter, scanned);
 		else
-			rtw_warn_on(1);
+			WARN_ON(1);
 	}
 
 check_bss:
@@ -479,7 +479,7 @@ void rtw_cfg80211_indicate_disconnect(struct adapter *padapter)
 		return;
 	}
 
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true)
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		return;
 
 	if (!padapter->mlmepriv.not_indic_disco) {
@@ -768,7 +768,7 @@ static int rtw_cfg80211_set_encryption(struct net_device *dev, struct ieee_param
 		struct sta_info *psta, *pbcmc_sta;
 		struct sta_priv *pstapriv = &padapter->stapriv;
 
-		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE | WIFI_MP_STATE) == true) { /* sta mode */
+		if (check_fwstate(pmlmepriv, WIFI_STATION_STATE | WIFI_MP_STATE)) { /* sta mode */
 			psta = rtw_get_stainfo(pstapriv, get_bssid(pmlmepriv));
 			if (psta) {
 				/* Jeff: don't disable ieee8021x_blocked while clearing key */
@@ -892,15 +892,15 @@ static int cfg80211_rtw_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 		memcpy(param->u.crypt.key, (u8 *)params->key, params->key_len);
 	}
 
-	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) == true) {
+	if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
 		ret =  rtw_cfg80211_set_encryption(ndev, param, param_len);
-	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
+	} else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
 		if (mac_addr)
 			memcpy(param->sta_addr, mac_addr, ETH_ALEN);
 
 		ret = rtw_cfg80211_ap_set_encryption(ndev, param, param_len);
-	} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) == true
-		|| check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE) == true) {
+	} else if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
+		   check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
 		ret =  rtw_cfg80211_set_encryption(ndev, param, param_len);
 	}
 
@@ -1033,7 +1033,7 @@ static int cfg80211_rtw_change_iface(struct wiphy *wiphy,
 	struct mlme_ext_priv *pmlmeext = &(padapter->mlmeextpriv);
 	int ret = 0;
 
-	if (adapter_to_dvobj(padapter)->processing_dev_remove == true) {
+	if (adapter_to_dvobj(padapter)->processing_dev_remove) {
 		ret = -EPERM;
 		goto exit;
 	}
@@ -1074,7 +1074,7 @@ static int cfg80211_rtw_change_iface(struct wiphy *wiphy,
 
 	rtw_wdev->iftype = type;
 
-	if (rtw_set_802_11_infrastructure_mode(padapter, networkType) == false) {
+	if (!rtw_set_802_11_infrastructure_mode(padapter, networkType)) {
 		rtw_wdev->iftype = old_type;
 		ret = -EPERM;
 		goto exit;
@@ -1209,8 +1209,9 @@ static int cfg80211_rtw_scan(struct wiphy *wiphy
 	pwdev_priv->scan_request = request;
 	spin_unlock_bh(&pwdev_priv->scan_req_lock);
 
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) == true) {
-		if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS | _FW_UNDER_SURVEY | _FW_UNDER_LINKING) == true) {
+	if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
+		if (check_fwstate(pmlmepriv, WIFI_UNDER_WPS | _FW_UNDER_SURVEY |
+					     _FW_UNDER_LINKING)) {
 			need_indicate_scan_done = true;
 			goto check_need_indicate_scan_done;
 		}
@@ -1225,10 +1226,10 @@ static int cfg80211_rtw_scan(struct wiphy *wiphy
 	if (request->ie && request->ie_len > 0)
 		rtw_cfg80211_set_probe_req_wpsp2pie(padapter, (u8 *)request->ie, request->ie_len);
 
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY) == true) {
+	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY)) {
 		need_indicate_scan_done = true;
 		goto check_need_indicate_scan_done;
-	} else if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING) == true) {
+	} else if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) {
 		ret = -EBUSY;
 		goto check_need_indicate_scan_done;
 	}
@@ -1284,7 +1285,7 @@ static int cfg80211_rtw_scan(struct wiphy *wiphy
 	}
 	spin_unlock_bh(&pmlmepriv->lock);
 
-	if (_status == false)
+	if (!_status)
 		ret = -1;
 
 check_need_indicate_scan_done:
@@ -1584,7 +1585,7 @@ static int cfg80211_rtw_join_ibss(struct wiphy *wiphy, struct net_device *ndev,
 	ret = rtw_cfg80211_set_auth_type(psecuritypriv, NL80211_AUTHTYPE_OPEN_SYSTEM);
 	rtw_set_802_11_authentication_mode(padapter, psecuritypriv->ndisauthtype);
 
-	if (rtw_set_802_11_ssid(padapter, &ndis_ssid) == false) {
+	if (!rtw_set_802_11_ssid(padapter, &ndis_ssid)) {
 		ret = -1;
 		goto exit;
 	}
@@ -1610,7 +1611,7 @@ static int cfg80211_rtw_leave_ibss(struct wiphy *wiphy, struct net_device *ndev)
 
 		rtw_wdev->iftype = NL80211_IFTYPE_STATION;
 
-		if (rtw_set_802_11_infrastructure_mode(padapter, Ndis802_11Infrastructure) == false) {
+		if (!rtw_set_802_11_infrastructure_mode(padapter, Ndis802_11Infrastructure)) {
 			rtw_wdev->iftype = old_type;
 			ret = -EPERM;
 			goto leave_ibss;
@@ -1634,7 +1635,7 @@ static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev,
 
 	padapter->mlmepriv.not_indic_disco = true;
 
-	if (adapter_wdev_data(padapter)->block == true) {
+	if (adapter_wdev_data(padapter)->block) {
 		ret = -EBUSY;
 		goto exit;
 	}
@@ -1664,11 +1665,11 @@ static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev,
 	ndis_ssid.ssid_length = sme->ssid_len;
 	memcpy(ndis_ssid.ssid, (u8 *)sme->ssid, sme->ssid_len);
 
-	if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING) == true) {
+	if (check_fwstate(pmlmepriv, _FW_UNDER_LINKING)) {
 		ret = -EBUSY;
 		goto exit;
 	}
-	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY) == true)
+	if (check_fwstate(pmlmepriv, _FW_UNDER_SURVEY))
 		rtw_scan_abort(padapter);
 
 	psecuritypriv->ndisencryptstatus = Ndis802_11EncryptionDisabled;
@@ -1761,7 +1762,7 @@ static int cfg80211_rtw_connect(struct wiphy *wiphy, struct net_device *ndev,
 
 	/* rtw_set_802_11_encryption_mode(padapter, padapter->securitypriv.ndisencryptstatus); */
 
-	if (rtw_set_802_11_connect(padapter, (u8 *)sme->bssid, &ndis_ssid) == false) {
+	if (!rtw_set_802_11_connect(padapter, (u8 *)sme->bssid, &ndis_ssid)) {
 		ret = -1;
 		goto exit;
 	}
@@ -2067,7 +2068,7 @@ static netdev_tx_t rtw_cfg80211_monitor_if_xmit_entry(struct sk_buff *skb, struc
 		u32 len = skb->len;
 		u8 category, action;
 
-		if (rtw_action_frame_parse(buf, len, &category, &action) == false)
+		if (!rtw_action_frame_parse(buf, len, &category, &action))
 			goto fail;
 
 		/* starting alloc mgmt frame to dump it */
@@ -2249,7 +2250,7 @@ static int rtw_add_beacon(struct adapter *adapter, const u8 *head, size_t head_l
 	uint len, wps_ielen = 0;
 	struct mlme_priv *pmlmepriv = &(adapter->mlmepriv);
 
-	if (check_fwstate(pmlmepriv, WIFI_AP_STATE) != true)
+	if (!check_fwstate(pmlmepriv, WIFI_AP_STATE))
 		return -EINVAL;
 
 	if (head_len < 24)
@@ -2343,7 +2344,7 @@ static int cfg80211_rtw_del_station(struct wiphy *wiphy,
 	struct sta_priv *pstapriv = &padapter->stapriv;
 	const u8 *mac = params->mac;
 
-	if (check_fwstate(pmlmepriv, (_FW_LINKED | WIFI_AP_STATE)) != true)
+	if (!check_fwstate(pmlmepriv, _FW_LINKED | WIFI_AP_STATE))
 		return -EINVAL;
 
 	if (!mac) {
@@ -2429,7 +2430,7 @@ static int	cfg80211_rtw_dump_station(struct wiphy *wiphy,
 	spin_lock_bh(&pstapriv->asoc_list_lock);
 	psta = rtw_sta_info_get_by_idx(idx, pstapriv);
 	spin_unlock_bh(&pstapriv->asoc_list_lock);
-	if (psta == NULL) {
+	if (!psta) {
 		ret = -ENOENT;
 		goto exit;
 	}
@@ -2549,7 +2550,7 @@ static int cfg80211_rtw_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 	/* indicate ack before issue frame to avoid racing with rsp frame */
 	rtw_cfg80211_mgmt_tx_status(padapter, *cookie, buf, len, ack, GFP_KERNEL);
 
-	if (rtw_action_frame_parse(buf, len, &category, &action) == false)
+	if (!rtw_action_frame_parse(buf, len, &category, &action))
 		goto exit;
 
 	rtw_ps_deny(padapter, PS_DENY_MGNT_TX);
diff --git a/drivers/staging/rtl8723bs/os_dep/os_intfs.c b/drivers/staging/rtl8723bs/os_dep/os_intfs.c
index e943dcea1a21b..f31196f54b3e0 100644
--- a/drivers/staging/rtl8723bs/os_dep/os_intfs.c
+++ b/drivers/staging/rtl8723bs/os_dep/os_intfs.c
@@ -288,8 +288,8 @@ static int rtw_net_set_mac_address(struct net_device *pnetdev, void *p)
 static struct net_device_stats *rtw_net_get_stats(struct net_device *pnetdev)
 {
 	struct adapter *padapter = rtw_netdev_priv(pnetdev);
-	struct xmit_priv *pxmitpriv = &(padapter->xmitpriv);
-	struct recv_priv *precvpriv = &(padapter->recvpriv);
+	struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
+	struct recv_priv *precvpriv = &padapter->recvpriv;
 
 	padapter->stats.tx_packets = pxmitpriv->tx_pkts;/* pxmitpriv->tx_pkts++; */
 	padapter->stats.rx_packets = precvpriv->rx_pkts;/* precvpriv->rx_pkts++; */
@@ -550,7 +550,7 @@ static void rtw_init_default_value(struct adapter *padapter)
 	rtw_update_registrypriv_dev_network(padapter);
 
 	/* hal_priv */
-	rtw_hal_def_value_init(padapter);
+	rtl8723bs_init_default_value(padapter);
 
 	/* misc. */
 	RTW_ENABLE_FUNC(padapter, DF_RX_BIT);
@@ -609,7 +609,7 @@ void rtw_reset_drv_sw(struct adapter *padapter)
 	struct pwrctrl_priv *pwrctrlpriv = adapter_to_pwrctl(padapter);
 
 	/* hal_priv */
-	rtw_hal_def_value_init(padapter);
+	rtl8723bs_init_default_value(padapter);
 
 	RTW_ENABLE_FUNC(padapter, DF_RX_BIT);
 	RTW_ENABLE_FUNC(padapter, DF_TX_BIT);
@@ -636,6 +636,8 @@ void rtw_reset_drv_sw(struct adapter *padapter)
 
 u8 rtw_init_drv_sw(struct adapter *padapter)
 {
+	int res;
+
 	rtw_init_default_value(padapter);
 
 	rtw_init_hal_com_default_value(padapter);
@@ -653,7 +655,8 @@ u8 rtw_init_drv_sw(struct adapter *padapter)
 
 	init_mlme_ext_priv(padapter);
 
-	if (_rtw_init_xmit_priv(&padapter->xmitpriv, padapter) == _FAIL)
+	res = _rtw_init_xmit_priv(&padapter->xmitpriv, padapter);
+	if (res)
 		goto free_mlme_ext;
 
 	if (_rtw_init_recv_priv(&padapter->recvpriv, padapter) == _FAIL)
@@ -674,7 +677,7 @@ u8 rtw_init_drv_sw(struct adapter *padapter)
 
 	rtw_init_pwrctrl_priv(padapter);
 
-	rtw_hal_dm_init(padapter);
+	rtl8723b_init_dm_priv(padapter);
 
 	return _SUCCESS;
 
@@ -734,7 +737,7 @@ u8 rtw_free_drv_sw(struct adapter *padapter)
 
 	/* kfree((void *)padapter); */
 
-	rtw_hal_free_data(padapter);
+	rtw_hal_data_deinit(padapter);
 
 	/* free the old_pnetdev */
 	if (padapter->rereg_nd_name_priv.old_pnetdev) {
@@ -986,12 +989,10 @@ void rtw_dev_unload(struct adapter *padapter)
 			rtw_stop_drv_threads(padapter);
 
 		while (atomic_read(&pcmdpriv->cmdthd_running)) {
-			if (cnt > 5) {
+			if (cnt > 5)
 				break;
-			} else {
-				cnt++;
-				msleep(10);
-			}
+			cnt++;
+			msleep(10);
 		}
 
 		/* check the status of IPS */
@@ -1140,7 +1141,7 @@ static int rtw_resume_process_normal(struct adapter *padapter)
 		if (ret)
 			goto exit;
 	}
-	rtw_hal_disable_interrupt(padapter);
+	rtw_sdio_disable_interrupt(padapter);
 	/* if (sdio_alloc_irq(adapter_to_dvobj(padapter)) != _SUCCESS) */
 	if ((padapter->intf_alloc_irq) && (padapter->intf_alloc_irq(adapter_to_dvobj(padapter)) != _SUCCESS)) {
 		ret = -1;
diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
index d0feb28b7043e..c43a0391a5ca7 100644
--- a/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
+++ b/drivers/staging/rtl8723bs/os_dep/sdio_intf.c
@@ -6,6 +6,7 @@
  ******************************************************************************/
 #include <drv_types.h>
 #include <hal_btcoex.h>
+#include <hal_data.h>
 #include <linux/jiffies.h>
 
 #ifndef dev_to_sdio_func
@@ -144,9 +145,8 @@ static void sdio_deinit(struct dvobj_priv *dvobj)
 		sdio_claim_host(func);
 		sdio_disable_func(func);
 
-		if (dvobj->irq_alloc) {
+		if (dvobj->irq_alloc)
 			sdio_release_irq(func);
-		}
 
 		sdio_release_host(func);
 	}
@@ -203,8 +203,7 @@ static void sd_intf_start(struct adapter *padapter)
 	if (!padapter)
 		return;
 
-	/*  hal dep */
-	rtw_hal_enable_interrupt(padapter);
+	rtw_sdio_enable_interrupt(padapter);
 }
 
 static void sd_intf_stop(struct adapter *padapter)
@@ -212,8 +211,7 @@ static void sd_intf_stop(struct adapter *padapter)
 	if (!padapter)
 		return;
 
-	/*  hal dep */
-	rtw_hal_disable_interrupt(padapter);
+	rtw_sdio_disable_interrupt(padapter);
 }
 
 
@@ -263,14 +261,14 @@ static struct adapter *rtw_sdio_if1_init(struct dvobj_priv *dvobj, const struct
 	if (rtw_init_io_priv(padapter, sdio_set_intf_ops) == _FAIL)
 		goto free_hal_data;
 
-	rtw_hal_read_chip_version(padapter);
+	rtl8723b_read_chip_version(padapter);
 
-	rtw_hal_chip_configure(padapter);
+	rtl8723bs_interface_configure(padapter);
 
 	hal_btcoex_Initialize((void *)padapter);
 
 	/* 3 6. read efuse/eeprom data */
-	rtw_hal_read_chip_info(padapter);
+	rtw_read_adapter_info(padapter);
 
 	/* 3 7. init driver common data */
 	if (rtw_init_drv_sw(padapter) == _FAIL)
@@ -282,7 +280,7 @@ static struct adapter *rtw_sdio_if1_init(struct dvobj_priv *dvobj, const struct
 	/*  set mac addr */
 	rtw_macaddr_cfg(&psdio->func->dev, padapter->eeprompriv.mac_addr);
 
-	rtw_hal_disable_interrupt(padapter);
+	rtw_sdio_disable_interrupt(padapter);
 
 	status = _SUCCESS;
 
@@ -401,7 +399,7 @@ static void rtw_dev_remove(struct sdio_func *func)
 
 	LeaveAllPowerSaveMode(padapter);
 
-	rtw_btcoex_HaltNotify(padapter);
+	rtw_btcoex_halt_notify(padapter);
 
 	rtw_sdio_if1_deinit(padapter);
 
diff --git a/drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c b/drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c
index 3ea9fdfa14f86..e9a2f3f7ec747 100644
--- a/drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c
+++ b/drivers/staging/rtl8723bs/os_dep/sdio_ops_linux.c
@@ -224,7 +224,7 @@ u32 sd_read32(struct intf_hdl *pintfhdl, u32 addr, s32 *err)
 			if ((-ESHUTDOWN == *err) || (-ENODEV == *err))
 				padapter->bSurpriseRemoved = true;
 
-			if (rtw_inc_and_chk_continual_io_error(psdiodev) == true) {
+			if (rtw_inc_and_chk_continual_io_error(psdiodev)) {
 				padapter->bSurpriseRemoved = true;
 				break;
 			}
@@ -300,7 +300,7 @@ void sd_write32(struct intf_hdl *pintfhdl, u32 addr, u32 v, s32 *err)
 			if ((-ESHUTDOWN == *err) || (-ENODEV == *err))
 				padapter->bSurpriseRemoved = true;
 
-			if (rtw_inc_and_chk_continual_io_error(psdiodev) == true) {
+			if (rtw_inc_and_chk_continual_io_error(psdiodev)) {
 				padapter->bSurpriseRemoved = true;
 				break;
 			}
diff --git a/drivers/staging/rtl8723bs/os_dep/wifi_regd.c b/drivers/staging/rtl8723bs/os_dep/wifi_regd.c
index f9c4d487badf0..c48a1e3ba4acc 100644
--- a/drivers/staging/rtl8723bs/os_dep/wifi_regd.c
+++ b/drivers/staging/rtl8723bs/os_dep/wifi_regd.c
@@ -83,23 +83,19 @@ static void _rtw_reg_apply_flags(struct wiphy *wiphy)
 }
 
 static int _rtw_reg_notifier_apply(struct wiphy *wiphy,
-				   struct regulatory_request *request,
-				   struct rtw_regulatory *reg)
+				   struct regulatory_request *request)
 {
 	/* Hard code flags */
 	_rtw_reg_apply_flags(wiphy);
 	return 0;
 }
 
-static const struct ieee80211_regdomain *_rtw_regdomain_select(struct
-							       rtw_regulatory
-							       *reg)
+static const struct ieee80211_regdomain *_rtw_regdomain_select(void)
 {
 	return &rtw_regdom_rd;
 }
 
-static void _rtw_regd_init_wiphy(struct rtw_regulatory *reg,
-				 struct wiphy *wiphy,
+static void _rtw_regd_init_wiphy(struct wiphy *wiphy,
 				 void (*reg_notifier)(struct wiphy *wiphy,
 						      struct
 						      regulatory_request *
@@ -113,7 +109,7 @@ static void _rtw_regd_init_wiphy(struct rtw_regulatory *reg,
 	wiphy->regulatory_flags &= ~REGULATORY_STRICT_REG;
 	wiphy->regulatory_flags &= ~REGULATORY_DISABLE_BEACON_HINTS;
 
-	regd = _rtw_regdomain_select(reg);
+	regd = _rtw_regdomain_select();
 	wiphy_apply_custom_regulatory(wiphy, regd);
 
 	/* Hard code flags */
@@ -124,12 +120,10 @@ void rtw_regd_init(struct wiphy *wiphy,
 		   void (*reg_notifier)(struct wiphy *wiphy,
 					struct regulatory_request *request))
 {
-	_rtw_regd_init_wiphy(NULL, wiphy, reg_notifier);
+	_rtw_regd_init_wiphy(wiphy, reg_notifier);
 }
 
 void rtw_reg_notifier(struct wiphy *wiphy, struct regulatory_request *request)
 {
-	struct rtw_regulatory *reg = NULL;
-
-	_rtw_reg_notifier_apply(wiphy, request, reg);
+	_rtw_reg_notifier_apply(wiphy, request);
 }
diff --git a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c
index dc0b77f38b1ae..d5bb1c7932fc3 100644
--- a/drivers/staging/rtl8723bs/os_dep/xmit_linux.c
+++ b/drivers/staging/rtl8723bs/os_dep/xmit_linux.c
@@ -46,19 +46,6 @@ signed int rtw_endofpktfile(struct pkt_file *pfile)
 	return false;
 }
 
-int rtw_os_xmit_resource_alloc(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 alloc_sz, u8 flag)
-{
-	if (alloc_sz > 0) {
-		pxmitbuf->pallocated_buf = kzalloc(alloc_sz, GFP_KERNEL);
-		if (!pxmitbuf->pallocated_buf)
-			return _FAIL;
-
-		pxmitbuf->pbuf = (u8 *)N_BYTE_ALIGMENT((SIZE_PTR)(pxmitbuf->pallocated_buf), XMITBUF_ALIGN_SZ);
-	}
-
-	return _SUCCESS;
-}
-
 void rtw_os_xmit_resource_free(struct adapter *padapter, struct xmit_buf *pxmitbuf, u32 free_sz, u8 flag)
 {
 	if (free_sz > 0)
@@ -188,7 +175,7 @@ void _rtw_xmit_entry(struct sk_buff *pkt, struct net_device *pnetdev)
 	struct mlme_priv *pmlmepriv = &padapter->mlmepriv;
 	s32 res = 0;
 
-	if (rtw_if_up(padapter) == false)
+	if (!rtw_if_up(padapter))
 		goto drop_packet;
 
 	rtw_check_xmit_resource(padapter, pkt);
diff --git a/drivers/staging/sm750fb/ddk750_power.c b/drivers/staging/sm750fb/ddk750_power.c
index 12834f78eef78..eaba3bc2e01a6 100644
--- a/drivers/staging/sm750fb/ddk750_power.c
+++ b/drivers/staging/sm750fb/ddk750_power.c
@@ -126,20 +126,3 @@ void sm750_enable_gpio(unsigned int enable)
 
 	sm750_set_current_gate(gate);
 }
-
-/*
- * This function enable/disable the I2C Engine
- */
-void sm750_enable_i2c(unsigned int enable)
-{
-	u32 gate;
-
-	/* Enable I2C Gate */
-	gate = peek32(CURRENT_GATE);
-	if (enable)
-		gate |= CURRENT_GATE_I2C;
-	else
-		gate &= ~CURRENT_GATE_I2C;
-
-	sm750_set_current_gate(gate);
-}
diff --git a/drivers/staging/sm750fb/ddk750_power.h b/drivers/staging/sm750fb/ddk750_power.h
index 5cbb11986bb89..1c4f054d72760 100644
--- a/drivers/staging/sm750fb/ddk750_power.h
+++ b/drivers/staging/sm750fb/ddk750_power.h
@@ -33,9 +33,4 @@ void sm750_enable_dma(unsigned int enable);
  */
 void sm750_enable_gpio(unsigned int enable);
 
-/*
- * This function enable/disable the I2C Engine
- */
-void sm750_enable_i2c(unsigned int enable);
-
 #endif
diff --git a/drivers/staging/sm750fb/sm750.c b/drivers/staging/sm750fb/sm750.c
index 9f3e3d37e82a1..996a586a3727d 100644
--- a/drivers/staging/sm750fb/sm750.c
+++ b/drivers/staging/sm750fb/sm750.c
@@ -531,7 +531,7 @@ static int lynxfb_ops_setcolreg(unsigned int regno,
 	var = &info->var;
 	ret = 0;
 
-	if (regno > 256) {
+	if (regno >= 256) {
 		dev_err(info->device, "regno = %d\n", regno);
 		return -EINVAL;
 	}
@@ -553,7 +553,7 @@ static int lynxfb_ops_setcolreg(unsigned int regno,
 		goto exit;
 	}
 
-	if (info->fix.visual == FB_VISUAL_TRUECOLOR && regno < 256) {
+	if (info->fix.visual == FB_VISUAL_TRUECOLOR) {
 		u32 val;
 
 		if (var->bits_per_pixel == 16 ||
@@ -622,26 +622,26 @@ static int sm750fb_set_drv(struct lynxfb_par *par)
 		output->paths = sm750_pnc;
 		crtc->channel = sm750_primary;
 		crtc->o_screen = 0;
-		crtc->v_screen = sm750_dev->pvMem;
+		crtc->v_screen = sm750_dev->vmem;
 		break;
 	case sm750_simul_sec:
 		output->paths = sm750_pnc;
 		crtc->channel = sm750_secondary;
 		crtc->o_screen = 0;
-		crtc->v_screen = sm750_dev->pvMem;
+		crtc->v_screen = sm750_dev->vmem;
 		break;
 	case sm750_dual_normal:
 		if (par->index == 0) {
 			output->paths = sm750_panel;
 			crtc->channel = sm750_primary;
 			crtc->o_screen = 0;
-			crtc->v_screen = sm750_dev->pvMem;
+			crtc->v_screen = sm750_dev->vmem;
 		} else {
 			output->paths = sm750_crt;
 			crtc->channel = sm750_secondary;
 			/* not consider of padding stuffs for o_screen,need fix */
 			crtc->o_screen = sm750_dev->vidmem_size >> 1;
-			crtc->v_screen = sm750_dev->pvMem + crtc->o_screen;
+			crtc->v_screen = sm750_dev->vmem + crtc->o_screen;
 		}
 		break;
 	case sm750_dual_swap:
@@ -649,7 +649,7 @@ static int sm750fb_set_drv(struct lynxfb_par *par)
 			output->paths = sm750_panel;
 			crtc->channel = sm750_secondary;
 			crtc->o_screen = 0;
-			crtc->v_screen = sm750_dev->pvMem;
+			crtc->v_screen = sm750_dev->vmem;
 		} else {
 			output->paths = sm750_crt;
 			crtc->channel = sm750_primary;
@@ -657,7 +657,7 @@ static int sm750fb_set_drv(struct lynxfb_par *par)
 			 * need fix
 			 */
 			crtc->o_screen = sm750_dev->vidmem_size >> 1;
-			crtc->v_screen = sm750_dev->pvMem + crtc->o_screen;
+			crtc->v_screen = sm750_dev->vmem + crtc->o_screen;
 		}
 		break;
 	default:
@@ -670,7 +670,7 @@ static int sm750fb_set_drv(struct lynxfb_par *par)
 static const struct fb_ops lynxfb_ops = {
 	.owner = THIS_MODULE,
 	FB_DEFAULT_IOMEM_OPS,
-	.fb_check_var =  lynxfb_ops_check_var,
+	.fb_check_var = lynxfb_ops_check_var,
 	.fb_set_par = lynxfb_ops_set_par,
 	.fb_setcolreg = lynxfb_ops_setcolreg,
 	.fb_blank = lynxfb_ops_blank,
@@ -680,7 +680,7 @@ static const struct fb_ops lynxfb_ops = {
 static const struct fb_ops lynxfb_ops_with_cursor = {
 	.owner = THIS_MODULE,
 	FB_DEFAULT_IOMEM_OPS,
-	.fb_check_var =  lynxfb_ops_check_var,
+	.fb_check_var = lynxfb_ops_check_var,
 	.fb_set_par = lynxfb_ops_set_par,
 	.fb_setcolreg = lynxfb_ops_setcolreg,
 	.fb_blank = lynxfb_ops_blank,
@@ -691,7 +691,7 @@ static const struct fb_ops lynxfb_ops_with_cursor = {
 static const struct fb_ops lynxfb_ops_accel = {
 	.owner = THIS_MODULE,
 	__FB_DEFAULT_IOMEM_OPS_RDWR,
-	.fb_check_var =  lynxfb_ops_check_var,
+	.fb_check_var = lynxfb_ops_check_var,
 	.fb_set_par = lynxfb_ops_set_par,
 	.fb_setcolreg = lynxfb_ops_setcolreg,
 	.fb_blank = lynxfb_ops_blank,
@@ -705,7 +705,7 @@ static const struct fb_ops lynxfb_ops_accel = {
 static const struct fb_ops lynxfb_ops_accel_with_cursor = {
 	.owner = THIS_MODULE,
 	__FB_DEFAULT_IOMEM_OPS_RDWR,
-	.fb_check_var =  lynxfb_ops_check_var,
+	.fb_check_var = lynxfb_ops_check_var,
 	.fb_set_par = lynxfb_ops_set_par,
 	.fb_setcolreg = lynxfb_ops_setcolreg,
 	.fb_blank = lynxfb_ops_blank,
@@ -731,7 +731,7 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 		lynx750_ext, NULL, vesa_modes,
 	};
 	int cdb[] = {ARRAY_SIZE(lynx750_ext), 0, VESA_MODEDB_SIZE};
-	static const char *fix_id[2] = {
+	static const char * const fix_id[2] = {
 		"sm750_fb1", "sm750_fb2",
 	};
 
@@ -761,7 +761,7 @@ static int lynxfb_set_fbinfo(struct fb_info *info, int index)
 	crtc->cursor.max_h = 64;
 	crtc->cursor.max_w = 64;
 	crtc->cursor.size = crtc->cursor.max_h * crtc->cursor.max_w * 2 / 8;
-	crtc->cursor.vstart = sm750_dev->pvMem + crtc->cursor.offset;
+	crtc->cursor.vstart = sm750_dev->vmem + crtc->cursor.offset;
 
 	memset_io(crtc->cursor.vstart, 0, crtc->cursor.size);
 	if (!g_hwcursor)
@@ -859,9 +859,9 @@ static void sm750fb_setup(struct sm750_dev *sm750_dev, char *src)
 	sm750_dev->init_parm.chip_clk = 0;
 	sm750_dev->init_parm.mem_clk = 0;
 	sm750_dev->init_parm.master_clk = 0;
-	sm750_dev->init_parm.powerMode = 0;
+	sm750_dev->init_parm.power_mode = 0;
 	sm750_dev->init_parm.setAllEngOff = 0;
-	sm750_dev->init_parm.resetMemory = 1;
+	sm750_dev->init_parm.reset_memory = 1;
 
 	/* defaultly turn g_hwcursor on for both view */
 	g_hwcursor = 3;
@@ -880,11 +880,11 @@ static void sm750fb_setup(struct sm750_dev *sm750_dev, char *src)
 		} else if (!strncmp(opt, "nocrt", strlen("nocrt"))) {
 			sm750_dev->nocrt = 1;
 		} else if (!strncmp(opt, "36bit", strlen("36bit"))) {
-			sm750_dev->pnltype = sm750_doubleTFT;
+			sm750_dev->pnltype = SM750_DOUBLE_TFT;
 		} else if (!strncmp(opt, "18bit", strlen("18bit"))) {
-			sm750_dev->pnltype = sm750_dualTFT;
+			sm750_dev->pnltype = SM750_DUAL_TFT;
 		} else if (!strncmp(opt, "24bit", strlen("24bit"))) {
-			sm750_dev->pnltype = sm750_24TFT;
+			sm750_dev->pnltype = SM750_24TFT;
 		} else if (!strncmp(opt, "nohwc0", strlen("nohwc0"))) {
 			g_hwcursor &= ~0x1;
 		} else if (!strncmp(opt, "nohwc1", strlen("nohwc1"))) {
@@ -1028,7 +1028,7 @@ static int lynxfb_pci_probe(struct pci_dev *pdev,
 		sm750_dev->mtrr.vram = arch_phys_wc_add(sm750_dev->vidmem_start,
 							sm750_dev->vidmem_size);
 
-	memset_io(sm750_dev->pvMem, 0, sm750_dev->vidmem_size);
+	memset_io(sm750_dev->vmem, 0, sm750_dev->vidmem_size);
 
 	pci_set_drvdata(pdev, sm750_dev);
 
@@ -1060,7 +1060,7 @@ static void lynxfb_pci_remove(struct pci_dev *pdev)
 	arch_phys_wc_del(sm750_dev->mtrr.vram);
 
 	iounmap(sm750_dev->pvReg);
-	iounmap(sm750_dev->pvMem);
+	iounmap(sm750_dev->vmem);
 	pci_release_region(pdev, 1);
 	kfree(g_settings);
 }
diff --git a/drivers/staging/sm750fb/sm750.h b/drivers/staging/sm750fb/sm750.h
index 67b9bfa23f41a..d2c522e67f26a 100644
--- a/drivers/staging/sm750fb/sm750.h
+++ b/drivers/staging/sm750fb/sm750.h
@@ -13,9 +13,9 @@
 #endif
 
 enum sm750_pnltype {
-	sm750_24TFT = 0,	/* 24bit tft */
-	sm750_dualTFT = 2,	/* dual 18 bit tft */
-	sm750_doubleTFT = 1,	/* 36 bit double pixel tft */
+	SM750_24TFT = 0,	/* 24bit tft */
+	SM750_DUAL_TFT = 2,	/* dual 18 bit tft */
+	SM750_DOUBLE_TFT = 1,	/* 36 bit double pixel tft */
 };
 
 /* vga channel is not concerned  */
@@ -39,13 +39,13 @@ enum sm750_path {
 };
 
 struct init_status {
-	ushort powerMode;
+	ushort power_mode;
 	/* below three clocks are in unit of MHZ*/
 	ushort chip_clk;
 	ushort mem_clk;
 	ushort master_clk;
 	ushort setAllEngOff;
-	ushort resetMemory;
+	ushort reset_memory;
 };
 
 struct lynx_accel {
@@ -98,7 +98,7 @@ struct sm750_dev {
 	__u32 vidmem_size;
 	__u32 vidreg_size;
 	void __iomem *pvReg;
-	unsigned char __iomem *pvMem;
+	unsigned char __iomem *vmem;
 	/* locks*/
 	spinlock_t slock;
 
diff --git a/drivers/staging/sm750fb/sm750_accel.c b/drivers/staging/sm750fb/sm750_accel.c
index 0f94d859e91c6..ec2f0a6aa57d8 100644
--- a/drivers/staging/sm750fb/sm750_accel.c
+++ b/drivers/staging/sm750fb/sm750_accel.c
@@ -90,14 +90,15 @@ int sm750_hw_fillrect(struct lynx_accel *accel,
 		      u32 color, u32 rop)
 {
 	u32 de_ctrl;
+	int ret;
 
-	if (accel->de_wait() != 0) {
+	ret = accel->de_wait();
+	if (ret) {
 		/*
-		 * int time wait and always busy,seems hardware
-		 * got something error
+		 * Timeout waiting and engine always busy, seems like a hardware issue
 		 */
 		pr_debug("De engine always busy\n");
-		return -1;
+		return ret;
 	}
 
 	write_dpr(accel, DE_WINDOW_DESTINATION_BASE, base); /* dpr40 */
@@ -154,6 +155,7 @@ int sm750_hw_copyarea(struct lynx_accel *accel,
 		      unsigned int rop2)
 {
 	unsigned int direction, de_ctrl;
+	int ret;
 
 	direction = LEFT_TO_RIGHT;
 	/* Direction of ROP2 operation: 1 = Left to Right, (-1) = Right to Left */
@@ -263,8 +265,9 @@ int sm750_hw_copyarea(struct lynx_accel *accel,
 		   DE_WINDOW_WIDTH_DST_MASK) |
 		  (source_pitch / Bpp & DE_WINDOW_WIDTH_SRC_MASK)); /* dpr3c */
 
-	if (accel->de_wait() != 0)
-		return -1;
+	ret = accel->de_wait();
+	if (ret)
+		return ret;
 
 	write_dpr(accel, DE_SOURCE,
 		  ((sx << DE_SOURCE_X_K1_SHIFT) & DE_SOURCE_X_K1_MASK) |
@@ -326,14 +329,16 @@ int sm750_hw_imageblit(struct lynx_accel *accel, const char *src_buf,
 	unsigned int de_ctrl = 0;
 	unsigned char remain[4];
 	int i, j;
+	int ret;
 
 	start_bit &= 7; /* Just make sure the start bit is within legal range */
 	bytes_per_scan = (width + start_bit + 7) / 8;
 	words_per_scan = bytes_per_scan & ~3;
 	bytes_remain = bytes_per_scan & 3;
 
-	if (accel->de_wait() != 0)
-		return -1;
+	ret = accel->de_wait();
+	if (ret)
+		return ret;
 
 	/*
 	 * 2D Source Base.
diff --git a/drivers/staging/sm750fb/sm750_cursor.c b/drivers/staging/sm750fb/sm750_cursor.c
index 7ede144905c92..f0338e6e76b1c 100644
--- a/drivers/staging/sm750fb/sm750_cursor.c
+++ b/drivers/staging/sm750fb/sm750_cursor.c
@@ -130,46 +130,3 @@ void sm750_hw_cursor_set_data(struct lynx_cursor *cursor, u16 rop,
 		}
 	}
 }
-
-void sm750_hw_cursor_set_data2(struct lynx_cursor *cursor, u16 rop,
-			       const u8 *pcol, const u8 *pmsk)
-{
-	int i, j, count, pitch, offset;
-	u8 color, mask;
-	u16 data;
-	void __iomem *pbuffer, *pstart;
-
-	/*  in byte*/
-	pitch = cursor->w >> 3;
-
-	/* in byte	*/
-	count = pitch * cursor->h;
-
-	/* in byte */
-	offset = cursor->max_w * 2 / 8;
-
-	data = 0;
-	pstart = cursor->vstart;
-	pbuffer = pstart;
-
-	for (i = 0; i < count; i++) {
-		color = *pcol++;
-		mask = *pmsk++;
-		data = 0;
-
-		for (j = 0; j < 8; j++) {
-			if (mask & (1 << j))
-				data |= ((color & (1 << j)) ? 1 : 2) << (j * 2);
-		}
-		iowrite16(data, pbuffer);
-
-		/* assume pitch is 1,2,4,8,...*/
-		if (!(i & (pitch - 1))) {
-			/* need a return */
-			pstart += offset;
-			pbuffer = pstart;
-		} else {
-			pbuffer += sizeof(u16);
-		}
-	}
-}
diff --git a/drivers/staging/sm750fb/sm750_cursor.h b/drivers/staging/sm750fb/sm750_cursor.h
index 88fa02f6377a4..51ba0da0270c2 100644
--- a/drivers/staging/sm750fb/sm750_cursor.h
+++ b/drivers/staging/sm750fb/sm750_cursor.h
@@ -10,6 +10,4 @@ void sm750_hw_cursor_set_pos(struct lynx_cursor *cursor, int x, int y);
 void sm750_hw_cursor_set_color(struct lynx_cursor *cursor, u32 fg, u32 bg);
 void sm750_hw_cursor_set_data(struct lynx_cursor *cursor, u16 rop,
 			      const u8 *data, const u8 *mask);
-void sm750_hw_cursor_set_data2(struct lynx_cursor *cursor, u16 rop,
-			       const u8 *data, const u8 *mask);
 #endif
diff --git a/drivers/staging/sm750fb/sm750_hw.c b/drivers/staging/sm750fb/sm750_hw.c
index a2798d428663e..6f7c354a34a1b 100644
--- a/drivers/staging/sm750fb/sm750_hw.c
+++ b/drivers/staging/sm750fb/sm750_hw.c
@@ -29,8 +29,6 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 {
 	int ret;
 
-	ret = 0;
-
 	sm750_dev->vidreg_start = pci_resource_start(pdev, 1);
 	sm750_dev->vidreg_size = SZ_2M;
 
@@ -66,9 +64,9 @@ int hw_sm750_map(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 	sm750_dev->vidmem_size = ddk750_get_vm_size();
 
 	/* reserve the vidmem space of smi adaptor */
-	sm750_dev->pvMem =
+	sm750_dev->vmem =
 		ioremap_wc(sm750_dev->vidmem_start, sm750_dev->vidmem_size);
-	if (!sm750_dev->pvMem) {
+	if (!sm750_dev->vmem) {
 		dev_err(&pdev->dev, "Map video memory failed\n");
 		ret = -EFAULT;
 		goto err_unmap_reg;
@@ -128,12 +126,12 @@ int hw_sm750_inithw(struct sm750_dev *sm750_dev, struct pci_dev *pdev)
 		      ~(PANEL_DISPLAY_CTRL_DUAL_DISPLAY |
 			PANEL_DISPLAY_CTRL_DOUBLE_PIXEL);
 		switch (sm750_dev->pnltype) {
-		case sm750_24TFT:
+		case SM750_24TFT:
 			break;
-		case sm750_doubleTFT:
+		case SM750_DOUBLE_TFT:
 			val |= PANEL_DISPLAY_CTRL_DOUBLE_PIXEL;
 			break;
-		case sm750_dualTFT:
+		case SM750_DUAL_TFT:
 			val |= PANEL_DISPLAY_CTRL_DUAL_DISPLAY;
 			break;
 		}
@@ -243,7 +241,6 @@ int hw_sm750_crtc_set_mode(struct lynxfb_crtc *crtc,
 	struct sm750_dev *sm750_dev;
 	struct lynxfb_par *par;
 
-	ret = 0;
 	par = container_of(crtc, struct lynxfb_par, crtc);
 	sm750_dev = par->dev;
 
@@ -501,8 +498,8 @@ int hw_sm750le_de_wait(void)
 		    (DE_STATE2_DE_FIFO_EMPTY | DE_STATE2_DE_MEM_FIFO_EMPTY))
 			return 0;
 	}
-	/* timeout error */
-	return -1;
+
+	return -ETIMEDOUT;
 }
 
 int hw_sm750_de_wait(void)
@@ -519,8 +516,8 @@ int hw_sm750_de_wait(void)
 		    (SYSTEM_CTRL_DE_FIFO_EMPTY | SYSTEM_CTRL_DE_MEM_FIFO_EMPTY))
 			return 0;
 	}
-	/* timeout error */
-	return -1;
+
+	return -ETIMEDOUT;
 }
 
 int hw_sm750_pan_display(struct lynxfb_crtc *crtc,
diff --git a/drivers/staging/vme_user/vme.c b/drivers/staging/vme_user/vme.c
index 7220aba7b919b..b5c66b66ce322 100644
--- a/drivers/staging/vme_user/vme.c
+++ b/drivers/staging/vme_user/vme.c
@@ -253,7 +253,7 @@ struct vme_resource *vme_slave_request(struct vme_dev *vdev, u32 address,
 {
 	struct vme_bridge *bridge;
 	struct vme_slave_resource *allocated_image = NULL;
-	struct vme_slave_resource *slave_image = NULL;
+	struct vme_slave_resource *slave_image;
 	struct vme_resource *resource = NULL;
 
 	bridge = vdev->bridge;
@@ -274,7 +274,7 @@ struct vme_resource *vme_slave_request(struct vme_dev *vdev, u32 address,
 		mutex_lock(&slave_image->mtx);
 		if (((slave_image->address_attr & address) == address) &&
 		    ((slave_image->cycle_attr & cycle) == cycle) &&
-		    (slave_image->locked == 0)) {
+		    !slave_image->locked) {
 			slave_image->locked = 1;
 			mutex_unlock(&slave_image->mtx);
 			allocated_image = slave_image;
@@ -419,7 +419,7 @@ void vme_slave_free(struct vme_resource *resource)
 
 	/* Unlock image */
 	mutex_lock(&slave_image->mtx);
-	if (slave_image->locked == 0)
+	if (!slave_image->locked)
 		dev_err(bridge->parent, "Image is already free\n");
 
 	slave_image->locked = 0;
@@ -447,7 +447,7 @@ struct vme_resource *vme_master_request(struct vme_dev *vdev, u32 address,
 {
 	struct vme_bridge *bridge;
 	struct vme_master_resource *allocated_image = NULL;
-	struct vme_master_resource *master_image = NULL;
+	struct vme_master_resource *master_image;
 	struct vme_resource *resource = NULL;
 
 	bridge = vdev->bridge;
@@ -469,7 +469,7 @@ struct vme_resource *vme_master_request(struct vme_dev *vdev, u32 address,
 		if (((master_image->address_attr & address) == address) &&
 		    ((master_image->cycle_attr & cycle) == cycle) &&
 		    ((master_image->width_attr & dwidth) == dwidth) &&
-		    (master_image->locked == 0)) {
+		    !master_image->locked) {
 			master_image->locked = 1;
 			spin_unlock(&master_image->lock);
 			allocated_image = master_image;
@@ -793,7 +793,7 @@ void vme_master_free(struct vme_resource *resource)
 
 	/* Unlock image */
 	spin_lock(&master_image->lock);
-	if (master_image->locked == 0)
+	if (!master_image->locked)
 		dev_err(bridge->parent, "Image is already free\n");
 
 	master_image->locked = 0;
@@ -818,7 +818,7 @@ struct vme_resource *vme_dma_request(struct vme_dev *vdev, u32 route)
 {
 	struct vme_bridge *bridge;
 	struct vme_dma_resource *allocated_ctrlr = NULL;
-	struct vme_dma_resource *dma_ctrlr = NULL;
+	struct vme_dma_resource *dma_ctrlr;
 	struct vme_resource *resource = NULL;
 
 	/* XXX Not checking resource attributes */
@@ -841,7 +841,7 @@ struct vme_resource *vme_dma_request(struct vme_dev *vdev, u32 route)
 		/* Find an unlocked and compatible controller */
 		mutex_lock(&dma_ctrlr->mtx);
 		if (((dma_ctrlr->route_attr & route) == route) &&
-		    (dma_ctrlr->locked == 0)) {
+		    !dma_ctrlr->locked) {
 			dma_ctrlr->locked = 1;
 			mutex_unlock(&dma_ctrlr->mtx);
 			allocated_ctrlr = dma_ctrlr;
@@ -1365,7 +1365,7 @@ void vme_irq_free(struct vme_dev *vdev, int level, int statid)
 	bridge->irq[level - 1].count--;
 
 	/* Disable IRQ level if no more interrupts attached at this level*/
-	if (bridge->irq[level - 1].count == 0)
+	if (!bridge->irq[level - 1].count)
 		bridge->irq_set(bridge, level, 0, 1);
 
 	bridge->irq[level - 1].callback[statid].func = NULL;
@@ -1426,7 +1426,7 @@ struct vme_resource *vme_lm_request(struct vme_dev *vdev)
 {
 	struct vme_bridge *bridge;
 	struct vme_lm_resource *allocated_lm = NULL;
-	struct vme_lm_resource *lm = NULL;
+	struct vme_lm_resource *lm;
 	struct vme_resource *resource = NULL;
 
 	bridge = vdev->bridge;
@@ -1445,7 +1445,7 @@ struct vme_resource *vme_lm_request(struct vme_dev *vdev)
 
 		/* Find an unlocked controller */
 		mutex_lock(&lm->mtx);
-		if (lm->locked == 0) {
+		if (!lm->locked) {
 			lm->locked = 1;
 			mutex_unlock(&lm->mtx);
 			allocated_lm = lm;
@@ -1766,7 +1766,7 @@ int vme_register_bridge(struct vme_bridge *bridge)
 
 	mutex_lock(&vme_buses_lock);
 	for (i = 0; i < sizeof(vme_bus_numbers) * 8; i++) {
-		if ((vme_bus_numbers & (1 << i)) == 0) {
+		if (!(vme_bus_numbers & (1 << i))) {
 			vme_bus_numbers |= (1 << i);
 			bridge->num = i;
 			INIT_LIST_HEAD(&bridge->devices);
diff --git a/drivers/target/loopback/tcm_loop.c b/drivers/target/loopback/tcm_loop.c
index a25fd826b5420..1102973457513 100644
--- a/drivers/target/loopback/tcm_loop.c
+++ b/drivers/target/loopback/tcm_loop.c
@@ -393,6 +393,7 @@ static int tcm_loop_driver_probe(struct device *dev)
 	if (error) {
 		pr_err("%s: scsi_add_host failed\n", __func__);
 		scsi_host_put(sh);
+		tl_hba->sh = NULL;
 		return -ENODEV;
 	}
 	return 0;
@@ -406,8 +407,10 @@ static void tcm_loop_driver_remove(struct device *dev)
 	tl_hba = to_tcm_loop_hba(dev);
 	sh = tl_hba->sh;
 
-	scsi_remove_host(sh);
-	scsi_host_put(sh);
+	if (sh) {
+		scsi_remove_host(sh);
+		scsi_host_put(sh);
+	}
 }
 
 static void tcm_loop_release_adapter(struct device *dev)
@@ -436,6 +439,11 @@ static int tcm_loop_setup_hba_bus(struct tcm_loop_hba *tl_hba, int tcm_loop_host
 		return -ENODEV;
 	}
 
+	if (!tl_hba->sh) {
+		device_unregister(&tl_hba->dev);
+		return -ENODEV;
+	}
+
 	return 0;
 }
 
diff --git a/drivers/tee/amdtee/amdtee_private.h b/drivers/tee/amdtee/amdtee_private.h
index d87050033894b..2c5ba02258b8a 100644
--- a/drivers/tee/amdtee/amdtee_private.h
+++ b/drivers/tee/amdtee/amdtee_private.h
@@ -65,13 +65,9 @@ struct amdtee_session {
 /**
  * struct amdtee_context_data - AMD-TEE driver context data
  * @sess_list:    Keeps track of sessions opened in current TEE context
- * @shm_list:     Keeps track of buffers allocated and mapped in current TEE
- *                context
  */
 struct amdtee_context_data {
 	struct list_head sess_list;
-	struct list_head shm_list;
-	struct mutex shm_mutex;   /* synchronizes access to @shm_list */
 };
 
 struct amdtee_driver_data {
@@ -83,17 +79,6 @@ struct shmem_desc {
 	u64 size;
 };
 
-/**
- * struct amdtee_shm_data - Shared memory data
- * @kaddr:	Kernel virtual address of shared memory
- * @buf_id:	Buffer id of memory mapped by TEE_CMD_ID_MAP_SHARED_MEM
- */
-struct amdtee_shm_data {
-	struct  list_head shm_node;
-	void    *kaddr;
-	u32     buf_id;
-};
-
 /**
  * struct amdtee_ta_data - Keeps track of all TAs loaded in AMD Secure
  *			   Processor
@@ -168,5 +153,4 @@ int handle_invoke_cmd(struct tee_ioctl_invoke_arg *arg, u32 sinfo,
 
 struct tee_shm_pool *amdtee_config_shm(void);
 
-u32 get_buffer_id(struct tee_shm *shm);
 #endif /*AMDTEE_PRIVATE_H*/
diff --git a/drivers/tee/amdtee/call.c b/drivers/tee/amdtee/call.c
index 441b2ceaafc32..23ccd0f037a7c 100644
--- a/drivers/tee/amdtee/call.c
+++ b/drivers/tee/amdtee/call.c
@@ -45,7 +45,7 @@ static int tee_params_to_amd_params(struct tee_param *tee, u32 count,
 
 		/* It is assumed that all values are within 2^32-1 */
 		if (type > TEE_OP_PARAM_TYPE_VALUE_INOUT) {
-			u32 buf_id = get_buffer_id(tee[i].u.memref.shm);
+			u32 buf_id = (u32)tee[i].u.memref.shm->sec_world_id;
 
 			amd->params[i].mref.buf_id = buf_id;
 			amd->params[i].mref.offset = tee[i].u.memref.shm_offs;
diff --git a/drivers/tee/amdtee/core.c b/drivers/tee/amdtee/core.c
index a1347d04b3ac0..0e56f4cf2697b 100644
--- a/drivers/tee/amdtee/core.c
+++ b/drivers/tee/amdtee/core.c
@@ -43,8 +43,6 @@ static int amdtee_open(struct tee_context *ctx)
 		return -ENOMEM;
 
 	INIT_LIST_HEAD(&ctxdata->sess_list);
-	INIT_LIST_HEAD(&ctxdata->shm_list);
-	mutex_init(&ctxdata->shm_mutex);
 
 	ctx->data = ctxdata;
 	return 0;
@@ -87,7 +85,6 @@ static void amdtee_release(struct tee_context *ctx)
 		list_del(&sess->list_node);
 		release_session(sess);
 	}
-	mutex_destroy(&ctxdata->shm_mutex);
 	kfree(ctxdata);
 
 	ctx->data = NULL;
@@ -152,23 +149,6 @@ static struct amdtee_session *find_session(struct amdtee_context_data *ctxdata,
 	return NULL;
 }
 
-u32 get_buffer_id(struct tee_shm *shm)
-{
-	struct amdtee_context_data *ctxdata = shm->ctx->data;
-	struct amdtee_shm_data *shmdata;
-	u32 buf_id = 0;
-
-	mutex_lock(&ctxdata->shm_mutex);
-	list_for_each_entry(shmdata, &ctxdata->shm_list, shm_node)
-		if (shmdata->kaddr == shm->kaddr) {
-			buf_id = shmdata->buf_id;
-			break;
-		}
-	mutex_unlock(&ctxdata->shm_mutex);
-
-	return buf_id;
-}
-
 static DEFINE_MUTEX(drv_mutex);
 static int copy_ta_binary(struct tee_context *ctx, void *ptr, void **ta,
 			  size_t *ta_size)
@@ -342,8 +322,6 @@ int amdtee_close_session(struct tee_context *ctx, u32 session)
 
 int amdtee_map_shmem(struct tee_shm *shm)
 {
-	struct amdtee_context_data *ctxdata;
-	struct amdtee_shm_data *shmnode;
 	struct shmem_desc shmem;
 	int rc, count;
 	u32 buf_id;
@@ -351,10 +329,6 @@ int amdtee_map_shmem(struct tee_shm *shm)
 	if (!shm)
 		return -EINVAL;
 
-	shmnode = kmalloc_obj(*shmnode);
-	if (!shmnode)
-		return -ENOMEM;
-
 	count = 1;
 	shmem.kaddr = shm->kaddr;
 	shmem.size = shm->size;
@@ -366,44 +340,26 @@ int amdtee_map_shmem(struct tee_shm *shm)
 	rc = handle_map_shmem(count, &shmem, &buf_id);
 	if (rc) {
 		pr_err("map_shmem failed: ret = %d\n", rc);
-		kfree(shmnode);
 		return rc;
 	}
 
-	shmnode->kaddr = shm->kaddr;
-	shmnode->buf_id = buf_id;
-	ctxdata = shm->ctx->data;
-	mutex_lock(&ctxdata->shm_mutex);
-	list_add(&shmnode->shm_node, &ctxdata->shm_list);
-	mutex_unlock(&ctxdata->shm_mutex);
+	shm->sec_world_id = buf_id;
 
-	pr_debug("buf_id :[%x] kaddr[%p]\n", shmnode->buf_id, shmnode->kaddr);
+	pr_debug("buf_id :[%x] kaddr[%p]\n", buf_id, shm->kaddr);
 
 	return 0;
 }
 
 void amdtee_unmap_shmem(struct tee_shm *shm)
 {
-	struct amdtee_context_data *ctxdata;
-	struct amdtee_shm_data *shmnode;
 	u32 buf_id;
 
 	if (!shm)
 		return;
 
-	buf_id = get_buffer_id(shm);
-	/* Unmap the shared memory from TEE */
+	buf_id = (u32)shm->sec_world_id;
 	handle_unmap_shmem(buf_id);
-
-	ctxdata = shm->ctx->data;
-	mutex_lock(&ctxdata->shm_mutex);
-	list_for_each_entry(shmnode, &ctxdata->shm_list, shm_node)
-		if (buf_id == shmnode->buf_id) {
-			list_del(&shmnode->shm_node);
-			kfree(shmnode);
-			break;
-		}
-	mutex_unlock(&ctxdata->shm_mutex);
+	shm->sec_world_id = 0;
 }
 
 int amdtee_invoke_func(struct tee_context *ctx,
diff --git a/drivers/tee/optee/call.c b/drivers/tee/optee/call.c
index 9effe88049e9d..e046aff618286 100644
--- a/drivers/tee/optee/call.c
+++ b/drivers/tee/optee/call.c
@@ -602,7 +602,8 @@ static bool is_normal_memory(pgprot_t p)
 	return (((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEALLOC) ||
 		((pgprot_val(p) & L_PTE_MT_MASK) == L_PTE_MT_WRITEBACK));
 #elif defined(CONFIG_ARM64)
-	return (pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL);
+	return ((pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL)) ||
+	       ((pgprot_val(p) & PTE_ATTRINDX_MASK) == PTE_ATTRINDX(MT_NORMAL_TAGGED));
 #else
 #error "Unsupported architecture"
 #endif
diff --git a/drivers/tee/optee/optee_msg.h b/drivers/tee/optee/optee_msg.h
index 838e1d4a22f04..7d9b12e71c035 100644
--- a/drivers/tee/optee/optee_msg.h
+++ b/drivers/tee/optee/optee_msg.h
@@ -103,9 +103,9 @@
 
 /**
  * struct optee_msg_param_tmem - temporary memory reference parameter
- * @buf_ptr:	Address of the buffer
- * @size:	Size of the buffer
- * @shm_ref:	Temporary shared memory reference, pointer to a struct tee_shm
+ * @buf_ptr:	address of the buffer
+ * @size:	size of the buffer
+ * @shm_ref:	temporary shared memory reference, pointer to a struct tee_shm
  *
  * Secure and normal world communicates pointers as physical address
  * instead of the virtual address. This is because secure and normal world
@@ -122,9 +122,9 @@ struct optee_msg_param_tmem {
 
 /**
  * struct optee_msg_param_rmem - registered memory reference parameter
- * @offs:	Offset into shared memory reference
- * @size:	Size of the buffer
- * @shm_ref:	Shared memory reference, pointer to a struct tee_shm
+ * @offs:	offset into shared memory reference
+ * @size:	size of the buffer
+ * @shm_ref:	shared memory reference, pointer to a struct tee_shm
  */
 struct optee_msg_param_rmem {
 	u64 offs;
@@ -134,12 +134,12 @@ struct optee_msg_param_rmem {
 
 /**
  * struct optee_msg_param_fmem - FF-A memory reference parameter
- * @offs_lower:	   Lower bits of offset into shared memory reference
- * @offs_upper:	   Upper bits of offset into shared memory reference
- * @internal_offs: Internal offset into the first page of shared memory
- *		   reference
- * @size:	   Size of the buffer
- * @global_id:	   Global identifier of the shared memory
+ * @offs_low:		lower bits of offset into shared memory reference
+ * @offs_high:		higher bits of offset into shared memory reference
+ * @internal_offs:	internal offset into the first page of shared memory
+ *			reference
+ * @size:		size of the buffer
+ * @global_id:		global identifier of the shared memory
  */
 struct optee_msg_param_fmem {
 	u32 offs_low;
@@ -151,6 +151,9 @@ struct optee_msg_param_fmem {
 
 /**
  * struct optee_msg_param_value - opaque value parameter
+ * @a: first opaque value
+ * @b: second opaque value
+ * @c: third opaque value
  *
  * Value parameters are passed unchecked between normal and secure world.
  */
@@ -168,6 +171,7 @@ struct optee_msg_param_value {
  * @fmem:	parameter by FF-A registered memory reference
  * @value:	parameter by opaque value
  * @octets:	parameter by octet string
+ * @u:		union holding OP-TEE msg parameter
  *
  * @attr & OPTEE_MSG_ATTR_TYPE_MASK indicates if tmem, rmem or value is used in
  * the union. OPTEE_MSG_ATTR_TYPE_VALUE_* indicates value or octets,
@@ -189,16 +193,18 @@ struct optee_msg_param {
 
 /**
  * struct optee_msg_arg - call argument
- * @cmd: Command, one of OPTEE_MSG_CMD_* or OPTEE_MSG_RPC_CMD_*
- * @func: Trusted Application function, specific to the Trusted Application,
- *	     used if cmd == OPTEE_MSG_CMD_INVOKE_COMMAND
- * @session: In parameter for all OPTEE_MSG_CMD_* except
- *	     OPTEE_MSG_CMD_OPEN_SESSION where it's an output parameter instead
- * @cancel_id: Cancellation id, a unique value to identify this request
- * @ret: return value
- * @ret_origin: origin of the return value
- * @num_params: number of parameters supplied to the OS Command
- * @params: the parameters supplied to the OS Command
+ * @cmd:	command, one of OPTEE_MSG_CMD_* or OPTEE_MSG_RPC_CMD_*
+ * @func:	Trusted Application function, specific to the Trusted
+ *		Application, used if cmd == OPTEE_MSG_CMD_INVOKE_COMMAND
+ * @session:	in parameter for all OPTEE_MSG_CMD_* except
+ *		OPTEE_MSG_CMD_OPEN_SESSION where it's an output parameter
+ *		instead
+ * @cancel_id:	cancellation id, a unique value to identify this request
+ * @pad:	padding for alignment
+ * @ret:	return value
+ * @ret_origin:	origin of the return value
+ * @num_params:	number of parameters supplied to the OS Command
+ * @params:	the parameters supplied to the OS Command
  *
  * All normal calls to Trusted OS uses this struct. If cmd requires further
  * information than what these fields hold it can be passed as a parameter
diff --git a/drivers/tee/optee/optee_private.h b/drivers/tee/optee/optee_private.h
index acd3051c4879d..aefe1e6f56891 100644
--- a/drivers/tee/optee/optee_private.h
+++ b/drivers/tee/optee/optee_private.h
@@ -47,11 +47,11 @@ typedef void (optee_invoke_fn)(unsigned long, unsigned long, unsigned long,
 				unsigned long, unsigned long,
 				struct arm_smccc_res *);
 
-/*
+/**
  * struct optee_call_waiter - TEE entry may need to wait for a free TEE thread
- * @list_node		Reference in waiters list
- * @c			Waiting completion reference
- * @sys_thread		True if waiter belongs to a system thread
+ * @list_node:	reference in waiters list
+ * @c:		waiting completion reference
+ * @sys_thread:	true if waiter belongs to a system thread
  */
 struct optee_call_waiter {
 	struct list_head list_node;
@@ -59,13 +59,13 @@ struct optee_call_waiter {
 	bool sys_thread;
 };
 
-/*
+/**
  * struct optee_call_queue - OP-TEE call queue management
- * @mutex			Serializes access to this struct
- * @waiters			List of threads waiting to enter OP-TEE
- * @total_thread_count		Overall number of thread context in OP-TEE or 0
- * @free_thread_count		Number of threads context free in OP-TEE
- * @sys_thread_req_count	Number of registered system thread sessions
+ * @mutex:			serializes access to this struct
+ * @waiters:			list of threads waiting to enter OP-TEE
+ * @total_thread_count:		overall number of thread context in OP-TEE or 0
+ * @free_thread_count:		number of threads context free in OP-TEE
+ * @sys_thread_req_count:	number of registered system thread sessions
  */
 struct optee_call_queue {
 	/* Serializes access to this struct */
@@ -96,17 +96,17 @@ struct optee_shm_arg_cache {
 
 /**
  * struct optee_supp - supplicant synchronization struct
- * @ctx			the context of current connected supplicant.
- *			if !NULL the supplicant device is available for use,
- *			else busy
- * @mutex:		held while accessing content of this struct
- * @req_id:		current request id if supplicant is doing synchronous
- *			communication, else -1
- * @reqs:		queued request not yet retrieved by supplicant
- * @idr:		IDR holding all requests currently being processed
- *			by supplicant
- * @reqs_c:		completion used by supplicant when waiting for a
- *			request to be queued.
+ * @mutex:	held while accessing content of this struct
+ * @ctx:	the context of current connected supplicant.
+ *		if !NULL the supplicant device is available for use,
+ *		else busy
+ * @req_id:	current request id if supplicant is doing synchronous
+ *		communication, else -1
+ * @reqs:	queued request not yet retrieved by supplicant
+ * @idr:	IDR holding all requests currently being processed
+ *		by supplicant
+ * @reqs_c:	completion used by supplicant when waiting for a
+ *		request to be queued.
  */
 struct optee_supp {
 	/* Serializes access to this struct */
@@ -119,25 +119,25 @@ struct optee_supp {
 	struct completion reqs_c;
 };
 
-/*
+/**
  * struct optee_pcpu - per cpu notif private struct passed to work functions
- * @optee		optee device reference
+ * @optee:	optee device reference
  */
 struct optee_pcpu {
 	struct optee *optee;
 };
 
-/*
+/**
  * struct optee_smc - optee smc communication struct
- * @invoke_fn		handler function to invoke secure monitor
- * @memremaped_shm	virtual address of memory in shared memory pool
+ * @invoke_fn:		handler function to invoke secure monitor
+ * @memremaped_shm:	virtual address of memory in shared memory pool
  * @sec_caps:		secure world capabilities defined by
  *			OPTEE_SMC_SEC_CAP_* in optee_smc.h
- * @notif_irq		interrupt used as async notification by OP-TEE or 0
- * @optee_pcpu		per_cpu optee instance for per cpu work or NULL
- * @notif_pcpu_wq	workqueue for per cpu asynchronous notification or NULL
- * @notif_pcpu_work	work for per cpu asynchronous notification
- * @notif_cpuhp_state   CPU hotplug state assigned for pcpu interrupt management
+ * @notif_irq:		interrupt used as async notification by OP-TEE or 0
+ * @optee_pcpu:		per_cpu optee instance for per cpu work or NULL
+ * @notif_pcpu_wq:	workqueue for per cpu asynchronous notification or NULL
+ * @notif_pcpu_work:	work for per cpu asynchronous notification
+ * @notif_cpuhp_state:	CPU hotplug state assigned for pcpu interrupt management
  */
 struct optee_smc {
 	optee_invoke_fn *invoke_fn;
@@ -151,13 +151,15 @@ struct optee_smc {
 };
 
 /**
- * struct optee_ffa_data -  FFA communication struct
- * @ffa_dev		FFA device, contains the destination id, the id of
+ * struct optee_ffa -  FFA communication struct
+ * @ffa_dev:		FFA device, contains the destination id, the id of
  *			OP-TEE in secure world
- * @bottom_half_value	Notification ID used for bottom half signalling or
+ * @bottom_half_value:	notification ID used for bottom half signalling or
  *			U32_MAX if unused
- * @mutex		Serializes access to @global_ids
- * @global_ids		FF-A shared memory global handle translation
+ * @mutex:		serializes access to @global_ids
+ * @global_ids:		FF-A shared memory global handle translation
+ * @notif_wq:		workqueue for FF-A asynchronous notification
+ * @notif_work:		work for FF-A asynchronous notification
  */
 struct optee_ffa {
 	struct ffa_device *ffa_dev;
@@ -222,26 +224,32 @@ struct optee_ops {
 
 /**
  * struct optee - main service struct
- * @supp_teedev:	supplicant device
- * @teedev:		client device
- * @ops:		internal callbacks for different ways to reach secure
- *			world
- * @ctx:		driver internal TEE context
- * @smc:		specific to SMC ABI
- * @ffa:		specific to FF-A ABI
- * @call_queue:		queue of threads waiting to call @invoke_fn
- * @notif:		notification synchronization struct
- * @supp:		supplicant synchronization struct for RPC to supplicant
- * @pool:		shared memory pool
- * @mutex:		mutex protecting @rpmb_dev
- * @rpmb_dev:		current RPMB device or NULL
- * @rpmb_scan_bus_done	flag if device registation of RPMB dependent devices
- *			was already done
- * @rpmb_scan_bus_work	workq to for an RPMB device and to scan optee bus
- *			and register RPMB dependent optee drivers
- * @rpc_param_count:	If > 0 number of RPC parameters to make room for
- * @scan_bus_done	flag if device registation was already done.
- * @scan_bus_work	workq to scan optee bus and register optee drivers
+ * @supp_teedev:		supplicant device
+ * @teedev:			client device
+ * @ops:			internal callbacks for different ways to reach
+ *				secure world
+ * @ctx:			driver internal TEE context
+ * @smc:			specific to SMC ABI
+ * @ffa:			specific to FF-A ABI
+ * @shm_arg_cache:		shared memory cache argument
+ * @call_queue:			queue of threads waiting to call @invoke_fn
+ * @notif:			notification synchronization struct
+ * @supp:			supplicant synchronization struct for RPC to
+ *				supplicant
+ * @pool:			shared memory pool
+ * @rpmb_dev_mutex:		mutex protecting @rpmb_dev
+ * @rpmb_dev:			current RPMB device or NULL
+ * @rpmb_intf:			RPMB notifier block
+ * @rpc_param_count:		if > 0 number of RPC parameters to make room for
+ * @scan_bus_done:		flag if device registation was already done
+ * @rpmb_scan_bus_done:		flag if device registation of RPMB dependent
+ *				devices was already done
+ * @in_kernel_rpmb_routing:	flag if OP-TEE supports in-kernel RPMB routing
+ * @scan_bus_work:		workq to scan optee bus and register optee
+ *				drivers
+ * @rpmb_scan_bus_work:		workq to for an RPMB device and to scan optee
+ *				bus and register RPMB dependent optee drivers
+ * @revision:			OP-TEE OS revision
  */
 struct optee {
 	struct tee_device *supp_teedev;
diff --git a/drivers/tee/optee/supp.c b/drivers/tee/optee/supp.c
index a3d11b1f90fa5..06747e90c2309 100644
--- a/drivers/tee/optee/supp.c
+++ b/drivers/tee/optee/supp.c
@@ -10,7 +10,11 @@
 struct optee_supp_req {
 	struct list_head link;
 
+	int id;
+
 	bool in_queue;
+	bool processed;
+
 	u32 func;
 	u32 ret;
 	size_t num_params;
@@ -19,6 +23,9 @@ struct optee_supp_req {
 	struct completion c;
 };
 
+/* It is temporary request used for revoked pending request in supp->idr. */
+#define INVALID_REQ_PTR ((struct optee_supp_req *)ERR_PTR(-EBADF))
+
 void optee_supp_init(struct optee_supp *supp)
 {
 	memset(supp, 0, sizeof(*supp));
@@ -39,21 +46,23 @@ void optee_supp_release(struct optee_supp *supp)
 {
 	int id;
 	struct optee_supp_req *req;
-	struct optee_supp_req *req_tmp;
 
 	mutex_lock(&supp->mutex);
 
-	/* Abort all request retrieved by supplicant */
+	/* Abort all request */
 	idr_for_each_entry(&supp->idr, req, id) {
 		idr_remove(&supp->idr, id);
-		req->ret = TEEC_ERROR_COMMUNICATION;
-		complete(&req->c);
-	}
+		/* Skip if request was already marked invalid */
+		if (IS_ERR(req))
+			continue;
 
-	/* Abort all queued requests */
-	list_for_each_entry_safe(req, req_tmp, &supp->reqs, link) {
-		list_del(&req->link);
-		req->in_queue = false;
+		/* For queued requests where supplicant has not seen it */
+		if (req->in_queue) {
+			list_del(&req->link);
+			req->in_queue = false;
+		}
+
+		req->processed = true;
 		req->ret = TEEC_ERROR_COMMUNICATION;
 		complete(&req->c);
 	}
@@ -100,8 +109,16 @@ u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params,
 
 	/* Insert the request in the request list */
 	mutex_lock(&supp->mutex);
+	req->id = idr_alloc(&supp->idr, req, 1, 0, GFP_KERNEL);
+	if (req->id < 0) {
+		mutex_unlock(&supp->mutex);
+		kfree(req);
+		return TEEC_ERROR_OUT_OF_MEMORY;
+	}
+
 	list_add_tail(&req->link, &supp->reqs);
 	req->in_queue = true;
+	req->processed = false;
 	mutex_unlock(&supp->mutex);
 
 	/* Tell an eventual waiter there's a new request */
@@ -117,21 +134,43 @@ u32 optee_supp_thrd_req(struct tee_context *ctx, u32 func, size_t num_params,
 	if (wait_for_completion_killable(&req->c)) {
 		mutex_lock(&supp->mutex);
 		if (req->in_queue) {
+			/* Supplicant has not seen this request yet. */
+			idr_remove(&supp->idr, req->id);
 			list_del(&req->link);
 			req->in_queue = false;
+
+			ret = TEEC_ERROR_COMMUNICATION;
+		} else if (req->processed) {
+			/*
+			 * Supplicant has processed this request. Ignore the
+			 * kill signal for now and submit the result. req is not
+			 * in supp->reqs (removed by supp_pop_entry()) nor in
+			 * supp->idr (removed by supp_pop_req()).
+			 */
+			ret = req->ret;
+		} else {
+			/*
+			 * Supplicant is in the middle of processing this
+			 * request. Replace req with INVALID_REQ_PTR so that
+			 * the ID remains busy, causing optee_supp_send() to
+			 * fail on the next call to supp_pop_req() with this ID.
+			 */
+			idr_replace(&supp->idr, INVALID_REQ_PTR, req->id);
+			ret = TEEC_ERROR_COMMUNICATION;
 		}
+
 		mutex_unlock(&supp->mutex);
-		req->ret = TEEC_ERROR_COMMUNICATION;
+	} else {
+		ret = req->ret;
 	}
 
-	ret = req->ret;
 	kfree(req);
 
 	return ret;
 }
 
 static struct optee_supp_req  *supp_pop_entry(struct optee_supp *supp,
-					      int num_params, int *id)
+					      int num_params)
 {
 	struct optee_supp_req *req;
 
@@ -153,10 +192,6 @@ static struct optee_supp_req  *supp_pop_entry(struct optee_supp *supp,
 		return ERR_PTR(-EINVAL);
 	}
 
-	*id = idr_alloc(&supp->idr, req, 1, 0, GFP_KERNEL);
-	if (*id < 0)
-		return ERR_PTR(-ENOMEM);
-
 	list_del(&req->link);
 	req->in_queue = false;
 
@@ -214,7 +249,6 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 	struct optee *optee = tee_get_drvdata(teedev);
 	struct optee_supp *supp = &optee->supp;
 	struct optee_supp_req *req = NULL;
-	int id;
 	size_t num_meta;
 	int rc;
 
@@ -224,15 +258,11 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 
 	while (true) {
 		mutex_lock(&supp->mutex);
-		req = supp_pop_entry(supp, *num_params - num_meta, &id);
+		req = supp_pop_entry(supp, *num_params - num_meta);
+		if (req)
+			break; /* Keep mutex held. */
 		mutex_unlock(&supp->mutex);
 
-		if (req) {
-			if (IS_ERR(req))
-				return PTR_ERR(req);
-			break;
-		}
-
 		/*
 		 * If we didn't get a request we'll block in
 		 * wait_for_completion() to avoid needless spinning.
@@ -245,6 +275,13 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 			return -ERESTARTSYS;
 	}
 
+	/* supp->mutex held and req != NULL. */
+
+	if (IS_ERR(req)) {
+		mutex_unlock(&supp->mutex);
+		return PTR_ERR(req);
+	}
+
 	if (num_meta) {
 		/*
 		 * tee-supplicant support meta parameters -> requsts can be
@@ -252,13 +289,11 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 		 */
 		param->attr = TEE_IOCTL_PARAM_ATTR_TYPE_VALUE_INOUT |
 			      TEE_IOCTL_PARAM_ATTR_META;
-		param->u.value.a = id;
+		param->u.value.a = req->id;
 		param->u.value.b = 0;
 		param->u.value.c = 0;
 	} else {
-		mutex_lock(&supp->mutex);
-		supp->req_id = id;
-		mutex_unlock(&supp->mutex);
+		supp->req_id = req->id;
 	}
 
 	*func = req->func;
@@ -266,6 +301,7 @@ int optee_supp_recv(struct tee_context *ctx, u32 *func, u32 *num_params,
 	memcpy(param + num_meta, req->param,
 	       sizeof(struct tee_param) * req->num_params);
 
+	mutex_unlock(&supp->mutex);
 	return 0;
 }
 
@@ -297,12 +333,17 @@ static struct optee_supp_req *supp_pop_req(struct optee_supp *supp,
 	if (!req)
 		return ERR_PTR(-ENOENT);
 
+	/* optee_supp_thrd_req() already returned to optee. */
+	if (IS_ERR(req))
+		goto failed_req;
+
 	if ((num_params - nm) != req->num_params)
 		return ERR_PTR(-EINVAL);
 
+	*num_meta = nm;
+failed_req:
 	idr_remove(&supp->idr, id);
 	supp->req_id = -1;
-	*num_meta = nm;
 
 	return req;
 }
@@ -328,10 +369,9 @@ int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params,
 
 	mutex_lock(&supp->mutex);
 	req = supp_pop_req(supp, num_params, param, &num_meta);
-	mutex_unlock(&supp->mutex);
-
 	if (IS_ERR(req)) {
-		/* Something is wrong, let supplicant restart. */
+		mutex_unlock(&supp->mutex);
+		/* Something is wrong, let supplicant handel it. */
 		return PTR_ERR(req);
 	}
 
@@ -355,9 +395,10 @@ int optee_supp_send(struct tee_context *ctx, u32 ret, u32 num_params,
 		}
 	}
 	req->ret = ret;
-
+	req->processed = true;
 	/* Let the requesting thread continue */
 	complete(&req->c);
+	mutex_unlock(&supp->mutex);
 
 	return 0;
 }
diff --git a/drivers/tee/qcomtee/core.c b/drivers/tee/qcomtee/core.c
index b1cb50e434f00..60fe3b5776e36 100644
--- a/drivers/tee/qcomtee/core.c
+++ b/drivers/tee/qcomtee/core.c
@@ -306,8 +306,10 @@ int qcomtee_object_user_init(struct qcomtee_object *object,
 		break;
 	case QCOMTEE_OBJECT_TYPE_CB:
 		object->ops = ops;
-		if (!object->ops->dispatch)
-			return -EINVAL;
+		if (!object->ops->dispatch) {
+			ret = -EINVAL;
+			break;
+		}
 
 		/* If failed, "no-name". */
 		object->name = kvasprintf_const(GFP_KERNEL, fmt, ap);
diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c
index ef9642d726728..1aac50c7c1de7 100644
--- a/drivers/tee/tee_core.c
+++ b/drivers/tee/tee_core.c
@@ -530,11 +530,24 @@ static int params_to_user(struct tee_ioctl_param __user *uparams,
 	return 0;
 }
 
+static void free_params(struct tee_param *params, size_t num_params)
+{
+	size_t n;
+
+	if (!params)
+		return;
+
+	for (n = 0; n < num_params; n++)
+		if (tee_param_is_memref(params + n) && params[n].u.memref.shm)
+			tee_shm_put(params[n].u.memref.shm);
+
+	kfree(params);
+}
+
 static int tee_ioctl_open_session(struct tee_context *ctx,
 				  struct tee_ioctl_buf_data __user *ubuf)
 {
 	int rc;
-	size_t n;
 	struct tee_ioctl_buf_data buf;
 	struct tee_ioctl_open_session_arg __user *uarg;
 	struct tee_ioctl_open_session_arg arg;
@@ -595,16 +608,7 @@ static int tee_ioctl_open_session(struct tee_context *ctx,
 	 */
 	if (rc && have_session && ctx->teedev->desc->ops->close_session)
 		ctx->teedev->desc->ops->close_session(ctx, arg.session);
-
-	if (params) {
-		/* Decrease ref count for all valid shared memory pointers */
-		for (n = 0; n < arg.num_params; n++)
-			if (tee_param_is_memref(params + n) &&
-			    params[n].u.memref.shm)
-				tee_shm_put(params[n].u.memref.shm);
-		kfree(params);
-	}
-
+	free_params(params, arg.num_params);
 	return rc;
 }
 
@@ -612,7 +616,6 @@ static int tee_ioctl_invoke(struct tee_context *ctx,
 			    struct tee_ioctl_buf_data __user *ubuf)
 {
 	int rc;
-	size_t n;
 	struct tee_ioctl_buf_data buf;
 	struct tee_ioctl_invoke_arg __user *uarg;
 	struct tee_ioctl_invoke_arg arg;
@@ -657,14 +660,7 @@ static int tee_ioctl_invoke(struct tee_context *ctx,
 	}
 	rc = params_to_user(uparams, arg.num_params, params);
 out:
-	if (params) {
-		/* Decrease ref count for all valid shared memory pointers */
-		for (n = 0; n < arg.num_params; n++)
-			if (tee_param_is_memref(params + n) &&
-			    params[n].u.memref.shm)
-				tee_shm_put(params[n].u.memref.shm);
-		kfree(params);
-	}
+	free_params(params, arg.num_params);
 	return rc;
 }
 
@@ -672,7 +668,6 @@ static int tee_ioctl_object_invoke(struct tee_context *ctx,
 				   struct tee_ioctl_buf_data __user *ubuf)
 {
 	int rc;
-	size_t n;
 	struct tee_ioctl_buf_data buf;
 	struct tee_ioctl_object_invoke_arg __user *uarg;
 	struct tee_ioctl_object_invoke_arg arg;
@@ -716,14 +711,7 @@ static int tee_ioctl_object_invoke(struct tee_context *ctx,
 	}
 	rc = params_to_user(uparams, arg.num_params, params);
 out:
-	if (params) {
-		/* Decrease ref count for all valid shared memory pointers */
-		for (n = 0; n < arg.num_params; n++)
-			if (tee_param_is_memref(params + n) &&
-			    params[n].u.memref.shm)
-				tee_shm_put(params[n].u.memref.shm);
-		kfree(params);
-	}
+	free_params(params, arg.num_params);
 	return rc;
 }
 
@@ -846,9 +834,15 @@ static int tee_ioctl_supp_recv(struct tee_context *ctx,
 		return -ENOMEM;
 
 	rc = params_from_user(ctx, params, num_params, uarg->params);
-	if (rc)
-		goto out;
+	if (rc) {
+		free_params(params, num_params);
+		return rc;
+	}
 
+	/*
+	 * supp_recv() may consume and replace the supplied parameters, so the
+	 * final cleanup cannot use free_params() like the other ioctl paths.
+	 */
 	rc = ctx->teedev->desc->ops->supp_recv(ctx, &func, &num_params, params);
 	if (rc)
 		goto out;
diff --git a/drivers/tee/tee_shm.c b/drivers/tee/tee_shm.c
index e9ea9f80cfd9a..6742b3579c86d 100644
--- a/drivers/tee/tee_shm.c
+++ b/drivers/tee/tee_shm.c
@@ -435,7 +435,7 @@ register_shm_helper(struct tee_context *ctx, struct iov_iter *iter, u32 flags,
 	num_pages = iov_iter_npages(iter, INT_MAX);
 	if (!num_pages) {
 		ret = ERR_PTR(-ENOMEM);
-		goto err_ctx_put;
+		goto err_free_shm;
 	}
 
 	shm->pages = kzalloc_objs(*shm->pages, num_pages);
diff --git a/drivers/thermal/intel/intel_hfi.c b/drivers/thermal/intel/intel_hfi.c
index 8c4ae75231f8e..3273b8fe3d4d5 100644
--- a/drivers/thermal/intel/intel_hfi.c
+++ b/drivers/thermal/intel/intel_hfi.c
@@ -41,6 +41,7 @@
 #include <linux/topology.h>
 #include <linux/workqueue.h>
 
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 
 #include "intel_hfi.h"
diff --git a/drivers/thermal/intel/x86_pkg_temp_thermal.c b/drivers/thermal/intel/x86_pkg_temp_thermal.c
index 540109761f0a4..d1dd2f5910e40 100644
--- a/drivers/thermal/intel/x86_pkg_temp_thermal.c
+++ b/drivers/thermal/intel/x86_pkg_temp_thermal.c
@@ -20,6 +20,7 @@
 #include <linux/debugfs.h>
 
 #include <asm/cpu_device_id.h>
+#include <asm/cpuid/api.h>
 #include <asm/msr.h>
 
 #include "thermal_interrupt.h"
diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c
index 2f4e2dc46b8f7..9b9fa51067bd4 100644
--- a/drivers/thermal/thermal_core.c
+++ b/drivers/thermal/thermal_core.c
@@ -116,78 +116,24 @@ static int thermal_set_governor(struct thermal_zone_device *tz,
 	return ret;
 }
 
-int thermal_register_governor(struct thermal_governor *governor)
+static int __init thermal_register_governor(struct thermal_governor *governor)
 {
-	int err;
-	const char *name;
-	struct thermal_zone_device *pos;
 
 	if (!governor)
 		return -EINVAL;
 
-	guard(mutex)(&thermal_governor_lock);
-
-	err = -EBUSY;
-	if (!__find_governor(governor->name)) {
-		bool match_default;
-
-		err = 0;
-		list_add(&governor->governor_list, &thermal_governor_list);
-		match_default = !strncmp(governor->name,
-					 DEFAULT_THERMAL_GOVERNOR,
-					 THERMAL_NAME_LENGTH);
-
-		if (!def_governor && match_default)
-			def_governor = governor;
-	}
-
-	guard(mutex)(&thermal_list_lock);
-
-	list_for_each_entry(pos, &thermal_tz_list, node) {
-		/*
-		 * only thermal zones with specified tz->tzp->governor_name
-		 * may run with tz->govenor unset
-		 */
-		if (pos->governor)
-			continue;
+	if (__find_governor(governor->name))
+		return -EBUSY;
 
-		name = pos->tzp->governor_name;
+	list_add(&governor->governor_list, &thermal_governor_list);
 
-		if (!strncasecmp(name, governor->name, THERMAL_NAME_LENGTH)) {
-			int ret;
-
-			ret = thermal_set_governor(pos, governor);
-			if (ret)
-				dev_err(&pos->device,
-					"Failed to set governor %s for thermal zone %s: %d\n",
-					governor->name, pos->type, ret);
-		}
-	}
-
-	return err;
-}
-
-void thermal_unregister_governor(struct thermal_governor *governor)
-{
-	struct thermal_zone_device *pos;
-
-	if (!governor)
-		return;
-
-	guard(mutex)(&thermal_governor_lock);
-
-	if (!__find_governor(governor->name))
-		return;
-
-	list_del(&governor->governor_list);
+	if (strncmp(governor->name, DEFAULT_THERMAL_GOVERNOR, THERMAL_NAME_LENGTH))
+		return 0;
 
-	guard(mutex)(&thermal_list_lock);
+	if (!def_governor)
+		def_governor = governor;
 
-	list_for_each_entry(pos, &thermal_tz_list, node) {
-		if (!strncasecmp(pos->governor->name, governor->name,
-				 THERMAL_NAME_LENGTH))
-			thermal_set_governor(pos, NULL);
-	}
+	return 0;
 }
 
 int thermal_zone_device_set_policy(struct thermal_zone_device *tz,
@@ -225,40 +171,34 @@ int thermal_build_list_of_policies(char *buf)
 
 static void __init thermal_unregister_governors(void)
 {
-	struct thermal_governor **governor;
+	struct thermal_governor *gov, *pos;
 
-	for_each_governor_table(governor)
-		thermal_unregister_governor(*governor);
+	guard(mutex)(&thermal_governor_lock);
+
+	list_for_each_entry_safe(gov, pos, &thermal_governor_list, governor_list)
+		list_del(&gov->governor_list);
 }
 
 static int __init thermal_register_governors(void)
 {
-	int ret = 0;
 	struct thermal_governor **governor;
 
+	guard(mutex)(&thermal_governor_lock);
+
 	for_each_governor_table(governor) {
+		int ret;
+
 		ret = thermal_register_governor(*governor);
 		if (ret) {
 			pr_err("Failed to register governor: '%s'",
 			       (*governor)->name);
-			break;
+			return ret;
 		}
 
-		pr_info("Registered thermal governor '%s'",
-			(*governor)->name);
+		pr_info("Registered thermal governor '%s'", (*governor)->name);
 	}
 
-	if (ret) {
-		struct thermal_governor **gov;
-
-		for_each_governor_table(gov) {
-			if (gov == governor)
-				break;
-			thermal_unregister_governor(*gov);
-		}
-	}
-
-	return ret;
+	return 0;
 }
 
 static int __thermal_zone_device_set_mode(struct thermal_zone_device *tz,
@@ -949,34 +889,7 @@ static void thermal_unbind_cdev_from_trip(struct thermal_zone_device *tz,
 	kfree(pos);
 }
 
-static void thermal_release(struct device *dev)
-{
-	struct thermal_zone_device *tz;
-	struct thermal_cooling_device *cdev;
-
-	if (!strncmp(dev_name(dev), "thermal_zone",
-		     sizeof("thermal_zone") - 1)) {
-		tz = to_thermal_zone(dev);
-		thermal_zone_destroy_device_groups(tz);
-		thermal_set_governor(tz, NULL);
-		ida_destroy(&tz->ida);
-		mutex_destroy(&tz->lock);
-		complete(&tz->removal);
-	} else if (!strncmp(dev_name(dev), "cooling_device",
-			    sizeof("cooling_device") - 1)) {
-		cdev = to_cooling_device(dev);
-		thermal_cooling_device_destroy_sysfs(cdev);
-		kfree_const(cdev->type);
-		ida_free(&thermal_cdev_ida, cdev->id);
-		kfree(cdev);
-	}
-}
-
-static const struct class thermal_class = {
-	.name = "thermal",
-	.dev_release = thermal_release,
-};
-static bool thermal_class_unavailable __ro_after_init = true;
+static struct class *thermal_class __ro_after_init;
 
 static inline
 void print_bind_err_msg(struct thermal_zone_device *tz,
@@ -1040,42 +953,35 @@ static void thermal_cooling_device_init_complete(struct thermal_cooling_device *
 		thermal_zone_cdev_bind(tz, cdev);
 }
 
-/**
- * __thermal_cooling_device_register() - register a new thermal cooling device
- * @np:		a pointer to a device tree node.
- * @type:	the thermal cooling device type.
- * @devdata:	device private data.
- * @ops:	standard thermal cooling devices callbacks.
- *
- * This interface function adds a new thermal cooling device (fan/processor/...)
- * to /sys/class/thermal/ folder as cooling_device[0-*]. It tries to bind itself
- * to all the thermal zone devices registered at the same time.
- * It also gives the opportunity to link the cooling device to a device tree
- * node, so that it can be bound to a thermal zone created out of device tree.
- *
- * Return: a pointer to the created struct thermal_cooling_device or an
- * ERR_PTR. Caller must check return value with IS_ERR*() helpers.
- */
+static void thermal_cdev_release(struct device *dev)
+{
+	struct thermal_cooling_device *cdev = to_cooling_device(dev);
+
+	thermal_cooling_device_destroy_sysfs(cdev);
+	kfree_const(cdev->type);
+	ida_free(&thermal_cdev_ida, cdev->id);
+	kfree(cdev);
+}
+
 static struct thermal_cooling_device *
-__thermal_cooling_device_register(struct device_node *np,
-				  const char *type, void *devdata,
-				  const struct thermal_cooling_device_ops *ops)
+thermal_cooling_device_alloc(const char *type, const struct thermal_cooling_device_ops *ops)
 {
 	struct thermal_cooling_device *cdev;
-	unsigned long current_state;
 	int ret;
 
 	if (!ops || !ops->get_max_state || !ops->get_cur_state ||
 	    !ops->set_cur_state)
 		return ERR_PTR(-EINVAL);
 
-	if (thermal_class_unavailable)
+	if (!thermal_class)
 		return ERR_PTR(-ENODEV);
 
 	cdev = kzalloc_obj(*cdev);
 	if (!cdev)
 		return ERR_PTR(-ENOMEM);
 
+	cdev->ops = ops;
+
 	ret = ida_alloc(&thermal_cdev_ida, GFP_KERNEL);
 	if (ret < 0)
 		goto out_kfree_cdev;
@@ -1087,17 +993,35 @@ __thermal_cooling_device_register(struct device_node *np,
 		goto out_ida_remove;
 	}
 
+	return cdev;
+
+out_ida_remove:
+	ida_free(&thermal_cdev_ida, cdev->id);
+out_kfree_cdev:
+	kfree(cdev);
+	return ERR_PTR(ret);
+}
+
+static int thermal_cooling_device_add(struct thermal_cooling_device *cdev, void *devdata)
+{
+	unsigned long current_state;
+	int ret;
+
 	mutex_init(&cdev->lock);
 	INIT_LIST_HEAD(&cdev->thermal_instances);
-	cdev->np = np;
-	cdev->ops = ops;
 	cdev->updated = false;
-	cdev->device.class = &thermal_class;
+	cdev->device.class = thermal_class;
+	cdev->device.release = thermal_cdev_release;
+	device_initialize(&cdev->device);
 	cdev->devdata = devdata;
 
+	ret = dev_set_name(&cdev->device, "cooling_device%d", cdev->id);
+	if (ret)
+		goto out_put_device;
+
 	ret = cdev->ops->get_max_state(cdev, &cdev->max_state);
 	if (ret)
-		goto out_cdev_type;
+		goto out_put_device;
 
 	/*
 	 * The cooling device's current state is only needed for debug
@@ -1113,33 +1037,62 @@ __thermal_cooling_device_register(struct device_node *np,
 
 	thermal_cooling_device_setup_sysfs(cdev);
 
-	ret = dev_set_name(&cdev->device, "cooling_device%d", cdev->id);
+	ret = device_add(&cdev->device);
 	if (ret)
-		goto out_cooling_dev;
-
-	ret = device_register(&cdev->device);
-	if (ret) {
-		/* thermal_release() handles rest of the cleanup */
-		put_device(&cdev->device);
-		return ERR_PTR(ret);
-	}
+		goto out_put_device;
 
 	if (current_state <= cdev->max_state)
 		thermal_debug_cdev_add(cdev, current_state);
 
 	thermal_cooling_device_init_complete(cdev);
 
-	return cdev;
+	return 0;
 
-out_cooling_dev:
-	thermal_cooling_device_destroy_sysfs(cdev);
-out_cdev_type:
-	kfree_const(cdev->type);
-out_ida_remove:
-	ida_free(&thermal_cdev_ida, cdev->id);
-out_kfree_cdev:
-	kfree(cdev);
-	return ERR_PTR(ret);
+out_put_device:
+	/*
+	 * The device core will release the memory via
+	 * thermal_release() after put_device() is called in the error
+	 * path
+	 */
+	put_device(&cdev->device);
+	return ret;
+}
+
+/**
+ * __thermal_cooling_device_register() - register a new thermal cooling device
+ * @np:		a pointer to a device tree node.
+ * @type:	the thermal cooling device type.
+ * @devdata:	device private data.
+ * @ops:	standard thermal cooling devices callbacks.
+ *
+ * This interface function adds a new thermal cooling device (fan/processor/...)
+ * to /sys/class/thermal/ folder as cooling_device[0-*]. It tries to bind itself
+ * to all the thermal zone devices registered at the same time.
+ * It also gives the opportunity to link the cooling device to a device tree
+ * node, so that it can be bound to a thermal zone created out of device tree.
+ *
+ * Return: a pointer to the created struct thermal_cooling_device or an
+ * ERR_PTR. Caller must check return value with IS_ERR*() helpers.
+ */
+static struct thermal_cooling_device *
+__thermal_cooling_device_register(struct device_node *np,
+				  const char *type, void *devdata,
+				  const struct thermal_cooling_device_ops *ops)
+{
+	struct thermal_cooling_device *cdev;
+	int ret;
+
+	cdev = thermal_cooling_device_alloc(type, ops);
+	if (IS_ERR(cdev))
+		return cdev;
+
+	cdev->np = np;
+
+	ret = thermal_cooling_device_add(cdev, devdata);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return cdev;
 }
 
 /**
@@ -1470,6 +1423,17 @@ static void thermal_zone_init_complete(struct thermal_zone_device *tz)
 	__thermal_zone_device_update(tz, THERMAL_EVENT_UNSPECIFIED);
 }
 
+static void thermal_zone_device_release(struct device *dev)
+{
+	struct thermal_zone_device *tz = to_thermal_zone(dev);
+
+	thermal_zone_destroy_device_groups(tz);
+	thermal_set_governor(tz, NULL);
+	ida_destroy(&tz->ida);
+	mutex_destroy(&tz->lock);
+	complete(&tz->removal);
+}
+
 /**
  * thermal_zone_device_register_with_trips() - register a new thermal zone device
  * @type:	the thermal zone device type
@@ -1540,7 +1504,7 @@ thermal_zone_device_register_with_trips(const char *type,
 	if (polling_delay && passive_delay > polling_delay)
 		return ERR_PTR(-EINVAL);
 
-	if (thermal_class_unavailable)
+	if (!thermal_class)
 		return ERR_PTR(-ENODEV);
 
 	tz = kzalloc_flex(*tz, trips, num_trips);
@@ -1576,7 +1540,8 @@ thermal_zone_device_register_with_trips(const char *type,
 	if (!tz->ops.critical)
 		tz->ops.critical = thermal_zone_device_critical;
 
-	tz->device.class = &thermal_class;
+	tz->device.class = thermal_class;
+	tz->device.release = thermal_zone_device_release;
 	tz->devdata = devdata;
 	tz->num_trips = num_trips;
 	for_each_trip_desc(tz, td) {
@@ -1837,7 +1802,7 @@ static void __thermal_pm_prepare(void)
 
 void thermal_pm_prepare(void)
 {
-	if (thermal_class_unavailable)
+	if (!thermal_class)
 		return;
 
 	__thermal_pm_prepare();
@@ -1868,7 +1833,7 @@ void thermal_pm_complete(void)
 {
 	struct thermal_zone_device *tz;
 
-	if (thermal_class_unavailable)
+	if (!thermal_class)
 		return;
 
 	guard(mutex)(&thermal_list_lock);
@@ -1881,6 +1846,7 @@ void thermal_pm_complete(void)
 
 static int __init thermal_init(void)
 {
+	struct class *tc;
 	int result;
 
 	thermal_debug_init();
@@ -1896,20 +1862,20 @@ static int __init thermal_init(void)
 	}
 
 	result = thermal_register_governors();
-	if (result)
-		goto destroy_workqueue;
-
-	result = class_register(&thermal_class);
 	if (result)
 		goto unregister_governors;
 
-	thermal_class_unavailable = false;
+	tc = class_create("thermal");
+	if (IS_ERR(tc)) {
+		result = PTR_ERR(tc);
+		goto unregister_governors;
+	}
 
+	thermal_class = tc;
 	return 0;
 
 unregister_governors:
 	thermal_unregister_governors();
-destroy_workqueue:
 	destroy_workqueue(thermal_wq);
 unregister_netlink:
 	thermal_netlink_exit();
diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h
index d3acff602f9ce..0acb7d9587ca5 100644
--- a/drivers/thermal/thermal_core.h
+++ b/drivers/thermal/thermal_core.h
@@ -258,8 +258,6 @@ struct thermal_instance {
 #define to_cooling_device(_dev)	\
 	container_of(_dev, struct thermal_cooling_device, device)
 
-int thermal_register_governor(struct thermal_governor *);
-void thermal_unregister_governor(struct thermal_governor *);
 int thermal_zone_device_set_policy(struct thermal_zone_device *, char *);
 int thermal_build_list_of_policies(char *buf);
 void __thermal_zone_device_update(struct thermal_zone_device *tz,
diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c
index b624892bc6d66..386dfb9f559e3 100644
--- a/drivers/thermal/thermal_hwmon.c
+++ b/drivers/thermal/thermal_hwmon.c
@@ -19,27 +19,19 @@
 #include "thermal_hwmon.h"
 #include "thermal_core.h"
 
+/*
+ * Needs to be large enough to hold a thermal zone type string followed by an
+ * underline character and a 32-bit integer in decimal representation.
+ */
+#define THERMAL_HWMON_NAME_LENGTH (THERMAL_NAME_LENGTH + 11)
+
 /* hwmon sys I/F */
 /* thermal zone devices with the same type share one hwmon device */
 struct thermal_hwmon_device {
-	char type[THERMAL_NAME_LENGTH];
+	char name[THERMAL_HWMON_NAME_LENGTH];
 	struct device *device;
-	int count;
-	struct list_head tz_list;
 	struct list_head node;
-};
-
-struct thermal_hwmon_attr {
-	struct device_attribute attr;
-	char name[16];
-};
-
-/* one temperature input for each thermal zone */
-struct thermal_hwmon_temp {
-	struct list_head hwmon_node;
 	struct thermal_zone_device *tz;
-	struct thermal_hwmon_attr temp_input;	/* hwmon sys attr */
-	struct thermal_hwmon_attr temp_crit;	/* hwmon sys attr */
 };
 
 static LIST_HEAD(thermal_hwmon_list);
@@ -47,19 +39,14 @@ static LIST_HEAD(thermal_hwmon_list);
 static DEFINE_MUTEX(thermal_hwmon_list_lock);
 
 static ssize_t
-temp_input_show(struct device *dev, struct device_attribute *attr, char *buf)
+temp1_input_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
+	struct thermal_hwmon_device *hwmon = dev_get_drvdata(dev);
+	struct thermal_zone_device *tz = hwmon->tz;
 	int temperature;
 	int ret;
-	struct thermal_hwmon_attr *hwmon_attr
-			= container_of(attr, struct thermal_hwmon_attr, attr);
-	struct thermal_hwmon_temp *temp
-			= container_of(hwmon_attr, struct thermal_hwmon_temp,
-				       temp_input);
-	struct thermal_zone_device *tz = temp->tz;
 
 	ret = thermal_zone_get_temp(tz, &temperature);
-
 	if (ret)
 		return ret;
 
@@ -67,14 +54,10 @@ temp_input_show(struct device *dev, struct device_attribute *attr, char *buf)
 }
 
 static ssize_t
-temp_crit_show(struct device *dev, struct device_attribute *attr, char *buf)
+temp1_crit_show(struct device *dev, struct device_attribute *attr, char *buf)
 {
-	struct thermal_hwmon_attr *hwmon_attr
-			= container_of(attr, struct thermal_hwmon_attr, attr);
-	struct thermal_hwmon_temp *temp
-			= container_of(hwmon_attr, struct thermal_hwmon_temp,
-				       temp_crit);
-	struct thermal_zone_device *tz = temp->tz;
+	struct thermal_hwmon_device *hwmon = dev_get_drvdata(dev);
+	struct thermal_zone_device *tz = hwmon->tz;
 	int temperature;
 	int ret;
 
@@ -87,166 +70,97 @@ temp_crit_show(struct device *dev, struct device_attribute *attr, char *buf)
 	return sysfs_emit(buf, "%d\n", temperature);
 }
 
+static DEVICE_ATTR_RO(temp1_input);
+static DEVICE_ATTR_RO(temp1_crit);
 
-static struct thermal_hwmon_device *
-thermal_hwmon_lookup_by_type(const struct thermal_zone_device *tz)
+static struct attribute *thermal_hwmon_attrs[] = {
+	&dev_attr_temp1_input.attr,
+	&dev_attr_temp1_crit.attr,
+	NULL,
+};
+
+static umode_t thermal_hwmon_attr_is_visible(struct kobject *kobj,
+					     struct attribute *a, int n)
 {
-	struct thermal_hwmon_device *hwmon;
-	char type[THERMAL_NAME_LENGTH];
+	if (a == &dev_attr_temp1_input.attr)
+		return a->mode;
 
-	mutex_lock(&thermal_hwmon_list_lock);
-	list_for_each_entry(hwmon, &thermal_hwmon_list, node) {
-		strscpy(type, tz->type);
-		strreplace(type, '-', '_');
-		if (!strcmp(hwmon->type, type)) {
-			mutex_unlock(&thermal_hwmon_list_lock);
-			return hwmon;
-		}
+	if (a == &dev_attr_temp1_crit.attr) {
+		struct thermal_hwmon_device *hwmon = dev_get_drvdata(kobj_to_dev(kobj));
+		struct thermal_zone_device *tz = hwmon->tz;
+		int dummy;
+
+		if (tz->ops.get_crit_temp && !tz->ops.get_crit_temp(tz, &dummy))
+			return a->mode;
 	}
-	mutex_unlock(&thermal_hwmon_list_lock);
 
-	return NULL;
+	return 0;
 }
 
-/* Find the temperature input matching a given thermal zone */
-static struct thermal_hwmon_temp *
-thermal_hwmon_lookup_temp(const struct thermal_hwmon_device *hwmon,
-			  const struct thermal_zone_device *tz)
-{
-	struct thermal_hwmon_temp *temp;
-
-	mutex_lock(&thermal_hwmon_list_lock);
-	list_for_each_entry(temp, &hwmon->tz_list, hwmon_node)
-		if (temp->tz == tz) {
-			mutex_unlock(&thermal_hwmon_list_lock);
-			return temp;
-		}
-	mutex_unlock(&thermal_hwmon_list_lock);
-
-	return NULL;
-}
+static const struct attribute_group thermal_hwmon_group = {
+	.attrs	= thermal_hwmon_attrs,
+	.is_visible = thermal_hwmon_attr_is_visible,
+};
 
-static bool thermal_zone_crit_temp_valid(struct thermal_zone_device *tz)
-{
-	int temp;
-	return tz->ops.get_crit_temp && !tz->ops.get_crit_temp(tz, &temp);
-}
+__ATTRIBUTE_GROUPS(thermal_hwmon);
 
 int thermal_add_hwmon_sysfs(struct thermal_zone_device *tz)
 {
 	struct thermal_hwmon_device *hwmon;
-	struct thermal_hwmon_temp *temp;
-	int new_hwmon_device = 1;
-	int result;
-
-	hwmon = thermal_hwmon_lookup_by_type(tz);
-	if (hwmon) {
-		new_hwmon_device = 0;
-		goto register_sys_interface;
-	}
 
 	hwmon = kzalloc_obj(*hwmon);
 	if (!hwmon)
 		return -ENOMEM;
 
-	INIT_LIST_HEAD(&hwmon->tz_list);
-	strscpy(hwmon->type, tz->type, THERMAL_NAME_LENGTH);
-	strreplace(hwmon->type, '-', '_');
+	hwmon->tz = tz;
+	/*
+	 * Append the thermal zone ID preceded by an underline character to the
+	 * type to disambiguate the sensors command output.
+	 */
+	scnprintf(hwmon->name, THERMAL_HWMON_NAME_LENGTH, "%s_%d", tz->type, tz->id);
+	strreplace(hwmon->name, '-', '_');
 	hwmon->device = hwmon_device_register_for_thermal(&tz->device,
-							  hwmon->type, hwmon);
+							  hwmon->name, hwmon,
+							  thermal_hwmon_groups);
 	if (IS_ERR(hwmon->device)) {
-		result = PTR_ERR(hwmon->device);
-		goto free_mem;
-	}
-
- register_sys_interface:
-	temp = kzalloc_obj(*temp);
-	if (!temp) {
-		result = -ENOMEM;
-		goto unregister_name;
-	}
+		int result = PTR_ERR(hwmon->device);
 
-	temp->tz = tz;
-	hwmon->count++;
-
-	snprintf(temp->temp_input.name, sizeof(temp->temp_input.name),
-		 "temp%d_input", hwmon->count);
-	temp->temp_input.attr.attr.name = temp->temp_input.name;
-	temp->temp_input.attr.attr.mode = 0444;
-	temp->temp_input.attr.show = temp_input_show;
-	sysfs_attr_init(&temp->temp_input.attr.attr);
-	result = device_create_file(hwmon->device, &temp->temp_input.attr);
-	if (result)
-		goto free_temp_mem;
-
-	if (thermal_zone_crit_temp_valid(tz)) {
-		snprintf(temp->temp_crit.name,
-				sizeof(temp->temp_crit.name),
-				"temp%d_crit", hwmon->count);
-		temp->temp_crit.attr.attr.name = temp->temp_crit.name;
-		temp->temp_crit.attr.attr.mode = 0444;
-		temp->temp_crit.attr.show = temp_crit_show;
-		sysfs_attr_init(&temp->temp_crit.attr.attr);
-		result = device_create_file(hwmon->device,
-					    &temp->temp_crit.attr);
-		if (result)
-			goto unregister_input;
+		kfree(hwmon);
+		return result;
 	}
 
+	/* The list is needed for hwmon lookup during removal. */
 	mutex_lock(&thermal_hwmon_list_lock);
-	if (new_hwmon_device)
-		list_add_tail(&hwmon->node, &thermal_hwmon_list);
-	list_add_tail(&temp->hwmon_node, &hwmon->tz_list);
+	list_add_tail(&hwmon->node, &thermal_hwmon_list);
 	mutex_unlock(&thermal_hwmon_list_lock);
 
 	return 0;
-
- unregister_input:
-	device_remove_file(hwmon->device, &temp->temp_input.attr);
- free_temp_mem:
-	kfree(temp);
- unregister_name:
-	if (new_hwmon_device)
-		hwmon_device_unregister(hwmon->device);
- free_mem:
-	kfree(hwmon);
-
-	return result;
 }
 EXPORT_SYMBOL_GPL(thermal_add_hwmon_sysfs);
 
-void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz)
+static struct thermal_hwmon_device *
+thermal_hwmon_lookup(const struct thermal_zone_device *tz)
 {
 	struct thermal_hwmon_device *hwmon;
-	struct thermal_hwmon_temp *temp;
 
-	hwmon = thermal_hwmon_lookup_by_type(tz);
-	if (unlikely(!hwmon)) {
-		/* Should never happen... */
-		dev_dbg(&tz->device, "hwmon device lookup failed!\n");
-		return;
+	list_for_each_entry(hwmon, &thermal_hwmon_list, node) {
+		if (hwmon->tz == tz)
+			return hwmon;
 	}
+	return NULL;
+}
 
-	temp = thermal_hwmon_lookup_temp(hwmon, tz);
-	if (unlikely(!temp)) {
-		/* Should never happen... */
-		dev_dbg(&tz->device, "temperature input lookup failed!\n");
-		return;
-	}
+void thermal_remove_hwmon_sysfs(struct thermal_zone_device *tz)
+{
+	struct thermal_hwmon_device *hwmon;
 
-	device_remove_file(hwmon->device, &temp->temp_input.attr);
-	if (thermal_zone_crit_temp_valid(tz))
-		device_remove_file(hwmon->device, &temp->temp_crit.attr);
+	scoped_guard(mutex, &thermal_hwmon_list_lock) {
+		hwmon = thermal_hwmon_lookup(tz);
+		if (!hwmon)
+			return;
 
-	mutex_lock(&thermal_hwmon_list_lock);
-	list_del(&temp->hwmon_node);
-	kfree(temp);
-	if (!list_empty(&hwmon->tz_list)) {
-		mutex_unlock(&thermal_hwmon_list_lock);
-		return;
+		list_del(&hwmon->node);
 	}
-	list_del(&hwmon->node);
-	mutex_unlock(&thermal_hwmon_list_lock);
 
 	hwmon_device_unregister(hwmon->device);
 	kfree(hwmon);
diff --git a/drivers/thunderbolt/Kconfig b/drivers/thunderbolt/Kconfig
index db3b0bef48f4c..294b3227a5458 100644
--- a/drivers/thunderbolt/Kconfig
+++ b/drivers/thunderbolt/Kconfig
@@ -18,6 +18,10 @@ menuconfig USB4
 
 if USB4
 
+config USB4_CONFIGFS
+	def_tristate USB4
+	depends on CONFIGFS_FS && !(USB4=y && CONFIGFS_FS=m)
+
 config USB4_DEBUGFS_WRITE
 	bool "Enable write by debugfs to configuration spaces (DANGEROUS)"
 	help
@@ -60,4 +64,15 @@ config USB4_DMA_TEST
 	  To compile this driver a module, choose M here. The module will be
 	  called thunderbolt_dma_test.
 
+config USB4_STREAM
+	tristate "Stream data over Thunderbolt/USB4 cable"
+	depends on USB4_CONFIGFS
+	help
+	  This adds support for USB4STREAM protocol that allows two
+	  hosts to stream data directly over Thunderbolt/USB4 cable
+	  through /dev/tbstreamX devices.
+
+	  To compile this driver a module, choose M here. The module will be
+	  called thunderbolt_stream.
+
 endif # USB4
diff --git a/drivers/thunderbolt/Makefile b/drivers/thunderbolt/Makefile
index b44b32dcb8322..c792b8084ffa3 100644
--- a/drivers/thunderbolt/Makefile
+++ b/drivers/thunderbolt/Makefile
@@ -7,8 +7,12 @@ thunderbolt-objs += usb4_port.o nvm.o retimer.o quirks.o clx.o
 
 thunderbolt-${CONFIG_ACPI} += acpi.o
 thunderbolt-$(CONFIG_DEBUG_FS) += debugfs.o
+thunderbolt-$(CONFIG_USB4_CONFIGFS) += configfs.o
 thunderbolt-${CONFIG_USB4_KUNIT_TEST} += test.o
 CFLAGS_test.o += $(DISABLE_STRUCTLEAK_PLUGIN)
 
 thunderbolt_dma_test-${CONFIG_USB4_DMA_TEST} += dma_test.o
 obj-$(CONFIG_USB4_DMA_TEST) += thunderbolt_dma_test.o
+
+thunderbolt_stream-${CONFIG_USB4_STREAM} += stream.o
+obj-$(CONFIG_USB4_STREAM) += thunderbolt_stream.o
diff --git a/drivers/thunderbolt/configfs.c b/drivers/thunderbolt/configfs.c
new file mode 100644
index 0000000000000..dc6bc363dfe80
--- /dev/null
+++ b/drivers/thunderbolt/configfs.c
@@ -0,0 +1,61 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ConfigFS support
+ *
+ * Copyright (C) 2026, Intel Corporation
+ * Author: Mika Westerberg <mika.westerberg@linux.intel.com>
+ */
+
+#include <linux/configfs.h>
+#include <linux/export.h>
+
+#include "tb.h"
+
+static const struct config_item_type tb_root_group_type = {
+	.ct_owner = THIS_MODULE,
+};
+
+static struct configfs_subsystem tb_configfs = {
+	.su_group = {
+		.cg_item = {
+			.ci_namebuf = "thunderbolt",
+			.ci_type = &tb_root_group_type,
+		},
+	},
+};
+
+/**
+ * tb_configfs_register_group() - Register Thunderbolt ConfigFS group
+ * @group: Group to register.
+ *
+ * Registers the new @group under Thunderbolt subsystem ConfigFS.
+ *
+ * Return: 0% in case of success, negative errno otherwise.
+ */
+int tb_configfs_register_group(struct config_group *group)
+{
+	return configfs_register_group(&tb_configfs.su_group, group);
+}
+EXPORT_SYMBOL_GPL(tb_configfs_register_group);
+
+/**
+ * tb_configfs_unregister_group() - Unregister previously registered group
+ * @group: Group to unregister.
+ */
+void tb_configfs_unregister_group(struct config_group *group)
+{
+	configfs_unregister_group(group);
+}
+EXPORT_SYMBOL_GPL(tb_configfs_unregister_group);
+
+int tb_configfs_init(void)
+{
+	config_group_init(&tb_configfs.su_group);
+	mutex_init(&tb_configfs.su_mutex);
+	return configfs_register_subsystem(&tb_configfs);
+}
+
+void tb_configfs_exit(void)
+{
+	configfs_unregister_subsystem(&tb_configfs);
+}
diff --git a/drivers/thunderbolt/debugfs.c b/drivers/thunderbolt/debugfs.c
index 042f6a0d0f7f0..569163a26afa2 100644
--- a/drivers/thunderbolt/debugfs.c
+++ b/drivers/thunderbolt/debugfs.c
@@ -1780,6 +1780,8 @@ static void margining_port_remove(struct tb_port *port)
 
 	if (!port->usb4)
 		return;
+	if (!port->usb4->margining)
+		return;
 
 	snprintf(dir_name, sizeof(dir_name), "port%d", port->port);
 	parent = debugfs_lookup(dir_name, port->sw->debugfs_dir);
@@ -2361,8 +2363,10 @@ static int sb_regs_show(struct tb_port *port, const struct sb_reg *sb_regs,
 		memset(data, 0, sizeof(data));
 		ret = usb4_port_sb_read(port, target, index, regs->reg, data,
 					regs->size);
-		if (ret)
-			return ret;
+		if (ret) {
+			seq_printf(s, "0x%02x <not accessible>\n", regs->reg);
+			continue;
+		}
 
 		seq_printf(s, "0x%02x", regs->reg);
 		for (j = 0; j < regs->size; j++)
diff --git a/drivers/thunderbolt/dma_test.c b/drivers/thunderbolt/dma_test.c
index b4aa79d482a0b..7877319b1b03a 100644
--- a/drivers/thunderbolt/dma_test.c
+++ b/drivers/thunderbolt/dma_test.c
@@ -87,7 +87,6 @@ static const char * const dma_test_result_names[] = {
  * @error_code: Error code of the last run
  * @complete: Used to wait for the Rx to complete
  * @lock: Lock serializing access to this structure
- * @debugfs_dir: dentry of this dma_test
  */
 struct dma_test {
 	const struct tb_service *svc;
@@ -108,7 +107,6 @@ struct dma_test {
 	enum dma_test_test_error error_code;
 	struct completion complete;
 	struct mutex lock;
-	struct dentry *debugfs_dir;
 };
 
 /* DMA test property directory UUID: 3188cd10-6523-4a5a-a682-fdca07a248d8 */
@@ -157,6 +155,8 @@ static int dma_test_start_rings(struct dma_test *dt)
 		dt->tx_ring = ring;
 		e2e_tx_hop = ring->hop;
 
+		tb_ring_throttling(ring, 128000);
+
 		ret = tb_xdomain_alloc_out_hopid(xd, -1);
 		if (ret < 0) {
 			dma_test_free_rings(dt);
@@ -164,6 +164,7 @@ static int dma_test_start_rings(struct dma_test *dt)
 		}
 
 		dt->tx_hopid = ret;
+
 	}
 
 	if (dt->packets_to_receive) {
@@ -182,6 +183,8 @@ static int dma_test_start_rings(struct dma_test *dt)
 
 		dt->rx_ring = ring;
 
+		tb_ring_throttling(ring, 128000);
+
 		ret = tb_xdomain_alloc_in_hopid(xd, -1);
 		if (ret < 0) {
 			dma_test_free_rings(dt);
@@ -619,18 +622,18 @@ DEFINE_SHOW_ATTRIBUTE(status);
 
 static void dma_test_debugfs_init(struct tb_service *svc)
 {
-	struct dma_test *dt = tb_service_get_drvdata(svc);
+	struct dentry *debugfs_dir;
 
-	dt->debugfs_dir = debugfs_create_dir("dma_test", svc->debugfs_dir);
+	debugfs_dir = debugfs_create_dir("dma_test", svc->debugfs_dir);
 
-	debugfs_create_file("lanes", 0600, dt->debugfs_dir, svc, &lanes_fops);
-	debugfs_create_file("speed", 0600, dt->debugfs_dir, svc, &speed_fops);
-	debugfs_create_file("packets_to_receive", 0600, dt->debugfs_dir, svc,
+	debugfs_create_file("lanes", 0600, debugfs_dir, svc, &lanes_fops);
+	debugfs_create_file("speed", 0600, debugfs_dir, svc, &speed_fops);
+	debugfs_create_file("packets_to_receive", 0600, debugfs_dir, svc,
 			    &packets_to_receive_fops);
-	debugfs_create_file("packets_to_send", 0600, dt->debugfs_dir, svc,
+	debugfs_create_file("packets_to_send", 0600, debugfs_dir, svc,
 			    &packets_to_send_fops);
-	debugfs_create_file("status", 0400, dt->debugfs_dir, svc, &status_fops);
-	debugfs_create_file("test", 0200, dt->debugfs_dir, svc, &test_fops);
+	debugfs_create_file("status", 0400, debugfs_dir, svc, &status_fops);
+	debugfs_create_file("test", 0200, debugfs_dir, svc, &test_fops);
 }
 
 static int dma_test_probe(struct tb_service *svc, const struct tb_service_id *id)
@@ -658,7 +661,7 @@ static void dma_test_remove(struct tb_service *svc)
 	struct dma_test *dt = tb_service_get_drvdata(svc);
 
 	mutex_lock(&dt->lock);
-	debugfs_remove_recursive(dt->debugfs_dir);
+	debugfs_lookup_and_remove("dma_test", svc->debugfs_dir);
 	mutex_unlock(&dt->lock);
 }
 
diff --git a/drivers/thunderbolt/domain.c b/drivers/thunderbolt/domain.c
index 317780b999929..b381f184340ec 100644
--- a/drivers/thunderbolt/domain.c
+++ b/drivers/thunderbolt/domain.c
@@ -319,12 +319,15 @@ const struct bus_type tb_bus_type = {
 static void tb_domain_release(struct device *dev)
 {
 	struct tb *tb = container_of(dev, struct tb, dev);
+	struct tb_nhi *nhi = tb->nhi;
 
 	tb_ctl_free(tb->ctl);
 	destroy_workqueue(tb->wq);
 	ida_free(&tb_domain_ida, tb->index);
 	mutex_destroy(&tb->lock);
 	kfree(tb);
+
+	complete(&nhi->domain_released);
 }
 
 const struct device_type tb_domain_type = {
@@ -850,10 +853,41 @@ int tb_domain_disconnect_all_paths(struct tb *tb)
 	return bus_for_each_dev(&tb_bus_type, NULL, tb, disconnect_xdomain);
 }
 
+struct unregister_context {
+	const struct tb *tb;
+	int n;
+};
+
+static int unregister_unplugged_xdomain(struct device *dev, void *data)
+{
+	struct unregister_context *ctx = data;
+	struct tb_xdomain *xd;
+
+	xd = tb_to_xdomain(dev);
+	if (xd && xd->tb == ctx->tb && xd->is_unplugged) {
+		tb_xdomain_unregister(xd);
+		ctx->n++;
+	}
+	return 0;
+}
+
+int tb_domain_unregister_unplugged_xdomains(struct tb *tb)
+{
+	struct unregister_context ctx;
+
+	ctx.tb = tb_domain_get(tb);
+	ctx.n = 0;
+	bus_for_each_dev(&tb_bus_type, NULL, &ctx, unregister_unplugged_xdomain);
+	tb_domain_put(tb);
+
+	return ctx.n;
+}
+
 int tb_domain_init(void)
 {
 	int ret;
 
+	tb_configfs_init();
 	tb_debugfs_init();
 	tb_acpi_init();
 
@@ -883,4 +917,5 @@ void tb_domain_exit(void)
 	tb_xdomain_exit();
 	tb_acpi_exit();
 	tb_debugfs_exit();
+	tb_configfs_exit();
 }
diff --git a/drivers/thunderbolt/icm.c b/drivers/thunderbolt/icm.c
index 9d95bf3ab44c1..c492995166f7a 100644
--- a/drivers/thunderbolt/icm.c
+++ b/drivers/thunderbolt/icm.c
@@ -587,6 +587,11 @@ static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 	struct icm_fr_pkg_approve_xdomain request;
 	int ret;
 
+	if (atomic_read(&xd->ntunnels) >= 1) {
+		tb_warn(tb, "only one tunnel is supported by the firmware\n");
+		return -EOPNOTSUPP;
+	}
+
 	memset(&request, 0, sizeof(request));
 	request.hdr.code = ICM_APPROVE_XDOMAIN;
 	request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
@@ -738,6 +743,7 @@ static void remove_xdomain(struct tb_xdomain *xd)
 
 	sw = tb_to_switch(xd->dev.parent);
 	tb_port_at(xd->route, sw)->xdomain = NULL;
+	xd->is_unplugged = true;
 	tb_xdomain_remove(xd);
 }
 
@@ -1157,6 +1163,11 @@ static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 	struct icm_tr_pkg_approve_xdomain request;
 	int ret;
 
+	if (atomic_read(&xd->ntunnels) >= 1) {
+		tb_warn(tb, "only one tunnel is supported by the firmware\n");
+		return -EOPNOTSUPP;
+	}
+
 	memset(&request, 0, sizeof(request));
 	request.hdr.code = ICM_APPROVE_XDOMAIN;
 	request.route_hi = upper_32_bits(xd->route);
@@ -1762,6 +1773,8 @@ static void icm_handle_notification(struct work_struct *work)
 
 	kfree(n->pkg);
 	kfree(n);
+
+	tb_domain_unregister_unplugged_xdomains(tb);
 }
 
 static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
@@ -2112,6 +2125,8 @@ static void icm_rescan_work(struct work_struct *work)
 	if (tb->root_switch)
 		icm_free_unplugged_children(tb->root_switch);
 	mutex_unlock(&tb->lock);
+
+	tb_domain_unregister_unplugged_xdomains(tb);
 }
 
 static void icm_complete(struct tb *tb)
diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c
index 2bb2e79ca3cb3..a0a789bfb680f 100644
--- a/drivers/thunderbolt/nhi.c
+++ b/drivers/thunderbolt/nhi.c
@@ -93,7 +93,7 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 	u32 old, new;
 
 	if (ring->irq > 0) {
-		u32 step, shift, ivr, misc;
+		u32 step, shift, ivr, misc, itr;
 		void __iomem *ivr_base;
 		int auto_clear_bit;
 		int index;
@@ -131,6 +131,12 @@ static void ring_interrupt_active(struct tb_ring *ring, bool active)
 		if (active)
 			ivr |= ring->vector << shift;
 		iowrite32(ivr, ivr_base + step);
+
+		/* Throttling is specified in 256ns increments */
+		itr = DIV_ROUND_UP(ring->interval_nsec, 256);
+		itr &= REG_INT_THROTTLING_RATE_INTERVAL_MASK;
+		iowrite32(itr, ring->nhi->iobase + REG_INT_THROTTLING_RATE +
+			  ring->vector * 4);
 	}
 
 	old = ioread32(ring->nhi->iobase + reg);
@@ -319,6 +325,8 @@ static void ring_work(struct work_struct *work)
 		if (frame->callback)
 			frame->callback(ring, frame, canceled);
 	}
+
+	wake_up(&ring->wait);
 }
 
 int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame)
@@ -595,6 +603,7 @@ static struct tb_ring *tb_ring_alloc(struct tb_nhi *nhi, u32 hop, int size,
 	INIT_LIST_HEAD(&ring->queue);
 	INIT_LIST_HEAD(&ring->in_flight);
 	INIT_WORK(&ring->work, ring_work);
+	init_waitqueue_head(&ring->wait);
 
 	ring->nhi = nhi;
 	ring->hop = hop;
@@ -754,6 +763,31 @@ void tb_ring_start(struct tb_ring *ring)
 }
 EXPORT_SYMBOL_GPL(tb_ring_start);
 
+static bool tb_ring_empty(struct tb_ring *ring)
+{
+	guard(spinlock_irqsave)(&ring->lock);
+	return list_empty(&ring->in_flight);
+}
+
+/**
+ * tb_ring_flush() - Waits for a ring to be empty
+ * @ring: Ring to wait
+ * @timeout_msec: Timeout in ms how long to wait.
+ *
+ * This can be called before stopping a ring to make sure all the frames
+ * submitted prior have been completed.
+ *
+ * Return: %true if the ring is empty now, %false otherwise.
+ */
+bool tb_ring_flush(struct tb_ring *ring, unsigned int timeout_msec)
+{
+	if (!wait_event_timeout(ring->wait, tb_ring_empty(ring),
+				msecs_to_jiffies(timeout_msec)))
+		return false;
+	return tb_ring_empty(ring);
+}
+EXPORT_SYMBOL_GPL(tb_ring_flush);
+
 /**
  * tb_ring_stop() - shutdown a ring
  * @ring: Ring to stop
@@ -854,6 +888,26 @@ void tb_ring_free(struct tb_ring *ring)
 }
 EXPORT_SYMBOL_GPL(tb_ring_free);
 
+/**
+ * tb_ring_throttling() - Configure throttling for ring interrupt
+ * @ring: Ring to configure
+ * @interval_nsec: Interval counter for moderation (in ns), %0 disables
+ *
+ * Enables or disables ring interrupt throttling. The ring must be
+ * stopped for this to be called. Granularity is 256 ns.
+ *
+ * Return: %0 on success, negative errno otherwise.
+ */
+int tb_ring_throttling(struct tb_ring *ring, unsigned int interval_nsec)
+{
+	guard(spinlock_irqsave)(&ring->lock);
+	if (WARN_ON_ONCE(ring->running))
+		return -EBUSY;
+	ring->interval_nsec = interval_nsec;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(tb_ring_throttling);
+
 /**
  * nhi_mailbox_cmd() - Send a command through NHI mailbox
  * @nhi: Pointer to the NHI structure
@@ -1035,22 +1089,6 @@ static int nhi_poweroff_noirq(struct device *dev)
 	return __nhi_suspend_noirq(dev, wakeup);
 }
 
-static void nhi_enable_int_throttling(struct tb_nhi *nhi)
-{
-	/* Throttling is specified in 256ns increments */
-	u32 throttle = DIV_ROUND_UP(128 * NSEC_PER_USEC, 256);
-	unsigned int i;
-
-	/*
-	 * Configure interrupt throttling for all vectors even if we
-	 * only use few.
-	 */
-	for (i = 0; i < MSIX_MAX_VECS; i++) {
-		u32 reg = REG_INT_THROTTLING_RATE + i * 4;
-		iowrite32(throttle, nhi->iobase + reg);
-	}
-}
-
 static int nhi_resume_noirq(struct device *dev)
 {
 	struct pci_dev *pdev = to_pci_dev(dev);
@@ -1065,13 +1103,10 @@ static int nhi_resume_noirq(struct device *dev)
 	 */
 	if (!pci_device_is_present(pdev)) {
 		nhi->going_away = true;
-	} else {
-		if (nhi->ops && nhi->ops->resume_noirq) {
-			ret = nhi->ops->resume_noirq(nhi);
-			if (ret)
-				return ret;
-		}
-		nhi_enable_int_throttling(tb->nhi);
+	} else if (nhi->ops && nhi->ops->resume_noirq) {
+		ret = nhi->ops->resume_noirq(nhi);
+		if (ret)
+			return ret;
 	}
 
 	return tb_domain_resume_noirq(tb);
@@ -1133,7 +1168,6 @@ static int nhi_runtime_resume(struct device *dev)
 			return ret;
 	}
 
-	nhi_enable_int_throttling(nhi);
 	return tb_domain_runtime_resume(tb);
 }
 
@@ -1271,8 +1305,6 @@ static int nhi_init_msi(struct tb_nhi *nhi)
 	/* In case someone left them on. */
 	nhi_disable_interrupts(nhi);
 
-	nhi_enable_int_throttling(nhi);
-
 	ida_init(&nhi->msix_ida);
 
 	/*
@@ -1401,6 +1433,8 @@ static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 	dev_dbg(dev, "NHI initialized, starting thunderbolt\n");
 
+	init_completion(&nhi->domain_released);
+
 	res = tb_domain_add(tb, host_reset);
 	if (res) {
 		/*
@@ -1408,6 +1442,7 @@ static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 		 * activated. Do a proper shutdown.
 		 */
 		tb_domain_put(tb);
+		wait_for_completion(&nhi->domain_released);
 		nhi_shutdown(nhi);
 		return res;
 	}
@@ -1433,6 +1468,7 @@ static void nhi_remove(struct pci_dev *pdev)
 	pm_runtime_forbid(&pdev->dev);
 
 	tb_domain_remove(tb);
+	wait_for_completion(&nhi->domain_released);
 	nhi_shutdown(nhi);
 }
 
diff --git a/drivers/thunderbolt/nhi_regs.h b/drivers/thunderbolt/nhi_regs.h
index cf5222bee9713..d6a197fabc74a 100644
--- a/drivers/thunderbolt/nhi_regs.h
+++ b/drivers/thunderbolt/nhi_regs.h
@@ -101,7 +101,8 @@ struct ring_desc {
 
 #define REG_RING_INTERRUPT_MASK_CLEAR_BASE	0x38208
 
-#define REG_INT_THROTTLING_RATE	0x38c00
+#define REG_INT_THROTTLING_RATE			0x38c00
+#define REG_INT_THROTTLING_RATE_INTERVAL_MASK	GENMASK(15, 0)
 
 /* Interrupt Vector Allocation */
 #define REG_INT_VEC_ALLOC_BASE	0x38c40
diff --git a/drivers/thunderbolt/path.c b/drivers/thunderbolt/path.c
index 8713ea0f47c1b..b2c322e76b8ad 100644
--- a/drivers/thunderbolt/path.c
+++ b/drivers/thunderbolt/path.c
@@ -412,7 +412,8 @@ static int __tb_path_deactivate_hop(struct tb_port *port, int hop_index,
 				 * in the USB4 spec so we clear them
 				 * only for pre-USB4 adapters.
 				 */
-				if (!tb_switch_is_usb4(port->sw)) {
+				if (tb_port_is_null(port) ||
+				    !tb_switch_is_usb4(port->sw)) {
 					hop.ingress_fc = 0;
 					hop.ingress_shared_buffer = 0;
 				}
@@ -483,7 +484,7 @@ void tb_path_deactivate(struct tb_path *path)
  * tb_path_activate() - activate a path
  * @path: Path to activate
  *
- * Activate a path starting with the last hop and iterating backwards. The
+ * Activate a path starting with the first hop and ending on the last hop. The
  * caller must fill path->hops before calling tb_path_activate().
  *
  * Return: %0 on success, negative errno otherwise.
@@ -525,22 +526,25 @@ int tb_path_activate(struct tb_path *path)
 	}
 
 	/* Activate hops. */
-	for (i = path->path_length - 1; i >= 0; i--) {
+	for (i = 0; i < path->path_length; i++) {
 		struct tb_regs_hop hop = { 0 };
 
 		/* If it is left active deactivate it first */
 		__tb_path_deactivate_hop(path->hops[i].in_port,
 				path->hops[i].in_hop_index, path->clear_fc);
 
-		/* dword 0 */
+		/* Needed for USB4 routers, read path config space before write */
+		res = tb_port_read(path->hops[i].in_port, &hop, TB_CFG_HOPS,
+				   2 * path->hops[i].in_hop_index, 2);
+		if (res)
+			goto err;
+
 		hop.next_hop = path->hops[i].next_hop_index;
 		hop.out_port = path->hops[i].out_port->port;
-		hop.initial_credits = path->hops[i].initial_credits;
 		hop.pmps = path->hops[i].pm_support;
 		hop.unknown1 = 0;
 		hop.enable = 1;
 
-		/* dword 1 */
 		out_mask = (i == path->path_length - 1) ?
 				TB_PATH_DESTINATION : TB_PATH_INTERNAL;
 		in_mask = (i == 0) ? TB_PATH_SOURCE : TB_PATH_INTERNAL;
@@ -550,12 +554,21 @@ int tb_path_activate(struct tb_path *path)
 		hop.drop_packages = path->drop_packages;
 		hop.counter = path->hops[i].in_counter_index;
 		hop.counter_enable = path->hops[i].in_counter_index != -1;
-		hop.ingress_fc = path->ingress_fc_enable & in_mask;
 		hop.egress_fc = path->egress_fc_enable & out_mask;
-		hop.ingress_shared_buffer = path->ingress_shared_buffer
-					    & in_mask;
-		hop.egress_shared_buffer = path->egress_shared_buffer
-					    & out_mask;
+		hop.egress_shared_buffer = path->egress_shared_buffer & out_mask;
+		/*
+		 * Protocol adapters IFC and ISE bits, and Path Credits
+		 * Allocated are vendor defined in the USB4 spec so we
+		 * program them only for pre-USB4 and lane adapters.
+		 */
+		if (tb_port_is_null(path->hops[i].in_port) ||
+		    !tb_switch_is_usb4(path->hops[i].in_port->sw)) {
+			hop.initial_credits = path->hops[i].initial_credits;
+			hop.ingress_fc = path->ingress_fc_enable & in_mask;
+			hop.ingress_shared_buffer =
+				path->ingress_shared_buffer & in_mask;
+		}
+
 		hop.unknown3 = 0;
 
 		tb_port_dbg(path->hops[i].in_port, "Writing hop %d\n", i);
@@ -563,7 +576,7 @@ int tb_path_activate(struct tb_path *path)
 		res = tb_port_write(path->hops[i].in_port, &hop, TB_CFG_HOPS,
 				    2 * path->hops[i].in_hop_index, 2);
 		if (res) {
-			__tb_path_deactivate_hops(path, i);
+			__tb_path_deactivate_hops(path, 0);
 			__tb_path_deallocate_nfc(path, 0);
 			goto err;
 		}
diff --git a/drivers/thunderbolt/property.c b/drivers/thunderbolt/property.c
index 50cbfc92fe65e..fb19df8fca4e1 100644
--- a/drivers/thunderbolt/property.c
+++ b/drivers/thunderbolt/property.c
@@ -8,6 +8,7 @@
  */
 
 #include <linux/err.h>
+#include <linux/overflow.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/uuid.h>
@@ -34,10 +35,12 @@ struct tb_property_dir_entry {
 };
 
 #define TB_PROPERTY_ROOTDIR_MAGIC	0x55584401
+#define TB_PROPERTY_MAX_DEPTH		8
 
 static struct tb_property_dir *__tb_property_parse_dir(const u32 *block,
 	size_t block_len, unsigned int dir_offset, size_t dir_len,
-	bool is_root);
+	bool is_root, unsigned int depth);
+static struct tb_property *tb_property_copy(const struct tb_property *property);
 
 static inline void parse_dwdata(void *dst, const void *src, size_t dwords)
 {
@@ -52,13 +55,16 @@ static inline void format_dwdata(void *dst, const void *src, size_t dwords)
 static bool tb_property_entry_valid(const struct tb_property_entry *entry,
 				  size_t block_len)
 {
+	u32 end;
+
 	switch (entry->type) {
 	case TB_PROPERTY_TYPE_DIRECTORY:
 	case TB_PROPERTY_TYPE_DATA:
 	case TB_PROPERTY_TYPE_TEXT:
 		if (entry->length > block_len)
 			return false;
-		if (entry->value + entry->length > block_len)
+		if (check_add_overflow(entry->value, entry->length, &end) ||
+		    end > block_len)
 			return false;
 		break;
 
@@ -93,7 +99,8 @@ tb_property_alloc(const char *key, enum tb_property_type type)
 }
 
 static struct tb_property *tb_property_parse(const u32 *block, size_t block_len,
-					const struct tb_property_entry *entry)
+					const struct tb_property_entry *entry,
+					unsigned int depth)
 {
 	char key[TB_PROPERTY_KEY_SIZE + 1];
 	struct tb_property *property;
@@ -114,7 +121,7 @@ static struct tb_property *tb_property_parse(const u32 *block, size_t block_len,
 	switch (property->type) {
 	case TB_PROPERTY_TYPE_DIRECTORY:
 		dir = __tb_property_parse_dir(block, block_len, entry->value,
-					      entry->length, false);
+					      entry->length, false, depth + 1);
 		if (!dir) {
 			kfree(property);
 			return NULL;
@@ -159,21 +166,31 @@ static struct tb_property *tb_property_parse(const u32 *block, size_t block_len,
 }
 
 static struct tb_property_dir *__tb_property_parse_dir(const u32 *block,
-	size_t block_len, unsigned int dir_offset, size_t dir_len, bool is_root)
+	size_t block_len, unsigned int dir_offset, size_t dir_len, bool is_root,
+	unsigned int depth)
 {
 	const struct tb_property_entry *entries;
 	size_t i, content_len, nentries;
 	unsigned int content_offset;
 	struct tb_property_dir *dir;
 
+	if (depth > TB_PROPERTY_MAX_DEPTH)
+		return NULL;
+
 	dir = kzalloc_obj(*dir);
 	if (!dir)
 		return NULL;
 
+	INIT_LIST_HEAD(&dir->properties);
+
 	if (is_root) {
 		content_offset = dir_offset + 2;
 		content_len = dir_len;
 	} else {
+		if (dir_len < 4) {
+			tb_property_free_dir(dir);
+			return NULL;
+		}
 		dir->uuid = kmemdup(&block[dir_offset], sizeof(*dir->uuid),
 				    GFP_KERNEL);
 		if (!dir->uuid) {
@@ -187,12 +204,10 @@ static struct tb_property_dir *__tb_property_parse_dir(const u32 *block,
 	entries = (const struct tb_property_entry *)&block[content_offset];
 	nentries = content_len / (sizeof(*entries) / 4);
 
-	INIT_LIST_HEAD(&dir->properties);
-
 	for (i = 0; i < nentries; i++) {
 		struct tb_property *property;
 
-		property = tb_property_parse(block, block_len, &entries[i]);
+		property = tb_property_parse(block, block_len, &entries[i], depth);
 		if (!property) {
 			tb_property_free_dir(dir);
 			return NULL;
@@ -231,7 +246,7 @@ struct tb_property_dir *tb_property_parse_dir(const u32 *block,
 		return NULL;
 
 	return __tb_property_parse_dir(block, block_len, 0, rootdir->length,
-				       true);
+				       true, 0);
 }
 
 /**
@@ -507,17 +522,9 @@ ssize_t tb_property_format_dir(const struct tb_property_dir *dir, u32 *block,
 	return ret < 0 ? ret : 0;
 }
 
-/**
- * tb_property_copy_dir() - Take a deep copy of directory
- * @dir: Directory to copy
- *
- * The resulting directory needs to be released by calling tb_property_free_dir().
- *
- * Return: Pointer to &struct tb_property_dir, %NULL in case of failure.
- */
-struct tb_property_dir *tb_property_copy_dir(const struct tb_property_dir *dir)
+static struct tb_property_dir *copy_dir(const struct tb_property_dir *dir)
 {
-	struct tb_property *property, *p = NULL;
+	struct tb_property *property, *p;
 	struct tb_property_dir *d;
 
 	if (!dir)
@@ -528,56 +535,131 @@ struct tb_property_dir *tb_property_copy_dir(const struct tb_property_dir *dir)
 		return NULL;
 
 	list_for_each_entry(property, &dir->properties, list) {
-		struct tb_property *p;
-
-		p = tb_property_alloc(property->key, property->type);
+		p = tb_property_copy(property);
 		if (!p)
 			goto err_free;
+		list_add_tail(&p->list, &d->properties);
+	}
 
-		p->length = property->length;
+	return d;
 
-		switch (property->type) {
-		case TB_PROPERTY_TYPE_DIRECTORY:
-			p->value.dir = tb_property_copy_dir(property->value.dir);
-			if (!p->value.dir)
-				goto err_free;
-			break;
+err_free:
+	tb_property_free_dir(d);
+	return NULL;
+}
 
-		case TB_PROPERTY_TYPE_DATA:
-			p->value.data = kmemdup(property->value.data,
-						property->length * 4,
-						GFP_KERNEL);
-			if (!p->value.data)
-				goto err_free;
-			break;
+static struct tb_property *tb_property_copy(const struct tb_property *property)
+{
+	struct tb_property *p;
 
-		case TB_PROPERTY_TYPE_TEXT:
-			p->value.text = kzalloc(p->length * 4, GFP_KERNEL);
-			if (!p->value.text)
-				goto err_free;
-			strcpy(p->value.text, property->value.text);
-			break;
+	p = tb_property_alloc(property->key, property->type);
+	if (!p)
+		return NULL;
 
-		case TB_PROPERTY_TYPE_VALUE:
-			p->value.immediate = property->value.immediate;
-			break;
+	p->length = property->length;
+	switch (property->type) {
+	case TB_PROPERTY_TYPE_DIRECTORY:
+		p->value.dir = copy_dir(property->value.dir);
+		if (!p->value.dir)
+			goto err_free;
+		break;
 
-		default:
-			break;
-		}
+	case TB_PROPERTY_TYPE_DATA:
+		p->value.data = kmemdup(property->value.data,
+					property->length * 4,
+					GFP_KERNEL);
+		if (!p->value.data)
+			goto err_free;
+		break;
 
-		list_add_tail(&p->list, &d->properties);
+	case TB_PROPERTY_TYPE_TEXT:
+		p->value.text = kzalloc(p->length * 4, GFP_KERNEL);
+		if (!p->value.text)
+			goto err_free;
+		strcpy(p->value.text, property->value.text);
+		break;
+
+	case TB_PROPERTY_TYPE_VALUE:
+		p->value.immediate = property->value.immediate;
+		break;
+
+	default:
+		break;
 	}
 
-	return d;
+	return p;
 
 err_free:
 	kfree(p);
-	tb_property_free_dir(d);
-
 	return NULL;
 }
 
+/**
+ * tb_property_copy_dir() - Take a deep copy of directory
+ * @dir: Directory to copy
+ *
+ * The resulting directory needs to be released by calling tb_property_free_dir().
+ *
+ * Return: Pointer to &struct tb_property_dir, %NULL in case of failure.
+ */
+struct tb_property_dir *tb_property_copy_dir(const struct tb_property_dir *dir)
+{
+	return copy_dir(dir);
+}
+EXPORT_SYMBOL_GPL(tb_property_copy_dir);
+
+/**
+ * tb_property_merge_dir() - Merges directory into parent
+ * @parent: Directory to merge @dir
+ * @dir: Directory that is merged
+ * @replace: Replace existing entries
+ *
+ * This will merge @dir into @parent. Both must have same UUID. The
+ * properties in @dir will overwrite overlapping properties in @parent
+ * if @replace is %true. Contents of @dir is copied (so if it is not
+ * needed afterwards it needs to relesed by calling tb_property_free_dir()).
+ */
+int tb_property_merge_dir(struct tb_property_dir *parent,
+			  const struct tb_property_dir *dir,
+			  bool replace)
+{
+	const struct tb_property *property;
+
+	if (WARN_ON(parent == dir))
+		return -EINVAL;
+
+	if (!uuid_equal(parent->uuid, dir->uuid))
+		return -EINVAL;
+
+	list_for_each_entry(property, &dir->properties, list) {
+		struct tb_property *p, *tmp;
+
+		tmp = tb_property_copy(property);
+		if (!tmp)
+			return -ENOMEM;
+
+		p = tb_property_find(parent, property->key, property->type);
+		if (p) {
+			if (replace) {
+				/*
+				 * Found existing property in parent so
+				 * replace with the new one.
+				 */
+				list_replace(&p->list, &tmp->list);
+				tb_property_free(p);
+			} else {
+				tb_property_free(tmp);
+				continue;
+			}
+		} else {
+			list_add_tail(&tmp->list, &parent->properties);
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(tb_property_merge_dir);
+
 /**
  * tb_property_add_immediate() - Add immediate property to directory
  * @parent: Directory to add the property
diff --git a/drivers/thunderbolt/stream.c b/drivers/thunderbolt/stream.c
new file mode 100644
index 0000000000000..c1f5c55583d06
--- /dev/null
+++ b/drivers/thunderbolt/stream.c
@@ -0,0 +1,1698 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Stream data over Thunderbolt/USB4 cable
+ *
+ * Copyright (C) 2026, Intel Corporation
+ * Authors: Alan Borzeszkowski <alan.borzeszkowski@linux.intel.com>
+ *	    Mika Westerberg <mika.westerberg@linux.intel.com>
+ */
+
+#define pr_fmt(fmt) "tbstream: " fmt
+
+#include <linux/configfs.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/idr.h>
+#include <linux/miscdevice.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/poll.h>
+#include <linux/sizes.h>
+#include <linux/thunderbolt.h>
+#include <linux/uaccess.h>
+#include <linux/uio.h>
+#include <linux/uuid.h>
+#include <linux/wait.h>
+
+/*
+ * USB4STREAM - Stream data directly over Thunderbolt/USB4 cable
+ *
+ * HopIDs are configured by the user. In Linux this is done through
+ * ConfigFS. Once that is done paths are be established the first time
+ * the stream is opened. Typically the read side is opened first to make
+ * sure all the data will be received.
+ *
+ * End-to-end flow control is mandatory on both sides.
+ *
+ * Data is sent to the other side as tunneled DATA packets. All the data
+ * is owned by the user and passed as-is from the writer to the reader.
+ *
+ * Once the stream device is closed, a CLOSE packet is sent to the peer
+ * so it can take the necessary action. On Linux this typically results
+ * in EOF being returned to the reader.
+ *
+ * Tunneled packet types:
+ *
+ * +-------+---------+------------------+
+ * |  PDF  |  Type   | Payload size     |
+ * +-------+---------+------------------+
+ * |   2   | DATA    | up to 4 KiB      |
+ * |   3   | CLOSE   | up to 256 bytes  |
+ * +-------+---------+------------------+
+ *
+ * Each stream can optionally publish configuration values under its own
+ * XDomain property directory. The name of the directory is the name of
+ * the stream in question and the UUID is up to the stream. For example
+ * if the stream exposes video output then the directory name could be
+ * "video".
+ *
+ * Below values are reserved and can be used by the stream:
+ *
+ * +----------+-----------+-------------------------+
+ * |   Key    |   Type    | Contents                |
+ * +----------+-----------+-------------------------+
+ * | inhopid  | IMMEDIATE | Configured input HopID  |
+ * | outhopid | IMMEDIATE | Configured output HopID |
+ * +----------+-----------+-------------------------+
+ *
+ * It is allowed to add more stream specific properties as well if the
+ * above are not enough.
+ */
+
+#define TBSTREAM_DEV_RING_SIZE		256
+#define TBSTREAM_DEV_MIN_RING_SIZE	32
+#define TBSTREAM_DEV_MAX_RING_SIZE	4096
+#define TBSTREAM_DEV_THROTTLING		8192
+#define TBSTREAM_DEV_MAX_THROTTLING	16776960
+
+/**
+ * enum tbstream_frame_pdf - PDF numbers for tunneled frames
+ * @TBSTREAM_FRAME_START: PDF of the start of the frame
+ * @TBSTREAM_DATA: PDF of the DATA frame
+ * @TBSTREAM_CLOSE: PDF of the CLOSE frame
+ */
+enum tbstream_frame_pdf {
+	TBSTREAM_FRAME_START = 1,
+	TBSTREAM_DATA,
+	TBSTREAM_CLOSE,
+};
+
+/**
+ * struct tbstream_frame - Frame submitted to/from the rings
+ * @sdev: Pointer to the stream device
+ * @page: Page holding the packet
+ * @offset: Offset inside @page if partial read is done
+ * @completed: %true if the RX frame is completed
+ * @frame: Underlying frame structure
+ */
+struct tbstream_frame {
+	struct tbstream_dev *sdev;
+	struct page *page;
+	unsigned int offset;
+	bool completed;
+	struct ring_frame frame;
+};
+
+/**
+ * struct tbstream_ring - Stream RX/TX ring structure
+ * @ring: Pointer to the API ring
+ * @prod: Current value of producer
+ * @cons: Current value of consumer
+ * @frames: Holds the ring frames
+ */
+struct tbstream_ring {
+	struct tb_ring *ring;
+	unsigned long prod;
+	unsigned long cons;
+	struct tbstream_frame *frames;
+};
+
+/**
+ * struct tbstream_dev - Stream character device
+ * @group: ConfigFS group for this device
+ * @stream: Pointer to the stream if it is attached (%NULL otherwise)
+ * @misc: Character device used for tunneling
+ * @kref: Reference count
+ * @index: Unique identifier for the character device
+ * @in_hopid: In HopID
+ * @out_hopid: Out HopID
+ * @ring_size: Size of the rings
+ * @throttling: Interrupt throttling rate in ns
+ * @users: Number of times @cdev has been opened
+ * @closed: CLOSE packet was received
+ * @removed: Userspace removed the ConfigFS group underneath.
+ * @wait: Waitqueue for open, read and write
+ * @lock: Lock protecting this structure
+ * @tx_ring: Transmit ring
+ * @rx_ring: Receive ring
+ * @list: Stream devices are linked through this
+ */
+struct tbstream_dev {
+	struct config_group group;
+	struct tbstream *stream;
+	struct miscdevice misc;
+	struct kref kref;
+	int index;
+	int in_hopid;
+	int out_hopid;
+	unsigned int ring_size;
+	unsigned int throttling;
+	int users;
+	bool closed;
+	bool removed;
+	wait_queue_head_t wait;
+	struct mutex lock;
+	struct tbstream_ring tx_ring;
+	struct tbstream_ring rx_ring;
+	struct list_head list;
+};
+
+/**
+ * struct tbstream_group - Config group for stream
+ * @group: ConfigFS group for @stream
+ * @stream: Stream the ConfigFS group is attached to. %NULL if there is
+ *	    no stream attached.
+ * @lock: Lock protecting this structure
+ * @dev_list: List of stream devices
+ *
+ * This is the ConfigFS directory for one connection to another host.
+ * There can be several &struct stream_dev linked through @dev_list of
+ * this structure. Reference count managed through @group.
+ */
+struct tbstream_group {
+	struct config_group group;
+	struct tbstream *stream;
+	struct mutex lock;
+	struct list_head dev_list;
+};
+
+/**
+ * struct tbstream - Stream service private data
+ * @kref: Reference count
+ * @svc: Pointer to the service device
+ * @list: Streams are linked through this in @stream_list
+ *
+ * This represents the actual physical connection between two hosts.
+ */
+struct tbstream {
+	struct kref kref;
+	struct tb_service *svc;
+	struct list_head list;
+};
+
+static DEFINE_IDA(tbstream_indices);
+
+/* Protects tbstream_list */
+static DEFINE_MUTEX(tbstream_lock);
+static LIST_HEAD(tbstream_list);
+
+/* Serializes tbstream_get()/put() */
+static DEFINE_MUTEX(tbstream_kref_lock);
+
+/* Serializes tbstream_dev_get()/put() */
+static DEFINE_MUTEX(tbstream_dev_kref_lock);
+
+/* Stream property directory UUID: 3a1cb984-c4d9-4469-a277-ce2fdfd11f0d */
+static const uuid_t tbstream_dir_uuid =
+	UUID_INIT(0x3a1cb984, 0xc4d9, 0x4469,
+		  0xa2, 0x77, 0xce, 0x2f, 0xdf, 0xd1, 0x1f, 0x0d);
+
+static struct tb_property_dir *tbstream_dir;
+
+static void tbstream_release(struct kref *kref)
+{
+	struct tbstream *stream = container_of(kref, typeof(*stream), kref);
+
+	tb_service_put(stream->svc);
+	kfree(stream);
+}
+
+static void tbstream_put(struct tbstream *stream)
+{
+	if (stream) {
+		guard(mutex)(&tbstream_kref_lock);
+		kref_put(&stream->kref, tbstream_release);
+	}
+}
+
+static struct tbstream *tbstream_get(struct tbstream *stream)
+{
+	if (stream) {
+		guard(mutex)(&tbstream_kref_lock);
+		kref_get(&stream->kref);
+	}
+	return stream;
+}
+
+static inline bool tbstream_valid(const struct tbstream *stream)
+{
+	if (stream)
+		return !tb_service_parent(stream->svc)->is_unplugged;
+	return false;
+}
+
+static void tbstream_ring_free(struct tbstream_ring *ring)
+{
+	struct device *dma_dev = tb_ring_dma_device(ring->ring);
+	enum dma_data_direction dir;
+	int i;
+
+	if (ring->ring->is_tx)
+		dir = DMA_TO_DEVICE;
+	else
+		dir = DMA_FROM_DEVICE;
+
+	for (i = 0; i < tb_ring_size(ring->ring); i++) {
+		struct tbstream_frame *sf = &ring->frames[i];
+
+		if (sf->frame.buffer_phy)
+			dma_unmap_page(dma_dev, sf->frame.buffer_phy,
+				       tb_ring_frame_size(&sf->frame), dir);
+		sf->frame.buffer_phy = 0;
+		if (sf->page)
+			__free_page(sf->page);
+		sf->page = NULL;
+	}
+
+	ring->prod = 0;
+	ring->cons = 0;
+	kfree(ring->frames);
+}
+
+static inline bool tbstream_ring_available(const struct tbstream_ring *ring)
+{
+	return ring->prod > ring->cons;
+}
+
+static inline struct tb_xdomain *tbstream_dev_xdomain(struct tbstream_dev *sdev)
+{
+	if (sdev->stream)
+		return tb_service_parent(sdev->stream->svc);
+	return NULL;
+}
+
+static void tbstream_dev_release(struct kref *kref)
+{
+	struct tbstream_dev *sdev = container_of(kref, struct tbstream_dev, kref);
+
+	if (sdev->stream) {
+		struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
+
+		if (sdev->out_hopid > 0)
+			tb_xdomain_release_out_hopid(xd, sdev->out_hopid);
+		if (sdev->in_hopid > 0)
+			tb_xdomain_release_in_hopid(xd, sdev->in_hopid);
+
+		tbstream_put(sdev->stream);
+	}
+	ida_free(&tbstream_indices, sdev->index);
+	kfree(sdev->misc.name);
+	kfree(sdev);
+}
+
+static inline void tbstream_dev_put(struct tbstream_dev *sdev)
+{
+	guard(mutex)(&tbstream_dev_kref_lock);
+	kref_put(&sdev->kref, tbstream_dev_release);
+}
+
+static inline struct tbstream_dev *tbstream_dev_get(struct tbstream_dev *sdev)
+{
+	guard(mutex)(&tbstream_dev_kref_lock);
+	kref_get(&sdev->kref);
+	return sdev;
+}
+
+static inline struct tbstream_dev *to_tbstream_dev(struct miscdevice *misc)
+{
+	return container_of(misc, struct tbstream_dev, misc);
+}
+
+static inline int tbstream_dev_valid(const struct tbstream_dev *sdev)
+{
+	const struct tbstream *stream = sdev->stream;
+
+	if (!tbstream_valid(stream))
+		return -ENXIO;
+	if (sdev->in_hopid <= 0 || sdev->out_hopid <= 0)
+		return -EINVAL;
+	return 0;
+}
+
+static inline bool tbstream_dev_removed(const struct tbstream_dev *sdev)
+{
+	return sdev->removed;
+}
+
+static inline bool tbstream_dev_closed(const struct tbstream_dev *sdev)
+{
+	return sdev->closed;
+}
+
+static void
+tbstream_dev_rx_callback(struct tb_ring *ring, struct ring_frame *frame,
+			 bool canceled)
+{
+	struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame);
+	struct tbstream_dev *sdev = sf->sdev;
+
+	if (canceled)
+		return;
+
+	sf->completed = true;
+	sdev->rx_ring.prod++;
+
+	if (sf->frame.flags & RING_DESC_CRC_ERROR)
+		pr_warn("RX CRC error\n");
+	else if (sf->frame.flags & RING_DESC_BUFFER_OVERRUN)
+		pr_warn("RX buffer overrun\n");
+	else
+		wake_up_interruptible_poll(&sdev->wait, EPOLLIN | EPOLLRDNORM);
+}
+
+static struct tbstream_frame *
+tbstream_dev_completed_rx(struct tbstream_dev *sdev)
+{
+	struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring);
+	struct tbstream_frame *sf;
+	int index;
+
+	index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring);
+	sf = &sdev->rx_ring.frames[index];
+	if (!sf->completed)
+		return NULL;
+
+	dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy,
+				tb_ring_frame_size(&sf->frame),
+				DMA_FROM_DEVICE);
+	return sf;
+}
+
+static int tbstream_dev_consume_rx(struct tbstream_dev *sdev)
+{
+	struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring);
+	struct tbstream_frame *sf;
+	int index;
+
+	index = sdev->rx_ring.cons % tb_ring_size(sdev->rx_ring.ring);
+	sdev->rx_ring.cons++;
+
+	sf = &sdev->rx_ring.frames[index];
+	sf->completed = false;
+	sf->offset = 0;
+	sf->frame.size = 0;
+
+	dma_sync_single_for_device(dma_dev, sf->frame.buffer_phy,
+				   tb_ring_frame_size(&sf->frame),
+				   DMA_FROM_DEVICE);
+
+	return tb_ring_rx(sdev->rx_ring.ring, &sf->frame);
+}
+
+static int tbstream_dev_alloc_rx_buffers(struct tbstream_dev *sdev)
+{
+	size_t ring_size = tb_ring_size(sdev->rx_ring.ring);
+	int i;
+
+	sdev->rx_ring.frames = kcalloc(ring_size, sizeof(struct tbstream_frame),
+				       GFP_KERNEL);
+	if (!sdev->rx_ring.frames)
+		return -ENOMEM;
+
+	for (i = 0; i < ring_size; i++) {
+		struct device *dma_dev = tb_ring_dma_device(sdev->rx_ring.ring);
+		struct tbstream_frame *sf = &sdev->rx_ring.frames[i];
+		dma_addr_t dma_addr;
+
+		sf->page = alloc_page(GFP_KERNEL);
+		if (!sf->page)
+			return -ENOMEM;
+
+		dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE,
+					DMA_FROM_DEVICE);
+		if (dma_mapping_error(dma_dev, dma_addr)) {
+			__free_page(sf->page);
+			sf->page = NULL;
+			return -ENOMEM;
+		}
+
+		sf->sdev = sdev;
+		sf->frame.callback = tbstream_dev_rx_callback;
+		sf->frame.buffer_phy = dma_addr;
+
+		tb_ring_rx(sdev->rx_ring.ring, &sf->frame);
+	}
+
+	sdev->rx_ring.cons = 0;
+	sdev->rx_ring.prod = 0;
+	return 0;
+}
+
+static void
+tbstream_dev_tx_callback(struct tb_ring *ring, struct ring_frame *frame,
+			 bool canceled)
+{
+	struct tbstream_frame *sf = container_of(frame, typeof(*sf), frame);
+	struct tbstream_dev *sdev = sf->sdev;
+
+	if (canceled)
+		return;
+
+	sdev->tx_ring.prod++;
+	if (sf->frame.eof == TBSTREAM_DATA)
+		wake_up_interruptible_poll(&sdev->wait, EPOLLOUT | EPOLLWRNORM);
+}
+
+static int tbstream_dev_alloc_tx_buffers(struct tbstream_dev *sdev)
+{
+	struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring);
+	size_t ring_size = tb_ring_size(sdev->tx_ring.ring);
+	int i;
+
+	sdev->tx_ring.frames = kcalloc(ring_size, sizeof(struct tbstream_frame),
+				       GFP_KERNEL);
+	if (!sdev->tx_ring.frames)
+		return -ENOMEM;
+
+	for (i = 0; i < ring_size; i++) {
+		struct tbstream_frame *sf = &sdev->tx_ring.frames[i];
+		dma_addr_t dma_addr;
+
+		sf->page = alloc_page(GFP_KERNEL);
+		if (!sf->page)
+			return -ENOMEM;
+
+		dma_addr = dma_map_page(dma_dev, sf->page, 0, TB_MAX_FRAME_SIZE,
+					DMA_TO_DEVICE);
+		if (dma_mapping_error(dma_dev, dma_addr)) {
+			__free_page(sf->page);
+			sf->page = NULL;
+			return -ENOMEM;
+		}
+
+		sf->sdev = sdev;
+		sf->frame.callback = tbstream_dev_tx_callback;
+		sf->frame.buffer_phy = dma_addr;
+		sf->frame.sof = TBSTREAM_FRAME_START;
+	}
+
+	sdev->tx_ring.cons = 0;
+	sdev->tx_ring.prod = ring_size - 1;
+	return 0;
+}
+
+static struct tbstream_frame *
+tbstream_dev_alloc_tx(struct tbstream_dev *sdev, enum tbstream_frame_pdf pdf,
+		      struct iov_iter *from, size_t size)
+{
+	struct device *dma_dev = tb_ring_dma_device(sdev->tx_ring.ring);
+	struct tbstream_frame *sf;
+	int index;
+
+	if (!tbstream_ring_available(&sdev->tx_ring))
+		return ERR_PTR(-ENOBUFS);
+
+	index = sdev->tx_ring.cons % tb_ring_size(sdev->tx_ring.ring);
+	sdev->tx_ring.cons++;
+
+	sf = &sdev->tx_ring.frames[index];
+	sf->frame.size = size < TB_MAX_FRAME_SIZE ? size : 0;
+	sf->frame.eof = pdf;
+
+	dma_sync_single_for_cpu(dma_dev, sf->frame.buffer_phy, size,
+				DMA_TO_DEVICE);
+	if (pdf == TBSTREAM_DATA) {
+		if (copy_page_from_iter(sf->page, 0, size, from) != size)
+			return ERR_PTR(-EFAULT);
+	} else {
+		memset(page_address(sf->page), 0, size);
+	}
+	dma_sync_single_for_device(dma_dev, sf->frame.buffer_phy, size,
+				   DMA_TO_DEVICE);
+	return sf;
+}
+
+static int
+tbstream_dev_send_data(struct tbstream_dev *sdev, struct iov_iter *from,
+		       size_t size)
+{
+	struct tbstream_frame *sf;
+
+	sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_DATA, from, size);
+	if (IS_ERR(sf))
+		return PTR_ERR(sf);
+	return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
+}
+
+static int tbstream_dev_send_close(struct tbstream_dev *sdev)
+{
+	struct tbstream_frame *sf;
+
+	sf = tbstream_dev_alloc_tx(sdev, TBSTREAM_CLOSE, NULL, SZ_256);
+	if (IS_ERR(sf))
+		return PTR_ERR(sf);
+	return tb_ring_tx(sdev->tx_ring.ring, &sf->frame);
+}
+
+static int tbstream_dev_start(struct tbstream_dev *sdev)
+{
+	struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
+	u16 sof_mask, eof_mask;
+	struct tb_ring *ring;
+	int ret, e2e_tx_hop;
+
+	ring = tb_ring_alloc_tx(xd->tb->nhi, -1, sdev->ring_size,
+				RING_FLAG_FRAME | RING_FLAG_E2E);
+	if (!ring)
+		return -ENOMEM;
+	sdev->tx_ring.ring = ring;
+
+	ret = tbstream_dev_alloc_tx_buffers(sdev);
+	if (ret)
+		goto err_free_tx;
+
+	e2e_tx_hop = ring->hop;
+	sof_mask = BIT(TBSTREAM_FRAME_START);
+	eof_mask = BIT(TBSTREAM_DATA) | BIT(TBSTREAM_CLOSE);
+
+	ring = tb_ring_alloc_rx(xd->tb->nhi, -1, sdev->ring_size,
+				RING_FLAG_FRAME | RING_FLAG_E2E, e2e_tx_hop,
+				sof_mask, eof_mask, NULL, NULL);
+	if (!ring) {
+		ret = -ENOMEM;
+		goto err_free_tx_buffers;
+	}
+	sdev->rx_ring.ring = ring;
+
+	ret = tb_xdomain_enable_paths(xd, sdev->out_hopid,
+				     sdev->tx_ring.ring->hop,
+				     sdev->in_hopid,
+				     sdev->rx_ring.ring->hop);
+	if (ret)
+		goto err_free_rx;
+
+	tb_ring_throttling(sdev->tx_ring.ring, sdev->throttling);
+	tb_ring_throttling(sdev->rx_ring.ring, sdev->throttling);
+
+	tb_ring_start(sdev->tx_ring.ring);
+	tb_ring_start(sdev->rx_ring.ring);
+
+	ret = tbstream_dev_alloc_rx_buffers(sdev);
+	if (ret)
+		goto err_stop;
+	return 0;
+
+err_stop:
+	tb_ring_stop(sdev->rx_ring.ring);
+	tb_ring_stop(sdev->tx_ring.ring);
+err_free_rx:
+	tb_ring_free(sdev->rx_ring.ring);
+err_free_tx_buffers:
+	tbstream_ring_free(&sdev->tx_ring);
+err_free_tx:
+	tb_ring_free(sdev->tx_ring.ring);
+
+	return ret;
+}
+
+static void tbstream_dev_stop(struct tbstream_dev *sdev)
+{
+	struct tb_xdomain *xd;
+
+	/* Wait for the ring to complete any outstanding frames */
+	tb_ring_flush(sdev->tx_ring.ring, 500);
+	tb_ring_stop(sdev->tx_ring.ring);
+	tb_ring_flush(sdev->rx_ring.ring, 500);
+	tb_ring_stop(sdev->rx_ring.ring);
+
+	xd = tbstream_dev_xdomain(sdev);
+	if (xd) {
+		tb_xdomain_disable_paths(xd, sdev->out_hopid,
+					 sdev->tx_ring.ring->hop,
+					 sdev->in_hopid,
+					 sdev->rx_ring.ring->hop);
+	}
+
+	tbstream_ring_free(&sdev->rx_ring);
+	tb_ring_free(sdev->rx_ring.ring);
+	sdev->rx_ring.ring = NULL;
+	tbstream_ring_free(&sdev->tx_ring);
+	tb_ring_free(sdev->tx_ring.ring);
+	sdev->tx_ring.ring = NULL;
+}
+
+static ssize_t
+tbstream_dev_fops_read_iter(struct kiocb *kiocb, struct iov_iter *to)
+{
+	struct file *file = kiocb->ki_filp;
+	struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
+	size_t nbytes;
+	int ret;
+
+	ret = tbstream_dev_valid(sdev);
+	if (ret)
+		return ret;
+
+	if (mutex_lock_interruptible(&sdev->lock))
+		return -ERESTARTSYS;
+
+	while (!tbstream_ring_available(&sdev->rx_ring)) {
+		mutex_unlock(&sdev->lock);
+
+		if (file->f_flags & O_NONBLOCK)
+			return -EAGAIN;
+		ret = wait_event_interruptible(sdev->wait,
+				tbstream_ring_available(&sdev->rx_ring) ||
+				tbstream_dev_valid(sdev) != 0 ||
+				tbstream_dev_closed(sdev) ||
+				tbstream_dev_removed(sdev));
+		if (ret)
+			return ret;
+
+		ret = tbstream_dev_valid(sdev);
+		if (ret)
+			return ret;
+
+		if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev))
+			return 0;
+
+		if (mutex_lock_interruptible(&sdev->lock))
+			return -ERESTARTSYS;
+	}
+
+	nbytes = 0;
+	while (nbytes < iov_iter_count(to)) {
+		struct tbstream_frame *sf;
+		size_t size, sf_size;
+
+		sf = tbstream_dev_completed_rx(sdev);
+		if (!sf)
+			break;
+		/*
+		 * CLOSE tunneled packet. If userspace already read
+		 * something then we stop processing now and return
+		 * those bytes. Next time the first frame will be CLOSE
+		 * in which case we return EOF to the user.
+		 */
+		if (sf->frame.eof == TBSTREAM_CLOSE) {
+			if (!nbytes) {
+				tbstream_dev_consume_rx(sdev);
+				sdev->closed = true;
+			}
+			break;
+		}
+
+		sf_size = tb_ring_frame_size(&sf->frame);
+		size = min(iov_iter_count(to) - nbytes, sf_size);
+
+		if (copy_page_to_iter(sf->page, sf->offset, size, to) != size) {
+			ret = -EFAULT;
+			break;
+		}
+
+		/*
+		 * If not all data from the frame is read so leave it in
+		 * place and update the offset accordingly so next read
+		 * gets the rest.
+		 */
+		if (size < sf_size) {
+			sf->offset += size;
+			sf->frame.size = sf_size - size;
+		} else {
+			ret = tbstream_dev_consume_rx(sdev);
+			if (ret)
+				break;
+		}
+
+		nbytes += size;
+	}
+
+	mutex_unlock(&sdev->lock);
+	if (ret)
+		return ret;
+	return nbytes;
+}
+
+static ssize_t
+tbstream_dev_fops_write_iter(struct kiocb *kiocb, struct iov_iter *from)
+{
+	struct file *file = kiocb->ki_filp;
+	struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
+	size_t nbytes;
+	int ret;
+
+	ret = tbstream_dev_valid(sdev);
+	if (ret)
+		return ret;
+
+	if (mutex_lock_interruptible(&sdev->lock))
+		return -ERESTARTSYS;
+
+	while (!tbstream_ring_available(&sdev->tx_ring)) {
+		mutex_unlock(&sdev->lock);
+
+		if (file->f_flags & O_NONBLOCK)
+			return -EAGAIN;
+		ret = wait_event_interruptible(sdev->wait,
+				tbstream_ring_available(&sdev->tx_ring) ||
+				tbstream_dev_valid(sdev) != 0 ||
+				tbstream_dev_closed(sdev) ||
+				tbstream_dev_removed(sdev));
+		if (ret)
+			return ret;
+
+		ret = tbstream_dev_valid(sdev);
+		if (ret)
+			return ret;
+
+		if (tbstream_dev_closed(sdev) || tbstream_dev_removed(sdev))
+			return -ENXIO;
+
+		if (mutex_lock_interruptible(&sdev->lock))
+			return -ERESTARTSYS;
+	}
+
+	nbytes = 0;
+	while (nbytes < iov_iter_count(from)) {
+		size_t size;
+
+		size = min(iov_iter_count(from) - nbytes, TB_MAX_FRAME_SIZE);
+		ret = tbstream_dev_send_data(sdev, from, size);
+		if (ret) {
+			/*
+			 * If there are no more buffers we are done for
+			 * this write.
+			 */
+			if (ret == -ENOBUFS)
+				ret = 0;
+			break;
+		}
+
+		nbytes += size;
+	}
+
+	mutex_unlock(&sdev->lock);
+	if (ret)
+		return ret;
+	return nbytes;
+}
+
+static __poll_t
+tbstream_dev_fops_poll(struct file *file, struct poll_table_struct *wait)
+{
+	struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
+	__poll_t mask = 0;
+
+	poll_wait(file, &sdev->wait, wait);
+	guard(mutex)(&sdev->lock);
+	if (tbstream_dev_valid(sdev) != 0) {
+		mask |= EPOLLHUP | EPOLLERR;
+	} else {
+		if (tbstream_ring_available(&sdev->tx_ring))
+			mask |= EPOLLOUT | EPOLLWRNORM;
+		if (tbstream_ring_available(&sdev->rx_ring))
+			mask |= EPOLLIN | EPOLLRDNORM;
+	}
+	return mask;
+}
+
+static int tbstream_dev_fops_open(struct inode *inode, struct file *file)
+{
+	struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
+	int ret;
+
+	tbstream_dev_get(sdev);
+
+	if (mutex_lock_interruptible(&sdev->lock)) {
+		tbstream_dev_put(sdev);
+		return -ERESTARTSYS;
+	}
+
+	/*
+	 * If there is no stream attached yet, block until it appears
+	 * unless this is opened in non-blocking mode.
+	 */
+	while ((ret = tbstream_dev_valid(sdev))) {
+		mutex_unlock(&sdev->lock);
+
+		if (ret != -ENXIO || (file->f_flags & O_NONBLOCK))
+			goto err_put;
+
+		ret = wait_event_interruptible(sdev->wait,
+				tbstream_dev_valid(sdev) == 0 ||
+				tbstream_dev_removed(sdev));
+		if (ret)
+			goto err_put;
+
+		if (tbstream_dev_removed(sdev)) {
+			ret = -ENXIO;
+			goto err_put;
+		}
+
+		if (mutex_lock_interruptible(&sdev->lock)) {
+			ret = -ERESTARTSYS;
+			goto err_put;
+		}
+	}
+
+	/* Only on first open we allocate rings and enable paths */
+	if (!sdev->users++) {
+		ret = tbstream_dev_start(sdev);
+		if (ret) {
+			sdev->users--;
+			goto err_unlock;
+		}
+		sdev->closed = false;
+	}
+
+	mutex_unlock(&sdev->lock);
+	return 0;
+
+err_unlock:
+	mutex_unlock(&sdev->lock);
+err_put:
+	tbstream_dev_put(sdev);
+
+	return ret;
+}
+
+static int tbstream_dev_fops_release(struct inode *inode, struct file *file)
+{
+	struct tbstream_dev *sdev = to_tbstream_dev(file->private_data);
+
+	mutex_lock(&sdev->lock);
+	if (--sdev->users == 0) {
+		/*
+		 * Send CLOSE tunneled packet to notify the other end
+		 * that we are closing the file. We do this twice if the
+		 * first one fails.
+		 */
+		tbstream_dev_send_close(sdev);
+		tbstream_dev_stop(sdev);
+	}
+	mutex_unlock(&sdev->lock);
+
+	tbstream_dev_put(sdev);
+	return 0;
+}
+
+static const struct file_operations tbstream_dev_fops = {
+	.owner = THIS_MODULE,
+	.llseek = noop_llseek,
+	.read_iter = tbstream_dev_fops_read_iter,
+	.write_iter = tbstream_dev_fops_write_iter,
+	.poll = tbstream_dev_fops_poll,
+	.open = tbstream_dev_fops_open,
+	.release = tbstream_dev_fops_release,
+};
+
+static inline struct tbstream_dev *
+tbstream_dev_from_group(struct config_group *group)
+{
+	return container_of(group, struct tbstream_dev, group);
+}
+
+static ssize_t tbstream_dev_index_show(struct config_item *item, char *buf)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+
+	return sysfs_emit(buf, "%d\n", sdev->index);
+}
+CONFIGFS_ATTR_RO(tbstream_dev_, index);
+
+static ssize_t tbstream_dev_in_hopid_show(struct config_item *item, char *buf)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+
+	return sysfs_emit(buf, "%d\n", sdev->in_hopid);
+}
+
+/* svc->lock must be held */
+static void service_remove_properties(struct tb_service *svc, const char *name)
+{
+	struct tb_property *p;
+
+	if (!svc->local_properties)
+		return;
+
+	p = tb_property_find(svc->local_properties, name,
+			     TB_PROPERTY_TYPE_DIRECTORY);
+	if (p) {
+		tb_property_free_dir(p->value.dir);
+		tb_property_remove(p);
+
+		dev_dbg(&svc->dev, "removed local directory %s\n", name);
+
+		/*
+		 * Is the service directory empty already? If it is then
+		 * we can release it as well.
+		 */
+		tb_property_for_each(svc->local_properties, p) {
+			if (p->type == TB_PROPERTY_TYPE_DIRECTORY)
+				return;
+		}
+
+		tb_property_free_dir(svc->local_properties);
+		svc->local_properties = NULL;
+	}
+}
+
+static int service_update_properties(struct tb_service *svc, const char *name,
+				     int in_hopid, int out_hopid)
+{
+	struct tb_property_dir *dir;
+	struct tb_property *p;
+
+	guard(mutex)(&svc->lock);
+
+	if (in_hopid < 8 || out_hopid < 8) {
+		service_remove_properties(svc, name);
+		return 0;
+	}
+
+	if (!svc->local_properties) {
+		/*
+		 * Add the service directory first time we
+		 * populate the entries.
+		 */
+		svc->local_properties = tb_property_copy_dir(tbstream_dir);
+		if (!svc->local_properties)
+			return -ENOMEM;
+	}
+
+	p = tb_property_find(svc->local_properties, name,
+			     TB_PROPERTY_TYPE_DIRECTORY);
+	if (p) {
+		dir = p->value.dir;
+
+		p = tb_property_find(dir, "inhopid", TB_PROPERTY_TYPE_VALUE);
+		if (p && p->value.immediate != in_hopid)
+			p->value.immediate = in_hopid;
+		p = tb_property_find(dir, "outhopid", TB_PROPERTY_TYPE_VALUE);
+		if (p && p->value.immediate != out_hopid)
+			p->value.immediate = out_hopid;
+
+		dev_dbg(&svc->dev,
+			"updated local directory %s: in HopID %d, out HopID %d\n",
+			name, in_hopid, out_hopid);
+	} else {
+		uuid_t uuid;
+		int ret;
+
+		uuid_gen(&uuid);
+		dir = tb_property_create_dir(&uuid);
+		if (!dir)
+			return -ENOMEM;
+
+		tb_property_add_immediate(dir, "inhopid", in_hopid);
+		tb_property_add_immediate(dir, "outhopid", out_hopid);
+
+		ret = tb_property_add_dir(svc->local_properties, name, dir);
+		if (ret) {
+			tb_property_free_dir(dir);
+			return ret;
+		}
+
+		dev_dbg(&svc->dev,
+			"added local directory %s: in HopID %d, out HopID %d\n",
+			name, in_hopid, out_hopid);
+	}
+
+	return 0;
+}
+
+static int tbstream_dev_update_properties(struct tbstream_dev *sdev)
+{
+	struct tbstream *stream;
+	int ret;
+
+	stream = tbstream_get(sdev->stream);
+	if (!stream)
+		return 0;
+
+	ret = service_update_properties(stream->svc,
+					config_item_name(&sdev->group.cg_item),
+					sdev->in_hopid, sdev->out_hopid);
+	if (!ret)
+		tb_service_properties_changed(stream->svc);
+
+	tbstream_put(stream);
+	return ret;
+}
+
+static int tbstream_dev_alloc_in_hopid(struct tbstream_dev *sdev, int hopid)
+{
+	struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
+	int ret;
+
+	if (sdev->in_hopid > 0 && sdev->in_hopid != hopid)
+		tb_xdomain_release_in_hopid(xd, sdev->in_hopid);
+	if (!hopid) {
+		sdev->in_hopid = hopid;
+		return 0;
+	}
+	ret = tb_xdomain_alloc_in_hopid(xd, hopid);
+	if (ret < 0)
+		return ret;
+	/*
+	 * If specific HopID was asked by the user and we did not get
+	 * that one then release and return error instead.
+	 */
+	if (hopid > 0 && hopid != ret) {
+		tb_xdomain_release_in_hopid(xd, ret);
+		return -EBUSY;
+	}
+	sdev->in_hopid = ret;
+	return 0;
+}
+
+static int tbstream_dev_alloc_out_hopid(struct tbstream_dev *sdev, int hopid)
+{
+	struct tb_xdomain *xd = tbstream_dev_xdomain(sdev);
+	int ret;
+
+	if (sdev->out_hopid > 0 && sdev->out_hopid != hopid)
+		tb_xdomain_release_out_hopid(xd, sdev->out_hopid);
+	if (!hopid) {
+		sdev->out_hopid = hopid;
+		return 0;
+	}
+	ret = tb_xdomain_alloc_out_hopid(xd, hopid);
+	if (ret < 0)
+		return ret;
+	if (hopid > 0 && hopid != ret) {
+		tb_xdomain_release_out_hopid(xd, ret);
+		return -EBUSY;
+	}
+	sdev->out_hopid = ret;
+	return 0;
+}
+
+static ssize_t
+tbstream_dev_in_hopid_store(struct config_item *item, const char *buf,
+			    size_t count)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+	int ret, in_hopid;
+
+	ret = kstrtoint(buf, 0, &in_hopid);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&sdev->lock);
+	if (sdev->users)
+		return -EBUSY;
+	if (sdev->stream) {
+		ret = tbstream_dev_alloc_in_hopid(sdev, in_hopid);
+		if (ret)
+			return ret;
+		ret = tbstream_dev_update_properties(sdev);
+	} else {
+		sdev->in_hopid = in_hopid;
+	}
+	return ret ? ret : count;
+}
+CONFIGFS_ATTR(tbstream_dev_, in_hopid);
+
+static ssize_t tbstream_dev_out_hopid_show(struct config_item *item, char *buf)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+
+	return sysfs_emit(buf, "%d\n", sdev->out_hopid);
+}
+
+static ssize_t
+tbstream_dev_out_hopid_store(struct config_item *item, const char *buf,
+			     size_t count)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+	int ret, out_hopid;
+
+	ret = kstrtoint(buf, 0, &out_hopid);
+	if (ret)
+		return ret;
+
+	guard(mutex)(&sdev->lock);
+	if (sdev->users)
+		return -EBUSY;
+	if (sdev->stream) {
+		ret = tbstream_dev_alloc_out_hopid(sdev, out_hopid);
+		if (ret)
+			return ret;
+		ret = tbstream_dev_update_properties(sdev);
+	} else {
+		sdev->out_hopid = out_hopid;
+	}
+	return ret ? ret : count;
+}
+CONFIGFS_ATTR(tbstream_dev_, out_hopid);
+
+static ssize_t tbstream_dev_ring_size_show(struct config_item *item, char *buf)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+
+	return sysfs_emit(buf, "%u\n", sdev->ring_size);
+}
+
+static ssize_t
+tbstream_dev_ring_size_store(struct config_item *item, const char *buf,
+			     size_t count)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+	unsigned int ring_size;
+	int ret;
+
+	ret = kstrtouint(buf, 0, &ring_size);
+	if (ret)
+		return ret;
+
+	if (ring_size < TBSTREAM_DEV_MIN_RING_SIZE ||
+	    ring_size > TBSTREAM_DEV_MAX_RING_SIZE)
+		return -EINVAL;
+
+	guard(mutex)(&sdev->lock);
+	if (sdev->users)
+		return -EBUSY;
+	sdev->ring_size = ring_size;
+	return count;
+}
+CONFIGFS_ATTR(tbstream_dev_, ring_size);
+
+static ssize_t tbstream_dev_throttling_show(struct config_item *item, char *buf)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+
+	return sysfs_emit(buf, "%u\n", sdev->throttling);
+}
+
+static ssize_t
+tbstream_dev_throttling_store(struct config_item *item, const char *buf,
+			      size_t count)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+	unsigned int throttling;
+	int ret;
+
+	ret = kstrtouint(buf, 0, &throttling);
+	if (ret)
+		return ret;
+
+	if (throttling > TBSTREAM_DEV_MAX_THROTTLING)
+		return -EINVAL;
+
+	guard(mutex)(&sdev->lock);
+	if (sdev->users)
+		return -EBUSY;
+	sdev->throttling = throttling;
+	return count;
+}
+CONFIGFS_ATTR(tbstream_dev_, throttling);
+
+static struct configfs_attribute *tbstream_dev_attrs[] = {
+	&tbstream_dev_attr_index,
+	&tbstream_dev_attr_in_hopid,
+	&tbstream_dev_attr_out_hopid,
+	&tbstream_dev_attr_ring_size,
+	&tbstream_dev_attr_throttling,
+	NULL,
+};
+
+static void tbstream_dev_item_release(struct config_item *item)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(group);
+
+	misc_deregister(&sdev->misc);
+	tbstream_dev_put(sdev);
+}
+
+static struct configfs_item_operations tbstream_dev_item_ops = {
+	.release = tbstream_dev_item_release,
+};
+
+static const struct config_item_type tbstream_dev_type = {
+	.ct_owner = THIS_MODULE,
+	.ct_item_ops = &tbstream_dev_item_ops,
+	.ct_attrs = tbstream_dev_attrs,
+};
+
+static void service_get_hopids(struct tb_service *svc, const char *name,
+			       int *in_hopid, int *out_hopid)
+{
+	struct tb_property_dir *dir;
+	struct tb_property *p;
+
+	guard(mutex)(&svc->lock);
+
+	/* See if we have directory entry with the matching name */
+	p = tb_property_find(svc->remote_properties, name,
+			     TB_PROPERTY_TYPE_DIRECTORY);
+	if (!p)
+		return;
+
+	dir = p->value.dir;
+
+	/*
+	 * We need to reverse the HopIDs on our end so that in becomes
+	 * out and vice versa.
+	 */
+	p = tb_property_find(dir, "inhopid", TB_PROPERTY_TYPE_VALUE);
+	if (p && p->value.immediate >= 8)
+		*out_hopid = p->value.immediate;
+	p = tb_property_find(dir, "outhopid", TB_PROPERTY_TYPE_VALUE);
+	if (p && p->value.immediate >= 8)
+		*in_hopid = p->value.immediate;
+}
+
+static void
+tbstream_dev_attach_stream(struct tbstream_dev *sdev, struct tbstream_group *sg)
+{
+	const char *name = config_item_name(&sdev->group.cg_item);
+	struct tbstream *stream;
+
+	stream = tbstream_get(sg->stream);
+	if (!stream)
+		return;
+
+	scoped_guard(mutex, &sdev->lock) {
+		sdev->stream = stream;
+		/*
+		 * If there is no existing configuration (or automatic
+		 * configuration is being used) check if the other side
+		 * has configuration for this and use it.
+		 */
+		if (sdev->in_hopid <= 0 && sdev->out_hopid <= 0)
+			service_get_hopids(stream->svc, name, &sdev->in_hopid,
+					   &sdev->out_hopid);
+		if (sdev->in_hopid)
+			tbstream_dev_alloc_in_hopid(sdev, sdev->in_hopid);
+		if (sdev->out_hopid)
+			tbstream_dev_alloc_out_hopid(sdev, sdev->out_hopid);
+	}
+
+	service_update_properties(stream->svc, name, sdev->in_hopid,
+				  sdev->out_hopid);
+	tb_service_properties_changed(stream->svc);
+
+	/* Notify any openerers that the stream is now attached */
+	wake_up_interruptible(&sdev->wait);
+}
+
+static void tbstream_dev_detach_stream(struct tbstream_dev *sdev)
+{
+	const char *name = config_item_name(&sdev->group.cg_item);
+	struct tbstream *stream;
+	struct tb_xdomain *xd;
+
+	scoped_guard(mutex, &sdev->lock) {
+		stream = sdev->stream;
+		if (!stream)
+			return;
+		sdev->stream = NULL;
+		xd = tb_service_parent(stream->svc);
+		if (sdev->out_hopid > 0)
+			tb_xdomain_release_out_hopid(xd, sdev->out_hopid);
+		if (sdev->in_hopid > 0)
+			tb_xdomain_release_in_hopid(xd, sdev->in_hopid);
+	}
+
+	service_update_properties(stream->svc, name, 0, 0);
+	tb_service_properties_changed(stream->svc);
+
+	tbstream_put(stream);
+
+	/* Notify any task that the stream is not valid anymore */
+	wake_up_interruptible_poll(&sdev->wait, EPOLLHUP | EPOLLERR);
+}
+
+static inline struct tbstream_group *
+to_tbstream_group(struct config_group *group)
+{
+	return container_of(group, struct tbstream_group, group);
+}
+
+static struct config_group *
+tbstream_dev_make_group(struct config_group *group, const char *name)
+{
+	struct tbstream_group *sg = to_tbstream_group(group);
+	struct tbstream_dev *sdev;
+	int ret, index;
+
+	/*
+	 * We want the names to be suitable for passing as property
+	 * directory names.
+	 */
+	if (strlen(name) > TB_PROPERTY_KEY_SIZE)
+		return ERR_PTR(-ENAMETOOLONG);
+
+	sdev = kzalloc_obj(*sdev, GFP_KERNEL);
+	if (!sdev)
+		return ERR_PTR(-ENOMEM);
+
+	index = ida_alloc(&tbstream_indices, GFP_KERNEL);
+	if (index < 0) {
+		kfree(sdev);
+		return ERR_PTR(index);
+	}
+
+	sdev->index = index;
+	sdev->ring_size = TBSTREAM_DEV_RING_SIZE;
+	sdev->throttling = TBSTREAM_DEV_THROTTLING;
+	mutex_init(&sdev->lock);
+	init_waitqueue_head(&sdev->wait);
+	INIT_LIST_HEAD(&sdev->list);
+	/* This point forward tbstream_dev_put() must be used to release sdev */
+	kref_init(&sdev->kref);
+
+	config_group_init_type_name(&sdev->group, name, &tbstream_dev_type);
+
+	scoped_guard(mutex, &sg->lock)
+		list_add_tail(&sdev->list, &sg->dev_list);
+
+	tbstream_dev_attach_stream(sdev, sg);
+
+	sdev->misc.name = kasprintf(GFP_KERNEL, "tbstream%d", index);
+	sdev->misc.minor = MISC_DYNAMIC_MINOR;
+	sdev->misc.fops = &tbstream_dev_fops;
+
+	ret = misc_register(&sdev->misc);
+	if (ret) {
+		tbstream_dev_detach_stream(sdev);
+		scoped_guard(mutex, &sg->lock)
+			list_del(&sdev->list);
+		/* Calls tbstream_dev_put() */
+		config_group_put(&sdev->group);
+		return ERR_PTR(ret);
+	}
+
+	return &sdev->group;
+}
+
+static void
+tbstream_dev_drop_item(struct config_group *group, struct config_item *item)
+{
+	struct config_group *sdev_group = to_config_group(item);
+	struct tbstream_dev *sdev = tbstream_dev_from_group(sdev_group);
+	struct tbstream_group *sg = to_tbstream_group(group);
+
+	scoped_guard(mutex, &sg->lock)
+		list_del(&sdev->list);
+	/* Notify any task that the underlying group was removed  */
+	sdev->removed = true;
+	wake_up_interruptible_poll(&sdev->wait, EPOLLHUP | EPOLLERR);
+	config_item_put(item);
+}
+
+static struct configfs_group_operations tbstream_dev_group_ops = {
+	.make_group = tbstream_dev_make_group,
+	.drop_item = tbstream_dev_drop_item,
+};
+
+static void tbstream_item_release(struct config_item *item)
+{
+	struct config_group *group = to_config_group(item);
+	struct tbstream_group *sg = to_tbstream_group(group);
+
+	tbstream_put(sg->stream);
+	kfree(sg);
+}
+
+static struct configfs_item_operations tbstream_item_ops = {
+	.release = tbstream_item_release,
+};
+
+static const struct config_item_type tbstream_dev_group_type = {
+	.ct_owner = THIS_MODULE,
+	.ct_group_ops = &tbstream_dev_group_ops,
+	.ct_item_ops = &tbstream_item_ops,
+};
+
+static struct config_group *
+tbstream_make_group(struct config_group *group, const char *name)
+{
+	struct tbstream_group *sg;
+	struct tbstream *stream;
+	int domain, index;
+	u64 route;
+
+	/* Make sure the format is correct */
+	if (sscanf(name, "%u-%llx.%u", &domain, &route, &index) != 3)
+		return ERR_PTR(-EINVAL);
+
+	sg = kzalloc_obj(*sg, GFP_KERNEL);
+	if (!sg)
+		return ERR_PTR(-ENOMEM);
+
+	mutex_init(&sg->lock);
+	INIT_LIST_HEAD(&sg->dev_list);
+
+	guard(mutex)(&tbstream_lock);
+	list_for_each_entry(stream, &tbstream_list, list) {
+		tbstream_get(stream);
+		if (sysfs_streq(name, dev_name(&stream->svc->dev))) {
+			sg->stream = stream;
+			break;
+		}
+		tbstream_put(stream);
+	}
+
+	config_group_init_type_name(&sg->group, name, &tbstream_dev_group_type);
+	return &sg->group;
+}
+
+static struct configfs_group_operations tbstream_group_ops = {
+	.make_group = tbstream_make_group,
+};
+
+static const struct config_item_type tbstream_group_type = {
+	.ct_owner = THIS_MODULE,
+	.ct_group_ops = &tbstream_group_ops,
+};
+
+static struct config_group tbstream_group = {
+	.cg_item = {
+		.ci_namebuf = "stream",
+		.ci_type = &tbstream_group_type,
+	},
+};
+
+/* Returns reference count increased */
+static struct tbstream_group *tbstream_group_find(struct tbstream *stream)
+{
+	const char *name = dev_name(&stream->svc->dev);
+	struct config_item *item;
+
+	guard(mutex)(&tbstream_group.cg_subsys->su_mutex);
+	item = config_group_find_item(&tbstream_group, name);
+	if (!item)
+		return NULL;
+	return to_tbstream_group(to_config_group(item));
+}
+
+static void tbstream_group_attach_stream(struct tbstream *stream)
+{
+	struct tbstream_group *sg;
+	struct tbstream_dev *sdev;
+
+	sg = tbstream_group_find(stream);
+	if (!sg)
+		return;
+
+	guard(mutex)(&sg->lock);
+	if (WARN_ON(sg->stream)) {
+		config_group_put(&sg->group);
+		return;
+	}
+	sg->stream = tbstream_get(stream);
+	/*
+	 * If there are existing stream devices, attach the stream to
+	 * them now.
+	 */
+	list_for_each_entry(sdev, &sg->dev_list, list) {
+		tbstream_dev_get(sdev);
+		tbstream_dev_attach_stream(sdev, sg);
+		tbstream_dev_put(sdev);
+	}
+
+	config_group_put(&sg->group);
+}
+
+static void tbstream_group_detach_stream(struct tbstream *stream)
+{
+	struct tbstream_group *sg;
+	struct tbstream_dev *sdev;
+
+	sg = tbstream_group_find(stream);
+	if (!sg)
+		return;
+
+	guard(mutex)(&sg->lock);
+	if (sg->stream) {
+		/* Detach this stream from the stream devices */
+		list_for_each_entry_reverse(sdev, &sg->dev_list, list) {
+			tbstream_dev_get(sdev);
+			tbstream_dev_detach_stream(sdev);
+			tbstream_dev_put(sdev);
+		}
+		tbstream_put(sg->stream);
+		sg->stream = NULL;
+	}
+
+	config_group_put(&sg->group);
+}
+
+static int tbstream_probe(struct tb_service *svc, const struct tb_service_id *id)
+{
+	struct tbstream *stream;
+
+	stream = kzalloc_obj(*stream, GFP_KERNEL);
+	if (!stream)
+		return -ENOMEM;
+
+	/* After this point, release stream by calling tbstream_put() */
+	kref_init(&stream->kref);
+	stream->svc = tb_service_get(svc);
+	INIT_LIST_HEAD(&stream->list);
+
+	scoped_guard(mutex, &tbstream_lock)
+		list_add_tail(&stream->list, &tbstream_list);
+
+	tbstream_group_attach_stream(stream);
+	tb_service_set_drvdata(svc, stream);
+	return 0;
+}
+
+static void tbstream_remove(struct tb_service *svc)
+{
+	struct tbstream *stream = tb_service_get_drvdata(svc);
+
+	tbstream_group_detach_stream(stream);
+	scoped_guard(mutex, &tbstream_lock)
+		list_del(&stream->list);
+	tbstream_put(stream);
+}
+
+static int __maybe_unused tbstream_suspend(struct device *dev)
+{
+	struct tb_service *svc = tb_to_service(dev);
+	struct tbstream *stream = tb_service_get_drvdata(svc);
+	struct tbstream_group *sg;
+	struct tbstream_dev *sdev;
+
+	sg = tbstream_group_find(stream);
+	if (!sg)
+		return 0;
+
+	list_for_each_entry_reverse(sdev, &sg->dev_list, list) {
+		tbstream_dev_get(sdev);
+		/* Stop the stream (if it was open) */
+		if (sdev->users)
+			tbstream_dev_stop(sdev);
+		tbstream_dev_put(sdev);
+	}
+
+	config_group_put(&sg->group);
+	return 0;
+}
+
+static int __maybe_unused tbstream_resume(struct device *dev)
+{
+	struct tb_service *svc = tb_to_service(dev);
+	struct tbstream *stream = tb_service_get_drvdata(svc);
+	struct tbstream_group *sg;
+	struct tbstream_dev *sdev;
+
+	sg = tbstream_group_find(stream);
+	if (!sg)
+		return 0;
+
+	list_for_each_entry(sdev, &sg->dev_list, list) {
+		tbstream_dev_get(sdev);
+		if (sdev->users) {
+			int ret;
+
+			ret = tbstream_dev_start(sdev);
+			if (ret) {
+				tbstream_dev_put(sdev);
+				config_group_put(&sg->group);
+				return ret;
+			}
+		}
+		tbstream_dev_put(sdev);
+	}
+
+	config_group_put(&sg->group);
+	return 0;
+}
+
+static const struct dev_pm_ops tbstream_pm_ops = {
+	SET_SYSTEM_SLEEP_PM_OPS(tbstream_suspend, tbstream_resume)
+};
+
+static const struct tb_service_id tbstream_ids[] = {
+	{ TB_SERVICE("stream", 1) },
+	{ },
+};
+MODULE_DEVICE_TABLE(tbsvc, tbstream_ids);
+
+static struct tb_service_driver tbstream_driver = {
+	.driver = {
+		.owner = THIS_MODULE,
+		.name = "thunderbolt_stream",
+		.pm = &tbstream_pm_ops,
+	},
+	.probe = tbstream_probe,
+	.remove = tbstream_remove,
+	.id_table = tbstream_ids,
+};
+
+static int __init tbstream_init(void)
+{
+	int ret;
+
+	tbstream_dir = tb_property_create_dir(&tbstream_dir_uuid);
+	if (!tbstream_dir)
+		return -ENOMEM;
+
+	tb_property_add_immediate(tbstream_dir, "prtcid", 1);
+	tb_property_add_immediate(tbstream_dir, "prtcvers", 1);
+	tb_property_add_immediate(tbstream_dir, "prtcrevs", 0);
+	tb_property_add_immediate(tbstream_dir, "prtcstns", 0);
+
+	ret = tb_register_property_dir("stream", tbstream_dir);
+	if (ret)
+		goto err_free_dir;
+
+	config_group_init(&tbstream_group);
+	ret = tb_configfs_register_group(&tbstream_group);
+	if (ret)
+		goto err_unregister_dir;
+
+	ret = tb_register_service_driver(&tbstream_driver);
+	if (ret)
+		goto err_unregister_group;
+	return 0;
+
+err_unregister_group:
+	tb_configfs_unregister_group(&tbstream_group);
+err_unregister_dir:
+	tb_unregister_property_dir("stream", tbstream_dir);
+err_free_dir:
+	tb_property_free_dir(tbstream_dir);
+	return ret;
+}
+module_init(tbstream_init);
+
+static void __exit tbstream_exit(void)
+{
+	tb_unregister_service_driver(&tbstream_driver);
+	tb_configfs_unregister_group(&tbstream_group);
+	tb_unregister_property_dir("stream", tbstream_dir);
+	tb_property_free_dir(tbstream_dir);
+	ida_destroy(&tbstream_indices);
+}
+module_exit(tbstream_exit);
+
+MODULE_AUTHOR("Alan Borzeszkowski <alan.borzeszkowski@linux.intel.com>");
+MODULE_AUTHOR("Mika Westerberg <mika.westerberg@linux.intel.com>");
+MODULE_DESCRIPTION("Stream data over Thunderbolt/USB4 cable");
+MODULE_LICENSE("GPL");
diff --git a/drivers/thunderbolt/switch.c b/drivers/thunderbolt/switch.c
index c2ad58b19e7b1..d7c53eb3221b0 100644
--- a/drivers/thunderbolt/switch.c
+++ b/drivers/thunderbolt/switch.c
@@ -1620,6 +1620,12 @@ static int tb_switch_reset_host(struct tb_switch *sw)
 				ret = tb_port_reset(port);
 				if (ret)
 					return ret;
+				/*
+				 * USB4 Lane 1 adapters do not have accessible
+				 * path config space.
+				 */
+				if (tb_switch_is_usb4(sw) && !port->usb4)
+					continue;
 			} else if (tb_port_is_usb3_down(port) ||
 				   tb_port_is_usb3_up(port)) {
 				tb_usb3_port_enable(port, false);
@@ -1761,8 +1767,6 @@ int tb_switch_wait_for_bit(struct tb_switch *sw, u32 offset, u32 bit,
 /*
  * tb_plug_events_active() - enable/disable plug events on a switch
  *
- * Also configures a sane plug_events_delay of 255ms.
- *
  * Return: %0 on success, negative errno otherwise.
  */
 static int tb_plug_events_active(struct tb_switch *sw, bool active)
@@ -1773,11 +1777,6 @@ static int tb_plug_events_active(struct tb_switch *sw, bool active)
 	if (tb_switch_is_icm(sw) || tb_switch_is_usb4(sw))
 		return 0;
 
-	sw->config.plug_events_delay = 0xff;
-	res = tb_sw_write(sw, ((u32 *) &sw->config) + 4, TB_CFG_SWITCH, 4, 1);
-	if (res)
-		return res;
-
 	res = tb_sw_read(sw, &data, TB_CFG_SWITCH, sw->cap_plug_events + 1, 1);
 	if (res)
 		return res;
@@ -2639,6 +2638,8 @@ int tb_switch_configure(struct tb_switch *sw)
 
 	sw->config.enabled = 1;
 
+	/* Set Notification Timeout to 255 ms for all routers */
+	sw->config.plug_events_delay = 0xff;
 	if (tb_switch_is_usb4(sw)) {
 		/*
 		 * For USB4 devices, we need to program the CM version
@@ -2650,7 +2651,6 @@ int tb_switch_configure(struct tb_switch *sw)
 			sw->config.cmuv = ROUTER_CS_4_CMUV_V1;
 		else
 			sw->config.cmuv = ROUTER_CS_4_CMUV_V2;
-		sw->config.plug_events_delay = 0xa;
 
 		/* Enumerate the switch */
 		ret = tb_sw_write(sw, (u32 *)&sw->config + 1, TB_CFG_SWITCH,
@@ -2671,7 +2671,7 @@ int tb_switch_configure(struct tb_switch *sw)
 
 		/* Enumerate the switch */
 		ret = tb_sw_write(sw, (u32 *)&sw->config + 1, TB_CFG_SWITCH,
-				  ROUTER_CS_1, 3);
+				  ROUTER_CS_1, 4);
 	}
 	if (ret)
 		return ret;
@@ -2971,14 +2971,14 @@ static int tb_switch_lane_bonding_enable(struct tb_switch *sw)
 	int ret;
 
 	if (!tb_switch_lane_bonding_possible(sw))
-		return 0;
+		return -EOPNOTSUPP;
 
 	up = tb_upstream_port(sw);
 	down = tb_switch_downstream_port(sw);
 
 	if (!tb_port_width_supported(up, TB_LINK_WIDTH_DUAL) ||
 	    !tb_port_width_supported(down, TB_LINK_WIDTH_DUAL))
-		return 0;
+		return -EOPNOTSUPP;
 
 	/*
 	 * Both lanes need to be in CL0. Here we assume lane 0 already be in
@@ -3625,6 +3625,20 @@ int tb_switch_resume(struct tb_switch *sw, bool runtime)
 				tb_port_warn(port,
 					     "lost during suspend, disconnecting\n");
 				tb_sw_set_unplugged(port->remote->sw);
+			} else if (port->xdomain) {
+				/*
+				 * If the user replaced the XDomain with
+				 * another router, this will succeed in
+				 * which case we must remove the XDomain
+				 * before adding the new router.
+				 */
+				err = tb_cfg_get_upstream_port(sw->tb->ctl,
+							       port->xdomain->route);
+				if (err > 0) {
+					tb_port_warn(port,
+						     "XDomain was disconnected\n");
+					port->xdomain->is_unplugged = true;
+				}
 			}
 		}
 	}
diff --git a/drivers/thunderbolt/tb.c b/drivers/thunderbolt/tb.c
index c69c323e6952a..72a0dd27937e8 100644
--- a/drivers/thunderbolt/tb.c
+++ b/drivers/thunderbolt/tb.c
@@ -98,7 +98,7 @@ static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug)
 	if (!ev)
 		return;
 
-	ev->tb = tb;
+	ev->tb = tb_domain_get(tb);
 	ev->route = route;
 	ev->port = port;
 	ev->unplug = unplug;
@@ -2524,9 +2524,14 @@ static void tb_handle_hotplug(struct work_struct *work)
 out:
 	mutex_unlock(&tb->lock);
 
+	tb_domain_unregister_unplugged_xdomains(tb);
+
 	pm_runtime_mark_last_busy(&tb->dev);
 	pm_runtime_put_autosuspend(&tb->dev);
 
+	/* Undo the refcount increased in tb_queue_hotplug() */
+	tb_domain_put(tb);
+
 	kfree(ev);
 }
 
@@ -2844,6 +2849,9 @@ static void tb_handle_dp_bandwidth_request(struct work_struct *work)
 
 		/* Update other clients about the allocation change */
 		tb_recalc_estimated_bandwidth(tb);
+
+		tb_dbg(tb, "checking if more DP tunnels can be established now\n");
+		tb_tunnel_dp(tb);
 	}
 
 put_sw:
@@ -2949,6 +2957,7 @@ static void tb_stop(struct tb *tb)
 		tb_tunnel_put(tunnel);
 	}
 	tb_switch_remove(tb->root_switch);
+	tb->root_switch = NULL;
 	tcm->hotplug_active = false; /* signal tb_handle_hotplug to quit */
 }
 
@@ -3110,6 +3119,24 @@ static void tb_restore_children(struct tb_switch *sw)
 	}
 }
 
+static void tb_free_unplugged_xdomains(struct tb_switch *sw)
+{
+	struct tb_port *port;
+
+	tb_switch_for_each_port(sw, port) {
+		if (tb_is_upstream_port(port))
+			continue;
+		if (port->xdomain && port->xdomain->is_unplugged) {
+			tb_retimer_remove_all(port);
+			tb_xdomain_remove(port->xdomain);
+			tb_port_unconfigure_xdomain(port);
+			port->xdomain = NULL;
+		} else if (port->remote) {
+			tb_free_unplugged_xdomains(port->remote->sw);
+		}
+	}
+}
+
 static int tb_resume_noirq(struct tb *tb)
 {
 	struct tb_cm *tcm = tb_priv(tb);
@@ -3129,6 +3156,7 @@ static int tb_resume_noirq(struct tb *tb)
 	tb_switch_resume(tb->root_switch, false);
 	tb_free_invalid_tunnels(tb);
 	tb_free_unplugged_children(tb->root_switch);
+	tb_free_unplugged_xdomains(tb->root_switch);
 	tb_restore_children(tb->root_switch);
 
 	/*
@@ -3171,28 +3199,6 @@ static int tb_resume_noirq(struct tb *tb)
 	return 0;
 }
 
-static int tb_free_unplugged_xdomains(struct tb_switch *sw)
-{
-	struct tb_port *port;
-	int ret = 0;
-
-	tb_switch_for_each_port(sw, port) {
-		if (tb_is_upstream_port(port))
-			continue;
-		if (port->xdomain && port->xdomain->is_unplugged) {
-			tb_retimer_remove_all(port);
-			tb_xdomain_remove(port->xdomain);
-			tb_port_unconfigure_xdomain(port);
-			port->xdomain = NULL;
-			ret++;
-		} else if (port->remote) {
-			ret += tb_free_unplugged_xdomains(port->remote->sw);
-		}
-	}
-
-	return ret;
-}
-
 static int tb_freeze_noirq(struct tb *tb)
 {
 	struct tb_cm *tcm = tb_priv(tb);
@@ -3212,14 +3218,14 @@ static int tb_thaw_noirq(struct tb *tb)
 static void tb_complete(struct tb *tb)
 {
 	/*
-	 * Release any unplugged XDomains and if there is a case where
+	 * Unregister unplugged XDomains and if there is a case where
 	 * another domain is swapped in place of unplugged XDomain we
 	 * need to run another rescan.
 	 */
-	mutex_lock(&tb->lock);
-	if (tb_free_unplugged_xdomains(tb->root_switch))
-		tb_scan_switch(tb->root_switch);
-	mutex_unlock(&tb->lock);
+	if (tb_domain_unregister_unplugged_xdomains(tb)) {
+		scoped_guard(mutex, &tb->lock)
+			tb_scan_switch(tb->root_switch);
+	}
 }
 
 static int tb_runtime_suspend(struct tb *tb)
@@ -3246,11 +3252,11 @@ static void tb_remove_work(struct work_struct *work)
 	struct tb *tb = tcm_to_tb(tcm);
 
 	mutex_lock(&tb->lock);
-	if (tb->root_switch) {
+	if (tb->root_switch)
 		tb_free_unplugged_children(tb->root_switch);
-		tb_free_unplugged_xdomains(tb->root_switch);
-	}
 	mutex_unlock(&tb->lock);
+
+	tb_free_unplugged_xdomains(tb->root_switch);
 }
 
 static int tb_runtime_resume(struct tb *tb)
diff --git a/drivers/thunderbolt/tb.h b/drivers/thunderbolt/tb.h
index 217c3114bec8b..003db653418ae 100644
--- a/drivers/thunderbolt/tb.h
+++ b/drivers/thunderbolt/tb.h
@@ -793,6 +793,7 @@ int tb_domain_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,
 				       int transmit_path, int transmit_ring,
 				       int receive_path, int receive_ring);
 int tb_domain_disconnect_all_paths(struct tb *tb);
+int tb_domain_unregister_unplugged_xdomains(struct tb *tb);
 
 static inline struct tb *tb_domain_get(struct tb *tb)
 {
@@ -1263,6 +1264,7 @@ struct tb_xdomain *tb_xdomain_alloc(struct tb *tb, struct device *parent,
 				    const uuid_t *remote_uuid);
 void tb_xdomain_add(struct tb_xdomain *xd);
 void tb_xdomain_remove(struct tb_xdomain *xd);
+void tb_xdomain_unregister(struct tb_xdomain *xd);
 struct tb_xdomain *tb_xdomain_find_by_link_depth(struct tb *tb, u8 link,
 						 u8 depth);
 
@@ -1479,6 +1481,7 @@ int usb4_dp_port_allocate_bandwidth(struct tb_port *port, int bw);
 int usb4_dp_port_requested_bandwidth(struct tb_port *port);
 
 int usb4_pci_port_set_ext_encapsulation(struct tb_port *port, bool enable);
+int usb4_pci_port_ltssm_state(struct tb_port *port);
 
 static inline bool tb_is_usb4_port_device(const struct device *dev)
 {
@@ -1556,4 +1559,12 @@ static inline void tb_retimer_debugfs_init(struct tb_retimer *rt) { }
 static inline void tb_retimer_debugfs_remove(struct tb_retimer *rt) { }
 #endif
 
+#if IS_REACHABLE(CONFIG_CONFIGFS_FS)
+int tb_configfs_init(void);
+void tb_configfs_exit(void);
+#else
+static inline int tb_configfs_init(void) { return 0; }
+static inline void tb_configfs_exit(void) { }
+#endif
+
 #endif
diff --git a/drivers/thunderbolt/tb_regs.h b/drivers/thunderbolt/tb_regs.h
index c0bf136236e67..92d893634d2bc 100644
--- a/drivers/thunderbolt/tb_regs.h
+++ b/drivers/thunderbolt/tb_regs.h
@@ -182,8 +182,7 @@ struct tb_regs_switch_header {
 	/* DWORD 4 */
 	u32 plug_events_delay:8; /*
 				  * RW, pause between plug events in
-				  * milliseconds. Writing 0x00 is interpreted
-				  * as 255ms.
+				  * milliseconds.
 				  */
 	u32 cmuv:8;
 	u32 __unknown4:8;
@@ -216,6 +215,7 @@ struct tb_regs_switch_header {
 #define ROUTER_CS_6_WOPS			BIT(2)
 #define ROUTER_CS_6_WOUS			BIT(3)
 #define ROUTER_CS_6_HCI				BIT(18)
+#define ROUTER_CS_6_RR				BIT(24)
 #define ROUTER_CS_6_CR				BIT(25)
 #define ROUTER_CS_7				0x07
 #define ROUTER_CS_9				0x09
@@ -473,10 +473,25 @@ struct tb_regs_port_header {
 
 /* PCIe adapter registers */
 #define ADP_PCIE_CS_0				0x00
+#define ADP_PCIE_CS_0_LTSSM_MASK		GENMASK(28, 25)
 #define ADP_PCIE_CS_0_PE			BIT(31)
 #define ADP_PCIE_CS_1				0x01
 #define ADP_PCIE_CS_1_EE			BIT(0)
 
+enum tb_pcie_ltssm_state {
+	USB4_PCIE_LTSSM_DETECT,
+	USB4_PCIE_LTSSM_POLLING,
+	USB4_PCIE_LTSSM_CONFIG,
+	USB4_PCIE_LTSSM_CONFIG_IDLE,
+	USB4_PCIE_LTSSM_RECOVERY,
+	USB4_PCIE_LTSSM_RECOVERY_IDLE,
+	USB4_PCIE_LTSSM_L0,
+	USB4_PCIE_LTSSM_L1,
+	USB4_PCIE_LTSSM_L2,
+	USB4_PCIE_LTSSM_DISABLED,
+	USB4_PCIE_LTSSM_HOT_RESET,
+};
+
 /* USB adapter registers */
 #define ADP_USB3_CS_0				0x00
 #define ADP_USB3_CS_0_V				BIT(30)
diff --git a/drivers/thunderbolt/test.c b/drivers/thunderbolt/test.c
index 1f4318249c226..fa8bee0ccf8b6 100644
--- a/drivers/thunderbolt/test.c
+++ b/drivers/thunderbolt/test.c
@@ -2852,7 +2852,214 @@ static void tb_test_property_copy(struct kunit *test)
 	tb_property_free_dir(src);
 }
 
+/*
+ * Reproducers for three memory-safety defects in
+ * drivers/thunderbolt/property.c reached from a crafted XDomain
+ * PROPERTIES_RESPONSE payload.  Without the fix these trip KASAN or
+ * smash the kernel stack; with the fix each returns NULL cleanly.
+ *
+ * The on-wire entry layout matches struct tb_property_entry in
+ * property.c (private to that translation unit): u32 key_hi, u32
+ * key_lo, then a packed u32 = (type << 24) | (reserved << 16) |
+ * length, then u32 value.  Each entry is 4 dwords.
+ */
+
+static void tb_test_property_parse_u32_wrap(struct kunit *test)
+{
+	/*
+	 * 0x102 dwords: enough for the entry's length field (0x100) to
+	 * pass the "entry->length > block_len" gate so the wrap check
+	 * is actually exercised.  parse_dwdata's downstream OOB read
+	 * lands ~16 GiB past the allocation regardless.
+	 */
+	u32 *block = kunit_kzalloc(test, 0x102 * sizeof(u32), GFP_KERNEL);
+	struct tb_property_dir *dir;
+
+	KUNIT_ASSERT_NOT_NULL(test, block);
+
+	block[0] = 0x55584401;	/* "UXD" v1 magic */
+	block[1] = 0x00000004;	/* Root directory length: one entry */
+
+	/*
+	 * DATA entry whose value 0xffffff00 + length 0x100 wrap to 0
+	 * in u32, passing the sum <= block_len guard even though the
+	 * real offset is far past the allocation.
+	 */
+	block[2] = 0x61616161;	/* key_hi */
+	block[3] = 0x61616161;	/* key_lo */
+	block[4] = 0x64000100;	/* type=DATA, reserved=0, length=0x100 */
+	block[5] = 0xffffff00;	/* value */
+
+	dir = tb_property_parse_dir(block, 0x102);
+	KUNIT_EXPECT_NULL(test, dir);
+	tb_property_free_dir(dir);
+}
+
+static void tb_test_property_parse_recursion(struct kunit *test)
+{
+	/*
+	 * 10 dwords: rootdir header (2) + parent DIRECTORY entry (4) +
+	 * the child entry that lives at dir_offset(2) + UUID(4) = 6,
+	 * occupying block[6..9].  Each recursive level re-reads the
+	 * same block[6..9] as its first child entry, which is itself
+	 * a DIRECTORY pointing at offset 2.
+	 */
+	u32 *block = kunit_kzalloc(test, 10 * sizeof(u32), GFP_KERNEL);
+	struct tb_property_dir *dir;
+
+	KUNIT_ASSERT_NOT_NULL(test, block);
+
+	block[0] = 0x55584401;	/* "UXD" v1 magic */
+	block[1] = 0x00000004;	/* Root directory length: one entry */
+
+	/*
+	 * DIRECTORY entry pointing at dir_offset = 2 with length = 8.
+	 * Non-root parse derives content_offset = 6, content_len = 4,
+	 * nentries = 1.  block[6..9] is read both as the parent's UUID
+	 * (kmemdup'd into dir->uuid) and as the single child entry --
+	 * which is itself a DIRECTORY pointing at offset 2, so the
+	 * recursion never terminates and the kernel stack is exhausted.
+	 */
+	block[2] = 0x61616161;	/* key_hi */
+	block[3] = 0x61616161;	/* key_lo */
+	block[4] = 0x44000008;	/* type=DIRECTORY, reserved=0, length=8 */
+	block[5] = 0x00000002;	/* value = dir_offset */
+
+	block[6] = 0x62626262;	/* doubles as UUID dword 0 / child key_hi */
+	block[7] = 0x62626262;	/* doubles as UUID dword 1 / child key_lo */
+	block[8] = 0x44000008;	/* type=DIRECTORY, reserved=0, length=8 */
+	block[9] = 0x00000002;	/* value = dir_offset (back at parent) */
+
+	dir = tb_property_parse_dir(block, 10);
+	KUNIT_EXPECT_NULL(test, dir);
+	tb_property_free_dir(dir);
+}
+
+static void tb_test_property_parse_dir_len_underflow(struct kunit *test)
+{
+	/*
+	 * Allocate exactly 7 dwords (28 bytes) so the kmalloc-32 chunk
+	 * leaves a 4-byte slab redzone tail that KASAN-Generic can flag.
+	 * With block_len = 7, dir_offset = 4, dir_len = 3, the non-root
+	 * UUID kmemdup reads 16 bytes from byte 16, so bytes 28..31 fall
+	 * in the redzone and trip a KASAN slab-out-of-bounds report on
+	 * the pre-fix kernel.  Sizing the buffer at a power of two (32,
+	 * 64, ...) puts the over-read into the slab cache tail where
+	 * KASAN's generic shadow does not flag it, and the test reduces
+	 * to the downstream content_len = dir_len - 4 underflow path
+	 * which also returns NULL.
+	 */
+	u32 *block = kunit_kzalloc(test, 7 * sizeof(u32), GFP_KERNEL);
+	struct tb_property_dir *dir;
+
+	KUNIT_ASSERT_NOT_NULL(test, block);
+
+	block[0] = 0x55584401;	/* "UXD" v1 magic */
+	block[1] = 0x00000004;	/* Root directory length: one entry */
+
+	/*
+	 * DIRECTORY entry with length = 3 pointing at dir_offset = 4.
+	 * tb_property_entry_valid() permits value(4) + length(3) <=
+	 * block_len(7).  Non-root parse begins with a kmemdup of 4
+	 * dwords from dir_offset for the UUID; that read runs past the
+	 * 28-byte allocation before the dir_len < 4 reject would fire.
+	 */
+	block[2] = 0x61616161;	/* key_hi */
+	block[3] = 0x61616161;	/* key_lo */
+	block[4] = 0x44000003;	/* type=DIRECTORY, reserved=0, length=3 */
+	block[5] = 0x00000004;	/* value = dir_offset */
+	/* block[6] is the start of the four UUID dwords; block[7..] is OOB. */
+
+	dir = tb_property_parse_dir(block, 7);
+	KUNIT_EXPECT_NULL(test, dir);
+	tb_property_free_dir(dir);
+}
+
+static void tb_test_property_merge(struct kunit *test)
+{
+	struct tb_property_dir *dir1, *dir2, *dir3;
+	struct tb_property *p;
+	uuid_t uuid;
+	int ret;
+
+	dir1 = tb_property_create_dir(&network_dir_uuid);
+	KUNIT_ASSERT_NOT_NULL(test, dir1);
+	ret = tb_property_add_immediate(dir1, "prtcid", 1);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	ret = tb_property_add_immediate(dir1, "prtcvers", 1);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	ret = tb_property_add_immediate(dir1, "prtcrevs", 0);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	ret = tb_property_add_immediate(dir1, "prtcstns", 0);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	dir2 = tb_property_create_dir(&network_dir_uuid);
+	KUNIT_ASSERT_NOT_NULL(test, dir2);
+	ret = tb_property_add_text(dir2, "descr", "This is text");
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	/* This replaces the value in dir1 */
+	ret = tb_property_add_immediate(dir2, "prtcvers", 0x1234);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	uuid_gen(&uuid);
+	dir3 = tb_property_create_dir(&uuid);
+	KUNIT_ASSERT_NOT_NULL(test, dir3);
+	ret = tb_property_add_immediate(dir3, "value0", 0);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	ret = tb_property_add_text(dir3, "value1", "Text value");
+	KUNIT_EXPECT_EQ(test, ret, 0);
+	ret = tb_property_add_dir(dir2, "my", dir3);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	ret = tb_property_merge_dir(dir1, dir2, true);
+	KUNIT_EXPECT_EQ(test, ret, 0);
+
+	p = tb_property_get_next(dir1, NULL);
+	KUNIT_ASSERT_NOT_NULL(test, p);
+	KUNIT_ASSERT_STREQ(test, &p->key[0], "prtcid");
+	KUNIT_ASSERT_EQ(test, p->type, TB_PROPERTY_TYPE_VALUE);
+	KUNIT_ASSERT_EQ(test, p->length, 1);
+	KUNIT_ASSERT_EQ(test, p->value.immediate, 1);
+	p = tb_property_get_next(dir1, p);
+	KUNIT_ASSERT_NOT_NULL(test, p);
+	KUNIT_ASSERT_STREQ(test, &p->key[0], "prtcvers");
+	KUNIT_ASSERT_EQ(test, p->type, TB_PROPERTY_TYPE_VALUE);
+	KUNIT_ASSERT_EQ(test, p->length, 1);
+	KUNIT_ASSERT_EQ(test, p->value.immediate, 0x1234);
+	p = tb_property_get_next(dir1, p);
+	KUNIT_ASSERT_NOT_NULL(test, p);
+	KUNIT_ASSERT_STREQ(test, &p->key[0], "prtcrevs");
+	KUNIT_ASSERT_EQ(test, p->type, TB_PROPERTY_TYPE_VALUE);
+	KUNIT_ASSERT_EQ(test, p->length, 1);
+	KUNIT_ASSERT_EQ(test, p->value.immediate, 0);
+	p = tb_property_get_next(dir1, p);
+	KUNIT_ASSERT_NOT_NULL(test, p);
+	KUNIT_ASSERT_STREQ(test, &p->key[0], "prtcstns");
+	KUNIT_ASSERT_EQ(test, p->type, TB_PROPERTY_TYPE_VALUE);
+	KUNIT_ASSERT_EQ(test, p->length, 1);
+	KUNIT_ASSERT_EQ(test, p->value.immediate, 0);
+	p = tb_property_get_next(dir1, p);
+	KUNIT_ASSERT_NOT_NULL(test, p);
+	KUNIT_ASSERT_STREQ(test, &p->key[0], "descr");
+	KUNIT_ASSERT_EQ(test, p->type, TB_PROPERTY_TYPE_TEXT);
+	KUNIT_ASSERT_EQ(test, p->length, 4);
+	KUNIT_ASSERT_STREQ(test, p->value.text, "This is text");
+	p = tb_property_get_next(dir1, p);
+	KUNIT_ASSERT_NOT_NULL(test, p);
+	KUNIT_ASSERT_STREQ(test, &p->key[0], "my");
+	KUNIT_ASSERT_EQ(test, p->type, TB_PROPERTY_TYPE_DIRECTORY);
+	compare_dirs(test, p->value.dir, dir3);
+	p = tb_property_get_next(dir1, p);
+	KUNIT_ASSERT_NULL(test, p);
+
+	tb_property_free_dir(dir2);
+	tb_property_free_dir(dir1);
+}
+
 static struct kunit_case tb_test_cases[] = {
+	KUNIT_CASE(tb_test_property_parse_u32_wrap),
+	KUNIT_CASE(tb_test_property_parse_recursion),
+	KUNIT_CASE(tb_test_property_parse_dir_len_underflow),
 	KUNIT_CASE(tb_test_path_basic),
 	KUNIT_CASE(tb_test_path_not_connected_walk),
 	KUNIT_CASE(tb_test_path_single_hop_walk),
@@ -2892,6 +3099,7 @@ static struct kunit_case tb_test_cases[] = {
 	KUNIT_CASE(tb_test_property_parse),
 	KUNIT_CASE(tb_test_property_format),
 	KUNIT_CASE(tb_test_property_copy),
+	KUNIT_CASE(tb_test_property_merge),
 	{ }
 };
 
diff --git a/drivers/thunderbolt/tunnel.c b/drivers/thunderbolt/tunnel.c
index f38f7753b6e4f..b7f32305f14a6 100644
--- a/drivers/thunderbolt/tunnel.c
+++ b/drivers/thunderbolt/tunnel.c
@@ -290,6 +290,40 @@ static inline void tb_tunnel_changed(struct tb_tunnel *tunnel)
 			tunnel->src_port, tunnel->dst_port);
 }
 
+static int tb_pci_port_ltssm_state_detect(struct tb_port *port)
+{
+	ktime_t timeout = ktime_add_ms(ktime_get(), 500);
+
+	do {
+		int ret;
+
+		ret = usb4_pci_port_ltssm_state(port);
+		if (ret < 0)
+			return ret;
+		if (ret == USB4_PCIE_LTSSM_DETECT)
+			return 0;
+
+		fsleep(50);
+	} while (ktime_before(ktime_get(), timeout));
+
+	return -ETIMEDOUT;
+}
+
+static int tb_pci_pre_activate(struct tb_tunnel *tunnel)
+{
+	struct tb_port *down = tunnel->src_port;
+	struct tb_port *up = tunnel->dst_port;
+	int ret;
+
+	ret = tb_switch_is_usb4(down->sw) ?
+		tb_pci_port_ltssm_state_detect(down) : 0;
+	if (ret)
+		return ret;
+
+	return tb_switch_is_usb4(up->sw) ?
+		tb_pci_port_ltssm_state_detect(up) : 0;
+}
+
 static int tb_pci_set_ext_encapsulation(struct tb_tunnel *tunnel, bool enable)
 {
 	struct tb_port *port = tb_upstream_port(tunnel->dst_port->sw);
@@ -505,6 +539,7 @@ struct tb_tunnel *tb_tunnel_alloc_pci(struct tb *tb, struct tb_port *up,
 	if (!tunnel)
 		return NULL;
 
+	tunnel->pre_activate = tb_pci_pre_activate;
 	tunnel->activate = tb_pci_activate;
 	tunnel->src_port = down;
 	tunnel->dst_port = up;
diff --git a/drivers/thunderbolt/usb4.c b/drivers/thunderbolt/usb4.c
index 9e810b2ae0b5f..5fd4fe070f258 100644
--- a/drivers/thunderbolt/usb4.c
+++ b/drivers/thunderbolt/usb4.c
@@ -294,7 +294,12 @@ int usb4_switch_setup(struct tb_switch *sw)
 	/* TBT3 supported by the CM */
 	val &= ~ROUTER_CS_5_CNS;
 
-	return tb_sw_write(sw, &val, TB_CFG_SWITCH, ROUTER_CS_5, 1);
+	ret = tb_sw_write(sw, &val, TB_CFG_SWITCH, ROUTER_CS_5, 1);
+	if (ret)
+		return ret;
+
+	return tb_switch_wait_for_bit(sw, ROUTER_CS_6, ROUTER_CS_6_RR,
+				      ROUTER_CS_6_RR, 500);
 }
 
 /**
@@ -304,7 +309,7 @@ int usb4_switch_setup(struct tb_switch *sw)
  * Sets configuration valid bit for the router. Must be called before
  * any tunnels can be set through the router and after
  * usb4_switch_setup() has been called. Can be called to host and device
- * routers (does nothing for the latter).
+ * routers (does nothing for the former).
  *
  * Return: %0 on success, negative errno otherwise.
  */
@@ -327,7 +332,7 @@ int usb4_switch_configuration_valid(struct tb_switch *sw)
 		return ret;
 
 	return tb_switch_wait_for_bit(sw, ROUTER_CS_6, ROUTER_CS_6_CR,
-				      ROUTER_CS_6_CR, 50);
+				      ROUTER_CS_6_CR, 500);
 }
 
 /**
@@ -3145,3 +3150,27 @@ int usb4_pci_port_set_ext_encapsulation(struct tb_port *port, bool enable)
 	return tb_port_write(port, &val, TB_CFG_PORT,
 			     port->cap_adap + ADP_PCIE_CS_1, 1);
 }
+
+/**
+ * usb4_pci_port_ltssm_state() - Read PCIe adapter LTSSM state
+ * @port: PCIe adapter
+ *
+ * Return:
+ * * LTSSM state of @port.
+ * * Negative errno - On failure.
+ */
+int usb4_pci_port_ltssm_state(struct tb_port *port)
+{
+	u32 val;
+	int ret;
+
+	if (!tb_port_is_pcie_down(port) && !tb_port_is_pcie_up(port))
+		return -EINVAL;
+
+	ret = tb_port_read(port, &val, TB_CFG_PORT,
+			   port->cap_adap + ADP_PCIE_CS_0, 1);
+	if (ret)
+		return ret;
+
+	return FIELD_GET(ADP_PCIE_CS_0_LTSSM_MASK, val);
+}
diff --git a/drivers/thunderbolt/xdomain.c b/drivers/thunderbolt/xdomain.c
index 754808c43f006..781d88d06b935 100644
--- a/drivers/thunderbolt/xdomain.c
+++ b/drivers/thunderbolt/xdomain.c
@@ -136,6 +136,7 @@ static int __tb_xdomain_response(struct tb_ctl *ctl, const void *response,
 				 size_t size, enum tb_cfg_pkg_type type)
 {
 	struct tb_cfg_request *req;
+	int ret;
 
 	req = tb_cfg_request_alloc();
 	if (!req)
@@ -147,7 +148,11 @@ static int __tb_xdomain_response(struct tb_ctl *ctl, const void *response,
 	req->request_size = size;
 	req->request_type = type;
 
-	return tb_cfg_request(ctl, req, response_ready, req);
+	ret = tb_cfg_request(ctl, req, response_ready, req);
+	if (ret)
+		tb_cfg_request_put(req);
+
+	return ret;
 }
 
 /**
@@ -535,29 +540,19 @@ static int tb_xdp_link_state_status_request(struct tb_ctl *ctl, u64 route,
 }
 
 static int tb_xdp_link_state_status_response(struct tb *tb, struct tb_ctl *ctl,
-					     struct tb_xdomain *xd, u8 sequence)
+					     struct tb_xdomain *xd, u8 sequence,
+					     u8 slw, u8 sls, u8 tls, u8 tlw)
 {
 	struct tb_xdp_link_state_status_response res;
-	struct tb_port *port = tb_xdomain_downstream_port(xd);
-	u32 val[2];
-	int ret;
 
 	memset(&res, 0, sizeof(res));
 	tb_xdp_fill_header(&res.hdr, xd->route, sequence,
 			   LINK_STATE_STATUS_RESPONSE, sizeof(res));
 
-	ret = tb_port_read(port, val, TB_CFG_PORT,
-			   port->cap_phy + LANE_ADP_CS_0, ARRAY_SIZE(val));
-	if (ret)
-		return ret;
-
-	res.slw = (val[0] & LANE_ADP_CS_0_SUPPORTED_WIDTH_MASK) >>
-			LANE_ADP_CS_0_SUPPORTED_WIDTH_SHIFT;
-	res.sls = (val[0] & LANE_ADP_CS_0_SUPPORTED_SPEED_MASK) >>
-			LANE_ADP_CS_0_SUPPORTED_SPEED_SHIFT;
-	res.tls = val[1] & LANE_ADP_CS_1_TARGET_SPEED_MASK;
-	res.tlw = (val[1] & LANE_ADP_CS_1_TARGET_WIDTH_MASK) >>
-			LANE_ADP_CS_1_TARGET_WIDTH_SHIFT;
+	res.slw = slw;
+	res.sls = sls;
+	res.tls = tls;
+	res.tlw = tlw;
 
 	return __tb_xdomain_response(ctl, &res, sizeof(res),
 				     TB_CFG_PKG_XDOMAIN_RESP);
@@ -645,6 +640,32 @@ void tb_unregister_protocol_handler(struct tb_protocol_handler *handler)
 }
 EXPORT_SYMBOL_GPL(tb_unregister_protocol_handler);
 
+static int update_service_properties(struct device *dev, void *data)
+{
+	struct tb_property_dir *root = data;
+	struct tb_service *svc;
+	struct tb_property *p;
+
+	svc = tb_to_service(dev);
+	if (!svc)
+		return 0;
+
+	guard(mutex)(&svc->lock);
+
+	/*
+	 * Replace the static service properties with the dynamic one.
+	 * Typically this is the same but service drivers can add their
+	 * own dynamic properties here too.
+	 */
+	p = tb_property_find(root, svc->key, TB_PROPERTY_TYPE_DIRECTORY);
+	if (!p)
+		return 0;
+	if (svc->local_properties)
+		return tb_property_merge_dir(p->value.dir,
+					     svc->local_properties, false);
+	return 0;
+}
+
 static void update_property_block(struct tb_xdomain *xd)
 {
 	mutex_lock(&xdomain_lock);
@@ -669,6 +690,9 @@ static void update_property_block(struct tb_xdomain *xd)
 		tb_property_add_text(dir, "deviceid", utsname()->nodename);
 		tb_property_add_immediate(dir, "maxhopid", xd->local_max_hopid);
 
+		/* Add service specific dynamic properties */
+		device_for_each_child(&xd->dev, dir, update_service_properties);
+
 		ret = tb_property_format_dir(dir, NULL, 0);
 		if (ret < 0) {
 			dev_warn(&xd->dev, "local property block creation failed\n");
@@ -745,7 +769,7 @@ static void tb_xdp_handle_request(struct work_struct *work)
 
 	mutex_lock(&tb->lock);
 	if (tb->root_switch)
-		uuid = tb->root_switch->uuid;
+		uuid = kmemdup(tb->root_switch->uuid, sizeof(*uuid), GFP_KERNEL);
 	else
 		uuid = NULL;
 	mutex_unlock(&tb->lock);
@@ -804,8 +828,47 @@ static void tb_xdp_handle_request(struct work_struct *work)
 		       route);
 
 		if (xd) {
-			ret = tb_xdp_link_state_status_response(tb, ctl, xd,
-								sequence);
+			struct tb_port *port = tb_xdomain_downstream_port(xd);
+			u8 slw, sls, tls, tlw;
+			u32 val[2];
+
+			/*
+			 * Read the adapter supported and target widths
+			 * and speeds.
+			 */
+			ret = tb_port_read(port, val, TB_CFG_PORT,
+					   port->cap_phy + LANE_ADP_CS_0,
+					   ARRAY_SIZE(val));
+			if (ret)
+				break;
+
+			slw = (val[0] & LANE_ADP_CS_0_SUPPORTED_WIDTH_MASK) >>
+				LANE_ADP_CS_0_SUPPORTED_WIDTH_SHIFT;
+			sls = (val[0] & LANE_ADP_CS_0_SUPPORTED_SPEED_MASK) >>
+				LANE_ADP_CS_0_SUPPORTED_SPEED_SHIFT;
+			tls = val[1] & LANE_ADP_CS_1_TARGET_SPEED_MASK;
+			tlw = (val[1] & LANE_ADP_CS_1_TARGET_WIDTH_MASK) >>
+				LANE_ADP_CS_1_TARGET_WIDTH_SHIFT;
+
+			/*
+			 * When we have higher UUID, we are supposed to
+			 * return ERROR_NOT_READY if the tlw is not yet
+			 * set according to the Inter-Domain spec for
+			 * USB4 v2.
+			 */
+			if (xd->state == XDOMAIN_STATE_BONDING_UUID_HIGH &&
+			    xd->target_link_width &&
+			    xd->target_link_width != tlw) {
+				tb_dbg(tb, "%llx: target link width not yet set %#x != %#x\n",
+				       route, tlw, xd->target_link_width);
+				tb_xdp_error_response(ctl, route, sequence,
+						      ERROR_NOT_READY);
+			} else {
+				tb_dbg(tb, "%llx: replying with target link width set to %#x\n",
+				       route, tlw);
+				ret = tb_xdp_link_state_status_response(tb, ctl,
+					xd, sequence, slw, sls, tls, tlw);
+			}
 		} else {
 			tb_xdp_error_response(ctl, route, sequence,
 					      ERROR_NOT_READY);
@@ -846,6 +909,7 @@ static void tb_xdp_handle_request(struct work_struct *work)
 	}
 
 out:
+	kfree(uuid);
 	kfree(xw->pkg);
 	kfree(xw);
 
@@ -901,6 +965,40 @@ void tb_unregister_service_driver(struct tb_service_driver *drv)
 }
 EXPORT_SYMBOL_GPL(tb_unregister_service_driver);
 
+static int update_xdomain(struct device *dev, void *data)
+{
+	struct tb_xdomain *xd;
+
+	xd = tb_to_xdomain(dev);
+	if (xd) {
+		queue_delayed_work(xd->tb->wq, &xd->properties_changed_work,
+				   msecs_to_jiffies(50));
+	}
+
+	return 0;
+}
+
+/**
+ * tb_service_properties_changed() - Notify the other host about changes
+ * @svc: Service whose properties changed
+ *
+ * Notifies the other host that service properties may have been
+ * changed. This should be called whenever @svc->local_properties is
+ * updated.
+ */
+void tb_service_properties_changed(struct tb_service *svc)
+{
+	struct tb_xdomain *xd = tb_service_parent(svc);
+
+	if (xd->is_unplugged)
+		return;
+
+	scoped_guard(mutex, &xdomain_lock)
+		xdomain_property_block_gen++;
+	update_xdomain(&xd->dev, NULL);
+}
+EXPORT_SYMBOL_GPL(tb_service_properties_changed);
+
 static ssize_t key_show(struct device *dev, struct device_attribute *attr,
 			char *buf)
 {
@@ -1000,10 +1098,11 @@ static void tb_service_release(struct device *dev)
 	struct tb_service *svc = container_of(dev, struct tb_service, dev);
 	struct tb_xdomain *xd = tb_service_parent(svc);
 
-	tb_service_debugfs_remove(svc);
+	tb_property_free_dir(svc->remote_properties);
 	ida_free(&xd->service_ids, svc->id);
 	kfree(svc->key);
 	kfree(svc);
+	tb_xdomain_put(xd);
 }
 
 const struct device_type tb_service_type = {
@@ -1014,6 +1113,24 @@ const struct device_type tb_service_type = {
 };
 EXPORT_SYMBOL_GPL(tb_service_type);
 
+static void update_service(struct tb_service *svc, struct tb_property *property)
+{
+	struct tb_property_dir *dir = property->value.dir;
+
+	guard(mutex)(&svc->lock);
+	tb_property_free_dir(svc->remote_properties);
+	svc->remote_properties = tb_property_copy_dir(dir);
+	kobject_uevent(&svc->dev.kobj, KOBJ_CHANGE);
+}
+
+static void __unregister_service(struct device *dev)
+{
+	struct tb_service *svc = tb_to_service(dev);
+
+	tb_service_debugfs_remove(svc);
+	device_unregister(&svc->dev);
+}
+
 static int remove_missing_service(struct device *dev, void *data)
 {
 	struct tb_xdomain *xd = data;
@@ -1025,7 +1142,7 @@ static int remove_missing_service(struct device *dev, void *data)
 
 	if (!tb_property_find(xd->remote_properties, svc->key,
 			      TB_PROPERTY_TYPE_DIRECTORY))
-		device_unregister(dev);
+		__unregister_service(dev);
 
 	return 0;
 }
@@ -1066,6 +1183,12 @@ static int populate_service(struct tb_service *svc,
 	if (!svc->key)
 		return -ENOMEM;
 
+	svc->remote_properties = tb_property_copy_dir(dir);
+	if (!svc->remote_properties) {
+		kfree(svc->key);
+		return -ENOMEM;
+	}
+
 	return 0;
 }
 
@@ -1090,6 +1213,7 @@ static void enumerate_services(struct tb_xdomain *xd)
 		/* If the service exists already we are fine */
 		dev = device_find_child(&xd->dev, p, find_service);
 		if (dev) {
+			update_service(tb_to_service(dev), p);
 			put_device(dev);
 			continue;
 		}
@@ -1112,12 +1236,14 @@ static void enumerate_services(struct tb_xdomain *xd)
 		svc->id = id;
 		svc->dev.bus = &tb_bus_type;
 		svc->dev.type = &tb_service_type;
-		svc->dev.parent = &xd->dev;
+		svc->dev.parent = get_device(&xd->dev);
+		mutex_init(&svc->lock);
 		dev_set_name(&svc->dev, "%s.%d", dev_name(&xd->dev), svc->id);
 
 		tb_service_debugfs_init(svc);
 
 		if (device_register(&svc->dev)) {
+			tb_service_debugfs_remove(svc);
 			put_device(&svc->dev);
 			break;
 		}
@@ -1931,7 +2057,13 @@ static void tb_xdomain_link_exit(struct tb_xdomain *xd)
 	if (tb_port_get_link_generation(down) >= 4) {
 		down->bonded = false;
 		down->dual_link_port->bonded = false;
-	} else if (xd->link_width > TB_LINK_WIDTH_SINGLE) {
+		return;
+	}
+
+	if (!xd->bonding_possible)
+		return;
+
+	if (xd->link_width > TB_LINK_WIDTH_SINGLE) {
 		/*
 		 * Just return port structures back to way they were and
 		 * update credits. No need to update userspace because
@@ -1988,6 +2120,7 @@ struct tb_xdomain *tb_xdomain_alloc(struct tb *tb, struct device *parent,
 	INIT_DELAYED_WORK(&xd->state_work, tb_xdomain_state_work);
 	INIT_DELAYED_WORK(&xd->properties_changed_work,
 			  tb_xdomain_properties_changed);
+	atomic_set(&xd->ntunnels, 0);
 
 	xd->local_uuid = kmemdup(local_uuid, sizeof(uuid_t), GFP_KERNEL);
 	if (!xd->local_uuid)
@@ -2050,45 +2183,58 @@ void tb_xdomain_add(struct tb_xdomain *xd)
 
 static int unregister_service(struct device *dev, void *data)
 {
-	device_unregister(dev);
+	__unregister_service(dev);
 	return 0;
 }
 
 /**
- * tb_xdomain_remove() - Remove XDomain from the bus
+ * tb_xdomain_remove() - Remove XDomain
  * @xd: XDomain to remove
  *
- * This will stop all ongoing configuration work and remove the XDomain
- * along with any services from the bus. When the last reference to @xd
- * is released the object will be released as well.
+ * This will stop all ongoing configuration work. XDomain is not removed
+ * from the bus if it was added. That needs to be done separately by
+ * calling tb_xdomain_unregister().
+ *
+ * Called with @tb->lock held.
  */
 void tb_xdomain_remove(struct tb_xdomain *xd)
 {
 	tb_xdomain_debugfs_remove(xd);
-
 	stop_handshake(xd);
-
-	device_for_each_child_reverse(&xd->dev, xd, unregister_service);
-
 	tb_xdomain_link_exit(xd);
 
-	/*
-	 * Undo runtime PM here explicitly because it is possible that
-	 * the XDomain was never added to the bus and thus device_del()
-	 * is not called for it (device_del() would handle this otherwise).
-	 */
-	pm_runtime_disable(&xd->dev);
-	pm_runtime_put_noidle(&xd->dev);
-	pm_runtime_set_suspended(&xd->dev);
-
 	if (!device_is_registered(&xd->dev)) {
+		/*
+		 * Undo runtime PM here explicitly because it is
+		 * possible that the XDomain was never added to the bus
+		 * and thus device_del() is not called for it
+		 * (device_del() would handle this otherwise).
+		 */
+		pm_runtime_disable(&xd->dev);
+		pm_runtime_put_noidle(&xd->dev);
+		pm_runtime_set_suspended(&xd->dev);
 		put_device(&xd->dev);
-	} else {
-		dev_info(&xd->dev, "host disconnected\n");
-		device_unregister(&xd->dev);
 	}
 }
 
+/**
+ * tb_xdomain_unregister() - Unregister XDomain
+ * @xd: XDomain to unregister
+ *
+ * This will unregister the XDomain along with any services from the
+ * bus. When the last reference to @xd is released the object will be
+ * released as well.
+ */
+void tb_xdomain_unregister(struct tb_xdomain *xd)
+{
+	lockdep_assert_not_held(&xd->tb->lock);
+
+	device_for_each_child_reverse(&xd->dev, xd, unregister_service);
+
+	dev_info(&xd->dev, "host disconnected\n");
+	device_unregister(&xd->dev);
+}
+
 /**
  * tb_xdomain_lane_bonding_enable() - Enable lane bonding on XDomain
  * @xd: XDomain connection
@@ -2265,9 +2411,15 @@ int tb_xdomain_enable_paths(struct tb_xdomain *xd, int transmit_path,
 			    int transmit_ring, int receive_path,
 			    int receive_ring)
 {
-	return tb_domain_approve_xdomain_paths(xd->tb, xd, transmit_path,
-					       transmit_ring, receive_path,
-					       receive_ring);
+	int ret;
+
+	ret = tb_domain_approve_xdomain_paths(xd->tb, xd, transmit_path,
+					      transmit_ring, receive_path,
+					      receive_ring);
+	if (ret)
+		return ret;
+	atomic_inc(&xd->ntunnels);
+	return 0;
 }
 EXPORT_SYMBOL_GPL(tb_xdomain_enable_paths);
 
@@ -2290,9 +2442,15 @@ int tb_xdomain_disable_paths(struct tb_xdomain *xd, int transmit_path,
 			     int transmit_ring, int receive_path,
 			     int receive_ring)
 {
-	return tb_domain_disconnect_xdomain_paths(xd->tb, xd, transmit_path,
-						  transmit_ring, receive_path,
-						  receive_ring);
+	int ret;
+
+	ret = tb_domain_disconnect_xdomain_paths(xd->tb, xd, transmit_path,
+						 transmit_ring, receive_path,
+						 receive_ring);
+	if (ret)
+		return ret;
+	atomic_dec(&xd->ntunnels);
+	return 0;
 }
 EXPORT_SYMBOL_GPL(tb_xdomain_disable_paths);
 
@@ -2308,6 +2466,9 @@ static struct tb_xdomain *switch_find_xdomain(struct tb_switch *sw,
 {
 	struct tb_port *port;
 
+	if (!sw)
+		return NULL;
+
 	tb_switch_for_each_port(sw, port) {
 		struct tb_xdomain *xd;
 
@@ -2470,19 +2631,6 @@ bool tb_xdomain_handle_request(struct tb *tb, enum tb_cfg_pkg_type type,
 	return ret > 0;
 }
 
-static int update_xdomain(struct device *dev, void *data)
-{
-	struct tb_xdomain *xd;
-
-	xd = tb_to_xdomain(dev);
-	if (xd) {
-		queue_delayed_work(xd->tb->wq, &xd->properties_changed_work,
-				   msecs_to_jiffies(50));
-	}
-
-	return 0;
-}
-
 static void update_all_xdomains(void)
 {
 	bus_for_each_dev(&tb_bus_type, NULL, NULL, update_xdomain);
diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig
index 149f3d53b7608..df6832a4c237c 100644
--- a/drivers/tty/Kconfig
+++ b/drivers/tty/Kconfig
@@ -231,21 +231,12 @@ config MOXA_SMARTIO
 	  This driver can also be built as a module. The module will be called
 	  mxser. If you want to do that, say M here.
 
-config SYNCLINK_GT
-	tristate "SyncLink GT/AC support"
-	depends on SERIAL_NONSTANDARD && PCI
-	depends on BROKEN
-	help
-	  Support for SyncLink GT and SyncLink AC families of
-	  synchronous and asynchronous serial adapters
-	  manufactured by Microgate Systems, Ltd. (www.microgate.com)
-
 config N_HDLC
 	tristate "HDLC line discipline support"
 	depends on SERIAL_NONSTANDARD
 	help
 	  Allows synchronous HDLC communications with tty device drivers that
-	  support synchronous HDLC such as the Microgate SyncLink adapter.
+	  support synchronous HDLC.
 
 	  This driver can be built as a module ( = code which can be
 	  inserted in and removed from the running kernel whenever you want).
diff --git a/drivers/tty/Makefile b/drivers/tty/Makefile
index 07aca5184a55d..8ca1a0a2229fa 100644
--- a/drivers/tty/Makefile
+++ b/drivers/tty/Makefile
@@ -21,7 +21,6 @@ obj-$(CONFIG_MOXA_INTELLIO)	+= moxa.o
 obj-$(CONFIG_MOXA_SMARTIO)	+= mxser.o
 obj-$(CONFIG_NOZOMI)		+= nozomi.o
 obj-$(CONFIG_NULL_TTY)	        += ttynull.o
-obj-$(CONFIG_SYNCLINK_GT)	+= synclink_gt.o
 obj-$(CONFIG_PPC_EPAPR_HV_BYTECHAN) += ehv_bytechan.o
 obj-$(CONFIG_GOLDFISH_TTY)	+= goldfish.o
 obj-$(CONFIG_MIPS_EJTAG_FDC_TTY) += mips_ejtag_fdc.o
diff --git a/drivers/tty/n_hdlc.c b/drivers/tty/n_hdlc.c
index 98eefa2cede4a..895b850a59503 100644
--- a/drivers/tty/n_hdlc.c
+++ b/drivers/tty/n_hdlc.c
@@ -4,8 +4,6 @@
  * Written by Paul Fulghum paulkf@microgate.com
  * for Microgate Corporation
  *
- * Microgate and SyncLink are registered trademarks of Microgate Corporation
- *
  * Adapted from ppp.c, written by Michael Callahan <callahan@maths.ox.ac.uk>,
  *	Al Longyear <longyear@netcom.com>,
  *	Paul Mackerras <Paul.Mackerras@cs.anu.edu.au>
@@ -54,11 +52,6 @@
  * this line discipline (or another line discipline that is frame
  * oriented such as N_PPP).
  *
- * The SyncLink driver (synclink.c) implements both asynchronous
- * (using standard line discipline N_TTY) and synchronous HDLC
- * (using N_HDLC) communications, with the latter using the above
- * conventions.
- *
  * This implementation is very basic and does not maintain
  * any statistics. The main point is to enforce the raw data
  * and frame orientation of HDLC communications.
diff --git a/drivers/tty/serial/8250/8250_dw.c b/drivers/tty/serial/8250/8250_dw.c
index 94beadb4024df..55e40c10f46a2 100644
--- a/drivers/tty/serial/8250/8250_dw.c
+++ b/drivers/tty/serial/8250/8250_dw.c
@@ -34,22 +34,11 @@
 
 #include "8250_dwlib.h"
 
-/* Offsets for the DesignWare specific registers */
-#define DW_UART_USR	0x1f /* UART Status Register */
-#define DW_UART_DMASA	0xa8 /* DMA Software Ack */
-
 #define OCTEON_UART_USR	0x27 /* UART Status Register */
 
 #define RZN1_UART_TDMACR 0x10c /* DMA Control Register Transmit Mode */
 #define RZN1_UART_RDMACR 0x110 /* DMA Control Register Receive Mode */
 
-/* DesignWare specific register fields */
-#define DW_UART_IIR_IID			GENMASK(3, 0)
-
-#define DW_UART_MCR_SIRE		BIT(6)
-
-#define DW_UART_USR_BUSY		BIT(0)
-
 /* Renesas specific register fields */
 #define RZN1_UART_xDMACR_DMA_EN		BIT(0)
 #define RZN1_UART_xDMACR_1_WORD_BURST	(0 << 1)
@@ -948,7 +937,15 @@ static const struct dw8250_platform_data dw8250_armada_38x_data = {
 
 static const struct dw8250_platform_data dw8250_renesas_rzn1_data = {
 	.usr_reg = DW_UART_USR,
-	.cpr_value = 0x00012f32,
+	.cpr_value = FIELD_PREP_CONST(DW_UART_CPR_ABP_DATA_WIDTH, 2) |
+		     DW_UART_CPR_AFCE_MODE |
+		     DW_UART_CPR_THRE_MODE |
+		     DW_UART_CPR_ADDITIONAL_FEATURES |
+		     DW_UART_CPR_FIFO_ACCESS |
+		     DW_UART_CPR_FIFO_STAT |
+		     DW_UART_CPR_SHADOW |
+		     DW_UART_CPR_DMA_EXTRA |
+		     DW_UART_CPR_FIFO_MODE_FROM_SIZE(16),
 	.quirks = DW_UART_QUIRK_CPR_VALUE | DW_UART_QUIRK_IS_DMA_FC,
 };
 
@@ -962,6 +959,15 @@ static const struct dw8250_platform_data dw8250_intc10ee = {
 	.quirks = DW_UART_QUIRK_IER_KICK,
 };
 
+static const struct dw8250_platform_data dw8250_ultrarisc_dp1000_data = {
+	.usr_reg = DW_UART_USR,
+	.cpr_value = FIELD_PREP_CONST(DW_UART_CPR_ABP_DATA_WIDTH, 2) |
+		     DW_UART_CPR_THRE_MODE |
+		     DW_UART_CPR_DMA_EXTRA |
+		     DW_UART_CPR_FIFO_MODE_FROM_SIZE(32),
+	.quirks = DW_UART_QUIRK_CPR_VALUE,
+};
+
 static const struct of_device_id dw8250_of_match[] = {
 	{ .compatible = "snps,dw-apb-uart", .data = &dw8250_dw_apb },
 	{ .compatible = "cavium,octeon-3860-uart", .data = &dw8250_octeon_3860_data },
@@ -969,6 +975,7 @@ static const struct of_device_id dw8250_of_match[] = {
 	{ .compatible = "renesas,rzn1-uart", .data = &dw8250_renesas_rzn1_data },
 	{ .compatible = "sophgo,sg2044-uart", .data = &dw8250_skip_set_rate_data },
 	{ .compatible = "starfive,jh7100-uart", .data = &dw8250_skip_set_rate_data },
+	{ .compatible = "ultrarisc,dp1000-uart", .data = &dw8250_ultrarisc_dp1000_data },
 	{ /* Sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, dw8250_of_match);
diff --git a/drivers/tty/serial/8250/8250_dwlib.c b/drivers/tty/serial/8250/8250_dwlib.c
index b055d89cfb393..8859e66d2d71c 100644
--- a/drivers/tty/serial/8250/8250_dwlib.c
+++ b/drivers/tty/serial/8250/8250_dwlib.c
@@ -13,55 +13,6 @@
 
 #include "8250_dwlib.h"
 
-/* Offsets for the DesignWare specific registers */
-#define DW_UART_TCR	0xac /* Transceiver Control Register (RS485) */
-#define DW_UART_DE_EN	0xb0 /* Driver Output Enable Register */
-#define DW_UART_RE_EN	0xb4 /* Receiver Output Enable Register */
-#define DW_UART_DLF	0xc0 /* Divisor Latch Fraction Register */
-#define DW_UART_RAR	0xc4 /* Receive Address Register */
-#define DW_UART_TAR	0xc8 /* Transmit Address Register */
-#define DW_UART_LCR_EXT	0xcc /* Line Extended Control Register */
-#define DW_UART_CPR	0xf4 /* Component Parameter Register */
-#define DW_UART_UCV	0xf8 /* UART Component Version */
-
-/* Receive / Transmit Address Register bits */
-#define DW_UART_ADDR_MASK		GENMASK(7, 0)
-
-/* Line Status Register bits */
-#define DW_UART_LSR_ADDR_RCVD		BIT(8)
-
-/* Transceiver Control Register bits */
-#define DW_UART_TCR_RS485_EN		BIT(0)
-#define DW_UART_TCR_RE_POL		BIT(1)
-#define DW_UART_TCR_DE_POL		BIT(2)
-#define DW_UART_TCR_XFER_MODE		GENMASK(4, 3)
-#define DW_UART_TCR_XFER_MODE_DE_DURING_RE	FIELD_PREP(DW_UART_TCR_XFER_MODE, 0)
-#define DW_UART_TCR_XFER_MODE_SW_DE_OR_RE	FIELD_PREP(DW_UART_TCR_XFER_MODE, 1)
-#define DW_UART_TCR_XFER_MODE_DE_OR_RE		FIELD_PREP(DW_UART_TCR_XFER_MODE, 2)
-
-/* Line Extended Control Register bits */
-#define DW_UART_LCR_EXT_DLS_E		BIT(0)
-#define DW_UART_LCR_EXT_ADDR_MATCH	BIT(1)
-#define DW_UART_LCR_EXT_SEND_ADDR	BIT(2)
-#define DW_UART_LCR_EXT_TRANSMIT_MODE	BIT(3)
-
-/* Component Parameter Register bits */
-#define DW_UART_CPR_ABP_DATA_WIDTH	GENMASK(1, 0)
-#define DW_UART_CPR_AFCE_MODE		BIT(4)
-#define DW_UART_CPR_THRE_MODE		BIT(5)
-#define DW_UART_CPR_SIR_MODE		BIT(6)
-#define DW_UART_CPR_SIR_LP_MODE		BIT(7)
-#define DW_UART_CPR_ADDITIONAL_FEATURES	BIT(8)
-#define DW_UART_CPR_FIFO_ACCESS		BIT(9)
-#define DW_UART_CPR_FIFO_STAT		BIT(10)
-#define DW_UART_CPR_SHADOW		BIT(11)
-#define DW_UART_CPR_ENCODED_PARMS	BIT(12)
-#define DW_UART_CPR_DMA_EXTRA		BIT(13)
-#define DW_UART_CPR_FIFO_MODE		GENMASK(23, 16)
-
-/* Helper for FIFO size calculation */
-#define DW_UART_CPR_FIFO_SIZE(a)	(FIELD_GET(DW_UART_CPR_FIFO_MODE, (a)) * 16)
-
 /*
  * divisor = div(I) + div(F)
  * "I" means integer, "F" means fractional
diff --git a/drivers/tty/serial/8250/8250_dwlib.h b/drivers/tty/serial/8250/8250_dwlib.h
index 7dd2a8e7b7808..1fe52332e774b 100644
--- a/drivers/tty/serial/8250/8250_dwlib.h
+++ b/drivers/tty/serial/8250/8250_dwlib.h
@@ -1,11 +1,82 @@
 /* SPDX-License-Identifier: GPL-2.0+ */
 /* Synopsys DesignWare 8250 library header file. */
 
+#ifndef _SERIAL_8250_DWLIB_H_
+#define _SERIAL_8250_DWLIB_H_
+
+#include <linux/bitfield.h>
+#include <linux/bits.h>
+#include <linux/build_bug.h>
 #include <linux/io.h>
 #include <linux/types.h>
 
 #include "8250.h"
 
+/* Offsets for the DesignWare specific registers */
+#define DW_UART_USR	0x1f /* UART Status Register */
+#define DW_UART_DMASA	0xa8 /* DMA Software Ack */
+#define DW_UART_TCR	0xac /* Transceiver Control Register (RS485) */
+#define DW_UART_DE_EN	0xb0 /* Driver Output Enable Register */
+#define DW_UART_RE_EN	0xb4 /* Receiver Output Enable Register */
+#define DW_UART_DLF	0xc0 /* Divisor Latch Fraction Register */
+#define DW_UART_RAR	0xc4 /* Receive Address Register */
+#define DW_UART_TAR	0xc8 /* Transmit Address Register */
+#define DW_UART_LCR_EXT	0xcc /* Line Extended Control Register */
+#define DW_UART_CPR	0xf4 /* Component Parameter Register */
+#define DW_UART_UCV	0xf8 /* UART Component Version */
+
+/* Interrupt ID Register bits */
+#define DW_UART_IIR_IID			GENMASK(3, 0)
+
+/* Modem Control Register bits */
+#define DW_UART_MCR_SIRE		BIT(6)
+
+/* Line Status Register bits */
+#define DW_UART_LSR_ADDR_RCVD		BIT(8)
+
+/* UART Status Register bits */
+#define DW_UART_USR_BUSY		BIT(0)
+
+/* Transceiver Control Register bits */
+#define DW_UART_TCR_RS485_EN		BIT(0)
+#define DW_UART_TCR_RE_POL		BIT(1)
+#define DW_UART_TCR_DE_POL		BIT(2)
+#define DW_UART_TCR_XFER_MODE		GENMASK(4, 3)
+#define DW_UART_TCR_XFER_MODE_DE_DURING_RE	FIELD_PREP(DW_UART_TCR_XFER_MODE, 0)
+#define DW_UART_TCR_XFER_MODE_SW_DE_OR_RE	FIELD_PREP(DW_UART_TCR_XFER_MODE, 1)
+#define DW_UART_TCR_XFER_MODE_DE_OR_RE		FIELD_PREP(DW_UART_TCR_XFER_MODE, 2)
+
+/* Receive / Transmit Address Register bits */
+#define DW_UART_ADDR_MASK		GENMASK(7, 0)
+
+/* Line Extended Control Register bits */
+#define DW_UART_LCR_EXT_DLS_E		BIT(0)
+#define DW_UART_LCR_EXT_ADDR_MATCH	BIT(1)
+#define DW_UART_LCR_EXT_SEND_ADDR	BIT(2)
+#define DW_UART_LCR_EXT_TRANSMIT_MODE	BIT(3)
+
+/* Component Parameter Register bits */
+#define DW_UART_CPR_ABP_DATA_WIDTH	GENMASK(1, 0)
+#define DW_UART_CPR_AFCE_MODE		BIT(4)
+#define DW_UART_CPR_THRE_MODE		BIT(5)
+#define DW_UART_CPR_SIR_MODE		BIT(6)
+#define DW_UART_CPR_SIR_LP_MODE		BIT(7)
+#define DW_UART_CPR_ADDITIONAL_FEATURES	BIT(8)
+#define DW_UART_CPR_FIFO_ACCESS		BIT(9)
+#define DW_UART_CPR_FIFO_STAT		BIT(10)
+#define DW_UART_CPR_SHADOW		BIT(11)
+#define DW_UART_CPR_ENCODED_PARMS	BIT(12)
+#define DW_UART_CPR_DMA_EXTRA		BIT(13)
+#define DW_UART_CPR_FIFO_MODE		GENMASK(23, 16)
+
+/* Helpers for FIFO size calculation */
+#define DW_UART_CPR_FIFO_SIZE(a)	(FIELD_GET(DW_UART_CPR_FIFO_MODE, (a)) * 16)
+#define DW_UART_CPR_FIFO_MODE_FROM_SIZE(size)			\
+	(FIELD_PREP_CONST(DW_UART_CPR_FIFO_MODE,		\
+			  BUILD_BUG_ON_ZERO((size) > 2048) +	\
+			  BUILD_BUG_ON_ZERO((size) % 16) +	\
+			  ((size) / 16)))
+
 struct dw8250_port_data {
 	/* Port properties */
 	int			line;
@@ -38,3 +109,5 @@ static inline void dw8250_writel_ext(struct uart_port *p, int offset, u32 reg)
 	else
 		writel(reg, p->membase + offset);
 }
+
+#endif /* _SERIAL_8250_DWLIB_H_ */
diff --git a/drivers/tty/serial/8250/8250_mtk.c b/drivers/tty/serial/8250/8250_mtk.c
index 5875a7b9b4b10..e6a56cf54ae0c 100644
--- a/drivers/tty/serial/8250/8250_mtk.c
+++ b/drivers/tty/serial/8250/8250_mtk.c
@@ -19,6 +19,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/tty.h>
 #include <linux/tty_flip.h>
+#include <linux/units.h>
 
 #include "8250.h"
 
@@ -521,6 +522,7 @@ static int mtk8250_probe(struct platform_device *pdev)
 	struct mtk8250_data *data;
 	struct resource *regs;
 	int irq, err;
+	struct fwnode_handle *fwnode = dev_fwnode(&pdev->dev);
 
 	irq = platform_get_irq(pdev, 0);
 	if (irq < 0)
@@ -543,12 +545,13 @@ static int mtk8250_probe(struct platform_device *pdev)
 
 	data->clk_count = 0;
 
-	if (pdev->dev.of_node) {
+	if (is_of_node(fwnode)) {
 		err = mtk8250_probe_of(pdev, &uart.port, data);
 		if (err)
 			return err;
-	} else
+	} else if (!fwnode) {
 		return -ENODEV;
+	}
 
 	spin_lock_init(&uart.port.lock);
 	uart.port.mapbase = regs->start;
@@ -564,14 +567,18 @@ static int mtk8250_probe(struct platform_device *pdev)
 	uart.port.startup = mtk8250_startup;
 	uart.port.set_termios = mtk8250_set_termios;
 	uart.port.uartclk = clk_get_rate(data->uart_clk);
+	if (!uart.port.uartclk)
+		uart.port.uartclk = 26 * HZ_PER_MHZ;
 #ifdef CONFIG_SERIAL_8250_DMA
 	if (data->dma)
 		uart.dma = data->dma;
 #endif
 
-	/* Disable Rate Fix function */
-	writel(0x0, uart.port.membase +
+	if (is_of_node(fwnode)) {
+		/* Disable Rate Fix function */
+		writel(0x0, uart.port.membase +
 			(MTK_UART_RATE_FIX << uart.port.regshift));
+	}
 
 	platform_set_drvdata(pdev, data);
 
@@ -649,11 +656,19 @@ static const struct of_device_id mtk8250_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, mtk8250_of_match);
 
+static const struct acpi_device_id mtk8250_acpi_match[] = {
+	{ "MTKI0511" },
+	{ "NVDA0240" },
+	{}
+};
+MODULE_DEVICE_TABLE(acpi, mtk8250_acpi_match);
+
 static struct platform_driver mtk8250_platform_driver = {
 	.driver = {
 		.name		= "mt6577-uart",
 		.pm		= &mtk8250_pm_ops,
 		.of_match_table	= mtk8250_of_match,
+		.acpi_match_table = mtk8250_acpi_match,
 	},
 	.probe			= mtk8250_probe,
 	.remove			= mtk8250_remove,
diff --git a/drivers/tty/serial/8250/8250_pci.c b/drivers/tty/serial/8250/8250_pci.c
index 2fbd8f2603b5f..3e5bc9e8d2694 100644
--- a/drivers/tty/serial/8250/8250_pci.c
+++ b/drivers/tty/serial/8250/8250_pci.c
@@ -143,16 +143,15 @@ struct serial_private {
 #define PCIE_DEVICE_ID_AX99100		0x9100
 
 static const struct pci_device_id pci_use_msi[] = {
-	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9900,
-			 0xA000, 0x1000) },
-	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9912,
-			 0xA000, 0x1000) },
-	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9922,
-			 0xA000, 0x1000) },
-	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_ASIX, PCI_DEVICE_ID_ASIX_AX99100,
-			 0xA000, 0x1000) },
-	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_HP_3PAR, PCI_DEVICE_ID_HPE_PCI_SERIAL,
-			 PCI_ANY_ID, PCI_ANY_ID) },
+	{ PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9900,
+			  0xA000, 0x1000) },
+	{ PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9912,
+			  0xA000, 0x1000) },
+	{ PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9922,
+			  0xA000, 0x1000) },
+	{ PCI_VDEVICE_SUB(ASIX, PCI_DEVICE_ID_ASIX_AX99100,
+			  0xA000, 0x1000) },
+	{ PCI_VDEVICE(HP_3PAR, PCI_DEVICE_ID_HPE_PCI_SERIAL) },
 	{ PCI_DEVICE_SUB(PCIE_VENDOR_ID_ASIX, PCIE_DEVICE_ID_AX99100,
 			 0xA000, 0x1000) },
 	{ }
@@ -4040,42 +4039,42 @@ static const struct pci_device_id blacklist[] = {
 
 	/* multi-io cards handled by parport_serial */
 	/* WCH CH353 2S1P */
-	{ PCI_VDEVICE(WCHCN, 0x7053), REPORT_CONFIG(PARPORT_SERIAL), },
+	{ PCI_VDEVICE(WCHCN, 0x7053), .driver_data = REPORT_CONFIG(PARPORT_SERIAL), },
 	/* WCH CH353 1S1P */
-	{ PCI_VDEVICE(WCHCN, 0x5053), REPORT_CONFIG(PARPORT_SERIAL), },
+	{ PCI_VDEVICE(WCHCN, 0x5053), .driver_data = REPORT_CONFIG(PARPORT_SERIAL), },
 	/* WCH CH382 2S1P */
-	{ PCI_VDEVICE(WCHIC, 0x3250), REPORT_CONFIG(PARPORT_SERIAL), },
+	{ PCI_VDEVICE(WCHIC, 0x3250), .driver_data = REPORT_CONFIG(PARPORT_SERIAL), },
 
 	/* Intel platforms with MID UART */
-	{ PCI_VDEVICE(INTEL, 0x081b), REPORT_8250_CONFIG(MID), },
-	{ PCI_VDEVICE(INTEL, 0x081c), REPORT_8250_CONFIG(MID), },
-	{ PCI_VDEVICE(INTEL, 0x081d), REPORT_8250_CONFIG(MID), },
-	{ PCI_VDEVICE(INTEL, 0x1191), REPORT_8250_CONFIG(MID), },
-	{ PCI_VDEVICE(INTEL, 0x18d8), REPORT_8250_CONFIG(MID), },
-	{ PCI_VDEVICE(INTEL, 0x19d8), REPORT_8250_CONFIG(MID), },
+	{ PCI_VDEVICE(INTEL, 0x081b), .driver_data = REPORT_8250_CONFIG(MID), },
+	{ PCI_VDEVICE(INTEL, 0x081c), .driver_data = REPORT_8250_CONFIG(MID), },
+	{ PCI_VDEVICE(INTEL, 0x081d), .driver_data = REPORT_8250_CONFIG(MID), },
+	{ PCI_VDEVICE(INTEL, 0x1191), .driver_data = REPORT_8250_CONFIG(MID), },
+	{ PCI_VDEVICE(INTEL, 0x18d8), .driver_data = REPORT_8250_CONFIG(MID), },
+	{ PCI_VDEVICE(INTEL, 0x19d8), .driver_data = REPORT_8250_CONFIG(MID), },
 
 	/* Intel platforms with DesignWare UART */
-	{ PCI_VDEVICE(INTEL, 0x0936), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x0f0a), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x0f0c), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x228a), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x228c), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x4b96), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x4b97), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x4b98), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x4b99), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x4b9a), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x4b9b), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x9ce3), REPORT_8250_CONFIG(LPSS), },
-	{ PCI_VDEVICE(INTEL, 0x9ce4), REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x0936), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x0f0a), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x0f0c), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x228a), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x228c), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x4b96), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x4b97), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x4b98), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x4b99), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x4b9a), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x4b9b), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x9ce3), .driver_data = REPORT_8250_CONFIG(LPSS), },
+	{ PCI_VDEVICE(INTEL, 0x9ce4), .driver_data = REPORT_8250_CONFIG(LPSS), },
 
 	/* Exar devices */
-	{ PCI_VDEVICE(EXAR, PCI_ANY_ID), REPORT_8250_CONFIG(EXAR), },
-	{ PCI_VDEVICE(COMMTECH, PCI_ANY_ID), REPORT_8250_CONFIG(EXAR), },
+	{ PCI_VDEVICE(EXAR, PCI_ANY_ID), .driver_data = REPORT_8250_CONFIG(EXAR), },
+	{ PCI_VDEVICE(COMMTECH, PCI_ANY_ID), .driver_data = REPORT_8250_CONFIG(EXAR), },
 
 	/* Pericom devices */
-	{ PCI_VDEVICE(PERICOM, PCI_ANY_ID), REPORT_8250_CONFIG(PERICOM), },
-	{ PCI_VDEVICE(ACCESSIO, PCI_ANY_ID), REPORT_8250_CONFIG(PERICOM), },
+	{ PCI_VDEVICE(PERICOM, PCI_ANY_ID), .driver_data = REPORT_8250_CONFIG(PERICOM), },
+	{ PCI_VDEVICE(ACCESSIO, PCI_ANY_ID), .driver_data = REPORT_8250_CONFIG(PERICOM), },
 
 	/* End of the black list */
 	{ }
@@ -4448,713 +4447,753 @@ static SIMPLE_DEV_PM_OPS(pciserial_pm_ops, pciserial_suspend_one,
 			 pciserial_resume_one);
 
 static const struct pci_device_id serial_pci_tbl[] = {
-	{	PCI_VENDOR_ID_ADVANTECH, PCI_DEVICE_ID_ADVANTECH_PCI1600,
-		PCI_DEVICE_ID_ADVANTECH_PCI1600_1611, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	/* Advantech use PCI_DEVICE_ID_ADVANTECH_PCI3620 (0x3620) as 'PCI_SUBVENDOR_ID' */
-	{	PCI_VENDOR_ID_ADVANTECH, PCI_DEVICE_ID_ADVANTECH_PCI3620,
-		PCI_DEVICE_ID_ADVANTECH_PCI3620, 0x0001, 0, 0,
-		pbn_b2_8_921600 },
-	/* Advantech also use 0x3618 and 0xf618 */
-	{	PCI_VENDOR_ID_ADVANTECH, PCI_DEVICE_ID_ADVANTECH_PCI3618,
-		PCI_DEVICE_ID_ADVANTECH_PCI3618, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_ADVANTECH, PCI_DEVICE_ID_ADVANTECH_PCIf618,
-		PCI_DEVICE_ID_ADVANTECH_PCI3618, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V960,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_232, 0, 0,
-		pbn_b1_8_1382400 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V960,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH4_232, 0, 0,
-		pbn_b1_4_1382400 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V960,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH2_232, 0, 0,
-		pbn_b1_2_1382400 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_232, 0, 0,
-		pbn_b1_8_1382400 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH4_232, 0, 0,
-		pbn_b1_4_1382400 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH2_232, 0, 0,
-		pbn_b1_2_1382400 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_485, 0, 0,
-		pbn_b1_8_921600 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_485_4_4, 0, 0,
-		pbn_b1_8_921600 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH4_485, 0, 0,
-		pbn_b1_4_921600 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH4_485_2_2, 0, 0,
-		pbn_b1_4_921600 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH2_485, 0, 0,
-		pbn_b1_2_921600 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_485_2_6, 0, 0,
-		pbn_b1_8_921600 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH081101V1, 0, 0,
-		pbn_b1_8_921600 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH041101V1, 0, 0,
-		pbn_b1_4_921600 },
-	{	PCI_VENDOR_ID_V3, PCI_DEVICE_ID_V3_V351,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_BH2_20MHZ, 0, 0,
-		pbn_b1_2_1250000 },
-	{	PCI_VENDOR_ID_OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_TITAN_2, 0, 0,
-		pbn_b0_2_1843200 },
-	{	PCI_VENDOR_ID_OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954,
-		PCI_SUBVENDOR_ID_CONNECT_TECH,
-		PCI_SUBDEVICE_ID_CONNECT_TECH_TITAN_4, 0, 0,
-		pbn_b0_4_1843200 },
-	{	PCI_VENDOR_ID_OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954,
-		PCI_VENDOR_ID_AFAVLAB,
-		PCI_SUBDEVICE_ID_AFAVLAB_P061, 0, 0,
-		pbn_b0_4_1152000 },
-	{	PCI_VENDOR_ID_SEALEVEL, PCI_DEVICE_ID_SEALEVEL_U530,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_1_115200 },
-	{	PCI_VENDOR_ID_SEALEVEL, PCI_DEVICE_ID_SEALEVEL_UCOMM2,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_2_115200 },
-	{	PCI_VENDOR_ID_SEALEVEL, PCI_DEVICE_ID_SEALEVEL_UCOMM422,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_4_115200 },
-	{	PCI_VENDOR_ID_SEALEVEL, PCI_DEVICE_ID_SEALEVEL_UCOMM232,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_2_115200 },
-	{	PCI_VENDOR_ID_SEALEVEL, PCI_DEVICE_ID_SEALEVEL_COMM4,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_4_115200 },
-	{	PCI_VENDOR_ID_SEALEVEL, PCI_DEVICE_ID_SEALEVEL_COMM8,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_8_115200 },
-	{	PCI_VENDOR_ID_SEALEVEL, PCI_DEVICE_ID_SEALEVEL_7803,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_8_460800 },
-	{	PCI_VENDOR_ID_SEALEVEL, PCI_DEVICE_ID_SEALEVEL_UCOMM8,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_8_115200 },
-
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_GTEK_SERIAL2,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_2_115200 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_SPCOM200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_2_921600 },
-	/*
-	 * VScom SPCOM800, from sl@s.pl
-	 */
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_SPCOM800,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_8_921600 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_1077,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_4_921600 },
-	/* Unknown card - subdevice 0x1584 */
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_VENDOR_ID_PLX,
-		PCI_SUBDEVICE_ID_UNKNOWN_0x1584, 0, 0,
-		pbn_b2_4_115200 },
-	/* Unknown card - subdevice 0x1588 */
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_VENDOR_ID_PLX,
-		PCI_SUBDEVICE_ID_UNKNOWN_0x1588, 0, 0,
-		pbn_b2_8_115200 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_SUBVENDOR_ID_KEYSPAN,
-		PCI_SUBDEVICE_ID_KEYSPAN_SX2, 0, 0,
-		pbn_panacom },
-	{	PCI_VENDOR_ID_PANACOM, PCI_DEVICE_ID_PANACOM_QUADMODEM,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_panacom4 },
-	{	PCI_VENDOR_ID_PANACOM, PCI_DEVICE_ID_PANACOM_DUALMODEM,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_panacom2 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9030,
-		PCI_VENDOR_ID_ESDGMBH,
-		PCI_DEVICE_ID_ESDGMBH_CPCIASIO4, 0, 0,
-		pbn_b2_4_115200 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_SUBVENDOR_ID_CHASE_PCIFAST,
-		PCI_SUBDEVICE_ID_CHASE_PCIFAST4, 0, 0,
-		pbn_b2_4_460800 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_SUBVENDOR_ID_CHASE_PCIFAST,
-		PCI_SUBDEVICE_ID_CHASE_PCIFAST8, 0, 0,
-		pbn_b2_8_460800 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_SUBVENDOR_ID_CHASE_PCIFAST,
-		PCI_SUBDEVICE_ID_CHASE_PCIFAST16, 0, 0,
-		pbn_b2_16_460800 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_SUBVENDOR_ID_CHASE_PCIFAST,
-		PCI_SUBDEVICE_ID_CHASE_PCIFAST16FMC, 0, 0,
-		pbn_b2_16_460800 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_SUBVENDOR_ID_CHASE_PCIRAS,
-		PCI_SUBDEVICE_ID_CHASE_PCIRAS4, 0, 0,
-		pbn_b2_4_460800 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_SUBVENDOR_ID_CHASE_PCIRAS,
-		PCI_SUBDEVICE_ID_CHASE_PCIRAS8, 0, 0,
-		pbn_b2_8_460800 },
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9050,
-		PCI_SUBVENDOR_ID_EXSYS,
-		PCI_SUBDEVICE_ID_EXSYS_4055, 0, 0,
-		pbn_b2_4_115200 },
-	/*
-	 * Megawolf Romulus PCI Serial Card, from Mike Hudson
-	 * (Exoray@isys.ca)
-	 */
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_ROMULUS,
-		0x10b5, 0x106a, 0, 0,
-		pbn_plx_romulus },
+	{
+		PCI_VDEVICE_SUB(ADVANTECH, PCI_DEVICE_ID_ADVANTECH_PCI1600,
+				PCI_DEVICE_ID_ADVANTECH_PCI1600_1611, PCI_ANY_ID),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		/* Advantech use PCI_DEVICE_ID_ADVANTECH_PCI3620 (0x3620) as 'PCI_SUBVENDOR_ID' */
+		PCI_VDEVICE_SUB(ADVANTECH, PCI_DEVICE_ID_ADVANTECH_PCI3620,
+				PCI_DEVICE_ID_ADVANTECH_PCI3620, 0x0001),
+		.driver_data = pbn_b2_8_921600,
+	}, {
+		/* Advantech also use 0x3618 and 0xf618 */
+		PCI_VDEVICE_SUB(ADVANTECH, PCI_DEVICE_ID_ADVANTECH_PCI3618,
+				PCI_DEVICE_ID_ADVANTECH_PCI3618, PCI_ANY_ID),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(ADVANTECH, PCI_DEVICE_ID_ADVANTECH_PCIf618,
+				PCI_DEVICE_ID_ADVANTECH_PCI3618, PCI_ANY_ID),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V960,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_232),
+		.driver_data = pbn_b1_8_1382400,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V960,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH4_232),
+		.driver_data = pbn_b1_4_1382400,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V960,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH2_232),
+		.driver_data = pbn_b1_2_1382400,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_232),
+		.driver_data = pbn_b1_8_1382400,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH4_232),
+		.driver_data = pbn_b1_4_1382400,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH2_232),
+		.driver_data = pbn_b1_2_1382400,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_485),
+		.driver_data = pbn_b1_8_921600,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_485_4_4),
+		.driver_data = pbn_b1_8_921600,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH4_485),
+		.driver_data = pbn_b1_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH4_485_2_2),
+		.driver_data = pbn_b1_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH2_485),
+		.driver_data = pbn_b1_2_921600,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH8_485_2_6),
+		.driver_data = pbn_b1_8_921600,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH081101V1),
+		.driver_data = pbn_b1_8_921600,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH041101V1),
+		.driver_data = pbn_b1_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(V3, PCI_DEVICE_ID_V3_V351,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_BH2_20MHZ),
+		.driver_data = pbn_b1_2_1250000,
+	}, {
+		PCI_VDEVICE_SUB(OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_TITAN_2),
+		.driver_data = pbn_b0_2_1843200,
+	}, {
+		PCI_VDEVICE_SUB(OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954,
+				PCI_SUBVENDOR_ID_CONNECT_TECH, PCI_SUBDEVICE_ID_CONNECT_TECH_TITAN_4),
+		.driver_data = pbn_b0_4_1843200,
+	}, {
+		PCI_VDEVICE_SUB(OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954,
+				PCI_VENDOR_ID_AFAVLAB, PCI_SUBDEVICE_ID_AFAVLAB_P061),
+		.driver_data = pbn_b0_4_1152000,
+	}, {
+		PCI_VDEVICE(SEALEVEL, PCI_DEVICE_ID_SEALEVEL_U530),
+		.driver_data = pbn_b2_bt_1_115200,
+	}, {
+		PCI_VDEVICE(SEALEVEL, PCI_DEVICE_ID_SEALEVEL_UCOMM2),
+		.driver_data = pbn_b2_bt_2_115200,
+	}, {
+		PCI_VDEVICE(SEALEVEL, PCI_DEVICE_ID_SEALEVEL_UCOMM422),
+		.driver_data = pbn_b2_bt_4_115200,
+	}, {
+		PCI_VDEVICE(SEALEVEL, PCI_DEVICE_ID_SEALEVEL_UCOMM232),
+		.driver_data = pbn_b2_bt_2_115200,
+	}, {
+		PCI_VDEVICE(SEALEVEL, PCI_DEVICE_ID_SEALEVEL_COMM4),
+		.driver_data = pbn_b2_bt_4_115200,
+	}, {
+		PCI_VDEVICE(SEALEVEL, PCI_DEVICE_ID_SEALEVEL_COMM8),
+		.driver_data = pbn_b2_8_115200,
+	}, {
+		PCI_VDEVICE(SEALEVEL, PCI_DEVICE_ID_SEALEVEL_7803),
+		.driver_data = pbn_b2_8_460800,
+	}, {
+		PCI_VDEVICE(SEALEVEL, PCI_DEVICE_ID_SEALEVEL_UCOMM8),
+		.driver_data = pbn_b2_8_115200,
+	}, {
+		PCI_VDEVICE(PLX, PCI_DEVICE_ID_PLX_GTEK_SERIAL2),
+		.driver_data = pbn_b2_bt_2_115200,
+	}, {
+		PCI_VDEVICE(PLX, PCI_DEVICE_ID_PLX_SPCOM200),
+		.driver_data = pbn_b2_bt_2_921600,
+	}, {
+		/* VScom SPCOM800, from sl@s.pl */
+		PCI_VDEVICE(PLX, PCI_DEVICE_ID_PLX_SPCOM800),
+		.driver_data = pbn_b2_8_921600,
+	}, {
+		PCI_VDEVICE(PLX, PCI_DEVICE_ID_PLX_1077),
+		.driver_data = pbn_b2_4_921600,
+	}, {
+		/* Unknown card - subdevice 0x1584 */
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_VENDOR_ID_PLX, PCI_SUBDEVICE_ID_UNKNOWN_0x1584),
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		/* Unknown card - subdevice 0x1588 */
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_VENDOR_ID_PLX, PCI_SUBDEVICE_ID_UNKNOWN_0x1588),
+		.driver_data = pbn_b2_8_115200,
+	}, {
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_SUBVENDOR_ID_KEYSPAN, PCI_SUBDEVICE_ID_KEYSPAN_SX2),
+		.driver_data = pbn_panacom,
+	}, {
+		PCI_VDEVICE(PANACOM, PCI_DEVICE_ID_PANACOM_QUADMODEM),
+		.driver_data = pbn_panacom4,
+	}, {
+		PCI_VDEVICE(PANACOM, PCI_DEVICE_ID_PANACOM_DUALMODEM),
+		.driver_data = pbn_panacom2,
+	}, {
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9030,
+				PCI_VENDOR_ID_ESDGMBH, PCI_DEVICE_ID_ESDGMBH_CPCIASIO4),
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_SUBVENDOR_ID_CHASE_PCIFAST, PCI_SUBDEVICE_ID_CHASE_PCIFAST4),
+		.driver_data = pbn_b2_4_460800,
+	}, {
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_SUBVENDOR_ID_CHASE_PCIFAST, PCI_SUBDEVICE_ID_CHASE_PCIFAST8),
+		.driver_data = pbn_b2_8_460800,
+	}, {
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_SUBVENDOR_ID_CHASE_PCIFAST, PCI_SUBDEVICE_ID_CHASE_PCIFAST16),
+		.driver_data = pbn_b2_16_460800,
+	}, {
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_SUBVENDOR_ID_CHASE_PCIFAST, PCI_SUBDEVICE_ID_CHASE_PCIFAST16FMC),
+		.driver_data = pbn_b2_16_460800,
+	}, {
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_SUBVENDOR_ID_CHASE_PCIRAS, PCI_SUBDEVICE_ID_CHASE_PCIRAS4),
+		.driver_data = pbn_b2_4_460800,
+	}, {
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_SUBVENDOR_ID_CHASE_PCIRAS, PCI_SUBDEVICE_ID_CHASE_PCIRAS8),
+		.driver_data = pbn_b2_8_460800,
+	}, {
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9050,
+				PCI_SUBVENDOR_ID_EXSYS, PCI_SUBDEVICE_ID_EXSYS_4055),
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		/*
+		 * Megawolf Romulus PCI Serial Card, from Mike Hudson
+		 * (Exoray@isys.ca)
+		 */
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_ROMULUS,
+				0x10b5, 0x106a),
+		.driver_data = pbn_plx_romulus,
+	},
+
 	/*
 	 * Quatech cards. These actually have configurable clocks but for
 	 * now we just use the default.
 	 *
 	 * 100 series are RS232, 200 series RS422,
 	 */
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_QSC100,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_4_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_DSC100,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_2_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_DSC100E,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_DSC200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_2_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_DSC200E,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_QSC200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_4_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_ESC100D,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_8_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_ESC100M,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_8_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_QSCP100,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_4_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_DSCP100,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_2_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_QSCP200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_4_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_DSCP200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_2_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_QSCLP100,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_4_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_DSCLP100,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_SSCLP100,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_1_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_QSCLP200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_4_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_DSCLP200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_SSCLP200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_1_115200 },
-	{	PCI_VENDOR_ID_QUATECH, PCI_DEVICE_ID_QUATECH_ESCLP100,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_8_115200 },
-
-	{	PCI_VENDOR_ID_SPECIALIX, PCI_DEVICE_ID_OXSEMI_16PCI954,
-		PCI_VENDOR_ID_SPECIALIX, PCI_SUBDEVICE_ID_SPECIALIX_SPEED4,
-		0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954,
-		PCI_SUBVENDOR_ID_SIIG, PCI_SUBDEVICE_ID_SIIG_QUARTET_SERIAL,
-		0, 0,
-		pbn_b0_4_1152000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0x9505,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_921600 },
-
+	{
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_QSC100),
+		.driver_data = pbn_b1_4_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_DSC100),
+		.driver_data = pbn_b1_2_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_DSC100E),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_DSC200),
+		.driver_data = pbn_b1_2_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_DSC200E),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_QSC200),
+		.driver_data = pbn_b1_4_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_ESC100D),
+		.driver_data = pbn_b1_8_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_ESC100M),
+		.driver_data = pbn_b1_8_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_QSCP100),
+		.driver_data = pbn_b1_4_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_DSCP100),
+		.driver_data = pbn_b1_2_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_QSCP200),
+		.driver_data = pbn_b1_4_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_DSCP200),
+		.driver_data = pbn_b1_2_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_QSCLP100),
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_DSCLP100),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_SSCLP100),
+		.driver_data = pbn_b2_1_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_QSCLP200),
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_DSCLP200),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_SSCLP200),
+		.driver_data = pbn_b2_1_115200,
+	}, {
+		PCI_VDEVICE(QUATECH, PCI_DEVICE_ID_QUATECH_ESCLP100),
+		.driver_data = pbn_b0_8_115200,
+	}, {
+		PCI_VDEVICE_SUB(SPECIALIX, PCI_DEVICE_ID_OXSEMI_16PCI954,
+				PCI_VENDOR_ID_SPECIALIX, PCI_SUBDEVICE_ID_SPECIALIX_SPEED4),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954,
+				PCI_SUBVENDOR_ID_SIIG, PCI_SUBDEVICE_ID_SIIG_QUARTET_SERIAL),
+		.driver_data = pbn_b0_4_1152000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0x9505),
+		.driver_data = pbn_b0_bt_2_921600,
+	}, {
 		/*
 		 * The below card is a little controversial since it is the
 		 * subject of a PCI vendor/device ID clash.  (See
 		 * www.ussg.iu.edu/hypermail/linux/kernel/0303.1/0516.html).
 		 * For now just used the hex ID 0x950a.
 		 */
-	{	PCI_VENDOR_ID_OXSEMI, 0x950a,
-		PCI_SUBVENDOR_ID_SIIG, PCI_SUBDEVICE_ID_SIIG_DUAL_00,
-		0, 0, pbn_b0_2_115200 },
-	{	PCI_VENDOR_ID_OXSEMI, 0x950a,
-		PCI_SUBVENDOR_ID_SIIG, PCI_SUBDEVICE_ID_SIIG_DUAL_30,
-		0, 0, pbn_b0_2_115200 },
-	{	PCI_VENDOR_ID_OXSEMI, 0x950a,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_2_1130000 },
-	{	PCI_VENDOR_ID_OXSEMI, PCI_DEVICE_ID_OXSEMI_C950,
-		PCI_VENDOR_ID_OXSEMI, PCI_SUBDEVICE_ID_OXSEMI_C950, 0, 0,
-		pbn_b0_1_921600 },
-	{	PCI_VENDOR_ID_OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_115200 },
-	{	PCI_VENDOR_ID_OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI952,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_921600 },
-	{	PCI_VENDOR_ID_OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI958,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_8_1152000 },
+		PCI_VDEVICE_SUB(OXSEMI, 0x950a,
+				PCI_SUBVENDOR_ID_SIIG, PCI_SUBDEVICE_ID_SIIG_DUAL_00),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		PCI_VDEVICE_SUB(OXSEMI, 0x950a,
+				PCI_SUBVENDOR_ID_SIIG, PCI_SUBDEVICE_ID_SIIG_DUAL_30),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0x950a),
+		.driver_data = pbn_b0_2_1130000,
+	}, {
+		PCI_VDEVICE_SUB(OXSEMI, PCI_DEVICE_ID_OXSEMI_C950,
+				PCI_VENDOR_ID_OXSEMI, PCI_SUBDEVICE_ID_OXSEMI_C950),
+		.driver_data = pbn_b0_1_921600,
+	}, {
+		PCI_VDEVICE(OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI954),
+		.driver_data = pbn_b0_4_115200,
+	}, {
+		PCI_VDEVICE(OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI952),
+		.driver_data = pbn_b0_bt_2_921600,
+	}, {
+		PCI_VDEVICE(OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI958),
+		.driver_data = pbn_b2_8_1152000,
+	},
 
 	/*
 	 * Oxford Semiconductor Inc. Tornado PCI express device range.
 	 */
-	{	PCI_VENDOR_ID_OXSEMI, 0xc101,    /* OXPCIe952 1 Legacy UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc105,    /* OXPCIe952 1 Legacy UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc11b,    /* OXPCIe952 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc11f,    /* OXPCIe952 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc120,    /* OXPCIe952 1 Legacy UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc124,    /* OXPCIe952 1 Legacy UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc138,    /* OXPCIe952 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc13d,    /* OXPCIe952 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc140,    /* OXPCIe952 1 Legacy UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc141,    /* OXPCIe952 1 Legacy UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc144,    /* OXPCIe952 1 Legacy UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc145,    /* OXPCIe952 1 Legacy UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc158,    /* OXPCIe952 2 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_2_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc15d,    /* OXPCIe952 2 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_2_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc208,    /* OXPCIe954 4 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_4_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc20d,    /* OXPCIe954 4 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_4_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc308,    /* OXPCIe958 8 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_8_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc30d,    /* OXPCIe958 8 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_8_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc40b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc40f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc41b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc41f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc42b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc42f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc43b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc43f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc44b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc44f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc45b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc45f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc46b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc46f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc47b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc47f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc48b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc48f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc49b,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc49f,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc4ab,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc4af,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc4bb,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc4bf,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc4cb,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_OXSEMI, 0xc4cf,    /* OXPCIe200 1 Native UART */
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_1_15625000 },
+	{
+		PCI_VDEVICE(OXSEMI, 0xc101),    /* OXPCIe952 1 Legacy UART */
+		.driver_data = pbn_b0_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc105),    /* OXPCIe952 1 Legacy UART */
+		.driver_data = pbn_b0_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc11b),    /* OXPCIe952 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc11f),    /* OXPCIe952 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc120),    /* OXPCIe952 1 Legacy UART */
+		.driver_data = pbn_b0_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc124),    /* OXPCIe952 1 Legacy UART */
+		.driver_data = pbn_b0_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc138),    /* OXPCIe952 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc13d),    /* OXPCIe952 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc140),    /* OXPCIe952 1 Legacy UART */
+		.driver_data = pbn_b0_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc141),    /* OXPCIe952 1 Legacy UART */
+		.driver_data = pbn_b0_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc144),    /* OXPCIe952 1 Legacy UART */
+		.driver_data = pbn_b0_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc145),    /* OXPCIe952 1 Legacy UART */
+		.driver_data = pbn_b0_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc158),    /* OXPCIe952 2 Native UART */
+		.driver_data = pbn_oxsemi_2_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc15d),    /* OXPCIe952 2 Native UART */
+		.driver_data = pbn_oxsemi_2_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc208),    /* OXPCIe954 4 Native UART */
+		.driver_data = pbn_oxsemi_4_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc20d),    /* OXPCIe954 4 Native UART */
+		.driver_data = pbn_oxsemi_4_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc308),    /* OXPCIe958 8 Native UART */
+		.driver_data = pbn_oxsemi_8_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc30d),    /* OXPCIe958 8 Native UART */
+		.driver_data = pbn_oxsemi_8_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc40b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc40f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc41b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc41f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc42b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc42f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc43b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc43f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc44b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc44f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc45b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc45f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc46b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc46f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc47b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc47f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc48b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc48f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc49b),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc49f),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc4ab),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc4af),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc4bb),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc4bf),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc4cb),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		PCI_VDEVICE(OXSEMI, 0xc4cf),    /* OXPCIe200 1 Native UART */
+		.driver_data = pbn_oxsemi_1_15625000,
+	},
 	/*
 	 * Mainpine Inc. IQ Express "Rev3" utilizing OxSemi Tornado
 	 */
-	{	PCI_VENDOR_ID_MAINPINE, 0x4000,	/* IQ Express 1 Port V.34 Super-G3 Fax */
-		PCI_VENDOR_ID_MAINPINE, 0x4001, 0, 0,
-		pbn_oxsemi_1_15625000 },
-	{	PCI_VENDOR_ID_MAINPINE, 0x4000,	/* IQ Express 2 Port V.34 Super-G3 Fax */
-		PCI_VENDOR_ID_MAINPINE, 0x4002, 0, 0,
-		pbn_oxsemi_2_15625000 },
-	{	PCI_VENDOR_ID_MAINPINE, 0x4000,	/* IQ Express 4 Port V.34 Super-G3 Fax */
-		PCI_VENDOR_ID_MAINPINE, 0x4004, 0, 0,
-		pbn_oxsemi_4_15625000 },
-	{	PCI_VENDOR_ID_MAINPINE, 0x4000,	/* IQ Express 8 Port V.34 Super-G3 Fax */
-		PCI_VENDOR_ID_MAINPINE, 0x4008, 0, 0,
-		pbn_oxsemi_8_15625000 },
+	{
+		/* IQ Express 1 Port V.34 Super-G3 Fax */
+		PCI_VDEVICE_SUB(MAINPINE, 0x4000,
+				PCI_VENDOR_ID_MAINPINE, 0x4001),
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		/* IQ Express 2 Port V.34 Super-G3 Fax */
+		PCI_VDEVICE_SUB(MAINPINE, 0x4000,
+				PCI_VENDOR_ID_MAINPINE, 0x4002),
+		.driver_data = pbn_oxsemi_2_15625000,
+	}, {
+		/* IQ Express 4 Port V.34 Super-G3 Fax */
+		PCI_VDEVICE_SUB(MAINPINE, 0x4000,
+				PCI_VENDOR_ID_MAINPINE, 0x4004),
+		.driver_data = pbn_oxsemi_4_15625000,
+	}, {
+		/* IQ Express 8 Port V.34 Super-G3 Fax */
+		PCI_VDEVICE_SUB(MAINPINE, 0x4000,
+				PCI_VENDOR_ID_MAINPINE, 0x4008),
+		.driver_data = pbn_oxsemi_8_15625000,
+	},
 
 	/*
 	 * Digi/IBM PCIe 2-port Async EIA-232 Adapter utilizing OxSemi Tornado
 	 */
-	{	PCI_VENDOR_ID_DIGI, PCIE_DEVICE_ID_NEO_2_OX_IBM,
-		PCI_SUBVENDOR_ID_IBM, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_2_15625000 },
+	{
+		PCI_VDEVICE_SUB(DIGI, PCIE_DEVICE_ID_NEO_2_OX_IBM,
+				PCI_SUBVENDOR_ID_IBM, PCI_ANY_ID),
+		.driver_data = pbn_oxsemi_2_15625000,
+	},
+
 	/*
 	 * EndRun Technologies. PCI express device range.
 	 * EndRun PTP/1588 has 2 Native UARTs utilizing OxSemi 952.
 	 */
-	{	PCI_VENDOR_ID_ENDRUN, PCI_DEVICE_ID_ENDRUN_1588,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi_2_15625000 },
+	{
+		PCI_VDEVICE(ENDRUN, PCI_DEVICE_ID_ENDRUN_1588),
+		.driver_data = pbn_oxsemi_2_15625000,
+	},
 
 	/*
 	 * SBS Technologies, Inc. P-Octal and PMC-OCTPRO cards,
 	 * from skokodyn@yahoo.com
 	 */
-	{	PCI_VENDOR_ID_SBSMODULARIO, PCI_DEVICE_ID_OCTPRO,
-		PCI_SUBVENDOR_ID_SBSMODULARIO, PCI_SUBDEVICE_ID_OCTPRO232, 0, 0,
-		pbn_sbsxrsio },
-	{	PCI_VENDOR_ID_SBSMODULARIO, PCI_DEVICE_ID_OCTPRO,
-		PCI_SUBVENDOR_ID_SBSMODULARIO, PCI_SUBDEVICE_ID_OCTPRO422, 0, 0,
-		pbn_sbsxrsio },
-	{	PCI_VENDOR_ID_SBSMODULARIO, PCI_DEVICE_ID_OCTPRO,
-		PCI_SUBVENDOR_ID_SBSMODULARIO, PCI_SUBDEVICE_ID_POCTAL232, 0, 0,
-		pbn_sbsxrsio },
-	{	PCI_VENDOR_ID_SBSMODULARIO, PCI_DEVICE_ID_OCTPRO,
-		PCI_SUBVENDOR_ID_SBSMODULARIO, PCI_SUBDEVICE_ID_POCTAL422, 0, 0,
-		pbn_sbsxrsio },
+	{
+		PCI_VDEVICE_SUB(SBSMODULARIO, PCI_DEVICE_ID_OCTPRO,
+				PCI_SUBVENDOR_ID_SBSMODULARIO, PCI_SUBDEVICE_ID_OCTPRO232),
+		.driver_data = pbn_sbsxrsio,
+	}, {
+		PCI_VDEVICE_SUB(SBSMODULARIO, PCI_DEVICE_ID_OCTPRO,
+				PCI_SUBVENDOR_ID_SBSMODULARIO, PCI_SUBDEVICE_ID_OCTPRO422),
+		.driver_data = pbn_sbsxrsio,
+	}, {
+		PCI_VDEVICE_SUB(SBSMODULARIO, PCI_DEVICE_ID_OCTPRO,
+				PCI_SUBVENDOR_ID_SBSMODULARIO, PCI_SUBDEVICE_ID_POCTAL232),
+		.driver_data = pbn_sbsxrsio,
+	}, {
+		PCI_VDEVICE_SUB(SBSMODULARIO, PCI_DEVICE_ID_OCTPRO,
+				PCI_SUBVENDOR_ID_SBSMODULARIO, PCI_SUBDEVICE_ID_POCTAL422),
+		.driver_data = pbn_sbsxrsio,
+	},
 
 	/*
 	 * Digitan DS560-558, from jimd@esoft.com
 	 */
-	{	PCI_VENDOR_ID_ATT, PCI_DEVICE_ID_ATT_VENUS_MODEM,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_1_115200 },
+	{
+		PCI_VDEVICE(ATT, PCI_DEVICE_ID_ATT_VENUS_MODEM),
+		.driver_data = pbn_b1_1_115200,
+	},
 
 	/*
 	 * Titan Electronic cards
 	 *  The 400L and 800L have a custom setup quirk.
 	 */
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_100,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_2_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_400,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_800B,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_100L,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_1_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_200L,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_bt_2_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_400L,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_800L,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_8_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_200I,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b4_bt_2_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_400I,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b4_bt_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_800I,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b4_bt_8_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_400EH,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_800EH,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_800EHB,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_100E,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_titan_1_4000000 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_200E,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_titan_2_4000000 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_400E,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_titan_4_4000000 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_800E,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_titan_8_4000000 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_200EI,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_titan_2_4000000 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_200EISI,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_titan_2_4000000 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_200V3,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_400V3,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_410V3,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_800V3,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_TITAN, PCI_DEVICE_ID_TITAN_800V3B,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_4_921600 },
-
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_1S_10x_550,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_1_460800 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_1S_10x_650,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_1_460800 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_1S_10x_850,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_1_460800 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_2S_10x_550,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_2_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_2S_10x_650,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_2_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_2S_10x_850,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_2_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_4S_10x_550,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_4_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_4S_10x_650,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_4_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_4S_10x_850,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_4_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_1S_20x_550,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_1S_20x_650,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_1S_20x_850,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_2S_20x_550,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_2S_20x_650,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_2S_20x_850,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_4S_20x_550,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_4_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_4S_20x_650,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_4_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_4S_20x_850,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_4_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_8S_20x_550,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_8_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_8S_20x_650,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_8_921600 },
-	{	PCI_VENDOR_ID_SIIG, PCI_DEVICE_ID_SIIG_8S_20x_850,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_8_921600 },
+	{
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_100),
+		.driver_data = pbn_b0_1_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_200),
+		.driver_data = pbn_b0_2_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_400),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_800B),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_100L),
+		.driver_data = pbn_b1_1_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_200L),
+		.driver_data = pbn_b1_bt_2_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_400L),
+		.driver_data = pbn_b0_bt_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_800L),
+		.driver_data = pbn_b0_bt_8_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_200I),
+		.driver_data = pbn_b4_bt_2_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_400I),
+		.driver_data = pbn_b4_bt_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_800I),
+		.driver_data = pbn_b4_bt_8_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_400EH),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_800EH),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_800EHB),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_100E),
+		.driver_data = pbn_titan_1_4000000,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_200E),
+		.driver_data = pbn_titan_2_4000000,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_400E),
+		.driver_data = pbn_titan_4_4000000,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_800E),
+		.driver_data = pbn_titan_8_4000000,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_200EI),
+		.driver_data = pbn_titan_2_4000000,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_200EISI),
+		.driver_data = pbn_titan_2_4000000,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_200V3),
+		.driver_data = pbn_b0_bt_2_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_400V3),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_410V3),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_800V3),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE(TITAN, PCI_DEVICE_ID_TITAN_800V3B),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_1S_10x_550),
+		.driver_data = pbn_b2_1_460800,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_1S_10x_650),
+		.driver_data = pbn_b2_1_460800,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_1S_10x_850),
+		.driver_data = pbn_b2_1_460800,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_2S_10x_550),
+		.driver_data = pbn_b2_bt_2_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_2S_10x_650),
+		.driver_data = pbn_b2_bt_2_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_2S_10x_850),
+		.driver_data = pbn_b2_bt_2_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_4S_10x_550),
+		.driver_data = pbn_b2_bt_4_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_4S_10x_650),
+		.driver_data = pbn_b2_bt_4_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_4S_10x_850),
+		.driver_data = pbn_b2_bt_4_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_1S_20x_550),
+		.driver_data = pbn_b0_1_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_1S_20x_650),
+		.driver_data = pbn_b0_1_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_1S_20x_850),
+		.driver_data = pbn_b0_1_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_2S_20x_550),
+		.driver_data = pbn_b0_bt_2_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_2S_20x_650),
+		.driver_data = pbn_b0_bt_2_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_2S_20x_850),
+		.driver_data = pbn_b0_bt_2_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_4S_20x_550),
+		.driver_data = pbn_b0_bt_4_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_4S_20x_650),
+		.driver_data = pbn_b0_bt_4_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_4S_20x_850),
+		.driver_data = pbn_b0_bt_4_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_8S_20x_550),
+		.driver_data = pbn_b0_bt_8_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_8S_20x_650),
+		.driver_data = pbn_b0_bt_8_921600,
+	}, {
+		PCI_VDEVICE(SIIG, PCI_DEVICE_ID_SIIG_8S_20x_850),
+		.driver_data = pbn_b0_bt_8_921600,
+	},
 
 	/*
 	 * Computone devices submitted by Doug McNash dmcnash@computone.com
 	 */
-	{	PCI_VENDOR_ID_COMPUTONE, PCI_DEVICE_ID_COMPUTONE_PG,
-		PCI_SUBVENDOR_ID_COMPUTONE, PCI_SUBDEVICE_ID_COMPUTONE_PG4,
-		0, 0, pbn_computone_4 },
-	{	PCI_VENDOR_ID_COMPUTONE, PCI_DEVICE_ID_COMPUTONE_PG,
-		PCI_SUBVENDOR_ID_COMPUTONE, PCI_SUBDEVICE_ID_COMPUTONE_PG8,
-		0, 0, pbn_computone_8 },
-	{	PCI_VENDOR_ID_COMPUTONE, PCI_DEVICE_ID_COMPUTONE_PG,
-		PCI_SUBVENDOR_ID_COMPUTONE, PCI_SUBDEVICE_ID_COMPUTONE_PG6,
-		0, 0, pbn_computone_6 },
-
-	{	PCI_VENDOR_ID_OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI95N,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_oxsemi },
-	{	PCI_VENDOR_ID_TIMEDIA, PCI_DEVICE_ID_TIMEDIA_1889,
-		PCI_VENDOR_ID_TIMEDIA, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_1_921600 },
+	{
+		PCI_VDEVICE_SUB(COMPUTONE, PCI_DEVICE_ID_COMPUTONE_PG,
+				PCI_SUBVENDOR_ID_COMPUTONE, PCI_SUBDEVICE_ID_COMPUTONE_PG4),
+		.driver_data = pbn_computone_4,
+	}, {
+		PCI_VDEVICE_SUB(COMPUTONE, PCI_DEVICE_ID_COMPUTONE_PG,
+				PCI_SUBVENDOR_ID_COMPUTONE, PCI_SUBDEVICE_ID_COMPUTONE_PG8),
+		.driver_data = pbn_computone_8,
+	}, {
+		PCI_VDEVICE_SUB(COMPUTONE, PCI_DEVICE_ID_COMPUTONE_PG,
+				PCI_SUBVENDOR_ID_COMPUTONE, PCI_SUBDEVICE_ID_COMPUTONE_PG6),
+		.driver_data = pbn_computone_6,
+	},
+
+	{
+		PCI_VDEVICE(OXSEMI, PCI_DEVICE_ID_OXSEMI_16PCI95N),
+		.driver_data = pbn_oxsemi,
+	}, {
+		PCI_VDEVICE_SUB(TIMEDIA, PCI_DEVICE_ID_TIMEDIA_1889,
+				PCI_VENDOR_ID_TIMEDIA, PCI_ANY_ID),
+		.driver_data = pbn_b0_bt_1_921600,
+	},
 
 	/*
 	 * Sunix PCI serial boards
 	 */
-	{	PCI_VENDOR_ID_SUNIX, PCI_DEVICE_ID_SUNIX_1999,
-		PCI_VENDOR_ID_SUNIX, 0x0001, 0, 0,
-		pbn_sunix_pci_1s },
-	{	PCI_VENDOR_ID_SUNIX, PCI_DEVICE_ID_SUNIX_1999,
-		PCI_VENDOR_ID_SUNIX, 0x0002, 0, 0,
-		pbn_sunix_pci_2s },
-	{	PCI_VENDOR_ID_SUNIX, PCI_DEVICE_ID_SUNIX_1999,
-		PCI_VENDOR_ID_SUNIX, 0x0004, 0, 0,
-		pbn_sunix_pci_4s },
-	{	PCI_VENDOR_ID_SUNIX, PCI_DEVICE_ID_SUNIX_1999,
-		PCI_VENDOR_ID_SUNIX, 0x0084, 0, 0,
-		pbn_sunix_pci_4s },
-	{	PCI_VENDOR_ID_SUNIX, PCI_DEVICE_ID_SUNIX_1999,
-		PCI_VENDOR_ID_SUNIX, 0x0008, 0, 0,
-		pbn_sunix_pci_8s },
-	{	PCI_VENDOR_ID_SUNIX, PCI_DEVICE_ID_SUNIX_1999,
-		PCI_VENDOR_ID_SUNIX, 0x0088, 0, 0,
-		pbn_sunix_pci_8s },
-	{	PCI_VENDOR_ID_SUNIX, PCI_DEVICE_ID_SUNIX_1999,
-		PCI_VENDOR_ID_SUNIX, 0x0010, 0, 0,
-		pbn_sunix_pci_16s },
+	{
+		PCI_VDEVICE_SUB(SUNIX, PCI_DEVICE_ID_SUNIX_1999,
+				PCI_VENDOR_ID_SUNIX, 0x0001),
+		.driver_data = pbn_sunix_pci_1s,
+	}, {
+		PCI_VDEVICE_SUB(SUNIX, PCI_DEVICE_ID_SUNIX_1999,
+				PCI_VENDOR_ID_SUNIX, 0x0002),
+		.driver_data = pbn_sunix_pci_2s,
+	}, {
+		PCI_VDEVICE_SUB(SUNIX, PCI_DEVICE_ID_SUNIX_1999,
+				PCI_VENDOR_ID_SUNIX, 0x0004),
+		.driver_data = pbn_sunix_pci_4s,
+	}, {
+		PCI_VDEVICE_SUB(SUNIX, PCI_DEVICE_ID_SUNIX_1999,
+				PCI_VENDOR_ID_SUNIX, 0x0084),
+		.driver_data = pbn_sunix_pci_4s,
+	}, {
+		PCI_VDEVICE_SUB(SUNIX, PCI_DEVICE_ID_SUNIX_1999,
+				PCI_VENDOR_ID_SUNIX, 0x0008),
+		.driver_data = pbn_sunix_pci_8s,
+	}, {
+		PCI_VDEVICE_SUB(SUNIX, PCI_DEVICE_ID_SUNIX_1999,
+				PCI_VENDOR_ID_SUNIX, 0x0088),
+		.driver_data = pbn_sunix_pci_8s,
+	}, {
+		PCI_VDEVICE_SUB(SUNIX, PCI_DEVICE_ID_SUNIX_1999,
+				PCI_VENDOR_ID_SUNIX, 0x0010),
+		.driver_data = pbn_sunix_pci_16s,
+	},
 
 	/*
 	 * AFAVLAB serial card, from Harald Welte <laforge@gnumonks.org>
 	 */
-	{	PCI_VENDOR_ID_AFAVLAB, PCI_DEVICE_ID_AFAVLAB_P028,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_8_115200 },
-	{	PCI_VENDOR_ID_AFAVLAB, PCI_DEVICE_ID_AFAVLAB_P030,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_8_115200 },
-
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_DSERIAL,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_115200 },
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_QUATRO_A,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_115200 },
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_QUATRO_B,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_115200 },
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_OCTO_A,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_4_460800 },
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_OCTO_B,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_4_460800 },
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_PORT_PLUS,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_460800 },
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_QUAD_A,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_460800 },
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_QUAD_B,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_2_460800 },
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_SSERIAL,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_1_115200 },
-	{	PCI_VENDOR_ID_LAVA, PCI_DEVICE_ID_LAVA_PORT_650,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_bt_1_460800 },
+	{
+		PCI_VDEVICE(AFAVLAB, PCI_DEVICE_ID_AFAVLAB_P028),
+		.driver_data = pbn_b0_bt_8_115200,
+	}, {
+		PCI_VDEVICE(AFAVLAB, PCI_DEVICE_ID_AFAVLAB_P030),
+		.driver_data = pbn_b0_bt_8_115200,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_DSERIAL),
+		.driver_data = pbn_b0_bt_2_115200,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_QUATRO_A),
+		.driver_data = pbn_b0_bt_2_115200,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_QUATRO_B),
+		.driver_data = pbn_b0_bt_2_115200,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_OCTO_A),
+		.driver_data = pbn_b0_bt_4_460800,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_OCTO_B),
+		.driver_data = pbn_b0_bt_4_460800,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_PORT_PLUS),
+		.driver_data = pbn_b0_bt_2_460800,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_QUAD_A),
+		.driver_data = pbn_b0_bt_2_460800,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_QUAD_B),
+		.driver_data = pbn_b0_bt_2_460800,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_SSERIAL),
+		.driver_data = pbn_b0_bt_1_115200,
+	}, {
+		PCI_VDEVICE(LAVA, PCI_DEVICE_ID_LAVA_PORT_650),
+		.driver_data = pbn_b0_bt_1_460800,
+	},
 
 	/*
 	 * Korenix Jetcard F0/F1 cards (JC1204, JC1208, JC1404, JC1408).
@@ -5164,560 +5203,533 @@ static const struct pci_device_id serial_pci_tbl[] = {
 	 * Note that JC140x are RS422/485 cards which require ox950
 	 * ACR = 0x10, and as such are not currently fully supported.
 	 */
-	{	PCI_VENDOR_ID_KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF0,
-		0x1204, 0x0004, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF0,
-		0x1208, 0x0004, 0, 0,
-		pbn_b0_4_921600 },
-/*	{	PCI_VENDOR_ID_KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF0,
-		0x1402, 0x0002, 0, 0,
-		pbn_b0_2_921600 }, */
-/*	{	PCI_VENDOR_ID_KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF0,
-		0x1404, 0x0004, 0, 0,
-		pbn_b0_4_921600 }, */
-	{	PCI_VENDOR_ID_KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF1,
-		0x1208, 0x0004, 0, 0,
-		pbn_b0_4_921600 },
-
-	{	PCI_VENDOR_ID_KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF2,
-		0x1204, 0x0004, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF2,
-		0x1208, 0x0004, 0, 0,
-		pbn_b0_4_921600 },
-	{	PCI_VENDOR_ID_KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF3,
-		0x1208, 0x0004, 0, 0,
-		pbn_b0_4_921600 },
+	{
+		PCI_VDEVICE_SUB(KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF0,
+				0x1204, 0x0004),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF0,
+				0x1208, 0x0004),
+		.driver_data = pbn_b0_4_921600,
+	},
+/*	{
+		PCI_VDEVICE_SUB(KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF0,
+				0x1402, 0x0002),
+		.driver_data = pbn_b0_2_921600,
+	}, */
+/*	{
+		PCI_VDEVICE_SUB(KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF0,
+				0x1404, 0x0004),
+		.driver_data = pbn_b0_4_921600,
+	}, */
+	{
+		PCI_VDEVICE_SUB(KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF1,
+				0x1208, 0x0004),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF2,
+				0x1204, 0x0004),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF2,
+				0x1208, 0x0004),
+		.driver_data = pbn_b0_4_921600,
+	}, {
+		PCI_VDEVICE_SUB(KORENIX, PCI_DEVICE_ID_KORENIX_JETCARDF3,
+				0x1208, 0x0004),
+		.driver_data = pbn_b0_4_921600,
+	},
 	/*
 	 * Dell Remote Access Card 4 - Tim_T_Murphy@Dell.com
 	 */
-	{	PCI_VENDOR_ID_DELL, PCI_DEVICE_ID_DELL_RAC4,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_1_1382400 },
+	{
+		PCI_VDEVICE(DELL, PCI_DEVICE_ID_DELL_RAC4),
+		.driver_data = pbn_b1_1_1382400,
+	},
 
 	/*
 	 * Dell Remote Access Card III - Tim_T_Murphy@Dell.com
 	 */
-	{	PCI_VENDOR_ID_DELL, PCI_DEVICE_ID_DELL_RACIII,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_1_1382400 },
+	{
+		PCI_VDEVICE(DELL, PCI_DEVICE_ID_DELL_RACIII),
+		.driver_data = pbn_b1_1_1382400,
+	},
 
 	/*
 	 * RAStel 2 port modem, gerg@moreton.com.au
 	 */
-	{	PCI_VENDOR_ID_MORETON, PCI_DEVICE_ID_RASTEL_2PORT,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_bt_2_115200 },
+	{
+		PCI_VDEVICE(MORETON, PCI_DEVICE_ID_RASTEL_2PORT),
+		.driver_data = pbn_b2_bt_2_115200,
+	},
 
 	/*
 	 * EKF addition for i960 Boards form EKF with serial port
 	 */
-	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_80960_RP,
-		0xE4BF, PCI_ANY_ID, 0, 0,
-		pbn_intel_i960 },
+	{
+		PCI_VDEVICE_SUB(INTEL, PCI_DEVICE_ID_INTEL_80960_RP,
+				0xE4BF, PCI_ANY_ID),
+		.driver_data = pbn_intel_i960,
+	},
 
 	/*
 	 * Xircom Cardbus/Ethernet combos
 	 */
-	{	PCI_VENDOR_ID_XIRCOM, PCI_DEVICE_ID_XIRCOM_X3201_MDM,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_115200 },
+	{
+		PCI_VDEVICE(XIRCOM, PCI_DEVICE_ID_XIRCOM_X3201_MDM),
+		.driver_data = pbn_b0_1_115200,
+	},
 	/*
 	 * Xircom RBM56G cardbus modem - Dirk Arnold (temp entry)
 	 */
-	{	PCI_VENDOR_ID_XIRCOM, PCI_DEVICE_ID_XIRCOM_RBM56G,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_115200 },
+	{
+		PCI_VDEVICE(XIRCOM, PCI_DEVICE_ID_XIRCOM_RBM56G),
+		.driver_data = pbn_b0_1_115200,
+	},
 
 	/*
 	 * Untested PCI modems, sent in from various folks...
 	 */
 
-	/*
-	 * Elsa Model 56K PCI Modem, from Andreas Rath <arh@01019freenet.de>
-	 */
-	{	PCI_VENDOR_ID_ROCKWELL, 0x1004,
-		0x1048, 0x1500, 0, 0,
-		pbn_b1_1_115200 },
-
-	{	PCI_VENDOR_ID_SGI, PCI_DEVICE_ID_SGI_IOC3,
-		0xFF00, 0, 0, 0,
-		pbn_sgi_ioc3 },
+	{
+		/* Elsa Model 56K PCI Modem, from Andreas Rath <arh@01019freenet.de> */
+		PCI_VDEVICE_SUB(ROCKWELL, 0x1004, 0x1048, 0x1500),
+		.driver_data = pbn_b1_1_115200,
+	}, {
+		PCI_VDEVICE_SUB(SGI, PCI_DEVICE_ID_SGI_IOC3, 0xFF00, 0),
+		.driver_data = pbn_sgi_ioc3,
+	},
 
 	/*
 	 * HP Diva card
 	 */
-	{	PCI_VENDOR_ID_HP, PCI_DEVICE_ID_HP_DIVA,
-		PCI_VENDOR_ID_HP, PCI_DEVICE_ID_HP_DIVA_RMP3, 0, 0,
-		pbn_b1_1_115200 },
-	{	PCI_VENDOR_ID_HP, PCI_DEVICE_ID_HP_DIVA,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_5_115200 },
-	{	PCI_VENDOR_ID_HP, PCI_DEVICE_ID_HP_DIVA_AUX,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_1_115200 },
-	/* HPE PCI serial device */
-	{	PCI_VENDOR_ID_HP_3PAR, PCI_DEVICE_ID_HPE_PCI_SERIAL,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_1_115200 },
-
-	{	PCI_VENDOR_ID_DCI, PCI_DEVICE_ID_DCI_PCCOM2,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b3_2_115200 },
-	{	PCI_VENDOR_ID_DCI, PCI_DEVICE_ID_DCI_PCCOM4,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b3_4_115200 },
-	{	PCI_VENDOR_ID_DCI, PCI_DEVICE_ID_DCI_PCCOM8,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b3_8_115200 },
-	/*
-	 * Topic TP560 Data/Fax/Voice 56k modem (reported by Evan Clarke)
-	 */
-	{	PCI_VENDOR_ID_TOPIC, PCI_DEVICE_ID_TOPIC_TP560,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b0_1_115200 },
-	/*
-	 * ITE
-	 */
-	{	PCI_VENDOR_ID_ITE, PCI_DEVICE_ID_ITE_8872,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b1_bt_1_115200 },
-
-	/*
-	 * IntaShield IS-100
-	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0D60,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b2_1_115200 },
-	/*
-	 * IntaShield IS-200
-	 */
-	{	PCI_VENDOR_ID_INTASHIELD, PCI_DEVICE_ID_INTASHIELD_IS200,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,	/* 135a.0d80 */
-		pbn_b2_2_115200 },
-	/*
-	 * IntaShield IS-400
-	 */
-	{	PCI_VENDOR_ID_INTASHIELD, PCI_DEVICE_ID_INTASHIELD_IS400,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,    /* 135a.0dc0 */
-		pbn_b2_4_115200 },
-	/*
-	 * IntaShield IX-100
-	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4027,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_1_15625000 },
-	/*
-	 * IntaShield IX-200
-	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4028,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_2_15625000 },
-	/*
-	 * IntaShield IX-400
-	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4029,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_4_15625000 },
+	{
+		PCI_VDEVICE_SUB(HP, PCI_DEVICE_ID_HP_DIVA,
+				PCI_VENDOR_ID_HP, PCI_DEVICE_ID_HP_DIVA_RMP3),
+		.driver_data = pbn_b1_1_115200,
+	}, {
+		PCI_VDEVICE(HP, PCI_DEVICE_ID_HP_DIVA),
+		.driver_data = pbn_b0_5_115200,
+	}, {
+		PCI_VDEVICE(HP, PCI_DEVICE_ID_HP_DIVA_AUX),
+		.driver_data = pbn_b2_1_115200,
+	}, {
+		/* HPE PCI serial device */
+		PCI_VDEVICE(HP_3PAR, PCI_DEVICE_ID_HPE_PCI_SERIAL),
+		.driver_data = pbn_b1_1_115200,
+	}, {
+		PCI_VDEVICE(DCI, PCI_DEVICE_ID_DCI_PCCOM2),
+		.driver_data = pbn_b3_2_115200,
+	}, {
+		PCI_VDEVICE(DCI, PCI_DEVICE_ID_DCI_PCCOM4),
+		.driver_data = pbn_b3_4_115200,
+	}, {
+		PCI_VDEVICE(DCI, PCI_DEVICE_ID_DCI_PCCOM8),
+		.driver_data = pbn_b3_8_115200,
+	}, {
+		/* Topic TP560 Data/Fax/Voice 56k modem (reported by Evan Clarke) */
+		PCI_VDEVICE(TOPIC, PCI_DEVICE_ID_TOPIC_TP560),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		/* ITE */
+		PCI_VDEVICE(ITE, PCI_DEVICE_ID_ITE_8872),
+		.driver_data = pbn_b1_bt_1_115200,
+	}, {
+		/* IntaShield IS-100 */
+		PCI_VDEVICE(INTASHIELD, 0x0D60),
+		.driver_data = pbn_b2_1_115200,
+	}, {
+		/* IntaShield IS-200; 135a.0d80 */
+		PCI_VDEVICE(INTASHIELD, PCI_DEVICE_ID_INTASHIELD_IS200),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		/* IntaShield IS-400; 135a.0dc0 */
+		PCI_VDEVICE(INTASHIELD, PCI_DEVICE_ID_INTASHIELD_IS400),
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		/* IntaShield IX-100 */
+		PCI_VDEVICE(INTASHIELD, 0x4027),
+		.driver_data = pbn_oxsemi_1_15625000,
+	}, {
+		/* IntaShield IX-200 */
+		PCI_VDEVICE(INTASHIELD, 0x4028),
+		.driver_data = pbn_oxsemi_2_15625000,
+	}, {
+		/* IntaShield IX-400 */
+		PCI_VDEVICE(INTASHIELD, 0x4029),
+		.driver_data = pbn_oxsemi_4_15625000,
+	},
 	/* Brainboxes Devices */
 	/*
 	* Brainboxes UC-101
 	*/
-	{       PCI_VENDOR_ID_INTASHIELD, 0x0BA1,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{       PCI_VENDOR_ID_INTASHIELD, 0x0BA2,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{       PCI_VENDOR_ID_INTASHIELD, 0x0BA3,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0BA1),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0BA2),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0BA3),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UC-235/246
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0AA1,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_1_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0AA2,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_1_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0AA1),
+		.driver_data = pbn_b2_1_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0AA2),
+		.driver_data = pbn_b2_1_115200,
+	},
 	/*
 	 * Brainboxes UC-253/UC-734
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0CA1,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0CA1),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UC-260/271/701/756
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0D21,
-		PCI_ANY_ID, PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_MULTISERIAL << 8, 0xffff00,
-		pbn_b2_4_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0E34,
-		PCI_ANY_ID, PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_MULTISERIAL << 8, 0xffff00,
-		pbn_b2_4_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0D21),
+		.class = PCI_CLASS_COMMUNICATION_MULTISERIAL << 8,
+		.class_mask = 0xffff00,
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0E34),
+		.class = PCI_CLASS_COMMUNICATION_MULTISERIAL << 8,
+		.class_mask = 0xffff00,
+		.driver_data = pbn_b2_4_115200,
+	},
 	/*
 	 * Brainboxes UC-268
 	 */
-	{       PCI_VENDOR_ID_INTASHIELD, 0x0841,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_4_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0841),
+		.driver_data = pbn_b2_4_115200,
+	},
 	/*
 	 * Brainboxes UC-275/279
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0881,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_8_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0881),
+		.driver_data = pbn_b2_8_115200,
+	},
 	/*
 	 * Brainboxes UC-302
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x08E1,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x08E2,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x08E3,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x08E1),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x08E2),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x08E3),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UC-310
 	 */
-	{       PCI_VENDOR_ID_INTASHIELD, 0x08C1,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x08C1),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UC-313
 	 */
-	{       PCI_VENDOR_ID_INTASHIELD, 0x08A1,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{       PCI_VENDOR_ID_INTASHIELD, 0x08A2,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{       PCI_VENDOR_ID_INTASHIELD, 0x08A3,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x08A1),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x08A2),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x08A3),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UC-320/324
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0A61,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_1_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0A61),
+		.driver_data = pbn_b2_1_115200,
+	},
 	/*
 	 * Brainboxes UC-346
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0B01,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_4_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0B02,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_4_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0B01),
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0B02),
+		.driver_data = pbn_b2_4_115200,
+	},
 	/*
 	 * Brainboxes UC-357
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0A81,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0A82,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0A83,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0A81),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0A82),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0A83),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UC-368
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0C41,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_4_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0C42,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_4_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0C43,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_4_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0C41),
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0C42),
+		.driver_data = pbn_b2_4_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0C43),
+		.driver_data = pbn_b2_4_115200,
+	},
 	/*
 	 * Brainboxes UC-420
 	 */
-	{       PCI_VENDOR_ID_INTASHIELD, 0x0921,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_4_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0921),
+		.driver_data = pbn_b2_4_115200,
+	},
 	/*
 	 * Brainboxes UC-607
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x09A1,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x09A2,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x09A3,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x09A1),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x09A2),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x09A3),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UC-836
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0D41,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_4_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0D41),
+		.driver_data = pbn_b2_4_115200,
+	},
 	/*
 	 * Brainboxes UP-189
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0AC1,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0AC2,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0AC3,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0AC1),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0AC2),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0AC3),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UP-200
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0B21,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0B22,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0B23,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0B21),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0B22),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0B23),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UP-869
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0C01,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0C02,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0C03,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0C01),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0C02),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0C03),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes UP-880
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0C21,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0C22,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0C23,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_2_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0C21),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0C22),
+		.driver_data = pbn_b2_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x0C23),
+		.driver_data = pbn_b2_2_115200,
+	},
 	/*
 	 * Brainboxes PX-101
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4005,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b0_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4019,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_2_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4005),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x4019),
+		.driver_data = pbn_oxsemi_2_15625000,
+	},
 	/*
 	 * Brainboxes PX-235/246
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4004,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b0_1_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4016,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_1_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4004),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x4016),
+		.driver_data = pbn_oxsemi_1_15625000,
+	},
 	/*
 	 * Brainboxes PX-203/PX-257
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4006,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b0_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4015,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_2_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4006),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x4015),
+		.driver_data = pbn_oxsemi_2_15625000,
+	},
 	/*
 	 * Brainboxes PX-260/PX-701
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x400A,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_4_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x400A),
+		.driver_data = pbn_oxsemi_4_15625000,
+	},
 	/*
 	 * Brainboxes PX-275/279
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x0E41,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b2_8_115200 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x0E41),
+		.driver_data = pbn_b2_8_115200,
+	},
 	/*
 	 * Brainboxes PX-310
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x400E,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_2_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x400E),
+		.driver_data = pbn_oxsemi_2_15625000,
+	},
 	/*
 	 * Brainboxes PX-313
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x400C,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_2_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x400C),
+		.driver_data = pbn_oxsemi_2_15625000,
+	},
 	/*
 	 * Brainboxes PX-320/324/PX-376/PX-387
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x400B,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_1_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x400B),
+		.driver_data = pbn_oxsemi_1_15625000,
+	},
 	/*
 	 * Brainboxes PX-335/346
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x400F,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_4_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x400F),
+		.driver_data = pbn_oxsemi_4_15625000,
+	},
 	/*
 	 * Brainboxes PX-368
 	 */
-	{       PCI_VENDOR_ID_INTASHIELD, 0x4010,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_4_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4010),
+		.driver_data = pbn_oxsemi_4_15625000,
+	},
 	/*
 	 * Brainboxes PX-420
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4000,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b0_4_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4011,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_4_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4000),
+		.driver_data = pbn_b0_4_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x4011),
+		.driver_data = pbn_oxsemi_4_15625000,
+	},
 	/*
 	 * Brainboxes PX-475
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x401D,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_1_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x401D),
+		.driver_data = pbn_oxsemi_1_15625000,
+	},
 	/*
 	 * Brainboxes PX-803/PX-857
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4009,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b0_2_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4018,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_2_15625000 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x401E,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_2_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4009),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x4018),
+		.driver_data = pbn_oxsemi_2_15625000,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x401E),
+		.driver_data = pbn_oxsemi_2_15625000,
+	},
 	/*
 	 * Brainboxes PX-820
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4002,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b0_4_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4013,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_4_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4002),
+		.driver_data = pbn_b0_4_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x4013),
+		.driver_data = pbn_oxsemi_4_15625000,
+	},
 	/*
 	 * Brainboxes PX-835/PX-846
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4008,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_b0_1_115200 },
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4017,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_1_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4008),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE(INTASHIELD, 0x4017),
+		.driver_data = pbn_oxsemi_1_15625000,
+	},
 	/*
 	 * Brainboxes XC-235
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4026,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_1_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4026),
+		.driver_data = pbn_oxsemi_1_15625000,
+	},
 	/*
 	 * Brainboxes XC-475
 	 */
-	{	PCI_VENDOR_ID_INTASHIELD, 0x4021,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0,
-		pbn_oxsemi_1_15625000 },
+	{
+		PCI_VDEVICE(INTASHIELD, 0x4021),
+		.driver_data = pbn_oxsemi_1_15625000,
+	},
 
 	/*
 	 * Perle PCI-RAS cards
 	 */
-	{       PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9030,
-		PCI_SUBVENDOR_ID_PERLE, PCI_SUBDEVICE_ID_PCI_RAS4,
-		0, 0, pbn_b2_4_921600 },
-	{       PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_9030,
-		PCI_SUBVENDOR_ID_PERLE, PCI_SUBDEVICE_ID_PCI_RAS8,
-		0, 0, pbn_b2_8_921600 },
+	{
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9030,
+				PCI_SUBVENDOR_ID_PERLE, PCI_SUBDEVICE_ID_PCI_RAS4),
+		.driver_data = pbn_b2_4_921600,
+	},
+	{
+		PCI_VDEVICE_SUB(PLX, PCI_DEVICE_ID_PLX_9030,
+				PCI_SUBVENDOR_ID_PERLE, PCI_SUBDEVICE_ID_PCI_RAS8),
+		.driver_data = pbn_b2_8_921600,
+	},
 
 	/*
 	 * Mainpine series cards: Fairly standard layout but fools
@@ -5725,375 +5737,343 @@ static const struct pci_device_id serial_pci_tbl[] = {
 	 * unmatched communications subclasses in the PCI Express case
 	 */
 
-	{	/* RockForceDUO */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x0200,
-		0, 0, pbn_b0_2_115200 },
-	{	/* RockForceQUATRO */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x0300,
-		0, 0, pbn_b0_4_115200 },
-	{	/* RockForceDUO+ */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x0400,
-		0, 0, pbn_b0_2_115200 },
-	{	/* RockForceQUATRO+ */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x0500,
-		0, 0, pbn_b0_4_115200 },
-	{	/* RockForce+ */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x0600,
-		0, 0, pbn_b0_2_115200 },
-	{	/* RockForce+ */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x0700,
-		0, 0, pbn_b0_4_115200 },
-	{	/* RockForceOCTO+ */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x0800,
-		0, 0, pbn_b0_8_115200 },
-	{	/* RockForceDUO+ */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x0C00,
-		0, 0, pbn_b0_2_115200 },
-	{	/* RockForceQUARTRO+ */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x0D00,
-		0, 0, pbn_b0_4_115200 },
-	{	/* RockForceOCTO+ */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x1D00,
-		0, 0, pbn_b0_8_115200 },
-	{	/* RockForceD1 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x2000,
-		0, 0, pbn_b0_1_115200 },
-	{	/* RockForceF1 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x2100,
-		0, 0, pbn_b0_1_115200 },
-	{	/* RockForceD2 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x2200,
-		0, 0, pbn_b0_2_115200 },
-	{	/* RockForceF2 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x2300,
-		0, 0, pbn_b0_2_115200 },
-	{	/* RockForceD4 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x2400,
-		0, 0, pbn_b0_4_115200 },
-	{	/* RockForceF4 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x2500,
-		0, 0, pbn_b0_4_115200 },
-	{	/* RockForceD8 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x2600,
-		0, 0, pbn_b0_8_115200 },
-	{	/* RockForceF8 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x2700,
-		0, 0, pbn_b0_8_115200 },
-	{	/* IQ Express D1 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x3000,
-		0, 0, pbn_b0_1_115200 },
-	{	/* IQ Express F1 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x3100,
-		0, 0, pbn_b0_1_115200 },
-	{	/* IQ Express D2 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x3200,
-		0, 0, pbn_b0_2_115200 },
-	{	/* IQ Express F2 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x3300,
-		0, 0, pbn_b0_2_115200 },
-	{	/* IQ Express D4 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x3400,
-		0, 0, pbn_b0_4_115200 },
-	{	/* IQ Express F4 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x3500,
-		0, 0, pbn_b0_4_115200 },
-	{	/* IQ Express D8 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x3C00,
-		0, 0, pbn_b0_8_115200 },
-	{	/* IQ Express F8 */
-		PCI_VENDOR_ID_MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
-		PCI_VENDOR_ID_MAINPINE, 0x3D00,
-		0, 0, pbn_b0_8_115200 },
-
+	{
+		/* RockForceDUO */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x0200),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		/* RockForceQUATRO */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x0300),
+		.driver_data = pbn_b0_4_115200,
+	}, {
+		/* RockForceDUO+ */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x0400),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		/* RockForceQUATRO+ */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x0500),
+			.driver_data = pbn_b0_4_115200,
+	}, {
+		/* RockForce+ */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x0600),
+			.driver_data = pbn_b0_2_115200,
+	}, {
+		/* RockForce+ */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x0700),
+			.driver_data = pbn_b0_4_115200,
+	}, {
+		/* RockForceOCTO+ */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x0800),
+			.driver_data = pbn_b0_8_115200,
+	}, {
+		/* RockForceDUO+ */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x0C00),
+			.driver_data = pbn_b0_2_115200,
+	}, {
+		/* RockForceQUARTRO+ */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x0D00),
+			.driver_data = pbn_b0_4_115200,
+	}, {
+		/* RockForceOCTO+ */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x1D00),
+			.driver_data = pbn_b0_8_115200,
+	}, {
+		/* RockForceD1 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x2000),
+			.driver_data = pbn_b0_1_115200,
+	}, {
+		/* RockForceF1 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x2100),
+			.driver_data = pbn_b0_1_115200,
+	}, {	/* RockForceD2 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x2200),
+			.driver_data = pbn_b0_2_115200,
+	}, {	/* RockForceF2 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x2300),
+			.driver_data = pbn_b0_2_115200,
+	}, {
+		/* RockForceD4 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x2400),
+			.driver_data = pbn_b0_4_115200,
+	}, {
+		/* RockForceF4 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x2500),
+			.driver_data = pbn_b0_4_115200,
+	}, {
+		/* RockForceD8 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x2600),
+			.driver_data = pbn_b0_8_115200,
+	}, {
+		/* RockForceF8 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x2700),
+			.driver_data = pbn_b0_8_115200,
+	}, {
+		/* IQ Express D1 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x3000),
+			.driver_data = pbn_b0_1_115200,
+	}, {
+		/* IQ Express F1 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x3100),
+			.driver_data = pbn_b0_1_115200,
+	}, {
+		/* IQ Express D2 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x3200),
+			.driver_data = pbn_b0_2_115200,
+	}, {
+		/* IQ Express F2 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x3300),
+			.driver_data = pbn_b0_2_115200,
+	}, {
+		/* IQ Express D4 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x3400),
+			.driver_data = pbn_b0_4_115200,
+	}, {
+		/* IQ Express F4 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x3500),
+			.driver_data = pbn_b0_4_115200,
+	}, {
+		/* IQ Express D8 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x3C00),
+			.driver_data = pbn_b0_8_115200,
+	}, {
+		/* IQ Express F8 */
+		PCI_VDEVICE_SUB(MAINPINE, PCI_DEVICE_ID_MAINPINE_PBRIDGE,
+				PCI_VENDOR_ID_MAINPINE, 0x3D00),
+			.driver_data = pbn_b0_8_115200,
+	},
 
-	/*
-	 * PA Semi PA6T-1682M on-chip UART
-	 */
-	{	PCI_VENDOR_ID_PASEMI, 0xa004,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_pasemi_1682M },
+	{
+		/* PA Semi PA6T-1682M on-chip UART */
+		PCI_VDEVICE(PASEMI, 0xa004),
+		.driver_data = pbn_pasemi_1682M,
+	},
 
 	/*
 	 * National Instruments
 	 */
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI23216,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_16_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI2328,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_8_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI2324,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_bt_4_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI2322,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_bt_2_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI2324I,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_bt_4_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI2322I,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_bt_2_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8420_23216,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_16_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8420_2328,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_8_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8420_2324,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_bt_4_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8420_2322,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_bt_2_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8422_2324,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_bt_4_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8422_2322,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_b1_bt_2_115200 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8430_2322,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_2 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI8430_2322,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_2 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8430_2324,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_4 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI8430_2324,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_4 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8430_2328,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_8 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI8430_2328,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_8 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8430_23216,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_16 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI8430_23216,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_16 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8432_2322,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_2 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI8432_2322,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_2 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PXI8432_2324,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_4 },
-	{	PCI_VENDOR_ID_NI, PCI_DEVICE_ID_NI_PCI8432_2324,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_ni8430_4 },
+	{
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI23216),
+		.driver_data = pbn_b1_16_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI2328),
+		.driver_data = pbn_b1_8_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI2324),
+		.driver_data = pbn_b1_bt_4_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI2322),
+		.driver_data = pbn_b1_bt_2_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI2324I),
+		.driver_data = pbn_b1_bt_4_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI2322I),
+		.driver_data = pbn_b1_bt_2_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8420_23216),
+		.driver_data = pbn_b1_16_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8420_2328),
+		.driver_data = pbn_b1_8_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8420_2324),
+		.driver_data = pbn_b1_bt_4_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8420_2322),
+		.driver_data = pbn_b1_bt_2_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8422_2324),
+		.driver_data = pbn_b1_bt_4_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8422_2322),
+		.driver_data = pbn_b1_bt_2_115200,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8430_2322),
+		.driver_data = pbn_ni8430_2,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI8430_2322),
+		.driver_data = pbn_ni8430_2,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8430_2324),
+		.driver_data = pbn_ni8430_4,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI8430_2324),
+		.driver_data = pbn_ni8430_4,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8430_2328),
+		.driver_data = pbn_ni8430_8,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI8430_2328),
+		.driver_data = pbn_ni8430_8,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8430_23216),
+		.driver_data = pbn_ni8430_16,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI8430_23216),
+		.driver_data = pbn_ni8430_16,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8432_2322),
+		.driver_data = pbn_ni8430_2,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI8432_2322),
+		.driver_data = pbn_ni8430_2,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PXI8432_2324),
+		.driver_data = pbn_ni8430_4,
+	}, {
+		PCI_VDEVICE(NI, PCI_DEVICE_ID_NI_PCI8432_2324),
+		.driver_data = pbn_ni8430_4,
+	},
 
 	/*
 	 * MOXA
 	 */
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP102E),	    pbn_moxa_2 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP102EL),    pbn_moxa_2 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP102N),	    pbn_moxa_2 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP104EL_A),  pbn_moxa_4 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP104N),	    pbn_moxa_4 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP112N),	    pbn_moxa_2 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP114EL),    pbn_moxa_4 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP114N),	    pbn_moxa_4 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP116E_A_A), pbn_moxa_8 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP116E_A_B), pbn_moxa_8 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP118EL_A),  pbn_moxa_8 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP118E_A_I), pbn_moxa_8 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP132EL),    pbn_moxa_2 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP132N),     pbn_moxa_2 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP134EL_A),  pbn_moxa_4 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP134N),	    pbn_moxa_4 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP138E_A),   pbn_moxa_8 },
-	{ PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP168EL_A),  pbn_moxa_8 },
+	{
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP102E),
+		.driver_data = pbn_moxa_2,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP102EL),
+		.driver_data = pbn_moxa_2,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP102N),
+		.driver_data = pbn_moxa_2,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP104EL_A),
+		.driver_data = pbn_moxa_4,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP104N),
+		.driver_data = pbn_moxa_4,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP112N),
+		.driver_data = pbn_moxa_2,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP114EL),
+		.driver_data = pbn_moxa_4,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP114N),
+		.driver_data = pbn_moxa_4,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP116E_A_A),
+		.driver_data = pbn_moxa_8,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP116E_A_B),
+		.driver_data = pbn_moxa_8,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP118EL_A),
+		.driver_data = pbn_moxa_8,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP118E_A_I),
+		.driver_data = pbn_moxa_8,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP132EL),
+		.driver_data = pbn_moxa_2,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP132N),
+		.driver_data = pbn_moxa_2,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP134EL_A),
+		.driver_data = pbn_moxa_4,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP134N),
+		.driver_data = pbn_moxa_4,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP138E_A),
+		.driver_data = pbn_moxa_8,
+	}, {
+		PCI_VDEVICE(MOXA, PCI_DEVICE_ID_MOXA_CP168EL_A),
+		.driver_data = pbn_moxa_8,
+	},
 
 	/*
 	* ADDI-DATA GmbH communication cards <info@addi-data.com>
 	*/
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7500,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_4_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7420,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_2_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7300,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_AMCC,
-		PCI_DEVICE_ID_AMCC_ADDIDATA_APCI7800,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b1_8_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7500_2,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_4_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7420_2,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_2_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7300_2,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7500_3,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_4_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7420_3,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_2_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7300_3,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCI7800_3,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_8_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCIe7500,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_ADDIDATA_PCIe_4_3906250 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCIe7420,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_ADDIDATA_PCIe_2_3906250 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCIe7300,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_ADDIDATA_PCIe_1_3906250 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_APCIe7800,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_ADDIDATA_PCIe_8_3906250 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_CPCI7500,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_4_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_CPCI7500_NG,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_4_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_CPCI7420_NG,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_2_115200 },
-
-	{	PCI_VENDOR_ID_ADDIDATA,
-		PCI_DEVICE_ID_ADDIDATA_CPCI7300_NG,
-		PCI_ANY_ID,
-		PCI_ANY_ID,
-		0,
-		0,
-		pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9835,
-		PCI_VENDOR_ID_IBM, 0x0299,
-		0, 0, pbn_b0_bt_2_115200 },
+	{
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7500),
+		.driver_data = pbn_b0_4_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7420),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7300),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE(AMCC, PCI_DEVICE_ID_AMCC_ADDIDATA_APCI7800),
+		.driver_data = pbn_b1_8_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7500_2),
+		.driver_data = pbn_b0_4_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7420_2),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7300_2),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7500_3),
+		.driver_data = pbn_b0_4_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7420_3),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7300_3),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCI7800_3),
+		.driver_data = pbn_b0_8_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCIe7500),
+		.driver_data = pbn_ADDIDATA_PCIe_4_3906250,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCIe7420),
+		.driver_data = pbn_ADDIDATA_PCIe_2_3906250,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCIe7300),
+		.driver_data = pbn_ADDIDATA_PCIe_1_3906250,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_APCIe7800),
+		.driver_data = pbn_ADDIDATA_PCIe_8_3906250,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_CPCI7500),
+		.driver_data = pbn_b0_4_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_CPCI7500_NG),
+		.driver_data = pbn_b0_4_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_CPCI7420_NG),
+		.driver_data = pbn_b0_2_115200,
+	}, {
+		PCI_VDEVICE(ADDIDATA, PCI_DEVICE_ID_ADDIDATA_CPCI7300_NG),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9835,
+				PCI_VENDOR_ID_IBM, 0x0299),
+		.driver_data = pbn_b0_bt_2_115200,
+	},
 
 	/*
 	 * other NetMos 9835 devices are most likely handled by the
@@ -6101,157 +6081,183 @@ static const struct pci_device_id serial_pci_tbl[] = {
 	 * before adding them here.
 	 */
 
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9901,
-		0xA000, 0x1000,
-		0, 0, pbn_b0_1_115200 },
-
-	/* the 9901 is a rebranded 9912 */
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9912,
-		0xA000, 0x1000,
-		0, 0, pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9922,
-		0xA000, 0x1000,
-		0, 0, pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9904,
-		0xA000, 0x1000,
-		0, 0, pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9900,
-		0xA000, 0x1000,
-		0, 0, pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9900,
-		0xA000, 0x3002,
-		0, 0, pbn_NETMOS9900_2s_115200 },
-
-	{	PCIE_VENDOR_ID_ASIX, PCIE_DEVICE_ID_AX99100,
-		0xA000, 0x1000,
-		0, 0, pbn_b0_1_115200 },
+	{
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9901,
+				0xA000, 0x1000),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		/* the 9901 is a rebranded 9912 */
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9912,
+				0xA000, 0x1000),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9922,
+				0xA000, 0x1000),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9904,
+				0xA000, 0x1000),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9900,
+				0xA000, 0x1000),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9900,
+				0xA000, 0x3002),
+		.driver_data = pbn_NETMOS9900_2s_115200,
+	}, {
+		PCI_DEVICE_SUB(PCIE_VENDOR_ID_ASIX, PCIE_DEVICE_ID_AX99100,
+			       0xA000, 0x1000),
+		.driver_data = pbn_b0_1_115200,
+	},
 
 	/*
 	 * Best Connectivity and Rosewill PCI Multi I/O cards
 	 */
 
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9865,
-		0xA000, 0x1000,
-		0, 0, pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9865,
-		0xA000, 0x3002,
-		0, 0, pbn_b0_bt_2_115200 },
-
-	{	PCI_VENDOR_ID_NETMOS, PCI_DEVICE_ID_NETMOS_9865,
-		0xA000, 0x3004,
-		0, 0, pbn_b0_bt_4_115200 },
+	{
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9865,
+				0xA000, 0x1000),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9865,
+				0xA000, 0x3002),
+		.driver_data = pbn_b0_bt_2_115200,
+	}, {
+		PCI_VDEVICE_SUB(NETMOS, PCI_DEVICE_ID_NETMOS_9865,
+				0xA000, 0x3004),
+		.driver_data = pbn_b0_bt_4_115200,
+	},
 
 	/*
 	 * ASIX AX99100 PCIe to Multi I/O Controller
 	 */
-	{	PCI_VENDOR_ID_ASIX, PCI_DEVICE_ID_ASIX_AX99100,
-		0xA000, 0x1000,
-		0, 0, pbn_b0_1_115200 },
+	{
+		PCI_VDEVICE_SUB(ASIX, PCI_DEVICE_ID_ASIX_AX99100,
+				0xA000, 0x1000),
+		.driver_data = pbn_b0_1_115200,
+	},
 
 	/* Intel CE4100 */
-	{	PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_CE4100_UART,
-		PCI_ANY_ID,  PCI_ANY_ID, 0, 0,
-		pbn_ce4100_1_115200 },
+	{
+		PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_CE4100_UART),
+		.driver_data = pbn_ce4100_1_115200,
+	},
 
 	/*
 	 * Cronyx Omega PCI
 	 */
-	{	PCI_VENDOR_ID_PLX, PCI_DEVICE_ID_PLX_CRONYX_OMEGA,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_omegapci },
+	{
+		PCI_VDEVICE(PLX, PCI_DEVICE_ID_PLX_CRONYX_OMEGA),
+		.driver_data = pbn_omegapci,
+	},
 
 	/*
 	 * Broadcom TruManage
 	 */
-	{	PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_BROADCOM_TRUMANAGE,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		pbn_brcm_trumanage },
+	{
+		PCI_VDEVICE(BROADCOM, PCI_DEVICE_ID_BROADCOM_TRUMANAGE),
+		.driver_data = pbn_brcm_trumanage,
+	},
 
 	/*
 	 * AgeStar as-prs2-009
 	 */
-	{	PCI_VENDOR_ID_AGESTAR, PCI_DEVICE_ID_AGESTAR_9375,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0, pbn_b0_bt_2_115200 },
+	{
+		PCI_VDEVICE(AGESTAR, PCI_DEVICE_ID_AGESTAR_9375),
+		.driver_data = pbn_b0_bt_2_115200,
+	},
 
 	/*
 	 * WCH CH353 series devices: The 2S1P is handled by parport_serial
 	 * so not listed here.
 	 */
-	{	PCI_VENDOR_ID_WCHCN, PCI_DEVICE_ID_WCHCN_CH353_4S,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0, pbn_b0_bt_4_115200 },
-
-	{	PCI_VENDOR_ID_WCHCN, PCI_DEVICE_ID_WCHCN_CH353_2S1PF,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0, pbn_b0_bt_2_115200 },
-
-	{	PCI_VENDOR_ID_WCHCN, PCI_DEVICE_ID_WCHCN_CH355_4S,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0, pbn_b0_bt_4_115200 },
-
-	{	PCI_VENDOR_ID_WCHIC, PCI_DEVICE_ID_WCHIC_CH382_2S,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0, pbn_wch382_2 },
-
-	{	PCI_VENDOR_ID_WCHIC, PCI_DEVICE_ID_WCHIC_CH384_4S,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0, pbn_wch384_4 },
+	{
+		PCI_VDEVICE(WCHCN, PCI_DEVICE_ID_WCHCN_CH353_4S),
+		.driver_data = pbn_b0_bt_4_115200,
+	}, {
+		PCI_VDEVICE(WCHCN, PCI_DEVICE_ID_WCHCN_CH353_2S1PF),
+		.driver_data = pbn_b0_bt_2_115200,
+	}, {
+		PCI_VDEVICE(WCHCN, PCI_DEVICE_ID_WCHCN_CH355_4S),
+		.driver_data = pbn_b0_bt_4_115200,
+	}, {
+		PCI_VDEVICE(WCHIC, PCI_DEVICE_ID_WCHIC_CH382_2S),
+		.driver_data = pbn_wch382_2,
+	}, {
+		PCI_VDEVICE(WCHIC, PCI_DEVICE_ID_WCHIC_CH384_4S),
+		.driver_data = pbn_wch384_4,
+	}, {
+		PCI_VDEVICE(WCHIC, PCI_DEVICE_ID_WCHIC_CH384_8S),
+		.driver_data = pbn_wch384_8,
+	},
 
-	{	PCI_VENDOR_ID_WCHIC, PCI_DEVICE_ID_WCHIC_CH384_8S,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0, pbn_wch384_8 },
 	/*
 	 * Realtek RealManage
 	 */
-	{	PCI_VENDOR_ID_REALTEK, 0x816a,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0, pbn_b0_1_115200 },
-
-	{	PCI_VENDOR_ID_REALTEK, 0x816b,
-		PCI_ANY_ID, PCI_ANY_ID,
-		0, 0, pbn_b0_1_115200 },
+	{
+		PCI_VDEVICE(REALTEK, 0x816a),
+		.driver_data = pbn_b0_1_115200,
+	}, {
+		PCI_VDEVICE(REALTEK, 0x816b),
+		.driver_data = pbn_b0_1_115200,
+	},
 
 	/* Systembase Multi I/O cards */
-	{ PCI_VDEVICE(SYSTEMBASE, 0x0008), pbn_b0_8_921600 },
+	{
+		PCI_VDEVICE(SYSTEMBASE, 0x0008),
+		.driver_data = pbn_b0_8_921600,
+	},
 
 	/* Fintek PCI serial cards */
-	{ PCI_DEVICE(0x1c29, 0x1104), .driver_data = pbn_fintek_4 },
-	{ PCI_DEVICE(0x1c29, 0x1108), .driver_data = pbn_fintek_8 },
-	{ PCI_DEVICE(0x1c29, 0x1112), .driver_data = pbn_fintek_12 },
-	{ PCI_DEVICE(0x1c29, 0x1204), .driver_data = pbn_fintek_F81504A },
-	{ PCI_DEVICE(0x1c29, 0x1208), .driver_data = pbn_fintek_F81508A },
-	{ PCI_DEVICE(0x1c29, 0x1212), .driver_data = pbn_fintek_F81512A },
+	{
+		PCI_DEVICE(0x1c29, 0x1104),
+		.driver_data = pbn_fintek_4,
+	}, {
+		PCI_DEVICE(0x1c29, 0x1108),
+		.driver_data = pbn_fintek_8,
+	}, {
+		PCI_DEVICE(0x1c29, 0x1112),
+		.driver_data = pbn_fintek_12,
+	}, {
+		PCI_DEVICE(0x1c29, 0x1204),
+		.driver_data = pbn_fintek_F81504A,
+	}, {
+		PCI_DEVICE(0x1c29, 0x1208),
+		.driver_data = pbn_fintek_F81508A,
+	}, {
+		PCI_DEVICE(0x1c29, 0x1212),
+		.driver_data = pbn_fintek_F81512A,
+	},
 
 	/* MKS Tenta SCOM-080x serial cards */
-	{ PCI_DEVICE(0x1601, 0x0800), .driver_data = pbn_b0_4_1250000 },
-	{ PCI_DEVICE(0x1601, 0xa801), .driver_data = pbn_b0_4_1250000 },
+	{
+		PCI_DEVICE(0x1601, 0x0800), .driver_data = pbn_b0_4_1250000,
+	}, {
+		PCI_DEVICE(0x1601, 0xa801), .driver_data = pbn_b0_4_1250000,
+	},
 
 	/* Amazon PCI serial device */
-	{ PCI_DEVICE(0x1d0f, 0x8250), .driver_data = pbn_b0_1_115200 },
+	{
+		PCI_DEVICE(0x1d0f, 0x8250), .driver_data = pbn_b0_1_115200,
+	},
 
 	/*
 	 * These entries match devices with class COMMUNICATION_SERIAL,
 	 * COMMUNICATION_MODEM or COMMUNICATION_MULTISERIAL
 	 */
-	{	PCI_ANY_ID, PCI_ANY_ID,
-		PCI_ANY_ID, PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_SERIAL << 8,
-		0xffff00, pbn_default },
-	{	PCI_ANY_ID, PCI_ANY_ID,
-		PCI_ANY_ID, PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_MODEM << 8,
-		0xffff00, pbn_default },
-	{	PCI_ANY_ID, PCI_ANY_ID,
-		PCI_ANY_ID, PCI_ANY_ID,
-		PCI_CLASS_COMMUNICATION_MULTISERIAL << 8,
-		0xffff00, pbn_default },
-	{ 0, }
+	{
+		PCI_DEVICE_CLASS(PCI_CLASS_COMMUNICATION_SERIAL << 8, 0xffff00),
+		.driver_data = pbn_default,
+	}, {
+		PCI_DEVICE_CLASS(PCI_CLASS_COMMUNICATION_MODEM << 8, 0xffff00),
+		.driver_data = pbn_default,
+	}, {
+		PCI_DEVICE_CLASS(PCI_CLASS_COMMUNICATION_MULTISERIAL << 8, 0xffff00),
+		.driver_data = pbn_default,
+	},
+	{ }
 };
 
 static pci_ers_result_t serial8250_io_error_detected(struct pci_dev *dev,
diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
index af78cc02f38e7..e4e6a53ebea39 100644
--- a/drivers/tty/serial/8250/8250_port.c
+++ b/drivers/tty/serial/8250/8250_port.c
@@ -458,6 +458,8 @@ static void set_io_from_upio(struct uart_port *p)
 		p->serial_out = io_serial_out;
 		break;
 #endif
+	case UPIO_AU:
+		break;
 	default:
 		WARN(p->iotype != UPIO_PORT || p->iobase,
 		     "Unsupported UART type %x\n", p->iotype);
@@ -1988,7 +1990,7 @@ static void wait_for_xmitr(struct uart_8250_port *up, int bits)
 	wait_for_lsr(up, bits);
 
 	/* Wait up to 1s for flow control if necessary */
-	if (up->port.flags & UPF_CONS_FLOW) {
+	if (uart_cons_flow_enabled(&up->port)) {
 		for (tmout = 1000000; tmout; tmout--) {
 			unsigned int msr = serial_in(up, UART_MSR);
 			up->msr_saved_flags |= msr & MSR_SAVE_FLAGS;
@@ -3351,7 +3353,7 @@ void serial8250_console_write(struct uart_8250_port *up, const char *s,
 		 * it regardless of the CTS state. Therefore, only use fifo
 		 * if we don't use control flow.
 		 */
-		!(up->port.flags & UPF_CONS_FLOW);
+		!uart_cons_flow_enabled(&up->port);
 
 	if (likely(use_fifo))
 		serial8250_console_fifo_write(up, s, count);
diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index 9aa61c93d7bc6..f834e5d292fd7 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -321,7 +321,7 @@ config SERIAL_MAX3100
 
 config SERIAL_MAX310X
 	tristate "MAX310X support"
-	depends on SPI_MASTER
+	depends on SPI_MASTER || I2C
 	select SERIAL_CORE
 	select REGMAP_SPI if SPI_MASTER
 	select REGMAP_I2C if I2C
@@ -913,13 +913,13 @@ config SERIAL_MSM_CONSOLE
 	select SERIAL_EARLYCON
 
 config SERIAL_QCOM_GENI
-	tristate "QCOM on-chip GENI based serial port support"
+	tristate "Qualcomm on-chip GENI based serial port support"
 	depends on ARCH_QCOM || COMPILE_TEST
 	depends on QCOM_GENI_SE
 	select SERIAL_CORE
 
 config SERIAL_QCOM_GENI_CONSOLE
-	bool "QCOM GENI Serial Console support"
+	bool "Qualcomm GENI Serial Console support"
 	depends on SERIAL_QCOM_GENI
 	select SERIAL_CORE_CONSOLE
 	select SERIAL_EARLYCON
diff --git a/drivers/tty/serial/apbuart.c b/drivers/tty/serial/apbuart.c
index 3e46341cfff8a..afb04d727203e 100644
--- a/drivers/tty/serial/apbuart.c
+++ b/drivers/tty/serial/apbuart.c
@@ -210,8 +210,6 @@ static void apbuart_set_termios(struct uart_port *port,
 
 	/* Ask the core to calculate the divisor for us. */
 	baud = uart_get_baud_rate(port, termios, old, 0, port->uartclk / 16);
-	if (baud == 0)
-		panic("invalid baudrate %i\n", port->uartclk / 16);
 
 	/* uart_get_divisor calc a *16 uart freq, apbuart is *8 */
 	quot = (uart_get_divisor(port, baud)) * 2;
diff --git a/drivers/tty/serial/bcm63xx_uart.c b/drivers/tty/serial/bcm63xx_uart.c
index 51df9d2d8bfc5..544695cb184c3 100644
--- a/drivers/tty/serial/bcm63xx_uart.c
+++ b/drivers/tty/serial/bcm63xx_uart.c
@@ -675,7 +675,7 @@ static void wait_for_xmitr(struct uart_port *port)
 	}
 
 	/* Wait up to 1s for flow control if necessary */
-	if (port->flags & UPF_CONS_FLOW) {
+	if (uart_cons_flow_enabled(port)) {
 		tmout = 1000000;
 		while (--tmout) {
 			unsigned int val;
diff --git a/drivers/tty/serial/fsl_lpuart.c b/drivers/tty/serial/fsl_lpuart.c
index 1bd7ec9c81ea4..b7919c05f0fb8 100644
--- a/drivers/tty/serial/fsl_lpuart.c
+++ b/drivers/tty/serial/fsl_lpuart.c
@@ -1379,7 +1379,8 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
 
 	if (!nent) {
 		dev_err(sport->port.dev, "DMA Rx mapping error\n");
-		return -EINVAL;
+		ret = -EINVAL;
+		goto err_free_buf;
 	}
 
 	dma_rx_sconfig.src_addr = lpuart_dma_datareg_addr(sport);
@@ -1391,7 +1392,7 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
 	if (ret < 0) {
 		dev_err(sport->port.dev,
 				"DMA Rx slave config failed, err = %d\n", ret);
-		return ret;
+		goto err_unmap_sg;
 	}
 
 	sport->dma_rx_desc = dmaengine_prep_dma_cyclic(chan,
@@ -1402,7 +1403,8 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
 				 DMA_PREP_INTERRUPT);
 	if (!sport->dma_rx_desc) {
 		dev_err(sport->port.dev, "Cannot prepare cyclic DMA\n");
-		return -EFAULT;
+		ret = -ENOMEM;
+		goto err_unmap_sg;
 	}
 
 	sport->dma_rx_desc->callback = lpuart_dma_rx_complete;
@@ -1426,6 +1428,13 @@ static inline int lpuart_start_rx_dma(struct lpuart_port *sport)
 	}
 
 	return 0;
+
+err_unmap_sg:
+	dma_unmap_sg(chan->device->dev, &sport->rx_sgl, 1, DMA_FROM_DEVICE);
+err_free_buf:
+	kfree(ring->buf);
+	ring->buf = NULL;
+	return ret;
 }
 
 static void lpuart_dma_rx_free(struct uart_port *port)
diff --git a/drivers/tty/serial/icom.c b/drivers/tty/serial/icom.c
index bcdc072084860..b6399d86a0bc3 100644
--- a/drivers/tty/serial/icom.c
+++ b/drivers/tty/serial/icom.c
@@ -1396,8 +1396,6 @@ static void icom_set_termios(struct uart_port *port, struct ktermios *termios,
 	baud = uart_get_baud_rate(port, termios, old_termios,
 				  icom_acfg_baud[0],
 				  icom_acfg_baud[BAUD_TABLE_LIMIT]);
-	if (!baud)
-		baud = 9600;	/* B0 transition handled in rs_set_termios */
 
 	for (index = 0; index < BAUD_TABLE_LIMIT; index++) {
 		if (icom_acfg_baud[index] == baud) {
diff --git a/drivers/tty/serial/jsm/jsm_driver.c b/drivers/tty/serial/jsm/jsm_driver.c
index 4b73e51f83fb5..8cd9b981bf200 100644
--- a/drivers/tty/serial/jsm/jsm_driver.c
+++ b/drivers/tty/serial/jsm/jsm_driver.c
@@ -296,25 +296,25 @@ static void jsm_remove_one(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id jsm_pci_tbl[] = {
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_2DB9), 0, 0, 0 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_2DB9PRI), 0, 0, 1 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_2RJ45), 0, 0, 2 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_2RJ45PRI), 0, 0, 3 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCIE_DEVICE_ID_NEO_4_IBM), 0, 0, 4 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_DIGI_NEO_8), 0, 0, 5 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_4), 0, 0, 6 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_1_422), 0, 0, 7 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_1_422_485), 0, 0, 8 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_NEO_2_422_485), 0, 0, 9 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCIE_DEVICE_ID_NEO_8), 0, 0, 10 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCIE_DEVICE_ID_NEO_4), 0, 0, 11 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCIE_DEVICE_ID_NEO_4RJ45), 0, 0, 12 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCIE_DEVICE_ID_NEO_8RJ45), 0, 0, 13 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_CLASSIC_4), 0, 0, 14 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_CLASSIC_4_422), 0, 0, 15 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_CLASSIC_8), 0, 0, 16 },
-	{ PCI_DEVICE(PCI_VENDOR_ID_DIGI, PCI_DEVICE_ID_CLASSIC_8_422), 0, 0, 17 },
-	{ 0, }
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_NEO_2DB9) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_NEO_2DB9PRI) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_NEO_2RJ45) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_NEO_2RJ45PRI) },
+	{ PCI_VDEVICE(DIGI, PCIE_DEVICE_ID_NEO_4_IBM) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_DIGI_NEO_8) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_NEO_4) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_NEO_1_422) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_NEO_1_422_485) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_NEO_2_422_485) },
+	{ PCI_VDEVICE(DIGI, PCIE_DEVICE_ID_NEO_8) },
+	{ PCI_VDEVICE(DIGI, PCIE_DEVICE_ID_NEO_4) },
+	{ PCI_VDEVICE(DIGI, PCIE_DEVICE_ID_NEO_4RJ45) },
+	{ PCI_VDEVICE(DIGI, PCIE_DEVICE_ID_NEO_8RJ45) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_CLASSIC_4) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_CLASSIC_4_422) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_CLASSIC_8) },
+	{ PCI_VDEVICE(DIGI, PCI_DEVICE_ID_CLASSIC_8_422) },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, jsm_pci_tbl);
 
diff --git a/drivers/tty/serial/max310x.c b/drivers/tty/serial/max310x.c
index ac7d3f197c3a5..9f423b3b4201d 100644
--- a/drivers/tty/serial/max310x.c
+++ b/drivers/tty/serial/max310x.c
@@ -9,6 +9,7 @@
  *  Based on max3107.c, by Aavamobile
  */
 
+#include <linux/bitfield.h>
 #include <linux/bitops.h>
 #include <linux/clk.h>
 #include <linux/delay.h>
@@ -241,12 +242,12 @@
 #define MAX310X_WRITE_BIT		0x80
 
 /* Port startup definitions */
-#define MAX310X_PORT_STARTUP_WAIT_RETRIES	20 /* Number of retries */
-#define MAX310X_PORT_STARTUP_WAIT_DELAY_MS	10 /* Delay between retries */
+#define MAX310X_PORT_STARTUP_SLEEP_US	10000  /* Delay between retries */
+#define MAX310X_PORT_STARTUP_TIMEOUT_US	(20 * MAX310X_PORT_STARTUP_SLEEP_US) /* Total timeout */
 
 /* Crystal-related definitions */
-#define MAX310X_XTAL_WAIT_RETRIES	20 /* Number of retries */
-#define MAX310X_XTAL_WAIT_DELAY_MS	10 /* Delay between retries */
+#define MAX310X_XTAL_SLEEP_US		10000  /* Delay between retries */
+#define MAX310X_XTAL_TIMEOUT_US		(20 * MAX310X_XTAL_SLEEP_US) /* Total timeout */
 
 /* MAX3107 specific */
 #define MAX3107_REV_ID			(0xa0)
@@ -258,6 +259,17 @@
 #define MAX14830_BRGCFG_CLKDIS_BIT	(1 << 6) /* Clock Disable */
 #define MAX14830_REV_ID			(0xb0)
 
+struct max310x_clk_config_t {
+	u8 prediv; /* Predivider */
+	u8 pll_mult; /* PLL multiplier */
+	unsigned int fref; /*
+			    * Reference clock for fractional baud rate generator:
+			    *   PLL enabled:  (freq / prediv) x pll_mult
+			    *   PLL disabled: freq
+			    */
+	unsigned int err; /* Computed error for selected parameters */
+};
+
 struct max310x_if_cfg {
 	int (*extended_reg_enable)(struct device *dev, bool enable);
 	u8 rev_id_offset;
@@ -545,70 +557,91 @@ static int max310x_set_baud(struct uart_port *port, int baud)
 	return (16*port->uartclk) / (c*(16*div + frac));
 }
 
-static int max310x_update_best_err(unsigned long f, long *besterr)
+static void max310x_try_cfg(unsigned int fdiv, u8 div, u8 pll_mult,
+			    struct max310x_clk_config_t *cfg)
 {
-	/* Use baudrate 115200 for calculate error */
-	long err = f % (460800 * 16);
+	unsigned int fmul = fdiv * pll_mult;
+	unsigned int err;
 
-	if ((*besterr < 0) || (*besterr > err)) {
-		*besterr = err;
-		return 0;
+	/* Use high-enough baudrate to calculate error */
+	err = fmul % (460800 * 16);
+
+	if (cfg->err > err) {
+		cfg->err = err;
+		cfg->pll_mult = pll_mult;
+		cfg->prediv = div;
+		cfg->fref = fmul;
 	}
+}
 
-	return 1;
+static u8 max310x_pll_mult_to_id(u8 pll_mult)
+{
+	switch (pll_mult) {
+	case 144:	return 3;
+	case 96:	return 2;
+	case 48:	return 1;
+	case 6:
+	default:	return 0;
+	}
 }
 
-static s32 max310x_set_ref_clk(struct device *dev, struct max310x_port *s,
-			       unsigned long freq, bool xtal)
+/*
+ * From table 7 in datasheet: PLLFactor Selector Guide
+ *
+ * +-----------+----------------+-------------------+-------------------+
+ * | PLLFactor | MULTIPLICATION |      fPLLIN       |       fREF        |
+ * |  (1 & 0)  |     FACTOR     +---------+---------+---------+---------+
+ * |           |                |   MIN   |   MAX   |   MIN   |   MAX   |
+ * +-----------+----------------+---------+---------+---------+---------+
+ * |     0     |        6       |  500kHz |  800kHz |   3MHz  |  4.8MHz |
+ * +-----------+----------------+---------+---------+---------+---------+
+ * |     1     |       48       |  850kHz |  1.2MHz | 40.8MHz |  56MHz  |
+ * +-----------+----------------+---------+---------+---------+---------+
+ * |     2     |       96       |  425kHz |   1MHz  | 40.8MHz |  96MHz  |
+ * +-----------+----------------+---------+---------+---------+---------+
+ * |     3     |      144       |  390kHz |  667kHz |  56MHz  |  96MHz  |
+ * +-----------+----------------+---------+---------+---------+---------+
+ */
+static int max310x_set_ref_clk(struct device *dev, struct max310x_port *s,
+			       unsigned int freq, unsigned int *fref, bool xtal)
 {
-	unsigned int div, clksrc, pllcfg = 0;
-	long besterr = -1;
-	unsigned long fdiv, fmul, bestfreq = freq;
+	unsigned int div, fdiv, clksrc, val;
+	struct max310x_clk_config_t cfg;
+
+	cfg.err = UINT_MAX;
+	cfg.prediv = 0;
+	cfg.fref = freq;
 
 	/* First, update error without PLL */
-	max310x_update_best_err(freq, &besterr);
+	max310x_try_cfg(freq, 1, 1, &cfg);
 
 	/* Try all possible PLL dividers */
-	for (div = 1; (div <= 63) && besterr; div++) {
+	for (div = 1; (div <= 63) && cfg.err; div++) {
 		fdiv = DIV_ROUND_CLOSEST(freq, div);
 
-		/* Try multiplier 6 */
-		fmul = fdiv * 6;
 		if ((fdiv >= 500000) && (fdiv <= 800000))
-			if (!max310x_update_best_err(fmul, &besterr)) {
-				pllcfg = (0 << 6) | div;
-				bestfreq = fmul;
-			}
-		/* Try multiplier 48 */
-		fmul = fdiv * 48;
-		if ((fdiv >= 850000) && (fdiv <= 1200000))
-			if (!max310x_update_best_err(fmul, &besterr)) {
-				pllcfg = (1 << 6) | div;
-				bestfreq = fmul;
-			}
-		/* Try multiplier 96 */
-		fmul = fdiv * 96;
+			max310x_try_cfg(fdiv, div, 6, &cfg);   /* PLL x6 */
+		else if ((fdiv >= 850000) && (fdiv <= 1200000))
+			max310x_try_cfg(fdiv, div, 48, &cfg);  /* PLL x48 */
+
 		if ((fdiv >= 425000) && (fdiv <= 1000000))
-			if (!max310x_update_best_err(fmul, &besterr)) {
-				pllcfg = (2 << 6) | div;
-				bestfreq = fmul;
-			}
-		/* Try multiplier 144 */
-		fmul = fdiv * 144;
+			max310x_try_cfg(fdiv, div, 96, &cfg);  /* PLL x96 */
+
 		if ((fdiv >= 390000) && (fdiv <= 667000))
-			if (!max310x_update_best_err(fmul, &besterr)) {
-				pllcfg = (3 << 6) | div;
-				bestfreq = fmul;
-			}
+			max310x_try_cfg(fdiv, div, 144, &cfg); /* PLL x144 */
 	}
 
 	/* Configure clock source */
 	clksrc = MAX310X_CLKSRC_EXTCLK_BIT | (xtal ? MAX310X_CLKSRC_CRYST_BIT : 0);
 
 	/* Configure PLL */
-	if (pllcfg) {
+	if (cfg.prediv) {
+		u8 pll_id = max310x_pll_mult_to_id(cfg.pll_mult);
+
 		clksrc |= MAX310X_CLKSRC_PLL_BIT;
-		regmap_write(s->regmap, MAX310X_PLLCFG_REG, pllcfg);
+		val = FIELD_PREP(MAX310X_PLLCFG_PLLFACTOR_MASK, pll_id) |
+			FIELD_PREP(MAX310X_PLLCFG_PREDIV_MASK, cfg.prediv);
+		regmap_write(s->regmap, MAX310X_PLLCFG_REG, val);
 	} else
 		clksrc |= MAX310X_CLKSRC_PLLBYP_BIT;
 
@@ -616,23 +649,20 @@ static s32 max310x_set_ref_clk(struct device *dev, struct max310x_port *s,
 
 	/* Wait for crystal */
 	if (xtal) {
-		bool stable = false;
-		unsigned int try = 0, val = 0;
+		int ret;
 
-		do {
-			msleep(MAX310X_XTAL_WAIT_DELAY_MS);
-			regmap_read(s->regmap, MAX310X_STS_IRQSTS_REG, &val);
-
-			if (val & MAX310X_STS_CLKREADY_BIT)
-				stable = true;
-		} while (!stable && (++try < MAX310X_XTAL_WAIT_RETRIES));
-
-		if (!stable)
+		ret = regmap_read_poll_timeout(s->regmap, MAX310X_STS_IRQSTS_REG,
+					       val, val & MAX310X_STS_CLKREADY_BIT,
+					       MAX310X_XTAL_SLEEP_US,
+					       MAX310X_XTAL_TIMEOUT_US);
+		if (ret)
 			return dev_err_probe(dev, -EAGAIN,
 					     "clock is not stable\n");
 	}
 
-	return bestfreq;
+	*fref = cfg.fref;
+
+	return 0;
 }
 
 static void max310x_batch_write(struct uart_port *port, u8 *txbuf, unsigned int len)
@@ -748,8 +778,6 @@ static void max310x_handle_rx(struct uart_port *port, unsigned int rxlen)
 static void max310x_handle_tx(struct uart_port *port)
 {
 	struct tty_port *tport = &port->state->port;
-	unsigned int txlen, to_send;
-	unsigned char *tail;
 
 	if (unlikely(port->x_char)) {
 		max310x_port_write(port, MAX310X_THR_REG, port->x_char);
@@ -766,6 +794,9 @@ static void max310x_handle_tx(struct uart_port *port)
 	 * We could do that in one SPI transaction, but meh.
 	 */
 	while (!kfifo_is_empty(&tport->xmit_fifo)) {
+		unsigned int txlen, to_send;
+		unsigned char *tail;
+
 		/* Limit to space available in TX FIFO */
 		txlen = max310x_port_read(port, MAX310X_TXFIFOLVL_REG);
 		txlen = port->fifosize - txlen;
@@ -1268,9 +1299,10 @@ static int max310x_probe(struct device *dev, const struct max310x_devtype *devty
 			 const struct max310x_if_cfg *if_cfg,
 			 struct regmap *regmaps[], int irq)
 {
-	int i, ret, fmin, fmax, freq;
+	unsigned int fmin, fmax, freq;
+	int i, ret;
 	struct max310x_port *s;
-	s32 uartclk = 0;
+	unsigned int uartclk = 0;
 	bool xtal;
 
 	for (i = 0; i < devtype->nr; i++)
@@ -1330,8 +1362,7 @@ static int max310x_probe(struct device *dev, const struct max310x_devtype *devty
 		goto out_clk;
 
 	for (i = 0; i < devtype->nr; i++) {
-		bool started = false;
-		unsigned int try = 0, val = 0;
+		unsigned int val;
 
 		/* Reset port */
 		regmap_write(regmaps[i], MAX310X_MODE2_REG,
@@ -1340,15 +1371,11 @@ static int max310x_probe(struct device *dev, const struct max310x_devtype *devty
 		regmap_write(regmaps[i], MAX310X_MODE2_REG, 0);
 
 		/* Wait for port startup */
-		do {
-			msleep(MAX310X_PORT_STARTUP_WAIT_DELAY_MS);
-			regmap_read(regmaps[i], MAX310X_BRGDIVLSB_REG, &val);
-
-			if (val == 0x01)
-				started = true;
-		} while (!started && (++try < MAX310X_PORT_STARTUP_WAIT_RETRIES));
-
-		if (!started) {
+		ret = regmap_read_poll_timeout(regmaps[i], MAX310X_BRGDIVLSB_REG,
+					       val, val == 0x01,
+					       MAX310X_PORT_STARTUP_SLEEP_US,
+					       MAX310X_PORT_STARTUP_TIMEOUT_US);
+		if (ret) {
 			ret = dev_err_probe(dev, -EAGAIN, "port reset failed\n");
 			goto out_uart;
 		}
@@ -1356,11 +1383,9 @@ static int max310x_probe(struct device *dev, const struct max310x_devtype *devty
 		regmap_write(regmaps[i], MAX310X_MODE1_REG, devtype->mode1);
 	}
 
-	uartclk = max310x_set_ref_clk(dev, s, freq, xtal);
-	if (uartclk < 0) {
-		ret = uartclk;
+	ret = max310x_set_ref_clk(dev, s, freq, &uartclk, xtal);
+	if (ret < 0)
 		goto out_uart;
-	}
 
 	dev_dbg(dev, "Reference clock set to %i Hz\n", uartclk);
 
diff --git a/drivers/tty/serial/mxs-auart.c b/drivers/tty/serial/mxs-auart.c
index 693b491f1e755..697318dbb146b 100644
--- a/drivers/tty/serial/mxs-auart.c
+++ b/drivers/tty/serial/mxs-auart.c
@@ -1520,7 +1520,7 @@ static int mxs_auart_init_gpios(struct mxs_auart_port *s, struct device *dev)
 
 	for (i = 0; i < UART_GPIO_MAX; i++) {
 		gpiod = mctrl_gpio_to_gpiod(s->gpios, i);
-		if (gpiod && (gpiod_get_direction(gpiod) == 1))
+		if (gpiod && (gpiod_get_direction(gpiod) == GPIO_LINE_DIRECTION_IN))
 			s->gpio_irq[i] = gpiod_to_irq(gpiod);
 		else
 			s->gpio_irq[i] = -EINVAL;
diff --git a/drivers/tty/serial/omap-serial.c b/drivers/tty/serial/omap-serial.c
index 0b85f47ff19e0..a689d190940cf 100644
--- a/drivers/tty/serial/omap-serial.c
+++ b/drivers/tty/serial/omap-serial.c
@@ -1092,7 +1092,7 @@ static void __maybe_unused wait_for_xmitr(struct uart_omap_port *up)
 	} while (!uart_lsr_tx_empty(status));
 
 	/* Wait up to 1s for flow control if necessary */
-	if (up->port.flags & UPF_CONS_FLOW) {
+	if (uart_cons_flow_enabled(&up->port)) {
 		for (tmout = 1000000; tmout; tmout--) {
 			unsigned int msr = serial_in(up, UART_MSR);
 
diff --git a/drivers/tty/serial/pch_uart.c b/drivers/tty/serial/pch_uart.c
index 6729d8e83c3c5..80e31c4d9536d 100644
--- a/drivers/tty/serial/pch_uart.c
+++ b/drivers/tty/serial/pch_uart.c
@@ -689,8 +689,7 @@ static void pch_request_dma(struct uart_port *port)
 	if (!chan) {
 		dev_err(priv->port.dev, "%s:dma_request_channel FAILS(Tx)\n",
 			__func__);
-		pci_dev_put(dma_dev);
-		return;
+		goto err_pci_get;
 	}
 	priv->chan_tx = chan;
 
@@ -704,18 +703,26 @@ static void pch_request_dma(struct uart_port *port)
 	if (!chan) {
 		dev_err(priv->port.dev, "%s:dma_request_channel FAILS(Rx)\n",
 			__func__);
-		dma_release_channel(priv->chan_tx);
-		priv->chan_tx = NULL;
-		pci_dev_put(dma_dev);
-		return;
+		goto err_req_tx;
 	}
 
 	/* Get Consistent memory for DMA */
 	priv->rx_buf_virt = dma_alloc_coherent(port->dev, port->fifosize,
 				    &priv->rx_buf_dma, GFP_KERNEL);
+	if (!priv->rx_buf_virt)
+		goto err_req_rx;
 	priv->chan_rx = chan;
 
 	pci_dev_put(dma_dev);
+	return;
+
+err_req_rx:
+	dma_release_channel(chan);
+err_req_tx:
+	dma_release_channel(priv->chan_tx);
+	priv->chan_tx = NULL;
+err_pci_get:
+	pci_dev_put(dma_dev);
 }
 
 static void pch_dma_rx_complete(void *arg)
@@ -1444,7 +1451,7 @@ static void wait_for_xmitr(struct eg20t_port *up, int bits)
 	}
 
 	/* Wait up to 1s for flow control if necessary */
-	if (up->port.flags & UPF_CONS_FLOW) {
+	if (uart_cons_flow_enabled(&up->port)) {
 		unsigned int tmout;
 		for (tmout = 1000000; tmout; tmout--) {
 			unsigned int msr = ioread8(up->membase + UART_MSR);
diff --git a/drivers/tty/serial/pxa.c b/drivers/tty/serial/pxa.c
index fea0255067ccd..10fc8990579b5 100644
--- a/drivers/tty/serial/pxa.c
+++ b/drivers/tty/serial/pxa.c
@@ -573,7 +573,7 @@ static void wait_for_xmitr(struct uart_pxa_port *up)
 	} while (!uart_lsr_tx_empty(status));
 
 	/* Wait up to 1s for flow control if necessary */
-	if (up->port.flags & UPF_CONS_FLOW) {
+	if (uart_cons_flow_enabled(&up->port)) {
 		tmout = 1000000;
 		while (--tmout &&
 		       ((serial_in(up, UART_MSR) & UART_MSR_CTS) == 0))
diff --git a/drivers/tty/serial/qcom_geni_serial.c b/drivers/tty/serial/qcom_geni_serial.c
index b365dd5da3cb7..d81b539cff7f0 100644
--- a/drivers/tty/serial/qcom_geni_serial.c
+++ b/drivers/tty/serial/qcom_geni_serial.c
@@ -50,7 +50,7 @@
 #define TX_STOP_BIT_LEN_2		2
 
 /* SE_UART_RX_TRANS_CFG */
-#define UART_RX_PAR_EN			BIT(3)
+#define UART_RX_PAR_EN			BIT(4)
 
 /* SE_UART_RX_WORD_LEN */
 #define RX_WORD_LEN_MASK		GENMASK(9, 0)
@@ -1031,8 +1031,20 @@ static void qcom_geni_serial_handle_tx_dma(struct uart_port *uport)
 {
 	struct qcom_geni_serial_port *port = to_dev_port(uport);
 	struct tty_port *tport = &uport->state->port;
+	unsigned int fifo_len = kfifo_len(&tport->xmit_fifo);
+
+	/*
+	 * Only advance the kfifo if it still contains the bytes that were
+	 * transferred. uart_flush_buffer() may have run before this IRQ
+	 * fired: it calls kfifo_reset() under the port lock, making
+	 * fifo_len = 0 while tx_remaining remains non-zero. Calling
+	 * uart_xmit_advance() in that case would underflow kfifo->out past
+	 * kfifo->in, making kfifo_len() wrap to UART_XMIT_SIZE - tx_remaining
+	 * and triggering a spurious large DMA transfer of stale data.
+	 */
+	if (fifo_len >= port->tx_remaining)
+		uart_xmit_advance(uport, port->tx_remaining);
 
-	uart_xmit_advance(uport, port->tx_remaining);
 	geni_se_tx_dma_unprep(&port->se, port->tx_dma_addr, port->tx_remaining);
 	port->tx_dma_addr = 0;
 	port->tx_remaining = 0;
@@ -1992,6 +2004,7 @@ static int qcom_geni_serial_resume(struct device *dev)
 	return ret;
 }
 
+#if IS_ENABLED(CONFIG_SERIAL_QCOM_GENI_CONSOLE)
 static const struct qcom_geni_device_data qcom_geni_console_data = {
 	.console = true,
 	.mode = GENI_SE_FIFO,
@@ -2000,14 +2013,6 @@ static const struct qcom_geni_device_data qcom_geni_console_data = {
 	.power_state = geni_serial_resource_state,
 };
 
-static const struct qcom_geni_device_data qcom_geni_uart_data = {
-	.console = false,
-	.mode = GENI_SE_DMA,
-	.resources_init = geni_serial_resource_init,
-	.set_rate = geni_serial_set_rate,
-	.power_state = geni_serial_resource_state,
-};
-
 static const struct qcom_geni_device_data sa8255p_qcom_geni_console_data = {
 	.console = true,
 	.mode = GENI_SE_FIFO,
@@ -2019,6 +2024,15 @@ static const struct qcom_geni_device_data sa8255p_qcom_geni_console_data = {
 	.resources_init = geni_serial_pwr_init,
 	.set_rate = geni_serial_set_level,
 };
+#endif
+
+static const struct qcom_geni_device_data qcom_geni_uart_data = {
+	.console = false,
+	.mode = GENI_SE_DMA,
+	.resources_init = geni_serial_resource_init,
+	.set_rate = geni_serial_set_rate,
+	.power_state = geni_serial_resource_state,
+};
 
 static const struct qcom_geni_device_data sa8255p_qcom_geni_uart_data = {
 	.console = false,
@@ -2039,6 +2053,7 @@ static const struct dev_pm_ops qcom_geni_serial_pm_ops = {
 };
 
 static const struct of_device_id qcom_geni_serial_match_table[] = {
+#if IS_ENABLED(CONFIG_SERIAL_QCOM_GENI_CONSOLE)
 	{
 		.compatible = "qcom,geni-debug-uart",
 		.data = &qcom_geni_console_data,
@@ -2047,6 +2062,7 @@ static const struct of_device_id qcom_geni_serial_match_table[] = {
 		.compatible = "qcom,sa8255p-geni-debug-uart",
 		.data = &sa8255p_qcom_geni_console_data,
 	},
+#endif
 	{
 		.compatible = "qcom,geni-uart",
 		.data = &qcom_geni_uart_data,
diff --git a/drivers/tty/serial/samsung_tty.c b/drivers/tty/serial/samsung_tty.c
index e27806bf2cf3e..2f94fc798cffb 100644
--- a/drivers/tty/serial/samsung_tty.c
+++ b/drivers/tty/serial/samsung_tty.c
@@ -319,7 +319,7 @@ static void s3c24xx_serial_stop_tx(struct uart_port *port)
 	ourport->tx_enabled = 0;
 	ourport->tx_in_progress = 0;
 
-	if (port->flags & UPF_CONS_FLOW)
+	if (uart_cons_flow_enabled(port))
 		s3c24xx_serial_rx_enable(port);
 
 	ourport->tx_mode = 0;
@@ -493,7 +493,7 @@ static void s3c24xx_serial_start_tx(struct uart_port *port)
 	struct tty_port *tport = &port->state->port;
 
 	if (!ourport->tx_enabled) {
-		if (port->flags & UPF_CONS_FLOW)
+		if (uart_cons_flow_enabled(port))
 			s3c24xx_serial_rx_disable(port);
 
 		ourport->tx_enabled = 1;
@@ -781,7 +781,7 @@ static void s3c24xx_serial_rx_drain_fifo(struct s3c24xx_uart_port *ourport)
 		uerstat = rd_regl(port, S3C2410_UERSTAT);
 		ch = rd_reg(port, S3C2410_URXH);
 
-		if (port->flags & UPF_CONS_FLOW) {
+		if (uart_cons_flow_enabled(port)) {
 			bool txe = s3c24xx_serial_txempty_nofifo(port);
 
 			if (ourport->rx_enabled) {
@@ -1830,7 +1830,7 @@ static int s3c24xx_serial_init_port(struct s3c24xx_uart_port *ourport,
 
 	if (cfg->uart_flags & UPF_CONS_FLOW) {
 		dev_dbg(port->dev, "enabling flow control\n");
-		port->flags |= UPF_CONS_FLOW;
+		uart_set_cons_flow_enabled(port, true);
 	}
 
 	/* sort our the physical and virtual addresses for each UART */
diff --git a/drivers/tty/serial/serial_base_bus.c b/drivers/tty/serial/serial_base_bus.c
index a12935f6b9922..5f23284a8778b 100644
--- a/drivers/tty/serial/serial_base_bus.c
+++ b/drivers/tty/serial/serial_base_bus.c
@@ -74,7 +74,7 @@ static int serial_base_device_init(struct uart_port *port,
 	dev->parent = parent_dev;
 	dev->bus = &serial_base_bus_type;
 	dev->release = release;
-	dev->of_node_reused = true;
+	dev_set_of_node_reused(dev);
 
 	device_set_node(dev, fwnode_handle_get(dev_fwnode(parent_dev)));
 
diff --git a/drivers/tty/serial/serial_core.c b/drivers/tty/serial/serial_core.c
index 89cebdd278410..075a69164aa7c 100644
--- a/drivers/tty/serial/serial_core.c
+++ b/drivers/tty/serial/serial_core.c
@@ -462,10 +462,10 @@ EXPORT_SYMBOL(uart_update_timeout);
  *
  * Decode the termios structure into a numeric baud rate, taking account of the
  * magic 38400 baud rate (with spd_* flags), and mapping the %B0 rate to 9600
- * baud.
+ * baud or min argument, whichever is greater.
  *
  * If the new baud rate is invalid, try the @old termios setting. If it's still
- * invalid, we try 9600 baud. If that is also invalid 0 is returned.
+ * invalid, clip to the nearest chip supported rate.
  *
  * The @termios structure is updated to reflect the baud rate we're actually
  * going to be using. Don't do this for the case where B0 is requested ("hang
@@ -480,7 +480,6 @@ uart_get_baud_rate(struct uart_port *port, struct ktermios *termios,
 	unsigned int try;
 	unsigned int baud;
 	unsigned int altbaud;
-	int hung_up = 0;
 	upf_t flags = port->flags & UPF_SPD_MASK;
 
 	switch (flags) {
@@ -514,24 +513,20 @@ uart_get_baud_rate(struct uart_port *port, struct ktermios *termios,
 		/*
 		 * Special case: B0 rate.
 		 */
-		if (baud == 0) {
-			hung_up = 1;
-			baud = 9600;
-		}
+		if (baud == 0)
+			return max(min, 9600);
 
 		if (baud >= min && baud <= max)
 			return baud;
 
 		/*
-		 * Oops, the quotient was zero.  Try again with
+		 * If the range cannot be met then try again with
 		 * the old baud rate if possible.
 		 */
 		termios->c_cflag &= ~CBAUD;
 		if (old) {
 			baud = tty_termios_baud_rate(old);
-			if (!hung_up)
-				tty_termios_encode_baud_rate(termios,
-								baud, baud);
+			tty_termios_encode_baud_rate(termios, baud, baud);
 			old = NULL;
 			continue;
 		}
@@ -540,15 +535,16 @@ uart_get_baud_rate(struct uart_port *port, struct ktermios *termios,
 		 * As a last resort, if the range cannot be met then clip to
 		 * the nearest chip supported rate.
 		 */
-		if (!hung_up) {
-			if (baud <= min)
-				tty_termios_encode_baud_rate(termios,
-							min + 1, min + 1);
-			else
-				tty_termios_encode_baud_rate(termios,
-							max - 1, max - 1);
-		}
+		if (baud <= min)
+			baud = min + 1;
+		else
+			baud = max - 1;
+
+		tty_termios_encode_baud_rate(termios, baud, baud);
 	}
+
+	/* Should never happen */
+	WARN_ON(1);
 	return 0;
 }
 EXPORT_SYMBOL(uart_get_baud_rate);
diff --git a/drivers/tty/serial/serial_txx9.c b/drivers/tty/serial/serial_txx9.c
index 436a559234dfe..4ae9a45c8e3a3 100644
--- a/drivers/tty/serial/serial_txx9.c
+++ b/drivers/tty/serial/serial_txx9.c
@@ -422,7 +422,7 @@ static void wait_for_xmitr(struct uart_port *up)
 		udelay(1);
 
 	/* Wait up to 1s for flow control if necessary */
-	if (up->flags & UPF_CONS_FLOW) {
+	if (uart_cons_flow_enabled(up)) {
 		tmout = 1000000;
 		while (--tmout &&
 		       (sio_in(up, TXX9_SICISR) & TXX9_SICISR_CTSS))
@@ -857,7 +857,7 @@ serial_txx9_console_write(struct console *co, const char *s, unsigned int count)
 	 *	Disable flow-control if enabled (and unnecessary)
 	 */
 	flcr = sio_in(up, TXX9_SIFLCR);
-	if (!(up->flags & UPF_CONS_FLOW) && (flcr & TXX9_SIFLCR_TES))
+	if (!uart_cons_flow_enabled(up) && (flcr & TXX9_SIFLCR_TES))
 		sio_out(up, TXX9_SIFLCR, flcr & ~TXX9_SIFLCR_TES);
 
 	uart_console_write(up, s, count, serial_txx9_console_putchar);
diff --git a/drivers/tty/serial/sh-sci.c b/drivers/tty/serial/sh-sci.c
index 6c819b6b24258..787e7cdc5e9c3 100644
--- a/drivers/tty/serial/sh-sci.c
+++ b/drivers/tty/serial/sh-sci.c
@@ -3025,7 +3025,7 @@ int sci_request_port(struct uart_port *port)
 
 	ret = sci_remap_port(port);
 	if (unlikely(ret != 0)) {
-		release_resource(res);
+		release_mem_region(port->mapbase, sport->reg_size);
 		return ret;
 	}
 
@@ -3369,7 +3369,7 @@ static int sci_init_single(struct platform_device *dev,
 	}
 
 	port->type		= SCI_PUBLIC_PORT_ID(p->type);
-	port->flags		= UPF_FIXED_PORT | UPF_BOOT_AUTOCONF | p->flags;
+	port->flags		= UPF_FIXED_PORT | UPF_BOOT_AUTOCONF;
 	port->fifosize		= sci_port->params->fifosize;
 
 	if (p->type == PORT_SCI && !dev->dev.of_node) {
diff --git a/drivers/tty/serial/sunsu.c b/drivers/tty/serial/sunsu.c
index 6505a1930da9a..4ff3e5c41dab9 100644
--- a/drivers/tty/serial/sunsu.c
+++ b/drivers/tty/serial/sunsu.c
@@ -1245,7 +1245,7 @@ static void wait_for_xmitr(struct uart_sunsu_port *up)
 	} while (!uart_lsr_tx_empty(status));
 
 	/* Wait up to 1s for flow control if necessary */
-	if (up->port.flags & UPF_CONS_FLOW) {
+	if (uart_cons_flow_enabled(&up->port)) {
 		tmout = 1000000;
 		while (--tmout &&
 		       ((serial_in(up, UART_MSR) & UART_MSR_CTS) == 0))
diff --git a/drivers/tty/serial/zs.c b/drivers/tty/serial/zs.c
index 72a3c0d90f409..31e12645f66b1 100644
--- a/drivers/tty/serial/zs.c
+++ b/drivers/tty/serial/zs.c
@@ -680,9 +680,9 @@ static void zs_status_handle(struct zs_port *zport, struct zs_port *zport_a)
 			uart_handle_dcd_change(uport,
 					       zport->mctrl & TIOCM_CAR);
 		if (delta & TIOCM_RNG)
-			uport->icount.dsr++;
-		if (delta & TIOCM_DSR)
 			uport->icount.rng++;
+		if (delta & TIOCM_DSR)
+			uport->icount.dsr++;
 
 		if (delta)
 			wake_up_interruptible(&uport->state->port.delta_msr_wait);
diff --git a/drivers/tty/synclink_gt.c b/drivers/tty/synclink_gt.c
deleted file mode 100644
index bf4f50e0ce94a..0000000000000
--- a/drivers/tty/synclink_gt.c
+++ /dev/null
@@ -1,5038 +0,0 @@
-// SPDX-License-Identifier: GPL-1.0+
-/*
- * Device driver for Microgate SyncLink GT serial adapters.
- *
- * written by Paul Fulghum for Microgate Corporation
- * paulkf@microgate.com
- *
- * Microgate and SyncLink are trademarks of Microgate Corporation
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
- * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
- * OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * DEBUG OUTPUT DEFINITIONS
- *
- * uncomment lines below to enable specific types of debug output
- *
- * DBGINFO   information - most verbose output
- * DBGERR    serious errors
- * DBGBH     bottom half service routine debugging
- * DBGISR    interrupt service routine debugging
- * DBGDATA   output receive and transmit data
- * DBGTBUF   output transmit DMA buffers and registers
- * DBGRBUF   output receive DMA buffers and registers
- */
-
-#define DBGINFO(fmt) if (debug_level >= DEBUG_LEVEL_INFO) printk fmt
-#define DBGERR(fmt) if (debug_level >= DEBUG_LEVEL_ERROR) printk fmt
-#define DBGBH(fmt) if (debug_level >= DEBUG_LEVEL_BH) printk fmt
-#define DBGISR(fmt) if (debug_level >= DEBUG_LEVEL_ISR) printk fmt
-#define DBGDATA(info, buf, size, label) if (debug_level >= DEBUG_LEVEL_DATA) trace_block((info), (buf), (size), (label))
-/*#define DBGTBUF(info) dump_tbufs(info)*/
-/*#define DBGRBUF(info) dump_rbufs(info)*/
-
-
-#include <linux/module.h>
-#include <linux/errno.h>
-#include <linux/signal.h>
-#include <linux/sched.h>
-#include <linux/timer.h>
-#include <linux/interrupt.h>
-#include <linux/pci.h>
-#include <linux/tty.h>
-#include <linux/tty_flip.h>
-#include <linux/serial.h>
-#include <linux/major.h>
-#include <linux/string.h>
-#include <linux/fcntl.h>
-#include <linux/ptrace.h>
-#include <linux/ioport.h>
-#include <linux/mm.h>
-#include <linux/seq_file.h>
-#include <linux/slab.h>
-#include <linux/netdevice.h>
-#include <linux/vmalloc.h>
-#include <linux/init.h>
-#include <linux/delay.h>
-#include <linux/ioctl.h>
-#include <linux/termios.h>
-#include <linux/bitops.h>
-#include <linux/workqueue.h>
-#include <linux/hdlc.h>
-#include <linux/synclink.h>
-
-#include <asm/io.h>
-#include <asm/irq.h>
-#include <asm/dma.h>
-#include <asm/types.h>
-#include <linux/uaccess.h>
-
-#if defined(CONFIG_HDLC) || (defined(CONFIG_HDLC_MODULE) && defined(CONFIG_SYNCLINK_GT_MODULE))
-#define SYNCLINK_GENERIC_HDLC 1
-#else
-#define SYNCLINK_GENERIC_HDLC 0
-#endif
-
-/*
- * module identification
- */
-static const char driver_name[] = "SyncLink GT";
-static const char tty_dev_prefix[] = "ttySLG";
-MODULE_DESCRIPTION("Device driver for Microgate SyncLink GT serial adapters");
-MODULE_LICENSE("GPL");
-#define MAX_DEVICES 32
-
-static const struct pci_device_id pci_table[] = {
-	{ PCI_VDEVICE(MICROGATE, SYNCLINK_GT_DEVICE_ID) },
-	{ PCI_VDEVICE(MICROGATE, SYNCLINK_GT2_DEVICE_ID) },
-	{ PCI_VDEVICE(MICROGATE, SYNCLINK_GT4_DEVICE_ID) },
-	{ PCI_VDEVICE(MICROGATE, SYNCLINK_AC_DEVICE_ID) },
-	{ 0 }, /* terminate list */
-};
-MODULE_DEVICE_TABLE(pci, pci_table);
-
-static int  init_one(struct pci_dev *dev,const struct pci_device_id *ent);
-static void remove_one(struct pci_dev *dev);
-static struct pci_driver pci_driver = {
-	.name		= "synclink_gt",
-	.id_table	= pci_table,
-	.probe		= init_one,
-	.remove		= remove_one,
-};
-
-static bool pci_registered;
-
-/*
- * module configuration and status
- */
-static struct slgt_info *slgt_device_list;
-static int slgt_device_count;
-
-static int ttymajor;
-static int debug_level;
-static int maxframe[MAX_DEVICES];
-
-module_param(ttymajor, int, 0);
-module_param(debug_level, int, 0);
-module_param_array(maxframe, int, NULL, 0);
-
-MODULE_PARM_DESC(ttymajor, "TTY major device number override: 0=auto assigned");
-MODULE_PARM_DESC(debug_level, "Debug syslog output: 0=disabled, 1 to 5=increasing detail");
-MODULE_PARM_DESC(maxframe, "Maximum frame size used by device (4096 to 65535)");
-
-/*
- * tty support and callbacks
- */
-static struct tty_driver *serial_driver;
-
-static void wait_until_sent(struct tty_struct *tty, int timeout);
-static void flush_buffer(struct tty_struct *tty);
-static void tx_release(struct tty_struct *tty);
-
-/*
- * generic HDLC support
- */
-#define dev_to_port(D) (dev_to_hdlc(D)->priv)
-
-
-/*
- * device specific structures, macros and functions
- */
-
-#define SLGT_MAX_PORTS 4
-#define SLGT_REG_SIZE  256
-
-/*
- * conditional wait facility
- */
-struct cond_wait {
-	struct cond_wait *next;
-	wait_queue_head_t q;
-	wait_queue_entry_t wait;
-	unsigned int data;
-};
-static void flush_cond_wait(struct cond_wait **head);
-
-/*
- * DMA buffer descriptor and access macros
- */
-struct slgt_desc
-{
-	__le16 count;
-	__le16 status;
-	__le32 pbuf;  /* physical address of data buffer */
-	__le32 next;  /* physical address of next descriptor */
-
-	/* driver book keeping */
-	char *buf;          /* virtual  address of data buffer */
-    	unsigned int pdesc; /* physical address of this descriptor */
-	dma_addr_t buf_dma_addr;
-	unsigned short buf_count;
-};
-
-#define set_desc_buffer(a,b) (a).pbuf = cpu_to_le32((unsigned int)(b))
-#define set_desc_next(a,b) (a).next   = cpu_to_le32((unsigned int)(b))
-#define set_desc_count(a,b)(a).count  = cpu_to_le16((unsigned short)(b))
-#define set_desc_eof(a,b)  (a).status = cpu_to_le16((b) ? (le16_to_cpu((a).status) | BIT0) : (le16_to_cpu((a).status) & ~BIT0))
-#define set_desc_status(a, b) (a).status = cpu_to_le16((unsigned short)(b))
-#define desc_count(a)      (le16_to_cpu((a).count))
-#define desc_status(a)     (le16_to_cpu((a).status))
-#define desc_complete(a)   (le16_to_cpu((a).status) & BIT15)
-#define desc_eof(a)        (le16_to_cpu((a).status) & BIT2)
-#define desc_crc_error(a)  (le16_to_cpu((a).status) & BIT1)
-#define desc_abort(a)      (le16_to_cpu((a).status) & BIT0)
-#define desc_residue(a)    ((le16_to_cpu((a).status) & 0x38) >> 3)
-
-struct _input_signal_events {
-	int ri_up;
-	int ri_down;
-	int dsr_up;
-	int dsr_down;
-	int dcd_up;
-	int dcd_down;
-	int cts_up;
-	int cts_down;
-};
-
-/*
- * device instance data structure
- */
-struct slgt_info {
-	void *if_ptr;		/* General purpose pointer (used by SPPP) */
-	struct tty_port port;
-
-	struct slgt_info *next_device;	/* device list link */
-
-	char device_name[25];
-	struct pci_dev *pdev;
-
-	int port_count;  /* count of ports on adapter */
-	int adapter_num; /* adapter instance number */
-	int port_num;    /* port instance number */
-
-	/* array of pointers to port contexts on this adapter */
-	struct slgt_info *port_array[SLGT_MAX_PORTS];
-
-	int			line;		/* tty line instance number */
-
-	struct mgsl_icount	icount;
-
-	int			timeout;
-	int			x_char;		/* xon/xoff character */
-	unsigned int		read_status_mask;
-	unsigned int 		ignore_status_mask;
-
-	wait_queue_head_t	status_event_wait_q;
-	wait_queue_head_t	event_wait_q;
-	struct timer_list	tx_timer;
-	struct timer_list	rx_timer;
-
-	unsigned int            gpio_present;
-	struct cond_wait        *gpio_wait_q;
-
-	spinlock_t lock;	/* spinlock for synchronizing with ISR */
-
-	struct work_struct task;
-	u32 pending_bh;
-	bool bh_requested;
-	bool bh_running;
-
-	int isr_overflow;
-	bool irq_requested;	/* true if IRQ requested */
-	bool irq_occurred;	/* for diagnostics use */
-
-	/* device configuration */
-
-	unsigned int bus_type;
-	unsigned int irq_level;
-	unsigned long irq_flags;
-
-	unsigned char __iomem * reg_addr;  /* memory mapped registers address */
-	u32 phys_reg_addr;
-	bool reg_addr_requested;
-
-	MGSL_PARAMS params;       /* communications parameters */
-	u32 idle_mode;
-	u32 max_frame_size;       /* as set by device config */
-
-	unsigned int rbuf_fill_level;
-	unsigned int rx_pio;
-	unsigned int if_mode;
-	unsigned int base_clock;
-	unsigned int xsync;
-	unsigned int xctrl;
-
-	/* device status */
-
-	bool rx_enabled;
-	bool rx_restart;
-
-	bool tx_enabled;
-	bool tx_active;
-
-	unsigned char signals;    /* serial signal states */
-	int init_error;  /* initialization error */
-
-	unsigned char *tx_buf;
-	int tx_count;
-
-	bool drop_rts_on_tx_done;
-	struct	_input_signal_events	input_signal_events;
-
-	int dcd_chkcount;	/* check counts to prevent */
-	int cts_chkcount;	/* too many IRQs if a signal */
-	int dsr_chkcount;	/* is floating */
-	int ri_chkcount;
-
-	char *bufs;		/* virtual address of DMA buffer lists */
-	dma_addr_t bufs_dma_addr; /* physical address of buffer descriptors */
-
-	unsigned int rbuf_count;
-	struct slgt_desc *rbufs;
-	unsigned int rbuf_current;
-	unsigned int rbuf_index;
-	unsigned int rbuf_fill_index;
-	unsigned short rbuf_fill_count;
-
-	unsigned int tbuf_count;
-	struct slgt_desc *tbufs;
-	unsigned int tbuf_current;
-	unsigned int tbuf_start;
-
-	unsigned char *tmp_rbuf;
-	unsigned int tmp_rbuf_count;
-
-	/* SPPP/Cisco HDLC device parts */
-
-	int netcount;
-	spinlock_t netlock;
-#if SYNCLINK_GENERIC_HDLC
-	struct net_device *netdev;
-#endif
-
-};
-
-static const MGSL_PARAMS default_params = {
-	.mode            = MGSL_MODE_HDLC,
-	.loopback        = 0,
-	.flags           = HDLC_FLAG_UNDERRUN_ABORT15,
-	.encoding        = HDLC_ENCODING_NRZI_SPACE,
-	.clock_speed     = 0,
-	.addr_filter     = 0xff,
-	.crc_type        = HDLC_CRC_16_CCITT,
-	.preamble_length = HDLC_PREAMBLE_LENGTH_8BITS,
-	.preamble        = HDLC_PREAMBLE_PATTERN_NONE,
-	.data_rate       = 9600,
-	.data_bits       = 8,
-	.stop_bits       = 1,
-	.parity          = ASYNC_PARITY_NONE
-};
-
-
-#define BH_RECEIVE  1
-#define BH_TRANSMIT 2
-#define BH_STATUS   4
-#define IO_PIN_SHUTDOWN_LIMIT 100
-
-#define DMABUFSIZE 256
-#define DESC_LIST_SIZE 4096
-
-#define MASK_PARITY  BIT1
-#define MASK_FRAMING BIT0
-#define MASK_BREAK   BIT14
-#define MASK_OVERRUN BIT4
-
-#define GSR   0x00 /* global status */
-#define JCR   0x04 /* JTAG control */
-#define IODR  0x08 /* GPIO direction */
-#define IOER  0x0c /* GPIO interrupt enable */
-#define IOVR  0x10 /* GPIO value */
-#define IOSR  0x14 /* GPIO interrupt status */
-#define TDR   0x80 /* tx data */
-#define RDR   0x80 /* rx data */
-#define TCR   0x82 /* tx control */
-#define TIR   0x84 /* tx idle */
-#define TPR   0x85 /* tx preamble */
-#define RCR   0x86 /* rx control */
-#define VCR   0x88 /* V.24 control */
-#define CCR   0x89 /* clock control */
-#define BDR   0x8a /* baud divisor */
-#define SCR   0x8c /* serial control */
-#define SSR   0x8e /* serial status */
-#define RDCSR 0x90 /* rx DMA control/status */
-#define TDCSR 0x94 /* tx DMA control/status */
-#define RDDAR 0x98 /* rx DMA descriptor address */
-#define TDDAR 0x9c /* tx DMA descriptor address */
-#define XSR   0x40 /* extended sync pattern */
-#define XCR   0x44 /* extended control */
-
-#define RXIDLE      BIT14
-#define RXBREAK     BIT14
-#define IRQ_TXDATA  BIT13
-#define IRQ_TXIDLE  BIT12
-#define IRQ_TXUNDER BIT11 /* HDLC */
-#define IRQ_RXDATA  BIT10
-#define IRQ_RXIDLE  BIT9  /* HDLC */
-#define IRQ_RXBREAK BIT9  /* async */
-#define IRQ_RXOVER  BIT8
-#define IRQ_DSR     BIT7
-#define IRQ_CTS     BIT6
-#define IRQ_DCD     BIT5
-#define IRQ_RI      BIT4
-#define IRQ_ALL     0x3ff0
-#define IRQ_MASTER  BIT0
-
-#define slgt_irq_on(info, mask) \
-	wr_reg16((info), SCR, (unsigned short)(rd_reg16((info), SCR) | (mask)))
-#define slgt_irq_off(info, mask) \
-	wr_reg16((info), SCR, (unsigned short)(rd_reg16((info), SCR) & ~(mask)))
-
-static __u8  rd_reg8(struct slgt_info *info, unsigned int addr);
-static void  wr_reg8(struct slgt_info *info, unsigned int addr, __u8 value);
-static __u16 rd_reg16(struct slgt_info *info, unsigned int addr);
-static void  wr_reg16(struct slgt_info *info, unsigned int addr, __u16 value);
-static __u32 rd_reg32(struct slgt_info *info, unsigned int addr);
-static void  wr_reg32(struct slgt_info *info, unsigned int addr, __u32 value);
-
-static void  msc_set_vcr(struct slgt_info *info);
-
-static int  startup_hw(struct slgt_info *info);
-static int  block_til_ready(struct tty_struct *tty, struct file * filp,struct slgt_info *info);
-static void shutdown_hw(struct slgt_info *info);
-static void program_hw(struct slgt_info *info);
-static void change_params(struct slgt_info *info);
-
-static int  adapter_test(struct slgt_info *info);
-
-static void reset_port(struct slgt_info *info);
-static void async_mode(struct slgt_info *info);
-static void sync_mode(struct slgt_info *info);
-
-static void rx_stop(struct slgt_info *info);
-static void rx_start(struct slgt_info *info);
-static void reset_rbufs(struct slgt_info *info);
-static void free_rbufs(struct slgt_info *info, unsigned int first, unsigned int last);
-static bool rx_get_frame(struct slgt_info *info);
-static bool rx_get_buf(struct slgt_info *info);
-
-static void tx_start(struct slgt_info *info);
-static void tx_stop(struct slgt_info *info);
-static void tx_set_idle(struct slgt_info *info);
-static unsigned int tbuf_bytes(struct slgt_info *info);
-static void reset_tbufs(struct slgt_info *info);
-static void tdma_reset(struct slgt_info *info);
-static bool tx_load(struct slgt_info *info, const u8 *buf, unsigned int count);
-
-static void get_gtsignals(struct slgt_info *info);
-static void set_gtsignals(struct slgt_info *info);
-static void set_rate(struct slgt_info *info, u32 data_rate);
-
-static void bh_transmit(struct slgt_info *info);
-static void isr_txeom(struct slgt_info *info, unsigned short status);
-
-static void tx_timeout(struct timer_list *t);
-static void rx_timeout(struct timer_list *t);
-
-/*
- * ioctl handlers
- */
-static int  get_stats(struct slgt_info *info, struct mgsl_icount __user *user_icount);
-static int  get_params(struct slgt_info *info, MGSL_PARAMS __user *params);
-static int  set_params(struct slgt_info *info, MGSL_PARAMS __user *params);
-static int  get_txidle(struct slgt_info *info, int __user *idle_mode);
-static int  set_txidle(struct slgt_info *info, int idle_mode);
-static int  tx_enable(struct slgt_info *info, int enable);
-static int  tx_abort(struct slgt_info *info);
-static int  rx_enable(struct slgt_info *info, int enable);
-static int  modem_input_wait(struct slgt_info *info,int arg);
-static int  wait_mgsl_event(struct slgt_info *info, int __user *mask_ptr);
-static int  get_interface(struct slgt_info *info, int __user *if_mode);
-static int  set_interface(struct slgt_info *info, int if_mode);
-static int  set_gpio(struct slgt_info *info, struct gpio_desc __user *gpio);
-static int  get_gpio(struct slgt_info *info, struct gpio_desc __user *gpio);
-static int  wait_gpio(struct slgt_info *info, struct gpio_desc __user *gpio);
-static int  get_xsync(struct slgt_info *info, int __user *if_mode);
-static int  set_xsync(struct slgt_info *info, int if_mode);
-static int  get_xctrl(struct slgt_info *info, int __user *if_mode);
-static int  set_xctrl(struct slgt_info *info, int if_mode);
-
-/*
- * driver functions
- */
-static void release_resources(struct slgt_info *info);
-
-/*
- * DEBUG OUTPUT CODE
- */
-#ifndef DBGINFO
-#define DBGINFO(fmt)
-#endif
-#ifndef DBGERR
-#define DBGERR(fmt)
-#endif
-#ifndef DBGBH
-#define DBGBH(fmt)
-#endif
-#ifndef DBGISR
-#define DBGISR(fmt)
-#endif
-
-#ifdef DBGDATA
-static void trace_block(struct slgt_info *info, const char *data, int count, const char *label)
-{
-	int i;
-	int linecount;
-	printk("%s %s data:\n",info->device_name, label);
-	while(count) {
-		linecount = (count > 16) ? 16 : count;
-		for(i=0; i < linecount; i++)
-			printk("%02X ",(unsigned char)data[i]);
-		for(;i<17;i++)
-			printk("   ");
-		for(i=0;i<linecount;i++) {
-			if (data[i]>=040 && data[i]<=0176)
-				printk("%c",data[i]);
-			else
-				printk(".");
-		}
-		printk("\n");
-		data  += linecount;
-		count -= linecount;
-	}
-}
-#else
-#define DBGDATA(info, buf, size, label)
-#endif
-
-#ifdef DBGTBUF
-static void dump_tbufs(struct slgt_info *info)
-{
-	int i;
-	printk("tbuf_current=%d\n", info->tbuf_current);
-	for (i=0 ; i < info->tbuf_count ; i++) {
-		printk("%d: count=%04X status=%04X\n",
-			i, le16_to_cpu(info->tbufs[i].count), le16_to_cpu(info->tbufs[i].status));
-	}
-}
-#else
-#define DBGTBUF(info)
-#endif
-
-#ifdef DBGRBUF
-static void dump_rbufs(struct slgt_info *info)
-{
-	int i;
-	printk("rbuf_current=%d\n", info->rbuf_current);
-	for (i=0 ; i < info->rbuf_count ; i++) {
-		printk("%d: count=%04X status=%04X\n",
-			i, le16_to_cpu(info->rbufs[i].count), le16_to_cpu(info->rbufs[i].status));
-	}
-}
-#else
-#define DBGRBUF(info)
-#endif
-
-static inline int sanity_check(struct slgt_info *info, char *devname, const char *name)
-{
-#ifdef SANITY_CHECK
-	if (!info) {
-		printk("null struct slgt_info for (%s) in %s\n", devname, name);
-		return 1;
-	}
-#else
-	if (!info)
-		return 1;
-#endif
-	return 0;
-}
-
-/*
- * line discipline callback wrappers
- *
- * The wrappers maintain line discipline references
- * while calling into the line discipline.
- *
- * ldisc_receive_buf  - pass receive data to line discipline
- */
-static void ldisc_receive_buf(struct tty_struct *tty,
-			      const __u8 *data, char *flags, int count)
-{
-	struct tty_ldisc *ld;
-	if (!tty)
-		return;
-	ld = tty_ldisc_ref(tty);
-	if (ld) {
-		if (ld->ops->receive_buf)
-			ld->ops->receive_buf(tty, data, flags, count);
-		tty_ldisc_deref(ld);
-	}
-}
-
-/* tty callbacks */
-
-static int open(struct tty_struct *tty, struct file *filp)
-{
-	struct slgt_info *info;
-	int retval, line;
-	unsigned long flags;
-
-	line = tty->index;
-	if (line >= slgt_device_count) {
-		DBGERR(("%s: open with invalid line #%d.\n", driver_name, line));
-		return -ENODEV;
-	}
-
-	info = slgt_device_list;
-	while(info && info->line != line)
-		info = info->next_device;
-	if (sanity_check(info, tty->name, "open"))
-		return -ENODEV;
-	if (info->init_error) {
-		DBGERR(("%s init error=%d\n", info->device_name, info->init_error));
-		return -ENODEV;
-	}
-
-	tty->driver_data = info;
-	info->port.tty = tty;
-
-	DBGINFO(("%s open, old ref count = %d\n", info->device_name, info->port.count));
-
-	mutex_lock(&info->port.mutex);
-
-	spin_lock_irqsave(&info->netlock, flags);
-	if (info->netcount) {
-		retval = -EBUSY;
-		spin_unlock_irqrestore(&info->netlock, flags);
-		mutex_unlock(&info->port.mutex);
-		goto cleanup;
-	}
-	info->port.count++;
-	spin_unlock_irqrestore(&info->netlock, flags);
-
-	if (info->port.count == 1) {
-		/* 1st open on this device, init hardware */
-		retval = startup_hw(info);
-		if (retval < 0) {
-			mutex_unlock(&info->port.mutex);
-			goto cleanup;
-		}
-	}
-	mutex_unlock(&info->port.mutex);
-	retval = block_til_ready(tty, filp, info);
-	if (retval) {
-		DBGINFO(("%s block_til_ready rc=%d\n", info->device_name, retval));
-		goto cleanup;
-	}
-
-	retval = 0;
-
-cleanup:
-	if (retval) {
-		if (tty->count == 1)
-			info->port.tty = NULL; /* tty layer will release tty struct */
-		if(info->port.count)
-			info->port.count--;
-	}
-
-	DBGINFO(("%s open rc=%d\n", info->device_name, retval));
-	return retval;
-}
-
-static void close(struct tty_struct *tty, struct file *filp)
-{
-	struct slgt_info *info = tty->driver_data;
-
-	if (sanity_check(info, tty->name, "close"))
-		return;
-	DBGINFO(("%s close entry, count=%d\n", info->device_name, info->port.count));
-
-	if (tty_port_close_start(&info->port, tty, filp) == 0)
-		goto cleanup;
-
-	mutex_lock(&info->port.mutex);
-	if (tty_port_initialized(&info->port))
- 		wait_until_sent(tty, info->timeout);
-	flush_buffer(tty);
-	tty_ldisc_flush(tty);
-
-	shutdown_hw(info);
-	mutex_unlock(&info->port.mutex);
-
-	tty_port_close_end(&info->port, tty);
-	info->port.tty = NULL;
-cleanup:
-	DBGINFO(("%s close exit, count=%d\n", tty->driver->name, info->port.count));
-}
-
-static void hangup(struct tty_struct *tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "hangup"))
-		return;
-	DBGINFO(("%s hangup\n", info->device_name));
-
-	flush_buffer(tty);
-
-	mutex_lock(&info->port.mutex);
-	shutdown_hw(info);
-
-	spin_lock_irqsave(&info->port.lock, flags);
-	info->port.count = 0;
-	info->port.tty = NULL;
-	spin_unlock_irqrestore(&info->port.lock, flags);
-	tty_port_set_active(&info->port, false);
-	mutex_unlock(&info->port.mutex);
-
-	wake_up_interruptible(&info->port.open_wait);
-}
-
-static void set_termios(struct tty_struct *tty,
-			const struct ktermios *old_termios)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	DBGINFO(("%s set_termios\n", tty->driver->name));
-
-	change_params(info);
-
-	/* Handle transition to B0 status */
-	if ((old_termios->c_cflag & CBAUD) && !C_BAUD(tty)) {
-		info->signals &= ~(SerialSignal_RTS | SerialSignal_DTR);
-		spin_lock_irqsave(&info->lock,flags);
-		set_gtsignals(info);
-		spin_unlock_irqrestore(&info->lock,flags);
-	}
-
-	/* Handle transition away from B0 status */
-	if (!(old_termios->c_cflag & CBAUD) && C_BAUD(tty)) {
-		info->signals |= SerialSignal_DTR;
-		if (!C_CRTSCTS(tty) || !tty_throttled(tty))
-			info->signals |= SerialSignal_RTS;
-		spin_lock_irqsave(&info->lock,flags);
-	 	set_gtsignals(info);
-		spin_unlock_irqrestore(&info->lock,flags);
-	}
-
-	/* Handle turning off CRTSCTS */
-	if ((old_termios->c_cflag & CRTSCTS) && !C_CRTSCTS(tty)) {
-		tty->hw_stopped = false;
-		tx_release(tty);
-	}
-}
-
-static void update_tx_timer(struct slgt_info *info)
-{
-	/*
-	 * use worst case speed of 1200bps to calculate transmit timeout
-	 * based on data in buffers (tbuf_bytes) and FIFO (128 bytes)
-	 */
-	if (info->params.mode == MGSL_MODE_HDLC) {
-		int timeout  = (tbuf_bytes(info) * 7) + 1000;
-		mod_timer(&info->tx_timer, jiffies + msecs_to_jiffies(timeout));
-	}
-}
-
-static ssize_t write(struct tty_struct *tty, const u8 *buf, size_t count)
-{
-	int ret = 0;
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "write"))
-		return -EIO;
-
-	DBGINFO(("%s write count=%zu\n", info->device_name, count));
-
-	if (!info->tx_buf || (count > info->max_frame_size))
-		return -EIO;
-
-	if (!count || tty->flow.stopped || tty->hw_stopped)
-		return 0;
-
-	spin_lock_irqsave(&info->lock, flags);
-
-	if (info->tx_count) {
-		/* send accumulated data from send_char() */
-		if (!tx_load(info, info->tx_buf, info->tx_count))
-			goto cleanup;
-		info->tx_count = 0;
-	}
-
-	if (tx_load(info, buf, count))
-		ret = count;
-
-cleanup:
-	spin_unlock_irqrestore(&info->lock, flags);
-	DBGINFO(("%s write rc=%d\n", info->device_name, ret));
-	return ret;
-}
-
-static int put_char(struct tty_struct *tty, u8 ch)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-	int ret = 0;
-
-	if (sanity_check(info, tty->name, "put_char"))
-		return 0;
-	DBGINFO(("%s put_char(%u)\n", info->device_name, ch));
-	if (!info->tx_buf)
-		return 0;
-	spin_lock_irqsave(&info->lock,flags);
-	if (info->tx_count < info->max_frame_size) {
-		info->tx_buf[info->tx_count++] = ch;
-		ret = 1;
-	}
-	spin_unlock_irqrestore(&info->lock,flags);
-	return ret;
-}
-
-static void send_xchar(struct tty_struct *tty, char ch)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "send_xchar"))
-		return;
-	DBGINFO(("%s send_xchar(%d)\n", info->device_name, ch));
-	info->x_char = ch;
-	if (ch) {
-		spin_lock_irqsave(&info->lock,flags);
-		if (!info->tx_enabled)
-		 	tx_start(info);
-		spin_unlock_irqrestore(&info->lock,flags);
-	}
-}
-
-static void wait_until_sent(struct tty_struct *tty, int timeout)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long orig_jiffies, char_time;
-
-	if (!info )
-		return;
-	if (sanity_check(info, tty->name, "wait_until_sent"))
-		return;
-	DBGINFO(("%s wait_until_sent entry\n", info->device_name));
-	if (!tty_port_initialized(&info->port))
-		goto exit;
-
-	orig_jiffies = jiffies;
-
-	/* Set check interval to 1/5 of estimated time to
-	 * send a character, and make it at least 1. The check
-	 * interval should also be less than the timeout.
-	 * Note: use tight timings here to satisfy the NIST-PCTS.
-	 */
-
-	if (info->params.data_rate) {
-	       	char_time = info->timeout/(32 * 5);
-		if (!char_time)
-			char_time++;
-	} else
-		char_time = 1;
-
-	if (timeout)
-		char_time = min_t(unsigned long, char_time, timeout);
-
-	while (info->tx_active) {
-		msleep_interruptible(jiffies_to_msecs(char_time));
-		if (signal_pending(current))
-			break;
-		if (timeout && time_after(jiffies, orig_jiffies + timeout))
-			break;
-	}
-exit:
-	DBGINFO(("%s wait_until_sent exit\n", info->device_name));
-}
-
-static unsigned int write_room(struct tty_struct *tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned int ret;
-
-	if (sanity_check(info, tty->name, "write_room"))
-		return 0;
-	ret = (info->tx_active) ? 0 : HDLC_MAX_FRAME_SIZE;
-	DBGINFO(("%s write_room=%u\n", info->device_name, ret));
-	return ret;
-}
-
-static void flush_chars(struct tty_struct *tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "flush_chars"))
-		return;
-	DBGINFO(("%s flush_chars entry tx_count=%d\n", info->device_name, info->tx_count));
-
-	if (info->tx_count <= 0 || tty->flow.stopped ||
-	    tty->hw_stopped || !info->tx_buf)
-		return;
-
-	DBGINFO(("%s flush_chars start transmit\n", info->device_name));
-
-	spin_lock_irqsave(&info->lock,flags);
-	if (info->tx_count && tx_load(info, info->tx_buf, info->tx_count))
-		info->tx_count = 0;
-	spin_unlock_irqrestore(&info->lock,flags);
-}
-
-static void flush_buffer(struct tty_struct *tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "flush_buffer"))
-		return;
-	DBGINFO(("%s flush_buffer\n", info->device_name));
-
-	spin_lock_irqsave(&info->lock, flags);
-	info->tx_count = 0;
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	tty_wakeup(tty);
-}
-
-/*
- * throttle (stop) transmitter
- */
-static void tx_hold(struct tty_struct *tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "tx_hold"))
-		return;
-	DBGINFO(("%s tx_hold\n", info->device_name));
-	spin_lock_irqsave(&info->lock,flags);
-	if (info->tx_enabled && info->params.mode == MGSL_MODE_ASYNC)
-	 	tx_stop(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-}
-
-/*
- * release (start) transmitter
- */
-static void tx_release(struct tty_struct *tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "tx_release"))
-		return;
-	DBGINFO(("%s tx_release\n", info->device_name));
-	spin_lock_irqsave(&info->lock, flags);
-	if (info->tx_count && tx_load(info, info->tx_buf, info->tx_count))
-		info->tx_count = 0;
-	spin_unlock_irqrestore(&info->lock, flags);
-}
-
-/*
- * Service an IOCTL request
- *
- * Arguments
- *
- * 	tty	pointer to tty instance data
- * 	cmd	IOCTL command code
- * 	arg	command argument/context
- *
- * Return 0 if success, otherwise error code
- */
-static int ioctl(struct tty_struct *tty,
-		 unsigned int cmd, unsigned long arg)
-{
-	struct slgt_info *info = tty->driver_data;
-	void __user *argp = (void __user *)arg;
-	int ret;
-
-	if (sanity_check(info, tty->name, "ioctl"))
-		return -ENODEV;
-	DBGINFO(("%s ioctl() cmd=%08X\n", info->device_name, cmd));
-
-	if (cmd != TIOCMIWAIT) {
-		if (tty_io_error(tty))
-		    return -EIO;
-	}
-
-	switch (cmd) {
-	case MGSL_IOCWAITEVENT:
-		return wait_mgsl_event(info, argp);
-	case TIOCMIWAIT:
-		return modem_input_wait(info,(int)arg);
-	case MGSL_IOCSGPIO:
-		return set_gpio(info, argp);
-	case MGSL_IOCGGPIO:
-		return get_gpio(info, argp);
-	case MGSL_IOCWAITGPIO:
-		return wait_gpio(info, argp);
-	case MGSL_IOCGXSYNC:
-		return get_xsync(info, argp);
-	case MGSL_IOCSXSYNC:
-		return set_xsync(info, (int)arg);
-	case MGSL_IOCGXCTRL:
-		return get_xctrl(info, argp);
-	case MGSL_IOCSXCTRL:
-		return set_xctrl(info, (int)arg);
-	}
-	mutex_lock(&info->port.mutex);
-	switch (cmd) {
-	case MGSL_IOCGPARAMS:
-		ret = get_params(info, argp);
-		break;
-	case MGSL_IOCSPARAMS:
-		ret = set_params(info, argp);
-		break;
-	case MGSL_IOCGTXIDLE:
-		ret = get_txidle(info, argp);
-		break;
-	case MGSL_IOCSTXIDLE:
-		ret = set_txidle(info, (int)arg);
-		break;
-	case MGSL_IOCTXENABLE:
-		ret = tx_enable(info, (int)arg);
-		break;
-	case MGSL_IOCRXENABLE:
-		ret = rx_enable(info, (int)arg);
-		break;
-	case MGSL_IOCTXABORT:
-		ret = tx_abort(info);
-		break;
-	case MGSL_IOCGSTATS:
-		ret = get_stats(info, argp);
-		break;
-	case MGSL_IOCGIF:
-		ret = get_interface(info, argp);
-		break;
-	case MGSL_IOCSIF:
-		ret = set_interface(info,(int)arg);
-		break;
-	default:
-		ret = -ENOIOCTLCMD;
-	}
-	mutex_unlock(&info->port.mutex);
-	return ret;
-}
-
-static int get_icount(struct tty_struct *tty,
-				struct serial_icounter_struct *icount)
-
-{
-	struct slgt_info *info = tty->driver_data;
-	struct mgsl_icount cnow;	/* kernel counter temps */
-	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock,flags);
-	cnow = info->icount;
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	icount->cts = cnow.cts;
-	icount->dsr = cnow.dsr;
-	icount->rng = cnow.rng;
-	icount->dcd = cnow.dcd;
-	icount->rx = cnow.rx;
-	icount->tx = cnow.tx;
-	icount->frame = cnow.frame;
-	icount->overrun = cnow.overrun;
-	icount->parity = cnow.parity;
-	icount->brk = cnow.brk;
-	icount->buf_overrun = cnow.buf_overrun;
-
-	return 0;
-}
-
-/*
- * support for 32 bit ioctl calls on 64 bit systems
- */
-#ifdef CONFIG_COMPAT
-static long get_params32(struct slgt_info *info, struct MGSL_PARAMS32 __user *user_params)
-{
-	struct MGSL_PARAMS32 tmp_params;
-
-	DBGINFO(("%s get_params32\n", info->device_name));
-	memset(&tmp_params, 0, sizeof(tmp_params));
-	tmp_params.mode            = (compat_ulong_t)info->params.mode;
-	tmp_params.loopback        = info->params.loopback;
-	tmp_params.flags           = info->params.flags;
-	tmp_params.encoding        = info->params.encoding;
-	tmp_params.clock_speed     = (compat_ulong_t)info->params.clock_speed;
-	tmp_params.addr_filter     = info->params.addr_filter;
-	tmp_params.crc_type        = info->params.crc_type;
-	tmp_params.preamble_length = info->params.preamble_length;
-	tmp_params.preamble        = info->params.preamble;
-	tmp_params.data_rate       = (compat_ulong_t)info->params.data_rate;
-	tmp_params.data_bits       = info->params.data_bits;
-	tmp_params.stop_bits       = info->params.stop_bits;
-	tmp_params.parity          = info->params.parity;
-	if (copy_to_user(user_params, &tmp_params, sizeof(struct MGSL_PARAMS32)))
-		return -EFAULT;
-	return 0;
-}
-
-static long set_params32(struct slgt_info *info, struct MGSL_PARAMS32 __user *new_params)
-{
-	struct MGSL_PARAMS32 tmp_params;
-	unsigned long flags;
-
-	DBGINFO(("%s set_params32\n", info->device_name));
-	if (copy_from_user(&tmp_params, new_params, sizeof(struct MGSL_PARAMS32)))
-		return -EFAULT;
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (tmp_params.mode == MGSL_MODE_BASE_CLOCK) {
-		info->base_clock = tmp_params.clock_speed;
-	} else {
-		info->params.mode            = tmp_params.mode;
-		info->params.loopback        = tmp_params.loopback;
-		info->params.flags           = tmp_params.flags;
-		info->params.encoding        = tmp_params.encoding;
-		info->params.clock_speed     = tmp_params.clock_speed;
-		info->params.addr_filter     = tmp_params.addr_filter;
-		info->params.crc_type        = tmp_params.crc_type;
-		info->params.preamble_length = tmp_params.preamble_length;
-		info->params.preamble        = tmp_params.preamble;
-		info->params.data_rate       = tmp_params.data_rate;
-		info->params.data_bits       = tmp_params.data_bits;
-		info->params.stop_bits       = tmp_params.stop_bits;
-		info->params.parity          = tmp_params.parity;
-	}
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	program_hw(info);
-
-	return 0;
-}
-
-static long slgt_compat_ioctl(struct tty_struct *tty,
-			 unsigned int cmd, unsigned long arg)
-{
-	struct slgt_info *info = tty->driver_data;
-	int rc;
-
-	if (sanity_check(info, tty->name, "compat_ioctl"))
-		return -ENODEV;
-	DBGINFO(("%s compat_ioctl() cmd=%08X\n", info->device_name, cmd));
-
-	switch (cmd) {
-	case MGSL_IOCSPARAMS32:
-		rc = set_params32(info, compat_ptr(arg));
-		break;
-
-	case MGSL_IOCGPARAMS32:
-		rc = get_params32(info, compat_ptr(arg));
-		break;
-
-	case MGSL_IOCGPARAMS:
-	case MGSL_IOCSPARAMS:
-	case MGSL_IOCGTXIDLE:
-	case MGSL_IOCGSTATS:
-	case MGSL_IOCWAITEVENT:
-	case MGSL_IOCGIF:
-	case MGSL_IOCSGPIO:
-	case MGSL_IOCGGPIO:
-	case MGSL_IOCWAITGPIO:
-	case MGSL_IOCGXSYNC:
-	case MGSL_IOCGXCTRL:
-		rc = ioctl(tty, cmd, (unsigned long)compat_ptr(arg));
-		break;
-	default:
-		rc = ioctl(tty, cmd, arg);
-	}
-	DBGINFO(("%s compat_ioctl() cmd=%08X rc=%d\n", info->device_name, cmd, rc));
-	return rc;
-}
-#else
-#define slgt_compat_ioctl NULL
-#endif /* ifdef CONFIG_COMPAT */
-
-/*
- * proc fs support
- */
-static inline void line_info(struct seq_file *m, struct slgt_info *info)
-{
-	char stat_buf[30];
-	unsigned long flags;
-
-	seq_printf(m, "%s: IO=%08X IRQ=%d MaxFrameSize=%u\n",
-		      info->device_name, info->phys_reg_addr,
-		      info->irq_level, info->max_frame_size);
-
-	/* output current serial signal states */
-	spin_lock_irqsave(&info->lock,flags);
-	get_gtsignals(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	stat_buf[0] = 0;
-	stat_buf[1] = 0;
-	if (info->signals & SerialSignal_RTS)
-		strcat(stat_buf, "|RTS");
-	if (info->signals & SerialSignal_CTS)
-		strcat(stat_buf, "|CTS");
-	if (info->signals & SerialSignal_DTR)
-		strcat(stat_buf, "|DTR");
-	if (info->signals & SerialSignal_DSR)
-		strcat(stat_buf, "|DSR");
-	if (info->signals & SerialSignal_DCD)
-		strcat(stat_buf, "|CD");
-	if (info->signals & SerialSignal_RI)
-		strcat(stat_buf, "|RI");
-
-	if (info->params.mode != MGSL_MODE_ASYNC) {
-		seq_printf(m, "\tHDLC txok:%d rxok:%d",
-			       info->icount.txok, info->icount.rxok);
-		if (info->icount.txunder)
-			seq_printf(m, " txunder:%d", info->icount.txunder);
-		if (info->icount.txabort)
-			seq_printf(m, " txabort:%d", info->icount.txabort);
-		if (info->icount.rxshort)
-			seq_printf(m, " rxshort:%d", info->icount.rxshort);
-		if (info->icount.rxlong)
-			seq_printf(m, " rxlong:%d", info->icount.rxlong);
-		if (info->icount.rxover)
-			seq_printf(m, " rxover:%d", info->icount.rxover);
-		if (info->icount.rxcrc)
-			seq_printf(m, " rxcrc:%d", info->icount.rxcrc);
-	} else {
-		seq_printf(m, "\tASYNC tx:%d rx:%d",
-			       info->icount.tx, info->icount.rx);
-		if (info->icount.frame)
-			seq_printf(m, " fe:%d", info->icount.frame);
-		if (info->icount.parity)
-			seq_printf(m, " pe:%d", info->icount.parity);
-		if (info->icount.brk)
-			seq_printf(m, " brk:%d", info->icount.brk);
-		if (info->icount.overrun)
-			seq_printf(m, " oe:%d", info->icount.overrun);
-	}
-
-	/* Append serial signal status to end */
-	seq_printf(m, " %s\n", stat_buf+1);
-
-	seq_printf(m, "\ttxactive=%d bh_req=%d bh_run=%d pending_bh=%x\n",
-		       info->tx_active,info->bh_requested,info->bh_running,
-		       info->pending_bh);
-}
-
-/* Called to print information about devices
- */
-static int synclink_gt_proc_show(struct seq_file *m, void *v)
-{
-	struct slgt_info *info;
-
-	seq_puts(m, "synclink_gt driver\n");
-
-	info = slgt_device_list;
-	while( info ) {
-		line_info(m, info);
-		info = info->next_device;
-	}
-	return 0;
-}
-
-/*
- * return count of bytes in transmit buffer
- */
-static unsigned int chars_in_buffer(struct tty_struct *tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned int count;
-	if (sanity_check(info, tty->name, "chars_in_buffer"))
-		return 0;
-	count = tbuf_bytes(info);
-	DBGINFO(("%s chars_in_buffer()=%u\n", info->device_name, count));
-	return count;
-}
-
-/*
- * signal remote device to throttle send data (our receive data)
- */
-static void throttle(struct tty_struct * tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "throttle"))
-		return;
-	DBGINFO(("%s throttle\n", info->device_name));
-	if (I_IXOFF(tty))
-		send_xchar(tty, STOP_CHAR(tty));
-	if (C_CRTSCTS(tty)) {
-		spin_lock_irqsave(&info->lock,flags);
-		info->signals &= ~SerialSignal_RTS;
-		set_gtsignals(info);
-		spin_unlock_irqrestore(&info->lock,flags);
-	}
-}
-
-/*
- * signal remote device to stop throttling send data (our receive data)
- */
-static void unthrottle(struct tty_struct * tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "unthrottle"))
-		return;
-	DBGINFO(("%s unthrottle\n", info->device_name));
-	if (I_IXOFF(tty)) {
-		if (info->x_char)
-			info->x_char = 0;
-		else
-			send_xchar(tty, START_CHAR(tty));
-	}
-	if (C_CRTSCTS(tty)) {
-		spin_lock_irqsave(&info->lock,flags);
-		info->signals |= SerialSignal_RTS;
-		set_gtsignals(info);
-		spin_unlock_irqrestore(&info->lock,flags);
-	}
-}
-
-/*
- * set or clear transmit break condition
- * break_state	-1=set break condition, 0=clear
- */
-static int set_break(struct tty_struct *tty, int break_state)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned short value;
-	unsigned long flags;
-
-	if (sanity_check(info, tty->name, "set_break"))
-		return -EINVAL;
-	DBGINFO(("%s set_break(%d)\n", info->device_name, break_state));
-
-	spin_lock_irqsave(&info->lock,flags);
-	value = rd_reg16(info, TCR);
- 	if (break_state == -1)
-		value |= BIT6;
-	else
-		value &= ~BIT6;
-	wr_reg16(info, TCR, value);
-	spin_unlock_irqrestore(&info->lock,flags);
-	return 0;
-}
-
-#if SYNCLINK_GENERIC_HDLC
-
-/**
- * hdlcdev_attach - called by generic HDLC layer when protocol selected (PPP, frame relay, etc.)
- * @dev:      pointer to network device structure
- * @encoding: serial encoding setting
- * @parity:   FCS setting
- *
- * Set encoding and frame check sequence (FCS) options.
- *
- * Return: 0 if success, otherwise error code
- */
-static int hdlcdev_attach(struct net_device *dev, unsigned short encoding,
-			  unsigned short parity)
-{
-	struct slgt_info *info = dev_to_port(dev);
-	unsigned char  new_encoding;
-	unsigned short new_crctype;
-
-	/* return error if TTY interface open */
-	if (info->port.count)
-		return -EBUSY;
-
-	DBGINFO(("%s hdlcdev_attach\n", info->device_name));
-
-	switch (encoding)
-	{
-	case ENCODING_NRZ:        new_encoding = HDLC_ENCODING_NRZ; break;
-	case ENCODING_NRZI:       new_encoding = HDLC_ENCODING_NRZI_SPACE; break;
-	case ENCODING_FM_MARK:    new_encoding = HDLC_ENCODING_BIPHASE_MARK; break;
-	case ENCODING_FM_SPACE:   new_encoding = HDLC_ENCODING_BIPHASE_SPACE; break;
-	case ENCODING_MANCHESTER: new_encoding = HDLC_ENCODING_BIPHASE_LEVEL; break;
-	default: return -EINVAL;
-	}
-
-	switch (parity)
-	{
-	case PARITY_NONE:            new_crctype = HDLC_CRC_NONE; break;
-	case PARITY_CRC16_PR1_CCITT: new_crctype = HDLC_CRC_16_CCITT; break;
-	case PARITY_CRC32_PR1_CCITT: new_crctype = HDLC_CRC_32_CCITT; break;
-	default: return -EINVAL;
-	}
-
-	info->params.encoding = new_encoding;
-	info->params.crc_type = new_crctype;
-
-	/* if network interface up, reprogram hardware */
-	if (info->netcount)
-		program_hw(info);
-
-	return 0;
-}
-
-/**
- * hdlcdev_xmit - called by generic HDLC layer to send a frame
- * @skb: socket buffer containing HDLC frame
- * @dev: pointer to network device structure
- */
-static netdev_tx_t hdlcdev_xmit(struct sk_buff *skb,
-				      struct net_device *dev)
-{
-	struct slgt_info *info = dev_to_port(dev);
-	unsigned long flags;
-
-	DBGINFO(("%s hdlc_xmit\n", dev->name));
-
-	if (!skb->len)
-		return NETDEV_TX_OK;
-
-	/* stop sending until this frame completes */
-	netif_stop_queue(dev);
-
-	/* update network statistics */
-	dev->stats.tx_packets++;
-	dev->stats.tx_bytes += skb->len;
-
-	/* save start time for transmit timeout detection */
-	netif_trans_update(dev);
-
-	spin_lock_irqsave(&info->lock, flags);
-	tx_load(info, skb->data, skb->len);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	/* done with socket buffer, so free it */
-	dev_kfree_skb(skb);
-
-	return NETDEV_TX_OK;
-}
-
-/**
- * hdlcdev_open - called by network layer when interface enabled
- * @dev: pointer to network device structure
- *
- * Claim resources and initialize hardware.
- *
- * Return: 0 if success, otherwise error code
- */
-static int hdlcdev_open(struct net_device *dev)
-{
-	struct slgt_info *info = dev_to_port(dev);
-	int rc;
-	unsigned long flags;
-
-	DBGINFO(("%s hdlcdev_open\n", dev->name));
-
-	/* arbitrate between network and tty opens */
-	spin_lock_irqsave(&info->netlock, flags);
-	if (info->port.count != 0 || info->netcount != 0) {
-		DBGINFO(("%s hdlc_open busy\n", dev->name));
-		spin_unlock_irqrestore(&info->netlock, flags);
-		return -EBUSY;
-	}
-	info->netcount=1;
-	spin_unlock_irqrestore(&info->netlock, flags);
-
-	/* claim resources and init adapter */
-	if ((rc = startup_hw(info)) != 0) {
-		spin_lock_irqsave(&info->netlock, flags);
-		info->netcount=0;
-		spin_unlock_irqrestore(&info->netlock, flags);
-		return rc;
-	}
-
-	/* generic HDLC layer open processing */
-	rc = hdlc_open(dev);
-	if (rc) {
-		shutdown_hw(info);
-		spin_lock_irqsave(&info->netlock, flags);
-		info->netcount = 0;
-		spin_unlock_irqrestore(&info->netlock, flags);
-		return rc;
-	}
-
-	/* assert RTS and DTR, apply hardware settings */
-	info->signals |= SerialSignal_RTS | SerialSignal_DTR;
-	program_hw(info);
-
-	/* enable network layer transmit */
-	netif_trans_update(dev);
-	netif_start_queue(dev);
-
-	/* inform generic HDLC layer of current DCD status */
-	spin_lock_irqsave(&info->lock, flags);
-	get_gtsignals(info);
-	spin_unlock_irqrestore(&info->lock, flags);
-	if (info->signals & SerialSignal_DCD)
-		netif_carrier_on(dev);
-	else
-		netif_carrier_off(dev);
-	return 0;
-}
-
-/**
- * hdlcdev_close - called by network layer when interface is disabled
- * @dev:  pointer to network device structure
- *
- * Shutdown hardware and release resources.
- *
- * Return: 0 if success, otherwise error code
- */
-static int hdlcdev_close(struct net_device *dev)
-{
-	struct slgt_info *info = dev_to_port(dev);
-	unsigned long flags;
-
-	DBGINFO(("%s hdlcdev_close\n", dev->name));
-
-	netif_stop_queue(dev);
-
-	/* shutdown adapter and release resources */
-	shutdown_hw(info);
-
-	hdlc_close(dev);
-
-	spin_lock_irqsave(&info->netlock, flags);
-	info->netcount=0;
-	spin_unlock_irqrestore(&info->netlock, flags);
-
-	return 0;
-}
-
-/**
- * hdlcdev_ioctl - called by network layer to process IOCTL call to network device
- * @dev: pointer to network device structure
- * @ifr: pointer to network interface request structure
- * @cmd: IOCTL command code
- *
- * Return: 0 if success, otherwise error code
- */
-static int hdlcdev_ioctl(struct net_device *dev, struct if_settings *ifs)
-{
-	const size_t size = sizeof(sync_serial_settings);
-	sync_serial_settings new_line;
-	sync_serial_settings __user *line = ifs->ifs_ifsu.sync;
-	struct slgt_info *info = dev_to_port(dev);
-	unsigned int flags;
-
-	DBGINFO(("%s hdlcdev_ioctl\n", dev->name));
-
-	/* return error if TTY interface open */
-	if (info->port.count)
-		return -EBUSY;
-
-	memset(&new_line, 0, sizeof(new_line));
-
-	switch (ifs->type) {
-	case IF_GET_IFACE: /* return current sync_serial_settings */
-
-		ifs->type = IF_IFACE_SYNC_SERIAL;
-		if (ifs->size < size) {
-			ifs->size = size; /* data size wanted */
-			return -ENOBUFS;
-		}
-
-		flags = info->params.flags & (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_RXC_DPLL |
-					      HDLC_FLAG_RXC_BRG    | HDLC_FLAG_RXC_TXCPIN |
-					      HDLC_FLAG_TXC_TXCPIN | HDLC_FLAG_TXC_DPLL |
-					      HDLC_FLAG_TXC_BRG    | HDLC_FLAG_TXC_RXCPIN);
-
-		switch (flags){
-		case (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_TXCPIN): new_line.clock_type = CLOCK_EXT; break;
-		case (HDLC_FLAG_RXC_BRG    | HDLC_FLAG_TXC_BRG):    new_line.clock_type = CLOCK_INT; break;
-		case (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_BRG):    new_line.clock_type = CLOCK_TXINT; break;
-		case (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_RXCPIN): new_line.clock_type = CLOCK_TXFROMRX; break;
-		default: new_line.clock_type = CLOCK_DEFAULT;
-		}
-
-		new_line.clock_rate = info->params.clock_speed;
-		new_line.loopback   = info->params.loopback ? 1:0;
-
-		if (copy_to_user(line, &new_line, size))
-			return -EFAULT;
-		return 0;
-
-	case IF_IFACE_SYNC_SERIAL: /* set sync_serial_settings */
-
-		if(!capable(CAP_NET_ADMIN))
-			return -EPERM;
-		if (copy_from_user(&new_line, line, size))
-			return -EFAULT;
-
-		switch (new_line.clock_type)
-		{
-		case CLOCK_EXT:      flags = HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_TXCPIN; break;
-		case CLOCK_TXFROMRX: flags = HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_RXCPIN; break;
-		case CLOCK_INT:      flags = HDLC_FLAG_RXC_BRG    | HDLC_FLAG_TXC_BRG;    break;
-		case CLOCK_TXINT:    flags = HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_TXC_BRG;    break;
-		case CLOCK_DEFAULT:  flags = info->params.flags &
-					     (HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_RXC_DPLL |
-					      HDLC_FLAG_RXC_BRG    | HDLC_FLAG_RXC_TXCPIN |
-					      HDLC_FLAG_TXC_TXCPIN | HDLC_FLAG_TXC_DPLL |
-					      HDLC_FLAG_TXC_BRG    | HDLC_FLAG_TXC_RXCPIN); break;
-		default: return -EINVAL;
-		}
-
-		if (new_line.loopback != 0 && new_line.loopback != 1)
-			return -EINVAL;
-
-		info->params.flags &= ~(HDLC_FLAG_RXC_RXCPIN | HDLC_FLAG_RXC_DPLL |
-					HDLC_FLAG_RXC_BRG    | HDLC_FLAG_RXC_TXCPIN |
-					HDLC_FLAG_TXC_TXCPIN | HDLC_FLAG_TXC_DPLL |
-					HDLC_FLAG_TXC_BRG    | HDLC_FLAG_TXC_RXCPIN);
-		info->params.flags |= flags;
-
-		info->params.loopback = new_line.loopback;
-
-		if (flags & (HDLC_FLAG_RXC_BRG | HDLC_FLAG_TXC_BRG))
-			info->params.clock_speed = new_line.clock_rate;
-		else
-			info->params.clock_speed = 0;
-
-		/* if network interface up, reprogram hardware */
-		if (info->netcount)
-			program_hw(info);
-		return 0;
-
-	default:
-		return hdlc_ioctl(dev, ifs);
-	}
-}
-
-/**
- * hdlcdev_tx_timeout - called by network layer when transmit timeout is detected
- * @dev: pointer to network device structure
- * @txqueue: unused
- */
-static void hdlcdev_tx_timeout(struct net_device *dev, unsigned int txqueue)
-{
-	struct slgt_info *info = dev_to_port(dev);
-	unsigned long flags;
-
-	DBGINFO(("%s hdlcdev_tx_timeout\n", dev->name));
-
-	dev->stats.tx_errors++;
-	dev->stats.tx_aborted_errors++;
-
-	spin_lock_irqsave(&info->lock,flags);
-	tx_stop(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	netif_wake_queue(dev);
-}
-
-/**
- * hdlcdev_tx_done - called by device driver when transmit completes
- * @info: pointer to device instance information
- *
- * Reenable network layer transmit if stopped.
- */
-static void hdlcdev_tx_done(struct slgt_info *info)
-{
-	if (netif_queue_stopped(info->netdev))
-		netif_wake_queue(info->netdev);
-}
-
-/**
- * hdlcdev_rx - called by device driver when frame received
- * @info: pointer to device instance information
- * @buf:  pointer to buffer contianing frame data
- * @size: count of data bytes in buf
- *
- * Pass frame to network layer.
- */
-static void hdlcdev_rx(struct slgt_info *info, char *buf, int size)
-{
-	struct sk_buff *skb = dev_alloc_skb(size);
-	struct net_device *dev = info->netdev;
-
-	DBGINFO(("%s hdlcdev_rx\n", dev->name));
-
-	if (skb == NULL) {
-		DBGERR(("%s: can't alloc skb, drop packet\n", dev->name));
-		dev->stats.rx_dropped++;
-		return;
-	}
-
-	skb_put_data(skb, buf, size);
-
-	skb->protocol = hdlc_type_trans(skb, dev);
-
-	dev->stats.rx_packets++;
-	dev->stats.rx_bytes += size;
-
-	netif_rx(skb);
-}
-
-static const struct net_device_ops hdlcdev_ops = {
-	.ndo_open       = hdlcdev_open,
-	.ndo_stop       = hdlcdev_close,
-	.ndo_start_xmit = hdlc_start_xmit,
-	.ndo_siocwandev = hdlcdev_ioctl,
-	.ndo_tx_timeout = hdlcdev_tx_timeout,
-};
-
-/**
- * hdlcdev_init - called by device driver when adding device instance
- * @info: pointer to device instance information
- *
- * Do generic HDLC initialization.
- *
- * Return: 0 if success, otherwise error code
- */
-static int hdlcdev_init(struct slgt_info *info)
-{
-	int rc;
-	struct net_device *dev;
-	hdlc_device *hdlc;
-
-	/* allocate and initialize network and HDLC layer objects */
-
-	dev = alloc_hdlcdev(info);
-	if (!dev) {
-		printk(KERN_ERR "%s hdlc device alloc failure\n", info->device_name);
-		return -ENOMEM;
-	}
-
-	/* for network layer reporting purposes only */
-	dev->mem_start = info->phys_reg_addr;
-	dev->mem_end   = info->phys_reg_addr + SLGT_REG_SIZE - 1;
-	dev->irq       = info->irq_level;
-
-	/* network layer callbacks and settings */
-	dev->netdev_ops	    = &hdlcdev_ops;
-	dev->watchdog_timeo = 10 * HZ;
-	dev->tx_queue_len   = 50;
-
-	/* generic HDLC layer callbacks and settings */
-	hdlc         = dev_to_hdlc(dev);
-	hdlc->attach = hdlcdev_attach;
-	hdlc->xmit   = hdlcdev_xmit;
-
-	/* register objects with HDLC layer */
-	rc = register_hdlc_device(dev);
-	if (rc) {
-		printk(KERN_WARNING "%s:unable to register hdlc device\n",__FILE__);
-		free_netdev(dev);
-		return rc;
-	}
-
-	info->netdev = dev;
-	return 0;
-}
-
-/**
- * hdlcdev_exit - called by device driver when removing device instance
- * @info: pointer to device instance information
- *
- * Do generic HDLC cleanup.
- */
-static void hdlcdev_exit(struct slgt_info *info)
-{
-	if (!info->netdev)
-		return;
-	unregister_hdlc_device(info->netdev);
-	free_netdev(info->netdev);
-	info->netdev = NULL;
-}
-
-#endif /* ifdef CONFIG_HDLC */
-
-/*
- * get async data from rx DMA buffers
- */
-static void rx_async(struct slgt_info *info)
-{
- 	struct mgsl_icount *icount = &info->icount;
-	unsigned int start, end;
-	unsigned char *p;
-	unsigned char status;
-	struct slgt_desc *bufs = info->rbufs;
-	int i, count;
-	int chars = 0;
-	int stat;
-	unsigned char ch;
-
-	start = end = info->rbuf_current;
-
-	while(desc_complete(bufs[end])) {
-		count = desc_count(bufs[end]) - info->rbuf_index;
-		p     = bufs[end].buf + info->rbuf_index;
-
-		DBGISR(("%s rx_async count=%d\n", info->device_name, count));
-		DBGDATA(info, p, count, "rx");
-
-		for(i=0 ; i < count; i+=2, p+=2) {
-			ch = *p;
-			icount->rx++;
-
-			stat = 0;
-
-			status = *(p + 1) & (BIT1 + BIT0);
-			if (status) {
-				if (status & BIT1)
-					icount->parity++;
-				else if (status & BIT0)
-					icount->frame++;
-				/* discard char if tty control flags say so */
-				if (status & info->ignore_status_mask)
-					continue;
-				if (status & BIT1)
-					stat = TTY_PARITY;
-				else if (status & BIT0)
-					stat = TTY_FRAME;
-			}
-			tty_insert_flip_char(&info->port, ch, stat);
-			chars++;
-		}
-
-		if (i < count) {
-			/* receive buffer not completed */
-			info->rbuf_index += i;
-			mod_timer(&info->rx_timer, jiffies + 1);
-			break;
-		}
-
-		info->rbuf_index = 0;
-		free_rbufs(info, end, end);
-
-		if (++end == info->rbuf_count)
-			end = 0;
-
-		/* if entire list searched then no frame available */
-		if (end == start)
-			break;
-	}
-
-	if (chars)
-		tty_flip_buffer_push(&info->port);
-}
-
-/*
- * return next bottom half action to perform
- */
-static int bh_action(struct slgt_info *info)
-{
-	unsigned long flags;
-	int rc;
-
-	spin_lock_irqsave(&info->lock,flags);
-
-	if (info->pending_bh & BH_RECEIVE) {
-		info->pending_bh &= ~BH_RECEIVE;
-		rc = BH_RECEIVE;
-	} else if (info->pending_bh & BH_TRANSMIT) {
-		info->pending_bh &= ~BH_TRANSMIT;
-		rc = BH_TRANSMIT;
-	} else if (info->pending_bh & BH_STATUS) {
-		info->pending_bh &= ~BH_STATUS;
-		rc = BH_STATUS;
-	} else {
-		/* Mark BH routine as complete */
-		info->bh_running = false;
-		info->bh_requested = false;
-		rc = 0;
-	}
-
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	return rc;
-}
-
-/*
- * perform bottom half processing
- */
-static void bh_handler(struct work_struct *work)
-{
-	struct slgt_info *info = container_of(work, struct slgt_info, task);
-	int action;
-
-	info->bh_running = true;
-
-	while((action = bh_action(info))) {
-		switch (action) {
-		case BH_RECEIVE:
-			DBGBH(("%s bh receive\n", info->device_name));
-			switch(info->params.mode) {
-			case MGSL_MODE_ASYNC:
-				rx_async(info);
-				break;
-			case MGSL_MODE_HDLC:
-				while(rx_get_frame(info));
-				break;
-			case MGSL_MODE_RAW:
-			case MGSL_MODE_MONOSYNC:
-			case MGSL_MODE_BISYNC:
-			case MGSL_MODE_XSYNC:
-				while(rx_get_buf(info));
-				break;
-			}
-			/* restart receiver if rx DMA buffers exhausted */
-			if (info->rx_restart)
-				rx_start(info);
-			break;
-		case BH_TRANSMIT:
-			bh_transmit(info);
-			break;
-		case BH_STATUS:
-			DBGBH(("%s bh status\n", info->device_name));
-			info->ri_chkcount = 0;
-			info->dsr_chkcount = 0;
-			info->dcd_chkcount = 0;
-			info->cts_chkcount = 0;
-			break;
-		default:
-			DBGBH(("%s unknown action\n", info->device_name));
-			break;
-		}
-	}
-	DBGBH(("%s bh_handler exit\n", info->device_name));
-}
-
-static void bh_transmit(struct slgt_info *info)
-{
-	struct tty_struct *tty = info->port.tty;
-
-	DBGBH(("%s bh_transmit\n", info->device_name));
-	if (tty)
-		tty_wakeup(tty);
-}
-
-static void dsr_change(struct slgt_info *info, unsigned short status)
-{
-	if (status & BIT3) {
-		info->signals |= SerialSignal_DSR;
-		info->input_signal_events.dsr_up++;
-	} else {
-		info->signals &= ~SerialSignal_DSR;
-		info->input_signal_events.dsr_down++;
-	}
-	DBGISR(("dsr_change %s signals=%04X\n", info->device_name, info->signals));
-	if ((info->dsr_chkcount)++ == IO_PIN_SHUTDOWN_LIMIT) {
-		slgt_irq_off(info, IRQ_DSR);
-		return;
-	}
-	info->icount.dsr++;
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-	info->pending_bh |= BH_STATUS;
-}
-
-static void cts_change(struct slgt_info *info, unsigned short status)
-{
-	if (status & BIT2) {
-		info->signals |= SerialSignal_CTS;
-		info->input_signal_events.cts_up++;
-	} else {
-		info->signals &= ~SerialSignal_CTS;
-		info->input_signal_events.cts_down++;
-	}
-	DBGISR(("cts_change %s signals=%04X\n", info->device_name, info->signals));
-	if ((info->cts_chkcount)++ == IO_PIN_SHUTDOWN_LIMIT) {
-		slgt_irq_off(info, IRQ_CTS);
-		return;
-	}
-	info->icount.cts++;
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-	info->pending_bh |= BH_STATUS;
-
-	if (tty_port_cts_enabled(&info->port)) {
-		if (info->port.tty) {
-			if (info->port.tty->hw_stopped) {
-				if (info->signals & SerialSignal_CTS) {
-					info->port.tty->hw_stopped = false;
-					info->pending_bh |= BH_TRANSMIT;
-					return;
-				}
-			} else {
-				if (!(info->signals & SerialSignal_CTS))
-					info->port.tty->hw_stopped = true;
-			}
-		}
-	}
-}
-
-static void dcd_change(struct slgt_info *info, unsigned short status)
-{
-	if (status & BIT1) {
-		info->signals |= SerialSignal_DCD;
-		info->input_signal_events.dcd_up++;
-	} else {
-		info->signals &= ~SerialSignal_DCD;
-		info->input_signal_events.dcd_down++;
-	}
-	DBGISR(("dcd_change %s signals=%04X\n", info->device_name, info->signals));
-	if ((info->dcd_chkcount)++ == IO_PIN_SHUTDOWN_LIMIT) {
-		slgt_irq_off(info, IRQ_DCD);
-		return;
-	}
-	info->icount.dcd++;
-#if SYNCLINK_GENERIC_HDLC
-	if (info->netcount) {
-		if (info->signals & SerialSignal_DCD)
-			netif_carrier_on(info->netdev);
-		else
-			netif_carrier_off(info->netdev);
-	}
-#endif
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-	info->pending_bh |= BH_STATUS;
-
-	if (tty_port_check_carrier(&info->port)) {
-		if (info->signals & SerialSignal_DCD)
-			wake_up_interruptible(&info->port.open_wait);
-		else {
-			if (info->port.tty)
-				tty_hangup(info->port.tty);
-		}
-	}
-}
-
-static void ri_change(struct slgt_info *info, unsigned short status)
-{
-	if (status & BIT0) {
-		info->signals |= SerialSignal_RI;
-		info->input_signal_events.ri_up++;
-	} else {
-		info->signals &= ~SerialSignal_RI;
-		info->input_signal_events.ri_down++;
-	}
-	DBGISR(("ri_change %s signals=%04X\n", info->device_name, info->signals));
-	if ((info->ri_chkcount)++ == IO_PIN_SHUTDOWN_LIMIT) {
-		slgt_irq_off(info, IRQ_RI);
-		return;
-	}
-	info->icount.rng++;
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-	info->pending_bh |= BH_STATUS;
-}
-
-static void isr_rxdata(struct slgt_info *info)
-{
-	unsigned int count = info->rbuf_fill_count;
-	unsigned int i = info->rbuf_fill_index;
-	unsigned short reg;
-
-	while (rd_reg16(info, SSR) & IRQ_RXDATA) {
-		reg = rd_reg16(info, RDR);
-		DBGISR(("isr_rxdata %s RDR=%04X\n", info->device_name, reg));
-		if (desc_complete(info->rbufs[i])) {
-			/* all buffers full */
-			rx_stop(info);
-			info->rx_restart = true;
-			continue;
-		}
-		info->rbufs[i].buf[count++] = (unsigned char)reg;
-		/* async mode saves status byte to buffer for each data byte */
-		if (info->params.mode == MGSL_MODE_ASYNC)
-			info->rbufs[i].buf[count++] = (unsigned char)(reg >> 8);
-		if (count == info->rbuf_fill_level || (reg & BIT10)) {
-			/* buffer full or end of frame */
-			set_desc_count(info->rbufs[i], count);
-			set_desc_status(info->rbufs[i], BIT15 | (reg >> 8));
-			info->rbuf_fill_count = count = 0;
-			if (++i == info->rbuf_count)
-				i = 0;
-			info->pending_bh |= BH_RECEIVE;
-		}
-	}
-
-	info->rbuf_fill_index = i;
-	info->rbuf_fill_count = count;
-}
-
-static void isr_serial(struct slgt_info *info)
-{
-	unsigned short status = rd_reg16(info, SSR);
-
-	DBGISR(("%s isr_serial status=%04X\n", info->device_name, status));
-
-	wr_reg16(info, SSR, status); /* clear pending */
-
-	info->irq_occurred = true;
-
-	if (info->params.mode == MGSL_MODE_ASYNC) {
-		if (status & IRQ_TXIDLE) {
-			if (info->tx_active)
-				isr_txeom(info, status);
-		}
-		if (info->rx_pio && (status & IRQ_RXDATA))
-			isr_rxdata(info);
-		if ((status & IRQ_RXBREAK) && (status & RXBREAK)) {
-			info->icount.brk++;
-			/* process break detection if tty control allows */
-			if (info->port.tty) {
-				if (!(status & info->ignore_status_mask)) {
-					if (info->read_status_mask & MASK_BREAK) {
-						tty_insert_flip_char(&info->port, 0, TTY_BREAK);
-						if (info->port.flags & ASYNC_SAK)
-							do_SAK(info->port.tty);
-					}
-				}
-			}
-		}
-	} else {
-		if (status & (IRQ_TXIDLE + IRQ_TXUNDER))
-			isr_txeom(info, status);
-		if (info->rx_pio && (status & IRQ_RXDATA))
-			isr_rxdata(info);
-		if (status & IRQ_RXIDLE) {
-			if (status & RXIDLE)
-				info->icount.rxidle++;
-			else
-				info->icount.exithunt++;
-			wake_up_interruptible(&info->event_wait_q);
-		}
-
-		if (status & IRQ_RXOVER)
-			rx_start(info);
-	}
-
-	if (status & IRQ_DSR)
-		dsr_change(info, status);
-	if (status & IRQ_CTS)
-		cts_change(info, status);
-	if (status & IRQ_DCD)
-		dcd_change(info, status);
-	if (status & IRQ_RI)
-		ri_change(info, status);
-}
-
-static void isr_rdma(struct slgt_info *info)
-{
-	unsigned int status = rd_reg32(info, RDCSR);
-
-	DBGISR(("%s isr_rdma status=%08x\n", info->device_name, status));
-
-	/* RDCSR (rx DMA control/status)
-	 *
-	 * 31..07  reserved
-	 * 06      save status byte to DMA buffer
-	 * 05      error
-	 * 04      eol (end of list)
-	 * 03      eob (end of buffer)
-	 * 02      IRQ enable
-	 * 01      reset
-	 * 00      enable
-	 */
-	wr_reg32(info, RDCSR, status);	/* clear pending */
-
-	if (status & (BIT5 + BIT4)) {
-		DBGISR(("%s isr_rdma rx_restart=1\n", info->device_name));
-		info->rx_restart = true;
-	}
-	info->pending_bh |= BH_RECEIVE;
-}
-
-static void isr_tdma(struct slgt_info *info)
-{
-	unsigned int status = rd_reg32(info, TDCSR);
-
-	DBGISR(("%s isr_tdma status=%08x\n", info->device_name, status));
-
-	/* TDCSR (tx DMA control/status)
-	 *
-	 * 31..06  reserved
-	 * 05      error
-	 * 04      eol (end of list)
-	 * 03      eob (end of buffer)
-	 * 02      IRQ enable
-	 * 01      reset
-	 * 00      enable
-	 */
-	wr_reg32(info, TDCSR, status);	/* clear pending */
-
-	if (status & (BIT5 + BIT4 + BIT3)) {
-		// another transmit buffer has completed
-		// run bottom half to get more send data from user
-		info->pending_bh |= BH_TRANSMIT;
-	}
-}
-
-/*
- * return true if there are unsent tx DMA buffers, otherwise false
- *
- * if there are unsent buffers then info->tbuf_start
- * is set to index of first unsent buffer
- */
-static bool unsent_tbufs(struct slgt_info *info)
-{
-	unsigned int i = info->tbuf_current;
-	bool rc = false;
-
-	/*
-	 * search backwards from last loaded buffer (precedes tbuf_current)
-	 * for first unsent buffer (desc_count > 0)
-	 */
-
-	do {
-		if (i)
-			i--;
-		else
-			i = info->tbuf_count - 1;
-		if (!desc_count(info->tbufs[i]))
-			break;
-		info->tbuf_start = i;
-		rc = true;
-	} while (i != info->tbuf_current);
-
-	return rc;
-}
-
-static void isr_txeom(struct slgt_info *info, unsigned short status)
-{
-	DBGISR(("%s txeom status=%04x\n", info->device_name, status));
-
-	slgt_irq_off(info, IRQ_TXDATA + IRQ_TXIDLE + IRQ_TXUNDER);
-	tdma_reset(info);
-	if (status & IRQ_TXUNDER) {
-		unsigned short val = rd_reg16(info, TCR);
-		wr_reg16(info, TCR, (unsigned short)(val | BIT2)); /* set reset bit */
-		wr_reg16(info, TCR, val); /* clear reset bit */
-	}
-
-	if (info->tx_active) {
-		if (info->params.mode != MGSL_MODE_ASYNC) {
-			if (status & IRQ_TXUNDER)
-				info->icount.txunder++;
-			else if (status & IRQ_TXIDLE)
-				info->icount.txok++;
-		}
-
-		if (unsent_tbufs(info)) {
-			tx_start(info);
-			update_tx_timer(info);
-			return;
-		}
-		info->tx_active = false;
-
-		timer_delete(&info->tx_timer);
-
-		if (info->params.mode != MGSL_MODE_ASYNC && info->drop_rts_on_tx_done) {
-			info->signals &= ~SerialSignal_RTS;
-			info->drop_rts_on_tx_done = false;
-			set_gtsignals(info);
-		}
-
-#if SYNCLINK_GENERIC_HDLC
-		if (info->netcount)
-			hdlcdev_tx_done(info);
-		else
-#endif
-		{
-			if (info->port.tty && (info->port.tty->flow.stopped || info->port.tty->hw_stopped)) {
-				tx_stop(info);
-				return;
-			}
-			info->pending_bh |= BH_TRANSMIT;
-		}
-	}
-}
-
-static void isr_gpio(struct slgt_info *info, unsigned int changed, unsigned int state)
-{
-	struct cond_wait *w, *prev;
-
-	/* wake processes waiting for specific transitions */
-	for (w = info->gpio_wait_q, prev = NULL ; w != NULL ; w = w->next) {
-		if (w->data & changed) {
-			w->data = state;
-			wake_up_interruptible(&w->q);
-			if (prev != NULL)
-				prev->next = w->next;
-			else
-				info->gpio_wait_q = w->next;
-		} else
-			prev = w;
-	}
-}
-
-/* interrupt service routine
- *
- * 	irq	interrupt number
- * 	dev_id	device ID supplied during interrupt registration
- */
-static irqreturn_t slgt_interrupt(int dummy, void *dev_id)
-{
-	struct slgt_info *info = dev_id;
-	unsigned int gsr;
-	unsigned int i;
-
-	DBGISR(("slgt_interrupt irq=%d entry\n", info->irq_level));
-
-	while((gsr = rd_reg32(info, GSR) & 0xffffff00)) {
-		DBGISR(("%s gsr=%08x\n", info->device_name, gsr));
-		info->irq_occurred = true;
-		for(i=0; i < info->port_count ; i++) {
-			if (info->port_array[i] == NULL)
-				continue;
-			spin_lock(&info->port_array[i]->lock);
-			if (gsr & (BIT8 << i))
-				isr_serial(info->port_array[i]);
-			if (gsr & (BIT16 << (i*2)))
-				isr_rdma(info->port_array[i]);
-			if (gsr & (BIT17 << (i*2)))
-				isr_tdma(info->port_array[i]);
-			spin_unlock(&info->port_array[i]->lock);
-		}
-	}
-
-	if (info->gpio_present) {
-		unsigned int state;
-		unsigned int changed;
-		spin_lock(&info->lock);
-		while ((changed = rd_reg32(info, IOSR)) != 0) {
-			DBGISR(("%s iosr=%08x\n", info->device_name, changed));
-			/* read latched state of GPIO signals */
-			state = rd_reg32(info, IOVR);
-			/* clear pending GPIO interrupt bits */
-			wr_reg32(info, IOSR, changed);
-			for (i=0 ; i < info->port_count ; i++) {
-				if (info->port_array[i] != NULL)
-					isr_gpio(info->port_array[i], changed, state);
-			}
-		}
-		spin_unlock(&info->lock);
-	}
-
-	for(i=0; i < info->port_count ; i++) {
-		struct slgt_info *port = info->port_array[i];
-		if (port == NULL)
-			continue;
-		spin_lock(&port->lock);
-		if ((port->port.count || port->netcount) &&
-		    port->pending_bh && !port->bh_running &&
-		    !port->bh_requested) {
-			DBGISR(("%s bh queued\n", port->device_name));
-			schedule_work(&port->task);
-			port->bh_requested = true;
-		}
-		spin_unlock(&port->lock);
-	}
-
-	DBGISR(("slgt_interrupt irq=%d exit\n", info->irq_level));
-	return IRQ_HANDLED;
-}
-
-static int startup_hw(struct slgt_info *info)
-{
-	DBGINFO(("%s startup\n", info->device_name));
-
-	if (tty_port_initialized(&info->port))
-		return 0;
-
-	if (!info->tx_buf) {
-		info->tx_buf = kmalloc(info->max_frame_size, GFP_KERNEL);
-		if (!info->tx_buf) {
-			DBGERR(("%s can't allocate tx buffer\n", info->device_name));
-			return -ENOMEM;
-		}
-	}
-
-	info->pending_bh = 0;
-
-	memset(&info->icount, 0, sizeof(info->icount));
-
-	/* program hardware for current parameters */
-	change_params(info);
-
-	if (info->port.tty)
-		clear_bit(TTY_IO_ERROR, &info->port.tty->flags);
-
-	tty_port_set_initialized(&info->port, true);
-
-	return 0;
-}
-
-/*
- *  called by close() and hangup() to shutdown hardware
- */
-static void shutdown_hw(struct slgt_info *info)
-{
-	unsigned long flags;
-
-	if (!tty_port_initialized(&info->port))
-		return;
-
-	DBGINFO(("%s shutdown\n", info->device_name));
-
-	/* clear status wait queue because status changes */
-	/* can't happen after shutting down the hardware */
-	wake_up_interruptible(&info->status_event_wait_q);
-	wake_up_interruptible(&info->event_wait_q);
-
-	timer_delete_sync(&info->tx_timer);
-	timer_delete_sync(&info->rx_timer);
-
-	kfree(info->tx_buf);
-	info->tx_buf = NULL;
-
-	spin_lock_irqsave(&info->lock,flags);
-
-	tx_stop(info);
-	rx_stop(info);
-
-	slgt_irq_off(info, IRQ_ALL | IRQ_MASTER);
-
- 	if (!info->port.tty || info->port.tty->termios.c_cflag & HUPCL) {
-		info->signals &= ~(SerialSignal_RTS | SerialSignal_DTR);
-		set_gtsignals(info);
-	}
-
-	flush_cond_wait(&info->gpio_wait_q);
-
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	if (info->port.tty)
-		set_bit(TTY_IO_ERROR, &info->port.tty->flags);
-
-	tty_port_set_initialized(&info->port, false);
-}
-
-static void program_hw(struct slgt_info *info)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock,flags);
-
-	rx_stop(info);
-	tx_stop(info);
-
-	if (info->params.mode != MGSL_MODE_ASYNC ||
-	    info->netcount)
-		sync_mode(info);
-	else
-		async_mode(info);
-
-	set_gtsignals(info);
-
-	info->dcd_chkcount = 0;
-	info->cts_chkcount = 0;
-	info->ri_chkcount = 0;
-	info->dsr_chkcount = 0;
-
-	slgt_irq_on(info, IRQ_DCD | IRQ_CTS | IRQ_DSR | IRQ_RI);
-	get_gtsignals(info);
-
-	if (info->netcount ||
-	    (info->port.tty && info->port.tty->termios.c_cflag & CREAD))
-		rx_start(info);
-
-	spin_unlock_irqrestore(&info->lock,flags);
-}
-
-/*
- * reconfigure adapter based on new parameters
- */
-static void change_params(struct slgt_info *info)
-{
-	unsigned cflag;
-	int bits_per_char;
-
-	if (!info->port.tty)
-		return;
-	DBGINFO(("%s change_params\n", info->device_name));
-
-	cflag = info->port.tty->termios.c_cflag;
-
-	/* if B0 rate (hangup) specified then negate RTS and DTR */
-	/* otherwise assert RTS and DTR */
- 	if (cflag & CBAUD)
-		info->signals |= SerialSignal_RTS | SerialSignal_DTR;
-	else
-		info->signals &= ~(SerialSignal_RTS | SerialSignal_DTR);
-
-	/* byte size and parity */
-
-	info->params.data_bits = tty_get_char_size(cflag);
-	info->params.stop_bits = (cflag & CSTOPB) ? 2 : 1;
-
-	if (cflag & PARENB)
-		info->params.parity = (cflag & PARODD) ? ASYNC_PARITY_ODD : ASYNC_PARITY_EVEN;
-	else
-		info->params.parity = ASYNC_PARITY_NONE;
-
-	/* calculate number of jiffies to transmit a full
-	 * FIFO (32 bytes) at specified data rate
-	 */
-	bits_per_char = info->params.data_bits +
-			info->params.stop_bits + 1;
-
-	info->params.data_rate = tty_get_baud_rate(info->port.tty);
-
-	if (info->params.data_rate) {
-		info->timeout = (32*HZ*bits_per_char) /
-				info->params.data_rate;
-	}
-	info->timeout += HZ/50;		/* Add .02 seconds of slop */
-
-	tty_port_set_cts_flow(&info->port, cflag & CRTSCTS);
-	tty_port_set_check_carrier(&info->port, ~cflag & CLOCAL);
-
-	/* process tty input control flags */
-
-	info->read_status_mask = IRQ_RXOVER;
-	if (I_INPCK(info->port.tty))
-		info->read_status_mask |= MASK_PARITY | MASK_FRAMING;
-	if (I_BRKINT(info->port.tty) || I_PARMRK(info->port.tty))
-		info->read_status_mask |= MASK_BREAK;
-	if (I_IGNPAR(info->port.tty))
-		info->ignore_status_mask |= MASK_PARITY | MASK_FRAMING;
-	if (I_IGNBRK(info->port.tty)) {
-		info->ignore_status_mask |= MASK_BREAK;
-		/* If ignoring parity and break indicators, ignore
-		 * overruns too.  (For real raw support).
-		 */
-		if (I_IGNPAR(info->port.tty))
-			info->ignore_status_mask |= MASK_OVERRUN;
-	}
-
-	program_hw(info);
-}
-
-static int get_stats(struct slgt_info *info, struct mgsl_icount __user *user_icount)
-{
-	DBGINFO(("%s get_stats\n",  info->device_name));
-	if (!user_icount) {
-		memset(&info->icount, 0, sizeof(info->icount));
-	} else {
-		if (copy_to_user(user_icount, &info->icount, sizeof(struct mgsl_icount)))
-			return -EFAULT;
-	}
-	return 0;
-}
-
-static int get_params(struct slgt_info *info, MGSL_PARAMS __user *user_params)
-{
-	DBGINFO(("%s get_params\n", info->device_name));
-	if (copy_to_user(user_params, &info->params, sizeof(MGSL_PARAMS)))
-		return -EFAULT;
-	return 0;
-}
-
-static int set_params(struct slgt_info *info, MGSL_PARAMS __user *new_params)
-{
- 	unsigned long flags;
-	MGSL_PARAMS tmp_params;
-
-	DBGINFO(("%s set_params\n", info->device_name));
-	if (copy_from_user(&tmp_params, new_params, sizeof(MGSL_PARAMS)))
-		return -EFAULT;
-
-	spin_lock_irqsave(&info->lock, flags);
-	if (tmp_params.mode == MGSL_MODE_BASE_CLOCK)
-		info->base_clock = tmp_params.clock_speed;
-	else
-		memcpy(&info->params, &tmp_params, sizeof(MGSL_PARAMS));
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	program_hw(info);
-
-	return 0;
-}
-
-static int get_txidle(struct slgt_info *info, int __user *idle_mode)
-{
-	DBGINFO(("%s get_txidle=%d\n", info->device_name, info->idle_mode));
-	if (put_user(info->idle_mode, idle_mode))
-		return -EFAULT;
-	return 0;
-}
-
-static int set_txidle(struct slgt_info *info, int idle_mode)
-{
- 	unsigned long flags;
-	DBGINFO(("%s set_txidle(%d)\n", info->device_name, idle_mode));
-	spin_lock_irqsave(&info->lock,flags);
-	info->idle_mode = idle_mode;
-	if (info->params.mode != MGSL_MODE_ASYNC)
-		tx_set_idle(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-	return 0;
-}
-
-static int tx_enable(struct slgt_info *info, int enable)
-{
- 	unsigned long flags;
-	DBGINFO(("%s tx_enable(%d)\n", info->device_name, enable));
-	spin_lock_irqsave(&info->lock,flags);
-	if (enable) {
-		if (!info->tx_enabled)
-			tx_start(info);
-	} else {
-		if (info->tx_enabled)
-			tx_stop(info);
-	}
-	spin_unlock_irqrestore(&info->lock,flags);
-	return 0;
-}
-
-/*
- * abort transmit HDLC frame
- */
-static int tx_abort(struct slgt_info *info)
-{
- 	unsigned long flags;
-	DBGINFO(("%s tx_abort\n", info->device_name));
-	spin_lock_irqsave(&info->lock,flags);
-	tdma_reset(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-	return 0;
-}
-
-static int rx_enable(struct slgt_info *info, int enable)
-{
- 	unsigned long flags;
-	unsigned int rbuf_fill_level;
-	DBGINFO(("%s rx_enable(%08x)\n", info->device_name, enable));
-	spin_lock_irqsave(&info->lock,flags);
-	/*
-	 * enable[31..16] = receive DMA buffer fill level
-	 * 0 = noop (leave fill level unchanged)
-	 * fill level must be multiple of 4 and <= buffer size
-	 */
-	rbuf_fill_level = ((unsigned int)enable) >> 16;
-	if (rbuf_fill_level) {
-		if ((rbuf_fill_level > DMABUFSIZE) || (rbuf_fill_level % 4)) {
-			spin_unlock_irqrestore(&info->lock, flags);
-			return -EINVAL;
-		}
-		info->rbuf_fill_level = rbuf_fill_level;
-		if (rbuf_fill_level < 128)
-			info->rx_pio = 1; /* PIO mode */
-		else
-			info->rx_pio = 0; /* DMA mode */
-		rx_stop(info); /* restart receiver to use new fill level */
-	}
-
-	/*
-	 * enable[1..0] = receiver enable command
-	 * 0 = disable
-	 * 1 = enable
-	 * 2 = enable or force hunt mode if already enabled
-	 */
-	enable &= 3;
-	if (enable) {
-		if (!info->rx_enabled)
-			rx_start(info);
-		else if (enable == 2) {
-			/* force hunt mode (write 1 to RCR[3]) */
-			wr_reg16(info, RCR, rd_reg16(info, RCR) | BIT3);
-		}
-	} else {
-		if (info->rx_enabled)
-			rx_stop(info);
-	}
-	spin_unlock_irqrestore(&info->lock,flags);
-	return 0;
-}
-
-/*
- *  wait for specified event to occur
- */
-static int wait_mgsl_event(struct slgt_info *info, int __user *mask_ptr)
-{
- 	unsigned long flags;
-	int s;
-	int rc=0;
-	struct mgsl_icount cprev, cnow;
-	int events;
-	int mask;
-	struct	_input_signal_events oldsigs, newsigs;
-	DECLARE_WAITQUEUE(wait, current);
-
-	if (get_user(mask, mask_ptr))
-		return -EFAULT;
-
-	DBGINFO(("%s wait_mgsl_event(%d)\n", info->device_name, mask));
-
-	spin_lock_irqsave(&info->lock,flags);
-
-	/* return immediately if state matches requested events */
-	get_gtsignals(info);
-	s = info->signals;
-
-	events = mask &
-		( ((s & SerialSignal_DSR) ? MgslEvent_DsrActive:MgslEvent_DsrInactive) +
- 		  ((s & SerialSignal_DCD) ? MgslEvent_DcdActive:MgslEvent_DcdInactive) +
-		  ((s & SerialSignal_CTS) ? MgslEvent_CtsActive:MgslEvent_CtsInactive) +
-		  ((s & SerialSignal_RI)  ? MgslEvent_RiActive :MgslEvent_RiInactive) );
-	if (events) {
-		spin_unlock_irqrestore(&info->lock,flags);
-		goto exit;
-	}
-
-	/* save current irq counts */
-	cprev = info->icount;
-	oldsigs = info->input_signal_events;
-
-	/* enable hunt and idle irqs if needed */
-	if (mask & (MgslEvent_ExitHuntMode+MgslEvent_IdleReceived)) {
-		unsigned short val = rd_reg16(info, SCR);
-		if (!(val & IRQ_RXIDLE))
-			wr_reg16(info, SCR, (unsigned short)(val | IRQ_RXIDLE));
-	}
-
-	set_current_state(TASK_INTERRUPTIBLE);
-	add_wait_queue(&info->event_wait_q, &wait);
-
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	for(;;) {
-		schedule();
-		if (signal_pending(current)) {
-			rc = -ERESTARTSYS;
-			break;
-		}
-
-		/* get current irq counts */
-		spin_lock_irqsave(&info->lock,flags);
-		cnow = info->icount;
-		newsigs = info->input_signal_events;
-		set_current_state(TASK_INTERRUPTIBLE);
-		spin_unlock_irqrestore(&info->lock,flags);
-
-		/* if no change, wait aborted for some reason */
-		if (newsigs.dsr_up   == oldsigs.dsr_up   &&
-		    newsigs.dsr_down == oldsigs.dsr_down &&
-		    newsigs.dcd_up   == oldsigs.dcd_up   &&
-		    newsigs.dcd_down == oldsigs.dcd_down &&
-		    newsigs.cts_up   == oldsigs.cts_up   &&
-		    newsigs.cts_down == oldsigs.cts_down &&
-		    newsigs.ri_up    == oldsigs.ri_up    &&
-		    newsigs.ri_down  == oldsigs.ri_down  &&
-		    cnow.exithunt    == cprev.exithunt   &&
-		    cnow.rxidle      == cprev.rxidle) {
-			rc = -EIO;
-			break;
-		}
-
-		events = mask &
-			( (newsigs.dsr_up   != oldsigs.dsr_up   ? MgslEvent_DsrActive:0)   +
-			  (newsigs.dsr_down != oldsigs.dsr_down ? MgslEvent_DsrInactive:0) +
-			  (newsigs.dcd_up   != oldsigs.dcd_up   ? MgslEvent_DcdActive:0)   +
-			  (newsigs.dcd_down != oldsigs.dcd_down ? MgslEvent_DcdInactive:0) +
-			  (newsigs.cts_up   != oldsigs.cts_up   ? MgslEvent_CtsActive:0)   +
-			  (newsigs.cts_down != oldsigs.cts_down ? MgslEvent_CtsInactive:0) +
-			  (newsigs.ri_up    != oldsigs.ri_up    ? MgslEvent_RiActive:0)    +
-			  (newsigs.ri_down  != oldsigs.ri_down  ? MgslEvent_RiInactive:0)  +
-			  (cnow.exithunt    != cprev.exithunt   ? MgslEvent_ExitHuntMode:0) +
-			  (cnow.rxidle      != cprev.rxidle     ? MgslEvent_IdleReceived:0) );
-		if (events)
-			break;
-
-		cprev = cnow;
-		oldsigs = newsigs;
-	}
-
-	remove_wait_queue(&info->event_wait_q, &wait);
-	set_current_state(TASK_RUNNING);
-
-
-	if (mask & (MgslEvent_ExitHuntMode + MgslEvent_IdleReceived)) {
-		spin_lock_irqsave(&info->lock,flags);
-		if (!waitqueue_active(&info->event_wait_q)) {
-			/* disable enable exit hunt mode/idle rcvd IRQs */
-			wr_reg16(info, SCR,
-				(unsigned short)(rd_reg16(info, SCR) & ~IRQ_RXIDLE));
-		}
-		spin_unlock_irqrestore(&info->lock,flags);
-	}
-exit:
-	if (rc == 0)
-		rc = put_user(events, mask_ptr);
-	return rc;
-}
-
-static int get_interface(struct slgt_info *info, int __user *if_mode)
-{
-	DBGINFO(("%s get_interface=%x\n", info->device_name, info->if_mode));
-	if (put_user(info->if_mode, if_mode))
-		return -EFAULT;
-	return 0;
-}
-
-static int set_interface(struct slgt_info *info, int if_mode)
-{
- 	unsigned long flags;
-	unsigned short val;
-
-	DBGINFO(("%s set_interface=%x)\n", info->device_name, if_mode));
-	spin_lock_irqsave(&info->lock,flags);
-	info->if_mode = if_mode;
-
-	msc_set_vcr(info);
-
-	/* TCR (tx control) 07  1=RTS driver control */
-	val = rd_reg16(info, TCR);
-	if (info->if_mode & MGSL_INTERFACE_RTS_EN)
-		val |= BIT7;
-	else
-		val &= ~BIT7;
-	wr_reg16(info, TCR, val);
-
-	spin_unlock_irqrestore(&info->lock,flags);
-	return 0;
-}
-
-static int get_xsync(struct slgt_info *info, int __user *xsync)
-{
-	DBGINFO(("%s get_xsync=%x\n", info->device_name, info->xsync));
-	if (put_user(info->xsync, xsync))
-		return -EFAULT;
-	return 0;
-}
-
-/*
- * set extended sync pattern (1 to 4 bytes) for extended sync mode
- *
- * sync pattern is contained in least significant bytes of value
- * most significant byte of sync pattern is oldest (1st sent/detected)
- */
-static int set_xsync(struct slgt_info *info, int xsync)
-{
-	unsigned long flags;
-
-	DBGINFO(("%s set_xsync=%x)\n", info->device_name, xsync));
-	spin_lock_irqsave(&info->lock, flags);
-	info->xsync = xsync;
-	wr_reg32(info, XSR, xsync);
-	spin_unlock_irqrestore(&info->lock, flags);
-	return 0;
-}
-
-static int get_xctrl(struct slgt_info *info, int __user *xctrl)
-{
-	DBGINFO(("%s get_xctrl=%x\n", info->device_name, info->xctrl));
-	if (put_user(info->xctrl, xctrl))
-		return -EFAULT;
-	return 0;
-}
-
-/*
- * set extended control options
- *
- * xctrl[31:19] reserved, must be zero
- * xctrl[18:17] extended sync pattern length in bytes
- *              00 = 1 byte  in xsr[7:0]
- *              01 = 2 bytes in xsr[15:0]
- *              10 = 3 bytes in xsr[23:0]
- *              11 = 4 bytes in xsr[31:0]
- * xctrl[16]    1 = enable terminal count, 0=disabled
- * xctrl[15:0]  receive terminal count for fixed length packets
- *              value is count minus one (0 = 1 byte packet)
- *              when terminal count is reached, receiver
- *              automatically returns to hunt mode and receive
- *              FIFO contents are flushed to DMA buffers with
- *              end of frame (EOF) status
- */
-static int set_xctrl(struct slgt_info *info, int xctrl)
-{
-	unsigned long flags;
-
-	DBGINFO(("%s set_xctrl=%x)\n", info->device_name, xctrl));
-	spin_lock_irqsave(&info->lock, flags);
-	info->xctrl = xctrl;
-	wr_reg32(info, XCR, xctrl);
-	spin_unlock_irqrestore(&info->lock, flags);
-	return 0;
-}
-
-/*
- * set general purpose IO pin state and direction
- *
- * user_gpio fields:
- * state   each bit indicates a pin state
- * smask   set bit indicates pin state to set
- * dir     each bit indicates a pin direction (0=input, 1=output)
- * dmask   set bit indicates pin direction to set
- */
-static int set_gpio(struct slgt_info *info, struct gpio_desc __user *user_gpio)
-{
- 	unsigned long flags;
-	struct gpio_desc gpio;
-	__u32 data;
-
-	if (!info->gpio_present)
-		return -EINVAL;
-	if (copy_from_user(&gpio, user_gpio, sizeof(gpio)))
-		return -EFAULT;
-	DBGINFO(("%s set_gpio state=%08x smask=%08x dir=%08x dmask=%08x\n",
-		 info->device_name, gpio.state, gpio.smask,
-		 gpio.dir, gpio.dmask));
-
-	spin_lock_irqsave(&info->port_array[0]->lock, flags);
-	if (gpio.dmask) {
-		data = rd_reg32(info, IODR);
-		data |= gpio.dmask & gpio.dir;
-		data &= ~(gpio.dmask & ~gpio.dir);
-		wr_reg32(info, IODR, data);
-	}
-	if (gpio.smask) {
-		data = rd_reg32(info, IOVR);
-		data |= gpio.smask & gpio.state;
-		data &= ~(gpio.smask & ~gpio.state);
-		wr_reg32(info, IOVR, data);
-	}
-	spin_unlock_irqrestore(&info->port_array[0]->lock, flags);
-
-	return 0;
-}
-
-/*
- * get general purpose IO pin state and direction
- */
-static int get_gpio(struct slgt_info *info, struct gpio_desc __user *user_gpio)
-{
-	struct gpio_desc gpio;
-	if (!info->gpio_present)
-		return -EINVAL;
-	gpio.state = rd_reg32(info, IOVR);
-	gpio.smask = 0xffffffff;
-	gpio.dir   = rd_reg32(info, IODR);
-	gpio.dmask = 0xffffffff;
-	if (copy_to_user(user_gpio, &gpio, sizeof(gpio)))
-		return -EFAULT;
-	DBGINFO(("%s get_gpio state=%08x dir=%08x\n",
-		 info->device_name, gpio.state, gpio.dir));
-	return 0;
-}
-
-/*
- * conditional wait facility
- */
-static void init_cond_wait(struct cond_wait *w, unsigned int data)
-{
-	init_waitqueue_head(&w->q);
-	init_waitqueue_entry(&w->wait, current);
-	w->data = data;
-}
-
-static void add_cond_wait(struct cond_wait **head, struct cond_wait *w)
-{
-	set_current_state(TASK_INTERRUPTIBLE);
-	add_wait_queue(&w->q, &w->wait);
-	w->next = *head;
-	*head = w;
-}
-
-static void remove_cond_wait(struct cond_wait **head, struct cond_wait *cw)
-{
-	struct cond_wait *w, *prev;
-	remove_wait_queue(&cw->q, &cw->wait);
-	set_current_state(TASK_RUNNING);
-	for (w = *head, prev = NULL ; w != NULL ; prev = w, w = w->next) {
-		if (w == cw) {
-			if (prev != NULL)
-				prev->next = w->next;
-			else
-				*head = w->next;
-			break;
-		}
-	}
-}
-
-static void flush_cond_wait(struct cond_wait **head)
-{
-	while (*head != NULL) {
-		wake_up_interruptible(&(*head)->q);
-		*head = (*head)->next;
-	}
-}
-
-/*
- * wait for general purpose I/O pin(s) to enter specified state
- *
- * user_gpio fields:
- * state - bit indicates target pin state
- * smask - set bit indicates watched pin
- *
- * The wait ends when at least one watched pin enters the specified
- * state. When 0 (no error) is returned, user_gpio->state is set to the
- * state of all GPIO pins when the wait ends.
- *
- * Note: Each pin may be a dedicated input, dedicated output, or
- * configurable input/output. The number and configuration of pins
- * varies with the specific adapter model. Only input pins (dedicated
- * or configured) can be monitored with this function.
- */
-static int wait_gpio(struct slgt_info *info, struct gpio_desc __user *user_gpio)
-{
- 	unsigned long flags;
-	int rc = 0;
-	struct gpio_desc gpio;
-	struct cond_wait wait;
-	u32 state;
-
-	if (!info->gpio_present)
-		return -EINVAL;
-	if (copy_from_user(&gpio, user_gpio, sizeof(gpio)))
-		return -EFAULT;
-	DBGINFO(("%s wait_gpio() state=%08x smask=%08x\n",
-		 info->device_name, gpio.state, gpio.smask));
-	/* ignore output pins identified by set IODR bit */
-	if ((gpio.smask &= ~rd_reg32(info, IODR)) == 0)
-		return -EINVAL;
-	init_cond_wait(&wait, gpio.smask);
-
-	spin_lock_irqsave(&info->port_array[0]->lock, flags);
-	/* enable interrupts for watched pins */
-	wr_reg32(info, IOER, rd_reg32(info, IOER) | gpio.smask);
-	/* get current pin states */
-	state = rd_reg32(info, IOVR);
-
-	if (gpio.smask & ~(state ^ gpio.state)) {
-		/* already in target state */
-		gpio.state = state;
-	} else {
-		/* wait for target state */
-		add_cond_wait(&info->gpio_wait_q, &wait);
-		spin_unlock_irqrestore(&info->port_array[0]->lock, flags);
-		schedule();
-		if (signal_pending(current))
-			rc = -ERESTARTSYS;
-		else
-			gpio.state = wait.data;
-		spin_lock_irqsave(&info->port_array[0]->lock, flags);
-		remove_cond_wait(&info->gpio_wait_q, &wait);
-	}
-
-	/* disable all GPIO interrupts if no waiting processes */
-	if (info->gpio_wait_q == NULL)
-		wr_reg32(info, IOER, 0);
-	spin_unlock_irqrestore(&info->port_array[0]->lock, flags);
-
-	if ((rc == 0) && copy_to_user(user_gpio, &gpio, sizeof(gpio)))
-		rc = -EFAULT;
-	return rc;
-}
-
-static int modem_input_wait(struct slgt_info *info,int arg)
-{
- 	unsigned long flags;
-	int rc;
-	struct mgsl_icount cprev, cnow;
-	DECLARE_WAITQUEUE(wait, current);
-
-	/* save current irq counts */
-	spin_lock_irqsave(&info->lock,flags);
-	cprev = info->icount;
-	add_wait_queue(&info->status_event_wait_q, &wait);
-	set_current_state(TASK_INTERRUPTIBLE);
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	for(;;) {
-		schedule();
-		if (signal_pending(current)) {
-			rc = -ERESTARTSYS;
-			break;
-		}
-
-		/* get new irq counts */
-		spin_lock_irqsave(&info->lock,flags);
-		cnow = info->icount;
-		set_current_state(TASK_INTERRUPTIBLE);
-		spin_unlock_irqrestore(&info->lock,flags);
-
-		/* if no change, wait aborted for some reason */
-		if (cnow.rng == cprev.rng && cnow.dsr == cprev.dsr &&
-		    cnow.dcd == cprev.dcd && cnow.cts == cprev.cts) {
-			rc = -EIO;
-			break;
-		}
-
-		/* check for change in caller specified modem input */
-		if ((arg & TIOCM_RNG && cnow.rng != cprev.rng) ||
-		    (arg & TIOCM_DSR && cnow.dsr != cprev.dsr) ||
-		    (arg & TIOCM_CD  && cnow.dcd != cprev.dcd) ||
-		    (arg & TIOCM_CTS && cnow.cts != cprev.cts)) {
-			rc = 0;
-			break;
-		}
-
-		cprev = cnow;
-	}
-	remove_wait_queue(&info->status_event_wait_q, &wait);
-	set_current_state(TASK_RUNNING);
-	return rc;
-}
-
-/*
- *  return state of serial control and status signals
- */
-static int tiocmget(struct tty_struct *tty)
-{
-	struct slgt_info *info = tty->driver_data;
-	unsigned int result;
- 	unsigned long flags;
-
-	spin_lock_irqsave(&info->lock,flags);
- 	get_gtsignals(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	result = ((info->signals & SerialSignal_RTS) ? TIOCM_RTS:0) +
-		((info->signals & SerialSignal_DTR) ? TIOCM_DTR:0) +
-		((info->signals & SerialSignal_DCD) ? TIOCM_CAR:0) +
-		((info->signals & SerialSignal_RI)  ? TIOCM_RNG:0) +
-		((info->signals & SerialSignal_DSR) ? TIOCM_DSR:0) +
-		((info->signals & SerialSignal_CTS) ? TIOCM_CTS:0);
-
-	DBGINFO(("%s tiocmget value=%08X\n", info->device_name, result));
-	return result;
-}
-
-/*
- * set modem control signals (DTR/RTS)
- *
- * 	cmd	signal command: TIOCMBIS = set bit TIOCMBIC = clear bit
- *		TIOCMSET = set/clear signal values
- * 	value	bit mask for command
- */
-static int tiocmset(struct tty_struct *tty,
-		    unsigned int set, unsigned int clear)
-{
-	struct slgt_info *info = tty->driver_data;
- 	unsigned long flags;
-
-	DBGINFO(("%s tiocmset(%x,%x)\n", info->device_name, set, clear));
-
-	if (set & TIOCM_RTS)
-		info->signals |= SerialSignal_RTS;
-	if (set & TIOCM_DTR)
-		info->signals |= SerialSignal_DTR;
-	if (clear & TIOCM_RTS)
-		info->signals &= ~SerialSignal_RTS;
-	if (clear & TIOCM_DTR)
-		info->signals &= ~SerialSignal_DTR;
-
-	spin_lock_irqsave(&info->lock,flags);
-	set_gtsignals(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-	return 0;
-}
-
-static bool carrier_raised(struct tty_port *port)
-{
-	unsigned long flags;
-	struct slgt_info *info = container_of(port, struct slgt_info, port);
-
-	spin_lock_irqsave(&info->lock,flags);
-	get_gtsignals(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	return info->signals & SerialSignal_DCD;
-}
-
-static void dtr_rts(struct tty_port *port, bool active)
-{
-	unsigned long flags;
-	struct slgt_info *info = container_of(port, struct slgt_info, port);
-
-	spin_lock_irqsave(&info->lock,flags);
-	if (active)
-		info->signals |= SerialSignal_RTS | SerialSignal_DTR;
-	else
-		info->signals &= ~(SerialSignal_RTS | SerialSignal_DTR);
-	set_gtsignals(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-}
-
-
-/*
- *  block current process until the device is ready to open
- */
-static int block_til_ready(struct tty_struct *tty, struct file *filp,
-			   struct slgt_info *info)
-{
-	DECLARE_WAITQUEUE(wait, current);
-	int		retval;
-	bool		do_clocal = false;
-	unsigned long	flags;
-	bool		cd;
-	struct tty_port *port = &info->port;
-
-	DBGINFO(("%s block_til_ready\n", tty->driver->name));
-
-	if (filp->f_flags & O_NONBLOCK || tty_io_error(tty)) {
-		/* nonblock mode is set or port is not enabled */
-		tty_port_set_active(port, true);
-		return 0;
-	}
-
-	if (C_CLOCAL(tty))
-		do_clocal = true;
-
-	/* Wait for carrier detect and the line to become
-	 * free (i.e., not in use by the callout).  While we are in
-	 * this loop, port->count is dropped by one, so that
-	 * close() knows when to free things.  We restore it upon
-	 * exit, either normal or abnormal.
-	 */
-
-	retval = 0;
-	add_wait_queue(&port->open_wait, &wait);
-
-	spin_lock_irqsave(&info->lock, flags);
-	port->count--;
-	spin_unlock_irqrestore(&info->lock, flags);
-	port->blocked_open++;
-
-	while (1) {
-		if (C_BAUD(tty) && tty_port_initialized(port))
-			tty_port_raise_dtr_rts(port);
-
-		set_current_state(TASK_INTERRUPTIBLE);
-
-		if (tty_hung_up_p(filp) || !tty_port_initialized(port)) {
-			retval = (port->flags & ASYNC_HUP_NOTIFY) ?
-					-EAGAIN : -ERESTARTSYS;
-			break;
-		}
-
-		cd = tty_port_carrier_raised(port);
-		if (do_clocal || cd)
-			break;
-
-		if (signal_pending(current)) {
-			retval = -ERESTARTSYS;
-			break;
-		}
-
-		DBGINFO(("%s block_til_ready wait\n", tty->driver->name));
-		tty_unlock(tty);
-		schedule();
-		tty_lock(tty);
-	}
-
-	set_current_state(TASK_RUNNING);
-	remove_wait_queue(&port->open_wait, &wait);
-
-	if (!tty_hung_up_p(filp))
-		port->count++;
-	port->blocked_open--;
-
-	if (!retval)
-		tty_port_set_active(port, true);
-
-	DBGINFO(("%s block_til_ready ready, rc=%d\n", tty->driver->name, retval));
-	return retval;
-}
-
-/*
- * allocate buffers used for calling line discipline receive_buf
- * directly in synchronous mode
- * note: add 5 bytes to max frame size to allow appending
- * 32-bit CRC and status byte when configured to do so
- */
-static int alloc_tmp_rbuf(struct slgt_info *info)
-{
-	info->tmp_rbuf = kmalloc(info->max_frame_size + 5, GFP_KERNEL);
-	if (info->tmp_rbuf == NULL)
-		return -ENOMEM;
-
-	return 0;
-}
-
-static void free_tmp_rbuf(struct slgt_info *info)
-{
-	kfree(info->tmp_rbuf);
-	info->tmp_rbuf = NULL;
-}
-
-/*
- * allocate DMA descriptor lists.
- */
-static int alloc_desc(struct slgt_info *info)
-{
-	unsigned int i;
-	unsigned int pbufs;
-
-	/* allocate memory to hold descriptor lists */
-	info->bufs = dma_alloc_coherent(&info->pdev->dev, DESC_LIST_SIZE,
-					&info->bufs_dma_addr, GFP_KERNEL);
-	if (info->bufs == NULL)
-		return -ENOMEM;
-
-	info->rbufs = (struct slgt_desc*)info->bufs;
-	info->tbufs = ((struct slgt_desc*)info->bufs) + info->rbuf_count;
-
-	pbufs = (unsigned int)info->bufs_dma_addr;
-
-	/*
-	 * Build circular lists of descriptors
-	 */
-
-	for (i=0; i < info->rbuf_count; i++) {
-		/* physical address of this descriptor */
-		info->rbufs[i].pdesc = pbufs + (i * sizeof(struct slgt_desc));
-
-		/* physical address of next descriptor */
-		if (i == info->rbuf_count - 1)
-			info->rbufs[i].next = cpu_to_le32(pbufs);
-		else
-			info->rbufs[i].next = cpu_to_le32(pbufs + ((i+1) * sizeof(struct slgt_desc)));
-		set_desc_count(info->rbufs[i], DMABUFSIZE);
-	}
-
-	for (i=0; i < info->tbuf_count; i++) {
-		/* physical address of this descriptor */
-		info->tbufs[i].pdesc = pbufs + ((info->rbuf_count + i) * sizeof(struct slgt_desc));
-
-		/* physical address of next descriptor */
-		if (i == info->tbuf_count - 1)
-			info->tbufs[i].next = cpu_to_le32(pbufs + info->rbuf_count * sizeof(struct slgt_desc));
-		else
-			info->tbufs[i].next = cpu_to_le32(pbufs + ((info->rbuf_count + i + 1) * sizeof(struct slgt_desc)));
-	}
-
-	return 0;
-}
-
-static void free_desc(struct slgt_info *info)
-{
-	if (info->bufs != NULL) {
-		dma_free_coherent(&info->pdev->dev, DESC_LIST_SIZE,
-				  info->bufs, info->bufs_dma_addr);
-		info->bufs  = NULL;
-		info->rbufs = NULL;
-		info->tbufs = NULL;
-	}
-}
-
-static int alloc_bufs(struct slgt_info *info, struct slgt_desc *bufs, int count)
-{
-	int i;
-	for (i=0; i < count; i++) {
-		bufs[i].buf = dma_alloc_coherent(&info->pdev->dev, DMABUFSIZE,
-						 &bufs[i].buf_dma_addr, GFP_KERNEL);
-		if (!bufs[i].buf)
-			return -ENOMEM;
-		bufs[i].pbuf  = cpu_to_le32((unsigned int)bufs[i].buf_dma_addr);
-	}
-	return 0;
-}
-
-static void free_bufs(struct slgt_info *info, struct slgt_desc *bufs, int count)
-{
-	int i;
-	for (i=0; i < count; i++) {
-		if (bufs[i].buf == NULL)
-			continue;
-		dma_free_coherent(&info->pdev->dev, DMABUFSIZE, bufs[i].buf,
-				  bufs[i].buf_dma_addr);
-		bufs[i].buf = NULL;
-	}
-}
-
-static int alloc_dma_bufs(struct slgt_info *info)
-{
-	info->rbuf_count = 32;
-	info->tbuf_count = 32;
-
-	if (alloc_desc(info) < 0 ||
-	    alloc_bufs(info, info->rbufs, info->rbuf_count) < 0 ||
-	    alloc_bufs(info, info->tbufs, info->tbuf_count) < 0 ||
-	    alloc_tmp_rbuf(info) < 0) {
-		DBGERR(("%s DMA buffer alloc fail\n", info->device_name));
-		return -ENOMEM;
-	}
-	reset_rbufs(info);
-	return 0;
-}
-
-static void free_dma_bufs(struct slgt_info *info)
-{
-	if (info->bufs) {
-		free_bufs(info, info->rbufs, info->rbuf_count);
-		free_bufs(info, info->tbufs, info->tbuf_count);
-		free_desc(info);
-	}
-	free_tmp_rbuf(info);
-}
-
-static int claim_resources(struct slgt_info *info)
-{
-	if (request_mem_region(info->phys_reg_addr, SLGT_REG_SIZE, "synclink_gt") == NULL) {
-		DBGERR(("%s reg addr conflict, addr=%08X\n",
-			info->device_name, info->phys_reg_addr));
-		info->init_error = DiagStatus_AddressConflict;
-		goto errout;
-	}
-	else
-		info->reg_addr_requested = true;
-
-	info->reg_addr = ioremap(info->phys_reg_addr, SLGT_REG_SIZE);
-	if (!info->reg_addr) {
-		DBGERR(("%s can't map device registers, addr=%08X\n",
-			info->device_name, info->phys_reg_addr));
-		info->init_error = DiagStatus_CantAssignPciResources;
-		goto errout;
-	}
-	return 0;
-
-errout:
-	release_resources(info);
-	return -ENODEV;
-}
-
-static void release_resources(struct slgt_info *info)
-{
-	if (info->irq_requested) {
-		free_irq(info->irq_level, info);
-		info->irq_requested = false;
-	}
-
-	if (info->reg_addr_requested) {
-		release_mem_region(info->phys_reg_addr, SLGT_REG_SIZE);
-		info->reg_addr_requested = false;
-	}
-
-	if (info->reg_addr) {
-		iounmap(info->reg_addr);
-		info->reg_addr = NULL;
-	}
-}
-
-/* Add the specified device instance data structure to the
- * global linked list of devices and increment the device count.
- */
-static void add_device(struct slgt_info *info)
-{
-	char *devstr;
-
-	info->next_device = NULL;
-	info->line = slgt_device_count;
-	sprintf(info->device_name, "%s%d", tty_dev_prefix, info->line);
-
-	if (info->line < MAX_DEVICES) {
-		if (maxframe[info->line])
-			info->max_frame_size = maxframe[info->line];
-	}
-
-	slgt_device_count++;
-
-	if (!slgt_device_list)
-		slgt_device_list = info;
-	else {
-		struct slgt_info *current_dev = slgt_device_list;
-		while(current_dev->next_device)
-			current_dev = current_dev->next_device;
-		current_dev->next_device = info;
-	}
-
-	if (info->max_frame_size < 4096)
-		info->max_frame_size = 4096;
-	else if (info->max_frame_size > 65535)
-		info->max_frame_size = 65535;
-
-	switch(info->pdev->device) {
-	case SYNCLINK_GT_DEVICE_ID:
-		devstr = "GT";
-		break;
-	case SYNCLINK_GT2_DEVICE_ID:
-		devstr = "GT2";
-		break;
-	case SYNCLINK_GT4_DEVICE_ID:
-		devstr = "GT4";
-		break;
-	case SYNCLINK_AC_DEVICE_ID:
-		devstr = "AC";
-		info->params.mode = MGSL_MODE_ASYNC;
-		break;
-	default:
-		devstr = "(unknown model)";
-	}
-	printk("SyncLink %s %s IO=%08x IRQ=%d MaxFrameSize=%u\n",
-		devstr, info->device_name, info->phys_reg_addr,
-		info->irq_level, info->max_frame_size);
-
-#if SYNCLINK_GENERIC_HDLC
-	hdlcdev_init(info);
-#endif
-}
-
-static const struct tty_port_operations slgt_port_ops = {
-	.carrier_raised = carrier_raised,
-	.dtr_rts = dtr_rts,
-};
-
-/*
- *  allocate device instance structure, return NULL on failure
- */
-static struct slgt_info *alloc_dev(int adapter_num, int port_num, struct pci_dev *pdev)
-{
-	struct slgt_info *info;
-
-	info = kzalloc_obj(struct slgt_info);
-
-	if (!info) {
-		DBGERR(("%s device alloc failed adapter=%d port=%d\n",
-			driver_name, adapter_num, port_num));
-	} else {
-		tty_port_init(&info->port);
-		info->port.ops = &slgt_port_ops;
-		INIT_WORK(&info->task, bh_handler);
-		info->max_frame_size = 4096;
-		info->base_clock = 14745600;
-		info->rbuf_fill_level = DMABUFSIZE;
-		init_waitqueue_head(&info->status_event_wait_q);
-		init_waitqueue_head(&info->event_wait_q);
-		spin_lock_init(&info->netlock);
-		memcpy(&info->params,&default_params,sizeof(MGSL_PARAMS));
-		info->idle_mode = HDLC_TXIDLE_FLAGS;
-		info->adapter_num = adapter_num;
-		info->port_num = port_num;
-
-		timer_setup(&info->tx_timer, tx_timeout, 0);
-		timer_setup(&info->rx_timer, rx_timeout, 0);
-
-		/* Copy configuration info to device instance data */
-		info->pdev = pdev;
-		info->irq_level = pdev->irq;
-		info->phys_reg_addr = pci_resource_start(pdev,0);
-
-		info->bus_type = MGSL_BUS_TYPE_PCI;
-		info->irq_flags = IRQF_SHARED;
-
-		info->init_error = -1; /* assume error, set to 0 on successful init */
-	}
-
-	return info;
-}
-
-static void device_init(int adapter_num, struct pci_dev *pdev)
-{
-	struct slgt_info *port_array[SLGT_MAX_PORTS];
-	int i;
-	int port_count = 1;
-
-	if (pdev->device == SYNCLINK_GT2_DEVICE_ID)
-		port_count = 2;
-	else if (pdev->device == SYNCLINK_GT4_DEVICE_ID)
-		port_count = 4;
-
-	/* allocate device instances for all ports */
-	for (i=0; i < port_count; ++i) {
-		port_array[i] = alloc_dev(adapter_num, i, pdev);
-		if (port_array[i] == NULL) {
-			for (--i; i >= 0; --i) {
-				tty_port_destroy(&port_array[i]->port);
-				kfree(port_array[i]);
-			}
-			return;
-		}
-	}
-
-	/* give copy of port_array to all ports and add to device list  */
-	for (i=0; i < port_count; ++i) {
-		memcpy(port_array[i]->port_array, port_array, sizeof(port_array));
-		add_device(port_array[i]);
-		port_array[i]->port_count = port_count;
-		spin_lock_init(&port_array[i]->lock);
-	}
-
-	/* Allocate and claim adapter resources */
-	if (!claim_resources(port_array[0])) {
-
-		alloc_dma_bufs(port_array[0]);
-
-		/* copy resource information from first port to others */
-		for (i = 1; i < port_count; ++i) {
-			port_array[i]->irq_level = port_array[0]->irq_level;
-			port_array[i]->reg_addr  = port_array[0]->reg_addr;
-			alloc_dma_bufs(port_array[i]);
-		}
-
-		if (request_irq(port_array[0]->irq_level,
-					slgt_interrupt,
-					port_array[0]->irq_flags,
-					port_array[0]->device_name,
-					port_array[0]) < 0) {
-			DBGERR(("%s request_irq failed IRQ=%d\n",
-				port_array[0]->device_name,
-				port_array[0]->irq_level));
-		} else {
-			port_array[0]->irq_requested = true;
-			adapter_test(port_array[0]);
-			for (i=1 ; i < port_count ; i++) {
-				port_array[i]->init_error = port_array[0]->init_error;
-				port_array[i]->gpio_present = port_array[0]->gpio_present;
-			}
-		}
-	}
-
-	for (i = 0; i < port_count; ++i) {
-		struct slgt_info *info = port_array[i];
-		tty_port_register_device(&info->port, serial_driver, info->line,
-				&info->pdev->dev);
-	}
-}
-
-static int init_one(struct pci_dev *dev,
-			      const struct pci_device_id *ent)
-{
-	if (pci_enable_device(dev)) {
-		printk("error enabling pci device %p\n", dev);
-		return -EIO;
-	}
-	pci_set_master(dev);
-	device_init(slgt_device_count, dev);
-	return 0;
-}
-
-static void remove_one(struct pci_dev *dev)
-{
-}
-
-static const struct tty_operations ops = {
-	.open = open,
-	.close = close,
-	.write = write,
-	.put_char = put_char,
-	.flush_chars = flush_chars,
-	.write_room = write_room,
-	.chars_in_buffer = chars_in_buffer,
-	.flush_buffer = flush_buffer,
-	.ioctl = ioctl,
-	.compat_ioctl = slgt_compat_ioctl,
-	.throttle = throttle,
-	.unthrottle = unthrottle,
-	.send_xchar = send_xchar,
-	.break_ctl = set_break,
-	.wait_until_sent = wait_until_sent,
-	.set_termios = set_termios,
-	.stop = tx_hold,
-	.start = tx_release,
-	.hangup = hangup,
-	.tiocmget = tiocmget,
-	.tiocmset = tiocmset,
-	.get_icount = get_icount,
-	.proc_show = synclink_gt_proc_show,
-};
-
-static void slgt_cleanup(void)
-{
-	struct slgt_info *info;
-	struct slgt_info *tmp;
-
-	if (serial_driver) {
-		for (info=slgt_device_list ; info != NULL ; info=info->next_device)
-			tty_unregister_device(serial_driver, info->line);
-		tty_unregister_driver(serial_driver);
-		tty_driver_kref_put(serial_driver);
-	}
-
-	/* reset devices */
-	info = slgt_device_list;
-	while(info) {
-		reset_port(info);
-		info = info->next_device;
-	}
-
-	/* release devices */
-	info = slgt_device_list;
-	while(info) {
-#if SYNCLINK_GENERIC_HDLC
-		hdlcdev_exit(info);
-#endif
-		free_dma_bufs(info);
-		free_tmp_rbuf(info);
-		if (info->port_num == 0)
-			release_resources(info);
-		tmp = info;
-		info = info->next_device;
-		tty_port_destroy(&tmp->port);
-		kfree(tmp);
-	}
-
-	if (pci_registered)
-		pci_unregister_driver(&pci_driver);
-}
-
-/*
- *  Driver initialization entry point.
- */
-static int __init slgt_init(void)
-{
-	int rc;
-
-	serial_driver = tty_alloc_driver(MAX_DEVICES, TTY_DRIVER_REAL_RAW |
-			TTY_DRIVER_DYNAMIC_DEV);
-	if (IS_ERR(serial_driver)) {
-		printk("%s can't allocate tty driver\n", driver_name);
-		return PTR_ERR(serial_driver);
-	}
-
-	/* Initialize the tty_driver structure */
-
-	serial_driver->driver_name = "synclink_gt";
-	serial_driver->name = tty_dev_prefix;
-	serial_driver->major = ttymajor;
-	serial_driver->minor_start = 64;
-	serial_driver->type = TTY_DRIVER_TYPE_SERIAL;
-	serial_driver->subtype = SERIAL_TYPE_NORMAL;
-	serial_driver->init_termios = tty_std_termios;
-	serial_driver->init_termios.c_cflag =
-		B9600 | CS8 | CREAD | HUPCL | CLOCAL;
-	serial_driver->init_termios.c_ispeed = 9600;
-	serial_driver->init_termios.c_ospeed = 9600;
-	tty_set_operations(serial_driver, &ops);
-	if ((rc = tty_register_driver(serial_driver)) < 0) {
-		DBGERR(("%s can't register serial driver\n", driver_name));
-		tty_driver_kref_put(serial_driver);
-		serial_driver = NULL;
-		goto error;
-	}
-
-	slgt_device_count = 0;
-	if ((rc = pci_register_driver(&pci_driver)) < 0) {
-		printk("%s pci_register_driver error=%d\n", driver_name, rc);
-		goto error;
-	}
-	pci_registered = true;
-
-	return 0;
-
-error:
-	slgt_cleanup();
-	return rc;
-}
-
-static void __exit slgt_exit(void)
-{
-	slgt_cleanup();
-}
-
-module_init(slgt_init);
-module_exit(slgt_exit);
-
-/*
- * register access routines
- */
-
-static inline void __iomem *calc_regaddr(struct slgt_info *info,
-					 unsigned int addr)
-{
-	void __iomem *reg_addr = info->reg_addr + addr;
-
-	if (addr >= 0x80)
-		reg_addr += info->port_num * 32;
-	else if (addr >= 0x40)
-		reg_addr += info->port_num * 16;
-
-	return reg_addr;
-}
-
-static __u8 rd_reg8(struct slgt_info *info, unsigned int addr)
-{
-	return readb(calc_regaddr(info, addr));
-}
-
-static void wr_reg8(struct slgt_info *info, unsigned int addr, __u8 value)
-{
-	writeb(value, calc_regaddr(info, addr));
-}
-
-static __u16 rd_reg16(struct slgt_info *info, unsigned int addr)
-{
-	return readw(calc_regaddr(info, addr));
-}
-
-static void wr_reg16(struct slgt_info *info, unsigned int addr, __u16 value)
-{
-	writew(value, calc_regaddr(info, addr));
-}
-
-static __u32 rd_reg32(struct slgt_info *info, unsigned int addr)
-{
-	return readl(calc_regaddr(info, addr));
-}
-
-static void wr_reg32(struct slgt_info *info, unsigned int addr, __u32 value)
-{
-	writel(value, calc_regaddr(info, addr));
-}
-
-static void rdma_reset(struct slgt_info *info)
-{
-	unsigned int i;
-
-	/* set reset bit */
-	wr_reg32(info, RDCSR, BIT1);
-
-	/* wait for enable bit cleared */
-	for(i=0 ; i < 1000 ; i++)
-		if (!(rd_reg32(info, RDCSR) & BIT0))
-			break;
-}
-
-static void tdma_reset(struct slgt_info *info)
-{
-	unsigned int i;
-
-	/* set reset bit */
-	wr_reg32(info, TDCSR, BIT1);
-
-	/* wait for enable bit cleared */
-	for(i=0 ; i < 1000 ; i++)
-		if (!(rd_reg32(info, TDCSR) & BIT0))
-			break;
-}
-
-/*
- * enable internal loopback
- * TxCLK and RxCLK are generated from BRG
- * and TxD is looped back to RxD internally.
- */
-static void enable_loopback(struct slgt_info *info)
-{
-	/* SCR (serial control) BIT2=loopback enable */
-	wr_reg16(info, SCR, (unsigned short)(rd_reg16(info, SCR) | BIT2));
-
-	if (info->params.mode != MGSL_MODE_ASYNC) {
-		/* CCR (clock control)
-		 * 07..05  tx clock source (010 = BRG)
-		 * 04..02  rx clock source (010 = BRG)
-		 * 01      auxclk enable   (0 = disable)
-		 * 00      BRG enable      (1 = enable)
-		 *
-		 * 0100 1001
-		 */
-		wr_reg8(info, CCR, 0x49);
-
-		/* set speed if available, otherwise use default */
-		if (info->params.clock_speed)
-			set_rate(info, info->params.clock_speed);
-		else
-			set_rate(info, 3686400);
-	}
-}
-
-/*
- *  set baud rate generator to specified rate
- */
-static void set_rate(struct slgt_info *info, u32 rate)
-{
-	unsigned int div;
-	unsigned int osc = info->base_clock;
-
-	/* div = osc/rate - 1
-	 *
-	 * Round div up if osc/rate is not integer to
-	 * force to next slowest rate.
-	 */
-
-	if (rate) {
-		div = osc/rate;
-		if (!(osc % rate) && div)
-			div--;
-		wr_reg16(info, BDR, (unsigned short)div);
-	}
-}
-
-static void rx_stop(struct slgt_info *info)
-{
-	unsigned short val;
-
-	/* disable and reset receiver */
-	val = rd_reg16(info, RCR) & ~BIT1;          /* clear enable bit */
-	wr_reg16(info, RCR, (unsigned short)(val | BIT2)); /* set reset bit */
-	wr_reg16(info, RCR, val);                  /* clear reset bit */
-
-	slgt_irq_off(info, IRQ_RXOVER + IRQ_RXDATA + IRQ_RXIDLE);
-
-	/* clear pending rx interrupts */
-	wr_reg16(info, SSR, IRQ_RXIDLE + IRQ_RXOVER);
-
-	rdma_reset(info);
-
-	info->rx_enabled = false;
-	info->rx_restart = false;
-}
-
-static void rx_start(struct slgt_info *info)
-{
-	unsigned short val;
-
-	slgt_irq_off(info, IRQ_RXOVER + IRQ_RXDATA);
-
-	/* clear pending rx overrun IRQ */
-	wr_reg16(info, SSR, IRQ_RXOVER);
-
-	/* reset and disable receiver */
-	val = rd_reg16(info, RCR) & ~BIT1; /* clear enable bit */
-	wr_reg16(info, RCR, (unsigned short)(val | BIT2)); /* set reset bit */
-	wr_reg16(info, RCR, val);                  /* clear reset bit */
-
-	rdma_reset(info);
-	reset_rbufs(info);
-
-	if (info->rx_pio) {
-		/* rx request when rx FIFO not empty */
-		wr_reg16(info, SCR, (unsigned short)(rd_reg16(info, SCR) & ~BIT14));
-		slgt_irq_on(info, IRQ_RXDATA);
-		if (info->params.mode == MGSL_MODE_ASYNC) {
-			/* enable saving of rx status */
-			wr_reg32(info, RDCSR, BIT6);
-		}
-	} else {
-		/* rx request when rx FIFO half full */
-		wr_reg16(info, SCR, (unsigned short)(rd_reg16(info, SCR) | BIT14));
-		/* set 1st descriptor address */
-		wr_reg32(info, RDDAR, info->rbufs[0].pdesc);
-
-		if (info->params.mode != MGSL_MODE_ASYNC) {
-			/* enable rx DMA and DMA interrupt */
-			wr_reg32(info, RDCSR, (BIT2 + BIT0));
-		} else {
-			/* enable saving of rx status, rx DMA and DMA interrupt */
-			wr_reg32(info, RDCSR, (BIT6 + BIT2 + BIT0));
-		}
-	}
-
-	slgt_irq_on(info, IRQ_RXOVER);
-
-	/* enable receiver */
-	wr_reg16(info, RCR, (unsigned short)(rd_reg16(info, RCR) | BIT1));
-
-	info->rx_restart = false;
-	info->rx_enabled = true;
-}
-
-static void tx_start(struct slgt_info *info)
-{
-	if (!info->tx_enabled) {
-		wr_reg16(info, TCR,
-			 (unsigned short)((rd_reg16(info, TCR) | BIT1) & ~BIT2));
-		info->tx_enabled = true;
-	}
-
-	if (desc_count(info->tbufs[info->tbuf_start])) {
-		info->drop_rts_on_tx_done = false;
-
-		if (info->params.mode != MGSL_MODE_ASYNC) {
-			if (info->params.flags & HDLC_FLAG_AUTO_RTS) {
-				get_gtsignals(info);
-				if (!(info->signals & SerialSignal_RTS)) {
-					info->signals |= SerialSignal_RTS;
-					set_gtsignals(info);
-					info->drop_rts_on_tx_done = true;
-				}
-			}
-
-			slgt_irq_off(info, IRQ_TXDATA);
-			slgt_irq_on(info, IRQ_TXUNDER + IRQ_TXIDLE);
-			/* clear tx idle and underrun status bits */
-			wr_reg16(info, SSR, (unsigned short)(IRQ_TXIDLE + IRQ_TXUNDER));
-		} else {
-			slgt_irq_off(info, IRQ_TXDATA);
-			slgt_irq_on(info, IRQ_TXIDLE);
-			/* clear tx idle status bit */
-			wr_reg16(info, SSR, IRQ_TXIDLE);
-		}
-		/* set 1st descriptor address and start DMA */
-		wr_reg32(info, TDDAR, info->tbufs[info->tbuf_start].pdesc);
-		wr_reg32(info, TDCSR, BIT2 + BIT0);
-		info->tx_active = true;
-	}
-}
-
-static void tx_stop(struct slgt_info *info)
-{
-	unsigned short val;
-
-	timer_delete(&info->tx_timer);
-
-	tdma_reset(info);
-
-	/* reset and disable transmitter */
-	val = rd_reg16(info, TCR) & ~BIT1;          /* clear enable bit */
-	wr_reg16(info, TCR, (unsigned short)(val | BIT2)); /* set reset bit */
-
-	slgt_irq_off(info, IRQ_TXDATA + IRQ_TXIDLE + IRQ_TXUNDER);
-
-	/* clear tx idle and underrun status bit */
-	wr_reg16(info, SSR, (unsigned short)(IRQ_TXIDLE + IRQ_TXUNDER));
-
-	reset_tbufs(info);
-
-	info->tx_enabled = false;
-	info->tx_active = false;
-}
-
-static void reset_port(struct slgt_info *info)
-{
-	if (!info->reg_addr)
-		return;
-
-	tx_stop(info);
-	rx_stop(info);
-
-	info->signals &= ~(SerialSignal_RTS | SerialSignal_DTR);
-	set_gtsignals(info);
-
-	slgt_irq_off(info, IRQ_ALL | IRQ_MASTER);
-}
-
-static void reset_adapter(struct slgt_info *info)
-{
-	int i;
-	for (i=0; i < info->port_count; ++i) {
-		if (info->port_array[i])
-			reset_port(info->port_array[i]);
-	}
-}
-
-static void async_mode(struct slgt_info *info)
-{
-  	unsigned short val;
-
-	slgt_irq_off(info, IRQ_ALL | IRQ_MASTER);
-	tx_stop(info);
-	rx_stop(info);
-
-	/* TCR (tx control)
-	 *
-	 * 15..13  mode, 010=async
-	 * 12..10  encoding, 000=NRZ
-	 * 09      parity enable
-	 * 08      1=odd parity, 0=even parity
-	 * 07      1=RTS driver control
-	 * 06      1=break enable
-	 * 05..04  character length
-	 *         00=5 bits
-	 *         01=6 bits
-	 *         10=7 bits
-	 *         11=8 bits
-	 * 03      0=1 stop bit, 1=2 stop bits
-	 * 02      reset
-	 * 01      enable
-	 * 00      auto-CTS enable
-	 */
-	val = 0x4000;
-
-	if (info->if_mode & MGSL_INTERFACE_RTS_EN)
-		val |= BIT7;
-
-	if (info->params.parity != ASYNC_PARITY_NONE) {
-		val |= BIT9;
-		if (info->params.parity == ASYNC_PARITY_ODD)
-			val |= BIT8;
-	}
-
-	switch (info->params.data_bits)
-	{
-	case 6: val |= BIT4; break;
-	case 7: val |= BIT5; break;
-	case 8: val |= BIT5 + BIT4; break;
-	}
-
-	if (info->params.stop_bits != 1)
-		val |= BIT3;
-
-	if (info->params.flags & HDLC_FLAG_AUTO_CTS)
-		val |= BIT0;
-
-	wr_reg16(info, TCR, val);
-
-	/* RCR (rx control)
-	 *
-	 * 15..13  mode, 010=async
-	 * 12..10  encoding, 000=NRZ
-	 * 09      parity enable
-	 * 08      1=odd parity, 0=even parity
-	 * 07..06  reserved, must be 0
-	 * 05..04  character length
-	 *         00=5 bits
-	 *         01=6 bits
-	 *         10=7 bits
-	 *         11=8 bits
-	 * 03      reserved, must be zero
-	 * 02      reset
-	 * 01      enable
-	 * 00      auto-DCD enable
-	 */
-	val = 0x4000;
-
-	if (info->params.parity != ASYNC_PARITY_NONE) {
-		val |= BIT9;
-		if (info->params.parity == ASYNC_PARITY_ODD)
-			val |= BIT8;
-	}
-
-	switch (info->params.data_bits)
-	{
-	case 6: val |= BIT4; break;
-	case 7: val |= BIT5; break;
-	case 8: val |= BIT5 + BIT4; break;
-	}
-
-	if (info->params.flags & HDLC_FLAG_AUTO_DCD)
-		val |= BIT0;
-
-	wr_reg16(info, RCR, val);
-
-	/* CCR (clock control)
-	 *
-	 * 07..05  011 = tx clock source is BRG/16
-	 * 04..02  010 = rx clock source is BRG
-	 * 01      0 = auxclk disabled
-	 * 00      1 = BRG enabled
-	 *
-	 * 0110 1001
-	 */
-	wr_reg8(info, CCR, 0x69);
-
-	msc_set_vcr(info);
-
-	/* SCR (serial control)
-	 *
-	 * 15  1=tx req on FIFO half empty
-	 * 14  1=rx req on FIFO half full
-	 * 13  tx data  IRQ enable
-	 * 12  tx idle  IRQ enable
-	 * 11  rx break on IRQ enable
-	 * 10  rx data  IRQ enable
-	 * 09  rx break off IRQ enable
-	 * 08  overrun  IRQ enable
-	 * 07  DSR      IRQ enable
-	 * 06  CTS      IRQ enable
-	 * 05  DCD      IRQ enable
-	 * 04  RI       IRQ enable
-	 * 03  0=16x sampling, 1=8x sampling
-	 * 02  1=txd->rxd internal loopback enable
-	 * 01  reserved, must be zero
-	 * 00  1=master IRQ enable
-	 */
-	val = BIT15 + BIT14 + BIT0;
-	/* JCR[8] : 1 = x8 async mode feature available */
-	if ((rd_reg32(info, JCR) & BIT8) && info->params.data_rate &&
-	    ((info->base_clock < (info->params.data_rate * 16)) ||
-	     (info->base_clock % (info->params.data_rate * 16)))) {
-		/* use 8x sampling */
-		val |= BIT3;
-		set_rate(info, info->params.data_rate * 8);
-	} else {
-		/* use 16x sampling */
-		set_rate(info, info->params.data_rate * 16);
-	}
-	wr_reg16(info, SCR, val);
-
-	slgt_irq_on(info, IRQ_RXBREAK | IRQ_RXOVER);
-
-	if (info->params.loopback)
-		enable_loopback(info);
-}
-
-static void sync_mode(struct slgt_info *info)
-{
-	unsigned short val;
-
-	slgt_irq_off(info, IRQ_ALL | IRQ_MASTER);
-	tx_stop(info);
-	rx_stop(info);
-
-	/* TCR (tx control)
-	 *
-	 * 15..13  mode
-	 *         000=HDLC/SDLC
-	 *         001=raw bit synchronous
-	 *         010=asynchronous/isochronous
-	 *         011=monosync byte synchronous
-	 *         100=bisync byte synchronous
-	 *         101=xsync byte synchronous
-	 * 12..10  encoding
-	 * 09      CRC enable
-	 * 08      CRC32
-	 * 07      1=RTS driver control
-	 * 06      preamble enable
-	 * 05..04  preamble length
-	 * 03      share open/close flag
-	 * 02      reset
-	 * 01      enable
-	 * 00      auto-CTS enable
-	 */
-	val = BIT2;
-
-	switch(info->params.mode) {
-	case MGSL_MODE_XSYNC:
-		val |= BIT15 + BIT13;
-		break;
-	case MGSL_MODE_MONOSYNC: val |= BIT14 + BIT13; break;
-	case MGSL_MODE_BISYNC:   val |= BIT15; break;
-	case MGSL_MODE_RAW:      val |= BIT13; break;
-	}
-	if (info->if_mode & MGSL_INTERFACE_RTS_EN)
-		val |= BIT7;
-
-	switch(info->params.encoding)
-	{
-	case HDLC_ENCODING_NRZB:          val |= BIT10; break;
-	case HDLC_ENCODING_NRZI_MARK:     val |= BIT11; break;
-	case HDLC_ENCODING_NRZI:          val |= BIT11 + BIT10; break;
-	case HDLC_ENCODING_BIPHASE_MARK:  val |= BIT12; break;
-	case HDLC_ENCODING_BIPHASE_SPACE: val |= BIT12 + BIT10; break;
-	case HDLC_ENCODING_BIPHASE_LEVEL: val |= BIT12 + BIT11; break;
-	case HDLC_ENCODING_DIFF_BIPHASE_LEVEL: val |= BIT12 + BIT11 + BIT10; break;
-	}
-
-	switch (info->params.crc_type & HDLC_CRC_MASK)
-	{
-	case HDLC_CRC_16_CCITT: val |= BIT9; break;
-	case HDLC_CRC_32_CCITT: val |= BIT9 + BIT8; break;
-	}
-
-	if (info->params.preamble != HDLC_PREAMBLE_PATTERN_NONE)
-		val |= BIT6;
-
-	switch (info->params.preamble_length)
-	{
-	case HDLC_PREAMBLE_LENGTH_16BITS: val |= BIT5; break;
-	case HDLC_PREAMBLE_LENGTH_32BITS: val |= BIT4; break;
-	case HDLC_PREAMBLE_LENGTH_64BITS: val |= BIT5 + BIT4; break;
-	}
-
-	if (info->params.flags & HDLC_FLAG_AUTO_CTS)
-		val |= BIT0;
-
-	wr_reg16(info, TCR, val);
-
-	/* TPR (transmit preamble) */
-
-	switch (info->params.preamble)
-	{
-	case HDLC_PREAMBLE_PATTERN_FLAGS: val = 0x7e; break;
-	case HDLC_PREAMBLE_PATTERN_ONES:  val = 0xff; break;
-	case HDLC_PREAMBLE_PATTERN_ZEROS: val = 0x00; break;
-	case HDLC_PREAMBLE_PATTERN_10:    val = 0x55; break;
-	case HDLC_PREAMBLE_PATTERN_01:    val = 0xaa; break;
-	default:                          val = 0x7e; break;
-	}
-	wr_reg8(info, TPR, (unsigned char)val);
-
-	/* RCR (rx control)
-	 *
-	 * 15..13  mode
-	 *         000=HDLC/SDLC
-	 *         001=raw bit synchronous
-	 *         010=asynchronous/isochronous
-	 *         011=monosync byte synchronous
-	 *         100=bisync byte synchronous
-	 *         101=xsync byte synchronous
-	 * 12..10  encoding
-	 * 09      CRC enable
-	 * 08      CRC32
-	 * 07..03  reserved, must be 0
-	 * 02      reset
-	 * 01      enable
-	 * 00      auto-DCD enable
-	 */
-	val = 0;
-
-	switch(info->params.mode) {
-	case MGSL_MODE_XSYNC:
-		val |= BIT15 + BIT13;
-		break;
-	case MGSL_MODE_MONOSYNC: val |= BIT14 + BIT13; break;
-	case MGSL_MODE_BISYNC:   val |= BIT15; break;
-	case MGSL_MODE_RAW:      val |= BIT13; break;
-	}
-
-	switch(info->params.encoding)
-	{
-	case HDLC_ENCODING_NRZB:          val |= BIT10; break;
-	case HDLC_ENCODING_NRZI_MARK:     val |= BIT11; break;
-	case HDLC_ENCODING_NRZI:          val |= BIT11 + BIT10; break;
-	case HDLC_ENCODING_BIPHASE_MARK:  val |= BIT12; break;
-	case HDLC_ENCODING_BIPHASE_SPACE: val |= BIT12 + BIT10; break;
-	case HDLC_ENCODING_BIPHASE_LEVEL: val |= BIT12 + BIT11; break;
-	case HDLC_ENCODING_DIFF_BIPHASE_LEVEL: val |= BIT12 + BIT11 + BIT10; break;
-	}
-
-	switch (info->params.crc_type & HDLC_CRC_MASK)
-	{
-	case HDLC_CRC_16_CCITT: val |= BIT9; break;
-	case HDLC_CRC_32_CCITT: val |= BIT9 + BIT8; break;
-	}
-
-	if (info->params.flags & HDLC_FLAG_AUTO_DCD)
-		val |= BIT0;
-
-	wr_reg16(info, RCR, val);
-
-	/* CCR (clock control)
-	 *
-	 * 07..05  tx clock source
-	 * 04..02  rx clock source
-	 * 01      auxclk enable
-	 * 00      BRG enable
-	 */
-	val = 0;
-
-	if (info->params.flags & HDLC_FLAG_TXC_BRG)
-	{
-		// when RxC source is DPLL, BRG generates 16X DPLL
-		// reference clock, so take TxC from BRG/16 to get
-		// transmit clock at actual data rate
-		if (info->params.flags & HDLC_FLAG_RXC_DPLL)
-			val |= BIT6 + BIT5;	/* 011, txclk = BRG/16 */
-		else
-			val |= BIT6;	/* 010, txclk = BRG */
-	}
-	else if (info->params.flags & HDLC_FLAG_TXC_DPLL)
-		val |= BIT7;	/* 100, txclk = DPLL Input */
-	else if (info->params.flags & HDLC_FLAG_TXC_RXCPIN)
-		val |= BIT5;	/* 001, txclk = RXC Input */
-
-	if (info->params.flags & HDLC_FLAG_RXC_BRG)
-		val |= BIT3;	/* 010, rxclk = BRG */
-	else if (info->params.flags & HDLC_FLAG_RXC_DPLL)
-		val |= BIT4;	/* 100, rxclk = DPLL */
-	else if (info->params.flags & HDLC_FLAG_RXC_TXCPIN)
-		val |= BIT2;	/* 001, rxclk = TXC Input */
-
-	if (info->params.clock_speed)
-		val |= BIT1 + BIT0;
-
-	wr_reg8(info, CCR, (unsigned char)val);
-
-	if (info->params.flags & (HDLC_FLAG_TXC_DPLL + HDLC_FLAG_RXC_DPLL))
-	{
-		// program DPLL mode
-		switch(info->params.encoding)
-		{
-		case HDLC_ENCODING_BIPHASE_MARK:
-		case HDLC_ENCODING_BIPHASE_SPACE:
-			val = BIT7; break;
-		case HDLC_ENCODING_BIPHASE_LEVEL:
-		case HDLC_ENCODING_DIFF_BIPHASE_LEVEL:
-			val = BIT7 + BIT6; break;
-		default: val = BIT6;	// NRZ encodings
-		}
-		wr_reg16(info, RCR, (unsigned short)(rd_reg16(info, RCR) | val));
-
-		// DPLL requires a 16X reference clock from BRG
-		set_rate(info, info->params.clock_speed * 16);
-	}
-	else
-		set_rate(info, info->params.clock_speed);
-
-	tx_set_idle(info);
-
-	msc_set_vcr(info);
-
-	/* SCR (serial control)
-	 *
-	 * 15  1=tx req on FIFO half empty
-	 * 14  1=rx req on FIFO half full
-	 * 13  tx data  IRQ enable
-	 * 12  tx idle  IRQ enable
-	 * 11  underrun IRQ enable
-	 * 10  rx data  IRQ enable
-	 * 09  rx idle  IRQ enable
-	 * 08  overrun  IRQ enable
-	 * 07  DSR      IRQ enable
-	 * 06  CTS      IRQ enable
-	 * 05  DCD      IRQ enable
-	 * 04  RI       IRQ enable
-	 * 03  reserved, must be zero
-	 * 02  1=txd->rxd internal loopback enable
-	 * 01  reserved, must be zero
-	 * 00  1=master IRQ enable
-	 */
-	wr_reg16(info, SCR, BIT15 + BIT14 + BIT0);
-
-	if (info->params.loopback)
-		enable_loopback(info);
-}
-
-/*
- *  set transmit idle mode
- */
-static void tx_set_idle(struct slgt_info *info)
-{
-	unsigned char val;
-	unsigned short tcr;
-
-	/* if preamble enabled (tcr[6] == 1) then tx idle size = 8 bits
-	 * else tcr[5:4] = tx idle size: 00 = 8 bits, 01 = 16 bits
-	 */
-	tcr = rd_reg16(info, TCR);
-	if (info->idle_mode & HDLC_TXIDLE_CUSTOM_16) {
-		/* disable preamble, set idle size to 16 bits */
-		tcr = (tcr & ~(BIT6 + BIT5)) | BIT4;
-		/* MSB of 16 bit idle specified in tx preamble register (TPR) */
-		wr_reg8(info, TPR, (unsigned char)((info->idle_mode >> 8) & 0xff));
-	} else if (!(tcr & BIT6)) {
-		/* preamble is disabled, set idle size to 8 bits */
-		tcr &= ~(BIT5 + BIT4);
-	}
-	wr_reg16(info, TCR, tcr);
-
-	if (info->idle_mode & (HDLC_TXIDLE_CUSTOM_8 | HDLC_TXIDLE_CUSTOM_16)) {
-		/* LSB of custom tx idle specified in tx idle register */
-		val = (unsigned char)(info->idle_mode & 0xff);
-	} else {
-		/* standard 8 bit idle patterns */
-		switch(info->idle_mode)
-		{
-		case HDLC_TXIDLE_FLAGS:          val = 0x7e; break;
-		case HDLC_TXIDLE_ALT_ZEROS_ONES:
-		case HDLC_TXIDLE_ALT_MARK_SPACE: val = 0xaa; break;
-		case HDLC_TXIDLE_ZEROS:
-		case HDLC_TXIDLE_SPACE:          val = 0x00; break;
-		default:                         val = 0xff;
-		}
-	}
-
-	wr_reg8(info, TIR, val);
-}
-
-/*
- * get state of V24 status (input) signals
- */
-static void get_gtsignals(struct slgt_info *info)
-{
-	unsigned short status = rd_reg16(info, SSR);
-
-	/* clear all serial signals except RTS and DTR */
-	info->signals &= SerialSignal_RTS | SerialSignal_DTR;
-
-	if (status & BIT3)
-		info->signals |= SerialSignal_DSR;
-	if (status & BIT2)
-		info->signals |= SerialSignal_CTS;
-	if (status & BIT1)
-		info->signals |= SerialSignal_DCD;
-	if (status & BIT0)
-		info->signals |= SerialSignal_RI;
-}
-
-/*
- * set V.24 Control Register based on current configuration
- */
-static void msc_set_vcr(struct slgt_info *info)
-{
-	unsigned char val = 0;
-
-	/* VCR (V.24 control)
-	 *
-	 * 07..04  serial IF select
-	 * 03      DTR
-	 * 02      RTS
-	 * 01      LL
-	 * 00      RL
-	 */
-
-	switch(info->if_mode & MGSL_INTERFACE_MASK)
-	{
-	case MGSL_INTERFACE_RS232:
-		val |= BIT5; /* 0010 */
-		break;
-	case MGSL_INTERFACE_V35:
-		val |= BIT7 + BIT6 + BIT5; /* 1110 */
-		break;
-	case MGSL_INTERFACE_RS422:
-		val |= BIT6; /* 0100 */
-		break;
-	}
-
-	if (info->if_mode & MGSL_INTERFACE_MSB_FIRST)
-		val |= BIT4;
-	if (info->signals & SerialSignal_DTR)
-		val |= BIT3;
-	if (info->signals & SerialSignal_RTS)
-		val |= BIT2;
-	if (info->if_mode & MGSL_INTERFACE_LL)
-		val |= BIT1;
-	if (info->if_mode & MGSL_INTERFACE_RL)
-		val |= BIT0;
-	wr_reg8(info, VCR, val);
-}
-
-/*
- * set state of V24 control (output) signals
- */
-static void set_gtsignals(struct slgt_info *info)
-{
-	unsigned char val = rd_reg8(info, VCR);
-	if (info->signals & SerialSignal_DTR)
-		val |= BIT3;
-	else
-		val &= ~BIT3;
-	if (info->signals & SerialSignal_RTS)
-		val |= BIT2;
-	else
-		val &= ~BIT2;
-	wr_reg8(info, VCR, val);
-}
-
-/*
- * free range of receive DMA buffers (i to last)
- */
-static void free_rbufs(struct slgt_info *info, unsigned int i, unsigned int last)
-{
-	int done = 0;
-
-	while(!done) {
-		/* reset current buffer for reuse */
-		info->rbufs[i].status = 0;
-		set_desc_count(info->rbufs[i], info->rbuf_fill_level);
-		if (i == last)
-			done = 1;
-		if (++i == info->rbuf_count)
-			i = 0;
-	}
-	info->rbuf_current = i;
-}
-
-/*
- * mark all receive DMA buffers as free
- */
-static void reset_rbufs(struct slgt_info *info)
-{
-	free_rbufs(info, 0, info->rbuf_count - 1);
-	info->rbuf_fill_index = 0;
-	info->rbuf_fill_count = 0;
-}
-
-/*
- * pass receive HDLC frame to upper layer
- *
- * return true if frame available, otherwise false
- */
-static bool rx_get_frame(struct slgt_info *info)
-{
-	unsigned int start, end;
-	unsigned short status;
-	unsigned int framesize = 0;
-	unsigned long flags;
-	struct tty_struct *tty = info->port.tty;
-	unsigned char addr_field = 0xff;
-	unsigned int crc_size = 0;
-
-	switch (info->params.crc_type & HDLC_CRC_MASK) {
-	case HDLC_CRC_16_CCITT: crc_size = 2; break;
-	case HDLC_CRC_32_CCITT: crc_size = 4; break;
-	}
-
-check_again:
-
-	framesize = 0;
-	addr_field = 0xff;
-	start = end = info->rbuf_current;
-
-	for (;;) {
-		if (!desc_complete(info->rbufs[end]))
-			goto cleanup;
-
-		if (framesize == 0 && info->params.addr_filter != 0xff)
-			addr_field = info->rbufs[end].buf[0];
-
-		framesize += desc_count(info->rbufs[end]);
-
-		if (desc_eof(info->rbufs[end]))
-			break;
-
-		if (++end == info->rbuf_count)
-			end = 0;
-
-		if (end == info->rbuf_current) {
-			if (info->rx_enabled){
-				spin_lock_irqsave(&info->lock,flags);
-				rx_start(info);
-				spin_unlock_irqrestore(&info->lock,flags);
-			}
-			goto cleanup;
-		}
-	}
-
-	/* status
-	 *
-	 * 15      buffer complete
-	 * 14..06  reserved
-	 * 05..04  residue
-	 * 02      eof (end of frame)
-	 * 01      CRC error
-	 * 00      abort
-	 */
-	status = desc_status(info->rbufs[end]);
-
-	/* ignore CRC bit if not using CRC (bit is undefined) */
-	if ((info->params.crc_type & HDLC_CRC_MASK) == HDLC_CRC_NONE)
-		status &= ~BIT1;
-
-	if (framesize == 0 ||
-		 (addr_field != 0xff && addr_field != info->params.addr_filter)) {
-		free_rbufs(info, start, end);
-		goto check_again;
-	}
-
-	if (framesize < (2 + crc_size) || status & BIT0) {
-		info->icount.rxshort++;
-		framesize = 0;
-	} else if (status & BIT1) {
-		info->icount.rxcrc++;
-		if (!(info->params.crc_type & HDLC_CRC_RETURN_EX))
-			framesize = 0;
-	}
-
-#if SYNCLINK_GENERIC_HDLC
-	if (framesize == 0) {
-		info->netdev->stats.rx_errors++;
-		info->netdev->stats.rx_frame_errors++;
-	}
-#endif
-
-	DBGBH(("%s rx frame status=%04X size=%d\n",
-		info->device_name, status, framesize));
-	DBGDATA(info, info->rbufs[start].buf, min_t(int, framesize, info->rbuf_fill_level), "rx");
-
-	if (framesize) {
-		if (!(info->params.crc_type & HDLC_CRC_RETURN_EX)) {
-			framesize -= crc_size;
-			crc_size = 0;
-		}
-
-		if (framesize > info->max_frame_size + crc_size)
-			info->icount.rxlong++;
-		else {
-			/* copy dma buffer(s) to contiguous temp buffer */
-			int copy_count = framesize;
-			int i = start;
-			unsigned char *p = info->tmp_rbuf;
-			info->tmp_rbuf_count = framesize;
-
-			info->icount.rxok++;
-
-			while(copy_count) {
-				int partial_count = min_t(int, copy_count, info->rbuf_fill_level);
-				memcpy(p, info->rbufs[i].buf, partial_count);
-				p += partial_count;
-				copy_count -= partial_count;
-				if (++i == info->rbuf_count)
-					i = 0;
-			}
-
-			if (info->params.crc_type & HDLC_CRC_RETURN_EX) {
-				*p = (status & BIT1) ? RX_CRC_ERROR : RX_OK;
-				framesize++;
-			}
-
-#if SYNCLINK_GENERIC_HDLC
-			if (info->netcount)
-				hdlcdev_rx(info,info->tmp_rbuf, framesize);
-			else
-#endif
-				ldisc_receive_buf(tty, info->tmp_rbuf, NULL,
-						  framesize);
-		}
-	}
-	free_rbufs(info, start, end);
-	return true;
-
-cleanup:
-	return false;
-}
-
-/*
- * pass receive buffer (RAW synchronous mode) to tty layer
- * return true if buffer available, otherwise false
- */
-static bool rx_get_buf(struct slgt_info *info)
-{
-	unsigned int i = info->rbuf_current;
-	unsigned int count;
-
-	if (!desc_complete(info->rbufs[i]))
-		return false;
-	count = desc_count(info->rbufs[i]);
-	switch(info->params.mode) {
-	case MGSL_MODE_MONOSYNC:
-	case MGSL_MODE_BISYNC:
-	case MGSL_MODE_XSYNC:
-		/* ignore residue in byte synchronous modes */
-		if (desc_residue(info->rbufs[i]))
-			count--;
-		break;
-	}
-	DBGDATA(info, info->rbufs[i].buf, count, "rx");
-	DBGINFO(("rx_get_buf size=%d\n", count));
-	if (count)
-		ldisc_receive_buf(info->port.tty, info->rbufs[i].buf, NULL,
-				  count);
-	free_rbufs(info, i, i);
-	return true;
-}
-
-static void reset_tbufs(struct slgt_info *info)
-{
-	unsigned int i;
-	info->tbuf_current = 0;
-	for (i=0 ; i < info->tbuf_count ; i++) {
-		info->tbufs[i].status = 0;
-		info->tbufs[i].count  = 0;
-	}
-}
-
-/*
- * return number of free transmit DMA buffers
- */
-static unsigned int free_tbuf_count(struct slgt_info *info)
-{
-	unsigned int count = 0;
-	unsigned int i = info->tbuf_current;
-
-	do
-	{
-		if (desc_count(info->tbufs[i]))
-			break; /* buffer in use */
-		++count;
-		if (++i == info->tbuf_count)
-			i=0;
-	} while (i != info->tbuf_current);
-
-	/* if tx DMA active, last zero count buffer is in use */
-	if (count && (rd_reg32(info, TDCSR) & BIT0))
-		--count;
-
-	return count;
-}
-
-/*
- * return number of bytes in unsent transmit DMA buffers
- * and the serial controller tx FIFO
- */
-static unsigned int tbuf_bytes(struct slgt_info *info)
-{
-	unsigned int total_count = 0;
-	unsigned int i = info->tbuf_current;
-	unsigned int reg_value;
-	unsigned int count;
-	unsigned int active_buf_count = 0;
-
-	/*
-	 * Add descriptor counts for all tx DMA buffers.
-	 * If count is zero (cleared by DMA controller after read),
-	 * the buffer is complete or is actively being read from.
-	 *
-	 * Record buf_count of last buffer with zero count starting
-	 * from current ring position. buf_count is mirror
-	 * copy of count and is not cleared by serial controller.
-	 * If DMA controller is active, that buffer is actively
-	 * being read so add to total.
-	 */
-	do {
-		count = desc_count(info->tbufs[i]);
-		if (count)
-			total_count += count;
-		else if (!total_count)
-			active_buf_count = info->tbufs[i].buf_count;
-		if (++i == info->tbuf_count)
-			i = 0;
-	} while (i != info->tbuf_current);
-
-	/* read tx DMA status register */
-	reg_value = rd_reg32(info, TDCSR);
-
-	/* if tx DMA active, last zero count buffer is in use */
-	if (reg_value & BIT0)
-		total_count += active_buf_count;
-
-	/* add tx FIFO count = reg_value[15..8] */
-	total_count += (reg_value >> 8) & 0xff;
-
-	/* if transmitter active add one byte for shift register */
-	if (info->tx_active)
-		total_count++;
-
-	return total_count;
-}
-
-/*
- * load data into transmit DMA buffer ring and start transmitter if needed
- * return true if data accepted, otherwise false (buffers full)
- */
-static bool tx_load(struct slgt_info *info, const u8 *buf, unsigned int size)
-{
-	unsigned short count;
-	unsigned int i;
-	struct slgt_desc *d;
-
-	/* check required buffer space */
-	if (DIV_ROUND_UP(size, DMABUFSIZE) > free_tbuf_count(info))
-		return false;
-
-	DBGDATA(info, buf, size, "tx");
-
-	/*
-	 * copy data to one or more DMA buffers in circular ring
-	 * tbuf_start   = first buffer for this data
-	 * tbuf_current = next free buffer
-	 *
-	 * Copy all data before making data visible to DMA controller by
-	 * setting descriptor count of the first buffer.
-	 * This prevents an active DMA controller from reading the first DMA
-	 * buffers of a frame and stopping before the final buffers are filled.
-	 */
-
-	info->tbuf_start = i = info->tbuf_current;
-
-	while (size) {
-		d = &info->tbufs[i];
-
-		count = (unsigned short)((size > DMABUFSIZE) ? DMABUFSIZE : size);
-		memcpy(d->buf, buf, count);
-
-		size -= count;
-		buf  += count;
-
-		/*
-		 * set EOF bit for last buffer of HDLC frame or
-		 * for every buffer in raw mode
-		 */
-		if ((!size && info->params.mode == MGSL_MODE_HDLC) ||
-		    info->params.mode == MGSL_MODE_RAW)
-			set_desc_eof(*d, 1);
-		else
-			set_desc_eof(*d, 0);
-
-		/* set descriptor count for all but first buffer */
-		if (i != info->tbuf_start)
-			set_desc_count(*d, count);
-		d->buf_count = count;
-
-		if (++i == info->tbuf_count)
-			i = 0;
-	}
-
-	info->tbuf_current = i;
-
-	/* set first buffer count to make new data visible to DMA controller */
-	d = &info->tbufs[info->tbuf_start];
-	set_desc_count(*d, d->buf_count);
-
-	/* start transmitter if needed and update transmit timeout */
-	if (!info->tx_active)
-		tx_start(info);
-	update_tx_timer(info);
-
-	return true;
-}
-
-static int register_test(struct slgt_info *info)
-{
-	static unsigned short patterns[] =
-		{0x0000, 0xffff, 0xaaaa, 0x5555, 0x6969, 0x9696};
-	static unsigned int count = ARRAY_SIZE(patterns);
-	unsigned int i;
-	int rc = 0;
-
-	for (i=0 ; i < count ; i++) {
-		wr_reg16(info, TIR, patterns[i]);
-		wr_reg16(info, BDR, patterns[(i+1)%count]);
-		if ((rd_reg16(info, TIR) != patterns[i]) ||
-		    (rd_reg16(info, BDR) != patterns[(i+1)%count])) {
-			rc = -ENODEV;
-			break;
-		}
-	}
-	info->gpio_present = (rd_reg32(info, JCR) & BIT5) ? 1 : 0;
-	info->init_error = rc ? 0 : DiagStatus_AddressFailure;
-	return rc;
-}
-
-static int irq_test(struct slgt_info *info)
-{
-	unsigned long timeout;
-	unsigned long flags;
-	struct tty_struct *oldtty = info->port.tty;
-	u32 speed = info->params.data_rate;
-
-	info->params.data_rate = 921600;
-	info->port.tty = NULL;
-
-	spin_lock_irqsave(&info->lock, flags);
-	async_mode(info);
-	slgt_irq_on(info, IRQ_TXIDLE);
-
-	/* enable transmitter */
-	wr_reg16(info, TCR,
-		(unsigned short)(rd_reg16(info, TCR) | BIT1));
-
-	/* write one byte and wait for tx idle */
-	wr_reg16(info, TDR, 0);
-
-	/* assume failure */
-	info->init_error = DiagStatus_IrqFailure;
-	info->irq_occurred = false;
-
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	timeout=100;
-	while(timeout-- && !info->irq_occurred)
-		msleep_interruptible(10);
-
-	spin_lock_irqsave(&info->lock,flags);
-	reset_port(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	info->params.data_rate = speed;
-	info->port.tty = oldtty;
-
-	info->init_error = info->irq_occurred ? 0 : DiagStatus_IrqFailure;
-	return info->irq_occurred ? 0 : -ENODEV;
-}
-
-static int loopback_test_rx(struct slgt_info *info)
-{
-	unsigned char *src, *dest;
-	int count;
-
-	if (desc_complete(info->rbufs[0])) {
-		count = desc_count(info->rbufs[0]);
-		src   = info->rbufs[0].buf;
-		dest  = info->tmp_rbuf;
-
-		for( ; count ; count-=2, src+=2) {
-			/* src=data byte (src+1)=status byte */
-			if (!(*(src+1) & (BIT9 + BIT8))) {
-				*dest = *src;
-				dest++;
-				info->tmp_rbuf_count++;
-			}
-		}
-		DBGDATA(info, info->tmp_rbuf, info->tmp_rbuf_count, "rx");
-		return 1;
-	}
-	return 0;
-}
-
-static int loopback_test(struct slgt_info *info)
-{
-#define TESTFRAMESIZE 20
-
-	unsigned long timeout;
-	u16 count;
-	unsigned char buf[TESTFRAMESIZE];
-	int rc = -ENODEV;
-	unsigned long flags;
-
-	struct tty_struct *oldtty = info->port.tty;
-	MGSL_PARAMS params;
-
-	memcpy(&params, &info->params, sizeof(params));
-
-	info->params.mode = MGSL_MODE_ASYNC;
-	info->params.data_rate = 921600;
-	info->params.loopback = 1;
-	info->port.tty = NULL;
-
-	/* build and send transmit frame */
-	for (count = 0; count < TESTFRAMESIZE; ++count)
-		buf[count] = (unsigned char)count;
-
-	info->tmp_rbuf_count = 0;
-	memset(info->tmp_rbuf, 0, TESTFRAMESIZE);
-
-	/* program hardware for HDLC and enabled receiver */
-	spin_lock_irqsave(&info->lock,flags);
-	async_mode(info);
-	rx_start(info);
-	tx_load(info, buf, count);
-	spin_unlock_irqrestore(&info->lock, flags);
-
-	/* wait for receive complete */
-	for (timeout = 100; timeout; --timeout) {
-		msleep_interruptible(10);
-		if (loopback_test_rx(info)) {
-			rc = 0;
-			break;
-		}
-	}
-
-	/* verify received frame length and contents */
-	if (!rc && (info->tmp_rbuf_count != count ||
-		  memcmp(buf, info->tmp_rbuf, count))) {
-		rc = -ENODEV;
-	}
-
-	spin_lock_irqsave(&info->lock,flags);
-	reset_adapter(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-
-	memcpy(&info->params, &params, sizeof(info->params));
-	info->port.tty = oldtty;
-
-	info->init_error = rc ? DiagStatus_DmaFailure : 0;
-	return rc;
-}
-
-static int adapter_test(struct slgt_info *info)
-{
-	DBGINFO(("testing %s\n", info->device_name));
-	if (register_test(info) < 0) {
-		printk("register test failure %s addr=%08X\n",
-			info->device_name, info->phys_reg_addr);
-	} else if (irq_test(info) < 0) {
-		printk("IRQ test failure %s IRQ=%d\n",
-			info->device_name, info->irq_level);
-	} else if (loopback_test(info) < 0) {
-		printk("loopback test failure %s\n", info->device_name);
-	}
-	return info->init_error;
-}
-
-/*
- * transmit timeout handler
- */
-static void tx_timeout(struct timer_list *t)
-{
-	struct slgt_info *info = timer_container_of(info, t, tx_timer);
-	unsigned long flags;
-
-	DBGINFO(("%s tx_timeout\n", info->device_name));
-	if(info->tx_active && info->params.mode == MGSL_MODE_HDLC) {
-		info->icount.txtimeout++;
-	}
-	spin_lock_irqsave(&info->lock,flags);
-	tx_stop(info);
-	spin_unlock_irqrestore(&info->lock,flags);
-
-#if SYNCLINK_GENERIC_HDLC
-	if (info->netcount)
-		hdlcdev_tx_done(info);
-	else
-#endif
-		bh_transmit(info);
-}
-
-/*
- * receive buffer polling timer
- */
-static void rx_timeout(struct timer_list *t)
-{
-	struct slgt_info *info = timer_container_of(info, t, rx_timer);
-	unsigned long flags;
-
-	DBGINFO(("%s rx_timeout\n", info->device_name));
-	spin_lock_irqsave(&info->lock, flags);
-	info->pending_bh |= BH_RECEIVE;
-	spin_unlock_irqrestore(&info->lock, flags);
-	bh_handler(&info->task);
-}
-
diff --git a/drivers/tty/vt/vt.c b/drivers/tty/vt/vt.c
index e99636ab9db50..3ca5e3dc5ac09 100644
--- a/drivers/tty/vt/vt.c
+++ b/drivers/tty/vt/vt.c
@@ -3978,9 +3978,6 @@ int __init vty_init(const struct file_operations *console_fops)
 		panic("Couldn't register console driver\n");
 	kbd_init();
 	console_map_init();
-#ifdef CONFIG_MDA_CONSOLE
-	mda_console_init();
-#endif
 	return 0;
 }
 
diff --git a/drivers/ufs/core/ufs-sysfs.c b/drivers/ufs/core/ufs-sysfs.c
index 99af3c73f1af7..d9dc4cc3452ee 100644
--- a/drivers/ufs/core/ufs-sysfs.c
+++ b/drivers/ufs/core/ufs-sysfs.c
@@ -594,8 +594,13 @@ static ssize_t device_lvl_exception_id_show(struct device *dev,
 	u64 exception_id;
 	int err;
 
+	if (hba->dev_info.wspecversion < 0x410)
+		return -EOPNOTSUPP;
+
 	ufshcd_rpm_get_sync(hba);
-	err = ufshcd_read_device_lvl_exception_id(hba, &exception_id);
+	err = ufshcd_query_attr_qword(hba, UPIU_QUERY_OPCODE_READ_ATTR,
+				      QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID,
+				      0, 0, &exception_id);
 	ufshcd_rpm_put_sync(hba);
 
 	if (err)
@@ -1670,6 +1675,12 @@ static inline bool ufshcd_is_wb_attrs(enum attr_idn idn)
 		idn <= QUERY_ATTR_IDN_CURR_WB_BUFF_SIZE;
 }
 
+static inline bool ufshcd_is_qword_attr(enum attr_idn idn)
+{
+	return idn == QUERY_ATTR_IDN_TIMESTAMP ||
+	       idn == QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID;
+}
+
 static int wb_read_resize_attrs(struct ufs_hba *hba,
 			enum attr_idn idn, u32 *attr_val)
 {
@@ -1736,6 +1747,7 @@ static ssize_t _name##_show(struct device *dev,				\
 	struct device_attribute *attr, char *buf)			\
 {									\
 	struct ufs_hba *hba = dev_get_drvdata(dev);			\
+	u64 qword_value;						\
 	u32 value;							\
 	int ret;							\
 	u8 index = 0;							\
@@ -1748,14 +1760,24 @@ static ssize_t _name##_show(struct device *dev,				\
 	if (ufshcd_is_wb_attrs(QUERY_ATTR_IDN##_uname))			\
 		index = ufshcd_wb_get_query_index(hba);			\
 	ufshcd_rpm_get_sync(hba);					\
-	ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR,	\
-		QUERY_ATTR_IDN##_uname, index, 0, &value);		\
+	if (ufshcd_is_qword_attr(QUERY_ATTR_IDN##_uname))		\
+		ret = ufshcd_query_attr_qword(hba,			\
+			UPIU_QUERY_OPCODE_READ_ATTR,			\
+			QUERY_ATTR_IDN##_uname,				\
+			index, 0, &qword_value);			\
+	else								\
+		ret = ufshcd_query_attr(hba,				\
+			UPIU_QUERY_OPCODE_READ_ATTR,			\
+			QUERY_ATTR_IDN##_uname, index, 0, &value);	\
 	ufshcd_rpm_put_sync(hba);					\
 	if (ret) {							\
 		ret = -EINVAL;						\
 		goto out;						\
 	}								\
-	ret = sysfs_emit(buf, "0x%08X\n", value);			\
+	if (ufshcd_is_qword_attr(QUERY_ATTR_IDN##_uname))		\
+		ret = sysfs_emit(buf, "0x%016llX\n", qword_value);	\
+	else								\
+		ret = sysfs_emit(buf, "0x%08X\n", value);		\
 out:									\
 	up(&hba->host_sem);						\
 	return ret;							\
diff --git a/drivers/ufs/core/ufs-txeq.c b/drivers/ufs/core/ufs-txeq.c
index b2dc891243538..4b264adfdf49a 100644
--- a/drivers/ufs/core/ufs-txeq.c
+++ b/drivers/ufs/core/ufs-txeq.c
@@ -14,6 +14,9 @@
 #include <ufs/unipro.h>
 #include "ufshcd-priv.h"
 
+#define TX_EQ_SETTING_MASK		0x7
+#define TX_EQ_SETTINGS_VALID_BIT	BIT(15)
+
 static bool use_adaptive_txeq;
 module_param(use_adaptive_txeq, bool, 0644);
 MODULE_PARM_DESC(use_adaptive_txeq, "Find and apply optimal TX Equalization settings before changing Power Mode (default: false)");
@@ -40,6 +43,28 @@ static bool txeq_presets_selected[UFS_TX_EQ_PRESET_MAX] = {[0 ... (UFS_TX_EQ_PRE
 module_param_array(txeq_presets_selected, bool, NULL, 0644);
 MODULE_PARM_DESC(txeq_presets_selected, "Use only the selected Presets out of the 8 TX Equalization Presets for TX EQTR");
 
+static int txeq_setting_sel_set(const char *val, const struct kernel_param *kp)
+{
+	return param_set_uint_minmax(val, kp, 0, 1);
+}
+
+static const struct kernel_param_ops txeq_setting_sel_ops = {
+	.set = txeq_setting_sel_set,
+	.get = param_get_uint,
+};
+
+static unsigned int txeq_setting_sel;
+module_param_cb(txeq_setting_sel, &txeq_setting_sel_ops, &txeq_setting_sel, 0644);
+MODULE_PARM_DESC(txeq_setting_sel, "The qTxEQGnSettings and wTxEQGnSettingsExt Attributes selector used to retrieve and store TX Equalization settings");
+
+static bool retrieve_txeq_setting = true;
+module_param(retrieve_txeq_setting, bool, 0644);
+MODULE_PARM_DESC(retrieve_txeq_setting, "Retrieve TX Equalization settings from qTxEQGnSettings and wTxEQGnSettingsExt Attributes (default: true)");
+
+static bool store_txeq_setting = true;
+module_param(store_txeq_setting, bool, 0644);
+MODULE_PARM_DESC(store_txeq_setting, "Store the optimal TX Equalization settings to qTxEQGnSettings and wTxEQGnSettingsExt Attributes (default: true)");
+
 /*
  * ufs_tx_eq_preset - Table of minimum required list of presets.
  *
@@ -117,6 +142,126 @@ static const u32 pa_tx_eq_setting[UFS_HS_GEAR_MAX] = {
 	PA_TXEQG6SETTING
 };
 
+/*
+ * Decode Device TX Equalization PreShoot value based on qTxEQGnSettings bit assignment:
+ * bit[3:0]: Device TX Logical LANE 0 PreShoot
+ * bit[7:4]: Device TX Logical LANE 1 PreShoot
+ */
+static inline u8 tx_eq_device_preshoot_decode(u64 eq, u8 lane)
+{
+	return (u8)((eq >> (lane * TX_HS_PRESHOOT_SHIFT)) & TX_EQ_SETTING_MASK);
+}
+
+/*
+ * Decode Device TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment:
+ * bit[19:16]: Device TX Logical LANE 0 DeEmphasis
+ * bit[23:20]: Device TX Logical LANE 1 DeEmphasis
+ */
+static inline u8 tx_eq_device_deemphasis_decode(u64 eq, u8 lane)
+{
+	return (u8)((eq >> (lane * TX_HS_DEEMPHASIS_SHIFT + 16)) & TX_EQ_SETTING_MASK);
+}
+
+/*
+ * Decode Host TX Equalization PreShoot value based on qTxEQGnSettings bit assignment:
+ * bit[35:32]: Host TX Logical LANE 0 PreShoot
+ * bit[39:36]: Host TX Logical LANE 1 PreShoot
+ */
+static inline u8 tx_eq_host_preshoot_decode(u64 eq, u8 lane)
+{
+	return (u8)((eq >> (lane * TX_HS_PRESHOOT_SHIFT + 32)) & TX_EQ_SETTING_MASK);
+}
+
+/*
+ * Decode Host TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment:
+ * bit[51:48]: Host TX Logical LANE 0 DeEmphasis
+ * bit[55:52]: Host TX Logical LANE 1 DeEmphasis
+ */
+static inline u8 tx_eq_host_deemphasis_decode(u64 eq, u8 lane)
+{
+	return (u8)((eq >> (lane * TX_HS_DEEMPHASIS_SHIFT + 48)) & TX_EQ_SETTING_MASK);
+}
+
+/*
+ * Decode Device TX precode_en indication based on wTxEQGnSettingsExt bit assignment:
+ * bit[0]: PreCodeEn for Device TX Logical LANE 0
+ * bit[1]: PreCodeEn for Device TX Logical LANE 1
+ */
+static inline bool tx_eq_device_precode_en_decode(u16 eq_ext, u8 lane)
+{
+	return eq_ext & BIT(lane);
+}
+
+/*
+ * Decode Host TX precode_en indication based on wTxEQGnSettingsExt bit assignment:
+ * bit[4]: PreCodeEn for Device RX Logical LANE 0
+ * bit[5]: PreCodeEn for Device RX Logical LANE 1
+ */
+static inline bool tx_eq_host_precode_en_decode(u16 eq_ext, u8 lane)
+{
+	return eq_ext & BIT(lane + 4);
+}
+
+/*
+ * Encode Device TX Equalization PreShoot value based on qTxEQGnSettings bit assignment:
+ * bit[3:0]: Device TX Logical LANE 0 PreShoot
+ * bit[7:4]: Device TX Logical LANE 1 PreShoot
+ */
+static inline u64 tx_eq_device_preshoot_encode(u64 val, u8 lane)
+{
+	return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_PRESHOOT_SHIFT);
+}
+
+/*
+ * Encode Device TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment:
+ * bit[19:16]: Device TX Logical LANE 0 DeEmphasis
+ * bit[23:20]: Device TX Logical LANE 1 DeEmphasis
+ */
+static inline u64 tx_eq_device_deemphasis_encode(u64 val, u8 lane)
+{
+	return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_DEEMPHASIS_SHIFT + 16);
+}
+
+/*
+ * Encode Host TX Equalization PreShoot value based on qTxEQGnSettings bit assignment:
+ * bit[35:32]: Host TX Logical LANE 0 PreShoot
+ * bit[39:36]: Host TX Logical LANE 1 PreShoot
+ */
+static inline u64 tx_eq_host_preshoot_encode(u64 val, u8 lane)
+{
+	return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_PRESHOOT_SHIFT + 32);
+}
+
+/*
+ * Encode Host TX Equalization DeEmphasis value based on qTxEQGnSettings bit assignment:
+ * bit[51:48]: Host TX Logical LANE 0 DeEmphasis
+ * bit[55:52]: Host TX Logical LANE 1 DeEmphasis
+ */
+static inline u64 tx_eq_host_deemphasis_encode(u64 val, u8 lane)
+{
+	return (val & TX_EQ_SETTING_MASK) << (lane * TX_HS_DEEMPHASIS_SHIFT + 48);
+}
+
+/*
+ * Encode Device precode_en based on wTxEQGnSettingsExt bit assignment:
+ * bit[0]: PreCodeEn for Device TX Logical LANE 0
+ * bit[1]: PreCodeEn for Device TX Logical LANE 1
+ */
+static inline u16 tx_eq_device_precode_en_encode(bool en, u8 lane)
+{
+	return (u16)en << lane;
+}
+
+/*
+ * Encode Host precode_en based on wTxEQGnSettingsExt bit assignment:
+ * bit[4]: PreCodeEn for Device RX Logical LANE 0
+ * bit[5]: PreCodeEn for Device RX Logical LANE 1
+ */
+static inline u16 tx_eq_host_precode_en_encode(bool en, u8 lane)
+{
+	return (u16)en << (lane + 4);
+}
+
 /**
  * ufshcd_configure_precoding - Configure Pre-Coding for all active lanes
  * @hba: per adapter instance
@@ -1164,6 +1309,7 @@ int ufshcd_config_tx_eq_settings(struct ufs_hba *hba,
 
 		/* Mark TX Equalization settings as valid */
 		params->is_valid = true;
+		params->is_trained = true;
 		params->is_applied = false;
 	}
 
@@ -1291,3 +1437,144 @@ int ufshcd_retrain_tx_eq(struct ufs_hba *hba, u32 gear)
 
 	return ret;
 }
+
+/**
+ * ufshcd_extract_tx_eq_settings_attrs - Extract TX Equalization settings from UFS attributes
+ * @hba: per adapter instance
+ * @gear: target gear
+ *
+ * This function extracts previously stored TX Equalization settings from UFS
+ * attributes qTxEQGnSettings and wTxEQGnSettingsExt. These attributes contain
+ * the optimal TX Equalization parameters (PreShoot, DeEmphasis, and PreCoding
+ * enable) that were determined during a previous EQTR procedure.
+ *
+ * The function reads:
+ * 1. qTxEQGnSettings (64-bit): Main attribute containing PreShoot and
+ *    DeEmphasis values for both host and device TX lanes
+ * 2. wTxEQGnSettingsExt (16-bit): Extended attribute containing PreCoding
+ *    enable flags and validity indicator
+ */
+static void ufshcd_extract_tx_eq_settings_attrs(struct ufs_hba *hba, u8 gear)
+{
+	struct ufshcd_tx_eq_params *params;
+	u32 lane, eq_ext;
+	int ret;
+	u64 eq;
+
+	ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_READ_ATTR,
+				QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS_EXT, gear - 1,
+				(u8)txeq_setting_sel, &eq_ext);
+	if (ret)
+		return;
+
+	dev_dbg(hba->dev, "%s: HS-G%u wTxEQGnSettingsExt (Selector %u) = 0x%08x\n",
+		__func__, gear, txeq_setting_sel, eq_ext);
+
+	if (!(eq_ext & TX_EQ_SETTINGS_VALID_BIT))
+		return;
+
+	ret = ufshcd_query_attr_qword(hba, UPIU_QUERY_OPCODE_READ_ATTR,
+				      QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS,
+				      gear - 1, (u8)txeq_setting_sel, &eq);
+	if (ret)
+		return;
+
+	dev_dbg(hba->dev, "%s: HS-G%u qTxEQGnSettings (Selector %u) = 0x%016llx\n",
+		__func__, gear, txeq_setting_sel, eq);
+
+	params = &hba->tx_eq_params[gear - 1];
+
+	for (lane = 0; lane < UFS_MAX_LANES; lane++) {
+		params->host[lane].preshoot = tx_eq_host_preshoot_decode(eq, lane);
+		params->host[lane].deemphasis = tx_eq_host_deemphasis_decode(eq, lane);
+		params->host[lane].precode_en = tx_eq_host_precode_en_decode(eq_ext, lane);
+
+		params->device[lane].preshoot = tx_eq_device_preshoot_decode(eq, lane);
+		params->device[lane].deemphasis = tx_eq_device_deemphasis_decode(eq, lane);
+		params->device[lane].precode_en = tx_eq_device_precode_en_decode(eq_ext, lane);
+	}
+
+	params->is_valid = true;
+}
+
+void ufshcd_retrieve_tx_eq_settings(struct ufs_hba *hba)
+{
+	u8 gear = (u8)adaptive_txeq_gear;
+
+	if (!hba->max_pwr_info.is_valid || !ufshcd_is_tx_eq_supported(hba) ||
+	    !use_adaptive_txeq || !retrieve_txeq_setting)
+		return;
+
+	for (; gear <= UFS_HS_GEAR_MAX; gear++)
+		ufshcd_extract_tx_eq_settings_attrs(hba, gear);
+}
+
+/**
+ * ufshcd_update_tx_eq_settings_attrs - Update TX EQ settings in UFS attributes
+ * @hba: per adapter instance
+ * @gear: target gear
+ *
+ * This function stores the optimal TX Equalization settings obtained from
+ * TX EQTR procedure into UFS device attributes for future fast-path retrieval.
+ * The settings are stored in two complementary attributes:
+ *
+ * 1. qTxEQGnSettings (64-bit): Main attribute containing PreShoot and
+ *    DeEmphasis values for both host and device TX lanes
+ * 2. wTxEQGnSettingsExt (16-bit): Extended attribute containing PreCoding
+ *    enable flags and validity indicator
+ */
+static void ufshcd_update_tx_eq_settings_attrs(struct ufs_hba *hba, u8 gear)
+{
+	struct ufshcd_tx_eq_params *params;
+	u32 lane, eq_ext = 0;
+	u64 eq = 0;
+	int ret;
+
+	params = &hba->tx_eq_params[gear - 1];
+	if (!params->is_valid || !params->is_trained)
+		return;
+
+	for (lane = 0; lane < UFS_MAX_LANES; lane++) {
+		eq |= tx_eq_host_preshoot_encode((u64)params->host[lane].preshoot, lane);
+		eq |= tx_eq_host_deemphasis_encode((u64)params->host[lane].deemphasis, lane);
+		eq_ext |= tx_eq_host_precode_en_encode(params->host[lane].precode_en, lane);
+
+		eq |= tx_eq_device_preshoot_encode((u64)params->device[lane].preshoot, lane);
+		eq |= tx_eq_device_deemphasis_encode((u64)params->device[lane].deemphasis, lane);
+		eq_ext |= tx_eq_device_precode_en_encode(params->device[lane].precode_en, lane);
+	}
+
+	/* Set validity flag to indicate valid settings are stored */
+	eq_ext |= TX_EQ_SETTINGS_VALID_BIT;
+
+	/* Write qTxEQGnSettings */
+	ret = ufshcd_query_attr_qword(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
+				      QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS,
+				      gear - 1, (u8)txeq_setting_sel, &eq);
+	if (ret)
+		return;
+
+	/* Write wTxEQGnSettingsExt */
+	ret = ufshcd_query_attr(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
+				QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS_EXT, gear - 1,
+				(u8)txeq_setting_sel, &eq_ext);
+	if (ret)
+		return;
+
+	dev_dbg(hba->dev, "%s: Saved HS-G%u qTxEQGnSettings (Selector %u) = 0x%016llx\n",
+		__func__, gear, txeq_setting_sel, eq);
+	dev_dbg(hba->dev, "%s: Saved HS-G%u wTxEQGnSettingsExt (Selector %u) = 0x%08x\n",
+		__func__, gear, txeq_setting_sel, eq_ext);
+}
+
+void ufshcd_store_tx_eq_settings(struct ufs_hba *hba)
+{
+	u8 gear = (u8)adaptive_txeq_gear;
+
+	if (!hba->max_pwr_info.is_valid || !ufshcd_is_tx_eq_supported(hba) ||
+	    !use_adaptive_txeq || !store_txeq_setting)
+		return;
+
+	for (; gear <= UFS_HS_GEAR_MAX; gear++)
+		ufshcd_update_tx_eq_settings_attrs(hba, gear);
+}
diff --git a/drivers/ufs/core/ufshcd-priv.h b/drivers/ufs/core/ufshcd-priv.h
index 0a72148cb053e..70f90d97f2177 100644
--- a/drivers/ufs/core/ufshcd-priv.h
+++ b/drivers/ufs/core/ufshcd-priv.h
@@ -60,6 +60,8 @@ int ufshcd_query_attr_retry(struct ufs_hba *hba, enum query_opcode opcode,
 			    u32 *attr_val);
 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
 		      enum attr_idn idn, u8 index, u8 selector, u32 *attr_val);
+int ufshcd_query_attr_qword(struct ufs_hba *hba, enum query_opcode opcode,
+			    enum attr_idn idn, u8 index, u8 sel, u64 *attr_val);
 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
 	enum flag_idn idn, u8 index, bool *flag_res);
 void ufshcd_auto_hibern8_update(struct ufs_hba *hba, u32 ahit);
@@ -106,7 +108,6 @@ int ufshcd_exec_raw_upiu_cmd(struct ufs_hba *hba,
 			     enum query_opcode desc_op);
 
 int ufshcd_wb_toggle(struct ufs_hba *hba, bool enable);
-int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id);
 
 int ufshcd_uic_tx_eqtr(struct ufs_hba *hba, int gear);
 void ufshcd_apply_valid_tx_eq_settings(struct ufs_hba *hba);
@@ -117,6 +118,8 @@ void ufshcd_print_tx_eq_params(struct ufs_hba *hba);
 bool ufshcd_is_txeq_presets_used(struct ufs_hba *hba);
 bool ufshcd_is_txeq_preset_selected(u8 preshoot, u8 deemphasis);
 int ufshcd_retrain_tx_eq(struct ufs_hba *hba, u32 gear);
+void ufshcd_retrieve_tx_eq_settings(struct ufs_hba *hba);
+void ufshcd_store_tx_eq_settings(struct ufs_hba *hba);
 
 /* Wrapper functions for safely calling variant operations */
 static inline const char *ufshcd_get_var_name(struct ufs_hba *hba)
diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c
index c3f08957d179a..06b965080fb0e 100644
--- a/drivers/ufs/core/ufshcd.c
+++ b/drivers/ufs/core/ufshcd.c
@@ -3611,6 +3611,67 @@ int ufshcd_query_attr_retry(struct ufs_hba *hba,
 	return ret;
 }
 
+/**
+ * ufshcd_query_attr_qword - Function of sending query requests for quad-word attributes
+ * @hba: per-adapter instance
+ * @opcode: attribute opcode
+ * @idn: attribute idn to access
+ * @index: index field
+ * @sel: selector field
+ * @attr_val: the attribute value after the query request completes
+ *
+ * Return: 0 for success, non-zero in case of failure.
+ */
+int ufshcd_query_attr_qword(struct ufs_hba *hba, enum query_opcode opcode,
+			    enum attr_idn idn, u8 index, u8 sel, u64 *attr_val)
+{
+	struct utp_upiu_query_v4_0 *upiu_req;
+	struct utp_upiu_query_v4_0 *upiu_resp;
+	struct ufs_query_req *request = NULL;
+	struct ufs_query_res *response = NULL;
+	int err;
+
+	if (!attr_val) {
+		dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
+			__func__, opcode);
+		return -EINVAL;
+	}
+
+	ufshcd_dev_man_lock(hba);
+
+	ufshcd_init_query(hba, &request, &response, opcode, idn, index, sel);
+
+	switch (opcode) {
+	case UPIU_QUERY_OPCODE_WRITE_ATTR:
+		request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
+		upiu_req = (struct utp_upiu_query_v4_0 *)&request->upiu_req;
+		put_unaligned_be64(*attr_val, &upiu_req->osf3);
+		break;
+	case UPIU_QUERY_OPCODE_READ_ATTR:
+		request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
+		break;
+	default:
+		dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
+			__func__, opcode);
+		err = -EINVAL;
+		goto out_unlock;
+	}
+
+	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout);
+	if (err) {
+		dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, selector %d, err = %d\n",
+			__func__, opcode, idn, index, sel, err);
+		goto out_unlock;
+	}
+
+	upiu_resp = (struct utp_upiu_query_v4_0 *)response;
+	*attr_val = get_unaligned_be64(&upiu_resp->osf3);
+
+out_unlock:
+	ufshcd_dev_man_unlock(hba);
+	return err;
+}
+
 /*
  * Return: 0 upon success; > 0 in case the UFS device reported an OCS error;
  * < 0 if another error occurred.
@@ -5157,6 +5218,35 @@ void ufshcd_update_evt_hist(struct ufs_hba *hba, u32 id, u32 val)
 }
 EXPORT_SYMBOL_GPL(ufshcd_update_evt_hist);
 
+static int ufshcd_validate_link_params(struct ufs_hba *hba)
+{
+	int ret, val;
+
+	ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
+			     &val);
+	if (ret)
+		return ret;
+
+	if (val != hba->lanes_per_direction) {
+		dev_err(hba->dev, "Tx lane mismatch [config,reported] [%d,%d]\n",
+			hba->lanes_per_direction, val);
+		return -ENOLINK;
+	}
+
+	ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
+			     &val);
+	if (ret)
+		return ret;
+
+	if (val != hba->lanes_per_direction) {
+		dev_err(hba->dev, "Rx lane mismatch [config,reported] [%d,%d]\n",
+			hba->lanes_per_direction, val);
+		return -ENOLINK;
+	}
+
+	return 0;
+}
+
 /**
  * ufshcd_link_startup - Initialize unipro link startup
  * @hba: per adapter instance
@@ -5230,6 +5320,10 @@ static int ufshcd_link_startup(struct ufs_hba *hba)
 			goto out;
 	}
 
+	ret = ufshcd_validate_link_params(hba);
+	if (ret)
+		goto out;
+
 	/* Include any host controller configuration via UIC commands */
 	ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
 	if (ret)
@@ -6224,46 +6318,6 @@ static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
 				__func__, err);
 }
 
-/*
- * Return: 0 upon success; > 0 in case the UFS device reported an OCS error;
- * < 0 if another error occurred.
- */
-int ufshcd_read_device_lvl_exception_id(struct ufs_hba *hba, u64 *exception_id)
-{
-	struct utp_upiu_query_v4_0 *upiu_resp;
-	struct ufs_query_req *request = NULL;
-	struct ufs_query_res *response = NULL;
-	int err;
-
-	if (hba->dev_info.wspecversion < 0x410)
-		return -EOPNOTSUPP;
-
-	ufshcd_hold(hba);
-	mutex_lock(&hba->dev_cmd.lock);
-
-	ufshcd_init_query(hba, &request, &response,
-			  UPIU_QUERY_OPCODE_READ_ATTR,
-			  QUERY_ATTR_IDN_DEV_LVL_EXCEPTION_ID, 0, 0);
-
-	request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
-
-	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout);
-
-	if (err) {
-		dev_err(hba->dev, "%s: failed to read device level exception %d\n",
-			__func__, err);
-		goto out;
-	}
-
-	upiu_resp = (struct utp_upiu_query_v4_0 *)response;
-	*exception_id = get_unaligned_be64(&upiu_resp->osf3);
-out:
-	mutex_unlock(&hba->dev_cmd.lock);
-	ufshcd_release(hba);
-
-	return err;
-}
-
 static int __ufshcd_wb_toggle(struct ufs_hba *hba, bool set, enum flag_idn idn)
 {
 	u8 index;
@@ -9107,41 +9161,28 @@ static int ufshcd_device_params_init(struct ufs_hba *hba)
 		dev_err(hba->dev,
 			"%s: Failed getting max supported power mode\n",
 			__func__);
+
+	ufshcd_retrieve_tx_eq_settings(hba);
 out:
 	return ret;
 }
 
 static void ufshcd_set_timestamp_attr(struct ufs_hba *hba)
 {
-	int err;
-	struct ufs_query_req *request = NULL;
-	struct ufs_query_res *response = NULL;
 	struct ufs_dev_info *dev_info = &hba->dev_info;
-	struct utp_upiu_query_v4_0 *upiu_data;
+	u64 ts_ns;
+	int err;
 
 	if (dev_info->wspecversion < 0x400 ||
 	    hba->dev_quirks & UFS_DEVICE_QUIRK_NO_TIMESTAMP_SUPPORT)
 		return;
 
-	ufshcd_dev_man_lock(hba);
-
-	ufshcd_init_query(hba, &request, &response,
-			  UPIU_QUERY_OPCODE_WRITE_ATTR,
-			  QUERY_ATTR_IDN_TIMESTAMP, 0, 0);
-
-	request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
-
-	upiu_data = (struct utp_upiu_query_v4_0 *)&request->upiu_req;
-
-	put_unaligned_be64(ktime_get_real_ns(), &upiu_data->osf3);
-
-	err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, dev_cmd_timeout);
-
+	ts_ns = ktime_get_real_ns();
+	err = ufshcd_query_attr_qword(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
+				      QUERY_ATTR_IDN_TIMESTAMP, 0, 0, &ts_ns);
 	if (err)
 		dev_err(hba->dev, "%s: failed to set timestamp %d\n",
 			__func__, err);
-
-	ufshcd_dev_man_unlock(hba);
 }
 
 /**
@@ -10768,6 +10809,9 @@ static void ufshcd_wl_shutdown(struct scsi_device *sdev)
 
 	/* Turn on everything while shutting down */
 	ufshcd_rpm_get_sync(hba);
+
+	ufshcd_store_tx_eq_settings(hba);
+
 	scsi_device_quiesce(sdev);
 	shost_for_each_device(sdev, hba->host) {
 		if (sdev == hba->ufs_device_wlun)
diff --git a/drivers/ufs/host/Kconfig b/drivers/ufs/host/Kconfig
index 964ae70e7390d..ff170c0b6da0c 100644
--- a/drivers/ufs/host/Kconfig
+++ b/drivers/ufs/host/Kconfig
@@ -55,7 +55,7 @@ config SCSI_UFS_DWC_TC_PLATFORM
 	  If unsure, say N.
 
 config SCSI_UFS_QCOM
-	tristate "QCOM specific hooks to UFS controller platform driver"
+	tristate "Qualcomm specific hooks to UFS controller platform driver"
 	depends on SCSI_UFSHCD_PLATFORM && ARCH_QCOM
 	depends on GENERIC_MSI_IRQ
 	depends on RESET_CONTROLLER
diff --git a/drivers/ufs/host/tc-dwc-g210-pci.c b/drivers/ufs/host/tc-dwc-g210-pci.c
index 0167d8bef71a1..c6d89f9c44ae0 100644
--- a/drivers/ufs/host/tc-dwc-g210-pci.c
+++ b/drivers/ufs/host/tc-dwc-g210-pci.c
@@ -114,8 +114,8 @@ static const struct dev_pm_ops tc_dwc_g210_pci_pm_ops = {
 };
 
 static const struct pci_device_id tc_dwc_g210_pci_tbl[] = {
-	{ PCI_VENDOR_ID_SYNOPSYS, 0xB101, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_SYNOPSYS, 0xB102, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
+	{ PCI_VDEVICE(SYNOPSYS, 0xB101) },
+	{ PCI_VDEVICE(SYNOPSYS, 0xB102) },
 	{ }	/* terminate list */
 };
 
diff --git a/drivers/ufs/host/ufs-exynos.c b/drivers/ufs/host/ufs-exynos.c
index 77a6c8e44485b..b2f65c4655251 100644
--- a/drivers/ufs/host/ufs-exynos.c
+++ b/drivers/ufs/host/ufs-exynos.c
@@ -97,6 +97,10 @@
 #define UFS_EXYNOSAUTO_RD_SHARABLE	BIT(1)
 #define UFS_EXYNOSAUTO_SHARABLE		(UFS_EXYNOSAUTO_WR_SHARABLE | \
 					 UFS_EXYNOSAUTO_RD_SHARABLE)
+#define UFS_EXYNOSAUTOV920_WR_SHARABLE	BIT(3)
+#define UFS_EXYNOSAUTOV920_RD_SHARABLE	BIT(2)
+#define UFS_EXYNOSAUTOV920_SHARABLE	(UFS_EXYNOSAUTOV920_WR_SHARABLE |\
+					 UFS_EXYNOSAUTOV920_RD_SHARABLE)
 #define UFS_GS101_WR_SHARABLE		BIT(1)
 #define UFS_GS101_RD_SHARABLE		BIT(0)
 #define UFS_GS101_SHARABLE		(UFS_GS101_WR_SHARABLE | \
@@ -417,6 +421,95 @@ static int exynos7_ufs_post_pwr_change(struct exynos_ufs *ufs,
 	return 0;
 }
 
+static int exynosautov920_ufs_pre_link(struct exynos_ufs *ufs)
+{
+	struct ufs_hba *hba = ufs->hba;
+	int i;
+	u32 tx_line_reset_period, rx_line_reset_period;
+
+	rx_line_reset_period = (RX_LINE_RESET_TIME * ufs->mclk_rate)
+				/ NSEC_PER_MSEC;
+	tx_line_reset_period = (TX_LINE_RESET_TIME * ufs->mclk_rate)
+				/ NSEC_PER_MSEC;
+
+	unipro_writel(ufs, 0x5f, 0x44);
+
+	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(0x202), 0x02);
+
+	for_each_ufs_rx_lane(ufs, i) {
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD, i),
+			       DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD_EN, i), 0x0);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE2, i),
+			       (rx_line_reset_period >> 16) & 0xFF);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE1, i),
+			       (rx_line_reset_period >> 8) & 0xFF);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE0, i),
+			       (rx_line_reset_period) & 0xFF);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x2f, i), 0x69);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x84, i), 0x1);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x25, i), 0xf6);
+	}
+
+	for_each_ufs_tx_lane(ufs, i) {
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD, i),
+			       DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD_EN, i),
+			       0x02);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE2, i),
+			       (tx_line_reset_period >> 16) & 0xFF);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE1, i),
+			       (tx_line_reset_period >> 8) & 0xFF);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE0, i),
+			       (tx_line_reset_period) & 0xFF);
+
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x04, i), 0x1);
+		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x7f, i), 0x0);
+	}
+
+	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0x0);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(0xa011), 0x8000);
+
+	return 0;
+}
+
+static int exynosautov920_ufs_post_link(struct exynos_ufs *ufs)
+{
+	struct ufs_hba *hba = ufs->hba;
+
+	ufshcd_dme_set(hba, UIC_ARG_MIB(0x9529), 0x1);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(0x15a4), 0x3e8);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(0x9529), 0x0);
+
+	return 0;
+}
+
+static int exynosautov920_ufs_pre_pwr_change(struct exynos_ufs *ufs,
+					     struct ufs_pa_layer_attr *pwr)
+{
+	struct ufs_hba *hba = ufs->hba;
+
+	ufshcd_dme_set(hba, UIC_ARG_MIB(0x15d4), 0x1);
+
+	ufshcd_dme_set(hba, UIC_ARG_MIB(DL_FC0PROTTIMEOUTVAL), 8064);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(DL_TC0REPLAYTIMEOUTVAL), 28224);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(DL_AFC0REQTIMEOUTVAL), 20160);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 12000);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 32000);
+	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 16000);
+
+	unipro_writel(ufs, 8064, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER0);
+	unipro_writel(ufs, 28224, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER1);
+	unipro_writel(ufs, 20160, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER2);
+	unipro_writel(ufs, 12000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER0);
+	unipro_writel(ufs, 32000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER1);
+	unipro_writel(ufs, 16000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER2);
+
+	return 0;
+}
+
 /*
  * exynos_ufs_auto_ctrl_hcc - HCI core clock control by h/w
  * Control should be disabled in the below cases
@@ -2201,6 +2294,21 @@ static const struct exynos_ufs_drv_data gs101_ufs_drvs = {
 	.suspend		= gs101_ufs_suspend,
 };
 
+static const struct exynos_ufs_drv_data exynosautov920_ufs_drvs = {
+	.uic_attr               = &exynos7_uic_attr,
+	.quirks                 = UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING,
+	.opts                   = EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL |
+				  EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR |
+				  EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX |
+				  EXYNOS_UFS_OPT_TIMER_TICK_SELECT,
+	.iocc_mask		= UFS_EXYNOSAUTOV920_SHARABLE,
+	.drv_init               = exynosauto_ufs_drv_init,
+	.post_hce_enable        = exynosauto_ufs_post_hce_enable,
+	.pre_link               = exynosautov920_ufs_pre_link,
+	.post_link              = exynosautov920_ufs_post_link,
+	.pre_pwr_change         = exynosautov920_ufs_pre_pwr_change,
+};
+
 static const struct of_device_id exynos_ufs_of_match[] = {
 	{ .compatible = "google,gs101-ufs",
 	  .data	      = &gs101_ufs_drvs },
@@ -2210,6 +2318,8 @@ static const struct of_device_id exynos_ufs_of_match[] = {
 	  .data	      = &exynosauto_ufs_drvs },
 	{ .compatible = "samsung,exynosautov9-ufs-vh",
 	  .data	      = &exynosauto_ufs_vh_drvs },
+	{ .compatible = "samsung,exynosautov920-ufs",
+	  .data       = &exynosautov920_ufs_drvs },
 	{ .compatible = "tesla,fsd-ufs",
 	  .data       = &fsd_ufs_drvs },
 	{},
diff --git a/drivers/ufs/host/ufs-qcom.c b/drivers/ufs/host/ufs-qcom.c
index bc037db46624a..9664b8ae67b78 100644
--- a/drivers/ufs/host/ufs-qcom.c
+++ b/drivers/ufs/host/ufs-qcom.c
@@ -177,14 +177,14 @@ static int ufs_qcom_ice_init(struct ufs_qcom_host *host)
 	int i;
 
 	ice = devm_of_qcom_ice_get(dev);
-	if (ice == ERR_PTR(-EOPNOTSUPP)) {
+	if (IS_ERR(ice)) {
+		if (ice != ERR_PTR(-EOPNOTSUPP))
+			return PTR_ERR(ice);
+
 		dev_warn(dev, "Disabling inline encryption support\n");
-		ice = NULL;
+		return 0;
 	}
 
-	if (IS_ERR_OR_NULL(ice))
-		return PTR_ERR_OR_ZERO(ice);
-
 	host->ice = ice;
 
 	/* Initialize the blk_crypto_profile */
@@ -705,6 +705,13 @@ static int ufs_qcom_cfg_timers(struct ufs_hba *hba, bool is_pre_scale_up, unsign
 	return 0;
 }
 
+static void ufs_qcom_link_startup_post_change(struct ufs_hba *hba)
+{
+	if (ufshcd_is_auto_hibern8_supported(hba))
+		ufshcd_rmwl(hba, UFS_HW_CLK_CTRL_EN, UFS_HW_CLK_CTRL_EN,
+			    UFS_AH8_CFG);
+}
+
 static int ufs_qcom_link_startup_notify(struct ufs_hba *hba,
 					enum ufs_notify_change_status status)
 {
@@ -730,6 +737,9 @@ static int ufs_qcom_link_startup_notify(struct ufs_hba *hba,
 		 */
 		err = ufshcd_disable_host_tx_lcc(hba);
 
+		break;
+	case POST_CHANGE:
+		ufs_qcom_link_startup_post_change(hba);
 		break;
 	default:
 		break;
diff --git a/drivers/ufs/host/ufs-qcom.h b/drivers/ufs/host/ufs-qcom.h
index 5d083331a7f43..e20b3ca505775 100644
--- a/drivers/ufs/host/ufs-qcom.h
+++ b/drivers/ufs/host/ufs-qcom.h
@@ -268,6 +268,17 @@ enum {
  */
 #define NUM_TX_R1W1 13
 
+/* bit definitions for UFS_AH8_CFG register */
+#define CC_UFS_SYS_CLK_REQ_EN          BIT(2)
+#define CC_UFS_ICE_CORE_CLK_REQ_EN     BIT(3)
+#define CC_UFS_UNIPRO_CORE_CLK_REQ_EN  BIT(4)
+#define CC_UFS_AUXCLK_REQ_EN           BIT(5)
+
+#define UFS_HW_CLK_CTRL_EN	(CC_UFS_SYS_CLK_REQ_EN |\
+				CC_UFS_ICE_CORE_CLK_REQ_EN |\
+				CC_UFS_UNIPRO_CORE_CLK_REQ_EN |\
+				CC_UFS_AUXCLK_REQ_EN)
+
 static inline void
 ufs_qcom_get_controller_revision(struct ufs_hba *hba,
 				 u8 *major, u16 *minor, u16 *step)
diff --git a/drivers/ufs/host/ufshcd-pci.c b/drivers/ufs/host/ufshcd-pci.c
index effa3c7a01c57..13293e83064ce 100644
--- a/drivers/ufs/host/ufshcd-pci.c
+++ b/drivers/ufs/host/ufshcd-pci.c
@@ -680,21 +680,20 @@ static const struct dev_pm_ops ufshcd_pci_pm_ops = {
 };
 
 static const struct pci_device_id ufshcd_pci_tbl[] = {
-	{ PCI_VENDOR_ID_REDHAT, 0x0013, PCI_ANY_ID, PCI_ANY_ID, 0, 0,
-		(kernel_ulong_t)&ufs_qemu_hba_vops },
-	{ PCI_VENDOR_ID_SAMSUNG, 0xC00C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VDEVICE(INTEL, 0x9DFA), (kernel_ulong_t)&ufs_intel_cnl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0x4B41), (kernel_ulong_t)&ufs_intel_ehl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0x4B43), (kernel_ulong_t)&ufs_intel_ehl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0x98FA), (kernel_ulong_t)&ufs_intel_lkf_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0x51FF), (kernel_ulong_t)&ufs_intel_adl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0x54FF), (kernel_ulong_t)&ufs_intel_adl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0x7E47), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0xA847), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0x7747), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0xE447), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0x4D47), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
-	{ PCI_VDEVICE(INTEL, 0xD335), (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+	{ PCI_VDEVICE(REDHAT, 0x0013), .driver_data = (kernel_ulong_t)&ufs_qemu_hba_vops },
+	{ PCI_VDEVICE(SAMSUNG, 0xC00C), .driver_data = 0 },
+	{ PCI_VDEVICE(INTEL, 0x9DFA), .driver_data = (kernel_ulong_t)&ufs_intel_cnl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0x4B41), .driver_data = (kernel_ulong_t)&ufs_intel_ehl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0x4B43), .driver_data = (kernel_ulong_t)&ufs_intel_ehl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0x98FA), .driver_data = (kernel_ulong_t)&ufs_intel_lkf_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0x51FF), .driver_data = (kernel_ulong_t)&ufs_intel_adl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0x54FF), .driver_data = (kernel_ulong_t)&ufs_intel_adl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0x7E47), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0xA847), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0x7747), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0xE447), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0x4D47), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
+	{ PCI_VDEVICE(INTEL, 0xD335), .driver_data = (kernel_ulong_t)&ufs_intel_mtl_hba_vops },
 	{ }	/* terminate list */
 };
 
diff --git a/drivers/usb/Makefile b/drivers/usb/Makefile
index 6f3c861498875..eecbd631fdab5 100644
--- a/drivers/usb/Makefile
+++ b/drivers/usb/Makefile
@@ -14,8 +14,6 @@ obj-$(CONFIG_USB_DWC2)		+= dwc2/
 obj-$(CONFIG_USB_ISP1760)	+= isp1760/
 
 obj-$(CONFIG_USB_CDNS_SUPPORT)	+= cdns3/
-obj-$(CONFIG_USB_CDNS3)		+= cdns3/
-obj-$(CONFIG_USB_CDNSP_PCI)	+= cdns3/
 
 obj-$(CONFIG_USB_FOTG210)	+= fotg210/
 
diff --git a/drivers/usb/cdns3/Kconfig b/drivers/usb/cdns3/Kconfig
index 0a514b5915272..39ad23d1ada89 100644
--- a/drivers/usb/cdns3/Kconfig
+++ b/drivers/usb/cdns3/Kconfig
@@ -1,6 +1,9 @@
 config USB_CDNS_SUPPORT
 	tristate "Cadence USB Support"
-	depends on USB_SUPPORT && (USB || USB_GADGET) && HAS_DMA
+	depends on USB_SUPPORT && HAS_DMA
+	depends on USB || USB_GADGET
+	depends on USB if !USB_GADGET
+	depends on USB_GADGET if !USB
 	select USB_XHCI_PLATFORM if USB_XHCI_HCD
 	select USB_ROLE_SWITCH
 	help
@@ -8,44 +11,49 @@ config USB_CDNS_SUPPORT
 	  dual-role controller.
 	  It supports: dual-role switch, Host-only, and Peripheral-only.
 
-config USB_CDNS_HOST
-	bool
-
 if USB_CDNS_SUPPORT
 
 config USB_CDNS3
-	tristate "Cadence USB3 Dual-Role Controller"
+	tristate "Cadence USB dual-role controller (USBSS and USBSSP)"
 	depends on USB_CDNS_SUPPORT
 	help
-	  Say Y here if your system has a Cadence USB3 dual-role controller.
-	  It supports: dual-role switch, Host-only, and Peripheral-only.
+	  Say Y or M here if your system has an on-chip Cadence USB
+	  dual-role controller. This covers both USBSS (USB 3.0) and
+	  USBSSP (SuperSpeed Plus) IP; the driver detects the variant at
+	  runtime.
 
-	  If you choose to build this driver is a dynamically linked
-	  as module, the module will be called cdns3.ko.
-endif
+	  The core driver (core, DRD, generic platform binding for the
+	  "cdns,usb3" device tree compatible, optional host and gadget)
+	  builds as one module named cdns.ko when built as a loadable
+	  module.
+
+	  It supports: dual-role switch, Host-only, and Peripheral-only.
 
 if USB_CDNS3
 
-config USB_CDNS3_GADGET
-	bool "Cadence USB3 device controller"
-	depends on USB_GADGET=y || USB_GADGET=USB_CDNS3
+config USB_CDNS3_HOST
+	bool "Cadence USB host controller (xHCI)"
+	depends on USB=y || USB=USB_CDNS3
 	help
-	  Say Y here to enable device controller functionality of the
-	  Cadence USBSS-DEV driver.
+	  Say Y here to enable host controller functionality for Cadence
+	  USBSS and USBSSP dual-role controllers.
 
-	  This controller supports FF, HS and SS mode. It doesn't support
-	  LS and SSP mode.
+	  The host controller is xHCI compliant and uses the standard
+	  xHCI driver.
 
-config USB_CDNS3_HOST
-	bool "Cadence USB3 host controller"
-	depends on USB=y || USB=USB_CDNS3
-	select USB_CDNS_HOST
+config USB_CDNS3_GADGET
+	bool "Cadence USB device controller (USBSS and USBSSP)"
+	depends on USB_GADGET=y || USB_GADGET=USB_CDNS3
 	help
-	  Say Y here to enable host controller functionality of the
-	  Cadence driver.
+	  Say Y here to include Cadence USB device (gadget) support for
+	  both USBSS (USB 3.0) and USBSSP (SuperSpeed Plus) IP in the
+	  cdns.ko module. The implementation is selected at runtime from
+	  the detected controller version.
 
-	  Host controller is compliant with XHCI so it will use
-	  standard XHCI driver.
+	  USBSS gadget supports FF, HS and SS mode (not LS or SSP).
+	  USBSSP gadget supports FF, HS, SS and SSP mode (not LS).
+
+comment "Platform glue driver support"
 
 config USB_CDNS3_PCI_WRAP
 	tristate "Cadence USB3 support on PCIe-based platforms"
@@ -58,6 +66,17 @@ config USB_CDNS3_PCI_WRAP
 	  If you choose to build this driver as module it will
 	  be dynamically linked and module will be called cdns3-pci.ko
 
+config USB_CDNSP_PCI
+	tristate "Cadence USBSSP support on PCIe-based platforms"
+	depends on USB_PCI && ACPI
+	default USB_CDNS3
+	help
+	  If you're using the USBSSP Core IP with a PCIe, please say
+	  'Y' or 'M' here.
+
+	  If you choose to build this driver as module it will
+	  be dynamically linked and module will be called cdnsp-pci.ko
+
 config USB_CDNS3_TI
 	tristate "Cadence USB3 support on TI platforms"
 	depends on ARCH_K3 || COMPILE_TEST
@@ -81,6 +100,7 @@ config USB_CDNS3_IMX
 config USB_CDNS3_STARFIVE
 	tristate "Cadence USB3 support on StarFive SoC platforms"
 	depends on ARCH_STARFIVE || COMPILE_TEST
+	default USB_CDNS3
 	help
 	  Say 'Y' or 'M' here if you are building for StarFive SoCs
 	  platforms that contain Cadence USB3 controller core.
@@ -89,45 +109,7 @@ config USB_CDNS3_STARFIVE
 
 	  If you choose to build this driver as module it will
 	  be dynamically linked and module will be called cdns3-starfive.ko
-endif
-
-if USB_CDNS_SUPPORT
-
-config USB_CDNSP_PCI
-	tristate "Cadence CDNSP Dual-Role Controller"
-	depends on USB_CDNS_SUPPORT && USB_PCI && ACPI
-	help
-	  Say Y here if your system has a Cadence CDNSP dual-role controller.
-	  It supports: dual-role switch Host-only, and Peripheral-only.
-
-	  If you choose to build this driver is a dynamically linked
-	  module, the module will be called cdnsp.ko.
-endif
-
-if USB_CDNSP_PCI
-
-config USB_CDNSP_GADGET
-	bool "Cadence CDNSP device controller"
-	depends on USB_GADGET=y || USB_GADGET=USB_CDNSP_PCI
-	help
-	  Say Y here to enable device controller functionality of the
-	  Cadence CDNSP-DEV driver.
-
-	  Cadence CDNSP Device Controller in device mode is
-	  very similar to XHCI controller. Therefore some algorithms
-	  used has been taken from host driver.
-	  This controller supports FF, HS, SS and SSP mode.
-	  It doesn't support LS.
-
-config USB_CDNSP_HOST
-	bool "Cadence CDNSP host controller"
-	depends on USB=y || USB=USB_CDNSP_PCI
-	select USB_CDNS_HOST
-	help
-	  Say Y here to enable host controller functionality of the
-	  Cadence driver.
 
-	  Host controller is compliant with XHCI so it uses
-	  standard XHCI driver.
+endif # USB_CDNS3
 
-endif
+endif # USB_CDNS_SUPPORT
diff --git a/drivers/usb/cdns3/Makefile b/drivers/usb/cdns3/Makefile
index 48dfae75b5aaf..b2e4ba6a49a36 100644
--- a/drivers/usb/cdns3/Makefile
+++ b/drivers/usb/cdns3/Makefile
@@ -3,42 +3,28 @@
 CFLAGS_cdns3-trace.o				:= -I$(src)
 CFLAGS_cdnsp-trace.o				:= -I$(src)
 
-cdns-usb-common-y				:= core.o drd.o
-cdns3-y						:= cdns3-plat.o
+obj-$(CONFIG_USB_CDNS3)			+= cdns.o
 
-ifeq ($(CONFIG_USB),m)
-obj-m						+= cdns-usb-common.o
-obj-m						+= cdns3.o
-else
-obj-$(CONFIG_USB_CDNS_SUPPORT)			+= cdns-usb-common.o
-obj-$(CONFIG_USB_CDNS3)				+= cdns3.o
-endif
+cdns-y					:= core.o drd.o cdns3-plat.o
+cdns-$(CONFIG_USB_CDNS3_HOST)		+= host.o
 
-cdns-usb-common-$(CONFIG_USB_CDNS_HOST) 	+= host.o
-cdns3-$(CONFIG_USB_CDNS3_GADGET)		+= cdns3-gadget.o cdns3-ep0.o
+ifneq ($(CONFIG_USB_CDNS3_GADGET),)
+cdns-y					+= cdns3-gadget.o cdns3-ep0.o \
+					   cdnsp-ring.o cdnsp-gadget.o \
+					   cdnsp-mem.o cdnsp-ep0.o
+endif
 
+ifneq ($(CONFIG_TRACING),)
 ifneq ($(CONFIG_USB_CDNS3_GADGET),)
-cdns3-$(CONFIG_TRACING)				+= cdns3-trace.o
+cdns-y					+= cdns3-trace.o cdnsp-trace.o
+endif
 endif
 
+##
+# Platform-specific glue layers (PCI wrappers, SoC integration)
+##
 obj-$(CONFIG_USB_CDNS3_PCI_WRAP)		+= cdns3-pci-wrap.o
+obj-$(CONFIG_USB_CDNSP_PCI)			+= cdnsp-pci.o
 obj-$(CONFIG_USB_CDNS3_TI)			+= cdns3-ti.o
 obj-$(CONFIG_USB_CDNS3_IMX)			+= cdns3-imx.o
 obj-$(CONFIG_USB_CDNS3_STARFIVE)		+= cdns3-starfive.o
-
-cdnsp-udc-pci-y					:= cdnsp-pci.o
-
-ifdef CONFIG_USB_CDNSP_PCI
-ifeq ($(CONFIG_USB),m)
-obj-m						+= cdnsp-udc-pci.o
-else
-obj-$(CONFIG_USB_CDNSP_PCI) 			+= cdnsp-udc-pci.o
-endif
-endif
-
-cdnsp-udc-pci-$(CONFIG_USB_CDNSP_GADGET)	+= cdnsp-ring.o cdnsp-gadget.o \
-						   cdnsp-mem.o cdnsp-ep0.o
-
-ifneq ($(CONFIG_USB_CDNSP_GADGET),)
-cdnsp-udc-pci-$(CONFIG_TRACING)			+= cdnsp-trace.o
-endif
diff --git a/drivers/usb/cdns3/cdns3-gadget.c b/drivers/usb/cdns3/cdns3-gadget.c
index 8382231af3574..90c2aea93cce9 100644
--- a/drivers/usb/cdns3/cdns3-gadget.c
+++ b/drivers/usb/cdns3/cdns3-gadget.c
@@ -3512,3 +3512,4 @@ int cdns3_gadget_init(struct cdns *cdns)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(cdns3_gadget_init);
diff --git a/drivers/usb/cdns3/cdns3-plat.c b/drivers/usb/cdns3/cdns3-plat.c
index 735df88774e43..3fe3109a3688a 100644
--- a/drivers/usb/cdns3/cdns3-plat.c
+++ b/drivers/usb/cdns3/cdns3-plat.c
@@ -21,6 +21,7 @@
 
 #include "core.h"
 #include "gadget-export.h"
+#include "host-export.h"
 #include "drd.h"
 
 static int set_phy_power_on(struct cdns *cdns)
@@ -44,6 +45,19 @@ static void set_phy_power_off(struct cdns *cdns)
 	phy_power_off(cdns->usb2_phy);
 }
 
+static int cdns3_plat_gadget_init(struct cdns *cdns)
+{
+	if (cdns->version < CDNSP_CONTROLLER_V2)
+		return cdns3_gadget_init(cdns);
+	else
+		return cdnsp_gadget_init(cdns);
+}
+
+static int cdns3_plat_host_init(struct cdns *cdns)
+{
+	return cdns_host_init(cdns);
+}
+
 /**
  * cdns3_plat_probe - probe for cdns3 core device
  * @pdev: Pointer to cdns3 core platform device
@@ -64,6 +78,8 @@ static int cdns3_plat_probe(struct platform_device *pdev)
 
 	cdns->dev = dev;
 	cdns->pdata = dev_get_platdata(dev);
+	if (cdns->pdata && cdns->pdata->override_apb_timeout)
+		cdns->override_apb_timeout = cdns->pdata->override_apb_timeout;
 
 	platform_set_drvdata(pdev, cdns);
 
@@ -143,12 +159,16 @@ static int cdns3_plat_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_phy_power_on;
 
-	cdns->gadget_init = cdns3_gadget_init;
-
 	ret = cdns_init(cdns);
 	if (ret)
 		goto err_cdns_init;
 
+	cdns->gadget_init = cdns3_plat_gadget_init;
+	cdns->host_init = cdns3_plat_host_init;
+	ret = cdns_core_init_role(cdns);
+	if (ret)
+		goto err_cdns_init;
+
 	device_set_wakeup_capable(dev, true);
 	pm_runtime_set_active(dev);
 	pm_runtime_enable(dev);
@@ -336,6 +356,3 @@ static struct platform_driver cdns3_driver = {
 module_platform_driver(cdns3_driver);
 
 MODULE_ALIAS("platform:cdns3");
-MODULE_AUTHOR("Pawel Laszczak <pawell@cadence.com>");
-MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("Cadence USB3 DRD Controller Driver");
diff --git a/drivers/usb/cdns3/cdnsp-gadget.c b/drivers/usb/cdns3/cdnsp-gadget.c
index 6b3815f8a6e5b..a5275c2fb43b8 100644
--- a/drivers/usb/cdns3/cdnsp-gadget.c
+++ b/drivers/usb/cdns3/cdnsp-gadget.c
@@ -124,20 +124,28 @@ void cdnsp_set_link_state(struct cdnsp_device *pdev,
 }
 
 static void cdnsp_disable_port(struct cdnsp_device *pdev,
-			       __le32 __iomem *port_regs)
+			       struct cdnsp_port *port)
 {
-	u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
+	u32 temp;
+
+	if (!port->exist)
+		return;
 
-	writel(temp | PORT_PED, port_regs);
+	temp = cdnsp_port_state_to_neutral(readl(&port->regs->portsc));
+	writel(temp | PORT_PED, &port->regs->portsc);
 }
 
 static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
-					__le32 __iomem *port_regs)
+					struct cdnsp_port *port)
 {
-	u32 portsc = readl(port_regs);
+	u32 portsc;
+
+	if (!port->exist)
+		return;
 
+	portsc = readl(&port->regs->portsc);
 	writel(cdnsp_port_state_to_neutral(portsc) |
-	       (portsc & PORT_CHANGE_BITS), port_regs);
+	       (portsc & PORT_CHANGE_BITS), &port->regs->portsc);
 }
 
 static void cdnsp_set_apb_timeout_value(struct cdnsp_device *pdev)
@@ -944,7 +952,7 @@ void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
 				 struct usb_request *req,
 				 int enable)
 {
-	if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
+	if (pdev->active_port == &pdev->usb3_port || !pdev->gadget.lpm_capable)
 		return;
 
 	trace_cdnsp_lpm(enable);
@@ -1310,20 +1318,26 @@ static int cdnsp_run(struct cdnsp_device *pdev,
 		break;
 	}
 
-	if (speed >= USB_SPEED_SUPER) {
+	if (pdev->usb3_port.exist && speed >= USB_SPEED_SUPER) {
 		writel(temp, &pdev->port3x_regs->mode_addr);
 		cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
 				     XDEV_RXDETECT);
 	} else {
-		cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
+		cdnsp_disable_port(pdev, &pdev->usb3_port);
 	}
 
-	cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
-			     XDEV_RXDETECT);
+	if (pdev->usb2_port.exist) {
+		cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
+				     XDEV_RXDETECT);
+		writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
+	}
+
+	if (pdev->eusb_port.exist)
+		cdnsp_set_link_state(pdev, &pdev->eusb_port.regs->portsc,
+				     XDEV_RXDETECT);
 
 	cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
 
-	writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
 
 	ret = cdnsp_start(pdev);
 	if (ret) {
@@ -1469,8 +1483,10 @@ static void cdnsp_stop(struct cdnsp_device *pdev)
 			cdnsp_ep_dequeue(&pdev->eps[0], req);
 	}
 
-	cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
-	cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
+	cdnsp_disable_port(pdev, &pdev->usb2_port);
+	cdnsp_disable_port(pdev, &pdev->usb3_port);
+	cdnsp_disable_port(pdev, &pdev->eusb_port);
+
 	cdnsp_disable_slot(pdev);
 	cdnsp_halt(pdev);
 
@@ -1479,8 +1495,9 @@ static void cdnsp_stop(struct cdnsp_device *pdev)
 	temp = readl(&pdev->ir_set->irq_pending);
 	writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
 
-	cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
-	cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
+	cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port);
+	cdnsp_clear_port_change_bit(pdev, &pdev->eusb_port);
+	cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port);
 
 	/* Clear interrupt line */
 	temp = readl(&pdev->ir_set->irq_pending);
@@ -2075,3 +2092,4 @@ int cdnsp_gadget_init(struct cdns *cdns)
 
 	return 0;
 }
+EXPORT_SYMBOL_GPL(cdnsp_gadget_init);
diff --git a/drivers/usb/cdns3/cdnsp-gadget.h b/drivers/usb/cdns3/cdnsp-gadget.h
index a91cca509db08..c44bca348a419 100644
--- a/drivers/usb/cdns3/cdnsp-gadget.h
+++ b/drivers/usb/cdns3/cdnsp-gadget.h
@@ -1474,6 +1474,7 @@ struct cdnsp_device {
 	unsigned int link_state;
 
 	struct cdnsp_port usb2_port;
+	struct cdnsp_port eusb_port;
 	struct cdnsp_port usb3_port;
 	struct cdnsp_port *active_port;
 	u16 test_mode;
diff --git a/drivers/usb/cdns3/cdnsp-mem.c b/drivers/usb/cdns3/cdnsp-mem.c
index a2a1b21f2ef87..5d8cdc91927dd 100644
--- a/drivers/usb/cdns3/cdnsp-mem.c
+++ b/drivers/usb/cdns3/cdnsp-mem.c
@@ -1088,11 +1088,9 @@ void cdnsp_mem_cleanup(struct cdnsp_device *pdev)
 			  pdev->dcbaa, pdev->dcbaa->dma);
 
 	pdev->dcbaa = NULL;
-
-	pdev->usb2_port.exist = 0;
-	pdev->usb3_port.exist = 0;
-	pdev->usb2_port.port_num = 0;
-	pdev->usb3_port.port_num = 0;
+	memset(&pdev->usb2_port, 0, sizeof(struct cdnsp_port));
+	memset(&pdev->eusb_port, 0, sizeof(struct cdnsp_port));
+	memset(&pdev->usb3_port, 0, sizeof(struct cdnsp_port));
 	pdev->active_port = NULL;
 }
 
@@ -1133,6 +1131,18 @@ static void cdnsp_add_in_port(struct cdnsp_device *pdev,
 	port_offset = CDNSP_EXT_PORT_OFF(temp);
 	port_count = CDNSP_EXT_PORT_COUNT(temp);
 
+	if (port == &pdev->eusb_port) {
+		/*
+		 * If controller has usb2 + eusb port then eusb is as
+		 * second port
+		 */
+		if (port_count == 2)
+			port_offset++;
+
+		if (port_count == 1 && pdev->usb2_port.exist)
+			return;
+	}
+
 	trace_cdnsp_port_info(addr, port_offset, port_count, port->maj_rev);
 
 	port->port_num = port_offset;
@@ -1152,13 +1162,10 @@ static int cdnsp_setup_port_arrays(struct cdnsp_device *pdev)
 	base = &pdev->cap_regs->hc_capbase;
 	offset = cdnsp_find_next_ext_cap(base, 0,
 					 EXT_CAP_CFG_DEV_20PORT_CAP_ID);
-	pdev->port20_regs = base + offset;
-
-	offset = cdnsp_find_next_ext_cap(base, 0, D_XEC_CFG_3XPORT_CAP);
-	pdev->port3x_regs =  base + offset;
+	if (offset)
+		pdev->port20_regs = base + offset;
 
 	offset = 0;
-	base = &pdev->cap_regs->hc_capbase;
 
 	/* Driver expects max 2 extended protocol capability. */
 	for (i = 0; i < 2; i++) {
@@ -1173,26 +1180,46 @@ static int cdnsp_setup_port_arrays(struct cdnsp_device *pdev)
 			cdnsp_add_in_port(pdev, &pdev->usb3_port,
 					  base + offset);
 
-		if (CDNSP_EXT_PORT_MAJOR(temp) == 0x02 &&
-		    !pdev->usb2_port.port_num)
-			cdnsp_add_in_port(pdev, &pdev->usb2_port,
-					  base + offset);
+		if (CDNSP_EXT_PORT_MAJOR(temp) == 0x02) {
+			if (!pdev->usb2_port.port_num && pdev->port20_regs)
+				cdnsp_add_in_port(pdev, &pdev->usb2_port,
+						  base + offset);
+
+			if (!pdev->eusb_port.port_num)
+				cdnsp_add_in_port(pdev, &pdev->eusb_port,
+						  base + offset);
+		}
 	}
 
-	if (!pdev->usb2_port.exist || !pdev->usb3_port.exist) {
-		dev_err(pdev->dev, "Error: Only one port detected\n");
+	if (!pdev->usb2_port.exist && !pdev->eusb_port.exist &&
+	    !pdev->usb3_port.exist) {
+		dev_err(pdev->dev, "Error: No port detected\n");
 		return -ENODEV;
 	}
 
-	trace_cdnsp_init("Found USB 2.0 ports and  USB 3.0 ports.");
+	if (pdev->usb2_port.exist) {
+		pdev->usb2_port.regs = (struct cdnsp_port_regs __iomem *)
+				       (&pdev->op_regs->port_reg_base + NUM_PORT_REGS *
+					(pdev->usb2_port.port_num - 1));
+		trace_cdnsp_init("Found USB 2.0 port.");
+	}
 
-	pdev->usb2_port.regs = (struct cdnsp_port_regs __iomem *)
-			       (&pdev->op_regs->port_reg_base + NUM_PORT_REGS *
-				(pdev->usb2_port.port_num - 1));
+	if (pdev->eusb_port.exist) {
+		pdev->eusb_port.regs = (struct cdnsp_port_regs __iomem *)
+				       (&pdev->op_regs->port_reg_base + NUM_PORT_REGS *
+					(pdev->eusb_port.port_num - 1));
+		trace_cdnsp_init("Found eUSB 2.0 port.");
+	}
+
+	if (pdev->usb3_port.exist) {
+		offset = cdnsp_find_next_ext_cap(base, 0, D_XEC_CFG_3XPORT_CAP);
+		pdev->port3x_regs =  base + offset;
 
-	pdev->usb3_port.regs = (struct cdnsp_port_regs __iomem *)
-			       (&pdev->op_regs->port_reg_base + NUM_PORT_REGS *
-				(pdev->usb3_port.port_num - 1));
+		pdev->usb3_port.regs = (struct cdnsp_port_regs __iomem *)
+				       (&pdev->op_regs->port_reg_base + NUM_PORT_REGS *
+					(pdev->usb3_port.port_num - 1));
+		trace_cdnsp_init("Found USB 3.x port.");
+	}
 
 	return 0;
 }
diff --git a/drivers/usb/cdns3/cdnsp-pci.c b/drivers/usb/cdns3/cdnsp-pci.c
index 566d94e49102e..432007cfe6954 100644
--- a/drivers/usb/cdns3/cdnsp-pci.c
+++ b/drivers/usb/cdns3/cdnsp-pci.c
@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
- * Cadence PCI Glue driver.
+ * Cadence USBSSP PCI Glue driver.
  *
  * Copyright (C) 2019 Cadence.
  *
@@ -16,7 +16,19 @@
 #include <linux/pci.h>
 
 #include "core.h"
-#include "gadget-export.h"
+
+struct cdnsp_wrap {
+	struct platform_device *plat_dev;
+	struct resource dev_res[6];
+	int devfn;
+};
+
+#define RES_IRQ_HOST_ID		0
+#define RES_IRQ_PERIPHERAL_ID	1
+#define RES_IRQ_OTG_ID		2
+#define RES_HOST_ID		3
+#define RES_DEV_ID		4
+#define RES_DRD_ID		5
 
 #define PCI_BAR_HOST		0
 #define PCI_BAR_OTG		0
@@ -26,16 +38,16 @@
 #define PCI_DEV_FN_OTG		1
 
 #define PCI_DRIVER_NAME		"cdns-pci-usbssp"
-#define PLAT_DRIVER_NAME	"cdns-usbssp"
+#define PLAT_DRIVER_NAME	"cdns-usb3"
 
-#define CHICKEN_APB_TIMEOUT_VALUE       0x1C20
+#define CHICKEN_APB_TIMEOUT_VALUE	0x1C20
 
 static struct pci_dev *cdnsp_get_second_fun(struct pci_dev *pdev)
 {
 	/*
 	 * Gets the second function.
-	 * Platform has two function. The fist keeps resources for
-	 * Host/Device while the secon keeps resources for DRD/OTG.
+	 * Platform has two function. The first keeps resources for
+	 * Host/Device while the second keeps resources for DRD/OTG.
 	 */
 	if (pdev->device == PCI_DEVICE_ID_CDNS_USBSSP)
 		return pci_get_device(pdev->vendor, PCI_DEVICE_ID_CDNS_USBSS, NULL);
@@ -48,11 +60,12 @@ static struct pci_dev *cdnsp_get_second_fun(struct pci_dev *pdev)
 static int cdnsp_pci_probe(struct pci_dev *pdev,
 			   const struct pci_device_id *id)
 {
-	struct device *dev = &pdev->dev;
-	struct pci_dev *func;
+	struct platform_device_info plat_info;
+	static struct cdns3_platform_data pdata;
+	struct cdnsp_wrap *wrap;
 	struct resource *res;
-	struct cdns *cdnsp;
-	int ret;
+	struct pci_dev *func;
+	int ret = 0;
 
 	/*
 	 * For GADGET/HOST PCI (devfn) function number is 0,
@@ -79,146 +92,105 @@ static int cdnsp_pci_probe(struct pci_dev *pdev,
 	}
 
 	pci_set_master(pdev);
+
 	if (pci_is_enabled(func)) {
-		cdnsp = pci_get_drvdata(func);
+		wrap = pci_get_drvdata(func);
 	} else {
-		cdnsp = kzalloc_obj(*cdnsp);
-		if (!cdnsp) {
+		wrap = kzalloc_obj(*wrap);
+		if (!wrap) {
 			ret = -ENOMEM;
 			goto put_pci;
 		}
 	}
 
-	/* For GADGET device function number is 0. */
-	if (pdev->devfn == 0) {
-		resource_size_t rsrc_start, rsrc_len;
-
-		/* Function 0: host(BAR_0) + device(BAR_1).*/
-		dev_dbg(dev, "Initialize resources\n");
-		rsrc_start = pci_resource_start(pdev, PCI_BAR_DEV);
-		rsrc_len = pci_resource_len(pdev, PCI_BAR_DEV);
-		res = devm_request_mem_region(dev, rsrc_start, rsrc_len, "dev");
-		if (!res) {
-			dev_dbg(dev, "controller already in use\n");
-			ret = -EBUSY;
-			goto free_cdnsp;
-		}
-
-		cdnsp->dev_regs = devm_ioremap(dev, rsrc_start, rsrc_len);
-		if (!cdnsp->dev_regs) {
-			dev_dbg(dev, "error mapping memory\n");
-			ret = -EFAULT;
-			goto free_cdnsp;
-		}
-
-		cdnsp->dev_irq = pdev->irq;
-		dev_dbg(dev, "USBSS-DEV physical base addr: %pa\n",
-			&rsrc_start);
-
-		res = &cdnsp->xhci_res[0];
-		res->start = pci_resource_start(pdev, PCI_BAR_HOST);
-		res->end = pci_resource_end(pdev, PCI_BAR_HOST);
-		res->name = "xhci";
-		res->flags = IORESOURCE_MEM;
-		dev_dbg(dev, "USBSS-XHCI physical base addr: %pa\n",
-			&res->start);
-
-		/* Interrupt for XHCI, */
-		res = &cdnsp->xhci_res[1];
-		res->start = pdev->irq;
-		res->name = "host";
-		res->flags = IORESOURCE_IRQ;
+	res = wrap->dev_res;
+
+	if (pdev->devfn == PCI_DEV_FN_HOST_DEVICE) {
+		/* Function 0: host(BAR_0) + device(BAR_2). */
+		dev_dbg(&pdev->dev, "Initialize Device resources\n");
+		res[RES_DEV_ID].start = pci_resource_start(pdev, PCI_BAR_DEV);
+		res[RES_DEV_ID].end = pci_resource_end(pdev, PCI_BAR_DEV);
+		res[RES_DEV_ID].name = "dev";
+		res[RES_DEV_ID].flags = IORESOURCE_MEM;
+		dev_dbg(&pdev->dev, "USBSSP-DEV physical base addr: %pa\n",
+			&res[RES_DEV_ID].start);
+
+		res[RES_HOST_ID].start = pci_resource_start(pdev, PCI_BAR_HOST);
+		res[RES_HOST_ID].end = pci_resource_end(pdev, PCI_BAR_HOST);
+		res[RES_HOST_ID].name = "xhci";
+		res[RES_HOST_ID].flags = IORESOURCE_MEM;
+		dev_dbg(&pdev->dev, "USBSSP-XHCI physical base addr: %pa\n",
+			&res[RES_HOST_ID].start);
+
+		/* Interrupt for XHCI */
+		wrap->dev_res[RES_IRQ_HOST_ID].start = pdev->irq;
+		wrap->dev_res[RES_IRQ_HOST_ID].name = "host";
+		wrap->dev_res[RES_IRQ_HOST_ID].flags = IORESOURCE_IRQ;
+
+		/* Interrupt for device. It's the same as for HOST. */
+		wrap->dev_res[RES_IRQ_PERIPHERAL_ID].start = pdev->irq;
+		wrap->dev_res[RES_IRQ_PERIPHERAL_ID].name = "peripheral";
+		wrap->dev_res[RES_IRQ_PERIPHERAL_ID].flags = IORESOURCE_IRQ;
 	} else {
-		res = &cdnsp->otg_res;
-		res->start = pci_resource_start(pdev, PCI_BAR_OTG);
-		res->end =   pci_resource_end(pdev, PCI_BAR_OTG);
-		res->name = "otg";
-		res->flags = IORESOURCE_MEM;
-		dev_dbg(dev, "CDNSP-DRD physical base addr: %pa\n",
-			&res->start);
+		res[RES_DRD_ID].start = pci_resource_start(pdev, PCI_BAR_OTG);
+		res[RES_DRD_ID].end = pci_resource_end(pdev, PCI_BAR_OTG);
+		res[RES_DRD_ID].name = "otg";
+		res[RES_DRD_ID].flags = IORESOURCE_MEM;
+		dev_dbg(&pdev->dev, "CDNSP-DRD physical base addr: %pa\n",
+			&res[RES_DRD_ID].start);
 
 		/* Interrupt for OTG/DRD. */
-		cdnsp->otg_irq = pdev->irq;
+		wrap->dev_res[RES_IRQ_OTG_ID].start = pdev->irq;
+		wrap->dev_res[RES_IRQ_OTG_ID].name = "otg";
+		wrap->dev_res[RES_IRQ_OTG_ID].flags = IORESOURCE_IRQ;
 	}
 
-	/*
-	 * Cadence PCI based platform require some longer timeout for APB
-	 * to fixes domain clock synchronization issue after resuming
-	 * controller from L1 state.
-	 */
-	cdnsp->override_apb_timeout = CHICKEN_APB_TIMEOUT_VALUE;
-	pci_set_drvdata(pdev, cdnsp);
-
 	if (pci_is_enabled(func)) {
-		cdnsp->dev = dev;
-		cdnsp->gadget_init = cdnsp_gadget_init;
-
-		ret = cdns_init(cdnsp);
-		if (ret)
-			goto free_cdnsp;
+		/* set up platform device info */
+		pdata.override_apb_timeout = CHICKEN_APB_TIMEOUT_VALUE;
+		memset(&plat_info, 0, sizeof(plat_info));
+		plat_info.parent = &pdev->dev;
+		plat_info.fwnode = pdev->dev.fwnode;
+		plat_info.name = PLAT_DRIVER_NAME;
+		plat_info.id = pdev->devfn;
+		plat_info.res = wrap->dev_res;
+		plat_info.num_res = ARRAY_SIZE(wrap->dev_res);
+		plat_info.dma_mask = pdev->dma_mask;
+		plat_info.data = &pdata;
+		plat_info.size_data = sizeof(pdata);
+		wrap->devfn = pdev->devfn;
+		/* register platform device */
+		wrap->plat_dev = platform_device_register_full(&plat_info);
+		if (IS_ERR(wrap->plat_dev)) {
+			ret = PTR_ERR(wrap->plat_dev);
+			kfree(wrap);
+			goto put_pci;
+		}
 	}
 
-	device_wakeup_enable(&pdev->dev);
-	if (pci_dev_run_wake(pdev))
-		pm_runtime_put_noidle(&pdev->dev);
-
-	return 0;
-
-free_cdnsp:
-	if (!pci_is_enabled(func))
-		kfree(cdnsp);
-
+	pci_set_drvdata(pdev, wrap);
 put_pci:
 	pci_dev_put(func);
-
 	return ret;
 }
 
 static void cdnsp_pci_remove(struct pci_dev *pdev)
 {
-	struct cdns *cdnsp;
+	struct cdnsp_wrap *wrap;
 	struct pci_dev *func;
 
 	func = cdnsp_get_second_fun(pdev);
-	cdnsp = (struct cdns *)pci_get_drvdata(pdev);
+	wrap = pci_get_drvdata(pdev);
 
-	if (pci_dev_run_wake(pdev))
-		pm_runtime_get_noresume(&pdev->dev);
+	if (wrap->devfn == pdev->devfn)
+		platform_device_unregister(wrap->plat_dev);
 
-	if (pci_is_enabled(func)) {
-		cdns_remove(cdnsp);
-	} else {
-		kfree(cdnsp);
-	}
+	if (!pci_is_enabled(func))
+		kfree(wrap);
 
 	pci_dev_put(func);
 }
 
-static int __maybe_unused cdnsp_pci_suspend(struct device *dev)
-{
-	struct cdns *cdns = dev_get_drvdata(dev);
-
-	return cdns_suspend(cdns);
-}
-
-static int __maybe_unused cdnsp_pci_resume(struct device *dev)
-{
-	struct cdns *cdns = dev_get_drvdata(dev);
-	unsigned long flags;
-	int ret;
-
-	spin_lock_irqsave(&cdns->lock, flags);
-	ret = cdns_resume(cdns);
-	spin_unlock_irqrestore(&cdns->lock, flags);
-	cdns_set_active(cdns, 1);
-
-	return ret;
-}
-
-static const struct dev_pm_ops cdnsp_pci_pm_ops = {
-	SET_SYSTEM_SLEEP_PM_OPS(cdnsp_pci_suspend, cdnsp_pci_resume)
-};
-
 static const struct pci_device_id cdnsp_pci_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_CDNS, PCI_DEVICE_ID_CDNS_USBSSP),
 	  .class = PCI_CLASS_SERIAL_USB_DEVICE },
@@ -230,13 +202,10 @@ static const struct pci_device_id cdnsp_pci_ids[] = {
 };
 
 static struct pci_driver cdnsp_pci_driver = {
-	.name = "cdnsp-pci",
+	.name = PCI_DRIVER_NAME,
 	.id_table = cdnsp_pci_ids,
 	.probe = cdnsp_pci_probe,
 	.remove = cdnsp_pci_remove,
-	.driver = {
-		.pm = &cdnsp_pci_pm_ops,
-	}
 };
 
 module_pci_driver(cdnsp_pci_driver);
@@ -245,4 +214,4 @@ MODULE_DEVICE_TABLE(pci, cdnsp_pci_ids);
 MODULE_ALIAS("pci:cdnsp");
 MODULE_AUTHOR("Pawel Laszczak <pawell@cadence.com>");
 MODULE_LICENSE("GPL v2");
-MODULE_DESCRIPTION("Cadence CDNSP PCI driver");
+MODULE_DESCRIPTION("Cadence CDNSP PCI wrapper");
diff --git a/drivers/usb/cdns3/cdnsp-ring.c b/drivers/usb/cdns3/cdnsp-ring.c
index 0758f171f73ec..715658c981ff8 100644
--- a/drivers/usb/cdns3/cdnsp-ring.c
+++ b/drivers/usb/cdns3/cdnsp-ring.c
@@ -259,7 +259,7 @@ static bool cdnsp_room_on_ring(struct cdnsp_device *pdev,
  */
 static void cdnsp_force_l0_go(struct cdnsp_device *pdev)
 {
-	if (pdev->active_port == &pdev->usb2_port && pdev->gadget.lpm_capable)
+	if (pdev->active_port != &pdev->usb3_port && pdev->gadget.lpm_capable)
 		cdnsp_set_link_state(pdev, &pdev->active_port->regs->portsc, XDEV_U0);
 }
 
@@ -763,6 +763,8 @@ static int cdnsp_update_port_id(struct cdnsp_device *pdev, u32 port_id)
 
 	if (port_id == pdev->usb2_port.port_num) {
 		port = &pdev->usb2_port;
+	} else if (port_id == pdev->eusb_port.port_num) {
+		port = &pdev->eusb_port;
 	} else if (port_id == pdev->usb3_port.port_num) {
 		port  = &pdev->usb3_port;
 	} else {
@@ -779,7 +781,8 @@ static int cdnsp_update_port_id(struct cdnsp_device *pdev, u32 port_id)
 		cdnsp_enable_slot(pdev);
 	}
 
-	if (port_id == pdev->usb2_port.port_num)
+	if ((pdev->usb2_port.exist && port_id == pdev->usb2_port.port_num) ||
+	    (pdev->eusb_port.exist && port_id == pdev->eusb_port.port_num))
 		cdnsp_set_usb2_hardware_lpm(pdev, NULL, 1);
 	else
 		writel(PORT_U1_TIMEOUT(1) | PORT_U2_TIMEOUT(1),
@@ -808,7 +811,7 @@ static void cdnsp_handle_port_status(struct cdnsp_device *pdev,
 
 	port_regs = pdev->active_port->regs;
 
-	if (port_id == pdev->usb2_port.port_num)
+	if (port_id == pdev->usb2_port.port_num || port_id == pdev->eusb_port.port_num)
 		port2 = true;
 
 new_event:
diff --git a/drivers/usb/cdns3/core.c b/drivers/usb/cdns3/core.c
index f0e32227c0b79..6a8d1fefbc0d0 100644
--- a/drivers/usb/cdns3/core.c
+++ b/drivers/usb/cdns3/core.c
@@ -21,7 +21,6 @@
 #include <linux/pm_runtime.h>
 
 #include "core.h"
-#include "host-export.h"
 #include "drd.h"
 
 static int cdns_idle_init(struct cdns *cdns);
@@ -80,7 +79,7 @@ static void cdns_exit_roles(struct cdns *cdns)
  *
  * Returns 0 on success otherwise negative errno
  */
-static int cdns_core_init_role(struct cdns *cdns)
+int cdns_core_init_role(struct cdns *cdns)
 {
 	struct device *dev = cdns->dev;
 	enum usb_dr_mode best_dr_mode;
@@ -96,23 +95,13 @@ static int cdns_core_init_role(struct cdns *cdns)
 	 * can be restricted later depending on strap pin configuration.
 	 */
 	if (dr_mode == USB_DR_MODE_UNKNOWN) {
-		if (cdns->version == CDNSP_CONTROLLER_V2) {
-			if (IS_ENABLED(CONFIG_USB_CDNSP_HOST) &&
-			    IS_ENABLED(CONFIG_USB_CDNSP_GADGET))
-				dr_mode = USB_DR_MODE_OTG;
-			else if (IS_ENABLED(CONFIG_USB_CDNSP_HOST))
-				dr_mode = USB_DR_MODE_HOST;
-			else if (IS_ENABLED(CONFIG_USB_CDNSP_GADGET))
-				dr_mode = USB_DR_MODE_PERIPHERAL;
-		} else {
-			if (IS_ENABLED(CONFIG_USB_CDNS3_HOST) &&
-			    IS_ENABLED(CONFIG_USB_CDNS3_GADGET))
-				dr_mode = USB_DR_MODE_OTG;
-			else if (IS_ENABLED(CONFIG_USB_CDNS3_HOST))
-				dr_mode = USB_DR_MODE_HOST;
-			else if (IS_ENABLED(CONFIG_USB_CDNS3_GADGET))
-				dr_mode = USB_DR_MODE_PERIPHERAL;
-		}
+		if (IS_ENABLED(CONFIG_USB_CDNS3_HOST) &&
+		    IS_ENABLED(CONFIG_USB_CDNS3_GADGET))
+			dr_mode = USB_DR_MODE_OTG;
+		else if (IS_ENABLED(CONFIG_USB_CDNS3_HOST))
+			dr_mode = USB_DR_MODE_HOST;
+		else if (IS_ENABLED(CONFIG_USB_CDNS3_GADGET))
+			dr_mode = USB_DR_MODE_PERIPHERAL;
 	}
 
 	/*
@@ -137,11 +126,8 @@ static int cdns_core_init_role(struct cdns *cdns)
 	dr_mode = best_dr_mode;
 
 	if (dr_mode == USB_DR_MODE_OTG || dr_mode == USB_DR_MODE_HOST) {
-		if ((cdns->version == CDNSP_CONTROLLER_V2 &&
-		     IS_ENABLED(CONFIG_USB_CDNSP_HOST)) ||
-		    (cdns->version < CDNSP_CONTROLLER_V2 &&
-		     IS_ENABLED(CONFIG_USB_CDNS3_HOST)))
-			ret = cdns_host_init(cdns);
+		if (cdns->host_init)
+			ret = cdns->host_init(cdns);
 		else
 			ret = -ENXIO;
 
@@ -197,11 +183,14 @@ static int cdns_core_init_role(struct cdns *cdns)
 		goto err;
 	}
 
+	dev_dbg(dev, "Cadence USB3 core: probe succeed\n");
+
 	return 0;
 err:
 	cdns_exit_roles(cdns);
 	return ret;
 }
+EXPORT_SYMBOL_GPL(cdns_core_init_role);
 
 /**
  * cdns_hw_role_state_machine  - role switch state machine based on hw events.
@@ -469,14 +458,8 @@ int cdns_init(struct cdns *cdns)
 	if (ret)
 		goto init_failed;
 
-	ret = cdns_core_init_role(cdns);
-	if (ret)
-		goto init_failed;
-
 	spin_lock_init(&cdns->lock);
 
-	dev_dbg(dev, "Cadence USB3 core: probe succeed\n");
-
 	return 0;
 init_failed:
 	cdns_drd_exit(cdns);
@@ -576,5 +559,5 @@ EXPORT_SYMBOL_GPL(cdns_set_active);
 MODULE_AUTHOR("Peter Chen <peter.chen@nxp.com>");
 MODULE_AUTHOR("Pawel Laszczak <pawell@cadence.com>");
 MODULE_AUTHOR("Roger Quadros <rogerq@ti.com>");
-MODULE_DESCRIPTION("Cadence USBSS and USBSSP DRD Driver");
+MODULE_DESCRIPTION("Cadence USBSS/USBSSP DRD driver (core, DRD, platform, optional host/gadget)");
 MODULE_LICENSE("GPL");
diff --git a/drivers/usb/cdns3/core.h b/drivers/usb/cdns3/core.h
index 801be9e61340e..bca973b999a4d 100644
--- a/drivers/usb/cdns3/core.h
+++ b/drivers/usb/cdns3/core.h
@@ -45,6 +45,7 @@ struct cdns3_platform_data {
 	unsigned long quirks;
 #define CDNS3_DEFAULT_PM_RUNTIME_ALLOW	BIT(0)
 #define CDNS3_DRD_SUSPEND_RESIDENCY_ENABLE	BIT(1)
+	u32 override_apb_timeout;	/* 0 = use default (e.g. for PCI) */
 };
 
 /**
@@ -82,6 +83,7 @@ struct cdns3_platform_data {
  * @override_apb_timeout: hold value of APB timeout. For value 0 the default
  *                        value in CHICKEN_BITS_3 will be preserved.
  * @gadget_init: pointer to gadget initialization function
+ * @host_init: pointer to host initialization function
  */
 struct cdns {
 	struct device			*dev;
@@ -120,13 +122,14 @@ struct cdns {
 	spinlock_t			lock;
 	struct xhci_plat_priv		*xhci_plat_data;
 	u32                             override_apb_timeout;
-
 	int (*gadget_init)(struct cdns *cdns);
+	int (*host_init)(struct cdns *cdns);
 };
 
 int cdns_hw_role_switch(struct cdns *cdns);
 int cdns_init(struct cdns *cdns);
 int cdns_remove(struct cdns *cdns);
+int cdns_core_init_role(struct cdns *cdns);
 
 #ifdef CONFIG_PM_SLEEP
 int cdns_resume(struct cdns *cdns);
diff --git a/drivers/usb/cdns3/gadget-export.h b/drivers/usb/cdns3/gadget-export.h
index c37b6269b0010..60c3177db62c1 100644
--- a/drivers/usb/cdns3/gadget-export.h
+++ b/drivers/usb/cdns3/gadget-export.h
@@ -10,9 +10,10 @@
 #ifndef __LINUX_CDNS3_GADGET_EXPORT
 #define __LINUX_CDNS3_GADGET_EXPORT
 
-#if IS_ENABLED(CONFIG_USB_CDNSP_GADGET)
+#if IS_ENABLED(CONFIG_USB_CDNS3_GADGET)
 
 int cdnsp_gadget_init(struct cdns *cdns);
+int cdns3_gadget_init(struct cdns *cdns);
 #else
 
 static inline int cdnsp_gadget_init(struct cdns *cdns)
@@ -20,13 +21,6 @@ static inline int cdnsp_gadget_init(struct cdns *cdns)
 	return -ENXIO;
 }
 
-#endif /* CONFIG_USB_CDNSP_GADGET */
-
-#if IS_ENABLED(CONFIG_USB_CDNS3_GADGET)
-
-int cdns3_gadget_init(struct cdns *cdns);
-#else
-
 static inline int cdns3_gadget_init(struct cdns *cdns)
 {
 	return -ENXIO;
diff --git a/drivers/usb/cdns3/host-export.h b/drivers/usb/cdns3/host-export.h
index cf92173ecf008..34fd1f1ad59da 100644
--- a/drivers/usb/cdns3/host-export.h
+++ b/drivers/usb/cdns3/host-export.h
@@ -9,7 +9,7 @@
 #ifndef __LINUX_CDNS3_HOST_EXPORT
 #define __LINUX_CDNS3_HOST_EXPORT
 
-#if IS_ENABLED(CONFIG_USB_CDNS_HOST)
+#if IS_ENABLED(CONFIG_USB_CDNS3_HOST)
 
 int cdns_host_init(struct cdns *cdns);
 
@@ -22,6 +22,6 @@ static inline int cdns_host_init(struct cdns *cdns)
 
 static inline void cdns_host_exit(struct cdns *cdns) { }
 
-#endif /* USB_CDNS_HOST */
+#endif /* CONFIG_USB_CDNS3_HOST */
 
 #endif /* __LINUX_CDNS3_HOST_EXPORT */
diff --git a/drivers/usb/core/config.c b/drivers/usb/core/config.c
index 417140b012bb9..45e20c6d76c04 100644
--- a/drivers/usb/core/config.c
+++ b/drivers/usb/core/config.c
@@ -56,8 +56,7 @@ static void usb_parse_ssp_isoc_endpoint_companion(struct device *ddev,
 	desc = (struct usb_ssp_isoc_ep_comp_descriptor *) buffer;
 	if (size < USB_DT_SSP_ISOC_EP_COMP_SIZE ||
 	    desc->bDescriptorType != USB_DT_SSP_ISOC_ENDPOINT_COMP) {
-		dev_notice(ddev, "Invalid SuperSpeedPlus isoc endpoint companion"
-			 "for config %d interface %d altsetting %d ep %d.\n",
+		dev_notice(ddev, "Invalid SuperSpeedPlus isoc endpoint companion for config %d interface %d altsetting %d ep 0x%X.\n",
 			 cfgno, inum, asnum, ep->desc.bEndpointAddress);
 		return;
 	}
@@ -91,7 +90,7 @@ static void usb_parse_eusb2_isoc_endpoint_companion(struct device *ddev,
 		size -= h->bLength;
 	}
 
-	dev_notice(ddev, "No eUSB2 isoc ep %d companion for config %d interface %d altsetting %d\n",
+	dev_notice(ddev, "No eUSB2 isoc ep 0x%X companion for config %d interface %d altsetting %d\n",
 		   ep->desc.bEndpointAddress, cfgno, inum, asnum);
 }
 
@@ -115,9 +114,7 @@ static void usb_parse_ss_endpoint_companion(struct device *ddev, int cfgno,
 	}
 
 	if (desc->bDescriptorType != USB_DT_SS_ENDPOINT_COMP) {
-		dev_notice(ddev, "No SuperSpeed endpoint companion for config %d "
-				" interface %d altsetting %d ep %d: "
-				"using minimum values\n",
+		dev_notice(ddev, "No SuperSpeed endpoint companion for config %d interface %d altsetting %d ep 0x%X: using minimum values\n",
 				cfgno, inum, asnum, ep->desc.bEndpointAddress);
 
 		/* Fill in some default values.
@@ -141,42 +138,32 @@ static void usb_parse_ss_endpoint_companion(struct device *ddev, int cfgno,
 
 	/* Check the various values */
 	if (usb_endpoint_xfer_control(&ep->desc) && desc->bMaxBurst != 0) {
-		dev_notice(ddev, "Control endpoint with bMaxBurst = %d in "
-				"config %d interface %d altsetting %d ep %d: "
-				"setting to zero\n", desc->bMaxBurst,
-				cfgno, inum, asnum, ep->desc.bEndpointAddress);
+		dev_notice(ddev, "Control endpoint with bMaxBurst = %d in config %d interface %d altsetting %d ep 0x%X: setting to zero\n",
+				desc->bMaxBurst, cfgno, inum, asnum, ep->desc.bEndpointAddress);
 		ep->ss_ep_comp.bMaxBurst = 0;
 	} else if (desc->bMaxBurst > 15) {
-		dev_notice(ddev, "Endpoint with bMaxBurst = %d in "
-				"config %d interface %d altsetting %d ep %d: "
-				"setting to 15\n", desc->bMaxBurst,
-				cfgno, inum, asnum, ep->desc.bEndpointAddress);
+		dev_notice(ddev, "Endpoint with bMaxBurst = %d in config %d interface %d altsetting %d ep 0x%X: setting to 15\n",
+				desc->bMaxBurst, cfgno, inum, asnum, ep->desc.bEndpointAddress);
 		ep->ss_ep_comp.bMaxBurst = 15;
 	}
 
 	if ((usb_endpoint_xfer_control(&ep->desc) ||
 			usb_endpoint_xfer_int(&ep->desc)) &&
 				desc->bmAttributes != 0) {
-		dev_notice(ddev, "%s endpoint with bmAttributes = %d in "
-				"config %d interface %d altsetting %d ep %d: "
-				"setting to zero\n",
+		dev_notice(ddev, "%s endpoint with bmAttributes = %d in config %d interface %d altsetting %d ep 0x%X: setting to zero\n",
 				usb_endpoint_xfer_control(&ep->desc) ? "Control" : "Bulk",
 				desc->bmAttributes,
 				cfgno, inum, asnum, ep->desc.bEndpointAddress);
 		ep->ss_ep_comp.bmAttributes = 0;
 	} else if (usb_endpoint_xfer_bulk(&ep->desc) &&
 			desc->bmAttributes > 16) {
-		dev_notice(ddev, "Bulk endpoint with more than 65536 streams in "
-				"config %d interface %d altsetting %d ep %d: "
-				"setting to max\n",
+		dev_notice(ddev, "Bulk endpoint with more than 65536 streams in config %d interface %d altsetting %d ep 0x%X: setting to max\n",
 				cfgno, inum, asnum, ep->desc.bEndpointAddress);
 		ep->ss_ep_comp.bmAttributes = 16;
 	} else if (usb_endpoint_xfer_isoc(&ep->desc) &&
 		   !USB_SS_SSP_ISOC_COMP(desc->bmAttributes) &&
 		   USB_SS_MULT(desc->bmAttributes) > 3) {
-		dev_notice(ddev, "Isoc endpoint has Mult of %d in "
-				"config %d interface %d altsetting %d ep %d: "
-				"setting to 3\n",
+		dev_notice(ddev, "Isoc endpoint has Mult of %d in config %d interface %d altsetting %d ep 0x%X: setting to 3\n",
 				USB_SS_MULT(desc->bmAttributes),
 				cfgno, inum, asnum, ep->desc.bEndpointAddress);
 		ep->ss_ep_comp.bmAttributes = 2;
@@ -191,10 +178,15 @@ static void usb_parse_ss_endpoint_companion(struct device *ddev, int cfgno,
 			(desc->bMaxBurst + 1);
 	else
 		max_tx = 999999;
-	if (le16_to_cpu(desc->wBytesPerInterval) > max_tx) {
-		dev_notice(ddev, "%s endpoint with wBytesPerInterval of %d in "
-				"config %d interface %d altsetting %d ep %d: "
-				"setting to %d\n",
+	/*
+	 * wBytesPerInterval > max_tx is bogus, but USB3 spec doesn't forbid the opposite.
+	 * Experience shows that wBytesPerInterval < wMaxPacketSize on common interrupt IN
+	 * endpoints is usually bogus too, and recent HCs enforce interrupt BW limits.
+	 */
+	if (le16_to_cpu(desc->wBytesPerInterval) > max_tx ||
+	    (le16_to_cpu(desc->wBytesPerInterval) < usb_endpoint_maxp(&ep->desc) &&
+	     usb_endpoint_is_int_in(&ep->desc))) {
+		dev_notice(ddev, "%s endpoint with wBytesPerInterval of %d in config %d interface %d altsetting %d ep 0x%X: setting to %d\n",
 				usb_endpoint_xfer_isoc(&ep->desc) ? "Isoc" : "Int",
 				le16_to_cpu(desc->wBytesPerInterval),
 				cfgno, inum, asnum, ep->desc.bEndpointAddress,
diff --git a/drivers/usb/core/hcd.c b/drivers/usb/core/hcd.c
index 89221f1ce7694..b181b43a35dc0 100644
--- a/drivers/usb/core/hcd.c
+++ b/drivers/usb/core/hcd.c
@@ -328,9 +328,7 @@ static const u8 ss_rh_config_descriptor[] = {
 	USB_DT_ENDPOINT, /* __u8 ep_bDescriptorType; Endpoint */
 	0x81,       /*  __u8  ep_bEndpointAddress; IN Endpoint 1 */
 	0x03,       /*  __u8  ep_bmAttributes; Interrupt */
-		    /* __le16 ep_wMaxPacketSize; 1 + (MAX_ROOT_PORTS / 8)
-		     * see hub.c:hub_configure() for details. */
-	(USB_MAXCHILDREN + 1 + 7) / 8, 0x00,
+	0x02, 0x00, /* __le16 ep_wMaxPacketSize; 2 bytes per USB3 10.15.1 */
 	0x0c,       /*  __u8  ep_bInterval; (256ms -- usb 2.0 spec) */
 
 	/* one SuperSpeed endpoint companion descriptor */
diff --git a/drivers/usb/gadget/composite.c b/drivers/usb/gadget/composite.c
index a902184bdf828..dc36643745963 100644
--- a/drivers/usb/gadget/composite.c
+++ b/drivers/usb/gadget/composite.c
@@ -2172,7 +2172,10 @@ composite_setup(struct usb_gadget *gadget, const struct usb_ctrlrequest *ctrl)
 				sizeof(url_descriptor->URL)
 				- WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH + landing_page_offset);
 
-			if (w_length < WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH + landing_page_length)
+			if (w_length < WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH)
+				landing_page_length = landing_page_offset;
+			else if (w_length <
+				 WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH + landing_page_length)
 				landing_page_length = w_length
 				- WEBUSB_URL_DESCRIPTOR_HEADER_LENGTH + landing_page_offset;
 
diff --git a/drivers/usb/gadget/udc/aspeed-vhub/dev.c b/drivers/usb/gadget/udc/aspeed-vhub/dev.c
index 2ecd049dacc2d..8b9449d16324e 100644
--- a/drivers/usb/gadget/udc/aspeed-vhub/dev.c
+++ b/drivers/usb/gadget/udc/aspeed-vhub/dev.c
@@ -593,7 +593,7 @@ int ast_vhub_init_dev(struct ast_vhub *vhub, unsigned int idx)
 		d->gadget.max_speed = USB_SPEED_HIGH;
 	d->gadget.speed = USB_SPEED_UNKNOWN;
 	d->gadget.dev.of_node = vhub->pdev->dev.of_node;
-	d->gadget.dev.of_node_reused = true;
+	dev_set_of_node_reused(&d->gadget.dev);
 
 	rc = usb_add_gadget_udc(d->port_dev, &d->gadget);
 	if (rc != 0)
diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c
index e8861eaad9077..60340ff9edbf0 100644
--- a/drivers/usb/gadget/udc/core.c
+++ b/drivers/usb/gadget/udc/core.c
@@ -712,6 +712,9 @@ static int usb_gadget_connect_locked(struct usb_gadget *gadget)
 		goto out;
 	}
 
+	if (gadget->connected)
+		goto out;
+
 	if (gadget->deactivated || !gadget->udc->allow_connect || !gadget->udc->started) {
 		/*
 		 * If the gadget isn't usable (because it is deactivated,
@@ -885,8 +888,10 @@ int usb_gadget_activate(struct usb_gadget *gadget)
 	 * If gadget has been connected before deactivation, or became connected
 	 * while it was being deactivated, we call usb_gadget_connect().
 	 */
-	if (gadget->connected)
+	if (gadget->connected) {
+		gadget->connected = false;
 		ret = usb_gadget_connect_locked(gadget);
+	}
 
 unlock:
 	mutex_unlock(&gadget->udc->connect_lock);
diff --git a/drivers/usb/gadget/udc/pxa25x_udc.c b/drivers/usb/gadget/udc/pxa25x_udc.c
index b3d58d7c3a774..594d671937634 100644
--- a/drivers/usb/gadget/udc/pxa25x_udc.c
+++ b/drivers/usb/gadget/udc/pxa25x_udc.c
@@ -12,7 +12,7 @@
 /* #define VERBOSE_DEBUG */
 
 #include <linux/device.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/ioport.h>
@@ -261,24 +261,12 @@ static void nuke (struct pxa25x_ep *, int status);
 /* one GPIO should control a D+ pullup, so host sees this device (or not) */
 static void pullup_off(void)
 {
-	struct pxa2xx_udc_mach_info		*mach = the_controller->mach;
-	int off_level = mach->gpio_pullup_inverted;
-
-	if (gpio_is_valid(mach->gpio_pullup))
-		gpio_set_value(mach->gpio_pullup, off_level);
-	else if (mach->udc_command)
-		mach->udc_command(PXA2XX_UDC_CMD_DISCONNECT);
+	gpiod_set_value(the_controller->pullup_gpio, 0);
 }
 
 static void pullup_on(void)
 {
-	struct pxa2xx_udc_mach_info		*mach = the_controller->mach;
-	int on_level = !mach->gpio_pullup_inverted;
-
-	if (gpio_is_valid(mach->gpio_pullup))
-		gpio_set_value(mach->gpio_pullup, on_level);
-	else if (mach->udc_command)
-		mach->udc_command(PXA2XX_UDC_CMD_CONNECT);
+	gpiod_set_value(the_controller->pullup_gpio, 1);
 }
 
 #if defined(CONFIG_CPU_BIG_ENDIAN)
@@ -1190,8 +1178,7 @@ static int pxa25x_udc_pullup(struct usb_gadget *_gadget, int is_active)
 
 	udc = container_of(_gadget, struct pxa25x_udc, gadget);
 
-	/* not all boards support pullup control */
-	if (!gpio_is_valid(udc->mach->gpio_pullup) && !udc->mach->udc_command)
+	if (!udc->pullup_gpio)
 		return -EOPNOTSUPP;
 
 	udc->pullup = (is_active != 0);
@@ -2343,19 +2330,17 @@ static int pxa25x_udc_probe(struct platform_device *pdev)
 
 	/* other non-static parts of init */
 	dev->dev = &pdev->dev;
-	dev->mach = dev_get_platdata(&pdev->dev);
 
 	dev->transceiver = devm_usb_get_phy(&pdev->dev, USB_PHY_TYPE_USB2);
 
-	if (gpio_is_valid(dev->mach->gpio_pullup)) {
-		retval = devm_gpio_request_one(&pdev->dev, dev->mach->gpio_pullup,
-					       GPIOF_OUT_INIT_LOW, "pca25x_udc GPIO PULLUP");
-		if (retval) {
-			dev_dbg(&pdev->dev,
-				"can't get pullup gpio %d, err: %d\n",
-				dev->mach->gpio_pullup, retval);
-			goto err;
-		}
+	dev->pullup_gpio = devm_gpiod_get_index_optional(&pdev->dev, "pullup", 0,
+						    GPIOD_OUT_HIGH);
+	if (IS_ERR(dev->pullup_gpio)) {
+		dev_dbg(&pdev->dev,
+			"can't get pullup gpio err: %ld\n",
+			PTR_ERR(dev->pullup_gpio));
+		retval = PTR_ERR(dev->pullup_gpio);
+		goto err;
 	}
 
 	timer_setup(&dev->timer, udc_watchdog, 0);
@@ -2439,7 +2424,7 @@ static int pxa25x_udc_suspend(struct platform_device *dev, pm_message_t state)
 	struct pxa25x_udc	*udc = platform_get_drvdata(dev);
 	unsigned long flags;
 
-	if (!gpio_is_valid(udc->mach->gpio_pullup) && !udc->mach->udc_command)
+	if (!udc->pullup_gpio)
 		WARNING("USB host won't detect disconnect!\n");
 	udc->suspended = 1;
 
diff --git a/drivers/usb/gadget/udc/pxa25x_udc.h b/drivers/usb/gadget/udc/pxa25x_udc.h
index 6ab6047edc834..3452cf54286c8 100644
--- a/drivers/usb/gadget/udc/pxa25x_udc.h
+++ b/drivers/usb/gadget/udc/pxa25x_udc.h
@@ -112,7 +112,7 @@ struct pxa25x_udc {
 
 	struct device				*dev;
 	struct clk				*clk;
-	struct pxa2xx_udc_mach_info		*mach;
+	struct gpio_desc			*pullup_gpio;
 	struct usb_phy				*transceiver;
 	u64					dma_mask;
 	struct pxa25x_ep			ep [PXA_UDC_NUM_ENDPOINTS];
diff --git a/drivers/usb/gadget/udc/pxa27x_udc.c b/drivers/usb/gadget/udc/pxa27x_udc.c
index 1abea0d48c35d..640f81988c041 100644
--- a/drivers/usb/gadget/udc/pxa27x_udc.c
+++ b/drivers/usb/gadget/udc/pxa27x_udc.c
@@ -17,7 +17,6 @@
 #include <linux/proc_fs.h>
 #include <linux/clk.h>
 #include <linux/irq.h>
-#include <linux/gpio.h>
 #include <linux/gpio/consumer.h>
 #include <linux/slab.h>
 #include <linux/string_choices.h>
@@ -1423,14 +1422,7 @@ static const struct usb_ep_ops pxa_ep_ops = {
  */
 static void dplus_pullup(struct pxa_udc *udc, int on)
 {
-	if (udc->gpiod) {
-		gpiod_set_value(udc->gpiod, on);
-	} else if (udc->udc_command) {
-		if (on)
-			udc->udc_command(PXA2XX_UDC_CMD_CONNECT);
-		else
-			udc->udc_command(PXA2XX_UDC_CMD_DISCONNECT);
-	}
+	gpiod_set_value(udc->gpiod, on);
 	udc->pullup_on = on;
 }
 
@@ -1521,7 +1513,7 @@ static int pxa_udc_pullup(struct usb_gadget *_gadget, int is_active)
 	struct pxa_udc *udc = to_gadget_udc(_gadget);
 	int ret;
 
-	if (!udc->gpiod && !udc->udc_command)
+	if (!udc->gpiod)
 		return -EOPNOTSUPP;
 
 	dplus_pullup(udc, is_active);
@@ -2380,26 +2372,11 @@ MODULE_DEVICE_TABLE(of, udc_pxa_dt_ids);
 static int pxa_udc_probe(struct platform_device *pdev)
 {
 	struct pxa_udc *udc = &memory;
-	int retval = 0, gpio;
-	struct pxa2xx_udc_mach_info *mach = dev_get_platdata(&pdev->dev);
-
-	if (mach) {
-		gpio = mach->gpio_pullup;
-		if (gpio_is_valid(gpio)) {
-			retval = devm_gpio_request_one(&pdev->dev, gpio,
-						       GPIOF_OUT_INIT_LOW,
-						       "USB D+ pullup");
-			if (retval)
-				return retval;
-			udc->gpiod = gpio_to_desc(mach->gpio_pullup);
-
-			if (mach->gpio_pullup_inverted ^ gpiod_is_active_low(udc->gpiod))
-				gpiod_toggle_active_low(udc->gpiod);
-		}
-		udc->udc_command = mach->udc_command;
-	} else {
-		udc->gpiod = devm_gpiod_get(&pdev->dev, NULL, GPIOD_ASIS);
-	}
+	int retval = 0;
+
+	udc->gpiod = devm_gpiod_get(&pdev->dev, NULL, GPIOD_ASIS);
+	if (IS_ERR(udc->gpiod))
+		return PTR_ERR(udc->gpiod);
 
 	udc->regs = devm_platform_ioremap_resource(pdev, 0);
 	if (IS_ERR(udc->regs))
diff --git a/drivers/usb/gadget/udc/pxa27x_udc.h b/drivers/usb/gadget/udc/pxa27x_udc.h
index 31bf79ce931c6..2c28b691010a4 100644
--- a/drivers/usb/gadget/udc/pxa27x_udc.h
+++ b/drivers/usb/gadget/udc/pxa27x_udc.h
@@ -426,7 +426,6 @@ struct udc_stats {
  * @usb_gadget: udc gadget structure
  * @driver: bound gadget (zero, g_ether, g_mass_storage, ...)
  * @dev: device
- * @udc_command: machine specific function to activate D+ pullup
  * @gpiod: gpio descriptor of gpio for D+ pullup (or NULL if none)
  * @transceiver: external transceiver to handle vbus sense and D+ pullup
  * @ep0state: control endpoint state machine state
@@ -452,7 +451,6 @@ struct pxa_udc {
 	struct usb_gadget			gadget;
 	struct usb_gadget_driver		*driver;
 	struct device				*dev;
-	void					(*udc_command)(int);
 	struct gpio_desc			*gpiod;
 	struct usb_phy				*transceiver;
 
diff --git a/drivers/usb/misc/Kconfig b/drivers/usb/misc/Kconfig
index 0b56b773dbdf7..91b62acb537b3 100644
--- a/drivers/usb/misc/Kconfig
+++ b/drivers/usb/misc/Kconfig
@@ -145,7 +145,7 @@ config USB_APPLEDISPLAY
 	  Displays over USB. This driver provides a sysfs interface.
 
 config USB_QCOM_EUD
-	tristate "QCOM Embedded USB Debugger(EUD) Driver"
+	tristate "Qualcomm Embedded USB Debugger(EUD) Driver"
 	depends on ARCH_QCOM || COMPILE_TEST
 	select QCOM_SCM
 	select USB_ROLE_SWITCH
diff --git a/drivers/usb/misc/onboard_usb_dev.c b/drivers/usb/misc/onboard_usb_dev.c
index 7cdbdfe07a763..11508ed4df251 100644
--- a/drivers/usb/misc/onboard_usb_dev.c
+++ b/drivers/usb/misc/onboard_usb_dev.c
@@ -665,6 +665,7 @@ static const struct usb_device_id onboard_dev_id_table[] = {
 	{ USB_DEVICE(VENDOR_ID_GENESYS, 0x0608) }, /* Genesys Logic GL850G USB 2.0 HUB */
 	{ USB_DEVICE(VENDOR_ID_GENESYS, 0x0610) }, /* Genesys Logic GL852G USB 2.0 HUB */
 	{ USB_DEVICE(VENDOR_ID_GENESYS, 0x0620) }, /* Genesys Logic GL3523 USB 3.1 HUB */
+	{ USB_DEVICE(VENDOR_ID_GENESYS, 0x0625) }, /* Genesys Logic GL3590 USB 3.2 HUB */
 	{ USB_DEVICE(VENDOR_ID_MICROCHIP, 0x2412) }, /* USB2412 USB 2.0 HUB */
 	{ USB_DEVICE(VENDOR_ID_MICROCHIP, 0x2514) }, /* USB2514B USB 2.0 HUB */
 	{ USB_DEVICE(VENDOR_ID_MICROCHIP, 0x2517) }, /* USB2517 USB 2.0 HUB */
diff --git a/drivers/usb/misc/onboard_usb_dev.h b/drivers/usb/misc/onboard_usb_dev.h
index ac1aa3e122ad9..3523f8f8a1496 100644
--- a/drivers/usb/misc/onboard_usb_dev.h
+++ b/drivers/usb/misc/onboard_usb_dev.h
@@ -101,6 +101,13 @@ static const struct onboard_dev_pdata genesys_gl852g_data = {
 	.is_hub = true,
 };
 
+static const struct onboard_dev_pdata genesys_gl3590_data = {
+	.reset_us = 50,
+	.num_supplies = 2,
+	.supply_names = { "vdd", "vdd12" },
+	.is_hub = true,
+};
+
 static const struct onboard_dev_pdata usb_a_conn_data = {
 	.num_supplies = 1,
 	.supply_names = { "vbus" },
@@ -146,6 +153,7 @@ static const struct of_device_id onboard_dev_match[] = {
 	{ .compatible = "usb5e3,608", .data = &genesys_gl850g_data, },
 	{ .compatible = "usb5e3,610", .data = &genesys_gl852g_data, },
 	{ .compatible = "usb5e3,620", .data = &genesys_gl852g_data, },
+	{ .compatible = "usb5e3,625", .data = &genesys_gl3590_data, },
 	{ .compatible = "usb5e3,626", .data = &genesys_gl852g_data, },
 	{ .compatible = "usbbda,179", .data = &realtek_rtl8188etv_data, },
 	{ .compatible = "usbbda,411", .data = &realtek_rts5411_data, },
diff --git a/drivers/usb/serial/ark3116.c b/drivers/usb/serial/ark3116.c
index d974da43fba32..3a574b6ee0c4d 100644
--- a/drivers/usb/serial/ark3116.c
+++ b/drivers/usb/serial/ark3116.c
@@ -28,7 +28,6 @@
 #include <linux/usb/serial.h>
 #include <linux/serial.h>
 #include <linux/serial_reg.h>
-#include <linux/uaccess.h>
 #include <linux/mutex.h>
 #include <linux/spinlock.h>
 
diff --git a/drivers/usb/serial/belkin_sa.c b/drivers/usb/serial/belkin_sa.c
index 38ac910b1082d..cfff292631aa3 100644
--- a/drivers/usb/serial/belkin_sa.c
+++ b/drivers/usb/serial/belkin_sa.c
@@ -23,11 +23,9 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include "belkin_sa.h"
@@ -194,6 +192,9 @@ static void belkin_sa_read_int_callback(struct urb *urb)
 
 	usb_serial_debug_data(&port->dev, __func__, urb->actual_length, data);
 
+	if (urb->actual_length < BELKIN_SA_MSR_INDEX + 1)
+		goto exit;
+
 	/* Handle known interrupt data */
 	/* ignore data[0] and data[1] */
 
diff --git a/drivers/usb/serial/cyberjack.c b/drivers/usb/serial/cyberjack.c
index 4e8ceb23c27d7..57e20eb02a461 100644
--- a/drivers/usb/serial/cyberjack.c
+++ b/drivers/usb/serial/cyberjack.c
@@ -28,11 +28,9 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 
diff --git a/drivers/usb/serial/cypress_m8.c b/drivers/usb/serial/cypress_m8.c
index afff1a0f4298b..1d61968ab69a6 100644
--- a/drivers/usb/serial/cypress_m8.c
+++ b/drivers/usb/serial/cypress_m8.c
@@ -25,7 +25,6 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
@@ -35,7 +34,6 @@
 #include <linux/serial.h>
 #include <linux/kfifo.h>
 #include <linux/delay.h>
-#include <linux/uaccess.h>
 #include <linux/unaligned.h>
 
 #include "cypress_m8.h"
diff --git a/drivers/usb/serial/digi_acceleport.c b/drivers/usb/serial/digi_acceleport.c
index d515df045c4c8..6899aebfd6ae9 100644
--- a/drivers/usb/serial/digi_acceleport.c
+++ b/drivers/usb/serial/digi_acceleport.c
@@ -15,11 +15,9 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/wait.h>
 #include <linux/sched/signal.h>
@@ -1229,15 +1227,34 @@ static int digi_port_init(struct usb_serial_port *port, unsigned port_num)
 static int digi_startup(struct usb_serial *serial)
 {
 	struct digi_serial *serial_priv;
+	int oob_port_num;
 	int ret;
+	int i;
+
+	/*
+	 * The port bulk-out buffers must be large enough for header and
+	 * buffered data.
+	 */
+	for (i = 0; i < serial->type->num_ports; i++) {
+		if (serial->port[i]->bulk_out_size < DIGI_OUT_BUF_SIZE + 2)
+			return -EINVAL;
+	}
+
+	/*
+	 * The OOB port bulk-out buffer must be large enough for the two
+	 * commands in digi_set_modem_signals().
+	 */
+	oob_port_num = serial->type->num_ports;
+	if (serial->port[oob_port_num]->bulk_out_size < 8)
+		return -EINVAL;
 
 	serial_priv = kzalloc_obj(*serial_priv);
 	if (!serial_priv)
 		return -ENOMEM;
 
 	spin_lock_init(&serial_priv->ds_serial_lock);
-	serial_priv->ds_oob_port_num = serial->type->num_ports;
-	serial_priv->ds_oob_port = serial->port[serial_priv->ds_oob_port_num];
+	serial_priv->ds_oob_port_num = oob_port_num;
+	serial_priv->ds_oob_port = serial->port[oob_port_num];
 
 	ret = digi_port_init(serial_priv->ds_oob_port,
 						serial_priv->ds_oob_port_num);
diff --git a/drivers/usb/serial/empeg.c b/drivers/usb/serial/empeg.c
index aedcf7ebd2693..4937575caff17 100644
--- a/drivers/usb/serial/empeg.c
+++ b/drivers/usb/serial/empeg.c
@@ -16,11 +16,8 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
-#include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 
diff --git a/drivers/usb/serial/f81232.c b/drivers/usb/serial/f81232.c
index 9262a2ac97f58..dcceb95d7d99f 100644
--- a/drivers/usb/serial/f81232.c
+++ b/drivers/usb/serial/f81232.c
@@ -11,13 +11,10 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/serial.h>
 #include <linux/module.h>
-#include <linux/moduleparam.h>
 #include <linux/mutex.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/serial_reg.h>
diff --git a/drivers/usb/serial/f81534.c b/drivers/usb/serial/f81534.c
index 685930ac83833..0037767e4bbf7 100644
--- a/drivers/usb/serial/f81534.c
+++ b/drivers/usb/serial/f81534.c
@@ -28,7 +28,6 @@
 #include <linux/usb/serial.h>
 #include <linux/serial_reg.h>
 #include <linux/module.h>
-#include <linux/uaccess.h>
 
 /* Serial Port register Address */
 #define F81534_UART_BASE_ADDRESS	0x1200
diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c
index af14548fa03d9..88dd32da82c2b 100644
--- a/drivers/usb/serial/ftdi_sio.c
+++ b/drivers/usb/serial/ftdi_sio.c
@@ -31,7 +31,6 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
diff --git a/drivers/usb/serial/garmin_gps.c b/drivers/usb/serial/garmin_gps.c
index 7205483a0115c..8020149f5658d 100644
--- a/drivers/usb/serial/garmin_gps.c
+++ b/drivers/usb/serial/garmin_gps.c
@@ -15,12 +15,9 @@
 #include <linux/slab.h>
 #include <linux/timer.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
-#include <linux/atomic.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 
diff --git a/drivers/usb/serial/generic.c b/drivers/usb/serial/generic.c
index 757f0a586ddb1..6eaf74930aa3f 100644
--- a/drivers/usb/serial/generic.c
+++ b/drivers/usb/serial/generic.c
@@ -17,7 +17,6 @@
 #include <linux/moduleparam.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
-#include <linux/uaccess.h>
 #include <linux/kfifo.h>
 #include <linux/serial.h>
 
diff --git a/drivers/usb/serial/io_edgeport.c b/drivers/usb/serial/io_edgeport.c
index 3f5889145e519..34ccf78205371 100644
--- a/drivers/usb/serial/io_edgeport.c
+++ b/drivers/usb/serial/io_edgeport.c
@@ -25,12 +25,12 @@
  *
  */
 
+#include <linux/atomic.h>
 #include <linux/kernel.h>
 #include <linux/jiffies.h>
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
diff --git a/drivers/usb/serial/io_ti.c b/drivers/usb/serial/io_ti.c
index cb55370e036f7..28e16515c7f1a 100644
--- a/drivers/usb/serial/io_ti.c
+++ b/drivers/usb/serial/io_ti.c
@@ -18,7 +18,6 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
@@ -28,7 +27,6 @@
 #include <linux/kfifo.h>
 #include <linux/ioctl.h>
 #include <linux/firmware.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 
diff --git a/drivers/usb/serial/ipaq.c b/drivers/usb/serial/ipaq.c
index 3c6a9b9b9c2b9..c01bdbcb3f552 100644
--- a/drivers/usb/serial/ipaq.c
+++ b/drivers/usb/serial/ipaq.c
@@ -10,11 +10,8 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
-#include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 
diff --git a/drivers/usb/serial/ipw.c b/drivers/usb/serial/ipw.c
index 83709d678b3ae..5be153f2c68d0 100644
--- a/drivers/usb/serial/ipw.c
+++ b/drivers/usb/serial/ipw.c
@@ -36,12 +36,10 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
-#include <linux/uaccess.h>
 #include "usb-wwan.h"
 
 #define DRIVER_AUTHOR	"Roelf Diedericks"
diff --git a/drivers/usb/serial/ir-usb.c b/drivers/usb/serial/ir-usb.c
index 12e928d25ba1c..aac84539730de 100644
--- a/drivers/usb/serial/ir-usb.c
+++ b/drivers/usb/serial/ir-usb.c
@@ -25,11 +25,9 @@
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/usb/irda.h>
diff --git a/drivers/usb/serial/iuu_phoenix.c b/drivers/usb/serial/iuu_phoenix.c
index 0ca111b111c71..3b1f7546fb5c0 100644
--- a/drivers/usb/serial/iuu_phoenix.c
+++ b/drivers/usb/serial/iuu_phoenix.c
@@ -14,13 +14,11 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/serial.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include "iuu_phoenix.h"
diff --git a/drivers/usb/serial/keyspan.c b/drivers/usb/serial/keyspan.c
index 46448843541ae..4d3746c7a94e7 100644
--- a/drivers/usb/serial/keyspan.c
+++ b/drivers/usb/serial/keyspan.c
@@ -29,11 +29,9 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/usb/ezusb.h>
@@ -1187,6 +1185,10 @@ static void usa49wg_indat_callback(struct urb *urb)
 	len = 0;
 
 	while (i < urb->actual_length) {
+		if (urb->actual_length - i < 3) {
+			dev_warn_ratelimited(&urb->dev->dev, "malformed indat packet\n");
+			break;
+		}
 
 		/* Check port number from message */
 		if (data[i] >= serial->num_ports) {
diff --git a/drivers/usb/serial/keyspan_pda.c b/drivers/usb/serial/keyspan_pda.c
index 5036600dd334c..3b99f9676c352 100644
--- a/drivers/usb/serial/keyspan_pda.c
+++ b/drivers/usb/serial/keyspan_pda.c
@@ -15,12 +15,10 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
 #include <linux/workqueue.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/usb/ezusb.h>
diff --git a/drivers/usb/serial/kl5kusb105.c b/drivers/usb/serial/kl5kusb105.c
index ed8531a647684..e7917871ddf16 100644
--- a/drivers/usb/serial/kl5kusb105.c
+++ b/drivers/usb/serial/kl5kusb105.c
@@ -35,10 +35,8 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
-#include <linux/uaccess.h>
 #include <linux/unaligned.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
diff --git a/drivers/usb/serial/kobil_sct.c b/drivers/usb/serial/kobil_sct.c
index 6126afd67a7b0..4a622cf8e6aed 100644
--- a/drivers/usb/serial/kobil_sct.c
+++ b/drivers/usb/serial/kobil_sct.c
@@ -23,11 +23,9 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/ioctl.h>
diff --git a/drivers/usb/serial/mct_u232.c b/drivers/usb/serial/mct_u232.c
index 18844b92bd088..a6de18917ec9f 100644
--- a/drivers/usb/serial/mct_u232.c
+++ b/drivers/usb/serial/mct_u232.c
@@ -21,11 +21,9 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/unaligned.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
@@ -378,6 +376,7 @@ static int mct_u232_port_probe(struct usb_serial_port *port)
 {
 	struct usb_serial *serial = port->serial;
 	struct mct_u232_private *priv;
+	u16 pid;
 
 	/* check first to simplify error handling */
 	if (!serial->port[1] || !serial->port[1]->interrupt_in_urb) {
@@ -385,6 +384,16 @@ static int mct_u232_port_probe(struct usb_serial_port *port)
 		return -ENODEV;
 	}
 
+	/*
+	 * Compensate for a hardware bug: although the Sitecom U232-P25
+	 * device reports a maximum output packet size of 32 bytes,
+	 * it seems to be able to accept only 16 bytes (and that's what
+	 * SniffUSB says too...)
+	 */
+	pid = le16_to_cpu(serial->dev->descriptor.idProduct);
+	if (pid == MCT_U232_SITECOM_PID)
+		port->bulk_out_size = min(16, port->bulk_out_size);
+
 	priv = kzalloc_obj(*priv);
 	if (!priv)
 		return -ENOMEM;
@@ -410,7 +419,6 @@ static void mct_u232_port_remove(struct usb_serial_port *port)
 
 static int  mct_u232_open(struct tty_struct *tty, struct usb_serial_port *port)
 {
-	struct usb_serial *serial = port->serial;
 	struct mct_u232_private *priv = usb_get_serial_port_data(port);
 	int retval = 0;
 	unsigned int control_state;
@@ -418,15 +426,6 @@ static int  mct_u232_open(struct tty_struct *tty, struct usb_serial_port *port)
 	unsigned char last_lcr;
 	unsigned char last_msr;
 
-	/* Compensate for a hardware bug: although the Sitecom U232-P25
-	 * device reports a maximum output packet size of 32 bytes,
-	 * it seems to be able to accept only 16 bytes (and that's what
-	 * SniffUSB says too...)
-	 */
-	if (le16_to_cpu(serial->dev->descriptor.idProduct)
-						== MCT_U232_SITECOM_PID)
-		port->bulk_out_size = 16;
-
 	/* Do a defined restart: the normal serial device seems to
 	 * always turn on DTR and RTS here, so do the same. I'm not
 	 * sure if this is really necessary. But it should not harm
@@ -543,6 +542,11 @@ static void mct_u232_read_int_callback(struct urb *urb)
 		goto exit;
 	}
 
+	if (urb->actual_length < 2) {
+		dev_warn_ratelimited(&port->dev, "short interrupt-in packet\n");
+		goto exit;
+	}
+
 	/*
 	 * The interrupt-in pipe signals exceptional conditions (modem line
 	 * signal changes and errors). data[0] holds MSR, data[1] holds LSR.
diff --git a/drivers/usb/serial/metro-usb.c b/drivers/usb/serial/metro-usb.c
index bc834cc485501..35473544f1c83 100644
--- a/drivers/usb/serial/metro-usb.c
+++ b/drivers/usb/serial/metro-usb.c
@@ -13,11 +13,8 @@
 #include <linux/usb.h>
 #include <linux/errno.h>
 #include <linux/slab.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
-#include <linux/moduleparam.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb/serial.h>
 
 #define DRIVER_DESC "Metrologic Instruments Inc. - USB-POS driver"
diff --git a/drivers/usb/serial/mos7720.c b/drivers/usb/serial/mos7720.c
index 94459408e7fb5..104f43a8f4d81 100644
--- a/drivers/usb/serial/mos7720.c
+++ b/drivers/usb/serial/mos7720.c
@@ -17,11 +17,12 @@
  *	Copyright (C) 2000 Inside Out Networks, All rights reserved.
  *	Copyright (C) 2001-2002 Greg Kroah-Hartman <greg@kroah.com>
  */
+
+#include <linux/atomic.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
diff --git a/drivers/usb/serial/mos7840.c b/drivers/usb/serial/mos7840.c
index c0d0c24b074b0..ef803e74ab4bc 100644
--- a/drivers/usb/serial/mos7840.c
+++ b/drivers/usb/serial/mos7840.c
@@ -13,7 +13,6 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/serial.h>
diff --git a/drivers/usb/serial/mxuport.c b/drivers/usb/serial/mxuport.c
index ad5fdf55a02e1..9bdc5e2fee964 100644
--- a/drivers/usb/serial/mxuport.c
+++ b/drivers/usb/serial/mxuport.c
@@ -20,9 +20,7 @@
 #include <linux/serial_reg.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/unaligned.h>
@@ -146,11 +144,6 @@
 #define MX_WAIT_FOR_LOW_WATER		0x0040
 #define MX_WAIT_FOR_SEND_NEXT		0x0080
 
-#define MX_UPORT_2_PORT			BIT(0)
-#define MX_UPORT_4_PORT			BIT(1)
-#define MX_UPORT_8_PORT			BIT(2)
-#define MX_UPORT_16_PORT		BIT(3)
-
 /* This structure holds all of the local port information */
 struct mxuport_port {
 	u8 mcr_state;		/* Last MCR state */
@@ -159,26 +152,31 @@ struct mxuport_port {
 	spinlock_t spinlock;	/* Protects msr_state */
 };
 
+/* Encode number of ports (2..16 or undefined) */
+#define MX_PORTS_MASK			GENMASK(3, 0)
+#define MX_PORTS_OFFSET			1
+#define MX_PORTS(n)			(((n) - MX_PORTS_OFFSET) & MX_PORTS_MASK)
+
 /* Table of devices that work with this driver */
 static const struct usb_device_id mxuport_idtable[] = {
 	{ USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1250_PID),
-	  .driver_info = MX_UPORT_2_PORT },
+	  .driver_info = MX_PORTS(2) },
 	{ USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1251_PID),
-	  .driver_info = MX_UPORT_2_PORT },
+	  .driver_info = MX_PORTS(2) },
 	{ USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1410_PID),
-	  .driver_info = MX_UPORT_4_PORT },
+	  .driver_info = MX_PORTS(4) },
 	{ USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1450_PID),
-	  .driver_info = MX_UPORT_4_PORT },
+	  .driver_info = MX_PORTS(4) },
 	{ USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1451_PID),
-	  .driver_info = MX_UPORT_4_PORT },
+	  .driver_info = MX_PORTS(4) },
 	{ USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1618_PID),
-	  .driver_info = MX_UPORT_8_PORT },
+	  .driver_info = MX_PORTS(8) },
 	{ USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1658_PID),
-	  .driver_info = MX_UPORT_8_PORT },
+	  .driver_info = MX_PORTS(8) },
 	{ USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1613_PID),
-	  .driver_info = MX_UPORT_16_PORT },
+	  .driver_info = MX_PORTS(16) },
 	{ USB_DEVICE(MX_USBSERIAL_VID, MX_UPORT1653_PID),
-	  .driver_info = MX_UPORT_16_PORT },
+	  .driver_info = MX_PORTS(16) },
 	{}			/* Terminating entry */
 };
 
@@ -942,14 +940,8 @@ static int mxuport_calc_num_ports(struct usb_serial *serial,
 	int num_ports;
 	int i;
 
-	if (features & MX_UPORT_2_PORT) {
-		num_ports = 2;
-	} else if (features & MX_UPORT_4_PORT) {
-		num_ports = 4;
-	} else if (features & MX_UPORT_8_PORT) {
-		num_ports = 8;
-	} else if (features & MX_UPORT_16_PORT) {
-		num_ports = 16;
+	if (features & MX_PORTS_MASK) {
+		num_ports = (features & MX_PORTS_MASK) + MX_PORTS_OFFSET;
 	} else {
 		dev_warn(&serial->interface->dev,
 				"unknown device, assuming two ports\n");
diff --git a/drivers/usb/serial/omninet.c b/drivers/usb/serial/omninet.c
index aa1e9745f9679..1f35a7e7654fe 100644
--- a/drivers/usb/serial/omninet.c
+++ b/drivers/usb/serial/omninet.c
@@ -14,10 +14,8 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 
diff --git a/drivers/usb/serial/opticon.c b/drivers/usb/serial/opticon.c
index e2bed477ad577..152dfc901cd8f 100644
--- a/drivers/usb/serial/opticon.c
+++ b/drivers/usb/serial/opticon.c
@@ -10,14 +10,12 @@
 
 #include <linux/kernel.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/slab.h>
 #include <linux/tty_flip.h>
 #include <linux/serial.h>
 #include <linux/module.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
-#include <linux/uaccess.h>
 
 #define CONTROL_RTS			0x02
 #define RESEND_CTS_STATE	0x03
diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
index 42e4cecd28aca..504955bde306c 100644
--- a/drivers/usb/serial/option.c
+++ b/drivers/usb/serial/option.c
@@ -32,7 +32,6 @@
 #include <linux/jiffies.h>
 #include <linux/errno.h>
 #include <linux/tty.h>
-#include <linux/tty_flip.h>
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/bitops.h>
@@ -2450,6 +2449,12 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x30) },	/* MeiG Smart SRM825WN (Diag) */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x40) },	/* MeiG Smart SRM825WN (AT) */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d38, 0xff, 0xff, 0x60) },	/* MeiG Smart SRM825WN (NMEA) */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d63, 0xff, 0xff, 0x30) },	/* MeiG SRM813Q (Diag) */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d63, 0xff, 0xff, 0x40) },	/* MeiG SRM813Q (AT) */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d64, 0xff, 0xff, 0x30) },	/* MeiG SRM813Q (Diag) */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d64, 0xff, 0xff, 0x40) },	/* MeiG SRM813Q (AT) */
+	{ USB_DEVICE_AND_INTERFACE_INFO(0x2dee, 0x4d64, 0xff, 0xff, 0x60) },	/* MeiG SRM813Q (NMEA) */
+
 	{ USB_DEVICE_INTERFACE_CLASS(0x2df3, 0x9d03, 0xff) },			/* LongSung M5710 */
 	{ USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1404, 0xff) },			/* GosunCn GM500 RNDIS */
 	{ USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1405, 0xff) },			/* GosunCn GM500 MBIM */
@@ -2470,7 +2475,8 @@ static const struct usb_device_id option_ids[] = {
 	{ USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0302, 0xff) },			/* Rolling RW101R-GL (laptop MBIM) */
 	{ USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0802, 0xff),			/* Rolling RW350-GL (laptop MBIM) */
 	  .driver_info = RSVD(5) },
-	{ USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x1003, 0xff) },			/* Rolling RW135R-GL (laptop MBIM) */
+	{ USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x1003, 0xff),			/* Rolling RW135R-GL (laptop MBIM) */
+	  .driver_info = RSVD(5) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x30) },	/* NetPrisma LCUK54-WWD for Global */
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0x00, 0x40) },
 	{ USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x40) },
diff --git a/drivers/usb/serial/oti6858.c b/drivers/usb/serial/oti6858.c
index 3ef5b5d8ce1a6..c6364e8ff71f9 100644
--- a/drivers/usb/serial/oti6858.c
+++ b/drivers/usb/serial/oti6858.c
@@ -38,15 +38,12 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/serial.h>
 #include <linux/module.h>
-#include <linux/moduleparam.h>
 #include <linux/spinlock.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
-#include <linux/uaccess.h>
 #include <linux/kfifo.h>
 #include "oti6858.h"
 
diff --git a/drivers/usb/serial/pl2303.c b/drivers/usb/serial/pl2303.c
index 50dc838e81158..0bcbdcea52afe 100644
--- a/drivers/usb/serial/pl2303.c
+++ b/drivers/usb/serial/pl2303.c
@@ -15,13 +15,10 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/serial.h>
 #include <linux/module.h>
-#include <linux/moduleparam.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/unaligned.h>
diff --git a/drivers/usb/serial/qcserial.c b/drivers/usb/serial/qcserial.c
index 1a930dc668e44..105c0a0eae863 100644
--- a/drivers/usb/serial/qcserial.c
+++ b/drivers/usb/serial/qcserial.c
@@ -8,7 +8,6 @@
  */
 
 #include <linux/tty.h>
-#include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
diff --git a/drivers/usb/serial/quatech2.c b/drivers/usb/serial/quatech2.c
index b05c655b3cc4f..dc2b39810a26c 100644
--- a/drivers/usb/serial/quatech2.c
+++ b/drivers/usb/serial/quatech2.c
@@ -13,14 +13,12 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/serial.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/serial_reg.h>
-#include <linux/uaccess.h>
 
 /* default urb timeout for usb operations */
 #define QT2_USB_TIMEOUT USB_CTRL_SET_TIMEOUT
diff --git a/drivers/usb/serial/safe_serial.c b/drivers/usb/serial/safe_serial.c
index 238b54993446c..a82b6453d79c0 100644
--- a/drivers/usb/serial/safe_serial.c
+++ b/drivers/usb/serial/safe_serial.c
@@ -64,11 +64,9 @@
 #include <linux/errno.h>
 #include <linux/gfp.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 
diff --git a/drivers/usb/serial/spcp8x5.c b/drivers/usb/serial/spcp8x5.c
index 2fbb48f19efd7..c11d64bf08fba 100644
--- a/drivers/usb/serial/spcp8x5.c
+++ b/drivers/usb/serial/spcp8x5.c
@@ -14,8 +14,6 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
-#include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
 #include <linux/usb.h>
diff --git a/drivers/usb/serial/ssu100.c b/drivers/usb/serial/ssu100.c
index da6316410b773..b0d51558b73ce 100644
--- a/drivers/usb/serial/ssu100.c
+++ b/drivers/usb/serial/ssu100.c
@@ -9,14 +9,12 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/serial.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/serial_reg.h>
-#include <linux/uaccess.h>
 
 #define QT_OPEN_CLOSE_CHANNEL       0xca
 #define QT_SET_GET_DEVICE           0xc2
diff --git a/drivers/usb/serial/symbolserial.c b/drivers/usb/serial/symbolserial.c
index 7a5aa39172a8d..8823671a9c7aa 100644
--- a/drivers/usb/serial/symbolserial.c
+++ b/drivers/usb/serial/symbolserial.c
@@ -10,12 +10,10 @@
 #include <linux/kernel.h>
 #include <linux/tty.h>
 #include <linux/slab.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
-#include <linux/uaccess.h>
 
 static const struct usb_device_id id_table[] = {
 	{ USB_DEVICE(0x05e0, 0x0600) },
diff --git a/drivers/usb/serial/ti_usb_3410_5052.c b/drivers/usb/serial/ti_usb_3410_5052.c
index b3591d6d7645a..b3f00e3158158 100644
--- a/drivers/usb/serial/ti_usb_3410_5052.c
+++ b/drivers/usb/serial/ti_usb_3410_5052.c
@@ -18,7 +18,6 @@
 #include <linux/firmware.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
@@ -26,7 +25,6 @@
 #include <linux/serial.h>
 #include <linux/kfifo.h>
 #include <linux/mutex.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 
diff --git a/drivers/usb/serial/usb-serial.c b/drivers/usb/serial/usb-serial.c
index 0e072fd87c3d2..17edc057a3110 100644
--- a/drivers/usb/serial/usb-serial.c
+++ b/drivers/usb/serial/usb-serial.c
@@ -22,14 +22,11 @@
 #include <linux/slab.h>
 #include <linux/tty.h>
 #include <linux/tty_driver.h>
-#include <linux/tty_flip.h>
 #include <linux/module.h>
-#include <linux/moduleparam.h>
 #include <linux/seq_file.h>
 #include <linux/spinlock.h>
 #include <linux/mutex.h>
 #include <linux/list.h>
-#include <linux/uaccess.h>
 #include <linux/serial.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
diff --git a/drivers/usb/serial/usb_wwan.c b/drivers/usb/serial/usb_wwan.c
index e752ffa4dc621..a183cc4515edb 100644
--- a/drivers/usb/serial/usb_wwan.c
+++ b/drivers/usb/serial/usb_wwan.c
@@ -27,7 +27,6 @@
 #include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/bitops.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/cdc.h>
 #include <linux/usb/serial.h>
diff --git a/drivers/usb/serial/visor.c b/drivers/usb/serial/visor.c
index 062a38fe0c1cc..5706c46c49c27 100644
--- a/drivers/usb/serial/visor.c
+++ b/drivers/usb/serial/visor.c
@@ -15,12 +15,8 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
-#include <linux/tty_flip.h>
 #include <linux/module.h>
-#include <linux/moduleparam.h>
 #include <linux/spinlock.h>
-#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
 #include <linux/usb/cdc.h>
diff --git a/drivers/usb/serial/whiteheat.c b/drivers/usb/serial/whiteheat.c
index 6e2b52b2b5f80..a2d83f69e00e0 100644
--- a/drivers/usb/serial/whiteheat.c
+++ b/drivers/usb/serial/whiteheat.c
@@ -16,13 +16,9 @@
 #include <linux/errno.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
-#include <linux/tty_driver.h>
-#include <linux/tty_flip.h>
 #include <linux/module.h>
 #include <linux/spinlock.h>
 #include <linux/mutex.h>
-#include <linux/uaccess.h>
-#include <asm/termbits.h>
 #include <linux/usb.h>
 #include <linux/serial_reg.h>
 #include <linux/serial.h>
diff --git a/drivers/usb/serial/wishbone-serial.c b/drivers/usb/serial/wishbone-serial.c
index 670d573f6b634..4a1fed4cb57b6 100644
--- a/drivers/usb/serial/wishbone-serial.c
+++ b/drivers/usb/serial/wishbone-serial.c
@@ -11,7 +11,6 @@
 #include <linux/module.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
-#include <linux/uaccess.h>
 
 #define GSI_VENDOR_OPENCLOSE 0xB0
 
diff --git a/drivers/usb/serial/xr_serial.c b/drivers/usb/serial/xr_serial.c
index 9fc4082de7701..352c765d88034 100644
--- a/drivers/usb/serial/xr_serial.c
+++ b/drivers/usb/serial/xr_serial.c
@@ -16,6 +16,7 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/tty.h>
+#include <linux/uaccess.h>
 #include <linux/usb.h>
 #include <linux/usb/cdc.h>
 #include <linux/usb/serial.h>
diff --git a/drivers/usb/serial/xsens_mt.c b/drivers/usb/serial/xsens_mt.c
index 382b3698c1d5f..151a6e893ba20 100644
--- a/drivers/usb/serial/xsens_mt.c
+++ b/drivers/usb/serial/xsens_mt.c
@@ -10,7 +10,6 @@
 #include <linux/module.h>
 #include <linux/usb.h>
 #include <linux/usb/serial.h>
-#include <linux/uaccess.h>
 
 #define XSENS_VID 0x2639
 
diff --git a/drivers/usb/typec/altmodes/displayport.c b/drivers/usb/typec/altmodes/displayport.c
index 35d9c30869900..263a89c5f3243 100644
--- a/drivers/usb/typec/altmodes/displayport.c
+++ b/drivers/usb/typec/altmodes/displayport.c
@@ -405,6 +405,8 @@ static int dp_altmode_vdm(struct typec_altmode *alt,
 				dp->state = DP_STATE_EXIT_PRIME;
 			break;
 		case DP_CMD_STATUS_UPDATE:
+			if (count < 2)
+				break;
 			dp->data.status = *vdo;
 			ret = dp_altmode_status_update(dp);
 			break;
diff --git a/drivers/usb/typec/mux/intel_pmc_mux.c b/drivers/usb/typec/mux/intel_pmc_mux.c
index 1698428654abb..e22b070a140f9 100644
--- a/drivers/usb/typec/mux/intel_pmc_mux.c
+++ b/drivers/usb/typec/mux/intel_pmc_mux.c
@@ -151,13 +151,14 @@ struct pmc_usb {
 	u8 num_ports;
 	struct device *dev;
 	struct intel_scu_ipc_dev *ipc;
-	struct pmc_usb_port *port;
 	struct acpi_device *iom_adev;
 	void __iomem *iom_base;
 	u32 iom_port_status_offset;
 	u8 iom_port_status_size;
 
 	struct dentry *dentry;
+
+	struct pmc_usb_port port[] __counted_by(num_ports);
 };
 
 static struct dentry *pmc_mux_debugfs_root;
@@ -731,27 +732,25 @@ static int pmc_usb_probe(struct platform_device *pdev)
 {
 	struct fwnode_handle *fwnode = NULL;
 	struct pmc_usb *pmc;
+	u8 num_ports;
 	int i = 0;
 	int ret;
 
-	pmc = devm_kzalloc(&pdev->dev, sizeof(*pmc), GFP_KERNEL);
-	if (!pmc)
-		return -ENOMEM;
-
 	device_for_each_child_node(&pdev->dev, fwnode)
-		pmc->num_ports++;
+		num_ports++;
 
 	/* The IOM microcontroller has a limitation of max 4 ports. */
-	if (pmc->num_ports > 4) {
+	if (num_ports > 4) {
 		dev_err(&pdev->dev, "driver limited to 4 ports\n");
 		return -ERANGE;
 	}
 
-	pmc->port = devm_kcalloc(&pdev->dev, pmc->num_ports,
-				 sizeof(struct pmc_usb_port), GFP_KERNEL);
-	if (!pmc->port)
+	pmc = devm_kzalloc(&pdev->dev, struct_size(pmc, port, num_ports), GFP_KERNEL);
+	if (!pmc)
 		return -ENOMEM;
 
+	pmc->num_ports = num_ports;
+
 	pmc->ipc = devm_intel_scu_ipc_dev_get(&pdev->dev);
 	if (!pmc->ipc)
 		return -EPROBE_DEFER;
diff --git a/drivers/usb/typec/mux/ps883x.c b/drivers/usb/typec/mux/ps883x.c
index 1256252eceedc..f52443638ee28 100644
--- a/drivers/usb/typec/mux/ps883x.c
+++ b/drivers/usb/typec/mux/ps883x.c
@@ -61,19 +61,110 @@ struct ps883x_retimer {
 	struct mutex lock; /* protect non-concurrent retimer & switch */
 
 	enum typec_orientation orientation;
-	u8 cfg0;
-	u8 cfg1;
-	u8 cfg2;
+	bool in_reset;
 };
 
+static int ps883x_enable_vregs(struct ps883x_retimer *retimer)
+{
+	struct device *dev = &retimer->client->dev;
+	int ret;
+
+	ret = regulator_enable(retimer->vdd33_supply);
+	if (ret) {
+		dev_err(dev, "cannot enable VDD 3.3V regulator: %d\n", ret);
+		return ret;
+	}
+
+	ret = regulator_enable(retimer->vdd33_cap_supply);
+	if (ret) {
+		dev_err(dev, "cannot enable VDD 3.3V CAP regulator: %d\n", ret);
+		goto err_vdd33_disable;
+	}
+
+	usleep_range(4000, 10000);
+
+	ret = regulator_enable(retimer->vdd_supply);
+	if (ret) {
+		dev_err(dev, "cannot enable VDD regulator: %d\n", ret);
+		goto err_vdd33_cap_disable;
+	}
+
+	ret = regulator_enable(retimer->vddar_supply);
+	if (ret) {
+		dev_err(dev, "cannot enable VDD AR regulator: %d\n", ret);
+		goto err_vdd_disable;
+	}
+
+	ret = regulator_enable(retimer->vddat_supply);
+	if (ret) {
+		dev_err(dev, "cannot enable VDD AT regulator: %d\n", ret);
+		goto err_vddar_disable;
+	}
+
+	ret = regulator_enable(retimer->vddio_supply);
+	if (ret) {
+		dev_err(dev, "cannot enable VDD IO regulator: %d\n", ret);
+		goto err_vddat_disable;
+	}
+
+	return 0;
+
+err_vddat_disable:
+	regulator_disable(retimer->vddat_supply);
+err_vddar_disable:
+	regulator_disable(retimer->vddar_supply);
+err_vdd_disable:
+	regulator_disable(retimer->vdd_supply);
+err_vdd33_cap_disable:
+	regulator_disable(retimer->vdd33_cap_supply);
+err_vdd33_disable:
+	regulator_disable(retimer->vdd33_supply);
+
+	return ret;
+}
+
+static void ps883x_disable_vregs(struct ps883x_retimer *retimer)
+{
+	regulator_disable(retimer->vddio_supply);
+	regulator_disable(retimer->vddat_supply);
+	regulator_disable(retimer->vddar_supply);
+	regulator_disable(retimer->vdd_supply);
+	regulator_disable(retimer->vdd33_cap_supply);
+	regulator_disable(retimer->vdd33_supply);
+}
+
+static void ps883x_reset(struct ps883x_retimer *retimer)
+{
+	if (retimer->in_reset)
+		return;
+
+	gpiod_set_value(retimer->reset_gpio, 1);
+	ps883x_disable_vregs(retimer);
+	retimer->in_reset = true;
+}
+
 static int ps883x_configure(struct ps883x_retimer *retimer, int cfg0,
-			    int cfg1, int cfg2)
+			    int cfg1, int cfg2, bool reset)
 {
 	struct device *dev = &retimer->client->dev;
 	int ret;
 
-	if (retimer->cfg0 == cfg0 && retimer->cfg1 == cfg1 && retimer->cfg2 == cfg2)
+	if (reset) {
+		ps883x_reset(retimer);
+
 		return 0;
+	} else if (retimer->in_reset) {
+		ret = ps883x_enable_vregs(retimer);
+		if (ret)
+			return ret;
+
+		gpiod_set_value(retimer->reset_gpio, 0);
+
+		/* firmware initialization delay */
+		msleep(60);
+
+		retimer->in_reset = false;
+	}
 
 	ret = regmap_write(retimer->regmap, REG_USB_PORT_CONN_STATUS_0, cfg0);
 	if (ret) {
@@ -93,10 +184,6 @@ static int ps883x_configure(struct ps883x_retimer *retimer, int cfg0,
 		return ret;
 	}
 
-	retimer->cfg0 = cfg0;
-	retimer->cfg1 = cfg1;
-	retimer->cfg2 = cfg2;
-
 	return 0;
 }
 
@@ -107,6 +194,7 @@ static int ps883x_set(struct ps883x_retimer *retimer, struct typec_retimer_state
 	int cfg0 = CONN_STATUS_0_CONNECTION_PRESENT;
 	int cfg1 = 0x00;
 	int cfg2 = 0x00;
+	bool reset = false;
 
 	if (retimer->orientation == TYPEC_ORIENTATION_REVERSE)
 		cfg0 |= CONN_STATUS_0_ORIENTATION_REVERSED;
@@ -148,9 +236,13 @@ static int ps883x_set(struct ps883x_retimer *retimer, struct typec_retimer_state
 		}
 	} else {
 		switch (state->mode) {
+		/* SAFE can be transient or point to an actual disconnect */
 		case TYPEC_STATE_SAFE:
+			reset = retimer->orientation == TYPEC_ORIENTATION_NONE;
+			break;
 		/* USB2 pins don't even go through this chip */
 		case TYPEC_MODE_USB2:
+			reset = true;
 			break;
 		case TYPEC_STATE_USB:
 		case TYPEC_MODE_USB3:
@@ -171,7 +263,7 @@ static int ps883x_set(struct ps883x_retimer *retimer, struct typec_retimer_state
 		}
 	}
 
-	return ps883x_configure(retimer, cfg0, cfg1, cfg2);
+	return ps883x_configure(retimer, cfg0, cfg1, cfg2, reset);
 }
 
 static int ps883x_sw_set(struct typec_switch_dev *sw,
@@ -184,11 +276,19 @@ static int ps883x_sw_set(struct typec_switch_dev *sw,
 	if (ret)
 		return ret;
 
-	mutex_lock(&retimer->lock);
+	guard(mutex)(&retimer->lock);
 
 	if (retimer->orientation != orientation) {
 		retimer->orientation = orientation;
 
+		/*
+		 * Orientation notifications usually come prior to mode switch
+		 * events. If the retimer is already in reset, we still want to
+		 * cache the new orientation value for the subsequent ps883x_set().
+		 */
+		if (retimer->in_reset)
+			return 0;
+
 		ret = regmap_assign_bits(retimer->regmap, REG_USB_PORT_CONN_STATUS_0,
 					 CONN_STATUS_0_ORIENTATION_REVERSED,
 					 orientation == TYPEC_ORIENTATION_REVERSE);
@@ -196,8 +296,6 @@ static int ps883x_sw_set(struct typec_switch_dev *sw,
 			dev_err(&retimer->client->dev, "failed to set orientation: %d\n", ret);
 	}
 
-	mutex_unlock(&retimer->lock);
-
 	return ret;
 }
 
@@ -222,75 +320,6 @@ static int ps883x_retimer_set(struct typec_retimer *rtmr,
 	return typec_mux_set(retimer->typec_mux, &mux_state);
 }
 
-static int ps883x_enable_vregs(struct ps883x_retimer *retimer)
-{
-	struct device *dev = &retimer->client->dev;
-	int ret;
-
-	ret = regulator_enable(retimer->vdd33_supply);
-	if (ret) {
-		dev_err(dev, "cannot enable VDD 3.3V regulator: %d\n", ret);
-		return ret;
-	}
-
-	ret = regulator_enable(retimer->vdd33_cap_supply);
-	if (ret) {
-		dev_err(dev, "cannot enable VDD 3.3V CAP regulator: %d\n", ret);
-		goto err_vdd33_disable;
-	}
-
-	usleep_range(4000, 10000);
-
-	ret = regulator_enable(retimer->vdd_supply);
-	if (ret) {
-		dev_err(dev, "cannot enable VDD regulator: %d\n", ret);
-		goto err_vdd33_cap_disable;
-	}
-
-	ret = regulator_enable(retimer->vddar_supply);
-	if (ret) {
-		dev_err(dev, "cannot enable VDD AR regulator: %d\n", ret);
-		goto err_vdd_disable;
-	}
-
-	ret = regulator_enable(retimer->vddat_supply);
-	if (ret) {
-		dev_err(dev, "cannot enable VDD AT regulator: %d\n", ret);
-		goto err_vddar_disable;
-	}
-
-	ret = regulator_enable(retimer->vddio_supply);
-	if (ret) {
-		dev_err(dev, "cannot enable VDD IO regulator: %d\n", ret);
-		goto err_vddat_disable;
-	}
-
-	return 0;
-
-err_vddat_disable:
-	regulator_disable(retimer->vddat_supply);
-err_vddar_disable:
-	regulator_disable(retimer->vddar_supply);
-err_vdd_disable:
-	regulator_disable(retimer->vdd_supply);
-err_vdd33_cap_disable:
-	regulator_disable(retimer->vdd33_cap_supply);
-err_vdd33_disable:
-	regulator_disable(retimer->vdd33_supply);
-
-	return ret;
-}
-
-static void ps883x_disable_vregs(struct ps883x_retimer *retimer)
-{
-	regulator_disable(retimer->vddio_supply);
-	regulator_disable(retimer->vddat_supply);
-	regulator_disable(retimer->vddar_supply);
-	regulator_disable(retimer->vdd_supply);
-	regulator_disable(retimer->vdd33_cap_supply);
-	regulator_disable(retimer->vdd33_supply);
-}
-
 static int ps883x_get_vregs(struct ps883x_retimer *retimer)
 {
 	struct device *dev = &retimer->client->dev;
@@ -422,6 +451,9 @@ static int ps883x_retimer_probe(struct i2c_client *client)
 		}
 	}
 
+	/* Keep the retimer in reset until a Type-C notification comes */
+	ps883x_reset(retimer);
+
 	sw_desc.drvdata = retimer;
 	sw_desc.fwnode = dev_fwnode(dev);
 	sw_desc.set = ps883x_sw_set;
diff --git a/drivers/usb/typec/tcpm/tcpci_maxim_core.c b/drivers/usb/typec/tcpm/tcpci_maxim_core.c
index c0ee7e6959edf..7324139d51c8e 100644
--- a/drivers/usb/typec/tcpm/tcpci_maxim_core.c
+++ b/drivers/usb/typec/tcpm/tcpci_maxim_core.c
@@ -181,6 +181,15 @@ static void process_rx(struct max_tcpci_chip *chip, u16 status)
 	rx_buf_ptr = rx_buf + TCPC_RECEIVE_BUFFER_RX_BYTE_BUF_OFFSET;
 	msg.header = cpu_to_le16(*(u16 *)rx_buf_ptr);
 	rx_buf_ptr = rx_buf_ptr + sizeof(msg.header);
+
+	if (count < TCPC_RECEIVE_BUFFER_RX_BYTE_BUF_OFFSET + sizeof(msg.header) +
+		    pd_header_cnt_le(msg.header) * sizeof(msg.payload[0])) {
+		max_tcpci_write16(chip, TCPC_ALERT, TCPC_ALERT_RX_STATUS);
+		dev_err(chip->dev, "Invalid TCPC_RX_BYTE_CNT %d for header cnt %d\n",
+			count, pd_header_cnt_le(msg.header));
+		return;
+	}
+
 	for (payload_index = 0; payload_index < pd_header_cnt_le(msg.header); payload_index++,
 	     rx_buf_ptr += sizeof(msg.payload[0]))
 		msg.payload[payload_index] = cpu_to_le32(*(u32 *)rx_buf_ptr);
diff --git a/drivers/usb/typec/tcpm/tcpm.c b/drivers/usb/typec/tcpm/tcpm.c
index 55fee96d3342a..ed5f745a82313 100644
--- a/drivers/usb/typec/tcpm/tcpm.c
+++ b/drivers/usb/typec/tcpm/tcpm.c
@@ -1855,6 +1855,9 @@ static void svdm_consume_identity(struct tcpm_port *port, const u32 *p, int cnt)
 	u32 vdo = p[VDO_INDEX_IDH];
 	u32 product = p[VDO_INDEX_PRODUCT];
 
+	if (cnt <= VDO_INDEX_PRODUCT)
+		return;
+
 	memset(&port->mode_data, 0, sizeof(port->mode_data));
 
 	port->partner_ident.id_header = vdo;
@@ -1875,6 +1878,9 @@ static void svdm_consume_identity_sop_prime(struct tcpm_port *port, const u32 *p
 	u32 product = p[VDO_INDEX_PRODUCT];
 	int svdm_version;
 
+	if (cnt <= VDO_INDEX_CABLE_1)
+		return;
+
 	/*
 	 * Attempt to consume identity only if cable currently is not set
 	 */
@@ -1898,7 +1904,7 @@ static void svdm_consume_identity_sop_prime(struct tcpm_port *port, const u32 *p
 	switch (port->negotiated_rev_prime) {
 	case PD_REV30:
 		port->cable_desc.pd_revision = 0x0300;
-		if (port->cable_desc.active)
+		if (port->cable_desc.active && cnt > VDO_INDEX_CABLE_2)
 			port->cable_ident.vdo[1] = p[VDO_INDEX_CABLE_2];
 		break;
 	case PD_REV20:
@@ -1986,23 +1992,19 @@ static void svdm_consume_modes(struct tcpm_port *port, const u32 *p, int cnt,
 	switch (rx_sop_type) {
 	case TCPC_TX_SOP_PRIME:
 		pmdata = &port->mode_data_prime;
-		if (pmdata->altmodes >= ARRAY_SIZE(port->plug_prime_altmode)) {
-			/* Already logged in svdm_consume_svids() */
-			return;
-		}
 		break;
 	case TCPC_TX_SOP:
 		pmdata = &port->mode_data;
-		if (pmdata->altmodes >= ARRAY_SIZE(port->partner_altmode)) {
-			/* Already logged in svdm_consume_svids() */
-			return;
-		}
 		break;
 	default:
 		return;
 	}
 
 	for (i = 1; i < cnt; i++) {
+		if (pmdata->altmodes >= ALTMODE_DISCOVERY_MAX) {
+			/* Already logged in svdm_consume_svids() */
+			return;
+		}
 		paltmode = &pmdata->altmode_desc[pmdata->altmodes];
 		memset(paltmode, 0, sizeof(*paltmode));
 
diff --git a/drivers/usb/typec/tcpm/wcove.c b/drivers/usb/typec/tcpm/wcove.c
index 759c982bb16a9..0e5a3e277c3e4 100644
--- a/drivers/usb/typec/tcpm/wcove.c
+++ b/drivers/usb/typec/tcpm/wcove.c
@@ -444,9 +444,11 @@ static int wcove_start_toggling(struct tcpc_dev *tcpc,
 	return regmap_write(wcove->regmap, USBC_CONTROL1, usbc_ctrl);
 }
 
-static int wcove_read_rx_buffer(struct wcove_typec *wcove, void *msg)
+static int wcove_read_rx_buffer(struct wcove_typec *wcove,
+				struct pd_message *msg)
 {
-	unsigned int info;
+	unsigned int info, val, len;
+	u8 *buf = (u8 *)msg;
 	int ret;
 	int i;
 
@@ -454,12 +456,13 @@ static int wcove_read_rx_buffer(struct wcove_typec *wcove, void *msg)
 	if (ret)
 		return ret;
 
-	/* FIXME: Check that USBC_RXINFO_RXBYTES(info) matches the header */
+	len = min(USBC_RXINFO_RXBYTES(info), sizeof(*msg));
 
-	for (i = 0; i < USBC_RXINFO_RXBYTES(info); i++) {
-		ret = regmap_read(wcove->regmap, USBC_RX_DATA + i, msg + i);
+	for (i = 0; i < len; i++) {
+		ret = regmap_read(wcove->regmap, USBC_RX_DATA + i, &val);
 		if (ret)
 			return ret;
+		buf[i] = val;
 	}
 
 	return regmap_write(wcove->regmap, USBC_RXSTATUS,
diff --git a/drivers/usb/typec/tipd/core.c b/drivers/usb/typec/tipd/core.c
index 43faec794b95a..d0b769333bd99 100644
--- a/drivers/usb/typec/tipd/core.c
+++ b/drivers/usb/typec/tipd/core.c
@@ -1835,6 +1835,7 @@ static int tps6598x_probe(struct i2c_client *client)
 		goto err_role_put;
 
 	if (status & TPS_STATUS_PLUG_PRESENT) {
+		ret = -EINVAL;
 		if (!tps6598x_read_power_status(tps))
 			goto err_unregister_port;
 		if (!tps->data->read_data_status(tps))
diff --git a/drivers/usb/typec/ucsi/displayport.c b/drivers/usb/typec/ucsi/displayport.c
index 8aae80b457d74..67a0991a7b769 100644
--- a/drivers/usb/typec/ucsi/displayport.c
+++ b/drivers/usb/typec/ucsi/displayport.c
@@ -240,6 +240,10 @@ static int ucsi_displayport_vdm(struct typec_altmode *alt,
 				dp->header |= VDO_CMDT(CMDT_RSP_ACK);
 			break;
 		case DP_CMD_CONFIGURE:
+			if (count < 2) {
+				dp->header |= VDO_CMDT(CMDT_RSP_NAK);
+				break;
+			}
 			dp->data.conf = *data;
 			if (ucsi_displayport_configure(dp)) {
 				dp->header |= VDO_CMDT(CMDT_RSP_NAK);
diff --git a/drivers/usb/typec/ucsi/ucsi.c b/drivers/usb/typec/ucsi/ucsi.c
index 5b7ad9e99cb94..539dc706798d5 100644
--- a/drivers/usb/typec/ucsi/ucsi.c
+++ b/drivers/usb/typec/ucsi/ucsi.c
@@ -1380,13 +1380,22 @@ static void ucsi_handle_connector_change(struct work_struct *work)
  */
 void ucsi_connector_change(struct ucsi *ucsi, u8 num)
 {
-	struct ucsi_connector *con = &ucsi->connector[num - 1];
+	struct ucsi_connector *con;
 
 	if (!(ucsi->ntfy & UCSI_ENABLE_NTFY_CONNECTOR_CHANGE)) {
 		dev_dbg(ucsi->dev, "Early connector change event\n");
 		return;
 	}
 
+	if (!num || num > ucsi->cap.num_connectors) {
+		dev_warn_ratelimited(ucsi->dev,
+				     "Bogus connector change on %u (max %u)\n",
+				     num, ucsi->cap.num_connectors);
+		return;
+	}
+
+	con = &ucsi->connector[num - 1];
+
 	if (!test_and_set_bit(EVENT_PENDING, &ucsi->flags))
 		schedule_work(&con->work);
 }
diff --git a/drivers/usb/typec/ucsi/ucsi_ccg.c b/drivers/usb/typec/ucsi/ucsi_ccg.c
index 199799b319c2a..4463c1ae96bd4 100644
--- a/drivers/usb/typec/ucsi/ucsi_ccg.c
+++ b/drivers/usb/typec/ucsi/ucsi_ccg.c
@@ -1243,6 +1243,11 @@ static int do_flash(struct ucsi_ccg *uc, enum enum_flash_mode mode)
 	 *****************************************************************/
 
 	p = strnchr(fw->data, fw->size, ':');
+	if (!p) {
+		dev_err(dev, "Bad FW format: no ':' record header found\n");
+		err = -EINVAL;
+		goto release_mem;
+	}
 	while (p < eof) {
 		s = strnchr(p + 1, eof - p - 1, ':');
 
diff --git a/drivers/usb/usbip/usbip_common.h b/drivers/usb/usbip/usbip_common.h
index 282efca64a012..be4c5e65a7f8c 100644
--- a/drivers/usb/usbip/usbip_common.h
+++ b/drivers/usb/usbip/usbip_common.h
@@ -282,9 +282,7 @@ struct usbip_device {
 		void (*unusable)(struct usbip_device *);
 	} eh_ops;
 
-#ifdef CONFIG_KCOV
-	u64 kcov_handle;
-#endif
+	struct kcov_common_handle_id kcov_handle;
 };
 
 #define kthread_get_run(threadfn, data, namefmt, ...)			   \
@@ -339,29 +337,4 @@ static inline int interface_to_devnum(struct usb_interface *interface)
 	return udev->devnum;
 }
 
-#ifdef CONFIG_KCOV
-
-static inline void usbip_kcov_handle_init(struct usbip_device *ud)
-{
-	ud->kcov_handle = kcov_common_handle();
-}
-
-static inline void usbip_kcov_remote_start(struct usbip_device *ud)
-{
-	kcov_remote_start_common(ud->kcov_handle);
-}
-
-static inline void usbip_kcov_remote_stop(void)
-{
-	kcov_remote_stop();
-}
-
-#else /* CONFIG_KCOV */
-
-static inline void usbip_kcov_handle_init(struct usbip_device *ud) { }
-static inline void usbip_kcov_remote_start(struct usbip_device *ud) { }
-static inline void usbip_kcov_remote_stop(void) { }
-
-#endif /* CONFIG_KCOV */
-
 #endif /* __USBIP_COMMON_H */
diff --git a/drivers/usb/usbip/vhci_rx.c b/drivers/usb/usbip/vhci_rx.c
index a75f4a898a412..a678e7c898375 100644
--- a/drivers/usb/usbip/vhci_rx.c
+++ b/drivers/usb/usbip/vhci_rx.c
@@ -261,9 +261,9 @@ int vhci_rx_loop(void *data)
 		if (usbip_event_happened(ud))
 			break;
 
-		usbip_kcov_remote_start(ud);
+		kcov_remote_start_common(ud->kcov_handle);
 		vhci_rx_pdu(ud);
-		usbip_kcov_remote_stop();
+		kcov_remote_stop();
 	}
 
 	return 0;
diff --git a/drivers/usb/usbip/vhci_sysfs.c b/drivers/usb/usbip/vhci_sysfs.c
index 5bc8c47788d45..b98d14c43d13d 100644
--- a/drivers/usb/usbip/vhci_sysfs.c
+++ b/drivers/usb/usbip/vhci_sysfs.c
@@ -425,7 +425,7 @@ static ssize_t attach_store(struct device *dev, struct device_attribute *attr,
 	vdev->ud.tcp_rx     = tcp_rx;
 	vdev->ud.tcp_tx     = tcp_tx;
 	vdev->ud.status     = VDEV_ST_NOTASSIGNED;
-	usbip_kcov_handle_init(&vdev->ud);
+	vdev->ud.kcov_handle = kcov_common_handle();
 
 	spin_unlock(&vdev->ud.lock);
 	spin_unlock_irqrestore(&vhci->lock, flags);
diff --git a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
index bb121f635b9f7..86362ec424a50 100644
--- a/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
+++ b/drivers/vfio/pci/hisilicon/hisi_acc_vfio_pci.c
@@ -81,13 +81,10 @@ static int qm_get_vft(struct hisi_qm *qm, u32 *base)
 	u32 qp_num;
 	int ret;
 
-	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1);
+	ret = hisi_qm_mb_read(qm, &sqc_vft, QM_MB_CMD_SQC_VFT_V2, 0);
 	if (ret)
 		return ret;
 
-	sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
-		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
-		  QM_XQC_ADDR_OFFSET);
 	*base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2);
 	qp_num = (QM_SQC_VFT_NUM_MASK_V2 &
 		  (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1;
@@ -95,36 +92,6 @@ static int qm_get_vft(struct hisi_qm *qm, u32 *base)
 	return qp_num;
 }
 
-static int qm_get_sqc(struct hisi_qm *qm, u64 *addr)
-{
-	int ret;
-
-	ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, 0, 0, 1);
-	if (ret)
-		return ret;
-
-	*addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
-		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
-		  QM_XQC_ADDR_OFFSET);
-
-	return 0;
-}
-
-static int qm_get_cqc(struct hisi_qm *qm, u64 *addr)
-{
-	int ret;
-
-	ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, 0, 0, 1);
-	if (ret)
-		return ret;
-
-	*addr = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) |
-		  ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) <<
-		  QM_XQC_ADDR_OFFSET);
-
-	return 0;
-}
-
 static void qm_xqc_reg_offsets(struct hisi_qm *qm,
 			       u32 *eqc_addr, u32 *aeqc_addr)
 {
@@ -575,13 +542,13 @@ static int vf_qm_read_data(struct hisi_qm *vf_qm, struct acc_vf_data *vf_data)
 	vf_data->aeqe_dma |= vf_data->qm_aeqc_dw[QM_XQC_ADDR_LOW];
 
 	/* Through SQC_BT/CQC_BT to get sqc and cqc address */
-	ret = qm_get_sqc(vf_qm, &vf_data->sqc_dma);
+	ret = hisi_qm_mb_read(vf_qm, &vf_data->sqc_dma, QM_MB_CMD_SQC_BT, 0);
 	if (ret) {
 		dev_err(dev, "failed to read SQC addr!\n");
 		return ret;
 	}
 
-	ret = qm_get_cqc(vf_qm, &vf_data->cqc_dma);
+	ret = hisi_qm_mb_read(vf_qm, &vf_data->cqc_dma, QM_MB_CMD_CQC_BT, 0);
 	if (ret) {
 		dev_err(dev, "failed to read CQC addr!\n");
 		return ret;
diff --git a/drivers/vfio/pci/nvgrace-gpu/main.c b/drivers/vfio/pci/nvgrace-gpu/main.c
index fa056b69f899a..15e2f03c6cd44 100644
--- a/drivers/vfio/pci/nvgrace-gpu/main.c
+++ b/drivers/vfio/pci/nvgrace-gpu/main.c
@@ -61,6 +61,7 @@ struct nvgrace_gpu_pci_core_device {
 	struct mem_region resmem;
 	/* Lock to control device memory kernel mapping */
 	struct mutex remap_lock;
+	void __iomem *bar0_base;
 	bool has_mig_hw_bug;
 	/* GPU has just been reset */
 	bool reset_done;
@@ -171,6 +172,7 @@ static int nvgrace_gpu_open_device(struct vfio_device *core_vdev)
 	struct nvgrace_gpu_pci_core_device *nvdev =
 		container_of(core_vdev, struct nvgrace_gpu_pci_core_device,
 			     core_device.vdev);
+	void __iomem *io;
 	int ret;
 
 	ret = vfio_pci_core_enable(vdev);
@@ -184,14 +186,14 @@ static int nvgrace_gpu_open_device(struct vfio_device *core_vdev)
 
 	/*
 	 * GPU readiness is checked by reading the BAR0 registers.
-	 *
-	 * ioremap BAR0 to ensure that the BAR0 mapping is present before
-	 * register reads on first fault before establishing any GPU
-	 * memory mapping.
+	 * The BAR map was just set up by vfio_pci_core_enable(), so
+	 * bail early if that wasn't successful:
 	 */
-	ret = vfio_pci_core_setup_barmap(vdev, 0);
-	if (ret)
+	io = vfio_pci_core_get_iomap(vdev, 0);
+	if (IS_ERR(io)) {
+		ret = PTR_ERR(io);
 		goto error_exit;
+	}
 
 	if (nvdev->resmem.memlength) {
 		ret = nvgrace_gpu_vfio_pci_register_pfn_range(core_vdev, &nvdev->resmem);
@@ -204,6 +206,8 @@ static int nvgrace_gpu_open_device(struct vfio_device *core_vdev)
 		goto register_mem_failed;
 
 	vfio_pci_core_finish_enable(vdev);
+	nvdev->bar0_base = io;
+
 	return 0;
 
 register_mem_failed:
@@ -220,6 +224,8 @@ static void nvgrace_gpu_close_device(struct vfio_device *core_vdev)
 		container_of(core_vdev, struct nvgrace_gpu_pci_core_device,
 			     core_device.vdev);
 
+	nvdev->bar0_base = NULL;
+
 	if (nvdev->resmem.memlength)
 		unregister_pfn_address_space(&nvdev->resmem.pfn_address_space);
 
@@ -275,7 +281,7 @@ nvgrace_gpu_check_device_ready(struct nvgrace_gpu_pci_core_device *nvdev)
 	if (!__vfio_pci_memory_enabled(vdev))
 		return -EIO;
 
-	ret = nvgrace_gpu_wait_device_ready(vdev->barmap[0]);
+	ret = nvgrace_gpu_wait_device_ready(nvdev->bar0_base);
 	if (ret)
 		return ret;
 
diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c
index 050e7542952ed..041243a84d817 100644
--- a/drivers/vfio/pci/vfio_pci_core.c
+++ b/drivers/vfio/pci/vfio_pci_core.c
@@ -1762,7 +1762,7 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
 	struct pci_dev *pdev = vdev->pdev;
 	unsigned int index;
 	u64 phys_len, req_len, pgoff, req_start;
-	int ret;
+	void __iomem *bar_io;
 
 	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
 
@@ -1796,12 +1796,11 @@ int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma
 		return -EINVAL;
 
 	/*
-	 * Even though we don't make use of the barmap for the mmap,
-	 * we need to request the region and the barmap tracks that.
+	 * Ensure the BAR resource region is reserved for use.
 	 */
-	ret = vfio_pci_core_setup_barmap(vdev, index);
-	if (ret)
-		return ret;
+	bar_io = vfio_pci_core_get_iomap(vdev, index);
+	if (IS_ERR(bar_io))
+		return PTR_ERR(bar_io);
 
 	vma->vm_private_data = vdev;
 	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
diff --git a/drivers/vfio/pci/vfio_pci_dmabuf.c b/drivers/vfio/pci/vfio_pci_dmabuf.c
index 1a177ce7de546..c16f460c01d68 100644
--- a/drivers/vfio/pci/vfio_pci_dmabuf.c
+++ b/drivers/vfio/pci/vfio_pci_dmabuf.c
@@ -248,7 +248,7 @@ int vfio_pci_core_feature_dma_buf(struct vfio_pci_core_device *vdev, u32 flags,
 	 * else.  Check that PCI resources have been claimed for it.
 	 */
 	if (get_dma_buf.region_index >= VFIO_PCI_ROM_REGION_INDEX ||
-	    vfio_pci_core_setup_barmap(vdev, get_dma_buf.region_index))
+	    IS_ERR(vfio_pci_core_get_iomap(vdev, get_dma_buf.region_index)))
 		return -ENODEV;
 
 	dma_ranges = memdup_array_user(&arg->dma_ranges, get_dma_buf.nr_ranges,
diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c
index 3bfbb879a0051..7f14dd46de17a 100644
--- a/drivers/vfio/pci/vfio_pci_rdwr.c
+++ b/drivers/vfio/pci/vfio_pci_rdwr.c
@@ -198,19 +198,6 @@ ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
 }
 EXPORT_SYMBOL_GPL(vfio_pci_core_do_io_rw);
 
-/*
- * The barmap is set up in vfio_pci_core_enable().  Callers use this
- * function to check that the BAR resources are requested or that the
- * pci_iomap() was done.
- */
-int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
-{
-	if (IS_ERR(vdev->barmap[bar]))
-		return PTR_ERR(vdev->barmap[bar]);
-	return 0;
-}
-EXPORT_SYMBOL_GPL(vfio_pci_core_setup_barmap);
-
 ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
 			size_t count, loff_t *ppos, bool iswrite)
 {
@@ -262,13 +249,11 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
 		 */
 		max_width = VFIO_PCI_IO_WIDTH_4;
 	} else {
-		int ret = vfio_pci_core_setup_barmap(vdev, bar);
-		if (ret) {
-			done = ret;
+		io = vfio_pci_core_get_iomap(vdev, bar);
+		if (IS_ERR(io)) {
+			done = PTR_ERR(io);
 			goto out;
 		}
-
-		io = vdev->barmap[bar];
 	}
 
 	if (bar == vdev->msix_bar) {
@@ -423,6 +408,7 @@ int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
 	loff_t pos = offset & VFIO_PCI_OFFSET_MASK;
 	int ret, bar = VFIO_PCI_OFFSET_TO_INDEX(offset);
 	struct vfio_pci_ioeventfd *ioeventfd;
+	void __iomem *io;
 
 	/* Only support ioeventfds into BARs */
 	if (bar > VFIO_PCI_BAR5_REGION_INDEX)
@@ -440,9 +426,9 @@ int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
 	if (count == 8)
 		return -EINVAL;
 
-	ret = vfio_pci_core_setup_barmap(vdev, bar);
-	if (ret)
-		return ret;
+	io = vfio_pci_core_get_iomap(vdev, bar);
+	if (IS_ERR(io))
+		return PTR_ERR(io);
 
 	mutex_lock(&vdev->ioeventfds_lock);
 
@@ -479,7 +465,7 @@ int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
 	}
 
 	ioeventfd->vdev = vdev;
-	ioeventfd->addr = vdev->barmap[bar] + pos;
+	ioeventfd->addr = io + pos;
 	ioeventfd->data = data;
 	ioeventfd->pos = pos;
 	ioeventfd->bar = bar;
diff --git a/drivers/vfio/pci/virtio/legacy_io.c b/drivers/vfio/pci/virtio/legacy_io.c
index f022301e60d6e..74ff302edc9f0 100644
--- a/drivers/vfio/pci/virtio/legacy_io.c
+++ b/drivers/vfio/pci/virtio/legacy_io.c
@@ -298,19 +298,18 @@ int virtiovf_pci_ioctl_get_region_info(struct vfio_device *core_vdev,
 static int virtiovf_set_notify_addr(struct virtiovf_pci_core_device *virtvdev)
 {
 	struct vfio_pci_core_device *core_device = &virtvdev->core_device;
-	int ret;
+	void __iomem *io;
 
 	/*
 	 * Setup the BAR where the 'notify' exists to be used by vfio as well
 	 * This will let us mmap it only once and use it when needed.
 	 */
-	ret = vfio_pci_core_setup_barmap(core_device,
-					 virtvdev->notify_bar);
-	if (ret)
-		return ret;
+	io = vfio_pci_core_get_iomap(core_device,
+				     virtvdev->notify_bar);
+	if (IS_ERR(io))
+		return PTR_ERR(io);
 
-	virtvdev->notify_addr = core_device->barmap[virtvdev->notify_bar] +
-			virtvdev->notify_offset;
+	virtvdev->notify_addr = io + virtvdev->notify_offset;
 	return 0;
 }
 
diff --git a/drivers/vfio/pci/xe/main.c b/drivers/vfio/pci/xe/main.c
index 4ecadbbfd86ec..cbff5af385ef5 100644
--- a/drivers/vfio/pci/xe/main.c
+++ b/drivers/vfio/pci/xe/main.c
@@ -135,8 +135,6 @@ static void xe_vfio_pci_reset_done(struct pci_dev *pdev)
 	}
 	spin_unlock(&xe_vdev->reset_lock);
 	xe_vfio_pci_state_mutex_unlock(xe_vdev);
-
-	xe_vfio_pci_reset(xe_vdev);
 }
 
 static const struct pci_error_handlers xe_vfio_pci_err_handlers = {
diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c
index c6536cad9c4f9..db341c9226739 100644
--- a/drivers/vhost/net.c
+++ b/drivers/vhost/net.c
@@ -176,13 +176,21 @@ static void *vhost_net_buf_consume(struct vhost_net_buf *rxq)
 	return ret;
 }
 
-static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)
+static int vhost_net_buf_produce(struct sock *sk,
+				 struct vhost_net_virtqueue *nvq)
 {
+	struct file *file = sk->sk_socket->file;
 	struct vhost_net_buf *rxq = &nvq->rxq;
 
 	rxq->head = 0;
-	rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,
-					      VHOST_NET_BATCH);
+	spin_lock(&nvq->rx_ring->consumer_lock);
+	rxq->tail = __ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,
+					       VHOST_NET_BATCH);
+
+	if (rxq->tail)
+		tun_wake_queue(file, rxq->tail);
+
+	spin_unlock(&nvq->rx_ring->consumer_lock);
 	return rxq->tail;
 }
 
@@ -209,14 +217,15 @@ static int vhost_net_buf_peek_len(void *ptr)
 	return __skb_array_len_with_tag(ptr);
 }
 
-static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)
+static int vhost_net_buf_peek(struct sock *sk,
+			      struct vhost_net_virtqueue *nvq)
 {
 	struct vhost_net_buf *rxq = &nvq->rxq;
 
 	if (!vhost_net_buf_is_empty(rxq))
 		goto out;
 
-	if (!vhost_net_buf_produce(nvq))
+	if (!vhost_net_buf_produce(sk, nvq))
 		return 0;
 
 out:
@@ -995,7 +1004,7 @@ static int peek_head_len(struct vhost_net_virtqueue *rvq, struct sock *sk)
 	unsigned long flags;
 
 	if (rvq->rx_ring)
-		return vhost_net_buf_peek(rvq);
+		return vhost_net_buf_peek(sk, rvq);
 
 	spin_lock_irqsave(&sk->sk_receive_queue.lock, flags);
 	head = skb_peek(&sk->sk_receive_queue);
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 4fe99765c5c73..0192ade6e7491 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -44,7 +44,7 @@ struct vhost_worker {
 	/* Used to serialize device wide flushing with worker swapping. */
 	struct mutex		mutex;
 	struct llist_head	work_list;
-	u64			kcov_handle;
+	struct kcov_common_handle_id kcov_handle;
 	u32			id;
 	int			attachment_cnt;
 	bool			killed;
diff --git a/drivers/video/backlight/Kconfig b/drivers/video/backlight/Kconfig
index a7a3fbaf7c29e..7aa1c4b21111f 100644
--- a/drivers/video/backlight/Kconfig
+++ b/drivers/video/backlight/Kconfig
@@ -260,7 +260,7 @@ config BACKLIGHT_PWM
 
 config BACKLIGHT_CGBC
 	tristate "Congatec Board Controller (CGBC) backlight support"
-	depends on MFD_CGBC && X86
+	depends on MFD_CGBC
 	help
 	  Say Y here to enable support for LCD backlight control on Congatec
 	  x86-based boards via the CGBC (Congatec Board Controller).
@@ -282,6 +282,13 @@ config BACKLIGHT_DA9052
 	help
 	  Enable the Backlight Driver for DA9052-BC and DA9053-AA/Bx PMICs.
 
+config BACKLIGHT_MAX25014
+	tristate "Backlight driver for Maxim MAX25014"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  If you are using a MAX25014 chip as a backlight driver say Y to enable it.
+
 config BACKLIGHT_MAX8925
 	tristate "Backlight driver for MAX8925"
 	depends on MFD_MAX8925
diff --git a/drivers/video/backlight/Makefile b/drivers/video/backlight/Makefile
index 794820a98ed49..21c8313cfb121 100644
--- a/drivers/video/backlight/Makefile
+++ b/drivers/video/backlight/Makefile
@@ -47,6 +47,7 @@ obj-$(CONFIG_BACKLIGHT_LOCOMO)		+= locomolcd.o
 obj-$(CONFIG_BACKLIGHT_LP855X)		+= lp855x_bl.o
 obj-$(CONFIG_BACKLIGHT_LP8788)		+= lp8788_bl.o
 obj-$(CONFIG_BACKLIGHT_LV5207LP)	+= lv5207lp.o
+obj-$(CONFIG_BACKLIGHT_MAX25014)	+= max25014.o
 obj-$(CONFIG_BACKLIGHT_MAX8925)		+= max8925_bl.o
 obj-$(CONFIG_BACKLIGHT_MP3309C)		+= mp3309c.o
 obj-$(CONFIG_BACKLIGHT_MT6370)		+= mt6370-backlight.o
diff --git a/drivers/video/backlight/ktd2801-backlight.c b/drivers/video/backlight/ktd2801-backlight.c
index 17eac1b3bce4a..1b1307e03b204 100644
--- a/drivers/video/backlight/ktd2801-backlight.c
+++ b/drivers/video/backlight/ktd2801-backlight.c
@@ -53,6 +53,7 @@ static int ktd2801_update_status(struct backlight_device *bd)
 }
 
 static const struct backlight_ops ktd2801_backlight_ops = {
+	.options = BL_CORE_SUSPENDRESUME,
 	.update_status = ktd2801_update_status,
 };
 
diff --git a/drivers/video/backlight/max25014.c b/drivers/video/backlight/max25014.c
new file mode 100644
index 0000000000000..3ee45617261f3
--- /dev/null
+++ b/drivers/video/backlight/max25014.c
@@ -0,0 +1,377 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Backlight driver for Maxim MAX25014
+ *
+ * Copyright (C) 2025 GOcontroll B.V.
+ * Author: Maud Spierings <maudspierings@gocontroll.com>
+ */
+
+#include <linux/backlight.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#define MAX25014_ISET_DEFAULT_100 11
+#define MAX_BRIGHTNESS            100
+#define MIN_BRIGHTNESS            0
+#define TON_MAX                   130720 /* @153Hz */
+#define TON_STEP                  1307 /* @153Hz */
+#define TON_MIN                   0
+
+#define MAX25014_DEV_ID           0x00
+#define MAX25014_REV_ID           0x01
+#define MAX25014_ISET             0x02
+#define MAX25014_IMODE            0x03
+#define MAX25014_TON1H            0x04
+#define MAX25014_TON1L            0x05
+#define MAX25014_TON2H            0x06
+#define MAX25014_TON2L            0x07
+#define MAX25014_TON3H            0x08
+#define MAX25014_TON3L            0x09
+#define MAX25014_TON4H            0x0A
+#define MAX25014_TON4L            0x0B
+#define MAX25014_TON_1_4_LSB      0x0C
+#define MAX25014_SETTING          0x12
+#define MAX25014_DISABLE          0x13
+#define MAX25014_BSTMON           0x14
+#define MAX25014_IOUT1            0x15
+#define MAX25014_IOUT2            0x16
+#define MAX25014_IOUT3            0x17
+#define MAX25014_IOUT4            0x18
+#define MAX25014_OPEN             0x1B
+#define MAX25014_SHORTGND         0x1C
+#define MAX25014_SHORTED_LED      0x1D
+#define MAX25014_MASK             0x1E
+#define MAX25014_DIAG             0x1F
+
+#define MAX25014_ISET_ENA         BIT(5)
+#define MAX25014_ISET_PSEN        BIT(4)
+#define MAX25014_IMODE_HDIM       BIT(2)
+#define MAX25014_SETTING_FPWM     GENMASK(6, 4)
+#define MAX25014_DISABLE_DIS_MASK GENMASK(3, 0)
+#define MAX25014_DIAG_OT          BIT(0)
+#define MAX25014_DIAG_OTW         BIT(1)
+#define MAX25014_DIAG_HW_RST      BIT(2)
+#define MAX25014_DIAG_BSTOV       BIT(3)
+#define MAX25014_DIAG_BSTUV       BIT(4)
+#define MAX25014_DIAG_IREFOOR     BIT(5)
+
+struct max25014 {
+	struct i2c_client *client;
+	struct backlight_device *bl;
+	struct regmap *regmap;
+	struct gpio_desc *enable;
+	uint32_t iset;
+	uint8_t strings_mask;
+};
+
+static const struct regmap_config max25014_regmap_config = {
+	.reg_bits = 8,
+	.val_bits = 8,
+	.max_register = MAX25014_DIAG,
+};
+
+static int max25014_initial_power_state(struct max25014 *maxim)
+{
+	uint32_t val;
+	int ret;
+
+	ret = regmap_read(maxim->regmap, MAX25014_ISET, &val);
+	if (ret)
+		return ret;
+
+	return val & MAX25014_ISET_ENA ? BACKLIGHT_POWER_ON : BACKLIGHT_POWER_OFF;
+}
+
+static int max25014_check_errors(struct max25014 *maxim)
+{
+	uint32_t val;
+	uint8_t i;
+	int ret;
+
+	ret = regmap_read(maxim->regmap, MAX25014_OPEN, &val);
+	if (ret)
+		return ret;
+	if (val) {
+		dev_err(&maxim->client->dev, "Open led strings detected on:\n");
+		for (i = 0; i < 4; i++) {
+			if (val & 1 << i)
+				dev_err(&maxim->client->dev, "string %d\n", i + 1);
+		}
+		return -EIO;
+	}
+
+	ret = regmap_read(maxim->regmap, MAX25014_SHORTGND, &val);
+	if (ret)
+		return ret;
+	if (val) {
+		dev_err(&maxim->client->dev, "Short to ground detected on:\n");
+		for (i = 0; i < 4; i++) {
+			if (val & 1 << i)
+				dev_err(&maxim->client->dev, "string %d\n", i + 1);
+		}
+		return -EIO;
+	}
+
+	ret = regmap_read(maxim->regmap, MAX25014_SHORTED_LED, &val);
+	if (ret)
+		return ret;
+	if (val) {
+		dev_err(&maxim->client->dev, "Shorted led detected on:\n");
+		for (i = 0; i < 4; i++) {
+			if (val & 1 << i)
+				dev_err(&maxim->client->dev, "string %d\n", i + 1);
+		}
+		return -EIO;
+	}
+
+	ret = regmap_read(maxim->regmap, MAX25014_DIAG, &val);
+	if (ret)
+		return ret;
+	/*
+	 * The HW_RST bit always starts at 1 after power up.
+	 * It is reset on first read, does not indicate an error.
+	 */
+	if (val && val != MAX25014_DIAG_HW_RST) {
+		if (val & MAX25014_DIAG_OT)
+			dev_err(&maxim->client->dev,
+				"Overtemperature shutdown\n");
+		if (val & MAX25014_DIAG_OTW)
+			dev_err(&maxim->client->dev,
+				 "Chip is getting too hot (>125C)\n");
+		if (val & MAX25014_DIAG_BSTOV)
+			dev_err(&maxim->client->dev,
+				"Boost converter overvoltage\n");
+		if (val & MAX25014_DIAG_BSTUV)
+			dev_err(&maxim->client->dev,
+				"Boost converter undervoltage\n");
+		if (val & MAX25014_DIAG_IREFOOR)
+			dev_err(&maxim->client->dev, "IREF out of range\n");
+		return -EIO;
+	}
+	return 0;
+}
+
+/*
+ * 1. disable unused strings
+ * 2. set dim mode
+ * 3. set setting register
+ * 4. enable the backlight
+ */
+static int max25014_configure(struct max25014 *maxim, int initial_state)
+{
+	uint32_t val;
+	int ret;
+
+	ret = regmap_read(maxim->regmap, MAX25014_DISABLE, &val);
+	if (ret)
+		return ret;
+
+	/*
+	 * Strings can only be disabled when MAX25014_ISET_ENA == 0, check if
+	 * it needs to be changed at all to prevent the backlight flashing when
+	 * it is configured correctly by the bootloader
+	 */
+	if (!((val & MAX25014_DISABLE_DIS_MASK) == maxim->strings_mask)) {
+		if (initial_state == BACKLIGHT_POWER_ON) {
+			ret = regmap_write(maxim->regmap, MAX25014_ISET, 0);
+			if (ret)
+				return ret;
+		}
+		ret = regmap_write(maxim->regmap, MAX25014_DISABLE, maxim->strings_mask);
+		if (ret)
+			return ret;
+	}
+
+	ret = regmap_write(maxim->regmap, MAX25014_IMODE, MAX25014_IMODE_HDIM);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(maxim->regmap, MAX25014_SETTING, &val);
+	if (ret)
+		return ret;
+
+	ret = regmap_write(maxim->regmap, MAX25014_SETTING,
+			   val & ~MAX25014_SETTING_FPWM);
+	if (ret)
+		return ret;
+
+	return regmap_write(maxim->regmap, MAX25014_ISET,
+			   maxim->iset | MAX25014_ISET_ENA |
+			   MAX25014_ISET_PSEN);
+}
+
+static int max25014_update_status(struct backlight_device *bl_dev)
+{
+	struct max25014 *maxim = bl_get_data(bl_dev);
+	uint32_t reg;
+	int ret;
+
+	reg  = TON_STEP * backlight_get_brightness(bl_dev);
+
+	/*
+	 * 18 bit number lowest, 2 bits in first register,
+	 * next lowest 8 in the L register, next 8 in the H register
+	 * Seemingly setting the strength of only one string controls all of
+	 * them, individual settings don't affect the outcome.
+	 */
+	ret = regmap_write(maxim->regmap, MAX25014_TON_1_4_LSB, reg & 0b00000011);
+	if (ret != 0)
+		return ret;
+	ret = regmap_write(maxim->regmap, MAX25014_TON1L, (reg >> 2) & 0b11111111);
+	if (ret != 0)
+		return ret;
+	return regmap_write(maxim->regmap, MAX25014_TON1H, (reg >> 10) & 0b11111111);
+}
+
+static const struct backlight_ops max25014_bl_ops = {
+	.options = BL_CORE_SUSPENDRESUME,
+	.update_status = max25014_update_status,
+};
+
+static int max25014_parse_dt(struct max25014 *maxim,
+			     uint32_t *initial_brightness)
+{
+	struct device *dev = &maxim->client->dev;
+	struct device_node *node = dev->of_node;
+	uint32_t strings[4];
+	int res, i;
+
+	res = of_property_count_u32_elems(node, "maxim,strings");
+	if (res == 4) {
+		of_property_read_u32_array(node, "maxim,strings", strings, 4);
+		for (i = 0; i < 4; i++) {
+			if (strings[i] == 0)
+				maxim->strings_mask |= 1 << i;
+	}
+	} else {
+		maxim->strings_mask = 0;
+	}
+
+	*initial_brightness = 50U;
+	of_property_read_u32(node, "default-brightness", initial_brightness);
+
+	maxim->iset = MAX25014_ISET_DEFAULT_100;
+	of_property_read_u32(node, "maxim,iset", &maxim->iset);
+
+	if (maxim->iset > 15)
+		return dev_err_probe(dev, -EINVAL,
+				     "Invalid iset, should be a value from 0-15, entered was %d\n",
+				     maxim->iset);
+
+	if (*initial_brightness > 100)
+		return dev_err_probe(dev, -EINVAL,
+				     "Invalid initial brightness, should be a value from 0-100, entered was %d\n",
+				     *initial_brightness);
+
+	return 0;
+}
+
+static int max25014_probe(struct i2c_client *cl)
+{
+	const struct i2c_device_id *id = i2c_client_get_device_id(cl);
+	struct backlight_properties props;
+	uint32_t initial_brightness = 50;
+	struct backlight_device *bl;
+	struct max25014 *maxim;
+	int ret;
+
+	maxim = devm_kzalloc(&cl->dev, sizeof(struct max25014), GFP_KERNEL);
+	if (!maxim)
+		return -ENOMEM;
+
+	maxim->client = cl;
+
+	ret = max25014_parse_dt(maxim, &initial_brightness);
+	if (ret)
+		return ret;
+
+	ret = devm_regulator_get_enable(&maxim->client->dev, "power");
+	if (ret)
+		return dev_err_probe(&maxim->client->dev, ret,
+				     "failed to get power-supply");
+
+	maxim->enable = devm_gpiod_get_optional(&maxim->client->dev, "enable",
+						GPIOD_OUT_HIGH);
+	if (IS_ERR(maxim->enable))
+		return dev_err_probe(&maxim->client->dev, PTR_ERR(maxim->enable),
+				    "failed to get enable gpio\n");
+
+	/* Datasheet Electrical Characteristics tSTARTUP 2ms */
+	fsleep(2000);
+
+	maxim->regmap = devm_regmap_init_i2c(cl, &max25014_regmap_config);
+	if (IS_ERR(maxim->regmap))
+		return dev_err_probe(&maxim->client->dev, PTR_ERR(maxim->regmap),
+			"failed to initialize the i2c regmap\n");
+
+	i2c_set_clientdata(cl, maxim);
+
+	ret = max25014_check_errors(maxim);
+	if (ret) /* error is already reported in the above function */
+		return ret;
+
+	ret = max25014_initial_power_state(maxim);
+	if (ret < 0)
+		return dev_err_probe(&maxim->client->dev, ret, "Could not get enabled state\n");
+
+	memset(&props, 0, sizeof(struct backlight_properties));
+	props.type = BACKLIGHT_PLATFORM;
+	props.max_brightness = MAX_BRIGHTNESS;
+	props.brightness = initial_brightness;
+	props.scale = BACKLIGHT_SCALE_LINEAR;
+	props.power = ret;
+
+	ret = max25014_configure(maxim, ret);
+	if (ret)
+		return dev_err_probe(&maxim->client->dev, ret, "device config error");
+
+	bl = devm_backlight_device_register(&maxim->client->dev, id->name,
+					    &maxim->client->dev, maxim,
+					    &max25014_bl_ops, &props);
+	if (IS_ERR(bl))
+		return dev_err_probe(&maxim->client->dev, PTR_ERR(bl),
+				    "failed to register backlight\n");
+
+	maxim->bl = bl;
+
+	backlight_update_status(maxim->bl);
+
+	return 0;
+}
+
+static void max25014_remove(struct i2c_client *cl)
+{
+	struct max25014 *maxim = i2c_get_clientdata(cl);
+
+	backlight_device_set_brightness(maxim->bl, 0);
+	gpiod_set_value_cansleep(maxim->enable, 0);
+}
+
+static const struct of_device_id max25014_dt_ids[] = {
+	{ .compatible = "maxim,max25014", },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, max25014_dt_ids);
+
+static const struct i2c_device_id max25014_ids[] = {
+	{ "max25014" },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, max25014_ids);
+
+static struct i2c_driver max25014_driver = {
+	.driver = {
+		.name = KBUILD_MODNAME,
+		.of_match_table = of_match_ptr(max25014_dt_ids),
+	},
+	.probe = max25014_probe,
+	.remove = max25014_remove,
+	.id_table = max25014_ids,
+};
+module_i2c_driver(max25014_driver);
+
+MODULE_DESCRIPTION("Maxim MAX25014 backlight driver");
+MODULE_AUTHOR("Maud Spierings <maudspierings@gocontroll.com>");
+MODULE_LICENSE("GPL");
diff --git a/drivers/video/console/Kconfig b/drivers/video/console/Kconfig
index 12f54480f57f8..9f81af3506daa 100644
--- a/drivers/video/console/Kconfig
+++ b/drivers/video/console/Kconfig
@@ -23,21 +23,6 @@ config VGA_CONSOLE
 
 	  Say Y.
 
-config MDA_CONSOLE
-	depends on VGA_CONSOLE && ISA
-	tristate "MDA text console (dual-headed)"
-	help
-	  Say Y here if you have an old MDA or monochrome Hercules graphics
-	  adapter in your system acting as a second head ( = video card). You
-	  will then be able to use two monitors with your Linux system. Do not
-	  say Y here if your MDA card is the primary card in your system; the
-	  normal VGA driver will handle it.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called mdacon.
-
-	  If unsure, say N.
-
 config SGI_NEWPORT_CONSOLE
 	tristate "SGI Newport Console support"
 	depends on SGI_IP22 && HAS_IOMEM
diff --git a/drivers/video/console/Makefile b/drivers/video/console/Makefile
index fd79016a0d95b..f1000605210c4 100644
--- a/drivers/video/console/Makefile
+++ b/drivers/video/console/Makefile
@@ -7,4 +7,3 @@ obj-$(CONFIG_DUMMY_CONSOLE)       += dummycon.o
 obj-$(CONFIG_SGI_NEWPORT_CONSOLE) += newport_con.o
 obj-$(CONFIG_STI_CONSOLE)         += sticon.o
 obj-$(CONFIG_VGA_CONSOLE)         += vgacon.o
-obj-$(CONFIG_MDA_CONSOLE)         += mdacon.o
diff --git a/drivers/video/console/mdacon.c b/drivers/video/console/mdacon.c
deleted file mode 100644
index d52cd99cd18b4..0000000000000
--- a/drivers/video/console/mdacon.c
+++ /dev/null
@@ -1,566 +0,0 @@
-/*
- *  linux/drivers/video/mdacon.c -- Low level MDA based console driver
- *
- *	(c) 1998 Andrew Apted <ajapted@netspace.net.au>
- *
- *      including portions (c) 1995-1998 Patrick Caulfield.
- *
- *      slight improvements (c) 2000 Edward Betts <edward@debian.org>
- *
- *  This file is based on the VGA console driver (vgacon.c):
- *	
- *	Created 28 Sep 1997 by Geert Uytterhoeven
- *
- *	Rewritten by Martin Mares <mj@ucw.cz>, July 1998
- *
- *  and on the old console.c, vga.c and vesa_blank.c drivers:
- *
- *	Copyright (C) 1991, 1992  Linus Torvalds
- *			    1995  Jay Estabrook
- *
- *  This file is subject to the terms and conditions of the GNU General Public
- *  License.  See the file COPYING in the main directory of this archive for
- *  more details.
- *
- *  Changelog:
- *  Paul G. (03/2001) Fix mdacon= boot prompt to use __setup().
- */
-
-#include <linux/types.h>
-#include <linux/fs.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/console.h>
-#include <linux/string.h>
-#include <linux/kd.h>
-#include <linux/vt_kern.h>
-#include <linux/vt_buffer.h>
-#include <linux/selection.h>
-#include <linux/spinlock.h>
-#include <linux/ioport.h>
-#include <linux/delay.h>
-#include <linux/init.h>
-
-#include <asm/io.h>
-#include <asm/vga.h>
-
-static DEFINE_SPINLOCK(mda_lock);
-
-/* description of the hardware layout */
-
-static u16		*mda_vram_base;		/* Base of video memory */
-static unsigned long	mda_vram_len;		/* Size of video memory */
-static unsigned int	mda_num_columns;	/* Number of text columns */
-static unsigned int	mda_num_lines;		/* Number of text lines */
-
-static unsigned int	mda_index_port;		/* Register select port */
-static unsigned int	mda_value_port;		/* Register value port */
-static unsigned int	mda_mode_port;		/* Mode control port */
-static unsigned int	mda_status_port;	/* Status and Config port */
-static unsigned int	mda_gfx_port;		/* Graphics control port */
-
-/* current hardware state */
-
-static int	mda_cursor_loc=-1;
-static int	mda_cursor_size_from=-1;
-static int	mda_cursor_size_to=-1;
-
-static enum { TYPE_MDA, TYPE_HERC, TYPE_HERCPLUS, TYPE_HERCCOLOR } mda_type;
-static char *mda_type_name;
-
-/* console information */
-
-static int	mda_first_vc = 13;
-static int	mda_last_vc  = 16;
-
-static struct vc_data	*mda_display_fg = NULL;
-
-module_param(mda_first_vc, int, 0);
-MODULE_PARM_DESC(mda_first_vc, "First virtual console. Default: 13");
-module_param(mda_last_vc, int, 0);
-MODULE_PARM_DESC(mda_last_vc, "Last virtual console. Default: 16");
-
-/* MDA register values
- */
-
-#define MDA_CURSOR_BLINKING	0x00
-#define MDA_CURSOR_OFF		0x20
-#define MDA_CURSOR_SLOWBLINK	0x60
-
-#define MDA_MODE_GRAPHICS	0x02
-#define MDA_MODE_VIDEO_EN	0x08
-#define MDA_MODE_BLINK_EN	0x20
-#define MDA_MODE_GFX_PAGE1	0x80
-
-#define MDA_STATUS_HSYNC	0x01
-#define MDA_STATUS_VSYNC	0x80
-#define MDA_STATUS_VIDEO	0x08
-
-#define MDA_CONFIG_COL132	0x08
-#define MDA_GFX_MODE_EN		0x01
-#define MDA_GFX_PAGE_EN		0x02
-
-
-/*
- * MDA could easily be classified as "pre-dinosaur hardware".
- */
-
-static void write_mda_b(unsigned int val, unsigned char reg)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mda_lock, flags);	
-
-	outb_p(reg, mda_index_port); 
-	outb_p(val, mda_value_port);
-
-	spin_unlock_irqrestore(&mda_lock, flags);
-}
-
-static void write_mda_w(unsigned int val, unsigned char reg)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mda_lock, flags);
-
-	outb_p(reg,   mda_index_port); outb_p(val >> 8,   mda_value_port);
-	outb_p(reg+1, mda_index_port); outb_p(val & 0xff, mda_value_port);
-
-	spin_unlock_irqrestore(&mda_lock, flags);
-}
-
-#ifdef TEST_MDA_B
-static int test_mda_b(unsigned char val, unsigned char reg)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&mda_lock, flags);
-
-	outb_p(reg, mda_index_port); 
-	outb  (val, mda_value_port);
-
-	udelay(20); val = (inb_p(mda_value_port) == val);
-
-	spin_unlock_irqrestore(&mda_lock, flags);
-	return val;
-}
-#endif
-
-static inline void mda_set_cursor(unsigned int location) 
-{
-	if (mda_cursor_loc == location)
-		return;
-
-	write_mda_w(location >> 1, 0x0e);
-
-	mda_cursor_loc = location;
-}
-
-static inline void mda_set_cursor_size(int from, int to)
-{
-	if (mda_cursor_size_from==from && mda_cursor_size_to==to)
-		return;
-	
-	if (from > to) {
-		write_mda_b(MDA_CURSOR_OFF, 0x0a);	/* disable cursor */
-	} else {
-		write_mda_b(from, 0x0a);	/* cursor start */
-		write_mda_b(to,   0x0b);	/* cursor end */
-	}
-
-	mda_cursor_size_from = from;
-	mda_cursor_size_to   = to;
-}
-
-
-#ifndef MODULE
-static int __init mdacon_setup(char *str)
-{
-	/* command line format: mdacon=<first>,<last> */
-
-	int ints[3];
-
-	str = get_options(str, ARRAY_SIZE(ints), ints);
-
-	if (ints[0] < 2)
-		return 0;
-
-	if (ints[1] < 1 || ints[1] > MAX_NR_CONSOLES || 
-	    ints[2] < 1 || ints[2] > MAX_NR_CONSOLES)
-		return 0;
-
-	mda_first_vc = ints[1];
-	mda_last_vc  = ints[2];
-	return 1;
-}
-
-__setup("mdacon=", mdacon_setup);
-#endif
-
-static int mda_detect(void)
-{
-	int count=0;
-	u16 *p, p_save;
-	u16 *q, q_save;
-
-	/* do a memory check */
-
-	p = mda_vram_base;
-	q = mda_vram_base + 0x01000 / 2;
-
-	p_save = scr_readw(p);
-	q_save = scr_readw(q);
-
-	scr_writew(0xAA55, p);
-	if (scr_readw(p) == 0xAA55)
-		count++;
-
-	scr_writew(0x55AA, p);
-	if (scr_readw(p) == 0x55AA)
-		count++;
-
-	scr_writew(p_save, p);
-
-	if (count != 2) {
-		return 0;
-	}
-
-	/* check if we have 4K or 8K */
-
-	scr_writew(0xA55A, q);
-	scr_writew(0x0000, p);
-	if (scr_readw(q) == 0xA55A)
-		count++;
-	
-	scr_writew(0x5AA5, q);
-	scr_writew(0x0000, p);
-	if (scr_readw(q) == 0x5AA5)
-		count++;
-
-	scr_writew(p_save, p);
-	scr_writew(q_save, q);
-	
-	if (count == 4) {
-		mda_vram_len = 0x02000;
-	}
-	
-	/* Ok, there is definitely a card registering at the correct
-	 * memory location, so now we do an I/O port test.
-	 */
-
-#ifdef TEST_MDA_B
-	/* Edward: These two mess `tests' mess up my cursor on bootup */
-
-	/* cursor low register */
-	if (!test_mda_b(0x66, 0x0f))
-		return 0;
-
-	/* cursor low register */
-	if (!test_mda_b(0x99, 0x0f))
-		return 0;
-#endif
-
-	/* See if the card is a Hercules, by checking whether the vsync
-	 * bit of the status register is changing.  This test lasts for
-	 * approximately 1/10th of a second.
-	 */
-	
-	p_save = q_save = inb_p(mda_status_port) & MDA_STATUS_VSYNC;
-
-	for (count = 0; count < 50000 && p_save == q_save; count++) {
-		q_save = inb(mda_status_port) & MDA_STATUS_VSYNC;
-		udelay(2);
-	}
-
-	if (p_save != q_save) {
-		switch (inb_p(mda_status_port) & 0x70) {
-		case 0x10:
-			mda_type = TYPE_HERCPLUS;
-			mda_type_name = "HerculesPlus";
-			break;
-		case 0x50:
-			mda_type = TYPE_HERCCOLOR;
-			mda_type_name = "HerculesColor";
-			break;
-		default:
-			mda_type = TYPE_HERC;
-			mda_type_name = "Hercules";
-			break;
-		}
-	}
-
-	return 1;
-}
-
-static void mda_initialize(void)
-{
-	write_mda_b(97, 0x00);		/* horizontal total */
-	write_mda_b(80, 0x01);		/* horizontal displayed */
-	write_mda_b(82, 0x02);		/* horizontal sync pos */
-	write_mda_b(15, 0x03);		/* horizontal sync width */
-
-	write_mda_b(25, 0x04);		/* vertical total */
-	write_mda_b(6,  0x05);		/* vertical total adjust */
-	write_mda_b(25, 0x06);		/* vertical displayed */
-	write_mda_b(25, 0x07);		/* vertical sync pos */
-
-	write_mda_b(2,  0x08);		/* interlace mode */
-	write_mda_b(13, 0x09);		/* maximum scanline */
-	write_mda_b(12, 0x0a);		/* cursor start */
-	write_mda_b(13, 0x0b);		/* cursor end */
-
-	write_mda_w(0x0000, 0x0c);	/* start address */
-	write_mda_w(0x0000, 0x0e);	/* cursor location */
-
-	outb_p(MDA_MODE_VIDEO_EN | MDA_MODE_BLINK_EN, mda_mode_port);
-	outb_p(0x00, mda_status_port);
-	outb_p(0x00, mda_gfx_port);
-}
-
-static const char *mdacon_startup(void)
-{
-	mda_num_columns = 80;
-	mda_num_lines   = 25;
-
-	mda_vram_len  = 0x01000;
-	mda_vram_base = (u16 *)VGA_MAP_MEM(0xb0000, mda_vram_len);
-
-	mda_index_port  = 0x3b4;
-	mda_value_port  = 0x3b5;
-	mda_mode_port   = 0x3b8;
-	mda_status_port = 0x3ba;
-	mda_gfx_port    = 0x3bf;
-
-	mda_type = TYPE_MDA;
-	mda_type_name = "MDA";
-
-	if (! mda_detect()) {
-		printk("mdacon: MDA card not detected.\n");
-		return NULL;
-	}
-
-	if (mda_type != TYPE_MDA) {
-		mda_initialize();
-	}
-
-	/* cursor looks ugly during boot-up, so turn it off */
-	mda_set_cursor(mda_vram_len - 1);
-
-	printk("mdacon: %s with %ldK of memory detected.\n",
-		mda_type_name, mda_vram_len/1024);
-
-	return "MDA-2";
-}
-
-static void mdacon_init(struct vc_data *c, bool init)
-{
-	c->vc_complement_mask = 0x0800;	 /* reverse video */
-	c->vc_display_fg = &mda_display_fg;
-
-	if (init) {
-		c->vc_cols = mda_num_columns;
-		c->vc_rows = mda_num_lines;
-	} else
-		vc_resize(c, mda_num_columns, mda_num_lines);
-
-	/* make the first MDA console visible */
-
-	if (mda_display_fg == NULL)
-		mda_display_fg = c;
-}
-
-static void mdacon_deinit(struct vc_data *c)
-{
-	/* con_set_default_unimap(c->vc_num); */
-
-	if (mda_display_fg == c)
-		mda_display_fg = NULL;
-}
-
-static inline u16 mda_convert_attr(u16 ch)
-{
-	u16 attr = 0x0700;
-
-	/* Underline and reverse-video are mutually exclusive on MDA.
-	 * Since reverse-video is used for cursors and selected areas,
-	 * it takes precedence. 
-	 */
-
-	if (ch & 0x0800)	attr = 0x7000;	/* reverse */
-	else if (ch & 0x0400)	attr = 0x0100;	/* underline */
-
-	return ((ch & 0x0200) << 2) | 		/* intensity */ 
-		(ch & 0x8000) |			/* blink */ 
-		(ch & 0x00ff) | attr;
-}
-
-static u8 mdacon_build_attr(struct vc_data *c, u8 color,
-			    enum vc_intensity intensity,
-			    bool blink, bool underline, bool reverse,
-			    bool italic)
-{
-	/* The attribute is just a bit vector:
-	 *
-	 *	Bit 0..1 : intensity (0..2)
-	 *	Bit 2    : underline
-	 *	Bit 3    : reverse
-	 *	Bit 7    : blink
-	 */
-
-	return (intensity & VCI_MASK) |
-		(underline << 2) |
-		(reverse << 3) |
-		(italic << 4) |
-		(blink << 7);
-}
-
-static void mdacon_invert_region(struct vc_data *c, u16 *p, int count)
-{
-	for (; count > 0; count--) {
-		scr_writew(scr_readw(p) ^ 0x0800, p);
-		p++;
-	}
-}
-
-static inline u16 *mda_addr(unsigned int x, unsigned int y)
-{
-	return mda_vram_base + y * mda_num_columns + x;
-}
-
-static void mdacon_putcs(struct vc_data *c, const u16 *s, unsigned int count,
-			 unsigned int y, unsigned int x)
-{
-	u16 *dest = mda_addr(x, y);
-
-	for (; count > 0; count--) {
-		scr_writew(mda_convert_attr(scr_readw(s++)), dest++);
-	}
-}
-
-static void mdacon_clear(struct vc_data *c, unsigned int y, unsigned int x,
-			 unsigned int width)
-{
-	u16 *dest = mda_addr(x, y);
-	u16 eattr = mda_convert_attr(c->vc_video_erase_char);
-
-	scr_memsetw(dest, eattr, width * 2);
-}
-
-static bool mdacon_switch(struct vc_data *c)
-{
-	return true;	/* redrawing needed */
-}
-
-static bool mdacon_blank(struct vc_data *c, enum vesa_blank_mode blank,
-			 bool mode_switch)
-{
-	if (mda_type == TYPE_MDA) {
-		if (blank) 
-			scr_memsetw(mda_vram_base,
-				mda_convert_attr(c->vc_video_erase_char),
-				c->vc_screenbuf_size);
-		/* Tell console.c that it has to restore the screen itself */
-		return true;
-	} else {
-		if (blank)
-			outb_p(0x00, mda_mode_port);	/* disable video */
-		else
-			outb_p(MDA_MODE_VIDEO_EN | MDA_MODE_BLINK_EN, 
-				mda_mode_port);
-		return false;
-	}
-}
-
-static void mdacon_cursor(struct vc_data *c, bool enable)
-{
-	if (!enable) {
-		mda_set_cursor(mda_vram_len - 1);
-		return;
-	}
-
-	mda_set_cursor(c->state.y * mda_num_columns * 2 + c->state.x * 2);
-
-	switch (CUR_SIZE(c->vc_cursor_type)) {
-
-		case CUR_LOWER_THIRD:	mda_set_cursor_size(10, 13); break;
-		case CUR_LOWER_HALF:	mda_set_cursor_size(7,  13); break;
-		case CUR_TWO_THIRDS:	mda_set_cursor_size(4,  13); break;
-		case CUR_BLOCK:		mda_set_cursor_size(1,  13); break;
-		case CUR_NONE:		mda_set_cursor_size(14, 13); break;
-		default:		mda_set_cursor_size(12, 13); break;
-	}
-}
-
-static bool mdacon_scroll(struct vc_data *c, unsigned int t, unsigned int b,
-		enum con_scroll dir, unsigned int lines)
-{
-	u16 eattr = mda_convert_attr(c->vc_video_erase_char);
-
-	if (!lines)
-		return false;
-
-	if (lines > c->vc_rows)   /* maximum realistic size */
-		lines = c->vc_rows;
-
-	switch (dir) {
-
-	case SM_UP:
-		scr_memmovew(mda_addr(0, t), mda_addr(0, t + lines),
-				(b-t-lines)*mda_num_columns*2);
-		scr_memsetw(mda_addr(0, b - lines), eattr,
-				lines*mda_num_columns*2);
-		break;
-
-	case SM_DOWN:
-		scr_memmovew(mda_addr(0, t + lines), mda_addr(0, t),
-				(b-t-lines)*mda_num_columns*2);
-		scr_memsetw(mda_addr(0, t), eattr, lines*mda_num_columns*2);
-		break;
-	}
-
-	return false;
-}
-
-
-/*
- *  The console `switch' structure for the MDA based console
- */
-
-static const struct consw mda_con = {
-	.owner =		THIS_MODULE,
-	.con_startup =		mdacon_startup,
-	.con_init =		mdacon_init,
-	.con_deinit =		mdacon_deinit,
-	.con_clear =		mdacon_clear,
-	.con_putcs =		mdacon_putcs,
-	.con_cursor =		mdacon_cursor,
-	.con_scroll =		mdacon_scroll,
-	.con_switch =		mdacon_switch,
-	.con_blank =		mdacon_blank,
-	.con_build_attr =	mdacon_build_attr,
-	.con_invert_region =	mdacon_invert_region,
-};
-
-int __init mda_console_init(void)
-{
-	int err;
-
-	if (mda_first_vc > mda_last_vc)
-		return 1;
-	console_lock();
-	err = do_take_over_console(&mda_con, mda_first_vc-1, mda_last_vc-1, 0);
-	console_unlock();
-	return err;
-}
-
-static void __exit mda_console_exit(void)
-{
-	give_up_console(&mda_con);
-}
-
-module_init(mda_console_init);
-module_exit(mda_console_exit);
-
-MODULE_DESCRIPTION("MDA based console driver");
-MODULE_LICENSE("GPL");
-
diff --git a/drivers/video/fbdev/Kconfig b/drivers/video/fbdev/Kconfig
index 1c73d560f196c..085d3a202148b 100644
--- a/drivers/video/fbdev/Kconfig
+++ b/drivers/video/fbdev/Kconfig
@@ -453,19 +453,6 @@ config FB_N411
 	  This enables support for the Apollo display controller in its
 	  Hecuba form using the n411 devkit.
 
-config FB_HGA
-	tristate "Hercules mono graphics support"
-	depends on FB && X86
-	select FB_IOMEM_FOPS
-	help
-	  Say Y here if you have a Hercules mono graphics card.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called hgafb.
-
-	  As this card technology is at least 25 years old,
-	  most people will answer N here.
-
 config FB_GBE
 	bool "SGI Graphics Backend frame buffer support"
 	depends on (FB = y) && HAS_IOMEM
diff --git a/drivers/video/fbdev/Makefile b/drivers/video/fbdev/Makefile
index 36a18d958ba0d..0b17c878154d4 100644
--- a/drivers/video/fbdev/Makefile
+++ b/drivers/video/fbdev/Makefile
@@ -59,7 +59,6 @@ obj-$(CONFIG_FB_ATARI)            += atafb.o c2p_iplan2.o atafb_mfb.o \
 obj-$(CONFIG_FB_MAC)              += macfb.o
 obj-$(CONFIG_FB_HECUBA)           += hecubafb.o
 obj-$(CONFIG_FB_N411)             += n411.o
-obj-$(CONFIG_FB_HGA)              += hgafb.o
 obj-$(CONFIG_FB_XVR500)           += sunxvr500.o
 obj-$(CONFIG_FB_XVR2500)          += sunxvr2500.o
 obj-$(CONFIG_FB_XVR1000)          += sunxvr1000.o
diff --git a/drivers/video/fbdev/arkfb.c b/drivers/video/fbdev/arkfb.c
index 866c1165704ed..195dbf4a5142d 100644
--- a/drivers/video/fbdev/arkfb.c
+++ b/drivers/video/fbdev/arkfb.c
@@ -1167,8 +1167,8 @@ static const struct dev_pm_ops ark_pci_pm_ops = {
 /* List of boards that we are trying to support */
 
 static const struct pci_device_id ark_devices[] = {
-	{PCI_DEVICE(0xEDD8, 0xA099)},
-	{0, 0, 0, 0, 0, 0, 0}
+	{ PCI_DEVICE(0xEDD8, 0xA099) },
+	{ }
 };
 
 
diff --git a/drivers/video/fbdev/atmel_lcdfb.c b/drivers/video/fbdev/atmel_lcdfb.c
index 9dfbc53102107..87406a5a2dcf3 100644
--- a/drivers/video/fbdev/atmel_lcdfb.c
+++ b/drivers/video/fbdev/atmel_lcdfb.c
@@ -21,7 +21,6 @@
 #include <linux/gpio/consumer.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/of_device.h>
 #include <video/of_videomode.h>
 #include <video/of_display_timing.h>
 #include <linux/regulator/consumer.h>
@@ -56,7 +55,7 @@ struct atmel_lcdfb_info {
 
 	struct atmel_lcdfb_pdata pdata;
 
-	struct atmel_lcdfb_config *config;
+	const struct atmel_lcdfb_config *config;
 	struct regulator	*reg_lcd;
 };
 
@@ -930,8 +929,7 @@ static int atmel_lcdfb_of_init(struct atmel_lcdfb_info *sinfo)
 	int ret;
 	int i;
 
-	sinfo->config = (struct atmel_lcdfb_config*)
-		of_match_device(atmel_lcdfb_dt_ids, dev)->data;
+	sinfo->config = of_device_get_match_data(dev);
 
 	display_np = of_parse_phandle(np, "display", 0);
 	if (!display_np) {
diff --git a/drivers/video/fbdev/aty/aty128fb.c b/drivers/video/fbdev/aty/aty128fb.c
index f55b4c7609a8c..bcb10e66221c8 100644
--- a/drivers/video/fbdev/aty/aty128fb.c
+++ b/drivers/video/fbdev/aty/aty128fb.c
@@ -180,101 +180,54 @@ static const struct dev_pm_ops aty128_pci_pm_ops = {
 
 /* supported Rage128 chipsets */
 static const struct pci_device_id aty128_pci_tbl[] = {
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_LE,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_M3_pci },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_LF,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_M3 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_MF,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_M4 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_ML,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_M4 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PA,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PB,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PC,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PD,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro_pci },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PE,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PF,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PG,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PH,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PI,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PJ,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PK,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PM,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PN,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PO,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PP,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro_pci },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PQ,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PR,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro_pci },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PS,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PT,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PU,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PV,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PW,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_PX,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pro },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_RE,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pci },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_RF,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_RG,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_RK,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pci },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_RL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_SE,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_SF,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_pci },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_SG,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_SH,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_SK,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_SL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_SM,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_SN,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128 },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_TF,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_ultra },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_TL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_ultra },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_TR,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_ultra },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_TS,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_ultra },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_TT,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_ultra },
-	{ PCI_VENDOR_ID_ATI, PCI_DEVICE_ID_ATI_RAGE128_TU,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, rage_128_ultra },
-	{ 0, }
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_LE), .driver_data = rage_M3_pci },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_LF), .driver_data = rage_M3 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_MF), .driver_data = rage_M4 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_ML), .driver_data = rage_M4 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PA), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PB), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PC), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PD), .driver_data = rage_128_pro_pci },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PE), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PF), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PG), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PH), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PI), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PJ), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PK), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PL), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PM), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PN), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PO), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PP), .driver_data = rage_128_pro_pci },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PQ), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PR), .driver_data = rage_128_pro_pci },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PS), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PT), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PU), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PV), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PW), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_PX), .driver_data = rage_128_pro },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_RE), .driver_data = rage_128_pci },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_RF), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_RG), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_RK), .driver_data = rage_128_pci },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_RL), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_SE), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_SF), .driver_data = rage_128_pci },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_SG), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_SH), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_SK), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_SL), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_SM), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_SN), .driver_data = rage_128 },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_TF), .driver_data = rage_128_ultra },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_TL), .driver_data = rage_128_ultra },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_TR), .driver_data = rage_128_ultra },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_TS), .driver_data = rage_128_ultra },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_TT), .driver_data = rage_128_ultra },
+	{ PCI_VDEVICE(ATI, PCI_DEVICE_ID_ATI_RAGE128_TU), .driver_data = rage_128_ultra },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, aty128_pci_tbl);
diff --git a/drivers/video/fbdev/aty/radeon_base.c b/drivers/video/fbdev/aty/radeon_base.c
index cb006484831b6..b6b058cee7516 100644
--- a/drivers/video/fbdev/aty/radeon_base.c
+++ b/drivers/video/fbdev/aty/radeon_base.c
@@ -95,7 +95,7 @@
 #define MIN_MAPPED_VRAM	(1024*768*1)
 
 #define CHIP_DEF(id, family, flags)					\
-	{ PCI_VENDOR_ID_ATI, id, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (flags) | (CHIP_FAMILY_##family) }
+	{ PCI_DEVICE(PCI_VENDOR_ID_ATI, id), .driver_data = (flags) | (CHIP_FAMILY_##family) }
 
 static const struct pci_device_id radeonfb_pci_table[] = {
         /* Radeon Xpress 200m */
@@ -2476,6 +2476,7 @@ static int radeonfb_pci_register(struct pci_dev *pdev,
 	return 0;
 err_unmap_fb:
 	iounmap(rinfo->fb_base);
+	fb_destroy_modelist(&info->modelist);
 err_unmap_rom:
 	kfree(rinfo->mon1_EDID);
 	kfree(rinfo->mon2_EDID);
diff --git a/drivers/video/fbdev/broadsheetfb.c b/drivers/video/fbdev/broadsheetfb.c
index c8ba098a8c42f..582f1ee4c9b6d 100644
--- a/drivers/video/fbdev/broadsheetfb.c
+++ b/drivers/video/fbdev/broadsheetfb.c
@@ -1072,12 +1072,14 @@ static int broadsheetfb_probe(struct platform_device *dev)
 	info->flags = FBINFO_VIRTFB;
 
 	info->fbdefio = &broadsheetfb_defio;
-	fb_deferred_io_init(info);
+	retval = fb_deferred_io_init(info);
+	if (retval)
+		goto err_vfree;
 
 	retval = fb_alloc_cmap(&info->cmap, 16, 0);
 	if (retval < 0) {
 		dev_err(&dev->dev, "Failed to allocate colormap\n");
-		goto err_vfree;
+		goto err_fbdefio;
 	}
 
 	/* set cmap */
@@ -1121,6 +1123,8 @@ static int broadsheetfb_probe(struct platform_device *dev)
 	board->cleanup(par);
 err_cmap:
 	fb_dealloc_cmap(&info->cmap);
+err_fbdefio:
+	fb_deferred_io_cleanup(info);
 err_vfree:
 	vfree(videomemory);
 err_fb_rel:
diff --git a/drivers/video/fbdev/carminefb.c b/drivers/video/fbdev/carminefb.c
index bd4bff6a24844..fca50b7961eb3 100644
--- a/drivers/video/fbdev/carminefb.c
+++ b/drivers/video/fbdev/carminefb.c
@@ -589,6 +589,7 @@ static int alloc_carmine_fb(void __iomem *regs, void __iomem *smem_base,
 	return 0;
 
 err_dealloc_cmap:
+	fb_destroy_modelist(&info->modelist);
 	fb_dealloc_cmap(&info->cmap);
 err_free_fb:
 	framebuffer_release(info);
@@ -753,9 +754,8 @@ static void carminefb_remove(struct pci_dev *dev)
 
 #define PCI_VENDOR_ID_FUJITU_LIMITED 0x10cf
 static struct pci_device_id carmine_devices[] = {
-{
-	PCI_DEVICE(PCI_VENDOR_ID_FUJITU_LIMITED, 0x202b)},
-	{0, 0, 0, 0, 0, 0, 0}
+	{ PCI_DEVICE(PCI_VENDOR_ID_FUJITU_LIMITED, 0x202b) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, carmine_devices);
diff --git a/drivers/video/fbdev/chipsfb.c b/drivers/video/fbdev/chipsfb.c
index 33caf0b99a452..946e30fcb6a41 100644
--- a/drivers/video/fbdev/chipsfb.c
+++ b/drivers/video/fbdev/chipsfb.c
@@ -526,4 +526,5 @@ static void __exit chipsfb_exit(void)
 	pci_unregister_driver(&chipsfb_driver);
 }
 
+MODULE_DESCRIPTION("Chips & Technologies 65550 frame buffer driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/video/fbdev/cirrusfb.c b/drivers/video/fbdev/cirrusfb.c
index e29217e476ea4..2693b5cc053f3 100644
--- a/drivers/video/fbdev/cirrusfb.c
+++ b/drivers/video/fbdev/cirrusfb.c
@@ -253,7 +253,7 @@ static const struct cirrusfb_board_info_rec {
 
 #ifdef CONFIG_PCI
 #define CHIP(id, btype) \
-	{ PCI_VENDOR_ID_CIRRUS, id, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (btype) }
+	{ PCI_VDEVICE(CIRRUS, id), .driver_data = (btype) }
 
 static struct pci_device_id cirrusfb_pci_table[] = {
 	CHIP(PCI_DEVICE_ID_CIRRUS_5436, BT_ALPINE),
diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c
index b0e3e765360d1..91a525a570dc2 100644
--- a/drivers/video/fbdev/core/fbcon.c
+++ b/drivers/video/fbdev/core/fbcon.c
@@ -1440,8 +1440,7 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var,
 	struct vc_data **default_mode, *vc;
 	struct vc_data *svc;
 	struct fbcon_par *par = info->fbcon_par;
-	int rows, cols;
-	unsigned long ret = 0;
+	int rows, cols, ret;
 
 	p = &fb_display[unit];
 
@@ -2602,8 +2601,9 @@ void fbcon_suspended(struct fb_info *info)
 		return;
 	vc = vc_cons[par->currcon].d;
 
-	/* Clear cursor, restore saved data */
-	fbcon_cursor(vc, false);
+	/* Clear cursor, restore saved data when in text mode */
+	if (con_is_visible(vc) && (vc->vc_mode == KD_TEXT))
+		fbcon_cursor(vc, false);
 }
 
 void fbcon_resumed(struct fb_info *info)
@@ -2615,7 +2615,9 @@ void fbcon_resumed(struct fb_info *info)
 		return;
 	vc = vc_cons[par->currcon].d;
 
-	update_screen(vc);
+	/* Update screen when in text mode only */
+	if (con_is_visible(vc) && (vc->vc_mode == KD_TEXT))
+		update_screen(vc);
 }
 
 static void fbcon_modechanged(struct fb_info *info)
diff --git a/drivers/video/fbdev/cyber2000fb.c b/drivers/video/fbdev/cyber2000fb.c
index 2d12f8e96c7ed..3daba94c6c13e 100644
--- a/drivers/video/fbdev/cyber2000fb.c
+++ b/drivers/video/fbdev/cyber2000fb.c
@@ -1796,16 +1796,22 @@ static int __maybe_unused cyberpro_pci_resume(struct device *dev)
 
 static struct pci_device_id cyberpro_pci_table[] = {
 /*	Not yet
- *	{ PCI_VENDOR_ID_INTERG, PCI_DEVICE_ID_INTERG_1682,
- *		PCI_ANY_ID, PCI_ANY_ID, 0, 0, ID_IGA_1682 },
+ *	{
+ *		PCI_VDEVICE(INTERG, PCI_DEVICE_ID_INTERG_1682),
+ *		.driver_data = ID_IGA_1682,
+ *	},
  */
-	{ PCI_VENDOR_ID_INTERG, PCI_DEVICE_ID_INTERG_2000,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0, ID_CYBERPRO_2000 },
-	{ PCI_VENDOR_ID_INTERG, PCI_DEVICE_ID_INTERG_2010,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0, ID_CYBERPRO_2010 },
-	{ PCI_VENDOR_ID_INTERG, PCI_DEVICE_ID_INTERG_5000,
-		PCI_ANY_ID, PCI_ANY_ID, 0, 0, ID_CYBERPRO_5000 },
-	{ 0, }
+	{
+		PCI_VDEVICE(INTERG, PCI_DEVICE_ID_INTERG_2000),
+		.driver_data = ID_CYBERPRO_2000,
+	}, {
+		PCI_VDEVICE(INTERG, PCI_DEVICE_ID_INTERG_2010),
+		.driver_data = ID_CYBERPRO_2010,
+	}, {
+		PCI_VDEVICE(INTERG, PCI_DEVICE_ID_INTERG_5000),
+		.driver_data = ID_CYBERPRO_5000,
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, cyberpro_pci_table);
diff --git a/drivers/video/fbdev/efifb.c b/drivers/video/fbdev/efifb.c
index 47ebc0107209e..ad8dec7807c3a 100644
--- a/drivers/video/fbdev/efifb.c
+++ b/drivers/video/fbdev/efifb.c
@@ -377,6 +377,7 @@ static int efifb_probe(struct platform_device *dev)
 	if (fb_get_options("efifb", &option))
 		return -ENODEV;
 	efifb_setup(si, option);
+	kfree(option);
 
 	/* We don't get linelength from UGA Draw Protocol, only from
 	 * EFI Graphics Protocol.  So if it's not in DMI, and it's not
diff --git a/drivers/video/fbdev/geode/gx1fb_core.c b/drivers/video/fbdev/geode/gx1fb_core.c
index a1919c1934aca..7cca46891aeff 100644
--- a/drivers/video/fbdev/geode/gx1fb_core.c
+++ b/drivers/video/fbdev/geode/gx1fb_core.c
@@ -423,10 +423,12 @@ static void __init gx1fb_setup(char *options)
 #endif
 
 static struct pci_device_id gx1fb_id_table[] = {
-	{ PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5530_VIDEO,
-	  PCI_ANY_ID, PCI_ANY_ID, PCI_BASE_CLASS_DISPLAY << 16,
-	  0xff0000, 0 },
-	{ 0, }
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5530_VIDEO),
+		.class = PCI_BASE_CLASS_DISPLAY << 16,
+		.class_mask = 0xff0000,
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, gx1fb_id_table);
diff --git a/drivers/video/fbdev/hecubafb.c b/drivers/video/fbdev/hecubafb.c
index 3547d58a29cf2..dd2af980f3d8b 100644
--- a/drivers/video/fbdev/hecubafb.c
+++ b/drivers/video/fbdev/hecubafb.c
@@ -192,7 +192,9 @@ static int hecubafb_probe(struct platform_device *dev)
 	info->flags = FBINFO_VIRTFB;
 
 	info->fbdefio = &hecubafb_defio;
-	fb_deferred_io_init(info);
+	retval = fb_deferred_io_init(info);
+	if (retval)
+		goto err_fbdefio;
 
 	retval = register_framebuffer(info);
 	if (retval < 0)
@@ -209,6 +211,8 @@ static int hecubafb_probe(struct platform_device *dev)
 
 	return 0;
 err_fbreg:
+	fb_deferred_io_cleanup(info);
+err_fbdefio:
 	framebuffer_release(info);
 err_fballoc:
 	vfree(videomemory);
diff --git a/drivers/video/fbdev/hgafb.c b/drivers/video/fbdev/hgafb.c
deleted file mode 100644
index d32fd1c5217c6..0000000000000
--- a/drivers/video/fbdev/hgafb.c
+++ /dev/null
@@ -1,685 +0,0 @@
-/*
- * linux/drivers/video/hgafb.c -- Hercules graphics adaptor frame buffer device
- *
- *      Created 25 Nov 1999 by Ferenc Bakonyi (fero@drama.obuda.kando.hu)
- *      Based on skeletonfb.c by Geert Uytterhoeven and
- *               mdacon.c by Andrew Apted
- *
- * History:
- *
- * - Revision 0.1.8 (23 Oct 2002): Ported to new framebuffer api.
- *
- * - Revision 0.1.7 (23 Jan 2001): fix crash resulting from MDA only cards
- *				   being detected as Hercules.	 (Paul G.)
- * - Revision 0.1.6 (17 Aug 2000): new style structs
- *                                 documentation
- * - Revision 0.1.5 (13 Mar 2000): spinlocks instead of saveflags();cli();etc
- *                                 minor fixes
- * - Revision 0.1.4 (24 Jan 2000): fixed a bug in hga_card_detect() for
- *                                  HGA-only systems
- * - Revision 0.1.3 (22 Jan 2000): modified for the new fb_info structure
- *                                 screen is cleared after rmmod
- *                                 virtual resolutions
- *                                 module parameter 'nologo={0|1}'
- *                                 the most important: boot logo :)
- * - Revision 0.1.0  (6 Dec 1999): faster scrolling and minor fixes
- * - First release  (25 Nov 1999)
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License.  See the file COPYING in the main directory of this archive
- * for more details.
- */
-
-#include <linux/module.h>
-#include <linux/kernel.h>
-#include <linux/errno.h>
-#include <linux/spinlock.h>
-#include <linux/string.h>
-#include <linux/mm.h>
-#include <linux/delay.h>
-#include <linux/fb.h>
-#include <linux/init.h>
-#include <linux/ioport.h>
-#include <linux/platform_device.h>
-#include <asm/io.h>
-#include <asm/vga.h>
-
-#if 0
-#define DPRINTK(args...) printk(KERN_DEBUG __FILE__": " ##args)
-#else
-#define DPRINTK(args...)
-#endif
-
-#if 0
-#define CHKINFO(ret) if (info != &fb_info) { printk(KERN_DEBUG __FILE__": This should never happen, line:%d \n", __LINE__); return ret; }
-#else
-#define CHKINFO(ret)
-#endif
-
-/* Description of the hardware layout */
-
-static void __iomem *hga_vram;			/* Base of video memory */
-static unsigned long hga_vram_len;		/* Size of video memory */
-
-#define HGA_ROWADDR(row) ((row%4)*8192 + (row>>2)*90)
-#define HGA_TXT			0
-#define HGA_GFX			1
-
-static inline u8 __iomem * rowaddr(struct fb_info *info, u_int row)
-{
-	return info->screen_base + HGA_ROWADDR(row);
-}
-
-static int hga_mode = -1;			/* 0 = txt, 1 = gfx mode */
-
-static enum { TYPE_HERC, TYPE_HERCPLUS, TYPE_HERCCOLOR } hga_type;
-static char *hga_type_name;
-
-#define HGA_INDEX_PORT		0x3b4		/* Register select port */
-#define HGA_VALUE_PORT		0x3b5		/* Register value port */
-#define HGA_MODE_PORT		0x3b8		/* Mode control port */
-#define HGA_STATUS_PORT		0x3ba		/* Status and Config port */
-#define HGA_GFX_PORT		0x3bf		/* Graphics control port */
-
-/* HGA register values */
-
-#define HGA_CURSOR_BLINKING	0x00
-#define HGA_CURSOR_OFF		0x20
-#define HGA_CURSOR_SLOWBLINK	0x60
-
-#define HGA_MODE_GRAPHICS	0x02
-#define HGA_MODE_VIDEO_EN	0x08
-#define HGA_MODE_BLINK_EN	0x20
-#define HGA_MODE_GFX_PAGE1	0x80
-
-#define HGA_STATUS_HSYNC	0x01
-#define HGA_STATUS_VSYNC	0x80
-#define HGA_STATUS_VIDEO	0x08
-
-#define HGA_CONFIG_COL132	0x08
-#define HGA_GFX_MODE_EN		0x01
-#define HGA_GFX_PAGE_EN		0x02
-
-/* Global locks */
-
-static DEFINE_SPINLOCK(hga_reg_lock);
-
-/* Framebuffer driver structures */
-
-static const struct fb_var_screeninfo hga_default_var = {
-	.xres		= 720,
-	.yres 		= 348,
-	.xres_virtual 	= 720,
-	.yres_virtual	= 348,
-	.bits_per_pixel = 1,
-	.red 		= {0, 1, 0},
-	.green 		= {0, 1, 0},
-	.blue 		= {0, 1, 0},
-	.transp 	= {0, 0, 0},
-	.height 	= -1,
-	.width 		= -1,
-};
-
-static struct fb_fix_screeninfo hga_fix = {
-	.id 		= "HGA",
-	.type 		= FB_TYPE_PACKED_PIXELS,	/* (not sure) */
-	.visual 	= FB_VISUAL_MONO10,
-	.xpanstep 	= 8,
-	.ypanstep 	= 8,
-	.line_length 	= 90,
-	.accel 		= FB_ACCEL_NONE
-};
-
-/* Don't assume that tty1 will be the initial current console. */
-static int release_io_port = 0;
-static int release_io_ports = 0;
-static bool nologo = 0;
-
-/* -------------------------------------------------------------------------
- *
- * Low level hardware functions
- *
- * ------------------------------------------------------------------------- */
-
-static void write_hga_b(unsigned int val, unsigned char reg)
-{
-	outb_p(reg, HGA_INDEX_PORT);
-	outb_p(val, HGA_VALUE_PORT);
-}
-
-static void write_hga_w(unsigned int val, unsigned char reg)
-{
-	outb_p(reg,   HGA_INDEX_PORT); outb_p(val >> 8,   HGA_VALUE_PORT);
-	outb_p(reg+1, HGA_INDEX_PORT); outb_p(val & 0xff, HGA_VALUE_PORT);
-}
-
-static int test_hga_b(unsigned char val, unsigned char reg)
-{
-	outb_p(reg, HGA_INDEX_PORT);
-	outb  (val, HGA_VALUE_PORT);
-	udelay(20); val = (inb_p(HGA_VALUE_PORT) == val);
-	return val;
-}
-
-static void hga_clear_screen(void)
-{
-	unsigned char fillchar = 0xbf; /* magic */
-	unsigned long flags;
-
-	spin_lock_irqsave(&hga_reg_lock, flags);
-	if (hga_mode == HGA_TXT)
-		fillchar = ' ';
-	else if (hga_mode == HGA_GFX)
-		fillchar = 0x00;
-	spin_unlock_irqrestore(&hga_reg_lock, flags);
-	if (fillchar != 0xbf)
-		memset_io(hga_vram, fillchar, hga_vram_len);
-}
-
-static void hga_txt_mode(void)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&hga_reg_lock, flags);
-	outb_p(HGA_MODE_VIDEO_EN | HGA_MODE_BLINK_EN, HGA_MODE_PORT);
-	outb_p(0x00, HGA_GFX_PORT);
-	outb_p(0x00, HGA_STATUS_PORT);
-
-	write_hga_b(0x61, 0x00);	/* horizontal total */
-	write_hga_b(0x50, 0x01);	/* horizontal displayed */
-	write_hga_b(0x52, 0x02);	/* horizontal sync pos */
-	write_hga_b(0x0f, 0x03);	/* horizontal sync width */
-
-	write_hga_b(0x19, 0x04);	/* vertical total */
-	write_hga_b(0x06, 0x05);	/* vertical total adjust */
-	write_hga_b(0x19, 0x06);	/* vertical displayed */
-	write_hga_b(0x19, 0x07);	/* vertical sync pos */
-
-	write_hga_b(0x02, 0x08);	/* interlace mode */
-	write_hga_b(0x0d, 0x09);	/* maximum scanline */
-	write_hga_b(0x0c, 0x0a);	/* cursor start */
-	write_hga_b(0x0d, 0x0b);	/* cursor end */
-
-	write_hga_w(0x0000, 0x0c);	/* start address */
-	write_hga_w(0x0000, 0x0e);	/* cursor location */
-
-	hga_mode = HGA_TXT;
-	spin_unlock_irqrestore(&hga_reg_lock, flags);
-}
-
-static void hga_gfx_mode(void)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&hga_reg_lock, flags);
-	outb_p(0x00, HGA_STATUS_PORT);
-	outb_p(HGA_GFX_MODE_EN, HGA_GFX_PORT);
-	outb_p(HGA_MODE_VIDEO_EN | HGA_MODE_GRAPHICS, HGA_MODE_PORT);
-
-	write_hga_b(0x35, 0x00);	/* horizontal total */
-	write_hga_b(0x2d, 0x01);	/* horizontal displayed */
-	write_hga_b(0x2e, 0x02);	/* horizontal sync pos */
-	write_hga_b(0x07, 0x03);	/* horizontal sync width */
-
-	write_hga_b(0x5b, 0x04);	/* vertical total */
-	write_hga_b(0x02, 0x05);	/* vertical total adjust */
-	write_hga_b(0x57, 0x06);	/* vertical displayed */
-	write_hga_b(0x57, 0x07);	/* vertical sync pos */
-
-	write_hga_b(0x02, 0x08);	/* interlace mode */
-	write_hga_b(0x03, 0x09);	/* maximum scanline */
-	write_hga_b(0x00, 0x0a);	/* cursor start */
-	write_hga_b(0x00, 0x0b);	/* cursor end */
-
-	write_hga_w(0x0000, 0x0c);	/* start address */
-	write_hga_w(0x0000, 0x0e);	/* cursor location */
-
-	hga_mode = HGA_GFX;
-	spin_unlock_irqrestore(&hga_reg_lock, flags);
-}
-
-static void hga_show_logo(struct fb_info *info)
-{
-/*
-	void __iomem *dest = hga_vram;
-	char *logo = linux_logo_bw;
-	int x, y;
-
-	for (y = 134; y < 134 + 80 ; y++) * this needs some cleanup *
-		for (x = 0; x < 10 ; x++)
-			writeb(~*(logo++),(dest + HGA_ROWADDR(y) + x + 40));
-*/
-}
-
-static void hga_pan(unsigned int xoffset, unsigned int yoffset)
-{
-	unsigned int base;
-	unsigned long flags;
-
-	base = (yoffset / 8) * 90 + xoffset;
-	spin_lock_irqsave(&hga_reg_lock, flags);
-	write_hga_w(base, 0x0c);	/* start address */
-	spin_unlock_irqrestore(&hga_reg_lock, flags);
-	DPRINTK("hga_pan: base:%d\n", base);
-}
-
-static void hga_blank(int blank_mode)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&hga_reg_lock, flags);
-	if (blank_mode) {
-		outb_p(0x00, HGA_MODE_PORT);	/* disable video */
-	} else {
-		outb_p(HGA_MODE_VIDEO_EN | HGA_MODE_GRAPHICS, HGA_MODE_PORT);
-	}
-	spin_unlock_irqrestore(&hga_reg_lock, flags);
-}
-
-static int hga_card_detect(struct platform_device *pdev)
-{
-	int count = 0;
-	void __iomem *p, *q;
-	unsigned short p_save, q_save;
-
-	hga_vram_len  = 0x08000;
-
-	if (!devm_request_mem_region(&pdev->dev, 0xb0000, hga_vram_len, "hgafb")) {
-		dev_err(&pdev->dev, "cannot reserve video memory at 0xb0000\n");
-		return -EBUSY;
-	}
-
-	hga_vram = ioremap(0xb0000, hga_vram_len);
-	if (!hga_vram)
-		return -ENOMEM;
-
-	if (request_region(0x3b0, 12, "hgafb"))
-		release_io_ports = 1;
-	if (request_region(0x3bf, 1, "hgafb"))
-		release_io_port = 1;
-
-	/* do a memory check */
-
-	p = hga_vram;
-	q = hga_vram + 0x01000;
-
-	p_save = readw(p); q_save = readw(q);
-
-	writew(0xaa55, p); if (readw(p) == 0xaa55) count++;
-	writew(0x55aa, p); if (readw(p) == 0x55aa) count++;
-	writew(p_save, p);
-
-	if (count != 2)
-		goto error;
-
-	/* Ok, there is definitely a card registering at the correct
-	 * memory location, so now we do an I/O port test.
-	 */
-
-	if (!test_hga_b(0x66, 0x0f))	    /* cursor low register */
-		goto error;
-
-	if (!test_hga_b(0x99, 0x0f))     /* cursor low register */
-		goto error;
-
-	/* See if the card is a Hercules, by checking whether the vsync
-	 * bit of the status register is changing.  This test lasts for
-	 * approximately 1/10th of a second.
-	 */
-
-	p_save = q_save = inb_p(HGA_STATUS_PORT) & HGA_STATUS_VSYNC;
-
-	for (count=0; count < 50000 && p_save == q_save; count++) {
-		q_save = inb(HGA_STATUS_PORT) & HGA_STATUS_VSYNC;
-		udelay(2);
-	}
-
-	if (p_save == q_save)
-		goto error;
-
-	switch (inb_p(HGA_STATUS_PORT) & 0x70) {
-		case 0x10:
-			hga_type = TYPE_HERCPLUS;
-			hga_type_name = "HerculesPlus";
-			break;
-		case 0x50:
-			hga_type = TYPE_HERCCOLOR;
-			hga_type_name = "HerculesColor";
-			break;
-		default:
-			hga_type = TYPE_HERC;
-			hga_type_name = "Hercules";
-			break;
-	}
-	return 0;
-error:
-	if (release_io_ports)
-		release_region(0x3b0, 12);
-	if (release_io_port)
-		release_region(0x3bf, 1);
-
-	iounmap(hga_vram);
-
-	pr_err("hgafb: HGA card not detected.\n");
-
-	return -EINVAL;
-}
-
-/**
- *	hgafb_open - open the framebuffer device
- *	@info: pointer to fb_info object containing info for current hga board
- *	@init: open by console system or userland.
- *
- *	Returns: %0
- */
-
-static int hgafb_open(struct fb_info *info, int init)
-{
-	hga_gfx_mode();
-	hga_clear_screen();
-	if (!nologo) hga_show_logo(info);
-	return 0;
-}
-
-/**
- *	hgafb_release - open the framebuffer device
- *	@info: pointer to fb_info object containing info for current hga board
- *	@init: open by console system or userland.
- *
- *	Returns: %0
- */
-
-static int hgafb_release(struct fb_info *info, int init)
-{
-	hga_txt_mode();
-	hga_clear_screen();
-	return 0;
-}
-
-/**
- *	hgafb_setcolreg - set color registers
- *	@regno:register index to set
- *	@red:red value, unused
- *	@green:green value, unused
- *	@blue:blue value, unused
- *	@transp:transparency value, unused
- *	@info:unused
- *
- *	This callback function is used to set the color registers of a HGA
- *	board. Since we have only two fixed colors only @regno is checked.
- *	A zero is returned on success and 1 for failure.
- *
- *	Returns: %0
- */
-
-static int hgafb_setcolreg(u_int regno, u_int red, u_int green, u_int blue,
-			   u_int transp, struct fb_info *info)
-{
-	if (regno > 1)
-		return 1;
-	return 0;
-}
-
-/**
- *	hgafb_pan_display - pan or wrap the display
- *	@var:contains new xoffset, yoffset and vmode values
- *	@info:pointer to fb_info object containing info for current hga board
- *
- *	This function looks only at xoffset, yoffset and the %FB_VMODE_YWRAP
- *	flag in @var. If input parameters are correct it calls hga_pan() to
- *	program the hardware. @info->var is updated to the new values.
- *
- *	Returns: %0 on success or %-EINVAL for failure.
- */
-
-static int hgafb_pan_display(struct fb_var_screeninfo *var,
-			     struct fb_info *info)
-{
-	if (var->vmode & FB_VMODE_YWRAP) {
-		if (var->yoffset >= info->var.yres_virtual ||
-		    var->xoffset)
-			return -EINVAL;
-	} else {
-		if (var->xoffset + info->var.xres > info->var.xres_virtual
-		 || var->yoffset + info->var.yres > info->var.yres_virtual
-		 || var->yoffset % 8)
-			return -EINVAL;
-	}
-
-	hga_pan(var->xoffset, var->yoffset);
-	return 0;
-}
-
-/**
- *	hgafb_blank - (un)blank the screen
- *	@blank_mode:blanking method to use
- *	@info:unused
- *
- *	Blank the screen if blank_mode != 0, else unblank.
- *	Implements VESA suspend and powerdown modes on hardware that supports
- *	disabling hsync/vsync:
- *		@blank_mode == 2 means suspend vsync,
- *		@blank_mode == 3 means suspend hsync,
- *		@blank_mode == 4 means powerdown.
- *
- * Returns: %0
- */
-
-static int hgafb_blank(int blank_mode, struct fb_info *info)
-{
-	hga_blank(blank_mode);
-	return 0;
-}
-
-/*
- * Accel functions
- */
-static void hgafb_fillrect(struct fb_info *info, const struct fb_fillrect *rect)
-{
-	u_int rows, y;
-	u8 __iomem *dest;
-
-	y = rect->dy;
-
-	for (rows = rect->height; rows--; y++) {
-		dest = rowaddr(info, y) + (rect->dx >> 3);
-		switch (rect->rop) {
-		case ROP_COPY:
-			memset_io(dest, rect->color, (rect->width >> 3));
-			break;
-		case ROP_XOR:
-			fb_writeb(~(fb_readb(dest)), dest);
-			break;
-		}
-	}
-}
-
-static void hgafb_copyarea(struct fb_info *info, const struct fb_copyarea *area)
-{
-	u_int rows, y1, y2;
-	u8 __iomem *src;
-	u8 __iomem *dest;
-
-	if (area->dy <= area->sy) {
-		y1 = area->sy;
-		y2 = area->dy;
-
-		for (rows = area->height; rows--; ) {
-			src = rowaddr(info, y1) + (area->sx >> 3);
-			dest = rowaddr(info, y2) + (area->dx >> 3);
-			memmove(dest, src, (area->width >> 3));
-			y1++;
-			y2++;
-		}
-	} else {
-		y1 = area->sy + area->height - 1;
-		y2 = area->dy + area->height - 1;
-
-		for (rows = area->height; rows--;) {
-			src = rowaddr(info, y1) + (area->sx >> 3);
-			dest = rowaddr(info, y2) + (area->dx >> 3);
-			memmove(dest, src, (area->width >> 3));
-			y1--;
-			y2--;
-		}
-	}
-}
-
-static void hgafb_imageblit(struct fb_info *info, const struct fb_image *image)
-{
-	u8 __iomem *dest;
-	u8 *cdat = (u8 *) image->data;
-	u_int rows, y = image->dy;
-	u_int x;
-	u8 d;
-
-	for (rows = image->height; rows--; y++) {
-		for (x = 0; x < image->width; x+= 8) {
-			d = *cdat++;
-			dest = rowaddr(info, y) + ((image->dx + x)>> 3);
-			fb_writeb(d, dest);
-		}
-	}
-}
-
-static const struct fb_ops hgafb_ops = {
-	.owner		= THIS_MODULE,
-	.fb_open	= hgafb_open,
-	.fb_release	= hgafb_release,
-	__FB_DEFAULT_IOMEM_OPS_RDWR,
-	.fb_setcolreg	= hgafb_setcolreg,
-	.fb_pan_display	= hgafb_pan_display,
-	.fb_blank	= hgafb_blank,
-	.fb_fillrect	= hgafb_fillrect,
-	.fb_copyarea	= hgafb_copyarea,
-	.fb_imageblit	= hgafb_imageblit,
-	__FB_DEFAULT_IOMEM_OPS_MMAP,
-};
-
-/* ------------------------------------------------------------------------- *
- *
- * Functions in fb_info
- *
- * ------------------------------------------------------------------------- */
-
-/* ------------------------------------------------------------------------- */
-
-	/*
-	 *  Initialization
-	 */
-
-static int hgafb_probe(struct platform_device *pdev)
-{
-	struct fb_info *info;
-	int ret;
-
-	ret = hga_card_detect(pdev);
-	if (ret)
-		return ret;
-
-	printk(KERN_INFO "hgafb: %s with %ldK of memory detected.\n",
-		hga_type_name, hga_vram_len/1024);
-
-	info = framebuffer_alloc(0, &pdev->dev);
-	if (!info) {
-		iounmap(hga_vram);
-		return -ENOMEM;
-	}
-
-	hga_fix.smem_start = (unsigned long)hga_vram;
-	hga_fix.smem_len = hga_vram_len;
-
-	info->flags = FBINFO_HWACCEL_YPAN;
-	info->var = hga_default_var;
-	info->fix = hga_fix;
-	info->monspecs.hfmin = 0;
-	info->monspecs.hfmax = 0;
-	info->monspecs.vfmin = 10000;
-	info->monspecs.vfmax = 10000;
-	info->monspecs.dpms = 0;
-	info->fbops = &hgafb_ops;
-	info->screen_base = hga_vram;
-
-        if (register_framebuffer(info) < 0) {
-		framebuffer_release(info);
-		iounmap(hga_vram);
-		return -EINVAL;
-	}
-
-	fb_info(info, "%s frame buffer device\n", info->fix.id);
-	platform_set_drvdata(pdev, info);
-	return 0;
-}
-
-static void hgafb_remove(struct platform_device *pdev)
-{
-	struct fb_info *info = platform_get_drvdata(pdev);
-
-	hga_txt_mode();
-	hga_clear_screen();
-
-	if (info) {
-		unregister_framebuffer(info);
-		framebuffer_release(info);
-	}
-
-	iounmap(hga_vram);
-
-	if (release_io_ports)
-		release_region(0x3b0, 12);
-
-	if (release_io_port)
-		release_region(0x3bf, 1);
-}
-
-static struct platform_driver hgafb_driver = {
-	.probe = hgafb_probe,
-	.remove = hgafb_remove,
-	.driver = {
-		.name = "hgafb",
-	},
-};
-
-static struct platform_device *hgafb_device;
-
-static int __init hgafb_init(void)
-{
-	int ret;
-
-	if (fb_get_options("hgafb", NULL))
-		return -ENODEV;
-
-	ret = platform_driver_register(&hgafb_driver);
-
-	if (!ret) {
-		hgafb_device = platform_device_register_simple("hgafb", 0, NULL, 0);
-
-		if (IS_ERR(hgafb_device)) {
-			platform_driver_unregister(&hgafb_driver);
-			ret = PTR_ERR(hgafb_device);
-		}
-	}
-
-	return ret;
-}
-
-static void __exit hgafb_exit(void)
-{
-	platform_device_unregister(hgafb_device);
-	platform_driver_unregister(&hgafb_driver);
-}
-
-/* -------------------------------------------------------------------------
- *
- *  Modularization
- *
- * ------------------------------------------------------------------------- */
-
-MODULE_AUTHOR("Ferenc Bakonyi <fero@drama.obuda.kando.hu>");
-MODULE_DESCRIPTION("FBDev driver for Hercules Graphics Adaptor");
-MODULE_LICENSE("GPL");
-
-module_param(nologo, bool, 0);
-MODULE_PARM_DESC(nologo, "Disables startup logo if != 0 (default=0)");
-module_init(hgafb_init);
-module_exit(hgafb_exit);
diff --git a/drivers/video/fbdev/i740fb.c b/drivers/video/fbdev/i740fb.c
index 9b74dae71472c..c14a19382769a 100644
--- a/drivers/video/fbdev/i740fb.c
+++ b/drivers/video/fbdev/i740fb.c
@@ -1152,6 +1152,7 @@ static int i740fb_probe(struct pci_dev *dev, const struct pci_device_id *ent)
 	fb_dealloc_cmap(&info->cmap);
 err_alloc_cmap:
 err_find_mode:
+	fb_destroy_modelist(&info->modelist);
 	if (par->ddc_registered)
 		i2c_del_adapter(&par->ddc_adapter);
 	pci_iounmap(dev, par->regs);
diff --git a/drivers/video/fbdev/imxfb.c b/drivers/video/fbdev/imxfb.c
index a077bf346bdf4..7a021da0a32a2 100644
--- a/drivers/video/fbdev/imxfb.c
+++ b/drivers/video/fbdev/imxfb.c
@@ -30,7 +30,6 @@
 #include <linux/lcd.h>
 #include <linux/math64.h>
 #include <linux/of.h>
-#include <linux/of_device.h>
 #include <linux/bitfield.h>
 
 #include <linux/regulator/consumer.h>
@@ -880,7 +879,6 @@ static int imxfb_probe(struct platform_device *pdev)
 	struct lcd_device *lcd;
 	struct fb_info *info;
 	struct imx_fb_videomode *m;
-	const struct of_device_id *of_id;
 	struct device_node *display_np;
 	int ret, i;
 	int bytes_per_pixel;
@@ -891,9 +889,7 @@ static int imxfb_probe(struct platform_device *pdev)
 	if (ret < 0)
 		return ret;
 
-	of_id = of_match_device(imxfb_of_dev_id, &pdev->dev);
-	if (of_id)
-		pdev->id_entry = of_id->data;
+	pdev->id_entry = of_device_get_match_data(&pdev->dev);
 
 	info = framebuffer_alloc(sizeof(struct imxfb_info), &pdev->dev);
 	if (!info)
diff --git a/drivers/video/fbdev/kyro/fbdev.c b/drivers/video/fbdev/kyro/fbdev.c
index c8b1dfa456a33..d756b3603fa6a 100644
--- a/drivers/video/fbdev/kyro/fbdev.c
+++ b/drivers/video/fbdev/kyro/fbdev.c
@@ -645,9 +645,8 @@ static int kyrofb_ioctl(struct fb_info *info,
 }
 
 static const struct pci_device_id kyrofb_pci_tbl[] = {
-	{ PCI_VENDOR_ID_ST, PCI_DEVICE_ID_STG4000,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ 0, }
+	{ PCI_DEVICE(PCI_VENDOR_ID_ST, PCI_DEVICE_ID_STG4000) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, kyrofb_pci_tbl);
diff --git a/drivers/video/fbdev/matrox/matroxfb_base.c b/drivers/video/fbdev/matrox/matroxfb_base.c
index e1a4bc7c23184..ac04a19b68492 100644
--- a/drivers/video/fbdev/matrox/matroxfb_base.c
+++ b/drivers/video/fbdev/matrox/matroxfb_base.c
@@ -1642,8 +1642,8 @@ static int initMatrox2(struct matrox_fb_info *minfo, struct board *b)
 	int err;
 
 	static const struct pci_device_id intel_82437[] = {
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82437) },
-		{ },
+		{ PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_82437) },
+		{ }
 	};
 
 	DBG(__func__)
@@ -2135,35 +2135,23 @@ static void pci_remove_matrox(struct pci_dev* pdev) {
 
 static const struct pci_device_id matroxfb_devices[] = {
 #ifdef CONFIG_FB_MATROX_MILLENIUM
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_MIL,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_MIL_2,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_MIL_2_AGP,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_MIL) },
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_MIL_2) },
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_MIL_2_AGP) },
 #endif
 #ifdef CONFIG_FB_MATROX_MYSTIQUE
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_MYS,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_MYS) },
 #endif
 #ifdef CONFIG_FB_MATROX_G
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_G100_MM,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_G100_AGP,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_G200_PCI,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
-	{PCI_VENDOR_ID_MATROX,	0x0532,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_G200_AGP,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_G400,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
-	{PCI_VENDOR_ID_MATROX,	PCI_DEVICE_ID_MATROX_G550,
-		PCI_ANY_ID,	PCI_ANY_ID,	0, 0, 0},
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_G100_MM) },
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_G100_AGP) },
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_G200_PCI) },
+	{ PCI_VDEVICE(MATROX, 0x0532) },
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_G200_AGP) },
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_G400) },
+	{ PCI_VDEVICE(MATROX, PCI_DEVICE_ID_MATROX_G550) },
 #endif
-	{0,			0,
-		0,		0,		0, 0, 0}
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, matroxfb_devices);
diff --git a/drivers/video/fbdev/matrox/matroxfb_maven.c b/drivers/video/fbdev/matrox/matroxfb_maven.c
index 2ea65da6075ca..fe057a0b57ec7 100644
--- a/drivers/video/fbdev/matrox/matroxfb_maven.c
+++ b/drivers/video/fbdev/matrox/matroxfb_maven.c
@@ -1282,7 +1282,7 @@ static void maven_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id maven_id[] = {
-	{ "maven" },
+	{ .name = "maven" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, maven_id);
diff --git a/drivers/video/fbdev/metronomefb.c b/drivers/video/fbdev/metronomefb.c
index 6f0942c6e5f1c..83c614963a0a6 100644
--- a/drivers/video/fbdev/metronomefb.c
+++ b/drivers/video/fbdev/metronomefb.c
@@ -645,12 +645,14 @@ static int metronomefb_probe(struct platform_device *dev)
 	info->flags = FBINFO_VIRTFB;
 
 	info->fbdefio = &metronomefb_defio;
-	fb_deferred_io_init(info);
+	retval = fb_deferred_io_init(info);
+	if (retval)
+		goto err_free_irq;
 
 	retval = fb_alloc_cmap(&info->cmap, 8, 0);
 	if (retval < 0) {
 		dev_err(&dev->dev, "Failed to allocate colormap\n");
-		goto err_free_irq;
+		goto err_fbdefio;
 	}
 
 	/* set cmap */
@@ -673,6 +675,8 @@ static int metronomefb_probe(struct platform_device *dev)
 
 err_cmap:
 	fb_dealloc_cmap(&info->cmap);
+err_fbdefio:
+	fb_deferred_io_cleanup(info);
 err_free_irq:
 	board->cleanup(par);
 err_csum_table:
diff --git a/drivers/video/fbdev/neofb.c b/drivers/video/fbdev/neofb.c
index c1cd028b8991c..e0b8d4d6ce79b 100644
--- a/drivers/video/fbdev/neofb.c
+++ b/drivers/video/fbdev/neofb.c
@@ -2126,34 +2126,25 @@ static void neofb_remove(struct pci_dev *dev)
 }
 
 static const struct pci_device_id neofb_devices[] = {
-	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2070,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2070},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_CHIP_NM2070), .driver_data = FB_ACCEL_NEOMAGIC_NM2070 },
 
-	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2090,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2090},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_CHIP_NM2090), .driver_data = FB_ACCEL_NEOMAGIC_NM2090 },
 
-	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2093,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2093},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_CHIP_NM2093), .driver_data = FB_ACCEL_NEOMAGIC_NM2093 },
 
-	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2097,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2097},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_CHIP_NM2097), .driver_data = FB_ACCEL_NEOMAGIC_NM2097 },
 
-	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2160,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2160},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_CHIP_NM2160), .driver_data = FB_ACCEL_NEOMAGIC_NM2160 },
 
-	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2200,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2200},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_CHIP_NM2200), .driver_data = FB_ACCEL_NEOMAGIC_NM2200 },
 
-	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2230,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2230},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_CHIP_NM2230), .driver_data = FB_ACCEL_NEOMAGIC_NM2230 },
 
-	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2360,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2360},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_CHIP_NM2360), .driver_data = FB_ACCEL_NEOMAGIC_NM2360 },
 
-	{PCI_VENDOR_ID_NEOMAGIC, PCI_CHIP_NM2380,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_NEOMAGIC_NM2380},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_CHIP_NM2380), .driver_data = FB_ACCEL_NEOMAGIC_NM2380 },
 
-	{0, 0, 0, 0, 0, 0, 0}
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, neofb_devices);
diff --git a/drivers/video/fbdev/nvidia/nvidia.c b/drivers/video/fbdev/nvidia/nvidia.c
index 72b85f475605d..7d20c4087aeb5 100644
--- a/drivers/video/fbdev/nvidia/nvidia.c
+++ b/drivers/video/fbdev/nvidia/nvidia.c
@@ -58,9 +58,12 @@
 #define MAX_CURS		32
 
 static const struct pci_device_id nvidiafb_pci_tbl[] = {
-	{PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID,
-	 PCI_BASE_CLASS_DISPLAY << 16, 0xff0000, 0},
-	{ 0, }
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID),
+		.class = PCI_BASE_CLASS_DISPLAY << 16,
+		.class_mask = 0xff0000
+	},
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, nvidiafb_pci_tbl);
 
@@ -1421,6 +1424,7 @@ static int nvidiafb_probe(struct pci_dev *pd, const struct pci_device_id *ent)
 
 err_out_iounmap_fb:
 	iounmap(info->screen_base);
+	fb_destroy_modelist(&info->modelist);
 err_out_free_base1:
 	fb_destroy_modedb(info->monspecs.modedb);
 	nvidia_delete_i2c_busses(par);
diff --git a/drivers/video/fbdev/pm2fb.c b/drivers/video/fbdev/pm2fb.c
index f34429829b7d7..412ff249b5c79 100644
--- a/drivers/video/fbdev/pm2fb.c
+++ b/drivers/video/fbdev/pm2fb.c
@@ -1748,13 +1748,10 @@ static void pm2fb_remove(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id pm2fb_id_table[] = {
-	{ PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_TVP4020,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_3DLABS, PCI_DEVICE_ID_3DLABS_PERMEDIA2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_3DLABS, PCI_DEVICE_ID_3DLABS_PERMEDIA2V,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ 0, }
+	{ PCI_VDEVICE(TI, PCI_DEVICE_ID_TI_TVP4020) },
+	{ PCI_VDEVICE(3DLABS, PCI_DEVICE_ID_3DLABS_PERMEDIA2) },
+	{ PCI_VDEVICE(3DLABS, PCI_DEVICE_ID_3DLABS_PERMEDIA2V) },
+	{ }
 };
 
 static struct pci_driver pm2fb_driver = {
diff --git a/drivers/video/fbdev/pm3fb.c b/drivers/video/fbdev/pm3fb.c
index 6e55e42514d63..6f552ae36219a 100644
--- a/drivers/video/fbdev/pm3fb.c
+++ b/drivers/video/fbdev/pm3fb.c
@@ -1486,9 +1486,8 @@ static void pm3fb_remove(struct pci_dev *dev)
 }
 
 static const struct pci_device_id pm3fb_id_table[] = {
-	{ PCI_VENDOR_ID_3DLABS, 0x0a,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ 0, }
+	{ PCI_VDEVICE(3DLABS, 0x000a) },
+	{ }
 };
 
 /* For PCI drivers */
diff --git a/drivers/video/fbdev/pvr2fb.c b/drivers/video/fbdev/pvr2fb.c
index 3f6384e631b14..9428716e2dc40 100644
--- a/drivers/video/fbdev/pvr2fb.c
+++ b/drivers/video/fbdev/pvr2fb.c
@@ -993,9 +993,8 @@ static void pvr2fb_pci_remove(struct pci_dev *pdev)
 }
 
 static const struct pci_device_id pvr2fb_pci_tbl[] = {
-	{ PCI_VENDOR_ID_NEC, PCI_DEVICE_ID_NEC_NEON250,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ 0, },
+	{ PCI_VDEVICE(NEC, PCI_DEVICE_ID_NEC_NEON250) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, pvr2fb_pci_tbl);
diff --git a/drivers/video/fbdev/riva/fbdev.c b/drivers/video/fbdev/riva/fbdev.c
index 1e377b2ec0898..2268fea4d8072 100644
--- a/drivers/video/fbdev/riva/fbdev.c
+++ b/drivers/video/fbdev/riva/fbdev.c
@@ -103,92 +103,50 @@ static int rivafb_blank(int blank, struct fb_info *info);
  * ------------------------------------------------------------------------- */
 
 static const struct pci_device_id rivafb_pci_tbl[] = {
-	{ PCI_VENDOR_ID_NVIDIA_SGS, PCI_DEVICE_ID_NVIDIA_SGS_RIVA128,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_TNT,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_TNT2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_UTNT2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_VTNT2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_UVTNT2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_ITNT2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE_SDR,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE_DDR,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_MX,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_MX2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_GO,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO2_MXR,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_GTS,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_GTS2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_ULTRA,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO2_PRO,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_MX_460,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_MX_440,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA_SGS, PCI_DEVICE_ID_NVIDIA_SGS_RIVA128) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_TNT) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_TNT2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_UTNT2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_VTNT2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_UVTNT2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_ITNT2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE_SDR) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE_DDR) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_MX) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_MX2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_GO) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO2_MXR) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_GTS) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_GTS2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE2_ULTRA) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO2_PRO) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_MX_460) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_MX_440) },
 	// NF2/IGP version, GeForce 4 MX, NV18
-	{ PCI_VENDOR_ID_NVIDIA, 0x01f0,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_MX_420,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_440_GO,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_420_GO,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_420_GO_M32,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_500XGL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_440_GO_M64,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_200,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_550XGL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_500_GOGL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_IGEFORCE2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE3,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE3_1,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE3_2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO_DDC,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_TI_4600,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_TI_4400,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_TI_4200,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
- 	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_900XGL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_750XGL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_700XGL,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE_FX_GO_5200,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },
-	{ 0, } /* terminate list */
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, 0x01f0) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_MX_420) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_440_GO) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_420_GO) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_420_GO_M32) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_500XGL) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_440_GO_M64) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_200) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_550XGL) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_500_GOGL) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_IGEFORCE2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE3) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE3_1) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE3_2) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO_DDC) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_TI_4600) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_TI_4400) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE4_TI_4200) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_900XGL) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_750XGL) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_QUADRO4_700XGL) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_NVIDIA, PCI_DEVICE_ID_NVIDIA_GEFORCE_FX_GO_5200) },
+	{ } /* terminate list */
 };
 MODULE_DEVICE_TABLE(pci, rivafb_pci_tbl);
 
diff --git a/drivers/video/fbdev/s3fb.c b/drivers/video/fbdev/s3fb.c
index ba30e5568cabc..dc1f9b627185d 100644
--- a/drivers/video/fbdev/s3fb.c
+++ b/drivers/video/fbdev/s3fb.c
@@ -1446,6 +1446,7 @@ static int s3_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
 err_alloc_cmap:
 err_find_mode:
 #ifdef CONFIG_FB_S3_DDC
+	fb_destroy_modelist(&info->modelist);
 	if (par->ddc_registered)
 		i2c_del_adapter(&par->ddc_adapter);
 	if (par->mmio)
@@ -1563,24 +1564,24 @@ static const struct dev_pm_ops s3_pci_pm_ops = {
 /* List of boards that we are trying to support */
 
 static const struct pci_device_id s3_devices[] = {
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8810), .driver_data = CHIP_XXX_TRIO},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8811), .driver_data = CHIP_XXX_TRIO},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8812), .driver_data = CHIP_M65_AURORA64VP},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8814), .driver_data = CHIP_767_TRIO64UVP},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8901), .driver_data = CHIP_XXX_TRIO64V2_DXGX},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8902), .driver_data = CHIP_551_PLATO_PX},
-
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x5631), .driver_data = CHIP_325_VIRGE},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x883D), .driver_data = CHIP_988_VIRGE_VX},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8A01), .driver_data = CHIP_XXX_VIRGE_DXGX},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8A10), .driver_data = CHIP_357_VIRGE_GX2},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8A11), .driver_data = CHIP_359_VIRGE_GX2P},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8A12), .driver_data = CHIP_359_VIRGE_GX2P},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8A13), .driver_data = CHIP_36X_TRIO3D_1X_2X},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8904), .driver_data = CHIP_365_TRIO3D},
-	{PCI_DEVICE(PCI_VENDOR_ID_S3, 0x8C01), .driver_data = CHIP_260_VIRGE_MX},
-
-	{0, 0, 0, 0, 0, 0, 0}
+	{ PCI_VDEVICE(S3, 0x8810), .driver_data = CHIP_XXX_TRIO },
+	{ PCI_VDEVICE(S3, 0x8811), .driver_data = CHIP_XXX_TRIO },
+	{ PCI_VDEVICE(S3, 0x8812), .driver_data = CHIP_M65_AURORA64VP },
+	{ PCI_VDEVICE(S3, 0x8814), .driver_data = CHIP_767_TRIO64UVP },
+	{ PCI_VDEVICE(S3, 0x8901), .driver_data = CHIP_XXX_TRIO64V2_DXGX },
+	{ PCI_VDEVICE(S3, 0x8902), .driver_data = CHIP_551_PLATO_PX },
+
+	{ PCI_VDEVICE(S3, 0x5631), .driver_data = CHIP_325_VIRGE },
+	{ PCI_VDEVICE(S3, 0x883D), .driver_data = CHIP_988_VIRGE_VX },
+	{ PCI_VDEVICE(S3, 0x8A01), .driver_data = CHIP_XXX_VIRGE_DXGX },
+	{ PCI_VDEVICE(S3, 0x8A10), .driver_data = CHIP_357_VIRGE_GX2 },
+	{ PCI_VDEVICE(S3, 0x8A11), .driver_data = CHIP_359_VIRGE_GX2P },
+	{ PCI_VDEVICE(S3, 0x8A12), .driver_data = CHIP_359_VIRGE_GX2P },
+	{ PCI_VDEVICE(S3, 0x8A13), .driver_data = CHIP_36X_TRIO3D_1X_2X },
+	{ PCI_VDEVICE(S3, 0x8904), .driver_data = CHIP_365_TRIO3D },
+	{ PCI_VDEVICE(S3, 0x8C01), .driver_data = CHIP_260_VIRGE_MX },
+
+	{ }
 };
 
 
diff --git a/drivers/video/fbdev/savage/savagefb_driver.c b/drivers/video/fbdev/savage/savagefb_driver.c
index c2f79357c8da0..7789196d2eb5c 100644
--- a/drivers/video/fbdev/savage/savagefb_driver.c
+++ b/drivers/video/fbdev/savage/savagefb_driver.c
@@ -2449,76 +2449,78 @@ static const struct dev_pm_ops savagefb_pm_ops = {
 };
 
 static const struct pci_device_id savagefb_devices[] = {
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_MX128,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_MX64,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_MX64C,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_IX128SDR,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_IX128DDR,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_IX64SDR,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_IX64DDR,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_IXCSDR,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SUPSAV_IXCDDR,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SUPERSAVAGE},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SAVAGE4,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SAVAGE4},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SAVAGE3D,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SAVAGE3D},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SAVAGE3D_MV,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SAVAGE3D_MV},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SAVAGE2000,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SAVAGE2000},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SAVAGE_MX_MV,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SAVAGE_MX_MV},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SAVAGE_MX,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SAVAGE_MX},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SAVAGE_IX_MV,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SAVAGE_IX_MV},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_SAVAGE_IX,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_SAVAGE_IX},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_PROSAVAGE_PM,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_PROSAVAGE_PM},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_PROSAVAGE_KM,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_PROSAVAGE_KM},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_S3TWISTER_P,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_S3TWISTER_P},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_S3TWISTER_K,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_S3TWISTER_K},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_PROSAVAGE_DDR,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_PROSAVAGE_DDR},
-
-	{PCI_VENDOR_ID_S3, PCI_CHIP_PROSAVAGE_DDRK,
-	 PCI_ANY_ID, PCI_ANY_ID, 0, 0, FB_ACCEL_PROSAVAGE_DDRK},
-
-	{0, 0, 0, 0, 0, 0, 0}
+	{
+		PCI_VDEVICE(S3, PCI_CHIP_SUPSAV_MX128),
+		.driver_data = FB_ACCEL_SUPERSAVAGE,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SUPSAV_MX64),
+		.driver_data = FB_ACCEL_SUPERSAVAGE,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SUPSAV_MX64C),
+		.driver_data = FB_ACCEL_SUPERSAVAGE,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SUPSAV_IX128SDR),
+		.driver_data = FB_ACCEL_SUPERSAVAGE,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SUPSAV_IX128DDR),
+		.driver_data = FB_ACCEL_SUPERSAVAGE,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SUPSAV_IX64SDR),
+		.driver_data = FB_ACCEL_SUPERSAVAGE,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SUPSAV_IX64DDR),
+		.driver_data = FB_ACCEL_SUPERSAVAGE,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SUPSAV_IXCSDR),
+		.driver_data = FB_ACCEL_SUPERSAVAGE,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SUPSAV_IXCDDR),
+		.driver_data = FB_ACCEL_SUPERSAVAGE,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SAVAGE4),
+		.driver_data = FB_ACCEL_SAVAGE4,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SAVAGE3D),
+		.driver_data = FB_ACCEL_SAVAGE3D,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SAVAGE3D_MV),
+		.driver_data = FB_ACCEL_SAVAGE3D_MV,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SAVAGE2000),
+		.driver_data = FB_ACCEL_SAVAGE2000,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SAVAGE_MX_MV),
+		.driver_data = FB_ACCEL_SAVAGE_MX_MV,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SAVAGE_MX),
+		.driver_data = FB_ACCEL_SAVAGE_MX,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SAVAGE_IX_MV),
+		.driver_data = FB_ACCEL_SAVAGE_IX_MV,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_SAVAGE_IX),
+		.driver_data = FB_ACCEL_SAVAGE_IX,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_PROSAVAGE_PM),
+		.driver_data = FB_ACCEL_PROSAVAGE_PM,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_PROSAVAGE_KM),
+		.driver_data = FB_ACCEL_PROSAVAGE_KM,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_S3TWISTER_P),
+		.driver_data = FB_ACCEL_S3TWISTER_P,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_S3TWISTER_K),
+		.driver_data = FB_ACCEL_S3TWISTER_K,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_PROSAVAGE_DDR),
+		.driver_data = FB_ACCEL_PROSAVAGE_DDR,
+	}, {
+		PCI_VDEVICE(S3, PCI_CHIP_PROSAVAGE_DDRK),
+		.driver_data = FB_ACCEL_PROSAVAGE_DDRK,
+	},
+
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, savagefb_devices);
diff --git a/drivers/video/fbdev/sis/sis_main.h b/drivers/video/fbdev/sis/sis_main.h
index 0965db9fad6af..4ca487f482055 100644
--- a/drivers/video/fbdev/sis/sis_main.h
+++ b/drivers/video/fbdev/sis/sis_main.h
@@ -102,22 +102,22 @@ static struct sisfb_chip_info {
 
 static struct pci_device_id sisfb_pci_table[] = {
 #ifdef CONFIG_FB_SIS_300
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_300,     PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_540_VGA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 1},
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_630_VGA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 2},
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_300), .driver_data = 0 },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_540_VGA), .driver_data = 1 },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_630_VGA), .driver_data = 2 },
 #endif
 #ifdef CONFIG_FB_SIS_315
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_315H,    PCI_ANY_ID, PCI_ANY_ID, 0, 0, 3},
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_315,     PCI_ANY_ID, PCI_ANY_ID, 0, 0, 4},
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_315PRO,  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 5},
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_550_VGA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 6},
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_650_VGA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 7},
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_330,     PCI_ANY_ID, PCI_ANY_ID, 0, 0, 8},
-	{ PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_660_VGA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 9},
-	{ PCI_VENDOR_ID_XGI,PCI_DEVICE_ID_XGI_20,     PCI_ANY_ID, PCI_ANY_ID, 0, 0,10},
-	{ PCI_VENDOR_ID_XGI,PCI_DEVICE_ID_XGI_40,     PCI_ANY_ID, PCI_ANY_ID, 0, 0,11},
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_315H), .driver_data = 3 },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_315), .driver_data = 4 },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_315PRO), .driver_data = 5 },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_550_VGA), .driver_data = 6 },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_650_VGA), .driver_data = 7 },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_330), .driver_data = 8 },
+	{ PCI_VDEVICE(SI, PCI_DEVICE_ID_SI_660_VGA), .driver_data = 9 },
+	{ PCI_VDEVICE(XGI, PCI_DEVICE_ID_XGI_20), .driver_data = 10 },
+	{ PCI_VDEVICE(XGI, PCI_DEVICE_ID_XGI_40), .driver_data = 11 },
 #endif
-	{ 0 }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, sisfb_pci_table);
diff --git a/drivers/video/fbdev/sm501fb.c b/drivers/video/fbdev/sm501fb.c
index fee4b9f84592f..1ee7842517b85 100644
--- a/drivers/video/fbdev/sm501fb.c
+++ b/drivers/video/fbdev/sm501fb.c
@@ -2048,6 +2048,9 @@ static int sm501fb_probe(struct platform_device *pdev)
 	framebuffer_release(info->fb[HEAD_CRT]);
 
 err_alloc:
+#if defined(CONFIG_OF)
+	kfree(info->edid_data);
+#endif
 	kfree(info);
 
 	return ret;
diff --git a/drivers/video/fbdev/sm712.h b/drivers/video/fbdev/sm712.h
index c7ebf03b8d537..83fe25fc61f2c 100644
--- a/drivers/video/fbdev/sm712.h
+++ b/drivers/video/fbdev/sm712.h
@@ -101,7 +101,7 @@ struct modeinit {
 #define mmio_addr		0x00800000
 #define seqw17()		smtc_seqw(0x17, 0x30)
 #define big_pixel_depth(p, d)	{if (p == 24) {p = 32; d = 32; } }
-#define big_swap(p)		((p & 0xff00ff00 >> 8) | (p & 0x00ff00ff << 8))
+#define big_swap(p)		(((p & 0xff00ff00) >> 8) | ((p & 0x00ff00ff) << 8))
 #else
 #define pal_rgb(r, g, b, val)	val
 #define big_addr		0
diff --git a/drivers/video/fbdev/ssd1307fb.c b/drivers/video/fbdev/ssd1307fb.c
index 83dd31fa1fab5..644b8d97b3811 100644
--- a/drivers/video/fbdev/ssd1307fb.c
+++ b/drivers/video/fbdev/ssd1307fb.c
@@ -784,10 +784,10 @@ static void ssd1307fb_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ssd1307fb_i2c_id[] = {
-	{ "ssd1305fb" },
-	{ "ssd1306fb" },
-	{ "ssd1307fb" },
-	{ "ssd1309fb" },
+	{ .name = "ssd1305fb" },
+	{ .name = "ssd1306fb" },
+	{ .name = "ssd1307fb" },
+	{ .name = "ssd1309fb" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ssd1307fb_i2c_id);
diff --git a/drivers/video/fbdev/sunxvr2500.c b/drivers/video/fbdev/sunxvr2500.c
index 42426d09b935f..b7587ff4df856 100644
--- a/drivers/video/fbdev/sunxvr2500.c
+++ b/drivers/video/fbdev/sunxvr2500.c
@@ -38,8 +38,7 @@ static int s3d_get_props(struct s3d_info *sp)
 	sp->depth = of_getintprop_default(sp->of_node, "depth", 8);
 
 	if (!sp->width || !sp->height) {
-		printk(KERN_ERR "s3d: Critical properties missing for %s\n",
-		       pci_name(sp->pdev));
+		pci_err(sp->pdev, "Critical properties missing\n");
 		return -EINVAL;
 	}
 
@@ -107,7 +106,7 @@ static int s3d_set_fbinfo(struct s3d_info *sp)
 	var->transp.length = 0;
 
 	if (fb_alloc_cmap(&info->cmap, 256, 0)) {
-		printk(KERN_ERR "s3d: Cannot allocate color map.\n");
+		pci_err(sp->pdev, "Cannot allocate color map\n");
 		return -ENOMEM;
 	}
 
@@ -127,8 +126,7 @@ static int s3d_pci_register(struct pci_dev *pdev,
 
 	err = pci_enable_device(pdev);
 	if (err < 0) {
-		printk(KERN_ERR "s3d: Cannot enable PCI device %s\n",
-		       pci_name(pdev));
+		pci_err(pdev, "Cannot enable PCI device\n");
 		goto err_out;
 	}
 
@@ -143,8 +141,7 @@ static int s3d_pci_register(struct pci_dev *pdev,
 	sp->pdev = pdev;
 	sp->of_node = pci_device_to_OF_node(pdev);
 	if (!sp->of_node) {
-		printk(KERN_ERR "s3d: Cannot find OF node of %s\n",
-		       pci_name(pdev));
+		pci_err(pdev, "Cannot find OF node\n");
 		err = -ENODEV;
 		goto err_release_fb;
 	}
@@ -153,8 +150,7 @@ static int s3d_pci_register(struct pci_dev *pdev,
 
 	err = pci_request_region(pdev, 1, "s3d framebuffer");
 	if (err < 0) {
-		printk("s3d: Cannot request region 1 for %s\n",
-		       pci_name(pdev));
+		pci_err(pdev, "Cannot request region 1\n");
 		goto err_release_fb;
 	}
 
@@ -194,12 +190,11 @@ static int s3d_pci_register(struct pci_dev *pdev,
 
 	pci_set_drvdata(pdev, info);
 
-	printk("s3d: Found device at %s\n", pci_name(pdev));
+	pci_info(pdev, "Found device\n");
 
 	err = register_framebuffer(info);
 	if (err < 0) {
-		printk(KERN_ERR "s3d: Could not register framebuffer %s\n",
-		       pci_name(pdev));
+		pci_err(pdev, "Could not register framebuffer\n");
 		goto err_unmap_fb;
 	}
 
diff --git a/drivers/video/fbdev/tdfxfb.c b/drivers/video/fbdev/tdfxfb.c
index 4c4e53aaea3ad..cc6a074f3165b 100644
--- a/drivers/video/fbdev/tdfxfb.c
+++ b/drivers/video/fbdev/tdfxfb.c
@@ -124,16 +124,17 @@ static int tdfxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id);
 static void tdfxfb_remove(struct pci_dev *pdev);
 
 static const struct pci_device_id tdfxfb_id_table[] = {
-	{ PCI_VENDOR_ID_3DFX, PCI_DEVICE_ID_3DFX_BANSHEE,
-	  PCI_ANY_ID, PCI_ANY_ID, PCI_BASE_CLASS_DISPLAY << 16,
-	  0xff0000, 0 },
-	{ PCI_VENDOR_ID_3DFX, PCI_DEVICE_ID_3DFX_VOODOO3,
-	  PCI_ANY_ID, PCI_ANY_ID, PCI_BASE_CLASS_DISPLAY << 16,
-	  0xff0000, 0 },
-	{ PCI_VENDOR_ID_3DFX, PCI_DEVICE_ID_3DFX_VOODOO5,
-	  PCI_ANY_ID, PCI_ANY_ID, PCI_BASE_CLASS_DISPLAY << 16,
-	  0xff0000, 0 },
-	{ 0, }
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_3DFX, PCI_DEVICE_ID_3DFX_BANSHEE),
+		.class = PCI_BASE_CLASS_DISPLAY << 16, .class_mask = 0xff0000,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_3DFX, PCI_DEVICE_ID_3DFX_VOODOO3),
+		.class = PCI_BASE_CLASS_DISPLAY << 16, .class_mask = 0xff0000,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_3DFX, PCI_DEVICE_ID_3DFX_VOODOO5),
+		.class = PCI_BASE_CLASS_DISPLAY << 16, .class_mask = 0xff0000,
+	},
+	{ }
 };
 
 static struct pci_driver tdfxfb_driver = {
@@ -1551,6 +1552,7 @@ static int tdfxfb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 
 out_err_iobase:
 #ifdef CONFIG_FB_3DFX_I2C
+	fb_destroy_modelist(&info->modelist);
 	tdfxfb_delete_i2c_busses(default_par);
 #endif
 	arch_phys_wc_del(default_par->wc_cookie);
diff --git a/drivers/video/fbdev/tridentfb.c b/drivers/video/fbdev/tridentfb.c
index 17b7253b8fbec..9f055ba776c81 100644
--- a/drivers/video/fbdev/tridentfb.c
+++ b/drivers/video/fbdev/tridentfb.c
@@ -1706,6 +1706,7 @@ static int trident_pci_probe(struct pci_dev *dev,
 	return 0;
 
 out_unmap2:
+	fb_destroy_modelist(&info->modelist);
 	if (default_par->ddc_registered)
 		i2c_del_adapter(&default_par->ddc_adapter);
 	kfree(info->pixmap.addr);
@@ -1736,28 +1737,28 @@ static void trident_pci_remove(struct pci_dev *dev)
 
 /* List of boards that we are trying to support */
 static const struct pci_device_id trident_devices[] = {
-	{PCI_VENDOR_ID_TRIDENT,	BLADE3D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEi7, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEi7D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEi1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEi1D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEAi1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEAi1D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEE4, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	TGUI9440, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	TGUI9660, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	IMAGE975, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	IMAGE985, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBER9320, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBER9388, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBER9520, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBER9525DVD, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBER9397, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBER9397DVD, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEXPAi1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEXPm8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{PCI_VENDOR_ID_TRIDENT,	CYBERBLADEXPm16, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
-	{0,}
+	{ PCI_VDEVICE(TRIDENT, BLADE3D) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEi7) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEi7D) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEi1) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEi1D) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEAi1) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEAi1D) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEE4) },
+	{ PCI_VDEVICE(TRIDENT, TGUI9440) },
+	{ PCI_VDEVICE(TRIDENT, TGUI9660) },
+	{ PCI_VDEVICE(TRIDENT, IMAGE975) },
+	{ PCI_VDEVICE(TRIDENT, IMAGE985) },
+	{ PCI_VDEVICE(TRIDENT, CYBER9320) },
+	{ PCI_VDEVICE(TRIDENT, CYBER9388) },
+	{ PCI_VDEVICE(TRIDENT, CYBER9520) },
+	{ PCI_VDEVICE(TRIDENT, CYBER9525DVD) },
+	{ PCI_VDEVICE(TRIDENT, CYBER9397) },
+	{ PCI_VDEVICE(TRIDENT, CYBER9397DVD) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEXPAi1) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEXPm8) },
+	{ PCI_VDEVICE(TRIDENT, CYBERBLADEXPm16) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, trident_devices);
diff --git a/drivers/video/fbdev/uvesafb.c b/drivers/video/fbdev/uvesafb.c
index 88667fccc27b9..9d82326c744f2 100644
--- a/drivers/video/fbdev/uvesafb.c
+++ b/drivers/video/fbdev/uvesafb.c
@@ -1694,14 +1694,14 @@ static int uvesafb_probe(struct platform_device *dev)
 	i = uvesafb_vbe_init_mode(info);
 	if (i < 0) {
 		err = -EINVAL;
-		goto out;
+		goto out_mode;
 	} else {
 		mode = &par->vbe_modes[i];
 	}
 
 	if (fb_alloc_cmap(&info->cmap, 256, 0) < 0) {
 		err = -ENXIO;
-		goto out;
+		goto out_mode;
 	}
 
 	uvesafb_init_info(info, mode);
diff --git a/drivers/video/fbdev/vesafb.c b/drivers/video/fbdev/vesafb.c
index f84f4db244bfe..f1902056bd733 100644
--- a/drivers/video/fbdev/vesafb.c
+++ b/drivers/video/fbdev/vesafb.c
@@ -269,6 +269,7 @@ static int vesafb_probe(struct platform_device *dev)
 	/* ignore error return of fb_get_options */
 	fb_get_options("vesafb", &option);
 	vesafb_setup(option);
+	kfree(option);
 
 	if (si->orig_video_isVGA != VIDEO_TYPE_VLFB)
 		return -ENODEV;
diff --git a/drivers/video/fbdev/vt8623fb.c b/drivers/video/fbdev/vt8623fb.c
index df984f3a7ff64..9708d968970a5 100644
--- a/drivers/video/fbdev/vt8623fb.c
+++ b/drivers/video/fbdev/vt8623fb.c
@@ -900,8 +900,8 @@ static const struct dev_pm_ops vt8623_pci_pm_ops = {
 /* List of boards that we are trying to support */
 
 static const struct pci_device_id vt8623_devices[] = {
-	{PCI_DEVICE(PCI_VENDOR_ID_VIA, 0x3122)},
-	{0, 0, 0, 0, 0, 0, 0}
+	{ PCI_DEVICE(PCI_VENDOR_ID_VIA, 0x3122) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, vt8623_devices);
diff --git a/drivers/virt/acrn/hsm.c b/drivers/virt/acrn/hsm.c
index 74f2086fa59f2..f170ff4617fdd 100644
--- a/drivers/virt/acrn/hsm.c
+++ b/drivers/virt/acrn/hsm.c
@@ -16,6 +16,7 @@
 #include <linux/slab.h>
 
 #include <asm/acrn.h>
+#include <asm/cpuid/api.h>
 #include <asm/hypervisor.h>
 
 #include "acrn_drv.h"
diff --git a/drivers/virt/acrn/irqfd.c b/drivers/virt/acrn/irqfd.c
index acf8cd5f8f8c2..aab15f94166ae 100644
--- a/drivers/virt/acrn/irqfd.c
+++ b/drivers/virt/acrn/irqfd.c
@@ -206,7 +206,7 @@ int acrn_irqfd_init(struct acrn_vm *vm)
 {
 	INIT_LIST_HEAD(&vm->irqfds);
 	mutex_init(&vm->irqfds_lock);
-	vm->irqfd_wq = alloc_workqueue("acrn_irqfd-%u", 0, 0, vm->vmid);
+	vm->irqfd_wq = alloc_workqueue("acrn_irqfd-%u", WQ_PERCPU, 0, vm->vmid);
 	if (!vm->irqfd_wq)
 		return -ENOMEM;
 
diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c
index 910a1de0d5a72..d186ae55cf63b 100644
--- a/drivers/virt/coco/sev-guest/sev-guest.c
+++ b/drivers/virt/coco/sev-guest/sev-guest.c
@@ -176,6 +176,7 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques
 	struct snp_guest_req req = {};
 	int ret, npages = 0, resp_len;
 	sockptr_t certs_address;
+	u64 pfn;
 
 	if (sockptr_is_null(io->req_data) || sockptr_is_null(io->resp_data))
 		return -EINVAL;
@@ -215,10 +216,11 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques
 	if (!req.certs_data)
 		return -ENOMEM;
 
+	pfn = PHYS_PFN(virt_to_phys(req.certs_data));
 	ret = set_memory_decrypted((unsigned long)req.certs_data, npages);
 	if (ret) {
 		pr_err("failed to mark page shared, ret=%d\n", ret);
-		free_pages_exact(req.certs_data, npages << PAGE_SHIFT);
+		snp_leak_pages(pfn, npages);
 		return -EFAULT;
 	}
 
@@ -272,10 +274,12 @@ static int get_ext_report(struct snp_guest_dev *snp_dev, struct snp_guest_reques
 	kfree(report_resp);
 e_free_data:
 	if (npages) {
-		if (set_memory_encrypted((unsigned long)req.certs_data, npages))
+		if (set_memory_encrypted((unsigned long)req.certs_data, npages)) {
 			WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n");
-		else
+			snp_leak_pages(pfn, npages);
+		} else {
 			free_pages_exact(req.certs_data, npages << PAGE_SHIFT);
+		}
 	}
 	return ret;
 }
diff --git a/drivers/virt/vmgenid.c b/drivers/virt/vmgenid.c
index 66135eac3abff..2cf0096aa217f 100644
--- a/drivers/virt/vmgenid.c
+++ b/drivers/virt/vmgenid.c
@@ -75,7 +75,8 @@ static int vmgenid_add_acpi(struct device *dev, struct vmgenid_state *state)
 	phys_addr = (obj->package.elements[0].integer.value << 0) |
 		    (obj->package.elements[1].integer.value << 32);
 
-	virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE, MEMREMAP_WB);
+	virt_addr = devm_memremap(&device->dev, phys_addr, VMGENID_SIZE,
+				  MEMREMAP_WB | MEMREMAP_DEC);
 	if (IS_ERR(virt_addr)) {
 		ret = PTR_ERR(virt_addr);
 		goto out;
diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig
index dc78729ba2a5d..08cb8612d41fe 100644
--- a/drivers/watchdog/Kconfig
+++ b/drivers/watchdog/Kconfig
@@ -250,7 +250,7 @@ config DA9062_WATCHDOG
 
 config GPIO_WATCHDOG
 	tristate "Watchdog device controlled through GPIO-line"
-	depends on OF_GPIO
+	depends on GPIOLIB && (ACPI || OF)
 	select WATCHDOG_CORE
 	help
 	  If you say yes here you get support for watchdog device
@@ -882,7 +882,7 @@ config TEGRA_WATCHDOG
 	  module will be called tegra_wdt.
 
 config QCOM_WDT
-	tristate "QCOM watchdog"
+	tristate "Qualcomm watchdog"
 	depends on HAS_IOMEM
 	depends on ARCH_QCOM || COMPILE_TEST
 	select WATCHDOG_CORE
@@ -1093,7 +1093,7 @@ config SPRD_WATCHDOG
 	  by Spreadtrum system.
 
 config PM8916_WATCHDOG
-	tristate "QCOM PM8916 pmic watchdog"
+	tristate "Qualcomm PM8916 pmic watchdog"
 	depends on OF && MFD_SPMI_PMIC
 	select WATCHDOG_CORE
 	help
@@ -1270,19 +1270,6 @@ config GEODE_WDT
 	  You can compile this driver directly into the kernel, or use
 	  it as a module.  The module will be called geodewdt.
 
-config SC520_WDT
-	tristate "AMD Elan SC520 processor Watchdog"
-	depends on MELAN || COMPILE_TEST
-	help
-	  This is the driver for the hardware watchdog built in to the
-	  AMD "Elan" SC520 microcomputer commonly used in embedded systems.
-	  This watchdog simply watches your kernel to make sure it doesn't
-	  freeze, and if it does, it reboots your computer after a certain
-	  amount of time.
-
-	  You can compile this driver directly into the kernel, or use
-	  it as a module.  The module will be called sc520_wdt.
-
 config SBC_FITPC2_WATCHDOG
 	tristate "Compulab SBC-FITPC2 watchdog"
 	depends on (X86 || COMPILE_TEST) && HAS_IOPORT
@@ -1547,7 +1534,7 @@ config NV_TCO
 
 config RDC321X_WDT
 	tristate "RDC R-321x SoC watchdog"
-	depends on X86_RDC321X || COMPILE_TEST
+	depends on X86_32 || COMPILE_TEST
 	depends on PCI
 	help
 	  This is the driver for the built in hardware watchdog
@@ -1712,19 +1699,6 @@ config W83977F_WDT
 	  To compile this driver as a module, choose M here: the
 	  module will be called w83977f_wdt.
 
-config MACHZ_WDT
-	tristate "ZF MachZ Watchdog"
-	depends on (X86 || COMPILE_TEST) && HAS_IOPORT
-	help
-	  If you are using a ZF Micro MachZ processor, say Y here, otherwise
-	  N.  This is the driver for the watchdog timer built-in on that
-	  processor using ZF-Logic interface.  This watchdog simply watches
-	  your kernel to make sure it doesn't freeze, and if it does, it
-	  reboots your computer after a certain amount of time.
-
-	  To compile this driver as a module, choose M here: the
-	  module will be called machzwd.
-
 config SBC_EPX_C3_WATCHDOG
 	tristate "Winsystems SBC EPX-C3 watchdog"
 	depends on (X86 || COMPILE_TEST) && HAS_IOPORT
@@ -2011,7 +1985,7 @@ config MT7621_WDT
 config PIC32_WDT
 	tristate "Microchip PIC32 hardware watchdog"
 	select WATCHDOG_CORE
-	depends on MACH_PIC32 || (MIPS && COMPILE_TEST)
+	depends on MACH_PIC32 || COMPILE_TEST
 	help
 	  Watchdog driver for the built in watchdog hardware in a PIC32.
 
@@ -2024,7 +1998,7 @@ config PIC32_WDT
 config PIC32_DMT
 	tristate "Microchip PIC32 Deadman Timer"
 	select WATCHDOG_CORE
-	depends on MACH_PIC32 || (MIPS && COMPILE_TEST)
+	depends on MACH_PIC32 || COMPILE_TEST
 	help
 	  Watchdog driver for PIC32 instruction fetch counting timer. This
 	  specific timer is typically be used in mission critical and safety
@@ -2132,6 +2106,15 @@ config WATCHDOG_RTAS
 
 # RISC-V Architecture
 
+config ATCWDT200_WATCHDOG
+	tristate "Andes ATCWDT200 Watchdog support"
+	depends on ARCH_ANDES || COMPILE_TEST
+	help
+	  Driver for the Andes ATCWDT200 watchdog timer. It provides access to
+	  programmable reset and interrupt counters with clock-source dependent
+	  timing. The driver automatically detects the supported IntTime bit-width
+	  and is fully integrated with the Linux Watchdog Framework.
+
 config STARFIVE_WATCHDOG
 	tristate "StarFive Watchdog support"
 	depends on ARCH_STARFIVE || COMPILE_TEST
@@ -2270,10 +2253,11 @@ config MIXCOMWD
 	  Most people will say N.
 
 config WDT
-	tristate "WDT Watchdog timer"
+	tristate "ICS WDT500P/501P Watchdog timer"
 	depends on ISA
 	help
-	  If you have a WDT500P or WDT501P watchdog board, say Y here,
+	  If you have an Industrial Computer Source (ICS) WDT500P or WDT501P
+	  watchdog board, say Y here,
 	  otherwise N. It is not possible to probe for this board, which means
 	  that you have to inform the kernel about the IO port and IRQ that
 	  is needed (you can do this via the io and irq parameters)
@@ -2304,10 +2288,11 @@ config PCIPCWATCHDOG
 	  Most people will say N.
 
 config WDTPCI
-	tristate "PCI-WDT500/501 Watchdog timer"
+	tristate "ICS PCI-WDT500/501 Watchdog timer"
 	depends on PCI && HAS_IOPORT
 	help
-	  If you have a PCI-WDT500/501 watchdog board, say Y here, otherwise N.
+	  If you have an Industrial Computer Source (ICS) PCI-WDT500/501 watchdog
+	  board, say Y here, otherwise N.
 
 	  If you have a PCI-WDT501 watchdog board then you can enable the
 	  temperature sensor by setting the type parameter to 501.
@@ -2354,4 +2339,17 @@ config KEEMBAY_WATCHDOG
 	  To compile this driver as a module, choose M here: the
 	  module will be called keembay_wdt.
 
+config GUNYAH_WATCHDOG
+	tristate "Qualcomm Gunyah Watchdog"
+	depends on ARCH_QCOM || COMPILE_TEST
+	depends on HAVE_ARM_SMCCC
+	select WATCHDOG_CORE
+	help
+	  Say Y here to include support for watchdog timer provided by the
+	  Gunyah hypervisor. The driver uses ARM SMC Calling Convention (SMCCC)
+	  to interact with Gunyah Watchdog.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called gunyah_wdt.
+
 endif # WATCHDOG
diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile
index d2fb16b9f9ce7..bc1d52220f223 100644
--- a/drivers/watchdog/Makefile
+++ b/drivers/watchdog/Makefile
@@ -103,6 +103,7 @@ obj-$(CONFIG_MSC313E_WATCHDOG) += msc313e_wdt.o
 obj-$(CONFIG_APPLE_WATCHDOG) += apple_wdt.o
 obj-$(CONFIG_SUNPLUS_WATCHDOG) += sunplus_wdt.o
 obj-$(CONFIG_MARVELL_GTI_WDT) += marvell_gti_wdt.o
+obj-$(CONFIG_GUNYAH_WATCHDOG) += gunyah_wdt.o
 
 # X86 (i386 + ia64 + x86_64) Architecture
 obj-$(CONFIG_ACQUIRE_WDT) += acquirewdt.o
@@ -116,7 +117,6 @@ obj-$(CONFIG_EXAR_WDT) += exar_wdt.o
 obj-$(CONFIG_F71808E_WDT) += f71808e_wdt.o
 obj-$(CONFIG_SP5100_TCO) += sp5100_tco.o
 obj-$(CONFIG_GEODE_WDT) += geodewdt.o
-obj-$(CONFIG_SC520_WDT) += sc520_wdt.o
 obj-$(CONFIG_SBC_FITPC2_WATCHDOG) += sbc_fitpc2_wdt.o
 obj-$(CONFIG_EUROTECH_WDT) += eurotechwdt.o
 obj-$(CONFIG_IB700_WDT) += ib700wdt.o
@@ -146,7 +146,6 @@ obj-$(CONFIG_VIA_WDT) += via_wdt.o
 obj-$(CONFIG_W83627HF_WDT) += w83627hf_wdt.o
 obj-$(CONFIG_W83877F_WDT) += w83877f_wdt.o
 obj-$(CONFIG_W83977F_WDT) += w83977f_wdt.o
-obj-$(CONFIG_MACHZ_WDT) += machzwd.o
 obj-$(CONFIG_SBC_EPX_C3_WATCHDOG) += sbc_epx_c3.o
 obj-$(CONFIG_INTEL_MID_WATCHDOG) += intel-mid_wdt.o
 obj-$(CONFIG_INTEL_OC_WATCHDOG) += intel_oc_wdt.o
@@ -198,6 +197,7 @@ obj-$(CONFIG_PSERIES_WDT) += pseries-wdt.o
 obj-$(CONFIG_WATCHDOG_RTAS) += wdrtas.o
 
 # RISC-V Architecture
+obj-$(CONFIG_ATCWDT200_WATCHDOG) += atcwdt200_wdt.o
 obj-$(CONFIG_STARFIVE_WATCHDOG) += starfive-wdt.o
 
 # S390 Architecture
diff --git a/drivers/watchdog/apple_wdt.c b/drivers/watchdog/apple_wdt.c
index 66a158f67a712..6b9b0f9b05ced 100644
--- a/drivers/watchdog/apple_wdt.c
+++ b/drivers/watchdog/apple_wdt.c
@@ -218,6 +218,7 @@ static int apple_wdt_suspend(struct device *dev)
 static DEFINE_SIMPLE_DEV_PM_OPS(apple_wdt_pm_ops, apple_wdt_suspend, apple_wdt_resume);
 
 static const struct of_device_id apple_wdt_of_match[] = {
+	{ .compatible = "apple,t8103-wdt" },
 	{ .compatible = "apple,wdt" },
 	{},
 };
diff --git a/drivers/watchdog/at91sam9_wdt.h b/drivers/watchdog/at91sam9_wdt.h
index 298d545df1a15..2020694f8f6f3 100644
--- a/drivers/watchdog/at91sam9_wdt.h
+++ b/drivers/watchdog/at91sam9_wdt.h
@@ -9,6 +9,8 @@
  * Watchdog Timer (WDT) - System peripherals regsters.
  * Based on AT91SAM9261 datasheet revision D.
  * Based on SAM9X60 datasheet.
+ * Based on SAMA7G5 datasheet.
+ * Based on SAM9X75 datasheet.
  *
  */
 
@@ -27,10 +29,10 @@
 #define  AT91_SAM9X60_PERIODRST	BIT(4)		/* Period Reset */
 #define  AT91_SAM9X60_RPTHRST	BIT(5)		/* Minimum Restart Period */
 #define  AT91_WDT_WDFIEN	BIT(12)		/* Fault Interrupt Enable */
-#define  AT91_SAM9X60_WDDIS	BIT(12)		/* Watchdog Disable */
+#define  AT91_SAM9X60_WDDIS	BIT(12)		/* Watchdog Disable (SAM9X60, SAMA7G5, SAM9X75) */
 #define  AT91_WDT_WDRSTEN	BIT(13)		/* Reset Processor */
 #define  AT91_WDT_WDRPROC	BIT(14)		/* Timer Restart */
-#define  AT91_WDT_WDDIS		BIT(15)		/* Watchdog Disable */
+#define  AT91_WDT_WDDIS		BIT(15)		/* Watchdog Disable (SAMA5, AT91SAM9261) */
 #define  AT91_WDT_WDD		(0xfffUL << 16)		/* Delta Value */
 #define  AT91_WDT_SET_WDD(x)	(((x) << 16) & AT91_WDT_WDD)
 #define  AT91_WDT_WDDBGHLT	BIT(28)		/* Debug Halt */
diff --git a/drivers/watchdog/atcwdt200_wdt.c b/drivers/watchdog/atcwdt200_wdt.c
new file mode 100644
index 0000000000000..8e3b18aea368f
--- /dev/null
+++ b/drivers/watchdog/atcwdt200_wdt.c
@@ -0,0 +1,580 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Andes ATCWDT200 watchdog timer driver.
+ *
+ * Copyright (C) 2025 Andes Technology Corporation
+ */
+
+#include <linux/bitfield.h>
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/dev_printk.h>
+#include <linux/math64.h>
+#include <linux/minmax.h>
+#include <linux/moduleparam.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_platform.h>
+#include <linux/platform_device.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/watchdog.h>
+
+/* Register definitions */
+#define REG_CTRL		0x10
+#define REG_RESTART		0x14
+#define REG_WRITE_EN		0x18
+#define REG_STATUS		0x1C
+
+/* Control Register */
+#define CTRL_RST_TIME_MSK	GENMASK(10, 8)
+#define CTRL_RST_TIME_SET(x)	FIELD_PREP(CTRL_RST_TIME_MSK, x)
+#define CTRL_INT_TIME_MSK	GENMASK(7, 4)
+#define CTRL_INT_TIME_SET(x)	FIELD_PREP(CTRL_INT_TIME_MSK, x)
+#define CTRL_INT_TIME_GET(x)	FIELD_GET(CTRL_INT_TIME_MSK, x)
+#define CTRL_RST_EN		BIT(3)
+#define CTRL_CLK_SEL		BIT(1)
+#define CTRL_CLK_SEL_PCLK	1
+#define CTRL_CLK_SEL_SET(x)	FIELD_PREP(CTRL_CLK_SEL, x)
+#define CTRL_WDT_EN		BIT(0)
+
+/* Restart Register */
+#define RESTART_MAGIC		0xCAFE
+
+/* Write Enable Register */
+#define WRITE_EN_MAGIC		0x5AA5
+
+/* Status Register */
+#define STATUS_INT_EXPIRED	BIT(1)
+
+/* The default timeout value in seconds */
+#define ATCWDT_TIMEOUT		4
+
+/* Define the array size for each timer type */
+#define TMR_SZ_RST		8
+#define TMR_SZ_INT_16		8
+#define TMR_SZ_INT_32		16
+
+#define DRV_NAME		"atcwdt200"
+/**
+ * enum timer_type - Supported timer types for ATCWDT200 watchdog driver
+ * @TMR_RST:      Reset timer (non-interrupt).
+ * @TMR_INT_16:   16-bit interrupt timer supported by hardware.
+ * @TMR_INT_32:   32-bit interrupt timer supported by hardware.
+ * @TMR_UNKNOWN:  Timer type cannot be determined.
+ */
+enum timer_type {
+	TMR_RST,
+	TMR_INT_16,
+	TMR_INT_32,
+	TMR_UNKNOWN
+};
+
+static unsigned int timeout = ATCWDT_TIMEOUT;
+static bool nowayout = WATCHDOG_NOWAYOUT;
+
+/**
+ * struct atcwdt_drv - ATCWDT200 watchdog driver private data
+ * @wdt_dev:        Watchdog device used by the watchdog framework.
+ * @regmap:         Register map for accessing hardware registers.
+ * @clk:            Hardware clock used by the watchdog timer.
+ * @lock:           Spinlock protecting register accesses and driver state.
+ * @clk_freq:       Input clock frequency of the ATCWDT200.
+ * @clk_src:        Selected clock source for the watchdog timer.
+ * @int_timer_type: Detected interrupt timer type (16-bit, 32-bit, or unknown).
+ */
+struct atcwdt_drv {
+	struct watchdog_device	wdt_dev;
+	struct regmap		*regmap;
+	struct clk		*clk;
+	spinlock_t		lock;
+	unsigned int		clk_freq;
+	unsigned char		clk_src;
+	unsigned char		int_timer_type;
+};
+
+static const struct watchdog_info atcwdt_info = {
+	.identity = DRV_NAME,
+	.options = WDIOF_SETTIMEOUT |
+		   WDIOF_KEEPALIVEPING |
+		   WDIOF_MAGICCLOSE,
+};
+
+/**
+ * atcwdt_get_index - Get the interval value for the specified timer type
+ * @index: The index of the interval in the array
+ * @timer_type: The type of timer, which can be TMR_RST, TMR_INT_16, or
+ *              TMR_INT_32.
+ *
+ * This function retrieves the interval value based on the timer type and
+ * ensures the index stays within the valid range for the given timer type.
+ * For TMR_RST:
+ *  - The maximum array size is 8 (index range: 0-7).
+ * For TMR_INT_16:
+ *  - The maximum array size is 8 (index range: 0-7).
+ * For TMR_INT_32:
+ *  - The maximum array size is 16 (index range: 0-15).
+ *
+ * If the index exceeds the maximum array size, the function will return
+ * the last element of the respective array.
+ */
+static inline unsigned char atcwdt_get_index(unsigned char index,
+					     enum timer_type timer_type)
+{
+	static const unsigned char rst_timer_interval[TMR_SZ_RST] = {
+		7, 8, 9, 10, 11, 12, 13, 14};
+	static const unsigned char int_timer_interval[TMR_SZ_INT_32] = {
+		6, 8, 10, 11, 12, 13, 14, 15, 17, 19, 21, 23, 25, 27, 29, 31};
+	unsigned char array_index;
+
+	if (timer_type == TMR_RST) {
+		array_index = min(index, TMR_SZ_RST - 1);
+		return rst_timer_interval[array_index];
+	}
+
+	if (timer_type == TMR_INT_32)
+		array_index = min(index, TMR_SZ_INT_32 - 1);
+	else
+		array_index = min(index, TMR_SZ_INT_16 - 1);
+
+	return int_timer_interval[array_index];
+}
+
+/**
+ * atcwdt_get_clock_period - Calculate the closest clock period based on a
+ *                           given tick count
+ * @tick: The target tick count to match
+ * @timer_type: The type of timer, which can be TMR_RST, TMR_INT_16, or
+ *              TMR_INT_32.
+ * @index: Pointer to store the index of the selected parameter
+ *
+ * This function calculates the closest clock period to the given tick count
+ * by iterating through the timer parameters and selecting the one that
+ * minimizes the difference between the target tick count and the calculated
+ * clock period. The function determines the index of the closest parameter
+ * and returns the difference between the target tick count and the selected
+ * clock period.
+ *
+ * Return: The difference between the target tick count and the selected
+ * clock period.
+ */
+static long long atcwdt_get_clock_period(long long tick,
+					 enum timer_type timer_type,
+					 unsigned char *index)
+{
+	long long result;
+	unsigned char size;
+	char i;
+
+	if (timer_type == TMR_RST)
+		size = TMR_SZ_RST;
+	else if (timer_type == TMR_INT_32)
+		size = TMR_SZ_INT_32;
+	else
+		size = TMR_SZ_INT_16;
+
+	*index = size - 1;
+	for (i = 0; i < size; i++) {
+		result = tick - (1LL << atcwdt_get_index(i, timer_type));
+
+		if (result <= 1) {
+			*index = i;
+			break;
+		}
+	}
+
+	return result;
+}
+
+/**
+ * atcwdt_get_timeout_params - Calculate optimal parameters for Watchdog Timer
+ * @drv_data: Pointer to the Watchdog driver data structure
+ * @timeout: Desired timeout value (in seconds)
+ * @int_timer_params: Pointer to store the calculated interrupt timer
+ *                    parameter index
+ * @rst_timer_params: Pointer to store the calculated reset timer parameter
+ *                    index
+ *
+ * This function calculates the optimal parameter combination for the
+ * interrupt timer and reset timer of the Watchdog Timer to achieve a
+ * timeout value closest to, but not less than the specified timeout.
+ *
+ * Algorithm:
+ * 1. The parameters for both the interrupt timer and reset timer are
+ *    predefined as a series of options represented as powers of 2.
+ * 2. The function first determines the interrupt timer's parameter index
+ *    that provides a time closest to and not exceeding the desired timeout.
+ * 3. Based on the selected interrupt timer, it calculates the required
+ *    reset timer parameter to ensure the total timeout matches the target.
+ *
+ * Return: The calculated parameter indices are stored in the provided
+ *         pointers.
+ */
+static void atcwdt_get_timeout_params(struct atcwdt_drv *drv_data,
+				      unsigned int timeout,
+				      unsigned char *int_timer_params,
+				      unsigned char *rst_timer_params)
+{
+	long long rest_time_ms;
+	long long result;
+	long long tick;
+	unsigned char rst_index;
+	unsigned char int_index;
+	unsigned char above;
+	unsigned char below;
+
+	tick = (long long)timeout * drv_data->clk_freq;
+	result = atcwdt_get_clock_period(tick,
+					 drv_data->int_timer_type,
+					 &above);
+	if (result == 0 || above == 0) {
+		*int_timer_params = above;
+		*rst_timer_params = 0;
+		return;
+	}
+	below = above - 1;
+
+	int_index = atcwdt_get_index(below, drv_data->int_timer_type);
+	rest_time_ms = timeout * 1000LL
+		       - div64_s64(1000LL << int_index, drv_data->clk_freq);
+
+	result = atcwdt_get_clock_period(rest_time_ms * drv_data->clk_freq,
+					 TMR_RST,
+					 &rst_index);
+
+	if (result > 1) {
+		*int_timer_params = above;
+		*rst_timer_params = 0;
+	} else {
+		*int_timer_params = below;
+		*rst_timer_params = rst_index;
+	}
+}
+
+/**
+ * atcwdt_get_int_timer_type - Get the supported interrupt timer type.
+ * @drv_data: Pointer to the watchdog driver data structure.
+ *
+ * This function tests the writable bits in the IntTime field of the control
+ * register to determine the interrupt timer type supported by the hardware.
+ *
+ * Note: This function must only be called when the ATCWDT200 watchdog is
+ * disabled. If the watchdog is enabled, this function returns TMR_UNKNOWN.
+ *
+ * Returns: The interrupt timer type supported by the hardware.
+ */
+static int atcwdt_get_int_timer_type(struct atcwdt_drv *drv_data)
+{
+	struct device *dev = drv_data->wdt_dev.parent;
+	unsigned int val;
+	int ret  = 0;
+
+	spin_lock(&drv_data->lock);
+	regmap_read(drv_data->regmap, REG_CTRL, &val);
+	if (val & CTRL_WDT_EN) {
+		spin_unlock(&drv_data->lock);
+		return TMR_UNKNOWN;
+	}
+
+	/*
+	 * Configures the IntTime field with the maximum mask value
+	 * (CTRL_INT_TIME_MSK), reads its value from the control register
+	 * to identify the maximum writable bits.
+	 */
+	regmap_write(drv_data->regmap, REG_WRITE_EN, WRITE_EN_MAGIC);
+	regmap_write(drv_data->regmap, REG_CTRL, CTRL_INT_TIME_MSK);
+	regmap_read(drv_data->regmap, REG_CTRL, &val);
+	spin_unlock(&drv_data->lock);
+
+	val = CTRL_INT_TIME_GET(val);
+	switch (val) {
+	case 7:
+		drv_data->int_timer_type = TMR_INT_16;
+		break;
+	case 15:
+		drv_data->int_timer_type = TMR_INT_32;
+		break;
+	default:
+		drv_data->int_timer_type = TMR_UNKNOWN;
+		ret = dev_err_probe(dev, -ENODEV,
+				    "Failed to detect interrupt timer type\n");
+	}
+
+	return ret;
+}
+
+static int atcwdt_ping(struct watchdog_device *wdt_dev)
+{
+	struct atcwdt_drv *drv_data = watchdog_get_drvdata(wdt_dev);
+
+	spin_lock(&drv_data->lock);
+	regmap_write(drv_data->regmap, REG_WRITE_EN, WRITE_EN_MAGIC);
+	regmap_write(drv_data->regmap, REG_RESTART, RESTART_MAGIC);
+	regmap_update_bits(drv_data->regmap, REG_STATUS, STATUS_INT_EXPIRED,
+			   STATUS_INT_EXPIRED);
+	spin_unlock(&drv_data->lock);
+
+	return 0;
+}
+
+static int atcwdt_set_timeout(struct watchdog_device *wdt_dev,
+			      unsigned int timeout)
+{
+	struct atcwdt_drv *drv_data = watchdog_get_drvdata(wdt_dev);
+	unsigned int value;
+	unsigned char  rst_val;
+	unsigned char  int_val;
+
+	wdt_dev->timeout = timeout;
+	atcwdt_get_timeout_params(drv_data, timeout, &int_val, &rst_val);
+
+	spin_lock(&drv_data->lock);
+	regmap_write(drv_data->regmap, REG_WRITE_EN, WRITE_EN_MAGIC);
+
+	value = CTRL_RST_TIME_SET(rst_val) |
+		CTRL_INT_TIME_SET(int_val) |
+		CTRL_CLK_SEL_SET(drv_data->clk_src);
+	regmap_update_bits(drv_data->regmap,
+			   REG_CTRL,
+			   CTRL_RST_TIME_MSK |
+			   CTRL_INT_TIME_MSK |
+			   CTRL_CLK_SEL,
+			   value);
+
+	spin_unlock(&drv_data->lock);
+	atcwdt_ping(wdt_dev);
+
+	return 0;
+}
+
+static int atcwdt_start(struct watchdog_device *wdt_dev)
+{
+	struct atcwdt_drv *drv_data = watchdog_get_drvdata(wdt_dev);
+
+	atcwdt_set_timeout(wdt_dev, wdt_dev->timeout);
+
+	spin_lock(&drv_data->lock);
+	regmap_write(drv_data->regmap, REG_WRITE_EN, WRITE_EN_MAGIC);
+	regmap_update_bits(drv_data->regmap,
+			   REG_CTRL,
+			   CTRL_RST_EN | CTRL_WDT_EN,
+			   CTRL_RST_EN | CTRL_WDT_EN);
+
+	spin_unlock(&drv_data->lock);
+
+	return 0;
+}
+
+static int atcwdt_stop(struct watchdog_device *wdt_dev)
+{
+	struct atcwdt_drv *drv_data = watchdog_get_drvdata(wdt_dev);
+
+	spin_lock(&drv_data->lock);
+	regmap_write(drv_data->regmap, REG_WRITE_EN, WRITE_EN_MAGIC);
+	regmap_update_bits(drv_data->regmap,
+			   REG_CTRL,
+			   CTRL_RST_EN | CTRL_WDT_EN,
+			   0);
+	spin_unlock(&drv_data->lock);
+
+	return 0;
+}
+
+static int atcwdt_restart(struct watchdog_device *wdt_dev,
+			  unsigned long action, void *data)
+{
+	struct atcwdt_drv *drv_data = watchdog_get_drvdata(wdt_dev);
+
+	atcwdt_set_timeout(wdt_dev, 0);
+
+	spin_lock(&drv_data->lock);
+	regmap_write(drv_data->regmap, REG_WRITE_EN, WRITE_EN_MAGIC);
+	regmap_update_bits(drv_data->regmap,
+			   REG_CTRL,
+			   CTRL_RST_EN | CTRL_WDT_EN,
+			   CTRL_RST_EN | CTRL_WDT_EN);
+	spin_unlock(&drv_data->lock);
+
+	return 0;
+}
+
+static const struct watchdog_ops atcwdt_ops = {
+	.owner = THIS_MODULE,
+	.start = atcwdt_start,
+	.stop = atcwdt_stop,
+	.ping = atcwdt_ping,
+	.set_timeout = atcwdt_set_timeout,
+	.restart = atcwdt_restart,
+};
+
+static int atcwdt_init_resource(struct platform_device *pdev,
+				struct atcwdt_drv *drv_data)
+{
+	struct device *dev = &pdev->dev;
+	void __iomem *base;
+	const struct regmap_config cfg = {
+		.name = "atcwdt",
+		.reg_bits = 32,
+		.val_bits = 32,
+		.cache_type = REGCACHE_NONE,
+		.reg_stride = 4,
+		.max_register = REG_STATUS,
+	};
+
+	base = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(base))
+		return dev_err_probe(dev, PTR_ERR(base),
+				     "Failed to ioremap I/O resource\n");
+
+	drv_data->regmap = devm_regmap_init_mmio(dev, base, &cfg);
+	if (IS_ERR(drv_data->regmap))
+		return dev_err_probe(dev, PTR_ERR(drv_data->regmap),
+				     "Failed to create regmap\n");
+
+	return 0;
+}
+
+static int atcwdt_enable_clk(struct atcwdt_drv *drv_data)
+{
+	struct device *dev = drv_data->wdt_dev.parent;
+	unsigned int val;
+	int clk_src;
+
+	drv_data->clk = devm_clk_get_enabled(dev, NULL);
+	if (IS_ERR(drv_data->clk))
+		return dev_err_probe(dev, PTR_ERR(drv_data->clk),
+				     "Failed to get watchdog clock\n");
+
+	drv_data->clk_freq = clk_get_rate(drv_data->clk);
+	if (!drv_data->clk_freq)
+		return dev_err_probe(dev, -EINVAL,
+				     "Failed to get clock rate\n");
+
+	clk_src = device_property_read_u32(dev, "andestech,clock-source", &val);
+	drv_data->clk_src = (!clk_src && val != 0) ? CTRL_CLK_SEL_PCLK : 0;
+
+	return 0;
+}
+
+static int atcwdt_init_wdt_device(struct device *dev,
+				  struct atcwdt_drv *drv_data)
+{
+	struct watchdog_device *wdd = &drv_data->wdt_dev;
+
+	wdd->parent = dev;
+	wdd->info = &atcwdt_info;
+	wdd->ops = &atcwdt_ops;
+	wdd->timeout = ATCWDT_TIMEOUT;
+	wdd->min_timeout = 1;
+
+	watchdog_set_nowayout(wdd, nowayout);
+	watchdog_set_drvdata(wdd, drv_data);
+
+	return 0;
+}
+
+static void atcwdt_calc_max_timeout(struct atcwdt_drv *drv_data)
+{
+	unsigned char rst_idx = atcwdt_get_index(0xFF, TMR_RST);
+	unsigned char int_idx = atcwdt_get_index(0xFF,
+						 drv_data->int_timer_type);
+
+	drv_data->wdt_dev.max_timeout =
+		((1U << rst_idx) + (1U << int_idx)) / drv_data->clk_freq;
+}
+
+static int atcwdt_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct atcwdt_drv *drv_data;
+	int ret;
+
+	drv_data = devm_kzalloc(dev, sizeof(*drv_data), GFP_KERNEL);
+	if (!drv_data)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, drv_data);
+	spin_lock_init(&drv_data->lock);
+
+	ret = atcwdt_init_wdt_device(dev, drv_data);
+	if (ret)
+		return ret;
+
+	ret = atcwdt_init_resource(pdev, drv_data);
+	if (ret)
+		return ret;
+
+	ret = atcwdt_enable_clk(drv_data);
+	if (ret)
+		return ret;
+
+	ret = atcwdt_get_int_timer_type(drv_data);
+	if (ret)
+		return ret;
+
+	atcwdt_calc_max_timeout(drv_data);
+
+	ret = devm_watchdog_register_device(dev, &drv_data->wdt_dev);
+
+	return ret;
+}
+
+static int atcwdt_suspend(struct device *dev)
+{
+	struct atcwdt_drv *drv_data = dev_get_drvdata(dev);
+
+	if (watchdog_active(&drv_data->wdt_dev)) {
+		atcwdt_stop(&drv_data->wdt_dev);
+		clk_disable_unprepare(drv_data->clk);
+	}
+
+	return 0;
+}
+
+static int atcwdt_resume(struct device *dev)
+{
+	struct atcwdt_drv *drv_data = dev_get_drvdata(dev);
+	int ret = 0;
+
+	if (watchdog_active(&drv_data->wdt_dev)) {
+		ret = clk_prepare_enable(drv_data->clk);
+		if (ret)
+			return ret;
+		atcwdt_start(&drv_data->wdt_dev);
+		atcwdt_ping(&drv_data->wdt_dev);
+	}
+
+	return ret;
+}
+
+static const struct of_device_id atcwdt_match[] = {
+	{ .compatible = "andestech,ae350-wdt" },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, atcwdt_match);
+
+static DEFINE_SIMPLE_DEV_PM_OPS(atcwdt_pm_ops, atcwdt_suspend, atcwdt_resume);
+
+static struct platform_driver atcwdt_driver = {
+	.probe = atcwdt_probe,
+	.driver = {
+		.name = DRV_NAME,
+		.of_match_table = atcwdt_match,
+		.pm = pm_sleep_ptr(&atcwdt_pm_ops),
+	},
+};
+
+module_platform_driver(atcwdt_driver);
+
+module_param(timeout, uint, 0);
+MODULE_PARM_DESC(timeout, "Watchdog timeout in seconds (default="
+		 __MODULE_STRING(ATCWDT_TIMEOUT) ")");
+
+module_param(nowayout, bool, 0);
+MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
+		 __MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("CL Wang <cl634@andestech.com>");
+MODULE_DESCRIPTION("Andes ATCWDT200 Watchdog timer driver");
diff --git a/drivers/watchdog/bcm2835_wdt.c b/drivers/watchdog/bcm2835_wdt.c
index 9fcfee63905b9..6fd8b1b8e386a 100644
--- a/drivers/watchdog/bcm2835_wdt.c
+++ b/drivers/watchdog/bcm2835_wdt.c
@@ -19,6 +19,7 @@
 #include <linux/platform_device.h>
 #include <linux/of_address.h>
 #include <linux/of_platform.h>
+#include <linux/reboot.h>
 
 #define PM_RSTC				0x1c
 #define PM_RSTS				0x20
@@ -49,8 +50,6 @@ struct bcm2835_wdt {
 	spinlock_t		lock;
 };
 
-static struct bcm2835_wdt *bcm2835_power_off_wdt;
-
 static unsigned int heartbeat;
 static bool nowayout = WATCHDOG_NOWAYOUT;
 
@@ -150,9 +149,9 @@ static struct watchdog_device bcm2835_wdt_wdd = {
  * indicate to bootcode.bin not to reboot, then most of the chip will be
  * powered off.
  */
-static void bcm2835_power_off(void)
+static int bcm2835_power_off(struct sys_off_data *data)
 {
-	struct bcm2835_wdt *wdt = bcm2835_power_off_wdt;
+	struct bcm2835_wdt *wdt = data->cb_data;
 	u32 val;
 
 	/*
@@ -166,6 +165,8 @@ static void bcm2835_power_off(void)
 
 	/* Continue with normal reset mechanism */
 	__bcm2835_restart(wdt);
+
+	return NOTIFY_DONE;
 }
 
 static int bcm2835_wdt_probe(struct platform_device *pdev)
@@ -206,28 +207,17 @@ static int bcm2835_wdt_probe(struct platform_device *pdev)
 	if (err)
 		return err;
 
-	if (of_device_is_system_power_controller(pdev->dev.parent->of_node)) {
-		if (!pm_power_off) {
-			pm_power_off = bcm2835_power_off;
-			bcm2835_power_off_wdt = wdt;
-		} else {
-			dev_info(dev, "Poweroff handler already present!\n");
-		}
-	}
+	if (of_device_is_system_power_controller(pdev->dev.parent->of_node))
+		devm_register_sys_off_handler(dev, SYS_OFF_MODE_POWER_OFF,
+					      SYS_OFF_PRIO_DEFAULT,
+					      bcm2835_power_off, wdt);
 
 	dev_info(dev, "Broadcom BCM2835 watchdog timer");
 	return 0;
 }
 
-static void bcm2835_wdt_remove(struct platform_device *pdev)
-{
-	if (pm_power_off == bcm2835_power_off)
-		pm_power_off = NULL;
-}
-
 static struct platform_driver bcm2835_wdt_driver = {
 	.probe		= bcm2835_wdt_probe,
-	.remove		= bcm2835_wdt_remove,
 	.driver = {
 		.name =		"bcm2835-wdt",
 	},
diff --git a/drivers/watchdog/gunyah_wdt.c b/drivers/watchdog/gunyah_wdt.c
new file mode 100644
index 0000000000000..49dfef459e847
--- /dev/null
+++ b/drivers/watchdog/gunyah_wdt.c
@@ -0,0 +1,261 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include <linux/arm-smccc.h>
+#include <linux/delay.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/watchdog.h>
+
+#define GUNYAH_WDT_SMCCC_CALL_VAL(func_id) \
+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_32,\
+			   ARM_SMCCC_OWNER_VENDOR_HYP, func_id)
+
+/* SMCCC function IDs for watchdog operations */
+#define GUNYAH_WDT_CONTROL   GUNYAH_WDT_SMCCC_CALL_VAL(0x0005)
+#define GUNYAH_WDT_STATUS    GUNYAH_WDT_SMCCC_CALL_VAL(0x0006)
+#define GUNYAH_WDT_PING      GUNYAH_WDT_SMCCC_CALL_VAL(0x0007)
+#define GUNYAH_WDT_SET_TIME  GUNYAH_WDT_SMCCC_CALL_VAL(0x0008)
+
+/*
+ * Control values for GUNYAH_WDT_CONTROL.
+ * Bit 0 is used to enable or disable the watchdog. If this bit is set,
+ * then the watchdog is enabled and vice versa.
+ * Bit 1 should always be set to 1 as this bit is reserved in Gunyah and
+ * it's expected to be 1.
+ */
+#define WDT_CTRL_ENABLE  (BIT(1) | BIT(0))
+#define WDT_CTRL_DISABLE BIT(1)
+
+enum gunyah_error {
+	GUNYAH_ERROR_OK				= 0,
+	GUNYAH_ERROR_UNIMPLEMENTED		= -1,
+	GUNYAH_ERROR_ARG_INVAL			= 1,
+};
+
+/**
+ * gunyah_error_remap() - Remap Gunyah hypervisor errors into a Linux error code
+ * @gunyah_error: Gunyah hypercall return value
+ */
+static inline int gunyah_error_remap(enum gunyah_error gunyah_error)
+{
+	switch (gunyah_error) {
+	case GUNYAH_ERROR_OK:
+		return 0;
+	case GUNYAH_ERROR_UNIMPLEMENTED:
+		return -EOPNOTSUPP;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int gunyah_wdt_call(unsigned long func_id, unsigned long arg1,
+			   unsigned long arg2)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_smc(func_id, arg1, arg2, &res);
+	return gunyah_error_remap(res.a0);
+}
+
+static int gunyah_wdt_start(struct watchdog_device *wdd)
+{
+	unsigned int timeout_ms;
+	struct device *dev = wdd->parent;
+	int ret;
+
+	ret = gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_DISABLE, 0);
+	if (ret && watchdog_active(wdd)) {
+		dev_err(dev, "%s: Failed to stop gunyah wdt %d\n", __func__, ret);
+		return ret;
+	}
+
+	timeout_ms = wdd->timeout * 1000;
+	ret = gunyah_wdt_call(GUNYAH_WDT_SET_TIME, timeout_ms, timeout_ms);
+	if (ret) {
+		dev_err(dev, "%s: Failed to set timeout for gunyah wdt %d\n",
+			__func__, ret);
+		return ret;
+	}
+
+	ret = gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_ENABLE, 0);
+	if (ret)
+		dev_err(dev, "%s: Failed to start gunyah wdt %d\n", __func__, ret);
+
+	return ret;
+}
+
+static int gunyah_wdt_stop(struct watchdog_device *wdd)
+{
+	return gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_DISABLE, 0);
+}
+
+static int gunyah_wdt_ping(struct watchdog_device *wdd)
+{
+	return gunyah_wdt_call(GUNYAH_WDT_PING, 0, 0);
+}
+
+static int gunyah_wdt_set_timeout(struct watchdog_device *wdd,
+				  unsigned int timeout_sec)
+{
+	wdd->timeout = timeout_sec;
+
+	if (watchdog_active(wdd))
+		return gunyah_wdt_start(wdd);
+
+	return 0;
+}
+
+static int gunyah_wdt_get_time_since_last_ping(void)
+{
+	struct arm_smccc_res res;
+
+	arm_smccc_1_1_smc(GUNYAH_WDT_STATUS, 0, 0, &res);
+	if (res.a0)
+		return gunyah_error_remap(res.a0);
+
+	return res.a2 / 1000;
+}
+
+static unsigned int gunyah_wdt_get_timeleft(struct watchdog_device *wdd)
+{
+	int seconds_since_last_ping;
+
+	seconds_since_last_ping = gunyah_wdt_get_time_since_last_ping();
+	if (seconds_since_last_ping < 0 ||
+	    seconds_since_last_ping > wdd->timeout)
+		return 0;
+
+	return wdd->timeout - seconds_since_last_ping;
+}
+
+static int gunyah_wdt_restart(struct watchdog_device *wdd,
+			      unsigned long action, void *data)
+{
+	/* Set timeout to 1ms and send a ping */
+	gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_DISABLE, 0);
+	gunyah_wdt_call(GUNYAH_WDT_SET_TIME, 1, 1);
+	gunyah_wdt_call(GUNYAH_WDT_CONTROL, WDT_CTRL_ENABLE, 0);
+	gunyah_wdt_call(GUNYAH_WDT_PING, 0, 0);
+
+	/* Wait to make sure reset occurs */
+	mdelay(100);
+
+	return 0;
+}
+
+static const struct watchdog_info gunyah_wdt_info = {
+	.identity = "Gunyah Watchdog",
+	.options = WDIOF_SETTIMEOUT
+		 | WDIOF_KEEPALIVEPING
+		 | WDIOF_MAGICCLOSE,
+};
+
+static const struct watchdog_ops gunyah_wdt_ops = {
+	.owner = THIS_MODULE,
+	.start = gunyah_wdt_start,
+	.stop = gunyah_wdt_stop,
+	.ping = gunyah_wdt_ping,
+	.set_timeout = gunyah_wdt_set_timeout,
+	.get_timeleft = gunyah_wdt_get_timeleft,
+	.restart = gunyah_wdt_restart
+};
+
+static int gunyah_wdt_probe(struct platform_device *pdev)
+{
+	struct watchdog_device *wdd;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	ret = gunyah_wdt_call(GUNYAH_WDT_STATUS, 0, 0);
+	if (ret == -EOPNOTSUPP)
+		return -ENODEV;
+
+	if (ret)
+		return dev_err_probe(dev, ret, "status check failed\n");
+
+	wdd = devm_kzalloc(dev, sizeof(*wdd), GFP_KERNEL);
+	if (!wdd)
+		return -ENOMEM;
+
+	wdd->info = &gunyah_wdt_info;
+	wdd->ops = &gunyah_wdt_ops;
+	wdd->parent = dev;
+
+	/*
+	 * Although Gunyah expects 16-bit unsigned int values as timeout values
+	 * in milliseconds, values above 0x8000 are reserved. This limits the
+	 * max timeout value to 32 seconds.
+	 */
+	wdd->max_timeout = 32; /* seconds */
+	wdd->min_timeout = 1; /* seconds */
+	wdd->timeout = wdd->max_timeout;
+
+	gunyah_wdt_stop(wdd);
+	platform_set_drvdata(pdev, wdd);
+	watchdog_set_restart_priority(wdd, 0);
+
+	return devm_watchdog_register_device(dev, wdd);
+}
+
+static void gunyah_wdt_remove(struct platform_device *pdev)
+{
+	struct watchdog_device *wdd = platform_get_drvdata(pdev);
+
+	gunyah_wdt_stop(wdd);
+}
+
+static int gunyah_wdt_suspend(struct device *dev)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+
+	if (watchdog_active(wdd))
+		gunyah_wdt_stop(wdd);
+
+	return 0;
+}
+
+static int gunyah_wdt_resume(struct device *dev)
+{
+	struct watchdog_device *wdd = dev_get_drvdata(dev);
+
+	if (watchdog_active(wdd))
+		gunyah_wdt_start(wdd);
+
+	return 0;
+}
+
+static DEFINE_SIMPLE_DEV_PM_OPS(gunyah_wdt_pm_ops, gunyah_wdt_suspend, gunyah_wdt_resume);
+
+/*
+ * Gunyah watchdog is a vendor-specific hypervisor interface provided by the
+ * Gunyah hypervisor. Using QCOM SCM driver to detect Gunyah watchdog SMCCC
+ * hypervisor service and register platform device when the service is available
+ * allows this driver to operate independently of the devicetree and avoids
+ * adding the non-hardware nodes to the devicetree.
+ */
+static const struct platform_device_id gunyah_wdt_id[] = {
+	{ .name = "gunyah-wdt" },
+	{}
+};
+MODULE_DEVICE_TABLE(platform, gunyah_wdt_id);
+
+static struct platform_driver gunyah_wdt_driver = {
+	.driver = {
+		.name = "gunyah-wdt",
+		.pm = pm_sleep_ptr(&gunyah_wdt_pm_ops),
+	},
+	.id_table = gunyah_wdt_id,
+	.probe = gunyah_wdt_probe,
+	.remove = gunyah_wdt_remove,
+};
+
+module_platform_driver(gunyah_wdt_driver);
+
+MODULE_DESCRIPTION("Gunyah Watchdog Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/watchdog/hpwdt.c b/drivers/watchdog/hpwdt.c
index 2a848c35c14d2..8af1fad2de0bd 100644
--- a/drivers/watchdog/hpwdt.c
+++ b/drivers/watchdog/hpwdt.c
@@ -23,6 +23,7 @@
 #include <linux/watchdog.h>
 #ifdef CONFIG_HPWDT_NMI_DECODING
 #include <asm/nmi.h>
+#include <asm/uv/uv.h>
 #endif
 #include <linux/crash_dump.h>
 
@@ -159,24 +160,31 @@ static int hpwdt_set_pretimeout(struct watchdog_device *wdd, unsigned int req)
 	return 0;
 }
 
-static int hpwdt_my_nmi(void)
-{
-	return ioread8(hpwdt_nmistat) & 0x6;
-}
+#define NMISTAT_EASR	BIT(0)
+#define NMISTAT_EWDOG	BIT(1)
+#define NMISTAT_RTRAP	BIT(2)
+#define NMISTAT_DBELL	BIT(3)
+#define NMISTAT_EMSWDG	BIT(4)
+#define NMISTAT_GPI	BIT(5)
+#define NMISTAT_NMIOUT	BIT(7)
 
 /*
  *	NMI Handler
  */
 static int hpwdt_pretimeout(unsigned int ulReason, struct pt_regs *regs)
 {
-	unsigned int mynmi = hpwdt_my_nmi();
-	static char panic_msg[] =
+	u8 nmistat = ioread8(hpwdt_nmistat);
+	bool mynmi = (nmistat & (NMISTAT_EWDOG | NMISTAT_RTRAP)) != 0;
+	static char panic_msg_default[] =
 		"00: An NMI occurred. Depending on your system the reason "
 		"for the NMI is logged in any one of the following resources:\n"
 		"1. Integrated Management Log (IML)\n"
 		"2. OA Syslog\n"
 		"3. OA Forward Progress Log\n"
 		"4. iLO Event Log";
+	static char panic_msg_uv[] =
+		"00: A watchdog NMI occurred.";
+	char *panic_msg = is_uv_system() ? panic_msg_uv : panic_msg_default;
 
 	if (ulReason == NMI_UNKNOWN && !mynmi)
 		return NMI_DONE;
@@ -190,7 +198,7 @@ static int hpwdt_pretimeout(unsigned int ulReason, struct pt_regs *regs)
 		hpwdt_ping_ticks(SECS_TO_TICKS(val));
 	}
 
-	hex_byte_pack(panic_msg, mynmi);
+	hex_byte_pack(panic_msg, nmistat);
 	nmi_panic(regs, panic_msg);
 
 	return NMI_HANDLED;
diff --git a/drivers/watchdog/imx7ulp_wdt.c b/drivers/watchdog/imx7ulp_wdt.c
index 03479110453ce..855dc9d5083a5 100644
--- a/drivers/watchdog/imx7ulp_wdt.c
+++ b/drivers/watchdog/imx7ulp_wdt.c
@@ -56,6 +56,7 @@ MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default="
 struct imx_wdt_hw_feature {
 	bool prescaler_enable;
 	bool post_rcs_wait;
+	bool cpu_lpm_auto_cg;
 	u32 wdog_clock_rate;
 };
 
@@ -360,7 +361,7 @@ static int __maybe_unused imx7ulp_wdt_suspend_noirq(struct device *dev)
 {
 	struct imx7ulp_wdt_device *imx7ulp_wdt = dev_get_drvdata(dev);
 
-	if (watchdog_active(&imx7ulp_wdt->wdd))
+	if (watchdog_active(&imx7ulp_wdt->wdd) && !imx7ulp_wdt->hw->cpu_lpm_auto_cg)
 		imx7ulp_wdt_stop(&imx7ulp_wdt->wdd);
 
 	clk_disable_unprepare(imx7ulp_wdt->clk);
@@ -408,10 +409,17 @@ static const struct imx_wdt_hw_feature imx93_wdt_hw = {
 	.wdog_clock_rate = 125,
 };
 
+static const struct imx_wdt_hw_feature imx94_wdt_hw = {
+	.prescaler_enable = true,
+	.wdog_clock_rate = 125,
+	.cpu_lpm_auto_cg = true,
+};
+
 static const struct of_device_id imx7ulp_wdt_dt_ids[] = {
 	{ .compatible = "fsl,imx7ulp-wdt", .data = &imx7ulp_wdt_hw, },
 	{ .compatible = "fsl,imx8ulp-wdt", .data = &imx8ulp_wdt_hw, },
 	{ .compatible = "fsl,imx93-wdt", .data = &imx93_wdt_hw, },
+	{ .compatible = "fsl,imx94-wdt", .data = &imx94_wdt_hw, },
 	{ /* sentinel */ }
 };
 MODULE_DEVICE_TABLE(of, imx7ulp_wdt_dt_ids);
diff --git a/drivers/watchdog/lenovo_se10_wdt.c b/drivers/watchdog/lenovo_se10_wdt.c
index cd0500e5080b4..503e220263f95 100644
--- a/drivers/watchdog/lenovo_se10_wdt.c
+++ b/drivers/watchdog/lenovo_se10_wdt.c
@@ -178,7 +178,7 @@ static int se10_wdt_probe(struct platform_device *pdev)
 		return -EBUSY;
 
 	chip_id = get_chipID();
-	if (chip_id != 0x5632) {
+	if (chip_id != 0x5632 && chip_id != 0x5652) {
 		release_region(CFG_PORT, CFG_SIZE);
 		return -ENODEV;
 	}
@@ -224,7 +224,7 @@ static struct platform_driver se10_wdt_driver = {
 	.probe  = se10_wdt_probe,
 };
 
-static int se10_create_platform_device(const struct dmi_system_id *id)
+static int se10_create_platform_device(void)
 {
 	int err;
 
@@ -233,9 +233,10 @@ static int se10_create_platform_device(const struct dmi_system_id *id)
 		return -ENOMEM;
 
 	err = platform_device_add(se10_pdev);
-	if (err)
+	if (err) {
 		platform_device_put(se10_pdev);
-
+		se10_pdev = NULL;
+	}
 	return err;
 }
 
@@ -246,7 +247,6 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = {
 			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "12NH"),
 		},
-		.callback = se10_create_platform_device,
 	},
 	{
 		.ident = "LENOVO-SE10",
@@ -254,7 +254,6 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = {
 			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "12NJ"),
 		},
-		.callback = se10_create_platform_device,
 	},
 	{
 		.ident = "LENOVO-SE10",
@@ -262,7 +261,6 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = {
 			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "12NK"),
 		},
-		.callback = se10_create_platform_device,
 	},
 	{
 		.ident = "LENOVO-SE10",
@@ -270,7 +268,6 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = {
 			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "12NL"),
 		},
-		.callback = se10_create_platform_device,
 	},
 	{
 		.ident = "LENOVO-SE10",
@@ -278,7 +275,62 @@ static const struct dmi_system_id se10_dmi_table[] __initconst = {
 			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
 			DMI_MATCH(DMI_PRODUCT_NAME, "12NM"),
 		},
-		.callback = se10_create_platform_device,
+	},
+	{
+		.ident = "LENOVO-SE10-G2",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "13LJ"),
+		},
+	},
+	{
+		.ident = "LENOVO-SE10-G2",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "13LK"),
+		},
+	},
+	{
+		.ident = "LENOVO-SE10-G2",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "13S1"),
+		},
+	},
+	{
+		.ident = "LENOVO-SE10-G2",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "13S2"),
+		},
+	},
+	{
+		.ident = "LENOVO-SE10-G2",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "13S3"),
+		},
+	},
+	{
+		.ident = "LENOVO-SE10-G2",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "13S4"),
+		},
+	},
+	{
+		.ident = "LENOVO-SE10-G2",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "13S5"),
+		},
+	},
+	{
+		.ident = "LENOVO-SE10-G2",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "LENOVO"),
+			DMI_MATCH(DMI_PRODUCT_NAME, "13S6"),
+		},
 	},
 	{}
 };
@@ -286,10 +338,20 @@ MODULE_DEVICE_TABLE(dmi, se10_dmi_table);
 
 static int __init se10_wdt_init(void)
 {
+	int err;
+
 	if (!dmi_check_system(se10_dmi_table))
 		return -ENODEV;
 
-	return platform_driver_register(&se10_wdt_driver);
+	err = platform_driver_register(&se10_wdt_driver);
+	if (err)
+		return err;
+
+	err = se10_create_platform_device();
+	if (err)
+		platform_driver_unregister(&se10_wdt_driver);
+
+	return err;
 }
 
 static void __exit se10_wdt_exit(void)
diff --git a/drivers/watchdog/machzwd.c b/drivers/watchdog/machzwd.c
deleted file mode 100644
index 0ae8e5bc10ae9..0000000000000
--- a/drivers/watchdog/machzwd.c
+++ /dev/null
@@ -1,452 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *  MachZ ZF-Logic Watchdog Timer driver for Linux
- *
- *  The author does NOT admit liability nor provide warranty for
- *  any of this software. This material is provided "AS-IS" in
- *  the hope that it may be useful for others.
- *
- *  Author: Fernando Fuganti <fuganti@conectiva.com.br>
- *
- *  Based on sbc60xxwdt.c by Jakob Oestergaard
- *
- *  We have two timers (wd#1, wd#2) driven by a 32 KHz clock with the
- *  following periods:
- *      wd#1 - 2 seconds;
- *      wd#2 - 7.2 ms;
- *  After the expiration of wd#1, it can generate a NMI, SCI, SMI, or
- *  a system RESET and it starts wd#2 that unconditionally will RESET
- *  the system when the counter reaches zero.
- *
- *  14-Dec-2001 Matt Domsch <Matt_Domsch@dell.com>
- *      Added nowayout module option to override CONFIG_WATCHDOG_NOWAYOUT
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/types.h>
-#include <linux/timer.h>
-#include <linux/jiffies.h>
-#include <linux/miscdevice.h>
-#include <linux/watchdog.h>
-#include <linux/fs.h>
-#include <linux/ioport.h>
-#include <linux/notifier.h>
-#include <linux/reboot.h>
-#include <linux/init.h>
-#include <linux/io.h>
-#include <linux/uaccess.h>
-
-
-/* ports */
-#define ZF_IOBASE	0x218
-#define INDEX		0x218
-#define DATA_B		0x219
-#define DATA_W		0x21A
-#define DATA_D		0x21A
-
-/* indexes */			/* size */
-#define ZFL_VERSION	0x02	/* 16   */
-#define CONTROL		0x10	/* 16   */
-#define STATUS		0x12	/* 8    */
-#define COUNTER_1	0x0C	/* 16   */
-#define COUNTER_2	0x0E	/* 8    */
-#define PULSE_LEN	0x0F	/* 8    */
-
-/* controls */
-#define ENABLE_WD1	0x0001
-#define ENABLE_WD2	0x0002
-#define RESET_WD1	0x0010
-#define RESET_WD2	0x0020
-#define GEN_SCI		0x0100
-#define GEN_NMI		0x0200
-#define GEN_SMI		0x0400
-#define GEN_RESET	0x0800
-
-
-/* utilities */
-
-#define WD1	0
-#define WD2	1
-
-#define zf_writew(port, data)  { outb(port, INDEX); outw(data, DATA_W); }
-#define zf_writeb(port, data)  { outb(port, INDEX); outb(data, DATA_B); }
-#define zf_get_ZFL_version()   zf_readw(ZFL_VERSION)
-
-
-static unsigned short zf_readw(unsigned char port)
-{
-	outb(port, INDEX);
-	return inw(DATA_W);
-}
-
-
-MODULE_AUTHOR("Fernando Fuganti <fuganti@conectiva.com.br>");
-MODULE_DESCRIPTION("MachZ ZF-Logic Watchdog driver");
-MODULE_LICENSE("GPL");
-
-static bool nowayout = WATCHDOG_NOWAYOUT;
-module_param(nowayout, bool, 0);
-MODULE_PARM_DESC(nowayout,
-		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
-
-#define PFX "machzwd"
-
-static const struct watchdog_info zf_info = {
-	.options		= WDIOF_KEEPALIVEPING | WDIOF_MAGICCLOSE,
-	.firmware_version	= 1,
-	.identity		= "ZF-Logic watchdog",
-};
-
-
-/*
- * action refers to action taken when watchdog resets
- * 0 = GEN_RESET
- * 1 = GEN_SMI
- * 2 = GEN_NMI
- * 3 = GEN_SCI
- * defaults to GEN_RESET (0)
- */
-static int action;
-module_param(action, int, 0);
-MODULE_PARM_DESC(action, "after watchdog resets, generate: "
-				"0 = RESET(*)  1 = SMI  2 = NMI  3 = SCI");
-
-static void zf_ping(struct timer_list *unused);
-
-static int zf_action = GEN_RESET;
-static unsigned long zf_is_open;
-static char zf_expect_close;
-static DEFINE_SPINLOCK(zf_port_lock);
-static DEFINE_TIMER(zf_timer, zf_ping);
-static unsigned long next_heartbeat;
-
-
-/* timeout for user land heart beat (10 seconds) */
-#define ZF_USER_TIMEO (HZ*10)
-
-/* timeout for hardware watchdog (~500ms) */
-#define ZF_HW_TIMEO (HZ/2)
-
-/* number of ticks on WD#1 (driven by a 32KHz clock, 2s) */
-#define ZF_CTIMEOUT 0xffff
-
-#ifndef ZF_DEBUG
-#define dprintk(format, args...)
-#else
-#define dprintk(format, args...)					\
-	pr_debug(":%s:%d: " format, __func__, __LINE__ , ## args)
-#endif
-
-
-static inline void zf_set_status(unsigned char new)
-{
-	zf_writeb(STATUS, new);
-}
-
-
-/* CONTROL register functions */
-
-static inline unsigned short zf_get_control(void)
-{
-	return zf_readw(CONTROL);
-}
-
-static inline void zf_set_control(unsigned short new)
-{
-	zf_writew(CONTROL, new);
-}
-
-
-/* WD#? counter functions */
-/*
- *	Just set counter value
- */
-
-static inline void zf_set_timer(unsigned short new, unsigned char n)
-{
-	switch (n) {
-	case WD1:
-		zf_writew(COUNTER_1, new);
-		fallthrough;
-	case WD2:
-		zf_writeb(COUNTER_2, new > 0xff ? 0xff : new);
-		fallthrough;
-	default:
-		return;
-	}
-}
-
-/*
- * stop hardware timer
- */
-static void zf_timer_off(void)
-{
-	unsigned int ctrl_reg = 0;
-	unsigned long flags;
-
-	/* stop internal ping */
-	timer_delete_sync(&zf_timer);
-
-	spin_lock_irqsave(&zf_port_lock, flags);
-	/* stop watchdog timer */
-	ctrl_reg = zf_get_control();
-	ctrl_reg |= (ENABLE_WD1|ENABLE_WD2);	/* disable wd1 and wd2 */
-	ctrl_reg &= ~(ENABLE_WD1|ENABLE_WD2);
-	zf_set_control(ctrl_reg);
-	spin_unlock_irqrestore(&zf_port_lock, flags);
-
-	pr_info("Watchdog timer is now disabled\n");
-}
-
-
-/*
- * start hardware timer
- */
-static void zf_timer_on(void)
-{
-	unsigned int ctrl_reg = 0;
-	unsigned long flags;
-
-	spin_lock_irqsave(&zf_port_lock, flags);
-
-	zf_writeb(PULSE_LEN, 0xff);
-
-	zf_set_timer(ZF_CTIMEOUT, WD1);
-
-	/* user land ping */
-	next_heartbeat = jiffies + ZF_USER_TIMEO;
-
-	/* start the timer for internal ping */
-	mod_timer(&zf_timer, jiffies + ZF_HW_TIMEO);
-
-	/* start watchdog timer */
-	ctrl_reg = zf_get_control();
-	ctrl_reg |= (ENABLE_WD1|zf_action);
-	zf_set_control(ctrl_reg);
-	spin_unlock_irqrestore(&zf_port_lock, flags);
-
-	pr_info("Watchdog timer is now enabled\n");
-}
-
-
-static void zf_ping(struct timer_list *unused)
-{
-	unsigned int ctrl_reg = 0;
-	unsigned long flags;
-
-	zf_writeb(COUNTER_2, 0xff);
-
-	if (time_before(jiffies, next_heartbeat)) {
-		dprintk("time_before: %ld\n", next_heartbeat - jiffies);
-		/*
-		 * reset event is activated by transition from 0 to 1 on
-		 * RESET_WD1 bit and we assume that it is already zero...
-		 */
-
-		spin_lock_irqsave(&zf_port_lock, flags);
-		ctrl_reg = zf_get_control();
-		ctrl_reg |= RESET_WD1;
-		zf_set_control(ctrl_reg);
-
-		/* ...and nothing changes until here */
-		ctrl_reg &= ~(RESET_WD1);
-		zf_set_control(ctrl_reg);
-		spin_unlock_irqrestore(&zf_port_lock, flags);
-
-		mod_timer(&zf_timer, jiffies + ZF_HW_TIMEO);
-	} else
-		pr_crit("I will reset your machine\n");
-}
-
-static ssize_t zf_write(struct file *file, const char __user *buf, size_t count,
-								loff_t *ppos)
-{
-	/* See if we got the magic character */
-	if (count) {
-		/*
-		 * no need to check for close confirmation
-		 * no way to disable watchdog ;)
-		 */
-		if (!nowayout) {
-			size_t ofs;
-			/*
-			 * note: just in case someone wrote the magic character
-			 * five months ago...
-			 */
-			zf_expect_close = 0;
-
-			/* now scan */
-			for (ofs = 0; ofs != count; ofs++) {
-				char c;
-				if (get_user(c, buf + ofs))
-					return -EFAULT;
-				if (c == 'V') {
-					zf_expect_close = 42;
-					dprintk("zf_expect_close = 42\n");
-				}
-			}
-		}
-
-		/*
-		 * Well, anyhow someone wrote to us,
-		 * we should return that favour
-		 */
-		next_heartbeat = jiffies + ZF_USER_TIMEO;
-		dprintk("user ping at %ld\n", jiffies);
-	}
-	return count;
-}
-
-static long zf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
-	void __user *argp = (void __user *)arg;
-	int __user *p = argp;
-	switch (cmd) {
-	case WDIOC_GETSUPPORT:
-		if (copy_to_user(argp, &zf_info, sizeof(zf_info)))
-			return -EFAULT;
-		break;
-	case WDIOC_GETSTATUS:
-	case WDIOC_GETBOOTSTATUS:
-		return put_user(0, p);
-	case WDIOC_KEEPALIVE:
-		zf_ping(NULL);
-		break;
-	default:
-		return -ENOTTY;
-	}
-	return 0;
-}
-
-static int zf_open(struct inode *inode, struct file *file)
-{
-	if (test_and_set_bit(0, &zf_is_open))
-		return -EBUSY;
-	if (nowayout)
-		__module_get(THIS_MODULE);
-	zf_timer_on();
-	return stream_open(inode, file);
-}
-
-static int zf_close(struct inode *inode, struct file *file)
-{
-	if (zf_expect_close == 42)
-		zf_timer_off();
-	else {
-		timer_delete(&zf_timer);
-		pr_err("device file closed unexpectedly. Will not stop the WDT!\n");
-	}
-	clear_bit(0, &zf_is_open);
-	zf_expect_close = 0;
-	return 0;
-}
-
-/*
- * Notifier for system down
- */
-
-static int zf_notify_sys(struct notifier_block *this, unsigned long code,
-								void *unused)
-{
-	if (code == SYS_DOWN || code == SYS_HALT)
-		zf_timer_off();
-	return NOTIFY_DONE;
-}
-
-static const struct file_operations zf_fops = {
-	.owner		= THIS_MODULE,
-	.write		= zf_write,
-	.unlocked_ioctl = zf_ioctl,
-	.compat_ioctl	= compat_ptr_ioctl,
-	.open		= zf_open,
-	.release	= zf_close,
-};
-
-static struct miscdevice zf_miscdev = {
-	.minor = WATCHDOG_MINOR,
-	.name = "watchdog",
-	.fops = &zf_fops,
-};
-
-
-/*
- * The device needs to learn about soft shutdowns in order to
- * turn the timebomb registers off.
- */
-static struct notifier_block zf_notifier = {
-	.notifier_call = zf_notify_sys,
-};
-
-static void __init zf_show_action(int act)
-{
-	static const char * const str[] = { "RESET", "SMI", "NMI", "SCI" };
-
-	pr_info("Watchdog using action = %s\n", str[act]);
-}
-
-static int __init zf_init(void)
-{
-	int ret;
-
-	pr_info("MachZ ZF-Logic Watchdog driver initializing\n");
-
-	ret = zf_get_ZFL_version();
-	if (!ret || ret == 0xffff) {
-		pr_warn("no ZF-Logic found\n");
-		return -ENODEV;
-	}
-
-	if (action <= 3 && action >= 0)
-		zf_action = zf_action >> action;
-	else
-		action = 0;
-
-	zf_show_action(action);
-
-	if (!request_region(ZF_IOBASE, 3, "MachZ ZFL WDT")) {
-		pr_err("cannot reserve I/O ports at %d\n", ZF_IOBASE);
-		ret = -EBUSY;
-		goto no_region;
-	}
-
-	ret = register_reboot_notifier(&zf_notifier);
-	if (ret) {
-		pr_err("can't register reboot notifier (err=%d)\n", ret);
-		goto no_reboot;
-	}
-
-	ret = misc_register(&zf_miscdev);
-	if (ret) {
-		pr_err("can't misc_register on minor=%d\n", WATCHDOG_MINOR);
-		goto no_misc;
-	}
-
-	zf_set_status(0);
-	zf_set_control(0);
-
-	return 0;
-
-no_misc:
-	unregister_reboot_notifier(&zf_notifier);
-no_reboot:
-	release_region(ZF_IOBASE, 3);
-no_region:
-	return ret;
-}
-
-
-static void __exit zf_exit(void)
-{
-	zf_timer_off();
-
-	misc_deregister(&zf_miscdev);
-	unregister_reboot_notifier(&zf_notifier);
-	release_region(ZF_IOBASE, 3);
-}
-
-module_init(zf_init);
-module_exit(zf_exit);
diff --git a/drivers/watchdog/menz69_wdt.c b/drivers/watchdog/menz69_wdt.c
index 6e5e4e5c0b566..3fe23451135d7 100644
--- a/drivers/watchdog/menz69_wdt.c
+++ b/drivers/watchdog/menz69_wdt.c
@@ -163,5 +163,4 @@ module_mcb_driver(men_z069_driver);
 MODULE_AUTHOR("Johannes Thumshirn <jth@kernel.org>");
 MODULE_DESCRIPTION("Watchdog driver for the MEN z069 IP-Core");
 MODULE_LICENSE("GPL v2");
-MODULE_ALIAS("mcb:16z069");
 MODULE_IMPORT_NS("MCB");
diff --git a/drivers/watchdog/ni903x_wdt.c b/drivers/watchdog/ni903x_wdt.c
index 8b1b9baa914e5..c72a9ee9cb8e1 100644
--- a/drivers/watchdog/ni903x_wdt.c
+++ b/drivers/watchdog/ni903x_wdt.c
@@ -183,9 +183,14 @@ static int ni903x_acpi_probe(struct platform_device *pdev)
 	struct device *dev = &pdev->dev;
 	struct watchdog_device *wdd;
 	struct ni903x_wdt *wdt;
+	acpi_handle handle;
 	acpi_status status;
 	int ret;
 
+	handle = ACPI_HANDLE(dev);
+	if (!handle)
+		return -ENODEV;
+
 	wdt = devm_kzalloc(dev, sizeof(*wdt), GFP_KERNEL);
 	if (!wdt)
 		return -ENOMEM;
@@ -193,7 +198,7 @@ static int ni903x_acpi_probe(struct platform_device *pdev)
 	platform_set_drvdata(pdev, wdt);
 	wdt->dev = dev;
 
-	status = acpi_walk_resources(ACPI_HANDLE(dev), METHOD_NAME__CRS,
+	status = acpi_walk_resources(handle, METHOD_NAME__CRS,
 				     ni903x_resources, wdt);
 	if (ACPI_FAILURE(status) || wdt->io_base == 0) {
 		dev_err(dev, "failed to get resources\n");
diff --git a/drivers/watchdog/qcom-wdt.c b/drivers/watchdog/qcom-wdt.c
index dfaac5995c84c..49bd04841f0c8 100644
--- a/drivers/watchdog/qcom-wdt.c
+++ b/drivers/watchdog/qcom-wdt.c
@@ -9,6 +9,7 @@
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_address.h>
 #include <linux/platform_device.h>
 #include <linux/watchdog.h>
 
@@ -42,6 +43,7 @@ struct qcom_wdt_match_data {
 	const u32 *offset;
 	bool pretimeout;
 	u32 max_tick_count;
+	u32 wdt_reason_val;
 };
 
 struct qcom_wdt {
@@ -185,6 +187,7 @@ static const struct qcom_wdt_match_data match_data_ipq5424 = {
 	.offset = reg_offset_data_kpss,
 	.pretimeout = true,
 	.max_tick_count = 0xFFFFFU,
+	.wdt_reason_val = 5,
 };
 
 static const struct qcom_wdt_match_data match_data_kpss = {
@@ -193,6 +196,40 @@ static const struct qcom_wdt_match_data match_data_kpss = {
 	.max_tick_count = 0xFFFFFU,
 };
 
+static int qcom_wdt_get_bootstatus(struct device *dev, struct qcom_wdt *wdt,
+				   u32 val)
+{
+	struct device_node *imem;
+	struct resource res;
+	void __iomem *addr;
+	int ret;
+
+	imem = of_parse_phandle(dev->of_node, "sram", 0);
+	if (!imem) {
+		/* Read the EXPIRED_STATUS bit as a fallback */
+		if (readl(wdt_addr(wdt, WDT_STS)) & 1)
+			wdt->wdd.bootstatus = WDIOF_CARDRESET;
+
+		return 0;
+	}
+
+	ret = of_address_to_resource(imem, 0, &res);
+	of_node_put(imem);
+	if (ret)
+		return ret;
+
+	addr = ioremap(res.start, resource_size(&res));
+	if (!addr)
+		return -ENOMEM;
+
+	if (readl(addr) == val)
+		wdt->wdd.bootstatus = WDIOF_CARDRESET;
+
+	iounmap(addr);
+
+	return 0;
+}
+
 static int qcom_wdt_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
@@ -273,8 +310,9 @@ static int qcom_wdt_probe(struct platform_device *pdev)
 	wdt->wdd.parent = dev;
 	wdt->layout = data->offset;
 
-	if (readl(wdt_addr(wdt, WDT_STS)) & 1)
-		wdt->wdd.bootstatus = WDIOF_CARDRESET;
+	ret = qcom_wdt_get_bootstatus(dev, wdt, data->wdt_reason_val);
+	if (ret)
+		dev_err(dev, "failed to get the bootstatus, %d\n", ret);
 
 	/*
 	 * If 'timeout-sec' unspecified in devicetree, assume a 30 second
diff --git a/drivers/watchdog/realtek_otto_wdt.c b/drivers/watchdog/realtek_otto_wdt.c
index 2c30ddd574c59..01b3ef89bacfc 100644
--- a/drivers/watchdog/realtek_otto_wdt.c
+++ b/drivers/watchdog/realtek_otto_wdt.c
@@ -114,12 +114,6 @@ static int otto_wdt_tick_ms(struct otto_wdt_ctrl *ctrl, int prescale)
  * the value stored in those fields. This means each phase will run for at least
  * one tick, so small values need to be clamped to correctly reflect the timeout.
  */
-static inline unsigned int div_round_ticks(unsigned int val, unsigned int tick_duration,
-		unsigned int min_ticks)
-{
-	return max(min_ticks, DIV_ROUND_UP(val, tick_duration));
-}
-
 static int otto_wdt_determine_timeouts(struct watchdog_device *wdev, unsigned int timeout,
 		unsigned int pretimeout)
 {
@@ -140,9 +134,9 @@ static int otto_wdt_determine_timeouts(struct watchdog_device *wdev, unsigned in
 			return -EINVAL;
 
 		tick_ms = otto_wdt_tick_ms(ctrl, prescale);
-		total_ticks = div_round_ticks(timeout_ms, tick_ms, 2);
-		phase1_ticks = div_round_ticks(timeout_ms - pretimeout_ms, tick_ms, 1);
-		phase2_ticks = total_ticks - phase1_ticks;
+		total_ticks = max(2, DIV_ROUND_UP(timeout_ms, tick_ms));
+		phase2_ticks = max(1, pretimeout_ms / tick_ms);
+		phase1_ticks = total_ticks - phase2_ticks;
 
 		prescale_next++;
 	} while (phase1_ticks > OTTO_WDT_PHASE_TICKS_MAX
@@ -302,15 +296,15 @@ static int otto_wdt_probe(struct platform_device *pdev)
 	if (IS_ERR(ctrl->base))
 		return PTR_ERR(ctrl->base);
 
+	ret = otto_wdt_probe_clk(ctrl);
+	if (ret)
+		return ret;
+
 	/* Clear any old interrupts and reset initial state */
 	iowrite32(OTTO_WDT_INTR_PHASE_1 | OTTO_WDT_INTR_PHASE_2,
 			ctrl->base + OTTO_WDT_REG_INTR);
 	iowrite32(OTTO_WDT_CTRL_DEFAULT, ctrl->base + OTTO_WDT_REG_CTRL);
 
-	ret = otto_wdt_probe_clk(ctrl);
-	if (ret)
-		return ret;
-
 	ctrl->irq_phase1 = platform_get_irq_byname(pdev, "phase1");
 	if (ctrl->irq_phase1 < 0)
 		return ctrl->irq_phase1;
diff --git a/drivers/watchdog/rti_wdt.c b/drivers/watchdog/rti_wdt.c
index be7d7db475911..c3c7715140ea8 100644
--- a/drivers/watchdog/rti_wdt.c
+++ b/drivers/watchdog/rti_wdt.c
@@ -35,7 +35,8 @@
 #define RTIWWDRXCTRL	0xa4
 #define RTIWWDSIZECTRL	0xa8
 
-#define RTIWWDRX_NMI	0xa
+#define RTIWWDRXN_RST	0x5
+#define RTIWWDRXN_NMI	0xa
 
 #define RTIWWDSIZE_50P		0x50
 #define RTIWWDSIZE_25P		0x500
@@ -63,22 +64,29 @@
 
 static int heartbeat;
 
+struct rti_wdt_data {
+	bool nmi;
+};
+
 /*
  * struct to hold data for each WDT device
  * @base - base io address of WD device
  * @freq - source clock frequency of WDT
  * @wdd  - hold watchdog device as is in WDT core
+ * @nmi - Set if this WDT instance supports generating NMI
  */
 struct rti_wdt_device {
 	void __iomem		*base;
 	unsigned long		freq;
 	struct watchdog_device	wdd;
+	bool			nmi;
 };
 
 static int rti_wdt_start(struct watchdog_device *wdd)
 {
 	u32 timer_margin;
 	struct rti_wdt_device *wdt = watchdog_get_drvdata(wdd);
+	u8 reaction;
 	int ret;
 
 	ret = pm_runtime_resume_and_get(wdd->parent);
@@ -101,8 +109,13 @@ static int rti_wdt_start(struct watchdog_device *wdd)
 	 */
 	wdd->min_hw_heartbeat_ms = 520 * wdd->timeout + MAX_HW_ERROR;
 
-	/* Generate NMI when wdt expires */
-	writel_relaxed(RTIWWDRX_NMI, wdt->base + RTIWWDRXCTRL);
+	/* When WDT expires, generate NMI or reset if NMI not supported */
+	if (wdt->nmi)
+		reaction = RTIWWDRXN_NMI;
+	else
+		reaction = RTIWWDRXN_RST;
+
+	writel_relaxed(reaction, wdt->base + RTIWWDRXCTRL);
 
 	/* Open window size 50%; this is the largest window size available */
 	writel_relaxed(RTIWWDSIZE_50P, wdt->base + RTIWWDSIZECTRL);
@@ -210,6 +223,7 @@ static int rti_wdt_probe(struct platform_device *pdev)
 {
 	int ret = 0;
 	struct device *dev = &pdev->dev;
+	const struct rti_wdt_data *data;
 	struct watchdog_device *wdd;
 	struct rti_wdt_device *wdt;
 	struct clk *clk;
@@ -254,6 +268,14 @@ static int rti_wdt_probe(struct platform_device *pdev)
 	wdd->timeout = DEFAULT_HEARTBEAT;
 	wdd->parent = dev;
 
+	data = device_get_match_data(dev);
+	if (!data) {
+		ret = -ENODEV;
+		goto err_iomap;
+	}
+
+	wdt->nmi = data->nmi;
+
 	watchdog_set_drvdata(wdd, wdt);
 	watchdog_set_nowayout(wdd, 1);
 	watchdog_set_restart_priority(wdd, 128);
@@ -361,8 +383,17 @@ static void rti_wdt_remove(struct platform_device *pdev)
 	pm_runtime_disable(&pdev->dev);
 }
 
+static const struct rti_wdt_data rti_wdt_j7_data = {
+	.nmi = true,
+};
+
+static const struct rti_wdt_data rti_wdt_am62l_data = {
+	.nmi = false,
+};
+
 static const struct of_device_id rti_wdt_of_match[] = {
-	{ .compatible = "ti,j7-rti-wdt", },
+	{ .compatible = "ti,j7-rti-wdt", .data = &rti_wdt_j7_data },
+	{ .compatible = "ti,am62l-rti-wdt", .data = &rti_wdt_am62l_data },
 	{},
 };
 MODULE_DEVICE_TABLE(of, rti_wdt_of_match);
diff --git a/drivers/watchdog/rzn1_wdt.c b/drivers/watchdog/rzn1_wdt.c
index 96fd04fbc2a26..4fdc5363ba98f 100644
--- a/drivers/watchdog/rzn1_wdt.c
+++ b/drivers/watchdog/rzn1_wdt.c
@@ -79,14 +79,6 @@ static int rzn1_wdt_start(struct watchdog_device *w)
 	return 0;
 }
 
-static irqreturn_t rzn1_wdt_irq(int irq, void *_wdt)
-{
-	pr_crit("RZN1 Watchdog. Initiating system reboot\n");
-	emergency_restart();
-
-	return IRQ_HANDLED;
-}
-
 static struct watchdog_info rzn1_wdt_info = {
 	.identity = "RZ/N1 Watchdog",
 	.options = WDIOF_MAGICCLOSE | WDIOF_SETTIMEOUT | WDIOF_KEEPALIVEPING,
@@ -102,11 +94,9 @@ static int rzn1_wdt_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
 	struct rzn1_watchdog *wdt;
-	struct device_node *np = dev->of_node;
-	struct clk *clk;
 	unsigned long clk_rate;
+	struct clk *clk;
 	int ret;
-	int irq;
 
 	wdt = devm_kzalloc(dev, sizeof(*wdt), GFP_KERNEL);
 	if (!wdt)
@@ -116,28 +106,13 @@ static int rzn1_wdt_probe(struct platform_device *pdev)
 	if (IS_ERR(wdt->base))
 		return PTR_ERR(wdt->base);
 
-	irq = platform_get_irq(pdev, 0);
-	if (irq < 0)
-		return irq;
-
-	ret = devm_request_irq(dev, irq, rzn1_wdt_irq, 0,
-			       np->name, wdt);
-	if (ret) {
-		dev_err(dev, "failed to request irq %d\n", irq);
-		return ret;
-	}
-
 	clk = devm_clk_get_enabled(dev, NULL);
-	if (IS_ERR(clk)) {
-		dev_err(dev, "failed to get the clock\n");
-		return PTR_ERR(clk);
-	}
+	if (IS_ERR(clk))
+		return dev_err_probe(dev, PTR_ERR(clk), "failed to get the clock\n");
 
 	clk_rate = clk_get_rate(clk);
-	if (!clk_rate) {
-		dev_err(dev, "failed to get the clock rate\n");
-		return -EINVAL;
-	}
+	if (!clk_rate)
+		return dev_err_probe(dev, -EINVAL, "failed to get the clock rate\n");
 
 	wdt->clk_rate_khz = clk_rate / 1000;
 	wdt->wdtdev.info = &rzn1_wdt_info;
diff --git a/drivers/watchdog/s32g_wdt.c b/drivers/watchdog/s32g_wdt.c
index ad55063060af6..6422a694fc65f 100644
--- a/drivers/watchdog/s32g_wdt.c
+++ b/drivers/watchdog/s32g_wdt.c
@@ -56,8 +56,7 @@ MODULE_PARM_DESC(early_enable,
 
 static const struct watchdog_info s32g_wdt_info = {
 	.identity = "s32g watchdog",
-	.options = WDIOF_KEEPALIVEPING | WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE |
-	WDIOC_GETTIMEOUT | WDIOC_GETTIMELEFT,
+	.options = WDIOF_KEEPALIVEPING | WDIOF_SETTIMEOUT | WDIOF_MAGICCLOSE,
 };
 
 static struct s32g_wdt_device *wdd_to_s32g_wdt(struct watchdog_device *wdd)
diff --git a/drivers/watchdog/sama5d4_wdt.c b/drivers/watchdog/sama5d4_wdt.c
index 13e72918338a2..704b786cc2ec6 100644
--- a/drivers/watchdog/sama5d4_wdt.c
+++ b/drivers/watchdog/sama5d4_wdt.c
@@ -30,6 +30,7 @@ struct sama5d4_wdt {
 	void __iomem		*reg_base;
 	u32			mr;
 	u32			ir;
+	u32			wddis_mask;
 	unsigned long		last_ping;
 	bool			need_irq;
 	bool			sam9x60_support;
@@ -48,7 +49,10 @@ MODULE_PARM_DESC(nowayout,
 	"Watchdog cannot be stopped once started (default="
 	__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
 
-#define wdt_enabled (!(wdt->mr & AT91_WDT_WDDIS))
+static inline bool wdt_enabled(struct sama5d4_wdt *wdt)
+{
+	return !(wdt->mr & wdt->wddis_mask);
+}
 
 #define wdt_read(wdt, field) \
 	readl_relaxed((wdt)->reg_base + (field))
@@ -81,12 +85,9 @@ static int sama5d4_wdt_start(struct watchdog_device *wdd)
 {
 	struct sama5d4_wdt *wdt = watchdog_get_drvdata(wdd);
 
-	if (wdt->sam9x60_support) {
+	if (wdt->sam9x60_support)
 		writel_relaxed(wdt->ir, wdt->reg_base + AT91_SAM9X60_IER);
-		wdt->mr &= ~AT91_SAM9X60_WDDIS;
-	} else {
-		wdt->mr &= ~AT91_WDT_WDDIS;
-	}
+	wdt->mr &= ~wdt->wddis_mask;
 	wdt_write(wdt, AT91_WDT_MR, wdt->mr);
 
 	return 0;
@@ -96,12 +97,9 @@ static int sama5d4_wdt_stop(struct watchdog_device *wdd)
 {
 	struct sama5d4_wdt *wdt = watchdog_get_drvdata(wdd);
 
-	if (wdt->sam9x60_support) {
+	if (wdt->sam9x60_support)
 		writel_relaxed(wdt->ir, wdt->reg_base + AT91_SAM9X60_IDR);
-		wdt->mr |= AT91_SAM9X60_WDDIS;
-	} else {
-		wdt->mr |= AT91_WDT_WDDIS;
-	}
+	wdt->mr |= wdt->wddis_mask;
 	wdt_write(wdt, AT91_WDT_MR, wdt->mr);
 
 	return 0;
@@ -117,7 +115,7 @@ static int sama5d4_wdt_ping(struct watchdog_device *wdd)
 }
 
 static int sama5d4_wdt_set_timeout(struct watchdog_device *wdd,
-				 unsigned int timeout)
+				    unsigned int timeout)
 {
 	struct sama5d4_wdt *wdt = watchdog_get_drvdata(wdd);
 	u32 value = WDT_SEC2TICKS(timeout);
@@ -140,8 +138,8 @@ static int sama5d4_wdt_set_timeout(struct watchdog_device *wdd,
 	 * If the watchdog is enabled, then the timeout can be updated. Else,
 	 * wait that the user enables it.
 	 */
-	if (wdt_enabled)
-		wdt_write(wdt, AT91_WDT_MR, wdt->mr & ~AT91_WDT_WDDIS);
+	if (wdt_enabled(wdt))
+		wdt_write(wdt, AT91_WDT_MR, wdt->mr & ~wdt->wddis_mask);
 
 	wdd->timeout = timeout;
 
@@ -184,10 +182,7 @@ static int of_sama5d4_wdt_init(struct device_node *np, struct sama5d4_wdt *wdt)
 {
 	const char *tmp;
 
-	if (wdt->sam9x60_support)
-		wdt->mr = AT91_SAM9X60_WDDIS;
-	else
-		wdt->mr = AT91_WDT_WDDIS;
+	wdt->mr = wdt->wddis_mask;
 
 	if (!of_property_read_string(np, "atmel,watchdog-type", &tmp) &&
 	    !strcmp(tmp, "software"))
@@ -213,15 +208,11 @@ static int sama5d4_wdt_init(struct sama5d4_wdt *wdt)
 	 * If the watchdog is already running, we can safely update it.
 	 * Else, we have to disable it properly.
 	 */
-	if (!wdt_enabled) {
+	if (!wdt_enabled(wdt)) {
 		reg = wdt_read(wdt, AT91_WDT_MR);
-		if (wdt->sam9x60_support && (!(reg & AT91_SAM9X60_WDDIS)))
-			wdt_write_nosleep(wdt, AT91_WDT_MR,
-					  reg | AT91_SAM9X60_WDDIS);
-		else if (!wdt->sam9x60_support &&
-			 (!(reg & AT91_WDT_WDDIS)))
+		if (!(reg & wdt->wddis_mask))
 			wdt_write_nosleep(wdt, AT91_WDT_MR,
-					  reg | AT91_WDT_WDDIS);
+					  reg | wdt->wddis_mask);
 	}
 
 	if (wdt->sam9x60_support) {
@@ -273,6 +264,9 @@ static int sama5d4_wdt_probe(struct platform_device *pdev)
 	    of_device_is_compatible(dev->of_node, "microchip,sama7g5-wdt"))
 		wdt->sam9x60_support = true;
 
+	wdt->wddis_mask = wdt->sam9x60_support ? AT91_SAM9X60_WDDIS
+						: AT91_WDT_WDDIS;
+
 	watchdog_set_drvdata(wdd, wdt);
 
 	regs = devm_platform_ioremap_resource(pdev, 0);
@@ -306,8 +300,8 @@ static int sama5d4_wdt_probe(struct platform_device *pdev)
 	watchdog_init_timeout(wdd, wdt_timeout, dev);
 
 	reg = wdt_read(wdt, AT91_WDT_MR);
-	if (!(reg & AT91_WDT_WDDIS)) {
-		wdt->mr &= ~AT91_WDT_WDDIS;
+	if (!(reg & wdt->wddis_mask)) {
+		wdt->mr &= ~wdt->wddis_mask;
 		set_bit(WDOG_HW_RUNNING, &wdd->status);
 	}
 
diff --git a/drivers/watchdog/sc520_wdt.c b/drivers/watchdog/sc520_wdt.c
deleted file mode 100644
index 005f62e4a4fb3..0000000000000
--- a/drivers/watchdog/sc520_wdt.c
+++ /dev/null
@@ -1,430 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- *	AMD Elan SC520 processor Watchdog Timer driver
- *
- *	Based on acquirewdt.c by Alan Cox,
- *	     and sbc60xxwdt.c by Jakob Oestergaard <jakob@unthought.net>
- *
- *	The authors do NOT admit liability nor provide warranty for
- *	any of this software. This material is provided "AS-IS" in
- *	the hope that it may be useful for others.
- *
- *	(c) Copyright 2001    Scott Jennings <linuxdrivers@oro.net>
- *           9/27 - 2001      [Initial release]
- *
- *	Additional fixes Alan Cox
- *	-	Fixed formatting
- *	-	Removed debug printks
- *	-	Fixed SMP built kernel deadlock
- *	-	Switched to private locks not lock_kernel
- *	-	Used ioremap/writew/readw
- *	-	Added NOWAYOUT support
- *	4/12 - 2002 Changes by Rob Radez <rob@osinvestor.com>
- *	-	Change comments
- *	-	Eliminate fop_llseek
- *	-	Change CONFIG_WATCHDOG_NOWAYOUT semantics
- *	-	Add KERN_* tags to printks
- *	-	fix possible wdt_is_open race
- *	-	Report proper capabilities in watchdog_info
- *	-	Add WDIOC_{GETSTATUS, GETBOOTSTATUS, SETTIMEOUT,
- *		GETTIMEOUT, SETOPTIONS} ioctls
- *	09/8 - 2003 Changes by Wim Van Sebroeck <wim@iguana.be>
- *	-	cleanup of trailing spaces
- *	-	added extra printk's for startup problems
- *	-	use module_param
- *	-	made timeout (the emulated heartbeat) a module_param
- *	-	made the keepalive ping an internal subroutine
- *	3/27 - 2004 Changes by Sean Young <sean@mess.org>
- *	-	set MMCR_BASE to 0xfffef000
- *	-	CBAR does not need to be read
- *	-	removed debugging printks
- *
- *  This WDT driver is different from most other Linux WDT
- *  drivers in that the driver will ping the watchdog by itself,
- *  because this particular WDT has a very short timeout (1.6
- *  seconds) and it would be insane to count on any userspace
- *  daemon always getting scheduled within that time frame.
- *
- *  This driver uses memory mapped IO, and spinlock.
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/types.h>
-#include <linux/timer.h>
-#include <linux/miscdevice.h>
-#include <linux/watchdog.h>
-#include <linux/fs.h>
-#include <linux/ioport.h>
-#include <linux/notifier.h>
-#include <linux/reboot.h>
-#include <linux/init.h>
-#include <linux/jiffies.h>
-#include <linux/io.h>
-#include <linux/uaccess.h>
-
-
-/*
- * The AMD Elan SC520 timeout value is 492us times a power of 2 (0-7)
- *
- *   0: 492us    2: 1.01s    4: 4.03s   6: 16.22s
- *   1: 503ms    3: 2.01s    5: 8.05s   7: 32.21s
- *
- * We will program the SC520 watchdog for a timeout of 2.01s.
- * If we reset the watchdog every ~250ms we should be safe.
- */
-
-#define WDT_INTERVAL (HZ/4+1)
-
-/*
- * We must not require too good response from the userspace daemon.
- * Here we require the userspace daemon to send us a heartbeat
- * char to /dev/watchdog every 30 seconds.
- */
-
-#define WATCHDOG_TIMEOUT 30		/* 30 sec default timeout */
-/* in seconds, will be multiplied by HZ to get seconds to wait for a ping */
-static int timeout = WATCHDOG_TIMEOUT;
-module_param(timeout, int, 0);
-MODULE_PARM_DESC(timeout,
-	"Watchdog timeout in seconds. (1 <= timeout <= 3600, default="
-				__MODULE_STRING(WATCHDOG_TIMEOUT) ")");
-
-static bool nowayout = WATCHDOG_NOWAYOUT;
-module_param(nowayout, bool, 0);
-MODULE_PARM_DESC(nowayout,
-		"Watchdog cannot be stopped once started (default="
-				__MODULE_STRING(WATCHDOG_NOWAYOUT) ")");
-
-/*
- * AMD Elan SC520 - Watchdog Timer Registers
- */
-#define MMCR_BASE	0xfffef000	/* The default base address */
-#define OFFS_WDTMRCTL	0xCB0	/* Watchdog Timer Control Register */
-
-/* WDT Control Register bit definitions */
-#define WDT_EXP_SEL_01	0x0001	/* [01] Time-out = 496 us (with 33 Mhz clk). */
-#define WDT_EXP_SEL_02	0x0002	/* [02] Time-out = 508 ms (with 33 Mhz clk). */
-#define WDT_EXP_SEL_03	0x0004	/* [03] Time-out = 1.02 s (with 33 Mhz clk). */
-#define WDT_EXP_SEL_04	0x0008	/* [04] Time-out = 2.03 s (with 33 Mhz clk). */
-#define WDT_EXP_SEL_05	0x0010	/* [05] Time-out = 4.07 s (with 33 Mhz clk). */
-#define WDT_EXP_SEL_06	0x0020	/* [06] Time-out = 8.13 s (with 33 Mhz clk). */
-#define WDT_EXP_SEL_07	0x0040	/* [07] Time-out = 16.27s (with 33 Mhz clk). */
-#define WDT_EXP_SEL_08	0x0080	/* [08] Time-out = 32.54s (with 33 Mhz clk). */
-#define WDT_IRQ_FLG	0x1000	/* [12] Interrupt Request Flag */
-#define WDT_WRST_ENB	0x4000	/* [14] Watchdog Timer Reset Enable */
-#define WDT_ENB		0x8000	/* [15] Watchdog Timer Enable */
-
-static __u16 __iomem *wdtmrctl;
-
-static void wdt_timer_ping(struct timer_list *);
-static DEFINE_TIMER(timer, wdt_timer_ping);
-static unsigned long next_heartbeat;
-static unsigned long wdt_is_open;
-static char wdt_expect_close;
-static DEFINE_SPINLOCK(wdt_spinlock);
-
-/*
- *	Whack the dog
- */
-
-static void wdt_timer_ping(struct timer_list *unused)
-{
-	/* If we got a heartbeat pulse within the WDT_US_INTERVAL
-	 * we agree to ping the WDT
-	 */
-	if (time_before(jiffies, next_heartbeat)) {
-		/* Ping the WDT */
-		spin_lock(&wdt_spinlock);
-		writew(0xAAAA, wdtmrctl);
-		writew(0x5555, wdtmrctl);
-		spin_unlock(&wdt_spinlock);
-
-		/* Re-set the timer interval */
-		mod_timer(&timer, jiffies + WDT_INTERVAL);
-	} else
-		pr_warn("Heartbeat lost! Will not ping the watchdog\n");
-}
-
-/*
- *	Utility routines
- */
-
-static void wdt_config(int writeval)
-{
-	unsigned long flags;
-
-	/* buy some time (ping) */
-	spin_lock_irqsave(&wdt_spinlock, flags);
-	readw(wdtmrctl);	/* ensure write synchronization */
-	writew(0xAAAA, wdtmrctl);
-	writew(0x5555, wdtmrctl);
-	/* unlock WDT = make WDT configuration register writable one time */
-	writew(0x3333, wdtmrctl);
-	writew(0xCCCC, wdtmrctl);
-	/* write WDT configuration register */
-	writew(writeval, wdtmrctl);
-	spin_unlock_irqrestore(&wdt_spinlock, flags);
-}
-
-static int wdt_startup(void)
-{
-	next_heartbeat = jiffies + (timeout * HZ);
-
-	/* Start the timer */
-	mod_timer(&timer, jiffies + WDT_INTERVAL);
-
-	/* Start the watchdog */
-	wdt_config(WDT_ENB | WDT_WRST_ENB | WDT_EXP_SEL_04);
-
-	pr_info("Watchdog timer is now enabled\n");
-	return 0;
-}
-
-static int wdt_turnoff(void)
-{
-	/* Stop the timer */
-	timer_delete_sync(&timer);
-
-	/* Stop the watchdog */
-	wdt_config(0);
-
-	pr_info("Watchdog timer is now disabled...\n");
-	return 0;
-}
-
-static int wdt_keepalive(void)
-{
-	/* user land ping */
-	next_heartbeat = jiffies + (timeout * HZ);
-	return 0;
-}
-
-static int wdt_set_heartbeat(int t)
-{
-	if ((t < 1) || (t > 3600))	/* arbitrary upper limit */
-		return -EINVAL;
-
-	timeout = t;
-	return 0;
-}
-
-/*
- *	/dev/watchdog handling
- */
-
-static ssize_t fop_write(struct file *file, const char __user *buf,
-						size_t count, loff_t *ppos)
-{
-	/* See if we got the magic character 'V' and reload the timer */
-	if (count) {
-		if (!nowayout) {
-			size_t ofs;
-
-			/* note: just in case someone wrote the magic character
-			 * five months ago... */
-			wdt_expect_close = 0;
-
-			/* now scan */
-			for (ofs = 0; ofs != count; ofs++) {
-				char c;
-				if (get_user(c, buf + ofs))
-					return -EFAULT;
-				if (c == 'V')
-					wdt_expect_close = 42;
-			}
-		}
-
-		/* Well, anyhow someone wrote to us, we should
-		   return that favour */
-		wdt_keepalive();
-	}
-	return count;
-}
-
-static int fop_open(struct inode *inode, struct file *file)
-{
-	/* Just in case we're already talking to someone... */
-	if (test_and_set_bit(0, &wdt_is_open))
-		return -EBUSY;
-	if (nowayout)
-		__module_get(THIS_MODULE);
-
-	/* Good, fire up the show */
-	wdt_startup();
-	return stream_open(inode, file);
-}
-
-static int fop_close(struct inode *inode, struct file *file)
-{
-	if (wdt_expect_close == 42)
-		wdt_turnoff();
-	else {
-		pr_crit("Unexpected close, not stopping watchdog!\n");
-		wdt_keepalive();
-	}
-	clear_bit(0, &wdt_is_open);
-	wdt_expect_close = 0;
-	return 0;
-}
-
-static long fop_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
-{
-	void __user *argp = (void __user *)arg;
-	int __user *p = argp;
-	static const struct watchdog_info ident = {
-		.options = WDIOF_KEEPALIVEPING | WDIOF_SETTIMEOUT
-							| WDIOF_MAGICCLOSE,
-		.firmware_version = 1,
-		.identity = "SC520",
-	};
-
-	switch (cmd) {
-	case WDIOC_GETSUPPORT:
-		return copy_to_user(argp, &ident, sizeof(ident)) ? -EFAULT : 0;
-	case WDIOC_GETSTATUS:
-	case WDIOC_GETBOOTSTATUS:
-		return put_user(0, p);
-	case WDIOC_SETOPTIONS:
-	{
-		int new_options, retval = -EINVAL;
-
-		if (get_user(new_options, p))
-			return -EFAULT;
-
-		if (new_options & WDIOS_DISABLECARD) {
-			wdt_turnoff();
-			retval = 0;
-		}
-
-		if (new_options & WDIOS_ENABLECARD) {
-			wdt_startup();
-			retval = 0;
-		}
-
-		return retval;
-	}
-	case WDIOC_KEEPALIVE:
-		wdt_keepalive();
-		return 0;
-	case WDIOC_SETTIMEOUT:
-	{
-		int new_timeout;
-
-		if (get_user(new_timeout, p))
-			return -EFAULT;
-
-		if (wdt_set_heartbeat(new_timeout))
-			return -EINVAL;
-
-		wdt_keepalive();
-	}
-		fallthrough;
-	case WDIOC_GETTIMEOUT:
-		return put_user(timeout, p);
-	default:
-		return -ENOTTY;
-	}
-}
-
-static const struct file_operations wdt_fops = {
-	.owner		= THIS_MODULE,
-	.write		= fop_write,
-	.open		= fop_open,
-	.release	= fop_close,
-	.unlocked_ioctl	= fop_ioctl,
-	.compat_ioctl	= compat_ptr_ioctl,
-};
-
-static struct miscdevice wdt_miscdev = {
-	.minor	= WATCHDOG_MINOR,
-	.name	= "watchdog",
-	.fops	= &wdt_fops,
-};
-
-/*
- *	Notifier for system down
- */
-
-static int wdt_notify_sys(struct notifier_block *this, unsigned long code,
-	void *unused)
-{
-	if (code == SYS_DOWN || code == SYS_HALT)
-		wdt_turnoff();
-	return NOTIFY_DONE;
-}
-
-/*
- *	The WDT needs to learn about soft shutdowns in order to
- *	turn the timebomb registers off.
- */
-
-static struct notifier_block wdt_notifier = {
-	.notifier_call = wdt_notify_sys,
-};
-
-static void __exit sc520_wdt_unload(void)
-{
-	if (!nowayout)
-		wdt_turnoff();
-
-	/* Deregister */
-	misc_deregister(&wdt_miscdev);
-	unregister_reboot_notifier(&wdt_notifier);
-	iounmap(wdtmrctl);
-}
-
-static int __init sc520_wdt_init(void)
-{
-	int rc = -EBUSY;
-
-	/* Check that the timeout value is within it's range ;
-	   if not reset to the default */
-	if (wdt_set_heartbeat(timeout)) {
-		wdt_set_heartbeat(WATCHDOG_TIMEOUT);
-		pr_info("timeout value must be 1 <= timeout <= 3600, using %d\n",
-			WATCHDOG_TIMEOUT);
-	}
-
-	wdtmrctl = ioremap(MMCR_BASE + OFFS_WDTMRCTL, 2);
-	if (!wdtmrctl) {
-		pr_err("Unable to remap memory\n");
-		rc = -ENOMEM;
-		goto err_out_region2;
-	}
-
-	rc = register_reboot_notifier(&wdt_notifier);
-	if (rc) {
-		pr_err("cannot register reboot notifier (err=%d)\n", rc);
-		goto err_out_ioremap;
-	}
-
-	rc = misc_register(&wdt_miscdev);
-	if (rc) {
-		pr_err("cannot register miscdev on minor=%d (err=%d)\n",
-		       WATCHDOG_MINOR, rc);
-		goto err_out_notifier;
-	}
-
-	pr_info("WDT driver for SC520 initialised. timeout=%d sec (nowayout=%d)\n",
-		timeout, nowayout);
-
-	return 0;
-
-err_out_notifier:
-	unregister_reboot_notifier(&wdt_notifier);
-err_out_ioremap:
-	iounmap(wdtmrctl);
-err_out_region2:
-	return rc;
-}
-
-module_init(sc520_wdt_init);
-module_exit(sc520_wdt_unload);
-
-MODULE_AUTHOR("Scott and Bill Jennings");
-MODULE_DESCRIPTION(
-	"Driver for watchdog timer in AMD \"Elan\" SC520 uProcessor");
-MODULE_LICENSE("GPL");
diff --git a/drivers/watchdog/smsc37b787_wdt.c b/drivers/watchdog/smsc37b787_wdt.c
index 3011e1af00f98..9f2c2ab4e2f15 100644
--- a/drivers/watchdog/smsc37b787_wdt.c
+++ b/drivers/watchdog/smsc37b787_wdt.c
@@ -36,7 +36,7 @@
  *  mknod /dev/watchdog c 10 130
  *
  * For an example userspace keep-alive daemon, see:
- *   Documentation/watchdog/wdt.rst
+ *   samples/watchdog/watchdog-simple.c
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
diff --git a/drivers/watchdog/sp5100_tco.c b/drivers/watchdog/sp5100_tco.c
index 2bd3dc25cb030..7e99c3b1f3676 100644
--- a/drivers/watchdog/sp5100_tco.c
+++ b/drivers/watchdog/sp5100_tco.c
@@ -92,7 +92,8 @@ static enum tco_reg_layout tco_reg_layout(struct pci_dev *dev)
 	    dev->device == PCI_DEVICE_ID_ATI_SBX00_SMBUS &&
 	    dev->revision < 0x40) {
 		return sp5100;
-	} else if (dev->vendor == PCI_VENDOR_ID_AMD &&
+	} else if ((dev->vendor == PCI_VENDOR_ID_AMD ||
+		    dev->vendor == PCI_VENDOR_ID_HYGON) &&
 	    sp5100_tco_pci->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS &&
 	    sp5100_tco_pci->revision >= AMD_ZEN_SMBUS_PCI_REV) {
 		return efch_mmio;
diff --git a/drivers/watchdog/sprd_wdt.c b/drivers/watchdog/sprd_wdt.c
index 4e689b6ff1418..aacf04616fefe 100644
--- a/drivers/watchdog/sprd_wdt.c
+++ b/drivers/watchdog/sprd_wdt.c
@@ -320,10 +320,9 @@ static int sprd_wdt_probe(struct platform_device *pdev)
 	watchdog_init_timeout(&wdt->wdd, 0, dev);
 
 	ret = devm_watchdog_register_device(dev, &wdt->wdd);
-	if (ret) {
-		sprd_wdt_disable(wdt);
+	if (ret)
 		return ret;
-	}
+
 	platform_set_drvdata(pdev, wdt);
 
 	return 0;
diff --git a/drivers/watchdog/ziirave_wdt.c b/drivers/watchdog/ziirave_wdt.c
index 5c6e3fa001d88..f3bb935c08c2e 100644
--- a/drivers/watchdog/ziirave_wdt.c
+++ b/drivers/watchdog/ziirave_wdt.c
@@ -718,7 +718,7 @@ static void ziirave_wdt_remove(struct i2c_client *client)
 }
 
 static const struct i2c_device_id ziirave_wdt_id[] = {
-	{ "rave-wdt" },
+	{ .name = "rave-wdt" },
 	{ }
 };
 MODULE_DEVICE_TABLE(i2c, ziirave_wdt_id);
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index bc9a41662efc4..6ea945508a89a 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -40,6 +40,7 @@
 #include <linux/ktime.h>
 
 #ifdef CONFIG_X86
+#include <asm/cpuid/api.h>
 #include <asm/desc.h>
 #include <asm/ptrace.h>
 #include <asm/idtentry.h>
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index a6abf1ccd54ce..35f879dc5dfb8 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -59,6 +59,7 @@
 #include <xen/swiotlb-xen.h>
 #include <xen/balloon.h>
 #ifdef CONFIG_X86
+#include <asm/cpuid/api.h>
 #include <asm/xen/cpuid.h>
 #endif
 #include <xen/mem-reservation.h>
diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c
index 82b0a34ded70d..c202e7c553a68 100644
--- a/drivers/xen/xenbus/xenbus_xs.c
+++ b/drivers/xen/xenbus/xenbus_xs.c
@@ -47,6 +47,9 @@
 #include <linux/rwsem.h>
 #include <linux/mutex.h>
 #include <asm/xen/hypervisor.h>
+#ifdef CONFIG_X86
+#include <asm/cpuid/api.h>
+#endif
 #include <xen/xenbus.h>
 #include <xen/xen.h>
 #include "xenbus.h"
diff --git a/drivers/zorro/zorro-sysfs.c b/drivers/zorro/zorro-sysfs.c
index 4e967754d8ad4..9989240fb76b8 100644
--- a/drivers/zorro/zorro-sysfs.c
+++ b/drivers/zorro/zorro-sysfs.c
@@ -15,6 +15,7 @@
 #include <linux/zorro.h>
 #include <linux/stat.h>
 #include <linux/string.h>
+#include <linux/sysfs.h>
 
 #include <asm/byteorder.h>
 
@@ -29,7 +30,7 @@ static ssize_t name##_show(struct device *dev,				\
 	struct zorro_dev *z;						\
 									\
 	z = to_zorro_dev(dev);						\
-	return sprintf(buf, format_string, z->field);			\
+	return sysfs_emit(buf, format_string, z->field);		\
 }									\
 static DEVICE_ATTR_RO(name);
 
@@ -44,7 +45,7 @@ static ssize_t serial_show(struct device *dev, struct device_attribute *attr,
 	struct zorro_dev *z;
 
 	z = to_zorro_dev(dev);
-	return sprintf(buf, "0x%08x\n", be32_to_cpu(z->rom.er_SerialNumber));
+	return sysfs_emit(buf, "0x%08x\n", be32_to_cpu(z->rom.er_SerialNumber));
 }
 static DEVICE_ATTR_RO(serial);
 
@@ -53,10 +54,10 @@ static ssize_t resource_show(struct device *dev, struct device_attribute *attr,
 {
 	struct zorro_dev *z = to_zorro_dev(dev);
 
-	return sprintf(buf, "0x%08lx 0x%08lx 0x%08lx\n",
-		       (unsigned long)zorro_resource_start(z),
-		       (unsigned long)zorro_resource_end(z),
-		       zorro_resource_flags(z));
+	return sysfs_emit(buf, "0x%08lx 0x%08lx 0x%08lx\n",
+			  (unsigned long)zorro_resource_start(z),
+			  (unsigned long)zorro_resource_end(z),
+			  zorro_resource_flags(z));
 }
 static DEVICE_ATTR_RO(resource);
 
@@ -65,7 +66,7 @@ static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
 {
 	struct zorro_dev *z = to_zorro_dev(dev);
 
-	return sprintf(buf, ZORRO_DEVICE_MODALIAS_FMT "\n", z->id);
+	return sysfs_emit(buf, ZORRO_DEVICE_MODALIAS_FMT "\n", z->id);
 }
 static DEVICE_ATTR_RO(modalias);
 
diff --git a/fs/9p/v9fs_vfs.h b/fs/9p/v9fs_vfs.h
index d3aefbec4de6e..34c115d7c2502 100644
--- a/fs/9p/v9fs_vfs.h
+++ b/fs/9p/v9fs_vfs.h
@@ -75,17 +75,4 @@ static inline void v9fs_invalidate_inode_attr(struct inode *inode)
 
 int v9fs_open_to_dotl_flags(int flags);
 
-static inline void v9fs_i_size_write(struct inode *inode, loff_t i_size)
-{
-	/*
-	 * 32-bit need the lock, concurrent updates could break the
-	 * sequences and make i_size_read() loop forever.
-	 * 64-bit updates are atomic and can skip the locking.
-	 */
-	if (sizeof(i_size) > sizeof(long))
-		spin_lock(&inode->i_lock);
-	i_size_write(inode, i_size);
-	if (sizeof(i_size) > sizeof(long))
-		spin_unlock(&inode->i_lock);
-}
 #endif
diff --git a/fs/9p/vfs_inode.c b/fs/9p/vfs_inode.c
index d1508b1fe1092..f468acb8ee7df 100644
--- a/fs/9p/vfs_inode.c
+++ b/fs/9p/vfs_inode.c
@@ -1141,11 +1141,13 @@ v9fs_stat2inode(struct p9_wstat *stat, struct inode *inode,
 	mode |= inode->i_mode & ~S_IALLUGO;
 	inode->i_mode = mode;
 
-	v9inode->netfs.remote_i_size = stat->length;
+	spin_lock(&inode->i_lock);
+	netfs_write_remote_i_size(inode, stat->length);
 	if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE))
-		v9fs_i_size_write(inode, stat->length);
+		i_size_write(inode, stat->length);
 	/* not real number of blocks, but 512 byte ones ... */
 	inode->i_blocks = (stat->length + 512 - 1) >> 9;
+	spin_unlock(&inode->i_lock);
 	v9inode->cache_validity &= ~V9FS_INO_INVALID_ATTR;
 }
 
diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c
index 71796a89bcf47..141fb54db65d2 100644
--- a/fs/9p/vfs_inode_dotl.c
+++ b/fs/9p/vfs_inode_dotl.c
@@ -634,10 +634,12 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode,
 		mode |= inode->i_mode & ~S_IALLUGO;
 		inode->i_mode = mode;
 
-		v9inode->netfs.remote_i_size = stat->st_size;
+		spin_lock(&inode->i_lock);
+		netfs_write_remote_i_size(inode, stat->st_size);
 		if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE))
-			v9fs_i_size_write(inode, stat->st_size);
+			i_size_write(inode, stat->st_size);
 		inode->i_blocks = stat->st_blocks;
+		spin_unlock(&inode->i_lock);
 	} else {
 		if (stat->st_result_mask & P9_STATS_ATIME) {
 			inode_set_atime(inode, stat->st_atime_sec,
@@ -662,13 +664,15 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode,
 			mode |= inode->i_mode & ~S_IALLUGO;
 			inode->i_mode = mode;
 		}
+		spin_lock(&inode->i_lock);
 		if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE) &&
 		    stat->st_result_mask & P9_STATS_SIZE) {
-			v9inode->netfs.remote_i_size = stat->st_size;
-			v9fs_i_size_write(inode, stat->st_size);
+			netfs_write_remote_i_size(inode, stat->st_size);
+			i_size_write(inode, stat->st_size);
 		}
 		if (stat->st_result_mask & P9_STATS_BLOCKS)
 			inode->i_blocks = stat->st_blocks;
+		spin_unlock(&inode->i_lock);
 	}
 	if (stat->st_result_mask & P9_STATS_GEN)
 		inode->i_generation = stat->st_gen;
diff --git a/fs/Kconfig b/fs/Kconfig
index 43cb06de297ff..cf6ae64776e62 100644
--- a/fs/Kconfig
+++ b/fs/Kconfig
@@ -78,7 +78,7 @@ config FS_DAX
 	  --map=mem:
 	  https://docs.pmem.io/ndctl-user-guide/ndctl-man-pages/ndctl-create-namespace
 
-          For ndctl to work CONFIG_DEV_DAX needs to be enabled as well. For most
+	  For ndctl to work CONFIG_DEV_DAX needs to be enabled as well. For most
 	  file systems DAX support needs to be manually enabled globally or
 	  per-inode using a mount option as well.  See the file documentation in
 	  Documentation/filesystems/dax.rst for details.
@@ -116,8 +116,8 @@ config FILE_LOCKING
 	default y
 	help
 	  This option enables standard file locking support, required
-          for filesystems like NFS and for the flock() system
-          call. Disabling this option saves about 11k.
+	  for filesystems like NFS and for the flock() system
+	  call. Disabling this option saves about 11k.
 
 source "fs/crypto/Kconfig"
 
diff --git a/fs/affs/affs.h b/fs/affs/affs.h
index a0caf6ace8601..44a3f69d275f9 100644
--- a/fs/affs/affs.h
+++ b/fs/affs/affs.h
@@ -227,11 +227,6 @@ static inline bool affs_validblock(struct super_block *sb, int block)
 	       block < AFFS_SB(sb)->s_partition_size);
 }
 
-static inline void
-affs_set_blocksize(struct super_block *sb, int size)
-{
-	sb_set_blocksize(sb, size);
-}
 static inline struct buffer_head *
 affs_bread(struct super_block *sb, int block)
 {
diff --git a/fs/affs/super.c b/fs/affs/super.c
index 079f36e1ddec1..b232251aa7bbd 100644
--- a/fs/affs/super.c
+++ b/fs/affs/super.c
@@ -358,7 +358,8 @@ static int affs_fill_super(struct super_block *sb, struct fs_context *fc)
 	size = bdev_nr_sectors(sb->s_bdev);
 	pr_debug("initial blocksize=%d, #blocks=%d\n", 512, size);
 
-	affs_set_blocksize(sb, PAGE_SIZE);
+	if (!sb_set_blocksize(sb, PAGE_SIZE))
+		return -EINVAL;
 	/* Try to find root block. Its location depends on the block size. */
 
 	i = bdev_logical_block_size(sb->s_bdev);
@@ -374,7 +375,8 @@ static int affs_fill_super(struct super_block *sb, struct fs_context *fc)
 		if (ctx->root_block < 0)
 			sbi->s_root_block = (ctx->reserved + size - 1) / 2;
 		pr_debug("setting blocksize to %d\n", blocksize);
-		affs_set_blocksize(sb, blocksize);
+		if (!sb_set_blocksize(sb, blocksize))
+			return -EINVAL;
 		sbi->s_partition_size = size;
 
 		/* The root block location that was calculated above is not
diff --git a/fs/afs/Makefile b/fs/afs/Makefile
index b49b8fe682f39..0d8f1982d596c 100644
--- a/fs/afs/Makefile
+++ b/fs/afs/Makefile
@@ -30,6 +30,7 @@ kafs-y := \
 	server.o \
 	server_list.o \
 	super.o \
+	symlink.o \
 	validation.o \
 	vlclient.o \
 	vl_alias.o \
diff --git a/fs/afs/dir.c b/fs/afs/dir.c
index aaaa55878ffd2..498b99ccdf0e2 100644
--- a/fs/afs/dir.c
+++ b/fs/afs/dir.c
@@ -44,6 +44,8 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 static int afs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 		      struct dentry *old_dentry, struct inode *new_dir,
 		      struct dentry *new_dentry, unsigned int flags);
+static int afs_dir_writepages(struct address_space *mapping,
+			      struct writeback_control *wbc);
 
 const struct file_operations afs_dir_file_operations = {
 	.open		= afs_dir_open,
@@ -68,7 +70,7 @@ const struct inode_operations afs_dir_inode_operations = {
 };
 
 const struct address_space_operations afs_dir_aops = {
-	.writepages	= afs_single_writepages,
+	.writepages	= afs_dir_writepages,
 };
 
 const struct dentry_operations afs_fs_dentry_operations = {
@@ -233,22 +235,13 @@ static ssize_t afs_do_read_single(struct afs_vnode *dvnode, struct file *file)
 	struct iov_iter iter;
 	ssize_t ret;
 	loff_t i_size;
-	bool is_dir = (S_ISDIR(dvnode->netfs.inode.i_mode) &&
-		       !test_bit(AFS_VNODE_MOUNTPOINT, &dvnode->flags));
 
 	i_size = i_size_read(&dvnode->netfs.inode);
-	if (is_dir) {
-		if (i_size < AFS_DIR_BLOCK_SIZE)
-			return afs_bad(dvnode, afs_file_error_dir_small);
-		if (i_size > AFS_DIR_BLOCK_SIZE * 1024) {
-			trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big);
-			return -EFBIG;
-		}
-	} else {
-		if (i_size > AFSPATHMAX) {
-			trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big);
-			return -EFBIG;
-		}
+	if (i_size < AFS_DIR_BLOCK_SIZE)
+		return afs_bad(dvnode, afs_file_error_dir_small);
+	if (i_size > AFS_DIR_BLOCK_SIZE * 1024) {
+		trace_afs_file_error(dvnode, -EFBIG, afs_file_error_dir_big);
+		return -EFBIG;
 	}
 
 	/* Expand the storage.  TODO: Shrink the storage too. */
@@ -277,24 +270,18 @@ static ssize_t afs_do_read_single(struct afs_vnode *dvnode, struct file *file)
 			 * buffer.
 			 */
 			ret = -ESTALE;
-		} else if (is_dir) {
+		} else {
 			int ret2 = afs_dir_check(dvnode);
 
 			if (ret2 < 0)
 				ret = ret2;
-		} else if (i_size < folioq_folio_size(dvnode->directory, 0)) {
-			/* NUL-terminate a symlink. */
-			char *symlink = kmap_local_folio(folioq_folio(dvnode->directory, 0), 0);
-
-			symlink[i_size] = 0;
-			kunmap_local(symlink);
 		}
 	}
 
 	return ret;
 }
 
-ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file)
+static ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file)
 {
 	ssize_t ret;
 
@@ -1763,13 +1750,20 @@ static int afs_link(struct dentry *from, struct inode *dir,
 	return ret;
 }
 
+static void afs_symlink_put(struct afs_operation *op)
+{
+	kfree(op->create.symlink);
+	op->create.symlink = NULL;
+	afs_create_put(op);
+}
+
 static const struct afs_operation_ops afs_symlink_operation = {
 	.issue_afs_rpc	= afs_fs_symlink,
 	.issue_yfs_rpc	= yfs_fs_symlink,
 	.success	= afs_create_success,
 	.aborted	= afs_check_for_remote_deletion,
 	.edit_dir	= afs_create_edit_dir,
-	.put		= afs_create_put,
+	.put		= afs_symlink_put,
 };
 
 /*
@@ -1779,7 +1773,9 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 		       struct dentry *dentry, const char *content)
 {
 	struct afs_operation *op;
+	struct afs_symlink *symlink;
 	struct afs_vnode *dvnode = AFS_FS_I(dir);
+	size_t clen = strlen(content);
 	int ret;
 
 	_enter("{%llx:%llu},{%pd},%s",
@@ -1791,12 +1787,20 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 		goto error;
 
 	ret = -EINVAL;
-	if (strlen(content) >= AFSPATHMAX)
+	if (clen >= AFSPATHMAX)
+		goto error;
+
+	ret = -ENOMEM;
+	symlink = kmalloc_flex(struct afs_symlink, content, clen + 1, GFP_KERNEL);
+	if (!symlink)
 		goto error;
+	refcount_set(&symlink->ref, 1);
+	memcpy(symlink->content, content, clen + 1);
 
 	op = afs_alloc_operation(NULL, dvnode->volume);
 	if (IS_ERR(op)) {
 		ret = PTR_ERR(op);
+		kfree(symlink);
 		goto error;
 	}
 
@@ -1808,7 +1812,7 @@ static int afs_symlink(struct mnt_idmap *idmap, struct inode *dir,
 	op->dentry		= dentry;
 	op->ops			= &afs_symlink_operation;
 	op->create.reason	= afs_edit_dir_for_symlink;
-	op->create.symlink	= content;
+	op->create.symlink	= symlink;
 	op->mtime		= current_time(dir);
 	ret = afs_do_sync_operation(op);
 	afs_dir_unuse_cookie(dvnode, ret);
@@ -2192,28 +2196,33 @@ static int afs_rename(struct mnt_idmap *idmap, struct inode *old_dir,
 }
 
 /*
- * Write the file contents to the cache as a single blob.
+ * Write the directory contents to the cache as a single blob.
  */
-int afs_single_writepages(struct address_space *mapping,
-			  struct writeback_control *wbc)
+static int afs_dir_writepages(struct address_space *mapping,
+			      struct writeback_control *wbc)
 {
 	struct afs_vnode *dvnode = AFS_FS_I(mapping->host);
 	struct iov_iter iter;
-	bool is_dir = (S_ISDIR(dvnode->netfs.inode.i_mode) &&
-		       !test_bit(AFS_VNODE_MOUNTPOINT, &dvnode->flags));
 	int ret = 0;
 
 	/* Need to lock to prevent the folio queue and folios from being thrown
 	 * away.
 	 */
-	down_read(&dvnode->validate_lock);
+	if (!down_read_trylock(&dvnode->validate_lock)) {
+		if (wbc->sync_mode == WB_SYNC_NONE) {
+			/* The VFS will have undirtied the inode. */
+			netfs_single_mark_inode_dirty(&dvnode->netfs.inode);
+			return 0;
+		}
+		down_read(&dvnode->validate_lock);
+	}
 
-	if (is_dir ?
-	    test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags) :
-	    atomic64_read(&dvnode->cb_expires_at) != AFS_NO_CB_PROMISE) {
+	if (test_bit(AFS_VNODE_DIR_VALID, &dvnode->flags)) {
 		iov_iter_folio_queue(&iter, ITER_SOURCE, dvnode->directory, 0, 0,
 				     i_size_read(&dvnode->netfs.inode));
 		ret = netfs_writeback_single(mapping, wbc, &iter);
+		if (ret == 1)
+			ret = 0; /* Skipped write due to lock conflict. */
 	}
 
 	up_read(&dvnode->validate_lock);
diff --git a/fs/afs/file.c b/fs/afs/file.c
index 85696ac984cc3..0467742bfeee3 100644
--- a/fs/afs/file.c
+++ b/fs/afs/file.c
@@ -427,21 +427,35 @@ static void afs_free_request(struct netfs_io_request *rreq)
 	afs_put_wb_key(rreq->netfs_priv2);
 }
 
-static void afs_update_i_size(struct inode *inode, loff_t new_i_size)
+/*
+ * Set the file size and block count, taking ->cb_lock and ->i_lock to maintain
+ * coherency and prevent 64-bit tearing on 32-bit arches.
+ *
+ * Also, estimate the number of 512 bytes blocks used, rounded up to nearest 1K
+ * for consistency with other AFS clients.
+ */
+void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size)
 {
-	struct afs_vnode *vnode = AFS_FS_I(inode);
+	struct inode *inode = &vnode->netfs.inode;
 	loff_t i_size;
 
 	write_seqlock(&vnode->cb_lock);
-	i_size = i_size_read(&vnode->netfs.inode);
+	spin_lock(&inode->i_lock);
+	i_size = i_size_read(inode);
 	if (new_i_size > i_size) {
-		i_size_write(&vnode->netfs.inode, new_i_size);
-		inode_set_bytes(&vnode->netfs.inode, new_i_size);
+		i_size_write(inode, new_i_size);
+		inode_set_bytes(inode, round_up(new_i_size, 1024));
 	}
+	spin_unlock(&inode->i_lock);
 	write_sequnlock(&vnode->cb_lock);
 	fscache_update_cookie(afs_vnode_cache(vnode), NULL, &new_i_size);
 }
 
+static void afs_update_i_size(struct inode *inode, loff_t new_i_size)
+{
+	afs_set_i_size(AFS_FS_I(inode), new_i_size);
+}
+
 static void afs_netfs_invalidate_cache(struct netfs_io_request *wreq)
 {
 	struct afs_vnode *vnode = AFS_FS_I(wreq->inode);
diff --git a/fs/afs/fsclient.c b/fs/afs/fsclient.c
index 95494d5f2b8a9..a2ffd60889f89 100644
--- a/fs/afs/fsclient.c
+++ b/fs/afs/fsclient.c
@@ -886,7 +886,7 @@ void afs_fs_symlink(struct afs_operation *op)
 	namesz = name->len;
 	padsz = (4 - (namesz & 3)) & 3;
 
-	c_namesz = strlen(op->create.symlink);
+	c_namesz = strlen(op->create.symlink->content);
 	c_padsz = (4 - (c_namesz & 3)) & 3;
 
 	reqsz = (6 * 4) + namesz + padsz + c_namesz + c_padsz + (6 * 4);
@@ -910,7 +910,7 @@ void afs_fs_symlink(struct afs_operation *op)
 		bp = (void *) bp + padsz;
 	}
 	*bp++ = htonl(c_namesz);
-	memcpy(bp, op->create.symlink, c_namesz);
+	memcpy(bp, op->create.symlink->content, c_namesz);
 	bp = (void *) bp + c_namesz;
 	if (c_padsz > 0) {
 		memset(bp, 0, c_padsz);
diff --git a/fs/afs/inode.c b/fs/afs/inode.c
index a5173434f7862..3f48458694baa 100644
--- a/fs/afs/inode.c
+++ b/fs/afs/inode.c
@@ -25,96 +25,6 @@
 #include "internal.h"
 #include "afs_fs.h"
 
-void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op)
-{
-	size_t size = strlen(op->create.symlink) + 1;
-	size_t dsize = 0;
-	char *p;
-
-	if (netfs_alloc_folioq_buffer(NULL, &vnode->directory, &dsize, size,
-				      mapping_gfp_mask(vnode->netfs.inode.i_mapping)) < 0)
-		return;
-
-	vnode->directory_size = dsize;
-	p = kmap_local_folio(folioq_folio(vnode->directory, 0), 0);
-	memcpy(p, op->create.symlink, size);
-	kunmap_local(p);
-	set_bit(AFS_VNODE_DIR_READ, &vnode->flags);
-	netfs_single_mark_inode_dirty(&vnode->netfs.inode);
-}
-
-static void afs_put_link(void *arg)
-{
-	struct folio *folio = virt_to_folio(arg);
-
-	kunmap_local(arg);
-	folio_put(folio);
-}
-
-const char *afs_get_link(struct dentry *dentry, struct inode *inode,
-			 struct delayed_call *callback)
-{
-	struct afs_vnode *vnode = AFS_FS_I(inode);
-	struct folio *folio;
-	char *content;
-	ssize_t ret;
-
-	if (!dentry) {
-		/* RCU pathwalk. */
-		if (!test_bit(AFS_VNODE_DIR_READ, &vnode->flags) || !afs_check_validity(vnode))
-			return ERR_PTR(-ECHILD);
-		goto good;
-	}
-
-	if (test_bit(AFS_VNODE_DIR_READ, &vnode->flags))
-		goto fetch;
-
-	ret = afs_validate(vnode, NULL);
-	if (ret < 0)
-		return ERR_PTR(ret);
-
-	if (!test_and_clear_bit(AFS_VNODE_ZAP_DATA, &vnode->flags) &&
-	    test_bit(AFS_VNODE_DIR_READ, &vnode->flags))
-		goto good;
-
-fetch:
-	ret = afs_read_single(vnode, NULL);
-	if (ret < 0)
-		return ERR_PTR(ret);
-	set_bit(AFS_VNODE_DIR_READ, &vnode->flags);
-
-good:
-	folio = folioq_folio(vnode->directory, 0);
-	folio_get(folio);
-	content = kmap_local_folio(folio, 0);
-	set_delayed_call(callback, afs_put_link, content);
-	return content;
-}
-
-int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
-{
-	DEFINE_DELAYED_CALL(done);
-	const char *content;
-	int len;
-
-	content = afs_get_link(dentry, d_inode(dentry), &done);
-	if (IS_ERR(content)) {
-		do_delayed_call(&done);
-		return PTR_ERR(content);
-	}
-
-	len = umin(strlen(content), buflen);
-	if (copy_to_user(buffer, content, len))
-		len = -EFAULT;
-	do_delayed_call(&done);
-	return len;
-}
-
-static const struct inode_operations afs_symlink_inode_operations = {
-	.get_link	= afs_get_link,
-	.readlink	= afs_readlink,
-};
-
 static noinline void dump_vnode(struct afs_vnode *vnode, struct afs_vnode *parent_vnode)
 {
 	static unsigned long once_only;
@@ -214,7 +124,7 @@ static int afs_inode_init_from_status(struct afs_operation *op,
 			inode->i_mode	= S_IFLNK | status->mode;
 			inode->i_op	= &afs_symlink_inode_operations;
 		}
-		inode->i_mapping->a_ops	= &afs_dir_aops;
+		inode->i_mapping->a_ops	= &afs_symlink_aops;
 		inode_nohighmem(inode);
 		mapping_set_release_always(inode->i_mapping);
 		break;
@@ -224,7 +134,8 @@ static int afs_inode_init_from_status(struct afs_operation *op,
 		return afs_protocol_error(NULL, afs_eproto_file_type);
 	}
 
-	afs_set_i_size(vnode, status->size);
+	i_size_write(inode, status->size);
+	inode_set_bytes(inode, status->size);
 	afs_set_netfs_context(vnode);
 
 	vnode->invalid_before	= status->data_version;
@@ -253,7 +164,8 @@ static void afs_apply_status(struct afs_operation *op,
 {
 	struct afs_file_status *status = &vp->scb.status;
 	struct afs_vnode *vnode = vp->vnode;
-	struct inode *inode = &vnode->netfs.inode;
+	struct netfs_inode *ictx = &vnode->netfs;
+	struct inode *inode = &ictx->inode;
 	struct timespec64 t;
 	umode_t mode;
 	bool unexpected_jump = false;
@@ -336,6 +248,8 @@ static void afs_apply_status(struct afs_operation *op,
 	}
 
 	if (data_changed) {
+		unsigned long long zero_point, size = status->size;
+
 		inode_set_iversion_raw(inode, status->data_version);
 
 		/* Only update the size if the data version jumped.  If the
@@ -343,16 +257,25 @@ static void afs_apply_status(struct afs_operation *op,
 		 * idea of what the size should be that's not the same as
 		 * what's on the server.
 		 */
-		vnode->netfs.remote_i_size = status->size;
-		if (change_size || status->size > i_size_read(inode)) {
-			afs_set_i_size(vnode, status->size);
+		spin_lock(&inode->i_lock);
+
+		if (change_size || size > i_size_read(inode)) {
+			/* We can read the sizes directly as we hold i_lock. */
+			zero_point = ictx->_zero_point;
+
 			if (unexpected_jump)
-				vnode->netfs.zero_point = status->size;
+				zero_point = size;
+			netfs_write_sizes(inode, size, size, zero_point);
+			inode_set_bytes(inode, size);
 			inode_set_ctime_to_ts(inode, t);
 			inode_set_atime_to_ts(inode, t);
+		} else {
+			netfs_write_remote_i_size(inode, size);
 		}
+		spin_unlock(&inode->i_lock);
+
 		if (op->ops == &afs_fetch_data_operation)
-			op->fetch.subreq->rreq->i_size = status->size;
+			op->fetch.subreq->rreq->i_size = size;
 	}
 }
 
@@ -709,7 +632,7 @@ int afs_getattr(struct mnt_idmap *idmap, const struct path *path,
 		 * it, but we need to give userspace the server's size.
 		 */
 		if (S_ISDIR(inode->i_mode))
-			stat->size = vnode->netfs.remote_i_size;
+			stat->size = netfs_read_remote_i_size(inode);
 	} while (read_seqretry(&vnode->cb_lock, seq));
 
 	return 0;
@@ -756,12 +679,14 @@ void afs_evict_inode(struct inode *inode)
 			.range_end = LLONG_MAX,
 		};
 
-		afs_single_writepages(inode->i_mapping, &wbc);
+		inode->i_mapping->a_ops->writepages(inode->i_mapping, &wbc);
 	}
 
 	netfs_wait_for_outstanding_io(inode);
 	truncate_inode_pages_final(&inode->i_data);
 	netfs_free_folioq_buffer(vnode->directory);
+	if (vnode->symlink)
+		afs_evict_symlink(vnode);
 
 	afs_set_cache_aux(vnode, &aux);
 	netfs_clear_inode_writeback(inode, &aux);
@@ -889,7 +814,7 @@ int afs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 		 */
 		if (!(attr->ia_valid & (supported & ~ATTR_SIZE & ~ATTR_MTIME)) &&
 		    attr->ia_size < i_size &&
-		    attr->ia_size > vnode->netfs.remote_i_size) {
+		    attr->ia_size > netfs_read_remote_i_size(inode)) {
 			truncate_setsize(inode, attr->ia_size);
 			netfs_resize_file(&vnode->netfs, size, false);
 			fscache_resize_cookie(afs_vnode_cache(vnode),
diff --git a/fs/afs/internal.h b/fs/afs/internal.h
index 599353c333372..0b72a85662999 100644
--- a/fs/afs/internal.h
+++ b/fs/afs/internal.h
@@ -710,6 +710,7 @@ struct afs_vnode {
 #define AFS_VNODE_DIR_READ	11		/* Set if we've read a dir's contents */
 
 	struct folio_queue	*directory;	/* Directory contents */
+	struct afs_symlink __rcu *symlink;	/* Symlink content */
 	struct list_head	wb_keys;	/* List of keys available for writeback */
 	struct list_head	pending_locks;	/* locks waiting to be granted */
 	struct list_head	granted_locks;	/* locks granted on this file */
@@ -776,6 +777,15 @@ struct afs_permits {
 	struct afs_permit	permits[] __counted_by(nr_permits);	/* List of permits sorted by key pointer */
 };
 
+/*
+ * Copy of symlink content for normal use.
+ */
+struct afs_symlink {
+	struct rcu_head		rcu;
+	refcount_t		ref;
+	char			content[];
+};
+
 /*
  * Error prioritisation and accumulation.
  */
@@ -887,7 +897,7 @@ struct afs_operation {
 		struct {
 			int	reason;		/* enum afs_edit_dir_reason */
 			mode_t	mode;
-			const char *symlink;
+			struct afs_symlink *symlink;
 		} create;
 		struct {
 			bool	need_rehash;
@@ -1098,13 +1108,10 @@ extern const struct inode_operations afs_dir_inode_operations;
 extern const struct address_space_operations afs_dir_aops;
 extern const struct dentry_operations afs_fs_dentry_operations;
 
-ssize_t afs_read_single(struct afs_vnode *dvnode, struct file *file);
 ssize_t afs_read_dir(struct afs_vnode *dvnode, struct file *file)
 	__acquires(&dvnode->validate_lock);
 extern void afs_d_release(struct dentry *);
 extern void afs_check_for_remote_deletion(struct afs_operation *);
-int afs_single_writepages(struct address_space *mapping,
-			  struct writeback_control *wbc);
 
 /*
  * dir_edit.c
@@ -1157,6 +1164,7 @@ extern int afs_open(struct inode *, struct file *);
 extern int afs_release(struct inode *, struct file *);
 void afs_fetch_data_async_rx(struct work_struct *work);
 void afs_fetch_data_immediate_cancel(struct afs_call *call);
+void afs_set_i_size(struct afs_vnode *vnode, loff_t new_i_size);
 
 /*
  * flock.c
@@ -1246,10 +1254,6 @@ extern void afs_fs_probe_cleanup(struct afs_net *);
  */
 extern const struct afs_operation_ops afs_fetch_status_operation;
 
-void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op);
-const char *afs_get_link(struct dentry *dentry, struct inode *inode,
-			 struct delayed_call *callback);
-int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen);
 extern void afs_vnode_commit_status(struct afs_operation *, struct afs_vnode_param *);
 extern int afs_fetch_status(struct afs_vnode *, struct key *, bool, afs_access_t *);
 extern int afs_ilookup5_test_by_fid(struct inode *, void *);
@@ -1599,6 +1603,21 @@ void afs_detach_volume_from_servers(struct afs_volume *volume, struct afs_server
 extern int __init afs_fs_init(void);
 extern void afs_fs_exit(void);
 
+/*
+ * symlink.c
+ */
+extern const struct inode_operations afs_symlink_inode_operations;
+extern const struct address_space_operations afs_symlink_aops;
+
+void afs_invalidate_symlink(struct afs_vnode *vnode);
+void afs_evict_symlink(struct afs_vnode *vnode);
+void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op);
+const char *afs_get_link(struct dentry *dentry, struct inode *inode,
+			 struct delayed_call *callback);
+int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen);
+int afs_symlink_writepages(struct address_space *mapping,
+			   struct writeback_control *wbc);
+
 /*
  * validation.c
  */
@@ -1758,16 +1777,6 @@ static inline void afs_update_dentry_version(struct afs_operation *op,
 			(void *)(unsigned long)dir_vp->scb.status.data_version;
 }
 
-/*
- * Set the file size and block count.  Estimate the number of 512 bytes blocks
- * used, rounded up to nearest 1K for consistency with other AFS clients.
- */
-static inline void afs_set_i_size(struct afs_vnode *vnode, u64 size)
-{
-	i_size_write(&vnode->netfs.inode, size);
-	vnode->netfs.inode.i_blocks = ((size + 1023) >> 10) << 1;
-}
-
 /*
  * Check for a conflicting operation on a directory that we just unlinked from.
  * If someone managed to sneak a link or an unlink in on the file we just
diff --git a/fs/afs/symlink.c b/fs/afs/symlink.c
new file mode 100644
index 0000000000000..ed5868369f372
--- /dev/null
+++ b/fs/afs/symlink.c
@@ -0,0 +1,278 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/* AFS filesystem symbolic link handling
+ *
+ * Copyright (C) 2026 Red Hat, Inc. All Rights Reserved.
+ * Written by David Howells (dhowells@redhat.com)
+ */
+
+#include <linux/kernel.h>
+#include <linux/fs.h>
+#include <linux/namei.h>
+#include <linux/pagemap.h>
+#include <linux/iov_iter.h>
+#include "internal.h"
+
+static void afs_put_symlink(struct afs_symlink *symlink)
+{
+	if (refcount_dec_and_test(&symlink->ref))
+		kfree_rcu(symlink, rcu);
+}
+
+static void afs_replace_symlink(struct afs_vnode *vnode, struct afs_symlink *symlink)
+{
+	struct afs_symlink *old;
+
+	old = rcu_replace_pointer(vnode->symlink, symlink,
+				  lockdep_is_held(&vnode->validate_lock));
+	if (old)
+		afs_put_symlink(old);
+}
+
+/*
+ * In the event that a third-party update of a symlink occurs, dispose of the
+ * copy of the old contents.  Called under ->validate_lock.
+ */
+void afs_invalidate_symlink(struct afs_vnode *vnode)
+{
+	afs_replace_symlink(vnode, NULL);
+}
+
+/*
+ * Dispose of a symlink copy during inode deletion.
+ */
+void afs_evict_symlink(struct afs_vnode *vnode)
+{
+	struct afs_symlink *old;
+
+	old = rcu_replace_pointer(vnode->symlink, NULL, true);
+	if (old)
+		afs_put_symlink(old);
+
+}
+
+/*
+ * Set up a locally created symlink inode for immediate write to the cache.
+ */
+void afs_init_new_symlink(struct afs_vnode *vnode, struct afs_operation *op)
+{
+	struct afs_symlink *symlink = op->create.symlink;
+	size_t dsize = 0;
+	size_t size = strlen(symlink->content) + 1;
+	char *p;
+
+	rcu_assign_pointer(vnode->symlink, symlink);
+	op->create.symlink = NULL;
+
+	if (!fscache_cookie_enabled(netfs_i_cookie(&vnode->netfs)))
+		return;
+
+	if (netfs_alloc_folioq_buffer(NULL, &vnode->directory, &dsize, size,
+				      mapping_gfp_mask(vnode->netfs.inode.i_mapping)) < 0)
+		return;
+
+	vnode->directory_size = dsize;
+	p = kmap_local_folio(folioq_folio(vnode->directory, 0), 0);
+	memcpy(p, symlink->content, size);
+	kunmap_local(p);
+	netfs_single_mark_inode_dirty(&vnode->netfs.inode);
+}
+
+/*
+ * Read a symlink in a single download.
+ */
+static ssize_t afs_do_read_symlink(struct afs_vnode *vnode)
+{
+	struct afs_symlink *symlink;
+	struct iov_iter iter;
+	ssize_t ret;
+	loff_t i_size;
+
+	i_size = i_size_read(&vnode->netfs.inode);
+	if (i_size > PAGE_SIZE - 1) {
+		trace_afs_file_error(vnode, -EFBIG, afs_file_error_dir_big);
+		return -EFBIG;
+	}
+
+	if (!vnode->directory) {
+		size_t cur_size = 0;
+
+		ret = netfs_alloc_folioq_buffer(NULL,
+						&vnode->directory, &cur_size, PAGE_SIZE,
+						mapping_gfp_mask(vnode->netfs.inode.i_mapping));
+		vnode->directory_size = PAGE_SIZE - 1;
+		if (ret < 0)
+			return ret;
+	}
+
+	iov_iter_folio_queue(&iter, ITER_DEST, vnode->directory, 0, 0, PAGE_SIZE);
+
+	/* AFS requires us to perform the read of a symlink as a single unit to
+	 * avoid issues with the content being changed between reads.
+	 */
+	ret = netfs_read_single(&vnode->netfs.inode, NULL, &iter);
+	if (ret >= 0) {
+		i_size = ret;
+		if (i_size > PAGE_SIZE - 1) {
+			trace_afs_file_error(vnode, -EFBIG, afs_file_error_dir_big);
+			return -EFBIG;
+		}
+		vnode->directory_size = i_size;
+
+		/* Copy the symlink. */
+		symlink = kmalloc_flex(struct afs_symlink, content, i_size + 1,
+				       GFP_KERNEL);
+		if (!symlink)
+			return -ENOMEM;
+
+		refcount_set(&symlink->ref, 1);
+		symlink->content[i_size] = 0;
+
+		const char *s = kmap_local_folio(folioq_folio(vnode->directory, 0), 0);
+
+		memcpy(symlink->content, s, i_size);
+		kunmap_local(s);
+
+		afs_replace_symlink(vnode, symlink);
+	}
+
+	if (!fscache_cookie_enabled(netfs_i_cookie(&vnode->netfs))) {
+		netfs_free_folioq_buffer(vnode->directory);
+		vnode->directory = NULL;
+		vnode->directory_size = 0;
+	}
+
+	return ret;
+}
+
+static ssize_t afs_read_symlink(struct afs_vnode *vnode)
+{
+	ssize_t ret;
+
+	fscache_use_cookie(afs_vnode_cache(vnode), false);
+	ret = afs_do_read_symlink(vnode);
+	fscache_unuse_cookie(afs_vnode_cache(vnode), NULL, NULL);
+	return ret;
+}
+
+static void afs_put_link(void *arg)
+{
+	afs_put_symlink(arg);
+}
+
+const char *afs_get_link(struct dentry *dentry, struct inode *inode,
+			 struct delayed_call *callback)
+{
+	struct afs_symlink *symlink;
+	struct afs_vnode *vnode = AFS_FS_I(inode);
+	ssize_t ret;
+
+	if (!dentry) {
+		/* RCU pathwalk. */
+		symlink = rcu_dereference(vnode->symlink);
+		if (!symlink || !afs_check_validity(vnode))
+			return ERR_PTR(-ECHILD);
+		set_delayed_call(callback, NULL, NULL);
+		return symlink->content;
+	}
+
+	if (vnode->symlink) {
+		ret = afs_validate(vnode, NULL);
+		if (ret < 0)
+			return ERR_PTR(ret);
+
+		down_read(&vnode->validate_lock);
+		if (vnode->symlink)
+			goto good;
+		up_read(&vnode->validate_lock);
+	}
+
+	if (down_write_killable(&vnode->validate_lock) < 0)
+		return ERR_PTR(-ERESTARTSYS);
+	if (!vnode->symlink) {
+		ret = afs_read_symlink(vnode);
+		if (ret < 0) {
+			up_write(&vnode->validate_lock);
+			return ERR_PTR(ret);
+		}
+	}
+
+	downgrade_write(&vnode->validate_lock);
+	
+good:
+	symlink = rcu_dereference_protected(vnode->symlink,
+					    lockdep_is_held(&vnode->validate_lock));
+	refcount_inc(&symlink->ref);
+	up_read(&vnode->validate_lock);
+
+	set_delayed_call(callback, afs_put_link, symlink);
+	return symlink->content;
+}
+
+int afs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
+{
+	DEFINE_DELAYED_CALL(done);
+	const char *content;
+	int len;
+
+	content = afs_get_link(dentry, d_inode(dentry), &done);
+	if (IS_ERR(content)) {
+		do_delayed_call(&done);
+		return PTR_ERR(content);
+	}
+
+	len = umin(strlen(content), buflen);
+	if (copy_to_user(buffer, content, len))
+		len = -EFAULT;
+	do_delayed_call(&done);
+	return len;
+}
+
+/*
+ * Write the symlink contents to the cache as a single blob.  We then throw
+ * away the page we used to receive it.
+ */
+int afs_symlink_writepages(struct address_space *mapping,
+			   struct writeback_control *wbc)
+{
+	struct afs_vnode *vnode = AFS_FS_I(mapping->host);
+	struct iov_iter iter;
+	int ret = 0;
+
+	if (!down_read_trylock(&vnode->validate_lock)) {
+		if (wbc->sync_mode == WB_SYNC_NONE) {
+			/* The VFS will have undirtied the inode. */
+			netfs_single_mark_inode_dirty(&vnode->netfs.inode);
+			return 0;
+		}
+		down_read(&vnode->validate_lock);
+	}
+
+	if (vnode->directory &&
+	    atomic64_read(&vnode->cb_expires_at) != AFS_NO_CB_PROMISE) {
+		iov_iter_folio_queue(&iter, ITER_SOURCE, vnode->directory, 0, 0,
+				     i_size_read(&vnode->netfs.inode));
+		ret = netfs_writeback_single(mapping, wbc, &iter);
+	}
+
+	if (ret == 0) {
+		mutex_lock(&vnode->netfs.wb_lock);
+		netfs_free_folioq_buffer(vnode->directory);
+		vnode->directory = NULL;
+		vnode->directory_size = 0;
+		mutex_unlock(&vnode->netfs.wb_lock);
+	} else if (ret == 1) {
+		ret = 0; /* Skipped write due to lock conflict. */
+	}
+
+	up_read(&vnode->validate_lock);
+	return ret;
+}
+
+const struct inode_operations afs_symlink_inode_operations = {
+	.get_link	= afs_get_link,
+	.readlink	= afs_readlink,
+};
+
+const struct address_space_operations afs_symlink_aops = {
+	.writepages	= afs_symlink_writepages,
+};
diff --git a/fs/afs/validation.c b/fs/afs/validation.c
index 0ba8336c90250..e997563af658b 100644
--- a/fs/afs/validation.c
+++ b/fs/afs/validation.c
@@ -465,11 +465,17 @@ int afs_validate(struct afs_vnode *vnode, struct key *key)
 	vnode->cb_ro_snapshot = cb_ro_snapshot;
 	vnode->cb_scrub = cb_scrub;
 
-	/* if the vnode's data version number changed then its contents are
-	 * different */
+	/* If the vnode's data version number changed then its contents are
+	 * different.  Note that afs_apply_status() doesn't set ZAP_DATA on
+	 * directories.
+	 */
 	zap |= test_and_clear_bit(AFS_VNODE_ZAP_DATA, &vnode->flags);
-	if (zap)
-		afs_zap_data(vnode);
+	if (zap) {
+		if (S_ISREG(vnode->netfs.inode.i_mode))
+			afs_zap_data(vnode);
+		else if (S_ISLNK(vnode->netfs.inode.i_mode))
+			afs_invalidate_symlink(vnode);
+	}
 	up_write(&vnode->validate_lock);
 	_leave(" = 0");
 	return 0;
diff --git a/fs/afs/write.c b/fs/afs/write.c
index fcfed9d24e0a3..7f34b939706a0 100644
--- a/fs/afs/write.c
+++ b/fs/afs/write.c
@@ -142,7 +142,7 @@ static void afs_issue_write_worker(struct work_struct *work)
 	afs_begin_vnode_operation(op);
 
 	op->store.write_iter	= &subreq->io_iter;
-	op->store.i_size	= umax(pos + len, vnode->netfs.remote_i_size);
+	op->store.i_size	= umax(pos + len, netfs_read_remote_i_size(&vnode->netfs.inode));
 	op->mtime		= inode_get_mtime(&vnode->netfs.inode);
 
 	afs_wait_for_operation(op);
diff --git a/fs/afs/yfsclient.c b/fs/afs/yfsclient.c
index 24fb562ebd33a..d941179730a98 100644
--- a/fs/afs/yfsclient.c
+++ b/fs/afs/yfsclient.c
@@ -960,7 +960,7 @@ void yfs_fs_symlink(struct afs_operation *op)
 
 	_enter("");
 
-	contents_sz = strlen(op->create.symlink);
+	contents_sz = strlen(op->create.symlink->content);
 	call = afs_alloc_flat_call(op->net, &yfs_RXYFSSymlink,
 				   sizeof(__be32) +
 				   sizeof(struct yfs_xdr_RPCFlags) +
@@ -981,7 +981,7 @@ void yfs_fs_symlink(struct afs_operation *op)
 	bp = xdr_encode_u32(bp, 0); /* RPC flags */
 	bp = xdr_encode_YFSFid(bp, &dvp->fid);
 	bp = xdr_encode_name(bp, name);
-	bp = xdr_encode_string(bp, op->create.symlink, contents_sz);
+	bp = xdr_encode_string(bp, op->create.symlink->content, contents_sz);
 	bp = xdr_encode_YFSStoreStatus(bp, &mode, &op->mtime);
 	yfs_check_req(call, bp);
 
diff --git a/fs/befs/linuxvfs.c b/fs/befs/linuxvfs.c
index c12caae9a9679..ee0cbae521b9f 100644
--- a/fs/befs/linuxvfs.c
+++ b/fs/befs/linuxvfs.c
@@ -860,7 +860,8 @@ befs_fill_super(struct super_block *sb, struct fs_context *fc)
 	 */
 	sb->s_magic = BEFS_SUPER_MAGIC;
 	/* Set real blocksize of fs */
-	sb_set_blocksize(sb, (ulong) befs_sb->block_size);
+	if (!sb_set_blocksize(sb, (ulong) befs_sb->block_size))
+		goto unacquire_priv_sbp;
 	sb->s_op = &befs_sops;
 	sb->s_export_op = &befs_export_operations;
 	sb->s_time_min = 0;
diff --git a/fs/bfs/inode.c b/fs/bfs/inode.c
index 19e49c8cf7505..9c3e90390824c 100644
--- a/fs/bfs/inode.c
+++ b/fs/bfs/inode.c
@@ -346,7 +346,8 @@ static int bfs_fill_super(struct super_block *s, struct fs_context *fc)
 	s->s_time_min = 0;
 	s->s_time_max = U32_MAX;
 
-	sb_set_blocksize(s, BFS_BSIZE);
+	if (!sb_set_blocksize(s, BFS_BSIZE))
+		goto out;
 
 	sbh = sb_bread(s, 0);
 	if (!sbh)
diff --git a/fs/bpf_fs_kfuncs.c b/fs/bpf_fs_kfuncs.c
index e4e51a1d0de28..8642a754b37dc 100644
--- a/fs/bpf_fs_kfuncs.c
+++ b/fs/bpf_fs_kfuncs.c
@@ -100,7 +100,7 @@ static bool match_security_bpf_prefix(const char *name__str)
 
 static int bpf_xattr_read_permission(const char *name, struct inode *inode)
 {
-	if (WARN_ON(!inode))
+	if (!inode)
 		return -EINVAL;
 
 	/* Allow reading xattr with user. and security.bpf. prefix */
@@ -170,7 +170,7 @@ __bpf_kfunc_end_defs();
 
 static int bpf_xattr_write_permission(const char *name, struct inode *inode)
 {
-	if (WARN_ON(!inode))
+	if (!inode)
 		return -EINVAL;
 
 	/* Only allow setting and removing security.bpf. xattrs */
@@ -200,7 +200,7 @@ int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str,
 				const struct bpf_dynptr *value_p, int flags)
 {
 
-	struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p;
+	const struct bpf_dynptr_kern *value_ptr = (struct bpf_dynptr_kern *)value_p;
 	struct inode *inode = d_inode(dentry);
 	const void *value;
 	u32 value_len;
@@ -289,6 +289,9 @@ __bpf_kfunc int bpf_set_dentry_xattr(struct dentry *dentry, const char *name__st
 	struct inode *inode = d_inode(dentry);
 	int ret;
 
+	if (!inode)
+		return -EINVAL;
+
 	inode_lock(inode);
 	ret = bpf_set_dentry_xattr_locked(dentry, name__str, value_p, flags);
 	inode_unlock(inode);
@@ -314,6 +317,9 @@ __bpf_kfunc int bpf_remove_dentry_xattr(struct dentry *dentry, const char *name_
 	struct inode *inode = d_inode(dentry);
 	int ret;
 
+	if (!inode)
+		return -EINVAL;
+
 	inode_lock(inode);
 	ret = bpf_remove_dentry_xattr_locked(dentry, name__str);
 	inode_unlock(inode);
diff --git a/fs/btrfs/defrag.c b/fs/btrfs/defrag.c
index 7e2db5d3a4d4c..a8d49d9ca981c 100644
--- a/fs/btrfs/defrag.c
+++ b/fs/btrfs/defrag.c
@@ -860,9 +860,6 @@ static struct folio *defrag_prepare_one_folio(struct btrfs_inode *inode, pgoff_t
 		return folio;
 
 	/*
-	 * Since we can defragment files opened read-only, we can encounter
-	 * transparent huge pages here (see CONFIG_READ_ONLY_THP_FOR_FS).
-	 *
 	 * The IO for such large folios is not fully tested, thus return
 	 * an error to reject such folios unless it's an experimental build.
 	 *
diff --git a/fs/btrfs/extent_map.c b/fs/btrfs/extent_map.c
index 9284c0a81befb..6b79bff241f21 100644
--- a/fs/btrfs/extent_map.c
+++ b/fs/btrfs/extent_map.c
@@ -1246,7 +1246,9 @@ static struct btrfs_inode *find_first_inode_to_shrink(struct btrfs_root *root,
 		write_unlock(&tree->lock);
 next:
 		from = btrfs_ino(inode) + 1;
-		cond_resched_lock(&root->inodes.xa_lock);
+		xa_unlock(&root->inodes);
+		cond_resched();
+		xa_lock(&root->inodes);
 	}
 	xa_unlock(&root->inodes);
 
diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c
index cf1cb5c4db757..8c171ed07008b 100644
--- a/fs/btrfs/file.c
+++ b/fs/btrfs/file.c
@@ -633,7 +633,7 @@ int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
 							 trans->transid);
 			btrfs_set_file_extent_num_bytes(leaf, fi,
 							end - other_start);
-			return 0;
+			goto mark_dirty;
 		}
 	}
 
@@ -661,7 +661,7 @@ int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
 							other_end - start);
 			btrfs_set_file_extent_offset(leaf, fi,
 						     start - orig_offset);
-			return 0;
+			goto mark_dirty;
 		}
 	}
 
@@ -788,7 +788,12 @@ int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
 		}
 	}
 
-	return 0;
+mark_dirty:
+	ret = btrfs_inode_set_file_extent_range(inode, start, end - start);
+	if (ret)
+		btrfs_abort_transaction(trans, ret);
+
+	return ret;
 }
 
 /*
diff --git a/fs/btrfs/fs.h b/fs/btrfs/fs.h
index a4758d94b32e9..a8aa086a4df86 100644
--- a/fs/btrfs/fs.h
+++ b/fs/btrfs/fs.h
@@ -155,6 +155,7 @@ enum {
 	BTRFS_FS_LOG_RECOVERING,
 	BTRFS_FS_OPEN,
 	BTRFS_FS_QUOTA_ENABLED,
+	BTRFS_FS_SQUOTA_ENABLING,
 	BTRFS_FS_UPDATE_UUID_TREE_GEN,
 	BTRFS_FS_CREATING_FREE_SPACE_TREE,
 	BTRFS_FS_BTREE_ERR,
diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index 75136a1727101..509042adee078 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -4745,7 +4745,7 @@ static void btrfs_prune_dentries(struct btrfs_root *root)
 
 	inode = btrfs_find_first_inode(root, min_ino);
 	while (inode) {
-		if (icount_read(&inode->vfs_inode) > 1)
+		if (icount_read_once(&inode->vfs_inode) > 1)
 			d_prune_aliases(&inode->vfs_inode);
 
 		min_ino = btrfs_ino(inode) + 1;
@@ -10699,7 +10699,9 @@ struct btrfs_inode *btrfs_find_first_inode(struct btrfs_root *root, u64 min_ino)
 			break;
 
 		from = btrfs_ino(inode) + 1;
-		cond_resched_lock(&root->inodes.xa_lock);
+		xa_unlock(&root->inodes);
+		cond_resched();
+		xa_lock(&root->inodes);
 	}
 	xa_unlock(&root->inodes);
 
diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c
index cdf736d3a4e5b..6838faceb6d5c 100644
--- a/fs/btrfs/qgroup.c
+++ b/fs/btrfs/qgroup.c
@@ -1107,7 +1107,13 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
 	if (simple) {
 		fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE;
 		btrfs_set_fs_incompat(fs_info, SIMPLE_QUOTA);
-		btrfs_set_qgroup_status_enable_gen(leaf, ptr, trans->transid);
+		/*
+		 * Set the enable generation to the next transaction, as we cannot
+		 * ensure that extents written during this transaction will see any
+		 * state we have set here. So we should treat all extents of the
+		 * transaction as coming in before squotas was enabled.
+		 */
+		btrfs_set_qgroup_status_enable_gen(leaf, ptr, trans->transid + 1);
 	} else {
 		fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
 	}
@@ -1210,7 +1216,15 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
 		goto out_free_path;
 	}
 
-	fs_info->qgroup_enable_gen = trans->transid;
+	/*
+	 * Set fs_info->qgroup_enable_gen and BTRFS_FS_SQUOTA_ENABLING
+	 * under the transaction handle. We want to ensure that all extents in
+	 * the next transaction definitely see them.
+	 */
+	if (simple) {
+		fs_info->qgroup_enable_gen = trans->transid + 1;
+		set_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags);
+	}
 
 	mutex_unlock(&fs_info->qgroup_ioctl_lock);
 	/*
@@ -1224,9 +1238,15 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
 	 */
 	ret = btrfs_commit_transaction(trans);
 	trans = NULL;
+
 	mutex_lock(&fs_info->qgroup_ioctl_lock);
-	if (ret)
+	if (ret) {
+		if (simple) {
+			clear_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags);
+			fs_info->qgroup_enable_gen = 0;
+		}
 		goto out_free_path;
+	}
 
 	/*
 	 * Set quota enabled flag after committing the transaction, to avoid
@@ -1236,6 +1256,8 @@ int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
 	spin_lock(&fs_info->qgroup_lock);
 	fs_info->quota_root = quota_root;
 	set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
+	if (simple)
+		clear_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags);
 	spin_unlock(&fs_info->qgroup_lock);
 
 	/* Skip rescan for simple qgroups. */
@@ -1715,32 +1737,24 @@ int btrfs_create_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid)
 	return ret;
 }
 
-static bool can_delete_parent_qgroup(struct btrfs_qgroup *qgroup)
-
+static bool can_delete_parent_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup)
 {
 	ASSERT(btrfs_qgroup_level(qgroup->qgroupid));
+	if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
+		squota_check_parent_usage(fs_info, qgroup);
 	return list_empty(&qgroup->members);
 }
 
 /*
- * Return true if we can delete the squota qgroup and false otherwise.
- *
- * Rules for whether we can delete:
- *
- * A subvolume qgroup can be removed iff the subvolume is fully deleted, which
- * is iff there is 0 usage in the qgroup.
- *
- * A higher level qgroup can be removed iff it has no members.
- * Note: We audit its usage to warn on inconsitencies without blocking deletion.
+ * Because a shared extent can outlive its owning subvolume, we cannot delete a
+ * subvol squota qgroup until all of the extents it owns are gone, even if the
+ * subvolume itself has been deleted.
  */
-static bool can_delete_squota_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup *qgroup)
+static bool can_delete_squota_subvol_qgroup(struct btrfs_fs_info *fs_info,
+					    struct btrfs_qgroup *qgroup)
 {
 	ASSERT(btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE);
-
-	if (btrfs_qgroup_level(qgroup->qgroupid) > 0) {
-		squota_check_parent_usage(fs_info, qgroup);
-		return can_delete_parent_qgroup(qgroup);
-	}
+	ASSERT(btrfs_qgroup_level(qgroup->qgroupid) == 0);
 
 	return !(qgroup->rfer || qgroup->excl || qgroup->rfer_cmpr || qgroup->excl_cmpr);
 }
@@ -1754,14 +1768,11 @@ static int can_delete_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup
 {
 	struct btrfs_key key;
 	BTRFS_PATH_AUTO_FREE(path);
-
-	/* Since squotas cannot be inconsistent, they have special rules for deletion. */
-	if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
-		return can_delete_squota_qgroup(fs_info, qgroup);
+	int ret;
 
 	/* For higher level qgroup, we can only delete it if it has no child. */
 	if (btrfs_qgroup_level(qgroup->qgroupid))
-		return can_delete_parent_qgroup(qgroup);
+		return can_delete_parent_qgroup(fs_info, qgroup);
 
 	/*
 	 * For level-0 qgroups, we can only delete it if it has no subvolume
@@ -1777,10 +1788,21 @@ static int can_delete_qgroup(struct btrfs_fs_info *fs_info, struct btrfs_qgroup
 		return -ENOMEM;
 
 	/*
-	 * The @ret from btrfs_find_root() exactly matches our definition for
-	 * the return value, thus can be returned directly.
+	 * Any subvol qgroup, regardless of mode, cannot be deleted if the
+	 * subvol still exists.
 	 */
-	return btrfs_find_root(fs_info->tree_root, &key, path, NULL, NULL);
+	ret = btrfs_find_root(fs_info->tree_root, &key, path, NULL, NULL);
+	/*
+	 * btrfs_find_root returns <0 on error, 0 if found, and >0 if not,
+	 * so the "found" and "error" cases match our desired return values.
+	 */
+	if (ret <= 0)
+		return ret;
+
+	/* Squotas require additional checks, even if the subvol is deleted. */
+	if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
+		return can_delete_squota_subvol_qgroup(fs_info, qgroup);
+	return 1;
 }
 
 int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid)
@@ -4922,7 +4944,8 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info,
 	u64 num_bytes = delta->num_bytes;
 	const int sign = (delta->is_inc ? 1 : -1);
 
-	if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE)
+	if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE &&
+	    !test_bit(BTRFS_FS_SQUOTA_ENABLING, &fs_info->flags))
 		return 0;
 
 	if (!btrfs_is_fstree(root))
@@ -4934,8 +4957,9 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info,
 
 	spin_lock(&fs_info->qgroup_lock);
 	qgroup = find_qgroup_rb(fs_info, root);
-	if (!qgroup) {
-		ret = -ENOENT;
+	if (WARN_ON_ONCE(!qgroup)) {
+		btrfs_warn(fs_info, "squota failed to find qgroup for root %llu", root);
+		ret = 0;
 		goto out;
 	}
 
@@ -4944,8 +4968,19 @@ int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info,
 	list_for_each_entry(qg, &qgroup_list, iterator) {
 		struct btrfs_qgroup_list *glist;
 
-		qg->excl += num_bytes * sign;
-		qg->rfer += num_bytes * sign;
+		ASSERT(qg->excl == qg->rfer);
+		if (WARN_ON_ONCE(sign < 0 && qg->excl < num_bytes)) {
+			btrfs_warn(fs_info,
+				   "squota underflow qg %hu/%llu excl %llu num_bytes %llu",
+				   btrfs_qgroup_level(qg->qgroupid),
+				   btrfs_qgroup_subvolid(qg->qgroupid),
+				   qg->excl, num_bytes);
+			qg->excl = 0;
+			qg->rfer = 0;
+		} else {
+			qg->excl += num_bytes * sign;
+			qg->rfer += num_bytes * sign;
+		}
 		qgroup_dirty(fs_info, qg);
 
 		list_for_each_entry(glist, &qg->groups, next_group)
diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index 08ee8f316d96d..dabc9522e8814 100644
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -1410,7 +1410,7 @@ static void generate_pq_vertical_step(struct btrfs_raid_bio *rbio, unsigned int
 				rbio_qstripe_paddr(rbio, sector_nr, step_nr));
 
 		assert_rbio(rbio);
-		raid6_call.gen_syndrome(rbio->real_stripes, step, pointers);
+		raid6_gen_syndrome(rbio->real_stripes, step, pointers);
 	} else {
 		/* raid5 */
 		memcpy(pointers[rbio->nr_data], pointers[0], step);
@@ -1987,10 +1987,10 @@ static void recover_vertical_step(struct btrfs_raid_bio *rbio,
 		}
 
 		if (failb == rbio->real_stripes - 2) {
-			raid6_datap_recov(rbio->real_stripes, step,
+			raid6_recov_datap(rbio->real_stripes, step,
 					  faila, pointers);
 		} else {
-			raid6_2data_recov(rbio->real_stripes, step,
+			raid6_recov_2data(rbio->real_stripes, step,
 					  faila, failb, pointers);
 		}
 	} else {
@@ -2644,7 +2644,7 @@ static bool verify_one_parity_step(struct btrfs_raid_bio *rbio,
 	if (has_qstripe) {
 		assert_rbio(rbio);
 		/* RAID6, call the library function to fill in our P/Q. */
-		raid6_call.gen_syndrome(rbio->real_stripes, step, pointers);
+		raid6_gen_syndrome(rbio->real_stripes, step, pointers);
 	} else {
 		/* RAID5. */
 		memcpy(pointers[nr_data], pointers[0], step);
diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c
index 1b83ed0e0a63f..2937db690b40b 100644
--- a/fs/cachefiles/namei.c
+++ b/fs/cachefiles/namei.c
@@ -130,6 +130,8 @@ struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache,
 		ret = cachefiles_inject_write_error();
 		if (ret == 0) {
 			subdir = vfs_mkdir(&nop_mnt_idmap, d_inode(dir), subdir, 0700, NULL);
+			if (IS_ERR(subdir))
+				ret = PTR_ERR(subdir);
 		} else {
 			end_creating(subdir);
 			subdir = ERR_PTR(ret);
diff --git a/fs/ceph/file.c b/fs/ceph/file.c
index d54d71669176b..bdc734837df40 100644
--- a/fs/ceph/file.c
+++ b/fs/ceph/file.c
@@ -996,6 +996,10 @@ int ceph_atomic_open(struct inode *dir, struct dentry *dentry,
 			ceph_init_inode_acls(newino, &as_ctx);
 			file->f_mode |= FMODE_CREATED;
 		}
+		if ((flags & OPENAT2_REGULAR) && !d_is_reg(dentry)) {
+			err = -EFTYPE;
+			goto out_req;
+		}
 		err = finish_open(file, dentry, ceph_open);
 	}
 out_req:
diff --git a/fs/ceph/mds_client.c b/fs/ceph/mds_client.c
index ed17e0023705e..0edb6a2515012 100644
--- a/fs/ceph/mds_client.c
+++ b/fs/ceph/mds_client.c
@@ -2267,7 +2267,7 @@ static int trim_caps_cb(struct inode *inode, int mds, void *arg)
 			int count;
 			dput(dentry);
 			d_prune_aliases(inode);
-			count = icount_read(inode);
+			count = icount_read_once(inode);
 			if (count == 1)
 				(*remaining)--;
 			doutc(cl, "%p %llx.%llx cap %p pruned, count now %d\n",
diff --git a/fs/coredump.c b/fs/coredump.c
index bb6fdb1f458e9..c0c919621a7af 100644
--- a/fs/coredump.c
+++ b/fs/coredump.c
@@ -1488,7 +1488,8 @@ static int proc_dostring_coredump(const struct ctl_table *table, int write,
 		return -EINVAL;
 	}
 
-	validate_coredump_safety();
+	if (strncmp(old_core_pattern, core_pattern, CORENAME_MAX_SIZE))
+		validate_coredump_safety();
 	return error;
 }
 
diff --git a/fs/cramfs/README b/fs/cramfs/README
index 778df5c4d70bb..c0052a11aa28a 100644
--- a/fs/cramfs/README
+++ b/fs/cramfs/README
@@ -104,94 +104,4 @@ Tools
 -----
 
 The cramfs user-space tools, including mkcramfs and cramfsck, are
-located at <http://sourceforge.net/projects/cramfs/>.
-
-
-Future Development
-==================
-
-Block Size
-----------
-
-(Block size in cramfs refers to the size of input data that is
-compressed at a time.  It's intended to be somewhere around
-PAGE_SIZE for cramfs_read_folio's convenience.)
-
-The superblock ought to indicate the block size that the fs was
-written for, since comments in <linux/pagemap.h> indicate that
-PAGE_SIZE may grow in future (if I interpret the comment
-correctly).
-
-Currently, mkcramfs #define's PAGE_SIZE as 4096 and uses that
-for blksize, whereas Linux-2.3.39 uses its PAGE_SIZE, which in
-turn is defined as PAGE_SIZE (which can be as large as 32KB on arm).
-This discrepancy is a bug, though it's not clear which should be
-changed.
-
-One option is to change mkcramfs to take its PAGE_SIZE from
-<asm/page.h>.  Personally I don't like this option, but it does
-require the least amount of change: just change `#define
-PAGE_SIZE (4096)' to `#include <asm/page.h>'.  The disadvantage
-is that the generated cramfs cannot always be shared between different
-kernels, not even necessarily kernels of the same architecture if
-PAGE_SIZE is subject to change between kernel versions
-(currently possible with arm and ia64).
-
-The remaining options try to make cramfs more sharable.
-
-One part of that is addressing endianness.  The two options here are
-`always use little-endian' (like ext2fs) or `writer chooses
-endianness; kernel adapts at runtime'.  Little-endian wins because of
-code simplicity and little CPU overhead even on big-endian machines.
-
-The cost of swabbing is changing the code to use the le32_to_cpu
-etc. macros as used by ext2fs.  We don't need to swab the compressed
-data, only the superblock, inodes and block pointers.
-
-
-The other part of making cramfs more sharable is choosing a block
-size.  The options are:
-
-  1. Always 4096 bytes.
-
-  2. Writer chooses blocksize; kernel adapts but rejects blocksize >
-     PAGE_SIZE.
-
-  3. Writer chooses blocksize; kernel adapts even to blocksize >
-     PAGE_SIZE.
-
-It's easy enough to change the kernel to use a smaller value than
-PAGE_SIZE: just make cramfs_read_folio read multiple blocks.
-
-The cost of option 1 is that kernels with a larger PAGE_SIZE
-value don't get as good compression as they can.
-
-The cost of option 2 relative to option 1 is that the code uses
-variables instead of #define'd constants.  The gain is that people
-with kernels having larger PAGE_SIZE can make use of that if
-they don't mind their cramfs being inaccessible to kernels with
-smaller PAGE_SIZE values.
-
-Option 3 is easy to implement if we don't mind being CPU-inefficient:
-e.g. get read_folio to decompress to a buffer of size MAX_BLKSIZE (which
-must be no larger than 32KB) and discard what it doesn't need.
-Getting read_folio to read into all the covered pages is harder.
-
-The main advantage of option 3 over 1, 2, is better compression.  The
-cost is greater complexity.  Probably not worth it, but I hope someone
-will disagree.  (If it is implemented, then I'll re-use that code in
-e2compr.)
-
-
-Another cost of 2 and 3 over 1 is making mkcramfs use a different
-block size, but that just means adding and parsing a -b option.
-
-
-Inode Size
-----------
-
-Given that cramfs will probably be used for CDs etc. as well as just
-silicon ROMs, it might make sense to expand the inode a little from
-its current 12 bytes.  Inodes other than the root inode are followed
-by filename, so the expansion doesn't even have to be a multiple of 4
-bytes.
+located at <https://github.com/npitre/cramfs-tools>.
diff --git a/fs/dcache.c b/fs/dcache.c
index 2c61aeea41f45..d6f5053132053 100644
--- a/fs/dcache.c
+++ b/fs/dcache.c
@@ -426,9 +426,16 @@ static inline void __d_clear_type_and_inode(struct dentry *dentry)
 		this_cpu_inc(nr_dentry_negative);
 }
 
+#define DENTRY_WARN_ONCE(condition, dentry) \
+	WARN_ONCE((condition), "dentry=%p d_flags=0x%x\n", (dentry), (dentry)->d_flags)
+#define D_FLAG_VERIFY(dentry, x) \
+	DENTRY_WARN_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x), (dentry))
+
 static void dentry_free(struct dentry *dentry)
 {
-	WARN_ON(d_really_is_positive(dentry));
+	DENTRY_WARN_ONCE(d_really_is_positive(dentry), dentry);
+	DENTRY_WARN_ONCE(dentry->d_lockref.count >= 0, dentry);
+	D_FLAG_VERIFY(dentry, 0);
 	if (unlikely(dname_external(dentry))) {
 		struct external_name *p = external_name(dentry);
 		if (likely(atomic_dec_and_test(&p->count))) {
@@ -495,7 +502,6 @@ static void dentry_unlink_inode(struct dentry * dentry)
  * These helper functions make sure we always follow the
  * rules. d_lock must be held by the caller.
  */
-#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
 static void d_lru_add(struct dentry *dentry)
 {
 	D_FLAG_VERIFY(dentry, 0);
@@ -1820,10 +1826,10 @@ static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
 		name = &slash_name;
 		dname = dentry->d_shortname.string;
 	} else if (name->len > DNAME_INLINE_LEN-1) {
-		size_t size = offsetof(struct external_name, name[1]);
-		struct external_name *p = kmalloc(size + name->len,
-						  GFP_KERNEL_ACCOUNT |
-						  __GFP_RECLAIMABLE);
+		struct external_name *p;
+
+		p = kmalloc_flex(*p, name, name->len + 1,
+				 GFP_KERNEL_ACCOUNT | __GFP_RECLAIMABLE);
 		if (!p) {
 			kmem_cache_free(dentry_cache, dentry); 
 			return NULL;
@@ -2100,6 +2106,10 @@ void d_instantiate_new(struct dentry *entry, struct inode *inode)
 	__d_instantiate(entry, inode);
 	spin_unlock(&entry->d_lock);
 	WARN_ON(!(inode_state_read(inode) & I_NEW));
+	/*
+	 * Paired with igrab_from_hash()
+	 */
+	smp_wmb();
 	inode_state_clear(inode, I_NEW | I_CREATING);
 	inode_wake_up_bit(inode, __I_NEW);
 	spin_unlock(&inode->i_lock);
diff --git a/fs/dlm/lowcomms.c b/fs/dlm/lowcomms.c
index 5b61427879195..2aff1c7c17de4 100644
--- a/fs/dlm/lowcomms.c
+++ b/fs/dlm/lowcomms.c
@@ -271,7 +271,8 @@ static struct connection *__find_con(int nodeid, int r)
 {
 	struct connection *con;
 
-	hlist_for_each_entry_rcu(con, &connection_hash[r], list) {
+	hlist_for_each_entry_srcu(con, &connection_hash[r], list,
+			srcu_read_lock_held(&connections_srcu)) {
 		if (con->nodeid == nodeid)
 			return con;
 	}
@@ -426,7 +427,8 @@ static int addr_to_nodeid(struct sockaddr_storage *addr, int *nodeid,
 
 	idx = srcu_read_lock(&connections_srcu);
 	for (i = 0; i < CONN_HASH_SIZE; i++) {
-		hlist_for_each_entry_rcu(con, &connection_hash[i], list) {
+		hlist_for_each_entry_srcu(con, &connection_hash[i], list,
+				srcu_read_lock_held(&connections_srcu)) {
 			WARN_ON_ONCE(!con->addr_count);
 
 			spin_lock(&con->addrs_lock);
@@ -1729,7 +1731,8 @@ void dlm_lowcomms_shutdown(void)
 
 	idx = srcu_read_lock(&connections_srcu);
 	for (i = 0; i < CONN_HASH_SIZE; i++) {
-		hlist_for_each_entry_rcu(con, &connection_hash[i], list) {
+		hlist_for_each_entry_srcu(con, &connection_hash[i], list,
+				srcu_read_lock_held(&connections_srcu)) {
 			shutdown_connection(con, true);
 			stop_connection_io(con);
 			flush_workqueue(process_workqueue);
@@ -1968,7 +1971,8 @@ void dlm_lowcomms_exit(void)
 
 	idx = srcu_read_lock(&connections_srcu);
 	for (i = 0; i < CONN_HASH_SIZE; i++) {
-		hlist_for_each_entry_rcu(con, &connection_hash[i], list) {
+		hlist_for_each_entry_srcu(con, &connection_hash[i], list,
+				srcu_read_lock_held(&connections_srcu)) {
 			spin_lock(&connections_lock);
 			hlist_del_rcu(&con->list);
 			spin_unlock(&connections_lock);
diff --git a/fs/dlm/memory.c b/fs/dlm/memory.c
index 5c35cc67aca4c..ee55994ce90d8 100644
--- a/fs/dlm/memory.c
+++ b/fs/dlm/memory.c
@@ -48,8 +48,10 @@ int __init dlm_memory_init(void)
 	if (!rsb_cache)
 		goto rsb;
 
-	cb_cache = kmem_cache_create("dlm_cb", sizeof(struct dlm_callback),
+	cb_cache = kmem_cache_create_usercopy("dlm_cb", sizeof(struct dlm_callback),
 				     __alignof__(struct dlm_callback), 0,
+					 offsetof(struct dlm_callback, lvbptr),
+					 sizeof_field(struct dlm_callback, lvbptr),
 				     NULL);
 	if (!cb_cache)
 		goto cb;
diff --git a/fs/dlm/midcomms.c b/fs/dlm/midcomms.c
index d54bdd8fc4f2e..8964164600d2d 100644
--- a/fs/dlm/midcomms.c
+++ b/fs/dlm/midcomms.c
@@ -275,7 +275,8 @@ static struct midcomms_node *__find_node(int nodeid, int r)
 {
 	struct midcomms_node *node;
 
-	hlist_for_each_entry_rcu(node, &node_hash[r], hlist) {
+	hlist_for_each_entry_srcu(node, &node_hash[r], hlist,
+			srcu_read_lock_held(&nodes_srcu)) {
 		if (node->nodeid == nodeid)
 			return node;
 	}
@@ -355,6 +356,7 @@ int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr)
 	if (!node)
 		return -ENOMEM;
 
+	node->debugfs = dlm_create_debug_comms_file(nodeid, node);
 	node->nodeid = nodeid;
 	spin_lock_init(&node->state_lock);
 	spin_lock_init(&node->send_queue_lock);
@@ -368,7 +370,6 @@ int dlm_midcomms_addr(int nodeid, struct sockaddr_storage *addr)
 	hlist_add_head_rcu(&node->hlist, &node_hash[r]);
 	spin_unlock_bh(&nodes_lock);
 
-	node->debugfs = dlm_create_debug_comms_file(nodeid, node);
 	return 0;
 }
 
@@ -968,10 +969,10 @@ static void midcomms_new_msg_cb(void *data)
 	atomic_inc(&mh->node->send_queue_cnt);
 
 	spin_lock_bh(&mh->node->send_queue_lock);
+	/* need to be locked with list_add_tail_rcu() because list is ordered */
+	mh->seq = atomic_fetch_inc(&mh->node->seq_send);
 	list_add_tail_rcu(&mh->list, &mh->node->send_queue);
 	spin_unlock_bh(&mh->node->send_queue_lock);
-
-	mh->seq = atomic_fetch_inc(&mh->node->seq_send);
 }
 
 static struct dlm_msg *dlm_midcomms_get_msg_3_2(struct dlm_mhandle *mh, int nodeid,
@@ -1165,7 +1166,8 @@ void dlm_midcomms_exit(void)
 
 	idx = srcu_read_lock(&nodes_srcu);
 	for (i = 0; i < CONN_HASH_SIZE; i++) {
-		hlist_for_each_entry_rcu(node, &node_hash[i], hlist) {
+		hlist_for_each_entry_srcu(node, &node_hash[i], hlist,
+				srcu_read_lock_held(&nodes_srcu)) {
 			dlm_delete_debug_comms_file(node->debugfs);
 
 			spin_lock(&nodes_lock);
@@ -1325,7 +1327,8 @@ void dlm_midcomms_version_wait(void)
 
 	idx = srcu_read_lock(&nodes_srcu);
 	for (i = 0; i < CONN_HASH_SIZE; i++) {
-		hlist_for_each_entry_rcu(node, &node_hash[i], hlist) {
+		hlist_for_each_entry_srcu(node, &node_hash[i], hlist,
+				srcu_read_lock_held(&nodes_srcu)) {
 			ret = wait_event_timeout(node->shutdown_wait,
 						 node->version != DLM_VERSION_NOT_SET ||
 						 node->state == DLM_CLOSED ||
@@ -1396,7 +1399,8 @@ void dlm_midcomms_shutdown(void)
 	mutex_lock(&close_lock);
 	idx = srcu_read_lock(&nodes_srcu);
 	for (i = 0; i < CONN_HASH_SIZE; i++) {
-		hlist_for_each_entry_rcu(node, &node_hash[i], hlist) {
+		hlist_for_each_entry_srcu(node, &node_hash[i], hlist,
+				srcu_read_lock_held(&nodes_srcu)) {
 			midcomms_shutdown(node);
 		}
 	}
@@ -1404,7 +1408,8 @@ void dlm_midcomms_shutdown(void)
 	dlm_lowcomms_shutdown();
 
 	for (i = 0; i < CONN_HASH_SIZE; i++) {
-		hlist_for_each_entry_rcu(node, &node_hash[i], hlist) {
+		hlist_for_each_entry_srcu(node, &node_hash[i], hlist,
+				srcu_read_lock_held(&nodes_srcu)) {
 			midcomms_node_reset(node);
 		}
 	}
diff --git a/fs/ecryptfs/keystore.c b/fs/ecryptfs/keystore.c
index 0be746493e560..ebebc9551f1ff 100644
--- a/fs/ecryptfs/keystore.c
+++ b/fs/ecryptfs/keystore.c
@@ -1718,7 +1718,6 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat,
 			      struct dentry *ecryptfs_dentry)
 {
 	size_t i = 0;
-	size_t found_auth_tok;
 	size_t next_packet_is_auth_tok_packet;
 	LIST_HEAD(auth_tok_list);
 	struct ecryptfs_auth_tok *matching_auth_tok;
@@ -1822,7 +1821,6 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat,
 	 * the metadata. There may be several potential matches, but
 	 * just one will be sufficient to decrypt to get the FEK. */
 find_next_matching_auth_tok:
-	found_auth_tok = 0;
 	list_for_each_entry(auth_tok_list_item, &auth_tok_list, list) {
 		candidate_auth_tok = &auth_tok_list_item->auth_tok;
 		if (unlikely(ecryptfs_verbosity > 0)) {
@@ -1843,17 +1841,13 @@ int ecryptfs_parse_packet_set(struct ecryptfs_crypt_stat *crypt_stat,
 					       &matching_auth_tok,
 					       crypt_stat->mount_crypt_stat,
 					       candidate_auth_tok_sig);
-		if (!rc) {
-			found_auth_tok = 1;
+		if (!rc)
 			goto found_matching_auth_tok;
-		}
-	}
-	if (!found_auth_tok) {
-		ecryptfs_printk(KERN_ERR, "Could not find a usable "
-				"authentication token\n");
-		rc = -EIO;
-		goto out_wipe_list;
 	}
+	ecryptfs_printk(KERN_ERR,
+			"Could not find a usable authentication token\n");
+	rc = -EIO;
+	goto out_wipe_list;
 found_matching_auth_tok:
 	if (candidate_auth_tok->token_type == ECRYPTFS_PRIVATE_KEY) {
 		memcpy(&(candidate_auth_tok->token.private_key),
diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c
index 41e311019a251..df7ea019526d7 100644
--- a/fs/erofs/xattr.c
+++ b/fs/erofs/xattr.c
@@ -89,13 +89,11 @@ static int erofs_init_inode_xattrs(struct inode *inode)
 	    vi->xattr_isize - sizeof(struct erofs_xattr_ibody_header)) {
 		erofs_err(sb, "invalid h_shared_count %u @ nid %llu",
 			  vi->xattr_shared_count, vi->nid);
-		erofs_put_metabuf(&buf);
 		ret = -EFSCORRUPTED;
 		goto out_unlock;
 	}
 	vi->xattr_shared_xattrs = kmalloc_objs(uint, vi->xattr_shared_count);
 	if (!vi->xattr_shared_xattrs) {
-		erofs_put_metabuf(&buf);
 		ret = -ENOMEM;
 		goto out_unlock;
 	}
@@ -112,12 +110,12 @@ static int erofs_init_inode_xattrs(struct inode *inode)
 		}
 		vi->xattr_shared_xattrs[i] = le32_to_cpu(*xattr_id);
 	}
-	erofs_put_metabuf(&buf);
 
 	/* paired with smp_mb() at the beginning of the function. */
 	smp_mb();
 	set_bit(EROFS_I_EA_INITED_BIT, &vi->flags);
 out_unlock:
+	erofs_put_metabuf(&buf);
 	clear_and_wake_up_bit(EROFS_I_BL_XATTR_BIT, &vi->flags);
 	return ret;
 }
diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c
index 43bb5a6a9924b..27ab7bd844ec7 100644
--- a/fs/erofs/zdata.c
+++ b/fs/erofs/zdata.c
@@ -1509,8 +1509,15 @@ static void z_erofs_fill_bio_vec(struct bio_vec *bvec,
 	DBG_BUGON(z_erofs_is_shortlived_page(bvec->bv_page));
 
 	folio = page_folio(zbv.page);
-	/* For preallocated managed folios, add them to page cache here */
+	/*
+	 * Preallocated folios are added to the managed cache here rather than
+	 * in z_erofs_bind_cache() in order to keep these folios locked in
+	 * increasing (physical) address order.
+	 * Clear folio->private before these folios become visible to others in
+	 * the managed cache to avoid duplicate additions for unaligned extents.
+	 */
 	if (folio->private == Z_EROFS_PREALLOCATED_FOLIO) {
+		folio->private = NULL;
 		tocache = true;
 		goto out_tocache;
 	}
@@ -1546,14 +1553,8 @@ static void z_erofs_fill_bio_vec(struct bio_vec *bvec,
 			}
 			return;
 		}
-		/*
-		 * Already linked with another pcluster, which only appears in
-		 * crafted images by fuzzers for now.  But handle this anyway.
-		 */
-		tocache = false;	/* use temporary short-lived pages */
 	} else {
 		DBG_BUGON(1); /* referenced managed folios can't be truncated */
-		tocache = true;
 	}
 	folio_unlock(folio);
 	folio_put(folio);
diff --git a/fs/eventpoll.c b/fs/eventpoll.c
index a3090b446af10..a569e98d4a996 100644
--- a/fs/eventpoll.c
+++ b/fs/eventpoll.c
@@ -41,45 +41,170 @@
 #include <net/busy_poll.h>
 
 /*
- * LOCKING:
- * There are three level of locking required by epoll :
- *
- * 1) epnested_mutex (mutex)
- * 2) ep->mtx (mutex)
- * 3) ep->lock (spinlock)
- *
- * The acquire order is the one listed above, from 1 to 3.
- * We need a spinlock (ep->lock) because we manipulate objects
- * from inside the poll callback, that might be triggered from
- * a wake_up() that in turn might be called from IRQ context.
- * So we can't sleep inside the poll callback and hence we need
- * a spinlock. During the event transfer loop (from kernel to
- * user space) we could end up sleeping due a copy_to_user(), so
- * we need a lock that will allow us to sleep. This lock is a
- * mutex (ep->mtx). It is acquired during the event transfer loop,
- * during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file().
- * The epnested_mutex is acquired when inserting an epoll fd onto another
- * epoll fd. We do this so that we walk the epoll tree and ensure that this
- * insertion does not create a cycle of epoll file descriptors, which
- * could lead to deadlock. We need a global mutex to prevent two
- * simultaneous inserts (A into B and B into A) from racing and
- * constructing a cycle without either insert observing that it is
- * going to.
- * It is necessary to acquire multiple "ep->mtx"es at once in the
- * case when one epoll fd is added to another. In this case, we
- * always acquire the locks in the order of nesting (i.e. after
- * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired
- * before e2->mtx). Since we disallow cycles of epoll file
- * descriptors, this ensures that the mutexes are well-ordered. In
- * order to communicate this nesting to lockdep, when walking a tree
- * of epoll file descriptors, we use the current recursion depth as
- * the lockdep subkey.
- * It is possible to drop the "ep->mtx" and to use the global
- * mutex "epnested_mutex" (together with "ep->lock") to have it working,
- * but having "ep->mtx" will make the interface more scalable.
- * Events that require holding "epnested_mutex" are very rare, while for
- * normal operations the epoll private "ep->mtx" will guarantee
- * a better scalability.
+ * fs/eventpoll.c - Efficient event polling ("epoll") kernel implementation.
+ *
+ *
+ * Overview
+ * --------
+ *
+ * Each epoll_create(2) returns an anonymous [eventpoll] file whose
+ * ->private_data is a struct eventpoll. Each EPOLL_CTL_ADD installs
+ * a struct epitem linking one (watched file, fd) pair back to that
+ * eventpoll via the watched file's f_op->poll() wait queue(s). When
+ * the watched file signals readiness, ep_poll_callback() fires and
+ * marks the epitem ready. epoll_wait(2) drains the ready list under
+ * ep->mtx, re-queueing items in level-triggered mode.
+ *
+ * epoll instances can watch other epoll instances up to EP_MAX_NESTS
+ * deep; cycles are forbidden and detected at EPOLL_CTL_ADD time.
+ *
+ *
+ * Locking
+ * -------
+ *
+ * Three levels, acquired from outer to inner:
+ *
+ *   epnested_mutex   (global; rare; taken only for EPOLL_CTL_ADD
+ *                     loop / path checks)
+ *     > ep->mtx     (per-eventpoll; sleepable; serializes most ops)
+ *       > ep->lock  (per-eventpoll; IRQ-safe spinlock)
+ *
+ *   file->f_lock    (per-file; NOT IRQ-safe; guards f_ep hlist ops;
+ *                    nested inside ep->mtx, outside ep->lock)
+ *
+ * Rationale:
+ *   - ep->lock is a spinlock because ep_poll_callback() is called from
+ *     wake_up() which may run in hard-IRQ context. All ep->lock
+ *     critical sections use spin_lock_irqsave().
+ *   - ep->mtx is a sleepable mutex because the event delivery loop
+ *     calls copy_to_user(), and ep_insert() may sleep in
+ *     kmem_cache_alloc() and f_op->poll().
+ *   - epnested_mutex is global because cycle detection needs a global
+ *     view of the epoll topology; a per-object scheme would let two
+ *     concurrent inserts (A into B, B into A) construct a cycle
+ *     without either observer seeing it.
+ *   - Per-ep ep->mtx is preferred for scalability elsewhere. Events
+ *     that require epnested_mutex are rare.
+ *
+ * When EPOLL_CTL_ADD nests one eventpoll inside another we acquire
+ * ep->mtx on both: outer first, target second. Since cycles are
+ * forbidden the set of live ep->mtx holds is always a strict chain,
+ * communicated to lockdep via mutex_lock_nested() subclasses derived
+ * from the current recursion depth.
+ *
+ *
+ * Field protection
+ * ----------------
+ *
+ * struct eventpoll:
+ *   mtx              - self
+ *   rbr              - ep->mtx
+ *   ovflist, rdllist - ep->lock (IRQ-safe)
+ *   wq               - ep->lock for queue mutation
+ *   poll_wait        - internal waitqueue spinlock
+ *   refs             - file->f_lock for adds; ep->mtx for removes;
+ *                      RCU for readers (hlist_del_rcu + kfree_rcu(ep))
+ *   ws               - ep->mtx
+ *   gen, loop_check_depth - epnested_mutex
+ *   file, user       - immutable after setup
+ *   refcount         - atomic (refcount_t)
+ *   napi_*           - READ_ONCE / WRITE_ONCE
+ *
+ * struct epitem:
+ *   rbn / rcu union  - rbn: ep->mtx (while epi is linked in ep->rbr).
+ *                      rcu: written only by kfree_rcu(epi) on the free
+ *                      path; otherwise untouched by epoll code.
+ *   rdllink, next    - ep->lock
+ *   ffd, ep          - immutable after ep_insert()
+ *   pwqlist          - ep->mtx for writes; POLLFREE clears pwq->whead
+ *                      via smp_store_release(), see below
+ *   fllink           - file->f_lock for mutation; hlist_del_rcu +
+ *                      kfree_rcu(epi) for safe RCU readers
+ *   ws               - RCU (rcu_assign_pointer /
+ *                      rcu_dereference_check(mtx))
+ *   event            - ep->mtx for writes; lockless read in
+ *                      ep_poll_callback pairs with smp_mb() in
+ *                      ep_modify()
+ *
+ *
+ * Ready-list state machine
+ * ------------------------
+ *
+ * Readiness is tracked in two lists under ep->lock:
+ *
+ *   rdllist   - doubly-linked FIFO; the "current" ready list.
+ *   ovflist   - singly-linked LIFO; used during a scan to catch
+ *               events that arrive while rdllist is being iterated
+ *               without ep->lock.
+ *
+ * Encoded in ep->ovflist:
+ *   EP_UNACTIVE_PTR - no scan active; callback appends to rdllist.
+ *   NULL            - scan active, no spill yet.
+ *   pointer to epi  - scan active with spilled items (LIFO).
+ *
+ * Encoded in epi->ovflist_next:
+ *   EP_UNACTIVE_PTR - epi is not on ovflist.
+ *   otherwise       - next epi on ovflist (NULL at tail).
+ *
+ * ep_start_scan() flips "not scanning" to "scanning" and splices
+ * rdllist into a caller-local scan_batch. ep_done_scan() drains ovflist
+ * back to rdllist (list_add head-insert reverses LIFO to FIFO),
+ * flips back to "not scanning", and re-splices any items the caller
+ * left in scan_batch (e.g., level-triggered re-queues).
+ *
+ *
+ * Removal paths
+ * -------------
+ *
+ * Three paths dispose of epitems and/or eventpolls:
+ *
+ *   A. ep_remove()              - EPOLL_CTL_DEL and ep_insert()
+ *                                 rollback. Caller holds ep->mtx.
+ *   B. ep_clear_and_put()       - close of the epoll fd itself
+ *                                 (ep_eventpoll_release).
+ *   C. eventpoll_release_file() - close of a watched file, invoked
+ *                                 from __fput().
+ *
+ * Coordination:
+ *   A and C exclude each other via the watched file's refcount.
+ *   A pins the file with epi_fget() before touching file->f_ep or
+ *   file->f_lock; if the pin fails, __fput() is in flight and C
+ *   will clean this epi up. See the epi_fget() block comment.
+ *   A and B both hold ep->mtx serially. B walks the rbtree with
+ *   rb_next() captured before ep_remove() erases the current node.
+ *   B and C both take ep->mtx; the loser sees fewer entries or an
+ *   empty file->f_ep.
+ *
+ * Within every path the internal order is strict:
+ *   ep_unregister_pollwait()  - drain pwqlist; synchronizes with any
+ *                                in-flight ep_poll_callback via the
+ *                                watched wait-queue head's lock.
+ *   ep_remove_file()          - hlist_del_rcu of epi->fllink and,
+ *                                if last watcher, clear file->f_ep,
+ *                                under file->f_lock.
+ *   ep_remove_epi()           - rb_erase, rdllist unlink (ep->lock),
+ *                                wakeup_source_unregister,
+ *                                kfree_rcu(epi).
+ *
+ * kfree_rcu(epi) defers the free past RCU readers in
+ * reverse_path_check_proc(); kfree_rcu(ep) defers past readers in
+ * ep_get_upwards_depth_proc().
+ *
+ *
+ * POLLFREE handshake
+ * ------------------
+ *
+ * When a subsystem tears down a wait-queue head that an epitem is
+ * registered on (binder, signalfd, ...), it wakes the callback with
+ * POLLFREE and must RCU-defer the head's free. The store/load pair:
+ *
+ *   ep_poll_callback() POLLFREE branch:
+ *     smp_store_release(&pwq->whead, NULL)
+ *
+ *   ep_remove_wait_queue():
+ *     smp_load_acquire(&pwq->whead)
+ *
+ * See those sites for the full argument.
  */
 
 /* Epoll private bits inside the event mask */
@@ -99,11 +224,6 @@
 
 #define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry))
 
-struct epoll_filefd {
-	struct file *file;
-	int fd;
-} __packed;
-
 /* Wait structure used by the poll hooks */
 struct eppoll_entry {
 	/* List header used to link this structure to the "struct epitem" */
@@ -136,17 +256,19 @@ struct epitem {
 		struct rcu_head rcu;
 	};
 
-	/* List header used to link this structure to the eventpoll ready list */
+	/* Link on the owning eventpoll's ready list (ep->rdllist). */
 	struct list_head rdllink;
 
 	/*
-	 * Works together "struct eventpoll"->ovflist in keeping the
-	 * single linked chain of items.
+	 * Link on the owning eventpoll's scan-overflow list (ep->ovflist),
+	 * EP_UNACTIVE_PTR when not linked. See epi_on_ovflist() /
+	 * epi_clear_ovflist() and the "Ready-list state machine" section
+	 * in the top-of-file banner.
 	 */
-	struct epitem *next;
+	struct epitem *ovflist_next;
 
 	/* The file descriptor information this item refers to */
-	struct epoll_filefd ffd;
+	struct epoll_key ffd;
 
 	/* List containing poll wait queues */
 	struct eppoll_entry *pwqlist;
@@ -247,13 +369,77 @@ struct ep_pqueue {
 /* Maximum number of epoll watched descriptors, per user */
 static long max_user_watches __read_mostly;
 
-/* Used for cycles detection */
+/*
+ * Cycle and path-length checks at EPOLL_CTL_ADD
+ * ---------------------------------------------
+ *
+ * When EPOLL_CTL_ADD creates a link that either targets an eventpoll
+ * file or extends an existing chain of eventpolls, two checks run:
+ *
+ *   1. no cycle is being formed -- ep_loop_check() walks downward
+ *      from the candidate target, and ep_get_upwards_depth_proc()
+ *      walks upward from the outer ep, both bounded by EP_MAX_NESTS.
+ *   2. no file accumulates more than path_limits[depth] wakeup paths
+ *      of a given length -- reverse_path_check().
+ *
+ * Both need a global view of the epoll topology and must be atomic
+ * with the insertion, so the check is serialized by epnested_mutex
+ * and carries its scratch state on a stack-allocated struct
+ * ep_ctl_ctx scoped to one do_epoll_ctl() call. Non-nested inserts
+ * skip this machinery entirely and take only ep->mtx.
+ *
+ *   epnested_mutex     Serializes the whole check.
+ *   loop_check_gen     Global monotonic stamp, bumped at the start of
+ *                      a check and again at the end. ep->gen caches
+ *                      the value under which ep was last visited by
+ *                      ep_loop_check_proc() or
+ *                      ep_get_upwards_depth_proc(); the post-check
+ *                      bump ensures those cached stamps can no longer
+ *                      equal loop_check_gen, so the
+ *                      "ep->gen == loop_check_gen" trigger in
+ *                      ep_ctl_lock() only fires while another check
+ *                      is in flight.
+ *
+ * struct ep_ctl_ctx carries the rest (inserting_into, tfile_check_list,
+ * path_count[]) through the walk; see its declaration below.
+ *
+ * Commits fdcfce93073d ("eventpoll: Fix integer overflow in
+ * ep_loop_check_proc()") and f2e467a48287 ("eventpoll: Fix
+ * semi-unbounded recursion") hardened the walk; any refactor must
+ * preserve both bail-outs.
+ */
 static DEFINE_MUTEX(epnested_mutex);
-
 static u64 loop_check_gen = 0;
 
-/* Used to check for epoll file descriptor inclusion loops */
-static struct eventpoll *inserting_into;
+#define PATH_ARR_SIZE 5
+
+/*
+ * Per-do_epoll_ctl() scratch for the loop / path checks. Allocated on
+ * the caller's stack; populated by ep_ctl_lock() and the downward
+ * walk; consumed by reverse_path_check(); released by ep_ctl_unlock().
+ * Only valid while the caller holds epnested_mutex.
+ */
+struct ep_ctl_ctx {
+	/*
+	 * Outer eventpoll for one ep_loop_check(); if the downward walk
+	 * reaches it the insert would form a cycle.
+	 */
+	struct eventpoll *inserting_into;
+
+	/*
+	 * Singly-linked list of epitems_head objects collected during
+	 * ep_loop_check_proc(), then walked by reverse_path_check().
+	 * NULL means empty.
+	 */
+	struct epitems_head *tfile_check_list;
+
+	/*
+	 * Per-depth wakeup-path tally used by reverse_path_check_proc();
+	 * reinitialized to zero at the start of each reverse_path_check()
+	 * iteration.
+	 */
+	int path_count[PATH_ARR_SIZE];
+};
 
 /* Slab cache used to allocate "struct epitem" */
 static struct kmem_cache *epi_cache __ro_after_init;
@@ -262,14 +448,15 @@ static struct kmem_cache *epi_cache __ro_after_init;
 static struct kmem_cache *pwq_cache __ro_after_init;
 
 /*
- * List of files with newly added links, where we may need to limit the number
- * of emanating paths. Protected by the epnested_mutex.
+ * Wrapper anchor for file->f_ep when the watched file is not itself an
+ * eventpoll; for the epoll-watches-epoll case, file->f_ep points at
+ * &watched_ep->refs directly. The ->next field threads
+ * ctx->tfile_check_list during one EPOLL_CTL_ADD path check.
  */
 struct epitems_head {
 	struct hlist_head epitems;
 	struct epitems_head *next;
 };
-static struct epitems_head *tfile_check_list = EP_UNACTIVE_PTR;
 
 static struct kmem_cache *ephead_cache __ro_after_init;
 
@@ -279,14 +466,14 @@ static inline void free_ephead(struct epitems_head *head)
 		kmem_cache_free(ephead_cache, head);
 }
 
-static void list_file(struct file *file)
+static void list_file(struct file *file, struct ep_ctl_ctx *ctx)
 {
 	struct epitems_head *head;
 
 	head = container_of(file->f_ep, struct epitems_head, epitems);
 	if (!head->next) {
-		head->next = tfile_check_list;
-		tfile_check_list = head;
+		head->next = ctx->tfile_check_list;
+		ctx->tfile_check_list = head;
 	}
 }
 
@@ -334,29 +521,20 @@ static void __init epoll_sysctls_init(void)
 
 static const struct file_operations eventpoll_fops;
 
-static inline int is_file_epoll(struct file *f)
+bool is_file_epoll(struct file *f)
 {
 	return f->f_op == &eventpoll_fops;
 }
 
-/* Setup the structure that is used as key for the RB tree */
-static inline void ep_set_ffd(struct epoll_filefd *ffd,
-			      struct file *file, int fd)
-{
-	ffd->file = file;
-	ffd->fd = fd;
-}
-
 /* Compare RB tree keys */
-static inline int ep_cmp_ffd(struct epoll_filefd *p1,
-			     struct epoll_filefd *p2)
+static inline int ep_cmp_ffd(struct epoll_key *p1, struct epoll_key *p2)
 {
 	return (p1->file > p2->file ? +1:
 	        (p1->file < p2->file ? -1 : p1->fd - p2->fd));
 }
 
-/* Tells us if the item is currently linked */
-static inline int ep_is_linked(struct epitem *epi)
+/* True iff @epi is on its owning ep's ready list. */
+static inline bool ep_is_linked(struct epitem *epi)
 {
 	return !list_empty(&epi->rdllink);
 }
@@ -372,18 +550,47 @@ static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p)
 	return container_of(p, struct eppoll_entry, wait)->base;
 }
 
-/**
- * ep_events_available - Checks if ready events might be available.
- *
- * @ep: Pointer to the eventpoll context.
- *
- * Return: a value different than %zero if ready events are available,
- *          or %zero otherwise.
+/*
+ * Ready-list / ovflist state (see "Ready-list state machine" in the
+ * top-of-file banner for the full state machine). EP_UNACTIVE_PTR is
+ * the sentinel; these wrappers name each transition and each test so
+ * call sites do not need to know the sentinel's value.
  */
-static inline int ep_events_available(struct eventpoll *ep)
+
+/* True iff @ep is between ep_enter_scan() and ep_exit_scan(). */
+static inline bool ep_is_scanning(struct eventpoll *ep)
 {
-	return !list_empty_careful(&ep->rdllist) ||
-		READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR;
+	return READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR;
+}
+
+/* Called by ep_start_scan(): divert ep_poll_callback() to ovflist. */
+static inline void ep_enter_scan(struct eventpoll *ep)
+{
+	WRITE_ONCE(ep->ovflist, NULL);
+}
+
+/* Called by ep_done_scan(): redirect ep_poll_callback() back to rdllist. */
+static inline void ep_exit_scan(struct eventpoll *ep)
+{
+	WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR);
+}
+
+/* True iff @epi is currently linked on its ep's ovflist. */
+static inline bool epi_on_ovflist(const struct epitem *epi)
+{
+	return epi->ovflist_next != EP_UNACTIVE_PTR;
+}
+
+/* Mark @epi as not on any ovflist (init and post-drain). */
+static inline void epi_clear_ovflist(struct epitem *epi)
+{
+	epi->ovflist_next = EP_UNACTIVE_PTR;
+}
+
+/* True iff @ep has ready events that epoll_wait() might harvest. */
+static inline bool ep_events_available(struct eventpoll *ep)
+{
+	return !list_empty_careful(&ep->rdllist) || ep_is_scanning(ep);
 }
 
 #ifdef CONFIG_NET_RX_BUSY_POLL
@@ -659,10 +866,15 @@ static void ep_remove_wait_queue(struct eppoll_entry *pwq)
 
 	rcu_read_lock();
 	/*
-	 * If it is cleared by POLLFREE, it should be rcu-safe.
-	 * If we read NULL we need a barrier paired with
-	 * smp_store_release() in ep_poll_callback(), otherwise
-	 * we rely on whead->lock.
+	 * POLLFREE handshake, acquire side; see "POLLFREE handshake"
+	 * at the top of this file.
+	 *
+	 * A NULL load is paired with the smp_store_release(&whead, NULL)
+	 * in ep_poll_callback()'s POLLFREE branch: the teardown is
+	 * complete and we must not touch whead again. On a non-NULL load
+	 * rcu_read_lock() keeps the waitqueue memory alive (POLLFREE
+	 * firers RCU-defer the free) and whead->lock inside
+	 * remove_wait_queue() serializes us against the store side.
 	 */
 	whead = smp_load_acquire(&pwq->whead);
 	if (whead)
@@ -723,7 +935,7 @@ static inline void ep_pm_stay_awake_rcu(struct epitem *epi)
  * ep->mutex needs to be held because we could be hit by
  * eventpoll_release_file() and epoll_ctl().
  */
-static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist)
+static void ep_start_scan(struct eventpoll *ep, struct list_head *scan_batch)
 {
 	/*
 	 * Steal the ready list, and re-init the original one to the
@@ -735,13 +947,13 @@ static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist)
 	 */
 	lockdep_assert_irqs_enabled();
 	spin_lock_irq(&ep->lock);
-	list_splice_init(&ep->rdllist, txlist);
-	WRITE_ONCE(ep->ovflist, NULL);
+	list_splice_init(&ep->rdllist, scan_batch);
+	ep_enter_scan(ep);
 	spin_unlock_irq(&ep->lock);
 }
 
 static void ep_done_scan(struct eventpoll *ep,
-			 struct list_head *txlist)
+			 struct list_head *scan_batch)
 {
 	struct epitem *epi, *nepi;
 
@@ -751,34 +963,29 @@ static void ep_done_scan(struct eventpoll *ep,
 	 * other events might have been queued by the poll callback.
 	 * We re-insert them inside the main ready-list here.
 	 */
-	for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;
-	     nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {
+	for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL; ) {
+		nepi = epi->ovflist_next;
+		epi_clear_ovflist(epi);
 		/*
-		 * We need to check if the item is already in the list.
-		 * During the "sproc" callback execution time, items are
-		 * queued into ->ovflist but the "txlist" might already
-		 * contain them, and the list_splice() below takes care of them.
+		 * Skip items that the caller already returned via @scan_batch
+		 * -- the list_splice() below takes care of those.
 		 */
 		if (!ep_is_linked(epi)) {
 			/*
-			 * ->ovflist is LIFO, so we have to reverse it in order
-			 * to keep in FIFO.
+			 * ovflist is LIFO; list_add() head-insert here
+			 * reverses the iteration order into FIFO.
 			 */
 			list_add(&epi->rdllink, &ep->rdllist);
 			ep_pm_stay_awake(epi);
 		}
 	}
-	/*
-	 * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after
-	 * releasing the lock, events will be queued in the normal way inside
-	 * ep->rdllist.
-	 */
-	WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR);
+	/* Back out of scan mode; callbacks target ep->rdllist again. */
+	ep_exit_scan(ep);
 
 	/*
-	 * Quickly re-inject items left on "txlist".
+	 * Quickly re-inject items left on "scan_batch".
 	 */
-	list_splice(txlist, &ep->rdllist);
+	list_splice(scan_batch, &ep->rdllist);
 	__pm_relax(ep->ws);
 
 	if (!list_empty(&ep->rdllist)) {
@@ -795,9 +1002,10 @@ static void ep_get(struct eventpoll *ep)
 }
 
 /*
- * Returns true if the event poll can be disposed
+ * Drop a reference to @ep; returns true iff it was the last, in which
+ * case the caller is responsible for ep_free().
  */
-static bool ep_refcount_dec_and_test(struct eventpoll *ep)
+static bool ep_put(struct eventpoll *ep)
 {
 	if (!refcount_dec_and_test(&ep->refcount))
 		return false;
@@ -817,22 +1025,23 @@ static void ep_free(struct eventpoll *ep)
 }
 
 /*
- * The ffd.file pointer may be in the process of being torn down due to
- * being closed, but we may not have finished eventpoll_release() yet.
- *
- * Normally, even with the atomic_long_inc_not_zero, the file may have
- * been free'd and then gotten re-allocated to something else (since
- * files are not RCU-delayed, they are SLAB_TYPESAFE_BY_RCU).
- *
- * But for epoll, users hold the ep->mtx mutex, and as such any file in
- * the process of being free'd will block in eventpoll_release_file()
- * and thus the underlying file allocation will not be free'd, and the
- * file re-use cannot happen.
- *
- * For the same reason we can avoid a rcu_read_lock() around the
- * operation - 'ffd.file' cannot go away even if the refcount has
- * reached zero (but we must still not call out to ->poll() functions
- * etc).
+ * Pin @epi->ffd.file for operations that require both safe dereference
+ * and exclusion from __fput().
+ *
+ * struct file uses SLAB_TYPESAFE_BY_RCU, so a freed slot can be
+ * reassigned at any time. The bare load of epi->ffd.file is safe here
+ * because the caller holds ep->mtx and eventpoll_release_file() blocks
+ * on that mutex while tearing down the epi, so the backing file
+ * allocation cannot be freed and reused under us. An rcu_read_lock()
+ * is therefore unnecessary for the load.
+ *
+ * A successful file_ref_get() additionally blocks __fput() from
+ * starting on this file: once the refcount has reached zero it cannot
+ * come back. ep_remove() relies on that to touch file->f_lock and
+ * file->f_ep without racing eventpoll_release_file() (see commit
+ * a6dc643c6931). A NULL return means __fput() is already in flight;
+ * the caller must bail without touching the file, and
+ * eventpoll_release_file() will clean the epi up from its side.
  */
 static struct file *epi_fget(const struct epitem *epi)
 {
@@ -858,7 +1067,13 @@ static void ep_remove_file(struct eventpoll *ep, struct epitem *epi,
 	spin_lock(&file->f_lock);
 	head = file->f_ep;
 	if (hlist_is_singular_node(&epi->fllink, head)) {
-		/* See eventpoll_release() for details. */
+		/*
+		 * Last watcher: publish NULL so the eventpoll_release()
+		 * fastpath in include/linux/eventpoll.h can skip the slow
+		 * path on a future __fput(). Safe because every f_ep writer
+		 * either holds a pin on @file via epi_fget() or is __fput()
+		 * itself -- see the comment in eventpoll_release().
+		 */
 		WRITE_ONCE(file->f_ep, NULL);
 		if (!is_file_epoll(file)) {
 			struct epitems_head *v;
@@ -919,47 +1134,82 @@ static void ep_remove(struct eventpoll *ep, struct epitem *epi)
 
 	ep_remove_file(ep, epi, file);
 	ep_remove_epi(ep, epi);
-	WARN_ON_ONCE(ep_refcount_dec_and_test(ep));
+	WARN_ON_ONCE(ep_put(ep));
 }
 
-static void ep_clear_and_put(struct eventpoll *ep)
+/*
+ * Pass 1 of ep_clear_and_put(): drain every epi's pwqlist.
+ * ep_unregister_pollwait() takes each watched wait-queue head's lock,
+ * which synchronizes with any in-flight ep_poll_callback(); after
+ * this returns no callback can still be about to dereference an epi
+ * on this ep. Must strictly precede ep_drain_tree() -- fusing the
+ * two walks would let a callback queued on epi_i still fire after
+ * epi_{i+k} had already been freed.
+ */
+static void ep_drain_pollwaits(struct eventpoll *ep)
 {
-	struct rb_node *rbp, *next;
+	struct rb_node *rbp;
 	struct epitem *epi;
 
-	/* We need to release all tasks waiting for these file */
-	if (waitqueue_active(&ep->poll_wait))
-		ep_poll_safewake(ep, NULL, 0);
-
-	mutex_lock(&ep->mtx);
+	lockdep_assert_held(&ep->mtx);
 
-	/*
-	 * Walks through the whole tree by unregistering poll callbacks.
-	 */
 	for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
 		epi = rb_entry(rbp, struct epitem, rbn);
 
 		ep_unregister_pollwait(ep, epi);
 		cond_resched();
 	}
+}
+
+/*
+ * Pass 2 of ep_clear_and_put(): ep_remove() every epi. The per-epi
+ * pwqlist is already empty (ep_drain_pollwaits ran), but the rest of
+ * ep_remove() still runs: epi_fget() pin, f_ep clear under f_lock,
+ * rbtree erase, rdllist unlink, kfree_rcu(epi). rb_next() is captured
+ * before each erase so the iteration is stable.
+ *
+ * A concurrent eventpoll_release_file() (removal path C) on a watched
+ * file serializes with us via ep->mtx; ep_remove() transparently
+ * hands off any epi whose file is in __fput() by bailing when
+ * epi_fget() returns NULL, and path C will clean that epi up.
+ */
+static void ep_drain_tree(struct eventpoll *ep)
+{
+	struct rb_node *rbp, *next;
+	struct epitem *epi;
+
+	lockdep_assert_held(&ep->mtx);
 
-	/*
-	 * Walks through the whole tree and try to free each "struct epitem".
-	 * Note that ep_remove() will not remove the epitem in case of a
-	 * racing eventpoll_release_file(); the latter will do the removal.
-	 * At this point we are sure no poll callbacks will be lingering around.
-	 * Since we still own a reference to the eventpoll struct, the loop can't
-	 * dispose it.
-	 */
 	for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = next) {
 		next = rb_next(rbp);
 		epi = rb_entry(rbp, struct epitem, rbn);
 		ep_remove(ep, epi);
 		cond_resched();
 	}
+}
+
+/*
+ * Removal path B (see "Removal paths" in the top-of-file banner):
+ * close of the epoll fd itself, reached via ep_eventpoll_release().
+ *
+ * Two passes under ep->mtx: first ep_drain_pollwaits() quiesces
+ * in-flight callbacks, then ep_drain_tree() frees the epis. The
+ * ep->refcount is kept > 0 across the walk by the ep file's own
+ * share, which we drop below; ep_free() runs iff we were the last
+ * holder after the tree drained.
+ */
+static void ep_clear_and_put(struct eventpoll *ep)
+{
+	/* Release any threads blocked in poll-on-ep. */
+	if (waitqueue_active(&ep->poll_wait))
+		ep_poll_safewake(ep, NULL, 0);
 
+	mutex_lock(&ep->mtx);
+	ep_drain_pollwaits(ep);
+	ep_drain_tree(ep);
 	mutex_unlock(&ep->mtx);
-	if (ep_refcount_dec_and_test(ep))
+
+	if (ep_put(ep))
 		ep_free(ep);
 }
 
@@ -999,7 +1249,7 @@ static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, int depth
 static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int depth)
 {
 	struct eventpoll *ep = file->private_data;
-	LIST_HEAD(txlist);
+	LIST_HEAD(scan_batch);
 	struct epitem *epi, *tmp;
 	poll_table pt;
 	__poll_t res = 0;
@@ -1014,8 +1264,8 @@ static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int dep
 	 * the ready list.
 	 */
 	mutex_lock_nested(&ep->mtx, depth);
-	ep_start_scan(ep, &txlist);
-	list_for_each_entry_safe(epi, tmp, &txlist, rdllink) {
+	ep_start_scan(ep, &scan_batch);
+	list_for_each_entry_safe(epi, tmp, &scan_batch, rdllink) {
 		if (ep_item_poll(epi, &pt, depth + 1)) {
 			res = EPOLLIN | EPOLLRDNORM;
 			break;
@@ -1029,7 +1279,7 @@ static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int dep
 			list_del_init(&epi->rdllink);
 		}
 	}
-	ep_done_scan(ep, &txlist);
+	ep_done_scan(ep, &scan_batch);
 	mutex_unlock(&ep->mtx);
 	return res;
 }
@@ -1138,7 +1388,7 @@ void eventpoll_release_file(struct file *file)
 
 		mutex_unlock(&ep->mtx);
 
-		if (ep_refcount_dec_and_test(ep))
+		if (ep_put(ep))
 			ep_free(ep);
 		goto again;
 	}
@@ -1159,7 +1409,7 @@ static int ep_alloc(struct eventpoll **pep)
 	init_waitqueue_head(&ep->poll_wait);
 	INIT_LIST_HEAD(&ep->rdllist);
 	ep->rbr = RB_ROOT_CACHED;
-	ep->ovflist = EP_UNACTIVE_PTR;
+	ep->ovflist = EP_UNACTIVE_PTR;	/* not scanning */
 	ep->user = get_current_user();
 	refcount_set(&ep->refcount, 1);
 
@@ -1173,17 +1423,15 @@ static int ep_alloc(struct eventpoll **pep)
  * are protected by the "mtx" mutex, and ep_find() must be called with
  * "mtx" held.
  */
-static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)
+static struct epitem *ep_find(struct eventpoll *ep, struct epoll_key *tf)
 {
 	int kcmp;
 	struct rb_node *rbp;
 	struct epitem *epi, *epir = NULL;
-	struct epoll_filefd ffd;
 
-	ep_set_ffd(&ffd, file, fd);
 	for (rbp = ep->rbr.rb_root.rb_node; rbp; ) {
 		epi = rb_entry(rbp, struct epitem, rbn);
-		kcmp = ep_cmp_ffd(&ffd, &epi->ffd);
+		kcmp = ep_cmp_ffd(tf, &epi->ffd);
 		if (kcmp > 0)
 			rbp = rbp->rb_right;
 		else if (kcmp < 0)
@@ -1197,50 +1445,6 @@ static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)
 	return epir;
 }
 
-#ifdef CONFIG_KCMP
-static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff)
-{
-	struct rb_node *rbp;
-	struct epitem *epi;
-
-	for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
-		epi = rb_entry(rbp, struct epitem, rbn);
-		if (epi->ffd.fd == tfd) {
-			if (toff == 0)
-				return epi;
-			else
-				toff--;
-		}
-		cond_resched();
-	}
-
-	return NULL;
-}
-
-struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd,
-				     unsigned long toff)
-{
-	struct file *file_raw;
-	struct eventpoll *ep;
-	struct epitem *epi;
-
-	if (!is_file_epoll(file))
-		return ERR_PTR(-EINVAL);
-
-	ep = file->private_data;
-
-	mutex_lock(&ep->mtx);
-	epi = ep_find_tfd(ep, tfd, toff);
-	if (epi)
-		file_raw = epi->ffd.file;
-	else
-		file_raw = ERR_PTR(-ENOENT);
-	mutex_unlock(&ep->mtx);
-
-	return file_raw;
-}
-#endif /* CONFIG_KCMP */
-
 /*
  * This is the callback that is passed to the wait queue wakeup
  * mechanism. It is called by the stored file descriptors when they
@@ -1283,9 +1487,9 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v
 	 * semantics). All the events that happen during that period of time are
 	 * chained in ep->ovflist and requeued later on.
 	 */
-	if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) {
-		if (epi->next == EP_UNACTIVE_PTR) {
-			epi->next = READ_ONCE(ep->ovflist);
+	if (ep_is_scanning(ep)) {
+		if (!epi_on_ovflist(epi)) {
+			epi->ovflist_next = READ_ONCE(ep->ovflist);
 			WRITE_ONCE(ep->ovflist, epi);
 			ep_pm_stay_awake_rcu(epi);
 		}
@@ -1336,17 +1540,24 @@ static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, v
 
 	if (pollflags & POLLFREE) {
 		/*
-		 * If we race with ep_remove_wait_queue() it can miss
-		 * ->whead = NULL and do another remove_wait_queue() after
-		 * us, so we can't use __remove_wait_queue().
+		 * POLLFREE handshake, release side; see "POLLFREE handshake"
+		 * at the top of this file.
+		 *
+		 * Unlink our wait entry with list_del_init rather than
+		 * __remove_wait_queue: a concurrent ep_remove_wait_queue()
+		 * that already loaded a non-NULL whead may still call
+		 * remove_wait_queue() after us, and list_del_init() tolerates
+		 * the second delete.
+		 *
+		 * smp_store_release(&whead, NULL) publishes the teardown to
+		 * ep_remove_wait_queue()'s smp_load_acquire(). Before this
+		 * store, a racing ep_clear_and_put() / ep_remove() reaches
+		 * ep_remove_wait_queue() which sees whead != NULL and takes
+		 * whead->lock -- the same lock held by our caller, so it
+		 * serializes behind us. Once whead is zeroed, nothing else
+		 * protects ep / epi / wait.
 		 */
 		list_del_init(&wait->entry);
-		/*
-		 * ->whead != NULL protects us from the race with
-		 * ep_clear_and_put() or ep_remove(), ep_remove_wait_queue()
-		 * takes whead->lock held by the caller. Once we nullify it,
-		 * nothing protects ep/epi or even wait.
-		 */
 		smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL);
 	}
 
@@ -1407,41 +1618,40 @@ static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)
 
 
 
-#define PATH_ARR_SIZE 5
 /*
- * These are the number paths of length 1 to 5, that we are allowing to emanate
- * from a single file of interest. For example, we allow 1000 paths of length
- * 1, to emanate from each file of interest. This essentially represents the
- * potential wakeup paths, which need to be limited in order to avoid massive
- * uncontrolled wakeup storms. The common use case should be a single ep which
- * is connected to n file sources. In this case each file source has 1 path
- * of length 1. Thus, the numbers below should be more than sufficient. These
- * path limits are enforced during an EPOLL_CTL_ADD operation, since a modify
- * and delete can't add additional paths. Protected by the epnested_mutex.
+ * Upper bound on wakeup paths emanating from any one watched file,
+ * indexed by path depth (1..PATH_ARR_SIZE). For example, we allow
+ * 1000 paths of length 1 from each watched file. These caps limit
+ * the wakeup amplification that can be built from epoll-watches-
+ * epoll topologies without rejecting reasonable usage.
+ *
+ * Enforced at EPOLL_CTL_ADD; CTL_MOD and CTL_DEL cannot add paths.
+ * The running tallies live in ctx->path_count[] and are protected by
+ * epnested_mutex.
  */
 static const int path_limits[PATH_ARR_SIZE] = { 1000, 500, 100, 50, 10 };
-static int path_count[PATH_ARR_SIZE];
 
-static int path_count_inc(int nests)
+static int path_count_inc(struct ep_ctl_ctx *ctx, int nests)
 {
 	/* Allow an arbitrary number of depth 1 paths */
 	if (nests == 0)
 		return 0;
 
-	if (++path_count[nests] > path_limits[nests])
+	if (++ctx->path_count[nests] > path_limits[nests])
 		return -1;
 	return 0;
 }
 
-static void path_count_init(void)
+static void path_count_init(struct ep_ctl_ctx *ctx)
 {
 	int i;
 
 	for (i = 0; i < PATH_ARR_SIZE; i++)
-		path_count[i] = 0;
+		ctx->path_count[i] = 0;
 }
 
-static int reverse_path_check_proc(struct hlist_head *refs, int depth)
+static int reverse_path_check_proc(struct ep_ctl_ctx *ctx,
+				   struct hlist_head *refs, int depth)
 {
 	int error = 0;
 	struct epitem *epi;
@@ -1453,9 +1663,9 @@ static int reverse_path_check_proc(struct hlist_head *refs, int depth)
 	hlist_for_each_entry_rcu(epi, refs, fllink) {
 		struct hlist_head *refs = &epi->ep->refs;
 		if (hlist_empty(refs))
-			error = path_count_inc(depth);
+			error = path_count_inc(ctx, depth);
 		else
-			error = reverse_path_check_proc(refs, depth + 1);
+			error = reverse_path_check_proc(ctx, refs, depth + 1);
 		if (error != 0)
 			break;
 	}
@@ -1463,24 +1673,23 @@ static int reverse_path_check_proc(struct hlist_head *refs, int depth)
 }
 
 /**
- * reverse_path_check - The tfile_check_list is list of epitem_head, which have
- *                      links that are proposed to be newly added. We need to
- *                      make sure that those added links don't add too many
- *                      paths such that we will spend all our time waking up
- *                      eventpoll objects.
+ * reverse_path_check - ctx->tfile_check_list is a list of epitems_head
+ *                      anchoring files with newly proposed links; make
+ *                      sure those links don't push any path-length bucket
+ *                      over its limit in path_limits[].
  *
  * Return: %zero if the proposed links don't create too many paths,
  *	    %-1 otherwise.
  */
-static int reverse_path_check(void)
+static int reverse_path_check(struct ep_ctl_ctx *ctx)
 {
 	struct epitems_head *p;
 
-	for (p = tfile_check_list; p != EP_UNACTIVE_PTR; p = p->next) {
+	for (p = ctx->tfile_check_list; p; p = p->next) {
 		int error;
-		path_count_init();
+		path_count_init(ctx);
 		rcu_read_lock();
-		error = reverse_path_check_proc(&p->epitems, 0);
+		error = reverse_path_check_proc(ctx, &p->epitems, 0);
 		rcu_read_unlock();
 		if (error)
 			return error;
@@ -1526,7 +1735,7 @@ static noinline void ep_destroy_wakeup_source(struct epitem *epi)
 	wakeup_source_unregister(ws);
 }
 
-static int attach_epitem(struct file *file, struct epitem *epi)
+static int ep_attach_file(struct file *file, struct epitem *epi)
 {
 	struct epitems_head *to_free = NULL;
 	struct hlist_head *head = NULL;
@@ -1561,69 +1770,115 @@ static int attach_epitem(struct file *file, struct epitem *epi)
 }
 
 /*
- * Must be called with "mtx" held.
+ * Charge the user's epoll_watches quota, allocate a fresh epitem for
+ * @tf, and initialize its fields. The returned item is not yet linked
+ * into any data structure; the caller must install it via
+ * ep_register_epitem() (which takes over on success) or kmem_cache_free()
+ * it and decrement epoll_watches on its own.
+ *
+ * Returns ERR_PTR(-ENOSPC) if the quota is exceeded, ERR_PTR(-ENOMEM)
+ * if the slab allocation fails.
  */
-static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
-		     struct file *tfile, int fd, int full_check)
+static struct epitem *ep_alloc_epitem(struct eventpoll *ep,
+				      const struct epoll_event *event,
+				      struct epoll_key *tf)
 {
-	int error, pwake = 0;
-	__poll_t revents;
 	struct epitem *epi;
-	struct ep_pqueue epq;
-	struct eventpoll *tep = NULL;
-
-	if (is_file_epoll(tfile))
-		tep = tfile->private_data;
-
-	lockdep_assert_irqs_enabled();
 
 	if (unlikely(percpu_counter_compare(&ep->user->epoll_watches,
 					    max_user_watches) >= 0))
-		return -ENOSPC;
+		return ERR_PTR(-ENOSPC);
 	percpu_counter_inc(&ep->user->epoll_watches);
 
-	if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL))) {
+	epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL);
+	if (unlikely(!epi)) {
 		percpu_counter_dec(&ep->user->epoll_watches);
-		return -ENOMEM;
+		return ERR_PTR(-ENOMEM);
 	}
 
-	/* Item initialization follow here ... */
 	INIT_LIST_HEAD(&epi->rdllink);
 	epi->ep = ep;
-	ep_set_ffd(&epi->ffd, tfile, fd);
+	epi->ffd = *tf;
 	epi->event = *event;
-	epi->next = EP_UNACTIVE_PTR;
+	epi_clear_ovflist(epi);
+
+	return epi;
+}
+
+/*
+ * Install @epi into its target file's f_ep hlist and into @ep's rbtree,
+ * taking one additional reference on @ep for the lifetime of the item.
+ *
+ * If @tep is non-NULL, the target file is itself an eventpoll; we hold
+ * tep->mtx at subclass 1 across the attach + rbtree insert to serialize
+ * with the target side. RB tree ops are protected by @ep->mtx, which
+ * the caller already holds.
+ *
+ * On failure the epi is freed and the epoll_watches counter decremented,
+ * matching ep_alloc_epitem()'s allocation. After this returns
+ * successfully, ep_insert()'s later error paths use ep_remove() for
+ * unwind; that cannot drop @ep's refcount to zero because the ep file
+ * itself still holds the original reference.
+ */
+static int ep_register_epitem(struct ep_ctl_ctx *ctx, struct eventpoll *ep,
+			      struct epitem *epi, struct eventpoll *tep,
+			      int full_check)
+{
+	struct file *tfile = epi->ffd.file;
+	int error;
 
 	if (tep)
 		mutex_lock_nested(&tep->mtx, 1);
-	/* Add the current item to the list of active epoll hook for this file */
-	if (unlikely(attach_epitem(tfile, epi) < 0)) {
+
+	error = ep_attach_file(tfile, epi);
+	if (unlikely(error)) {
 		if (tep)
 			mutex_unlock(&tep->mtx);
 		kmem_cache_free(epi_cache, epi);
 		percpu_counter_dec(&ep->user->epoll_watches);
-		return -ENOMEM;
+		return error;
 	}
 
 	if (full_check && !tep)
-		list_file(tfile);
+		list_file(tfile, ctx);
 
-	/*
-	 * Add the current item to the RB tree. All RB tree operations are
-	 * protected by "mtx", and ep_insert() is called with "mtx" held.
-	 */
 	ep_rbtree_insert(ep, epi);
+
 	if (tep)
 		mutex_unlock(&tep->mtx);
 
-	/*
-	 * ep_remove() calls in the later error paths can't lead to
-	 * ep_free() as the ep file itself still holds an ep reference.
-	 */
 	ep_get(ep);
+	return 0;
+}
+
+/*
+ * Must be called with "mtx" held.
+ */
+static int ep_insert(struct ep_ctl_ctx *ctx, struct eventpoll *ep,
+		     const struct epoll_event *event, struct epoll_key *tf,
+		     int full_check)
+{
+	int error, pwake = 0;
+	__poll_t revents;
+	struct epitem *epi;
+	struct ep_pqueue epq;
+	struct eventpoll *tep = NULL;
+
+	if (is_file_epoll(tf->file))
+		tep = tf->file->private_data;
+
+	lockdep_assert_irqs_enabled();
+
+	epi = ep_alloc_epitem(ep, event, tf);
+	if (IS_ERR(epi))
+		return PTR_ERR(epi);
 
-	/* now check if we've created too many backpaths */
-	if (unlikely(full_check && reverse_path_check())) {
+	error = ep_register_epitem(ctx, ep, epi, tep, full_check);
+	if (error)
+		return error;
+
+	/* Reject the insert if the new link would create too many back-paths. */
+	if (unlikely(full_check && reverse_path_check(ctx))) {
 		ep_remove(ep, epi);
 		return -EINVAL;
 	}
@@ -1649,28 +1904,21 @@ static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
 	 */
 	revents = ep_item_poll(epi, &epq.pt, 1);
 
-	/*
-	 * We have to check if something went wrong during the poll wait queue
-	 * install process. Namely an allocation for a wait queue failed due
-	 * high memory pressure.
-	 */
+	/* ep_ptable_queue_proc() signals allocation failure by clearing epq.epi. */
 	if (unlikely(!epq.epi)) {
 		ep_remove(ep, epi);
 		return -ENOMEM;
 	}
 
-	/* We have to drop the new item inside our item list to keep track of it */
+	/* Drop the new item onto the ready list if it is already ready. */
 	spin_lock_irq(&ep->lock);
 
-	/* record NAPI ID of new item if present */
 	ep_set_busy_poll_napi_id(epi);
 
-	/* If the file is already "ready" we drop it inside the ready list */
 	if (revents && !ep_is_linked(epi)) {
 		list_add_tail(&epi->rdllink, &ep->rdllist);
 		ep_pm_stay_awake(epi);
 
-		/* Notify waiting tasks that events are available */
 		if (waitqueue_active(&ep->wq))
 			wake_up(&ep->wq);
 		if (waitqueue_active(&ep->poll_wait))
@@ -1762,11 +2010,87 @@ static int ep_modify(struct eventpoll *ep, struct epitem *epi,
 	return 0;
 }
 
+/*
+ * Attempt to deliver one event for @epi into @*uevents.
+ *
+ * Returns 1 if an event was delivered (with *uevents advanced to the
+ * next slot), 0 if the re-poll reported no caller-requested events
+ * (@epi drops out of the ready list; a future callback will re-add
+ * it), or -EFAULT if copy_to_user() faulted (in which case @epi is
+ * re-inserted at the head of @scan_batch so ep_done_scan() merges it
+ * back to rdllist for the next attempt).
+ *
+ * PM bookkeeping and level-triggered re-queue are handled here.
+ * Caller holds ep->mtx and the scan is active.
+ */
+static int ep_deliver_event(struct eventpoll *ep, struct epitem *epi,
+			    poll_table *pt,
+			    struct epoll_event __user **uevents,
+			    struct list_head *scan_batch)
+{
+	struct epoll_event __user *next;
+	struct wakeup_source *ws;
+	__poll_t revents;
+
+	/*
+	 * Activate ep->ws before deactivating epi->ws to prevent
+	 * triggering auto-suspend here (in case we reactivate epi->ws
+	 * below).  Rearranging to delay the deactivation would let
+	 * epi->ws drift out of sync with ep_is_linked().
+	 */
+	ws = ep_wakeup_source(epi);
+	if (ws) {
+		if (ws->active)
+			__pm_stay_awake(ep->ws);
+		__pm_relax(ws);
+	}
+
+	list_del_init(&epi->rdllink);
+
+	/*
+	 * Re-poll under ep->mtx so userspace cannot change the item
+	 * out from under us. If no caller-requested events remain,
+	 * @epi stays off the ready list; the poll callback will
+	 * re-queue it when events next appear.
+	 */
+	revents = ep_item_poll(epi, pt, 1);
+	if (!revents)
+		return 0;
+
+	next = epoll_put_uevent(revents, epi->event.data, *uevents);
+	if (!next) {
+		/*
+		 * copy_to_user() faulted: put the item back so
+		 * ep_done_scan() splices it onto rdllist for the next
+		 * attempt.
+		 */
+		list_add(&epi->rdllink, scan_batch);
+		ep_pm_stay_awake(epi);
+		return -EFAULT;
+	}
+	*uevents = next;
+
+	if (epi->event.events & EPOLLONESHOT) {
+		epi->event.events &= EP_PRIVATE_BITS;
+	} else if (!(epi->event.events & EPOLLET)) {
+		/*
+		 * Level-triggered: re-queue so the next epoll_wait()
+		 * rechecks availability. We are the sole writer to
+		 * rdllist here -- epoll_ctl() callers are locked out
+		 * by ep->mtx, and the poll callback queues to ovflist
+		 * during scans.
+		 */
+		list_add_tail(&epi->rdllink, &ep->rdllist);
+		ep_pm_stay_awake(epi);
+	}
+	return 1;
+}
+
 static int ep_send_events(struct eventpoll *ep,
 			  struct epoll_event __user *events, int maxevents)
 {
 	struct epitem *epi, *tmp;
-	LIST_HEAD(txlist);
+	LIST_HEAD(scan_batch);
 	poll_table pt;
 	int res = 0;
 
@@ -1781,74 +2105,28 @@ static int ep_send_events(struct eventpoll *ep,
 	init_poll_funcptr(&pt, NULL);
 
 	mutex_lock(&ep->mtx);
-	ep_start_scan(ep, &txlist);
+	ep_start_scan(ep, &scan_batch);
 
 	/*
-	 * We can loop without lock because we are passed a task private list.
-	 * Items cannot vanish during the loop we are holding ep->mtx.
+	 * We can loop without lock because we are passed a task-private
+	 * scan_batch; items cannot vanish while we hold ep->mtx.
 	 */
-	list_for_each_entry_safe(epi, tmp, &txlist, rdllink) {
-		struct wakeup_source *ws;
-		__poll_t revents;
+	list_for_each_entry_safe(epi, tmp, &scan_batch, rdllink) {
+		int delivered;
 
 		if (res >= maxevents)
 			break;
 
-		/*
-		 * Activate ep->ws before deactivating epi->ws to prevent
-		 * triggering auto-suspend here (in case we reactive epi->ws
-		 * below).
-		 *
-		 * This could be rearranged to delay the deactivation of epi->ws
-		 * instead, but then epi->ws would temporarily be out of sync
-		 * with ep_is_linked().
-		 */
-		ws = ep_wakeup_source(epi);
-		if (ws) {
-			if (ws->active)
-				__pm_stay_awake(ep->ws);
-			__pm_relax(ws);
-		}
-
-		list_del_init(&epi->rdllink);
-
-		/*
-		 * If the event mask intersect the caller-requested one,
-		 * deliver the event to userspace. Again, we are holding ep->mtx,
-		 * so no operations coming from userspace can change the item.
-		 */
-		revents = ep_item_poll(epi, &pt, 1);
-		if (!revents)
-			continue;
-
-		events = epoll_put_uevent(revents, epi->event.data, events);
-		if (!events) {
-			list_add(&epi->rdllink, &txlist);
-			ep_pm_stay_awake(epi);
+		delivered = ep_deliver_event(ep, epi, &pt, &events, &scan_batch);
+		if (delivered < 0) {
 			if (!res)
-				res = -EFAULT;
+				res = delivered;
 			break;
 		}
-		res++;
-		if (epi->event.events & EPOLLONESHOT)
-			epi->event.events &= EP_PRIVATE_BITS;
-		else if (!(epi->event.events & EPOLLET)) {
-			/*
-			 * If this file has been added with Level
-			 * Trigger mode, we need to insert back inside
-			 * the ready list, so that the next call to
-			 * epoll_wait() will check again the events
-			 * availability. At this point, no one can insert
-			 * into ep->rdllist besides us. The epoll_ctl()
-			 * callers are locked out by
-			 * ep_send_events() holding "mtx" and the
-			 * poll callback will queue them in ep->ovflist.
-			 */
-			list_add_tail(&epi->rdllink, &ep->rdllist);
-			ep_pm_stay_awake(epi);
-		}
+		res += delivered;
 	}
-	ep_done_scan(ep, &txlist);
+
+	ep_done_scan(ep, &scan_batch);
 	mutex_unlock(&ep->mtx);
 
 	return res;
@@ -1938,7 +2216,8 @@ static int ep_schedule_timeout(ktime_t *to)
 static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
 		   int maxevents, struct timespec64 *timeout)
 {
-	int res, eavail, timed_out = 0;
+	int res, timed_out = 0;
+	bool eavail;
 	u64 slack = 0;
 	wait_queue_entry_t wait;
 	ktime_t expires, *to = NULL;
@@ -2036,7 +2315,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
 		 * If timed out and still on the wait queue, recheck eavail
 		 * carefully under lock, below.
 		 */
-		eavail = 1;
+		eavail = true;
 
 		if (!list_empty_careful(&wait.entry)) {
 			spin_lock_irq(&ep->lock);
@@ -2066,7 +2345,8 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
  * Return: depth of the subtree, or a value bigger than EP_MAX_NESTS if we found
  * a loop or went too deep.
  */
-static int ep_loop_check_proc(struct eventpoll *ep, int depth)
+static int ep_loop_check_proc(struct ep_ctl_ctx *ctx,
+			      struct eventpoll *ep, int depth)
 {
 	int result = 0;
 	struct rb_node *rbp;
@@ -2082,22 +2362,23 @@ static int ep_loop_check_proc(struct eventpoll *ep, int depth)
 		if (unlikely(is_file_epoll(epi->ffd.file))) {
 			struct eventpoll *ep_tovisit;
 			ep_tovisit = epi->ffd.file->private_data;
-			if (ep_tovisit == inserting_into || depth > EP_MAX_NESTS)
+			if (ep_tovisit == ctx->inserting_into ||
+			    depth > EP_MAX_NESTS)
 				result = EP_MAX_NESTS+1;
 			else
-				result = max(result, ep_loop_check_proc(ep_tovisit, depth + 1) + 1);
+				result = max(result,
+					     ep_loop_check_proc(ctx, ep_tovisit,
+								depth + 1) + 1);
 			if (result > EP_MAX_NESTS)
 				break;
 		} else {
 			/*
-			 * If we've reached a file that is not associated with
-			 * an ep, then we need to check if the newly added
-			 * links are going to add too many wakeup paths. We do
-			 * this by adding it to the tfile_check_list, if it's
-			 * not already there, and calling reverse_path_check()
-			 * during ep_insert().
+			 * A non-epoll leaf. Queue it for the companion
+			 * reverse_path_check() that runs after this walk so
+			 * any new links we propose don't add too many wakeup
+			 * paths.
 			 */
-			list_file(epi->ffd.file);
+			list_file(epi->ffd.file, ctx);
 		}
 	}
 	ep->loop_check_depth = result;
@@ -2126,22 +2407,24 @@ static int ep_get_upwards_depth_proc(struct eventpoll *ep, int depth)
  *                 into another epoll file (represented by @ep) does not create
  *                 closed loops or too deep chains.
  *
- * @ep: Pointer to the epoll we are inserting into.
- * @to: Pointer to the epoll to be inserted.
+ * @ctx: Per-CTL_ADD scratch context.
+ * @ep:  Pointer to the epoll we are inserting into.
+ * @to:  Pointer to the epoll to be inserted.
  *
  * Return: %zero if adding the epoll @to inside the epoll @from
  * does not violate the constraints, or %-1 otherwise.
  */
-static int ep_loop_check(struct eventpoll *ep, struct eventpoll *to)
+static int ep_loop_check(struct ep_ctl_ctx *ctx, struct eventpoll *ep,
+			 struct eventpoll *to)
 {
 	int depth, upwards_depth;
 
-	inserting_into = ep;
+	ctx->inserting_into = ep;
 	/*
 	 * Check how deep down we can get from @to, and whether it is possible
 	 * to loop up to @ep.
 	 */
-	depth = ep_loop_check_proc(to, 0);
+	depth = ep_loop_check_proc(ctx, to, 0);
 	if (depth > EP_MAX_NESTS)
 		return -1;
 	/* Check how far up we can go from @ep. */
@@ -2152,12 +2435,12 @@ static int ep_loop_check(struct eventpoll *ep, struct eventpoll *to)
 	return (depth+1+upwards_depth > EP_MAX_NESTS) ? -1 : 0;
 }
 
-static void clear_tfile_check_list(void)
+static void clear_tfile_check_list(struct ep_ctl_ctx *ctx)
 {
 	rcu_read_lock();
-	while (tfile_check_list != EP_UNACTIVE_PTR) {
-		struct epitems_head *head = tfile_check_list;
-		tfile_check_list = head->next;
+	while (ctx->tfile_check_list) {
+		struct epitems_head *head = ctx->tfile_check_list;
+		ctx->tfile_check_list = head->next;
 		unlist_file(head);
 	}
 	rcu_read_unlock();
@@ -2223,38 +2506,105 @@ static inline void ep_take_care_of_epollwakeup(struct epoll_event *epev)
 }
 #endif
 
-static inline int epoll_mutex_lock(struct mutex *mutex, int depth,
-				   bool nonblock)
+static inline int epoll_mutex_lock(struct mutex *mutex, bool nonblock)
 {
 	if (!nonblock) {
-		mutex_lock_nested(mutex, depth);
+		mutex_lock(mutex);
 		return 0;
 	}
-	if (mutex_trylock(mutex))
+	return mutex_trylock(mutex) ? 0 : -EAGAIN;
+}
+
+/*
+ * Acquire the locks required for do_epoll_ctl() on @ep for @op.
+ *
+ * Always takes ep->mtx. For EPOLL_CTL_ADD, additionally runs the
+ * loop / path check under epnested_mutex when the topology can
+ * change: @ep is already watched (epfile->f_ep non-NULL), @ep was
+ * recently loop-checked (ep->gen == loop_check_gen), or @tfile is
+ * itself an eventpoll.
+ *
+ * Return value encodes both outcome and lock state:
+ *
+ *   0        success; ep->mtx held.
+ *   1        success; ep->mtx held AND the full check ran under
+ *            epnested_mutex (which is also still held). The value
+ *            doubles as the @full_check argument to ep_insert().
+ *   -errno   failure; no locks held.
+ *
+ * The caller releases what was taken with ep_ctl_unlock(ep, ret).
+ *
+ * Holding epnested_mutex on add is what prevents two racing
+ * EPOLL_CTL_ADDs on different eps from building a cycle without
+ * either walker observing it.
+ */
+static int ep_ctl_lock(struct ep_ctl_ctx *ctx, struct eventpoll *ep, int op,
+		       struct file *epfile, struct file *tfile, bool nonblock)
+{
+	struct eventpoll *tep;
+	int error;
+
+	error = epoll_mutex_lock(&ep->mtx, nonblock);
+	if (error)
+		return error;
+
+	if (op != EPOLL_CTL_ADD)
+		return 0;
+	if (!READ_ONCE(epfile->f_ep) && ep->gen != loop_check_gen &&
+	    !is_file_epoll(tfile))
 		return 0;
-	return -EAGAIN;
+
+	/* Full check needed: drop ep->mtx so we can take epnested_mutex. */
+	mutex_unlock(&ep->mtx);
+	error = epoll_mutex_lock(&epnested_mutex, nonblock);
+	if (error)
+		return error;
+
+	loop_check_gen++;
+
+	if (is_file_epoll(tfile)) {
+		tep = tfile->private_data;
+		if (ep_loop_check(ctx, ep, tep) != 0) {
+			error = -ELOOP;
+			goto err_unlock_nested;
+		}
+	}
+
+	error = epoll_mutex_lock(&ep->mtx, nonblock);
+	if (error)
+		goto err_unlock_nested;
+
+	return 1;
+
+err_unlock_nested:
+	clear_tfile_check_list(ctx);
+	loop_check_gen++;
+	mutex_unlock(&epnested_mutex);
+	return error;
 }
 
-int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,
-		 bool nonblock)
+static void ep_ctl_unlock(struct ep_ctl_ctx *ctx, struct eventpoll *ep,
+			  int full_check)
+{
+	mutex_unlock(&ep->mtx);
+	if (full_check) {
+		clear_tfile_check_list(ctx);
+		loop_check_gen++;
+		mutex_unlock(&epnested_mutex);
+	}
+}
+
+int do_epoll_ctl_file(struct file *f, int op, struct epoll_key *tf,
+		      struct epoll_event *epds, bool nonblock)
 {
 	int error;
-	int full_check = 0;
+	int full_check;
 	struct eventpoll *ep;
 	struct epitem *epi;
-	struct eventpoll *tep = NULL;
-
-	CLASS(fd, f)(epfd);
-	if (fd_empty(f))
-		return -EBADF;
-
-	/* Get the "struct file *" for the target file */
-	CLASS(fd, tf)(fd);
-	if (fd_empty(tf))
-		return -EBADF;
+	struct ep_ctl_ctx ctx = { };
 
 	/* The target file descriptor must support poll */
-	if (!file_can_poll(fd_file(tf)))
+	if (!file_can_poll(tf->file))
 		return -EPERM;
 
 	/* Check if EPOLLWAKEUP is allowed */
@@ -2262,85 +2612,43 @@ int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,
 		ep_take_care_of_epollwakeup(epds);
 
 	/*
-	 * We have to check that the file structure underneath the file descriptor
-	 * the user passed to us _is_ an eventpoll file. And also we do not permit
+	 * The @f file must itself be an eventpoll, and we do not permit
 	 * adding an epoll file descriptor inside itself.
 	 */
-	error = -EINVAL;
-	if (fd_file(f) == fd_file(tf) || !is_file_epoll(fd_file(f)))
-		goto error_tgt_fput;
+	if (f == tf->file || !is_file_epoll(f))
+		return -EINVAL;
 
 	/*
 	 * epoll adds to the wakeup queue at EPOLL_CTL_ADD time only,
 	 * so EPOLLEXCLUSIVE is not allowed for a EPOLL_CTL_MOD operation.
-	 * Also, we do not currently supported nested exclusive wakeups.
+	 * Also, nested exclusive wakeups are not supported.
 	 */
 	if (ep_op_has_event(op) && (epds->events & EPOLLEXCLUSIVE)) {
 		if (op == EPOLL_CTL_MOD)
-			goto error_tgt_fput;
-		if (op == EPOLL_CTL_ADD && (is_file_epoll(fd_file(tf)) ||
+			return -EINVAL;
+		if (op == EPOLL_CTL_ADD && (is_file_epoll(tf->file) ||
 				(epds->events & ~EPOLLEXCLUSIVE_OK_BITS)))
-			goto error_tgt_fput;
+			return -EINVAL;
 	}
 
-	/*
-	 * At this point it is safe to assume that the "private_data" contains
-	 * our own data structure.
-	 */
-	ep = fd_file(f)->private_data;
+	ep = f->private_data;
 
-	/*
-	 * When we insert an epoll file descriptor inside another epoll file
-	 * descriptor, there is the chance of creating closed loops, which are
-	 * better be handled here, than in more critical paths. While we are
-	 * checking for loops we also determine the list of files reachable
-	 * and hang them on the tfile_check_list, so we can check that we
-	 * haven't created too many possible wakeup paths.
-	 *
-	 * We do not need to take the global 'epumutex' on EPOLL_CTL_ADD when
-	 * the epoll file descriptor is attaching directly to a wakeup source,
-	 * unless the epoll file descriptor is nested. The purpose of taking the
-	 * 'epnested_mutex' on add is to prevent complex toplogies such as loops and
-	 * deep wakeup paths from forming in parallel through multiple
-	 * EPOLL_CTL_ADD operations.
-	 */
-	error = epoll_mutex_lock(&ep->mtx, 0, nonblock);
-	if (error)
-		goto error_tgt_fput;
-	if (op == EPOLL_CTL_ADD) {
-		if (READ_ONCE(fd_file(f)->f_ep) || ep->gen == loop_check_gen ||
-		    is_file_epoll(fd_file(tf))) {
-			mutex_unlock(&ep->mtx);
-			error = epoll_mutex_lock(&epnested_mutex, 0, nonblock);
-			if (error)
-				goto error_tgt_fput;
-			loop_check_gen++;
-			full_check = 1;
-			if (is_file_epoll(fd_file(tf))) {
-				tep = fd_file(tf)->private_data;
-				error = -ELOOP;
-				if (ep_loop_check(ep, tep) != 0)
-					goto error_tgt_fput;
-			}
-			error = epoll_mutex_lock(&ep->mtx, 0, nonblock);
-			if (error)
-				goto error_tgt_fput;
-		}
-	}
+	full_check = ep_ctl_lock(&ctx, ep, op, f, tf->file, nonblock);
+	if (full_check < 0)
+		return full_check;
 
 	/*
-	 * Try to lookup the file inside our RB tree. Since we grabbed "mtx"
-	 * above, we can be sure to be able to use the item looked up by
-	 * ep_find() till we release the mutex.
+	 * Look the target up in ep's RB tree. We hold ep->mtx, so the
+	 * item stays valid until we release.
 	 */
-	epi = ep_find(ep, fd_file(tf), fd);
+	epi = ep_find(ep, tf);
 
 	error = -EINVAL;
 	switch (op) {
 	case EPOLL_CTL_ADD:
 		if (!epi) {
 			epds->events |= EPOLLERR | EPOLLHUP;
-			error = ep_insert(ep, epds, fd_file(tf), fd, full_check);
+			error = ep_insert(&ctx, ep, epds, tf, full_check);
 		} else
 			error = -EEXIST;
 		break;
@@ -2366,17 +2674,30 @@ int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,
 			error = -ENOENT;
 		break;
 	}
-	mutex_unlock(&ep->mtx);
 
-error_tgt_fput:
-	if (full_check) {
-		clear_tfile_check_list();
-		loop_check_gen++;
-		mutex_unlock(&epnested_mutex);
-	}
+	ep_ctl_unlock(&ctx, ep, full_check);
 	return error;
 }
 
+int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,
+		 bool nonblock)
+{
+	struct epoll_key efd;
+
+	CLASS(fd, f)(epfd);
+	if (fd_empty(f))
+		return -EBADF;
+
+	/* Get the "struct file *" for the target file */
+	CLASS(fd, tf)(fd);
+	if (fd_empty(tf))
+		return -EBADF;
+
+	efd.file = fd_file(tf);
+	efd.fd = fd;
+	return do_epoll_ctl_file(fd_file(f), op, &efd, epds, nonblock);
+}
+
 /*
  * The following function implements the controller interface for
  * the eventpoll file that enables the insertion/removal/change of
@@ -2527,6 +2848,50 @@ SYSCALL_DEFINE6(epoll_pwait2, int, epfd, struct epoll_event __user *, events,
 			      sigmask, sigsetsize);
 }
 
+#ifdef CONFIG_KCMP
+static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff)
+{
+	struct rb_node *rbp;
+	struct epitem *epi;
+
+	for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
+		epi = rb_entry(rbp, struct epitem, rbn);
+		if (epi->ffd.fd == tfd) {
+			if (toff == 0)
+				return epi;
+			else
+				toff--;
+		}
+		cond_resched();
+	}
+
+	return NULL;
+}
+
+struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd,
+				     unsigned long toff)
+{
+	struct file *file_raw;
+	struct eventpoll *ep;
+	struct epitem *epi;
+
+	if (!is_file_epoll(file))
+		return ERR_PTR(-EINVAL);
+
+	ep = file->private_data;
+
+	mutex_lock(&ep->mtx);
+	epi = ep_find_tfd(ep, tfd, toff);
+	if (epi)
+		file_raw = epi->ffd.file;
+	else
+		file_raw = ERR_PTR(-ENOENT);
+	mutex_unlock(&ep->mtx);
+
+	return file_raw;
+}
+#endif /* CONFIG_KCMP */
+
 #ifdef CONFIG_COMPAT
 static int do_compat_epoll_pwait(int epfd, struct epoll_event __user *events,
 				 int maxevents, struct timespec64 *timeout,
diff --git a/fs/exec.c b/fs/exec.c
index ba12b4c466f6d..2889b7cf808d7 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1975,9 +1975,11 @@ COMPAT_SYSCALL_DEFINE5(execveat, int, fd,
 static int proc_dointvec_minmax_coredump(const struct ctl_table *table, int write,
 		void *buffer, size_t *lenp, loff_t *ppos)
 {
-	int error = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+	int error, old = READ_ONCE(suid_dumpable);
 
-	if (!error && write)
+	error = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+
+	if (!error && write && (old != READ_ONCE(suid_dumpable)))
 		validate_coredump_safety();
 	return error;
 }
diff --git a/fs/exfat/dir.c b/fs/exfat/dir.c
index ac008ccaa97de..561fd23492183 100644
--- a/fs/exfat/dir.c
+++ b/fs/exfat/dir.c
@@ -1027,12 +1027,12 @@ int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei,
 				continue;
 			}
 
-			brelse(bh);
 			if (entry_type == TYPE_EXTEND) {
 				unsigned short entry_uniname[16], unichar;
 
 				if (step != DIRENT_STEP_NAME ||
 				    name_len >= MAX_NAME_LENGTH) {
+					brelse(bh);
 					step = DIRENT_STEP_FILE;
 					continue;
 				}
@@ -1043,6 +1043,7 @@ int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei,
 					uniname += EXFAT_FILE_NAME_LEN;
 
 				len = exfat_extract_uni_name(ep, entry_uniname);
+				brelse(bh);
 				name_len += len;
 
 				unichar = *(uniname+len);
@@ -1061,6 +1062,7 @@ int exfat_find_dir_entry(struct super_block *sb, struct exfat_inode_info *ei,
 				continue;
 			}
 
+			brelse(bh);
 			if (entry_type &
 					(TYPE_CRITICAL_SEC | TYPE_BENIGN_SEC)) {
 				if (step == DIRENT_STEP_SECD) {
diff --git a/fs/exfat/exfat_fs.h b/fs/exfat/exfat_fs.h
index 89ef5368277f8..aff4dcd4e75a5 100644
--- a/fs/exfat/exfat_fs.h
+++ b/fs/exfat/exfat_fs.h
@@ -496,6 +496,8 @@ int exfat_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 int exfat_getattr(struct mnt_idmap *idmap, const struct path *path,
 		  struct kstat *stat, unsigned int request_mask,
 		  unsigned int query_flags);
+struct file_kattr;
+int exfat_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
 int exfat_file_fsync(struct file *file, loff_t start, loff_t end, int datasync);
 long exfat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
 long exfat_compat_ioctl(struct file *filp, unsigned int cmd,
diff --git a/fs/exfat/file.c b/fs/exfat/file.c
index 354bdcfe4abcd..91e5511945d11 100644
--- a/fs/exfat/file.c
+++ b/fs/exfat/file.c
@@ -14,6 +14,7 @@
 #include <linux/writeback.h>
 #include <linux/filelock.h>
 #include <linux/falloc.h>
+#include <linux/fileattr.h>
 
 #include "exfat_raw.h"
 #include "exfat_fs.h"
@@ -323,6 +324,18 @@ int exfat_getattr(struct mnt_idmap *idmap, const struct path *path,
 	return 0;
 }
 
+int exfat_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
+{
+	/*
+	 * exFAT compares filenames through an upcase table, so lookup
+	 * is always case-insensitive. Long names are stored in UTF-16
+	 * with case intact; CASENONPRESERVING stays clear.
+	 */
+	fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
+	fa->flags |= FS_CASEFOLD_FL;
+	return 0;
+}
+
 int exfat_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 		  struct iattr *attr)
 {
@@ -817,6 +830,7 @@ const struct file_operations exfat_file_operations = {
 };
 
 const struct inode_operations exfat_file_inode_operations = {
-	.setattr     = exfat_setattr,
-	.getattr     = exfat_getattr,
+	.setattr	= exfat_setattr,
+	.getattr	= exfat_getattr,
+	.fileattr_get	= exfat_fileattr_get,
 };
diff --git a/fs/exfat/namei.c b/fs/exfat/namei.c
index 2c5636634b4a4..833b31d0f955d 100644
--- a/fs/exfat/namei.c
+++ b/fs/exfat/namei.c
@@ -705,71 +705,44 @@ static int exfat_find(struct inode *dir, const struct qstr *qname,
 	return 0;
 }
 
-static int exfat_d_anon_disconn(struct dentry *dentry)
-{
-	return IS_ROOT(dentry) && (dentry->d_flags & DCACHE_DISCONNECTED);
-}
-
 static struct dentry *exfat_lookup(struct inode *dir, struct dentry *dentry,
 		unsigned int flags)
 {
 	struct super_block *sb = dir->i_sb;
-	struct inode *inode;
+	struct inode *inode = NULL;
 	struct dentry *alias;
 	struct exfat_dir_entry info;
 	int err;
 	loff_t i_pos;
-	mode_t i_mode;
 
 	mutex_lock(&EXFAT_SB(sb)->s_lock);
 	err = exfat_find(dir, &dentry->d_name, &info);
 	if (err) {
-		if (err == -ENOENT) {
-			inode = NULL;
-			goto out;
-		}
-		goto unlock;
+		if (unlikely(err != -ENOENT))
+			inode = ERR_PTR(err);
+		goto out;
 	}
 
 	i_pos = exfat_make_i_pos(&info);
 	inode = exfat_build_inode(sb, &info, i_pos);
-	err = PTR_ERR_OR_ZERO(inode);
-	if (err)
-		goto unlock;
+	if (IS_ERR(inode) || S_ISDIR(inode->i_mode))
+		goto out;
 
-	i_mode = inode->i_mode;
 	alias = d_find_alias(inode);
 
 	/*
 	 * Checking "alias->d_parent == dentry->d_parent" to make sure
 	 * FS is not corrupted (especially double linked dir).
 	 */
-	if (alias && alias->d_parent == dentry->d_parent &&
-			!exfat_d_anon_disconn(alias)) {
-
+	if (alias && alias->d_parent == dentry->d_parent) {
 		/*
-		 * Unhashed alias is able to exist because of revalidate()
-		 * called by lookup_fast. You can easily make this status
-		 * by calling create and lookup concurrently
-		 * In such case, we reuse an alias instead of new dentry
+		 * This inode has a hashed alias dentry with different
+		 * name. This means, the user did ->lookup() by an
+		 * another name (longname vs 8.3 alias of it) in past.
+		 *
+		 * Switch to new one for reason of locality if possible.
 		 */
-		if (d_unhashed(alias)) {
-			WARN_ON(alias->d_name.hash_len !=
-				dentry->d_name.hash_len);
-			exfat_info(sb, "rehashed a dentry(%p) in read lookup",
-				   alias);
-			d_drop(dentry);
-			d_rehash(alias);
-		} else if (!S_ISDIR(i_mode)) {
-			/*
-			 * This inode has non anonymous-DCACHE_DISCONNECTED
-			 * dentry. This means, the user did ->lookup() by an
-			 * another name (longname vs 8.3 alias of it) in past.
-			 *
-			 * Switch to new one for reason of locality if possible.
-			 */
-			d_move(alias, dentry);
-		}
+		d_move(alias, dentry);
 		iput(inode);
 		mutex_unlock(&EXFAT_SB(sb)->s_lock);
 		return alias;
@@ -781,9 +754,6 @@ static struct dentry *exfat_lookup(struct inode *dir, struct dentry *dentry,
 		exfat_d_version_set(dentry, inode_query_iversion(dir));
 
 	return d_splice_alias(inode, dentry);
-unlock:
-	mutex_unlock(&EXFAT_SB(sb)->s_lock);
-	return ERR_PTR(err);
 }
 
 /* remove an entry, BUT don't truncate */
@@ -1311,4 +1281,5 @@ const struct inode_operations exfat_dir_inode_operations = {
 	.rename		= exfat_rename,
 	.setattr	= exfat_setattr,
 	.getattr	= exfat_getattr,
+	.fileattr_get	= exfat_fileattr_get,
 };
diff --git a/fs/exfat/nls.c b/fs/exfat/nls.c
index 57db08a5271cf..055447edcf9a6 100644
--- a/fs/exfat/nls.c
+++ b/fs/exfat/nls.c
@@ -769,13 +769,18 @@ int exfat_create_upcase_table(struct super_block *sb)
 
 			tbl_clu  = le32_to_cpu(ep->dentry.upcase.start_clu);
 			tbl_size = le64_to_cpu(ep->dentry.upcase.size);
-
-			sector = exfat_cluster_to_sector(sbi, tbl_clu);
-			num_sectors = ((tbl_size - 1) >> blksize_bits) + 1;
-			ret = exfat_load_upcase_table(sb, sector, num_sectors,
-				le32_to_cpu(ep->dentry.upcase.checksum));
-
+			if (tbl_size) {
+				sector = exfat_cluster_to_sector(sbi, tbl_clu);
+				num_sectors = ((tbl_size - 1) >> blksize_bits) + 1;
+				ret = exfat_load_upcase_table(sb, sector, num_sectors,
+					le32_to_cpu(ep->dentry.upcase.checksum));
+			} else {
+				exfat_fs_error(sb,
+					       "bad upcase table size (0 bytes). Please run fsck");
+				ret = -EINVAL;
+			}
 			brelse(bh);
+
 			if (ret && ret != -EIO) {
 				/* free memory from exfat_load_upcase_table call */
 				exfat_free_upcase_table(sbi);
@@ -790,6 +795,8 @@ int exfat_create_upcase_table(struct super_block *sb)
 			return -EIO;
 	}
 
+	exfat_fs_error(sb, "no upcase table entry. Please run fsck");
+
 load_default:
 	/* load default upcase table */
 	return exfat_load_default_upcase_table(sb);
diff --git a/fs/ext4/ialloc.c b/fs/ext4/ialloc.c
index 3fd8f00998521..8c80d50875167 100644
--- a/fs/ext4/ialloc.c
+++ b/fs/ext4/ialloc.c
@@ -252,10 +252,10 @@ void ext4_free_inode(handle_t *handle, struct inode *inode)
 		       "nonexistent device\n", __func__, __LINE__);
 		return;
 	}
-	if (icount_read(inode) > 1) {
+	if (icount_read_once(inode) > 1) {
 		ext4_msg(sb, KERN_ERR, "%s:%d: inode #%llu: count=%d",
 			 __func__, __LINE__, inode->i_ino,
-			 icount_read(inode));
+			 icount_read_once(inode));
 		return;
 	}
 	if (inode->i_nlink) {
diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
index c00a6b6ebcbd8..8b074cac4c754 100644
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -778,7 +778,7 @@ void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
 	__remove_ino_entry(sbi, ino, type);
 }
 
-/* mode should be APPEND_INO, UPDATE_INO or TRANS_DIR_INO */
+/* mode should be APPEND_INO, UPDATE_INO, LARGE_FOLIO_IO, or TRANS_DIR_INO */
 bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode)
 {
 	struct inode_management *im = &sbi->im[mode];
diff --git a/fs/f2fs/compress.c b/fs/f2fs/compress.c
index 881e76158b967..caf522d667d61 100644
--- a/fs/f2fs/compress.c
+++ b/fs/f2fs/compress.c
@@ -14,6 +14,7 @@
 #include <linux/lz4.h>
 #include <linux/zstd.h>
 #include <linux/folio_batch.h>
+#include <linux/fserror.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -760,6 +761,7 @@ void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task)
 
 		/* Avoid f2fs_commit_super in irq context */
 		f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION);
+		fserror_report_file_metadata(dic->inode, ret, GFP_NOFS);
 		goto out_release;
 	}
 
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 8d4f1e75dee3e..bdcf396038f89 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -20,6 +20,7 @@
 #include <linux/sched/signal.h>
 #include <linux/fiemap.h>
 #include <linux/iomap.h>
+#include <linux/fserror.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -377,9 +378,10 @@ static void f2fs_write_end_io(struct bio *bio)
 
 		if (unlikely(bio->bi_status != BLK_STS_OK)) {
 			mapping_set_error(folio->mapping, -EIO);
-			if (type == F2FS_WB_CP_DATA)
+			if (type == F2FS_WB_CP_DATA) {
 				f2fs_stop_checkpoint(sbi, true,
 						STOP_CP_REASON_WRITE_FAIL);
+			}
 		}
 
 		if (is_node_folio(folio)) {
@@ -509,6 +511,8 @@ static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages)
 		bio->bi_private = sbi;
 		bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi,
 						fio->type, fio->temp);
+		bio->bi_write_stream = f2fs_io_type_to_write_stream(bdev, fio->type,
+								    fio->temp);
 	}
 	iostat_alloc_and_bind_ctx(sbi, bio, NULL);
 
@@ -943,6 +947,35 @@ void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
 	}
 }
 
+void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi)
+{
+	struct bio_entry *be, *tmp;
+	struct f2fs_bio_info *io;
+	enum temp_type temp;
+
+	for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
+		LIST_HEAD(list);
+
+		io = sbi->write_io[DATA] + temp;
+
+		/* A lockless list_empty() check is safe here: any bios from
+		 * other kworkers that we miss will be submitted by those
+		 * kworkers accordingly.
+		 */
+		if (list_empty(&io->bio_list))
+			continue;
+
+		f2fs_down_write(&io->bio_list_lock);
+		list_splice_init(&io->bio_list, &list);
+		f2fs_up_write(&io->bio_list_lock);
+
+		list_for_each_entry_safe(be, tmp, &list, list) {
+			f2fs_submit_write_bio(sbi, be->bio, DATA);
+			del_bio_entry(be);
+		}
+	}
+}
+
 int f2fs_merge_page_bio(struct f2fs_io_info *fio)
 {
 	struct bio *bio = *fio->bio;
@@ -1748,6 +1781,7 @@ int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag)
 			err = -EFSCORRUPTED;
 			f2fs_handle_error(sbi,
 					ERROR_CORRUPTED_CLUSTER);
+			fserror_report_file_metadata(inode, err, GFP_NOFS);
 			goto sync_out;
 		}
 
@@ -2497,7 +2531,7 @@ static int f2fs_read_data_large_folio(struct inode *inode,
 	int ret = 0;
 	bool folio_in_bio;
 
-	if (!IS_IMMUTABLE(inode) || f2fs_compressed_file(inode)) {
+	if (f2fs_compressed_file(inode)) {
 		if (folio)
 			folio_unlock(folio);
 		return -EOPNOTSUPP;
diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c
index 38802ee2e40de..b1697194c3c4d 100644
--- a/fs/f2fs/dir.c
+++ b/fs/f2fs/dir.c
@@ -11,6 +11,7 @@
 #include <linux/filelock.h>
 #include <linux/sched/signal.h>
 #include <linux/unicode.h>
+#include <linux/fserror.h>
 #include "f2fs.h"
 #include "node.h"
 #include "acl.h"
@@ -1020,6 +1021,7 @@ int f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
 			set_sbi_flag(sbi, SBI_NEED_FSCK);
 			err = -EFSCORRUPTED;
 			f2fs_handle_error(sbi, ERROR_CORRUPTED_DIRENT);
+			fserror_report_file_metadata(d->inode, err, GFP_NOFS);
 			goto out;
 		}
 
diff --git a/fs/f2fs/extent_cache.c b/fs/f2fs/extent_cache.c
index d2e006420f040..61f6b97143663 100644
--- a/fs/f2fs/extent_cache.c
+++ b/fs/f2fs/extent_cache.c
@@ -119,10 +119,9 @@ static bool __may_extent_tree(struct inode *inode, enum extent_type type)
 	if (!__init_may_extent_tree(inode, type))
 		return false;
 
-	if (is_inode_flag_set(inode, FI_NO_EXTENT))
-		return false;
-
 	if (type == EX_READ) {
+		if (is_inode_flag_set(inode, FI_NO_EXTENT))
+			return false;
 		if (is_inode_flag_set(inode, FI_COMPRESSED_FILE) &&
 				 !f2fs_sb_has_readonly(F2FS_I_SB(inode)))
 			return false;
@@ -645,14 +644,10 @@ static unsigned int __destroy_extent_node(struct inode *inode,
 
 	while (atomic_read(&et->node_cnt)) {
 		write_lock(&et->lock);
-		if (!is_inode_flag_set(inode, FI_NO_EXTENT))
-			set_inode_flag(inode, FI_NO_EXTENT);
 		node_cnt += __free_extent_tree(sbi, et, nr_shrink);
 		write_unlock(&et->lock);
 	}
 
-	f2fs_bug_on(sbi, atomic_read(&et->node_cnt));
-
 	return node_cnt;
 }
 
@@ -691,12 +686,12 @@ static void __update_extent_tree_range(struct inode *inode,
 
 	write_lock(&et->lock);
 
-	if (is_inode_flag_set(inode, FI_NO_EXTENT)) {
-		write_unlock(&et->lock);
-		return;
-	}
-
 	if (type == EX_READ) {
+		if (is_inode_flag_set(inode, FI_NO_EXTENT)) {
+			write_unlock(&et->lock);
+			return;
+		}
+
 		prev = et->largest;
 		dei.len = 0;
 
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 91f506e7c9cfb..f0a54883b8829 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -96,6 +96,15 @@ extern const char *f2fs_fault_name[FAULT_MAX];
 #define DEFAULT_FAILURE_RETRY_COUNT		1
 #endif
 
+enum {
+	REPORT_FAULT_NEED_FSCK,
+	REPORT_FAULT_STOP_CP,
+	REPORT_FAULT_MAX,
+};
+
+void f2fs_fault_report(struct super_block *sb, unsigned int err_code,
+			const char *func, unsigned int data);
+
 /*
  * For mount options
  */
@@ -379,6 +388,7 @@ enum {
 /* for the list of ino */
 enum {
 	ORPHAN_INO,		/* for orphan ino list */
+	LARGE_FOLIO_INO,	/* for large folio case */
 	APPEND_INO,		/* for append ino list */
 	UPDATE_INO,		/* for update ino list */
 	TRANS_DIR_INO,		/* for transactions dir ino list */
@@ -2125,12 +2135,12 @@ static inline void f2fs_update_time(struct f2fs_sb_info *sbi, int type)
 {
 	unsigned long now = jiffies;
 
-	sbi->last_time[type] = now;
+	WRITE_ONCE(sbi->last_time[type], now);
 
 	/* DISCARD_TIME and GC_TIME are based on REQ_TIME */
 	if (type == REQ_TIME) {
-		sbi->last_time[DISCARD_TIME] = now;
-		sbi->last_time[GC_TIME] = now;
+		WRITE_ONCE(sbi->last_time[DISCARD_TIME], now);
+		WRITE_ONCE(sbi->last_time[GC_TIME], now);
 	}
 }
 
@@ -2138,7 +2148,7 @@ static inline bool f2fs_time_over(struct f2fs_sb_info *sbi, int type)
 {
 	unsigned long interval = sbi->interval_time[type] * HZ;
 
-	return time_after(jiffies, sbi->last_time[type] + interval);
+	return time_after(jiffies, READ_ONCE(sbi->last_time[type]) + interval);
 }
 
 static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi,
@@ -2148,7 +2158,7 @@ static inline unsigned int f2fs_time_to_wait(struct f2fs_sb_info *sbi,
 	unsigned int wait_ms = 0;
 	long delta;
 
-	delta = (sbi->last_time[type] + interval) - jiffies;
+	delta = (READ_ONCE(sbi->last_time[type]) + interval) - jiffies;
 	if (delta > 0)
 		wait_ms = jiffies_to_msecs(delta);
 
@@ -2279,11 +2289,18 @@ static inline bool is_sbi_flag_set(struct f2fs_sb_info *sbi, unsigned int type)
 	return test_bit(type, &sbi->s_flag);
 }
 
-static inline void set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
+static inline void __set_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
 {
 	set_bit(type, &sbi->s_flag);
 }
 
+#define set_sbi_flag(sbi, type)				\
+do {							\
+	__set_sbi_flag(sbi, type);			\
+	if ((type) == SBI_NEED_FSCK)			\
+		f2fs_fault_report(sbi->sb, REPORT_FAULT_NEED_FSCK, __func__, __LINE__);	\
+} while (0)
+
 static inline void clear_sbi_flag(struct f2fs_sb_info *sbi, unsigned int type)
 {
 	clear_bit(type, &sbi->s_flag);
@@ -4061,6 +4078,8 @@ void f2fs_destroy_segment_manager_caches(void);
 int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint);
 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
 			enum page_type type, enum temp_type temp);
+u8 f2fs_io_type_to_write_stream(struct block_device *bdev,
+				enum page_type type, enum temp_type temp);
 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi);
 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
 			unsigned int segno);
@@ -4153,6 +4172,7 @@ void f2fs_submit_merged_write_folio(struct f2fs_sb_info *sbi,
 				struct folio *folio, enum page_type type);
 void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi,
 					struct bio **bio, struct folio *folio);
+void f2fs_submit_all_merged_ipu_writes(struct f2fs_sb_info *sbi);
 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi);
 int f2fs_submit_page_bio(struct f2fs_io_info *fio);
 int f2fs_merge_page_bio(struct f2fs_io_info *fio);
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index fb12c5c9affda..71385ca4163df 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -2071,9 +2071,16 @@ static long f2fs_fallocate(struct file *file, int mode,
 
 static int f2fs_release_file(struct inode *inode, struct file *filp)
 {
-	if (atomic_dec_and_test(&F2FS_I(inode)->open_count))
+	if (atomic_dec_and_test(&F2FS_I(inode)->open_count)) {
 		f2fs_remove_donate_inode(inode);
-
+		/*
+		 * In order to get large folio as soon as possible, let's drop
+		 * inode cache asap. See also f2fs_drop_inode.
+		 */
+		if (f2fs_exist_written_data(F2FS_I_SB(inode),
+					    inode->i_ino, LARGE_FOLIO_INO))
+                       d_drop(filp->f_path.dentry);
+	}
 	/*
 	 * f2fs_release_file is called at every close calls. So we should
 	 * not drop any inmemory pages by close called by other process.
@@ -5076,6 +5083,8 @@ static void f2fs_dio_write_submit_io(const struct iomap_iter *iter,
 	enum temp_type temp = f2fs_get_segment_temp(sbi, type);
 
 	bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, DATA, temp);
+	bio->bi_write_stream =
+		f2fs_io_type_to_write_stream(bio->bi_bdev, DATA, temp);
 	blk_crypto_submit_bio(bio);
 }
 
diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index ba93010924c06..99bc598898251 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -1754,9 +1754,8 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi,
 	unsigned int end_segno = start_segno + SEGS_PER_SEC(sbi);
 	unsigned int sec_end_segno;
 	int seg_freed = 0, migrated = 0;
-	unsigned char type = IS_DATASEG(get_seg_entry(sbi, segno)->type) ?
-						SUM_TYPE_DATA : SUM_TYPE_NODE;
-	unsigned char data_type = (type == SUM_TYPE_DATA) ? DATA : NODE;
+	unsigned char type;
+	unsigned char data_type;
 	int submitted = 0, sum_blk_cnt;
 
 	if (__is_large_section(sbi)) {
@@ -1855,6 +1854,12 @@ static int do_garbage_collect(struct f2fs_sb_info *sbi,
 					migrated >= sbi->migration_granularity)
 				continue;
 
+			if (migrated == 0) {
+				type = IS_DATASEG(get_seg_entry(sbi, cur_segno)->type) ?
+							SUM_TYPE_DATA : SUM_TYPE_NODE;
+				data_type = (type == SUM_TYPE_DATA) ? DATA : NODE;
+			}
+
 			sum = SUM_BLK_PAGE_ADDR(sbi, sum_folio, cur_segno);
 			if (type != GET_SUM_TYPE(sum_footer(sbi, sum))) {
 				f2fs_err(sbi, "Inconsistent segment (%u) type "
diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c
index 7aabfc9b43cb8..099f720897016 100644
--- a/fs/f2fs/inline.c
+++ b/fs/f2fs/inline.c
@@ -9,6 +9,7 @@
 #include <linux/fs.h>
 #include <linux/f2fs_fs.h>
 #include <linux/fiemap.h>
+#include <linux/fserror.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -179,6 +180,7 @@ int f2fs_convert_inline_folio(struct dnode_of_data *dn, struct folio *folio)
 		f2fs_warn(fio.sbi, "%s: corrupted inline inode ino=%llu, i_addr[0]:0x%x, run fsck to fix.",
 			  __func__, dn->inode->i_ino, dn->data_blkaddr);
 		f2fs_handle_error(fio.sbi, ERROR_INVALID_BLKADDR);
+		fserror_report_file_metadata(dn->inode, -EFSCORRUPTED, GFP_NOFS);
 		return -EFSCORRUPTED;
 	}
 
@@ -435,6 +437,7 @@ static int f2fs_move_inline_dirents(struct inode *dir, struct folio *ifolio,
 			  __func__, dir->i_ino, dn.data_blkaddr);
 		f2fs_handle_error(F2FS_F_SB(folio), ERROR_INVALID_BLKADDR);
 		err = -EFSCORRUPTED;
+		fserror_report_file_metadata(dn.inode, err, GFP_NOFS);
 		goto out;
 	}
 
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index c6dcda447882f..12f982f87f151 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -11,6 +11,7 @@
 #include <linux/sched/mm.h>
 #include <linux/lz4.h>
 #include <linux/zstd.h>
+#include <linux/fserror.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -480,6 +481,7 @@ static int do_read_inode(struct inode *inode)
 		f2fs_folio_put(node_folio, true);
 		set_sbi_flag(sbi, SBI_NEED_FSCK);
 		f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
+		fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS);
 		return -EFSCORRUPTED;
 	}
 
@@ -541,6 +543,7 @@ static int do_read_inode(struct inode *inode)
 	if (!sanity_check_extent_cache(inode, node_folio)) {
 		f2fs_folio_put(node_folio, true);
 		f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
+		fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS);
 		return -EFSCORRUPTED;
 	}
 
@@ -565,6 +568,20 @@ static bool is_meta_ino(struct f2fs_sb_info *sbi, unsigned int ino)
 		ino == F2FS_COMPRESS_INO(sbi);
 }
 
+static void f2fs_mapping_set_large_folio(struct inode *inode)
+{
+	struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
+
+	if (f2fs_compressed_file(inode))
+		return;
+	if (f2fs_quota_file(sbi, inode->i_ino))
+		return;
+	if (IS_IMMUTABLE(inode) ||
+	    (f2fs_exist_written_data(sbi, inode->i_ino, LARGE_FOLIO_INO) &&
+	     !(inode->i_mode & S_IWUGO)))
+	    mapping_set_folio_min_order(inode->i_mapping, 0);
+}
+
 struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
 {
 	struct f2fs_sb_info *sbi = F2FS_SB(sb);
@@ -583,6 +600,7 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
 			trace_f2fs_iget_exit(inode, ret);
 			iput(inode);
 			f2fs_handle_error(sbi, ERROR_CORRUPTED_INODE);
+			fserror_report_file_metadata(inode, ret, GFP_NOFS);
 			return ERR_PTR(ret);
 		}
 
@@ -620,9 +638,7 @@ struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
 		inode->i_op = &f2fs_file_inode_operations;
 		inode->i_fop = &f2fs_file_operations;
 		inode->i_mapping->a_ops = &f2fs_dblock_aops;
-		if (IS_IMMUTABLE(inode) && !f2fs_compressed_file(inode) &&
-		    !f2fs_quota_file(sbi, inode->i_ino))
-			mapping_set_folio_min_order(inode->i_mapping, 0);
+		f2fs_mapping_set_large_folio(inode);
 	} else if (S_ISDIR(inode->i_mode)) {
 		inode->i_op = &f2fs_dir_inode_operations;
 		inode->i_fop = &f2fs_dir_operations;
@@ -787,6 +803,7 @@ void f2fs_update_inode_page(struct inode *inode)
 		if (err == -ENOMEM || ++count <= DEFAULT_RETRY_IO_COUNT)
 			goto retry;
 stop_checkpoint:
+		fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS);
 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_UPDATE_INODE);
 		return;
 	}
@@ -895,6 +912,7 @@ void f2fs_evict_inode(struct inode *inode)
 	f2fs_remove_ino_entry(sbi, inode->i_ino, APPEND_INO);
 	f2fs_remove_ino_entry(sbi, inode->i_ino, UPDATE_INO);
 	f2fs_remove_ino_entry(sbi, inode->i_ino, FLUSH_INO);
+	f2fs_remove_ino_entry(sbi, inode->i_ino, LARGE_FOLIO_INO);
 
 	if (!is_sbi_flag_set(sbi, SBI_IS_FREEZING)) {
 		sb_start_intwrite(inode->i_sb);
diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
index 4e5bd9e4cfc32..a464d5ee11244 100644
--- a/fs/f2fs/node.c
+++ b/fs/f2fs/node.c
@@ -12,6 +12,7 @@
 #include <linux/blkdev.h>
 #include <linux/folio_batch.h>
 #include <linux/swap.h>
+#include <linux/fserror.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -72,7 +73,11 @@ bool f2fs_available_free_memory(struct f2fs_sb_info *sbi, int type)
 				sizeof(struct free_nid)) >> PAGE_SHIFT;
 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
 	} else if (type == NAT_ENTRIES) {
-		mem_size = (nm_i->nat_cnt[TOTAL_NAT] *
+		/*
+		 * nat_cnt[] is heuristic accounting. Sample it locklessly here
+		 * to avoid taking nat_tree_lock in the balance path.
+		 */
+		mem_size = (data_race(READ_ONCE(nm_i->nat_cnt[TOTAL_NAT])) *
 				sizeof(struct nat_entry)) >> PAGE_SHIFT;
 		res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
 		if (excess_cached_nats(sbi))
@@ -1265,6 +1270,8 @@ int f2fs_truncate_inode_blocks(struct inode *inode, pgoff_t from)
 		if (err == -ENOENT) {
 			set_sbi_flag(F2FS_F_SB(folio), SBI_NEED_FSCK);
 			f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
+			fserror_report_file_metadata(dn.inode, -EFSCORRUPTED,
+								GFP_NOFS);
 			f2fs_err_ratelimited(sbi,
 				"truncate node fail, ino:%llu, nid:%u, "
 				"offset[0]:%d, offset[1]:%d, nofs:%d",
@@ -1556,6 +1563,8 @@ int f2fs_sanity_check_node_footer(struct f2fs_sb_info *sbi,
 		next_blkaddr_of_node(folio));
 
 	f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER);
+	fserror_report_file_metadata(folio->mapping->host,
+			-EFSCORRUPTED, in_irq ? GFP_NOWAIT : GFP_NOFS);
 	return -EFSCORRUPTED;
 }
 
@@ -1778,6 +1787,7 @@ static bool __write_node_folio(struct folio *folio, bool atomic, bool do_fsync,
 
 	if (f2fs_sanity_check_node_footer(sbi, folio, nid,
 					NODE_TYPE_REGULAR, false)) {
+		fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS);
 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_NID);
 		goto redirty_out;
 	}
@@ -2703,6 +2713,8 @@ bool f2fs_alloc_nid(struct f2fs_sb_info *sbi, nid_t *nid)
 			spin_unlock(&nm_i->nid_list_lock);
 			f2fs_err(sbi, "Corrupted nid %u in free_nid_list",
 								i->nid);
+			fserror_report_metadata(sbi->sb, -EFSCORRUPTED,
+								GFP_NOFS);
 			f2fs_stop_checkpoint(sbi, false,
 					STOP_CP_REASON_CORRUPTED_NID);
 			return false;
diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h
index bcf2034e42639..5e114f3520998 100644
--- a/fs/f2fs/node.h
+++ b/fs/f2fs/node.h
@@ -129,13 +129,17 @@ static inline void raw_nat_from_node_info(struct f2fs_nat_entry *raw_ne,
 
 static inline bool excess_dirty_nats(struct f2fs_sb_info *sbi)
 {
-	return NM_I(sbi)->nat_cnt[DIRTY_NAT] >= NM_I(sbi)->max_nid *
+	/* nat_cnt[] is heuristic accounting sampled locklessly here. */
+	return data_race(READ_ONCE(NM_I(sbi)->nat_cnt[DIRTY_NAT])) >=
+					NM_I(sbi)->max_nid *
 					NM_I(sbi)->dirty_nats_ratio / 100;
 }
 
 static inline bool excess_cached_nats(struct f2fs_sb_info *sbi)
 {
-	return NM_I(sbi)->nat_cnt[TOTAL_NAT] >= DEF_NAT_CACHE_THRESHOLD;
+	/* nat_cnt[] is heuristic accounting sampled locklessly here. */
+	return data_race(READ_ONCE(NM_I(sbi)->nat_cnt[TOTAL_NAT])) >=
+					DEF_NAT_CACHE_THRESHOLD;
 }
 
 enum mem_type {
diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c
index 3d3dacec94825..89af8407b6673 100644
--- a/fs/f2fs/recovery.c
+++ b/fs/f2fs/recovery.c
@@ -9,6 +9,7 @@
 #include <linux/fs.h>
 #include <linux/f2fs_fs.h>
 #include <linux/sched/mm.h>
+#include <linux/fserror.h>
 #include "f2fs.h"
 #include "node.h"
 #include "segment.h"
@@ -679,6 +680,7 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
 			  ofs_of_node(folio));
 		err = -EFSCORRUPTED;
 		f2fs_handle_error(sbi, ERROR_INCONSISTENT_FOOTER);
+		fserror_report_file_metadata(dn.inode, err, GFP_NOFS);
 		goto err;
 	}
 
diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
index 788f8b0502492..7c8ac62b1b0d8 100644
--- a/fs/f2fs/segment.c
+++ b/fs/f2fs/segment.c
@@ -17,6 +17,7 @@
 #include <linux/freezer.h>
 #include <linux/sched/signal.h>
 #include <linux/random.h>
+#include <linux/fserror.h>
 
 #include "f2fs.h"
 #include "segment.h"
@@ -462,6 +463,14 @@ void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
 			.should_migrate_blocks = false,
 			.err_gc_skipped = false,
 			.nr_free_secs = 1 };
+
+		/*
+		 * Submit all cached OPU/IPU DATA bios before triggering
+		 * foreground GC to avoid potential deadlocks.
+		 */
+		f2fs_submit_merged_write(sbi, DATA);
+		f2fs_submit_all_merged_ipu_writes(sbi);
+
 		f2fs_down_write_trace(&sbi->gc_lock, &gc_control.lc);
 		stat_inc_gc_call_count(sbi, FOREGROUND);
 		f2fs_gc(sbi, &gc_control);
@@ -2896,6 +2905,7 @@ static int get_new_segment(struct f2fs_sb_info *sbi,
 	/* set it as dirty segment in free segmap */
 	if (test_bit(segno, free_i->free_segmap)) {
 		ret = -EFSCORRUPTED;
+		fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS);
 		f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_CORRUPTED_FREE_BITMAP);
 		goto out_unlock;
 	}
@@ -3636,6 +3646,19 @@ enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
 	}
 }
 
+u8 f2fs_io_type_to_write_stream(struct block_device *bdev,
+				enum page_type type, enum temp_type temp)
+{
+	unsigned short nr = bdev_max_write_streams(bdev);
+
+	if (type != DATA || !nr)
+		return 0;
+	if (nr < NR_TEMP_TYPE)
+		return temp == COLD ? nr : HOT + 1;
+
+	return temp + 1;
+}
+
 static int __get_segment_type_2(struct f2fs_io_info *fio)
 {
 	if (fio->type == DATA)
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index ccf806b676f53..ada8098f8b335 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -29,6 +29,7 @@
 #include <linux/lz4.h>
 #include <linux/ctype.h>
 #include <linux/fs_parser.h>
+#include <linux/fserror.h>
 
 #include "f2fs.h"
 #include "node.h"
@@ -1844,6 +1845,13 @@ static int f2fs_drop_inode(struct inode *inode)
 			return 1;
 		}
 	}
+	/*
+	 * In order to get large folio as soon as possible, let's drop
+	 * inode cache asap. See also f2fs_release_file.
+	 */
+	if (f2fs_exist_written_data(sbi, inode->i_ino, LARGE_FOLIO_INO) &&
+	    !is_inode_flag_set(inode, FI_DIRTY_INODE))
+		return 1;
 
 	/*
 	 * This is to avoid a deadlock condition like below.
@@ -4635,6 +4643,8 @@ static void f2fs_record_stop_reason(struct f2fs_sb_info *sbi)
 		f2fs_err_ratelimited(sbi,
 			"f2fs_commit_super fails to record stop_reason, err:%d",
 			err);
+
+	fserror_report_shutdown(sbi->sb, GFP_NOFS);
 }
 
 void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag)
@@ -4649,6 +4659,27 @@ void f2fs_save_errors(struct f2fs_sb_info *sbi, unsigned char flag)
 	spin_unlock_irqrestore(&sbi->error_lock, flags);
 }
 
+static void f2fs_report_fserror(struct f2fs_sb_info *sbi, unsigned char error)
+{
+	switch (error) {
+	case ERROR_INVALID_BLKADDR:
+	case ERROR_CORRUPTED_INODE:
+	case ERROR_INCONSISTENT_SUMMARY:
+	case ERROR_INCONSISTENT_SUM_TYPE:
+	case ERROR_CORRUPTED_JOURNAL:
+	case ERROR_INCONSISTENT_NODE_COUNT:
+	case ERROR_INCONSISTENT_BLOCK_COUNT:
+	case ERROR_INVALID_CURSEG:
+	case ERROR_INCONSISTENT_SIT:
+	case ERROR_INVALID_NODE_REFERENCE:
+	case ERROR_INCONSISTENT_NAT:
+		fserror_report_metadata(sbi->sb, -EFSCORRUPTED, GFP_NOFS);
+		break;
+	default:
+		return;
+	}
+}
+
 void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error)
 {
 	f2fs_save_errors(sbi, error);
@@ -4658,6 +4689,8 @@ void f2fs_handle_error(struct f2fs_sb_info *sbi, unsigned char error)
 	if (!test_bit(error, (unsigned long *)sbi->errors))
 		return;
 	schedule_work(&sbi->s_error_work);
+
+	f2fs_report_fserror(sbi, error);
 }
 
 static bool system_going_down(void)
@@ -4724,9 +4757,18 @@ static void f2fs_handle_critical_error(struct f2fs_sb_info *sbi,
 	 */
 }
 
+void f2fs_fault_report(struct super_block *sb, unsigned int err_code,
+			const char *func, unsigned int data)
+{
+	trace_f2fs_fault_report(sb, err_code, func, data);
+}
+
 void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
 						unsigned char reason)
 {
+	if (reason != STOP_CP_REASON_SHUTDOWN)
+		f2fs_fault_report(sbi->sb, REPORT_FAULT_STOP_CP, __func__, reason);
+
 	f2fs_build_fault_attr(sbi, 0, 0, FAULT_ALL);
 	if (!end_io)
 		f2fs_flush_merged_writes(sbi);
@@ -5107,6 +5149,13 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 		goto free_page_array_cache;
 	}
 
+	/*
+	 * Initialize ino entry info early so f2fs_drop_inode ->
+	 * f2fs_exist_written_data can safely take im->ino_lock if mount
+	 * fails after this point and triggers iput on cleanup.
+	 */
+	f2fs_init_ino_entry_info(sbi);
+
 	err = f2fs_get_valid_checkpoint(sbi);
 	if (err) {
 		f2fs_err(sbi, "Failed to get valid F2FS checkpoint");
@@ -5151,8 +5200,6 @@ static int f2fs_fill_super(struct super_block *sb, struct fs_context *fc)
 
 	f2fs_init_extent_cache_info(sbi);
 
-	f2fs_init_ino_entry_info(sbi);
-
 	f2fs_init_fsync_node_info(sbi);
 
 	/* setup checkpoint request control and start checkpoint issue thread */
diff --git a/fs/f2fs/sysfs.c b/fs/f2fs/sysfs.c
index 352e96ad5c3a5..665687244c939 100644
--- a/fs/f2fs/sysfs.c
+++ b/fs/f2fs/sysfs.c
@@ -1399,6 +1399,7 @@ F2FS_FEATURE_RO_ATTR(pin_file);
 F2FS_FEATURE_RO_ATTR(linear_lookup);
 #endif
 F2FS_FEATURE_RO_ATTR(packed_ssa);
+F2FS_FEATURE_RO_ATTR(fserror);
 
 #define ATTR_LIST(name) (&f2fs_attr_##name.attr)
 static struct attribute *f2fs_attrs[] = {
@@ -1566,6 +1567,7 @@ static struct attribute *f2fs_feat_attrs[] = {
 	BASE_ATTR_LIST(linear_lookup),
 #endif
 	BASE_ATTR_LIST(packed_ssa),
+	BASE_ATTR_LIST(fserror),
 	NULL,
 };
 ATTRIBUTE_GROUPS(f2fs_feat);
diff --git a/fs/f2fs/verity.c b/fs/f2fs/verity.c
index 92ebcc19cab09..39f4825154452 100644
--- a/fs/f2fs/verity.c
+++ b/fs/f2fs/verity.c
@@ -25,6 +25,7 @@
  */
 
 #include <linux/f2fs_fs.h>
+#include <linux/fserror.h>
 
 #include "f2fs.h"
 #include "xattr.h"
@@ -243,6 +244,7 @@ static int f2fs_get_verity_descriptor(struct inode *inode, void *buf,
 		f2fs_warn(F2FS_I_SB(inode), "invalid verity xattr");
 		f2fs_handle_error(F2FS_I_SB(inode),
 				ERROR_CORRUPTED_VERITY_XATTR);
+		fserror_report_file_metadata(inode, -EFSCORRUPTED, GFP_NOFS);
 		return -EFSCORRUPTED;
 	}
 	if (buf_size) {
diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c
index 610d5810074dc..84273936f2a0d 100644
--- a/fs/f2fs/xattr.c
+++ b/fs/f2fs/xattr.c
@@ -19,6 +19,7 @@
 #include <linux/f2fs_fs.h>
 #include <linux/security.h>
 #include <linux/posix_acl_xattr.h>
+#include <linux/fserror.h>
 #include "f2fs.h"
 #include "xattr.h"
 #include "segment.h"
@@ -44,6 +45,16 @@ static void xattr_free(struct f2fs_sb_info *sbi, void *xattr_addr,
 		kfree(xattr_addr);
 }
 
+static int f2fs_xattr_fadvise_get(struct inode *inode, void *buffer)
+{
+	if (!buffer)
+		goto out;
+	if (mapping_large_folio_support(inode->i_mapping))
+		*((unsigned int *)buffer) |= BIT(F2FS_XATTR_FADV_LARGEFOLIO);
+out:
+	return sizeof(unsigned int);
+}
+
 static int f2fs_xattr_generic_get(const struct xattr_handler *handler,
 		struct dentry *unused, struct inode *inode,
 		const char *name, void *buffer, size_t size)
@@ -61,10 +72,29 @@ static int f2fs_xattr_generic_get(const struct xattr_handler *handler,
 	default:
 		return -EINVAL;
 	}
+	if (handler->flags == F2FS_XATTR_INDEX_USER &&
+	    !strcmp(name, "fadvise"))
+		return f2fs_xattr_fadvise_get(inode, buffer);
+
 	return f2fs_getxattr(inode, handler->flags, name,
 			     buffer, size, NULL);
 }
 
+static int f2fs_xattr_fadvise_set(struct inode *inode, const void *value)
+{
+	unsigned int new_fadvise;
+
+	new_fadvise = *(unsigned int *)value;
+
+	if (new_fadvise & BIT(F2FS_XATTR_FADV_LARGEFOLIO))
+		f2fs_add_ino_entry(F2FS_I_SB(inode),
+				inode->i_ino, LARGE_FOLIO_INO);
+	else
+		f2fs_remove_ino_entry(F2FS_I_SB(inode),
+				inode->i_ino, LARGE_FOLIO_INO);
+	return 0;
+}
+
 static int f2fs_xattr_generic_set(const struct xattr_handler *handler,
 		struct mnt_idmap *idmap,
 		struct dentry *unused, struct inode *inode,
@@ -84,6 +114,10 @@ static int f2fs_xattr_generic_set(const struct xattr_handler *handler,
 	default:
 		return -EINVAL;
 	}
+	if (handler->flags == F2FS_XATTR_INDEX_USER &&
+	    !strcmp(name, "fadvise"))
+		return f2fs_xattr_fadvise_set(inode, value);
+
 	return f2fs_setxattr(inode, handler->flags, name,
 					value, size, NULL, flags);
 }
@@ -371,6 +405,7 @@ static int lookup_all_xattrs(struct inode *inode, struct folio *ifolio,
 		err = -ENODATA;
 		f2fs_handle_error(F2FS_I_SB(inode),
 					ERROR_CORRUPTED_XATTR);
+		fserror_report_file_metadata(inode, err, GFP_NOFS);
 		goto out;
 	}
 check:
@@ -590,6 +625,8 @@ ssize_t f2fs_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size)
 			set_sbi_flag(F2FS_I_SB(inode), SBI_NEED_FSCK);
 			f2fs_handle_error(F2FS_I_SB(inode),
 						ERROR_CORRUPTED_XATTR);
+			fserror_report_file_metadata(inode,
+						-EFSCORRUPTED, GFP_NOFS);
 			break;
 		}
 
@@ -677,6 +714,7 @@ static int __f2fs_setxattr(struct inode *inode, int index,
 		error = -EFSCORRUPTED;
 		f2fs_handle_error(F2FS_I_SB(inode),
 					ERROR_CORRUPTED_XATTR);
+		fserror_report_file_metadata(inode, error, GFP_NOFS);
 		goto exit;
 	}
 
@@ -705,6 +743,7 @@ static int __f2fs_setxattr(struct inode *inode, int index,
 			error = -EFSCORRUPTED;
 			f2fs_handle_error(F2FS_I_SB(inode),
 						ERROR_CORRUPTED_XATTR);
+			fserror_report_file_metadata(inode, error, GFP_NOFS);
 			goto exit;
 		}
 		last = XATTR_NEXT_ENTRY(last);
@@ -842,4 +881,4 @@ int __init f2fs_init_xattr_cache(void)
 void f2fs_destroy_xattr_cache(void)
 {
 	kmem_cache_destroy(inline_xattr_slab);
-}
\ No newline at end of file
+}
diff --git a/fs/f2fs/xattr.h b/fs/f2fs/xattr.h
index bce3d93e4755b..455f460d014e2 100644
--- a/fs/f2fs/xattr.h
+++ b/fs/f2fs/xattr.h
@@ -24,6 +24,7 @@
 #define F2FS_XATTR_REFCOUNT_MAX         1024
 
 /* Name indexes */
+#define F2FS_USER_FADVISE_NAME			"user.fadvise"
 #define F2FS_SYSTEM_ADVISE_NAME			"system.advise"
 #define F2FS_XATTR_INDEX_USER			1
 #define F2FS_XATTR_INDEX_POSIX_ACL_ACCESS	2
@@ -39,6 +40,11 @@
 #define F2FS_XATTR_NAME_ENCRYPTION_CONTEXT	"c"
 #define F2FS_XATTR_NAME_VERITY			"v"
 
+/* used for F2FS_USER_FADVISE_NAME */
+enum {
+	F2FS_XATTR_FADV_LARGEFOLIO,
+};
+
 struct f2fs_xattr_header {
 	__le32  h_magic;        /* magic number for identification */
 	__le32  h_refcount;     /* reference count */
diff --git a/fs/fat/fat.h b/fs/fat/fat.h
index 5a58f0bf8ce83..99ed9228a677b 100644
--- a/fs/fat/fat.h
+++ b/fs/fat/fat.h
@@ -10,6 +10,8 @@
 #include <linux/fs_context.h>
 #include <linux/fs_parser.h>
 
+struct file_kattr;
+
 /*
  * vfat shortname flags
  */
@@ -408,6 +410,7 @@ extern void fat_truncate_blocks(struct inode *inode, loff_t offset);
 extern int fat_getattr(struct mnt_idmap *idmap,
 		       const struct path *path, struct kstat *stat,
 		       u32 request_mask, unsigned int flags);
+int fat_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
 extern int fat_file_fsync(struct file *file, loff_t start, loff_t end,
 			  int datasync);
 
diff --git a/fs/fat/fat_test.c b/fs/fat/fat_test.c
index 886bf044a9f1d..4eeed9dca5494 100644
--- a/fs/fat/fat_test.c
+++ b/fs/fat/fat_test.c
@@ -20,6 +20,37 @@ static void fat_checksum_test(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, fat_checksum("ABCDEFGHA  "), (u8)98);
 }
 
+static void fat_clus_to_blknr_test(struct kunit *test)
+{
+	struct msdos_sb_info sbi = {
+		.sec_per_clus = 4,
+		.data_start = 100,
+	};
+
+	KUNIT_EXPECT_EQ(test, (sector_t)100,
+			fat_clus_to_blknr(&sbi, FAT_START_ENT));
+	KUNIT_EXPECT_EQ(test, (sector_t)112, fat_clus_to_blknr(&sbi, 5));
+}
+
+static void fat_get_blknr_offset_test(struct kunit *test)
+{
+	struct msdos_sb_info sbi = {
+		.dir_per_block = 16,
+		.dir_per_block_bits = 4,
+	};
+
+	sector_t blknr;
+	int offset;
+
+	fat_get_blknr_offset(&sbi, 0, &blknr, &offset);
+	KUNIT_EXPECT_EQ(test, (sector_t)0, blknr);
+	KUNIT_EXPECT_EQ(test, 0, offset);
+
+	fat_get_blknr_offset(&sbi, (10 << 4) | 7, &blknr, &offset);
+	KUNIT_EXPECT_EQ(test, (sector_t)10, blknr);
+	KUNIT_EXPECT_EQ(test, 7, offset);
+}
+
 struct fat_timestamp_testcase {
 	const char *name;
 	struct timespec64 ts;
@@ -341,6 +372,8 @@ static void fat_truncate_atime_test(struct kunit *test)
 
 static struct kunit_case fat_test_cases[] = {
 	KUNIT_CASE(fat_checksum_test),
+	KUNIT_CASE(fat_clus_to_blknr_test),
+	KUNIT_CASE(fat_get_blknr_offset_test),
 	KUNIT_CASE_PARAM(fat_time_fat2unix_test, fat_time_gen_params),
 	KUNIT_CASE_PARAM(fat_time_unix2fat_test, fat_time_gen_params),
 	KUNIT_CASE_PARAM(fat_time_unix2fat_clamp_test,
diff --git a/fs/fat/file.c b/fs/fat/file.c
index becccdd2e501a..37e7049b4c8c4 100644
--- a/fs/fat/file.c
+++ b/fs/fat/file.c
@@ -17,6 +17,7 @@
 #include <linux/fsnotify.h>
 #include <linux/security.h>
 #include <linux/falloc.h>
+#include <linux/fileattr.h>
 #include "fat.h"
 
 static long fat_fallocate(struct file *file, int mode,
@@ -398,6 +399,40 @@ void fat_truncate_blocks(struct inode *inode, loff_t offset)
 	fat_flush_inodes(inode->i_sb, inode, NULL);
 }
 
+int fat_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
+{
+	struct msdos_sb_info *sbi = MSDOS_SB(dentry->d_sb);
+	bool case_sensitive;
+
+	/*
+	 * FAT filesystems are case-insensitive by default. VFAT
+	 * becomes case-sensitive when mounted with 'check=strict',
+	 * which installs vfat_dentry_ops. MSDOS has no such option;
+	 * its 'nocase' mount option selects case-sensitive matching.
+	 *
+	 * VFAT long filename entries preserve case. Without VFAT, only
+	 * uppercased 8.3 short names are stored. MSDOS with 'nocase'
+	 * also preserves case.
+	 */
+	if (sbi->options.isvfat)
+		case_sensitive = sbi->options.name_check == 's';
+	else
+		case_sensitive = sbi->options.nocase;
+
+	if (!case_sensitive) {
+		fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
+		fa->flags |= FS_CASEFOLD_FL;
+		if (!sbi->options.isvfat)
+			fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING;
+	}
+	if (d_inode(dentry)->i_flags & S_IMMUTABLE) {
+		fa->fsx_xflags |= FS_XFLAG_IMMUTABLE;
+		fa->flags |= FS_IMMUTABLE_FL;
+	}
+	return 0;
+}
+EXPORT_SYMBOL_GPL(fat_fileattr_get);
+
 int fat_getattr(struct mnt_idmap *idmap, const struct path *path,
 		struct kstat *stat, u32 request_mask, unsigned int flags)
 {
@@ -575,5 +610,6 @@ EXPORT_SYMBOL_GPL(fat_setattr);
 const struct inode_operations fat_file_inode_operations = {
 	.setattr	= fat_setattr,
 	.getattr	= fat_getattr,
+	.fileattr_get	= fat_fileattr_get,
 	.update_time	= fat_update_time,
 };
diff --git a/fs/fat/namei_msdos.c b/fs/fat/namei_msdos.c
index 4cc65f330fb7e..0fd2971ad4b13 100644
--- a/fs/fat/namei_msdos.c
+++ b/fs/fat/namei_msdos.c
@@ -644,6 +644,7 @@ static const struct inode_operations msdos_dir_inode_operations = {
 	.rename		= msdos_rename,
 	.setattr	= fat_setattr,
 	.getattr	= fat_getattr,
+	.fileattr_get	= fat_fileattr_get,
 	.update_time	= fat_update_time,
 };
 
diff --git a/fs/fat/namei_vfat.c b/fs/fat/namei_vfat.c
index 918b3756674c3..e909447873e36 100644
--- a/fs/fat/namei_vfat.c
+++ b/fs/fat/namei_vfat.c
@@ -1185,6 +1185,7 @@ static const struct inode_operations vfat_dir_inode_operations = {
 	.rename		= vfat_rename2,
 	.setattr	= fat_setattr,
 	.getattr	= fat_getattr,
+	.fileattr_get	= fat_fileattr_get,
 	.update_time	= fat_update_time,
 };
 
diff --git a/fs/fcntl.c b/fs/fcntl.c
index beab8080badf6..7e953d4a807de 100644
--- a/fs/fcntl.c
+++ b/fs/fcntl.c
@@ -1169,9 +1169,9 @@ static int __init fcntl_init(void)
 	 * Exceptions: O_NONBLOCK is a two bit define on parisc; O_NDELAY
 	 * is defined as O_NONBLOCK on some platforms and not on others.
 	 */
-	BUILD_BUG_ON(20 - 1 /* for O_RDONLY being 0 */ !=
+	BUILD_BUG_ON(22 - 1 /* for O_RDONLY being 0 */ !=
 		HWEIGHT32(
-			(VALID_OPEN_FLAGS & ~(O_NONBLOCK | O_NDELAY)) |
+			(VALID_OPENAT2_FLAGS & ~(O_NONBLOCK | O_NDELAY)) |
 			__FMODE_EXEC));
 
 	fasync_cache = kmem_cache_create("fasync_cache",
diff --git a/fs/file.c b/fs/file.c
index 2c81c0b162d05..e5c75b22e0c7c 100644
--- a/fs/file.c
+++ b/fs/file.c
@@ -544,24 +544,23 @@ struct files_struct init_files = {
 static unsigned int find_next_fd(struct fdtable *fdt, unsigned int start)
 {
 	unsigned int maxfd = fdt->max_fds; /* always multiple of BITS_PER_LONG */
-	unsigned int maxbit = maxfd / BITS_PER_LONG;
-	unsigned int bitbit = start / BITS_PER_LONG;
+	unsigned int max_fds_words = maxfd / BITS_PER_LONG;
+	unsigned int fds_word_idx = start / BITS_PER_LONG;
 	unsigned int bit;
 
 	/*
 	 * Try to avoid looking at the second level bitmap
 	 */
-	bit = find_next_zero_bit(&fdt->open_fds[bitbit], BITS_PER_LONG,
+	bit = find_next_zero_bit(&fdt->open_fds[fds_word_idx], BITS_PER_LONG,
 				 start & (BITS_PER_LONG - 1));
 	if (bit < BITS_PER_LONG)
-		return bit + bitbit * BITS_PER_LONG;
+		return bit + (fds_word_idx * BITS_PER_LONG);
 
-	bitbit = find_next_zero_bit(fdt->full_fds_bits, maxbit, bitbit) * BITS_PER_LONG;
-	if (bitbit >= maxfd)
+	bit = BITS_PER_LONG *
+		find_next_zero_bit(fdt->full_fds_bits, max_fds_words, fds_word_idx + 1);
+	if (bit >= maxfd)
 		return maxfd;
-	if (bitbit > start)
-		start = bitbit;
-	return find_next_zero_bit(fdt->open_fds, maxfd, start);
+	return find_next_zero_bit(fdt->open_fds, maxfd, bit);
 }
 
 /*
diff --git a/fs/file_attr.c b/fs/file_attr.c
index da983e105d708..bfb00d256dd56 100644
--- a/fs/file_attr.c
+++ b/fs/file_attr.c
@@ -15,12 +15,10 @@
  * @fa:		fileattr pointer
  * @xflags:	FS_XFLAG_* flags
  *
- * Set ->fsx_xflags, ->fsx_valid and ->flags (translated xflags).  All
- * other fields are zeroed.
+ * Set ->fsx_xflags, ->fsx_valid and ->flags (translated xflags).
  */
 void fileattr_fill_xflags(struct file_kattr *fa, u32 xflags)
 {
-	memset(fa, 0, sizeof(*fa));
 	fa->fsx_valid = true;
 	fa->fsx_xflags = xflags;
 	if (fa->fsx_xflags & FS_XFLAG_IMMUTABLE)
@@ -39,6 +37,8 @@ void fileattr_fill_xflags(struct file_kattr *fa, u32 xflags)
 		fa->flags |= FS_PROJINHERIT_FL;
 	if (fa->fsx_xflags & FS_XFLAG_VERITY)
 		fa->flags |= FS_VERITY_FL;
+	if (fa->fsx_xflags & FS_XFLAG_CASEFOLD)
+		fa->flags |= FS_CASEFOLD_FL;
 }
 EXPORT_SYMBOL(fileattr_fill_xflags);
 
@@ -48,11 +48,9 @@ EXPORT_SYMBOL(fileattr_fill_xflags);
  * @flags:	FS_*_FL flags
  *
  * Set ->flags, ->flags_valid and ->fsx_xflags (translated flags).
- * All other fields are zeroed.
  */
 void fileattr_fill_flags(struct file_kattr *fa, u32 flags)
 {
-	memset(fa, 0, sizeof(*fa));
 	fa->flags_valid = true;
 	fa->flags = flags;
 	if (fa->flags & FS_SYNC_FL)
@@ -71,6 +69,8 @@ void fileattr_fill_flags(struct file_kattr *fa, u32 flags)
 		fa->fsx_xflags |= FS_XFLAG_PROJINHERIT;
 	if (fa->flags & FS_VERITY_FL)
 		fa->fsx_xflags |= FS_XFLAG_VERITY;
+	if (fa->flags & FS_CASEFOLD_FL)
+		fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
 }
 EXPORT_SYMBOL(fileattr_fill_flags);
 
@@ -325,7 +325,7 @@ int ioctl_setflags(struct file *file, unsigned int __user *argp)
 {
 	struct mnt_idmap *idmap = file_mnt_idmap(file);
 	struct dentry *dentry = file->f_path.dentry;
-	struct file_kattr fa;
+	struct file_kattr fa = {};
 	unsigned int flags;
 	int err;
 
@@ -357,7 +357,7 @@ int ioctl_fssetxattr(struct file *file, void __user *argp)
 {
 	struct mnt_idmap *idmap = file_mnt_idmap(file);
 	struct dentry *dentry = file->f_path.dentry;
-	struct file_kattr fa;
+	struct file_kattr fa = {};
 	int err;
 
 	err = copy_fsxattr_from_user(&fa, argp);
@@ -431,7 +431,7 @@ SYSCALL_DEFINE5(file_setattr, int, dfd, const char __user *, filename,
 	struct path filepath __free(path_put) = {};
 	unsigned int lookup_flags = 0;
 	struct file_attr fattr;
-	struct file_kattr fa;
+	struct file_kattr fa = {};
 	int error;
 
 	BUILD_BUG_ON(sizeof(struct file_attr) < FILE_ATTR_SIZE_VER0);
diff --git a/fs/filesystems.c b/fs/filesystems.c
index 0c7d2b7ac26c8..771fc31a69b8f 100644
--- a/fs/filesystems.c
+++ b/fs/filesystems.c
@@ -17,22 +17,49 @@
 #include <linux/slab.h>
 #include <linux/uaccess.h>
 #include <linux/fs_parser.h>
+#include <linux/rculist.h>
 
 /*
- * Handling of filesystem drivers list.
- * Rules:
- *	Inclusion to/removals from/scanning of list are protected by spinlock.
- *	During the unload module must call unregister_filesystem().
- *	We can access the fields of list element if:
- *		1) spinlock is held or
- *		2) we hold the reference to the module.
- *	The latter can be guaranteed by call of try_module_get(); if it
- *	returned 0 we must skip the element, otherwise we got the reference.
- *	Once the reference is obtained we can drop the spinlock.
+ * Read-mostly filesystem drivers list.
+ *
+ * Readers walk under rcu_read_lock(); writers take file_systems_lock
+ * and publish via _rcu hlist primitives.  unregister_filesystem()
+ * synchronize_rcu()s after unlock so the embedded file_system_type
+ * can't go away under a reader.  To keep using a filesystem after
+ * the RCU section ends, take a module reference via try_module_get().
+ */
+static HLIST_HEAD(file_systems);
+static DEFINE_SPINLOCK(file_systems_lock);
+
+#ifdef CONFIG_PROC_FS
+/*
+ * Cache a stringified version of the filesystem list.
+ *
+ * The fs list gets queried a lot by userspace because of libselinux, including
+ * rather surprising programs (would you guess *sed* is on the list?). In order
+ * to reduce the overhead we cache the resulting string, which normally hangs
+ * around below 512 bytes in size.
+ *
+ * As the list almost never changes, its creation is not particularly optimized
+ * to keep things simple.
+ *
+ * We sort it out on read in order to not introduce a failure point for fs
+ * registration (in principle we may be unable to alloc memory for the list).
  */
+struct file_systems_string {
+	struct rcu_head rcu;
+	unsigned long gen;
+	size_t len;
+	char string[];
+};
+
+static unsigned long file_systems_gen;
+static struct file_systems_string __rcu *file_systems_string;
 
-static struct file_system_type *file_systems;
-static DEFINE_RWLOCK(file_systems_lock);
+static void invalidate_filesystems_string(void);
+#else
+static inline void invalidate_filesystems_string(void) { }
+#endif
 
 /* WARNING: This can be used only if we _already_ own a reference */
 struct file_system_type *get_filesystem(struct file_system_type *fs)
@@ -46,14 +73,15 @@ void put_filesystem(struct file_system_type *fs)
 	module_put(fs->owner);
 }
 
-static struct file_system_type **find_filesystem(const char *name, unsigned len)
+static struct file_system_type *find_filesystem(const char *name, unsigned len)
 {
-	struct file_system_type **p;
-	for (p = &file_systems; *p; p = &(*p)->next)
-		if (strncmp((*p)->name, name, len) == 0 &&
-		    !(*p)->name[len])
-			break;
-	return p;
+	struct file_system_type *fs;
+
+	hlist_for_each_entry_rcu(fs, &file_systems, list,
+				 lockdep_is_held(&file_systems_lock))
+		if (strncmp(fs->name, name, len) == 0 && !fs->name[len])
+			return fs;
+	return NULL;
 }
 
 /**
@@ -64,33 +92,27 @@ static struct file_system_type **find_filesystem(const char *name, unsigned len)
  *	is aware of for mount and other syscalls. Returns 0 on success,
  *	or a negative errno code on an error.
  *
- *	The &struct file_system_type that is passed is linked into the kernel 
+ *	The &struct file_system_type that is passed is linked into the kernel
  *	structures and must not be freed until the file system has been
  *	unregistered.
  */
- 
-int register_filesystem(struct file_system_type * fs)
+int register_filesystem(struct file_system_type *fs)
 {
-	int res = 0;
-	struct file_system_type ** p;
-
 	if (fs->parameters &&
 	    !fs_validate_description(fs->name, fs->parameters))
 		return -EINVAL;
 
 	BUG_ON(strchr(fs->name, '.'));
-	if (fs->next)
+	if (!hlist_unhashed_lockless(&fs->list))
 		return -EBUSY;
-	write_lock(&file_systems_lock);
-	p = find_filesystem(fs->name, strlen(fs->name));
-	if (*p)
-		res = -EBUSY;
-	else
-		*p = fs;
-	write_unlock(&file_systems_lock);
-	return res;
-}
 
+	guard(spinlock)(&file_systems_lock);
+	if (find_filesystem(fs->name, strlen(fs->name)))
+		return -EBUSY;
+	hlist_add_tail_rcu(&fs->list, &file_systems);
+	invalidate_filesystems_string();
+	return 0;
+}
 EXPORT_SYMBOL(register_filesystem);
 
 /**
@@ -100,94 +122,79 @@ EXPORT_SYMBOL(register_filesystem);
  *	Remove a file system that was previously successfully registered
  *	with the kernel. An error is returned if the file system is not found.
  *	Zero is returned on a success.
- *	
+ *
  *	Once this function has returned the &struct file_system_type structure
  *	may be freed or reused.
  */
- 
-int unregister_filesystem(struct file_system_type * fs)
+int unregister_filesystem(struct file_system_type *fs)
 {
-	struct file_system_type ** tmp;
-
-	write_lock(&file_systems_lock);
-	tmp = &file_systems;
-	while (*tmp) {
-		if (fs == *tmp) {
-			*tmp = fs->next;
-			fs->next = NULL;
-			write_unlock(&file_systems_lock);
-			synchronize_rcu();
-			return 0;
-		}
-		tmp = &(*tmp)->next;
+	scoped_guard(spinlock, &file_systems_lock) {
+		if (hlist_unhashed(&fs->list))
+			return -EINVAL;
+		hlist_del_init_rcu(&fs->list);
+		invalidate_filesystems_string();
 	}
-	write_unlock(&file_systems_lock);
-
-	return -EINVAL;
+	synchronize_rcu();
+	return 0;
 }
-
 EXPORT_SYMBOL(unregister_filesystem);
 
 #ifdef CONFIG_SYSFS_SYSCALL
-static int fs_index(const char __user * __name)
+static int fs_index(const char __user *__name)
 {
-	struct file_system_type * tmp;
+	struct file_system_type *p;
 	char *name __free(kfree) = strndup_user(__name, PATH_MAX);
-	int err, index;
+	int index = 0;
 
 	if (IS_ERR(name))
 		return PTR_ERR(name);
 
-	err = -EINVAL;
-	read_lock(&file_systems_lock);
-	for (tmp=file_systems, index=0 ; tmp ; tmp=tmp->next, index++) {
-		if (strcmp(tmp->name, name) == 0) {
-			err = index;
-			break;
-		}
+	guard(rcu)();
+	hlist_for_each_entry_rcu(p, &file_systems, list) {
+		if (strcmp(p->name, name) == 0)
+			return index;
+		index++;
 	}
-	read_unlock(&file_systems_lock);
-	return err;
+	return -EINVAL;
 }
 
-static int fs_name(unsigned int index, char __user * buf)
+static int fs_name(unsigned int index, char __user *buf)
 {
-	struct file_system_type * tmp;
-	int len, res = -EINVAL;
-
-	read_lock(&file_systems_lock);
-	for (tmp = file_systems; tmp; tmp = tmp->next, index--) {
-		if (index == 0) {
-			if (try_module_get(tmp->owner))
-				res = 0;
+	struct file_system_type *p, *found = NULL;
+	int len, res;
+
+	scoped_guard(rcu) {
+		hlist_for_each_entry_rcu(p, &file_systems, list) {
+			if (index--)
+				continue;
+			if (try_module_get(p->owner))
+				found = p;
 			break;
 		}
 	}
-	read_unlock(&file_systems_lock);
-	if (res)
-		return res;
+	if (!found)
+		return -EINVAL;
 
 	/* OK, we got the reference, so we can safely block */
-	len = strlen(tmp->name) + 1;
-	res = copy_to_user(buf, tmp->name, len) ? -EFAULT : 0;
-	put_filesystem(tmp);
+	len = strlen(found->name) + 1;
+	res = copy_to_user(buf, found->name, len) ? -EFAULT : 0;
+	put_filesystem(found);
 	return res;
 }
 
 static int fs_maxindex(void)
 {
-	struct file_system_type * tmp;
-	int index;
+	struct file_system_type *p;
+	int index = 0;
 
-	read_lock(&file_systems_lock);
-	for (tmp = file_systems, index = 0 ; tmp ; tmp = tmp->next, index++)
-		;
-	read_unlock(&file_systems_lock);
+	guard(rcu)();
+	hlist_for_each_entry_rcu(p, &file_systems, list)
+		index++;
 	return index;
 }
 
 /*
- * Whee.. Weird sysv syscall. 
+ * Whee.. Weird sysv syscall.
  */
 SYSCALL_DEFINE3(sysfs, int, option, unsigned long, arg1, unsigned long, arg2)
 {
@@ -216,8 +223,8 @@ int __init list_bdev_fs_names(char *buf, size_t size)
 	size_t len;
 	int count = 0;
 
-	read_lock(&file_systems_lock);
-	for (p = file_systems; p; p = p->next) {
+	guard(rcu)();
+	hlist_for_each_entry_rcu(p, &file_systems, list) {
 		if (!(p->fs_flags & FS_REQUIRES_DEV))
 			continue;
 		len = strlen(p->name) + 1;
@@ -230,30 +237,145 @@ int __init list_bdev_fs_names(char *buf, size_t size)
 		size -= len;
 		count++;
 	}
-	read_unlock(&file_systems_lock);
 	return count;
 }
 
 #ifdef CONFIG_PROC_FS
-static int filesystems_proc_show(struct seq_file *m, void *v)
+static void invalidate_filesystems_string(void)
+{
+	struct file_systems_string *old;
+
+	lockdep_assert_held_write(&file_systems_lock);
+	file_systems_gen++;
+	old = rcu_replace_pointer(file_systems_string, NULL,
+			   lockdep_is_held(&file_systems_lock));
+	if (old)
+		kfree_rcu(old, rcu);
+}
+
+static __cold noinline int regen_filesystems_string(void)
+{
+	struct file_system_type *p;
+	struct file_systems_string *old, *new;
+	size_t newlen, usedlen;
+	unsigned long gen;
+
+retry:
+	newlen = 0;
+
+	/* pre-calc space for each fs */
+	spin_lock(&file_systems_lock);
+	gen = file_systems_gen;
+	hlist_for_each_entry_rcu(p, &file_systems, list) {
+		if (!(p->fs_flags & FS_REQUIRES_DEV))
+			newlen += strlen("nodev");
+		newlen += strlen("\t") + strlen(p->name) +  strlen("\n");
+	}
+	spin_unlock(&file_systems_lock);
+
+	new = kmalloc(offsetof(struct file_systems_string, string) + newlen + 1,
+		      GFP_KERNEL);
+	if (!new)
+		return -ENOMEM;
+
+	new->gen = gen;
+	new->len = newlen;
+	new->string[newlen] = '\0';
+
+	spin_lock(&file_systems_lock);
+	old = file_systems_string;
+
+	/*
+	 * Did someone beat us to it?
+	 */
+	if (old && old->gen == file_systems_gen) {
+		kfree(new);
+		return 0;
+	}
+
+	/*
+	 * Did the list change in the meantime?
+	 */
+	if (gen != file_systems_gen) {
+		kfree(new);
+		goto retry;
+	}
+
+	/*
+	 * Populate the string.
+	 *
+	 * We know we have just enough space because we calculated the right
+	 * size the previous time we had the lock and confirmed the list has
+	 * not changed after reacquiring it.
+	 */
+	usedlen = 0;
+	hlist_for_each_entry_rcu(p, &file_systems, list) {
+		usedlen += sprintf(&new->string[usedlen], "%s\t%s\n",
+				   (p->fs_flags & FS_REQUIRES_DEV) ? "" : "nodev",
+				   p->name);
+	}
+
+	if (WARN_ON_ONCE(new->len != strlen(new->string))) {
+		/*
+		 * Should never happen of course, keep this in case someone changes string
+		 * generation above and messes it up.
+		 */
+		spin_unlock(&file_systems_lock);
+		if (old)
+			kfree_rcu(old, rcu);
+		return -EINVAL;
+	}
+
+	/*
+	 * Paired with consume fence in READ_ONCE() in filesystems_proc_show()
+	 */
+	smp_store_release(&file_systems_string, new);
+	spin_unlock(&file_systems_lock);
+	if (old)
+		kfree_rcu(old, rcu);
+	return 0;
+}
+
+static __cold noinline int filesystems_proc_show_fallback(struct seq_file *m, void *v)
 {
-	struct file_system_type * tmp;
+	struct file_system_type *p;
 
-	read_lock(&file_systems_lock);
-	tmp = file_systems;
-	while (tmp) {
+	guard(rcu)();
+	hlist_for_each_entry_rcu(p, &file_systems, list) {
 		seq_printf(m, "%s\t%s\n",
-			(tmp->fs_flags & FS_REQUIRES_DEV) ? "" : "nodev",
-			tmp->name);
-		tmp = tmp->next;
+			   (p->fs_flags & FS_REQUIRES_DEV) ? "" : "nodev",
+			   p->name);
 	}
-	read_unlock(&file_systems_lock);
 	return 0;
 }
 
+static int filesystems_proc_show(struct seq_file *m, void *v)
+{
+	struct file_systems_string *fss;
+
+	for (;;) {
+		scoped_guard(rcu) {
+			fss = rcu_dereference(file_systems_string);
+			if (likely(fss)) {
+				seq_write(m, fss->string, fss->len);
+				return 0;
+			}
+		}
+
+		int err = regen_filesystems_string();
+		if (unlikely(err))
+			return filesystems_proc_show_fallback(m, v);
+	}
+}
+
 static int __init proc_filesystems_init(void)
 {
-	proc_create_single("filesystems", 0, NULL, filesystems_proc_show);
+	struct proc_dir_entry *pde;
+
+	pde = proc_create_single("filesystems", 0, NULL, filesystems_proc_show);
+	if (!pde)
+		return -ENOMEM;
+	proc_make_permanent(pde);
 	return 0;
 }
 module_init(proc_filesystems_init);
@@ -263,11 +385,10 @@ static struct file_system_type *__get_fs_type(const char *name, int len)
 {
 	struct file_system_type *fs;
 
-	read_lock(&file_systems_lock);
-	fs = *(find_filesystem(name, len));
+	guard(rcu)();
+	fs = find_filesystem(name, len);
 	if (fs && !try_module_get(fs->owner))
 		fs = NULL;
-	read_unlock(&file_systems_lock);
 	return fs;
 }
 
@@ -291,5 +412,4 @@ struct file_system_type *get_fs_type(const char *name)
 	}
 	return fs;
 }
-
 EXPORT_SYMBOL(get_fs_type);
diff --git a/fs/fuse/Makefile b/fs/fuse/Makefile
index 22ad9538dfc4b..245e67852b03e 100644
--- a/fs/fuse/Makefile
+++ b/fs/fuse/Makefile
@@ -11,7 +11,8 @@ obj-$(CONFIG_CUSE) += cuse.o
 obj-$(CONFIG_VIRTIO_FS) += virtiofs.o
 
 fuse-y := trace.o	# put trace.o first so we see ftrace errors sooner
-fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o
+fuse-y += dev.o dir.o file.o inode.o control.o xattr.o acl.o readdir.o ioctl.o req_timeout.o req.o
+fuse-y += poll.o notify.o
 fuse-y += iomode.o
 fuse-$(CONFIG_FUSE_DAX) += dax.o
 fuse-$(CONFIG_FUSE_PASSTHROUGH) += passthrough.o backing.o
diff --git a/fs/fuse/args.h b/fs/fuse/args.h
new file mode 100644
index 0000000000000..ecfe51a192af0
--- /dev/null
+++ b/fs/fuse/args.h
@@ -0,0 +1,65 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _FS_FUSE_ARGS_H
+#define _FS_FUSE_ARGS_H
+
+#include <linux/types.h>
+
+struct fuse_mount;
+
+/** One input argument of a request */
+struct fuse_in_arg {
+	unsigned size;
+	const void *value;
+};
+
+/** One output argument of a request */
+struct fuse_arg {
+	unsigned size;
+	void *value;
+};
+
+struct fuse_args {
+	u64 nodeid;
+	u32 opcode;
+	u32 uid;
+	u32 gid;
+	u32 pid;
+	u8 in_numargs;
+	u8 out_numargs;
+	u8 ext_idx;
+	bool force:1;
+	bool noreply:1;
+	bool nocreds:1;
+	bool in_pages:1;
+	bool out_pages:1;
+	bool user_pages:1;
+	bool out_argvar:1;
+	bool page_zeroing:1;
+	bool page_replace:1;
+	bool may_block:1;
+	bool is_ext:1;
+	bool is_pinned:1;
+	bool invalidate_vmap:1;
+	bool abort_on_kill:1;
+	struct fuse_in_arg in_args[4];
+	struct fuse_arg out_args[2];
+	void (*end)(struct fuse_args *args, int error);
+	/* Used for kvec iter backed by vmalloc address */
+	void *vmap_base;
+};
+
+/** FUSE folio descriptor */
+struct fuse_folio_desc {
+	unsigned int length;
+	unsigned int offset;
+};
+
+struct fuse_args_pages {
+	struct fuse_args args;
+	struct folio **folios;
+	struct fuse_folio_desc *descs;
+	unsigned int num_folios;
+};
+
+#endif /* _FS_FUSE_ARGS_H */
diff --git a/fs/fuse/backing.c b/fs/fuse/backing.c
index d95dfa48483f0..472b6afa7dfff 100644
--- a/fs/fuse/backing.c
+++ b/fs/fuse/backing.c
@@ -5,6 +5,7 @@
  * Copyright (c) 2023 CTERA Networks.
  */
 
+#include "dev.h"
 #include "fuse_i.h"
 
 #include <linux/file.h>
@@ -118,7 +119,7 @@ int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map)
 		goto out_fput;
 
 	fb->file = file;
-	fb->cred = prepare_creds();
+	fb->cred = get_current_cred();
 	refcount_set(&fb->count, 1);
 
 	res = fuse_backing_id_alloc(fc, fb);
diff --git a/fs/fuse/control.c b/fs/fuse/control.c
index f902a7fb4630c..925a154884996 100644
--- a/fs/fuse/control.c
+++ b/fs/fuse/control.c
@@ -7,6 +7,7 @@
 */
 
 #include "fuse_i.h"
+#include "dev.h"
 
 #include <linux/init.h>
 #include <linux/module.h>
@@ -37,9 +38,7 @@ static ssize_t fuse_conn_abort_write(struct file *file, const char __user *buf,
 {
 	struct fuse_conn *fc = fuse_ctl_file_conn_get(file);
 	if (fc) {
-		if (fc->abort_err)
-			fc->aborted = true;
-		fuse_abort_conn(fc);
+		fuse_chan_abort(fc->chan, fc->abort_err);
 		fuse_conn_put(fc);
 	}
 	return count;
@@ -57,7 +56,7 @@ static ssize_t fuse_conn_waiting_read(struct file *file, char __user *buf,
 		if (!fc)
 			return 0;
 
-		value = atomic_read(&fc->num_waiting);
+		value = fuse_chan_num_waiting(fc->chan);
 		file->private_data = (void *)value;
 		fuse_conn_put(fc);
 	}
@@ -111,7 +110,7 @@ static ssize_t fuse_conn_max_background_read(struct file *file,
 	if (!fc)
 		return 0;
 
-	val = READ_ONCE(fc->max_background);
+	val = fuse_chan_max_background(fc->chan);
 	fuse_conn_put(fc);
 
 	return fuse_conn_limit_read(file, buf, len, ppos, val);
@@ -129,12 +128,7 @@ static ssize_t fuse_conn_max_background_write(struct file *file,
 	if (ret > 0) {
 		struct fuse_conn *fc = fuse_ctl_file_conn_get(file);
 		if (fc) {
-			spin_lock(&fc->bg_lock);
-			fc->max_background = val;
-			fc->blocked = fc->num_background >= fc->max_background;
-			if (!fc->blocked)
-				wake_up(&fc->blocked_waitq);
-			spin_unlock(&fc->bg_lock);
+			fuse_chan_max_background_set(fc->chan, val);
 			fuse_conn_put(fc);
 		}
 	}
diff --git a/fs/fuse/cuse.c b/fs/fuse/cuse.c
index 174333633471b..bac327cfc7f14 100644
--- a/fs/fuse/cuse.c
+++ b/fs/fuse/cuse.c
@@ -51,6 +51,7 @@
 #include <linux/uio.h>
 #include <linux/user_namespace.h>
 
+#include "dev.h"
 #include "fuse_i.h"
 #include "fuse_dev_i.h"
 
@@ -306,6 +307,7 @@ struct cuse_init_args {
 	struct cuse_init_out out;
 	struct folio *folio;
 	struct fuse_folio_desc desc;
+	struct fuse_conn *fc;
 };
 
 /**
@@ -319,11 +321,10 @@ struct cuse_init_args {
  * required data structures for it.  Please read the comment at the
  * top of this file for high level overview.
  */
-static void cuse_process_init_reply(struct fuse_mount *fm,
-				    struct fuse_args *args, int error)
+static void cuse_process_init_reply(struct fuse_args *args, int error)
 {
-	struct fuse_conn *fc = fm->fc;
 	struct cuse_init_args *ia = container_of(args, typeof(*ia), ap.args);
+	struct fuse_conn *fc = ia->fc;
 	struct fuse_args_pages *ap = &ia->ap;
 	struct cuse_conn *cc = fc_to_cc(fc), *pos;
 	struct cuse_init_out *arg = &ia->out;
@@ -423,7 +424,7 @@ static void cuse_process_init_reply(struct fuse_mount *fm,
 err_region:
 	unregister_chrdev_region(devt, 1);
 err:
-	fuse_abort_conn(fc);
+	fuse_chan_abort(fc->chan, false);
 	goto out;
 }
 
@@ -466,6 +467,7 @@ static int cuse_send_init(struct cuse_conn *cc)
 	ap->descs = &ia->desc;
 	ia->folio = folio;
 	ia->desc.length = ap->args.out_args[1].size;
+	ia->fc = &cc->fc;
 	ap->args.end = cuse_process_init_reply;
 
 	rc = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
@@ -502,8 +504,12 @@ static int cuse_channel_open(struct inode *inode, struct file *file)
 {
 	struct fuse_dev *fud;
 	struct cuse_conn *cc;
+	struct fuse_chan *fch __free(fuse_chan_free) = fuse_dev_chan_new();
 	int rc;
 
+	if (!fch)
+		return -ENOMEM;
+
 	/* set up cuse_conn */
 	cc = kzalloc_obj(*cc);
 	if (!cc)
@@ -513,18 +519,16 @@ static int cuse_channel_open(struct inode *inode, struct file *file)
 	 * Limit the cuse channel to requests that can
 	 * be represented in file->f_cred->user_ns.
 	 */
-	fuse_conn_init(&cc->fc, &cc->fm, file->f_cred->user_ns,
-		       &fuse_dev_fiq_ops, NULL);
-
+	fuse_conn_init(&cc->fc, &cc->fm, file->f_cred->user_ns, no_free_ptr(fch));
 	cc->fc.release = cuse_fc_release;
-	fud = fuse_dev_alloc_install(&cc->fc);
+	fud = fuse_dev_alloc_install(cc->fc.chan);
 	fuse_conn_put(&cc->fc);
 	if (!fud)
 		return -ENOMEM;
 
 	INIT_LIST_HEAD(&cc->list);
 
-	cc->fc.initialized = 1;
+	cc->fc.chan->initialized = 1;
 	rc = cuse_send_init(cc);
 	if (rc) {
 		fuse_dev_put(fud);
@@ -549,7 +553,7 @@ static int cuse_channel_open(struct inode *inode, struct file *file)
 static int cuse_channel_release(struct inode *inode, struct file *file)
 {
 	struct fuse_dev *fud = __fuse_get_dev(file);
-	struct cuse_conn *cc = fc_to_cc(fud->fc);
+	struct cuse_conn *cc = fc_to_cc(fud->chan->conn);
 
 	/* remove from the conntbl, no more access from this point on */
 	mutex_lock(&cuse_lock);
@@ -581,7 +585,7 @@ static ssize_t cuse_class_waiting_show(struct device *dev,
 {
 	struct cuse_conn *cc = dev_get_drvdata(dev);
 
-	return sprintf(buf, "%d\n", atomic_read(&cc->fc.num_waiting));
+	return sprintf(buf, "%d\n", atomic_read(&cc->fc.chan->num_waiting));
 }
 static DEVICE_ATTR(waiting, 0400, cuse_class_waiting_show, NULL);
 
@@ -591,7 +595,7 @@ static ssize_t cuse_class_abort_store(struct device *dev,
 {
 	struct cuse_conn *cc = dev_get_drvdata(dev);
 
-	fuse_abort_conn(&cc->fc);
+	fuse_chan_abort(cc->fc.chan, false);
 	return count;
 }
 static DEVICE_ATTR(abort, 0200, NULL, cuse_class_abort_store);
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index 5dda7080f4a90..be4e66ed633c4 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -6,9 +6,9 @@
   See the file COPYING.
 */
 
+#include "dev.h"
+#include "args.h"
 #include "dev_uring_i.h"
-#include "fuse_i.h"
-#include "fuse_dev_i.h"
 
 #include <linux/init.h>
 #include <linux/module.h>
@@ -30,118 +30,26 @@
 MODULE_ALIAS_MISCDEV(FUSE_MINOR);
 MODULE_ALIAS("devname:fuse");
 
-static struct kmem_cache *fuse_req_cachep;
-
-const unsigned long fuse_timeout_timer_freq =
-	secs_to_jiffies(FUSE_TIMEOUT_TIMER_FREQ);
-
-bool fuse_request_expired(struct fuse_conn *fc, struct list_head *list)
-{
-	struct fuse_req *req;
-
-	req = list_first_entry_or_null(list, struct fuse_req, list);
-	if (!req)
-		return false;
-	return time_is_before_jiffies(req->create_time + fc->timeout.req_timeout);
-}
-
-static bool fuse_fpq_processing_expired(struct fuse_conn *fc, struct list_head *processing)
-{
-	int i;
-
-	for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
-		if (fuse_request_expired(fc, &processing[i]))
-			return true;
-
-	return false;
-}
-
-/*
- * Check if any requests aren't being completed by the time the request timeout
- * elapses. To do so, we:
- * - check the fiq pending list
- * - check the bg queue
- * - check the fpq io and processing lists
- *
- * To make this fast, we only check against the head request on each list since
- * these are generally queued in order of creation time (eg newer requests get
- * queued to the tail). We might miss a few edge cases (eg requests transitioning
- * between lists, re-sent requests at the head of the pending list having a
- * later creation time than other requests on that list, etc.) but that is fine
- * since if the request never gets fulfilled, it will eventually be caught.
- */
-void fuse_check_timeout(struct work_struct *work)
-{
-	struct delayed_work *dwork = to_delayed_work(work);
-	struct fuse_conn *fc = container_of(dwork, struct fuse_conn,
-					    timeout.work);
-	struct fuse_iqueue *fiq = &fc->iq;
-	struct fuse_dev *fud;
-	struct fuse_pqueue *fpq;
-	bool expired = false;
-
-	if (!atomic_read(&fc->num_waiting))
-	    goto out;
-
-	spin_lock(&fiq->lock);
-	expired = fuse_request_expired(fc, &fiq->pending);
-	spin_unlock(&fiq->lock);
-	if (expired)
-		goto abort_conn;
-
-	spin_lock(&fc->bg_lock);
-	expired = fuse_request_expired(fc, &fc->bg_queue);
-	spin_unlock(&fc->bg_lock);
-	if (expired)
-		goto abort_conn;
-
-	spin_lock(&fc->lock);
-	if (!fc->connected) {
-		spin_unlock(&fc->lock);
-		return;
-	}
-	list_for_each_entry(fud, &fc->devices, entry) {
-		fpq = &fud->pq;
-		spin_lock(&fpq->lock);
-		if (fuse_request_expired(fc, &fpq->io) ||
-		    fuse_fpq_processing_expired(fc, fpq->processing)) {
-			spin_unlock(&fpq->lock);
-			spin_unlock(&fc->lock);
-			goto abort_conn;
-		}
-
-		spin_unlock(&fpq->lock);
-	}
-	spin_unlock(&fc->lock);
+static DECLARE_WAIT_QUEUE_HEAD(fuse_dev_waitq);
 
-	if (fuse_uring_request_expired(fc))
-	    goto abort_conn;
-
-out:
-	queue_delayed_work(system_percpu_wq, &fc->timeout.work,
-			   fuse_timeout_timer_freq);
-	return;
-
-abort_conn:
-	fuse_abort_conn(fc);
-}
+static struct kmem_cache *fuse_req_cachep;
 
-static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req)
+static void fuse_request_init(struct fuse_chan *fch, struct fuse_req *req)
 {
 	INIT_LIST_HEAD(&req->list);
 	INIT_LIST_HEAD(&req->intr_entry);
 	init_waitqueue_head(&req->waitq);
 	refcount_set(&req->count, 1);
 	__set_bit(FR_PENDING, &req->flags);
-	req->fm = fm;
+	req->chan = fch;
 	req->create_time = jiffies;
 }
 
-static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags)
+static struct fuse_req *fuse_request_alloc(struct fuse_chan *fch, gfp_t flags)
 {
 	struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags);
 	if (req)
-		fuse_request_init(fm, req);
+		fuse_request_init(fch, req);
 
 	return req;
 }
@@ -162,110 +70,86 @@ static void __fuse_put_request(struct fuse_req *req)
 	refcount_dec(&req->count);
 }
 
-void fuse_set_initialized(struct fuse_conn *fc)
+void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param)
 {
+	if (param) {
+		fch->minor = param->minor;
+		fch->max_write = param->max_write;
+		fch->max_pages = param->max_pages;
+	}
+
 	/* Make sure stores before this are seen on another CPU */
 	smp_wmb();
-	fc->initialized = 1;
+	fch->initialized = 1;
+	wake_up_all(&fch->blocked_waitq);
 }
 
-static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
+static bool fuse_block_alloc(struct fuse_chan *fch, bool for_background)
 {
-	return !fc->initialized || (for_background && fc->blocked) ||
-	       (fc->io_uring && fc->connected && !fuse_uring_ready(fc));
+	return !fch->initialized || (for_background && fch->blocked) ||
+	       (fch->io_uring && fch->connected && !fuse_uring_ready(fch));
 }
 
-static void fuse_drop_waiting(struct fuse_conn *fc)
+static void fuse_drop_waiting(struct fuse_chan *fch)
 {
 	/*
-	 * lockess check of fc->connected is okay, because atomic_dec_and_test()
-	 * provides a memory barrier matched with the one in fuse_wait_aborted()
+	 * lockess check of fch->connected is okay, because atomic_dec_and_test()
+	 * provides a memory barrier matched with the one in fuse_chan_wait_aborted()
 	 * to ensure no wake-up is missed.
 	 */
-	if (atomic_dec_and_test(&fc->num_waiting) &&
-	    !READ_ONCE(fc->connected)) {
+	if (atomic_dec_and_test(&fch->num_waiting) &&
+	    !READ_ONCE(fch->connected)) {
 		/* wake up aborters */
-		wake_up_all(&fc->blocked_waitq);
+		wake_up_all(&fch->blocked_waitq);
 	}
 }
 
 static void fuse_put_request(struct fuse_req *req);
 
-static struct fuse_req *fuse_get_req(struct mnt_idmap *idmap,
-				     struct fuse_mount *fm,
-				     bool for_background)
+static struct fuse_req *fuse_get_req(struct fuse_chan *fch, bool for_background)
 {
-	struct fuse_conn *fc = fm->fc;
 	struct fuse_req *req;
-	bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP);
-	kuid_t fsuid;
-	kgid_t fsgid;
 	int err;
 
-	atomic_inc(&fc->num_waiting);
+	atomic_inc(&fch->num_waiting);
 
-	if (fuse_block_alloc(fc, for_background)) {
+	if (fuse_block_alloc(fch, for_background)) {
 		err = -EINTR;
-		if (wait_event_state_exclusive(fc->blocked_waitq,
-				!fuse_block_alloc(fc, for_background),
+		if (wait_event_state_exclusive(fch->blocked_waitq,
+				!fuse_block_alloc(fch, for_background),
 				(TASK_KILLABLE | TASK_FREEZABLE)))
 			goto out;
 	}
-	/* Matches smp_wmb() in fuse_set_initialized() */
+
+	/* Matches smp_wmb() in fuse_chan_set_initialized() */
 	smp_rmb();
 
 	err = -ENOTCONN;
-	if (!fc->connected)
-		goto out;
-
-	err = -ECONNREFUSED;
-	if (fc->conn_error)
+	if (!fch->connected)
 		goto out;
 
-	req = fuse_request_alloc(fm, GFP_KERNEL);
+	req = fuse_request_alloc(fch, GFP_KERNEL);
 	err = -ENOMEM;
 	if (!req) {
 		if (for_background)
-			wake_up(&fc->blocked_waitq);
+			wake_up(&fch->blocked_waitq);
 		goto out;
 	}
 
-	req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
-
 	__set_bit(FR_WAITING, &req->flags);
 	if (for_background)
 		__set_bit(FR_BACKGROUND, &req->flags);
 
-	/*
-	 * Keep the old behavior when idmappings support was not
-	 * declared by a FUSE server.
-	 *
-	 * For those FUSE servers who support idmapped mounts,
-	 * we send UID/GID only along with "inode creation"
-	 * fuse requests, otherwise idmap == &invalid_mnt_idmap and
-	 * req->in.h.{u,g}id will be equal to FUSE_INVALID_UIDGID.
-	 */
-	fsuid = no_idmap ? current_fsuid() : mapped_fsuid(idmap, fc->user_ns);
-	fsgid = no_idmap ? current_fsgid() : mapped_fsgid(idmap, fc->user_ns);
-	req->in.h.uid = from_kuid(fc->user_ns, fsuid);
-	req->in.h.gid = from_kgid(fc->user_ns, fsgid);
-
-	if (no_idmap && unlikely(req->in.h.uid == ((uid_t)-1) ||
-				 req->in.h.gid == ((gid_t)-1))) {
-		fuse_put_request(req);
-		return ERR_PTR(-EOVERFLOW);
-	}
-
 	return req;
 
  out:
-	fuse_drop_waiting(fc);
+	fuse_drop_waiting(fch);
 	return ERR_PTR(err);
 }
 
 static void fuse_put_request(struct fuse_req *req)
 {
-	struct fuse_conn *fc = req->fm->fc;
+	struct fuse_chan *fch = req->chan;
 
 	if (refcount_dec_and_test(&req->count)) {
 		if (test_bit(FR_BACKGROUND, &req->flags)) {
@@ -273,15 +157,15 @@ static void fuse_put_request(struct fuse_req *req)
 			 * We get here in the unlikely case that a background
 			 * request was allocated but not sent
 			 */
-			spin_lock(&fc->bg_lock);
-			if (!fc->blocked)
-				wake_up(&fc->blocked_waitq);
-			spin_unlock(&fc->bg_lock);
+			spin_lock(&fch->bg_lock);
+			if (!fch->blocked)
+				wake_up(&fch->blocked_waitq);
+			spin_unlock(&fch->bg_lock);
 		}
 
 		if (test_bit(FR_WAITING, &req->flags)) {
 			__clear_bit(FR_WAITING, &req->flags);
-			fuse_drop_waiting(fc);
+			fuse_drop_waiting(fch);
 		}
 
 		fuse_request_free(req);
@@ -335,6 +219,11 @@ __releases(fiq->lock)
 	spin_unlock(&fiq->lock);
 }
 
+struct fuse_forget_link *fuse_alloc_forget(void)
+{
+	return kzalloc_obj(struct fuse_forget_link, GFP_KERNEL_ACCOUNT);
+}
+
 void fuse_dev_queue_forget(struct fuse_iqueue *fiq,
 			   struct fuse_forget_link *forget)
 {
@@ -406,12 +295,271 @@ static void fuse_dev_queue_req(struct fuse_iqueue *fiq, struct fuse_req *req)
 	}
 }
 
-const struct fuse_iqueue_ops fuse_dev_fiq_ops = {
+static const struct fuse_iqueue_ops fuse_dev_fiq_ops = {
 	.send_forget	= fuse_dev_queue_forget,
 	.send_interrupt	= fuse_dev_queue_interrupt,
 	.send_req	= fuse_dev_queue_req,
 };
-EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops);
+
+void fuse_iqueue_init(struct fuse_iqueue *fiq, const struct fuse_iqueue_ops *ops, void *priv)
+{
+	spin_lock_init(&fiq->lock);
+	init_waitqueue_head(&fiq->waitq);
+	INIT_LIST_HEAD(&fiq->pending);
+	INIT_LIST_HEAD(&fiq->interrupts);
+	fiq->forget_list_tail = &fiq->forget_list_head;
+	fiq->connected = 1;
+	fiq->ops = ops;
+	fiq->priv = priv;
+}
+EXPORT_SYMBOL_GPL(fuse_iqueue_init);
+
+void fuse_chan_release(struct fuse_chan *fch)
+{
+	struct fuse_iqueue *fiq = &fch->iq;
+
+	if (fiq->ops->release)
+		fiq->ops->release(fiq);
+
+	if (fch->timeout.req_timeout)
+		cancel_delayed_work_sync(&fch->timeout.work);
+}
+
+void fuse_chan_free(struct fuse_chan *fch)
+{
+	WARN_ON(!list_empty(&fch->devices));
+	kfree(fch->pq_prealloc);
+	kfree(fch);
+}
+EXPORT_SYMBOL_GPL(fuse_chan_free);
+
+struct fuse_chan *fuse_chan_new(void)
+{
+	struct fuse_chan *fch = kzalloc_obj(struct fuse_chan);
+	if (!fch)
+		return NULL;
+
+	spin_lock_init(&fch->lock);
+	INIT_LIST_HEAD(&fch->devices);
+	spin_lock_init(&fch->bg_lock);
+	INIT_LIST_HEAD(&fch->bg_queue);
+	init_waitqueue_head(&fch->blocked_waitq);
+	atomic_set(&fch->num_waiting, 0);
+	fch->max_background = FUSE_DEFAULT_MAX_BACKGROUND;
+	fch->initialized = 0;
+	fch->blocked = 0;
+	fch->connected = 1;
+	fch->timeout.req_timeout = 0;
+
+	return fch;
+}
+EXPORT_SYMBOL_GPL(fuse_chan_new);
+
+struct list_head *fuse_pqueue_alloc(void)
+{
+	struct list_head *pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE);
+
+	if (pq) {
+		for (int i = 0; i < FUSE_PQ_HASH_SIZE; i++)
+			INIT_LIST_HEAD(&pq[i]);
+	}
+	return pq;
+}
+
+struct fuse_chan *fuse_dev_chan_new(void)
+{
+	struct fuse_chan *fch __free(kfree) = fuse_chan_new();
+	if (!fch)
+		return NULL;
+
+	fch->pq_prealloc = fuse_pqueue_alloc();
+	if (!fch->pq_prealloc)
+		return NULL;
+
+	fuse_iqueue_init(&fch->iq, &fuse_dev_fiq_ops, NULL);
+
+	return no_free_ptr(fch);
+}
+EXPORT_SYMBOL_GPL(fuse_dev_chan_new);
+
+unsigned int fuse_chan_num_background(struct fuse_chan *fch)
+{
+	return fch->num_background;
+}
+
+unsigned int fuse_chan_max_background(struct fuse_chan *fch)
+{
+	return READ_ONCE(fch->max_background);
+}
+
+void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val)
+{
+	spin_lock(&fch->bg_lock);
+	fch->max_background = val;
+	fch->blocked = fch->num_background >= fch->max_background;
+	if (!fch->blocked)
+		wake_up(&fch->blocked_waitq);
+	spin_unlock(&fch->bg_lock);
+}
+
+unsigned int fuse_chan_num_waiting(struct fuse_chan *fch)
+{
+	return atomic_read(&fch->num_waiting);
+}
+
+void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc)
+{
+	fch->conn = fc;
+}
+
+void fuse_chan_io_uring_enable(struct fuse_chan *fch)
+{
+	fch->io_uring = 1;
+}
+
+void fuse_pqueue_init(struct fuse_pqueue *fpq)
+{
+	spin_lock_init(&fpq->lock);
+	INIT_LIST_HEAD(&fpq->io);
+	fpq->connected = 1;
+	fpq->processing = NULL;
+}
+
+static struct fuse_dev *fuse_dev_alloc_no_pq(void)
+{
+	struct fuse_dev *fud;
+
+	fud = kzalloc_obj(struct fuse_dev);
+	if (!fud)
+		return NULL;
+
+	refcount_set(&fud->ref, 1);
+	fuse_pqueue_init(&fud->pq);
+
+	return fud;
+}
+
+struct fuse_dev *fuse_dev_alloc(void)
+{
+	struct fuse_dev *fud __free(kfree) = fuse_dev_alloc_no_pq();
+	if (!fud)
+		return NULL;
+
+	fud->pq.processing = fuse_pqueue_alloc();
+	if (!fud->pq.processing)
+		return NULL;
+
+	return no_free_ptr(fud);
+}
+EXPORT_SYMBOL_GPL(fuse_dev_alloc);
+
+/*
+ * Installs @fch into @fud, return true on success.  "Consumes" @pq in either case.
+ */
+static bool fuse_dev_install_with_pq(struct fuse_dev *fud, struct fuse_chan *fch,
+				     struct list_head *pq)
+{
+	struct fuse_chan *old_fch;
+
+	guard(spinlock)(&fch->lock);
+	/*
+	 * Pairs with:
+	 *  - xchg() in fuse_dev_release()
+	 *  - smp_load_acquire() in fuse_dev_fc_get()
+	 */
+	old_fch = cmpxchg(&fud->chan, NULL, fch);
+	if (old_fch) {
+		/*
+		 * failed to set fud->chan because
+		 *  - it was already set to a different fc
+		 *  - it was set to disconneted
+		 */
+		kfree(pq);
+		return false;
+	}
+	if (pq) {
+		WARN_ON(fud->pq.processing);
+		fud->pq.processing = pq;
+	}
+	list_add_tail(&fud->entry, &fch->devices);
+	fuse_conn_get(fch->conn);
+	wake_up_all(&fuse_dev_waitq);
+	return true;
+}
+
+void fuse_dev_install(struct fuse_dev *fud, struct fuse_chan *fch)
+{
+	struct list_head *pq = fch->pq_prealloc;
+
+	fch->pq_prealloc = NULL;
+	if (!fuse_dev_install_with_pq(fud, fch, pq)) {
+		/* Channel is not usable without a dev */
+		fuse_chan_abort(fch, false);
+	}
+}
+EXPORT_SYMBOL_GPL(fuse_dev_install);
+
+struct fuse_dev *fuse_dev_alloc_install(struct fuse_chan *fch)
+{
+	struct fuse_dev *fud;
+
+	fud = fuse_dev_alloc_no_pq();
+	if (!fud)
+		return NULL;
+
+	fuse_dev_install(fud, fch);
+	return fud;
+}
+EXPORT_SYMBOL_GPL(fuse_dev_alloc_install);
+
+void fuse_dev_put(struct fuse_dev *fud)
+{
+	struct fuse_chan *fch;
+
+	if (!refcount_dec_and_test(&fud->ref))
+		return;
+
+	fch = fuse_dev_chan_get(fud);
+	if (fch && fch != FUSE_DEV_CHAN_DISCONNECTED) {
+		/* This is the virtiofs case (fuse_dev_release() not called) */
+		spin_lock(&fch->lock);
+		list_del(&fud->entry);
+		spin_unlock(&fch->lock);
+
+		fuse_conn_put(fch->conn);
+	}
+	kfree(fud->pq.processing);
+	kfree(fud);
+}
+EXPORT_SYMBOL_GPL(fuse_dev_put);
+
+bool fuse_dev_is_installed(struct fuse_dev *fud)
+{
+	struct fuse_chan *fch = fuse_dev_chan_get(fud);
+
+	return fch != NULL && fch != FUSE_DEV_CHAN_DISCONNECTED;
+}
+
+/*
+ * Checks if @fc matches the one installed in @fud
+ */
+bool fuse_dev_verify(struct fuse_dev *fud, struct fuse_chan *fch)
+{
+	return fuse_dev_chan_get(fud) == fch;
+}
+
+bool fuse_dev_is_sync_init(struct fuse_dev *fud)
+{
+	return fud->sync_init;
+}
+
+struct fuse_dev *fuse_dev_grab(struct file *file)
+{
+	struct fuse_dev *fud = fuse_file_to_fud(file);
+
+	refcount_inc(&fud->ref);
+	return fud;
+}
 
 static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req)
 {
@@ -421,10 +569,10 @@ static void fuse_send_one(struct fuse_iqueue *fiq, struct fuse_req *req)
 	fiq->ops->send_req(fiq, req);
 }
 
-void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
-		       u64 nodeid, u64 nlookup)
+void fuse_chan_queue_forget(struct fuse_chan *fch, struct fuse_forget_link *forget,
+			    u64 nodeid, u64 nlookup)
 {
-	struct fuse_iqueue *fiq = &fc->iq;
+	struct fuse_iqueue *fiq = &fch->iq;
 
 	forget->forget_one.nodeid = nodeid;
 	forget->forget_one.nlookup = nlookup;
@@ -432,17 +580,17 @@ void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
 	fiq->ops->send_forget(fiq, forget);
 }
 
-static void flush_bg_queue(struct fuse_conn *fc)
+static void flush_bg_queue(struct fuse_chan *fch)
 {
-	struct fuse_iqueue *fiq = &fc->iq;
+	struct fuse_iqueue *fiq = &fch->iq;
 
-	while (fc->active_background < fc->max_background &&
-	       !list_empty(&fc->bg_queue)) {
+	while (fch->active_background < fch->max_background &&
+	       !list_empty(&fch->bg_queue)) {
 		struct fuse_req *req;
 
-		req = list_first_entry(&fc->bg_queue, struct fuse_req, list);
+		req = list_first_entry(&fch->bg_queue, struct fuse_req, list);
 		list_del(&req->list);
-		fc->active_background++;
+		fch->active_background++;
 		fuse_send_one(fiq, req);
 	}
 }
@@ -457,9 +605,8 @@ static void flush_bg_queue(struct fuse_conn *fc)
  */
 void fuse_request_end(struct fuse_req *req)
 {
-	struct fuse_mount *fm = req->fm;
-	struct fuse_conn *fc = fm->fc;
-	struct fuse_iqueue *fiq = &fc->iq;
+	struct fuse_chan *fch = req->chan;
+	struct fuse_iqueue *fiq = &fch->iq;
 
 	if (test_and_set_bit(FR_FINISHED, &req->flags))
 		goto put_request;
@@ -478,33 +625,33 @@ void fuse_request_end(struct fuse_req *req)
 	WARN_ON(test_bit(FR_PENDING, &req->flags));
 	WARN_ON(test_bit(FR_SENT, &req->flags));
 	if (test_bit(FR_BACKGROUND, &req->flags)) {
-		spin_lock(&fc->bg_lock);
+		spin_lock(&fch->bg_lock);
 		clear_bit(FR_BACKGROUND, &req->flags);
-		if (fc->num_background == fc->max_background) {
-			fc->blocked = 0;
-			wake_up(&fc->blocked_waitq);
-		} else if (!fc->blocked) {
+		if (fch->num_background == fch->max_background) {
+			fch->blocked = 0;
+			wake_up(&fch->blocked_waitq);
+		} else if (!fch->blocked) {
 			/*
 			 * Wake up next waiter, if any.  It's okay to use
 			 * waitqueue_active(), as we've already synced up
-			 * fc->blocked with waiters with the wake_up() call
+			 * fch->blocked with waiters with the wake_up() call
 			 * above.
 			 */
-			if (waitqueue_active(&fc->blocked_waitq))
-				wake_up(&fc->blocked_waitq);
+			if (waitqueue_active(&fch->blocked_waitq))
+				wake_up(&fch->blocked_waitq);
 		}
 
-		fc->num_background--;
-		fc->active_background--;
-		flush_bg_queue(fc);
-		spin_unlock(&fc->bg_lock);
+		fch->num_background--;
+		fch->active_background--;
+		flush_bg_queue(fch);
+		spin_unlock(&fch->bg_lock);
 	} else {
 		/* Wake up waiter sleeping in request_wait_answer() */
 		wake_up(&req->waitq);
 	}
 
 	if (test_bit(FR_ASYNC, &req->flags))
-		req->args->end(fm, req->args, req->out.h.error);
+		req->args->end(req->args, req->out.h.error);
 put_request:
 	fuse_put_request(req);
 }
@@ -512,7 +659,7 @@ EXPORT_SYMBOL_GPL(fuse_request_end);
 
 static int queue_interrupt(struct fuse_req *req)
 {
-	struct fuse_iqueue *fiq = &req->fm->fc->iq;
+	struct fuse_iqueue *fiq = &req->chan->iq;
 
 	/* Check for we've sent request to interrupt this req */
 	if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags)))
@@ -543,11 +690,11 @@ bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock)
 
 static void request_wait_answer(struct fuse_req *req)
 {
-	struct fuse_conn *fc = req->fm->fc;
-	struct fuse_iqueue *fiq = &fc->iq;
+	struct fuse_chan *fch = req->chan;
+	struct fuse_iqueue *fiq = &fch->iq;
 	int err;
 
-	if (!fc->no_interrupt) {
+	if (!fch->no_interrupt) {
 		/* Any signal may interrupt this */
 		err = wait_event_interruptible(req->waitq,
 					test_bit(FR_FINISHED, &req->flags));
@@ -571,7 +718,7 @@ static void request_wait_answer(struct fuse_req *req)
 			return;
 
 		if (req->args->abort_on_kill) {
-			fuse_abort_conn(fc);
+			fuse_chan_abort(fch, false);
 			return;
 		}
 
@@ -592,7 +739,7 @@ static void request_wait_answer(struct fuse_req *req)
 
 static void __fuse_request_send(struct fuse_req *req)
 {
-	struct fuse_iqueue *fiq = &req->fm->fc->iq;
+	struct fuse_iqueue *fiq = &req->chan->iq;
 
 	BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
 
@@ -606,12 +753,12 @@ static void __fuse_request_send(struct fuse_req *req)
 	smp_rmb();
 }
 
-static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
+static void fuse_adjust_compat(struct fuse_chan *fch, struct fuse_args *args)
 {
-	if (fc->minor < 4 && args->opcode == FUSE_STATFS)
+	if (fch->minor < 4 && args->opcode == FUSE_STATFS)
 		args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE;
 
-	if (fc->minor < 9) {
+	if (fch->minor < 9) {
 		switch (args->opcode) {
 		case FUSE_LOOKUP:
 		case FUSE_CREATE:
@@ -627,7 +774,7 @@ static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
 			break;
 		}
 	}
-	if (fc->minor < 12) {
+	if (fch->minor < 12) {
 		switch (args->opcode) {
 		case FUSE_CREATE:
 			args->in_args[0].size = sizeof(struct fuse_open_in);
@@ -639,25 +786,13 @@ static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
 	}
 }
 
-static void fuse_force_creds(struct fuse_req *req)
-{
-	struct fuse_conn *fc = req->fm->fc;
-
-	if (!req->fm->sb || req->fm->sb->s_iflags & SB_I_NOIDMAP) {
-		req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid());
-		req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid());
-	} else {
-		req->in.h.uid = FUSE_INVALID_UIDGID;
-		req->in.h.gid = FUSE_INVALID_UIDGID;
-	}
-
-	req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
-}
-
 static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args)
 {
 	req->in.h.opcode = args->opcode;
 	req->in.h.nodeid = args->nodeid;
+	req->in.h.uid = args->uid;
+	req->in.h.gid = args->gid;
+	req->in.h.pid = args->pid;
 	req->args = args;
 	if (args->is_ext)
 		req->in.h.total_extlen = args->in_args[args->ext_idx].size / 8;
@@ -665,33 +800,26 @@ static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args)
 		__set_bit(FR_ASYNC, &req->flags);
 }
 
-ssize_t __fuse_simple_request(struct mnt_idmap *idmap,
-			      struct fuse_mount *fm,
-			      struct fuse_args *args)
+ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args)
 {
-	struct fuse_conn *fc = fm->fc;
 	struct fuse_req *req;
 	ssize_t ret;
 
 	if (args->force) {
-		atomic_inc(&fc->num_waiting);
-		req = fuse_request_alloc(fm, GFP_KERNEL | __GFP_NOFAIL);
-
-		if (!args->nocreds)
-			fuse_force_creds(req);
+		atomic_inc(&fch->num_waiting);
+		req = fuse_request_alloc(fch, GFP_KERNEL | __GFP_NOFAIL);
 
 		__set_bit(FR_WAITING, &req->flags);
 		if (!args->abort_on_kill)
 			__set_bit(FR_FORCE, &req->flags);
 	} else {
-		WARN_ON(args->nocreds);
-		req = fuse_get_req(idmap, fm, false);
+		req = fuse_get_req(fch, false);
 		if (IS_ERR(req))
 			return PTR_ERR(req);
 	}
 
-	/* Needs to be done after fuse_get_req() so that fc->minor is valid */
-	fuse_adjust_compat(fc, args);
+	/* Needs to be done after fuse_get_req() so that fch->minor is valid */
+	fuse_adjust_compat(fch, args);
 	fuse_args_to_req(req, args);
 
 	if (!args->noreply)
@@ -708,10 +836,9 @@ ssize_t __fuse_simple_request(struct mnt_idmap *idmap,
 }
 
 #ifdef CONFIG_FUSE_IO_URING
-static bool fuse_request_queue_background_uring(struct fuse_conn *fc,
-					       struct fuse_req *req)
+static bool fuse_request_queue_background_uring(struct fuse_req *req)
 {
-	struct fuse_iqueue *fiq = &fc->iq;
+	struct fuse_iqueue *fiq = &req->chan->iq;
 
 	req->in.h.len = sizeof(struct fuse_in_header) +
 		fuse_len_args(req->args->in_numargs,
@@ -727,50 +854,46 @@ static bool fuse_request_queue_background_uring(struct fuse_conn *fc,
  */
 static int fuse_request_queue_background(struct fuse_req *req)
 {
-	struct fuse_mount *fm = req->fm;
-	struct fuse_conn *fc = fm->fc;
+	struct fuse_chan *fch = req->chan;
 	bool queued = false;
 
 	WARN_ON(!test_bit(FR_BACKGROUND, &req->flags));
 	if (!test_bit(FR_WAITING, &req->flags)) {
 		__set_bit(FR_WAITING, &req->flags);
-		atomic_inc(&fc->num_waiting);
+		atomic_inc(&fch->num_waiting);
 	}
 	__set_bit(FR_ISREPLY, &req->flags);
 
 #ifdef CONFIG_FUSE_IO_URING
-	if (fuse_uring_ready(fc))
-		return fuse_request_queue_background_uring(fc, req);
+	if (fuse_uring_ready(fch))
+		return fuse_request_queue_background_uring(req);
 #endif
 
-	spin_lock(&fc->bg_lock);
-	if (likely(fc->connected)) {
-		fc->num_background++;
-		if (fc->num_background == fc->max_background)
-			fc->blocked = 1;
-		list_add_tail(&req->list, &fc->bg_queue);
-		flush_bg_queue(fc);
+	spin_lock(&fch->bg_lock);
+	if (likely(fch->connected)) {
+		fch->num_background++;
+		if (fch->num_background == fch->max_background)
+			fch->blocked = 1;
+		list_add_tail(&req->list, &fch->bg_queue);
+		flush_bg_queue(fch);
 		queued = true;
 	}
-	spin_unlock(&fc->bg_lock);
+	spin_unlock(&fch->bg_lock);
 
 	return queued;
 }
 
-int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args,
-			    gfp_t gfp_flags)
+int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags)
 {
 	struct fuse_req *req;
 
 	if (args->force) {
-		WARN_ON(!args->nocreds);
-		req = fuse_request_alloc(fm, gfp_flags);
+		req = fuse_request_alloc(fch, gfp_flags);
 		if (!req)
 			return -ENOMEM;
 		__set_bit(FR_BACKGROUND, &req->flags);
 	} else {
-		WARN_ON(args->nocreds);
-		req = fuse_get_req(&invalid_mnt_idmap, fm, true);
+		req = fuse_get_req(fch, true);
 		if (IS_ERR(req))
 			return PTR_ERR(req);
 	}
@@ -784,15 +907,13 @@ int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(fuse_simple_background);
 
-static int fuse_simple_notify_reply(struct fuse_mount *fm,
-				    struct fuse_args *args, u64 unique)
+int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique)
 {
 	struct fuse_req *req;
-	struct fuse_iqueue *fiq = &fm->fc->iq;
+	struct fuse_iqueue *fiq = &fch->iq;
 
-	req = fuse_get_req(&invalid_mnt_idmap, fm, false);
+	req = fuse_get_req(fch, false);
 	if (IS_ERR(req))
 		return PTR_ERR(req);
 
@@ -1122,8 +1243,8 @@ static int fuse_ref_folio(struct fuse_copy_state *cs, struct folio *folio,
  * Copy a folio in the request to/from the userspace buffer.  Must be
  * done atomically
  */
-static int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
-			   unsigned offset, unsigned count, int zeroing)
+int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
+		    unsigned offset, unsigned count, int zeroing)
 {
 	int err;
 	struct folio *folio = *foliop;
@@ -1204,7 +1325,7 @@ static int fuse_copy_folios(struct fuse_copy_state *cs, unsigned nbytes,
 }
 
 /* Copy a single argument in the request to/from userspace buffer */
-static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
+int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
 {
 	while (size) {
 		if (!cs->len) {
@@ -1256,7 +1377,7 @@ static int request_pending(struct fuse_iqueue *fiq)
  */
 static int fuse_read_interrupt(struct fuse_iqueue *fiq,
 			       struct fuse_copy_state *cs,
-			       size_t nbytes, struct fuse_req *req)
+			       struct fuse_req *req)
 __releases(fiq->lock)
 {
 	struct fuse_in_header ih;
@@ -1273,8 +1394,6 @@ __releases(fiq->lock)
 	arg.unique = req->in.h.unique;
 
 	spin_unlock(&fiq->lock);
-	if (nbytes < reqsize)
-		return -EINVAL;
 
 	err = fuse_copy_one(cs, &ih, sizeof(ih));
 	if (!err)
@@ -1307,8 +1426,7 @@ static struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
 }
 
 static int fuse_read_single_forget(struct fuse_iqueue *fiq,
-				   struct fuse_copy_state *cs,
-				   size_t nbytes)
+				   struct fuse_copy_state *cs)
 __releases(fiq->lock)
 {
 	int err;
@@ -1325,8 +1443,6 @@ __releases(fiq->lock)
 
 	spin_unlock(&fiq->lock);
 	kfree(forget);
-	if (nbytes < ih.len)
-		return -EINVAL;
 
 	err = fuse_copy_one(cs, &ih, sizeof(ih));
 	if (!err)
@@ -1354,11 +1470,6 @@ __releases(fiq->lock)
 		.len = sizeof(ih) + sizeof(arg),
 	};
 
-	if (nbytes < ih.len) {
-		spin_unlock(&fiq->lock);
-		return -EINVAL;
-	}
-
 	max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
 	head = fuse_dequeue_forget(fiq, max_forgets, &count);
 	spin_unlock(&fiq->lock);
@@ -1388,13 +1499,13 @@ __releases(fiq->lock)
 	return ih.len;
 }
 
-static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
+static int fuse_read_forget(struct fuse_chan *fch, struct fuse_iqueue *fiq,
 			    struct fuse_copy_state *cs,
 			    size_t nbytes)
 __releases(fiq->lock)
 {
-	if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
-		return fuse_read_single_forget(fiq, cs, nbytes);
+	if (fch->minor < 16 || fiq->forget_list_head.next->next == NULL)
+		return fuse_read_single_forget(fiq, cs);
 	else
 		return fuse_read_batch_forget(fiq, cs, nbytes);
 }
@@ -1412,8 +1523,8 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
 				struct fuse_copy_state *cs, size_t nbytes)
 {
 	ssize_t err;
-	struct fuse_conn *fc = fud->fc;
-	struct fuse_iqueue *fiq = &fc->iq;
+	struct fuse_chan *fch = fud->chan;
+	struct fuse_iqueue *fiq = &fch->iq;
 	struct fuse_pqueue *fpq = &fud->pq;
 	struct fuse_req *req;
 	struct fuse_args *args;
@@ -1435,7 +1546,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
 	if (nbytes < max_t(size_t, FUSE_MIN_READ_BUFFER,
 			   sizeof(struct fuse_in_header) +
 			   sizeof(struct fuse_write_in) +
-			   fc->max_write))
+			   fch->max_write))
 		return -EINVAL;
 
  restart:
@@ -1454,19 +1565,19 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
 	}
 
 	if (!fiq->connected) {
-		err = fc->aborted ? -ECONNABORTED : -ENODEV;
+		err = fch->abort_with_err ? -ECONNABORTED : -ENODEV;
 		goto err_unlock;
 	}
 
 	if (!list_empty(&fiq->interrupts)) {
 		req = list_entry(fiq->interrupts.next, struct fuse_req,
 				 intr_entry);
-		return fuse_read_interrupt(fiq, cs, nbytes, req);
+		return fuse_read_interrupt(fiq, cs, req);
 	}
 
 	if (forget_pending(fiq)) {
 		if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
-			return fuse_read_forget(fc, fiq, cs, nbytes);
+			return fuse_read_forget(fch, fiq, cs, nbytes);
 
 		if (fiq->forget_batch <= -8)
 			fiq->forget_batch = 16;
@@ -1492,7 +1603,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
 	spin_lock(&fpq->lock);
 	/*
 	 *  Must not put request on fpq->io queue after having been shut down by
-	 *  fuse_abort_conn()
+	 *  fuse_chan_abort()
 	 */
 	if (!fpq->connected) {
 		req->out.h.error = err = -ECONNABORTED;
@@ -1510,7 +1621,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
 	spin_lock(&fpq->lock);
 	clear_bit(FR_LOCKED, &req->flags);
 	if (!fpq->connected) {
-		err = fc->aborted ? -ECONNABORTED : -ENODEV;
+		err = fch->abort_with_err ? -ECONNABORTED : -ENODEV;
 		goto out_end;
 	}
 	if (err) {
@@ -1548,7 +1659,7 @@ static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
 
 static int fuse_dev_open(struct inode *inode, struct file *file)
 {
-	struct fuse_dev *fud = fuse_dev_alloc();
+	struct fuse_dev *fud = fuse_dev_alloc_no_pq();
 
 	if (!fud)
 		return -ENOMEM;
@@ -1562,9 +1673,15 @@ struct fuse_dev *fuse_get_dev(struct file *file)
 	struct fuse_dev *fud = fuse_file_to_fud(file);
 	int err;
 
-	err = wait_event_interruptible(fuse_dev_waitq, fuse_dev_fc_get(fud) != NULL);
-	if (err)
-		return ERR_PTR(err);
+	if (unlikely(!fuse_dev_chan_get(fud))) {
+		/* only block waiting for mount if sync init was requested */
+		if (!fud->sync_init)
+			return ERR_PTR(-EPERM);
+
+		err = wait_event_interruptible(fuse_dev_waitq, fuse_dev_chan_get(fud) != NULL);
+		if (err)
+			return ERR_PTR(err);
+	}
 
 	return fud;
 }
@@ -1636,344 +1753,6 @@ static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
 	return ret;
 }
 
-static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
-			    struct fuse_copy_state *cs)
-{
-	struct fuse_notify_poll_wakeup_out outarg;
-	int err;
-
-	if (size != sizeof(outarg))
-		return -EINVAL;
-
-	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
-	if (err)
-		return err;
-
-	fuse_copy_finish(cs);
-	return fuse_notify_poll_wakeup(fc, &outarg);
-}
-
-static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
-				   struct fuse_copy_state *cs)
-{
-	struct fuse_notify_inval_inode_out outarg;
-	int err;
-
-	if (size != sizeof(outarg))
-		return -EINVAL;
-
-	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
-	if (err)
-		return err;
-	fuse_copy_finish(cs);
-
-	down_read(&fc->killsb);
-	err = fuse_reverse_inval_inode(fc, outarg.ino,
-				       outarg.off, outarg.len);
-	up_read(&fc->killsb);
-	return err;
-}
-
-static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
-				   struct fuse_copy_state *cs)
-{
-	struct fuse_notify_inval_entry_out outarg;
-	int err;
-	char *buf;
-	struct qstr name;
-
-	if (size < sizeof(outarg))
-		return -EINVAL;
-
-	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
-	if (err)
-		return err;
-
-	if (outarg.namelen > fc->name_max)
-		return -ENAMETOOLONG;
-
-	err = -EINVAL;
-	if (size != sizeof(outarg) + outarg.namelen + 1)
-		return -EINVAL;
-
-	buf = kzalloc(outarg.namelen + 1, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	name.name = buf;
-	name.len = outarg.namelen;
-	err = fuse_copy_one(cs, buf, outarg.namelen + 1);
-	if (err)
-		goto err;
-	fuse_copy_finish(cs);
-	buf[outarg.namelen] = 0;
-
-	down_read(&fc->killsb);
-	err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags);
-	up_read(&fc->killsb);
-err:
-	kfree(buf);
-	return err;
-}
-
-static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
-			      struct fuse_copy_state *cs)
-{
-	struct fuse_notify_delete_out outarg;
-	int err;
-	char *buf;
-	struct qstr name;
-
-	if (size < sizeof(outarg))
-		return -EINVAL;
-
-	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
-	if (err)
-		return err;
-
-	if (outarg.namelen > fc->name_max)
-		return -ENAMETOOLONG;
-
-	if (size != sizeof(outarg) + outarg.namelen + 1)
-		return -EINVAL;
-
-	buf = kzalloc(outarg.namelen + 1, GFP_KERNEL);
-	if (!buf)
-		return -ENOMEM;
-
-	name.name = buf;
-	name.len = outarg.namelen;
-	err = fuse_copy_one(cs, buf, outarg.namelen + 1);
-	if (err)
-		goto err;
-	fuse_copy_finish(cs);
-	buf[outarg.namelen] = 0;
-
-	down_read(&fc->killsb);
-	err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0);
-	up_read(&fc->killsb);
-err:
-	kfree(buf);
-	return err;
-}
-
-static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
-			     struct fuse_copy_state *cs)
-{
-	struct fuse_notify_store_out outarg;
-	struct inode *inode;
-	struct address_space *mapping;
-	u64 nodeid;
-	int err;
-	unsigned int num;
-	loff_t file_size;
-	loff_t pos;
-	loff_t end;
-
-	if (size < sizeof(outarg))
-		return -EINVAL;
-
-	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
-	if (err)
-		return err;
-
-	if (size - sizeof(outarg) != outarg.size)
-		return -EINVAL;
-
-	if (outarg.offset >= MAX_LFS_FILESIZE)
-		return -EINVAL;
-
-	nodeid = outarg.nodeid;
-	pos = outarg.offset;
-	num = min(outarg.size, MAX_LFS_FILESIZE - pos);
-
-	down_read(&fc->killsb);
-
-	err = -ENOENT;
-	inode = fuse_ilookup(fc, nodeid,  NULL);
-	if (!inode)
-		goto out_up_killsb;
-
-	mapping = inode->i_mapping;
-	file_size = i_size_read(inode);
-	end = pos + num;
-	if (end > file_size) {
-		file_size = end;
-		fuse_write_update_attr(inode, file_size, num);
-	}
-
-	while (num) {
-		struct folio *folio;
-		unsigned int folio_offset;
-		unsigned int nr_bytes;
-		pgoff_t index = pos >> PAGE_SHIFT;
-
-		folio = filemap_grab_folio(mapping, index);
-		err = PTR_ERR(folio);
-		if (IS_ERR(folio))
-			goto out_iput;
-
-		folio_offset = offset_in_folio(folio, pos);
-		nr_bytes = min(num, folio_size(folio) - folio_offset);
-
-		err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0);
-		if (!folio_test_uptodate(folio) && !err && folio_offset == 0 &&
-		    (nr_bytes == folio_size(folio) || file_size == end)) {
-			folio_zero_segment(folio, nr_bytes, folio_size(folio));
-			folio_mark_uptodate(folio);
-		}
-		folio_unlock(folio);
-		folio_put(folio);
-
-		if (err)
-			goto out_iput;
-
-		pos += nr_bytes;
-		num -= nr_bytes;
-	}
-
-	err = 0;
-
-out_iput:
-	iput(inode);
-out_up_killsb:
-	up_read(&fc->killsb);
-	return err;
-}
-
-struct fuse_retrieve_args {
-	struct fuse_args_pages ap;
-	struct fuse_notify_retrieve_in inarg;
-};
-
-static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args,
-			      int error)
-{
-	struct fuse_retrieve_args *ra =
-		container_of(args, typeof(*ra), ap.args);
-
-	release_pages(ra->ap.folios, ra->ap.num_folios);
-	kfree(ra);
-}
-
-static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
-			 struct fuse_notify_retrieve_out *outarg)
-{
-	int err;
-	struct address_space *mapping = inode->i_mapping;
-	loff_t file_size;
-	unsigned int num;
-	unsigned int offset;
-	size_t total_len = 0;
-	unsigned int num_pages;
-	struct fuse_conn *fc = fm->fc;
-	struct fuse_retrieve_args *ra;
-	size_t args_size = sizeof(*ra);
-	struct fuse_args_pages *ap;
-	struct fuse_args *args;
-	loff_t pos = outarg->offset;
-
-	offset = offset_in_page(pos);
-	file_size = i_size_read(inode);
-
-	num = min(outarg->size, fc->max_write);
-	if (pos > file_size)
-		num = 0;
-	else if (num > file_size - pos)
-		num = file_size - pos;
-
-	num_pages = DIV_ROUND_UP(num + offset, PAGE_SIZE);
-	num_pages = min(num_pages, fc->max_pages);
-	num = min(num, num_pages << PAGE_SHIFT);
-
-	args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0]));
-
-	ra = kzalloc(args_size, GFP_KERNEL);
-	if (!ra)
-		return -ENOMEM;
-
-	ap = &ra->ap;
-	ap->folios = (void *) (ra + 1);
-	ap->descs = (void *) (ap->folios + num_pages);
-
-	args = &ap->args;
-	args->nodeid = outarg->nodeid;
-	args->opcode = FUSE_NOTIFY_REPLY;
-	args->in_numargs = 3;
-	args->in_pages = true;
-	args->end = fuse_retrieve_end;
-
-	while (num && ap->num_folios < num_pages) {
-		struct folio *folio;
-		unsigned int folio_offset;
-		unsigned int nr_bytes;
-		pgoff_t index = pos >> PAGE_SHIFT;
-
-		folio = filemap_get_folio(mapping, index);
-		if (IS_ERR(folio))
-			break;
-
-		folio_offset = offset_in_folio(folio, pos);
-		nr_bytes = min(folio_size(folio) - folio_offset, num);
-
-		ap->folios[ap->num_folios] = folio;
-		ap->descs[ap->num_folios].offset = folio_offset;
-		ap->descs[ap->num_folios].length = nr_bytes;
-		ap->num_folios++;
-
-		pos += nr_bytes;
-		num -= nr_bytes;
-		total_len += nr_bytes;
-	}
-	ra->inarg.offset = outarg->offset;
-	ra->inarg.size = total_len;
-	fuse_set_zero_arg0(args);
-	args->in_args[1].size = sizeof(ra->inarg);
-	args->in_args[1].value = &ra->inarg;
-	args->in_args[2].size = total_len;
-
-	err = fuse_simple_notify_reply(fm, args, outarg->notify_unique);
-	if (err)
-		fuse_retrieve_end(fm, args, err);
-
-	return err;
-}
-
-static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
-				struct fuse_copy_state *cs)
-{
-	struct fuse_notify_retrieve_out outarg;
-	struct fuse_mount *fm;
-	struct inode *inode;
-	u64 nodeid;
-	int err;
-
-	if (size != sizeof(outarg))
-		return -EINVAL;
-
-	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
-	if (err)
-		return err;
-
-	fuse_copy_finish(cs);
-
-	if (outarg.offset >= MAX_LFS_FILESIZE)
-		return -EINVAL;
-
-	down_read(&fc->killsb);
-	err = -ENOENT;
-	nodeid = outarg.nodeid;
-
-	inode = fuse_ilookup(fc, nodeid, &fm);
-	if (inode) {
-		err = fuse_retrieve(fm, inode, &outarg);
-		iput(inode);
-	}
-	up_read(&fc->killsb);
-
-	return err;
-}
-
 /*
  * Resending all processing queue requests.
  *
@@ -1987,21 +1766,21 @@ static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
  * if the FUSE daemon takes careful measures to avoid processing duplicated
  * non-idempotent requests.
  */
-static void fuse_resend(struct fuse_conn *fc)
+void fuse_chan_resend(struct fuse_chan *fch)
 {
 	struct fuse_dev *fud;
 	struct fuse_req *req, *next;
-	struct fuse_iqueue *fiq = &fc->iq;
+	struct fuse_iqueue *fiq = &fch->iq;
 	LIST_HEAD(to_queue);
 	unsigned int i;
 
-	spin_lock(&fc->lock);
-	if (!fc->connected) {
-		spin_unlock(&fc->lock);
+	spin_lock(&fch->lock);
+	if (!fch->connected) {
+		spin_unlock(&fch->lock);
 		return;
 	}
 
-	list_for_each_entry(fud, &fc->devices, entry) {
+	list_for_each_entry(fud, &fch->devices, entry) {
 		struct fuse_pqueue *fpq = &fud->pq;
 
 		spin_lock(&fpq->lock);
@@ -2009,7 +1788,7 @@ static void fuse_resend(struct fuse_conn *fc)
 			list_splice_tail_init(&fpq->processing[i], &to_queue);
 		spin_unlock(&fpq->lock);
 	}
-	spin_unlock(&fc->lock);
+	spin_unlock(&fch->lock);
 
 	list_for_each_entry_safe(req, next, &to_queue, list) {
 		set_bit(FR_PENDING, &req->flags);
@@ -2031,108 +1810,6 @@ static void fuse_resend(struct fuse_conn *fc)
 	fuse_dev_wake_and_unlock(fiq);
 }
 
-static int fuse_notify_resend(struct fuse_conn *fc)
-{
-	fuse_resend(fc);
-	return 0;
-}
-
-/*
- * Increments the fuse connection epoch.  This will result of dentries from
- * previous epochs to be invalidated.  Additionally, if inval_wq is set, a work
- * queue is scheduled to trigger the invalidation.
- */
-static int fuse_notify_inc_epoch(struct fuse_conn *fc)
-{
-	atomic_inc(&fc->epoch);
-	if (inval_wq)
-		schedule_work(&fc->epoch_work);
-
-	return 0;
-}
-
-static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size,
-			     struct fuse_copy_state *cs)
-{
-	struct fuse_notify_prune_out outarg;
-	const unsigned int batch = 512;
-	u64 *nodeids __free(kfree) = kmalloc(sizeof(u64) * batch, GFP_KERNEL);
-	unsigned int num, i;
-	int err;
-
-	if (!nodeids)
-		return -ENOMEM;
-
-	if (size < sizeof(outarg))
-		return -EINVAL;
-
-	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
-	if (err)
-		return err;
-
-	if (size - sizeof(outarg) != outarg.count * sizeof(u64))
-		return -EINVAL;
-
-	for (; outarg.count; outarg.count -= num) {
-		num = min(batch, outarg.count);
-		err = fuse_copy_one(cs, nodeids, num * sizeof(u64));
-		if (err)
-			return err;
-
-		scoped_guard(rwsem_read, &fc->killsb) {
-			for (i = 0; i < num; i++)
-				fuse_try_prune_one_inode(fc, nodeids[i]);
-		}
-	}
-	return 0;
-}
-
-static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
-		       unsigned int size, struct fuse_copy_state *cs)
-{
-	/*
-	 * Only allow notifications during while the connection is in an
-	 * initialized and connected state
-	 */
-	if (!fc->initialized || !fc->connected)
-		return -EINVAL;
-
-	/* Don't try to move folios (yet) */
-	cs->move_folios = false;
-
-	switch (code) {
-	case FUSE_NOTIFY_POLL:
-		return fuse_notify_poll(fc, size, cs);
-
-	case FUSE_NOTIFY_INVAL_INODE:
-		return fuse_notify_inval_inode(fc, size, cs);
-
-	case FUSE_NOTIFY_INVAL_ENTRY:
-		return fuse_notify_inval_entry(fc, size, cs);
-
-	case FUSE_NOTIFY_STORE:
-		return fuse_notify_store(fc, size, cs);
-
-	case FUSE_NOTIFY_RETRIEVE:
-		return fuse_notify_retrieve(fc, size, cs);
-
-	case FUSE_NOTIFY_DELETE:
-		return fuse_notify_delete(fc, size, cs);
-
-	case FUSE_NOTIFY_RESEND:
-		return fuse_notify_resend(fc);
-
-	case FUSE_NOTIFY_INC_EPOCH:
-		return fuse_notify_inc_epoch(fc);
-
-	case FUSE_NOTIFY_PRUNE:
-		return fuse_notify_prune(fc, size, cs);
-
-	default:
-		return -EINVAL;
-	}
-}
-
 /* Look up request on processing list by unique ID */
 struct fuse_req *fuse_request_find(struct fuse_pqueue *fpq, u64 unique)
 {
@@ -2185,7 +1862,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
 				 struct fuse_copy_state *cs, size_t nbytes)
 {
 	int err;
-	struct fuse_conn *fc = fud->fc;
+	struct fuse_chan *fch = fud->chan;
 	struct fuse_pqueue *fpq = &fud->pq;
 	struct fuse_req *req;
 	struct fuse_out_header oh;
@@ -2207,7 +1884,18 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
 	 * and error contains notification code.
 	 */
 	if (!oh.unique) {
-		err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
+		/*
+		 * Only allow notifications during while the connection is in an
+		 * initialized and connected state
+		 */
+		err = -EINVAL;
+		if (!fch->initialized || !fch->connected)
+			goto copy_finish;
+
+		/* Don't try to move folios (yet) */
+		cs->move_folios = false;
+
+		err = fuse_notify(fch->conn, oh.error, nbytes - sizeof(oh), cs);
 		goto copy_finish;
 	}
 
@@ -2235,7 +1923,7 @@ static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
 		if (nbytes != sizeof(struct fuse_out_header))
 			err = -EINVAL;
 		else if (oh.error == -ENOSYS)
-			fc->no_interrupt = 1;
+			fch->no_interrupt = 1;
 		else if (oh.error == -EAGAIN)
 			err = queue_interrupt(req);
 
@@ -2395,7 +2083,7 @@ static __poll_t fuse_dev_poll(struct file *file, poll_table *wait)
 	if (IS_ERR(fud))
 		return EPOLLERR;
 
-	fiq = &fud->fc->iq;
+	fiq = &fud->chan->iq;
 	poll_wait(file, &fiq->waitq, wait);
 
 	spin_lock(&fiq->lock);
@@ -2421,21 +2109,6 @@ void fuse_dev_end_requests(struct list_head *head)
 	}
 }
 
-static void end_polls(struct fuse_conn *fc)
-{
-	struct rb_node *p;
-
-	p = rb_first(&fc->polled_files);
-
-	while (p) {
-		struct fuse_file *ff;
-		ff = rb_entry(p, struct fuse_file, polled_node);
-		wake_up_interruptible_all(&ff->poll_wait);
-
-		p = rb_next(p);
-	}
-}
-
 /*
  * Abort all requests.
  *
@@ -2454,27 +2127,29 @@ static void end_polls(struct fuse_conn *fc)
  * is OK, the request will in that case be removed from the list before we touch
  * it.
  */
-void fuse_abort_conn(struct fuse_conn *fc)
+void fuse_chan_abort(struct fuse_chan *fch, bool abort_with_err)
 {
-	struct fuse_iqueue *fiq = &fc->iq;
+	struct fuse_iqueue *fiq = &fch->iq;
+
+	fch->abort_with_err = abort_with_err;
 
-	spin_lock(&fc->lock);
-	if (fc->connected) {
+	spin_lock(&fch->lock);
+	if (fch->connected) {
 		struct fuse_dev *fud;
 		struct fuse_req *req, *next;
 		LIST_HEAD(to_end);
 		unsigned int i;
 
-		if (fc->timeout.req_timeout)
-			cancel_delayed_work(&fc->timeout.work);
+		if (fch->timeout.req_timeout)
+			cancel_delayed_work(&fch->timeout.work);
 
-		/* Background queuing checks fc->connected under bg_lock */
-		spin_lock(&fc->bg_lock);
-		fc->connected = 0;
-		spin_unlock(&fc->bg_lock);
+		/* Background queuing checks fch->connected under bg_lock */
+		spin_lock(&fch->bg_lock);
+		fch->connected = 0;
+		spin_unlock(&fch->bg_lock);
 
-		fuse_set_initialized(fc);
-		list_for_each_entry(fud, &fc->devices, entry) {
+		fuse_chan_set_initialized(fch, 0);
+		list_for_each_entry(fud, &fch->devices, entry) {
 			struct fuse_pqueue *fpq = &fud->pq;
 
 			spin_lock(&fpq->lock);
@@ -2495,11 +2170,11 @@ void fuse_abort_conn(struct fuse_conn *fc)
 						      &to_end);
 			spin_unlock(&fpq->lock);
 		}
-		spin_lock(&fc->bg_lock);
-		fc->blocked = 0;
-		fc->max_background = UINT_MAX;
-		flush_bg_queue(fc);
-		spin_unlock(&fc->bg_lock);
+		spin_lock(&fch->bg_lock);
+		fch->blocked = 0;
+		fch->max_background = UINT_MAX;
+		flush_bg_queue(fch);
+		spin_unlock(&fch->bg_lock);
 
 		spin_lock(&fiq->lock);
 		fiq->connected = 0;
@@ -2511,39 +2186,39 @@ void fuse_abort_conn(struct fuse_conn *fc)
 		wake_up_all(&fiq->waitq);
 		spin_unlock(&fiq->lock);
 		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
-		end_polls(fc);
-		wake_up_all(&fc->blocked_waitq);
-		spin_unlock(&fc->lock);
+		fuse_end_polls(fch->conn);
+		wake_up_all(&fch->blocked_waitq);
+		spin_unlock(&fch->lock);
 
 		fuse_dev_end_requests(&to_end);
 
 		/*
-		 * fc->lock must not be taken to avoid conflicts with io-uring
+		 * fch->lock must not be taken to avoid conflicts with io-uring
 		 * locks
 		 */
-		fuse_uring_abort(fc);
+		fuse_uring_abort(fch);
 	} else {
-		spin_unlock(&fc->lock);
+		spin_unlock(&fch->lock);
 	}
 }
-EXPORT_SYMBOL_GPL(fuse_abort_conn);
+EXPORT_SYMBOL_GPL(fuse_chan_abort);
 
-void fuse_wait_aborted(struct fuse_conn *fc)
+void fuse_chan_wait_aborted(struct fuse_chan *fch)
 {
 	/* matches implicit memory barrier in fuse_drop_waiting() */
 	smp_mb();
-	wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0);
+	wait_event(fch->blocked_waitq, fuse_chan_num_waiting(fch) == 0);
 
-	fuse_uring_wait_stopped_queues(fc);
+	fuse_uring_wait_stopped_queues(fch);
 }
 
 int fuse_dev_release(struct inode *inode, struct file *file)
 {
 	struct fuse_dev *fud = fuse_file_to_fud(file);
 	/* Pairs with cmpxchg() in fuse_dev_install() */
-	struct fuse_conn *fc = xchg(&fud->fc, FUSE_DEV_FC_DISCONNECTED);
+	struct fuse_chan *fch = xchg(&fud->chan, FUSE_DEV_CHAN_DISCONNECTED);
 
-	if (fc) {
+	if (fch) {
 		struct fuse_pqueue *fpq = &fud->pq;
 		LIST_HEAD(to_end);
 		unsigned int i;
@@ -2557,17 +2232,17 @@ int fuse_dev_release(struct inode *inode, struct file *file)
 
 		fuse_dev_end_requests(&to_end);
 
-		spin_lock(&fc->lock);
+		spin_lock(&fch->lock);
 		list_del(&fud->entry);
 		/* Are we the last open device? */
-		last = list_empty(&fc->devices);
-		spin_unlock(&fc->lock);
+		last = list_empty(&fch->devices);
+		spin_unlock(&fch->lock);
 
 		if (last) {
-			WARN_ON(fc->iq.fasync != NULL);
-			fuse_abort_conn(fc);
+			WARN_ON(fch->iq.fasync != NULL);
+			fuse_chan_abort(fch, false);
 		}
-		fuse_conn_put(fc);
+		fuse_conn_put(fch->conn);
 	}
 	fuse_dev_put(fud);
 	return 0;
@@ -2582,13 +2257,14 @@ static int fuse_dev_fasync(int fd, struct file *file, int on)
 		return PTR_ERR(fud);
 
 	/* No locking - fasync_helper does its own locking */
-	return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
+	return fasync_helper(fd, file, on, &fud->chan->iq.fasync);
 }
 
 static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp)
 {
 	int oldfd;
 	struct fuse_dev *fud, *new_fud;
+	struct list_head *pq;
 
 	if (get_user(oldfd, argp))
 		return -EFAULT;
@@ -2608,12 +2284,14 @@ static long fuse_dev_ioctl_clone(struct file *file, __u32 __user *argp)
 	if (IS_ERR(fud))
 		return PTR_ERR(fud);
 
+	pq = fuse_pqueue_alloc();
+	if (!pq)
+		return -ENOMEM;
+
 	new_fud = fuse_file_to_fud(file);
-	if (fuse_dev_fc_get(new_fud))
+	if (!fuse_dev_install_with_pq(new_fud, fud->chan, pq))
 		return -EINVAL;
 
-	fuse_dev_install(new_fud, fud->fc);
-
 	return 0;
 }
 
@@ -2632,7 +2310,7 @@ static long fuse_dev_ioctl_backing_open(struct file *file,
 	if (copy_from_user(&map, argp, sizeof(map)))
 		return -EFAULT;
 
-	return fuse_backing_open(fud->fc, &map);
+	return fuse_backing_open(fud->chan->conn, &map);
 }
 
 static long fuse_dev_ioctl_backing_close(struct file *file, __u32 __user *argp)
@@ -2649,21 +2327,18 @@ static long fuse_dev_ioctl_backing_close(struct file *file, __u32 __user *argp)
 	if (get_user(backing_id, argp))
 		return -EFAULT;
 
-	return fuse_backing_close(fud->fc, backing_id);
+	return fuse_backing_close(fud->chan->conn, backing_id);
 }
 
 static long fuse_dev_ioctl_sync_init(struct file *file)
 {
-	int err = -EINVAL;
 	struct fuse_dev *fud = fuse_file_to_fud(file);
 
-	mutex_lock(&fuse_mutex);
-	if (!fuse_dev_fc_get(fud)) {
-		fud->sync_init = true;
-		err = 0;
-	}
-	mutex_unlock(&fuse_mutex);
-	return err;
+	if (fuse_dev_chan_get(fud))
+		return -EINVAL;
+
+	fud->sync_init = true;
+	return 0;
 }
 
 static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
@@ -2696,7 +2371,7 @@ static void fuse_dev_show_fdinfo(struct seq_file *seq, struct file *file)
 	if (!fud)
 		return;
 
-	seq_printf(seq, "fuse_connection:\t%u\n", fud->fc->dev);
+	seq_printf(seq, "fuse_connection:\t%u\n", fuse_conn_get_id(fud->chan->conn));
 }
 #endif
 
diff --git a/fs/fuse/dev.h b/fs/fuse/dev.h
new file mode 100644
index 0000000000000..aed69fd14c41d
--- /dev/null
+++ b/fs/fuse/dev.h
@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _FS_FUSE_DEV_H
+#define _FS_FUSE_DEV_H
+
+#include <linux/cleanup.h>
+
+/** Maximum number of outstanding background requests */
+#define FUSE_DEFAULT_MAX_BACKGROUND 12
+
+struct fuse_conn;
+struct fuse_chan;
+struct fuse_dev;
+struct fuse_args;
+struct fuse_copy_state;
+struct fuse_backing_map;
+struct file;
+struct folio;
+enum fuse_notify_code;
+
+struct fuse_chan_param {
+	unsigned int minor;
+	unsigned int max_write;
+	unsigned int max_pages;
+};
+
+struct fuse_chan *fuse_chan_new(void);
+struct fuse_chan *fuse_dev_chan_new(void);
+void fuse_chan_release(struct fuse_chan *fch);
+void fuse_chan_free(struct fuse_chan *fch);
+unsigned int fuse_chan_num_background(struct fuse_chan *fch);
+unsigned int fuse_chan_max_background(struct fuse_chan *fch);
+void fuse_chan_max_background_set(struct fuse_chan *fch, unsigned int val);
+unsigned int fuse_chan_num_waiting(struct fuse_chan *fch);
+void fuse_chan_set_fc(struct fuse_chan *fch, struct fuse_conn *fc);
+void fuse_chan_set_initialized(struct fuse_chan *fch, struct fuse_chan_param *param);
+void fuse_chan_io_uring_enable(struct fuse_chan *fch);
+ssize_t fuse_chan_send(struct fuse_chan *fch, struct fuse_args *args);
+int fuse_chan_send_bg(struct fuse_chan *fch, struct fuse_args *args, gfp_t gfp_flags);
+int fuse_chan_send_notify_reply(struct fuse_chan *fch, struct fuse_args *args, u64 unique);
+void fuse_chan_resend(struct fuse_chan *fch);
+
+struct fuse_forget_link *fuse_alloc_forget(void);
+void fuse_chan_queue_forget(struct fuse_chan *fch, struct fuse_forget_link *forget,
+			    u64 nodeid, u64 nlookup);
+
+DEFINE_FREE(fuse_chan_free, struct fuse_chan *, if (_T) fuse_chan_free(_T))
+
+/**
+ * Initialize the client device
+ */
+int fuse_dev_init(void);
+
+/**
+ * Cleanup the client device
+ */
+void fuse_dev_cleanup(void);
+
+void fuse_dev_install(struct fuse_dev *fud, struct fuse_chan *fch);
+bool fuse_dev_verify(struct fuse_dev *fud, struct fuse_chan *fch);
+void fuse_dev_put(struct fuse_dev *fud);
+bool fuse_dev_is_installed(struct fuse_dev *fud);
+bool fuse_dev_is_sync_init(struct fuse_dev *fud);
+struct fuse_dev *fuse_dev_grab(struct file *file);
+
+void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout);
+
+/* Abort all requests */
+void fuse_chan_abort(struct fuse_chan *fch, bool abort_with_err);
+void fuse_chan_wait_aborted(struct fuse_chan *fch);
+
+/**
+ * Acquire reference to fuse_conn
+ */
+struct fuse_conn *fuse_conn_get(struct fuse_conn *fc);
+
+/**
+ * Release reference to fuse_conn
+ */
+void fuse_conn_put(struct fuse_conn *fc);
+
+dev_t fuse_conn_get_id(struct fuse_conn *fc);
+
+void fuse_end_polls(struct fuse_conn *fc);
+int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
+		unsigned int size, struct fuse_copy_state *cs);
+
+int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map);
+int fuse_backing_close(struct fuse_conn *fc, int backing_id);
+
+int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size);
+int fuse_copy_folio(struct fuse_copy_state *cs, struct folio **foliop,
+		    unsigned offset, unsigned count, int zeroing);
+void fuse_copy_finish(struct fuse_copy_state *cs);
+
+#ifdef CONFIG_FUSE_IO_URING
+bool fuse_uring_enabled(void);
+void fuse_uring_destruct(struct fuse_chan *fch);
+#else /* CONFIG_FUSE_IO_URING */
+static inline bool fuse_uring_enabled(void)
+{
+	return false;
+}
+
+static inline void fuse_uring_destruct(struct fuse_chan *fch)
+{
+}
+#endif /* CONFIG_FUSE_IO_URING */
+
+#endif /* _FS_FUSE_DEV_H */
diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index 7b9822e8837bc..e467b23e68959 100644
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -4,9 +4,9 @@
  * Copyright (c) 2023-2024 DataDirect Networks.
  */
 
-#include "fuse_i.h"
+#include "dev.h"
+#include "args.h"
 #include "dev_uring_i.h"
-#include "fuse_dev_i.h"
 #include "fuse_trace.h"
 
 #include <linux/fs.h>
@@ -51,10 +51,10 @@ static struct fuse_ring_ent *uring_cmd_to_ring_ent(struct io_uring_cmd *cmd)
 static void fuse_uring_flush_bg(struct fuse_ring_queue *queue)
 {
 	struct fuse_ring *ring = queue->ring;
-	struct fuse_conn *fc = ring->fc;
+	struct fuse_chan *fch = ring->chan;
 
 	lockdep_assert_held(&queue->lock);
-	lockdep_assert_held(&fc->bg_lock);
+	lockdep_assert_held(&fch->bg_lock);
 
 	/*
 	 * Allow one bg request per queue, ignoring global fc limits.
@@ -62,14 +62,14 @@ static void fuse_uring_flush_bg(struct fuse_ring_queue *queue)
 	 * eliminates the need for remote queue wake-ups when global
 	 * limits are met but this queue has no more waiting requests.
 	 */
-	while ((fc->active_background < fc->max_background ||
+	while ((fch->active_background < fch->max_background ||
 		!queue->active_background) &&
 	       (!list_empty(&queue->fuse_req_bg_queue))) {
 		struct fuse_req *req;
 
 		req = list_first_entry(&queue->fuse_req_bg_queue,
 				       struct fuse_req, list);
-		fc->active_background++;
+		fch->active_background++;
 		queue->active_background++;
 
 		list_move_tail(&req->list, &queue->fuse_req_queue);
@@ -81,7 +81,7 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,
 {
 	struct fuse_ring_queue *queue = ent->queue;
 	struct fuse_ring *ring = queue->ring;
-	struct fuse_conn *fc = ring->fc;
+	struct fuse_chan *fch = ring->chan;
 
 	lockdep_assert_not_held(&queue->lock);
 	spin_lock(&queue->lock);
@@ -89,9 +89,9 @@ static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,
 	list_del_init(&req->list);
 	if (test_bit(FR_BACKGROUND, &req->flags)) {
 		queue->active_background--;
-		spin_lock(&fc->bg_lock);
+		spin_lock(&fch->bg_lock);
 		fuse_uring_flush_bg(queue);
-		spin_unlock(&fc->bg_lock);
+		spin_unlock(&fch->bg_lock);
 	}
 
 	spin_unlock(&queue->lock);
@@ -123,7 +123,7 @@ void fuse_uring_abort_end_requests(struct fuse_ring *ring)
 {
 	int qid;
 	struct fuse_ring_queue *queue;
-	struct fuse_conn *fc = ring->fc;
+	struct fuse_chan *fch = ring->chan;
 
 	for (qid = 0; qid < ring->nr_queues; qid++) {
 		queue = READ_ONCE(ring->queues[qid]);
@@ -132,17 +132,17 @@ void fuse_uring_abort_end_requests(struct fuse_ring *ring)
 
 		queue->stopped = true;
 
-		WARN_ON_ONCE(ring->fc->max_background != UINT_MAX);
+		WARN_ON_ONCE(fch->max_background != UINT_MAX);
 		spin_lock(&queue->lock);
-		spin_lock(&fc->bg_lock);
+		spin_lock(&fch->bg_lock);
 		fuse_uring_flush_bg(queue);
-		spin_unlock(&fc->bg_lock);
+		spin_unlock(&fch->bg_lock);
 		spin_unlock(&queue->lock);
 		fuse_uring_abort_end_queue_requests(queue);
 	}
 }
 
-static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *list)
+static bool ent_list_request_expired(struct fuse_chan *fch, struct list_head *list)
 {
 	struct fuse_ring_ent *ent;
 	struct fuse_req *req;
@@ -154,12 +154,12 @@ static bool ent_list_request_expired(struct fuse_conn *fc, struct list_head *lis
 	req = ent->fuse_req;
 
 	return time_is_before_jiffies(req->create_time +
-				      fc->timeout.req_timeout);
+				      fch->timeout.req_timeout);
 }
 
-bool fuse_uring_request_expired(struct fuse_conn *fc)
+bool fuse_uring_request_expired(struct fuse_chan *fch)
 {
-	struct fuse_ring *ring = fc->ring;
+	struct fuse_ring *ring = fch->ring;
 	struct fuse_ring_queue *queue;
 	int qid;
 
@@ -172,10 +172,10 @@ bool fuse_uring_request_expired(struct fuse_conn *fc)
 			continue;
 
 		spin_lock(&queue->lock);
-		if (fuse_request_expired(fc, &queue->fuse_req_queue) ||
-		    fuse_request_expired(fc, &queue->fuse_req_bg_queue) ||
-		    ent_list_request_expired(fc, &queue->ent_w_req_queue) ||
-		    ent_list_request_expired(fc, &queue->ent_in_userspace)) {
+		if (fuse_request_expired(fch, &queue->fuse_req_queue) ||
+		    fuse_request_expired(fch, &queue->fuse_req_bg_queue) ||
+		    ent_list_request_expired(fch, &queue->ent_w_req_queue) ||
+		    ent_list_request_expired(fch, &queue->ent_in_userspace)) {
 			spin_unlock(&queue->lock);
 			return true;
 		}
@@ -185,9 +185,9 @@ bool fuse_uring_request_expired(struct fuse_conn *fc)
 	return false;
 }
 
-void fuse_uring_destruct(struct fuse_conn *fc)
+void fuse_uring_destruct(struct fuse_chan *fch)
 {
-	struct fuse_ring *ring = fc->ring;
+	struct fuse_ring *ring = fch->ring;
 	int qid;
 
 	if (!ring)
@@ -218,20 +218,20 @@ void fuse_uring_destruct(struct fuse_conn *fc)
 
 	kfree(ring->queues);
 	kfree(ring);
-	fc->ring = NULL;
+	fch->ring = NULL;
 }
 
 /*
  * Basic ring setup for this connection based on the provided configuration
  */
-static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc)
+static struct fuse_ring *fuse_uring_create(struct fuse_chan *fch)
 {
 	struct fuse_ring *ring;
 	size_t nr_queues = num_possible_cpus();
 	struct fuse_ring *res = NULL;
 	size_t max_payload_size;
 
-	ring = kzalloc_obj(*fc->ring, GFP_KERNEL_ACCOUNT);
+	ring = kzalloc_obj(*ring, GFP_KERNEL_ACCOUNT);
 	if (!ring)
 		return NULL;
 
@@ -240,25 +240,25 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc)
 	if (!ring->queues)
 		goto out_err;
 
-	max_payload_size = max(FUSE_MIN_READ_BUFFER, fc->max_write);
-	max_payload_size = max(max_payload_size, fc->max_pages * PAGE_SIZE);
+	max_payload_size = max(FUSE_MIN_READ_BUFFER, fch->max_write);
+	max_payload_size = max(max_payload_size, fch->max_pages * PAGE_SIZE);
 
-	spin_lock(&fc->lock);
-	if (fc->ring) {
+	spin_lock(&fch->lock);
+	if (fch->ring) {
 		/* race, another thread created the ring in the meantime */
-		spin_unlock(&fc->lock);
-		res = fc->ring;
+		spin_unlock(&fch->lock);
+		res = fch->ring;
 		goto out_err;
 	}
 
 	init_waitqueue_head(&ring->stop_waitq);
 
 	ring->nr_queues = nr_queues;
-	ring->fc = fc;
+	ring->chan = fch;
 	ring->max_payload_sz = max_payload_size;
-	smp_store_release(&fc->ring, ring);
+	smp_store_release(&fch->ring, ring);
 
-	spin_unlock(&fc->lock);
+	spin_unlock(&fch->lock);
 	return ring;
 
 out_err:
@@ -270,14 +270,14 @@ static struct fuse_ring *fuse_uring_create(struct fuse_conn *fc)
 static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
 						       int qid)
 {
-	struct fuse_conn *fc = ring->fc;
+	struct fuse_chan *fch = ring->chan;
 	struct fuse_ring_queue *queue;
 	struct list_head *pq;
 
 	queue = kzalloc_obj(*queue, GFP_KERNEL_ACCOUNT);
 	if (!queue)
 		return NULL;
-	pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE);
+	pq = fuse_pqueue_alloc();
 	if (!pq) {
 		kfree(queue);
 		return NULL;
@@ -295,12 +295,12 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
 	INIT_LIST_HEAD(&queue->fuse_req_bg_queue);
 	INIT_LIST_HEAD(&queue->ent_released);
 
-	queue->fpq.processing = pq;
 	fuse_pqueue_init(&queue->fpq);
+	queue->fpq.processing = pq;
 
-	spin_lock(&fc->lock);
+	spin_lock(&fch->lock);
 	if (ring->queues[qid]) {
-		spin_unlock(&fc->lock);
+		spin_unlock(&fch->lock);
 		kfree(queue->fpq.processing);
 		kfree(queue);
 		return ring->queues[qid];
@@ -310,7 +310,7 @@ static struct fuse_ring_queue *fuse_uring_create_queue(struct fuse_ring *ring,
 	 * write_once and lock as the caller mostly doesn't take the lock at all
 	 */
 	WRITE_ONCE(ring->queues[qid], queue);
-	spin_unlock(&fc->lock);
+	spin_unlock(&fch->lock);
 
 	return queue;
 }
@@ -531,8 +531,7 @@ static void fuse_uring_prepare_cancel(struct io_uring_cmd *cmd, int issue_flags,
  * Checks for errors and stores it into the request
  */
 static int fuse_uring_out_header_has_err(struct fuse_out_header *oh,
-					 struct fuse_req *req,
-					 struct fuse_conn *fc)
+					 struct fuse_req *req)
 {
 	int err;
 
@@ -812,7 +811,6 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
 			      unsigned int issue_flags)
 {
 	struct fuse_ring *ring = ent->queue->ring;
-	struct fuse_conn *fc = ring->fc;
 	ssize_t err = 0;
 
 	err = copy_from_user(&req->out.h, &ent->headers->in_out,
@@ -822,7 +820,7 @@ static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,
 		goto out;
 	}
 
-	err = fuse_uring_out_header_has_err(&req->out.h, req, fc);
+	err = fuse_uring_out_header_has_err(&req->out.h, req);
 	if (err) {
 		/* req->out.h.error already set */
 		goto out;
@@ -873,13 +871,13 @@ static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent)
 
 /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */
 static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
-				   struct fuse_conn *fc)
+				   struct fuse_chan *fch)
 {
 	const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe128_cmd(cmd->sqe,
 								       struct fuse_uring_cmd_req);
 	struct fuse_ring_ent *ent;
 	int err;
-	struct fuse_ring *ring = fc->ring;
+	struct fuse_ring *ring = fch->ring;
 	struct fuse_ring_queue *queue;
 	uint64_t commit_id = READ_ONCE(cmd_req->commit_id);
 	unsigned int qid = READ_ONCE(cmd_req->qid);
@@ -898,7 +896,7 @@ static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
 		return err;
 	fpq = &queue->fpq;
 
-	if (!READ_ONCE(fc->connected) || READ_ONCE(queue->stopped))
+	if (!READ_ONCE(fch->connected) || READ_ONCE(queue->stopped))
 		return err;
 
 	spin_lock(&queue->lock);
@@ -981,8 +979,8 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ent,
 {
 	struct fuse_ring_queue *queue = ent->queue;
 	struct fuse_ring *ring = queue->ring;
-	struct fuse_conn *fc = ring->fc;
-	struct fuse_iqueue *fiq = &fc->iq;
+	struct fuse_chan *fch = ring->chan;
+	struct fuse_iqueue *fiq = &fch->iq;
 
 	fuse_uring_prepare_cancel(cmd, issue_flags, ent);
 
@@ -997,7 +995,7 @@ static void fuse_uring_do_register(struct fuse_ring_ent *ent,
 		if (ready) {
 			WRITE_ONCE(fiq->ops, &fuse_io_uring_ops);
 			WRITE_ONCE(ring->ready, true);
-			wake_up_all(&fc->blocked_waitq);
+			wake_up_all(&fch->blocked_waitq);
 		}
 	}
 }
@@ -1078,11 +1076,11 @@ fuse_uring_create_ring_ent(struct io_uring_cmd *cmd,
  * entry as "ready to get fuse requests" on the queue
  */
 static int fuse_uring_register(struct io_uring_cmd *cmd,
-			       unsigned int issue_flags, struct fuse_conn *fc)
+			       unsigned int issue_flags, struct fuse_chan *fch)
 {
 	const struct fuse_uring_cmd_req *cmd_req = io_uring_sqe128_cmd(cmd->sqe,
 								       struct fuse_uring_cmd_req);
-	struct fuse_ring *ring = smp_load_acquire(&fc->ring);
+	struct fuse_ring *ring = smp_load_acquire(&fch->ring);
 	struct fuse_ring_queue *queue;
 	struct fuse_ring_ent *ent;
 	int err;
@@ -1090,7 +1088,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd,
 
 	err = -ENOMEM;
 	if (!ring) {
-		ring = fuse_uring_create(fc);
+		ring = fuse_uring_create(fch);
 		if (!ring)
 			return err;
 	}
@@ -1128,7 +1126,7 @@ static int fuse_uring_register(struct io_uring_cmd *cmd,
 int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
 {
 	struct fuse_dev *fud;
-	struct fuse_conn *fc;
+	struct fuse_chan *fch;
 	u32 cmd_op = cmd->cmd_op;
 	int err;
 
@@ -1146,39 +1144,39 @@ int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
 		pr_info_ratelimited("No fuse device found\n");
 		return PTR_ERR(fud);
 	}
-	fc = fud->fc;
+	fch = fud->chan;
 
 	/* Once a connection has io-uring enabled on it, it can't be disabled */
-	if (!enable_uring && !fc->io_uring) {
+	if (!enable_uring && !fch->io_uring) {
 		pr_info_ratelimited("fuse-io-uring is disabled\n");
 		return -EOPNOTSUPP;
 	}
 
-	if (fc->aborted)
+	if (fch->abort_with_err)
 		return -ECONNABORTED;
-	if (!fc->connected)
+	if (!fch->connected)
 		return -ENOTCONN;
 
 	/*
 	 * fuse_uring_register() needs the ring to be initialized,
 	 * we need to know the max payload size
 	 */
-	if (!fc->initialized)
+	if (!fch->initialized)
 		return -EAGAIN;
 
 	switch (cmd_op) {
 	case FUSE_IO_URING_CMD_REGISTER:
-		err = fuse_uring_register(cmd, issue_flags, fc);
+		err = fuse_uring_register(cmd, issue_flags, fch);
 		if (err) {
 			pr_info_once("FUSE_IO_URING_CMD_REGISTER failed err=%d\n",
 				     err);
-			fc->io_uring = 0;
-			wake_up_all(&fc->blocked_waitq);
+			fch->io_uring = 0;
+			wake_up_all(&fch->blocked_waitq);
 			return err;
 		}
 		break;
 	case FUSE_IO_URING_CMD_COMMIT_AND_FETCH:
-		err = fuse_uring_commit_fetch(cmd, issue_flags, fc);
+		err = fuse_uring_commit_fetch(cmd, issue_flags, fch);
 		if (err) {
 			pr_info_once("FUSE_IO_URING_COMMIT_AND_FETCH failed err=%d\n",
 				     err);
@@ -1261,8 +1259,7 @@ static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent)
 /* queue a fuse request and send it if a ring entry is available */
 void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req)
 {
-	struct fuse_conn *fc = req->fm->fc;
-	struct fuse_ring *ring = fc->ring;
+	struct fuse_ring *ring = req->chan->ring;
 	struct fuse_ring_queue *queue;
 	struct fuse_ring_ent *ent = NULL;
 	int err;
@@ -1304,8 +1301,8 @@ void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req)
 
 bool fuse_uring_queue_bq_req(struct fuse_req *req)
 {
-	struct fuse_conn *fc = req->fm->fc;
-	struct fuse_ring *ring = fc->ring;
+	struct fuse_chan *fch = req->chan;
+	struct fuse_ring *ring = fch->ring;
 	struct fuse_ring_queue *queue;
 	struct fuse_ring_ent *ent = NULL;
 
@@ -1325,12 +1322,12 @@ bool fuse_uring_queue_bq_req(struct fuse_req *req)
 
 	ent = list_first_entry_or_null(&queue->ent_avail_queue,
 				       struct fuse_ring_ent, list);
-	spin_lock(&fc->bg_lock);
-	fc->num_background++;
-	if (fc->num_background == fc->max_background)
-		fc->blocked = 1;
+	spin_lock(&fch->bg_lock);
+	fch->num_background++;
+	if (fch->num_background == fch->max_background)
+		fch->blocked = 1;
 	fuse_uring_flush_bg(queue);
-	spin_unlock(&fc->bg_lock);
+	spin_unlock(&fch->bg_lock);
 
 	/*
 	 * Due to bg_queue flush limits there might be other bg requests
diff --git a/fs/fuse/dev_uring_i.h b/fs/fuse/dev_uring_i.h
index 51a563922ce14..368f4d0790ebb 100644
--- a/fs/fuse/dev_uring_i.h
+++ b/fs/fuse/dev_uring_i.h
@@ -7,7 +7,7 @@
 #ifndef _FS_FUSE_DEV_URING_I_H
 #define _FS_FUSE_DEV_URING_I_H
 
-#include "fuse_i.h"
+#include "fuse_dev_i.h"
 
 #ifdef CONFIG_FUSE_IO_URING
 
@@ -101,13 +101,13 @@ struct fuse_ring_queue {
 	bool stopped;
 };
 
-/**
+/*
  * Describes if uring is for communication and holds alls the data needed
  * for uring communication
  */
 struct fuse_ring {
 	/* back pointer */
-	struct fuse_conn *fc;
+	struct fuse_chan *chan;
 
 	/* number of ring queues */
 	size_t nr_queues;
@@ -135,19 +135,17 @@ struct fuse_ring {
 	bool ready;
 };
 
-bool fuse_uring_enabled(void);
-void fuse_uring_destruct(struct fuse_conn *fc);
 void fuse_uring_stop_queues(struct fuse_ring *ring);
 void fuse_uring_abort_end_requests(struct fuse_ring *ring);
 int fuse_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags);
 void fuse_uring_queue_fuse_req(struct fuse_iqueue *fiq, struct fuse_req *req);
 bool fuse_uring_queue_bq_req(struct fuse_req *req);
 bool fuse_uring_remove_pending_req(struct fuse_req *req);
-bool fuse_uring_request_expired(struct fuse_conn *fc);
+bool fuse_uring_request_expired(struct fuse_chan *fch);
 
-static inline void fuse_uring_abort(struct fuse_conn *fc)
+static inline void fuse_uring_abort(struct fuse_chan *fch)
 {
-	struct fuse_ring *ring = fc->ring;
+	struct fuse_ring *ring = fch->ring;
 
 	if (ring == NULL)
 		return;
@@ -158,40 +156,31 @@ static inline void fuse_uring_abort(struct fuse_conn *fc)
 	}
 }
 
-static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc)
+static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch)
 {
-	struct fuse_ring *ring = fc->ring;
+	struct fuse_ring *ring = fch->ring;
 
 	if (ring)
 		wait_event(ring->stop_waitq,
 			   atomic_read(&ring->queue_refs) == 0);
 }
 
-static inline bool fuse_uring_ready(struct fuse_conn *fc)
+static inline bool fuse_uring_ready(struct fuse_chan *fch)
 {
-	return fc->ring && fc->ring->ready;
+	return fch->ring && fch->ring->ready;
 }
 
 #else /* CONFIG_FUSE_IO_URING */
 
-static inline void fuse_uring_destruct(struct fuse_conn *fc)
+static inline void fuse_uring_abort(struct fuse_chan *fch)
 {
 }
 
-static inline bool fuse_uring_enabled(void)
-{
-	return false;
-}
-
-static inline void fuse_uring_abort(struct fuse_conn *fc)
-{
-}
-
-static inline void fuse_uring_wait_stopped_queues(struct fuse_conn *fc)
+static inline void fuse_uring_wait_stopped_queues(struct fuse_chan *fch)
 {
 }
 
-static inline bool fuse_uring_ready(struct fuse_conn *fc)
+static inline bool fuse_uring_ready(struct fuse_chan *fch)
 {
 	return false;
 }
@@ -201,7 +190,7 @@ static inline bool fuse_uring_remove_pending_req(struct fuse_req *req)
 	return false;
 }
 
-static inline bool fuse_uring_request_expired(struct fuse_conn *fc)
+static inline bool fuse_uring_request_expired(struct fuse_chan *fch)
 {
 	return false;
 }
diff --git a/fs/fuse/dir.c b/fs/fuse/dir.c
index b658b6baf72fe..be41c14ef3292 100644
--- a/fs/fuse/dir.c
+++ b/fs/fuse/dir.c
@@ -6,6 +6,7 @@
   See the file COPYING.
 */
 
+#include "dev.h"
 #include "fuse_i.h"
 
 #include <linux/pagemap.h>
@@ -317,7 +318,7 @@ void fuse_invalidate_attr(struct inode *inode)
 
 static void fuse_dir_changed(struct inode *dir)
 {
-	fuse_invalidate_attr(dir);
+	fuse_invalidate_attr_mask(dir, FUSE_STATX_MODDIR);
 	inode_maybe_inc_iversion(dir, false);
 }
 
@@ -430,7 +431,7 @@ static int fuse_dentry_revalidate(struct inode *dir, const struct qstr *name,
 			fi = get_fuse_inode(inode);
 			if (outarg.nodeid != get_node_id(inode) ||
 			    (bool) IS_AUTOMOUNT(inode) != (bool) (outarg.attr.flags & FUSE_ATTR_SUBMOUNT)) {
-				fuse_queue_forget(fm->fc, forget,
+				fuse_chan_queue_forget(fm->fc->chan, forget,
 						  outarg.nodeid, 1);
 				goto invalid;
 			}
@@ -593,7 +594,7 @@ int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name
 			   attr_version, evict_ctr);
 	err = -ENOMEM;
 	if (!*inode) {
-		fuse_queue_forget(fm->fc, forget, outarg->nodeid, 1);
+		fuse_chan_queue_forget(fm->fc->chan, forget, outarg->nodeid, 1);
 		goto out;
 	}
 	err = 0;
@@ -837,7 +838,6 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir,
 	if (!forget)
 		goto out_err;
 
-	err = -ENOMEM;
 	ff = fuse_file_alloc(fm, true);
 	if (!ff)
 		goto out_put_forget_req;
@@ -894,7 +894,7 @@ static int fuse_create_open(struct mnt_idmap *idmap, struct inode *dir,
 	if (!inode) {
 		flags &= ~(O_CREAT | O_EXCL | O_TRUNC);
 		fuse_sync_release(NULL, ff, flags);
-		fuse_queue_forget(fm->fc, forget, outentry.nodeid, 1);
+		fuse_chan_queue_forget(fm->fc->chan, forget, outentry.nodeid, 1);
 		err = -ENOMEM;
 		goto out_err;
 	}
@@ -1019,7 +1019,7 @@ static struct dentry *create_new_entry(struct mnt_idmap *idmap, struct fuse_moun
 	inode = fuse_iget(dir->i_sb, outarg.nodeid, outarg.generation,
 			  &outarg.attr, ATTR_TIMEOUT(&outarg), 0, 0);
 	if (!inode) {
-		fuse_queue_forget(fm->fc, forget, outarg.nodeid, 1);
+		fuse_chan_queue_forget(fm->fc->chan, forget, outarg.nodeid, 1);
 		return ERR_PTR(-ENOMEM);
 	}
 	kfree(forget);
diff --git a/fs/fuse/file.c b/fs/fuse/file.c
index c59452d60b8da..7833ac451dda9 100644
--- a/fs/fuse/file.c
+++ b/fs/fuse/file.c
@@ -7,6 +7,7 @@
 */
 
 #include "fuse_i.h"
+#include "dev.h"
 
 #include <linux/pagemap.h>
 #include <linux/slab.h>
@@ -91,8 +92,7 @@ static struct fuse_file *fuse_file_get(struct fuse_file *ff)
 	return ff;
 }
 
-static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args,
-			     int error)
+static void fuse_release_end(struct fuse_args *args, int error)
 {
 	struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
 
@@ -112,10 +112,10 @@ static void fuse_file_put(struct fuse_file *ff, bool sync)
 		if (!args) {
 			/* Do nothing when server does not implement 'opendir' */
 		} else if (args->opcode == FUSE_RELEASE && ff->fm->fc->no_open) {
-			fuse_release_end(ff->fm, args, 0);
+			fuse_release_end(args, 0);
 		} else if (sync) {
 			fuse_simple_request(ff->fm, args);
-			fuse_release_end(ff->fm, args, 0);
+			fuse_release_end(args, 0);
 		} else {
 			/*
 			 * DAX inodes may need to issue a number of synchronous
@@ -126,7 +126,7 @@ static void fuse_file_put(struct fuse_file *ff, bool sync)
 			args->end = fuse_release_end;
 			if (fuse_simple_background(ff->fm, args,
 						   GFP_KERNEL | __GFP_NOFAIL))
-				fuse_release_end(ff->fm, args, -ENOTCONN);
+				fuse_release_end(args, -ENOTCONN);
 		}
 		kfree(ff);
 	}
@@ -709,8 +709,7 @@ static void fuse_io_free(struct fuse_io_args *ia)
 	kfree(ia);
 }
 
-static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
-				  int err)
+static void fuse_aio_complete_req(struct fuse_args *args, int err)
 {
 	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
 	struct fuse_io_priv *io = ia->io;
@@ -758,7 +757,7 @@ static ssize_t fuse_async_req_send(struct fuse_mount *fm,
 	ia->ap.args.may_block = io->should_dirty;
 	err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
 	if (err)
-		fuse_aio_complete_req(fm, &ia->ap.args, err);
+		fuse_aio_complete_req(&ia->ap.args, err);
 
 	return num_bytes;
 }
@@ -902,7 +901,7 @@ static int fuse_handle_readahead(struct folio *folio,
 		ia = NULL;
 	}
 	if (!ia) {
-		if (fc->num_background >= fc->congestion_threshold &&
+		if (fuse_chan_num_background(fc->chan) >= fc->congestion_threshold &&
 		    rac->ra->async_size >= readahead_count(rac))
 			/*
 			 * Congested and only async pages left, so skip the
@@ -1001,8 +1000,7 @@ static int fuse_iomap_read_folio_range(const struct iomap_iter *iter,
 	return fuse_do_readfolio(file, folio, off, len);
 }
 
-static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
-			       int err)
+static void fuse_readpages_end(struct fuse_args *args, int err)
 {
 	int i;
 	struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
@@ -1068,7 +1066,7 @@ static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file,
 		res = fuse_simple_request(fm, &ap->args);
 		err = res < 0 ? res : 0;
 	}
-	fuse_readpages_end(fm, &ap->args, err);
+	fuse_readpages_end(&ap->args, err);
 }
 
 static void fuse_readahead(struct readahead_control *rac)
@@ -1985,8 +1983,7 @@ __acquires(fi->lock)
 	}
 }
 
-static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args,
-			       int error)
+static void fuse_writepage_end(struct fuse_args *args, int error)
 {
 	struct fuse_writepage_args *wpa =
 		container_of(args, typeof(*wpa), ia.ap.args);
@@ -2176,7 +2173,10 @@ static bool fuse_folios_need_send(struct fuse_conn *fc, loff_t pos,
 
 	WARN_ON(!ap->num_folios);
 
-	/* Reached max pages */
+	/* Reached max pages or max folio slots */
+	if (ap->num_folios >= fc->max_pages)
+		return true;
+
 	if (DIV_ROUND_UP(bytes, PAGE_SIZE) > fc->max_pages)
 		return true;
 
@@ -2294,7 +2294,7 @@ static int fuse_writepages(struct address_space *mapping,
 		return -EIO;
 
 	if (wbc->sync_mode == WB_SYNC_NONE &&
-	    fc->num_background >= fc->congestion_threshold)
+	    fuse_chan_num_background(fc->chan) >= fc->congestion_threshold)
 		return 0;
 
 	return iomap_writepages(&wpc);
@@ -2680,125 +2680,6 @@ static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
 	return retval;
 }
 
-/*
- * All files which have been polled are linked to RB tree
- * fuse_conn->polled_files which is indexed by kh.  Walk the tree and
- * find the matching one.
- */
-static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
-					      struct rb_node **parent_out)
-{
-	struct rb_node **link = &fc->polled_files.rb_node;
-	struct rb_node *last = NULL;
-
-	while (*link) {
-		struct fuse_file *ff;
-
-		last = *link;
-		ff = rb_entry(last, struct fuse_file, polled_node);
-
-		if (kh < ff->kh)
-			link = &last->rb_left;
-		else if (kh > ff->kh)
-			link = &last->rb_right;
-		else
-			return link;
-	}
-
-	if (parent_out)
-		*parent_out = last;
-	return link;
-}
-
-/*
- * The file is about to be polled.  Make sure it's on the polled_files
- * RB tree.  Note that files once added to the polled_files tree are
- * not removed before the file is released.  This is because a file
- * polled once is likely to be polled again.
- */
-static void fuse_register_polled_file(struct fuse_conn *fc,
-				      struct fuse_file *ff)
-{
-	spin_lock(&fc->lock);
-	if (RB_EMPTY_NODE(&ff->polled_node)) {
-		struct rb_node **link, *parent;
-
-		link = fuse_find_polled_node(fc, ff->kh, &parent);
-		BUG_ON(*link);
-		rb_link_node(&ff->polled_node, parent, link);
-		rb_insert_color(&ff->polled_node, &fc->polled_files);
-	}
-	spin_unlock(&fc->lock);
-}
-
-__poll_t fuse_file_poll(struct file *file, poll_table *wait)
-{
-	struct fuse_file *ff = file->private_data;
-	struct fuse_mount *fm = ff->fm;
-	struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
-	struct fuse_poll_out outarg;
-	FUSE_ARGS(args);
-	int err;
-
-	if (fm->fc->no_poll)
-		return DEFAULT_POLLMASK;
-
-	poll_wait(file, &ff->poll_wait, wait);
-	inarg.events = mangle_poll(poll_requested_events(wait));
-
-	/*
-	 * Ask for notification iff there's someone waiting for it.
-	 * The client may ignore the flag and always notify.
-	 */
-	if (waitqueue_active(&ff->poll_wait)) {
-		inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
-		fuse_register_polled_file(fm->fc, ff);
-	}
-
-	args.opcode = FUSE_POLL;
-	args.nodeid = ff->nodeid;
-	args.in_numargs = 1;
-	args.in_args[0].size = sizeof(inarg);
-	args.in_args[0].value = &inarg;
-	args.out_numargs = 1;
-	args.out_args[0].size = sizeof(outarg);
-	args.out_args[0].value = &outarg;
-	err = fuse_simple_request(fm, &args);
-
-	if (!err)
-		return demangle_poll(outarg.revents);
-	if (err == -ENOSYS) {
-		fm->fc->no_poll = 1;
-		return DEFAULT_POLLMASK;
-	}
-	return EPOLLERR;
-}
-EXPORT_SYMBOL_GPL(fuse_file_poll);
-
-/*
- * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
- * wakes up the poll waiters.
- */
-int fuse_notify_poll_wakeup(struct fuse_conn *fc,
-			    struct fuse_notify_poll_wakeup_out *outarg)
-{
-	u64 kh = outarg->kh;
-	struct rb_node **link;
-
-	spin_lock(&fc->lock);
-
-	link = fuse_find_polled_node(fc, kh, NULL);
-	if (*link) {
-		struct fuse_file *ff;
-
-		ff = rb_entry(*link, struct fuse_file, polled_node);
-		wake_up_interruptible_sync(&ff->poll_wait);
-	}
-
-	spin_unlock(&fc->lock);
-	return 0;
-}
-
 static void fuse_do_truncate(struct file *file)
 {
 	struct inode *inode = file->f_mapping->host;
diff --git a/fs/fuse/fuse_dev_i.h b/fs/fuse/fuse_dev_i.h
index 910f883cd090f..9ce987826deda 100644
--- a/fs/fuse/fuse_dev_i.h
+++ b/fs/fuse/fuse_dev_i.h
@@ -6,20 +6,311 @@
 #ifndef _FS_FUSE_DEV_I_H
 #define _FS_FUSE_DEV_I_H
 
+#include <linux/fuse.h>
 #include <linux/types.h>
+#include <linux/refcount.h>
+#include <linux/wait.h>
+#include <linux/workqueue.h>
+#include <linux/fs.h>
 
 /* Ordinary requests have even IDs, while interrupts IDs are odd */
 #define FUSE_INT_REQ_BIT (1ULL << 0)
 #define FUSE_REQ_ID_STEP (1ULL << 1)
 
-extern struct wait_queue_head fuse_dev_waitq;
-
 struct fuse_arg;
 struct fuse_args;
 struct fuse_pqueue;
-struct fuse_req;
 struct fuse_iqueue;
-struct fuse_forget_link;
+
+/**
+ * enum fuse_req_flag - Request flags
+ *
+ * @FR_ISREPLY:		set if the request has reply
+ * @FR_FORCE:		force sending of the request even if interrupted
+ * @FR_BACKGROUND:	request is sent in the background
+ * @FR_WAITING:		request is counted as "waiting"
+ * @FR_ABORTED:		the request was aborted
+ * @FR_INTERRUPTED:	the request has been interrupted
+ * @FR_LOCKED:		data is being copied to/from the request
+ * @FR_PENDING:		request is not yet in userspace
+ * @FR_SENT:		request is in userspace, waiting for an answer
+ * @FR_FINISHED:	request is finished
+ * @FR_PRIVATE:		request is on private list
+ * @FR_ASYNC:		request is asynchronous
+ * @FR_URING:		request is handled through fuse-io-uring
+ */
+enum fuse_req_flag {
+	FR_ISREPLY,
+	FR_FORCE,
+	FR_BACKGROUND,
+	FR_WAITING,
+	FR_ABORTED,
+	FR_INTERRUPTED,
+	FR_LOCKED,
+	FR_PENDING,
+	FR_SENT,
+	FR_FINISHED,
+	FR_PRIVATE,
+	FR_ASYNC,
+	FR_URING,
+};
+
+/**
+ * struct fuse_req - A request to the client
+ *
+ * .waitq.lock protects the following fields:
+ *   - FR_ABORTED
+ *   - FR_LOCKED (may also be modified under fpq->lock, tested under both)
+ */
+struct fuse_req {
+	/**
+	 * @list: This can be on either pending processing or io lists in
+	 * fuse_conn
+	 */
+	struct list_head list;
+
+	/** @intr_entry: Entry on the interrupts list  */
+	struct list_head intr_entry;
+
+	/** @args: Input/output arguments */
+	struct fuse_args *args;
+
+	/** @count: refcount */
+	refcount_t count;
+
+	/** @flags: Request flags, updated with test/set/clear_bit() */
+	unsigned long flags;
+
+	/** @in: The request input header */
+	struct {
+		/** @in.h: The request input header */
+		struct fuse_in_header h;
+	} in;
+
+	/** @out: The request output header */
+	struct {
+		/** @out.h: The request output header */
+		struct fuse_out_header h;
+	} out;
+
+	/** @waitq: Used to wake up the task waiting for completion of request */
+	wait_queue_head_t waitq;
+
+#if IS_ENABLED(CONFIG_VIRTIO_FS)
+	/**
+	 * @argbuf: virtio-fs's physically contiguous buffer for in and out
+	 * args
+	 */
+	void *argbuf;
+#endif
+
+	/** @chan: fuse_chan this request belongs to */
+	struct fuse_chan *chan;
+
+#ifdef CONFIG_FUSE_IO_URING
+	void *ring_entry;
+	void *ring_queue;
+#endif
+	/** @create_time: When (in jiffies) the request was created */
+	unsigned long create_time;
+};
+
+/* One forget request */
+struct fuse_forget_link {
+	struct fuse_forget_one forget_one;
+	struct fuse_forget_link *next;
+};
+
+/**
+ * struct fuse_iqueue_ops - Input queue callbacks
+ *
+ * Input queue signalling is device-specific.  For example, the /dev/fuse file
+ * uses fiq->waitq and fasync to wake processes that are waiting on queue
+ * readiness.  These callbacks allow other device types to respond to input
+ * queue activity.
+ */
+struct fuse_iqueue_ops {
+	/**
+	 * @send_forget: Send one forget
+	 */
+	void (*send_forget)(struct fuse_iqueue *fiq, struct fuse_forget_link *link);
+
+	/**
+	 * @send_interrupt: Send interrupt for request
+	 */
+	void (*send_interrupt)(struct fuse_iqueue *fiq, struct fuse_req *req);
+
+	/**
+	 * @send_req: Send one request
+	 */
+	void (*send_req)(struct fuse_iqueue *fiq, struct fuse_req *req);
+
+	/**
+	 * @release: Clean up when fuse_iqueue is destroyed
+	 */
+	void (*release)(struct fuse_iqueue *fiq);
+};
+
+struct fuse_iqueue {
+	/** Connection established */
+	unsigned connected;
+
+	/** Lock protecting accesses to members of this structure */
+	spinlock_t lock;
+
+	/** Readers of the connection are waiting on this */
+	wait_queue_head_t waitq;
+
+	/** The next unique request id */
+	u64 reqctr;
+
+	/** The list of pending requests */
+	struct list_head pending;
+
+	/** Pending interrupts */
+	struct list_head interrupts;
+
+	/** Queue of pending forgets */
+	struct fuse_forget_link forget_list_head;
+	struct fuse_forget_link *forget_list_tail;
+
+	/** Batching of FORGET requests (positive indicates FORGET batch) */
+	int forget_batch;
+
+	/** O_ASYNC requests */
+	struct fasync_struct *fasync;
+
+	/** Device-specific callbacks */
+	const struct fuse_iqueue_ops *ops;
+
+	/** Device-specific state */
+	void *priv;
+};
+
+struct fuse_chan {
+	/** Lock protecting:
+	    - devices
+	    - connected
+	    - ring
+	    - ring->queues[qid]
+	 */
+	spinlock_t lock;
+
+	/* back pointer: fc->chan->conn == fc */
+	struct fuse_conn *conn;
+
+	/** Input queue */
+	struct fuse_iqueue iq;
+
+	/** List of device instances belonging to this connection */
+	struct list_head devices;
+
+	/** Maximum number of outstanding background requests */
+	unsigned max_background;
+
+	/** Number of requests currently in the background */
+	unsigned num_background;
+
+	/** Number of background requests currently queued for userspace */
+	unsigned active_background;
+
+	/** The list of background requests set aside for later queuing */
+	struct list_head bg_queue;
+
+	/** Protects: max_background, num_background, active_background, bg_queue, blocked */
+	spinlock_t bg_lock;
+
+	/** Flag indicating that INIT reply has been received. Allocating
+	 * any fuse request will be suspended until the flag is set */
+	int initialized;
+
+	/** Flag indicating if connection is blocked.  This will be
+	    the case before the INIT reply is received, and if there
+	    are too many outstading backgrounds requests */
+	int blocked;
+
+	/** waitq for blocked connection */
+	wait_queue_head_t blocked_waitq;
+
+	/** Connection established, cleared on umount, connection
+	    abort and device release */
+	unsigned connected;
+
+	/** The number of requests waiting for completion */
+	atomic_t num_waiting;
+
+	/** Is interrupt not implemented by fs? */
+	bool no_interrupt;
+
+	/* Use io_uring for communication */
+	unsigned int io_uring;
+
+	/* Negotiated minor version */
+	unsigned int minor;
+
+	/* Maximum write size */
+	unsigned int max_write;
+
+	/* Maximum number of pages that can be used in a single request */
+	unsigned int max_pages;
+
+	/* Before being installed into fud, contains the preallocated pq array*/
+	struct list_head *pq_prealloc;
+
+	/** Connection aborted via sysfs, respond with ECONNABORTED on device I/O */
+	bool abort_with_err;
+
+#ifdef CONFIG_FUSE_IO_URING
+	/**  uring connection information*/
+	struct fuse_ring *ring;
+#endif
+
+	/** Only used if the connection opts into request timeouts */
+	struct {
+		/* Worker for checking if any requests have timed out */
+		struct delayed_work work;
+
+		/* Request timeout (in jiffies). 0 = no timeout */
+		unsigned int req_timeout;
+	} timeout;
+};
+
+#define FUSE_PQ_HASH_BITS 8
+#define FUSE_PQ_HASH_SIZE (1 << FUSE_PQ_HASH_BITS)
+
+struct fuse_pqueue {
+	/** Connection established */
+	unsigned connected;
+
+	/** Lock protecting accessess to  members of this structure */
+	spinlock_t lock;
+
+	/** Hash table of requests being processed */
+	struct list_head *processing;
+
+	/** The list of requests under I/O */
+	struct list_head io;
+};
+
+/**
+ * struct fuse_dev - Fuse device instance
+ */
+struct fuse_dev {
+	/** @ref: Reference count of this object */
+	refcount_t ref;
+
+	/** @sync_init: Issue FUSE_INIT synchronously */
+	bool sync_init;
+
+	/** @chan: Fuse channel for this device */
+	struct fuse_chan *chan;
+
+	/** @pq: Processing queue */
+	struct fuse_pqueue pq;
+
+	/** @entry: list entry on fch->devices */
+	struct list_head entry;
+};
 
 struct fuse_copy_state {
 	struct fuse_req *req;
@@ -39,21 +330,21 @@ struct fuse_copy_state {
 	} ring;
 };
 
-/* fud->fc gets assigned to this value when /dev/fuse is closed */
-#define FUSE_DEV_FC_DISCONNECTED ((struct fuse_conn *) 1)
+/* fud->chan gets assigned to this value when /dev/fuse is closed */
+#define FUSE_DEV_CHAN_DISCONNECTED ((struct fuse_chan *) 1)
 
 /*
- * Lockless access is OK, because fud->fc is set once during mount and is valid
+ * Lockless access is OK, because fud->chan is set once during mount and is valid
  * until the file is released.
  *
- * fud->fc is set to FUSE_DEV_FC_DISCONNECTED only after the containing file is
+ * fud->chan is set to FUSE_DEV_CHAN_DISCONNECTED only after the containing file is
  * released, so result is safe to dereference in most cases.  Exceptions are:
  * fuse_dev_put() and fuse_fill_super_common().
  */
-static inline struct fuse_conn *fuse_dev_fc_get(struct fuse_dev *fud)
+static inline struct fuse_chan *fuse_dev_chan_get(struct fuse_dev *fud)
 {
 	/* Pairs with xchg() in fuse_dev_install() */
-	return smp_load_acquire(&fud->fc);
+	return smp_load_acquire(&fud->chan);
 }
 
 static inline struct fuse_dev *fuse_file_to_fud(struct file *file)
@@ -65,12 +356,14 @@ static inline struct fuse_dev *__fuse_get_dev(struct file *file)
 {
 	struct fuse_dev *fud = fuse_file_to_fud(file);
 
-	if (!fuse_dev_fc_get(fud))
+	if (!fuse_dev_chan_get(fud))
 		return NULL;
 
 	return fud;
 }
 
+void fuse_iqueue_init(struct fuse_iqueue *fiq, const struct fuse_iqueue_ops *ops, void *priv);
+
 struct fuse_dev *fuse_get_dev(struct file *file);
 
 unsigned int fuse_req_hash(u64 unique);
@@ -80,7 +373,11 @@ void fuse_dev_end_requests(struct list_head *head);
 
 void fuse_copy_init(struct fuse_copy_state *cs, bool write,
 			   struct iov_iter *iter);
-void fuse_copy_finish(struct fuse_copy_state *cs);
+/*
+ * Return the number of bytes in an arguments list
+ */
+unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args);
+
 int fuse_copy_args(struct fuse_copy_state *cs, unsigned int numargs,
 		   unsigned int argpages, struct fuse_arg *args,
 		   int zeroing);
@@ -91,7 +388,34 @@ void fuse_dev_queue_forget(struct fuse_iqueue *fiq,
 void fuse_dev_queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req);
 bool fuse_remove_pending_req(struct fuse_req *req, spinlock_t *lock);
 
-bool fuse_request_expired(struct fuse_conn *fc, struct list_head *list);
+bool fuse_request_expired(struct fuse_chan *fch, struct list_head *list);
+
+/*
+ * Assign a unique id to a fuse request
+ */
+void fuse_request_assign_unique(struct fuse_iqueue *fiq, struct fuse_req *req);
+
+/*
+ * Get the next unique ID for a request
+ */
+u64 fuse_get_unique(struct fuse_iqueue *fiq);
+
+struct fuse_dev *fuse_dev_alloc_install(struct fuse_chan *fch);
+struct fuse_dev *fuse_dev_alloc(void);
+
+int fuse_dev_release(struct inode *inode, struct file *file);
+
+struct list_head *fuse_pqueue_alloc(void);
+
+/*
+ * Initialize the fuse processing queue
+ */
+void fuse_pqueue_init(struct fuse_pqueue *fpq);
+
+/*
+ * End a finished request
+ */
+void fuse_request_end(struct fuse_req *req);
 
 #endif
 
diff --git a/fs/fuse/fuse_i.h b/fs/fuse/fuse_i.h
index 17423d4e3cfa6..55a3841b28899 100644
--- a/fs/fuse/fuse_i.h
+++ b/fs/fuse/fuse_i.h
@@ -13,6 +13,7 @@
 # define pr_fmt(fmt) "fuse: " fmt
 #endif
 
+#include "args.h"
 #include <linux/fuse.h>
 #include <linux/fs.h>
 #include <linux/mount.h>
@@ -48,12 +49,6 @@
 /** Number of dentries for each connection in the control filesystem */
 #define FUSE_CTL_NUM_DENTRIES 5
 
-/* Frequency (in seconds) of request timeout checks, if opted into */
-#define FUSE_TIMEOUT_TIMER_FREQ 15
-
-/** Frequency (in jiffies) of request timeout checks, if opted into */
-extern const unsigned long fuse_timeout_timer_freq;
-
 /*
  * Dentries invalidation workqueue period, in seconds.  The value of this
  * parameter shall be >= FUSE_DENTRY_INVAL_FREQ_MIN seconds, or 0 (zero), in
@@ -63,16 +58,6 @@ extern unsigned inval_wq __read_mostly;
 
 /** Maximum of max_pages received in init_out */
 extern unsigned int fuse_max_pages_limit;
-/*
- * Default timeout (in seconds) for the server to reply to a request
- * before the connection is aborted, if no timeout was specified on mount.
- */
-extern unsigned int fuse_default_req_timeout;
-/*
- * Max timeout (in seconds) for the server to reply to a request before
- * the connection is aborted.
- */
-extern unsigned int fuse_max_req_timeout;
 
 /** List of active connections */
 extern struct list_head fuse_conn_list;
@@ -84,143 +69,168 @@ extern struct mutex fuse_mutex;
 extern unsigned int max_user_bgreq;
 extern unsigned int max_user_congthresh;
 
-/* One forget request */
-struct fuse_forget_link {
-	struct fuse_forget_one forget_one;
-	struct fuse_forget_link *next;
-};
+struct fuse_forget_link;
 
-/* Submount lookup tracking */
+/**
+ * struct fuse_submount_lookup - Submount lookup tracking
+ */
 struct fuse_submount_lookup {
-	/** Refcount */
+	/** @count: Refcount */
 	refcount_t count;
 
-	/** Unique ID, which identifies the inode between userspace
-	 * and kernel */
+	/**
+	 * @nodeid: Unique ID, which identifies the inode between userspace
+	 * and kernel
+	 */
 	u64 nodeid;
 
-	/** The request used for sending the FORGET message */
+	/** @forget: The request used for sending the FORGET message */
 	struct fuse_forget_link *forget;
 };
 
-/** Container for data related to mapping to backing file */
+/* Container for data related to mapping to backing file */
 struct fuse_backing {
 	struct file *file;
-	struct cred *cred;
+	const struct cred *cred;
 
-	/** refcount */
+	/* refcount */
 	refcount_t count;
 	struct rcu_head rcu;
 };
 
-/** FUSE inode */
+/**
+ * struct fuse_inode - FUSE inode
+ */
 struct fuse_inode {
-	/** Inode data */
+	/** @inode: Inode data */
 	struct inode inode;
 
-	/** Unique ID, which identifies the inode between userspace
-	 * and kernel */
+	/**
+	 * @nodeid: Unique ID, which identifies the inode between userspace
+	 * and kernel
+	 */
 	u64 nodeid;
 
-	/** Number of lookups on this inode */
+	/** @nlookup: Number of lookups on this inode */
 	u64 nlookup;
 
-	/** The request used for sending the FORGET message */
+	/** @forget: The request used for sending the FORGET message */
 	struct fuse_forget_link *forget;
 
-	/** Time in jiffies until the file attributes are valid */
+	/** @i_time: Time in jiffies until the file attributes are valid */
 	u64 i_time;
 
-	/* Which attributes are invalid */
+	/** @inval_mask: Which attributes are invalid */
 	u32 inval_mask;
 
-	/** The sticky bit in inode->i_mode may have been removed, so
-	    preserve the original mode */
+	/**
+	 * @orig_i_mode: The sticky bit in inode->i_mode may have been removed,
+	 * so preserve the original mode
+	 */
 	umode_t orig_i_mode;
 
-	/* Cache birthtime */
+	/** @i_btime: Cache birthtime */
 	struct timespec64 i_btime;
 
-	/** 64 bit inode number */
+	/** @orig_ino: 64-bit inode number */
 	u64 orig_ino;
 
-	/** Version of last attribute change */
+	/** @attr_version: Version of last attribute change */
 	u64 attr_version;
 
 	union {
 		/* read/write io cache (regular file only) */
 		struct {
-			/* Files usable in writepage.  Protected by fi->lock */
+			/**
+			 * @write_files: Files usable in writepage.
+			 * Protected by fi->lock
+			 */
 			struct list_head write_files;
 
-			/* Writepages pending on truncate or fsync */
+			/**
+			 * @queued_writes: Writepages pending on truncate or
+			 * fsync
+			 */
 			struct list_head queued_writes;
 
-			/* Number of sent writes, a negative bias
-			 * (FUSE_NOWRITE) means more writes are blocked */
+			/**
+			 * @writectr: Number of sent writes, a negative bias
+			 * (FUSE_NOWRITE) means more writes are blocked
+			 */
 			int writectr;
 
-			/** Number of files/maps using page cache */
+			/** @iocachectr: Number of files/maps using page cache */
 			int iocachectr;
 
-			/* Waitq for writepage completion */
+			/** @page_waitq: Waitq for writepage completion */
 			wait_queue_head_t page_waitq;
 
-			/* waitq for direct-io completion */
+			/** @direct_io_waitq: waitq for direct-io completion */
 			wait_queue_head_t direct_io_waitq;
 		};
 
-		/* readdir cache (directory only) */
+		/** @rdc: readdir cache (directory only) */
 		struct {
-			/* true if fully cached */
+			/** @cached: true if fully cached */
 			bool cached;
 
-			/* size of cache */
+			/** @size: size of cache */
 			loff_t size;
 
-			/* position at end of cache (position of next entry) */
+			/**
+			 * @pos: position at end of cache (position of next
+			 * entry)
+			 */
 			loff_t pos;
 
-			/* version of the cache */
+			/** @version: version of the cache */
 			u64 version;
 
-			/* modification time of directory when cache was
-			 * started */
+			/**
+			 * @mtime: modification time of directory when cache was
+			 * started
+			 */
 			struct timespec64 mtime;
 
-			/* iversion of directory when cache was started */
+			/**
+			 * @iversion: iversion of directory when cache was
+			 * started
+			 */
 			u64 iversion;
 
-			/* protects above fields */
+			/** @lock: protects above fields */
 			spinlock_t lock;
 		} rdc;
 	};
 
-	/** Miscellaneous bits describing inode state */
+	/** @state: Miscellaneous bits describing inode state */
 	unsigned long state;
 
-	/** Lock for serializing lookup and readdir for back compatibility*/
+	/**
+	 * @mutex: Lock for serializing lookup and readdir for back
+	 * compatibility
+	 */
 	struct mutex mutex;
 
-	/** Lock to protect write related fields */
+	/** @lock: Lock to protect write-related fields */
 	spinlock_t lock;
 
 #ifdef CONFIG_FUSE_DAX
-	/*
-	 * Dax specific inode data
+	/**
+	 * @dax: Dax specific inode data
 	 */
 	struct fuse_inode_dax *dax;
 #endif
-	/** Submount specific lookup tracking */
+	/** @submount_lookup: Submount specific lookup tracking */
 	struct fuse_submount_lookup *submount_lookup;
 #ifdef CONFIG_FUSE_PASSTHROUGH
-	/** Reference to backing file in passthrough mode */
+	/** @fb: Reference to backing file in passthrough mode */
 	struct fuse_backing *fb;
 #endif
 
-	/*
-	 * The underlying inode->i_blkbits value will not be modified,
-	 * so preserve the blocksize specified by the server.
+	/**
+	 * @cached_i_blkbits: The underlying inode->i_blkbits value will not
+	 * be modified, so preserve the blocksize specified by the server.
 	 */
 	u8 cached_i_blkbits;
 };
@@ -250,116 +260,67 @@ struct fuse_conn;
 struct fuse_mount;
 union fuse_file_args;
 
-/** FUSE specific file data */
+/**
+ * struct fuse_file - FUSE-specific file data
+ */
 struct fuse_file {
-	/** Fuse connection for this file */
+	/** @fm: Fuse connection for this file */
 	struct fuse_mount *fm;
 
-	/* Argument space reserved for open/release */
+	/** @args: Argument space reserved for open/release */
 	union fuse_file_args *args;
 
-	/** Kernel file handle guaranteed to be unique */
+	/** @kh: Kernel file handle guaranteed to be unique */
 	u64 kh;
 
-	/** File handle used by userspace */
+	/** @fh: File handle used by userspace */
 	u64 fh;
 
-	/** Node id of this file */
+	/** @nodeid: Node id of this file */
 	u64 nodeid;
 
-	/** Refcount */
+	/** @count: Refcount */
 	refcount_t count;
 
-	/** FOPEN_* flags returned by open */
+	/** @open_flags: FOPEN_* flags returned by open */
 	u32 open_flags;
 
-	/** Entry on inode's write_files list */
+	/** @write_entry: Entry on inode's write_files list */
 	struct list_head write_entry;
 
-	/* Readdir related */
+	/** @readdir: Readdir-related */
 	struct {
-		/* Dir stream position */
+		/** @pos: Dir stream position */
 		loff_t pos;
 
-		/* Offset in cache */
+		/** @cache_off: Offset in cache */
 		loff_t cache_off;
 
-		/* Version of cache we are reading */
+		/** @version: Version of cache we are reading */
 		u64 version;
 
 	} readdir;
 
-	/** RB node to be linked on fuse_conn->polled_files */
+	/** @polled_node: RB node to be linked on fuse_conn->polled_files */
 	struct rb_node polled_node;
 
-	/** Wait queue head for poll */
+	/** @poll_wait: Wait queue head for poll */
 	wait_queue_head_t poll_wait;
 
-	/** Does file hold a fi->iocachectr refcount? */
+	/** @iomode: Does file hold a fi->iocachectr refcount? */
 	enum { IOM_NONE, IOM_CACHED, IOM_UNCACHED } iomode;
 
 #ifdef CONFIG_FUSE_PASSTHROUGH
-	/** Reference to backing file in passthrough mode */
+	/** @passthrough: Reference to backing file in passthrough mode */
 	struct file *passthrough;
+	/** @cred: passthrough file credentials */
 	const struct cred *cred;
 #endif
 
-	/** Has flock been performed on this file? */
+	/** @flock: Has flock been performed on this file? */
 	bool flock:1;
 };
 
-/** One input argument of a request */
-struct fuse_in_arg {
-	unsigned size;
-	const void *value;
-};
-
-/** One output argument of a request */
-struct fuse_arg {
-	unsigned size;
-	void *value;
-};
-
-/** FUSE folio descriptor */
-struct fuse_folio_desc {
-	unsigned int length;
-	unsigned int offset;
-};
-
-struct fuse_args {
-	uint64_t nodeid;
-	uint32_t opcode;
-	uint8_t in_numargs;
-	uint8_t out_numargs;
-	uint8_t ext_idx;
-	bool force:1;
-	bool noreply:1;
-	bool nocreds:1;
-	bool in_pages:1;
-	bool out_pages:1;
-	bool user_pages:1;
-	bool out_argvar:1;
-	bool page_zeroing:1;
-	bool page_replace:1;
-	bool may_block:1;
-	bool is_ext:1;
-	bool is_pinned:1;
-	bool invalidate_vmap:1;
-	bool abort_on_kill:1;
-	struct fuse_in_arg in_args[4];
-	struct fuse_arg out_args[2];
-	void (*end)(struct fuse_mount *fm, struct fuse_args *args, int error);
-	/* Used for kvec iter backed by vmalloc address */
-	void *vmap_base;
-};
-
-struct fuse_args_pages {
-	struct fuse_args args;
-	struct folio **folios;
-	struct fuse_folio_desc *descs;
-	unsigned int num_folios;
-};
-
 struct fuse_release_args {
 	struct fuse_args args;
 	struct fuse_release_in inarg;
@@ -399,200 +360,6 @@ struct fuse_io_priv {
 	.iocb = i,			\
 }
 
-/**
- * Request flags
- *
- * FR_ISREPLY:		set if the request has reply
- * FR_FORCE:		force sending of the request even if interrupted
- * FR_BACKGROUND:	request is sent in the background
- * FR_WAITING:		request is counted as "waiting"
- * FR_ABORTED:		the request was aborted
- * FR_INTERRUPTED:	the request has been interrupted
- * FR_LOCKED:		data is being copied to/from the request
- * FR_PENDING:		request is not yet in userspace
- * FR_SENT:		request is in userspace, waiting for an answer
- * FR_FINISHED:		request is finished
- * FR_PRIVATE:		request is on private list
- * FR_ASYNC:		request is asynchronous
- * FR_URING:		request is handled through fuse-io-uring
- */
-enum fuse_req_flag {
-	FR_ISREPLY,
-	FR_FORCE,
-	FR_BACKGROUND,
-	FR_WAITING,
-	FR_ABORTED,
-	FR_INTERRUPTED,
-	FR_LOCKED,
-	FR_PENDING,
-	FR_SENT,
-	FR_FINISHED,
-	FR_PRIVATE,
-	FR_ASYNC,
-	FR_URING,
-};
-
-/**
- * A request to the client
- *
- * .waitq.lock protects the following fields:
- *   - FR_ABORTED
- *   - FR_LOCKED (may also be modified under fc->lock, tested under both)
- */
-struct fuse_req {
-	/** This can be on either pending processing or io lists in
-	    fuse_conn */
-	struct list_head list;
-
-	/** Entry on the interrupts list  */
-	struct list_head intr_entry;
-
-	/* Input/output arguments */
-	struct fuse_args *args;
-
-	/** refcount */
-	refcount_t count;
-
-	/* Request flags, updated with test/set/clear_bit() */
-	unsigned long flags;
-
-	/* The request input header */
-	struct {
-		struct fuse_in_header h;
-	} in;
-
-	/* The request output header */
-	struct {
-		struct fuse_out_header h;
-	} out;
-
-	/** Used to wake up the task waiting for completion of request*/
-	wait_queue_head_t waitq;
-
-#if IS_ENABLED(CONFIG_VIRTIO_FS)
-	/** virtio-fs's physically contiguous buffer for in and out args */
-	void *argbuf;
-#endif
-
-	/** fuse_mount this request belongs to */
-	struct fuse_mount *fm;
-
-#ifdef CONFIG_FUSE_IO_URING
-	void *ring_entry;
-	void *ring_queue;
-#endif
-	/** When (in jiffies) the request was created */
-	unsigned long create_time;
-};
-
-struct fuse_iqueue;
-
-/**
- * Input queue callbacks
- *
- * Input queue signalling is device-specific.  For example, the /dev/fuse file
- * uses fiq->waitq and fasync to wake processes that are waiting on queue
- * readiness.  These callbacks allow other device types to respond to input
- * queue activity.
- */
-struct fuse_iqueue_ops {
-	/**
-	 * Send one forget
-	 */
-	void (*send_forget)(struct fuse_iqueue *fiq, struct fuse_forget_link *link);
-
-	/**
-	 * Send interrupt for request
-	 */
-	void (*send_interrupt)(struct fuse_iqueue *fiq, struct fuse_req *req);
-
-	/**
-	 * Send one request
-	 */
-	void (*send_req)(struct fuse_iqueue *fiq, struct fuse_req *req);
-
-	/**
-	 * Clean up when fuse_iqueue is destroyed
-	 */
-	void (*release)(struct fuse_iqueue *fiq);
-};
-
-/** /dev/fuse input queue operations */
-extern const struct fuse_iqueue_ops fuse_dev_fiq_ops;
-
-struct fuse_iqueue {
-	/** Connection established */
-	unsigned connected;
-
-	/** Lock protecting accesses to members of this structure */
-	spinlock_t lock;
-
-	/** Readers of the connection are waiting on this */
-	wait_queue_head_t waitq;
-
-	/** The next unique request id */
-	u64 reqctr;
-
-	/** The list of pending requests */
-	struct list_head pending;
-
-	/** Pending interrupts */
-	struct list_head interrupts;
-
-	/** Queue of pending forgets */
-	struct fuse_forget_link forget_list_head;
-	struct fuse_forget_link *forget_list_tail;
-
-	/** Batching of FORGET requests (positive indicates FORGET batch) */
-	int forget_batch;
-
-	/** O_ASYNC requests */
-	struct fasync_struct *fasync;
-
-	/** Device-specific callbacks */
-	const struct fuse_iqueue_ops *ops;
-
-	/** Device-specific state */
-	void *priv;
-};
-
-#define FUSE_PQ_HASH_BITS 8
-#define FUSE_PQ_HASH_SIZE (1 << FUSE_PQ_HASH_BITS)
-
-struct fuse_pqueue {
-	/** Connection established */
-	unsigned connected;
-
-	/** Lock protecting accessess to  members of this structure */
-	spinlock_t lock;
-
-	/** Hash table of requests being processed */
-	struct list_head *processing;
-
-	/** The list of requests under I/O */
-	struct list_head io;
-};
-
-/**
- * Fuse device instance
- */
-struct fuse_dev {
-	/** Reference count of this object */
-	refcount_t ref;
-
-	/** Issue FUSE_INIT synchronously */
-	bool sync_init;
-
-	/** Fuse connection for this device */
-	struct fuse_conn *fc;
-
-	/** Processing queue */
-	struct fuse_pqueue pq;
-
-	/** list entry on fc->devices */
-	struct list_head entry;
-};
-
 enum fuse_dax_mode {
 	FUSE_DAX_INODE_DEFAULT,	/* default */
 	FUSE_DAX_ALWAYS,	/* "-o dax=always" */
@@ -637,133 +404,135 @@ struct fuse_sync_bucket {
 };
 
 /**
- * A Fuse connection.
+ * struct fuse_conn - A Fuse connection.
  *
  * This structure is created, when the root filesystem is mounted, and
  * is destroyed, when the client device is closed and the last
  * fuse_mount is destroyed.
  */
 struct fuse_conn {
-	/** Lock protecting accessess to  members of this structure */
+	/**
+	 * @lock: Lock protecting:
+	 * - polled_files
+	 * - backing_files_map
+	 * - curr_bucket
+	 */
 	spinlock_t lock;
 
-	/** Refcount */
+	/** @count: Refcount */
 	refcount_t count;
 
-	/** Current epoch for up-to-date dentries */
+	/** @epoch: Current epoch for up-to-date dentries */
 	atomic_t epoch;
 
+	/** @epoch_work: Used to invalidate dentries from old epochs */
 	struct work_struct epoch_work;
 
+	/** @rcu: Used to delay freeing fuse_conn, making it safe */
 	struct rcu_head rcu;
 
-	/** The user id for this mount */
+	/** @user_id: The user id for this mount */
 	kuid_t user_id;
 
-	/** The group id for this mount */
+	/** @group_id: The group id for this mount */
 	kgid_t group_id;
 
-	/** The pid namespace for this mount */
+	/** @pid_ns: The pid namespace for this mount */
 	struct pid_namespace *pid_ns;
 
-	/** The user namespace for this mount */
+	/** @user_ns: The user namespace for this mount */
 	struct user_namespace *user_ns;
 
-	/** Maximum read size */
+	/** @max_read: Maximum read size */
 	unsigned max_read;
 
-	/** Maximum write size */
+	/** @max_write: Maximum write size */
 	unsigned max_write;
 
-	/** Maximum number of pages that can be used in a single request */
+	/**
+	 * @max_pages: Maximum number of pages that can be used in a
+	 * single request
+	 */
 	unsigned int max_pages;
 
-	/** Constrain ->max_pages to this value during feature negotiation */
+	/**
+	 * @max_pages_limit: Constrain ->max_pages to this value during
+	 * feature negotiation
+	 */
 	unsigned int max_pages_limit;
 
-	/** Input queue */
-	struct fuse_iqueue iq;
+	/** @chan: transport layer object */
+	struct fuse_chan *chan;
 
-	/** The next unique kernel file handle */
+	/** @khctr: The next unique kernel file handle */
 	atomic64_t khctr;
 
-	/** rbtree of fuse_files waiting for poll events indexed by ph */
+	/**
+	 * @polled_files: rbtree of fuse_files waiting for poll events
+	 * indexed by ph
+	 */
 	struct rb_root polled_files;
 
-	/** Maximum number of outstanding background requests */
-	unsigned max_background;
-
-	/** Number of background requests at which congestion starts */
+	/**
+	 * @congestion_threshold: Number of background requests at which
+	 * congestion starts
+	 */
 	unsigned congestion_threshold;
 
-	/** Number of requests currently in the background */
-	unsigned num_background;
-
-	/** Number of background requests currently queued for userspace */
-	unsigned active_background;
-
-	/** The list of background requests set aside for later queuing */
-	struct list_head bg_queue;
-
-	/** Protects: max_background, congestion_threshold, num_background,
-	 * active_background, bg_queue, blocked */
-	spinlock_t bg_lock;
-
-	/** Flag indicating that INIT reply has been received. Allocating
-	 * any fuse request will be suspended until the flag is set */
-	int initialized;
-
-	/** Flag indicating if connection is blocked.  This will be
-	    the case before the INIT reply is received, and if there
-	    are too many outstading backgrounds requests */
-	int blocked;
-
-	/** waitq for blocked connection */
-	wait_queue_head_t blocked_waitq;
-
-	/** Connection established, cleared on umount, connection
-	    abort and device release */
-	unsigned connected;
-
-	/** Connection aborted via sysfs */
-	bool aborted;
-
-	/** Connection failed (version mismatch).  Cannot race with
-	    setting other bitfields since it is only set once in INIT
-	    reply, before any other request, and never cleared */
+	/**
+	 * @conn_error: Connection failed (version mismatch).  Cannot race with
+	 * setting other bitfields since it is only set once in INIT
+	 * reply, before any other request, and never cleared
+	 */
 	unsigned conn_error:1;
 
-	/** Connection successful.  Only set in INIT */
+	/** @conn_init: Connection successful.  Only set in INIT */
 	unsigned conn_init:1;
 
-	/** Do readahead asynchronously?  Only set in INIT */
+	/** @async_read: Do readahead asynchronously?  Only set in INIT */
 	unsigned async_read:1;
 
-	/** Return an unique read error after abort.  Only set in INIT */
+	/**
+	 * @abort_err: Return an unique read error after abort.
+	 * Only set in INIT
+	 */
 	unsigned abort_err:1;
 
-	/** Do not send separate SETATTR request before open(O_TRUNC)  */
+	/**
+	 * @atomic_o_trunc: Do not send separate SETATTR request before
+	 * open(O_TRUNC)
+	 */
 	unsigned atomic_o_trunc:1;
 
-	/** Filesystem supports NFS exporting.  Only set in INIT */
+	/**
+	 * @export_support: Filesystem supports NFS exporting.
+	 * Only set in INIT
+	 */
 	unsigned export_support:1;
 
-	/** write-back cache policy (default is write-through) */
+	/** @writeback_cache: write-back cache policy (default is write-through) */
 	unsigned writeback_cache:1;
 
-	/** allow parallel lookups and readdir (default is serialized) */
+	/**
+	 * @parallel_dirops: allow parallel lookups and readdir (default is
+	 * serialized)
+	 */
 	unsigned parallel_dirops:1;
 
-	/** handle fs handles killing suid/sgid/cap on write/chown/trunc */
+	/**
+	 * @handle_killpriv: handle fs handles killing suid/sgid/cap on
+	 * write/chown/trunc
+	 */
 	unsigned handle_killpriv:1;
 
-	/** cache READLINK responses in page cache */
+	/** @cache_symlinks: cache READLINK responses in page cache */
 	unsigned cache_symlinks:1;
 
-	/* show legacy mount options */
+	/** @legacy_opts_show: show legacy mount options */
 	unsigned int legacy_opts_show:1;
 
-	/*
+	/**
+	 * @handle_killpriv_v2:
 	 * fs kills suid/sgid/cap on write/chown/trunc. suid is killed on
 	 * write/trunc only if caller did not have CAP_FSETID.  sgid is killed
 	 * on write/truncate only if caller did not have CAP_FSETID as well as
@@ -776,224 +545,219 @@ struct fuse_conn {
 	 * and hence races in setting them will not cause malfunction
 	 */
 
-	/** Is open/release not implemented by fs? */
+	/** @no_open: Is open/release not implemented by fs? */
 	unsigned no_open:1;
 
-	/** Is opendir/releasedir not implemented by fs? */
+	/** @no_opendir: Is opendir/releasedir not implemented by fs? */
 	unsigned no_opendir:1;
 
-	/** Is fsync not implemented by fs? */
+	/** @no_fsync: Is fsync not implemented by fs? */
 	unsigned no_fsync:1;
 
-	/** Is fsyncdir not implemented by fs? */
+	/** @no_fsyncdir: Is fsyncdir not implemented by fs? */
 	unsigned no_fsyncdir:1;
 
-	/** Is flush not implemented by fs? */
+	/** @no_flush: Is flush not implemented by fs? */
 	unsigned no_flush:1;
 
-	/** Is setxattr not implemented by fs? */
+	/** @no_setxattr: Is setxattr not implemented by fs? */
 	unsigned no_setxattr:1;
 
-	/** Does file server support extended setxattr */
+	/** @setxattr_ext: Does file server support extended setxattr */
 	unsigned setxattr_ext:1;
 
-	/** Is getxattr not implemented by fs? */
+	/** @no_getxattr: Is getxattr not implemented by fs? */
 	unsigned no_getxattr:1;
 
-	/** Is listxattr not implemented by fs? */
+	/** @no_listxattr: Is listxattr not implemented by fs? */
 	unsigned no_listxattr:1;
 
-	/** Is removexattr not implemented by fs? */
+	/** @no_removexattr: Is removexattr not implemented by fs? */
 	unsigned no_removexattr:1;
 
-	/** Are posix file locking primitives not implemented by fs? */
+	/** @no_lock: Are posix file locking primitives not implemented by fs? */
 	unsigned no_lock:1;
 
-	/** Is access not implemented by fs? */
+	/** @no_access: Is access not implemented by fs? */
 	unsigned no_access:1;
 
-	/** Is create not implemented by fs? */
+	/** @no_create: Is create not implemented by fs? */
 	unsigned no_create:1;
 
-	/** Is interrupt not implemented by fs? */
-	unsigned no_interrupt:1;
-
-	/** Is bmap not implemented by fs? */
+	/** @no_bmap: Is bmap not implemented by fs? */
 	unsigned no_bmap:1;
 
-	/** Is poll not implemented by fs? */
+	/** @no_poll: Is poll not implemented by fs? */
 	unsigned no_poll:1;
 
-	/** Do multi-page cached writes */
+	/** @big_writes: Do multi-page cached writes */
 	unsigned big_writes:1;
 
-	/** Don't apply umask to creation modes */
+	/** @dont_mask: Don't apply umask to creation modes */
 	unsigned dont_mask:1;
 
-	/** Are BSD file locking primitives not implemented by fs? */
+	/** @no_flock: Are BSD file locking primitives not implemented by fs? */
 	unsigned no_flock:1;
 
-	/** Is fallocate not implemented by fs? */
+	/** @no_fallocate: Is fallocate not implemented by fs? */
 	unsigned no_fallocate:1;
 
-	/** Is rename with flags implemented by fs? */
+	/** @no_rename2: Is rename with flags implemented by fs? */
 	unsigned no_rename2:1;
 
-	/** Use enhanced/automatic page cache invalidation. */
+	/** @auto_inval_data: Use enhanced/automatic page cache invalidation. */
 	unsigned auto_inval_data:1;
 
-	/** Filesystem is fully responsible for page cache invalidation. */
+	/**
+	 * @explicit_inval_data: Filesystem is fully responsible for page cache
+	 * invalidation.
+	 */
 	unsigned explicit_inval_data:1;
 
-	/** Does the filesystem support readdirplus? */
+	/** @do_readdirplus: Does the filesystem support readdirplus? */
 	unsigned do_readdirplus:1;
 
-	/** Does the filesystem want adaptive readdirplus? */
+	/** @readdirplus_auto: Does the filesystem want adaptive readdirplus? */
 	unsigned readdirplus_auto:1;
 
-	/** Does the filesystem support asynchronous direct-IO submission? */
+	/**
+	 * @async_dio: Does the filesystem support asynchronous direct-IO
+	 * submission?
+	 */
 	unsigned async_dio:1;
 
-	/** Is lseek not implemented by fs? */
+	/** @no_lseek: Is lseek not implemented by fs? */
 	unsigned no_lseek:1;
 
-	/** Does the filesystem support posix acls? */
+	/** @posix_acl: Does the filesystem support posix acls? */
 	unsigned posix_acl:1;
 
-	/** Check permissions based on the file mode or not? */
+	/**
+	 * @default_permissions: Check permissions based on the file mode
+	 * or not?
+	 */
 	unsigned default_permissions:1;
 
-	/** Allow other than the mounter user to access the filesystem ? */
+	/**
+	 * @allow_other: Allow other than the mounter user to access the
+	 * filesystem ?
+	 */
 	unsigned allow_other:1;
 
-	/** Does the filesystem support copy_file_range? */
+	/** @no_copy_file_range: Does the filesystem support copy_file_range? */
 	unsigned no_copy_file_range:1;
 
-	/** Does the filesystem support copy_file_range_64? */
+	/**
+	 * @no_copy_file_range_64: Does the filesystem support
+	 * copy_file_range_64?
+	 */
 	unsigned no_copy_file_range_64:1;
 
-	/* Send DESTROY request */
+	/** @destroy: Send DESTROY request */
 	unsigned int destroy:1;
 
-	/* Delete dentries that have gone stale */
+	/** @delete_stale: Delete dentries that have gone stale */
 	unsigned int delete_stale:1;
 
-	/** Do not create entry in fusectl fs */
+	/** @no_control: Do not create entry in fusectl fs */
 	unsigned int no_control:1;
 
-	/** Do not allow MNT_FORCE umount */
+	/** @no_force_umount: Do not allow MNT_FORCE umount */
 	unsigned int no_force_umount:1;
 
-	/* Auto-mount submounts announced by the server */
+	/** @auto_submounts: Auto-mount submounts announced by the server */
 	unsigned int auto_submounts:1;
 
-	/* Propagate syncfs() to server */
+	/** @sync_fs: Propagate syncfs() to server */
 	unsigned int sync_fs:1;
 
-	/* Initialize security xattrs when creating a new inode */
+	/** @init_security: Initialize security xattrs when creating a new inode */
 	unsigned int init_security:1;
 
-	/* Add supplementary group info when creating a new inode */
+	/**
+	 * @create_supp_group: Add supplementary group info when creating
+	 * a new inode
+	 */
 	unsigned int create_supp_group:1;
 
-	/* Does the filesystem support per inode DAX? */
+	/** @inode_dax: Does the filesystem support per inode DAX? */
 	unsigned int inode_dax:1;
 
-	/* Is tmpfile not implemented by fs? */
+	/** @no_tmpfile: Is tmpfile not implemented by fs? */
 	unsigned int no_tmpfile:1;
 
-	/* Relax restrictions to allow shared mmap in FOPEN_DIRECT_IO mode */
+	/**
+	 * @direct_io_allow_mmap: Relax restrictions to allow shared mmap
+	 * in FOPEN_DIRECT_IO mode
+	 */
 	unsigned int direct_io_allow_mmap:1;
 
-	/* Is statx not implemented by fs? */
+	/** @no_statx: Is statx not implemented by fs? */
 	unsigned int no_statx:1;
 
-	/** Passthrough support for read/write IO */
+	/** @passthrough: Passthrough support for read/write IO */
 	unsigned int passthrough:1;
 
-	/* Use pages instead of pointer for kernel I/O */
+	/** @use_pages_for_kvec_io: Use pages instead of pointer for kernel I/O */
 	unsigned int use_pages_for_kvec_io:1;
 
-	/* Is link not implemented by fs? */
+	/** @no_link: Is link not implemented by fs? */
 	unsigned int no_link:1;
 
-	/* Is synchronous FUSE_INIT allowed? */
+	/** @sync_init: Is synchronous FUSE_INIT allowed? */
 	unsigned int sync_init:1;
 
-	/* Use io_uring for communication */
-	unsigned int io_uring;
-
-	/** Maximum stack depth for passthrough backing files */
+	/** @max_stack_depth: Maximum stack depth for passthrough backing files */
 	int max_stack_depth;
 
-	/** The number of requests waiting for completion */
-	atomic_t num_waiting;
-
-	/** Negotiated minor version */
+	/** @minor: Negotiated minor version */
 	unsigned minor;
 
-	/** Entry on the fuse_conn_list */
+	/** @entry: Entry on the fuse_conn_list */
 	struct list_head entry;
 
-	/** Device ID from the root super block */
+	/** @dev: Device ID from the root super block */
 	dev_t dev;
 
-	/** Key for lock owner ID scrambling */
+	/** @scramble_key: Key for lock owner ID scrambling */
 	u32 scramble_key[4];
 
-	/** Version counter for attribute changes */
+	/** @attr_version: Version counter for attribute changes */
 	atomic64_t attr_version;
 
-	/** Version counter for evict inode */
+	/** @evict_ctr: Version counter for evict inode */
 	atomic64_t evict_ctr;
 
-	/* maximum file name length */
+	/** @name_max: maximum file name length */
 	u32 name_max;
 
-	/** Called on final put */
+	/** @release: Called on final put */
 	void (*release)(struct fuse_conn *);
 
 	/**
-	 * Read/write semaphore to hold when accessing the sb of any
+	 * @killsb: Read/write semaphore to hold when accessing the sb of any
 	 * fuse_mount belonging to this connection
 	 */
 	struct rw_semaphore killsb;
 
-	/** List of device instances belonging to this connection */
-	struct list_head devices;
-
 #ifdef CONFIG_FUSE_DAX
-	/* Dax mode */
+	/** @dax_mode: Dax mode */
 	enum fuse_dax_mode dax_mode;
 
-	/* Dax specific conn data, non-NULL if DAX is enabled */
+	/** @dax: Dax specific conn data, non-NULL if DAX is enabled */
 	struct fuse_conn_dax *dax;
 #endif
 
-	/** List of filesystems using this connection */
+	/** @mounts: List of filesystems using this connection */
 	struct list_head mounts;
 
-	/* New writepages go into this bucket */
+	/** @curr_bucket: New writepages go into this bucket */
 	struct fuse_sync_bucket __rcu *curr_bucket;
 
 #ifdef CONFIG_FUSE_PASSTHROUGH
-	/** IDR for backing files ids */
+	/** @backing_files_map: IDR for backing files ids */
 	struct idr backing_files_map;
 #endif
-
-#ifdef CONFIG_FUSE_IO_URING
-	/**  uring connection information*/
-	struct fuse_ring *ring;
-#endif
-
-	/** Only used if the connection opts into request timeouts */
-	struct {
-		/* Worker for checking if any requests have timed out */
-		struct delayed_work work;
-
-		/* Request timeout (in jiffies). 0 = no timeout */
-		unsigned int req_timeout;
-	} timeout;
 };
 
 /*
@@ -1136,7 +900,7 @@ extern const struct file_operations fuse_dev_operations;
 
 extern const struct dentry_operations fuse_dentry_operations;
 
-/**
+/*
  * Get a filled in inode
  */
 struct inode *fuse_iget(struct super_block *sb, u64 nodeid,
@@ -1147,14 +911,6 @@ struct inode *fuse_iget(struct super_block *sb, u64 nodeid,
 int fuse_lookup_name(struct super_block *sb, u64 nodeid, const struct qstr *name,
 		     struct fuse_entry_out *outarg, struct inode **inode);
 
-/**
- * Send FORGET command
- */
-void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
-		       u64 nodeid, u64 nlookup);
-
-struct fuse_forget_link *fuse_alloc_forget(void);
-
 /*
  * Initialize READ or READDIR request
  */
@@ -1186,44 +942,44 @@ int fuse_finish_open(struct inode *inode, struct file *file);
 void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff,
 		       unsigned int flags);
 
-/**
+/*
  * Send RELEASE or RELEASEDIR request
  */
 void fuse_release_common(struct file *file, bool isdir);
 
-/**
+/*
  * Send FSYNC or FSYNCDIR request
  */
 int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
 		      int datasync, int opcode);
 
-/**
+/*
  * Notify poll wakeup
  */
 int fuse_notify_poll_wakeup(struct fuse_conn *fc,
 			    struct fuse_notify_poll_wakeup_out *outarg);
 
-/**
+/*
  * Initialize file operations on a regular file
  */
 void fuse_init_file_inode(struct inode *inode, unsigned int flags);
 
-/**
+/*
  * Initialize inode operations on regular files and special files
  */
 void fuse_init_common(struct inode *inode);
 
-/**
+/*
  * Initialize inode and file operations on a directory
  */
 void fuse_init_dir(struct inode *inode);
 
-/**
+/*
  * Initialize inode operations on a symlink
  */
 void fuse_init_symlink(struct inode *inode);
 
-/**
+/*
  * Change attributes of an inode
  */
 void fuse_change_attributes(struct inode *inode, struct fuse_attr *attr,
@@ -1237,20 +993,10 @@ void fuse_change_attributes_common(struct inode *inode, struct fuse_attr *attr,
 
 u32 fuse_get_cache_mask(struct inode *inode);
 
-/**
- * Initialize the client device
- */
-int fuse_dev_init(void);
-
-/**
- * Cleanup the client device
- */
-void fuse_dev_cleanup(void);
-
 int fuse_ctl_init(void);
 void __exit fuse_ctl_cleanup(void);
 
-/**
+/*
  * Simple request sending that does request allocation and freeing
  */
 ssize_t __fuse_simple_request(struct mnt_idmap *idmap,
@@ -1271,30 +1017,14 @@ static inline ssize_t fuse_simple_idmap_request(struct mnt_idmap *idmap,
 
 int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args,
 			   gfp_t gfp_flags);
-
-/**
- * Assign a unique id to a fuse request
- */
-void fuse_request_assign_unique(struct fuse_iqueue *fiq, struct fuse_req *req);
-
-/**
- * End a finished request
- */
-void fuse_request_end(struct fuse_req *req);
-
-/* Abort all requests */
-void fuse_abort_conn(struct fuse_conn *fc);
-void fuse_wait_aborted(struct fuse_conn *fc);
-
-/* Check if any requests timed out */
-void fuse_check_timeout(struct work_struct *work);
+int fuse_simple_notify_reply(struct fuse_mount *fm, struct fuse_args *args, u64 unique);
 
 void fuse_dentry_tree_init(void);
 void fuse_dentry_tree_cleanup(void);
 
 void fuse_epoch_work(struct work_struct *work);
 
-/**
+/*
  * Invalidate inode attributes
  */
 
@@ -1304,6 +1034,9 @@ void fuse_epoch_work(struct work_struct *work);
 /* Attributes possibly changed on data and/or size modification */
 #define FUSE_STATX_MODSIZE	(FUSE_STATX_MODIFY | STATX_SIZE)
 
+/* Attributes possibly changed on directory modification */
+#define FUSE_STATX_MODDIR	(FUSE_STATX_MODSIZE | STATX_NLINK)
+
 void fuse_invalidate_attr(struct inode *inode);
 void fuse_invalidate_attr_mask(struct inode *inode, u32 mask);
 
@@ -1317,45 +1050,26 @@ u64 fuse_time_to_jiffies(u64 sec, u32 nsec);
 
 void fuse_change_entry_timeout(struct dentry *entry, struct fuse_entry_out *o);
 
-/**
- * Acquire reference to fuse_conn
- */
-struct fuse_conn *fuse_conn_get(struct fuse_conn *fc);
-
-/**
- * Initialize the fuse processing queue
- */
-void fuse_pqueue_init(struct fuse_pqueue *fpq);
-
-/**
+/*
  * Initialize fuse_conn
  */
 void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
-		    struct user_namespace *user_ns,
-		    const struct fuse_iqueue_ops *fiq_ops, void *fiq_priv);
+		    struct user_namespace *user_ns, struct fuse_chan *fch);
 
-/**
- * Release reference to fuse_conn
- */
-void fuse_conn_put(struct fuse_conn *fc);
-
-struct fuse_dev *fuse_dev_alloc_install(struct fuse_conn *fc);
-struct fuse_dev *fuse_dev_alloc(void);
-void fuse_dev_install(struct fuse_dev *fud, struct fuse_conn *fc);
-void fuse_dev_put(struct fuse_dev *fud);
 int fuse_send_init(struct fuse_mount *fm);
 
 /**
- * Fill in superblock and initialize fuse connection
+ * fuse_fill_super_common - Fill in superblock and initialize fuse connection
  * @sb: partially-initialized superblock to fill in
  * @ctx: mount context
  */
 int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx);
 
-/*
- * Remove the mount from the connection
+/**
+ * fuse_mount_remove - Remove the mount from the connection
+ * @fm: fuse_mount to remove
  *
- * Returns whether this was the last mount
+ * Returns: whether this was the last mount
  */
 bool fuse_mount_remove(struct fuse_mount *fm);
 
@@ -1373,23 +1087,25 @@ void fuse_conn_destroy(struct fuse_mount *fm);
 void fuse_mount_destroy(struct fuse_mount *fm);
 
 /**
- * Add connection to control filesystem
+ * fuse_ctl_add_conn - Add connection to control filesystem
+ * @fc: Fuse connection to add
  */
 int fuse_ctl_add_conn(struct fuse_conn *fc);
 
 /**
- * Remove connection from control filesystem
+ * fuse_ctl_remove_conn - Remove connection from control filesystem
+ * @fc: Fuse connection to remove
  */
 void fuse_ctl_remove_conn(struct fuse_conn *fc);
 
-/**
+/*
  * Is file type valid?
  */
 int fuse_valid_type(int m);
 
 bool fuse_invalid_attr(struct fuse_attr *attr);
 
-/**
+/*
  * Is current process allowed to perform filesystem operation?
  */
 bool fuse_allow_current_process(struct fuse_conn *fc);
@@ -1406,7 +1122,7 @@ void fuse_flush_writepages(struct inode *inode);
 void fuse_set_nowrite(struct inode *inode);
 void fuse_release_nowrite(struct inode *inode);
 
-/**
+/*
  * Scan all fuse_mounts belonging to fc to find the first where
  * ilookup5() returns a result.  Return that result and the
  * respective fuse_mount in *fm (unless fm is NULL).
@@ -1416,13 +1132,13 @@ void fuse_release_nowrite(struct inode *inode);
 struct inode *fuse_ilookup(struct fuse_conn *fc, u64 nodeid,
 			   struct fuse_mount **fm);
 
-/**
+/*
  * File-system tells the kernel to invalidate cache for the given node id.
  */
 int fuse_reverse_inval_inode(struct fuse_conn *fc, u64 nodeid,
 			     loff_t offset, loff_t len);
 
-/**
+/*
  * File-system tells the kernel to invalidate parent attributes and
  * the dentry matching parent/name.
  *
@@ -1444,7 +1160,7 @@ void fuse_try_prune_one_inode(struct fuse_conn *fc, u64 nodeid);
 int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
 		 bool isdir);
 
-/**
+/*
  * fuse_direct_io() flags
  */
 
@@ -1461,7 +1177,6 @@ long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
 long fuse_ioctl_common(struct file *file, unsigned int cmd,
 		       unsigned long arg, unsigned int flags);
 __poll_t fuse_file_poll(struct file *file, poll_table *wait);
-int fuse_dev_release(struct inode *inode, struct file *file);
 
 bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written);
 
@@ -1471,8 +1186,6 @@ int fuse_write_inode(struct inode *inode, struct writeback_control *wbc);
 int fuse_do_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 		    struct iattr *attr, struct file *file);
 
-void fuse_set_initialized(struct fuse_conn *fc);
-
 void fuse_unlock_inode(struct inode *inode, bool locked);
 bool fuse_lock_inode(struct inode *inode);
 
@@ -1494,15 +1207,6 @@ int fuse_set_acl(struct mnt_idmap *, struct dentry *dentry,
 /* readdir.c */
 int fuse_readdir(struct file *file, struct dir_context *ctx);
 
-/**
- * Return the number of bytes in an arguments list
- */
-unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args);
-
-/**
- * Get the next unique ID for a request
- */
-u64 fuse_get_unique(struct fuse_iqueue *fiq);
 void fuse_free_conn(struct fuse_conn *fc);
 
 /* dax.c */
@@ -1570,8 +1274,6 @@ static inline struct fuse_backing *fuse_backing_lookup(struct fuse_conn *fc,
 
 void fuse_backing_files_init(struct fuse_conn *fc);
 void fuse_backing_files_free(struct fuse_conn *fc);
-int fuse_backing_open(struct fuse_conn *fc, struct fuse_backing_map *map);
-int fuse_backing_close(struct fuse_conn *fc, int backing_id);
 
 /* passthrough.c */
 static inline struct fuse_backing *fuse_inode_backing(struct fuse_inode *fi)
diff --git a/fs/fuse/fuse_trace.h b/fs/fuse/fuse_trace.h
index bbe9ddd8c7169..4e34ddb172ed9 100644
--- a/fs/fuse/fuse_trace.h
+++ b/fs/fuse/fuse_trace.h
@@ -90,7 +90,7 @@ TRACE_EVENT(fuse_request_send,
 	),
 
 	TP_fast_assign(
-		__entry->connection	=	req->fm->fc->dev;
+		__entry->connection	=	req->chan->conn->dev;
 		__entry->unique		=	req->in.h.unique;
 		__entry->opcode		=	req->in.h.opcode;
 		__entry->len		=	req->in.h.len;
@@ -114,7 +114,7 @@ TRACE_EVENT(fuse_request_end,
 	),
 
 	TP_fast_assign(
-		__entry->connection	=	req->fm->fc->dev;
+		__entry->connection	=	req->chan->conn->dev;
 		__entry->unique		=	req->in.h.unique;
 		__entry->len		=	req->out.h.len;
 		__entry->error		=	req->out.h.error;
diff --git a/fs/fuse/inode.c b/fs/fuse/inode.c
index deddfffb037fb..0897f8e62b4d1 100644
--- a/fs/fuse/inode.c
+++ b/fs/fuse/inode.c
@@ -6,9 +6,8 @@
   See the file COPYING.
 */
 
+#include "dev.h"
 #include "fuse_i.h"
-#include "fuse_dev_i.h"
-#include "dev_uring_i.h"
 
 #include <linux/dax.h>
 #include <linux/pagemap.h>
@@ -35,14 +34,11 @@ MODULE_LICENSE("GPL");
 static struct kmem_cache *fuse_inode_cachep;
 struct list_head fuse_conn_list;
 DEFINE_MUTEX(fuse_mutex);
-DECLARE_WAIT_QUEUE_HEAD(fuse_dev_waitq);
 
 static int set_global_limit(const char *val, const struct kernel_param *kp);
 
 unsigned int fuse_max_pages_limit = 256;
 /* default is no timeout */
-unsigned int fuse_default_req_timeout;
-unsigned int fuse_max_req_timeout;
 
 unsigned int max_user_bgreq;
 module_param_call(max_user_bgreq, set_global_limit, param_get_uint,
@@ -62,9 +58,6 @@ MODULE_PARM_DESC(max_user_congthresh,
 
 #define FUSE_DEFAULT_BLKSIZE 512
 
-/** Maximum number of outstanding background requests */
-#define FUSE_DEFAULT_MAX_BACKGROUND 12
-
 /** Congestion starts at 75% of maximum */
 #define FUSE_DEFAULT_CONGESTION_THRESHOLD (FUSE_DEFAULT_MAX_BACKGROUND * 3 / 4)
 
@@ -72,11 +65,6 @@ MODULE_PARM_DESC(max_user_congthresh,
 static struct file_system_type fuseblk_fs_type;
 #endif
 
-struct fuse_forget_link *fuse_alloc_forget(void)
-{
-	return kzalloc_obj(struct fuse_forget_link, GFP_KERNEL_ACCOUNT);
-}
-
 static struct fuse_submount_lookup *fuse_alloc_submount_lookup(void)
 {
 	struct fuse_submount_lookup *sl;
@@ -150,7 +138,7 @@ static void fuse_cleanup_submount_lookup(struct fuse_conn *fc,
 	if (!refcount_dec_and_test(&sl->count))
 		return;
 
-	fuse_queue_forget(fc, sl->forget, sl->nodeid, 1);
+	fuse_chan_queue_forget(fc->chan, sl->forget, sl->nodeid, 1);
 	sl->forget = NULL;
 	kfree(sl);
 }
@@ -173,8 +161,8 @@ static void fuse_evict_inode(struct inode *inode)
 		if (FUSE_IS_DAX(inode))
 			fuse_dax_inode_cleanup(inode);
 		if (fi->nlookup) {
-			fuse_queue_forget(fc, fi->forget, fi->nodeid,
-					  fi->nlookup);
+			fuse_chan_queue_forget(fc->chan, fi->forget, fi->nodeid,
+					       fi->nlookup);
 			fi->forget = NULL;
 		}
 
@@ -624,7 +612,7 @@ static void fuse_umount_begin(struct super_block *sb)
 	if (fc->no_force_umount)
 		return;
 
-	fuse_abort_conn(fc);
+	fuse_chan_abort(fc->chan, false);
 
 	// Only retire block-device-based superblocks.
 	if (sb->s_bdev != NULL)
@@ -688,11 +676,9 @@ static struct fuse_sync_bucket *fuse_sync_bucket_alloc(void)
 	struct fuse_sync_bucket *bucket;
 
 	bucket = kzalloc_obj(*bucket, GFP_KERNEL | __GFP_NOFAIL);
-	if (bucket) {
-		init_waitqueue_head(&bucket->waitq);
-		/* Initial active count */
-		atomic_set(&bucket->count, 1);
-	}
+	init_waitqueue_head(&bucket->waitq);
+	/* Initial active count */
+	atomic_set(&bucket->count, 1);
 	return bucket;
 }
 
@@ -816,8 +802,7 @@ static int fuse_opt_fd(struct fs_context *fsc, struct file *file)
 	if (file->f_cred->user_ns != fsc->user_ns)
 		return invalfc(fsc, "wrong user namespace for fuse device");
 
-	ctx->fud = file->private_data;
-	refcount_inc(&ctx->fud->ref);
+	ctx->fud = fuse_dev_grab(file);
 
 	return 0;
 }
@@ -970,56 +955,19 @@ static int fuse_show_options(struct seq_file *m, struct dentry *root)
 	return 0;
 }
 
-static void fuse_iqueue_init(struct fuse_iqueue *fiq,
-			     const struct fuse_iqueue_ops *ops,
-			     void *priv)
-{
-	memset(fiq, 0, sizeof(struct fuse_iqueue));
-	spin_lock_init(&fiq->lock);
-	init_waitqueue_head(&fiq->waitq);
-	INIT_LIST_HEAD(&fiq->pending);
-	INIT_LIST_HEAD(&fiq->interrupts);
-	fiq->forget_list_tail = &fiq->forget_list_head;
-	fiq->connected = 1;
-	fiq->ops = ops;
-	fiq->priv = priv;
-}
-
-void fuse_pqueue_init(struct fuse_pqueue *fpq)
-{
-	unsigned int i;
-
-	spin_lock_init(&fpq->lock);
-	for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
-		INIT_LIST_HEAD(&fpq->processing[i]);
-	INIT_LIST_HEAD(&fpq->io);
-	fpq->connected = 1;
-}
-
 void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
-		    struct user_namespace *user_ns,
-		    const struct fuse_iqueue_ops *fiq_ops, void *fiq_priv)
+		    struct user_namespace *user_ns, struct fuse_chan *fch)
 {
 	memset(fc, 0, sizeof(*fc));
 	spin_lock_init(&fc->lock);
-	spin_lock_init(&fc->bg_lock);
 	init_rwsem(&fc->killsb);
 	refcount_set(&fc->count, 1);
 	atomic_set(&fc->epoch, 1);
 	INIT_WORK(&fc->epoch_work, fuse_epoch_work);
-	init_waitqueue_head(&fc->blocked_waitq);
-	fuse_iqueue_init(&fc->iq, fiq_ops, fiq_priv);
-	INIT_LIST_HEAD(&fc->bg_queue);
 	INIT_LIST_HEAD(&fc->entry);
-	INIT_LIST_HEAD(&fc->devices);
-	atomic_set(&fc->num_waiting, 0);
-	fc->max_background = FUSE_DEFAULT_MAX_BACKGROUND;
 	fc->congestion_threshold = FUSE_DEFAULT_CONGESTION_THRESHOLD;
 	atomic64_set(&fc->khctr, 0);
 	fc->polled_files = RB_ROOT;
-	fc->blocked = 0;
-	fc->initialized = 0;
-	fc->connected = 1;
 	atomic64_set(&fc->attr_version, 1);
 	atomic64_set(&fc->evict_ctr, 1);
 	get_random_bytes(&fc->scramble_key, sizeof(fc->scramble_key));
@@ -1028,7 +976,6 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
 	fc->max_pages = FUSE_DEFAULT_MAX_PAGES_PER_REQ;
 	fc->max_pages_limit = fuse_max_pages_limit;
 	fc->name_max = FUSE_NAME_LOW_MAX;
-	fc->timeout.req_timeout = 0;
 
 	if (IS_ENABLED(CONFIG_FUSE_PASSTHROUGH))
 		fuse_backing_files_init(fc);
@@ -1036,6 +983,8 @@ void fuse_conn_init(struct fuse_conn *fc, struct fuse_mount *fm,
 	INIT_LIST_HEAD(&fc->mounts);
 	list_add(&fm->fc_entry, &fc->mounts);
 	fm->fc = fc;
+	fuse_chan_set_fc(fch, fc);
+	fc->chan = fch;
 }
 EXPORT_SYMBOL_GPL(fuse_conn_init);
 
@@ -1043,7 +992,8 @@ static void delayed_release(struct rcu_head *p)
 {
 	struct fuse_conn *fc = container_of(p, struct fuse_conn, rcu);
 
-	fuse_uring_destruct(fc);
+	fuse_uring_destruct(fc->chan);
+	fuse_chan_free(fc->chan);
 
 	put_user_ns(fc->user_ns);
 	fc->release(fc);
@@ -1051,7 +1001,6 @@ static void delayed_release(struct rcu_head *p)
 
 void fuse_conn_put(struct fuse_conn *fc)
 {
-	struct fuse_iqueue *fiq = &fc->iq;
 	struct fuse_sync_bucket *bucket;
 
 	if (!refcount_dec_and_test(&fc->count))
@@ -1059,11 +1008,8 @@ void fuse_conn_put(struct fuse_conn *fc)
 
 	if (IS_ENABLED(CONFIG_FUSE_DAX))
 		fuse_dax_conn_free(fc);
-	if (fc->timeout.req_timeout)
-		cancel_delayed_work_sync(&fc->timeout.work);
 	cancel_work_sync(&fc->epoch_work);
-	if (fiq->ops->release)
-		fiq->ops->release(fiq);
+	fuse_chan_release(fc->chan);
 	put_pid_ns(fc->pid_ns);
 	bucket = rcu_dereference_protected(fc->curr_bucket, 1);
 	if (bucket) {
@@ -1083,6 +1029,11 @@ struct fuse_conn *fuse_conn_get(struct fuse_conn *fc)
 }
 EXPORT_SYMBOL_GPL(fuse_conn_get);
 
+dev_t fuse_conn_get_id(struct fuse_conn *fc)
+{
+	return fc->dev;
+}
+
 static struct inode *fuse_get_root_inode(struct super_block *sb, unsigned int mode)
 {
 	struct fuse_attr attr;
@@ -1294,12 +1245,13 @@ static void process_init_limits(struct fuse_conn *fc, struct fuse_init_out *arg)
 	sanitize_global_limit(&max_user_bgreq);
 	sanitize_global_limit(&max_user_congthresh);
 
-	spin_lock(&fc->bg_lock);
 	if (arg->max_background) {
-		fc->max_background = arg->max_background;
+		unsigned int max_background = max_background = arg->max_background;
+
+		if (!cap_sys_admin && max_background > max_user_bgreq)
+			max_background = max_user_bgreq;
 
-		if (!cap_sys_admin && fc->max_background > max_user_bgreq)
-			fc->max_background = max_user_bgreq;
+		fuse_chan_max_background_set(fc->chan, max_background);
 	}
 	if (arg->congestion_threshold) {
 		fc->congestion_threshold = arg->congestion_threshold;
@@ -1308,48 +1260,20 @@ static void process_init_limits(struct fuse_conn *fc, struct fuse_init_out *arg)
 		    fc->congestion_threshold > max_user_congthresh)
 			fc->congestion_threshold = max_user_congthresh;
 	}
-	spin_unlock(&fc->bg_lock);
-}
-
-static void set_request_timeout(struct fuse_conn *fc, unsigned int timeout)
-{
-	fc->timeout.req_timeout = secs_to_jiffies(timeout);
-	INIT_DELAYED_WORK(&fc->timeout.work, fuse_check_timeout);
-	queue_delayed_work(system_percpu_wq, &fc->timeout.work,
-			   fuse_timeout_timer_freq);
-}
-
-static void init_server_timeout(struct fuse_conn *fc, unsigned int timeout)
-{
-	if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout)
-		return;
-
-	if (!timeout)
-		timeout = fuse_default_req_timeout;
-
-	if (fuse_max_req_timeout) {
-		if (timeout)
-			timeout = min(fuse_max_req_timeout, timeout);
-		else
-			timeout = fuse_max_req_timeout;
-	}
-
-	timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout);
-
-	set_request_timeout(fc, timeout);
 }
 
 struct fuse_init_args {
 	struct fuse_args args;
 	struct fuse_init_in in;
 	struct fuse_init_out out;
+	struct fuse_mount *fm;
 };
 
-static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
-			       int error)
+static void process_init_reply(struct fuse_args *args, int error)
 {
-	struct fuse_conn *fc = fm->fc;
 	struct fuse_init_args *ia = container_of(args, typeof(*ia), args);
+	struct fuse_mount *fm = ia->fm;
+	struct fuse_conn *fc = fm->fc;
 	struct fuse_init_out *arg = &ia->out;
 	bool ok = true;
 
@@ -1481,7 +1405,7 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
 					ok = false;
 			}
 			if (flags & FUSE_OVER_IO_URING && fuse_uring_enabled())
-				fc->io_uring = 1;
+				fuse_chan_io_uring_enable(fc->chan);
 
 			if (flags & FUSE_REQUEST_TIMEOUT)
 				timeout = arg->request_timeout;
@@ -1491,7 +1415,7 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
 			fc->no_flock = 1;
 		}
 
-		init_server_timeout(fc, timeout);
+		fuse_init_server_timeout(fc->chan, timeout);
 
 		fm->sb->s_bdi->ra_pages =
 				min(fm->sb->s_bdi->ra_pages, ra_pages);
@@ -1505,10 +1429,15 @@ static void process_init_reply(struct fuse_mount *fm, struct fuse_args *args,
 	if (!ok) {
 		fc->conn_init = 0;
 		fc->conn_error = 1;
+		fuse_chan_set_initialized(fc->chan, NULL);
+	} else {
+		struct fuse_chan_param cp = {
+			.minor = fc->minor,
+			.max_write = fc->max_write,
+			.max_pages = fc->max_pages,
+		};
+		fuse_chan_set_initialized(fc->chan, &cp);
 	}
-
-	fuse_set_initialized(fc);
-	wake_up_all(&fc->blocked_waitq);
 }
 
 static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm)
@@ -1518,6 +1447,7 @@ static struct fuse_init_args *fuse_new_init(struct fuse_mount *fm)
 
 	ia = kzalloc_obj(*ia, GFP_KERNEL | __GFP_NOFAIL);
 
+	ia->fm = fm;
 	ia->in.major = FUSE_KERNEL_VERSION;
 	ia->in.minor = FUSE_KERNEL_MINOR_VERSION;
 	ia->in.max_readahead = fm->sb->s_bdi->ra_pages * PAGE_SIZE;
@@ -1591,7 +1521,7 @@ int fuse_send_init(struct fuse_mount *fm)
 		if (!err)
 			return 0;
 	}
-	process_init_reply(fm, &ia->args, err);
+	process_init_reply(&ia->args, err);
 	if (fm->fc->conn_error)
 		return -ENOTCONN;
 	return 0;
@@ -1600,7 +1530,6 @@ EXPORT_SYMBOL_GPL(fuse_send_init);
 
 void fuse_free_conn(struct fuse_conn *fc)
 {
-	WARN_ON(!list_empty(&fc->devices));
 	kfree(fc);
 }
 EXPORT_SYMBOL_GPL(fuse_free_conn);
@@ -1643,89 +1572,6 @@ static int fuse_bdi_init(struct fuse_conn *fc, struct super_block *sb)
 	return 0;
 }
 
-struct fuse_dev *fuse_dev_alloc(void)
-{
-	struct fuse_dev *fud;
-	struct list_head *pq;
-
-	fud = kzalloc_obj(struct fuse_dev);
-	if (!fud)
-		return NULL;
-
-	refcount_set(&fud->ref, 1);
-	pq = kzalloc_objs(struct list_head, FUSE_PQ_HASH_SIZE);
-	if (!pq) {
-		kfree(fud);
-		return NULL;
-	}
-
-	fud->pq.processing = pq;
-	fuse_pqueue_init(&fud->pq);
-
-	return fud;
-}
-EXPORT_SYMBOL_GPL(fuse_dev_alloc);
-
-void fuse_dev_install(struct fuse_dev *fud, struct fuse_conn *fc)
-{
-	struct fuse_conn *old_fc;
-
-	spin_lock(&fc->lock);
-	/*
-	 * Pairs with:
-	 *  - xchg() in fuse_dev_release()
-	 *  - smp_load_acquire() in fuse_dev_fc_get()
-	 */
-	old_fc = cmpxchg(&fud->fc, NULL, fc);
-	if (old_fc) {
-		/*
-		 * failed to set fud->fc because
-		 *  - it was already set to a different fc
-		 *  - it was set to disconneted
-		 */
-		fc->connected = 0;
-	} else {
-		list_add_tail(&fud->entry, &fc->devices);
-		fuse_conn_get(fc);
-	}
-	spin_unlock(&fc->lock);
-}
-EXPORT_SYMBOL_GPL(fuse_dev_install);
-
-struct fuse_dev *fuse_dev_alloc_install(struct fuse_conn *fc)
-{
-	struct fuse_dev *fud;
-
-	fud = fuse_dev_alloc();
-	if (!fud)
-		return NULL;
-
-	fuse_dev_install(fud, fc);
-	return fud;
-}
-EXPORT_SYMBOL_GPL(fuse_dev_alloc_install);
-
-void fuse_dev_put(struct fuse_dev *fud)
-{
-	struct fuse_conn *fc;
-
-	if (!refcount_dec_and_test(&fud->ref))
-		return;
-
-	fc = fuse_dev_fc_get(fud);
-	if (fc && fc != FUSE_DEV_FC_DISCONNECTED) {
-		/* This is the virtiofs case (fuse_dev_release() not called) */
-		spin_lock(&fc->lock);
-		list_del(&fud->entry);
-		spin_unlock(&fc->lock);
-
-		fuse_conn_put(fc);
-	}
-	kfree(fud->pq.processing);
-	kfree(fud);
-}
-EXPORT_SYMBOL_GPL(fuse_dev_put);
-
 static void fuse_fill_attr_from_inode(struct fuse_attr *attr,
 				      const struct fuse_inode *fi)
 {
@@ -1941,9 +1787,9 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx)
 	mutex_lock(&fuse_mutex);
 	err = -EINVAL;
 	if (fud) {
-		if (fuse_dev_fc_get(fud))
+		if (fuse_dev_is_installed(fud))
 			goto err_unlock;
-		if (fud->sync_init)
+		if (fuse_dev_is_sync_init(fud))
 			fc->sync_init = 1;
 	}
 
@@ -1953,10 +1799,9 @@ int fuse_fill_super_common(struct super_block *sb, struct fuse_fs_context *ctx)
 
 	list_add_tail(&fc->entry, &fuse_conn_list);
 	sb->s_root = root_dentry;
-	if (fud) {
-		fuse_dev_install(fud, fc);
-		wake_up_all(&fuse_dev_waitq);
-	}
+	if (fud)
+		fuse_dev_install(fud, fc->chan);
+
 	mutex_unlock(&fuse_mutex);
 	return 0;
 
@@ -2001,9 +1846,7 @@ static int fuse_set_no_super(struct super_block *sb, struct fs_context *fsc)
 
 static int fuse_test_super(struct super_block *sb, struct fs_context *fsc)
 {
-	struct fuse_dev *fud = fsc->sget_key;
-
-	return fuse_dev_fc_get(fud) == get_fuse_conn_super(sb);
+	return fuse_dev_verify(fsc->sget_key, get_fuse_conn_super(sb)->chan);
 }
 
 static int fuse_get_tree(struct fs_context *fsc)
@@ -2012,8 +1855,12 @@ static int fuse_get_tree(struct fs_context *fsc)
 	struct fuse_conn *fc;
 	struct fuse_mount *fm;
 	struct super_block *sb;
+	struct fuse_chan *fch __free(fuse_chan_free) = fuse_dev_chan_new();
 	int err;
 
+	if (!fch)
+		return -ENOMEM;
+
 	fc = kmalloc_obj(*fc);
 	if (!fc)
 		return -ENOMEM;
@@ -2024,7 +1871,7 @@ static int fuse_get_tree(struct fs_context *fsc)
 		return -ENOMEM;
 	}
 
-	fuse_conn_init(fc, fm, fsc->user_ns, &fuse_dev_fiq_ops, NULL);
+	fuse_conn_init(fc, fm, fsc->user_ns, no_free_ptr(fch));
 	fc->release = fuse_free_conn;
 
 	fsc->s_fs_info = fm;
@@ -2045,7 +1892,7 @@ static int fuse_get_tree(struct fs_context *fsc)
 	 * Allow creating a fuse mount with an already initialized fuse
 	 * connection
 	 */
-	if (fuse_dev_fc_get(ctx->fud)) {
+	if (fuse_dev_is_installed(ctx->fud)) {
 		fsc->sget_key = ctx->fud;
 		sb = sget_fc(fsc, fuse_test_super, fuse_set_no_super);
 		err = PTR_ERR_OR_ZERO(sb);
@@ -2116,8 +1963,8 @@ void fuse_conn_destroy(struct fuse_mount *fm)
 	if (fc->destroy)
 		fuse_send_destroy(fm);
 
-	fuse_abort_conn(fc);
-	fuse_wait_aborted(fc);
+	fuse_chan_abort(fc->chan, false);
+	fuse_chan_wait_aborted(fc->chan);
 
 	if (!list_empty(&fc->entry)) {
 		mutex_lock(&fuse_mutex);
diff --git a/fs/fuse/notify.c b/fs/fuse/notify.c
new file mode 100644
index 0000000000000..f200a33f05332
--- /dev/null
+++ b/fs/fuse/notify.c
@@ -0,0 +1,434 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "dev.h"
+#include "fuse_i.h"
+#include <linux/pagemap.h>
+
+static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
+			    struct fuse_copy_state *cs)
+{
+	struct fuse_notify_poll_wakeup_out outarg;
+	int err;
+
+	if (size != sizeof(outarg))
+		return -EINVAL;
+
+	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
+	if (err)
+		return err;
+
+	fuse_copy_finish(cs);
+	return fuse_notify_poll_wakeup(fc, &outarg);
+}
+
+static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
+				   struct fuse_copy_state *cs)
+{
+	struct fuse_notify_inval_inode_out outarg;
+	int err;
+
+	if (size != sizeof(outarg))
+		return -EINVAL;
+
+	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
+	if (err)
+		return err;
+	fuse_copy_finish(cs);
+
+	down_read(&fc->killsb);
+	err = fuse_reverse_inval_inode(fc, outarg.ino,
+				       outarg.off, outarg.len);
+	up_read(&fc->killsb);
+	return err;
+}
+
+static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
+				   struct fuse_copy_state *cs)
+{
+	struct fuse_notify_inval_entry_out outarg;
+	int err;
+	char *buf;
+	struct qstr name;
+
+	if (size < sizeof(outarg))
+		return -EINVAL;
+
+	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
+	if (err)
+		return err;
+
+	if (outarg.namelen > fc->name_max)
+		return -ENAMETOOLONG;
+
+	err = -EINVAL;
+	if (size != sizeof(outarg) + outarg.namelen + 1)
+		return -EINVAL;
+
+	buf = kzalloc(outarg.namelen + 1, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	name.name = buf;
+	name.len = outarg.namelen;
+	err = fuse_copy_one(cs, buf, outarg.namelen + 1);
+	if (err)
+		goto err;
+	fuse_copy_finish(cs);
+	buf[outarg.namelen] = 0;
+
+	down_read(&fc->killsb);
+	err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name, outarg.flags);
+	up_read(&fc->killsb);
+err:
+	kfree(buf);
+	return err;
+}
+
+static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
+			      struct fuse_copy_state *cs)
+{
+	struct fuse_notify_delete_out outarg;
+	int err;
+	char *buf;
+	struct qstr name;
+
+	if (size < sizeof(outarg))
+		return -EINVAL;
+
+	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
+	if (err)
+		return err;
+
+	if (outarg.namelen > fc->name_max)
+		return -ENAMETOOLONG;
+
+	if (size != sizeof(outarg) + outarg.namelen + 1)
+		return -EINVAL;
+
+	buf = kzalloc(outarg.namelen + 1, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	name.name = buf;
+	name.len = outarg.namelen;
+	err = fuse_copy_one(cs, buf, outarg.namelen + 1);
+	if (err)
+		goto err;
+	fuse_copy_finish(cs);
+	buf[outarg.namelen] = 0;
+
+	down_read(&fc->killsb);
+	err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name, 0);
+	up_read(&fc->killsb);
+err:
+	kfree(buf);
+	return err;
+}
+
+static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
+			     struct fuse_copy_state *cs)
+{
+	struct fuse_notify_store_out outarg;
+	struct inode *inode;
+	struct address_space *mapping;
+	u64 nodeid;
+	int err;
+	unsigned int num;
+	loff_t file_size;
+	loff_t pos;
+	loff_t end;
+
+	if (size < sizeof(outarg))
+		return -EINVAL;
+
+	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
+	if (err)
+		return err;
+
+	if (size - sizeof(outarg) != outarg.size)
+		return -EINVAL;
+
+	if (outarg.offset >= MAX_LFS_FILESIZE)
+		return -EINVAL;
+
+	nodeid = outarg.nodeid;
+	pos = outarg.offset;
+	num = min(outarg.size, MAX_LFS_FILESIZE - pos);
+
+	down_read(&fc->killsb);
+
+	err = -ENOENT;
+	inode = fuse_ilookup(fc, nodeid,  NULL);
+	if (!inode)
+		goto out_up_killsb;
+
+	mapping = inode->i_mapping;
+	file_size = i_size_read(inode);
+	end = pos + num;
+	if (end > file_size) {
+		file_size = end;
+		fuse_write_update_attr(inode, file_size, num);
+	}
+
+	while (num) {
+		struct folio *folio;
+		unsigned int folio_offset;
+		unsigned int nr_bytes;
+		pgoff_t index = pos >> PAGE_SHIFT;
+
+		folio = filemap_grab_folio(mapping, index);
+		err = PTR_ERR(folio);
+		if (IS_ERR(folio))
+			goto out_iput;
+
+		folio_offset = offset_in_folio(folio, pos);
+		nr_bytes = min(num, folio_size(folio) - folio_offset);
+
+		err = fuse_copy_folio(cs, &folio, folio_offset, nr_bytes, 0);
+		if (!folio_test_uptodate(folio) && !err && folio_offset == 0 &&
+		    (nr_bytes == folio_size(folio) || file_size == end)) {
+			folio_zero_segment(folio, nr_bytes, folio_size(folio));
+			folio_mark_uptodate(folio);
+		}
+		folio_unlock(folio);
+		folio_put(folio);
+
+		if (err)
+			goto out_iput;
+
+		pos += nr_bytes;
+		num -= nr_bytes;
+	}
+
+	err = 0;
+
+out_iput:
+	iput(inode);
+out_up_killsb:
+	up_read(&fc->killsb);
+	return err;
+}
+
+struct fuse_retrieve_args {
+	struct fuse_args_pages ap;
+	struct fuse_notify_retrieve_in inarg;
+};
+
+static void fuse_retrieve_end(struct fuse_args *args, int error)
+{
+	struct fuse_retrieve_args *ra =
+		container_of(args, typeof(*ra), ap.args);
+
+	release_pages(ra->ap.folios, ra->ap.num_folios);
+	kfree(ra);
+}
+
+static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
+			 struct fuse_notify_retrieve_out *outarg)
+{
+	int err;
+	struct address_space *mapping = inode->i_mapping;
+	loff_t file_size;
+	unsigned int num;
+	unsigned int offset;
+	size_t total_len = 0;
+	unsigned int num_pages;
+	struct fuse_conn *fc = fm->fc;
+	struct fuse_retrieve_args *ra;
+	size_t args_size = sizeof(*ra);
+	struct fuse_args_pages *ap;
+	struct fuse_args *args;
+	loff_t pos = outarg->offset;
+
+	offset = offset_in_page(pos);
+	file_size = i_size_read(inode);
+
+	num = min(outarg->size, fc->max_write);
+	if (pos > file_size)
+		num = 0;
+	else if (num > file_size - pos)
+		num = file_size - pos;
+
+	num_pages = DIV_ROUND_UP(num + offset, PAGE_SIZE);
+	num_pages = min(num_pages, fc->max_pages);
+	num = min(num, num_pages << PAGE_SHIFT);
+
+	args_size += num_pages * (sizeof(ap->folios[0]) + sizeof(ap->descs[0]));
+
+	ra = kzalloc(args_size, GFP_KERNEL);
+	if (!ra)
+		return -ENOMEM;
+
+	ap = &ra->ap;
+	ap->folios = (void *) (ra + 1);
+	ap->descs = (void *) (ap->folios + num_pages);
+
+	args = &ap->args;
+	args->nodeid = outarg->nodeid;
+	args->opcode = FUSE_NOTIFY_REPLY;
+	args->in_numargs = 3;
+	args->in_pages = true;
+	args->end = fuse_retrieve_end;
+
+	while (num && ap->num_folios < num_pages) {
+		struct folio *folio;
+		unsigned int folio_offset;
+		unsigned int nr_bytes;
+		pgoff_t index = pos >> PAGE_SHIFT;
+
+		folio = filemap_get_folio(mapping, index);
+		if (IS_ERR(folio))
+			break;
+
+		folio_offset = offset_in_folio(folio, pos);
+		nr_bytes = min(folio_size(folio) - folio_offset, num);
+
+		ap->folios[ap->num_folios] = folio;
+		ap->descs[ap->num_folios].offset = folio_offset;
+		ap->descs[ap->num_folios].length = nr_bytes;
+		ap->num_folios++;
+
+		pos += nr_bytes;
+		num -= nr_bytes;
+		total_len += nr_bytes;
+	}
+	ra->inarg.offset = outarg->offset;
+	ra->inarg.size = total_len;
+	fuse_set_zero_arg0(args);
+	args->in_args[1].size = sizeof(ra->inarg);
+	args->in_args[1].value = &ra->inarg;
+	args->in_args[2].size = total_len;
+
+	err = fuse_simple_notify_reply(fm, args, outarg->notify_unique);
+	if (err)
+		fuse_retrieve_end(args, err);
+
+	return err;
+}
+
+static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
+				struct fuse_copy_state *cs)
+{
+	struct fuse_notify_retrieve_out outarg;
+	struct fuse_mount *fm;
+	struct inode *inode;
+	u64 nodeid;
+	int err;
+
+	if (size != sizeof(outarg))
+		return -EINVAL;
+
+	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
+	if (err)
+		return err;
+
+	fuse_copy_finish(cs);
+
+	if (outarg.offset >= MAX_LFS_FILESIZE)
+		return -EINVAL;
+
+	down_read(&fc->killsb);
+	err = -ENOENT;
+	nodeid = outarg.nodeid;
+
+	inode = fuse_ilookup(fc, nodeid, &fm);
+	if (inode) {
+		err = fuse_retrieve(fm, inode, &outarg);
+		iput(inode);
+	}
+	up_read(&fc->killsb);
+
+	return err;
+}
+
+static int fuse_notify_resend(struct fuse_conn *fc)
+{
+	fuse_chan_resend(fc->chan);
+	return 0;
+}
+
+/*
+ * Increments the fuse connection epoch.  This will result of dentries from
+ * previous epochs to be invalidated.  Additionally, if inval_wq is set, a work
+ * queue is scheduled to trigger the invalidation.
+ */
+static int fuse_notify_inc_epoch(struct fuse_conn *fc)
+{
+	atomic_inc(&fc->epoch);
+	if (inval_wq)
+		schedule_work(&fc->epoch_work);
+
+	return 0;
+}
+
+static int fuse_notify_prune(struct fuse_conn *fc, unsigned int size,
+			     struct fuse_copy_state *cs)
+{
+	struct fuse_notify_prune_out outarg;
+	const unsigned int batch = 512;
+	u64 *nodeids __free(kfree) = kmalloc(sizeof(u64) * batch, GFP_KERNEL);
+	unsigned int num, i;
+	int err;
+
+	if (!nodeids)
+		return -ENOMEM;
+
+	if (size < sizeof(outarg))
+		return -EINVAL;
+
+	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
+	if (err)
+		return err;
+
+	if (size - sizeof(outarg) != outarg.count * sizeof(u64))
+		return -EINVAL;
+
+	for (; outarg.count; outarg.count -= num) {
+		num = min(batch, outarg.count);
+		err = fuse_copy_one(cs, nodeids, num * sizeof(u64));
+		if (err)
+			return err;
+
+		scoped_guard(rwsem_read, &fc->killsb) {
+			for (i = 0; i < num; i++)
+				fuse_try_prune_one_inode(fc, nodeids[i]);
+		}
+	}
+	return 0;
+}
+
+int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
+		unsigned int size, struct fuse_copy_state *cs)
+{
+	switch (code) {
+	case FUSE_NOTIFY_POLL:
+		return fuse_notify_poll(fc, size, cs);
+
+	case FUSE_NOTIFY_INVAL_INODE:
+		return fuse_notify_inval_inode(fc, size, cs);
+
+	case FUSE_NOTIFY_INVAL_ENTRY:
+		return fuse_notify_inval_entry(fc, size, cs);
+
+	case FUSE_NOTIFY_STORE:
+		return fuse_notify_store(fc, size, cs);
+
+	case FUSE_NOTIFY_RETRIEVE:
+		return fuse_notify_retrieve(fc, size, cs);
+
+	case FUSE_NOTIFY_DELETE:
+		return fuse_notify_delete(fc, size, cs);
+
+	case FUSE_NOTIFY_RESEND:
+		return fuse_notify_resend(fc);
+
+	case FUSE_NOTIFY_INC_EPOCH:
+		return fuse_notify_inc_epoch(fc);
+
+	case FUSE_NOTIFY_PRUNE:
+		return fuse_notify_prune(fc, size, cs);
+
+	default:
+		return -EINVAL;
+	}
+}
diff --git a/fs/fuse/poll.c b/fs/fuse/poll.c
new file mode 100644
index 0000000000000..bce3ee2e861eb
--- /dev/null
+++ b/fs/fuse/poll.c
@@ -0,0 +1,141 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "dev.h"
+#include "fuse_i.h"
+
+void fuse_end_polls(struct fuse_conn *fc)
+{
+	struct rb_node *p;
+
+	spin_lock(&fc->lock);
+	p = rb_first(&fc->polled_files);
+
+	while (p) {
+		struct fuse_file *ff;
+		ff = rb_entry(p, struct fuse_file, polled_node);
+		wake_up_interruptible_all(&ff->poll_wait);
+
+		p = rb_next(p);
+	}
+	spin_unlock(&fc->lock);
+}
+
+/*
+ * All files which have been polled are linked to RB tree
+ * fuse_conn->polled_files which is indexed by kh.  Walk the tree and
+ * find the matching one.
+ */
+static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
+					      struct rb_node **parent_out)
+{
+	struct rb_node **link = &fc->polled_files.rb_node;
+	struct rb_node *last = NULL;
+
+	while (*link) {
+		struct fuse_file *ff;
+
+		last = *link;
+		ff = rb_entry(last, struct fuse_file, polled_node);
+
+		if (kh < ff->kh)
+			link = &last->rb_left;
+		else if (kh > ff->kh)
+			link = &last->rb_right;
+		else
+			return link;
+	}
+
+	if (parent_out)
+		*parent_out = last;
+	return link;
+}
+
+/*
+ * The file is about to be polled.  Make sure it's on the polled_files
+ * RB tree.  Note that files once added to the polled_files tree are
+ * not removed before the file is released.  This is because a file
+ * polled once is likely to be polled again.
+ */
+static void fuse_register_polled_file(struct fuse_conn *fc,
+				      struct fuse_file *ff)
+{
+	spin_lock(&fc->lock);
+	if (RB_EMPTY_NODE(&ff->polled_node)) {
+		struct rb_node **link, *parent;
+
+		link = fuse_find_polled_node(fc, ff->kh, &parent);
+		BUG_ON(*link);
+		rb_link_node(&ff->polled_node, parent, link);
+		rb_insert_color(&ff->polled_node, &fc->polled_files);
+	}
+	spin_unlock(&fc->lock);
+}
+
+__poll_t fuse_file_poll(struct file *file, poll_table *wait)
+{
+	struct fuse_file *ff = file->private_data;
+	struct fuse_mount *fm = ff->fm;
+	struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
+	struct fuse_poll_out outarg;
+	FUSE_ARGS(args);
+	int err;
+
+	if (fm->fc->no_poll)
+		return DEFAULT_POLLMASK;
+
+	poll_wait(file, &ff->poll_wait, wait);
+	inarg.events = mangle_poll(poll_requested_events(wait));
+
+	/*
+	 * Ask for notification iff there's someone waiting for it.
+	 * The client may ignore the flag and always notify.
+	 */
+	if (waitqueue_active(&ff->poll_wait)) {
+		inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
+		fuse_register_polled_file(fm->fc, ff);
+	}
+
+	args.opcode = FUSE_POLL;
+	args.nodeid = ff->nodeid;
+	args.in_numargs = 1;
+	args.in_args[0].size = sizeof(inarg);
+	args.in_args[0].value = &inarg;
+	args.out_numargs = 1;
+	args.out_args[0].size = sizeof(outarg);
+	args.out_args[0].value = &outarg;
+	err = fuse_simple_request(fm, &args);
+
+	if (!err)
+		return demangle_poll(outarg.revents);
+	if (err == -ENOSYS) {
+		fm->fc->no_poll = 1;
+		return DEFAULT_POLLMASK;
+	}
+	return EPOLLERR;
+}
+EXPORT_SYMBOL_GPL(fuse_file_poll);
+
+/*
+ * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
+ * wakes up the poll waiters.
+ */
+int fuse_notify_poll_wakeup(struct fuse_conn *fc,
+			    struct fuse_notify_poll_wakeup_out *outarg)
+{
+	u64 kh = outarg->kh;
+	struct rb_node **link;
+
+	spin_lock(&fc->lock);
+
+	link = fuse_find_polled_node(fc, kh, NULL);
+	if (*link) {
+		struct fuse_file *ff;
+
+		ff = rb_entry(*link, struct fuse_file, polled_node);
+		wake_up_interruptible_sync(&ff->poll_wait);
+	}
+
+	spin_unlock(&fc->lock);
+	return 0;
+}
+
diff --git a/fs/fuse/req.c b/fs/fuse/req.c
new file mode 100644
index 0000000000000..a01ee743d31e9
--- /dev/null
+++ b/fs/fuse/req.c
@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "dev.h"
+#include "fuse_i.h"
+
+static int fuse_fill_creds(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap)
+{
+	struct fuse_conn *fc = fm->fc;
+	bool no_idmap = !fm->sb || (fm->sb->s_iflags & SB_I_NOIDMAP);
+	kuid_t fsuid = mapped_fsuid(idmap, fc->user_ns);
+	kgid_t fsgid = mapped_fsgid(idmap, fc->user_ns);
+
+	args->pid = pid_nr_ns(task_pid(current), fc->pid_ns);
+
+	if (args->force) {
+		if (args->nocreds)
+			return 0;
+
+		if (no_idmap) {
+			args->uid = from_kuid_munged(fc->user_ns, current_fsuid());
+			args->gid = from_kgid_munged(fc->user_ns, current_fsgid());
+		} else {
+			args->uid = FUSE_INVALID_UIDGID;
+			args->gid = FUSE_INVALID_UIDGID;
+		}
+		return 0;
+	}
+
+	WARN_ON(args->nocreds);
+	/*
+	 * Keep the old behavior when idmappings support was not
+	 * declared by a FUSE server.
+	 *
+	 * For those FUSE servers who support idmapped mounts, we send UID/GID
+	 * only along with "inode creation" fuse requests, otherwise idmap ==
+	 * &invalid_mnt_idmap and req->in.h.{u,g}id will be equal to
+	 * FUSE_INVALID_UIDGID.
+	 */
+	if (no_idmap) {
+		fsuid = current_fsuid();
+		fsgid = current_fsgid();
+	}
+	args->uid = from_kuid(fc->user_ns, fsuid);
+	args->gid = from_kgid(fc->user_ns, fsgid);
+
+	if (no_idmap && unlikely(args->uid == ((uid_t)-1) || args->gid == ((gid_t)-1)))
+		return -EOVERFLOW;
+
+	return 0;
+}
+
+static int fuse_req_prep(struct fuse_mount *fm, struct fuse_args *args, struct mnt_idmap *idmap)
+{
+	if (!args->force && fm->fc->conn_error)
+		return -ECONNREFUSED;
+
+	return fuse_fill_creds(fm, args, idmap);
+}
+
+ssize_t __fuse_simple_request(struct mnt_idmap *idmap, struct fuse_mount *fm,
+			      struct fuse_args *args)
+{
+	struct fuse_conn *fc = fm->fc;
+	int err = fuse_req_prep(fm, args, idmap);
+
+	if (err)
+		return err;
+
+	return fuse_chan_send(fc->chan, args);
+}
+
+int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args, gfp_t gfp_flags)
+{
+	struct fuse_conn *fc = fm->fc;
+	int err;
+
+	WARN_ON(args->force && !args->nocreds);
+
+	err = fuse_req_prep(fm, args, &invalid_mnt_idmap);
+	if (err)
+		return err;
+
+	return fuse_chan_send_bg(fc->chan, args, gfp_flags);
+}
+EXPORT_SYMBOL_GPL(fuse_simple_background);
+
+int fuse_simple_notify_reply(struct fuse_mount *fm, struct fuse_args *args, u64 unique)
+{
+	struct fuse_conn *fc = fm->fc;
+	int err;
+
+	WARN_ON(args->force && !args->nocreds);
+
+	err = fuse_req_prep(fm, args, &invalid_mnt_idmap);
+	if (err)
+		return err;
+
+	return fuse_chan_send_notify_reply(fc->chan, args, unique);
+}
diff --git a/fs/fuse/req_timeout.c b/fs/fuse/req_timeout.c
new file mode 100644
index 0000000000000..6cc6fc4913437
--- /dev/null
+++ b/fs/fuse/req_timeout.c
@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include "dev.h"
+#include "sysctl.h"
+#include "fuse_dev_i.h"
+#include "dev_uring_i.h"
+
+/* Frequency (in seconds) of request timeout checks, if opted into */
+#define FUSE_TIMEOUT_TIMER_FREQ 15
+
+/* Frequency (in jiffies) of request timeout checks, if opted into */
+static const unsigned long fuse_timeout_timer_freq =
+	secs_to_jiffies(FUSE_TIMEOUT_TIMER_FREQ);
+
+/*
+ * Default timeout (in seconds) for the server to reply to a request
+ * before the connection is aborted, if no timeout was specified on mount.
+ *
+ * Exported via sysctl
+ */
+unsigned int fuse_default_req_timeout;
+
+/*
+ * Max timeout (in seconds) for the server to reply to a request before
+ * the connection is aborted.
+ *
+ * Exported via sysctl
+ */
+unsigned int fuse_max_req_timeout;
+
+bool fuse_request_expired(struct fuse_chan *fch, struct list_head *list)
+{
+	struct fuse_req *req;
+
+	req = list_first_entry_or_null(list, struct fuse_req, list);
+	if (!req)
+		return false;
+	return time_is_before_jiffies(req->create_time + fch->timeout.req_timeout);
+}
+
+static bool fuse_fpq_processing_expired(struct fuse_chan *fch, struct list_head *processing)
+{
+	int i;
+
+	for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
+		if (fuse_request_expired(fch, &processing[i]))
+			return true;
+
+	return false;
+}
+
+/*
+ * Check if any requests aren't being completed by the time the request timeout
+ * elapses. To do so, we:
+ * - check the fiq pending list
+ * - check the bg queue
+ * - check the fpq io and processing lists
+ *
+ * To make this fast, we only check against the head request on each list since
+ * these are generally queued in order of creation time (eg newer requests get
+ * queued to the tail). We might miss a few edge cases (eg requests transitioning
+ * between lists, re-sent requests at the head of the pending list having a
+ * later creation time than other requests on that list, etc.) but that is fine
+ * since if the request never gets fulfilled, it will eventually be caught.
+ */
+static void fuse_check_timeout(struct work_struct *work)
+{
+	struct delayed_work *dwork = to_delayed_work(work);
+	struct fuse_chan *fch = container_of(dwork, struct fuse_chan, timeout.work);
+	struct fuse_iqueue *fiq = &fch->iq;
+	struct fuse_dev *fud;
+	struct fuse_pqueue *fpq;
+	bool expired = false;
+
+	if (!atomic_read(&fch->num_waiting))
+		goto out;
+
+	spin_lock(&fiq->lock);
+	expired = fuse_request_expired(fch, &fiq->pending);
+	spin_unlock(&fiq->lock);
+	if (expired)
+		goto chan_abort;
+
+	spin_lock(&fch->bg_lock);
+	expired = fuse_request_expired(fch, &fch->bg_queue);
+	spin_unlock(&fch->bg_lock);
+	if (expired)
+		goto chan_abort;
+
+	spin_lock(&fch->lock);
+	if (!fch->connected) {
+		spin_unlock(&fch->lock);
+		return;
+	}
+	list_for_each_entry(fud, &fch->devices, entry) {
+		fpq = &fud->pq;
+		spin_lock(&fpq->lock);
+		if (fuse_request_expired(fch, &fpq->io) ||
+		    fuse_fpq_processing_expired(fch, fpq->processing)) {
+			spin_unlock(&fpq->lock);
+			spin_unlock(&fch->lock);
+			goto chan_abort;
+		}
+
+		spin_unlock(&fpq->lock);
+	}
+	spin_unlock(&fch->lock);
+
+	if (fuse_uring_request_expired(fch))
+		goto chan_abort;
+
+out:
+	queue_delayed_work(system_percpu_wq, &fch->timeout.work,
+			   fuse_timeout_timer_freq);
+	return;
+
+chan_abort:
+	fuse_chan_abort(fch, false);
+}
+
+static void set_request_timeout(struct fuse_chan *fch, unsigned int timeout)
+{
+	fch->timeout.req_timeout = secs_to_jiffies(timeout);
+	INIT_DELAYED_WORK(&fch->timeout.work, fuse_check_timeout);
+	queue_delayed_work(system_percpu_wq, &fch->timeout.work,
+			   fuse_timeout_timer_freq);
+}
+
+void fuse_init_server_timeout(struct fuse_chan *fch, unsigned int timeout)
+{
+	if (!timeout && !fuse_max_req_timeout && !fuse_default_req_timeout)
+		return;
+
+	if (!timeout)
+		timeout = fuse_default_req_timeout;
+
+	if (fuse_max_req_timeout) {
+		if (timeout)
+			timeout = min(fuse_max_req_timeout, timeout);
+		else
+			timeout = fuse_max_req_timeout;
+	}
+
+	timeout = max(FUSE_TIMEOUT_TIMER_FREQ, timeout);
+
+	set_request_timeout(fch, timeout);
+}
+
diff --git a/fs/fuse/sysctl.c b/fs/fuse/sysctl.c
index e2d921abcb883..74eca5ce9a2ce 100644
--- a/fs/fuse/sysctl.c
+++ b/fs/fuse/sysctl.c
@@ -6,6 +6,7 @@
  */
 #include <linux/sysctl.h>
 
+#include "sysctl.h"
 #include "fuse_i.h"
 
 static struct ctl_table_header *fuse_table_header;
diff --git a/fs/fuse/sysctl.h b/fs/fuse/sysctl.h
new file mode 100644
index 0000000000000..948d884171331
--- /dev/null
+++ b/fs/fuse/sysctl.h
@@ -0,0 +1,9 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _FS_FUSE_SYSCTL_H
+#define _FS_FUSE_SYSCTL_H
+
+extern unsigned int fuse_default_req_timeout;
+extern unsigned int fuse_max_req_timeout;
+
+#endif /* _FS_FUSE_SYSCTL_H */
diff --git a/fs/fuse/virtio_fs.c b/fs/fuse/virtio_fs.c
index 12300651a0f1a..a4cf813cebfcc 100644
--- a/fs/fuse/virtio_fs.c
+++ b/fs/fuse/virtio_fs.c
@@ -19,6 +19,7 @@
 #include <linux/highmem.h>
 #include <linux/cleanup.h>
 #include <linux/uio.h>
+#include "dev.h"
 #include "fuse_i.h"
 #include "fuse_dev_i.h"
 
@@ -1519,7 +1520,7 @@ static void virtio_fs_send_req(struct fuse_iqueue *fiq, struct fuse_req *req)
 		if (ret == -ENOSPC) {
 			/*
 			 * Virtqueue full. Retry submission from worker
-			 * context as we might be holding fc->bg_lock.
+			 * context as we might be holding fc->chan->bg_lock.
 			 */
 			spin_lock(&fsvq->lock);
 			list_add_tail(&req->list, &fsvq->queued_reqs);
@@ -1562,7 +1563,7 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc)
 {
 	struct fuse_mount *fm = get_fuse_mount_super(sb);
 	struct fuse_conn *fc = fm->fc;
-	struct virtio_fs *fs = fc->iq.priv;
+	struct virtio_fs *fs = fc->chan->iq.priv;
 	struct fuse_fs_context *ctx = fsc->fs_private;
 	unsigned int i;
 	int err;
@@ -1606,7 +1607,7 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc)
 	for (i = 0; i < fs->nvqs; i++) {
 		struct virtio_fs_vq *fsvq = &fs->vqs[i];
 
-		fuse_dev_install(fsvq->fud, fc);
+		fuse_dev_install(fsvq->fud, fc->chan);
 	}
 
 	/* Previous unmount will stop all queues. Start these again */
@@ -1625,7 +1626,7 @@ static int virtio_fs_fill_super(struct super_block *sb, struct fs_context *fsc)
 static void virtio_fs_conn_destroy(struct fuse_mount *fm)
 {
 	struct fuse_conn *fc = fm->fc;
-	struct virtio_fs *vfs = fc->iq.priv;
+	struct virtio_fs *vfs = fc->chan->iq.priv;
 	struct virtio_fs_vq *fsvq = &vfs->vqs[VQ_HIPRIO];
 
 	/* Stop dax worker. Soon evict_inodes() will be called which
@@ -1673,7 +1674,7 @@ static int virtio_fs_test_super(struct super_block *sb,
 	struct fuse_mount *fsc_fm = fsc->s_fs_info;
 	struct fuse_mount *sb_fm = get_fuse_mount_super(sb);
 
-	return fsc_fm->fc->iq.priv == sb_fm->fc->iq.priv;
+	return fsc_fm->fc->chan->iq.priv == sb_fm->fc->chan->iq.priv;
 }
 
 static int virtio_fs_get_tree(struct fs_context *fsc)
@@ -1683,13 +1684,17 @@ static int virtio_fs_get_tree(struct fs_context *fsc)
 	struct fuse_conn *fc = NULL;
 	struct fuse_mount *fm;
 	unsigned int virtqueue_size;
+	struct fuse_chan *fch __free(fuse_chan_free) = fuse_chan_new();
 	int err = -EIO;
 
+	if (!fch)
+		return -ENOMEM;
+
 	if (!fsc->source)
 		return invalf(fsc, "No source specified");
 
 	/* This gets a reference on virtio_fs object. This ptr gets installed
-	 * in fc->iq->priv. Once fuse_conn is going away, it calls ->put()
+	 * in chan->iq->priv. Once fuse_conn is going away, it calls ->put()
 	 * to drop the reference to this object.
 	 */
 	fs = virtio_fs_find_instance(fsc->source);
@@ -1711,7 +1716,9 @@ static int virtio_fs_get_tree(struct fs_context *fsc)
 	if (!fm)
 		goto out_err;
 
-	fuse_conn_init(fc, fm, fsc->user_ns, &virtio_fs_fiq_ops, fs);
+	fuse_iqueue_init(&fch->iq, &virtio_fs_fiq_ops, fs);
+	fuse_conn_init(fc, fm, fsc->user_ns, no_free_ptr(fch));
+
 	fc->release = fuse_free_conn;
 	fc->delete_stale = true;
 	fc->auto_submounts = true;
diff --git a/fs/gfs2/inode.c b/fs/gfs2/inode.c
index e9bf4879c07f7..4ccbd95fc31ac 100644
--- a/fs/gfs2/inode.c
+++ b/fs/gfs2/inode.c
@@ -738,6 +738,12 @@ static int gfs2_create_inode(struct inode *dir, struct dentry *dentry,
 	inode = gfs2_dir_search(dir, &dentry->d_name, !S_ISREG(mode) || excl);
 	error = PTR_ERR(inode);
 	if (!IS_ERR(inode)) {
+		if (file && (file->f_flags & OPENAT2_REGULAR) && !S_ISREG(inode->i_mode)) {
+			iput(inode);
+			inode = NULL;
+			error = -EFTYPE;
+			goto fail_gunlock;
+		}
 		if (S_ISDIR(inode->i_mode)) {
 			iput(inode);
 			inode = NULL;
diff --git a/fs/gfs2/quota.c b/fs/gfs2/quota.c
index 5290865f27f14..91e9975d25e89 100644
--- a/fs/gfs2/quota.c
+++ b/fs/gfs2/quota.c
@@ -254,9 +254,13 @@ static struct gfs2_quota_data *qd_alloc(unsigned hash, struct gfs2_sbd *sdp, str
 	return NULL;
 }
 
-static struct gfs2_quota_data *gfs2_qd_search_bucket(unsigned int hash,
-						     const struct gfs2_sbd *sdp,
-						     struct kqid qid)
+/*
+ * Lookup variant for callers which already hold qd_lock + bucket lock.
+ */
+static struct gfs2_quota_data *
+gfs2_qd_search_bucket_noref(unsigned int hash,
+			    const struct gfs2_sbd *sdp,
+			    struct kqid qid)
 {
 	struct gfs2_quota_data *qd;
 	struct hlist_bl_node *h;
@@ -264,12 +268,22 @@ static struct gfs2_quota_data *gfs2_qd_search_bucket(unsigned int hash,
 	hlist_bl_for_each_entry_rcu(qd, h, &qd_hash_table[hash], qd_hlist) {
 		if (!qid_eq(qd->qd_id, qid))
 			continue;
-		if (qd->qd_sbd != sdp)
-			continue;
-		if (lockref_get_not_dead(&qd->qd_lockref)) {
-			list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru);
+		if (qd->qd_sbd == sdp)
 			return qd;
-		}
+	}
+
+	return NULL;
+}
+
+static struct gfs2_quota_data *
+gfs2_qd_search_bucket(unsigned int hash, const struct gfs2_sbd *sdp, struct kqid qid)
+{
+	struct gfs2_quota_data *qd;
+
+	qd = gfs2_qd_search_bucket_noref(hash, sdp, qid);
+	if (qd && lockref_get_not_dead(&qd->qd_lockref)) {
+		list_lru_del_obj(&gfs2_qd_lru, &qd->qd_lru);
+		return qd;
 	}
 
 	return NULL;
@@ -1433,7 +1447,7 @@ int gfs2_quota_init(struct gfs2_sbd *sdp)
 
 		qc = (struct gfs2_quota_change *)(bh->b_data + sizeof(struct gfs2_meta_header));
 		for (y = 0; y < sdp->sd_qc_per_block && slot < sdp->sd_quota_slots;
-		     y++, slot++) {
+		     y++, slot++, qc++) {
 			struct gfs2_quota_data *old_qd, *qd;
 			s64 qc_change = be64_to_cpu(qc->qc_change);
 			u32 qc_flags = be32_to_cpu(qc->qc_flags);
@@ -1441,7 +1455,6 @@ int gfs2_quota_init(struct gfs2_sbd *sdp)
 						USRQUOTA : GRPQUOTA;
 			struct kqid qc_id = make_kqid(&init_user_ns, qtype,
 						      be32_to_cpu(qc->qc_id));
-			qc++;
 			if (!qc_change)
 				continue;
 
@@ -1458,7 +1471,7 @@ int gfs2_quota_init(struct gfs2_sbd *sdp)
 
 			spin_lock(&qd_lock);
 			spin_lock_bucket(hash);
-			old_qd = gfs2_qd_search_bucket(hash, sdp, qc_id);
+			old_qd = gfs2_qd_search_bucket_noref(hash, sdp, qc_id);
 			if (old_qd) {
 				fs_err(sdp, "Corruption found in quota_change%u"
 					    "file: duplicate identifier in "
@@ -1467,7 +1480,6 @@ int gfs2_quota_init(struct gfs2_sbd *sdp)
 
 				spin_unlock_bucket(hash);
 				spin_unlock(&qd_lock);
-				qd_put(old_qd);
 
 				gfs2_glock_put(qd->qd_gl);
 				kmem_cache_free(gfs2_quotad_cachep, qd);
diff --git a/fs/gfs2/super.c b/fs/gfs2/super.c
index a2ea121331f18..4d854556b5299 100644
--- a/fs/gfs2/super.c
+++ b/fs/gfs2/super.c
@@ -643,6 +643,7 @@ static void gfs2_put_super(struct super_block *sb)
 	gfs2_delete_debugfs_file(sdp);
 
 	gfs2_sys_fs_del(sdp);
+	rcu_barrier();
 	free_sbd(sdp);
 }
 
diff --git a/fs/hfs/dir.c b/fs/hfs/dir.c
index f5e7efe924e78..c4c6e1623f55d 100644
--- a/fs/hfs/dir.c
+++ b/fs/hfs/dir.c
@@ -328,4 +328,5 @@ const struct inode_operations hfs_dir_inode_operations = {
 	.rmdir		= hfs_remove,
 	.rename		= hfs_rename,
 	.setattr	= hfs_inode_setattr,
+	.fileattr_get	= hfs_fileattr_get,
 };
diff --git a/fs/hfs/hfs_fs.h b/fs/hfs/hfs_fs.h
index ac0e83f77a0f1..1b23448c9a48b 100644
--- a/fs/hfs/hfs_fs.h
+++ b/fs/hfs/hfs_fs.h
@@ -177,6 +177,8 @@ extern int hfs_get_block(struct inode *inode, sector_t block,
 extern const struct address_space_operations hfs_aops;
 extern const struct address_space_operations hfs_btree_aops;
 
+struct file_kattr;
+int hfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
 int hfs_write_begin(const struct kiocb *iocb, struct address_space *mapping,
 		    loff_t pos, unsigned int len, struct folio **foliop,
 		    void **fsdata);
diff --git a/fs/hfs/inode.c b/fs/hfs/inode.c
index 89b33a9d46d5c..f41cc261684d9 100644
--- a/fs/hfs/inode.c
+++ b/fs/hfs/inode.c
@@ -18,6 +18,7 @@
 #include <linux/uio.h>
 #include <linux/xattr.h>
 #include <linux/blkdev.h>
+#include <linux/fileattr.h>
 
 #include "hfs_fs.h"
 #include "btree.h"
@@ -699,6 +700,18 @@ static int hfs_file_fsync(struct file *filp, loff_t start, loff_t end,
 	return ret;
 }
 
+int hfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
+{
+	/*
+	 * HFS compares filenames using Mac OS Roman case folding, so
+	 * lookup is always case-insensitive. Names are stored on disk
+	 * with case intact; CASENONPRESERVING stays clear.
+	 */
+	fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
+	fa->flags |= FS_CASEFOLD_FL;
+	return 0;
+}
+
 static const struct file_operations hfs_file_operations = {
 	.llseek		= generic_file_llseek,
 	.read_iter	= generic_file_read_iter,
@@ -715,4 +728,5 @@ static const struct inode_operations hfs_file_inode_operations = {
 	.lookup		= hfs_file_lookup,
 	.setattr	= hfs_inode_setattr,
 	.listxattr	= generic_listxattr,
+	.fileattr_get	= hfs_fileattr_get,
 };
diff --git a/fs/hfsplus/inode.c b/fs/hfsplus/inode.c
index d05891ec492e3..5565c14b4bf69 100644
--- a/fs/hfsplus/inode.c
+++ b/fs/hfsplus/inode.c
@@ -740,6 +740,7 @@ int hfsplus_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 {
 	struct inode *inode = d_inode(dentry);
 	struct hfsplus_inode_info *hip = HFSPLUS_I(inode);
+	struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb);
 	unsigned int flags = 0;
 
 	if (inode->i_flags & S_IMMUTABLE)
@@ -748,6 +749,8 @@ int hfsplus_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
 		flags |= FS_APPEND_FL;
 	if (hip->userflags & HFSPLUS_FLG_NODUMP)
 		flags |= FS_NODUMP_FL;
+	if (test_bit(HFSPLUS_SB_CASEFOLD, &sbi->flags))
+		flags |= FS_CASEFOLD_FL;
 
 	fileattr_fill_flags(fa, flags);
 
@@ -759,13 +762,24 @@ int hfsplus_fileattr_set(struct mnt_idmap *idmap,
 {
 	struct inode *inode = d_inode(dentry);
 	struct hfsplus_inode_info *hip = HFSPLUS_I(inode);
+	struct hfsplus_sb_info *sbi = HFSPLUS_SB(inode->i_sb);
+	unsigned int allowed = FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL;
 	unsigned int new_fl = 0;
 
 	if (fileattr_has_fsx(fa))
 		return -EOPNOTSUPP;
 
+	/*
+	 * FS_CASEFOLD_FL reflects HFSPLUS_SB_CASEFOLD, a mount-time
+	 * property. Accept it as a no-op so chattr's RMW round-trip
+	 * succeeds; reject any attempt to enable it on a volume that
+	 * was not formatted case-insensitive.
+	 */
+	if (test_bit(HFSPLUS_SB_CASEFOLD, &sbi->flags))
+		allowed |= FS_CASEFOLD_FL;
+
 	/* don't silently ignore unsupported ext2 flags */
-	if (fa->flags & ~(FS_IMMUTABLE_FL|FS_APPEND_FL|FS_NODUMP_FL))
+	if (fa->flags & ~allowed)
 		return -EOPNOTSUPP;
 
 	if (fa->flags & FS_IMMUTABLE_FL)
diff --git a/fs/hpfs/inode.c b/fs/hpfs/inode.c
index 0e932cc8be1b2..1b4fcf760aadc 100644
--- a/fs/hpfs/inode.c
+++ b/fs/hpfs/inode.c
@@ -184,7 +184,7 @@ void hpfs_write_inode(struct inode *i)
 	struct hpfs_inode_info *hpfs_inode = hpfs_i(i);
 	struct inode *parent;
 	if (i->i_ino == hpfs_sb(i->i_sb)->sb_root) return;
-	if (hpfs_inode->i_rddir_off && !icount_read(i)) {
+	if (hpfs_inode->i_rddir_off && !icount_read_once(i)) {
 		if (*hpfs_inode->i_rddir_off)
 			pr_err("write_inode: some position still there\n");
 		kfree(hpfs_inode->i_rddir_off);
diff --git a/fs/hpfs/super.c b/fs/hpfs/super.c
index c16d5d4caeade..8fbdbf0806276 100644
--- a/fs/hpfs/super.c
+++ b/fs/hpfs/super.c
@@ -523,7 +523,8 @@ static int hpfs_fill_super(struct super_block *s, struct fs_context *fc)
 	hpfs_lock(s);
 
 	/*sbi->sb_mounting = 1;*/
-	sb_set_blocksize(s, 512);
+	if (!sb_set_blocksize(s, 512))
+		goto bail0;
 	sbi->sb_fs_size = -1;
 	if (!(bootblock = hpfs_map_sector(s, 0, &bh0, 0))) goto bail1;
 	if (!(superblock = hpfs_map_sector(s, 16, &bh1, 1))) goto bail2;
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 8b05bec08e049..78d61bf2bd9bb 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -96,15 +96,8 @@ static const struct fs_parameter_spec hugetlb_fs_parameters[] = {
 #define PGOFF_LOFFT_MAX \
 	(((1UL << (PAGE_SHIFT + 1)) - 1) <<  (BITS_PER_LONG - (PAGE_SHIFT + 1)))
 
-static int hugetlb_file_mmap_prepare_success(const struct vm_area_struct *vma)
+static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
 {
-	/* Unfortunate we have to reassign vma->vm_private_data. */
-	return hugetlb_vma_lock_alloc((struct vm_area_struct *)vma);
-}
-
-static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
-{
-	struct file *file = desc->file;
 	struct inode *inode = file_inode(file);
 	loff_t len, vma_len;
 	int ret;
@@ -119,8 +112,8 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
 	 * way when do_mmap unwinds (may be important on powerpc
 	 * and ia64).
 	 */
-	vma_desc_set_flags(desc, VMA_HUGETLB_BIT, VMA_DONTEXPAND_BIT);
-	desc->vm_ops = &hugetlb_vm_ops;
+	vma_set_flags(vma, VMA_HUGETLB_BIT, VMA_DONTEXPAND_BIT);
+	vma->vm_ops = &hugetlb_vm_ops;
 
 	/*
 	 * page based offset in vm_pgoff could be sufficiently large to
@@ -129,16 +122,16 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
 	 * sizeof(unsigned long).  So, only check in those instances.
 	 */
 	if (sizeof(unsigned long) == sizeof(loff_t)) {
-		if (desc->pgoff & PGOFF_LOFFT_MAX)
+		if (vma->vm_pgoff & PGOFF_LOFFT_MAX)
 			return -EINVAL;
 	}
 
 	/* must be huge page aligned */
-	if (desc->pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
+	if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
 		return -EINVAL;
 
-	vma_len = (loff_t)vma_desc_size(desc);
-	len = vma_len + ((loff_t)desc->pgoff << PAGE_SHIFT);
+	vma_len = (loff_t)(vma->vm_end - vma->vm_start);
+	len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
 	/* check for overflow */
 	if (len < vma_len)
 		return -EINVAL;
@@ -148,7 +141,7 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
 
 	ret = -ENOMEM;
 
-	vma_flags = desc->vma_flags;
+	vma_flags = vma->flags;
 	/*
 	 * for SHM_HUGETLB, the pages are reserved in the shmget() call so skip
 	 * reserving here. Note: only for SHM hugetlbfs file, the inode
@@ -158,30 +151,17 @@ static int hugetlbfs_file_mmap_prepare(struct vm_area_desc *desc)
 		vma_flags_set(&vma_flags, VMA_NORESERVE_BIT);
 
 	if (hugetlb_reserve_pages(inode,
-			desc->pgoff >> huge_page_order(h),
-			len >> huge_page_shift(h), desc,
-			vma_flags) < 0)
+				vma->vm_pgoff >> huge_page_order(h),
+				len >> huge_page_shift(h), vma,
+				vma_flags) < 0)
 		goto out;
 
 	ret = 0;
-	if (vma_desc_test(desc, VMA_WRITE_BIT) && inode->i_size < len)
+	if (vma_test(vma, VMA_WRITE_BIT) && inode->i_size < len)
 		i_size_write(inode, len);
 out:
 	inode_unlock(inode);
 
-	if (!ret) {
-		/* Allocate the VMA lock after we set it up. */
-		desc->action.success_hook = hugetlb_file_mmap_prepare_success;
-		/*
-		 * We cannot permit the rmap finding this VMA in the time
-		 * between the VMA being inserted into the VMA tree and the
-		 * completion/success hook being invoked.
-		 *
-		 * This is because we establish a per-VMA hugetlb lock which can
-		 * be raced by rmap.
-		 */
-		desc->action.hide_from_rmap_until_complete = true;
-	}
 	return ret;
 }
 
@@ -1227,7 +1207,7 @@ static void init_once(void *foo)
 
 static const struct file_operations hugetlbfs_file_operations = {
 	.read_iter		= hugetlbfs_read_iter,
-	.mmap_prepare		= hugetlbfs_file_mmap_prepare,
+	.mmap			= hugetlbfs_file_mmap,
 	.fsync			= noop_fsync,
 	.get_unmapped_area	= hugetlb_get_unmapped_area,
 	.llseek			= default_llseek,
diff --git a/fs/inode.c b/fs/inode.c
index 6a3cbc7dcd28c..31c5b9ee3a81d 100644
--- a/fs/inode.c
+++ b/fs/inode.c
@@ -53,11 +53,7 @@
  *   inode->i_lock
  *
  * inode_hash_lock
- *   inode->i_sb->s_inode_list_lock
  *   inode->i_lock
- *
- * iunique_lock
- *   inode_hash_lock
  */
 
 static unsigned int i_hash_mask __ro_after_init;
@@ -279,9 +275,6 @@ int inode_init_always_gfp(struct super_block *sb, struct inode *inode, gfp_t gfp
 	mapping->flags = 0;
 	mapping->wb_err = 0;
 	atomic_set(&mapping->i_mmap_writable, 0);
-#ifdef CONFIG_READ_ONLY_THP_FOR_FS
-	atomic_set(&mapping->nr_thps, 0);
-#endif
 	mapping_set_gfp_mask(mapping, GFP_HIGHUSER_MOVABLE);
 	mapping->writeback_index = 0;
 	init_rwsem(&mapping->invalidate_lock);
@@ -518,15 +511,6 @@ static void init_once(void *foo)
 	inode_init_once(inode);
 }
 
-/*
- * get additional reference to inode; caller must already hold one.
- */
-void ihold(struct inode *inode)
-{
-	WARN_ON(atomic_inc_return(&inode->i_count) < 2);
-}
-EXPORT_SYMBOL(ihold);
-
 struct wait_queue_head *inode_bit_waitqueue(struct wait_bit_queue_entry *wqe,
 					    struct inode *inode, u32 bit)
 {
@@ -902,7 +886,7 @@ void evict_inodes(struct super_block *sb)
 again:
 	spin_lock(&sb->s_inode_list_lock);
 	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
-		if (icount_read(inode))
+		if (icount_read_once(inode))
 			continue;
 
 		spin_lock(&inode->i_lock);
@@ -1032,6 +1016,7 @@ long prune_icache_sb(struct super_block *sb, struct shrink_control *sc)
 }
 
 static void __wait_on_freeing_inode(struct inode *inode, bool hash_locked, bool rcu_locked);
+static bool igrab_from_hash(struct inode *inode);
 
 /*
  * Called with the inode lock held.
@@ -1056,6 +1041,11 @@ static struct inode *find_inode(struct super_block *sb,
 			continue;
 		if (!test(inode, data))
 			continue;
+		if (igrab_from_hash(inode)) {
+			rcu_read_unlock();
+			*isnew = false;
+			return inode;
+		}
 		spin_lock(&inode->i_lock);
 		if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) {
 			__wait_on_freeing_inode(inode, hash_locked, true);
@@ -1098,6 +1088,11 @@ static struct inode *find_inode_fast(struct super_block *sb,
 			continue;
 		if (inode->i_sb != sb)
 			continue;
+		if (igrab_from_hash(inode)) {
+			rcu_read_unlock();
+			*isnew = false;
+			return inode;
+		}
 		spin_lock(&inode->i_lock);
 		if (inode_state_read(inode) & (I_FREEING | I_WILL_FREE)) {
 			__wait_on_freeing_inode(inode, hash_locked, true);
@@ -1215,6 +1210,10 @@ void unlock_new_inode(struct inode *inode)
 	lockdep_annotate_inode_mutex_key(inode);
 	spin_lock(&inode->i_lock);
 	WARN_ON(!(inode_state_read(inode) & I_NEW));
+	/*
+	 * Paired with igrab_from_hash()
+	 */
+	smp_wmb();
 	inode_state_clear(inode, I_NEW | I_CREATING);
 	inode_wake_up_bit(inode, __I_NEW);
 	spin_unlock(&inode->i_lock);
@@ -1226,6 +1225,10 @@ void discard_new_inode(struct inode *inode)
 	lockdep_annotate_inode_mutex_key(inode);
 	spin_lock(&inode->i_lock);
 	WARN_ON(!(inode_state_read(inode) & I_NEW));
+	/*
+	 * Paired with igrab_from_hash()
+	 */
+	smp_wmb();
 	inode_state_clear(inode, I_NEW);
 	inode_wake_up_bit(inode, __I_NEW);
 	spin_unlock(&inode->i_lock);
@@ -1572,8 +1575,27 @@ ino_t iunique(struct super_block *sb, ino_t max_reserved)
 }
 EXPORT_SYMBOL(iunique);
 
+/**
+ * ihold - get a reference on the inode, provided you already have one
+ * @inode:	inode to operate on
+ */
+void ihold(struct inode *inode)
+{
+	VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode);
+	WARN_ON(atomic_inc_return(&inode->i_count) < 2);
+}
+EXPORT_SYMBOL(ihold);
+
 struct inode *igrab(struct inode *inode)
 {
+	/*
+	 * Read commentary above igrab_from_hash() for an explanation why this works.
+	 */
+	if (atomic_add_unless(&inode->i_count, 1, 0)) {
+		VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode);
+		return inode;
+	}
+
 	spin_lock(&inode->i_lock);
 	if (!(inode_state_read(inode) & (I_FREEING | I_WILL_FREE))) {
 		__iget(inode);
@@ -1591,6 +1613,43 @@ struct inode *igrab(struct inode *inode)
 }
 EXPORT_SYMBOL(igrab);
 
+/*
+ * igrab_from_hash - special inode refcount acquire primitive for the inode hash
+ *
+ * It provides lockless refcount acquire in the common case of no problematic
+ * flags being set and the count being > 0.
+ *
+ * There are 4 state flags to worry about and the routine makes sure to not bump the
+ * ref if any of them is present.
+ *
+ * I_NEW and I_CREATING can only legally get set *before* the inode becomes visible
+ * during lookup. Thus if the flags are not spotted, they are guaranteed to not be
+ * a factor. However, we need an acquire fence before returning the inode just
+ * in case we raced against clearing the state to make sure our consumer picks up
+ * any other changes made prior. atomic_add_unless provides a full fence, which
+ * takes care of it.
+ *
+ * I_FREEING and I_WILL_FREE can only legally get set if ->i_count == 0 and it is
+ * illegal to bump the ref if either is present. Consequently if atomic_add_unless
+ * managed to replace a non-0 value with a bigger one, we have a guarantee neither
+ * of these flags is set. Note this means explicitly checking of these flags below
+ * is not necessary, it is only done because it does not cost anything on top of the
+ * load which already needs to be done to handle the other flags.
+ */
+static bool igrab_from_hash(struct inode *inode)
+{
+	if (inode_state_read_once(inode) & (I_NEW | I_CREATING | I_FREEING | I_WILL_FREE))
+		return false;
+	/*
+	 * Paired with routines clearing I_NEW
+	 */
+	if (atomic_add_unless(&inode->i_count, 1, 0)) {
+		VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_WILL_FREE), inode);
+		return true;
+	}
+	return false;
+}
+
 /**
  * ilookup5_nowait - search for an inode in the inode cache
  * @sb:		super block of file system to search
@@ -1920,7 +1979,7 @@ static void iput_final(struct inode *inode)
 	int drop;
 
 	WARN_ON(inode_state_read(inode) & I_NEW);
-	VFS_BUG_ON_INODE(atomic_read(&inode->i_count) != 0, inode);
+	VFS_BUG_ON_INODE(icount_read(inode) != 0, inode);
 
 	if (op->drop_inode)
 		drop = op->drop_inode(inode);
@@ -1939,7 +1998,7 @@ static void iput_final(struct inode *inode)
 	 * Re-check ->i_count in case the ->drop_inode() hooks played games.
 	 * Note we only execute this if the verdict was to drop the inode.
 	 */
-	VFS_BUG_ON_INODE(atomic_read(&inode->i_count) != 0, inode);
+	VFS_BUG_ON_INODE(icount_read(inode) != 0, inode);
 
 	if (drop) {
 		inode_state_set(inode, I_FREEING);
@@ -1983,7 +2042,7 @@ void iput(struct inode *inode)
 	 * equal to one, then two CPUs racing to further drop it can both
 	 * conclude it's fine.
 	 */
-	VFS_BUG_ON_INODE(atomic_read(&inode->i_count) < 1, inode);
+	VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode);
 
 	if (atomic_add_unless(&inode->i_count, -1, 1))
 		return;
@@ -2017,7 +2076,7 @@ EXPORT_SYMBOL(iput);
 void iput_not_last(struct inode *inode)
 {
 	VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode);
-	VFS_BUG_ON_INODE(atomic_read(&inode->i_count) < 2, inode);
+	VFS_BUG_ON_INODE(icount_read_once(inode) < 2, inode);
 
 	WARN_ON(atomic_sub_return(1, &inode->i_count) == 0);
 }
@@ -2124,7 +2183,13 @@ static int inode_update_cmtime(struct inode *inode, unsigned int flags)
 			    inode_iversion_need_inc(inode))
 				return -EAGAIN;
 		} else {
-			if (inode_maybe_inc_iversion(inode, !!dirty))
+			/*
+			 * Don't force iversion increment for pure lazytime
+			 * updates (I_DIRTY_TIME only), let I_VERSION_QUERIED
+			 * dictate whether the increment is needed.
+			 */
+			if (inode_maybe_inc_iversion(inode,
+						     dirty != I_DIRTY_TIME))
 				dirty |= I_DIRTY_SYNC;
 		}
 	}
@@ -3040,7 +3105,7 @@ void dump_inode(struct inode *inode, const char *reason)
 	}
 
 	state = inode_state_read_once(inode);
-	count = atomic_read(&inode->i_count);
+	count = icount_read_once(inode);
 
 	if (!sb ||
 	    get_kernel_nofault(s_type, &sb->s_type) || !s_type ||
diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c
index d7b648421a70f..c98cf19fbe08a 100644
--- a/fs/iomap/buffered-io.c
+++ b/fs/iomap/buffered-io.c
@@ -1058,7 +1058,6 @@ static bool iomap_write_end_inline(const struct iomap_iter *iter,
 	void *addr;
 
 	WARN_ON_ONCE(!folio_test_uptodate(folio));
-	BUG_ON(!iomap_inline_data_valid(iomap));
 
 	if (WARN_ON_ONCE(!iomap->inline_data))
 		return false;
@@ -1543,6 +1542,8 @@ static int iomap_zero_iter(struct iomap_iter *iter, bool *did_zero,
 		size_t offset;
 		bool ret;
 
+		balance_dirty_pages_ratelimited(iter->inode->i_mapping);
+
 		bytes = min_t(u64, SIZE_MAX, bytes);
 		status = iomap_write_begin(iter, write_ops, &folio, &offset,
 				&bytes);
diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c
index b36ee619cdcdd..c1067de5ea451 100644
--- a/fs/iomap/direct-io.c
+++ b/fs/iomap/direct-io.c
@@ -603,9 +603,6 @@ static int iomap_dio_inline_iter(struct iomap_iter *iomi, struct iomap_dio *dio)
 	if (WARN_ON_ONCE(!inline_data))
 		return -EIO;
 
-	if (WARN_ON_ONCE(!iomap_inline_data_valid(iomap)))
-		return -EIO;
-
 	if (dio->flags & IOMAP_DIO_WRITE) {
 		loff_t size = iomi->inode->i_size;
 
diff --git a/fs/iomap/iter.c b/fs/iomap/iter.c
index c04796f6e57fa..e4a29829591a7 100644
--- a/fs/iomap/iter.c
+++ b/fs/iomap/iter.c
@@ -6,17 +6,13 @@
 #include <linux/iomap.h>
 #include "trace.h"
 
-static inline void iomap_iter_reset_iomap(struct iomap_iter *iter)
+static inline void iomap_iter_clean_fbatch(struct iomap_iter *iter)
 {
 	if (iter->iomap.flags & IOMAP_F_FOLIO_BATCH) {
 		folio_batch_release(iter->fbatch);
 		folio_batch_reinit(iter->fbatch);
 		iter->iomap.flags &= ~IOMAP_F_FOLIO_BATCH;
 	}
-
-	iter->status = 0;
-	memset(&iter->iomap, 0, sizeof(iter->iomap));
-	memset(&iter->srcmap, 0, sizeof(iter->srcmap));
 }
 
 /* Advance the current iterator position and decrement the remaining length */
@@ -102,10 +98,14 @@ int iomap_iter(struct iomap_iter *iter, const struct iomap_ops *ops)
 		ret = 0;
 	else
 		ret = 1;
-	iomap_iter_reset_iomap(iter);
+	iomap_iter_clean_fbatch(iter);
+	iter->status = 0;
 	if (ret <= 0)
 		return ret;
 
+	memset(&iter->iomap, 0, sizeof(iter->iomap));
+	memset(&iter->srcmap, 0, sizeof(iter->srcmap));
+
 begin:
 	ret = ops->iomap_begin(iter->inode, iter->pos, iter->len, iter->flags,
 			       &iter->iomap, &iter->srcmap);
diff --git a/fs/isofs/dir.c b/fs/isofs/dir.c
index 2fd9948d606e9..55385a72a4ce0 100644
--- a/fs/isofs/dir.c
+++ b/fs/isofs/dir.c
@@ -14,6 +14,7 @@
 #include <linux/gfp.h>
 #include <linux/filelock.h>
 #include "isofs.h"
+#include <linux/fileattr.h>
 
 int isofs_name_translate(struct iso_directory_record *de, char *new, struct inode *inode)
 {
@@ -267,6 +268,20 @@ static int isofs_readdir(struct file *file, struct dir_context *ctx)
 	return result;
 }
 
+int isofs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
+{
+	struct isofs_sb_info *sbi = ISOFS_SB(dentry->d_sb);
+
+	if (sbi->s_check == 'r') {
+		fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
+		fa->flags |= FS_CASEFOLD_FL;
+	}
+	if (!sbi->s_joliet_level && !sbi->s_rock &&
+	    (sbi->s_mapping == 'n' || sbi->s_mapping == 'a'))
+		fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING;
+	return 0;
+}
+
 const struct file_operations isofs_dir_operations =
 {
 	.llseek = generic_file_llseek,
@@ -281,6 +296,7 @@ const struct file_operations isofs_dir_operations =
 const struct inode_operations isofs_dir_inode_operations =
 {
 	.lookup = isofs_lookup,
+	.fileattr_get = isofs_fileattr_get,
 };
 
 
diff --git a/fs/isofs/inode.c b/fs/isofs/inode.c
index efee53717f1cd..337836a0a1704 100644
--- a/fs/isofs/inode.c
+++ b/fs/isofs/inode.c
@@ -818,7 +818,8 @@ static int isofs_fill_super(struct super_block *s, struct fs_context *fc)
 	 * entries.  By forcing the blocksize in this way, we ensure
 	 * that we will never be required to do this.
 	 */
-	sb_set_blocksize(s, orig_zonesize);
+	if (!sb_set_blocksize(s, orig_zonesize))
+		goto out_freesbi;
 
 	sbi->s_nls_iocharset = NULL;
 
diff --git a/fs/isofs/isofs.h b/fs/isofs/isofs.h
index 5065558375333..0ec8b24a42edc 100644
--- a/fs/isofs/isofs.h
+++ b/fs/isofs/isofs.h
@@ -197,6 +197,9 @@ isofs_normalize_block_and_offset(struct iso_directory_record* de,
 	}
 }
 
+struct file_kattr;
+int isofs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
+
 extern const struct inode_operations isofs_dir_inode_operations;
 extern const struct file_operations isofs_dir_operations;
 extern const struct address_space_operations isofs_symlink_aops;
diff --git a/fs/jfs/namei.c b/fs/jfs/namei.c
index 60c4a0e0fca5e..442d626792622 100644
--- a/fs/jfs/namei.c
+++ b/fs/jfs/namei.c
@@ -309,7 +309,7 @@ static struct dentry *jfs_mkdir(struct mnt_idmap *idmap, struct inode *dip,
       out1:
 
 	jfs_info("jfs_mkdir: rc:%d", rc);
-	return ERR_PTR(rc);
+	return rc ? ERR_PTR(rc) : NULL;
 }
 
 /*
diff --git a/fs/jfs/super.c b/fs/jfs/super.c
index 61575f7397aea..8180d83d33fed 100644
--- a/fs/jfs/super.c
+++ b/fs/jfs/super.c
@@ -491,7 +491,8 @@ static int jfs_fill_super(struct super_block *sb, struct fs_context *fc)
 	/*
 	 * Initialize blocksize to 4K.
 	 */
-	sb_set_blocksize(sb, PSIZE);
+	if (!sb_set_blocksize(sb, PSIZE))
+		goto out_unload;
 
 	/*
 	 * Set method vectors.
diff --git a/fs/minix/inode.c b/fs/minix/inode.c
index 9c6bac2489077..03a69b13950db 100644
--- a/fs/minix/inode.c
+++ b/fs/minix/inode.c
@@ -292,7 +292,8 @@ static int minix_fill_super(struct super_block *s, struct fs_context *fc)
 		sbi->s_namelen = 60;
 		sbi->s_version = MINIX_V3;
 		sbi->s_mount_state = MINIX_VALID_FS;
-		sb_set_blocksize(s, m3s->s_blocksize);
+		if (!sb_set_blocksize(s, m3s->s_blocksize))
+			goto out;
 		s->s_max_links = MINIX2_LINK_MAX;
 	} else
 		goto out_no_fs;
diff --git a/fs/mnt_idmapping.c b/fs/mnt_idmapping.c
index 6472c4ea3d1e6..cb61fbdb52e90 100644
--- a/fs/mnt_idmapping.c
+++ b/fs/mnt_idmapping.c
@@ -375,6 +375,8 @@ int statmount_mnt_idmap(struct mnt_idmap *idmap, struct seq_file *seq, bool uid_
 			continue;
 
 		seq_printf(seq, "%u %u %u", extent->first, lower, extent->count);
+		if (seq_has_overflowed(seq))
+			return -EAGAIN;
 
 		seq->count++; /* mappings are separated by \0 */
 		if (seq_has_overflowed(seq))
diff --git a/fs/mount.h b/fs/mount.h
index e0816c11a1989..5df134d56d475 100644
--- a/fs/mount.h
+++ b/fs/mount.h
@@ -25,6 +25,7 @@ struct mnt_namespace {
 	__u32			n_fsnotify_mask;
 	struct fsnotify_mark_connector __rcu *n_fsnotify_marks;
 #endif
+	struct hlist_head	mnt_visible_mounts; /* SB_I_USERNS_VISIBLE mounts */
 	unsigned int		nr_mounts; /* # of mounts in the namespace */
 	unsigned int		pending_mounts;
 	refcount_t		passive; /* number references not pinning @mounts */
@@ -90,6 +91,7 @@ struct mount {
 	int mnt_expiry_mark;		/* true if marked for expiry */
 	struct hlist_head mnt_pins;
 	struct hlist_head mnt_stuck_children;
+	struct hlist_node mnt_ns_visible; /* link in ns->mnt_visible_mounts */
 	struct mount *overmount;	/* mounted on ->mnt_root */
 } __randomize_layout;
 
@@ -207,6 +209,8 @@ static inline void move_from_ns(struct mount *mnt)
 		ns->mnt_first_node = rb_next(&mnt->mnt_node);
 	rb_erase(&mnt->mnt_node, &ns->mounts);
 	RB_CLEAR_NODE(&mnt->mnt_node);
+	if (!hlist_unhashed(&mnt->mnt_ns_visible))
+		hlist_del_init(&mnt->mnt_ns_visible);
 }
 
 bool has_locked_children(struct mount *mnt, struct dentry *dentry);
diff --git a/fs/namei.c b/fs/namei.c
index c7fac83c9a85e..a7882e8da48d9 100644
--- a/fs/namei.c
+++ b/fs/namei.c
@@ -2955,15 +2955,16 @@ void end_dirop(struct dentry *de)
 EXPORT_SYMBOL(end_dirop);
 
 /* does lookup, returns the object with parent locked */
-static struct dentry *__start_removing_path(int dfd, struct filename *name,
-					   struct path *path)
+struct dentry *start_removing_path(const char *name, struct path *path)
 {
+	CLASS(filename_kernel, filename)(name);
 	struct path parent_path __free(path_put) = {};
 	struct dentry *d;
 	struct qstr last;
 	int type, error;
 
-	error = filename_parentat(dfd, name, 0, &parent_path, &last, &type);
+	error = filename_parentat(AT_FDCWD, filename, 0, &parent_path, &last,
+			&type);
 	if (error)
 		return ERR_PTR(error);
 	if (unlikely(type != LAST_NORM))
@@ -3023,21 +3024,6 @@ struct dentry *kern_path_parent(const char *name, struct path *path)
 	return d;
 }
 
-struct dentry *start_removing_path(const char *name, struct path *path)
-{
-	CLASS(filename_kernel, filename)(name);
-	return __start_removing_path(AT_FDCWD, filename, path);
-}
-
-struct dentry *start_removing_user_path_at(int dfd,
-					   const char __user *name,
-					   struct path *path)
-{
-	CLASS(filename, filename)(name);
-	return __start_removing_path(dfd, filename, path);
-}
-EXPORT_SYMBOL(start_removing_user_path_at);
-
 int kern_path(const char *name, unsigned int flags, struct path *path)
 {
 	CLASS(filename_kernel, filename)(name);
@@ -3617,7 +3603,6 @@ int path_pts(struct path *path)
 	 */
 	struct dentry *parent = dget_parent(path->dentry);
 	struct dentry *child;
-	struct qstr this = QSTR_INIT("pts", 3);
 
 	if (unlikely(!path_connected(path->mnt, parent))) {
 		dput(parent);
@@ -3625,7 +3610,7 @@ int path_pts(struct path *path)
 	}
 	dput(path->dentry);
 	path->dentry = parent;
-	child = d_hash_and_lookup(parent, &this);
+	child = d_hash_and_lookup(parent, &QSTR("pts"));
 	if (IS_ERR_OR_NULL(child))
 		return -ENOENT;
 
@@ -4679,6 +4664,10 @@ static int do_open(struct nameidata *nd,
 		if (unlikely(error))
 			return error;
 	}
+
+	if ((open_flag & OPENAT2_REGULAR) && !d_is_reg(nd->path.dentry))
+		return -EFTYPE;
+
 	if ((nd->flags & LOOKUP_DIRECTORY) && !d_can_lookup(nd->path.dentry))
 		return -ENOTDIR;
 
diff --git a/fs/namespace.c b/fs/namespace.c
index fe919abd2f011..9a66a806a9b8a 100644
--- a/fs/namespace.c
+++ b/fs/namespace.c
@@ -321,6 +321,7 @@ static struct mount *alloc_vfsmnt(const char *name)
 		INIT_HLIST_NODE(&mnt->mnt_slave);
 		INIT_HLIST_NODE(&mnt->mnt_mp_list);
 		INIT_HLIST_HEAD(&mnt->mnt_stuck_children);
+		INIT_HLIST_NODE(&mnt->mnt_ns_visible);
 		RB_CLEAR_NODE(&mnt->mnt_node);
 		mnt->mnt.mnt_idmap = &nop_mnt_idmap;
 	}
@@ -1098,6 +1099,10 @@ static void mnt_add_to_ns(struct mnt_namespace *ns, struct mount *mnt)
 	rb_link_node(&mnt->mnt_node, parent, link);
 	rb_insert_color(&mnt->mnt_node, &ns->mounts);
 
+	if ((mnt->mnt.mnt_sb->s_type->fs_flags & FS_USERNS_MOUNT_RESTRICTED) &&
+	    mnt->mnt.mnt_root == mnt->mnt.mnt_sb->s_root)
+		hlist_add_head(&mnt->mnt_ns_visible, &ns->mnt_visible_mounts);
+
 	mnt_notify_add(mnt);
 }
 
@@ -6340,20 +6345,26 @@ static bool mnt_already_visible(struct mnt_namespace *ns,
 				int *new_mnt_flags)
 {
 	int new_flags = *new_mnt_flags;
-	struct mount *mnt, *n;
+	struct mount *mnt;
+
+	/* Don't acquire namespace semaphore without a good reason. */
+	if (hlist_empty(&ns->mnt_visible_mounts))
+		return false;
 
 	guard(namespace_shared)();
-	rbtree_postorder_for_each_entry_safe(mnt, n, &ns->mounts, mnt_node) {
+	hlist_for_each_entry(mnt, &ns->mnt_visible_mounts, mnt_ns_visible) {
+		const struct super_block *sb_visible = mnt->mnt.mnt_sb;
 		struct mount *child;
 		int mnt_flags;
 
-		if (mnt->mnt.mnt_sb->s_type != sb->s_type)
+		if (sb_visible->s_type != sb->s_type)
 			continue;
 
-		/* This mount is not fully visible if it's root directory
-		 * is not the root directory of the filesystem.
+		/*
+		 * Restricted variants are not compatible with anything, even
+		 * other restricted variants.
 		 */
-		if (mnt->mnt.mnt_root != mnt->mnt.mnt_sb->s_root)
+		if (sb_visible->s_iflags & SB_I_RESTRICTED_VARIANT)
 			continue;
 
 		/* A local view of the mount flags */
@@ -6405,16 +6416,23 @@ static bool mount_too_revealing(const struct super_block *sb, int *new_mnt_flags
 		return false;
 
 	/* Can this filesystem be too revealing? */
-	s_iflags = sb->s_iflags;
-	if (!(s_iflags & SB_I_USERNS_VISIBLE))
+	if (!(sb->s_type->fs_flags & FS_USERNS_MOUNT_RESTRICTED))
 		return false;
 
+	s_iflags = sb->s_iflags;
 	if ((s_iflags & required_iflags) != required_iflags) {
 		WARN_ONCE(1, "Expected s_iflags to contain 0x%lx\n",
 			  required_iflags);
 		return true;
 	}
 
+	/*
+	 * Restricted variants don't need an already visible mount because they
+	 * don't expose the full filesystem view.
+	 */
+	if (s_iflags & SB_I_RESTRICTED_VARIANT)
+		return false;
+
 	return !mnt_already_visible(ns, sb, new_mnt_flags);
 }
 
diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c
index a8c0d86118c58..76d0f6a29abab 100644
--- a/fs/netfs/buffered_read.c
+++ b/fs/netfs/buffered_read.c
@@ -156,9 +156,8 @@ static void netfs_read_cache_to_pagecache(struct netfs_io_request *rreq,
 			netfs_cache_read_terminated, subreq);
 }
 
-static void netfs_queue_read(struct netfs_io_request *rreq,
-			     struct netfs_io_subrequest *subreq,
-			     bool last_subreq)
+void netfs_queue_read(struct netfs_io_request *rreq,
+		      struct netfs_io_subrequest *subreq)
 {
 	struct netfs_io_stream *stream = &rreq->io_streams[0];
 
@@ -169,7 +168,8 @@ static void netfs_queue_read(struct netfs_io_request *rreq,
 	 * remove entries off of the front.
 	 */
 	spin_lock(&rreq->lock);
-	list_add_tail(&subreq->rreq_link, &stream->subrequests);
+	/* Write IN_PROGRESS before pointer to new subreq */
+	list_add_tail_release(&subreq->rreq_link, &stream->subrequests);
 	if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
 		if (!stream->active) {
 			stream->collected_to = subreq->start;
@@ -178,11 +178,6 @@ static void netfs_queue_read(struct netfs_io_request *rreq,
 		}
 	}
 
-	if (last_subreq) {
-		smp_wmb(); /* Write lists before ALL_QUEUED. */
-		set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
-	}
-
 	spin_unlock(&rreq->lock);
 }
 
@@ -214,7 +209,6 @@ static void netfs_issue_read(struct netfs_io_request *rreq,
 static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
 				    struct readahead_control *ractl)
 {
-	struct netfs_inode *ictx = netfs_inode(rreq->inode);
 	unsigned long long start = rreq->start;
 	ssize_t size = rreq->len;
 	int ret = 0;
@@ -233,10 +227,13 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
 		subreq->start	= start;
 		subreq->len	= size;
 
+		netfs_queue_read(rreq, subreq);
+
 		source = netfs_cache_prepare_read(rreq, subreq, rreq->i_size);
 		subreq->source = source;
 		if (source == NETFS_DOWNLOAD_FROM_SERVER) {
-			unsigned long long zp = umin(ictx->zero_point, rreq->i_size);
+			unsigned long long zero_point = netfs_read_zero_point(rreq->inode);
+			unsigned long long zp = umin(zero_point, rreq->i_size);
 			size_t len = subreq->len;
 
 			if (unlikely(rreq->origin == NETFS_READ_SINGLE))
@@ -252,7 +249,8 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
 				pr_err("ZERO-LEN READ: R=%08x[%x] l=%zx/%zx s=%llx z=%llx i=%llx",
 				       rreq->debug_id, subreq->debug_index,
 				       subreq->len, size,
-				       subreq->start, ictx->zero_point, rreq->i_size);
+				       subreq->start, zero_point, rreq->i_size);
+				netfs_cancel_read(subreq, ret);
 				break;
 			}
 			subreq->len = len;
@@ -261,12 +259,7 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
 			if (rreq->netfs_ops->prepare_read) {
 				ret = rreq->netfs_ops->prepare_read(subreq);
 				if (ret < 0) {
-					subreq->error = ret;
-					/* Not queued - release both refs. */
-					netfs_put_subrequest(subreq,
-							     netfs_sreq_trace_put_cancel);
-					netfs_put_subrequest(subreq,
-							     netfs_sreq_trace_put_cancel);
+					netfs_cancel_read(subreq, ret);
 					break;
 				}
 				trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
@@ -289,24 +282,29 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq,
 
 		pr_err("Unexpected read source %u\n", source);
 		WARN_ON_ONCE(1);
+		netfs_cancel_read(subreq, ret);
 		break;
 
 	issue:
 		slice = netfs_prepare_read_iterator(subreq, ractl);
 		if (slice < 0) {
 			ret = slice;
-			subreq->error = ret;
-			trace_netfs_sreq(subreq, netfs_sreq_trace_cancel);
-			/* Not queued - release both refs. */
-			netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel);
-			netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel);
+			netfs_cancel_read(subreq, ret);
 			break;
 		}
-		size -= slice;
 		start += slice;
+		size -= slice;
+		if (size <= 0) {
+			smp_wmb(); /* Write lists before ALL_QUEUED. */
+			set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+		}
 
-		netfs_queue_read(rreq, subreq, size <= 0);
 		netfs_issue_read(rreq, subreq);
+
+		if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags))
+			netfs_wait_for_paused_read(rreq);
+		if (test_bit(NETFS_RREQ_FAILED, &rreq->flags))
+			break;
 		cond_resched();
 	} while (size > 0);
 
@@ -397,6 +395,7 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
 {
 	struct netfs_io_request *rreq;
 	struct address_space *mapping = folio->mapping;
+	struct netfs_group *group = netfs_folio_group(folio);
 	struct netfs_folio *finfo = netfs_folio_info(folio);
 	struct netfs_inode *ctx = netfs_inode(mapping->host);
 	struct folio *sink = NULL;
@@ -458,14 +457,20 @@ static int netfs_read_gaps(struct file *file, struct folio *folio)
 
 	netfs_read_to_pagecache(rreq, NULL);
 
-	if (sink)
-		folio_put(sink);
-
 	ret = netfs_wait_for_read(rreq);
 	if (ret >= 0) {
+		if (group)
+			folio_change_private(folio, group);
+		else
+			folio_detach_private(folio);
+		kfree(finfo);
+		trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
 		flush_dcache_folio(folio);
 		folio_mark_uptodate(folio);
 	}
+
+	if (sink)
+		folio_put(sink);
 	folio_unlock(folio);
 	netfs_put_request(rreq, netfs_rreq_trace_put_return);
 	return ret < 0 ? ret : 0;
@@ -498,10 +503,10 @@ int netfs_read_folio(struct file *file, struct folio *folio)
 	struct netfs_inode *ctx = netfs_inode(mapping->host);
 	int ret;
 
-	if (folio_test_dirty(folio)) {
-		trace_netfs_folio(folio, netfs_folio_trace_read_gaps);
+	folio_wait_writeback(folio);
+
+	if (folio_test_dirty(folio))
 		return netfs_read_gaps(file, folio);
-	}
 
 	_enter("%lx", folio->index);
 
@@ -667,7 +672,7 @@ int netfs_write_begin(struct netfs_inode *ctx,
 		ret = PTR_ERR(rreq);
 		goto error;
 	}
-	rreq->no_unlock_folio	= folio->index;
+	rreq->no_unlock_folio	= folio;
 	__set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags);
 
 	ret = netfs_begin_cache_read(rreq, ctx);
@@ -684,9 +689,9 @@ int netfs_write_begin(struct netfs_inode *ctx,
 
 	netfs_read_to_pagecache(rreq, NULL);
 	ret = netfs_wait_for_read(rreq);
+	netfs_put_request(rreq, netfs_rreq_trace_put_return);
 	if (ret < 0)
 		goto error;
-	netfs_put_request(rreq, netfs_rreq_trace_put_return);
 
 have_folio:
 	ret = folio_wait_private_2_killable(folio);
@@ -733,7 +738,7 @@ int netfs_prefetch_for_write(struct file *file, struct folio *folio,
 		goto error;
 	}
 
-	rreq->no_unlock_folio = folio->index;
+	rreq->no_unlock_folio = folio;
 	__set_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags);
 	ret = netfs_begin_cache_read(rreq, ctx);
 	if (ret == -ENOMEM || ret == -EINTR || ret == -ERESTARTSYS)
diff --git a/fs/netfs/buffered_write.c b/fs/netfs/buffered_write.c
index 05ea5b0cc0e8b..6bde3320bcec6 100644
--- a/fs/netfs/buffered_write.c
+++ b/fs/netfs/buffered_write.c
@@ -12,24 +12,6 @@
 #include <linux/slab.h>
 #include "internal.h"
 
-static void __netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
-{
-	if (netfs_group)
-		folio_attach_private(folio, netfs_get_group(netfs_group));
-}
-
-static void netfs_set_group(struct folio *folio, struct netfs_group *netfs_group)
-{
-	void *priv = folio_get_private(folio);
-
-	if (unlikely(priv != netfs_group)) {
-		if (netfs_group && (!priv || priv == NETFS_FOLIO_COPY_TO_CACHE))
-			folio_attach_private(folio, netfs_get_group(netfs_group));
-		else if (!netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE)
-			folio_detach_private(folio);
-	}
-}
-
 /*
  * Grab a folio for writing and lock it.  Attempt to allocate as large a folio
  * as possible to hold as much of the remaining length as possible in one go.
@@ -149,6 +131,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 	}
 
 	do {
+		enum netfs_folio_trace trace;
 		struct netfs_folio *finfo;
 		struct netfs_group *group;
 		unsigned long long fpos;
@@ -156,6 +139,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 		size_t offset;	/* Offset into pagecache folio */
 		size_t part;	/* Bytes to write to folio */
 		size_t copied;	/* Bytes copied from user */
+		void *priv;
 
 		offset = pos & (max_chunk - 1);
 		part = min(max_chunk - offset, iov_iter_count(iter));
@@ -201,73 +185,99 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 			goto error_folio_unlock;
 		}
 
-		/* Decide how we should modify a folio.  We might be attempting
-		 * to do write-streaming, in which case we don't want to a
-		 * local RMW cycle if we can avoid it.  If we're doing local
-		 * caching or content crypto, we award that priority over
-		 * avoiding RMW.  If the file is open readably, then we also
-		 * assume that we may want to read what we wrote.
-		 */
 		finfo = netfs_folio_info(folio);
 		group = netfs_folio_group(folio);
 
+		/* If the requested group differs from the group set on the
+		 * page, then we need to flush out the folio if it has a group
+		 * set (ie. is non-NULL).  Note that COPY_TO_CACHE is a special
+		 * case, being a netfs annotation rather than an actual group.
+		 *
+		 * The filesystem isn't permitted to mix writes with groups and
+		 * writes without groups as the NULL group is used to indicate
+		 * that no group is set.
+		 */
 		if (unlikely(group != netfs_group) &&
-		    group != NETFS_FOLIO_COPY_TO_CACHE)
+		    group != NETFS_FOLIO_COPY_TO_CACHE &&
+		    group) {
+			WARN_ON_ONCE(!netfs_group);
 			goto flush_content;
+		}
 
+		/* Decide how we should modify a folio.  We might be attempting
+		 * to do write-streaming, as we don't want to a local RMW cycle
+		 * if we can avoid it.  If we're doing local caching or content
+		 * crypto, we award that priority over avoiding RMW.  If the
+		 * file is open readably, then we let ->read_folio() fill in
+		 * the gaps.
+		 */
 		if (folio_test_uptodate(folio)) {
 			if (mapping_writably_mapped(mapping))
 				flush_dcache_folio(folio);
 			copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
 			if (unlikely(copied == 0))
 				goto copy_failed;
-			netfs_set_group(folio, netfs_group);
-			trace_netfs_folio(folio, netfs_folio_is_uptodate);
-			goto copied;
+			trace = netfs_folio_is_uptodate;
+			goto copied_uptodate;
 		}
 
 		/* If the page is above the zero-point then we assume that the
 		 * server would just return a block of zeros or a short read if
 		 * we try to read it.
 		 */
-		if (fpos >= ctx->zero_point) {
+		if (fpos >= netfs_read_zero_point(inode)) {
 			folio_zero_segment(folio, 0, offset);
 			copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
 			if (unlikely(copied == 0))
 				goto copy_failed;
 			folio_zero_segment(folio, offset + copied, flen);
-			__netfs_set_group(folio, netfs_group);
-			folio_mark_uptodate(folio);
-			trace_netfs_folio(folio, netfs_modify_and_clear);
-			goto copied;
+			if (finfo)
+				trace = netfs_modify_and_clear_rm_finfo;
+			else
+				trace = netfs_modify_and_clear;
+			goto mark_uptodate;
 		}
 
 		/* See if we can write a whole folio in one go. */
 		if (!maybe_trouble && offset == 0 && part >= flen) {
 			copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
-			if (unlikely(copied == 0))
+			if (likely(copied == part)) {
+				if (finfo)
+					trace = netfs_whole_folio_modify_filled;
+				else
+					trace = netfs_whole_folio_modify;
+				goto mark_uptodate;
+			}
+			if (copied == 0)
 				goto copy_failed;
-			if (unlikely(copied < part)) {
+			if (!finfo || copied <= finfo->dirty_offset) {
 				maybe_trouble = true;
 				iov_iter_revert(iter, copied);
 				copied = 0;
 				folio_unlock(folio);
 				goto retry;
 			}
-			__netfs_set_group(folio, netfs_group);
-			folio_mark_uptodate(folio);
-			trace_netfs_folio(folio, netfs_whole_folio_modify);
+
+			/* We overwrote some existing dirty data, so we have to
+			 * accept the partial write.
+			 */
+			finfo->dirty_len += finfo->dirty_offset;
+			if (finfo->dirty_len == flen) {
+				trace = netfs_whole_folio_modify_filled_efault;
+				goto mark_uptodate;
+			}
+			if (copied > finfo->dirty_len)
+				finfo->dirty_len = copied;
+			finfo->dirty_offset = 0;
+			trace = netfs_whole_folio_modify_efault;
 			goto copied;
 		}
 
 		/* We don't want to do a streaming write on a file that loses
 		 * caching service temporarily because the backing store got
-		 * culled and we don't really want to get a streaming write on
-		 * a file that's open for reading as ->read_folio() then has to
-		 * be able to flush it.
+		 * culled.
 		 */
-		if ((file->f_mode & FMODE_READ) ||
-		    netfs_is_cache_enabled(ctx)) {
+		if (netfs_is_cache_enabled(ctx)) {
 			if (finfo) {
 				netfs_stat(&netfs_n_wh_wstream_conflict);
 				goto flush_content;
@@ -282,11 +292,11 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 			copied = copy_folio_from_iter_atomic(folio, offset, part, iter);
 			if (unlikely(copied == 0))
 				goto copy_failed;
-			netfs_set_group(folio, netfs_group);
-			trace_netfs_folio(folio, netfs_just_prefetch);
-			goto copied;
+			trace = netfs_just_prefetch;
+			goto copied_uptodate;
 		}
 
+		/* Do a streaming write on a folio that has nothing in it yet. */
 		if (!finfo) {
 			ret = -EIO;
 			if (WARN_ON(folio_get_private(folio)))
@@ -295,10 +305,8 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 			if (unlikely(copied == 0))
 				goto copy_failed;
 			if (offset == 0 && copied == flen) {
-				__netfs_set_group(folio, netfs_group);
-				folio_mark_uptodate(folio);
-				trace_netfs_folio(folio, netfs_streaming_filled_page);
-				goto copied;
+				trace = netfs_streaming_filled_page;
+				goto mark_uptodate;
 			}
 
 			finfo = kzalloc_obj(*finfo);
@@ -312,7 +320,7 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 			finfo->dirty_len = copied;
 			folio_attach_private(folio, (void *)((unsigned long)finfo |
 							     NETFS_FOLIO_INFO));
-			trace_netfs_folio(folio, netfs_streaming_write);
+			trace = netfs_streaming_write;
 			goto copied;
 		}
 
@@ -326,16 +334,10 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 				goto copy_failed;
 			finfo->dirty_len += copied;
 			if (finfo->dirty_offset == 0 && finfo->dirty_len == flen) {
-				if (finfo->netfs_group)
-					folio_change_private(folio, finfo->netfs_group);
-				else
-					folio_detach_private(folio);
-				folio_mark_uptodate(folio);
-				kfree(finfo);
-				trace_netfs_folio(folio, netfs_streaming_cont_filled_page);
-			} else {
-				trace_netfs_folio(folio, netfs_streaming_write_cont);
+				trace = netfs_streaming_cont_filled_page;
+				goto mark_uptodate;
 			}
+			trace = netfs_streaming_write_cont;
 			goto copied;
 		}
 
@@ -349,7 +351,38 @@ ssize_t netfs_perform_write(struct kiocb *iocb, struct iov_iter *iter,
 			goto out;
 		continue;
 
+		/* Mark a folio as being up to data when we've filled it
+		 * completely.  If the folio has a group attached, then it must
+		 * be the same group, otherwise we should have flushed it out
+		 * above.  We have to get rid of the netfs_folio struct if
+		 * there was one.
+		 */
+	mark_uptodate:
+		folio_mark_uptodate(folio);
+
+	copied_uptodate:
+		priv = folio_get_private(folio);
+		if (likely(priv == netfs_group)) {
+			/* Already set correctly; no change required. */
+		} else if (priv == NETFS_FOLIO_COPY_TO_CACHE) {
+			if (!netfs_group)
+				folio_detach_private(folio);
+			else
+				folio_change_private(folio, netfs_get_group(netfs_group));
+		} else if (!priv) {
+			folio_attach_private(folio, netfs_get_group(netfs_group));
+		} else {
+			WARN_ON_ONCE(!finfo);
+			if (netfs_group)
+				/* finfo->netfs_group has a ref */
+				folio_change_private(folio, netfs_group);
+			else
+				folio_detach_private(folio);
+			kfree(finfo);
+		}
+
 	copied:
+		trace_netfs_folio(folio, trace);
 		flush_dcache_folio(folio);
 
 		/* Update the inode size if we moved the EOF marker */
@@ -510,6 +543,7 @@ vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_gr
 	struct inode *inode = file_inode(file);
 	struct netfs_inode *ictx = netfs_inode(inode);
 	vm_fault_t ret = VM_FAULT_NOPAGE;
+	void *priv;
 	int err;
 
 	_enter("%lx", folio->index);
@@ -530,7 +564,9 @@ vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_gr
 	}
 
 	group = netfs_folio_group(folio);
-	if (group != netfs_group && group != NETFS_FOLIO_COPY_TO_CACHE) {
+	if (group &&
+	    group != netfs_group &&
+	    group != NETFS_FOLIO_COPY_TO_CACHE) {
 		folio_unlock(folio);
 		err = filemap_fdatawrite_range(mapping,
 					       folio_pos(folio),
@@ -552,7 +588,19 @@ vm_fault_t netfs_page_mkwrite(struct vm_fault *vmf, struct netfs_group *netfs_gr
 		trace_netfs_folio(folio, netfs_folio_trace_mkwrite_plus);
 	else
 		trace_netfs_folio(folio, netfs_folio_trace_mkwrite);
-	netfs_set_group(folio, netfs_group);
+
+	priv = folio_get_private(folio);
+	if (priv != netfs_group) {
+		if (!netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE)
+			folio_detach_private(folio);
+		else if (netfs_group && priv == NETFS_FOLIO_COPY_TO_CACHE)
+			folio_change_private(folio, netfs_get_group(netfs_group));
+		else if (netfs_group && !priv)
+			folio_attach_private(folio, netfs_get_group(netfs_group));
+		else
+			WARN_ON_ONCE(1);
+	}
+
 	file_update_time(file);
 	set_bit(NETFS_ICTX_MODIFIED_ATTR, &ictx->flags);
 	if (ictx->ops->post_modify)
diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c
index f72e6da88cca7..6a8fb0d55e040 100644
--- a/fs/netfs/direct_read.c
+++ b/fs/netfs/direct_read.c
@@ -45,12 +45,11 @@ static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq)
  * Perform a read to a buffer from the server, slicing up the region to be read
  * according to the network rsize.
  */
-static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
+static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
 {
-	struct netfs_io_stream *stream = &rreq->io_streams[0];
 	unsigned long long start = rreq->start;
 	ssize_t size = rreq->len;
-	int ret = 0;
+	int ret;
 
 	do {
 		struct netfs_io_subrequest *subreq;
@@ -58,7 +57,10 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
 
 		subreq = netfs_alloc_subrequest(rreq);
 		if (!subreq) {
-			ret = -ENOMEM;
+			/* Stash the error in the request if there's not
+			 * already an error set.
+			 */
+			cmpxchg(&rreq->error, 0, -ENOMEM);
 			break;
 		}
 
@@ -66,25 +68,13 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
 		subreq->start	= start;
 		subreq->len	= size;
 
-		__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
-
-		spin_lock(&rreq->lock);
-		list_add_tail(&subreq->rreq_link, &stream->subrequests);
-		if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
-			if (!stream->active) {
-				stream->collected_to = subreq->start;
-				/* Store list pointers before active flag */
-				smp_store_release(&stream->active, true);
-			}
-		}
-		trace_netfs_sreq(subreq, netfs_sreq_trace_added);
-		spin_unlock(&rreq->lock);
+		netfs_queue_read(rreq, subreq);
 
 		netfs_stat(&netfs_n_rh_download);
 		if (rreq->netfs_ops->prepare_read) {
 			ret = rreq->netfs_ops->prepare_read(subreq);
 			if (ret < 0) {
-				netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel);
+				netfs_cancel_read(subreq, ret);
 				break;
 			}
 		}
@@ -113,8 +103,6 @@ static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
 		set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
 		netfs_wake_collector(rreq);
 	}
-
-	return ret;
 }
 
 /*
@@ -137,21 +125,17 @@ static ssize_t netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync)
 	// TODO: Use bounce buffer if requested
 
 	inode_dio_begin(rreq->inode);
+	netfs_dispatch_unbuffered_reads(rreq);
 
-	ret = netfs_dispatch_unbuffered_reads(rreq);
-
-	if (!rreq->submitted) {
-		netfs_put_request(rreq, netfs_rreq_trace_put_no_submit);
-		inode_dio_end(rreq->inode);
-		ret = 0;
-		goto out;
-	}
+	/* The collector will get run, even if we don't manage to submit any
+	 * subreqs, so we shouldn't call inode_dio_end() here.
+	 */
 
 	if (sync)
 		ret = netfs_wait_for_read(rreq);
 	else
 		ret = -EIOCBQUEUED;
-out:
+
 	_leave(" = %zd", ret);
 	return ret;
 }
diff --git a/fs/netfs/direct_write.c b/fs/netfs/direct_write.c
index f9ab69de3e298..25f8ceb15fad6 100644
--- a/fs/netfs/direct_write.c
+++ b/fs/netfs/direct_write.c
@@ -376,8 +376,10 @@ ssize_t netfs_unbuffered_write_iter(struct kiocb *iocb, struct iov_iter *from)
 	if (ret < 0)
 		goto out;
 	end = iocb->ki_pos + iov_iter_count(from);
-	if (end > ictx->zero_point)
-		ictx->zero_point = end;
+	spin_lock(&inode->i_lock);
+	if (end > ictx->_zero_point)
+		netfs_write_zero_point(inode, end);
+	spin_unlock(&inode->i_lock);
 
 	fscache_invalidate(netfs_i_cookie(ictx), NULL, i_size_read(inode),
 			   FSCACHE_INVAL_DIO_WRITE);
diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h
index d436e20d34185..645996ecfc803 100644
--- a/fs/netfs/internal.h
+++ b/fs/netfs/internal.h
@@ -23,6 +23,8 @@
 /*
  * buffered_read.c
  */
+void netfs_queue_read(struct netfs_io_request *rreq,
+		      struct netfs_io_subrequest *subreq);
 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
 int netfs_prefetch_for_write(struct file *file, struct folio *folio,
 			     size_t offset, size_t len);
@@ -108,6 +110,7 @@ static inline void netfs_see_subrequest(struct netfs_io_subrequest *subreq,
  */
 bool netfs_read_collection(struct netfs_io_request *rreq);
 void netfs_read_collection_worker(struct work_struct *work);
+void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error);
 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error);
 
 /*
diff --git a/fs/netfs/iterator.c b/fs/netfs/iterator.c
index 154a14bb2d7f7..b375567e0520e 100644
--- a/fs/netfs/iterator.c
+++ b/fs/netfs/iterator.c
@@ -22,7 +22,7 @@
  *
  * Extract the page fragments from the given amount of the source iterator and
  * build up a second iterator that refers to all of those bits.  This allows
- * the original iterator to disposed of.
+ * the original iterator to be disposed of.
  *
  * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be
  * allowed on the pages extracted.
@@ -43,7 +43,7 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
 	unsigned int max_pages;
 	unsigned int npages = 0;
 	unsigned int i;
-	ssize_t ret;
+	ssize_t ret = 0;
 	size_t count = orig_len, offset, len;
 	size_t bv_size, pg_size;
 
@@ -67,26 +67,29 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
 		ret = iov_iter_extract_pages(orig, &pages, count,
 					     max_pages - npages, extraction_flags,
 					     &offset);
-		if (ret < 0) {
-			pr_err("Couldn't get user pages (rc=%zd)\n", ret);
+		if (unlikely(ret <= 0)) {
+			ret = ret ?: -EIO;
 			break;
 		}
 
-		if (ret > count) {
-			pr_err("get_pages rc=%zd more than %zu\n", ret, count);
+		if (WARN(ret > count,
+			 "%s: extract_pages overrun %zd > %zu bytes\n",
+			 __func__, ret, count)) {
+			ret = -EIO;
 			break;
 		}
 
-		count -= ret;
-		ret += offset;
-		cur_npages = DIV_ROUND_UP(ret, PAGE_SIZE);
-
-		if (npages + cur_npages > max_pages) {
-			pr_err("Out of bvec array capacity (%u vs %u)\n",
-			       npages + cur_npages, max_pages);
+		cur_npages = DIV_ROUND_UP(offset + ret, PAGE_SIZE);
+		if (WARN(cur_npages > max_pages - npages,
+			 "%s: extract_pages overrun %u > %u pages\n",
+			 __func__, npages + cur_npages, max_pages)) {
+			ret = -EIO;
 			break;
 		}
 
+		count -= ret;
+		ret += offset;
+
 		for (i = 0; i < cur_npages; i++) {
 			len = ret > PAGE_SIZE ? PAGE_SIZE : ret;
 			bvec_set_page(bv + npages + i, *pages++, len - offset, offset);
@@ -97,6 +100,18 @@ ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len,
 		npages += cur_npages;
 	}
 
+	/* Note: Don't try to clean up after EIO.  Either we got no pages, so
+	 * nothing to clean up, or we got a buffer overrun, memory corruption
+	 * and can't trust the stuff in the buffer (a WARN was emitted).
+	 */
+
+	if (ret < 0 && (ret == -ENOMEM || npages == 0)) {
+		for (i = 0; i < npages; i++)
+			unpin_user_page(bv[i].bv_page);
+		kvfree(bv);
+		return ret;
+	}
+
 	iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count);
 	return npages;
 }
diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c
index 6df89c92b10b0..5d554512ed23a 100644
--- a/fs/netfs/misc.c
+++ b/fs/netfs/misc.c
@@ -211,18 +211,25 @@ EXPORT_SYMBOL(netfs_clear_inode_writeback);
 void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
 {
 	struct netfs_folio *finfo;
-	struct netfs_inode *ctx = netfs_inode(folio_inode(folio));
+	struct inode *inode = folio_inode(folio);
+	struct netfs_inode *ctx = netfs_inode(inode);
 	size_t flen = folio_size(folio);
 
 	_enter("{%lx},%zx,%zx", folio->index, offset, length);
 
 	if (offset == 0 && length == flen) {
-		unsigned long long i_size = i_size_read(&ctx->inode);
+		unsigned long long i_size, remote_i_size, zero_point;
 		unsigned long long fpos = folio_pos(folio), end;
 
+		netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
 		end = umin(fpos + flen, i_size);
-		if (fpos < i_size && end > ctx->zero_point)
-			ctx->zero_point = end;
+		if (fpos < i_size && end > zero_point) {
+			spin_lock(&inode->i_lock);
+			end = umin(fpos + flen, inode->i_size);
+			if (fpos < i_size && end > ctx->_zero_point)
+				netfs_write_zero_point(inode, end);
+			spin_unlock(&inode->i_lock);
+		}
 	}
 
 	folio_wait_private_2(folio); /* [DEPRECATED] */
@@ -255,7 +262,8 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
 				goto erase_completely;
 			/* Move the start of the data. */
 			finfo->dirty_len = fend - iend;
-			finfo->dirty_offset = offset;
+			finfo->dirty_offset = iend;
+			trace_netfs_folio(folio, netfs_folio_trace_invalidate_front);
 			return;
 		}
 
@@ -264,12 +272,14 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
 		 */
 		if (iend >= fend) {
 			finfo->dirty_len = offset - fstart;
+			trace_netfs_folio(folio, netfs_folio_trace_invalidate_tail);
 			return;
 		}
 
 		/* A partial write was split.  The caller has already zeroed
 		 * it, so just absorb the hole.
 		 */
+		trace_netfs_folio(folio, netfs_folio_trace_invalidate_middle);
 	}
 	return;
 
@@ -277,8 +287,9 @@ void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
 	netfs_put_group(netfs_folio_group(folio));
 	folio_detach_private(folio);
 	folio_clear_uptodate(folio);
+	folio_cancel_dirty(folio);
 	kfree(finfo);
-	return;
+	trace_netfs_folio(folio, netfs_folio_trace_invalidate_all);
 }
 EXPORT_SYMBOL(netfs_invalidate_folio);
 
@@ -292,15 +303,22 @@ EXPORT_SYMBOL(netfs_invalidate_folio);
  */
 bool netfs_release_folio(struct folio *folio, gfp_t gfp)
 {
-	struct netfs_inode *ctx = netfs_inode(folio_inode(folio));
-	unsigned long long end;
+	struct inode *inode = folio_inode(folio);
+	struct netfs_inode *ctx = netfs_inode(inode);
+	unsigned long long i_size, remote_i_size, zero_point, end;
 
 	if (folio_test_dirty(folio))
 		return false;
 
-	end = umin(folio_next_pos(folio), i_size_read(&ctx->inode));
-	if (end > ctx->zero_point)
-		ctx->zero_point = end;
+	netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
+	end = folio_next_pos(folio);
+	if (end > zero_point) {
+		spin_lock(&inode->i_lock);
+		end = umin(end, ctx->_remote_i_size);
+		if (end > ctx->_zero_point)
+			netfs_write_zero_point(inode, end);
+		spin_unlock(&inode->i_lock);
+	}
 
 	if (folio_test_private(folio))
 		return false;
@@ -356,6 +374,7 @@ void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq,
 	DEFINE_WAIT(myself);
 
 	list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
+		smp_rmb(); /* Read ->next before IN_PROGRESS. */
 		if (!netfs_check_subreq_in_progress(subreq))
 			continue;
 
diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c
index e5f6665b3341e..23660a5901246 100644
--- a/fs/netfs/read_collect.c
+++ b/fs/netfs/read_collect.c
@@ -83,7 +83,7 @@ static void netfs_unlock_read_folio(struct netfs_io_request *rreq,
 	}
 
 just_unlock:
-	if (folio->index == rreq->no_unlock_folio &&
+	if (folio == rreq->no_unlock_folio &&
 	    test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) {
 		_debug("no unlock");
 	} else {
@@ -205,8 +205,10 @@ static void netfs_collect_read_results(struct netfs_io_request *rreq)
 	 * in progress.  The issuer thread may be adding stuff to the tail
 	 * whilst we're doing this.
 	 */
-	front = list_first_entry_or_null(&stream->subrequests,
-					 struct netfs_io_subrequest, rreq_link);
+	front = list_first_entry_or_null_acquire(&stream->subrequests,
+						 struct netfs_io_subrequest, rreq_link);
+	/* Read first subreq pointer before IN_PROGRESS flag. */
+
 	while (front) {
 		size_t transferred;
 
@@ -575,6 +577,17 @@ void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq)
 }
 EXPORT_SYMBOL(netfs_read_subreq_terminated);
 
+/*
+ * Cancel a read subrequest due to preparation failure.
+ */
+void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error)
+{
+	trace_netfs_sreq(subreq, netfs_sreq_trace_cancel);
+	subreq->error = error;
+	__set_bit(NETFS_SREQ_FAILED, &subreq->flags);
+	netfs_read_subreq_terminated(subreq);
+}
+
 /*
  * Handle termination of a read from the cache.
  */
diff --git a/fs/netfs/read_retry.c b/fs/netfs/read_retry.c
index cca9ac43c0773..f59a70f3a086b 100644
--- a/fs/netfs/read_retry.c
+++ b/fs/netfs/read_retry.c
@@ -175,7 +175,9 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
 			list_for_each_entry_safe_from(subreq, tmp,
 						      &stream->subrequests, rreq_link) {
 				trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous);
+				spin_lock(&rreq->lock);
 				list_del(&subreq->rreq_link);
+				spin_unlock(&rreq->lock);
 				netfs_put_subrequest(subreq, netfs_sreq_trace_put_done);
 				if (subreq == to)
 					break;
@@ -203,8 +205,10 @@ static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
 					     refcount_read(&subreq->ref),
 					     netfs_sreq_trace_new);
 
+			spin_lock(&rreq->lock);
 			list_add(&subreq->rreq_link, &to->rreq_link);
-			to = list_next_entry(to, rreq_link);
+			spin_unlock(&rreq->lock);
+			to = subreq;
 			trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
 
 			stream->sreq_max_len	= umin(len, rreq->rsize);
@@ -288,8 +292,15 @@ void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
 			struct folio *folio = folioq_folio(p, slot);
 
 			if (folio && !folioq_is_marked2(p, slot)) {
-				trace_netfs_folio(folio, netfs_folio_trace_abandon);
-				folio_unlock(folio);
+				if (folio == rreq->no_unlock_folio &&
+				    test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO,
+					     &rreq->flags)) {
+					_debug("no unlock");
+				} else {
+					trace_netfs_folio(folio,
+						netfs_folio_trace_abandon);
+					folio_unlock(folio);
+				}
 			}
 		}
 	}
diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c
index d0e23bc42445f..8833550d2eb60 100644
--- a/fs/netfs/read_single.c
+++ b/fs/netfs/read_single.c
@@ -89,7 +89,6 @@ static void netfs_single_read_cache(struct netfs_io_request *rreq,
  */
 static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
 {
-	struct netfs_io_stream *stream = &rreq->io_streams[0];
 	struct netfs_io_subrequest *subreq;
 	int ret = 0;
 
@@ -102,14 +101,7 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
 	subreq->len	= rreq->len;
 	subreq->io_iter	= rreq->buffer.iter;
 
-	__set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
-
-	spin_lock(&rreq->lock);
-	list_add_tail(&subreq->rreq_link, &stream->subrequests);
-	trace_netfs_sreq(subreq, netfs_sreq_trace_added);
-	/* Store list pointers before active flag */
-	smp_store_release(&stream->active, true);
-	spin_unlock(&rreq->lock);
+	netfs_queue_read(rreq, subreq);
 
 	netfs_single_cache_prepare_read(rreq, subreq);
 	switch (subreq->source) {
@@ -121,10 +113,14 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
 				goto cancel;
 		}
 
+		smp_wmb(); /* Write lists before ALL_QUEUED. */
+		set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
 		rreq->netfs_ops->issue_read(subreq);
 		rreq->submitted += subreq->len;
 		break;
 	case NETFS_READ_FROM_CACHE:
+		smp_wmb(); /* Write lists before ALL_QUEUED. */
+		set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
 		trace_netfs_sreq(subreq, netfs_sreq_trace_submit);
 		netfs_single_read_cache(rreq, subreq);
 		rreq->submitted += subreq->len;
@@ -134,14 +130,15 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq)
 		pr_warn("Unexpected single-read source %u\n", subreq->source);
 		WARN_ON_ONCE(true);
 		ret = -EIO;
-		break;
+		goto cancel;
 	}
 
-	smp_wmb(); /* Write lists before ALL_QUEUED. */
-	set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
 	return ret;
 cancel:
-	netfs_put_subrequest(subreq, netfs_sreq_trace_put_cancel);
+	netfs_cancel_read(subreq, ret);
+	smp_wmb(); /* Write lists before ALL_QUEUED. */
+	set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
+	netfs_wake_collector(rreq);
 	return ret;
 }
 
diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c
index b194447f4b111..24fc2bb2f8a47 100644
--- a/fs/netfs/write_collect.c
+++ b/fs/netfs/write_collect.c
@@ -57,7 +57,8 @@ static void netfs_dump_request(const struct netfs_io_request *rreq)
 int netfs_folio_written_back(struct folio *folio)
 {
 	enum netfs_folio_trace why = netfs_folio_trace_clear;
-	struct netfs_inode *ictx = netfs_inode(folio->mapping->host);
+	struct inode *inode = folio_inode(folio);
+	struct netfs_inode *ictx = netfs_inode(inode);
 	struct netfs_folio *finfo;
 	struct netfs_group *group = NULL;
 	int gcount = 0;
@@ -69,8 +70,10 @@ int netfs_folio_written_back(struct folio *folio)
 		unsigned long long fend;
 
 		fend = folio_pos(folio) + finfo->dirty_offset + finfo->dirty_len;
-		if (fend > ictx->zero_point)
-			ictx->zero_point = fend;
+		spin_lock(&ictx->inode.i_lock);
+		if (fend > ictx->_zero_point)
+			netfs_write_zero_point(inode, fend);
+		spin_unlock(&ictx->inode.i_lock);
 
 		folio_detach_private(folio);
 		group = finfo->netfs_group;
@@ -228,8 +231,10 @@ static void netfs_collect_write_results(struct netfs_io_request *wreq)
 		if (!smp_load_acquire(&stream->active))
 			continue;
 
-		front = list_first_entry_or_null(&stream->subrequests,
-						 struct netfs_io_subrequest, rreq_link);
+		front = list_first_entry_or_null_acquire(&stream->subrequests,
+							 struct netfs_io_subrequest, rreq_link);
+		/* Read first subreq pointer before IN_PROGRESS flag. */
+
 		while (front) {
 			trace_netfs_collect_sreq(wreq, front);
 			//_debug("sreq [%x] %llx %zx/%zx",
diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c
index 2db688f941251..c03c7cc45e471 100644
--- a/fs/netfs/write_issue.c
+++ b/fs/netfs/write_issue.c
@@ -204,7 +204,8 @@ void netfs_prepare_write(struct netfs_io_request *wreq,
 	 * remove entries off of the front.
 	 */
 	spin_lock(&wreq->lock);
-	list_add_tail(&subreq->rreq_link, &stream->subrequests);
+	/* Write IN_PROGRESS before pointer to new subreq */
+	list_add_tail_release(&subreq->rreq_link, &stream->subrequests);
 	if (list_is_first(&subreq->rreq_link, &stream->subrequests)) {
 		if (!stream->active) {
 			stream->collected_to = subreq->start;
@@ -413,12 +414,7 @@ static int netfs_write_folio(struct netfs_io_request *wreq,
 	if (streamw)
 		netfs_issue_write(wreq, cache);
 
-	/* Flip the page to the writeback state and unlock.  If we're called
-	 * from write-through, then the page has already been put into the wb
-	 * state.
-	 */
-	if (wreq->origin == NETFS_WRITEBACK)
-		folio_start_writeback(folio);
+	folio_start_writeback(folio);
 	folio_unlock(folio);
 
 	if (fgroup == NETFS_FOLIO_COPY_TO_CACHE) {
@@ -646,29 +642,41 @@ int netfs_advance_writethrough(struct netfs_io_request *wreq, struct writeback_c
 			       struct folio *folio, size_t copied, bool to_page_end,
 			       struct folio **writethrough_cache)
 {
+	int ret;
+
 	_enter("R=%x ic=%zu ws=%u cp=%zu tp=%u",
 	       wreq->debug_id, wreq->buffer.iter.count, wreq->wsize, copied, to_page_end);
 
-	if (!*writethrough_cache) {
-		if (folio_test_dirty(folio))
-			/* Sigh.  mmap. */
-			folio_clear_dirty_for_io(folio);
+	/* The folio is locked. */
 
+	if (*writethrough_cache != folio) {
+		if (*writethrough_cache) {
+			/* Did the folio get moved? */
+			folio_put(*writethrough_cache);
+			*writethrough_cache = NULL;
+		}
 		/* We can make multiple writes to the folio... */
-		folio_start_writeback(folio);
 		if (wreq->len == 0)
 			trace_netfs_folio(folio, netfs_folio_trace_wthru);
 		else
 			trace_netfs_folio(folio, netfs_folio_trace_wthru_plus);
 		*writethrough_cache = folio;
+		folio_get(folio);
 	}
 
 	wreq->len += copied;
-	if (!to_page_end)
+
+	if (!to_page_end) {
+		folio_mark_dirty(folio);
+		folio_unlock(folio);
 		return 0;
+	}
 
+	ret = netfs_write_folio(wreq, wbc, folio);
+	folio_put(*writethrough_cache);
 	*writethrough_cache = NULL;
-	return netfs_write_folio(wreq, wbc, folio);
+	wreq->submitted = wreq->len;
+	return ret;
 }
 
 /*
@@ -682,8 +690,12 @@ ssize_t netfs_end_writethrough(struct netfs_io_request *wreq, struct writeback_c
 
 	_enter("R=%x", wreq->debug_id);
 
-	if (writethrough_cache)
+	if (writethrough_cache) {
+		folio_lock(writethrough_cache);
 		netfs_write_folio(wreq, wbc, writethrough_cache);
+		folio_put(writethrough_cache);
+		wreq->submitted = wreq->len;
+	}
 
 	netfs_end_issue_write(wreq);
 
@@ -818,6 +830,9 @@ static int netfs_write_folio_single(struct netfs_io_request *wreq,
  *
  * Write a monolithic, non-pagecache object back to the server and/or
  * the cache.
+ *
+ * Return: 0 if successful; 1 if skipped due to lock conflict and WB_SYNC_NONE;
+ * or a negative error code.
  */
 int netfs_writeback_single(struct address_space *mapping,
 			   struct writeback_control *wbc,
@@ -834,8 +849,10 @@ int netfs_writeback_single(struct address_space *mapping,
 
 	if (!mutex_trylock(&ictx->wb_lock)) {
 		if (wbc->sync_mode == WB_SYNC_NONE) {
+			/* The VFS will have undirtied the inode. */
+			netfs_single_mark_inode_dirty(&ictx->inode);
 			netfs_stat(&netfs_n_wb_lock_skip);
-			return 0;
+			return 1;
 		}
 		netfs_stat(&netfs_n_wb_lock_wait);
 		mutex_lock(&ictx->wb_lock);
diff --git a/fs/netfs/write_retry.c b/fs/netfs/write_retry.c
index 29489a23a2209..32735abfa03f0 100644
--- a/fs/netfs/write_retry.c
+++ b/fs/netfs/write_retry.c
@@ -130,7 +130,9 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
 			list_for_each_entry_safe_from(subreq, tmp,
 						      &stream->subrequests, rreq_link) {
 				trace_netfs_sreq(subreq, netfs_sreq_trace_discard);
+				spin_lock(&wreq->lock);
 				list_del(&subreq->rreq_link);
+				spin_unlock(&wreq->lock);
 				netfs_put_subrequest(subreq, netfs_sreq_trace_put_done);
 				if (subreq == to)
 					break;
@@ -153,8 +155,10 @@ static void netfs_retry_write_stream(struct netfs_io_request *wreq,
 					     netfs_sreq_trace_new);
 			trace_netfs_sreq(subreq, netfs_sreq_trace_split);
 
+			spin_lock(&wreq->lock);
 			list_add(&subreq->rreq_link, &to->rreq_link);
-			to = list_next_entry(to, rreq_link);
+			spin_unlock(&wreq->lock);
+			to = subreq;
 			trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
 
 			stream->sreq_max_len	= len;
diff --git a/fs/nfs/client.c b/fs/nfs/client.c
index be02bb227741d..3db2f18315b87 100644
--- a/fs/nfs/client.c
+++ b/fs/nfs/client.c
@@ -914,6 +914,7 @@ static void nfs_server_set_fsinfo(struct nfs_server *server,
  */
 static int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *mntfh, struct nfs_fattr *fattr)
 {
+	struct nfs_pathconf pathinfo = { };
 	struct nfs_fsinfo fsinfo;
 	struct nfs_client *clp = server->nfs_client;
 	int error;
@@ -933,15 +934,23 @@ static int nfs_probe_fsinfo(struct nfs_server *server, struct nfs_fh *mntfh, str
 
 	nfs_server_set_fsinfo(server, &fsinfo);
 
-	/* Get some general file system info */
-	if (server->namelen == 0) {
-		struct nfs_pathconf pathinfo;
+	pathinfo.fattr = fattr;
+	nfs_fattr_init(fattr);
 
-		pathinfo.fattr = fattr;
-		nfs_fattr_init(fattr);
+	/* Clear before probing so a failed RPC does not retain stale bits. */
+	if (clp->rpc_ops->version < 4)
+		server->caps &= ~(NFS_CAP_CASE_INSENSITIVE |
+				  NFS_CAP_CASE_NONPRESERVING);
 
-		if (clp->rpc_ops->pathconf(server, mntfh, &pathinfo) >= 0)
+	if (clp->rpc_ops->pathconf(server, mntfh, &pathinfo) >= 0) {
+		if (server->namelen == 0)
 			server->namelen = pathinfo.max_namelen;
+		if (clp->rpc_ops->version < 4) {
+			if (pathinfo.case_insensitive)
+				server->caps |= NFS_CAP_CASE_INSENSITIVE;
+			if (!pathinfo.case_preserving)
+				server->caps |= NFS_CAP_CASE_NONPRESERVING;
+		}
 	}
 
 	if (clp->rpc_ops->discover_trunking != NULL &&
diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c
index e9ce1883288c5..7a1e65953f3aa 100644
--- a/fs/nfs/dir.c
+++ b/fs/nfs/dir.c
@@ -2194,6 +2194,10 @@ int nfs_atomic_open(struct inode *dir, struct dentry *dentry,
 			break;
 		case -EISDIR:
 		case -ENOTDIR:
+			if (open_flags & OPENAT2_REGULAR) {
+				err = -EFTYPE;
+				break;
+			}
 			goto no_open;
 		case -ELOOP:
 			if (!(open_flags & O_NOFOLLOW))
diff --git a/fs/nfs/dns_resolve.c b/fs/nfs/dns_resolve.c
index 2ed2126201f41..acd3511c04a6c 100644
--- a/fs/nfs/dns_resolve.c
+++ b/fs/nfs/dns_resolve.c
@@ -156,7 +156,7 @@ static int nfs_dns_upcall(struct cache_detail *cd,
 	if (!nfs_cache_upcall(cd, key->hostname))
 		return 0;
 	clear_bit(CACHE_PENDING, &ch->flags);
-	return sunrpc_cache_pipe_upcall_timeout(cd, ch);
+	return sunrpc_cache_upcall_warn(cd, ch);
 }
 
 static int nfs_dns_match(struct cache_head *ca,
diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index e26030e736966..6227df9ae6f1d 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -41,6 +41,7 @@
 #include <linux/freezer.h>
 #include <linux/uaccess.h>
 #include <linux/iversion.h>
+#include <linux/fileattr.h>
 
 #include "nfs4_fs.h"
 #include "callback.h"
@@ -608,7 +609,7 @@ nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr)
 		inode->i_sb->s_id,
 		(unsigned long long)NFS_FILEID(inode),
 		nfs_display_fhandle_hash(fh),
-		icount_read(inode));
+		icount_read_once(inode));
 
 out:
 	return inode;
@@ -1095,6 +1096,20 @@ int nfs_getattr(struct mnt_idmap *idmap, const struct path *path,
 }
 EXPORT_SYMBOL_GPL(nfs_getattr);
 
+int nfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
+{
+	struct inode *inode = d_inode(dentry);
+
+	if (nfs_server_capable(inode, NFS_CAP_CASE_INSENSITIVE)) {
+		fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
+		fa->flags |= FS_CASEFOLD_FL;
+	}
+	if (nfs_server_capable(inode, NFS_CAP_CASE_NONPRESERVING))
+		fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING;
+	return 0;
+}
+EXPORT_SYMBOL_GPL(nfs_fileattr_get);
+
 static void nfs_init_lock_context(struct nfs_lock_context *l_ctx)
 {
 	refcount_set(&l_ctx->count, 1);
@@ -2255,7 +2270,7 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr)
 	dfprintk(VFS, "NFS: %s(%s/%llu fh_crc=0x%08x ct=%d info=0x%llx)\n",
 			__func__, inode->i_sb->s_id, inode->i_ino,
 			nfs_display_fhandle_hash(NFS_FH(inode)),
-			icount_read(inode), fattr->valid);
+			icount_read_once(inode), fattr->valid);
 
 	if (!(fattr->valid & NFS_ATTR_FATTR_FILEID)) {
 		/* Only a mounted-on-fileid? Just exit */
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 18d46b0e71ddc..ec2b3d9843989 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -451,6 +451,9 @@ extern void nfs_set_cache_invalid(struct inode *inode, unsigned long flags);
 extern bool nfs_check_cache_invalid(struct inode *, unsigned long);
 extern int nfs_wait_bit_killable(struct wait_bit_key *key, int mode);
 
+struct file_kattr;
+int nfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
+
 #if IS_ENABLED(CONFIG_NFS_LOCALIO)
 /* localio.c */
 struct nfs_local_dio {
diff --git a/fs/nfs/namespace.c b/fs/nfs/namespace.c
index af9be0c5f5163..6d0073c24771b 100644
--- a/fs/nfs/namespace.c
+++ b/fs/nfs/namespace.c
@@ -246,11 +246,13 @@ nfs_namespace_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
 const struct inode_operations nfs_mountpoint_inode_operations = {
 	.getattr	= nfs_getattr,
 	.setattr	= nfs_setattr,
+	.fileattr_get	= nfs_fileattr_get,
 };
 
 const struct inode_operations nfs_referral_inode_operations = {
 	.getattr	= nfs_namespace_getattr,
 	.setattr	= nfs_namespace_setattr,
+	.fileattr_get	= nfs_fileattr_get,
 };
 
 static void nfs_expire_automounts(struct work_struct *work)
diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c
index 95d7cd564b746..b80d0c5efc279 100644
--- a/fs/nfs/nfs3proc.c
+++ b/fs/nfs/nfs3proc.c
@@ -1053,6 +1053,7 @@ static const struct inode_operations nfs3_dir_inode_operations = {
 	.permission	= nfs_permission,
 	.getattr	= nfs_getattr,
 	.setattr	= nfs_setattr,
+	.fileattr_get	= nfs_fileattr_get,
 #ifdef CONFIG_NFS_V3_ACL
 	.listxattr	= nfs3_listxattr,
 	.get_inode_acl	= nfs3_get_acl,
@@ -1064,6 +1065,7 @@ static const struct inode_operations nfs3_file_inode_operations = {
 	.permission	= nfs_permission,
 	.getattr	= nfs_getattr,
 	.setattr	= nfs_setattr,
+	.fileattr_get	= nfs_fileattr_get,
 #ifdef CONFIG_NFS_V3_ACL
 	.listxattr	= nfs3_listxattr,
 	.get_inode_acl	= nfs3_get_acl,
diff --git a/fs/nfs/nfs3xdr.c b/fs/nfs/nfs3xdr.c
index e17d729084125..e745e78faab0a 100644
--- a/fs/nfs/nfs3xdr.c
+++ b/fs/nfs/nfs3xdr.c
@@ -2276,8 +2276,11 @@ static int decode_pathconf3resok(struct xdr_stream *xdr,
 	if (unlikely(!p))
 		return -EIO;
 	result->max_link = be32_to_cpup(p++);
-	result->max_namelen = be32_to_cpup(p);
-	/* ignore remaining fields */
+	result->max_namelen = be32_to_cpup(p++);
+	p++;	/* ignore no_trunc */
+	p++;	/* ignore chown_restricted */
+	result->case_insensitive = be32_to_cpup(p++) != 0;
+	result->case_preserving = be32_to_cpup(p) != 0;
 	return 0;
 }
 
diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c
index a9b8d482d2894..c48281db38689 100644
--- a/fs/nfs/nfs4proc.c
+++ b/fs/nfs/nfs4proc.c
@@ -3933,7 +3933,8 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f
 		server->caps &=
 			~(NFS_CAP_ACLS | NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS |
 			  NFS_CAP_SECURITY_LABEL | NFS_CAP_FS_LOCATIONS |
-			  NFS_CAP_OPEN_XOR | NFS_CAP_DELEGTIME);
+			  NFS_CAP_OPEN_XOR | NFS_CAP_DELEGTIME |
+			  NFS_CAP_CASE_INSENSITIVE | NFS_CAP_CASE_NONPRESERVING);
 		server->fattr_valid = NFS_ATTR_FATTR_V4;
 		if (res.attr_bitmask[0] & FATTR4_WORD0_ACL &&
 				res.acl_bitmask & ACL4_SUPPORT_ALLOW_ACL)
@@ -3944,8 +3945,9 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f
 			server->caps |= NFS_CAP_SYMLINKS;
 		if (res.case_insensitive)
 			server->caps |= NFS_CAP_CASE_INSENSITIVE;
-		if (res.case_preserving)
-			server->caps |= NFS_CAP_CASE_PRESERVING;
+		if ((res.attr_bitmask[0] & FATTR4_WORD0_CASE_PRESERVING) &&
+		    !res.case_preserving)
+			server->caps |= NFS_CAP_CASE_NONPRESERVING;
 #ifdef CONFIG_NFS_V4_SECURITY_LABEL
 		if (res.attr_bitmask[2] & FATTR4_WORD2_SECURITY_LABEL)
 			server->caps |= NFS_CAP_SECURITY_LABEL;
@@ -10562,13 +10564,10 @@ static ssize_t nfs4_listxattr(struct dentry *dentry, char *list, size_t size)
 		left -= error;
 	}
 
-	error2 = security_inode_listsecurity(d_inode(dentry), list, left);
+	error2 = security_inode_listsecurity(d_inode(dentry), &list, &left);
 	if (error2 < 0)
 		return error2;
-	if (list) {
-		list += error2;
-		left -= error2;
-	}
+	error2 = size - error - left;
 
 	error3 = nfs4_listxattr_nfs4_user(d_inode(dentry), list, left);
 	if (error3 < 0)
@@ -10617,6 +10616,7 @@ static const struct inode_operations nfs4_dir_inode_operations = {
 	.getattr	= nfs_getattr,
 	.setattr	= nfs_setattr,
 	.listxattr	= nfs4_listxattr,
+	.fileattr_get	= nfs_fileattr_get,
 };
 
 static const struct inode_operations nfs4_file_inode_operations = {
@@ -10624,6 +10624,7 @@ static const struct inode_operations nfs4_file_inode_operations = {
 	.getattr	= nfs_getattr,
 	.setattr	= nfs_setattr,
 	.listxattr	= nfs4_listxattr,
+	.fileattr_get	= nfs_fileattr_get,
 };
 
 static struct nfs_server *nfs4_clone_server(struct nfs_server *source,
diff --git a/fs/nfs/proc.c b/fs/nfs/proc.c
index 70795684b8e84..03c2c1f31be9a 100644
--- a/fs/nfs/proc.c
+++ b/fs/nfs/proc.c
@@ -598,6 +598,7 @@ nfs_proc_pathconf(struct nfs_server *server, struct nfs_fh *fhandle,
 {
 	info->max_link = 0;
 	info->max_namelen = NFS2_MAXNAMLEN;
+	info->case_preserving = true;
 	return 0;
 }
 
@@ -718,12 +719,14 @@ static const struct inode_operations nfs_dir_inode_operations = {
 	.permission	= nfs_permission,
 	.getattr	= nfs_getattr,
 	.setattr	= nfs_setattr,
+	.fileattr_get	= nfs_fileattr_get,
 };
 
 static const struct inode_operations nfs_file_inode_operations = {
 	.permission	= nfs_permission,
 	.getattr	= nfs_getattr,
 	.setattr	= nfs_setattr,
+	.fileattr_get	= nfs_fileattr_get,
 };
 
 const struct nfs_rpc_ops nfs_v2_clientops = {
diff --git a/fs/nfs/symlink.c b/fs/nfs/symlink.c
index 58146e9354020..74a072896f8d9 100644
--- a/fs/nfs/symlink.c
+++ b/fs/nfs/symlink.c
@@ -22,6 +22,8 @@
 #include <linux/mm.h>
 #include <linux/string.h>
 
+#include "internal.h"
+
 /* Symlink caching in the page cache is even more simplistic
  * and straight-forward than readdir caching.
  */
@@ -74,4 +76,5 @@ const struct inode_operations nfs_symlink_inode_operations = {
 	.get_link	= nfs_get_link,
 	.getattr	= nfs_getattr,
 	.setattr	= nfs_setattr,
+	.fileattr_get	= nfs_fileattr_get,
 };
diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c
index 9d829c84f374e..5be7721c22c23 100644
--- a/fs/nfsd/blocklayout.c
+++ b/fs/nfsd/blocklayout.c
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2014-2016 Christoph Hellwig.
  */
-#include <linux/exportfs.h>
+#include <linux/exportfs_block.h>
 #include <linux/iomap.h>
 #include <linux/slab.h>
 #include <linux/pr.h>
@@ -32,8 +32,8 @@ nfsd4_block_map_extent(struct inode *inode, const struct svc_fh *fhp,
 	u32 device_generation = 0;
 	int error;
 
-	error = sb->s_export_op->map_blocks(inode, offset, length, &iomap,
-			iomode != IOMODE_READ, &device_generation);
+	error = sb->s_export_op->block_ops->map_blocks(inode, offset, length,
+			&iomap, iomode != IOMODE_READ, &device_generation);
 	if (error) {
 		if (error == -ENXIO)
 			return nfserr_layoutunavailable;
@@ -179,23 +179,20 @@ static __be32
 nfsd4_block_commit_blocks(struct inode *inode, struct nfsd4_layoutcommit *lcp,
 		struct iomap *iomaps, int nr_iomaps)
 {
-	struct timespec64 mtime = inode_get_mtime(inode);
-	struct iattr iattr = { .ia_valid = 0 };
 	int error;
 
-	if (lcp->lc_mtime.tv_nsec == UTIME_NOW ||
-	    timespec64_compare(&lcp->lc_mtime, &mtime) < 0)
-		lcp->lc_mtime = current_time(inode);
-	iattr.ia_valid |= ATTR_ATIME | ATTR_CTIME | ATTR_MTIME;
-	iattr.ia_atime = iattr.ia_ctime = iattr.ia_mtime = lcp->lc_mtime;
-
-	if (lcp->lc_size_chg) {
-		iattr.ia_valid |= ATTR_SIZE;
-		iattr.ia_size = lcp->lc_newsize;
-	}
-
-	error = inode->i_sb->s_export_op->commit_blocks(inode, iomaps,
-			nr_iomaps, &iattr);
+	/*
+	 * This ignores the client provided mtime in loca_time_modify, as a
+	 * fully client specified mtime doesn't really fit into the Linux
+	 * multi-grain timestamp architecture.
+	 *
+	 * RFC 8881 Section 18.42 makes it clear that the client provided
+	 * timestamp is a "may" condition, and clients that want to force a
+	 * specific timestamp should send a separate SETATTR in the compound.
+	 */
+	error = inode->i_sb->s_export_op->block_ops->commit_blocks(inode,
+			iomaps, nr_iomaps,
+			lcp->lc_size_chg ? lcp->lc_newsize : 0);
 	kfree(iomaps);
 	return nfserrno(error);
 }
@@ -218,8 +215,8 @@ nfsd4_block_get_device_info_simple(struct super_block *sb,
 
 	b->type = PNFS_BLOCK_VOLUME_SIMPLE;
 	b->simple.sig_len = PNFS_BLOCK_UUID_LEN;
-	return sb->s_export_op->get_uuid(sb, b->simple.sig, &b->simple.sig_len,
-			&b->simple.offset);
+	return sb->s_export_op->block_ops->get_uuid(sb, b->simple.sig,
+			&b->simple.sig_len, &b->simple.offset);
 }
 
 static __be32
diff --git a/fs/nfsd/export.c b/fs/nfsd/export.c
index 665153f1720e0..eb020054f9a3f 100644
--- a/fs/nfsd/export.c
+++ b/fs/nfsd/export.c
@@ -17,6 +17,8 @@
 #include <linux/module.h>
 #include <linux/exportfs.h>
 #include <linux/sunrpc/svc_xprt.h>
+#include <net/genetlink.h>
+#include <uapi/linux/nfsd_netlink.h>
 
 #include "nfsd.h"
 #include "nfsfh.h"
@@ -24,6 +26,7 @@
 #include "pnfs.h"
 #include "filecache.h"
 #include "trace.h"
+#include "netlink.h"
 
 #define NFSDDBG_FACILITY	NFSDDBG_EXPORT
 
@@ -64,7 +67,7 @@ static void expkey_put(struct kref *ref)
 
 static int expkey_upcall(struct cache_detail *cd, struct cache_head *h)
 {
-	return sunrpc_cache_pipe_upcall(cd, h);
+	return sunrpc_cache_upcall(cd, h);
 }
 
 static void expkey_request(struct cache_detail *cd,
@@ -263,12 +266,18 @@ static void expkey_flush(void)
 	mutex_unlock(&nfsd_mutex);
 }
 
+static int expkey_notify(struct cache_detail *cd, struct cache_head *h)
+{
+	return nfsd_cache_notify(cd, h, NFSD_CACHE_TYPE_EXPKEY);
+}
+
 static const struct cache_detail svc_expkey_cache_template = {
 	.owner		= THIS_MODULE,
 	.hash_size	= EXPKEY_HASHMAX,
 	.name		= "nfsd.fh",
 	.cache_put	= expkey_put,
 	.cache_upcall	= expkey_upcall,
+	.cache_notify	= expkey_notify,
 	.cache_request	= expkey_request,
 	.cache_parse	= expkey_parse,
 	.cache_show	= expkey_show,
@@ -319,6 +328,266 @@ svc_expkey_update(struct cache_detail *cd, struct svc_expkey *new,
 		return NULL;
 }
 
+/**
+ * nfsd_nl_expkey_get_reqs_dumpit - dump pending expkey requests
+ * @skb: reply buffer
+ * @cb: netlink metadata and command arguments
+ *
+ * Walk the expkey cache's pending request list and create a netlink
+ * message with a nested entry for each cache_request, containing the
+ * seqno, client string, fsidtype and fsid.
+ *
+ * Uses cb->args[0] as a seqno cursor for dump continuation across
+ * multiple netlink messages.
+ *
+ * Returns the size of the reply or a negative errno.
+ */
+int nfsd_nl_expkey_get_reqs_dumpit(struct sk_buff *skb,
+				   struct netlink_callback *cb)
+{
+	struct nfsd_net *nn;
+	struct cache_detail *cd;
+	struct cache_head **items;
+	u64 *seqnos;
+	int cnt, i, emitted;
+	void *hdr;
+	int ret;
+
+	nn = net_generic(sock_net(skb->sk), nfsd_net_id);
+
+	mutex_lock(&nfsd_mutex);
+
+	cd = nn->svc_expkey_cache;
+	if (!cd) {
+		ret = -ENODEV;
+		goto out_unlock;
+	}
+
+	cnt = sunrpc_cache_requests_count(cd);
+	if (!cnt) {
+		ret = 0;
+		goto out_unlock;
+	}
+
+	items = kcalloc(cnt, sizeof(*items), GFP_KERNEL);
+	seqnos = kcalloc(cnt, sizeof(*seqnos), GFP_KERNEL);
+	if (!items || !seqnos) {
+		ret = -ENOMEM;
+		goto out_alloc;
+	}
+
+	cnt = sunrpc_cache_requests_snapshot(cd, items, seqnos, cnt,
+					     cb->args[0]);
+	if (!cnt) {
+		ret = 0;
+		goto out_alloc;
+	}
+
+	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			  cb->nlh->nlmsg_seq, &nfsd_nl_family,
+			  NLM_F_MULTI, NFSD_CMD_EXPKEY_GET_REQS);
+	if (!hdr) {
+		ret = -ENOBUFS;
+		goto out_put;
+	}
+
+	emitted = 0;
+	for (i = 0; i < cnt; i++) {
+		struct svc_expkey *ek;
+		struct nlattr *nest;
+
+		ek = container_of(items[i], struct svc_expkey, h);
+
+		nest = nla_nest_start(skb, NFSD_A_EXPKEY_REQS_REQUESTS);
+		if (!nest)
+			break;
+
+		if (nla_put_u64_64bit(skb, NFSD_A_EXPKEY_SEQNO,
+				      seqnos[i], 0) ||
+		    nla_put_string(skb, NFSD_A_EXPKEY_CLIENT,
+				   ek->ek_client->name) ||
+		    nla_put_u8(skb, NFSD_A_EXPKEY_FSIDTYPE,
+			       ek->ek_fsidtype) ||
+		    nla_put(skb, NFSD_A_EXPKEY_FSID,
+			    key_len(ek->ek_fsidtype), ek->ek_fsid)) {
+			nla_nest_cancel(skb, nest);
+			break;
+		}
+
+		nla_nest_end(skb, nest);
+		cb->args[0] = seqnos[i];
+		emitted++;
+	}
+
+	if (!emitted) {
+		genlmsg_cancel(skb, hdr);
+		ret = -EMSGSIZE;
+		goto out_put;
+	}
+
+	genlmsg_end(skb, hdr);
+	ret = skb->len;
+out_put:
+	for (i = 0; i < cnt; i++)
+		cache_put(items[i], cd);
+out_alloc:
+	kfree(seqnos);
+	kfree(items);
+out_unlock:
+	mutex_unlock(&nfsd_mutex);
+	return ret;
+}
+
+/**
+ * nfsd_nl_parse_one_expkey - parse one expkey entry from netlink
+ * @cd: cache_detail for the expkey cache
+ * @attr: nested attribute containing expkey fields
+ *
+ * Parses one expkey entry from a netlink message and updates the
+ * cache. Mirrors the logic in expkey_parse().
+ *
+ * Returns 0 on success or a negative errno.
+ */
+static int nfsd_nl_parse_one_expkey(struct cache_detail *cd,
+				    struct nlattr *attr)
+{
+	struct nlattr *tb[NFSD_A_EXPKEY_PATH + 1];
+	struct auth_domain *dom = NULL;
+	struct svc_expkey key;
+	struct svc_expkey *ek = NULL;
+	struct timespec64 boot;
+	int err;
+	u8 fsidtype;
+	int fsid_len;
+
+	err = nla_parse_nested(tb, NFSD_A_EXPKEY_PATH, attr,
+			       nfsd_expkey_nl_policy, NULL);
+	if (err)
+		return err;
+
+	/* client (required) */
+	if (!tb[NFSD_A_EXPKEY_CLIENT])
+		return -EINVAL;
+
+	dom = auth_domain_find(nla_data(tb[NFSD_A_EXPKEY_CLIENT]));
+	if (!dom)
+		return -ENOENT;
+
+	/* fsidtype (required) */
+	if (!tb[NFSD_A_EXPKEY_FSIDTYPE]) {
+		err = -EINVAL;
+		goto out_dom;
+	}
+	fsidtype = nla_get_u8(tb[NFSD_A_EXPKEY_FSIDTYPE]);
+	if (key_len(fsidtype) == 0) {
+		err = -EINVAL;
+		goto out_dom;
+	}
+
+	/* fsid (required) */
+	if (!tb[NFSD_A_EXPKEY_FSID]) {
+		err = -EINVAL;
+		goto out_dom;
+	}
+	fsid_len = nla_len(tb[NFSD_A_EXPKEY_FSID]);
+	if (fsid_len != key_len(fsidtype)) {
+		err = -EINVAL;
+		goto out_dom;
+	}
+
+	/* expiry (required, wallclock seconds) */
+	if (!tb[NFSD_A_EXPKEY_EXPIRY]) {
+		err = -EINVAL;
+		goto out_dom;
+	}
+
+	key.h.flags = 0;
+	getboottime64(&boot);
+	key.h.expiry_time = nla_get_u64(tb[NFSD_A_EXPKEY_EXPIRY]) -
+			    boot.tv_sec;
+	key.ek_client = dom;
+	key.ek_fsidtype = fsidtype;
+	memcpy(key.ek_fsid, nla_data(tb[NFSD_A_EXPKEY_FSID]), fsid_len);
+
+	ek = svc_expkey_lookup(cd, &key);
+	if (!ek) {
+		err = -ENOMEM;
+		goto out_dom;
+	}
+
+	if (tb[NFSD_A_EXPKEY_NEGATIVE]) {
+		set_bit(CACHE_NEGATIVE, &key.h.flags);
+		ek = svc_expkey_update(cd, &key, ek);
+		if (ek)
+			trace_nfsd_expkey_update(ek, NULL);
+		else
+			err = -ENOMEM;
+	} else if (tb[NFSD_A_EXPKEY_PATH]) {
+		err = kern_path(nla_data(tb[NFSD_A_EXPKEY_PATH]), 0,
+				&key.ek_path);
+		if (err)
+			goto out_ek;
+		ek = svc_expkey_update(cd, &key, ek);
+		if (ek)
+			trace_nfsd_expkey_update(ek,
+					nla_data(tb[NFSD_A_EXPKEY_PATH]));
+		else
+			err = -ENOMEM;
+		path_put(&key.ek_path);
+	} else {
+		err = -EINVAL;
+		goto out_ek;
+	}
+
+	cache_flush();
+
+out_ek:
+	if (ek)
+		cache_put(&ek->h, cd);
+out_dom:
+	auth_domain_put(dom);
+	return err;
+}
+
+/**
+ * nfsd_nl_expkey_set_reqs_doit - respond to expkey requests
+ * @skb: reply buffer
+ * @info: netlink metadata and command arguments
+ *
+ * Parse one or more expkey cache responses from userspace and
+ * update the expkey cache accordingly.
+ *
+ * Returns 0 on success or a negative errno.
+ */
+int nfsd_nl_expkey_set_reqs_doit(struct sk_buff *skb,
+				 struct genl_info *info)
+{
+	struct nfsd_net *nn;
+	struct cache_detail *cd;
+	const struct nlattr *attr;
+	int rem, ret = 0;
+
+	nn = net_generic(genl_info_net(info), nfsd_net_id);
+
+	mutex_lock(&nfsd_mutex);
+
+	cd = nn->svc_expkey_cache;
+	if (!cd) {
+		ret = -ENODEV;
+		goto out_unlock;
+	}
+
+	nlmsg_for_each_attr_type(attr, NFSD_A_EXPKEY_REQS_REQUESTS,
+				 info->nlhdr, GENL_HDRLEN, rem) {
+		ret = nfsd_nl_parse_one_expkey(cd, (struct nlattr *)attr);
+		if (ret)
+			break;
+	}
+
+out_unlock:
+	mutex_unlock(&nfsd_mutex);
+	return ret;
+}
 
 #define	EXPORT_HASHBITS		8
 #define	EXPORT_HASHMAX		(1<< EXPORT_HASHBITS)
@@ -386,9 +655,445 @@ static void svc_export_put(struct kref *ref)
 	queue_rcu_work(nfsd_export_wq, &exp->ex_rwork);
 }
 
+/**
+ * nfsd_nl_svc_export_get_reqs_dumpit - dump pending svc_export requests
+ * @skb: reply buffer
+ * @cb: netlink metadata and command arguments
+ *
+ * Walk the svc_export cache's pending request list and create a netlink
+ * message with a nested entry for each cache_request, containing the
+ * seqno, client string, and path.
+ *
+ * Uses cb->args[0] as a seqno cursor for dump continuation across
+ * multiple netlink messages.
+ *
+ * Returns the size of the reply or a negative errno.
+ */
+int nfsd_nl_svc_export_get_reqs_dumpit(struct sk_buff *skb,
+				       struct netlink_callback *cb)
+{
+	struct nfsd_net *nn;
+	struct cache_detail *cd;
+	struct cache_head **items;
+	u64 *seqnos;
+	int cnt, i, emitted;
+	char *pathbuf;
+	void *hdr;
+	int ret;
+
+	nn = net_generic(sock_net(skb->sk), nfsd_net_id);
+
+	mutex_lock(&nfsd_mutex);
+
+	cd = nn->svc_export_cache;
+	if (!cd) {
+		ret = -ENODEV;
+		goto out_unlock;
+	}
+
+	cnt = sunrpc_cache_requests_count(cd);
+	if (!cnt) {
+		ret = 0;
+		goto out_unlock;
+	}
+
+	items = kcalloc(cnt, sizeof(*items), GFP_KERNEL);
+	seqnos = kcalloc(cnt, sizeof(*seqnos), GFP_KERNEL);
+	pathbuf = kmalloc(PATH_MAX, GFP_KERNEL);
+	if (!items || !seqnos || !pathbuf) {
+		ret = -ENOMEM;
+		goto out_alloc;
+	}
+
+	cnt = sunrpc_cache_requests_snapshot(cd, items, seqnos, cnt,
+					     cb->args[0]);
+	if (!cnt) {
+		ret = 0;
+		goto out_alloc;
+	}
+
+	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			  cb->nlh->nlmsg_seq, &nfsd_nl_family,
+			  NLM_F_MULTI, NFSD_CMD_SVC_EXPORT_GET_REQS);
+	if (!hdr) {
+		ret = -ENOBUFS;
+		goto out_put;
+	}
+
+	emitted = 0;
+	for (i = 0; i < cnt; i++) {
+		struct svc_export *exp;
+		struct nlattr *nest;
+		char *pth;
+
+		exp = container_of(items[i], struct svc_export, h);
+
+		pth = d_path(&exp->ex_path, pathbuf, PATH_MAX);
+		if (IS_ERR(pth))
+			continue;
+
+		nest = nla_nest_start(skb,
+				      NFSD_A_SVC_EXPORT_REQS_REQUESTS);
+		if (!nest)
+			break;
+
+		if (nla_put_u64_64bit(skb, NFSD_A_SVC_EXPORT_SEQNO,
+				      seqnos[i], 0) ||
+		    nla_put_string(skb, NFSD_A_SVC_EXPORT_CLIENT,
+				   exp->ex_client->name) ||
+		    nla_put_string(skb, NFSD_A_SVC_EXPORT_PATH, pth)) {
+			nla_nest_cancel(skb, nest);
+			break;
+		}
+
+		nla_nest_end(skb, nest);
+		cb->args[0] = seqnos[i];
+		emitted++;
+	}
+
+	if (!emitted) {
+		genlmsg_cancel(skb, hdr);
+		ret = -EMSGSIZE;
+		goto out_put;
+	}
+
+	genlmsg_end(skb, hdr);
+	ret = skb->len;
+out_put:
+	for (i = 0; i < cnt; i++)
+		cache_put(items[i], cd);
+out_alloc:
+	kfree(pathbuf);
+	kfree(seqnos);
+	kfree(items);
+out_unlock:
+	mutex_unlock(&nfsd_mutex);
+	return ret;
+}
+
+/**
+ * nfsd_nl_parse_fslocations - parse fslocations from netlink
+ * @attr: NFSD_A_SVC_EXPORT_FSLOCATIONS nested attribute
+ * @fsloc: fslocations struct to fill in
+ *
+ * Returns 0 on success or a negative errno.
+ */
+static int nfsd_nl_parse_fslocations(struct nlattr *attr,
+				     struct nfsd4_fs_locations *fsloc)
+{
+	struct nlattr *loc_attr;
+	int rem, count = 0;
+	int err;
+
+	if (fsloc->locations)
+		return -EINVAL;
+
+	/* Count locations first */
+	nla_for_each_nested_type(loc_attr, NFSD_A_FSLOCATIONS_LOCATION,
+				 attr, rem)
+		count++;
+
+	if (count > MAX_FS_LOCATIONS)
+		return -EINVAL;
+	if (!count)
+		return 0;
+
+	fsloc->locations = kcalloc(count, sizeof(struct nfsd4_fs_location),
+				   GFP_KERNEL);
+	if (!fsloc->locations)
+		return -ENOMEM;
+
+	nla_for_each_nested_type(loc_attr, NFSD_A_FSLOCATIONS_LOCATION,
+				 attr, rem) {
+		struct nlattr *tb[NFSD_A_FSLOCATION_PATH + 1];
+		struct nfsd4_fs_location *loc;
+
+		err = nla_parse_nested(tb, NFSD_A_FSLOCATION_PATH, loc_attr,
+				       nfsd_fslocation_nl_policy, NULL);
+		if (err)
+			goto out_free;
+
+		if (!tb[NFSD_A_FSLOCATION_HOST] ||
+		    !tb[NFSD_A_FSLOCATION_PATH]) {
+			err = -EINVAL;
+			goto out_free;
+		}
+
+		loc = &fsloc->locations[fsloc->locations_count++];
+		loc->hosts = kstrdup(nla_data(tb[NFSD_A_FSLOCATION_HOST]),
+				     GFP_KERNEL);
+		loc->path = kstrdup(nla_data(tb[NFSD_A_FSLOCATION_PATH]),
+				    GFP_KERNEL);
+		if (!loc->hosts || !loc->path) {
+			err = -ENOMEM;
+			goto out_free;
+		}
+	}
+
+	return 0;
+out_free:
+	nfsd4_fslocs_free(fsloc);
+	return err;
+}
+
+static struct svc_export *svc_export_update(struct svc_export *new,
+					    struct svc_export *old);
+static struct svc_export *svc_export_lookup(struct svc_export *);
+static int check_export(const struct path *path, int *flags,
+			unsigned char *uuid);
+
+/**
+ * nfsd_nl_parse_one_export - parse one svc_export entry from a netlink message
+ * @cd: cache_detail for the svc_export cache
+ * @attr: nested attribute containing svc-export fields
+ *
+ * Parses one svc-export entry from a netlink message and updates the
+ * cache. Mirrors the logic in svc_export_parse().
+ *
+ * Returns 0 on success or a negative errno.
+ */
+static int nfsd_nl_parse_one_export(struct cache_detail *cd,
+				    struct nlattr *attr)
+{
+	struct nlattr *tb[NFSD_A_SVC_EXPORT_FSID + 1];
+	struct auth_domain *dom = NULL;
+	struct svc_export exp = {}, *expp;
+	struct nlattr *secinfo_attr;
+	struct timespec64 boot;
+	int err, rem;
+
+	err = nla_parse_nested(tb, NFSD_A_SVC_EXPORT_FSID, attr,
+			       nfsd_svc_export_nl_policy, NULL);
+	if (err)
+		return err;
+
+	/* client (required) */
+	if (!tb[NFSD_A_SVC_EXPORT_CLIENT])
+		return -EINVAL;
+
+	dom = auth_domain_find(nla_data(tb[NFSD_A_SVC_EXPORT_CLIENT]));
+	if (!dom)
+		return -ENOENT;
+
+	/* path (required) */
+	if (!tb[NFSD_A_SVC_EXPORT_PATH]) {
+		err = -EINVAL;
+		goto out_dom;
+	}
+
+	err = kern_path(nla_data(tb[NFSD_A_SVC_EXPORT_PATH]), 0,
+			&exp.ex_path);
+	if (err)
+		goto out_dom;
+
+	exp.ex_client = dom;
+	exp.cd = cd;
+	exp.ex_devid_map = NULL;
+	exp.ex_xprtsec_modes = NFSEXP_XPRTSEC_ALL;
+
+	/* expiry (required, wallclock seconds) */
+	if (!tb[NFSD_A_SVC_EXPORT_EXPIRY]) {
+		err = -EINVAL;
+		goto out_path;
+	}
+	getboottime64(&boot);
+	exp.h.expiry_time = nla_get_u64(tb[NFSD_A_SVC_EXPORT_EXPIRY]) -
+			    boot.tv_sec;
+
+	if (tb[NFSD_A_SVC_EXPORT_NEGATIVE]) {
+		set_bit(CACHE_NEGATIVE, &exp.h.flags);
+	} else {
+		/* flags */
+		if (tb[NFSD_A_SVC_EXPORT_FLAGS])
+			exp.ex_flags = nla_get_u32(tb[NFSD_A_SVC_EXPORT_FLAGS]);
+
+		/* anon uid */
+		if (tb[NFSD_A_SVC_EXPORT_ANON_UID]) {
+			u32 uid = nla_get_u32(tb[NFSD_A_SVC_EXPORT_ANON_UID]);
+
+			exp.ex_anon_uid = make_kuid(current_user_ns(), uid);
+		}
+
+		/* anon gid */
+		if (tb[NFSD_A_SVC_EXPORT_ANON_GID]) {
+			u32 gid = nla_get_u32(tb[NFSD_A_SVC_EXPORT_ANON_GID]);
+
+			exp.ex_anon_gid = make_kgid(current_user_ns(), gid);
+		}
+
+		/* fsid */
+		if (tb[NFSD_A_SVC_EXPORT_FSID])
+			exp.ex_fsid = nla_get_s32(tb[NFSD_A_SVC_EXPORT_FSID]);
+
+		/* fslocations */
+		if (tb[NFSD_A_SVC_EXPORT_FSLOCATIONS]) {
+			struct nlattr *fsl = tb[NFSD_A_SVC_EXPORT_FSLOCATIONS];
+
+			err = nfsd_nl_parse_fslocations(fsl,
+							&exp.ex_fslocs);
+			if (err)
+				goto out_path;
+		}
+
+		/* uuid */
+		if (tb[NFSD_A_SVC_EXPORT_UUID]) {
+			if (nla_len(tb[NFSD_A_SVC_EXPORT_UUID]) !=
+			    EX_UUID_LEN) {
+				err = -EINVAL;
+				goto out_fslocs;
+			}
+			exp.ex_uuid = kmemdup(nla_data(tb[NFSD_A_SVC_EXPORT_UUID]),
+					      EX_UUID_LEN, GFP_KERNEL);
+			if (!exp.ex_uuid) {
+				err = -ENOMEM;
+				goto out_fslocs;
+			}
+		}
+
+		/* secinfo (multi-attr) */
+		nla_for_each_nested_type(secinfo_attr,
+					 NFSD_A_SVC_EXPORT_SECINFO,
+					 attr, rem) {
+			struct nlattr *ftb[NFSD_A_AUTH_FLAVOR_FLAGS + 1];
+			struct exp_flavor_info *f;
+
+			if (exp.ex_nflavors >= MAX_SECINFO_LIST) {
+				err = -EINVAL;
+				goto out_uuid;
+			}
+
+			err = nla_parse_nested(ftb,
+					       NFSD_A_AUTH_FLAVOR_FLAGS,
+					       secinfo_attr,
+					       nfsd_auth_flavor_nl_policy,
+					       NULL);
+			if (err)
+				goto out_uuid;
+
+			f = &exp.ex_flavors[exp.ex_nflavors++];
+
+			if (ftb[NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR])
+				f->pseudoflavor = nla_get_u32(ftb[NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR]);
+
+			if (ftb[NFSD_A_AUTH_FLAVOR_FLAGS])
+				f->flags = nla_get_u32(ftb[NFSD_A_AUTH_FLAVOR_FLAGS]);
+
+			/* Only some flags are allowed to differ between flavors: */
+			if (~NFSEXP_SECINFO_FLAGS & (f->flags ^ exp.ex_flags)) {
+				err = -EINVAL;
+				goto out_uuid;
+			}
+		}
+
+		/* xprtsec (multi-attr u32) */
+		if (tb[NFSD_A_SVC_EXPORT_XPRTSEC]) {
+			struct nlattr *xp_attr;
+
+			exp.ex_xprtsec_modes = 0;
+			nla_for_each_nested_type(xp_attr,
+						 NFSD_A_SVC_EXPORT_XPRTSEC,
+						 attr, rem) {
+				u32 mode = nla_get_u32(xp_attr);
+
+				if (mode > NFSEXP_XPRTSEC_MTLS) {
+					err = -EINVAL;
+					goto out_uuid;
+				}
+				exp.ex_xprtsec_modes |= mode;
+			}
+		}
+
+		err = check_export(&exp.ex_path, &exp.ex_flags,
+				   exp.ex_uuid);
+		if (err)
+			goto out_uuid;
+
+		if (exp.h.expiry_time < seconds_since_boot())
+			goto out_uuid;
+
+		err = -EINVAL;
+		if (!uid_valid(exp.ex_anon_uid))
+			goto out_uuid;
+		if (!gid_valid(exp.ex_anon_gid))
+			goto out_uuid;
+		err = 0;
+
+		nfsd4_setup_layout_type(&exp);
+	}
+
+	expp = svc_export_lookup(&exp);
+	if (!expp) {
+		err = -ENOMEM;
+		goto out_uuid;
+	}
+	expp = svc_export_update(&exp, expp);
+	if (expp) {
+		trace_nfsd_export_update(expp);
+		cache_flush();
+		exp_put(expp);
+	} else {
+		err = -ENOMEM;
+	}
+
+out_uuid:
+	kfree(exp.ex_uuid);
+out_fslocs:
+	nfsd4_fslocs_free(&exp.ex_fslocs);
+out_path:
+	path_put(&exp.ex_path);
+out_dom:
+	auth_domain_put(dom);
+	return err;
+}
+
+/**
+ * nfsd_nl_svc_export_set_reqs_doit - respond to svc_export requests
+ * @skb: reply buffer
+ * @info: netlink metadata and command arguments
+ *
+ * Parse one or more svc_export cache responses from userspace and
+ * update the export cache accordingly.
+ *
+ * Returns 0 on success or a negative errno.
+ */
+int nfsd_nl_svc_export_set_reqs_doit(struct sk_buff *skb,
+				     struct genl_info *info)
+{
+	struct nfsd_net *nn;
+	struct cache_detail *cd;
+	const struct nlattr *attr;
+	int rem, ret = 0;
+
+	nn = net_generic(genl_info_net(info), nfsd_net_id);
+
+	mutex_lock(&nfsd_mutex);
+
+	cd = nn->svc_export_cache;
+	if (!cd) {
+		ret = -ENODEV;
+		goto out_unlock;
+	}
+
+	nlmsg_for_each_attr_type(attr, NFSD_A_SVC_EXPORT_REQS_REQUESTS,
+				 info->nlhdr, GENL_HDRLEN, rem) {
+		ret = nfsd_nl_parse_one_export(cd, (struct nlattr *)attr);
+		if (ret)
+			break;
+	}
+
+out_unlock:
+	mutex_unlock(&nfsd_mutex);
+	return ret;
+}
+
 static int svc_export_upcall(struct cache_detail *cd, struct cache_head *h)
 {
-	return sunrpc_cache_pipe_upcall(cd, h);
+	return sunrpc_cache_upcall(cd, h);
+}
+
+static int svc_export_notify(struct cache_detail *cd, struct cache_head *h)
+{
+	return nfsd_cache_notify(cd, h, NFSD_CACHE_TYPE_SVC_EXPORT);
 }
 
 static void svc_export_request(struct cache_detail *cd,
@@ -410,10 +1115,6 @@ static void svc_export_request(struct cache_detail *cd,
 	(*bpp)[-1] = '\n';
 }
 
-static struct svc_export *svc_export_update(struct svc_export *new,
-					    struct svc_export *old);
-static struct svc_export *svc_export_lookup(struct svc_export *);
-
 static int check_export(const struct path *path, int *flags, unsigned char *uuid)
 {
 	struct inode *inode = d_inode(path->dentry);
@@ -735,7 +1436,8 @@ static int svc_export_parse(struct cache_detail *cd, char *mesg, int mlen)
 			goto out4;
 		err = 0;
 
-		nfsd4_setup_layout_type(&exp);
+		if (exp.ex_flags & NFSEXP_PNFS)
+			nfsd4_setup_layout_type(&exp);
 	}
 
 	expp = svc_export_lookup(&exp);
@@ -907,6 +1609,7 @@ static const struct cache_detail svc_export_cache_template = {
 	.name		= "nfsd.export",
 	.cache_put	= svc_export_put,
 	.cache_upcall	= svc_export_upcall,
+	.cache_notify	= svc_export_notify,
 	.cache_request	= svc_export_request,
 	.cache_parse	= svc_export_parse,
 	.cache_show	= svc_export_show,
diff --git a/fs/nfsd/netlink.c b/fs/nfsd/netlink.c
index 81c943345d13d..f99add477cc7a 100644
--- a/fs/nfsd/netlink.c
+++ b/fs/nfsd/netlink.c
@@ -12,11 +12,51 @@
 #include <uapi/linux/nfsd_netlink.h>
 
 /* Common nested types */
+const struct nla_policy nfsd_auth_flavor_nl_policy[NFSD_A_AUTH_FLAVOR_FLAGS + 1] = {
+	[NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR] = { .type = NLA_U32, },
+	[NFSD_A_AUTH_FLAVOR_FLAGS] = NLA_POLICY_MASK(NLA_U32, 0x3ffff),
+};
+
+const struct nla_policy nfsd_expkey_nl_policy[NFSD_A_EXPKEY_PATH + 1] = {
+	[NFSD_A_EXPKEY_SEQNO] = { .type = NLA_U64, },
+	[NFSD_A_EXPKEY_CLIENT] = { .type = NLA_NUL_STRING, },
+	[NFSD_A_EXPKEY_FSIDTYPE] = { .type = NLA_U8, },
+	[NFSD_A_EXPKEY_FSID] = { .type = NLA_BINARY, },
+	[NFSD_A_EXPKEY_NEGATIVE] = { .type = NLA_FLAG, },
+	[NFSD_A_EXPKEY_EXPIRY] = { .type = NLA_U64, },
+	[NFSD_A_EXPKEY_PATH] = { .type = NLA_NUL_STRING, },
+};
+
+const struct nla_policy nfsd_fslocation_nl_policy[NFSD_A_FSLOCATION_PATH + 1] = {
+	[NFSD_A_FSLOCATION_HOST] = { .type = NLA_NUL_STRING, },
+	[NFSD_A_FSLOCATION_PATH] = { .type = NLA_NUL_STRING, },
+};
+
+const struct nla_policy nfsd_fslocations_nl_policy[NFSD_A_FSLOCATIONS_LOCATION + 1] = {
+	[NFSD_A_FSLOCATIONS_LOCATION] = NLA_POLICY_NESTED(nfsd_fslocation_nl_policy),
+};
+
 const struct nla_policy nfsd_sock_nl_policy[NFSD_A_SOCK_TRANSPORT_NAME + 1] = {
 	[NFSD_A_SOCK_ADDR] = { .type = NLA_BINARY, },
 	[NFSD_A_SOCK_TRANSPORT_NAME] = { .type = NLA_NUL_STRING, },
 };
 
+const struct nla_policy nfsd_svc_export_nl_policy[NFSD_A_SVC_EXPORT_FSID + 1] = {
+	[NFSD_A_SVC_EXPORT_SEQNO] = { .type = NLA_U64, },
+	[NFSD_A_SVC_EXPORT_CLIENT] = { .type = NLA_NUL_STRING, },
+	[NFSD_A_SVC_EXPORT_PATH] = { .type = NLA_NUL_STRING, },
+	[NFSD_A_SVC_EXPORT_NEGATIVE] = { .type = NLA_FLAG, },
+	[NFSD_A_SVC_EXPORT_EXPIRY] = { .type = NLA_U64, },
+	[NFSD_A_SVC_EXPORT_ANON_UID] = { .type = NLA_U32, },
+	[NFSD_A_SVC_EXPORT_ANON_GID] = { .type = NLA_U32, },
+	[NFSD_A_SVC_EXPORT_FSLOCATIONS] = NLA_POLICY_NESTED(nfsd_fslocations_nl_policy),
+	[NFSD_A_SVC_EXPORT_UUID] = { .type = NLA_BINARY, },
+	[NFSD_A_SVC_EXPORT_SECINFO] = NLA_POLICY_NESTED(nfsd_auth_flavor_nl_policy),
+	[NFSD_A_SVC_EXPORT_XPRTSEC] = NLA_POLICY_MASK(NLA_U32, 0x7),
+	[NFSD_A_SVC_EXPORT_FLAGS] = NLA_POLICY_MASK(NLA_U32, 0x3ffff),
+	[NFSD_A_SVC_EXPORT_FSID] = { .type = NLA_S32, },
+};
+
 const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1] = {
 	[NFSD_A_VERSION_MAJOR] = { .type = NLA_U32, },
 	[NFSD_A_VERSION_MINOR] = { .type = NLA_U32, },
@@ -48,6 +88,21 @@ static const struct nla_policy nfsd_pool_mode_set_nl_policy[NFSD_A_POOL_MODE_MOD
 	[NFSD_A_POOL_MODE_MODE] = { .type = NLA_NUL_STRING, },
 };
 
+/* NFSD_CMD_SVC_EXPORT_SET_REQS - do */
+static const struct nla_policy nfsd_svc_export_set_reqs_nl_policy[NFSD_A_SVC_EXPORT_REQS_REQUESTS + 1] = {
+	[NFSD_A_SVC_EXPORT_REQS_REQUESTS] = NLA_POLICY_NESTED(nfsd_svc_export_nl_policy),
+};
+
+/* NFSD_CMD_EXPKEY_SET_REQS - do */
+static const struct nla_policy nfsd_expkey_set_reqs_nl_policy[NFSD_A_EXPKEY_REQS_REQUESTS + 1] = {
+	[NFSD_A_EXPKEY_REQS_REQUESTS] = NLA_POLICY_NESTED(nfsd_expkey_nl_policy),
+};
+
+/* NFSD_CMD_CACHE_FLUSH - do */
+static const struct nla_policy nfsd_cache_flush_nl_policy[NFSD_A_CACHE_FLUSH_MASK + 1] = {
+	[NFSD_A_CACHE_FLUSH_MASK] = NLA_POLICY_MASK(NLA_U32, 0x3),
+};
+
 /* Ops table for nfsd */
 static const struct genl_split_ops nfsd_nl_ops[] = {
 	{
@@ -103,6 +158,42 @@ static const struct genl_split_ops nfsd_nl_ops[] = {
 		.doit	= nfsd_nl_pool_mode_get_doit,
 		.flags	= GENL_CMD_CAP_DO,
 	},
+	{
+		.cmd	= NFSD_CMD_SVC_EXPORT_GET_REQS,
+		.dumpit	= nfsd_nl_svc_export_get_reqs_dumpit,
+		.flags	= GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= NFSD_CMD_SVC_EXPORT_SET_REQS,
+		.doit		= nfsd_nl_svc_export_set_reqs_doit,
+		.policy		= nfsd_svc_export_set_reqs_nl_policy,
+		.maxattr	= NFSD_A_SVC_EXPORT_REQS_REQUESTS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd	= NFSD_CMD_EXPKEY_GET_REQS,
+		.dumpit	= nfsd_nl_expkey_get_reqs_dumpit,
+		.flags	= GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= NFSD_CMD_EXPKEY_SET_REQS,
+		.doit		= nfsd_nl_expkey_set_reqs_doit,
+		.policy		= nfsd_expkey_set_reqs_nl_policy,
+		.maxattr	= NFSD_A_EXPKEY_REQS_REQUESTS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= NFSD_CMD_CACHE_FLUSH,
+		.doit		= nfsd_nl_cache_flush_doit,
+		.policy		= nfsd_cache_flush_nl_policy,
+		.maxattr	= NFSD_A_CACHE_FLUSH_MASK,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+};
+
+static const struct genl_multicast_group nfsd_nl_mcgrps[] = {
+	[NFSD_NLGRP_NONE] = { "none", },
+	[NFSD_NLGRP_EXPORTD] = { "exportd", },
 };
 
 struct genl_family nfsd_nl_family __ro_after_init = {
@@ -113,4 +204,6 @@ struct genl_family nfsd_nl_family __ro_after_init = {
 	.module		= THIS_MODULE,
 	.split_ops	= nfsd_nl_ops,
 	.n_split_ops	= ARRAY_SIZE(nfsd_nl_ops),
+	.mcgrps		= nfsd_nl_mcgrps,
+	.n_mcgrps	= ARRAY_SIZE(nfsd_nl_mcgrps),
 };
diff --git a/fs/nfsd/netlink.h b/fs/nfsd/netlink.h
index 478117ff6b8c0..cc89732ed71bd 100644
--- a/fs/nfsd/netlink.h
+++ b/fs/nfsd/netlink.h
@@ -13,7 +13,12 @@
 #include <uapi/linux/nfsd_netlink.h>
 
 /* Common nested types */
+extern const struct nla_policy nfsd_auth_flavor_nl_policy[NFSD_A_AUTH_FLAVOR_FLAGS + 1];
+extern const struct nla_policy nfsd_expkey_nl_policy[NFSD_A_EXPKEY_PATH + 1];
+extern const struct nla_policy nfsd_fslocation_nl_policy[NFSD_A_FSLOCATION_PATH + 1];
+extern const struct nla_policy nfsd_fslocations_nl_policy[NFSD_A_FSLOCATIONS_LOCATION + 1];
 extern const struct nla_policy nfsd_sock_nl_policy[NFSD_A_SOCK_TRANSPORT_NAME + 1];
+extern const struct nla_policy nfsd_svc_export_nl_policy[NFSD_A_SVC_EXPORT_FSID + 1];
 extern const struct nla_policy nfsd_version_nl_policy[NFSD_A_VERSION_ENABLED + 1];
 
 int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb,
@@ -26,6 +31,19 @@ int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info);
 int nfsd_nl_listener_get_doit(struct sk_buff *skb, struct genl_info *info);
 int nfsd_nl_pool_mode_set_doit(struct sk_buff *skb, struct genl_info *info);
 int nfsd_nl_pool_mode_get_doit(struct sk_buff *skb, struct genl_info *info);
+int nfsd_nl_svc_export_get_reqs_dumpit(struct sk_buff *skb,
+				       struct netlink_callback *cb);
+int nfsd_nl_svc_export_set_reqs_doit(struct sk_buff *skb,
+				     struct genl_info *info);
+int nfsd_nl_expkey_get_reqs_dumpit(struct sk_buff *skb,
+				   struct netlink_callback *cb);
+int nfsd_nl_expkey_set_reqs_doit(struct sk_buff *skb, struct genl_info *info);
+int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info);
+
+enum {
+	NFSD_NLGRP_NONE,
+	NFSD_NLGRP_EXPORTD,
+};
 
 extern struct genl_family nfsd_nl_family;
 
diff --git a/fs/nfsd/nfs3proc.c b/fs/nfsd/nfs3proc.c
index 42adc5461db03..12b9172c6be19 100644
--- a/fs/nfsd/nfs3proc.c
+++ b/fs/nfsd/nfs3proc.c
@@ -710,23 +710,43 @@ nfsd3_proc_pathconf(struct svc_rqst *rqstp)
 	resp->p_name_max = 255;		/* at least */
 	resp->p_no_trunc = 0;
 	resp->p_chown_restricted = 1;
-	resp->p_case_insensitive = 0;
-	resp->p_case_preserving = 1;
+	resp->p_case_insensitive = false;
+	resp->p_case_preserving = true;
 
 	resp->status = fh_verify(rqstp, &argp->fh, 0, NFSD_MAY_NOP);
 
 	if (resp->status == nfs_ok) {
 		struct super_block *sb = argp->fh.fh_dentry->d_sb;
+		int err;
 
-		/* Note that we don't care for remote fs's here */
-		switch (sb->s_magic) {
-		case EXT2_SUPER_MAGIC:
+		if (sb->s_magic == EXT2_SUPER_MAGIC) {
 			resp->p_link_max = EXT2_LINK_MAX;
 			resp->p_name_max = EXT2_NAME_LEN;
+		}
+
+		err = nfsd_get_case_info(argp->fh.fh_dentry,
+					 &resp->p_case_insensitive,
+					 &resp->p_case_preserving);
+		/*
+		 * RFC 1813 lists NFS3ERR_STALE, NFS3ERR_BADHANDLE, and
+		 * NFS3ERR_SERVERFAULT as the only PATHCONF errors.
+		 */
+		switch (err) {
+		case 0:
+		case -EOPNOTSUPP:
+			/* Both arms leave the output booleans valid. */
+			break;
+		case -EACCES:
+		case -EPERM:
+			/*
+			 * Policy denied the query. Report STALE so the
+			 * handle is unusable without implying a server
+			 * malfunction.
+			 */
+			resp->status = nfserr_stale;
 			break;
-		case MSDOS_SUPER_MAGIC:
-			resp->p_case_insensitive = 1;
-			resp->p_case_preserving  = 0;
+		default:
+			resp->status = nfserr_serverfault;
 			break;
 		}
 	}
diff --git a/fs/nfsd/nfs4idmap.c b/fs/nfsd/nfs4idmap.c
index ba06d3d3e6dd9..71ba61b5d0a3a 100644
--- a/fs/nfsd/nfs4idmap.c
+++ b/fs/nfsd/nfs4idmap.c
@@ -126,7 +126,7 @@ idtoname_hash(struct ent *ent)
 static int
 idtoname_upcall(struct cache_detail *cd, struct cache_head *h)
 {
-	return sunrpc_cache_pipe_upcall_timeout(cd, h);
+	return sunrpc_cache_upcall_warn(cd, h);
 }
 
 static void
@@ -306,7 +306,7 @@ nametoid_hash(struct ent *ent)
 static int
 nametoid_upcall(struct cache_detail *cd, struct cache_head *h)
 {
-	return sunrpc_cache_pipe_upcall_timeout(cd, h);
+	return sunrpc_cache_upcall_warn(cd, h);
 }
 
 static void
diff --git a/fs/nfsd/nfs4layouts.c b/fs/nfsd/nfs4layouts.c
index 69e41105efdd5..c3543d4567024 100644
--- a/fs/nfsd/nfs4layouts.c
+++ b/fs/nfsd/nfs4layouts.c
@@ -2,7 +2,7 @@
 /*
  * Copyright (c) 2014 Christoph Hellwig.
  */
-#include <linux/blkdev.h>
+#include <linux/exportfs_block.h>
 #include <linux/kmod.h>
 #include <linux/file.h>
 #include <linux/jhash.h>
@@ -127,30 +127,17 @@ nfsd4_set_deviceid(struct nfsd4_deviceid *id, const struct svc_fh *fhp,
 
 void nfsd4_setup_layout_type(struct svc_export *exp)
 {
-#if defined(CONFIG_NFSD_BLOCKLAYOUT) || defined(CONFIG_NFSD_SCSILAYOUT)
 	struct super_block *sb = exp->ex_path.mnt->mnt_sb;
-#endif
-
-	if (!(exp->ex_flags & NFSEXP_PNFS))
-		return;
+	expfs_block_layouts_t block_supported = exportfs_layouts_supported(sb);
 
-#ifdef CONFIG_NFSD_FLEXFILELAYOUT
-	exp->ex_layout_types |= 1 << LAYOUT_FLEX_FILES;
-#endif
-#ifdef CONFIG_NFSD_BLOCKLAYOUT
-	if (sb->s_export_op->get_uuid &&
-	    sb->s_export_op->map_blocks &&
-	    sb->s_export_op->commit_blocks)
+	if (IS_ENABLED(CONFIG_NFSD_FLEXFILELAYOUT))
+		exp->ex_layout_types |= 1 << LAYOUT_FLEX_FILES;
+	if (IS_ENABLED(CONFIG_NFSD_BLOCKLAYOUT) &&
+	    (block_supported & EXPFS_BLOCK_IN_BAND_ID))
 		exp->ex_layout_types |= 1 << LAYOUT_BLOCK_VOLUME;
-#endif
-#ifdef CONFIG_NFSD_SCSILAYOUT
-	if (sb->s_export_op->map_blocks &&
-	    sb->s_export_op->commit_blocks &&
-	    sb->s_bdev &&
-	    sb->s_bdev->bd_disk->fops->pr_ops &&
-	    sb->s_bdev->bd_disk->fops->get_unique_id)
+	if (IS_ENABLED(CONFIG_NFSD_SCSILAYOUT) &&
+	    (block_supported & EXPFS_BLOCK_OUT_OF_BAND_ID))
 		exp->ex_layout_types |= 1 << LAYOUT_SCSI;
-#endif
 }
 
 void nfsd4_close_layout(struct nfs4_layout_stateid *ls)
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 6837b63d98645..3c2eb03f78c6d 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -1376,7 +1376,8 @@ static void destroy_delegation(struct nfs4_delegation *dp)
  * stateid or it's called from a laundromat thread (nfsd4_landromat()) that
  * determined that this specific state has expired and needs to be revoked
  * (both mark state with the appropriate stid sc_status mode). It is also
- * assumed that a reference was taken on the @dp state.
+ * assumed that a reference was taken on the @dp state. This function
+ * consumes that reference.
  *
  * If this function finds that the @dp state is SC_STATUS_FREED it means
  * that a FREE_STATEID operation for this stateid has been processed and
@@ -1839,6 +1840,10 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb)
 					mutex_unlock(&stp->st_mutex);
 					break;
 				case SC_TYPE_DELEG:
+					/* Extra reference guards against concurrent
+					 * FREE_STATEID; revoke_delegation() consumes
+					 * it, otherwise release it directly.
+					 */
 					refcount_inc(&stid->sc_count);
 					dp = delegstateid(stid);
 					spin_lock(&nn->deleg_lock);
@@ -1848,6 +1853,8 @@ void nfsd4_revoke_states(struct nfsd_net *nn, struct super_block *sb)
 					spin_unlock(&nn->deleg_lock);
 					if (dp)
 						revoke_delegation(dp);
+					else
+						nfs4_put_stid(stid);
 					break;
 				case SC_TYPE_LAYOUT:
 					ls = layoutstateid(stid);
diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c
index 2a0946c630e1d..319007b79d49f 100644
--- a/fs/nfsd/nfs4xdr.c
+++ b/fs/nfsd/nfs4xdr.c
@@ -3158,6 +3158,8 @@ struct nfsd4_fattr_args {
 	u32			rdattr_err;
 	bool			contextsupport;
 	bool			ignore_crossmnt;
+	bool			case_insensitive;
+	bool			case_preserving;
 };
 
 typedef __be32(*nfsd4_enc_attr)(struct xdr_stream *xdr,
@@ -3356,6 +3358,33 @@ static __be32 nfsd4_encode_fattr4_acl(struct xdr_stream *xdr,
 	return nfs_ok;
 }
 
+static __be32 nfsd4_encode_fattr4_case_insensitive(struct xdr_stream *xdr,
+					const struct nfsd4_fattr_args *args)
+{
+	return nfsd4_encode_bool(xdr, args->case_insensitive);
+}
+
+static __be32 nfsd4_encode_fattr4_case_preserving(struct xdr_stream *xdr,
+					const struct nfsd4_fattr_args *args)
+{
+	return nfsd4_encode_bool(xdr, args->case_preserving);
+}
+
+static __be32 nfsd4_encode_fattr4_homogeneous(struct xdr_stream *xdr,
+					const struct nfsd4_fattr_args *args)
+{
+	/*
+	 * Casefold-capable filesystems (e.g. ext4 or f2fs with the
+	 * casefold feature) attach a Unicode encoding at mount time
+	 * but apply case folding per directory.  The per-file-system
+	 * case_insensitive and case_preserving values can therefore
+	 * legitimately differ across objects that share the same fsid.
+	 * Report FATTR4_HOMOGENEOUS = FALSE on such filesystems to
+	 * keep that variation consistent with RFC 8881 Section 5.8.2.16.
+	 */
+	return nfsd4_encode_bool(xdr, !sb_has_encoding(args->dentry->d_sb));
+}
+
 static __be32 nfsd4_encode_fattr4_filehandle(struct xdr_stream *xdr,
 					     const struct nfsd4_fattr_args *args)
 {
@@ -3748,8 +3777,8 @@ static const nfsd4_enc_attr nfsd4_enc_fattr4_encode_ops[] = {
 	[FATTR4_ACLSUPPORT]		= nfsd4_encode_fattr4_aclsupport,
 	[FATTR4_ARCHIVE]		= nfsd4_encode_fattr4__noop,
 	[FATTR4_CANSETTIME]		= nfsd4_encode_fattr4__true,
-	[FATTR4_CASE_INSENSITIVE]	= nfsd4_encode_fattr4__false,
-	[FATTR4_CASE_PRESERVING]	= nfsd4_encode_fattr4__true,
+	[FATTR4_CASE_INSENSITIVE]	= nfsd4_encode_fattr4_case_insensitive,
+	[FATTR4_CASE_PRESERVING]	= nfsd4_encode_fattr4_case_preserving,
 	[FATTR4_CHOWN_RESTRICTED]	= nfsd4_encode_fattr4__true,
 	[FATTR4_FILEHANDLE]		= nfsd4_encode_fattr4_filehandle,
 	[FATTR4_FILEID]			= nfsd4_encode_fattr4_fileid,
@@ -3758,7 +3787,7 @@ static const nfsd4_enc_attr nfsd4_enc_fattr4_encode_ops[] = {
 	[FATTR4_FILES_TOTAL]		= nfsd4_encode_fattr4_files_total,
 	[FATTR4_FS_LOCATIONS]		= nfsd4_encode_fattr4_fs_locations,
 	[FATTR4_HIDDEN]			= nfsd4_encode_fattr4__noop,
-	[FATTR4_HOMOGENEOUS]		= nfsd4_encode_fattr4__true,
+	[FATTR4_HOMOGENEOUS]		= nfsd4_encode_fattr4_homogeneous,
 	[FATTR4_MAXFILESIZE]		= nfsd4_encode_fattr4_maxfilesize,
 	[FATTR4_MAXLINK]		= nfsd4_encode_fattr4_maxlink,
 	[FATTR4_MAXNAME]		= nfsd4_encode_fattr4_maxname,
@@ -3968,6 +3997,23 @@ nfsd4_encode_fattr4(struct svc_rqst *rqstp, struct xdr_stream *xdr,
 		args.fhp = tempfh;
 	} else
 		args.fhp = fhp;
+	if (attrmask[0] & (FATTR4_WORD0_CASE_INSENSITIVE |
+			   FATTR4_WORD0_CASE_PRESERVING)) {
+		err = nfsd_get_case_info(dentry, &args.case_insensitive,
+					 &args.case_preserving);
+		/*
+		 * Per RFC 8881 Section 18.7.3, an attribute advertised
+		 * in SUPPORTED_ATTRS must come back with a value or the
+		 * GETATTR must fail. nfsd_get_case_info() fills POSIX
+		 * defaults and returns -EOPNOTSUPP when the underlying
+		 * filesystem does not expose case state; encode those
+		 * defaults so the reply agrees with what SUPPORTED_ATTRS
+		 * advertises. Other errors fail the operation as the
+		 * spec requires.
+		 */
+		if (err && err != -EOPNOTSUPP)
+			goto out_nfserr;
+	}
 
 	if (attrmask[0] & FATTR4_WORD0_ACL) {
 		err = nfsd4_get_nfs4_acl(rqstp, dentry, &args.acl);
diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c
index 39e7012a60d8e..92f958509399f 100644
--- a/fs/nfsd/nfsctl.c
+++ b/fs/nfsd/nfsctl.c
@@ -10,6 +10,7 @@
 #include <linux/ctype.h>
 #include <linux/fs_context.h>
 
+#include <linux/sunrpc/cache.h>
 #include <linux/sunrpc/svcsock.h>
 #include <linux/lockd/bind.h>
 #include <linux/sunrpc/addr.h>
@@ -1412,6 +1413,21 @@ static int create_proc_exports_entry(void)
 
 unsigned int nfsd_net_id;
 
+struct nfsd_genl_rqstp {
+	struct sockaddr		rq_daddr;
+	struct sockaddr		rq_saddr;
+	unsigned long		rq_flags;
+	ktime_t			rq_stime;
+	__be32			rq_xid;
+	u32			rq_vers;
+	u32			rq_prog;
+	u32			rq_proc;
+
+	/* NFSv4 compound */
+	u32			rq_opcnt;
+	u32			rq_opnum[16];
+};
+
 static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb,
 					    struct netlink_callback *cb,
 					    struct nfsd_genl_rqstp *genl_rqstp)
@@ -2187,6 +2203,69 @@ int nfsd_nl_pool_mode_get_doit(struct sk_buff *skb, struct genl_info *info)
 	return err;
 }
 
+/**
+ * nfsd_nl_cache_flush_doit - flush nfsd caches via netlink
+ * @skb: reply buffer
+ * @info: netlink metadata and command arguments
+ *
+ * Flush the svc_export and/or expkey caches. If NFSD_A_CACHE_FLUSH_MASK
+ * is provided, only flush the caches indicated by the bitmask (bit 0 =
+ * svc_export, bit 1 = expkey). If omitted, flush both.
+ *
+ * Return 0 on success or a negative errno.
+ */
+int nfsd_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	struct net *net = genl_info_net(info);
+	struct nfsd_net *nn = net_generic(net, nfsd_net_id);
+	u32 mask = ~0U;
+
+	if (info->attrs[NFSD_A_CACHE_FLUSH_MASK])
+		mask = nla_get_u32(info->attrs[NFSD_A_CACHE_FLUSH_MASK]);
+
+	mutex_lock(&nfsd_mutex);
+
+	if ((mask & NFSD_CACHE_TYPE_SVC_EXPORT) &&
+	    nn->svc_export_cache)
+		cache_purge(nn->svc_export_cache);
+
+	if ((mask & NFSD_CACHE_TYPE_EXPKEY) &&
+	    nn->svc_expkey_cache)
+		cache_purge(nn->svc_expkey_cache);
+
+	mutex_unlock(&nfsd_mutex);
+
+	return 0;
+}
+
+int nfsd_cache_notify(struct cache_detail *cd, struct cache_head *h, u32 cache_type)
+{
+	struct genlmsghdr *hdr;
+	struct sk_buff *msg;
+
+	if (!genl_has_listeners(&nfsd_nl_family, cd->net, NFSD_NLGRP_EXPORTD))
+		return -ENOLINK;
+
+	msg = genlmsg_new(nla_total_size(sizeof(u32)), GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	hdr = genlmsg_put(msg, 0, 0, &nfsd_nl_family, 0, NFSD_CMD_CACHE_NOTIFY);
+	if (!hdr) {
+		nlmsg_free(msg);
+		return -ENOMEM;
+	}
+
+	if (nla_put_u32(msg, NFSD_A_CACHE_NOTIFY_CACHE_TYPE, cache_type)) {
+		nlmsg_free(msg);
+		return -ENOMEM;
+	}
+
+	genlmsg_end(msg, hdr);
+	return genlmsg_multicast_netns(&nfsd_nl_family, cd->net, msg, 0,
+				       NFSD_NLGRP_EXPORTD, GFP_KERNEL);
+}
+
 /**
  * nfsd_net_init - Prepare the nfsd_net portion of a new net namespace
  * @net: a freshly-created network namespace
diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h
index 7c009f07c90b5..709da3c7a5fa7 100644
--- a/fs/nfsd/nfsd.h
+++ b/fs/nfsd/nfsd.h
@@ -60,21 +60,6 @@ struct readdir_cd {
 /* Maximum number of operations per session compound */
 #define NFSD_MAX_OPS_PER_COMPOUND	200
 
-struct nfsd_genl_rqstp {
-	struct sockaddr		rq_daddr;
-	struct sockaddr		rq_saddr;
-	unsigned long		rq_flags;
-	ktime_t			rq_stime;
-	__be32			rq_xid;
-	u32			rq_vers;
-	u32			rq_prog;
-	u32			rq_proc;
-
-	/* NFSv4 compound */
-	u32			rq_opcnt;
-	u32			rq_opnum[16];
-};
-
 extern struct svc_program	nfsd_programs[];
 extern const struct svc_version	nfsd_version2, nfsd_version3, nfsd_version4;
 extern struct mutex		nfsd_mutex;
@@ -123,7 +108,7 @@ struct dentry *nfsd_client_mkdir(struct nfsd_net *nn,
 				 const struct tree_descr *,
 				 struct dentry **fdentries);
 void nfsd_client_rmdir(struct dentry *dentry);
-
+int nfsd_cache_notify(struct cache_detail *cd, struct cache_head *h, u32 cache_type);
 
 #if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
 #ifdef CONFIG_NFSD_V2_ACL
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index eafdf7b7890fd..85ff418127c7b 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -32,6 +32,7 @@
 #include <linux/writeback.h>
 #include <linux/security.h>
 #include <linux/sunrpc/xdr.h>
+#include <linux/fileattr.h>
 
 #include "xdr3.h"
 
@@ -2891,3 +2892,90 @@ nfsd_permission(struct svc_cred *cred, struct svc_export *exp,
 
 	return err? nfserrno(err) : 0;
 }
+
+/**
+ * nfsd_get_case_info - get case sensitivity info for a dentry
+ * @dentry: dentry to query
+ * @case_insensitive: set to true if name comparison ignores case
+ * @case_preserving: set to true if case is preserved on disk
+ *
+ * On casefold-capable filesystems the flag lives on the directory,
+ * not on its entries, so for a non-directory @dentry the parent is
+ * queried instead. A directory (including an export root, whose
+ * parent lies outside the export) is queried as-is so its own
+ * contents' lookup behavior is reported. NFSD advertises
+ * fattr4_homogeneous as FALSE, so per-directory answers may differ
+ * within an export.
+ *
+ * The probe runs with kernel credentials. case_insensitive and
+ * case_preserving describe the directory's structural lookup
+ * behavior, not the caller's identity; running under the calling
+ * client's mapped credentials would let per-client MAC policy on
+ * the parent directory turn this query into NFS4ERR_ACCESS even
+ * though the underlying property is the same for every client.
+ *
+ * When the filesystem does not expose case-folding state (no
+ * ->fileattr_get, or the callback returns -EOPNOTSUPP /
+ * -ENOIOCTLCMD / -ENOTTY / -EINVAL), the outputs are filled with
+ * POSIX defaults (case-sensitive, case-preserving) on the premise
+ * that a filesystem with case-folding support wires up
+ * fileattr_get.
+ *
+ * Return: 0 with outputs filled, -EOPNOTSUPP with outputs filled
+ *         to POSIX defaults, or a negative errno (e.g., -EIO,
+ *         -ESTALE, -ENOMEM) with outputs unmodified.
+ */
+int
+nfsd_get_case_info(struct dentry *dentry, bool *case_insensitive,
+		   bool *case_preserving)
+{
+	struct file_kattr fa = {};
+	const struct cred *saved;
+	struct cred *probe;
+	struct dentry *cd;
+	bool put = false;
+	int err;
+
+	if (d_is_dir(dentry)) {
+		cd = dentry;
+	} else {
+		cd = dget_parent(dentry);
+		put = true;
+	}
+
+	probe = prepare_creds();
+	if (!probe) {
+		err = -ENOMEM;
+		goto out;
+	}
+	probe->fsuid = GLOBAL_ROOT_UID;
+	probe->fsgid = GLOBAL_ROOT_GID;
+	saved = override_creds(probe);
+
+	err = vfs_fileattr_get(cd, &fa);
+
+	put_cred(revert_creds(saved));
+out:
+	if (put)
+		dput(cd);
+	switch (err) {
+	case 0:
+		*case_insensitive = fa.fsx_xflags & FS_XFLAG_CASEFOLD;
+		*case_preserving =
+			!(fa.fsx_xflags & FS_XFLAG_CASENONPRESERVING);
+		return 0;
+	case -EINVAL:
+	case -ENOTTY:
+	case -ENOIOCTLCMD:
+	case -EOPNOTSUPP:
+		/*
+		 * Filesystem does not expose case state.
+		 * Report POSIX defaults.
+		 */
+		*case_insensitive = false;
+		*case_preserving = true;
+		return -EOPNOTSUPP;
+	default:
+		return err;
+	}
+}
diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h
index 702a844f2106b..e09ea04a51b99 100644
--- a/fs/nfsd/vfs.h
+++ b/fs/nfsd/vfs.h
@@ -156,6 +156,9 @@ __be32		nfsd_readdir(struct svc_rqst *, struct svc_fh *,
 			     loff_t *, struct readdir_cd *, nfsd_filldir_t);
 __be32		nfsd_statfs(struct svc_rqst *, struct svc_fh *,
 				struct kstatfs *, int access);
+int		nfsd_get_case_info(struct dentry *dentry,
+				   bool *case_insensitive,
+				   bool *case_preserving);
 
 __be32		nfsd_permission(struct svc_cred *cred, struct svc_export *exp,
 				struct dentry *dentry, int acc);
diff --git a/fs/nfsd/xdr3.h b/fs/nfsd/xdr3.h
index 522067b7fd755..a7c9714b0b0e1 100644
--- a/fs/nfsd/xdr3.h
+++ b/fs/nfsd/xdr3.h
@@ -209,8 +209,8 @@ struct nfsd3_pathconfres {
 	__u32			p_name_max;
 	__u32			p_no_trunc;
 	__u32			p_chown_restricted;
-	__u32			p_case_insensitive;
-	__u32			p_case_preserving;
+	bool			p_case_insensitive;
+	bool			p_case_preserving;
 };
 
 struct nfsd3_commitres {
diff --git a/fs/nsfs.c b/fs/nsfs.c
index 51e8c9430477b..160018c4fb362 100644
--- a/fs/nsfs.c
+++ b/fs/nsfs.c
@@ -266,7 +266,7 @@ static long ns_ioctl(struct file *filp, unsigned int ioctl,
 		else
 			tsk = find_task_by_pid_ns(arg, pid_ns);
 		if (!tsk)
-			break;
+			return ret;
 
 		switch (ioctl) {
 		case NS_GET_PID_FROM_PIDNS:
diff --git a/fs/ntfs/Makefile b/fs/ntfs/Makefile
index 0ce4d9a9388af..e120c2e69862a 100644
--- a/fs/ntfs/Makefile
+++ b/fs/ntfs/Makefile
@@ -5,6 +5,6 @@ obj-$(CONFIG_NTFS_FS) += ntfs.o
 ntfs-y := aops.o attrib.o collate.o dir.o file.o index.o inode.o \
 	  mft.o mst.o namei.o runlist.o super.o unistr.o attrlist.o ea.o \
 	  upcase.o bitmap.o lcnalloc.o logfile.o reparse.o compress.o \
-	  iomap.o debug.o sysctl.o quota.o object_id.o bdev-io.o
+	  iomap.o debug.o sysctl.o object_id.o bdev-io.o
 
 ccflags-$(CONFIG_NTFS_DEBUG) += -DDEBUG
diff --git a/fs/ntfs/attrib.c b/fs/ntfs/attrib.c
index 97b660eaa00c1..9d676375f25cb 100644
--- a/fs/ntfs/attrib.c
+++ b/fs/ntfs/attrib.c
@@ -16,6 +16,7 @@
  * Copyright (c) 2010 Erik Larsson
  */
 
+#include <linux/string_choices.h>
 #include <linux/writeback.h>
 #include <linux/iomap.h>
 
@@ -583,24 +584,13 @@ static u32 ntfs_resident_attr_min_value_length(const __le32 type)
 	case AT_STANDARD_INFORMATION:
 		return offsetof(struct standard_information, ver) +
 		       sizeof(((struct standard_information *)0)->ver.v1.reserved12);
-	case AT_ATTRIBUTE_LIST:
-		return offsetof(struct attr_list_entry, name);
 	case AT_FILE_NAME:
-		return offsetof(struct file_name_attr, file_name);
-	case AT_OBJECT_ID:
-		return sizeof(struct guid);
-	case AT_SECURITY_DESCRIPTOR:
-		return sizeof(struct security_descriptor_relative);
+		return offsetof(struct file_name_attr, file_name) +
+			sizeof(__le16) * 1;
 	case AT_VOLUME_INFORMATION:
 		return sizeof(struct volume_information);
-	case AT_INDEX_ROOT:
-		return sizeof(struct index_root);
-	case AT_REPARSE_POINT:
-		return offsetof(struct reparse_point, reparse_data);
 	case AT_EA_INFORMATION:
 		return sizeof(struct ea_information);
-	case AT_EA:
-		return offsetof(struct ea_attr, ea_name) + 1;
 	default:
 		return 0;
 	}
@@ -672,6 +662,9 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name,
 	__le16 *upcase = vol->upcase;
 	u32 upcase_len = vol->upcase_len;
 	unsigned int space;
+	u16 name_offset;
+	u32 attr_len;
+	u32 name_size;
 
 	/*
 	 * Iterate over attributes in mft record starting at @ctx->attr, or the
@@ -699,6 +692,20 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name,
 			return -ENOENT;
 		if (unlikely(!a->length))
 			break;
+		if (a->name_length) {
+			name_offset = le16_to_cpu(a->name_offset);
+			attr_len = le32_to_cpu(a->length);
+			name_size = a->name_length * sizeof(__le16);
+
+			if (name_offset > attr_len ||
+			    attr_len - name_offset < name_size) {
+				ntfs_error(vol->sb,
+					   "Corrupt attribute name in MFT record %llu\n",
+					   ctx->ntfs_ino->mft_no);
+				break;
+			}
+		}
+
 		if (type == AT_UNUSED)
 			return 0;
 		if (a->type != type)
@@ -712,14 +719,6 @@ static int ntfs_attr_find(const __le32 type, const __le16 *name,
 			if (a->name_length)
 				return -ENOENT;
 		} else {
-			if (a->name_length && ((le16_to_cpu(a->name_offset) +
-					       a->name_length * sizeof(__le16)) >
-						le32_to_cpu(a->length))) {
-				ntfs_error(vol->sb, "Corrupt attribute name in MFT record %llu\n",
-					   ctx->ntfs_ino->mft_no);
-				break;
-			}
-
 			if (!ntfs_are_names_equal(name, name_len,
 					(__le16 *)((u8 *)a + le16_to_cpu(a->name_offset)),
 					a->name_length, ic, upcase, upcase_len)) {
@@ -1857,7 +1856,7 @@ int ntfs_attr_make_non_resident(struct ntfs_inode *ni, const u32 data_size)
 		if (IS_ERR(rl)) {
 			err = PTR_ERR(rl);
 			ntfs_debug("Failed to allocate cluster%s, error code %i.",
-					ntfs_bytes_to_cluster(vol, new_size) > 1 ? "s" : "",
+					str_plural(ntfs_bytes_to_cluster(vol, new_size)),
 					err);
 			goto folio_err_out;
 		}
@@ -2924,12 +2923,12 @@ int ntfs_attr_open(struct ntfs_inode *ni, const __le32 type,
 	struct ntfs_inode *base_ni;
 	int err;
 
-	ntfs_debug("Entering for inode %lld, attr 0x%x.\n",
-			(unsigned long long)ni->mft_no, type);
-
 	if (!ni || !ni->vol)
 		return -EINVAL;
 
+	ntfs_debug("Entering for inode %lld, attr 0x%x.\n",
+			ni->mft_no, type);
+
 	if (NInoAttr(ni))
 		base_ni = ni->ext.base_ntfs_ino;
 	else
diff --git a/fs/ntfs/attrlist.c b/fs/ntfs/attrlist.c
index bd501e8a628c4..c2594d4c83b06 100644
--- a/fs/ntfs/attrlist.c
+++ b/fs/ntfs/attrlist.c
@@ -119,15 +119,14 @@ int ntfs_attrlist_entry_add(struct ntfs_inode *ni, struct attr_record *attr)
 	struct mft_record *ni_mrec;
 	u8 *old_al;
 
-	ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n",
-			(long long) ni->mft_no,
-			(unsigned int) le32_to_cpu(attr->type));
-
 	if (!ni || !attr) {
 		ntfs_debug("Invalid arguments.\n");
 		return -EINVAL;
 	}
 
+	ntfs_debug("Entering for inode 0x%llx, attr 0x%x.\n",
+			ni->mft_no, (unsigned int) le32_to_cpu(attr->type));
+
 	ni_mrec = map_mft_record(ni);
 	if (IS_ERR(ni_mrec)) {
 		ntfs_debug("Invalid arguments.\n");
diff --git a/fs/ntfs/bdev-io.c b/fs/ntfs/bdev-io.c
index 67e65c88d6818..27d7c2767a33f 100644
--- a/fs/ntfs/bdev-io.c
+++ b/fs/ntfs/bdev-io.c
@@ -97,6 +97,8 @@ int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size
 		idx_end++;
 
 	for (; idx < idx_end; idx++, from = 0) {
+		u32 len;
+
 		folio = read_mapping_folio(sb->s_bdev->bd_mapping, idx, NULL);
 		if (IS_ERR(folio)) {
 			ntfs_error(sb, "Unable to read %ld page", idx);
@@ -105,9 +107,10 @@ int ntfs_bdev_write(struct super_block *sb, void *buf, loff_t start, size_t size
 
 		offset = (loff_t)idx << PAGE_SHIFT;
 		to = min_t(u32, end - offset, PAGE_SIZE);
+		len = to - from;
 
-		memcpy_to_folio(folio, from, buf + buf_off, to);
-		buf_off += to;
+		memcpy_to_folio(folio, from, buf + buf_off, len);
+		buf_off += len;
 		folio_mark_uptodate(folio);
 		folio_mark_dirty(folio);
 		folio_put(folio);
diff --git a/fs/ntfs/index.c b/fs/ntfs/index.c
index a547bdcfa4561..146e011c1a418 100644
--- a/fs/ntfs/index.c
+++ b/fs/ntfs/index.c
@@ -677,11 +677,11 @@ static int ntfs_ib_read(struct ntfs_index_context *icx, s64 vcn, struct index_bl
 
 static int ntfs_icx_parent_inc(struct ntfs_index_context *icx)
 {
-	icx->pindex++;
-	if (icx->pindex >= MAX_PARENT_VCN) {
+	if (icx->pindex >= MAX_PARENT_VCN - 1) {
 		ntfs_error(icx->idx_ni->vol->sb, "Index is over %d level deep", MAX_PARENT_VCN);
 		return -EOPNOTSUPP;
 	}
+	icx->pindex++;
 	return 0;
 }
 
@@ -1970,6 +1970,7 @@ struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_ind
 {
 	struct index_entry *entry;
 	struct index_block *ib;
+	int err;
 	s64 vcn;
 
 	entry = ie;
@@ -1979,14 +1980,20 @@ struct index_entry *ntfs_index_walk_down(struct index_entry *ie, struct ntfs_ind
 			ib = kvzalloc(ictx->block_size, GFP_NOFS);
 			if (!ib)
 				return ERR_PTR(-ENOMEM);
-			/* down from level zero */
+			/*
+			 * Descending from root index (level 0) to the first
+			 * child level. is_in_root == true implies pindex == 0,
+			 * so advance to level 1.
+			 */
+			ictx->pindex = 1;
 			ictx->ir = NULL;
 			ictx->ib = ib;
-			ictx->pindex = 1;
 			ictx->is_in_root = false;
 		} else {
 			/* down from non-zero level */
-			ictx->pindex++;
+			err = ntfs_icx_parent_inc(ictx);
+			if (err)
+				return ERR_PTR(err);
 		}
 
 		ictx->parent_pos[ictx->pindex] = 0;
diff --git a/fs/ntfs/inode.c b/fs/ntfs/inode.c
index 16890d411194d..360bebd1ee3fe 100644
--- a/fs/ntfs/inode.c
+++ b/fs/ntfs/inode.c
@@ -2582,8 +2582,8 @@ int ntfs_inode_sync_filename(struct ntfs_inode *ni)
 
 		mutex_lock_nested(&index_ni->mrec_lock, NTFS_INODE_MUTEX_PARENT);
 		if (NInoBeingDeleted(ni)) {
-			iput(index_vi);
 			mutex_unlock(&index_ni->mrec_lock);
+			iput(index_vi);
 			continue;
 		}
 
@@ -2591,8 +2591,8 @@ int ntfs_inode_sync_filename(struct ntfs_inode *ni)
 		if (!ictx) {
 			ntfs_error(sb, "Failed to get index ctx, inode %llu",
 					index_ni->mft_no);
-			iput(index_vi);
 			mutex_unlock(&index_ni->mrec_lock);
+			iput(index_vi);
 			continue;
 		}
 
@@ -2601,8 +2601,8 @@ int ntfs_inode_sync_filename(struct ntfs_inode *ni)
 			ntfs_debug("Index lookup failed, inode %llu",
 					index_ni->mft_no);
 			ntfs_index_ctx_put(ictx);
-			iput(index_vi);
 			mutex_unlock(&index_ni->mrec_lock);
+			iput(index_vi);
 			continue;
 		}
 		/* Update flags and file size. */
diff --git a/fs/ntfs/logfile.c b/fs/ntfs/logfile.c
index 3f8d1640f1d50..d3f25d8e29f9d 100644
--- a/fs/ntfs/logfile.c
+++ b/fs/ntfs/logfile.c
@@ -710,6 +710,9 @@ bool ntfs_empty_logfile(struct inode *log_vi)
 		if (unlikely(lcn == LCN_RL_NOT_MAPPED)) {
 			vcn = rl->vcn;
 			kvfree(empty_buf);
+			empty_buf = NULL;
+			kfree(ra);
+			ra = NULL;
 			goto map_vcn;
 		}
 		/* If this run is not valid abort with an error. */
@@ -753,7 +756,7 @@ bool ntfs_empty_logfile(struct inode *log_vi)
 		} while (start < end);
 	} while ((++rl)->vcn < end_vcn);
 	up_write(&log_ni->runlist.lock);
-	kfree(empty_buf);
+	kvfree(empty_buf);
 	kfree(ra);
 	truncate_inode_pages(log_vi->i_mapping, 0);
 	/* Set the flag so we do not have to do it again on remount. */
diff --git a/fs/ntfs/mft.c b/fs/ntfs/mft.c
index 7d989267a82b9..a7d10ee41b344 100644
--- a/fs/ntfs/mft.c
+++ b/fs/ntfs/mft.c
@@ -30,6 +30,8 @@ int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m,
 {
 	struct attr_record *a;
 	struct super_block *sb = vol->sb;
+	u16 attrs_offset;
+	u32 bytes_in_use;
 
 	if (!ntfs_is_file_record(m->magic)) {
 		ntfs_error(sb, "Record %llu has no FILE magic (0x%x)\n",
@@ -65,7 +67,16 @@ int ntfs_mft_record_check(const struct ntfs_volume *vol, struct mft_record *m,
 		goto err_out;
 	}
 
-	a = (struct attr_record *)((char *)m + le16_to_cpu(m->attrs_offset));
+	attrs_offset = le16_to_cpu(m->attrs_offset);
+	bytes_in_use = le32_to_cpu(m->bytes_in_use);
+
+	if (attrs_offset > bytes_in_use ||
+	    bytes_in_use - attrs_offset < sizeof_field(struct attr_record, type)) {
+		ntfs_error(sb, "Record %llu has corrupt attribute offset\n", mft_no);
+		goto err_out;
+	}
+
+	a = (struct attr_record *)((char *)m + attrs_offset);
 	if ((char *)a < (char *)m || (char *)a > (char *)m + vol->mft_record_size) {
 		ntfs_error(sb, "Record %llu is corrupt\n", mft_no);
 		goto err_out;
@@ -449,7 +460,7 @@ static void ntfs_bio_end_io(struct bio *bio)
 int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no,
 		struct mft_record *m)
 {
-	u8 *kmirr = NULL;
+	u8 *kmirr;
 	struct folio *folio;
 	unsigned int folio_ofs, lcn_folio_off = 0;
 	int err = 0;
@@ -479,6 +490,7 @@ int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no,
 	kmirr = kmap_local_folio(folio, 0) + folio_ofs;
 	/* Copy the mst protected mft record to the mirror. */
 	memcpy(kmirr, m, vol->mft_record_size);
+	kunmap_local(kmirr);
 
 	if (vol->cluster_size_bits > PAGE_SHIFT) {
 		lcn_folio_off = folio->index << PAGE_SHIFT;
@@ -490,20 +502,22 @@ int ntfs_sync_mft_mirror(struct ntfs_volume *vol, const u64 mft_no,
 		NTFS_B_TO_SECTOR(vol, NTFS_CLU_TO_B(vol, vol->mftmirr_lcn) +
 				 lcn_folio_off + folio_ofs);
 
-	if (!bio_add_folio(bio, folio, vol->mft_record_size, folio_ofs)) {
+	if (bio_add_folio(bio, folio, vol->mft_record_size, folio_ofs))
+		err = submit_bio_wait(bio);
+	else
 		err = -EIO;
-		bio_put(bio);
-		goto unlock_folio;
-	}
+	bio_put(bio);
 
-	bio->bi_end_io = ntfs_bio_end_io;
-	submit_bio(bio);
-	/* Current state: all buffers are clean, unlocked, and uptodate. */
+	/*
+	 * The in-memory mirror is now valid because we just memcpy()'d the
+	 * mst-protected mft record into it.  Mark the folio uptodate even on
+	 * write error so a subsequent read_mapping_folio() does not refetch
+	 * the stale on-disk mirror and overwrite this copy.  The error is
+	 * propagated to the caller via @err.
+	 */
 	folio_mark_uptodate(folio);
 
-unlock_folio:
 	folio_unlock(folio);
-	kunmap_local(kmirr);
 	folio_put(folio);
 	if (likely(!err)) {
 		ntfs_debug("Done.");
@@ -588,20 +602,36 @@ int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int syn
 		}
 
 		/* Synchronize the mft mirror now if not @sync. */
-		if (!sync && ni->mft_no < vol->mftmirr_size)
-			ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m);
+		if (!sync && ni->mft_no < vol->mftmirr_size) {
+			int sub_err = ntfs_sync_mft_mirror(vol, ni->mft_no,
+							   fixup_m);
+			if (unlikely(sub_err) && !err)
+				err = sub_err;
+		}
 
-		folio_get(folio);
-		bio->bi_private = folio;
-		bio->bi_end_io = ntfs_bio_end_io;
-		submit_bio(bio);
+		if (sync) {
+			int sub_err = submit_bio_wait(bio);
+
+			bio_put(bio);
+			if (unlikely(sub_err) && !err)
+				err = sub_err;
+		} else {
+			folio_get(folio);
+			bio->bi_private = folio;
+			bio->bi_end_io = ntfs_bio_end_io;
+			submit_bio(bio);
+		}
 		offset += vol->cluster_size;
 		i++;
 	}
 
 	/* If @sync, now synchronize the mft mirror. */
-	if (sync && ni->mft_no < vol->mftmirr_size)
-		ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m);
+	if (sync && ni->mft_no < vol->mftmirr_size) {
+		int sub_err = ntfs_sync_mft_mirror(vol, ni->mft_no, fixup_m);
+
+		if (unlikely(sub_err) && !err)
+			err = sub_err;
+	}
 	kunmap_local(kaddr);
 	if (unlikely(err)) {
 		/* I/O error during writing.  This is really bad! */
@@ -617,10 +647,10 @@ int write_mft_record_nolock(struct ntfs_inode *ni, struct mft_record *m, int syn
 	bio_put(bio);
 err_out:
 	/*
-	 * Current state: all buffers are clean, unlocked, and uptodate.
-	 * The caller should mark the base inode as bad so that no more i/o
-	 * happens.  ->drop_inode() will still be invoked so all extent inodes
-	 * and other allocated memory will be freed.
+	 * The caller should mark the base inode as bad so no more I/O
+	 * happens. ->drop_inode() will still be invoked so all extent inodes
+	 * and other allocated memory will be freed. ENOMEM is retried by
+	 * redirtying the mft record below.
 	 */
 	if (err == -ENOMEM) {
 		ntfs_error(vol->sb,
@@ -833,7 +863,7 @@ static bool ntfs_may_write_mft_record(struct ntfs_volume *vol, const u64 mft_no,
 		vi = igrab(mft_vi);
 		WARN_ON(vi != mft_vi);
 	} else {
-		vi = find_inode_nowait(sb, mft_no, ntfs_test_inode_wb, &na);
+		vi = find_inode_nowait(sb, na.mft_no, ntfs_test_inode_wb, &na);
 		if (na.state == NI_BeingDeleted || na.state == NI_BeingCreated)
 			return false;
 	}
@@ -1034,7 +1064,7 @@ static s64 ntfs_mft_bitmap_find_and_alloc_free_rec_nolock(struct ntfs_volume *vo
 				b = ffz((unsigned long)*byte);
 				if (b < 8 && b >= (bit & 7)) {
 					ll = data_pos + (bit & ~7ull) + b;
-					if (unlikely(ll > (1ll << 32))) {
+					if (unlikely(ll >= (1ll << 32))) {
 						folio_unlock(folio);
 						kunmap_local(buf);
 						folio_put(folio);
@@ -2721,8 +2751,11 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w
 	ntfs_debug("Entering for inode 0x%llx, attribute type 0x%x, folio index 0x%lx.",
 			ni->mft_no, ni->type, folio->index);
 
-	if (!locked_nis || !ref_inos)
+	if (!locked_nis || !ref_inos) {
+		folio_redirty_for_writepage(wbc, folio);
+		folio_unlock(folio);
 		return -ENOMEM;
+	}
 
 	/* We have to zero every time due to mmap-at-end-of-file. */
 	if (folio->index >= (i_size >> folio_shift(folio)))
@@ -2840,9 +2873,13 @@ static int ntfs_write_mft_block(struct folio *folio, struct writeback_control *w
 			}
 			prev_mft_ofs = mft_ofs;
 
-			if (mft_no < vol->mftmirr_size)
-				ntfs_sync_mft_mirror(vol, mft_no,
+			if (mft_no < vol->mftmirr_size) {
+				int sub_err = ntfs_sync_mft_mirror(vol, mft_no,
 						(struct mft_record *)(kaddr + mft_ofs));
+
+				if (unlikely(sub_err) && !err)
+					err = sub_err;
+			}
 		} else if (ref_inos[nr_ref_inos])
 			nr_ref_inos++;
 	}
diff --git a/fs/ntfs/namei.c b/fs/ntfs/namei.c
index 96c450e62efca..c4f82846c58c3 100644
--- a/fs/ntfs/namei.c
+++ b/fs/ntfs/namei.c
@@ -344,9 +344,9 @@ static int ntfs_sd_add_everyone(struct ntfs_inode *ni)
 	sd_len = sizeof(struct security_descriptor_relative) + 2 *
 		(sizeof(struct ntfs_sid) + 8) + sizeof(struct ntfs_acl) +
 		sizeof(struct ntfs_ace) + 4;
-	sd = kmalloc(sd_len, GFP_NOFS);
+	sd = kzalloc(sd_len, GFP_NOFS);
 	if (!sd)
-		return -1;
+		return -ENOMEM;
 
 	sd->revision = 1;
 	sd->control = SE_DACL_PRESENT | SE_SELF_RELATIVE;
diff --git a/fs/ntfs/quota.c b/fs/ntfs/quota.c
deleted file mode 100644
index b443243b58fbf..0000000000000
--- a/fs/ntfs/quota.c
+++ /dev/null
@@ -1,95 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * NTFS kernel quota ($Quota) handling.
- *
- * Copyright (c) 2004 Anton Altaparmakov
- */
-
-#include "index.h"
-#include "quota.h"
-#include "debug.h"
-#include "ntfs.h"
-
-/*
- * ntfs_mark_quotas_out_of_date - mark the quotas out of date on an ntfs volume
- * @vol:	ntfs volume on which to mark the quotas out of date
- *
- * Mark the quotas out of date on the ntfs volume @vol and return 'true' on
- * success and 'false' on error.
- */
-bool ntfs_mark_quotas_out_of_date(struct ntfs_volume *vol)
-{
-	struct ntfs_index_context *ictx;
-	struct quota_control_entry *qce;
-	const __le32 qid = QUOTA_DEFAULTS_ID;
-	int err;
-
-	ntfs_debug("Entering.");
-	if (NVolQuotaOutOfDate(vol))
-		goto done;
-	if (!vol->quota_ino || !vol->quota_q_ino) {
-		ntfs_error(vol->sb, "Quota inodes are not open.");
-		return false;
-	}
-	inode_lock(vol->quota_q_ino);
-	ictx = ntfs_index_ctx_get(NTFS_I(vol->quota_q_ino), I30, 4);
-	if (!ictx) {
-		ntfs_error(vol->sb, "Failed to get index context.");
-		goto err_out;
-	}
-	err = ntfs_index_lookup(&qid, sizeof(qid), ictx);
-	if (err) {
-		if (err == -ENOENT)
-			ntfs_error(vol->sb, "Quota defaults entry is not present.");
-		else
-			ntfs_error(vol->sb, "Lookup of quota defaults entry failed.");
-		goto err_out;
-	}
-	if (ictx->data_len < offsetof(struct quota_control_entry, sid)) {
-		ntfs_error(vol->sb, "Quota defaults entry size is invalid.  Run chkdsk.");
-		goto err_out;
-	}
-	qce = (struct quota_control_entry *)ictx->data;
-	if (le32_to_cpu(qce->version) != QUOTA_VERSION) {
-		ntfs_error(vol->sb,
-			"Quota defaults entry version 0x%x is not supported.",
-			le32_to_cpu(qce->version));
-		goto err_out;
-	}
-	ntfs_debug("Quota defaults flags = 0x%x.", le32_to_cpu(qce->flags));
-	/* If quotas are already marked out of date, no need to do anything. */
-	if (qce->flags & QUOTA_FLAG_OUT_OF_DATE)
-		goto set_done;
-	/*
-	 * If quota tracking is neither requested, nor enabled and there are no
-	 * pending deletes, no need to mark the quotas out of date.
-	 */
-	if (!(qce->flags & (QUOTA_FLAG_TRACKING_ENABLED |
-			QUOTA_FLAG_TRACKING_REQUESTED |
-			QUOTA_FLAG_PENDING_DELETES)))
-		goto set_done;
-	/*
-	 * Set the QUOTA_FLAG_OUT_OF_DATE bit thus marking quotas out of date.
-	 * This is verified on WinXP to be sufficient to cause windows to
-	 * rescan the volume on boot and update all quota entries.
-	 */
-	qce->flags |= QUOTA_FLAG_OUT_OF_DATE;
-	/* Ensure the modified flags are written to disk. */
-	ntfs_index_entry_mark_dirty(ictx);
-set_done:
-	ntfs_index_ctx_put(ictx);
-	inode_unlock(vol->quota_q_ino);
-	/*
-	 * We set the flag so we do not try to mark the quotas out of date
-	 * again on remount.
-	 */
-	NVolSetQuotaOutOfDate(vol);
-done:
-	ntfs_debug("Done.");
-	return true;
-err_out:
-	if (ictx)
-		ntfs_index_ctx_put(ictx);
-	inode_unlock(vol->quota_q_ino);
-	return false;
-}
diff --git a/fs/ntfs/quota.h b/fs/ntfs/quota.h
deleted file mode 100644
index 4b7322661a326..0000000000000
--- a/fs/ntfs/quota.h
+++ /dev/null
@@ -1,15 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Defines for NTFS kernel quota ($Quota) handling.
- *
- * Copyright (c) 2004 Anton Altaparmakov
- */
-
-#ifndef _LINUX_NTFS_QUOTA_H
-#define _LINUX_NTFS_QUOTA_H
-
-#include "volume.h"
-
-bool ntfs_mark_quotas_out_of_date(struct ntfs_volume *vol);
-
-#endif /* _LINUX_NTFS_QUOTA_H */
diff --git a/fs/ntfs/runlist.c b/fs/ntfs/runlist.c
index da21dbeaaf667..e7de3d01257e7 100644
--- a/fs/ntfs/runlist.c
+++ b/fs/ntfs/runlist.c
@@ -2056,10 +2056,11 @@ struct runlist_element *ntfs_rl_collapse_range(struct runlist_element *dst_rl, i
 	 * consists of holes.
 	 */
 	merge_cnt = 0;
-	i = new_1st_cnt == 0 ? 1 : new_1st_cnt;
-	if (ntfs_rle_lcn_contiguous(&new_rl[i - 1], &new_rl[i])) {
-		/* Merge right and left */
-		s_rl =  &new_rl[new_1st_cnt - 1];
+	if (new_1st_cnt > 0 &&
+	    ntfs_rle_lcn_contiguous(&new_rl[new_1st_cnt - 1],
+				    &new_rl[new_1st_cnt])) {
+		/* Merge right and left. */
+		s_rl = &new_rl[new_1st_cnt - 1];
 		s_rl->length += s_rl[1].length;
 		merge_cnt = 1;
 	}
diff --git a/fs/ntfs/super.c b/fs/ntfs/super.c
index 22dc7865eca79..c2db3c3b60587 100644
--- a/fs/ntfs/super.c
+++ b/fs/ntfs/super.c
@@ -17,7 +17,6 @@
 
 #include "sysctl.h"
 #include "logfile.h"
-#include "quota.h"
 #include "index.h"
 #include "ntfs.h"
 #include "ea.h"
@@ -240,8 +239,7 @@ static int ntfs_reconfigure(struct fs_context *fc)
 	 * flags are set.  Also, empty the logfile journal as it would become
 	 * stale as soon as something is written to the volume and mark the
 	 * volume dirty so that chkdsk is run if the volume is not umounted
-	 * cleanly.  Finally, mark the quotas out of date so Windows rescans
-	 * the volume on boot and updates them.
+	 * cleanly.
 	 *
 	 * When remounting read-only, mark the volume clean if no volume errors
 	 * have occurred.
@@ -274,12 +272,6 @@ static int ntfs_reconfigure(struct fs_context *fc)
 			NVolSetErrors(vol);
 			return -EROFS;
 		}
-		if (!ntfs_mark_quotas_out_of_date(vol)) {
-			ntfs_error(sb, "Failed to mark quotas out of date%s",
-					es);
-			NVolSetErrors(vol);
-			return -EROFS;
-		}
 	} else if (!sb_rdonly(sb) && (fc->sb_flags & SB_RDONLY)) {
 		/* Remounting read-only. */
 		if (!NVolErrors(vol)) {
@@ -413,6 +405,7 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label)
 {
 	struct ntfs_inode *vol_ni = NTFS_I(vol->vol_ino);
 	struct ntfs_attr_search_ctx *ctx;
+	char *new_label;
 	__le16 *uname;
 	int uname_len, ret;
 
@@ -425,7 +418,7 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label)
 		return uname_len;
 	}
 
-	if (uname_len  > NTFS_MAX_LABEL_LEN) {
+	if (uname_len > NTFS_MAX_LABEL_LEN) {
 		ntfs_error(vol->sb,
 			   "Volume label is too long (max %d characters).",
 			   NTFS_MAX_LABEL_LEN);
@@ -433,11 +426,22 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label)
 		return -EINVAL;
 	}
 
+	/*
+	 * Allocate the in-memory label copy up front. If kstrdup() fails we
+	 * bail out before touching on-disk metadata, so the in-memory label
+	 * and the on-disk label stay in sync.
+	 */
+	new_label = kstrdup(label, GFP_KERNEL);
+	if (!new_label) {
+		kvfree(uname);
+		return -ENOMEM;
+	}
+
 	mutex_lock(&vol_ni->mrec_lock);
 	ctx = ntfs_attr_get_search_ctx(vol_ni, NULL);
 	if (!ctx) {
 		ret = -ENOMEM;
-		goto  out;
+		goto out;
 	}
 
 	if (!ntfs_attr_lookup(AT_VOLUME_NAME, NULL, 0, 0, 0, NULL, 0,
@@ -450,12 +454,14 @@ int ntfs_write_volume_label(struct ntfs_volume *vol, char *label)
 out:
 	mutex_unlock(&vol_ni->mrec_lock);
 	kvfree(uname);
-	mark_inode_dirty_sync(vol->vol_ino);
 
 	if (ret >= 0) {
 		kfree(vol->volume_label);
-		vol->volume_label = kstrdup(label, GFP_KERNEL);
+		vol->volume_label = new_label;
+		mark_inode_dirty_sync(vol->vol_ino);
 		ret = 0;
+	} else {
+		kfree(new_label);
 	}
 	return ret;
 }
@@ -979,6 +985,13 @@ static bool check_mft_mirror(struct ntfs_volume *vol)
 			    ntfs_is_baad_recordp((__le32 *)kmirr))
 				bytes = vol->mft_record_size;
 		}
+		/* Compare the two records. */
+		if (memcmp(kmft, kmirr, bytes)) {
+			ntfs_error(sb,
+				   "$MFT and $MFTMirr record %i do not match.  Run chkdsk.",
+				   i);
+			goto mm_unmap_out;
+		}
 		kmft += vol->mft_record_size;
 		kmirr += vol->mft_record_size;
 	} while (++i < vol->mftmirr_size);
@@ -1154,73 +1167,6 @@ static int check_windows_hibernation_status(struct ntfs_volume *vol)
 	return ret;
 }
 
-/*
- * load_and_init_quota - load and setup the quota file for a volume if present
- * @vol:	ntfs super block describing device whose quota file to load
- *
- * Return 'true' on success or 'false' on error.  If $Quota is not present, we
- * leave vol->quota_ino as NULL and return success.
- */
-static bool load_and_init_quota(struct ntfs_volume *vol)
-{
-	static const __le16 Quota[7] = { cpu_to_le16('$'),
-			cpu_to_le16('Q'), cpu_to_le16('u'),
-			cpu_to_le16('o'), cpu_to_le16('t'),
-			cpu_to_le16('a'), 0 };
-	static __le16 Q[3] = { cpu_to_le16('$'),
-			cpu_to_le16('Q'), 0 };
-	struct ntfs_name *name = NULL;
-	u64 mref;
-	struct inode *tmp_ino;
-
-	ntfs_debug("Entering.");
-	/*
-	 * Find the inode number for the quota file by looking up the filename
-	 * $Quota in the extended system files directory $Extend.
-	 */
-	inode_lock(vol->extend_ino);
-	mref = ntfs_lookup_inode_by_name(NTFS_I(vol->extend_ino), Quota, 6,
-			&name);
-	inode_unlock(vol->extend_ino);
-	kfree(name);
-	if (IS_ERR_MREF(mref)) {
-		/*
-		 * If the file does not exist, quotas are disabled and have
-		 * never been enabled on this volume, just return success.
-		 */
-		if (MREF_ERR(mref) == -ENOENT) {
-			ntfs_debug("$Quota not present.  Volume does not have quotas enabled.");
-			/*
-			 * No need to try to set quotas out of date if they are
-			 * not enabled.
-			 */
-			NVolSetQuotaOutOfDate(vol);
-			return true;
-		}
-		/* A real error occurred. */
-		ntfs_error(vol->sb, "Failed to find inode number for $Quota.");
-		return false;
-	}
-	/* Get the inode. */
-	tmp_ino = ntfs_iget(vol->sb, MREF(mref));
-	if (IS_ERR(tmp_ino)) {
-		if (!IS_ERR(tmp_ino))
-			iput(tmp_ino);
-		ntfs_error(vol->sb, "Failed to load $Quota.");
-		return false;
-	}
-	vol->quota_ino = tmp_ino;
-	/* Get the $Q index allocation attribute. */
-	tmp_ino = ntfs_index_iget(vol->quota_ino, Q, 2);
-	if (IS_ERR(tmp_ino)) {
-		ntfs_error(vol->sb, "Failed to load $Quota/$Q index.");
-		return false;
-	}
-	vol->quota_q_ino = tmp_ino;
-	ntfs_debug("Done.");
-	return true;
-}
-
 /*
  * load_and_init_attrdef - load the attribute definitions table for a volume
  * @vol:	ntfs super block describing device whose attrdef to load
@@ -1638,18 +1584,6 @@ static bool load_system_files(struct ntfs_volume *vol)
 		ntfs_error(sb, "Failed to load $Extend.");
 		goto iput_sec_err_out;
 	}
-	/* Find the quota file, load it if present, and set it up. */
-	if (!load_and_init_quota(vol) &&
-	    vol->on_errors == ON_ERRORS_REMOUNT_RO) {
-		static const char *es1 = "Failed to load $Quota";
-		static const char *es2 = ".  Run chkdsk.";
-
-		sb->s_flags |= SB_RDONLY;
-		ntfs_error(sb, "%s.  Mounting read-only%s", es1, es2);
-		/* This will prevent a read-write remount. */
-		NVolSetErrors(vol);
-	}
-
 	return true;
 
 iput_sec_err_out:
@@ -1671,7 +1605,7 @@ static bool load_system_files(struct ntfs_volume *vol)
 iput_upcase_err_out:
 	vol->upcase_len = 0;
 	mutex_lock(&ntfs_lock);
-	if (vol->upcase == default_upcase) {
+	if (vol->upcase && vol->upcase == default_upcase) {
 		ntfs_nr_upcase_users--;
 		vol->upcase = NULL;
 	}
@@ -1701,7 +1635,7 @@ static void ntfs_volume_free(struct ntfs_volume *vol)
 	 * the number of upcase users if we are a user.
 	 */
 	mutex_lock(&ntfs_lock);
-	if (vol->upcase == default_upcase) {
+	if (vol->upcase && vol->upcase == default_upcase) {
 		ntfs_nr_upcase_users--;
 		vol->upcase = NULL;
 	}
@@ -1747,10 +1681,6 @@ static void ntfs_put_super(struct super_block *sb)
 
 	/* NTFS 3.0+ specific. */
 	if (vol->major_ver >= 3) {
-		if (vol->quota_q_ino)
-			ntfs_commit_inode(vol->quota_q_ino);
-		if (vol->quota_ino)
-			ntfs_commit_inode(vol->quota_ino);
 		if (vol->extend_ino)
 			ntfs_commit_inode(vol->extend_ino);
 		if (vol->secure_ino)
@@ -1799,14 +1729,6 @@ static void ntfs_put_super(struct super_block *sb)
 
 	/* NTFS 3.0+ specific clean up. */
 	if (vol->major_ver >= 3) {
-		if (vol->quota_q_ino) {
-			iput(vol->quota_q_ino);
-			vol->quota_q_ino = NULL;
-		}
-		if (vol->quota_ino) {
-			iput(vol->quota_ino);
-			vol->quota_ino = NULL;
-		}
 		if (vol->extend_ino) {
 			iput(vol->extend_ino);
 			vol->extend_ino = NULL;
@@ -2455,14 +2377,6 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc)
 	vol->vol_ino = NULL;
 	/* NTFS 3.0+ specific clean up. */
 	if (vol->major_ver >= 3) {
-		if (vol->quota_q_ino) {
-			iput(vol->quota_q_ino);
-			vol->quota_q_ino = NULL;
-		}
-		if (vol->quota_ino) {
-			iput(vol->quota_ino);
-			vol->quota_ino = NULL;
-		}
 		if (vol->extend_ino) {
 			iput(vol->extend_ino);
 			vol->extend_ino = NULL;
@@ -2494,7 +2408,7 @@ static int ntfs_fill_super(struct super_block *sb, struct fs_context *fc)
 	}
 	vol->upcase_len = 0;
 	mutex_lock(&ntfs_lock);
-	if (vol->upcase == default_upcase) {
+	if (vol->upcase && vol->upcase == default_upcase) {
 		ntfs_nr_upcase_users--;
 		vol->upcase = NULL;
 	}
diff --git a/fs/ntfs/volume.h b/fs/ntfs/volume.h
index af41427ec622c..e13e1423b2a92 100644
--- a/fs/ntfs/volume.h
+++ b/fs/ntfs/volume.h
@@ -76,8 +76,6 @@
  * @root_ino: The VFS inode of the root directory.
  * @secure_ino: The VFS inode of $Secure (NTFS3.0+ only, otherwise NULL).
  * @extend_ino: The VFS inode of $Extend (NTFS3.0+ only, otherwise NULL).
- * @quota_ino: The VFS inode of $Quota.
- * @quota_q_ino: Attribute inode for $Quota/$Q.
  * @nls_map: NLS (National Language Support) table.
  * @nls_utf8: NLS table for UTF-8.
  * @free_waitq: Wait queue for threads waiting for free clusters or MFT records.
@@ -141,8 +139,6 @@ struct ntfs_volume {
 	struct inode *root_ino;
 	struct inode *secure_ino;
 	struct inode *extend_ino;
-	struct inode *quota_ino;
-	struct inode *quota_q_ino;
 	struct nls_table *nls_map;
 	bool nls_utf8;
 	wait_queue_head_t free_waitq;
@@ -165,7 +161,6 @@ struct ntfs_volume {
  *				Otherwise be case insensitive but still
  *				create file names in POSIX namespace.
  * NV_LogFileEmpty		LogFile journal is empty.
- * NV_QuotaOutOfDate		Quota is out of date.
  * NV_UsnJrnlStamped		UsnJrnl has been stamped.
  * NV_ReadOnly			Volume is mounted read-only.
  * NV_Compression		Volume supports compression.
@@ -186,7 +181,6 @@ enum {
 	NV_ShowSystemFiles,
 	NV_CaseSensitive,
 	NV_LogFileEmpty,
-	NV_QuotaOutOfDate,
 	NV_UsnJrnlStamped,
 	NV_ReadOnly,
 	NV_Compression,
@@ -223,7 +217,6 @@ DEFINE_NVOL_BIT_OPS(Errors)
 DEFINE_NVOL_BIT_OPS(ShowSystemFiles)
 DEFINE_NVOL_BIT_OPS(CaseSensitive)
 DEFINE_NVOL_BIT_OPS(LogFileEmpty)
-DEFINE_NVOL_BIT_OPS(QuotaOutOfDate)
 DEFINE_NVOL_BIT_OPS(UsnJrnlStamped)
 DEFINE_NVOL_BIT_OPS(ReadOnly)
 DEFINE_NVOL_BIT_OPS(Compression)
diff --git a/fs/ntfs3/dir.c b/fs/ntfs3/dir.c
index d99ab086ef6fe..a052ba9250e73 100644
--- a/fs/ntfs3/dir.c
+++ b/fs/ntfs3/dir.c
@@ -489,10 +489,17 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx)
 			goto out;
 	}
 
+	/*
+	 * Keep directory metadata stable for the whole walk. Loading subrecords
+	 * once is not enough if concurrent writeback can still compact ATTR_LIST
+	 * entries and free the record that ntfs_read_hdr() is currently walking.
+	 */
+	ni_lock(ni);
+
 	root = indx_get_root(&ni->dir, ni, NULL, NULL);
 	if (!root) {
 		err = -EINVAL;
-		goto out;
+		goto out_unlock;
 	}
 
 	if (pos >= sbi->record_size) {
@@ -503,7 +510,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx)
 		 */
 		err = ntfs_read_hdr(sbi, ni, &root->ihdr, 0, pos, name, ctx);
 		if (err)
-			goto out;
+			goto out_unlock;
 		bit = 0;
 	}
 
@@ -514,7 +521,7 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx)
 		/* Get the next used index. */
 		err = indx_used_bit(&ni->dir, ni, &bit);
 		if (err)
-			goto out;
+			goto out_unlock;
 
 		if (bit == MINUS_ONE_T) {
 			/* no more used indexes. end of dir. */
@@ -524,13 +531,13 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx)
 		if (bit >= max_bit) {
 			/* Corrupted directory. */
 			err = -EINVAL;
-			goto out;
+			goto out_unlock;
 		}
 
 		err = indx_read_ra(&ni->dir, ni, bit << ni->dir.idx2vbn_bits,
 				   &node, &file->f_ra);
 		if (err)
-			goto out;
+			goto out_unlock;
 
 		/*
 		 * Add each name from index in 'ctx'.
@@ -539,9 +546,12 @@ static int ntfs_readdir(struct file *file, struct dir_context *ctx)
 				    ((u64)bit << index_bits) + sbi->record_size,
 				    pos, name, ctx);
 		if (err)
-			goto out;
+			goto out_unlock;
 	}
 
+out_unlock:
+	ni_unlock(ni);
+
 out:
 	kfree(name);
 	put_indx_node(node);
diff --git a/fs/ntfs3/file.c b/fs/ntfs3/file.c
index b041639ab406f..ad9350d7fc3fd 100644
--- a/fs/ntfs3/file.c
+++ b/fs/ntfs3/file.c
@@ -180,6 +180,34 @@ long ntfs_compat_ioctl(struct file *filp, u32 cmd, unsigned long arg)
 }
 #endif
 
+/*
+ * ntfs_fileattr_get - inode_operations::fileattr_get
+ */
+int ntfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
+{
+	struct inode *inode = d_inode(dentry);
+	struct ntfs_sb_info *sbi = inode->i_sb->s_fs_info;
+
+	/* Avoid any operation if inode is bad. */
+	if (unlikely(is_bad_ni(ntfs_i(inode))))
+		return -EINVAL;
+
+	/*
+	 * NTFS preserves case (the default). Case sensitivity depends on
+	 * mount options: with "nocase", NTFS is case-insensitive;
+	 * otherwise it is case-sensitive.
+	 */
+	if (sbi->options->nocase) {
+		fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
+		fa->flags |= FS_CASEFOLD_FL;
+	}
+	if (inode->i_flags & S_IMMUTABLE) {
+		fa->fsx_xflags |= FS_XFLAG_IMMUTABLE;
+		fa->flags |= FS_IMMUTABLE_FL;
+	}
+	return 0;
+}
+
 /*
  * ntfs_getattr - inode_operations::getattr
  */
@@ -1547,6 +1575,7 @@ const struct inode_operations ntfs_file_inode_operations = {
 	.get_acl	= ntfs_get_acl,
 	.set_acl	= ntfs_set_acl,
 	.fiemap		= ntfs_fiemap,
+	.fileattr_get	= ntfs_fileattr_get,
 };
 
 const struct file_operations ntfs_file_operations = {
diff --git a/fs/ntfs3/fslog.c b/fs/ntfs3/fslog.c
index acfa18b84401e..0d1be4da3e4f9 100644
--- a/fs/ntfs3/fslog.c
+++ b/fs/ntfs3/fslog.c
@@ -3368,7 +3368,10 @@ static int do_action(struct ntfs_log *log, struct OPEN_ATTR_ENRTY *oe,
 		memmove(Add2Ptr(attr, aoff), data, dlen);
 
 		if (run_get_highest_vcn(le64_to_cpu(attr->nres.svcn),
-					attr_run(attr), &t64)) {
+					attr_run(attr),
+				        le32_to_cpu(attr->size) - 
+				                le16_to_cpu(attr->nres.run_off),	
+					&t64)) {
 			goto dirty_vol;
 		}
 
diff --git a/fs/ntfs3/index.c b/fs/ntfs3/index.c
index 5344b29b0577f..d65984d9dc079 100644
--- a/fs/ntfs3/index.c
+++ b/fs/ntfs3/index.c
@@ -1506,6 +1506,7 @@ static int indx_add_allocate(struct ntfs_index *indx, struct ntfs_inode *ni,
 
 	if (bit != MINUS_ONE_T) {
 		bmp = NULL;
+		bmp_size = bmp_size_v = 0;
 	} else {
 		if (bmp->non_res) {
 			bmp_size = le64_to_cpu(bmp->nres.data_size);
@@ -1742,6 +1743,22 @@ static int indx_insert_into_root(struct ntfs_index *indx, struct ntfs_inode *ni,
 	hdr_used = le32_to_cpu(hdr->used);
 	hdr_total = le32_to_cpu(hdr->total);
 
+	/*
+	 * The destination INDEX_BUFFER has 'hdr_total' bytes of payload
+	 * available after the header, of which 'hdr_used' are already
+	 * consumed by the single terminal END entry installed by
+	 * indx_new(). A crafted image can present a resident root whose
+	 * non-last entries (summing to 'to_move') exceed what fits in
+	 * this buffer; copying them unchecked would overrun the
+	 * kmalloc(1u << indx->index_bits) allocation backing the new
+	 * buffer. Reject the copy in that case.
+	 */
+	if (to_move > hdr_total - hdr_used) {
+		err = -EINVAL;
+		ntfs_set_state(sbi, NTFS_DIRTY_ERROR);
+		goto out_put_n;
+	}
+
 	/* Copy root entries into new buffer. */
 	hdr_insert_head(hdr, re, to_move);
 
@@ -1846,6 +1863,20 @@ indx_insert_into_buffer(struct ntfs_index *indx, struct ntfs_inode *ni,
 	memcpy(up_e, sp, sp_size);
 
 	used1 = le32_to_cpu(hdr1->used);
+
+	/*
+	 * hdr_find_split does not validate per-entry sizes, so a crafted
+	 * NTFS_DE whose le16 size field is out of range can place sp such
+	 * that (PtrOffset(hdr1, sp) + sp_size) exceeds used1. Without this
+	 * guard the u32 'used = used1 - to_copy - sp_size' underflows and
+	 * the subsequent memmove count becomes a near-4-GiB value,
+	 * triggering an out-of-bounds kernel write.
+	 */
+	if (PtrOffset(hdr1, sp) + sp_size > used1) {
+		err = -EINVAL;
+		goto out;
+	}
+
 	hdr1_saved = kmemdup(hdr1, used1, GFP_NOFS);
 	if (!hdr1_saved) {
 		err = -ENOMEM;
@@ -2022,13 +2053,21 @@ int indx_insert_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
 static struct indx_node *indx_find_buffer(struct ntfs_index *indx,
 					  struct ntfs_inode *ni,
 					  const struct INDEX_ROOT *root,
-					  __le64 vbn, struct indx_node *n)
+					  __le64 vbn, struct indx_node *n,
+					  int depth)
 {
 	int err;
 	const struct NTFS_DE *e;
 	struct indx_node *r;
 	const struct INDEX_HDR *hdr = n ? &n->index->ihdr : &root->ihdr;
 
+	/*
+	 * Limit recursion depth to prevent stack overflow from crafted
+	 * images.  Use the same bound as the fnd->nodes array (20).
+	 */
+	if (depth > ARRAY_SIZE(((struct ntfs_fnd *)NULL)->nodes))
+		return ERR_PTR(-EINVAL);
+
 	/* Step 1: Scan one level. */
 	for (e = hdr_first_de(hdr);; e = hdr_next_de(hdr, e)) {
 		if (!e)
@@ -2049,7 +2088,8 @@ static struct indx_node *indx_find_buffer(struct ntfs_index *indx,
 			if (err)
 				return ERR_PTR(err);
 
-			r = indx_find_buffer(indx, ni, root, vbn, n);
+			r = indx_find_buffer(indx, ni, root, vbn, n,
+					     depth + 1);
 			if (r)
 				return r;
 		}
@@ -2462,7 +2502,7 @@ int indx_delete_entry(struct ntfs_index *indx, struct ntfs_inode *ni,
 
 		fnd_clear(fnd);
 
-		in = indx_find_buffer(indx, ni, root, sub_vbn, NULL);
+		in = indx_find_buffer(indx, ni, root, sub_vbn, NULL, 0);
 		if (IS_ERR(in)) {
 			err = PTR_ERR(in);
 			goto out;
diff --git a/fs/ntfs3/lznt.c b/fs/ntfs3/lznt.c
index fdc9b2ebf3410..f818d97850049 100644
--- a/fs/ntfs3/lznt.c
+++ b/fs/ntfs3/lznt.c
@@ -240,7 +240,7 @@ static inline ssize_t decompress_chunk(u8 *unc, u8 *unc_end, const u8 *cmpr,
 		if (up - unc > LZNT_CHUNK_SIZE)
 			return -EINVAL;
 		/* Correct index */
-		while (unc + s_max_off[index] < up)
+		while (index < ARRAY_SIZE(s_max_off) - 1 && unc + s_max_off[index] < up)
 			index += 1;
 
 		/* Check the current flag for zero. */
diff --git a/fs/ntfs3/namei.c b/fs/ntfs3/namei.c
index b2af8f695e60f..d0740957b7df6 100644
--- a/fs/ntfs3/namei.c
+++ b/fs/ntfs3/namei.c
@@ -340,7 +340,7 @@ static int ntfs_rename(struct mnt_idmap *idmap, struct inode *dir,
 			ntfs_sync_inode(dir);
 
 		if (IS_DIRSYNC(new_dir))
-			ntfs_sync_inode(inode);
+			ntfs_sync_inode(new_dir);
 	}
 
 	if (dir_ni != new_dir_ni)
@@ -518,6 +518,7 @@ const struct inode_operations ntfs_dir_inode_operations = {
 	.getattr	= ntfs_getattr,
 	.listxattr	= ntfs_listxattr,
 	.fiemap		= ntfs_fiemap,
+	.fileattr_get	= ntfs_fileattr_get,
 };
 
 const struct inode_operations ntfs_special_inode_operations = {
diff --git a/fs/ntfs3/ntfs_fs.h b/fs/ntfs3/ntfs_fs.h
index bbf3b6a1dcbee..ecdd803c6167e 100644
--- a/fs/ntfs3/ntfs_fs.h
+++ b/fs/ntfs3/ntfs_fs.h
@@ -529,6 +529,7 @@ bool dir_is_empty(struct inode *dir);
 extern const struct file_operations ntfs_dir_operations;
 
 /* Globals from file.c */
+int ntfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
 int ntfs_getattr(struct mnt_idmap *idmap, const struct path *path,
 		 struct kstat *stat, u32 request_mask, u32 flags);
 int ntfs_setattr(struct mnt_idmap *idmap, struct dentry *dentry,
@@ -877,7 +878,8 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino,
 #else
 #define run_unpack_ex run_unpack
 #endif
-int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn);
+int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, 
+		       u64 *highest_vcn);
 int run_clone(const struct runs_tree *run, struct runs_tree *new_run);
 bool run_remove_range(struct runs_tree *run, CLST vcn, CLST len, CLST *done);
 CLST run_len(const struct runs_tree *run);
diff --git a/fs/ntfs3/run.c b/fs/ntfs3/run.c
index 1ce7d92fb2748..19aa044fd1fcc 100644
--- a/fs/ntfs3/run.c
+++ b/fs/ntfs3/run.c
@@ -1205,18 +1205,23 @@ int run_unpack_ex(struct runs_tree *run, struct ntfs_sb_info *sbi, CLST ino,
  * Return the highest vcn from a mapping pairs array
  * it used while replaying log file.
  */
-int run_get_highest_vcn(CLST vcn, const u8 *run_buf, u64 *highest_vcn)
+int run_get_highest_vcn(CLST vcn, const u8 *run_buf, size_t run_buf_size, 
+		       u64 *highest_vcn)
 {
+	const u8 *run_last = run_buf + run_buf_size;
 	u64 vcn64 = vcn;
 	u8 size_size;
 
-	while ((size_size = *run_buf & 0xF)) {
+	while (run_buf < run_last && (size_size = *run_buf & 0xF)) {
 		u8 offset_size = *run_buf++ >> 4;
 		u64 len;
 
 		if (size_size > 8 || offset_size > 8)
 			return -EINVAL;
 
+		if (run_buf + size_size + offset_size > run_last) 
+			return -EINVAL;
+
 		len = run_unpack_s64(run_buf, size_size, 0);
 		if (!len)
 			return -EINVAL;
diff --git a/fs/ntfs3/super.c b/fs/ntfs3/super.c
index 004f599375597..3305fe406cb22 100644
--- a/fs/ntfs3/super.c
+++ b/fs/ntfs3/super.c
@@ -1174,7 +1174,10 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size,
 	rec->total = cpu_to_le32(sbi->record_size);
 	((struct ATTRIB *)Add2Ptr(rec, ao))->type = ATTR_END;
 
-	sb_set_blocksize(sb, min_t(u32, sbi->cluster_size, PAGE_SIZE));
+	if (!sb_set_blocksize(sb, min_t(u32, sbi->cluster_size, PAGE_SIZE))) {
+		err = -EINVAL;
+		goto out;
+	}
 
 	sbi->block_mask = sb->s_blocksize - 1;
 	sbi->blocks_per_cluster = sbi->cluster_size >> sb->s_blocksize_bits;
@@ -1225,7 +1228,8 @@ static int ntfs_init_from_boot(struct super_block *sb, u32 sector_size,
 			/*
 			 * Try alternative boot (last sector)
 			 */
-			sb_set_blocksize(sb, block_size);
+			if (!sb_set_blocksize(sb, block_size))
+				return -EINVAL;
 			hint = "Alternative boot";
 			dev_size = dev_size0; /* restore original size. */
 			goto read_boot;
diff --git a/fs/ntfs3/xattr.c b/fs/ntfs3/xattr.c
index 9eeac0ab2b714..7e5118247660d 100644
--- a/fs/ntfs3/xattr.c
+++ b/fs/ntfs3/xattr.c
@@ -867,7 +867,9 @@ static noinline int ntfs_setxattr(const struct xattr_handler *handler,
 	if (!strcmp(name, SYSTEM_DOS_ATTRIB)) {
 		if (sizeof(u8) != size)
 			goto out;
-		new_fa = cpu_to_le32(*(u8 *)value);
+		/* system.dos_attrib only covers the low DOS attribute byte. */
+		new_fa = (ni->std_fa & ~cpu_to_le32(0xff)) |
+			 cpu_to_le32(*(u8 *)value);
 		goto set_new_fa;
 	}
 
diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c
index 7df9921c1a389..d6e977ba65656 100644
--- a/fs/ocfs2/file.c
+++ b/fs/ocfs2/file.c
@@ -444,21 +444,26 @@ int ocfs2_truncate_file(struct inode *inode,
 	struct ocfs2_dinode *fe = NULL;
 	struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
 
-	/* We trust di_bh because it comes from ocfs2_inode_lock(), which
-	 * already validated it */
+	/*
+	 * On local mounts ocfs2_inode_lock_update() skips the inode
+	 * refresh path, so truncation still needs to reject an inode
+	 * state that no longer matches di_bh.
+	 */
 	fe = (struct ocfs2_dinode *) di_bh->b_data;
 
 	trace_ocfs2_truncate_file((unsigned long long)OCFS2_I(inode)->ip_blkno,
 				  (unsigned long long)le64_to_cpu(fe->i_size),
 				  (unsigned long long)new_i_size);
 
-	mlog_bug_on_msg(le64_to_cpu(fe->i_size) != i_size_read(inode),
-			"Inode %llu, inode i_size = %lld != di "
-			"i_size = %llu, i_flags = 0x%x\n",
-			(unsigned long long)OCFS2_I(inode)->ip_blkno,
-			i_size_read(inode),
-			(unsigned long long)le64_to_cpu(fe->i_size),
-			le32_to_cpu(fe->i_flags));
+	if (unlikely(le64_to_cpu(fe->i_size) != i_size_read(inode))) {
+		status = ocfs2_error(inode->i_sb,
+				     "Inode %llu has inconsistent i_size: inode = %lld, dinode = %llu, i_flags = 0x%x\n",
+				     (unsigned long long)OCFS2_I(inode)->ip_blkno,
+				     i_size_read(inode),
+				     (unsigned long long)le64_to_cpu(fe->i_size),
+				     le32_to_cpu(fe->i_flags));
+		goto bail;
+	}
 
 	if (new_i_size > le64_to_cpu(fe->i_size)) {
 		trace_ocfs2_truncate_file_error(
diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c
index a510a0eb1adcc..432eac01c1763 100644
--- a/fs/ocfs2/inode.c
+++ b/fs/ocfs2/inode.c
@@ -13,6 +13,7 @@
 #include <linux/pagemap.h>
 #include <linux/quotaops.h>
 #include <linux/iversion.h>
+#include <linux/fs_dirent.h>
 
 #include <asm/byteorder.h>
 
@@ -64,7 +65,40 @@ static int ocfs2_filecheck_read_inode_block_full(struct inode *inode,
 static int ocfs2_filecheck_validate_inode_block(struct super_block *sb,
 						struct buffer_head *bh);
 static int ocfs2_filecheck_repair_inode_block(struct super_block *sb,
-					      struct buffer_head *bh);
+						      struct buffer_head *bh);
+
+static bool ocfs2_valid_inode_mode(umode_t mode)
+{
+	return fs_umode_to_ftype(mode) != FT_UNKNOWN;
+}
+
+static bool ocfs2_dinode_has_unexpected_rdev(struct ocfs2_dinode *di)
+{
+	umode_t mode = le16_to_cpu(di->i_mode);
+
+	if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL)
+		return false;
+
+	return !S_ISCHR(mode) && !S_ISBLK(mode) && di->id1.dev1.i_rdev != 0;
+}
+
+static bool ocfs2_dinode_has_size_without_clusters(struct super_block *sb,
+						   struct ocfs2_dinode *di)
+{
+	umode_t mode = le16_to_cpu(di->i_mode);
+
+	if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL)
+		return false;
+	if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL)
+		return false;
+	if (!le64_to_cpu(di->i_size) || le32_to_cpu(di->i_clusters))
+		return false;
+
+	if (S_ISDIR(mode))
+		return true;
+
+	return !ocfs2_sparse_alloc(OCFS2_SB(sb)) && S_ISREG(mode);
+}
 
 void ocfs2_set_inode_flags(struct inode *inode)
 {
@@ -1494,6 +1528,86 @@ int ocfs2_validate_inode_block(struct super_block *sb,
 		goto bail;
 	}
 
+	/*
+	 * Reject dinodes whose i_mode does not name one of the seven
+	 * canonical POSIX file types.  ocfs2_populate_inode() copies
+	 * i_mode verbatim into inode->i_mode and then dispatches via
+	 * switch (mode & S_IFMT) to file/dir/symlink/special_file iops;
+	 * an unrecognised type falls into ocfs2_special_file_iops with
+	 * init_special_inode(), which interprets i_rdev.  Constrain the
+	 * type here so the dispatch only ever sees a value mkfs.ocfs2 /
+	 * VFS can produce.
+	 */
+	if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) {
+		rc = ocfs2_error(sb,
+				 "Invalid dinode #%llu: mode 0%o has unknown file type\n",
+				 (unsigned long long)bh->b_blocknr,
+				 le16_to_cpu(di->i_mode));
+		goto bail;
+	}
+
+	/*
+	 * id1.dev1.i_rdev is the device-number arm of the id1 union and
+	 * is only meaningful for character and block device inodes.  For
+	 * any other regular user-visible file type the on-disk value
+	 * must be zero.  ocfs2_populate_inode() currently runs
+	 *
+	 *     inode->i_rdev = huge_decode_dev(le64_to_cpu(fe->id1.dev1.i_rdev));
+	 *
+	 * unconditionally, before the S_IFMT switch decides whether the
+	 * inode is a special file.  As a result, an i_rdev value present
+	 * on a non-device inode is silently published into the in-core
+	 * inode; a subsequent forced re-read or in-core mode mutation
+	 * (cluster peer with raw write access to the shared LUN,
+	 * on-disk corruption, or a separately forged dinode) can then
+	 * expose the attacker-controlled device number to
+	 * init_special_inode() without ever showing an unusual i_mode
+	 * at validation time.
+	 *
+	 * System inodes (OCFS2_SYSTEM_FL) legitimately use the bitmap1
+	 * and journal1 arms of the same union (allocator i_used /
+	 * i_total counters and the journal ij_flags /
+	 * ij_recovery_generation pair); those bytes are not an i_rdev
+	 * and must not be checked here.  Restrict the cross-check to
+	 * non-system inodes, which is the full attacker-controllable
+	 * surface.
+	 */
+	if (ocfs2_dinode_has_unexpected_rdev(di)) {
+		rc = ocfs2_error(sb,
+				 "Invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n",
+				 (unsigned long long)bh->b_blocknr,
+				 le16_to_cpu(di->i_mode),
+				 (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev));
+		goto bail;
+	}
+
+	/*
+	 * Non-inline directories must not have i_size without allocated
+	 * clusters: directory growth adds storage before advancing i_size,
+	 * and readdir walks i_size block-by-block.  A forged directory
+	 * with zero clusters and a huge i_size would repeatedly fault on
+	 * holes while advancing through the claimed size.
+	 *
+	 * Non-inline regular files have the same invariant on non-sparse
+	 * volumes.  Sparse regular files are different: truncate can
+	 * legitimately grow i_size without allocating clusters, so keep
+	 * the sparse-alloc carveout for S_IFREG only.  System inodes and
+	 * inline-data dinodes have their own storage rules.
+	 */
+	if (ocfs2_dinode_has_size_without_clusters(sb, di)) {
+		if (S_ISDIR(le16_to_cpu(di->i_mode)))
+			rc = ocfs2_error(sb,
+					 "Invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n",
+					 (unsigned long long)bh->b_blocknr,
+					 (unsigned long long)le64_to_cpu(di->i_size));
+		else
+			rc = ocfs2_error(sb,
+					 "Invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n",
+					 (unsigned long long)bh->b_blocknr,
+					 (unsigned long long)le64_to_cpu(di->i_size));
+		goto bail;
+	}
+
 	if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) {
 		struct ocfs2_inline_data *data = &di->id2.i_data;
 
@@ -1624,6 +1738,40 @@ static int ocfs2_filecheck_validate_inode_block(struct super_block *sb,
 		     (unsigned long long)bh->b_blocknr,
 		     le32_to_cpu(di->i_fs_generation));
 		rc = -OCFS2_FILECHECK_ERR_GENERATION;
+		goto bail;
+	}
+
+	if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) {
+		mlog(ML_ERROR,
+		     "Filecheck: invalid dinode #%llu: mode 0%o has unknown file type\n",
+		     (unsigned long long)bh->b_blocknr,
+		     le16_to_cpu(di->i_mode));
+		rc = -OCFS2_FILECHECK_ERR_INVALIDINO;
+		goto bail;
+	}
+
+	if (ocfs2_dinode_has_unexpected_rdev(di)) {
+		mlog(ML_ERROR,
+		     "Filecheck: invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n",
+		     (unsigned long long)bh->b_blocknr,
+		     le16_to_cpu(di->i_mode),
+		     (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev));
+		rc = -OCFS2_FILECHECK_ERR_INVALIDINO;
+		goto bail;
+	}
+
+	if (ocfs2_dinode_has_size_without_clusters(sb, di)) {
+		if (S_ISDIR(le16_to_cpu(di->i_mode)))
+			mlog(ML_ERROR,
+			     "Filecheck: invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n",
+			     (unsigned long long)bh->b_blocknr,
+			     (unsigned long long)le64_to_cpu(di->i_size));
+		else
+			mlog(ML_ERROR,
+			     "Filecheck: invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n",
+			     (unsigned long long)bh->b_blocknr,
+			     (unsigned long long)le64_to_cpu(di->i_size));
+		rc = -OCFS2_FILECHECK_ERR_INVALIDINO;
 	}
 
 bail:
@@ -1812,4 +1960,3 @@ const struct ocfs2_caching_operations ocfs2_inode_caching_ops = {
 	.co_io_lock		= ocfs2_inode_cache_io_lock,
 	.co_io_unlock		= ocfs2_inode_cache_io_unlock,
 };
-
diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c
index f9bf3bac085db..fc54cc798ce35 100644
--- a/fs/ocfs2/journal.c
+++ b/fs/ocfs2/journal.c
@@ -1022,11 +1022,8 @@ static int ocfs2_journal_toggle_dirty(struct ocfs2_super *osb,
 	struct ocfs2_dinode *fe;
 
 	fe = (struct ocfs2_dinode *)bh->b_data;
-
-	/* The journal bh on the osb always comes from ocfs2_journal_init()
-	 * and was validated there inside ocfs2_inode_lock_full().  It's a
-	 * code bug if we mess it up. */
-	BUG_ON(!OCFS2_IS_VALID_DINODE(fe));
+	if (WARN_ON(!OCFS2_IS_VALID_DINODE(fe)))
+		return -EIO;
 
 	flags = le32_to_cpu(fe->id1.journal1.ij_flags);
 	if (dirty)
diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h
index 7b50e03dfa664..62cad6522c7a3 100644
--- a/fs/ocfs2/ocfs2.h
+++ b/fs/ocfs2/ocfs2.h
@@ -494,8 +494,6 @@ struct ocfs2_super
 	struct rb_root	osb_rf_lock_tree;
 	struct ocfs2_refcount_tree *osb_ref_tree_lru;
 
-	struct mutex system_file_mutex;
-
 	/*
 	 * OCFS2 needs to schedule several different types of work which
 	 * require cluster locking, disk I/O, recovery waits, etc. Since these
diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c
index 12cbb4fccda0d..f55810c59b1b1 100644
--- a/fs/ocfs2/quota_local.c
+++ b/fs/ocfs2/quota_local.c
@@ -302,7 +302,7 @@ static int ocfs2_add_recovery_chunk(struct super_block *sb,
 	if (!rc)
 		return -ENOMEM;
 	rc->rc_chunk = chunk;
-	rc->rc_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS);
+	rc->rc_bitmap = kzalloc(sb->s_blocksize, GFP_NOFS);
 	if (!rc->rc_bitmap) {
 		kfree(rc);
 		return -ENOMEM;
diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c
index b875f01c97564..bcac614ff02a8 100644
--- a/fs/ocfs2/super.c
+++ b/fs/ocfs2/super.c
@@ -1224,7 +1224,6 @@ static struct file_system_type ocfs2_fs_type = {
 	.name           = "ocfs2",
 	.kill_sb        = kill_block_super,
 	.fs_flags       = FS_REQUIRES_DEV|FS_RENAME_DOES_D_MOVE,
-	.next           = NULL,
 	.init_fs_context = ocfs2_init_fs_context,
 	.parameters	= ocfs2_param_spec,
 };
@@ -1997,8 +1996,6 @@ static int ocfs2_initialize_super(struct super_block *sb,
 	spin_lock_init(&osb->osb_xattr_lock);
 	ocfs2_init_steal_slots(osb);
 
-	mutex_init(&osb->system_file_mutex);
-
 	atomic_set(&osb->alloc_stats.moves, 0);
 	atomic_set(&osb->alloc_stats.local_data, 0);
 	atomic_set(&osb->alloc_stats.bitmap_data, 0);
diff --git a/fs/ocfs2/sysfile.c b/fs/ocfs2/sysfile.c
index d53a6cc866bef..67e492f4b828b 100644
--- a/fs/ocfs2/sysfile.c
+++ b/fs/ocfs2/sysfile.c
@@ -98,11 +98,9 @@ struct inode *ocfs2_get_system_file_inode(struct ocfs2_super *osb,
 	} else
 		arr = get_local_system_inode(osb, type, slot);
 
-	mutex_lock(&osb->system_file_mutex);
 	if (arr && ((inode = *arr) != NULL)) {
 		/* get a ref in addition to the array ref */
 		inode = igrab(inode);
-		mutex_unlock(&osb->system_file_mutex);
 		BUG_ON(!inode);
 
 		return inode;
@@ -112,11 +110,10 @@ struct inode *ocfs2_get_system_file_inode(struct ocfs2_super *osb,
 	inode = _ocfs2_get_system_file_inode(osb, type, slot);
 
 	/* add one more if putting into array for first time */
-	if (arr && inode) {
-		*arr = igrab(inode);
-		BUG_ON(!*arr);
+	if (inode && arr && !*arr && !cmpxchg(&(*arr), NULL, inode)) {
+		inode = igrab(inode);
+		BUG_ON(!inode);
 	}
-	mutex_unlock(&osb->system_file_mutex);
 	return inode;
 }
 
diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c
index 86cfd4c2adf92..fcddd3c13acdd 100644
--- a/fs/ocfs2/xattr.c
+++ b/fs/ocfs2/xattr.c
@@ -950,15 +950,51 @@ static int ocfs2_xattr_list_entries(struct inode *inode,
 	return result;
 }
 
+static int ocfs2_xattr_ibody_lookup_header(struct inode *inode,
+					   struct ocfs2_dinode *di,
+					   struct ocfs2_xattr_header **header)
+{
+	u16 xattr_count;
+	size_t max_entries;
+	u16 inline_size = le16_to_cpu(di->i_xattr_inline_size);
+
+	if (inline_size > inode->i_sb->s_blocksize ||
+	    inline_size < sizeof(struct ocfs2_xattr_header)) {
+		ocfs2_error(inode->i_sb,
+			    "Invalid xattr inline size %u in inode %llu\n",
+			    inline_size,
+			    (unsigned long long)OCFS2_I(inode)->ip_blkno);
+		return -EFSCORRUPTED;
+	}
+
+	*header = (struct ocfs2_xattr_header *)
+		((void *)di + inode->i_sb->s_blocksize - inline_size);
+
+	xattr_count = le16_to_cpu((*header)->xh_count);
+	max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) /
+		      sizeof(struct ocfs2_xattr_entry);
+
+	if (xattr_count > max_entries) {
+		ocfs2_error(inode->i_sb,
+			    "xattr entry count %u exceeds maximum %zu in inode %llu\n",
+			    xattr_count, max_entries,
+			    (unsigned long long)OCFS2_I(inode)->ip_blkno);
+		return -EFSCORRUPTED;
+	}
+
+	return 0;
+}
+
 int ocfs2_has_inline_xattr_value_outside(struct inode *inode,
 					 struct ocfs2_dinode *di)
 {
 	struct ocfs2_xattr_header *xh;
+	int ret;
 	int i;
 
-	xh = (struct ocfs2_xattr_header *)
-		 ((void *)di + inode->i_sb->s_blocksize -
-		 le16_to_cpu(di->i_xattr_inline_size));
+	ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xh);
+	if (ret)
+		return 1;
 
 	for (i = 0; i < le16_to_cpu(xh->xh_count); i++)
 		if (!ocfs2_xattr_is_local(&xh->xh_entries[i]))
@@ -975,39 +1011,13 @@ static int ocfs2_xattr_ibody_list(struct inode *inode,
 	struct ocfs2_xattr_header *header = NULL;
 	struct ocfs2_inode_info *oi = OCFS2_I(inode);
 	int ret = 0;
-	u16 xattr_count;
-	size_t max_entries;
-	u16 inline_size;
 
 	if (!(oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL))
 		return ret;
 
-	inline_size = le16_to_cpu(di->i_xattr_inline_size);
-
-	/* Validate inline size is reasonable */
-	if (inline_size > inode->i_sb->s_blocksize ||
-	    inline_size < sizeof(struct ocfs2_xattr_header)) {
-		ocfs2_error(inode->i_sb,
-			    "Invalid xattr inline size %u in inode %llu\n",
-			    inline_size,
-			    (unsigned long long)OCFS2_I(inode)->ip_blkno);
-		return -EFSCORRUPTED;
-	}
-
-	header = (struct ocfs2_xattr_header *)
-		 ((void *)di + inode->i_sb->s_blocksize - inline_size);
-
-	xattr_count = le16_to_cpu(header->xh_count);
-	max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) /
-		       sizeof(struct ocfs2_xattr_entry);
-
-	if (xattr_count > max_entries) {
-		ocfs2_error(inode->i_sb,
-			    "xattr entry count %u exceeds maximum %zu in inode %llu\n",
-			    xattr_count, max_entries,
-			    (unsigned long long)OCFS2_I(inode)->ip_blkno);
-		return -EFSCORRUPTED;
-	}
+	ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header);
+	if (ret)
+		return ret;
 
 	ret = ocfs2_xattr_list_entries(inode, header, buffer, buffer_size);
 
@@ -1200,8 +1210,9 @@ static int ocfs2_xattr_ibody_get(struct inode *inode,
 		return -ENODATA;
 
 	xs->end = (void *)di + inode->i_sb->s_blocksize;
-	xs->header = (struct ocfs2_xattr_header *)
-			(xs->end - le16_to_cpu(di->i_xattr_inline_size));
+	ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xs->header);
+	if (ret)
+		return ret;
 	xs->base = (void *)xs->header;
 	xs->here = xs->header->xh_entries;
 
@@ -2465,9 +2476,9 @@ static int ocfs2_xattr_ibody_remove(struct inode *inode,
 		.vb_access = ocfs2_journal_access_di,
 	};
 
-	header = (struct ocfs2_xattr_header *)
-		 ((void *)di + inode->i_sb->s_blocksize -
-		 le16_to_cpu(di->i_xattr_inline_size));
+	ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header);
+	if (ret)
+		return ret;
 
 	ret = ocfs2_remove_value_outside(inode, &vb, header,
 					 ref_ci, ref_root_bh);
@@ -2726,12 +2737,14 @@ static int ocfs2_xattr_ibody_find(struct inode *inode,
 
 	xs->xattr_bh = xs->inode_bh;
 	xs->end = (void *)di + inode->i_sb->s_blocksize;
-	if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)
-		xs->header = (struct ocfs2_xattr_header *)
-			(xs->end - le16_to_cpu(di->i_xattr_inline_size));
-	else
+	if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) {
+		ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xs->header);
+		if (ret)
+			return ret;
+	} else {
 		xs->header = (struct ocfs2_xattr_header *)
 			(xs->end - OCFS2_SB(inode->i_sb)->s_xattr_inline_size);
+	}
 	xs->base = (void *)xs->header;
 	xs->here = xs->header->xh_entries;
 
@@ -6003,14 +6016,17 @@ static int ocfs2_xattr_inline_attach_refcount(struct inode *inode,
 				struct ocfs2_cached_dealloc_ctxt *dealloc)
 {
 	struct ocfs2_dinode *di = (struct ocfs2_dinode *)fe_bh->b_data;
-	struct ocfs2_xattr_header *header = (struct ocfs2_xattr_header *)
-				(fe_bh->b_data + inode->i_sb->s_blocksize -
-				le16_to_cpu(di->i_xattr_inline_size));
+	struct ocfs2_xattr_header *header;
+	int ret;
 	struct ocfs2_xattr_value_buf vb = {
 		.vb_bh = fe_bh,
 		.vb_access = ocfs2_journal_access_di,
 	};
 
+	ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header);
+	if (ret)
+		return ret;
+
 	return ocfs2_xattr_attach_refcount_normal(inode, &vb, header,
 						  ref_ci, ref_root_bh, dealloc);
 }
@@ -6495,12 +6511,10 @@ static int ocfs2_reflink_xattr_inline(struct ocfs2_xattr_reflink *args)
 	handle_t *handle;
 	struct ocfs2_super *osb = OCFS2_SB(args->old_inode->i_sb);
 	struct ocfs2_dinode *di = (struct ocfs2_dinode *)args->old_bh->b_data;
-	int inline_size = le16_to_cpu(di->i_xattr_inline_size);
-	int header_off = osb->sb->s_blocksize - inline_size;
-	struct ocfs2_xattr_header *xh = (struct ocfs2_xattr_header *)
-					(args->old_bh->b_data + header_off);
-	struct ocfs2_xattr_header *new_xh = (struct ocfs2_xattr_header *)
-					(args->new_bh->b_data + header_off);
+	int inline_size;
+	int header_off;
+	struct ocfs2_xattr_header *xh;
+	struct ocfs2_xattr_header *new_xh;
 	struct ocfs2_alloc_context *meta_ac = NULL;
 	struct ocfs2_inode_info *new_oi;
 	struct ocfs2_dinode *new_di;
@@ -6509,6 +6523,15 @@ static int ocfs2_reflink_xattr_inline(struct ocfs2_xattr_reflink *args)
 		.vb_access = ocfs2_journal_access_di,
 	};
 
+	ret = ocfs2_xattr_ibody_lookup_header(args->old_inode, di, &xh);
+	if (ret)
+		goto out;
+
+	inline_size = le16_to_cpu(di->i_xattr_inline_size);
+	header_off = osb->sb->s_blocksize - inline_size;
+	new_xh = (struct ocfs2_xattr_header *)
+		(args->new_bh->b_data + header_off);
+
 	ret = ocfs2_reflink_lock_xattr_allocators(osb, xh, args->ref_root_bh,
 						  &credits, &meta_ac);
 	if (ret) {
diff --git a/fs/omfs/inode.c b/fs/omfs/inode.c
index 834cae1e62233..1d915ef72119f 100644
--- a/fs/omfs/inode.c
+++ b/fs/omfs/inode.c
@@ -478,7 +478,8 @@ static int omfs_fill_super(struct super_block *sb, struct fs_context *fc)
 	sb->s_time_min = 0;
 	sb->s_time_max = U64_MAX / MSEC_PER_SEC;
 
-	sb_set_blocksize(sb, 0x200);
+	if (!sb_set_blocksize(sb, 0x200))
+		goto end;
 
 	bh = sb_bread(sb, 0);
 	if (!bh)
@@ -530,7 +531,8 @@ static int omfs_fill_super(struct super_block *sb, struct fs_context *fc)
 	 * Use sys_blocksize as the fs block since it is smaller than a
 	 * page while the fs blocksize can be larger.
 	 */
-	sb_set_blocksize(sb, sbi->s_sys_blocksize);
+	if (!sb_set_blocksize(sb, sbi->s_sys_blocksize))
+		goto out_brelse_bh;
 
 	/*
 	 * ...and the difference goes into a shift.  sys_blocksize is always
diff --git a/fs/open.c b/fs/open.c
index 681d405bc61eb..834ae63f4e9e5 100644
--- a/fs/open.c
+++ b/fs/open.c
@@ -960,7 +960,7 @@ static int do_dentry_open(struct file *f,
 	if (f->f_mapping->a_ops && f->f_mapping->a_ops->direct_IO)
 		f->f_mode |= FMODE_CAN_ODIRECT;
 
-	f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC);
+	f->f_flags &= ~(O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC | OPENAT2_REGULAR);
 	f->f_iocb_flags = iocb_flags(f);
 
 	file_ra_state_init(&f->f_ra, f->f_mapping->host->i_mapping);
@@ -968,33 +968,6 @@ static int do_dentry_open(struct file *f,
 	if ((f->f_flags & O_DIRECT) && !(f->f_mode & FMODE_CAN_ODIRECT))
 		return -EINVAL;
 
-	/*
-	 * XXX: Huge page cache doesn't support writing yet. Drop all page
-	 * cache for this file before processing writes.
-	 */
-	if (f->f_mode & FMODE_WRITE) {
-		/*
-		 * Depends on full fence from get_write_access() to synchronize
-		 * against collapse_file() regarding i_writecount and nr_thps
-		 * updates. Ensures subsequent insertion of THPs into the page
-		 * cache will fail.
-		 */
-		if (filemap_nr_thps(inode->i_mapping)) {
-			struct address_space *mapping = inode->i_mapping;
-
-			filemap_invalidate_lock(inode->i_mapping);
-			/*
-			 * unmap_mapping_range just need to be called once
-			 * here, because the private pages is not need to be
-			 * unmapped mapping (e.g. data segment of dynamic
-			 * shared libraries here).
-			 */
-			unmap_mapping_range(mapping, 0, 0, 0);
-			truncate_inode_pages(mapping, 0);
-			filemap_invalidate_unlock(inode->i_mapping);
-		}
-	}
-
 	return 0;
 
 cleanup_all:
@@ -1158,7 +1131,7 @@ struct file *kernel_file_open(const struct path *path, int flags,
 EXPORT_SYMBOL_GPL(kernel_file_open);
 
 #define WILL_CREATE(flags)	(flags & (O_CREAT | __O_TMPFILE))
-#define O_PATH_FLAGS		(O_DIRECTORY | O_NOFOLLOW | O_PATH | O_CLOEXEC)
+#define O_PATH_FLAGS		(O_DIRECTORY | O_NOFOLLOW | O_PATH | O_CLOEXEC | O_EMPTYPATH)
 
 inline struct open_how build_open_how(int flags, umode_t mode)
 {
@@ -1183,7 +1156,7 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op)
 	int lookup_flags = 0;
 	int acc_mode = ACC_MODE(flags);
 
-	BUILD_BUG_ON_MSG(upper_32_bits(VALID_OPEN_FLAGS),
+	BUILD_BUG_ON_MSG(upper_32_bits(VALID_OPENAT2_FLAGS),
 			 "struct open_flags doesn't yet handle flags > 32 bits");
 
 	/*
@@ -1196,7 +1169,7 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op)
 	 * values before calling build_open_flags(), but openat2(2) checks all
 	 * of its arguments.
 	 */
-	if (flags & ~VALID_OPEN_FLAGS)
+	if (flags & ~VALID_OPENAT2_FLAGS)
 		return -EINVAL;
 	if (how->resolve & ~VALID_RESOLVE_FLAGS)
 		return -EINVAL;
@@ -1236,6 +1209,11 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op)
 		if (!(acc_mode & MAY_WRITE))
 			return -EINVAL;
 	}
+
+	if ((flags & (O_DIRECTORY | OPENAT2_REGULAR)) ==
+	    (O_DIRECTORY | OPENAT2_REGULAR))
+		return -EINVAL;
+
 	if (flags & O_PATH) {
 		/* O_PATH only permits certain other flags to be set. */
 		if (flags & ~O_PATH_FLAGS)
@@ -1279,6 +1257,8 @@ inline int build_open_flags(const struct open_how *how, struct open_flags *op)
 		lookup_flags |= LOOKUP_DIRECTORY;
 	if (!(flags & O_NOFOLLOW))
 		lookup_flags |= LOOKUP_FOLLOW;
+	if (flags & O_EMPTYPATH)
+		lookup_flags |= LOOKUP_EMPTY;
 
 	if (how->resolve & RESOLVE_NO_XDEV)
 		lookup_flags |= LOOKUP_NO_XDEV;
@@ -1360,7 +1340,7 @@ static int do_sys_openat2(int dfd, const char __user *filename,
 	if (unlikely(err))
 		return err;
 
-	CLASS(filename, name)(filename);
+	CLASS(filename_flags, name)(filename, op.lookup_flags);
 	return FD_ADD(how->flags, do_file_open(dfd, name, &op));
 }
 
diff --git a/fs/orangefs/namei.c b/fs/orangefs/namei.c
index bec5475de094d..75e65e72c2d64 100644
--- a/fs/orangefs/namei.c
+++ b/fs/orangefs/namei.c
@@ -362,7 +362,7 @@ static struct dentry *orangefs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
 	__orangefs_setattr(dir, &iattr);
 out:
 	op_release(new_op);
-	return ERR_PTR(ret);
+	return ret ? ERR_PTR(ret) : NULL;
 }
 
 static int orangefs_rename(struct mnt_idmap *idmap,
diff --git a/fs/proc/base.c b/fs/proc/base.c
index d9acfa89c894b..49344de101582 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -3543,28 +3543,42 @@ struct dentry *proc_pid_lookup(struct dentry *dentry, unsigned int flags)
 struct tgid_iter {
 	unsigned int tgid;
 	struct task_struct *task;
+	struct pid_namespace *const pid_ns;
 };
-static struct tgid_iter next_tgid(struct pid_namespace *ns, struct tgid_iter iter)
+
+static struct tgid_iter
+make_tgid_iter(unsigned int init_tgid, struct pid_namespace *pid_ns)
 {
-	struct pid *pid;
+	return (struct tgid_iter){
+		.tgid = init_tgid - 1,
+		.pid_ns = pid_ns,
+	};
+}
+
+static bool next_tgid(struct tgid_iter *it)
+{
+	if (it->task) {
+		put_task_struct(it->task);
+		it->task = NULL;
+	}
 
-	if (iter.task)
-		put_task_struct(iter.task);
 	rcu_read_lock();
-retry:
-	iter.task = NULL;
-	pid = find_ge_pid(iter.tgid, ns);
-	if (pid) {
-		iter.tgid = pid_nr_ns(pid, ns);
-		iter.task = pid_task(pid, PIDTYPE_TGID);
-		if (!iter.task) {
-			iter.tgid += 1;
-			goto retry;
+	while (1) {
+		it->tgid += 1;
+		const auto pid = find_ge_pid(it->tgid, it->pid_ns);
+		if (pid) {
+			it->tgid = pid_nr_ns(pid, it->pid_ns);
+			it->task = pid_task(pid, PIDTYPE_TGID);
+			if (it->task) {
+				get_task_struct(it->task);
+				rcu_read_unlock();
+				return true;
+			}
+		} else {
+			rcu_read_unlock();
+			return false;
 		}
-		get_task_struct(iter.task);
 	}
-	rcu_read_unlock();
-	return iter;
 }
 
 #define TGID_OFFSET (FIRST_PROCESS_ENTRY + 2)
@@ -3572,9 +3586,8 @@ static struct tgid_iter next_tgid(struct pid_namespace *ns, struct tgid_iter ite
 /* for the /proc/ directory itself, after non-process stuff has been done */
 int proc_pid_readdir(struct file *file, struct dir_context *ctx)
 {
-	struct tgid_iter iter;
 	struct proc_fs_info *fs_info = proc_sb_info(file_inode(file)->i_sb);
-	struct pid_namespace *ns = proc_pid_ns(file_inode(file)->i_sb);
+	struct pid_namespace *pid_ns = proc_pid_ns(file_inode(file)->i_sb);
 	loff_t pos = ctx->pos;
 
 	if (pos >= PID_MAX_LIMIT + TGID_OFFSET)
@@ -3590,11 +3603,9 @@ int proc_pid_readdir(struct file *file, struct dir_context *ctx)
 			return 0;
 		ctx->pos = pos = pos + 1;
 	}
-	iter.tgid = pos - TGID_OFFSET;
-	iter.task = NULL;
-	for (iter = next_tgid(ns, iter);
-	     iter.task;
-	     iter.tgid += 1, iter = next_tgid(ns, iter)) {
+
+	auto iter = make_tgid_iter(pos - TGID_OFFSET, pid_ns);
+	while (next_tgid(&iter)) {
 		char name[10 + 1];
 		unsigned int len;
 
diff --git a/fs/proc/generic.c b/fs/proc/generic.c
index 8bb81e58c9d8c..adc9b9a092b03 100644
--- a/fs/proc/generic.c
+++ b/fs/proc/generic.c
@@ -427,9 +427,13 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent,
 	if (xlate_proc_name(name, parent, &fn) != 0)
 		goto out;
 	qstr.name = fn;
-	qstr.len = strlen(fn);
-	if (qstr.len == 0 || qstr.len >= 256) {
-		WARN(1, "name len %u\n", qstr.len);
+	qstr.len = strnlen(fn, NAME_MAX + 1);
+	if (qstr.len == 0) {
+		WARN(1, "empty name\n");
+		return NULL;
+	}
+	if (qstr.len > NAME_MAX) {
+		WARN(1, "name too long\n");
 		return NULL;
 	}
 	if (qstr.len == 1 && fn[0] == '.') {
@@ -841,3 +845,13 @@ ssize_t proc_simple_write(struct file *f, const char __user *ubuf, size_t size,
 	kfree(buf);
 	return ret == 0 ? size : ret;
 }
+
+/*
+ * Not exported to modules:
+ * modules' /proc files aren't permanent because modules aren't permanent.
+ */
+void impl_proc_make_permanent(struct proc_dir_entry *pde)
+{
+	if (pde)
+		pde_make_permanent(pde);
+}
diff --git a/fs/proc/internal.h b/fs/proc/internal.h
index 64dc44832808a..1edbabbdbc5d7 100644
--- a/fs/proc/internal.h
+++ b/fs/proc/internal.h
@@ -79,8 +79,11 @@ static inline bool pde_is_permanent(const struct proc_dir_entry *pde)
 	return pde->flags & PROC_ENTRY_PERMANENT;
 }
 
+/* This is for builtin code, not even for modules which are compiled in. */
 static inline void pde_make_permanent(struct proc_dir_entry *pde)
 {
+	/* Ensure magic flag does something. */
+	static_assert(PROC_ENTRY_PERMANENT != 0);
 	pde->flags |= PROC_ENTRY_PERMANENT;
 }
 
diff --git a/fs/proc/proc_net.c b/fs/proc/proc_net.c
index 184cddeb8215c..00cc385bce212 100644
--- a/fs/proc/proc_net.c
+++ b/fs/proc/proc_net.c
@@ -23,6 +23,7 @@
 #include <linux/uidgid.h>
 #include <net/net_namespace.h>
 #include <linux/seq_file.h>
+#include <linux/security.h>
 
 #include "internal.h"
 
@@ -270,6 +271,7 @@ static struct net *get_proc_task_net(struct inode *dir)
 	struct task_struct *task;
 	struct nsproxy *ns;
 	struct net *net = NULL;
+	struct proc_fs_info *fs_info = proc_sb_info(dir->i_sb);
 
 	rcu_read_lock();
 	task = pid_task(proc_pid(dir), PIDTYPE_PID);
@@ -282,6 +284,12 @@ static struct net *get_proc_task_net(struct inode *dir)
 	}
 	rcu_read_unlock();
 
+	if (net && (fs_info->pidonly == PROC_PIDONLY_ON) &&
+	    security_capable(fs_info->mounter_cred, net->user_ns, CAP_NET_ADMIN, CAP_OPT_NONE) < 0) {
+		put_net(net);
+		net = NULL;
+	}
+
 	return net;
 }
 
diff --git a/fs/proc/root.c b/fs/proc/root.c
index 0f91005594710..99adddfeb4a44 100644
--- a/fs/proc/root.c
+++ b/fs/proc/root.c
@@ -223,12 +223,17 @@ static int proc_parse_param(struct fs_context *fc, struct fs_parameter *param)
 	return 0;
 }
 
-static void proc_apply_options(struct proc_fs_info *fs_info,
+static int proc_apply_options(struct proc_fs_info *fs_info,
 			       struct fs_context *fc,
 			       struct user_namespace *user_ns)
 {
 	struct proc_fs_context *ctx = fc->fs_private;
 
+	if ((ctx->mask & (1 << Opt_subset)) &&
+	    fc->purpose == FS_CONTEXT_FOR_RECONFIGURE &&
+	    ctx->pidonly != fs_info->pidonly)
+		return invalf(fc, "proc: subset=pid cannot be changed\n");
+
 	if (ctx->mask & (1 << Opt_gid))
 		fs_info->pid_gid = make_kgid(user_ns, ctx->gid);
 	if (ctx->mask & (1 << Opt_hidepid))
@@ -240,6 +245,7 @@ static void proc_apply_options(struct proc_fs_info *fs_info,
 		put_pid_ns(fs_info->pid_ns);
 		fs_info->pid_ns = get_pid_ns(ctx->pid_ns);
 	}
+	return 0;
 }
 
 static int proc_fill_super(struct super_block *s, struct fs_context *fc)
@@ -254,10 +260,13 @@ static int proc_fill_super(struct super_block *s, struct fs_context *fc)
 		return -ENOMEM;
 
 	fs_info->pid_ns = get_pid_ns(ctx->pid_ns);
-	proc_apply_options(fs_info, fc, current_user_ns());
+	fs_info->mounter_cred = get_cred(fc->cred);
+	ret = proc_apply_options(fs_info, fc, current_user_ns());
+	if (ret)
+		return ret;
 
 	/* User space would break if executables or devices appear on proc */
-	s->s_iflags |= SB_I_USERNS_VISIBLE | SB_I_NOEXEC | SB_I_NODEV;
+	s->s_iflags |= SB_I_NOEXEC | SB_I_NODEV;
 	s->s_flags |= SB_NODIRATIME | SB_NOSUID | SB_NOEXEC;
 	s->s_blocksize = 1024;
 	s->s_blocksize_bits = 10;
@@ -266,6 +275,9 @@ static int proc_fill_super(struct super_block *s, struct fs_context *fc)
 	s->s_time_gran = 1;
 	s->s_fs_info = fs_info;
 
+	if (fs_info->pidonly == PROC_PIDONLY_ON)
+		s->s_iflags |= SB_I_RESTRICTED_VARIANT;
+
 	/*
 	 * procfs isn't actually a stacking filesystem; however, there is
 	 * too much magic going on inside it to permit stacking things on
@@ -303,8 +315,7 @@ static int proc_reconfigure(struct fs_context *fc)
 
 	sync_filesystem(sb);
 
-	proc_apply_options(fs_info, fc, current_user_ns());
-	return 0;
+	return proc_apply_options(fs_info, fc, current_user_ns());
 }
 
 static int proc_get_tree(struct fs_context *fc)
@@ -350,6 +361,7 @@ static void proc_kill_sb(struct super_block *sb)
 	kill_anon_super(sb);
 	if (fs_info) {
 		put_pid_ns(fs_info->pid_ns);
+		put_cred(fs_info->mounter_cred);
 		kfree_rcu(fs_info, rcu);
 	}
 }
@@ -359,7 +371,7 @@ static struct file_system_type proc_fs_type = {
 	.init_fs_context	= proc_init_fs_context,
 	.parameters		= proc_fs_parameters,
 	.kill_sb		= proc_kill_sb,
-	.fs_flags		= FS_USERNS_MOUNT | FS_DISALLOW_NOTIFY_PERM,
+	.fs_flags		= FS_USERNS_MOUNT | FS_USERNS_MOUNT_RESTRICTED | FS_DISALLOW_NOTIFY_PERM,
 };
 
 void __init proc_root_init(void)
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 751b9ba160fbf..1e3a15bf46f4e 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -132,6 +132,22 @@ static void release_task_mempolicy(struct proc_maps_private *priv)
 
 #ifdef CONFIG_PER_VMA_LOCK
 
+static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
+{
+	int ret = mmap_read_lock_killable(lock_ctx->mm);
+
+	if (!ret)
+		lock_ctx->mmap_locked = true;
+
+	return ret;
+}
+
+static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
+{
+	mmap_read_unlock(lock_ctx->mm);
+	lock_ctx->mmap_locked = false;
+}
+
 static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx)
 {
 	lock_ctx->locked_vma = NULL;
@@ -146,25 +162,11 @@ static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx)
 	}
 }
 
-static const struct seq_operations proc_pid_maps_op;
-
 static inline bool lock_vma_range(struct seq_file *m,
 				  struct proc_maps_locking_ctx *lock_ctx)
 {
-	/*
-	 * smaps and numa_maps perform page table walk, therefore require
-	 * mmap_lock but maps can be read with locking just the vma and
-	 * walking the vma tree under rcu read protection.
-	 */
-	if (m->op != &proc_pid_maps_op) {
-		if (mmap_read_lock_killable(lock_ctx->mm))
-			return false;
-
-		lock_ctx->mmap_locked = true;
-	} else {
-		rcu_read_lock();
-		reset_lock_ctx(lock_ctx);
-	}
+	rcu_read_lock();
+	reset_lock_ctx(lock_ctx);
 
 	return true;
 }
@@ -172,7 +174,7 @@ static inline bool lock_vma_range(struct seq_file *m,
 static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx)
 {
 	if (lock_ctx->mmap_locked) {
-		mmap_read_unlock(lock_ctx->mm);
+		unlock_ctx_mm(lock_ctx);
 	} else {
 		unlock_ctx_vma(lock_ctx);
 		rcu_read_unlock();
@@ -213,17 +215,45 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,
 	return true;
 }
 
+static inline void drop_rcu(struct proc_maps_private *priv)
+{
+	if (priv->lock_ctx.mmap_locked)
+		return;
+
+	rcu_read_unlock();
+}
+
+static inline void reacquire_rcu(struct proc_maps_private *priv)
+{
+	if (priv->lock_ctx.mmap_locked)
+		return;
+
+	rcu_read_lock();
+	/* Reinitialize the iterator. */
+	vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end);
+}
+
 #else /* CONFIG_PER_VMA_LOCK */
 
+static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
+{
+	return mmap_read_lock_killable(lock_ctx->mm);
+}
+
+static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx)
+{
+	mmap_read_unlock(lock_ctx->mm);
+}
+
 static inline bool lock_vma_range(struct seq_file *m,
 				  struct proc_maps_locking_ctx *lock_ctx)
 {
-	return mmap_read_lock_killable(lock_ctx->mm) == 0;
+	return lock_ctx_mm(lock_ctx) == 0;
 }
 
 static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx)
 {
-	mmap_read_unlock(lock_ctx->mm);
+	unlock_ctx_mm(lock_ctx);
 }
 
 static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv,
@@ -238,6 +268,9 @@ static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv,
 	return false;
 }
 
+static inline void drop_rcu(struct proc_maps_private *priv) {}
+static inline void reacquire_rcu(struct proc_maps_private *priv) {}
+
 #endif /* CONFIG_PER_VMA_LOCK */
 
 static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos)
@@ -538,12 +571,10 @@ static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx)
 
 static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx)
 {
-	if (lock_ctx->mmap_locked) {
-		mmap_read_unlock(lock_ctx->mm);
-		lock_ctx->mmap_locked = false;
-	} else {
+	if (lock_ctx->mmap_locked)
+		unlock_ctx_mm(lock_ctx);
+	else
 		unlock_ctx_vma(lock_ctx);
-	}
 }
 
 static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx,
@@ -1280,21 +1311,75 @@ static const struct mm_walk_ops smaps_shmem_walk_ops = {
 	.walk_lock		= PGWALK_RDLOCK,
 };
 
+#ifdef CONFIG_PER_VMA_LOCK
+
+static const struct mm_walk_ops smaps_walk_vma_lock_ops = {
+	.pmd_entry		= smaps_pte_range,
+	.hugetlb_entry		= smaps_hugetlb_range,
+	.walk_lock		= PGWALK_VMA_RDLOCK_VERIFY,
+};
+
+static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = {
+	.pmd_entry		= smaps_pte_range,
+	.hugetlb_entry		= smaps_hugetlb_range,
+	.pte_hole		= smaps_pte_hole,
+	.walk_lock		= PGWALK_VMA_RDLOCK_VERIFY,
+};
+
+static inline const struct mm_walk_ops *
+get_smaps_walk_ops(struct proc_maps_private *priv)
+{
+	if (priv->lock_ctx.mmap_locked)
+		return &smaps_walk_ops;
+	return &smaps_walk_vma_lock_ops;
+}
+
+static inline const struct mm_walk_ops *
+get_smaps_shmem_walk_ops(struct proc_maps_private *priv)
+{
+	if (priv->lock_ctx.mmap_locked)
+		return  &smaps_shmem_walk_ops;
+	return &smaps_shmem_walk_vma_lock_ops;
+}
+
+#else /* CONFIG_PER_VMA_LOCK */
+
+static inline const struct mm_walk_ops *
+get_smaps_walk_ops(struct proc_maps_private *priv)
+{
+	return &smaps_walk_ops;
+}
+
+static inline const struct mm_walk_ops *
+get_smaps_shmem_walk_ops(struct proc_maps_private *priv)
+{
+	return &smaps_shmem_walk_ops;
+}
+
+#endif /* CONFIG_PER_VMA_LOCK */
+
 /*
  * Gather mem stats from @vma with the indicated beginning
  * address @start, and keep them in @mss.
  *
  * Use vm_start of @vma as the beginning address if @start is 0.
  */
-static void smap_gather_stats(struct vm_area_struct *vma,
-		struct mem_size_stats *mss, unsigned long start)
+static void smap_gather_stats(struct proc_maps_private *priv,
+			      struct vm_area_struct *vma,
+			      struct mem_size_stats *mss, unsigned long start)
 {
-	const struct mm_walk_ops *ops = &smaps_walk_ops;
+	const struct mm_walk_ops *ops = get_smaps_walk_ops(priv);
 
 	/* Invalid start */
 	if (start >= vma->vm_end)
 		return;
 
+	if (vma == get_gate_vma(priv->lock_ctx.mm))
+		return;
+
+	/* Might sleep. Drop RCU read lock but keep the VMA locked. */
+	drop_rcu(priv);
+
 	if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
 		/*
 		 * For shared or readonly shmem mappings we know that all
@@ -1312,15 +1397,16 @@ static void smap_gather_stats(struct vm_area_struct *vma,
 					!(vma->vm_flags & VM_WRITE))) {
 			mss->swap += shmem_swapped;
 		} else {
-			ops = &smaps_shmem_walk_ops;
+			ops = get_smaps_shmem_walk_ops(priv);
 		}
 	}
 
-	/* mmap_lock is held in m_start */
 	if (!start)
 		walk_page_vma(vma, ops, mss);
 	else
 		walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);
+
+	reacquire_rcu(priv);
 }
 
 #define SEQ_PUT_DEC(str, val) \
@@ -1369,10 +1455,11 @@ static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,
 
 static int show_smap(struct seq_file *m, void *v)
 {
+	struct proc_maps_private *priv = m->private;
 	struct vm_area_struct *vma = v;
 	struct mem_size_stats mss = {};
 
-	smap_gather_stats(vma, &mss, 0);
+	smap_gather_stats(priv, vma, &mss, 0);
 
 	show_map_vma(m, vma);
 
@@ -1413,7 +1500,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v)
 		goto out_put_task;
 	}
 
-	ret = mmap_read_lock_killable(mm);
+	ret = lock_ctx_mm(&priv->lock_ctx);
 	if (ret)
 		goto out_put_mm;
 
@@ -1425,7 +1512,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v)
 
 	vma_start = vma->vm_start;
 	do {
-		smap_gather_stats(vma, &mss, 0);
+		smap_gather_stats(priv, vma, &mss, 0);
 		last_vma_end = vma->vm_end;
 
 		/*
@@ -1434,8 +1521,8 @@ static int show_smaps_rollup(struct seq_file *m, void *v)
 		 */
 		if (mmap_lock_is_contended(mm)) {
 			vma_iter_invalidate(&vmi);
-			mmap_read_unlock(mm);
-			ret = mmap_read_lock_killable(mm);
+			unlock_ctx_mm(&priv->lock_ctx);
+			ret = lock_ctx_mm(&priv->lock_ctx);
 			if (ret) {
 				release_task_mempolicy(priv);
 				goto out_put_mm;
@@ -1484,14 +1571,14 @@ static int show_smaps_rollup(struct seq_file *m, void *v)
 
 			/* Case 1 and 2 above */
 			if (vma->vm_start >= last_vma_end) {
-				smap_gather_stats(vma, &mss, 0);
+				smap_gather_stats(priv, vma, &mss, 0);
 				last_vma_end = vma->vm_end;
 				continue;
 			}
 
 			/* Case 4 above */
 			if (vma->vm_end > last_vma_end) {
-				smap_gather_stats(vma, &mss, last_vma_end);
+				smap_gather_stats(priv, vma, &mss, last_vma_end);
 				last_vma_end = vma->vm_end;
 			}
 		}
@@ -1505,7 +1592,7 @@ static int show_smaps_rollup(struct seq_file *m, void *v)
 	__show_smap(m, &mss, true);
 
 	release_task_mempolicy(priv);
-	mmap_read_unlock(mm);
+	unlock_ctx_mm(&priv->lock_ctx);
 
 out_put_mm:
 	mmput(mm);
@@ -3291,6 +3378,31 @@ static const struct mm_walk_ops show_numa_ops = {
 	.walk_lock = PGWALK_RDLOCK,
 };
 
+#ifdef CONFIG_PER_VMA_LOCK
+static const struct mm_walk_ops show_numa_vma_lock_ops = {
+	.hugetlb_entry = gather_hugetlb_stats,
+	.pmd_entry = gather_pte_stats,
+	.walk_lock = PGWALK_VMA_RDLOCK_VERIFY,
+};
+
+static inline const struct mm_walk_ops *
+get_show_numa_ops(struct proc_maps_private *priv)
+{
+	if (priv->lock_ctx.mmap_locked)
+		return &show_numa_ops;
+	return &show_numa_vma_lock_ops;
+}
+
+#else /* CONFIG_PER_VMA_LOCK */
+
+static inline const struct mm_walk_ops *
+get_show_numa_ops(struct proc_maps_private *priv)
+{
+	return &show_numa_ops;
+}
+
+#endif /* CONFIG_PER_VMA_LOCK */
+
 /*
  * Display pages allocated per node and memory policy via /proc.
  */
@@ -3335,8 +3447,13 @@ static int show_numa_map(struct seq_file *m, void *v)
 	if (is_vm_hugetlb_page(vma))
 		seq_puts(m, " huge");
 
-	/* mmap_lock is held by m_start */
-	walk_page_vma(vma, &show_numa_ops, md);
+	/* Skip walking pages if gate VMA */
+	if (vma != get_gate_vma(proc_priv->lock_ctx.mm)) {
+		/* Might sleep. Drop RCU read lock but keep the VMA locked. */
+		drop_rcu(proc_priv);
+		walk_page_vma(vma, get_show_numa_ops(proc_priv), md);
+		reacquire_rcu(proc_priv);
+	}
 
 	if (!md->pages)
 		goto out;
diff --git a/fs/qnx4/inode.c b/fs/qnx4/inode.c
index 4deb0eeadbdef..42fcd500fad21 100644
--- a/fs/qnx4/inode.c
+++ b/fs/qnx4/inode.c
@@ -202,7 +202,8 @@ static int qnx4_fill_super(struct super_block *s, struct fs_context *fc)
 		return -ENOMEM;
 	s->s_fs_info = qs;
 
-	sb_set_blocksize(s, QNX4_BLOCK_SIZE);
+	if (!sb_set_blocksize(s, QNX4_BLOCK_SIZE))
+		return -EINVAL;
 
 	s->s_op = &qnx4_sops;
 	s->s_magic = QNX4_SUPER_MAGIC;
diff --git a/fs/resctrl/internal.h b/fs/resctrl/internal.h
index 1a9b29119f88f..48af75b9dc855 100644
--- a/fs/resctrl/internal.h
+++ b/fs/resctrl/internal.h
@@ -408,6 +408,8 @@ void __check_limbo(struct rdt_l3_mon_domain *d, bool force_free);
 
 void resctrl_file_fflags_init(const char *config, unsigned long fflags);
 
+void resctrl_file_mode_init(const char *config, umode_t mode);
+
 void rdt_staged_configs_clear(void);
 
 bool closid_allocated(unsigned int closid);
diff --git a/fs/resctrl/monitor.c b/fs/resctrl/monitor.c
index 9fd901c78dc66..0e6a389a16bf6 100644
--- a/fs/resctrl/monitor.c
+++ b/fs/resctrl/monitor.c
@@ -1211,9 +1211,10 @@ static int rdtgroup_alloc_assign_cntr(struct rdt_resource *r, struct rdt_l3_mon_
  * NULL; otherwise, assign the counter to the specified domain @d.
  *
  * If all counters in a domain are already in use, rdtgroup_alloc_assign_cntr()
- * will fail. The assignment process will abort at the first failure encountered
- * during domain traversal, which may result in the event being only partially
- * assigned.
+ * will fail. When attempting to assign counters to all domains, carry on trying
+ * to assign counters after a failure since only some domains may have counters
+ * and the goal is to assign counters where possible. If any counter assignment
+ * fails, return the error from the last failing assignment.
  *
  * Return:
  * 0 on success, < 0 on failure.
@@ -1226,9 +1227,11 @@ static int rdtgroup_assign_cntr_event(struct rdt_l3_mon_domain *d, struct rdtgro
 
 	if (!d) {
 		list_for_each_entry(d, &r->mon_domains, hdr.list) {
-			ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt);
-			if (ret)
-				return ret;
+			int err;
+
+			err = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt);
+			if (err)
+				ret = err;
 		}
 	} else {
 		ret = rdtgroup_alloc_assign_cntr(r, d, rdtgrp, mevt);
@@ -1422,6 +1425,11 @@ ssize_t event_filter_write(struct kernfs_open_file *of, char *buf, size_t nbytes
 		ret = -EINVAL;
 		goto out_unlock;
 	}
+	if (!r->mon.mbm_cntr_configurable) {
+		rdt_last_cmd_puts("event_filter is not configurable\n");
+		ret = -EPERM;
+		goto out_unlock;
+	}
 
 	ret = resctrl_parse_mem_transactions(buf, &evt_cfg);
 	if (!ret && mevt->evt_cfg != evt_cfg) {
@@ -1451,7 +1459,7 @@ int resctrl_mbm_assign_mode_show(struct kernfs_open_file *of,
 		else
 			seq_puts(s, "[default]\n");
 
-		if (!IS_ENABLED(CONFIG_RESCTRL_ASSIGN_FIXED)) {
+		if (!r->mon.mbm_cntr_assign_fixed) {
 			if (enabled)
 				seq_puts(s, "default\n");
 			else
@@ -1502,6 +1510,12 @@ ssize_t resctrl_mbm_assign_mode_write(struct kernfs_open_file *of, char *buf,
 	}
 
 	if (enable != resctrl_arch_mbm_cntr_assign_enabled(r)) {
+		if (r->mon.mbm_cntr_assign_fixed) {
+			ret = -EINVAL;
+			rdt_last_cmd_puts("Counter assignment mode is not configurable\n");
+			goto out_unlock;
+		}
+
 		ret = resctrl_arch_mbm_cntr_assign_set(r, enable);
 		if (ret)
 			goto out_unlock;
@@ -1886,6 +1900,8 @@ int resctrl_l3_mon_resource_init(void)
 		resctrl_file_fflags_init("available_mbm_cntrs",
 					 RFTYPE_MON_INFO | RFTYPE_RES_CACHE);
 		resctrl_file_fflags_init("event_filter", RFTYPE_ASSIGN_CONFIG);
+		if (r->mon.mbm_cntr_configurable)
+			resctrl_file_mode_init("event_filter", 0644);
 		resctrl_file_fflags_init("mbm_assign_on_mkdir", RFTYPE_MON_INFO |
 					 RFTYPE_RES_CACHE);
 		resctrl_file_fflags_init("mbm_L3_assignments", RFTYPE_MON_BASE);
diff --git a/fs/resctrl/rdtgroup.c b/fs/resctrl/rdtgroup.c
index 5dfdaa6f9d8ff..af2cbab14497e 100644
--- a/fs/resctrl/rdtgroup.c
+++ b/fs/resctrl/rdtgroup.c
@@ -2022,7 +2022,7 @@ static struct rftype res_common_files[] = {
 	},
 	{
 		.name		= "event_filter",
-		.mode		= 0644,
+		.mode		= 0444,
 		.kf_ops		= &rdtgroup_kf_single_ops,
 		.seq_show	= event_filter_show,
 		.write		= event_filter_write,
@@ -2215,6 +2215,15 @@ void resctrl_file_fflags_init(const char *config, unsigned long fflags)
 		rft->fflags = fflags;
 }
 
+void resctrl_file_mode_init(const char *config, umode_t mode)
+{
+	struct rftype *rft;
+
+	rft = rdtgroup_get_rftype_by_name(config);
+	if (rft)
+		rft->mode = mode;
+}
+
 /**
  * rdtgroup_kn_mode_restrict - Restrict user access to named resctrl file
  * @r: The resource group with which the file is associated.
@@ -2331,22 +2340,19 @@ static int resctrl_mkdir_event_configs(struct rdt_resource *r, struct kernfs_nod
 			continue;
 
 		kn_subdir2 = kernfs_create_dir(kn_subdir, mevt->name, kn_subdir->mode, mevt);
-		if (IS_ERR(kn_subdir2)) {
-			ret = PTR_ERR(kn_subdir2);
-			goto out;
-		}
+		if (IS_ERR(kn_subdir2))
+			return PTR_ERR(kn_subdir2);
 
 		ret = rdtgroup_kn_set_ugid(kn_subdir2);
 		if (ret)
-			goto out;
+			return ret;
 
 		ret = rdtgroup_add_files(kn_subdir2, RFTYPE_ASSIGN_CONFIG);
 		if (ret)
-			break;
+			return ret;
 	}
 
-out:
-	return ret;
+	return 0;
 }
 
 static int rdtgroup_mkdir_info_resdir(void *priv, char *name,
@@ -2510,10 +2516,13 @@ static void mba_sc_domain_destroy(struct rdt_resource *r,
 }
 
 /*
- * MBA software controller is supported only if
- * MBM is supported and MBA is in linear scale,
- * and the MBM monitor scope is the same as MBA
- * control scope.
+ * The MBA software controller is supported only if MBM is supported and MBA is
+ * in linear scale, and the MBM monitor scope is the same as MBA control scope.
+ *
+ * The software controller cannot be supported when the MBM counters are
+ * assignable.  There is no guarantee that MBM counters are assigned to the
+ * event backing the software controller in all monitoring domains of all
+ * monitoring groups.
  */
 static bool supports_mba_mbps(void)
 {
@@ -2522,7 +2531,8 @@ static bool supports_mba_mbps(void)
 
 	return (resctrl_is_mbm_enabled() &&
 		r->alloc_capable && is_mba_linear() &&
-		r->ctrl_scope == rmbm->mon_scope);
+		r->ctrl_scope == rmbm->mon_scope &&
+		!rmbm->mon.mbm_cntr_assignable);
 }
 
 /*
@@ -2937,7 +2947,7 @@ static int rdt_parse_param(struct fs_context *fc, struct fs_parameter *param)
 		ctx->enable_cdpl2 = true;
 		return 0;
 	case Opt_mba_mbps:
-		msg = "mba_MBps requires MBM and linear scale MBA at L3 scope";
+		msg = "mba_MBps requires MBM (mbm_event mode not supported) and linear scale MBA at L3 scope";
 		if (!supports_mba_mbps())
 			return invalfc(fc, msg);
 		ctx->enable_mba_mbps = true;
diff --git a/fs/select.c b/fs/select.c
index 75978b18f48f8..bf71c9838dfe1 100644
--- a/fs/select.c
+++ b/fs/select.c
@@ -708,6 +708,17 @@ static int kern_select(int n, fd_set __user *inp, fd_set __user *outp,
 		if (copy_from_user(&tv, tvp, sizeof(tv)))
 			return -EFAULT;
 
+		/*
+		 * Reject negative components before normalisation. The seconds
+		 * sum below is performed in signed long and a crafted negative
+		 * timeval can wrap to a positive value that passes
+		 * timespec64_valid() and turns into an effectively-infinite
+		 * deadline via timespec64_add_safe()'s saturation, instead of
+		 * the -EINVAL POSIX requires for negative timeouts.
+		 */
+		if (tv.tv_sec < 0 || tv.tv_usec < 0)
+			return -EINVAL;
+
 		to = &end_time;
 		if (poll_select_set_timeout(to,
 				tv.tv_sec + (tv.tv_usec / USEC_PER_SEC),
diff --git a/fs/smb/client/cached_dir.c b/fs/smb/client/cached_dir.c
index 88d5e9a32f28b..47a51b6b397ea 100644
--- a/fs/smb/client/cached_dir.c
+++ b/fs/smb/client/cached_dir.c
@@ -691,9 +691,11 @@ bool cached_dir_lease_break(struct cifs_tcon *tcon, __u8 lease_key[16])
 			cfid->on_list = false;
 			cfids->num_entries--;
 
+			spin_lock(&tcon->tc_lock);
 			++tcon->tc_count;
 			trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count,
 					    netfs_trace_tcon_ref_get_cached_lease_break);
+			spin_unlock(&tcon->tc_lock);
 			queue_work(cfid_put_wq, &cfid->put_work);
 			spin_unlock(&cfids->cfid_list_lock);
 			return true;
@@ -851,7 +853,9 @@ void free_cached_dirs(struct cached_fids *cfids)
 	if (cfids == NULL)
 		return;
 
+	flush_workqueue(cfid_put_wq);
 	cancel_delayed_work_sync(&cfids->laundromat_work);
+	flush_workqueue(serverclose_wq);
 
 	spin_lock(&cfids->cfid_list_lock);
 	list_for_each_entry_safe(cfid, q, &cfids->entries, entry) {
diff --git a/fs/smb/client/cifs_spnego.c b/fs/smb/client/cifs_spnego.c
index 3a41bbada04c7..44c4072756804 100644
--- a/fs/smb/client/cifs_spnego.c
+++ b/fs/smb/client/cifs_spnego.c
@@ -8,6 +8,7 @@
  */
 
 #include <linux/list.h>
+#include <linux/cred.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <keys/user-type.h>
@@ -40,12 +41,27 @@ cifs_spnego_key_destroy(struct key *key)
 	kfree(key->payload.data[0]);
 }
 
+static int
+cifs_spnego_key_vet_description(const char *description)
+{
+	/*
+	 * cifs.spnego descriptions are authority-bearing inputs to cifs.upcall.
+	 * They are only valid when produced by CIFS while using the private
+	 * spnego_cred installed below.  Do not let userspace create this type
+	 * of key through request_key(2)/add_key(2), since the helper treats
+	 * pid/uid/creduid/upcall_target as kernel-originating fields.
+	 */
+	if (current_cred() != spnego_cred)
+		return -EPERM;
+	return 0;
+}
 
 /*
  * keytype for CIFS spnego keys
  */
 struct key_type cifs_spnego_key_type = {
 	.name		= "cifs.spnego",
+	.vet_description = cifs_spnego_key_vet_description,
 	.instantiate	= cifs_spnego_key_instantiate,
 	.destroy	= cifs_spnego_key_destroy,
 	.describe	= user_describe,
diff --git a/fs/smb/client/cifs_swn.c b/fs/smb/client/cifs_swn.c
index 9753a432d0998..9951817d0d7ff 100644
--- a/fs/smb/client/cifs_swn.c
+++ b/fs/smb/client/cifs_swn.c
@@ -28,10 +28,54 @@ struct cifs_swn_reg {
 	bool net_name_notify;
 	bool share_name_notify;
 	bool ip_notify;
+};
 
-	struct cifs_tcon *tcon;
+struct cifs_swn_reg_info {
+	int id;
+	unsigned int ref_count;
+	const char *net_name;
+	const char *share_name;
+	bool net_name_notify;
+	bool share_name_notify;
+	bool ip_notify;
 };
 
+static void cifs_swn_snapshot_reg(struct cifs_swn_reg *swnreg,
+				  struct cifs_swn_reg_info *info)
+{
+	info->id = swnreg->id;
+	info->ref_count = kref_read(&swnreg->ref_count);
+	info->net_name = swnreg->net_name;
+	info->share_name = swnreg->share_name;
+	info->net_name_notify = swnreg->net_name_notify;
+	info->share_name_notify = swnreg->share_name_notify;
+	info->ip_notify = swnreg->ip_notify;
+}
+
+static int cifs_swn_dup_reg(struct cifs_swn_reg *swnreg,
+			    struct cifs_swn_reg_info *info)
+{
+	cifs_swn_snapshot_reg(swnreg, info);
+
+	info->net_name = kstrdup(swnreg->net_name, GFP_KERNEL);
+	if (!info->net_name)
+		return -ENOMEM;
+
+	info->share_name = kstrdup(swnreg->share_name, GFP_KERNEL);
+	if (!info->share_name) {
+		kfree(info->net_name);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+static void cifs_swn_free_reg_info(struct cifs_swn_reg_info *info)
+{
+	kfree(info->net_name);
+	kfree(info->share_name);
+}
+
 static int cifs_swn_auth_info_krb(struct cifs_tcon *tcon, struct sk_buff *skb)
 {
 	int ret;
@@ -73,7 +117,8 @@ static int cifs_swn_auth_info_ntlm(struct cifs_tcon *tcon, struct sk_buff *skb)
  * The authentication information to connect to the witness service is bundled
  * into the message.
  */
-static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg)
+static int cifs_swn_send_register_message(struct cifs_swn_reg_info *swnreg,
+					  struct cifs_tcon *tcon)
 {
 	struct sk_buff *skb;
 	struct genlmsghdr *hdr;
@@ -109,10 +154,10 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg)
 	 * told to switch to it (client move message). In these cases we unregister from the
 	 * server address and register to the new address when we receive the notification.
 	 */
-	if (swnreg->tcon->ses->server->use_swn_dstaddr)
-		addr = &swnreg->tcon->ses->server->swn_dstaddr;
+	if (tcon->ses->server->use_swn_dstaddr)
+		addr = &tcon->ses->server->swn_dstaddr;
 	else
-		addr = &swnreg->tcon->ses->server->dstaddr;
+		addr = &tcon->ses->server->dstaddr;
 
 	ret = nla_put(skb, CIFS_GENL_ATTR_SWN_IP, sizeof(struct sockaddr_storage), addr);
 	if (ret < 0)
@@ -136,10 +181,10 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg)
 			goto nlmsg_fail;
 	}
 
-	authtype = cifs_select_sectype(swnreg->tcon->ses->server, swnreg->tcon->ses->sectype);
+	authtype = cifs_select_sectype(tcon->ses->server, tcon->ses->sectype);
 	switch (authtype) {
 	case Kerberos:
-		ret = cifs_swn_auth_info_krb(swnreg->tcon, skb);
+		ret = cifs_swn_auth_info_krb(tcon, skb);
 		if (ret < 0) {
 			cifs_dbg(VFS, "%s: Failed to get kerberos auth info: %d\n", __func__, ret);
 			goto nlmsg_fail;
@@ -147,7 +192,7 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg)
 		break;
 	case NTLMv2:
 	case RawNTLMSSP:
-		ret = cifs_swn_auth_info_ntlm(swnreg->tcon, skb);
+		ret = cifs_swn_auth_info_ntlm(tcon, skb);
 		if (ret < 0) {
 			cifs_dbg(VFS, "%s: Failed to get NTLM auth info: %d\n", __func__, ret);
 			goto nlmsg_fail;
@@ -176,7 +221,8 @@ static int cifs_swn_send_register_message(struct cifs_swn_reg *swnreg)
 /*
  * Sends an uregister message to the userspace daemon based on the registration
  */
-static int cifs_swn_send_unregister_message(struct cifs_swn_reg *swnreg)
+static int cifs_swn_send_unregister_message(struct cifs_swn_reg_info *swnreg,
+					    struct cifs_tcon *tcon)
 {
 	struct sk_buff *skb;
 	struct genlmsghdr *hdr;
@@ -205,7 +251,7 @@ static int cifs_swn_send_unregister_message(struct cifs_swn_reg *swnreg)
 		goto nlmsg_fail;
 
 	ret = nla_put(skb, CIFS_GENL_ATTR_SWN_IP, sizeof(struct sockaddr_storage),
-			&swnreg->tcon->ses->server->dstaddr);
+			&tcon->ses->server->dstaddr);
 	if (ret < 0)
 		goto nlmsg_fail;
 
@@ -241,6 +287,88 @@ static int cifs_swn_send_unregister_message(struct cifs_swn_reg *swnreg)
 	return ret;
 }
 
+/*
+ * Allocation-free mirror of extract_hostname() + extract_sharename() from
+ * fs/smb/client/unc.c.  Those helpers kmalloc(GFP_KERNEL); this runs under
+ * cifs_tcp_ses_lock and tcon->tc_lock, both spinlocks, so we mirror their
+ * parsing in place against the caller's stable net_name/share_name strings.
+ * Keep in sync with unc.c.
+ */
+static bool cifs_swn_tcon_matches(struct cifs_tcon *tcon,
+				  const char *net_name,
+				  const char *share_name)
+{
+	const char *unc = tcon->tree_name;
+	const char *host, *share, *delim;
+	size_t host_len, share_len;
+
+	if (!tcon->use_witness)
+		return false;
+
+	/* extract_hostname: require strlen(unc) >= 3 */
+	if (strnlen(unc, 3) < 3)
+		return false;
+	/* extract_hostname: skip all leading '\' characters */
+	for (host = unc; *host == '\\'; host++)
+		;
+	if (!*host)
+		return false;
+	delim = strchr(host, '\\');
+	if (!delim)
+		return false;
+	host_len = delim - host;
+	if (strlen(net_name) != host_len ||
+	    strncasecmp(host, net_name, host_len))
+		return false;
+
+	/* extract_sharename: start at unc + 2, then first '\' onward */
+	share = unc + 2;
+	delim = strchr(share, '\\');
+	if (!delim)
+		return false;
+	share = delim + 1;
+	share_len = strlen(share);
+
+	return strlen(share_name) == share_len &&
+	       !strncasecmp(share, share_name, share_len);
+}
+
+/*
+ * One SWN registration id represents one net/share name pair.  Multiple
+ * mounted tcons can therefore share the id.  Pick a live representative at
+ * use time instead of caching the first tcon pointer in the registration.
+ */
+static struct cifs_tcon *cifs_swn_get_tcon(struct cifs_swn_reg_info *swnreg)
+{
+	struct TCP_Server_Info *server;
+	struct cifs_ses *ses;
+	struct cifs_tcon *tcon;
+
+	spin_lock(&cifs_tcp_ses_lock);
+	list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) {
+		list_for_each_entry(ses, &server->smb_ses_list, smb_ses_list) {
+			list_for_each_entry(tcon, &ses->tcon_list, tcon_list) {
+				spin_lock(&tcon->tc_lock);
+				if (tcon->status == TID_EXITING ||
+				    !cifs_swn_tcon_matches(tcon, swnreg->net_name,
+							   swnreg->share_name)) {
+					spin_unlock(&tcon->tc_lock);
+					continue;
+				}
+				++tcon->tc_count;
+				trace_smb3_tcon_ref(tcon->debug_id,
+						    tcon->tc_count,
+						    netfs_trace_tcon_ref_get_swn_notify);
+				spin_unlock(&tcon->tc_lock);
+				spin_unlock(&cifs_tcp_ses_lock);
+				return tcon;
+			}
+		}
+	}
+	spin_unlock(&cifs_tcp_ses_lock);
+	return NULL;
+}
+
 /*
  * Try to find a matching registration for the tcon's server name and share name.
  * Calls to this function must be protected by cifs_swnreg_idr_mutex.
@@ -347,8 +475,6 @@ static struct cifs_swn_reg *cifs_get_swn_reg(struct cifs_tcon *tcon)
 	reg->net_name_notify = true;
 	reg->share_name_notify = true;
 	reg->ip_notify = (tcon->capabilities & SMB2_SHARE_CAP_SCALEOUT);
-
-	reg->tcon = tcon;
 unlock:
 	mutex_unlock(&cifs_swnreg_idr_mutex);
 
@@ -368,11 +494,6 @@ static struct cifs_swn_reg *cifs_get_swn_reg(struct cifs_tcon *tcon)
 static void cifs_swn_reg_release(struct kref *ref)
 {
 	struct cifs_swn_reg *swnreg = container_of(ref, struct cifs_swn_reg, ref_count);
-	int ret;
-
-	ret = cifs_swn_send_unregister_message(swnreg);
-	if (ret < 0)
-		cifs_dbg(VFS, "%s: Failed to send unregister message: %d\n", __func__, ret);
 
 	idr_remove(&cifs_swnreg_idr, swnreg->id);
 	kfree(swnreg->net_name);
@@ -380,23 +501,33 @@ static void cifs_swn_reg_release(struct kref *ref)
 	kfree(swnreg);
 }
 
-static void cifs_put_swn_reg(struct cifs_swn_reg *swnreg)
+static void cifs_put_swn_reg_locked(struct cifs_swn_reg *swnreg,
+				    struct cifs_tcon *tcon)
 {
-	mutex_lock(&cifs_swnreg_idr_mutex);
+	if (kref_read(&swnreg->ref_count) == 1) {
+		struct cifs_swn_reg_info swnreg_info;
+		int ret;
+
+		cifs_swn_snapshot_reg(swnreg, &swnreg_info);
+		ret = cifs_swn_send_unregister_message(&swnreg_info, tcon);
+		if (ret < 0)
+			cifs_dbg(VFS, "%s: Failed to send unregister message: %d\n",
+				 __func__, ret);
+	}
+
 	kref_put(&swnreg->ref_count, cifs_swn_reg_release);
-	mutex_unlock(&cifs_swnreg_idr_mutex);
 }
 
-static int cifs_swn_resource_state_changed(struct cifs_swn_reg *swnreg, const char *name, int state)
+static int cifs_swn_resource_state_changed(struct cifs_tcon *tcon, const char *name, int state)
 {
 	switch (state) {
 	case CIFS_SWN_RESOURCE_STATE_UNAVAILABLE:
 		cifs_dbg(FYI, "%s: resource name '%s' become unavailable\n", __func__, name);
-		cifs_signal_cifsd_for_reconnect(swnreg->tcon->ses->server, true);
+		cifs_signal_cifsd_for_reconnect(tcon->ses->server, true);
 		break;
 	case CIFS_SWN_RESOURCE_STATE_AVAILABLE:
 		cifs_dbg(FYI, "%s: resource name '%s' become available\n", __func__, name);
-		cifs_signal_cifsd_for_reconnect(swnreg->tcon->ses->server, true);
+		cifs_signal_cifsd_for_reconnect(tcon->ses->server, true);
 		break;
 	case CIFS_SWN_RESOURCE_STATE_UNKNOWN:
 		cifs_dbg(FYI, "%s: resource name '%s' changed to unknown state\n", __func__, name);
@@ -502,7 +633,7 @@ static int cifs_swn_reconnect(struct cifs_tcon *tcon, struct sockaddr_storage *a
 	return ret;
 }
 
-static int cifs_swn_client_move(struct cifs_swn_reg *swnreg, struct sockaddr_storage *addr)
+static int cifs_swn_client_move(struct cifs_tcon *tcon, struct sockaddr_storage *addr)
 {
 	struct sockaddr_in *ipv4 = (struct sockaddr_in *)addr;
 	struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)addr;
@@ -512,14 +643,17 @@ static int cifs_swn_client_move(struct cifs_swn_reg *swnreg, struct sockaddr_sto
 	else if (addr->ss_family == AF_INET6)
 		cifs_dbg(FYI, "%s: move to %pI6\n", __func__, &ipv6->sin6_addr);
 
-	return cifs_swn_reconnect(swnreg->tcon, addr);
+	return cifs_swn_reconnect(tcon, addr);
 }
 
 int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info)
 {
 	struct cifs_swn_reg *swnreg;
+	struct cifs_swn_reg_info swnreg_info;
+	struct cifs_tcon *tcon;
 	char name[256];
 	int type;
+	int ret = 0;
 
 	if (info->attrs[CIFS_GENL_ATTR_SWN_REGISTRATION_ID]) {
 		int swnreg_id;
@@ -527,21 +661,34 @@ int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info)
 		swnreg_id = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_REGISTRATION_ID]);
 		mutex_lock(&cifs_swnreg_idr_mutex);
 		swnreg = idr_find(&cifs_swnreg_idr, swnreg_id);
-		mutex_unlock(&cifs_swnreg_idr_mutex);
 		if (swnreg == NULL) {
+			mutex_unlock(&cifs_swnreg_idr_mutex);
 			cifs_dbg(FYI, "%s: registration id %d not found\n", __func__, swnreg_id);
 			return -EINVAL;
 		}
+		ret = cifs_swn_dup_reg(swnreg, &swnreg_info);
+		mutex_unlock(&cifs_swnreg_idr_mutex);
+		if (ret)
+			return ret;
 	} else {
 		cifs_dbg(FYI, "%s: missing registration id attribute\n", __func__);
 		return -EINVAL;
 	}
 
+	tcon = cifs_swn_get_tcon(&swnreg_info);
+	if (!tcon) {
+		cifs_dbg(FYI, "%s: registration id %d has no live tcon\n",
+			 __func__, swnreg_info.id);
+		ret = -ENODEV;
+		goto free_info;
+	}
+
 	if (info->attrs[CIFS_GENL_ATTR_SWN_NOTIFICATION_TYPE]) {
 		type = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_NOTIFICATION_TYPE]);
 	} else {
 		cifs_dbg(FYI, "%s: missing notification type attribute\n", __func__);
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 
 	switch (type) {
@@ -553,15 +700,18 @@ int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info)
 					sizeof(name));
 		} else {
 			cifs_dbg(FYI, "%s: missing resource name attribute\n", __func__);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto out;
 		}
 		if (info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_STATE]) {
 			state = nla_get_u32(info->attrs[CIFS_GENL_ATTR_SWN_RESOURCE_STATE]);
 		} else {
 			cifs_dbg(FYI, "%s: missing resource state attribute\n", __func__);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto out;
 		}
-		return cifs_swn_resource_state_changed(swnreg, name, state);
+		ret = cifs_swn_resource_state_changed(tcon, name, state);
+		break;
 	}
 	case CIFS_SWN_NOTIFICATION_CLIENT_MOVE: {
 		struct sockaddr_storage addr;
@@ -570,28 +720,36 @@ int cifs_swn_notify(struct sk_buff *skb, struct genl_info *info)
 			nla_memcpy(&addr, info->attrs[CIFS_GENL_ATTR_SWN_IP], sizeof(addr));
 		} else {
 			cifs_dbg(FYI, "%s: missing IP address attribute\n", __func__);
-			return -EINVAL;
+			ret = -EINVAL;
+			goto out;
 		}
-		return cifs_swn_client_move(swnreg, &addr);
+		ret = cifs_swn_client_move(tcon, &addr);
+		break;
 	}
 	default:
 		cifs_dbg(FYI, "%s: unknown notification type %d\n", __func__, type);
 		break;
 	}
 
-	return 0;
+out:
+	cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify);
+free_info:
+	cifs_swn_free_reg_info(&swnreg_info);
+	return ret;
 }
 
 int cifs_swn_register(struct cifs_tcon *tcon)
 {
 	struct cifs_swn_reg *swnreg;
+	struct cifs_swn_reg_info swnreg_info;
 	int ret;
 
 	swnreg = cifs_get_swn_reg(tcon);
 	if (IS_ERR(swnreg))
 		return PTR_ERR(swnreg);
 
-	ret = cifs_swn_send_register_message(swnreg);
+	cifs_swn_snapshot_reg(swnreg, &swnreg_info);
+	ret = cifs_swn_send_register_message(&swnreg_info, tcon);
 	if (ret < 0) {
 		cifs_dbg(VFS, "%s: Failed to send swn register message: %d\n", __func__, ret);
 		/* Do not put the swnreg or return error, the echo task will retry */
@@ -612,35 +770,68 @@ int cifs_swn_unregister(struct cifs_tcon *tcon)
 		return PTR_ERR(swnreg);
 	}
 
+	cifs_put_swn_reg_locked(swnreg, tcon);
 	mutex_unlock(&cifs_swnreg_idr_mutex);
 
-	cifs_put_swn_reg(swnreg);
-
 	return 0;
 }
 
-void cifs_swn_dump(struct seq_file *m)
+/*
+ * Snapshot one registration under cifs_swnreg_idr_mutex and return.  Callers
+ * intentionally do the per-registration network/genlmsg work without the
+ * mutex held, both to keep the critical section short and to avoid nesting
+ * cifs_swnreg_idr_mutex inside the higher tc_lock when a live tcon is then
+ * pinned for the send.
+ */
+static int cifs_swn_get_next_reg_info(int *id, struct cifs_swn_reg_info *info)
 {
 	struct cifs_swn_reg *swnreg;
+	int ret = 0;
+
+	mutex_lock(&cifs_swnreg_idr_mutex);
+	swnreg = idr_get_next(&cifs_swnreg_idr, id);
+	if (swnreg) {
+		ret = cifs_swn_dup_reg(swnreg, info);
+		if (!ret) {
+			*id = swnreg->id + 1;
+			ret = 1;
+		}
+	}
+	mutex_unlock(&cifs_swnreg_idr_mutex);
+
+	return ret;
+}
+
+void cifs_swn_dump(struct seq_file *m)
+{
+	struct cifs_swn_reg_info swnreg_info;
+	struct cifs_tcon *tcon;
 	struct sockaddr_in *sa;
 	struct sockaddr_in6 *sa6;
-	int id;
+	int id = 0;
+	int ret;
 
 	seq_puts(m, "Witness registrations:");
 
-	mutex_lock(&cifs_swnreg_idr_mutex);
-	idr_for_each_entry(&cifs_swnreg_idr, swnreg, id) {
+	while ((ret = cifs_swn_get_next_reg_info(&id, &swnreg_info)) > 0) {
 		seq_printf(m, "\nId: %u Refs: %u Network name: '%s'%s Share name: '%s'%s Ip address: ",
-				id, kref_read(&swnreg->ref_count),
-				swnreg->net_name, swnreg->net_name_notify ? "(y)" : "(n)",
-				swnreg->share_name, swnreg->share_name_notify ? "(y)" : "(n)");
-		switch (swnreg->tcon->ses->server->dstaddr.ss_family) {
+			   swnreg_info.id, swnreg_info.ref_count,
+			   swnreg_info.net_name, swnreg_info.net_name_notify ? "(y)" : "(n)",
+			   swnreg_info.share_name, swnreg_info.share_name_notify ? "(y)" : "(n)");
+
+		tcon = cifs_swn_get_tcon(&swnreg_info);
+		if (!tcon) {
+			seq_puts(m, "(no live tcon)");
+			goto next;
+		}
+
+		switch (tcon->ses->server->dstaddr.ss_family) {
 		case AF_INET:
-			sa = (struct sockaddr_in *) &swnreg->tcon->ses->server->dstaddr;
+			sa = (struct sockaddr_in *)&tcon->ses->server->dstaddr;
 			seq_printf(m, "%pI4", &sa->sin_addr.s_addr);
 			break;
 		case AF_INET6:
-			sa6 = (struct sockaddr_in6 *) &swnreg->tcon->ses->server->dstaddr;
+			sa6 = (struct sockaddr_in6 *)&tcon->ses->server->dstaddr;
 			seq_printf(m, "%pI6", &sa6->sin6_addr.s6_addr);
 			if (sa6->sin6_scope_id)
 				seq_printf(m, "%%%u", sa6->sin6_scope_id);
@@ -648,23 +839,38 @@ void cifs_swn_dump(struct seq_file *m)
 		default:
 			seq_puts(m, "(unknown)");
 		}
-		seq_printf(m, "%s", swnreg->ip_notify ? "(y)" : "(n)");
+		cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify);
+next:
+		seq_printf(m, "%s", swnreg_info.ip_notify ? "(y)" : "(n)");
+		cifs_swn_free_reg_info(&swnreg_info);
 	}
-	mutex_unlock(&cifs_swnreg_idr_mutex);
+	if (ret < 0)
+		seq_printf(m, "\nFailed to snapshot witness registration: %d", ret);
 	seq_puts(m, "\n");
 }
 
 void cifs_swn_check(void)
 {
-	struct cifs_swn_reg *swnreg;
-	int id;
+	struct cifs_swn_reg_info swnreg_info;
+	struct cifs_tcon *tcon;
+	int id = 0;
 	int ret;
 
-	mutex_lock(&cifs_swnreg_idr_mutex);
-	idr_for_each_entry(&cifs_swnreg_idr, swnreg, id) {
-		ret = cifs_swn_send_register_message(swnreg);
+	while ((ret = cifs_swn_get_next_reg_info(&id, &swnreg_info)) > 0) {
+		tcon = cifs_swn_get_tcon(&swnreg_info);
+		if (!tcon) {
+			cifs_dbg(FYI, "%s: registration id %d has no live tcon\n",
+				 __func__, swnreg_info.id);
+			goto free_info;
+		}
+
+		ret = cifs_swn_send_register_message(&swnreg_info, tcon);
 		if (ret < 0)
 			cifs_dbg(FYI, "%s: Failed to send register message: %d\n", __func__, ret);
+		cifs_put_tcon(tcon, netfs_trace_tcon_ref_put_swn_notify);
+free_info:
+		cifs_swn_free_reg_info(&swnreg_info);
 	}
-	mutex_unlock(&cifs_swnreg_idr_mutex);
+	if (ret < 0)
+		cifs_dbg(FYI, "%s: Failed to snapshot registration: %d\n", __func__, ret);
 }
diff --git a/fs/smb/client/cifsacl.c b/fs/smb/client/cifsacl.c
index 786dbbc43c5b9..42a3115359dac 100644
--- a/fs/smb/client/cifsacl.c
+++ b/fs/smb/client/cifsacl.c
@@ -275,6 +275,73 @@ cifs_copy_sid(struct smb_sid *dst, const struct smb_sid *src)
 	return size;
 }
 
+static int parse_sid(const struct smb_sid *psid, const char *end_of_acl)
+{
+	unsigned int sid_len;
+
+	/* SID must contain the fixed header before num_subauth is trusted. */
+	if (end_of_acl < (const char *)psid + CIFS_SID_BASE_SIZE) {
+		cifs_dbg(VFS, "ACL too small to parse SID %p\n", psid);
+		return -EINVAL;
+	}
+	if (psid->num_subauth > SID_MAX_SUB_AUTHORITIES) {
+		cifs_dbg(VFS, "SID contains too many subauthorities %u\n",
+			 psid->num_subauth);
+		return -EINVAL;
+	}
+
+	sid_len = CIFS_SID_BASE_SIZE + psid->num_subauth * sizeof(__le32);
+	if (end_of_acl < (const char *)psid + sid_len) {
+		cifs_dbg(VFS, "ACL too small to parse SID %p\n", psid);
+		return -EINVAL;
+	}
+
+#ifdef CONFIG_CIFS_DEBUG2
+	if (psid->num_subauth) {
+		int i;
+
+		cifs_dbg(FYI, "SID revision %d num_auth %d\n",
+			 psid->revision, psid->num_subauth);
+
+		for (i = 0; i < psid->num_subauth; i++) {
+			cifs_dbg(FYI, "SID sub_auth[%d]: 0x%x\n",
+				 i, le32_to_cpu(psid->sub_auth[i]));
+		}
+
+		cifs_dbg(FYI, "RID 0x%x\n",
+			 le32_to_cpu(psid->sub_auth[psid->num_subauth - 1]));
+	}
+#endif
+
+	return 0;
+}
+
+static int sid_from_sd(const struct smb_ntsd *pntsd, __u32 secdesclen,
+		       __u32 sid_offset, struct smb_sid **sid)
+{
+	struct smb_sid *psid;
+	char *end_of_acl;
+
+	if (secdesclen < sizeof(struct smb_ntsd)) {
+		cifs_dbg(VFS, "ACL too small to parse security descriptor\n");
+		return -EINVAL;
+	}
+	end_of_acl = (char *)pntsd + secdesclen;
+
+	if (sid_offset < sizeof(struct smb_ntsd) ||
+	    sid_offset > secdesclen - CIFS_SID_BASE_SIZE) {
+		cifs_dbg(VFS, "Server returned illegal SID offset\n");
+		return -EINVAL;
+	}
+
+	psid = (struct smb_sid *)((char *)pntsd + sid_offset);
+	if (parse_sid(psid, end_of_acl))
+		return -EINVAL;
+
+	*sid = psid;
+	return 0;
+}
+
 static int
 id_to_sid(unsigned int cid, uint sidtype, struct smb_sid *ssid)
 {
@@ -515,14 +582,14 @@ exit_cifs_idmap(void)
 }
 
 /* copy ntsd, owner sid, and group sid from a security descriptor to another */
-static __u32 copy_sec_desc(const struct smb_ntsd *pntsd,
-				struct smb_ntsd *pnntsd,
-				__u32 sidsoffset,
-				struct smb_sid *pownersid,
-				struct smb_sid *pgrpsid)
+static int copy_sec_desc(const struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd,
+			 __u32 sidsoffset, __u32 secdesclen,
+			 __u32 *pnsecdesclen, struct smb_sid *pownersid,
+			 struct smb_sid *pgrpsid)
 {
 	struct smb_sid *owner_sid_ptr, *group_sid_ptr;
 	struct smb_sid *nowner_sid_ptr, *ngroup_sid_ptr;
+	int rc;
 
 	/* copy security descriptor control portion */
 	pnntsd->revision = pntsd->revision;
@@ -533,28 +600,34 @@ static __u32 copy_sec_desc(const struct smb_ntsd *pntsd,
 	pnntsd->gsidoffset = cpu_to_le32(sidsoffset + sizeof(struct smb_sid));
 
 	/* copy owner sid */
-	if (pownersid)
+	if (pownersid) {
 		owner_sid_ptr = pownersid;
-	else
-		owner_sid_ptr = (struct smb_sid *)((char *)pntsd +
-				le32_to_cpu(pntsd->osidoffset));
+	} else {
+		rc = sid_from_sd(pntsd, secdesclen,
+				 le32_to_cpu(pntsd->osidoffset), &owner_sid_ptr);
+		if (rc)
+			return rc;
+	}
 	nowner_sid_ptr = (struct smb_sid *)((char *)pnntsd + sidsoffset);
 	cifs_copy_sid(nowner_sid_ptr, owner_sid_ptr);
 
 	/* copy group sid */
-	if (pgrpsid)
+	if (pgrpsid) {
 		group_sid_ptr = pgrpsid;
-	else
-		group_sid_ptr = (struct smb_sid *)((char *)pntsd +
-				le32_to_cpu(pntsd->gsidoffset));
+	} else {
+		rc = sid_from_sd(pntsd, secdesclen,
+				 le32_to_cpu(pntsd->gsidoffset), &group_sid_ptr);
+		if (rc)
+			return rc;
+	}
 	ngroup_sid_ptr = (struct smb_sid *)((char *)pnntsd + sidsoffset +
 					sizeof(struct smb_sid));
 	cifs_copy_sid(ngroup_sid_ptr, group_sid_ptr);
 
-	return sidsoffset + (2 * sizeof(struct smb_sid));
+	*pnsecdesclen = sidsoffset + (2 * sizeof(struct smb_sid));
+	return 0;
 }
 
-
 /*
    change posix mode to reflect permissions
    pmode is the existing mode (we only want to overwrite part of this
@@ -1232,38 +1305,6 @@ static int set_chmod_dacl(struct smb_acl *pdacl, struct smb_acl *pndacl,
 	return 0;
 }
 
-static int parse_sid(struct smb_sid *psid, char *end_of_acl)
-{
-	/* BB need to add parm so we can store the SID BB */
-
-	/* validate that we do not go past end of ACL - sid must be at least 8
-	   bytes long (assuming no sub-auths - e.g. the null SID */
-	if (end_of_acl < (char *)psid + 8) {
-		cifs_dbg(VFS, "ACL too small to parse SID %p\n", psid);
-		return -EINVAL;
-	}
-
-#ifdef CONFIG_CIFS_DEBUG2
-	if (psid->num_subauth) {
-		int i;
-		cifs_dbg(FYI, "SID revision %d num_auth %d\n",
-			 psid->revision, psid->num_subauth);
-
-		for (i = 0; i < psid->num_subauth; i++) {
-			cifs_dbg(FYI, "SID sub_auth[%d]: 0x%x\n",
-				 i, le32_to_cpu(psid->sub_auth[i]));
-		}
-
-		/* BB add length check to make sure that we do not have huge
-			num auths and therefore go off the end */
-		cifs_dbg(FYI, "RID 0x%x\n",
-			 le32_to_cpu(psid->sub_auth[psid->num_subauth-1]));
-	}
-#endif
-
-	return 0;
-}
-
 static bool dacl_offset_valid(unsigned int acl_len, __u32 dacloffset)
 {
 	if (acl_len < sizeof(struct smb_acl))
@@ -1284,23 +1325,25 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb,
 	int rc = 0;
 	struct smb_sid *owner_sid_ptr, *group_sid_ptr;
 	struct smb_acl *dacl_ptr; /* no need for SACL ptr */
-	char *end_of_acl = ((char *)pntsd) + acl_len;
-	__u32 dacloffset;
+	char *end_of_acl;
+	__u32 dacloffset, osidoffset, gsidoffset;
 
 	if (pntsd == NULL)
 		return smb_EIO(smb_eio_trace_null_pointers);
+	if (acl_len < (int)sizeof(struct smb_ntsd)) {
+		cifs_dbg(VFS, "ACL too small to parse security descriptor\n");
+		return -EINVAL;
+	}
+	end_of_acl = ((char *)pntsd) + acl_len;
 
-	owner_sid_ptr = (struct smb_sid *)((char *)pntsd +
-				le32_to_cpu(pntsd->osidoffset));
-	group_sid_ptr = (struct smb_sid *)((char *)pntsd +
-				le32_to_cpu(pntsd->gsidoffset));
+	osidoffset = le32_to_cpu(pntsd->osidoffset);
+	gsidoffset = le32_to_cpu(pntsd->gsidoffset);
 	dacloffset = le32_to_cpu(pntsd->dacloffset);
 	cifs_dbg(NOISY, "revision %d type 0x%x ooffset 0x%x goffset 0x%x sacloffset 0x%x dacloffset 0x%x\n",
-		 pntsd->revision, pntsd->type, le32_to_cpu(pntsd->osidoffset),
-		 le32_to_cpu(pntsd->gsidoffset),
+		 pntsd->revision, pntsd->type, osidoffset, gsidoffset,
 		 le32_to_cpu(pntsd->sacloffset), dacloffset);
 /*	cifs_dump_mem("owner_sid: ", owner_sid_ptr, 64); */
-	rc = parse_sid(owner_sid_ptr, end_of_acl);
+	rc = sid_from_sd(pntsd, acl_len, osidoffset, &owner_sid_ptr);
 	if (rc) {
 		cifs_dbg(FYI, "%s: Error %d parsing Owner SID\n", __func__, rc);
 		return rc;
@@ -1312,9 +1355,9 @@ static int parse_sec_desc(struct cifs_sb_info *cifs_sb,
 		return rc;
 	}
 
-	rc = parse_sid(group_sid_ptr, end_of_acl);
+	rc = sid_from_sd(pntsd, acl_len, gsidoffset, &group_sid_ptr);
 	if (rc) {
-		cifs_dbg(FYI, "%s: Error %d mapping Owner SID to gid\n",
+		cifs_dbg(FYI, "%s: Error %d parsing Group SID\n",
 			 __func__, rc);
 		return rc;
 	}
@@ -1354,8 +1397,15 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd,
 	struct smb_sid *nowner_sid_ptr = NULL, *ngroup_sid_ptr = NULL;
 	struct smb_acl *dacl_ptr = NULL;  /* no need for SACL ptr */
 	struct smb_acl *ndacl_ptr = NULL; /* no need for SACL ptr */
-	char *end_of_acl = ((char *)pntsd) + secdesclen;
+	char *end_of_acl;
 	u16 size = 0;
+	__u32 osidoffset, gsidoffset;
+
+	if (secdesclen < sizeof(struct smb_ntsd)) {
+		cifs_dbg(VFS, "ACL too small to parse security descriptor\n");
+		return -EINVAL;
+	}
+	end_of_acl = ((char *)pntsd) + secdesclen;
 
 	dacloffset = le32_to_cpu(pntsd->dacloffset);
 	if (dacloffset) {
@@ -1370,10 +1420,18 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd,
 			return rc;
 	}
 
-	owner_sid_ptr = (struct smb_sid *)((char *)pntsd +
-			le32_to_cpu(pntsd->osidoffset));
-	group_sid_ptr = (struct smb_sid *)((char *)pntsd +
-			le32_to_cpu(pntsd->gsidoffset));
+	osidoffset = le32_to_cpu(pntsd->osidoffset);
+	gsidoffset = le32_to_cpu(pntsd->gsidoffset);
+	rc = sid_from_sd(pntsd, secdesclen, osidoffset, &owner_sid_ptr);
+	if (rc) {
+		cifs_dbg(FYI, "%s: Error %d parsing Owner SID\n", __func__, rc);
+		return rc;
+	}
+	rc = sid_from_sd(pntsd, secdesclen, gsidoffset, &group_sid_ptr);
+	if (rc) {
+		cifs_dbg(FYI, "%s: Error %d parsing Group SID\n", __func__, rc);
+		return rc;
+	}
 
 	if (pnmode && *pnmode != NO_CHANGE_64) { /* chmod */
 		ndacloffset = sizeof(struct smb_ntsd);
@@ -1389,8 +1447,10 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd,
 
 		sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size);
 		/* copy the non-dacl portion of secdesc */
-		*pnsecdesclen = copy_sec_desc(pntsd, pnntsd, sidsoffset,
-				NULL, NULL);
+		rc = copy_sec_desc(pntsd, pnntsd, sidsoffset, secdesclen,
+				   pnsecdesclen, NULL, NULL);
+		if (rc)
+			return rc;
 
 		*aclflag |= CIFS_ACL_DACL;
 	} else {
@@ -1467,8 +1527,10 @@ static int build_sec_desc(struct smb_ntsd *pntsd, struct smb_ntsd *pnntsd,
 
 		sidsoffset = ndacloffset + le16_to_cpu(ndacl_ptr->size);
 		/* copy the non-dacl portion of secdesc */
-		*pnsecdesclen = copy_sec_desc(pntsd, pnntsd, sidsoffset,
-				nowner_sid_ptr, ngroup_sid_ptr);
+		rc = copy_sec_desc(pntsd, pnntsd, sidsoffset, secdesclen,
+				   pnsecdesclen, nowner_sid_ptr, ngroup_sid_ptr);
+		if (rc)
+			goto chown_chgrp_exit;
 
 chown_chgrp_exit:
 		/* errors could jump here. So make sure we return soon after this */
diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c
index 9f76b0347fa9d..b14ae32835530 100644
--- a/fs/smb/client/cifsfs.c
+++ b/fs/smb/client/cifsfs.c
@@ -30,6 +30,7 @@
 #include <linux/xattr.h>
 #include <linux/mm.h>
 #include <linux/key-type.h>
+#include <linux/fileattr.h>
 #include <uapi/linux/magic.h>
 #include <net/ipv6.h>
 #include "cifsfs.h"
@@ -306,6 +307,8 @@ static void cifs_kill_sb(struct super_block *sb)
 
 		/* Wait for all pending oplock breaks to complete */
 		flush_workqueue(cifsoplockd_wq);
+		/* Wait for all opened files to release */
+		flush_workqueue(deferredclose_wq);
 
 		/* finally release root dentry */
 		dput(cifs_sb->root);
@@ -434,7 +437,8 @@ cifs_alloc_inode(struct super_block *sb)
 	spin_lock_init(&cifs_inode->writers_lock);
 	cifs_inode->writers = 0;
 	cifs_inode->netfs.inode.i_blkbits = 14;  /* 2**14 = CIFS_MAX_MSGSIZE */
-	cifs_inode->netfs.remote_i_size = 0;
+	cifs_inode->netfs._remote_i_size = 0;
+	cifs_inode->netfs._zero_point = 0;
 	cifs_inode->uniqueid = 0;
 	cifs_inode->createtime = 0;
 	cifs_inode->epoch = 0;
@@ -1162,6 +1166,56 @@ struct file_system_type smb3_fs_type = {
 MODULE_ALIAS_FS("smb3");
 MODULE_ALIAS("smb3");
 
+int cifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
+{
+	struct cifs_sb_info *cifs_sb = CIFS_SB(dentry->d_sb);
+	struct cifs_tcon *tcon = cifs_sb_master_tcon(cifs_sb);
+	struct inode *inode = d_inode(dentry);
+	u32 attrs;
+
+	/* Preserve FS_COMPR_FL previously reported by cifs_ioctl(). */
+	if (CIFS_I(inode)->cifsAttrs & ATTR_COMPRESSED)
+		fa->flags |= FS_COMPR_FL;
+
+	/*
+	 * FS_CASEFOLD_FL is defined by UAPI as a folder attribute,
+	 * and userspace tools (e.g., lsattr) display it only on
+	 * directories. Confine the case-handling bits to directories
+	 * to match that convention; for non-directories the share's
+	 * case semantics are still discoverable through the parent.
+	 */
+	if (!S_ISDIR(inode->i_mode))
+		return 0;
+
+	/*
+	 * The server's FS_ATTRIBUTE_INFORMATION response, cached on
+	 * the tcon at mount, reflects the share's case-handling
+	 * semantics after any POSIX extensions negotiation. Prefer
+	 * it over the client-local nocase mount option, which only
+	 * governs dentry comparison on this superblock.
+	 *
+	 * QueryFSInfo is best-effort at mount; when it did not
+	 * populate fsAttrInfo, MaxPathNameComponentLength remains
+	 * zero. In that case fall back to nocase so the reporting
+	 * matches the comparison behavior installed on the sb.
+	 */
+	if (le32_to_cpu(tcon->fsAttrInfo.MaxPathNameComponentLength) == 0) {
+		if (tcon->nocase) {
+			fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
+			fa->flags |= FS_CASEFOLD_FL;
+		}
+		return 0;
+	}
+	attrs = le32_to_cpu(tcon->fsAttrInfo.Attributes);
+	if (!(attrs & FILE_CASE_SENSITIVE_SEARCH)) {
+		fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
+		fa->flags |= FS_CASEFOLD_FL;
+	}
+	if (!(attrs & FILE_CASE_PRESERVED_NAMES))
+		fa->fsx_xflags |= FS_XFLAG_CASENONPRESERVING;
+	return 0;
+}
+
 const struct inode_operations cifs_dir_inode_ops = {
 	.create = cifs_create,
 	.atomic_open = cifs_atomic_open,
@@ -1180,6 +1234,7 @@ const struct inode_operations cifs_dir_inode_ops = {
 	.listxattr = cifs_listxattr,
 	.get_acl = cifs_get_acl,
 	.set_acl = cifs_set_acl,
+	.fileattr_get = cifs_fileattr_get,
 };
 
 const struct inode_operations cifs_file_inode_ops = {
@@ -1190,6 +1245,7 @@ const struct inode_operations cifs_file_inode_ops = {
 	.fiemap = cifs_fiemap,
 	.get_acl = cifs_get_acl,
 	.set_acl = cifs_set_acl,
+	.fileattr_get = cifs_fileattr_get,
 };
 
 const char *cifs_get_link(struct dentry *dentry, struct inode *inode,
@@ -1303,7 +1359,8 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off,
 	struct cifsFileInfo *smb_file_src = src_file->private_data;
 	struct cifsFileInfo *smb_file_target = dst_file->private_data;
 	struct cifs_tcon *target_tcon, *src_tcon;
-	unsigned long long destend, fstart, fend, old_size, new_size;
+	unsigned long long i_size, new_size;
+	unsigned long long destend, fstart, fend;
 	unsigned int xid;
 	int rc;
 
@@ -1347,7 +1404,7 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off,
 	 * Advance the EOF marker after the flush above to the end of the range
 	 * if it's short of that.
 	 */
-	if (src_cifsi->netfs.remote_i_size < off + len) {
+	if (netfs_read_remote_i_size(src_inode) < off + len) {
 		rc = cifs_precopy_set_eof(src_inode, src_cifsi, src_tcon, xid, off + len);
 		if (rc < 0)
 			goto unlock;
@@ -1368,22 +1425,24 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off,
 	rc = cifs_flush_folio(target_inode, destend, &fstart, &fend, false);
 	if (rc)
 		goto unlock;
-	if (fend > target_cifsi->netfs.zero_point)
-		target_cifsi->netfs.zero_point = fend + 1;
-	old_size = target_cifsi->netfs.remote_i_size;
+
+	spin_lock(&target_inode->i_lock);
+	if (fend > target_cifsi->netfs._zero_point)
+		netfs_write_zero_point(target_inode, fend + 1);
+	i_size = target_inode->i_size;
+	spin_unlock(&target_inode->i_lock);
 
 	/* Discard all the folios that overlap the destination region. */
 	cifs_dbg(FYI, "about to discard pages %llx-%llx\n", fstart, fend);
 	truncate_inode_pages_range(&target_inode->i_data, fstart, fend);
 
-	fscache_invalidate(cifs_inode_cookie(target_inode), NULL,
-			   i_size_read(target_inode), 0);
+	fscache_invalidate(cifs_inode_cookie(target_inode), NULL, i_size, 0);
 
 	rc = -EOPNOTSUPP;
 	if (target_tcon->ses->server->ops->duplicate_extents) {
 		rc = target_tcon->ses->server->ops->duplicate_extents(xid,
 			smb_file_src, smb_file_target, off, len, destoff);
-		if (rc == 0 && new_size > old_size) {
+		if (rc == 0 && new_size > i_size) {
 			truncate_setsize(target_inode, new_size);
 			fscache_resize_cookie(cifs_inode_cookie(target_inode),
 					      new_size);
@@ -1402,8 +1461,12 @@ static loff_t cifs_remap_file_range(struct file *src_file, loff_t off,
 					rc = -EINVAL;
 			}
 		}
-		if (rc == 0 && new_size > target_cifsi->netfs.zero_point)
-			target_cifsi->netfs.zero_point = new_size;
+		if (rc == 0) {
+			spin_lock(&target_inode->i_lock);
+			if (new_size > target_cifsi->netfs._zero_point)
+				netfs_write_zero_point(target_inode, new_size);
+			spin_unlock(&target_inode->i_lock);
+		}
 	}
 
 	/* force revalidate of size and timestamps of target file now
@@ -1474,7 +1537,7 @@ ssize_t cifs_file_copychunk_range(unsigned int xid,
 	 * Advance the EOF marker after the flush above to the end of the range
 	 * if it's short of that.
 	 */
-	if (src_cifsi->netfs.remote_i_size < off + len) {
+	if (netfs_read_remote_i_size(src_inode) < off + len) {
 		rc = cifs_precopy_set_eof(src_inode, src_cifsi, src_tcon, xid, off + len);
 		if (rc < 0)
 			goto unlock;
@@ -1502,8 +1565,12 @@ ssize_t cifs_file_copychunk_range(unsigned int xid,
 			fscache_resize_cookie(cifs_inode_cookie(target_inode),
 					      i_size_read(target_inode));
 		}
-		if (rc > 0 && destoff + rc > target_cifsi->netfs.zero_point)
-			target_cifsi->netfs.zero_point = destoff + rc;
+		if (rc > 0) {
+			spin_lock(&target_inode->i_lock);
+			if (destoff + rc > target_cifsi->netfs._zero_point)
+				netfs_write_zero_point(target_inode, destoff + rc);
+			spin_unlock(&target_inode->i_lock);
+		}
 	}
 
 	file_accessed(src_file);
diff --git a/fs/smb/client/cifsfs.h b/fs/smb/client/cifsfs.h
index c455b15f27782..9d85224fafab5 100644
--- a/fs/smb/client/cifsfs.h
+++ b/fs/smb/client/cifsfs.h
@@ -89,6 +89,9 @@ extern const struct inode_operations cifs_file_inode_ops;
 extern const struct inode_operations cifs_symlink_inode_ops;
 extern const struct inode_operations cifs_namespace_inode_operations;
 
+struct file_kattr;
+int cifs_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
+
 
 /* Functions related to files and directories */
 extern const struct netfs_request_ops cifs_req_ops;
diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c
index 3990a90122640..9e27bfa7376b1 100644
--- a/fs/smb/client/cifssmb.c
+++ b/fs/smb/client/cifssmb.c
@@ -1465,6 +1465,7 @@ cifs_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 	struct cifs_io_subrequest *rdata = mid->callback_data;
 	struct netfs_inode *ictx = netfs_inode(rdata->rreq->inode);
 	struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink);
+	struct inode *inode = &ictx->inode;
 	struct smb_rqst rqst = { .rq_iov = rdata->iov,
 				 .rq_nvec = 1,
 				 .rq_iter = rdata->subreq.io_iter };
@@ -1538,7 +1539,7 @@ cifs_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 	} else {
 		size_t trans = rdata->subreq.transferred + rdata->got_bytes;
 		if (trans < rdata->subreq.len &&
-		    rdata->subreq.start + trans >= ictx->remote_i_size) {
+		    rdata->subreq.start + trans >= netfs_read_remote_i_size(inode)) {
 			rdata->result = 0;
 			__set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
 		} else if (rdata->got_bytes > 0) {
diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c
index dcde25da468df..c55b4c040a303 100644
--- a/fs/smb/client/connect.c
+++ b/fs/smb/client/connect.c
@@ -2605,6 +2605,11 @@ cifs_put_tcon(struct cifs_tcon *tcon, enum smb3_tcon_ref_trace trace)
 	cifs_dbg(FYI, "%s: tc_count=%d\n", __func__, tcon->tc_count);
 	spin_lock(&cifs_tcp_ses_lock);
 	spin_lock(&tcon->tc_lock);
+	if (tcon->status == TID_EXITING) {
+		spin_unlock(&tcon->tc_lock);
+		spin_unlock(&cifs_tcp_ses_lock);
+		return;
+	}
 	trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count - 1, trace);
 	if (--tcon->tc_count > 0) {
 		spin_unlock(&tcon->tc_lock);
diff --git a/fs/smb/client/dir.c b/fs/smb/client/dir.c
index e4295a5b55b34..ea1c3ea33aa6b 100644
--- a/fs/smb/client/dir.c
+++ b/fs/smb/client/dir.c
@@ -241,6 +241,12 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry,
 				goto cifs_create_get_file_info;
 			}
 
+			if ((oflags & OPENAT2_REGULAR) && !S_ISREG(newinode->i_mode)) {
+				CIFSSMBClose(xid, tcon, fid->netfid);
+				iput(newinode);
+				return -EFTYPE;
+			}
+
 			if (S_ISDIR(newinode->i_mode)) {
 				CIFSSMBClose(xid, tcon, fid->netfid);
 				iput(newinode);
@@ -458,9 +464,15 @@ static int __cifs_do_create(struct inode *dir, struct dentry *direntry,
 		goto out_err;
 	}
 
-	if (newinode && S_ISDIR(newinode->i_mode)) {
-		rc = -EISDIR;
-		goto out_err;
+	if (newinode) {
+		if ((oflags & OPENAT2_REGULAR) && !S_ISREG(newinode->i_mode)) {
+			rc = -EFTYPE;
+			goto out_err;
+		}
+		if (S_ISDIR(newinode->i_mode)) {
+			rc = -EISDIR;
+			goto out_err;
+		}
 	}
 
 	*inode = newinode;
diff --git a/fs/smb/client/file.c b/fs/smb/client/file.c
index 664a2c2230890..b60344125f271 100644
--- a/fs/smb/client/file.c
+++ b/fs/smb/client/file.c
@@ -2517,18 +2517,23 @@ int cifs_lock(struct file *file, int cmd, struct file_lock *flock)
 void cifs_write_subrequest_terminated(struct cifs_io_subrequest *wdata, ssize_t result)
 {
 	struct netfs_io_request *wreq = wdata->rreq;
-	struct netfs_inode *ictx = netfs_inode(wreq->inode);
+	struct inode *inode = wreq->inode;
+	struct netfs_inode *ictx = netfs_inode(inode);
 	loff_t wrend;
 
 	if (result > 0) {
+		spin_lock(&inode->i_lock);
+
 		wrend = wdata->subreq.start + wdata->subreq.transferred + result;
 
-		if (wrend > ictx->zero_point &&
+		if (wrend > ictx->_zero_point &&
 		    (wdata->rreq->origin == NETFS_UNBUFFERED_WRITE ||
 		     wdata->rreq->origin == NETFS_DIO_WRITE))
-			ictx->zero_point = wrend;
-		if (wrend > ictx->remote_i_size)
+			netfs_write_zero_point(inode, wrend);
+		if (wrend > ictx->_remote_i_size)
 			netfs_resize_file(ictx, wrend, true);
+
+		spin_unlock(&inode->i_lock);
 	}
 
 	netfs_write_subrequest_terminated(&wdata->subreq, result);
diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c
index 4c8b1b9ade8b8..2f86158f85d7b 100644
--- a/fs/smb/client/fs_context.c
+++ b/fs/smb/client/fs_context.c
@@ -420,7 +420,7 @@ static int parse_symlink_flavor(struct fs_context *fc, char *value,
 #define DUP_CTX_STR(field)						\
 do {									\
 	if (ctx->field) {						\
-		new_ctx->field = kstrdup(ctx->field, GFP_ATOMIC);	\
+		new_ctx->field = kstrdup(ctx->field, GFP_KERNEL);	\
 		if (new_ctx->field == NULL) {				\
 			smb3_cleanup_fs_context_contents(new_ctx);	\
 			return -ENOMEM;					\
diff --git a/fs/smb/client/inode.c b/fs/smb/client/inode.c
index 16a5310155d5a..51fb7c418d52a 100644
--- a/fs/smb/client/inode.c
+++ b/fs/smb/client/inode.c
@@ -28,6 +28,23 @@
 #include "cached_dir.h"
 #include "reparse.h"
 
+static void cifs_invalidate_cached_dir(struct cifs_tcon *tcon,
+				       struct dentry *parent)
+{
+	struct cached_fid *parent_cfid = NULL;
+
+	if (!tcon || !parent)
+		return;
+
+	if (!open_cached_dir_by_dentry(tcon, parent, &parent_cfid)) {
+		mutex_lock(&parent_cfid->dirents.de_mutex);
+		parent_cfid->dirents.is_valid = false;
+		parent_cfid->dirents.is_failed = true;
+		mutex_unlock(&parent_cfid->dirents.de_mutex);
+		close_cached_dir(parent_cfid);
+	}
+}
+
 /*
  * Set parameters for the netfs library
  */
@@ -119,7 +136,7 @@ cifs_revalidate_cache(struct inode *inode, struct cifs_fattr *fattr)
 	fattr->cf_mtime = timestamp_truncate(fattr->cf_mtime, inode);
 	mtime = inode_get_mtime(inode);
 	if (timespec64_equal(&mtime, &fattr->cf_mtime) &&
-	    cifs_i->netfs.remote_i_size == fattr->cf_eof) {
+	    netfs_read_remote_i_size(inode) == fattr->cf_eof) {
 		cifs_dbg(FYI, "%s: inode %llu is unchanged\n",
 			 __func__, cifs_i->uniqueid);
 		return;
@@ -173,12 +190,12 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr,
 		CIFS_I(inode)->time = 0; /* force reval */
 		return -ESTALE;
 	}
-	if (inode_state_read_once(inode) & I_NEW)
-		CIFS_I(inode)->netfs.zero_point = fattr->cf_eof;
-
 	cifs_revalidate_cache(inode, fattr);
 
 	spin_lock(&inode->i_lock);
+	if (inode_state_read_once(inode) & I_NEW)
+		netfs_write_zero_point(inode, fattr->cf_eof);
+
 	fattr->cf_mtime = timestamp_truncate(fattr->cf_mtime, inode);
 	fattr->cf_atime = timestamp_truncate(fattr->cf_atime, inode);
 	fattr->cf_ctime = timestamp_truncate(fattr->cf_ctime, inode);
@@ -212,7 +229,7 @@ cifs_fattr_to_inode(struct inode *inode, struct cifs_fattr *fattr,
 	else
 		clear_bit(CIFS_INO_DELETE_PENDING, &cifs_i->flags);
 
-	cifs_i->netfs.remote_i_size = fattr->cf_eof;
+	netfs_write_remote_i_size(inode, fattr->cf_eof);
 	/*
 	 * Can't safely change the file size here if the client is writing to
 	 * it due to potential races.
@@ -2067,6 +2084,9 @@ static int __cifs_unlink(struct inode *dir, struct dentry *dentry, bool sillyren
 		cifs_set_file_info(inode, attrs, xid, full_path, origattr);
 
 out_reval:
+	if (!rc && dentry->d_parent)
+		cifs_invalidate_cached_dir(tcon, dentry->d_parent);
+
 	if (inode) {
 		cifs_inode = CIFS_I(inode);
 		cifs_inode->time = 0;	/* will force revalidate to get info
@@ -2378,7 +2398,6 @@ int cifs_rmdir(struct inode *inode, struct dentry *direntry)
 	}
 
 	rc = server->ops->rmdir(xid, tcon, full_path, cifs_sb);
-	cifs_put_tlink(tlink);
 
 	cifsInode = CIFS_I(d_inode(direntry));
 
@@ -2388,6 +2407,8 @@ int cifs_rmdir(struct inode *inode, struct dentry *direntry)
 		i_size_write(d_inode(direntry), 0);
 		clear_nlink(d_inode(direntry));
 		spin_unlock(&d_inode(direntry)->i_lock);
+		if (direntry->d_parent)
+			cifs_invalidate_cached_dir(tcon, direntry->d_parent);
 	}
 
 	/* force revalidate to go get info when needed */
@@ -2402,6 +2423,7 @@ int cifs_rmdir(struct inode *inode, struct dentry *direntry)
 
 	inode_set_ctime_current(d_inode(direntry));
 	inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode));
+	cifs_put_tlink(tlink);
 
 rmdir_exit:
 	free_dentry_path(page);
@@ -2668,6 +2690,12 @@ cifs_rename2(struct mnt_idmap *idmap, struct inode *source_dir,
 	}
 
 	/* force revalidate to go get info when needed */
+	if (!rc) {
+		cifs_invalidate_cached_dir(tcon, source_dentry->d_parent);
+		if (target_dentry->d_parent != source_dentry->d_parent)
+			cifs_invalidate_cached_dir(tcon, target_dentry->d_parent);
+	}
+
 	CIFS_I(source_dir)->time = CIFS_I(target_dir)->time = 0;
 
 cifs_rename_exit:
@@ -2772,7 +2800,9 @@ cifs_revalidate_mapping(struct inode *inode)
 		if (cifs_sb_flags(cifs_sb) & CIFS_MOUNT_RW_CACHE)
 			goto skip_invalidate;
 
-		cifs_inode->netfs.zero_point = cifs_inode->netfs.remote_i_size;
+		spin_lock(&inode->i_lock);
+		netfs_write_zero_point(inode, netfs_inode(inode)->_remote_i_size);
+		spin_unlock(&inode->i_lock);
 		rc = filemap_invalidate_inode(inode, true, 0, LLONG_MAX);
 		if (rc) {
 			cifs_dbg(VFS, "%s: invalidate inode %p failed with rc %d\n",
@@ -2843,7 +2873,7 @@ int cifs_revalidate_dentry_attr(struct dentry *dentry)
 	}
 
 	cifs_dbg(FYI, "Update attributes: %s inode 0x%p count %d dentry: 0x%p d_time %ld jiffies %ld\n",
-		 full_path, inode, icount_read(inode),
+		 full_path, inode, icount_read_once(inode),
 		 dentry, cifs_get_time(dentry), jiffies);
 
 again:
diff --git a/fs/smb/client/namespace.c b/fs/smb/client/namespace.c
index 52a520349cb76..52a51b032fae3 100644
--- a/fs/smb/client/namespace.c
+++ b/fs/smb/client/namespace.c
@@ -294,4 +294,5 @@ struct vfsmount *cifs_d_automount(struct path *path)
 }
 
 const struct inode_operations cifs_namespace_inode_operations = {
+	.fileattr_get	= cifs_fileattr_get,
 };
diff --git a/fs/smb/client/netlink.c b/fs/smb/client/netlink.c
index 147d9409252cd..0dd10913c37a0 100644
--- a/fs/smb/client/netlink.c
+++ b/fs/smb/client/netlink.c
@@ -33,13 +33,17 @@ static const struct nla_policy cifs_genl_policy[CIFS_GENL_ATTR_MAX + 1] = {
 static const struct genl_ops cifs_genl_ops[] = {
 	{
 		.cmd = CIFS_GENL_CMD_SWN_NOTIFY,
+		.flags = GENL_ADMIN_PERM,
 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
 		.doit = cifs_swn_notify,
 	},
 };
 
 static const struct genl_multicast_group cifs_genl_mcgrps[] = {
-	[CIFS_GENL_MCGRP_SWN] = { .name = CIFS_GENL_MCGRP_SWN_NAME },
+	[CIFS_GENL_MCGRP_SWN] = {
+		.name = CIFS_GENL_MCGRP_SWN_NAME,
+		.flags = GENL_MCAST_CAP_NET_ADMIN,
+	},
 };
 
 struct genl_family cifs_genl_family = {
diff --git a/fs/smb/client/readdir.c b/fs/smb/client/readdir.c
index be22bbc4a65a0..e860fa08b5e30 100644
--- a/fs/smb/client/readdir.c
+++ b/fs/smb/client/readdir.c
@@ -143,7 +143,8 @@ cifs_prime_dcache(struct dentry *parent, struct qstr *name,
 						fattr->cf_rdev = inode->i_rdev;
 						fattr->cf_uid = inode->i_uid;
 						fattr->cf_gid = inode->i_gid;
-						fattr->cf_eof = CIFS_I(inode)->netfs.remote_i_size;
+						fattr->cf_eof =
+							netfs_read_remote_i_size(inode);
 						fattr->cf_symlink_target = NULL;
 					} else {
 						CIFS_I(inode)->time = 0;
diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c
index 3738204984f5a..d4875f9532b4d 100644
--- a/fs/smb/client/smb2ops.c
+++ b/fs/smb/client/smb2ops.c
@@ -3402,8 +3402,7 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon,
 	struct inode *inode = file_inode(file);
 	struct cifsInodeInfo *cifsi = CIFS_I(inode);
 	struct cifsFileInfo *cfile = file->private_data;
-	struct netfs_inode *ictx = netfs_inode(inode);
-	unsigned long long i_size, new_size, remote_size;
+	unsigned long long i_size, new_size, remote_i_size, zero_point;
 	long rc;
 	unsigned int xid;
 
@@ -3414,9 +3413,8 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon,
 
 	filemap_invalidate_lock(inode->i_mapping);
 
-	i_size = i_size_read(inode);
-	remote_size = ictx->remote_i_size;
-	if (offset + len >= remote_size && offset < i_size) {
+	netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
+	if (offset + len >= remote_i_size && offset < i_size) {
 		unsigned long long top = umin(offset + len, i_size);
 
 		rc = filemap_write_and_wait_range(inode->i_mapping, offset, top - 1);
@@ -3449,9 +3447,11 @@ static long smb3_zero_range(struct file *file, struct cifs_tcon *tcon,
 				  cfile->fid.volatile_fid, cfile->pid, new_size);
 		if (rc >= 0) {
 			truncate_setsize(inode, new_size);
+			spin_lock(&inode->i_lock);
 			netfs_resize_file(&cifsi->netfs, new_size, true);
-			if (offset < cifsi->netfs.zero_point)
-				cifsi->netfs.zero_point = offset;
+			if (offset < cifsi->netfs._zero_point)
+				netfs_write_zero_point(inode, offset);
+			spin_unlock(&inode->i_lock);
 			fscache_resize_cookie(cifs_inode_cookie(inode), new_size);
 		}
 	}
@@ -3474,7 +3474,7 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon,
 	struct inode *inode = file_inode(file);
 	struct cifsFileInfo *cfile = file->private_data;
 	struct file_zero_data_information fsctl_buf;
-	unsigned long long end = offset + len, i_size, remote_i_size;
+	unsigned long long end = offset + len, i_size, remote_i_size, zero_point;
 	long rc;
 	unsigned int xid;
 	__u8 set_sparse = 1;
@@ -3516,14 +3516,17 @@ static long smb3_punch_hole(struct file *file, struct cifs_tcon *tcon,
 	 * that we locally hole-punch the tail of the dirty data, the proposed
 	 * EOF update will end up in the wrong place.
 	 */
-	i_size = i_size_read(inode);
-	remote_i_size = netfs_inode(inode)->remote_i_size;
+	netfs_read_sizes(inode, &i_size, &remote_i_size, &zero_point);
+
 	if (end > remote_i_size && i_size > remote_i_size) {
 		unsigned long long extend_to = umin(end, i_size);
 		rc = SMB2_set_eof(xid, tcon, cfile->fid.persistent_fid,
 				  cfile->fid.volatile_fid, cfile->pid, extend_to);
-		if (rc >= 0)
-			netfs_inode(inode)->remote_i_size = extend_to;
+		if (rc >= 0) {
+			spin_lock(&inode->i_lock);
+			netfs_write_remote_i_size(inode, extend_to);
+			spin_unlock(&inode->i_lock);
+		}
 	}
 
 unlock:
@@ -3787,7 +3790,6 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon,
 	struct inode *inode = file_inode(file);
 	struct cifsInodeInfo *cifsi = CIFS_I(inode);
 	struct cifsFileInfo *cfile = file->private_data;
-	struct netfs_inode *ictx = &cifsi->netfs;
 	loff_t old_eof, new_eof;
 
 	xid = get_xid();
@@ -3805,7 +3807,9 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon,
 		goto out_2;
 
 	truncate_pagecache_range(inode, off, old_eof);
-	ictx->zero_point = old_eof;
+	spin_lock(&inode->i_lock);
+	netfs_write_zero_point(inode, old_eof);
+	spin_unlock(&inode->i_lock);
 	netfs_wait_for_outstanding_io(inode);
 
 	rc = smb2_copychunk_range(xid, cfile, cfile, off + len,
@@ -3822,8 +3826,10 @@ static long smb3_collapse_range(struct file *file, struct cifs_tcon *tcon,
 	rc = 0;
 
 	truncate_setsize(inode, new_eof);
+	spin_lock(&inode->i_lock);
 	netfs_resize_file(&cifsi->netfs, new_eof, true);
-	ictx->zero_point = new_eof;
+	netfs_write_zero_point(inode, new_eof);
+	spin_unlock(&inode->i_lock);
 	fscache_resize_cookie(cifs_inode_cookie(inode), new_eof);
 out_2:
 	filemap_invalidate_unlock(inode->i_mapping);
@@ -3866,13 +3872,17 @@ static long smb3_insert_range(struct file *file, struct cifs_tcon *tcon,
 		goto out_2;
 
 	truncate_setsize(inode, new_eof);
+	spin_lock(&inode->i_lock);
 	netfs_resize_file(&cifsi->netfs, i_size_read(inode), true);
+	spin_unlock(&inode->i_lock);
 	fscache_resize_cookie(cifs_inode_cookie(inode), i_size_read(inode));
 
 	rc = smb2_copychunk_range(xid, cfile, cfile, off, count, off + len);
 	if (rc < 0)
 		goto out_2;
-	cifsi->netfs.zero_point = new_eof;
+	spin_lock(&inode->i_lock);
+	netfs_write_zero_point(inode, new_eof);
+	spin_unlock(&inode->i_lock);
 
 	rc = smb3_zero_data(file, tcon, off, len, xid);
 	if (rc < 0)
@@ -4696,9 +4706,15 @@ cifs_copy_folioq_to_iter(struct folio_queue *folioq, size_t data_size,
 {
 	for (; folioq; folioq = folioq->next) {
 		for (int s = 0; s < folioq_count(folioq); s++) {
-			struct folio *folio = folioq_folio(folioq, s);
-			size_t fsize = folio_size(folio);
-			size_t n, len = umin(fsize - skip, data_size);
+			struct folio *folio;
+			size_t fsize, n, len;
+
+			if (data_size == 0)
+				return 0;
+
+			folio = folioq_folio(folioq, s);
+			fsize = folio_size(folio);
+			len = umin(fsize - skip, data_size);
 
 			n = copy_folio_to_iter(folio, skip, len, iter);
 			if (n != len) {
@@ -4711,6 +4727,12 @@ cifs_copy_folioq_to_iter(struct folio_queue *folioq, size_t data_size,
 		}
 	}
 
+	if (data_size != 0) {
+		cifs_dbg(VFS, "%s: short copy, %zu bytes missing\n",
+			 __func__, data_size);
+		return smb_EIO2(smb_eio_trace_rx_copy_to_iter, 0, data_size);
+	}
+
 	return 0;
 }
 
@@ -4826,7 +4848,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
 		}
 
 		/* Copy the data to the output I/O iterator. */
-		rdata->result = cifs_copy_folioq_to_iter(buffer, buffer_len,
+		rdata->result = cifs_copy_folioq_to_iter(buffer, data_len,
 							 cur_off, &rdata->subreq.io_iter);
 		if (rdata->result != 0) {
 			if (is_offloaded)
@@ -4835,7 +4857,7 @@ handle_read_data(struct TCP_Server_Info *server, struct mid_q_entry *mid,
 				dequeue_mid(server, mid, rdata->result);
 			return 0;
 		}
-		rdata->got_bytes = buffer_len;
+		rdata->got_bytes = data_len;
 
 	} else if (!check_add_overflow(data_offset, data_len, &end_off) &&
 		   buf_len >= end_off) {
diff --git a/fs/smb/client/smb2pdu.c b/fs/smb/client/smb2pdu.c
index 995fcdd306815..3bd300347f16e 100644
--- a/fs/smb/client/smb2pdu.c
+++ b/fs/smb/client/smb2pdu.c
@@ -4608,6 +4608,7 @@ smb2_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 	struct netfs_inode *ictx = netfs_inode(rdata->rreq->inode);
 	struct cifs_tcon *tcon = tlink_tcon(rdata->req->cfile->tlink);
 	struct smb2_hdr *shdr = (struct smb2_hdr *)rdata->iov[0].iov_base;
+	struct inode *inode = &ictx->inode;
 	struct cifs_credits credits = {
 		.value = 0,
 		.instance = 0,
@@ -4721,7 +4722,7 @@ smb2_readv_callback(struct TCP_Server_Info *server, struct mid_q_entry *mid)
 	} else {
 		size_t trans = rdata->subreq.transferred + rdata->got_bytes;
 		if (trans < rdata->subreq.len &&
-		    rdata->subreq.start + trans >= ictx->remote_i_size) {
+		    rdata->subreq.start + trans >= netfs_read_remote_i_size(inode)) {
 			__set_bit(NETFS_SREQ_HIT_EOF, &rdata->subreq.flags);
 			rdata->result = 0;
 		}
diff --git a/fs/smb/client/smb2transport.c b/fs/smb/client/smb2transport.c
index e8eeff9e50d67..1143ee52470a7 100644
--- a/fs/smb/client/smb2transport.c
+++ b/fs/smb/client/smb2transport.c
@@ -169,7 +169,9 @@ smb2_find_smb_sess_tcon_unlocked(struct cifs_ses *ses, __u32  tid)
 	list_for_each_entry(tcon, &ses->tcon_list, tcon_list) {
 		if (tcon->tid != tid)
 			continue;
+		spin_lock(&tcon->tc_lock);
 		++tcon->tc_count;
+		spin_unlock(&tcon->tc_lock);
 		trace_smb3_tcon_ref(tcon->debug_id, tcon->tc_count,
 				    netfs_trace_tcon_ref_get_find_sess_tcon);
 		return tcon;
diff --git a/fs/smb/client/trace.h b/fs/smb/client/trace.h
index b99ec5a417fad..5b21ad3c15fb5 100644
--- a/fs/smb/client/trace.h
+++ b/fs/smb/client/trace.h
@@ -181,6 +181,7 @@
 	EM(netfs_trace_tcon_ref_get_find,		"GET Find  ") \
 	EM(netfs_trace_tcon_ref_get_find_sess_tcon,	"GET FndSes") \
 	EM(netfs_trace_tcon_ref_get_reconnect_server,	"GET Reconn") \
+	EM(netfs_trace_tcon_ref_get_swn_notify,		"GET SwnNot") \
 	EM(netfs_trace_tcon_ref_new,			"NEW       ") \
 	EM(netfs_trace_tcon_ref_new_ipc,		"NEW Ipc   ") \
 	EM(netfs_trace_tcon_ref_new_reconnect_server,	"NEW Reconn") \
@@ -192,6 +193,7 @@
 	EM(netfs_trace_tcon_ref_put_mnt_ctx,		"PUT MntCtx") \
 	EM(netfs_trace_tcon_ref_put_dfs_refer,		"PUT DfsRfr") \
 	EM(netfs_trace_tcon_ref_put_reconnect_server,	"PUT Reconn") \
+	EM(netfs_trace_tcon_ref_put_swn_notify,		"PUT SwnNot") \
 	EM(netfs_trace_tcon_ref_put_tlink,		"PUT Tlink ") \
 	EM(netfs_trace_tcon_ref_see_cancelled_close,	"SEE Cn-Cls") \
 	EM(netfs_trace_tcon_ref_see_fscache_collision,	"SEE FV-CO!") \
diff --git a/fs/smb/server/oplock.c b/fs/smb/server/oplock.c
index 8feca02ddbf25..0f5c18520eff0 100644
--- a/fs/smb/server/oplock.c
+++ b/fs/smb/server/oplock.c
@@ -481,8 +481,12 @@ static inline int compare_guid_key(struct oplock_info *opinfo,
 				   const char *guid1, const char *key1)
 {
 	const char *guid2, *key2;
+	struct ksmbd_conn *conn;
 
-	guid2 = opinfo->conn->ClientGUID;
+	conn = READ_ONCE(opinfo->conn);
+	if (!conn)
+		return 0;
+	guid2 = conn->ClientGUID;
 	key2 = opinfo->o_lease->lease_key;
 	if (!memcmp(guid1, guid2, SMB2_CLIENT_GUID_SIZE) &&
 	    !memcmp(key1, key2, SMB2_LEASE_KEY_SIZE))
diff --git a/fs/smb/server/smb2pdu.c b/fs/smb/server/smb2pdu.c
index 62d4399a993d8..406ede2dd9e73 100644
--- a/fs/smb/server/smb2pdu.c
+++ b/fs/smb/server/smb2pdu.c
@@ -14,6 +14,7 @@
 #include <linux/falloc.h>
 #include <linux/mount.h>
 #include <linux/filelock.h>
+#include <linux/fileattr.h>
 
 #include "glob.h"
 #include "smbfsctl.h"
@@ -5550,16 +5551,33 @@ static int smb2_get_info_filesystem(struct ksmbd_work *work,
 	case FS_ATTRIBUTE_INFORMATION:
 	{
 		FILE_SYSTEM_ATTRIBUTE_INFO *info;
+		struct file_kattr fa = {};
 		size_t sz;
+		u32 attrs;
+		int err;
 
 		info = (FILE_SYSTEM_ATTRIBUTE_INFO *)rsp->Buffer;
-		info->Attributes = cpu_to_le32(FILE_SUPPORTS_OBJECT_IDS |
-					       FILE_PERSISTENT_ACLS |
-					       FILE_UNICODE_ON_DISK |
-					       FILE_CASE_PRESERVED_NAMES |
-					       FILE_CASE_SENSITIVE_SEARCH |
-					       FILE_SUPPORTS_BLOCK_REFCOUNTING);
+		attrs = FILE_SUPPORTS_OBJECT_IDS |
+			FILE_PERSISTENT_ACLS |
+			FILE_UNICODE_ON_DISK |
+			FILE_SUPPORTS_BLOCK_REFCOUNTING;
+
+		err = vfs_fileattr_get(path.dentry, &fa);
+		/*
+		 * -EINVAL, -EOPNOTSUPP: ntfs-3g and other FUSE
+		 * filesystems that lack FS_IOC_FSGETXATTR support.
+		 */
+		if (err && err != -ENOIOCTLCMD && err != -ENOTTY &&
+		    err != -EINVAL && err != -EOPNOTSUPP) {
+			path_put(&path);
+			return err;
+		}
+		if (!(fa.fsx_xflags & FS_XFLAG_CASEFOLD))
+			attrs |= FILE_CASE_SENSITIVE_SEARCH;
+		if (!(fa.fsx_xflags & FS_XFLAG_CASENONPRESERVING))
+			attrs |= FILE_CASE_PRESERVED_NAMES;
 
+		info->Attributes = cpu_to_le32(attrs);
 		info->Attributes |= cpu_to_le32(server_conf.share_fake_fscaps);
 
 		if (test_share_config_flag(work->tcon->share_conf,
diff --git a/fs/smb/server/smbacl.c b/fs/smb/server/smbacl.c
index c1d1f34581d69..9161e9d7ed247 100644
--- a/fs/smb/server/smbacl.c
+++ b/fs/smb/server/smbacl.c
@@ -643,8 +643,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 		ntace = (struct smb_ace *)((char *)pndace + *size);
 		ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, flags,
 				pace->e_perm, 0777);
-		if (check_add_overflow(*size, ace_sz, size))
+		if (check_add_overflow(*size, ace_sz, size)) {
+			kfree(sid);
 			break;
+		}
 		(*num_aces)++;
 		if (pace->e_tag == ACL_USER)
 			ntace->access_req |=
@@ -655,8 +657,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 			ntace = (struct smb_ace *)((char *)pndace + *size);
 			ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED,
 					0x03, pace->e_perm, 0777);
-			if (check_add_overflow(*size, ace_sz, size))
+			if (check_add_overflow(*size, ace_sz, size)) {
+				kfree(sid);
 				break;
+			}
 			(*num_aces)++;
 			if (pace->e_tag == ACL_USER)
 				ntace->access_req |=
@@ -698,8 +702,10 @@ static void set_posix_acl_entries_dacl(struct mnt_idmap *idmap,
 		ntace = (struct smb_ace *)((char *)pndace + *size);
 		ace_sz = fill_ace_for_sid(ntace, sid, ACCESS_ALLOWED, 0x0b,
 				pace->e_perm, 0777);
-		if (check_add_overflow(*size, ace_sz, size))
+		if (check_add_overflow(*size, ace_sz, size)) {
+			kfree(sid);
 			break;
+		}
 		(*num_aces)++;
 		if (pace->e_tag == ACL_USER)
 			ntace->access_req |=
diff --git a/fs/smb/server/vfs_cache.c b/fs/smb/server/vfs_cache.c
index 354c4d8a1cfbd..913164c958b15 100644
--- a/fs/smb/server/vfs_cache.c
+++ b/fs/smb/server/vfs_cache.c
@@ -81,7 +81,7 @@ static int proc_show_files(struct seq_file *m, void *v)
 	read_lock(&global_ft.lock);
 	idr_for_each_entry(global_ft.idr, fp, id) {
 		seq_printf(m, "%#-10x %#-10llx %#-10llx %#-10x",
-			   fp->tcon->id,
+			   fp->tcon ? fp->tcon->id : 0,
 			   fp->persistent_id,
 			   fp->volatile_id,
 			   atomic_read(&fp->refcount));
diff --git a/fs/smb/smbdirect/debug.c b/fs/smb/smbdirect/debug.c
index 05ba7c8d165ee..3445843445bff 100644
--- a/fs/smb/smbdirect/debug.c
+++ b/fs/smb/smbdirect/debug.c
@@ -40,7 +40,7 @@ void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc,
 
 	seq_puts(m, "\n");
 	seq_printf(m, "Conn keep_alive_interval: %u ",
-		   sp->keepalive_interval_msec * 1000);
+		   sp->keepalive_interval_msec / 1000);
 	seq_printf(m, "max_readwrite_size: %u rdma_readwrite_threshold: %u",
 		   sp->max_read_write_size,
 		   rdma_readwrite_threshold);
diff --git a/fs/super.c b/fs/super.c
index 378e81efe643b..5d46a0d5b6169 100644
--- a/fs/super.c
+++ b/fs/super.c
@@ -882,7 +882,6 @@ void drop_super_exclusive(struct super_block *sb)
 	super_unlock_excl(sb);
 	put_super(sb);
 }
-EXPORT_SYMBOL(drop_super_exclusive);
 
 enum super_iter_flags_t {
 	SUPER_ITER_EXCL		= (1U << 0),
diff --git a/fs/sync.c b/fs/sync.c
index 942a60cfedfbf..4a84dd837b863 100644
--- a/fs/sync.c
+++ b/fs/sync.c
@@ -266,8 +266,7 @@ int sync_file_range(struct file *file, loff_t offset, loff_t nbytes,
 
 	i_mode = file_inode(file)->i_mode;
 	ret = -ESPIPE;
-	if (!S_ISREG(i_mode) && !S_ISBLK(i_mode) && !S_ISDIR(i_mode) &&
-			!S_ISLNK(i_mode))
+	if (!S_ISREG(i_mode) && !S_ISBLK(i_mode) && !S_ISDIR(i_mode))
 		goto out;
 
 	mapping = file->f_mapping;
diff --git a/fs/sysfs/file.c b/fs/sysfs/file.c
index 5709cede1d756..cd5bb0f9fee6d 100644
--- a/fs/sysfs/file.c
+++ b/fs/sysfs/file.c
@@ -70,9 +70,8 @@ static int sysfs_kf_seq_show(struct seq_file *sf, void *v)
 	 * The code works fine with PAGE_SIZE return but it's likely to
 	 * indicate truncated result or overflow in normal use cases.
 	 */
-	if (count >= (ssize_t)PAGE_SIZE) {
-		printk("fill_read_buffer: %pS returned bad count\n",
-				ops->show);
+	if (count >= PAGE_SIZE) {
+		WARN(1, "OOB write or bad count %zd at %pS\n", count, ops->show);
 		/* Try to struggle along */
 		count = PAGE_SIZE - 1;
 	}
@@ -120,6 +119,10 @@ static ssize_t sysfs_kf_read(struct kernfs_open_file *of, char *buf,
 	len = ops->show(kobj, of->kn->priv, buf);
 	if (len < 0)
 		return len;
+	if (len >= (ssize_t)PAGE_SIZE) {
+		printk("fill_read_buffer: %pS returned bad count\n", ops->show);
+		len = PAGE_SIZE - 1;
+	}
 	if (pos) {
 		if (len <= pos)
 			return 0;
diff --git a/fs/sysfs/group.c b/fs/sysfs/group.c
index 182e54e575ee9..4e1e4f18a1669 100644
--- a/fs/sysfs/group.c
+++ b/fs/sysfs/group.c
@@ -188,7 +188,7 @@ static int internal_create_group(struct kobject *kobj, int update,
 	kernfs_get(kn);
 	error = create_files(kn, kobj, uid, gid, grp, update);
 	if (error) {
-		if (grp->name)
+		if (grp->name && !update)
 			kernfs_remove(kn);
 	}
 	kernfs_put(kn);
diff --git a/fs/sysfs/mount.c b/fs/sysfs/mount.c
index b199e8ff79b1f..88c10823fcafa 100644
--- a/fs/sysfs/mount.c
+++ b/fs/sysfs/mount.c
@@ -23,20 +23,6 @@
 static struct kernfs_root *sysfs_root;
 struct kernfs_node *sysfs_root_kn;
 
-static int sysfs_get_tree(struct fs_context *fc)
-{
-	struct kernfs_fs_context *kfc = fc->fs_private;
-	int ret;
-
-	ret = kernfs_get_tree(fc);
-	if (ret)
-		return ret;
-
-	if (kfc->new_sb_created)
-		fc->root->d_sb->s_iflags |= SB_I_USERNS_VISIBLE;
-	return 0;
-}
-
 static void sysfs_fs_context_free(struct fs_context *fc)
 {
 	struct kernfs_fs_context *kfc = fc->fs_private;
@@ -49,7 +35,7 @@ static void sysfs_fs_context_free(struct fs_context *fc)
 
 static const struct fs_context_operations sysfs_fs_context_ops = {
 	.free		= sysfs_fs_context_free,
-	.get_tree	= sysfs_get_tree,
+	.get_tree	= kernfs_get_tree,
 };
 
 static int sysfs_init_fs_context(struct fs_context *fc)
@@ -93,7 +79,7 @@ static struct file_system_type sysfs_fs_type = {
 	.name			= "sysfs",
 	.init_fs_context	= sysfs_init_fs_context,
 	.kill_sb		= sysfs_kill_sb,
-	.fs_flags		= FS_USERNS_MOUNT,
+	.fs_flags		= FS_USERNS_MOUNT | FS_USERNS_MOUNT_RESTRICTED,
 };
 
 int __init sysfs_init(void)
diff --git a/fs/timerfd.c b/fs/timerfd.c
index 73104f36bcae0..fe845af0b74e6 100644
--- a/fs/timerfd.c
+++ b/fs/timerfd.c
@@ -55,6 +55,15 @@ static inline bool isalarm(struct timerfd_ctx *ctx)
 		ctx->clockid == CLOCK_BOOTTIME_ALARM;
 }
 
+static void __timerfd_triggered(struct timerfd_ctx *ctx)
+{
+	lockdep_assert_held(&ctx->wqh.lock);
+
+	ctx->expired = 1;
+	ctx->ticks++;
+	wake_up_locked_poll(&ctx->wqh, EPOLLIN);
+}
+
 /*
  * This gets called when the timer event triggers. We set the "expired"
  * flag, but we do not re-arm the timer (in case it's necessary,
@@ -62,13 +71,8 @@ static inline bool isalarm(struct timerfd_ctx *ctx)
  */
 static void timerfd_triggered(struct timerfd_ctx *ctx)
 {
-	unsigned long flags;
-
-	spin_lock_irqsave(&ctx->wqh.lock, flags);
-	ctx->expired = 1;
-	ctx->ticks++;
-	wake_up_locked_poll(&ctx->wqh, EPOLLIN);
-	spin_unlock_irqrestore(&ctx->wqh.lock, flags);
+	guard(spinlock_irqsave)(&ctx->wqh.lock);
+	__timerfd_triggered(ctx);
 }
 
 static enum hrtimer_restart timerfd_tmrproc(struct hrtimer *htmr)
@@ -184,15 +188,54 @@ static ktime_t timerfd_get_remaining(struct timerfd_ctx *ctx)
 	return remaining < 0 ? 0: remaining;
 }
 
+static void timerfd_alarm_start(struct timerfd_ctx *ctx, ktime_t exp, bool relative)
+{
+	/* Start the timer. If it's expired already, handle the callback. */
+	if (!alarm_start_timer(&ctx->t.alarm, exp, relative))
+		__timerfd_triggered(ctx);
+}
+
+static u64 timerfd_alarm_restart(struct timerfd_ctx *ctx)
+{
+	/* -1 to account for ctx->ticks++ in __timerfd_triggered() */
+	u64 ticks = alarm_forward_now(&ctx->t.alarm, ctx->tintv) - 1;
+
+	timerfd_alarm_start(ctx, alarm_get_expires(&ctx->t.alarm), false);
+	return ticks;
+}
+
+static void timerfd_hrtimer_start(struct timerfd_ctx *ctx, ktime_t exp,
+				  const enum hrtimer_mode mode)
+{
+	/* Start the timer. If it's expired already, handle the callback. */
+	if (!hrtimer_start_range_ns_user(&ctx->t.tmr, exp, 0, mode))
+		__timerfd_triggered(ctx);
+}
+
+static u64 timerfd_hrtimer_restart(struct timerfd_ctx *ctx)
+{
+	/* -1 to account for ctx->ticks++ in __timerfd_triggered() */
+	u64 ticks = hrtimer_forward_now(&ctx->t.tmr, ctx->tintv) - 1;
+
+	timerfd_hrtimer_start(ctx, hrtimer_get_expires(&ctx->t.tmr), HRTIMER_MODE_ABS);
+	return ticks;
+}
+
+static u64 timerfd_restart(struct timerfd_ctx *ctx)
+{
+	if (isalarm(ctx))
+		return timerfd_alarm_restart(ctx);
+	return timerfd_hrtimer_restart(ctx);
+}
+
 static int timerfd_setup(struct timerfd_ctx *ctx, int flags,
 			 const struct itimerspec64 *ktmr)
 {
+	int clockid = ctx->clockid;
 	enum hrtimer_mode htmode;
 	ktime_t texp;
-	int clockid = ctx->clockid;
 
-	htmode = (flags & TFD_TIMER_ABSTIME) ?
-		HRTIMER_MODE_ABS: HRTIMER_MODE_REL;
+	htmode = (flags & TFD_TIMER_ABSTIME) ? HRTIMER_MODE_ABS: HRTIMER_MODE_REL;
 
 	texp = timespec64_to_ktime(ktmr->it_value);
 	ctx->expired = 0;
@@ -206,20 +249,15 @@ static int timerfd_setup(struct timerfd_ctx *ctx, int flags,
 			   timerfd_alarmproc);
 	} else {
 		hrtimer_setup(&ctx->t.tmr, timerfd_tmrproc, clockid, htmode);
-		hrtimer_set_expires(&ctx->t.tmr, texp);
 	}
 
 	if (texp != 0) {
 		if (flags & TFD_TIMER_ABSTIME)
 			texp = timens_ktime_to_host(clockid, texp);
-		if (isalarm(ctx)) {
-			if (flags & TFD_TIMER_ABSTIME)
-				alarm_start(&ctx->t.alarm, texp);
-			else
-				alarm_start_relative(&ctx->t.alarm, texp);
-		} else {
-			hrtimer_start(&ctx->t.tmr, texp, htmode);
-		}
+		if (isalarm(ctx))
+			timerfd_alarm_start(ctx, texp, !(flags & TFD_TIMER_ABSTIME));
+		else
+			timerfd_hrtimer_start(ctx, texp, htmode);
 
 		if (timerfd_canceled(ctx))
 			return -ECANCELED;
@@ -287,27 +325,19 @@ static ssize_t timerfd_read_iter(struct kiocb *iocb, struct iov_iter *to)
 	}
 
 	if (ctx->ticks) {
-		ticks = ctx->ticks;
+		unsigned int expired = ctx->expired;
 
-		if (ctx->expired && ctx->tintv) {
-			/*
-			 * If tintv != 0, this is a periodic timer that
-			 * needs to be re-armed. We avoid doing it in the timer
-			 * callback to avoid DoS attacks specifying a very
-			 * short timer period.
-			 */
-			if (isalarm(ctx)) {
-				ticks += alarm_forward_now(
-					&ctx->t.alarm, ctx->tintv) - 1;
-				alarm_restart(&ctx->t.alarm);
-			} else {
-				ticks += hrtimer_forward_now(&ctx->t.tmr,
-							     ctx->tintv) - 1;
-				hrtimer_restart(&ctx->t.tmr);
-			}
-		}
+		ticks = ctx->ticks;
 		ctx->expired = 0;
 		ctx->ticks = 0;
+
+		/*
+		 * If tintv != 0, this is a periodic timer that needs to be
+		 * re-armed. We avoid doing it in the timer callback to avoid
+		 * DoS attacks specifying a very short timer period.
+		 */
+		if (expired && ctx->tintv)
+			ticks += timerfd_restart(ctx);
 	}
 	spin_unlock_irq(&ctx->wqh.lock);
 	if (ticks) {
@@ -526,18 +556,7 @@ static int do_timerfd_gettime(int ufd, struct itimerspec64 *t)
 	spin_lock_irq(&ctx->wqh.lock);
 	if (ctx->expired && ctx->tintv) {
 		ctx->expired = 0;
-
-		if (isalarm(ctx)) {
-			ctx->ticks +=
-				alarm_forward_now(
-					&ctx->t.alarm, ctx->tintv) - 1;
-			alarm_restart(&ctx->t.alarm);
-		} else {
-			ctx->ticks +=
-				hrtimer_forward_now(&ctx->t.tmr, ctx->tintv)
-				- 1;
-			hrtimer_restart(&ctx->t.tmr);
-		}
+		ctx->ticks += timerfd_restart(ctx);
 	}
 	t->it_value = ktime_to_timespec64(timerfd_get_remaining(ctx));
 	t->it_interval = ktime_to_timespec64(ctx->tintv);
diff --git a/fs/ubifs/super.c b/fs/ubifs/super.c
index 9a77d8b64ffa7..38972786817e7 100644
--- a/fs/ubifs/super.c
+++ b/fs/ubifs/super.c
@@ -358,7 +358,7 @@ static void ubifs_evict_inode(struct inode *inode)
 		goto out;
 
 	dbg_gen("inode %llu, mode %#x", inode->i_ino, (int)inode->i_mode);
-	ubifs_assert(c, !icount_read(inode));
+	ubifs_assert(c, !icount_read_once(inode));
 
 	truncate_inode_pages_final(&inode->i_data);
 
diff --git a/fs/udf/balloc.c b/fs/udf/balloc.c
index 807c493ed0cd5..cc6dc6e1d84de 100644
--- a/fs/udf/balloc.c
+++ b/fs/udf/balloc.c
@@ -82,8 +82,9 @@ static int load_block_bitmap(struct super_block *sb,
 	int nr_groups = bitmap->s_nr_groups;
 
 	if (block_group >= nr_groups) {
-		udf_debug("block_group (%u) > nr_groups (%d)\n",
+		udf_debug("block_group (%u) >= nr_groups (%d)\n",
 			  block_group, nr_groups);
+		return -EFSCORRUPTED;
 	}
 
 	if (bitmap->s_block_bitmap[block_group]) {
@@ -662,7 +663,7 @@ void udf_free_blocks(struct super_block *sb, struct inode *inode,
 
 	if (check_add_overflow(bloc->logicalBlockNum, offset, &blk) ||
 	    check_add_overflow(blk, count, &blk) ||
-	    bloc->logicalBlockNum + count > map->s_partition_len) {
+	    blk > map->s_partition_len) {
 		udf_debug("Invalid request to free blocks: (%d, %u), off %u, "
 			  "len %u, partition len %u\n",
 			  partition, bloc->logicalBlockNum, offset, count,
diff --git a/fs/udf/super.c b/fs/udf/super.c
index cad2e15d633b6..be4085da2104c 100644
--- a/fs/udf/super.c
+++ b/fs/udf/super.c
@@ -2321,7 +2321,7 @@ static int udf_fill_super(struct super_block *sb, struct fs_context *fc)
 
 error_out:
 	iput(sbi->s_vat_inode);
-	unload_nls(uopt->nls_map);
+	unload_nls(sbi->s_nls_map);
 	if (lvid_open)
 		udf_close_lvid(sb);
 	brelse(sbi->s_lvid_bh);
diff --git a/fs/unicode/utf8-core.c b/fs/unicode/utf8-core.c
index 543c60c124615..8c4b8e27e9562 100644
--- a/fs/unicode/utf8-core.c
+++ b/fs/unicode/utf8-core.c
@@ -138,28 +138,6 @@ int utf8_casefold_hash(const struct unicode_map *um, const void *salt,
 }
 EXPORT_SYMBOL(utf8_casefold_hash);
 
-int utf8_normalize(const struct unicode_map *um, const struct qstr *str,
-		   unsigned char *dest, size_t dlen)
-{
-	struct utf8cursor cur;
-	ssize_t nlen = 0;
-
-	if (utf8ncursor(&cur, um, UTF8_NFDI, str->name, str->len) < 0)
-		return -EINVAL;
-
-	for (nlen = 0; nlen < dlen; nlen++) {
-		int c = utf8byte(&cur);
-
-		dest[nlen] = c;
-		if (!c)
-			return nlen;
-		if (c == -1)
-			break;
-	}
-	return -EINVAL;
-}
-EXPORT_SYMBOL(utf8_normalize);
-
 static const struct utf8data *find_table_version(const struct utf8data *table,
 		size_t nr_entries, unsigned int version)
 {
diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 4b53dc4a32664..390e4b7d9cb9f 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -786,6 +786,8 @@ void mremap_userfaultfd_fail(struct vm_userfaultfd_ctx *vm_ctx)
 	if (!ctx)
 		return;
 
+	atomic_dec(&ctx->mmap_changing);
+	VM_WARN_ON_ONCE(atomic_read(&ctx->mmap_changing) < 0);
 	userfaultfd_ctx_put(ctx);
 }
 
diff --git a/fs/vboxsf/dir.c b/fs/vboxsf/dir.c
index 42bedc4ec7af7..c5bd3271aa961 100644
--- a/fs/vboxsf/dir.c
+++ b/fs/vboxsf/dir.c
@@ -477,4 +477,5 @@ const struct inode_operations vboxsf_dir_iops = {
 	.symlink = vboxsf_dir_symlink,
 	.getattr = vboxsf_getattr,
 	.setattr = vboxsf_setattr,
+	.fileattr_get = vboxsf_fileattr_get,
 };
diff --git a/fs/vboxsf/file.c b/fs/vboxsf/file.c
index 7a7a3fbb26514..943953867e18e 100644
--- a/fs/vboxsf/file.c
+++ b/fs/vboxsf/file.c
@@ -222,7 +222,8 @@ const struct file_operations vboxsf_reg_fops = {
 
 const struct inode_operations vboxsf_reg_iops = {
 	.getattr = vboxsf_getattr,
-	.setattr = vboxsf_setattr
+	.setattr = vboxsf_setattr,
+	.fileattr_get = vboxsf_fileattr_get,
 };
 
 static int vboxsf_read_folio(struct file *file, struct folio *folio)
@@ -389,5 +390,6 @@ static const char *vboxsf_get_link(struct dentry *dentry, struct inode *inode,
 }
 
 const struct inode_operations vboxsf_lnk_iops = {
-	.get_link = vboxsf_get_link
+	.get_link = vboxsf_get_link,
+	.fileattr_get = vboxsf_fileattr_get,
 };
diff --git a/fs/vboxsf/super.c b/fs/vboxsf/super.c
index a618cb093e007..a61fbab51d370 100644
--- a/fs/vboxsf/super.c
+++ b/fs/vboxsf/super.c
@@ -185,6 +185,13 @@ static int vboxsf_fill_super(struct super_block *sb, struct fs_context *fc)
 	if (err)
 		goto fail_unmap;
 
+	/*
+	 * A failed query leaves sbi->case_insensitive false, so the
+	 * mount defaults to reporting case-sensitive behavior. Do not
+	 * fail the mount over an advisory attribute.
+	 */
+	vboxsf_query_case_sensitive(sbi);
+
 	sb->s_magic = VBOXSF_SUPER_MAGIC;
 	sb->s_blocksize = 1024;
 	sb->s_maxbytes = MAX_LFS_FILESIZE;
diff --git a/fs/vboxsf/utils.c b/fs/vboxsf/utils.c
index 440e8c50629d3..298bfc93255c0 100644
--- a/fs/vboxsf/utils.c
+++ b/fs/vboxsf/utils.c
@@ -11,6 +11,7 @@
 #include <linux/sizes.h>
 #include <linux/pagemap.h>
 #include <linux/vfs.h>
+#include <linux/fileattr.h>
 #include "vfsmod.h"
 
 struct inode *vboxsf_new_inode(struct super_block *sb)
@@ -567,3 +568,32 @@ int vboxsf_dir_read_all(struct vboxsf_sbi *sbi, struct vboxsf_dir_info *sf_d,
 
 	return err;
 }
+
+int vboxsf_query_case_sensitive(struct vboxsf_sbi *sbi)
+{
+	struct shfl_volinfo volinfo = {};
+	u32 buf_len;
+	int err;
+
+	buf_len = sizeof(volinfo);
+	err = vboxsf_fsinfo(sbi->root, 0, SHFL_INFO_GET | SHFL_INFO_VOLUME,
+			    &buf_len, &volinfo);
+	if (err)
+		return err;
+	if (buf_len < sizeof(volinfo))
+		return 0;
+
+	sbi->case_insensitive = !volinfo.properties.case_sensitive;
+	return 0;
+}
+
+int vboxsf_fileattr_get(struct dentry *dentry, struct file_kattr *fa)
+{
+	struct vboxsf_sbi *sbi = VBOXSF_SBI(dentry->d_sb);
+
+	if (sbi->case_insensitive) {
+		fa->fsx_xflags |= FS_XFLAG_CASEFOLD;
+		fa->flags |= FS_CASEFOLD_FL;
+	}
+	return 0;
+}
diff --git a/fs/vboxsf/vfsmod.h b/fs/vboxsf/vfsmod.h
index 05973eb89d528..b61afd0ce842d 100644
--- a/fs/vboxsf/vfsmod.h
+++ b/fs/vboxsf/vfsmod.h
@@ -47,6 +47,7 @@ struct vboxsf_sbi {
 	u32 next_generation;
 	u32 root;
 	int bdi_id;
+	bool case_insensitive;
 };
 
 /* per-inode information */
@@ -111,6 +112,11 @@ void vboxsf_dir_info_free(struct vboxsf_dir_info *p);
 int vboxsf_dir_read_all(struct vboxsf_sbi *sbi, struct vboxsf_dir_info *sf_d,
 			u64 handle);
 
+int vboxsf_query_case_sensitive(struct vboxsf_sbi *sbi);
+
+struct file_kattr;
+int vboxsf_fileattr_get(struct dentry *dentry, struct file_kattr *fa);
+
 /* from vboxsf_wrappers.c */
 int vboxsf_connect(void);
 void vboxsf_disconnect(void);
diff --git a/fs/verity/measure.c b/fs/verity/measure.c
index 6a35623ebdf09..265fa0253e3dc 100644
--- a/fs/verity/measure.c
+++ b/fs/verity/measure.c
@@ -118,9 +118,9 @@ __bpf_kfunc_start_defs();
  *
  * Return: 0 on success, a negative value on error.
  */
-__bpf_kfunc int bpf_get_fsverity_digest(struct file *file, struct bpf_dynptr *digest_p)
+__bpf_kfunc int bpf_get_fsverity_digest(struct file *file, const struct bpf_dynptr *digest_p)
 {
-	struct bpf_dynptr_kern *digest_ptr = (struct bpf_dynptr_kern *)digest_p;
+	const struct bpf_dynptr_kern *digest_ptr = (struct bpf_dynptr_kern *)digest_p;
 	const struct inode *inode = file_inode(file);
 	u32 dynptr_sz = __bpf_dynptr_size(digest_ptr);
 	struct fsverity_digest *arg;
diff --git a/fs/xattr.c b/fs/xattr.c
index 09ecbaaa16608..0bc3b47e6936c 100644
--- a/fs/xattr.c
+++ b/fs/xattr.c
@@ -510,9 +510,12 @@ vfs_listxattr(struct dentry *dentry, char *list, size_t size)
 	if (inode->i_op->listxattr) {
 		error = inode->i_op->listxattr(dentry, list, size);
 	} else {
-		error = security_inode_listsecurity(inode, list, size);
-		if (size && error > size)
-			error = -ERANGE;
+		ssize_t remaining = size;
+
+		error = security_inode_listsecurity(inode, &list, &remaining);
+		if (error)
+			return error;
+		error = size - remaining;
 	}
 	return error;
 }
@@ -1540,7 +1543,7 @@ ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs,
 	if (err)
 		return err;
 
-	err = security_inode_listsecurity(inode, buffer, remaining_size);
+	err = security_inode_listsecurity(inode, &buffer, &remaining_size);
 	if (err < 0)
 		return err;
 
diff --git a/fs/xfs/libxfs/xfs_inode_util.c b/fs/xfs/libxfs/xfs_inode_util.c
index 551fa51befb65..82be54b6f8d3a 100644
--- a/fs/xfs/libxfs/xfs_inode_util.c
+++ b/fs/xfs/libxfs/xfs_inode_util.c
@@ -130,6 +130,8 @@ xfs_ip2xflags(
 
 	if (xfs_inode_has_attr_fork(ip))
 		flags |= FS_XFLAG_HASATTR;
+	if (xfs_has_asciici(ip->i_mount))
+		flags |= FS_XFLAG_CASEFOLD;
 	return flags;
 }
 
diff --git a/fs/xfs/xfs_export.c b/fs/xfs/xfs_export.c
index e3e3c3c898400..9b2ad3786b190 100644
--- a/fs/xfs/xfs_export.c
+++ b/fs/xfs/xfs_export.c
@@ -244,8 +244,6 @@ const struct export_operations xfs_export_operations = {
 	.get_parent		= xfs_fs_get_parent,
 	.commit_metadata	= xfs_fs_nfs_commit_metadata,
 #ifdef CONFIG_EXPORTFS_BLOCK_OPS
-	.get_uuid		= xfs_fs_get_uuid,
-	.map_blocks		= xfs_fs_map_blocks,
-	.commit_blocks		= xfs_fs_commit_blocks,
+	.block_ops		= &xfs_export_block_ops,
 #endif
 };
diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c
index 9978ac1422fc4..ddf2707c88945 100644
--- a/fs/xfs/xfs_inode.c
+++ b/fs/xfs/xfs_inode.c
@@ -1040,7 +1040,7 @@ xfs_itruncate_extents_flags(
 	int			error = 0;
 
 	xfs_assert_ilocked(ip, XFS_ILOCK_EXCL);
-	if (icount_read(VFS_I(ip)))
+	if (icount_read_once(VFS_I(ip)))
 		xfs_assert_ilocked(ip, XFS_IOLOCK_EXCL);
 	if (whichfork == XFS_DATA_FORK)
 		ASSERT(new_size <= XFS_ISIZE(ip));
diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c
index 46e234863644f..f8216f74679fd 100644
--- a/fs/xfs/xfs_ioctl.c
+++ b/fs/xfs/xfs_ioctl.c
@@ -517,7 +517,7 @@ xfs_ioc_fsgetxattra(
 	xfs_inode_t		*ip,
 	void			__user *arg)
 {
-	struct file_kattr	fa;
+	struct file_kattr	fa = {};
 
 	xfs_ilock(ip, XFS_ILOCK_SHARED);
 	xfs_fill_fsxattr(ip, XFS_ATTR_FORK, &fa);
@@ -755,9 +755,23 @@ xfs_fileattr_set(
 	trace_xfs_ioctl_setattr(ip);
 
 	if (!fa->fsx_valid) {
-		if (fa->flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL |
-				  FS_NOATIME_FL | FS_NODUMP_FL |
-				  FS_SYNC_FL | FS_DAX_FL | FS_PROJINHERIT_FL))
+		unsigned int allowed = FS_IMMUTABLE_FL | FS_APPEND_FL |
+				       FS_NOATIME_FL | FS_NODUMP_FL |
+				       FS_SYNC_FL | FS_DAX_FL |
+				       FS_PROJINHERIT_FL;
+
+		/*
+		 * FS_CASEFOLD_FL reflects the ASCIICI superblock feature,
+		 * a read-only property. Accept it as a no-op so chattr's
+		 * RMW round-trip succeeds; reject any attempt to enable
+		 * it on a non-ASCIICI filesystem. xfs_flags2diflags()
+		 * has no clause for CASEFOLD, so the bit is dropped from
+		 * the on-disk diflags regardless.
+		 */
+		if (xfs_has_asciici(mp))
+			allowed |= FS_CASEFOLD_FL;
+
+		if (fa->flags & ~allowed)
 			return -EOPNOTSUPP;
 	}
 
diff --git a/fs/xfs/xfs_pnfs.c b/fs/xfs/xfs_pnfs.c
index 221e55887a2a4..266a07601e8d0 100644
--- a/fs/xfs/xfs_pnfs.c
+++ b/fs/xfs/xfs_pnfs.c
@@ -13,6 +13,7 @@
 #include "xfs_bmap.h"
 #include "xfs_iomap.h"
 #include "xfs_pnfs.h"
+#include <linux/exportfs_block.h>
 
 /*
  * Ensure that we do not have any outstanding pNFS layouts that can be used by
@@ -45,11 +46,22 @@ xfs_break_leased_layouts(
 	return error;
 }
 
+static expfs_block_layouts_t
+xfs_fs_layouts_supported(
+	struct super_block	*sb)
+{
+	expfs_block_layouts_t	supported = EXPFS_BLOCK_IN_BAND_ID;
+
+	if (exportfs_bdev_supports_out_of_band_id(sb->s_bdev))
+		supported |= EXPFS_BLOCK_OUT_OF_BAND_ID;
+	return supported;
+}
+
 /*
  * Get a unique ID including its location so that the client can identify
  * the exported device.
  */
-int
+static int
 xfs_fs_get_uuid(
 	struct super_block	*sb,
 	u8			*buf,
@@ -104,7 +116,7 @@ xfs_fs_map_update_inode(
 /*
  * Get a layout for the pNFS client.
  */
-int
+static int
 xfs_fs_map_blocks(
 	struct inode		*inode,
 	loff_t			offset,
@@ -252,28 +264,27 @@ xfs_pnfs_validate_isize(
  * to manually flush the cache here similar to what the fsync code path does
  * for datasyncs on files that have no dirty metadata.
  */
-int
+static int
 xfs_fs_commit_blocks(
 	struct inode		*inode,
 	struct iomap		*maps,
 	int			nr_maps,
-	struct iattr		*iattr)
+	loff_t			new_size)
 {
 	struct xfs_inode	*ip = XFS_I(inode);
 	struct xfs_mount	*mp = ip->i_mount;
 	struct xfs_trans	*tp;
+	struct timespec64	now;
 	bool			update_isize = false;
 	int			error, i;
 	loff_t			size;
 
-	ASSERT(iattr->ia_valid & (ATTR_ATIME|ATTR_CTIME|ATTR_MTIME));
-
 	xfs_ilock(ip, XFS_IOLOCK_EXCL);
 
 	size = i_size_read(inode);
-	if ((iattr->ia_valid & ATTR_SIZE) && iattr->ia_size > size) {
+	if (new_size > size) {
 		update_isize = true;
-		size = iattr->ia_size;
+		size = new_size;
 	}
 
 	for (i = 0; i < nr_maps; i++) {
@@ -318,11 +329,13 @@ xfs_fs_commit_blocks(
 	xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
 	xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
 
-	ASSERT(!(iattr->ia_valid & (ATTR_UID | ATTR_GID)));
-	setattr_copy(&nop_mnt_idmap, inode, iattr);
+	now = inode_set_ctime_current(inode);
+	inode_set_atime_to_ts(inode, now);
+	inode_set_mtime_to_ts(inode, now);
+
 	if (update_isize) {
-		i_size_write(inode, iattr->ia_size);
-		ip->i_disk_size = iattr->ia_size;
+		i_size_write(inode, new_size);
+		ip->i_disk_size = new_size;
 	}
 
 	xfs_trans_set_sync(tp);
@@ -332,3 +345,10 @@ xfs_fs_commit_blocks(
 	xfs_iunlock(ip, XFS_IOLOCK_EXCL);
 	return error;
 }
+
+const struct exportfs_block_ops xfs_export_block_ops = {
+	.layouts_supported	= xfs_fs_layouts_supported,
+	.get_uuid		= xfs_fs_get_uuid,
+	.map_blocks		= xfs_fs_map_blocks,
+	.commit_blocks		= xfs_fs_commit_blocks,
+};
diff --git a/fs/xfs/xfs_pnfs.h b/fs/xfs/xfs_pnfs.h
index 940c6c2ad88c5..bf43b2009e4cc 100644
--- a/fs/xfs/xfs_pnfs.h
+++ b/fs/xfs/xfs_pnfs.h
@@ -2,13 +2,9 @@
 #ifndef _XFS_PNFS_H
 #define _XFS_PNFS_H 1
 
-#ifdef CONFIG_EXPORTFS_BLOCK_OPS
-int xfs_fs_get_uuid(struct super_block *sb, u8 *buf, u32 *len, u64 *offset);
-int xfs_fs_map_blocks(struct inode *inode, loff_t offset, u64 length,
-		struct iomap *iomap, bool write, u32 *device_generation);
-int xfs_fs_commit_blocks(struct inode *inode, struct iomap *maps, int nr_maps,
-		struct iattr *iattr);
+#include <linux/exportfs_block.h>
 
+#ifdef CONFIG_EXPORTFS_BLOCK_OPS
 int xfs_break_leased_layouts(struct inode *inode, uint *iolock,
 		bool *did_unlock);
 #else
@@ -18,4 +14,7 @@ xfs_break_leased_layouts(struct inode *inode, uint *iolock, bool *did_unlock)
 	return 0;
 }
 #endif /* CONFIG_EXPORTFS_BLOCK_OPS */
+
+extern const struct exportfs_block_ops xfs_export_block_ops;
+
 #endif /* _XFS_PNFS_H */
diff --git a/fs/xfs/xfs_trace.h b/fs/xfs/xfs_trace.h
index 1c098cfc5c00d..f87c738d84b24 100644
--- a/fs/xfs/xfs_trace.h
+++ b/fs/xfs/xfs_trace.h
@@ -1158,7 +1158,7 @@ DECLARE_EVENT_CLASS(xfs_iref_class,
 	TP_fast_assign(
 		__entry->dev = VFS_I(ip)->i_sb->s_dev;
 		__entry->ino = ip->i_ino;
-		__entry->count = icount_read(VFS_I(ip));
+		__entry->count = icount_read_once(VFS_I(ip));
 		__entry->pincount = atomic_read(&ip->i_pincount);
 		__entry->iflags = ip->i_flags;
 		__entry->caller_ip = caller_ip;
diff --git a/fs/zonefs/super.c b/fs/zonefs/super.c
index 9b646cb5335d0..ff43d6d1ea30a 100644
--- a/fs/zonefs/super.c
+++ b/fs/zonefs/super.c
@@ -610,10 +610,14 @@ static long zonefs_fname_to_fno(const struct qstr *fname)
 		return c - '0';
 
 	for (i = 0, rname = name + len - 1; i < len; i++, rname--) {
+		long digit;
+
 		c = *rname;
 		if (!isdigit(c))
 			return -ENOENT;
-		fno += (c - '0') * shift;
+		digit = (c - '0') * shift;
+		if (check_add_overflow(fno, digit, &fno))
+			return -ENOENT;
 		shift *= 10;
 	}
 
diff --git a/include/acpi/acpi_bus.h b/include/acpi/acpi_bus.h
index c41d9a7565cf1..f44247693bcf9 100644
--- a/include/acpi/acpi_bus.h
+++ b/include/acpi/acpi_bus.h
@@ -189,6 +189,10 @@ struct acpi_driver {
  * -----------
  */
 
+bool acpi_of_match_device(const struct acpi_device *adev,
+			  const struct of_device_id *of_match_table,
+			  const struct of_device_id **of_id);
+
 /* Status (_STA) */
 
 struct acpi_device_status {
@@ -994,6 +998,13 @@ int acpi_scan_add_dep(acpi_handle handle, struct acpi_handle_list *dep_devices);
 u32 arch_acpi_add_auto_dep(acpi_handle handle);
 #else	/* CONFIG_ACPI */
 
+static inline bool acpi_of_match_device(const struct acpi_device *adev,
+					const struct of_device_id *of_match_table,
+					const struct of_device_id **of_id)
+{
+	return false;
+}
+
 static inline int register_acpi_bus_type(void *bus) { return 0; }
 static inline int unregister_acpi_bus_type(void *bus) { return 0; }
 
diff --git a/include/asm-generic/barrier.h b/include/asm-generic/barrier.h
index d4f581c1e21da..b99cb57dfcccc 100644
--- a/include/asm-generic/barrier.h
+++ b/include/asm-generic/barrier.h
@@ -235,7 +235,7 @@ do {									\
 /**
  * smp_cond_load_relaxed() - (Spin) wait for cond with no ordering guarantees
  * @ptr: pointer to the variable to wait on
- * @cond: boolean expression to wait for
+ * @cond_expr: boolean expression to wait for
  *
  * Equivalent to using READ_ONCE() on the condition variable.
  *
@@ -259,7 +259,7 @@ do {									\
 /**
  * smp_cond_load_acquire() - (Spin) wait for cond with ACQUIRE ordering
  * @ptr: pointer to the variable to wait on
- * @cond: boolean expression to wait for
+ * @cond_expr: boolean expression to wait for
  *
  * Equivalent to using smp_load_acquire() on the condition variable but employs
  * the control dependency of the wait to reduce the barrier on many platforms.
diff --git a/include/asm-generic/bitops/lock.h b/include/asm-generic/bitops/lock.h
index 14d4ec8c5152d..ffb73b6129e73 100644
--- a/include/asm-generic/bitops/lock.h
+++ b/include/asm-generic/bitops/lock.h
@@ -16,16 +16,16 @@
  * It can be used to implement bit locks.
  */
 static __always_inline int
-arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *p)
+arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *addr)
 {
 	long old;
 	unsigned long mask = BIT_MASK(nr);
 
-	p += BIT_WORD(nr);
-	if (READ_ONCE(*p) & mask)
+	addr += BIT_WORD(nr);
+	if (READ_ONCE(*addr) & mask)
 		return 1;
 
-	old = raw_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)p);
+	old = raw_atomic_long_fetch_or_acquire(mask, (atomic_long_t *)addr);
 	return !!(old & mask);
 }
 
@@ -38,10 +38,10 @@ arch_test_and_set_bit_lock(unsigned int nr, volatile unsigned long *p)
  * This operation is atomic and provides release barrier semantics.
  */
 static __always_inline void
-arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *p)
+arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *addr)
 {
-	p += BIT_WORD(nr);
-	raw_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)p);
+	addr += BIT_WORD(nr);
+	raw_atomic_long_fetch_andnot_release(BIT_MASK(nr), (atomic_long_t *)addr);
 }
 
 /**
@@ -56,14 +56,14 @@ arch_clear_bit_unlock(unsigned int nr, volatile unsigned long *p)
  * See for example x86's implementation.
  */
 static inline void
-arch___clear_bit_unlock(unsigned int nr, volatile unsigned long *p)
+arch___clear_bit_unlock(unsigned int nr, volatile unsigned long *addr)
 {
 	unsigned long old;
 
-	p += BIT_WORD(nr);
-	old = READ_ONCE(*p);
+	addr += BIT_WORD(nr);
+	old = READ_ONCE(*addr);
 	old &= ~BIT_MASK(nr);
-	raw_atomic_long_set_release((atomic_long_t *)p, old);
+	raw_atomic_long_set_release((atomic_long_t *)addr, old);
 }
 
 #ifndef arch_xor_unlock_is_negative_byte
diff --git a/include/asm-generic/pgalloc.h b/include/asm-generic/pgalloc.h
index 57137d3ac1592..051aa1331051c 100644
--- a/include/asm-generic/pgalloc.h
+++ b/include/asm-generic/pgalloc.h
@@ -4,7 +4,7 @@
 
 #ifdef CONFIG_MMU
 
-#define GFP_PGTABLE_KERNEL	(GFP_KERNEL | __GFP_ZERO)
+#define GFP_PGTABLE_KERNEL	(GFP_KERNEL | __GFP_ZERO | __GFP_SKIP_KASAN)
 #define GFP_PGTABLE_USER	(GFP_PGTABLE_KERNEL | __GFP_ACCOUNT)
 
 /**
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index bdcc2778ac64f..cb41cc6a00241 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -240,6 +240,23 @@ static inline void tlb_remove_table(struct mmu_gather *tlb, void *table)
 }
 #endif /* CONFIG_MMU_GATHER_TABLE_FREE */
 
+/**
+ * tlb_table_flush_implies_ipi_broadcast - does TLB flush imply IPI sync
+ *
+ * When page table operations require synchronization with software/lockless
+ * walkers, they flush the TLB (tlb->freed_tables or tlb->unshared_tables)
+ * then call tlb_remove_table_sync_{one,rcu}(). If the flush already sent
+ * IPIs to all CPUs, the sync call is redundant.
+ *
+ * Returns false by default. Architectures can override by defining this.
+ */
+#ifndef tlb_table_flush_implies_ipi_broadcast
+static inline bool tlb_table_flush_implies_ipi_broadcast(void)
+{
+	return false;
+}
+#endif
+
 #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
 /*
  * This allows an architecture that does not use the linux page-tables for
diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h
index b276f783494c4..4569045e7139f 100644
--- a/include/asm-generic/uaccess.h
+++ b/include/asm-generic/uaccess.h
@@ -91,8 +91,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 	memcpy((void __force *)to, from, n);
 	return 0;
 }
-#define INLINE_COPY_FROM_USER
-#define INLINE_COPY_TO_USER
+#define INLINE_COPY_USER
 #endif /* CONFIG_UACCESS_MEMCPY */
 
 /*
diff --git a/include/crypto/df_sp80090a.h b/include/crypto/df_sp80090a.h
index cb5d6fe15d40c..e594fb718eb80 100644
--- a/include/crypto/df_sp80090a.h
+++ b/include/crypto/df_sp80090a.h
@@ -9,6 +9,31 @@
 
 #include <crypto/internal/cipher.h>
 #include <crypto/aes.h>
+#include <linux/list.h>
+
+/*
+ * Concatenation Helper and string operation helper
+ *
+ * SP800-90A requires the concatenation of different data. To avoid copying
+ * buffers around or allocate additional memory, the following data structure
+ * is used to point to the original memory with its size. In addition, it
+ * is used to build a linked list. The linked list defines the concatenation
+ * of individual buffers. The order of memory block referenced in that
+ * linked list determines the order of concatenation.
+ */
+struct drbg_string {
+	const unsigned char *buf;
+	size_t len;
+	struct list_head list;
+};
+
+static inline void drbg_string_fill(struct drbg_string *string,
+				    const unsigned char *buf, size_t len)
+{
+	string->buf = buf;
+	string->len = len;
+	INIT_LIST_HEAD(&string->list);
+}
 
 static inline int crypto_drbg_ctr_df_datalen(u8 statelen, u8 blocklen)
 {
diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h
deleted file mode 100644
index 2d42518cbdce8..0000000000000
--- a/include/crypto/drbg.h
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * DRBG based on NIST SP800-90A
- *
- * Copyright Stephan Mueller <smueller@chronox.de>, 2014
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, and the entire permission notice in its entirety,
- *    including the disclaimer of warranties.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. The name of the author may not be used to endorse or promote
- *    products derived from this software without specific prior
- *    written permission.
- *
- * ALTERNATIVELY, this product may be distributed under the terms of
- * the GNU General Public License, in which case the provisions of the GPL are
- * required INSTEAD OF the above restrictions.  (This clause is
- * necessary due to a potential bad interaction between the GPL and
- * the restrictions contained in a BSD-style copyright.)
- *
- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
- * WHICH ARE HEREBY DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE
- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
- * DAMAGE.
- */
-
-#ifndef _DRBG_H
-#define _DRBG_H
-
-
-#include <linux/random.h>
-#include <linux/scatterlist.h>
-#include <crypto/hash.h>
-#include <crypto/skcipher.h>
-#include <linux/module.h>
-#include <linux/crypto.h>
-#include <linux/slab.h>
-#include <crypto/internal/drbg.h>
-#include <crypto/internal/rng.h>
-#include <crypto/rng.h>
-#include <linux/fips.h>
-#include <linux/mutex.h>
-#include <linux/list.h>
-#include <linux/workqueue.h>
-
-struct drbg_state;
-typedef uint32_t drbg_flag_t;
-
-struct drbg_core {
-	drbg_flag_t flags;	/* flags for the cipher */
-	__u8 statelen;		/* maximum state length */
-	__u8 blocklen_bytes;	/* block size of output in bytes */
-	char cra_name[CRYPTO_MAX_ALG_NAME]; /* mapping to kernel crypto API */
-	 /* kernel crypto API backend cipher name */
-	char backend_cra_name[CRYPTO_MAX_ALG_NAME];
-};
-
-struct drbg_state_ops {
-	int (*update)(struct drbg_state *drbg, struct list_head *seed,
-		      int reseed);
-	int (*generate)(struct drbg_state *drbg,
-			unsigned char *buf, unsigned int buflen,
-			struct list_head *addtl);
-	int (*crypto_init)(struct drbg_state *drbg);
-	int (*crypto_fini)(struct drbg_state *drbg);
-
-};
-
-struct drbg_test_data {
-	struct drbg_string *testentropy; /* TEST PARAMETER: test entropy */
-};
-
-enum drbg_seed_state {
-	DRBG_SEED_STATE_UNSEEDED,
-	DRBG_SEED_STATE_PARTIAL, /* Seeded with !rng_is_initialized() */
-	DRBG_SEED_STATE_FULL,
-};
-
-struct drbg_state {
-	struct mutex drbg_mutex;	/* lock around DRBG */
-	unsigned char *V;	/* internal state 10.1.1.1 1a) */
-	unsigned char *Vbuf;
-	/* hash: static value 10.1.1.1 1b) hmac / ctr: key */
-	unsigned char *C;
-	unsigned char *Cbuf;
-	/* Number of RNG requests since last reseed -- 10.1.1.1 1c) */
-	size_t reseed_ctr;
-	size_t reseed_threshold;
-	 /* some memory the DRBG can use for its operation */
-	unsigned char *scratchpad;
-	unsigned char *scratchpadbuf;
-	void *priv_data;	/* Cipher handle */
-
-	struct crypto_skcipher *ctr_handle;	/* CTR mode cipher handle */
-	struct skcipher_request *ctr_req;	/* CTR mode request handle */
-	__u8 *outscratchpadbuf;			/* CTR mode output scratchpad */
-        __u8 *outscratchpad;			/* CTR mode aligned outbuf */
-	struct crypto_wait ctr_wait;		/* CTR mode async wait obj */
-	struct scatterlist sg_in, sg_out;	/* CTR mode SGLs */
-
-	enum drbg_seed_state seeded;		/* DRBG fully seeded? */
-	unsigned long last_seed_time;
-	bool pr;		/* Prediction resistance enabled? */
-	bool fips_primed;	/* Continuous test primed? */
-	unsigned char *prev;	/* FIPS 140-2 continuous test value */
-	struct crypto_rng *jent;
-	const struct drbg_state_ops *d_ops;
-	const struct drbg_core *core;
-	struct drbg_string test_data;
-};
-
-static inline __u8 drbg_statelen(struct drbg_state *drbg)
-{
-	if (drbg && drbg->core)
-		return drbg->core->statelen;
-	return 0;
-}
-
-static inline __u8 drbg_blocklen(struct drbg_state *drbg)
-{
-	if (drbg && drbg->core)
-		return drbg->core->blocklen_bytes;
-	return 0;
-}
-
-static inline __u8 drbg_keylen(struct drbg_state *drbg)
-{
-	if (drbg && drbg->core)
-		return (drbg->core->statelen - drbg->core->blocklen_bytes);
-	return 0;
-}
-
-static inline size_t drbg_max_request_bytes(struct drbg_state *drbg)
-{
-	/* SP800-90A requires the limit 2**19 bits, but we return bytes */
-	return (1 << 16);
-}
-
-static inline size_t drbg_max_addtl(struct drbg_state *drbg)
-{
-	/* SP800-90A requires 2**35 bytes additional info str / pers str */
-#if (__BITS_PER_LONG == 32)
-	/*
-	 * SP800-90A allows smaller maximum numbers to be returned -- we
-	 * return SIZE_MAX - 1 to allow the verification of the enforcement
-	 * of this value in drbg_healthcheck_sanity.
-	 */
-	return (SIZE_MAX - 1);
-#else
-	return (1UL<<35);
-#endif
-}
-
-static inline size_t drbg_max_requests(struct drbg_state *drbg)
-{
-	/* SP800-90A requires 2**48 maximum requests before reseeding */
-	return (1<<20);
-}
-
-/*
- * This is a wrapper to the kernel crypto API function of
- * crypto_rng_generate() to allow the caller to provide additional data.
- *
- * @drng DRBG handle -- see crypto_rng_get_bytes
- * @outbuf output buffer -- see crypto_rng_get_bytes
- * @outlen length of output buffer -- see crypto_rng_get_bytes
- * @addtl_input additional information string input buffer
- * @addtllen length of additional information string buffer
- *
- * return
- *	see crypto_rng_get_bytes
- */
-static inline int crypto_drbg_get_bytes_addtl(struct crypto_rng *drng,
-			unsigned char *outbuf, unsigned int outlen,
-			struct drbg_string *addtl)
-{
-	return crypto_rng_generate(drng, addtl->buf, addtl->len,
-				   outbuf, outlen);
-}
-
-/*
- * TEST code
- *
- * This is a wrapper to the kernel crypto API function of
- * crypto_rng_generate() to allow the caller to provide additional data and
- * allow furnishing of test_data
- *
- * @drng DRBG handle -- see crypto_rng_get_bytes
- * @outbuf output buffer -- see crypto_rng_get_bytes
- * @outlen length of output buffer -- see crypto_rng_get_bytes
- * @addtl_input additional information string input buffer
- * @addtllen length of additional information string buffer
- * @test_data filled test data
- *
- * return
- *	see crypto_rng_get_bytes
- */
-static inline int crypto_drbg_get_bytes_addtl_test(struct crypto_rng *drng,
-			unsigned char *outbuf, unsigned int outlen,
-			struct drbg_string *addtl,
-			struct drbg_test_data *test_data)
-{
-	crypto_rng_set_entropy(drng, test_data->testentropy->buf,
-			       test_data->testentropy->len);
-	return crypto_rng_generate(drng, addtl->buf, addtl->len,
-				   outbuf, outlen);
-}
-
-/*
- * TEST code
- *
- * This is a wrapper to the kernel crypto API function of
- * crypto_rng_reset() to allow the caller to provide test_data
- *
- * @drng DRBG handle -- see crypto_rng_reset
- * @pers personalization string input buffer
- * @perslen length of additional information string buffer
- * @test_data filled test data
- *
- * return
- *	see crypto_rng_reset
- */
-static inline int crypto_drbg_reset_test(struct crypto_rng *drng,
-					 struct drbg_string *pers,
-					 struct drbg_test_data *test_data)
-{
-	crypto_rng_set_entropy(drng, test_data->testentropy->buf,
-			       test_data->testentropy->len);
-	return crypto_rng_reset(drng, pers->buf, pers->len);
-}
-
-/* DRBG type flags */
-#define DRBG_CTR	((drbg_flag_t)1<<0)
-#define DRBG_HMAC	((drbg_flag_t)1<<1)
-#define DRBG_HASH	((drbg_flag_t)1<<2)
-#define DRBG_TYPE_MASK	(DRBG_CTR | DRBG_HMAC | DRBG_HASH)
-/* DRBG strength flags */
-#define DRBG_STRENGTH128	((drbg_flag_t)1<<3)
-#define DRBG_STRENGTH192	((drbg_flag_t)1<<4)
-#define DRBG_STRENGTH256	((drbg_flag_t)1<<5)
-#define DRBG_STRENGTH_MASK	(DRBG_STRENGTH128 | DRBG_STRENGTH192 | \
-				 DRBG_STRENGTH256)
-
-enum drbg_prefixes {
-	DRBG_PREFIX0 = 0x00,
-	DRBG_PREFIX1,
-	DRBG_PREFIX2,
-	DRBG_PREFIX3
-};
-
-#endif /* _DRBG_H */
diff --git a/include/crypto/internal/drbg.h b/include/crypto/internal/drbg.h
deleted file mode 100644
index 371e52dcee6c5..0000000000000
--- a/include/crypto/internal/drbg.h
+++ /dev/null
@@ -1,54 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-
-/*
- * NIST SP800-90A DRBG derivation function
- *
- * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
- */
-
-#ifndef _INTERNAL_DRBG_H
-#define _INTERNAL_DRBG_H
-
-/*
- * Convert an integer into a byte representation of this integer.
- * The byte representation is big-endian
- *
- * @val value to be converted
- * @buf buffer holding the converted integer -- caller must ensure that
- *      buffer size is at least 32 bit
- */
-static inline void drbg_cpu_to_be32(__u32 val, unsigned char *buf)
-{
-	struct s {
-		__be32 conv;
-	};
-	struct s *conversion = (struct s *)buf;
-
-	conversion->conv = cpu_to_be32(val);
-}
-
-/*
- * Concatenation Helper and string operation helper
- *
- * SP800-90A requires the concatenation of different data. To avoid copying
- * buffers around or allocate additional memory, the following data structure
- * is used to point to the original memory with its size. In addition, it
- * is used to build a linked list. The linked list defines the concatenation
- * of individual buffers. The order of memory block referenced in that
- * linked list determines the order of concatenation.
- */
-struct drbg_string {
-	const unsigned char *buf;
-	size_t len;
-	struct list_head list;
-};
-
-static inline void drbg_string_fill(struct drbg_string *string,
-				    const unsigned char *buf, size_t len)
-{
-	string->buf = buf;
-	string->len = len;
-	INIT_LIST_HEAD(&string->list);
-}
-
-#endif //_INTERNAL_DRBG_H
diff --git a/include/crypto/krb5.h b/include/crypto/krb5.h
index 71dd38f59be1d..aac3ecf88467c 100644
--- a/include/crypto/krb5.h
+++ b/include/crypto/krb5.h
@@ -121,9 +121,12 @@ size_t crypto_krb5_how_much_buffer(const struct krb5_enctype *krb5,
 size_t crypto_krb5_how_much_data(const struct krb5_enctype *krb5,
 				 enum krb5_crypto_mode mode,
 				 size_t *_buffer_size, size_t *_offset);
-void crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
-				   enum krb5_crypto_mode mode,
-				   size_t *_offset, size_t *_len);
+int crypto_krb5_where_is_the_data(const struct krb5_enctype *krb5,
+				  enum krb5_crypto_mode mode,
+				  size_t *_offset, size_t *_len);
+int crypto_krb5_check_data_len(const struct krb5_enctype *krb5,
+			       enum krb5_crypto_mode mode,
+			       size_t len, size_t min_content);
 struct crypto_aead *crypto_krb5_prepare_encryption(const struct krb5_enctype *krb5,
 						   const struct krb5_buffer *TK,
 						   u32 usage, gfp_t gfp);
diff --git a/include/crypto/pkcs7.h b/include/crypto/pkcs7.h
index 38ec7f5f90411..bd83202cd805c 100644
--- a/include/crypto/pkcs7.h
+++ b/include/crypto/pkcs7.h
@@ -25,6 +25,10 @@ extern void pkcs7_free_message(struct pkcs7_message *pkcs7);
 extern int pkcs7_get_content_data(const struct pkcs7_message *pkcs7,
 				  const void **_data, size_t *_datalen,
 				  size_t *_headerlen);
+extern int pkcs7_get_authattr(const struct pkcs7_message *pkcs7,
+			      enum OID oid,
+			      const void **_data, size_t *_len);
+
 
 /*
  * pkcs7_trust.c
diff --git a/include/drm/bridge/analogix_dp.h b/include/drm/bridge/analogix_dp.h
index cf17646c1310e..854af692229b9 100644
--- a/include/drm/bridge/analogix_dp.h
+++ b/include/drm/bridge/analogix_dp.h
@@ -27,16 +27,13 @@ static inline bool is_rockchip(enum analogix_dp_devtype type)
 struct analogix_dp_plat_data {
 	enum analogix_dp_devtype dev_type;
 	struct drm_panel *panel;
+	struct drm_bridge *next_bridge;
 	struct drm_encoder *encoder;
 	struct drm_connector *connector;
-	bool skip_connector;
+	const struct component_ops *ops;
 
 	int (*power_on)(struct analogix_dp_plat_data *);
 	int (*power_off)(struct analogix_dp_plat_data *);
-	int (*attach)(struct analogix_dp_plat_data *, struct drm_bridge *,
-		      struct drm_connector *);
-	int (*get_modes)(struct analogix_dp_plat_data *,
-			 struct drm_connector *);
 };
 
 int analogix_dp_resume(struct analogix_dp_device *dp);
@@ -52,5 +49,6 @@ int analogix_dp_stop_crc(struct drm_connector *connector);
 
 struct analogix_dp_plat_data *analogix_dp_aux_to_plat_data(struct drm_dp_aux *aux);
 struct drm_dp_aux *analogix_dp_get_aux(struct analogix_dp_device *dp);
+int analogix_dp_finish_probe(struct analogix_dp_device *dp);
 
 #endif /* _ANALOGIX_DP_H_ */
diff --git a/include/drm/bridge/dw_hdmi.h b/include/drm/bridge/dw_hdmi.h
index 336f062e1f9d9..8500dd4f99d8c 100644
--- a/include/drm/bridge/dw_hdmi.h
+++ b/include/drm/bridge/dw_hdmi.h
@@ -126,6 +126,12 @@ struct dw_hdmi_phy_ops {
 struct dw_hdmi_plat_data {
 	struct regmap *regm;
 
+	/*
+	 * The HDMI output port number must be 1 if the port is described
+	 * in the device tree. 0 if the device tree does not describe the
+	 * next component (legacy mode, i.e. without
+	 * DRM_BRIDGE_ATTACH_NO_CONNECTOR flag when attaching bridge).
+	 */
 	unsigned int output_port;
 
 	unsigned long input_bus_encoding;
diff --git a/include/drm/bridge/imx.h b/include/drm/bridge/imx.h
deleted file mode 100644
index b93f719fe0e7e..0000000000000
--- a/include/drm/bridge/imx.h
+++ /dev/null
@@ -1,17 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0+
-/*
- * Copyright (C) 2012 Sascha Hauer, Pengutronix
- */
-
-#ifndef DRM_IMX_BRIDGE_H
-#define DRM_IMX_BRIDGE_H
-
-struct device;
-struct device_node;
-struct drm_bridge;
-
-struct drm_bridge *devm_imx_drm_legacy_bridge(struct device *dev,
-					      struct device_node *np,
-					      int type);
-
-#endif
diff --git a/include/drm/bridge/of-display-mode-bridge.h b/include/drm/bridge/of-display-mode-bridge.h
new file mode 100644
index 0000000000000..89fcfedf68d80
--- /dev/null
+++ b/include/drm/bridge/of-display-mode-bridge.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright (C) 2012 Sascha Hauer, Pengutronix
+ */
+
+#ifndef DRM_OF_DISPLAY_MODE_BRIDGE_H
+#define DRM_OF_DISPLAY_MODE_BRIDGE_H
+
+struct device;
+struct device_node;
+struct drm_bridge;
+
+struct drm_bridge *devm_drm_of_display_mode_bridge(struct device *dev,
+						   struct device_node *np,
+						   int type);
+
+#endif
diff --git a/include/drm/display/drm_dp.h b/include/drm/display/drm_dp.h
index 8b15d3eeb716a..829e4d98d61cd 100644
--- a/include/drm/display/drm_dp.h
+++ b/include/drm/display/drm_dp.h
@@ -718,6 +718,12 @@
 #define DP_EXTENDED_DPRX_SLEEP_WAKE_TIMEOUT_GRANT	    0x119   /* 1.4a */
 # define DP_DPRX_SLEEP_WAKE_TIMEOUT_PERIOD_GRANTED	    (1 << 0)
 
+#define PANEL_REPLAY_CONFIG3			0x11a /* DP 2.1 */
+# define DP_PR_AS_SDP_SETUP_TIME_MASK		(3 << 6)
+# define DP_PR_AS_SDP_SETUP_TIME_T1		(0 << 6)
+# define DP_PR_AS_SDP_SETUP_TIME_DYNAMIC	(1 << 6) /* DP 2.1 Table 2-227 */
+# define DP_PR_AS_SDP_SETUP_TIME_T2		(2 << 6)
+
 #define DP_FEC_CONFIGURATION		    0x120    /* 1.4 */
 # define DP_FEC_READY			    (1 << 0)
 # define DP_FEC_ERR_COUNT_SEL_MASK	    (7 << 1)
@@ -1202,12 +1208,11 @@
 # define DP_DPRX_SLEEP_WAKE_TIMEOUT_PERIOD_80_MS	0x04
 # define DP_DPRX_SLEEP_WAKE_TIMEOUT_PERIOD_100_MS	0x05
 
-#define DP_DPRX_FEATURE_ENUMERATION_LIST_CONT_1         0x2214 /* 2.0 E11 */
-# define DP_ADAPTIVE_SYNC_SDP_SUPPORTED    (1 << 0)
-# define DP_ADAPTIVE_SYNC_SDP_OPERATION_MODE		GENMASK(1, 0)
-# define DP_ADAPTIVE_SYNC_SDP_LENGTH				GENMASK(5, 0)
-# define DP_AS_SDP_FIRST_HALF_LINE_OR_3840_PIXEL_CYCLE_WINDOW_NOT_SUPPORTED (1 << 1)
-# define DP_VSC_EXT_SDP_FRAMEWORK_VERSION_1_SUPPORTED  (1 << 4)
+#define DP_DPRX_FEATURE_ENUMERATION_LIST_CONT_1					0x2214 /* 2.0 E11 */
+# define DP_ADAPTIVE_SYNC_SDP_SUPPORTED						BIT(0)
+# define DP_AS_SDP_FIRST_HALF_LINE_OR_3840_PIXEL_CYCLE_WINDOW_NOT_SUPPORTED	BIT(1)
+# define DP_AS_SDP_FAVT_PAYLOAD_FIELDS_PARSING_SUPPORTED			BIT(2) /* 2.1 */
+# define DP_VSC_EXT_SDP_FRAMEWORK_VERSION_1_SUPPORTED				BIT(4)
 
 #define DP_128B132B_SUPPORTED_LINK_RATES       0x2215 /* 2.0 */
 # define DP_UHBR10                             (1 << 0)
@@ -1870,4 +1875,7 @@ enum operation_mode {
 	DP_AS_SDP_FAVT_TRR_REACHED = 0x03
 };
 
+#define DP_AS_SDP_OPERATION_MODE_MASK	GENMASK(1, 0)
+#define DP_AS_SDP_LENGTH_MASK		GENMASK(5, 0)
+
 #endif /* _DRM_DP_H_ */
diff --git a/include/drm/display/drm_dp_helper.h b/include/drm/display/drm_dp_helper.h
index 1d0acd58f4867..8c2d77a032f06 100644
--- a/include/drm/display/drm_dp_helper.h
+++ b/include/drm/display/drm_dp_helper.h
@@ -126,6 +126,7 @@ struct drm_dp_as_sdp {
 	int duration_decr_ms;
 	bool target_rr_divider;
 	enum operation_mode mode;
+	int coasting_vtotal;
 };
 
 void drm_dp_as_sdp_log(struct drm_printer *p,
diff --git a/include/drm/display/drm_dp_mst_helper.h b/include/drm/display/drm_dp_mst_helper.h
index 2cfe1d4bfc960..27658bfb50016 100644
--- a/include/drm/display/drm_dp_mst_helper.h
+++ b/include/drm/display/drm_dp_mst_helper.h
@@ -916,13 +916,13 @@ void drm_dp_mst_connector_early_unregister(struct drm_connector *connector,
 					   struct drm_dp_mst_port *port);
 
 struct drm_dp_mst_topology_state *
-drm_atomic_get_mst_topology_state(struct drm_atomic_state *state,
+drm_atomic_get_mst_topology_state(struct drm_atomic_commit *state,
 				  struct drm_dp_mst_topology_mgr *mgr);
 struct drm_dp_mst_topology_state *
-drm_atomic_get_old_mst_topology_state(struct drm_atomic_state *state,
+drm_atomic_get_old_mst_topology_state(struct drm_atomic_commit *state,
 				      struct drm_dp_mst_topology_mgr *mgr);
 struct drm_dp_mst_topology_state *
-drm_atomic_get_new_mst_topology_state(struct drm_atomic_state *state,
+drm_atomic_get_new_mst_topology_state(struct drm_atomic_commit *state,
 				      struct drm_dp_mst_topology_mgr *mgr);
 struct drm_dp_mst_atomic_payload *
 drm_atomic_get_mst_payload_state(struct drm_dp_mst_topology_state *state,
@@ -931,31 +931,31 @@ bool drm_dp_mst_port_downstream_of_parent(struct drm_dp_mst_topology_mgr *mgr,
 					  struct drm_dp_mst_port *port,
 					  struct drm_dp_mst_port *parent);
 int __must_check
-drm_dp_atomic_find_time_slots(struct drm_atomic_state *state,
+drm_dp_atomic_find_time_slots(struct drm_atomic_commit *state,
 			      struct drm_dp_mst_topology_mgr *mgr,
 			      struct drm_dp_mst_port *port, int pbn);
-int drm_dp_mst_atomic_enable_dsc(struct drm_atomic_state *state,
+int drm_dp_mst_atomic_enable_dsc(struct drm_atomic_commit *state,
 				 struct drm_dp_mst_port *port,
 				 int pbn, bool enable);
 int __must_check
-drm_dp_mst_add_affected_dsc_crtcs(struct drm_atomic_state *state,
+drm_dp_mst_add_affected_dsc_crtcs(struct drm_atomic_commit *state,
 				  struct drm_dp_mst_topology_mgr *mgr);
 int __must_check
-drm_dp_atomic_release_time_slots(struct drm_atomic_state *state,
+drm_dp_atomic_release_time_slots(struct drm_atomic_commit *state,
 				 struct drm_dp_mst_topology_mgr *mgr,
 				 struct drm_dp_mst_port *port);
-void drm_dp_mst_atomic_wait_for_dependencies(struct drm_atomic_state *state);
-int __must_check drm_dp_mst_atomic_setup_commit(struct drm_atomic_state *state);
+void drm_dp_mst_atomic_wait_for_dependencies(struct drm_atomic_commit *state);
+int __must_check drm_dp_mst_atomic_setup_commit(struct drm_atomic_commit *state);
 int drm_dp_send_power_updown_phy(struct drm_dp_mst_topology_mgr *mgr,
 				 struct drm_dp_mst_port *port, bool power_up);
 int drm_dp_send_query_stream_enc_status(struct drm_dp_mst_topology_mgr *mgr,
 		struct drm_dp_mst_port *port,
 		struct drm_dp_query_stream_enc_status_ack_reply *status);
-int __must_check drm_dp_mst_atomic_check_mgr(struct drm_atomic_state *state,
+int __must_check drm_dp_mst_atomic_check_mgr(struct drm_atomic_commit *state,
 					     struct drm_dp_mst_topology_mgr *mgr,
 					     struct drm_dp_mst_topology_state *mst_state,
 					     struct drm_dp_mst_port **failing_port);
-int __must_check drm_dp_mst_atomic_check(struct drm_atomic_state *state);
+int __must_check drm_dp_mst_atomic_check(struct drm_atomic_commit *state);
 int __must_check drm_dp_mst_root_conn_atomic_check(struct drm_connector_state *new_conn_state,
 						   struct drm_dp_mst_topology_mgr *mgr);
 
@@ -982,7 +982,7 @@ extern const struct drm_private_state_funcs drm_dp_mst_topology_state_funcs;
 /**
  * __drm_dp_mst_state_iter_get - private atomic state iterator function for
  * macro-internal use
- * @state: &struct drm_atomic_state pointer
+ * @state: &struct drm_atomic_commit pointer
  * @mgr: pointer to the &struct drm_dp_mst_topology_mgr iteration cursor
  * @old_state: optional pointer to the old &struct drm_dp_mst_topology_state
  * iteration cursor
@@ -999,7 +999,7 @@ extern const struct drm_private_state_funcs drm_dp_mst_topology_state_funcs;
  * drm_dp_mst_topology_mgr, false otherwise.
  */
 static inline bool
-__drm_dp_mst_state_iter_get(struct drm_atomic_state *state,
+__drm_dp_mst_state_iter_get(struct drm_atomic_commit *state,
 			    struct drm_dp_mst_topology_mgr **mgr,
 			    struct drm_dp_mst_topology_state **old_state,
 			    struct drm_dp_mst_topology_state **new_state,
@@ -1022,7 +1022,7 @@ __drm_dp_mst_state_iter_get(struct drm_atomic_state *state,
 /**
  * for_each_oldnew_mst_mgr_in_state - iterate over all DP MST topology
  * managers in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @mgr: &struct drm_dp_mst_topology_mgr iteration cursor
  * @old_state: &struct drm_dp_mst_topology_state iteration cursor for the old
  * state
@@ -1041,7 +1041,7 @@ __drm_dp_mst_state_iter_get(struct drm_atomic_state *state,
 /**
  * for_each_old_mst_mgr_in_state - iterate over all DP MST topology managers
  * in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @mgr: &struct drm_dp_mst_topology_mgr iteration cursor
  * @old_state: &struct drm_dp_mst_topology_state iteration cursor for the old
  * state
@@ -1058,7 +1058,7 @@ __drm_dp_mst_state_iter_get(struct drm_atomic_state *state,
 /**
  * for_each_new_mst_mgr_in_state - iterate over all DP MST topology managers
  * in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @mgr: &struct drm_dp_mst_topology_mgr iteration cursor
  * @new_state: &struct drm_dp_mst_topology_state iteration cursor for the new
  * state
diff --git a/include/drm/display/drm_dp_tunnel.h b/include/drm/display/drm_dp_tunnel.h
index 87212c8479150..57f5e90ba8fda 100644
--- a/include/drm/display/drm_dp_tunnel.h
+++ b/include/drm/display/drm_dp_tunnel.h
@@ -14,7 +14,7 @@ struct drm_dp_aux;
 
 struct drm_device;
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_dp_tunnel_mgr;
 struct drm_dp_tunnel_state;
 
@@ -53,6 +53,7 @@ int drm_dp_tunnel_destroy(struct drm_dp_tunnel *tunnel);
 int drm_dp_tunnel_enable_bw_alloc(struct drm_dp_tunnel *tunnel);
 int drm_dp_tunnel_disable_bw_alloc(struct drm_dp_tunnel *tunnel);
 bool drm_dp_tunnel_bw_alloc_is_enabled(const struct drm_dp_tunnel *tunnel);
+bool drm_dp_tunnel_pr_optimization_supported(const struct drm_dp_tunnel *tunnel);
 int drm_dp_tunnel_alloc_bw(struct drm_dp_tunnel *tunnel, int bw);
 int drm_dp_tunnel_get_allocated_bw(struct drm_dp_tunnel *tunnel);
 int drm_dp_tunnel_update_state(struct drm_dp_tunnel *tunnel);
@@ -69,25 +70,25 @@ int drm_dp_tunnel_available_bw(const struct drm_dp_tunnel *tunnel);
 const char *drm_dp_tunnel_name(const struct drm_dp_tunnel *tunnel);
 
 struct drm_dp_tunnel_state *
-drm_dp_tunnel_atomic_get_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_state(struct drm_atomic_commit *state,
 			       struct drm_dp_tunnel *tunnel);
 
 struct drm_dp_tunnel_state *
-drm_dp_tunnel_atomic_get_old_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_old_state(struct drm_atomic_commit *state,
 				   const struct drm_dp_tunnel *tunnel);
 
 struct drm_dp_tunnel_state *
-drm_dp_tunnel_atomic_get_new_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_new_state(struct drm_atomic_commit *state,
 				   const struct drm_dp_tunnel *tunnel);
 
-int drm_dp_tunnel_atomic_set_stream_bw(struct drm_atomic_state *state,
+int drm_dp_tunnel_atomic_set_stream_bw(struct drm_atomic_commit *state,
 				       struct drm_dp_tunnel *tunnel,
 				       u8 stream_id, int bw);
-int drm_dp_tunnel_atomic_get_group_streams_in_state(struct drm_atomic_state *state,
+int drm_dp_tunnel_atomic_get_group_streams_in_state(struct drm_atomic_commit *state,
 						    const struct drm_dp_tunnel *tunnel,
 						    u32 *stream_mask);
 
-int drm_dp_tunnel_atomic_check_stream_bws(struct drm_atomic_state *state,
+int drm_dp_tunnel_atomic_check_stream_bws(struct drm_atomic_commit *state,
 					  u32 *failed_stream_mask);
 
 int drm_dp_tunnel_atomic_get_required_bw(const struct drm_dp_tunnel_state *tunnel_state);
@@ -140,6 +141,11 @@ static inline bool drm_dp_tunnel_bw_alloc_is_enabled(const struct drm_dp_tunnel
 	return false;
 }
 
+static inline bool drm_dp_tunnel_pr_optimization_supported(const struct drm_dp_tunnel *tunnel)
+{
+	return false;
+}
+
 static inline int
 drm_dp_tunnel_alloc_bw(struct drm_dp_tunnel *tunnel, int bw)
 {
@@ -192,21 +198,21 @@ drm_dp_tunnel_name(const struct drm_dp_tunnel *tunnel)
 }
 
 static inline struct drm_dp_tunnel_state *
-drm_dp_tunnel_atomic_get_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_state(struct drm_atomic_commit *state,
 			       struct drm_dp_tunnel *tunnel)
 {
 	return ERR_PTR(-EOPNOTSUPP);
 }
 
 static inline struct drm_dp_tunnel_state *
-drm_dp_tunnel_atomic_get_new_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_new_state(struct drm_atomic_commit *state,
 				   const struct drm_dp_tunnel *tunnel)
 {
 	return ERR_PTR(-EOPNOTSUPP);
 }
 
 static inline int
-drm_dp_tunnel_atomic_set_stream_bw(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_set_stream_bw(struct drm_atomic_commit *state,
 				   struct drm_dp_tunnel *tunnel,
 				   u8 stream_id, int bw)
 {
@@ -214,7 +220,7 @@ drm_dp_tunnel_atomic_set_stream_bw(struct drm_atomic_state *state,
 }
 
 static inline int
-drm_dp_tunnel_atomic_get_group_streams_in_state(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_get_group_streams_in_state(struct drm_atomic_commit *state,
 						const struct drm_dp_tunnel *tunnel,
 						u32 *stream_mask)
 {
@@ -222,7 +228,7 @@ drm_dp_tunnel_atomic_get_group_streams_in_state(struct drm_atomic_state *state,
 }
 
 static inline int
-drm_dp_tunnel_atomic_check_stream_bws(struct drm_atomic_state *state,
+drm_dp_tunnel_atomic_check_stream_bws(struct drm_atomic_commit *state,
 				      u32 *failed_stream_mask)
 {
 	return -EOPNOTSUPP;
diff --git a/include/drm/display/drm_hdmi_state_helper.h b/include/drm/display/drm_hdmi_state_helper.h
index 2349c0d0f00fb..0adc30c55ec91 100644
--- a/include/drm/display/drm_hdmi_state_helper.h
+++ b/include/drm/display/drm_hdmi_state_helper.h
@@ -3,7 +3,7 @@
 #ifndef DRM_HDMI_STATE_HELPER_H_
 #define DRM_HDMI_STATE_HELPER_H_
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_connector;
 struct drm_connector_state;
 struct drm_display_mode;
@@ -15,13 +15,13 @@ void __drm_atomic_helper_connector_hdmi_reset(struct drm_connector *connector,
 					      struct drm_connector_state *new_conn_state);
 
 int drm_atomic_helper_connector_hdmi_check(struct drm_connector *connector,
-					   struct drm_atomic_state *state);
+					   struct drm_atomic_commit *state);
 
 int drm_atomic_helper_connector_hdmi_update_audio_infoframe(struct drm_connector *connector,
 							    struct hdmi_audio_infoframe *frame);
 int drm_atomic_helper_connector_hdmi_clear_audio_infoframe(struct drm_connector *connector);
 int drm_atomic_helper_connector_hdmi_update_infoframes(struct drm_connector *connector,
-						       struct drm_atomic_state *state);
+						       struct drm_atomic_commit *state);
 void drm_atomic_helper_connector_hdmi_hotplug(struct drm_connector *connector,
 					      enum drm_connector_status status);
 void drm_atomic_helper_connector_hdmi_force(struct drm_connector *connector);
diff --git a/include/drm/drm_atomic.h b/include/drm/drm_atomic.h
index 8883290cd0140..1a80a8cdf2693 100644
--- a/include/drm/drm_atomic.h
+++ b/include/drm/drm_atomic.h
@@ -169,9 +169,9 @@ struct __drm_planes_state {
 	 * @state_to_destroy:
 	 *
 	 * Used to track the @drm_plane_state we will need to free when
-	 * tearing down the associated &drm_atomic_state in
+	 * tearing down the associated &drm_atomic_commit in
 	 * $drm_mode_config_funcs.atomic_state_clear or
-	 * drm_atomic_state_default_clear().
+	 * drm_atomic_commit_default_clear().
 	 *
 	 * Before a commit, and the call to
 	 * drm_atomic_helper_swap_state() in particular, it points to
@@ -190,9 +190,9 @@ struct __drm_crtcs_state {
 	 * @state_to_destroy:
 	 *
 	 * Used to track the @drm_crtc_state we will need to free when
-	 * tearing down the associated &drm_atomic_state in
+	 * tearing down the associated &drm_atomic_commit in
 	 * $drm_mode_config_funcs.atomic_state_clear or
-	 * drm_atomic_state_default_clear().
+	 * drm_atomic_commit_default_clear().
 	 *
 	 * Before a commit, and the call to
 	 * drm_atomic_helper_swap_state() in particular, it points to
@@ -224,9 +224,9 @@ struct __drm_connnectors_state {
 	 * @state_to_destroy:
 	 *
 	 * Used to track the @drm_connector_state we will need to free
-	 * when tearing down the associated &drm_atomic_state in
+	 * when tearing down the associated &drm_atomic_commit in
 	 * $drm_mode_config_funcs.atomic_state_clear or
-	 * drm_atomic_state_default_clear().
+	 * drm_atomic_commit_default_clear().
 	 *
 	 * Before a commit, and the call to
 	 * drm_atomic_helper_swap_state() in particular, it points to
@@ -402,9 +402,9 @@ struct drm_private_obj {
  */
 struct drm_private_state {
 	/**
-	 * @state: backpointer to global drm_atomic_state
+	 * @state: backpointer to global drm_atomic_commit
 	 */
-	struct drm_atomic_state *state;
+	struct drm_atomic_commit *state;
 
 	/**
 	 * @obj: backpointer to the private object
@@ -419,9 +419,9 @@ struct __drm_private_objs_state {
 	 * @state_to_destroy:
 	 *
 	 * Used to track the @drm_private_state we will need to free
-	 * when tearing down the associated &drm_atomic_state in
+	 * when tearing down the associated &drm_atomic_commit in
 	 * $drm_mode_config_funcs.atomic_state_clear or
-	 * drm_atomic_state_default_clear().
+	 * drm_atomic_commit_default_clear().
 	 *
 	 * Before a commit, and the call to
 	 * drm_atomic_helper_swap_state() in particular, it points to
@@ -434,7 +434,7 @@ struct __drm_private_objs_state {
 };
 
 /**
- * struct drm_atomic_state - Atomic commit structure
+ * struct drm_atomic_commit - Atomic commit structure
  *
  * This structure is the kernel counterpart of @drm_mode_atomic and represents
  * an atomic commit that transitions from an old to a new display state. It
@@ -446,25 +446,25 @@ struct __drm_private_objs_state {
  * drm_atomic_get_plane_state(), drm_atomic_get_connector_state(), or for
  * private state structures, drm_atomic_get_private_obj_state().
  *
- * NOTE: struct drm_atomic_state first started as a single collection of
+ * NOTE: struct drm_atomic_commit first started as a single collection of
  * entities state pointers (drm_plane_state, drm_crtc_state, etc.).
  *
  * At atomic_check time, you could get the state about to be committed
- * from drm_atomic_state, and the one currently running from the
+ * from drm_atomic_commit, and the one currently running from the
  * entities state pointer (drm_crtc.state, for example). After the call
  * to drm_atomic_helper_swap_state(), the entities state pointer would
- * contain the state previously checked, and the drm_atomic_state
+ * contain the state previously checked, and the drm_atomic_commit
  * structure the old state.
  *
- * Over time, and in order to avoid confusion, drm_atomic_state has
+ * Over time, and in order to avoid confusion, drm_atomic_commit has
  * grown to have both the old state (ie, the state we replace) and the
  * new state (ie, the state we want to apply). Those names are stable
  * during the commit process, which makes it easier to reason about.
  *
  * You can still find some traces of that evolution through some hooks
- * or callbacks taking a drm_atomic_state parameter called names like
+ * or callbacks taking a drm_atomic_commit parameter called names like
  * "old_state". This doesn't necessarily mean that the previous
- * drm_atomic_state is passed, but rather that this used to be the state
+ * drm_atomic_commit is passed, but rather that this used to be the state
  * collection we were replacing after drm_atomic_helper_swap_state(),
  * but the variable name was never updated.
  *
@@ -472,12 +472,12 @@ struct __drm_private_objs_state {
  * first started to pass the entity state only. However, it was pretty
  * cumbersome for drivers, and especially CRTCs, to retrieve the states
  * of other components. Thus, we switched to passing the whole
- * drm_atomic_state as a parameter to those operations. Similarly, the
+ * drm_atomic_commit as a parameter to those operations. Similarly, the
  * transition isn't complete yet, and one might still find atomic
- * operations taking a drm_atomic_state pointer, or a component state
+ * operations taking a drm_atomic_commit pointer, or a component state
  * pointer. The former is the preferred form.
  */
-struct drm_atomic_state {
+struct drm_atomic_commit {
 	/**
 	 * @ref:
 	 *
@@ -679,61 +679,61 @@ static inline void drm_crtc_commit_put(struct drm_crtc_commit *commit)
 
 int drm_crtc_commit_wait(struct drm_crtc_commit *commit);
 
-struct drm_atomic_state * __must_check
-drm_atomic_state_alloc(struct drm_device *dev);
-void drm_atomic_state_clear(struct drm_atomic_state *state);
+struct drm_atomic_commit * __must_check
+drm_atomic_commit_alloc(struct drm_device *dev);
+void drm_atomic_commit_clear(struct drm_atomic_commit *state);
 
 /**
- * drm_atomic_state_get - acquire a reference to the atomic state
+ * drm_atomic_commit_get - acquire a reference to the atomic state
  * @state: The atomic state
  *
  * Returns a new reference to the @state
  */
-static inline struct drm_atomic_state *
-drm_atomic_state_get(struct drm_atomic_state *state)
+static inline struct drm_atomic_commit *
+drm_atomic_commit_get(struct drm_atomic_commit *state)
 {
 	kref_get(&state->ref);
 	return state;
 }
 
-void __drm_atomic_state_free(struct kref *ref);
+void __drm_atomic_commit_free(struct kref *ref);
 
 /**
- * drm_atomic_state_put - release a reference to the atomic state
+ * drm_atomic_commit_put - release a reference to the atomic state
  * @state: The atomic state
  *
  * This releases a reference to @state which is freed after removing the
  * final reference. No locking required and callable from any context.
  */
-static inline void drm_atomic_state_put(struct drm_atomic_state *state)
+static inline void drm_atomic_commit_put(struct drm_atomic_commit *state)
 {
-	kref_put(&state->ref, __drm_atomic_state_free);
+	kref_put(&state->ref, __drm_atomic_commit_free);
 }
 
 int  __must_check
-drm_atomic_state_init(struct drm_device *dev, struct drm_atomic_state *state);
-void drm_atomic_state_default_clear(struct drm_atomic_state *state);
-void drm_atomic_state_default_release(struct drm_atomic_state *state);
+drm_atomic_commit_init(struct drm_device *dev, struct drm_atomic_commit *state);
+void drm_atomic_commit_default_clear(struct drm_atomic_commit *state);
+void drm_atomic_commit_default_release(struct drm_atomic_commit *state);
 
 struct drm_crtc_state * __must_check
-drm_atomic_get_crtc_state(struct drm_atomic_state *state,
+drm_atomic_get_crtc_state(struct drm_atomic_commit *state,
 			  struct drm_crtc *crtc);
 struct drm_plane_state * __must_check
-drm_atomic_get_plane_state(struct drm_atomic_state *state,
+drm_atomic_get_plane_state(struct drm_atomic_commit *state,
 			   struct drm_plane *plane);
 struct drm_colorop_state *
-drm_atomic_get_colorop_state(struct drm_atomic_state *state,
+drm_atomic_get_colorop_state(struct drm_atomic_commit *state,
 			     struct drm_colorop *colorop);
 
 struct drm_colorop_state *
-drm_atomic_get_old_colorop_state(struct drm_atomic_state *state,
+drm_atomic_get_old_colorop_state(struct drm_atomic_commit *state,
 				 struct drm_colorop *colorop);
 struct drm_colorop_state *
-drm_atomic_get_new_colorop_state(struct drm_atomic_state *state,
+drm_atomic_get_new_colorop_state(struct drm_atomic_commit *state,
 				 struct drm_colorop *colorop);
 
 struct drm_connector_state * __must_check
-drm_atomic_get_connector_state(struct drm_atomic_state *state,
+drm_atomic_get_connector_state(struct drm_atomic_commit *state,
 			       struct drm_connector *connector);
 
 int drm_atomic_private_obj_init(struct drm_device *dev,
@@ -742,30 +742,30 @@ int drm_atomic_private_obj_init(struct drm_device *dev,
 void drm_atomic_private_obj_fini(struct drm_private_obj *obj);
 
 struct drm_private_state * __must_check
-drm_atomic_get_private_obj_state(struct drm_atomic_state *state,
+drm_atomic_get_private_obj_state(struct drm_atomic_commit *state,
 				 struct drm_private_obj *obj);
 struct drm_private_state *
-drm_atomic_get_old_private_obj_state(const struct drm_atomic_state *state,
+drm_atomic_get_old_private_obj_state(const struct drm_atomic_commit *state,
 				     struct drm_private_obj *obj);
 struct drm_private_state *
-drm_atomic_get_new_private_obj_state(const struct drm_atomic_state *state,
+drm_atomic_get_new_private_obj_state(const struct drm_atomic_commit *state,
 				     struct drm_private_obj *obj);
 
 struct drm_connector *
-drm_atomic_get_old_connector_for_encoder(const struct drm_atomic_state *state,
+drm_atomic_get_old_connector_for_encoder(const struct drm_atomic_commit *state,
 					 struct drm_encoder *encoder);
 struct drm_connector *
-drm_atomic_get_new_connector_for_encoder(const struct drm_atomic_state *state,
+drm_atomic_get_new_connector_for_encoder(const struct drm_atomic_commit *state,
 					 struct drm_encoder *encoder);
 struct drm_connector *
 drm_atomic_get_connector_for_encoder(const struct drm_encoder *encoder,
 				     struct drm_modeset_acquire_ctx *ctx);
 
 struct drm_crtc *
-drm_atomic_get_old_crtc_for_encoder(struct drm_atomic_state *state,
+drm_atomic_get_old_crtc_for_encoder(struct drm_atomic_commit *state,
 					 struct drm_encoder *encoder);
 struct drm_crtc *
-drm_atomic_get_new_crtc_for_encoder(struct drm_atomic_state *state,
+drm_atomic_get_new_crtc_for_encoder(struct drm_atomic_commit *state,
 					 struct drm_encoder *encoder);
 
 /**
@@ -777,7 +777,7 @@ drm_atomic_get_new_crtc_for_encoder(struct drm_atomic_state *state,
  * NULL if the CRTC is not part of the global atomic state.
  */
 static inline struct drm_crtc_state *
-drm_atomic_get_old_crtc_state(const struct drm_atomic_state *state,
+drm_atomic_get_old_crtc_state(const struct drm_atomic_commit *state,
 			      struct drm_crtc *crtc)
 {
 	return state->crtcs[drm_crtc_index(crtc)].old_state;
@@ -791,7 +791,7 @@ drm_atomic_get_old_crtc_state(const struct drm_atomic_state *state,
  * NULL if the CRTC is not part of the global atomic state.
  */
 static inline struct drm_crtc_state *
-drm_atomic_get_new_crtc_state(const struct drm_atomic_state *state,
+drm_atomic_get_new_crtc_state(const struct drm_atomic_commit *state,
 			      struct drm_crtc *crtc)
 {
 	return state->crtcs[drm_crtc_index(crtc)].new_state;
@@ -806,7 +806,7 @@ drm_atomic_get_new_crtc_state(const struct drm_atomic_state *state,
  * NULL if the plane is not part of the global atomic state.
  */
 static inline struct drm_plane_state *
-drm_atomic_get_old_plane_state(const struct drm_atomic_state *state,
+drm_atomic_get_old_plane_state(const struct drm_atomic_commit *state,
 			       struct drm_plane *plane)
 {
 	return state->planes[drm_plane_index(plane)].old_state;
@@ -821,7 +821,7 @@ drm_atomic_get_old_plane_state(const struct drm_atomic_state *state,
  * NULL if the plane is not part of the global atomic state.
  */
 static inline struct drm_plane_state *
-drm_atomic_get_new_plane_state(const struct drm_atomic_state *state,
+drm_atomic_get_new_plane_state(const struct drm_atomic_commit *state,
 			       struct drm_plane *plane)
 {
 	return state->planes[drm_plane_index(plane)].new_state;
@@ -836,7 +836,7 @@ drm_atomic_get_new_plane_state(const struct drm_atomic_state *state,
  * or NULL if the connector is not part of the global atomic state.
  */
 static inline struct drm_connector_state *
-drm_atomic_get_old_connector_state(const struct drm_atomic_state *state,
+drm_atomic_get_old_connector_state(const struct drm_atomic_commit *state,
 				   struct drm_connector *connector)
 {
 	int index = drm_connector_index(connector);
@@ -856,7 +856,7 @@ drm_atomic_get_old_connector_state(const struct drm_atomic_state *state,
  * or NULL if the connector is not part of the global atomic state.
  */
 static inline struct drm_connector_state *
-drm_atomic_get_new_connector_state(const struct drm_atomic_state *state,
+drm_atomic_get_new_connector_state(const struct drm_atomic_commit *state,
 				   struct drm_connector *connector)
 {
 	int index = drm_connector_index(connector);
@@ -894,7 +894,7 @@ drm_atomic_get_new_connector_state(const struct drm_atomic_state *state,
  * Read-only pointer to the current plane state.
  */
 static inline const struct drm_plane_state *
-__drm_atomic_get_current_plane_state(const struct drm_atomic_state *state,
+__drm_atomic_get_current_plane_state(const struct drm_atomic_commit *state,
 				     struct drm_plane *plane)
 {
 	struct drm_plane_state *plane_state;
@@ -910,27 +910,27 @@ __drm_atomic_get_current_plane_state(const struct drm_atomic_state *state,
 }
 
 int __must_check
-drm_atomic_add_encoder_bridges(struct drm_atomic_state *state,
+drm_atomic_add_encoder_bridges(struct drm_atomic_commit *state,
 			       struct drm_encoder *encoder);
 int __must_check
-drm_atomic_add_affected_connectors(struct drm_atomic_state *state,
+drm_atomic_add_affected_connectors(struct drm_atomic_commit *state,
 				   struct drm_crtc *crtc);
 int __must_check
-drm_atomic_add_affected_planes(struct drm_atomic_state *state,
+drm_atomic_add_affected_planes(struct drm_atomic_commit *state,
 			       struct drm_crtc *crtc);
 int __must_check
-drm_atomic_add_affected_colorops(struct drm_atomic_state *state,
+drm_atomic_add_affected_colorops(struct drm_atomic_commit *state,
 				 struct drm_plane *plane);
 
-int __must_check drm_atomic_check_only(struct drm_atomic_state *state);
-int __must_check drm_atomic_commit(struct drm_atomic_state *state);
-int __must_check drm_atomic_nonblocking_commit(struct drm_atomic_state *state);
+int __must_check drm_atomic_check_only(struct drm_atomic_commit *state);
+int __must_check drm_atomic_commit(struct drm_atomic_commit *state);
+int __must_check drm_atomic_nonblocking_commit(struct drm_atomic_commit *state);
 
 void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_oldnew_connector_in_state - iterate over all connectors in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @connector: &struct drm_connector iteration cursor
  * @old_connector_state: &struct drm_connector_state iteration cursor for the
  * 	old state
@@ -954,7 +954,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_old_connector_in_state - iterate over all connectors in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @connector: &struct drm_connector iteration cursor
  * @old_connector_state: &struct drm_connector_state iteration cursor for the
  * 	old state
@@ -975,7 +975,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_new_connector_in_state - iterate over all connectors in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @connector: &struct drm_connector iteration cursor
  * @new_connector_state: &struct drm_connector_state iteration cursor for the
  * 	new state
@@ -997,7 +997,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_oldnew_crtc_in_state - iterate over all CRTCs in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @crtc: &struct drm_crtc iteration cursor
  * @old_crtc_state: &struct drm_crtc_state iteration cursor for the old state
  * @new_crtc_state: &struct drm_crtc_state iteration cursor for the new state
@@ -1021,7 +1021,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_old_crtc_in_state - iterate over all CRTCs in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @crtc: &struct drm_crtc iteration cursor
  * @old_crtc_state: &struct drm_crtc_state iteration cursor for the old state
  * @__i: int iteration cursor, for macro-internal use
@@ -1041,7 +1041,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_new_crtc_in_state - iterate over all CRTCs in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @crtc: &struct drm_crtc iteration cursor
  * @new_crtc_state: &struct drm_crtc_state iteration cursor for the new state
  * @__i: int iteration cursor, for macro-internal use
@@ -1062,7 +1062,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_oldnew_colorop_in_state - iterate over all colorops in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @colorop: &struct drm_colorop iteration cursor
  * @old_colorop_state: &struct drm_colorop_state iteration cursor for the old state
  * @new_colorop_state: &struct drm_colorop_state iteration cursor for the new state
@@ -1085,7 +1085,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_new_colorop_in_state - iterate over all colorops in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @colorop: &struct drm_colorop iteration cursor
  * @new_colorop_state: &struct drm_colorop_state iteration cursor for the new state
  * @__i: int iteration cursor, for macro-internal use
@@ -1106,7 +1106,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_oldnew_plane_in_state - iterate over all planes in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @plane: &struct drm_plane iteration cursor
  * @old_plane_state: &struct drm_plane_state iteration cursor for the old state
  * @new_plane_state: &struct drm_plane_state iteration cursor for the new state
@@ -1129,7 +1129,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 /**
  * for_each_oldnew_plane_in_state_reverse - iterate over all planes in an atomic
  * update in reverse order
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @plane: &struct drm_plane iteration cursor
  * @old_plane_state: &struct drm_plane_state iteration cursor for the old state
  * @new_plane_state: &struct drm_plane_state iteration cursor for the new state
@@ -1151,7 +1151,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 /**
  * for_each_new_plane_in_state_reverse - other than only tracking new state,
  * it's the same as for_each_oldnew_plane_in_state_reverse
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @plane: &struct drm_plane iteration cursor
  * @new_plane_state: &struct drm_plane_state iteration cursor for the new state
  * @__i: int iteration cursor, for macro-internal use
@@ -1166,7 +1166,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_old_plane_in_state - iterate over all planes in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @plane: &struct drm_plane iteration cursor
  * @old_plane_state: &struct drm_plane_state iteration cursor for the old state
  * @__i: int iteration cursor, for macro-internal use
@@ -1184,7 +1184,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 			      (old_plane_state) = (__state)->planes[__i].old_state, 1))
 /**
  * for_each_new_plane_in_state - iterate over all planes in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @plane: &struct drm_plane iteration cursor
  * @new_plane_state: &struct drm_plane_state iteration cursor for the new state
  * @__i: int iteration cursor, for macro-internal use
@@ -1205,7 +1205,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_oldnew_private_obj_in_state - iterate over all private objects in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @obj: &struct drm_private_obj iteration cursor
  * @old_obj_state: &struct drm_private_state iteration cursor for the old state
  * @new_obj_state: &struct drm_private_state iteration cursor for the new state
@@ -1225,7 +1225,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_old_private_obj_in_state - iterate over all private objects in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @obj: &struct drm_private_obj iteration cursor
  * @old_obj_state: &struct drm_private_state iteration cursor for the old state
  * @__i: int iteration cursor, for macro-internal use
@@ -1243,7 +1243,7 @@ void drm_state_dump(struct drm_device *dev, struct drm_printer *p);
 
 /**
  * for_each_new_private_obj_in_state - iterate over all private objects in an atomic update
- * @__state: &struct drm_atomic_state pointer
+ * @__state: &struct drm_atomic_commit pointer
  * @obj: &struct drm_private_obj iteration cursor
  * @new_obj_state: &struct drm_private_state iteration cursor for the new state
  * @__i: int iteration cursor, for macro-internal use
@@ -1365,13 +1365,13 @@ drm_priv_to_bridge_state(struct drm_private_state *priv)
 }
 
 struct drm_bridge_state *
-drm_atomic_get_bridge_state(struct drm_atomic_state *state,
+drm_atomic_get_bridge_state(struct drm_atomic_commit *state,
 			    struct drm_bridge *bridge);
 struct drm_bridge_state *
-drm_atomic_get_old_bridge_state(const struct drm_atomic_state *state,
+drm_atomic_get_old_bridge_state(const struct drm_atomic_commit *state,
 				struct drm_bridge *bridge);
 struct drm_bridge_state *
-drm_atomic_get_new_bridge_state(const struct drm_atomic_state *state,
+drm_atomic_get_new_bridge_state(const struct drm_atomic_commit *state,
 				struct drm_bridge *bridge);
 
 #endif /* DRM_ATOMIC_H_ */
diff --git a/include/drm/drm_atomic_helper.h b/include/drm/drm_atomic_helper.h
index e154ee4f0696c..b84152810abb9 100644
--- a/include/drm/drm_atomic_helper.h
+++ b/include/drm/drm_atomic_helper.h
@@ -43,14 +43,14 @@
  */
 #define DRM_PLANE_NO_SCALING (1<<16)
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_private_obj;
 struct drm_private_state;
 
 int drm_atomic_helper_check_modeset(struct drm_device *dev,
-				struct drm_atomic_state *state);
+				struct drm_atomic_commit *state);
 int drm_atomic_helper_check_wb_connector_state(struct drm_connector *connector,
-					       struct drm_atomic_state *state);
+					       struct drm_atomic_commit *state);
 int drm_atomic_helper_check_plane_state(struct drm_plane_state *plane_state,
 					const struct drm_crtc_state *crtc_state,
 					int min_scale,
@@ -58,92 +58,92 @@ int drm_atomic_helper_check_plane_state(struct drm_plane_state *plane_state,
 					bool can_position,
 					bool can_update_disabled);
 int drm_atomic_helper_check_planes(struct drm_device *dev,
-			       struct drm_atomic_state *state);
+			       struct drm_atomic_commit *state);
 int drm_atomic_helper_check_crtc_primary_plane(struct drm_crtc_state *crtc_state);
 void drm_atomic_helper_commit_encoder_bridge_disable(struct drm_device *dev,
-						     struct drm_atomic_state *state);
+						     struct drm_atomic_commit *state);
 void drm_atomic_helper_commit_crtc_disable(struct drm_device *dev,
-					   struct drm_atomic_state *state);
+					   struct drm_atomic_commit *state);
 void drm_atomic_helper_commit_encoder_bridge_post_disable(struct drm_device *dev,
-							  struct drm_atomic_state *state);
+							  struct drm_atomic_commit *state);
 int drm_atomic_helper_check(struct drm_device *dev,
-			    struct drm_atomic_state *state);
-void drm_atomic_helper_commit_tail(struct drm_atomic_state *state);
-void drm_atomic_helper_commit_tail_rpm(struct drm_atomic_state *state);
+			    struct drm_atomic_commit *state);
+void drm_atomic_helper_commit_tail(struct drm_atomic_commit *state);
+void drm_atomic_helper_commit_tail_rpm(struct drm_atomic_commit *state);
 int drm_atomic_helper_commit(struct drm_device *dev,
-			     struct drm_atomic_state *state,
+			     struct drm_atomic_commit *state,
 			     bool nonblock);
 int drm_atomic_helper_async_check(struct drm_device *dev,
-				  struct drm_atomic_state *state);
+				  struct drm_atomic_commit *state);
 void drm_atomic_helper_async_commit(struct drm_device *dev,
-				    struct drm_atomic_state *state);
+				    struct drm_atomic_commit *state);
 
 int drm_atomic_helper_wait_for_fences(struct drm_device *dev,
-					struct drm_atomic_state *state,
+					struct drm_atomic_commit *state,
 					bool pre_swap);
 
 void drm_atomic_helper_wait_for_vblanks(struct drm_device *dev,
-					struct drm_atomic_state *old_state);
+					struct drm_atomic_commit *old_state);
 
 void drm_atomic_helper_wait_for_flip_done(struct drm_device *dev,
-					  struct drm_atomic_state *old_state);
+					  struct drm_atomic_commit *old_state);
 
 void
 drm_atomic_helper_update_legacy_modeset_state(struct drm_device *dev,
-					      struct drm_atomic_state *old_state);
+					      struct drm_atomic_commit *old_state);
 
 void
-drm_atomic_helper_calc_timestamping_constants(struct drm_atomic_state *state);
+drm_atomic_helper_calc_timestamping_constants(struct drm_atomic_commit *state);
 
 void drm_atomic_helper_commit_crtc_set_mode(struct drm_device *dev,
-					    struct drm_atomic_state *state);
+					    struct drm_atomic_commit *state);
 
 void drm_atomic_helper_commit_modeset_disables(struct drm_device *dev,
-					       struct drm_atomic_state *state);
+					       struct drm_atomic_commit *state);
 
 void drm_atomic_helper_commit_writebacks(struct drm_device *dev,
-					 struct drm_atomic_state *state);
+					 struct drm_atomic_commit *state);
 
 void drm_atomic_helper_commit_encoder_bridge_pre_enable(struct drm_device *dev,
-							struct drm_atomic_state *state);
+							struct drm_atomic_commit *state);
 
 void drm_atomic_helper_commit_crtc_enable(struct drm_device *dev,
-					  struct drm_atomic_state *state);
+					  struct drm_atomic_commit *state);
 
 void drm_atomic_helper_commit_encoder_bridge_enable(struct drm_device *dev,
-						    struct drm_atomic_state *state);
+						    struct drm_atomic_commit *state);
 
 void drm_atomic_helper_commit_modeset_enables(struct drm_device *dev,
-					  struct drm_atomic_state *old_state);
+					  struct drm_atomic_commit *old_state);
 
 int drm_atomic_helper_prepare_planes(struct drm_device *dev,
-				     struct drm_atomic_state *state);
+				     struct drm_atomic_commit *state);
 void drm_atomic_helper_unprepare_planes(struct drm_device *dev,
-					struct drm_atomic_state *state);
+					struct drm_atomic_commit *state);
 
 #define DRM_PLANE_COMMIT_ACTIVE_ONLY			BIT(0)
 #define DRM_PLANE_COMMIT_NO_DISABLE_AFTER_MODESET	BIT(1)
 
 void drm_atomic_helper_commit_planes(struct drm_device *dev,
-				     struct drm_atomic_state *state,
+				     struct drm_atomic_commit *state,
 				     uint32_t flags);
 void drm_atomic_helper_cleanup_planes(struct drm_device *dev,
-				      struct drm_atomic_state *old_state);
+				      struct drm_atomic_commit *old_state);
 void drm_atomic_helper_commit_planes_on_crtc(struct drm_crtc_state *old_crtc_state);
 void
 drm_atomic_helper_disable_planes_on_crtc(struct drm_crtc_state *old_crtc_state,
 					 bool atomic);
 
-int __must_check drm_atomic_helper_swap_state(struct drm_atomic_state *state,
+int __must_check drm_atomic_helper_swap_state(struct drm_atomic_commit *state,
 					      bool stall);
 
 /* nonblocking commit helpers */
-int drm_atomic_helper_setup_commit(struct drm_atomic_state *state,
+int drm_atomic_helper_setup_commit(struct drm_atomic_commit *state,
 				   bool nonblock);
-void drm_atomic_helper_wait_for_dependencies(struct drm_atomic_state *state);
-void drm_atomic_helper_fake_vblank(struct drm_atomic_state *state);
-void drm_atomic_helper_commit_hw_done(struct drm_atomic_state *state);
-void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *state);
+void drm_atomic_helper_wait_for_dependencies(struct drm_atomic_commit *state);
+void drm_atomic_helper_fake_vblank(struct drm_atomic_commit *state);
+void drm_atomic_helper_commit_hw_done(struct drm_atomic_commit *state);
+void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_commit *state);
 
 /* implementations for legacy interfaces */
 int drm_atomic_helper_update_plane(struct drm_plane *plane,
@@ -164,14 +164,14 @@ int drm_atomic_helper_disable_all(struct drm_device *dev,
 int drm_atomic_helper_reset_crtc(struct drm_crtc *crtc,
 				 struct drm_modeset_acquire_ctx *ctx);
 void drm_atomic_helper_shutdown(struct drm_device *dev);
-struct drm_atomic_state *
+struct drm_atomic_commit *
 drm_atomic_helper_duplicate_state(struct drm_device *dev,
 				  struct drm_modeset_acquire_ctx *ctx);
-struct drm_atomic_state *drm_atomic_helper_suspend(struct drm_device *dev);
-int drm_atomic_helper_commit_duplicated_state(struct drm_atomic_state *state,
+struct drm_atomic_commit *drm_atomic_helper_suspend(struct drm_device *dev);
+int drm_atomic_helper_commit_duplicated_state(struct drm_atomic_commit *state,
 					      struct drm_modeset_acquire_ctx *ctx);
 int drm_atomic_helper_resume(struct drm_device *dev,
-			     struct drm_atomic_state *state);
+			     struct drm_atomic_commit *state);
 
 int drm_atomic_helper_page_flip(struct drm_crtc *crtc,
 				struct drm_framebuffer *fb,
diff --git a/include/drm/drm_atomic_state_helper.h b/include/drm/drm_atomic_state_helper.h
index 900672c6ea90b..61a3b38ad49fd 100644
--- a/include/drm/drm_atomic_state_helper.h
+++ b/include/drm/drm_atomic_state_helper.h
@@ -26,7 +26,7 @@
 
 #include <linux/types.h>
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_bridge;
 struct drm_bridge_state;
 struct drm_crtc;
@@ -73,7 +73,7 @@ void __drm_atomic_helper_connector_reset(struct drm_connector *connector,
 void drm_atomic_helper_connector_reset(struct drm_connector *connector);
 void drm_atomic_helper_connector_tv_reset(struct drm_connector *connector);
 int drm_atomic_helper_connector_tv_check(struct drm_connector *connector,
-					 struct drm_atomic_state *state);
+					 struct drm_atomic_commit *state);
 void drm_atomic_helper_connector_tv_margins_reset(struct drm_connector *connector);
 void
 __drm_atomic_helper_connector_duplicate_state(struct drm_connector *connector,
diff --git a/include/drm/drm_blend.h b/include/drm/drm_blend.h
index c7e888767c81c..bebbb77a8f214 100644
--- a/include/drm/drm_blend.h
+++ b/include/drm/drm_blend.h
@@ -31,7 +31,7 @@
 #define DRM_MODE_BLEND_COVERAGE		1
 #define DRM_MODE_BLEND_PIXEL_NONE	2
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_crtc;
 struct drm_device;
 struct drm_plane;
@@ -56,7 +56,7 @@ int drm_plane_create_zpos_property(struct drm_plane *plane,
 int drm_plane_create_zpos_immutable_property(struct drm_plane *plane,
 					     unsigned int zpos);
 int drm_atomic_normalize_zpos(struct drm_device *dev,
-			      struct drm_atomic_state *state);
+			      struct drm_atomic_commit *state);
 int drm_plane_create_blend_mode_property(struct drm_plane *plane,
 					 unsigned int supported_modes);
 void drm_crtc_attach_background_color_property(struct drm_crtc *crtc);
diff --git a/include/drm/drm_bridge.h b/include/drm/drm_bridge.h
index a8d67bd9ee505..4ba3a5deef9a6 100644
--- a/include/drm/drm_bridge.h
+++ b/include/drm/drm_bridge.h
@@ -317,7 +317,7 @@ struct drm_bridge_funcs {
 	 * The @atomic_pre_enable callback is optional.
 	 */
 	void (*atomic_pre_enable)(struct drm_bridge *bridge,
-				  struct drm_atomic_state *state);
+				  struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_enable:
@@ -337,7 +337,7 @@ struct drm_bridge_funcs {
 	 * The @atomic_enable callback is optional.
 	 */
 	void (*atomic_enable)(struct drm_bridge *bridge,
-			      struct drm_atomic_state *state);
+			      struct drm_atomic_commit *state);
 	/**
 	 * @atomic_disable:
 	 *
@@ -354,7 +354,7 @@ struct drm_bridge_funcs {
 	 * The @atomic_disable callback is optional.
 	 */
 	void (*atomic_disable)(struct drm_bridge *bridge,
-			       struct drm_atomic_state *state);
+			       struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_post_disable:
@@ -373,7 +373,7 @@ struct drm_bridge_funcs {
 	 * The @atomic_post_disable callback is optional.
 	 */
 	void (*atomic_post_disable)(struct drm_bridge *bridge,
-				    struct drm_atomic_state *state);
+				    struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_duplicate_state:
@@ -1327,6 +1327,8 @@ int drm_bridge_attach(struct drm_encoder *encoder, struct drm_bridge *bridge,
 #ifdef CONFIG_OF
 struct drm_bridge *of_drm_find_and_get_bridge(struct device_node *np);
 struct drm_bridge *of_drm_find_bridge(struct device_node *np);
+struct drm_bridge *of_drm_get_bridge_by_endpoint(const struct device_node *np,
+						 int port, int endpoint);
 #else
 static inline struct drm_bridge *of_drm_find_and_get_bridge(struct device_node *np)
 {
@@ -1336,6 +1338,11 @@ static inline struct drm_bridge *of_drm_find_bridge(struct device_node *np)
 {
 	return NULL;
 }
+static inline struct drm_bridge *of_drm_get_bridge_by_endpoint(const struct device_node *np,
+							       int port, int endpoint)
+{
+	return ERR_PTR(-ENODEV);
+}
 #endif
 
 static inline bool drm_bridge_is_last(struct drm_bridge *bridge)
@@ -1457,26 +1464,37 @@ drm_bridge_chain_get_last_bridge(struct drm_encoder *encoder)
 						      struct drm_bridge, chain_node));
 }
 
-/**
- * drm_bridge_get_next_bridge_and_put - Get the next bridge in the chain
- *                                      and put the previous
- * @bridge: bridge object
- *
- * Same as drm_bridge_get_next_bridge() but additionally puts the @bridge.
- *
- * RETURNS:
- * the next bridge in the chain after @bridge, or NULL if @bridge is the last.
- */
-static inline struct drm_bridge *
-drm_bridge_get_next_bridge_and_put(struct drm_bridge *bridge)
+/* Internal to drm_for_each_bridge_in_chain*() */
+static inline struct drm_bridge *__drm_for_each_bridge_in_chain_next(struct drm_bridge *bridge)
 {
 	struct drm_bridge *next = drm_bridge_get_next_bridge(bridge);
 
+	if (!next)
+		mutex_unlock(&bridge->encoder->bridge_chain_mutex);
+
 	drm_bridge_put(bridge);
 
 	return next;
 }
 
+/* Internal to drm_for_each_bridge_in_chain*() */
+DEFINE_FREE(__drm_for_each_bridge_in_chain_cleanup, struct drm_bridge *,
+	if (_T) { mutex_unlock(&_T->encoder->bridge_chain_mutex); drm_bridge_put(_T); })
+
+/* Internal to drm_for_each_bridge_in_chain_scoped() */
+static inline struct drm_bridge *
+__drm_for_each_bridge_in_chain_scoped_start(struct drm_encoder *encoder)
+{
+	mutex_lock(&encoder->bridge_chain_mutex);
+
+	struct drm_bridge *bridge = drm_bridge_chain_get_first_bridge(encoder);
+
+	if (!bridge)
+		mutex_unlock(&encoder->bridge_chain_mutex);
+
+	return bridge;
+}
+
 /**
  * drm_for_each_bridge_in_chain_scoped - iterate over all bridges attached
  *                                       to an encoder
@@ -1486,14 +1504,24 @@ drm_bridge_get_next_bridge_and_put(struct drm_bridge *bridge)
  *
  * Iterate over all bridges present in the bridge chain attached to @encoder.
  *
- * Automatically gets/puts the bridge reference while iterating, and puts
- * the reference even if returning or breaking in the middle of the loop.
+ * Automatically gets/puts the bridge reference while iterating and locks
+ * the encoder chain mutex to prevent chain modifications while iterating.
  */
-#define drm_for_each_bridge_in_chain_scoped(encoder, bridge)		\
-	for (struct drm_bridge *bridge __free(drm_bridge_put) =		\
-	     drm_bridge_chain_get_first_bridge(encoder);		\
-	     bridge;							\
-	     bridge = drm_bridge_get_next_bridge_and_put(bridge))
+#define drm_for_each_bridge_in_chain_scoped(encoder, bridge)				\
+	for (struct drm_bridge *bridge __free(__drm_for_each_bridge_in_chain_cleanup) =	\
+		__drm_for_each_bridge_in_chain_scoped_start((encoder));			\
+	     bridge;									\
+	     bridge = __drm_for_each_bridge_in_chain_next(bridge))			\
+
+/* Internal to drm_for_each_bridge_in_chain_from() */
+static inline struct drm_bridge *
+__drm_for_each_bridge_in_chain_from_start(struct drm_bridge *bridge)
+{
+	drm_bridge_get(bridge);
+	mutex_lock(&bridge->encoder->bridge_chain_mutex);
+
+	return bridge;
+}
 
 /**
  * drm_for_each_bridge_in_chain_from - iterate over all bridges starting
@@ -1505,14 +1533,14 @@ drm_bridge_get_next_bridge_and_put(struct drm_bridge *bridge)
  * Iterate over all bridges in the encoder chain starting from
  * @first_bridge, included.
  *
- * Automatically gets/puts the bridge reference while iterating, and puts
- * the reference even if returning or breaking in the middle of the loop.
+ * Automatically gets/puts the bridge reference while iterating and locks
+ * the encoder chain mutex to prevent chain modifications while iterating.
  */
-#define drm_for_each_bridge_in_chain_from(first_bridge, bridge)		\
-	for (struct drm_bridge *bridge __free(drm_bridge_put) =		\
-		     drm_bridge_get(first_bridge);			\
-	     bridge;							\
-	     bridge = drm_bridge_get_next_bridge_and_put(bridge))
+#define drm_for_each_bridge_in_chain_from(first_bridge, bridge)				\
+	for (struct drm_bridge *bridge __free(__drm_for_each_bridge_in_chain_cleanup) =	\
+		__drm_for_each_bridge_in_chain_from_start(first_bridge);		\
+	     bridge;									\
+	     bridge = __drm_for_each_bridge_in_chain_next(bridge))			\
 
 enum drm_mode_status
 drm_bridge_chain_mode_valid(struct drm_bridge *bridge,
@@ -1526,13 +1554,13 @@ int drm_atomic_bridge_chain_check(struct drm_bridge *bridge,
 				  struct drm_crtc_state *crtc_state,
 				  struct drm_connector_state *conn_state);
 void drm_atomic_bridge_chain_disable(struct drm_bridge *bridge,
-				     struct drm_atomic_state *state);
+				     struct drm_atomic_commit *state);
 void drm_atomic_bridge_chain_post_disable(struct drm_bridge *bridge,
-					  struct drm_atomic_state *state);
+					  struct drm_atomic_commit *state);
 void drm_atomic_bridge_chain_pre_enable(struct drm_bridge *bridge,
-					struct drm_atomic_state *state);
+					struct drm_atomic_commit *state);
 void drm_atomic_bridge_chain_enable(struct drm_bridge *bridge,
-				    struct drm_atomic_state *state);
+				    struct drm_atomic_commit *state);
 
 u32 *
 drm_atomic_helper_bridge_propagate_bus_fmt(struct drm_bridge *bridge,
diff --git a/include/drm/drm_colorop.h b/include/drm/drm_colorop.h
index bd082854ca74c..c873199c60da9 100644
--- a/include/drm/drm_colorop.h
+++ b/include/drm/drm_colorop.h
@@ -183,8 +183,8 @@ struct drm_colorop_state {
 	 */
 	struct drm_property_blob *data;
 
-	/** @state: backpointer to global drm_atomic_state */
-	struct drm_atomic_state *state;
+	/** @state: backpointer to global drm_atomic_commit */
+	struct drm_atomic_commit *state;
 };
 
 /**
diff --git a/include/drm/drm_connector.h b/include/drm/drm_connector.h
index f83f28cae2075..5ad62c207d009 100644
--- a/include/drm/drm_connector.h
+++ b/include/drm/drm_connector.h
@@ -1098,8 +1098,8 @@ struct drm_connector_state {
 	 */
 	enum drm_link_status link_status;
 
-	/** @state: backpointer to global drm_atomic_state */
-	struct drm_atomic_state *state;
+	/** @state: backpointer to global drm_atomic_commit */
+	struct drm_atomic_commit *state;
 
 	/**
 	 * @commit: Tracks the pending commit to prevent use-after-free conditions.
@@ -2344,7 +2344,7 @@ struct drm_connector {
 	 *
 	 * This is protected by &drm_mode_config.connection_mutex. Note that
 	 * nonblocking atomic commits access the current connector state without
-	 * taking locks. Either by going through the &struct drm_atomic_state
+	 * taking locks. Either by going through the &struct drm_atomic_commit
 	 * pointers, see for_each_oldnew_connector_in_state(),
 	 * for_each_old_connector_in_state() and
 	 * for_each_new_connector_in_state(). Or through careful ordering of
@@ -2556,7 +2556,7 @@ int drm_connector_attach_vrr_capable_property(
 void drm_connector_attach_panel_type_property(struct drm_connector *connector);
 int drm_connector_attach_broadcast_rgb_property(struct drm_connector *connector);
 int drm_connector_attach_colorspace_property(struct drm_connector *connector);
-int drm_connector_attach_hdr_output_metadata_property(struct drm_connector *connector);
+void drm_connector_attach_hdr_output_metadata_property(struct drm_connector *connector);
 bool drm_connector_atomic_hdr_metadata_equal(struct drm_connector_state *old_state,
 					     struct drm_connector_state *new_state);
 int drm_mode_create_aspect_ratio_property(struct drm_device *dev);
diff --git a/include/drm/drm_crtc.h b/include/drm/drm_crtc.h
index 312fc1e745d22..c6dbe8b7db9ee 100644
--- a/include/drm/drm_crtc.h
+++ b/include/drm/drm_crtc.h
@@ -58,7 +58,7 @@ struct drm_crtc;
 struct drm_pending_vblank_event;
 struct drm_plane;
 struct drm_bridge;
-struct drm_atomic_state;
+struct drm_atomic_commit;
 
 struct drm_crtc_helper_funcs;
 struct drm_plane_helper_funcs;
@@ -406,8 +406,8 @@ struct drm_crtc_state {
 	 */
 	struct drm_crtc_commit *commit;
 
-	/** @state: backpointer to global drm_atomic_state */
-	struct drm_atomic_state *state;
+	/** @state: backpointer to global drm_atomic_commit */
+	struct drm_atomic_commit *state;
 };
 
 /**
@@ -1124,7 +1124,7 @@ struct drm_crtc {
 	 *
 	 * This is protected by @mutex. Note that nonblocking atomic commits
 	 * access the current CRTC state without taking locks. Either by going
-	 * through the &struct drm_atomic_state pointers, see
+	 * through the &struct drm_atomic_commit pointers, see
 	 * for_each_oldnew_crtc_in_state(), for_each_old_crtc_in_state() and
 	 * for_each_new_crtc_in_state(). Or through careful ordering of atomic
 	 * commit operations as implemented in the atomic helpers, see
@@ -1217,7 +1217,7 @@ struct drm_crtc {
  * @num_connectors: size of @connectors array
  *
  * This represents a modeset configuration for the legacy SETCRTC ioctl and is
- * also used internally. Atomic drivers instead use &drm_atomic_state.
+ * also used internally. Atomic drivers instead use &drm_atomic_commit.
  */
 struct drm_mode_set {
 	struct drm_framebuffer *fb;
diff --git a/include/drm/drm_crtc_helper.h b/include/drm/drm_crtc_helper.h
index 8c886fc46ef22..855da5733c478 100644
--- a/include/drm/drm_crtc_helper.h
+++ b/include/drm/drm_crtc_helper.h
@@ -35,7 +35,7 @@
 
 #include <linux/types.h>
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_connector;
 struct drm_crtc;
 struct drm_device;
@@ -53,7 +53,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
 			      int x, int y,
 			      struct drm_framebuffer *old_fb);
 int drm_crtc_helper_atomic_check(struct drm_crtc *crtc,
-				 struct drm_atomic_state *state);
+				 struct drm_atomic_commit *state);
 bool drm_helper_crtc_in_use(struct drm_crtc *crtc);
 bool drm_helper_encoder_in_use(struct drm_encoder *encoder);
 
diff --git a/include/drm/drm_damage_helper.h b/include/drm/drm_damage_helper.h
index a58cbcd112769..3661aeab2cd3a 100644
--- a/include/drm/drm_damage_helper.h
+++ b/include/drm/drm_damage_helper.h
@@ -64,7 +64,7 @@ struct drm_atomic_helper_damage_iter {
 	bool full_update;
 };
 
-void drm_atomic_helper_check_plane_damage(struct drm_atomic_state *state,
+void drm_atomic_helper_check_plane_damage(struct drm_atomic_commit *state,
 					  struct drm_plane_state *plane_state);
 int drm_atomic_helper_dirtyfb(struct drm_framebuffer *fb,
 			      struct drm_file *file_priv, unsigned int flags,
diff --git a/include/drm/drm_debugfs_crc.h b/include/drm/drm_debugfs_crc.h
index 1b4c98c2f8383..1cb71c03bf44e 100644
--- a/include/drm/drm_debugfs_crc.h
+++ b/include/drm/drm_debugfs_crc.h
@@ -49,7 +49,7 @@ struct drm_crtc_crc_entry {
  * @lock: protects the fields in this struct
  * @source: name of the currently configured source of CRCs
  * @opened: whether userspace has opened the data file for reading
- * @overflow: whether an overflow occured.
+ * @overflow: whether an overflow occurred
  * @entries: array of entries, with size of %DRM_CRC_ENTRIES_NR
  * @head: head of circular queue
  * @tail: tail of circular queue
diff --git a/include/drm/drm_device.h b/include/drm/drm_device.h
index bc78fb77cc279..768a8dae83c52 100644
--- a/include/drm/drm_device.h
+++ b/include/drm/drm_device.h
@@ -375,6 +375,13 @@ struct drm_device {
 	 * Root directory for debugfs files.
 	 */
 	struct dentry *debugfs_root;
+
+	/**
+	 * @gem_lru_mutex:
+	 *
+	 * Lock protecting movement of GEM objects between LRUs.
+	 */
+	struct mutex gem_lru_mutex;
 };
 
 void drm_dev_set_dma_dev(struct drm_device *dev, struct device *dma_dev);
diff --git a/include/drm/drm_encoder.h b/include/drm/drm_encoder.h
index 977a9381c8ba9..eded7c34481a2 100644
--- a/include/drm/drm_encoder.h
+++ b/include/drm/drm_encoder.h
@@ -25,6 +25,7 @@
 
 #include <linux/list.h>
 #include <linux/ctype.h>
+#include <linux/mutex.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_mode.h>
 #include <drm/drm_mode_object.h>
@@ -189,6 +190,9 @@ struct drm_encoder {
 	 */
 	struct list_head bridge_chain;
 
+	/** @bridge_chain_mutex: protect bridge_chain from changes while iterating */
+	struct mutex bridge_chain_mutex;
+
 	const struct drm_encoder_funcs *funcs;
 	const struct drm_encoder_helper_funcs *helper_private;
 
diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h
index 86f5846154f7d..8a704f6a65c15 100644
--- a/include/drm/drm_gem.h
+++ b/include/drm/drm_gem.h
@@ -245,17 +245,11 @@ struct drm_gem_object_funcs {
  * for lockless &shrinker.count_objects, and provides
  * &drm_gem_lru_scan for driver's &shrinker.scan_objects
  * implementation.
+ *
+ * Any access to this kind of object must be done with
+ * drm_device::gem_lru_mutex held.
  */
 struct drm_gem_lru {
-	/**
-	 * @lock:
-	 *
-	 * Lock protecting movement of GEM objects between LRUs.  All
-	 * LRUs that the object can move between should be protected
-	 * by the same lock.
-	 */
-	struct mutex *lock;
-
 	/**
 	 * @count:
 	 *
@@ -453,6 +447,9 @@ struct drm_gem_object {
 	 * @lru:
 	 *
 	 * The current LRU list that the GEM object is on.
+	 *
+	 * Access to this field must be done with drm_device::gem_lru_mutex
+	 * held.
 	 */
 	struct drm_gem_lru *lru;
 };
@@ -610,12 +607,13 @@ void drm_gem_unlock_reservations(struct drm_gem_object **objs, int count,
 int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
 			    u32 handle, u64 *offset);
 
-void drm_gem_lru_init(struct drm_gem_lru *lru, struct mutex *lock);
+void drm_gem_lru_init(struct drm_gem_lru *lru);
 void drm_gem_lru_remove(struct drm_gem_object *obj);
 void drm_gem_lru_move_tail_locked(struct drm_gem_lru *lru, struct drm_gem_object *obj);
 void drm_gem_lru_move_tail(struct drm_gem_lru *lru, struct drm_gem_object *obj);
 unsigned long
-drm_gem_lru_scan(struct drm_gem_lru *lru,
+drm_gem_lru_scan(struct drm_device *dev,
+		 struct drm_gem_lru *lru,
 		 unsigned int nr_to_scan,
 		 unsigned long *remaining,
 		 bool (*shrink)(struct drm_gem_object *obj, struct ww_acquire_ctx *ticket),
diff --git a/include/drm/drm_gpusvm.h b/include/drm/drm_gpusvm.h
index 2578ac92a8d4c..8a4d7134a9a72 100644
--- a/include/drm/drm_gpusvm.h
+++ b/include/drm/drm_gpusvm.h
@@ -6,6 +6,7 @@
 #ifndef __DRM_GPUSVM_H__
 #define __DRM_GPUSVM_H__
 
+#include <linux/dma-mapping.h>
 #include <linux/kref.h>
 #include <linux/interval_tree.h>
 #include <linux/mmu_notifier.h>
@@ -136,17 +137,16 @@ struct drm_gpusvm_pages_flags {
  * @dma_addr: Device address array
  * @dpagemap: The struct drm_pagemap of the device pages we're dma-mapping.
  *            Note this is assuming only one drm_pagemap per range is allowed.
+ * @state: DMA IOVA state for mapping.
+ * @state_offset: DMA IOVA offset for mapping.
  * @notifier_seq: Notifier sequence number of the range's pages
- * @flags: Flags for range
- * @flags.migrate_devmem: Flag indicating whether the range can be migrated to device memory
- * @flags.unmapped: Flag indicating if the range has been unmapped
- * @flags.partial_unmap: Flag indicating if the range has been partially unmapped
- * @flags.has_devmem_pages: Flag indicating if the range has devmem pages
- * @flags.has_dma_mapping: Flag indicating if the range has a DMA mapping
+ * @flags: Flags for the range; see &struct drm_gpusvm_pages_flags
  */
 struct drm_gpusvm_pages {
 	struct drm_pagemap_addr *dma_addr;
 	struct drm_pagemap *dpagemap;
+	struct dma_iova_state state;
+	unsigned long state_offset;
 	unsigned long notifier_seq;
 	struct drm_gpusvm_pages_flags flags;
 };
diff --git a/include/drm/drm_kunit_helpers.h b/include/drm/drm_kunit_helpers.h
index 4948379237e96..e653bc7b1e40b 100644
--- a/include/drm/drm_kunit_helpers.h
+++ b/include/drm/drm_kunit_helpers.h
@@ -97,7 +97,7 @@ __drm_kunit_helper_alloc_drm_device(struct kunit *test,
 						      offsetof(_type, _member),	\
 						      _feat))
 
-struct drm_atomic_state *
+struct drm_atomic_commit *
 drm_kunit_helper_atomic_state_alloc(struct kunit *test,
 				    struct drm_device *drm,
 				    struct drm_modeset_acquire_ctx *ctx);
diff --git a/include/drm/drm_mipi_dbi.h b/include/drm/drm_mipi_dbi.h
index 07374eb5d88e8..78ab23d844ce9 100644
--- a/include/drm/drm_mipi_dbi.h
+++ b/include/drm/drm_mipi_dbi.h
@@ -220,9 +220,9 @@ int mipi_dbi_buf_copy(void *dst, struct iosys_map *src, struct drm_framebuffer *
 	.disable_plane = drm_atomic_helper_disable_plane
 
 int drm_mipi_dbi_plane_helper_atomic_check(struct drm_plane *plane,
-					   struct drm_atomic_state *state);
+					   struct drm_atomic_commit *state);
 void drm_mipi_dbi_plane_helper_atomic_update(struct drm_plane *plane,
-					     struct drm_atomic_state *state);
+					     struct drm_atomic_commit *state);
 
 #define DRM_MIPI_DBI_PLANE_HELPER_FUNCS \
 	DRM_GEM_SHADOW_PLANE_HELPER_FUNCS, \
@@ -243,9 +243,9 @@ void drm_mipi_dbi_plane_helper_atomic_update(struct drm_plane *plane,
 enum drm_mode_status drm_mipi_dbi_crtc_helper_mode_valid(struct drm_crtc *crtc,
 							 const struct drm_display_mode *mode);
 int drm_mipi_dbi_crtc_helper_atomic_check(struct drm_crtc *crtc,
-					  struct drm_atomic_state *state);
+					  struct drm_atomic_commit *state);
 void drm_mipi_dbi_crtc_helper_atomic_disable(struct drm_crtc *crtc,
-					     struct drm_atomic_state *state);
+					     struct drm_atomic_commit *state);
 
 #define DRM_MIPI_DBI_CRTC_HELPER_FUNCS \
 	.mode_valid = drm_mipi_dbi_crtc_helper_mode_valid, \
diff --git a/include/drm/drm_mipi_dsi.h b/include/drm/drm_mipi_dsi.h
index 2ab651a36115d..b429acde4f71a 100644
--- a/include/drm/drm_mipi_dsi.h
+++ b/include/drm/drm_mipi_dsi.h
@@ -393,6 +393,7 @@ void mipi_dsi_dcs_set_page_address_multi(struct mipi_dsi_multi_context *ctx,
 void mipi_dsi_dcs_set_tear_scanline_multi(struct mipi_dsi_multi_context *ctx,
 					  u16 scanline);
 void mipi_dsi_dcs_set_tear_off_multi(struct mipi_dsi_multi_context *ctx);
+void mipi_dsi_shutdown_peripheral_multi(struct mipi_dsi_multi_context *ctx);
 
 /**
  * mipi_dsi_generic_write_seq_multi - transmit data using a generic write packet
diff --git a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h
index 687c0ee163d25..e584652ddf676 100644
--- a/include/drm/drm_mode_config.h
+++ b/include/drm/drm_mode_config.h
@@ -33,7 +33,7 @@
 
 struct drm_file;
 struct drm_device;
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_mode_fb_cmd2;
 struct drm_format_info;
 struct drm_display_mode;
@@ -158,7 +158,7 @@ struct drm_mode_config_funcs {
 	 * error conditions which don't have to be checked at the in this
 	 * callback.
 	 *
-	 * See the documentation for &struct drm_atomic_state for how exactly
+	 * See the documentation for &struct drm_atomic_commit for how exactly
 	 * an atomic modeset update is described.
 	 *
 	 * Drivers using the atomic helpers can implement this hook using
@@ -184,7 +184,7 @@ struct drm_mode_config_funcs {
 	 *    treated equally.
 	 */
 	int (*atomic_check)(struct drm_device *dev,
-			    struct drm_atomic_state *state);
+			    struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_commit:
@@ -194,7 +194,7 @@ struct drm_mode_config_funcs {
 	 * calling this function, and that nothing has been changed in the
 	 * interim.
 	 *
-	 * See the documentation for &struct drm_atomic_state for how exactly
+	 * See the documentation for &struct drm_atomic_commit for how exactly
 	 * an atomic modeset update is described.
 	 *
 	 * Drivers using the atomic helpers can implement this hook using
@@ -266,31 +266,31 @@ struct drm_mode_config_funcs {
 	 * additional modeset locks).
 	 */
 	int (*atomic_commit)(struct drm_device *dev,
-			     struct drm_atomic_state *state,
+			     struct drm_atomic_commit *state,
 			     bool nonblock);
 
 	/**
 	 * @atomic_state_alloc:
 	 *
 	 * This optional hook can be used by drivers that want to subclass struct
-	 * &drm_atomic_state to be able to track their own driver-private global
+	 * &drm_atomic_commit to be able to track their own driver-private global
 	 * state easily. If this hook is implemented, drivers must also
 	 * implement @atomic_state_clear and @atomic_state_free.
 	 *
-	 * Subclassing of &drm_atomic_state is deprecated in favour of using
+	 * Subclassing of &drm_atomic_commit is deprecated in favour of using
 	 * &drm_private_state and &drm_private_obj.
 	 *
 	 * RETURNS:
 	 *
-	 * A new &drm_atomic_state on success or NULL on failure.
+	 * A new &drm_atomic_commit on success or NULL on failure.
 	 */
-	struct drm_atomic_state *(*atomic_state_alloc)(struct drm_device *dev);
+	struct drm_atomic_commit *(*atomic_state_alloc)(struct drm_device *dev);
 
 	/**
 	 * @atomic_state_clear:
 	 *
 	 * This hook must clear any driver private state duplicated into the
-	 * passed-in &drm_atomic_state. This hook is called when the caller
+	 * passed-in &drm_atomic_commit. This hook is called when the caller
 	 * encountered a &drm_modeset_lock deadlock and needs to drop all
 	 * already acquired locks as part of the deadlock avoidance dance
 	 * implemented in drm_modeset_backoff().
@@ -299,28 +299,28 @@ struct drm_mode_config_funcs {
 	 * update might change it, and the drm atomic interfaces always apply
 	 * updates as relative changes to the current state.
 	 *
-	 * Drivers that implement this must call drm_atomic_state_default_clear()
+	 * Drivers that implement this must call drm_atomic_commit_default_clear()
 	 * to clear common state.
 	 *
-	 * Subclassing of &drm_atomic_state is deprecated in favour of using
+	 * Subclassing of &drm_atomic_commit is deprecated in favour of using
 	 * &drm_private_state and &drm_private_obj.
 	 */
-	void (*atomic_state_clear)(struct drm_atomic_state *state);
+	void (*atomic_state_clear)(struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_state_free:
 	 *
-	 * This hook needs driver private resources and the &drm_atomic_state
-	 * itself. Note that the core first calls drm_atomic_state_clear() to
+	 * This hook needs driver private resources and the &drm_atomic_commit
+	 * itself. Note that the core first calls drm_atomic_commit_clear() to
 	 * avoid code duplicate between the clear and free hooks.
 	 *
 	 * Drivers that implement this must call
-	 * drm_atomic_state_default_release() to release common resources.
+	 * drm_atomic_commit_default_release() to release common resources.
 	 *
-	 * Subclassing of &drm_atomic_state is deprecated in favour of using
+	 * Subclassing of &drm_atomic_commit is deprecated in favour of using
 	 * &drm_private_state and &drm_private_obj.
 	 */
-	void (*atomic_state_free)(struct drm_atomic_state *state);
+	void (*atomic_state_free)(struct drm_atomic_commit *state);
 };
 
 /**
@@ -985,7 +985,7 @@ struct drm_mode_config {
 	 * Set by drm_mode_config_helper_suspend() and cleared by
 	 * drm_mode_config_helper_resume().
 	 */
-	struct drm_atomic_state *suspend_state;
+	struct drm_atomic_commit *suspend_state;
 
 	const struct drm_mode_config_helper_funcs *helper_private;
 };
diff --git a/include/drm/drm_modeset_helper_vtables.h b/include/drm/drm_modeset_helper_vtables.h
index 3e68213958dd8..ca6268945c28d 100644
--- a/include/drm/drm_modeset_helper_vtables.h
+++ b/include/drm/drm_modeset_helper_vtables.h
@@ -332,7 +332,7 @@ struct drm_crtc_helper_funcs {
 	 * deadlock.
 	 */
 	int (*atomic_check)(struct drm_crtc *crtc,
-			    struct drm_atomic_state *state);
+			    struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_begin:
@@ -353,7 +353,7 @@ struct drm_crtc_helper_funcs {
 	 * optional.
 	 */
 	void (*atomic_begin)(struct drm_crtc *crtc,
-			     struct drm_atomic_state *state);
+			     struct drm_atomic_commit *state);
 	/**
 	 * @atomic_flush:
 	 *
@@ -377,7 +377,7 @@ struct drm_crtc_helper_funcs {
 	 * optional.
 	 */
 	void (*atomic_flush)(struct drm_crtc *crtc,
-			     struct drm_atomic_state *state);
+			     struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_enable:
@@ -399,7 +399,7 @@ struct drm_crtc_helper_funcs {
 	 * This function is optional.
 	 */
 	void (*atomic_enable)(struct drm_crtc *crtc,
-			      struct drm_atomic_state *state);
+			      struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_disable:
@@ -419,7 +419,7 @@ struct drm_crtc_helper_funcs {
 	 * This function is optional.
 	 */
 	void (*atomic_disable)(struct drm_crtc *crtc,
-			       struct drm_atomic_state *state);
+			       struct drm_atomic_commit *state);
 
 	/**
 	 * @get_scanout_position:
@@ -713,7 +713,7 @@ struct drm_encoder_helper_funcs {
 	 * @atomic_enable.
 	 */
 	void (*atomic_disable)(struct drm_encoder *encoder,
-			       struct drm_atomic_state *state);
+			       struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_enable:
@@ -736,7 +736,7 @@ struct drm_encoder_helper_funcs {
 	 * @atomic_disable.
 	 */
 	void (*atomic_enable)(struct drm_encoder *encoder,
-			      struct drm_atomic_state *state);
+			      struct drm_atomic_commit *state);
 
 	/**
 	 * @disable:
@@ -809,7 +809,7 @@ struct drm_encoder_helper_funcs {
 	 *
 	 * This function is called in the check phase of an atomic update. The
 	 * driver is not allowed to change anything outside of the free-standing
-	 * state objects passed-in or assembled in the overall &drm_atomic_state
+	 * state objects passed-in or assembled in the overall &drm_atomic_commit
 	 * update tracking structure.
 	 *
 	 * Also beware that userspace can request its own custom modes, neither
@@ -1044,7 +1044,7 @@ struct drm_connector_helper_funcs {
 	 *
 	 * This function is called in the check phase of an atomic update. The
 	 * driver is not allowed to change anything outside of the
-	 * &drm_atomic_state update tracking structure passed in.
+	 * &drm_atomic_commit update tracking structure passed in.
 	 *
 	 * RETURNS:
 	 *
@@ -1054,7 +1054,7 @@ struct drm_connector_helper_funcs {
 	 * for this.
 	 */
 	struct drm_encoder *(*atomic_best_encoder)(struct drm_connector *connector,
-						   struct drm_atomic_state *state);
+						   struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_check:
@@ -1077,7 +1077,7 @@ struct drm_connector_helper_funcs {
 	 *
 	 * This function is called in the check phase of an atomic update. The
 	 * driver is not allowed to change anything outside of the free-standing
-	 * state objects passed-in or assembled in the overall &drm_atomic_state
+	 * state objects passed-in or assembled in the overall &drm_atomic_commit
 	 * update tracking structure.
 	 *
 	 * RETURNS:
@@ -1088,7 +1088,7 @@ struct drm_connector_helper_funcs {
 	 * deadlock.
 	 */
 	int (*atomic_check)(struct drm_connector *connector,
-			    struct drm_atomic_state *state);
+			    struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_commit:
@@ -1103,7 +1103,7 @@ struct drm_connector_helper_funcs {
 	 * This callback is used by the atomic modeset helpers.
 	 */
 	void (*atomic_commit)(struct drm_connector *connector,
-			      struct drm_atomic_state *state);
+			      struct drm_atomic_commit *state);
 
 	/**
 	 * @prepare_writeback_job:
@@ -1299,7 +1299,7 @@ struct drm_plane_helper_funcs {
 	 *
 	 * This function is called in the check phase of an atomic update. The
 	 * driver is not allowed to change anything outside of the
-	 * &drm_atomic_state update tracking structure.
+	 * &drm_atomic_commit update tracking structure.
 	 *
 	 * RETURNS:
 	 *
@@ -1309,7 +1309,7 @@ struct drm_plane_helper_funcs {
 	 * deadlock.
 	 */
 	int (*atomic_check)(struct drm_plane *plane,
-			    struct drm_atomic_state *state);
+			    struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_update:
@@ -1326,7 +1326,7 @@ struct drm_plane_helper_funcs {
 	 * This callback is used by the atomic modeset helpers, but it is optional.
 	 */
 	void (*atomic_update)(struct drm_plane *plane,
-			      struct drm_atomic_state *state);
+			      struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_enable:
@@ -1351,7 +1351,7 @@ struct drm_plane_helper_funcs {
 	 * implement the complete plane update in @atomic_update.
 	 */
 	void (*atomic_enable)(struct drm_plane *plane,
-			      struct drm_atomic_state *state);
+			      struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_disable:
@@ -1376,7 +1376,7 @@ struct drm_plane_helper_funcs {
 	 * optional. It's intended to reverse the effects of @atomic_enable.
 	 */
 	void (*atomic_disable)(struct drm_plane *plane,
-			       struct drm_atomic_state *state);
+			       struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_async_check:
@@ -1400,7 +1400,7 @@ struct drm_plane_helper_funcs {
 	 * can not be applied in asynchronous manner.
 	 */
 	int (*atomic_async_check)(struct drm_plane *plane,
-				  struct drm_atomic_state *state, bool flip);
+				  struct drm_atomic_commit *state, bool flip);
 
 	/**
 	 * @atomic_async_update:
@@ -1437,7 +1437,7 @@ struct drm_plane_helper_funcs {
 	 *    for deferring if needed, until a common solution is created.
 	 */
 	void (*atomic_async_update)(struct drm_plane *plane,
-				    struct drm_atomic_state *state);
+				    struct drm_atomic_commit *state);
 
 	/**
 	 * @get_scanout_buffer:
@@ -1530,7 +1530,7 @@ struct drm_mode_config_helper_funcs {
 	 * This hook is optional, the default implementation is
 	 * drm_atomic_helper_commit_tail().
 	 */
-	void (*atomic_commit_tail)(struct drm_atomic_state *state);
+	void (*atomic_commit_tail)(struct drm_atomic_commit *state);
 
 	/**
 	 * @atomic_commit_setup:
@@ -1551,7 +1551,7 @@ struct drm_mode_config_helper_funcs {
 	 *
 	 * This hook is optional.
 	 */
-	int (*atomic_commit_setup)(struct drm_atomic_state *state);
+	int (*atomic_commit_setup)(struct drm_atomic_commit *state);
 };
 
 #endif
diff --git a/include/drm/drm_of.h b/include/drm/drm_of.h
index f2f2bf82eff90..7bcc0ccfe0f4b 100644
--- a/include/drm/drm_of.h
+++ b/include/drm/drm_of.h
@@ -62,6 +62,10 @@ int drm_of_get_data_lanes_count_ep(const struct device_node *port,
 				   int port_reg, int reg,
 				   const unsigned int min,
 				   const unsigned int max);
+int drm_of_get_data_lanes_count_remote(const struct device_node *port,
+				       int port_reg, int reg,
+				       const unsigned int min,
+				       const unsigned int max);
 #else
 static inline uint32_t drm_of_crtc_port_mask(struct drm_device *dev,
 					  struct device_node *port)
@@ -140,6 +144,15 @@ drm_of_get_data_lanes_count_ep(const struct device_node *port,
 {
 	return -EINVAL;
 }
+
+static inline int
+drm_of_get_data_lanes_count_remote(const struct device_node *port,
+				   int port_reg, int reg,
+				   const unsigned int min,
+				   const unsigned int max)
+{
+	return -EINVAL;
+}
 #endif
 
 #if IS_ENABLED(CONFIG_OF) && IS_ENABLED(CONFIG_DRM_MIPI_DSI)
diff --git a/include/drm/drm_pagemap.h b/include/drm/drm_pagemap.h
index 75e6ca58922d5..95eb4b66b0577 100644
--- a/include/drm/drm_pagemap.h
+++ b/include/drm/drm_pagemap.h
@@ -329,17 +329,12 @@ struct drm_pagemap_devmem {
  * struct drm_pagemap_migrate_details - Details to govern migration.
  * @timeslice_ms: The time requested for the migrated pagemap pages to
  * be present in @mm before being allowed to be migrated back.
- * @can_migrate_same_pagemap: Whether the copy function as indicated by
- * the @source_peer_migrates flag, can migrate device pages within a
- * single drm_pagemap.
- * @source_peer_migrates: Whether on p2p migration, The source drm_pagemap
- * should use the copy_to_ram() callback rather than the destination
- * drm_pagemap should use the copy_to_devmem() callback.
+ * @can_migrate_same_pagemap: Whether the copy function can migrate
+ * device pages within a single drm_pagemap.
  */
 struct drm_pagemap_migrate_details {
 	unsigned long timeslice_ms;
 	u32 can_migrate_same_pagemap : 1;
-	u32 source_peer_migrates : 1;
 };
 
 #if IS_ENABLED(CONFIG_ZONE_DEVICE)
diff --git a/include/drm/drm_panel.h b/include/drm/drm_panel.h
index 2407bfa60236f..86b3f9c65c926 100644
--- a/include/drm/drm_panel.h
+++ b/include/drm/drm_panel.h
@@ -320,15 +320,12 @@ void *__devm_drm_panel_alloc(struct device *dev, size_t size, size_t offset,
 					offsetof(type, member), funcs, \
 					connector_type))
 
-void drm_panel_init(struct drm_panel *panel, struct device *dev,
-		    const struct drm_panel_funcs *funcs,
-		    int connector_type);
-
 struct drm_panel *drm_panel_get(struct drm_panel *panel);
 void drm_panel_put(struct drm_panel *panel);
 
 void drm_panel_add(struct drm_panel *panel);
 void drm_panel_remove(struct drm_panel *panel);
+int devm_drm_panel_add(struct device *dev, struct drm_panel *panel);
 
 void drm_panel_prepare(struct drm_panel *panel);
 void drm_panel_unprepare(struct drm_panel *panel);
diff --git a/include/drm/drm_plane.h b/include/drm/drm_plane.h
index 703ef4d1bbbcf..419c88c873a6f 100644
--- a/include/drm/drm_plane.h
+++ b/include/drm/drm_plane.h
@@ -259,8 +259,8 @@ struct drm_plane_state {
 	 */
 	struct drm_crtc_commit *commit;
 
-	/** @state: backpointer to global drm_atomic_state */
-	struct drm_atomic_state *state;
+	/** @state: backpointer to global drm_atomic_commit */
+	struct drm_atomic_commit *state;
 
 	/**
 	 * @color_mgmt_changed: Color management properties have changed. Used
@@ -739,7 +739,7 @@ struct drm_plane {
 	 *
 	 * This is protected by @mutex. Note that nonblocking atomic commits
 	 * access the current plane state without taking locks. Either by going
-	 * through the &struct drm_atomic_state pointers, see
+	 * through the &struct drm_atomic_commit pointers, see
 	 * for_each_oldnew_plane_in_state(), for_each_old_plane_in_state() and
 	 * for_each_new_plane_in_state(). Or through careful ordering of atomic
 	 * commit operations as implemented in the atomic helpers, see
diff --git a/include/drm/drm_ras.h b/include/drm/drm_ras.h
index 5d50209e51db4..f2a787bc4f64a 100644
--- a/include/drm/drm_ras.h
+++ b/include/drm/drm_ras.h
@@ -58,6 +58,17 @@ struct drm_ras_node {
 	int (*query_error_counter)(struct drm_ras_node *node, u32 error_id,
 				   const char **name, u32 *val);
 
+	/**
+	 * @clear_error_counter:
+	 *
+	 * This callback is used by drm_ras to clear a specific error counter.
+	 * Driver should implement this callback to support clearing error counters
+	 * of a node.
+	 *
+	 * Returns: 0 on success, negative error code on failure.
+	 */
+	int (*clear_error_counter)(struct drm_ras_node *node, u32 error_id);
+
 	/** @priv: Driver private data */
 	void *priv;
 };
diff --git a/include/drm/drm_self_refresh_helper.h b/include/drm/drm_self_refresh_helper.h
index 520235c20708b..95c190125b604 100644
--- a/include/drm/drm_self_refresh_helper.h
+++ b/include/drm/drm_self_refresh_helper.h
@@ -8,11 +8,11 @@
 #ifndef DRM_SELF_REFRESH_HELPER_H_
 #define DRM_SELF_REFRESH_HELPER_H_
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_crtc;
 
-void drm_self_refresh_helper_alter_state(struct drm_atomic_state *state);
-void drm_self_refresh_helper_update_avg_times(struct drm_atomic_state *state,
+void drm_self_refresh_helper_alter_state(struct drm_atomic_commit *state);
+void drm_self_refresh_helper_update_avg_times(struct drm_atomic_commit *state,
 					unsigned int commit_time_ms,
 					unsigned int new_self_refresh_mask);
 
diff --git a/include/drm/drm_vblank_helper.h b/include/drm/drm_vblank_helper.h
index fcd8a9b358463..28051f08d0f42 100644
--- a/include/drm/drm_vblank_helper.h
+++ b/include/drm/drm_vblank_helper.h
@@ -6,7 +6,7 @@
 #include <linux/hrtimer_types.h>
 #include <linux/types.h>
 
-struct drm_atomic_state;
+struct drm_atomic_commit;
 struct drm_crtc;
 
 /*
@@ -14,11 +14,11 @@ struct drm_crtc;
  */
 
 void drm_crtc_vblank_atomic_flush(struct drm_crtc *crtc,
-				  struct drm_atomic_state *state);
+				  struct drm_atomic_commit *state);
 void drm_crtc_vblank_atomic_enable(struct drm_crtc *crtc,
-				   struct drm_atomic_state *state);
+				   struct drm_atomic_commit *state);
 void drm_crtc_vblank_atomic_disable(struct drm_crtc *crtc,
-				    struct drm_atomic_state *crtc_state);
+				    struct drm_atomic_commit *crtc_state);
 
 /**
  * DRM_CRTC_HELPER_VBLANK_FUNCS - Default implementation for VBLANK helpers
diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h
index 78e07c2507c7a..d61c19e78182c 100644
--- a/include/drm/gpu_scheduler.h
+++ b/include/drm/gpu_scheduler.h
@@ -25,11 +25,14 @@
 #define _DRM_GPU_SCHEDULER_H_
 
 #include <drm/spsc_queue.h>
+#include <linux/average.h>
 #include <linux/dma-fence.h>
 #include <linux/completion.h>
 #include <linux/xarray.h>
 #include <linux/workqueue.h>
 
+DECLARE_EWMA(drm_sched_avgtime, 6, 4);
+
 #define MAX_WAIT_SCHED_ENTITY_Q_EMPTY msecs_to_jiffies(1000)
 
 /**
@@ -63,6 +66,7 @@ struct drm_file;
  * to an array, and as such should start at 0.
  */
 enum drm_sched_priority {
+	DRM_SCHED_PRIORITY_INVALID = -1, /* Internal marker - do not use. */
 	DRM_SCHED_PRIORITY_KERNEL,
 	DRM_SCHED_PRIORITY_HIGH,
 	DRM_SCHED_PRIORITY_NORMAL,
@@ -71,6 +75,8 @@ enum drm_sched_priority {
 	DRM_SCHED_PRIORITY_COUNT
 };
 
+struct drm_sched_entity_stats;
+
 /**
  * struct drm_sched_entity - A wrapper around a job queue (typically
  * attached to the DRM file_priv).
@@ -109,6 +115,11 @@ struct drm_sched_entity {
 	 */
 	struct drm_sched_rq		*rq;
 
+	/**
+	 * @stats: Stats object reference held by the entity and jobs.
+	 */
+	struct drm_sched_entity_stats	*stats;
+
 	/**
 	 * @sched_list:
 	 *
@@ -238,24 +249,21 @@ struct drm_sched_entity {
 /**
  * struct drm_sched_rq - queue of entities to be scheduled.
  *
- * @sched: the scheduler to which this rq belongs to.
- * @lock: protects @entities, @rb_tree_root and @current_entity.
- * @current_entity: the entity which is to be scheduled.
+ * @lock: protects @entities, @rb_tree_root and @head_prio.
  * @entities: list of the entities to be scheduled.
  * @rb_tree_root: root of time based priority queue of entities for FIFO scheduling
+ * @head_prio: priority of the top tree element.
  *
  * Run queue is a set of entities scheduling command submissions for
  * one specific ring. It implements the scheduling policy that selects
  * the next entity to emit commands from.
  */
 struct drm_sched_rq {
-	struct drm_gpu_scheduler	*sched;
-
 	spinlock_t			lock;
 	/* Following members are protected by the @lock: */
-	struct drm_sched_entity		*current_entity;
 	struct list_head		entities;
 	struct rb_root_cached		rb_tree_root;
+	enum drm_sched_priority		head_prio;
 };
 
 /**
@@ -338,13 +346,6 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f);
  * to schedule the job.
  */
 struct drm_sched_job {
-	/**
-	 * @submit_ts:
-	 *
-	 * When the job was pushed into the entity queue.
-	 */
-	ktime_t                         submit_ts;
-
 	/**
 	 * @sched:
 	 *
@@ -357,6 +358,11 @@ struct drm_sched_job {
 	struct drm_sched_fence		*s_fence;
 	struct drm_sched_entity         *entity;
 
+	/**
+	 * @entity_stats: Stats object reference held by the job and entity.
+	 */
+	struct drm_sched_entity_stats	*entity_stats;
+
 	enum drm_sched_priority		s_priority;
 	u32				credits;
 	/** @last_dependency: tracks @dependencies as they signal */
@@ -543,15 +549,14 @@ struct drm_sched_backend_ops {
  * @credit_count: the current credit count of this scheduler
  * @timeout: the time after which a job is removed from the scheduler.
  * @name: name of the ring for which this scheduler is being used.
- * @num_rqs: Number of run-queues. This is at most DRM_SCHED_PRIORITY_COUNT,
- *           as there's usually one run-queue per priority, but could be less.
- * @sched_rq: An allocated array of run-queues of size @num_rqs;
+ * @rq: Scheduler run queue.
  * @job_scheduled: once drm_sched_entity_flush() is called the scheduler
  *                 waits on this wait queue until all the scheduled jobs are
  *                 finished.
  * @job_id_count: used to assign unique id to the each job.
  * @submit_wq: workqueue used to queue @work_run_job and @work_free_job
  * @timeout_wq: workqueue used to queue @work_tdr
+ * @avg_job_us: Average job duration.
  * @work_run_job: work which calls run_job op of each scheduler.
  * @work_free_job: work which calls free_job op of each scheduler.
  * @work_tdr: schedules a delayed call to @drm_sched_job_timedout after the
@@ -576,12 +581,12 @@ struct drm_gpu_scheduler {
 	atomic_t			credit_count;
 	long				timeout;
 	const char			*name;
-	u32                             num_rqs;
-	struct drm_sched_rq             **sched_rq;
+	struct drm_sched_rq             rq;
 	wait_queue_head_t		job_scheduled;
 	atomic64_t			job_id_count;
 	struct workqueue_struct		*submit_wq;
 	struct workqueue_struct		*timeout_wq;
+	struct ewma_drm_sched_avgtime   avg_job_us;
 	struct work_struct		work_run_job;
 	struct work_struct		work_free_job;
 	struct delayed_work		work_tdr;
@@ -603,8 +608,6 @@ struct drm_gpu_scheduler {
  * @ops: backend operations provided by the driver
  * @submit_wq: workqueue to use for submission. If NULL, an ordered wq is
  *	       allocated and used.
- * @num_rqs: Number of run-queues. This may be at most DRM_SCHED_PRIORITY_COUNT,
- *	     as there's usually one run-queue per priority, but may be less.
  * @credit_limit: the number of credits this scheduler can hold from all jobs
  * @hang_limit: number of times to allow a job to hang before dropping it.
  *		This mechanism is DEPRECATED. Set it to 0.
@@ -618,7 +621,6 @@ struct drm_sched_init_args {
 	const struct drm_sched_backend_ops *ops;
 	struct workqueue_struct *submit_wq;
 	struct workqueue_struct *timeout_wq;
-	u32 num_rqs;
 	u32 credit_limit;
 	unsigned int hang_limit;
 	long timeout;
@@ -691,6 +693,7 @@ int drm_sched_entity_init(struct drm_sched_entity *entity,
 			  unsigned int num_sched_list,
 			  atomic_t *guilty);
 long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout);
+void drm_sched_entity_kill(struct drm_sched_entity *entity);
 void drm_sched_entity_fini(struct drm_sched_entity *entity);
 void drm_sched_entity_destroy(struct drm_sched_entity *entity);
 void drm_sched_entity_set_priority(struct drm_sched_entity *entity,
diff --git a/include/drm/intel/display_parent_interface.h b/include/drm/intel/display_parent_interface.h
index 97ec94a2e7493..39991afeb173d 100644
--- a/include/drm/intel/display_parent_interface.h
+++ b/include/drm/intel/display_parent_interface.h
@@ -6,8 +6,8 @@
 
 #include <linux/types.h>
 
+enum vlv_iosf_sb_unit;
 struct dma_fence;
-struct drm_crtc;
 struct drm_device;
 struct drm_file;
 struct drm_framebuffer;
@@ -15,6 +15,8 @@ struct drm_gem_object;
 struct drm_mode_fb_cmd2;
 struct drm_plane_state;
 struct drm_scanout_buffer;
+struct fb_info;
+struct i915_gtt_view;
 struct i915_vma;
 struct intel_dpt;
 struct intel_dsb_buffer;
@@ -23,10 +25,22 @@ struct intel_hdcp_gsc_context;
 struct intel_initial_plane_config;
 struct intel_panic;
 struct intel_stolen_node;
+struct iosys_map;
 struct ref_tracker;
 struct seq_file;
 struct vm_area_struct;
 
+struct intel_fb_pin_params {
+	const struct i915_gtt_view *view;
+	unsigned int alignment;
+	unsigned int phys_alignment;
+	unsigned int vtd_guard;
+	bool needs_cpu_lmem_access;
+	bool needs_low_address;
+	bool needs_physical;
+	bool needs_fence;
+};
+
 /* Keep struct definitions sorted */
 
 struct intel_display_bo_interface {
@@ -43,6 +57,12 @@ struct intel_display_bo_interface {
 	struct drm_gem_object *(*framebuffer_lookup)(struct drm_device *drm,
 						     struct drm_file *filp,
 						     const struct drm_mode_fb_cmd2 *user_mode_cmd);
+#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION)
+	struct drm_gem_object *(*fbdev_create)(struct drm_device *drm, int size);
+	void (*fbdev_destroy)(struct drm_gem_object *obj);
+	int (*fbdev_fill_info)(struct drm_gem_object *obj, struct fb_info *info, struct i915_vma *vma);
+	u32 (*fbdev_pitch_align)(u32 stride);
+#endif
 };
 
 struct intel_display_dpt_interface {
@@ -62,6 +82,32 @@ struct intel_display_dsb_interface {
 	void (*flush_map)(struct intel_dsb_buffer *dsb_buf);
 };
 
+struct intel_display_fb_pin_interface {
+	int (*ggtt_pin)(struct drm_gem_object *obj,
+			const struct intel_fb_pin_params *pin_params,
+			struct i915_vma **out_ggtt_vma,
+			u32 *out_offset,
+			int *out_fence_id);
+	void (*ggtt_unpin)(struct i915_vma *ggtt_vma,
+			   int fence_id);
+	int (*dpt_pin)(struct drm_gem_object *obj,
+		       struct intel_dpt *dpt,
+		       const struct intel_fb_pin_params *pin_params,
+		       struct i915_vma **out_dpt_vma,
+		       struct i915_vma **out_ggtt_vma,
+		       u32 *out_offset);
+	void (*dpt_unpin)(struct intel_dpt *dpt,
+			  struct i915_vma *dpt_vma,
+			  struct i915_vma *ggtt_vma);
+	struct i915_vma *(*reuse_vma)(struct i915_vma *old_ggtt_vma,
+				      struct drm_gem_object *old_obj,
+				      const struct i915_gtt_view *old_view,
+				      struct drm_gem_object *new_obj,
+				      const struct i915_gtt_view *new_view,
+				      u32 *out_offset);
+	void (*get_map)(struct i915_vma *vma, struct iosys_map *map);
+};
+
 struct intel_display_frontbuffer_interface {
 	struct intel_frontbuffer *(*get)(struct drm_gem_object *obj);
 	void (*ref)(struct intel_frontbuffer *front);
@@ -79,11 +125,10 @@ struct intel_display_hdcp_interface {
 };
 
 struct intel_display_initial_plane_interface {
-	void (*vblank_wait)(struct drm_crtc *crtc);
 	struct drm_gem_object *(*alloc_obj)(struct drm_device *drm, struct intel_initial_plane_config *plane_config);
 	int (*setup)(struct drm_plane_state *plane_state, struct intel_initial_plane_config *plane_config,
 		     struct drm_framebuffer *fb, struct i915_vma *vma);
-	void (*config_fini)(struct intel_initial_plane_config *plane_configs);
+	void (*config_fini)(struct intel_initial_plane_config *plane_config);
 };
 
 struct intel_display_irq_interface {
@@ -176,8 +221,11 @@ struct intel_display_stolen_interface {
 	void (*node_free)(const struct intel_stolen_node *node);
 };
 
-struct intel_display_vma_interface {
-	int (*fence_id)(const struct i915_vma *vma);
+struct intel_display_vlv_iosf_interface {
+	void (*get)(struct drm_device *drm, unsigned long unit_mask);
+	void (*put)(struct drm_device *drm, unsigned long unit_mask);
+	u32 (*read)(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr);
+	int (*write)(struct drm_device *drm, enum vlv_iosf_sb_unit unit, u32 addr, u32 val);
 };
 
 /**
@@ -202,6 +250,9 @@ struct intel_display_parent_interface {
 	/** @dsb: DSB buffer interface */
 	const struct intel_display_dsb_interface *dsb;
 
+	/** @fb_pin: Framebuffer pin interface */
+	const struct intel_display_fb_pin_interface *fb_pin;
+
 	/** @frontbuffer: Frontbuffer interface */
 	const struct intel_display_frontbuffer_interface *frontbuffer;
 
@@ -235,8 +286,8 @@ struct intel_display_parent_interface {
 	/** @stolen: Stolen memory. */
 	const struct intel_display_stolen_interface *stolen;
 
-	/** @vma: VMA interface. Optional. */
-	const struct intel_display_vma_interface *vma;
+	/** @vlv_iosf: VLV IOSF sideband. Optional. */
+	const struct intel_display_vlv_iosf_interface *vlv_iosf;
 
 	/* Generic independent functions */
 	struct {
diff --git a/include/drm/intel/mchbar_regs.h b/include/drm/intel/mchbar_regs.h
new file mode 100644
index 0000000000000..ca0d421be16c3
--- /dev/null
+++ b/include/drm/intel/mchbar_regs.h
@@ -0,0 +1,273 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#ifndef __INTEL_MCHBAR_REGS__
+#define __INTEL_MCHBAR_REGS__
+
+#include "i915_reg_defs.h"
+
+/*
+ * MCHBAR mirror.
+ *
+ * This mirrors the MCHBAR MMIO space whose location is determined by
+ * device 0 function 0's pci config register 0x44 or 0x48 and matches it in
+ * every way.  It is not accessible from the CP register read instructions.
+ *
+ * Starting from Haswell, you can't write registers using the MCHBAR mirror,
+ * just read. On MTL+ the mirror no longer exists.
+ */
+
+#define MCHBAR_MIRROR_BASE			0x10000
+#define MCHBAR_MIRROR_END			0x13fff
+
+#define MCHBAR_MIRROR_BASE_SNB			0x140000
+#define MCHBAR_MIRROR_END_SNB			0x147fff
+#define MCHBAR_MIRROR_END_ICL_RKL		0x14ffff
+#define MCHBAR_MIRROR_END_TGL			0x15ffff
+
+#define CTG_STOLEN_RESERVED			_MMIO(MCHBAR_MIRROR_BASE + 0x34)
+#define ELK_STOLEN_RESERVED			_MMIO(MCHBAR_MIRROR_BASE + 0x48)
+#define   G4X_STOLEN_RESERVED_ADDR1_MASK	(0xFFFF << 16)
+#define   G4X_STOLEN_RESERVED_ADDR2_MASK	(0xFFF << 4)
+#define   G4X_STOLEN_RESERVED_ENABLE		(1 << 0)
+
+/* Pineview MCH register contains DDR3 setting */
+#define CSHRDDR3CTL				_MMIO(MCHBAR_MIRROR_BASE + 0x1a8)
+#define   CSHRDDR3CTL_DDR3			(1 << 2)
+
+/* 915-945 and GM965 MCH register controlling DRAM channel access */
+#define DCC					_MMIO(MCHBAR_MIRROR_BASE + 0x200)
+#define   DCC_ADDRESSING_MODE_SINGLE_CHANNEL	(0 << 0)
+#define   DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC	(1 << 0)
+#define   DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED	(2 << 0)
+#define   DCC_ADDRESSING_MODE_MASK		(3 << 0)
+#define   DCC_CHANNEL_XOR_DISABLE		(1 << 10)
+#define   DCC_CHANNEL_XOR_BIT_17		(1 << 9)
+#define DCC2					_MMIO(MCHBAR_MIRROR_BASE + 0x204)
+#define   DCC2_MODIFIED_ENHANCED_DISABLE	(1 << 20)
+
+/* 965 MCH register controlling DRAM channel configuration */
+#define C0DRB3_BW				_MMIO(MCHBAR_MIRROR_BASE + 0x206)
+#define C1DRB3_BW				_MMIO(MCHBAR_MIRROR_BASE + 0x606)
+
+/* Clocking configuration register */
+#define CLKCFG					_MMIO(MCHBAR_MIRROR_BASE + 0xc00)
+#define CLKCFG_FSB_400				(0 << 0)	/* hrawclk 100 */
+#define CLKCFG_FSB_400_ALT			(5 << 0)	/* hrawclk 100 */
+#define CLKCFG_FSB_533				(1 << 0)	/* hrawclk 133 */
+#define CLKCFG_FSB_667				(3 << 0)	/* hrawclk 166 */
+#define CLKCFG_FSB_800				(2 << 0)	/* hrawclk 200 */
+#define CLKCFG_FSB_1067				(6 << 0)	/* hrawclk 266 */
+#define CLKCFG_FSB_1067_ALT			(0 << 0)	/* hrawclk 266 */
+#define CLKCFG_FSB_1333				(7 << 0)	/* hrawclk 333 */
+#define CLKCFG_FSB_1333_ALT			(4 << 0)	/* hrawclk 333 */
+#define CLKCFG_FSB_1600_ALT			(6 << 0)	/* hrawclk 400 */
+#define CLKCFG_FSB_MASK				(7 << 0)
+#define CLKCFG_MEM_533				(1 << 4)
+#define CLKCFG_MEM_667				(2 << 4)
+#define CLKCFG_MEM_800				(3 << 4)
+#define CLKCFG_MEM_MASK				(7 << 4)
+
+#define HPLLVCO_MOBILE				_MMIO(MCHBAR_MIRROR_BASE + 0xc0f)
+#define HPLLVCO					_MMIO(MCHBAR_MIRROR_BASE + 0xc38)
+
+#define TSC1					_MMIO(MCHBAR_MIRROR_BASE + 0x1001)
+#define   TSE					(1 << 0)
+#define TR1					_MMIO(MCHBAR_MIRROR_BASE + 0x1006)
+#define TSFS					_MMIO(MCHBAR_MIRROR_BASE + 0x1020)
+#define   TSFS_SLOPE_MASK			0x0000ff00
+#define   TSFS_SLOPE_SHIFT			8
+#define   TSFS_INTR_MASK			0x000000ff
+
+/* Memory latency timer register */
+#define MLTR_ILK				_MMIO(MCHBAR_MIRROR_BASE + 0x1222)
+/* the unit of memory self-refresh latency time is 0.5us */
+#define   MLTR_WM2_MASK				REG_GENMASK(13, 8)
+#define   MLTR_WM1_MASK				REG_GENMASK(5, 0)
+
+#define CSIPLL0					_MMIO(MCHBAR_MIRROR_BASE + 0x2c10)
+#define DDRMPLL1				_MMIO(MCHBAR_MIRROR_BASE + 0x2c20)
+
+#define ILK_GDSR				_MMIO(MCHBAR_MIRROR_BASE + 0x2ca4)
+#define  ILK_GRDOM_FULL				(0 << 1)
+#define  ILK_GRDOM_RENDER			(1 << 1)
+#define  ILK_GRDOM_MEDIA			(3 << 1)
+#define  ILK_GRDOM_MASK				(3 << 1)
+#define  ILK_GRDOM_RESET_ENABLE			(1 << 0)
+
+#define BXT_D_CR_DRP0_DUNIT8			0x1000
+#define BXT_D_CR_DRP0_DUNIT9			0x1200
+#define   BXT_D_CR_DRP0_DUNIT_START		8
+#define   BXT_D_CR_DRP0_DUNIT_END		11
+#define BXT_D_CR_DRP0_DUNIT(x)			_MMIO(MCHBAR_MIRROR_BASE_SNB + \
+						      _PICK_EVEN((x) - 8, BXT_D_CR_DRP0_DUNIT8,\
+								 BXT_D_CR_DRP0_DUNIT9))
+#define   BXT_DRAM_RANK_MASK			0x3
+#define   BXT_DRAM_RANK_SINGLE			0x1
+#define   BXT_DRAM_RANK_DUAL			0x3
+#define   BXT_DRAM_WIDTH_MASK			(0x3 << 4)
+#define   BXT_DRAM_WIDTH_SHIFT			4
+#define   BXT_DRAM_WIDTH_X8			(0x0 << 4)
+#define   BXT_DRAM_WIDTH_X16			(0x1 << 4)
+#define   BXT_DRAM_WIDTH_X32			(0x2 << 4)
+#define   BXT_DRAM_WIDTH_X64			(0x3 << 4)
+#define   BXT_DRAM_SIZE_MASK			(0x7 << 6)
+#define   BXT_DRAM_SIZE_SHIFT			6
+#define   BXT_DRAM_SIZE_4GBIT			(0x0 << 6)
+#define   BXT_DRAM_SIZE_6GBIT			(0x1 << 6)
+#define   BXT_DRAM_SIZE_8GBIT			(0x2 << 6)
+#define   BXT_DRAM_SIZE_12GBIT			(0x3 << 6)
+#define   BXT_DRAM_SIZE_16GBIT			(0x4 << 6)
+#define   BXT_DRAM_TYPE_MASK			(0x7 << 22)
+#define   BXT_DRAM_TYPE_SHIFT			22
+#define   BXT_DRAM_TYPE_DDR3			(0x0 << 22)
+#define   BXT_DRAM_TYPE_LPDDR3			(0x1 << 22)
+#define   BXT_DRAM_TYPE_LPDDR4			(0x2 << 22)
+#define   BXT_DRAM_TYPE_DDR4			(0x4 << 22)
+
+#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4000)
+#define   DG1_DRAM_T_RDPRE_MASK			REG_GENMASK(16, 11)
+#define   DG1_DRAM_T_RP_MASK			REG_GENMASK(6, 0)
+#define MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x4004)
+#define   DG1_DRAM_T_RCD_MASK			REG_GENMASK(15, 9)
+#define   DG1_DRAM_T_RAS_MASK			REG_GENMASK(8, 1)
+
+#define SKL_MAD_INTER_CHANNEL_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5000)
+#define   SKL_DRAM_DDR_TYPE_MASK		REG_GENMASK(1, 0)
+#define   SKL_DRAM_DDR_TYPE_DDR4		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 0)
+#define   SKL_DRAM_DDR_TYPE_DDR3		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 1)
+#define   SKL_DRAM_DDR_TYPE_LPDDR3		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 2)
+#define   SKL_DRAM_DDR_TYPE_LPDDR4		REG_FIELD_PREP(SKL_DRAM_DDR_TYPE_MASK, 3)
+
+/* snb MCH registers for reading the DRAM channel configuration */
+#define MAD_DIMM_C0				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5004)
+#define MAD_DIMM_C1				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5008)
+#define MAD_DIMM_C2				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C)
+#define   MAD_DIMM_ECC_MASK			(0x3 << 24)
+#define   MAD_DIMM_ECC_OFF			(0x0 << 24)
+#define   MAD_DIMM_ECC_IO_ON_LOGIC_OFF		(0x1 << 24)
+#define   MAD_DIMM_ECC_IO_OFF_LOGIC_ON		(0x2 << 24)
+#define   MAD_DIMM_ECC_ON			(0x3 << 24)
+#define   MAD_DIMM_ENH_INTERLEAVE		(0x1 << 22)
+#define   MAD_DIMM_RANK_INTERLEAVE		(0x1 << 21)
+#define   MAD_DIMM_B_WIDTH_X16			(0x1 << 20) /* X8 chips if unset */
+#define   MAD_DIMM_A_WIDTH_X16			(0x1 << 19) /* X8 chips if unset */
+#define   MAD_DIMM_B_DUAL_RANK			(0x1 << 18)
+#define   MAD_DIMM_A_DUAL_RANK			(0x1 << 17)
+#define   MAD_DIMM_A_SELECT			(0x1 << 16)
+/* DIMM sizes are in multiples of 256mb. */
+#define   MAD_DIMM_B_SIZE_SHIFT			8
+#define   MAD_DIMM_B_SIZE_MASK			(0xff << MAD_DIMM_B_SIZE_SHIFT)
+#define   MAD_DIMM_A_SIZE_SHIFT			0
+#define   MAD_DIMM_A_SIZE_MASK			(0xff << MAD_DIMM_A_SIZE_SHIFT)
+
+#define SKL_MAD_DIMM_CH0_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x500C)
+#define SKL_MAD_DIMM_CH1_0_0_0_MCHBAR_MCMAIN	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5010)
+#define   SKL_DIMM_S_RANK_MASK			REG_GENMASK(26, 26)
+#define   SKL_DIMM_S_RANK_1			REG_FIELD_PREP(SKL_DIMM_S_RANK_MASK, 0)
+#define   SKL_DIMM_S_RANK_2			REG_FIELD_PREP(SKL_DIMM_S_RANK_MASK, 1)
+#define   SKL_DIMM_S_WIDTH_MASK			REG_GENMASK(25, 24)
+#define   SKL_DIMM_S_WIDTH_X8			REG_FIELD_PREP(SKL_DIMM_S_WIDTH_MASK, 0)
+#define   SKL_DIMM_S_WIDTH_X16			REG_FIELD_PREP(SKL_DIMM_S_WIDTH_MASK, 1)
+#define   SKL_DIMM_S_WIDTH_X32			REG_FIELD_PREP(SKL_DIMM_S_WIDTH_MASK, 2)
+#define   SKL_DIMM_S_SIZE_MASK			REG_GENMASK(21, 16)
+#define   SKL_DIMM_L_RANK_MASK			REG_GENMASK(10, 10)
+#define   SKL_DIMM_L_RANK_1			REG_FIELD_PREP(SKL_DIMM_L_RANK_MASK, 0)
+#define   SKL_DIMM_L_RANK_2			REG_FIELD_PREP(SKL_DIMM_L_RANK_MASK, 1)
+#define   SKL_DIMM_L_WIDTH_MASK			REG_GENMASK(9, 8)
+#define   SKL_DIMM_L_WIDTH_X8			REG_FIELD_PREP(SKL_DIMM_L_WIDTH_MASK, 0)
+#define   SKL_DIMM_L_WIDTH_X16			REG_FIELD_PREP(SKL_DIMM_L_WIDTH_MASK, 1)
+#define   SKL_DIMM_L_WIDTH_X32			REG_FIELD_PREP(SKL_DIMM_L_WIDTH_MASK, 2)
+#define   SKL_DIMM_L_SIZE_MASK			REG_GENMASK(5, 0)
+#define   ICL_DIMM_S_RANK_MASK			REG_GENMASK(27, 26)
+#define   ICL_DIMM_S_RANK_1			REG_FIELD_PREP(ICL_DIMM_S_RANK_MASK, 0)
+#define   ICL_DIMM_S_RANK_2			REG_FIELD_PREP(ICL_DIMM_S_RANK_MASK, 1)
+#define   ICL_DIMM_S_WIDTH_MASK			REG_GENMASK(25, 24)
+#define   ICL_DIMM_S_WIDTH_X8			REG_FIELD_PREP(ICL_DIMM_S_WIDTH_MASK, 0)
+#define   ICL_DIMM_S_WIDTH_X16			REG_FIELD_PREP(ICL_DIMM_S_WIDTH_MASK, 1)
+#define   ICL_DIMM_S_WIDTH_X32			REG_FIELD_PREP(ICL_DIMM_S_WIDTH_MASK, 2)
+#define   ICL_DIMM_S_SIZE_MASK			REG_GENMASK(22, 16)
+#define   ICL_DIMM_L_RANK_MASK			REG_GENMASK(10, 9)
+#define   ICL_DIMM_L_RANK_1			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 0)
+#define   ICL_DIMM_L_RANK_2			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 1)
+#define   ICL_DIMM_L_RANK_3			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 2)
+#define   ICL_DIMM_L_RANK_4			REG_FIELD_PREP(ICL_DIMM_L_RANK_MASK, 3)
+#define   ICL_DIMM_L_WIDTH_MASK			REG_GENMASK(8, 7)
+#define   ICL_DIMM_L_WIDTH_X8			REG_FIELD_PREP(ICL_DIMM_L_WIDTH_MASK, 0)
+#define   ICL_DIMM_L_WIDTH_X16			REG_FIELD_PREP(ICL_DIMM_L_WIDTH_MASK, 1)
+#define   ICL_DIMM_L_WIDTH_X32			REG_FIELD_PREP(ICL_DIMM_L_WIDTH_MASK, 2)
+#define   ICL_DIMM_L_SIZE_MASK			REG_GENMASK(6, 0)
+
+#define SA_PERF_STATUS_0_0_0_MCHBAR_PC		_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5918)
+#define  DG1_QCLK_RATIO_MASK			REG_GENMASK(9, 2)
+#define  DG1_QCLK_REFERENCE			REG_BIT(10)
+
+/*
+ * *_PACKAGE_POWER_SKU - SKU power and timing parameters.
+ */
+#define PCU_PACKAGE_POWER_SKU			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5930)
+#define   PKG_PKG_TDP				GENMASK_ULL(14, 0)
+#define   PKG_MIN_PWR				GENMASK_ULL(30, 16)
+#define   PKG_MAX_PWR				GENMASK_ULL(46, 32)
+#define   PKG_MAX_WIN				GENMASK_ULL(54, 48)
+#define     PKG_MAX_WIN_X			GENMASK_ULL(54, 53)
+#define     PKG_MAX_WIN_Y			GENMASK_ULL(52, 48)
+
+#define PCU_PACKAGE_POWER_SKU_UNIT		_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5938)
+#define   PKG_PWR_UNIT				REG_GENMASK(3, 0)
+#define   PKG_ENERGY_UNIT			REG_GENMASK(12, 8)
+#define   PKG_TIME_UNIT				REG_GENMASK(19, 16)
+#define PCU_PACKAGE_ENERGY_STATUS              _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x593c)
+
+#define GEN6_GT_PERF_STATUS			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5948)
+
+#define PCU_PACKAGE_TEMPERATURE			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5978)
+#define   TEMP_MASK				REG_GENMASK(7, 0)
+
+#define GEN6_RP_STATE_LIMITS			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5994)
+#define GEN6_RP_STATE_CAP			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5998)
+#define   RP0_CAP_MASK				REG_GENMASK(7, 0)
+#define   RP1_CAP_MASK				REG_GENMASK(15, 8)
+#define   RPN_CAP_MASK				REG_GENMASK(23, 16)
+
+#define GEN10_FREQ_INFO_REC			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5ef0)
+#define   RPE_MASK				REG_GENMASK(15, 8)
+#define PCU_PACKAGE_RAPL_LIMIT			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x59a0)
+#define   PKG_PWR_LIM_1				REG_GENMASK(14, 0)
+#define   PKG_PWR_LIM_1_EN			REG_BIT(15)
+#define   PKG_PWR_LIM_1_TIME			REG_GENMASK(23, 17)
+#define   PKG_PWR_LIM_1_TIME_X			REG_GENMASK(23, 22)
+#define   PKG_PWR_LIM_1_TIME_Y			REG_GENMASK(21, 17)
+
+/* snb MCH registers for priority tuning */
+#define MCH_SSKPD				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5d10)
+#define   SSKPD_NEW_WM0_MASK_HSW		REG_GENMASK64(63, 56)
+#define   SSKPD_WM4_MASK_HSW			REG_GENMASK64(40, 32)
+#define   SSKPD_WM3_MASK_HSW			REG_GENMASK64(28, 20)
+#define   SSKPD_WM2_MASK_HSW			REG_GENMASK64(19, 12)
+#define   SSKPD_WM1_MASK_HSW			REG_GENMASK64(11, 4)
+#define   SSKPD_OLD_WM0_MASK_HSW		REG_GENMASK64(3, 0)
+#define   SSKPD_WM3_MASK_SNB			REG_GENMASK(29, 24)
+#define   SSKPD_WM2_MASK_SNB			REG_GENMASK(21, 16)
+#define   SSKPD_WM1_MASK_SNB			REG_GENMASK(13, 8)
+#define   SSKPD_WM0_MASK_SNB			REG_GENMASK(5, 0)
+
+/* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */
+#define DCLK					_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04)
+#define SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU	_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04)
+#define   DG1_GEAR_TYPE				REG_BIT(16)
+
+/*
+ * Please see hsw_read_dcomp() and hsw_write_dcomp() before using this register,
+ * since on HSW we can't write to it using intel_uncore_write.
+ */
+#define D_COMP_HSW				_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5f0c)
+#define  D_COMP_RCOMP_IN_PROGRESS		(1 << 9)
+#define  D_COMP_COMP_FORCE			(1 << 8)
+#define  D_COMP_COMP_DISABLE			(1 << 0)
+
+#define BXT_GT_PERF_STATUS			_MMIO(MCHBAR_MIRROR_BASE_SNB + 0x7070)
+
+#endif /* __INTEL_MCHBAR_REGS */
diff --git a/include/drm/intel/pci_config.h b/include/drm/intel/pci_config.h
new file mode 100644
index 0000000000000..ebe040828e204
--- /dev/null
+++ b/include/drm/intel/pci_config.h
@@ -0,0 +1,110 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#ifndef __INTEL_PCI_CONFIG_H__
+#define __INTEL_PCI_CONFIG_H__
+
+/* PCI BARs */
+#define GEN2_GMADR_BAR				0
+#define GEN2_MMADR_BAR				1 /* MMIO+GTT, despite the name */
+#define GEN2_IO_BAR				2 /* 85x/865 */
+
+#define GEN3_MMADR_BAR				0 /* MMIO only */
+#define GEN3_IO_BAR				1
+#define GEN3_GMADR_BAR				2
+#define GEN3_GTTADR_BAR				3 /* GTT only */
+
+#define GEN4_GTTMMADR_BAR			0 /* MMIO+GTT */
+#define GEN4_GMADR_BAR				2
+#define GEN4_IO_BAR				4
+
+#define GEN12_LMEM_BAR				2
+
+static inline int intel_mmio_bar(int graphics_ver)
+{
+	switch (graphics_ver) {
+	case 2: return GEN2_MMADR_BAR;
+	case 3: return GEN3_MMADR_BAR;
+	default: return GEN4_GTTMMADR_BAR;
+	}
+}
+
+/* BSM in include/drm/intel/i915_drm.h */
+
+#define MCHBAR_I915				0x44
+#define MCHBAR_I965				0x48
+#define   MCHBAR_SIZE				(4 * 4096)
+
+#define DEVEN					0x54
+#define   DEVEN_MCHBAR_EN			(1 << 28)
+
+#define HPLLCC					0xc0 /* 85x only */
+#define   GC_CLOCK_CONTROL_MASK			(0x7 << 0)
+#define   GC_CLOCK_133_200			(0 << 0)
+#define   GC_CLOCK_100_200			(1 << 0)
+#define   GC_CLOCK_100_133			(2 << 0)
+#define   GC_CLOCK_133_266			(3 << 0)
+#define   GC_CLOCK_133_200_2			(4 << 0)
+#define   GC_CLOCK_133_266_2			(5 << 0)
+#define   GC_CLOCK_166_266			(6 << 0)
+#define   GC_CLOCK_166_250			(7 << 0)
+
+#define I915_GDRST				0xc0
+#define   GRDOM_FULL				(0 << 2)
+#define   GRDOM_RENDER				(1 << 2)
+#define   GRDOM_MEDIA				(3 << 2)
+#define   GRDOM_MASK				(3 << 2)
+#define   GRDOM_RESET_STATUS			(1 << 1)
+#define   GRDOM_RESET_ENABLE			(1 << 0)
+
+/* BSpec only has register offset, PCI device and bit found empirically */
+#define I830_CLOCK_GATE				0xc8 /* device 0 */
+#define   I830_L2_CACHE_CLOCK_GATE_DISABLE	(1 << 2)
+
+#define GCDGMBUS				0xcc
+
+#define GCFGC2					0xda
+#define GCFGC					0xf0 /* 915+ only */
+#define   GC_LOW_FREQUENCY_ENABLE		(1 << 7)
+#define   GC_DISPLAY_CLOCK_190_200_MHZ		(0 << 4)
+#define   GC_DISPLAY_CLOCK_333_320_MHZ		(4 << 4)
+#define   GC_DISPLAY_CLOCK_267_MHZ_PNV		(0 << 4)
+#define   GC_DISPLAY_CLOCK_333_MHZ_PNV		(1 << 4)
+#define   GC_DISPLAY_CLOCK_444_MHZ_PNV		(2 << 4)
+#define   GC_DISPLAY_CLOCK_200_MHZ_PNV		(5 << 4)
+#define   GC_DISPLAY_CLOCK_133_MHZ_PNV		(6 << 4)
+#define   GC_DISPLAY_CLOCK_167_MHZ_PNV		(7 << 4)
+#define   GC_DISPLAY_CLOCK_MASK			(7 << 4)
+#define   GM45_GC_RENDER_CLOCK_MASK		(0xf << 0)
+#define   GM45_GC_RENDER_CLOCK_266_MHZ		(8 << 0)
+#define   GM45_GC_RENDER_CLOCK_320_MHZ		(9 << 0)
+#define   GM45_GC_RENDER_CLOCK_400_MHZ		(0xb << 0)
+#define   GM45_GC_RENDER_CLOCK_533_MHZ		(0xc << 0)
+#define   I965_GC_RENDER_CLOCK_MASK		(0xf << 0)
+#define   I965_GC_RENDER_CLOCK_267_MHZ		(2 << 0)
+#define   I965_GC_RENDER_CLOCK_333_MHZ		(3 << 0)
+#define   I965_GC_RENDER_CLOCK_444_MHZ		(4 << 0)
+#define   I965_GC_RENDER_CLOCK_533_MHZ		(5 << 0)
+#define   I945_GC_RENDER_CLOCK_MASK		(7 << 0)
+#define   I945_GC_RENDER_CLOCK_166_MHZ		(0 << 0)
+#define   I945_GC_RENDER_CLOCK_200_MHZ		(1 << 0)
+#define   I945_GC_RENDER_CLOCK_250_MHZ		(3 << 0)
+#define   I945_GC_RENDER_CLOCK_400_MHZ		(5 << 0)
+#define   I915_GC_RENDER_CLOCK_MASK		(7 << 0)
+#define   I915_GC_RENDER_CLOCK_166_MHZ		(0 << 0)
+#define   I915_GC_RENDER_CLOCK_200_MHZ		(1 << 0)
+#define   I915_GC_RENDER_CLOCK_333_MHZ		(4 << 0)
+
+#define ASLE					0xe4
+#define ASLS					0xfc
+
+#define SWSCI					0xe8
+#define   SWSCI_SCISEL				(1 << 15)
+#define   SWSCI_GSSCIE				(1 << 0)
+
+/* legacy/combination backlight modes, also called LBB */
+#define LBPC					0xf4
+
+#endif /* __INTEL_PCI_CONFIG_H__ */
diff --git a/include/drm/intel/pciids.h b/include/drm/intel/pciids.h
index 33b91cb2e6845..e32ef763427c7 100644
--- a/include/drm/intel/pciids.h
+++ b/include/drm/intel/pciids.h
@@ -898,7 +898,11 @@
 
 /* CRI */
 #define INTEL_CRI_IDS(MACRO__, ...) \
-	MACRO__(0x674C, ## __VA_ARGS__)
+	MACRO__(0x674C, ## __VA_ARGS__), \
+	MACRO__(0x674D, ## __VA_ARGS__), \
+	MACRO__(0x674E, ## __VA_ARGS__), \
+	MACRO__(0x674F, ## __VA_ARGS__), \
+	MACRO__(0x6750, ## __VA_ARGS__)
 
 /* NVL-P */
 #define INTEL_NVLP_IDS(MACRO__, ...) \
diff --git a/include/drm/intel/vlv_iosf_sb_regs.h b/include/drm/intel/vlv_iosf_sb_regs.h
new file mode 100644
index 0000000000000..42d1def5534b1
--- /dev/null
+++ b/include/drm/intel/vlv_iosf_sb_regs.h
@@ -0,0 +1,192 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Copyright © 2022 Intel Corporation
+ */
+
+#ifndef _VLV_IOSF_SB_REGS_H_
+#define _VLV_IOSF_SB_REGS_H_
+
+enum vlv_iosf_sb_unit {
+	VLV_IOSF_SB_BUNIT,
+	VLV_IOSF_SB_CCK,
+	VLV_IOSF_SB_CCU,
+	VLV_IOSF_SB_DPIO,
+	VLV_IOSF_SB_DPIO_2,
+	VLV_IOSF_SB_FLISDSI,
+	VLV_IOSF_SB_GPIO,
+	VLV_IOSF_SB_NC,
+	VLV_IOSF_SB_PUNIT,
+};
+
+/* See configdb bunit SB addr map */
+#define BUNIT_REG_BISOC				0x11
+
+/* PUNIT_REG_*SSPM0 */
+#define   _SSPM0_SSC(val)			((val) << 0)
+#define   SSPM0_SSC_MASK			_SSPM0_SSC(0x3)
+#define   SSPM0_SSC_PWR_ON			_SSPM0_SSC(0x0)
+#define   SSPM0_SSC_CLK_GATE			_SSPM0_SSC(0x1)
+#define   SSPM0_SSC_RESET			_SSPM0_SSC(0x2)
+#define   SSPM0_SSC_PWR_GATE			_SSPM0_SSC(0x3)
+#define   _SSPM0_SSS(val)			((val) << 24)
+#define   SSPM0_SSS_MASK			_SSPM0_SSS(0x3)
+#define   SSPM0_SSS_PWR_ON			_SSPM0_SSS(0x0)
+#define   SSPM0_SSS_CLK_GATE			_SSPM0_SSS(0x1)
+#define   SSPM0_SSS_RESET			_SSPM0_SSS(0x2)
+#define   SSPM0_SSS_PWR_GATE			_SSPM0_SSS(0x3)
+
+/* PUNIT_REG_*SSPM1 */
+#define   SSPM1_FREQSTAT_SHIFT			24
+#define   SSPM1_FREQSTAT_MASK			(0x1f << SSPM1_FREQSTAT_SHIFT)
+#define   SSPM1_FREQGUAR_SHIFT			8
+#define   SSPM1_FREQGUAR_MASK			(0x1f << SSPM1_FREQGUAR_SHIFT)
+#define   SSPM1_FREQ_SHIFT			0
+#define   SSPM1_FREQ_MASK			(0x1f << SSPM1_FREQ_SHIFT)
+
+#define PUNIT_REG_VEDSSPM0			0x32
+#define PUNIT_REG_VEDSSPM1			0x33
+
+#define PUNIT_REG_DSPSSPM			0x36
+#define   DSPFREQSTAT_SHIFT_CHV			24
+#define   DSPFREQSTAT_MASK_CHV			(0x1f << DSPFREQSTAT_SHIFT_CHV)
+#define   DSPFREQGUAR_SHIFT_CHV			8
+#define   DSPFREQGUAR_MASK_CHV			(0x1f << DSPFREQGUAR_SHIFT_CHV)
+#define   DSPFREQSTAT_SHIFT			30
+#define   DSPFREQSTAT_MASK			(0x3 << DSPFREQSTAT_SHIFT)
+#define   DSPFREQGUAR_SHIFT			14
+#define   DSPFREQGUAR_MASK			(0x3 << DSPFREQGUAR_SHIFT)
+#define   DSP_MAXFIFO_PM5_STATUS		(1 << 22) /* chv */
+#define   DSP_AUTO_CDCLK_GATE_DISABLE		(1 << 7) /* chv */
+#define   DSP_MAXFIFO_PM5_ENABLE		(1 << 6) /* chv */
+#define   _DP_SSC(val, pipe)			((val) << (2 * (pipe)))
+#define   DP_SSC_MASK(pipe)			_DP_SSC(0x3, (pipe))
+#define   DP_SSC_PWR_ON(pipe)			_DP_SSC(0x0, (pipe))
+#define   DP_SSC_CLK_GATE(pipe)			_DP_SSC(0x1, (pipe))
+#define   DP_SSC_RESET(pipe)			_DP_SSC(0x2, (pipe))
+#define   DP_SSC_PWR_GATE(pipe)			_DP_SSC(0x3, (pipe))
+#define   _DP_SSS(val, pipe)			((val) << (2 * (pipe) + 16))
+#define   DP_SSS_MASK(pipe)			_DP_SSS(0x3, (pipe))
+#define   DP_SSS_PWR_ON(pipe)			_DP_SSS(0x0, (pipe))
+#define   DP_SSS_CLK_GATE(pipe)			_DP_SSS(0x1, (pipe))
+#define   DP_SSS_RESET(pipe)			_DP_SSS(0x2, (pipe))
+#define   DP_SSS_PWR_GATE(pipe)			_DP_SSS(0x3, (pipe))
+
+#define PUNIT_REG_ISPSSPM0			0x39
+#define PUNIT_REG_ISPSSPM1			0x3a
+
+#define PUNIT_REG_PWRGT_CTRL			0x60
+#define PUNIT_REG_PWRGT_STATUS			0x61
+#define   PUNIT_PWRGT_MASK(pw_idx)		(3 << ((pw_idx) * 2))
+#define   PUNIT_PWRGT_PWR_ON(pw_idx)		(0 << ((pw_idx) * 2))
+#define   PUNIT_PWRGT_CLK_GATE(pw_idx)		(1 << ((pw_idx) * 2))
+#define   PUNIT_PWRGT_RESET(pw_idx)		(2 << ((pw_idx) * 2))
+#define   PUNIT_PWRGT_PWR_GATE(pw_idx)		(3 << ((pw_idx) * 2))
+
+#define PUNIT_PWGT_IDX_RENDER			0
+#define PUNIT_PWGT_IDX_MEDIA			1
+#define PUNIT_PWGT_IDX_DISP2D			3
+#define PUNIT_PWGT_IDX_DPIO_CMN_BC		5
+#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_01	6
+#define PUNIT_PWGT_IDX_DPIO_TX_B_LANES_23	7
+#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_01	8
+#define PUNIT_PWGT_IDX_DPIO_TX_C_LANES_23	9
+#define PUNIT_PWGT_IDX_DPIO_RX0			10
+#define PUNIT_PWGT_IDX_DPIO_RX1			11
+#define PUNIT_PWGT_IDX_DPIO_CMN_D		12
+
+#define PUNIT_REG_GPU_LFM			0xd3
+#define PUNIT_REG_GPU_FREQ_REQ			0xd4
+#define PUNIT_REG_GPU_FREQ_STS			0xd8
+#define   GPLLENABLE				(1 << 4)
+#define   GENFREQSTATUS				(1 << 0)
+#define PUNIT_REG_MEDIA_TURBO_FREQ_REQ		0xdc
+#define PUNIT_REG_CZ_TIMESTAMP			0xce
+
+#define PUNIT_FUSE_BUS2				0xf6 /* bits 47:40 */
+#define PUNIT_FUSE_BUS1				0xf5 /* bits 55:48 */
+
+#define FB_GFX_FMAX_AT_VMAX_FUSE		0x136
+#define FB_GFX_FREQ_FUSE_MASK			0xff
+#define FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT	24
+#define FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT	16
+#define FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT	8
+
+#define FB_GFX_FMIN_AT_VMIN_FUSE		0x137
+#define FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT		8
+
+#define PUNIT_REG_DDR_SETUP2			0x139
+#define   FORCE_DDR_FREQ_REQ_ACK		(1 << 8)
+#define   FORCE_DDR_LOW_FREQ			(1 << 1)
+#define   FORCE_DDR_HIGH_FREQ			(1 << 0)
+
+#define PUNIT_GPU_STATUS_REG			0xdb
+#define PUNIT_GPU_STATUS_MAX_FREQ_SHIFT	16
+#define PUNIT_GPU_STATUS_MAX_FREQ_MASK		0xff
+#define PUNIT_GPU_STATIS_GFX_MIN_FREQ_SHIFT	8
+#define PUNIT_GPU_STATUS_GFX_MIN_FREQ_MASK	0xff
+
+#define PUNIT_GPU_DUTYCYCLE_REG		0xdf
+#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT	8
+#define PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK	0xff
+
+#define IOSF_NC_FB_GFX_FREQ_FUSE		0x1c
+#define   FB_GFX_MAX_FREQ_FUSE_SHIFT		3
+#define   FB_GFX_MAX_FREQ_FUSE_MASK		0x000007f8
+#define   FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT	11
+#define   FB_GFX_FGUARANTEED_FREQ_FUSE_MASK	0x0007f800
+#define IOSF_NC_FB_GFX_FMAX_FUSE_HI		0x34
+#define   FB_FMAX_VMIN_FREQ_HI_MASK		0x00000007
+#define IOSF_NC_FB_GFX_FMAX_FUSE_LO		0x30
+#define   FB_FMAX_VMIN_FREQ_LO_SHIFT		27
+#define   FB_FMAX_VMIN_FREQ_LO_MASK		0xf8000000
+
+#define VLV_TURBO_SOC_OVERRIDE		0x04
+#define   VLV_OVERRIDE_EN		1
+#define   VLV_SOC_TDP_EN		(1 << 1)
+#define   VLV_BIAS_CPU_125_SOC_875	(6 << 2)
+#define   CHV_BIAS_CPU_50_SOC_50	(3 << 2)
+
+/* vlv2 north clock has */
+#define CCK_FUSE_REG				0x8
+#define  CCK_FUSE_HPLL_FREQ_MASK		0x3
+#define CCK_REG_DSI_PLL_FUSE			0x44
+#define CCK_REG_DSI_PLL_CONTROL			0x48
+#define  DSI_PLL_VCO_EN				(1 << 31)
+#define  DSI_PLL_LDO_GATE			(1 << 30)
+#define  DSI_PLL_P1_POST_DIV_SHIFT		17
+#define  DSI_PLL_P1_POST_DIV_MASK		(0x1ff << 17)
+#define  DSI_PLL_P2_MUX_DSI0_DIV2		(1 << 13)
+#define  DSI_PLL_P3_MUX_DSI1_DIV2		(1 << 12)
+#define  DSI_PLL_MUX_MASK			(3 << 9)
+#define  DSI_PLL_MUX_DSI0_DSIPLL		(0 << 10)
+#define  DSI_PLL_MUX_DSI0_CCK			(1 << 10)
+#define  DSI_PLL_MUX_DSI1_DSIPLL		(0 << 9)
+#define  DSI_PLL_MUX_DSI1_CCK			(1 << 9)
+#define  DSI_PLL_CLK_GATE_MASK			(0xf << 5)
+#define  DSI_PLL_CLK_GATE_DSI0_DSIPLL		(1 << 8)
+#define  DSI_PLL_CLK_GATE_DSI1_DSIPLL		(1 << 7)
+#define  DSI_PLL_CLK_GATE_DSI0_CCK		(1 << 6)
+#define  DSI_PLL_CLK_GATE_DSI1_CCK		(1 << 5)
+#define  DSI_PLL_LOCK				(1 << 0)
+#define CCK_REG_DSI_PLL_DIVIDER			0x4c
+#define  DSI_PLL_LFSR				(1 << 31)
+#define  DSI_PLL_FRACTION_EN			(1 << 30)
+#define  DSI_PLL_FRAC_COUNTER_SHIFT		27
+#define  DSI_PLL_FRAC_COUNTER_MASK		(7 << 27)
+#define  DSI_PLL_USYNC_CNT_SHIFT		18
+#define  DSI_PLL_USYNC_CNT_MASK			(0x1ff << 18)
+#define  DSI_PLL_N1_DIV_SHIFT			16
+#define  DSI_PLL_N1_DIV_MASK			(3 << 16)
+#define  DSI_PLL_M1_DIV_SHIFT			0
+#define  DSI_PLL_M1_DIV_MASK			(0x1ff << 0)
+#define CCK_CZ_CLOCK_CONTROL			0x62
+#define CCK_GPLL_CLOCK_CONTROL			0x67
+#define CCK_DISPLAY_CLOCK_CONTROL		0x6b
+#define CCK_DISPLAY_REF_CLOCK_CONTROL		0x6c
+#define  CCK_TRUNK_FORCE_ON			(1 << 17)
+#define  CCK_TRUNK_FORCE_OFF			(1 << 16)
+#define  CCK_FREQUENCY_STATUS			(0x1f << 8)
+#define  CCK_FREQUENCY_STATUS_SHIFT		8
+#define  CCK_FREQUENCY_VALUES			(0x1f << 0)
+
+#endif /* _VLV_IOSF_SB_REGS_H_ */
diff --git a/include/drm/ttm/ttm_placement.h b/include/drm/ttm/ttm_placement.h
index b510a4812609f..ab2639e42c546 100644
--- a/include/drm/ttm/ttm_placement.h
+++ b/include/drm/ttm/ttm_placement.h
@@ -81,8 +81,8 @@
  * Structure indicating a possible place to put an object.
  */
 struct ttm_place {
-	unsigned	fpfn;
-	unsigned	lpfn;
+	uint64_t	fpfn;
+	uint64_t	lpfn;
 	uint32_t	mem_type;
 	uint32_t	flags;
 };
diff --git a/include/dt-bindings/arm/qcom,ids.h b/include/dt-bindings/arm/qcom,ids.h
index 336f7bb7188a8..1af73c0ad41cd 100644
--- a/include/dt-bindings/arm/qcom,ids.h
+++ b/include/dt-bindings/arm/qcom,ids.h
@@ -290,6 +290,7 @@
 #define QCOM_ID_IPQ5424			651
 #define QCOM_ID_QCM6690			657
 #define QCOM_ID_QCS6690			658
+#define QCOM_ID_SM7750			659
 #define QCOM_ID_SM8850			660
 #define QCOM_ID_IPQ5404			671
 #define QCOM_ID_QCS9100			667
@@ -297,13 +298,23 @@
 #define QCOM_ID_QCS8275			675
 #define QCOM_ID_QCS9075			676
 #define QCOM_ID_QCS615			680
+#define QCOM_ID_SA8797P			690
 #define QCOM_ID_CQ7790M			731
 #define QCOM_ID_CQ7790S			732
+#define QCOM_ID_CQ2390M			756
+#define QCOM_ID_CQ2390S			758
+#define QCOM_ID_IQ2390S			759
 #define QCOM_ID_IPQ5200			765
 #define QCOM_ID_IPQ5210			766
 #define QCOM_ID_QCF2200			767
 #define QCOM_ID_QCF3200			768
 #define QCOM_ID_QCF3210			769
+#define QCOM_ID_IPQ9620			770
+#define QCOM_ID_IPQ9650			771
+#define QCOM_ID_IPQ9610			778
+#define QCOM_ID_IPQ9630			779
+#define QCOM_ID_IPQ9640			780
+#define QCOM_ID_IPQ9670			781
 
 /*
  * The board type and revision information, used by Qualcomm bootloaders and
diff --git a/include/dt-bindings/clock/qcom,hawi-gcc.h b/include/dt-bindings/clock/qcom,hawi-gcc.h
new file mode 100644
index 0000000000000..6cd7fa0884f53
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,hawi-gcc.h
@@ -0,0 +1,253 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_GCC_HAWI_H
+#define _DT_BINDINGS_CLK_QCOM_GCC_HAWI_H
+
+/* GCC clocks */
+#define GCC_AGGRE_NOC_PCIE_AXI_CLK				0
+#define GCC_AGGRE_STARDUSTNOC_USB3_PRIM_AXI_CLK			1
+#define GCC_AGGRE_UFS_PHY_AXI_CLK				2
+#define GCC_BOOT_ROM_AHB_CLK					3
+#define GCC_CAM_BIST_MCLK_AHB_CLK				4
+#define GCC_CAMERA_AHB_CLK					5
+#define GCC_CAMERA_HF_AXI_CLK					6
+#define GCC_CAMERA_RSC_CORE_CLK					7
+#define GCC_CAMERA_SF_AXI_CLK					8
+#define GCC_CAMERA_XO_CLK					9
+#define GCC_CFG_NOC_PCIE_ANOC_AHB_CLK				10
+#define GCC_CFG_NOC_USB3_PRIM_AXI_CLK				11
+#define GCC_CNOC_PCIE_SF_AXI_CLK				12
+#define GCC_EVA_AHB_CLK						13
+#define GCC_EVA_AXI0_CLK					14
+#define GCC_EVA_AXI0C_CLK					15
+#define GCC_EVA_XO_CLK						16
+#define GCC_GP1_CLK						17
+#define GCC_GP1_CLK_SRC						18
+#define GCC_GP2_CLK						19
+#define GCC_GP2_CLK_SRC						20
+#define GCC_GP3_CLK						21
+#define GCC_GP3_CLK_SRC						22
+#define GCC_GPLL0						23
+#define GCC_GPLL0_OUT_EVEN					24
+#define GCC_GPLL4						25
+#define GCC_GPLL5						26
+#define GCC_GPLL7						27
+#define GCC_GPLL9						28
+#define GCC_GPU_CFG_AHB_CLK					29
+#define GCC_GPU_GEMNOC_GFX_CLK					30
+#define GCC_GPU_GPLL0_CLK_SRC					31
+#define GCC_GPU_GPLL0_DIV_CLK_SRC				32
+#define GCC_GPU_RSC_CORE_CLK					33
+#define GCC_GPU_SMMU_VOTE_CLK					34
+#define GCC_MMU_TCU_VOTE_CLK					35
+#define GCC_PCIE_0_AUX_CLK					36
+#define GCC_PCIE_0_AUX_CLK_SRC					37
+#define GCC_PCIE_0_CFG_AHB_CLK					38
+#define GCC_PCIE_0_MSTR_AXI_CLK					39
+#define GCC_PCIE_0_PHY_AUX_CLK					40
+#define GCC_PCIE_0_PHY_AUX_CLK_SRC				41
+#define GCC_PCIE_0_PHY_RCHNG_CLK				42
+#define GCC_PCIE_0_PHY_RCHNG_CLK_SRC				43
+#define GCC_PCIE_0_PIPE_CLK					44
+#define GCC_PCIE_0_PIPE_CLK_SRC					45
+#define GCC_PCIE_0_PIPE_DIV2_CLK				46
+#define GCC_PCIE_0_PIPE_DIV_CLK_SRC				47
+#define GCC_PCIE_0_SLV_AXI_CLK					48
+#define GCC_PCIE_0_SLV_Q2A_AXI_CLK				49
+#define GCC_PCIE_1_AUX_CLK					50
+#define GCC_PCIE_1_AUX_CLK_SRC					51
+#define GCC_PCIE_1_CFG_AHB_CLK					52
+#define GCC_PCIE_1_MSTR_AXI_CLK					53
+#define GCC_PCIE_1_PHY_AUX_CLK					54
+#define GCC_PCIE_1_PHY_AUX_CLK_SRC				55
+#define GCC_PCIE_1_PHY_RCHNG_CLK				56
+#define GCC_PCIE_1_PHY_RCHNG_CLK_SRC				57
+#define GCC_PCIE_1_PIPE_CLK					58
+#define GCC_PCIE_1_PIPE_CLK_SRC					59
+#define GCC_PCIE_1_PIPE_DIV2_CLK				60
+#define GCC_PCIE_1_PIPE_DIV_CLK_SRC				61
+#define GCC_PCIE_1_RSC_CORE_CLK					62
+#define GCC_PCIE_1_SLV_AXI_CLK					63
+#define GCC_PCIE_1_SLV_Q2A_AXI_CLK				64
+#define GCC_PCIE_RSC_CORE_CLK					65
+#define GCC_PCIE_RSCC_CFG_AHB_CLK				66
+#define GCC_PCIE_RSCC_XO_CLK					67
+#define GCC_PDM2_CLK						68
+#define GCC_PDM2_CLK_SRC					69
+#define GCC_PDM_AHB_CLK						70
+#define GCC_PDM_XO4_CLK						71
+#define GCC_QUPV3_I2C_CORE_CLK					72
+#define GCC_QUPV3_I2C_S0_CLK					73
+#define GCC_QUPV3_I2C_S0_CLK_SRC				74
+#define GCC_QUPV3_I2C_S1_CLK					75
+#define GCC_QUPV3_I2C_S1_CLK_SRC				76
+#define GCC_QUPV3_I2C_S2_CLK					77
+#define GCC_QUPV3_I2C_S2_CLK_SRC				78
+#define GCC_QUPV3_I2C_S3_CLK					79
+#define GCC_QUPV3_I2C_S3_CLK_SRC				80
+#define GCC_QUPV3_I2C_S4_CLK					81
+#define GCC_QUPV3_I2C_S4_CLK_SRC				82
+#define GCC_QUPV3_I2C_S_AHB_CLK					83
+#define GCC_QUPV3_WRAP1_CORE_2X_CLK				84
+#define GCC_QUPV3_WRAP1_CORE_CLK				85
+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK				86
+#define GCC_QUPV3_WRAP1_QSPI_REF_CLK_SRC			87
+#define GCC_QUPV3_WRAP1_S0_CLK					88
+#define GCC_QUPV3_WRAP1_S0_CLK_SRC				89
+#define GCC_QUPV3_WRAP1_S1_CLK					90
+#define GCC_QUPV3_WRAP1_S1_CLK_SRC				91
+#define GCC_QUPV3_WRAP1_S2_CLK					92
+#define GCC_QUPV3_WRAP1_S2_CLK_SRC				93
+#define GCC_QUPV3_WRAP1_S3_CLK					94
+#define GCC_QUPV3_WRAP1_S3_CLK_SRC				95
+#define GCC_QUPV3_WRAP1_S4_CLK					96
+#define GCC_QUPV3_WRAP1_S4_CLK_SRC				97
+#define GCC_QUPV3_WRAP1_S5_CLK					98
+#define GCC_QUPV3_WRAP1_S5_CLK_SRC				99
+#define GCC_QUPV3_WRAP1_S6_CLK					100
+#define GCC_QUPV3_WRAP1_S6_CLK_SRC				101
+#define GCC_QUPV3_WRAP1_S7_CLK					102
+#define GCC_QUPV3_WRAP1_S7_CLK_SRC				103
+#define GCC_QUPV3_WRAP2_CORE_2X_CLK				104
+#define GCC_QUPV3_WRAP2_CORE_CLK				105
+#define GCC_QUPV3_WRAP2_S0_CLK					106
+#define GCC_QUPV3_WRAP2_S0_CLK_SRC				107
+#define GCC_QUPV3_WRAP2_S1_CLK					108
+#define GCC_QUPV3_WRAP2_S1_CLK_SRC				109
+#define GCC_QUPV3_WRAP2_S2_CLK					110
+#define GCC_QUPV3_WRAP2_S2_CLK_SRC				111
+#define GCC_QUPV3_WRAP2_S3_CLK					112
+#define GCC_QUPV3_WRAP2_S3_CLK_SRC				113
+#define GCC_QUPV3_WRAP2_S4_CLK					114
+#define GCC_QUPV3_WRAP2_S4_CLK_SRC				115
+#define GCC_QUPV3_WRAP3_CORE_2X_CLK				116
+#define GCC_QUPV3_WRAP3_CORE_CLK				117
+#define GCC_QUPV3_WRAP3_QSPI_REF_CLK				118
+#define GCC_QUPV3_WRAP3_QSPI_REF_CLK_SRC			119
+#define GCC_QUPV3_WRAP3_S0_CLK					120
+#define GCC_QUPV3_WRAP3_S0_CLK_SRC				121
+#define GCC_QUPV3_WRAP3_S1_CLK					122
+#define GCC_QUPV3_WRAP3_S1_CLK_SRC				123
+#define GCC_QUPV3_WRAP3_S2_CLK					124
+#define GCC_QUPV3_WRAP3_S2_CLK_SRC				125
+#define GCC_QUPV3_WRAP3_S3_CLK					126
+#define GCC_QUPV3_WRAP3_S3_CLK_SRC				127
+#define GCC_QUPV3_WRAP3_S4_CLK					128
+#define GCC_QUPV3_WRAP3_S4_CLK_SRC				129
+#define GCC_QUPV3_WRAP3_S5_CLK					130
+#define GCC_QUPV3_WRAP3_S5_CLK_SRC				131
+#define GCC_QUPV3_WRAP4_CORE_2X_CLK				132
+#define GCC_QUPV3_WRAP4_CORE_CLK				133
+#define GCC_QUPV3_WRAP4_S0_CLK					134
+#define GCC_QUPV3_WRAP4_S0_CLK_SRC				135
+#define GCC_QUPV3_WRAP4_S1_CLK					136
+#define GCC_QUPV3_WRAP4_S1_CLK_SRC				137
+#define GCC_QUPV3_WRAP4_S2_CLK					138
+#define GCC_QUPV3_WRAP4_S2_CLK_SRC				139
+#define GCC_QUPV3_WRAP4_S3_CLK					140
+#define GCC_QUPV3_WRAP4_S3_CLK_SRC				141
+#define GCC_QUPV3_WRAP4_S4_CLK					142
+#define GCC_QUPV3_WRAP4_S4_CLK_SRC				143
+#define GCC_QUPV3_WRAP_1_M_AXI_CLK				144
+#define GCC_QUPV3_WRAP_1_S_AHB_CLK				145
+#define GCC_QUPV3_WRAP_2_M_AHB_CLK				146
+#define GCC_QUPV3_WRAP_2_S_AHB_CLK				147
+#define GCC_QUPV3_WRAP_3_M_AHB_CLK				148
+#define GCC_QUPV3_WRAP_3_S_AHB_CLK				149
+#define GCC_QUPV3_WRAP_4_M_AHB_CLK				150
+#define GCC_QUPV3_WRAP_4_S_AHB_CLK				151
+#define GCC_SDCC2_AHB_CLK					152
+#define GCC_SDCC2_APPS_CLK					153
+#define GCC_SDCC2_APPS_CLK_SRC					154
+#define GCC_SDCC4_AHB_CLK					155
+#define GCC_SDCC4_APPS_CLK					156
+#define GCC_SDCC4_APPS_CLK_SRC					157
+#define GCC_UFS_PHY_AHB_CLK					158
+#define GCC_UFS_PHY_AXI_CLK					159
+#define GCC_UFS_PHY_AXI_CLK_SRC					160
+#define GCC_UFS_PHY_ICE_CORE_CLK				161
+#define GCC_UFS_PHY_ICE_CORE_CLK_SRC				162
+#define GCC_UFS_PHY_PHY_AUX_CLK					163
+#define GCC_UFS_PHY_PHY_AUX_CLK_SRC				164
+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK				165
+#define GCC_UFS_PHY_RX_SYMBOL_0_CLK_SRC				166
+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK				167
+#define GCC_UFS_PHY_RX_SYMBOL_1_CLK_SRC				168
+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK				169
+#define GCC_UFS_PHY_TX_SYMBOL_0_CLK_SRC				170
+#define GCC_UFS_PHY_UNIPRO_5_CORE_CLK				171
+#define GCC_UFS_PHY_UNIPRO_5_CORE_CLK_SRC			172
+#define GCC_USB30_PRIM_MASTER_CLK				173
+#define GCC_USB30_PRIM_MASTER_CLK_SRC				174
+#define GCC_USB30_PRIM_MOCK_UTMI_CLK				175
+#define GCC_USB30_PRIM_MOCK_UTMI_CLK_SRC			176
+#define GCC_USB30_PRIM_MOCK_UTMI_POSTDIV_CLK_SRC		177
+#define GCC_USB30_PRIM_SLEEP_CLK				178
+#define GCC_USB3_PRIM_PHY_AUX_CLK				179
+#define GCC_USB3_PRIM_PHY_AUX_CLK_SRC				180
+#define GCC_USB3_PRIM_PHY_COM_AUX_CLK				181
+#define GCC_USB3_PRIM_PHY_PIPE_CLK				182
+#define GCC_USB3_PRIM_PHY_PIPE_CLK_SRC				183
+#define GCC_VIDEO_AHB_CLK					184
+#define GCC_VIDEO_AXI0_CLK					185
+#define GCC_VIDEO_AXI0C_CLK					186
+#define GCC_VIDEO_XO_CLK					187
+
+/* GCC power domains */
+#define GCC_PCIE_0_GDSC						0
+#define GCC_PCIE_0_PHY_GDSC					1
+#define GCC_PCIE_1_GDSC						2
+#define GCC_PCIE_1_PHY_GDSC					3
+#define GCC_UFS_MEM_PHY_GDSC					4
+#define GCC_UFS_PHY_GDSC					5
+#define GCC_USB30_PRIM_GDSC					6
+#define GCC_USB3_PHY_GDSC					7
+
+/* GCC resets */
+#define GCC_CAMERA_BCR						0
+#define GCC_EVA_AXI0_CLK_ARES					1
+#define GCC_EVA_AXI0C_CLK_ARES					2
+#define GCC_EVA_BCR						3
+#define GCC_GPU_BCR						4
+#define GCC_PCIE_0_BCR						5
+#define GCC_PCIE_0_LINK_DOWN_BCR				6
+#define GCC_PCIE_0_NOCSR_COM_PHY_BCR				7
+#define GCC_PCIE_0_PHY_BCR					8
+#define GCC_PCIE_0_PHY_NOCSR_COM_PHY_BCR			9
+#define GCC_PCIE_1_BCR						10
+#define GCC_PCIE_1_LINK_DOWN_BCR				11
+#define GCC_PCIE_1_NOCSR_COM_PHY_BCR				12
+#define GCC_PCIE_1_PHY_BCR					13
+#define GCC_PCIE_1_PHY_NOCSR_COM_PHY_BCR			14
+#define GCC_PCIE_PHY_BCR					15
+#define GCC_PCIE_PHY_CFG_AHB_BCR				16
+#define GCC_PCIE_PHY_COM_BCR					17
+#define GCC_PCIE_RSCC_BCR					18
+#define GCC_PDM_BCR						19
+#define GCC_QUPV3_WRAPPER_1_BCR					20
+#define GCC_QUPV3_WRAPPER_2_BCR					21
+#define GCC_QUPV3_WRAPPER_3_BCR					22
+#define GCC_QUPV3_WRAPPER_4_BCR					23
+#define GCC_QUPV3_WRAPPER_I2C_BCR				24
+#define GCC_QUSB2PHY_PRIM_BCR					25
+#define GCC_QUSB2PHY_SEC_BCR					26
+#define GCC_SDCC2_BCR						27
+#define GCC_SDCC4_BCR						28
+#define GCC_TCSR_PCIE_BCR					29
+#define GCC_UFS_PHY_BCR						30
+#define GCC_USB30_PRIM_BCR					31
+#define GCC_USB3_DP_PHY_PRIM_BCR				32
+#define GCC_USB3_DP_PHY_SEC_BCR					33
+#define GCC_USB3_PHY_PRIM_BCR					34
+#define GCC_USB3_PHY_SEC_BCR					35
+#define GCC_USB3PHY_PHY_PRIM_BCR				36
+#define GCC_USB3PHY_PHY_SEC_BCR					37
+#define GCC_VIDEO_AXI0_CLK_ARES					38
+#define GCC_VIDEO_AXI0C_CLK_ARES				39
+#define GCC_VIDEO_BCR						40
+#define GCC_VIDEO_XO_CLK_ARES					41
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,hawi-tcsrcc.h b/include/dt-bindings/clock/qcom,hawi-tcsrcc.h
new file mode 100644
index 0000000000000..957bc5f75bb74
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,hawi-tcsrcc.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_TCSR_CC_HAWI_H
+#define _DT_BINDINGS_CLK_QCOM_TCSR_CC_HAWI_H
+
+/* TCSR_CC clocks */
+#define TCSR_PCIE_0_CLKREF_EN					0
+#define TCSR_PCIE_1_CLKREF_EN					1
+#define TCSR_UFS_CLKREF_EN					2
+#define TCSR_USB2_CLKREF_EN					3
+#define TCSR_USB3_CLKREF_EN					4
+
+#endif
diff --git a/include/dt-bindings/clock/qcom,ipq5332-cmn-pll.h b/include/dt-bindings/clock/qcom,ipq5332-cmn-pll.h
new file mode 100644
index 0000000000000..172330e43669e
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,ipq5332-cmn-pll.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLK_QCOM_IPQ5332_CMN_PLL_H
+#define _DT_BINDINGS_CLK_QCOM_IPQ5332_CMN_PLL_H
+
+/* CMN PLL core clock. */
+#define IPQ5332_CMN_PLL_CLK			0
+
+/* The output clocks from CMN PLL of IPQ5332. */
+#define IPQ5332_XO_24MHZ_CLK			1
+#define IPQ5332_SLEEP_32KHZ_CLK			2
+#define IPQ5332_PCS_31P25MHZ_CLK		3
+#define IPQ5332_NSS_300MHZ_CLK			4
+#define IPQ5332_PPE_200MHZ_CLK			5
+#define IPQ5332_ETH_50MHZ_CLK			6
+#endif
diff --git a/include/dt-bindings/clock/qcom,ipq9650-gcc.h b/include/dt-bindings/clock/qcom,ipq9650-gcc.h
new file mode 100644
index 0000000000000..afd17c00d96ee
--- /dev/null
+++ b/include/dt-bindings/clock/qcom,ipq9650-gcc.h
@@ -0,0 +1,172 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_CLOCK_IPQ_GCC_IPQ9650_H
+#define _DT_BINDINGS_CLOCK_IPQ_GCC_IPQ9650_H
+
+#define GCC_ADSS_PWM_CLK					0
+#define GCC_ADSS_PWM_CLK_SRC					1
+#define GCC_ANOC_PCIE0_1LANE_M_CLK				2
+#define GCC_ANOC_PCIE0_1LANE_S_CLK				3
+#define GCC_ANOC_PCIE1_2LANE_M_CLK				4
+#define GCC_ANOC_PCIE1_2LANE_S_CLK				5
+#define GCC_ANOC_PCIE2_2LANE_M_CLK				6
+#define GCC_ANOC_PCIE2_2LANE_S_CLK				7
+#define GCC_ANOC_PCIE3_2LANE_M_CLK				8
+#define GCC_ANOC_PCIE3_2LANE_S_CLK				9
+#define GCC_ANOC_PCIE4_1LANE_M_CLK				10
+#define GCC_ANOC_PCIE4_1LANE_S_CLK				11
+#define GCC_CMN_12GPLL_AHB_CLK					12
+#define GCC_CMN_12GPLL_APU_CLK					13
+#define GCC_CMN_12GPLL_SYS_CLK					14
+#define GCC_CMN_LDO_CLK						15
+#define GCC_MDIO_AHB_CLK					16
+#define GCC_NSSCC_CLK						17
+#define GCC_NSSCFG_CLK						18
+#define GCC_NSSNOC_ATB_CLK					19
+#define GCC_NSSNOC_MEMNOC_1_CLK					20
+#define GCC_NSSNOC_MEMNOC_BFDCD_CLK_SRC				21
+#define GCC_NSSNOC_MEMNOC_CLK					22
+#define GCC_NSSNOC_MEMNOC_DIV_CLK_SRC				23
+#define GCC_NSSNOC_NSSCC_CLK					24
+#define GCC_NSSNOC_PCNOC_1_CLK					25
+#define GCC_NSSNOC_QOSGEN_REF_CLK				26
+#define GCC_NSSNOC_SNOC_1_CLK					27
+#define GCC_NSSNOC_SNOC_CLK					28
+#define GCC_NSSNOC_TIMEOUT_REF_CLK				29
+#define GCC_NSSNOC_XO_DCD_CLK					30
+#define GCC_NSS_TS_CLK						31
+#define GCC_NSS_TS_CLK_SRC					32
+#define GCC_PCIE0_AHB_CLK					33
+#define GCC_PCIE0_AUX_CLK					34
+#define GCC_PCIE0_AXI_M_CLK					35
+#define GCC_PCIE0_AXI_M_CLK_SRC					36
+#define GCC_PCIE0_AXI_S_BRIDGE_CLK				37
+#define GCC_PCIE0_AXI_S_CLK					38
+#define GCC_PCIE0_AXI_S_CLK_SRC					39
+#define GCC_PCIE0_PIPE_CLK					40
+#define GCC_PCIE0_PIPE_CLK_SRC					41
+#define GCC_PCIE0_RCHNG_CLK					42
+#define GCC_PCIE0_RCHNG_CLK_SRC					43
+#define GCC_PCIE1_AHB_CLK					44
+#define GCC_PCIE1_AUX_CLK					45
+#define GCC_PCIE1_AXI_M_CLK					46
+#define GCC_PCIE1_AXI_M_CLK_SRC					47
+#define GCC_PCIE1_AXI_S_BRIDGE_CLK				48
+#define GCC_PCIE1_AXI_S_CLK					49
+#define GCC_PCIE1_AXI_S_CLK_SRC					50
+#define GCC_PCIE1_PIPE_CLK					51
+#define GCC_PCIE1_PIPE_CLK_SRC					52
+#define GCC_PCIE1_RCHNG_CLK					53
+#define GCC_PCIE1_RCHNG_CLK_SRC					54
+#define GCC_PCIE2_AHB_CLK					55
+#define GCC_PCIE2_AUX_CLK					56
+#define GCC_PCIE2_AXI_M_CLK					57
+#define GCC_PCIE2_AXI_M_CLK_SRC					58
+#define GCC_PCIE2_AXI_S_BRIDGE_CLK				59
+#define GCC_PCIE2_AXI_S_CLK					60
+#define GCC_PCIE2_AXI_S_CLK_SRC					61
+#define GCC_PCIE2_PIPE_CLK					62
+#define GCC_PCIE2_PIPE_CLK_SRC					63
+#define GCC_PCIE2_RCHNG_CLK					64
+#define GCC_PCIE2_RCHNG_CLK_SRC					65
+#define GCC_PCIE3_AHB_CLK					66
+#define GCC_PCIE3_AUX_CLK					67
+#define GCC_PCIE3_AXI_M_CLK					68
+#define GCC_PCIE3_AXI_M_CLK_SRC					69
+#define GCC_PCIE3_AXI_S_BRIDGE_CLK				70
+#define GCC_PCIE3_AXI_S_CLK					71
+#define GCC_PCIE3_AXI_S_CLK_SRC					72
+#define GCC_PCIE3_PIPE_CLK					73
+#define GCC_PCIE3_PIPE_CLK_SRC					74
+#define GCC_PCIE3_RCHNG_CLK					75
+#define GCC_PCIE3_RCHNG_CLK_SRC					76
+#define GCC_PCIE4_AHB_CLK					77
+#define GCC_PCIE4_AUX_CLK					78
+#define GCC_PCIE4_AXI_M_CLK					79
+#define GCC_PCIE4_AXI_M_CLK_SRC					80
+#define GCC_PCIE4_AXI_S_BRIDGE_CLK				81
+#define GCC_PCIE4_AXI_S_CLK					82
+#define GCC_PCIE4_AXI_S_CLK_SRC					83
+#define GCC_PCIE4_PIPE_CLK					84
+#define GCC_PCIE4_PIPE_CLK_SRC					85
+#define GCC_PCIE4_RCHNG_CLK					86
+#define GCC_PCIE4_RCHNG_CLK_SRC					87
+#define GCC_PCIE_AUX_CLK_SRC					88
+#define GCC_PCNOC_BFDCD_CLK_SRC					89
+#define GCC_QDSS_AT_CLK						90
+#define GCC_QDSS_AT_CLK_SRC					91
+#define GCC_QDSS_DAP_CLK					92
+#define GCC_QDSS_TSCTR_CLK_SRC					93
+#define GCC_QPIC_AHB_CLK					94
+#define GCC_QPIC_CLK						95
+#define GCC_QPIC_CLK_SRC					96
+#define GCC_QPIC_IO_MACRO_CLK					97
+#define GCC_QPIC_IO_MACRO_CLK_SRC				98
+#define GCC_QPIC_SLEEP_CLK					99
+#define GCC_QUPV3_2X_CORE_CLK					100
+#define GCC_QUPV3_2X_CORE_CLK_SRC				101
+#define GCC_QUPV3_AHB_MST_CLK					102
+#define GCC_QUPV3_AHB_SLV_CLK					103
+#define GCC_QUPV3_CORE_CLK					104
+#define GCC_QUPV3_SLEEP_CLK					105
+#define GCC_QUPV3_WRAP_SE0_CLK					106
+#define GCC_QUPV3_WRAP_SE0_CLK_SRC				107
+#define GCC_QUPV3_WRAP_SE1_CLK					108
+#define GCC_QUPV3_WRAP_SE1_CLK_SRC				109
+#define GCC_QUPV3_WRAP_SE2_CLK					110
+#define GCC_QUPV3_WRAP_SE2_CLK_SRC				111
+#define GCC_QUPV3_WRAP_SE3_CLK					112
+#define GCC_QUPV3_WRAP_SE3_CLK_SRC				113
+#define GCC_QUPV3_WRAP_SE4_CLK					114
+#define GCC_QUPV3_WRAP_SE4_CLK_SRC				115
+#define GCC_QUPV3_WRAP_SE5_CLK					116
+#define GCC_QUPV3_WRAP_SE5_CLK_SRC				117
+#define GCC_QUPV3_WRAP_SE6_CLK					118
+#define GCC_QUPV3_WRAP_SE6_CLK_SRC				119
+#define GCC_QUPV3_WRAP_SE7_CLK					120
+#define GCC_QUPV3_WRAP_SE7_CLK_SRC				121
+#define GCC_SDCC1_AHB_CLK					122
+#define GCC_SDCC1_APPS_CLK					123
+#define GCC_SDCC1_APPS_CLK_SRC					124
+#define GCC_SDCC1_ICE_CORE_CLK					125
+#define GCC_SDCC1_ICE_CORE_CLK_SRC				126
+#define GCC_SLEEP_CLK_SRC					127
+#define GCC_SNOC_USB_CLK					128
+#define GCC_SYSTEM_NOC_BFDCD_CLK_SRC				129
+#define GCC_TLMM_AHB_CLK					130
+#define GCC_TLMM_CLK						131
+#define GCC_UNIPHY0_AHB_CLK					132
+#define GCC_UNIPHY0_SYS_CLK					133
+#define GCC_UNIPHY1_AHB_CLK					134
+#define GCC_UNIPHY1_SYS_CLK					135
+#define GCC_UNIPHY2_AHB_CLK					136
+#define GCC_UNIPHY2_SYS_CLK					137
+#define GCC_UNIPHY_SYS_CLK_SRC					138
+#define GCC_USB0_AUX_CLK					139
+#define GCC_USB0_AUX_CLK_SRC					140
+#define GCC_USB0_EUD_AT_CLK					141
+#define GCC_USB0_MASTER_CLK					142
+#define GCC_USB0_MASTER_CLK_SRC					143
+#define GCC_USB0_MOCK_UTMI_CLK					144
+#define GCC_USB0_MOCK_UTMI_CLK_SRC				145
+#define GCC_USB0_MOCK_UTMI_DIV_CLK_SRC				146
+#define GCC_USB0_PHY_CFG_AHB_CLK				147
+#define GCC_USB0_PIPE_CLK					148
+#define GCC_USB0_PIPE_CLK_SRC					149
+#define GCC_USB0_SLEEP_CLK					150
+#define GCC_USB1_MASTER_CLK					151
+#define GCC_USB1_MOCK_UTMI_CLK					152
+#define GCC_USB1_MOCK_UTMI_CLK_SRC				153
+#define GCC_USB1_MOCK_UTMI_DIV_CLK_SRC				154
+#define GCC_USB1_PHY_CFG_AHB_CLK				155
+#define GCC_USB1_SLEEP_CLK					156
+#define GCC_XO_CLK_SRC						157
+#define GPLL0							158
+#define GPLL0_MAIN						159
+#define GPLL2							160
+#define GPLL2_OUT_MAIN						161
+#define GPLL4							162
+#endif
diff --git a/include/dt-bindings/clock/qcom,rpmh.h b/include/dt-bindings/clock/qcom,rpmh.h
index 0a7d1be0d1246..2d62d5d0b08de 100644
--- a/include/dt-bindings/clock/qcom,rpmh.h
+++ b/include/dt-bindings/clock/qcom,rpmh.h
@@ -33,5 +33,7 @@
 #define RPMH_HWKM_CLK				24
 #define RPMH_QLINK_CLK				25
 #define RPMH_QLINK_CLK_A			26
+#define RPMH_LN_BB_CLK4				27
+#define RPMH_LN_BB_CLK4_A			28
 
 #endif
diff --git a/include/dt-bindings/clock/r8a73a4-clock.h b/include/dt-bindings/clock/r8a73a4-clock.h
index 655440a3e7c68..028ecef81451c 100644
--- a/include/dt-bindings/clock/r8a73a4-clock.h
+++ b/include/dt-bindings/clock/r8a73a4-clock.h
@@ -23,6 +23,8 @@
 #define R8A73A4_CLK_ZX		13
 #define R8A73A4_CLK_ZS		14
 #define R8A73A4_CLK_HP		15
+#define R8A73A4_CLK_ZTR		16
+#define R8A73A4_CLK_ZT		17
 
 /* MSTP1 */
 #define R8A73A4_CLK_TMU0	25
diff --git a/include/dt-bindings/clock/r8a7740-clock.h b/include/dt-bindings/clock/r8a7740-clock.h
index 1b3fdb39cc426..8a8816b2ff6ac 100644
--- a/include/dt-bindings/clock/r8a7740-clock.h
+++ b/include/dt-bindings/clock/r8a7740-clock.h
@@ -24,6 +24,8 @@
 #define R8A7740_CLK_ZB		14
 #define R8A7740_CLK_M3		15
 #define R8A7740_CLK_CP		16
+#define R8A7740_CLK_ZTR		17
+#define R8A7740_CLK_ZT		18
 
 /* MSTP1 */
 #define R8A7740_CLK_CEU21	28
diff --git a/include/dt-bindings/clock/rockchip,rk3588-cru.h b/include/dt-bindings/clock/rockchip,rk3588-cru.h
index 0c7d3ca2d5bc0..7528034cff56b 100644
--- a/include/dt-bindings/clock/rockchip,rk3588-cru.h
+++ b/include/dt-bindings/clock/rockchip,rk3588-cru.h
@@ -734,6 +734,10 @@
 #define PCLK_AV1_PRE			719
 #define HCLK_SDIO_PRE			720
 #define PCLK_VO1GRF			721
+#define I2S0_8CH_MCLKOUT_TO_IO		722
+#define I2S1_8CH_MCLKOUT_TO_IO		723
+#define I2S2_2CH_MCLKOUT_TO_IO		724
+#define I2S3_2CH_MCLKOUT_TO_IO		725
 
 /* scmi-clocks indices */
 
diff --git a/include/dt-bindings/firmware/qcom,scm.h b/include/dt-bindings/firmware/qcom,scm.h
index 6de8b08e1e799..0d29d8d4829ce 100644
--- a/include/dt-bindings/firmware/qcom,scm.h
+++ b/include/dt-bindings/firmware/qcom,scm.h
@@ -35,5 +35,6 @@
 #define QCOM_SCM_VMID_NAV		0x2B
 #define QCOM_SCM_VMID_TVM		0x2D
 #define QCOM_SCM_VMID_OEMVM		0x31
+#define QCOM_SCM_VMID_CP_ADSP_SHARED	0x33
 
 #endif
diff --git a/include/dt-bindings/gpio/nvidia,tegra238-gpio.h b/include/dt-bindings/gpio/nvidia,tegra238-gpio.h
new file mode 100644
index 0000000000000..8a616a1df54c0
--- /dev/null
+++ b/include/dt-bindings/gpio/nvidia,tegra238-gpio.h
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause */
+/* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. */
+
+/*
+ * This header provides constants for binding nvidia,tegra238-gpio*.
+ *
+ * The first cell in Tegra's GPIO specifier is the GPIO ID. The macros below
+ * provide names for this.
+ *
+ * The second cell contains standard flag values specified in gpio.h.
+ */
+
+#ifndef _DT_BINDINGS_GPIO_TEGRA238_GPIO_H
+#define _DT_BINDINGS_GPIO_TEGRA238_GPIO_H
+
+#include <dt-bindings/gpio/gpio.h>
+
+/* GPIOs implemented by main GPIO controller */
+#define TEGRA238_MAIN_GPIO_PORT_A   0
+#define TEGRA238_MAIN_GPIO_PORT_B   1
+#define TEGRA238_MAIN_GPIO_PORT_C   2
+#define TEGRA238_MAIN_GPIO_PORT_D   3
+#define TEGRA238_MAIN_GPIO_PORT_E   4
+#define TEGRA238_MAIN_GPIO_PORT_F   5
+#define TEGRA238_MAIN_GPIO_PORT_G   6
+#define TEGRA238_MAIN_GPIO_PORT_H   7
+#define TEGRA238_MAIN_GPIO_PORT_J   8
+#define TEGRA238_MAIN_GPIO_PORT_K   9
+#define TEGRA238_MAIN_GPIO_PORT_L  10
+#define TEGRA238_MAIN_GPIO_PORT_M  11
+#define TEGRA238_MAIN_GPIO_PORT_N  12
+#define TEGRA238_MAIN_GPIO_PORT_P  13
+#define TEGRA238_MAIN_GPIO_PORT_Q  14
+#define TEGRA238_MAIN_GPIO_PORT_R  15
+#define TEGRA238_MAIN_GPIO_PORT_S  16
+#define TEGRA238_MAIN_GPIO_PORT_T  17
+#define TEGRA238_MAIN_GPIO_PORT_U  18
+#define TEGRA238_MAIN_GPIO_PORT_V  19
+#define TEGRA238_MAIN_GPIO_PORT_W  20
+#define TEGRA238_MAIN_GPIO_PORT_X  21
+
+#define TEGRA238_MAIN_GPIO(port, offset) \
+	((TEGRA238_MAIN_GPIO_PORT_##port * 8) + (offset))
+
+/* GPIOs implemented by AON GPIO controller */
+#define TEGRA238_AON_GPIO_PORT_AA 0
+#define TEGRA238_AON_GPIO_PORT_BB 1
+#define TEGRA238_AON_GPIO_PORT_CC 2
+#define TEGRA238_AON_GPIO_PORT_DD 3
+#define TEGRA238_AON_GPIO_PORT_EE 4
+#define TEGRA238_AON_GPIO_PORT_FF 5
+#define TEGRA238_AON_GPIO_PORT_GG 6
+#define TEGRA238_AON_GPIO_PORT_HH 7
+
+#define TEGRA238_AON_GPIO(port, offset) \
+	((TEGRA238_AON_GPIO_PORT_##port * 8) + (offset))
+
+#endif
diff --git a/include/dt-bindings/interconnect/qcom,eliza-rpmh.h b/include/dt-bindings/interconnect/qcom,eliza-rpmh.h
index 95db2fe647de5..dfe99feefb279 100644
--- a/include/dt-bindings/interconnect/qcom,eliza-rpmh.h
+++ b/include/dt-bindings/interconnect/qcom,eliza-rpmh.h
@@ -57,6 +57,7 @@
 #define SLAVE_PCIE_ANOC_CFG			27
 #define SLAVE_QDSS_STM				28
 #define SLAVE_TCU				29
+#define SLAVE_SDCC_1				30
 
 #define MASTER_GEM_NOC_CNOC			0
 #define MASTER_GEM_NOC_PCIE_SNOC		1
diff --git a/include/dt-bindings/interconnect/qcom,hawi-rpmh.h b/include/dt-bindings/interconnect/qcom,hawi-rpmh.h
new file mode 100644
index 0000000000000..a8b649679846c
--- /dev/null
+++ b/include/dt-bindings/interconnect/qcom,hawi-rpmh.h
@@ -0,0 +1,165 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_HAWI_H
+#define __DT_BINDINGS_INTERCONNECT_QCOM_HAWI_H
+
+#define MASTER_QSPI_0				0
+#define MASTER_QUP_2				1
+#define MASTER_QUP_3				2
+#define MASTER_QUP_4				3
+#define MASTER_CRYPTO				4
+#define MASTER_IPA				5
+#define MASTER_QUP_1				6
+#define MASTER_SOCCP_PROC			7
+#define MASTER_QDSS_ETR				8
+#define MASTER_QDSS_ETR_1			9
+#define MASTER_SDCC_2				10
+#define MASTER_SDCC_4				11
+#define MASTER_UFS_MEM				12
+#define MASTER_USB3				13
+#define SLAVE_A1NOC_SNOC			14
+
+#define MASTER_DDR_EFF_VETO			0
+#define MASTER_QUP_CORE_0			1
+#define MASTER_QUP_CORE_1			2
+#define MASTER_QUP_CORE_2			3
+#define MASTER_QUP_CORE_3			4
+#define MASTER_QUP_CORE_4			5
+#define SLAVE_DDR_EFF_VETO			6
+#define SLAVE_QUP_CORE_0			7
+#define SLAVE_QUP_CORE_1			8
+#define SLAVE_QUP_CORE_2			9
+#define SLAVE_QUP_CORE_3			10
+#define SLAVE_QUP_CORE_4			11
+
+#define MASTER_GEM_NOC_CNOC			0
+#define MASTER_GEM_NOC_PCIE_SNOC		1
+#define SLAVE_AOSS				2
+#define SLAVE_IPA_CFG				3
+#define SLAVE_IPC_ROUTER_FENCE			4
+#define SLAVE_SOCCP				5
+#define SLAVE_TME_CFG				6
+#define SLAVE_CNOC_CFG				7
+#define SLAVE_DDRSS_CFG				8
+#define SLAVE_IMEM				9
+#define SLAVE_PCIE_0				10
+#define SLAVE_PCIE_1				11
+
+#define MASTER_GIC				0
+#define MASTER_GPU_TCU				1
+#define MASTER_SYS_TCU				2
+#define MASTER_APPSS_PROC			3
+#define MASTER_GFX3D				4
+#define MASTER_LPASS_GEM_NOC			5
+#define MASTER_MSS_PROC				6
+#define MASTER_MNOC_HF_MEM_NOC			7
+#define MASTER_MNOC_SF_MEM_NOC			8
+#define MASTER_COMPUTE_NOC			9
+#define MASTER_ANOC_PCIE_GEM_NOC		10
+#define MASTER_QPACE				11
+#define MASTER_SNOC_SF_MEM_NOC			12
+#define MASTER_WLAN_Q6				13
+#define SLAVE_GEM_NOC_CNOC			14
+#define SLAVE_LLCC				15
+#define SLAVE_MEM_NOC_PCIE_SNOC			16
+
+#define MASTER_LPIAON_NOC_LLCLPI_NOC		0
+#define SLAVE_LPASS_LPI_CC			1
+#define SLAVE_LLCC_ISLAND			2
+#define SLAVE_SERVICE_LLCLPI_NOC		3
+#define SLAVE_SERVICE_LLCLPI_NOC_CHIPCX		4
+
+#define MASTER_LPIAON_NOC			0
+#define SLAVE_LPASS_GEM_NOC			1
+
+#define MASTER_LPASS_LPINOC			0
+#define SLAVE_LPIAON_NOC_LLCLPI_NOC		1
+#define SLAVE_LPIAON_NOC_LPASS_AG_NOC		2
+
+#define MASTER_LPASS_PROC			0
+#define SLAVE_LPICX_NOC_LPIAON_NOC		1
+
+#define MASTER_LLCC				0
+#define MASTER_DDR_RT				1
+#define SLAVE_EBI1				2
+#define SLAVE_DDR_RT				3
+
+#define MASTER_CAMNOC_HF			0
+#define MASTER_CAMNOC_NRT_ICP_SF		1
+#define MASTER_CAMNOC_RT_CDM_SF			2
+#define MASTER_CAMNOC_SF			3
+#define MASTER_MDP				4
+#define MASTER_MDSS_DCP				5
+#define MASTER_CDSP_HCP				6
+#define MASTER_VIDEO_CV_PROC			7
+#define MASTER_VIDEO_EVA			8
+#define MASTER_VIDEO_MVP			9
+#define MASTER_VIDEO_V_PROC			10
+#define SLAVE_MNOC_HF_MEM_NOC			11
+#define SLAVE_MNOC_SF_MEM_NOC			12
+
+#define MASTER_CDSP_PROC			0
+#define SLAVE_CDSP_MEM_NOC			1
+
+#define MASTER_PCIE_ANOC_CFG			0
+#define MASTER_PCIE_0				1
+#define MASTER_PCIE_1				2
+#define SLAVE_ANOC_PCIE_GEM_NOC			3
+#define SLAVE_SERVICE_PCIE_ANOC			4
+
+#define MASTER_CFG_CENTER			0
+#define MASTER_CFG_EAST				1
+#define MASTER_CFG_MM				2
+#define MASTER_CFG_NORTH			3
+#define MASTER_CFG_SOUTH			4
+#define MASTER_CFG_SOUTHWEST			5
+#define SLAVE_AHB2PHY_SOUTH			6
+#define SLAVE_BOOT_ROM				7
+#define SLAVE_CAMERA_CFG			8
+#define SLAVE_CLK_CTL				9
+#define SLAVE_CRYPTO_CFG			10
+#define SLAVE_DISPLAY_CFG			11
+#define SLAVE_EVA_CFG				12
+#define SLAVE_GFX3D_CFG				13
+#define SLAVE_I2C				14
+#define SLAVE_IMEM_CFG				15
+#define SLAVE_IPC_ROUTER_CFG			16
+#define SLAVE_IRIS_CFG				17
+#define SLAVE_CNOC_MSS				18
+#define SLAVE_PCIE_0_CFG			19
+#define SLAVE_PCIE_1_CFG			20
+#define SLAVE_PRNG				21
+#define SLAVE_QSPI_0				22
+#define SLAVE_QUP_1				23
+#define SLAVE_QUP_2				24
+#define SLAVE_QUP_3				25
+#define SLAVE_QUP_4				26
+#define SLAVE_SDCC_2				27
+#define SLAVE_SDCC_4				28
+#define SLAVE_TLMM				29
+#define SLAVE_UFS_MEM_CFG			30
+#define SLAVE_USB3				31
+#define SLAVE_VSENSE_CTRL_CFG			32
+#define SLAVE_PCIE_ANOC_CFG			33
+#define SLAVE_QDSS_CFG				34
+#define SLAVE_QDSS_STM				35
+#define SLAVE_TCSR				36
+#define SLAVE_TCU				37
+
+#define MASTER_CNOC_STARDUST			0
+#define SLAVE_STARDUST_CENTER_CFG		1
+#define SLAVE_STARDUST_EAST_CFG			2
+#define SLAVE_STARDUST_MM_CFG			3
+#define SLAVE_STARDUST_NORTH_CFG		4
+#define SLAVE_STARDUST_SOUTH_CFG		5
+#define SLAVE_STARDUST_SOUTHWEST_CFG		6
+
+#define MASTER_A1NOC_SNOC			0
+#define MASTER_APSS_NOC				1
+#define MASTER_CNOC_SNOC			2
+#define SLAVE_SNOC_GEM_NOC_SF			3
+
+#endif
diff --git a/include/dt-bindings/interconnect/qcom,nord-rpmh.h b/include/dt-bindings/interconnect/qcom,nord-rpmh.h
new file mode 100644
index 0000000000000..5bdce6a9bab79
--- /dev/null
+++ b/include/dt-bindings/interconnect/qcom,nord-rpmh.h
@@ -0,0 +1,217 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_NORD_H
+#define __DT_BINDINGS_INTERCONNECT_QCOM_NORD_H
+
+#define MASTER_QSPI_0				0
+#define MASTER_SAILSS_MD1			1
+#define MASTER_QUP_3				2
+#define SLAVE_A1NOC_SNOC			3
+
+#define MASTER_QUP_2				0
+#define MASTER_CRYPTO_CORE0			1
+#define MASTER_CRYPTO_CORE1			2
+#define MASTER_CRYPTO_CORE2			3
+#define MASTER_SDCC_4				4
+#define MASTER_UFS_MEM				5
+#define MASTER_USB2				6
+#define MASTER_USB3_0				7
+#define MASTER_USB3_1				8
+#define SLAVE_A1NOC_HSCNOC			9
+
+#define MASTER_IPA				0
+#define MASTER_SOCCP_AGGR_NOC			1
+#define MASTER_QDSS_ETR				2
+#define MASTER_QDSS_ETR_1			3
+#define SLAVE_A2NOC_SNOC			4
+
+#define MASTER_QUP_0				0
+#define MASTER_QUP_1				1
+#define MASTER_EMAC_0				2
+#define MASTER_EMAC_1				3
+#define SLAVE_A2NOC_HSCNOC			4
+
+#define MASTER_QUP_CORE_0			0
+#define MASTER_QUP_CORE_1			1
+#define MASTER_QUP_CORE_2			2
+#define MASTER_QUP_CORE_3			3
+#define SLAVE_QUP_CORE_0			4
+#define SLAVE_QUP_CORE_1			5
+#define SLAVE_QUP_CORE_2			6
+#define SLAVE_QUP_CORE_3			7
+
+#define MASTER_CNOC_CFG				0
+#define SLAVE_PS_ETH_0				1
+#define SLAVE_PS_ETH_1				2
+#define SLAVE_SHS_SERVER			3
+#define SLAVE_AHB2PHY_0				4
+#define SLAVE_AHB2PHY_1				5
+#define SLAVE_AHB2PHY_2				6
+#define SLAVE_AHB2PHY_3				7
+#define SLAVE_AHB2PHY_ETH_0			8
+#define SLAVE_AHB2PHY_ETH_1			9
+#define SLAVE_CAMERA_CFG			10
+#define SLAVE_CLK_CTL				11
+#define SLAVE_CRYPTO_0_CFG			12
+#define SLAVE_CRYPTO_1_CFG			13
+#define SLAVE_CRYPTO_2_CFG			14
+#define SLAVE_DISPLAY_1_CFG			15
+#define SLAVE_DISPLAY_CFG			16
+#define SLAVE_DPRX0				17
+#define SLAVE_DPRX1				18
+#define SLAVE_EVA_CFG				19
+#define SLAVE_GFX3D_CFG				20
+#define SLAVE_GFX3D_1_CFG			21
+#define SLAVE_I2C				22
+#define SLAVE_IMEM_CFG				23
+#define SLAVE_MCW_PCIE				24
+#define SLAVE_MM_RSCC				25
+#define SLAVE_NE_CLK_CTL			26
+#define SLAVE_NSPSS0_CFG			27
+#define SLAVE_NSPSS1_CFG			28
+#define SLAVE_NSPSS2_CFG			29
+#define SLAVE_NSPSS3_CFG			30
+#define SLAVE_NW_CLK_CTL			31
+#define SLAVE_PRNG				32
+#define SLAVE_QDSS_CFG				33
+#define SLAVE_QSPI_0				34
+#define SLAVE_QUP_0				35
+#define SLAVE_QUP_3				36
+#define SLAVE_QUP_1				37
+#define SLAVE_QUP_2				38
+#define SLAVE_SAFEDMA_CFG			39
+#define SLAVE_SDCC_4				40
+#define SLAVE_SE_CLK_CTL			41
+#define SLAVE_TCSR				42
+#define SLAVE_TLMM				43
+#define SLAVE_TSC_CFG				44
+#define SLAVE_UFS_MEM_CFG			45
+#define SLAVE_USB2				46
+#define SLAVE_USB3_0				47
+#define SLAVE_USB3_1				48
+#define SLAVE_VENUS_CFG				49
+#define SLAVE_COMPUTENOC_CFG			50
+#define SLAVE_PCIE_NOC_CFG			51
+#define SLAVE_QTC_CFG				52
+#define SLAVE_QDSS_STM				53
+#define SLAVE_SYS_TCU0_CFG			54
+#define SLAVE_SYS_TCU1_CFG			55
+#define SLAVE_SYS_TCU2_CFG			56
+
+#define MASTER_MM_RSCC				0
+#define MASTER_HSCNOC_CNOC			1
+#define SLAVE_AOSS				2
+#define SLAVE_HBCU				3
+#define SLAVE_IPA_CFG				4
+#define SLAVE_IPC_ROUTER_CFG			5
+#define SLAVE_SOCCP				6
+#define SLAVE_TME_CFG				7
+#define SLAVE_PCIE_DMA				8
+#define SLAVE_CNOC_CFG				9
+#define SLAVE_DDRSS_CFG				10
+#define SLAVE_IMEM				11
+
+#define MASTER_HPASS_PROC_0			0
+#define MASTER_HPASS_PROC_1			1
+#define MASTER_HPASS_PROC_2			2
+#define SLAVE_HPASS_AGNOC_AUDIO			3
+
+#define MASTER_GPU_TCU				0
+#define MASTER_QTC_TCU				1
+#define MASTER_SYS_TCU_0			2
+#define MASTER_SYS_TCU_1			3
+#define MASTER_SYS_TCU_2			4
+#define MASTER_APPSS_PROC			5
+#define MASTER_A1NOC_TILE_HSCNOC		6
+#define MASTER_A2NOC_TILE_HSCNOC		7
+#define MASTER_GFX3D				8
+#define MASTER_GFX3D_1				9
+#define MASTER_HPASS_ADAS_HSCNOC		10
+#define MASTER_HPASS_AUDIO_HSCNOC		11
+#define MASTER_MNOC_HF_MEM_NOC			12
+#define MASTER_MNOC_SF_MEM_NOC			13
+#define MASTER_NSP0_HSCNOC			14
+#define MASTER_NSP1_HSCNOC			15
+#define MASTER_NSP2_HSCNOC			16
+#define MASTER_NSP3_HSCNOC			17
+#define MASTER_ANOC_PCIE_GEM_NOC		18
+#define MASTER_SAILSS_MD0_HSCNOC		19
+#define MASTER_SNOC_SF_MEM_NOC			20
+#define MASTER_GIC				21
+#define SLAVE_HSCNOC_CNOC			22
+#define SLAVE_LLCC				23
+#define SLAVE_MEM_NOC_PCIE_SNOC			24
+
+#define MASTER_LLCC				0
+#define SLAVE_EBI1				1
+
+#define MASTER_CAMNOC_HF			0
+#define MASTER_CAMNOC_NRT_ICP_SF		1
+#define MASTER_CAMNOC_RT_CDM_SF			2
+#define MASTER_CAMNOC_SF			3
+#define MASTER_DPRX0				4
+#define MASTER_DPRX1				5
+#define MASTER_MDP0				6
+#define MASTER_MDP1				7
+#define MASTER_VIDEO_CV_PROC			8
+#define MASTER_VIDEO_EVA			9
+#define MASTER_VIDEO_MVP0			10
+#define MASTER_VIDEO_MVP1			11
+#define MASTER_VIDEO_V_PROC			12
+#define SLAVE_MNOC_HF_MEM_NOC			13
+#define SLAVE_MNOC_SF_MEM_NOC			14
+
+#define MASTER_NSP0_PROC			0
+#define SLAVE_NSP0_HSC_NOC			1
+
+#define MASTER_NSP1_PROC			0
+#define SLAVE_NSP1_HSC_NOC			1
+
+#define MASTER_NSP2_PROC			0
+#define SLAVE_NSP2_HSC_NOC			1
+
+#define MASTER_NSP3_PROC			0
+#define SLAVE_NSP3_HSC_NOC			1
+
+#define MASTER_PCIE_NOC_CFG			0
+#define SLAVE_PCIE_AHB2PHY_CFG			1
+#define SLAVE_PCIE_CFG_0			2
+#define SLAVE_PCIE_CFG_1			3
+#define SLAVE_PCIE_CFG_2			4
+#define SLAVE_PCIE_CFG_3			5
+#define SLAVE_PCIE_DMA_0_CFG			6
+#define SLAVE_PCIE_DMA_1_CFG			7
+#define SLAVE_PCIE_DMA_2_CFG			8
+
+#define MASTER_PCIE_DMA_0			0
+#define MASTER_PCIE_DMA_1			1
+#define MASTER_PCIE_DMA_2			2
+#define MASTER_PCIE_0				3
+#define MASTER_PCIE_1				4
+#define MASTER_PCIE_2				5
+#define MASTER_PCIE_3				6
+#define SLAVE_PCIE_HSCNOC			7
+#define SLAVE_PCIE_OBNOC_DMA			8
+
+#define MASTER_CNOC_PCIE_DMA			0
+#define MASTER_ANOC_PCIE_HSCNOC			1
+#define MASTER_PCIE_IBNOC_DMA			2
+#define SLAVE_PCIE_DMA_0			3
+#define SLAVE_PCIE_DMA_1			4
+#define SLAVE_PCIE_DMA_2			5
+#define SLAVE_PCIE_0				6
+#define SLAVE_PCIE_1				7
+#define SLAVE_PCIE_2				8
+#define SLAVE_PCIE_3				9
+
+#define MASTER_A1NOC_SNOC			0
+#define MASTER_A2NOC_SNOC			1
+#define MASTER_CNOC_SNOC			2
+#define MASTER_NSINOC_SNOC			3
+#define MASTER_SAFE_DMA				4
+#define SLAVE_SNOC_HSCNOC_SF			5
+
+#endif
diff --git a/include/dt-bindings/interconnect/qcom,shikra.h b/include/dt-bindings/interconnect/qcom,shikra.h
new file mode 100644
index 0000000000000..a42ea22ee162c
--- /dev/null
+++ b/include/dt-bindings/interconnect/qcom,shikra.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef __DT_BINDINGS_INTERCONNECT_QCOM_SHIKRA_H
+#define __DT_BINDINGS_INTERCONNECT_QCOM_SHIKRA_H
+
+#define MASTER_QUP_CORE_0			0
+#define SLAVE_QUP_CORE_0			1
+
+#define SNOC_CNOC_MAS				0
+#define MASTER_QDSS_DAP				1
+#define SLAVE_AHB2PHY_USB			2
+#define SLAVE_APSS_THROTTLE_CFG			3
+#define SLAVE_AUDIO				4
+#define SLAVE_BOOT_ROM				5
+#define SLAVE_CAMERA_NRT_THROTTLE_CFG		6
+#define SLAVE_CAMERA_CFG			7
+#define SLAVE_CDSP_THROTTLE_CFG			8
+#define SLAVE_CLK_CTL				9
+#define SLAVE_DSP_CFG				10
+#define SLAVE_RBCPR_CX_CFG			11
+#define SLAVE_RBCPR_MX_CFG			12
+#define SLAVE_CRYPTO_0_CFG			13
+#define SLAVE_DDR_SS_CFG			14
+#define SLAVE_DISPLAY_CFG			15
+#define SLAVE_EMAC0_CFG				16
+#define SLAVE_EMAC1_CFG				17
+#define SLAVE_GPU_CFG				18
+#define SLAVE_GPU_THROTTLE_CFG			19
+#define SLAVE_HWKM				20
+#define SLAVE_IMEM_CFG				21
+#define SLAVE_MAPSS				22
+#define SLAVE_MDSP_MPU_CFG			23
+#define SLAVE_MESSAGE_RAM			24
+#define SLAVE_MSS				25
+#define SLAVE_PCIE_CFG				26
+#define SLAVE_PDM				27
+#define SLAVE_PIMEM_CFG				28
+#define SLAVE_PKA_WRAPPER_CFG			29
+#define SLAVE_PMIC_ARB				30
+#define SLAVE_QDSS_CFG				31
+#define SLAVE_QM_CFG				32
+#define SLAVE_QM_MPU_CFG			33
+#define SLAVE_QPIC				34
+#define SLAVE_QUP_0				35
+#define SLAVE_RPM				36
+#define SLAVE_SDCC_1				37
+#define SLAVE_SDCC_2				38
+#define SLAVE_SECURITY				39
+#define SLAVE_SNOC_CFG				40
+#define SNOC_SF_THROTTLE_CFG			41
+#define SLAVE_TLMM				42
+#define SLAVE_TSCSS				43
+#define SLAVE_USB2				44
+#define SLAVE_USB3				45
+#define SLAVE_VENUS_CFG				46
+#define SLAVE_VENUS_THROTTLE_CFG		47
+#define SLAVE_VSENSE_CTRL_CFG			48
+#define SLAVE_SERVICE_CNOC			49
+
+#define MASTER_LLCC				0
+#define SLAVE_EBI_CH0				1
+
+#define MASTER_GRAPHICS_3D			0
+#define MASTER_MNOC_HF_MEM_NOC			1
+#define MASTER_ANOC_PCIE_MEM_NOC		2
+#define MASTER_SNOC_SF_MEM_NOC			3
+#define MASTER_AMPSS_M0				4
+#define MASTER_SYS_TCU				5
+#define SLAVE_LLCC				6
+#define SLAVE_MEMNOC_SNOC			7
+#define SLAVE_MEM_NOC_PCIE_SNOC			8
+
+#define MASTER_CAMNOC_SF			0
+#define MASTER_VIDEO_P0				1
+#define MASTER_VIDEO_PROC			2
+#define SLAVE_MMNRT_VIRT			3
+
+#define MASTER_CAMNOC_HF			0
+#define MASTER_MDP_PORT0			1
+#define MASTER_MMRT_VIRT			2
+#define SLAVE_MM_MEMNOC				3
+
+#define MASTER_SNOC_CFG				0
+#define MASTER_TIC				1
+#define MASTER_ANOC_SNOC			2
+#define MASTER_MEMNOC_PCIE			3
+#define MASTER_MEMNOC_SNOC			4
+#define MASTER_PIMEM				5
+#define MASTER_PCIE2_0				6
+#define MASTER_QDSS_BAM				7
+#define MASTER_QPIC				8
+#define MASTER_QUP_0				9
+#define CNOC_SNOC_MAS				10
+#define MASTER_AUDIO				11
+#define MASTER_EMAC_0				12
+#define MASTER_EMAC_1				13
+#define MASTER_QDSS_ETR				14
+#define MASTER_SDCC_1				15
+#define MASTER_SDCC_2				16
+#define MASTER_USB2_0				17
+#define MASTER_USB3				18
+#define MASTER_CRYPTO_CORE0			19
+#define SLAVE_APPSS				20
+#define SLAVE_MCUSS				21
+#define SLAVE_WCSS				22
+#define SLAVE_MEMNOC_SF				23
+#define SNOC_CNOC_SLV				24
+#define SLAVE_BOOTIMEM				25
+#define SLAVE_OCIMEM				26
+#define SLAVE_PIMEM				27
+#define SLAVE_SERVICE_SNOC			28
+#define SLAVE_PCIE2_0				29
+#define SLAVE_QDSS_STM				30
+#define SLAVE_TCU				31
+#define SLAVE_PCIE_MEMNOC			32
+#define SLAVE_ANOC_SNOC				33
+
+#endif
diff --git a/include/dt-bindings/leds/common.h b/include/dt-bindings/leds/common.h
index 4f017bea01233..b7bafbaf7df3a 100644
--- a/include/dt-bindings/leds/common.h
+++ b/include/dt-bindings/leds/common.h
@@ -1,4 +1,4 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */
 /*
  * This header provides macros for the common LEDs device tree bindings.
  *
diff --git a/include/dt-bindings/memory/nvidia,tegra238-mc.h b/include/dt-bindings/memory/nvidia,tegra238-mc.h
new file mode 100644
index 0000000000000..be24c0eb3f15e
--- /dev/null
+++ b/include/dt-bindings/memory/nvidia,tegra238-mc.h
@@ -0,0 +1,74 @@
+/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */
+/* Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. */
+
+#ifndef DT_BINDINGS_MEMORY_TEGRA238_MC_H
+#define DT_BINDINGS_MEMORY_TEGRA238_MC_H
+
+/* special clients */
+#define TEGRA238_SID_INVALID		0x0
+#define TEGRA238_SID_PASSTHROUGH	0x7f
+
+/* ISO stream IDs */
+#define TEGRA238_SID_ISO_NVDISPLAY	0x1
+#define TEGRA238_SID_ISO_APE0		0x2
+#define TEGRA238_SID_ISO_APE1		0x3
+
+/* NISO stream IDs */
+#define TEGRA238_SID_AON		0x1
+#define TEGRA238_SID_BPMP		0x2
+#define TEGRA238_SID_ETR		0x3
+#define TEGRA238_SID_FDE		0x4
+#define TEGRA238_SID_HC		0x5
+#define TEGRA238_SID_HDA		0x6
+#define TEGRA238_SID_NVDEC		0x7
+#define TEGRA238_SID_NVDISPLAY		0x8
+#define TEGRA238_SID_NVENC		0x9
+#define TEGRA238_SID_OFA		0xa
+#define TEGRA238_SID_PCIE0		0xb
+#define TEGRA238_SID_PCIE1		0xc
+#define TEGRA238_SID_PCIE2		0xd
+#define TEGRA238_SID_PCIE3		0xe
+#define TEGRA238_SID_HWMP_PMA		0xf
+#define TEGRA238_SID_PSC		0x10
+#define TEGRA238_SID_SDMMC1A		0x11
+#define TEGRA238_SID_SDMMC4A		0x12
+#define TEGRA238_SID_SES_SE0		0x13
+#define TEGRA238_SID_SES_SE1		0x14
+#define TEGRA238_SID_SES_SE2		0x15
+#define TEGRA238_SID_SEU1_SE0		0x16
+#define TEGRA238_SID_SEU1_SE1		0x17
+#define TEGRA238_SID_SEU1_SE2		0x18
+#define TEGRA238_SID_TSEC		0x19
+#define TEGRA238_SID_UFSHC		0x1a
+#define TEGRA238_SID_VIC		0x1b
+#define TEGRA238_SID_XUSB_HOST		0x1c
+#define TEGRA238_SID_XUSB_DEV		0x1d
+#define TEGRA238_SID_GPCDMA_0		0x1e
+#define TEGRA238_SID_SMMU_TEST		0x1f
+
+/* Host1x virtualization clients. */
+#define TEGRA238_SID_HOST1X_CTX0	0x20
+#define TEGRA238_SID_HOST1X_CTX1	0x21
+#define TEGRA238_SID_HOST1X_CTX2	0x22
+#define TEGRA238_SID_HOST1X_CTX3	0x23
+#define TEGRA238_SID_HOST1X_CTX4	0x24
+#define TEGRA238_SID_HOST1X_CTX5	0x25
+#define TEGRA238_SID_HOST1X_CTX6	0x26
+#define TEGRA238_SID_HOST1X_CTX7	0x27
+
+#define TEGRA238_SID_XUSB_VF0		0x28
+#define TEGRA238_SID_XUSB_VF1		0x29
+#define TEGRA238_SID_XUSB_VF2		0x2a
+#define TEGRA238_SID_XUSB_VF3		0x2b
+
+/* Host1x command buffers */
+#define TEGRA238_SID_HC_VM0		0x2c
+#define TEGRA238_SID_HC_VM1		0x2d
+#define TEGRA238_SID_HC_VM2		0x2e
+#define TEGRA238_SID_HC_VM3		0x2f
+#define TEGRA238_SID_HC_VM4		0x30
+#define TEGRA238_SID_HC_VM5		0x31
+#define TEGRA238_SID_HC_VM6		0x32
+#define TEGRA238_SID_HC_VM7		0x33
+
+#endif
diff --git a/include/dt-bindings/memory/tegra114-mc.h b/include/dt-bindings/memory/tegra114-mc.h
index dfe99c8a5ba57..5e0d6a1b91f2a 100644
--- a/include/dt-bindings/memory/tegra114-mc.h
+++ b/include/dt-bindings/memory/tegra114-mc.h
@@ -40,4 +40,71 @@
 #define TEGRA114_MC_RESET_VDE		14
 #define TEGRA114_MC_RESET_VI		15
 
+#define TEGRA114_MC_PTCR		0
+#define TEGRA114_MC_DISPLAY0A		1
+#define TEGRA114_MC_DISPLAY0AB		2
+#define TEGRA114_MC_DISPLAY0B		3
+#define TEGRA114_MC_DISPLAY0BB		4
+#define TEGRA114_MC_DISPLAY0C		5
+#define TEGRA114_MC_DISPLAY0CB		6
+#define TEGRA114_MC_DISPLAY1B		7
+#define TEGRA114_MC_DISPLAY1BB		8
+#define TEGRA114_MC_EPPUP		9
+#define TEGRA114_MC_G2PR		10
+#define TEGRA114_MC_G2SR		11
+#define TEGRA114_MC_MPEUNIFBR		12
+#define TEGRA114_MC_VIRUV		13
+#define TEGRA114_MC_AFIR		14
+#define TEGRA114_MC_AVPCARM7R		15
+#define TEGRA114_MC_DISPLAYHC		16
+#define TEGRA114_MC_DISPLAYHCB		17
+#define TEGRA114_MC_FDCDRD		18
+#define TEGRA114_MC_FDCDRD2		19
+#define TEGRA114_MC_G2DR		20
+#define TEGRA114_MC_HDAR		21
+#define TEGRA114_MC_HOST1XDMAR		22
+#define TEGRA114_MC_HOST1XR		23
+#define TEGRA114_MC_IDXSRD		24
+#define TEGRA114_MC_IDXSRD2		25
+#define TEGRA114_MC_MPE_IPRED		26
+#define TEGRA114_MC_MPEAMEMRD		27
+#define TEGRA114_MC_MPECSRD		28
+#define TEGRA114_MC_PPCSAHBDMAR		29
+#define TEGRA114_MC_PPCSAHBSLVR		30
+#define TEGRA114_MC_SATAR		31
+#define TEGRA114_MC_TEXSRD		32
+#define TEGRA114_MC_TEXSRD2		33
+#define TEGRA114_MC_VDEBSEVR		34
+#define TEGRA114_MC_VDEMBER		35
+#define TEGRA114_MC_VDEMCER		36
+#define TEGRA114_MC_VDETPER		37
+#define TEGRA114_MC_MPCORELPR		38
+#define TEGRA114_MC_MPCORER		39
+#define TEGRA114_MC_EPPU		40
+#define TEGRA114_MC_EPPV		41
+#define TEGRA114_MC_EPPY		42
+#define TEGRA114_MC_MPEUNIFBW		43
+#define TEGRA114_MC_VIWSB		44
+#define TEGRA114_MC_VIWU		45
+#define TEGRA114_MC_VIWV		46
+#define TEGRA114_MC_VIWY		47
+#define TEGRA114_MC_G2DW		48
+#define TEGRA114_MC_AFIW		49
+#define TEGRA114_MC_AVPCARM7W		50
+#define TEGRA114_MC_FDCDWR		51
+#define TEGRA114_MC_FDCDWR2		52
+#define TEGRA114_MC_HDAW		53
+#define TEGRA114_MC_HOST1XW		54
+#define TEGRA114_MC_ISPW		55
+#define TEGRA114_MC_MPCORELPW		56
+#define TEGRA114_MC_MPCOREW		57
+#define TEGRA114_MC_MPECSWR		58
+#define TEGRA114_MC_PPCSAHBDMAW		59
+#define TEGRA114_MC_PPCSAHBSLVW		60
+#define TEGRA114_MC_SATAW		61
+#define TEGRA114_MC_VDEBSEVW		62
+#define TEGRA114_MC_VDEDBGW		63
+#define TEGRA114_MC_VDEMBEW		64
+#define TEGRA114_MC_VDETPMW		65
+
 #endif
diff --git a/include/dt-bindings/pinctrl/mediatek,mt6392-pinfunc.h b/include/dt-bindings/pinctrl/mediatek,mt6392-pinfunc.h
new file mode 100644
index 0000000000000..c65278c8103d4
--- /dev/null
+++ b/include/dt-bindings/pinctrl/mediatek,mt6392-pinfunc.h
@@ -0,0 +1,39 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+#ifndef __DTS_MT6392_PINFUNC_H
+#define __DTS_MT6392_PINFUNC_H
+
+#include <dt-bindings/pinctrl/mt65xx.h>
+
+#define MT6392_PIN_0_INT__FUNC_GPIO0 (MTK_PIN_NO(0) | 0)
+#define MT6392_PIN_0_INT__FUNC_INT (MTK_PIN_NO(0) | 1)
+#define MT6392_PIN_0_INT__FUNC_TEST_CK2 (MTK_PIN_NO(0) | 5)
+#define MT6392_PIN_0_INT__FUNC_TEST_IN1 (MTK_PIN_NO(0) | 6)
+#define MT6392_PIN_0_INT__FUNC_TEST_OUT1 (MTK_PIN_NO(0) | 7)
+
+#define MT6392_PIN_1_SRCLKEN__FUNC_GPIO1 (MTK_PIN_NO(1) | 0)
+#define MT6392_PIN_1_SRCLKEN__FUNC_SRCLKEN (MTK_PIN_NO(1) | 1)
+#define MT6392_PIN_1_SRCLKEN__FUNC_TEST_CK0 (MTK_PIN_NO(1) | 5)
+#define MT6392_PIN_1_SRCLKEN__FUNC_TEST_IN2 (MTK_PIN_NO(1) | 6)
+#define MT6392_PIN_1_SRCLKEN__FUNC_TEST_OUT2 (MTK_PIN_NO(1) | 7)
+
+#define MT6392_PIN_2_RTC_32K1V8__FUNC_GPIO2 (MTK_PIN_NO(2) | 0)
+#define MT6392_PIN_2_RTC_32K1V8__FUNC_RTC_32K1V8 (MTK_PIN_NO(2) | 1)
+#define MT6392_PIN_2_RTC_32K1V8__FUNC_TEST_CK1 (MTK_PIN_NO(2) | 5)
+#define MT6392_PIN_2_RTC_32K1V8__FUNC_TEST_IN3 (MTK_PIN_NO(2) | 6)
+#define MT6392_PIN_2_RTC_32K1V8__FUNC_TEST_OUT3 (MTK_PIN_NO(2) | 7)
+
+#define MT6392_PIN_3_SPI_CLK__FUNC_GPIO3 (MTK_PIN_NO(3) | 0)
+#define MT6392_PIN_3_SPI_CLK__FUNC_SPI_CLK (MTK_PIN_NO(3) | 1)
+
+#define MT6392_PIN_4_SPI_CSN__FUNC_GPIO4 (MTK_PIN_NO(4) | 0)
+#define MT6392_PIN_4_SPI_CSN__FUNC_SPI_CSN (MTK_PIN_NO(4) | 1)
+
+#define MT6392_PIN_5_SPI_MOSI__FUNC_GPIO5 (MTK_PIN_NO(5) | 0)
+#define MT6392_PIN_5_SPI_MOSI__FUNC_SPI_MOSI (MTK_PIN_NO(5) | 1)
+
+#define MT6392_PIN_6_SPI_MISO__FUNC_GPIO6 (MTK_PIN_NO(6) | 0)
+#define MT6392_PIN_6_SPI_MISO__FUNC_SPI_MISO (MTK_PIN_NO(6) | 1)
+#define MT6392_PIN_6_SPI_MISO__FUNC_TEST_IN4 (MTK_PIN_NO(6) | 6)
+#define MT6392_PIN_6_SPI_MISO__FUNC_TEST_OUT4 (MTK_PIN_NO(6) | 7)
+
+#endif /* __DTS_MT6392_PINFUNC_H */
diff --git a/include/dt-bindings/pinctrl/renesas,r9a08g046-pinctrl.h b/include/dt-bindings/pinctrl/renesas,r9a08g046-pinctrl.h
new file mode 100644
index 0000000000000..5ec5bfc27c7df
--- /dev/null
+++ b/include/dt-bindings/pinctrl/renesas,r9a08g046-pinctrl.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * This header provides constants for Renesas RZ/G3L family pinctrl bindings.
+ *
+ * Copyright (C) 2026 Renesas Electronics Corp.
+ *
+ */
+
+#ifndef __DT_BINDINGS_PINCTRL_RENESAS_R9A08G046_PINCTRL_H__
+#define __DT_BINDINGS_PINCTRL_RENESAS_R9A08G046_PINCTRL_H__
+
+#include <dt-bindings/pinctrl/rzg2l-pinctrl.h>
+
+/* RZG3L_Px = Offset address of PFC_P_mn  - 0x22 */
+#define RZG3L_P2	2
+#define RZG3L_P3	3
+#define RZG3L_P5	5
+#define RZG3L_P6	6
+#define RZG3L_P7	7
+#define RZG3L_P8	8
+#define RZG3L_PA	10
+#define RZG3L_PB	11
+#define RZG3L_PC	12
+#define RZG3L_PD	13
+#define RZG3L_PE	14
+#define RZG3L_PF	15
+#define RZG3L_PG	16
+#define RZG3L_PH	17
+#define RZG3L_PJ	19
+#define RZG3L_PK	20
+#define RZG3L_PL	21
+#define RZG3L_PM	22
+#define RZG3L_PS	28
+
+#define RZG3L_PORT_PINMUX(b, p, f)	RZG2L_PORT_PINMUX(RZG3L_P##b, p, f)
+#define RZG3L_GPIO(port, pin)		RZG2L_GPIO(RZG3L_P##port, pin)
+
+#endif /* __DT_BINDINGS_PINCTRL_RENESAS_R9A08G046_PINCTRL_H__ */
diff --git a/include/dt-bindings/reset/qcom,ipq9650-gcc.h b/include/dt-bindings/reset/qcom,ipq9650-gcc.h
new file mode 100644
index 0000000000000..a2cbb114addd1
--- /dev/null
+++ b/include/dt-bindings/reset/qcom,ipq9650-gcc.h
@@ -0,0 +1,215 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#ifndef _DT_BINDINGS_RESET_IPQ_GCC_IPQ9650_H
+#define _DT_BINDINGS_RESET_IPQ_GCC_IPQ9650_H
+
+#define GCC_ADSS_BCR						0
+#define GCC_ADSS_PWM_CLK_ARES					1
+#define GCC_APC0_VOLTAGE_DROOP_DETECTOR_BCR			2
+#define GCC_APC0_VOLTAGE_DROOP_DETECTOR_GPLL0_CLK_ARES		3
+#define GCC_APSS_AHB_CLK_ARES					4
+#define GCC_APSS_ATB_CLK_ARES					5
+#define GCC_APSS_AXI_CLK_ARES					6
+#define GCC_APSS_TS_CLK_ARES					7
+#define GCC_BOOT_ROM_AHB_CLK_ARES				8
+#define GCC_BOOT_ROM_BCR					9
+#define GCC_CMN_12GPLL_AHB_CLK_ARES				10
+#define GCC_CMN_12GPLL_APU_CLK_ARES				11
+#define GCC_CMN_12GPLL_SYS_CLK_ARES				12
+#define GCC_CMN_BLK_BCR						13
+#define GCC_CMN_LDO_CLK_ARES					14
+#define GCC_CPUSS_TRIG_CLK_ARES					15
+#define GCC_GP1_CLK_ARES					16
+#define GCC_GP2_CLK_ARES					17
+#define GCC_GP3_CLK_ARES					18
+#define GCC_MDIO_AHB_CLK_ARES					19
+#define GCC_MDIO_BCR						20
+#define GCC_NSSCC_CLK_ARES					21
+#define GCC_NSSCFG_CLK_ARES					22
+#define GCC_NSSNOC_ATB_CLK_ARES					23
+#define GCC_NSSNOC_MEMNOC_1_CLK_ARES				24
+#define GCC_NSSNOC_MEMNOC_CLK_ARES				25
+#define GCC_NSSNOC_NSSCC_CLK_ARES				26
+#define GCC_NSSNOC_PCNOC_1_CLK_ARES				27
+#define GCC_NSSNOC_QOSGEN_REF_CLK_ARES				28
+#define GCC_NSSNOC_SNOC_1_CLK_ARES				29
+#define GCC_NSSNOC_SNOC_CLK_ARES				30
+#define GCC_NSSNOC_TIMEOUT_REF_CLK_ARES				31
+#define GCC_NSSNOC_XO_DCD_CLK_ARES				32
+#define GCC_NSS_BCR						33
+#define GCC_NSS_TS_CLK_ARES					34
+#define GCC_PCIE0PHY_PHY_BCR					35
+#define GCC_PCIE0_AHB_CLK_ARES					36
+#define GCC_PCIE0_AHB_RESET					37
+#define GCC_PCIE0_AUX_CLK_ARES					38
+#define GCC_PCIE0_AUX_RESET					39
+#define GCC_PCIE0_AXI_M_CLK_ARES				40
+#define GCC_PCIE0_AXI_M_RESET					41
+#define GCC_PCIE0_AXI_M_STICKY_RESET				42
+#define GCC_PCIE0_AXI_S_BRIDGE_CLK_ARES				43
+#define GCC_PCIE0_AXI_S_CLK_ARES				44
+#define GCC_PCIE0_AXI_S_RESET					45
+#define GCC_PCIE0_AXI_S_STICKY_RESET				46
+#define GCC_PCIE0_BCR						47
+#define GCC_PCIE0_CORE_STICKY_RESET				48
+#define GCC_PCIE0_LINK_DOWN_BCR					49
+#define GCC_PCIE0_PHY_BCR					50
+#define GCC_PCIE0_PIPE_CLK_ARES					51
+#define GCC_PCIE0_PIPE_RESET					52
+#define GCC_PCIE1PHY_PHY_BCR					53
+#define GCC_PCIE1_AHB_CLK_ARES					54
+#define GCC_PCIE1_AHB_RESET					55
+#define GCC_PCIE1_AUX_CLK_ARES					56
+#define GCC_PCIE1_AUX_RESET					57
+#define GCC_PCIE1_AXI_M_CLK_ARES				58
+#define GCC_PCIE1_AXI_M_RESET					59
+#define GCC_PCIE1_AXI_M_STICKY_RESET				60
+#define GCC_PCIE1_AXI_S_BRIDGE_CLK_ARES				61
+#define GCC_PCIE1_AXI_S_CLK_ARES				62
+#define GCC_PCIE1_AXI_S_RESET					63
+#define GCC_PCIE1_AXI_S_STICKY_RESET				64
+#define GCC_PCIE1_BCR						65
+#define GCC_PCIE1_CORE_STICKY_RESET				66
+#define GCC_PCIE1_LINK_DOWN_BCR					67
+#define GCC_PCIE1_PHY_BCR					68
+#define GCC_PCIE1_PIPE_CLK_ARES					69
+#define GCC_PCIE1_PIPE_RESET					70
+#define GCC_PCIE2PHY_PHY_BCR					71
+#define GCC_PCIE2_AHB_CLK_ARES					72
+#define GCC_PCIE2_AHB_RESET					73
+#define GCC_PCIE2_AUX_CLK_ARES					74
+#define GCC_PCIE2_AUX_RESET					75
+#define GCC_PCIE2_AXI_M_CLK_ARES				76
+#define GCC_PCIE2_AXI_M_RESET					77
+#define GCC_PCIE2_AXI_M_STICKY_RESET				78
+#define GCC_PCIE2_AXI_S_BRIDGE_CLK_ARES				79
+#define GCC_PCIE2_AXI_S_CLK_ARES				80
+#define GCC_PCIE2_AXI_S_RESET					81
+#define GCC_PCIE2_AXI_S_STICKY_RESET				82
+#define GCC_PCIE2_BCR						83
+#define GCC_PCIE2_CORE_STICKY_RESET				84
+#define GCC_PCIE2_LINK_DOWN_BCR					85
+#define GCC_PCIE2_PHY_BCR					86
+#define GCC_PCIE2_PIPE_CLK_ARES					87
+#define GCC_PCIE2_PIPE_RESET					88
+#define GCC_PCIE3PHY_PHY_BCR					89
+#define GCC_PCIE3_AHB_CLK_ARES					90
+#define GCC_PCIE3_AHB_RESET					91
+#define GCC_PCIE3_AUX_CLK_ARES					92
+#define GCC_PCIE3_AUX_RESET					93
+#define GCC_PCIE3_AXI_M_CLK_ARES				94
+#define GCC_PCIE3_AXI_M_RESET					95
+#define GCC_PCIE3_AXI_M_STICKY_RESET				96
+#define GCC_PCIE3_AXI_S_BRIDGE_CLK_ARES				97
+#define GCC_PCIE3_AXI_S_CLK_ARES				98
+#define GCC_PCIE3_AXI_S_RESET					99
+#define GCC_PCIE3_AXI_S_STICKY_RESET				100
+#define GCC_PCIE3_BCR						101
+#define GCC_PCIE3_CORE_STICKY_RESET				102
+#define GCC_PCIE3_LINK_DOWN_BCR					103
+#define GCC_PCIE3_PHY_BCR					104
+#define GCC_PCIE3_PIPE_CLK_ARES					105
+#define GCC_PCIE3_PIPE_RESET					106
+#define GCC_PCIE4PHY_PHY_BCR					107
+#define GCC_PCIE4_AHB_CLK_ARES					108
+#define GCC_PCIE4_AHB_RESET					109
+#define GCC_PCIE4_AUX_CLK_ARES					110
+#define GCC_PCIE4_AUX_RESET					111
+#define GCC_PCIE4_AXI_M_CLK_ARES				112
+#define GCC_PCIE4_AXI_M_RESET					113
+#define GCC_PCIE4_AXI_M_STICKY_RESET				114
+#define GCC_PCIE4_AXI_S_BRIDGE_CLK_ARES				115
+#define GCC_PCIE4_AXI_S_CLK_ARES				116
+#define GCC_PCIE4_AXI_S_RESET					117
+#define GCC_PCIE4_AXI_S_STICKY_RESET				118
+#define GCC_PCIE4_BCR						119
+#define GCC_PCIE4_CORE_STICKY_RESET				120
+#define GCC_PCIE4_LINK_DOWN_BCR					121
+#define GCC_PCIE4_PHY_BCR					122
+#define GCC_PCIE4_PIPE_CLK_ARES					123
+#define GCC_PCIE4_PIPE_RESET					124
+#define GCC_QDSS_APB2JTAG_CLK_ARES				125
+#define GCC_QDSS_AT_CLK_ARES					126
+#define GCC_QDSS_BCR						127
+#define GCC_QDSS_CFG_AHB_CLK_ARES				128
+#define GCC_QDSS_DAP_AHB_CLK_ARES				129
+#define GCC_QDSS_DAP_CLK_ARES					130
+#define GCC_QDSS_ETR_USB_CLK_ARES				131
+#define GCC_QDSS_EUD_AT_CLK_ARES				132
+#define GCC_QDSS_STM_CLK_ARES					133
+#define GCC_QDSS_TRACECLKIN_CLK_ARES				134
+#define GCC_QDSS_TSCTR_DIV16_CLK_ARES				135
+#define GCC_QDSS_TSCTR_DIV2_CLK_ARES				136
+#define GCC_QDSS_TSCTR_DIV3_CLK_ARES				137
+#define GCC_QDSS_TSCTR_DIV4_CLK_ARES				138
+#define GCC_QDSS_TSCTR_DIV8_CLK_ARES				139
+#define GCC_QDSS_TS_CLK_ARES					140
+#define GCC_QPIC_AHB_CLK_ARES					141
+#define GCC_QPIC_BCR						142
+#define GCC_QPIC_CLK_ARES					143
+#define GCC_QPIC_IO_MACRO_CLK_ARES				144
+#define GCC_QPIC_SLEEP_CLK_ARES					145
+#define GCC_QUPV3_2X_CORE_CLK_ARES				146
+#define GCC_QUPV3_AHB_MST_CLK_ARES				147
+#define GCC_QUPV3_AHB_SLV_CLK_ARES				148
+#define GCC_QUPV3_BCR						149
+#define GCC_QUPV3_CORE_CLK_ARES					150
+#define GCC_QUPV3_WRAP_SE0_BCR					151
+#define GCC_QUPV3_WRAP_SE0_CLK_ARES				152
+#define GCC_QUPV3_WRAP_SE1_BCR					153
+#define GCC_QUPV3_WRAP_SE1_CLK_ARES				154
+#define GCC_QUPV3_WRAP_SE2_BCR					155
+#define GCC_QUPV3_WRAP_SE2_CLK_ARES				156
+#define GCC_QUPV3_WRAP_SE3_BCR					157
+#define GCC_QUPV3_WRAP_SE3_CLK_ARES				158
+#define GCC_QUPV3_WRAP_SE4_BCR					159
+#define GCC_QUPV3_WRAP_SE4_CLK_ARES				160
+#define GCC_QUPV3_WRAP_SE5_BCR					161
+#define GCC_QUPV3_WRAP_SE5_CLK_ARES				162
+#define GCC_QUPV3_WRAP_SE6_BCR					163
+#define GCC_QUPV3_WRAP_SE6_CLK_ARES				164
+#define GCC_QUPV3_WRAP_SE7_BCR					165
+#define GCC_QUPV3_WRAP_SE7_CLK_ARES				166
+#define GCC_QUSB2_0_PHY_BCR					167
+#define GCC_QUSB2_1_PHY_BCR					168
+#define GCC_SDCC1_APPS_CLK_ARES					169
+#define GCC_SDCC1_ICE_CORE_CLK_ARES				170
+#define GCC_SDCC_BCR						171
+#define GCC_TLMM_AHB_CLK_ARES					172
+#define GCC_TLMM_BCR						173
+#define GCC_TLMM_CLK_ARES					174
+#define GCC_UNIPHY0_AHB_CLK_ARES				175
+#define GCC_UNIPHY0_BCR						176
+#define GCC_UNIPHY0_PMA_BCR					177
+#define GCC_UNIPHY0_SYS_CLK_ARES				178
+#define GCC_UNIPHY0_XPCS_ARES					179
+#define GCC_UNIPHY1_AHB_CLK_ARES				180
+#define GCC_UNIPHY1_BCR						181
+#define GCC_UNIPHY1_PMA_BCR					182
+#define GCC_UNIPHY1_SYS_CLK_ARES				183
+#define GCC_UNIPHY1_XPCS_ARES					184
+#define GCC_UNIPHY2_AHB_CLK_ARES				185
+#define GCC_UNIPHY2_BCR						186
+#define GCC_UNIPHY2_PMA_BCR					187
+#define GCC_UNIPHY2_SYS_CLK_ARES				188
+#define GCC_UNIPHY2_XPCS_ARES					189
+#define GCC_USB0_AUX_CLK_ARES					190
+#define GCC_USB0_MASTER_CLK_ARES				191
+#define GCC_USB0_MOCK_UTMI_CLK_ARES				192
+#define GCC_USB0_PHY_BCR					193
+#define GCC_USB0_PHY_CFG_AHB_CLK_ARES				194
+#define GCC_USB0_PIPE_CLK_ARES					195
+#define GCC_USB0_SLEEP_CLK_ARES					196
+#define GCC_USB1_BCR						197
+#define GCC_USB1_MASTER_CLK_ARES				198
+#define GCC_USB1_MOCK_UTMI_CLK_ARES				199
+#define GCC_USB1_PHY_CFG_AHB_CLK_ARES				200
+#define GCC_USB1_SLEEP_CLK_ARES					201
+#define GCC_USB3PHY_0_PHY_BCR					202
+#define GCC_USB_BCR						203
+#define GCC_UNIPHY1_XLGPCS_ARES					204
+#define GCC_UNIPHY2_XLGPCS_ARES					205
+#endif
diff --git a/include/dt-bindings/soc/renesas,r8a78000-mfis.h b/include/dt-bindings/soc/renesas,r8a78000-mfis.h
new file mode 100644
index 0000000000000..147a8aefc6438
--- /dev/null
+++ b/include/dt-bindings/soc/renesas,r8a78000-mfis.h
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */
+#ifndef _DT_BINDINGS_SOC_RENESAS_R8A78000_MFIS_H
+#define _DT_BINDINGS_SOC_RENESAS_R8A78000_MFIS_H
+
+/*
+ * Constants for the second mbox-cell of the Renesas MFIS IP core. To be treated
+ * as bit flags which can be ORed.
+ */
+
+/*
+ * MFIS HW design before r8a78001 requires a channel to be marked as either
+ * TX or RX.
+ */
+#define MFIS_CHANNEL_TX (0 << 0)
+#define MFIS_CHANNEL_RX (1 << 0)
+
+/*
+ * MFIS variants before r8a78001 work with pairs of IICR and EICR registers.
+ * Usually, it is specified in the datasheets which of the two a specific core
+ * should use. Then, it does not need extra description in DT. For plain MFIS
+ * of r8a78000, this is selectable, though. According to the system design and
+ * the firmware in use, these channels need to be marked. This is not needed
+ * with other versions of the MFIS, not even with MFIS-SCP of r8a78000.
+ */
+#define MFIS_CHANNEL_IICR (0 << 1)
+#define MFIS_CHANNEL_EICR (1 << 1)
+
+#endif /* _DT_BINDINGS_SOC_RENESAS_R8A78000_MFIS_H */
diff --git a/include/hyperv/hvhdk.h b/include/hyperv/hvhdk.h
index 5e83d37149662..0c89c62c97067 100644
--- a/include/hyperv/hvhdk.h
+++ b/include/hyperv/hvhdk.h
@@ -79,6 +79,7 @@ struct hv_vp_register_page {
 
 		u64 registers[18];
 	};
+	u8 reserved[8];
 	/* Volatile XMM registers (HV_X64_REGISTER_CLASS_XMM) */
 	union {
 		struct {
diff --git a/include/keys/trusted-type.h b/include/keys/trusted-type.h
index 03527162613f7..9f9940482da43 100644
--- a/include/keys/trusted-type.h
+++ b/include/keys/trusted-type.h
@@ -83,18 +83,21 @@ struct trusted_key_source {
 
 extern struct key_type key_type_trusted;
 
-#define TRUSTED_DEBUG 0
+#ifdef CONFIG_TRUSTED_KEYS_DEBUG
+extern bool trusted_debug;
 
-#if TRUSTED_DEBUG
 static inline void dump_payload(struct trusted_key_payload *p)
 {
-	pr_info("key_len %d\n", p->key_len);
-	print_hex_dump(KERN_INFO, "key ", DUMP_PREFIX_NONE,
-		       16, 1, p->key, p->key_len, 0);
-	pr_info("bloblen %d\n", p->blob_len);
-	print_hex_dump(KERN_INFO, "blob ", DUMP_PREFIX_NONE,
-		       16, 1, p->blob, p->blob_len, 0);
-	pr_info("migratable %d\n", p->migratable);
+	if (!trusted_debug)
+		return;
+
+	pr_debug("key_len %d\n", p->key_len);
+	print_hex_dump_debug("key ", DUMP_PREFIX_NONE,
+			     16, 1, p->key, p->key_len, 0);
+	pr_debug("bloblen %d\n", p->blob_len);
+	print_hex_dump_debug("blob ", DUMP_PREFIX_NONE,
+			     16, 1, p->blob, p->blob_len, 0);
+	pr_debug("migratable %d\n", p->migratable);
 }
 #else
 static inline void dump_payload(struct trusted_key_payload *p)
diff --git a/include/kunit/test-bug.h b/include/kunit/test-bug.h
index 47aa8f21ccce8..99869029fc686 100644
--- a/include/kunit/test-bug.h
+++ b/include/kunit/test-bug.h
@@ -10,6 +10,7 @@
 #define _KUNIT_TEST_BUG_H
 
 #include <linux/stddef.h> /* for NULL */
+#include <linux/types.h>  /* for bool */
 
 #if IS_ENABLED(CONFIG_KUNIT)
 
@@ -23,6 +24,7 @@ DECLARE_STATIC_KEY_FALSE(kunit_running);
 extern struct kunit_hooks_table {
 	__printf(3, 4) void (*fail_current_test)(const char*, int, const char*, ...);
 	void *(*get_static_stub_address)(struct kunit *test, void *real_fn_addr);
+	bool (*is_suppressed_warning)(bool count);
 } kunit_hooks;
 
 /**
@@ -60,9 +62,33 @@ static inline struct kunit *kunit_get_current_test(void)
 		}								\
 	} while (0)
 
+/**
+ * kunit_is_suppressed_warning() - Check if warnings are being suppressed
+ *                                 by the current KUnit test.
+ * @count: if true, increment the suppression counter on match.
+ *
+ * Returns true if the current task has active warning suppression.
+ * Uses the kunit_running static branch for zero overhead when no tests run.
+ *
+ * A single WARN*() may traverse multiple call sites in the warning path
+ * (e.g., __warn_printk() and __report_bug()). Pass @count = true at the
+ * primary suppression point to count each warning exactly once, and
+ * @count = false at secondary points to suppress output without
+ * inflating the count.
+ */
+static inline bool kunit_is_suppressed_warning(bool count)
+{
+	if (!static_branch_unlikely(&kunit_running))
+		return false;
+
+	return kunit_hooks.is_suppressed_warning &&
+	       kunit_hooks.is_suppressed_warning(count);
+}
+
 #else
 
 static inline struct kunit *kunit_get_current_test(void) { return NULL; }
+static inline bool kunit_is_suppressed_warning(bool count) { return false; }
 
 #define kunit_fail_current_test(fmt, ...) do {} while (0)
 
diff --git a/include/kunit/test.h b/include/kunit/test.h
index 9cd1594ab697d..be71612f61655 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -1795,4 +1795,102 @@ do {									       \
 // include resource.h themselves if they need it.
 #include <kunit/resource.h>
 
+/*
+ * Warning backtrace suppression API.
+ *
+ * Suppresses WARN*() backtraces on the current task while active. Two forms
+ * are provided:
+ *
+ * - Scoped: kunit_warning_suppress(test) { ... }
+ *   Suppression is active for the duration of the block. On normal exit,
+ *   the for-loop increment deactivates suppression. On early exit (break,
+ *   return, goto), the __cleanup attribute fires. On kthread_exit() (e.g.,
+ *   a failed KUnit assertion), kunit_add_action() cleans up at test
+ *   teardown. The suppression handle is only accessible inside the block,
+ *   so warning counts must be checked before the block exits.
+ *
+ * - Direct: kunit_start_suppress_warning() / kunit_end_suppress_warning()
+ *   The underlying functions, returning an explicit handle pointer. Use
+ *   when the handle needs to be retained (e.g., for post-suppression
+ *   count checks) or passed across helper functions.
+ */
+struct kunit_suppressed_warning;
+
+struct kunit_suppressed_warning *
+kunit_start_suppress_warning(struct kunit *test);
+void kunit_end_suppress_warning(struct kunit *test,
+				struct kunit_suppressed_warning *w);
+int kunit_suppressed_warning_count(struct kunit_suppressed_warning *w);
+void __kunit_suppress_auto_cleanup(struct kunit_suppressed_warning **wp);
+bool kunit_has_active_suppress_warning(void);
+
+/**
+ * kunit_warning_suppress() - Suppress WARN*() backtraces for the duration
+ *                            of a block.
+ * @test: The test context object.
+ *
+ * Scoped form of the suppression API. Suppression starts when the block is
+ * entered and ends automatically when the block exits through any path. See
+ * the section comment above for the cleanup guarantees on each exit path.
+ * Fails the test if suppression is already active; nesting is not supported.
+ *
+ * The warning count can be checked inside the block via
+ * KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(). The handle is not accessible
+ * after the block exits.
+ *
+ * Example::
+ *
+ *   kunit_warning_suppress(test) {
+ *       trigger_warning();
+ *       KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+ *   }
+ */
+#define kunit_warning_suppress(test)					\
+	for (struct kunit_suppressed_warning *__kunit_suppress		\
+	     __cleanup(__kunit_suppress_auto_cleanup) =			\
+	     kunit_start_suppress_warning(test);			\
+	     __kunit_suppress;						\
+	     kunit_end_suppress_warning(test, __kunit_suppress),	\
+	     __kunit_suppress = NULL)
+
+/**
+ * KUNIT_SUPPRESSED_WARNING_COUNT() - Returns the suppressed warning count.
+ *
+ * Returns the number of WARN*() calls suppressed since the current
+ * suppression block started, or 0 if the handle is NULL. Usable inside a
+ * kunit_warning_suppress() block.
+ */
+#define KUNIT_SUPPRESSED_WARNING_COUNT() \
+	kunit_suppressed_warning_count(__kunit_suppress)
+
+/**
+ * KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT() - Sets an expectation that the
+ *                                           suppressed warning count equals
+ *                                           @expected.
+ * @test: The test context object.
+ * @expected: an expression that evaluates to the expected warning count.
+ *
+ * Sets an expectation that the number of suppressed WARN*() calls equals
+ * @expected. This is semantically equivalent to
+ * KUNIT_EXPECT_EQ(@test, KUNIT_SUPPRESSED_WARNING_COUNT(), @expected).
+ * See KUNIT_EXPECT_EQ() for more information.
+ */
+#define KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, expected) \
+	KUNIT_EXPECT_EQ(test, KUNIT_SUPPRESSED_WARNING_COUNT(), expected)
+
+/**
+ * KUNIT_ASSERT_SUPPRESSED_WARNING_COUNT() - Sets an assertion that the
+ *                                           suppressed warning count equals
+ *                                           @expected.
+ * @test: The test context object.
+ * @expected: an expression that evaluates to the expected warning count.
+ *
+ * Sets an assertion that the number of suppressed WARN*() calls equals
+ * @expected. This is the same as KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(),
+ * except it causes an assertion failure (see KUNIT_ASSERT_TRUE()) when the
+ * assertion is not met.
+ */
+#define KUNIT_ASSERT_SUPPRESSED_WARNING_COUNT(test, expected) \
+	KUNIT_ASSERT_EQ(test, KUNIT_SUPPRESSED_WARNING_COUNT(), expected)
+
 #endif /* _KUNIT_TEST_H */
diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h
index bf8cc9589bd09..15a4f97f81051 100644
--- a/include/kvm/arm_arch_timer.h
+++ b/include/kvm/arm_arch_timer.h
@@ -66,11 +66,6 @@ struct arch_timer_context {
 	 */
 	bool				loaded;
 
-	/* Output level of the timer IRQ */
-	struct {
-		bool			level;
-	} irq;
-
 	/* Who am I? */
 	enum kvm_arch_timers		timer_id;
 
@@ -104,7 +99,7 @@ void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu);
 void kvm_timer_sync_nested(struct kvm_vcpu *vcpu);
 void kvm_timer_sync_user(struct kvm_vcpu *vcpu);
 bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu);
-void kvm_timer_update_run(struct kvm_vcpu *vcpu);
+bool kvm_timer_update_run(struct kvm_vcpu *vcpu);
 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu);
 
 void kvm_timer_init_vm(struct kvm *kvm);
diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h
index 0a36a3d5c8944..b5e5942204fc6 100644
--- a/include/kvm/arm_pmu.h
+++ b/include/kvm/arm_pmu.h
@@ -32,7 +32,6 @@ struct kvm_pmu {
 	struct kvm_pmc pmc[KVM_ARMV8_PMU_MAX_COUNTERS];
 	int irq_num;
 	bool created;
-	bool irq_level;
 };
 
 struct arm_pmu_entry {
@@ -54,7 +53,7 @@ void kvm_pmu_reprogram_counter_mask(struct kvm_vcpu *vcpu, u64 val);
 void kvm_pmu_flush_hwstate(struct kvm_vcpu *vcpu);
 void kvm_pmu_sync_hwstate(struct kvm_vcpu *vcpu);
 bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu);
-void kvm_pmu_update_run(struct kvm_vcpu *vcpu);
+bool kvm_pmu_update_run(struct kvm_vcpu *vcpu);
 void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val);
 void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val);
 void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu, u64 data,
@@ -131,7 +130,7 @@ static inline bool kvm_pmu_should_notify_user(struct kvm_vcpu *vcpu)
 {
 	return false;
 }
-static inline void kvm_pmu_update_run(struct kvm_vcpu *vcpu) {}
+static inline bool kvm_pmu_update_run(struct kvm_vcpu *vcpu) { return false; }
 static inline void kvm_pmu_software_increment(struct kvm_vcpu *vcpu, u64 val) {}
 static inline void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) {}
 static inline void kvm_pmu_set_counter_event_type(struct kvm_vcpu *vcpu,
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index 67effb91fa983..10d6c6c11bdff 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -798,6 +798,8 @@ int acpi_get_local_u64_address(acpi_handle handle, u64 *addr);
 int acpi_get_local_address(acpi_handle handle, u32 *addr);
 const char *acpi_get_subsystem_id(acpi_handle handle);
 
+struct device *acpi_bus_find_device_by_name(const char *name);
+
 #ifdef CONFIG_ACPI_MRRM
 int acpi_mrrm_max_mem_region(void);
 #endif
@@ -1106,6 +1108,11 @@ static inline const char *acpi_get_subsystem_id(acpi_handle handle)
 	return ERR_PTR(-ENODEV);
 }
 
+static inline struct device *acpi_bus_find_device_by_name(const char *name)
+{
+	return NULL;
+}
+
 static inline int acpi_register_wakeup_handler(int wake_irq,
 	bool (*wakeup)(void *context), void *context)
 {
diff --git a/include/linux/adi-axi-common.h b/include/linux/adi-axi-common.h
index 37962ba530dfc..e7ba393061ee6 100644
--- a/include/linux/adi-axi-common.h
+++ b/include/linux/adi-axi-common.h
@@ -51,6 +51,7 @@ enum adi_axi_fpga_technology {
 	ADI_AXI_FPGA_TECH_SERIES7,
 	ADI_AXI_FPGA_TECH_ULTRASCALE,
 	ADI_AXI_FPGA_TECH_ULTRASCALE_PLUS,
+	ADI_AXI_FPGA_TECH_VERSAL,
 };
 
 enum adi_axi_fpga_family {
@@ -71,6 +72,7 @@ enum adi_axi_fpga_speed_grade {
 	ADI_AXI_FPGA_SPEED_2    = 20,
 	ADI_AXI_FPGA_SPEED_2L   = 21,
 	ADI_AXI_FPGA_SPEED_2LV  = 22,
+	ADI_AXI_FPGA_SPEED_2MP  = 23,
 	ADI_AXI_FPGA_SPEED_3    = 30,
 };
 
diff --git a/include/linux/alarmtimer.h b/include/linux/alarmtimer.h
index 3ffa5341dce21..2014288ca2f4b 100644
--- a/include/linux/alarmtimer.h
+++ b/include/linux/alarmtimer.h
@@ -42,11 +42,14 @@ struct alarm {
 	void			*data;
 };
 
+static __always_inline ktime_t alarm_get_expires(struct alarm *alarm)
+{
+	return alarm->node.expires;
+}
+
 void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
 		void (*function)(struct alarm *, ktime_t));
-void alarm_start(struct alarm *alarm, ktime_t start);
-void alarm_start_relative(struct alarm *alarm, ktime_t start);
-void alarm_restart(struct alarm *alarm);
+bool alarm_start_timer(struct alarm *alarm, ktime_t expires, bool relative);
 int alarm_try_to_cancel(struct alarm *alarm);
 int alarm_cancel(struct alarm *alarm);
 
diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h
index 02de2ede560f3..068ba2e77c5d6 100644
--- a/include/linux/alloc_tag.h
+++ b/include/linux/alloc_tag.h
@@ -163,11 +163,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref)
 {
 	WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n");
 }
-void alloc_tag_add_early_pfn(unsigned long pfn);
+void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags);
 #else
 static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {}
 static inline void alloc_tag_sub_check(union codetag_ref *ref) {}
-static inline void alloc_tag_add_early_pfn(unsigned long pfn) {}
+static inline void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags) {}
 #endif
 
 /* Caller should verify both ref and tag to be valid */
diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h
index e7195750d21bb..4de81848fe2ee 100644
--- a/include/linux/arm-smccc.h
+++ b/include/linux/arm-smccc.h
@@ -90,6 +90,11 @@
 			   ARM_SMCCC_SMC_32,				\
 			   0, 2)
 
+#define ARM_SMCCC_ARCH_SOC_ID64						\
+	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
+			   ARM_SMCCC_SMC_64,				\
+			   0, 2)
+
 #define ARM_SMCCC_ARCH_WORKAROUND_1					\
 	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,				\
 			   ARM_SMCCC_SMC_32,				\
diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h
index 81e603839c4a5..17eca3dfc59e2 100644
--- a/include/linux/arm_ffa.h
+++ b/include/linux/arm_ffa.h
@@ -173,7 +173,7 @@ struct ffa_partition_info;
 #if IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT)
 struct ffa_device *
 ffa_device_register(const struct ffa_partition_info *part_info,
-		    const struct ffa_ops *ops);
+		    const struct ffa_ops *ops, struct device *parent);
 void ffa_device_unregister(struct ffa_device *ffa_dev);
 int ffa_driver_register(struct ffa_driver *driver, struct module *owner,
 			const char *mod_name);
@@ -184,7 +184,7 @@ bool ffa_device_is_valid(struct ffa_device *ffa_dev);
 #else
 static inline struct ffa_device *
 ffa_device_register(const struct ffa_partition_info *part_info,
-		    const struct ffa_ops *ops)
+		    const struct ffa_ops *ops, struct device *parent)
 {
 	return NULL;
 }
diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h
index bc09b55e36822..4e1ad8ccbcdde 100644
--- a/include/linux/auxiliary_bus.h
+++ b/include/linux/auxiliary_bus.h
@@ -62,6 +62,9 @@
  * @sysfs.irqs: irqs xarray contains irq indices which are used by the device,
  * @sysfs.lock: Synchronize irq sysfs creation,
  * @sysfs.irq_dir_exists: whether "irqs" directory exists,
+ * @registration_data_rust: private data owned by the registering (parent)
+ *                          driver; valid for as long as the device is
+ *                          registered with the driver core,
  *
  * An auxiliary_device represents a part of its parent device's functionality.
  * It is given a name that, combined with the registering drivers
@@ -148,6 +151,7 @@ struct auxiliary_device {
 		struct mutex lock; /* Synchronize irq sysfs creation */
 		bool irq_dir_exists;
 	} sysfs;
+	void *registration_data_rust;
 };
 
 /**
diff --git a/include/linux/bio.h b/include/linux/bio.h
index dc17780d6c1e3..9b670b3435211 100644
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -256,6 +256,12 @@ static inline struct folio *bio_first_folio_all(struct bio *bio)
 	return page_folio(bio_first_page_all(bio));
 }
 
+static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
+{
+	WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
+	return &bio->bi_io_vec[bio->bi_vcnt - 1];
+}
+
 /**
  * struct folio_iter - State for iterating all folios in a bio.
  * @folio: The current folio we're iterating.  NULL after the last folio.
@@ -479,17 +485,10 @@ int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen,
 		size_t minsize);
 void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty);
 
-extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
-			       struct bio *src, struct bvec_iter *src_iter);
 extern void bio_copy_data(struct bio *dst, struct bio *src);
 extern void bio_free_pages(struct bio *bio);
+void zero_fill_bio(struct bio *bio);
 void guard_bio_eod(struct bio *bio);
-void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);
-
-static inline void zero_fill_bio(struct bio *bio)
-{
-	zero_fill_bio_iter(bio, bio->bi_iter);
-}
 
 static inline void bio_release_pages(struct bio *bio, bool mark_dirty)
 {
diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h
index 54aeeef1f0ec7..14f86e455a678 100644
--- a/include/linux/bitfield.h
+++ b/include/linux/bitfield.h
@@ -44,7 +44,7 @@
  *  FIELD_MODIFY(REG_FIELD_C, &reg, c);
  */
 
-#define __bf_shf(x) (__builtin_ffsll(x) - 1)
+#define __bf_shf __builtin_ctzll
 
 #define __scalar_type_to_unsigned_cases(type)				\
 		unsigned type:	(unsigned type)0,			\
@@ -178,6 +178,22 @@
 		__FIELD_GET(_mask, _reg, "FIELD_GET: ");		\
 	})
 
+/**
+ * FIELD_GET_SIGNED() - extract a signed bitfield element
+ * @mask: shifted mask defining the field's length and position
+ * @reg:  value of entire bitfield
+ *
+ * Returns the sign-extended field specified by @_mask from the
+ * bitfield passed in as @reg by masking and shifting it down.
+ */
+#define FIELD_GET_SIGNED(mask, reg)					\
+	({								\
+		__BF_FIELD_CHECK(mask, reg, 0U, "FIELD_GET_SIGNED: ");	\
+		 ((__signed_scalar_typeof(mask))			\
+		  (((long long)(reg) << __builtin_clzll(mask)) >>	\
+		   (__builtin_clzll(mask) + __builtin_ctzll(mask))));	\
+	})
+
 /**
  * FIELD_MODIFY() - modify a bitfield element
  * @_mask: shifted mask defining the field's length and position
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 890128cdea1ce..17270a28c66d5 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1040,7 +1040,6 @@ extern const char *blk_op_str(enum req_op op);
 
 int blk_status_to_errno(blk_status_t status);
 blk_status_t errno_to_blk_status(int errno);
-const char *blk_status_to_str(blk_status_t status);
 
 /* only poll the hardware once, don't continue until a completion was found */
 #define BLK_POLL_ONESHOT		(1 << 0)
diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h
index 692a5acc2ffc4..1c7f3b74ffcf3 100644
--- a/include/linux/bootconfig.h
+++ b/include/linux/bootconfig.h
@@ -265,6 +265,9 @@ static inline struct xbc_node * __init xbc_node_get_subkey(struct xbc_node *node
 int __init xbc_node_compose_key_after(struct xbc_node *root,
 			struct xbc_node *node, char *buf, size_t size);
 
+/* Render key/value pairs under @root as a flat cmdline string */
+int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root);
+
 /**
  * xbc_node_compose_key() - Compose full key string of the XBC node
  * @node: An XBC node.
diff --git a/include/linux/bootmem_info.h b/include/linux/bootmem_info.h
index 492ceeb1cdf89..f724340755e55 100644
--- a/include/linux/bootmem_info.h
+++ b/include/linux/bootmem_info.h
@@ -82,7 +82,6 @@ static inline void get_page_bootmem(unsigned long info, struct page *page,
 
 static inline void free_bootmem_page(struct page *page)
 {
-	kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);
 	free_reserved_page(page);
 }
 #endif
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 01e2039648928..1df67a59cd972 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1151,6 +1151,11 @@ struct bpf_prog_offload {
 
 /* The longest tracepoint has 12 args.
  * See include/trace/bpf_probe.h
+ *
+ * Also reuse this macro for maximum number of arguments a BPF function
+ * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via
+ * stack arg slots. The JIT may map some stack arg slots to registers based
+ * on the native calling convention (e.g., arg 6 to R9 on x86-64).
  */
 #define MAX_BPF_FUNC_ARGS 12
 
@@ -1543,6 +1548,7 @@ void bpf_jit_uncharge_modmem(u32 size);
 bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
 bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog,
 				 int insn_idx);
+u16 bpf_out_stack_arg_cnt(const struct bpf_verifier_env *env, const struct bpf_prog *prog);
 #else
 static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link,
 					   struct bpf_trampoline *tr,
@@ -1730,6 +1736,7 @@ struct bpf_prog_aux {
 	struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE];
 	char name[BPF_OBJ_NAME_LEN];
 	u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64);
+	u16 stack_arg_sp_adjust;
 #ifdef CONFIG_SECURITY
 	void *security;
 #endif
@@ -2914,10 +2921,18 @@ int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size,
 			     size_t actual_size);
 
 /* verify correctness of eBPF program */
-int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size);
+struct bpf_log_attr;
+int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr,
+	      struct bpf_log_attr *attr_log);
 
 #ifndef CONFIG_BPF_JIT_ALWAYS_ON
-void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth);
+s32 bpf_call_args_imm(s16 idx);
+#else
+static inline s32 bpf_call_args_imm(s16 idx)
+{
+	return 0;
+}
 #endif
 
 struct btf *bpf_get_btf_vmlinux(void);
@@ -3079,6 +3094,56 @@ void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr);
 void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr);
 void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip);
 
+static __always_inline u32
+bpf_prog_run_array_sleepable(const struct bpf_prog_array *array,
+			     const void *ctx, bpf_prog_run_fn run_prog)
+{
+	const struct bpf_prog_array_item *item;
+	struct bpf_prog *prog;
+	struct bpf_run_ctx *old_run_ctx;
+	struct bpf_trace_run_ctx run_ctx;
+	u32 ret = 1;
+
+	if (unlikely(!array))
+		return ret;
+
+	migrate_disable();
+
+	run_ctx.is_uprobe = false;
+
+	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
+	item = &array->items[0];
+	while ((prog = READ_ONCE(item->prog))) {
+		/* Skip dummy_bpf_prog placeholder (len == 0) */
+		if (unlikely(!prog->len)) {
+			item++;
+			continue;
+		}
+
+		if (unlikely(!bpf_prog_get_recursion_context(prog))) {
+			bpf_prog_inc_misses_counter(prog);
+			bpf_prog_put_recursion_context(prog);
+			item++;
+			continue;
+		}
+
+		run_ctx.bpf_cookie = item->bpf_cookie;
+
+		if (!prog->sleepable) {
+			guard(rcu)();
+			ret &= run_prog(prog, ctx);
+		} else {
+			ret &= run_prog(prog, ctx);
+		}
+
+		bpf_prog_put_recursion_context(prog);
+		item++;
+	}
+	bpf_reset_run_ctx(old_run_ctx);
+	migrate_enable();
+	return ret;
+}
+
 #else /* !CONFIG_BPF_SYSCALL */
 static inline struct bpf_prog *bpf_prog_get(u32 ufd)
 {
@@ -3622,8 +3687,8 @@ static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map,
 struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags);
 struct bpf_key *bpf_lookup_system_key(u64 id);
 void bpf_key_put(struct bpf_key *bkey);
-int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
-			       struct bpf_dynptr *sig_p,
+int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p,
+			       const struct bpf_dynptr *sig_p,
 			       struct bpf_key *trusted_keyring);
 
 #else
@@ -3641,8 +3706,8 @@ static inline void bpf_key_put(struct bpf_key *bkey)
 {
 }
 
-static inline int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
-					     struct bpf_dynptr *sig_p,
+static inline int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p,
+					     const struct bpf_dynptr *sig_p,
 					     struct bpf_key *trusted_keyring)
 {
 	return -EOPNOTSUPP;
diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h
index b148f816f25b0..5cbad3b64130c 100644
--- a/include/linux/bpf_verifier.h
+++ b/include/linux/bpf_verifier.h
@@ -8,6 +8,7 @@
 #include <linux/btf.h> /* for struct btf and btf_id() */
 #include <linux/filter.h> /* for MAX_BPF_STACK */
 #include <linux/tnum.h>
+#include <linux/cnum.h>
 
 /* Maximum variable offset umax_value permitted when resolving memory accesses.
  * In practice this is far bigger than any realistic pointer offset; this limit
@@ -120,14 +121,8 @@ struct bpf_reg_state {
 	 * These refer to the same value as var_off, not necessarily the actual
 	 * contents of the register.
 	 */
-	s64 smin_value; /* minimum possible (s64)value */
-	s64 smax_value; /* maximum possible (s64)value */
-	u64 umin_value; /* minimum possible (u64)value */
-	u64 umax_value; /* maximum possible (u64)value */
-	s32 s32_min_value; /* minimum possible (s32)value */
-	s32 s32_max_value; /* maximum possible (s32)value */
-	u32 u32_min_value; /* minimum possible (u32)value */
-	u32 u32_max_value; /* maximum possible (u32)value */
+	struct cnum64 r64; /* 64-bit range as circular number */
+	struct cnum32 r32; /* 32-bit range as circular number */
 	/* For PTR_TO_PACKET, used to find other pointers with the same variable
 	 * offset, so they can share range knowledge.
 	 * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we
@@ -209,6 +204,66 @@ struct bpf_reg_state {
 	bool precise;
 };
 
+static inline s64 reg_smin(const struct bpf_reg_state *reg)
+{
+	return cnum64_smin(reg->r64);
+}
+
+static inline s64 reg_smax(const struct bpf_reg_state *reg)
+{
+	return cnum64_smax(reg->r64);
+}
+
+static inline u64 reg_umin(const struct bpf_reg_state *reg)
+{
+	return cnum64_umin(reg->r64);
+}
+
+static inline u64 reg_umax(const struct bpf_reg_state *reg)
+{
+	return cnum64_umax(reg->r64);
+}
+
+static inline s32 reg_s32_min(const struct bpf_reg_state *reg)
+{
+	return cnum32_smin(reg->r32);
+}
+
+static inline s32 reg_s32_max(const struct bpf_reg_state *reg)
+{
+	return cnum32_smax(reg->r32);
+}
+
+static inline u32 reg_u32_min(const struct bpf_reg_state *reg)
+{
+	return cnum32_umin(reg->r32);
+}
+
+static inline u32 reg_u32_max(const struct bpf_reg_state *reg)
+{
+	return cnum32_umax(reg->r32);
+}
+
+static inline void reg_set_srange32(struct bpf_reg_state *reg, s32 smin, s32 smax)
+{
+	reg->r32 = cnum32_from_srange(smin, smax);
+}
+
+static inline void reg_set_urange32(struct bpf_reg_state *reg, u32 umin, u32 umax)
+{
+	reg->r32 = cnum32_from_urange(umin, umax);
+}
+
+static inline void reg_set_srange64(struct bpf_reg_state *reg, s64 smin, s64 smax)
+{
+	reg->r64 = cnum64_from_srange(smin, smax);
+}
+
+static inline void reg_set_urange64(struct bpf_reg_state *reg, u64 umin, u64 umax)
+{
+	reg->r64 = cnum64_from_urange(umin, umax);
+}
+
 enum bpf_stack_slot_type {
 	STACK_INVALID,    /* nothing was stored in this stack slot */
 	STACK_SPILL,      /* register spilled into stack */
@@ -347,6 +402,7 @@ struct bpf_func_state {
 	bool in_callback_fn;
 	bool in_async_callback_fn;
 	bool in_exception_callback_fn;
+	bool no_stack_arg_load;
 	/* For callback calling functions that limit number of possible
 	 * callback executions (e.g. bpf_loop) keeps track of current
 	 * simulated iteration number.
@@ -372,46 +428,48 @@ struct bpf_func_state {
 	 * `stack`. allocated_stack is always a multiple of BPF_REG_SIZE.
 	 */
 	int allocated_stack;
+
+	u16 out_stack_arg_cnt; /* Number of outgoing on-stack argument slots */
+	struct bpf_reg_state *stack_arg_regs; /* Outgoing on-stack arguments */
 };
 
 #define MAX_CALL_FRAMES 8
 
-/* instruction history flags, used in bpf_jmp_history_entry.flags field */
+/* instruction history flags, used in bpf_jmp_history_entry.flags field.
+ * Frame number and SPI are stored in dedicated fields of bpf_jmp_history_entry.
+ */
 enum {
-	/* instruction references stack slot through PTR_TO_STACK register;
-	 * we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8)
-	 * and accessed stack slot's index in next 6 bits (MAX_BPF_STACK is 512,
-	 * 8 bytes per slot, so slot index (spi) is [0, 63])
-	 */
-	INSN_F_FRAMENO_MASK = 0x7, /* 3 bits */
+	INSN_F_STACK_ACCESS = BIT(0),
 
-	INSN_F_SPI_MASK = 0x3f, /* 6 bits */
-	INSN_F_SPI_SHIFT = 3, /* shifted 3 bits to the left */
+	INSN_F_DST_REG_STACK = BIT(1), /* dst_reg is PTR_TO_STACK */
+	INSN_F_SRC_REG_STACK = BIT(2), /* src_reg is PTR_TO_STACK */
 
-	INSN_F_STACK_ACCESS = BIT(9),
-
-	INSN_F_DST_REG_STACK = BIT(10), /* dst_reg is PTR_TO_STACK */
-	INSN_F_SRC_REG_STACK = BIT(11), /* src_reg is PTR_TO_STACK */
-	/* total 12 bits are used now. */
+	INSN_F_STACK_ARG_ACCESS = BIT(3),
 };
 
-static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES);
-static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8);
-
 struct bpf_jmp_history_entry {
-	u32 idx;
 	/* insn idx can't be bigger than 1 million */
+	u32 idx : 20;
+	u32 frame : 3;	/* stack access frame number */
+	u32 spi : 6;	/* stack slot index (0..63) */
+	u32 : 3;
 	u32 prev_idx : 20;
 	/* special INSN_F_xxx flags */
-	u32 flags : 12;
+	u32 flags : 4;
+	u32 : 8;
 	/* additional registers that need precision tracking when this
 	 * jump is backtracked, vector of six 10-bit records
 	 */
 	u64 linked_regs;
 };
 
-/* Maximum number of register states that can exist at once */
-#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) * MAX_CALL_FRAMES)
+static_assert(MAX_CALL_FRAMES <= (1 << 3));
+static_assert(MAX_BPF_STACK / 8 <= (1 << 6));
+
+/* Maximum number of bpf_reg_state objects that can exist at once */
+#define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS)
+#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE + \
+			  MAX_STACK_ARG_SLOTS) * MAX_CALL_FRAMES)
 struct bpf_verifier_state {
 	/* call stack tracking */
 	struct bpf_func_state *frame[MAX_CALL_FRAMES];
@@ -497,10 +555,23 @@ struct bpf_verifier_state {
 	u32 may_goto_depth;
 };
 
-#define bpf_get_spilled_reg(slot, frame, mask)				\
-	(((slot < frame->allocated_stack / BPF_REG_SIZE) &&		\
-	  ((1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & (mask))) \
-	 ? &frame->stack[slot].spilled_ptr : NULL)
+static inline struct bpf_reg_state *
+bpf_get_spilled_reg(int slot, struct bpf_func_state *frame, u32 mask)
+{
+	if (slot < frame->allocated_stack / BPF_REG_SIZE &&
+	    (1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & mask)
+		return &frame->stack[slot].spilled_ptr;
+	return NULL;
+}
+
+static inline struct bpf_reg_state *
+bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame)
+{
+	if (slot < frame->out_stack_arg_cnt &&
+	    frame->stack_arg_regs[slot].type != NOT_INIT)
+		return &frame->stack_arg_regs[slot];
+	return NULL;
+}
 
 /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */
 #define bpf_for_each_spilled_reg(iter, frame, reg, mask)			\
@@ -508,6 +579,12 @@ struct bpf_verifier_state {
 	     iter < frame->allocated_stack / BPF_REG_SIZE;		\
 	     iter++, reg = bpf_get_spilled_reg(iter, frame, mask))
 
+/* Iterate over 'frame', setting 'reg' to either NULL or a spilled stack arg. */
+#define bpf_for_each_spilled_stack_arg(iter, frame, reg)               \
+	for (iter = 0, reg = bpf_get_spilled_stack_arg(iter, frame);   \
+	     iter < frame->out_stack_arg_cnt;                          \
+	     iter++, reg = bpf_get_spilled_stack_arg(iter, frame))
+
 #define bpf_for_each_reg_in_vstate_mask(__vst, __state, __reg, __mask, __expr)   \
 	({                                                               \
 		struct bpf_verifier_state *___vstate = __vst;            \
@@ -525,6 +602,11 @@ struct bpf_verifier_state {
 					continue;                        \
 				(void)(__expr);                          \
 			}                                                \
+			bpf_for_each_spilled_stack_arg(___j, __state, __reg) { \
+				if (!__reg)                              \
+					continue;                        \
+				(void)(__expr);                          \
+			}						 \
 		}                                                        \
 	})
 
@@ -700,6 +782,22 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log)
 	return log && log->level;
 }
 
+struct bpf_log_attr {
+	char __user *ubuf;
+	u32 size;
+	u32 level;
+	u32 offsetof_true_size;
+	bpfptr_t uattr;
+};
+
+int bpf_log_attr_init(struct bpf_log_attr *log, u64 log_buf, u32 log_size, u32 log_level,
+		      u32 offsetof_log_true_size, bpfptr_t uattr, struct bpf_common_attr *common,
+		      bpfptr_t uattr_common, u32 size_common);
+struct bpf_verifier_log *bpf_log_attr_create_vlog(struct bpf_log_attr *attr_log,
+						  struct bpf_common_attr *common, bpfptr_t uattr,
+						  u32 size);
+int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *log);
+
 #define BPF_MAX_SUBPROGS 256
 
 struct bpf_subprog_arg_info {
@@ -724,11 +822,13 @@ struct bpf_subprog_info {
 	u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */
 	u16 stack_depth; /* max. stack depth used by this function */
 	u16 stack_extra;
+	u32 insn_processed;
 	/* offsets in range [stack_depth .. fastcall_stack_off)
 	 * are used for bpf_fastcall spills and fills.
 	 */
 	s16 fastcall_stack_off;
 	bool has_tail_call: 1;
+	bool might_throw: 1;
 	bool tail_call_reachable: 1;
 	bool has_ld_abs: 1;
 	bool is_cb: 1;
@@ -739,12 +839,21 @@ struct bpf_subprog_info {
 	bool keep_fastcall_stack: 1;
 	bool changes_pkt_data: 1;
 	bool might_sleep: 1;
-	u8 arg_cnt:3;
+	u8 arg_cnt:4;
 
 	enum priv_stack_mode priv_stack_mode;
-	struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS];
+	struct bpf_subprog_arg_info args[MAX_BPF_FUNC_ARGS];
+	u16 stack_arg_cnt; /* incoming + max outgoing */
+	u16 max_out_stack_arg_cnt;
 };
 
+static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub)
+{
+	if (sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS)
+		return sub->arg_cnt - MAX_BPF_FUNC_REG_ARGS;
+	return 0;
+}
+
 struct bpf_verifier_env;
 
 struct backtrack_state {
@@ -752,6 +861,7 @@ struct backtrack_state {
 	u32 frame;
 	u32 reg_masks[MAX_CALL_FRAMES];
 	u64 stack_masks[MAX_CALL_FRAMES];
+	u8 stack_arg_masks[MAX_CALL_FRAMES];
 };
 
 struct bpf_id_pair {
@@ -880,6 +990,8 @@ struct bpf_verifier_env {
 	u32 prev_insn_processed, insn_processed;
 	/* number of jmps, calls, exits analyzed so far */
 	u32 prev_jmps_processed, jmps_processed;
+	/* maximum combined stack depth */
+	u32 max_stack_depth;
 	/* total verification time */
 	u64 verification_time;
 	/* maximum number of verifier states kept in 'branching' instructions */
@@ -913,6 +1025,7 @@ struct bpf_verifier_env {
 	 * e.g., in reg_type_str() to generate reg_type string
 	 */
 	char tmp_str_buf[TMP_STR_BUF_LEN];
+	char tmp_arg_name[32];
 	struct bpf_insn insn_buf[INSN_BUF_SIZE];
 	struct bpf_insn epilogue_buf[INSN_BUF_SIZE];
 	struct bpf_scc_callchain callchain_buf;
@@ -1086,7 +1199,7 @@ struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx);
 void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self);
 void bpf_free_backedges(struct bpf_scc_visit *visit);
 int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
-			 int insn_flags, u64 linked_regs);
+			 int insn_flags, int spi, int frame, u64 linked_regs);
 void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist);
 void bpf_mark_reg_not_init(const struct bpf_verifier_env *env,
 			   struct bpf_reg_state *reg);
@@ -1149,6 +1262,11 @@ static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame,
 	bt->stack_masks[frame] |= 1ull << slot;
 }
 
+static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot)
+{
+	bt->stack_arg_masks[frame] |= 1 << slot;
+}
+
 static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg)
 {
 	return bt->reg_masks[frame] & (1 << reg);
@@ -1308,6 +1426,7 @@ void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask);
 bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog);
 
 int bpf_find_subprog(struct bpf_verifier_env *env, int off);
+bool bpf_is_throw_kfunc(struct bpf_insn *insn);
 int bpf_compute_const_regs(struct bpf_verifier_env *env);
 int bpf_prune_dead_branches(struct bpf_verifier_env *env);
 int bpf_check_cfg(struct bpf_verifier_env *env);
diff --git a/include/linux/btf.h b/include/linux/btf.h
index 48108471c5b13..240401d9b25b9 100644
--- a/include/linux/btf.h
+++ b/include/linux/btf.h
@@ -145,7 +145,8 @@ const char *btf_get_name(const struct btf *btf);
 void btf_get(struct btf *btf);
 void btf_put(struct btf *btf);
 const struct btf_header *btf_header(const struct btf *btf);
-int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_sz);
+struct bpf_log_attr;
+int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log);
 struct btf *btf_get_by_fd(int fd);
 int btf_get_info_by_fd(const struct btf *btf,
 		       const union bpf_attr *attr,
@@ -415,12 +416,12 @@ static inline bool btf_type_is_array(const struct btf_type *t)
 	return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY;
 }
 
-static inline u16 btf_type_vlen(const struct btf_type *t)
+static inline u32 btf_type_vlen(const struct btf_type *t)
 {
 	return BTF_INFO_VLEN(t->info);
 }
 
-static inline u16 btf_vlen(const struct btf_type *t)
+static inline u32 btf_vlen(const struct btf_type *t)
 {
 	return btf_type_vlen(t);
 }
diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h
index c8f4f0a0b874e..fc879ac4cc4f2 100644
--- a/include/linux/cacheinfo.h
+++ b/include/linux/cacheinfo.h
@@ -89,6 +89,7 @@ int populate_cache_leaves(unsigned int cpu);
 int cache_setup_acpi(unsigned int cpu);
 bool last_level_cache_is_valid(unsigned int cpu);
 bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y);
+struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu);
 int fetch_cache_info(unsigned int cpu);
 int detect_cache_attributes(unsigned int cpu);
 #ifndef CONFIG_ACPI_PPTT
diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h
index 50a784da7a81a..de2cd6238c2ae 100644
--- a/include/linux/cgroup-defs.h
+++ b/include/linux/cgroup-defs.h
@@ -253,6 +253,18 @@ struct cgroup_subsys_state {
 	 */
 	int nr_descendants;
 
+	/*
+	 * Hierarchical populated state. For cgroup->self, nr_populated_csets
+	 * counts populated csets linked via cgrp_cset_link.
+	 * nr_populated_children counts immediate-child csses whose own
+	 * populated state is nonzero. Protected by css_set_lock.
+	 */
+	int nr_populated_csets;
+	int nr_populated_children;
+
+	/* deferred kill_css_finish() queued by css_update_populated() */
+	struct work_struct kill_finish_work;
+
 	/*
 	 * A singly-linked list of css structures to be rstat flushed.
 	 * This is a scratch field to be used exclusively by
@@ -504,17 +516,12 @@ struct cgroup {
 	int max_descendants;
 
 	/*
-	 * Each non-empty css_set associated with this cgroup contributes
-	 * one to nr_populated_csets.  The counter is zero iff this cgroup
-	 * doesn't have any tasks.
-	 *
-	 * All children which have non-zero nr_populated_csets and/or
-	 * nr_populated_children of their own contribute one to either
-	 * nr_populated_domain_children or nr_populated_threaded_children
-	 * depending on their type.  Each counter is zero iff all cgroups
-	 * of the type in the subtree proper don't have any tasks.
+	 * Domain/threaded split of self.nr_populated_children: each counts
+	 * immediate-child cgroups whose subtree is populated and sums to
+	 * self.nr_populated_children. Kept as separate fields to allow readers
+	 * like cgroup_can_be_thread_root() unlocked access. Protected by
+	 * css_set_lock; updated by css_update_populated().
 	 */
-	int nr_populated_csets;
 	int nr_populated_domain_children;
 	int nr_populated_threaded_children;
 
@@ -611,9 +618,6 @@ struct cgroup {
 	/* used to wait for offlining of csses */
 	wait_queue_head_t offline_waitq;
 
-	/* defers killing csses after removal until cgroup is depopulated */
-	struct work_struct finish_destroy_work;
-
 	/* used to schedule release agent */
 	struct work_struct release_agent_work;
 
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index f6d037a30fd89..f2aa46a4f871e 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -640,11 +640,32 @@ static inline bool task_under_cgroup_hierarchy(struct task_struct *task,
 	return cgroup_is_descendant(cset->dfl_cgrp, ancestor);
 }
 
-/* no synchronization, the result can only be used as a hint */
+/*
+ * Populated counters: writes happen under css_set_lock. The accessors below
+ * may read unlocked. What an unpopulated result means depends on context:
+ *
+ * - No lock held. Just a snapshot. May race with concurrent updates and is
+ *   useful only as a hint.
+ *
+ * - cgroup_mutex held. Migration into the cgroup is blocked, so an observed
+ *   !populated stays !populated until cgroup_mutex is dropped.
+ *
+ * - CSS_DYING set. The css can no longer be repopulated, so !populated is
+ *   sticky once observed.
+ */
+static inline bool cgroup_has_tasks(struct cgroup *cgrp)
+{
+	return READ_ONCE(cgrp->self.nr_populated_csets);
+}
+
+static inline bool css_is_populated(struct cgroup_subsys_state *css)
+{
+	return READ_ONCE(css->nr_populated_csets) || READ_ONCE(css->nr_populated_children);
+}
+
 static inline bool cgroup_is_populated(struct cgroup *cgrp)
 {
-	return cgrp->nr_populated_csets + cgrp->nr_populated_domain_children +
-		cgrp->nr_populated_threaded_children;
+	return css_is_populated(&cgrp->self);
 }
 
 /* returns ino associated with a cgroup */
@@ -777,6 +798,7 @@ static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen)
 /*
  * cgroup scalable recursive statistics.
  */
+void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu);
 void css_rstat_updated(struct cgroup_subsys_state *css, int cpu);
 void css_rstat_flush(struct cgroup_subsys_state *css);
 
diff --git a/include/linux/cgroup_rdma.h b/include/linux/cgroup_rdma.h
index 80edae03c3138..404e746552ca8 100644
--- a/include/linux/cgroup_rdma.h
+++ b/include/linux/cgroup_rdma.h
@@ -24,6 +24,10 @@ struct rdma_cgroup {
 	 * that belongs to this cgroup.
 	 */
 	struct list_head		rpools;
+
+	/* Handles for rdma.events[.local] */
+	struct cgroup_file		events_file;
+	struct cgroup_file		events_local_file;
 };
 
 struct rdmacg_device {
diff --git a/include/linux/clk/renesas.h b/include/linux/clk/renesas.h
index c360df9fa735c..0949400f44de4 100644
--- a/include/linux/clk/renesas.h
+++ b/include/linux/clk/renesas.h
@@ -164,6 +164,26 @@ struct rzv2h_pll_div_pars {
 		.k = { .min = -32768, .max = 32767 },			\
 	}								\
 
+#define RZG3E_CPG_PLL_DSI0_LIMITS(name)					\
+	static const struct rzv2h_pll_limits (name) = {			\
+		.fout = { .min = 25 * MEGA, .max = 1218 * MEGA },	\
+		.fvco = { .min = 1600 * MEGA, .max = 3200 * MEGA },	\
+		.m = { .min = 64, .max = 533 },				\
+		.p = { .min = 1, .max = 4 },				\
+		.s = { .min = 0, .max = 6 },				\
+		.k = { .min = -32768, .max = 32767 },			\
+	}								\
+
+#define RZG3E_CPG_PLL_DSI1_LIMITS(name)					\
+	static const struct rzv2h_pll_limits (name) = {			\
+		.fout = { .min = 25 * MEGA, .max = 609 * MEGA },	\
+		.fvco = { .min = 1600 * MEGA, .max = 3200 * MEGA },	\
+		.m = { .min = 64, .max = 533 },				\
+		.p = { .min = 1, .max = 4 },				\
+		.s = { .min = 0, .max = 6 },				\
+		.k = { .min = -32768, .max = 32767 },			\
+	}								\
+
 #ifdef CONFIG_CLK_RZV2H
 bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits,
 			struct rzv2h_pll_pars *pars, u64 freq_millihz);
diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h
index 7c38190b10bf0..b12a6d19aa60a 100644
--- a/include/linux/clocksource.h
+++ b/include/linux/clocksource.h
@@ -236,8 +236,9 @@ clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 minsec);
  */
 extern int
 __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq);
-extern void
-__clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq);
+extern int
+__devm_clocksource_register_scale(struct device *dev, struct clocksource *cs,
+				  u32 scale, u32 freq);
 
 /*
  * Don't call this unless you are a default clocksource
@@ -258,14 +259,16 @@ static inline int clocksource_register_khz(struct clocksource *cs, u32 khz)
 	return __clocksource_register_scale(cs, 1000, khz);
 }
 
-static inline void __clocksource_update_freq_hz(struct clocksource *cs, u32 hz)
+static inline int devm_clocksource_register_hz(struct device *dev,
+					       struct clocksource *cs, u32 hz)
 {
-	__clocksource_update_freq_scale(cs, 1, hz);
+	return __devm_clocksource_register_scale(dev, cs, 1, hz);
 }
 
-static inline void __clocksource_update_freq_khz(struct clocksource *cs, u32 khz)
+static inline int devm_clocksource_register_khz(struct device *dev,
+						struct clocksource *cs, u32 khz)
 {
-	__clocksource_update_freq_scale(cs, 1000, khz);
+	return __devm_clocksource_register_scale(dev, cs, 1000, khz);
 }
 
 #ifdef CONFIG_ARCH_CLOCKSOURCE_INIT
diff --git a/include/linux/cnt32_to_63.h b/include/linux/cnt32_to_63.h
deleted file mode 100644
index 064428479f2d4..0000000000000
--- a/include/linux/cnt32_to_63.h
+++ /dev/null
@@ -1,104 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- *  Extend a 32-bit counter to 63 bits
- *
- *  Author:	Nicolas Pitre
- *  Created:	December 3, 2006
- *  Copyright:	MontaVista Software, Inc.
- */
-
-#ifndef __LINUX_CNT32_TO_63_H__
-#define __LINUX_CNT32_TO_63_H__
-
-#include <linux/compiler.h>
-#include <linux/types.h>
-#include <asm/byteorder.h>
-
-/* this is used only to give gcc a clue about good code generation */
-union cnt32_to_63 {
-	struct {
-#if defined(__LITTLE_ENDIAN)
-		u32 lo, hi;
-#elif defined(__BIG_ENDIAN)
-		u32 hi, lo;
-#endif
-	};
-	u64 val;
-};
-
-
-/**
- * cnt32_to_63 - Expand a 32-bit counter to a 63-bit counter
- * @cnt_lo: The low part of the counter
- *
- * Many hardware clock counters are only 32 bits wide and therefore have
- * a relatively short period making wrap-arounds rather frequent.  This
- * is a problem when implementing sched_clock() for example, where a 64-bit
- * non-wrapping monotonic value is expected to be returned.
- *
- * To overcome that limitation, let's extend a 32-bit counter to 63 bits
- * in a completely lock free fashion. Bits 0 to 31 of the clock are provided
- * by the hardware while bits 32 to 62 are stored in memory.  The top bit in
- * memory is used to synchronize with the hardware clock half-period.  When
- * the top bit of both counters (hardware and in memory) differ then the
- * memory is updated with a new value, incrementing it when the hardware
- * counter wraps around.
- *
- * Because a word store in memory is atomic then the incremented value will
- * always be in synch with the top bit indicating to any potential concurrent
- * reader if the value in memory is up to date or not with regards to the
- * needed increment.  And any race in updating the value in memory is harmless
- * as the same value would simply be stored more than once.
- *
- * The restrictions for the algorithm to work properly are:
- *
- * 1) this code must be called at least once per each half period of the
- *    32-bit counter;
- *
- * 2) this code must not be preempted for a duration longer than the
- *    32-bit counter half period minus the longest period between two
- *    calls to this code;
- *
- * Those requirements ensure proper update to the state bit in memory.
- * This is usually not a problem in practice, but if it is then a kernel
- * timer should be scheduled to manage for this code to be executed often
- * enough.
- *
- * And finally:
- *
- * 3) the cnt_lo argument must be seen as a globally incrementing value,
- *    meaning that it should be a direct reference to the counter data which
- *    can be evaluated according to a specific ordering within the macro,
- *    and not the result of a previous evaluation stored in a variable.
- *
- * For example, this is wrong:
- *
- *	u32 partial = get_hw_count();
- *	u64 full = cnt32_to_63(partial);
- *	return full;
- *
- * This is fine:
- *
- *	u64 full = cnt32_to_63(get_hw_count());
- *	return full;
- *
- * Note that the top bit (bit 63) in the returned value should be considered
- * as garbage.  It is not cleared here because callers are likely to use a
- * multiplier on the returned value which can get rid of the top bit
- * implicitly by making the multiplier even, therefore saving on a runtime
- * clear-bit instruction. Otherwise caller must remember to clear the top
- * bit explicitly.
- */
-#define cnt32_to_63(cnt_lo) \
-({ \
-	static u32 __m_cnt_hi; \
-	union cnt32_to_63 __x; \
-	__x.hi = __m_cnt_hi; \
- 	smp_rmb(); \
-	__x.lo = (cnt_lo); \
-	if (unlikely((s32)(__x.hi ^ __x.lo) < 0)) \
-		__m_cnt_hi = __x.hi = (__x.hi ^ 0x80000000) + (__x.hi >> 31); \
-	__x.val; \
-})
-
-#endif
diff --git a/include/linux/cnum.h b/include/linux/cnum.h
new file mode 100644
index 0000000000000..49b7d0c7645de
--- /dev/null
+++ b/include/linux/cnum.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#ifndef _LINUX_CNUM_H
+#define _LINUX_CNUM_H
+
+#include <linux/types.h>
+
+/*
+ * cnum32: a circular number.
+ * A unified representation for signed and unsigned ranges.
+ *
+ * Assume that a 32-bit range is a circle, with 0 being in the 12 o'clock
+ * position, numbers placed sequentially in clockwise order and U32_MAX
+ * in the 11 o'clock position. Signed values map onto the same circle:
+ * S32_MAX sits at 5 o'clock, S32_MIN sits at 6 o'clock (opposite 0),
+ * negative values occupy the left half and positive values the right half.
+ *
+ * @cnum32 represents an arc on this circle drawn clockwise.
+ * @base corresponds to the first value of the range.
+ * @size corresponds to the number of integers in the range excluding @base.
+ * (The @base is excluded to avoid integer overflow when representing the full
+ *  0..U32_MAX range, which corresponds to 2^32, which can't be stored in u32).
+ *
+ * For example: {U32_MAX, 1} corresponds to signed range [-1, 0],
+ *              {S32_MAX, 1} corresponds to unsigned range [S32_MAX, S32_MIN].
+ */
+struct cnum32 {
+	u32 base;
+	u32 size;
+};
+
+#define CNUM32_UNBOUNDED ((struct cnum32){ .base = 0, .size = U32_MAX })
+#define CNUM32_EMPTY ((struct cnum32){ .base = U32_MAX, .size = U32_MAX })
+
+struct cnum32 cnum32_from_urange(u32 min, u32 max);
+struct cnum32 cnum32_from_srange(s32 min, s32 max);
+u32 cnum32_umin(struct cnum32 cnum);
+u32 cnum32_umax(struct cnum32 cnum);
+s32 cnum32_smin(struct cnum32 cnum);
+s32 cnum32_smax(struct cnum32 cnum);
+struct cnum32 cnum32_intersect(struct cnum32 a, struct cnum32 b);
+void cnum32_intersect_with(struct cnum32 *dst, struct cnum32 src);
+void cnum32_intersect_with_urange(struct cnum32 *dst, u32 min, u32 max);
+void cnum32_intersect_with_srange(struct cnum32 *dst, s32 min, s32 max);
+bool cnum32_contains(struct cnum32 cnum, u32 v);
+bool cnum32_is_const(struct cnum32 cnum);
+bool cnum32_is_empty(struct cnum32 cnum);
+struct cnum32 cnum32_add(struct cnum32 a, struct cnum32 b);
+struct cnum32 cnum32_negate(struct cnum32 a);
+bool cnum32_is_subset(struct cnum32 outer, struct cnum32 inner);
+
+/* Same as cnum32 but for 64-bit ranges */
+struct cnum64 {
+	u64 base;
+	u64 size;
+};
+
+#define CNUM64_UNBOUNDED ((struct cnum64){ .base = 0, .size = U64_MAX })
+#define CNUM64_EMPTY ((struct cnum64){ .base = U64_MAX, .size = U64_MAX })
+
+struct cnum64 cnum64_from_urange(u64 min, u64 max);
+struct cnum64 cnum64_from_srange(s64 min, s64 max);
+u64 cnum64_umin(struct cnum64 cnum);
+u64 cnum64_umax(struct cnum64 cnum);
+s64 cnum64_smin(struct cnum64 cnum);
+s64 cnum64_smax(struct cnum64 cnum);
+struct cnum64 cnum64_intersect(struct cnum64 a, struct cnum64 b);
+void cnum64_intersect_with(struct cnum64 *dst, struct cnum64 src);
+void cnum64_intersect_with_urange(struct cnum64 *dst, u64 min, u64 max);
+void cnum64_intersect_with_srange(struct cnum64 *dst, s64 min, s64 max);
+bool cnum64_contains(struct cnum64 cnum, u64 v);
+bool cnum64_is_const(struct cnum64 cnum);
+bool cnum64_is_empty(struct cnum64 cnum);
+struct cnum64 cnum64_add(struct cnum64 a, struct cnum64 b);
+struct cnum64 cnum64_negate(struct cnum64 a);
+bool cnum64_is_subset(struct cnum64 outer, struct cnum64 inner);
+
+struct cnum32 cnum32_from_cnum64(struct cnum64 cnum);
+struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b);
+
+#endif /* _LINUX_CNUM_H */
diff --git a/include/linux/codetag.h b/include/linux/codetag.h
index 8ea2a5f7c98a5..ddae7484ca456 100644
--- a/include/linux/codetag.h
+++ b/include/linux/codetag.h
@@ -74,8 +74,9 @@ struct codetag_iterator {
 	.flags		= 0,				\
 }
 
-void codetag_lock_module_list(struct codetag_type *cttype, bool lock);
+void codetag_lock_module_list(struct codetag_type *cttype);
 bool codetag_trylock_module_list(struct codetag_type *cttype);
+void codetag_unlock_module_list(struct codetag_type *cttype);
 struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype);
 struct codetag *codetag_next_ct(struct codetag_iterator *iter);
 
diff --git a/include/linux/compat.h b/include/linux/compat.h
index 56cebaff0c910..a1ce6d559db94 100644
--- a/include/linux/compat.h
+++ b/include/linux/compat.h
@@ -72,6 +72,8 @@
 	__diag_push();								\
 	__diag_ignore(GCC, 8, "-Wattribute-alias",				\
 		      "Type aliasing is used to sanitize syscall arguments");\
+	__diag_ignore(clang, 23, "-Wattribute-alias",				\
+		      "Type aliasing is used to sanitize syscall arguments");\
 	asmlinkage long compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))	\
 		__attribute__((alias(__stringify(__se_compat_sys##name))));	\
 	ALLOW_ERROR_INJECTION(compat_sys##name, ERRNO);				\
diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h
index e1123dd284862..527e4e1360205 100644
--- a/include/linux/compiler-clang.h
+++ b/include/linux/compiler-clang.h
@@ -131,6 +131,12 @@
 #define __diag_str(s)		__diag_str1(s)
 #define __diag(s)		_Pragma(__diag_str(clang diagnostic s))
 
+#if CONFIG_CLANG_VERSION >= 230000
+#define __diag_clang_23(s)	__diag(s)
+#else
+#define __diag_clang_23(s)
+#endif
+
 #define __diag_clang_13(s)	__diag(s)
 
 #define __diag_ignore_all(option, comment) \
diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h
index a9317571e6aff..8302ebc2ea8c8 100644
--- a/include/linux/compiler-context-analysis.h
+++ b/include/linux/compiler-context-analysis.h
@@ -39,12 +39,14 @@
 # define __assumes_shared_ctx_lock(...)	__attribute__((assert_shared_capability(__VA_ARGS__)))
 
 /**
- * __guarded_by - struct member and globals attribute, declares variable
- *                only accessible within active context
+ * __guarded_by() - struct member and globals attribute, declares variable
+ *                  only accessible within active context
+ * @...: context lock instance pointer(s)
  *
  * Declares that the struct member or global variable is only accessible within
- * the context entered by the given context lock. Read operations on the data
- * require shared access, while write operations require exclusive access.
+ * the context entered by the given context lock(s). Read operations on the data
+ * require shared access to at least one of the context locks, while write
+ * operations require exclusive access to all listed context locks.
  *
  * .. code-block:: c
  *
@@ -52,17 +54,24 @@
  *		spinlock_t lock;
  *		long counter __guarded_by(&lock);
  *	};
+ *
+ *	struct some_state {
+ *		spinlock_t lock1, lock2;
+ *		long counter __guarded_by(&lock1, &lock2);
+ *	};
  */
 # define __guarded_by(...)		__attribute__((guarded_by(__VA_ARGS__)))
 
 /**
- * __pt_guarded_by - struct member and globals attribute, declares pointed-to
- *                   data only accessible within active context
+ * __pt_guarded_by() - struct member and globals attribute, declares pointed-to
+ *                     data only accessible within active context
+ * @...: context lock instance pointer(s)
  *
  * Declares that the data pointed to by the struct member pointer or global
  * pointer is only accessible within the context entered by the given context
- * lock. Read operations on the data require shared access, while write
- * operations require exclusive access.
+ * lock(s). Read operations on the data require shared access to at least one
+ * of the context locks, while write operations require exclusive access to all
+ * listed context locks.
  *
  * .. code-block:: c
  *
@@ -70,6 +79,11 @@
  *		spinlock_t lock;
  *		long *counter __pt_guarded_by(&lock);
  *	};
+ *
+ *	struct some_state {
+ *		spinlock_t lock1, lock2;
+ *		long *counter __pt_guarded_by(&lock1, &lock2);
+ *	};
  */
 # define __pt_guarded_by(...)		__attribute__((pt_guarded_by(__VA_ARGS__)))
 
diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h
index e8fd77593b68d..369966598a2c0 100644
--- a/include/linux/compiler_types.h
+++ b/include/linux/compiler_types.h
@@ -711,6 +711,10 @@ struct ftrace_likely_data {
 #define __diag_GCC(version, severity, string)
 #endif
 
+#ifndef __diag_clang
+#define __diag_clang(version, severity, string)
+#endif
+
 #define __diag_push()	__diag(push)
 #define __diag_pop()	__diag(pop)
 
diff --git a/include/linux/console.h b/include/linux/console.h
index 5520e4477ad7e..d624200cfc170 100644
--- a/include/linux/console.h
+++ b/include/linux/console.h
@@ -718,8 +718,6 @@ extern bool console_suspend_enabled;
 extern void console_suspend_all(void);
 extern void console_resume_all(void);
 
-int mda_console_init(void);
-
 void vcs_make_sysfs(int index);
 void vcs_remove_sysfs(int index);
 
diff --git a/include/linux/coresight.h b/include/linux/coresight.h
index 2131febebee93..add0579cad884 100644
--- a/include/linux/coresight.h
+++ b/include/linux/coresight.h
@@ -141,6 +141,8 @@ struct csdev_access {
 		.base		= (_addr),	\
 	})
 
+#define CORESIGHT_DESC_CPU_BOUND	BIT(0)
+
 /**
  * struct coresight_desc - description of a component required from drivers
  * @type:	as defined by @coresight_dev_type.
@@ -153,6 +155,8 @@ struct csdev_access {
  *		in the component's sysfs sub-directory.
  * @name:	name for the coresight device, also shown under sysfs.
  * @access:	Describe access to the device
+ * @flags:	The descritpion flags.
+ * @cpu:	The CPU this component is affined to.
  */
 struct coresight_desc {
 	enum coresight_dev_type type;
@@ -163,6 +167,8 @@ struct coresight_desc {
 	const struct attribute_group **groups;
 	const char *name;
 	struct csdev_access access;
+	u32 flags;
+	int cpu;
 };
 
 /**
@@ -251,6 +257,7 @@ struct coresight_trace_id_map {
  *		by @coresight_ops.
  * @access:	Device i/o access abstraction for this device.
  * @dev:	The device entity associated to this component.
+ * @path:	Activated path pointer (only used for source device).
  * @mode:	The device mode, i.e sysFS, Perf or disabled. This is actually
  *		an 'enum cs_mode' but stored in an atomic type. Access is always
  *		through atomic APIs, ensuring SMP-safe synchronisation between
@@ -260,6 +267,7 @@ struct coresight_trace_id_map {
  *		device's spinlock when the coresight_mutex held and mode ==
  *		CS_MODE_SYSFS. Otherwise it must be accessed from inside the
  *		spinlock.
+ * @cpu:	The CPU this component is affined to (-1 for not CPU bound).
  * @orphan:	true if the component has connections that haven't been linked.
  * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs
  *		by writing a 1 to the 'enable_sink' file.  A sink can be
@@ -284,8 +292,10 @@ struct coresight_device {
 	const struct coresight_ops *ops;
 	struct csdev_access access;
 	struct device dev;
+	struct coresight_path *path;
 	atomic_t mode;
 	int refcnt;
+	int cpu;
 	bool orphan;
 	/* sink specific fields */
 	bool sysfs_sink_activated;
@@ -334,9 +344,9 @@ struct coresight_path {
 };
 
 enum cs_mode {
-	CS_MODE_DISABLED,
-	CS_MODE_SYSFS,
-	CS_MODE_PERF,
+	CS_MODE_DISABLED = 0,
+	CS_MODE_SYSFS	 = BIT(0),
+	CS_MODE_PERF	 = BIT(1),
 };
 
 #define coresight_ops(csdev)	csdev->ops
@@ -387,15 +397,12 @@ struct coresight_ops_link {
 /**
  * struct coresight_ops_source - basic operations for a source
  * Operations available for sources.
- * @cpu_id:	returns the value of the CPU number this component
- *		is associated to.
  * @enable:	enables tracing for a source.
  * @disable:	disables tracing for a source.
  * @resume_perf: resumes tracing for a source in perf session.
  * @pause_perf:	pauses tracing for a source in perf session.
  */
 struct coresight_ops_source {
-	int (*cpu_id)(struct coresight_device *csdev);
 	int (*enable)(struct coresight_device *csdev, struct perf_event *event,
 		      enum cs_mode mode, struct coresight_path *path);
 	void (*disable)(struct coresight_device *csdev,
@@ -433,6 +440,8 @@ struct coresight_ops_panic {
 struct coresight_ops {
 	int (*trace_id)(struct coresight_device *csdev, enum cs_mode mode,
 			struct coresight_device *sink);
+	int (*pm_save_disable)(struct coresight_device *csdev);
+	void (*pm_restore_enable)(struct coresight_device *csdev);
 	const struct coresight_ops_sink *sink_ops;
 	const struct coresight_ops_link *link_ops;
 	const struct coresight_ops_source *source_ops;
@@ -602,6 +611,12 @@ static inline bool coresight_is_percpu_source(struct coresight_device *csdev)
 	       (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_PROC);
 }
 
+static inline bool coresight_is_software_source(struct coresight_device *csdev)
+{
+	return csdev && coresight_is_device_source(csdev) &&
+	       (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE);
+}
+
 static inline bool coresight_is_percpu_sink(struct coresight_device *csdev)
 {
 	return csdev && (csdev->type == CORESIGHT_DEV_TYPE_SINK) &&
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 22ba327ec2278..0fb3a2a62eb00 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -180,7 +180,6 @@ enum cpuhp_state {
 	CPUHP_AP_DUMMY_TIMER_STARTING,
 	CPUHP_AP_ARM_XEN_STARTING,
 	CPUHP_AP_ARM_XEN_RUNSTATE_STARTING,
-	CPUHP_AP_ARM_CORESIGHT_STARTING,
 	CPUHP_AP_ARM_CORESIGHT_CTI_STARTING,
 	CPUHP_AP_ARM64_ISNDEP_STARTING,
 	CPUHP_AP_SMPCFD_DYING,
@@ -200,6 +199,7 @@ enum cpuhp_state {
 	CPUHP_AP_IRQ_AFFINITY_ONLINE,
 	CPUHP_AP_BLK_MQ_ONLINE,
 	CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS,
+	CPUHP_AP_ARM_CORESIGHT_ONLINE,
 	CPUHP_AP_X86_INTEL_EPB_ONLINE,
 	CPUHP_AP_PERF_ONLINE,
 	CPUHP_AP_PERF_X86_ONLINE,
diff --git a/include/linux/damon.h b/include/linux/damon.h
index f2cdb7c3f5e6c..4d4f031bcb453 100644
--- a/include/linux/damon.h
+++ b/include/linux/damon.h
@@ -8,23 +8,18 @@
 #ifndef _DAMON_H_
 #define _DAMON_H_
 
+#include <linux/math64.h>
 #include <linux/memcontrol.h>
 #include <linux/mutex.h>
+#include <linux/prandom.h>
 #include <linux/time64.h>
 #include <linux/types.h>
-#include <linux/random.h>
 
 /* Minimal region size.  Every damon_region is aligned by this. */
 #define DAMON_MIN_REGION_SZ	PAGE_SIZE
 /* Max priority score for DAMON-based operation schemes */
 #define DAMOS_MAX_SCORE		(99)
 
-/* Get a random number in [l, r) */
-static inline unsigned long damon_rand(unsigned long l, unsigned long r)
-{
-	return l + get_random_u32_below(r - l);
-}
-
 /**
  * struct damon_addr_range - Represents an address region of [@start, @end).
  * @start:	Start address of the region (inclusive).
@@ -121,6 +116,7 @@ struct damon_target {
  * @DAMOS_PAGEOUT:	Reclaim the region.
  * @DAMOS_HUGEPAGE:	Call ``madvise()`` for the region with MADV_HUGEPAGE.
  * @DAMOS_NOHUGEPAGE:	Call ``madvise()`` for the region with MADV_NOHUGEPAGE.
+ * @DAMOS_COLLAPSE:	Call ``madvise()`` for the region with MADV_COLLAPSE.
  * @DAMOS_LRU_PRIO:	Prioritize the region on its LRU lists.
  * @DAMOS_LRU_DEPRIO:	Deprioritize the region on its LRU lists.
  * @DAMOS_MIGRATE_HOT:  Migrate the regions prioritizing warmer regions.
@@ -140,6 +136,7 @@ enum damos_action {
 	DAMOS_PAGEOUT,
 	DAMOS_HUGEPAGE,
 	DAMOS_NOHUGEPAGE,
+	DAMOS_COLLAPSE,
 	DAMOS_LRU_PRIO,
 	DAMOS_LRU_DEPRIO,
 	DAMOS_MIGRATE_HOT,
@@ -159,6 +156,8 @@ enum damos_action {
  * @DAMOS_QUOTA_NODE_MEMCG_FREE_BP:	MemFree ratio of a node for a cgroup.
  * @DAMOS_QUOTA_ACTIVE_MEM_BP:		Active to total LRU memory ratio.
  * @DAMOS_QUOTA_INACTIVE_MEM_BP:	Inactive to total LRU memory ratio.
+ * @DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP:	Scheme-eligible memory ratio of a
+ *					node in basis points (0-10000).
  * @NR_DAMOS_QUOTA_GOAL_METRICS:	Number of DAMOS quota goal metrics.
  *
  * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported.
@@ -172,6 +171,7 @@ enum damos_quota_goal_metric {
 	DAMOS_QUOTA_NODE_MEMCG_FREE_BP,
 	DAMOS_QUOTA_ACTIVE_MEM_BP,
 	DAMOS_QUOTA_INACTIVE_MEM_BP,
+	DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP,
 	NR_DAMOS_QUOTA_GOAL_METRICS,
 };
 
@@ -233,6 +233,8 @@ enum damos_quota_goal_tuner {
  * @goals:		Head of quota tuning goals (&damos_quota_goal) list.
  * @goal_tuner:		Goal-based @esz tuning algorithm to use.
  * @esz:		Effective size quota in bytes.
+ * @fail_charge_num:	Failed regions charge rate numerator.
+ * @fail_charge_denom:	Failed regions charge rate denominator.
  *
  * @weight_sz:		Weight of the region's size for prioritization.
  * @weight_nr_accesses:	Weight of the region's nr_accesses for prioritization.
@@ -262,6 +264,10 @@ enum damos_quota_goal_tuner {
  *
  * The resulting effective size quota in bytes is set to @esz.
  *
+ * For DAMOS action applying failed amount of regions, charging those same to
+ * those that the action has successfully applied may be unfair.  For the
+ * reason, 'the size * @fail_charge_num / @fail_charge_denom' is charged.
+ *
  * For selecting regions within the quota, DAMON prioritizes current scheme's
  * target memory regions using the &struct damon_operations->get_scheme_score.
  * You could customize the prioritization logic by setting &weight_sz,
@@ -276,6 +282,9 @@ struct damos_quota {
 	enum damos_quota_goal_tuner goal_tuner;
 	unsigned long esz;
 
+	unsigned int fail_charge_num;
+	unsigned int fail_charge_denom;
+
 	unsigned int weight_sz;
 	unsigned int weight_nr_accesses;
 	unsigned int weight_age;
@@ -787,6 +796,7 @@ struct damon_attrs {
  * @ops:	Set of monitoring operations for given use cases.
  * @addr_unit:	Scale factor for core to ops address conversion.
  * @min_region_sz:	Minimum region size.
+ * @pause:	Pause kdamond main loop.
  * @adaptive_targets:	Head of monitoring targets (&damon_target) list.
  * @schemes:		Head of schemes (&damos) list.
  */
@@ -840,11 +850,31 @@ struct damon_ctx {
 	struct damon_operations ops;
 	unsigned long addr_unit;
 	unsigned long min_region_sz;
+	bool pause;
 
 	struct list_head adaptive_targets;
 	struct list_head schemes;
+
+	/* Per-ctx PRNG state for damon_rand(); kdamond is the sole consumer. */
+	struct rnd_state rnd_state;
 };
 
+/* Get a random number in [@l, @r) using @ctx's lockless PRNG. */
+static inline unsigned long damon_rand(struct damon_ctx *ctx,
+				       unsigned long l, unsigned long r)
+{
+	unsigned long span = r - l;
+	u64 rnd;
+
+	if (span <= U32_MAX) {
+		rnd = prandom_u32_state(&ctx->rnd_state);
+		return l + (unsigned long)((rnd * span) >> 32);
+	}
+	rnd = ((u64)prandom_u32_state(&ctx->rnd_state) << 32) |
+	      prandom_u32_state(&ctx->rnd_state);
+	return l + mul_u64_u64_shr(rnd, span, 64);
+}
+
 static inline struct damon_region *damon_next_region(struct damon_region *r)
 {
 	return container_of(r->list.next, struct damon_region, list);
@@ -994,7 +1024,7 @@ int damon_kdamond_pid(struct damon_ctx *ctx);
 int damon_call(struct damon_ctx *ctx, struct damon_call_control *control);
 int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control);
 
-int damon_set_region_biggest_system_ram_default(struct damon_target *t,
+int damon_set_region_system_rams_default(struct damon_target *t,
 				unsigned long *start, unsigned long *end,
 				unsigned long addr_unit,
 				unsigned long min_region_sz);
diff --git a/include/linux/dcache.h b/include/linux/dcache.h
index 2577c05f84ec7..e1bf9b4d707ac 100644
--- a/include/linux/dcache.h
+++ b/include/linux/dcache.h
@@ -184,7 +184,7 @@ struct dentry_operations {
  * Locking rules for dentry_operations callbacks are to be found in
  * Documentation/filesystems/locking.rst. Keep it updated!
  *
- * FUrther descriptions are found in Documentation/filesystems/vfs.rst.
+ * Further descriptions are found in Documentation/filesystems/vfs.rst.
  * Keep it updated too!
  */
 
@@ -245,7 +245,7 @@ enum dentry_flags {
 extern seqlock_t rename_lock;
 
 /*
- * These are the low-level FS interfaces to the dcache..
+ * These are the low-level FS interfaces to the dcache.
  */
 extern void d_instantiate(struct dentry *, struct inode *);
 extern void d_instantiate_new(struct dentry *, struct inode *);
diff --git a/include/linux/delay.h b/include/linux/delay.h
index 46412c00033a9..68b2a69dd24d6 100644
--- a/include/linux/delay.h
+++ b/include/linux/delay.h
@@ -110,7 +110,7 @@ static const unsigned int max_slack_shift = 2;
  * fsleep - flexible sleep which autoselects the best mechanism
  * @usecs:	requested sleep duration in microseconds
  *
- * flseep() selects the best mechanism that will provide maximum 25% slack
+ * fsleep() selects the best mechanism that will provide maximum 25% slack
  * to the requested sleep duration. Therefore it uses:
  *
  * * udelay() loop for sleep durations <= 10 microseconds to avoid hrtimer
diff --git a/include/linux/device.h b/include/linux/device.h
index 9c8fde6a3d866..d54c86d777645 100644
--- a/include/linux/device.h
+++ b/include/linux/device.h
@@ -512,10 +512,40 @@ struct device_physical_location {
  *
  * @DEV_FLAG_READY_TO_PROBE: If set then device_add() has finished enough
  *		initialization that probe could be called.
+ * @DEV_FLAG_CAN_MATCH: The device has matched with a driver at least once or it
+ *		is in a bus (like AMBA) which can't check for matching drivers
+ *		until other devices probe successfully.
+ * @DEV_FLAG_DMA_IOMMU: Device is using default IOMMU implementation for DMA and
+ *		doesn't rely on dma_ops structure.
+ * @DEV_FLAG_DMA_SKIP_SYNC: DMA sync operations can be skipped for coherent
+ *		buffers.
+ * @DEV_FLAG_DMA_OPS_BYPASS: If set then the dma_ops are bypassed for the
+ *		streaming DMA operations (->map_* / ->unmap_* / ->sync_*), and
+ *		optional (if the coherent mask is large enough) also for dma
+ *		allocations. This flag is managed by the dma ops instance from
+ *		->dma_supported.
+ * @DEV_FLAG_STATE_SYNCED: The hardware state of this device has been synced to
+ *		match the software state of this device by calling the
+ *		driver/bus sync_state() callback.
+ * @DEV_FLAG_DMA_COHERENT: This particular device is dma coherent, even if the
+ *		architecture supports non-coherent devices.
+ * @DEV_FLAG_OF_NODE_REUSED: Set if the device-tree node is shared with an
+ *		ancestor device.
+ * @DEV_FLAG_OFFLINE_DISABLED: If set, the device is permanently online.
+ * @DEV_FLAG_OFFLINE: Set after successful invocation of bus type's .offline().
  * @DEV_FLAG_COUNT: Number of defined struct_device_flags.
  */
 enum struct_device_flags {
 	DEV_FLAG_READY_TO_PROBE = 0,
+	DEV_FLAG_CAN_MATCH = 1,
+	DEV_FLAG_DMA_IOMMU = 2,
+	DEV_FLAG_DMA_SKIP_SYNC = 3,
+	DEV_FLAG_DMA_OPS_BYPASS = 4,
+	DEV_FLAG_STATE_SYNCED = 5,
+	DEV_FLAG_DMA_COHERENT = 6,
+	DEV_FLAG_OF_NODE_REUSED = 7,
+	DEV_FLAG_OFFLINE_DISABLED = 8,
+	DEV_FLAG_OFFLINE = 9,
 
 	DEV_FLAG_COUNT
 };
@@ -594,27 +624,6 @@ enum struct_device_flags {
  * @removable:  Whether the device can be removed from the system. This
  *              should be set by the subsystem / bus driver that discovered
  *              the device.
- *
- * @offline_disabled: If set, the device is permanently online.
- * @offline:	Set after successful invocation of bus type's .offline().
- * @of_node_reused: Set if the device-tree node is shared with an ancestor
- *              device.
- * @state_synced: The hardware state of this device has been synced to match
- *		  the software state of this device by calling the driver/bus
- *		  sync_state() callback.
- * @can_match:	The device has matched with a driver at least once or it is in
- *		a bus (like AMBA) which can't check for matching drivers until
- *		other devices probe successfully.
- * @dma_coherent: this particular device is dma coherent, even if the
- *		architecture supports non-coherent devices.
- * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the
- *		streaming DMA operations (->map_* / ->unmap_* / ->sync_*),
- *		and optionall (if the coherent mask is large enough) also
- *		for dma allocations.  This flag is managed by the dma ops
- *		instance from ->dma_supported.
- * @dma_skip_sync: DMA sync operations can be skipped for coherent buffers.
- * @dma_iommu: Device is using default IOMMU implementation for DMA and
- *		doesn't rely on dma_ops structure.
  * @flags:	DEV_FLAG_XXX flags. Use atomic bitfield operations to modify.
  *
  * At the lowest level, every device in a Linux system is represented by an
@@ -719,26 +728,6 @@ struct device {
 
 	enum device_removable	removable;
 
-	bool			offline_disabled:1;
-	bool			offline:1;
-	bool			of_node_reused:1;
-	bool			state_synced:1;
-	bool			can_match:1;
-#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \
-    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \
-    defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL)
-	bool			dma_coherent:1;
-#endif
-#ifdef CONFIG_DMA_OPS_BYPASS
-	bool			dma_ops_bypass : 1;
-#endif
-#ifdef CONFIG_DMA_NEED_SYNC
-	bool			dma_skip_sync:1;
-#endif
-#ifdef CONFIG_IOMMU_DMA
-	bool			dma_iommu:1;
-#endif
-
 	DECLARE_BITMAP(flags, DEV_FLAG_COUNT);
 };
 
@@ -765,6 +754,15 @@ static inline bool dev_test_and_set_##accessor_name(struct device *dev) \
 }
 
 __create_dev_flag_accessors(ready_to_probe, DEV_FLAG_READY_TO_PROBE);
+__create_dev_flag_accessors(can_match, DEV_FLAG_CAN_MATCH);
+__create_dev_flag_accessors(dma_iommu, DEV_FLAG_DMA_IOMMU);
+__create_dev_flag_accessors(dma_skip_sync, DEV_FLAG_DMA_SKIP_SYNC);
+__create_dev_flag_accessors(dma_ops_bypass, DEV_FLAG_DMA_OPS_BYPASS);
+__create_dev_flag_accessors(state_synced, DEV_FLAG_STATE_SYNCED);
+__create_dev_flag_accessors(dma_coherent, DEV_FLAG_DMA_COHERENT);
+__create_dev_flag_accessors(of_node_reused, DEV_FLAG_OF_NODE_REUSED);
+__create_dev_flag_accessors(offline_disabled, DEV_FLAG_OFFLINE_DISABLED);
+__create_dev_flag_accessors(offline, DEV_FLAG_OFFLINE);
 
 #undef __create_dev_flag_accessors
 
@@ -1063,17 +1061,6 @@ static inline void device_lock_assert(struct device *dev)
 	lockdep_assert_held(&dev->mutex);
 }
 
-static inline bool dev_has_sync_state(struct device *dev)
-{
-	if (!dev)
-		return false;
-	if (dev->driver && dev->driver->sync_state)
-		return true;
-	if (dev->bus && dev->bus->sync_state)
-		return true;
-	return false;
-}
-
 static inline int dev_set_drv_sync_state(struct device *dev,
 					 void (*fn)(struct device *dev))
 {
diff --git a/include/linux/device/class.h b/include/linux/device/class.h
index 78ab8d2b3e307..9db1d61ba743e 100644
--- a/include/linux/device/class.h
+++ b/include/linux/device/class.h
@@ -34,7 +34,7 @@ struct fwnode_handle;
  * @class_release: Called to release this class.
  * @dev_release: Called to release the device.
  * @shutdown_pre: Called at shut-down time before driver shutdown.
- * @ns_type:	Callbacks so sysfs can detemine namespaces.
+ * @ns_type:	Callbacks so sysfs can determine namespaces.
  * @namespace:	Namespace of the device belongs to this class.
  * @get_ownership: Allows class to specify uid/gid of the sysfs directories
  *		for the devices belonging to the class. Usually tied to
diff --git a/include/linux/dio.h b/include/linux/dio.h
index 464331c4c4a77..894e50b917925 100644
--- a/include/linux/dio.h
+++ b/include/linux/dio.h
@@ -112,9 +112,7 @@ struct dio_driver {
 #define DIOII_END               0x20000000      /* end of DIO-II space */
 #define DIOII_DEVSIZE           0x00400000      /* size of a DIO-II device */
 
-/* Highest valid select code. If we add DIO-II support this should become
- * 256 for everything except HP320, which only has DIO.
- */
+/* highest valid select code */
 #define DIO_SCMAX (hp300_model == HP_320 ? 32 : 256)
 #define DIOII_SCBASE 132 /* lowest DIO-II select code */
 #define DIO_SCINHOLE(scode) (((scode) >= 32) && ((scode) < DIOII_SCBASE))
diff --git a/include/linux/dma-fence-array.h b/include/linux/dma-fence-array.h
index 370b3d2bba371..1b1d87579c382 100644
--- a/include/linux/dma-fence-array.h
+++ b/include/linux/dma-fence-array.h
@@ -81,13 +81,11 @@ to_dma_fence_array(struct dma_fence *fence)
 struct dma_fence_array *dma_fence_array_alloc(int num_fences);
 void dma_fence_array_init(struct dma_fence_array *array,
 			  int num_fences, struct dma_fence **fences,
-			  u64 context, unsigned seqno,
-			  bool signal_on_any);
+			  u64 context, unsigned seqno);
 
 struct dma_fence_array *dma_fence_array_create(int num_fences,
 					       struct dma_fence **fences,
-					       u64 context, unsigned seqno,
-					       bool signal_on_any);
+					       u64 context, unsigned seqno);
 
 bool dma_fence_match_context(struct dma_fence *fence, u64 context);
 
diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h
index 6a1832a73cad8..bcb5b5428aea9 100644
--- a/include/linux/dma-map-ops.h
+++ b/include/linux/dma-map-ops.h
@@ -225,7 +225,7 @@ int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start,
 extern bool dma_default_coherent;
 static inline bool dev_is_dma_coherent(struct device *dev)
 {
-	return dev->dma_coherent;
+	return dev_dma_coherent(dev);
 }
 #else
 #define dma_default_coherent true
@@ -240,8 +240,8 @@ static inline void dma_reset_need_sync(struct device *dev)
 {
 #ifdef CONFIG_DMA_NEED_SYNC
 	/* Reset it only once so that the function can be called on hotpath */
-	if (unlikely(dev->dma_skip_sync))
-		dev->dma_skip_sync = false;
+	if (unlikely(dev_dma_skip_sync(dev)))
+		dev_clear_dma_skip_sync(dev);
 #endif
 }
 
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index db8ab24a54f4a..cc0823a99cfdb 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -429,7 +429,7 @@ bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr);
 static inline bool dma_dev_need_sync(const struct device *dev)
 {
 	/* Always call DMA sync operations when debugging is enabled */
-	return !dev->dma_skip_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG);
+	return !dev_dma_skip_sync(dev) || IS_ENABLED(CONFIG_DMA_API_DEBUG);
 }
 
 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
diff --git a/include/linux/dpll.h b/include/linux/dpll.h
index f8037f1ab20b6..82dfadd9f593b 100644
--- a/include/linux/dpll.h
+++ b/include/linux/dpll.h
@@ -60,7 +60,20 @@ struct dpll_device_ops {
 				struct netlink_ext_ack *extack);
 };
 
+enum dpll_ffo_type {
+	DPLL_FFO_PORT_RXTX_RATE,
+	DPLL_FFO_PIN_DEVICE,
+
+	__DPLL_FFO_TYPE_MAX,
+};
+
+struct dpll_ffo_param {
+	enum dpll_ffo_type type;
+	s64 ffo;
+};
+
 struct dpll_pin_ops {
+	unsigned long supported_ffo;
 	int (*frequency_set)(const struct dpll_pin *pin, void *pin_priv,
 			     const struct dpll_device *dpll, void *dpll_priv,
 			     const u64 frequency,
@@ -85,6 +98,12 @@ struct dpll_pin_ops {
 				 const struct dpll_device *dpll,
 				 void *dpll_priv, enum dpll_pin_state *state,
 				 struct netlink_ext_ack *extack);
+	int (*operstate_on_dpll_get)(const struct dpll_pin *pin,
+				     void *pin_priv,
+				     const struct dpll_device *dpll,
+				     void *dpll_priv,
+				     enum dpll_pin_operstate *operstate,
+				     struct netlink_ext_ack *extack);
 	int (*state_on_pin_set)(const struct dpll_pin *pin, void *pin_priv,
 				const struct dpll_pin *parent_pin,
 				void *parent_pin_priv,
@@ -115,7 +134,8 @@ struct dpll_pin_ops {
 				struct netlink_ext_ack *extack);
 	int (*ffo_get)(const struct dpll_pin *pin, void *pin_priv,
 		       const struct dpll_device *dpll, void *dpll_priv,
-		       s64 *ffo, struct netlink_ext_ack *extack);
+		       struct dpll_ffo_param *ffo,
+		       struct netlink_ext_ack *extack);
 	int (*measured_freq_get)(const struct dpll_pin *pin, void *pin_priv,
 				 const struct dpll_device *dpll,
 				 void *dpll_priv, u64 *measured_freq,
diff --git a/include/linux/drbd.h b/include/linux/drbd.h
deleted file mode 100644
index 5468a2399d483..0000000000000
--- a/include/linux/drbd.h
+++ /dev/null
@@ -1,392 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
-  drbd.h
-  Kernel module for 2.6.x Kernels
-
-  This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
-
-  Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
-  Copyright (C) 2001-2008, Philipp Reisner <philipp.reisner@linbit.com>.
-  Copyright (C) 2001-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
-
-
-*/
-#ifndef DRBD_H
-#define DRBD_H
-#include <asm/types.h>
-
-#ifdef __KERNEL__
-#include <linux/types.h>
-#include <asm/byteorder.h>
-#else
-#include <sys/types.h>
-#include <sys/wait.h>
-#include <limits.h>
-
-/* Although the Linux source code makes a difference between
-   generic endianness and the bitfields' endianness, there is no
-   architecture as of Linux-2.6.24-rc4 where the bitfields' endianness
-   does not match the generic endianness. */
-
-#if __BYTE_ORDER == __LITTLE_ENDIAN
-#define __LITTLE_ENDIAN_BITFIELD
-#elif __BYTE_ORDER == __BIG_ENDIAN
-#define __BIG_ENDIAN_BITFIELD
-#else
-# error "sorry, weird endianness on this box"
-#endif
-
-#endif
-
-enum drbd_io_error_p {
-	EP_PASS_ON, /* FIXME should the better be named "Ignore"? */
-	EP_CALL_HELPER,
-	EP_DETACH
-};
-
-enum drbd_fencing_p {
-	FP_NOT_AVAIL = -1, /* Not a policy */
-	FP_DONT_CARE = 0,
-	FP_RESOURCE,
-	FP_STONITH
-};
-
-enum drbd_disconnect_p {
-	DP_RECONNECT,
-	DP_DROP_NET_CONF,
-	DP_FREEZE_IO
-};
-
-enum drbd_after_sb_p {
-	ASB_DISCONNECT,
-	ASB_DISCARD_YOUNGER_PRI,
-	ASB_DISCARD_OLDER_PRI,
-	ASB_DISCARD_ZERO_CHG,
-	ASB_DISCARD_LEAST_CHG,
-	ASB_DISCARD_LOCAL,
-	ASB_DISCARD_REMOTE,
-	ASB_CONSENSUS,
-	ASB_DISCARD_SECONDARY,
-	ASB_CALL_HELPER,
-	ASB_VIOLENTLY
-};
-
-enum drbd_on_no_data {
-	OND_IO_ERROR,
-	OND_SUSPEND_IO
-};
-
-enum drbd_on_congestion {
-	OC_BLOCK,
-	OC_PULL_AHEAD,
-	OC_DISCONNECT,
-};
-
-enum drbd_read_balancing {
-	RB_PREFER_LOCAL,
-	RB_PREFER_REMOTE,
-	RB_ROUND_ROBIN,
-	RB_LEAST_PENDING,
-	RB_CONGESTED_REMOTE,
-	RB_32K_STRIPING,
-	RB_64K_STRIPING,
-	RB_128K_STRIPING,
-	RB_256K_STRIPING,
-	RB_512K_STRIPING,
-	RB_1M_STRIPING,
-};
-
-/* KEEP the order, do not delete or insert. Only append. */
-enum drbd_ret_code {
-	ERR_CODE_BASE		= 100,
-	NO_ERROR		= 101,
-	ERR_LOCAL_ADDR		= 102,
-	ERR_PEER_ADDR		= 103,
-	ERR_OPEN_DISK		= 104,
-	ERR_OPEN_MD_DISK	= 105,
-	ERR_DISK_NOT_BDEV	= 107,
-	ERR_MD_NOT_BDEV		= 108,
-	ERR_DISK_TOO_SMALL	= 111,
-	ERR_MD_DISK_TOO_SMALL	= 112,
-	ERR_BDCLAIM_DISK	= 114,
-	ERR_BDCLAIM_MD_DISK	= 115,
-	ERR_MD_IDX_INVALID	= 116,
-	ERR_IO_MD_DISK		= 118,
-	ERR_MD_INVALID          = 119,
-	ERR_AUTH_ALG		= 120,
-	ERR_AUTH_ALG_ND		= 121,
-	ERR_NOMEM		= 122,
-	ERR_DISCARD_IMPOSSIBLE	= 123,
-	ERR_DISK_CONFIGURED	= 124,
-	ERR_NET_CONFIGURED	= 125,
-	ERR_MANDATORY_TAG	= 126,
-	ERR_MINOR_INVALID	= 127,
-	ERR_INTR		= 129, /* EINTR */
-	ERR_RESIZE_RESYNC	= 130,
-	ERR_NO_PRIMARY		= 131,
-	ERR_RESYNC_AFTER	= 132,
-	ERR_RESYNC_AFTER_CYCLE	= 133,
-	ERR_PAUSE_IS_SET	= 134,
-	ERR_PAUSE_IS_CLEAR	= 135,
-	ERR_PACKET_NR		= 137,
-	ERR_NO_DISK		= 138,
-	ERR_NOT_PROTO_C		= 139,
-	ERR_NOMEM_BITMAP	= 140,
-	ERR_INTEGRITY_ALG	= 141, /* DRBD 8.2 only */
-	ERR_INTEGRITY_ALG_ND	= 142, /* DRBD 8.2 only */
-	ERR_CPU_MASK_PARSE	= 143, /* DRBD 8.2 only */
-	ERR_CSUMS_ALG		= 144, /* DRBD 8.2 only */
-	ERR_CSUMS_ALG_ND	= 145, /* DRBD 8.2 only */
-	ERR_VERIFY_ALG		= 146, /* DRBD 8.2 only */
-	ERR_VERIFY_ALG_ND	= 147, /* DRBD 8.2 only */
-	ERR_CSUMS_RESYNC_RUNNING= 148, /* DRBD 8.2 only */
-	ERR_VERIFY_RUNNING	= 149, /* DRBD 8.2 only */
-	ERR_DATA_NOT_CURRENT	= 150,
-	ERR_CONNECTED		= 151, /* DRBD 8.3 only */
-	ERR_PERM		= 152,
-	ERR_NEED_APV_93		= 153,
-	ERR_STONITH_AND_PROT_A  = 154,
-	ERR_CONG_NOT_PROTO_A	= 155,
-	ERR_PIC_AFTER_DEP	= 156,
-	ERR_PIC_PEER_DEP	= 157,
-	ERR_RES_NOT_KNOWN	= 158,
-	ERR_RES_IN_USE		= 159,
-	ERR_MINOR_CONFIGURED    = 160,
-	ERR_MINOR_OR_VOLUME_EXISTS = 161,
-	ERR_INVALID_REQUEST	= 162,
-	ERR_NEED_APV_100	= 163,
-	ERR_NEED_ALLOW_TWO_PRI  = 164,
-	ERR_MD_UNCLEAN          = 165,
-	ERR_MD_LAYOUT_CONNECTED = 166,
-	ERR_MD_LAYOUT_TOO_BIG   = 167,
-	ERR_MD_LAYOUT_TOO_SMALL = 168,
-	ERR_MD_LAYOUT_NO_FIT    = 169,
-	ERR_IMPLICIT_SHRINK     = 170,
-	/* insert new ones above this line */
-	AFTER_LAST_ERR_CODE
-};
-
-#define DRBD_PROT_A   1
-#define DRBD_PROT_B   2
-#define DRBD_PROT_C   3
-
-enum drbd_role {
-	R_UNKNOWN = 0,
-	R_PRIMARY = 1,     /* role */
-	R_SECONDARY = 2,   /* role */
-	R_MASK = 3,
-};
-
-/* The order of these constants is important.
- * The lower ones (<C_WF_REPORT_PARAMS) indicate
- * that there is no socket!
- * >=C_WF_REPORT_PARAMS ==> There is a socket
- */
-enum drbd_conns {
-	C_STANDALONE,
-	C_DISCONNECTING,  /* Temporal state on the way to StandAlone. */
-	C_UNCONNECTED,    /* >= C_UNCONNECTED -> inc_net() succeeds */
-
-	/* These temporal states are all used on the way
-	 * from >= C_CONNECTED to Unconnected.
-	 * The 'disconnect reason' states
-	 * I do not allow to change between them. */
-	C_TIMEOUT,
-	C_BROKEN_PIPE,
-	C_NETWORK_FAILURE,
-	C_PROTOCOL_ERROR,
-	C_TEAR_DOWN,
-
-	C_WF_CONNECTION,
-	C_WF_REPORT_PARAMS, /* we have a socket */
-	C_CONNECTED,      /* we have introduced each other */
-	C_STARTING_SYNC_S,  /* starting full sync by admin request. */
-	C_STARTING_SYNC_T,  /* starting full sync by admin request. */
-	C_WF_BITMAP_S,
-	C_WF_BITMAP_T,
-	C_WF_SYNC_UUID,
-
-	/* All SyncStates are tested with this comparison
-	 * xx >= C_SYNC_SOURCE && xx <= C_PAUSED_SYNC_T */
-	C_SYNC_SOURCE,
-	C_SYNC_TARGET,
-	C_VERIFY_S,
-	C_VERIFY_T,
-	C_PAUSED_SYNC_S,
-	C_PAUSED_SYNC_T,
-
-	C_AHEAD,
-	C_BEHIND,
-
-	C_MASK = 31
-};
-
-enum drbd_disk_state {
-	D_DISKLESS,
-	D_ATTACHING,      /* In the process of reading the meta-data */
-	D_FAILED,         /* Becomes D_DISKLESS as soon as we told it the peer */
-			  /* when >= D_FAILED it is legal to access mdev->ldev */
-	D_NEGOTIATING,    /* Late attaching state, we need to talk to the peer */
-	D_INCONSISTENT,
-	D_OUTDATED,
-	D_UNKNOWN,       /* Only used for the peer, never for myself */
-	D_CONSISTENT,     /* Might be D_OUTDATED, might be D_UP_TO_DATE ... */
-	D_UP_TO_DATE,       /* Only this disk state allows applications' IO ! */
-	D_MASK = 15
-};
-
-union drbd_state {
-/* According to gcc's docs is the ...
- * The order of allocation of bit-fields within a unit (C90 6.5.2.1, C99 6.7.2.1).
- * Determined by ABI.
- * pointed out by Maxim Uvarov q<muvarov@ru.mvista.com>
- * even though we transmit as "cpu_to_be32(state)",
- * the offsets of the bitfields still need to be swapped
- * on different endianness.
- */
-	struct {
-#if defined(__LITTLE_ENDIAN_BITFIELD)
-		unsigned role:2 ;   /* 3/4	 primary/secondary/unknown */
-		unsigned peer:2 ;   /* 3/4	 primary/secondary/unknown */
-		unsigned conn:5 ;   /* 17/32	 cstates */
-		unsigned disk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
-		unsigned pdsk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
-		unsigned susp:1 ;   /* 2/2	 IO suspended no/yes (by user) */
-		unsigned aftr_isp:1 ; /* isp .. imposed sync pause */
-		unsigned peer_isp:1 ;
-		unsigned user_isp:1 ;
-		unsigned susp_nod:1 ; /* IO suspended because no data */
-		unsigned susp_fen:1 ; /* IO suspended because fence peer handler runs*/
-		unsigned _pad:9;   /* 0	 unused */
-#elif defined(__BIG_ENDIAN_BITFIELD)
-		unsigned _pad:9;
-		unsigned susp_fen:1 ;
-		unsigned susp_nod:1 ;
-		unsigned user_isp:1 ;
-		unsigned peer_isp:1 ;
-		unsigned aftr_isp:1 ; /* isp .. imposed sync pause */
-		unsigned susp:1 ;   /* 2/2	 IO suspended  no/yes */
-		unsigned pdsk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
-		unsigned disk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
-		unsigned conn:5 ;   /* 17/32	 cstates */
-		unsigned peer:2 ;   /* 3/4	 primary/secondary/unknown */
-		unsigned role:2 ;   /* 3/4	 primary/secondary/unknown */
-#else
-# error "this endianness is not supported"
-#endif
-	};
-	unsigned int i;
-};
-
-enum drbd_state_rv {
-	SS_CW_NO_NEED = 4,
-	SS_CW_SUCCESS = 3,
-	SS_NOTHING_TO_DO = 2,
-	SS_SUCCESS = 1,
-	SS_UNKNOWN_ERROR = 0, /* Used to sleep longer in _drbd_request_state */
-	SS_TWO_PRIMARIES = -1,
-	SS_NO_UP_TO_DATE_DISK = -2,
-	SS_NO_LOCAL_DISK = -4,
-	SS_NO_REMOTE_DISK = -5,
-	SS_CONNECTED_OUTDATES = -6,
-	SS_PRIMARY_NOP = -7,
-	SS_RESYNC_RUNNING = -8,
-	SS_ALREADY_STANDALONE = -9,
-	SS_CW_FAILED_BY_PEER = -10,
-	SS_IS_DISKLESS = -11,
-	SS_DEVICE_IN_USE = -12,
-	SS_NO_NET_CONFIG = -13,
-	SS_NO_VERIFY_ALG = -14,       /* drbd-8.2 only */
-	SS_NEED_CONNECTION = -15,    /* drbd-8.2 only */
-	SS_LOWER_THAN_OUTDATED = -16,
-	SS_NOT_SUPPORTED = -17,      /* drbd-8.2 only */
-	SS_IN_TRANSIENT_STATE = -18,  /* Retry after the next state change */
-	SS_CONCURRENT_ST_CHG = -19,   /* Concurrent cluster side state change! */
-	SS_O_VOL_PEER_PRI = -20,
-	SS_OUTDATE_WO_CONN = -21,
-	SS_AFTER_LAST_ERROR = -22,    /* Keep this at bottom */
-};
-
-#define SHARED_SECRET_MAX 64
-
-#define MDF_CONSISTENT		(1 << 0)
-#define MDF_PRIMARY_IND		(1 << 1)
-#define MDF_CONNECTED_IND	(1 << 2)
-#define MDF_FULL_SYNC		(1 << 3)
-#define MDF_WAS_UP_TO_DATE	(1 << 4)
-#define MDF_PEER_OUT_DATED	(1 << 5)
-#define MDF_CRASHED_PRIMARY	(1 << 6)
-#define MDF_AL_CLEAN		(1 << 7)
-#define MDF_AL_DISABLED		(1 << 8)
-
-#define MAX_PEERS 32
-
-enum drbd_uuid_index {
-	UI_CURRENT,
-	UI_BITMAP,
-	UI_HISTORY_START,
-	UI_HISTORY_END,
-	UI_SIZE,      /* nl-packet: number of dirty bits */
-	UI_FLAGS,     /* nl-packet: flags */
-	UI_EXTENDED_SIZE   /* Everything. */
-};
-
-#define HISTORY_UUIDS MAX_PEERS
-
-enum drbd_timeout_flag {
-	UT_DEFAULT      = 0,
-	UT_DEGRADED     = 1,
-	UT_PEER_OUTDATED = 2,
-};
-
-enum drbd_notification_type {
-	NOTIFY_EXISTS,
-	NOTIFY_CREATE,
-	NOTIFY_CHANGE,
-	NOTIFY_DESTROY,
-	NOTIFY_CALL,
-	NOTIFY_RESPONSE,
-
-	NOTIFY_CONTINUES = 0x8000,
-	NOTIFY_FLAGS = NOTIFY_CONTINUES,
-};
-
-enum drbd_peer_state {
-	P_INCONSISTENT = 3,
-	P_OUTDATED = 4,
-	P_DOWN = 5,
-	P_PRIMARY = 6,
-	P_FENCING = 7,
-};
-
-#define UUID_JUST_CREATED ((__u64)4)
-
-enum write_ordering_e {
-	WO_NONE,
-	WO_DRAIN_IO,
-	WO_BDEV_FLUSH,
-	WO_BIO_BARRIER
-};
-
-/* magic numbers used in meta data and network packets */
-#define DRBD_MAGIC 0x83740267
-#define DRBD_MAGIC_BIG 0x835a
-#define DRBD_MAGIC_100 0x8620ec20
-
-#define DRBD_MD_MAGIC_07   (DRBD_MAGIC+3)
-#define DRBD_MD_MAGIC_08   (DRBD_MAGIC+4)
-#define DRBD_MD_MAGIC_84_UNCLEAN	(DRBD_MAGIC+5)
-
-
-/* how I came up with this magic?
- * base64 decode "actlog==" ;) */
-#define DRBD_AL_MAGIC 0x69cb65a2
-
-/* these are of type "int" */
-#define DRBD_MD_INDEX_INTERNAL -1
-#define DRBD_MD_INDEX_FLEX_EXT -2
-#define DRBD_MD_INDEX_FLEX_INT -3
-
-#define DRBD_CPU_MASK_SIZE 32
-
-#endif
diff --git a/include/linux/drbd_config.h b/include/linux/drbd_config.h
deleted file mode 100644
index d215365c6bb1b..0000000000000
--- a/include/linux/drbd_config.h
+++ /dev/null
@@ -1,16 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * drbd_config.h
- * DRBD's compile time configuration.
- */
-
-#ifndef DRBD_CONFIG_H
-#define DRBD_CONFIG_H
-
-extern const char *drbd_buildtag(void);
-
-#define REL_VERSION "8.4.11"
-#define PRO_VERSION_MIN 86
-#define PRO_VERSION_MAX 101
-
-#endif
diff --git a/include/linux/drbd_genl.h b/include/linux/drbd_genl.h
deleted file mode 100644
index f53c534aba0cf..0000000000000
--- a/include/linux/drbd_genl.h
+++ /dev/null
@@ -1,536 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * General overview:
- * full generic netlink message:
- * |nlmsghdr|genlmsghdr|<payload>
- *
- * payload:
- * |optional fixed size family header|<sequence of netlink attributes>
- *
- * sequence of netlink attributes:
- * I chose to have all "top level" attributes NLA_NESTED,
- * corresponding to some real struct.
- * So we have a sequence of |tla, len|<nested nla sequence>
- *
- * nested nla sequence:
- * may be empty, or contain a sequence of netlink attributes
- * representing the struct fields.
- *
- * The tag number of any field (regardless of containing struct)
- * will be available as T_ ## field_name,
- * so you cannot have the same field name in two differnt structs.
- *
- * The tag numbers themselves are per struct, though,
- * so should always begin at 1 (not 0, that is the special "NLA_UNSPEC" type,
- * which we won't use here).
- * The tag numbers are used as index in the respective nla_policy array.
- *
- * GENL_struct(tag_name, tag_number, struct name, struct fields) - struct and policy
- *	genl_magic_struct.h
- *		generates the struct declaration,
- *		generates an entry in the tla enum,
- *	genl_magic_func.h
- *		generates an entry in the static tla policy
- *		with .type = NLA_NESTED
- *		generates the static <struct_name>_nl_policy definition,
- *		and static conversion functions
- *
- *	genl_magic_func.h
- *
- * GENL_mc_group(group)
- *	genl_magic_struct.h
- *		does nothing
- *	genl_magic_func.h
- *		defines and registers the mcast group,
- *		and provides a send helper
- *
- * GENL_notification(op_name, op_num, mcast_group, tla list)
- *	These are notifications to userspace.
- *
- *	genl_magic_struct.h
- *		generates an entry in the genl_ops enum,
- *	genl_magic_func.h
- *		does nothing
- *
- *	mcast group: the name of the mcast group this notification should be
- *	expected on
- *	tla list: the list of expected top level attributes,
- *	for documentation and sanity checking.
- *
- * GENL_op(op_name, op_num, flags and handler, tla list) - "genl operations"
- *	These are requests from userspace.
- *
- *	_op and _notification share the same "number space",
- *	op_nr will be assigned to "genlmsghdr->cmd"
- *
- *	genl_magic_struct.h
- *		generates an entry in the genl_ops enum,
- *	genl_magic_func.h
- *		generates an entry in the static genl_ops array,
- *		and static register/unregister functions to
- *		genl_register_family().
- *
- *	flags and handler:
- *		GENL_op_init( .doit = x, .dumpit = y, .flags = something)
- *		GENL_doit(x) => .dumpit = NULL, .flags = GENL_ADMIN_PERM
- *	tla list: the list of expected top level attributes,
- *	for documentation and sanity checking.
- */
-
-/*
- * STRUCTS
- */
-
-/* this is sent kernel -> userland on various error conditions, and contains
- * informational textual info, which is supposedly human readable.
- * The computer relevant return code is in the drbd_genlmsghdr.
- */
-GENL_struct(DRBD_NLA_CFG_REPLY, 1, drbd_cfg_reply,
-		/* "arbitrary" size strings, nla_policy.len = 0 */
-	__str_field(1, 0,	info_text, 0)
-)
-
-/* Configuration requests typically need a context to operate on.
- * Possible keys are device minor (fits in the drbd_genlmsghdr),
- * the replication link (aka connection) name,
- * and/or the replication group (aka resource) name,
- * and the volume id within the resource. */
-GENL_struct(DRBD_NLA_CFG_CONTEXT, 2, drbd_cfg_context,
-	__u32_field(1, 0,	ctx_volume)
-	__str_field(2, 0,	ctx_resource_name, 128)
-	__bin_field(3, 0,	ctx_my_addr, 128)
-	__bin_field(4, 0,	ctx_peer_addr, 128)
-)
-
-GENL_struct(DRBD_NLA_DISK_CONF, 3, disk_conf,
-	__str_field(1, DRBD_F_REQUIRED | DRBD_F_INVARIANT,	backing_dev,	128)
-	__str_field(2, DRBD_F_REQUIRED | DRBD_F_INVARIANT,	meta_dev,	128)
-	__s32_field(3, DRBD_F_REQUIRED | DRBD_F_INVARIANT,	meta_dev_idx)
-
-	/* use the resize command to try and change the disk_size */
-	__u64_field(4, DRBD_F_INVARIANT,	disk_size)
-	/* we could change the max_bio_bvecs,
-	 * but it won't propagate through the stack */
-	__u32_field(5, DRBD_F_INVARIANT,	max_bio_bvecs)
-
-	__u32_field_def(6, 0,	on_io_error, DRBD_ON_IO_ERROR_DEF)
-	__u32_field_def(7, 0,	fencing, DRBD_FENCING_DEF)
-
-	__u32_field_def(8,	0,	resync_rate, DRBD_RESYNC_RATE_DEF)
-	__s32_field_def(9,	0,	resync_after, DRBD_MINOR_NUMBER_DEF)
-	__u32_field_def(10,	0,	al_extents, DRBD_AL_EXTENTS_DEF)
-	__u32_field_def(11,	0,	c_plan_ahead, DRBD_C_PLAN_AHEAD_DEF)
-	__u32_field_def(12,	0,	c_delay_target, DRBD_C_DELAY_TARGET_DEF)
-	__u32_field_def(13,	0,	c_fill_target, DRBD_C_FILL_TARGET_DEF)
-	__u32_field_def(14,	0,	c_max_rate, DRBD_C_MAX_RATE_DEF)
-	__u32_field_def(15,	0,	c_min_rate, DRBD_C_MIN_RATE_DEF)
-	__u32_field_def(20,     0, disk_timeout, DRBD_DISK_TIMEOUT_DEF)
-	__u32_field_def(21,     0 /* OPTIONAL */,       read_balancing, DRBD_READ_BALANCING_DEF)
-	__u32_field_def(25,     0 /* OPTIONAL */,       rs_discard_granularity, DRBD_RS_DISCARD_GRANULARITY_DEF)
-
-	__flg_field_def(16, 0,	disk_barrier, DRBD_DISK_BARRIER_DEF)
-	__flg_field_def(17, 0,	disk_flushes, DRBD_DISK_FLUSHES_DEF)
-	__flg_field_def(18, 0,	disk_drain, DRBD_DISK_DRAIN_DEF)
-	__flg_field_def(19, 0,	md_flushes, DRBD_MD_FLUSHES_DEF)
-	__flg_field_def(23,     0 /* OPTIONAL */,	al_updates, DRBD_AL_UPDATES_DEF)
-	__flg_field_def(24,     0 /* OPTIONAL */,	discard_zeroes_if_aligned, DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF)
-	__flg_field_def(26,     0 /* OPTIONAL */,	disable_write_same, DRBD_DISABLE_WRITE_SAME_DEF)
-)
-
-GENL_struct(DRBD_NLA_RESOURCE_OPTS, 4, res_opts,
-	__str_field_def(1,	0,	cpu_mask,       DRBD_CPU_MASK_SIZE)
-	__u32_field_def(2,	0,	on_no_data, DRBD_ON_NO_DATA_DEF)
-)
-
-GENL_struct(DRBD_NLA_NET_CONF, 5, net_conf,
-	__str_field_def(1,	DRBD_F_SENSITIVE,
-						shared_secret,	SHARED_SECRET_MAX)
-	__str_field_def(2,	0,	cram_hmac_alg,	SHARED_SECRET_MAX)
-	__str_field_def(3,	0,	integrity_alg,	SHARED_SECRET_MAX)
-	__str_field_def(4,	0,	verify_alg,     SHARED_SECRET_MAX)
-	__str_field_def(5,	0,	csums_alg,	SHARED_SECRET_MAX)
-	__u32_field_def(6,	0,	wire_protocol, DRBD_PROTOCOL_DEF)
-	__u32_field_def(7,	0,	connect_int, DRBD_CONNECT_INT_DEF)
-	__u32_field_def(8,	0,	timeout, DRBD_TIMEOUT_DEF)
-	__u32_field_def(9,	0,	ping_int, DRBD_PING_INT_DEF)
-	__u32_field_def(10,	0,	ping_timeo, DRBD_PING_TIMEO_DEF)
-	__u32_field_def(11,	0,	sndbuf_size, DRBD_SNDBUF_SIZE_DEF)
-	__u32_field_def(12,	0,	rcvbuf_size, DRBD_RCVBUF_SIZE_DEF)
-	__u32_field_def(13,	0,	ko_count, DRBD_KO_COUNT_DEF)
-	__u32_field_def(14,	0,	max_buffers, DRBD_MAX_BUFFERS_DEF)
-	__u32_field_def(15,	0,	max_epoch_size, DRBD_MAX_EPOCH_SIZE_DEF)
-	__u32_field_def(16,	0,	unplug_watermark, DRBD_UNPLUG_WATERMARK_DEF)
-	__u32_field_def(17,	0,	after_sb_0p, DRBD_AFTER_SB_0P_DEF)
-	__u32_field_def(18,	0,	after_sb_1p, DRBD_AFTER_SB_1P_DEF)
-	__u32_field_def(19,	0,	after_sb_2p, DRBD_AFTER_SB_2P_DEF)
-	__u32_field_def(20,	0,	rr_conflict, DRBD_RR_CONFLICT_DEF)
-	__u32_field_def(21,	0,	on_congestion, DRBD_ON_CONGESTION_DEF)
-	__u32_field_def(22,	0,	cong_fill, DRBD_CONG_FILL_DEF)
-	__u32_field_def(23,	0,	cong_extents, DRBD_CONG_EXTENTS_DEF)
-	__flg_field_def(24, 0,	two_primaries, DRBD_ALLOW_TWO_PRIMARIES_DEF)
-	__flg_field(25, DRBD_F_INVARIANT,	discard_my_data)
-	__flg_field_def(26, 0,	tcp_cork, DRBD_TCP_CORK_DEF)
-	__flg_field_def(27, 0,	always_asbp, DRBD_ALWAYS_ASBP_DEF)
-	__flg_field(28, DRBD_F_INVARIANT,	tentative)
-	__flg_field_def(29,	0,	use_rle, DRBD_USE_RLE_DEF)
-	/* 9: __u32_field_def(30,	0,	fencing_policy, DRBD_FENCING_DEF) */
-	/* 9: __str_field_def(31,     0, name, SHARED_SECRET_MAX) */
-	/* 9: __u32_field(32,         DRBD_F_REQUIRED | DRBD_F_INVARIANT,     peer_node_id) */
-	__flg_field_def(33, 0 /* OPTIONAL */,	csums_after_crash_only, DRBD_CSUMS_AFTER_CRASH_ONLY_DEF)
-	__u32_field_def(34, 0 /* OPTIONAL */, sock_check_timeo, DRBD_SOCKET_CHECK_TIMEO_DEF)
-)
-
-GENL_struct(DRBD_NLA_SET_ROLE_PARMS, 6, set_role_parms,
-	__flg_field(1, 0,	assume_uptodate)
-)
-
-GENL_struct(DRBD_NLA_RESIZE_PARMS, 7, resize_parms,
-	__u64_field(1, 0,	resize_size)
-	__flg_field(2, 0,	resize_force)
-	__flg_field(3, 0,	no_resync)
-	__u32_field_def(4, 0 /* OPTIONAL */, al_stripes, DRBD_AL_STRIPES_DEF)
-	__u32_field_def(5, 0 /* OPTIONAL */, al_stripe_size, DRBD_AL_STRIPE_SIZE_DEF)
-)
-
-GENL_struct(DRBD_NLA_STATE_INFO, 8, state_info,
-	/* the reason of the broadcast,
-	 * if this is an event triggered broadcast. */
-	__u32_field(1, 0,	sib_reason)
-	__u32_field(2, DRBD_F_REQUIRED,	current_state)
-	__u64_field(3, 0,	capacity)
-	__u64_field(4, 0,	ed_uuid)
-
-	/* These are for broadcast from after state change work.
-	 * prev_state and new_state are from the moment the state change took
-	 * place, new_state is not neccessarily the same as current_state,
-	 * there may have been more state changes since.  Which will be
-	 * broadcasted soon, in their respective after state change work.  */
-	__u32_field(5, 0,	prev_state)
-	__u32_field(6, 0,	new_state)
-
-	/* if we have a local disk: */
-	__bin_field(7, 0,	uuids, (UI_SIZE*sizeof(__u64)))
-	__u32_field(8, 0,	disk_flags)
-	__u64_field(9, 0,	bits_total)
-	__u64_field(10, 0,	bits_oos)
-	/* and in case resync or online verify is active */
-	__u64_field(11, 0,	bits_rs_total)
-	__u64_field(12, 0,	bits_rs_failed)
-
-	/* for pre and post notifications of helper execution */
-	__str_field(13, 0,	helper, 32)
-	__u32_field(14, 0,	helper_exit_code)
-
-	__u64_field(15,                      0, send_cnt)
-	__u64_field(16,                      0, recv_cnt)
-	__u64_field(17,                      0, read_cnt)
-	__u64_field(18,                      0, writ_cnt)
-	__u64_field(19,                      0, al_writ_cnt)
-	__u64_field(20,                      0, bm_writ_cnt)
-	__u32_field(21,                      0, ap_bio_cnt)
-	__u32_field(22,                      0, ap_pending_cnt)
-	__u32_field(23,                      0, rs_pending_cnt)
-)
-
-GENL_struct(DRBD_NLA_START_OV_PARMS, 9, start_ov_parms,
-	__u64_field(1, 0,	ov_start_sector)
-	__u64_field(2, 0,	ov_stop_sector)
-)
-
-GENL_struct(DRBD_NLA_NEW_C_UUID_PARMS, 10, new_c_uuid_parms,
-	__flg_field(1, 0, clear_bm)
-)
-
-GENL_struct(DRBD_NLA_TIMEOUT_PARMS, 11, timeout_parms,
-	__u32_field(1,	DRBD_F_REQUIRED,	timeout_type)
-)
-
-GENL_struct(DRBD_NLA_DISCONNECT_PARMS, 12, disconnect_parms,
-	__flg_field(1, 0,	force_disconnect)
-)
-
-GENL_struct(DRBD_NLA_DETACH_PARMS, 13, detach_parms,
-	__flg_field(1, 0,	force_detach)
-)
-
-GENL_struct(DRBD_NLA_RESOURCE_INFO, 15, resource_info,
-	__u32_field(1, 0, res_role)
-	__flg_field(2, 0, res_susp)
-	__flg_field(3, 0, res_susp_nod)
-	__flg_field(4, 0, res_susp_fen)
-	/* __flg_field(5, 0, res_weak) */
-)
-
-GENL_struct(DRBD_NLA_DEVICE_INFO, 16, device_info,
-	__u32_field(1, 0, dev_disk_state)
-)
-
-GENL_struct(DRBD_NLA_CONNECTION_INFO, 17, connection_info,
-	__u32_field(1, 0, conn_connection_state)
-	__u32_field(2, 0, conn_role)
-)
-
-GENL_struct(DRBD_NLA_PEER_DEVICE_INFO, 18, peer_device_info,
-	__u32_field(1, 0, peer_repl_state)
-	__u32_field(2, 0, peer_disk_state)
-	__u32_field(3, 0, peer_resync_susp_user)
-	__u32_field(4, 0, peer_resync_susp_peer)
-	__u32_field(5, 0, peer_resync_susp_dependency)
-)
-
-GENL_struct(DRBD_NLA_RESOURCE_STATISTICS, 19, resource_statistics,
-	__u32_field(1, 0, res_stat_write_ordering)
-)
-
-GENL_struct(DRBD_NLA_DEVICE_STATISTICS, 20, device_statistics,
-	__u64_field(1, 0, dev_size)  /* (sectors) */
-	__u64_field(2, 0, dev_read)  /* (sectors) */
-	__u64_field(3, 0, dev_write)  /* (sectors) */
-	__u64_field(4, 0, dev_al_writes)  /* activity log writes (count) */
-	__u64_field(5, 0, dev_bm_writes)  /*  bitmap writes  (count) */
-	__u32_field(6, 0, dev_upper_pending)  /* application requests in progress */
-	__u32_field(7, 0, dev_lower_pending)  /* backing device requests in progress */
-	__flg_field(8, 0, dev_upper_blocked)
-	__flg_field(9, 0, dev_lower_blocked)
-	__flg_field(10, 0, dev_al_suspended)  /* activity log suspended */
-	__u64_field(11, 0, dev_exposed_data_uuid)
-	__u64_field(12, 0, dev_current_uuid)
-	__u32_field(13, 0, dev_disk_flags)
-	__bin_field(14, 0, history_uuids, HISTORY_UUIDS * sizeof(__u64))
-)
-
-GENL_struct(DRBD_NLA_CONNECTION_STATISTICS, 21, connection_statistics,
-	__flg_field(1, 0, conn_congested)
-)
-
-GENL_struct(DRBD_NLA_PEER_DEVICE_STATISTICS, 22, peer_device_statistics,
-	__u64_field(1, 0, peer_dev_received)  /* sectors */
-	__u64_field(2, 0, peer_dev_sent)  /* sectors */
-	__u32_field(3, 0, peer_dev_pending)  /* number of requests */
-	__u32_field(4, 0, peer_dev_unacked)  /* number of requests */
-	__u64_field(5, 0, peer_dev_out_of_sync)  /* sectors */
-	__u64_field(6, 0, peer_dev_resync_failed)  /* sectors */
-	__u64_field(7, 0, peer_dev_bitmap_uuid)
-	__u32_field(9, 0, peer_dev_flags)
-)
-
-GENL_struct(DRBD_NLA_NOTIFICATION_HEADER, 23, drbd_notification_header,
-	__u32_field(1, 0, nh_type)
-)
-
-GENL_struct(DRBD_NLA_HELPER, 24, drbd_helper_info,
-	__str_field(1, 0, helper_name, 32)
-	__u32_field(2, 0, helper_status)
-)
-
-/*
- * Notifications and commands (genlmsghdr->cmd)
- */
-GENL_mc_group(events)
-
-	/* kernel -> userspace announcement of changes */
-GENL_notification(
-	DRBD_EVENT, 1, events,
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_STATE_INFO, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NET_CONF, 0)
-	GENL_tla_expected(DRBD_NLA_DISK_CONF, 0)
-	GENL_tla_expected(DRBD_NLA_SYNCER_CONF, 0)
-)
-
-	/* query kernel for specific or all info */
-GENL_op(
-	DRBD_ADM_GET_STATUS, 2,
-	GENL_op_init(
-		.doit = drbd_adm_get_status,
-		.dumpit = drbd_adm_get_status_all,
-		/* anyone may ask for the status,
-		 * it is broadcasted anyways */
-	),
-	/* To select the object .doit.
-	 * Or a subset of objects in .dumpit. */
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, 0)
-)
-
-	/* add DRBD minor devices as volumes to resources */
-GENL_op(DRBD_ADM_NEW_MINOR, 5, GENL_doit(drbd_adm_new_minor),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_DEL_MINOR, 6, GENL_doit(drbd_adm_del_minor),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-
-	/* add or delete resources */
-GENL_op(DRBD_ADM_NEW_RESOURCE, 7, GENL_doit(drbd_adm_new_resource),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_DEL_RESOURCE, 8, GENL_doit(drbd_adm_del_resource),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-
-GENL_op(DRBD_ADM_RESOURCE_OPTS, 9,
-	GENL_doit(drbd_adm_resource_opts),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_RESOURCE_OPTS, 0)
-)
-
-GENL_op(
-	DRBD_ADM_CONNECT, 10,
-	GENL_doit(drbd_adm_connect),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NET_CONF, DRBD_F_REQUIRED)
-)
-
-GENL_op(
-	DRBD_ADM_CHG_NET_OPTS, 29,
-	GENL_doit(drbd_adm_net_opts),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NET_CONF, DRBD_F_REQUIRED)
-)
-
-GENL_op(DRBD_ADM_DISCONNECT, 11, GENL_doit(drbd_adm_disconnect),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-
-GENL_op(DRBD_ADM_ATTACH, 12,
-	GENL_doit(drbd_adm_attach),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_DISK_CONF, DRBD_F_REQUIRED)
-)
-
-GENL_op(DRBD_ADM_CHG_DISK_OPTS, 28,
-	GENL_doit(drbd_adm_disk_opts),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_DISK_OPTS, DRBD_F_REQUIRED)
-)
-
-GENL_op(
-	DRBD_ADM_RESIZE, 13,
-	GENL_doit(drbd_adm_resize),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_RESIZE_PARMS, 0)
-)
-
-GENL_op(
-	DRBD_ADM_PRIMARY, 14,
-	GENL_doit(drbd_adm_set_role),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_SET_ROLE_PARMS, DRBD_F_REQUIRED)
-)
-
-GENL_op(
-	DRBD_ADM_SECONDARY, 15,
-	GENL_doit(drbd_adm_set_role),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_SET_ROLE_PARMS, DRBD_F_REQUIRED)
-)
-
-GENL_op(
-	DRBD_ADM_NEW_C_UUID, 16,
-	GENL_doit(drbd_adm_new_c_uuid),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NEW_C_UUID_PARMS, 0)
-)
-
-GENL_op(
-	DRBD_ADM_START_OV, 17,
-	GENL_doit(drbd_adm_start_ov),
-	GENL_tla_expected(DRBD_NLA_START_OV_PARMS, 0)
-)
-
-GENL_op(DRBD_ADM_DETACH,	18, GENL_doit(drbd_adm_detach),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_DETACH_PARMS, 0))
-
-GENL_op(DRBD_ADM_INVALIDATE,	19, GENL_doit(drbd_adm_invalidate),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_INVAL_PEER,	20, GENL_doit(drbd_adm_invalidate_peer),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_PAUSE_SYNC,	21, GENL_doit(drbd_adm_pause_sync),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_RESUME_SYNC,	22, GENL_doit(drbd_adm_resume_sync),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_SUSPEND_IO,	23, GENL_doit(drbd_adm_suspend_io),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_RESUME_IO,	24, GENL_doit(drbd_adm_resume_io),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_OUTDATE,	25, GENL_doit(drbd_adm_outdate),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_GET_TIMEOUT_TYPE, 26, GENL_doit(drbd_adm_get_timeout_type),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-GENL_op(DRBD_ADM_DOWN,		27, GENL_doit(drbd_adm_down),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED))
-
-GENL_op(DRBD_ADM_GET_RESOURCES, 30,
-	 GENL_op_init(
-		 .dumpit = drbd_adm_dump_resources,
-	 ),
-	 GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, 0)
-	 GENL_tla_expected(DRBD_NLA_RESOURCE_INFO, 0)
-	 GENL_tla_expected(DRBD_NLA_RESOURCE_STATISTICS, 0))
-
-GENL_op(DRBD_ADM_GET_DEVICES, 31,
-	 GENL_op_init(
-		 .dumpit = drbd_adm_dump_devices,
-		 .done = drbd_adm_dump_devices_done,
-	 ),
-	 GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, 0)
-	 GENL_tla_expected(DRBD_NLA_DEVICE_INFO, 0)
-	 GENL_tla_expected(DRBD_NLA_DEVICE_STATISTICS, 0))
-
-GENL_op(DRBD_ADM_GET_CONNECTIONS, 32,
-	 GENL_op_init(
-		 .dumpit = drbd_adm_dump_connections,
-		 .done = drbd_adm_dump_connections_done,
-	 ),
-	 GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, 0)
-	 GENL_tla_expected(DRBD_NLA_CONNECTION_INFO, 0)
-	 GENL_tla_expected(DRBD_NLA_CONNECTION_STATISTICS, 0))
-
-GENL_op(DRBD_ADM_GET_PEER_DEVICES, 33,
-	 GENL_op_init(
-		 .dumpit = drbd_adm_dump_peer_devices,
-		 .done = drbd_adm_dump_peer_devices_done,
-	 ),
-	 GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, 0)
-	 GENL_tla_expected(DRBD_NLA_PEER_DEVICE_INFO, 0)
-	 GENL_tla_expected(DRBD_NLA_PEER_DEVICE_STATISTICS, 0))
-
-GENL_notification(
-	DRBD_RESOURCE_STATE, 34, events,
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_RESOURCE_INFO, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_RESOURCE_STATISTICS, DRBD_F_REQUIRED))
-
-GENL_notification(
-	DRBD_DEVICE_STATE, 35, events,
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_DEVICE_INFO, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_DEVICE_STATISTICS, DRBD_F_REQUIRED))
-
-GENL_notification(
-	DRBD_CONNECTION_STATE, 36, events,
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_CONNECTION_INFO, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_CONNECTION_STATISTICS, DRBD_F_REQUIRED))
-
-GENL_notification(
-	DRBD_PEER_DEVICE_STATE, 37, events,
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_PEER_DEVICE_INFO, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_PEER_DEVICE_STATISTICS, DRBD_F_REQUIRED))
-
-GENL_op(
-	DRBD_ADM_GET_INITIAL_STATE, 38,
-	GENL_op_init(
-	        .dumpit = drbd_adm_get_initial_state,
-	),
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, 0))
-
-GENL_notification(
-	DRBD_HELPER, 40, events,
-	GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)
-	GENL_tla_expected(DRBD_NLA_HELPER, DRBD_F_REQUIRED))
-
-GENL_notification(
-	DRBD_INITIAL_STATE_DONE, 41, events,
-	GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED))
diff --git a/include/linux/drbd_genl_api.h b/include/linux/drbd_genl_api.h
deleted file mode 100644
index 70682c0580270..0000000000000
--- a/include/linux/drbd_genl_api.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef DRBD_GENL_STRUCT_H
-#define DRBD_GENL_STRUCT_H
-
-/**
- * struct drbd_genlmsghdr - DRBD specific header used in NETLINK_GENERIC requests
- * @minor:
- *     For admin requests (user -> kernel): which minor device to operate on.
- *     For (unicast) replies or informational (broadcast) messages
- *     (kernel -> user): which minor device the information is about.
- *     If we do not operate on minors, but on connections or resources,
- *     the minor value shall be (~0), and the attribute DRBD_NLA_CFG_CONTEXT
- *     is used instead.
- * @flags: possible operation modifiers (relevant only for user->kernel):
- *     DRBD_GENL_F_SET_DEFAULTS
- * @volume:
- *     When creating a new minor (adding it to a resource), the resource needs
- *     to know which volume number within the resource this is supposed to be.
- *     The volume number corresponds to the same volume number on the remote side,
- *     whereas the minor number on the remote side may be different
- *     (union with flags).
- * @ret_code: kernel->userland unicast cfg reply return code (union with flags);
- */
-struct drbd_genlmsghdr {
-	__u32 minor;
-	union {
-	__u32 flags;
-	__s32 ret_code;
-	};
-};
-
-/* To be used in drbd_genlmsghdr.flags */
-enum {
-	DRBD_GENL_F_SET_DEFAULTS = 1,
-};
-
-enum drbd_state_info_bcast_reason {
-	SIB_GET_STATUS_REPLY = 1,
-	SIB_STATE_CHANGE = 2,
-	SIB_HELPER_PRE = 3,
-	SIB_HELPER_POST = 4,
-	SIB_SYNC_PROGRESS = 5,
-};
-
-/* hack around predefined gcc/cpp "linux=1",
- * we cannot possibly include <1/drbd_genl.h> */
-#undef linux
-
-#include <linux/drbd.h>
-#define GENL_MAGIC_VERSION	1
-#define GENL_MAGIC_FAMILY	drbd
-#define GENL_MAGIC_FAMILY_HDRSZ	sizeof(struct drbd_genlmsghdr)
-#define GENL_MAGIC_INCLUDE_FILE <linux/drbd_genl.h>
-#include <linux/genl_magic_struct.h>
-
-#endif
diff --git a/include/linux/drbd_limits.h b/include/linux/drbd_limits.h
deleted file mode 100644
index 5b042fb427e97..0000000000000
--- a/include/linux/drbd_limits.h
+++ /dev/null
@@ -1,251 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
-  drbd_limits.h
-  This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
-*/
-
-/*
- * Our current limitations.
- * Some of them are hard limits,
- * some of them are arbitrary range limits, that make it easier to provide
- * feedback about nonsense settings for certain configurable values.
- */
-
-#ifndef DRBD_LIMITS_H
-#define DRBD_LIMITS_H 1
-
-#define DEBUG_RANGE_CHECK 0
-
-#define DRBD_MINOR_COUNT_MIN 1U
-#define DRBD_MINOR_COUNT_MAX 255U
-#define DRBD_MINOR_COUNT_DEF 32U
-#define DRBD_MINOR_COUNT_SCALE '1'
-
-#define DRBD_VOLUME_MAX 65534U
-
-#define DRBD_DIALOG_REFRESH_MIN 0U
-#define DRBD_DIALOG_REFRESH_MAX 600U
-#define DRBD_DIALOG_REFRESH_SCALE '1'
-
-/* valid port number */
-#define DRBD_PORT_MIN 1U
-#define DRBD_PORT_MAX 0xffffU
-#define DRBD_PORT_SCALE '1'
-
-/* startup { */
-  /* if you want more than 3.4 days, disable */
-#define DRBD_WFC_TIMEOUT_MIN 0U
-#define DRBD_WFC_TIMEOUT_MAX 300000U
-#define DRBD_WFC_TIMEOUT_DEF 0U
-#define DRBD_WFC_TIMEOUT_SCALE '1'
-
-#define DRBD_DEGR_WFC_TIMEOUT_MIN 0U
-#define DRBD_DEGR_WFC_TIMEOUT_MAX 300000U
-#define DRBD_DEGR_WFC_TIMEOUT_DEF 0U
-#define DRBD_DEGR_WFC_TIMEOUT_SCALE '1'
-
-#define DRBD_OUTDATED_WFC_TIMEOUT_MIN 0U
-#define DRBD_OUTDATED_WFC_TIMEOUT_MAX 300000U
-#define DRBD_OUTDATED_WFC_TIMEOUT_DEF 0U
-#define DRBD_OUTDATED_WFC_TIMEOUT_SCALE '1'
-/* }*/
-
-/* net { */
-  /* timeout, unit centi seconds
-   * more than one minute timeout is not useful */
-#define DRBD_TIMEOUT_MIN 1U
-#define DRBD_TIMEOUT_MAX 600U
-#define DRBD_TIMEOUT_DEF 60U       /* 6 seconds */
-#define DRBD_TIMEOUT_SCALE '1'
-
- /* If backing disk takes longer than disk_timeout, mark the disk as failed */
-#define DRBD_DISK_TIMEOUT_MIN 0U    /* 0 = disabled */
-#define DRBD_DISK_TIMEOUT_MAX 6000U /* 10 Minutes */
-#define DRBD_DISK_TIMEOUT_DEF 0U    /* disabled */
-#define DRBD_DISK_TIMEOUT_SCALE '1'
-
-  /* active connection retries when C_WF_CONNECTION */
-#define DRBD_CONNECT_INT_MIN 1U
-#define DRBD_CONNECT_INT_MAX 120U
-#define DRBD_CONNECT_INT_DEF 10U   /* seconds */
-#define DRBD_CONNECT_INT_SCALE '1'
-
-  /* keep-alive probes when idle */
-#define DRBD_PING_INT_MIN 1U
-#define DRBD_PING_INT_MAX 120U
-#define DRBD_PING_INT_DEF 10U
-#define DRBD_PING_INT_SCALE '1'
-
- /* timeout for the ping packets.*/
-#define DRBD_PING_TIMEO_MIN  1U
-#define DRBD_PING_TIMEO_MAX  300U
-#define DRBD_PING_TIMEO_DEF  5U
-#define DRBD_PING_TIMEO_SCALE '1'
-
-  /* max number of write requests between write barriers */
-#define DRBD_MAX_EPOCH_SIZE_MIN 1U
-#define DRBD_MAX_EPOCH_SIZE_MAX 20000U
-#define DRBD_MAX_EPOCH_SIZE_DEF 2048U
-#define DRBD_MAX_EPOCH_SIZE_SCALE '1'
-
-  /* I don't think that a tcp send buffer of more than 10M is useful */
-#define DRBD_SNDBUF_SIZE_MIN  0U
-#define DRBD_SNDBUF_SIZE_MAX  (10U<<20)
-#define DRBD_SNDBUF_SIZE_DEF  0U
-#define DRBD_SNDBUF_SIZE_SCALE '1'
-
-#define DRBD_RCVBUF_SIZE_MIN  0U
-#define DRBD_RCVBUF_SIZE_MAX  (10U<<20)
-#define DRBD_RCVBUF_SIZE_DEF  0U
-#define DRBD_RCVBUF_SIZE_SCALE '1'
-
-  /* @4k PageSize -> 128kB - 512MB */
-#define DRBD_MAX_BUFFERS_MIN  32U
-#define DRBD_MAX_BUFFERS_MAX  131072U
-#define DRBD_MAX_BUFFERS_DEF  2048U
-#define DRBD_MAX_BUFFERS_SCALE '1'
-
-  /* @4k PageSize -> 4kB - 512MB */
-#define DRBD_UNPLUG_WATERMARK_MIN  1U
-#define DRBD_UNPLUG_WATERMARK_MAX  131072U
-#define DRBD_UNPLUG_WATERMARK_DEF (DRBD_MAX_BUFFERS_DEF/16)
-#define DRBD_UNPLUG_WATERMARK_SCALE '1'
-
-  /* 0 is disabled.
-   * 200 should be more than enough even for very short timeouts */
-#define DRBD_KO_COUNT_MIN  0U
-#define DRBD_KO_COUNT_MAX  200U
-#define DRBD_KO_COUNT_DEF  7U
-#define DRBD_KO_COUNT_SCALE '1'
-/* } */
-
-/* syncer { */
-  /* FIXME allow rate to be zero? */
-#define DRBD_RESYNC_RATE_MIN 1U
-/* channel bonding 10 GbE, or other hardware */
-#define DRBD_RESYNC_RATE_MAX (4 << 20)
-#define DRBD_RESYNC_RATE_DEF 250U
-#define DRBD_RESYNC_RATE_SCALE 'k'  /* kilobytes */
-
-#define DRBD_AL_EXTENTS_MIN  67U
-  /* we use u16 as "slot number", (u16)~0 is "FREE".
-   * If you use >= 292 kB on-disk ring buffer,
-   * this is the maximum you can use: */
-#define DRBD_AL_EXTENTS_MAX  0xfffeU
-#define DRBD_AL_EXTENTS_DEF  1237U
-#define DRBD_AL_EXTENTS_SCALE '1'
-
-#define DRBD_MINOR_NUMBER_MIN  -1
-#define DRBD_MINOR_NUMBER_MAX  ((1 << 20) - 1)
-#define DRBD_MINOR_NUMBER_DEF  -1
-#define DRBD_MINOR_NUMBER_SCALE '1'
-
-/* } */
-
-/* drbdsetup XY resize -d Z
- * you are free to reduce the device size to nothing, if you want to.
- * the upper limit with 64bit kernel, enough ram and flexible meta data
- * is 1 PiB, currently. */
-/* DRBD_MAX_SECTORS */
-#define DRBD_DISK_SIZE_MIN  0LLU
-#define DRBD_DISK_SIZE_MAX  (1LLU * (2LLU << 40))
-#define DRBD_DISK_SIZE_DEF  0LLU /* = disabled = no user size... */
-#define DRBD_DISK_SIZE_SCALE 's'  /* sectors */
-
-#define DRBD_ON_IO_ERROR_DEF EP_DETACH
-#define DRBD_FENCING_DEF FP_DONT_CARE
-#define DRBD_AFTER_SB_0P_DEF ASB_DISCONNECT
-#define DRBD_AFTER_SB_1P_DEF ASB_DISCONNECT
-#define DRBD_AFTER_SB_2P_DEF ASB_DISCONNECT
-#define DRBD_RR_CONFLICT_DEF ASB_DISCONNECT
-#define DRBD_ON_NO_DATA_DEF OND_IO_ERROR
-#define DRBD_ON_CONGESTION_DEF OC_BLOCK
-#define DRBD_READ_BALANCING_DEF RB_PREFER_LOCAL
-
-#define DRBD_MAX_BIO_BVECS_MIN 0U
-#define DRBD_MAX_BIO_BVECS_MAX 128U
-#define DRBD_MAX_BIO_BVECS_DEF 0U
-#define DRBD_MAX_BIO_BVECS_SCALE '1'
-
-#define DRBD_C_PLAN_AHEAD_MIN  0U
-#define DRBD_C_PLAN_AHEAD_MAX  300U
-#define DRBD_C_PLAN_AHEAD_DEF  20U
-#define DRBD_C_PLAN_AHEAD_SCALE '1'
-
-#define DRBD_C_DELAY_TARGET_MIN 1U
-#define DRBD_C_DELAY_TARGET_MAX 100U
-#define DRBD_C_DELAY_TARGET_DEF 10U
-#define DRBD_C_DELAY_TARGET_SCALE '1'
-
-#define DRBD_C_FILL_TARGET_MIN 0U
-#define DRBD_C_FILL_TARGET_MAX (1U<<20) /* 500MByte in sec */
-#define DRBD_C_FILL_TARGET_DEF 100U /* Try to place 50KiB in socket send buffer during resync */
-#define DRBD_C_FILL_TARGET_SCALE 's'  /* sectors */
-
-#define DRBD_C_MAX_RATE_MIN     250U
-#define DRBD_C_MAX_RATE_MAX     (4U << 20)
-#define DRBD_C_MAX_RATE_DEF     102400U
-#define DRBD_C_MAX_RATE_SCALE	'k'  /* kilobytes */
-
-#define DRBD_C_MIN_RATE_MIN     0U
-#define DRBD_C_MIN_RATE_MAX     (4U << 20)
-#define DRBD_C_MIN_RATE_DEF     250U
-#define DRBD_C_MIN_RATE_SCALE	'k'  /* kilobytes */
-
-#define DRBD_CONG_FILL_MIN	0U
-#define DRBD_CONG_FILL_MAX	(10U<<21) /* 10GByte in sectors */
-#define DRBD_CONG_FILL_DEF	0U
-#define DRBD_CONG_FILL_SCALE	's'  /* sectors */
-
-#define DRBD_CONG_EXTENTS_MIN	DRBD_AL_EXTENTS_MIN
-#define DRBD_CONG_EXTENTS_MAX	DRBD_AL_EXTENTS_MAX
-#define DRBD_CONG_EXTENTS_DEF	DRBD_AL_EXTENTS_DEF
-#define DRBD_CONG_EXTENTS_SCALE DRBD_AL_EXTENTS_SCALE
-
-#define DRBD_PROTOCOL_DEF DRBD_PROT_C
-
-#define DRBD_DISK_BARRIER_DEF	0U
-#define DRBD_DISK_FLUSHES_DEF	1U
-#define DRBD_DISK_DRAIN_DEF	1U
-#define DRBD_MD_FLUSHES_DEF	1U
-#define DRBD_TCP_CORK_DEF	1U
-#define DRBD_AL_UPDATES_DEF     1U
-
-/* We used to ignore the discard_zeroes_data setting.
- * To not change established (and expected) behaviour,
- * by default assume that, for discard_zeroes_data=0,
- * we can make that an effective discard_zeroes_data=1,
- * if we only explicitly zero-out unaligned partial chunks. */
-#define DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF 1U
-
-/* Some backends pretend to support WRITE SAME,
- * but fail such requests when they are actually submitted.
- * This is to tell DRBD to not even try. */
-#define DRBD_DISABLE_WRITE_SAME_DEF 0U
-
-#define DRBD_ALLOW_TWO_PRIMARIES_DEF	0U
-#define DRBD_ALWAYS_ASBP_DEF	0U
-#define DRBD_USE_RLE_DEF	1U
-#define DRBD_CSUMS_AFTER_CRASH_ONLY_DEF 0U
-
-#define DRBD_AL_STRIPES_MIN     1U
-#define DRBD_AL_STRIPES_MAX     1024U
-#define DRBD_AL_STRIPES_DEF     1U
-#define DRBD_AL_STRIPES_SCALE   '1'
-
-#define DRBD_AL_STRIPE_SIZE_MIN   4U
-#define DRBD_AL_STRIPE_SIZE_MAX   16777216U
-#define DRBD_AL_STRIPE_SIZE_DEF   32U
-#define DRBD_AL_STRIPE_SIZE_SCALE 'k' /* kilobytes */
-
-#define DRBD_SOCKET_CHECK_TIMEO_MIN 0U
-#define DRBD_SOCKET_CHECK_TIMEO_MAX DRBD_PING_TIMEO_MAX
-#define DRBD_SOCKET_CHECK_TIMEO_DEF 0U
-#define DRBD_SOCKET_CHECK_TIMEO_SCALE '1'
-
-#define DRBD_RS_DISCARD_GRANULARITY_MIN 0U
-#define DRBD_RS_DISCARD_GRANULARITY_MAX (1U<<20)  /* 1MiByte */
-#define DRBD_RS_DISCARD_GRANULARITY_DEF 0U     /* disabled by default */
-#define DRBD_RS_DISCARD_GRANULARITY_SCALE '1' /* bytes */
-
-#endif
diff --git a/include/linux/edac.h b/include/linux/edac.h
index deba46b3ee25d..e6b4e51130e5f 100644
--- a/include/linux/edac.h
+++ b/include/linux/edac.h
@@ -184,6 +184,7 @@ static inline char *mc_event_error_type(const unsigned int err_type)
  * @MEM_DDR5:		Unbuffered DDR5 RAM
  * @MEM_RDDR5:		Registered DDR5 RAM
  * @MEM_LRDDR5:		Load-Reduced DDR5 memory.
+ * @MEM_LPDDR5:		Low-Power DDR5 memory.
  * @MEM_NVDIMM:		Non-volatile RAM
  * @MEM_WIO2:		Wide I/O 2.
  * @MEM_HBM2:		High bandwidth Memory Gen 2.
@@ -216,6 +217,7 @@ enum mem_type {
 	MEM_DDR5,
 	MEM_RDDR5,
 	MEM_LRDDR5,
+	MEM_LPDDR5,
 	MEM_NVDIMM,
 	MEM_WIO2,
 	MEM_HBM2,
@@ -247,6 +249,7 @@ enum mem_type {
 #define MEM_FLAG_DDR5		BIT(MEM_DDR5)
 #define MEM_FLAG_RDDR5		BIT(MEM_RDDR5)
 #define MEM_FLAG_LRDDR5		BIT(MEM_LRDDR5)
+#define MEM_FLAG_LPDDR5		BIT(MEM_LPDDR5)
 #define MEM_FLAG_NVDIMM		BIT(MEM_NVDIMM)
 #define MEM_FLAG_WIO2		BIT(MEM_WIO2)
 #define MEM_FLAG_HBM2		BIT(MEM_HBM2)
diff --git a/include/linux/efi.h b/include/linux/efi.h
index 72e76ec546415..ccbc35479684a 100644
--- a/include/linux/efi.h
+++ b/include/linux/efi.h
@@ -61,7 +61,7 @@ typedef void *efi_handle_t;
 
 /*
  * The UEFI spec and EDK2 reference implementation both define EFI_GUID as
- * struct { u32 a; u16; b; u16 c; u8 d[8]; }; and so the implied alignment
+ * struct { u32 a; u16 b; u16 c; u8 d[8]; }; and so the implied alignment
  * is 32 bits not 8 bits like our guid_t. In some cases (i.e., on 32-bit ARM),
  * this means that firmware services invoked by the kernel may assume that
  * efi_guid_t* arguments are 32-bit aligned, and use memory accessors that
diff --git a/include/linux/eventpoll.h b/include/linux/eventpoll.h
index 728fb5dee5ede..de1c738aa8ad9 100644
--- a/include/linux/eventpoll.h
+++ b/include/linux/eventpoll.h
@@ -61,8 +61,16 @@ static inline void eventpoll_release(struct file *file)
 	eventpoll_release_file(file);
 }
 
+struct epoll_key {
+	struct file *file;
+	int fd;
+} __packed;
+
+int do_epoll_ctl_file(struct file *f, int op, struct epoll_key *tf,
+		      struct epoll_event *epds, bool nonblock);
 int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,
 		 bool nonblock);
+bool is_file_epoll(struct file *f);
 
 /* Tells if the epoll_ctl(2) operation needs an event copy from userspace */
 static inline int ep_op_has_event(int op)
diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h
index 8bcdba28b4060..c835bc64f4fad 100644
--- a/include/linux/exportfs.h
+++ b/include/linux/exportfs.h
@@ -6,9 +6,8 @@
 #include <linux/path.h>
 
 struct dentry;
-struct iattr;
+struct exportfs_block_ops;
 struct inode;
-struct iomap;
 struct super_block;
 struct vfsmount;
 
@@ -260,19 +259,13 @@ struct handle_to_path_ctx {
  * @commit_metadata:
  *    @commit_metadata should commit metadata changes to stable storage.
  *
- * @get_uuid:
- *    Get a filesystem unique signature exposed to clients.
- *
- * @map_blocks:
- *    Map and, if necessary, allocate blocks for a layout.
- *
- * @commit_blocks:
- *    Commit blocks in a layout once the client is done with them.
- *
  * @flags:
  *    Allows the filesystem to communicate to nfsd that it may want to do things
  *    differently when dealing with it.
  *
+ * @block_ops:
+ *    Operations for layout grants to block on the underlying device.
+ *
  * Locking rules:
  *    get_parent is called with child->d_inode->i_rwsem down
  *    get_name is not (which is possibly inconsistent)
@@ -290,12 +283,6 @@ struct export_operations {
 	struct dentry * (*get_parent)(struct dentry *child);
 	int (*commit_metadata)(struct inode *inode);
 
-	int (*get_uuid)(struct super_block *sb, u8 *buf, u32 *len, u64 *offset);
-	int (*map_blocks)(struct inode *inode, loff_t offset,
-			  u64 len, struct iomap *iomap,
-			  bool write, u32 *device_generation);
-	int (*commit_blocks)(struct inode *inode, struct iomap *iomaps,
-			     int nr_iomaps, struct iattr *iattr);
 	int (*permission)(struct handle_to_path_ctx *ctx, unsigned int oflags);
 	struct file * (*open)(const struct path *path, unsigned int oflags);
 #define	EXPORT_OP_NOWCC			(0x1) /* don't collect v3 wcc data */
@@ -308,6 +295,10 @@ struct export_operations {
 #define EXPORT_OP_FLUSH_ON_CLOSE	(0x20) /* fs flushes file data on close */
 #define EXPORT_OP_NOLOCKS		(0x40) /* no file locking support */
 	unsigned long	flags;
+
+#ifdef CONFIG_EXPORTFS_BLOCK_OPS
+	const struct exportfs_block_ops *block_ops;
+#endif
 };
 
 /**
diff --git a/include/linux/exportfs_block.h b/include/linux/exportfs_block.h
new file mode 100644
index 0000000000000..de519b7b599b3
--- /dev/null
+++ b/include/linux/exportfs_block.h
@@ -0,0 +1,88 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2014-2026 Christoph Hellwig.
+ *
+ * Support for exportfs-based layout grants for direct block device access.
+ */
+#ifndef LINUX_EXPORTFS_BLOCK_H
+#define LINUX_EXPORTFS_BLOCK_H 1
+
+#include <linux/blkdev.h>
+#include <linux/exportfs.h>
+#include <linux/fs.h>
+
+struct inode;
+struct iomap;
+struct super_block;
+
+/*
+ * There are the two types of block-style layout support:
+ *  - In-band implies a device identified by a unique cookie inside the actual
+ *    device address space checked by the ->get_uuid method as used by the pNFS
+ *    block layout.  This is a bit dangerous and deprecated.
+ *  - Out of band implies identification by out of band unique identifiers
+ *    specified by the storage protocol, which is much safer and used by the
+ *    pNFS SCSI/NVMe layouts.
+ */
+typedef unsigned int __bitwise expfs_block_layouts_t;
+#define EXPFS_BLOCK_FLAG(__bit) \
+	((__force expfs_block_layouts_t)(1u << __bit))
+#define EXPFS_BLOCK_IN_BAND_ID		EXPFS_BLOCK_FLAG(0)
+#define EXPFS_BLOCK_OUT_OF_BAND_ID	EXPFS_BLOCK_FLAG(1)
+
+struct exportfs_block_ops {
+	/*
+	 * Returns the EXPFS_BLOCK_* bitmap of supported layout types.
+	 */
+	expfs_block_layouts_t (*layouts_supported)(struct super_block *sb);
+
+	/*
+	 * Get the in-band device unique signature exposed to clients.
+	 */
+	int (*get_uuid)(struct super_block *sb, u8 *buf, u32 *len, u64 *offset);
+
+	/*
+	 * Map blocks for direct block access.
+	 * If @write is %true, also allocate the blocks for the range if needed.
+	 */
+	int (*map_blocks)(struct inode *inode, loff_t offset, u64 len,
+			struct iomap *iomap, bool write,
+			u32 *device_generation);
+
+	/*
+	 * Commit blocks previously handed out by ->map_blocks and written to by
+	 * the client.
+	 */
+	int (*commit_blocks)(struct inode *inode, struct iomap *iomaps,
+			int nr_iomaps, loff_t new_size);
+};
+
+static inline bool
+exportfs_bdev_supports_out_of_band_id(struct block_device *bdev)
+{
+	return bdev->bd_disk->fops->pr_ops &&
+		bdev->bd_disk->fops->get_unique_id;
+}
+
+#ifdef CONFIG_EXPORTFS_BLOCK_OPS
+static inline expfs_block_layouts_t
+exportfs_layouts_supported(struct super_block *sb)
+{
+	const struct exportfs_block_ops *bops = sb->s_export_op->block_ops;
+
+	if (!bops ||
+	    !bops->layouts_supported ||
+	    WARN_ON_ONCE(!bops->map_blocks) ||
+	    WARN_ON_ONCE(!bops->commit_blocks))
+		return 0;
+	return bops->layouts_supported(sb);
+}
+#else
+static inline expfs_block_layouts_t
+exportfs_layouts_supported(struct super_block *sb)
+{
+	return 0;
+}
+#endif /* CONFIG_EXPORTFS_BLOCK_OPS */
+
+#endif /* LINUX_EXPORTFS_BLOCK_H */
diff --git a/include/linux/fcntl.h b/include/linux/fcntl.h
index a332e79b32079..b354f31dd62ed 100644
--- a/include/linux/fcntl.h
+++ b/include/linux/fcntl.h
@@ -10,7 +10,9 @@
 	(O_RDONLY | O_WRONLY | O_RDWR | O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC | \
 	 O_APPEND | O_NDELAY | O_NONBLOCK | __O_SYNC | O_DSYNC | \
 	 FASYNC	| O_DIRECT | O_LARGEFILE | O_DIRECTORY | O_NOFOLLOW | \
-	 O_NOATIME | O_CLOEXEC | O_PATH | __O_TMPFILE)
+	 O_NOATIME | O_CLOEXEC | O_PATH | __O_TMPFILE | O_EMPTYPATH)
+
+#define VALID_OPENAT2_FLAGS (VALID_OPEN_FLAGS | OPENAT2_REGULAR)
 
 /* List of all valid flags for the how->resolve argument: */
 #define VALID_RESOLVE_FLAGS \
diff --git a/include/linux/fileattr.h b/include/linux/fileattr.h
index 3780904a63a6c..58044b5980162 100644
--- a/include/linux/fileattr.h
+++ b/include/linux/fileattr.h
@@ -16,7 +16,8 @@
 
 /* Read-only inode flags */
 #define FS_XFLAG_RDONLY_MASK \
-	(FS_XFLAG_PREALLOC | FS_XFLAG_HASATTR | FS_XFLAG_VERITY)
+	(FS_XFLAG_PREALLOC | FS_XFLAG_HASATTR | FS_XFLAG_VERITY | \
+	 FS_XFLAG_CASEFOLD | FS_XFLAG_CASENONPRESERVING)
 
 /* Flags to indicate valid value of fsx_ fields */
 #define FS_XFLAG_VALUES_MASK \
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 1ec6d5ba64cc4..67d337ede91b8 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -58,8 +58,9 @@ struct ctl_table_header;
 #define BPF_REG_H	BPF_REG_9	/* hlen, callee-saved */
 
 /* Kernel hidden auxiliary/helper register. */
-#define BPF_REG_AX		MAX_BPF_REG
-#define MAX_BPF_EXT_REG		(MAX_BPF_REG + 1)
+#define BPF_REG_PARAMS		MAX_BPF_REG
+#define BPF_REG_AX		(MAX_BPF_REG + 1)
+#define MAX_BPF_EXT_REG		(MAX_BPF_REG + 2)
 #define MAX_BPF_JIT_REG		MAX_BPF_EXT_REG
 
 /* unused opcode to mark special call to bpf_tail_call() helper */
@@ -748,6 +749,27 @@ static inline u32 bpf_prog_run_pin_on_cpu(const struct bpf_prog *prog,
 	return ret;
 }
 
+static inline bool is_stack_arg_ldx(const struct bpf_insn *insn)
+{
+	return insn->code == (BPF_LDX | BPF_MEM | BPF_DW) &&
+	       insn->src_reg == BPF_REG_PARAMS &&
+	       insn->off > 0 && insn->off % 8 == 0;
+}
+
+static inline bool is_stack_arg_st(const struct bpf_insn *insn)
+{
+	return insn->code == (BPF_ST | BPF_MEM | BPF_DW) &&
+	       insn->dst_reg == BPF_REG_PARAMS &&
+	       insn->off < 0 && insn->off % 8 == 0;
+}
+
+static inline bool is_stack_arg_stx(const struct bpf_insn *insn)
+{
+	return insn->code == (BPF_STX | BPF_MEM | BPF_DW) &&
+	       insn->dst_reg == BPF_REG_PARAMS &&
+	       insn->off < 0 && insn->off % 8 == 0;
+}
+
 #define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN
 
 struct bpf_skb_data_end {
@@ -1151,9 +1173,6 @@ bool sk_filter_charge(struct sock *sk, struct sk_filter *fp);
 void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp);
 
 u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
-#define __bpf_call_base_args \
-	((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \
-	 (void *)__bpf_call_base)
 
 struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog);
 void bpf_jit_compile(struct bpf_prog *prog);
@@ -1162,6 +1181,7 @@ bool bpf_jit_inlines_helper_call(s32 imm);
 bool bpf_jit_supports_subprog_tailcalls(void);
 bool bpf_jit_supports_percpu_insn(void);
 bool bpf_jit_supports_kfunc_call(void);
+bool bpf_jit_supports_stack_args(void);
 bool bpf_jit_supports_far_kfunc_call(void);
 bool bpf_jit_supports_exceptions(void);
 bool bpf_jit_supports_ptr_xchg(void);
diff --git a/include/linux/firmware.h b/include/linux/firmware.h
index aae1b85ffc10e..0fa3b027f02f1 100644
--- a/include/linux/firmware.h
+++ b/include/linux/firmware.h
@@ -110,6 +110,9 @@ int request_firmware_nowait(
 	struct module *module, bool uevent,
 	const char *name, struct device *device, gfp_t gfp, void *context,
 	void (*cont)(const struct firmware *fw, void *context));
+void request_firmware_nowait_cancel(struct device *device, void *context,
+				    void (*cont)(const struct firmware *fw,
+						 void *context));
 int request_firmware_direct(const struct firmware **fw, const char *name,
 			    struct device *device);
 int request_firmware_into_buf(const struct firmware **firmware_p,
@@ -157,6 +160,13 @@ static inline int request_firmware_nowait(
 	return -EINVAL;
 }
 
+static inline void request_firmware_nowait_cancel(struct device *device,
+						  void *context,
+						  void (*cont)(const struct firmware *fw,
+							       void *context))
+{
+}
+
 static inline void release_firmware(const struct firmware *fw)
 {
 }
diff --git a/include/linux/firmware/imx/se_api.h b/include/linux/firmware/imx/se_api.h
new file mode 100644
index 0000000000000..b1c4c9115d7b2
--- /dev/null
+++ b/include/linux/firmware/imx/se_api.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * Copyright 2025 NXP
+ */
+
+#ifndef __SE_API_H__
+#define __SE_API_H__
+
+#include <linux/types.h>
+
+#define SOC_ID_OF_IMX8ULP		0x084d
+#define SOC_ID_OF_IMX93			0x9300
+
+#endif /* __SE_API_H__ */
diff --git a/include/linux/firmware/samsung/exynos-acpm-protocol.h b/include/linux/firmware/samsung/exynos-acpm-protocol.h
index 13f17dc4443b8..d4db2796a6fbf 100644
--- a/include/linux/firmware/samsung/exynos-acpm-protocol.h
+++ b/include/linux/firmware/samsung/exynos-acpm-protocol.h
@@ -8,6 +8,7 @@
 #ifndef __EXYNOS_ACPM_PROTOCOL_H
 #define __EXYNOS_ACPM_PROTOCOL_H
 
+#include <linux/err.h>
 #include <linux/types.h>
 
 struct acpm_handle;
@@ -57,7 +58,7 @@ struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
 static inline struct acpm_handle *devm_acpm_get_by_node(struct device *dev,
 							struct device_node *np)
 {
-	return NULL;
+	return ERR_PTR(-ENODEV);
 }
 #endif
 
diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h
index d70dcd462b441..7e27b0f7bf7ed 100644
--- a/include/linux/firmware/xlnx-zynqmp.h
+++ b/include/linux/firmware/xlnx-zynqmp.h
@@ -542,6 +542,18 @@ enum pm_gem_config_type {
 	GEM_CONFIG_FIXED = 2,
 };
 
+/**
+ * enum pm_node_status - Device node status provided by xilpm fw
+ * @PM_NODE_UNUSED: Device is not used
+ * @PM_NODE_RUNNING: Device is power-on and out of reset
+ * @PM_NODE_HALT: Device is power-on but in the reset state
+ */
+enum pm_node_status {
+	PM_NODE_UNUSED = 0,
+	PM_NODE_RUNNING = 1,
+	PM_NODE_HALT = 12,
+};
+
 /**
  * struct zynqmp_pm_query_data - PM query data
  * @qid:	query ID
@@ -630,6 +642,8 @@ int zynqmp_pm_set_rpu_mode(u32 node_id, enum rpu_oper_mode rpu_mode);
 int zynqmp_pm_set_tcm_config(u32 node_id, enum rpu_tcm_comb tcm_mode);
 int zynqmp_pm_get_node_status(const u32 node, u32 *const status,
 			      u32 *const requirements, u32 *const usage);
+int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status,
+				  u32 *const requirements, u32 *const usage);
 int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value);
 int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config,
 			     u32 value);
@@ -939,6 +953,13 @@ static inline int zynqmp_pm_get_node_status(const u32 node, u32 *const status,
 	return -ENODEV;
 }
 
+static inline int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status,
+						u32 *const requirements,
+						u32 *const usage)
+{
+	return -ENODEV;
+}
+
 static inline int zynqmp_pm_set_sd_config(u32 node,
 					  enum pm_sd_config_type config,
 					  u32 value)
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 11559c513dfbb..6439aa13cce71 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -460,7 +460,6 @@ struct mapping_metadata_bhs {
  *   memory mappings.
  * @gfp_mask: Memory allocation flags to use for allocating pages.
  * @i_mmap_writable: Number of VM_SHARED, VM_MAYWRITE mappings.
- * @nr_thps: Number of THPs in the pagecache (non-shmem only).
  * @i_mmap: Tree of private and shared mappings.
  * @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable.
  * @nrpages: Number of page entries, protected by the i_pages lock.
@@ -476,10 +475,6 @@ struct address_space {
 	struct rw_semaphore	invalidate_lock;
 	gfp_t			gfp_mask;
 	atomic_t		i_mmap_writable;
-#ifdef CONFIG_READ_ONLY_THP_FOR_FS
-	/* number of thp, only for non-shmem files */
-	atomic_t		nr_thps;
-#endif
 	struct rb_root_cached	i_mmap;
 	unsigned long		nrpages;
 	pgoff_t			writeback_index;
@@ -2218,8 +2213,21 @@ static inline void mark_inode_dirty_sync(struct inode *inode)
 	__mark_inode_dirty(inode, I_DIRTY_SYNC);
 }
 
+/*
+ * returns the refcount on the inode. it can change arbitrarily.
+ */
+static inline int icount_read_once(const struct inode *inode)
+{
+	return atomic_read(&inode->i_count);
+}
+
+/*
+ * returns the refcount on the inode. The lock guarantees no 0->1 or 1->0 transitions
+ * of the count are going to take place, otherwise it changes arbitrarily.
+ */
 static inline int icount_read(const struct inode *inode)
 {
+	lockdep_assert_held(&inode->i_lock);
 	return atomic_read(&inode->i_count);
 }
 
@@ -2281,12 +2289,13 @@ struct file_system_type {
 #define FS_MGTIME		64	/* FS uses multigrain timestamps */
 #define FS_LBS			128	/* FS supports LBS */
 #define FS_POWER_FREEZE		256	/* Always freeze on suspend/hibernate */
+#define FS_USERNS_MOUNT_RESTRICTED 512	/* Restrict mount in userns if not already visible */
 #define FS_RENAME_DOES_D_MOVE	32768	/* FS will handle d_move() during rename() internally. */
 	int (*init_fs_context)(struct fs_context *);
 	const struct fs_parameter_spec *parameters;
 	void (*kill_sb) (struct super_block *);
 	struct module *owner;
-	struct file_system_type * next;
+	struct hlist_node list;
 	struct hlist_head fs_supers;
 
 	struct lock_class_key s_lock_key;
diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h
index 383050e7fdf57..a6cdc8f6de4e8 100644
--- a/include/linux/fs/super_types.h
+++ b/include/linux/fs/super_types.h
@@ -326,7 +326,7 @@ struct super_block {
 #define SB_I_STABLE_WRITES 0x00000008	/* don't modify blks until WB is done */
 
 /* sb->s_iflags to limit user namespace mounts */
-#define SB_I_USERNS_VISIBLE		0x00000010 /* fstype already mounted */
+#define SB_I_RESTRICTED_VARIANT		0x00000010
 #define SB_I_IMA_UNVERIFIABLE_SIGNATURE	0x00000020
 #define SB_I_UNTRUSTED_MOUNTER		0x00000040
 #define SB_I_EVM_HMAC_UNSUPPORTED	0x00000080
diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h
index 1da63f2d70401..9f671e87c80c0 100644
--- a/include/linux/fsl/mc.h
+++ b/include/linux/fsl/mc.h
@@ -357,9 +357,11 @@ int mc_send_command(struct fsl_mc_io *mc_io, struct fsl_mc_command *cmd);
 
 #ifdef CONFIG_FSL_MC_BUS
 #define dev_is_fsl_mc(_dev) ((_dev)->bus == &fsl_mc_bus_type)
+u32 fsl_mc_get_msi_id(struct device *dev);
 #else
 /* If fsl-mc bus is not present device cannot belong to fsl-mc bus */
 #define dev_is_fsl_mc(_dev) (0)
+#define fsl_mc_get_msi_id(_dev)	(0)
 #endif
 
 /* Macro to check if a device is a container device */
@@ -419,10 +421,6 @@ int __must_check fsl_mc_object_allocate(struct fsl_mc_device *mc_dev,
 
 void fsl_mc_object_free(struct fsl_mc_device *mc_adev);
 
-struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode,
-						struct msi_domain_info *info,
-						struct irq_domain *parent);
-
 int __must_check fsl_mc_allocate_irqs(struct fsl_mc_device *mc_dev);
 
 void fsl_mc_free_irqs(struct fsl_mc_device *mc_dev);
diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h
index 80b38fbf2121c..c30a9baafc0d9 100644
--- a/include/linux/fwnode.h
+++ b/include/linux/fwnode.h
@@ -208,9 +208,12 @@ struct fwnode_operations {
 static inline void fwnode_init(struct fwnode_handle *fwnode,
 			       const struct fwnode_operations *ops)
 {
+	fwnode->secondary = NULL;
 	fwnode->ops = ops;
+	fwnode->dev = NULL;
 	INIT_LIST_HEAD(&fwnode->consumers);
 	INIT_LIST_HEAD(&fwnode->suppliers);
+	fwnode->flags = 0;
 }
 
 static inline void fwnode_set_flag(struct fwnode_handle *fwnode,
diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h
index fc5d0b5edadc0..07ef1c8341a4f 100644
--- a/include/linux/generic_pt/common.h
+++ b/include/linux/generic_pt/common.h
@@ -134,6 +134,11 @@ enum pt_features {
 	 * significant amount of page table.
 	 */
 	PT_FEAT_FLUSH_RANGE_NO_GAPS,
+	/**
+	 * @PT_FEAT_DETAILED_GATHER: Fill in the struct iommu_iotlb_gather pt
+	 * sub structure with information about which levels were changed.
+	 */
+	PT_FEAT_DETAILED_GATHER,
 	/* private: */
 	PT_FEAT_FMT_START,
 };
@@ -188,6 +193,10 @@ enum {
 	 * Support the 64k contiguous page size following the Svnapot extension.
 	 */
 	PT_FEAT_RISCV_SVNAPOT_64K = PT_FEAT_FMT_START,
+	/*
+	 * Support Svpbmt extension: encode page-based memory type (PBMT) in PTEs.
+	 */
+	PT_FEAT_RISCV_SVPBMT,
 
 };
 
diff --git a/include/linux/genl_magic_func.h b/include/linux/genl_magic_func.h
deleted file mode 100644
index a7d36c9ea9242..0000000000000
--- a/include/linux/genl_magic_func.h
+++ /dev/null
@@ -1,413 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef GENL_MAGIC_FUNC_H
-#define GENL_MAGIC_FUNC_H
-
-#include <linux/args.h>
-#include <linux/build_bug.h>
-#include <linux/genl_magic_struct.h>
-
-/*
- * Magic: declare tla policy						{{{1
- * Magic: declare nested policies
- *									{{{2
- */
-#undef GENL_mc_group
-#define GENL_mc_group(group)
-
-#undef GENL_notification
-#define GENL_notification(op_name, op_num, mcast_group, tla_list)
-
-#undef GENL_op
-#define GENL_op(op_name, op_num, handler, tla_list)
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-	[tag_name] = { .type = NLA_NESTED },
-
-static struct nla_policy CONCATENATE(GENL_MAGIC_FAMILY, _tla_nl_policy)[] = {
-#include GENL_MAGIC_INCLUDE_FILE
-};
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-static struct nla_policy s_name ## _nl_policy[] __read_mostly =		\
-{ s_fields };
-
-#undef __field
-#define __field(attr_nr, attr_flag, name, nla_type, _type, __get,	\
-		 __put, __is_signed)					\
-	[attr_nr] = { .type = nla_type },
-
-#undef __array
-#define __array(attr_nr, attr_flag, name, nla_type, _type, maxlen,	\
-		__get, __put, __is_signed)				\
-	[attr_nr] = { .type = nla_type,					\
-		      .len = maxlen - (nla_type == NLA_NUL_STRING) },
-
-#include GENL_MAGIC_INCLUDE_FILE
-
-#ifndef __KERNEL__
-#ifndef pr_info
-#define pr_info(args...)	fprintf(stderr, args);
-#endif
-#endif
-
-#ifdef GENL_MAGIC_DEBUG
-static void dprint_field(const char *dir, int nla_type,
-		const char *name, void *valp)
-{
-	__u64 val = valp ? *(__u32 *)valp : 1;
-	switch (nla_type) {
-	case NLA_U8:  val = (__u8)val;
-	case NLA_U16: val = (__u16)val;
-	case NLA_U32: val = (__u32)val;
-		pr_info("%s attr %s: %d 0x%08x\n", dir,
-			name, (int)val, (unsigned)val);
-		break;
-	case NLA_U64:
-		val = *(__u64*)valp;
-		pr_info("%s attr %s: %lld 0x%08llx\n", dir,
-			name, (long long)val, (unsigned long long)val);
-		break;
-	case NLA_FLAG:
-		if (val)
-			pr_info("%s attr %s: set\n", dir, name);
-		break;
-	}
-}
-
-static void dprint_array(const char *dir, int nla_type,
-		const char *name, const char *val, unsigned len)
-{
-	switch (nla_type) {
-	case NLA_NUL_STRING:
-		if (len && val[len-1] == '\0')
-			len--;
-		pr_info("%s attr %s: [len:%u] '%s'\n", dir, name, len, val);
-		break;
-	default:
-		/* we can always show 4 byte,
-		 * thats what nlattr are aligned to. */
-		pr_info("%s attr %s: [len:%u] %02x%02x%02x%02x ...\n",
-			dir, name, len, val[0], val[1], val[2], val[3]);
-	}
-}
-
-#define DPRINT_TLA(a, op, b) pr_info("%s %s %s\n", a, op, b);
-
-/* Name is a member field name of the struct s.
- * If s is NULL (only parsing, no copy requested in *_from_attrs()),
- * nla is supposed to point to the attribute containing the information
- * corresponding to that struct member. */
-#define DPRINT_FIELD(dir, nla_type, name, s, nla)			\
-	do {								\
-		if (s)							\
-			dprint_field(dir, nla_type, #name, &s->name);	\
-		else if (nla)						\
-			dprint_field(dir, nla_type, #name,		\
-				(nla_type == NLA_FLAG) ? NULL		\
-						: nla_data(nla));	\
-	} while (0)
-
-#define	DPRINT_ARRAY(dir, nla_type, name, s, nla)			\
-	do {								\
-		if (s)							\
-			dprint_array(dir, nla_type, #name,		\
-					s->name, s->name ## _len);	\
-		else if (nla)						\
-			dprint_array(dir, nla_type, #name,		\
-					nla_data(nla), nla_len(nla));	\
-	} while (0)
-#else
-#define DPRINT_TLA(a, op, b) do {} while (0)
-#define DPRINT_FIELD(dir, nla_type, name, s, nla) do {} while (0)
-#define	DPRINT_ARRAY(dir, nla_type, name, s, nla) do {} while (0)
-#endif
-
-/*
- * Magic: provide conversion functions					{{{1
- * populate struct from attribute table:
- *									{{{2
- */
-
-/* processing of generic netlink messages is serialized.
- * use one static buffer for parsing of nested attributes */
-static struct nlattr *nested_attr_tb[128];
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-/* *_from_attrs functions are static, but potentially unused */		\
-static int __ ## s_name ## _from_attrs(struct s_name *s,		\
-		struct genl_info *info, bool exclude_invariants)	\
-{									\
-	const int maxtype = ARRAY_SIZE(s_name ## _nl_policy)-1;		\
-	struct nlattr *tla = info->attrs[tag_number];			\
-	struct nlattr **ntb = nested_attr_tb;				\
-	struct nlattr *nla;						\
-	int err;							\
-	BUILD_BUG_ON(ARRAY_SIZE(s_name ## _nl_policy) > ARRAY_SIZE(nested_attr_tb));	\
-	if (!tla)							\
-		return -ENOMSG;						\
-	DPRINT_TLA(#s_name, "<=-", #tag_name);				\
-	err = nla_parse_nested_deprecated(ntb, maxtype, tla,			\
-					  s_name ## _nl_policy, NULL);	\
-	if (err)							\
-		return err;						\
-									\
-	s_fields							\
-	return 0;							\
-}					__attribute__((unused))		\
-static int s_name ## _from_attrs(struct s_name *s,			\
-						struct genl_info *info)	\
-{									\
-	return __ ## s_name ## _from_attrs(s, info, false);		\
-}					__attribute__((unused))		\
-static int s_name ## _from_attrs_for_change(struct s_name *s,		\
-						struct genl_info *info)	\
-{									\
-	return __ ## s_name ## _from_attrs(s, info, true);		\
-}					__attribute__((unused))		\
-
-#define __assign(attr_nr, attr_flag, name, nla_type, type, assignment...)	\
-		nla = ntb[attr_nr];						\
-		if (nla) {						\
-			if (exclude_invariants && !!((attr_flag) & DRBD_F_INVARIANT)) {		\
-				pr_info("<< must not change invariant attr: %s\n", #name);	\
-				return -EEXIST;				\
-			}						\
-			assignment;					\
-		} else if (exclude_invariants && !!((attr_flag) & DRBD_F_INVARIANT)) {		\
-			/* attribute missing from payload, */		\
-			/* which was expected */			\
-		} else if ((attr_flag) & DRBD_F_REQUIRED) {		\
-			pr_info("<< missing attr: %s\n", #name);	\
-			return -ENOMSG;					\
-		}
-
-#undef __field
-#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put,	\
-		__is_signed)						\
-	__assign(attr_nr, attr_flag, name, nla_type, type,		\
-			if (s)						\
-				s->name = __get(nla);			\
-			DPRINT_FIELD("<<", nla_type, name, s, nla))
-
-/* validate_nla() already checked nla_len <= maxlen appropriately. */
-#undef __array
-#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen,	\
-		__get, __put, __is_signed)				\
-	__assign(attr_nr, attr_flag, name, nla_type, type,		\
-			if (s)						\
-				s->name ## _len =			\
-					__get(s->name, nla, maxlen);	\
-			DPRINT_ARRAY("<<", nla_type, name, s, nla))
-
-#include GENL_MAGIC_INCLUDE_FILE
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)
-
-/*
- * Magic: define op number to op name mapping				{{{1
- *									{{{2
- */
-static const char *CONCATENATE(GENL_MAGIC_FAMILY, _genl_cmd_to_str)(__u8 cmd)
-{
-	switch (cmd) {
-#undef GENL_op
-#define GENL_op(op_name, op_num, handler, tla_list)		\
-	case op_num: return #op_name;
-#include GENL_MAGIC_INCLUDE_FILE
-	default:
-		     return "unknown";
-	}
-}
-
-#ifdef __KERNEL__
-#include <linux/stringify.h>
-/*
- * Magic: define genl_ops						{{{1
- *									{{{2
- */
-
-#undef GENL_op
-#define GENL_op(op_name, op_num, handler, tla_list)		\
-{								\
-	handler							\
-	.cmd = op_name,						\
-},
-
-#define ZZZ_genl_ops		CONCATENATE(GENL_MAGIC_FAMILY, _genl_ops)
-static struct genl_ops ZZZ_genl_ops[] __read_mostly = {
-#include GENL_MAGIC_INCLUDE_FILE
-};
-
-#undef GENL_op
-#define GENL_op(op_name, op_num, handler, tla_list)
-
-/*
- * Define the genl_family, multicast groups,				{{{1
- * and provide register/unregister functions.
- *									{{{2
- */
-#define ZZZ_genl_family		CONCATENATE(GENL_MAGIC_FAMILY, _genl_family)
-static struct genl_family ZZZ_genl_family;
-/*
- * Magic: define multicast groups
- * Magic: define multicast group registration helper
- */
-#define ZZZ_genl_mcgrps		CONCATENATE(GENL_MAGIC_FAMILY, _genl_mcgrps)
-static const struct genl_multicast_group ZZZ_genl_mcgrps[] = {
-#undef GENL_mc_group
-#define GENL_mc_group(group) { .name = #group, },
-#include GENL_MAGIC_INCLUDE_FILE
-};
-
-enum CONCATENATE(GENL_MAGIC_FAMILY, group_ids) {
-#undef GENL_mc_group
-#define GENL_mc_group(group) CONCATENATE(GENL_MAGIC_FAMILY, _group_ ## group),
-#include GENL_MAGIC_INCLUDE_FILE
-};
-
-#undef GENL_mc_group
-#define GENL_mc_group(group)						\
-static int CONCATENATE(GENL_MAGIC_FAMILY, _genl_multicast_ ## group)(	\
-	struct sk_buff *skb, gfp_t flags)				\
-{									\
-	unsigned int group_id =						\
-		CONCATENATE(GENL_MAGIC_FAMILY, _group_ ## group);		\
-	return genlmsg_multicast(&ZZZ_genl_family, skb, 0,		\
-				 group_id, flags);			\
-}
-
-#include GENL_MAGIC_INCLUDE_FILE
-
-#undef GENL_mc_group
-#define GENL_mc_group(group)
-
-static struct genl_family ZZZ_genl_family __ro_after_init = {
-	.name = __stringify(GENL_MAGIC_FAMILY),
-	.version = GENL_MAGIC_VERSION,
-#ifdef GENL_MAGIC_FAMILY_HDRSZ
-	.hdrsize = NLA_ALIGN(GENL_MAGIC_FAMILY_HDRSZ),
-#endif
-	.maxattr = ARRAY_SIZE(CONCATENATE(GENL_MAGIC_FAMILY, _tla_nl_policy))-1,
-	.policy	= CONCATENATE(GENL_MAGIC_FAMILY, _tla_nl_policy),
-#ifdef GENL_MAGIC_FAMILY_PRE_DOIT
-	.pre_doit = GENL_MAGIC_FAMILY_PRE_DOIT,
-	.post_doit = GENL_MAGIC_FAMILY_POST_DOIT,
-#endif
-	.ops = ZZZ_genl_ops,
-	.n_ops = ARRAY_SIZE(ZZZ_genl_ops),
-	.mcgrps = ZZZ_genl_mcgrps,
-	.resv_start_op = 42, /* drbd is currently the only user */
-	.n_mcgrps = ARRAY_SIZE(ZZZ_genl_mcgrps),
-	.module = THIS_MODULE,
-};
-
-int CONCATENATE(GENL_MAGIC_FAMILY, _genl_register)(void)
-{
-	return genl_register_family(&ZZZ_genl_family);
-}
-
-void CONCATENATE(GENL_MAGIC_FAMILY, _genl_unregister)(void)
-{
-	genl_unregister_family(&ZZZ_genl_family);
-}
-
-/*
- * Magic: provide conversion functions					{{{1
- * populate skb from struct.
- *									{{{2
- */
-
-#undef GENL_op
-#define GENL_op(op_name, op_num, handler, tla_list)
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-static int s_name ## _to_skb(struct sk_buff *skb, struct s_name *s,	\
-		const bool exclude_sensitive)				\
-{									\
-	struct nlattr *tla = nla_nest_start(skb, tag_number);		\
-	if (!tla)							\
-		goto nla_put_failure;					\
-	DPRINT_TLA(#s_name, "-=>", #tag_name);				\
-	s_fields							\
-	nla_nest_end(skb, tla);						\
-	return 0;							\
-									\
-nla_put_failure:							\
-	if (tla)							\
-		nla_nest_cancel(skb, tla);				\
-        return -EMSGSIZE;						\
-}									\
-static inline int s_name ## _to_priv_skb(struct sk_buff *skb,		\
-		struct s_name *s)					\
-{									\
-	return s_name ## _to_skb(skb, s, 0);				\
-}									\
-static inline int s_name ## _to_unpriv_skb(struct sk_buff *skb,		\
-		struct s_name *s)					\
-{									\
-	return s_name ## _to_skb(skb, s, 1);				\
-}
-
-
-#undef __field
-#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put,	\
-		__is_signed)						\
-	if (!exclude_sensitive || !((attr_flag) & DRBD_F_SENSITIVE)) {	\
-		DPRINT_FIELD(">>", nla_type, name, s, NULL);		\
-		if (__put(skb, attr_nr, s->name))			\
-			goto nla_put_failure;				\
-	}
-
-#undef __array
-#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen,	\
-		__get, __put, __is_signed)				\
-	if (!exclude_sensitive || !((attr_flag) & DRBD_F_SENSITIVE)) {	\
-		DPRINT_ARRAY(">>",nla_type, name, s, NULL);		\
-		if (__put(skb, attr_nr, min_t(int, maxlen,		\
-			s->name ## _len + (nla_type == NLA_NUL_STRING)),\
-						s->name))		\
-			goto nla_put_failure;				\
-	}
-
-#include GENL_MAGIC_INCLUDE_FILE
-
-
-/* Functions for initializing structs to default values.  */
-
-#undef __field
-#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put,	\
-		__is_signed)
-#undef __array
-#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen,	\
-		__get, __put, __is_signed)
-#undef __u32_field_def
-#define __u32_field_def(attr_nr, attr_flag, name, default)		\
-	x->name = default;
-#undef __s32_field_def
-#define __s32_field_def(attr_nr, attr_flag, name, default)		\
-	x->name = default;
-#undef __flg_field_def
-#define __flg_field_def(attr_nr, attr_flag, name, default)		\
-	x->name = default;
-#undef __str_field_def
-#define __str_field_def(attr_nr, attr_flag, name, maxlen)		\
-	memset(x->name, 0, sizeof(x->name));				\
-	x->name ## _len = 0;
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-static void set_ ## s_name ## _defaults(struct s_name *x) __attribute__((unused)); \
-static void set_ ## s_name ## _defaults(struct s_name *x) {	\
-s_fields								\
-}
-
-#include GENL_MAGIC_INCLUDE_FILE
-
-#endif /* __KERNEL__ */
-
-/* }}}1 */
-#endif /* GENL_MAGIC_FUNC_H */
diff --git a/include/linux/genl_magic_struct.h b/include/linux/genl_magic_struct.h
deleted file mode 100644
index 2200cedd160aa..0000000000000
--- a/include/linux/genl_magic_struct.h
+++ /dev/null
@@ -1,272 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef GENL_MAGIC_STRUCT_H
-#define GENL_MAGIC_STRUCT_H
-
-#ifndef GENL_MAGIC_FAMILY
-# error "you need to define GENL_MAGIC_FAMILY before inclusion"
-#endif
-
-#ifndef GENL_MAGIC_VERSION
-# error "you need to define GENL_MAGIC_VERSION before inclusion"
-#endif
-
-#ifndef GENL_MAGIC_INCLUDE_FILE
-# error "you need to define GENL_MAGIC_INCLUDE_FILE before inclusion"
-#endif
-
-#include <linux/args.h>
-#include <linux/types.h>
-#include <net/genetlink.h>
-
-extern int CONCATENATE(GENL_MAGIC_FAMILY, _genl_register)(void);
-extern void CONCATENATE(GENL_MAGIC_FAMILY, _genl_unregister)(void);
-
-/*
- * Extension of genl attribute validation policies			{{{2
- */
-
-/*
- * Flags specific to drbd and not visible at the netlink layer, used in
- * <struct>_from_attrs and <struct>_to_skb:
- *
- * @DRBD_F_REQUIRED: Attribute is required; a request without this attribute is
- * invalid.
- *
- * @DRBD_F_SENSITIVE: Attribute includes sensitive information and must not be
- * included in unpriviledged get requests or broadcasts.
- *
- * @DRBD_F_INVARIANT: Attribute is set when an object is initially created, but
- * cannot subsequently be changed.
- */
-#define DRBD_F_REQUIRED (1 << 0)
-#define DRBD_F_SENSITIVE (1 << 1)
-#define DRBD_F_INVARIANT (1 << 2)
-
-
-/*									}}}1
- * MAGIC
- * multi-include macro expansion magic starts here
- */
-
-/* MAGIC helpers							{{{2 */
-
-static inline int nla_put_u64_0pad(struct sk_buff *skb, int attrtype, u64 value)
-{
-	return nla_put_64bit(skb, attrtype, sizeof(u64), &value, 0);
-}
-
-/* possible field types */
-#define __flg_field(attr_nr, attr_flag, name) \
-	__field(attr_nr, attr_flag, name, NLA_U8, char, \
-			nla_get_u8, nla_put_u8, false)
-#define __u8_field(attr_nr, attr_flag, name)	\
-	__field(attr_nr, attr_flag, name, NLA_U8, unsigned char, \
-			nla_get_u8, nla_put_u8, false)
-#define __u16_field(attr_nr, attr_flag, name)	\
-	__field(attr_nr, attr_flag, name, NLA_U16, __u16, \
-			nla_get_u16, nla_put_u16, false)
-#define __u32_field(attr_nr, attr_flag, name)	\
-	__field(attr_nr, attr_flag, name, NLA_U32, __u32, \
-			nla_get_u32, nla_put_u32, false)
-#define __s32_field(attr_nr, attr_flag, name)	\
-	__field(attr_nr, attr_flag, name, NLA_U32, __s32, \
-			nla_get_u32, nla_put_u32, true)
-#define __u64_field(attr_nr, attr_flag, name)	\
-	__field(attr_nr, attr_flag, name, NLA_U64, __u64, \
-			nla_get_u64, nla_put_u64_0pad, false)
-#define __str_field(attr_nr, attr_flag, name, maxlen) \
-	__array(attr_nr, attr_flag, name, NLA_NUL_STRING, char, maxlen, \
-			nla_strscpy, nla_put, false)
-#define __bin_field(attr_nr, attr_flag, name, maxlen) \
-	__array(attr_nr, attr_flag, name, NLA_BINARY, char, maxlen, \
-			nla_memcpy, nla_put, false)
-
-/* fields with default values */
-#define __flg_field_def(attr_nr, attr_flag, name, default) \
-	__flg_field(attr_nr, attr_flag, name)
-#define __u32_field_def(attr_nr, attr_flag, name, default) \
-	__u32_field(attr_nr, attr_flag, name)
-#define __s32_field_def(attr_nr, attr_flag, name, default) \
-	__s32_field(attr_nr, attr_flag, name)
-#define __str_field_def(attr_nr, attr_flag, name, maxlen) \
-	__str_field(attr_nr, attr_flag, name, maxlen)
-
-#define GENL_op_init(args...)	args
-#define GENL_doit(handler)		\
-	.doit = handler,		\
-	.flags = GENL_ADMIN_PERM,
-#define GENL_dumpit(handler)		\
-	.dumpit = handler,		\
-	.flags = GENL_ADMIN_PERM,
-
-/*									}}}1
- * Magic: define the enum symbols for genl_ops
- * Magic: define the enum symbols for top level attributes
- * Magic: define the enum symbols for nested attributes
- *									{{{2
- */
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)
-
-#undef GENL_mc_group
-#define GENL_mc_group(group)
-
-#undef GENL_notification
-#define GENL_notification(op_name, op_num, mcast_group, tla_list)	\
-	op_name = op_num,
-
-#undef GENL_op
-#define GENL_op(op_name, op_num, handler, tla_list)			\
-	op_name = op_num,
-
-enum {
-#include GENL_MAGIC_INCLUDE_FILE
-};
-
-#undef GENL_notification
-#define GENL_notification(op_name, op_num, mcast_group, tla_list)
-
-#undef GENL_op
-#define GENL_op(op_name, op_num, handler, attr_list)
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields) \
-		tag_name = tag_number,
-
-enum {
-#include GENL_MAGIC_INCLUDE_FILE
-};
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)	\
-enum {								\
-	s_fields						\
-};
-
-#undef __field
-#define __field(attr_nr, attr_flag, name, nla_type, type,	\
-		__get, __put, __is_signed)			\
-	T_ ## name = (__u16)(attr_nr),
-
-#undef __array
-#define __array(attr_nr, attr_flag, name, nla_type, type,	\
-		maxlen, __get, __put, __is_signed)		\
-	T_ ## name = (__u16)(attr_nr),
-
-#include GENL_MAGIC_INCLUDE_FILE
-
-/*									}}}1
- * Magic: compile time assert unique numbers for operations
- * Magic: -"- unique numbers for top level attributes
- * Magic: -"- unique numbers for nested attributes
- *									{{{2
- */
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)
-
-#undef GENL_op
-#define GENL_op(op_name, op_num, handler, attr_list)	\
-	case op_name:
-
-#undef GENL_notification
-#define GENL_notification(op_name, op_num, mcast_group, tla_list)	\
-	case op_name:
-
-static inline void ct_assert_unique_operations(void)
-{
-	switch (0) {
-#include GENL_MAGIC_INCLUDE_FILE
-	case 0:
-		;
-	}
-}
-
-#undef GENL_op
-#define GENL_op(op_name, op_num, handler, attr_list)
-
-#undef GENL_notification
-#define GENL_notification(op_name, op_num, mcast_group, tla_list)
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-		case tag_number:
-
-static inline void ct_assert_unique_top_level_attributes(void)
-{
-	switch (0) {
-#include GENL_MAGIC_INCLUDE_FILE
-	case 0:
-		;
-	}
-}
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-static inline void ct_assert_unique_ ## s_name ## _attributes(void)	\
-{									\
-	switch (0) {							\
-	s_fields							\
-	case 0:								\
-			;						\
-	}								\
-}
-
-#undef __field
-#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put,	\
-		__is_signed)						\
-	case attr_nr:
-
-#undef __array
-#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen,	\
-		__get, __put, __is_signed)				\
-	case attr_nr:
-
-#include GENL_MAGIC_INCLUDE_FILE
-
-/*									}}}1
- * Magic: declare structs
- * struct <name> {
- *	fields
- * };
- *									{{{2
- */
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-struct s_name { s_fields };
-
-#undef __field
-#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put,	\
-		__is_signed)						\
-	type name;
-
-#undef __array
-#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen,	\
-		__get, __put, __is_signed)				\
-	type name[maxlen];	\
-	__u32 name ## _len;
-
-#include GENL_MAGIC_INCLUDE_FILE
-
-#undef GENL_struct
-#define GENL_struct(tag_name, tag_number, s_name, s_fields)		\
-enum {									\
-	s_fields							\
-};
-
-#undef __field
-#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put,	\
-		is_signed)						\
-	F_ ## name ## _IS_SIGNED = is_signed,
-
-#undef __array
-#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen,	\
-		__get, __put, is_signed)				\
-	F_ ## name ## _IS_SIGNED = is_signed,
-
-#include GENL_MAGIC_INCLUDE_FILE
-
-/* }}}1 */
-#endif /* GENL_MAGIC_STRUCT_H */
diff --git a/include/linux/gfp.h b/include/linux/gfp.h
index 51ef13ed756eb..cdf95a9f0b87c 100644
--- a/include/linux/gfp.h
+++ b/include/linux/gfp.h
@@ -239,6 +239,8 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
 				struct page **page_array);
 #define __alloc_pages_bulk(...)			alloc_hooks(alloc_pages_bulk_noprof(__VA_ARGS__))
 
+void free_pages_bulk(struct page **page_array, unsigned long nr_pages);
+
 unsigned long alloc_pages_bulk_mempolicy_noprof(gfp_t gfp,
 				unsigned long nr_pages,
 				struct page **page_array);
@@ -467,6 +469,8 @@ void free_contig_frozen_range(unsigned long pfn, unsigned long nr_pages);
 void free_contig_range(unsigned long pfn, unsigned long nr_pages);
 #endif
 
+void __free_contig_range(unsigned long pfn, unsigned long nr_pages);
+
 DEFINE_FREE(free_page, void *, free_page((unsigned long)_T))
 
 #endif /* __LINUX_GFP_H */
diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h
index 6c75df30a281d..54ca0c88bab6e 100644
--- a/include/linux/gfp_types.h
+++ b/include/linux/gfp_types.h
@@ -273,17 +273,17 @@ enum {
  *
  * %__GFP_ZERO returns a zeroed page on success.
  *
- * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself
- * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that
- * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting
- * memory tags at the same time as zeroing memory has minimal additional
- * performance impact.
+ * %__GFP_ZEROTAGS zeroes memory tags at allocation time. Setting memory tags at
+ * the same time as zeroing memory (e.g., with __GFP_ZERO) has minimal
+ * additional performance impact. However, __GFP_ZEROTAGS also zeroes the tags
+ * even if memory is not getting zeroed at allocation time (e.g.,
+ * with init_on_free).
  *
  * %__GFP_SKIP_KASAN makes KASAN skip unpoisoning on page allocation.
  * Used for userspace and vmalloc pages; the latter are unpoisoned by
- * kasan_unpoison_vmalloc instead. For userspace pages, results in
- * poisoning being skipped as well, see should_skip_kasan_poison for
- * details. Only effective in HW_TAGS mode.
+ * kasan_unpoison_vmalloc instead. If passed to vmalloc, kasan_unpoison_vmalloc
+ * is skipped too. For userspace pages, results in poisoning being skipped as
+ * well, see should_skip_kasan_poison for details. Only effective in HW_TAGS mode.
  */
 #define __GFP_NOWARN	((__force gfp_t)___GFP_NOWARN)
 #define __GFP_COMP	((__force gfp_t)___GFP_COMP)
diff --git a/include/linux/gpio.h b/include/linux/gpio.h
index 8f85ddb264295..b0d4942a65de2 100644
--- a/include/linux/gpio.h
+++ b/include/linux/gpio.h
@@ -2,13 +2,11 @@
 /*
  * NOTE: This header *must not* be included.
  *
- * This is the LEGACY GPIO bulk include file, including legacy APIs. It is
- * used for GPIO drivers still referencing the global GPIO numberspace,
- * and should not be included in new code.
- *
  * If you're implementing a GPIO driver, only include <linux/gpio/driver.h>
  * If you're implementing a GPIO consumer, only include <linux/gpio/consumer.h>
+ * If you're using the legacy interfaces, include <linux/gpio/legacy.h>
  */
+
 #ifndef __LINUX_GPIO_H
 #define __LINUX_GPIO_H
 
@@ -18,159 +16,7 @@
 #endif
 
 #ifdef CONFIG_GPIOLIB_LEGACY
-
-struct device;
-
-/* make these flag values available regardless of GPIO kconfig options */
-#define GPIOF_IN		((1 << 0))
-#define GPIOF_OUT_INIT_LOW	((0 << 0) | (0 << 1))
-#define GPIOF_OUT_INIT_HIGH	((0 << 0) | (1 << 1))
-
-#ifdef CONFIG_GPIOLIB
-/*
- * "valid" GPIO numbers are nonnegative and may be passed to
- * setup routines like gpio_request().  Only some valid numbers
- * can successfully be requested and used.
- *
- * Invalid GPIO numbers are useful for indicating no-such-GPIO in
- * platform data and other tables.
- */
-static inline bool gpio_is_valid(int number)
-{
-	/* only non-negative numbers are valid */
-	return number >= 0;
-}
-
-/*
- * Platforms may implement their GPIO interface with library code,
- * at a small performance cost for non-inlined operations and some
- * extra memory (for code and for per-GPIO table entries).
- */
-
-/* Always use the library code for GPIO management calls,
- * or when sleeping may be involved.
- */
-int gpio_request(unsigned gpio, const char *label);
-void gpio_free(unsigned gpio);
-
-static inline int gpio_direction_input(unsigned gpio)
-{
-	return gpiod_direction_input(gpio_to_desc(gpio));
-}
-static inline int gpio_direction_output(unsigned gpio, int value)
-{
-	return gpiod_direction_output_raw(gpio_to_desc(gpio), value);
-}
-
-static inline int gpio_get_value_cansleep(unsigned gpio)
-{
-	return gpiod_get_raw_value_cansleep(gpio_to_desc(gpio));
-}
-static inline void gpio_set_value_cansleep(unsigned gpio, int value)
-{
-	gpiod_set_raw_value_cansleep(gpio_to_desc(gpio), value);
-}
-
-static inline int gpio_get_value(unsigned gpio)
-{
-	return gpiod_get_raw_value(gpio_to_desc(gpio));
-}
-static inline void gpio_set_value(unsigned gpio, int value)
-{
-	gpiod_set_raw_value(gpio_to_desc(gpio), value);
-}
-
-static inline int gpio_to_irq(unsigned gpio)
-{
-	return gpiod_to_irq(gpio_to_desc(gpio));
-}
-
-int gpio_request_one(unsigned gpio, unsigned long flags, const char *label);
-
-int devm_gpio_request_one(struct device *dev, unsigned gpio,
-			  unsigned long flags, const char *label);
-
-#else /* ! CONFIG_GPIOLIB */
-
-#include <linux/kernel.h>
-
-#include <asm/bug.h>
-#include <asm/errno.h>
-
-static inline bool gpio_is_valid(int number)
-{
-	return false;
-}
-
-static inline int gpio_request(unsigned gpio, const char *label)
-{
-	return -ENOSYS;
-}
-
-static inline int gpio_request_one(unsigned gpio,
-					unsigned long flags, const char *label)
-{
-	return -ENOSYS;
-}
-
-static inline void gpio_free(unsigned gpio)
-{
-	might_sleep();
-
-	/* GPIO can never have been requested */
-	WARN_ON(1);
-}
-
-static inline int gpio_direction_input(unsigned gpio)
-{
-	return -ENOSYS;
-}
-
-static inline int gpio_direction_output(unsigned gpio, int value)
-{
-	return -ENOSYS;
-}
-
-static inline int gpio_get_value(unsigned gpio)
-{
-	/* GPIO can never have been requested or set as {in,out}put */
-	WARN_ON(1);
-	return 0;
-}
-
-static inline void gpio_set_value(unsigned gpio, int value)
-{
-	/* GPIO can never have been requested or set as output */
-	WARN_ON(1);
-}
-
-static inline int gpio_get_value_cansleep(unsigned gpio)
-{
-	/* GPIO can never have been requested or set as {in,out}put */
-	WARN_ON(1);
-	return 0;
-}
-
-static inline void gpio_set_value_cansleep(unsigned gpio, int value)
-{
-	/* GPIO can never have been requested or set as output */
-	WARN_ON(1);
-}
-
-static inline int gpio_to_irq(unsigned gpio)
-{
-	/* GPIO can never have been requested or set as input */
-	WARN_ON(1);
-	return -EINVAL;
-}
-
-static inline int devm_gpio_request_one(struct device *dev, unsigned gpio,
-					unsigned long flags, const char *label)
-{
-	WARN_ON(1);
-	return -EINVAL;
-}
-
-#endif /* ! CONFIG_GPIOLIB */
+#include <linux/gpio/legacy.h>
 #endif /* CONFIG_GPIOLIB_LEGACY */
+
 #endif /* __LINUX_GPIO_H */
diff --git a/include/linux/gpio/consumer.h b/include/linux/gpio/consumer.h
index 3efb5cb1e1d16..fceeefd5f8939 100644
--- a/include/linux/gpio/consumer.h
+++ b/include/linux/gpio/consumer.h
@@ -111,6 +111,7 @@ void devm_gpiod_unhinge(struct device *dev, struct gpio_desc *desc);
 void devm_gpiod_put_array(struct device *dev, struct gpio_descs *descs);
 
 int gpiod_get_direction(struct gpio_desc *desc);
+bool gpiod_is_single_ended(struct gpio_desc *desc);
 int gpiod_direction_input(struct gpio_desc *desc);
 int gpiod_direction_output(struct gpio_desc *desc, int value);
 int gpiod_direction_output_raw(struct gpio_desc *desc, int value);
@@ -337,6 +338,10 @@ static inline int gpiod_get_direction(const struct gpio_desc *desc)
 	WARN_ON(desc);
 	return -ENOSYS;
 }
+static inline bool gpiod_is_single_ended(struct gpio_desc *desc)
+{
+	return false;
+}
 static inline int gpiod_direction_input(struct gpio_desc *desc)
 {
 	/* GPIO can never have been requested */
@@ -598,6 +603,15 @@ static inline int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc,
 }
 #endif /* CONFIG_GPIOLIB && CONFIG_HTE */
 
+static inline
+struct gpio_desc *fwnode_gpiod_get(struct fwnode_handle *fwnode,
+				   const char *con_id,
+				   enum gpiod_flags flags,
+				   const char *label)
+{
+	return fwnode_gpiod_get_index(fwnode, con_id, 0, flags, label);
+}
+
 static inline
 struct gpio_desc *devm_fwnode_gpiod_get(struct device *dev,
 					struct fwnode_handle *fwnode,
diff --git a/include/linux/gpio/legacy.h b/include/linux/gpio/legacy.h
new file mode 100644
index 0000000000000..557ef635935ea
--- /dev/null
+++ b/include/linux/gpio/legacy.h
@@ -0,0 +1,173 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * This is the LEGACY GPIO include file, used only for legacy APIs.
+ *
+ * No new code should use this, but instead use the linux/gpio/consumer.h
+ * interfaces directly.
+ */
+
+#ifndef __LINUX_GPIO_LEGACY_H
+#define __LINUX_GPIO_LEGACY_H
+
+#include <linux/types.h>
+
+#ifdef CONFIG_GPIOLIB_LEGACY
+
+struct device;
+
+/* make these flag values available regardless of GPIO kconfig options */
+#define GPIOF_IN		((1 << 0))
+#define GPIOF_OUT_INIT_LOW	((0 << 0) | (0 << 1))
+#define GPIOF_OUT_INIT_HIGH	((0 << 0) | (1 << 1))
+
+#ifdef CONFIG_GPIOLIB
+
+#include <linux/gpio/consumer.h>
+
+/*
+ * "valid" GPIO numbers are nonnegative and may be passed to
+ * setup routines like gpio_request().  Only some valid numbers
+ * can successfully be requested and used.
+ *
+ * Invalid GPIO numbers are useful for indicating no-such-GPIO in
+ * platform data and other tables.
+ */
+static inline bool gpio_is_valid(int number)
+{
+	/* only non-negative numbers are valid */
+	return number >= 0;
+}
+
+/*
+ * Platforms may implement their GPIO interface with library code,
+ * at a small performance cost for non-inlined operations and some
+ * extra memory (for code and for per-GPIO table entries).
+ */
+
+/* Always use the library code for GPIO management calls,
+ * or when sleeping may be involved.
+ */
+int gpio_request(unsigned gpio, const char *label);
+void gpio_free(unsigned gpio);
+
+static inline int gpio_direction_input(unsigned gpio)
+{
+	return gpiod_direction_input(gpio_to_desc(gpio));
+}
+static inline int gpio_direction_output(unsigned gpio, int value)
+{
+	return gpiod_direction_output_raw(gpio_to_desc(gpio), value);
+}
+
+static inline int gpio_get_value_cansleep(unsigned gpio)
+{
+	return gpiod_get_raw_value_cansleep(gpio_to_desc(gpio));
+}
+static inline void gpio_set_value_cansleep(unsigned gpio, int value)
+{
+	gpiod_set_raw_value_cansleep(gpio_to_desc(gpio), value);
+}
+
+static inline int gpio_get_value(unsigned gpio)
+{
+	return gpiod_get_raw_value(gpio_to_desc(gpio));
+}
+static inline void gpio_set_value(unsigned gpio, int value)
+{
+	gpiod_set_raw_value(gpio_to_desc(gpio), value);
+}
+
+static inline int gpio_to_irq(unsigned gpio)
+{
+	return gpiod_to_irq(gpio_to_desc(gpio));
+}
+
+int gpio_request_one(unsigned gpio, unsigned long flags, const char *label);
+
+int devm_gpio_request_one(struct device *dev, unsigned gpio,
+			  unsigned long flags, const char *label);
+
+#else /* ! CONFIG_GPIOLIB */
+
+#include <linux/kernel.h>
+
+#include <asm/bug.h>
+#include <asm/errno.h>
+
+static inline bool gpio_is_valid(int number)
+{
+	return false;
+}
+
+static inline int gpio_request(unsigned gpio, const char *label)
+{
+	return -ENOSYS;
+}
+
+static inline int gpio_request_one(unsigned gpio,
+					unsigned long flags, const char *label)
+{
+	return -ENOSYS;
+}
+
+static inline void gpio_free(unsigned gpio)
+{
+	might_sleep();
+
+	/* GPIO can never have been requested */
+	WARN_ON(1);
+}
+
+static inline int gpio_direction_input(unsigned gpio)
+{
+	return -ENOSYS;
+}
+
+static inline int gpio_direction_output(unsigned gpio, int value)
+{
+	return -ENOSYS;
+}
+
+static inline int gpio_get_value(unsigned gpio)
+{
+	/* GPIO can never have been requested or set as {in,out}put */
+	WARN_ON(1);
+	return 0;
+}
+
+static inline void gpio_set_value(unsigned gpio, int value)
+{
+	/* GPIO can never have been requested or set as output */
+	WARN_ON(1);
+}
+
+static inline int gpio_get_value_cansleep(unsigned gpio)
+{
+	/* GPIO can never have been requested or set as {in,out}put */
+	WARN_ON(1);
+	return 0;
+}
+
+static inline void gpio_set_value_cansleep(unsigned gpio, int value)
+{
+	/* GPIO can never have been requested or set as output */
+	WARN_ON(1);
+}
+
+static inline int gpio_to_irq(unsigned gpio)
+{
+	/* GPIO can never have been requested or set as input */
+	WARN_ON(1);
+	return -EINVAL;
+}
+
+static inline int devm_gpio_request_one(struct device *dev, unsigned gpio,
+					unsigned long flags, const char *label)
+{
+	WARN_ON(1);
+	return -EINVAL;
+}
+
+#endif /* ! CONFIG_GPIOLIB */
+#endif /* CONFIG_GPIOLIB_LEGACY */
+#endif /* __LINUX_GPIO_LEGAGY_H */
diff --git a/include/linux/gpio/regmap.h b/include/linux/gpio/regmap.h
index 12d154732ca93..06255756710d0 100644
--- a/include/linux/gpio/regmap.h
+++ b/include/linux/gpio/regmap.h
@@ -38,6 +38,12 @@ struct regmap;
  *			offset to a register/bitmask pair. If not
  *			given the default gpio_regmap_simple_xlate()
  *			is used.
+ * @fixed_direction_mask:
+ *			(Optional) Bitmap representing the GPIO lines that
+ *			make use of the @fixed_direction_output list to
+ *			enforce direction of the GPIO. If this is NULL
+ *			and @fixed_direction_output is defined, ALL GPIOs
+ *			are assumed to be fixed direction (out or in).
  * @fixed_direction_output:
  *			(Optional) Bitmap representing the fixed direction of
  *			the GPIO lines. Useful when there are GPIO lines with a
@@ -89,6 +95,7 @@ struct gpio_regmap_config {
 	int reg_stride;
 	int ngpio_per_reg;
 	struct irq_domain *irq_domain;
+	unsigned long *fixed_direction_mask;
 	unsigned long *fixed_direction_output;
 
 #ifdef CONFIG_REGMAP_IRQ
diff --git a/include/linux/gpu_buddy.h b/include/linux/gpu_buddy.h
index 5fa917ba5450b..71941a0396485 100644
--- a/include/linux/gpu_buddy.h
+++ b/include/linux/gpu_buddy.h
@@ -154,6 +154,7 @@ struct gpu_buddy_block {
  * @avail: Total free space currently available for allocation in bytes.
  * @clear_avail: Free space available in the clear tree (zeroed memory) in bytes.
  *               This is a subset of @avail.
+ * @lock_dep_map: Annotates gpu_buddy API with a driver provided lock.
  */
 struct gpu_buddy {
 /* private: */
@@ -179,8 +180,48 @@ struct gpu_buddy {
 	u64 size;
 	u64 avail;
 	u64 clear_avail;
+#ifdef CONFIG_LOCKDEP
+	struct lockdep_map *lock_dep_map;
+#endif
 };
 
+#ifdef CONFIG_LOCKDEP
+/**
+ * gpu_buddy_driver_set_lock() - Set the lock protecting accesses to GPU BUDDY
+ * @mm: Pointer to GPU buddy structure.
+ * @lock: the lock used to protect the gpu buddy. The locking primitive
+ * must contain a dep_map field.
+ *
+ * Call this to annotate gpu_buddy APIs which access/modify gpu_buddy manager
+ */
+#define gpu_buddy_driver_set_lock(mm, lock) \
+	do { \
+		struct gpu_buddy *__mm = (mm); \
+		if (!WARN(__mm->lock_dep_map, "GPU BUDDY MM lock should be set only once.")) \
+			__mm->lock_dep_map = &(lock)->dep_map; \
+	} while (0)
+#else
+#define gpu_buddy_driver_set_lock(mm, lock) do { (void)(mm); (void)(lock); } while (0)
+#endif
+
+#ifdef CONFIG_LOCKDEP
+/**
+ * gpu_buddy_driver_lock_held() - Assert GPU BUDDY manager lock is held
+ * @mm: Pointer to the GPU BUDDY structure.
+ *
+ * Ensure driver lock is held.
+ */
+static inline void gpu_buddy_driver_lock_held(struct gpu_buddy *mm)
+{
+	if (mm->lock_dep_map)
+		lockdep_assert(lock_is_held_type(mm->lock_dep_map, 0));
+}
+#else
+static inline void gpu_buddy_driver_lock_held(struct gpu_buddy *mm)
+{
+}
+#endif
+
 static inline u64
 gpu_buddy_block_offset(const struct gpu_buddy_block *block)
 {
diff --git a/include/linux/highmem-internal.h b/include/linux/highmem-internal.h
index 0574c21ca45da..bb71e7dba4f7f 100644
--- a/include/linux/highmem-internal.h
+++ b/include/linux/highmem-internal.h
@@ -262,7 +262,7 @@ static inline bool is_kmap_addr(const void *x)
  * @__addr:       Virtual address to be unmapped
  *
  * Unmaps an address previously mapped by kmap_atomic() and re-enables
- * pagefaults. Depending on PREEMP_RT configuration, re-enables also
+ * pagefaults. Depending on PREEMPT_RT configuration, re-enables also
  * migration and preemption. Users should not count on these side effects.
  *
  * Mappings should be unmapped in the reverse order that they were mapped.
diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index af03db851a1d9..d7aac9de1c8a4 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -347,10 +347,11 @@ static inline void clear_highpage_kasan_tagged(struct page *page)
 
 #ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGES
 
-/* Return false to let people know we did not initialize the pages */
-static inline bool tag_clear_highpages(struct page *page, int numpages)
+/* Returns true if the caller has to initialize the pages */
+static inline bool tag_clear_highpages(struct page *page, int numpages,
+		bool clear_pages)
 {
-	return false;
+	return clear_pages;
 }
 
 #endif
diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h
index 9ced498fefaa2..6862dea0acc52 100644
--- a/include/linux/hrtimer.h
+++ b/include/linux/hrtimer.h
@@ -206,6 +206,9 @@ static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
 extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
 				   u64 range_ns, const enum hrtimer_mode mode);
 
+extern bool hrtimer_start_range_ns_user(struct hrtimer *timer, ktime_t tim,
+					u64 range_ns, const enum hrtimer_mode mode);
+
 /**
  * hrtimer_start - (re)start an hrtimer
  * @timer:	the timer to be added
@@ -223,17 +226,28 @@ static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
 extern int hrtimer_cancel(struct hrtimer *timer);
 extern int hrtimer_try_to_cancel(struct hrtimer *timer);
 
-static inline void hrtimer_start_expires(struct hrtimer *timer,
-					 enum hrtimer_mode mode)
+static inline void hrtimer_start_expires(struct hrtimer *timer, enum hrtimer_mode mode)
 {
-	u64 delta;
 	ktime_t soft, hard;
+	u64 delta;
+
 	soft = hrtimer_get_softexpires(timer);
 	hard = hrtimer_get_expires(timer);
 	delta = ktime_to_ns(ktime_sub(hard, soft));
 	hrtimer_start_range_ns(timer, soft, delta, mode);
 }
 
+static inline bool hrtimer_start_expires_user(struct hrtimer *timer, enum hrtimer_mode mode)
+{
+	ktime_t soft, hard;
+	u64 delta;
+
+	soft = hrtimer_get_softexpires(timer);
+	hard = hrtimer_get_expires(timer);
+	delta = ktime_to_ns(ktime_sub(hard, soft));
+	return hrtimer_start_range_ns_user(timer, soft, delta, mode);
+}
+
 void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
 				   enum hrtimer_mode mode);
 
@@ -254,8 +268,8 @@ static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
 	return __hrtimer_get_remaining(timer, false);
 }
 
-extern u64 hrtimer_get_next_event(void);
-extern u64 hrtimer_next_event_without(const struct hrtimer *exclude);
+extern ktime_t hrtimer_get_next_event(void);
+extern ktime_t hrtimer_next_event_without(const struct hrtimer *exclude);
 
 extern bool hrtimer_active(const struct hrtimer *timer);
 
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 2949e5acff351..127f9e1e7604c 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -128,6 +128,8 @@ enum mthp_stat_item {
 	MTHP_STAT_ANON_FAULT_ALLOC,
 	MTHP_STAT_ANON_FAULT_FALLBACK,
 	MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE,
+	MTHP_STAT_COLLAPSE_ALLOC,
+	MTHP_STAT_COLLAPSE_ALLOC_FAILED,
 	MTHP_STAT_ZSWPOUT,
 	MTHP_STAT_SWPIN,
 	MTHP_STAT_SWPIN_FALLBACK,
@@ -142,6 +144,9 @@ enum mthp_stat_item {
 	MTHP_STAT_SPLIT_DEFERRED,
 	MTHP_STAT_NR_ANON,
 	MTHP_STAT_NR_ANON_PARTIALLY_MAPPED,
+	MTHP_STAT_COLLAPSE_EXCEED_SWAP,
+	MTHP_STAT_COLLAPSE_EXCEED_NONE,
+	MTHP_STAT_COLLAPSE_EXCEED_SHARED,
 	__MTHP_STAT_COUNT
 };
 
@@ -389,27 +394,6 @@ static inline int split_huge_page_to_order(struct page *page, unsigned int new_o
 	return split_huge_page_to_list_to_order(page, NULL, new_order);
 }
 
-/**
- * try_folio_split_to_order() - try to split a @folio at @page to @new_order
- * using non uniform split.
- * @folio: folio to be split
- * @page: split to @new_order at the given page
- * @new_order: the target split order
- *
- * Try to split a @folio at @page using non uniform split to @new_order, if
- * non uniform split is not supported, fall back to uniform split. After-split
- * folios are put back to LRU list. Use min_order_for_split() to get the lower
- * bound of @new_order.
- *
- * Return: 0 - split is successful, otherwise split failed.
- */
-static inline int try_folio_split_to_order(struct folio *folio,
-		struct page *page, unsigned int new_order)
-{
-	if (folio_check_splittable(folio, new_order, SPLIT_TYPE_NON_UNIFORM))
-		return split_huge_page_to_order(&folio->page, new_order);
-	return folio_split(folio, new_order, page, NULL);
-}
 static inline int split_huge_page(struct page *page)
 {
 	return split_huge_page_to_list_to_order(page, NULL, 0);
@@ -642,8 +626,8 @@ static inline int split_folio_to_list(struct folio *folio, struct list_head *lis
 	return -EINVAL;
 }
 
-static inline int try_folio_split_to_order(struct folio *folio,
-		struct page *page, unsigned int new_order)
+static inline int folio_split(struct folio *folio, unsigned int new_order,
+		struct page *page, struct list_head *list)
 {
 	VM_WARN_ON_ONCE_FOLIO(1, folio);
 	return -EINVAL;
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 93418625d3c5f..5957bc25efa85 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -148,7 +148,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
 			     struct folio **foliop);
 #endif /* CONFIG_USERFAULTFD */
 long hugetlb_reserve_pages(struct inode *inode, long from, long to,
-			   struct vm_area_desc *desc, vma_flags_t vma_flags);
+			   struct vm_area_struct *vma, vma_flags_t vma_flags);
 long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
 						long freed);
 bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list);
@@ -276,7 +276,6 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 void hugetlb_unshare_all_pmds(struct vm_area_struct *vma);
 void fixup_hugetlb_reservations(struct vm_area_struct *vma);
 void hugetlb_split(struct vm_area_struct *vma, unsigned long addr);
-int hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
 
 unsigned int arch_hugetlb_cma_order(void);
 
@@ -469,11 +468,6 @@ static inline void fixup_hugetlb_reservations(struct vm_area_struct *vma)
 
 static inline void hugetlb_split(struct vm_area_struct *vma, unsigned long addr) {}
 
-static inline int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
-{
-	return 0;
-}
-
 #endif /* !CONFIG_HUGETLB_PAGE */
 
 #ifndef pgd_write
diff --git a/include/linux/hugetlb_inline.h b/include/linux/hugetlb_inline.h
index 565b473fd1353..5c29cd3223a1e 100644
--- a/include/linux/hugetlb_inline.h
+++ b/include/linux/hugetlb_inline.h
@@ -6,23 +6,13 @@
 
 #ifdef CONFIG_HUGETLB_PAGE
 
-static inline bool is_vm_hugetlb_flags(vm_flags_t vm_flags)
-{
-	return !!(vm_flags & VM_HUGETLB);
-}
-
 static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
 {
-	return vma_flags_test_any(flags, VMA_HUGETLB_BIT);
+	return vma_flags_test(flags, VMA_HUGETLB_BIT);
 }
 
 #else
 
-static inline bool is_vm_hugetlb_flags(vm_flags_t vm_flags)
-{
-	return false;
-}
-
 static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
 {
 	return false;
@@ -32,7 +22,7 @@ static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags)
 
 static inline bool is_vm_hugetlb_page(const struct vm_area_struct *vma)
 {
-	return is_vm_hugetlb_flags(vma->vm_flags);
+	return is_vma_hugetlb_flags(&vma->flags);
 }
 
 #endif
diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h
index 301a83afbd663..78ea83b17fc43 100644
--- a/include/linux/hwmon.h
+++ b/include/linux/hwmon.h
@@ -13,6 +13,7 @@
 #define _HWMON_H_
 
 #include <linux/bitops.h>
+#include <linux/cleanup.h>
 
 struct device;
 struct attribute_group;
@@ -477,7 +478,8 @@ hwmon_device_register_with_info(struct device *dev,
 				const struct attribute_group **extra_groups);
 struct device *
 hwmon_device_register_for_thermal(struct device *dev, const char *name,
-				  void *drvdata);
+				  void *drvdata,
+				  const struct attribute_group **extra_groups);
 struct device *
 devm_hwmon_device_register_with_info(struct device *dev,
 				const char *name, void *drvdata,
@@ -495,6 +497,8 @@ char *devm_hwmon_sanitize_name(struct device *dev, const char *name);
 void hwmon_lock(struct device *dev);
 void hwmon_unlock(struct device *dev);
 
+DEFINE_GUARD(hwmon_lock, struct device *, hwmon_lock(_T), hwmon_unlock(_T))
+
 /**
  * hwmon_is_bad_char - Is the char invalid in a hwmon name
  * @ch: the char to be considered
diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h
index 964f1be8150c5..41a3d82f0722b 100644
--- a/include/linux/hyperv.h
+++ b/include/linux/hyperv.h
@@ -1336,6 +1336,9 @@ int vmbus_allocate_mmio(struct resource **new, struct hv_device *device_obj,
 			bool fb_overlap_ok);
 void vmbus_free_mmio(resource_size_t start, resource_size_t size);
 
+void vmbus_initiate_unload(bool crash);
+void vmbus_set_skip_unload(bool skip);
+
 /*
  * GUID definitions of various offer types - services offered to the guest.
  */
diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h
index a97b1d01f3acf..87d92fb86fab1 100644
--- a/include/linux/ieee80211-eht.h
+++ b/include/linux/ieee80211-eht.h
@@ -9,7 +9,7 @@
  * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net>
  * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH
  * Copyright (c) 2016 - 2017 Intel Deutschland GmbH
- * Copyright (c) 2018 - 2025 Intel Corporation
+ * Copyright (c) 2018 - 2026 Intel Corporation
  */
 
 #ifndef LINUX_IEEE80211_EHT_H
@@ -485,26 +485,28 @@ struct ieee80211_multi_link_elem {
 #define IEEE80211_MED_SYNC_DELAY_DEFAULT		0x10ac
 
 #define IEEE80211_EML_CAP_EMLSR_SUPP			0x0001
-#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY		0x000e
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_0US		0
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_32US		1
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_64US		2
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_128US		3
-#define  IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US		4
-#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY	0x0070
+#define IEEE80211_EML_CAP_EML_PADDING_DELAY		0x000e
+/* Described Tables 9-417i & 9-417k in 802.11be-2024, which have the same values */
+#define  IEEE80211_EML_CAP_EML_PADDING_DELAY_0US		0
+#define  IEEE80211_EML_CAP_EML_PADDING_DELAY_32US		1
+#define  IEEE80211_EML_CAP_EML_PADDING_DELAY_64US		2
+#define  IEEE80211_EML_CAP_EML_PADDING_DELAY_128US		3
+#define  IEEE80211_EML_CAP_EML_PADDING_DELAY_256US		4
+#define IEEE80211_EML_CAP_EML_TRANSITION_DELAY	0x0070
+/* Described in Table 9-417j in 802.11be-2024 */
 #define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_0US		0
 #define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_16US		1
 #define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_32US		2
 #define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US		3
 #define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_128US		4
 #define  IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US		5
+/* Described in Table 9-417l in 802.11be-2024 */
+#define  IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_0US		0
+#define  IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_32US		1
+#define  IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_64US		2
+#define  IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_128US		3
+#define  IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_256US		4
 #define IEEE80211_EML_CAP_EMLMR_SUPPORT			0x0080
-#define IEEE80211_EML_CAP_EMLMR_DELAY			0x0700
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_0US			0
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_32US			1
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_64US			2
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_128US			3
-#define  IEEE80211_EML_CAP_EMLMR_DELAY_256US			4
 #define IEEE80211_EML_CAP_TRANSITION_TIMEOUT		0x7800
 #define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_0			0
 #define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128US		1
@@ -517,7 +519,6 @@ struct ieee80211_multi_link_elem {
 #define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_16TU		8
 #define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_32TU		9
 #define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_64TU		10
-#define  IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU		11
 
 #define IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS		0x000f
 #define IEEE80211_MLD_CAP_OP_SRS_SUPPORT		0x0010
@@ -749,11 +750,13 @@ static inline u16 ieee80211_mle_get_mld_capa_op(const u8 *data)
 }
 
 /* Defined in Figure 9-1074t in P802.11be_D7.0 */
-#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_PARAM_UPDATE           0x0001
-#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_RECO_MAX_LINKS_MASK    0x001e
-#define IEEE80211_EHT_ML_EXT_MLD_CAPA_NSTR_UPDATE               0x0020
-#define IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK     0x0040
-#define IEEE80211_EHT_ML_EXT_MLD_CAPA_BTM_MLD_RECO_MULTI_AP     0x0080
+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_PARAM_UPDATE		0x0001
+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_RECO_MAX_LINKS_MASK	0x001e
+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_NSTR_UPDATE		0x0020
+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK	0x0040
+#define IEEE80211_EHT_ML_EXT_MLD_CAPA_BTM_MLD_RECO_MULTI_AP	0x0080
+/* defined by UHR Draft P802.11bn_D1.3 Figure 9-1147 */
+#define IEEE80211_UHR_ML_EXT_MLD_CAPA_ML_PM			0x0100
 
 /**
  * ieee80211_mle_get_ext_mld_capa_op - returns the extended MLD capabilities
@@ -1121,14 +1124,20 @@ static inline bool ieee80211_tid_to_link_map_size_ok(const u8 *data, size_t len)
 
 static inline u32 ieee80211_emlsr_pad_delay_in_us(u16 eml_cap)
 {
+	u32 emlsr_supp =
+		u16_get_bits(eml_cap, IEEE80211_EML_CAP_EMLSR_SUPP);
+
+	if (!emlsr_supp)
+		return 0;
+
 	/* IEEE Std 802.11be-2024 Table 9-417i—Encoding of the EMLSR
 	 * Padding Delay subfield.
 	 */
 	u32 pad_delay = u16_get_bits(eml_cap,
-				     IEEE80211_EML_CAP_EMLSR_PADDING_DELAY);
+				     IEEE80211_EML_CAP_EML_PADDING_DELAY);
 
 	if (!pad_delay ||
-	    pad_delay > IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US)
+	    pad_delay > IEEE80211_EML_CAP_EML_PADDING_DELAY_256US)
 		return 0;
 
 	return 32 * (1 << (pad_delay - 1));
@@ -1145,12 +1154,18 @@ static inline u32 ieee80211_emlsr_pad_delay_in_us(u16 eml_cap)
 
 static inline u32 ieee80211_emlsr_trans_delay_in_us(u16 eml_cap)
 {
+	u32 emlsr_supp =
+		u16_get_bits(eml_cap, IEEE80211_EML_CAP_EMLSR_SUPP);
+
+	if (!emlsr_supp)
+		return 0;
+
 	/* IEEE Std 802.11be-2024 Table 9-417j—Encoding of the EMLSR
 	 * Transition Delay subfield.
 	 */
 	u32 trans_delay =
 		u16_get_bits(eml_cap,
-			     IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY);
+			     IEEE80211_EML_CAP_EML_TRANSITION_DELAY);
 
 	/* invalid values also just use 0 */
 	if (!trans_delay ||
@@ -1161,11 +1176,73 @@ static inline u32 ieee80211_emlsr_trans_delay_in_us(u16 eml_cap)
 }
 
 /**
- * ieee80211_eml_trans_timeout_in_us - Fetch the EMLSR Transition
+ * ieee80211_emlmr_pad_delay_in_us - Fetch the EMLMR Padding delay
+ *	in microseconds
+ * @eml_cap: EML capabilities field value from common info field of
+ *	the Multi-link element
+ * Return: the EMLMR Padding delay (in microseconds) encoded in the
+ *	EML Capabilities field
+ */
+
+static inline u32 ieee80211_emlmr_pad_delay_in_us(u16 eml_cap)
+{
+	u32 emlmr_supp =
+		u16_get_bits(eml_cap, IEEE80211_EML_CAP_EMLMR_SUPPORT);
+
+	if (!emlmr_supp)
+		return 0;
+
+	/* IEEE Std 802.11be-2024 Table 9-417k—Encoding of the EMLMR
+	 * Padding Delay subfield.
+	 */
+	u32 pad_delay = u16_get_bits(eml_cap,
+				     IEEE80211_EML_CAP_EML_PADDING_DELAY);
+
+	if (!pad_delay ||
+	    pad_delay > IEEE80211_EML_CAP_EML_PADDING_DELAY_256US)
+		return 0;
+
+	return 32 * (1 << (pad_delay - 1));
+}
+
+/**
+ * ieee80211_emlmr_trans_delay_in_us - Fetch the EMLMR Transition
+ *	delay in microseconds
+ * @eml_cap: EML capabilities field value from common info field of
+ *	the Multi-link element
+ * Return: the EMLMR Transition delay (in microseconds) encoded in the
+ *	EML Capabilities field
+ */
+
+static inline u32 ieee80211_emlmr_trans_delay_in_us(u16 eml_cap)
+{
+	u32 emlmr_supp =
+		u16_get_bits(eml_cap, IEEE80211_EML_CAP_EMLMR_SUPPORT);
+
+	if (!emlmr_supp)
+		return 0;
+
+	/* IEEE Std 802.11be-2024 Table 9-417l—Encoding of the EMLMR
+	 * Transition Delay subfield.
+	 */
+	u32 trans_delay =
+		u16_get_bits(eml_cap,
+			     IEEE80211_EML_CAP_EML_TRANSITION_DELAY);
+
+	/* invalid values also just use 0 */
+	if (!trans_delay ||
+	    trans_delay > IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_256US)
+		return 0;
+
+	return 32 * (1 << (trans_delay - 1));
+}
+
+/**
+ * ieee80211_eml_trans_timeout_in_us - Fetch the EML Transition
  *	timeout value in microseconds
  * @eml_cap: EML capabilities field value from common info field of
  *	the Multi-link element
- * Return: the EMLSR Transition timeout (in microseconds) encoded in
+ * Return: the EML Transition timeout (in microseconds) encoded in
  *	the EML Capabilities field
  */
 
@@ -1178,7 +1255,7 @@ static inline u32 ieee80211_eml_trans_timeout_in_us(u16 eml_cap)
 				  IEEE80211_EML_CAP_TRANSITION_TIMEOUT);
 
 	/* invalid values also just use 0 */
-	if (!timeout || timeout > IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU)
+	if (!timeout || timeout > IEEE80211_EML_CAP_TRANSITION_TIMEOUT_64TU)
 		return 0;
 
 	return 128 * (1 << (timeout - 1));
diff --git a/include/linux/ieee80211-uhr.h b/include/linux/ieee80211-uhr.h
index d199f3ebdba0a..f4f4bd8256dff 100644
--- a/include/linux/ieee80211-uhr.h
+++ b/include/linux/ieee80211-uhr.h
@@ -372,6 +372,12 @@ ieee80211_uhr_npca_dis_subch_bitmap(const struct ieee80211_uhr_operation *oper)
 #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES	0x08
 #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES	0x10
 
+struct ieee80211_uhr_cap_dbe {
+	u8 cap;
+	/* present 0, 1 or 2 times depending on _PRES bits */
+	struct ieee80211_eht_mcs_nss_supp_bw eht_mcs_map[];
+} __packed;
+
 /**
  * enum ieee80211_uhr_dbe_max_supported_bw - DBE Maximum Supported Bandwidth
  *
@@ -394,12 +400,6 @@ struct ieee80211_uhr_cap_mac {
 	u8 mac_cap[5];
 } __packed;
 
-struct ieee80211_uhr_cap {
-	struct ieee80211_uhr_cap_mac mac;
-	/* DBE, PHY capabilities */
-	u8 variable[];
-} __packed;
-
 #define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_LE80	0x01
 #define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_LE80	0x02
 #define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_160	0x04
@@ -413,11 +413,18 @@ struct ieee80211_uhr_cap_phy {
 	u8 cap;
 } __packed;
 
+struct ieee80211_uhr_cap {
+	struct ieee80211_uhr_cap_mac mac;
+	struct ieee80211_uhr_cap_phy phy;
+	/* optional DBE capabilities */
+	u8 variable[];
+} __packed;
+
 static inline bool ieee80211_uhr_capa_size_ok(const u8 *data, u8 len,
 					      bool from_ap)
 {
 	const struct ieee80211_uhr_cap *cap = (const void *)data;
-	size_t needed = sizeof(*cap) + sizeof(struct ieee80211_uhr_cap_phy);
+	size_t needed = sizeof(*cap);
 
 	if (len < needed)
 		return false;
@@ -427,44 +434,24 @@ static inline bool ieee80211_uhr_capa_size_ok(const u8 *data, u8 len,
 	 * in the UHR MAC Capabilities Information field.
 	 */
 	if (from_ap && cap->mac.mac_cap[1] & IEEE80211_UHR_MAC_CAP1_DBE_SUPP) {
-		u8 dbe;
+		const struct ieee80211_uhr_cap_dbe *dbe;
 
-		needed += 1;
+		needed += sizeof(struct ieee80211_uhr_cap_dbe);
 		if (len < needed)
 			return false;
 
-		dbe = cap->variable[0];
+		dbe = (const void *)cap->variable;
 
-		if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES)
-			needed += 3;
+		if (dbe->cap & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES)
+			needed += sizeof(dbe->eht_mcs_map[0]);
 
-		if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES)
-			needed += 3;
+		if (dbe->cap & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES)
+			needed += sizeof(dbe->eht_mcs_map[0]);
 	}
 
 	return len >= needed;
 }
 
-static inline const struct ieee80211_uhr_cap_phy *
-ieee80211_uhr_phy_cap(const struct ieee80211_uhr_cap *cap, bool from_ap)
-{
-	u8 offs = 0;
-
-	if (from_ap && cap->mac.mac_cap[1] & IEEE80211_UHR_MAC_CAP1_DBE_SUPP) {
-		u8 dbe = cap->variable[0];
-
-		offs += 1;
-
-		if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES)
-			offs += 3;
-
-		if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES)
-			offs += 3;
-	}
-
-	return (const void *)&cap->variable[offs];
-}
-
 #define IEEE80211_SMD_INFO_CAPA_DL_DATA_FWD		0x01
 #define IEEE80211_SMD_INFO_CAPA_MAX_NUM_PREP		0x0E
 #define IEEE80211_SMD_INFO_CAPA_TYPE			0x10
diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h
index 23f9df9be8372..11106589acc6f 100644
--- a/include/linux/ieee80211.h
+++ b/include/linux/ieee80211.h
@@ -2236,6 +2236,7 @@ struct ieee80211_multiple_bssid_configuration {
 #define WLAN_AKM_SUITE_WFA_DPP			SUITE(WLAN_OUI_WFA, 2)
 
 #define WLAN_MAX_KEY_LEN		32
+#define WLAN_MAX_SECURE_LTF_KEYSEED_LEN	48
 
 #define WLAN_PMK_NAME_LEN		16
 #define WLAN_PMKID_LEN			16
diff --git a/include/linux/if_bridge.h b/include/linux/if_bridge.h
index c5fe3b2a53e82..ec9ffea1e46ed 100644
--- a/include/linux/if_bridge.h
+++ b/include/linux/if_bridge.h
@@ -61,6 +61,7 @@ struct br_ip_list {
 #define BR_PORT_LOCKED		BIT(21)
 #define BR_PORT_MAB		BIT(22)
 #define BR_NEIGH_VLAN_SUPPRESS	BIT(23)
+#define BR_NEIGH_FORWARD_GRAT	BIT(24)
 
 #define BR_DEFAULT_AGEING_TIME	(300 * HZ)
 
diff --git a/include/linux/if_tun.h b/include/linux/if_tun.h
index 80166eb62f41c..5f3e206c7a737 100644
--- a/include/linux/if_tun.h
+++ b/include/linux/if_tun.h
@@ -22,6 +22,7 @@ struct tun_msg_ctl {
 #if defined(CONFIG_TUN) || defined(CONFIG_TUN_MODULE)
 struct socket *tun_get_socket(struct file *);
 struct ptr_ring *tun_get_tx_ring(struct file *file);
+void tun_wake_queue(struct file *file, int consumed);
 
 static inline bool tun_is_xdp_frame(void *ptr)
 {
@@ -55,6 +56,8 @@ static inline struct ptr_ring *tun_get_tx_ring(struct file *f)
 	return ERR_PTR(-EINVAL);
 }
 
+static inline void tun_wake_queue(struct file *f, int consumed) {}
+
 static inline bool tun_is_xdp_frame(void *ptr)
 {
 	return false;
diff --git a/include/linux/igmp.h b/include/linux/igmp.h
index 073b30a9b8508..3a2d35a9f3078 100644
--- a/include/linux/igmp.h
+++ b/include/linux/igmp.h
@@ -92,15 +92,166 @@ struct ip_mc_list {
 	struct rcu_head		rcu;
 };
 
+/* RFC3376, relevant sections:
+ *  - 4.1.1. Maximum Response Code
+ *  - 4.1.7. QQIC (Querier's Query Interval Code)
+ *
+ * For both MRC and QQIC, values >= 128 use the same floating-point
+ * encoding as follows:
+ *
+ *  0 1 2 3 4 5 6 7
+ * +-+-+-+-+-+-+-+-+
+ * |1| exp | mant  |
+ * +-+-+-+-+-+-+-+-+
+ */
+#define IGMPV3_FP_EXP(value)		(((value) >> 4) & 0x07)
+#define IGMPV3_FP_MAN(value)		((value) & 0x0f)
+
+/* IGMPv3 floating-point exponential field min threshold */
+#define IGMPV3_EXP_MIN_THRESHOLD	128
+/* IGMPv3 FP max threshold (mant = 0xF, exp = 7) -> 31744 */
+#define IGMPV3_EXP_MAX_THRESHOLD	31744
+
+/* V3 exponential field encoding */
+
+/* IGMPv3 MRC/QQIC 8-bit exponential field encode
+ *
+ * RFC3376, 4.1.1 & 4.1.7. defines only the decoding formula:
+ *     MRT/QQI = (mant | 0x10) << (exp + 3)
+ *
+ * but does NOT define the encoding procedure. To derive exponent:
+ *
+ * For any value of mantissa and exponent, the decoding formula
+ * indicates that the "hidden bit" (0x10) is shifted 4 bits left
+ * to sit above the 4-bit mantissa. The RFC again shifts this
+ * entire block left by (exp + 3) to reconstruct the value.
+ * So, 'hidden bit' is the MSB which is shifted by (4 + exp + 3).
+ *
+ * Total left shift of the 'hidden bit' = 4 + (exp + 3) = exp + 7.
+ * This is the MSB at the 0-based bit position: (exp + 7).
+ * Since fls() is 1-based, fls(value) - 1 = exp + 7.
+ *
+ * Therefore:
+ *     exp  = fls(value) - 8
+ *     mant = (value >> (exp + 3)) & 0x0F
+ *
+ * Final encoding formula:
+ *     0x80 | (exp << 4) | mant
+ *
+ * Example (value = 3200):
+ *  0               1
+ *  0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |0 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0| (value = 3200)
+ * |        ^-^-mant^ ^..(exp+3)..^| exp = 4, mant = 9
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *
+ * Encoded:
+ *   0x80 | (4 << 4) | 9 = 0xC9
+ */
+static inline u8 igmpv3_exp_field_encode(unsigned long value)
+{
+	u8 mc_exp, mc_man;
+
+	/* MRC/QQIC < 128 is literal */
+	if (value < IGMPV3_EXP_MIN_THRESHOLD)
+		return value;
+
+	/* Saturate at max representable (mant = 0xF, exp = 7) -> 31744 */
+	if (value >= IGMPV3_EXP_MAX_THRESHOLD)
+		return 0xFF;
+
+	mc_exp  = fls(value) - 8;
+	mc_man = (value >> (mc_exp + 3)) & 0x0F;
+
+	return 0x80 | (mc_exp << 4) | mc_man;
+}
+
+/* Calculate Maximum Response Code from Max Resp Time
+ *
+ * RFC3376, relevant sections:
+ *  - 4.1.1. Maximum Response Code
+ *  - 8.3. Query Response Interval
+ *
+ * MRC represents the encoded form of Max Resp Time (MRT); once
+ * decoded, the resulting value is in units of 0.1 seconds (100 ms).
+ */
+static inline u8 igmpv3_mrc(unsigned long mrt)
+{
+	return igmpv3_exp_field_encode(mrt);
+}
+
+/* Calculate Querier's Query Interval Code from Querier's Query Interval
+ *
+ * RFC3376, relevant sections:
+ *  - 4.1.7. QQIC (Querier's Query Interval Code)
+ *  - 8.2. Query Interval
+ *  - 8.12. Older Version Querier Present Timeout
+ *    (the [Query Interval] in the last Query received)
+ *
+ * QQIC represents the encoded form of Querier's Query Interval (QQI);
+ * once decoded, the resulting value is in units of seconds.
+ */
+static inline u8 igmpv3_qqic(unsigned long qi)
+{
+	return igmpv3_exp_field_encode(qi);
+}
+
 /* V3 exponential field decoding */
-#define IGMPV3_MASK(value, nb) ((nb)>=32 ? (value) : ((1<<(nb))-1) & (value))
-#define IGMPV3_EXP(thresh, nbmant, nbexp, value) \
-	((value) < (thresh) ? (value) : \
-        ((IGMPV3_MASK(value, nbmant) | (1<<(nbmant))) << \
-         (IGMPV3_MASK((value) >> (nbmant), nbexp) + (nbexp))))
-
-#define IGMPV3_QQIC(value) IGMPV3_EXP(0x80, 4, 3, value)
-#define IGMPV3_MRC(value) IGMPV3_EXP(0x80, 4, 3, value)
+
+/* IGMPv3 MRC/QQIC 8-bit exponential field decode
+ *
+ * RFC3376, 4.1.1 & 4.1.7. defines the decoding formula:
+ *      0 1 2 3 4 5 6 7
+ *     +-+-+-+-+-+-+-+-+
+ *     |1| exp | mant  |
+ *     +-+-+-+-+-+-+-+-+
+ * Max Resp Time = (mant | 0x10) << (exp + 3)
+ * QQI = (mant | 0x10) << (exp + 3)
+ */
+static inline unsigned long igmpv3_exp_field_decode(const u8 code)
+{
+	if (code < IGMPV3_EXP_MIN_THRESHOLD) {
+		return code;
+	} else {
+		unsigned long mc_man, mc_exp;
+
+		mc_exp = IGMPV3_FP_EXP(code);
+		mc_man = IGMPV3_FP_MAN(code);
+
+		return (mc_man | 0x10) << (mc_exp + 3);
+	}
+}
+
+/* Calculate Max Resp Time from Maximum Response Code
+ *
+ * RFC3376, relevant sections:
+ *  - 4.1.1. Maximum Response Code
+ *  - 8.3. Query Response Interval
+ *
+ * After decode, MRC represents the Maximum Response Time (MRT) in
+ * units of 0.1 seconds (100 ms).
+ */
+static inline unsigned long igmpv3_mrt(const struct igmpv3_query *ih3)
+{
+	return igmpv3_exp_field_decode(ih3->code);
+}
+
+/* Calculate Querier's Query Interval from Querier's Query Interval Code
+ *
+ * RFC3376, relevant sections:
+ *  - 4.1.7. QQIC (Querier's Query Interval Code)
+ *  - 8.2. Query Interval
+ *  - 8.12. Older Version Querier Present Timeout
+ *    (the [Query Interval] in the last Query received)
+ *
+ * After decode, QQIC represents the Querier's Query Interval in units
+ * of seconds.
+ */
+static inline unsigned long igmpv3_qqi(const struct igmpv3_query *ih3)
+{
+	return igmpv3_exp_field_decode(ih3->qqic);
+}
 
 static inline int ip_mc_may_pull(struct sk_buff *skb, unsigned int len)
 {
diff --git a/include/linux/iio/backend.h b/include/linux/iio/backend.h
index 4d15c2a9802c1..3f95ed1fdf9eb 100644
--- a/include/linux/iio/backend.h
+++ b/include/linux/iio/backend.h
@@ -261,6 +261,7 @@ int iio_backend_extend_chan_spec(struct iio_backend *back,
 bool iio_backend_has_caps(struct iio_backend *back, u32 caps);
 void *iio_backend_get_priv(const struct iio_backend *conv);
 struct iio_backend *devm_iio_backend_get(struct device *dev, const char *name);
+struct iio_backend *devm_iio_backend_get_by_index(struct device *dev, unsigned int index);
 struct iio_backend *devm_iio_backend_fwnode_get(struct device *dev,
 						const char *name,
 						struct fwnode_handle *fwnode);
diff --git a/include/linux/iio/buffer.h b/include/linux/iio/buffer.h
index d37f82678f716..745c98ef4e04c 100644
--- a/include/linux/iio/buffer.h
+++ b/include/linux/iio/buffer.h
@@ -34,8 +34,16 @@ static inline int iio_push_to_buffers_with_timestamp(struct iio_dev *indio_dev,
 	void *data, int64_t timestamp)
 {
 	if (ACCESS_PRIVATE(indio_dev, scan_timestamp)) {
-		size_t ts_offset = indio_dev->scan_bytes / sizeof(int64_t) - 1;
-		((int64_t *)data)[ts_offset] = timestamp;
+		size_t ts_offset = ACCESS_PRIVATE(indio_dev, scan_timestamp_offset);
+
+		/*
+		 * The size of indio_dev->scan_bytes is always aligned to the
+		 * largest scan element's alignment (see iio_compute_scan_bytes()).
+		 * So there may be padding after the timestamp. ts_offset contains
+		 * the offset in bytes that was already computed for correctly
+		 * aligning the timestamp.
+		 */
+		*(int64_t *)(data + ts_offset) = timestamp;
 	}
 
 	return iio_push_to_buffers(indio_dev, data);
@@ -71,7 +79,7 @@ static inline int iio_push_to_buffers_with_ts(struct iio_dev *indio_dev,
 
 int iio_push_to_buffers_with_ts_unaligned(struct iio_dev *indio_dev,
 					  const void *data, size_t data_sz,
-					  int64_t timestamp);
+					  s64 timestamp);
 
 bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev,
 				   const unsigned long *mask);
diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h
index 2c91b7659ce93..96b05c86c325f 100644
--- a/include/linux/iio/iio.h
+++ b/include/linux/iio/iio.h
@@ -176,9 +176,32 @@ struct iio_event_spec {
 	unsigned long mask_shared_by_all;
 };
 
+/**
+ * define IIO_SCAN_FORMAT_SIGNED_INT - signed integer data format
+ *
+ * &iio_scan_type.format value for signed integers (two's complement).
+ */
+#define IIO_SCAN_FORMAT_SIGNED_INT	's'
+
+/**
+ * define IIO_SCAN_FORMAT_UNSIGNED_INT - unsigned integer data format
+ *
+ * &iio_scan_type.format value for unsigned integers.
+ */
+#define IIO_SCAN_FORMAT_UNSIGNED_INT	'u'
+
+/**
+ * define IIO_SCAN_FORMAT_FLOAT - floating-point data format
+ *
+ * &iio_scan_type.format value for IEEE 754 floating-point numbers.
+ */
+#define IIO_SCAN_FORMAT_FLOAT		'f'
+
 /**
  * struct iio_scan_type - specification for channel data format in buffer
- * @sign:		's' or 'u' to specify signed or unsigned
+ * @sign:		Deprecated, use @format instead.
+ * @format:		Data format, can have any of the IIO_SCAN_FORMAT_*
+ *			values.
  * @realbits:		Number of valid bits of data
  * @storagebits:	Realbits + padding
  * @shift:		Shift right by this before masking out realbits.
@@ -189,7 +212,10 @@ struct iio_event_spec {
  * @endianness:		little or big endian
  */
 struct iio_scan_type {
-	char	sign;
+	union {
+		char sign;
+		char format;
+	};
 	u8	realbits;
 	u8	storagebits;
 	u8	shift;
@@ -584,6 +610,8 @@ struct iio_buffer_setup_ops {
  *			and owner
  * @buffer:		[DRIVER] any buffer present
  * @scan_bytes:		[INTERN] num bytes captured to be fed to buffer demux
+ * @scan_timestamp_offset: [INTERN] cache of the offset (in bytes) for the
+ *			   timestamp in the scan buffer
  * @available_scan_masks: [DRIVER] optional array of allowed bitmasks. Sort the
  *			   array in order of preference, the most preferred
  *			   masks first.
@@ -610,6 +638,7 @@ struct iio_dev {
 
 	struct iio_buffer		*buffer;
 	int				scan_bytes;
+	unsigned int			__private scan_timestamp_offset;
 
 	const unsigned long		*available_scan_masks;
 	unsigned int			__private masklength;
diff --git a/include/linux/init.h b/include/linux/init.h
index 40331923b9f4a..6326c61e2332d 100644
--- a/include/linux/init.h
+++ b/include/linux/init.h
@@ -47,7 +47,7 @@
 #define __initdata	__section(".init.data")
 #define __initconst	__section(".init.rodata")
 #define __exitdata	__section(".exit.data")
-#define __exit_call	__used __section(".exitcall.exit")
+#define __exit_call	__maybe_unused __section(".exitcall.exit")
 
 /*
  * modpost check for section mismatches during the kernel build.
diff --git a/include/linux/instruction_pointer.h b/include/linux/instruction_pointer.h
index aa0b3ffea9353..ea5bc756bd99d 100644
--- a/include/linux/instruction_pointer.h
+++ b/include/linux/instruction_pointer.h
@@ -8,6 +8,30 @@
 
 #ifndef _THIS_IP_
 #define _THIS_IP_  ({ __label__ __here; __here: (unsigned long)&&__here; })
+/*
+ * The current generic definition of _THIS_IP_ is considered broken by GCC [1]
+ * and Clang [2]. In particular, the address of a label is only expected to be
+ * used with a computed goto.
+ *
+ *   [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120071
+ *   [2] https://github.com/llvm/llvm-project/issues/138272
+ *
+ * Mark it as broken, so that appropriate fallback options can be implemented
+ * for architectures that do not define their own _THIS_IP_.
+ */
+#define HAS_BROKEN_THIS_IP
+#endif
+
+/*
+ * _CODE_LOCATION_ provides a unique identifier for the current code location.
+ * When _THIS_IP_ is broken (generic version), we fall back to a static marker
+ * which guarantees uniqueness and resolves to a constant address at link time,
+ * avoiding runtime overhead and compiler optimizations breaking it.
+ */
+#ifdef HAS_BROKEN_THIS_IP
+#define _CODE_LOCATION_ ({ static const char __here; (unsigned long)&__here; })
+#else
+#define _CODE_LOCATION_ _THIS_IP_
 #endif
 
 #endif /* _LINUX_INSTRUCTION_POINTER_H */
diff --git a/include/linux/intel_vsec.h b/include/linux/intel_vsec.h
index 1fe5665a9d02a..07ea563f524ee 100644
--- a/include/linux/intel_vsec.h
+++ b/include/linux/intel_vsec.h
@@ -135,8 +135,6 @@ struct intel_vsec_platform_info {
  * struct intel_vsec_device - Auxbus specific device information
  * @auxdev:        auxbus device struct for auxbus access
  * @dev:           struct device associated with the device
- * @resource:      PCI discovery resources (BAR windows), one per discovery
- *                 instance. Valid only when @src == INTEL_VSEC_DISC_PCI
  * @acpi_disc:     ACPI discovery tables, each entry is two QWORDs
  *                 in little-endian format as defined by the PMT ACPI spec.
  *                 Valid only when @src == INTEL_VSEC_DISC_ACPI.
@@ -149,11 +147,12 @@ struct intel_vsec_platform_info {
  * @quirks:        specified quirks
  * @base_addr:     base address of entries (if specified)
  * @cap_id:        the enumerated id of the vsec feature
+ * @resource:      PCI discovery resources (BAR windows), one per discovery
+ *                 instance. Valid only when @src == INTEL_VSEC_DISC_PCI
  */
 struct intel_vsec_device {
 	struct auxiliary_device auxdev;
 	struct device *dev;
-	struct resource *resource;
 	u32 (*acpi_disc)[4];
 	enum intel_vsec_disc_source src;
 	struct ida *ida;
@@ -164,6 +163,7 @@ struct intel_vsec_device {
 	unsigned long quirks;
 	u64 base_addr;
 	unsigned long cap_id;
+	struct resource resource[] __counted_by(num_resources);
 };
 
 /**
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 244392026c6d5..23b8891d57045 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -446,6 +446,9 @@ struct io_ring_ctx {
 	/* Stores zcrx object pointers of type struct io_zcrx_ifq */
 	struct xarray			zcrx_ctxs;
 
+	/* Used for accounting references on pages in registered buffers */
+	struct xarray		hpage_acct;
+
 	u32			pers_next;
 	struct xarray		personalities;
 
diff --git a/include/linux/iomap.h b/include/linux/iomap.h
index 2c5685adf3a97..52bfd6783417c 100644
--- a/include/linux/iomap.h
+++ b/include/linux/iomap.h
@@ -142,16 +142,6 @@ static inline void *iomap_inline_data(const struct iomap *iomap, loff_t pos)
 	return iomap->inline_data + pos - iomap->offset;
 }
 
-/*
- * Check if the mapping's length is within the valid range for inline data.
- * This is used to guard against accessing data beyond the page inline_data
- * points at.
- */
-static inline bool iomap_inline_data_valid(const struct iomap *iomap)
-{
-	return iomap->length <= PAGE_SIZE - offset_in_page(iomap->inline_data);
-}
-
 /*
  * When get_folio succeeds, put_folio will always be called to do any
  * cleanup work necessary.  put_folio is responsible for unlocking and putting
diff --git a/include/linux/iommu-dma.h b/include/linux/iommu-dma.h
index a92b3ff9b9343..060f6e23ab3c8 100644
--- a/include/linux/iommu-dma.h
+++ b/include/linux/iommu-dma.h
@@ -7,12 +7,13 @@
 #ifndef _LINUX_IOMMU_DMA_H
 #define _LINUX_IOMMU_DMA_H
 
+#include <linux/device.h>
 #include <linux/dma-direction.h>
 
 #ifdef CONFIG_IOMMU_DMA
 static inline bool use_dma_iommu(struct device *dev)
 {
-	return dev->dma_iommu;
+	return dev_dma_iommu(dev);
 }
 #else
 static inline bool use_dma_iommu(struct device *dev)
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index e587d4ac4d331..bf8a77a164e4d 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -345,12 +345,6 @@ struct iommu_pages_list {
 /**
  * struct iommu_iotlb_gather - Range information for a pending IOTLB flush
  *
- * @start: IOVA representing the start of the range to be flushed
- * @end: IOVA representing the end of the range to be flushed (inclusive)
- * @pgsize: The interval at which to perform the flush
- * @freelist: Removed pages to free after sync
- * @queued: Indicates that the flush will be queued
- *
  * This structure is intended to be updated by multiple calls to the
  * ->unmap() function in struct iommu_ops before eventually being passed
  * into ->iotlb_sync(). Drivers can add pages to @freelist to be freed after
@@ -359,10 +353,44 @@ struct iommu_pages_list {
  * later instead of ->iotlb_sync(), so drivers may optimise accordingly.
  */
 struct iommu_iotlb_gather {
+	/** @start: IOVA representing the start of the range to be flushed */
 	unsigned long		start;
+	/**
+	 * @end: IOVA representing the end of the range to be
+	 *       flushed (inclusive)
+	 */
 	unsigned long		end;
-	size_t			pgsize;
+
+	union {
+		/**
+		 * @pgsize: The interval at which to perform the flush, only
+		 *          used by arm-smmu-v3
+		 */
+		size_t pgsize;
+		struct {
+			/**
+			 * @pt.leaf_levels_bitmap: Bitmap of generic_pt
+			 * levels where leaf entries were unmapped. Bit 0
+			 * means the leaf only level. If 0 no leafs
+			 * were unmapped.
+			 */
+			u8 leaf_levels_bitmap;
+			/**
+			 * @pt.table_levels_bitmap: Bitmap of generic_pt levels
+			 * of table entries that were removed. Bit 0 is never
+			 * set, bit 1 means a table of all leafs was removed.
+			 * When freelist is empty this must be 0.
+			 */
+			u8 table_levels_bitmap;
+		} pt;
+	};
+
+	/**
+	 * @freelist: Removed pages to free after sync, only used by
+	 *            iommupt
+	 */
 	struct iommu_pages_list	freelist;
+	/** @queued: True if the gather will be completed with a flush all */
 	bool			queued;
 };
 
diff --git a/include/linux/ioport.h b/include/linux/ioport.h
index 3c73c9c0d4f7f..f7930b3dfd0a5 100644
--- a/include/linux/ioport.h
+++ b/include/linux/ioport.h
@@ -261,7 +261,7 @@ extern int allocate_resource(struct resource *root, struct resource *new,
 struct resource *lookup_resource(struct resource *root, resource_size_t start);
 int adjust_resource(struct resource *res, resource_size_t start,
 		    resource_size_t size);
-resource_size_t resource_alignment(struct resource *res);
+resource_size_t resource_alignment(const struct resource *res);
 
 /**
  * resource_set_size - Calculate resource end address from size and start
diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h
index 0225121f30138..ea5fd2374ebe0 100644
--- a/include/linux/irqchip/arm-gic-v3.h
+++ b/include/linux/irqchip/arm-gic-v3.h
@@ -604,7 +604,7 @@
 
 #include <asm/arch_gicv3.h>
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 /*
  * We need a value to serve as a irq-type for LPIs. Choose one that will
diff --git a/include/linux/irqchip/arm-gic.h b/include/linux/irqchip/arm-gic.h
index d45fa19f9e470..849386dc5ec80 100644
--- a/include/linux/irqchip/arm-gic.h
+++ b/include/linux/irqchip/arm-gic.h
@@ -131,7 +131,7 @@
 #define GICV_PMR_PRIORITY_SHIFT		3
 #define GICV_PMR_PRIORITY_MASK		(0x1f << GICV_PMR_PRIORITY_SHIFT)
 
-#ifndef __ASSEMBLY__
+#ifndef __ASSEMBLER__
 
 #include <linux/irqdomain.h>
 
@@ -162,5 +162,5 @@ int gic_get_cpu_id(unsigned int cpu);
 void gic_migrate_target(unsigned int new_cpu_id);
 unsigned long gic_get_sgir_physaddr(void);
 
-#endif /* __ASSEMBLY */
+#endif /* __ASSEMBLER__ */
 #endif
diff --git a/include/linux/irqdomain_defs.h b/include/linux/irqdomain_defs.h
index 36653e2ee1c92..3a03bdfeeee93 100644
--- a/include/linux/irqdomain_defs.h
+++ b/include/linux/irqdomain_defs.h
@@ -17,7 +17,6 @@ enum irq_domain_bus_token {
 	DOMAIN_BUS_PLATFORM_MSI,
 	DOMAIN_BUS_NEXUS,
 	DOMAIN_BUS_IPI,
-	DOMAIN_BUS_FSL_MC_MSI,
 	DOMAIN_BUS_TI_SCI_INTA_MSI,
 	DOMAIN_BUS_WAKEUP,
 	DOMAIN_BUS_VMD_MSI,
diff --git a/include/linux/kcov.h b/include/linux/kcov.h
index 0143358874b07..895b761b2db15 100644
--- a/include/linux/kcov.h
+++ b/include/linux/kcov.h
@@ -21,8 +21,6 @@ enum kcov_mode {
 	KCOV_MODE_TRACE_PC = 2,
 	/* Collecting comparison operands mode. */
 	KCOV_MODE_TRACE_CMP = 3,
-	/* The process owns a KCOV remote reference. */
-	KCOV_MODE_REMOTE = 4,
 };
 
 #define KCOV_IN_CTXSW	(1 << 30)
@@ -43,11 +41,11 @@ do {						\
 /* See Documentation/dev-tools/kcov.rst for usage details. */
 void kcov_remote_start(u64 handle);
 void kcov_remote_stop(void);
-u64 kcov_common_handle(void);
+struct kcov_common_handle_id kcov_common_handle(void);
 
-static inline void kcov_remote_start_common(u64 id)
+static inline void kcov_remote_start_common(struct kcov_common_handle_id id)
 {
-	kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id));
+	kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id.val));
 }
 
 static inline void kcov_remote_start_usb(u64 id)
@@ -99,11 +97,11 @@ static inline void kcov_prepare_switch(struct task_struct *t) {}
 static inline void kcov_finish_switch(struct task_struct *t) {}
 static inline void kcov_remote_start(u64 handle) {}
 static inline void kcov_remote_stop(void) {}
-static inline u64 kcov_common_handle(void)
+static inline struct kcov_common_handle_id kcov_common_handle(void)
 {
-	return 0;
+	return (struct kcov_common_handle_id){};
 }
-static inline void kcov_remote_start_common(u64 id) {}
+static inline void kcov_remote_start_common(struct kcov_common_handle_id id) {}
 static inline void kcov_remote_start_usb(u64 id) {}
 static inline void kcov_remote_start_usb_softirq(u64 id) {}
 static inline void kcov_remote_stop_softirq(void) {}
diff --git a/include/linux/kcsan-checks.h b/include/linux/kcsan-checks.h
index 92f3843d9ebb8..e135dacaa90f3 100644
--- a/include/linux/kcsan-checks.h
+++ b/include/linux/kcsan-checks.h
@@ -282,7 +282,7 @@ static inline void __kcsan_disable_current(void) { }
  * @size: size of access
  */
 #define __kcsan_check_write(ptr, size)                                         \
-	__kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
+	__kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_WRITE)
 
 /**
  * __kcsan_check_read_write - check regular read-write access for races
@@ -308,7 +308,7 @@ static inline void __kcsan_disable_current(void) { }
  * @size: size of access
  */
 #define kcsan_check_write(ptr, size)                                           \
-	kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
+	kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_WRITE)
 
 /**
  * kcsan_check_read_write - check regular read-write access for races
@@ -331,7 +331,7 @@ static inline void __kcsan_disable_current(void) { }
 #define kcsan_check_atomic_read(ptr, size)                                     \
 	kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
 #define kcsan_check_atomic_write(ptr, size)                                    \
-	kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE)
+	kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE)
 #define kcsan_check_atomic_read_write(ptr, size)                               \
 	kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND)
 #endif
diff --git a/include/linux/kho/abi/kexec_handover.h b/include/linux/kho/abi/kexec_handover.h
index 7e847a2339b09..fb2d37417ad9c 100644
--- a/include/linux/kho/abi/kexec_handover.h
+++ b/include/linux/kho/abi/kexec_handover.h
@@ -64,7 +64,7 @@
  *   Root KHO Node (/):
  *     - compatible: "kho-v3"
  *
- *       Indentifies the overall KHO ABI version.
+ *       Identifies the overall KHO ABI version.
  *
  *     - preserved-memory-map: u64
  *
@@ -274,7 +274,7 @@ enum kho_radix_consts {
 	 * and 1 bitmap level.
 	 */
 	KHO_TREE_MAX_DEPTH =
-		DIV_ROUND_UP(KHO_ORDER_0_LOG2 - KHO_BITMAP_SIZE_LOG2,
+		DIV_ROUND_UP(KHO_ORDER_0_LOG2 - KHO_BITMAP_SIZE_LOG2 + 1,
 			     KHO_TABLE_SIZE_LOG2) + 1,
 };
 
diff --git a/include/linux/ks8851_mll.h b/include/linux/ks8851_mll.h
deleted file mode 100644
index 57c0a39ed796d..0000000000000
--- a/include/linux/ks8851_mll.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * ks8861_mll platform data struct definition
- * Copyright (c) 2012 BTicino S.p.A.
- */
-
-#ifndef _LINUX_KS8851_MLL_H
-#define _LINUX_KS8851_MLL_H
-
-#include <linux/if_ether.h>
-
-/**
- * struct ks8851_mll_platform_data - Platform data of the KS8851_MLL network driver
- * @macaddr:	The MAC address of the device, set to all 0:s to use the on in
- *		the chip.
- */
-struct ks8851_mll_platform_data {
-	u8 mac_addr[ETH_ALEN];
-};
-
-#endif
diff --git a/include/linux/kstrtox.h b/include/linux/kstrtox.h
index 6ea897222af1d..6c92828667704 100644
--- a/include/linux/kstrtox.h
+++ b/include/linux/kstrtox.h
@@ -142,10 +142,9 @@ static inline int __must_check kstrtos32_from_user(const char __user *s, size_t
  * Keep in mind above caveat.
  */
 
-extern unsigned long simple_strtoul(const char *,char **,unsigned int);
-extern unsigned long simple_strntoul(const char *,char **,unsigned int,size_t);
-extern long simple_strtol(const char *,char **,unsigned int);
-extern unsigned long long simple_strtoull(const char *,char **,unsigned int);
-extern long long simple_strtoll(const char *,char **,unsigned int);
+unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base);
+long simple_strtol(const char *cp, char **endp, unsigned int base);
+unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base);
+long long simple_strtoll(const char *cp, char **endp, unsigned int base);
 
 #endif	/* _LINUX_KSTRTOX_H */
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 4c14aee1fb063..61a3430957f21 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -1390,20 +1390,20 @@ void mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *mems
 void mark_page_dirty(struct kvm *kvm, gfn_t gfn);
 void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn);
 
-int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvm_host_map *map,
+int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map,
 		   bool writable);
 void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map);
 
-static inline int kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa,
+static inline int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn,
 			       struct kvm_host_map *map)
 {
-	return __kvm_vcpu_map(vcpu, gpa, map, true);
+	return __kvm_vcpu_map(vcpu, gfn, map, true);
 }
 
-static inline int kvm_vcpu_map_readonly(struct kvm_vcpu *vcpu, gpa_t gpa,
+static inline int kvm_vcpu_map_readonly(struct kvm_vcpu *vcpu, gfn_t gfn,
 					struct kvm_host_map *map)
 {
-	return __kvm_vcpu_map(vcpu, gpa, map, false);
+	return __kvm_vcpu_map(vcpu, gfn, map, false);
 }
 
 static inline void kvm_vcpu_map_mark_dirty(struct kvm_vcpu *vcpu,
@@ -1562,7 +1562,7 @@ void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc);
 void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc);
 #endif
 
-void kvm_mmu_invalidate_begin(struct kvm *kvm);
+void kvm_mmu_invalidate_start(struct kvm *kvm);
 void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end);
 void kvm_mmu_invalidate_end(struct kvm *kvm);
 bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range);
diff --git a/include/linux/led-class-multicolor.h b/include/linux/led-class-multicolor.h
index db9f34c6736e9..8a05836cae257 100644
--- a/include/linux/led-class-multicolor.h
+++ b/include/linux/led-class-multicolor.h
@@ -9,10 +9,31 @@
 #include <linux/leds.h>
 #include <dt-bindings/leds/common.h>
 
+/**
+ * struct mc_subled - Color component description.
+ * @color_index: Color ID.
+ * @brightness: Scaled intensity.
+ * @intensity: Current intensity.
+ * @max_intensity: Maximum supported intensity value.
+ * @channel: Channel index.
+ *
+ * Describes a color component of a multicolor LED. Many multicolor LEDs
+ * do not support global brightness control in hardware, so they use
+ * the brightness field in connection with led_mc_calc_color_components()
+ * to perform the intensity scaling in software.
+ * Such drivers should set max_intensity to 0 to signal the multicolor LED core
+ * that the maximum global brightness of the LED class device should be used for
+ * limiting incoming intensity values.
+ *
+ * Multicolor LEDs that do support global brightness control in hardware
+ * should instead set max_intensity to the maximum intensity value supported
+ * by the hardware for a given color component.
+ */
 struct mc_subled {
 	unsigned int color_index;
 	unsigned int brightness;
 	unsigned int intensity;
+	unsigned int max_intensity;
 	unsigned int channel;
 };
 
@@ -53,7 +74,14 @@ int led_classdev_multicolor_register_ext(struct device *parent,
  */
 void led_classdev_multicolor_unregister(struct led_classdev_mc *mcled_cdev);
 
-/* Calculate brightness for the monochrome LED cluster */
+/**
+ * led_mc_calc_color_components() - Calculates component brightness values of a LED cluster.
+ * @mcled_cdev - Multicolor LED class device of the LED cluster.
+ * @brightness - Global brightness of the LED cluster.
+ *
+ * Calculates the brightness values for each color component of a monochrome LED cluster,
+ * see Documentation/leds/leds-class-multicolor.rst for details.
+ */
 int led_mc_calc_color_components(struct led_classdev_mc *mcled_cdev,
 				 enum led_brightness brightness);
 
diff --git a/include/linux/libata.h b/include/linux/libata.h
index 5c085ef4eda7b..127229fbd1a63 100644
--- a/include/linux/libata.h
+++ b/include/linux/libata.h
@@ -371,6 +371,7 @@ enum {
 	/* return values for ->qc_defer */
 	ATA_DEFER_LINK		= 1,
 	ATA_DEFER_PORT		= 2,
+	ATA_DEFER_LINK_EXCL	= 3,
 
 	/* desc_len for ata_eh_info and context */
 	ATA_EH_DESC_LEN		= 80,
@@ -854,6 +855,9 @@ struct ata_link {
 	unsigned int		sata_spd;	/* current SATA PHY speed */
 	enum ata_lpm_policy	lpm_policy;
 
+	struct work_struct	deferred_qc_work;
+	struct ata_queued_cmd	*deferred_qc;
+
 	/* record runtime error info, protected by host_set lock */
 	struct ata_eh_info	eh_info;
 	/* EH context */
@@ -899,9 +903,6 @@ struct ata_port {
 	u64			qc_active;
 	int			nr_active_links; /* #links with active qcs */
 
-	struct work_struct	deferred_qc_work;
-	struct ata_queued_cmd	*deferred_qc;
-
 	struct ata_link		link;		/* host default link */
 	struct ata_link		*slave_link;	/* see ata_slave_link_init() */
 
diff --git a/include/linux/list.h b/include/linux/list.h
index 00ea8e5fb88b0..09d979976b3b8 100644
--- a/include/linux/list.h
+++ b/include/linux/list.h
@@ -191,6 +191,29 @@ static inline void list_add_tail(struct list_head *new, struct list_head *head)
 	__list_add(new, head->prev, head);
 }
 
+/**
+ * list_add_tail_release - add a new entry with release barrier
+ * @new: new entry to be added
+ * @head: list head to add it before
+ *
+ * Insert a new entry before the specified head, using a release barrier to set
+ * the ->next pointer that points to it.  This is useful for implementing
+ * queues, in particular one that the elements will be walked through forwards
+ * locklessly.
+ */
+static inline void list_add_tail_release(struct list_head *new,
+					 struct list_head *head)
+{
+	struct list_head *prev = head->prev;
+
+	if (__list_add_valid(new, prev, head)) {
+		new->next = head;
+		new->prev = prev;
+		head->prev = new;
+		smp_store_release(&prev->next, new);
+	}
+}
+
 /*
  * Delete a list entry by making the prev/next entries
  * point to each other.
@@ -644,6 +667,20 @@ static inline void list_splice_tail_init(struct list_head *list,
 	pos__ != head__ ? list_entry(pos__, type, member) : NULL; \
 })
 
+/**
+ * list_first_entry_or_null_acquire - get the first element from a list with barrier
+ * @ptr:	the list head to take the element from.
+ * @type:	the type of the struct this is embedded in.
+ * @member:	the name of the list_head within the struct.
+ *
+ * Note that if the list is empty, it returns NULL.
+ */
+#define list_first_entry_or_null_acquire(ptr, type, member) ({ \
+	struct list_head *head__ = (ptr); \
+	struct list_head *pos__ = smp_load_acquire(&head__->next); \
+	pos__ != head__ ? list_entry(pos__, type, member) : NULL; \
+})
+
 /**
  * list_last_entry_or_null - get the last element from a list
  * @ptr:	the list head to take the element from.
diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h
index 30c5a39ff9e9c..88722e5caf020 100644
--- a/include/linux/liveupdate.h
+++ b/include/linux/liveupdate.h
@@ -12,6 +12,7 @@
 #include <linux/kho/abi/luo.h>
 #include <linux/list.h>
 #include <linux/mutex.h>
+#include <linux/refcount.h>
 #include <linux/rwsem.h>
 #include <linux/types.h>
 #include <uapi/linux/liveupdate.h>
@@ -175,7 +176,7 @@ struct liveupdate_flb_ops {
  * @retrieved: True once the FLB's retrieve() callback has run.
  */
 struct luo_flb_private_state {
-	long count;
+	refcount_t count;
 	u64 data;
 	void *obj;
 	struct mutex lock;
@@ -239,6 +240,8 @@ void liveupdate_unregister_flb(struct liveupdate_file_handler *fh,
 			       struct liveupdate_flb *flb);
 
 int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp);
+void liveupdate_flb_put_incoming(struct liveupdate_flb *flb);
+
 int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp);
 
 #else /* CONFIG_LIVEUPDATE */
@@ -279,6 +282,10 @@ static inline int liveupdate_flb_get_incoming(struct liveupdate_flb *flb,
 	return -EOPNOTSUPP;
 }
 
+static inline void liveupdate_flb_put_incoming(struct liveupdate_flb *flb)
+{
+}
+
 static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb,
 					      void **objp)
 {
diff --git a/include/linux/llist.h b/include/linux/llist.h
index 607b2360c938c..8846b7709669f 100644
--- a/include/linux/llist.h
+++ b/include/linux/llist.h
@@ -26,8 +26,8 @@
  *
  *           |   add    | del_first |  del_all
  * add       |    -     |     -     |     -
- * del_first |          |     L     |     L
- * del_all   |          |           |     -
+ * del_first |    -     |     L     |     L
+ * del_all   |    -     |     -     |     -
  *
  * Where, a particular row's operation can happen concurrently with a column's
  * operation, with "-" being no lock needed, while "L" being lock is needed.
diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h
index 2b8dfb35caed3..c0e91727f87ef 100644
--- a/include/linux/lsm_hook_defs.h
+++ b/include/linux/lsm_hook_defs.h
@@ -176,8 +176,8 @@ LSM_HOOK(int, -EOPNOTSUPP, inode_getsecurity, struct mnt_idmap *idmap,
 	 struct inode *inode, const char *name, void **buffer, bool alloc)
 LSM_HOOK(int, -EOPNOTSUPP, inode_setsecurity, struct inode *inode,
 	 const char *name, const void *value, size_t size, int flags)
-LSM_HOOK(int, 0, inode_listsecurity, struct inode *inode, char *buffer,
-	 size_t buffer_size)
+LSM_HOOK(int, 0, inode_listsecurity, struct inode *inode, char **buffer,
+	 ssize_t *remaining_size)
 LSM_HOOK(void, LSM_RET_VOID, inode_getlsmprop, struct inode *inode,
 	 struct lsm_prop *prop)
 LSM_HOOK(int, 0, inode_copy_up, struct dentry *src, struct cred **new)
@@ -444,6 +444,11 @@ LSM_HOOK(int, 0, bpf_prog, struct bpf_prog *prog)
 LSM_HOOK(int, 0, bpf_map_create, struct bpf_map *map, union bpf_attr *attr,
 	 struct bpf_token *token, bool kernel)
 LSM_HOOK(void, LSM_RET_VOID, bpf_map_free, struct bpf_map *map)
+LSM_HOOK(int, 0, bpf_prog_load_post_integrity, struct bpf_prog *prog,
+	 union bpf_attr *attr, struct bpf_token *token, bool kernel,
+	 const struct lsm_id *lsmid, enum lsm_integrity_verdict verdict)
+LSM_HOOK(int, 0, bpf_prog_load_integrity, struct bpf_prog *prog,
+	 union bpf_attr *attr, struct bpf_token *token, bool kernel)
 LSM_HOOK(int, 0, bpf_prog_load, struct bpf_prog *prog, union bpf_attr *attr,
 	 struct bpf_token *token, bool kernel)
 LSM_HOOK(void, LSM_RET_VOID, bpf_prog_free, struct bpf_prog *prog)
diff --git a/include/linux/mailbox_controller.h b/include/linux/mailbox_controller.h
index dc93287a2a01a..26a238a6f941c 100644
--- a/include/linux/mailbox_controller.h
+++ b/include/linux/mailbox_controller.h
@@ -120,6 +120,7 @@ struct mbox_controller {
  * @txdone_method:	Way to detect TXDone chosen by the API
  * @cl:			Pointer to the current owner of this channel
  * @tx_complete:	Transmission completion
+ * @tx_status:		Transmission status
  * @active_req:		Currently active request hook
  * @msg_count:		No. of mssg currently queued
  * @msg_free:		Index of next available mssg slot
@@ -132,6 +133,7 @@ struct mbox_chan {
 	unsigned txdone_method;
 	struct mbox_client *cl;
 	struct completion tx_complete;
+	int tx_status;
 	void *active_req;
 	unsigned msg_count, msg_free;
 	void *msg_data[MBOX_TX_QUEUE_LEN];
diff --git a/include/linux/memblock.h b/include/linux/memblock.h
index b0f750d22a7b5..5afcd99aa8c14 100644
--- a/include/linux/memblock.h
+++ b/include/linux/memblock.h
@@ -613,11 +613,28 @@ static inline void memtest_report_meminfo(struct seq_file *m) { }
 #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH
 void memblock_set_kho_scratch_only(void);
 void memblock_clear_kho_scratch_only(void);
-void memmap_init_kho_scratch_pages(void);
+bool memblock_is_kho_scratch_memory(phys_addr_t addr);
+
+static inline enum migratetype kho_scratch_migratetype(unsigned long pfn,
+						       enum migratetype mt)
+{
+	if (memblock_is_kho_scratch_memory(PFN_PHYS(pfn)))
+		return MIGRATE_CMA;
+	return mt;
+}
 #else
 static inline void memblock_set_kho_scratch_only(void) { }
 static inline void memblock_clear_kho_scratch_only(void) { }
-static inline void memmap_init_kho_scratch_pages(void) {}
+static inline bool memblock_is_kho_scratch_memory(phys_addr_t addr)
+{
+	return false;
+}
+
+static inline enum migratetype kho_scratch_migratetype(unsigned long pfn,
+						       enum migratetype mt)
+{
+	return mt;
+}
 #endif
 
 #endif /* _LINUX_MEMBLOCK_H */
diff --git a/include/linux/memory.h b/include/linux/memory.h
index 5bb5599c6b2bf..463dc02f6cff0 100644
--- a/include/linux/memory.h
+++ b/include/linux/memory.h
@@ -158,7 +158,11 @@ int create_memory_block_devices(unsigned long start, unsigned long size,
 void remove_memory_block_devices(unsigned long start, unsigned long size);
 extern void memory_dev_init(void);
 extern int memory_notify(enum memory_block_state state, void *v);
-extern struct memory_block *find_memory_block(unsigned long section_nr);
+struct memory_block *memory_block_get(unsigned long block_id);
+static inline void memory_block_put(struct memory_block *mem)
+{
+	put_device(&mem->dev);
+}
 typedef int (*walk_memory_blocks_func_t)(struct memory_block *, void *);
 extern int walk_memory_blocks(unsigned long start, unsigned long size,
 			      void *arg, walk_memory_blocks_func_t func);
@@ -171,7 +175,6 @@ struct memory_group *memory_group_find_by_id(int mgid);
 typedef int (*walk_memory_groups_func_t)(struct memory_group *, void *);
 int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func,
 			       struct memory_group *excluded, void *arg);
-struct memory_block *find_memory_block_by_id(unsigned long block_id);
 #define hotplug_memory_notifier(fn, pri) ({		\
 	static __meminitdata struct notifier_block fn##_mem_nb =\
 		{ .notifier_call = fn, .priority = pri };\
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index 815e908c4135b..7c9d66729c609 100644
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -135,9 +135,10 @@ static inline bool movable_node_is_enabled(void)
 	return movable_node_enabled;
 }
 
-extern void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap);
+extern void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap,
+			       struct dev_pagemap *pgmap);
 extern void __remove_pages(unsigned long start_pfn, unsigned long nr_pages,
-			   struct vmem_altmap *altmap);
+			   struct vmem_altmap *altmap, struct dev_pagemap *pgmap);
 
 /* reasonably generic interface to expand the physical pages */
 extern int __add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages,
@@ -307,7 +308,8 @@ extern int sparse_add_section(int nid, unsigned long pfn,
 		unsigned long nr_pages, struct vmem_altmap *altmap,
 		struct dev_pagemap *pgmap);
 extern void sparse_remove_section(unsigned long pfn, unsigned long nr_pages,
-				  struct vmem_altmap *altmap);
+				  struct vmem_altmap *altmap,
+				  struct dev_pagemap *pgmap);
 extern struct zone *zone_for_pfn_range(enum mmop online_type,
 		int nid, struct memory_group *group, unsigned long start_pfn,
 		unsigned long nr_pages);
diff --git a/include/linux/mfd/bq257xx.h b/include/linux/mfd/bq257xx.h
index 1d6ddc7fb09fc..4ec72eb920f2c 100644
--- a/include/linux/mfd/bq257xx.h
+++ b/include/linux/mfd/bq257xx.h
@@ -98,7 +98,419 @@
 
 #define BQ25703_EN_OTG_MASK			BIT(12)
 
+#define BQ25792_REG00_MIN_SYS_VOLTAGE		0x00
+#define BQ25792_REG01_CHARGE_VOLTAGE_LIMIT	0x01
+#define BQ25792_REG03_CHARGE_CURRENT_LIMIT	0x03
+#define BQ25792_REG05_INPUT_VOLTAGE_LIMIT	0x05
+#define BQ25792_REG06_INPUT_CURRENT_LIMIT	0x06
+#define BQ25792_REG08_PRECHARGE_CONTROL		0x08
+#define BQ25792_REG09_TERMINATION_CONTROL	0x09
+#define BQ25792_REG0A_RECHARGE_CONTROL		0x0a
+#define BQ25792_REG0B_VOTG_REGULATION		0x0b
+#define BQ25792_REG0D_IOTG_REGULATION		0x0d
+#define BQ25792_REG0E_TIMER_CONTROL		0x0e
+#define BQ25792_REG0F_CHARGER_CONTROL_0		0x0f
+#define BQ25792_REG10_CHARGER_CONTROL_1		0x10
+#define BQ25792_REG11_CHARGER_CONTROL_2		0x11
+#define BQ25792_REG12_CHARGER_CONTROL_3		0x12
+#define BQ25792_REG13_CHARGER_CONTROL_4		0x13
+#define BQ25792_REG14_CHARGER_CONTROL_5		0x14
+/* REG15 reserved */
+#define BQ25792_REG16_TEMPERATURE_CONTROL	0x16
+#define BQ25792_REG17_NTC_CONTROL_0		0x17
+#define BQ25792_REG18_NTC_CONTROL_1		0x18
+#define BQ25792_REG19_ICO_CURRENT_LIMIT		0x19
+#define BQ25792_REG1B_CHARGER_STATUS_0		0x1b
+#define BQ25792_REG1C_CHARGER_STATUS_1		0x1c
+#define BQ25792_REG1D_CHARGER_STATUS_2		0x1d
+#define BQ25792_REG1E_CHARGER_STATUS_3		0x1e
+#define BQ25792_REG1F_CHARGER_STATUS_4		0x1f
+#define BQ25792_REG20_FAULT_STATUS_0		0x20
+#define BQ25792_REG21_FAULT_STATUS_1		0x21
+#define BQ25792_REG22_CHARGER_FLAG_0		0x22
+#define BQ25792_REG23_CHARGER_FLAG_1		0x23
+#define BQ25792_REG24_CHARGER_FLAG_2		0x24
+#define BQ25792_REG25_CHARGER_FLAG_3		0x25
+#define BQ25792_REG26_FAULT_FLAG_0		0x26
+#define BQ25792_REG27_FAULT_FLAG_1		0x27
+#define BQ25792_REG28_CHARGER_MASK_0		0x28
+#define BQ25792_REG29_CHARGER_MASK_1		0x29
+#define BQ25792_REG2A_CHARGER_MASK_2		0x2a
+#define BQ25792_REG2B_CHARGER_MASK_3		0x2b
+#define BQ25792_REG2C_FAULT_MASK_0		0x2c
+#define BQ25792_REG2D_FAULT_MASK_1		0x2d
+#define BQ25792_REG2E_ADC_CONTROL		0x2e
+#define BQ25792_REG2F_ADC_FUNCTION_DISABLE_0	0x2f
+#define BQ25792_REG30_ADC_FUNCTION_DISABLE_1	0x30
+#define BQ25792_REG31_IBUS_ADC			0x31
+#define BQ25792_REG33_IBAT_ADC			0x33
+#define BQ25792_REG35_VBUS_ADC			0x35
+#define BQ25792_REG37_VAC1_ADC			0x37
+#define BQ25792_REG39_VAC2_ADC			0x39
+#define BQ25792_REG3B_VBAT_ADC			0x3b
+#define BQ25792_REG3D_VSYS_ADC			0x3d
+#define BQ25792_REG3F_TS_ADC			0x3f
+#define BQ25792_REG41_TDIE_ADC			0x41
+#define BQ25792_REG43_DP_ADC			0x43
+#define BQ25792_REG45_DM_ADC			0x45
+#define BQ25792_REG47_DPDM_DRIVER		0x47
+#define BQ25792_REG48_PART_INFORMATION		0x48
+
+/* Minimal System Voltage */
+#define BQ25792_REG00_VSYSMIN_MASK		GENMASK(5, 0)
+
+#define BQ25792_MINVSYS_MIN_UV			2500000
+#define BQ25792_MINVSYS_STEP_UV			250000
+#define BQ25792_MINVSYS_MAX_UV			16000000
+
+/* Charge Voltage Limit */
+#define BQ25792_REG01_VREG_MASK			GENMASK(10, 0)
+
+#define BQ25792_VBATREG_MIN_UV			3000000
+#define BQ25792_VBATREG_STEP_UV			10000
+#define BQ25792_VBATREG_MAX_UV			18800000
+
+/* Charge Current Limit */
+#define BQ25792_REG03_ICHG_MASK			GENMASK(8, 0)
+
+#define BQ25792_ICHG_MIN_UA			50000
+#define BQ25792_ICHG_STEP_UA			10000
+#define BQ25792_ICHG_MAX_UA			5000000
+
+/* Input Voltage Limit */
+#define BQ25792_REG05_VINDPM_MASK		GENMASK(7, 0)
+
+/* Input Current Limit */
+#define BQ25792_REG06_IINDPM_MASK		GENMASK(8, 0)
+#define BQ25792_IINDPM_DEFAULT_UA		3000000
+#define BQ25792_IINDPM_STEP_UA			10000
+#define BQ25792_IINDPM_MIN_UA			100000
+#define BQ25792_IINDPM_MAX_UA			3300000
+
+/* Precharge Control */
+#define BQ25792_REG08_VBAT_LOWV_MASK		GENMASK(7, 6)
+#define BQ25792_REG08_IPRECHG_MASK		GENMASK(5, 0)
+
+/* Termination Control */
+#define BQ25792_REG09_REG_RST			BIT(6)
+#define BQ25792_REG09_ITERM_MASK		GENMASK(4, 0)
+
+/* Re-charge Control */
+#define BQ25792_REG0A_CELL_MASK			GENMASK(7, 6)
+#define BQ25792_REG0A_TRECHG_MASK		GENMASK(5, 4)
+#define BQ25792_REG0A_VRECHG_MASK		GENMASK(3, 0)
+
+/* VOTG regulation */
+#define BQ25792_REG0B_VOTG_MASK			GENMASK(10, 0)
+
+#define BQ25792_OTG_VOLT_MIN_UV			2800000
+#define BQ25792_OTG_VOLT_STEP_UV		10000
+#define BQ25792_OTG_VOLT_MAX_UV			22000000
+#define BQ25792_OTG_VOLT_NUM_VOLT		((BQ25792_OTG_VOLT_MAX_UV \
+						  - BQ25792_OTG_VOLT_MIN_UV) \
+						  / BQ25792_OTG_VOLT_STEP_UV + 1)
+
+/* IOTG regulation */
+#define BQ25792_REG0D_PRECHG_TMR		BIT(7)
+#define BQ25792_REG0D_IOTG_MASK			GENMASK(6, 0)
+
+#define BQ25792_OTG_CUR_MIN_UA		120000
+#define BQ25792_OTG_CUR_STEP_UA		40000
+#define BQ25792_OTG_CUR_MAX_UA		3320000
+
+/* Timer Control */
+#define BQ25792_REG0E_TOPOFF_TMR_MASK		GENMASK(7, 6)
+#define BQ25792_REG0E_EN_TRICHG_TMR		BIT(5)
+#define BQ25792_REG0E_EN_PRECHG_TMR		BIT(4)
+#define BQ25792_REG0E_EN_CHG_TMR		BIT(3)
+#define BQ25792_REG0E_CHG_TMR_MASK		GENMASK(2, 1)
+#define BQ25792_REG0E_TMR2X_EN			BIT(0)
+
+/* Charger Control 0 */
+#define BQ25792_REG0F_EN_AUTO_IBATDIS		BIT(7)
+#define BQ25792_REG0F_FORCE_IBATDIS		BIT(6)
+#define BQ25792_REG0F_EN_CHG			BIT(5)
+#define BQ25792_REG0F_EN_ICO			BIT(4)
+#define BQ25792_REG0F_FORCE_ICO			BIT(3)
+#define BQ25792_REG0F_EN_HIZ			BIT(2)
+#define BQ25792_REG0F_EN_TERM			BIT(1)
+/* bit0 reserved */
+
+/* Charger Control 1 */
+#define BQ25792_REG10_VAC_OVP_MASK		GENMASK(5, 4)
+#define BQ25792_REG10_WD_RST			BIT(3)
+#define BQ25792_REG10_WATCHDOG_MASK		GENMASK(2, 0)
+
+/* Charger Control 2 */
+#define BQ25792_REG11_FORCE_INDET		BIT(7)
+#define BQ25792_REG11_AUTO_INDET_EN		BIT(6)
+#define BQ25792_REG11_EN_12V			BIT(5)
+#define BQ25792_REG11_EN_9V			BIT(4)
+#define BQ25792_REG11_HVDCP_EN			BIT(3)
+#define BQ25792_REG11_SDRV_CTRL_MASK		GENMASK(2, 1)
+#define BQ25792_REG11_SDRV_DLY			BIT(0)
+
+/* Charger Control 3 */
+#define BQ25792_REG12_DIS_ACDRV			BIT(7)
+#define BQ25792_REG12_EN_OTG			BIT(6)
+#define BQ25792_REG12_PFM_OTG_DIS		BIT(5)
+#define BQ25792_REG12_PFM_FWD_DIS		BIT(4)
+#define BQ25792_REG12_WKUP_DLY			BIT(3)
+#define BQ25792_REG12_DIS_LDO			BIT(2)
+#define BQ25792_REG12_DIS_OTG_OOA		BIT(1)
+#define BQ25792_REG12_DIS_FWD_OOA		BIT(0)
+
+/* Charger Control 4 */
+#define BQ25792_REG13_EN_ACDRV2			BIT(7)
+#define BQ25792_REG13_EN_ACDRV1			BIT(6)
+#define BQ25792_REG13_PWM_FREQ			BIT(5)
+#define BQ25792_REG13_DIS_STAT			BIT(4)
+#define BQ25792_REG13_DIS_VSYS_SHORT		BIT(3)
+#define BQ25792_REG13_DIS_VOTG_UVP		BIT(2)
+#define BQ25792_REG13_FORCE_VINDPM_DET		BIT(1)
+#define BQ25792_REG13_EN_IBUS_OCP		BIT(0)
+
+/* Charger Control 5 */
+#define BQ25792_REG14_SFET_PRESENT		BIT(7)
+/* bit6 reserved */
+#define BQ25792_REG14_EN_IBAT			BIT(5)
+#define BQ25792_REG14_IBAT_REG_MASK		GENMASK(4, 3)
+#define BQ25792_REG14_EN_IINDPM			BIT(2)
+#define BQ25792_REG14_EN_EXTILIM		BIT(1)
+#define BQ25792_REG14_EN_BATOC			BIT(0)
+
+#define BQ25792_IBAT_3A				FIELD_PREP(BQ25792_REG14_IBAT_REG_MASK, 0)
+#define BQ25792_IBAT_4A				FIELD_PREP(BQ25792_REG14_IBAT_REG_MASK, 1)
+#define BQ25792_IBAT_5A				FIELD_PREP(BQ25792_REG14_IBAT_REG_MASK, 2)
+#define BQ25792_IBAT_UNLIM			FIELD_PREP(BQ25792_REG14_IBAT_REG_MASK, 3)
+
+/* Temperature Control */
+#define BQ25792_REG16_TREG_MASK			GENMASK(7, 6)
+#define BQ25792_REG16_TSHUT_MASK		GENMASK(5, 4)
+#define BQ25792_REG16_VBUS_PD_EN		BIT(3)
+#define BQ25792_REG16_VAC1_PD_EN		BIT(2)
+#define BQ25792_REG16_VAC2_PD_EN		BIT(1)
+
+/* NTC Control 0 */
+#define BQ25792_REG17_JEITA_VSET_MASK		GENMASK(7, 5)
+#define BQ25792_REG17_JEITA_ISETH_MASK		GENMASK(4, 3)
+#define BQ25792_REG17_JEITA_ISETC_MASK		GENMASK(2, 1)
+
+/* NTC Control 1 */
+#define BQ25792_REG18_TS_COOL_MASK		GENMASK(7, 6)
+#define BQ25792_REG18_TS_WARM_MASK		GENMASK(5, 4)
+#define BQ25792_REG18_BHOT_MASK			GENMASK(3, 2)
+#define BQ25792_REG18_BCOLD			BIT(1)
+#define BQ25792_REG18_TS_IGNORE			BIT(0)
+
+/* ICO Current Limit */
+#define BQ25792_REG19_ICO_ILIM_MASK		GENMASK(8, 0)
+
+/* Charger Status 0 */
+#define BQ25792_REG1B_IINDPM_STAT		BIT(7)
+#define BQ25792_REG1B_VINDPM_STAT		BIT(6)
+#define BQ25792_REG1B_WD_STAT			BIT(5)
+#define BQ25792_REG1B_POORSRC_STAT		BIT(4)
+#define BQ25792_REG1B_PG_STAT			BIT(3)
+#define BQ25792_REG1B_AC2_PRESENT_STAT		BIT(2)
+#define BQ25792_REG1B_AC1_PRESENT_STAT		BIT(1)
+#define BQ25792_REG1B_VBUS_PRESENT_STAT		BIT(0)
+
+/* Charger Status 1 */
+#define BQ25792_REG1C_CHG_STAT_MASK		GENMASK(7, 5)
+#define BQ25792_REG1C_VBUS_STAT_MASK		GENMASK(4, 1)
+#define BQ25792_REG1C_BC12_DONE_STAT		BIT(0)
+
+/* Charger Status 2 */
+#define BQ25792_REG1D_ICO_STAT_MASK		GENMASK(7, 6)
+#define BQ25792_REG1D_TREG_STAT			BIT(2)
+#define BQ25792_REG1D_DPDM_STAT			BIT(1)
+#define BQ25792_REG1D_VBAT_PRESENT_STAT		BIT(0)
+
+/* Charger Status 3 */
+#define BQ25792_REG1E_ACRB2_STAT		BIT(7)
+#define BQ25792_REG1E_ACRB1_STAT		BIT(6)
+#define BQ25792_REG1E_ADC_DONE_STAT		BIT(5)
+#define BQ25792_REG1E_VSYS_STAT			BIT(4)
+#define BQ25792_REG1E_CHG_TMR_STAT		BIT(3)
+#define BQ25792_REG1E_TRICHG_TMR_STAT		BIT(2)
+#define BQ25792_REG1E_PRECHG_TMR_STAT		BIT(1)
+
+/* Charger Status 4 */
+#define BQ25792_REG1F_VBATOTG_LOW_STAT		BIT(4)
+#define BQ25792_REG1F_TS_COLD_STAT		BIT(3)
+#define BQ25792_REG1F_TS_COOL_STAT		BIT(2)
+#define BQ25792_REG1F_TS_WARM_STAT		BIT(1)
+#define BQ25792_REG1F_TS_HOT_STAT		BIT(0)
+
+/* FAULT Status 0 */
+#define BQ25792_REG20_IBAT_REG_STAT		BIT(7)
+#define BQ25792_REG20_VBUS_OVP_STAT		BIT(6)
+#define BQ25792_REG20_VBAT_OVP_STAT		BIT(5)
+#define BQ25792_REG20_IBUS_OCP_STAT		BIT(4)
+#define BQ25792_REG20_IBAT_OCP_STAT		BIT(3)
+#define BQ25792_REG20_CONV_OCP_STAT		BIT(2)
+#define BQ25792_REG20_VAC2_OVP_STAT		BIT(1)
+#define BQ25792_REG20_VAC1_OVP_STAT		BIT(0)
+
+#define BQ25792_REG20_OVERVOLTAGE_MASK		(BQ25792_REG20_VBAT_OVP_STAT | \
+						 BQ25792_REG20_VAC2_OVP_STAT | \
+						 BQ25792_REG20_VAC1_OVP_STAT)
+#define BQ25792_REG20_OVERCURRENT_MASK		(BQ25792_REG20_IBAT_OCP_STAT | \
+						 BQ25792_REG20_CONV_OCP_STAT)
+
+/* FAULT Status 1 */
+#define BQ25792_REG21_VSYS_SHORT_STAT		BIT(7)
+#define BQ25792_REG21_VSYS_OVP_STAT		BIT(6)
+#define BQ25792_REG21_OTG_OVP_STAT		BIT(5)
+#define BQ25792_REG21_OTG_UVP_STAT		BIT(4)
+#define BQ25792_REG21_TSHUT_STAT		BIT(2)
+
+
+/* Charger Flag 0 */
+#define BQ25792_REG22_IINDPM_FLAG		BIT(7)
+#define BQ25792_REG22_VINDPM_FLAG		BIT(6)
+#define BQ25792_REG22_WD_FLAG			BIT(5)
+#define BQ25792_REG22_POORSRC_FLAG		BIT(4)
+#define BQ25792_REG22_PG_FLAG			BIT(3)
+#define BQ25792_REG22_AC2_PRESENT_FLAG		BIT(2)
+#define BQ25792_REG22_AC1_PRESENT_FLAG		BIT(1)
+#define BQ25792_REG22_VBUS_PRESENT_FLAG		BIT(0)
+
+/* Charger Flag 1 */
+#define BQ25792_REG23_CHG_FLAG			BIT(7)
+#define BQ25792_REG23_ICO_FLAG			BIT(6)
+#define BQ25792_REG23_VBUS_FLAG			BIT(4)
+#define BQ25792_REG23_TREG_FLAG			BIT(2)
+#define BQ25792_REG23_VBAT_PRESENT_FLAG		BIT(1)
+#define BQ25792_REG23_BC12_DONE_FLAG		BIT(0)
+
+/* Charger Flag 2 */
+#define BQ25792_REG24_DPDM_DONE_FLAG		BIT(6)
+#define BQ25792_REG24_ADC_DONE_FLAG		BIT(5)
+#define BQ25792_REG24_VSYS_FLAG			BIT(4)
+#define BQ25792_REG24_CHG_TMR_FLAG		BIT(3)
+#define BQ25792_REG24_TRICHG_TMR_FLAG		BIT(2)
+#define BQ25792_REG24_PRECHG_TMR_FLAG		BIT(1)
+#define BQ25792_REG24_TOPOFF_TMR_FLAG		BIT(0)
+
+/* Charger Flag 3 */
+#define BQ25792_REG25_VBATOTG_LOW_FLAG		BIT(4)
+#define BQ25792_REG25_TS_COLD_FLAG		BIT(3)
+#define BQ25792_REG25_TS_COOL_FLAG		BIT(2)
+#define BQ25792_REG25_TS_WARM_FLAG		BIT(1)
+#define BQ25792_REG25_TS_HOT_FLAG		BIT(0)
+
+/* FAULT Flag 0 */
+#define BQ25792_REG26_IBAT_REG_FLAG		BIT(7)
+#define BQ25792_REG26_VBUS_OVP_FLAG		BIT(6)
+#define BQ25792_REG26_VBAT_OVP_FLAG		BIT(5)
+#define BQ25792_REG26_IBUS_OCP_FLAG		BIT(4)
+#define BQ25792_REG26_IBAT_OCP_FLAG		BIT(3)
+#define BQ25792_REG26_CONV_OCP_FLAG		BIT(2)
+#define BQ25792_REG26_VAC2_OVP_FLAG		BIT(1)
+#define BQ25792_REG26_VAC1_OVP_FLAG		BIT(0)
+
+/* FAULT Flag 1 */
+#define BQ25792_REG27_VSYS_SHORT_FLAG		BIT(7)
+#define BQ25792_REG27_VSYS_OVP_FLAG		BIT(6)
+#define BQ25792_REG27_OTG_OVP_FLAG		BIT(5)
+#define BQ25792_REG27_OTG_UVP_FLAG		BIT(4)
+#define BQ25792_REG27_TSHUT_FLAG		BIT(2)
+
+/* Charger Mask 0 */
+#define BQ25792_REG28_IINDPM_MASK		BIT(7)
+#define BQ25792_REG28_VINDPM_MASK		BIT(6)
+#define BQ25792_REG28_WD_MASK			BIT(5)
+#define BQ25792_REG28_POORSRC_MASK		BIT(4)
+#define BQ25792_REG28_PG_MASK			BIT(3)
+#define BQ25792_REG28_AC2_PRESENT_MASK		BIT(2)
+#define BQ25792_REG28_AC1_PRESENT_MASK		BIT(1)
+#define BQ25792_REG28_VBUS_PRESENT_MASK		BIT(0)
+
+/* Charger Mask 1 */
+#define BQ25792_REG29_CHG_MASK			BIT(7)
+#define BQ25792_REG29_ICO_MASK			BIT(6)
+#define BQ25792_REG29_VBUS_MASK			BIT(4)
+#define BQ25792_REG29_TREG_MASK			BIT(2)
+#define BQ25792_REG29_VBAT_PRESENT_MASK		BIT(1)
+#define BQ25792_REG29_BC12_DONE_MASK		BIT(0)
+
+/* Charger Mask 2 */
+#define BQ25792_REG2A_DPDM_DONE_MASK		BIT(6)
+#define BQ25792_REG2A_ADC_DONE_MASK		BIT(5)
+#define BQ25792_REG2A_VSYS_MASK			BIT(4)
+#define BQ25792_REG2A_CHG_TMR_MASK		BIT(3)
+#define BQ25792_REG2A_TRICHG_TMR_MASK		BIT(2)
+#define BQ25792_REG2A_PRECHG_TMR_MASK		BIT(1)
+#define BQ25792_REG2A_TOPOFF_TMR_MASK		BIT(0)
+
+/* Charger Mask 3 */
+#define BQ25792_REG2B_VBATOTG_LOW_MASK		BIT(4)
+#define BQ25792_REG2B_TS_COLD_MASK		BIT(3)
+#define BQ25792_REG2B_TS_COOL_MASK		BIT(2)
+#define BQ25792_REG2B_TS_WARM_MASK		BIT(1)
+#define BQ25792_REG2B_TS_HOT_MASK		BIT(0)
+
+/* FAULT Mask 0 */
+#define BQ25792_REG2C_IBAT_REG_MASK		BIT(7)
+#define BQ25792_REG2C_VBUS_OVP_MASK		BIT(6)
+#define BQ25792_REG2C_VBAT_OVP_MASK		BIT(5)
+#define BQ25792_REG2C_IBUS_OCP_MASK		BIT(4)
+#define BQ25792_REG2C_IBAT_OCP_MASK		BIT(3)
+#define BQ25792_REG2C_CONV_OCP_MASK		BIT(2)
+#define BQ25792_REG2C_VAC2_OVP_MASK		BIT(1)
+#define BQ25792_REG2C_VAC1_OVP_MASK		BIT(0)
+
+/* FAULT Mask 1 */
+#define BQ25792_REG2D_VSYS_SHORT_MASK		BIT(7)
+#define BQ25792_REG2D_VSYS_OVP_MASK		BIT(6)
+#define BQ25792_REG2D_OTG_OVP_MASK		BIT(5)
+#define BQ25792_REG2D_OTG_UVP_MASK		BIT(4)
+#define BQ25792_REG2D_TSHUT_MASK		BIT(2)
+
+/* ADC Control */
+#define BQ25792_REG2E_ADC_EN			BIT(7)
+#define BQ25792_REG2E_ADC_RATE			BIT(6)
+#define BQ25792_REG2E_ADC_SAMPLE_MASK		GENMASK(5, 4)
+#define BQ25792_REG2E_ADC_AVG			BIT(3)
+#define BQ25792_REG2E_ADC_AVG_INIT		BIT(2)
+
+/* ADC Function Disable 0 */
+#define BQ25792_REG2F_IBUS_ADC_DIS		BIT(7)
+#define BQ25792_REG2F_IBAT_ADC_DIS		BIT(6)
+#define BQ25792_REG2F_VBUS_ADC_DIS		BIT(5)
+#define BQ25792_REG2F_VBAT_ADC_DIS		BIT(4)
+#define BQ25792_REG2F_VSYS_ADC_DIS		BIT(3)
+#define BQ25792_REG2F_TS_ADC_DIS		BIT(2)
+#define BQ25792_REG2F_TDIE_ADC_DIS		BIT(1)
+
+/* ADC Function Disable 1 */
+#define BQ25792_REG30_DP_ADC_DIS		BIT(7)
+#define BQ25792_REG30_DM_ADC_DIS		BIT(6)
+#define BQ25792_REG30_VAC2_ADC_DIS		BIT(5)
+#define BQ25792_REG30_VAC1_ADC_DIS		BIT(4)
+
+/* 0x31-0x45: ADC result registers (16-bit, RO): single full-width field */
+
+#define BQ25792_ADCVSYSVBAT_STEP_UV		1000
+#define BQ25792_ADCIBAT_STEP_UA			1000
+
+/* DPDM Driver */
+#define BQ25792_REG47_DPLUS_DAC_MASK		GENMASK(7, 5)
+#define BQ25792_REG47_DMINUS_DAC_MASK		GENMASK(4, 2)
+
+/* Part Information */
+#define BQ25792_REG48_PN_MASK			GENMASK(5, 3)
+#define BQ25792_REG48_DEV_REV_MASK		GENMASK(2, 0)
+
+enum bq257xx_type {
+	BQ25703A = 1,
+	BQ25792,
+};
+
 struct bq257xx_device {
 	struct i2c_client *client;
 	struct regmap *regmap;
+	enum bq257xx_type type;
 };
diff --git a/include/linux/mfd/cs5535.h b/include/linux/mfd/cs5535.h
new file mode 100644
index 0000000000000..2e4ebf5d06af7
--- /dev/null
+++ b/include/linux/mfd/cs5535.h
@@ -0,0 +1,8 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef __MFD_CS5535_H__
+#define __MFD_CS5535_H__
+
+extern const struct software_node cs5535_gpio_swnode;
+
+#endif /* __MFD_CS5535_H__ */
diff --git a/include/linux/mfd/ezx-pcap.h b/include/linux/mfd/ezx-pcap.h
deleted file mode 100644
index ea51b1cdca5a2..0000000000000
--- a/include/linux/mfd/ezx-pcap.h
+++ /dev/null
@@ -1,253 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/*
- * Copyright 2009 Daniel Ribeiro <drwyrm@gmail.com>
- *
- * For further information, please see http://wiki.openezx.org/PCAP2
- */
-
-#ifndef EZX_PCAP_H
-#define EZX_PCAP_H
-
-struct pcap_subdev {
-	int id;
-	const char *name;
-	void *platform_data;
-};
-
-struct pcap_platform_data {
-	unsigned int irq_base;
-	unsigned int config;
-	int gpio;
-	void (*init) (void *);	/* board specific init */
-	int num_subdevs;
-	struct pcap_subdev *subdevs;
-};
-
-struct pcap_chip;
-
-int ezx_pcap_write(struct pcap_chip *, u8, u32);
-int ezx_pcap_read(struct pcap_chip *, u8, u32 *);
-int ezx_pcap_set_bits(struct pcap_chip *, u8, u32, u32);
-int pcap_to_irq(struct pcap_chip *, int);
-int irq_to_pcap(struct pcap_chip *, int);
-int pcap_adc_async(struct pcap_chip *, u8, u32, u8[], void *, void *);
-void pcap_set_ts_bits(struct pcap_chip *, u32);
-
-#define PCAP_SECOND_PORT	1
-#define PCAP_CS_AH		2
-
-#define PCAP_REGISTER_WRITE_OP_BIT	0x80000000
-#define PCAP_REGISTER_READ_OP_BIT	0x00000000
-
-#define PCAP_REGISTER_VALUE_MASK	0x01ffffff
-#define PCAP_REGISTER_ADDRESS_MASK	0x7c000000
-#define PCAP_REGISTER_ADDRESS_SHIFT	26
-#define PCAP_REGISTER_NUMBER		32
-#define PCAP_CLEAR_INTERRUPT_REGISTER	0x01ffffff
-#define PCAP_MASK_ALL_INTERRUPT		0x01ffffff
-
-/* registers accessible by both pcap ports */
-#define PCAP_REG_ISR		0x0	/* Interrupt Status */
-#define PCAP_REG_MSR		0x1	/* Interrupt Mask */
-#define PCAP_REG_PSTAT		0x2	/* Processor Status */
-#define PCAP_REG_VREG2		0x6	/* Regulator Bank 2 Control */
-#define PCAP_REG_AUXVREG	0x7	/* Auxiliary Regulator Control */
-#define PCAP_REG_BATT		0x8	/* Battery Control */
-#define PCAP_REG_ADC		0x9	/* AD Control */
-#define PCAP_REG_ADR		0xa	/* AD Result */
-#define PCAP_REG_CODEC		0xb	/* Audio Codec Control */
-#define PCAP_REG_RX_AMPS	0xc	/* RX Audio Amplifiers Control */
-#define PCAP_REG_ST_DAC		0xd	/* Stereo DAC Control */
-#define PCAP_REG_BUSCTRL	0x14	/* Connectivity Control */
-#define PCAP_REG_PERIPH		0x15	/* Peripheral Control */
-#define PCAP_REG_LOWPWR		0x18	/* Regulator Low Power Control */
-#define PCAP_REG_TX_AMPS	0x1a	/* TX Audio Amplifiers Control */
-#define PCAP_REG_GP		0x1b	/* General Purpose */
-#define PCAP_REG_TEST1		0x1c
-#define PCAP_REG_TEST2		0x1d
-#define PCAP_REG_VENDOR_TEST1	0x1e
-#define PCAP_REG_VENDOR_TEST2	0x1f
-
-/* registers accessible by pcap port 1 only (a1200, e2 & e6) */
-#define PCAP_REG_INT_SEL	0x3	/* Interrupt Select */
-#define PCAP_REG_SWCTRL		0x4	/* Switching Regulator Control */
-#define PCAP_REG_VREG1		0x5	/* Regulator Bank 1 Control */
-#define PCAP_REG_RTC_TOD	0xe	/* RTC Time of Day */
-#define PCAP_REG_RTC_TODA	0xf	/* RTC Time of Day Alarm */
-#define PCAP_REG_RTC_DAY	0x10	/* RTC Day */
-#define PCAP_REG_RTC_DAYA	0x11	/* RTC Day Alarm */
-#define PCAP_REG_MTRTMR		0x12	/* AD Monitor Timer */
-#define PCAP_REG_PWR		0x13	/* Power Control */
-#define PCAP_REG_AUXVREG_MASK	0x16	/* Auxiliary Regulator Mask */
-#define PCAP_REG_VENDOR_REV	0x17
-#define PCAP_REG_PERIPH_MASK	0x19	/* Peripheral Mask */
-
-/* PCAP2 Interrupts */
-#define PCAP_NIRQS		23
-#define PCAP_IRQ_ADCDONE	0	/* ADC done port 1 */
-#define PCAP_IRQ_TS		1	/* Touch Screen */
-#define PCAP_IRQ_1HZ		2	/* 1HZ timer */
-#define PCAP_IRQ_WH		3	/* ADC above high limit */
-#define PCAP_IRQ_WL		4	/* ADC below low limit */
-#define PCAP_IRQ_TODA		5	/* Time of day alarm */
-#define PCAP_IRQ_USB4V		6	/* USB above 4V */
-#define PCAP_IRQ_ONOFF		7	/* On/Off button */
-#define PCAP_IRQ_ONOFF2		8	/* On/Off button 2 */
-#define PCAP_IRQ_USB1V		9	/* USB above 1V */
-#define PCAP_IRQ_MOBPORT	10
-#define PCAP_IRQ_MIC		11	/* Mic attach/HS button */
-#define PCAP_IRQ_HS		12	/* Headset attach */
-#define PCAP_IRQ_ST		13
-#define PCAP_IRQ_PC		14	/* Power Cut */
-#define PCAP_IRQ_WARM		15
-#define PCAP_IRQ_EOL		16	/* Battery End Of Life */
-#define PCAP_IRQ_CLK		17
-#define PCAP_IRQ_SYSRST		18	/* System Reset */
-#define PCAP_IRQ_DUMMY		19
-#define PCAP_IRQ_ADCDONE2	20	/* ADC done port 2 */
-#define PCAP_IRQ_SOFTRESET	21
-#define PCAP_IRQ_MNEXB		22
-
-/* voltage regulators */
-#define V1		0
-#define V2		1
-#define V3		2
-#define V4		3
-#define V5		4
-#define V6		5
-#define V7		6
-#define V8		7
-#define V9		8
-#define V10		9
-#define VAUX1		10
-#define VAUX2		11
-#define VAUX3		12
-#define VAUX4		13
-#define VSIM		14
-#define VSIM2		15
-#define VVIB		16
-#define SW1		17
-#define SW2		18
-#define SW3		19
-#define SW1S		20
-#define SW2S		21
-
-#define PCAP_BATT_DAC_MASK		0x000000ff
-#define PCAP_BATT_DAC_SHIFT		0
-#define PCAP_BATT_B_FDBK		(1 << 8)
-#define PCAP_BATT_EXT_ISENSE		(1 << 9)
-#define PCAP_BATT_V_COIN_MASK		0x00003c00
-#define PCAP_BATT_V_COIN_SHIFT		10
-#define PCAP_BATT_I_COIN		(1 << 14)
-#define PCAP_BATT_COIN_CH_EN		(1 << 15)
-#define PCAP_BATT_EOL_SEL_MASK		0x000e0000
-#define PCAP_BATT_EOL_SEL_SHIFT		17
-#define PCAP_BATT_EOL_CMP_EN		(1 << 20)
-#define PCAP_BATT_BATT_DET_EN		(1 << 21)
-#define PCAP_BATT_THERMBIAS_CTRL	(1 << 22)
-
-#define PCAP_ADC_ADEN			(1 << 0)
-#define PCAP_ADC_RAND			(1 << 1)
-#define PCAP_ADC_AD_SEL1		(1 << 2)
-#define PCAP_ADC_AD_SEL2		(1 << 3)
-#define PCAP_ADC_ADA1_MASK		0x00000070
-#define PCAP_ADC_ADA1_SHIFT		4
-#define PCAP_ADC_ADA2_MASK		0x00000380
-#define PCAP_ADC_ADA2_SHIFT		7
-#define PCAP_ADC_ATO_MASK		0x00003c00
-#define PCAP_ADC_ATO_SHIFT		10
-#define PCAP_ADC_ATOX			(1 << 14)
-#define PCAP_ADC_MTR1			(1 << 15)
-#define PCAP_ADC_MTR2			(1 << 16)
-#define PCAP_ADC_TS_M_MASK		0x000e0000
-#define PCAP_ADC_TS_M_SHIFT		17
-#define PCAP_ADC_TS_REF_LOWPWR		(1 << 20)
-#define PCAP_ADC_TS_REFENB		(1 << 21)
-#define PCAP_ADC_BATT_I_POLARITY	(1 << 22)
-#define PCAP_ADC_BATT_I_ADC		(1 << 23)
-
-#define PCAP_ADC_BANK_0			0
-#define PCAP_ADC_BANK_1			1
-/* ADC bank 0 */
-#define PCAP_ADC_CH_COIN		0
-#define PCAP_ADC_CH_BATT		1
-#define PCAP_ADC_CH_BPLUS		2
-#define PCAP_ADC_CH_MOBPORTB		3
-#define PCAP_ADC_CH_TEMPERATURE		4
-#define PCAP_ADC_CH_CHARGER_ID		5
-#define PCAP_ADC_CH_AD6			6
-/* ADC bank 1 */
-#define PCAP_ADC_CH_AD7			0
-#define PCAP_ADC_CH_AD8			1
-#define PCAP_ADC_CH_AD9			2
-#define PCAP_ADC_CH_TS_X1		3
-#define PCAP_ADC_CH_TS_X2		4
-#define PCAP_ADC_CH_TS_Y1		5
-#define PCAP_ADC_CH_TS_Y2		6
-
-#define PCAP_ADC_T_NOW			0
-#define PCAP_ADC_T_IN_BURST		1
-#define PCAP_ADC_T_OUT_BURST		2
-
-#define PCAP_ADC_ATO_IN_BURST		6
-#define PCAP_ADC_ATO_OUT_BURST		0
-
-#define PCAP_ADC_TS_M_XY		1
-#define PCAP_ADC_TS_M_PRESSURE		2
-#define PCAP_ADC_TS_M_PLATE_X		3
-#define PCAP_ADC_TS_M_PLATE_Y		4
-#define PCAP_ADC_TS_M_STANDBY		5
-#define PCAP_ADC_TS_M_NONTS		6
-
-#define PCAP_ADR_ADD1_MASK		0x000003ff
-#define PCAP_ADR_ADD1_SHIFT		0
-#define PCAP_ADR_ADD2_MASK		0x000ffc00
-#define PCAP_ADR_ADD2_SHIFT		10
-#define PCAP_ADR_ADINC1			(1 << 20)
-#define PCAP_ADR_ADINC2			(1 << 21)
-#define PCAP_ADR_ASC			(1 << 22)
-#define PCAP_ADR_ONESHOT		(1 << 23)
-
-#define PCAP_BUSCTRL_FSENB		(1 << 0)
-#define PCAP_BUSCTRL_USB_SUSPEND	(1 << 1)
-#define PCAP_BUSCTRL_USB_PU		(1 << 2)
-#define PCAP_BUSCTRL_USB_PD		(1 << 3)
-#define PCAP_BUSCTRL_VUSB_EN		(1 << 4)
-#define PCAP_BUSCTRL_USB_PS		(1 << 5)
-#define PCAP_BUSCTRL_VUSB_MSTR_EN	(1 << 6)
-#define PCAP_BUSCTRL_VBUS_PD_ENB	(1 << 7)
-#define PCAP_BUSCTRL_CURRLIM		(1 << 8)
-#define PCAP_BUSCTRL_RS232ENB		(1 << 9)
-#define PCAP_BUSCTRL_RS232_DIR		(1 << 10)
-#define PCAP_BUSCTRL_SE0_CONN		(1 << 11)
-#define PCAP_BUSCTRL_USB_PDM		(1 << 12)
-#define PCAP_BUSCTRL_BUS_PRI_ADJ	(1 << 24)
-
-/* leds */
-#define PCAP_LED0		0
-#define PCAP_LED1		1
-#define PCAP_BL0		2
-#define PCAP_BL1		3
-#define PCAP_LED_3MA		0
-#define PCAP_LED_4MA		1
-#define PCAP_LED_5MA		2
-#define PCAP_LED_9MA		3
-#define PCAP_LED_T_MASK		0xf
-#define PCAP_LED_C_MASK		0x3
-#define PCAP_BL_MASK		0x1f
-#define PCAP_BL0_SHIFT		0
-#define PCAP_LED0_EN		(1 << 5)
-#define PCAP_LED1_EN		(1 << 6)
-#define PCAP_LED0_T_SHIFT	7
-#define PCAP_LED1_T_SHIFT	11
-#define PCAP_LED0_C_SHIFT	15
-#define PCAP_LED1_C_SHIFT	17
-#define PCAP_BL1_SHIFT		20
-
-/* RTC */
-#define PCAP_RTC_DAY_MASK	0x3fff
-#define PCAP_RTC_TOD_MASK	0xffff
-#define PCAP_RTC_PC_MASK	0x7
-#define SEC_PER_DAY		86400
-
-#endif
diff --git a/include/linux/mfd/max77759.h b/include/linux/mfd/max77759.h
index ec19be9528778..7c0b13219d515 100644
--- a/include/linux/mfd/max77759.h
+++ b/include/linux/mfd/max77759.h
@@ -106,9 +106,9 @@
 #define MAX77759_CHGR_REG_CHG_CNFG_10                  0xc3
 #define MAX77759_CHGR_REG_CHG_CNFG_11                  0xc4
 #define MAX77759_CHGR_REG_CHG_CNFG_12                  0xc5
-/* Wireless Charging input channel select */
+/* Setting this enables the Wireless Charging input channel. */
 #define   MAX77759_CHGR_REG_CHG_CNFG_12_WCINSEL        BIT(6)
-/* CHGIN/USB input channel select */
+/* Setting this enables the CHGIN/USB input channel. */
 #define   MAX77759_CHGR_REG_CHG_CNFG_12_CHGINSEL       BIT(5)
 #define MAX77759_CHGR_REG_CHG_CNFG_13                  0xc6
 #define MAX77759_CHGR_REG_CHG_CNFG_14                  0xc7
@@ -204,7 +204,7 @@ enum max77759_chgr_chg_dtls_states {
 };
 
 enum max77759_chgr_mode {
-	MAX77759_CHGR_MODE_OFF,
+	MAX77759_CHGR_MODE_OFF = 0x0,
 	MAX77759_CHGR_MODE_CHG_BUCK_ON = 0x5,
 	MAX77759_CHGR_MODE_OTG_BOOST_ON = 0xA,
 };
diff --git a/include/linux/mfd/rohm-bd72720.h b/include/linux/mfd/rohm-bd72720.h
index ae7343bcab064..d8ddbf232bb3e 100644
--- a/include/linux/mfd/rohm-bd72720.h
+++ b/include/linux/mfd/rohm-bd72720.h
@@ -21,7 +21,6 @@ enum {
 	BD72720_BUCK8,
 	BD72720_BUCK9,
 	BD72720_BUCK10,
-	BD72720_BUCK11,
 	BD72720_LDO1,
 	BD72720_LDO2,
 	BD72720_LDO3,
diff --git a/include/linux/mfd/wm8994/pdata.h b/include/linux/mfd/wm8994/pdata.h
index 6e2962ef5b811..b95a56a338c3b 100644
--- a/include/linux/mfd/wm8994/pdata.h
+++ b/include/linux/mfd/wm8994/pdata.h
@@ -226,11 +226,6 @@ struct wm8994_pdata {
 	 * lines is mastered.
 	 */
 	int max_channels_clocked[WM8994_NUM_AIF];
-
-	/**
-	 * GPIO for the IRQ pin if host only supports edge triggering
-	 */
-	int irq_gpio;
 };
 
 #endif
diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 04b96c5abb57e..9a4bb25d8e0ab 100644
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -343,9 +343,11 @@ struct mlx5_cmd_mailbox {
 	struct mlx5_cmd_mailbox *next;
 };
 
+struct mlx5_dma_pool_page;
 struct mlx5_buf_list {
 	void		       *buf;
 	dma_addr_t		map;
+	struct mlx5_dma_pool_page *frag_page;
 };
 
 struct mlx5_frag_buf {
@@ -545,6 +547,7 @@ struct mlx5_debugfs_entries {
 	struct dentry *eq_debugfs;
 	struct dentry *cq_debugfs;
 	struct dentry *cmdif_debugfs;
+	struct dentry *frag_buf_dma_pools_debugfs;
 	struct dentry *pages_debugfs;
 	struct dentry *lag_debugfs;
 };
@@ -556,8 +559,15 @@ enum mlx5_func_type {
 	MLX5_HOST_PF,
 	MLX5_EC_VF,
 	MLX5_FUNC_TYPE_NUM,
+	MLX5_FUNC_TYPE_NONE = MLX5_FUNC_TYPE_NUM,
 };
 
+enum mlx5_page_mgt_mode {
+	MLX5_PAGE_MGT_MODE_FUNC_ID,
+	MLX5_PAGE_MGT_MODE_VHCA_ID,
+};
+
+struct mlx5_frag_buf_node_pools;
 struct mlx5_ft_pool;
 struct mlx5_priv {
 	/* IRQ table valid only for real pci devices PF or VF */
@@ -575,20 +585,23 @@ struct mlx5_priv {
 	u32			fw_pages_alloc_failed;
 	u32			give_pages_dropped;
 	u32			reclaim_pages_discard;
+	enum mlx5_page_mgt_mode	page_mgt_mode;
 
 	struct mlx5_core_health health;
 	struct list_head	traps;
 
 	struct mlx5_debugfs_entries dbg;
 
-	/* start: alloc staff */
+	/* start: alloc stuff */
 	/* protect buffer allocation according to numa node */
 	struct mutex            alloc_mutex;
 	int                     numa_node;
 
 	struct mutex            pgdir_mutex;
 	struct list_head        pgdir_list;
-	/* end: alloc staff */
+
+	struct mlx5_frag_buf_node_pools **frag_buf_node_pools;
+	/* end: alloc stuff */
 
 	struct mlx5_adev       **adev;
 	int			adev_idx;
@@ -1034,6 +1047,8 @@ void mlx5_pagealloc_start(struct mlx5_core_dev *dev);
 void mlx5_pagealloc_stop(struct mlx5_core_dev *dev);
 void mlx5_pages_debugfs_init(struct mlx5_core_dev *dev);
 void mlx5_pages_debugfs_cleanup(struct mlx5_core_dev *dev);
+void mlx5_pages_by_func_type_debugfs_init(struct mlx5_core_dev *dev);
+void mlx5_pages_by_func_type_debugfs_cleanup(struct mlx5_core_dev *dev);
 int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot);
 int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev);
 void mlx5_register_debugfs(void);
diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h
index 67256e776566c..a0dd162baa787 100644
--- a/include/linux/mlx5/eswitch.h
+++ b/include/linux/mlx5/eswitch.h
@@ -63,7 +63,13 @@ struct mlx5_eswitch_rep {
 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
 				      const struct mlx5_eswitch_rep_ops *ops,
 				      u8 rep_type);
+void
+mlx5_eswitch_register_vport_reps_nested(struct mlx5_eswitch *esw,
+					const struct mlx5_eswitch_rep_ops *ops,
+					u8 rep_type);
 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type);
+void mlx5_eswitch_unregister_vport_reps_nested(struct mlx5_eswitch *esw,
+					       u8 rep_type);
 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
 				 u16 vport_num,
 				 u8 rep_type);
@@ -217,7 +223,7 @@ static inline bool is_mdev_switchdev_mode(struct mlx5_core_dev *dev)
 static inline u16 mlx5_eswitch_manager_vport(struct mlx5_core_dev *dev)
 {
 	return mlx5_core_is_ecpf_esw_manager(dev) ?
-		MLX5_VPORT_ECPF : MLX5_VPORT_PF;
+		MLX5_VPORT_ECPF : MLX5_VPORT_HOST_PF;
 }
 
 #endif
diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index 49f3ad4b1a7c5..6a675f918c40a 100644
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -1935,7 +1935,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         max_flow_counter_31_16[0x10];
 	u8         max_wqe_sz_sq_dc[0x10];
 
-	u8         reserved_at_2e0[0x7];
+	u8         query_host_net_function_num_max[0x5];
+	u8         reserved_at_2e5[0x2];
 	u8         max_qp_mcg[0x19];
 
 	u8         reserved_at_300[0x10];
@@ -2027,7 +2028,7 @@ struct mlx5_ifc_cmd_hca_cap_bits {
 	u8         log_max_current_mc_list[0x5];
 	u8         reserved_at_3f8[0x1];
 	u8         silent_mode_query[0x1];
-	u8         reserved_at_3fa[0x1];
+	u8         query_host_net_function_v1[0x1];
 	u8         log_max_current_uc_list[0x5];
 
 	u8         general_obj_types[0x40];
@@ -8452,7 +8453,9 @@ struct mlx5_ifc_enable_hca_in_bits {
 	u8         op_mod[0x10];
 
 	u8         embedded_cpu_function[0x1];
-	u8         reserved_at_41[0xf];
+	u8         reserved_at_41[0x2];
+	u8         function_id_type[0x1];
+	u8         reserved_at_44[0xc];
 	u8         function_id[0x10];
 
 	u8         reserved_at_60[0x20];
@@ -8497,7 +8500,9 @@ struct mlx5_ifc_disable_hca_in_bits {
 	u8         op_mod[0x10];
 
 	u8         embedded_cpu_function[0x1];
-	u8         reserved_at_41[0xf];
+	u8         reserved_at_41[0x2];
+	u8         function_id_type[0x1];
+	u8         reserved_at_44[0xc];
 	u8         function_id[0x10];
 
 	u8         reserved_at_60[0x20];
@@ -12700,6 +12705,54 @@ struct mlx5_ifc_host_params_context_bits {
 	u8         reserved_at_80[0x180];
 };
 
+enum mlx5_ifc_vhca_state {
+	MLX5_VHCA_STATE_INVALID = 0x0,
+	MLX5_VHCA_STATE_ALLOCATED = 0x1,
+	MLX5_VHCA_STATE_ACTIVE = 0x2,
+	MLX5_VHCA_STATE_IN_USE = 0x3,
+	MLX5_VHCA_STATE_TEARDOWN_REQUEST = 0x4,
+};
+
+enum {
+	MLX5_PCI_PF_TYPE_EXTERNAL_HOST_PF = 0x0,
+	MLX5_PCI_PF_TYPE_SATELLITE_PF = 0x1,
+};
+
+struct mlx5_ifc_network_function_params_bits {
+	u8         host_number[0x8];
+	u8         pci_pf_type[0x4];
+	u8         reserved_at_c[0x4];
+	u8         pci_num_vfs[0x10];
+
+	u8         pci_total_vfs[0x10];
+	u8         pci_bus[0x8];
+	u8         pci_device_function[0x8];
+
+	u8         vhca_id[0x10];
+	u8         vhca_state[0x4];
+	u8         reserved_at_54[0xc];
+
+	u8         reserved_at_60[0xa];
+	u8         esw_vport_manual[0x1];
+	u8         pci_bus_assigned[0x1];
+	u8         pci_vf_info_valid[0x1];
+	u8         reserved_at_6d[0x13];
+
+	u8         pci_vf_stride[0x10];
+	u8         pci_first_vf_offset[0x10];
+
+	u8         reserved_at_a0[0x160];
+};
+
+union mlx5_ifc_net_function_params_bits {
+	struct mlx5_ifc_host_params_context_bits host_params_context;
+	struct mlx5_ifc_network_function_params_bits network_function_params;
+};
+
+enum {
+	MLX5_QUERY_ESW_FUNC_OP_MOD_LAYOUT_V1 = BIT(14),
+};
+
 struct mlx5_ifc_query_esw_functions_in_bits {
 	u8         opcode[0x10];
 	u8         reserved_at_10[0x10];
@@ -12716,12 +12769,16 @@ struct mlx5_ifc_query_esw_functions_out_bits {
 
 	u8         syndrome[0x20];
 
-	u8         reserved_at_40[0x40];
+	u8         reserved_at_40[0x20];
 
-	struct mlx5_ifc_host_params_context_bits host_params_context;
+	u8         net_function_num[0x8];
+	u8         reserved_at_68[0x18];
 
-	u8         reserved_at_280[0x180];
-	u8         host_sf_enable[][0x40];
+	union {
+		u8 reserved_at_80[0x380];
+		DECLARE_FLEX_ARRAY(union mlx5_ifc_net_function_params_bits,
+				   net_function_params);
+	};
 };
 
 struct mlx5_ifc_sf_partition_bits {
diff --git a/include/linux/mlx5/vport.h b/include/linux/mlx5/vport.h
index dfa2fe32217af..90641f67da468 100644
--- a/include/linux/mlx5/vport.h
+++ b/include/linux/mlx5/vport.h
@@ -51,8 +51,8 @@ enum {
 
 /* Vport number for each function must keep unchanged */
 enum {
-	MLX5_VPORT_PF			= 0x0,
-	MLX5_VPORT_FIRST_VF		= 0x1,
+	MLX5_VPORT_HOST_PF		= 0x0,
+	MLX5_VPORT_FIRST_HOST_VF	= 0x1,
 	MLX5_VPORT_ECPF			= 0xfffe,
 	MLX5_VPORT_UPLINK		= 0xffff
 };
diff --git a/include/linux/mm.h b/include/linux/mm.h
index af23453e9dbd0..9cedc5e75aa93 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1888,16 +1888,6 @@ static inline bool folio_mapped(const struct folio *folio)
 	return folio_mapcount(folio) >= 1;
 }
 
-/*
- * Return true if this page is mapped into pagetables.
- * For compound page it returns true if any sub-page of compound page is mapped,
- * even if this particular sub-page is not itself mapped by any PTE or PMD.
- */
-static inline bool page_mapped(const struct page *page)
-{
-	return folio_mapped(page_folio(page));
-}
-
 static inline struct page *virt_to_head_page(const void *x)
 {
 	struct page *page = virt_to_page(x);
@@ -4855,18 +4845,10 @@ static inline void print_vma_addr(char *prefix, unsigned long rip)
 }
 #endif
 
-void *sparse_buffer_alloc(unsigned long size);
 unsigned long section_map_size(void);
 struct page * __populate_section_memmap(unsigned long pfn,
 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
 		struct dev_pagemap *pgmap);
-pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
-p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node);
-pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node);
-pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
-pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
-			    struct vmem_altmap *altmap, unsigned long ptpfn,
-			    unsigned long flags);
 void *vmemmap_alloc_block(unsigned long size, int node);
 struct vmem_altmap;
 void *vmemmap_alloc_block_buf(unsigned long size, int node,
@@ -5174,9 +5156,10 @@ int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
  * DMA mapping IDs for page_pool
  *
  * When DMA-mapping a page, page_pool allocates an ID (from an xarray) and
- * stashes it in the upper bits of page->pp_magic. Non-PP pages can have
- * arbitrary kernel pointers stored in the same field as pp_magic (since
- * it overlaps with page->lru.next), so we must ensure that we cannot
+ * stashes it in the upper bits of page->pp_magic. We always want to be able to
+ * unambiguously identify page pool pages (using page_pool_page_is_pp()). Non-PP
+ * pages can have arbitrary kernel pointers stored in the same field as pp_magic
+ * (since it overlaps with page->lru.next), so we must ensure that we cannot
  * mistake a valid kernel pointer with any of the values we write into this
  * field.
  *
@@ -5211,6 +5194,26 @@ int arch_lock_shadow_stack_status(struct task_struct *t, unsigned long status);
 #define PP_DMA_INDEX_MASK GENMASK(PP_DMA_INDEX_BITS + PP_DMA_INDEX_SHIFT - 1, \
 				  PP_DMA_INDEX_SHIFT)
 
+/* Mask used for checking in page_pool_page_is_pp() below. page->pp_magic is
+ * OR'ed with PP_SIGNATURE after the allocation in order to preserve bit 0 for
+ * the head page of compound page and bit 1 for pfmemalloc page, as well as the
+ * bits used for the DMA index. page_is_pfmemalloc() is checked in
+ * __page_pool_put_page() to avoid recycling the pfmemalloc page.
+ */
+#define PP_MAGIC_MASK ~(PP_DMA_INDEX_MASK | 0x3UL)
+
+#ifdef CONFIG_PAGE_POOL
+static inline bool page_pool_page_is_pp(const struct page *page)
+{
+	return (page->pp_magic & PP_MAGIC_MASK) == PP_SIGNATURE;
+}
+#else
+static inline bool page_pool_page_is_pp(const struct page *page)
+{
+	return false;
+}
+#endif
+
 #define PAGE_SNAPSHOT_FAITHFUL (1 << 0)
 #define PAGE_SNAPSHOT_PG_BUDDY (1 << 1)
 #define PAGE_SNAPSHOT_PG_IDLE  (1 << 2)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index a308e2c23b829..c7db35be6a30a 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1222,6 +1222,8 @@ struct mm_struct {
 		/* MM CID related storage */
 		struct mm_mm_cid mm_cid;
 
+		/* sched_cache related statistics */
+		struct sched_cache_stat sc_stat;
 #ifdef CONFIG_MMU
 		atomic_long_t pgtables_bytes;	/* size of all page tables */
 #endif
@@ -1628,6 +1630,36 @@ static inline unsigned int mm_cid_size(void)
 # define MM_CID_STATIC_SIZE	0
 #endif /* CONFIG_SCHED_MM_CID */
 
+#ifdef CONFIG_SCHED_CACHE
+void mm_init_sched(struct mm_struct *mm,
+		   struct sched_cache_time __percpu *pcpu_sched);
+
+static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
+{
+	struct sched_cache_time __percpu *pcpu_sched =
+		alloc_percpu_noprof(struct sched_cache_time);
+
+	if (!pcpu_sched)
+		return -ENOMEM;
+
+	mm_init_sched(mm, pcpu_sched);
+	return 0;
+}
+
+#define mm_alloc_sched(...)	alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__))
+
+static inline void mm_destroy_sched(struct mm_struct *mm)
+{
+	free_percpu(mm->sc_stat.pcpu_sched);
+	mm->sc_stat.pcpu_sched = NULL;
+}
+#else /* !CONFIG_SCHED_CACHE */
+
+static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; }
+static inline void mm_destroy_sched(struct mm_struct *mm) { }
+
+#endif /* CONFIG_SCHED_CACHE */
+
 struct mmu_gather;
 extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm);
 extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm);
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 69c304b467dfb..a11a44eef521e 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -134,8 +134,8 @@ struct mmu_notifier_ops {
 	 * Invalidation of multiple concurrent ranges may be
 	 * optionally permitted by the driver. Either way the
 	 * establishment of sptes is forbidden in the range passed to
-	 * invalidate_range_begin/end for the whole duration of the
-	 * invalidate_range_begin/end critical section.
+	 * invalidate_range_start/end for the whole duration of the
+	 * invalidate_range_start/end critical section.
 	 *
 	 * invalidate_range_start() is called when all pages in the
 	 * range are still mapped and have at least a refcount of one.
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 9adb2ad21da59..1331a7b93f33c 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -177,9 +177,12 @@ static inline bool migratetype_is_mergeable(int mt)
 	return mt < MIGRATE_PCPTYPES;
 }
 
-#define for_each_migratetype_order(order, type) \
-	for (order = 0; order < NR_PAGE_ORDERS; order++) \
-		for (type = 0; type < MIGRATE_TYPES; type++)
+#define for_each_free_list(list, zone, order) 				\
+	for (order = 0; order < NR_PAGE_ORDERS; order++) 		\
+		for (unsigned int __type = 0; 				\
+		     __type < MIGRATE_TYPES &&				\
+			(list = &(zone)->free_area[order].free_list[__type], 1); \
+		     __type++)
 
 extern int page_group_by_mobility_disabled;
 
diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h
index 23ff24080dfd9..3b0c9a251a2e9 100644
--- a/include/linux/mod_devicetable.h
+++ b/include/linux/mod_devicetable.h
@@ -61,7 +61,10 @@ struct ieee1394_device_id {
 	__u32 model_id;
 	__u32 specifier_id;
 	__u32 version;
-	kernel_ulong_t driver_data;
+	union {
+		kernel_ulong_t driver_data;
+		const void *driver_data_ptr;
+	};
 };
 
 
diff --git a/include/linux/msi.h b/include/linux/msi.h
index fa41eed628682..a4613de11960d 100644
--- a/include/linux/msi.h
+++ b/include/linux/msi.h
@@ -444,7 +444,7 @@ struct msi_domain_info;
  *
  * @domain_alloc_irqs, @domain_free_irqs can be used to override the
  * default allocation/free functions (__msi_domain_alloc/free_irqs). This
- * is initially for a wrapper around XENs seperate MSI universe which can't
+ * is initially for a wrapper around XEN's separate MSI universe which can't
  * be wrapped into the regular irq domains concepts by mere mortals.  This
  * allows to universally use msi_domain_alloc/free_irqs without having to
  * special case XEN all over the place.
diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h
index 782984ba3a209..44f4347104d66 100644
--- a/include/linux/mtd/spinand.h
+++ b/include/linux/mtd/spinand.h
@@ -583,6 +583,7 @@ enum spinand_bus_interface {
  * @op_variants.read_cache: variants of the read-cache operation
  * @op_variants.write_cache: variants of the write-cache operation
  * @op_variants.update_cache: variants of the update-cache operation
+ * @op_variants.cont_read_cache: variants of the continuous read-cache operation
  * @vendor_ops: vendor specific operations
  * @select_target: function used to select a target/die. Required only for
  *		   multi-die chips
@@ -607,6 +608,7 @@ struct spinand_info {
 		const struct spinand_op_variants *read_cache;
 		const struct spinand_op_variants *write_cache;
 		const struct spinand_op_variants *update_cache;
+		const struct spinand_op_variants *cont_read_cache;
 	} op_variants;
 	const struct spinand_op_variants *vendor_ops;
 	int (*select_target)(struct spinand_device *spinand,
@@ -636,6 +638,14 @@ struct spinand_info {
 		.update_cache = __update,				\
 	}
 
+#define SPINAND_INFO_OP_VARIANTS_WITH_CONT(__read, __write, __update, __cont_read) \
+	{								\
+		.read_cache = __read,					\
+		.write_cache = __write,					\
+		.update_cache = __update,				\
+		.cont_read_cache = __cont_read,				\
+	}
+
 #define SPINAND_INFO_VENDOR_OPS(__ops)					\
 	.vendor_ops = __ops
 
@@ -691,8 +701,6 @@ struct spinand_info {
 struct spinand_dirmap {
 	struct spi_mem_dirmap_desc *wdesc;
 	struct spi_mem_dirmap_desc *rdesc;
-	struct spi_mem_dirmap_desc *wdesc_ecc;
-	struct spi_mem_dirmap_desc *rdesc_ecc;
 };
 
 /**
@@ -709,6 +717,7 @@ struct spinand_dirmap {
  * @read_cache: read cache op template
  * @write_cache: write cache op template
  * @update_cache: update cache op template
+ * @cont_read_cache: continuous read cache op template (optional)
  */
 struct spinand_mem_ops {
 	struct spi_mem_op reset;
@@ -723,6 +732,7 @@ struct spinand_mem_ops {
 	const struct spi_mem_op *read_cache;
 	const struct spi_mem_op *write_cache;
 	const struct spi_mem_op *update_cache;
+	const struct spi_mem_op *cont_read_cache;
 };
 
 /**
@@ -869,6 +879,8 @@ static inline void spinand_set_of_node(struct spinand_device *spinand,
 	nanddev_set_of_node(&spinand->base, np);
 }
 
+bool spinand_op_is_odtr(const struct spi_mem_op *op);
+
 int spinand_match_and_init(struct spinand_device *spinand,
 			   const struct spinand_info *table,
 			   unsigned int table_size,
diff --git a/include/linux/mux/consumer.h b/include/linux/mux/consumer.h
index a961861a503b3..449e38e6e2c51 100644
--- a/include/linux/mux/consumer.h
+++ b/include/linux/mux/consumer.h
@@ -60,7 +60,10 @@ struct mux_control *mux_control_get_optional(struct device *dev, const char *mux
 void mux_control_put(struct mux_control *mux);
 
 struct mux_control *devm_mux_control_get(struct device *dev, const char *mux_name);
-struct mux_state *devm_mux_state_get(struct device *dev, const char *mux_name);
+
+struct mux_state *
+devm_mux_state_get_from_np(struct device *dev, const char *mux_name, struct device_node *np);
+
 struct mux_state *devm_mux_state_get_optional(struct device *dev, const char *mux_name);
 struct mux_state *devm_mux_state_get_selected(struct device *dev, const char *mux_name);
 struct mux_state *devm_mux_state_get_optional_selected(struct device *dev, const char *mux_name);
@@ -161,4 +164,7 @@ static inline struct mux_state *devm_mux_state_get_optional_selected(struct devi
 
 #endif /* CONFIG_MULTIPLEXER */
 
+#define devm_mux_state_get(dev, mux_name)		\
+	devm_mux_state_get_from_np(dev, mux_name, NULL)
+
 #endif /* _LINUX_MUX_CONSUMER_H */
diff --git a/include/linux/namei.h b/include/linux/namei.h
index 2ad6dd9987b90..80488b3de0c98 100644
--- a/include/linux/namei.h
+++ b/include/linux/namei.h
@@ -61,7 +61,6 @@ extern struct dentry *start_creating_path(int, const char *, struct path *, unsi
 extern struct dentry *start_creating_user_path(int, const char __user *, struct path *, unsigned int);
 extern void end_creating_path(const struct path *, struct dentry *);
 extern struct dentry *start_removing_path(const char *, struct path *);
-extern struct dentry *start_removing_user_path_at(int , const char __user *, struct path *);
 static inline void end_removing_path(const struct path *path , struct dentry *dentry)
 {
 	end_creating_path(path, dentry);
diff --git a/include/linux/net/intel/libie/adminq.h b/include/linux/net/intel/libie/adminq.h
index ab13bd777a285..839114d8975a6 100644
--- a/include/linux/net/intel/libie/adminq.h
+++ b/include/linux/net/intel/libie/adminq.h
@@ -196,6 +196,7 @@ LIBIE_CHECK_STRUCT_LEN(16, libie_aqc_list_caps);
 #define LIBIE_AQC_BIT_ROCEV2_LAG			BIT(0)
 #define LIBIE_AQC_BIT_SRIOV_LAG				BIT(1)
 #define LIBIE_AQC_BIT_SRIOV_AA_LAG			BIT(2)
+#define LIBIE_AQC_CAPS_EEE				0x009B
 #define LIBIE_AQC_CAPS_FLEX10				0x00F1
 #define LIBIE_AQC_CAPS_CEM				0x00F2
 
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 0e1e581efc5ac..ee47f152b6674 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -1223,6 +1223,12 @@ struct netdev_net_notifier {
  *	tx queues stopped. This allows the netdevice to perform queue
  *	management safely.
  *
+ *	NB: Returning -EOPNOTSUPP for whatever commands means "this qdisc
+ *	is not offloaded (anymore, offloading may have silently stopped)",
+ *	and the offloading flag is cleared. Notably, this is also true for
+ *	dump queries (e.g. TC_*_STATS commands). If the underlying device does
+ *	not report any statistics but is still offloading, return 0 instead.
+ *
  *	Fiber Channel over Ethernet (FCoE) offload functions.
  * int (*ndo_fcoe_enable)(struct net_device *dev);
  *	Called when the FCoE protocol stack wants to start using LLD for FCoE
@@ -1788,6 +1794,12 @@ enum netdev_stat_type {
 	NETDEV_PCPU_STAT_DSTATS, /* struct pcpu_dstats */
 };
 
+enum netmem_tx_mode {
+	NETMEM_TX_NONE,		/* no netmem TX support */
+	NETMEM_TX_DMA,		/* DMA-capable netmem TX (real HW) */
+	NETMEM_TX_NO_DMA,	/* no DMA, e.g. passthrough for virtual devs */
+};
+
 enum netdev_reg_state {
 	NETREG_UNINITIALIZED = 0,
 	NETREG_REGISTERED,	/* completed register_netdevice */
@@ -1809,7 +1821,7 @@ enum netdev_reg_state {
  *	@lltx:		device supports lockless Tx. Deprecated for real HW
  *			drivers. Mainly used by logical interfaces, such as
  *			bonding and tunnels
- *	@netmem_tx:	device support netmem_tx.
+ *	@netmem_tx:	device netmem TX mode
  *
  *	@name:	This is the first field of the "visible" part of this structure
  *		(i.e. as seen by users in the "Space.c" file).  It is the name
@@ -2119,6 +2131,8 @@ enum netdev_reg_state {
  *
  *	FIXME: cleanup struct net_device such that network protocol info
  *	moves out.
+ *
+ *	@netdev_trace_buffer_list: Linked list for debugging refcount leak.
  */
 
 struct net_device {
@@ -2132,7 +2146,7 @@ struct net_device {
 	struct_group(priv_flags_fast,
 		unsigned long		priv_flags:32;
 		unsigned long		lltx:1;
-		unsigned long		netmem_tx:1;
+		unsigned long		netmem_tx:2;
 	);
 	const struct net_device_ops *netdev_ops;
 	const struct header_ops *header_ops;
@@ -2275,6 +2289,9 @@ struct net_device {
 #if IS_ENABLED(CONFIG_TLS_DEVICE)
 	const struct tlsdev_ops *tlsdev_ops;
 #endif
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+	struct list_head	netdev_trace_buffer_list;
+#endif
 
 	unsigned int		operstate;
 	unsigned char		link_mode;
@@ -3219,6 +3236,7 @@ enum netdev_cmd {
 	NETDEV_OFFLOAD_XSTATS_REPORT_USED,
 	NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
 	NETDEV_XDP_FEAT_CHANGE,
+	NETDEV_DEBUG_UNREGISTER,
 };
 const char *netdev_cmd_to_name(enum netdev_cmd cmd);
 
@@ -3378,7 +3396,6 @@ static inline struct net_device *first_net_device(struct net *net)
 		net_device_entry(net->dev_base_head.next);
 }
 
-int netdev_boot_setup_check(struct net_device *dev);
 struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type,
 				   const char *hwaddr);
 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
@@ -4441,9 +4458,15 @@ static inline bool dev_nit_active(const struct net_device *dev)
 
 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
 
+void save_netdev_trace_buffer(struct net_device *dev, int delta);
+int trim_netdev_trace(unsigned long *entries, int nr_entries);
+
 static inline void __dev_put(struct net_device *dev)
 {
 	if (dev) {
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+		save_netdev_trace_buffer(dev, -1);
+#endif
 #ifdef CONFIG_PCPU_DEV_REFCNT
 		this_cpu_dec(*dev->pcpu_refcnt);
 #else
@@ -4455,6 +4478,9 @@ static inline void __dev_put(struct net_device *dev)
 static inline void __dev_hold(struct net_device *dev)
 {
 	if (dev) {
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+		save_netdev_trace_buffer(dev, 1);
+#endif
 #ifdef CONFIG_PCPU_DEV_REFCNT
 		this_cpu_inc(*dev->pcpu_refcnt);
 #else
diff --git a/include/linux/netfs.h b/include/linux/netfs.h
index ba17ac5bf356a..243c0f7379388 100644
--- a/include/linux/netfs.h
+++ b/include/linux/netfs.h
@@ -62,8 +62,8 @@ struct netfs_inode {
 	struct fscache_cookie	*cache;
 #endif
 	struct mutex		wb_lock;	/* Writeback serialisation */
-	loff_t			remote_i_size;	/* Size of the remote file */
-	loff_t			zero_point;	/* Size after which we assume there's no data
+	loff_t			_remote_i_size;	/* Size of the remote file */
+	loff_t			_zero_point;	/* Size after which we assume there's no data
 						 * on the server */
 	atomic_t		io_count;	/* Number of outstanding reqs */
 	unsigned long		flags;
@@ -252,7 +252,7 @@ struct netfs_io_request {
 	unsigned long long	collected_to;	/* Point we've collected to */
 	unsigned long long	cleaned_to;	/* Position we've cleaned folios to */
 	unsigned long long	abandon_to;	/* Position to abandon folios to */
-	pgoff_t			no_unlock_folio; /* Don't unlock this folio after read */
+	const struct folio	*no_unlock_folio; /* Don't unlock this folio after read */
 	unsigned int		direct_bv_count; /* Number of elements in direct_bv[] */
 	unsigned int		debug_id;
 	unsigned int		rsize;		/* Maximum read size (0 for none) */
@@ -474,6 +474,254 @@ static inline struct netfs_inode *netfs_inode(struct inode *inode)
 	return container_of(inode, struct netfs_inode, inode);
 }
 
+/**
+ * netfs_read_remote_i_size - Read remote_i_size safely
+ * @inode: The inode to access
+ *
+ * Read remote_i_size safely without the potential for tearing on 32-bit
+ * arches.
+ *
+ * NOTE: in a 32bit arch with a preemptable kernel and an UP compile the
+ * i_size_read/write must be atomic with respect to the local cpu (unlike with
+ * preempt disabled), but they don't need to be atomic with respect to other
+ * cpus like in true SMP (so they need either to either locally disable irq
+ * around the read or for example on x86 they can be still implemented as a
+ * cmpxchg8b without the need of the lock prefix).  For SMP compiles and 64bit
+ * archs it makes no difference if preempt is enabled or not.
+ */
+static inline unsigned long long netfs_read_remote_i_size(const struct inode *inode)
+{
+	const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode);
+	unsigned long long remote_i_size;
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	unsigned int seq;
+
+	do {
+		seq = read_seqcount_begin(&inode->i_size_seqcount);
+		remote_i_size = ictx->_remote_i_size;
+	} while (read_seqcount_retry(&inode->i_size_seqcount, seq));
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+	preempt_disable();
+	remote_i_size = ictx->_remote_i_size;
+	preempt_enable();
+#else
+	/* Pairs with smp_store_release() in netfs_write_remote_i_size() */
+	remote_i_size = smp_load_acquire(&ictx->_remote_i_size);
+#endif
+	return remote_i_size;
+}
+
+/*
+ * netfs_write_remote_i_size - Set remote_i_size safely
+ * @inode: The inode to access
+ * @remote_i_size: The new value for the size of the file on the server
+ *
+ * Set remote_i_size safely without the potential for tearing on 32-bit arches.
+ *
+ * Context: The caller must hold inode->i_lock.
+ *
+ * NOTE: unlike netfs_read_remote_i_size(), netfs_write_remote_i_size() does
+ * need locking around it (normally i_rwsem), otherwise on 32bit/SMP an update
+ * of i_size_seqcount can be lost, resulting in subsequent i_size_read() calls
+ * spinning forever.
+ */
+static inline void netfs_write_remote_i_size(struct inode *inode,
+					     unsigned long long remote_i_size)
+{
+	struct netfs_inode *ictx = netfs_inode(inode);
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	write_seqcount_begin(&inode->i_size_seqcount);
+	ictx->_remote_i_size = remote_i_size;
+	write_seqcount_end(&inode->i_size_seqcount);
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+	preempt_disable();
+	ictx->_remote_i_size = remote_i_size;
+	preempt_enable();
+#else
+	/*
+	 * Pairs with smp_load_acquire() in netfs_read_remote_i_size() to
+	 * ensure changes related to inode size (such as page contents) are
+	 * visible before we see the changed inode size.
+	 */
+	smp_store_release(&ictx->_remote_i_size, remote_i_size);
+#endif
+}
+
+/**
+ * netfs_read_zero_point - Read zero_point safely
+ * @inode: The inode to access
+ *
+ * Read zero_point safely without the potential for tearing on 32-bit
+ * arches.
+ *
+ * NOTE: in a 32bit arch with a preemptable kernel and an UP compile the
+ * i_size_read/write must be atomic with respect to the local cpu (unlike with
+ * preempt disabled), but they don't need to be atomic with respect to other
+ * cpus like in true SMP (so they need either to either locally disable irq
+ * around the read or for example on x86 they can be still implemented as a
+ * cmpxchg8b without the need of the lock prefix).  For SMP compiles and 64bit
+ * archs it makes no difference if preempt is enabled or not.
+ */
+static inline unsigned long long netfs_read_zero_point(const struct inode *inode)
+{
+	struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode);
+	unsigned long long zero_point;
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	unsigned int seq;
+
+	do {
+		seq = read_seqcount_begin(&inode->i_size_seqcount);
+		zero_point = ictx->_zero_point;
+	} while (read_seqcount_retry(&inode->i_size_seqcount, seq));
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+	preempt_disable();
+	zero_point = ictx->_zero_point;
+	preempt_enable();
+#else
+	/* Pairs with smp_store_release() in netfs_write_zero_point() */
+	zero_point = smp_load_acquire(&ictx->_zero_point);
+#endif
+	return zero_point;
+}
+
+/*
+ * netfs_write_zero_point - Set zero_point safely
+ * @inode: The inode to access
+ * @zero_point: The new value for the point beyond which the server has no data
+ *
+ * Set zero_point safely without the potential for tearing on 32-bit arches.
+ *
+ * Context: The caller must hold inode->i_lock.
+ *
+ * NOTE: unlike netfs_read_zero_point(), netfs_write_zero_point() does need
+ * locking around it (normally i_rwsem), otherwise on 32bit/SMP an update of
+ * i_size_seqcount can be lost, resulting in subsequent read calls spinning
+ * forever.
+ */
+static inline void netfs_write_zero_point(struct inode *inode,
+					  unsigned long long zero_point)
+{
+	struct netfs_inode *ictx = netfs_inode(inode);
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	write_seqcount_begin(&inode->i_size_seqcount);
+	ictx->_zero_point = zero_point;
+	write_seqcount_end(&inode->i_size_seqcount);
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+	preempt_disable();
+	ictx->_zero_point = zero_point;
+	preempt_enable();
+#else
+	/*
+	 * Pairs with smp_load_acquire() in netfs_read_zero_point() to
+	 * ensure changes related to inode size (such as page contents) are
+	 * visible before we see the changed inode size.
+	 */
+	smp_store_release(&ictx->_zero_point, zero_point);
+#endif
+}
+
+/**
+ * netfs_read_sizes - Read remote_i_size and zero_point safely
+ * @inode: The inode to access
+ * @i_size: Where to return the local file size.
+ * @remote_i_size: Where to return the size of the file on the server
+ * @zero_point: Where to return the the point beyond which the server has no data
+ *
+ * Read remote_i_size and zero_point safely without the potential for tearing
+ * on 32-bit arches.
+ *
+ * NOTE: in a 32bit arch with a preemptable kernel and an UP compile the
+ * i_size_read/write must be atomic with respect to the local cpu (unlike with
+ * preempt disabled), but they don't need to be atomic with respect to other
+ * cpus like in true SMP (so they need either to either locally disable irq
+ * around the read or for example on x86 they can be still implemented as a
+ * cmpxchg8b without the need of the lock prefix).  For SMP compiles and 64bit
+ * archs it makes no difference if preempt is enabled or not.
+ */
+static inline void netfs_read_sizes(const struct inode *inode,
+				    unsigned long long *i_size,
+				    unsigned long long *remote_i_size,
+				    unsigned long long *zero_point)
+{
+	const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode);
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	unsigned int seq;
+
+	do {
+		seq = read_seqcount_begin(&inode->i_size_seqcount);
+		*i_size = inode->i_size;
+		*remote_i_size = ictx->_remote_i_size;
+		*zero_point = ictx->_zero_point;
+	} while (read_seqcount_retry(&inode->i_size_seqcount, seq));
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+	preempt_disable();
+	*i_size = inode->i_size;
+	*remote_i_size = ictx->_remote_i_size;
+	*zero_point = ictx->_zero_point;
+	preempt_enable();
+#else
+	/* Pairs with smp_store_release() in i_size_write() */
+	*i_size = smp_load_acquire(&inode->i_size);
+	/* Pairs with smp_store_release() in netfs_write_remote_i_size() */
+	*remote_i_size = smp_load_acquire(&ictx->_remote_i_size);
+	/* Pairs with smp_store_release() in netfs_write_zero_point() */
+	*zero_point = smp_load_acquire(&ictx->_zero_point);
+#endif
+}
+
+/*
+ * netfs_write_sizes - Set i_size, remote_i_size and zero_point safely
+ * @inode: The inode to access
+ * @i_size: The new value for the local size of the file
+ * @remote_i_size: The new value for the size of the file on the server
+ * @zero_point: The new value for the point beyond which the server has no data
+ *
+ * Set both remote_i_size and zero_point safely without the potential for
+ * tearing on 32-bit arches.
+ *
+ * Context: The caller must hold inode->i_lock.
+ *
+ * NOTE: unlike netfs_read_zero_point(), netfs_write_zero_point() does need
+ * locking around it (normally i_rwsem), otherwise on 32bit/SMP an update of
+ * i_size_seqcount can be lost, resulting in subsequent read calls spinning
+ * forever.
+ */
+static inline void netfs_write_sizes(struct inode *inode,
+				     unsigned long long i_size,
+				     unsigned long long remote_i_size,
+				     unsigned long long zero_point)
+{
+	struct netfs_inode *ictx = netfs_inode(inode);
+
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	write_seqcount_begin(&inode->i_size_seqcount);
+	inode->i_size = i_size;
+	ictx->_remote_i_size = remote_i_size;
+	ictx->_zero_point = zero_point;
+	write_seqcount_end(&inode->i_size_seqcount);
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+	preempt_disable();
+	inode->i_size = i_size;
+	ictx->_remote_i_size = remote_i_size;
+	ictx->_zero_point = zero_point;
+	preempt_enable();
+#else
+	/*
+	 * Pairs with smp_load_acquire() in i_size_read(),
+	 * netfs_read_remote_i_size() and netfs_read_zero_point() to ensure
+	 * changes related to inode size (such as page contents) are visible
+	 * before we see the changed inode size.
+	 */
+	smp_store_release(&inode->i_size, i_size);
+	smp_store_release(&ictx->_remote_i_size, remote_i_size);
+	smp_store_release(&ictx->_zero_point, zero_point);
+#endif
+}
+
 /**
  * netfs_inode_init - Initialise a netfslib inode context
  * @ctx: The netfs inode to initialise
@@ -488,8 +736,8 @@ static inline void netfs_inode_init(struct netfs_inode *ctx,
 				    bool use_zero_point)
 {
 	ctx->ops = ops;
-	ctx->remote_i_size = i_size_read(&ctx->inode);
-	ctx->zero_point = LLONG_MAX;
+	ctx->_remote_i_size = i_size_read(&ctx->inode);
+	ctx->_zero_point = LLONG_MAX;
 	ctx->flags = 0;
 	atomic_set(&ctx->io_count, 0);
 #if IS_ENABLED(CONFIG_FSCACHE)
@@ -498,7 +746,7 @@ static inline void netfs_inode_init(struct netfs_inode *ctx,
 	mutex_init(&ctx->wb_lock);
 	/* ->releasepage() drives zero_point */
 	if (use_zero_point) {
-		ctx->zero_point = ctx->remote_i_size;
+		ctx->_zero_point = ctx->_remote_i_size;
 		mapping_set_release_always(ctx->inode.i_mapping);
 	}
 }
@@ -511,13 +759,40 @@ static inline void netfs_inode_init(struct netfs_inode *ctx,
  *
  * Inform the netfs lib that a file got resized so that it can adjust its state.
  */
-static inline void netfs_resize_file(struct netfs_inode *ctx, loff_t new_i_size,
+static inline void netfs_resize_file(struct netfs_inode *ictx,
+				     unsigned long long new_i_size,
 				     bool changed_on_server)
 {
+#if BITS_PER_LONG==32 && defined(CONFIG_SMP)
+	struct inode *inode = &ictx->inode;
+
+	preempt_disable();
+	write_seqcount_begin(&inode->i_size_seqcount);
+	if (changed_on_server)
+		ictx->_remote_i_size = new_i_size;
+	if (new_i_size < ictx->_zero_point)
+		ictx->_zero_point = new_i_size;
+	write_seqcount_end(&inode->i_size_seqcount);
+	preempt_enable();
+#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION)
+	preempt_disable();
 	if (changed_on_server)
-		ctx->remote_i_size = new_i_size;
-	if (new_i_size < ctx->zero_point)
-		ctx->zero_point = new_i_size;
+		ictx->_remote_i_size = new_i_size;
+	if (new_i_size < ictx->_zero_point)
+		ictx->_zero_point = new_i_size;
+	preempt_enable();
+#else
+	/*
+	 * Pairs with smp_load_acquire() in netfs_read_remote_i_size and
+	 * netfs_read_zero_point() to ensure changes related to inode size
+	 * (such as page contents) are visible before we see the changed inode
+	 * size.
+	 */
+	if (changed_on_server)
+		smp_store_release(&ictx->_remote_i_size, new_i_size);
+	if (new_i_size < ictx->_zero_point)
+		smp_store_release(&ictx->_zero_point, new_i_size);
+#endif
 }
 
 /**
diff --git a/include/linux/netpoll.h b/include/linux/netpoll.h
index f22eec4660405..e4b8f1f91e54b 100644
--- a/include/linux/netpoll.h
+++ b/include/linux/netpoll.h
@@ -67,13 +67,13 @@ static inline void netpoll_poll_disable(struct net_device *dev) { return; }
 static inline void netpoll_poll_enable(struct net_device *dev) { return; }
 #endif
 
-int netpoll_send_udp(struct netpoll *np, const char *msg, int len);
 int __netpoll_setup(struct netpoll *np, struct net_device *ndev);
 int netpoll_setup(struct netpoll *np);
 void __netpoll_free(struct netpoll *np);
 void netpoll_cleanup(struct netpoll *np);
 void do_netpoll_cleanup(struct netpoll *np);
 netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb);
+void netpoll_zap_completion_queue(void);
 
 #ifdef CONFIG_NETPOLL
 static inline void *netpoll_poll_lock(struct napi_struct *napi)
diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h
index 4daee27fa5eb4..34d294774f8ce 100644
--- a/include/linux/nfs_fs_sb.h
+++ b/include/linux/nfs_fs_sb.h
@@ -306,7 +306,7 @@ struct nfs_server {
 #define NFS_CAP_ATOMIC_OPEN	(1U << 4)
 #define NFS_CAP_LGOPEN		(1U << 5)
 #define NFS_CAP_CASE_INSENSITIVE	(1U << 6)
-#define NFS_CAP_CASE_PRESERVING	(1U << 7)
+#define NFS_CAP_CASE_NONPRESERVING	(1U << 7)
 #define NFS_CAP_REBOOT_LAYOUTRETURN	(1U << 8)
 #define NFS_CAP_OFFLOAD_STATUS	(1U << 9)
 #define NFS_CAP_ZERO_RANGE	(1U << 10)
diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h
index fcbd21b5685f4..83ee991cde2be 100644
--- a/include/linux/nfs_xdr.h
+++ b/include/linux/nfs_xdr.h
@@ -182,6 +182,8 @@ struct nfs_pathconf {
 	struct nfs_fattr	*fattr; /* Post-op attributes */
 	__u32			max_link; /* max # of hard links */
 	__u32			max_namelen; /* max name length */
+	bool			case_insensitive;
+	bool			case_preserving;
 };
 
 struct nfs4_change_info {
diff --git a/include/linux/of.h b/include/linux/of.h
index 959786f8f1966..28153616e616a 100644
--- a/include/linux/of.h
+++ b/include/linux/of.h
@@ -1476,6 +1476,13 @@ static inline int of_property_read_s32(const struct device_node *np,
 	return of_property_read_u32(np, propname, (u32*) out_value);
 }
 
+static inline int of_property_read_s32_index(const struct device_node *np,
+					     const char *propname, u32 index,
+					     s32 *out_value)
+{
+	return of_property_read_u32_index(np, propname, index, (u32 *)out_value);
+}
+
 #define of_for_each_phandle(it, err, np, ln, cn, cc)			\
 	for (of_phandle_iterator_init((it), (np), (ln), (cn), (cc)),	\
 	     err = of_phandle_iterator_next(it);			\
diff --git a/include/linux/oid_registry.h b/include/linux/oid_registry.h
index ebce402854de4..bf852715aaea4 100644
--- a/include/linux/oid_registry.h
+++ b/include/linux/oid_registry.h
@@ -150,6 +150,9 @@ enum OID {
 	OID_id_ml_dsa_65,			/* 2.16.840.1.101.3.4.3.18 */
 	OID_id_ml_dsa_87,			/* 2.16.840.1.101.3.4.3.19 */
 
+	/* Hornet LSM */
+	OID_hornet_data,	  /* 2.25.316487325684022475439036912669789383960 */
+
 	OID__NR
 };
 
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index 0e03d816e8b9d..7223f6f4e2b40 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -923,7 +923,6 @@ enum pagetype {
 	PGTY_zsmalloc		= 0xf6,
 	PGTY_unaccepted		= 0xf7,
 	PGTY_large_kmalloc	= 0xf8,
-	PGTY_netpp		= 0xf9,
 
 	PGTY_mapcount_underflow = 0xff
 };
@@ -1056,11 +1055,6 @@ PAGE_TYPE_OPS(Zsmalloc, zsmalloc, zsmalloc)
 PAGE_TYPE_OPS(Unaccepted, unaccepted, unaccepted)
 PAGE_TYPE_OPS(LargeKmalloc, large_kmalloc, large_kmalloc)
 
-/*
- * Marks page_pool allocated pages.
- */
-PAGE_TYPE_OPS(Netpp, netpp, netpp)
-
 /**
  * PageHuge - Determine if the page belongs to hugetlbfs
  * @page: The page to test.
diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h
index e046278a01fa8..9a6c3ea17684d 100644
--- a/include/linux/pageblock-flags.h
+++ b/include/linux/pageblock-flags.h
@@ -36,12 +36,12 @@ enum pageblock_bits {
 
 #define NR_PAGEBLOCK_BITS (roundup_pow_of_two(__NR_PAGEBLOCK_BITS))
 
-#define MIGRATETYPE_MASK (BIT(PB_migrate_0)|BIT(PB_migrate_1)|BIT(PB_migrate_2))
+#define PAGEBLOCK_MIGRATETYPE_MASK (BIT(PB_migrate_0)|BIT(PB_migrate_1)|BIT(PB_migrate_2))
 
 #ifdef CONFIG_MEMORY_ISOLATION
-#define MIGRATETYPE_AND_ISO_MASK (MIGRATETYPE_MASK | BIT(PB_migrate_isolate))
+#define PAGEBLOCK_ISO_MASK	BIT(PB_migrate_isolate)
 #else
-#define MIGRATETYPE_AND_ISO_MASK MIGRATETYPE_MASK
+#define PAGEBLOCK_ISO_MASK	0
 #endif
 
 #if defined(CONFIG_HUGETLB_PAGE)
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 31a848485ad9d..627771e82eb16 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -513,39 +513,37 @@ static inline bool mapping_large_folio_support(const struct address_space *mappi
 	return mapping_max_folio_order(mapping) > 0;
 }
 
-/* Return the maximum folio size for this pagecache mapping, in bytes. */
-static inline size_t mapping_max_folio_size(const struct address_space *mapping)
+/**
+ * mapping_pmd_folio_support() - Check if a mapping supports PMD-sized folio
+ * @mapping: The address_space
+ *
+ * While some mappings support large folios, they might not support PMD-sized
+ * folios. This function checks whether a mapping supports PMD-sized folios.
+ * For example, khugepaged needs this information before attempting to
+ * collapsing THPs.
+ *
+ * Return: True if PMD-sized folios are supported, otherwise false.
+ */
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static inline bool mapping_pmd_folio_support(const struct address_space *mapping)
 {
-	return PAGE_SIZE << mapping_max_folio_order(mapping);
-}
+	/* AS_FOLIO_ORDER is only reasonable for pagecache folios */
+	VM_WARN_ON_ONCE((unsigned long)mapping & FOLIO_MAPPING_ANON);
 
-static inline int filemap_nr_thps(const struct address_space *mapping)
-{
-#ifdef CONFIG_READ_ONLY_THP_FOR_FS
-	return atomic_read(&mapping->nr_thps);
-#else
-	return 0;
-#endif
+	return mapping_min_folio_order(mapping) <= PMD_ORDER &&
+	       mapping_max_folio_order(mapping) >= PMD_ORDER;
 }
-
-static inline void filemap_nr_thps_inc(struct address_space *mapping)
-{
-#ifdef CONFIG_READ_ONLY_THP_FOR_FS
-	if (!mapping_large_folio_support(mapping))
-		atomic_inc(&mapping->nr_thps);
 #else
-	WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
-#endif
+static inline bool mapping_pmd_folio_support(const struct address_space *mapping)
+{
+	return false;
 }
+#endif
 
-static inline void filemap_nr_thps_dec(struct address_space *mapping)
+/* Return the maximum folio size for this pagecache mapping, in bytes. */
+static inline size_t mapping_max_folio_size(const struct address_space *mapping)
 {
-#ifdef CONFIG_READ_ONLY_THP_FOR_FS
-	if (!mapping_large_folio_support(mapping))
-		atomic_dec(&mapping->nr_thps);
-#else
-	WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0);
-#endif
+	return PAGE_SIZE << mapping_max_folio_order(mapping);
 }
 
 struct address_space *folio_mapping(const struct folio *folio);
@@ -1350,6 +1348,7 @@ struct readahead_control {
 	struct file_ra_state *ra;
 /* private: use the readahead_* accessors instead */
 	pgoff_t _index;
+	pgoff_t _max_index; /* limit readahead to _max_index, inclusive */
 	unsigned int _nr_pages;
 	unsigned int _batch_count;
 	bool dropbehind;
@@ -1363,6 +1362,7 @@ struct readahead_control {
 		.mapping = m,						\
 		.ra = r,						\
 		._index = i,						\
+		._max_index = ULONG_MAX,				\
 	}
 
 #define VM_READAHEAD_PAGES	(SZ_128K / PAGE_SIZE)
diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h
index 1eca1264815b2..f247cf9bcf1ac 100644
--- a/include/linux/pci-epc.h
+++ b/include/linux/pci-epc.h
@@ -61,6 +61,47 @@ struct pci_epc_map {
 	void __iomem	*virt_addr;
 };
 
+/**
+ * enum pci_epc_aux_resource_type - auxiliary resource type identifiers
+ * @PCI_EPC_AUX_DOORBELL_MMIO: Doorbell MMIO, that might be outside the DMA
+ *                             controller register window
+ *
+ * EPC backends may expose auxiliary blocks (e.g. DMA engines) by mapping their
+ * register windows and descriptor memories into BAR space. This enum
+ * identifies the type of each exposable resource.
+ */
+enum pci_epc_aux_resource_type {
+	PCI_EPC_AUX_DOORBELL_MMIO,
+};
+
+/**
+ * struct pci_epc_aux_resource - a physical auxiliary resource that may be
+ *                               exposed for peer use
+ * @type:       resource type, see enum pci_epc_aux_resource_type
+ * @phys_addr:  physical base address of the resource
+ * @size:       size of the resource in bytes
+ * @bar:        BAR number where this resource is already exposed to the RC
+ *              (NO_BAR if not)
+ * @bar_offset: offset within @bar where the resource starts (valid iff
+ *              @bar != NO_BAR)
+ * @u:          type-specific metadata
+ */
+struct pci_epc_aux_resource {
+	enum pci_epc_aux_resource_type type;
+	phys_addr_t phys_addr;
+	resource_size_t size;
+	enum pci_barno bar;
+	resource_size_t bar_offset;
+
+	union {
+		/* PCI_EPC_AUX_DOORBELL_MMIO */
+		struct {
+			int irq; /* IRQ number for the doorbell handler */
+			u32 data; /* write value to ring the doorbell */
+		} db_mmio;
+	} u;
+};
+
 /**
  * struct pci_epc_ops - set of function pointers for performing EPC operations
  * @write_header: ops to populate configuration space header
@@ -84,6 +125,9 @@ struct pci_epc_map {
  * @start: ops to start the PCI link
  * @stop: ops to stop the PCI link
  * @get_features: ops to get the features supported by the EPC
+ * @get_aux_resources_count: ops to get the number of controller-owned
+ *                           auxiliary resources
+ * @get_aux_resources: ops to retrieve controller-owned auxiliary resources
  * @owner: the module owner containing the ops
  */
 struct pci_epc_ops {
@@ -115,6 +159,11 @@ struct pci_epc_ops {
 	void	(*stop)(struct pci_epc *epc);
 	const struct pci_epc_features* (*get_features)(struct pci_epc *epc,
 						       u8 func_no, u8 vfunc_no);
+	int	(*get_aux_resources_count)(struct pci_epc *epc, u8 func_no,
+					   u8 vfunc_no);
+	int	(*get_aux_resources)(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+				     struct pci_epc_aux_resource *resources,
+				     int num_resources);
 	struct module *owner;
 };
 
@@ -343,6 +392,11 @@ int pci_epc_start(struct pci_epc *epc);
 void pci_epc_stop(struct pci_epc *epc);
 const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc,
 						    u8 func_no, u8 vfunc_no);
+int pci_epc_get_aux_resources_count(struct pci_epc *epc, u8 func_no,
+				    u8 vfunc_no);
+int pci_epc_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
+			      struct pci_epc_aux_resource *resources,
+			      int num_resources);
 enum pci_barno
 pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features);
 enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features
diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h
index 7737a7c03260d..8a6c64a358907 100644
--- a/include/linux/pci-epf.h
+++ b/include/linux/pci-epf.h
@@ -152,14 +152,41 @@ struct pci_epf_bar {
 	struct pci_epf_bar_submap	*submap;
 };
 
+enum pci_epf_doorbell_type {
+	PCI_EPF_DOORBELL_MSI = 0,
+	PCI_EPF_DOORBELL_EMBEDDED,
+};
+
 /**
  * struct pci_epf_doorbell_msg - represents doorbell message
- * @msg: MSI message
- * @virq: IRQ number of this doorbell MSI message
+ * @msg: Doorbell address/data pair to be mapped into BAR space.
+ *       For MSI-backed doorbells this is the MSI message, while for
+ *       "embedded" doorbells this represents an MMIO write that asserts
+ *       an interrupt on the EP side.
+ * @virq: IRQ number of this doorbell message
+ * @irq_flags: Required flags for request_irq()/request_threaded_irq().
+ *             Callers may OR-in additional flags (e.g. IRQF_ONESHOT).
+ * @type: Doorbell type.
+ * @bar: BAR number where the doorbell target is already exposed to the RC
+ *       (NO_BAR if not)
+ * @offset: offset within @bar for the doorbell target (valid iff
+ *          @bar != NO_BAR)
+ * @iova_base: Internal: base DMA address returned by dma_map_resource() for the
+ *             embedded doorbell MMIO window (used only for unmapping). Valid
+ *             when @type is PCI_EPF_DOORBELL_EMBEDDED and @iova_size is
+ *             non-zero.
+ * @iova_size: Internal: size of the dma_map_resource() mapping at @iova_base.
+ *             Zero when no mapping was created (e.g. pre-exposed fixed BAR).
  */
 struct pci_epf_doorbell_msg {
 	struct msi_msg msg;
 	int virq;
+	unsigned long irq_flags;
+	enum pci_epf_doorbell_type type;
+	enum pci_barno bar;
+	resource_size_t offset;
+	dma_addr_t iova_base;
+	size_t iova_size;
 };
 
 /**
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 2c4454583c115..709ab32adf1c1 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -27,6 +27,7 @@
 #include <linux/mod_devicetable.h>
 
 #include <linux/types.h>
+#include <linux/sizes.h>
 #include <linux/init.h>
 #include <linux/ioport.h>
 #include <linux/list.h>
@@ -508,15 +509,13 @@ struct pci_dev {
 	unsigned int	no_command_memory:1;	/* No PCI_COMMAND_MEMORY */
 	unsigned int	rom_bar_overlap:1;	/* ROM BAR disable broken */
 	unsigned int	rom_attr_enabled:1;	/* Display of ROM attribute enabled? */
-	unsigned int	non_mappable_bars:1;	/* BARs can't be mapped to user-space  */
+	unsigned int	non_mappable_bars:1;	/* BARs can't be mapped by CPU or peers */
 	pci_dev_flags_t dev_flags;
 	atomic_t	enable_cnt;	/* pci_enable_device has been called */
 
 	spinlock_t	pcie_cap_lock;		/* Protects RMW ops in capability accessors */
 	u32		saved_config_space[16]; /* Config space saved at suspend time */
 	struct hlist_head saved_cap_space;
-	struct bin_attribute *res_attr[DEVICE_COUNT_RESOURCE]; /* sysfs file for resources */
-	struct bin_attribute *res_attr_wc[DEVICE_COUNT_RESOURCE]; /* sysfs file for WC mapping of resources */
 
 #ifdef CONFIG_HOTPLUG_PCI_PCIE
 	unsigned int	broken_cmd_compl:1;	/* No compl for some cmds */
@@ -636,6 +635,7 @@ struct pci_host_bridge {
 	int		domain_nr;
 	struct list_head windows;	/* resource_entry */
 	struct list_head dma_ranges;	/* dma ranges resource list */
+	struct list_head ports;		/* Root Port list (pci_host_port) */
 #ifdef CONFIG_PCI_IDE
 	u16 nr_ide_streams; /* Max streams possibly active in @ide_stream_ida */
 	struct ida ide_stream_ida;
@@ -727,8 +727,6 @@ struct pci_bus {
 	pci_bus_flags_t bus_flags;	/* Inherited by child buses */
 	struct device		*bridge;
 	struct device		dev;
-	struct bin_attribute	*legacy_io;	/* Legacy I/O for this bus */
-	struct bin_attribute	*legacy_mem;	/* Legacy mem */
 	unsigned int		is_added:1;
 	unsigned int		unsafe_warn:1;	/* warned about RW1C config write */
 	unsigned int		flit_mode:1;	/* Link in Flit mode */
@@ -1171,6 +1169,10 @@ enum {
 /* These external functions are only available when PCI support is enabled */
 #ifdef CONFIG_PCI
 
+/* PCI legacy I/O port and memory address space sizes. */
+#define PCI_LEGACY_IO_SIZE	(SZ_64K - 1)
+#define PCI_LEGACY_MEM_SIZE	SZ_1M
+
 extern unsigned int pci_flags;
 
 static inline void pci_set_flags(int flags) { pci_flags = flags; }
@@ -2300,6 +2302,31 @@ int pci_iobar_pfn(struct pci_dev *pdev, int bar, struct vm_area_struct *vma);
 	CONCATENATE(__pci_dev_for_each_res, COUNT_ARGS(__VA_ARGS__)) 	\
 		    (dev, res, __VA_ARGS__)
 
+/**
+ * pci_resource_is_io - check if a PCI resource is of I/O port type.
+ * @dev: PCI device to check.
+ * @resno: The resource number (BAR index) to check.
+ *
+ * Returns true if the resource type is I/O port.
+ */
+static inline bool pci_resource_is_io(const struct pci_dev *dev, int resno)
+{
+	return resource_type(pci_resource_n(dev, resno)) == IORESOURCE_IO;
+}
+
+/**
+ * pci_resource_is_mem - check if a PCI resource is of memory type.
+ * @dev: PCI device to check.
+ * @resno: The resource number (BAR index) to check.
+ *
+ * Returns true if the resource type is memory, including
+ * prefetchable memory.
+ */
+static inline bool pci_resource_is_mem(const struct pci_dev *dev, int resno)
+{
+	return resource_type(pci_resource_n(dev, resno)) == IORESOURCE_MEM;
+}
+
 /*
  * Similar to the helpers above, these manipulate per-pci_dev
  * driver-specific data.  They are really just a wrapper around
@@ -2506,11 +2533,6 @@ int pcibios_alloc_irq(struct pci_dev *dev);
 void pcibios_free_irq(struct pci_dev *dev);
 resource_size_t pcibios_default_alignment(void);
 
-#if !defined(HAVE_PCI_MMAP) && !defined(ARCH_GENERIC_PCI_MMAP_RESOURCE)
-extern int pci_create_resource_files(struct pci_dev *dev);
-extern void pci_remove_resource_files(struct pci_dev *dev);
-#endif
-
 #if defined(CONFIG_PCI_MMCONFIG) || defined(CONFIG_ACPI_MCFG)
 void __init pci_mmcfg_early_init(void);
 void __init pci_mmcfg_late_init(void);
@@ -2540,7 +2562,7 @@ int pci_vfs_assigned(struct pci_dev *dev);
 int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs);
 int pci_sriov_get_totalvfs(struct pci_dev *dev);
 int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn);
-resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno);
+resource_size_t pci_iov_resource_size(const struct pci_dev *dev, int resno);
 int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size);
 u32 pci_iov_vf_bar_get_sizes(struct pci_dev *dev, int resno, int num_vfs);
 void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe);
@@ -2548,7 +2570,8 @@ void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe);
 /* Arch may override these (weak) */
 int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs);
 int pcibios_sriov_disable(struct pci_dev *pdev);
-resource_size_t pcibios_iov_resource_alignment(struct pci_dev *dev, int resno);
+resource_size_t pcibios_iov_resource_alignment(const struct pci_dev *dev,
+					       int resno);
 #else
 static inline int pci_iov_virtfn_bus(struct pci_dev *dev, int id)
 {
@@ -2593,7 +2616,8 @@ static inline int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs)
 static inline int pci_sriov_get_totalvfs(struct pci_dev *dev)
 { return 0; }
 #define pci_sriov_configure_simple	NULL
-static inline resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno)
+static inline resource_size_t pci_iov_resource_size(const struct pci_dev *dev,
+						    int resno)
 { return 0; }
 static inline int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size)
 { return -ENODEV; }
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 85bf8dd9f0874..bdb721dac0e32 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -36,7 +36,7 @@
 #define PCPU_BITMAP_BLOCK_BITS		(PCPU_BITMAP_BLOCK_SIZE >>	\
 					 PCPU_MIN_ALLOC_SHIFT)
 
-#ifdef CONFIG_RANDOM_KMALLOC_CACHES
+#ifdef CONFIG_KMALLOC_PARTITION_CACHES
 # if defined(CONFIG_LOCKDEP) && !defined(CONFIG_PAGE_SIZE_4KB)
 # define PERCPU_DYNAMIC_SIZE_SHIFT      13
 # else
diff --git a/include/linux/pinctrl/pinmux.h b/include/linux/pinctrl/pinmux.h
index 094bbe2fd6fd5..77664937eeb27 100644
--- a/include/linux/pinctrl/pinmux.h
+++ b/include/linux/pinctrl/pinmux.h
@@ -51,6 +51,8 @@ struct pinctrl_gpio_range;
  *	are handled by the pinmux subsystem. The @func_selector selects a
  *	certain function whereas @group_selector selects a certain set of pins
  *	to be used. On simple controllers the latter argument may be ignored
+ * @release_mux: Release software resources acquired by @set_mux. This callback
+ *	must not change hardware state to avoid glitches when switching mux.
  * @gpio_request_enable: requests and enables GPIO on a certain pin.
  *	Implement this only if you can mux every pin individually as GPIO. The
  *	affected GPIO range is passed along with an offset(pin number) into that
@@ -80,6 +82,9 @@ struct pinmux_ops {
 				  unsigned int selector);
 	int (*set_mux) (struct pinctrl_dev *pctldev, unsigned int func_selector,
 			unsigned int group_selector);
+	void (*release_mux) (struct pinctrl_dev *pctldev,
+			     unsigned int func_selector,
+			     unsigned int group_selector);
 	int (*gpio_request_enable) (struct pinctrl_dev *pctldev,
 				    struct pinctrl_gpio_range *range,
 				    unsigned int offset);
diff --git a/include/linux/platform_data/apds990x.h b/include/linux/platform_data/apds990x.h
deleted file mode 100644
index 37684f68c04f6..0000000000000
--- a/include/linux/platform_data/apds990x.h
+++ /dev/null
@@ -1,65 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * This file is part of the APDS990x sensor driver.
- * Chip is combined proximity and ambient light sensor.
- *
- * Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
- *
- * Contact: Samu Onkalo <samu.p.onkalo@nokia.com>
- */
-
-#ifndef __APDS990X_H__
-#define __APDS990X_H__
-
-
-#define APDS_IRLED_CURR_12mA	0x3
-#define APDS_IRLED_CURR_25mA	0x2
-#define APDS_IRLED_CURR_50mA	0x1
-#define APDS_IRLED_CURR_100mA	0x0
-
-/**
- * struct apds990x_chip_factors - defines effect of the cover window
- * @ga: Total glass attenuation
- * @cf1: clear channel factor 1 for raw to lux conversion
- * @irf1: IR channel factor 1 for raw to lux conversion
- * @cf2: clear channel factor 2 for raw to lux conversion
- * @irf2: IR channel factor 2 for raw to lux conversion
- * @df: device factor for conversion formulas
- *
- * Structure for tuning ALS calculation to match with environment.
- * Values depend on the material above the sensor and the sensor
- * itself. If the GA is zero, driver will use uncovered sensor default values
- * format: decimal value * APDS_PARAM_SCALE except df which is plain integer.
- */
-struct apds990x_chip_factors {
-	int ga;
-	int cf1;
-	int irf1;
-	int cf2;
-	int irf2;
-	int df;
-};
-#define APDS_PARAM_SCALE 4096
-
-/**
- * struct apds990x_platform_data - platform data for apsd990x.c driver
- * @cf: chip factor data
- * @pdrive: IR-led driving current
- * @ppcount: number of IR pulses used for proximity estimation
- * @setup_resources: interrupt line setup call back function
- * @release_resources: interrupt line release call back function
- *
- * Proximity detection result depends heavily on correct ppcount, pdrive
- * and cover window.
- *
- */
-
-struct apds990x_platform_data {
-	struct apds990x_chip_factors cf;
-	u8     pdrive;
-	u8     ppcount;
-	int    (*setup_resources)(void);
-	int    (*release_resources)(void);
-};
-
-#endif
diff --git a/include/linux/platform_data/asoc-pxa.h b/include/linux/platform_data/asoc-pxa.h
index 7b5b9e20fbf5e..0d5eaf4b100ca 100644
--- a/include/linux/platform_data/asoc-pxa.h
+++ b/include/linux/platform_data/asoc-pxa.h
@@ -6,27 +6,6 @@
 #include <sound/pcm.h>
 #include <sound/ac97_codec.h>
 
-/*
- * @reset_gpio: AC97 reset gpio (normally gpio113 or gpio95)
- *              a -1 value means no gpio will be used for reset
- * @codec_pdata: AC97 codec platform_data
-
- * reset_gpio should only be specified for pxa27x CPUs where a silicon
- * bug prevents correct operation of the reset line. If not specified,
- * the default behaviour on these CPUs is to consider gpio 113 as the
- * AC97 reset line, which is the default on most boards.
- */
-typedef struct {
-	int (*startup)(struct snd_pcm_substream *, void *);
-	void (*shutdown)(struct snd_pcm_substream *, void *);
-	void (*suspend)(void *);
-	void (*resume)(void *);
-	void *priv;
-	int reset_gpio;
-	void *codec_pdata[AC97_BUS_MAX_DEVICES];
-} pxa2xx_audio_ops_t;
-
-extern void pxa_set_ac97_info(pxa2xx_audio_ops_t *ops);
-extern void pxa27x_configure_ac97reset(int reset_gpio, bool to_gpio);
+extern void pxa27x_configure_ac97reset(struct gpio_desc *reset_gpio, bool to_gpio);
 
 #endif
diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h
index de14923720a53..6ed1c4c5ce2ef 100644
--- a/include/linux/platform_data/cros_ec_proto.h
+++ b/include/linux/platform_data/cros_ec_proto.h
@@ -228,6 +228,7 @@ struct cros_ec_platform {
 /**
  * struct cros_ec_dev - ChromeOS EC device entry point.
  * @class_dev: Device structure used in sysfs.
+ * @group: sysfs attributes groups for this EC.
  * @ec_dev: cros_ec_device structure to talk to the physical device.
  * @dev: Pointer to the platform device.
  * @debug_info: cros_ec_debugfs structure for debugging information.
@@ -237,6 +238,7 @@ struct cros_ec_platform {
  */
 struct cros_ec_dev {
 	struct device class_dev;
+	const struct attribute_group *group;
 	struct cros_ec_device *ec_dev;
 	struct device *dev;
 	struct cros_ec_debugfs *debug_info;
diff --git a/include/linux/platform_data/pxa2xx_udc.h b/include/linux/platform_data/pxa2xx_udc.h
index bc99cc6a3c5f8..c1e4d03bae2c4 100644
--- a/include/linux/platform_data/pxa2xx_udc.h
+++ b/include/linux/platform_data/pxa2xx_udc.h
@@ -10,21 +10,6 @@
 #ifndef PXA2XX_UDC_H
 #define PXA2XX_UDC_H
 
-struct pxa2xx_udc_mach_info {
-        int  (*udc_is_connected)(void);		/* do we see host? */
-        void (*udc_command)(int cmd);
-#define	PXA2XX_UDC_CMD_CONNECT		0	/* let host see us */
-#define	PXA2XX_UDC_CMD_DISCONNECT	1	/* so host won't see us */
-
-	/* Boards following the design guidelines in the developer's manual,
-	 * with on-chip GPIOs not Lubbock's weird hardware, can have a sane
-	 * VBUS IRQ and omit the methods above.  Store the GPIO number
-	 * here.  Note that sometimes the signals go through inverters...
-	 */
-	bool	gpio_pullup_inverted;
-	int	gpio_pullup;			/* high == pullup activated */
-};
-
 #ifdef CONFIG_PXA27x
 extern void pxa27x_clear_otgph(void);
 #else
diff --git a/include/linux/platform_data/spi-s3c64xx.h b/include/linux/platform_data/spi-s3c64xx.h
index 1d6e6c424fc69..f92bb4a1213be 100644
--- a/include/linux/platform_data/spi-s3c64xx.h
+++ b/include/linux/platform_data/spi-s3c64xx.h
@@ -30,6 +30,7 @@ struct s3c64xx_spi_csinfo {
  * @src_clk_nr: Clock source index for the CLK_CFG[SPI_CLKSEL] field.
  * @num_cs: Number of CS this controller emulates.
  * @no_cs: Used when CS line is not connected.
+ * @polling: Using polling mode when %true (no 'dmas' property in devicetree)
  * @cfg_gpio: Configure pins for this SPI controller.
  */
 struct s3c64xx_spi_info {
@@ -41,7 +42,7 @@ struct s3c64xx_spi_info {
 };
 
 /**
- * s3c64xx_spi_set_platdata - SPI Controller configure callback by the board
+ * s3c64xx_spi0_set_platdata - SPI Controller configure callback by the board
  *				initialization code.
  * @src_clk_nr: Clock the SPI controller is to use to generate SPI clocks.
  * @num_cs: Number of elements in the 'cs' array.
diff --git a/include/linux/platform_data/wiznet.h b/include/linux/platform_data/wiznet.h
deleted file mode 100644
index 1154c4db8a135..0000000000000
--- a/include/linux/platform_data/wiznet.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Ethernet driver for the WIZnet W5x00 chip.
- */
-
-#ifndef PLATFORM_DATA_WIZNET_H
-#define PLATFORM_DATA_WIZNET_H
-
-#include <linux/if_ether.h>
-
-struct wiznet_platform_data {
-	int	link_gpio;
-	u8	mac_addr[ETH_ALEN];
-};
-
-#ifndef CONFIG_WIZNET_BUS_SHIFT
-#define CONFIG_WIZNET_BUS_SHIFT 0
-#endif
-
-#define W5100_BUS_DIRECT_SIZE	(0x8000 << CONFIG_WIZNET_BUS_SHIFT)
-#define W5300_BUS_DIRECT_SIZE	(0x0400 << CONFIG_WIZNET_BUS_SHIFT)
-
-#endif /* PLATFORM_DATA_WIZNET_H */
diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h
index b299dc0128d65..f925614aebdbc 100644
--- a/include/linux/pm_domain.h
+++ b/include/linux/pm_domain.h
@@ -467,6 +467,10 @@ struct generic_pm_domain *of_genpd_remove_last(struct device_node *np);
 int of_genpd_parse_idle_states(struct device_node *dn,
 			       struct genpd_power_state **states, int *n);
 void of_genpd_sync_state(struct device_node *np);
+int of_genpd_add_child_ids(struct device_node *np,
+			   struct genpd_onecell_data *data);
+int of_genpd_remove_child_ids(struct device_node *np,
+			      struct genpd_onecell_data *data);
 
 int genpd_dev_pm_attach(struct device *dev);
 struct device *genpd_dev_pm_attach_by_id(struct device *dev,
@@ -536,6 +540,18 @@ struct generic_pm_domain *of_genpd_remove_last(struct device_node *np)
 {
 	return ERR_PTR(-EOPNOTSUPP);
 }
+
+static inline int of_genpd_add_child_ids(struct device_node *np,
+					 struct genpd_onecell_data *data)
+{
+	return -EOPNOTSUPP;
+}
+
+static inline int of_genpd_remove_child_ids(struct device_node *np,
+					    struct genpd_onecell_data *data)
+{
+	return -EOPNOTSUPP;
+}
 #endif /* CONFIG_PM_GENERIC_DOMAINS_OF */
 
 #ifdef CONFIG_PM
diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h
index 19d1c5e5f3350..47d7deaeed8ff 100644
--- a/include/linux/proc_fs.h
+++ b/include/linux/proc_fs.h
@@ -67,6 +67,7 @@ enum proc_pidonly {
 struct proc_fs_info {
 	struct pid_namespace *pid_ns;
 	kgid_t pid_gid;
+	const struct cred *mounter_cred;
 	enum proc_hidepid hide_pid;
 	enum proc_pidonly pidonly;
 	struct rcu_head rcu;
@@ -248,4 +249,16 @@ static inline struct pid_namespace *proc_pid_ns(struct super_block *sb)
 
 bool proc_ns_file(const struct file *file);
 
+#if defined CONFIG_PROC_FS && !defined MODULE
+void impl_proc_make_permanent(struct proc_dir_entry *pde);
+#endif
+
+static inline void proc_make_permanent(struct proc_dir_entry *pde)
+{
+	/* Don't give matches to modules. */
+#if defined CONFIG_PROC_FS && !defined MODULE
+	impl_proc_make_permanent(pde);
+#endif
+}
+
 #endif /* _LINUX_PROC_FS_H */
diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h
index d5099a2baca51..ce16bbc0b3083 100644
--- a/include/linux/psp-sev.h
+++ b/include/linux/psp-sev.h
@@ -902,6 +902,42 @@ struct snp_feature_info {
 /* Feature bits in EBX */
 #define SNP_SEV_TIO_SUPPORTED			BIT(1)
 
+/**
+ * struct sev_snp_tcb_version_genoa_milan
+ *
+ * @boot_loader: SVN of PSP bootloader
+ * @tee: SVN of PSP operating system
+ * @reserved: reserved
+ * @snp: SVN of SNP firmware
+ * @microcode: Lowest current patch level of all cores
+ */
+struct sev_snp_tcb_version_genoa_milan {
+	u8 boot_loader;
+	u8 tee;
+	u8 reserved[4];
+	u8 snp;
+	u8 microcode;
+};
+
+/**
+ * struct sev_snp_tcb_version_turin
+ *
+ * @fmc: SVN of FMC firmware
+ * @boot_loader: SVN of PSP bootloader
+ * @tee: SVN of PSP operating system
+ * @snp: SVN of SNP firmware
+ * @reserved: reserved
+ * @microcode: Lowest current patch level of all cores
+ */
+struct sev_snp_tcb_version_turin {
+	u8 fmc;
+	u8 boot_loader;
+	u8 tee;
+	u8 snp;
+	u8 reserved[3];
+	u8 microcode;
+};
+
 #ifdef CONFIG_CRYPTO_DEV_SP_PSP
 
 /**
@@ -1048,6 +1084,7 @@ void snp_free_firmware_page(void *addr);
 void sev_platform_shutdown(void);
 bool sev_is_snp_ciphertext_hiding_supported(void);
 u64 sev_get_snp_policy_bits(void);
+int sev_firmware_supported_vm_types(void);
 
 #else	/* !CONFIG_CRYPTO_DEV_SP_PSP */
 
diff --git a/include/linux/ptr_ring.h b/include/linux/ptr_ring.h
index d2c3629bbe451..c95e891903f05 100644
--- a/include/linux/ptr_ring.h
+++ b/include/linux/ptr_ring.h
@@ -96,6 +96,20 @@ static inline bool ptr_ring_full_bh(struct ptr_ring *r)
 	return ret;
 }
 
+/* Note: callers invoking this in a loop must use a compiler barrier,
+ * for example cpu_relax(). Callers must hold producer_lock.
+ */
+static inline int __ptr_ring_check_produce(struct ptr_ring *r)
+{
+	if (unlikely(!r->size))
+		return -EINVAL;
+
+	if (data_race(r->queue[r->producer]))
+		return -ENOSPC;
+
+	return 0;
+}
+
 /* Note: callers invoking this in a loop must use a compiler barrier,
  * for example cpu_relax(). Callers must hold producer_lock.
  * Callers are responsible for making sure pointer that is being queued
@@ -103,8 +117,10 @@ static inline bool ptr_ring_full_bh(struct ptr_ring *r)
  */
 static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr)
 {
-	if (unlikely(!r->size) || data_race(r->queue[r->producer]))
-		return -ENOSPC;
+	int p = __ptr_ring_check_produce(r);
+
+	if (p)
+		return p;
 
 	/* Make sure the pointer we are storing points to a valid data. */
 	/* Pairs with the dependency ordering in __ptr_ring_consume. */
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 2467b3be15c9e..b3bf9339cd86a 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -1,206 +1,24 @@
 /* SPDX-License-Identifier: GPL-2.0-or-later */
-/* -*- linux-c -*- ------------------------------------------------------- *
+/*
+ * Copyright 2003 H. Peter Anvin - All Rights Reserved
  *
- *   Copyright 2003 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-#ifndef LINUX_RAID_RAID6_H
-#define LINUX_RAID_RAID6_H
-
-#ifdef __KERNEL__
-
-#include <linux/blkdev.h>
-#include <linux/mm.h>
-
-/* This should be const but the raid6 code is too convoluted for that. */
-static inline void *raid6_get_zero_page(void)
-{
-	return page_address(ZERO_PAGE(0));
-}
-
-#else /* ! __KERNEL__ */
-/* Used for testing in user space */
-
-#include <errno.h>
-#include <inttypes.h>
-#include <stddef.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/time.h>
-#include <sys/types.h>
-
-/* Not standard, but glibc defines it */
-#define BITS_PER_LONG __WORDSIZE
-
-typedef uint8_t  u8;
-typedef uint16_t u16;
-typedef uint32_t u32;
-typedef uint64_t u64;
-
-#ifndef PAGE_SIZE
-# define PAGE_SIZE 4096
-#endif
-#ifndef PAGE_SHIFT
-# define PAGE_SHIFT 12
-#endif
-extern const char raid6_empty_zero_page[PAGE_SIZE];
-
-#define __init
-#define __exit
-#ifndef __attribute_const__
-# define __attribute_const__ __attribute__((const))
-#endif
-#define noinline __attribute__((noinline))
-
-#define preempt_enable()
-#define preempt_disable()
-#define cpu_has_feature(x) 1
-#define enable_kernel_altivec()
-#define disable_kernel_altivec()
-
-#undef	EXPORT_SYMBOL
-#define EXPORT_SYMBOL(sym)
-#undef	EXPORT_SYMBOL_GPL
-#define EXPORT_SYMBOL_GPL(sym)
-#define MODULE_LICENSE(licence)
-#define MODULE_DESCRIPTION(desc)
-#define subsys_initcall(x)
-#define module_exit(x)
-
-#define IS_ENABLED(x) (x)
-#define CONFIG_RAID6_PQ_BENCHMARK 1
-#endif /* __KERNEL__ */
-
-/* Routine choices */
-struct raid6_calls {
-	void (*gen_syndrome)(int, size_t, void **);
-	void (*xor_syndrome)(int, int, int, size_t, void **);
-	int  (*valid)(void);	/* Returns 1 if this routine set is usable */
-	const char *name;	/* Name of this routine set */
-	int priority;		/* Relative priority ranking if non-zero */
-};
-
-/* Selected algorithm */
-extern struct raid6_calls raid6_call;
-
-/* Various routine sets */
-extern const struct raid6_calls raid6_intx1;
-extern const struct raid6_calls raid6_intx2;
-extern const struct raid6_calls raid6_intx4;
-extern const struct raid6_calls raid6_intx8;
-extern const struct raid6_calls raid6_mmxx1;
-extern const struct raid6_calls raid6_mmxx2;
-extern const struct raid6_calls raid6_sse1x1;
-extern const struct raid6_calls raid6_sse1x2;
-extern const struct raid6_calls raid6_sse2x1;
-extern const struct raid6_calls raid6_sse2x2;
-extern const struct raid6_calls raid6_sse2x4;
-extern const struct raid6_calls raid6_altivec1;
-extern const struct raid6_calls raid6_altivec2;
-extern const struct raid6_calls raid6_altivec4;
-extern const struct raid6_calls raid6_altivec8;
-extern const struct raid6_calls raid6_avx2x1;
-extern const struct raid6_calls raid6_avx2x2;
-extern const struct raid6_calls raid6_avx2x4;
-extern const struct raid6_calls raid6_avx512x1;
-extern const struct raid6_calls raid6_avx512x2;
-extern const struct raid6_calls raid6_avx512x4;
-extern const struct raid6_calls raid6_s390vx8;
-extern const struct raid6_calls raid6_vpermxor1;
-extern const struct raid6_calls raid6_vpermxor2;
-extern const struct raid6_calls raid6_vpermxor4;
-extern const struct raid6_calls raid6_vpermxor8;
-extern const struct raid6_calls raid6_lsx;
-extern const struct raid6_calls raid6_lasx;
-extern const struct raid6_calls raid6_rvvx1;
-extern const struct raid6_calls raid6_rvvx2;
-extern const struct raid6_calls raid6_rvvx4;
-extern const struct raid6_calls raid6_rvvx8;
-
-struct raid6_recov_calls {
-	void (*data2)(int, size_t, int, int, void **);
-	void (*datap)(int, size_t, int, void **);
-	int  (*valid)(void);
-	const char *name;
-	int priority;
-};
-
-extern const struct raid6_recov_calls raid6_recov_intx1;
-extern const struct raid6_recov_calls raid6_recov_ssse3;
-extern const struct raid6_recov_calls raid6_recov_avx2;
-extern const struct raid6_recov_calls raid6_recov_avx512;
-extern const struct raid6_recov_calls raid6_recov_s390xc;
-extern const struct raid6_recov_calls raid6_recov_neon;
-extern const struct raid6_recov_calls raid6_recov_lsx;
-extern const struct raid6_recov_calls raid6_recov_lasx;
-extern const struct raid6_recov_calls raid6_recov_rvv;
-
-extern const struct raid6_calls raid6_neonx1;
-extern const struct raid6_calls raid6_neonx2;
-extern const struct raid6_calls raid6_neonx4;
-extern const struct raid6_calls raid6_neonx8;
-
-/* Algorithm list */
-extern const struct raid6_calls * const raid6_algos[];
-extern const struct raid6_recov_calls *const raid6_recov_algos[];
-int raid6_select_algo(void);
-
-/* Return values from chk_syndrome */
-#define RAID6_OK	0
-#define RAID6_P_BAD	1
-#define RAID6_Q_BAD	2
-#define RAID6_PQ_BAD	3
-
-/* Galois field tables */
-extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256)));
-extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256)));
-extern const u8 raid6_gfexp[256]      __attribute__((aligned(256)));
-extern const u8 raid6_gflog[256]      __attribute__((aligned(256)));
-extern const u8 raid6_gfinv[256]      __attribute__((aligned(256)));
-extern const u8 raid6_gfexi[256]      __attribute__((aligned(256)));
-
-/* Recovery routines */
-extern void (*raid6_2data_recov)(int disks, size_t bytes, int faila, int failb,
-		       void **ptrs);
-extern void (*raid6_datap_recov)(int disks, size_t bytes, int faila,
-			void **ptrs);
-void raid6_dual_recov(int disks, size_t bytes, int faila, int failb,
-		      void **ptrs);
-
-/* Some definitions to allow code to be compiled for testing in userspace */
-#ifndef __KERNEL__
-
-# define jiffies	raid6_jiffies()
-# define printk 	printf
-# define pr_err(format, ...) fprintf(stderr, format, ## __VA_ARGS__)
-# define pr_info(format, ...) fprintf(stdout, format, ## __VA_ARGS__)
-# define GFP_KERNEL	0
-# define __get_free_pages(x, y)	((unsigned long)mmap(NULL, PAGE_SIZE << (y), \
-						     PROT_READ|PROT_WRITE,   \
-						     MAP_PRIVATE|MAP_ANONYMOUS,\
-						     0, 0))
-# define free_pages(x, y)	munmap((void *)(x), PAGE_SIZE << (y))
+ * Public interface to the RAID6 P/Q calculation and recovery library.
+ */
+#ifndef LINUX_RAID_PQ_H
+#define LINUX_RAID_PQ_H
 
-static inline void cpu_relax(void)
-{
-	/* Nothing */
-}
+#include <linux/types.h>
 
-#undef  HZ
-#define HZ 1000
-static inline uint32_t raid6_jiffies(void)
-{
-	struct timeval tv;
-	gettimeofday(&tv, NULL);
-	return tv.tv_sec*1000 + tv.tv_usec/1000;
-}
+#define RAID6_MIN_DISKS		4
 
-static inline void *raid6_get_zero_page(void)
-{
-	return raid6_empty_zero_page;
-}
+void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs);
+void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes,
+		void **ptrs);
+bool raid6_can_xor_syndrome(void);
 
-#endif /* ! __KERNEL__ */
+void raid6_recov_2data(int disks, size_t bytes, int faila, int failb,
+		void **ptrs);
+void raid6_recov_datap(int disks, size_t bytes, int faila,
+		void **ptrs);
 
-#endif /* LINUX_RAID_RAID6_H */
+#endif /* LINUX_RAID_PQ_H */
diff --git a/include/linux/raid/pq_tables.h b/include/linux/raid/pq_tables.h
new file mode 100644
index 0000000000000..7b1ebe675677f
--- /dev/null
+++ b/include/linux/raid/pq_tables.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright 2003 H. Peter Anvin - All Rights Reserved
+ *
+ * Galois field tables for the Linux RAID6 P/Q parity algorithm.
+ */
+#ifndef _LINUX_RAID_PQ_TABLES_H
+#define _LINUX_RAID_PQ_TABLES_H
+
+#include <linux/types.h>
+
+extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256)));
+extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256)));
+extern const u8 raid6_gfexp[256]      __attribute__((aligned(256)));
+extern const u8 raid6_gflog[256]      __attribute__((aligned(256)));
+extern const u8 raid6_gfinv[256]      __attribute__((aligned(256)));
+extern const u8 raid6_gfexi[256]      __attribute__((aligned(256)));
+
+#endif /* _LINUX_RAID_PQ_TABLES_H */
diff --git a/include/linux/regulator/mt6359-regulator.h b/include/linux/regulator/mt6359-regulator.h
index 6d6e5a58f482b..ce2cd0fc9d95a 100644
--- a/include/linux/regulator/mt6359-regulator.h
+++ b/include/linux/regulator/mt6359-regulator.h
@@ -29,8 +29,7 @@ enum {
 	MT6359_ID_VCN18,
 	MT6359_ID_VFE28,
 	MT6359_ID_VCN13,
-	MT6359_ID_VCN33_1_BT,
-	MT6359_ID_VCN33_1_WIFI,
+	MT6359_ID_VCN33_1,
 	MT6359_ID_VAUX18,
 	MT6359_ID_VSRAM_OTHERS,
 	MT6359_ID_VEFUSE,
@@ -39,8 +38,7 @@ enum {
 	MT6359_ID_VBIF28,
 	MT6359_ID_VIO28,
 	MT6359_ID_VEMC,
-	MT6359_ID_VCN33_2_BT,
-	MT6359_ID_VCN33_2_WIFI,
+	MT6359_ID_VCN33_2,
 	MT6359_ID_VA12,
 	MT6359_ID_VA09,
 	MT6359_ID_VRF18,
@@ -51,6 +49,10 @@ enum {
 	MT6359_ID_VSRAM_PROC1,
 	MT6359_ID_VSIM2,
 	MT6359_ID_VSRAM_OTHERS_SSHUB,
+	MT6359_ID_VCN33_1_BT,
+	MT6359_ID_VCN33_1_WIFI,
+	MT6359_ID_VCN33_2_BT,
+	MT6359_ID_VCN33_2_WIFI,
 	MT6359_ID_RG_MAX,
 };
 
diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h
index 006e57fd7ca58..73ff522448a02 100644
--- a/include/linux/resctrl.h
+++ b/include/linux/resctrl.h
@@ -286,13 +286,15 @@ enum resctrl_schema_fmt {
 
 /**
  * struct resctrl_mon - Monitoring related data of a resctrl resource.
- * @num_rmid:		Number of RMIDs available.
- * @mbm_cfg_mask:	Memory transactions that can be tracked when bandwidth
- *			monitoring events can be configured.
- * @num_mbm_cntrs:	Number of assignable counters.
- * @mbm_cntr_assignable:Is system capable of supporting counter assignment?
- * @mbm_assign_on_mkdir:True if counters should automatically be assigned to MBM
- *			events of monitor groups created via mkdir.
+ * @num_rmid:			Number of RMIDs available.
+ * @mbm_cfg_mask:		Memory transactions that can be tracked when
+ *				bandwidth monitoring events can be configured.
+ * @num_mbm_cntrs:		Number of assignable counters.
+ * @mbm_cntr_assignable:	Is system capable of supporting counter assignment?
+ * @mbm_assign_on_mkdir:	True if counters should automatically be assigned to MBM
+ *				events of monitor groups created via mkdir.
+ * @mbm_cntr_configurable:	True if assignable counters are configurable.
+ * @mbm_cntr_assign_fixed:	True if the counter assignment mode is fixed.
  */
 struct resctrl_mon {
 	u32			num_rmid;
@@ -300,6 +302,8 @@ struct resctrl_mon {
 	int			num_mbm_cntrs;
 	bool			mbm_cntr_assignable;
 	bool			mbm_assign_on_mkdir;
+	bool			mbm_cntr_configurable;
+	bool			mbm_cntr_assign_fixed;
 };
 
 /**
diff --git a/include/linux/rhashtable-types.h b/include/linux/rhashtable-types.h
index fc2f596a6df1b..57c11ec9dc645 100644
--- a/include/linux/rhashtable-types.h
+++ b/include/linux/rhashtable-types.h
@@ -136,12 +136,26 @@ struct rhashtable_iter {
 	bool end_of_table;
 };
 
-int rhashtable_init_noprof(struct rhashtable *ht,
-		    const struct rhashtable_params *params);
+int __rhashtable_init_noprof(struct rhashtable *ht,
+		    const struct rhashtable_params *params,
+		    struct lock_class_key *key);
+#define rhashtable_init_noprof(ht, params)				\
+({									\
+	static struct lock_class_key __key;				\
+									\
+	__rhashtable_init_noprof(ht, params, &__key);			\
+})
 #define rhashtable_init(...)	alloc_hooks(rhashtable_init_noprof(__VA_ARGS__))
 
-int rhltable_init_noprof(struct rhltable *hlt,
-		  const struct rhashtable_params *params);
+int __rhltable_init_noprof(struct rhltable *hlt,
+		  const struct rhashtable_params *params,
+		  struct lock_class_key *key);
+#define rhltable_init_noprof(hlt, params)				\
+({									\
+	static struct lock_class_key __key;				\
+									\
+	__rhltable_init_noprof(hlt, params, &__key);			\
+})
 #define rhltable_init(...)	alloc_hooks(rhltable_init_noprof(__VA_ARGS__))
 
 #endif /* _LINUX_RHASHTABLE_TYPES_H */
diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h
index 78e7e588817cf..9e1f012f89db8 100644
--- a/include/linux/rtmutex.h
+++ b/include/linux/rtmutex.h
@@ -56,6 +56,8 @@ static inline struct task_struct *rt_mutex_owner(struct rt_mutex_base *lock)
 #endif
 extern void rt_mutex_base_init(struct rt_mutex_base *rtb);
 
+context_lock_struct(rt_mutex);
+
 /**
  * The rt_mutex structure
  *
@@ -108,8 +110,10 @@ do { \
 extern void __rt_mutex_init(struct rt_mutex *lock, const char *name, struct lock_class_key *key);
 
 #ifdef CONFIG_DEBUG_LOCK_ALLOC
-extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass);
-extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock);
+extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
+	__acquires(lock);
+extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock)
+	__acquires(lock);
 #define rt_mutex_lock(lock) rt_mutex_lock_nested(lock, 0)
 #define rt_mutex_lock_nest_lock(lock, nest_lock)			\
 	do {								\
@@ -118,15 +122,19 @@ extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *
 	} while (0)
 
 #else
-extern void rt_mutex_lock(struct rt_mutex *lock);
+extern void rt_mutex_lock(struct rt_mutex *lock) __acquires(lock);
 #define rt_mutex_lock_nested(lock, subclass) rt_mutex_lock(lock)
 #define rt_mutex_lock_nest_lock(lock, nest_lock) rt_mutex_lock(lock)
 #endif
 
-extern int rt_mutex_lock_interruptible(struct rt_mutex *lock);
-extern int rt_mutex_lock_killable(struct rt_mutex *lock);
-extern int rt_mutex_trylock(struct rt_mutex *lock);
+extern int rt_mutex_lock_interruptible(struct rt_mutex *lock)
+	__cond_acquires(0, lock);
+extern int rt_mutex_lock_killable(struct rt_mutex *lock)
+	__cond_acquires(0, lock);
+extern int rt_mutex_trylock(struct rt_mutex *lock)
+	__cond_acquires(true, lock);
 
-extern void rt_mutex_unlock(struct rt_mutex *lock);
+extern void rt_mutex_unlock(struct rt_mutex *lock)
+	__releases(lock);
 
 #endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index ee06cba5c6f53..67e46e5db16c4 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1411,6 +1411,13 @@ struct task_struct {
 	unsigned long			numa_pages_migrated;
 #endif /* CONFIG_NUMA_BALANCING */
 
+#ifdef CONFIG_SCHED_CACHE
+	struct callback_head		cache_work;
+	int				preferred_llc;
+	/* 1: task was enqueued to its preferred LLC, 0 otherwise */
+	int				pref_llc_queued;
+#endif
+
 	struct rseq_data		rseq;
 	struct sched_mm_cid		mm_cid;
 
@@ -1517,6 +1524,9 @@ struct task_struct {
 	/* KCOV descriptor wired with this task or NULL: */
 	struct kcov			*kcov;
 
+	/* KCOV descriptor for remote coverage collection from other tasks: */
+	struct kcov			*kcov_remote;
+
 	/* KCOV common handle for remote coverage collection: */
 	u64				kcov_handle;
 
@@ -2411,6 +2421,29 @@ static __always_inline int task_mm_cid(struct task_struct *t)
 }
 #endif
 
+#ifdef CONFIG_SCHED_CACHE
+
+struct sched_cache_time {
+	u64 runtime;
+	unsigned long epoch;
+};
+
+struct sched_cache_stat {
+	struct sched_cache_time __percpu *pcpu_sched;
+	raw_spinlock_t lock;
+	unsigned long epoch;
+	u64 nr_running_avg;
+	unsigned long next_scan;
+	unsigned long footprint;
+	int cpu;
+} ____cacheline_aligned_in_smp;
+
+#else
+
+struct sched_cache_stat { };
+
+#endif
+
 #ifndef MODULE
 #ifndef COMPILE_OFFSETS
 
diff --git a/include/linux/sched/clock.h b/include/linux/sched/clock.h
index 196f0ca351a25..39f0a7f94bfc9 100644
--- a/include/linux/sched/clock.h
+++ b/include/linux/sched/clock.h
@@ -33,6 +33,11 @@ extern u64 sched_clock_cpu(int cpu);
 extern void sched_clock_init(void);
 
 #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
+static inline int sched_clock_stable(void)
+{
+	return 1;
+}
+
 static inline void sched_clock_tick(void)
 {
 }
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index 2129e18ada58b..20b2343aa344b 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -207,6 +207,15 @@ struct sched_ext_entity {
 	u64			core_sched_at;	/* see scx_prio_less() */
 #endif
 
+	/*
+	 * Unique non-zero task ID assigned at fork. Persists across exec and
+	 * is never reused. Lets BPF schedulers identify tasks without storing
+	 * kernel pointers - arena-backed schedulers being one example. See
+	 * scx_bpf_tid_to_task().
+	 */
+	u64			tid;
+	struct rhash_head	tid_hash_node;	/* see SCX_OPS_TID_TO_TASK */
+
 	/* BPF scheduler modifiable fields */
 
 	/*
diff --git a/include/linux/sched/smt.h b/include/linux/sched/smt.h
index 166b19af956f8..cde6679c02781 100644
--- a/include/linux/sched/smt.h
+++ b/include/linux/sched/smt.h
@@ -4,16 +4,12 @@
 
 #include <linux/static_key.h>
 
-#ifdef CONFIG_SCHED_SMT
 extern struct static_key_false sched_smt_present;
 
 static __always_inline bool sched_smt_active(void)
 {
 	return static_branch_likely(&sched_smt_present);
 }
-#else
-static __always_inline bool sched_smt_active(void) { return false; }
-#endif
 
 void arch_smt_update(void);
 
diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h
index 36553e14866d0..b5d9d7c2b8add 100644
--- a/include/linux/sched/topology.h
+++ b/include/linux/sched/topology.h
@@ -67,7 +67,25 @@ struct sched_domain_shared {
 	atomic_t	ref;
 	atomic_t	nr_busy_cpus;
 	int		has_idle_cores;
-	int		nr_idle_scan;
+	union {
+		int	nr_idle_scan;
+		/*
+		 * Used during allocation to claim the sched_domain_shared
+		 * object at multiple levels.
+		 *
+		 * Note: between build and the first periodic LB tick, which
+		 * rewrites the union via update_idle_cpu_scan(), readers of
+		 * nr_idle_scan may observe the transient SD_* flag value as
+		 * the scan bound. The flag bits are small positive integers,
+		 * so the effect is just a slightly relaxed scan bound for one
+		 * window and self-heals on the first tick.
+		 */
+		int	alloc_flags;
+	};
+#ifdef CONFIG_SCHED_CACHE
+	unsigned long	util_avg;
+	unsigned long	capacity;
+#endif
 };
 
 struct sched_domain {
@@ -99,6 +117,12 @@ struct sched_domain {
 	u64 max_newidle_lb_cost;
 	unsigned long last_decay_max_lb_cost;
 
+#ifdef CONFIG_SCHED_CACHE
+	unsigned int llc_max;
+	unsigned int *llc_counts __counted_by_ptr(llc_max);
+	unsigned long llc_bytes;
+#endif
+
 #ifdef CONFIG_SCHEDSTATS
 	/* sched_balance_rq() stats */
 	unsigned int lb_count[CPU_MAX_IDLE_TYPES];
@@ -256,4 +280,10 @@ static inline int task_node(const struct task_struct *p)
 	return cpu_to_node(task_cpu(p));
 }
 
+#ifdef CONFIG_SCHED_CACHE
+extern void sched_update_llc_bytes(unsigned int cpu);
+#else
+static inline void sched_update_llc_bytes(unsigned int cpu) { }
+#endif
+
 #endif /* _LINUX_SCHED_TOPOLOGY_H */
diff --git a/include/linux/scmi_imx_protocol.h b/include/linux/scmi_imx_protocol.h
index 2407d7693b6ba..ab867463c08cc 100644
--- a/include/linux/scmi_imx_protocol.h
+++ b/include/linux/scmi_imx_protocol.h
@@ -52,6 +52,17 @@ struct scmi_imx_misc_ctrl_notify_report {
 	unsigned int		flags;
 };
 
+
+#define MISC_EXT_INFO_LEN_MAX	4
+struct scmi_imx_misc_reset_reason {
+	bool valid:1;
+	bool orig_valid:1;
+	bool err_valid:1;
+	u32 reason;
+	u32 origin;
+	u32 errid;
+};
+
 struct scmi_imx_misc_proto_ops {
 	int (*misc_ctrl_set)(const struct scmi_protocol_handle *ph, u32 id,
 			     u32 num, u32 *val);
@@ -61,6 +72,9 @@ struct scmi_imx_misc_proto_ops {
 				    u32 ctrl_id, u32 evt_id, u32 flags);
 	int (*misc_syslog)(const struct scmi_protocol_handle *ph, u16 *size,
 			   void *array);
+	int (*misc_reset_reason)(const struct scmi_protocol_handle *ph,
+				 bool system, struct scmi_imx_misc_reset_reason *boot_r,
+				 struct scmi_imx_misc_reset_reason *shut_r, u32 *extinfo);
 };
 
 /* See LMM_ATTRIBUTES in imx95.rst */
diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h
index aafaac1496b06..5ab73b1ab9aa4 100644
--- a/include/linux/scmi_protocol.h
+++ b/include/linux/scmi_protocol.h
@@ -15,10 +15,9 @@
 
 #define SCMI_MAX_STR_SIZE		64
 #define SCMI_SHORT_NAME_MAX_SIZE	16
-#define SCMI_MAX_NUM_RATES		16
 
 /**
- * struct scmi_revision_info - version information structure
+ * struct scmi_base_info - version information structure
  *
  * @major_ver: Major ABI version. Change here implies risk of backward
  *	compatibility break.
@@ -31,7 +30,7 @@
  * @vendor_id: A vendor identifier(Null terminated ASCII string)
  * @sub_vendor_id: A sub-vendor identifier(Null terminated ASCII string)
  */
-struct scmi_revision_info {
+struct scmi_base_info {
 	u16 major_ver;
 	u16 minor_ver;
 	u8 num_protocols;
@@ -41,27 +40,23 @@ struct scmi_revision_info {
 	char sub_vendor_id[SCMI_SHORT_NAME_MAX_SIZE];
 };
 
+struct scmi_clock_rates {
+	bool rate_discrete;
+	unsigned int num_rates;
+	u64 *rates;
+};
+
 struct scmi_clock_info {
 	char name[SCMI_MAX_STR_SIZE];
 	unsigned int enable_latency;
-	bool rate_discrete;
 	bool rate_changed_notifications;
 	bool rate_change_requested_notifications;
 	bool state_ctrl_forbidden;
 	bool rate_ctrl_forbidden;
 	bool parent_ctrl_forbidden;
 	bool extended_config;
-	union {
-		struct {
-			int num_rates;
-			u64 rates[SCMI_MAX_NUM_RATES];
-		} list;
-		struct {
-			u64 min_rate;
-			u64 max_rate;
-			u64 step_size;
-		} range;
-	};
+	u64 min_rate;
+	u64 max_rate;
 	int num_parents;
 	u32 *parents;
 };
@@ -91,6 +86,11 @@ enum scmi_clock_oem_config {
  * @info_get: get the information of the specified clock
  * @rate_get: request the current clock rate of a clock
  * @rate_set: set the clock rate of a clock
+ * @determine_rate: determine the effective rate that can be supported by a
+ *		    clock calculating the closest allowed rate.
+ *		    Note that @rate is an input/output parameter used both to
+ *		    describe the requested rate and report the closest match
+ * @all_rates_get: get the list of all available rates for the specified clock.
  * @enable: enables the specified clock
  * @disable: disables the specified clock
  * @state_get: get the status of the specified clock
@@ -108,6 +108,10 @@ struct scmi_clk_proto_ops {
 			u64 *rate);
 	int (*rate_set)(const struct scmi_protocol_handle *ph, u32 clk_id,
 			u64 rate);
+	int (*determine_rate)(const struct scmi_protocol_handle *ph, u32 clk_id,
+			      unsigned long *rate);
+	const struct scmi_clock_rates __must_check *(*all_rates_get)
+		(const struct scmi_protocol_handle *ph, u32 clk_id);
 	int (*enable)(const struct scmi_protocol_handle *ph, u32 clk_id,
 		      bool atomic);
 	int (*disable)(const struct scmi_protocol_handle *ph, u32 clk_id,
@@ -901,7 +905,7 @@ struct scmi_notify_ops {
  */
 struct scmi_handle {
 	struct device *dev;
-	struct scmi_revision_info *version;
+	struct scmi_base_info *version;
 
 	int __must_check (*devm_protocol_acquire)(struct scmi_device *sdev,
 						  u8 proto);
diff --git a/include/linux/security.h b/include/linux/security.h
index 41d7367cf4036..598cd2eb1dcd5 100644
--- a/include/linux/security.h
+++ b/include/linux/security.h
@@ -67,6 +67,7 @@ enum fs_value_type;
 struct watch;
 struct watch_notification;
 struct lsm_ctx;
+struct lsm_id;
 
 /* Default (no) options for the capable function */
 #define CAP_OPT_NONE 0x0
@@ -100,6 +101,17 @@ enum lsm_integrity_type {
 	LSM_INT_FSVERITY_BUILTINSIG_VALID,
 };
 
+enum lsm_integrity_verdict {
+	LSM_INT_VERDICT_NONE = 0,
+	LSM_INT_VERDICT_OK,
+	LSM_INT_VERDICT_UNSIGNED,
+	LSM_INT_VERDICT_PARTIALSIG,
+	LSM_INT_VERDICT_UNKNOWNKEY,
+	LSM_INT_VERDICT_UNEXPECTED,
+	LSM_INT_VERDICT_FAULT,
+	LSM_INT_VERDICT_BADSIG,
+};
+
 /*
  * These are reasons that can be passed to the security_locked_down()
  * LSM hook. Lockdown reasons that protect kernel integrity (ie, the
@@ -459,7 +471,7 @@ int security_inode_getsecurity(struct mnt_idmap *idmap,
 			       struct inode *inode, const char *name,
 			       void **buffer, bool alloc);
 int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags);
-int security_inode_listsecurity(struct inode *inode, char *buffer, size_t buffer_size);
+int security_inode_listsecurity(struct inode *inode, char **buffer, ssize_t *remaining_size);
 void security_inode_getlsmprop(struct inode *inode, struct lsm_prop *prop);
 int security_inode_copy_up(struct dentry *src, struct cred **new);
 int security_inode_copy_up_xattr(struct dentry *src, const char *name);
@@ -1097,7 +1109,8 @@ static inline int security_inode_setsecurity(struct inode *inode, const char *na
 	return -EOPNOTSUPP;
 }
 
-static inline int security_inode_listsecurity(struct inode *inode, char *buffer, size_t buffer_size)
+static inline int security_inode_listsecurity(struct inode *inode,
+					char **buffer, ssize_t *remaining_size)
 {
 	return 0;
 }
@@ -2303,6 +2316,12 @@ extern int security_bpf_prog(struct bpf_prog *prog);
 extern int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
 				   struct bpf_token *token, bool kernel);
 extern void security_bpf_map_free(struct bpf_map *map);
+extern int security_bpf_prog_load_post_integrity(struct bpf_prog *prog,
+					union bpf_attr *attr,
+					struct bpf_token *token,
+					bool kernel,
+					const struct lsm_id *lsmid,
+					enum lsm_integrity_verdict verdict);
 extern int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
 				  struct bpf_token *token, bool kernel);
 extern void security_bpf_prog_free(struct bpf_prog *prog);
@@ -2337,6 +2356,16 @@ static inline int security_bpf_map_create(struct bpf_map *map, union bpf_attr *a
 static inline void security_bpf_map_free(struct bpf_map *map)
 { }
 
+static inline int security_bpf_prog_load_post_integrity(struct bpf_prog *prog,
+					  union bpf_attr *attr,
+					  struct bpf_token *token,
+					  bool kernel,
+					  const struct lsm_id *lsmid,
+					  enum lsm_integrity_verdict verdict)
+{
+	return 0;
+}
+
 static inline int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
 					 struct bpf_token *token, bool kernel)
 {
diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
index 5a40252b83348..f865491c4f2c2 100644
--- a/include/linux/seqlock.h
+++ b/include/linux/seqlock.h
@@ -1259,14 +1259,15 @@ static __always_inline void __scoped_seqlock_cleanup(struct ss_tmp *sst)
 
 extern void __scoped_seqlock_invalid_target(void);
 
-#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || defined(CONFIG_KASAN)
+#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || \
+	defined(CONFIG_KASAN) || defined(CONFIG_UBSAN_ALIGNMENT)
 /*
  * For some reason some GCC-8 architectures (nios2, alpha) have trouble
  * determining that the ss_done state is impossible in __scoped_seqlock_next()
  * below.
  *
- * Similarly KASAN is known to confuse compilers enough to break this. But we
- * don't care about code quality for KASAN builds anyway.
+ * Similarly KASAN and UBSAN_ALIGNMENT are known to confuse compilers enough
+ * to break this. But we don't care about code quality for such builds anyway.
  */
 static inline void __scoped_seqlock_bug(void) { }
 #else
diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h
index 666430b478997..4f7bbdd900176 100644
--- a/include/linux/serial_core.h
+++ b/include/linux/serial_core.h
@@ -533,6 +533,7 @@ struct uart_port {
 #define UPF_HARD_FLOW		((__force upf_t) (UPF_AUTO_CTS | UPF_AUTO_RTS))
 /* Port has hardware-assisted s/w flow control */
 #define UPF_SOFT_FLOW		((__force upf_t) BIT_ULL(22))
+/* Deprecated: use uart_set_cons_flow_enabled()/uart_cons_flow_enabled() instead. */
 #define UPF_CONS_FLOW		((__force upf_t) BIT_ULL(23))
 #define UPF_SHARE_IRQ		((__force upf_t) BIT_ULL(24))
 #define UPF_EXAR_EFR		((__force upf_t) BIT_ULL(25))
@@ -567,6 +568,7 @@ struct uart_port {
 #define UPSTAT_SYNC_FIFO	((__force upstat_t) (1 << 5))
 
 	bool			hw_stopped;		/* sw-assisted CTS flow state */
+	bool			cons_flow;		/* user specified console flow control */
 	unsigned int		mctrl;			/* current modem ctrl settings */
 	unsigned int		frame_time;		/* frame timing in ns */
 	unsigned int		type;			/* port type */
@@ -1163,6 +1165,16 @@ static inline bool uart_softcts_mode(struct uart_port *uport)
 	return ((uport->status & mask) == UPSTAT_CTS_ENABLE);
 }
 
+static inline void uart_set_cons_flow_enabled(struct uart_port *uport, bool enabled)
+{
+	uport->cons_flow = enabled;
+}
+
+static inline bool uart_cons_flow_enabled(const struct uart_port *uport)
+{
+	return uport->cons_flow;
+}
+
 /*
  * The following are helper functions for the low level drivers.
  */
diff --git a/include/linux/serial_sci.h b/include/linux/serial_sci.h
index 0f2f50b8d28e2..36c795d61f7e8 100644
--- a/include/linux/serial_sci.h
+++ b/include/linux/serial_sci.h
@@ -51,7 +51,6 @@ struct plat_sci_port_ops {
  */
 struct plat_sci_port {
 	unsigned int	type;			/* SCI / SCIF / IRDA / HSCIF */
-	upf_t		flags;			/* UPF_* flags */
 
 	unsigned int	sampling_rate;
 	unsigned int	scscr;			/* SCSCR initialization */
diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index 93a0ba872ebe0..acb8dd961b45c 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -127,7 +127,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug,
 void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end);
 int shmem_unuse(unsigned int type);
 
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SHMEM)
 unsigned long shmem_allowable_huge_orders(struct inode *inode,
 				struct vm_area_struct *vma, pgoff_t index,
 				loff_t write_end, bool shmem_huge_force);
diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h
index 2bcf78a4de7b9..b8d1e76222c18 100644
--- a/include/linux/skbuff.h
+++ b/include/linux/skbuff.h
@@ -1082,9 +1082,7 @@ struct sk_buff {
 	__u16			network_header;
 	__u16			mac_header;
 
-#ifdef CONFIG_KCOV
-	u64			kcov_handle;
-#endif
+	struct kcov_common_handle_id kcov_handle;
 
 	); /* end headers group */
 
@@ -1313,7 +1311,8 @@ static inline bool skb_data_unref(const struct sk_buff *skb,
 	return true;
 }
 
-void __fix_address sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb,
+void __fix_address sk_skb_reason_drop(const struct sock *sk,
+				      struct sk_buff *skb,
 				      enum skb_drop_reason reason);
 
 static inline void
@@ -5437,20 +5436,14 @@ static inline void skb_reset_csum_not_inet(struct sk_buff *skb)
 }
 
 static inline void skb_set_kcov_handle(struct sk_buff *skb,
-				       const u64 kcov_handle)
+				       struct kcov_common_handle_id kcov_handle)
 {
-#ifdef CONFIG_KCOV
 	skb->kcov_handle = kcov_handle;
-#endif
 }
 
-static inline u64 skb_get_kcov_handle(struct sk_buff *skb)
+static inline struct kcov_common_handle_id skb_get_kcov_handle(struct sk_buff *skb)
 {
-#ifdef CONFIG_KCOV
 	return skb->kcov_handle;
-#else
-	return 0;
-#endif
 }
 
 static inline void skb_mark_for_recycle(struct sk_buff *skb)
diff --git a/include/linux/slab.h b/include/linux/slab.h
index 2b5ab488e96b0..7b46fa499b081 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -499,14 +499,80 @@ int kmem_cache_shrink(struct kmem_cache *s);
 				.usersize	= sizeof_field(struct __struct, __field),	\
 			}, (__flags))
 
+#ifdef CONFIG_KMALLOC_PARTITION_CACHES
+typedef struct { unsigned long v; } kmalloc_token_t;
+#ifdef CONFIG_KMALLOC_PARTITION_RANDOM
+extern unsigned long random_kmalloc_seed;
+#define __kmalloc_token(...) ((kmalloc_token_t){ .v = _CODE_LOCATION_ })
+#elif defined(CONFIG_KMALLOC_PARTITION_TYPED)
+#define __kmalloc_token(...) ((kmalloc_token_t){ .v = __builtin_infer_alloc_token(__VA_ARGS__) })
+#endif
+#define DECL_TOKEN_PARAM(_token)	, kmalloc_token_t (_token)
+#define _PASS_TOKEN_PARAM(_token)	, (_token)
+#define PASS_TOKEN_PARAM(_token)	(_token)
+#define DECL_TOKEN_PARAMS(_size, _token) size_t (_size), kmalloc_token_t (_token)
+#define PASS_TOKEN_PARAMS(_size, _token) (_size), (_token)
+#else /* !CONFIG_KMALLOC_PARTITION_CACHES */
+typedef struct {} kmalloc_token_t;
+#define __kmalloc_token(...) ((kmalloc_token_t){}) /* no-op */
+#define DECL_TOKEN_PARAM(_token)
+#define _PASS_TOKEN_PARAM(_token)
+#define PASS_TOKEN_PARAM(_token)	((kmalloc_token_t){})
+#define DECL_TOKEN_PARAMS(_size, _token) size_t (_size)
+#define PASS_TOKEN_PARAMS(_size, _token) (_size)
+#endif /* CONFIG_KMALLOC_PARTITION_CACHES */
+
 /*
  * Common kmalloc functions provided by all allocators
  */
-void * __must_check krealloc_node_align_noprof(const void *objp, size_t new_size,
+void * __must_check krealloc_node_align_noprof(const void *objp,
+					       DECL_TOKEN_PARAMS(new_size, token),
 					       unsigned long align,
 					       gfp_t flags, int nid) __realloc_size(2);
-#define krealloc_noprof(_o, _s, _f)	krealloc_node_align_noprof(_o, _s, 1, _f, NUMA_NO_NODE)
-#define krealloc_node_align(...)	alloc_hooks(krealloc_node_align_noprof(__VA_ARGS__))
+#define krealloc_noprof(_o, _s, _f)	krealloc_node_align_noprof(_o, PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), 1, _f, NUMA_NO_NODE)
+#if 0 /* kernel-doc */
+/**
+ * krealloc_node_align - reallocate memory. The contents will remain unchanged.
+ * @p: object to reallocate memory for.
+ * @new_size: how many bytes of memory are required.
+ * @align: desired alignment.
+ * @flags: the type of memory to allocate.
+ * @nid: NUMA node or NUMA_NO_NODE
+ *
+ * If @p is %NULL, krealloc() behaves exactly like kmalloc().  If @new_size
+ * is 0 and @p is not a %NULL pointer, the object pointed to is freed.
+ *
+ * Only alignments up to those guaranteed by kmalloc() will be honored. Please see
+ * Documentation/core-api/memory-allocation.rst for more details.
+ *
+ * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
+ * initial memory allocation, every subsequent call to this API for the same
+ * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
+ * __GFP_ZERO is not fully honored by this API.
+ *
+ * When slub_debug_orig_size() is off, krealloc() only knows about the bucket
+ * size of an allocation (but not the exact size it was allocated with) and
+ * hence implements the following semantics for shrinking and growing buffers
+ * with __GFP_ZERO::
+ *
+ *           new             bucket
+ *   0       size             size
+ *   |--------|----------------|
+ *   |  keep  |      zero      |
+ *
+ * Otherwise, the original allocation size 'orig_size' could be used to
+ * precisely clear the requested size, and the new size will also be stored
+ * as the new 'orig_size'.
+ *
+ * In any case, the contents of the object pointed to are preserved up to the
+ * lesser of the new and old sizes.
+ *
+ * Return: pointer to the allocated memory or %NULL in case of error
+ */
+void *krealloc_node_align(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid);
+#endif
+#define krealloc_node_align(p, new_size, align, flags, nid) \
+	alloc_hooks(krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(new_size, __kmalloc_token(new_size)), align, flags, nid))
 #define krealloc_node(_o, _s, _f, _n)	krealloc_node_align(_o, _s, 1, _f, _n)
 #define krealloc(...)			krealloc_node(__VA_ARGS__, NUMA_NO_NODE)
 
@@ -612,10 +678,10 @@ static inline unsigned int arch_slab_minalign(void)
 #define SLAB_OBJ_MIN_SIZE      (KMALLOC_MIN_SIZE < 16 ? \
                                (KMALLOC_MIN_SIZE) : 16)
 
-#ifdef CONFIG_RANDOM_KMALLOC_CACHES
-#define RANDOM_KMALLOC_CACHES_NR	15 // # of cache copies
+#ifdef CONFIG_KMALLOC_PARTITION_CACHES
+#define KMALLOC_PARTITION_CACHES_NR	15 // # of cache copies
 #else
-#define RANDOM_KMALLOC_CACHES_NR	0
+#define KMALLOC_PARTITION_CACHES_NR	0
 #endif
 
 /*
@@ -634,8 +700,8 @@ enum kmalloc_cache_type {
 #ifndef CONFIG_MEMCG
 	KMALLOC_CGROUP = KMALLOC_NORMAL,
 #endif
-	KMALLOC_RANDOM_START = KMALLOC_NORMAL,
-	KMALLOC_RANDOM_END = KMALLOC_RANDOM_START + RANDOM_KMALLOC_CACHES_NR,
+	KMALLOC_PARTITION_START = KMALLOC_NORMAL,
+	KMALLOC_PARTITION_END = KMALLOC_PARTITION_START + KMALLOC_PARTITION_CACHES_NR,
 #ifdef CONFIG_SLUB_TINY
 	KMALLOC_RECLAIM = KMALLOC_NORMAL,
 #else
@@ -662,19 +728,19 @@ extern kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES];
 	(IS_ENABLED(CONFIG_ZONE_DMA)   ? __GFP_DMA : 0) |	\
 	(IS_ENABLED(CONFIG_MEMCG) ? __GFP_ACCOUNT : 0))
 
-extern unsigned long random_kmalloc_seed;
-
-static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, unsigned long caller)
+static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, kmalloc_token_t token)
 {
 	/*
 	 * The most common case is KMALLOC_NORMAL, so test for it
 	 * with a single branch for all the relevant flags.
 	 */
 	if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0))
-#ifdef CONFIG_RANDOM_KMALLOC_CACHES
-		/* RANDOM_KMALLOC_CACHES_NR (=15) copies + the KMALLOC_NORMAL */
-		return KMALLOC_RANDOM_START + hash_64(caller ^ random_kmalloc_seed,
-						      ilog2(RANDOM_KMALLOC_CACHES_NR + 1));
+#ifdef CONFIG_KMALLOC_PARTITION_RANDOM
+		/* KMALLOC_PARTITION_CACHES_NR (=15) copies + the KMALLOC_NORMAL */
+		return KMALLOC_PARTITION_START + hash_64(token.v ^ random_kmalloc_seed,
+							 ilog2(KMALLOC_PARTITION_CACHES_NR + 1));
+#elif defined(CONFIG_KMALLOC_PARTITION_TYPED)
+		return KMALLOC_PARTITION_START + token.v;
 #else
 		return KMALLOC_NORMAL;
 #endif
@@ -858,16 +924,22 @@ unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf);
 #define PASS_BUCKET_PARAM(_b)		NULL
 #endif
 
+#define DECL_KMALLOC_PARAMS(_size, _b, _token) DECL_BUCKET_PARAMS(_size, _b) \
+					       DECL_TOKEN_PARAM(_token)
+
+#define PASS_KMALLOC_PARAMS(_size, _b, _token) PASS_BUCKET_PARAMS(_size, _b) \
+					       _PASS_TOKEN_PARAM(_token)
+
 /*
  * The following functions are not to be used directly and are intended only
  * for internal use from kmalloc() and kmalloc_node()
  * with the exception of kunit tests
  */
 
-void *__kmalloc_noprof(size_t size, gfp_t flags)
+void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags)
 				__assume_kmalloc_alignment __alloc_size(1);
 
-void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
+void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node)
 				__assume_kmalloc_alignment __alloc_size(1);
 
 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size)
@@ -883,6 +955,23 @@ void *__kmalloc_large_noprof(size_t size, gfp_t flags)
 void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
 				__assume_page_alignment __alloc_size(1);
 
+static __always_inline __alloc_size(1) void *_kmalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token)
+{
+	if (__builtin_constant_p(size) && size) {
+		unsigned int index;
+
+		if (size > KMALLOC_MAX_CACHE_SIZE)
+			return __kmalloc_large_noprof(size, flags);
+
+		index = kmalloc_index(size);
+		return __kmalloc_cache_noprof(
+				kmalloc_caches[kmalloc_type(flags, token)][index],
+				flags, size);
+	}
+	return __kmalloc_noprof(PASS_TOKEN_PARAMS(size, token), flags);
+}
+#define kmalloc_noprof(...)			_kmalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
+#if 0 /* kernel-doc */
 /**
  * kmalloc - allocate kernel memory
  * @size: how many bytes of memory are required.
@@ -938,25 +1027,27 @@ void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node)
  *	Try really hard to succeed the allocation but fail
  *	eventually.
  */
-static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t flags)
-{
-	if (__builtin_constant_p(size) && size) {
-		unsigned int index;
-
-		if (size > KMALLOC_MAX_CACHE_SIZE)
-			return __kmalloc_large_noprof(size, flags);
-
-		index = kmalloc_index(size);
-		return __kmalloc_cache_noprof(
-				kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
-				flags, size);
-	}
-	return __kmalloc_noprof(size, flags);
-}
-#define kmalloc(...)				alloc_hooks(kmalloc_noprof(__VA_ARGS__))
+void *kmalloc(size_t size, gfp_t flags);
+#endif
+#define kmalloc(size, flags)			alloc_hooks(kmalloc_noprof(size, flags))
 
-void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node);
-#define kmalloc_nolock(...)			alloc_hooks(kmalloc_nolock_noprof(__VA_ARGS__))
+void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node);
+#define kmalloc_nolock_noprof(_s, _f, _n)	_kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), _f, _n)
+#if 0 /* kernel-doc */
+/**
+ * kmalloc_nolock - Allocate an object of given size from any context.
+ * @size: size to allocate
+ * @gfp_flags: GFP flags. Only __GFP_ACCOUNT, __GFP_ZERO, __GFP_NO_OBJ_EXT
+ * allowed.
+ * @node: node number of the target node.
+ *
+ * Return: pointer to the new object or NULL in case of error.
+ * NULL does not mean EBUSY or EAGAIN. It means ENOMEM.
+ * There is no reason to call it again and expect !NULL.
+ */
+void *kmalloc_nolock(size_t size, gfp_t gfp_flags, int node);
+#endif
+#define kmalloc_nolock(size, gfp_flags, node)	alloc_hooks(kmalloc_nolock_noprof(size, gfp_flags, node))
 
 /**
  * __alloc_objs - Allocate objects of a given type using
@@ -1060,12 +1151,12 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node);
 	__alloc_flex(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT)
 
 #define kmem_buckets_alloc(_b, _size, _flags)	\
-	alloc_hooks(__kmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE))
+	alloc_hooks(__kmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE))
 
 #define kmem_buckets_alloc_track_caller(_b, _size, _flags)	\
-	alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE, _RET_IP_))
+	alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE, _RET_IP_))
 
-static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gfp_t flags, int node)
+static __always_inline __alloc_size(1) void *_kmalloc_node_noprof(size_t size, gfp_t flags, int node, kmalloc_token_t token)
 {
 	if (__builtin_constant_p(size) && size) {
 		unsigned int index;
@@ -1075,29 +1166,48 @@ static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gf
 
 		index = kmalloc_index(size);
 		return __kmalloc_cache_node_noprof(
-				kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index],
+				kmalloc_caches[kmalloc_type(flags, token)][index],
 				flags, node, size);
 	}
-	return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node);
+	return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(size, NULL, token), flags, node);
 }
+#define kmalloc_node_noprof(...)		_kmalloc_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
 #define kmalloc_node(...)			alloc_hooks(kmalloc_node_noprof(__VA_ARGS__))
 
+static inline __alloc_size(1, 2) void *_kmalloc_array_noprof(size_t n, size_t size, gfp_t flags, kmalloc_token_t token)
+{
+	size_t bytes;
+
+	if (unlikely(check_mul_overflow(n, size, &bytes)))
+		return NULL;
+	return _kmalloc_noprof(bytes, flags, token);
+}
+#define kmalloc_array_noprof(...)		_kmalloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
+#if 0 /* kernel-doc */
 /**
  * kmalloc_array - allocate memory for an array.
  * @n: number of elements.
  * @size: element size.
  * @flags: the type of memory to allocate (see kmalloc).
  */
-static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t size, gfp_t flags)
+void *kmalloc_array(size_t n, size_t size, gfp_t flags);
+#endif
+#define kmalloc_array(n, size, flags)		alloc_hooks(kmalloc_array_noprof(n, size, flags))
+
+static inline __realloc_size(2, 3) void * __must_check _krealloc_array_noprof(void *p,
+								       size_t new_n,
+								       size_t new_size,
+								       gfp_t flags, kmalloc_token_t token)
 {
 	size_t bytes;
 
-	if (unlikely(check_mul_overflow(n, size, &bytes)))
+	if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
 		return NULL;
-	return kmalloc_noprof(bytes, flags);
-}
-#define kmalloc_array(...)			alloc_hooks(kmalloc_array_noprof(__VA_ARGS__))
 
+	return krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(bytes, token), 1, flags, NUMA_NO_NODE);
+}
+#define krealloc_array_noprof(...)		_krealloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
+#if 0 /* kernel-doc */
 /**
  * krealloc_array - reallocate memory for an array.
  * @p: pointer to the memory chunk to reallocate
@@ -1115,19 +1225,9 @@ static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t siz
  * In any case, the contents of the object pointed to are preserved up to the
  * lesser of the new and old sizes.
  */
-static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(void *p,
-								       size_t new_n,
-								       size_t new_size,
-								       gfp_t flags)
-{
-	size_t bytes;
-
-	if (unlikely(check_mul_overflow(new_n, new_size, &bytes)))
-		return NULL;
-
-	return krealloc_noprof(p, bytes, flags);
-}
-#define krealloc_array(...)			alloc_hooks(krealloc_array_noprof(__VA_ARGS__))
+void *krealloc_array(void *p, size_t new_n, size_t new_size, gfp_t flags);
+#endif
+#define krealloc_array(p, new_n, new_size, flags) alloc_hooks(krealloc_array_noprof(p, new_n, new_size, flags))
 
 /**
  * kcalloc - allocate memory for an array. The memory is set to zero.
@@ -1137,10 +1237,10 @@ static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(voi
  */
 #define kcalloc(n, size, flags)		kmalloc_array(n, size, (flags) | __GFP_ZERO)
 
-void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node,
+void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node,
 					 unsigned long caller) __alloc_size(1);
 #define kmalloc_node_track_caller_noprof(size, flags, node, caller) \
-	__kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node, caller)
+	__kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(size, NULL, __kmalloc_token(size)), flags, node, caller)
 #define kmalloc_node_track_caller(...)		\
 	alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_))
 
@@ -1157,17 +1257,18 @@ void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flag
 #define kmalloc_track_caller_noprof(...)	\
 		kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_)
 
-static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags,
-							  int node)
+static inline __alloc_size(1, 2) void *_kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags,
+								  int node, kmalloc_token_t token)
 {
 	size_t bytes;
 
 	if (unlikely(check_mul_overflow(n, size, &bytes)))
 		return NULL;
 	if (__builtin_constant_p(n) && __builtin_constant_p(size))
-		return kmalloc_node_noprof(bytes, flags, node);
-	return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(bytes, NULL), flags, node);
+		return _kmalloc_node_noprof(bytes, flags, node, token);
+	return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), flags, node);
 }
+#define kmalloc_array_node_noprof(...)		_kmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
 #define kmalloc_array_node(...)			alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__))
 
 #define kcalloc_node(_n, _size, _flags, _node)	\
@@ -1178,44 +1279,73 @@ static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_
  */
 #define kmem_cache_zalloc(_k, _flags)		kmem_cache_alloc(_k, (_flags)|__GFP_ZERO)
 
+static inline __alloc_size(1) void *_kzalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token)
+{
+	return _kmalloc_noprof(size, flags | __GFP_ZERO, token);
+}
+#define kzalloc_noprof(...)			_kzalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
+#if 0 /* kernel-doc */
 /**
  * kzalloc - allocate memory. The memory is set to zero.
  * @size: how many bytes of memory are required.
  * @flags: the type of memory to allocate (see kmalloc).
  */
-static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags)
-{
-	return kmalloc_noprof(size, flags | __GFP_ZERO);
-}
-#define kzalloc(...)				alloc_hooks(kzalloc_noprof(__VA_ARGS__))
+void *kzalloc(size_t size, gfp_t flags);
+#endif
+#define kzalloc(size, flags)			alloc_hooks(kzalloc_noprof(size, flags))
 #define kzalloc_node(_size, _flags, _node)	kmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
 
-void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align,
+void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long align,
 			     gfp_t flags, int node) __alloc_size(1);
 #define kvmalloc_node_align_noprof(_size, _align, _flags, _node)	\
-	__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, NULL), _align, _flags, _node)
+	__kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, NULL, __kmalloc_token(_size)), _align, _flags, _node)
 #define kvmalloc_node_align(...)		\
 	alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__))
-#define kvmalloc_node(_s, _f, _n)		kvmalloc_node_align(_s, 1, _f, _n)
+#if 0 /* kernel-doc */
+/**
+ * kvmalloc_node - attempt to allocate physically contiguous memory, but upon
+ * failure, fall back to non-contiguous (vmalloc) allocation.
+ * @size: size of the request.
+ * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
+ * @node: numa node to allocate from
+ *
+ * Only alignments up to those guaranteed by kmalloc() will be honored. Please see
+ * Documentation/core-api/memory-allocation.rst for more details.
+ *
+ * Uses kmalloc to get the memory but if the allocation fails then falls back
+ * to the vmalloc allocator. Use kvfree for freeing the memory.
+ *
+ * GFP_NOWAIT and GFP_ATOMIC are supported, the __GFP_NORETRY modifier is not.
+ * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
+ * preferable to the vmalloc fallback, due to visible performance drawbacks.
+ *
+ * Return: pointer to the allocated memory of %NULL in case of failure
+ */
+void *kvmalloc_node(size_t size, gfp_t flags, int node);
+#endif
+#define kvmalloc_node(size, flags, node)	kvmalloc_node_align(size, 1, flags, node)
+#define kvmalloc_node_noprof(size, flags, node)	\
+	kvmalloc_node_align_noprof(size, 1, flags, node)
 #define kvmalloc(...)				kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE)
+#define kvmalloc_noprof(_size, _flags)		kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE)
 #define kvzalloc(_size, _flags)			kvmalloc(_size, (_flags)|__GFP_ZERO)
 
 #define kvzalloc_node(_size, _flags, _node)	kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node)
 
 #define kmem_buckets_valloc(_b, _size, _flags)	\
-	alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), 1, _flags, NUMA_NO_NODE))
+	alloc_hooks(__kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), 1, _flags, NUMA_NO_NODE))
 
 static inline __alloc_size(1, 2) void *
-kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node)
+_kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node, kmalloc_token_t token)
 {
 	size_t bytes;
 
 	if (unlikely(check_mul_overflow(n, size, &bytes)))
 		return NULL;
 
-	return kvmalloc_node_align_noprof(bytes, 1, flags, node);
+	return __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), 1, flags, node);
 }
-
+#define kvmalloc_array_node_noprof(...)		_kvmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))
 #define kvmalloc_array_noprof(...)		kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
 #define kvcalloc_node_noprof(_n,_s,_f,_node)	kvmalloc_array_node_noprof(_n,_s,(_f)|__GFP_ZERO,_node)
 #define kvcalloc_noprof(...)			kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_NODE)
@@ -1224,10 +1354,40 @@ kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node)
 #define kvcalloc_node(...)			alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__))
 #define kvcalloc(...)				alloc_hooks(kvcalloc_noprof(__VA_ARGS__))
 
-void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align,
+void *kvrealloc_node_align_noprof(const void *p, DECL_TOKEN_PARAMS(size, token), unsigned long align,
 				  gfp_t flags, int nid) __realloc_size(2);
-#define kvrealloc_node_align(...)		\
-	alloc_hooks(kvrealloc_node_align_noprof(__VA_ARGS__))
+#if 0 /* kernel-doc */
+/**
+ * kvrealloc_node_align - reallocate memory; contents remain unchanged
+ * @p: object to reallocate memory for
+ * @size: the size to reallocate
+ * @align: desired alignment
+ * @flags: the flags for the page level allocator
+ * @nid: NUMA node id
+ *
+ * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0
+ * and @p is not a %NULL pointer, the object pointed to is freed.
+ *
+ * Only alignments up to those guaranteed by kmalloc() will be honored. Please see
+ * Documentation/core-api/memory-allocation.rst for more details.
+ *
+ * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
+ * initial memory allocation, every subsequent call to this API for the same
+ * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
+ * __GFP_ZERO is not fully honored by this API.
+ *
+ * In any case, the contents of the object pointed to are preserved up to the
+ * lesser of the new and old sizes.
+ *
+ * This function must not be called concurrently with itself or kvfree() for the
+ * same memory allocation.
+ *
+ * Return: pointer to the allocated memory or %NULL in case of error
+ */
+void *kvrealloc_node_align(const void *p, size_t size, unsigned long align, gfp_t flags, int nid);
+#endif
+#define kvrealloc_node_align(p, size, align, flags, nid)	\
+	alloc_hooks(kvrealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(size, __kmalloc_token(size)), align, flags, nid))
 #define kvrealloc_node(_p, _s, _f, _n)		kvrealloc_node_align(_p, _s, 1, _f, _n)
 #define kvrealloc(...)				kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE)
 
diff --git a/include/linux/soc/airoha/airoha_offload.h b/include/linux/soc/airoha/airoha_offload.h
index d01ef4a6b3d7c..7589fccfeef6d 100644
--- a/include/linux/soc/airoha/airoha_offload.h
+++ b/include/linux/soc/airoha/airoha_offload.h
@@ -71,9 +71,9 @@ static inline void airoha_ppe_dev_check_skb(struct airoha_ppe_dev *dev,
 #define NPU_RX1_DESC_NUM	512
 
 /* CTRL */
-#define NPU_RX_DMA_DESC_LAST_MASK	BIT(27)
-#define NPU_RX_DMA_DESC_LEN_MASK	GENMASK(26, 14)
-#define NPU_RX_DMA_DESC_CUR_LEN_MASK	GENMASK(13, 1)
+#define NPU_RX_DMA_DESC_LAST_MASK	BIT(29)
+#define NPU_RX_DMA_DESC_LEN_MASK	GENMASK(28, 15)
+#define NPU_RX_DMA_DESC_CUR_LEN_MASK	GENMASK(14, 1)
 #define NPU_RX_DMA_DESC_DONE_MASK	BIT(0)
 /* INFO */
 #define NPU_RX_DMA_PKT_COUNT_MASK	GENMASK(31, 29)
diff --git a/include/linux/soc/mediatek/mtk-cmdq.h b/include/linux/soc/mediatek/mtk-cmdq.h
index a06b5a61f3375..03bb854625663 100644
--- a/include/linux/soc/mediatek/mtk-cmdq.h
+++ b/include/linux/soc/mediatek/mtk-cmdq.h
@@ -445,6 +445,24 @@ static inline int cmdq_pkt_jump(struct cmdq_pkt *pkt, dma_addr_t addr, u8 shift_
  */
 int cmdq_pkt_jump_rel(struct cmdq_pkt *pkt, s32 offset, u8 shift_pa);
 
+/**
+ * cmdq_pkt_jump_rel_temp() - Temporary wrapper for new CMDQ helper API
+ * @pkt:	the CMDQ packet
+ * @offset:	relative offset of target instruction buffer from current PC.
+ * @shift_pa:	[DEPRECATED] shift bits of physical address in CMDQ instruction.
+ *		This value is got by cmdq_get_shift_pa().
+ *
+ * This function is a temporary wrapper that was introduced only for ease of
+ * migration of the many users of the CMDQ API located in multiple kernel
+ * subsystems.
+ *
+ * This has to be removed after all users are migrated to the newer CMDQ API.
+ */
+static inline int cmdq_pkt_jump_rel_temp(struct cmdq_pkt *pkt, s32 offset, u8 shift_pa)
+{
+	return cmdq_pkt_jump_rel(pkt, offset, shift_pa);
+}
+
 /**
  * cmdq_pkt_eoc() - Append EOC and ask GCE to generate an IRQ at end of execution
  * @pkt:	The CMDQ packet
@@ -599,6 +617,12 @@ static inline int cmdq_pkt_jump_rel(struct cmdq_pkt *pkt, s32 offset, u8 shift_p
 	return -EINVAL;
 }
 
+/* This wrapper has to be removed after all users migrated to jump_rel */
+static inline int cmdq_pkt_jump_rel_temp(struct cmdq_pkt *pkt, s32 offset, u8 shift_pa)
+{
+	return -EINVAL;
+}
+
 static inline int cmdq_pkt_eoc(struct cmdq_pkt *pkt)
 {
 	return -EINVAL;
diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h
index 227125d843183..f3ed63e475ab1 100644
--- a/include/linux/soc/qcom/llcc-qcom.h
+++ b/include/linux/soc/qcom/llcc-qcom.h
@@ -89,18 +89,20 @@
 
 /**
  * struct llcc_slice_desc - Cache slice descriptor
- * @slice_id: llcc slice id
- * @slice_size: Size allocated for the llcc slice
+ * @slice_id: LLCC slice id
+ * @uid: Unique ID associated with the LLCC device
+ * @slice_size: Size allocated for the LLCC slice
  * @refcount: Atomic counter to track activate/deactivate calls
  */
 struct llcc_slice_desc {
 	u32 slice_id;
+	u32 uid;
 	size_t slice_size;
 	refcount_t refcount;
 };
 
 /**
- * struct llcc_edac_reg_data - llcc edac registers data for each error type
+ * struct llcc_edac_reg_data - LLCC EDAC registers data for each error type
  * @name: Name of the error
  * @reg_cnt: Number of registers
  * @count_mask: Mask value to get the error count
@@ -146,21 +148,23 @@ struct llcc_edac_reg_offset {
 };
 
 /**
- * struct llcc_drv_data - Data associated with the llcc driver
- * @regmaps: regmaps associated with the llcc device
- * @bcast_regmap: regmap associated with llcc broadcast OR offset
- * @bcast_and_regmap: regmap associated with llcc broadcast AND offset
+ * struct llcc_drv_data - Data associated with the LLCC driver
+ * @dev: device back-pointer for this LLCC instance
+ * @regmaps: regmaps associated with the LLCC device
+ * @bcast_regmap: regmap associated with LLCC broadcast OR offset
+ * @bcast_and_regmap: regmap associated with LLCC broadcast AND offset
  * @cfg: pointer to the data structure for slice configuration
  * @edac_reg_offset: Offset of the LLCC EDAC registers
  * @lock: mutex associated with each slice
  * @cfg_size: size of the config data table
- * @num_banks: Number of llcc banks
- * @ecc_irq: interrupt for llcc cache error detection and reporting
+ * @num_banks: Number of LLCC banks
+ * @ecc_irq: interrupt for LLCC cache error detection and reporting
  * @ecc_irq_configured: 'True' if firmware has already configured the irq propagation
- * @desc: Array pointer of pre-allocated LLCC slice descriptors
  * @version: Indicates the LLCC version
+ * @desc: Array pointer of pre-allocated LLCC slice descriptors
  */
 struct llcc_drv_data {
+	struct device *dev;
 	struct regmap **regmaps;
 	struct regmap *bcast_regmap;
 	struct regmap *bcast_and_regmap;
@@ -177,38 +181,38 @@ struct llcc_drv_data {
 
 #if IS_ENABLED(CONFIG_QCOM_LLCC)
 /**
- * llcc_slice_getd - get llcc slice descriptor
+ * llcc_slice_getd - get LLCC slice descriptor
  * @uid: usecase_id of the client
  */
 struct llcc_slice_desc *llcc_slice_getd(u32 uid);
 
 /**
- * llcc_slice_putd - llcc slice descritpor
- * @desc: Pointer to llcc slice descriptor
+ * llcc_slice_putd - LLCC slice descriptor
+ * @desc: Pointer to LLCC slice descriptor
  */
 void llcc_slice_putd(struct llcc_slice_desc *desc);
 
 /**
  * llcc_get_slice_id - get slice id
- * @desc: Pointer to llcc slice descriptor
+ * @desc: Pointer to LLCC slice descriptor
  */
 int llcc_get_slice_id(struct llcc_slice_desc *desc);
 
 /**
- * llcc_get_slice_size - llcc slice size
- * @desc: Pointer to llcc slice descriptor
+ * llcc_get_slice_size - LLCC slice size
+ * @desc: Pointer to LLCC slice descriptor
  */
 size_t llcc_get_slice_size(struct llcc_slice_desc *desc);
 
 /**
- * llcc_slice_activate - Activate the llcc slice
- * @desc: Pointer to llcc slice descriptor
+ * llcc_slice_activate - Activate the LLCC slice
+ * @desc: Pointer to LLCC slice descriptor
  */
 int llcc_slice_activate(struct llcc_slice_desc *desc);
 
 /**
- * llcc_slice_deactivate - Deactivate the llcc slice
- * @desc: Pointer to llcc slice descriptor
+ * llcc_slice_deactivate - Deactivate the LLCC slice
+ * @desc: Pointer to LLCC slice descriptor
  */
 int llcc_slice_deactivate(struct llcc_slice_desc *desc);
 
diff --git a/include/linux/soc/ti/knav_dma.h b/include/linux/soc/ti/knav_dma.h
index 18d806a8e52c8..eb1e6b014eaf4 100644
--- a/include/linux/soc/ti/knav_dma.h
+++ b/include/linux/soc/ti/knav_dma.h
@@ -75,7 +75,7 @@ enum knav_dma_desc_type {
  * struct knav_dma_tx_cfg:	Tx channel configuration
  * @filt_einfo:			Filter extended packet info
  * @filt_pswords:		Filter PS words present
- * @knav_dma_tx_priority:	Tx channel scheduling priority
+ * @priority:			Tx channel scheduling priority
  */
 struct knav_dma_tx_cfg {
 	bool				filt_einfo;
@@ -87,13 +87,13 @@ struct knav_dma_tx_cfg {
  * struct knav_dma_rx_cfg:	Rx flow configuration
  * @einfo_present:		Extended packet info present
  * @psinfo_present:		PS words present
- * @knav_dma_rx_err_mode:	Error during buffer starvation
- * @knav_dma_desc_type:	Host or Monolithic desc
+ * @err_mode:			Error during buffer starvation
+ * @desc_type:			Host or Monolithic desc
  * @psinfo_at_sop:		PS word located at start of packet
  * @sop_offset:			Start of packet offset
  * @dst_q:			Destination queue for a given flow
  * @thresh:			Rx flow size threshold
- * @fdq[]:			Free desc Queue array
+ * @fdq:			Free desc Queue array
  * @sz_thresh0:			RX packet size threshold 0
  * @sz_thresh1:			RX packet size threshold 1
  * @sz_thresh2:			RX packet size threshold 2
@@ -115,7 +115,8 @@ struct knav_dma_rx_cfg {
 
 /**
  * struct knav_dma_cfg:	Pktdma channel configuration
- * @sl_cfg:			Slave configuration
+ * @direction:			DMA transfer mode and direction
+ * @u:				union containing @tx or @rx
  * @tx:				Tx channel configuration
  * @rx:				Rx flow configuration
  */
diff --git a/include/linux/socket.h b/include/linux/socket.h
index ec4a0a0257939..1a4d0d128a13e 100644
--- a/include/linux/socket.h
+++ b/include/linux/socket.h
@@ -357,7 +357,7 @@ struct ucred {
 
 /* Flags to be cleared on entry by sendmsg and sendmmsg syscalls */
 #define MSG_INTERNAL_SENDMSG_FLAGS \
-	(MSG_SPLICE_PAGES | MSG_SENDPAGE_NOPOLICY | MSG_SENDPAGE_DECRYPTED)
+	(MSG_SPLICE_PAGES | MSG_SENDPAGE_NOPOLICY | MSG_SENDPAGE_DECRYPTED | MSG_NO_SHARED_FRAGS)
 
 /* Setsockoptions(2) level. Thanks to BSD these must match IPPROTO_xxx */
 #define SOL_IP		0
diff --git a/include/linux/sockptr.h b/include/linux/sockptr.h
index 3e6c8e9d67aef..9c2429c1a570a 100644
--- a/include/linux/sockptr.h
+++ b/include/linux/sockptr.h
@@ -87,24 +87,10 @@ static inline int copy_safe_from_sockptr(void *dst, size_t ksize,
 static inline int copy_struct_from_sockptr(void *dst, size_t ksize,
 		sockptr_t src, size_t usize)
 {
-	size_t size = min(ksize, usize);
-	size_t rest = max(ksize, usize) - size;
-
 	if (!sockptr_is_kernel(src))
-		return copy_struct_from_user(dst, ksize, src.user, size);
-
-	if (usize < ksize) {
-		memset(dst + size, 0, rest);
-	} else if (usize > ksize) {
-		char *p = src.kernel;
+		return copy_struct_from_user(dst, ksize, src.user, usize);
 
-		while (rest--) {
-			if (*p++)
-				return -E2BIG;
-		}
-	}
-	memcpy(dst, src.kernel, size);
-	return 0;
+	return copy_struct_from_bounce_buffer(dst, ksize, src.kernel, usize);
 }
 
 static inline int copy_to_sockptr_offset(sockptr_t dst, size_t offset,
@@ -121,6 +107,16 @@ static inline int copy_to_sockptr(sockptr_t dst, const void *src, size_t size)
 	return copy_to_sockptr_offset(dst, 0, src, size);
 }
 
+static inline int
+copy_struct_to_sockptr(sockptr_t dst, size_t usize, const void *src,
+		       size_t ksize, bool *ignored_trailing)
+{
+	if (!sockptr_is_kernel(dst))
+		return copy_struct_to_user(dst.user, usize, src, ksize, ignored_trailing);
+
+	return copy_struct_to_bounce_buffer(dst.kernel, usize, src, ksize, ignored_trailing);
+}
+
 static inline void *memdup_sockptr_noprof(sockptr_t src, size_t len)
 {
 	void *p = kmalloc_track_caller_noprof(len, GFP_USER | __GFP_NOWARN);
diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h
index 6147eb1fb210d..a46cbaec59491 100644
--- a/include/linux/soundwire/sdw.h
+++ b/include/linux/soundwire/sdw.h
@@ -1093,6 +1093,8 @@ int sdw_slave_get_current_bank(struct sdw_slave *sdev);
 
 int sdw_slave_get_scale_index(struct sdw_slave *slave, u8 *base);
 
+int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms);
+
 /* messaging and data APIs */
 int sdw_read(struct sdw_slave *slave, u32 addr);
 int sdw_write(struct sdw_slave *slave, u32 addr, u8 value);
@@ -1136,6 +1138,12 @@ static inline int sdw_slave_get_current_bank(struct sdw_slave *sdev)
 	return -EINVAL;
 }
 
+static inline int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms)
+{
+	WARN_ONCE(1, "SoundWire API is disabled");
+	return -EINVAL;
+}
+
 /* messaging and data APIs */
 static inline int sdw_read(struct sdw_slave *slave, u32 addr)
 {
diff --git a/include/linux/spi/spi-mem.h b/include/linux/spi/spi-mem.h
index c8e207522223f..722abd9aee3c4 100644
--- a/include/linux/spi/spi-mem.h
+++ b/include/linux/spi/spi-mem.h
@@ -227,6 +227,8 @@ struct spi_mem_op {
  * struct spi_mem_dirmap_info - Direct mapping information
  * @op_tmpl: operation template that should be used by the direct mapping when
  *	     the memory device is accessed
+ * @secondary_op_tmpl: secondary template, may be used as an alternative to the
+ *                     primary template (decided by the upper layer)
  * @offset: absolute offset this direct mapping is pointing to
  * @length: length in byte of this direct mapping
  *
@@ -237,7 +239,9 @@ struct spi_mem_op {
  * direction is directly encoded in the ->op_tmpl.data.dir field.
  */
 struct spi_mem_dirmap_info {
-	struct spi_mem_op op_tmpl;
+	struct spi_mem_op *op_tmpl;
+	struct spi_mem_op primary_op_tmpl;
+	struct spi_mem_op secondary_op_tmpl;
 	u64 offset;
 	u64 length;
 };
@@ -381,12 +385,18 @@ struct spi_controller_mem_ops {
  * @swap16: Supports swapping bytes on a 16 bit boundary when configured in
  *	    Octal DTR
  * @per_op_freq: Supports per operation frequency switching
+ * @no_cs_assertion: The controller may automatically deassert the CS if there
+ *                   is a pause in the transfer (eg. internal bus contention or
+ *                   DMA arbitration on an interconnect). Features such as NAND
+ *                   continuous reads shall not be leveraged.
  */
 struct spi_controller_mem_caps {
 	bool dtr;
 	bool ecc;
 	bool swap16;
 	bool per_op_freq;
+	bool secondary_op_tmpl;
+	bool no_cs_assertion;
 };
 
 #define spi_mem_controller_is_capable(ctlr, cap)	\
diff --git a/include/linux/stddef.h b/include/linux/stddef.h
index 80b6bfb944f0d..e1851c50c89b5 100644
--- a/include/linux/stddef.h
+++ b/include/linux/stddef.h
@@ -100,6 +100,71 @@ enum {
  * Creates a union between a flexible-array member (FAM) in a struct and a set
  * of additional members that would otherwise follow it.
  *
+ * Beware that, as this helper encloses TYPE NAME and MEMBERS in the same
+ * union, designated initializers for MEMBERS may overwrite portions
+ * previously initialized through NAME.
+ *
+ * For example::
+ *
+ *	struct flex {
+ *		size_t count;
+ *		u8 fam[];
+ *	};
+ *
+ *	struct composite {
+ *		...
+ *		__TRAILING_OVERLAP(struct flex, flex, fam, __packed,
+ *			u8 data;
+ *		);
+ *	} __packed;
+ *
+ *	static struct composite comp = {
+ *		.flex = {
+ *			.count = 1,
+ *		},
+ *		.data = 2,
+ *	};
+ *
+ * In the example above, .flex and .data initialize different views of the same
+ * union storage. Since .data is initialized last, it _may_ overwrite portions
+ * previously initialized through .flex, leading to .flex.count being zeroed
+ * out.
+ *
+ * A couple of alternatives are shown below.
+ *
+ * a) Initialize only one view of the overlapped storage and assign the rest
+ *    at runtime::
+ *
+ *	static struct composite comp = {
+ *		.flex = {
+ *			.count = 1,
+ *		},
+ *	};
+ *
+ *	static void foo(void)
+ *	{
+ *		comp.data = 2;
+ *		...
+ *	}
+ *
+ * b) Alternatively, replace designated initializers with runtime assignments::
+ *
+ *	static void foo(void)
+ *	{
+ *		struct composite comp;
+ *
+ *		comp.flex.count = 1;
+ *		comp.data = 2;
+ *		...
+ *	}
+ *
+ * Compiler Explorer test code: https://godbolt.org/z/voM4E36dT
+ *
+ * For another example of the above see commit 5e54510a9389 ("acpi: nfit:
+ * intel: avoid multiple -Wflex-array-member-not-at-end warnings")
+ *
+ * Link: https://git.kernel.org/linus/5e54510a9389caa9
+ *
  * @TYPE: Flexible structure type name, including "struct" keyword.
  * @NAME: Name for a variable to define.
  * @FAM: The flexible-array member within @TYPE
diff --git a/include/linux/sunrpc/cache.h b/include/linux/sunrpc/cache.h
index b1e595c2615bd..2735c332ddb73 100644
--- a/include/linux/sunrpc/cache.h
+++ b/include/linux/sunrpc/cache.h
@@ -80,6 +80,9 @@ struct cache_detail {
 	int			(*cache_upcall)(struct cache_detail *,
 						struct cache_head *);
 
+	int			(*cache_notify)(struct cache_detail *cd,
+						struct cache_head *h);
+
 	void			(*cache_request)(struct cache_detail *cd,
 						 struct cache_head *ch,
 						 char **bpp, int *blen);
@@ -189,9 +192,9 @@ sunrpc_cache_update(struct cache_detail *detail,
 		    struct cache_head *new, struct cache_head *old, int hash);
 
 extern int
-sunrpc_cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h);
+sunrpc_cache_upcall(struct cache_detail *detail, struct cache_head *h);
 extern int
-sunrpc_cache_pipe_upcall_timeout(struct cache_detail *detail,
+sunrpc_cache_upcall_warn(struct cache_detail *detail,
 				 struct cache_head *h);
 
 
@@ -248,6 +251,14 @@ extern int sunrpc_cache_register_pipefs(struct dentry *parent, const char *,
 extern void sunrpc_cache_unregister_pipefs(struct cache_detail *);
 extern void sunrpc_cache_unhash(struct cache_detail *, struct cache_head *);
 
+int sunrpc_cache_requests_count(struct cache_detail *cd);
+int sunrpc_cache_requests_snapshot(struct cache_detail *cd,
+				   struct cache_head **items,
+				   u64 *seqnos, int max,
+				   u64 min_seqno);
+int sunrpc_cache_notify(struct cache_detail *cd, struct cache_head *h,
+			u32 cache_type);
+
 /* Must store cache_detail in seq_file->private if using next three functions */
 extern void *cache_seq_start_rcu(struct seq_file *file, loff_t *pos);
 extern void *cache_seq_next_rcu(struct seq_file *file, void *p, loff_t *pos);
diff --git a/include/linux/switchtec.h b/include/linux/switchtec.h
index cdb58d61c1526..724da6c08bf7d 100644
--- a/include/linux/switchtec.h
+++ b/include/linux/switchtec.h
@@ -42,6 +42,7 @@ enum switchtec_gen {
 	SWITCHTEC_GEN3,
 	SWITCHTEC_GEN4,
 	SWITCHTEC_GEN5,
+	SWITCHTEC_GEN6,
 };
 
 struct mrpc_regs {
diff --git a/include/linux/synclink.h b/include/linux/synclink.h
deleted file mode 100644
index f1405b1c71ba1..0000000000000
--- a/include/linux/synclink.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * SyncLink Multiprotocol Serial Adapter Driver
- *
- * $Id: synclink.h,v 3.14 2006/07/17 20:15:43 paulkf Exp $
- *
- * Copyright (C) 1998-2000 by Microgate Corporation
- *
- * Redistribution of this file is permitted under
- * the terms of the GNU Public License (GPL)
- */
-#ifndef _SYNCLINK_H_
-#define _SYNCLINK_H_
-
-#include <uapi/linux/synclink.h>
-
-/* provide 32 bit ioctl compatibility on 64 bit systems */
-#ifdef CONFIG_COMPAT
-#include <linux/compat.h>
-struct MGSL_PARAMS32 {
-	compat_ulong_t	mode;
-	unsigned char	loopback;
-	unsigned short	flags;
-	unsigned char	encoding;
-	compat_ulong_t	clock_speed;
-	unsigned char	addr_filter;
-	unsigned short	crc_type;
-	unsigned char	preamble_length;
-	unsigned char	preamble;
-	compat_ulong_t	data_rate;
-	unsigned char	data_bits;
-	unsigned char	stop_bits;
-	unsigned char	parity;
-};
-#define MGSL_IOCSPARAMS32 _IOW(MGSL_MAGIC_IOC,0,struct MGSL_PARAMS32)
-#define MGSL_IOCGPARAMS32 _IOR(MGSL_MAGIC_IOC,1,struct MGSL_PARAMS32)
-#endif
-#endif /* _SYNCLINK_H_ */
diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h
index f5639d5ac3316..f3dfc3269188a 100644
--- a/include/linux/syscalls.h
+++ b/include/linux/syscalls.h
@@ -247,6 +247,8 @@ static inline int is_syscall_trace_event(struct trace_event_call *tp_event)
 	__diag_push();							\
 	__diag_ignore(GCC, 8, "-Wattribute-alias",			\
 		      "Type aliasing is used to sanitize syscall arguments");\
+	__diag_ignore(clang, 23, "-Wattribute-alias",			\
+		      "Type aliasing is used to sanitize syscall arguments");\
 	asmlinkage long sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))	\
 		__attribute__((alias(__stringify(__se_sys##name))));	\
 	ALLOW_ERROR_INJECTION(sys##name, ERRNO);			\
@@ -936,7 +938,8 @@ asmlinkage long sys_seccomp(unsigned int op, unsigned int flags,
 asmlinkage long sys_getrandom(char __user *buf, size_t count,
 			      unsigned int flags);
 asmlinkage long sys_memfd_create(const char __user *uname_ptr, unsigned int flags);
-asmlinkage long sys_bpf(int cmd, union bpf_attr __user *attr, unsigned int size);
+asmlinkage long sys_bpf(int cmd, union bpf_attr __user *attr, unsigned int size,
+			struct bpf_common_attr __user *attr_common, unsigned int size_common);
 asmlinkage long sys_execveat(int dfd, const char __user *filename,
 			const char __user *const __user *argv,
 			const char __user *const __user *envp, int flags);
diff --git a/include/linux/tcp.h b/include/linux/tcp.h
index 6982f10e826b4..8a6807082672a 100644
--- a/include/linux/tcp.h
+++ b/include/linux/tcp.h
@@ -253,15 +253,20 @@ struct tcp_sock {
 
 	/* TX read-write hotpath cache lines */
 	__cacheline_group_begin(tcp_sock_write_tx) ____cacheline_aligned;
-	u32	segs_out;	/* RFC4898 tcpEStatsPerfSegsOut
-				 * The total number of segments sent.
-				 */
-	u32	data_segs_out;	/* RFC4898 tcpEStatsPerfDataSegsOut
-				 * total number of data segments sent.
+	u32	delivered;	/* Total data packets delivered incl. rexmits */
+	u32	delivered_ce;	/* Like the above but only ECE marked packets */
+	u64	bytes_acked;	/* RFC4898 tcpEStatsAppHCThruOctetsAcked
+				 * sum(delta(snd_una)), or how many bytes
+				 * were acked.
 				 */
 	u64	bytes_sent;	/* RFC4898 tcpEStatsPerfHCDataOctetsOut
 				 * total number of data bytes sent.
 				 */
+	u64	first_tx_mstamp;  /* start of window send phase */
+	u64	delivered_mstamp; /* time we reached "delivered" */
+	u32	data_segs_out;	/* RFC4898 tcpEStatsPerfDataSegsOut
+				 * total number of data segments sent.
+				 */
 	u32	snd_sml;	/* Last byte of the most recently transmitted small packet */
 	u8	chrono_type;	/* current chronograph type */
 	u32	chrono_start;	/* Start time in jiffies of a TCP chrono */
@@ -271,6 +276,10 @@ struct tcp_sock {
 	u32	lsndtime;	/* timestamp of last sent data packet (for restart window) */
 	u32	mdev_us;	/* medium deviation			*/
 	u32	rtt_seq;	/* sequence number to update rttvar	*/
+	u32	max_packets_out;  /* max packets_out in last window */
+	u32	cwnd_usage_seq;  /* right edge of cwnd usage tracking flight */
+	u32	rate_delivered;    /* saved rate sample: packets delivered */
+	u32	rate_interval_us;  /* saved rate sample: time elapsed */
 	u64	tcp_wstamp_ns;	/* departure time for next sent data packet */
 	u64	accecn_opt_tstamp;	/* Last AccECN option sent timestamp */
 	struct list_head tsorted_sent_queue; /* time-sorted sent but un-SACKed skbs */
@@ -307,8 +316,6 @@ struct tcp_sock {
 	u32	srtt_us;	/* smoothed round trip time << 3 in usecs */
 	u32	packets_out;	/* Packets which are "in flight"	*/
 	u32	snd_up;		/* Urgent pointer		*/
-	u32	delivered;	/* Total data packets delivered incl. rexmits */
-	u32	delivered_ce;	/* Like the above but only ECE marked packets */
 	u32	received_ce;	/* Like the above but for rcvd CE marked pkts */
 	u32	received_ecn_bytes[3]; /* received byte counters for three ECN
 					* types: INET_ECN_ECT_1, INET_ECN_ECT_0,
@@ -324,6 +331,12 @@ struct tcp_sock {
  *      Options received (usually on last packet, some only on SYN packets).
  */
 	struct tcp_options_received rx_opt;
+	u32	segs_in;	/* RFC4898 tcpEStatsPerfSegsIn
+				 * total number of segments in.
+				 */
+	u32	segs_out;	/* RFC4898 tcpEStatsPerfSegsOut
+				 * The total number of segments sent.
+				 */
 	__cacheline_group_end(tcp_sock_write_txrx);
 
 	/* RX read-write hotpath cache lines */
@@ -333,26 +346,13 @@ struct tcp_sock {
 				 * sum(delta(rcv_nxt)), or how many bytes
 				 * were acked.
 				 */
-	u32	segs_in;	/* RFC4898 tcpEStatsPerfSegsIn
-				 * total number of segments in.
-				 */
 	u32	data_segs_in;	/* RFC4898 tcpEStatsPerfDataSegsIn
 				 * total number of data segments in.
 				 */
 	u32	rcv_wup;	/* rcv_nxt on last window update sent	*/
-	u32	max_packets_out;  /* max packets_out in last window */
-	u32	cwnd_usage_seq;  /* right edge of cwnd usage tracking flight */
-	u32	rate_delivered;    /* saved rate sample: packets delivered */
-	u32	rate_interval_us;  /* saved rate sample: time elapsed */
 	u32	rcv_rtt_last_tsecr;
 	u32	delivered_ecn_bytes[3];
 	u16	pkts_acked_ewma;/* Pkts acked EWMA for AccECN cep heuristic */
-	u64	first_tx_mstamp;  /* start of window send phase */
-	u64	delivered_mstamp; /* time we reached "delivered" */
-	u64	bytes_acked;	/* RFC4898 tcpEStatsAppHCThruOctetsAcked
-				 * sum(delta(snd_una)), or how many bytes
-				 * were acked.
-				 */
 	struct {
 		u32	rtt_us;
 		u32	seq;
diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h
index 051e429026904..307b8390fc670 100644
--- a/include/linux/thread_info.h
+++ b/include/linux/thread_info.h
@@ -92,7 +92,7 @@ static inline long set_restart_fn(struct restart_block *restart,
 #define THREAD_ALIGN	THREAD_SIZE
 #endif
 
-#define THREADINFO_GFP		(GFP_KERNEL_ACCOUNT | __GFP_ZERO)
+#define THREADINFO_GFP		(GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_SKIP_KASAN)
 
 /*
  * flag set/clear/test wrappers
diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h
index 0ba112175bb39..0be9b298e692c 100644
--- a/include/linux/thunderbolt.h
+++ b/include/linux/thunderbolt.h
@@ -13,6 +13,7 @@
 
 #include <linux/types.h>
 
+struct config_group;
 struct fwnode_handle;
 struct device;
 
@@ -153,6 +154,9 @@ struct tb_property_dir *tb_property_parse_dir(const u32 *block,
 ssize_t tb_property_format_dir(const struct tb_property_dir *dir, u32 *block,
 			       size_t block_len);
 struct tb_property_dir *tb_property_copy_dir(const struct tb_property_dir *dir);
+int tb_property_merge_dir(struct tb_property_dir *parent,
+			  const struct tb_property_dir *dir,
+			  bool replace);
 struct tb_property_dir *tb_property_create_dir(const uuid_t *uuid);
 void tb_property_free_dir(struct tb_property_dir *dir);
 int tb_property_add_immediate(struct tb_property_dir *parent, const char *key,
@@ -228,6 +232,7 @@ enum tb_link_width {
  *				changed notification
  * @bonding_possible: True if lane bonding is possible on local side
  * @target_link_width: Target link width from the remote host
+ * @ntunnels: Keeps track of how many tunnels go through this XDomain
  * @link: Root switch link the remote domain is connected (ICM only)
  * @depth: Depth in the chain the remote domain is connected (ICM only)
  *
@@ -273,6 +278,7 @@ struct tb_xdomain {
 	int properties_changed_retries;
 	bool bonding_possible;
 	u8 target_link_width;
+	atomic_t ntunnels;
 	u8 link;
 	u8 depth;
 };
@@ -392,6 +398,10 @@ void tb_unregister_protocol_handler(struct tb_protocol_handler *handler);
  * @prtcvers: Protocol version from the properties directory
  * @prtcrevs: Protocol software revision from the properties directory
  * @prtcstns: Protocol settings mask from the properties directory
+ * @lock: Protects this structure
+ * @local_properties: Properties owned by the service driver
+ * @remote_properties: Properties read from the remote service. These
+ *		       are read-only.
  * @debugfs_dir: Pointer to the service debugfs directory. Always created
  *		 when debugfs is enabled. Can be used by service drivers to
  *		 add their own entries under the service.
@@ -399,6 +409,9 @@ void tb_unregister_protocol_handler(struct tb_protocol_handler *handler);
  * Each domain exposes set of services it supports as collection of
  * properties. For each service there will be one corresponding
  * &struct tb_service. Service drivers are bound to these.
+ *
+ * Service drivers can add their own dynamic properties to
+ * @local_properties but whenever they do so @lock must be held.
  */
 struct tb_service {
 	struct device dev;
@@ -408,6 +421,9 @@ struct tb_service {
 	u32 prtcvers;
 	u32 prtcrevs;
 	u32 prtcstns;
+	struct mutex lock;
+	struct tb_property_dir *local_properties;
+	struct tb_property_dir *remote_properties;
 	struct dentry *debugfs_dir;
 };
 
@@ -476,6 +492,8 @@ static inline struct tb_xdomain *tb_service_parent(struct tb_service *svc)
 	return tb_to_xdomain(svc->dev.parent);
 }
 
+void tb_service_properties_changed(struct tb_service *svc);
+
 /**
  * struct tb_nhi - thunderbolt native host interface
  * @lock: Must be held during ring creation/destruction. Is acquired by
@@ -493,6 +511,7 @@ static inline struct tb_xdomain *tb_service_parent(struct tb_service *svc)
  *		    MSI-X is used.
  * @hop_count: Number of rings (end point hops) supported by NHI.
  * @quirks: NHI specific quirks if any
+ * @domain_released: Completed when domain has been fully released
  */
 struct tb_nhi {
 	spinlock_t lock;
@@ -507,6 +526,7 @@ struct tb_nhi {
 	struct work_struct interrupt_work;
 	u32 hop_count;
 	unsigned long quirks;
+	struct completion domain_released;
 };
 
 /**
@@ -535,6 +555,9 @@ struct tb_nhi {
  * @start_poll: Called when ring interrupt is triggered to start
  *		polling. Passing %NULL keeps the ring in interrupt mode.
  * @poll_data: Data passed to @start_poll
+ * @interval_nsec: Interval counter if interrupt throttling is to be
+ *		   used with this ring (in ns)
+ * @wait: Used to signal that the ring may be empty now
  */
 struct tb_ring {
 	spinlock_t lock;
@@ -558,6 +581,8 @@ struct tb_ring {
 	u16 eof_mask;
 	void (*start_poll)(void *data);
 	void *poll_data;
+	unsigned int interval_nsec;
+	wait_queue_head_t wait;
 };
 
 /* Leave ring interrupt enabled on suspend */
@@ -609,7 +634,20 @@ struct ring_frame {
 };
 
 /* Minimum size for ring_rx */
-#define TB_FRAME_SIZE		0x100
+#define TB_FRAME_SIZE		256
+#define TB_MAX_FRAME_SIZE	4096
+
+static inline size_t tb_ring_frame_size(const struct ring_frame *frame)
+{
+	if (frame->size)
+		return frame->size;
+	return TB_MAX_FRAME_SIZE;
+}
+
+static inline size_t tb_ring_size(const struct tb_ring *ring)
+{
+	return ring->size;
+}
 
 struct tb_ring *tb_ring_alloc_tx(struct tb_nhi *nhi, int hop, int size,
 				 unsigned int flags);
@@ -618,6 +656,7 @@ struct tb_ring *tb_ring_alloc_rx(struct tb_nhi *nhi, int hop, int size,
 				 u16 sof_mask, u16 eof_mask,
 				 void (*start_poll)(void *), void *poll_data);
 void tb_ring_start(struct tb_ring *ring);
+bool tb_ring_flush(struct tb_ring *ring, unsigned int timeout_msec);
 void tb_ring_stop(struct tb_ring *ring);
 void tb_ring_free(struct tb_ring *ring);
 
@@ -670,6 +709,8 @@ static inline int tb_ring_tx(struct tb_ring *ring, struct ring_frame *frame)
 struct ring_frame *tb_ring_poll(struct tb_ring *ring);
 void tb_ring_poll_complete(struct tb_ring *ring);
 
+int tb_ring_throttling(struct tb_ring *ring, unsigned int interval_nsec);
+
 /**
  * tb_ring_dma_device() - Return device used for DMA mapping
  * @ring: Ring whose DMA device is retrieved
@@ -687,6 +728,11 @@ static inline struct device *tb_ring_dma_device(struct tb_ring *ring)
 bool usb4_usb3_port_match(struct device *usb4_port_dev,
 			  const struct fwnode_handle *usb3_port_fwnode);
 
+#if IS_REACHABLE(CONFIG_CONFIGFS_FS)
+int tb_configfs_register_group(struct config_group *group);
+void tb_configfs_unregister_group(struct config_group *group);
+#endif
+
 #else /* CONFIG_USB4 */
 static inline bool usb4_usb3_port_match(struct device *usb4_port_dev,
 					const struct fwnode_handle *usb3_port_fwnode)
diff --git a/include/linux/tifm.h b/include/linux/tifm.h
index 44073d06710f6..752fcfae27fe3 100644
--- a/include/linux/tifm.h
+++ b/include/linux/tifm.h
@@ -97,7 +97,7 @@ struct tifm_dev {
 };
 
 struct tifm_driver {
-	struct tifm_device_id *id_table;
+	const struct tifm_device_id *id_table;
 	int                   (*probe)(struct tifm_dev *dev);
 	void                  (*remove)(struct tifm_dev *dev);
 	int                   (*suspend)(struct tifm_dev *dev,
diff --git a/include/linux/timb_gpio.h b/include/linux/timb_gpio.h
deleted file mode 100644
index 74f5e73bf6db6..0000000000000
--- a/include/linux/timb_gpio.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * timb_gpio.h timberdale FPGA GPIO driver, platform data definition
- * Copyright (c) 2009 Intel Corporation
- */
-
-#ifndef _LINUX_TIMB_GPIO_H
-#define _LINUX_TIMB_GPIO_H
-
-/**
- * struct timbgpio_platform_data - Platform data of the Timberdale GPIO driver
- * @gpio_base:		The number of the first GPIO pin, set to -1 for
- *			dynamic number allocation.
- * @nr_pins:		Number of pins that is supported by the hardware (1-32)
- * @irq_base:		If IRQ is supported by the hardware, this is the base
- *			number of IRQ:s. One IRQ per pin will be used. Set to
- *			-1 if IRQ:s is not supported.
- */
-struct timbgpio_platform_data {
-	int gpio_base;
-	int nr_pins;
-	int irq_base;
-};
-
-#endif
diff --git a/include/linux/topology.h b/include/linux/topology.h
index 6575af39fd10f..709a2dcf4c738 100644
--- a/include/linux/topology.h
+++ b/include/linux/topology.h
@@ -230,11 +230,24 @@ static inline int cpu_to_mem(int cpu)
 #define topology_drawer_cpumask(cpu)		cpumask_of(cpu)
 #endif
 
-#if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask)
+/*
+ * Defining cpu_smt_mask as cpumask_of that CPU helps to get
+ * rid of lot of ifdeffery all around the codebase in case of
+ * CONFIG_SCHED_SMT=n. It just means there are no other siblings, which
+ * is what is expected.
+ */
+#if defined(CONFIG_SCHED_SMT)
+# if !defined(cpu_smt_mask)
 static inline const struct cpumask *cpu_smt_mask(int cpu)
 {
 	return topology_sibling_cpumask(cpu);
 }
+# endif
+#else	/* !CONFIG_SCHED_SMT */
+static inline const struct cpumask *cpu_smt_mask(int cpu)
+{
+	return cpumask_of(cpu);
+}
 #endif
 
 #ifndef topology_is_primary_thread
diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h
index 40a43a4c7caf4..d49338c44014e 100644
--- a/include/linux/trace_events.h
+++ b/include/linux/trace_events.h
@@ -770,6 +770,7 @@ trace_trigger_soft_disabled(struct trace_event_file *file)
 
 #ifdef CONFIG_BPF_EVENTS
 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx);
+unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx);
 int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie);
 void perf_event_detach_bpf_prog(struct perf_event *event);
 int perf_event_query_prog_array(struct perf_event *event, void __user *info);
@@ -792,6 +793,11 @@ static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *c
 	return 1;
 }
 
+static inline unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)
+{
+	return 1;
+}
+
 static inline int
 perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie)
 {
diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h
index d2a7882c0b58b..23cad403bb8f5 100644
--- a/include/linux/tty_port.h
+++ b/include/linux/tty_port.h
@@ -6,10 +6,10 @@
 #include <linux/kref.h>
 #include <linux/mutex.h>
 #include <linux/tty_buffer.h>
+#include <linux/tty_driver.h>
 #include <linux/wait.h>
 
 struct attribute_group;
-struct tty_driver;
 struct tty_port;
 struct tty_struct;
 
diff --git a/include/linux/types.h b/include/linux/types.h
index 608050dbca6a7..93166b0b0617a 100644
--- a/include/linux/types.h
+++ b/include/linux/types.h
@@ -224,6 +224,12 @@ struct ustat {
 	char			f_fpack[6];
 };
 
+struct kcov_common_handle_id {
+#ifdef CONFIG_KCOV
+	u64 val;
+#endif
+};
+
 /**
  * struct callback_head - callback structure for use with RCU and task_work
  * @next: next update requests in a list
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index 56328601218c5..9d3fcef61f04f 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -84,7 +84,7 @@
  * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic())
  * that are used instead.  Out of those, __... ones are inlined.  Plain
  * copy_{to,from}_user() might or might not be inlined.  If you want them
- * inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER.
+ * inlined, have asm/uaccess.h define INLINE_COPY_USER.
  *
  * NOTE: only copy_from_user() zero-pads the destination in case of short copy.
  * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything
@@ -157,7 +157,7 @@ __copy_to_user(void __user *to, const void *from, unsigned long n)
 }
 
 /*
- * Architectures that #define INLINE_COPY_TO_USER use this function
+ * Architectures that #define INLINE_COPY_USER use this function
  * directly in the normal copy_to/from_user(), the other ones go
  * through an extern _copy_to/from_user(), which expands the same code
  * here.
@@ -190,10 +190,6 @@ _inline_copy_from_user(void *to, const void __user *from, unsigned long n)
 	memset(to + (n - res), 0, res);
 	return res;
 }
-#ifndef INLINE_COPY_FROM_USER
-extern __must_check unsigned long
-_copy_from_user(void *, const void __user *, unsigned long);
-#endif
 
 static inline __must_check unsigned long
 _inline_copy_to_user(void __user *to, const void *from, unsigned long n)
@@ -207,7 +203,13 @@ _inline_copy_to_user(void __user *to, const void *from, unsigned long n)
 	}
 	return n;
 }
-#ifndef INLINE_COPY_TO_USER
+#ifdef INLINE_COPY_USER
+# define _copy_to_user _inline_copy_to_user
+# define _copy_from_user _inline_copy_from_user
+#else
+extern __must_check unsigned long
+_copy_from_user(void *, const void __user *, unsigned long);
+
 extern __must_check unsigned long
 _copy_to_user(void __user *, const void *, unsigned long);
 #endif
@@ -217,11 +219,7 @@ copy_from_user(void *to, const void __user *from, unsigned long n)
 {
 	if (!check_copy_size(to, n, false))
 		return n;
-#ifdef INLINE_COPY_FROM_USER
-	return _inline_copy_from_user(to, from, n);
-#else
 	return _copy_from_user(to, from, n);
-#endif
 }
 
 static __always_inline unsigned long __must_check
@@ -229,12 +227,7 @@ copy_to_user(void __user *to, const void *from, unsigned long n)
 {
 	if (!check_copy_size(from, n, true))
 		return n;
-
-#ifdef INLINE_COPY_TO_USER
-	return _inline_copy_to_user(to, from, n);
-#else
 	return _copy_to_user(to, from, n);
-#endif
 }
 
 #ifndef copy_mc_to_kernel
@@ -510,7 +503,7 @@ copy_struct_to_user(void __user *dst, size_t usize, const void *src,
 			return -EFAULT;
 	}
 	if (ignored_trailing)
-		*ignored_trailing = ksize < usize &&
+		*ignored_trailing = usize < ksize &&
 			memchr_inv(src + size, 0, rest) != NULL;
 	/* Copy the interoperable parts of the struct. */
 	if (copy_to_user(dst, src, size))
@@ -518,6 +511,69 @@ copy_struct_to_user(void __user *dst, size_t usize, const void *src,
 	return 0;
 }
 
+static __always_inline void
+__copy_struct_generic_bounce_buffer(void *dst, size_t dstsize,
+				    const void *src, size_t srcsize,
+				    bool *ignored_trailing)
+{
+	size_t size = min(dstsize, srcsize);
+	size_t rest = max(dstsize, srcsize) - size;
+
+	/* Deal with trailing bytes. */
+	if (dstsize > srcsize)
+		memset(dst + size, 0, rest);
+	if (ignored_trailing)
+		*ignored_trailing = dstsize < srcsize &&
+			memchr_inv(src + size, 0, rest) != NULL;
+	/* Copy the interoperable parts of the struct. */
+	memcpy(dst, src, size);
+}
+
+/**
+ * This is like copy_struct_from_user(), but the
+ * src buffer was already copied into a kernel
+ * bounce buffer, so it will never return -EFAULT.
+ */
+static __always_inline __must_check int
+copy_struct_from_bounce_buffer(void *dst, size_t dstsize,
+			       const void *src, size_t srcsize)
+{
+	bool ignored_trailing;
+
+	/* Double check if ksize is larger than a known object size. */
+	if (WARN_ON_ONCE(dstsize > __builtin_object_size(dst, 1)))
+		return -E2BIG;
+
+	__copy_struct_generic_bounce_buffer(dst, dstsize,
+					    src, srcsize,
+					    &ignored_trailing);
+	if (unlikely(ignored_trailing))
+		return -E2BIG;
+
+	return 0;
+}
+
+/**
+ * This is like copy_struct_to_user(), but the
+ * dst buffer is a kernel bounce buffer instead
+ * of a direct userspace buffer, so it will never return -EFAULT.
+ */
+static __always_inline __must_check int
+copy_struct_to_bounce_buffer(void *dst, size_t dstsize,
+			     const void *src,
+			     size_t srcsize,
+			     bool *ignored_trailing)
+{
+	/* Double check if srcsize is larger than a known object size. */
+	if (WARN_ON_ONCE(srcsize > __builtin_object_size(src, 1)))
+		return -E2BIG;
+
+	__copy_struct_generic_bounce_buffer(dst, dstsize,
+					    src, srcsize,
+					    ignored_trailing);
+	return 0;
+}
+
 bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size);
 
 long copy_from_kernel_nofault(void *dst, const void *src, size_t size);
diff --git a/include/linux/unicode.h b/include/linux/unicode.h
index 5e6b212a2aeda..64fa44fe180c4 100644
--- a/include/linux/unicode.h
+++ b/include/linux/unicode.h
@@ -66,9 +66,6 @@ int utf8_strncasecmp_folded(const struct unicode_map *um,
 			    const struct qstr *cf,
 			    const struct qstr *s1);
 
-int utf8_normalize(const struct unicode_map *um, const struct qstr *str,
-		   unsigned char *dest, size_t dlen);
-
 int utf8_casefold(const struct unicode_map *um, const struct qstr *str,
 		  unsigned char *dest, size_t dlen);
 
diff --git a/include/linux/usb/uvc.h b/include/linux/usb/uvc.h
index 05bfebab42b67..99b070ab860f1 100644
--- a/include/linux/usb/uvc.h
+++ b/include/linux/usb/uvc.h
@@ -43,6 +43,16 @@
 #define UVC_GUID_MSXU_1_5 \
 	{0xdc, 0x95, 0x3f, 0x0f, 0x32, 0x26, 0x4e, 0x4c, \
 	 0x92, 0xc9, 0xa0, 0x47, 0x82, 0xf4, 0x3b, 0xc8}
+#define UVC_GUID_LOGITECH_MOTOR_CONTROL_V1 \
+	{0x82, 0x06, 0x61, 0x63, 0x70, 0x50, 0xab, 0x49, \
+	 0xb8, 0xcc, 0xb3, 0x85, 0x5e, 0x8d, 0x22, 0x56 }
+#define UVC_GUID_LOGITECH_PERIPHERAL \
+	{0x21, 0x2d, 0xe5, 0xff, 0x30, 0x80, 0x2c, 0x4e, \
+	 0x82, 0xd9, 0xf5, 0x87, 0xd0, 0x05, 0x40, 0xbd }
+#define UVC_GUID_LOGITECH_USER_HW_CONTROL_V1 \
+	{0x82, 0x06, 0x61, 0x63, 0x70, 0x50, 0xab, 0x49, \
+	 0xb8, 0xcc, 0xb3, 0x85, 0x5e, 0x8d, 0x22, 0x1f }
+
 
 /* https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/uvc-extensions-1-5#222-extension-unit-controls */
 #define UVC_MSXU_CONTROL_FOCUS			0x01
diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h
index 89165b769e5c1..7accd0eac457d 100644
--- a/include/linux/vfio_pci_core.h
+++ b/include/linux/vfio_pci_core.h
@@ -188,7 +188,6 @@ int vfio_pci_core_match_token_uuid(struct vfio_device *core_vdev,
 int vfio_pci_core_enable(struct vfio_pci_core_device *vdev);
 void vfio_pci_core_disable(struct vfio_pci_core_device *vdev);
 void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev);
-int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar);
 pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev,
 						pci_channel_state_t state);
 ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
@@ -234,6 +233,25 @@ static inline bool is_aligned_for_order(struct vm_area_struct *vma,
 			   !IS_ALIGNED(pfn, 1 << order)));
 }
 
+/*
+ * Returns a BAR's iomap base or an ERR_PTR() if, for example, the
+ * BAR isn't valid, its resource wasn't acquired, or its iomap
+ * failed.  This shall only be used after vfio_pci_core_enable()
+ * has set up the BAR maps and before vfio_pci_core_disable()
+ * tears them down.
+ */
+static inline void __iomem __must_check *
+vfio_pci_core_get_iomap(struct vfio_pci_core_device *vdev, unsigned int bar)
+{
+	if (WARN_ON_ONCE(bar >= PCI_STD_NUM_BARS))
+		return ERR_PTR(-EINVAL);
+
+	if (WARN_ON_ONCE(!vdev->barmap[bar]))
+		return ERR_PTR(-ENODEV);
+
+	return vdev->barmap[bar];
+}
+
 int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment,
 				 struct phys_vec *phys);
 
diff --git a/include/linux/vmpressure.h b/include/linux/vmpressure.h
index 6a2f51ebbfd35..faecd55224017 100644
--- a/include/linux/vmpressure.h
+++ b/include/linux/vmpressure.h
@@ -30,8 +30,8 @@ struct vmpressure {
 struct mem_cgroup;
 
 #ifdef CONFIG_MEMCG
-extern void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
-		       unsigned long scanned, unsigned long reclaimed);
+void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
+		unsigned long scanned, unsigned long reclaimed);
 extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio);
 
 extern void vmpressure_init(struct vmpressure *vmpr);
@@ -44,8 +44,9 @@ extern int vmpressure_register_event(struct mem_cgroup *memcg,
 extern void vmpressure_unregister_event(struct mem_cgroup *memcg,
 					struct eventfd_ctx *eventfd);
 #else
-static inline void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
-			      unsigned long scanned, unsigned long reclaimed) {}
+static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg,
+			      bool tree, unsigned long scanned,
+			      unsigned long reclaimed) {}
 static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg,
 				   int prio) {}
 #endif /* CONFIG_MEMCG */
diff --git a/include/linux/vt_buffer.h b/include/linux/vt_buffer.h
index b6eeb8cb60705..6c15c6a15f747 100644
--- a/include/linux/vt_buffer.h
+++ b/include/linux/vt_buffer.h
@@ -16,7 +16,7 @@
 
 #include <linux/string.h>
 
-#if IS_ENABLED(CONFIG_VGA_CONSOLE) || IS_ENABLED(CONFIG_MDA_CONSOLE)
+#if IS_ENABLED(CONFIG_VGA_CONSOLE)
 #include <asm/vga.h>
 #endif
 
diff --git a/include/media/i2c/lm3560.h b/include/media/i2c/lm3560.h
deleted file mode 100644
index 770d8c72c94ae..0000000000000
--- a/include/media/i2c/lm3560.h
+++ /dev/null
@@ -1,84 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * include/media/i2c/lm3560.h
- *
- * Copyright (C) 2013 Texas Instruments
- *
- * Contact: Daniel Jeong <gshark.jeong@gmail.com>
- *			Ldd-Mlp <ldd-mlp@list.ti.com>
- */
-
-#ifndef __LM3560_H__
-#define __LM3560_H__
-
-#include <media/v4l2-subdev.h>
-
-#define LM3559_NAME	"lm3559"
-#define LM3560_NAME	"lm3560"
-#define LM3560_I2C_ADDR	(0x53)
-
-/*  FLASH Brightness
- *	min 62500uA, step 62500uA, max 1000000uA
- */
-#define LM3560_FLASH_BRT_MIN 62500
-#define LM3560_FLASH_BRT_STEP 62500
-#define LM3560_FLASH_BRT_MAX 1000000
-#define LM3560_FLASH_BRT_uA_TO_REG(a)	\
-	((a) < LM3560_FLASH_BRT_MIN ? 0 :	\
-	 (((a) - LM3560_FLASH_BRT_MIN) / LM3560_FLASH_BRT_STEP))
-#define LM3560_FLASH_BRT_REG_TO_uA(a)		\
-	((a) * LM3560_FLASH_BRT_STEP + LM3560_FLASH_BRT_MIN)
-
-/*  FLASH TIMEOUT DURATION
- *	min 32ms, step 32ms, max 1024ms
- */
-#define LM3560_FLASH_TOUT_MIN 32
-#define LM3560_FLASH_TOUT_STEP 32
-#define LM3560_FLASH_TOUT_MAX 1024
-#define LM3560_FLASH_TOUT_ms_TO_REG(a)	\
-	((a) < LM3560_FLASH_TOUT_MIN ? 0 :	\
-	 (((a) - LM3560_FLASH_TOUT_MIN) / LM3560_FLASH_TOUT_STEP))
-#define LM3560_FLASH_TOUT_REG_TO_ms(a)		\
-	((a) * LM3560_FLASH_TOUT_STEP + LM3560_FLASH_TOUT_MIN)
-
-/*  TORCH BRT
- *	min 31250uA, step 31250uA, max 250000uA
- */
-#define LM3560_TORCH_BRT_MIN 31250
-#define LM3560_TORCH_BRT_STEP 31250
-#define LM3560_TORCH_BRT_MAX 250000
-#define LM3560_TORCH_BRT_uA_TO_REG(a)	\
-	((a) < LM3560_TORCH_BRT_MIN ? 0 :	\
-	 (((a) - LM3560_TORCH_BRT_MIN) / LM3560_TORCH_BRT_STEP))
-#define LM3560_TORCH_BRT_REG_TO_uA(a)		\
-	((a) * LM3560_TORCH_BRT_STEP + LM3560_TORCH_BRT_MIN)
-
-enum lm3560_led_id {
-	LM3560_LED0 = 0,
-	LM3560_LED1,
-	LM3560_LED_MAX
-};
-
-enum lm3560_peak_current {
-	LM3560_PEAK_1600mA = 0x00,
-	LM3560_PEAK_2300mA = 0x20,
-	LM3560_PEAK_3000mA = 0x40,
-	LM3560_PEAK_3600mA = 0x60
-};
-
-/* struct lm3560_platform_data
- *
- * @peak :  peak current
- * @max_flash_timeout: flash timeout
- * @max_flash_brt: flash mode led brightness
- * @max_torch_brt: torch mode led brightness
- */
-struct lm3560_platform_data {
-	enum lm3560_peak_current peak;
-
-	u32 max_flash_timeout;
-	u32 max_flash_brt[LM3560_LED_MAX];
-	u32 max_torch_brt[LM3560_LED_MAX];
-};
-
-#endif /* __LM3560_H__ */
diff --git a/include/media/media-entity.h b/include/media/media-entity.h
index b91ff6f8c3bb4..d9b72cd87d524 100644
--- a/include/media/media-entity.h
+++ b/include/media/media-entity.h
@@ -726,14 +726,12 @@ int media_entity_pads_init(struct media_entity *entity, u16 num_pads,
  * the entity (currently, it does nothing).
  *
  * Calling media_entity_cleanup() on a media_entity whose memory has been
- * zeroed but that has not been initialized with media_entity_pad_init() is
+ * zeroed but that has not been initialized with media_entity_pads_init() is
  * valid and is a no-op.
  */
-#if IS_ENABLED(CONFIG_MEDIA_CONTROLLER)
-static inline void media_entity_cleanup(struct media_entity *entity) {}
-#else
-#define media_entity_cleanup(entity) do { } while (false)
-#endif
+static inline void media_entity_cleanup(struct media_entity *entity)
+{
+}
 
 /**
  * media_get_pad_index() - retrieves a pad index from an entity
diff --git a/include/media/videobuf2-core.h b/include/media/videobuf2-core.h
index 4424d481d7f74..4b4f4c15c53a7 100644
--- a/include/media/videobuf2-core.h
+++ b/include/media/videobuf2-core.h
@@ -1093,8 +1093,8 @@ __poll_t vb2_core_poll(struct vb2_queue *q, struct file *file,
  * @ppos:	file handle position tracking pointer
  * @nonblock:	mode selector (1 means blocking calls, 0 means nonblocking)
  */
-size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
-		loff_t *ppos, int nonblock);
+ssize_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
+		 loff_t *ppos, int nonblock);
 /**
  * vb2_write() - implements write() syscall logic.
  * @q:		pointer to &struct vb2_queue with videobuf2 queue.
@@ -1103,8 +1103,8 @@ size_t vb2_read(struct vb2_queue *q, char __user *data, size_t count,
  * @ppos:	file handle position tracking pointer
  * @nonblock:	mode selector (1 means blocking calls, 0 means nonblocking)
  */
-size_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
-		loff_t *ppos, int nonblock);
+ssize_t vb2_write(struct vb2_queue *q, const char __user *data, size_t count,
+		  loff_t *ppos, int nonblock);
 
 /**
  * typedef vb2_thread_fnc - callback function for use with vb2_thread.
diff --git a/include/media/vsp1.h b/include/media/vsp1.h
index d9b91ff027618..98089e0a4385c 100644
--- a/include/media/vsp1.h
+++ b/include/media/vsp1.h
@@ -44,8 +44,9 @@ struct vsp1_du_lif_config {
 	void *callback_data;
 };
 
-int vsp1_du_setup_lif(struct device *dev, unsigned int pipe_index,
-		      const struct vsp1_du_lif_config *cfg);
+int vsp1_du_enable(struct device *dev, unsigned int pipe_index,
+		   const struct vsp1_du_lif_config *cfg);
+int vsp1_du_disable(struct device *dev, unsigned int pipe_index);
 
 /**
  * struct vsp1_du_atomic_config - VSP atomic configuration parameters
diff --git a/include/net/Space.h b/include/net/Space.h
deleted file mode 100644
index 6a0b6674d9308..0000000000000
--- a/include/net/Space.h
+++ /dev/null
@@ -1,7 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-/* A unified ethernet device probe.  This is the easiest way to have every
- * ethernet adaptor have the name "eth[0123...]".
- */
-
-struct net_device *ne_probe(int unit);
-struct net_device *cs89x0_probe(int unit);
diff --git a/include/net/act_api.h b/include/net/act_api.h
index d11b791079302..167435c5615e0 100644
--- a/include/net/act_api.h
+++ b/include/net/act_api.h
@@ -241,7 +241,7 @@ static inline void tcf_action_update_bstats(struct tc_action *a,
 static inline void tcf_action_inc_drop_qstats(struct tc_action *a)
 {
 	if (likely(a->cpu_qstats)) {
-		qstats_drop_inc(this_cpu_ptr(a->cpu_qstats));
+		qstats_cpu_drop_inc(a->cpu_qstats);
 		return;
 	}
 	atomic_inc(&a->tcfa_drops);
@@ -250,7 +250,7 @@ static inline void tcf_action_inc_drop_qstats(struct tc_action *a)
 static inline void tcf_action_inc_overlimit_qstats(struct tc_action *a)
 {
 	if (likely(a->cpu_qstats)) {
-		qstats_overlimit_inc(this_cpu_ptr(a->cpu_qstats));
+		qstats_cpu_overlimit_inc(a->cpu_qstats);
 		return;
 	}
 	atomic_inc(&a->tcfa_overlimits);
diff --git a/include/net/amt.h b/include/net/amt.h
index c881bc8b673b3..a0255491f5b05 100644
--- a/include/net/amt.h
+++ b/include/net/amt.h
@@ -331,7 +331,7 @@ struct amt_dev {
 	enum amt_status		status;
 	/* Generated key */
 	siphash_key_t		key;
-	struct socket	  __rcu *sock;
+	struct sock	  __rcu *sk;
 	u32			max_groups;
 	u32			max_sources;
 	u32			hash_buckets;
diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h
index 69eed69f7f265..3faea66b19799 100644
--- a/include/net/bluetooth/bluetooth.h
+++ b/include/net/bluetooth/bluetooth.h
@@ -398,6 +398,7 @@ void baswap(bdaddr_t *dst, const bdaddr_t *src);
 struct bt_sock {
 	struct sock sk;
 	struct list_head accept_q;
+	spinlock_t accept_q_lock; /* protects accept_q */
 	struct sock *parent;
 	unsigned long flags;
 	void (*skb_msg_name)(struct sk_buff *, void *, int *);
diff --git a/include/net/cfg80211.h b/include/net/cfg80211.h
index 9d3639ff9c28e..69dc9a978861a 100644
--- a/include/net/cfg80211.h
+++ b/include/net/cfg80211.h
@@ -830,6 +830,8 @@ struct vif_params {
  * @seq_len: length of @seq.
  * @vlan_id: vlan_id for VLAN group key (if nonzero)
  * @mode: key install mode (RX_TX, NO_TX or SET_TX)
+ * @ltf_keyseed: LTF key seed material
+ * @ltf_keyseed_len: length of LTF key seed material
  */
 struct key_params {
 	const u8 *key;
@@ -839,11 +841,18 @@ struct key_params {
 	u16 vlan_id;
 	u32 cipher;
 	enum nl80211_key_mode mode;
+	const u8 *ltf_keyseed;
+	size_t ltf_keyseed_len;
 };
 
 /**
  * struct cfg80211_chan_def - channel definition
  * @chan: the (control) channel
+ * @npca_chan: the NPCA primary channel
+ *	Note that if DBE is in use, this channel may appear to be
+ *	inside the primary half of the chandef. Implementations
+ *	can use the position of this channel to understand how
+ *	NPCA is used.
  * @width: channel width
  * @center_freq1: center frequency of first segment
  * @center_freq2: center frequency of second segment
@@ -856,18 +865,24 @@ struct key_params {
  * @punctured: mask of the punctured 20 MHz subchannels, with
  *	bits turned on being disabled (punctured); numbered
  *	from lower to higher frequency (like in the spec)
+ * @npca_punctured: NPCA puncturing bitmap, like @punctured but for
+ *	NPCA transmissions. If NPCA is used (@npca_chan is not %NULL)
+ *	this will be a superset of the @punctured bimap.
+ *	Note that if DBE is used, this bitmap is also shifted to be in
+ *	accordance with the overall chandef bandwidth.
  * @s1g_primary_2mhz: Indicates if the control channel pointed to
  *	by 'chan' exists as a 1MHz primary subchannel within an
  *	S1G 2MHz primary channel.
  */
 struct cfg80211_chan_def {
 	struct ieee80211_channel *chan;
+	struct ieee80211_channel *npca_chan;
 	enum nl80211_chan_width width;
 	u32 center_freq1;
 	u32 center_freq2;
 	struct ieee80211_edmg edmg;
 	u16 freq1_offset;
-	u16 punctured;
+	u16 punctured, npca_punctured;
 	bool s1g_primary_2mhz;
 };
 
@@ -1014,7 +1029,9 @@ cfg80211_chandef_identical(const struct cfg80211_chan_def *chandef1,
 		chandef1->freq1_offset == chandef2->freq1_offset &&
 		chandef1->center_freq2 == chandef2->center_freq2 &&
 		chandef1->punctured == chandef2->punctured &&
-		chandef1->s1g_primary_2mhz == chandef2->s1g_primary_2mhz);
+		chandef1->s1g_primary_2mhz == chandef2->s1g_primary_2mhz &&
+		chandef1->npca_chan == chandef2->npca_chan &&
+		chandef1->npca_punctured == chandef2->npca_punctured);
 }
 
 /**
@@ -1143,6 +1160,46 @@ int cfg80211_chandef_primary(const struct cfg80211_chan_def *chandef,
 			     enum nl80211_chan_width primary_chan_width,
 			     u16 *punctured);
 
+/**
+ * cfg80211_chandef_npca_valid - check that NPCA information is valid
+ * @wiphy: the wiphy to check for, for channel pointer lookup
+ * @chandef: the BSS channel chandef to check against
+ * @npca: NPCA information, can be %NULL in which case this
+ *	always returns %true
+ *
+ * Note that DBE must not have been configured into the chandef yet
+ * before checking NPCA, i.e. @chandef must represent the BSS channel.
+ *
+ * Returns: %true if the NPCA channel and puncturing bitmap are valid
+ *	according to the chandef, %false otherwise
+ */
+bool cfg80211_chandef_npca_valid(struct wiphy *wiphy,
+				 const struct cfg80211_chan_def *chandef,
+				 const struct ieee80211_uhr_npca_info *npca);
+
+/**
+ * cfg80211_chandef_add_npca - parse and add NPCA information to chandef
+ * @wiphy: the wiphy this will be used for, for channel pointer lookup
+ * @chandef: the chandef to modify, must be a valid chandef without NPCA
+ * @npca: the NPCA information, can be %NULL
+ *
+ * Returns: 0 if the NPCA information was added and the resulting
+ *	chandef is valid, a negative error code on errors
+ */
+int cfg80211_chandef_add_npca(struct wiphy *wiphy,
+			      struct cfg80211_chan_def *chandef,
+			      const struct ieee80211_uhr_npca_info *npca);
+
+/**
+ * cfg80211_chandef_add_dbe - parse and add DBE information to chandef
+ * @chandef: the chandef to expand
+ * @dbe: the DBE information, must be size-checked if not %NULL
+ *
+ * Returns: 0 for success, a negative error code otherwise
+ */
+int cfg80211_chandef_add_dbe(struct cfg80211_chan_def *chandef,
+			     const struct ieee80211_uhr_dbe_info *dbe);
+
 /**
  * nl80211_send_chandef - sends the channel definition.
  * @msg: the msg to send channel definition
@@ -1396,6 +1453,13 @@ struct cfg80211_rnr_elems {
  * @he_bss_color: BSS Color settings
  * @he_bss_color_valid: indicates whether bss color
  *	attribute is present in beacon data or not.
+ * @ht_required: stations must support HT
+ * @vht_required: stations must support VHT
+ * @ht_oper: HT operation element (or %NULL if HT isn't enabled)
+ * @vht_oper: VHT operation element (or %NULL if VHT isn't enabled)
+ * @he_oper: HE operation IE (or %NULL if HE isn't enabled)
+ * @eht_oper: EHT operation IE (or %NULL if EHT isn't enabled)
+ * @uhr_oper: UHR operation (or %NULL if UHR isn't enabled)
  */
 struct cfg80211_beacon_data {
 	unsigned int link_id;
@@ -1420,6 +1484,13 @@ struct cfg80211_beacon_data {
 	size_t civicloc_len;
 	struct cfg80211_he_bss_color he_bss_color;
 	bool he_bss_color_valid;
+
+	bool ht_required, vht_required;
+	const struct ieee80211_ht_operation *ht_oper;
+	const struct ieee80211_vht_operation *vht_oper;
+	const struct ieee80211_he_operation *he_oper;
+	const struct ieee80211_eht_operation *eht_oper;
+	const struct ieee80211_uhr_operation *uhr_oper;
 };
 
 struct mac_address {
@@ -1524,16 +1595,9 @@ struct cfg80211_s1g_short_beacon {
  * @vht_cap: VHT capabilities (or %NULL if VHT isn't enabled)
  * @he_cap: HE capabilities (or %NULL if HE isn't enabled)
  * @eht_cap: EHT capabilities (or %NULL if EHT isn't enabled)
- * @eht_oper: EHT operation IE (or %NULL if EHT isn't enabled)
- * @uhr_oper: UHR operation (or %NULL if UHR isn't enabled)
- * @ht_required: stations must support HT
- * @vht_required: stations must support VHT
  * @twt_responder: Enable Target Wait Time
- * @he_required: stations must support HE
- * @sae_h2e_required: stations must support direct H2E technique in SAE
  * @flags: flags, as defined in &enum nl80211_ap_settings_flags
  * @he_obss_pd: OBSS Packet Detection settings
- * @he_oper: HE operation IE (or %NULL if HE isn't enabled)
  * @fils_discovery: FILS discovery transmission parameters
  * @unsol_bcast_probe_resp: Unsolicited broadcast probe response parameters
  * @mbssid_config: AP settings for multiple bssid
@@ -1562,11 +1626,7 @@ struct cfg80211_ap_settings {
 	const struct ieee80211_ht_cap *ht_cap;
 	const struct ieee80211_vht_cap *vht_cap;
 	const struct ieee80211_he_cap_elem *he_cap;
-	const struct ieee80211_he_operation *he_oper;
 	const struct ieee80211_eht_cap_elem *eht_cap;
-	const struct ieee80211_eht_operation *eht_oper;
-	const struct ieee80211_uhr_operation *uhr_oper;
-	bool ht_required, vht_required, he_required, sae_h2e_required;
 	bool twt_responder;
 	u32 flags;
 	struct ieee80211_he_obss_pd he_obss_pd;
@@ -4433,6 +4493,25 @@ struct cfg80211_ftm_responder_stats {
  *	(must have either this or @rtt_avg)
  * @dist_variance: variance of distances measured (see also @rtt_variance)
  * @dist_spread: spread of distances measured (see also @rtt_spread)
+ * @tx_ltf_repetition_count: negotiated value of number of tx ltf repetitions
+ *	in NDP frames
+ * @rx_ltf_repetition_count: negotiated value of number of rx ltf repetitions
+ *	in NDP frames
+ * @max_time_between_measurements: the negotiated maximum interval (in units of
+ *	10 ms) by which the ISTA must complete the next measurement cycle.
+ * @min_time_between_measurements: the negotiated minimum interval (in units of
+ *	100 us) between two consecutive range measurements initiated by the
+ *	ISTA.
+ * @num_tx_spatial_streams: number of Tx space-time streams used in the NDP
+ *	frame during the measurement sounding phase.
+ * @num_rx_spatial_streams: number of Rx space-time streams used in the NDP
+ *	frame during the measurement sounding phase.
+ * @nominal_time: negotiated nominal duration between adjacent availability
+ *	windows in units of milliseconds (u32).
+ * @availability_window: negotiated availability window time used in this
+ *	session in units of milliseconds (u8).
+ * @chan_width: band width used for measurement.
+ * @preamble: preamble used for measurement.
  * @num_ftmr_attempts_valid: @num_ftmr_attempts is valid
  * @num_ftmr_successes_valid: @num_ftmr_successes is valid
  * @rssi_avg_valid: @rssi_avg is valid
@@ -4445,6 +4524,18 @@ struct cfg80211_ftm_responder_stats {
  * @dist_avg_valid: @dist_avg is valid
  * @dist_variance_valid: @dist_variance is valid
  * @dist_spread_valid: @dist_spread is valid
+ * @tx_ltf_repetition_count_valid: @tx_ltf_repetition_count is valid
+ * @rx_ltf_repetition_count_valid: @rx_ltf_repetition_count is valid
+ * @max_time_between_measurements_valid: @max_time_between_measurements is valid
+ * @min_time_between_measurements_valid: @min_time_between_measurements is valid
+ * @num_tx_spatial_streams_valid: @num_tx_spatial_streams is valid
+ * @num_rx_spatial_streams_valid: @num_rx_spatial_streams is valid
+ * @nominal_time_valid: @nominal_time is valid
+ * @availability_window_valid: @availability_window is valid
+ * @chan_width_valid: @chan_width is valid.
+ * @preamble_valid: @preamble is valid.
+ * @is_delayed_lmr: indicates if the reported LMR is of the current burst or the
+ *	previous burst, flag.
  */
 struct cfg80211_pmsr_ftm_result {
 	const u8 *lci;
@@ -4468,8 +4559,18 @@ struct cfg80211_pmsr_ftm_result {
 	s64 dist_avg;
 	s64 dist_variance;
 	s64 dist_spread;
+	u32 tx_ltf_repetition_count;
+	u32 rx_ltf_repetition_count;
+	u32 max_time_between_measurements;
+	u32 min_time_between_measurements;
+	u8 num_tx_spatial_streams;
+	u8 num_rx_spatial_streams;
+	u32 nominal_time;
+	u8 availability_window;
+	enum nl80211_chan_width chan_width;
+	enum nl80211_preamble preamble;
 
-	u16 num_ftmr_attempts_valid:1,
+	u32 num_ftmr_attempts_valid:1,
 	    num_ftmr_successes_valid:1,
 	    rssi_avg_valid:1,
 	    rssi_spread_valid:1,
@@ -4480,7 +4581,18 @@ struct cfg80211_pmsr_ftm_result {
 	    rtt_spread_valid:1,
 	    dist_avg_valid:1,
 	    dist_variance_valid:1,
-	    dist_spread_valid:1;
+	    dist_spread_valid:1,
+	    tx_ltf_repetition_count_valid:1,
+	    rx_ltf_repetition_count_valid:1,
+	    max_time_between_measurements_valid:1,
+	    min_time_between_measurements_valid:1,
+	    num_tx_spatial_streams_valid:1,
+	    num_rx_spatial_streams_valid:1,
+	    nominal_time_valid:1,
+	    availability_window_valid:1,
+	    chan_width_valid:1,
+	    preamble_valid:1,
+	    is_delayed_lmr:1;
 };
 
 /**
@@ -4524,7 +4636,8 @@ struct cfg80211_pmsr_result {
  * @burst_duration: burst duration. If @trigger_based or @non_trigger_based is
  *	set, this is the burst duration in milliseconds, and zero means the
  *	device should pick an appropriate value based on @ftms_per_burst.
- * @ftms_per_burst: number of FTMs per burst
+ * @ftms_per_burst: number of FTMs per burst. If set to 0, the firmware or
+ *	driver can automatically select an appropriate value.
  * @ftmr_retries: number of retries for FTM request
  * @request_lci: request LCI information
  * @request_civicloc: request civic location information
@@ -4541,6 +4654,48 @@ struct cfg80211_pmsr_result {
  * @bss_color: the bss color of the responder. Optional. Set to zero to
  *	indicate the driver should set the BSS color. Only valid if
  *	@non_trigger_based or @trigger_based is set.
+ * @request_type: ranging request type, one of
+ *	&enum nl80211_peer_measurement_ftm_req_type. Defaults to
+ *	%NL80211_PMSR_FTM_REQ_TYPE_INFRA if not specified.
+ * @min_time_between_measurements: minimum time between two consecutive range
+ *	measurements in units of 100 microseconds, for non-trigger based
+ *	ranging. Should be set as short as possible to minimize turnaround
+ *	time, since two-way ranging with delayed LMR requires two measurements.
+ *	Only valid if @non_trigger_based is set.
+ * @max_time_between_measurements: maximum time between two consecutive range
+ *	measurements in units of 10 milliseconds, for non-trigger based
+ *	ranging. Acts as a session timeout; if exceeded, the ranging session
+ *	should be terminated. Only valid if @non_trigger_based is set.
+ * @availability_window: duration of the availability window (AW) in units of
+ *	1 millisecond (0-255 ms). Only valid if @non_trigger_based is set.
+ *	If set to 0, the firmware or driver can automatically select an
+ *	appropriate value.
+ * @nominal_time: Nominal duration between adjacent availability windows
+ *	in units of milli seconds. Only valid if @non_trigger_based is set.
+ *	If set to 0, the firmware or driver can automatically select an
+ *	appropriate value.
+ * @num_measurements: number of Availability Windows (AWs) to schedule
+ *	for non-trigger-based ranging. Each AW may contain multiple FTM
+ *	exchanges as configured by @ftms_per_burst. Only valid if
+ *	@non_trigger_based is set. If set to 0, the firmware or driver
+ *	can automatically select an appropriate value.
+ * @ingress_distance: optional ingress threshold in units of mm. When set,
+ *	the measurement result of the peer needs to be indicated if the device
+ *	moves into this range. Measurement results need to be sent on a burst
+ *	index basis in this case.
+ * @egress_distance: optional egress threshold in units of mm. When set,
+ *	the measurement result of the peer needs to be indicated if the device
+ *	moves out of this range. Measurement results need to be sent on a burst
+ *	index basis in this case.
+ *	If neither or only one of @ingress_distance and @egress_distance
+ *	is set, only the specified threshold is used. If both are set, both
+ *	thresholds are applied. If neither is set, results are reported without
+ *	threshold filtering.
+ * @pd_suppress_range_results: flag to suppress ranging results for PD
+ *	requests. When set, the device performs ranging measurements to
+ *	provide ranging services to a peer (e.g. in RSTA role) but does
+ *	not report the measurement results to userspace. Only valid when
+ *	@request_type is %NL80211_PMSR_FTM_REQ_TYPE_PD.
  *
  * See also nl80211 for the respective attribute documentation.
  */
@@ -4560,6 +4715,16 @@ struct cfg80211_pmsr_ftm_request_peer {
 	u8 ftms_per_burst;
 	u8 ftmr_retries;
 	u8 bss_color;
+
+	u32 request_type;
+	u32 min_time_between_measurements;
+	u32 max_time_between_measurements;
+	u8 availability_window;
+	u32 nominal_time;
+	u32 num_measurements;
+	u64 ingress_distance;
+	u64 egress_distance;
+	u8 pd_suppress_range_results:1;
 };
 
 /**
@@ -5085,6 +5250,9 @@ struct mgmt_frame_regs {
  *	links by calling cfg80211_mlo_reconf_add_done(). When calling
  *	cfg80211_mlo_reconf_add_done() the bss pointer must be given for each
  *	link for which MLO reconfiguration 'add' operation was requested.
+ *
+ * @start_pd: Start the PD interface.
+ * @stop_pd: Stop the PD interface.
  */
 struct cfg80211_ops {
 	int	(*suspend)(struct wiphy *wiphy, struct cfg80211_wowlan *wow);
@@ -5267,7 +5435,7 @@ struct cfg80211_ops {
 				     struct wireless_dev *wdev,
 				     struct ieee80211_channel *chan,
 				     unsigned int duration,
-				     u64 *cookie);
+				     u64 *cookie, const u8 *rx_addr);
 	int	(*cancel_remain_on_channel)(struct wiphy *wiphy,
 					    struct wireless_dev *wdev,
 					    u64 cookie);
@@ -5461,6 +5629,8 @@ struct cfg80211_ops {
 				   struct cfg80211_ml_reconf_req *req);
 	int	(*set_epcs)(struct wiphy *wiphy, struct net_device *dev,
 			    bool val);
+	int	(*start_pd)(struct wiphy *wiphy, struct wireless_dev *wdev);
+	void	(*stop_pd)(struct wiphy *wiphy, struct wireless_dev *wdev);
 };
 
 /*
@@ -5852,6 +6022,7 @@ struct wiphy_vendor_command {
  * @extended_capabilities_len: length of the extended capabilities
  * @eml_capabilities: EML capabilities (for MLO)
  * @mld_capa_and_ops: MLD capabilities and operations (for MLO)
+ * @ext_mld_capa_and_ops: Extended MLD capabilities and operations (for MLO)
  */
 struct wiphy_iftype_ext_capab {
 	enum nl80211_iftype iftype;
@@ -5860,6 +6031,7 @@ struct wiphy_iftype_ext_capab {
 	u8 extended_capabilities_len;
 	u16 eml_capabilities;
 	u16 mld_capa_and_ops;
+	u16 ext_mld_capa_and_ops;
 };
 
 /**
@@ -5903,7 +6075,53 @@ cfg80211_get_iftype_ext_capa(struct wiphy *wiphy, enum nl80211_iftype type);
  *	(0 means unknown)
  * @ftm.max_total_ltf_rx: maximum total number of LTFs that can be received
  *	(0 means unknown)
- * @ftm.support_rsta: supports operating as RSTA in PMSR FTM request
+ * @ftm.ista: initiator role capabilities
+ * @ftm.ista.support_ntb: supports operating as ISTA in PMSR FTM request for
+ *	NTB ranging.
+ * @ftm.ista.support_tb: supports operating as ISTA in PMSR FTM request for
+ *	TB ranging.
+ * @ftm.ista.support_edca: supports operating as ISTA in PMSR FTM request for
+ *	EDCA based ranging.
+ * @ftm.ista.max_peers: maximum number of peers supported in the ISTA role.
+ *	If zero, no role-specific peer limit applies.
+ * @ftm.rsta: responder role capabilities
+ * @ftm.rsta.support_ntb: supports operating as RSTA in PMSR FTM request for
+ *	NTB ranging.
+ * @ftm.rsta.support_tb: supports operating as RSTA in PMSR FTM request for
+ *	TB ranging.
+ * @ftm.rsta.support_edca: supports operating as RSTA in PMSR FTM request for
+ *	EDCA based ranging.
+ * @ftm.rsta.max_peers: maximum number of peers supported in the RSTA role.
+ *	If zero, no role-specific peer limit applies.
+ * @ftm.max_no_of_tx_antennas: maximum number of transmit antennas supported for
+ *	EDCA based ranging (0 means unknown)
+ * @ftm.max_no_of_rx_antennas: maximum number of receive antennas supported for
+ *	EDCA based ranging (0 means unknown)
+ * @ftm.min_allowed_ranging_interval_edca: Minimum EDCA ranging
+ *	interval supported by the device in milli seconds. (0 means unknown).
+ *	Applications can use this value to estimate the burst period to be
+ *	given in the FTM request for the EDCA based ranging case. If
+ *	non-zero, this value will be used to validate the burst period in
+ *	the FTM request.
+ * @ftm.min_allowed_ranging_interval_ntb: Minimum NTB ranging
+ *	interval supported by the device in milli seconds. (0 means unknown).
+ *	Applications can use this value to estimate the burst period to be
+ *	given in the FTM request for the NTB ranging case. If non-zero,
+ *	this value will be used to validate the nominal time in the FTM
+ *	request.
+ * @ftm.type: ranging type capabilities
+ * @ftm.type.infra_support: supports infrastructure ranging (STA-to-AP or
+ *	AP-to-STA) as part of Proximity Detection
+ * @ftm.type.pd_support: supports peer-to-peer ranging as mentioned in the
+ *	specification "PR Implementation Consideration Draft 1.9 rev 1" where
+ *	PD stands for proximity detection
+ * @ftm.concurrent_ista_rsta_support: indicates if the device can
+ *	simultaneously act as initiator and responder in a multi-peer
+ *	measurement request. Only valid if @ftm.rsta_support is set.
+ * @ftm.pd_preambles: bitmap of preambles supported (&enum nl80211_preamble)
+ *	for PD ranging requests. Ignored if @ftm.type.pd_support is not set.
+ * @ftm.pd_bandwidths: bitmap of bandwidths supported (&enum nl80211_chan_width)
+ *	for PD ranging requests. Ignored if @ftm.type.pd_support is not set.
  */
 struct cfg80211_pmsr_capabilities {
 	unsigned int max_peers;
@@ -5929,7 +6147,29 @@ struct cfg80211_pmsr_capabilities {
 		u8 max_rx_sts;
 		u8 max_total_ltf_tx;
 		u8 max_total_ltf_rx;
-		u8 support_rsta:1;
+		struct {
+			u8 support_ntb:1,
+			   support_tb:1,
+			   support_edca:1;
+			u32 max_peers;
+		} ista;
+		struct {
+			u8 support_ntb:1,
+			   support_tb:1,
+			   support_edca:1;
+			u32 max_peers;
+		} rsta;
+		u8 max_no_of_tx_antennas;
+		u8 max_no_of_rx_antennas;
+		u32 min_allowed_ranging_interval_edca;
+		u32 min_allowed_ranging_interval_ntb;
+		struct {
+			u8 infra_support:1,
+			   pd_support:1;
+		} type;
+		u8 concurrent_ista_rsta_support:1;
+		u32 pd_preambles;
+		u32 pd_bandwidths;
 	} ftm;
 };
 
@@ -6024,7 +6264,7 @@ enum wiphy_nan_flags {
  *     nibble indicates the number of TX antennas and upper nibble indicates the
  *     number of RX antennas. Value 0 indicates the information is not
  *     available.
- * @max_channel_switch_time: maximum channel switch time in milliseconds.
+ * @max_channel_switch_time: maximum channel switch time in microseconds.
  * @dev_capabilities: NAN device capabilities as defined in Wi-Fi Aware (TM)
  *     specification Table 79 (Capabilities field).
  * @phy: Band-agnostic capabilities for NAN data interfaces. Since NAN
@@ -8302,6 +8542,7 @@ void cfg80211_auth_timeout(struct net_device *dev, const u8 *addr);
  *	as the AC bitmap in the QoS info field
  * @req_ies: information elements from the (Re)Association Request frame
  * @req_ies_len: length of req_ies data
+ * @assoc_encrypted: indicate if the (re)association exchange is encrypted.
  * @ap_mld_addr: AP MLD address (in case of MLO)
  * @links: per-link information indexed by link ID, use links[0] for
  *	non-MLO connections
@@ -8316,6 +8557,7 @@ struct cfg80211_rx_assoc_resp_data {
 	const u8 *req_ies;
 	size_t req_ies_len;
 	int uapsd_queues;
+	bool assoc_encrypted;
 	const u8 *ap_mld_addr;
 	struct {
 		u8 addr[ETH_ALEN] __aligned(2);
@@ -8835,6 +9077,9 @@ struct cfg80211_fils_resp_params {
  * @links.status: per-link status code, to report a status code that's not
  *	%WLAN_STATUS_SUCCESS for a given link, it must also be in the
  *	@valid_links bitmap and may have a BSS pointer (which is then released)
+ * @assoc_encrypted: The driver should set this flag to indicate that the
+ *	(Re)Association Request/Response frames are transmitted encrypted over
+ *	the air.
  */
 struct cfg80211_connect_resp_params {
 	int status;
@@ -8844,6 +9089,7 @@ struct cfg80211_connect_resp_params {
 	size_t resp_ie_len;
 	struct cfg80211_fils_resp_params fils;
 	enum nl80211_timeout_reason timeout_reason;
+	bool assoc_encrypted;
 
 	const u8 *ap_mld_addr;
 	u16 valid_links;
diff --git a/include/net/dsa.h b/include/net/dsa.h
index 8b6d34e8a6f0d..4cc67469cf2ec 100644
--- a/include/net/dsa.h
+++ b/include/net/dsa.h
@@ -1122,7 +1122,8 @@ struct dsa_switch_ops {
 	void	(*port_mirror_del)(struct dsa_switch *ds, int port,
 				   struct dsa_mall_mirror_tc_entry *mirror);
 	int	(*port_policer_add)(struct dsa_switch *ds, int port,
-				    const struct flow_action_police *policer);
+				    const struct flow_action_police *policer,
+				    struct netlink_ext_ack *extack);
 	void	(*port_policer_del)(struct dsa_switch *ds, int port);
 	int	(*port_setup_tc)(struct dsa_switch *ds, int port,
 				 enum tc_setup_type type, void *type_data);
diff --git a/include/net/dst.h b/include/net/dst.h
index 307073eae7f83..10b73fb45117c 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -95,8 +95,19 @@ struct dst_entry {
 #ifdef CONFIG_64BIT
 	struct lwtunnel_state   *lwtstate;
 #endif
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+	atomic_t dst_trace_seq;
+#endif
 };
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+void save_dst_trace_buffer(struct dst_entry *dst, int delta);
+void dump_dst_trace_buffer(const struct net_device *dev);
+#else
+static inline void save_dst_trace_buffer(struct dst_entry *dst, int delta) { };
+static inline void dump_dst_trace_buffer(const struct net_device *dev) { };
+#endif
+
 struct dst_metrics {
 	u32		metrics[RTAX_MAX];
 	refcount_t	refcnt;
@@ -244,7 +255,10 @@ static inline void dst_hold(struct dst_entry *dst)
 	 * the placement of __rcuref in struct dst_entry
 	 */
 	BUILD_BUG_ON(offsetof(struct dst_entry, __rcuref) & 63);
-	WARN_ON(!rcuref_get(&dst->__rcuref));
+	if (!rcuref_get(&dst->__rcuref))
+		WARN_ON(1);
+	else
+		save_dst_trace_buffer(dst, 1);
 }
 
 static inline void dst_use_noref(struct dst_entry *dst, unsigned long time)
@@ -308,7 +322,11 @@ static inline void skb_dst_copy(struct sk_buff *nskb, const struct sk_buff *oskb
  */
 static inline bool dst_hold_safe(struct dst_entry *dst)
 {
-	return rcuref_get(&dst->__rcuref);
+	const bool ret = rcuref_get(&dst->__rcuref);
+
+	if (ret)
+		save_dst_trace_buffer(dst, 1);
+	return ret;
 }
 
 /**
diff --git a/include/net/gen_stats.h b/include/net/gen_stats.h
index 7aa2b8e1fb298..856b5e90fbdb8 100644
--- a/include/net/gen_stats.h
+++ b/include/net/gen_stats.h
@@ -47,19 +47,19 @@ int gnet_stats_start_copy_compat(struct sk_buff *skb, int type,
 				 int padattr);
 
 int gnet_stats_copy_basic(struct gnet_dump *d,
-			  struct gnet_stats_basic_sync __percpu *cpu,
-			  struct gnet_stats_basic_sync *b, bool running);
+			  const struct gnet_stats_basic_sync __percpu *cpu,
+			  const struct gnet_stats_basic_sync *b, bool running);
 void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats,
-			  struct gnet_stats_basic_sync __percpu *cpu,
-			  struct gnet_stats_basic_sync *b, bool running);
+			  const struct gnet_stats_basic_sync __percpu *cpu,
+			  const struct gnet_stats_basic_sync *b, bool running);
 int gnet_stats_copy_basic_hw(struct gnet_dump *d,
-			     struct gnet_stats_basic_sync __percpu *cpu,
-			     struct gnet_stats_basic_sync *b, bool running);
+			     const struct gnet_stats_basic_sync __percpu *cpu,
+			     const struct gnet_stats_basic_sync *b, bool running);
 int gnet_stats_copy_rate_est(struct gnet_dump *d,
 			     struct net_rate_estimator __rcu **ptr);
 int gnet_stats_copy_queue(struct gnet_dump *d,
-			  struct gnet_stats_queue __percpu *cpu_q,
-			  struct gnet_stats_queue *q, __u32 qlen);
+			  const struct gnet_stats_queue __percpu *cpu_q,
+			  const struct gnet_stats_queue *q, __u32 qlen);
 void gnet_stats_add_queue(struct gnet_stats_queue *qstats,
 			  const struct gnet_stats_queue __percpu *cpu_q,
 			  const struct gnet_stats_queue *q);
diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h
index 02762ce73a0ce..a02e569813d2b 100644
--- a/include/net/ip_vs.h
+++ b/include/net/ip_vs.h
@@ -1186,8 +1186,9 @@ struct netns_ipvs {
 	struct timer_list	dest_trash_timer; /* expiration timer */
 	struct mutex		service_mutex;    /* service reconfig */
 	struct rw_semaphore	svc_resize_sem;   /* svc_table resizing */
+	struct rw_semaphore	svc_replace_sem;  /* svc_table replace */
 	struct delayed_work	svc_resize_work;  /* resize svc_table */
-	atomic_t		svc_table_changes;/* ++ on new table */
+	atomic_t		svc_table_changes;/* ++ on table changes */
 	/* Service counters */
 	atomic_t		num_services[IP_VS_AF_MAX];   /* Services */
 	atomic_t		fwm_services[IP_VS_AF_MAX];   /* Services */
diff --git a/include/net/mac80211.h b/include/net/mac80211.h
index 40cb20d9309c0..4fb579805e0fb 100644
--- a/include/net/mac80211.h
+++ b/include/net/mac80211.h
@@ -218,6 +218,8 @@ struct ieee80211_low_level_stats {
  *	bandwidth) OFDMA settings need to be changed
  * @IEEE80211_CHANCTX_CHANGE_PUNCTURING: The punctured channel(s) bitmap
  *	was changed.
+ * @IEEE80211_CHANCTX_CHANGE_NPCA: NPCA configuration changed
+ * @IEEE80211_CHANCTX_CHANGE_NPCA_PUNCT: NPCA puncturing changed
  */
 enum ieee80211_chanctx_change {
 	IEEE80211_CHANCTX_CHANGE_WIDTH		= BIT(0),
@@ -227,6 +229,8 @@ enum ieee80211_chanctx_change {
 	IEEE80211_CHANCTX_CHANGE_MIN_DEF	= BIT(4),
 	IEEE80211_CHANCTX_CHANGE_AP		= BIT(5),
 	IEEE80211_CHANCTX_CHANGE_PUNCTURING	= BIT(6),
+	IEEE80211_CHANCTX_CHANGE_NPCA		= BIT(7),
+	IEEE80211_CHANCTX_CHANGE_NPCA_PUNCT	= BIT(8),
 };
 
 /**
@@ -234,10 +238,13 @@ enum ieee80211_chanctx_change {
  * @oper: channel definition to use for operation
  * @ap: the channel definition of the AP, if any
  *	(otherwise the chan member is %NULL)
+ * @require_npca: If NPCA is configured, require it to
+ *	remain, this is used by AP interfaces
  */
 struct ieee80211_chan_req {
 	struct cfg80211_chan_def oper;
 	struct cfg80211_chan_def ap;
+	bool require_npca;
 };
 
 /**
@@ -366,6 +373,7 @@ struct ieee80211_vif_chanctx_switch {
  * @BSS_CHANGED_MLD_TTLM: negotiated TID to link mapping was changed
  * @BSS_CHANGED_TPE: transmit power envelope changed
  * @BSS_CHANGED_NAN_LOCAL_SCHED: NAN local schedule changed (NAN mode only)
+ * @BSS_CHANGED_NPCA: NPCA parameters changed
  */
 enum ieee80211_bss_change {
 	BSS_CHANGED_ASSOC		= 1<<0,
@@ -404,6 +412,7 @@ enum ieee80211_bss_change {
 	BSS_CHANGED_MLD_TTLM		= BIT_ULL(34),
 	BSS_CHANGED_TPE			= BIT_ULL(35),
 	BSS_CHANGED_NAN_LOCAL_SCHED	= BIT_ULL(36),
+	BSS_CHANGED_NPCA		= BIT_ULL(37),
 
 	/* when adding here, make sure to change ieee80211_reconfig */
 };
@@ -589,6 +598,26 @@ struct ieee80211_parsed_tpe {
 	struct ieee80211_parsed_tpe_psd psd_local[2], psd_reg_client[2];
 };
 
+/**
+ * struct ieee80211_bss_npca_params - NPCA parameters
+ * @min_dur_thresh: NPCA minimum duration threshold (512 + 128*n usec)
+ * @switch_delay: NPCA switch delay (units of 4 usec)
+ * @switch_back_delay: NPCA switch back delay (units of 4 usec)
+ * @init_qsrc: initial QSRC value
+ * @moplen: indicates MOPLEN NPCA is permitted in the BSS
+ * @enabled: NPCA is enabled for this link
+ *
+ * Note: the individual values (except @enabled) are in spec representation.
+ */
+struct ieee80211_bss_npca_params {
+	u32 min_dur_thresh:4,
+	    switch_delay:6,
+	    switch_back_delay:6,
+	    init_qsrc:2,
+	    moplen:1,
+	    enabled:1;
+};
+
 /**
  * struct ieee80211_bss_conf - holds the BSS's changing parameters
  *
@@ -763,6 +792,7 @@ struct ieee80211_parsed_tpe {
  *	(as opposed to hearing its value from another link's beacon).
  * @s1g_long_beacon_period: number of beacon intervals between each long
  *	beacon transmission.
+ * @npca: NPCA parameters
  */
 struct ieee80211_bss_conf {
 	struct ieee80211_vif *vif;
@@ -866,6 +896,8 @@ struct ieee80211_bss_conf {
 	u8 bss_param_ch_cnt_link_id;
 
 	u8 s1g_long_beacon_period;
+
+	struct ieee80211_bss_npca_params npca;
 };
 
 #define IEEE80211_NAN_MAX_CHANNELS 3
@@ -1747,18 +1779,18 @@ struct ieee80211_rx_status {
 			u8 he_ru:3;
 			u8 he_gi:2;
 			u8 he_dcm:1;
-		};
+		} __packed;
 		struct {
 			u8 ru:4;
 			u8 gi:2;
-		} eht;
+		} __packed eht;
 		struct {
 			u8 ru:4;
 			u8 gi:2;
 			u8 elr:1;
 			u8 im:1;
-		} uhr;
-	};
+		} __packed uhr;
+	} __packed;
 	u8 rate_idx;
 	u8 nss;
 	u8 rx_flags;
@@ -1771,6 +1803,8 @@ struct ieee80211_rx_status {
 	u8 link_valid:1, link_id:4;
 };
 
+static_assert(sizeof(struct ieee80211_rx_status) <= sizeof_field(struct sk_buff, cb));
+
 static inline u32
 ieee80211_rx_status_to_khz(struct ieee80211_rx_status *rx_status)
 {
@@ -2672,6 +2706,8 @@ struct ieee80211_link_sta {
  * @epp_peer: indicates that the peer is an EPP peer.
  * @nmi: For NDI stations, pointer to the NMI station of the peer.
  * @nan_sched: NAN peer schedule for this station. Valid only for NMI stations.
+ * @ext_mld_capa_ops: the MLD's extended MLD capabilities and operations
+ *	NOTE: currently only tracked for AP STAs
  */
 struct ieee80211_sta {
 	u8 addr[ETH_ALEN] __aligned(2);
@@ -2696,6 +2732,7 @@ struct ieee80211_sta {
 	struct ieee80211_txq *txq[IEEE80211_NUM_TIDS + 1];
 
 	u16 valid_links;
+	u16 ext_mld_capa_ops;
 	bool epp_peer;
 	struct ieee80211_link_sta deflink;
 	struct ieee80211_link_sta __rcu *link[IEEE80211_MLD_MAX_NUM_LINKS];
@@ -7889,6 +7926,35 @@ void ieee80211_nan_func_match(struct ieee80211_vif *vif,
  */
 void ieee80211_nan_sched_update_done(struct ieee80211_vif *vif);
 
+/**
+ * ieee80211_nan_cluster_joined - notify about NAN cluster join.
+ *
+ * This function is used to notify mac80211 about NAN cluster join.
+ *
+ * @vif: &struct ieee80211_vif pointer from the add_interface callback.
+ * @cluster_id: the cluster ID that was joined
+ * @new_cluster: true if this is a new cluster
+ * @gfp: allocation flags
+ */
+void ieee80211_nan_cluster_joined(struct ieee80211_vif *vif,
+				  const u8 *cluster_id, bool new_cluster,
+				  gfp_t gfp);
+
+/**
+ * ieee80211_nan_try_evacuate - try to evacuate a NAN channel
+ *
+ * This function tries to evacuate a NAN channel that is using the given
+ * channel context, to free up channel context resources.
+ *
+ * @hw: pointer as obtained from ieee80211_alloc_hw()
+ * @conf: the channel context configuration to try to evacuate. If %NULL,
+ *	the NAN channel that has the fewest slots scheduled will be evacuated.
+ *
+ * Return: %true if a channel was evacuated, %false otherwise
+ */
+bool ieee80211_nan_try_evacuate(struct ieee80211_hw *hw,
+				struct ieee80211_chanctx_conf *conf);
+
 /**
  * ieee80211_calc_rx_airtime - calculate estimated transmission airtime for RX.
  *
diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h
index 8f721cd4e4a7c..aa90a858c8e3f 100644
--- a/include/net/mana/mana.h
+++ b/include/net/mana/mana.h
@@ -507,7 +507,7 @@ struct mana_port_context {
 	bool tx_shortform_allowed;
 	u16 tx_vp_offset;
 
-	struct mana_tx_qp *tx_qp;
+	struct mana_tx_qp **tx_qp;
 
 	/* Indirection Table for RX & TX. The values are queue indexes */
 	u32 *indir_table;
diff --git a/include/net/mld.h b/include/net/mld.h
index c073598084934..23255cc47ced9 100644
--- a/include/net/mld.h
+++ b/include/net/mld.h
@@ -89,29 +89,196 @@ struct mld2_query {
 #define MLDV2_QQIC_EXP(value)	(((value) >> 4) & 0x07)
 #define MLDV2_QQIC_MAN(value)	((value) & 0x0f)
 
-#define MLD_EXP_MIN_LIMIT	32768UL
-#define MLDV1_MRD_MAX_COMPAT	(MLD_EXP_MIN_LIMIT - 1)
+/* MLDv2 QQIC floating-point exponential field min threshold */
+#define MLD_QQIC_MIN_THRESHOLD	128
+/* MLDv2 QQIC FP max threshold (mant = 0xF, exp = 7) -> 31744 */
+#define MLD_QQIC_MAX_THRESHOLD	31744
+/* MLDv2 MRC floating-point exponential field min threshold */
+#define MLD_MRC_MIN_THRESHOLD	32768UL
+/* MLDv2 MRC FP max threshold (mant = 0xFFF, exp = 7) -> 8387584 */
+#define MLD_MRC_MAX_THRESHOLD	8387584
+#define MLDV1_MRD_MAX_COMPAT	(MLD_MRC_MIN_THRESHOLD - 1)
 
 #define MLD_MAX_QUEUE		8
 #define MLD_MAX_SKBS		32
 
-static inline unsigned long mldv2_mrc(const struct mld2_query *mlh2)
+/* V2 exponential field encoding */
+
+/*
+ * Calculate Maximum Response Code from Maximum Response Delay
+ *
+ * MRC represents the 16-bit encoded form of Maximum Response Delay (MRD);
+ * once decoded, the resulting value is in milliseconds.
+ *
+ * RFC3810, 5.1.3. defines only the decoding formula:
+ *     Maximum Response Delay = (mant | 0x1000) << (exp + 3)
+ *
+ * but does NOT define the encoding procedure. To derive exponent:
+ *
+ * For the 16-bit MRC, the "hidden bit" (0x1000) is left shifted by 12 to
+ * sit above the 12-bit mantissa. The RFC then shifts this entire block
+ * left by (exp + 3) to reconstruct the value. So, 'hidden bit' is the
+ * MSB which is shifted by (12 + exp + 3).
+ *
+ * Total left shift of the hidden bit = 12 + (exp + 3) = exp + 15.
+ * This is the MSB at the 0-based bit position: (exp + 15).
+ * Since fls() is 1-based, fls(value) - 1 = exp + 15.
+ *
+ * Therefore:
+ *     exp  = fls(value) - 16
+ *     mant = (value >> (exp + 3)) & 0x0FFF
+ *
+ * Final encoding formula:
+ *     0x8000 | (exp << 12) | mant
+ *
+ * Example (value = 1311744):
+ *  0               1               2               3
+ *  0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |0 0 0 0 0 0 0 0 0 0 0 1 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0| 1311744
+ * |                      ^-^--------mant---------^ ^...(exp+3)...^| exp=5
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *
+ * Encoded:
+ *   0x8000 | (5 << 12) | 0x404 = 0xD404
+ */
+static inline u16 mldv2_mrc(unsigned long mrd)
+{
+	u16 mc_man, mc_exp;
+
+	/* MRC < 32768 is literal */
+	if (mrd < MLD_MRC_MIN_THRESHOLD)
+		return mrd;
+
+	/* Saturate at max representable (mant = 0xFFF, exp = 7) -> 8387584 */
+	if (mrd >= MLD_MRC_MAX_THRESHOLD)
+		return 0xFFFF;
+
+	mc_exp = fls(mrd) - 16;
+	mc_man = (mrd >> (mc_exp + 3)) & 0x0FFF;
+
+	return 0x8000 | (mc_exp << 12) | mc_man;
+}
+
+/*
+ * Calculate Querier's Query Interval Code from Querier's Query Interval
+ *
+ * QQIC represents the 8-bit encoded form of Querier's Query Interval (QQI);
+ * once decoded, the resulting value is in seconds.
+ *
+ * RFC3810, 5.1.9. defines only the decoding formula:
+ *     QQI = (mant | 0x10) << (exp + 3)
+ *
+ * but does NOT define the encoding procedure. To derive exponent:
+ *
+ * For any value of mantissa and exponent, the decoding formula indicates
+ * that the "hidden bit" (0x10) is shifted 4 bits left to sit above the
+ * 4-bit mantissa. The RFC again shifts this entire block left by (exp + 3)
+ * to reconstruct the value. So, 'hidden bit' is the MSB which is shifted
+ * by (4 + exp + 3).
+ *
+ * Total left shift of the 'hidden bit' = 4 + (exp + 3) = exp + 7.
+ * This is the MSB at the 0-based bit position: (exp + 7).
+ * Since fls() is 1-based, fls(value) - 1 = exp + 7.
+ *
+ * Therefore:
+ *     exp  = fls(value) - 8
+ *     mant = (value >> (exp + 3)) & 0x0F
+ *
+ * Final encoding formula:
+ *     0x80 | (exp << 4) | mant
+ *
+ * Example (value = 3200):
+ *  0               1
+ *  0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ * |0 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0| (value = 3200)
+ * |        ^-^-mant^ ^..(exp+3)..^| exp = 4, mant = 9
+ * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *
+ * Encoded:
+ *   0x80 | (4 << 4) | 9 = 0xC9
+ */
+static inline u8 mldv2_qqic(unsigned long value)
+{
+	u8 mc_man, mc_exp;
+
+	/* QQIC < 128 is literal */
+	if (value < MLD_QQIC_MIN_THRESHOLD)
+		return value;
+
+	/* Saturate at max representable (mant = 0xF, exp = 7) -> 31744 */
+	if (value >= MLD_QQIC_MAX_THRESHOLD)
+		return 0xFF;
+
+	mc_exp  = fls(value) - 8;
+	mc_man = (value >> (mc_exp + 3)) & 0x0F;
+
+	return 0x80 | (mc_exp << 4) | mc_man;
+}
+
+/* V2 exponential field decoding */
+
+/* Calculate Maximum Response Delay from Maximum Response Code
+ *
+ * RFC3810, relevant sections:
+ *  - 5.1.3. Maximum Response Code defines the decoding formula:
+ *      0 1 2 3 4 5 6 7 8 9 A B C D E F
+ *     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *     |1| exp |          mant         |
+ *     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ *    Maximum Response Delay = (mant | 0x1000) << (exp+3)
+ *  - 9.3. Query Response Interval
+ *
+ * After decode, MRC represents the Maximum Response Delay (MRD) in
+ * units of milliseconds.
+ */
+static inline unsigned long mldv2_mrd(const struct mld2_query *mlh2)
 {
-	/* RFC3810, 5.1.3. Maximum Response Code */
-	unsigned long ret, mc_mrc = ntohs(mlh2->mld2q_mrc);
+	unsigned long mc_mrc = ntohs(mlh2->mld2q_mrc);
 
-	if (mc_mrc < MLD_EXP_MIN_LIMIT) {
-		ret = mc_mrc;
+	if (mc_mrc < MLD_MRC_MIN_THRESHOLD) {
+		return mc_mrc;
 	} else {
 		unsigned long mc_man, mc_exp;
 
 		mc_exp = MLDV2_MRC_EXP(mc_mrc);
 		mc_man = MLDV2_MRC_MAN(mc_mrc);
 
-		ret = (mc_man | 0x1000) << (mc_exp + 3);
+		return (mc_man | 0x1000) << (mc_exp + 3);
 	}
+}
 
-	return ret;
+/* Calculate Querier's Query Interval from Querier's Query Interval Code
+ *
+ * RFC3810, relevant sections:
+ *  - 5.1.9. QQIC (Querier's Query Interval Code) defines the decoding formula:
+ *      0 1 2 3 4 5 6 7
+ *     +-+-+-+-+-+-+-+-+
+ *     |1| exp | mant  |
+ *     +-+-+-+-+-+-+-+-+
+ *    QQI = (mant | 0x10) << (exp + 3)
+ *  - 9.2. Query Interval
+ *  - 9.12. Older Version Querier Present Timeout
+ *    (the [Query Interval] in the last Query received)
+ *
+ * After decode, QQIC represents the Querier's Query Interval in units
+ * of seconds.
+ */
+static inline unsigned long mldv2_qqi(const struct mld2_query *mlh2)
+{
+	unsigned long qqic = mlh2->mld2q_qqic;
+
+	if (qqic < MLD_QQIC_MIN_THRESHOLD) {
+		return qqic;
+	} else {
+		unsigned long mc_man, mc_exp;
+
+		mc_exp = MLDV2_QQIC_EXP(qqic);
+		mc_man = MLDV2_QQIC_MAN(qqic);
+
+		return (mc_man | 0x10) << (mc_exp + 3);
+	}
 }
 
 #endif
diff --git a/include/net/neighbour.h b/include/net/neighbour.h
index 2dfee6d4258af..8860cc2175fc1 100644
--- a/include/net/neighbour.h
+++ b/include/net/neighbour.h
@@ -489,11 +489,15 @@ static inline int neigh_event_send(struct neighbour *neigh, struct sk_buff *skb)
 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
 static inline int neigh_hh_bridge(struct hh_cache *hh, struct sk_buff *skb)
 {
-	unsigned int seq, hh_alen;
+	unsigned int seq, hh_alen = HH_DATA_ALIGN(ETH_HLEN);
+	int err;
+
+	err = skb_cow_head(skb, hh_alen);
+	if (err)
+		return err;
 
 	do {
 		seq = read_seqbegin(&hh->hh_lock);
-		hh_alen = HH_DATA_ALIGN(ETH_HLEN);
 		memcpy(skb->data - hh_alen, hh->hh_data, ETH_ALEN + hh_alen - ETH_HLEN);
 	} while (read_seqretry(&hh->hh_lock, seq));
 	return 0;
diff --git a/include/net/net_shaper.h b/include/net/net_shaper.h
index 5c3f49b52fe96..3939b816b0011 100644
--- a/include/net/net_shaper.h
+++ b/include/net/net_shaper.h
@@ -53,6 +53,7 @@ struct net_shaper {
 
 	/* private: */
 	u32 leaves; /* accounted only for NODE scope */
+	bool valid;
 	struct rcu_head rcu;
 };
 
diff --git a/include/net/netfilter/nf_queue.h b/include/net/netfilter/nf_queue.h
index d17035d14d96c..3978c3174cdbe 100644
--- a/include/net/netfilter/nf_queue.h
+++ b/include/net/netfilter/nf_queue.h
@@ -14,6 +14,7 @@ struct nf_queue_entry {
 	struct list_head	list;
 	struct rhash_head	hash_node;
 	struct sk_buff		*skb;
+	struct net_device	*skb_dev;
 	unsigned int		id;
 	unsigned int		hook_index;	/* index in hook_entries->hook[] */
 #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
diff --git a/include/net/netlink.h b/include/net/netlink.h
index 546d10586576e..b5048dd1d5115 100644
--- a/include/net/netlink.h
+++ b/include/net/netlink.h
@@ -2260,7 +2260,10 @@ static inline struct nlattr *nla_nest_start(struct sk_buff *skb, int attrtype)
  */
 static inline int nla_nest_end(struct sk_buff *skb, struct nlattr *start)
 {
-	start->nla_len = skb_tail_pointer(skb) - (unsigned char *)start;
+	unsigned long diff = skb_tail_pointer(skb) - (unsigned char *)start;
+
+	DEBUG_NET_WARN_ON_ONCE(diff > U16_MAX);
+	start->nla_len = diff;
 	return skb->len;
 }
 
diff --git a/include/net/netmem.h b/include/net/netmem.h
index 78fe51e5756b1..bccacd21b6c37 100644
--- a/include/net/netmem.h
+++ b/include/net/netmem.h
@@ -94,20 +94,10 @@ enum net_iov_type {
  */
 struct net_iov {
 	struct netmem_desc desc;
-	unsigned int page_type;
 	enum net_iov_type type;
 	struct net_iov_area *owner;
 };
 
-/* Make sure 'the offset of page_type in struct page == the offset of
- * type in struct net_iov'.
- */
-#define NET_IOV_ASSERT_OFFSET(pg, iov)			\
-	static_assert(offsetof(struct page, pg) ==	\
-		      offsetof(struct net_iov, iov))
-NET_IOV_ASSERT_OFFSET(page_type, page_type);
-#undef NET_IOV_ASSERT_OFFSET
-
 struct net_iov_area {
 	/* Array of net_iovs for this area. */
 	struct net_iov *niovs;
@@ -127,11 +117,7 @@ static inline unsigned int net_iov_idx(const struct net_iov *niov)
 	return niov - net_iov_owner(niov)->niovs;
 }
 
-/* Initialize a niov: stamp the owning area, the memory provider type,
- * and the page_type "no type" sentinel expected by the page-type API
- * (see PAGE_TYPE_OPS in <linux/page-flags.h>) so that
- * page_pool_set_pp_info() can later call __SetPageNetpp() on a niov
- * cast to struct page.
+/* Initialize a niov: stamp the owning area, the memory provider type.
  */
 static inline void net_iov_init(struct net_iov *niov,
 				struct net_iov_area *owner,
@@ -139,7 +125,6 @@ static inline void net_iov_init(struct net_iov *niov,
 {
 	niov->owner = owner;
 	niov->type = type;
-	niov->page_type = UINT_MAX;
 }
 
 /* netmem */
@@ -245,7 +230,7 @@ static inline unsigned long netmem_pfn_trace(netmem_ref netmem)
  */
 #define pp_page_to_nmdesc(p)						\
 ({									\
-	DEBUG_NET_WARN_ON_ONCE(!PageNetpp(p));				\
+	DEBUG_NET_WARN_ON_ONCE(!page_pool_page_is_pp(p));		\
 	__pp_page_to_nmdesc(p);						\
 })
 
diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index 11159a50d6a14..d0ca932b1871c 100644
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -542,6 +542,21 @@ static inline int qdisc_qlen(const struct Qdisc *q)
 	return q->q.qlen;
 }
 
+static inline int qdisc_qlen_lockless(const struct Qdisc *q)
+{
+	return READ_ONCE(q->q.qlen);
+}
+
+static inline void qdisc_qlen_inc(struct Qdisc *q)
+{
+	WRITE_ONCE(q->q.qlen, q->q.qlen + 1);
+}
+
+static inline void qdisc_qlen_dec(struct Qdisc *q)
+{
+	WRITE_ONCE(q->q.qlen, q->q.qlen - 1);
+}
+
 static inline int qdisc_qlen_sum(const struct Qdisc *q)
 {
 	__u32 qlen = q->qstats.qlen;
@@ -549,9 +564,9 @@ static inline int qdisc_qlen_sum(const struct Qdisc *q)
 
 	if (qdisc_is_percpu_stats(q)) {
 		for_each_possible_cpu(i)
-			qlen += per_cpu_ptr(q->cpu_qstats, i)->qlen;
+			qlen += READ_ONCE(per_cpu_ptr(q->cpu_qstats, i)->qlen);
 	} else {
-		qlen += q->q.qlen;
+		qlen += qdisc_qlen_lockless(q);
 	}
 
 	return qlen;
@@ -937,6 +952,15 @@ static inline void _bstats_update(struct gnet_stats_basic_sync *bstats,
 	u64_stats_update_end(&bstats->syncp);
 }
 
+static inline void _bstats_set(struct gnet_stats_basic_sync *bstats,
+			       u64 bytes, u64 packets)
+{
+	u64_stats_update_begin(&bstats->syncp);
+	u64_stats_set(&bstats->bytes, bytes);
+	u64_stats_set(&bstats->packets, packets);
+	u64_stats_update_end(&bstats->syncp);
+}
+
 static inline void bstats_update(struct gnet_stats_basic_sync *bstats,
 				 const struct sk_buff *skb)
 {
@@ -955,10 +979,15 @@ static inline void qdisc_bstats_update(struct Qdisc *sch,
 	bstats_update(&sch->bstats, skb);
 }
 
+static inline void qstats_backlog_sub(struct Qdisc *sch, u32 val)
+{
+	WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog - val);
+}
+
 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch,
 					    const struct sk_buff *skb)
 {
-	sch->qstats.backlog -= qdisc_pkt_len(skb);
+	qstats_backlog_sub(sch, qdisc_pkt_len(skb));
 }
 
 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,
@@ -967,10 +996,15 @@ static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch,
 	this_cpu_sub(sch->cpu_qstats->backlog, qdisc_pkt_len(skb));
 }
 
+static inline void qstats_backlog_add(struct Qdisc *sch, u32 val)
+{
+	WRITE_ONCE(sch->qstats.backlog, sch->qstats.backlog + val);
+}
+
 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch,
 					    const struct sk_buff *skb)
 {
-	sch->qstats.backlog += qdisc_pkt_len(skb);
+	qstats_backlog_add(sch, qdisc_pkt_len(skb));
 }
 
 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch,
@@ -996,17 +1030,22 @@ static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch)
 
 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count)
 {
-	sch->qstats.drops += count;
+	WRITE_ONCE(sch->qstats.drops, sch->qstats.drops + count);
 }
 
 static inline void qstats_drop_inc(struct gnet_stats_queue *qstats)
 {
-	qstats->drops++;
+	WRITE_ONCE(qstats->drops, qstats->drops + 1);
+}
+
+static inline void qstats_cpu_drop_inc(struct gnet_stats_queue __percpu *qstats)
+{
+	this_cpu_inc(qstats->drops);
 }
 
-static inline void qstats_overlimit_inc(struct gnet_stats_queue *qstats)
+static inline void qstats_cpu_overlimit_inc(struct gnet_stats_queue __percpu *qstats)
 {
-	qstats->overlimits++;
+	this_cpu_inc(qstats->overlimits);
 }
 
 static inline void qdisc_qstats_drop(struct Qdisc *sch)
@@ -1021,23 +1060,23 @@ static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch)
 
 static inline void qdisc_qstats_overlimit(struct Qdisc *sch)
 {
-	sch->qstats.overlimits++;
+	WRITE_ONCE(sch->qstats.overlimits, sch->qstats.overlimits + 1);
 }
 
-static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch)
+static inline int qdisc_qstats_copy(struct gnet_dump *d, const struct Qdisc *sch)
 {
 	__u32 qlen = qdisc_qlen_sum(sch);
 
 	return gnet_stats_copy_queue(d, sch->cpu_qstats, &sch->qstats, qlen);
 }
 
-static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch,  __u32 *qlen,
-					     __u32 *backlog)
+static inline void qdisc_qstats_qlen_backlog(const struct Qdisc *sch,
+					     u32 *qlen, u32 *backlog)
 {
 	struct gnet_stats_queue qstats = { 0 };
 
 	gnet_stats_add_queue(&qstats, sch->cpu_qstats, &sch->qstats);
-	*qlen = qstats.qlen + qdisc_qlen(sch);
+	*qlen = qstats.qlen + qdisc_qlen_lockless(sch);
 	*backlog = qstats.backlog;
 }
 
@@ -1063,7 +1102,7 @@ static inline void __qdisc_enqueue_tail(struct sk_buff *skb,
 		qh->tail = skb;
 		qh->head = skb;
 	}
-	qh->qlen++;
+	WRITE_ONCE(qh->qlen, qh->qlen + 1);
 }
 
 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch)
@@ -1081,7 +1120,7 @@ static inline void __qdisc_enqueue_head(struct sk_buff *skb,
 	if (!qh->head)
 		qh->tail = skb;
 	qh->head = skb;
-	qh->qlen++;
+	WRITE_ONCE(qh->qlen, qh->qlen + 1);
 }
 
 static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh)
@@ -1090,7 +1129,7 @@ static inline struct sk_buff *__qdisc_dequeue_head(struct qdisc_skb_head *qh)
 
 	if (likely(skb != NULL)) {
 		qh->head = skb->next;
-		qh->qlen--;
+		WRITE_ONCE(qh->qlen, qh->qlen - 1);
 		if (qh->head == NULL)
 			qh->tail = NULL;
 		skb->next = NULL;
@@ -1105,7 +1144,7 @@ static inline struct sk_buff *qdisc_dequeue_internal(struct Qdisc *sch, bool dir
 
 	skb = __skb_dequeue(&sch->gso_skb);
 	if (skb) {
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 		qdisc_qstats_backlog_dec(sch, skb);
 		return skb;
 	}
@@ -1261,7 +1300,7 @@ static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch)
 			__skb_queue_head(&sch->gso_skb, skb);
 			/* it's still part of the queue */
 			qdisc_qstats_backlog_inc(sch, skb);
-			sch->q.qlen++;
+			qdisc_qlen_inc(sch);
 		}
 	}
 
@@ -1278,7 +1317,7 @@ static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch,
 	} else {
 		qdisc_qstats_backlog_dec(sch, skb);
 		qdisc_bstats_update(sch, skb);
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 	}
 }
 
@@ -1289,8 +1328,8 @@ static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch,
 		qdisc_qstats_cpu_qlen_inc(sch);
 		this_cpu_add(sch->cpu_qstats->backlog, pkt_len);
 	} else {
-		sch->qstats.backlog += pkt_len;
-		sch->q.qlen++;
+		qstats_backlog_add(sch, pkt_len);
+		qdisc_qlen_inc(sch);
 	}
 }
 
@@ -1306,7 +1345,7 @@ static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch)
 			qdisc_qstats_cpu_qlen_dec(sch);
 		} else {
 			qdisc_qstats_backlog_dec(sch, skb);
-			sch->q.qlen--;
+			qdisc_qlen_dec(sch);
 		}
 	} else {
 		skb = sch->dequeue(sch);
@@ -1327,7 +1366,7 @@ static inline void __qdisc_reset_queue(struct qdisc_skb_head *qh)
 
 		qh->head = NULL;
 		qh->tail = NULL;
-		qh->qlen = 0;
+		WRITE_ONCE(qh->qlen, 0);
 	}
 }
 
diff --git a/include/net/smc.h b/include/net/smc.h
index bfdc4c41f0198..a2bc3ab88075b 100644
--- a/include/net/smc.h
+++ b/include/net/smc.h
@@ -40,7 +40,6 @@ struct smcd_dev {
 	struct dibs_dev *dibs;
 	struct list_head list;
 	spinlock_t lock;
-	struct smc_connection **conn;
 	struct list_head vlan;
 	struct workqueue_struct *event_wq;
 	u8 pnetid[SMC_MAX_PNETID_LEN];
@@ -50,6 +49,7 @@ struct smcd_dev {
 	atomic_t lgr_cnt;
 	wait_queue_head_t lgrs_deleted;
 	u8 going_away : 1;
+	struct smc_connection *conn[];
 };
 
 #define SMC_HS_CTRL_NAME_MAX 16
diff --git a/include/net/sock.h b/include/net/sock.h
index dccd3738c3687..76bfd3e56d639 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -1850,8 +1850,17 @@ static inline struct sock *sk_clone_lock(const struct sock *sk, const gfp_t prio
 
 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
 			     gfp_t priority);
-void __sock_wfree(struct sk_buff *skb);
 void sock_wfree(struct sk_buff *skb);
+void __sock_wfree(struct sk_buff *skb);
+void tcp_wfree(struct sk_buff *skb);
+
+static inline bool is_skb_wmem(const struct sk_buff *skb)
+{
+	return skb->destructor == sock_wfree ||
+	       (IS_ENABLED(CONFIG_INET) && skb->destructor == __sock_wfree) ||
+	       (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree);
+}
+
 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
 			     gfp_t priority);
 void skb_orphan_partial(struct sk_buff *skb);
@@ -2196,8 +2205,12 @@ sk_dst_get(const struct sock *sk)
 
 	rcu_read_lock();
 	dst = rcu_dereference(sk->sk_dst_cache);
-	if (dst && !rcuref_get(&dst->__rcuref))
-		dst = NULL;
+	if (dst) {
+		if (!rcuref_get(&dst->__rcuref))
+			dst = NULL;
+		else
+			save_dst_trace_buffer(dst, 1);
+	}
 	rcu_read_unlock();
 	return dst;
 }
diff --git a/include/net/tcp.h b/include/net/tcp.h
index ecbadcb3a7446..f063eccbbba34 100644
--- a/include/net/tcp.h
+++ b/include/net/tcp.h
@@ -65,8 +65,6 @@ static inline void tcp_orphan_count_dec(void)
 	this_cpu_dec(tcp_orphan_count);
 }
 
-DECLARE_PER_CPU(u32, tcp_tw_isn);
-
 void tcp_time_wait(struct sock *sk, int state, int timeo);
 
 #define MAX_TCP_HEADER	L1_CACHE_ALIGN(128 + MAX_HEADER)
@@ -390,7 +388,6 @@ static inline bool tcp_release_cb_cond(struct sock *sk)
 	return false;
 }
 
-void tcp_wfree(struct sk_buff *skb);
 void tcp_write_timer_handler(struct sock *sk);
 void tcp_delack_timer_handler(struct sock *sk);
 int tcp_ioctl(struct sock *sk, int cmd, int *karg);
@@ -1015,6 +1012,16 @@ static inline u32 tcp_time_stamp_ts(const struct tcp_sock *tp)
 	return tcp_time_stamp_ms(tp);
 }
 
+/* Refresh clocks of a TCP socket,
+ * ensuring monotically increasing values.
+ */
+static inline void tcp_mstamp_refresh_inline(struct tcp_sock *tp)
+{
+	u64 val = tcp_clock_ns();
+
+	tp->tcp_clock_cache = val;
+	tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC);
+}
 void tcp_mstamp_refresh(struct tcp_sock *tp);
 
 static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
@@ -1102,10 +1109,13 @@ struct tcp_skb_cb {
 	__u32		seq;		/* Starting sequence number	*/
 	__u32		end_seq;	/* SEQ + FIN + SYN + datalen	*/
 	union {
-		/* Note :
+		/* Notes :
+		 *	tcp_tw_isn is used in input path only
+		 *	(isn chosen by tcp_timewait_state_process())
 		 * 	  tcp_gso_segs/size are used in write queue only,
 		 *	  cf tcp_skb_pcount()/tcp_skb_mss()
 		 */
+		u32		tcp_tw_isn;
 		struct {
 			u16	tcp_gso_segs;
 			u16	tcp_gso_size;
@@ -2017,40 +2027,6 @@ struct tcp6_pseudohdr {
 	__be32		protocol;	/* including padding */
 };
 
-/*
- * struct tcp_sigpool - per-CPU pool of ahash_requests
- * @scratch: per-CPU temporary area, that can be used between
- *	     tcp_sigpool_start() and tcp_sigpool_end() to perform
- *	     crypto request
- * @req: pre-allocated ahash request
- */
-struct tcp_sigpool {
-	void *scratch;
-	struct ahash_request *req;
-};
-
-int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size);
-void tcp_sigpool_get(unsigned int id);
-void tcp_sigpool_release(unsigned int id);
-int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp,
-			      const struct sk_buff *skb,
-			      unsigned int header_len);
-
-/**
- * tcp_sigpool_start - disable bh and start using tcp_sigpool_ahash
- * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
- * @c: returned tcp_sigpool for usage (uninitialized on failure)
- *
- * Returns: 0 on success, error otherwise.
- */
-int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c);
-/**
- * tcp_sigpool_end - enable bh and stop using tcp_sigpool
- * @c: tcp_sigpool context that was returned by tcp_sigpool_start()
- */
-void tcp_sigpool_end(struct tcp_sigpool *c);
-size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len);
-/* - functions */
 void tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
 			 const struct sock *sk, const struct sk_buff *skb);
 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
@@ -2510,9 +2486,9 @@ struct tcp_sock_af_ops {
 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
 					struct sock *addr_sk,
 					int sndid, int rcvid);
-	int (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key,
-			      const struct sock *sk,
-			      __be32 sisn, __be32 disn, bool send);
+	void (*ao_calc_key_sk)(struct tcp_ao_key *mkt, u8 *key,
+			       const struct sock *sk,
+			       __be32 sisn, __be32 disn, bool send);
 	int (*calc_ao_hash)(char *location, struct tcp_ao_key *ao,
 			    const struct sock *sk, const struct sk_buff *skb,
 			    const u8 *tkey, int hash_offset, u32 sne);
@@ -2533,7 +2509,7 @@ struct tcp_request_sock_ops {
 	struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
 					struct request_sock *req,
 					int sndid, int rcvid);
-	int (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk);
+	void (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk);
 	int (*ao_synack_hash)(char *ao_hash, struct tcp_ao_key *mkt,
 			      struct request_sock *req, const struct sk_buff *skb,
 			      int hash_offset, u32 sne);
diff --git a/include/net/tcp_ao.h b/include/net/tcp_ao.h
index 1e9e27d6e06ba..29fd7b735afaf 100644
--- a/include/net/tcp_ao.h
+++ b/include/net/tcp_ao.h
@@ -2,8 +2,7 @@
 #ifndef _TCP_AO_H
 #define _TCP_AO_H
 
-#define TCP_AO_KEY_ALIGN	1
-#define __tcp_ao_key_align __aligned(TCP_AO_KEY_ALIGN)
+#include <crypto/sha2.h> /* for SHA256_DIGEST_SIZE */
 
 union tcp_ao_addr {
 	struct in_addr  a4;
@@ -32,11 +31,27 @@ struct tcp_ao_counters {
 	atomic64_t	dropped_icmp;
 };
 
+enum tcp_ao_algo_id {
+	TCP_AO_ALGO_HMAC_SHA1 = 1, /* specified by RFC 5926 */
+	TCP_AO_ALGO_HMAC_SHA256, /* Linux extension */
+	TCP_AO_ALGO_AES_128_CMAC, /* specified by RFC 5926 */
+};
+
+/*
+ * This is the maximum untruncated MAC length, in bytes.  Note that the MACs
+ * actually get truncated to 20 or fewer bytes to fit in the TCP options space.
+ */
+#define TCP_AO_MAX_MAC_LEN		SHA256_DIGEST_SIZE
+
+#define TCP_AO_MAX_TRAFFIC_KEY_LEN	SHA256_DIGEST_SIZE
+
+struct tcp_ao_mac_ctx;
+
 struct tcp_ao_key {
 	struct hlist_node	node;
 	union tcp_ao_addr	addr;
-	u8			key[TCP_AO_MAXKEYLEN] __tcp_ao_key_align;
-	unsigned int		tcp_sigpool_id;
+	u8			key[TCP_AO_MAXKEYLEN];
+	enum tcp_ao_algo_id	algo;
 	unsigned int		digest_size;
 	int			l3index;
 	u8			prefixlen;
@@ -168,14 +183,15 @@ struct tcp6_ao_context {
 	__be32		disn;
 };
 
-struct tcp_sigpool;
 /* Established states are fast-path and there always is current_key/rnext_key */
 #define TCP_AO_ESTABLISHED (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | \
 			    TCPF_CLOSE_WAIT | TCPF_LAST_ACK | TCPF_CLOSING)
 
-int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
-			struct tcp_ao_key *key, struct tcphdr *th,
-			__u8 *hash_location);
+void tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
+			 struct tcp_ao_key *key, struct tcphdr *th,
+			 __u8 *hash_location);
+void tcp_ao_mac_update(struct tcp_ao_mac_ctx *mac_ctx, const void *data,
+		       size_t data_len);
 int tcp_ao_hash_skb(unsigned short int family,
 		    char *ao_hash, struct tcp_ao_key *key,
 		    const struct sock *sk, const struct sk_buff *skb,
@@ -188,8 +204,8 @@ struct tcp_ao_key *tcp_ao_established_key(const struct sock *sk,
 int tcp_ao_copy_all_matching(const struct sock *sk, struct sock *newsk,
 			     struct request_sock *req, struct sk_buff *skb,
 			     int family);
-int tcp_ao_calc_traffic_key(struct tcp_ao_key *mkt, u8 *key, void *ctx,
-			    unsigned int len, struct tcp_sigpool *hp);
+void tcp_ao_calc_traffic_key(const struct tcp_ao_key *mkt, u8 *traffic_key,
+			     const void *input, unsigned int input_len);
 void tcp_ao_destroy_sock(struct sock *sk, bool twsk);
 void tcp_ao_time_wait(struct tcp_timewait_sock *tcptw, struct tcp_sock *tp);
 bool tcp_ao_ignore_icmp(const struct sock *sk, int family, int type, int code);
@@ -222,11 +238,11 @@ struct tcp_ao_key *tcp_v4_ao_lookup(const struct sock *sk, struct sock *addr_sk,
 int tcp_v4_ao_synack_hash(char *ao_hash, struct tcp_ao_key *mkt,
 			  struct request_sock *req, const struct sk_buff *skb,
 			  int hash_offset, u32 sne);
-int tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
-			  const struct sock *sk,
-			  __be32 sisn, __be32 disn, bool send);
-int tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
-			   struct request_sock *req);
+void tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
+			   const struct sock *sk,
+			   __be32 sisn, __be32 disn, bool send);
+void tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
+			    struct request_sock *req);
 struct tcp_ao_key *tcp_v4_ao_lookup_rsk(const struct sock *sk,
 					struct request_sock *req,
 					int sndid, int rcvid);
@@ -234,16 +250,17 @@ int tcp_v4_ao_hash_skb(char *ao_hash, struct tcp_ao_key *key,
 		       const struct sock *sk, const struct sk_buff *skb,
 		       const u8 *tkey, int hash_offset, u32 sne);
 /* ipv6 specific functions */
-int tcp_v6_ao_hash_pseudoheader(struct tcp_sigpool *hp,
-				const struct in6_addr *daddr,
-				const struct in6_addr *saddr, int nbytes);
-int tcp_v6_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
-			   const struct sk_buff *skb, __be32 sisn, __be32 disn);
-int tcp_v6_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
-			  const struct sock *sk, __be32 sisn,
-			  __be32 disn, bool send);
-int tcp_v6_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
-			   struct request_sock *req);
+void tcp_v6_ao_hash_pseudoheader(struct tcp_ao_mac_ctx *mac_ctx,
+				 const struct in6_addr *daddr,
+				 const struct in6_addr *saddr, int nbytes);
+void tcp_v6_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
+			    const struct sk_buff *skb, __be32 sisn,
+			    __be32 disn);
+void tcp_v6_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
+			   const struct sock *sk, __be32 sisn,
+			   __be32 disn, bool send);
+void tcp_v6_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
+			    struct request_sock *req);
 struct tcp_ao_key *tcp_v6_ao_lookup(const struct sock *sk,
 				    struct sock *addr_sk, int sndid, int rcvid);
 struct tcp_ao_key *tcp_v6_ao_lookup_rsk(const struct sock *sk,
@@ -263,11 +280,10 @@ void tcp_ao_syncookie(struct sock *sk, const struct sk_buff *skb,
 		      struct request_sock *req, unsigned short int family);
 #else /* CONFIG_TCP_AO */
 
-static inline int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
-				      struct tcp_ao_key *key, struct tcphdr *th,
-				      __u8 *hash_location)
+static inline void tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
+				       struct tcp_ao_key *key,
+				       struct tcphdr *th, __u8 *hash_location)
 {
-	return 0;
 }
 
 static inline void tcp_ao_syncookie(struct sock *sk, const struct sk_buff *skb,
diff --git a/include/net/udp_tunnel.h b/include/net/udp_tunnel.h
index 47c23d4a17408..498b7b262fa94 100644
--- a/include/net/udp_tunnel.h
+++ b/include/net/udp_tunnel.h
@@ -94,7 +94,7 @@ struct udp_tunnel_sock_cfg {
 };
 
 /* Setup the given (UDP) sock to receive UDP encapsulated packets */
-void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
+void setup_udp_tunnel_sock(struct net *net, struct sock *sk,
 			   struct udp_tunnel_sock_cfg *sock_cfg);
 
 /* -- List of parsable UDP tunnel types --
@@ -127,12 +127,12 @@ struct udp_tunnel_info {
 };
 
 /* Notify network devices of offloadable types */
-void udp_tunnel_push_rx_port(struct net_device *dev, struct socket *sock,
+void udp_tunnel_push_rx_port(struct net_device *dev, struct sock *sk,
 			     unsigned short type);
-void udp_tunnel_drop_rx_port(struct net_device *dev, struct socket *sock,
+void udp_tunnel_drop_rx_port(struct net_device *dev, struct sock *sk,
 			     unsigned short type);
-void udp_tunnel_notify_add_rx_port(struct socket *sock, unsigned short type);
-void udp_tunnel_notify_del_rx_port(struct socket *sock, unsigned short type);
+void udp_tunnel_notify_add_rx_port(struct sock *sk, unsigned short type);
+void udp_tunnel_notify_del_rx_port(struct sock *sk, unsigned short type);
 
 /* Transmit the skb using UDP encapsulation. */
 void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb,
@@ -176,7 +176,7 @@ static inline void udp_tunnel_set_inner_protocol(struct sk_buff *skb,
 		skb_set_inner_protocol(skb, inner_proto);
 }
 
-void udp_tunnel_sock_release(struct socket *sock);
+void udp_tunnel_sock_release(struct sock *sk);
 
 struct rtable *udp_tunnel_dst_lookup(struct sk_buff *skb,
 				     struct net_device *dev,
@@ -188,7 +188,7 @@ struct rtable *udp_tunnel_dst_lookup(struct sk_buff *skb,
 struct dst_entry *udp_tunnel6_dst_lookup(struct sk_buff *skb,
 					 struct net_device *dev,
 					 struct net *net,
-					 struct socket *sock, int oif,
+					 struct sock *sk, int oif,
 					 struct in6_addr *saddr,
 					 const struct ip_tunnel_key *key,
 					 __be16 sport, __be16 dport, u8 dsfield,
diff --git a/include/net/vxlan.h b/include/net/vxlan.h
index 0ee50785f4f1c..dfba89695efcf 100644
--- a/include/net/vxlan.h
+++ b/include/net/vxlan.h
@@ -185,7 +185,8 @@ struct vxlan_metadata {
 /* per UDP socket information */
 struct vxlan_sock {
 	struct hlist_node hlist;
-	struct socket	 *sock;
+	struct sock	  *sk;
+	struct rcu_head	  rcu;
 	struct hlist_head vni_list[VNI_HASH_SIZE];
 	refcount_t	  refcnt;
 	u32		  flags;
@@ -448,7 +449,7 @@ static inline __be32 vxlan_compute_rco(unsigned int start, unsigned int offset)
 
 static inline unsigned short vxlan_get_sk_family(struct vxlan_sock *vs)
 {
-	return vs->sock->sk->sk_family;
+	return vs->sk->sk_family;
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 10d3edde6b2fa..874409127e292 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -715,6 +715,7 @@ struct xfrm_mgr {
 					   const struct xfrm_migrate *m,
 					   int num_bundles,
 					   const struct xfrm_kmaddress *k,
+					   struct net *net,
 					   const struct xfrm_encap_tmpl *encap);
 	bool			(*is_alive)(const struct km_event *c);
 };
@@ -1891,7 +1892,7 @@ int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol);
 #ifdef CONFIG_XFRM_MIGRATE
 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 	       const struct xfrm_migrate *m, int num_bundles,
-	       const struct xfrm_kmaddress *k,
+	       const struct xfrm_kmaddress *k, struct net *net,
 	       const struct xfrm_encap_tmpl *encap);
 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
 						u32 if_id);
diff --git a/include/rdma/uverbs_ioctl.h b/include/rdma/uverbs_ioctl.h
index e2af17da3e32c..c89428030d61a 100644
--- a/include/rdma/uverbs_ioctl.h
+++ b/include/rdma/uverbs_ioctl.h
@@ -635,6 +635,7 @@ struct uverbs_attr_bundle {
 		struct ib_uverbs_file *ufile;
 		struct ib_ucontext *context;
 		struct ib_uobject *uobject;
+		const struct uverbs_api_ioctl_method *method_elm;
 		DECLARE_BITMAP(attr_present, UVERBS_API_ATTR_BKEY_LEN);
 	);
 	struct uverbs_attr attrs[];
diff --git a/include/soc/bcm2835/raspberrypi-firmware.h b/include/soc/bcm2835/raspberrypi-firmware.h
index e1f87fbfe5542..975bef529854a 100644
--- a/include/soc/bcm2835/raspberrypi-firmware.h
+++ b/include/soc/bcm2835/raspberrypi-firmware.h
@@ -156,6 +156,31 @@ enum rpi_firmware_clk_id {
 	RPI_FIRMWARE_NUM_CLK_ID,
 };
 
+enum rpi_firmware_volt_id {
+	RPI_FIRMWARE_VOLT_ID_CORE = 1,
+	RPI_FIRMWARE_VOLT_ID_SDRAM_C = 2,
+	RPI_FIRMWARE_VOLT_ID_SDRAM_P = 3,
+	RPI_FIRMWARE_VOLT_ID_SDRAM_I = 4,
+	RPI_FIRMWARE_NUM_VOLT_ID,
+};
+
+/**
+ * struct rpi_firmware_get_voltage_request - Firmware request for a voltage
+ * @id:		ID of the voltage being queried
+ * @value:	Voltage in microvolts. Set by the firmware.
+ *
+ * Used by @RPI_FIRMWARE_GET_VOLTAGE.
+ */
+struct rpi_firmware_get_voltage_request {
+	__le32 id;
+	__le32 value;
+} __packed;
+
+#define RPI_FIRMWARE_GET_VOLTAGE_REQUEST(_id)	\
+	{					\
+		.id = cpu_to_le32(_id),		\
+	}
+
 /**
  * struct rpi_firmware_clk_rate_request - Firmware Request for a rate
  * @id:	ID of the clock being queried
diff --git a/include/soc/qcom/qcom-spmi-pmic.h b/include/soc/qcom/qcom-spmi-pmic.h
index 2cf9e2d8cd55f..997fa18d70fe8 100644
--- a/include/soc/qcom/qcom-spmi-pmic.h
+++ b/include/soc/qcom/qcom-spmi-pmic.h
@@ -50,9 +50,22 @@
 #define PMR735B_SUBTYPE		0x34
 #define PM6350_SUBTYPE		0x36
 #define PM4125_SUBTYPE		0x37
+#define PM8010_SUBTYPE		0x41
+#define PM8550VS_SUBTYPE	0x45
+#define PM8550VE_SUBTYPE	0x46
+#define PMR735D_SUBTYPE	0x48
+#define PM8550_SUBTYPE		0x49
+#define PMK8550_SUBTYPE	0x4a
 #define PMM8650AU_SUBTYPE       0x4e
 #define PMM8650AU_PSAIL_SUBTYPE 0x4f
-
+#define PM8750B_SUBTYPE	0x56
+#define PMD8028_SUBTYPE	0x57
+#define PMK8850_SUBTYPE	0x5c
+#define PMH0101_SUBTYPE	0x5d
+#define SMB2370_SUBTYPE	0x5f
+#define PMH0104_SUBTYPE	0x60
+#define PMH0110_SUBTYPE	0x61
+#define PMCX0102_SUBTYPE	0x62
 #define PMI8998_FAB_ID_SMIC	0x11
 #define PMI8998_FAB_ID_GF	0x30
 
diff --git a/include/soc/rockchip/rk3588_grf.h b/include/soc/rockchip/rk3588_grf.h
index 02a7b2432d994..db0092fc66ad9 100644
--- a/include/soc/rockchip/rk3588_grf.h
+++ b/include/soc/rockchip/rk3588_grf.h
@@ -19,4 +19,6 @@
 /* Whether the LPDDR5 is in 2:1 (= 0) or 4:1 (= 1) CKR a.k.a. DQS mode */
 #define RK3588_PMUGRF_OS_REG6_LP5_CKR			BIT(0)
 
+#define RK3588_SYSGRF_SOC_CON6			0x0318
+
 #endif /* __SOC_RK3588_GRF_H */
diff --git a/include/soc/tegra/mc.h b/include/soc/tegra/mc.h
index e6da035d1306d..25d465d704938 100644
--- a/include/soc/tegra/mc.h
+++ b/include/soc/tegra/mc.h
@@ -164,7 +164,7 @@ struct tegra_mc_ops {
 	 */
 	int (*probe)(struct tegra_mc *mc);
 	void (*remove)(struct tegra_mc *mc);
-	int (*resume)(struct tegra_mc *mc);
+	void (*resume)(struct tegra_mc *mc);
 	int (*probe_device)(struct tegra_mc *mc, struct device *dev);
 };
 
diff --git a/include/sound/ac97/codec.h b/include/sound/ac97/codec.h
index 882b849b9255e..69b404c354f50 100644
--- a/include/sound/ac97/codec.h
+++ b/include/sound/ac97/codec.h
@@ -108,6 +108,9 @@ static inline void ac97_set_drvdata(struct ac97_codec_device *adev,
 	dev_set_drvdata(ac97_codec_dev2dev(adev), data);
 }
 
-void *snd_ac97_codec_get_platdata(const struct ac97_codec_device *adev);
+static inline void *snd_ac97_codec_get_platdata(const struct ac97_codec_device *adev)
+{
+	return NULL;
+}
 
 #endif
diff --git a/include/sound/ac97/controller.h b/include/sound/ac97/controller.h
index 06b5afb7fa6b9..d5895ea3922bd 100644
--- a/include/sound/ac97/controller.h
+++ b/include/sound/ac97/controller.h
@@ -62,14 +62,13 @@ struct ac97_controller_ops {
 #if IS_ENABLED(CONFIG_AC97_BUS_NEW)
 struct ac97_controller *snd_ac97_controller_register(
 	const struct ac97_controller_ops *ops, struct device *dev,
-	unsigned short slots_available, void **codecs_pdata);
+	unsigned short slots_available);
 void snd_ac97_controller_unregister(struct ac97_controller *ac97_ctrl);
 #else
 static inline struct ac97_controller *
 snd_ac97_controller_register(const struct ac97_controller_ops *ops,
 			     struct device *dev,
-			     unsigned short slots_available,
-			     void **codecs_pdata)
+			     unsigned short slots_available)
 {
 	return ERR_PTR(-ENODEV);
 }
diff --git a/include/sound/emux_synth.h b/include/sound/emux_synth.h
index 3f7f365ed248c..2c0a976e00abb 100644
--- a/include/sound/emux_synth.h
+++ b/include/sound/emux_synth.h
@@ -125,7 +125,6 @@ struct snd_emux {
  */
 struct snd_emux_port {
 
-	struct snd_midi_channel_set chset;
 	struct snd_emux *emu;
 
 	char port_mode;			/* operation mode */
@@ -138,6 +137,7 @@ struct snd_emux_port {
 #if IS_ENABLED(CONFIG_SND_SEQUENCER_OSS)
 	struct snd_seq_oss_arg *oss_arg;
 #endif
+	struct snd_midi_channel_set chset;
 };
 
 /* port_mode */
diff --git a/include/sound/pxa2xx-lib.h b/include/sound/pxa2xx-lib.h
index 0a6f8dabf8c4c..2d86f62f94082 100644
--- a/include/sound/pxa2xx-lib.h
+++ b/include/sound/pxa2xx-lib.h
@@ -2,55 +2,7 @@
 #ifndef PXA2XX_LIB_H
 #define PXA2XX_LIB_H
 
-#include <uapi/sound/asound.h>
-#include <linux/platform_device.h>
-
-/* PCM */
-struct snd_pcm_substream;
-struct snd_pcm_hw_params;
-struct snd_soc_pcm_runtime;
-struct snd_pcm;
-struct snd_soc_component;
-
-extern int pxa2xx_pcm_hw_params(struct snd_pcm_substream *substream,
-				struct snd_pcm_hw_params *params);
-extern int pxa2xx_pcm_trigger(struct snd_pcm_substream *substream, int cmd);
-extern snd_pcm_uframes_t pxa2xx_pcm_pointer(struct snd_pcm_substream *substream);
-extern int pxa2xx_pcm_prepare(struct snd_pcm_substream *substream);
-extern int pxa2xx_pcm_open(struct snd_pcm_substream *substream);
-extern int pxa2xx_pcm_close(struct snd_pcm_substream *substream);
-extern int pxa2xx_pcm_preallocate_dma_buffer(struct snd_pcm *pcm);
-extern int pxa2xx_soc_pcm_new(struct snd_soc_component *component,
-			      struct snd_soc_pcm_runtime *rtd);
-extern int pxa2xx_soc_pcm_open(struct snd_soc_component *component,
-			       struct snd_pcm_substream *substream);
-extern int pxa2xx_soc_pcm_close(struct snd_soc_component *component,
-				struct snd_pcm_substream *substream);
-extern int pxa2xx_soc_pcm_hw_params(struct snd_soc_component *component,
-				    struct snd_pcm_substream *substream,
-				    struct snd_pcm_hw_params *params);
-extern int pxa2xx_soc_pcm_prepare(struct snd_soc_component *component,
-				  struct snd_pcm_substream *substream);
-extern int pxa2xx_soc_pcm_trigger(struct snd_soc_component *component,
-				  struct snd_pcm_substream *substream, int cmd);
-extern snd_pcm_uframes_t
-pxa2xx_soc_pcm_pointer(struct snd_soc_component *component,
-		       struct snd_pcm_substream *substream);
-
-/* AC97 */
-
-extern int pxa2xx_ac97_read(int slot, unsigned short reg);
-extern int pxa2xx_ac97_write(int slot, unsigned short reg, unsigned short val);
-
-extern bool pxa2xx_ac97_try_warm_reset(void);
-extern bool pxa2xx_ac97_try_cold_reset(void);
-extern void pxa2xx_ac97_finish_reset(void);
-
-extern int pxa2xx_ac97_hw_suspend(void);
-extern int pxa2xx_ac97_hw_resume(void);
-
-extern int pxa2xx_ac97_hw_probe(struct platform_device *dev);
-extern void pxa2xx_ac97_hw_remove(struct platform_device *dev);
+#include <linux/types.h>
 
 /* modem registers, used by touchscreen driver */
 u32 pxa2xx_ac97_read_modr(void);
diff --git a/include/sound/sdca.h b/include/sound/sdca.h
index 67ff3c88705d5..2bdf4e333e044 100644
--- a/include/sound/sdca.h
+++ b/include/sound/sdca.h
@@ -26,6 +26,8 @@ struct sdca_dev;
  * @name: Human-readable string.
  * @type: Function topology type.
  * @adr: ACPI address (used for SDCA register access).
+ * @duplicate: Internal flag to indicate if other functions of the same type
+ * exist.
  */
 struct sdca_function_desc {
 	struct fwnode_handle *node;
@@ -33,6 +35,8 @@ struct sdca_function_desc {
 	const char *name;
 	u32 type;
 	u8 adr;
+
+	bool duplicate;
 };
 
 /**
diff --git a/include/sound/sdca_asoc.h b/include/sound/sdca_asoc.h
index 46a61a52decc5..ca35d5a44370b 100644
--- a/include/sound/sdca_asoc.h
+++ b/include/sound/sdca_asoc.h
@@ -13,6 +13,7 @@
 struct device;
 struct regmap;
 struct sdca_function_data;
+struct sdca_pde_delay;
 struct snd_ctl_elem_value;
 struct snd_kcontrol;
 struct snd_kcontrol_new;
@@ -99,4 +100,9 @@ int sdca_asoc_q78_put_volsw(struct snd_kcontrol *kcontrol,
 			    struct snd_ctl_elem_value *ucontrol);
 int sdca_asoc_q78_get_volsw(struct snd_kcontrol *kcontrol,
 			    struct snd_ctl_elem_value *ucontrol);
+int sdca_asoc_pde_poll_actual_ps(struct device *dev, struct regmap *regmap,
+				 int function_id, int entity_id,
+			    int from_ps, int to_ps,
+			    const struct sdca_pde_delay *pde_delays,
+			    int num_delays);
 #endif // __SDCA_ASOC_H__
diff --git a/include/sound/sdca_function.h b/include/sound/sdca_function.h
index 0e871c786513f..b1489178b0ef8 100644
--- a/include/sound/sdca_function.h
+++ b/include/sound/sdca_function.h
@@ -1452,7 +1452,6 @@ static inline u32 sdca_range_search(struct sdca_control_range *range,
 }
 
 int sdca_parse_function(struct device *dev, struct sdw_slave *sdw,
-			struct sdca_function_desc *desc,
 			struct sdca_function_data *function);
 
 const char *sdca_find_terminal_name(enum sdca_terminal_type type);
diff --git a/include/sound/sdca_jack.h b/include/sound/sdca_jack.h
index 3ec22046d3ebc..181541f0f4d8c 100644
--- a/include/sound/sdca_jack.h
+++ b/include/sound/sdca_jack.h
@@ -18,10 +18,13 @@ struct snd_soc_jack;
  * struct jack_state - Jack state structure to keep data between interrupts
  * @kctl: Pointer to the ALSA control attached to this jack
  * @jack: Pointer to the ASoC jack struct for this jack
+ * @mask: Possible reported jack status bits for this jack
  */
 struct jack_state {
 	struct snd_kcontrol *kctl;
 	struct snd_soc_jack *jack;
+
+	unsigned int mask;
 };
 
 int sdca_jack_alloc_state(struct sdca_interrupt *interrupt);
diff --git a/include/sound/seq_midi_emul.h b/include/sound/seq_midi_emul.h
index 88799d1e1f53d..afc7654378701 100644
--- a/include/sound/seq_midi_emul.h
+++ b/include/sound/seq_midi_emul.h
@@ -55,14 +55,14 @@ struct snd_midi_channel_set {
 	int  client;			/* Client for this port */
 	int  port;			/* The port number */
 
-	int  max_channels;		/* Size of the channels array */
-	struct snd_midi_channel *channels;
-
 	unsigned char midi_mode;	/* MIDI operating mode */
 	unsigned char gs_master_volume;	/* SYSEX master volume: 0-127 */
 	unsigned char gs_chorus_mode;
 	unsigned char gs_reverb_mode;
 
+	int  max_channels;		/* Size of the channels array */
+	struct snd_midi_channel channels[] __counted_by(max_channels);
+
 };
 
 struct snd_midi_op {
diff --git a/include/sound/snd_wavefront.h b/include/sound/snd_wavefront.h
index 30f508a567661..ac749bb2b836d 100644
--- a/include/sound/snd_wavefront.h
+++ b/include/sound/snd_wavefront.h
@@ -12,6 +12,7 @@
 struct _snd_wavefront_midi;
 struct _snd_wavefront_card;
 struct _snd_wavefront;
+struct snd_wss;
 
 typedef struct _snd_wavefront_midi snd_wavefront_midi_t;
 typedef struct _snd_wavefront_card snd_wavefront_card_t;
@@ -46,6 +47,8 @@ extern void   snd_wavefront_midi_enable_virtual (snd_wavefront_card_t *);
 extern void   snd_wavefront_midi_disable_virtual (snd_wavefront_card_t *);
 extern void   snd_wavefront_midi_interrupt (snd_wavefront_card_t *);
 extern int    snd_wavefront_midi_start (snd_wavefront_card_t *);
+void snd_wavefront_midi_suspend(snd_wavefront_card_t *card);
+void snd_wavefront_midi_resume(snd_wavefront_card_t *card);
 
 struct _snd_wavefront {
 	unsigned long    irq;   /* "you were one, one of the few ..." */
@@ -93,6 +96,7 @@ struct _snd_wavefront {
 	int samples_used;                  /* how many */
 	char interrupts_are_midi;          /* h/w MPU interrupts enabled ? */
 	char rom_samples_rdonly;           /* can we write on ROM samples */
+	char midi_in_to_synth;             /* route external MIDI to synth */
 	spinlock_t irq_lock;
 	wait_queue_head_t interrupt_sleeper; 
 	snd_wavefront_midi_t midi;         /* ICS2115 MIDI interface */
@@ -101,6 +105,7 @@ struct _snd_wavefront {
 
 struct _snd_wavefront_card {
 	snd_wavefront_t wavefront;
+	struct snd_wss *chip;
 #ifdef CONFIG_PNP
 	struct pnp_dev *wss;
 	struct pnp_dev *ctrl;
@@ -110,8 +115,10 @@ struct _snd_wavefront_card {
 };
 
 extern void snd_wavefront_internal_interrupt (snd_wavefront_card_t *card);
+void snd_wavefront_cache_firmware(snd_wavefront_t *dev);
 extern int  snd_wavefront_start (snd_wavefront_t *dev);
 extern int  snd_wavefront_detect (snd_wavefront_card_t *card);
+int snd_wavefront_resume_synth(snd_wavefront_card_t *card);
 extern int  snd_wavefront_cmd (snd_wavefront_t *, int, unsigned char *,
 			       unsigned char *);
 
diff --git a/include/sound/soc-dapm.h b/include/sound/soc-dapm.h
index 4f8fb7622a139..f24eafefbdff2 100644
--- a/include/sound/soc-dapm.h
+++ b/include/sound/soc-dapm.h
@@ -673,6 +673,7 @@ int snd_soc_dapm_mux_update_power(struct snd_soc_dapm_context *dapm,
 /* dapm sys fs - used by the core */
 extern struct attribute *snd_soc_dapm_dev_attrs[];
 void snd_soc_dapm_debugfs_init(struct snd_soc_dapm_context *dapm, struct dentry *parent);
+void snd_soc_dapm_debugfs_pop_time(struct dentry *parent);
 
 /* dapm audio pin control and status */
 int snd_soc_dapm_enable_pin(struct snd_soc_dapm_context *dapm, const char *pin);
diff --git a/include/sound/soc.h b/include/sound/soc.h
index 5e3eb617d8323..967b68d4d142c 100644
--- a/include/sound/soc.h
+++ b/include/sound/soc.h
@@ -431,7 +431,7 @@ struct snd_soc_jack_pin;
 int snd_soc_register_card(struct snd_soc_card *card);
 void snd_soc_unregister_card(struct snd_soc_card *card);
 int devm_snd_soc_register_card(struct device *dev, struct snd_soc_card *card);
-int devm_snd_soc_register_deferrable_card(struct device *dev, struct snd_soc_card *card);
+#define devm_snd_soc_register_deferrable_card(d, c) devm_snd_soc_register_card(d, c)
 #ifdef CONFIG_PM_SLEEP
 int snd_soc_suspend(struct device *dev);
 int snd_soc_resume(struct device *dev);
@@ -1081,8 +1081,6 @@ struct snd_soc_card {
 #ifdef CONFIG_PM_SLEEP
 	struct work_struct deferred_resume_work;
 #endif
-	u32 pop_time;
-
 	/* bit field */
 	unsigned int instantiated:1;
 	unsigned int topology_shortname_created:1;
diff --git a/include/sound/soc_sdw_utils.h b/include/sound/soc_sdw_utils.h
index 4890831836734..79c21966220b2 100644
--- a/include/sound/soc_sdw_utils.h
+++ b/include/sound/soc_sdw_utils.h
@@ -223,6 +223,17 @@ int asoc_sdw_cs42l43_spk_init(struct snd_soc_card *card,
 			      struct asoc_sdw_codec_info *info,
 			      bool playback);
 
+/* es9356 codec support */
+int asoc_sdw_es9356_init(struct snd_soc_card *card,
+			       struct snd_soc_dai_link *dai_links,
+			       struct asoc_sdw_codec_info *info,
+			       bool playback);
+int asoc_sdw_es9356_amp_init(struct snd_soc_card *card,
+			       struct snd_soc_dai_link *dai_links,
+			       struct asoc_sdw_codec_info *info,
+			       bool playback);
+int asoc_sdw_es9356_exit(struct snd_soc_card *card, struct snd_soc_dai_link *dai_link);
+
 /* CS AMP support */
 int asoc_sdw_bridge_cs35l56_count_sidecar(struct snd_soc_card *card,
 					  int *num_dais, int *num_devs);
@@ -272,7 +283,14 @@ int asoc_sdw_ti_amp_init(struct snd_soc_card *card,
 			 struct asoc_sdw_codec_info *info,
 			 bool playback);
 int asoc_sdw_ti_spk_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
+int asoc_sdw_ti_tac5xx2_spk_rtd_init(struct snd_soc_pcm_runtime *rtd,
+				     struct snd_soc_dai *dai);
 int asoc_sdw_ti_amp_initial_settings(struct snd_soc_card *card,
 				     const char *name_prefix);
+int asoc_sdw_ti_dmic_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
+int asoc_sdw_ti_sdca_jack_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
+int asoc_sdw_es9356_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
+int asoc_sdw_es9356_spk_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
+int asoc_sdw_es9356_dmic_rtd_init(struct snd_soc_pcm_runtime *rtd, struct snd_soc_dai *dai);
 
 #endif
diff --git a/include/trace/bpf_probe.h b/include/trace/bpf_probe.h
index 9391d54d3f124..d1de8f9aa07fb 100644
--- a/include/trace/bpf_probe.h
+++ b/include/trace/bpf_probe.h
@@ -58,9 +58,7 @@ static notrace void							\
 __bpf_trace_##call(void *__data, proto)					\
 {									\
 	might_fault();							\
-	preempt_disable_notrace();					\
 	CONCATENATE(bpf_trace_run, COUNT_ARGS(args))(__data, CAST_TO_U64(args));	\
-	preempt_enable_notrace();					\
 }
 
 #undef DECLARE_EVENT_SYSCALL_CLASS
diff --git a/include/trace/events/amdxdna.h b/include/trace/events/amdxdna.h
index c6cb2da7b706c..71da24267e52e 100644
--- a/include/trace/events/amdxdna.h
+++ b/include/trace/events/amdxdna.h
@@ -30,26 +30,30 @@ TRACE_EVENT(amdxdna_debug_point,
 );
 
 TRACE_EVENT(xdna_job,
-	    TP_PROTO(struct drm_sched_job *sched_job, const char *name, const char *str, u64 seq),
+	    TP_PROTO(struct drm_sched_job *sched_job, const char *name,
+		     const char *str, u64 seq, u32 op),
 
-	    TP_ARGS(sched_job, name, str, seq),
+	    TP_ARGS(sched_job, name, str, seq, op),
 
 	    TP_STRUCT__entry(__string(name, name)
 			     __string(str, str)
 			     __field(u64, fence_context)
 			     __field(u64, fence_seqno)
-			     __field(u64, seq)),
+			     __field(u64, seq)
+			     __field(u32, op)),
 
 	    TP_fast_assign(__assign_str(name);
 			   __assign_str(str);
 			   __entry->fence_context = sched_job->s_fence->finished.context;
 			   __entry->fence_seqno = sched_job->s_fence->finished.seqno;
-			   __entry->seq = seq;),
+			   __entry->seq = seq;
+			   __entry->op = op;),
 
-	    TP_printk("fence=(context:%llu, seqno:%lld), %s seq#:%lld %s",
+	    TP_printk("fence=(context:%llu, seqno:%llu), %s seq#:%llu %s, op=%u",
 		      __entry->fence_context, __entry->fence_seqno,
 		      __get_str(name), __entry->seq,
-		      __get_str(str))
+		      __get_str(str),
+		      __entry->op)
 );
 
 DECLARE_EVENT_CLASS(xdna_mbox_msg,
@@ -81,18 +85,28 @@ DEFINE_EVENT(xdna_mbox_msg, mbox_set_head,
 	     TP_ARGS(name, chann_id, opcode, id)
 );
 
-TRACE_EVENT(mbox_irq_handle,
-	    TP_PROTO(char *name, int irq),
+DECLARE_EVENT_CLASS(xdna_mbox_name_id,
+		    TP_PROTO(char *name, int irq),
 
-	    TP_ARGS(name, irq),
+		    TP_ARGS(name, irq),
 
-	    TP_STRUCT__entry(__string(name, name)
-			     __field(int, irq)),
+		    TP_STRUCT__entry(__string(name, name)
+				     __field(int, irq)),
 
-	    TP_fast_assign(__assign_str(name);
-			   __entry->irq = irq;),
+		    TP_fast_assign(__assign_str(name);
+				   __entry->irq = irq;),
+
+		    TP_printk("%s.%d", __get_str(name), __entry->irq)
+);
+
+DEFINE_EVENT(xdna_mbox_name_id, mbox_irq_handle,
+	     TP_PROTO(char *name, int irq),
+	     TP_ARGS(name, irq)
+);
 
-	    TP_printk("%s.%d", __get_str(name), __entry->irq)
+DEFINE_EVENT(xdna_mbox_name_id, mbox_rx_worker,
+	     TP_PROTO(char *name, int irq),
+	     TP_ARGS(name, irq)
 );
 
 #endif /* !defined(_TRACE_AMDXDNA_H) || defined(TRACE_HEADER_MULTI_READ) */
diff --git a/include/trace/events/damon.h b/include/trace/events/damon.h
index 24fc402ab3c85..7e25f4469b81b 100644
--- a/include/trace/events/damon.h
+++ b/include/trace/events/damon.h
@@ -41,7 +41,7 @@ TRACE_EVENT(damos_stat_after_apply_interval,
 	),
 
 	TP_printk("ctx_idx=%u scheme_idx=%u nr_tried=%lu sz_tried=%lu "
-			"nr_applied=%lu sz_tried=%lu sz_ops_filter_passed=%lu "
+			"nr_applied=%lu sz_applied=%lu sz_ops_filter_passed=%lu "
 			"qt_exceeds=%lu nr_snapshots=%lu",
 			__entry->context_idx, __entry->scheme_idx,
 			__entry->nr_tried, __entry->sz_tried,
diff --git a/include/trace/events/dma_fence.h b/include/trace/events/dma_fence.h
index 3abba45c0601a..5b10a9e06fb4e 100644
--- a/include/trace/events/dma_fence.h
+++ b/include/trace/events/dma_fence.h
@@ -9,12 +9,40 @@
 
 struct dma_fence;
 
+DECLARE_EVENT_CLASS(dma_fence,
+
+	TP_PROTO(struct dma_fence *fence),
+
+	TP_ARGS(fence),
+
+	TP_STRUCT__entry(
+		__string(driver, dma_fence_driver_name(fence))
+		__string(timeline, dma_fence_timeline_name(fence))
+		__field(unsigned int, context)
+		__field(unsigned int, seqno)
+	),
+
+	TP_fast_assign(
+		__assign_str(driver);
+		__assign_str(timeline);
+		__entry->context = fence->context;
+		__entry->seqno = fence->seqno;
+	),
+
+	TP_printk("driver=%s timeline=%s context=%u seqno=%u",
+		  __get_str(driver), __get_str(timeline), __entry->context,
+		  __entry->seqno)
+);
+
 /*
  * Safe only for call sites which are guaranteed to not race with fence
- * signaling,holding the fence->lock and having checked for not signaled, or the
- * signaling path itself.
+ * signaling, holding the fence->lock and having checked for not signaled, or
+ * the signaling path itself.
+ *
+ * TODO: Remove the need for this event class when drivers switch to independent
+ *       fences.
  */
-DECLARE_EVENT_CLASS(dma_fence,
+DECLARE_EVENT_CLASS(dma_fence_ops,
 
 	TP_PROTO(struct dma_fence *fence),
 
@@ -46,7 +74,7 @@ DEFINE_EVENT(dma_fence, dma_fence_emit,
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(dma_fence, dma_fence_init,
+DEFINE_EVENT(dma_fence_ops, dma_fence_init,
 
 	TP_PROTO(struct dma_fence *fence),
 
@@ -60,14 +88,14 @@ DEFINE_EVENT(dma_fence, dma_fence_destroy,
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(dma_fence, dma_fence_enable_signal,
+DEFINE_EVENT(dma_fence_ops, dma_fence_enable_signal,
 
 	TP_PROTO(struct dma_fence *fence),
 
 	TP_ARGS(fence)
 );
 
-DEFINE_EVENT(dma_fence, dma_fence_signaled,
+DEFINE_EVENT(dma_fence_ops, dma_fence_signaled,
 
 	TP_PROTO(struct dma_fence *fence),
 
diff --git a/include/trace/events/f2fs.h b/include/trace/events/f2fs.h
index b5188d2671d73..270c1a2c24c44 100644
--- a/include/trace/events/f2fs.h
+++ b/include/trace/events/f2fs.h
@@ -2595,6 +2595,34 @@ DEFINE_EVENT(f2fs_priority_update, f2fs_priority_restore,
 	TP_ARGS(sbi, lock_name, is_write, p, orig_prio, new_prio)
 );
 
+TRACE_EVENT(f2fs_fault_report,
+
+	TP_PROTO(struct super_block *sb, unsigned int err_code,
+		const char *func, unsigned int data),
+
+	TP_ARGS(sb, err_code, func, data),
+
+	TP_STRUCT__entry(
+		__field(dev_t, dev)
+		__field(unsigned int, err_code)
+		__string(func, func)
+		__field(unsigned int, data)
+	),
+
+	TP_fast_assign(
+		__entry->dev		= sb->s_dev;
+		__entry->err_code	= err_code;
+		__assign_str(func);
+		__entry->data		= data;
+	),
+
+	TP_printk("dev = (%d,%d), err_code = %u, func = %s, data = %u",
+		show_dev(__entry->dev),
+		__entry->err_code,
+		__get_str(func),
+		__entry->data)
+);
+
 #endif /* _TRACE_F2FS_H */
 
  /* This part must be outside protection */
diff --git a/include/trace/events/filelock.h b/include/trace/events/filelock.h
index 1167748862449..c8c8847bb6f66 100644
--- a/include/trace/events/filelock.h
+++ b/include/trace/events/filelock.h
@@ -190,7 +190,7 @@ TRACE_EVENT(generic_add_lease,
 		__entry->i_ino = inode->i_ino;
 		__entry->wcount = atomic_read(&inode->i_writecount);
 		__entry->rcount = atomic_read(&inode->i_readcount);
-		__entry->icount = icount_read(inode);
+		__entry->icount = icount_read_once(inode);
 		__entry->owner = fl->c.flc_owner;
 		__entry->flags = fl->c.flc_flags;
 		__entry->type = fl->c.flc_type;
diff --git a/include/trace/events/huge_memory.h b/include/trace/events/huge_memory.h
index bcdc57eea2701..70c25136e7e80 100644
--- a/include/trace/events/huge_memory.h
+++ b/include/trace/events/huge_memory.h
@@ -39,7 +39,8 @@
 	EM( SCAN_STORE_FAILED,		"store_failed")			\
 	EM( SCAN_COPY_MC,		"copy_poisoned_page")		\
 	EM( SCAN_PAGE_FILLED,		"page_filled")			\
-	EMe(SCAN_PAGE_DIRTY_OR_WRITEBACK, "page_dirty_or_writeback")
+	EM(SCAN_PAGE_DIRTY_OR_WRITEBACK, "page_dirty_or_writeback")	\
+	EMe(SCAN_INVALID_PTES_NONE,	"invalid_ptes_none")
 
 #undef EM
 #undef EMe
@@ -89,40 +90,44 @@ TRACE_EVENT(mm_khugepaged_scan_pmd,
 
 TRACE_EVENT(mm_collapse_huge_page,
 
-	TP_PROTO(struct mm_struct *mm, int isolated, int status),
+	TP_PROTO(struct mm_struct *mm, int isolated, int status, unsigned int order),
 
-	TP_ARGS(mm, isolated, status),
+	TP_ARGS(mm, isolated, status, order),
 
 	TP_STRUCT__entry(
 		__field(struct mm_struct *, mm)
 		__field(int, isolated)
 		__field(int, status)
+		__field(unsigned int, order)
 	),
 
 	TP_fast_assign(
 		__entry->mm = mm;
 		__entry->isolated = isolated;
 		__entry->status = status;
+		__entry->order = order;
 	),
 
-	TP_printk("mm=%p, isolated=%d, status=%s",
+	TP_printk("mm=%p, isolated=%d, status=%s, order=%u",
 		__entry->mm,
 		__entry->isolated,
-		__print_symbolic(__entry->status, SCAN_STATUS))
+		__print_symbolic(__entry->status, SCAN_STATUS),
+		__entry->order)
 );
 
 TRACE_EVENT(mm_collapse_huge_page_isolate,
 
 	TP_PROTO(struct folio *folio, int none_or_zero,
-		 int referenced, int status),
+		 int referenced, int status, unsigned int order),
 
-	TP_ARGS(folio, none_or_zero, referenced, status),
+	TP_ARGS(folio, none_or_zero, referenced, status, order),
 
 	TP_STRUCT__entry(
 		__field(unsigned long, pfn)
 		__field(int, none_or_zero)
 		__field(int, referenced)
 		__field(int, status)
+		__field(unsigned int, order)
 	),
 
 	TP_fast_assign(
@@ -130,26 +135,30 @@ TRACE_EVENT(mm_collapse_huge_page_isolate,
 		__entry->none_or_zero = none_or_zero;
 		__entry->referenced = referenced;
 		__entry->status = status;
+		__entry->order = order;
 	),
 
-	TP_printk("scan_pfn=0x%lx, none_or_zero=%d, referenced=%d, status=%s",
+	TP_printk("scan_pfn=0x%lx, none_or_zero=%d, referenced=%d, status=%s, order=%u",
 		__entry->pfn,
 		__entry->none_or_zero,
 		__entry->referenced,
-		__print_symbolic(__entry->status, SCAN_STATUS))
+		__print_symbolic(__entry->status, SCAN_STATUS),
+		__entry->order)
 );
 
 TRACE_EVENT(mm_collapse_huge_page_swapin,
 
-	TP_PROTO(struct mm_struct *mm, int swapped_in, int referenced, int ret),
+	TP_PROTO(struct mm_struct *mm, int swapped_in, int referenced, int ret,
+		 unsigned int order),
 
-	TP_ARGS(mm, swapped_in, referenced, ret),
+	TP_ARGS(mm, swapped_in, referenced, ret, order),
 
 	TP_STRUCT__entry(
 		__field(struct mm_struct *, mm)
 		__field(int, swapped_in)
 		__field(int, referenced)
 		__field(int, ret)
+		__field(unsigned int, order)
 	),
 
 	TP_fast_assign(
@@ -157,13 +166,15 @@ TRACE_EVENT(mm_collapse_huge_page_swapin,
 		__entry->swapped_in = swapped_in;
 		__entry->referenced = referenced;
 		__entry->ret = ret;
+		__entry->order = order;
 	),
 
-	TP_printk("mm=%p, swapped_in=%d, referenced=%d, ret=%d",
+	TP_printk("mm=%p, swapped_in=%d, referenced=%d, ret=%d, order=%u",
 		__entry->mm,
 		__entry->swapped_in,
 		__entry->referenced,
-		__entry->ret)
+		__entry->ret,
+		__entry->order)
 );
 
 TRACE_EVENT(mm_khugepaged_scan_file,
diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h
index 8c936fc575d57..082cb03c61316 100644
--- a/include/trace/events/netfs.h
+++ b/include/trace/events/netfs.h
@@ -177,7 +177,11 @@
 	EM(netfs_folio_is_uptodate,		"mod-uptodate")	\
 	EM(netfs_just_prefetch,			"mod-prefetch")	\
 	EM(netfs_whole_folio_modify,		"mod-whole-f")	\
+	EM(netfs_whole_folio_modify_efault,	"mod-whole-f!")	\
+	EM(netfs_whole_folio_modify_filled,	"mod-whole-f+")	\
+	EM(netfs_whole_folio_modify_filled_efault, "mod-whole-f+!") \
 	EM(netfs_modify_and_clear,		"mod-n-clear")	\
+	EM(netfs_modify_and_clear_rm_finfo,	"mod-n-clear+")	\
 	EM(netfs_streaming_write,		"mod-streamw")	\
 	EM(netfs_streaming_write_cont,		"mod-streamw+")	\
 	EM(netfs_flush_content,			"flush")	\
@@ -194,6 +198,10 @@
 	EM(netfs_folio_trace_copy_to_cache,	"mark-copy")	\
 	EM(netfs_folio_trace_end_copy,		"end-copy")	\
 	EM(netfs_folio_trace_filled_gaps,	"filled-gaps")	\
+	EM(netfs_folio_trace_invalidate_all,	"inval-all")	\
+	EM(netfs_folio_trace_invalidate_front,	"inval-front")	\
+	EM(netfs_folio_trace_invalidate_middle,	"inval-mid")	\
+	EM(netfs_folio_trace_invalidate_tail,	"inval-tail")	\
 	EM(netfs_folio_trace_kill,		"kill")		\
 	EM(netfs_folio_trace_kill_cc,		"kill-cc")	\
 	EM(netfs_folio_trace_kill_g,		"kill-g")	\
diff --git a/include/trace/events/qcom_geni_spi.h b/include/trace/events/qcom_geni_spi.h
new file mode 100644
index 0000000000000..6d027adf2e1df
--- /dev/null
+++ b/include/trace/events/qcom_geni_spi.h
@@ -0,0 +1,103 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM qcom_geni_spi
+
+#if !defined(_TRACE_QCOM_GENI_SPI_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_QCOM_GENI_SPI_H
+
+#include <linux/tracepoint.h>
+
+TRACE_EVENT(geni_spi_setup_params,
+	    TP_PROTO(struct device *dev, u8 cs, u32 mode,
+		     u32 mode_changed, bool cs_changed),
+	    TP_ARGS(dev, cs, mode, mode_changed, cs_changed),
+
+	    TP_STRUCT__entry(__string(name, dev_name(dev))
+			     __field(u8, cs)
+			     __field(u32, mode)
+			     __field(u32, mode_changed)
+			     __field(bool, cs_changed)
+	    ),
+
+	    TP_fast_assign(__assign_str(name);
+			   __entry->cs = cs;
+			   __entry->mode = mode;
+			   __entry->mode_changed = mode_changed;
+			   __entry->cs_changed = cs_changed;
+	    ),
+
+	    TP_printk("%s: cs=%u mode=0x%08x mode_changed=0x%08x cs_changed=%d",
+		      __get_str(name), __entry->cs, __entry->mode,
+		      __entry->mode_changed, __entry->cs_changed)
+);
+
+TRACE_EVENT(geni_spi_clk_cfg,
+	    TP_PROTO(struct device *dev, unsigned long req_hz,
+		     unsigned long sclk_hz, unsigned int clk_idx,
+		     unsigned int clk_div, unsigned int bpw),
+	    TP_ARGS(dev, req_hz, sclk_hz, clk_idx, clk_div, bpw),
+
+	    TP_STRUCT__entry(__string(name, dev_name(dev))
+			     __field(unsigned long, req_hz)
+			     __field(unsigned long, sclk_hz)
+			     __field(unsigned int, clk_idx)
+			     __field(unsigned int, clk_div)
+			     __field(unsigned int, bpw)
+	    ),
+
+	    TP_fast_assign(__assign_str(name);
+			   __entry->req_hz = req_hz;
+			   __entry->sclk_hz = sclk_hz;
+			   __entry->clk_idx = clk_idx;
+			   __entry->clk_div = clk_div;
+			   __entry->bpw = bpw;
+	    ),
+
+	    TP_printk("%s: req_hz=%lu sclk_hz=%lu clk_idx=%u clk_div=%u bpw=%u",
+		      __get_str(name), __entry->req_hz, __entry->sclk_hz,
+		      __entry->clk_idx, __entry->clk_div, __entry->bpw)
+);
+
+TRACE_EVENT(geni_spi_transfer,
+	    TP_PROTO(struct device *dev, unsigned int len, u32 m_cmd),
+	    TP_ARGS(dev, len, m_cmd),
+
+	    TP_STRUCT__entry(__string(name, dev_name(dev))
+			     __field(unsigned int, len)
+			     __field(u32, m_cmd)
+	    ),
+
+	    TP_fast_assign(__assign_str(name);
+			   __entry->len = len;
+			   __entry->m_cmd = m_cmd;
+	    ),
+
+	    TP_printk("%s: len=%u m_cmd=0x%08x",
+		      __get_str(name), __entry->len, __entry->m_cmd)
+);
+
+TRACE_EVENT(geni_spi_irq,
+	    TP_PROTO(struct device *dev, u32 m_irq, u32 dma_tx, u32 dma_rx),
+	    TP_ARGS(dev, m_irq, dma_tx, dma_rx),
+
+	    TP_STRUCT__entry(__string(name, dev_name(dev))
+			     __field(u32, m_irq)
+			     __field(u32, dma_tx)
+			     __field(u32, dma_rx)
+	    ),
+
+	    TP_fast_assign(__assign_str(name);
+			   __entry->m_irq = m_irq;
+			   __entry->dma_tx = dma_tx;
+			   __entry->dma_rx = dma_rx;
+	    ),
+
+	    TP_printk("%s: m_irq=0x%08x dma_tx=0x%08x dma_rx=0x%08x",
+		      __get_str(name), __entry->m_irq, __entry->dma_tx,
+		      __entry->dma_rx)
+);
+
+#endif /* _TRACE_QCOM_GENI_SPI_H */
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
diff --git a/include/trace/events/rxrpc.h b/include/trace/events/rxrpc.h
index 573f2df3a2c99..704a10de66700 100644
--- a/include/trace/events/rxrpc.h
+++ b/include/trace/events/rxrpc.h
@@ -71,6 +71,7 @@
 	EM(rxkad_abort_resp_unknown_tkt,	"rxkad-resp-unknown-tkt") \
 	EM(rxkad_abort_resp_version,		"rxkad-resp-version")	\
 	/* RxGK security errors */					\
+	EM(rxgk_abort_1_short_header,		"rxgk1-short-hdr")	\
 	EM(rxgk_abort_1_verify_mic_eproto,	"rxgk1-vfy-mic-eproto")	\
 	EM(rxgk_abort_2_decrypt_eproto,		"rxgk2-dec-eproto")	\
 	EM(rxgk_abort_2_short_data,		"rxgk2-short-data")	\
diff --git a/include/trace/events/skb.h b/include/trace/events/skb.h
index b877133cd93a8..2945aa7fe9a7d 100644
--- a/include/trace/events/skb.h
+++ b/include/trace/events/skb.h
@@ -24,14 +24,14 @@ DEFINE_DROP_REASON(FN, FN)
 TRACE_EVENT(kfree_skb,
 
 	TP_PROTO(struct sk_buff *skb, void *location,
-		 enum skb_drop_reason reason, struct sock *rx_sk),
+		 enum skb_drop_reason reason, const struct sock *rx_sk),
 
 	TP_ARGS(skb, location, reason, rx_sk),
 
 	TP_STRUCT__entry(
 		__field(void *,		skbaddr)
 		__field(void *,		location)
-		__field(void *,		rx_sk)
+		__field(const void *,	rx_sk)
 		__field(unsigned short,	protocol)
 		__field(enum skb_drop_reason,	reason)
 	),
diff --git a/include/trace/events/timer.h b/include/trace/events/timer.h
index 07cbb9836b91e..ca82fd62dc30f 100644
--- a/include/trace/events/timer.h
+++ b/include/trace/events/timer.h
@@ -298,6 +298,19 @@ DECLARE_EVENT_CLASS(hrtimer_class,
 	TP_printk("hrtimer=%p", __entry->hrtimer)
 );
 
+/**
+ * hrtimer_start_expired - Invoked when a expired timer was started
+ * @hrtimer:	pointer to struct hrtimer
+ *
+ * Preceeded by a hrtimer_start tracepoint.
+ */
+DEFINE_EVENT(hrtimer_class, hrtimer_start_expired,
+
+	TP_PROTO(struct hrtimer *hrtimer),
+
+	TP_ARGS(hrtimer)
+);
+
 /**
  * hrtimer_expire_exit - called immediately after the hrtimer callback returns
  * @hrtimer:	pointer to struct hrtimer
diff --git a/include/trace/events/timer_migration.h b/include/trace/events/timer_migration.h
index 61171b13c687c..0b135e9301b1d 100644
--- a/include/trace/events/timer_migration.h
+++ b/include/trace/events/timer_migration.h
@@ -33,15 +33,16 @@ TRACE_EVENT(tmigr_group_set,
 
 TRACE_EVENT(tmigr_connect_child_parent,
 
-	TP_PROTO(struct tmigr_group *child),
+	TP_PROTO(struct tmigr_hierarchy *hier, struct tmigr_group *child),
 
-	TP_ARGS(child),
+	TP_ARGS(hier, child),
 
 	TP_STRUCT__entry(
 		__field( void *,	child		)
 		__field( void *,	parent		)
 		__field( unsigned int,	lvl		)
 		__field( unsigned int,	numa_node	)
+		__field( unsigned int,	capacity	)
 		__field( unsigned int,	num_children	)
 		__field( u32,		groupmask	)
 	),
@@ -51,26 +52,28 @@ TRACE_EVENT(tmigr_connect_child_parent,
 		__entry->parent		= child->parent;
 		__entry->lvl		= child->parent->level;
 		__entry->numa_node	= child->parent->numa_node;
+		__entry->capacity	= hier->capacity;
 		__entry->num_children	= child->parent->num_children;
 		__entry->groupmask	= child->groupmask;
 	),
 
-	TP_printk("group=%p groupmask=%0x parent=%p lvl=%d numa=%d num_children=%d",
-		  __entry->child,  __entry->groupmask, __entry->parent,
-		  __entry->lvl, __entry->numa_node, __entry->num_children)
+	TP_printk("group=%p groupmask=%0x parent=%p lvl=%d numa=%d capacity=%d num_children=%d",
+		  __entry->child,  __entry->groupmask, __entry->parent, __entry->lvl,
+		  __entry->numa_node, __entry->capacity, __entry->num_children)
 );
 
 TRACE_EVENT(tmigr_connect_cpu_parent,
 
-	TP_PROTO(struct tmigr_cpu *tmc),
+	TP_PROTO(struct tmigr_hierarchy *hier, struct tmigr_cpu *tmc),
 
-	TP_ARGS(tmc),
+	TP_ARGS(hier, tmc),
 
 	TP_STRUCT__entry(
 		__field( void *,	parent		)
 		__field( unsigned int,	cpu		)
 		__field( unsigned int,	lvl		)
 		__field( unsigned int,	numa_node	)
+		__field( unsigned int,	capacity	)
 		__field( unsigned int,	num_children	)
 		__field( u32,		groupmask	)
 	),
@@ -80,13 +83,14 @@ TRACE_EVENT(tmigr_connect_cpu_parent,
 		__entry->cpu		= tmc->cpuevt.cpu;
 		__entry->lvl		= tmc->tmgroup->level;
 		__entry->numa_node	= tmc->tmgroup->numa_node;
+		__entry->capacity	= hier->capacity;
 		__entry->num_children	= tmc->tmgroup->num_children;
 		__entry->groupmask	= tmc->groupmask;
 	),
 
-	TP_printk("cpu=%d groupmask=%0x parent=%p lvl=%d numa=%d num_children=%d",
-		  __entry->cpu,	 __entry->groupmask, __entry->parent,
-		  __entry->lvl, __entry->numa_node, __entry->num_children)
+	TP_printk("cpu=%d groupmask=%0x parent=%p lvl=%d numa=%d capacity=%d num_children=%d",
+		  __entry->cpu,	 __entry->groupmask, __entry->parent, __entry->lvl,
+		  __entry->numa_node, __entry->capacity, __entry->num_children)
 );
 
 DECLARE_EVENT_CLASS(tmigr_group_and_cpu,
diff --git a/include/trace/events/vmscan.h b/include/trace/events/vmscan.h
index 4445a8d9218de..b4bf7b8def1f5 100644
--- a/include/trace/events/vmscan.h
+++ b/include/trace/events/vmscan.h
@@ -96,6 +96,58 @@ TRACE_EVENT(mm_vmscan_kswapd_wake,
 		__entry->order)
 );
 
+TRACE_EVENT(mm_vmscan_balance_pgdat_begin,
+
+	TP_PROTO(int nid, int order, int highest_zoneidx),
+
+	TP_ARGS(nid, order, highest_zoneidx),
+
+	TP_STRUCT__entry(
+		__field(int, nid)
+		__field(int, order)
+		__field(int, highest_zoneidx)
+	),
+
+	TP_fast_assign(
+		__entry->nid = nid;
+		__entry->order = order;
+		__entry->highest_zoneidx = highest_zoneidx;
+	),
+
+	TP_printk("nid=%d order=%d highest_zoneidx=%-8s",
+		__entry->nid,
+		__entry->order,
+		__print_symbolic(__entry->highest_zoneidx, ZONE_TYPE))
+);
+
+TRACE_EVENT(mm_vmscan_balance_pgdat_end,
+
+	TP_PROTO(int nid, int order, int highest_zoneidx,
+		 unsigned long nr_reclaimed),
+
+	TP_ARGS(nid, order, highest_zoneidx, nr_reclaimed),
+
+	TP_STRUCT__entry(
+		__field(int, nid)
+		__field(int, order)
+		__field(int, highest_zoneidx)
+		__field(unsigned long, nr_reclaimed)
+	),
+
+	TP_fast_assign(
+		__entry->nid = nid;
+		__entry->order = order;
+		__entry->highest_zoneidx = highest_zoneidx;
+		__entry->nr_reclaimed = nr_reclaimed;
+	),
+
+	TP_printk("nid=%d order=%d highest_zoneidx=%-8s nr_reclaimed=%lu",
+		__entry->nid,
+		__entry->order,
+		__print_symbolic(__entry->highest_zoneidx, ZONE_TYPE),
+		__entry->nr_reclaimed)
+);
+
 TRACE_EVENT(mm_vmscan_wakeup_kswapd,
 
 	TP_PROTO(int nid, int zid, int order, gfp_t gfp_flags),
diff --git a/include/uapi/asm-generic/errno.h b/include/uapi/asm-generic/errno.h
index 92e7ae493ee31..bd78e69e0a43f 100644
--- a/include/uapi/asm-generic/errno.h
+++ b/include/uapi/asm-generic/errno.h
@@ -122,4 +122,6 @@
 
 #define EHWPOISON	133	/* Memory page has hardware error */
 
+#define EFTYPE		134	/* Wrong file type for the intended operation */
+
 #endif
diff --git a/include/uapi/asm-generic/fcntl.h b/include/uapi/asm-generic/fcntl.h
index 613475285643b..c4b0e7153fa87 100644
--- a/include/uapi/asm-generic/fcntl.h
+++ b/include/uapi/asm-generic/fcntl.h
@@ -88,6 +88,14 @@
 #define __O_TMPFILE	020000000
 #endif
 
+#ifndef OPENAT2_REGULAR
+#define OPENAT2_REGULAR	040000000
+#endif
+
+#ifndef O_EMPTYPATH
+#define O_EMPTYPATH	0100000000	/* allow empty path */
+#endif
+
 /* a horrid kludge trying to make sure that this will fail on old kernels */
 #define O_TMPFILE (__O_TMPFILE | O_DIRECTORY)
 
diff --git a/include/uapi/drm/amdxdna_accel.h b/include/uapi/drm/amdxdna_accel.h
index 61d3686fa3b1f..51a507561df6a 100644
--- a/include/uapi/drm/amdxdna_accel.h
+++ b/include/uapi/drm/amdxdna_accel.h
@@ -18,6 +18,7 @@ extern "C" {
 #define AMDXDNA_INVALID_CTX_HANDLE	0
 #define AMDXDNA_INVALID_BO_HANDLE	0
 #define AMDXDNA_INVALID_FENCE_HANDLE	0
+#define AMDXDNA_INVALID_DOORBELL_OFFSET	(~0U)
 
 /*
  * Define hardware context priority
@@ -29,7 +30,9 @@ extern "C" {
 
 enum amdxdna_device_type {
 	AMDXDNA_DEV_TYPE_UNKNOWN = -1,
-	AMDXDNA_DEV_TYPE_KMQ,
+	AMDXDNA_DEV_TYPE_KMQ = 0,
+	AMDXDNA_DEV_TYPE_UMQ = 1,
+	AMDXDNA_DEV_TYPE_PF = 2,
 };
 
 enum amdxdna_drm_ioctl_id {
@@ -42,7 +45,8 @@ enum amdxdna_drm_ioctl_id {
 	DRM_AMDXDNA_EXEC_CMD,
 	DRM_AMDXDNA_GET_INFO,
 	DRM_AMDXDNA_SET_STATE,
-	DRM_AMDXDNA_GET_ARRAY = 10,
+	DRM_AMDXDNA_WAIT_CMD,
+	DRM_AMDXDNA_GET_ARRAY,
 };
 
 /**
@@ -271,6 +275,21 @@ struct amdxdna_drm_exec_cmd {
 	__u64 seq;
 };
 
+/**
+ * struct amdxdna_drm_wait_cmd - Wait execution command.
+ *
+ * @hwctx: Context handle.
+ * @timeout: timeout in ms, 0 implies infinite wait.
+ * @seq: sequence number of the command returned by execute command.
+ *
+ * Wait a command specified by seq to be completed.
+ */
+struct amdxdna_drm_wait_cmd {
+	__u32 hwctx;
+	__u32 timeout;
+	__u64 seq;
+};
+
 /**
  * struct amdxdna_drm_query_aie_status - Query the status of the AIE hardware
  * @buffer: The user space buffer that will return the AIE status.
@@ -736,6 +755,10 @@ struct amdxdna_drm_set_power_mode {
 	DRM_IOWR(DRM_COMMAND_BASE + DRM_AMDXDNA_GET_ARRAY, \
 		 struct amdxdna_drm_get_array)
 
+#define DRM_IOCTL_AMDXDNA_WAIT_CMD \
+	DRM_IOW(DRM_COMMAND_BASE + DRM_AMDXDNA_WAIT_CMD, \
+		struct amdxdna_drm_wait_cmd)
+
 #if defined(__cplusplus)
 } /* extern c end */
 #endif
diff --git a/include/uapi/drm/drm.h b/include/uapi/drm/drm.h
index 27cc159c1d275..bc7ef7684099b 100644
--- a/include/uapi/drm/drm.h
+++ b/include/uapi/drm/drm.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: MIT */
 /*
  * Header for the Direct Rendering Manager
  *
@@ -11,25 +12,6 @@
  * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas.
  * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  * All rights reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #ifndef _DRM_H_
@@ -1323,6 +1305,13 @@ extern "C" {
  */
 #define DRM_IOCTL_MODE_GETFB2		DRM_IOWR(0xCE, struct drm_mode_fb_cmd2)
 
+/**
+ * DRM_IOCTL_SYNCOBJ_EVENTFD - Register an eventfd to be signalled by a syncobj.
+ *
+ * This can be used to integrate a syncobj in an event loop.
+ *
+ * The IOCTL argument is a struct drm_syncobj_eventfd.
+ */
 #define DRM_IOCTL_SYNCOBJ_EVENTFD	DRM_IOWR(0xCF, struct drm_syncobj_eventfd)
 
 /**
diff --git a/include/uapi/drm/drm_fourcc.h b/include/uapi/drm/drm_fourcc.h
index ac66fa93b5a3e..3a4d4dc635bf5 100644
--- a/include/uapi/drm/drm_fourcc.h
+++ b/include/uapi/drm/drm_fourcc.h
@@ -1,24 +1,6 @@
+/* SPDX-License-Identifier: MIT */
 /*
  * Copyright 2011 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #ifndef DRM_FOURCC_H
@@ -242,9 +224,9 @@ extern "C" {
  * [31:0] sign:exponent:mantissa 1:8:23
  */
 #define DRM_FORMAT_R32F          fourcc_code('R', ' ', ' ', 'F') /* [31:0] R 32 little endian */
-#define DRM_FORMAT_GR3232F       fourcc_code('G', 'R', ' ', 'F') /* [63:0] R:G 32:32 little endian */
-#define DRM_FORMAT_BGR323232F    fourcc_code('B', 'G', 'R', 'F') /* [95:0] R:G:B 32:32:32 little endian */
-#define DRM_FORMAT_ABGR32323232F fourcc_code('A', 'B', '8', 'F') /* [127:0] R:G:B:A 32:32:32:32 little endian */
+#define DRM_FORMAT_GR3232F       fourcc_code('G', 'R', ' ', 'F') /* [63:0] G:R 32:32 little endian */
+#define DRM_FORMAT_BGR323232F    fourcc_code('B', 'G', 'R', 'F') /* [95:0] B:G:R 32:32:32 little endian */
+#define DRM_FORMAT_ABGR32323232F fourcc_code('A', 'B', '8', 'F') /* [127:0] A:B:G:R 32:32:32:32 little endian */
 
 /*
  * RGBA format with 10-bit components packed in 64-bit per pixel, with 6 bits
@@ -264,6 +246,7 @@ extern "C" {
 #define DRM_FORMAT_XVUY8888	fourcc_code('X', 'V', 'U', 'Y') /* [31:0] X:Cr:Cb:Y 8:8:8:8 little endian */
 #define DRM_FORMAT_VUY888	fourcc_code('V', 'U', '2', '4') /* [23:0] Cr:Cb:Y 8:8:8 little endian */
 #define DRM_FORMAT_VUY101010	fourcc_code('V', 'U', '3', '0') /* Y followed by U then V, 10:10:10. Non-linear modifier only */
+#define DRM_FORMAT_XVUY2101010	fourcc_code('X', 'Y', '3', '0') /* [31:0] x:Cr:Cb:Y 2:10:10:10 little endian */
 
 /*
  * packed Y2xx indicate for each component, xx valid data occupy msb
@@ -379,6 +362,14 @@ extern "C" {
  */
 #define DRM_FORMAT_P030		fourcc_code('P', '0', '3', '0') /* 2x2 subsampled Cr:Cb plane 10 bits per channel packed */
 
+/*
+ * 2 plane YCbCr422.
+ * 3 10 bit components and 2 padding bits packed into 4 bytes.
+ * index 0 = Y plane, [31:0] x:Y2:Y1:Y0 2:10:10:10 little endian
+ * index 1 = Cr:Cb plane, [63:0] x:Cr2:Cb2:Cr1:x:Cb1:Cr0:Cb0 [2:10:10:10:2:10:10:10] little endian
+ */
+#define DRM_FORMAT_P230		fourcc_code('P', '2', '3', '0') /* 2x1 subsampled Cr:Cb plane 10 bits per channel packed */
+
 /* 3 plane non-subsampled (444) YCbCr
  * 16 bits per component, but only 10 bits are used and 6 bits are padded
  * index 0: Y plane, [15:0] Y:x [10:6] little endian
@@ -395,6 +386,15 @@ extern "C" {
  */
 #define DRM_FORMAT_Q401		fourcc_code('Q', '4', '0', '1')
 
+/*
+ * 3 plane non-subsampled (444) YCbCr LSB aligned
+ * 10 bpc, 30 bits per sample image data in a single contiguous buffer.
+ * index 0: Y plane,  [31:0] x:Y2:Y1:Y0    [2:10:10:10] little endian
+ * index 1: Cb plane, [31:0] x:Cb2:Cb1:Cb0 [2:10:10:10] little endian
+ * index 2: Cr plane, [31:0] x:Cr2:Cr1:Cr0 [2:10:10:10] little endian
+ */
+#define DRM_FORMAT_T430		fourcc_code('T', '4', '3', '0')
+
 /*
  * 3 plane YCbCr LSB aligned
  * In order to use these formats in a similar fashion to MSB aligned ones
@@ -451,6 +451,16 @@ extern "C" {
 #define DRM_FORMAT_YUV444	fourcc_code('Y', 'U', '2', '4') /* non-subsampled Cb (1) and Cr (2) planes */
 #define DRM_FORMAT_YVU444	fourcc_code('Y', 'V', '2', '4') /* non-subsampled Cr (1) and Cb (2) planes */
 
+/*
+ * Y-only (greyscale) formats
+ *
+ * The Y-only formats are handled similarly to the YCbCr formats in the display
+ * pipeline, with the Cb and Cr implicitly neutral (0.0 in nominal values). This
+ * also means that COLOR_RANGE property applies to the Y-only formats.
+ */
+
+#define DRM_FORMAT_Y8		fourcc_code('G', 'R', 'E', 'Y')  /* 8-bit Y-only */
+#define DRM_FORMAT_XYYY2101010	fourcc_code('Y', 'P', 'A', '4')  /* [31:0] x:Y2:Y1:Y0 2:10:10:10 little endian */
 
 /*
  * Format Modifiers:
diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h
index a4bdc4bd11bc1..381a3e857d4e9 100644
--- a/include/uapi/drm/drm_mode.h
+++ b/include/uapi/drm/drm_mode.h
@@ -1,27 +1,10 @@
+/* SPDX-License-Identifier: MIT */
 /*
  * Copyright (c) 2007 Dave Airlie <airlied@linux.ie>
  * Copyright (c) 2007 Jakob Bornecrantz <wallbraker@gmail.com>
  * Copyright (c) 2008 Red Hat Inc.
  * Copyright (c) 2007-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
  * Copyright (c) 2007-2008 Intel Corporation
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
- * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
- * IN THE SOFTWARE.
  */
 
 #ifndef _DRM_MODE_H
diff --git a/include/uapi/drm/drm_ras.h b/include/uapi/drm/drm_ras.h
index 5f40fa5b869d0..218a3ee868052 100644
--- a/include/uapi/drm/drm_ras.h
+++ b/include/uapi/drm/drm_ras.h
@@ -41,6 +41,7 @@ enum {
 enum {
 	DRM_RAS_CMD_LIST_NODES = 1,
 	DRM_RAS_CMD_GET_ERROR_COUNTER,
+	DRM_RAS_CMD_CLEAR_ERROR_COUNTER,
 
 	__DRM_RAS_CMD_MAX,
 	DRM_RAS_CMD_MAX = (__DRM_RAS_CMD_MAX - 1)
diff --git a/include/uapi/drm/drm_sarea.h b/include/uapi/drm/drm_sarea.h
index a951ced60ebe0..1e38d028332de 100644
--- a/include/uapi/drm/drm_sarea.h
+++ b/include/uapi/drm/drm_sarea.h
@@ -1,3 +1,4 @@
+/* SPDX-License-Identifier: MIT */
 /**
  * \file drm_sarea.h
  * \brief SAREA definitions
@@ -8,25 +9,6 @@
 /*
  * Copyright 2002 Tungsten Graphics, Inc., Cedar Park, Texas.
  * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
  */
 
 #ifndef _DRM_SAREA_H_
diff --git a/include/uapi/drm/virtgpu_drm.h b/include/uapi/drm/virtgpu_drm.h
index 9debb320c34be..95587e12aed57 100644
--- a/include/uapi/drm/virtgpu_drm.h
+++ b/include/uapi/drm/virtgpu_drm.h
@@ -98,6 +98,7 @@ struct drm_virtgpu_execbuffer {
 #define VIRTGPU_PARAM_CONTEXT_INIT 6 /* DRM_VIRTGPU_CONTEXT_INIT */
 #define VIRTGPU_PARAM_SUPPORTED_CAPSET_IDs 7 /* Bitmask of supported capability set ids */
 #define VIRTGPU_PARAM_EXPLICIT_DEBUG_NAME 8 /* Ability to set debug name from userspace */
+#define VIRTGPU_PARAM_BLOB_ALIGNMENT 9 /* Device alignment requirements for blobs */
 
 struct drm_virtgpu_getparam {
 	__u64 param;
@@ -200,6 +201,10 @@ struct drm_virtgpu_resource_create_blob {
 	__u32 cmd_size;
 	__u64 cmd;
 	__u64 blob_id;
+
+#define DRM_VIRTGPU_BLOB_FLAG_HINT_DEFER_MAPPING        0x0001
+	__u32 blob_hints;
+	__u32 pad2;
 };
 
 #define VIRTGPU_CONTEXT_PARAM_CAPSET_ID       0x0001
diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h
index ae2fda23ce7c7..48e9f1fdb78dd 100644
--- a/include/uapi/drm/xe_drm.h
+++ b/include/uapi/drm/xe_drm.h
@@ -83,6 +83,7 @@ extern "C" {
  *  - &DRM_IOCTL_XE_OBSERVATION
  *  - &DRM_IOCTL_XE_MADVISE
  *  - &DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS
+ *  - &DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY
  *  - &DRM_IOCTL_XE_VM_GET_PROPERTY
  */
 
@@ -167,7 +168,7 @@ extern "C" {
  * Typically the struct drm_xe_user_extension would be embedded in some uAPI
  * struct, and in this case we would feed it the head of the chain(i.e ext1),
  * which would then apply all of the above extensions.
-*/
+ */
 
 /**
  * struct drm_xe_user_extension - Base class for defining a chain of extensions
@@ -229,9 +230,9 @@ struct drm_xe_ext_set_property {
 /**
  * struct drm_xe_engine_class_instance - instance of an engine class
  *
- * It is returned as part of the @drm_xe_engine, but it also is used as
- * the input of engine selection for both @drm_xe_exec_queue_create and
- * @drm_xe_query_engine_cycles
+ * It is returned as part of the &struct drm_xe_engine, but it also is used as
+ * the input of engine selection for both &struct drm_xe_exec_queue_create and
+ * &struct drm_xe_query_engine_cycles
  *
  * The @engine_class can be:
  *  - %DRM_XE_ENGINE_CLASS_RENDER
@@ -264,7 +265,7 @@ struct drm_xe_engine_class_instance {
  * struct drm_xe_engine - describe hardware engine
  */
 struct drm_xe_engine {
-	/** @instance: The @drm_xe_engine_class_instance */
+	/** @instance: The &struct drm_xe_engine_class_instance */
 	struct drm_xe_engine_class_instance instance;
 
 	/** @reserved: Reserved */
@@ -274,9 +275,9 @@ struct drm_xe_engine {
 /**
  * struct drm_xe_query_engines - describe engines
  *
- * If a query is made with a struct @drm_xe_device_query where .query
+ * If a query is made with a &struct drm_xe_device_query where .query
  * is equal to %DRM_XE_DEVICE_QUERY_ENGINES, then the reply uses an array of
- * struct @drm_xe_query_engines in .data.
+ * &struct drm_xe_query_engines in .data.
  */
 struct drm_xe_query_engines {
 	/** @num_engines: number of engines returned in @engines */
@@ -349,7 +350,7 @@ struct drm_xe_mem_region {
 	 * is smaller than @total_size then this is referred to as a
 	 * small BAR system.
 	 *
-	 * On systems without small BAR (full BAR), the probed_size will
+	 * On systems without small BAR (full BAR), the @cpu_visible_size will
 	 * always equal the @total_size, since all of it will be CPU
 	 * accessible.
 	 *
@@ -410,7 +411,7 @@ struct drm_xe_query_mem_regions {
  *      device supports the userspace hint %DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION.
  *      This is exposed only on Xe2+.
  *    - %DRM_XE_QUERY_CONFIG_FLAG_HAS_DISABLE_STATE_CACHE_PERF_FIX - Flag is set
- *      if a queue can be creaed with
+ *      if a queue can be created with
  *      %DRM_XE_EXEC_QUEUE_SET_DISABLE_STATE_CACHE_PERF_FIX
  *  - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment
  *    required by this device, typically SZ_4K or SZ_64K
@@ -705,7 +706,10 @@ struct drm_xe_query_pxp_status {
  *    attributes.
  *  - %DRM_XE_DEVICE_QUERY_GT_TOPOLOGY
  *  - %DRM_XE_DEVICE_QUERY_ENGINE_CYCLES
+ *  - %DRM_XE_DEVICE_QUERY_UC_FW_VERSION
+ *  - %DRM_XE_DEVICE_QUERY_OA_UNITS
  *  - %DRM_XE_DEVICE_QUERY_PXP_STATUS
+ *  - %DRM_XE_DEVICE_QUERY_EU_STALL
  *
  * If size is set to 0, the driver fills it with the required size for
  * the requested type of data to query. If size is equal to the required
@@ -825,7 +829,7 @@ struct drm_xe_device_query {
  *
  * This ioctl supports setting the following properties via the
  * %DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY extension, which uses the
- * generic @drm_xe_ext_set_property struct:
+ * generic &struct drm_xe_ext_set_property:
  *
  *  - %DRM_XE_GEM_CREATE_SET_PROPERTY_PXP_TYPE - set the type of PXP session
  *    this object will be used with. Valid values are listed in enum
@@ -862,8 +866,7 @@ struct drm_xe_gem_create {
 #define DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM	(1 << 2)
 #define DRM_XE_GEM_CREATE_FLAG_NO_COMPRESSION		(1 << 3)
 	/**
-	 * @flags: Flags, currently a mask of memory instances of where BO can
-	 * be placed
+	 * @flags: Flags for the GEM object, see DRM_XE_GEM_CREATE_FLAG_*
 	 */
 	__u32 flags;
 
@@ -888,7 +891,7 @@ struct drm_xe_gem_create {
 #define DRM_XE_GEM_CPU_CACHING_WC                      2
 	/**
 	 * @cpu_caching: The CPU caching mode to select for this object. If
-	 * mmaping the object the mode selected here will also be used. The
+	 * mmapping the object the mode selected here will also be used. The
 	 * exception is when mapping system memory (including data evicted
 	 * to system) on discrete GPUs. The caching mode selected will
 	 * then be overridden to DRM_XE_GEM_CPU_CACHING_WB, and coherency
@@ -931,7 +934,7 @@ struct drm_xe_gem_create {
  *
  *     err = ioctl(fd, DRM_IOCTL_XE_GEM_MMAP_OFFSET, &mmo);
  *     map = mmap(NULL, size, PROT_WRITE, MAP_SHARED, fd, mmo.offset);
- *     map[i] = 0xdeadbeaf; // issue barrier
+ *     map[i] = 0xdeadbeef; // issue barrier
  */
 struct drm_xe_gem_mmap_offset {
 	/** @extensions: Pointer to the first extension struct, if any */
@@ -958,8 +961,8 @@ struct drm_xe_gem_mmap_offset {
  *  - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE - Map the whole virtual address
  *    space of the VM to scratch page. A vm_bind would overwrite the scratch
  *    page mapping. This flag is mutually exclusive with the
- *    %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception of on x2 and
- *    xe3 platform.
+ *    %DRM_XE_VM_CREATE_FLAG_FAULT_MODE flag, with an exception on Xe2 and
+ *    Xe3 platforms.
  *  - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts
  *    exec submissions to its exec_queues that don't have an upper time
  *    limit on the job execution time. But exec submissions to these
@@ -1045,7 +1048,7 @@ struct drm_xe_vm_destroy {
  *    set, no mappings are created rather the range is reserved for CPU address
  *    mirroring which will be populated on GPU page faults or prefetches. Only
  *    valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address
- *    mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO
+ *    mirror flag is only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO
  *    handle MBZ, and the BO offset MBZ.
  *  - %DRM_XE_VM_BIND_FLAG_MADVISE_AUTORESET - Can be used in combination with
  *    %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR to reset madvises when the underlying
@@ -1061,6 +1064,7 @@ struct drm_xe_vm_destroy {
  *    not invoke autoreset. Neither will stack variables going out of scope.
  *    Therefore it's recommended to always explicitly reset the madvises when
  *    freeing the memory backing a region used in a &DRM_IOCTL_XE_MADVISE call.
+ *
  *  - %DRM_XE_VM_BIND_FLAG_DECOMPRESS - Request on-device decompression for a MAP.
  *    When set on a MAP bind operation, request the driver schedule an on-device
  *    in-place decompression (via the migrate/resolve path) for the GPU mapping
@@ -1109,7 +1113,7 @@ struct drm_xe_vm_bind_op {
 	 *	ppGTT WT -> COH_NONE
 	 *	ppGTT WB -> COH_AT_LEAST_1WAY
 	 *
-	 * In practice UC/WC/WT should only ever used for scanout surfaces on
+	 * In practice UC/WC/WT should only ever be used for scanout surfaces on
 	 * such platforms (or perhaps in general for dma-buf if shared with
 	 * another device) since it is only the display engine that is actually
 	 * incoherent.  Everything else should typically use WB given that we
@@ -1199,10 +1203,10 @@ struct drm_xe_vm_bind_op {
 /**
  * struct drm_xe_vm_bind - Input of &DRM_IOCTL_XE_VM_BIND
  *
- * Below is an example of a minimal use of @drm_xe_vm_bind to
+ * Below is an example of a minimal use of &struct drm_xe_vm_bind to
  * asynchronously bind the buffer `data` at address `BIND_ADDRESS` to
  * illustrate `userptr`. It can be synchronized by using the example
- * provided for @drm_xe_sync.
+ * provided for &struct drm_xe_sync.
  *
  * .. code-block:: C
  *
@@ -1355,7 +1359,7 @@ struct drm_xe_vm_get_property {
  *
  * This ioctl supports setting the following properties via the
  * %DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY extension, which uses the
- * generic @drm_xe_ext_set_property struct:
+ * generic &struct drm_xe_ext_set_property:
  *
  *  - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY - set the queue priority.
  *    CAP_SYS_NICE is required to set a value above normal.
@@ -1366,7 +1370,7 @@ struct drm_xe_vm_get_property {
  *    drm_xe_pxp_session_type. %DRM_XE_PXP_TYPE_NONE is the default behavior, so
  *    there is no need to explicitly set that. When a queue of type
  *    %DRM_XE_PXP_TYPE_HWDRM is created, the PXP default HWDRM session
- *    (%XE_PXP_HWDRM_DEFAULT_SESSION) will be started, if isn't already running.
+ *    (%DRM_XE_PXP_HWDRM_DEFAULT_SESSION) will be started, if it isn't already running.
  *    The user is expected to query the PXP status via the query ioctl (see
  *    %DRM_XE_DEVICE_QUERY_PXP_STATUS) and to wait for PXP to be ready before
  *    attempting to create a queue with this property. When a queue is created
@@ -1390,9 +1394,9 @@ struct drm_xe_vm_get_property {
  *    enable render color cache keying on BTP+BTI instead of just BTI
  *    (only valid for render queues).
  *
- * The example below shows how to use @drm_xe_exec_queue_create to create
+ * The example below shows how to use &struct drm_xe_exec_queue_create to create
  * a simple exec_queue (no parallel submission) of class
- * &DRM_XE_ENGINE_CLASS_RENDER.
+ * %DRM_XE_ENGINE_CLASS_RENDER.
  *
  * .. code-block:: C
  *
@@ -1402,23 +1406,25 @@ struct drm_xe_vm_get_property {
  *     struct drm_xe_exec_queue_create exec_queue_create = {
  *          .extensions = 0,
  *          .vm_id = vm,
- *          .num_bb_per_exec = 1,
- *          .num_eng_per_bb = 1,
+ *          .width = 1,
+ *          .num_placements = 1,
  *          .instances = to_user_pointer(&instance),
  *     };
  *     ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create);
  *
- *     Allow users to provide a hint to kernel for cases demanding low latency
- *     profile. Please note it will have impact on power consumption. User can
- *     indicate low latency hint with flag while creating exec queue as
- *     mentioned below,
+ * Allow users to provide a hint to kernel for cases demanding low latency
+ * profile. Please note it will have impact on power consumption. User can
+ * indicate low latency hint with flag while creating exec queue as
+ * mentioned below:
+ *
+ * .. code-block:: C
  *
  *     struct drm_xe_exec_queue_create exec_queue_create = {
  *          .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT,
  *          .extensions = 0,
  *          .vm_id = vm,
- *          .num_bb_per_exec = 1,
- *          .num_eng_per_bb = 1,
+ *          .width = 1,
+ *          .num_placements = 1,
  *          .instances = to_user_pointer(&instance),
  *     };
  *     ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create);
@@ -1515,7 +1521,7 @@ struct drm_xe_exec_queue_get_property {
  * and the @flags can be:
  *  - %DRM_XE_SYNC_FLAG_SIGNAL
  *
- * A minimal use of @drm_xe_sync looks like this:
+ * A minimal use of &struct drm_xe_sync looks like this:
  *
  * .. code-block:: C
  *
@@ -1546,7 +1552,7 @@ struct drm_xe_sync {
 #define DRM_XE_SYNC_TYPE_SYNCOBJ		0x0
 #define DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ	0x1
 #define DRM_XE_SYNC_TYPE_USER_FENCE		0x2
-	/** @type: Type of the this sync object */
+	/** @type: Type of this sync object */
 	__u32 type;
 
 #define DRM_XE_SYNC_FLAG_SIGNAL	(1 << 0)
@@ -1559,9 +1565,9 @@ struct drm_xe_sync {
 
 		/**
 		 * @addr: Address of user fence. When sync is passed in via exec
-		 * IOCTL this is a GPU address in the VM. When sync passed in via
+		 * IOCTL this is a GPU address in the VM. When sync is passed in via
 		 * VM bind IOCTL this is a user pointer. In either case, it is
-		 * the users responsibility that this address is present and
+		 * the user's responsibility that this address is present and
 		 * mapped when the user fence is signalled. Must be qword
 		 * aligned.
 		 */
@@ -1581,10 +1587,10 @@ struct drm_xe_sync {
 /**
  * struct drm_xe_exec - Input of &DRM_IOCTL_XE_EXEC
  *
- * This is an example to use @drm_xe_exec for execution of the object
- * at BIND_ADDRESS (see example in @drm_xe_vm_bind) by an exec_queue
- * (see example in @drm_xe_exec_queue_create). It can be synchronized
- * by using the example provided for @drm_xe_sync.
+ * This is an example to use &struct drm_xe_exec for execution of the object
+ * at BIND_ADDRESS (see example in &struct drm_xe_vm_bind) by an exec_queue
+ * (see example in &struct drm_xe_exec_queue_create). It can be synchronized
+ * by using the example provided for &struct drm_xe_sync.
  *
  * .. code-block:: C
  *
@@ -1651,7 +1657,6 @@ struct drm_xe_exec {
  *
  * and the @flags can be:
  *  - %DRM_XE_UFENCE_WAIT_FLAG_ABSTIME
- *  - %DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP
  *
  * The @mask values can be for example:
  *  - 0xffu for u8
@@ -1664,7 +1669,7 @@ struct drm_xe_wait_user_fence {
 	__u64 extensions;
 
 	/**
-	 * @addr: user pointer address to wait on, must qword aligned
+	 * @addr: user pointer address to wait on, must be qword aligned
 	 */
 	__u64 addr;
 
@@ -1695,9 +1700,9 @@ struct drm_xe_wait_user_fence {
 	 * Without DRM_XE_UFENCE_WAIT_FLAG_ABSTIME flag set (relative timeout)
 	 * it contains timeout expressed in nanoseconds to wait (fence will
 	 * expire at now() + timeout).
-	 * When DRM_XE_UFENCE_WAIT_FLAG_ABSTIME flat is set (absolute timeout) wait
-	 * will end at timeout (uses system MONOTONIC_CLOCK).
-	 * Passing negative timeout leads to neverending wait.
+	 * When DRM_XE_UFENCE_WAIT_FLAG_ABSTIME flag is set (absolute timeout) wait
+	 * will end at timeout (uses system CLOCK_MONOTONIC).
+	 * Passing negative timeout leads to never ending wait.
 	 *
 	 * On relative timeout this value is updated with timeout left
 	 * (for restarting the call in case of signal delivery).
@@ -1741,7 +1746,7 @@ enum drm_xe_observation_op {
 };
 
 /**
- * struct drm_xe_observation_param - Input of &DRM_XE_OBSERVATION
+ * struct drm_xe_observation_param - Input of &DRM_IOCTL_XE_OBSERVATION
  *
  * The observation layer enables multiplexing observation streams of
  * multiple types. The actual params for a particular stream operation are
@@ -1751,25 +1756,25 @@ enum drm_xe_observation_op {
 struct drm_xe_observation_param {
 	/** @extensions: Pointer to the first extension struct, if any */
 	__u64 extensions;
-	/** @observation_type: observation stream type, of enum @drm_xe_observation_type */
+	/** @observation_type: observation stream type, of &enum drm_xe_observation_type */
 	__u64 observation_type;
-	/** @observation_op: observation stream op, of enum @drm_xe_observation_op */
+	/** @observation_op: observation stream op, of &enum drm_xe_observation_op */
 	__u64 observation_op;
 	/** @param: Pointer to actual stream params */
 	__u64 param;
 };
 
 /**
- * enum drm_xe_observation_ioctls - Observation stream fd ioctl's
+ * enum drm_xe_observation_ioctls - Observation stream fd ioctls
  *
  * Information exchanged between userspace and kernel for observation fd
- * ioctl's is stream type specific
+ * ioctls is stream type specific
  */
 enum drm_xe_observation_ioctls {
 	/** @DRM_XE_OBSERVATION_IOCTL_ENABLE: Enable data capture for an observation stream */
 	DRM_XE_OBSERVATION_IOCTL_ENABLE = _IO('i', 0x0),
 
-	/** @DRM_XE_OBSERVATION_IOCTL_DISABLE: Disable data capture for a observation stream */
+	/** @DRM_XE_OBSERVATION_IOCTL_DISABLE: Disable data capture for an observation stream */
 	DRM_XE_OBSERVATION_IOCTL_DISABLE = _IO('i', 0x1),
 
 	/** @DRM_XE_OBSERVATION_IOCTL_CONFIG: Change observation stream configuration */
@@ -1812,7 +1817,7 @@ struct drm_xe_oa_unit {
 	/** @oa_unit_id: OA unit ID */
 	__u32 oa_unit_id;
 
-	/** @oa_unit_type: OA unit type of @drm_xe_oa_unit_type */
+	/** @oa_unit_type: OA unit type of &enum drm_xe_oa_unit_type */
 	__u32 oa_unit_type;
 
 	/** @capabilities: OA capabilities bit-mask */
@@ -1875,7 +1880,7 @@ struct drm_xe_query_oa_units {
 	/** @pad: MBZ */
 	__u32 pad;
 	/**
-	 * @oa_units: struct @drm_xe_oa_unit array returned for this device.
+	 * @oa_units: &struct drm_xe_oa_unit array returned for this device.
 	 * Written below as a u64 array to avoid problems with nested flexible
 	 * arrays with some compilers
 	 */
@@ -1902,24 +1907,24 @@ enum drm_xe_oa_format_type {
 };
 
 /**
- * enum drm_xe_oa_property_id - OA stream property id's
+ * enum drm_xe_oa_property_id - OA stream property IDs
  *
- * Stream params are specified as a chain of @drm_xe_ext_set_property
- * struct's, with @property values from enum @drm_xe_oa_property_id and
- * @drm_xe_user_extension base.name set to @DRM_XE_OA_EXTENSION_SET_PROPERTY.
- * @param field in struct @drm_xe_observation_param points to the first
- * @drm_xe_ext_set_property struct.
+ * Stream params are specified as a chain of &struct drm_xe_ext_set_property
+ * structs, with property values from &enum drm_xe_oa_property_id and
+ * &struct drm_xe_user_extension base.name set to %DRM_XE_OA_EXTENSION_SET_PROPERTY.
+ * The param field in &struct drm_xe_observation_param points to the first
+ * &struct drm_xe_ext_set_property struct.
  *
  * Exactly the same mechanism is also used for stream reconfiguration using the
- * @DRM_XE_OBSERVATION_IOCTL_CONFIG observation stream fd ioctl, though only a
+ * %DRM_XE_OBSERVATION_IOCTL_CONFIG observation stream fd ioctl, though only a
  * subset of properties below can be specified for stream reconfiguration.
  */
 enum drm_xe_oa_property_id {
 #define DRM_XE_OA_EXTENSION_SET_PROPERTY	0
 	/**
 	 * @DRM_XE_OA_PROPERTY_OA_UNIT_ID: ID of the OA unit on which to open
-	 * the OA stream, see @oa_unit_id in 'struct
-	 * drm_xe_query_oa_units'. Defaults to 0 if not provided.
+	 * the OA stream, see oa_unit_id in &struct drm_xe_oa_unit.
+	 * Defaults to 0 if not provided.
 	 */
 	DRM_XE_OA_PROPERTY_OA_UNIT_ID = 1,
 
@@ -1932,7 +1937,7 @@ enum drm_xe_oa_property_id {
 
 	/**
 	 * @DRM_XE_OA_PROPERTY_OA_METRIC_SET: OA metrics defining contents of OA
-	 * reports, previously added via @DRM_XE_OBSERVATION_OP_ADD_CONFIG.
+	 * reports, previously added via %DRM_XE_OBSERVATION_OP_ADD_CONFIG.
 	 */
 	DRM_XE_OA_PROPERTY_OA_METRIC_SET,
 
@@ -1940,7 +1945,7 @@ enum drm_xe_oa_property_id {
 	DRM_XE_OA_PROPERTY_OA_FORMAT,
 	/*
 	 * OA_FORMAT's are specified the same way as in PRM/Bspec 52198/60942,
-	 * in terms of the following quantities: a. enum @drm_xe_oa_format_type
+	 * in terms of the following quantities: a. &enum drm_xe_oa_format_type
 	 * b. Counter select c. Counter size and d. BC report. Also refer to the
 	 * oa_formats array in drivers/gpu/drm/xe/xe_oa.c.
 	 */
@@ -1957,19 +1962,19 @@ enum drm_xe_oa_property_id {
 
 	/**
 	 * @DRM_XE_OA_PROPERTY_OA_DISABLED: A value of 1 will open the OA
-	 * stream in a DISABLED state (see @DRM_XE_OBSERVATION_IOCTL_ENABLE).
+	 * stream in a DISABLED state (see %DRM_XE_OBSERVATION_IOCTL_ENABLE).
 	 */
 	DRM_XE_OA_PROPERTY_OA_DISABLED,
 
 	/**
 	 * @DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID: Open the stream for a specific
-	 * @exec_queue_id. OA queries can be executed on this exec queue.
+	 * exec_queue_id. OA queries can be executed on this exec queue.
 	 */
 	DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID,
 
 	/**
 	 * @DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE: Optional engine instance to
-	 * pass along with @DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID or will default to 0.
+	 * pass along with %DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID or will default to 0.
 	 */
 	DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE,
 
@@ -1981,16 +1986,16 @@ enum drm_xe_oa_property_id {
 
 	/**
 	 * @DRM_XE_OA_PROPERTY_NUM_SYNCS: Number of syncs in the sync array
-	 * specified in @DRM_XE_OA_PROPERTY_SYNCS
+	 * specified in %DRM_XE_OA_PROPERTY_SYNCS
 	 */
 	DRM_XE_OA_PROPERTY_NUM_SYNCS,
 
 	/**
-	 * @DRM_XE_OA_PROPERTY_SYNCS: Pointer to struct @drm_xe_sync array
-	 * with array size specified via @DRM_XE_OA_PROPERTY_NUM_SYNCS. OA
+	 * @DRM_XE_OA_PROPERTY_SYNCS: Pointer to &struct drm_xe_sync array
+	 * with array size specified via %DRM_XE_OA_PROPERTY_NUM_SYNCS. OA
 	 * configuration will wait till input fences signal. Output fences
 	 * will signal after the new OA configuration takes effect. For
-	 * @DRM_XE_SYNC_TYPE_USER_FENCE, @addr is a user pointer, similar
+	 * %DRM_XE_SYNC_TYPE_USER_FENCE, addr is a user pointer, similar
 	 * to the VM bind case.
 	 */
 	DRM_XE_OA_PROPERTY_SYNCS,
@@ -2013,15 +2018,15 @@ enum drm_xe_oa_property_id {
 /**
  * struct drm_xe_oa_config - OA metric configuration
  *
- * Multiple OA configs can be added using @DRM_XE_OBSERVATION_OP_ADD_CONFIG. A
+ * Multiple OA configs can be added using %DRM_XE_OBSERVATION_OP_ADD_CONFIG. A
  * particular config can be specified when opening an OA stream using
- * @DRM_XE_OA_PROPERTY_OA_METRIC_SET property.
+ * %DRM_XE_OA_PROPERTY_OA_METRIC_SET property.
  */
 struct drm_xe_oa_config {
 	/** @extensions: Pointer to the first extension struct, if any */
 	__u64 extensions;
 
-	/** @uuid: String formatted like "%\08x-%\04x-%\04x-%\04x-%\012x" */
+	/** @uuid: String formatted like "%08x-%04x-%04x-%04x-%012x" */
 	char uuid[36];
 
 	/** @n_regs: Number of regs in @regs_ptr */
@@ -2036,7 +2041,7 @@ struct drm_xe_oa_config {
 
 /**
  * struct drm_xe_oa_stream_status - OA stream status returned from
- * @DRM_XE_OBSERVATION_IOCTL_STATUS observation stream fd ioctl. Userspace can
+ * %DRM_XE_OBSERVATION_IOCTL_STATUS observation stream fd ioctl. Userspace can
  * call the ioctl to query stream status in response to EIO errno from
  * observation fd read().
  */
@@ -2057,7 +2062,7 @@ struct drm_xe_oa_stream_status {
 
 /**
  * struct drm_xe_oa_stream_info - OA stream info returned from
- * @DRM_XE_OBSERVATION_IOCTL_INFO observation stream fd ioctl
+ * %DRM_XE_OBSERVATION_IOCTL_INFO observation stream fd ioctl
  */
 struct drm_xe_oa_stream_info {
 	/** @extensions: Pointer to the first extension struct, if any */
@@ -2094,27 +2099,27 @@ enum drm_xe_pxp_session_type {
  * enum drm_xe_eu_stall_property_id - EU stall sampling input property ids.
  *
  * These properties are passed to the driver at open as a chain of
- * @drm_xe_ext_set_property structures with @property set to these
- * properties' enums and @value set to the corresponding values of these
- * properties. @drm_xe_user_extension base.name should be set to
- * @DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY.
+ * &struct drm_xe_ext_set_property structures with property set to these
+ * properties' enums and value set to the corresponding values of these
+ * properties. &struct drm_xe_user_extension base.name should be set to
+ * %DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY.
  *
  * With the file descriptor obtained from open, user space must enable
- * the EU stall stream fd with @DRM_XE_OBSERVATION_IOCTL_ENABLE before
+ * the EU stall stream fd with %DRM_XE_OBSERVATION_IOCTL_ENABLE before
  * calling read(). EIO errno from read() indicates HW dropped data
  * due to full buffer.
  */
 enum drm_xe_eu_stall_property_id {
 #define DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY		0
 	/**
-	 * @DRM_XE_EU_STALL_PROP_GT_ID: @gt_id of the GT on which
+	 * @DRM_XE_EU_STALL_PROP_GT_ID: gt_id of the GT on which
 	 * EU stall data will be captured.
 	 */
 	DRM_XE_EU_STALL_PROP_GT_ID = 1,
 
 	/**
 	 * @DRM_XE_EU_STALL_PROP_SAMPLE_RATE: Sampling rate in
-	 * GPU cycles from @sampling_rates in struct @drm_xe_query_eu_stall
+	 * GPU cycles from sampling_rates in &struct drm_xe_query_eu_stall
 	 */
 	DRM_XE_EU_STALL_PROP_SAMPLE_RATE,
 
@@ -2129,9 +2134,9 @@ enum drm_xe_eu_stall_property_id {
 /**
  * struct drm_xe_query_eu_stall - Information about EU stall sampling.
  *
- * If a query is made with a struct @drm_xe_device_query where .query
- * is equal to @DRM_XE_DEVICE_QUERY_EU_STALL, then the reply uses
- * struct @drm_xe_query_eu_stall in .data.
+ * If a query is made with a &struct drm_xe_device_query where .query
+ * is equal to %DRM_XE_DEVICE_QUERY_EU_STALL, then the reply uses
+ * &struct drm_xe_query_eu_stall in .data.
  */
 struct drm_xe_query_eu_stall {
 	/** @extensions: Pointer to the first extension struct, if any */
@@ -2183,7 +2188,7 @@ struct drm_xe_query_eu_stall {
  *         .start = 0x100000,
  *         .range = 0x2000,
  *         .type = DRM_XE_MEM_RANGE_ATTR_ATOMIC,
- *         .atomic_val = DRM_XE_ATOMIC_DEVICE,
+ *         .atomic.val = DRM_XE_ATOMIC_DEVICE,
  *    };
  *
  *    ioctl(fd, DRM_IOCTL_XE_MADVISE, &madvise);
@@ -2242,7 +2247,7 @@ struct drm_xe_madvise {
 
 			/**
 			 * @preferred_mem_loc.region_instance : Region instance.
-			 * MBZ if @devmem_fd <= &DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE.
+			 * MBZ if @devmem_fd <= %DRM_XE_PREFERRED_LOC_DEFAULT_DEVICE.
 			 * Otherwise should point to the desired device
 			 * VRAM instance of the device indicated by
 			 * @preferred_mem_loc.devmem_fd.
@@ -2369,24 +2374,19 @@ struct drm_xe_madvise {
 };
 
 /**
- * struct drm_xe_mem_range_attr - Output of &DRM_IOCTL_XE_VM_QUERY_MEM_RANGES_ATTRS
+ * struct drm_xe_mem_range_attr - Output of &DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS
  *
  * This structure is provided by userspace and filled by KMD in response to the
- * DRM_IOCTL_XE_VM_QUERY_MEM_RANGES_ATTRS ioctl. It describes memory attributes of
- * a memory ranges within a user specified address range in a VM.
+ * DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS ioctl. It describes memory attributes of
+ * memory ranges within a user specified address range in a VM.
  *
  * The structure includes information such as atomic access policy,
  * page attribute table (PAT) index, and preferred memory location.
  * Userspace allocates an array of these structures and passes a pointer to the
- * ioctl to retrieve attributes for each memory ranges
- *
- * @extensions: Pointer to the first extension struct, if any
- * @start: Start address of the memory range
- * @end: End address of the virtual memory range
- *
+ * ioctl to retrieve attributes for each memory range.
  */
 struct drm_xe_mem_range_attr {
-	 /** @extensions: Pointer to the first extension struct, if any */
+	/** @extensions: Pointer to the first extension struct, if any */
 	__u64 extensions;
 
 	/** @start: start of the memory range */
@@ -2413,7 +2413,7 @@ struct drm_xe_mem_range_attr {
 		__u32 reserved;
 	} atomic;
 
-	 /** @pat_index: Page attribute table index */
+	/** @pat_index: Page attribute table index */
 	struct {
 		/** @pat_index.val: PAT index */
 		__u32 val;
@@ -2427,7 +2427,7 @@ struct drm_xe_mem_range_attr {
 };
 
 /**
- * struct drm_xe_vm_query_mem_range_attr - Input of &DRM_IOCTL_XE_VM_QUERY_MEM_ATTRIBUTES
+ * struct drm_xe_vm_query_mem_range_attr - Input of &DRM_IOCTL_XE_VM_QUERY_MEM_RANGE_ATTRS
  *
  * This structure is used to query memory attributes of memory regions
  * within a user specified address range in a VM. It provides detailed
@@ -2435,15 +2435,15 @@ struct drm_xe_mem_range_attr {
  * page attribute table (PAT) index, and preferred memory location.
  *
  * Userspace first calls the ioctl with @num_mem_ranges = 0,
- * @sizeof_mem_ranges_attr = 0 and @vector_of_vma_mem_attr = NULL to retrieve
+ * @sizeof_mem_range_attr = 0 and @vector_of_mem_attr = NULL to retrieve
  * the number of memory regions and size of each memory range attribute.
  * Then, it allocates a buffer of that size and calls the ioctl again to fill
  * the buffer with memory range attributes.
  *
  * If second call fails with -ENOSPC, it means memory ranges changed between
  * first call and now, retry IOCTL again with @num_mem_ranges = 0,
- * @sizeof_mem_ranges_attr = 0 and @vector_of_vma_mem_attr = NULL followed by
- * Second ioctl call.
+ * @sizeof_mem_range_attr = 0 and @vector_of_mem_attr = NULL followed by
+ * second ioctl call.
  *
  * Example:
  *
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index 552bc5d9afbd1..aec171ccb6efd 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -994,6 +994,7 @@ enum bpf_cmd {
 	BPF_PROG_STREAM_READ_BY_FD,
 	BPF_PROG_ASSOC_STRUCT_OPS,
 	__MAX_BPF_CMD,
+	BPF_COMMON_ATTRS = 1 << 16, /* Indicate carrying syscall common attrs. */
 };
 
 enum bpf_map_type {
@@ -1500,6 +1501,13 @@ struct bpf_stack_build_id {
 	};
 };
 
+struct bpf_common_attr {
+	__aligned_u64 log_buf;
+	__u32 log_size;
+	__u32 log_level;
+	__u32 log_true_size;
+};
+
 #define BPF_OBJ_NAME_LEN 16U
 
 enum {
diff --git a/include/uapi/linux/btf.h b/include/uapi/linux/btf.h
index 638615ebddc22..618167cab4e61 100644
--- a/include/uapi/linux/btf.h
+++ b/include/uapi/linux/btf.h
@@ -33,20 +33,22 @@ struct btf_header {
 	__u32	layout_len;	/* length of layout section	*/
 };
 
-/* Max # of type identifier */
-#define BTF_MAX_TYPE	0x000fffff
-/* Max offset into the string section */
-#define BTF_MAX_NAME_OFFSET	0x00ffffff
-/* Max # of struct/union/enum members or func args */
-#define BTF_MAX_VLEN	0xffff
+enum btf_max {
+	/* Max possible kind */
+	BTF_MAX_KIND =		0x0000007f,
+	/* Max # of type identifier */
+	BTF_MAX_TYPE =		0x000fffff,
+	/* Max offset into the string section */
+	BTF_MAX_NAME_OFFSET =	0x00ffffff,
+	/* Max # of struct/union/enum members or func args */
+	BTF_MAX_VLEN =		0x00ffffff,
+};
 
 struct btf_type {
 	__u32 name_off;
 	/* "info" bits arrangement
-	 * bits  0-15: vlen (e.g. # of struct's members)
-	 * bits 16-23: unused
-	 * bits 24-28: kind (e.g. int, ptr, array...etc)
-	 * bits 29-30: unused
+	 * bits  0-23: vlen (e.g. # of struct's members)
+	 * bits 24-30: kind (e.g. int, ptr, array...etc)
 	 * bit     31: kind_flag, currently used by
 	 *             struct, union, enum, fwd, enum64,
 	 *             decl_tag and type_tag
@@ -65,8 +67,8 @@ struct btf_type {
 	};
 };
 
-#define BTF_INFO_KIND(info)	(((info) >> 24) & 0x1f)
-#define BTF_INFO_VLEN(info)	((info) & 0xffff)
+#define BTF_INFO_KIND(info)	(((info) >> 24) & 0x7f)
+#define BTF_INFO_VLEN(info)	((info) & 0xffffff)
 #define BTF_INFO_KFLAG(info)	((info) >> 31)
 
 enum {
diff --git a/include/uapi/linux/cec-funcs.h b/include/uapi/linux/cec-funcs.h
index 189ecf0e13cdb..ba4b47de9bf02 100644
--- a/include/uapi/linux/cec-funcs.h
+++ b/include/uapi/linux/cec-funcs.h
@@ -1701,6 +1701,188 @@ static inline void cec_ops_request_current_latency(const struct cec_msg *msg,
 }
 
 
+/* Latency Indication Protocol Feature */
+/* Only for CEC 2.0 and up */
+static inline void cec_msg_request_lip_support(struct cec_msg *msg,
+					       int reply, __u16 phys_addr)
+{
+	msg->len = 4;
+	msg->msg[1] = CEC_MSG_REQUEST_LIP_SUPPORT;
+	msg->msg[2] = phys_addr >> 8;
+	msg->msg[3] = phys_addr & 0xff;
+	msg->reply = reply ? CEC_MSG_REPORT_LIP_SUPPORT : 0;
+}
+
+static inline void cec_ops_request_lip_support(const struct cec_msg *msg,
+					       __u16 *phys_addr)
+{
+	*phys_addr = (msg->msg[2] << 8) | msg->msg[3];
+}
+
+static inline void cec_msg_report_lip_support(struct cec_msg *msg, __u32 sqid)
+{
+	msg->len = 6;
+	msg->msg[1] = CEC_MSG_REPORT_LIP_SUPPORT;
+	msg->msg[2] = sqid >> 24;
+	msg->msg[3] = (sqid >> 16) & 0xff;
+	msg->msg[4] = (sqid >> 8) & 0xff;
+	msg->msg[5] = sqid & 0xff;
+}
+
+static inline void cec_ops_report_lip_support(const struct cec_msg *msg,
+					      __u32 *sqid)
+{
+	*sqid = (msg->msg[2] << 24) | (msg->msg[3] << 16) |
+		(msg->msg[4] << 8) | msg->msg[5];
+}
+
+static inline void cec_msg_request_audio_and_video_latency(struct cec_msg *msg,
+					       int reply, __u8 video_format,
+					       __u8 hdr_format, __u8 vrr_format,
+					       __u8 audio_format,
+					       __u8 audio_format_extension)
+{
+	msg->len = 6;
+	msg->msg[1] = CEC_MSG_REQUEST_AUDIO_AND_VIDEO_LATENCY;
+	msg->msg[2] = video_format;
+	msg->msg[3] = hdr_format;
+	msg->msg[4] = vrr_format;
+	msg->msg[5] = audio_format;
+	if (audio_format >= 1 && audio_format <= 31) {
+		msg->msg[6] = audio_format_extension;
+		msg->len++;
+	}
+	msg->reply = reply ? CEC_MSG_REPORT_AUDIO_AND_VIDEO_LATENCY : 0;
+}
+
+static inline void cec_ops_request_audio_and_video_latency(const struct cec_msg *msg,
+					       __u8 *video_format,
+					       __u8 *hdr_format,
+					       __u8 *vrr_format,
+					       __u8 *audio_format,
+					       __u8 *audio_format_extension)
+{
+	*video_format = msg->msg[2];
+	*hdr_format = msg->msg[3];
+	*vrr_format = msg->msg[4];
+	*audio_format = msg->msg[5];
+	*audio_format_extension = msg->len > 6 ? msg->msg[6] : 0;
+}
+
+static inline void cec_msg_report_audio_and_video_latency(struct cec_msg *msg,
+							  __u16 video_latency,
+							  __u16 audio_latency)
+{
+	msg->len = 6;
+	msg->msg[1] = CEC_MSG_REPORT_AUDIO_AND_VIDEO_LATENCY;
+	msg->msg[2] = video_latency >> 8;
+	msg->msg[3] = video_latency & 0xff;
+	msg->msg[4] = audio_latency >> 8;
+	msg->msg[5] = audio_latency & 0xff;
+}
+
+static inline void cec_ops_report_audio_and_video_latency(const struct cec_msg *msg,
+					      __u16 *video_latency,
+					      __u16 *audio_latency)
+{
+	*video_latency = (msg->msg[2] << 8) | msg->msg[3];
+	*audio_latency = (msg->msg[4] << 8) | msg->msg[5];
+}
+
+static inline void cec_msg_request_audio_latency(struct cec_msg *msg,
+						 int reply,
+						 __u8 audio_format,
+						 __u8 audio_format_extension)
+{
+	msg->len = 3;
+	msg->msg[1] = CEC_MSG_REQUEST_AUDIO_LATENCY;
+	msg->msg[2] = audio_format;
+	if (audio_format >= 1 && audio_format <= 31) {
+		msg->msg[3] = audio_format_extension;
+		msg->len++;
+	}
+	msg->reply = reply ? CEC_MSG_REPORT_AUDIO_LATENCY : 0;
+}
+
+static inline void cec_ops_request_audio_latency(const struct cec_msg *msg,
+						 __u8 *audio_format,
+						 __u8 *audio_format_extension)
+{
+	*audio_format = msg->msg[2];
+	*audio_format_extension = msg->len > 3 ? msg->msg[3] : 0;
+}
+
+static inline void cec_msg_report_audio_latency(struct cec_msg *msg,
+						__u16 audio_latency)
+{
+	msg->len = 4;
+	msg->msg[1] = CEC_MSG_REPORT_AUDIO_LATENCY;
+	msg->msg[2] = audio_latency >> 8;
+	msg->msg[3] = audio_latency & 0xff;
+}
+
+static inline void cec_ops_report_audio_latency(const struct cec_msg *msg,
+						__u16 *audio_latency)
+{
+	*audio_latency = (msg->msg[2] << 8) | msg->msg[3];
+}
+
+static inline void cec_msg_request_video_latency(struct cec_msg *msg,
+						 int reply, __u8 video_format,
+						 __u8 hdr_format,
+						 __u8 vrr_format)
+{
+	msg->len = 5;
+	msg->msg[1] = CEC_MSG_REQUEST_VIDEO_LATENCY;
+	msg->msg[2] = video_format;
+	msg->msg[3] = hdr_format;
+	msg->msg[4] = vrr_format;
+	msg->reply = reply ? CEC_MSG_REPORT_VIDEO_LATENCY : 0;
+}
+
+static inline void cec_ops_request_video_latency(const struct cec_msg *msg,
+						 __u8 *video_format,
+						 __u8 *hdr_format,
+						 __u8 *vrr_format)
+{
+	*video_format = msg->msg[2];
+	*hdr_format = msg->msg[3];
+	*vrr_format = msg->msg[4];
+}
+
+static inline void cec_msg_report_video_latency(struct cec_msg *msg,
+						__u16 video_latency)
+{
+	msg->len = 4;
+	msg->msg[1] = CEC_MSG_REPORT_VIDEO_LATENCY;
+	msg->msg[2] = video_latency >> 8;
+	msg->msg[3] = video_latency & 0xff;
+}
+
+static inline void cec_ops_report_video_latency(const struct cec_msg *msg,
+						__u16 *video_latency)
+{
+	*video_latency = (msg->msg[2] << 8) | msg->msg[3];
+}
+
+static inline void cec_msg_update_sqid(struct cec_msg *msg, __u32 sqid)
+{
+	msg->len = 6;
+	msg->msg[1] = CEC_MSG_UPDATE_SQID;
+	msg->msg[2] = sqid >> 24;
+	msg->msg[3] = (sqid >> 16) & 0xff;
+	msg->msg[4] = (sqid >> 8) & 0xff;
+	msg->msg[5] = sqid & 0xff;
+}
+
+static inline void cec_ops_update_sqid(const struct cec_msg *msg,
+				       __u32 *sqid)
+{
+	*sqid = (msg->msg[2] << 24) | (msg->msg[3] << 16) |
+		(msg->msg[4] << 8) | msg->msg[5];
+}
+
+
 /* Capability Discovery and Control Feature */
 static inline void cec_msg_cdc_hec_inquire_state(struct cec_msg *msg,
 						 __u16 phys_addr1,
diff --git a/include/uapi/linux/cec.h b/include/uapi/linux/cec.h
index b2af1dddd4d7b..81a05c9c07060 100644
--- a/include/uapi/linux/cec.h
+++ b/include/uapi/linux/cec.h
@@ -742,7 +742,7 @@ struct cec_event {
 #define CEC_OP_PRIM_DEVTYPE_PROCESSOR			7
 
 #define CEC_MSG_SET_MENU_LANGUAGE			0x32
-#define CEC_MSG_REPORT_FEATURES				0xa6	/* HDMI 2.0 */
+#define CEC_MSG_REPORT_FEATURES				0xa6	/* CEC 2.0 */
 /* All Device Types Operand (all_device_types) */
 #define CEC_OP_ALL_DEVTYPE_TV				0x80
 #define CEC_OP_ALL_DEVTYPE_RECORD			0x40
@@ -777,7 +777,7 @@ struct cec_event {
 #define CEC_OP_FEAT_DEV_SOURCE_HAS_ARC_RX		0x02
 #define CEC_OP_FEAT_DEV_HAS_SET_AUDIO_VOLUME_LEVEL	0x01
 
-#define CEC_MSG_GIVE_FEATURES				0xa5	/* HDMI 2.0 */
+#define CEC_MSG_GIVE_FEATURES				0xa5	/* CEC 2.0 */
 
 
 /* Deck Control Feature */
@@ -1067,7 +1067,7 @@ struct cec_event {
 #define CEC_OP_AUD_FMT_ID_CEA861			0
 #define CEC_OP_AUD_FMT_ID_CEA861_CXT			1
 
-#define CEC_MSG_SET_AUDIO_VOLUME_LEVEL			0x73
+#define CEC_MSG_SET_AUDIO_VOLUME_LEVEL			0x73	/* CEC 2.0 */
 
 /* Audio Rate Control Feature */
 #define CEC_MSG_SET_AUDIO_RATE				0x9a
@@ -1091,7 +1091,6 @@ struct cec_event {
 
 
 /* Dynamic Audio Lipsync Feature */
-/* Only for CEC 2.0 and up */
 #define CEC_MSG_REQUEST_CURRENT_LATENCY			0xa7
 #define CEC_MSG_REPORT_CURRENT_LATENCY			0xa8
 /* Low Latency Mode Operand (low_latency_mode) */
@@ -1104,6 +1103,30 @@ struct cec_event {
 #define CEC_OP_AUD_OUT_COMPENSATED_PARTIAL_DELAY	3
 
 
+/* Latency Indication Protocol Feature */
+#define CEC_MSG_REQUEST_LIP_SUPPORT			0x50	/* CEC 2.0 */
+#define CEC_MSG_REPORT_LIP_SUPPORT			0x51	/* CEC 2.0 */
+#define CEC_MSG_REQUEST_AUDIO_AND_VIDEO_LATENCY		0x52	/* CEC 2.0 */
+/* HDR Format Operand (hdr_format) */
+#define CEC_OP_HDR_FORMAT_GAMMA_SDR			0
+#define CEC_OP_HDR_FORMAT_GAMMA_HDR			1
+#define CEC_OP_HDR_FORMAT_PQ				2
+#define CEC_OP_HDR_FORMAT_HLG				3
+#define CEC_OP_HDR_FORMAT_DYNAMIC_HDR_TYPE_1		8
+#define CEC_OP_HDR_FORMAT_DYNAMIC_HDR_TYPE_2		9
+#define CEC_OP_HDR_FORMAT_DYNAMIC_HDR_TYPE_4		11
+#define CEC_OP_HDR_FORMAT_DV_SINK_LED			16
+#define CEC_OP_HDR_FORMAT_DV_SOURCE_LED			17
+#define CEC_OP_HDR_FORMAT_HDR10PLUS			24
+#define CEC_OP_HDR_FORMAT_ETSI_TS_103_433		32
+#define CEC_MSG_REPORT_AUDIO_AND_VIDEO_LATENCY		0x53	/* CEC 2.0 */
+#define CEC_MSG_REQUEST_AUDIO_LATENCY			0x54	/* CEC 2.0 */
+#define CEC_MSG_REPORT_AUDIO_LATENCY			0x55	/* CEC 2.0 */
+#define CEC_MSG_REQUEST_VIDEO_LATENCY			0x56	/* CEC 2.0 */
+#define CEC_MSG_REPORT_VIDEO_LATENCY			0x57	/* CEC 2.0 */
+#define CEC_MSG_UPDATE_SQID				0x58	/* CEC 2.0 */
+
+
 /* Capability Discovery and Control Feature */
 #define CEC_MSG_CDC_MESSAGE				0xf8
 /* Ethernet-over-HDMI: nobody ever does this... */
diff --git a/include/uapi/linux/dpll.h b/include/uapi/linux/dpll.h
index 871685f7c353b..cb363cccf2e2a 100644
--- a/include/uapi/linux/dpll.h
+++ b/include/uapi/linux/dpll.h
@@ -178,6 +178,28 @@ enum dpll_pin_state {
 	DPLL_PIN_STATE_MAX = (__DPLL_PIN_STATE_MAX - 1)
 };
 
+/**
+ * enum dpll_pin_operstate - defines possible operational states of a pin with
+ *   respect to its parent DPLL device, valid values for DPLL_A_PIN_OPERSTATE
+ *   attribute
+ * @DPLL_PIN_OPERSTATE_ACTIVE: pin is qualified and actively used by the DPLL
+ * @DPLL_PIN_OPERSTATE_STANDBY: pin is qualified but not actively used by the
+ *   DPLL
+ * @DPLL_PIN_OPERSTATE_NO_SIGNAL: pin does not have a valid signal
+ * @DPLL_PIN_OPERSTATE_QUAL_FAILED: pin signal failed qualification (e.g.
+ *   frequency or phase monitor)
+ */
+enum dpll_pin_operstate {
+	DPLL_PIN_OPERSTATE_ACTIVE = 1,
+	DPLL_PIN_OPERSTATE_STANDBY,
+	DPLL_PIN_OPERSTATE_NO_SIGNAL,
+	DPLL_PIN_OPERSTATE_QUAL_FAILED,
+
+	/* private: */
+	__DPLL_PIN_OPERSTATE_MAX,
+	DPLL_PIN_OPERSTATE_MAX = (__DPLL_PIN_OPERSTATE_MAX - 1)
+};
+
 /**
  * enum dpll_pin_capabilities - defines possible capabilities of a pin, valid
  *   flags on DPLL_A_PIN_CAPABILITIES attribute
@@ -257,6 +279,7 @@ enum dpll_a_pin {
 	DPLL_A_PIN_PHASE_ADJUST_GRAN,
 	DPLL_A_PIN_FRACTIONAL_FREQUENCY_OFFSET_PPT,
 	DPLL_A_PIN_MEASURED_FREQUENCY,
+	DPLL_A_PIN_OPERSTATE,
 
 	__DPLL_A_PIN_MAX,
 	DPLL_A_PIN_MAX = (__DPLL_A_PIN_MAX - 1)
diff --git a/include/uapi/linux/drbd.h b/include/uapi/linux/drbd.h
new file mode 100644
index 0000000000000..cf1ec3eb872f0
--- /dev/null
+++ b/include/uapi/linux/drbd.h
@@ -0,0 +1,413 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later WITH Linux-syscall-note */
+/*
+  drbd.h
+  Kernel module for 2.6.x Kernels
+
+  This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
+
+  Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
+  Copyright (C) 2001-2008, Philipp Reisner <philipp.reisner@linbit.com>.
+  Copyright (C) 2001-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
+
+
+*/
+#ifndef _UAPI_LINUX_DRBD_H
+#define _UAPI_LINUX_DRBD_H
+#include <linux/types.h>
+#include <asm/byteorder.h>
+
+enum drbd_io_error_p {
+	EP_PASS_ON, /* FIXME should the better be named "Ignore"? */
+	EP_CALL_HELPER,
+	EP_DETACH
+};
+
+enum drbd_fencing_p {
+	FP_NOT_AVAIL = -1, /* Not a policy */
+	FP_DONT_CARE = 0,
+	FP_RESOURCE,
+	FP_STONITH
+};
+
+enum drbd_disconnect_p {
+	DP_RECONNECT,
+	DP_DROP_NET_CONF,
+	DP_FREEZE_IO
+};
+
+enum drbd_after_sb_p {
+	ASB_DISCONNECT,
+	ASB_DISCARD_YOUNGER_PRI,
+	ASB_DISCARD_OLDER_PRI,
+	ASB_DISCARD_ZERO_CHG,
+	ASB_DISCARD_LEAST_CHG,
+	ASB_DISCARD_LOCAL,
+	ASB_DISCARD_REMOTE,
+	ASB_CONSENSUS,
+	ASB_DISCARD_SECONDARY,
+	ASB_CALL_HELPER,
+	ASB_VIOLENTLY
+};
+
+enum drbd_on_no_data {
+	OND_IO_ERROR,
+	OND_SUSPEND_IO
+};
+
+enum drbd_on_congestion {
+	OC_BLOCK,
+	OC_PULL_AHEAD,
+	OC_DISCONNECT,
+};
+
+enum drbd_read_balancing {
+	RB_PREFER_LOCAL,
+	RB_PREFER_REMOTE,
+	RB_ROUND_ROBIN,
+	RB_LEAST_PENDING,
+	RB_CONGESTED_REMOTE,
+	RB_32K_STRIPING,
+	RB_64K_STRIPING,
+	RB_128K_STRIPING,
+	RB_256K_STRIPING,
+	RB_512K_STRIPING,
+	RB_1M_STRIPING,
+};
+
+/* KEEP the order, do not delete or insert. Only append. */
+enum drbd_ret_code {
+	ERR_CODE_BASE		= 100,
+	NO_ERROR		= 101,
+	ERR_LOCAL_ADDR		= 102,
+	ERR_PEER_ADDR		= 103,
+	ERR_OPEN_DISK		= 104,
+	ERR_OPEN_MD_DISK	= 105,
+	ERR_DISK_NOT_BDEV	= 107,
+	ERR_MD_NOT_BDEV		= 108,
+	ERR_DISK_TOO_SMALL	= 111,
+	ERR_MD_DISK_TOO_SMALL	= 112,
+	ERR_BDCLAIM_DISK	= 114,
+	ERR_BDCLAIM_MD_DISK	= 115,
+	ERR_MD_IDX_INVALID	= 116,
+	ERR_IO_MD_DISK		= 118,
+	ERR_MD_INVALID          = 119,
+	ERR_AUTH_ALG		= 120,
+	ERR_AUTH_ALG_ND		= 121,
+	ERR_NOMEM		= 122,
+	ERR_DISCARD_IMPOSSIBLE	= 123,
+	ERR_DISK_CONFIGURED	= 124,
+	ERR_NET_CONFIGURED	= 125,
+	ERR_MANDATORY_TAG	= 126,
+	ERR_MINOR_INVALID	= 127,
+	ERR_INTR		= 129, /* EINTR */
+	ERR_RESIZE_RESYNC	= 130,
+	ERR_NO_PRIMARY		= 131,
+	ERR_RESYNC_AFTER	= 132,
+	ERR_RESYNC_AFTER_CYCLE	= 133,
+	ERR_PAUSE_IS_SET	= 134,
+	ERR_PAUSE_IS_CLEAR	= 135,
+	ERR_PACKET_NR		= 137,
+	ERR_NO_DISK		= 138,
+	ERR_NOT_PROTO_C		= 139,
+	ERR_NOMEM_BITMAP	= 140,
+	ERR_INTEGRITY_ALG	= 141, /* DRBD 8.2 only */
+	ERR_INTEGRITY_ALG_ND	= 142, /* DRBD 8.2 only */
+	ERR_CPU_MASK_PARSE	= 143, /* DRBD 8.2 only */
+	ERR_CSUMS_ALG		= 144, /* DRBD 8.2 only */
+	ERR_CSUMS_ALG_ND	= 145, /* DRBD 8.2 only */
+	ERR_VERIFY_ALG		= 146, /* DRBD 8.2 only */
+	ERR_VERIFY_ALG_ND	= 147, /* DRBD 8.2 only */
+	ERR_CSUMS_RESYNC_RUNNING= 148, /* DRBD 8.2 only */
+	ERR_VERIFY_RUNNING	= 149, /* DRBD 8.2 only */
+	ERR_DATA_NOT_CURRENT	= 150,
+	ERR_CONNECTED		= 151, /* DRBD 8.3 only */
+	ERR_PERM		= 152,
+	ERR_NEED_APV_93		= 153,
+	ERR_STONITH_AND_PROT_A  = 154,
+	ERR_CONG_NOT_PROTO_A	= 155,
+	ERR_PIC_AFTER_DEP	= 156,
+	ERR_PIC_PEER_DEP	= 157,
+	ERR_RES_NOT_KNOWN	= 158,
+	ERR_RES_IN_USE		= 159,
+	ERR_MINOR_CONFIGURED    = 160,
+	ERR_MINOR_OR_VOLUME_EXISTS = 161,
+	ERR_INVALID_REQUEST	= 162,
+	ERR_NEED_APV_100	= 163,
+	ERR_NEED_ALLOW_TWO_PRI  = 164,
+	ERR_MD_UNCLEAN          = 165,
+	ERR_MD_LAYOUT_CONNECTED = 166,
+	ERR_MD_LAYOUT_TOO_BIG   = 167,
+	ERR_MD_LAYOUT_TOO_SMALL = 168,
+	ERR_MD_LAYOUT_NO_FIT    = 169,
+	ERR_IMPLICIT_SHRINK     = 170,
+	/* insert new ones above this line */
+	AFTER_LAST_ERR_CODE
+};
+
+#define DRBD_PROT_A   1
+#define DRBD_PROT_B   2
+#define DRBD_PROT_C   3
+
+enum drbd_role {
+	R_UNKNOWN = 0,
+	R_PRIMARY = 1,     /* role */
+	R_SECONDARY = 2,   /* role */
+	R_MASK = 3,
+};
+
+/* The order of these constants is important.
+ * The lower ones (<C_WF_REPORT_PARAMS) indicate
+ * that there is no socket!
+ * >=C_WF_REPORT_PARAMS ==> There is a socket
+ */
+enum drbd_conns {
+	C_STANDALONE,
+	C_DISCONNECTING,  /* Temporal state on the way to StandAlone. */
+	C_UNCONNECTED,    /* >= C_UNCONNECTED -> inc_net() succeeds */
+
+	/* These temporal states are all used on the way
+	 * from >= C_CONNECTED to Unconnected.
+	 * The 'disconnect reason' states
+	 * I do not allow to change between them. */
+	C_TIMEOUT,
+	C_BROKEN_PIPE,
+	C_NETWORK_FAILURE,
+	C_PROTOCOL_ERROR,
+	C_TEAR_DOWN,
+
+	C_WF_CONNECTION,
+	C_WF_REPORT_PARAMS, /* we have a socket */
+	C_CONNECTED,      /* we have introduced each other */
+	C_STARTING_SYNC_S,  /* starting full sync by admin request. */
+	C_STARTING_SYNC_T,  /* starting full sync by admin request. */
+	C_WF_BITMAP_S,
+	C_WF_BITMAP_T,
+	C_WF_SYNC_UUID,
+
+	/* All SyncStates are tested with this comparison
+	 * xx >= C_SYNC_SOURCE && xx <= C_PAUSED_SYNC_T */
+	C_SYNC_SOURCE,
+	C_SYNC_TARGET,
+	C_VERIFY_S,
+	C_VERIFY_T,
+	C_PAUSED_SYNC_S,
+	C_PAUSED_SYNC_T,
+
+	C_AHEAD,
+	C_BEHIND,
+
+	C_MASK = 31
+};
+
+enum drbd_disk_state {
+	D_DISKLESS,
+	D_ATTACHING,      /* In the process of reading the meta-data */
+	D_FAILED,         /* Becomes D_DISKLESS as soon as we told it the peer */
+			  /* when >= D_FAILED it is legal to access mdev->ldev */
+	D_NEGOTIATING,    /* Late attaching state, we need to talk to the peer */
+	D_INCONSISTENT,
+	D_OUTDATED,
+	D_UNKNOWN,       /* Only used for the peer, never for myself */
+	D_CONSISTENT,     /* Might be D_OUTDATED, might be D_UP_TO_DATE ... */
+	D_UP_TO_DATE,       /* Only this disk state allows applications' IO ! */
+	D_MASK = 15
+};
+
+union drbd_state {
+/* According to gcc's docs is the ...
+ * The order of allocation of bit-fields within a unit (C90 6.5.2.1, C99 6.7.2.1).
+ * Determined by ABI.
+ * pointed out by Maxim Uvarov q<muvarov@ru.mvista.com>
+ * even though we transmit as "cpu_to_be32(state)",
+ * the offsets of the bitfields still need to be swapped
+ * on different endianness.
+ */
+	struct {
+#if defined(__LITTLE_ENDIAN_BITFIELD)
+		unsigned role:2 ;   /* 3/4	 primary/secondary/unknown */
+		unsigned peer:2 ;   /* 3/4	 primary/secondary/unknown */
+		unsigned conn:5 ;   /* 17/32	 cstates */
+		unsigned disk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
+		unsigned pdsk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
+		unsigned susp:1 ;   /* 2/2	 IO suspended no/yes (by user) */
+		unsigned aftr_isp:1 ; /* isp .. imposed sync pause */
+		unsigned peer_isp:1 ;
+		unsigned user_isp:1 ;
+		unsigned susp_nod:1 ; /* IO suspended because no data */
+		unsigned susp_fen:1 ; /* IO suspended because fence peer handler runs*/
+		unsigned _pad:9;   /* 0	 unused */
+#elif defined(__BIG_ENDIAN_BITFIELD)
+		unsigned _pad:9;
+		unsigned susp_fen:1 ;
+		unsigned susp_nod:1 ;
+		unsigned user_isp:1 ;
+		unsigned peer_isp:1 ;
+		unsigned aftr_isp:1 ; /* isp .. imposed sync pause */
+		unsigned susp:1 ;   /* 2/2	 IO suspended  no/yes */
+		unsigned pdsk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
+		unsigned disk:4 ;   /* 8/16	 from D_DISKLESS to D_UP_TO_DATE */
+		unsigned conn:5 ;   /* 17/32	 cstates */
+		unsigned peer:2 ;   /* 3/4	 primary/secondary/unknown */
+		unsigned role:2 ;   /* 3/4	 primary/secondary/unknown */
+#else
+# error "this endianness is not supported"
+#endif
+	};
+	unsigned int i;
+};
+
+enum drbd_state_rv {
+	SS_CW_NO_NEED = 4,
+	SS_CW_SUCCESS = 3,
+	SS_NOTHING_TO_DO = 2,
+	SS_SUCCESS = 1,
+	SS_UNKNOWN_ERROR = 0, /* Used to sleep longer in _drbd_request_state */
+	SS_TWO_PRIMARIES = -1,
+	SS_NO_UP_TO_DATE_DISK = -2,
+	SS_NO_LOCAL_DISK = -4,
+	SS_NO_REMOTE_DISK = -5,
+	SS_CONNECTED_OUTDATES = -6,
+	SS_PRIMARY_NOP = -7,
+	SS_RESYNC_RUNNING = -8,
+	SS_ALREADY_STANDALONE = -9,
+	SS_CW_FAILED_BY_PEER = -10,
+	SS_IS_DISKLESS = -11,
+	SS_DEVICE_IN_USE = -12,
+	SS_NO_NET_CONFIG = -13,
+	SS_NO_VERIFY_ALG = -14,       /* drbd-8.2 only */
+	SS_NEED_CONNECTION = -15,    /* drbd-8.2 only */
+	SS_LOWER_THAN_OUTDATED = -16,
+	SS_NOT_SUPPORTED = -17,      /* drbd-8.2 only */
+	SS_IN_TRANSIENT_STATE = -18,  /* Retry after the next state change */
+	SS_CONCURRENT_ST_CHG = -19,   /* Concurrent cluster side state change! */
+	SS_O_VOL_PEER_PRI = -20,
+	SS_OUTDATE_WO_CONN = -21,
+	SS_AFTER_LAST_ERROR = -22,    /* Keep this at bottom */
+};
+
+#define SHARED_SECRET_MAX 64
+
+#define MDF_CONSISTENT		(1 << 0)
+#define MDF_PRIMARY_IND		(1 << 1)
+#define MDF_CONNECTED_IND	(1 << 2)
+#define MDF_FULL_SYNC		(1 << 3)
+#define MDF_WAS_UP_TO_DATE	(1 << 4)
+#define MDF_PEER_OUT_DATED	(1 << 5)
+#define MDF_CRASHED_PRIMARY	(1 << 6)
+#define MDF_AL_CLEAN		(1 << 7)
+#define MDF_AL_DISABLED		(1 << 8)
+
+#define MAX_PEERS 32
+
+enum drbd_uuid_index {
+	UI_CURRENT,
+	UI_BITMAP,
+	UI_HISTORY_START,
+	UI_HISTORY_END,
+	UI_SIZE,      /* nl-packet: number of dirty bits */
+	UI_FLAGS,     /* nl-packet: flags */
+	UI_EXTENDED_SIZE   /* Everything. */
+};
+
+#define HISTORY_UUIDS MAX_PEERS
+
+#define DRBD_NL_UUIDS_SIZE		(UI_SIZE * sizeof(__u64))
+#define DRBD_NL_HISTORY_UUIDS_SIZE	(HISTORY_UUIDS * sizeof(__u64))
+
+enum drbd_timeout_flag {
+	UT_DEFAULT      = 0,
+	UT_DEGRADED     = 1,
+	UT_PEER_OUTDATED = 2,
+};
+
+enum drbd_notification_type {
+	NOTIFY_EXISTS,
+	NOTIFY_CREATE,
+	NOTIFY_CHANGE,
+	NOTIFY_DESTROY,
+	NOTIFY_CALL,
+	NOTIFY_RESPONSE,
+
+	NOTIFY_CONTINUES = 0x8000,
+	NOTIFY_FLAGS = NOTIFY_CONTINUES,
+};
+
+enum drbd_peer_state {
+	P_INCONSISTENT = 3,
+	P_OUTDATED = 4,
+	P_DOWN = 5,
+	P_PRIMARY = 6,
+	P_FENCING = 7,
+};
+
+#define UUID_JUST_CREATED ((__u64)4)
+
+enum write_ordering_e {
+	WO_NONE,
+	WO_DRAIN_IO,
+	WO_BDEV_FLUSH,
+	WO_BIO_BARRIER
+};
+
+/* magic numbers used in meta data and network packets */
+#define DRBD_MAGIC 0x83740267
+#define DRBD_MAGIC_BIG 0x835a
+#define DRBD_MAGIC_100 0x8620ec20
+
+#define DRBD_MD_MAGIC_07   (DRBD_MAGIC+3)
+#define DRBD_MD_MAGIC_08   (DRBD_MAGIC+4)
+#define DRBD_MD_MAGIC_84_UNCLEAN	(DRBD_MAGIC+5)
+
+
+/* how I came up with this magic?
+ * base64 decode "actlog==" ;) */
+#define DRBD_AL_MAGIC 0x69cb65a2
+
+/* these are of type "int" */
+#define DRBD_MD_INDEX_INTERNAL -1
+#define DRBD_MD_INDEX_FLEX_EXT -2
+#define DRBD_MD_INDEX_FLEX_INT -3
+
+#define DRBD_CPU_MASK_SIZE 32
+
+/**
+ * struct drbd_genlmsghdr - DRBD specific header used in NETLINK_GENERIC requests
+ * @minor:
+ *     For admin requests (user -> kernel): which minor device to operate on.
+ *     For (unicast) replies or informational (broadcast) messages
+ *     (kernel -> user): which minor device the information is about.
+ *     If we do not operate on minors, but on connections or resources,
+ *     the minor value shall be (~0), and the attribute DRBD_NLA_CFG_CONTEXT
+ *     is used instead.
+ * @flags: possible operation modifiers (relevant only for user->kernel):
+ *     DRBD_GENL_F_SET_DEFAULTS
+ * @volume:
+ *     When creating a new minor (adding it to a resource), the resource needs
+ *     to know which volume number within the resource this is supposed to be.
+ *     The volume number corresponds to the same volume number on the remote side,
+ *     whereas the minor number on the remote side may be different
+ *     (union with flags).
+ * @ret_code: kernel->userland unicast cfg reply return code (union with flags);
+ */
+struct drbd_genlmsghdr {
+	__u32 minor;
+	union {
+		__u32 flags;
+		__s32 ret_code;
+	};
+};
+
+/* To be used in drbd_genlmsghdr.flags */
+enum {
+	DRBD_GENL_F_SET_DEFAULTS = 1,
+};
+
+enum drbd_state_info_bcast_reason {
+	SIB_GET_STATUS_REPLY = 1,
+	SIB_STATE_CHANGE = 2,
+	SIB_HELPER_PRE = 3,
+	SIB_HELPER_POST = 4,
+	SIB_SYNC_PROGRESS = 5,
+};
+
+#endif /* _UAPI_LINUX_DRBD_H */
diff --git a/include/uapi/linux/drbd_genl.h b/include/uapi/linux/drbd_genl.h
new file mode 100644
index 0000000000000..8b25f08d8a90f
--- /dev/null
+++ b/include/uapi/linux/drbd_genl.h
@@ -0,0 +1,359 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+
+#ifndef _UAPI_LINUX_DRBD_GENL_H
+#define _UAPI_LINUX_DRBD_GENL_H
+
+#define DRBD_FAMILY_NAME	"drbd"
+#define DRBD_FAMILY_VERSION	1
+
+enum {
+	DRBD_NLA_CFG_REPLY = 1,
+	DRBD_NLA_CFG_CONTEXT,
+	DRBD_NLA_DISK_CONF,
+	DRBD_NLA_RESOURCE_OPTS,
+	DRBD_NLA_NET_CONF,
+	DRBD_NLA_SET_ROLE_PARMS,
+	DRBD_NLA_RESIZE_PARMS,
+	DRBD_NLA_STATE_INFO,
+	DRBD_NLA_START_OV_PARMS,
+	DRBD_NLA_NEW_C_UUID_PARMS,
+	DRBD_NLA_TIMEOUT_PARMS,
+	DRBD_NLA_DISCONNECT_PARMS,
+	DRBD_NLA_DETACH_PARMS,
+	DRBD_NLA_RESOURCE_INFO = 15,
+	DRBD_NLA_DEVICE_INFO,
+	DRBD_NLA_CONNECTION_INFO,
+	DRBD_NLA_PEER_DEVICE_INFO,
+	DRBD_NLA_RESOURCE_STATISTICS,
+	DRBD_NLA_DEVICE_STATISTICS,
+	DRBD_NLA_CONNECTION_STATISTICS,
+	DRBD_NLA_PEER_DEVICE_STATISTICS,
+	DRBD_NLA_NOTIFICATION_HEADER,
+	DRBD_NLA_HELPER,
+
+	__DRBD_NLA_MAX,
+	DRBD_NLA_MAX = (__DRBD_NLA_MAX - 1)
+};
+
+enum {
+	DRBD_A_DRBD_CFG_REPLY_INFO_TEXT = 1,
+
+	__DRBD_A_DRBD_CFG_REPLY_MAX,
+	DRBD_A_DRBD_CFG_REPLY_MAX = (__DRBD_A_DRBD_CFG_REPLY_MAX - 1)
+};
+
+enum {
+	DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME = 1,
+	DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME,
+	DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR,
+	DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR,
+
+	__DRBD_A_DRBD_CFG_CONTEXT_MAX,
+	DRBD_A_DRBD_CFG_CONTEXT_MAX = (__DRBD_A_DRBD_CFG_CONTEXT_MAX - 1)
+};
+
+enum {
+	DRBD_A_DISK_CONF_BACKING_DEV = 1,
+	DRBD_A_DISK_CONF_META_DEV,
+	DRBD_A_DISK_CONF_META_DEV_IDX,
+	DRBD_A_DISK_CONF_DISK_SIZE,
+	DRBD_A_DISK_CONF_MAX_BIO_BVECS,
+	DRBD_A_DISK_CONF_ON_IO_ERROR,
+	DRBD_A_DISK_CONF_FENCING,
+	DRBD_A_DISK_CONF_RESYNC_RATE,
+	DRBD_A_DISK_CONF_RESYNC_AFTER,
+	DRBD_A_DISK_CONF_AL_EXTENTS,
+	DRBD_A_DISK_CONF_C_PLAN_AHEAD,
+	DRBD_A_DISK_CONF_C_DELAY_TARGET,
+	DRBD_A_DISK_CONF_C_FILL_TARGET,
+	DRBD_A_DISK_CONF_C_MAX_RATE,
+	DRBD_A_DISK_CONF_C_MIN_RATE,
+	DRBD_A_DISK_CONF_DISK_BARRIER,
+	DRBD_A_DISK_CONF_DISK_FLUSHES,
+	DRBD_A_DISK_CONF_DISK_DRAIN,
+	DRBD_A_DISK_CONF_MD_FLUSHES,
+	DRBD_A_DISK_CONF_DISK_TIMEOUT,
+	DRBD_A_DISK_CONF_READ_BALANCING,
+	DRBD_A_DISK_CONF_AL_UPDATES = 23,
+	DRBD_A_DISK_CONF_DISCARD_ZEROES_IF_ALIGNED,
+	DRBD_A_DISK_CONF_RS_DISCARD_GRANULARITY,
+	DRBD_A_DISK_CONF_DISABLE_WRITE_SAME,
+
+	__DRBD_A_DISK_CONF_MAX,
+	DRBD_A_DISK_CONF_MAX = (__DRBD_A_DISK_CONF_MAX - 1)
+};
+
+enum {
+	DRBD_A_RES_OPTS_CPU_MASK = 1,
+	DRBD_A_RES_OPTS_ON_NO_DATA,
+
+	__DRBD_A_RES_OPTS_MAX,
+	DRBD_A_RES_OPTS_MAX = (__DRBD_A_RES_OPTS_MAX - 1)
+};
+
+enum {
+	DRBD_A_NET_CONF_SHARED_SECRET = 1,
+	DRBD_A_NET_CONF_CRAM_HMAC_ALG,
+	DRBD_A_NET_CONF_INTEGRITY_ALG,
+	DRBD_A_NET_CONF_VERIFY_ALG,
+	DRBD_A_NET_CONF_CSUMS_ALG,
+	DRBD_A_NET_CONF_WIRE_PROTOCOL,
+	DRBD_A_NET_CONF_CONNECT_INT,
+	DRBD_A_NET_CONF_TIMEOUT,
+	DRBD_A_NET_CONF_PING_INT,
+	DRBD_A_NET_CONF_PING_TIMEO,
+	DRBD_A_NET_CONF_SNDBUF_SIZE,
+	DRBD_A_NET_CONF_RCVBUF_SIZE,
+	DRBD_A_NET_CONF_KO_COUNT,
+	DRBD_A_NET_CONF_MAX_BUFFERS,
+	DRBD_A_NET_CONF_MAX_EPOCH_SIZE,
+	DRBD_A_NET_CONF_UNPLUG_WATERMARK,
+	DRBD_A_NET_CONF_AFTER_SB_0P,
+	DRBD_A_NET_CONF_AFTER_SB_1P,
+	DRBD_A_NET_CONF_AFTER_SB_2P,
+	DRBD_A_NET_CONF_RR_CONFLICT,
+	DRBD_A_NET_CONF_ON_CONGESTION,
+	DRBD_A_NET_CONF_CONG_FILL,
+	DRBD_A_NET_CONF_CONG_EXTENTS,
+	DRBD_A_NET_CONF_TWO_PRIMARIES,
+	DRBD_A_NET_CONF_DISCARD_MY_DATA,
+	DRBD_A_NET_CONF_TCP_CORK,
+	DRBD_A_NET_CONF_ALWAYS_ASBP,
+	DRBD_A_NET_CONF_TENTATIVE,
+	DRBD_A_NET_CONF_USE_RLE,
+	DRBD_A_NET_CONF_CSUMS_AFTER_CRASH_ONLY = 33,
+	DRBD_A_NET_CONF_SOCK_CHECK_TIMEO,
+
+	__DRBD_A_NET_CONF_MAX,
+	DRBD_A_NET_CONF_MAX = (__DRBD_A_NET_CONF_MAX - 1)
+};
+
+enum {
+	DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE = 1,
+
+	__DRBD_A_SET_ROLE_PARMS_MAX,
+	DRBD_A_SET_ROLE_PARMS_MAX = (__DRBD_A_SET_ROLE_PARMS_MAX - 1)
+};
+
+enum {
+	DRBD_A_RESIZE_PARMS_RESIZE_SIZE = 1,
+	DRBD_A_RESIZE_PARMS_RESIZE_FORCE,
+	DRBD_A_RESIZE_PARMS_NO_RESYNC,
+	DRBD_A_RESIZE_PARMS_AL_STRIPES,
+	DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE,
+
+	__DRBD_A_RESIZE_PARMS_MAX,
+	DRBD_A_RESIZE_PARMS_MAX = (__DRBD_A_RESIZE_PARMS_MAX - 1)
+};
+
+enum {
+	DRBD_A_STATE_INFO_SIB_REASON = 1,
+	DRBD_A_STATE_INFO_CURRENT_STATE,
+	DRBD_A_STATE_INFO_CAPACITY,
+	DRBD_A_STATE_INFO_ED_UUID,
+	DRBD_A_STATE_INFO_PREV_STATE,
+	DRBD_A_STATE_INFO_NEW_STATE,
+	DRBD_A_STATE_INFO_UUIDS,
+	DRBD_A_STATE_INFO_DISK_FLAGS,
+	DRBD_A_STATE_INFO_BITS_TOTAL,
+	DRBD_A_STATE_INFO_BITS_OOS,
+	DRBD_A_STATE_INFO_BITS_RS_TOTAL,
+	DRBD_A_STATE_INFO_BITS_RS_FAILED,
+	DRBD_A_STATE_INFO_HELPER,
+	DRBD_A_STATE_INFO_HELPER_EXIT_CODE,
+	DRBD_A_STATE_INFO_SEND_CNT,
+	DRBD_A_STATE_INFO_RECV_CNT,
+	DRBD_A_STATE_INFO_READ_CNT,
+	DRBD_A_STATE_INFO_WRIT_CNT,
+	DRBD_A_STATE_INFO_AL_WRIT_CNT,
+	DRBD_A_STATE_INFO_BM_WRIT_CNT,
+	DRBD_A_STATE_INFO_AP_BIO_CNT,
+	DRBD_A_STATE_INFO_AP_PENDING_CNT,
+	DRBD_A_STATE_INFO_RS_PENDING_CNT,
+
+	__DRBD_A_STATE_INFO_MAX,
+	DRBD_A_STATE_INFO_MAX = (__DRBD_A_STATE_INFO_MAX - 1)
+};
+
+enum {
+	DRBD_A_START_OV_PARMS_OV_START_SECTOR = 1,
+	DRBD_A_START_OV_PARMS_OV_STOP_SECTOR,
+
+	__DRBD_A_START_OV_PARMS_MAX,
+	DRBD_A_START_OV_PARMS_MAX = (__DRBD_A_START_OV_PARMS_MAX - 1)
+};
+
+enum {
+	DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM = 1,
+
+	__DRBD_A_NEW_C_UUID_PARMS_MAX,
+	DRBD_A_NEW_C_UUID_PARMS_MAX = (__DRBD_A_NEW_C_UUID_PARMS_MAX - 1)
+};
+
+enum {
+	DRBD_A_TIMEOUT_PARMS_TIMEOUT_TYPE = 1,
+
+	__DRBD_A_TIMEOUT_PARMS_MAX,
+	DRBD_A_TIMEOUT_PARMS_MAX = (__DRBD_A_TIMEOUT_PARMS_MAX - 1)
+};
+
+enum {
+	DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT = 1,
+
+	__DRBD_A_DISCONNECT_PARMS_MAX,
+	DRBD_A_DISCONNECT_PARMS_MAX = (__DRBD_A_DISCONNECT_PARMS_MAX - 1)
+};
+
+enum {
+	DRBD_A_DETACH_PARMS_FORCE_DETACH = 1,
+
+	__DRBD_A_DETACH_PARMS_MAX,
+	DRBD_A_DETACH_PARMS_MAX = (__DRBD_A_DETACH_PARMS_MAX - 1)
+};
+
+enum {
+	DRBD_A_RESOURCE_INFO_RES_ROLE = 1,
+	DRBD_A_RESOURCE_INFO_RES_SUSP,
+	DRBD_A_RESOURCE_INFO_RES_SUSP_NOD,
+	DRBD_A_RESOURCE_INFO_RES_SUSP_FEN,
+
+	__DRBD_A_RESOURCE_INFO_MAX,
+	DRBD_A_RESOURCE_INFO_MAX = (__DRBD_A_RESOURCE_INFO_MAX - 1)
+};
+
+enum {
+	DRBD_A_DEVICE_INFO_DEV_DISK_STATE = 1,
+
+	__DRBD_A_DEVICE_INFO_MAX,
+	DRBD_A_DEVICE_INFO_MAX = (__DRBD_A_DEVICE_INFO_MAX - 1)
+};
+
+enum {
+	DRBD_A_CONNECTION_INFO_CONN_CONNECTION_STATE = 1,
+	DRBD_A_CONNECTION_INFO_CONN_ROLE,
+
+	__DRBD_A_CONNECTION_INFO_MAX,
+	DRBD_A_CONNECTION_INFO_MAX = (__DRBD_A_CONNECTION_INFO_MAX - 1)
+};
+
+enum {
+	DRBD_A_PEER_DEVICE_INFO_PEER_REPL_STATE = 1,
+	DRBD_A_PEER_DEVICE_INFO_PEER_DISK_STATE,
+	DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_USER,
+	DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_PEER,
+	DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY,
+
+	__DRBD_A_PEER_DEVICE_INFO_MAX,
+	DRBD_A_PEER_DEVICE_INFO_MAX = (__DRBD_A_PEER_DEVICE_INFO_MAX - 1)
+};
+
+enum {
+	DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING = 1,
+
+	__DRBD_A_RESOURCE_STATISTICS_MAX,
+	DRBD_A_RESOURCE_STATISTICS_MAX = (__DRBD_A_RESOURCE_STATISTICS_MAX - 1)
+};
+
+enum {
+	DRBD_A_DEVICE_STATISTICS_DEV_SIZE = 1,
+	DRBD_A_DEVICE_STATISTICS_DEV_READ,
+	DRBD_A_DEVICE_STATISTICS_DEV_WRITE,
+	DRBD_A_DEVICE_STATISTICS_DEV_AL_WRITES,
+	DRBD_A_DEVICE_STATISTICS_DEV_BM_WRITES,
+	DRBD_A_DEVICE_STATISTICS_DEV_UPPER_PENDING,
+	DRBD_A_DEVICE_STATISTICS_DEV_LOWER_PENDING,
+	DRBD_A_DEVICE_STATISTICS_DEV_UPPER_BLOCKED,
+	DRBD_A_DEVICE_STATISTICS_DEV_LOWER_BLOCKED,
+	DRBD_A_DEVICE_STATISTICS_DEV_AL_SUSPENDED,
+	DRBD_A_DEVICE_STATISTICS_DEV_EXPOSED_DATA_UUID,
+	DRBD_A_DEVICE_STATISTICS_DEV_CURRENT_UUID,
+	DRBD_A_DEVICE_STATISTICS_DEV_DISK_FLAGS,
+	DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS,
+
+	__DRBD_A_DEVICE_STATISTICS_MAX,
+	DRBD_A_DEVICE_STATISTICS_MAX = (__DRBD_A_DEVICE_STATISTICS_MAX - 1)
+};
+
+enum {
+	DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED = 1,
+
+	__DRBD_A_CONNECTION_STATISTICS_MAX,
+	DRBD_A_CONNECTION_STATISTICS_MAX = (__DRBD_A_CONNECTION_STATISTICS_MAX - 1)
+};
+
+enum {
+	DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RECEIVED = 1,
+	DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_SENT,
+	DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_PENDING,
+	DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_UNACKED,
+	DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_OUT_OF_SYNC,
+	DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RESYNC_FAILED,
+	DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_BITMAP_UUID,
+	DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS = 9,
+
+	__DRBD_A_PEER_DEVICE_STATISTICS_MAX,
+	DRBD_A_PEER_DEVICE_STATISTICS_MAX = (__DRBD_A_PEER_DEVICE_STATISTICS_MAX - 1)
+};
+
+enum {
+	DRBD_A_DRBD_NOTIFICATION_HEADER_NH_TYPE = 1,
+
+	__DRBD_A_DRBD_NOTIFICATION_HEADER_MAX,
+	DRBD_A_DRBD_NOTIFICATION_HEADER_MAX = (__DRBD_A_DRBD_NOTIFICATION_HEADER_MAX - 1)
+};
+
+enum {
+	DRBD_A_DRBD_HELPER_INFO_HELPER_NAME = 1,
+	DRBD_A_DRBD_HELPER_INFO_HELPER_STATUS,
+
+	__DRBD_A_DRBD_HELPER_INFO_MAX,
+	DRBD_A_DRBD_HELPER_INFO_MAX = (__DRBD_A_DRBD_HELPER_INFO_MAX - 1)
+};
+
+enum {
+	DRBD_ADM_EVENT = 1,
+	DRBD_ADM_GET_STATUS,
+	DRBD_ADM_NEW_MINOR = 5,
+	DRBD_ADM_DEL_MINOR,
+	DRBD_ADM_NEW_RESOURCE,
+	DRBD_ADM_DEL_RESOURCE,
+	DRBD_ADM_RESOURCE_OPTS,
+	DRBD_ADM_CONNECT,
+	DRBD_ADM_DISCONNECT,
+	DRBD_ADM_ATTACH,
+	DRBD_ADM_RESIZE,
+	DRBD_ADM_PRIMARY,
+	DRBD_ADM_SECONDARY,
+	DRBD_ADM_NEW_C_UUID,
+	DRBD_ADM_START_OV,
+	DRBD_ADM_DETACH,
+	DRBD_ADM_INVALIDATE,
+	DRBD_ADM_INVAL_PEER,
+	DRBD_ADM_PAUSE_SYNC,
+	DRBD_ADM_RESUME_SYNC,
+	DRBD_ADM_SUSPEND_IO,
+	DRBD_ADM_RESUME_IO,
+	DRBD_ADM_OUTDATE,
+	DRBD_ADM_GET_TIMEOUT_TYPE,
+	DRBD_ADM_DOWN,
+	DRBD_ADM_CHG_DISK_OPTS,
+	DRBD_ADM_CHG_NET_OPTS,
+	DRBD_ADM_GET_RESOURCES,
+	DRBD_ADM_GET_DEVICES,
+	DRBD_ADM_GET_CONNECTIONS,
+	DRBD_ADM_GET_PEER_DEVICES,
+	DRBD_ADM_RESOURCE_STATE,
+	DRBD_ADM_DEVICE_STATE,
+	DRBD_ADM_CONNECTION_STATE,
+	DRBD_ADM_PEER_DEVICE_STATE,
+	DRBD_ADM_GET_INITIAL_STATE,
+	DRBD_ADM_HELPER = 40,
+	DRBD_ADM_INITIAL_STATE_DONE,
+
+	__DRBD_ADM_MAX,
+	DRBD_ADM_MAX = (__DRBD_ADM_MAX - 1)
+};
+
+#define DRBD_MCGRP_EVENTS	"events"
+
+#endif /* _UAPI_LINUX_DRBD_GENL_H */
diff --git a/include/uapi/linux/drbd_limits.h b/include/uapi/linux/drbd_limits.h
new file mode 100644
index 0000000000000..acefe84bc6026
--- /dev/null
+++ b/include/uapi/linux/drbd_limits.h
@@ -0,0 +1,251 @@
+/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */
+/*
+  drbd_limits.h
+  This file is part of DRBD by Philipp Reisner and Lars Ellenberg.
+*/
+
+/*
+ * Our current limitations.
+ * Some of them are hard limits,
+ * some of them are arbitrary range limits, that make it easier to provide
+ * feedback about nonsense settings for certain configurable values.
+ */
+
+#ifndef _UAPI_LINUX_DRBD_LIMITS_H
+#define _UAPI_LINUX_DRBD_LIMITS_H
+
+#include <linux/drbd.h>
+
+#define DRBD_MINOR_COUNT_MIN 1U
+#define DRBD_MINOR_COUNT_MAX 255U
+#define DRBD_MINOR_COUNT_DEF 32U
+#define DRBD_MINOR_COUNT_SCALE '1'
+
+#define DRBD_VOLUME_MAX 65534U
+
+#define DRBD_DIALOG_REFRESH_MIN 0U
+#define DRBD_DIALOG_REFRESH_MAX 600U
+#define DRBD_DIALOG_REFRESH_SCALE '1'
+
+/* valid port number */
+#define DRBD_PORT_MIN 1U
+#define DRBD_PORT_MAX 0xffffU
+#define DRBD_PORT_SCALE '1'
+
+/* startup { */
+  /* if you want more than 3.4 days, disable */
+#define DRBD_WFC_TIMEOUT_MIN 0U
+#define DRBD_WFC_TIMEOUT_MAX 300000U
+#define DRBD_WFC_TIMEOUT_DEF 0U
+#define DRBD_WFC_TIMEOUT_SCALE '1'
+
+#define DRBD_DEGR_WFC_TIMEOUT_MIN 0U
+#define DRBD_DEGR_WFC_TIMEOUT_MAX 300000U
+#define DRBD_DEGR_WFC_TIMEOUT_DEF 0U
+#define DRBD_DEGR_WFC_TIMEOUT_SCALE '1'
+
+#define DRBD_OUTDATED_WFC_TIMEOUT_MIN 0U
+#define DRBD_OUTDATED_WFC_TIMEOUT_MAX 300000U
+#define DRBD_OUTDATED_WFC_TIMEOUT_DEF 0U
+#define DRBD_OUTDATED_WFC_TIMEOUT_SCALE '1'
+/* }*/
+
+/* net { */
+  /* timeout, unit centi seconds
+   * more than one minute timeout is not useful */
+#define DRBD_TIMEOUT_MIN 1U
+#define DRBD_TIMEOUT_MAX 600U
+#define DRBD_TIMEOUT_DEF 60U       /* 6 seconds */
+#define DRBD_TIMEOUT_SCALE '1'
+
+ /* If backing disk takes longer than disk_timeout, mark the disk as failed */
+#define DRBD_DISK_TIMEOUT_MIN 0U    /* 0 = disabled */
+#define DRBD_DISK_TIMEOUT_MAX 6000U /* 10 Minutes */
+#define DRBD_DISK_TIMEOUT_DEF 0U    /* disabled */
+#define DRBD_DISK_TIMEOUT_SCALE '1'
+
+  /* active connection retries when C_WF_CONNECTION */
+#define DRBD_CONNECT_INT_MIN 1U
+#define DRBD_CONNECT_INT_MAX 120U
+#define DRBD_CONNECT_INT_DEF 10U   /* seconds */
+#define DRBD_CONNECT_INT_SCALE '1'
+
+  /* keep-alive probes when idle */
+#define DRBD_PING_INT_MIN 1U
+#define DRBD_PING_INT_MAX 120U
+#define DRBD_PING_INT_DEF 10U
+#define DRBD_PING_INT_SCALE '1'
+
+ /* timeout for the ping packets.*/
+#define DRBD_PING_TIMEO_MIN  1U
+#define DRBD_PING_TIMEO_MAX  300U
+#define DRBD_PING_TIMEO_DEF  5U
+#define DRBD_PING_TIMEO_SCALE '1'
+
+  /* max number of write requests between write barriers */
+#define DRBD_MAX_EPOCH_SIZE_MIN 1U
+#define DRBD_MAX_EPOCH_SIZE_MAX 20000U
+#define DRBD_MAX_EPOCH_SIZE_DEF 2048U
+#define DRBD_MAX_EPOCH_SIZE_SCALE '1'
+
+  /* I don't think that a tcp send buffer of more than 10M is useful */
+#define DRBD_SNDBUF_SIZE_MIN  0U
+#define DRBD_SNDBUF_SIZE_MAX  (10U<<20)
+#define DRBD_SNDBUF_SIZE_DEF  0U
+#define DRBD_SNDBUF_SIZE_SCALE '1'
+
+#define DRBD_RCVBUF_SIZE_MIN  0U
+#define DRBD_RCVBUF_SIZE_MAX  (10U<<20)
+#define DRBD_RCVBUF_SIZE_DEF  0U
+#define DRBD_RCVBUF_SIZE_SCALE '1'
+
+  /* @4k PageSize -> 128kB - 512MB */
+#define DRBD_MAX_BUFFERS_MIN  32U
+#define DRBD_MAX_BUFFERS_MAX  131072U
+#define DRBD_MAX_BUFFERS_DEF  2048U
+#define DRBD_MAX_BUFFERS_SCALE '1'
+
+  /* @4k PageSize -> 4kB - 512MB */
+#define DRBD_UNPLUG_WATERMARK_MIN  1U
+#define DRBD_UNPLUG_WATERMARK_MAX  131072U
+#define DRBD_UNPLUG_WATERMARK_DEF (DRBD_MAX_BUFFERS_DEF/16)
+#define DRBD_UNPLUG_WATERMARK_SCALE '1'
+
+  /* 0 is disabled.
+   * 200 should be more than enough even for very short timeouts */
+#define DRBD_KO_COUNT_MIN  0U
+#define DRBD_KO_COUNT_MAX  200U
+#define DRBD_KO_COUNT_DEF  7U
+#define DRBD_KO_COUNT_SCALE '1'
+/* } */
+
+/* syncer { */
+  /* FIXME allow rate to be zero? */
+#define DRBD_RESYNC_RATE_MIN 1U
+/* channel bonding 10 GbE, or other hardware */
+#define DRBD_RESYNC_RATE_MAX (4 << 20)
+#define DRBD_RESYNC_RATE_DEF 250U
+#define DRBD_RESYNC_RATE_SCALE 'k'  /* kilobytes */
+
+#define DRBD_AL_EXTENTS_MIN  67U
+  /* we use u16 as "slot number", (u16)~0 is "FREE".
+   * If you use >= 292 kB on-disk ring buffer,
+   * this is the maximum you can use: */
+#define DRBD_AL_EXTENTS_MAX  0xfffeU
+#define DRBD_AL_EXTENTS_DEF  1237U
+#define DRBD_AL_EXTENTS_SCALE '1'
+
+#define DRBD_MINOR_NUMBER_MIN  -1
+#define DRBD_MINOR_NUMBER_MAX  ((1 << 20) - 1)
+#define DRBD_MINOR_NUMBER_DEF  -1
+#define DRBD_MINOR_NUMBER_SCALE '1'
+
+/* } */
+
+/* drbdsetup XY resize -d Z
+ * you are free to reduce the device size to nothing, if you want to.
+ * the upper limit with 64bit kernel, enough ram and flexible meta data
+ * is 1 PiB, currently. */
+/* DRBD_MAX_SECTORS */
+#define DRBD_DISK_SIZE_MIN  0LLU
+#define DRBD_DISK_SIZE_MAX  (1LLU * (2LLU << 40))
+#define DRBD_DISK_SIZE_DEF  0LLU /* = disabled = no user size... */
+#define DRBD_DISK_SIZE_SCALE 's'  /* sectors */
+
+#define DRBD_ON_IO_ERROR_DEF EP_DETACH
+#define DRBD_FENCING_DEF FP_DONT_CARE
+#define DRBD_AFTER_SB_0P_DEF ASB_DISCONNECT
+#define DRBD_AFTER_SB_1P_DEF ASB_DISCONNECT
+#define DRBD_AFTER_SB_2P_DEF ASB_DISCONNECT
+#define DRBD_RR_CONFLICT_DEF ASB_DISCONNECT
+#define DRBD_ON_NO_DATA_DEF OND_IO_ERROR
+#define DRBD_ON_CONGESTION_DEF OC_BLOCK
+#define DRBD_READ_BALANCING_DEF RB_PREFER_LOCAL
+
+#define DRBD_MAX_BIO_BVECS_MIN 0U
+#define DRBD_MAX_BIO_BVECS_MAX 128U
+#define DRBD_MAX_BIO_BVECS_DEF 0U
+#define DRBD_MAX_BIO_BVECS_SCALE '1'
+
+#define DRBD_C_PLAN_AHEAD_MIN  0U
+#define DRBD_C_PLAN_AHEAD_MAX  300U
+#define DRBD_C_PLAN_AHEAD_DEF  20U
+#define DRBD_C_PLAN_AHEAD_SCALE '1'
+
+#define DRBD_C_DELAY_TARGET_MIN 1U
+#define DRBD_C_DELAY_TARGET_MAX 100U
+#define DRBD_C_DELAY_TARGET_DEF 10U
+#define DRBD_C_DELAY_TARGET_SCALE '1'
+
+#define DRBD_C_FILL_TARGET_MIN 0U
+#define DRBD_C_FILL_TARGET_MAX (1U<<20) /* 500MByte in sec */
+#define DRBD_C_FILL_TARGET_DEF 100U /* Try to place 50KiB in socket send buffer during resync */
+#define DRBD_C_FILL_TARGET_SCALE 's'  /* sectors */
+
+#define DRBD_C_MAX_RATE_MIN     250U
+#define DRBD_C_MAX_RATE_MAX     (4U << 20)
+#define DRBD_C_MAX_RATE_DEF     102400U
+#define DRBD_C_MAX_RATE_SCALE	'k'  /* kilobytes */
+
+#define DRBD_C_MIN_RATE_MIN     0U
+#define DRBD_C_MIN_RATE_MAX     (4U << 20)
+#define DRBD_C_MIN_RATE_DEF     250U
+#define DRBD_C_MIN_RATE_SCALE	'k'  /* kilobytes */
+
+#define DRBD_CONG_FILL_MIN	0U
+#define DRBD_CONG_FILL_MAX	(10U<<21) /* 10GByte in sectors */
+#define DRBD_CONG_FILL_DEF	0U
+#define DRBD_CONG_FILL_SCALE	's'  /* sectors */
+
+#define DRBD_CONG_EXTENTS_MIN	DRBD_AL_EXTENTS_MIN
+#define DRBD_CONG_EXTENTS_MAX	DRBD_AL_EXTENTS_MAX
+#define DRBD_CONG_EXTENTS_DEF	DRBD_AL_EXTENTS_DEF
+#define DRBD_CONG_EXTENTS_SCALE DRBD_AL_EXTENTS_SCALE
+
+#define DRBD_PROTOCOL_DEF DRBD_PROT_C
+
+#define DRBD_DISK_BARRIER_DEF	0U
+#define DRBD_DISK_FLUSHES_DEF	1U
+#define DRBD_DISK_DRAIN_DEF	1U
+#define DRBD_MD_FLUSHES_DEF	1U
+#define DRBD_TCP_CORK_DEF	1U
+#define DRBD_AL_UPDATES_DEF     1U
+
+/* We used to ignore the discard_zeroes_data setting.
+ * To not change established (and expected) behaviour,
+ * by default assume that, for discard_zeroes_data=0,
+ * we can make that an effective discard_zeroes_data=1,
+ * if we only explicitly zero-out unaligned partial chunks. */
+#define DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF 1U
+
+/* Some backends pretend to support WRITE SAME,
+ * but fail such requests when they are actually submitted.
+ * This is to tell DRBD to not even try. */
+#define DRBD_DISABLE_WRITE_SAME_DEF 0U
+
+#define DRBD_ALLOW_TWO_PRIMARIES_DEF	0U
+#define DRBD_ALWAYS_ASBP_DEF	0U
+#define DRBD_USE_RLE_DEF	1U
+#define DRBD_CSUMS_AFTER_CRASH_ONLY_DEF 0U
+
+#define DRBD_AL_STRIPES_MIN     1U
+#define DRBD_AL_STRIPES_MAX     1024U
+#define DRBD_AL_STRIPES_DEF     1U
+#define DRBD_AL_STRIPES_SCALE   '1'
+
+#define DRBD_AL_STRIPE_SIZE_MIN   4U
+#define DRBD_AL_STRIPE_SIZE_MAX   16777216U
+#define DRBD_AL_STRIPE_SIZE_DEF   32U
+#define DRBD_AL_STRIPE_SIZE_SCALE 'k' /* kilobytes */
+
+#define DRBD_SOCKET_CHECK_TIMEO_MIN 0U
+#define DRBD_SOCKET_CHECK_TIMEO_MAX DRBD_PING_TIMEO_MAX
+#define DRBD_SOCKET_CHECK_TIMEO_DEF 0U
+#define DRBD_SOCKET_CHECK_TIMEO_SCALE '1'
+
+#define DRBD_RS_DISCARD_GRANULARITY_MIN 0U
+#define DRBD_RS_DISCARD_GRANULARITY_MAX (1U<<20)  /* 1MiByte */
+#define DRBD_RS_DISCARD_GRANULARITY_DEF 0U     /* disabled by default */
+#define DRBD_RS_DISCARD_GRANULARITY_SCALE '1' /* bytes */
+
+#endif /* _UAPI_LINUX_DRBD_LIMITS_H */
diff --git a/include/uapi/linux/fs.h b/include/uapi/linux/fs.h
index 13f71202845e8..2ea4c81df08f1 100644
--- a/include/uapi/linux/fs.h
+++ b/include/uapi/linux/fs.h
@@ -254,6 +254,13 @@ struct file_attr {
 #define FS_XFLAG_DAX		0x00008000	/* use DAX for IO */
 #define FS_XFLAG_COWEXTSIZE	0x00010000	/* CoW extent size allocator hint */
 #define FS_XFLAG_VERITY		0x00020000	/* fs-verity enabled */
+/*
+ * Case handling flags (read-only, cannot be set via ioctl).
+ * Default (neither set) indicates POSIX semantics: case-sensitive
+ * lookups and case-preserving storage.
+ */
+#define FS_XFLAG_CASEFOLD	0x00040000	/* case-insensitive lookups */
+#define FS_XFLAG_CASENONPRESERVING 0x00080000	/* case not preserved */
 #define FS_XFLAG_HASATTR	0x80000000	/* no DIFLAG for this	*/
 
 /* the read-only stuff doesn't really belong here, but any other place is
diff --git a/include/uapi/linux/fsl_hypervisor.h b/include/uapi/linux/fsl_hypervisor.h
index 1e237fba951fe..ab4388441e80c 100644
--- a/include/uapi/linux/fsl_hypervisor.h
+++ b/include/uapi/linux/fsl_hypervisor.h
@@ -114,9 +114,9 @@ struct fsl_hv_ioctl_stop {
  * @target: the partition ID of the target partition, or -1 for this
  *          partition
  * @reserved: reserved, must be set to 0
- * @local_addr: user-space virtual address of a buffer in the local
+ * @local_vaddr: user-space virtual address of a buffer in the local
  *              partition
- * @remote_addr: guest physical address of a buffer in the
+ * @remote_paddr: guest physical address of a buffer in the
  *           remote partition
  * @count: the number of bytes to copy.  Both the local and remote
  *         buffers must be at least 'count' bytes long
diff --git a/include/uapi/linux/if_bridge.h b/include/uapi/linux/if_bridge.h
index e52f8207ab278..21a700c02ef76 100644
--- a/include/uapi/linux/if_bridge.h
+++ b/include/uapi/linux/if_bridge.h
@@ -526,6 +526,7 @@ enum {
 	BRIDGE_VLANDB_ENTRY_MCAST_N_GROUPS,
 	BRIDGE_VLANDB_ENTRY_MCAST_MAX_GROUPS,
 	BRIDGE_VLANDB_ENTRY_NEIGH_SUPPRESS,
+	BRIDGE_VLANDB_ENTRY_NEIGH_FORWARD_GRAT,
 	__BRIDGE_VLANDB_ENTRY_MAX,
 };
 #define BRIDGE_VLANDB_ENTRY_MAX (__BRIDGE_VLANDB_ENTRY_MAX - 1)
diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h
index 79ce4bc24cba6..46413392b402c 100644
--- a/include/uapi/linux/if_link.h
+++ b/include/uapi/linux/if_link.h
@@ -1085,6 +1085,22 @@ enum {
  *   Note that this option only takes effect when *IFLA_BRPORT_NEIGH_SUPPRESS*
  *   is enabled for a given port.
  *
+ * @IFLA_BRPORT_NEIGH_FORWARD_GRAT
+ *   Controls whether gratuitous ARP packets and unsolicited Neighbor
+ *   Advertisement packets are forwarded on a given port even when neighbor
+ *   suppression is enabled.
+ *   By default this flag is off, meaning gratuitous ARP and unsolicited NA
+ *   packets will be suppressed when neighbor suppression is enabled.
+ *   Setting this flag to on allows these packets to be forwarded even
+ *   when *IFLA_BRPORT_NEIGH_SUPPRESS* or *IFLA_BRPORT_NEIGH_VLAN_SUPPRESS*
+ *   is enabled.
+ *
+ *   Note that this option only takes effect when *IFLA_BRPORT_NEIGH_SUPPRESS*
+ *   or *IFLA_BRPORT_NEIGH_VLAN_SUPPRESS* is enabled for a given port.
+ *   When *IFLA_BRPORT_NEIGH_VLAN_SUPPRESS* is set, this port-level flag is
+ *   ignored and per-VLAN control is available via
+ *   *BRIDGE_VLANDB_ENTRY_NEIGH_FORWARD_GRAT*.
+ *
  * @IFLA_BRPORT_BACKUP_NHID
  *   The FDB nexthop object ID to attach to packets being redirected to a
  *   backup port that has VLAN tunnel mapping enabled (via the
@@ -1137,6 +1153,7 @@ enum {
 	IFLA_BRPORT_MCAST_MAX_GROUPS,
 	IFLA_BRPORT_NEIGH_VLAN_SUPPRESS,
 	IFLA_BRPORT_BACKUP_NHID,
+	IFLA_BRPORT_NEIGH_FORWARD_GRAT,
 	__IFLA_BRPORT_MAX
 };
 #define IFLA_BRPORT_MAX (__IFLA_BRPORT_MAX - 1)
diff --git a/include/uapi/linux/iio/types.h b/include/uapi/linux/iio/types.h
index 6d269b8442713..d7c2bb2236518 100644
--- a/include/uapi/linux/iio/types.h
+++ b/include/uapi/linux/iio/types.h
@@ -113,6 +113,7 @@ enum iio_modifier {
 	IIO_MOD_ACTIVE,
 	IIO_MOD_REACTIVE,
 	IIO_MOD_APPARENT,
+	IIO_MOD_QUATERNION_AXIS,
 };
 
 enum iio_event_type {
diff --git a/include/uapi/linux/io_uring/bpf_filter.h b/include/uapi/linux/io_uring/bpf_filter.h
index 1b461d792a7b9..ce7d78ab13b30 100644
--- a/include/uapi/linux/io_uring/bpf_filter.h
+++ b/include/uapi/linux/io_uring/bpf_filter.h
@@ -27,6 +27,22 @@ struct io_uring_bpf_ctx {
 			__u64	mode;
 			__u64	resolve;
 		} open;
+		/*
+		 * For CONNECT: fields are populated only when addr_len covers
+		 * them; unpopulated fields are zero from the caller-side memset
+		 * in io_uring_populate_bpf_ctx(). port and v4_addr are network
+		 * byte order. Filters may only issue BPF_LD|BPF_W|BPF_ABS at
+		 * 4-byte aligned offsets; load + mask for sub-word fields.
+		 */
+		struct {
+			__u32	family;	/* sa_family_t zero-extended */
+			__be16	port;
+			__u8	pad[2];
+			union {
+				__be32	v4_addr;
+				__u8	v6_addr[16];
+			};
+		} connect;
 	};
 };
 
diff --git a/include/uapi/linux/kfd_ioctl.h b/include/uapi/linux/kfd_ioctl.h
index e72359370857c..9584b5aab727a 100644
--- a/include/uapi/linux/kfd_ioctl.h
+++ b/include/uapi/linux/kfd_ioctl.h
@@ -48,9 +48,10 @@
  * - 1.20 - Trap handler support for expert scheduling mode available
  * - 1.21 - Debugger support to subscribe to LDS out-of-address exceptions
  * - 1.22 - Add queue creation with metadata ring base address
+ * - 1.23 - Add profiler control ioctl to enable/disable profiler on a process
  */
 #define KFD_IOCTL_MAJOR_VERSION 1
-#define KFD_IOCTL_MINOR_VERSION 22
+#define KFD_IOCTL_MINOR_VERSION 23
 
 struct kfd_ioctl_get_version_args {
 	__u32 major_version;	/* from KFD */
@@ -1558,6 +1559,36 @@ struct kfd_ioctl_dbg_trap_args {
 	};
 };
 
+#define KFD_IOC_PROFILER_VERSION_NUM 1
+enum kfd_profiler_ops {
+	KFD_IOC_PROFILER_PMC = 0,
+	KFD_IOC_PROFILER_VERSION = 2,
+	KFD_IOC_PROFILER_PTL_CONTROL = 3,
+};
+
+/**
+ * Enables/Disables GPU Specific profiler settings
+ */
+struct kfd_ioctl_pmc_settings {
+	__u32 gpu_id;             /* This is the user_gpu_id */
+	__u32 lock;               /* Lock GPU for Profiling */
+	__u32 perfcount_enable;   /* Force Perfcount Enable for queues on GPU */
+};
+
+struct kfd_ioctl_ptl_control {
+	__u32 gpu_id; /* user_gpu_id */
+	__u32 enable; /* set 1 to enable PTL, set 0 to disable PTL */
+};
+
+struct kfd_ioctl_profiler_args {
+	__u32 op;						/* kfd_profiler_op */
+	union {
+		struct kfd_ioctl_pmc_settings  pmc;
+		struct kfd_ioctl_ptl_control   ptl;
+		__u32 version;				/* KFD_IOC_PROFILER_VERSION_NUM */
+	};
+};
+
 #define AMDKFD_IOCTL_BASE 'K'
 #define AMDKFD_IO(nr)			_IO(AMDKFD_IOCTL_BASE, nr)
 #define AMDKFD_IOR(nr, type)		_IOR(AMDKFD_IOCTL_BASE, nr, type)
@@ -1681,7 +1712,10 @@ struct kfd_ioctl_dbg_trap_args {
 #define AMDKFD_IOC_CREATE_PROCESS		\
 		AMDKFD_IO(0x27)
 
+#define AMDKFD_IOC_PROFILER			\
+		AMDKFD_IOWR(0x28, struct kfd_ioctl_profiler_args)
+
 #define AMDKFD_COMMAND_START		0x01
-#define AMDKFD_COMMAND_END		0x28
+#define AMDKFD_COMMAND_END		0x29
 
 #endif
diff --git a/include/uapi/linux/liveupdate.h b/include/uapi/linux/liveupdate.h
index 30bc66ee9436a..3a9ff53b02e00 100644
--- a/include/uapi/linux/liveupdate.h
+++ b/include/uapi/linux/liveupdate.h
@@ -59,6 +59,7 @@ enum {
 	LIVEUPDATE_CMD_SESSION_PRESERVE_FD = LIVEUPDATE_CMD_SESSION_BASE,
 	LIVEUPDATE_CMD_SESSION_RETRIEVE_FD = 0x41,
 	LIVEUPDATE_CMD_SESSION_FINISH = 0x42,
+	LIVEUPDATE_CMD_SESSION_GET_NAME = 0x43,
 };
 
 /**
@@ -213,4 +214,24 @@ struct liveupdate_session_finish {
 #define LIVEUPDATE_SESSION_FINISH					\
 	_IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_SESSION_FINISH)
 
+/**
+ * struct liveupdate_session_get_name - ioctl(LIVEUPDATE_SESSION_GET_NAME)
+ * @size:  Input; sizeof(struct liveupdate_session_get_name)
+ * @reserved: Input; Must be zero. Reserved for future use.
+ * @name:  Output; A null-terminated string with the full session name.
+ *
+ * Retrieves the full name of the session associated with this file descriptor.
+ * This is useful because the kernel may truncate the name shown in /proc.
+ *
+ * Return: 0 on success, negative error code on failure.
+ */
+struct liveupdate_session_get_name {
+	__u32		size;
+	__u32		reserved;
+	__u8		name[LIVEUPDATE_SESSION_NAME_LENGTH];
+};
+
+#define LIVEUPDATE_SESSION_GET_NAME					\
+	_IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_SESSION_GET_NAME)
+
 #endif /* _UAPI_LIVEUPDATE_H */
diff --git a/include/uapi/linux/lsm.h b/include/uapi/linux/lsm.h
index 938593dfd5daf..2ff9bcdd551e2 100644
--- a/include/uapi/linux/lsm.h
+++ b/include/uapi/linux/lsm.h
@@ -65,6 +65,7 @@ struct lsm_ctx {
 #define LSM_ID_IMA		111
 #define LSM_ID_EVM		112
 #define LSM_ID_IPE		113
+#define LSM_ID_HORNET		114
 
 /*
  * LSM_ATTR_XXX definitions identify different LSM attributes
diff --git a/include/uapi/linux/mdio.h b/include/uapi/linux/mdio.h
index 8d769f100de60..b2541c948fc1e 100644
--- a/include/uapi/linux/mdio.h
+++ b/include/uapi/linux/mdio.h
@@ -23,6 +23,10 @@
 #define MDIO_MMD_DTEXS		5	/* DTE Extender Sublayer */
 #define MDIO_MMD_TC		6	/* Transmission Convergence */
 #define MDIO_MMD_AN		7	/* Auto-Negotiation */
+#define MDIO_MMD_SEP_PMA1	8	/* Separated PMA (1) */
+#define MDIO_MMD_SEP_PMA2	9	/* Separated PMA (2) */
+#define MDIO_MMD_SEP_PMA3	10	/* Separated PMA (3) */
+#define MDIO_MMD_SEP_PMA4	11	/* Separated PMA (4) */
 #define MDIO_MMD_POWER_UNIT	13	/* PHY Power Unit */
 #define MDIO_MMD_C22EXT		29	/* Clause 22 extension */
 #define MDIO_MMD_VEND1		30	/* Vendor specific 1 */
@@ -63,6 +67,8 @@
 					 * Lanes B-D are numbered 134-136. */
 #define MDIO_PMA_10GBR_FSRT_CSR	147	/* 10GBASE-R fast retrain status and control */
 #define MDIO_PMA_10GBR_FECABLE	170	/* 10GBASE-R FEC ability */
+#define MDIO_PMA_RSFEC_CTRL	200	/* RSFEC control */
+#define MDIO_PMA_RSFEC_LANE_MAP	206	/* RSFEC lane mapping */
 #define MDIO_PCS_10GBX_STAT1	24	/* 10GBASE-X PCS status 1 */
 #define MDIO_PCS_10GBRT_STAT1	32	/* 10GBASE-R/-T PCS status 1 */
 #define MDIO_PCS_10GBRT_STAT2	33	/* 10GBASE-R/-T PCS status 2 */
@@ -175,6 +181,10 @@
 #define MDIO_DEVS_DTEXS			MDIO_DEVS_PRESENT(MDIO_MMD_DTEXS)
 #define MDIO_DEVS_TC			MDIO_DEVS_PRESENT(MDIO_MMD_TC)
 #define MDIO_DEVS_AN			MDIO_DEVS_PRESENT(MDIO_MMD_AN)
+#define MDIO_DEVS_SEP_PMA1		MDIO_DEVS_PRESENT(MDIO_MMD_SEP_PMA1)
+#define MDIO_DEVS_SEP_PMA2		MDIO_DEVS_PRESENT(MDIO_MMD_SEP_PMA2)
+#define MDIO_DEVS_SEP_PMA3		MDIO_DEVS_PRESENT(MDIO_MMD_SEP_PMA3)
+#define MDIO_DEVS_SEP_PMA4		MDIO_DEVS_PRESENT(MDIO_MMD_SEP_PMA4)
 #define MDIO_DEVS_C22EXT		MDIO_DEVS_PRESENT(MDIO_MMD_C22EXT)
 #define MDIO_DEVS_VEND1			MDIO_DEVS_PRESENT(MDIO_MMD_VEND1)
 #define MDIO_DEVS_VEND2			MDIO_DEVS_PRESENT(MDIO_MMD_VEND2)
diff --git a/include/uapi/linux/nfsd_netlink.h b/include/uapi/linux/nfsd_netlink.h
index 97c7447f4d14d..060c43675599f 100644
--- a/include/uapi/linux/nfsd_netlink.h
+++ b/include/uapi/linux/nfsd_netlink.h
@@ -10,6 +10,53 @@
 #define NFSD_FAMILY_NAME	"nfsd"
 #define NFSD_FAMILY_VERSION	1
 
+enum nfsd_cache_type {
+	NFSD_CACHE_TYPE_SVC_EXPORT = 1,
+	NFSD_CACHE_TYPE_EXPKEY = 2,
+};
+
+/*
+ * These flags are ordered to match the NFSEXP_* flags in
+ * include/linux/nfsd/export.h
+ */
+enum nfsd_export_flags {
+	NFSD_EXPORT_FLAGS_READONLY = 1,
+	NFSD_EXPORT_FLAGS_INSECURE_PORT = 2,
+	NFSD_EXPORT_FLAGS_ROOTSQUASH = 4,
+	NFSD_EXPORT_FLAGS_ALLSQUASH = 8,
+	NFSD_EXPORT_FLAGS_ASYNC = 16,
+	NFSD_EXPORT_FLAGS_GATHERED_WRITES = 32,
+	NFSD_EXPORT_FLAGS_NOREADDIRPLUS = 64,
+	NFSD_EXPORT_FLAGS_SECURITY_LABEL = 128,
+	NFSD_EXPORT_FLAGS_SIGN_FH = 256,
+	NFSD_EXPORT_FLAGS_NOHIDE = 512,
+	NFSD_EXPORT_FLAGS_NOSUBTREECHECK = 1024,
+	NFSD_EXPORT_FLAGS_NOAUTHNLM = 2048,
+	NFSD_EXPORT_FLAGS_MSNFS = 4096,
+	NFSD_EXPORT_FLAGS_FSID = 8192,
+	NFSD_EXPORT_FLAGS_CROSSMOUNT = 16384,
+	NFSD_EXPORT_FLAGS_NOACL = 32768,
+	NFSD_EXPORT_FLAGS_V4ROOT = 65536,
+	NFSD_EXPORT_FLAGS_PNFS = 131072,
+};
+
+/*
+ * These flags are ordered to match the NFSEXP_XPRTSEC_* flags in
+ * include/linux/nfsd/export.h
+ */
+enum nfsd_xprtsec_mode {
+	NFSD_XPRTSEC_MODE_NONE = 1,
+	NFSD_XPRTSEC_MODE_TLS = 2,
+	NFSD_XPRTSEC_MODE_MTLS = 4,
+};
+
+enum {
+	NFSD_A_CACHE_NOTIFY_CACHE_TYPE = 1,
+
+	__NFSD_A_CACHE_NOTIFY_MAX,
+	NFSD_A_CACHE_NOTIFY_MAX = (__NFSD_A_CACHE_NOTIFY_MAX - 1)
+};
+
 enum {
 	NFSD_A_RPC_STATUS_XID = 1,
 	NFSD_A_RPC_STATUS_FLAGS,
@@ -81,6 +128,82 @@ enum {
 	NFSD_A_POOL_MODE_MAX = (__NFSD_A_POOL_MODE_MAX - 1)
 };
 
+enum {
+	NFSD_A_FSLOCATION_HOST = 1,
+	NFSD_A_FSLOCATION_PATH,
+
+	__NFSD_A_FSLOCATION_MAX,
+	NFSD_A_FSLOCATION_MAX = (__NFSD_A_FSLOCATION_MAX - 1)
+};
+
+enum {
+	NFSD_A_FSLOCATIONS_LOCATION = 1,
+
+	__NFSD_A_FSLOCATIONS_MAX,
+	NFSD_A_FSLOCATIONS_MAX = (__NFSD_A_FSLOCATIONS_MAX - 1)
+};
+
+enum {
+	NFSD_A_AUTH_FLAVOR_PSEUDOFLAVOR = 1,
+	NFSD_A_AUTH_FLAVOR_FLAGS,
+
+	__NFSD_A_AUTH_FLAVOR_MAX,
+	NFSD_A_AUTH_FLAVOR_MAX = (__NFSD_A_AUTH_FLAVOR_MAX - 1)
+};
+
+enum {
+	NFSD_A_SVC_EXPORT_SEQNO = 1,
+	NFSD_A_SVC_EXPORT_CLIENT,
+	NFSD_A_SVC_EXPORT_PATH,
+	NFSD_A_SVC_EXPORT_NEGATIVE,
+	NFSD_A_SVC_EXPORT_EXPIRY,
+	NFSD_A_SVC_EXPORT_ANON_UID,
+	NFSD_A_SVC_EXPORT_ANON_GID,
+	NFSD_A_SVC_EXPORT_FSLOCATIONS,
+	NFSD_A_SVC_EXPORT_UUID,
+	NFSD_A_SVC_EXPORT_SECINFO,
+	NFSD_A_SVC_EXPORT_XPRTSEC,
+	NFSD_A_SVC_EXPORT_FLAGS,
+	NFSD_A_SVC_EXPORT_FSID,
+
+	__NFSD_A_SVC_EXPORT_MAX,
+	NFSD_A_SVC_EXPORT_MAX = (__NFSD_A_SVC_EXPORT_MAX - 1)
+};
+
+enum {
+	NFSD_A_SVC_EXPORT_REQS_REQUESTS = 1,
+
+	__NFSD_A_SVC_EXPORT_REQS_MAX,
+	NFSD_A_SVC_EXPORT_REQS_MAX = (__NFSD_A_SVC_EXPORT_REQS_MAX - 1)
+};
+
+enum {
+	NFSD_A_EXPKEY_SEQNO = 1,
+	NFSD_A_EXPKEY_CLIENT,
+	NFSD_A_EXPKEY_FSIDTYPE,
+	NFSD_A_EXPKEY_FSID,
+	NFSD_A_EXPKEY_NEGATIVE,
+	NFSD_A_EXPKEY_EXPIRY,
+	NFSD_A_EXPKEY_PATH,
+
+	__NFSD_A_EXPKEY_MAX,
+	NFSD_A_EXPKEY_MAX = (__NFSD_A_EXPKEY_MAX - 1)
+};
+
+enum {
+	NFSD_A_EXPKEY_REQS_REQUESTS = 1,
+
+	__NFSD_A_EXPKEY_REQS_MAX,
+	NFSD_A_EXPKEY_REQS_MAX = (__NFSD_A_EXPKEY_REQS_MAX - 1)
+};
+
+enum {
+	NFSD_A_CACHE_FLUSH_MASK = 1,
+
+	__NFSD_A_CACHE_FLUSH_MAX,
+	NFSD_A_CACHE_FLUSH_MAX = (__NFSD_A_CACHE_FLUSH_MAX - 1)
+};
+
 enum {
 	NFSD_CMD_RPC_STATUS_GET = 1,
 	NFSD_CMD_THREADS_SET,
@@ -91,9 +214,18 @@ enum {
 	NFSD_CMD_LISTENER_GET,
 	NFSD_CMD_POOL_MODE_SET,
 	NFSD_CMD_POOL_MODE_GET,
+	NFSD_CMD_CACHE_NOTIFY,
+	NFSD_CMD_SVC_EXPORT_GET_REQS,
+	NFSD_CMD_SVC_EXPORT_SET_REQS,
+	NFSD_CMD_EXPKEY_GET_REQS,
+	NFSD_CMD_EXPKEY_SET_REQS,
+	NFSD_CMD_CACHE_FLUSH,
 
 	__NFSD_CMD_MAX,
 	NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1)
 };
 
+#define NFSD_MCGRP_NONE		"none"
+#define NFSD_MCGRP_EXPORTD	"exportd"
+
 #endif /* _UAPI_LINUX_NFSD_NETLINK_H */
diff --git a/include/uapi/linux/nl80211.h b/include/uapi/linux/nl80211.h
index 3d55bf4be36fe..9998f6c0a665c 100644
--- a/include/uapi/linux/nl80211.h
+++ b/include/uapi/linux/nl80211.h
@@ -729,7 +729,9 @@
  *	to remain on the channel. This command is also used as an event to
  *	notify when the requested duration starts (it may take a while for the
  *	driver to schedule this time due to other concurrent needs for the
- *	radio).
+ *	radio). An optional attribute %NL80211_ATTR_MAC can be used to filter
+ *	incoming frames during remain-on-channel, such that frames
+ *	addressed to the specified destination MAC are reported.
  *	When called, this operation returns a cookie (%NL80211_ATTR_COOKIE)
  *	that will be included with any events pertaining to this request;
  *	the cookie is also used to cancel the request.
@@ -1204,10 +1206,12 @@
  *	user space through the connect result as the user space would have
  *	initiated the connection through the connect request.
  *
- * @NL80211_CMD_STA_OPMODE_CHANGED: An event that notify station's
- *	ht opmode or vht opmode changes using any of %NL80211_ATTR_SMPS_MODE,
- *	%NL80211_ATTR_CHANNEL_WIDTH,%NL80211_ATTR_NSS attributes with its
- *	address(specified in %NL80211_ATTR_MAC).
+ * @NL80211_CMD_STA_OPMODE_CHANGED: An event that notifies that a station's
+ *	HT opmode or VHT opmode changed using any of %NL80211_ATTR_SMPS_MODE,
+ *	%NL80211_ATTR_CHANNEL_WIDTH, %NL80211_ATTR_NSS attributes with its
+ *	address (specified in %NL80211_ATTR_MAC).
+ *	Note that 80+80 and 160 MHz might not be differentiated, i.e. may
+ *	report %NL80211_CHAN_WIDTH_160 instead of %NL80211_CHAN_WIDTH_80P80.
  *
  * @NL80211_CMD_GET_FTM_RESPONDER_STATS: Retrieve FTM responder statistics, in
  *	the %NL80211_ATTR_FTM_RESPONDER_STATS attribute.
@@ -1415,6 +1419,12 @@
  *	identifying the evacuated channel.
  *	User space may reconfigure the local schedule in response to this
  *	notification.
+ * @NL80211_CMD_START_PD: Start PD operation, identified by its
+ *	%NL80211_ATTR_WDEV interface. This interface must have been previously
+ *	created with %NL80211_CMD_NEW_INTERFACE.
+ * @NL80211_CMD_STOP_PD: Stop the PD operation, identified by
+ *	its %NL80211_ATTR_WDEV interface.
+ *
  * @NL80211_CMD_MAX: highest used command number
  * @__NL80211_CMD_AFTER_LAST: internal use
  */
@@ -1690,6 +1700,9 @@ enum nl80211_commands {
 
 	NL80211_CMD_NAN_CHANNEL_EVAC,
 
+	NL80211_CMD_START_PD,
+	NL80211_CMD_STOP_PD,
+
 	/* add new commands above here */
 
 	/* used to define NL80211_CMD_MAX below */
@@ -2991,11 +3004,13 @@ enum nl80211_commands {
  * @NL80211_ATTR_EPCS: Flag attribute indicating that EPCS is enabled for a
  *	station interface.
  *
- * @NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS: Extended MLD capabilities and
- *	operations that userspace implements to use during association/ML
- *	link reconfig, currently only "BTM MLD Recommendation For Multiple
- *	APs Support". Drivers may set additional flags that they support
- *	in the kernel or device.
+ * @NL80211_ATTR_EXT_MLD_CAPA_AND_OPS: Extended MLD capabilities and operations.
+ *	For association and link reconfiguration, indicates extra capabilities
+ *	that userspace implements, currently only "BTM MLD Recommendation For
+ *	Multiple APs Support".
+ *	For wiphy information, additional flags that drivers will set, but
+ *	this is informational only for userspace (it's not expected to set
+ *	these.)
  *
  * @NL80211_ATTR_WIPHY_RADIO_INDEX: (int) Integer attribute denoting the index
  *	of the radio in interest. Internally a value of -1 is used to
@@ -3140,6 +3155,16 @@ enum nl80211_commands {
  *	association response etc., since it's abridged in the beacon. Used
  *	for START_AP etc.
  *
+ * @NL80211_ATTR_ASSOC_ENCRYPTED: Flag attribute, used only with the
+ *	%NL80211_CMD_CONNECT event in SME-in-driver mode. The driver should
+ *	set this flag to indicate that both the (Re)Association Request frame
+ *	and the corresponding (Re)Association Response frame are transmitted
+ *	encrypted over the air. Enhanced Privacy Protection (EPP), as defined
+ *	in IEEE P802.11bi/D4.0, mandates this encryption.
+ *
+ * @NL80211_ATTR_NPCA_PRIMARY_FREQ: NPCA primary channel (u32)
+ * @NL80211_ATTR_NPCA_PUNCT_BITMAP: NPCA puncturing bitmap (u32)
+ *
  * @NUM_NL80211_ATTR: total number of nl80211_attrs available
  * @NL80211_ATTR_MAX: highest attribute number currently defined
  * @__NL80211_ATTR_AFTER_LAST: internal use
@@ -3695,7 +3720,7 @@ enum nl80211_attrs {
 	NL80211_ATTR_MLO_RECONF_REM_LINKS,
 	NL80211_ATTR_EPCS,
 
-	NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS,
+	NL80211_ATTR_EXT_MLD_CAPA_AND_OPS,
 
 	NL80211_ATTR_WIPHY_RADIO_INDEX,
 
@@ -3733,6 +3758,11 @@ enum nl80211_attrs {
 	NL80211_ATTR_NAN_MAX_CHAN_SWITCH_TIME,
 	NL80211_ATTR_NAN_PEER_MAPS,
 
+	NL80211_ATTR_ASSOC_ENCRYPTED,
+
+	NL80211_ATTR_NPCA_PRIMARY_FREQ,
+	NL80211_ATTR_NPCA_PUNCT_BITMAP,
+
 	/* add attributes here, update the policy in nl80211.c */
 
 	__NL80211_ATTR_AFTER_LAST,
@@ -3747,6 +3777,7 @@ enum nl80211_attrs {
 #define NL80211_ATTR_SAE_DATA NL80211_ATTR_AUTH_DATA
 #define NL80211_ATTR_CSA_C_OFF_BEACON NL80211_ATTR_CNTDWN_OFFS_BEACON
 #define NL80211_ATTR_CSA_C_OFF_PRESP NL80211_ATTR_CNTDWN_OFFS_PRESP
+#define NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS NL80211_ATTR_EXT_MLD_CAPA_AND_OPS
 
 /*
  * Allow user space programs to use #ifdef on new attributes by defining them
@@ -3829,6 +3860,7 @@ enum nl80211_attrs {
  * @NL80211_IFTYPE_NAN_DATA: NAN data interface type (netdev); NAN data
  *	interfaces can only be brought up (IFF_UP) when a NAN interface
  *	already exists and NAN has been started (using %NL80211_CMD_START_NAN).
+ * @NL80211_IFTYPE_PD: PD device interface type (not a netdev)
  * @NL80211_IFTYPE_MAX: highest interface type number currently defined
  * @NUM_NL80211_IFTYPES: number of defined interface types
  *
@@ -3851,6 +3883,7 @@ enum nl80211_iftype {
 	NL80211_IFTYPE_OCB,
 	NL80211_IFTYPE_NAN,
 	NL80211_IFTYPE_NAN_DATA,
+	NL80211_IFTYPE_PD,
 
 	/* keep last */
 	NUM_NL80211_IFTYPES,
@@ -5790,6 +5823,18 @@ enum nl80211_key_default_types {
  * @NL80211_KEY_MODE: the mode from enum nl80211_key_mode.
  *	Defaults to @NL80211_KEY_RX_TX.
  * @NL80211_KEY_DEFAULT_BEACON: flag indicating default Beacon frame key
+ * @NL80211_KEY_LTF_SEED: LTF key seed is used by the driver to generate
+ *	secure LTF keys used in case of peer measurement request with FTM
+ *	request type as either %NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED
+ *	or %NL80211_PMSR_FTM_REQ_ATTR_TRIGGER_BASED. Secure LTF key seeds
+ *	will help enable PHY security in peer measurement session.
+ *	The LTF key seed is installed along with the TK (Temporal Key) using
+ *	%NL80211_CMD_NEW_KEY. The TK is configured using the
+ *	%NL80211_ATTR_KEY_DATA attribute, while the LTF key seed is configured
+ *	using this attribute. Both keys must be	configured before initiation
+ *	of peer measurement to ensure peer measurement session is secure.
+ *	Only valid if %NL80211_EXT_FEATURE_SET_KEY_LTF_SEED is set. This
+ *	attribute is restricted to pairwise keys (%NL80211_KEYTYPE_PAIRWISE).
  *
  * @__NL80211_KEY_AFTER_LAST: internal
  * @NL80211_KEY_MAX: highest key attribute
@@ -5806,6 +5851,7 @@ enum nl80211_key_attributes {
 	NL80211_KEY_DEFAULT_TYPES,
 	NL80211_KEY_MODE,
 	NL80211_KEY_DEFAULT_BEACON,
+	NL80211_KEY_LTF_SEED,
 
 	/* keep last */
 	__NL80211_KEY_AFTER_LAST,
@@ -7029,6 +7075,16 @@ enum nl80211_feature_flags {
  *	(NL80211_CMD_AUTHENTICATE) in non-AP STA mode, as specified in
  *	"IEEE P802.11bi/D4.0, 12.16.5".
  *
+ * @NL80211_EXT_FEATURE_ROC_ADDR_FILTER: Driver supports MAC address
+ *	filtering during remain-on-channel. When %NL80211_ATTR_MAC is
+ *	provided with %NL80211_CMD_REMAIN_ON_CHANNEL, the driver will
+ *	forward frames with a matching MAC address to userspace during
+ *	the off-channel period.
+ *
+ * @NL80211_EXT_FEATURE_SET_KEY_LTF_SEED: Driver supports installing the
+ *	LTF key seed via %NL80211_KEY_LTF_SEED. The seed is used to generate
+ *	secure LTF keys for secure LTF measurement sessions.
+ *
  * @NUM_NL80211_EXT_FEATURES: number of extended features.
  * @MAX_NL80211_EXT_FEATURES: highest extended feature index.
  */
@@ -7108,6 +7164,8 @@ enum nl80211_ext_feature_index {
 	NL80211_EXT_FEATURE_EPPKE,
 	NL80211_EXT_FEATURE_ASSOC_FRAME_ENCRYPTION,
 	NL80211_EXT_FEATURE_IEEE8021X_AUTH,
+	NL80211_EXT_FEATURE_ROC_ADDR_FILTER,
+	NL80211_EXT_FEATURE_SET_KEY_LTF_SEED,
 
 	/* add new features before the definition below */
 	NUM_NL80211_EXT_FEATURES,
@@ -7960,6 +8018,26 @@ enum nl80211_peer_measurement_resp {
 	NL80211_PMSR_RESP_ATTR_MAX = NUM_NL80211_PMSR_RESP_ATTRS - 1
 };
 
+/**
+ * enum nl80211_peer_measurement_ftm_req_type - FTM ranging request type,
+ *	used with %NL80211_PMSR_PEER_ATTR_REQ_TYPE
+ *
+ * @NL80211_PMSR_FTM_REQ_TYPE_INFRA: infrastructure ranging, i.e. STA-to-AP
+ * @NL80211_PMSR_FTM_REQ_TYPE_PD: peer-to-peer ranging as defined in the
+ *	Wi-Fi Alliance specification "Proximity Ranging (PR) Implementation
+ *	Consideration Draft 1.9 Rev 1"
+ * @NUM_NL80211_PMSR_FTM_REQ_TYPE: internal
+ * @NL80211_PMSR_FTM_REQ_TYPE_MAX: highest request type value
+ */
+enum nl80211_peer_measurement_ftm_req_type {
+	NL80211_PMSR_FTM_REQ_TYPE_INFRA,
+	NL80211_PMSR_FTM_REQ_TYPE_PD,
+
+	/* keep last */
+	NUM_NL80211_PMSR_FTM_REQ_TYPE,
+	NL80211_PMSR_FTM_REQ_TYPE_MAX = NUM_NL80211_PMSR_FTM_REQ_TYPE - 1
+};
+
 /**
  * enum nl80211_peer_measurement_peer_attrs - peer attributes for measurement
  * @__NL80211_PMSR_PEER_ATTR_INVALID: invalid
@@ -7973,6 +8051,9 @@ enum nl80211_peer_measurement_resp {
  * @NL80211_PMSR_PEER_ATTR_RESP: This is a nested attribute indexed by
  *	measurement type, with attributes from the
  *	&enum nl80211_peer_measurement_resp inside.
+ * @NL80211_PMSR_PEER_ATTR_REQ_TYPE: u32 attribute specifying the ranging
+ *	request type, using values from &enum nl80211_peer_measurement_ftm_req_type.
+ *	If absent, defaults to %NL80211_PMSR_FTM_REQ_TYPE_INFRA.
  *
  * @NUM_NL80211_PMSR_PEER_ATTRS: internal
  * @NL80211_PMSR_PEER_ATTR_MAX: highest attribute number
@@ -7984,6 +8065,7 @@ enum nl80211_peer_measurement_peer_attrs {
 	NL80211_PMSR_PEER_ATTR_CHAN,
 	NL80211_PMSR_PEER_ATTR_REQ,
 	NL80211_PMSR_PEER_ATTR_RESP,
+	NL80211_PMSR_PEER_ATTR_REQ_TYPE,
 
 	/* keep last */
 	NUM_NL80211_PMSR_PEER_ATTRS,
@@ -8010,6 +8092,18 @@ enum nl80211_peer_measurement_peer_attrs {
  *	meaningless, just a list of peers to measure with, with the
  *	sub-attributes taken from
  *	&enum nl80211_peer_measurement_peer_attrs.
+ * @NL80211_PMSR_ATTR_MAX_PEER_ISTA_ROLE: u32 attribute indicating the
+ *	maximum number of peers supported when the device operates in the
+ *	ISTA (Initiator STA) role. If absent, no role-specific peer limit
+ *	applies. The sum of %NL80211_PMSR_ATTR_MAX_PEER_ISTA_ROLE and
+ *	%NL80211_PMSR_ATTR_MAX_PEER_RSTA_ROLE is enforced when the device
+ *	supports concurrent ISTA/RSTA operation.
+ * @NL80211_PMSR_ATTR_MAX_PEER_RSTA_ROLE: u32 attribute indicating the
+ *	maximum number of peers supported when the device operates in the
+ *	RSTA (Responder STA) role. If absent, no role-specific peer limit
+ *	applies. The sum of %NL80211_PMSR_ATTR_MAX_PEER_ISTA_ROLE and
+ *	%NL80211_PMSR_ATTR_MAX_PEER_RSTA_ROLE is enforced when the device
+ *	supports concurrent ISTA/RSTA operation.
  *
  * @NUM_NL80211_PMSR_ATTR: internal
  * @NL80211_PMSR_ATTR_MAX: highest attribute number
@@ -8022,6 +8116,8 @@ enum nl80211_peer_measurement_attrs {
 	NL80211_PMSR_ATTR_RANDOMIZE_MAC_ADDR,
 	NL80211_PMSR_ATTR_TYPE_CAPA,
 	NL80211_PMSR_ATTR_PEERS,
+	NL80211_PMSR_ATTR_MAX_PEER_ISTA_ROLE,
+	NL80211_PMSR_ATTR_MAX_PEER_RSTA_ROLE,
 
 	/* keep last */
 	NUM_NL80211_PMSR_ATTR,
@@ -8080,6 +8176,54 @@ enum nl80211_peer_measurement_attrs {
  *	This limits the allowed combinations of LTF repetitions and STS.
  * @NL80211_PMSR_FTM_CAPA_ATTR_RSTA_SUPPORT: flag attribute indicating the
  *	device supports operating as the RSTA in PMSR FTM request
+ * @NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS: nested attribute containing ISTA
+ *	(initiator) role capabilities. Uses the same sub-attributes as
+ *	%NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS: nested attribute containing RSTA
+ *	(responder) role capabilities.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_NTB: flag attribute (used inside
+ *	%NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS or
+ *	%NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS) indicating NTB ranging support.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_TB: flag attribute (used inside
+ *	%NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS or
+ *	%NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS) indicating TB ranging support.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_EDCA: flag attribute (used inside
+ *	%NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS or
+ *	%NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS) indicating EDCA based ranging
+ *	support.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_TYPE_CAPS: nested attribute containing ranging
+ *	type capabilities. Uses sub-attributes from
+ *	&enum nl80211_peer_measurement_ftm_type_capa.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_CONCURRENT_ISTA_RSTA_SUPPORT: flag attribute
+ *	indicating that the device can simultaneously act as initiator and
+ *	responder in a multi-peer measurement request. Only valid if
+ *	@NL80211_PMSR_FTM_CAPA_ATTR_RSTA_SUPPORT is set.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_TX_ANTENNAS: u32 attribute indicating
+ *	the maximum number of transmit antennas supported for EDCA based ranging
+ *	(0 means unknown)
+ * @NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_RX_ANTENNAS: u32 attribute indicating
+ *	the maximum number of receive antennas supported for EDCA based ranging
+ *	(0 means unknown)
+ * @NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_EDCA: u32 attribute indicating
+ *	the minimum EDCA ranging interval supported by the device
+ *	in milli seconds. (0 means unknown). Applications can use this value
+ *	to estimate the burst period to be given in the FTM request for the
+ *	EDCA based ranging case. If non-zero, this value will be used to
+ *	validate the burst period in the FTM request.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_NTB: u32 attribute indicating
+ *	the minimum NTB ranging interval supported by the device
+ *	in milli seconds. (0 means unknown). Applications can use this value
+ *	to estimate the burst period to be given in the FTM request for the
+ *	NTB ranging case. If non-zero, this value will be used to validate
+ *	the nominal time in the FTM request.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_PD_PREAMBLES: u32 bitmap of values from
+ *	&enum nl80211_preamble indicating the supported preambles for PD
+ *	ranging requests. Only valid if %NL80211_PMSR_FTM_TYPE_CAPA_ATTR_PD_SUPPORT
+ *	is set.
+ * @NL80211_PMSR_FTM_CAPA_ATTR_PD_BANDWIDTHS: u32 bitmap of values from
+ *	&enum nl80211_chan_width indicating the supported channel bandwidths
+ *	for PD ranging requests. Only valid if
+ *	%NL80211_PMSR_FTM_TYPE_CAPA_ATTR_PD_SUPPORT is set.
  *
  * @NUM_NL80211_PMSR_FTM_CAPA_ATTR: internal
  * @NL80211_PMSR_FTM_CAPA_ATTR_MAX: highest attribute number
@@ -8105,12 +8249,52 @@ enum nl80211_peer_measurement_ftm_capa {
 	NL80211_PMSR_FTM_CAPA_ATTR_MAX_TOTAL_LTF_TX,
 	NL80211_PMSR_FTM_CAPA_ATTR_MAX_TOTAL_LTF_RX,
 	NL80211_PMSR_FTM_CAPA_ATTR_RSTA_SUPPORT,
+	NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS,
+	NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS,
+	NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_NTB,
+	NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_TB,
+	NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_EDCA,
+	NL80211_PMSR_FTM_CAPA_ATTR_TYPE_CAPS,
+	NL80211_PMSR_FTM_CAPA_ATTR_CONCURRENT_ISTA_RSTA_SUPPORT,
+	NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_TX_ANTENNAS,
+	NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_RX_ANTENNAS,
+	NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_EDCA,
+	NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_NTB,
+	NL80211_PMSR_FTM_CAPA_ATTR_PD_PREAMBLES,
+	NL80211_PMSR_FTM_CAPA_ATTR_PD_BANDWIDTHS,
 
 	/* keep last */
 	NUM_NL80211_PMSR_FTM_CAPA_ATTR,
 	NL80211_PMSR_FTM_CAPA_ATTR_MAX = NUM_NL80211_PMSR_FTM_CAPA_ATTR - 1
 };
 
+/**
+ * enum nl80211_peer_measurement_ftm_type_capa - FTM ranging type capability
+ *	sub-attributes, used inside %NL80211_PMSR_FTM_CAPA_ATTR_TYPE_CAPS
+ * @__NL80211_PMSR_FTM_TYPE_CAPA_ATTR_INVALID: invalid
+ *
+ * @NL80211_PMSR_FTM_TYPE_CAPA_ATTR_INFRA_SUPPORT: flag attribute indicating
+ *	that the device supports infrastructure ranging (STA-to-AP or
+ *	AP-to-STA) as part of Proximity Detection
+ * @NL80211_PMSR_FTM_TYPE_CAPA_ATTR_PD_SUPPORT: flag attribute indicating that
+ *	the device supports peer-to-peer ranging as mentioned in the
+ *	specification "PR Implementation Consideration Draft 1.9 rev 1" where
+ *	PD stands for proximity detection
+ *
+ * @NUM_NL80211_PMSR_FTM_TYPE_CAPA_ATTR: internal
+ * @NL80211_PMSR_FTM_TYPE_CAPA_ATTR_MAX: highest attribute number
+ */
+enum nl80211_peer_measurement_ftm_type_capa {
+	__NL80211_PMSR_FTM_TYPE_CAPA_ATTR_INVALID,
+
+	NL80211_PMSR_FTM_TYPE_CAPA_ATTR_INFRA_SUPPORT,
+	NL80211_PMSR_FTM_TYPE_CAPA_ATTR_PD_SUPPORT,
+
+	/* keep last */
+	NUM_NL80211_PMSR_FTM_TYPE_CAPA_ATTR,
+	NL80211_PMSR_FTM_TYPE_CAPA_ATTR_MAX = NUM_NL80211_PMSR_FTM_TYPE_CAPA_ATTR - 1
+};
+
 /**
  * enum nl80211_peer_measurement_ftm_req - FTM request attributes
  * @__NL80211_PMSR_FTM_REQ_ATTR_INVALID: invalid
@@ -8129,9 +8313,11 @@ enum nl80211_peer_measurement_ftm_capa {
  *	default 15 i.e. "no preference"). For non-EDCA ranging, this is the
  *	burst duration in milliseconds (optional with default 0, i.e. let the
  *	device decide).
- * @NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST: number of successful FTM frames
- *	requested per burst
+ * @NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST: (Optional) number of successful
+ *	FTM frames requested per burst
  *	(u8, 0-31, optional with default 0 i.e. "no preference")
+ *	If the attribute is absent ("no preference"), the driver or firmware can
+ *	choose a suitable value.
  * @NL80211_PMSR_FTM_REQ_ATTR_NUM_FTMR_RETRIES: number of FTMR frame retries
  *	(u8, default 3)
  * @NL80211_PMSR_FTM_REQ_ATTR_REQUEST_LCI: request LCI data (flag)
@@ -8165,6 +8351,50 @@ enum nl80211_peer_measurement_ftm_capa {
  *	Only valid if %NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK is set (so the
  *	RSTA will have the measurement results to report back in the FTM
  *	response).
+ * @NL80211_PMSR_FTM_REQ_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS: minimum time
+ *	between two consecutive range measurements in units of 100 microseconds,
+ *	for non-trigger based ranging (u32). Should be set as short as possible
+ *	to minimize turnaround time, since two-way ranging with delayed LMR
+ *	requires two measurements. Only valid if
+ *	%NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED is set.
+ * @NL80211_PMSR_FTM_REQ_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS: maximum time
+ *	between two consecutive range measurements in units of 10 milliseconds,
+ *	for non-trigger based ranging (u32). Acts as a session timeout; if
+ *	exceeded, the ranging session should be terminated. Only valid if
+ *	%NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED is set.
+ * @NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME: The nominal time field shall be
+ *	set to the nominal duration between adjacent Availability Windows in
+ *	units of milli seconds (u32). Mandatory if
+ *	%NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED is set.
+ * @NL80211_PMSR_FTM_REQ_ATTR_AW_DURATION: (Optional) The AW duration field
+ *	shall be set to the duration of the AW in units of 1ms (0-255 ms) (u32).
+ *	Only valid if %NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED is set.
+ *	If the attribute is absent ("no preference"), the driver or firmware
+ *	can choose a suitable value.
+ * @NL80211_PMSR_FTM_REQ_ATTR_NUM_MEASUREMENTS: (Optional) number of
+ *	Availability Windows (AWs) to schedule for non-trigger-based ranging.
+ *	Each AW may contain multiple FTM exchanges as configured by
+ *	%NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST. Only valid if
+ *	%NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED is set.
+ *	If the attribute is absent ("no preference"), the driver or firmware
+ *	can choose a suitable value.
+ * @NL80211_PMSR_FTM_REQ_ATTR_PAD: ignore, for u64/s64 padding only.
+ * @NL80211_PMSR_FTM_REQ_ATTR_INGRESS: optional u64 attribute in units of mm.
+ *	When specified, the measurement result of the peer needs to be
+ *	indicated if the device moves into this range.
+ * @NL80211_PMSR_FTM_REQ_ATTR_EGRESS: optional u64 attribute in units of mm.
+ *	When specified, the measurement result of the peer needs to be
+ *	indicated if the device moves out of this range.
+ *	If neither or only one of @NL80211_PMSR_FTM_REQ_ATTR_INGRESS and
+ *	@NL80211_PMSR_FTM_REQ_ATTR_EGRESS is specified, only the specified
+ *	threshold is used. If both are specified, both thresholds are applied.
+ *	If neither is specified, results are reported without threshold
+ *	filtering.
+ * @NL80211_PMSR_FTM_REQ_ATTR_PD_SUPPRESS_RESULTS: Flag to suppress ranging
+ *	results for PD requests. When set, ranging measurements are performed
+ *	but results are not reported to userspace, regardless of ranging role
+ *	or type. Only valid when %NL80211_PMSR_PEER_ATTR_REQ_TYPE is set to
+ *	%NL80211_PMSR_FTM_REQ_TYPE_PD.
  *
  * @NUM_NL80211_PMSR_FTM_REQ_ATTR: internal
  * @NL80211_PMSR_FTM_REQ_ATTR_MAX: highest attribute number
@@ -8186,6 +8416,15 @@ enum nl80211_peer_measurement_ftm_req {
 	NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK,
 	NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR,
 	NL80211_PMSR_FTM_REQ_ATTR_RSTA,
+	NL80211_PMSR_FTM_REQ_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS,
+	NL80211_PMSR_FTM_REQ_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS,
+	NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME,
+	NL80211_PMSR_FTM_REQ_ATTR_AW_DURATION,
+	NL80211_PMSR_FTM_REQ_ATTR_NUM_MEASUREMENTS,
+	NL80211_PMSR_FTM_REQ_ATTR_PAD,
+	NL80211_PMSR_FTM_REQ_ATTR_INGRESS,
+	NL80211_PMSR_FTM_REQ_ATTR_EGRESS,
+	NL80211_PMSR_FTM_REQ_ATTR_PD_SUPPRESS_RESULTS,
 
 	/* keep last */
 	NUM_NL80211_PMSR_FTM_REQ_ATTR,
@@ -8272,6 +8511,33 @@ enum nl80211_peer_measurement_ftm_failure_reasons {
  * @NL80211_PMSR_FTM_RESP_ATTR_PAD: ignore, for u64/s64 padding only
  * @NL80211_PMSR_FTM_RESP_ATTR_BURST_PERIOD: actual burst period used by
  *	the responder (similar to request, u16)
+ * @NL80211_PMSR_FTM_RESP_ATTR_TX_LTF_REPETITION_COUNT: negotiated value of
+ *	number of tx ltf repetitions in NDP frames (u32, optional)
+ * @NL80211_PMSR_FTM_RESP_ATTR_RX_LTF_REPETITION_COUNT: negotiated value of
+ *	number of rx ltf repetitions in NDP frames (u32, optional)
+ * @NL80211_PMSR_FTM_RESP_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS: negotiated value
+ *	where latest time by which the ISTA needs to complete the next round of
+ *	measurements, in units of 10 ms (u32, optional)
+ * @NL80211_PMSR_FTM_RESP_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS: negotiated
+ *	minimum time between two consecutive range measurements initiated by an
+ *	ISTA, in units of 100 us (u32, optional)
+ * @NL80211_PMSR_FTM_RESP_ATTR_NUM_TX_SPATIAL_STREAMS: number of Tx space-time
+ *	streams used in NDP frames during the measurement sounding phase
+ *	(u32, optional).
+ * @NL80211_PMSR_FTM_RESP_ATTR_NUM_RX_SPATIAL_STREAMS: number of Rx space-time
+ *	streams used in the NDP frames during the measurement sounding phase
+ *	(u32, optional)
+ * @NL80211_PMSR_FTM_RESP_ATTR_NOMINAL_TIME: negotiated nominal time used in
+ *	this session in milliseconds. (u32, optional)
+ * @NL80211_PMSR_FTM_RESP_ATTR_AVAILABILITY_WINDOW: negotiated availability
+ *	window time used in this session, in units of milli seconds.
+ *	(u32, optional)
+ * @NL80211_PMSR_FTM_RESP_ATTR_CHANNEL_WIDTH: u32 attribute indicating channel
+ *	width used for measurement, see &enum nl80211_chan_width (optional).
+ * @NL80211_PMSR_FTM_RESP_ATTR_PREAMBLE: u32 attribute indicating the preamble
+ *	type used for the measurement, see &enum nl80211_preamble (optional).
+ * @NL80211_PMSR_FTM_RESP_ATTR_IS_DELAYED_LMR: flag, indicates if the
+ *	current result is delayed LMR data.
  *
  * @NUM_NL80211_PMSR_FTM_RESP_ATTR: internal
  * @NL80211_PMSR_FTM_RESP_ATTR_MAX: highest attribute number
@@ -8301,6 +8567,17 @@ enum nl80211_peer_measurement_ftm_resp {
 	NL80211_PMSR_FTM_RESP_ATTR_CIVICLOC,
 	NL80211_PMSR_FTM_RESP_ATTR_PAD,
 	NL80211_PMSR_FTM_RESP_ATTR_BURST_PERIOD,
+	NL80211_PMSR_FTM_RESP_ATTR_TX_LTF_REPETITION_COUNT,
+	NL80211_PMSR_FTM_RESP_ATTR_RX_LTF_REPETITION_COUNT,
+	NL80211_PMSR_FTM_RESP_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS,
+	NL80211_PMSR_FTM_RESP_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS,
+	NL80211_PMSR_FTM_RESP_ATTR_NUM_TX_SPATIAL_STREAMS,
+	NL80211_PMSR_FTM_RESP_ATTR_NUM_RX_SPATIAL_STREAMS,
+	NL80211_PMSR_FTM_RESP_ATTR_NOMINAL_TIME,
+	NL80211_PMSR_FTM_RESP_ATTR_AVAILABILITY_WINDOW,
+	NL80211_PMSR_FTM_RESP_ATTR_CHANNEL_WIDTH,
+	NL80211_PMSR_FTM_RESP_ATTR_PREAMBLE,
+	NL80211_PMSR_FTM_RESP_ATTR_IS_DELAYED_LMR,
 
 	/* keep last */
 	NUM_NL80211_PMSR_FTM_RESP_ATTR,
diff --git a/include/uapi/linux/pkt_sched.h b/include/uapi/linux/pkt_sched.h
index 66e8072f44dff..490efd2885267 100644
--- a/include/uapi/linux/pkt_sched.h
+++ b/include/uapi/linux/pkt_sched.h
@@ -569,6 +569,16 @@ struct tc_netem_gemodel {
 #define NETEM_DIST_SCALE	8192
 #define NETEM_DIST_MAX		16384
 
+struct tc_netem_xstats {
+	__u64	delayed;	/* packets delayed */
+	__u64	dropped;	/* packets dropped by loss model      */
+	__u64	corrupted;	/* packets with bit errors injected   */
+	__u64	duplicated;	/* duplicate packets generated        */
+	__u64	reordered;	/* packets sent out of order          */
+	__u64	ecn_marked;	/* packets ECN CE-marked (not dropped)*/
+	__u64	allocation_errors;
+};
+
 /* DRR */
 
 enum {
diff --git a/include/uapi/linux/pps.h b/include/uapi/linux/pps.h
index 009ebcd8ced5e..1088dea65e12f 100644
--- a/include/uapi/linux/pps.h
+++ b/include/uapi/linux/pps.h
@@ -26,7 +26,7 @@
 #include <linux/types.h>
 
 #define PPS_VERSION		"5.3.6"
-#define PPS_MAX_SOURCES		16		/* should be enough... */
+#define PPS_MAX_SOURCES		256		/* should be enough... */
 
 /* Implementation note: the logical states ``assert'' and ``clear''
  * are implemented in terms of the chip register, i.e. ``assert''
diff --git a/include/uapi/linux/rkisp1-config.h b/include/uapi/linux/rkisp1-config.h
index b2d2a71f7baff..d97384abc1575 100644
--- a/include/uapi/linux/rkisp1-config.h
+++ b/include/uapi/linux/rkisp1-config.h
@@ -967,6 +967,92 @@ struct rkisp1_cif_isp_wdr_config {
 	__u8 use_iref;
 };
 
+/*
+ * enum rkisp1_cif_isp_cac_h_clip_mode - horizontal clipping mode
+ *
+ * @RKISP1_CIF_ISP_CAC_H_CLIP_MODE_4PX: +/- 4 pixels
+ * @RKISP1_CIF_ISP_CAC_H_CLIP_MODE_4_5PX: +/- 4/5 pixels depending on bayer position
+ */
+enum rkisp1_cif_isp_cac_h_clip_mode {
+	RKISP1_CIF_ISP_CAC_H_CLIP_MODE_4PX = 0,
+	RKISP1_CIF_ISP_CAC_H_CLIP_MODE_4_5PX = 1,
+};
+
+/**
+ * enum rkisp1_cif_isp_cac_v_clip_mode - vertical clipping mode
+ *
+ * @RKISP1_CIF_ISP_CAC_V_CLIP_MODE_2PX: +/- 2 pixels
+ * @RKISP1_CIF_ISP_CAC_V_CLIP_MODE_3PX: +/- 3 pixels
+ * @RKISP1_CIF_ISP_CAC_V_CLIP_MODE_3_4PX: +/- 3/4 pixels depending on bayer position
+ */
+enum rkisp1_cif_isp_cac_v_clip_mode {
+	RKISP1_CIF_ISP_CAC_V_CLIP_MODE_2PX = 0,
+	RKISP1_CIF_ISP_CAC_V_CLIP_MODE_3PX = 1,
+	RKISP1_CIF_ISP_CAC_V_CLIP_MODE_3_4PX = 2,
+};
+
+/**
+ * struct rkisp1_cif_isp_cac_config - chromatic aberration correction configuration
+ *
+ * The correction is carried out by shifting the red and blue pixels relative
+ * to the green ones, depending on the distance from the optical center:
+ *
+ * @h_count_start: horizontal coordinate of the optical center (13-bit unsigned integer; [1,8191])
+ * @v_count_start: vertical coordinate of the optical center (13-bit unsigned integer; [1,8191])
+ *
+ * For each pixel, the x/y distances from the optical center are calculated and
+ * then transformed into the [0,255] range based on the following formula:
+ *
+ *   (((d << 4) >> ns) * nf) >> 5
+ *
+ * where `d` is the distance, `ns` and `nf` are the normalization parameters:
+ *
+ * @x_nf: horizontal normalization scale parameter (5-bit unsigned integer; [0,31])
+ * @x_ns: horizontal normalization shift parameter (4-bit unsigned integer; [0,15])
+ *
+ * @y_nf: vertical normalization scale parameter (5-bit unsigned integer; [0,31])
+ * @y_ns: vertical normalization shift parameter (4-bit unsigned integer; [0,15])
+ *
+ * These parameters should be chosen based on the image resolution, the position
+ * of the optical center, and the shape of pixels, so that no normalized distance
+ * is larger than 255. If the pixels have square shape, the two sets of parameters
+ * should be equal.
+ *
+ * The actual amount of correction is calculated with a third degree polynomial:
+ *
+ *   c[0] * r + c[1] * r^2 + c[2] * r^3
+ *
+ * where `c` is the set of coefficients for the given color, and `r` is distance:
+ *
+ * @red: red coefficients (5.4 two's complement; [-16,15.9375])
+ * @blue: blue coefficients (5.4 two's complement; [-16,15.9375])
+ *
+ * Finally, the amount is clipped as requested:
+ *
+ * @h_clip_mode: maximum horizontal shift (from enum rkisp1_cif_isp_cac_h_clip_mode)
+ * @v_clip_mode: maximum vertical shift (from enum rkisp1_cif_isp_cac_v_clip_mode)
+ *
+ * A positive result will shift away from the optical center, while a negative
+ * one will shift towards the optical center. In the latter case, the pixel
+ * values at the edges are duplicated.
+ */
+struct rkisp1_cif_isp_cac_config {
+	__u8 h_clip_mode;
+	__u8 v_clip_mode;
+
+	__u16 h_count_start;
+	__u16 v_count_start;
+
+	__u16 red[3];
+	__u16 blue[3];
+
+	__u8 x_nf;
+	__u8 x_ns;
+
+	__u8 y_nf;
+	__u8 y_ns;
+};
+
 /*---------- PART2: Measurement Statistics ------------*/
 
 /**
@@ -1138,6 +1224,7 @@ struct rkisp1_stat_buffer {
  * @RKISP1_EXT_PARAMS_BLOCK_TYPE_COMPAND_EXPAND: Companding expand curve
  * @RKISP1_EXT_PARAMS_BLOCK_TYPE_COMPAND_COMPRESS: Companding compress curve
  * @RKISP1_EXT_PARAMS_BLOCK_TYPE_WDR: Wide dynamic range
+ * @RKISP1_EXT_PARAMS_BLOCK_TYPE_CAC: Chromatic aberration correction
  */
 enum rkisp1_ext_params_block_type {
 	RKISP1_EXT_PARAMS_BLOCK_TYPE_BLS,
@@ -1161,6 +1248,7 @@ enum rkisp1_ext_params_block_type {
 	RKISP1_EXT_PARAMS_BLOCK_TYPE_COMPAND_EXPAND,
 	RKISP1_EXT_PARAMS_BLOCK_TYPE_COMPAND_COMPRESS,
 	RKISP1_EXT_PARAMS_BLOCK_TYPE_WDR,
+	RKISP1_EXT_PARAMS_BLOCK_TYPE_CAC,
 };
 
 /* For backward compatibility */
@@ -1507,6 +1595,22 @@ struct rkisp1_ext_params_wdr_config {
 	struct rkisp1_cif_isp_wdr_config config;
 } __attribute__((aligned(8)));
 
+/**
+ * struct rkisp1_ext_params_cac_config - RkISP1 extensible params CAC config
+ *
+ * RkISP1 extensible parameters CAC block.
+ * Identified by :c:type:`RKISP1_EXT_PARAMS_BLOCK_TYPE_CAC`.
+ *
+ * @header: The RkISP1 extensible parameters header, see
+ *	    :c:type:`rkisp1_ext_params_block_header`
+ * @config: CAC configuration, see
+ *	    :c:type:`rkisp1_cif_isp_cac_config`
+ */
+struct rkisp1_ext_params_cac_config {
+	struct rkisp1_ext_params_block_header header;
+	struct rkisp1_cif_isp_cac_config config;
+} __attribute__((aligned(8)));
+
 /*
  * The rkisp1_ext_params_compand_curve_config structure is counted twice as it
  * is used for both the COMPAND_EXPAND and COMPAND_COMPRESS block types.
@@ -1532,14 +1636,15 @@ struct rkisp1_ext_params_wdr_config {
 	sizeof(struct rkisp1_ext_params_compand_bls_config)		+\
 	sizeof(struct rkisp1_ext_params_compand_curve_config)		+\
 	sizeof(struct rkisp1_ext_params_compand_curve_config)		+\
-	sizeof(struct rkisp1_ext_params_wdr_config))
+	sizeof(struct rkisp1_ext_params_wdr_config)			+\
+	sizeof(struct rkisp1_ext_params_cac_config))
 
 /**
- * enum rksip1_ext_param_buffer_version - RkISP1 extensible parameters version
+ * enum rkisp1_ext_param_buffer_version - RkISP1 extensible parameters version
  *
  * @RKISP1_EXT_PARAM_BUFFER_V1: First version of RkISP1 extensible parameters
  */
-enum rksip1_ext_param_buffer_version {
+enum rkisp1_ext_param_buffer_version {
 	RKISP1_EXT_PARAM_BUFFER_V1 = V4L2_ISP_PARAMS_VERSION_V1,
 };
 
@@ -1601,7 +1706,7 @@ enum rksip1_ext_param_buffer_version {
  *	+---------------------------------------------------------------------+
  *
  * @version: The RkISP1 extensible parameters buffer version, see
- *	     :c:type:`rksip1_ext_param_buffer_version`
+ *	     :c:type:`rkisp1_ext_param_buffer_version`
  * @data_size: The RkISP1 configuration data effective size, excluding this
  *	       header
  * @data: The RkISP1 extensible configuration data blocks
diff --git a/include/uapi/linux/rtnetlink.h b/include/uapi/linux/rtnetlink.h
index dab9493c791b8..27265fd31e5fc 100644
--- a/include/uapi/linux/rtnetlink.h
+++ b/include/uapi/linux/rtnetlink.h
@@ -840,6 +840,7 @@ enum {
 #define RTEXT_FILTER_CFM_CONFIG	(1 << 5)
 #define RTEXT_FILTER_CFM_STATUS	(1 << 6)
 #define RTEXT_FILTER_MST	(1 << 7)
+#define RTEXT_FILTER_NAME_ONLY	(1 << 8)
 
 /* End of information exported to user level */
 
diff --git a/include/uapi/linux/se_ioctl.h b/include/uapi/linux/se_ioctl.h
new file mode 100644
index 0000000000000..0c948bdc8c262
--- /dev/null
+++ b/include/uapi/linux/se_ioctl.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause*/
+/*
+ * Copyright 2025 NXP
+ */
+
+#ifndef SE_IOCTL_H
+#define SE_IOCTL_H
+
+#include <linux/types.h>
+
+#define SE_TYPE_STR_DBG			"dbg"
+#define SE_TYPE_STR_HSM			"hsm"
+#define SE_TYPE_ID_UNKWN		0x0
+#define SE_TYPE_ID_DBG			0x1
+#define SE_TYPE_ID_HSM			0x2
+/* IOCTL definitions. */
+
+struct se_ioctl_setup_iobuf {
+	void __user *user_buf;
+	__u32 length;
+	__u32 flags;
+	__u64 ele_addr;
+};
+
+struct se_ioctl_shared_mem_cfg {
+	__u32 base_offset;
+	__u32 size;
+};
+
+struct se_ioctl_get_if_info {
+	__u8 se_if_id;
+	__u8 interrupt_idx;
+	__u8 tz;
+	__u8 did;
+	__u8 cmd_tag;
+	__u8 rsp_tag;
+	__u8 success_tag;
+	__u8 base_api_ver;
+	__u8 fw_api_ver;
+};
+
+struct se_ioctl_cmd_snd_rcv_rsp_info {
+	__u32 __user *tx_buf;
+	int tx_buf_sz;
+	__u32 __user *rx_buf;
+	int rx_buf_sz;
+};
+
+struct se_ioctl_get_soc_info {
+	__u16 soc_id;
+	__u16 soc_rev;
+};
+
+/* IO Buffer Flags */
+#define SE_IO_BUF_FLAGS_IS_OUTPUT	(0x00u)
+#define SE_IO_BUF_FLAGS_IS_INPUT	(0x01u)
+#define SE_IO_BUF_FLAGS_USE_SEC_MEM	(0x02u)
+#define SE_IO_BUF_FLAGS_USE_SHORT_ADDR	(0x04u)
+#define SE_IO_BUF_FLAGS_IS_IN_OUT	(0x10u)
+
+/* IOCTLS */
+#define SE_IOCTL			0x0A /* like MISC_MAJOR. */
+
+/*
+ * ioctl to designated the current fd as logical-reciever.
+ * This is ioctl is send when the nvm-daemon, a slave to the
+ * firmware is started by the user.
+ */
+#define SE_IOCTL_ENABLE_CMD_RCV	_IO(SE_IOCTL, 0x01)
+
+/*
+ * ioctl to get the buffer allocated from the memory, which is shared
+ * between kernel and FW.
+ * Post allocation, the kernel tagged the allocated memory with:
+ *  Output
+ *  Input
+ *  Input-Output
+ *  Short address
+ *  Secure-memory
+ */
+#define SE_IOCTL_SETUP_IOBUF	_IOWR(SE_IOCTL, 0x03, struct se_ioctl_setup_iobuf)
+
+/*
+ * ioctl to get the mu information, that is used to exchange message
+ * with FW, from user-spaced.
+ */
+#define SE_IOCTL_GET_MU_INFO	_IOR(SE_IOCTL, 0x04, struct se_ioctl_get_if_info)
+/*
+ * ioctl to get SoC Info from user-space.
+ */
+#define SE_IOCTL_GET_SOC_INFO      _IOR(SE_IOCTL, 0x06, struct se_ioctl_get_soc_info)
+
+/*
+ * ioctl to send command and receive response from user-space.
+ */
+#define SE_IOCTL_CMD_SEND_RCV_RSP _IOWR(SE_IOCTL, 0x07, struct se_ioctl_cmd_snd_rcv_rsp_info)
+#endif
diff --git a/include/uapi/linux/sunrpc_netlink.h b/include/uapi/linux/sunrpc_netlink.h
new file mode 100644
index 0000000000000..34677f0ec2f95
--- /dev/null
+++ b/include/uapi/linux/sunrpc_netlink.h
@@ -0,0 +1,84 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/* Do not edit directly, auto-generated from: */
+/*	Documentation/netlink/specs/sunrpc_cache.yaml */
+/* YNL-GEN uapi header */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#ifndef _UAPI_LINUX_SUNRPC_NETLINK_H
+#define _UAPI_LINUX_SUNRPC_NETLINK_H
+
+#define SUNRPC_FAMILY_NAME	"sunrpc"
+#define SUNRPC_FAMILY_VERSION	1
+
+enum sunrpc_cache_type {
+	SUNRPC_CACHE_TYPE_IP_MAP = 1,
+	SUNRPC_CACHE_TYPE_UNIX_GID = 2,
+};
+
+enum {
+	SUNRPC_A_CACHE_NOTIFY_CACHE_TYPE = 1,
+
+	__SUNRPC_A_CACHE_NOTIFY_MAX,
+	SUNRPC_A_CACHE_NOTIFY_MAX = (__SUNRPC_A_CACHE_NOTIFY_MAX - 1)
+};
+
+enum {
+	SUNRPC_A_IP_MAP_SEQNO = 1,
+	SUNRPC_A_IP_MAP_CLASS,
+	SUNRPC_A_IP_MAP_ADDR,
+	SUNRPC_A_IP_MAP_DOMAIN,
+	SUNRPC_A_IP_MAP_NEGATIVE,
+	SUNRPC_A_IP_MAP_EXPIRY,
+
+	__SUNRPC_A_IP_MAP_MAX,
+	SUNRPC_A_IP_MAP_MAX = (__SUNRPC_A_IP_MAP_MAX - 1)
+};
+
+enum {
+	SUNRPC_A_IP_MAP_REQS_REQUESTS = 1,
+
+	__SUNRPC_A_IP_MAP_REQS_MAX,
+	SUNRPC_A_IP_MAP_REQS_MAX = (__SUNRPC_A_IP_MAP_REQS_MAX - 1)
+};
+
+enum {
+	SUNRPC_A_UNIX_GID_SEQNO = 1,
+	SUNRPC_A_UNIX_GID_UID,
+	SUNRPC_A_UNIX_GID_GIDS,
+	SUNRPC_A_UNIX_GID_NEGATIVE,
+	SUNRPC_A_UNIX_GID_EXPIRY,
+
+	__SUNRPC_A_UNIX_GID_MAX,
+	SUNRPC_A_UNIX_GID_MAX = (__SUNRPC_A_UNIX_GID_MAX - 1)
+};
+
+enum {
+	SUNRPC_A_UNIX_GID_REQS_REQUESTS = 1,
+
+	__SUNRPC_A_UNIX_GID_REQS_MAX,
+	SUNRPC_A_UNIX_GID_REQS_MAX = (__SUNRPC_A_UNIX_GID_REQS_MAX - 1)
+};
+
+enum {
+	SUNRPC_A_CACHE_FLUSH_MASK = 1,
+
+	__SUNRPC_A_CACHE_FLUSH_MAX,
+	SUNRPC_A_CACHE_FLUSH_MAX = (__SUNRPC_A_CACHE_FLUSH_MAX - 1)
+};
+
+enum {
+	SUNRPC_CMD_CACHE_NOTIFY = 1,
+	SUNRPC_CMD_IP_MAP_GET_REQS,
+	SUNRPC_CMD_IP_MAP_SET_REQS,
+	SUNRPC_CMD_UNIX_GID_GET_REQS,
+	SUNRPC_CMD_UNIX_GID_SET_REQS,
+	SUNRPC_CMD_CACHE_FLUSH,
+
+	__SUNRPC_CMD_MAX,
+	SUNRPC_CMD_MAX = (__SUNRPC_CMD_MAX - 1)
+};
+
+#define SUNRPC_MCGRP_NONE	"none"
+#define SUNRPC_MCGRP_EXPORTD	"exportd"
+
+#endif /* _UAPI_LINUX_SUNRPC_NETLINK_H */
diff --git a/include/uapi/linux/synclink.h b/include/uapi/linux/synclink.h
deleted file mode 100644
index 62f32d4e10219..0000000000000
--- a/include/uapi/linux/synclink.h
+++ /dev/null
@@ -1,301 +0,0 @@
-/* SPDX-License-Identifier: GPL-1.0+ WITH Linux-syscall-note */
-/*
- * SyncLink Multiprotocol Serial Adapter Driver
- *
- * $Id: synclink.h,v 3.14 2006/07/17 20:15:43 paulkf Exp $
- *
- * Copyright (C) 1998-2000 by Microgate Corporation
- *
- * Redistribution of this file is permitted under
- * the terms of the GNU Public License (GPL)
- */
-
-#ifndef _UAPI_SYNCLINK_H_
-#define _UAPI_SYNCLINK_H_
-#define SYNCLINK_H_VERSION 3.6
-
-#include <linux/types.h>
-
-#define BIT0	0x0001
-#define BIT1	0x0002
-#define BIT2	0x0004
-#define BIT3	0x0008
-#define BIT4	0x0010
-#define BIT5	0x0020
-#define BIT6	0x0040
-#define BIT7	0x0080
-#define BIT8	0x0100
-#define BIT9	0x0200
-#define BIT10	0x0400
-#define BIT11	0x0800
-#define BIT12	0x1000
-#define BIT13	0x2000
-#define BIT14	0x4000
-#define BIT15	0x8000
-#define BIT16	0x00010000
-#define BIT17	0x00020000
-#define BIT18	0x00040000
-#define BIT19	0x00080000
-#define BIT20	0x00100000
-#define BIT21	0x00200000
-#define BIT22	0x00400000
-#define BIT23	0x00800000
-#define BIT24	0x01000000
-#define BIT25	0x02000000
-#define BIT26	0x04000000
-#define BIT27	0x08000000
-#define BIT28	0x10000000
-#define BIT29	0x20000000
-#define BIT30	0x40000000
-#define BIT31	0x80000000
-
-
-#define HDLC_MAX_FRAME_SIZE	65535
-#define MAX_ASYNC_TRANSMIT	4096
-#define MAX_ASYNC_BUFFER_SIZE	4096
-
-#define ASYNC_PARITY_NONE		0
-#define ASYNC_PARITY_EVEN		1
-#define ASYNC_PARITY_ODD		2
-#define ASYNC_PARITY_SPACE		3
-
-#define HDLC_FLAG_UNDERRUN_ABORT7	0x0000
-#define HDLC_FLAG_UNDERRUN_ABORT15	0x0001
-#define HDLC_FLAG_UNDERRUN_FLAG		0x0002
-#define HDLC_FLAG_UNDERRUN_CRC		0x0004
-#define HDLC_FLAG_SHARE_ZERO		0x0010
-#define HDLC_FLAG_AUTO_CTS		0x0020
-#define HDLC_FLAG_AUTO_DCD		0x0040
-#define HDLC_FLAG_AUTO_RTS		0x0080
-#define HDLC_FLAG_RXC_DPLL		0x0100
-#define HDLC_FLAG_RXC_BRG		0x0200
-#define HDLC_FLAG_RXC_TXCPIN		0x8000
-#define HDLC_FLAG_RXC_RXCPIN		0x0000
-#define HDLC_FLAG_TXC_DPLL		0x0400
-#define HDLC_FLAG_TXC_BRG		0x0800
-#define HDLC_FLAG_TXC_TXCPIN		0x0000
-#define HDLC_FLAG_TXC_RXCPIN		0x0008
-#define HDLC_FLAG_DPLL_DIV8		0x1000
-#define HDLC_FLAG_DPLL_DIV16		0x2000
-#define HDLC_FLAG_DPLL_DIV32		0x0000
-#define HDLC_FLAG_HDLC_LOOPMODE		0x4000
-
-#define HDLC_CRC_NONE			0
-#define HDLC_CRC_16_CCITT		1
-#define HDLC_CRC_32_CCITT		2
-#define HDLC_CRC_MASK			0x00ff
-#define HDLC_CRC_RETURN_EX		0x8000
-
-#define RX_OK				0
-#define RX_CRC_ERROR			1
-
-#define HDLC_TXIDLE_FLAGS		0
-#define HDLC_TXIDLE_ALT_ZEROS_ONES	1
-#define HDLC_TXIDLE_ZEROS		2
-#define HDLC_TXIDLE_ONES		3
-#define HDLC_TXIDLE_ALT_MARK_SPACE	4
-#define HDLC_TXIDLE_SPACE		5
-#define HDLC_TXIDLE_MARK		6
-#define HDLC_TXIDLE_CUSTOM_8            0x10000000
-#define HDLC_TXIDLE_CUSTOM_16           0x20000000
-
-#define HDLC_ENCODING_NRZ			0
-#define HDLC_ENCODING_NRZB			1
-#define HDLC_ENCODING_NRZI_MARK			2
-#define HDLC_ENCODING_NRZI_SPACE		3
-#define HDLC_ENCODING_NRZI			HDLC_ENCODING_NRZI_SPACE
-#define HDLC_ENCODING_BIPHASE_MARK		4
-#define HDLC_ENCODING_BIPHASE_SPACE		5
-#define HDLC_ENCODING_BIPHASE_LEVEL		6
-#define HDLC_ENCODING_DIFF_BIPHASE_LEVEL	7
-
-#define HDLC_PREAMBLE_LENGTH_8BITS	0
-#define HDLC_PREAMBLE_LENGTH_16BITS	1
-#define HDLC_PREAMBLE_LENGTH_32BITS	2
-#define HDLC_PREAMBLE_LENGTH_64BITS	3
-
-#define HDLC_PREAMBLE_PATTERN_NONE	0
-#define HDLC_PREAMBLE_PATTERN_ZEROS	1
-#define HDLC_PREAMBLE_PATTERN_FLAGS	2
-#define HDLC_PREAMBLE_PATTERN_10	3
-#define HDLC_PREAMBLE_PATTERN_01	4
-#define HDLC_PREAMBLE_PATTERN_ONES	5
-
-#define MGSL_MODE_ASYNC		1
-#define MGSL_MODE_HDLC		2
-#define MGSL_MODE_MONOSYNC	3
-#define MGSL_MODE_BISYNC	4
-#define MGSL_MODE_RAW		6
-#define MGSL_MODE_BASE_CLOCK    7
-#define MGSL_MODE_XSYNC         8
-
-#define MGSL_BUS_TYPE_ISA	1
-#define MGSL_BUS_TYPE_EISA	2
-#define MGSL_BUS_TYPE_PCI	5
-
-#define MGSL_INTERFACE_MASK     0xf
-#define MGSL_INTERFACE_DISABLE  0
-#define MGSL_INTERFACE_RS232    1
-#define MGSL_INTERFACE_V35      2
-#define MGSL_INTERFACE_RS422    3
-#define MGSL_INTERFACE_RTS_EN   0x10
-#define MGSL_INTERFACE_LL       0x20
-#define MGSL_INTERFACE_RL       0x40
-#define MGSL_INTERFACE_MSB_FIRST 0x80
-
-typedef struct _MGSL_PARAMS
-{
-	/* Common */
-
-	unsigned long	mode;		/* Asynchronous or HDLC */
-	unsigned char	loopback;	/* internal loopback mode */
-
-	/* HDLC Only */
-
-	unsigned short	flags;
-	unsigned char	encoding;	/* NRZ, NRZI, etc. */
-	unsigned long	clock_speed;	/* external clock speed in bits per second */
-	unsigned char	addr_filter;	/* receive HDLC address filter, 0xFF = disable */
-	unsigned short	crc_type;	/* None, CRC16-CCITT, or CRC32-CCITT */
-	unsigned char	preamble_length;
-	unsigned char	preamble;
-
-	/* Async Only */
-
-	unsigned long	data_rate;	/* bits per second */
-	unsigned char	data_bits;	/* 7 or 8 data bits */
-	unsigned char	stop_bits;	/* 1 or 2 stop bits */
-	unsigned char	parity;		/* none, even, or odd */
-
-} MGSL_PARAMS, *PMGSL_PARAMS;
-
-#define MICROGATE_VENDOR_ID 0x13c0
-#define SYNCLINK_DEVICE_ID 0x0010
-#define MGSCC_DEVICE_ID 0x0020
-#define SYNCLINK_SCA_DEVICE_ID 0x0030
-#define SYNCLINK_GT_DEVICE_ID 0x0070
-#define SYNCLINK_GT4_DEVICE_ID 0x0080
-#define SYNCLINK_AC_DEVICE_ID  0x0090
-#define SYNCLINK_GT2_DEVICE_ID 0x00A0
-#define MGSL_MAX_SERIAL_NUMBER 30
-
-/*
-** device diagnostics status
-*/
-
-#define DiagStatus_OK				0
-#define DiagStatus_AddressFailure		1
-#define DiagStatus_AddressConflict		2
-#define DiagStatus_IrqFailure			3
-#define DiagStatus_IrqConflict			4
-#define DiagStatus_DmaFailure			5
-#define DiagStatus_DmaConflict			6
-#define DiagStatus_PciAdapterNotFound		7
-#define DiagStatus_CantAssignPciResources	8
-#define DiagStatus_CantAssignPciMemAddr		9
-#define DiagStatus_CantAssignPciIoAddr		10
-#define DiagStatus_CantAssignPciIrq		11
-#define DiagStatus_MemoryError			12
-
-#define SerialSignal_DCD            0x01     /* Data Carrier Detect */
-#define SerialSignal_TXD            0x02     /* Transmit Data */
-#define SerialSignal_RI             0x04     /* Ring Indicator */
-#define SerialSignal_RXD            0x08     /* Receive Data */
-#define SerialSignal_CTS            0x10     /* Clear to Send */
-#define SerialSignal_RTS            0x20     /* Request to Send */
-#define SerialSignal_DSR            0x40     /* Data Set Ready */
-#define SerialSignal_DTR            0x80     /* Data Terminal Ready */
-
-
-/*
- * Counters of the input lines (CTS, DSR, RI, CD) interrupts
- */
-struct mgsl_icount {
-	__u32	cts, dsr, rng, dcd, tx, rx;
-	__u32	frame, parity, overrun, brk;
-	__u32	buf_overrun;
-	__u32	txok;
-	__u32	txunder;
-	__u32	txabort;
-	__u32	txtimeout;
-	__u32	rxshort;
-	__u32	rxlong;
-	__u32	rxabort;
-	__u32	rxover;
-	__u32	rxcrc;
-	__u32	rxok;
-	__u32	exithunt;
-	__u32	rxidle;
-};
-
-struct gpio_desc {
-	__u32 state;
-	__u32 smask;
-	__u32 dir;
-	__u32 dmask;
-};
-
-#define DEBUG_LEVEL_DATA	1
-#define DEBUG_LEVEL_ERROR 	2
-#define DEBUG_LEVEL_INFO  	3
-#define DEBUG_LEVEL_BH    	4
-#define DEBUG_LEVEL_ISR		5
-
-/*
-** Event bit flags for use with MgslWaitEvent
-*/
-
-#define MgslEvent_DsrActive	0x0001
-#define MgslEvent_DsrInactive	0x0002
-#define MgslEvent_Dsr		0x0003
-#define MgslEvent_CtsActive	0x0004
-#define MgslEvent_CtsInactive	0x0008
-#define MgslEvent_Cts		0x000c
-#define MgslEvent_DcdActive	0x0010
-#define MgslEvent_DcdInactive	0x0020
-#define MgslEvent_Dcd		0x0030
-#define MgslEvent_RiActive	0x0040
-#define MgslEvent_RiInactive	0x0080
-#define MgslEvent_Ri		0x00c0
-#define MgslEvent_ExitHuntMode	0x0100
-#define MgslEvent_IdleReceived	0x0200
-
-/* Private IOCTL codes:
- *
- * MGSL_IOCSPARAMS	set MGSL_PARAMS structure values
- * MGSL_IOCGPARAMS	get current MGSL_PARAMS structure values
- * MGSL_IOCSTXIDLE	set current transmit idle mode
- * MGSL_IOCGTXIDLE	get current transmit idle mode
- * MGSL_IOCTXENABLE	enable or disable transmitter
- * MGSL_IOCRXENABLE	enable or disable receiver
- * MGSL_IOCTXABORT	abort transmitting frame (HDLC)
- * MGSL_IOCGSTATS	return current statistics
- * MGSL_IOCWAITEVENT	wait for specified event to occur
- * MGSL_LOOPTXDONE	transmit in HDLC LoopMode done
- * MGSL_IOCSIF          set the serial interface type
- * MGSL_IOCGIF          get the serial interface type
- */
-#define MGSL_MAGIC_IOC	'm'
-#define MGSL_IOCSPARAMS		_IOW(MGSL_MAGIC_IOC,0,struct _MGSL_PARAMS)
-#define MGSL_IOCGPARAMS		_IOR(MGSL_MAGIC_IOC,1,struct _MGSL_PARAMS)
-#define MGSL_IOCSTXIDLE		_IO(MGSL_MAGIC_IOC,2)
-#define MGSL_IOCGTXIDLE		_IO(MGSL_MAGIC_IOC,3)
-#define MGSL_IOCTXENABLE	_IO(MGSL_MAGIC_IOC,4)
-#define MGSL_IOCRXENABLE	_IO(MGSL_MAGIC_IOC,5)
-#define MGSL_IOCTXABORT		_IO(MGSL_MAGIC_IOC,6)
-#define MGSL_IOCGSTATS		_IO(MGSL_MAGIC_IOC,7)
-#define MGSL_IOCWAITEVENT	_IOWR(MGSL_MAGIC_IOC,8,int)
-#define MGSL_IOCCLRMODCOUNT	_IO(MGSL_MAGIC_IOC,15)
-#define MGSL_IOCLOOPTXDONE	_IO(MGSL_MAGIC_IOC,9)
-#define MGSL_IOCSIF		_IO(MGSL_MAGIC_IOC,10)
-#define MGSL_IOCGIF		_IO(MGSL_MAGIC_IOC,11)
-#define MGSL_IOCSGPIO		_IOW(MGSL_MAGIC_IOC,16,struct gpio_desc)
-#define MGSL_IOCGGPIO		_IOR(MGSL_MAGIC_IOC,17,struct gpio_desc)
-#define MGSL_IOCWAITGPIO	_IOWR(MGSL_MAGIC_IOC,18,struct gpio_desc)
-#define MGSL_IOCSXSYNC		_IO(MGSL_MAGIC_IOC, 19)
-#define MGSL_IOCGXSYNC		_IO(MGSL_MAGIC_IOC, 20)
-#define MGSL_IOCSXCTRL		_IO(MGSL_MAGIC_IOC, 21)
-#define MGSL_IOCGXCTRL		_IO(MGSL_MAGIC_IOC, 22)
-
-
-#endif /* _UAPI_SYNCLINK_H_ */
diff --git a/include/uapi/linux/tee.h b/include/uapi/linux/tee.h
index cab5cadca8ef9..5203977ed35d1 100644
--- a/include/uapi/linux/tee.h
+++ b/include/uapi/linux/tee.h
@@ -470,6 +470,7 @@ struct tee_ioctl_object_invoke_arg {
 	__u32 op;
 	__u32 ret;
 	__u32 num_params;
+	__u32 :32;
 	/* num_params tells the actual number of element in params */
 	struct tee_ioctl_param params[];
 };
diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h
index 68dd0c4e47b2d..affec0ab47815 100644
--- a/include/uapi/linux/v4l2-controls.h
+++ b/include/uapi/linux/v4l2-controls.h
@@ -464,6 +464,8 @@ enum v4l2_mpeg_video_intra_refresh_period_type {
 	V4L2_CID_MPEG_VIDEO_INTRA_REFRESH_PERIOD_TYPE_CYCLIC	= 1,
 };
 
+#define V4L2_CID_MPEG_VIDEO_BACKGROUND_DETECTION	(V4L2_CID_CODEC_BASE + 238)
+
 /* CIDs for the MPEG-2 Part 2 (H.262) codec */
 #define V4L2_CID_MPEG_VIDEO_MPEG2_LEVEL			(V4L2_CID_CODEC_BASE+270)
 enum v4l2_mpeg_video_mpeg2_level {
diff --git a/include/uapi/linux/virtio_gpu.h b/include/uapi/linux/virtio_gpu.h
index be109777d10db..4f530d90058c7 100644
--- a/include/uapi/linux/virtio_gpu.h
+++ b/include/uapi/linux/virtio_gpu.h
@@ -64,6 +64,14 @@
  * context_init and multiple timelines
  */
 #define VIRTIO_GPU_F_CONTEXT_INIT        4
+/*
+ * The device provides a valid blob_alignment
+ * field in its configuration and both
+ * VIRTIO_GPU_CMD_RESOURCE_CREATE_BLOB and
+ * VIRTIO_GPU_CMD_RESOURCE_MAP_BLOB requests
+ * must be aligned to that value.
+ */
+#define VIRTIO_GPU_F_BLOB_ALIGNMENT      5
 
 enum virtio_gpu_ctrl_type {
 	VIRTIO_GPU_UNDEFINED = 0,
@@ -365,6 +373,7 @@ struct virtio_gpu_config {
 	__le32 events_clear;
 	__le32 num_scanouts;
 	__le32 num_capsets;
+	__le32 blob_alignment;
 };
 
 /* simple formats for fbcon/X use */
diff --git a/include/ufs/ufs.h b/include/ufs/ufs.h
index 602aa34c98221..0d48e137d66d2 100644
--- a/include/ufs/ufs.h
+++ b/include/ufs/ufs.h
@@ -191,6 +191,8 @@ enum attr_idn {
 	QUERY_ATTR_IDN_WB_BUF_RESIZE_HINT	= 0x3C,
 	QUERY_ATTR_IDN_WB_BUF_RESIZE_EN		= 0x3D,
 	QUERY_ATTR_IDN_WB_BUF_RESIZE_STATUS	= 0x3E,
+	QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS        = 0x47,
+	QUERY_ATTR_IDN_TX_EQ_GN_SETTINGS_EXT    = 0x48,
 };
 
 /* Descriptor idn for Query requests */
diff --git a/include/ufs/ufshcd.h b/include/ufs/ufshcd.h
index cfbc75d8df836..f48d6416e2993 100644
--- a/include/ufs/ufshcd.h
+++ b/include/ufs/ufshcd.h
@@ -358,6 +358,7 @@ struct ufshcd_tx_eqtr_record {
  * @eqtr_record: Pointer to TX EQTR record
  * @is_valid: True if parameter contains valid TX Equalization settings
  * @is_applied: True if settings have been applied to UniPro of both sides
+ * @is_trained: True if parameters obtained from TX EQTR procedure
  */
 struct ufshcd_tx_eq_params {
 	struct ufshcd_tx_eq_settings host[UFS_MAX_LANES];
@@ -365,6 +366,7 @@ struct ufshcd_tx_eq_params {
 	struct ufshcd_tx_eqtr_record *eqtr_record;
 	bool is_valid;
 	bool is_applied;
+	bool is_trained;
 };
 
 /**
diff --git a/init/Kconfig b/init/Kconfig
index 2937c4d308aec..ca35184532dc4 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -170,6 +170,9 @@ config CC_HAS_BROKEN_COUNTED_BY_REF
 	# https://github.com/llvm/llvm-project/issues/182575
 	default y if CC_IS_CLANG && CLANG_VERSION < 220100
 
+config CC_HAS_ALLOC_TOKEN
+	def_bool $(cc-option,-falloc-token-max=123)
+
 config CC_HAS_MULTIDIMENSIONAL_NONSTRING
 	def_bool $(success,echo 'char tag[][4] __attribute__((__nonstring__)) = { };' | $(CC) $(CLANG_FLAGS) -x c - -c -o /dev/null -Werror)
 
@@ -1019,6 +1022,17 @@ config NUMA_BALANCING
 
 	  This system will be inactive on UMA systems.
 
+config SCHED_CACHE
+	bool "Cache aware load balance"
+	default y
+	depends on SMP
+	help
+	  When enabled, the scheduler will attempt to aggregate tasks from
+	  the same process onto a single Last Level Cache (LLC) domain when
+	  possible. This improves cache locality by keeping tasks that share
+	  resources within the same cache domain, reducing cache misses and
+	  lowering data access latency.
+
 config NUMA_BALANCING_DEFAULT_ENABLED
 	bool "Automatically enable NUMA aware memory/task placement"
 	default y
@@ -1050,14 +1064,14 @@ config PAGE_COUNTER
 	bool
 
 config CGROUP_FAVOR_DYNMODS
-        bool "Favor dynamic modification latency reduction by default"
-        help
-          This option enables the "favordynmods" mount option by default
-          which reduces the latencies of dynamic cgroup modifications such
-          as task migrations and controller on/offs at the cost of making
-          hot path operations such as forks and exits more expensive.
+	bool "Favor dynamic modification latency reduction by default"
+	help
+	  This option enables the "favordynmods" mount option by default
+	  which reduces the latencies of dynamic cgroup modifications such
+	  as task migrations and controller on/offs at the cost of making
+	  hot path operations such as forks and exits more expensive.
 
-          Say N if unsure.
+	  Say N if unsure.
 
 config MEMCG
 	bool "Memory controller"
@@ -1139,7 +1153,7 @@ config GROUP_SCHED_WEIGHT
 	def_bool n
 
 config GROUP_SCHED_BANDWIDTH
-        def_bool n
+	def_bool n
 
 config FAIR_GROUP_SCHED
 	bool "Group scheduling for SCHED_OTHER"
@@ -1645,10 +1659,10 @@ config LD_ORPHAN_WARN
 	depends on $(ld-option,--orphan-handling=error)
 
 config LD_ORPHAN_WARN_LEVEL
-        string
-        depends on LD_ORPHAN_WARN
-        default "error" if WERROR
-        default "warn"
+	string
+	depends on LD_ORPHAN_WARN
+	default "error" if WERROR
+	default "warn"
 
 config SYSCTL
 	bool
diff --git a/init/init_task.c b/init/init_task.c
index b5f48ebdc2b6e..3ecd66fbd563b 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -215,6 +215,10 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
 	.numa_group	= NULL,
 	.numa_faults	= NULL,
 #endif
+#ifdef CONFIG_SCHED_CACHE
+	.preferred_llc  = -1,
+	.pref_llc_queued  = 0,
+#endif
 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
 	.kasan_depth	= 1,
 #endif
diff --git a/init/initramfs.c b/init/initramfs.c
index 58db15fb18fd8..20a18fcda48ee 100644
--- a/init/initramfs.c
+++ b/init/initramfs.c
@@ -1,25 +1,28 @@
 // SPDX-License-Identifier: GPL-2.0
-#include <linux/init.h>
 #include <linux/async.h>
-#include <linux/export.h>
-#include <linux/fs.h>
-#include <linux/slab.h>
-#include <linux/types.h>
-#include <linux/fcntl.h>
 #include <linux/delay.h>
-#include <linux/string.h>
 #include <linux/dirent.h>
-#include <linux/syscalls.h>
-#include <linux/utime.h>
+#include <linux/export.h>
+#include <linux/fcntl.h>
 #include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/hex.h>
+#include <linux/init.h>
+#include <linux/init_syscalls.h>
 #include <linux/kstrtox.h>
 #include <linux/memblock.h>
 #include <linux/mm.h>
 #include <linux/namei.h>
-#include <linux/init_syscalls.h>
-#include <linux/umh.h>
-#include <linux/security.h>
 #include <linux/overflow.h>
+#include <linux/security.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/syscalls.h>
+#include <linux/types.h>
+#include <linux/umh.h>
+#include <linux/utime.h>
+
+#include <asm/byteorder.h>
 
 #include "do_mounts.h"
 #include "initramfs_internal.h"
@@ -190,26 +193,30 @@ static __initdata gid_t gid;
 static __initdata unsigned rdev;
 static __initdata u32 hdr_csum;
 
-static void __init parse_header(char *s)
+static int __init parse_header(char *s)
 {
-	unsigned long parsed[13];
-	int i;
+	__be32 header[13];
+	int ret;
 
-	for (i = 0, s += 6; i < 13; i++, s += 8)
-		parsed[i] = simple_strntoul(s, NULL, 16, 8);
+	ret = hex2bin((u8 *)header, s + 6, sizeof(header));
+	if (ret) {
+		error("damaged header");
+		return ret;
+	}
 
-	ino = parsed[0];
-	mode = parsed[1];
-	uid = parsed[2];
-	gid = parsed[3];
-	nlink = parsed[4];
-	mtime = parsed[5]; /* breaks in y2106 */
-	body_len = parsed[6];
-	major = parsed[7];
-	minor = parsed[8];
-	rdev = new_encode_dev(MKDEV(parsed[9], parsed[10]));
-	name_len = parsed[11];
-	hdr_csum = parsed[12];
+	ino = be32_to_cpu(header[0]);
+	mode = be32_to_cpu(header[1]);
+	uid = be32_to_cpu(header[2]);
+	gid = be32_to_cpu(header[3]);
+	nlink = be32_to_cpu(header[4]);
+	mtime = be32_to_cpu(header[5]); /* breaks in y2106 */
+	body_len = be32_to_cpu(header[6]);
+	major = be32_to_cpu(header[7]);
+	minor = be32_to_cpu(header[8]);
+	rdev = new_encode_dev(MKDEV(be32_to_cpu(header[9]), be32_to_cpu(header[10])));
+	name_len = be32_to_cpu(header[11]);
+	hdr_csum = be32_to_cpu(header[12]);
+	return 0;
 }
 
 /* Finite-state machine */
@@ -289,7 +296,8 @@ static int __init do_header(void)
 			error("no cpio magic");
 		return 1;
 	}
-	parse_header(collected);
+	if (parse_header(collected))
+		return 1;
 	next_header = this_header + N_ALIGN(name_len) + body_len;
 	next_header = (next_header + 3) & ~3;
 	state = SkipIt;
diff --git a/init/initramfs_test.c b/init/initramfs_test.c
index 2ce38d9a8fd0f..8a0ddc2db2c08 100644
--- a/init/initramfs_test.c
+++ b/init/initramfs_test.c
@@ -27,7 +27,18 @@ struct initramfs_test_cpio {
 	char *data;
 };
 
-static size_t fill_cpio(struct initramfs_test_cpio *cs, size_t csz, char *out)
+/* regular newc header format */
+#define CPIO_HDR_FMT "%s%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%s"
+/*
+ * Bogus newc header with "0x" prefixes on the uid, gid, and namesize values.
+ * parse_header()/simple_str[n]toul() accepted this, contrary to the initramfs
+ * specification. hex2bin() now fails.
+ */
+#define CPIO_HDR_OX_INJECT \
+	"%s%08x%08x0x%06x0X%06x%08x%08x%08x%08x%08x%08x%08x0x%06x%08x%s"
+
+static size_t fill_cpio(struct initramfs_test_cpio *cs, size_t csz,
+			bool inject_ox, char *out)
 {
 	int i;
 	size_t off = 0;
@@ -38,9 +49,8 @@ static size_t fill_cpio(struct initramfs_test_cpio *cs, size_t csz, char *out)
 		size_t thislen;
 
 		/* +1 to account for nulterm */
-		thislen = sprintf(pos, "%s"
-			"%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x%08x"
-			"%s",
+		thislen = sprintf(pos,
+			inject_ox ? CPIO_HDR_OX_INJECT : CPIO_HDR_FMT,
 			c->magic, c->ino, c->mode, c->uid, c->gid, c->nlink,
 			c->mtime, c->filesize, c->devmajor, c->devminor,
 			c->rdevmajor, c->rdevminor, c->namesize, c->csum,
@@ -102,7 +112,7 @@ static void __init initramfs_test_extract(struct kunit *test)
 	/* +3 to cater for any 4-byte end-alignment */
 	cpio_srcbuf = kzalloc(ARRAY_SIZE(c) * (CPIO_HDRLEN + PATH_MAX + 3),
 			      GFP_KERNEL);
-	len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf);
+	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
 	ktime_get_real_ts64(&ts_before);
 	err = unpack_to_rootfs(cpio_srcbuf, len);
@@ -177,7 +187,7 @@ static void __init initramfs_test_fname_overrun(struct kunit *test)
 	/* limit overrun to avoid crashes / filp_open() ENAMETOOLONG */
 	cpio_srcbuf[CPIO_HDRLEN + strlen(c[0].fname) + 20] = '\0';
 
-	len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf);
+	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 	/* overwrite trailing fname terminator and padding */
 	suffix_off = len - 1;
 	while (cpio_srcbuf[suffix_off] == '\0') {
@@ -219,7 +229,7 @@ static void __init initramfs_test_data(struct kunit *test)
 	cpio_srcbuf = kmalloc(CPIO_HDRLEN + c[0].namesize + c[0].filesize + 6,
 			      GFP_KERNEL);
 
-	len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf);
+	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
 	err = unpack_to_rootfs(cpio_srcbuf, len);
 	KUNIT_EXPECT_NULL(test, err);
@@ -274,7 +284,7 @@ static void __init initramfs_test_csum(struct kunit *test)
 
 	cpio_srcbuf = kmalloc(8192, GFP_KERNEL);
 
-	len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf);
+	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
 	err = unpack_to_rootfs(cpio_srcbuf, len);
 	KUNIT_EXPECT_NULL(test, err);
@@ -284,7 +294,7 @@ static void __init initramfs_test_csum(struct kunit *test)
 
 	/* mess up the csum and confirm that unpack fails */
 	c[0].csum--;
-	len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf);
+	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
 	err = unpack_to_rootfs(cpio_srcbuf, len);
 	KUNIT_EXPECT_NOT_NULL(test, err);
@@ -306,7 +316,7 @@ static void __init initramfs_test_hardlink(struct kunit *test)
 {
 	char *err, *cpio_srcbuf;
 	size_t len;
-	struct kstat st0, st1;
+	struct kstat st0 = {}, st1 = {};
 	struct initramfs_test_cpio c[] = { {
 		.magic = "070701",
 		.ino = 1,
@@ -330,7 +340,7 @@ static void __init initramfs_test_hardlink(struct kunit *test)
 
 	cpio_srcbuf = kmalloc(8192, GFP_KERNEL);
 
-	len = fill_cpio(c, ARRAY_SIZE(c), cpio_srcbuf);
+	len = fill_cpio(c, ARRAY_SIZE(c), false, cpio_srcbuf);
 
 	err = unpack_to_rootfs(cpio_srcbuf, len);
 	KUNIT_EXPECT_NULL(test, err);
@@ -371,7 +381,7 @@ static void __init initramfs_test_many(struct kunit *test)
 		};
 
 		c.namesize = 1 + sprintf(thispath, "initramfs_test_many-%d", i);
-		p += fill_cpio(&c, 1, p);
+		p += fill_cpio(&c, 1, false, p);
 	}
 
 	len = p - cpio_srcbuf;
@@ -425,7 +435,7 @@ static void __init initramfs_test_fname_pad(struct kunit *test)
 	} };
 
 	memcpy(tbufs->padded_fname, "padded_fname", sizeof("padded_fname"));
-	len = fill_cpio(c, ARRAY_SIZE(c), tbufs->cpio_srcbuf);
+	len = fill_cpio(c, ARRAY_SIZE(c), false, tbufs->cpio_srcbuf);
 
 	err = unpack_to_rootfs(tbufs->cpio_srcbuf, len);
 	KUNIT_EXPECT_NULL(test, err);
@@ -451,7 +461,7 @@ static void __init initramfs_test_fname_path_max(struct kunit *test)
 {
 	char *err;
 	size_t len;
-	struct kstat st0, st1;
+	struct kstat st0 = {}, st1 = {};
 	char fdata[] = "this file data will not be unpacked";
 	struct test_fname_path_max {
 		char fname_oversize[PATH_MAX + 1];
@@ -481,7 +491,7 @@ static void __init initramfs_test_fname_path_max(struct kunit *test)
 	memcpy(tbufs->fname_oversize, "fname_oversize",
 	       sizeof("fname_oversize") - 1);
 	memcpy(tbufs->fname_ok, "fname_ok", sizeof("fname_ok") - 1);
-	len = fill_cpio(c, ARRAY_SIZE(c), tbufs->cpio_src);
+	len = fill_cpio(c, ARRAY_SIZE(c), false, tbufs->cpio_src);
 
 	/* unpack skips over fname_oversize instead of returning an error */
 	err = unpack_to_rootfs(tbufs->cpio_src, len);
@@ -494,6 +504,45 @@ static void __init initramfs_test_fname_path_max(struct kunit *test)
 	kfree(tbufs);
 }
 
+static void __init initramfs_test_hdr_hex(struct kunit *test)
+{
+	char *err;
+	size_t len;
+	char fdata[] = "this file data will not be unpacked";
+	struct initramfs_test_bufs {
+		char cpio_src[(CPIO_HDRLEN + PATH_MAX + 3 + sizeof(fdata)) * 2];
+	} *tbufs = kzalloc(sizeof(struct initramfs_test_bufs), GFP_KERNEL);
+	struct initramfs_test_cpio c[] = { {
+		.magic = "070701",
+		.ino = 1,
+		.mode = S_IFREG | 0777,
+		.uid = 0x123456,
+		.gid = 0x123457,
+		.nlink = 1,
+		.namesize = sizeof("initramfs_test_hdr_hex_0"),
+		.fname = "initramfs_test_hdr_hex_0",
+		.filesize = sizeof(fdata),
+		.data = fdata,
+	}, {
+		.magic = "070701",
+		.ino = 2,
+		.mode = S_IFDIR | 0777,
+		.uid = 0x000056,
+		.gid = 0x000057,
+		.nlink = 1,
+		.namesize = sizeof("initramfs_test_hdr_hex_1"),
+		.fname = "initramfs_test_hdr_hex_1",
+	} };
+
+	/* inject_ox=true to add "0x" cpio field prefixes */
+	len = fill_cpio(c, ARRAY_SIZE(c), true, tbufs->cpio_src);
+
+	err = unpack_to_rootfs(tbufs->cpio_src, len);
+	KUNIT_EXPECT_NOT_NULL(test, err);
+
+	kfree(tbufs);
+}
+
 /*
  * The kunit_case/_suite struct cannot be marked as __initdata as this will be
  * used in debugfs to retrieve results after test has run.
@@ -507,6 +556,7 @@ static struct kunit_case __refdata initramfs_test_cases[] = {
 	KUNIT_CASE(initramfs_test_many),
 	KUNIT_CASE(initramfs_test_fname_pad),
 	KUNIT_CASE(initramfs_test_fname_path_max),
+	KUNIT_CASE(initramfs_test_hdr_hex),
 	{},
 };
 
diff --git a/init/main.c b/init/main.c
index 96f93bb06c490..e363232b428b4 100644
--- a/init/main.c
+++ b/init/main.c
@@ -324,51 +324,6 @@ static void * __init get_boot_config_from_initrd(size_t *_size)
 
 #ifdef CONFIG_BOOT_CONFIG
 
-static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata;
-
-#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0)
-
-static int __init xbc_snprint_cmdline(char *buf, size_t size,
-				      struct xbc_node *root)
-{
-	struct xbc_node *knode, *vnode;
-	char *end = buf + size;
-	const char *val, *q;
-	int ret;
-
-	xbc_node_for_each_key_value(root, knode, val) {
-		ret = xbc_node_compose_key_after(root, knode,
-					xbc_namebuf, XBC_KEYLEN_MAX);
-		if (ret < 0)
-			return ret;
-
-		vnode = xbc_node_get_child(knode);
-		if (!vnode) {
-			ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf);
-			if (ret < 0)
-				return ret;
-			buf += ret;
-			continue;
-		}
-		xbc_array_for_each_value(vnode, val) {
-			/*
-			 * For prettier and more readable /proc/cmdline, only
-			 * quote the value when necessary, i.e. when it contains
-			 * whitespace.
-			 */
-			q = strpbrk(val, " \t\r\n") ? "\"" : "";
-			ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ",
-				       xbc_namebuf, q, val, q);
-			if (ret < 0)
-				return ret;
-			buf += ret;
-		}
-	}
-
-	return buf - (end - size);
-}
-#undef rest
-
 /* Make an extra command line under given key word */
 static char * __init xbc_make_cmdline(const char *key)
 {
diff --git a/io_uring/epoll.c b/io_uring/epoll.c
index 8d4610246ba0a..eecd748cad018 100644
--- a/io_uring/epoll.c
+++ b/io_uring/epoll.c
@@ -51,10 +51,24 @@ int io_epoll_ctl_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
 {
 	struct io_epoll *ie = io_kiocb_to_cmd(req, struct io_epoll);
-	int ret;
 	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
+	struct epoll_key key;
+	int ret;
+
+	CLASS(fd, f)(ie->epfd);
+	if (fd_empty(f))
+		return -EBADF;
+
+	CLASS(fd, tf)(ie->fd);
+	if (fd_empty(tf))
+		return -EBADF;
+	/* disallow adding an epoll context to another epoll context */
+	if (ie->op == EPOLL_CTL_ADD && is_file_epoll(fd_file(tf)))
+		return -EINVAL;
 
-	ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
+	key.file = fd_file(tf);
+	key.fd = ie->fd;
+	ret = do_epoll_ctl_file(fd_file(f), ie->op, &key, &ie->event, force_nonblock);
 	if (force_nonblock && ret == -EAGAIN)
 		return -EAGAIN;
 
diff --git a/io_uring/fdinfo.c b/io_uring/fdinfo.c
index 001fb542dc11a..3ae4804765b94 100644
--- a/io_uring/fdinfo.c
+++ b/io_uring/fdinfo.c
@@ -224,7 +224,7 @@ static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
 		if (ctx->buf_table.nodes[i])
 			buf = ctx->buf_table.nodes[i]->buf;
 		if (buf)
-			seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf, buf->len);
+			seq_printf(m, "%5u: 0x%llx/%zu\n", i, buf->ubuf, buf->len);
 		else
 			seq_printf(m, "%5u: <none>\n", i);
 	}
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 036145ee466cd..7f4bc45e09d88 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -233,6 +233,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 		return NULL;
 
 	xa_init(&ctx->io_bl_xa);
+	xa_init(&ctx->hpage_acct);
 
 	/*
 	 * Use 5 bits less than the max cq entries, that should give us around
@@ -302,6 +303,7 @@ static __cold struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
 	io_free_alloc_caches(ctx);
 	kvfree(ctx->cancel_table.hbs);
 	xa_destroy(&ctx->io_bl_xa);
+	xa_destroy(&ctx->hpage_acct);
 	kfree(ctx);
 	return NULL;
 }
@@ -1738,10 +1740,9 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
 	const struct io_issue_def *def;
 	unsigned int sqe_flags;
 	int personality;
-	u8 opcode;
 
 	req->ctx = ctx;
-	req->opcode = opcode = READ_ONCE(sqe->opcode);
+	req->opcode = READ_ONCE(sqe->opcode);
 	/* same numerical values with corresponding REQ_F_*, safe to copy */
 	sqe_flags = READ_ONCE(sqe->flags);
 	req->flags = (__force io_req_flags_t) sqe_flags;
@@ -1751,13 +1752,13 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
 	req->cancel_seq_set = false;
 	req->async_data = NULL;
 
-	if (unlikely(opcode >= IORING_OP_LAST)) {
+	if (unlikely(req->opcode >= IORING_OP_LAST)) {
 		req->opcode = 0;
 		return io_init_fail_req(req, -EINVAL);
 	}
-	opcode = array_index_nospec(opcode, IORING_OP_LAST);
+	req->opcode = array_index_nospec(req->opcode, IORING_OP_LAST);
 
-	def = &io_issue_defs[opcode];
+	def = &io_issue_defs[req->opcode];
 	if (def->is_128 && !(ctx->flags & IORING_SETUP_SQE128)) {
 		/*
 		 * A 128b op on a non-128b SQ requires mixed SQE support as
@@ -2215,6 +2216,7 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
 	io_napi_free(ctx);
 	kvfree(ctx->cancel_table.hbs);
 	xa_destroy(&ctx->io_bl_xa);
+	xa_destroy(&ctx->hpage_acct);
 	kfree(ctx);
 }
 
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index e612a66ee80e9..1b657b7143733 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -312,7 +312,7 @@ static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx,
 	}
 
 	if (trace_io_uring_complete_enabled())
-		trace_io_uring_complete(req->ctx, req, cqe);
+		trace_call__io_uring_complete(req->ctx, req, cqe);
 	return true;
 }
 
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index 63061aa1cab94..dd54e43e9ddf4 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -21,7 +21,7 @@
 #define MAX_BIDS_PER_BGID (1 << 16)
 
 /* Mapped buffer ring, return io_uring_buf from head */
-#define io_ring_head_to_buf(br, head, mask)	&(br)->bufs[(head) & (mask)]
+#define io_ring_head_to_buf(br, head, mask)	(&(br)->bufs[(head) & (mask)])
 
 struct io_provide_buf {
 	struct file			*file;
diff --git a/io_uring/net.c b/io_uring/net.c
index 30cd22c0b934b..f01f1d25e930b 100644
--- a/io_uring/net.c
+++ b/io_uring/net.c
@@ -4,6 +4,7 @@
 #include <linux/file.h>
 #include <linux/slab.h>
 #include <linux/net.h>
+#include <linux/un.h>
 #include <linux/compat.h>
 #include <net/compat.h>
 #include <linux/io_uring.h>
@@ -1674,6 +1675,47 @@ void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
 	bctx->socket.protocol = sock->protocol;
 }
 
+void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
+{
+	struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect);
+	struct io_async_msghdr *iomsg = req->async_data;
+	struct sockaddr_storage *ss = &iomsg->addr;
+
+	/*
+	 * move_addr_to_kernel() skips the copy for addr_len == 0, so
+	 * iomsg->addr may hold stale data from a prior CONNECT. Bail
+	 * unless addr_len covers the family discriminator.
+	 */
+	if (conn->addr_len < (int)sizeof(sa_family_t))
+		return;
+
+	bctx->connect.family = ss->ss_family;
+	switch (ss->ss_family) {
+	case AF_INET: {
+		struct sockaddr_in *sin = (struct sockaddr_in *)ss;
+
+		if (conn->addr_len < (int)sizeof(*sin))
+			break;
+		bctx->connect.port = sin->sin_port;
+		bctx->connect.v4_addr = sin->sin_addr.s_addr;
+		break;
+	}
+	case AF_INET6: {
+		struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss;
+
+		if (conn->addr_len < (int)sizeof(*sin6))
+			break;
+		bctx->connect.port = sin6->sin6_port;
+		memcpy(bctx->connect.v6_addr, &sin6->sin6_addr,
+		       sizeof(bctx->connect.v6_addr));
+		break;
+	}
+	default:
+		/* family is set; per-family fields stay zero - family-only filtering */
+		break;
+	}
+}
+
 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
 	struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket);
@@ -1799,11 +1841,29 @@ int io_connect(struct io_kiocb *req, unsigned int issue_flags)
 	return IOU_COMPLETE;
 }
 
+/*
+ * Check if bind request would potentially end up with filename_create(),
+ * which in turn end up in mnt_want_write() which will grab the fs
+ * percpu start write sem. This can trigger a lockdep warning.
+ */
+static int io_bind_file_create(const struct io_async_msghdr *io, int addr_len)
+{
+	const struct sockaddr_un *sun;
+
+	if (io->addr.ss_family != AF_UNIX)
+		return 0;
+	if (addr_len <= offsetof(struct sockaddr_un, sun_path))
+		return 0;
+	sun = (const struct sockaddr_un *) &io->addr;
+	return sun->sun_path[0] != '\0';
+}
+
 int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
 	struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind);
 	struct sockaddr __user *uaddr;
 	struct io_async_msghdr *io;
+	int ret;
 
 	if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
 		return -EINVAL;
@@ -1814,7 +1874,12 @@ int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	io = io_msg_alloc_async(req);
 	if (unlikely(!io))
 		return -ENOMEM;
-	return move_addr_to_kernel(uaddr, bind->addr_len, &io->addr);
+	ret = move_addr_to_kernel(uaddr, bind->addr_len, &io->addr);
+	if (unlikely(ret))
+		return ret;
+	if (io_bind_file_create(io, bind->addr_len))
+		req->flags |= REQ_F_FORCE_ASYNC;
+	return 0;
 }
 
 int io_bind(struct io_kiocb *req, unsigned int issue_flags)
diff --git a/io_uring/net.h b/io_uring/net.h
index d4d1ddce50e3a..51fda715d3c0b 100644
--- a/io_uring/net.h
+++ b/io_uring/net.h
@@ -46,6 +46,7 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags);
 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 int io_socket(struct io_kiocb *req, unsigned int issue_flags);
 void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req);
+void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req);
 
 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 int io_connect(struct io_kiocb *req, unsigned int issue_flags);
@@ -69,4 +70,10 @@ static inline void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx,
 					  struct io_kiocb *req)
 {
 }
+
+static inline void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx,
+					   struct io_kiocb *req)
+{
+}
+
 #endif
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index c3ef52b708113..8ea6bd274607a 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -203,9 +203,11 @@ const struct io_issue_def io_issue_defs[] = {
 		.unbound_nonreg_file	= 1,
 		.pollout		= 1,
 #if defined(CONFIG_NET)
+		.filter_pdu_size	= sizeof_field(struct io_uring_bpf_ctx, connect),
 		.async_size		= sizeof(struct io_async_msghdr),
 		.prep			= io_connect_prep,
 		.issue			= io_connect,
+		.filter_populate	= io_connect_bpf_populate,
 #else
 		.prep			= io_eopnotsupp_prep,
 #endif
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 650303626be6e..7f553d115e365 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -28,7 +28,52 @@ struct io_rsrc_update {
 };
 
 static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
-			struct iovec *iov, struct page **last_hpage);
+						   struct iovec *iov);
+
+static int hpage_acct_ref(struct io_ring_ctx *ctx, struct page *hpage,
+			  bool *acct_new)
+{
+	unsigned long key = (unsigned long) hpage;
+	unsigned long count;
+	void *entry;
+	int ret;
+
+	lockdep_assert_held(&ctx->uring_lock);
+
+	entry = xa_load(&ctx->hpage_acct, key);
+	if (entry) {
+		*acct_new = false;
+		count = xa_to_value(entry) + 1;
+	} else {
+		ret = xa_reserve(&ctx->hpage_acct, key, GFP_KERNEL_ACCOUNT);
+		if (ret)
+			return ret;
+		*acct_new = true;
+		count = 1;
+	}
+	xa_store(&ctx->hpage_acct, key, xa_mk_value(count), GFP_KERNEL_ACCOUNT);
+	return 0;
+}
+
+static bool hpage_acct_unref(struct io_ring_ctx *ctx, struct page *hpage)
+{
+	unsigned long key = (unsigned long) hpage;
+	unsigned long count;
+	void *entry;
+
+	lockdep_assert_held(&ctx->uring_lock);
+
+	entry = xa_load(&ctx->hpage_acct, key);
+	if (WARN_ON_ONCE(!entry))
+		return false;
+	count = xa_to_value(entry);
+	if (count == 1) {
+		xa_erase(&ctx->hpage_acct, key);
+		return true;
+	}
+	xa_store(&ctx->hpage_acct, key, xa_mk_value(count - 1), GFP_KERNEL_ACCOUNT);
+	return false;
+}
 
 /* only define max */
 #define IORING_MAX_FIXED_FILES	(1U << 20)
@@ -88,9 +133,14 @@ int io_validate_user_buf_range(u64 uaddr, u64 ulen)
 	unsigned long tmp, base = (unsigned long)uaddr;
 	unsigned long acct_len = (unsigned long)PAGE_ALIGN(ulen);
 
-	/* arbitrary limit, but we need something */
-	if (ulen > SZ_1G || !ulen)
+	if (!ulen)
 		return -EFAULT;
+	/* 32-bit sanity checking */
+	if (ulen > ULONG_MAX || uaddr > ULONG_MAX)
+		return -EFAULT;
+	/* cap to 1TB for 64-bit */
+	if (ulen > SZ_1T)
+		return -EINVAL;
 	if (check_add_overflow(base, acct_len, &tmp))
 		return -EOVERFLOW;
 	return 0;
@@ -124,15 +174,53 @@ static void io_free_imu(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
 		kvfree(imu);
 }
 
+static unsigned long io_buffer_unaccount_pages(struct io_ring_ctx *ctx,
+					       struct io_mapped_ubuf *imu)
+{
+	struct page *seen = NULL;
+	unsigned long acct = 0;
+	int i;
+
+	if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user)
+		return 0;
+
+	for (i = 0; i < imu->nr_bvecs; i++) {
+		struct page *page = imu->bvec[i].bv_page;
+		struct page *hpage;
+
+		if (!PageCompound(page)) {
+			acct++;
+			continue;
+		}
+
+		hpage = compound_head(page);
+		if (hpage == seen)
+			continue;
+		seen = hpage;
+
+		/* Unaccount on last reference */
+		if (hpage_acct_unref(ctx, hpage))
+			acct += page_size(hpage) >> PAGE_SHIFT;
+		cond_resched();
+	}
+
+	return acct;
+}
+
 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf *imu)
 {
+	unsigned long acct_pages = 0;
+
+	/* Always decrement, so it works for cloned buffers too */
+	acct_pages = io_buffer_unaccount_pages(ctx, imu);
+
 	if (unlikely(refcount_read(&imu->refs) > 1)) {
 		if (!refcount_dec_and_test(&imu->refs))
 			return;
 	}
 
-	if (imu->acct_pages)
-		io_unaccount_mem(ctx->user, ctx->mm_account, imu->acct_pages);
+	if (acct_pages)
+		io_unaccount_mem(ctx->user, ctx->mm_account, acct_pages);
 	imu->release(imu->priv);
 	io_free_imu(ctx, imu);
 }
@@ -282,7 +370,6 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 {
 	u64 __user *tags = u64_to_user_ptr(up->tags);
 	struct iovec fast_iov, *iov;
-	struct page *last_hpage = NULL;
 	struct iovec __user *uvec;
 	u64 user_data = up->data;
 	__u32 done;
@@ -307,7 +394,7 @@ static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
 			err = -EFAULT;
 			break;
 		}
-		node = io_sqe_buffer_register(ctx, iov, &last_hpage);
+		node = io_sqe_buffer_register(ctx, iov);
 		if (IS_ERR(node)) {
 			err = PTR_ERR(node);
 			break;
@@ -605,76 +692,79 @@ int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
 }
 
 /*
- * Not super efficient, but this is just a registration time. And we do cache
- * the last compound head, so generally we'll only do a full search if we don't
- * match that one.
- *
- * We check if the given compound head page has already been accounted, to
- * avoid double accounting it. This allows us to account the full size of the
- * page, not just the constituent pages of a huge page.
+ * Undo hpage_acct_ref() calls made during io_buffer_account_pin() on failure.
+ * This operates on the pages array since imu->bvec isn't populated yet.
  */
-static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
-				  int nr_pages, struct page *hpage)
+static void io_buffer_unaccount_hpages(struct io_ring_ctx *ctx,
+				       struct page **pages, int nr_pages)
 {
-	int i, j;
+	struct page *seen = NULL;
+	int i;
+
+	if (!ctx->user)
+		return;
 
-	/* check current page array */
 	for (i = 0; i < nr_pages; i++) {
+		struct page *hpage;
+
 		if (!PageCompound(pages[i]))
 			continue;
-		if (compound_head(pages[i]) == hpage)
-			return true;
-	}
 
-	/* check previously registered pages */
-	for (i = 0; i < ctx->buf_table.nr; i++) {
-		struct io_rsrc_node *node = ctx->buf_table.nodes[i];
-		struct io_mapped_ubuf *imu;
-
-		if (!node)
+		hpage = compound_head(pages[i]);
+		if (hpage == seen)
 			continue;
-		imu = node->buf;
-		for (j = 0; j < imu->nr_bvecs; j++) {
-			if (!PageCompound(imu->bvec[j].bv_page))
-				continue;
-			if (compound_head(imu->bvec[j].bv_page) == hpage)
-				return true;
-		}
-	}
+		seen = hpage;
 
-	return false;
+		hpage_acct_unref(ctx, hpage);
+		cond_resched();
+	}
 }
 
 static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
-				 int nr_pages, struct io_mapped_ubuf *imu,
-				 struct page **last_hpage)
+				 int nr_pages)
 {
+	unsigned long acct_pages = 0;
+	struct page *seen = NULL;
 	int i, ret;
 
-	imu->acct_pages = 0;
+	if (!ctx->user)
+		return 0;
+
 	for (i = 0; i < nr_pages; i++) {
+		struct page *hpage;
+		bool acct_new;
+
 		if (!PageCompound(pages[i])) {
-			imu->acct_pages++;
-		} else {
-			struct page *hpage;
-
-			hpage = compound_head(pages[i]);
-			if (hpage == *last_hpage)
-				continue;
-			*last_hpage = hpage;
-			if (headpage_already_acct(ctx, pages, i, hpage))
-				continue;
-			imu->acct_pages += page_size(hpage) >> PAGE_SHIFT;
+			acct_pages++;
+			continue;
+		}
+
+		hpage = compound_head(pages[i]);
+		if (hpage == seen)
+			continue;
+		seen = hpage;
+
+		ret = hpage_acct_ref(ctx, hpage, &acct_new);
+		if (ret) {
+			io_buffer_unaccount_hpages(ctx, pages, i);
+			return ret;
 		}
+		if (acct_new)
+			acct_pages += page_size(hpage) >> PAGE_SHIFT;
+		cond_resched();
 	}
 
-	if (!imu->acct_pages)
-		return 0;
+	/* Try to account the memory */
+	if (acct_pages) {
+		ret = io_account_mem(ctx->user, ctx->mm_account, acct_pages);
+		if (ret) {
+			/* Undo the refs we just added */
+			io_buffer_unaccount_hpages(ctx, pages, nr_pages);
+			return ret;
+		}
+	}
 
-	ret = io_account_mem(ctx->user, ctx->mm_account, imu->acct_pages);
-	if (ret)
-		imu->acct_pages = 0;
-	return ret;
+	return 0;
 }
 
 static bool io_coalesce_buffer(struct page ***pages, int *nr_pages,
@@ -763,8 +853,7 @@ bool io_check_coalesce_buffer(struct page **page_array, int nr_pages,
 }
 
 static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
-						   struct iovec *iov,
-						   struct page **last_hpage)
+						   struct iovec *iov)
 {
 	struct io_mapped_ubuf *imu = NULL;
 	struct page **pages = NULL;
@@ -811,7 +900,7 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
 		goto done;
 
 	imu->nr_bvecs = nr_pages;
-	ret = io_buffer_account_pin(ctx, pages, nr_pages, imu, last_hpage);
+	ret = io_buffer_account_pin(ctx, pages, nr_pages);
 	if (ret)
 		goto done;
 
@@ -861,7 +950,6 @@ static struct io_rsrc_node *io_sqe_buffer_register(struct io_ring_ctx *ctx,
 int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
 			    unsigned int nr_args, u64 __user *tags)
 {
-	struct page *last_hpage = NULL;
 	struct io_rsrc_data data;
 	struct iovec fast_iov, *iov = &fast_iov;
 	const struct iovec __user *uvec;
@@ -904,7 +992,7 @@ int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
 			}
 		}
 
-		node = io_sqe_buffer_register(ctx, iov, &last_hpage);
+		node = io_sqe_buffer_register(ctx, iov);
 		if (IS_ERR(node)) {
 			ret = PTR_ERR(node);
 			break;
@@ -971,7 +1059,6 @@ int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
 
 	imu->ubuf = 0;
 	imu->len = blk_rq_bytes(rq);
-	imu->acct_pages = 0;
 	imu->folio_shift = PAGE_SHIFT;
 	refcount_set(&imu->refs, 1);
 	imu->release = release;
@@ -1136,6 +1223,56 @@ int io_import_reg_buf(struct io_kiocb *req, struct iov_iter *iter,
 	return io_import_fixed(ddir, iter, node->buf, buf_addr, len);
 }
 
+static int io_buffer_acct_cloned_hpages(struct io_ring_ctx *ctx,
+					struct io_mapped_ubuf *imu)
+{
+	struct page *seen = NULL;
+	int i, ret = 0;
+
+	if (imu->flags & IO_REGBUF_F_KBUF || !ctx->user)
+		return 0;
+
+	for (i = 0; i < imu->nr_bvecs; i++) {
+		struct page *page = imu->bvec[i].bv_page;
+		struct page *hpage;
+		bool acct_new;
+
+		if (!PageCompound(page))
+			continue;
+
+		hpage = compound_head(page);
+		if (hpage == seen)
+			continue;
+		seen = hpage;
+
+		/* Atomically add reference for cloned buffer */
+		ret = hpage_acct_ref(ctx, hpage, &acct_new);
+		if (ret)
+			break;
+
+		cond_resched();
+	}
+
+	if (!ret)
+		return 0;
+
+	/* Undo refs we added for bvecs [0..i) */
+	seen = NULL;
+	for (int j = 0; j < i; j++) {
+		struct page *p = imu->bvec[j].bv_page;
+		struct page *hp;
+
+		if (!PageCompound(p))
+			continue;
+		hp = compound_head(p);
+		if (hp == seen)
+			continue;
+		seen = hp;
+		hpage_acct_unref(ctx, hp);
+	}
+	return ret;
+}
+
 /* Lock two rings at once. The rings must be different! */
 static void lock_two_rings(struct io_ring_ctx *ctx1, struct io_ring_ctx *ctx2)
 {
@@ -1218,6 +1355,14 @@ static int io_clone_buffers(struct io_ring_ctx *ctx, struct io_ring_ctx *src_ctx
 
 			refcount_inc(&src_node->buf->refs);
 			dst_node->buf = src_node->buf;
+			/* track compound references to clones */
+			ret = io_buffer_acct_cloned_hpages(ctx, src_node->buf);
+			if (ret) {
+				refcount_dec(&src_node->buf->refs);
+				io_cache_free(&ctx->node_cache, dst_node);
+				io_rsrc_data_free(ctx, &data);
+				return ret;
+			}
 		}
 		data.nodes[off++] = dst_node;
 		i++;
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 44e3386f7c1ca..98ae8ef51009d 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -34,15 +34,14 @@ enum {
 
 struct io_mapped_ubuf {
 	u64		ubuf;
-	unsigned int	len;
+	size_t		len;
 	unsigned int	nr_bvecs;
 	unsigned int    folio_shift;
 	refcount_t	refs;
-	unsigned long	acct_pages;
-	void		(*release)(void *);
-	void		*priv;
 	u8		flags;
 	u8		dir;
+	void		(*release)(void *);
+	void		*priv;
 	struct bio_vec	bvec[] __counted_by(nr_bvecs);
 };
 
diff --git a/io_uring/timeout.c b/io_uring/timeout.c
index 6353a4d979dc0..c4dd26cf342da 100644
--- a/io_uring/timeout.c
+++ b/io_uring/timeout.c
@@ -417,8 +417,10 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
 	 * done in io_req_task_link_timeout(), if needed.
 	 */
 	if (prev) {
-		if (!req_ref_inc_not_zero(prev))
+		if (!req_ref_inc_not_zero(prev)) {
+			io_remove_next_linked(prev);
 			prev = NULL;
+		}
 	}
 	list_del(&timeout->list);
 	timeout->prev = prev;
diff --git a/io_uring/waitid.c b/io_uring/waitid.c
index d25d60aed6afc..32f68fd7fcdda 100644
--- a/io_uring/waitid.c
+++ b/io_uring/waitid.c
@@ -275,6 +275,7 @@ int io_waitid_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 	iw->options = READ_ONCE(sqe->file_index);
 	iw->head = NULL;
 	iw->infop = u64_to_user_ptr(READ_ONCE(sqe->addr2));
+	memset(&iw->info, 0, sizeof(iw->info));
 	return 0;
 }
 
diff --git a/ipc/shm.c b/ipc/shm.c
index a95dae4477076..b3e8a58e177d3 100644
--- a/ipc/shm.c
+++ b/ipc/shm.c
@@ -418,15 +418,17 @@ static int shm_try_destroy_orphaned(int id, void *p, void *data)
 	 * We want to destroy segments without users and with already
 	 * exit'ed originating process.
 	 *
-	 * As shp->* are changed under rwsem, it's safe to skip shp locking.
+	 * shm_nattch can be changed under shm_perm.lock without holding the
+	 * rwsem, so take the object lock before checking shm_may_destroy().
 	 */
 	if (!list_empty(&shp->shm_clist))
 		return 0;
 
-	if (shm_may_destroy(shp)) {
-		shm_lock_by_ptr(shp);
+	shm_lock_by_ptr(shp);
+	if (shm_may_destroy(shp))
 		shm_destroy(ns, shp);
-	}
+	else
+		shm_unlock(shp);
 	return 0;
 }
 
diff --git a/ipc/util.c b/ipc/util.c
index 9eb89820594ee..1737d776bc089 100644
--- a/ipc/util.c
+++ b/ipc/util.c
@@ -253,7 +253,7 @@ static inline int ipc_idr_alloc(struct ipc_ids *ids, struct kern_ipc_perm *new)
 	} else {
 		new->seq = ipcid_to_seqx(next_id);
 		idx = idr_alloc(&ids->ipcs_idr, new, ipcid_to_idx(next_id),
-				0, GFP_NOWAIT);
+				ipc_mni, GFP_NOWAIT);
 	}
 	if (idx >= 0)
 		new->id = (new->seq << ipcmni_seq_shift()) + idx;
diff --git a/kernel/acct.c b/kernel/acct.c
index cbbf79d718cf4..c440d43479ca5 100644
--- a/kernel/acct.c
+++ b/kernel/acct.c
@@ -249,7 +249,7 @@ static int acct_on(const char __user *name)
 		return -EINVAL;
 
 	/* Exclude procfs and sysfs. */
-	if (file_inode(file)->i_sb->s_iflags & SB_I_USERNS_VISIBLE)
+	if (file_inode(file)->i_sb->s_type->fs_flags & FS_USERNS_MOUNT_RESTRICTED)
 		return -EINVAL;
 
 	if (!(file->f_mode & FMODE_CAN_WRITE))
diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile
index 399007b67a923..4dc41bf5780cc 100644
--- a/kernel/bpf/Makefile
+++ b/kernel/bpf/Makefile
@@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse
 endif
 CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy)
 
-obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o liveness.o const_fold.o
+obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o cnum.o log.o token.o liveness.o const_fold.o
 obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o
 obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o
 obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o bpf_insn_array.o
diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index 49a8f7b1beef5..a497c5913bd49 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -729,7 +729,7 @@ static void arena_free_pages(struct bpf_arena *arena, long uaddr, long page_cnt,
 
 	llist_for_each_safe(pos, t, __llist_del_all(&free_pages)) {
 		page = llist_entry(pos, struct page, pcp_llist);
-		if (page_cnt == 1 && page_mapped(page)) /* mapped by some user process */
+		if (page_cnt == 1 && page_ref_count(page) > 1) /* maybe mapped by user space */
 			/* Optimization for the common case of page_cnt==1:
 			 * If page wasn't mapped into some user vma there
 			 * is no need to call zap_pages which is slow. When
diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c
index 5e25e03535094..dfb2110ab7335 100644
--- a/kernel/bpf/arraymap.c
+++ b/kernel/bpf/arraymap.c
@@ -827,7 +827,7 @@ const struct bpf_map_ops array_map_ops = {
 };
 
 const struct bpf_map_ops percpu_array_map_ops = {
-	.map_meta_equal = bpf_map_meta_equal,
+	.map_meta_equal = array_map_meta_equal,
 	.map_alloc_check = array_map_alloc_check,
 	.map_alloc = array_map_alloc,
 	.map_free = array_map_free,
diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c
index 854731dc93fe7..2e4ae0ef08609 100644
--- a/kernel/bpf/backtrack.c
+++ b/kernel/bpf/backtrack.c
@@ -9,7 +9,7 @@
 
 /* for any branch, call, exit record the history of jmps in the given state */
 int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur,
-			 int insn_flags, u64 linked_regs)
+			 int insn_flags, int spi, int frame, u64 linked_regs)
 {
 	u32 cnt = cur->jmp_history_cnt;
 	struct bpf_jmp_history_entry *p;
@@ -25,6 +25,8 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state
 				env, "insn history: insn_idx %d cur flags %x new flags %x",
 				env->insn_idx, env->cur_hist_ent->flags, insn_flags);
 		env->cur_hist_ent->flags |= insn_flags;
+		env->cur_hist_ent->spi = spi;
+		env->cur_hist_ent->frame = frame;
 		verifier_bug_if(env->cur_hist_ent->linked_regs != 0, env,
 				"insn history: insn_idx %d linked_regs: %#llx",
 				env->insn_idx, env->cur_hist_ent->linked_regs);
@@ -43,6 +45,8 @@ int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state
 	p->idx = env->insn_idx;
 	p->prev_idx = env->prev_insn_idx;
 	p->flags = insn_flags;
+	p->spi = spi;
+	p->frame = frame;
 	p->linked_regs = linked_regs;
 	cur->jmp_history_cnt = cnt;
 	env->cur_hist_ent = p;
@@ -64,16 +68,6 @@ static bool is_atomic_fetch_insn(const struct bpf_insn *insn)
 	       (insn->imm & BPF_FETCH);
 }
 
-static int insn_stack_access_spi(int insn_flags)
-{
-	return (insn_flags >> INSN_F_SPI_SHIFT) & INSN_F_SPI_MASK;
-}
-
-static int insn_stack_access_frameno(int insn_flags)
-{
-	return insn_flags & INSN_F_FRAMENO_MASK;
-}
-
 /* Backtrack one insn at a time. If idx is not at the top of recorded
  * history then previous instruction came from straight line execution.
  * Return -ENOENT if we exhausted all instructions within given state.
@@ -135,11 +129,21 @@ static inline u32 bt_empty(struct backtrack_state *bt)
 	int i;
 
 	for (i = 0; i <= bt->frame; i++)
-		mask |= bt->reg_masks[i] | bt->stack_masks[i];
+		mask |= bt->reg_masks[i] | bt->stack_masks[i] | bt->stack_arg_masks[i];
 
 	return mask == 0;
 }
 
+static inline void bt_clear_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot)
+{
+	bt->stack_arg_masks[frame] &= ~(1 << slot);
+}
+
+static inline bool bt_is_frame_stack_arg_slot_set(struct backtrack_state *bt, u32 frame, u32 slot)
+{
+	return bt->stack_arg_masks[frame] & (1 << slot);
+}
+
 static inline int bt_subprog_enter(struct backtrack_state *bt)
 {
 	if (bt->frame == MAX_CALL_FRAMES - 1) {
@@ -200,6 +204,11 @@ static inline u64 bt_stack_mask(struct backtrack_state *bt)
 	return bt->stack_masks[bt->frame];
 }
 
+static inline u8 bt_stack_arg_mask(struct backtrack_state *bt)
+{
+	return bt->stack_arg_masks[bt->frame];
+}
+
 static inline bool bt_is_reg_set(struct backtrack_state *bt, u32 reg)
 {
 	return bt->reg_masks[bt->frame] & (1 << reg);
@@ -341,6 +350,19 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 			return 0;
 		bt_clear_reg(bt, load_reg);
 
+		if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) {
+			spi = hist->spi;
+			/*
+			 * Stack arg read: callee reads from r11+off, but
+			 * the data lives in the caller's stack_arg_regs.
+			 * Set the mask in the caller frame so precision
+			 * is marked in the caller's slot at the callee
+			 * entry checkpoint.
+			 */
+			bt_set_frame_stack_arg_slot(bt, bt->frame - 1, spi);
+			return 0;
+		}
+
 		/* scalars can only be spilled into stack w/o losing precision.
 		 * Load from any other memory can be zero extended.
 		 * The desire to keep that precision is already indicated
@@ -353,8 +375,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 		 * that [fp - off] slot contains scalar that needs to be
 		 * tracked with precision
 		 */
-		spi = insn_stack_access_spi(hist->flags);
-		fr = insn_stack_access_frameno(hist->flags);
+		spi = hist->spi;
+		fr = hist->frame;
 		bpf_bt_set_frame_slot(bt, fr, spi);
 	} else if (class == BPF_STX || class == BPF_ST) {
 		if (bt_is_reg_set(bt, dreg))
@@ -363,11 +385,22 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 			 * encountered a case of pointer subtraction.
 			 */
 			return -ENOTSUPP;
+
+		if (hist && hist->flags & INSN_F_STACK_ARG_ACCESS) {
+			spi = hist->spi;
+			if (!bt_is_frame_stack_arg_slot_set(bt, bt->frame, spi))
+				return 0;
+			bt_clear_frame_stack_arg_slot(bt, bt->frame, spi);
+			if (class == BPF_STX)
+				bt_set_reg(bt, sreg);
+			return 0;
+		}
+
 		/* scalars can only be spilled into stack */
 		if (!hist || !(hist->flags & INSN_F_STACK_ACCESS))
 			return 0;
-		spi = insn_stack_access_spi(hist->flags);
-		fr = insn_stack_access_frameno(hist->flags);
+		spi = hist->spi;
+		fr = hist->frame;
 		if (!bt_is_frame_slot_set(bt, fr, spi))
 			return 0;
 		bt_clear_frame_slot(bt, fr, spi);
@@ -431,6 +464,12 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx,
 						bpf_bt_set_frame_reg(bt, bt->frame - 1, i);
 					}
 				}
+				if (bt_stack_arg_mask(bt)) {
+					verifier_bug(env,
+						     "static subprog leftover stack arg slots %x",
+						     bt_stack_arg_mask(bt));
+					return -EFAULT;
+				}
 				if (bt_subprog_exit(bt))
 					return -EFAULT;
 				return 0;
@@ -901,6 +940,17 @@ int bpf_mark_chain_precision(struct bpf_verifier_env *env,
 					*changed = true;
 				}
 			}
+			for (i = 0; i < func->out_stack_arg_cnt; i++) {
+				if (!bt_is_frame_stack_arg_slot_set(bt, fr, i))
+					continue;
+				reg = &func->stack_arg_regs[i];
+				if (reg->type != SCALAR_VALUE || reg->precise) {
+					bt_clear_frame_stack_arg_slot(bt, fr, i);
+				} else {
+					reg->precise = true;
+					*changed = true;
+				}
+			}
 			if (env->log.level & BPF_LOG_LEVEL2) {
 				fmt_reg_mask(env->tmp_str_buf, TMP_STR_BUF_LEN,
 					     bt_frame_reg_mask(bt, fr));
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index a62d785812076..17d4ab0a82068 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -182,7 +182,6 @@
 #define BITS_ROUNDUP_BYTES(bits) \
 	(BITS_ROUNDDOWN_BYTES(bits) + !!BITS_PER_BYTE_MASKED(bits))
 
-#define BTF_INFO_MASK 0x9f00ffff
 #define BTF_INT_MASK 0x0fffffff
 #define BTF_TYPE_ID_VALID(type_id) ((type_id) <= BTF_MAX_TYPE)
 #define BTF_STR_OFFSET_VALID(name_off) ((name_off) <= BTF_MAX_NAME_OFFSET)
@@ -289,7 +288,7 @@ enum verifier_phase {
 struct resolve_vertex {
 	const struct btf_type *t;
 	u32 type_id;
-	u16 next_member;
+	u32 next_member;
 };
 
 enum visit_state {
@@ -2031,7 +2030,7 @@ static int env_stack_push(struct btf_verifier_env *env,
 }
 
 static void env_stack_set_next_member(struct btf_verifier_env *env,
-				      u16 next_member)
+				      u32 next_member)
 {
 	env->stack[env->top_stack - 1].next_member = next_member;
 }
@@ -3293,7 +3292,7 @@ static s32 btf_struct_check_meta(struct btf_verifier_env *env,
 	struct btf *btf = env->btf;
 	u32 struct_size = t->size;
 	u32 offset;
-	u16 i;
+	u32 i;
 
 	meta_needed = btf_type_vlen(t) * sizeof(*member);
 	if (meta_left < meta_needed) {
@@ -3369,7 +3368,7 @@ static int btf_struct_resolve(struct btf_verifier_env *env,
 {
 	const struct btf_member *member;
 	int err;
-	u16 i;
+	u32 i;
 
 	/* Before continue resolving the next_member,
 	 * ensure the last member is indeed resolved to a
@@ -4447,7 +4446,7 @@ static s32 btf_enum_check_meta(struct btf_verifier_env *env,
 	const struct btf_enum *enums = btf_type_enum(t);
 	struct btf *btf = env->btf;
 	const char *fmt_str;
-	u16 i, nr_enums;
+	u32 i, nr_enums;
 	u32 meta_needed;
 
 	nr_enums = btf_type_vlen(t);
@@ -4555,7 +4554,7 @@ static s32 btf_enum64_check_meta(struct btf_verifier_env *env,
 	const struct btf_enum64 *enums = btf_type_enum64(t);
 	struct btf *btf = env->btf;
 	const char *fmt_str;
-	u16 i, nr_enums;
+	u32 i, nr_enums;
 	u32 meta_needed;
 
 	nr_enums = btf_type_vlen(t);
@@ -4683,7 +4682,7 @@ static void btf_func_proto_log(struct btf_verifier_env *env,
 			       const struct btf_type *t)
 {
 	const struct btf_param *args = (const struct btf_param *)(t + 1);
-	u16 nr_args = btf_type_vlen(t), i;
+	u32 nr_args = btf_type_vlen(t), i;
 
 	btf_verifier_log(env, "return=%u args=(", t->type);
 	if (!nr_args) {
@@ -4929,7 +4928,7 @@ static int btf_datasec_resolve(struct btf_verifier_env *env,
 {
 	const struct btf_var_secinfo *vsi;
 	struct btf *btf = env->btf;
-	u16 i;
+	u32 i;
 
 	env->resolve_mode = RESOLVE_TBD;
 	for_each_vsi_from(i, v->next_member, v->t, vsi) {
@@ -5183,7 +5182,7 @@ static int btf_func_proto_check(struct btf_verifier_env *env,
 	const struct btf_type *ret_type;
 	const struct btf_param *args;
 	const struct btf *btf;
-	u16 nr_args, i;
+	u32 nr_args, i;
 	int err;
 
 	btf = env->btf;
@@ -5278,7 +5277,7 @@ static int btf_func_check(struct btf_verifier_env *env,
 	const struct btf_type *proto_type;
 	const struct btf_param *args;
 	const struct btf *btf;
-	u16 nr_args, i;
+	u32 nr_args, i;
 
 	btf = env->btf;
 	proto_type = btf_type_by_id(btf, t->type);
@@ -5336,12 +5335,6 @@ static s32 btf_check_meta(struct btf_verifier_env *env,
 	}
 	meta_left -= sizeof(*t);
 
-	if (t->info & ~BTF_INFO_MASK) {
-		btf_verifier_log(env, "[%u] Invalid btf_info:%x",
-				 env->log_type_id, t->info);
-		return -EINVAL;
-	}
-
 	if (BTF_INFO_KIND(t->info) > BTF_KIND_MAX ||
 	    BTF_INFO_KIND(t->info) == BTF_KIND_UNKN) {
 		btf_verifier_log(env, "[%u] Invalid kind:%u",
@@ -5914,25 +5907,10 @@ static int btf_check_type_tags(struct btf_verifier_env *env,
 	return 0;
 }
 
-static int finalize_log(struct bpf_verifier_log *log, bpfptr_t uattr, u32 uattr_size)
-{
-	u32 log_true_size;
-	int err;
-
-	err = bpf_vlog_finalize(log, &log_true_size);
-
-	if (uattr_size >= offsetofend(union bpf_attr, btf_log_true_size) &&
-	    copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, btf_log_true_size),
-				  &log_true_size, sizeof(log_true_size)))
-		err = -EFAULT;
-
-	return err;
-}
-
-static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
+static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr,
+			     struct bpf_log_attr *attr_log)
 {
 	bpfptr_t btf_data = make_bpfptr(attr->btf, uattr.is_kernel);
-	char __user *log_ubuf = u64_to_user_ptr(attr->btf_log_buf);
 	struct btf_struct_metas *struct_meta_tab;
 	struct btf_verifier_env *env = NULL;
 	struct btf *btf = NULL;
@@ -5949,8 +5927,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uat
 	/* user could have requested verbose verifier output
 	 * and supplied buffer to store the verification trace
 	 */
-	err = bpf_vlog_init(&env->log, attr->btf_log_level,
-			    log_ubuf, attr->btf_log_size);
+	err = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size);
 	if (err)
 		goto errout_free;
 
@@ -6015,7 +5992,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uat
 		}
 	}
 
-	err = finalize_log(&env->log, uattr, uattr_size);
+	err = bpf_log_attr_finalize(attr_log, &env->log);
 	if (err)
 		goto errout_free;
 
@@ -6027,7 +6004,7 @@ static struct btf *btf_parse(const union bpf_attr *attr, bpfptr_t uattr, u32 uat
 	btf_free_struct_meta_tab(btf);
 errout:
 	/* overwrite err with -ENOSPC or -EFAULT */
-	ret = finalize_log(&env->log, uattr, uattr_size);
+	ret = bpf_log_attr_finalize(attr_log, &env->log);
 	if (ret)
 		err = ret;
 errout_free:
@@ -7887,10 +7864,23 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 	}
 	args = (const struct btf_param *)(t + 1);
 	nargs = btf_type_vlen(t);
+	sub->arg_cnt = nargs;
+	if (nargs > MAX_BPF_FUNC_ARGS) {
+		bpf_log(log, "kernel supports at most %d parameters, function %s has %d\n",
+			MAX_BPF_FUNC_ARGS, tname, nargs);
+		return -EFAULT;
+	}
 	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
-		if (!is_global)
-			return -EINVAL;
-		bpf_log(log, "Global function %s() with %d > %d args. Buggy compiler.\n",
+		if (!bpf_jit_supports_stack_args()) {
+			bpf_log(log, "JIT does not support function %s() with %d args\n",
+				tname, nargs);
+			return -EFAULT;
+		}
+		sub->stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS;
+	}
+
+	if (is_global && nargs > MAX_BPF_FUNC_REG_ARGS) {
+		bpf_log(log, "global function %s has %d > %d args, stack args not supported\n",
 			tname, nargs, MAX_BPF_FUNC_REG_ARGS);
 		return -EINVAL;
 	}
@@ -7973,7 +7963,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 				bpf_log(log, "arg#%d has invalid combination of tags\n", i);
 				return -EINVAL;
 			}
-			sub->args[i].arg_type = ARG_PTR_TO_DYNPTR | MEM_RDONLY;
+			sub->args[i].arg_type = ARG_PTR_TO_DYNPTR;
 			continue;
 		}
 		if (tags & ARG_TAG_TRUSTED) {
@@ -8074,7 +8064,6 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 		return -EINVAL;
 	}
 
-	sub->arg_cnt = nargs;
 	sub->args_cached = true;
 
 	return 0;
@@ -8196,12 +8185,12 @@ static int __btf_new_fd(struct btf *btf)
 	return anon_inode_getfd("btf", &btf_fops, btf, O_RDONLY | O_CLOEXEC);
 }
 
-int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
+int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log)
 {
 	struct btf *btf;
 	int ret;
 
-	btf = btf_parse(attr, uattr, uattr_size);
+	btf = btf_parse(attr, uattr, attr_log);
 	if (IS_ERR(btf))
 		return PTR_ERR(btf);
 
diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c
index 998f42a8189a4..26d37066465f3 100644
--- a/kernel/bpf/cfg.c
+++ b/kernel/bpf/cfg.c
@@ -64,11 +64,19 @@ static void mark_subprog_might_sleep(struct bpf_verifier_env *env, int off)
 	subprog->might_sleep = true;
 }
 
+static void mark_subprog_might_throw(struct bpf_verifier_env *env, int off)
+{
+	struct bpf_subprog_info *subprog;
+
+	subprog = bpf_find_containing_subprog(env, off);
+	subprog->might_throw = true;
+}
+
 /* 't' is an index of a call-site.
  * 'w' is a callee entry point.
  * Eventually this function would be called when env->cfg.insn_state[w] == EXPLORED.
  * Rely on DFS traversal order and absence of recursive calls to guarantee that
- * callee's change_pkt_data marks would be correct at that moment.
+ * callee's effect marks would be correct at that moment.
  */
 static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
 {
@@ -78,6 +86,7 @@ static void merge_callee_effects(struct bpf_verifier_env *env, int t, int w)
 	callee = bpf_find_containing_subprog(env, w);
 	caller->changes_pkt_data |= callee->changes_pkt_data;
 	caller->might_sleep |= callee->might_sleep;
+	caller->might_throw |= callee->might_throw;
 }
 
 enum {
@@ -509,6 +518,8 @@ static int visit_insn(int t, struct bpf_verifier_env *env)
 				mark_subprog_might_sleep(env, t);
 			if (ret == 0 && bpf_is_kfunc_pkt_changing(&meta))
 				mark_subprog_changes_pkt_data(env, t);
+			if (ret == 0 && bpf_is_throw_kfunc(insn))
+				mark_subprog_might_throw(env, t);
 		}
 		return visit_func_call_insn(t, insns, env, insn->src_reg == BPF_PSEUDO_CALL);
 
diff --git a/kernel/bpf/cnum.c b/kernel/bpf/cnum.c
new file mode 100644
index 0000000000000..86142cb2aee5f
--- /dev/null
+++ b/kernel/bpf/cnum.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#include <linux/bits.h>
+
+#define T 32
+#include "cnum_defs.h"
+#undef T
+
+#define T 64
+#include "cnum_defs.h"
+#undef T
+
+struct cnum32 cnum32_from_cnum64(struct cnum64 cnum)
+{
+	if (cnum64_is_empty(cnum))
+		return CNUM32_EMPTY;
+
+	if (cnum.size >= U32_MAX)
+		return (struct cnum32){ .base = 0, .size = U32_MAX };
+	else
+		return (struct cnum32){ .base = (u32)cnum.base, .size = cnum.size };
+}
+
+/*
+ * Suppose 'a' and 'b' are laid out as follows:
+ *
+ *                                                          64-bit number axis --->
+ *
+ * N*2^32                   (N+1)*2^32                (N+2)*2^32                (N+3)*2^32
+ * ||------|---|=====|-------||----------|=====|-------||----------|=====|----|--||
+ *         |   |< b >|                   |< b >|                   |< b >|    |
+ *         |   |                                                         |    |
+ *         |<--+--------------------------- a ---------------------------+--->|
+ *             |                                                         |
+ *             |<-------------------------- t -------------------------->|
+ *
+ * In such a case it is possible to infer a more tight representation t
+ * such that ∀ v ∈ a, (u32)v ∈ b: v ∈ t.
+ */
+struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b)
+{
+	/*
+	 * To simplify reasoning, rotate the circles so that [virtual] a1 starts
+	 * at u32 boundary, b1 represents b in this new frame of reference.
+	 */
+	struct cnum32 b1 = { b.base - (u32)a.base, b.size };
+	struct cnum64 t = a;
+	u64 d, b1_max;
+
+	if (cnum64_is_empty(a) || cnum32_is_empty(b))
+		return CNUM64_EMPTY;
+
+	if (cnum32_urange_overflow(b1)) {
+		b1_max = (u32)b1.base + (u32)b1.size; /* overflow here is fine and necessary */
+		if ((u32)a.size > b1_max && (u32)a.size < b1.base) {
+			/*
+			 * N*2^32                   (N+1)*2^32
+			 * ||=====|------------|=====||=====|---------|---|=====||
+			 *  |b1 ->|            |<- b1||b1 ->|         |   |<- b1|
+			 *  |<----------------- a1 ------------------>|
+			 *  |<-------------- t ------------>|<-- d -->| (after adjustment)
+			 *                                  ^
+			 *                                b1_max
+			 */
+			d = (u32)a.size - b1_max;
+			t.size -= d;
+		} else {
+			/*
+			 * No adjustments possible in the following cases:
+			 *
+			 * ||=====|------------|=====||===|=|-------------|=|===||
+			 *  |b1 ->|            |<- b1||b1 +>|             |<+ b1|
+			 *  |<----------------- a1 ------>|                 |
+			 *  |<----------------- (or) a1 ------------------->|
+			 */
+		}
+	} else {
+		if (t.size < b1.base)
+			/*
+			 * N*2^32                   (N+1)*2^32
+			 * ||----------|--|=======|--||------>
+			 *  |<-- a1 -->|  |<- b ->|
+			 */
+			return CNUM64_EMPTY;
+		/*
+		 * N*2^32                   (N+1)*2^32
+		 * ||-------------|========|-||-----| -------|========|-||
+		 *  |             |<- b1 ->|        |        |<- b1 ->|
+		 *  |<------------+ a1 ------------>|
+		 *                |<------ t ------>| (after adjustment)
+		 */
+		t.base += b1.base;
+		t.size -= b1.base;
+		b1_max = b1.base + b1.size;
+		d = 0;
+		if ((u32)a.size < b1.base)
+			/*
+			 * N*2^32                   (N+1)*2^32
+			 * ||-------------|========|-||------|-------|========|-||
+			 *  |             |<- b1 ->|         |       |<- b1 ->|
+			 *  |<------------+-- a1 --+-------->|
+			 *                |<- t  ->|<-- d -->| (after adjustment)
+			 */
+			d = (u32)a.size + (BIT_ULL(32) - b1_max);
+		else if ((u32)a.size >= b1_max)
+			/*
+			 * N*2^32                   (N+1)*2^32
+			 * ||--|========|------------||--|========|-------|-----||
+			 *  |  |<- b1 ->|                |<- b1 ->|       |
+			 *  |<-+------------------ a1 ------------+------>|
+			 *     |<-------------- t --------------->|<- d ->| (after adjustment)
+			 */
+			d = (u32)a.size - b1_max;
+		if (t.size < d)
+			return CNUM64_EMPTY;
+		t.size -= d;
+	}
+	return t;
+}
diff --git a/kernel/bpf/cnum_defs.h b/kernel/bpf/cnum_defs.h
new file mode 100644
index 0000000000000..a90e317e35780
--- /dev/null
+++ b/kernel/bpf/cnum_defs.h
@@ -0,0 +1,247 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
+
+#ifndef T
+#error "Define T (bit width: 32, 64) before including cnum_defs.h"
+#endif
+
+#include <linux/cnum.h>
+#include <linux/kernel.h>
+#include <linux/limits.h>
+#include <linux/minmax.h>
+#include <linux/compiler_types.h>
+
+#define cnum_t   __PASTE(cnum, T)
+#define ut       __PASTE(u, T)
+#define st       __PASTE(s, T)
+#define UT_MAX   __PASTE(__PASTE(U, T), _MAX)
+#define ST_MAX   __PASTE(__PASTE(S, T), _MAX)
+#define ST_MIN   __PASTE(__PASTE(S, T), _MIN)
+#define EMPTY    __PASTE(__PASTE(CNUM, T), _EMPTY)
+#define FN(name) __PASTE(__PASTE(cnum, T), __PASTE(_, name))
+
+struct cnum_t FN(from_urange)(ut min, ut max)
+{
+	return (struct cnum_t){ .base = min, .size = (ut)max - min };
+}
+
+struct cnum_t FN(from_srange)(st min, st max)
+{
+	ut size = (ut)max - (ut)min;
+	ut base = size == UT_MAX ? 0 : (ut)min;
+
+	return (struct cnum_t){ .base = base, .size = size };
+}
+
+/* True if this cnum represents two unsigned ranges. */
+static inline bool FN(urange_overflow)(struct cnum_t cnum)
+{
+	/* Same as cnum.base + cnum.size > UT_MAX but avoids overflow */
+	return cnum.size > UT_MAX - (ut)cnum.base;
+}
+
+/*
+ * cnum{T}_umin / cnum{T}_umax query an unsigned range represented by this cnum.
+ * If cnum represents a range crossing the UT_MAX/0 boundary, the unbound range
+ * [0..UT_MAX] is returned.
+ */
+ut FN(umin)(struct cnum_t cnum)
+{
+	return FN(urange_overflow)(cnum) ? 0 : cnum.base;
+}
+EXPORT_SYMBOL_GPL(FN(umin));
+
+ut FN(umax)(struct cnum_t cnum)
+{
+	return FN(urange_overflow)(cnum) ? UT_MAX : cnum.base + cnum.size;
+}
+EXPORT_SYMBOL_GPL(FN(umax));
+
+/* True if this cnum represents two signed ranges. */
+static inline bool FN(srange_overflow)(struct cnum_t cnum)
+{
+	return FN(contains)(cnum, (ut)ST_MAX) && FN(contains)(cnum, (ut)ST_MIN);
+}
+
+/*
+ * cnum{T}_smin / cnum{T}_smax query a signed range represented by this cnum.
+ * If cnum represents a range crossing the ST_MAX/ST_MIN boundary, the unbound range
+ * [ST_MIN..ST_MAX] is returned.
+ */
+st FN(smin)(struct cnum_t cnum)
+{
+	return FN(srange_overflow)(cnum)
+	       ? ST_MIN
+	       : min((st)cnum.base, (st)(cnum.base + cnum.size));
+}
+
+st FN(smax)(struct cnum_t cnum)
+{
+	return FN(srange_overflow)(cnum)
+	       ? ST_MAX
+	       : max((st)cnum.base, (st)(cnum.base + cnum.size));
+}
+
+/*
+ * Returns a possibly empty intersection of cnums 'a' and 'b'.
+ * If 'a' and 'b' intersect in two sub-arcs, the function over-approximates
+ * and returns either 'a' or 'b', whichever is smaller.
+ */
+struct cnum_t FN(intersect)(struct cnum_t a, struct cnum_t b)
+{
+	struct cnum_t b1;
+	ut dbase;
+
+	if (FN(is_empty)(a) || FN(is_empty)(b))
+		return EMPTY;
+
+	if (a.base > b.base)
+		swap(a, b);
+
+	/*
+	 * Rotate frame of reference such that a.base is 0.
+	 * 'b1' is 'b' in this frame of reference.
+	 */
+	dbase = b.base - a.base;
+	b1 = (struct cnum_t){ dbase, b.size };
+	if (FN(urange_overflow)(b1)) {
+		if (b1.base <= a.size) {
+			/*
+			 * Rotated frame (a.base at origin):
+			 *
+			 * 0                                       UT_MAX
+			 * |--------------------------------------------|
+			 * [=== a ==========================]           |
+			 * [= b1 tail =]  [========= b1 main ==========>]
+			 *                 ^-- b1.base <= a.size
+			 *
+			 * 'a' and 'b' intersect in two disjoint arcs,
+			 * can't represent as single cnum, over-approximate
+			 * the result.
+			 */
+			return a.size <= b.size ? a : b;
+		} else {
+			/*
+			 * Rotated frame (a.base at origin):
+			 *
+			 * 0                                       UT_MAX
+			 * |--------------------------------------------|
+			 * [=== a =============]  |                     |
+			 * [= b1 tail =]          [======= b1 main ====>]
+			 *                         ^-- b1.base > a.size
+			 *
+			 * Only 'b' tail intersects 'a'.
+			 */
+			return (struct cnum_t) {
+				.base = a.base,
+				.size = min(a.size, (ut)(b1.base + b1.size)),
+			};
+		}
+	} else if (a.size >= b1.base) {
+		/*
+		 * Rotated frame (a.base at origin):
+		 *
+		 * 0                                             UT_MAX
+		 * |--------------------------------------------------|
+		 * [=== a ==================================]         |
+		 *                   [== b1 =====================]
+		 *
+		 * 0                                             UT_MAX
+		 * |--------------------------------------------------|
+		 * [=== a ==================================]         |
+		 *                   [== b1 ====]
+		 *                   ^-- b1.base <= a.size
+		 *                   |<-- a.size - dbase -->|
+		 *
+		 * 'a' and 'b' intersect as one cnum.
+		 */
+		return (struct cnum_t) {
+			.base = b.base,
+			.size = min((ut)(a.size - dbase), b.size),
+		};
+	} else {
+		return EMPTY;
+	}
+}
+
+void FN(intersect_with)(struct cnum_t *dst, struct cnum_t src)
+{
+	*dst = FN(intersect)(*dst, src);
+}
+
+void FN(intersect_with_urange)(struct cnum_t *dst, ut min, ut max)
+{
+	FN(intersect_with)(dst, FN(from_urange)(min, max));
+}
+
+void FN(intersect_with_srange)(struct cnum_t *dst, st min, st max)
+{
+	FN(intersect_with)(dst, FN(from_srange)(min, max));
+}
+
+static inline struct cnum_t FN(normalize)(struct cnum_t cnum)
+{
+	if (cnum.size == UT_MAX && cnum.base != 0 && cnum.base != (ut)ST_MAX)
+		cnum.base = 0;
+	return cnum;
+}
+
+struct cnum_t FN(add)(struct cnum_t a, struct cnum_t b)
+{
+	if (FN(is_empty)(a) || FN(is_empty)(b))
+		return EMPTY;
+	if (a.size > UT_MAX - b.size)
+		return (struct cnum_t){ 0, (ut)UT_MAX };
+	else
+		return FN(normalize)((struct cnum_t){ a.base + b.base, a.size + b.size });
+}
+
+struct cnum_t FN(negate)(struct cnum_t a)
+{
+	if (FN(is_empty)(a))
+		return EMPTY;
+	return FN(normalize)((struct cnum_t){ -((ut)a.base + a.size), a.size });
+}
+
+bool FN(is_empty)(struct cnum_t cnum)
+{
+	return cnum.base == EMPTY.base && cnum.size == EMPTY.size;
+}
+
+bool FN(contains)(struct cnum_t cnum, ut v)
+{
+	if (FN(is_empty)(cnum))
+		return false;
+	if (FN(urange_overflow)(cnum))
+		return v >= cnum.base || v <= (ut)cnum.base + cnum.size;
+	else
+		return v >= cnum.base && v <= (ut)cnum.base + cnum.size;
+}
+
+bool FN(is_const)(struct cnum_t cnum)
+{
+	return cnum.size == 0;
+}
+
+bool FN(is_subset)(struct cnum_t bigger, struct cnum_t smaller)
+{
+	if (FN(is_empty(smaller)))
+		return true;
+	if (FN(is_empty(bigger)))
+		return false;
+	/* rotate both arcs such that 'bigger' starts at origin, hence does not overflow */
+	smaller.base -= bigger.base;
+	bigger.base = 0;
+	if (FN(urange_overflow)(smaller) && bigger.size < UT_MAX)
+		return false;
+	return smaller.base + smaller.size <= bigger.size;
+}
+
+#undef EMPTY
+#undef cnum_t
+#undef ut
+#undef st
+#undef UT_MAX
+#undef ST_MAX
+#undef ST_MIN
+#undef FN
diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c
index db73c4740b1e1..b2a19acadb91e 100644
--- a/kernel/bpf/const_fold.c
+++ b/kernel/bpf/const_fold.c
@@ -58,6 +58,14 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info *
 	u8 opcode = BPF_OP(insn->code) | BPF_SRC(insn->code);
 	int r;
 
+	/* Stack arg stores (r11-based) are outside the tracked register set. */
+	if (is_stack_arg_st(insn) || is_stack_arg_stx(insn))
+		return;
+	if (is_stack_arg_ldx(insn)) {
+		ci_out[insn->dst_reg] = unknown;
+		return;
+	}
+
 	switch (class) {
 	case BPF_ALU:
 	case BPF_ALU64:
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 8b018ff488750..39a660afe93a4 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -1299,8 +1299,8 @@ static int bpf_jit_blind_insn(const struct bpf_insn *from,
 	u32 imm_rnd = get_random_u32();
 	s16 off;
 
-	BUILD_BUG_ON(BPF_REG_AX  + 1 != MAX_BPF_JIT_REG);
-	BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG);
+	BUILD_BUG_ON(BPF_REG_PARAMS + 2 != MAX_BPF_JIT_REG);
+	BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG);
 
 	/* Constraints on AX register:
 	 *
@@ -1582,6 +1582,16 @@ bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struc
 	insn_idx += prog->aux->subprog_start;
 	return env->insn_aux_data[insn_idx].indirect_target;
 }
+
+u16 bpf_out_stack_arg_cnt(const struct bpf_verifier_env *env, const struct bpf_prog *prog)
+{
+	const struct bpf_subprog_info *sub;
+
+	if (!env)
+		return 0;
+	sub = &env->subprog_info[prog->aux->func_idx];
+	return sub->stack_arg_cnt - bpf_in_stack_arg_cnt(sub);
+}
 #endif /* CONFIG_BPF_JIT */
 
 /* Base function for offset calculation. Needs to go into .text section,
@@ -1771,6 +1781,9 @@ static u32 abs_s32(s32 x)
 	return x >= 0 ? (u32)x : -(u32)x;
 }
 
+static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5,
+				  const struct bpf_insn *insn);
+
 /**
  *	___bpf_prog_run - run eBPF program on a given context
  *	@regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers
@@ -2077,10 +2090,9 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)
 		CONT;
 
 	JMP_CALL_ARGS:
-		BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2,
-							    BPF_R3, BPF_R4,
-							    BPF_R5,
-							    insn + insn->off + 1);
+		BPF_R0 = interpreters_args[insn->off](BPF_R1, BPF_R2, BPF_R3,
+						      BPF_R4, BPF_R5,
+						      insn + insn->imm + 1);
 		CONT;
 
 	JMP_TAIL_CALL: {
@@ -2394,13 +2406,22 @@ EVAL4(PROG_NAME_LIST, 416, 448, 480, 512)
 #undef PROG_NAME_LIST
 
 #ifdef CONFIG_BPF_SYSCALL
-void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
+int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth)
 {
 	stack_depth = max_t(u32, stack_depth, 1);
-	insn->off = (s16) insn->imm;
-	insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] -
-		__bpf_call_base_args;
+	/* Prevent out-of-bounds read to interpreters_args */
+	if (stack_depth > MAX_BPF_STACK)
+		return -EINVAL;
+	insn->off = (round_up(stack_depth, 32) / 32) - 1;
 	insn->code = BPF_JMP | BPF_CALL_ARGS;
+	return 0;
+}
+
+s32 bpf_call_args_imm(s16 idx)
+{
+	if (WARN_ON_ONCE(idx < 0 || idx >= ARRAY_SIZE(interpreters_args)))
+		return 0;
+	return BPF_CALL_IMM(interpreters_args[idx]);
 }
 #endif
 #endif
@@ -3217,6 +3238,11 @@ bool __weak bpf_jit_supports_kfunc_call(void)
 	return false;
 }
 
+bool __weak bpf_jit_supports_stack_args(void)
+{
+	return false;
+}
+
 bool __weak bpf_jit_supports_far_kfunc_call(void)
 {
 	return false;
diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c
index fba9e8c008787..12739add2ddae 100644
--- a/kernel/bpf/fixups.c
+++ b/kernel/bpf/fixups.c
@@ -1250,9 +1250,9 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 		}
 		if (!bpf_pseudo_call(insn))
 			continue;
-		insn->off = env->insn_aux_data[i].call_imm;
-		subprog = bpf_find_subprog(env, i + insn->off + 1);
-		insn->imm = subprog;
+		insn->imm = env->insn_aux_data[i].call_imm;
+		subprog = bpf_find_subprog(env, i + insn->imm + 1);
+		insn->off = subprog;
 	}
 
 	prog->jited = 1;
@@ -1265,6 +1265,7 @@ static int jit_subprogs(struct bpf_verifier_env *env)
 	prog->aux->real_func_cnt = env->subprog_cnt;
 	prog->aux->bpf_exception_cb = (void *)func[env->exception_callback_subprog]->bpf_func;
 	prog->aux->exception_boundary = func[0]->aux->exception_boundary;
+	prog->aux->stack_arg_sp_adjust = func[0]->aux->stack_arg_sp_adjust;
 	bpf_prog_jit_attempt_done(prog);
 	return 0;
 out_free:
@@ -1378,9 +1379,21 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
 	struct bpf_prog *prog = env->prog;
 	struct bpf_insn *insn = prog->insnsi;
 	bool has_kfunc_call = bpf_prog_has_kfunc_call(prog);
-	int i, depth;
+	int depth;
 #endif
-	int err = 0;
+	int i, err = 0;
+
+	for (i = 0; i < env->subprog_cnt; i++) {
+		struct bpf_subprog_info *subprog = &env->subprog_info[i];
+		u16 outgoing = subprog->stack_arg_cnt - bpf_in_stack_arg_cnt(subprog);
+
+		if (subprog->max_out_stack_arg_cnt > outgoing) {
+			verbose(env,
+				"func#%d writes %u stack arg slots, but calls only require %u\n",
+				i, subprog->max_out_stack_arg_cnt, outgoing);
+			return -EINVAL;
+		}
+	}
 
 	if (env->prog->jit_requested &&
 	    !bpf_prog_is_offloaded(env->prog->aux)) {
@@ -1395,6 +1408,12 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
 		verbose(env, "calling kernel functions are not allowed in non-JITed programs\n");
 		return -EINVAL;
 	}
+	for (i = 0; i < env->subprog_cnt; i++) {
+		if (bpf_in_stack_arg_cnt(&env->subprog_info[i])) {
+			verbose(env, "stack args are not supported in non-JITed programs\n");
+			return -EINVAL;
+		}
+	}
 	if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
 		/* When JIT fails the progs with bpf2bpf calls and tail_calls
 		 * have to be rejected, since interpreter doesn't support them yet.
@@ -1416,7 +1435,12 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env)
 		depth = get_callee_stack_depth(env, insn, i);
 		if (depth < 0)
 			return depth;
-		bpf_patch_call_args(insn, depth);
+		err = bpf_patch_call_args(insn, depth);
+		if (err) {
+			verbose(env, "stack depth %d exceeds interpreter stack depth limit\n",
+				depth);
+			return err;
+		}
 	}
 	err = 0;
 #endif
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 2bb60200c2665..e2d6b5931932b 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -1944,7 +1944,7 @@ static const struct bpf_func_proto bpf_dynptr_read_proto = {
 	.ret_type	= RET_INTEGER,
 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
-	.arg3_type	= ARG_PTR_TO_DYNPTR | MEM_RDONLY,
+	.arg3_type	= ARG_PTR_TO_DYNPTR,
 	.arg4_type	= ARG_ANYTHING,
 	.arg5_type	= ARG_ANYTHING,
 };
@@ -2001,7 +2001,7 @@ static const struct bpf_func_proto bpf_dynptr_write_proto = {
 	.func		= bpf_dynptr_write,
 	.gpl_only	= false,
 	.ret_type	= RET_INTEGER,
-	.arg1_type	= ARG_PTR_TO_DYNPTR | MEM_RDONLY,
+	.arg1_type	= ARG_PTR_TO_DYNPTR,
 	.arg2_type	= ARG_ANYTHING,
 	.arg3_type	= ARG_PTR_TO_MEM | MEM_RDONLY,
 	.arg4_type	= ARG_CONST_SIZE_OR_ZERO,
@@ -2044,7 +2044,7 @@ static const struct bpf_func_proto bpf_dynptr_data_proto = {
 	.func		= bpf_dynptr_data,
 	.gpl_only	= false,
 	.ret_type	= RET_PTR_TO_DYNPTR_MEM_OR_NULL,
-	.arg1_type	= ARG_PTR_TO_DYNPTR | MEM_RDONLY,
+	.arg1_type	= ARG_PTR_TO_DYNPTR,
 	.arg2_type	= ARG_ANYTHING,
 	.arg3_type	= ARG_CONST_ALLOC_SIZE_OR_ZERO,
 };
@@ -3072,7 +3072,7 @@ __bpf_kfunc void *bpf_dynptr_slice_rdwr(const struct bpf_dynptr *p, u64 offset,
 	return bpf_dynptr_slice(p, offset, buffer__nullable, buffer__szk);
 }
 
-__bpf_kfunc int bpf_dynptr_adjust(const struct bpf_dynptr *p, u64 start, u64 end)
+__bpf_kfunc int bpf_dynptr_adjust(struct bpf_dynptr *p, u64 start, u64 end)
 {
 	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
 	u64 size;
@@ -3093,14 +3093,14 @@ __bpf_kfunc int bpf_dynptr_adjust(const struct bpf_dynptr *p, u64 start, u64 end
 
 __bpf_kfunc bool bpf_dynptr_is_null(const struct bpf_dynptr *p)
 {
-	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
+	const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
 
 	return !ptr->data;
 }
 
 __bpf_kfunc bool bpf_dynptr_is_rdonly(const struct bpf_dynptr *p)
 {
-	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
+	const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
 
 	if (!ptr->data)
 		return false;
@@ -3110,7 +3110,7 @@ __bpf_kfunc bool bpf_dynptr_is_rdonly(const struct bpf_dynptr *p)
 
 __bpf_kfunc u64 bpf_dynptr_size(const struct bpf_dynptr *p)
 {
-	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
+	const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
 
 	if (!ptr->data)
 		return -EINVAL;
@@ -3122,7 +3122,7 @@ __bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p,
 				 struct bpf_dynptr *clone__uninit)
 {
 	struct bpf_dynptr_kern *clone = (struct bpf_dynptr_kern *)clone__uninit;
-	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
+	const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
 
 	if (!ptr->data) {
 		bpf_dynptr_set_null(clone);
@@ -3145,11 +3145,11 @@ __bpf_kfunc int bpf_dynptr_clone(const struct bpf_dynptr *p,
  * Copies data from source dynptr to destination dynptr.
  * Returns 0 on success; negative error, otherwise.
  */
-__bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u64 dst_off,
-				struct bpf_dynptr *src_ptr, u64 src_off, u64 size)
+__bpf_kfunc int bpf_dynptr_copy(const struct bpf_dynptr *dst_ptr, u64 dst_off,
+				const struct bpf_dynptr *src_ptr, u64 src_off, u64 size)
 {
-	struct bpf_dynptr_kern *dst = (struct bpf_dynptr_kern *)dst_ptr;
-	struct bpf_dynptr_kern *src = (struct bpf_dynptr_kern *)src_ptr;
+	const struct bpf_dynptr_kern *dst = (struct bpf_dynptr_kern *)dst_ptr;
+	const struct bpf_dynptr_kern *src = (struct bpf_dynptr_kern *)src_ptr;
 	void *src_slice, *dst_slice;
 	char buf[256];
 	u64 off;
@@ -3200,9 +3200,9 @@ __bpf_kfunc int bpf_dynptr_copy(struct bpf_dynptr *dst_ptr, u64 dst_off,
  * at @offset with the constant byte @val.
  * Returns 0 on success; negative error, otherwise.
  */
-__bpf_kfunc int bpf_dynptr_memset(struct bpf_dynptr *p, u64 offset, u64 size, u8 val)
+__bpf_kfunc int bpf_dynptr_memset(const struct bpf_dynptr *p, u64 offset, u64 size, u8 val)
 {
-	struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
+	const struct bpf_dynptr_kern *ptr = (struct bpf_dynptr_kern *)p;
 	u64 chunk_sz, write_off;
 	char buf[256];
 	void* slice;
@@ -3301,7 +3301,7 @@ __bpf_kfunc void bpf_throw(u64 cookie)
 	 * which skips compiler generated instrumentation to do the same.
 	 */
 	kasan_unpoison_task_stack_below((void *)(long)ctx.sp);
-	ctx.aux->bpf_exception_cb(cookie, ctx.sp, ctx.bp, 0, 0);
+	ctx.aux->bpf_exception_cb(cookie, ctx.sp + ctx.aux->stack_arg_sp_adjust, ctx.bp, 0, 0);
 	WARN(1, "A call to BPF exception callback should never return\n");
 }
 
@@ -4214,13 +4214,13 @@ __bpf_kfunc void bpf_key_put(struct bpf_key *bkey)
  *
  * Return: 0 on success, a negative value on error.
  */
-__bpf_kfunc int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
-			       struct bpf_dynptr *sig_p,
+__bpf_kfunc int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p,
+			       const struct bpf_dynptr *sig_p,
 			       struct bpf_key *trusted_keyring)
 {
 #ifdef CONFIG_SYSTEM_DATA_VERIFICATION
-	struct bpf_dynptr_kern *data_ptr = (struct bpf_dynptr_kern *)data_p;
-	struct bpf_dynptr_kern *sig_ptr = (struct bpf_dynptr_kern *)sig_p;
+	const struct bpf_dynptr_kern *data_ptr = (struct bpf_dynptr_kern *)data_p;
+	const struct bpf_dynptr_kern *sig_ptr = (struct bpf_dynptr_kern *)sig_p;
 	const void *data, *sig;
 	u32 data_len, sig_len;
 	int ret;
@@ -4241,8 +4241,13 @@ __bpf_kfunc int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p,
 
 	data_len = __bpf_dynptr_size(data_ptr);
 	data = __bpf_dynptr_data(data_ptr, data_len);
+	if (!data)
+		return -EINVAL;
+
 	sig_len = __bpf_dynptr_size(sig_ptr);
 	sig = __bpf_dynptr_data(sig_ptr, sig_len);
+	if (!sig)
+		return -EINVAL;
 
 	return verify_pkcs7_signature(data, data_len, sig, sig_len,
 				      trusted_keyring->key,
diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c
index 58197d73b1201..0aadfbae0acc5 100644
--- a/kernel/bpf/liveness.c
+++ b/kernel/bpf/liveness.c
@@ -610,6 +610,21 @@ enum arg_track_state {
 /* Track callee stack slots fp-8 through fp-512 (64 slots of 8 bytes each) */
 #define MAX_ARG_SPILL_SLOTS 64
 
+/*
+ * Combined register + stack arg tracking: R0-R10 at indices 0-10,
+ * outgoing stack arg slots at indices MAX_BPF_REG..MAX_BPF_REG+6.
+ */
+#define MAX_AT_TRACK_REGS (MAX_BPF_REG + MAX_STACK_ARG_SLOTS)
+
+static int stack_arg_off_to_slot(s16 off)
+{
+	int aoff = off < 0 ? -off : off;
+
+	if (aoff / 8 > MAX_STACK_ARG_SLOTS)
+		return -1;
+	return aoff / 8 - 1;
+}
+
 static bool arg_is_visited(const struct arg_track *at)
 {
 	return at->frame != ARG_UNVISITED;
@@ -791,7 +806,9 @@ static bool arg_track_join(struct bpf_verifier_env *env, int idx, int target, in
 		return true;
 
 	verbose(env, "arg JOIN insn %d -> %d ", idx, target);
-	if (r >= 0)
+	if (r >= MAX_BPF_REG)
+		verbose(env, "sa%d: ", r - MAX_BPF_REG);
+	else if (r >= 0)
 		verbose(env, "r%d: ", r);
 	else
 		verbose(env, "fp%+d: ", r * 8);
@@ -1032,6 +1049,21 @@ static void arg_track_log(struct bpf_verifier_env *env, struct bpf_insn *insn, i
 		verbose(env, "\tr%d: ", i); verbose_arg_track(env, &at_in[i]);
 		verbose(env, " -> "); verbose_arg_track(env, &at_out[i]);
 	}
+	/* Log outgoing stack arg slot transitions at indices MAX_BPF_REG..MAX_AT_TRACK_REGS-1 */
+	for (i = 0; i < MAX_STACK_ARG_SLOTS; i++) {
+		int ai = MAX_BPF_REG + i;
+
+		if (arg_track_eq(&at_out[ai], &at_in[ai]))
+			continue;
+		if (!printed) {
+			verbose(env, "%3d: ", idx);
+			bpf_verbose_insn(env, insn);
+			bpf_vlog_reset(&env->log, env->log.end_pos - 1);
+			printed = true;
+		}
+		verbose(env, "\tsa%d: ", i); verbose_arg_track(env, &at_in[ai]);
+		verbose(env, " -> "); verbose_arg_track(env, &at_out[ai]);
+	}
 	for (i = 0; i < MAX_ARG_SPILL_SLOTS; i++) {
 		if (arg_track_eq(&at_stack_out[i], &at_stack_in[i]))
 			continue;
@@ -1062,6 +1094,7 @@ static bool can_be_local_fp(int depth, int regno, struct arg_track *at)
 static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			   int insn_idx,
 			   struct arg_track *at_out, struct arg_track *at_stack_out,
+			   const struct arg_track *at_stack_arg_entry,
 			   struct func_instance *instance,
 			   u32 *callsites)
 {
@@ -1071,9 +1104,21 @@ static void arg_track_xfer(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	struct arg_track *dst = &at_out[insn->dst_reg];
 	struct arg_track *src = &at_out[insn->src_reg];
 	struct arg_track none = { .frame = ARG_NONE };
-	int r;
-
-	if (class == BPF_ALU64 && BPF_SRC(insn->code) == BPF_K) {
+	int r, slot;
+
+	/* Handle stack arg stores and loads. */
+	if (is_stack_arg_st(insn) || is_stack_arg_stx(insn)) {
+		slot = stack_arg_off_to_slot(insn->off);
+		if (slot >= 0) {
+			if (is_stack_arg_stx(insn))
+				at_out[MAX_BPF_REG + slot] = at_out[insn->src_reg];
+			else
+				at_out[MAX_BPF_REG + slot] = none;
+		}
+	} else if (is_stack_arg_ldx(insn)) {
+		slot = stack_arg_off_to_slot(insn->off);
+		at_out[insn->dst_reg] = (slot >= 0) ? at_stack_arg_entry[slot] : none;
+	} else if (class == BPF_ALU64 && BPF_SRC(insn->code) == BPF_K) {
 		if (code == BPF_MOV) {
 			*dst = none;
 		} else if (dst->frame >= 0) {
@@ -1297,6 +1342,16 @@ static int record_load_store_access(struct bpf_verifier_env *env,
 	struct arg_track resolved, *ptr;
 	int oi;
 
+	/*
+	 * Stack arg insns use dst_reg/src_reg=BPF_REG_PARAMS(11). Since at[]
+	 * is extended to MAX_AT_TRACK_REGS, at[11] holds the arg_track for
+	 * outgoing stack arg slot 0 — not the pointer used for the memory
+	 * access. Skip so the slot's tracked value isn't confused with the
+	 * base register that record_stack_access() expects.
+	 */
+	if (is_stack_arg_stx(insn) || is_stack_arg_st(insn) || is_stack_arg_ldx(insn))
+		return 0;
+
 	switch (class) {
 	case BPF_LDX:
 		ptr = &at[insn->src_reg];
@@ -1343,6 +1398,42 @@ static int record_load_store_access(struct bpf_verifier_env *env,
 	return 0;
 }
 
+static int record_arg_access(struct bpf_verifier_env *env,
+			     struct func_instance *instance,
+			     struct bpf_insn *insn,
+			     struct arg_track *at, int arg_idx,
+			     int insn_idx)
+{
+	int depth = instance->depth;
+	int frame = at->frame;
+	int err = 0;
+	s64 bytes;
+
+	if (!arg_is_fp(at))
+		return 0;
+
+	if (bpf_helper_call(insn)) {
+		bytes = bpf_helper_stack_access_bytes(env, insn, arg_idx, insn_idx);
+	} else if (bpf_pseudo_kfunc_call(insn)) {
+		bytes = bpf_kfunc_stack_access_bytes(env, insn, arg_idx, insn_idx);
+	} else {
+		for (int f = 0; f <= depth; f++) {
+			err = mark_stack_read(instance, f, insn_idx, SPIS_ALL);
+			if (err)
+				return err;
+		}
+		return 0;
+	}
+	if (bytes == 0)
+		return 0;
+
+	if (frame >= 0 && frame <= depth)
+		err = record_stack_access(instance, at, bytes, frame, insn_idx);
+	else if (frame == ARG_IMPRECISE)
+		err = record_imprecise(instance, at->mask, insn_idx);
+	return err;
+}
+
 /* Record stack access for a given 'at' state of helper/kfunc 'insn' */
 static int record_call_access(struct bpf_verifier_env *env,
 			      struct func_instance *instance,
@@ -1350,9 +1441,8 @@ static int record_call_access(struct bpf_verifier_env *env,
 			      int insn_idx)
 {
 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
-	int depth = instance->depth;
 	struct bpf_call_summary cs;
-	int r, err = 0, num_params = 5;
+	int r, err, num_params = 5;
 
 	if (bpf_pseudo_call(insn))
 		return 0;
@@ -1360,32 +1450,15 @@ static int record_call_access(struct bpf_verifier_env *env,
 	if (bpf_get_call_summary(env, insn, &cs))
 		num_params = cs.num_params;
 
-	for (r = BPF_REG_1; r < BPF_REG_1 + num_params; r++) {
-		int frame = at[r].frame;
-		s64 bytes;
-
-		if (!arg_is_fp(&at[r]))
-			continue;
-
-		if (bpf_helper_call(insn)) {
-			bytes = bpf_helper_stack_access_bytes(env, insn, r - 1, insn_idx);
-		} else if (bpf_pseudo_kfunc_call(insn)) {
-			bytes = bpf_kfunc_stack_access_bytes(env, insn, r - 1, insn_idx);
-		} else {
-			for (int f = 0; f <= depth; f++) {
-				err = mark_stack_read(instance, f, insn_idx, SPIS_ALL);
-				if (err)
-					return err;
-			}
-			return 0;
-		}
-		if (bytes == 0)
-			continue;
+	for (r = BPF_REG_1; r < BPF_REG_1 + min(num_params, MAX_BPF_FUNC_REG_ARGS); r++) {
+		err = record_arg_access(env, instance, insn, &at[r], r - 1, insn_idx);
+		if (err)
+			return err;
+	}
 
-		if (frame >= 0 && frame <= depth)
-			err = record_stack_access(instance, &at[r], bytes, frame, insn_idx);
-		else if (frame == ARG_IMPRECISE)
-			err = record_imprecise(instance, at[r].mask, insn_idx);
+	for (r = 0; r < MAX_STACK_ARG_SLOTS && r < num_params - MAX_BPF_FUNC_REG_ARGS; r++) {
+		err = record_arg_access(env, instance, insn, &at[MAX_BPF_REG + r],
+					r + MAX_BPF_FUNC_REG_ARGS, insn_idx);
 		if (err)
 			return err;
 	}
@@ -1445,7 +1518,7 @@ static int find_callback_subprog(struct bpf_verifier_env *env,
 
 /* Per-subprog intermediate state kept alive across analysis phases */
 struct subprog_at_info {
-	struct arg_track (*at_in)[MAX_BPF_REG];
+	struct arg_track (*at_in)[MAX_AT_TRACK_REGS];
 	int len;
 };
 
@@ -1479,6 +1552,9 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env,
 			for (r = 0; r < MAX_BPF_REG - 1; r++)
 				if (arg_is_fp(&info->at_in[i][r]))
 					has_extra = true;
+			for (r = 0; r < MAX_STACK_ARG_SLOTS; r++)
+				if (arg_is_fp(&info->at_in[i][MAX_BPF_REG + r]))
+					has_extra = true;
 		}
 		if (is_ldx_stx_call) {
 			for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++)
@@ -1503,6 +1579,12 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env,
 				verbose(env, " r%d=", r);
 				verbose_arg_track(env, &info->at_in[i][r]);
 			}
+			for (r = 0; r < MAX_STACK_ARG_SLOTS; r++) {
+				if (!arg_is_fp(&info->at_in[i][MAX_BPF_REG + r]))
+					continue;
+				verbose(env, " sa%d=", r);
+				verbose_arg_track(env, &info->at_in[i][MAX_BPF_REG + r]);
+			}
 		}
 
 		if (is_ldx_stx_call) {
@@ -1525,7 +1607,7 @@ static void print_subprog_arg_access(struct bpf_verifier_env *env,
  * Runs forward fixed-point with arg_track_xfer(), then records
  * memory accesses in a single linear pass over converged state.
  *
- * @callee_entry: pre-populated entry state for R1-R5
+ * @callee_entry: pre-populated entry state for R1-R5 and stack args
  *                NULL for main (subprog 0).
  * @info:         stores at_in, len for debug printing.
  */
@@ -1543,10 +1625,11 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 	int end = env->subprog_info[subprog + 1].start;
 	int po_end = env->subprog_info[subprog + 1].postorder_start;
 	int len = end - start;
-	struct arg_track (*at_in)[MAX_BPF_REG] = NULL;
-	struct arg_track at_out[MAX_BPF_REG];
+	struct arg_track (*at_in)[MAX_AT_TRACK_REGS] = NULL;
+	struct arg_track at_out[MAX_AT_TRACK_REGS];
 	struct arg_track (*at_stack_in)[MAX_ARG_SPILL_SLOTS] = NULL;
 	struct arg_track *at_stack_out = NULL;
+	struct arg_track at_stack_arg_entry[MAX_STACK_ARG_SLOTS];
 	struct arg_track unvisited = { .frame = ARG_UNVISITED };
 	struct arg_track none = { .frame = ARG_NONE };
 	bool changed;
@@ -1565,13 +1648,13 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 		goto err_free;
 
 	for (i = 0; i < len; i++) {
-		for (r = 0; r < MAX_BPF_REG; r++)
+		for (r = 0; r < MAX_AT_TRACK_REGS; r++)
 			at_in[i][r] = unvisited;
 		for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++)
 			at_stack_in[i][r] = unvisited;
 	}
 
-	for (r = 0; r < MAX_BPF_REG; r++)
+	for (r = 0; r < MAX_AT_TRACK_REGS; r++)
 		at_in[0][r] = none;
 
 	/* Entry: R10 is always precisely the current frame's FP */
@@ -1587,6 +1670,10 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 	for (r = 0; r < MAX_ARG_SPILL_SLOTS; r++)
 		at_stack_in[0][r] = none;
 
+	/* Entry: incoming stack args from caller, or ARG_NONE for main */
+	for (r = 0; r < MAX_STACK_ARG_SLOTS; r++)
+		at_stack_arg_entry[r] = callee_entry ? callee_entry[MAX_BPF_REG + r] : none;
+
 	if (env->log.level & BPF_LOG_LEVEL2)
 		verbose(env, "subprog#%d: analyzing (depth %d)...\n", subprog, depth);
 
@@ -1605,7 +1692,8 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 		memcpy(at_out, at_in[i], sizeof(at_out));
 		memcpy(at_stack_out, at_stack_in[i], MAX_ARG_SPILL_SLOTS * sizeof(*at_stack_out));
 
-		arg_track_xfer(env, insn, idx, at_out, at_stack_out, instance, callsites);
+		arg_track_xfer(env, insn, idx, at_out, at_stack_out,
+			       at_stack_arg_entry, instance, callsites);
 		arg_track_log(env, insn, idx, at_in[i], at_stack_in[i], at_out, at_stack_out);
 
 		/* Propagate to successors within this subprogram */
@@ -1619,7 +1707,7 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 				continue;
 			ti = target - start;
 
-			for (r = 0; r < MAX_BPF_REG; r++)
+			for (r = 0; r < MAX_AT_TRACK_REGS; r++)
 				changed |= arg_track_join(env, idx, target, r,
 							  &at_in[ti][r], at_out[r]);
 
@@ -1674,11 +1762,14 @@ static int compute_subprog_args(struct bpf_verifier_env *env,
 	return err;
 }
 
-/* Return true if any of R1-R5 is derived from a frame pointer. */
+/* Return true if any of R1-R5 or stack args is derived from a frame pointer. */
 static bool has_fp_args(struct arg_track *args)
 {
 	for (int r = BPF_REG_1; r <= BPF_REG_5; r++)
-		if (args[r].frame != ARG_NONE)
+		if (arg_is_fp(&args[r]))
+			return true;
+	for (int r = 0; r < MAX_STACK_ARG_SLOTS; r++)
+		if (arg_is_fp(&args[MAX_BPF_REG + r]))
 			return true;
 	return false;
 }
@@ -1803,7 +1894,7 @@ static int analyze_subprog(struct bpf_verifier_env *env,
 	/* For each reachable call site in the subprog, recurse into callees */
 	for (int p = po_start; p < po_end; p++) {
 		int idx = env->cfg.insn_postorder[p];
-		struct arg_track callee_args[BPF_REG_5 + 1];
+		struct arg_track callee_args[MAX_AT_TRACK_REGS] = {};
 		struct arg_track none = { .frame = ARG_NONE };
 		struct bpf_insn *insn = &insns[idx];
 		struct func_instance *callee_instance;
@@ -1818,9 +1909,11 @@ static int analyze_subprog(struct bpf_verifier_env *env,
 			if (callee < 0)
 				continue;
 
-			/* Build entry args: R1-R5 from at_in at call site */
+			/* Build entry args: R1-R5 and stack args from at_in at call site */
 			for (int r = BPF_REG_1; r <= BPF_REG_5; r++)
 				callee_args[r] = info[subprog].at_in[j][r];
+			for (int r = 0; r < MAX_STACK_ARG_SLOTS; r++)
+				callee_args[MAX_BPF_REG + r] = info[subprog].at_in[j][MAX_BPF_REG + r];
 		} else if (bpf_calls_callback(env, idx)) {
 			callee = find_callback_subprog(env, insn, idx, &caller_reg, &cb_callee_reg);
 			if (callee == -2) {
@@ -1842,6 +1935,8 @@ static int analyze_subprog(struct bpf_verifier_env *env,
 
 			for (int r = BPF_REG_1; r <= BPF_REG_5; r++)
 				callee_args[r] = none;
+			for (int r = 0; r < MAX_STACK_ARG_SLOTS; r++)
+				callee_args[MAX_BPF_REG + r] = none;
 			callee_args[cb_callee_reg] = info[subprog].at_in[j][caller_reg];
 		} else {
 			continue;
@@ -2085,7 +2180,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env,
 			def = ALL_CALLER_SAVED_REGS;
 			use = def & ~BIT(BPF_REG_0);
 			if (bpf_get_call_summary(env, insn, &cs))
-				use = GENMASK(cs.num_params, 1);
+				use = GENMASK(min_t(u8, cs.num_params, MAX_BPF_FUNC_REG_ARGS), 1);
 			break;
 		default:
 			def = 0;
diff --git a/kernel/bpf/log.c b/kernel/bpf/log.c
index 011e4ec25acd2..62fe6ed18374e 100644
--- a/kernel/bpf/log.c
+++ b/kernel/bpf/log.c
@@ -13,17 +13,17 @@
 
 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
 
-static bool bpf_verifier_log_attr_valid(const struct bpf_verifier_log *log)
+static bool bpf_verifier_log_attr_valid(u32 log_level, char __user *log_buf, u32 log_size)
 {
 	/* ubuf and len_total should both be specified (or not) together */
-	if (!!log->ubuf != !!log->len_total)
+	if (!!log_buf != !!log_size)
 		return false;
 	/* log buf without log_level is meaningless */
-	if (log->ubuf && log->level == 0)
+	if (log_buf && log_level == 0)
 		return false;
-	if (log->level & ~BPF_LOG_MASK)
+	if (log_level & ~BPF_LOG_MASK)
 		return false;
-	if (log->len_total > UINT_MAX >> 2)
+	if (log_size > UINT_MAX >> 2)
 		return false;
 	return true;
 }
@@ -36,7 +36,7 @@ int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
 	log->len_total = log_size;
 
 	/* log attributes have to be sane */
-	if (!bpf_verifier_log_attr_valid(log))
+	if (!bpf_verifier_log_attr_valid(log_level, log_buf, log_size))
 		return -EINVAL;
 
 	return 0;
@@ -571,20 +571,20 @@ static void print_scalar_ranges(struct bpf_verifier_env *env,
 		u64 val;
 		bool omit;
 	} minmaxs[] = {
-		{"smin",   reg->smin_value,         reg->smin_value == S64_MIN},
-		{"smax",   reg->smax_value,         reg->smax_value == S64_MAX},
-		{"umin",   reg->umin_value,         reg->umin_value == 0},
-		{"umax",   reg->umax_value,         reg->umax_value == U64_MAX},
+		{"smin",   reg_smin(reg),         reg_smin(reg) == S64_MIN},
+		{"smax",   reg_smax(reg),         reg_smax(reg) == S64_MAX},
+		{"umin",   reg_umin(reg),         reg_umin(reg) == 0},
+		{"umax",   reg_umax(reg),         reg_umax(reg) == U64_MAX},
 		{"smin32",
-		 is_snum_decimal((s64)reg->s32_min_value)
-			 ? (s64)reg->s32_min_value
-			 : (u32)reg->s32_min_value, reg->s32_min_value == S32_MIN},
+		 is_snum_decimal((s64)reg_s32_min(reg))
+			 ? (s64)reg_s32_min(reg)
+			 : (u32)reg_s32_min(reg), reg_s32_min(reg) == S32_MIN},
 		{"smax32",
-		 is_snum_decimal((s64)reg->s32_max_value)
-			 ? (s64)reg->s32_max_value
-			 : (u32)reg->s32_max_value, reg->s32_max_value == S32_MAX},
-		{"umin32", reg->u32_min_value,      reg->u32_min_value == 0},
-		{"umax32", reg->u32_max_value,      reg->u32_max_value == U32_MAX},
+		 is_snum_decimal((s64)reg_s32_max(reg))
+			 ? (s64)reg_s32_max(reg)
+			 : (u32)reg_s32_max(reg), reg_s32_max(reg) == S32_MAX},
+		{"umin32", reg_u32_min(reg),      reg_u32_min(reg) == 0},
+		{"umax32", reg_u32_max(reg),      reg_u32_max(reg) == U32_MAX},
 	}, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)];
 	bool neg1, neg2;
 
@@ -825,3 +825,81 @@ void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_st
 	}
 	print_verifier_state(env, vstate, frameno, false);
 }
+
+int bpf_log_attr_init(struct bpf_log_attr *log, u64 log_buf, u32 log_size, u32 log_level,
+		      u32 offsetof_log_true_size, bpfptr_t uattr, struct bpf_common_attr *common,
+		      bpfptr_t uattr_common, u32 size_common)
+{
+	char __user *ubuf_common = u64_to_user_ptr(common->log_buf);
+	char __user *ubuf = u64_to_user_ptr(log_buf);
+
+	if (!bpf_verifier_log_attr_valid(common->log_level, ubuf_common, common->log_size) ||
+	    !bpf_verifier_log_attr_valid(log_level, ubuf, log_size))
+		return -EINVAL;
+
+	if (ubuf && ubuf_common && (ubuf != ubuf_common || log_size != common->log_size ||
+				    log_level != common->log_level))
+		return -EINVAL;
+
+	memset(log, 0, sizeof(*log));
+	log->ubuf = ubuf;
+	log->size = log_size;
+	log->level = log_level;
+	log->offsetof_true_size = offsetof_log_true_size;
+	log->uattr = uattr;
+
+	if (!ubuf && ubuf_common) {
+		log->ubuf = ubuf_common;
+		log->size = common->log_size;
+		log->level = common->log_level;
+		log->uattr = uattr_common;
+		log->offsetof_true_size = 0;
+		if (size_common >= offsetofend(struct bpf_common_attr, log_true_size))
+			log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size);
+	}
+	return 0;
+}
+
+struct bpf_verifier_log *bpf_log_attr_create_vlog(struct bpf_log_attr *attr_log,
+						  struct bpf_common_attr *common, bpfptr_t uattr,
+						  u32 size)
+{
+	struct bpf_verifier_log *log;
+	int err;
+
+	memset(attr_log, 0, sizeof(*attr_log));
+	attr_log->uattr = uattr;
+	if (size >= offsetofend(struct bpf_common_attr, log_true_size))
+		attr_log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size);
+
+	if (!size)
+		return NULL;
+
+	log = kzalloc_obj(*log, GFP_KERNEL);
+	if (!log)
+		return ERR_PTR(-ENOMEM);
+
+	err = bpf_vlog_init(log, common->log_level, u64_to_user_ptr(common->log_buf),
+			    common->log_size);
+	if (err) {
+		kfree(log);
+		return ERR_PTR(err);
+	}
+
+	return log;
+}
+
+int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *log)
+{
+	u32 log_true_size;
+	int err;
+
+	err = bpf_vlog_finalize(log, &log_true_size);
+
+	if (attr->offsetof_true_size &&
+	    copy_to_bpfptr_offset(attr->uattr, attr->offsetof_true_size, &log_true_size,
+				  sizeof(log_true_size)))
+		return -EFAULT;
+
+	return err;
+}
diff --git a/kernel/bpf/states.c b/kernel/bpf/states.c
index 8478d2c6ed5b6..8773381360095 100644
--- a/kernel/bpf/states.c
+++ b/kernel/bpf/states.c
@@ -2,6 +2,7 @@
 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */
 #include <linux/bpf.h>
 #include <linux/bpf_verifier.h>
+#include <linux/cnum.h>
 #include <linux/filter.h>
 
 #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
@@ -301,14 +302,8 @@ int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_s
 static bool range_within(const struct bpf_reg_state *old,
 			 const struct bpf_reg_state *cur)
 {
-	return old->umin_value <= cur->umin_value &&
-	       old->umax_value >= cur->umax_value &&
-	       old->smin_value <= cur->smin_value &&
-	       old->smax_value >= cur->smax_value &&
-	       old->u32_min_value <= cur->u32_min_value &&
-	       old->u32_max_value >= cur->u32_max_value &&
-	       old->s32_min_value <= cur->s32_min_value &&
-	       old->s32_max_value >= cur->s32_max_value;
+	return cnum64_is_subset(old->r64, cur->r64) &&
+	       cnum32_is_subset(old->r32, cur->r32);
 }
 
 /* If in the old state two registers had the same id, then they need to have
@@ -838,6 +833,32 @@ static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old,
 	return true;
 }
 
+/*
+ * Compare stack arg slots between old and current states.
+ * Outgoing stack args are path-local state and must agree for pruning.
+ */
+static bool stack_arg_safe(struct bpf_verifier_env *env, struct bpf_func_state *old,
+			   struct bpf_func_state *cur, struct bpf_idmap *idmap,
+			   enum exact_level exact)
+{
+	int i, nslots;
+
+	nslots = max(old->out_stack_arg_cnt, cur->out_stack_arg_cnt);
+	for (i = 0; i < nslots; i++) {
+		struct bpf_reg_state *old_arg, *cur_arg;
+		struct bpf_reg_state not_init = { .type = NOT_INIT };
+
+		old_arg = i < old->out_stack_arg_cnt ?
+			  &old->stack_arg_regs[i] : &not_init;
+		cur_arg = i < cur->out_stack_arg_cnt ?
+			  &cur->stack_arg_regs[i] : &not_init;
+		if (!regsafe(env, old_arg, cur_arg, idmap, exact))
+			return false;
+	}
+
+	return true;
+}
+
 static bool refsafe(struct bpf_verifier_state *old, struct bpf_verifier_state *cur,
 		    struct bpf_idmap *idmap)
 {
@@ -920,6 +941,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat
 	if (old->callback_depth > cur->callback_depth)
 		return false;
 
+	if (!old->no_stack_arg_load && cur->no_stack_arg_load)
+		return false;
+
 	for (i = 0; i < MAX_BPF_REG; i++)
 		if (((1 << i) & live_regs) &&
 		    !regsafe(env, &old->regs[i], &cur->regs[i],
@@ -929,6 +953,9 @@ static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_stat
 	if (!stacksafe(env, old, cur, &env->idmap_scratch, exact))
 		return false;
 
+	if (!stack_arg_safe(env, old, cur, &env->idmap_scratch, exact))
+		return false;
+
 	return true;
 }
 
@@ -1376,7 +1403,7 @@ int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx)
 			 */
 			err = 0;
 			if (bpf_is_jmp_point(env, env->insn_idx))
-				err = bpf_push_jmp_history(env, cur, 0, 0);
+				err = bpf_push_jmp_history(env, cur, 0, 0, 0, 0);
 			err = err ? : propagate_precision(env, &sl->state, cur, NULL);
 			if (err)
 				return err;
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index a3c0214ca9341..8d4ea700aac6b 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1359,7 +1359,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
 
 #define BPF_MAP_CREATE_LAST_FIELD excl_prog_hash_size
 /* called via syscall */
-static int map_create(union bpf_attr *attr, bpfptr_t uattr)
+static int __map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_verifier_log *log)
 {
 	const struct bpf_map_ops *ops;
 	struct bpf_token *token = NULL;
@@ -1371,8 +1371,10 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
 	int err;
 
 	err = CHECK_ATTR(BPF_MAP_CREATE);
-	if (err)
+	if (err) {
+		bpf_log(log, "Invalid attr.\n");
 		return -EINVAL;
+	}
 
 	/* check BPF_F_TOKEN_FD flag, remember if it's set, and then clear it
 	 * to avoid per-map type checks tripping on unknown flag
@@ -1381,17 +1383,25 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
 	attr->map_flags &= ~BPF_F_TOKEN_FD;
 
 	if (attr->btf_vmlinux_value_type_id) {
-		if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS ||
-		    attr->btf_key_type_id || attr->btf_value_type_id)
+		if (attr->map_type != BPF_MAP_TYPE_STRUCT_OPS) {
+			bpf_log(log, "btf_vmlinux_value_type_id can only be used with struct_ops maps.\n");
 			return -EINVAL;
+		}
+		if (attr->btf_key_type_id || attr->btf_value_type_id) {
+			bpf_log(log, "btf_vmlinux_value_type_id is mutually exclusive with btf_key_type_id and btf_value_type_id.\n");
+			return -EINVAL;
+		}
 	} else if (attr->btf_key_type_id && !attr->btf_value_type_id) {
+		bpf_log(log, "Invalid btf_value_type_id.\n");
 		return -EINVAL;
 	}
 
 	if (attr->map_type != BPF_MAP_TYPE_BLOOM_FILTER &&
 	    attr->map_type != BPF_MAP_TYPE_ARENA &&
-	    attr->map_extra != 0)
+	    attr->map_extra != 0) {
+		bpf_log(log, "Invalid map_extra.\n");
 		return -EINVAL;
+	}
 
 	f_flags = bpf_get_file_flag(attr->map_flags);
 	if (f_flags < 0)
@@ -1399,13 +1409,17 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
 
 	if (numa_node != NUMA_NO_NODE &&
 	    ((unsigned int)numa_node >= nr_node_ids ||
-	     !node_online(numa_node)))
+	     !node_online(numa_node))) {
+		bpf_log(log, "Invalid numa_node.\n");
 		return -EINVAL;
+	}
 
 	/* find map type and init map: hashtable vs rbtree vs bloom vs ... */
 	map_type = attr->map_type;
-	if (map_type >= ARRAY_SIZE(bpf_map_types))
+	if (map_type >= ARRAY_SIZE(bpf_map_types)) {
+		bpf_log(log, "Invalid map_type.\n");
 		return -EINVAL;
+	}
 	map_type = array_index_nospec(map_type, ARRAY_SIZE(bpf_map_types));
 	ops = bpf_map_types[map_type];
 	if (!ops)
@@ -1423,8 +1437,10 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
 
 	if (token_flag) {
 		token = bpf_token_get_from_fd(attr->map_token_fd);
-		if (IS_ERR(token))
+		if (IS_ERR(token)) {
+			bpf_log(log, "Invalid map_token_fd.\n");
 			return PTR_ERR(token);
+		}
 
 		/* if current token doesn't grant map creation permissions,
 		 * then we can't use this token, so ignore it and rely on
@@ -1507,8 +1523,10 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
 
 	err = bpf_obj_name_cpy(map->name, attr->map_name,
 			       sizeof(attr->map_name));
-	if (err < 0)
+	if (err < 0) {
+		bpf_log(log, "Invalid map_name.\n");
 		goto free_map;
+	}
 
 	preempt_disable();
 	map->cookie = gen_cookie_next(&bpf_map_cookie);
@@ -1531,6 +1549,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
 
 		btf = btf_get_by_fd(attr->btf_fd);
 		if (IS_ERR(btf)) {
+			bpf_log(log, "Invalid btf_fd.\n");
 			err = PTR_ERR(btf);
 			goto free_map;
 		}
@@ -1558,6 +1577,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
 		bpfptr_t uprog_hash = make_bpfptr(attr->excl_prog_hash, uattr.is_kernel);
 
 		if (attr->excl_prog_hash_size != SHA256_DIGEST_SIZE) {
+			bpf_log(log, "Invalid excl_prog_hash_size.\n");
 			err = -EINVAL;
 			goto free_map;
 		}
@@ -1573,6 +1593,7 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
 			goto free_map;
 		}
 	} else if (attr->excl_prog_hash_size) {
+		bpf_log(log, "Invalid excl_prog_hash_size.\n");
 		err = -EINVAL;
 		goto free_map;
 	}
@@ -1611,6 +1632,31 @@ static int map_create(union bpf_attr *attr, bpfptr_t uattr)
 	return err;
 }
 
+static int map_create(union bpf_attr *attr, bpfptr_t uattr, struct bpf_common_attr *attr_common,
+		      bpfptr_t uattr_common, u32 size_common)
+{
+	struct bpf_verifier_log *log;
+	struct bpf_log_attr attr_log;
+	int err, ret;
+
+	log = bpf_log_attr_create_vlog(&attr_log, attr_common, uattr_common, size_common);
+	if (IS_ERR(log))
+		return PTR_ERR(log);
+
+	err = __map_create(attr, uattr, log);
+
+	/* preserve original error even if log finalization is successful */
+	ret = bpf_log_attr_finalize(&attr_log, log);
+	if (ret) {
+		if (err >= 0)
+			close_fd(err);
+		err = ret;
+	}
+
+	kfree(log);
+	return err;
+}
+
 void bpf_map_inc(struct bpf_map *map)
 {
 	atomic64_inc(&map->refcnt);
@@ -2861,7 +2907,7 @@ static int bpf_prog_mark_insn_arrays_ready(struct bpf_prog *prog)
 /* last field in 'union bpf_attr' used by this command */
 #define BPF_PROG_LOAD_LAST_FIELD keyring_id
 
-static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
+static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log)
 {
 	enum bpf_prog_type type = attr->prog_type;
 	struct bpf_prog *prog, *dst_prog = NULL;
@@ -3079,7 +3125,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
 		goto free_prog_sec;
 
 	/* run eBPF verifier */
-	err = bpf_check(&prog, attr, uattr, uattr_size);
+	err = bpf_check(&prog, attr, uattr, attr_log);
 	if (err < 0)
 		goto free_used_maps;
 
@@ -4281,6 +4327,11 @@ static int bpf_raw_tp_link_attach(struct bpf_prog *prog,
 	if (!btp)
 		return -ENOENT;
 
+	if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) {
+		bpf_put_raw_tracepoint(btp);
+		return -EINVAL;
+	}
+
 	link = kzalloc_obj(*link, GFP_USER);
 	if (!link) {
 		err = -ENOMEM;
@@ -4919,6 +4970,29 @@ static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog,
 	return map;
 }
 
+static void prepare_dump_pseudo_call(struct bpf_insn *insn)
+{
+	s32 call_off = insn->imm;
+
+	/*
+	 * BPF_CALL_ARGS only exists for interpreter fallback.
+	 * 1. For interpreter (BPF_CALL_ARGS): insn->off is the index of
+	 *    interpreters_args array, so here using bpf_call_args_imm()
+	 *    to get the real address offset.
+	 * 2. For JIT (BPF_CALL): insn->off is the subprog id.
+	 */
+	if (insn->code == (BPF_JMP | BPF_CALL_ARGS))
+		insn->imm = bpf_call_args_imm(insn->off);
+	else
+		insn->imm = insn->off;
+
+	/* Avoid dumping a truncated and misleading pc-relative offset. */
+	if (call_off > S16_MAX || call_off < S16_MIN)
+		insn->off = 0;
+	else
+		insn->off = call_off;
+}
+
 static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
 					      const struct cred *f_cred)
 {
@@ -4944,6 +5018,9 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog,
 		}
 		if (code == (BPF_JMP | BPF_CALL) ||
 		    code == (BPF_JMP | BPF_CALL_ARGS)) {
+			/* Restore the legacy xlated dump layout. */
+			if (insns[i].src_reg == BPF_PSEUDO_CALL)
+				prepare_dump_pseudo_call(&insns[i]);
 			if (code == (BPF_JMP | BPF_CALL_ARGS))
 				insns[i].code = BPF_JMP | BPF_CALL;
 			if (!bpf_dump_raw_ok(f_cred))
@@ -5469,7 +5546,7 @@ static int bpf_obj_get_info_by_fd(const union bpf_attr *attr,
 
 #define BPF_BTF_LOAD_LAST_FIELD btf_token_fd
 
-static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
+static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log)
 {
 	struct bpf_token *token = NULL;
 
@@ -5496,7 +5573,7 @@ static int bpf_btf_load(const union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_
 
 	bpf_token_put(token);
 
-	return btf_new_fd(attr, uattr, uattr_size);
+	return btf_new_fd(attr, uattr, attr_log);
 }
 
 #define BPF_BTF_GET_FD_BY_ID_LAST_FIELD fd_by_id_token_fd
@@ -6206,8 +6283,12 @@ static int prog_assoc_struct_ops(union bpf_attr *attr)
 	return ret;
 }
 
-static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
+static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size,
+		     bpfptr_t uattr_common, unsigned int size_common)
 {
+	struct bpf_common_attr attr_common;
+	u32 offsetof_log_true_size = 0;
+	struct bpf_log_attr attr_log;
 	union bpf_attr attr;
 	int err;
 
@@ -6221,13 +6302,29 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
 	if (copy_from_bpfptr(&attr, uattr, size) != 0)
 		return -EFAULT;
 
+	memset(&attr_common, 0, sizeof(attr_common));
+	if (cmd & BPF_COMMON_ATTRS) {
+		err = bpf_check_uarg_tail_zero(uattr_common,
+					       offsetofend(struct bpf_common_attr, log_true_size),
+					       size_common);
+		if (err)
+			return err;
+
+		cmd &= ~BPF_COMMON_ATTRS;
+		size_common = min_t(u32, size_common, sizeof(attr_common));
+		if (copy_from_bpfptr(&attr_common, uattr_common, size_common) != 0)
+			return -EFAULT;
+	} else {
+		size_common = 0;
+	}
+
 	err = security_bpf(cmd, &attr, size, uattr.is_kernel);
 	if (err < 0)
 		return err;
 
 	switch (cmd) {
 	case BPF_MAP_CREATE:
-		err = map_create(&attr, uattr);
+		err = map_create(&attr, uattr, &attr_common, uattr_common, size_common);
 		break;
 	case BPF_MAP_LOOKUP_ELEM:
 		err = map_lookup_elem(&attr);
@@ -6245,7 +6342,12 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
 		err = map_freeze(&attr);
 		break;
 	case BPF_PROG_LOAD:
-		err = bpf_prog_load(&attr, uattr, size);
+		if (size >= offsetofend(union bpf_attr, log_true_size))
+			offsetof_log_true_size = offsetof(union bpf_attr, log_true_size);
+		err = bpf_log_attr_init(&attr_log, attr.log_buf, attr.log_size, attr.log_level,
+					offsetof_log_true_size, uattr, &attr_common, uattr_common,
+					size_common);
+		err = err ?: bpf_prog_load(&attr, uattr, &attr_log);
 		break;
 	case BPF_OBJ_PIN:
 		err = bpf_obj_pin(&attr);
@@ -6290,7 +6392,12 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
 		err = bpf_raw_tracepoint_open(&attr);
 		break;
 	case BPF_BTF_LOAD:
-		err = bpf_btf_load(&attr, uattr, size);
+		if (size >= offsetofend(union bpf_attr, btf_log_true_size))
+			offsetof_log_true_size = offsetof(union bpf_attr, btf_log_true_size);
+		err = bpf_log_attr_init(&attr_log, attr.btf_log_buf, attr.btf_log_size,
+					attr.btf_log_level, offsetof_log_true_size, uattr,
+					&attr_common, uattr_common, size_common);
+		err = err ?: bpf_btf_load(&attr, uattr, &attr_log);
 		break;
 	case BPF_BTF_GET_FD_BY_ID:
 		err = bpf_btf_get_fd_by_id(&attr);
@@ -6356,9 +6463,10 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size)
 	return err;
 }
 
-SYSCALL_DEFINE3(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size)
+SYSCALL_DEFINE5(bpf, int, cmd, union bpf_attr __user *, uattr, unsigned int, size,
+		struct bpf_common_attr __user *, uattr_common, unsigned int, size_common)
 {
-	return __sys_bpf(cmd, USER_BPFPTR(uattr), size);
+	return __sys_bpf(cmd, USER_BPFPTR(uattr), size, USER_BPFPTR(uattr_common), size_common);
 }
 
 static bool syscall_prog_is_valid_access(int off, int size,
@@ -6388,7 +6496,7 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
 	default:
 		return -EINVAL;
 	}
-	return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size);
+	return __sys_bpf(cmd, KERNEL_BPFPTR(attr), attr_size, KERNEL_BPFPTR(NULL), 0);
 }
 
 
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index f02254a21585f..a4298a25d4bae 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -685,7 +685,6 @@ static int bpf_trampoline_update(struct bpf_trampoline *tr, bool lock_direct_mut
 	if (err)
 		goto out_free;
 
-	WARN_ON(tr->cur_image && total == 0);
 	if (tr->cur_image)
 		/* progs already running at this address */
 		err = modify_fentry(tr, orig_flags, tr->cur_image->image,
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index 69d75515ed3f7..4e3aff5c9ed18 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -26,6 +26,7 @@
 #include <linux/poison.h>
 #include <linux/module.h>
 #include <linux/cpumask.h>
+#include <linux/cnum.h>
 #include <linux/bpf_mem_alloc.h>
 #include <net/xdp.h>
 #include <linux/trace_events.h>
@@ -261,6 +262,41 @@ struct bpf_kfunc_meta {
 
 struct btf *btf_vmlinux;
 
+typedef struct argno {
+	int argno;
+} argno_t;
+
+static argno_t argno_from_reg(u32 regno)
+{
+	return (argno_t){ .argno = regno };
+}
+
+static argno_t argno_from_arg(u32 arg)
+{
+	return (argno_t){ .argno = -arg };
+}
+
+static int reg_from_argno(argno_t a)
+{
+	if (a.argno >= 0)
+		return a.argno;
+	if (a.argno >= -MAX_BPF_FUNC_REG_ARGS)
+		return -a.argno;
+	return -1;
+}
+
+static int arg_from_argno(argno_t a)
+{
+	if (a.argno < 0)
+		return -a.argno;
+	return -1;
+}
+
+static int arg_idx_from_argno(argno_t a)
+{
+	return arg_from_argno(a) - 1;
+}
+
 static const char *btf_type_name(const struct btf *btf, u32 id)
 {
 	return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
@@ -290,12 +326,12 @@ static void verbose_invalid_scalar(struct bpf_verifier_env *env,
 	bool unknown = true;
 
 	verbose(env, "%s the register %s has", ctx, reg_name);
-	if (reg->smin_value > S64_MIN) {
-		verbose(env, " smin=%lld", reg->smin_value);
+	if (reg_smin(reg) > S64_MIN) {
+		verbose(env, " smin=%lld", reg_smin(reg));
 		unknown = false;
 	}
-	if (reg->smax_value < S64_MAX) {
-		verbose(env, " smax=%lld", reg->smax_value);
+	if (reg_smax(reg) < S64_MAX) {
+		verbose(env, " smax=%lld", reg_smax(reg));
 		unknown = false;
 	}
 	if (unknown)
@@ -442,7 +478,6 @@ static bool is_dynptr_ref_function(enum bpf_func_id func_id)
 static bool is_sync_callback_calling_kfunc(u32 btf_id);
 static bool is_async_callback_calling_kfunc(u32 btf_id);
 static bool is_callback_calling_kfunc(u32 btf_id);
-static bool is_bpf_throw_kfunc(struct bpf_insn *insn);
 
 static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);
 static bool is_task_work_add_kfunc(u32 func_id);
@@ -717,15 +752,6 @@ static int unmark_stack_slots_dynptr(struct bpf_verifier_env *env, struct bpf_re
 	struct bpf_func_state *state = bpf_func(env, reg);
 	int spi, ref_obj_id, i;
 
-	/*
-	 * This can only be set for PTR_TO_STACK, as CONST_PTR_TO_DYNPTR cannot
-	 * be released by any dynptr helper. Hence, unmark_stack_slots_dynptr
-	 * is safe to do directly.
-	 */
-	if (reg->type == CONST_PTR_TO_DYNPTR) {
-		verifier_bug(env, "CONST_PTR_TO_DYNPTR cannot be released");
-		return -EFAULT;
-	}
 	spi = dynptr_get_spi(env, reg);
 	if (spi < 0)
 		return spi;
@@ -1340,6 +1366,18 @@ static int copy_stack_state(struct bpf_func_state *dst, const struct bpf_func_st
 		return -ENOMEM;
 
 	dst->allocated_stack = src->allocated_stack;
+
+	/* copy stack args state */
+	n = src->out_stack_arg_cnt;
+	if (n) {
+		dst->stack_arg_regs = copy_array(dst->stack_arg_regs, src->stack_arg_regs, n,
+						 sizeof(struct bpf_reg_state),
+						 GFP_KERNEL_ACCOUNT);
+		if (!dst->stack_arg_regs)
+			return -ENOMEM;
+	}
+
+	dst->out_stack_arg_cnt = src->out_stack_arg_cnt;
 	return 0;
 }
 
@@ -1381,6 +1419,23 @@ static int grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state
 	return 0;
 }
 
+static int grow_stack_arg_slots(struct bpf_verifier_env *env,
+				struct bpf_func_state *state, int cnt)
+{
+	size_t old_n = state->out_stack_arg_cnt;
+
+	if (old_n >= cnt)
+		return 0;
+
+	state->stack_arg_regs = realloc_array(state->stack_arg_regs, old_n, cnt,
+					      sizeof(struct bpf_reg_state));
+	if (!state->stack_arg_regs)
+		return -ENOMEM;
+
+	state->out_stack_arg_cnt = cnt;
+	return 0;
+}
+
 /* Acquire a pointer id from the env and update the state->refs to include
  * this new pointer reference.
  * On success, returns a valid pointer id to associate with the register
@@ -1543,6 +1598,7 @@ static void free_func_state(struct bpf_func_state *state)
 {
 	if (!state)
 		return;
+	kfree(state->stack_arg_regs);
 	kfree(state->stack);
 	kfree(state);
 }
@@ -1751,6 +1807,22 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
 	return &elem->st;
 }
 
+static const char *reg_arg_name(struct bpf_verifier_env *env, argno_t argno)
+{
+	char *buf = env->tmp_arg_name;
+	int len = sizeof(env->tmp_arg_name);
+	int arg, regno = reg_from_argno(argno);
+
+	if (regno >= 0) {
+		snprintf(buf, len, "R%d", regno);
+	} else {
+		arg = arg_from_argno(argno);
+		snprintf(buf, len, "*(R11-%u)", (arg - MAX_BPF_FUNC_REG_ARGS) * BPF_REG_SIZE);
+	}
+
+	return buf;
+}
+
 static const int caller_saved[CALLER_SAVED_REGS] = {
 	BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
 };
@@ -1759,15 +1831,8 @@ static const int caller_saved[CALLER_SAVED_REGS] = {
 static void ___mark_reg_known(struct bpf_reg_state *reg, u64 imm)
 {
 	reg->var_off = tnum_const(imm);
-	reg->smin_value = (s64)imm;
-	reg->smax_value = (s64)imm;
-	reg->umin_value = imm;
-	reg->umax_value = imm;
-
-	reg->s32_min_value = (s32)imm;
-	reg->s32_max_value = (s32)imm;
-	reg->u32_min_value = (u32)imm;
-	reg->u32_max_value = (u32)imm;
+	reg->r64 = cnum64_from_urange(imm, imm);
+	reg->r32 = cnum32_from_urange((u32)imm, (u32)imm);
 }
 
 /* Mark the unknown part of a register (variable offset or scalar value) as
@@ -1786,10 +1851,7 @@ static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
 static void __mark_reg32_known(struct bpf_reg_state *reg, u64 imm)
 {
 	reg->var_off = tnum_const_subreg(reg->var_off, imm);
-	reg->s32_min_value = (s32)imm;
-	reg->s32_max_value = (s32)imm;
-	reg->u32_min_value = (u32)imm;
-	reg->u32_max_value = (u32)imm;
+	reg->r32 = cnum32_from_urange((u32)imm, (u32)imm);
 }
 
 /* Mark the 'variable offset' part of a register as zero.  This should be
@@ -1900,34 +1962,21 @@ static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
 	       tnum_equals_const(reg->var_off, 0);
 }
 
-/* Reset the min/max bounds of a register */
-static void __mark_reg_unbounded(struct bpf_reg_state *reg)
+static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
 {
-	reg->smin_value = S64_MIN;
-	reg->smax_value = S64_MAX;
-	reg->umin_value = 0;
-	reg->umax_value = U64_MAX;
-
-	reg->s32_min_value = S32_MIN;
-	reg->s32_max_value = S32_MAX;
-	reg->u32_min_value = 0;
-	reg->u32_max_value = U32_MAX;
+	reg->r32 = CNUM32_UNBOUNDED;
 }
 
 static void __mark_reg64_unbounded(struct bpf_reg_state *reg)
 {
-	reg->smin_value = S64_MIN;
-	reg->smax_value = S64_MAX;
-	reg->umin_value = 0;
-	reg->umax_value = U64_MAX;
+	reg->r64 = CNUM64_UNBOUNDED;
 }
 
-static void __mark_reg32_unbounded(struct bpf_reg_state *reg)
+/* Reset the min/max bounds of a register */
+static void __mark_reg_unbounded(struct bpf_reg_state *reg)
 {
-	reg->s32_min_value = S32_MIN;
-	reg->s32_max_value = S32_MAX;
-	reg->u32_min_value = 0;
-	reg->u32_max_value = U32_MAX;
+	__mark_reg64_unbounded(reg);
+	__mark_reg32_unbounded(reg);
 }
 
 static void reset_reg64_and_tnum(struct bpf_reg_state *reg)
@@ -1942,19 +1991,32 @@ static void reset_reg32_and_tnum(struct bpf_reg_state *reg)
 	reg->var_off = tnum_unknown;
 }
 
-static void __update_reg32_bounds(struct bpf_reg_state *reg)
+static struct cnum32 cnum32_from_tnum(struct tnum tnum)
 {
-	struct tnum var32_off = tnum_subreg(reg->var_off);
+	tnum = tnum_subreg(tnum);
+	if ((tnum.mask & S32_MIN) || (tnum.value & S32_MIN))
+		/* min signed is max(sign bit) | min(other bits) */
+		/* max signed is min(sign bit) | max(other bits) */
+		return cnum32_from_srange(tnum.value | (tnum.mask & S32_MIN),
+					  tnum.value | (tnum.mask & S32_MAX));
+	else
+		return cnum32_from_urange(tnum.value, (tnum.value | tnum.mask));
+}
 
-	/* min signed is max(sign bit) | min(other bits) */
-	reg->s32_min_value = max_t(s32, reg->s32_min_value,
-			var32_off.value | (var32_off.mask & S32_MIN));
-	/* max signed is min(sign bit) | max(other bits) */
-	reg->s32_max_value = min_t(s32, reg->s32_max_value,
-			var32_off.value | (var32_off.mask & S32_MAX));
-	reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)var32_off.value);
-	reg->u32_max_value = min(reg->u32_max_value,
-				 (u32)(var32_off.value | var32_off.mask));
+static struct cnum64 cnum64_from_tnum(struct tnum tnum)
+{
+	if ((tnum.mask & S64_MIN) || (tnum.value & S64_MIN))
+		/* min signed is max(sign bit) | min(other bits) */
+		/* max signed is min(sign bit) | max(other bits) */
+		return cnum64_from_srange(tnum.value | (tnum.mask & S64_MIN),
+					  tnum.value | (tnum.mask & S64_MAX));
+	else
+		return cnum64_from_urange(tnum.value, (tnum.value | tnum.mask));
+}
+
+static void __update_reg32_bounds(struct bpf_reg_state *reg)
+{
+	cnum32_intersect_with(&reg->r32, cnum32_from_tnum(reg->var_off));
 }
 
 static void __update_reg64_bounds(struct bpf_reg_state *reg)
@@ -1962,26 +2024,18 @@ static void __update_reg64_bounds(struct bpf_reg_state *reg)
 	u64 tnum_next, tmax;
 	bool umin_in_tnum;
 
-	/* min signed is max(sign bit) | min(other bits) */
-	reg->smin_value = max_t(s64, reg->smin_value,
-				reg->var_off.value | (reg->var_off.mask & S64_MIN));
-	/* max signed is min(sign bit) | max(other bits) */
-	reg->smax_value = min_t(s64, reg->smax_value,
-				reg->var_off.value | (reg->var_off.mask & S64_MAX));
-	reg->umin_value = max(reg->umin_value, reg->var_off.value);
-	reg->umax_value = min(reg->umax_value,
-			      reg->var_off.value | reg->var_off.mask);
+	cnum64_intersect_with(&reg->r64, cnum64_from_tnum(reg->var_off));
 
 	/* Check if u64 and tnum overlap in a single value */
-	tnum_next = tnum_step(reg->var_off, reg->umin_value);
-	umin_in_tnum = (reg->umin_value & ~reg->var_off.mask) == reg->var_off.value;
+	tnum_next = tnum_step(reg->var_off, reg_umin(reg));
+	umin_in_tnum = (reg_umin(reg) & ~reg->var_off.mask) == reg->var_off.value;
 	tmax = reg->var_off.value | reg->var_off.mask;
-	if (umin_in_tnum && tnum_next > reg->umax_value) {
+	if (umin_in_tnum && tnum_next > reg_umax(reg)) {
 		/* The u64 range and the tnum only overlap in umin.
 		 * u64:  ---[xxxxxx]-----
 		 * tnum: --xx----------x-
 		 */
-		___mark_reg_known(reg, reg->umin_value);
+		___mark_reg_known(reg, reg_umin(reg));
 	} else if (!umin_in_tnum && tnum_next == tmax) {
 		/* The u64 range and the tnum only overlap in the maximum value
 		 * represented by the tnum, called tmax.
@@ -1989,8 +2043,8 @@ static void __update_reg64_bounds(struct bpf_reg_state *reg)
 		 * tnum: xx-----x--------
 		 */
 		___mark_reg_known(reg, tmax);
-	} else if (!umin_in_tnum && tnum_next <= reg->umax_value &&
-		   tnum_step(reg->var_off, tnum_next) > reg->umax_value) {
+	} else if (!umin_in_tnum && tnum_next <= reg_umax(reg) &&
+		   tnum_step(reg->var_off, tnum_next) > reg_umax(reg)) {
 		/* The u64 range and the tnum only overlap in between umin
 		 * (excluded) and umax.
 		 * u64:  ---[xxxxxx]-----
@@ -2006,329 +2060,19 @@ static void __update_reg_bounds(struct bpf_reg_state *reg)
 	__update_reg64_bounds(reg);
 }
 
-/* Uses signed min/max values to inform unsigned, and vice-versa */
 static void deduce_bounds_32_from_64(struct bpf_reg_state *reg)
 {
-	/* If upper 32 bits of u64/s64 range don't change, we can use lower 32
-	 * bits to improve our u32/s32 boundaries.
-	 *
-	 * E.g., the case where we have upper 32 bits as zero ([10, 20] in
-	 * u64) is pretty trivial, it's obvious that in u32 we'll also have
-	 * [10, 20] range. But this property holds for any 64-bit range as
-	 * long as upper 32 bits in that entire range of values stay the same.
-	 *
-	 * E.g., u64 range [0x10000000A, 0x10000000F] ([4294967306, 4294967311]
-	 * in decimal) has the same upper 32 bits throughout all the values in
-	 * that range. As such, lower 32 bits form a valid [0xA, 0xF] ([10, 15])
-	 * range.
-	 *
-	 * Note also, that [0xA, 0xF] is a valid range both in u32 and in s32,
-	 * following the rules outlined below about u64/s64 correspondence
-	 * (which equally applies to u32 vs s32 correspondence). In general it
-	 * depends on actual hexadecimal values of 32-bit range. They can form
-	 * only valid u32, or only valid s32 ranges in some cases.
-	 *
-	 * So we use all these insights to derive bounds for subregisters here.
-	 */
-	if ((reg->umin_value >> 32) == (reg->umax_value >> 32)) {
-		/* u64 to u32 casting preserves validity of low 32 bits as
-		 * a range, if upper 32 bits are the same
-		 */
-		reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->umin_value);
-		reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->umax_value);
-
-		if ((s32)reg->umin_value <= (s32)reg->umax_value) {
-			reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
-			reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
-		}
-	}
-	if ((reg->smin_value >> 32) == (reg->smax_value >> 32)) {
-		/* low 32 bits should form a proper u32 range */
-		if ((u32)reg->smin_value <= (u32)reg->smax_value) {
-			reg->u32_min_value = max_t(u32, reg->u32_min_value, (u32)reg->smin_value);
-			reg->u32_max_value = min_t(u32, reg->u32_max_value, (u32)reg->smax_value);
-		}
-		/* low 32 bits should form a proper s32 range */
-		if ((s32)reg->smin_value <= (s32)reg->smax_value) {
-			reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
-			reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
-		}
-	}
-	/* Special case where upper bits form a small sequence of two
-	 * sequential numbers (in 32-bit unsigned space, so 0xffffffff to
-	 * 0x00000000 is also valid), while lower bits form a proper s32 range
-	 * going from negative numbers to positive numbers. E.g., let's say we
-	 * have s64 range [-1, 1] ([0xffffffffffffffff, 0x0000000000000001]).
-	 * Possible s64 values are {-1, 0, 1} ({0xffffffffffffffff,
-	 * 0x0000000000000000, 0x00000000000001}). Ignoring upper 32 bits,
-	 * we still get a valid s32 range [-1, 1] ([0xffffffff, 0x00000001]).
-	 * Note that it doesn't have to be 0xffffffff going to 0x00000000 in
-	 * upper 32 bits. As a random example, s64 range
-	 * [0xfffffff0fffffff0; 0xfffffff100000010], forms a valid s32 range
-	 * [-16, 16] ([0xfffffff0; 0x00000010]) in its 32 bit subregister.
-	 */
-	if ((u32)(reg->umin_value >> 32) + 1 == (u32)(reg->umax_value >> 32) &&
-	    (s32)reg->umin_value < 0 && (s32)reg->umax_value >= 0) {
-		reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->umin_value);
-		reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->umax_value);
-	}
-	if ((u32)(reg->smin_value >> 32) + 1 == (u32)(reg->smax_value >> 32) &&
-	    (s32)reg->smin_value < 0 && (s32)reg->smax_value >= 0) {
-		reg->s32_min_value = max_t(s32, reg->s32_min_value, (s32)reg->smin_value);
-		reg->s32_max_value = min_t(s32, reg->s32_max_value, (s32)reg->smax_value);
-	}
-}
-
-static void deduce_bounds_32_from_32(struct bpf_reg_state *reg)
-{
-	/* if u32 range forms a valid s32 range (due to matching sign bit),
-	 * try to learn from that
-	 */
-	if ((s32)reg->u32_min_value <= (s32)reg->u32_max_value) {
-		reg->s32_min_value = max_t(s32, reg->s32_min_value, reg->u32_min_value);
-		reg->s32_max_value = min_t(s32, reg->s32_max_value, reg->u32_max_value);
-	}
-	/* If we cannot cross the sign boundary, then signed and unsigned bounds
-	 * are the same, so combine.  This works even in the negative case, e.g.
-	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
-	 */
-	if ((u32)reg->s32_min_value <= (u32)reg->s32_max_value) {
-		reg->u32_min_value = max_t(u32, reg->s32_min_value, reg->u32_min_value);
-		reg->u32_max_value = min_t(u32, reg->s32_max_value, reg->u32_max_value);
-	} else {
-		if (reg->u32_max_value < (u32)reg->s32_min_value) {
-			/* See __reg64_deduce_bounds() for detailed explanation.
-			 * Refine ranges in the following situation:
-			 *
-			 * 0                                                   U32_MAX
-			 * |  [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx]              |
-			 * |----------------------------|----------------------------|
-			 * |xxxxx s32 range xxxxxxxxx]                       [xxxxxxx|
-			 * 0                     S32_MAX S32_MIN                    -1
-			 */
-			reg->s32_min_value = (s32)reg->u32_min_value;
-			reg->u32_max_value = min_t(u32, reg->u32_max_value, reg->s32_max_value);
-		} else if ((u32)reg->s32_max_value < reg->u32_min_value) {
-			/*
-			 * 0                                                   U32_MAX
-			 * |              [xxxxxxxxxxxxxx u32 range xxxxxxxxxxxxxx]  |
-			 * |----------------------------|----------------------------|
-			 * |xxxxxxxxx]                       [xxxxxxxxxxxx s32 range |
-			 * 0                     S32_MAX S32_MIN                    -1
-			 */
-			reg->s32_max_value = (s32)reg->u32_max_value;
-			reg->u32_min_value = max_t(u32, reg->u32_min_value, reg->s32_min_value);
-		}
-	}
-}
-
-static void deduce_bounds_64_from_64(struct bpf_reg_state *reg)
-{
-	/* If u64 range forms a valid s64 range (due to matching sign bit),
-	 * try to learn from that. Let's do a bit of ASCII art to see when
-	 * this is happening. Let's take u64 range first:
-	 *
-	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
-	 * |-------------------------------|--------------------------------|
-	 *
-	 * Valid u64 range is formed when umin and umax are anywhere in the
-	 * range [0, U64_MAX], and umin <= umax. u64 case is simple and
-	 * straightforward. Let's see how s64 range maps onto the same range
-	 * of values, annotated below the line for comparison:
-	 *
-	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
-	 * |-------------------------------|--------------------------------|
-	 * 0                        S64_MAX S64_MIN                        -1
-	 *
-	 * So s64 values basically start in the middle and they are logically
-	 * contiguous to the right of it, wrapping around from -1 to 0, and
-	 * then finishing as S64_MAX (0x7fffffffffffffff) right before
-	 * S64_MIN. We can try drawing the continuity of u64 vs s64 values
-	 * more visually as mapped to sign-agnostic range of hex values.
-	 *
-	 *  u64 start                                               u64 end
-	 *  _______________________________________________________________
-	 * /                                                               \
-	 * 0             0x7fffffffffffffff 0x8000000000000000        U64_MAX
-	 * |-------------------------------|--------------------------------|
-	 * 0                        S64_MAX S64_MIN                        -1
-	 *                                / \
-	 * >------------------------------   ------------------------------->
-	 * s64 continues...        s64 end   s64 start          s64 "midpoint"
-	 *
-	 * What this means is that, in general, we can't always derive
-	 * something new about u64 from any random s64 range, and vice versa.
-	 *
-	 * But we can do that in two particular cases. One is when entire
-	 * u64/s64 range is *entirely* contained within left half of the above
-	 * diagram or when it is *entirely* contained in the right half. I.e.:
-	 *
-	 * |-------------------------------|--------------------------------|
-	 *     ^                   ^            ^                 ^
-	 *     A                   B            C                 D
-	 *
-	 * [A, B] and [C, D] are contained entirely in their respective halves
-	 * and form valid contiguous ranges as both u64 and s64 values. [A, B]
-	 * will be non-negative both as u64 and s64 (and in fact it will be
-	 * identical ranges no matter the signedness). [C, D] treated as s64
-	 * will be a range of negative values, while in u64 it will be
-	 * non-negative range of values larger than 0x8000000000000000.
-	 *
-	 * Now, any other range here can't be represented in both u64 and s64
-	 * simultaneously. E.g., [A, C], [A, D], [B, C], [B, D] are valid
-	 * contiguous u64 ranges, but they are discontinuous in s64. [B, C]
-	 * in s64 would be properly presented as [S64_MIN, C] and [B, S64_MAX],
-	 * for example. Similarly, valid s64 range [D, A] (going from negative
-	 * to positive values), would be two separate [D, U64_MAX] and [0, A]
-	 * ranges as u64. Currently reg_state can't represent two segments per
-	 * numeric domain, so in such situations we can only derive maximal
-	 * possible range ([0, U64_MAX] for u64, and [S64_MIN, S64_MAX] for s64).
-	 *
-	 * So we use these facts to derive umin/umax from smin/smax and vice
-	 * versa only if they stay within the same "half". This is equivalent
-	 * to checking sign bit: lower half will have sign bit as zero, upper
-	 * half have sign bit 1. Below in code we simplify this by just
-	 * casting umin/umax as smin/smax and checking if they form valid
-	 * range, and vice versa. Those are equivalent checks.
-	 */
-	if ((s64)reg->umin_value <= (s64)reg->umax_value) {
-		reg->smin_value = max_t(s64, reg->smin_value, reg->umin_value);
-		reg->smax_value = min_t(s64, reg->smax_value, reg->umax_value);
-	}
-	/* If we cannot cross the sign boundary, then signed and unsigned bounds
-	 * are the same, so combine.  This works even in the negative case, e.g.
-	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
-	 */
-	if ((u64)reg->smin_value <= (u64)reg->smax_value) {
-		reg->umin_value = max_t(u64, reg->smin_value, reg->umin_value);
-		reg->umax_value = min_t(u64, reg->smax_value, reg->umax_value);
-	} else {
-		/* If the s64 range crosses the sign boundary, then it's split
-		 * between the beginning and end of the U64 domain. In that
-		 * case, we can derive new bounds if the u64 range overlaps
-		 * with only one end of the s64 range.
-		 *
-		 * In the following example, the u64 range overlaps only with
-		 * positive portion of the s64 range.
-		 *
-		 * 0                                                   U64_MAX
-		 * |  [xxxxxxxxxxxxxx u64 range xxxxxxxxxxxxxx]              |
-		 * |----------------------------|----------------------------|
-		 * |xxxxx s64 range xxxxxxxxx]                       [xxxxxxx|
-		 * 0                     S64_MAX S64_MIN                    -1
-		 *
-		 * We can thus derive the following new s64 and u64 ranges.
-		 *
-		 * 0                                                   U64_MAX
-		 * |  [xxxxxx u64 range xxxxx]                               |
-		 * |----------------------------|----------------------------|
-		 * |  [xxxxxx s64 range xxxxx]                               |
-		 * 0                     S64_MAX S64_MIN                    -1
-		 *
-		 * If they overlap in two places, we can't derive anything
-		 * because reg_state can't represent two ranges per numeric
-		 * domain.
-		 *
-		 * 0                                                   U64_MAX
-		 * |  [xxxxxxxxxxxxxxxxx u64 range xxxxxxxxxxxxxxxxx]        |
-		 * |----------------------------|----------------------------|
-		 * |xxxxx s64 range xxxxxxxxx]                    [xxxxxxxxxx|
-		 * 0                     S64_MAX S64_MIN                    -1
-		 *
-		 * The first condition below corresponds to the first diagram
-		 * above.
-		 */
-		if (reg->umax_value < (u64)reg->smin_value) {
-			reg->smin_value = (s64)reg->umin_value;
-			reg->umax_value = min_t(u64, reg->umax_value, reg->smax_value);
-		} else if ((u64)reg->smax_value < reg->umin_value) {
-			/* This second condition considers the case where the u64 range
-			 * overlaps with the negative portion of the s64 range:
-			 *
-			 * 0                                                   U64_MAX
-			 * |              [xxxxxxxxxxxxxx u64 range xxxxxxxxxxxxxx]  |
-			 * |----------------------------|----------------------------|
-			 * |xxxxxxxxx]                       [xxxxxxxxxxxx s64 range |
-			 * 0                     S64_MAX S64_MIN                    -1
-			 */
-			reg->smax_value = (s64)reg->umax_value;
-			reg->umin_value = max_t(u64, reg->umin_value, reg->smin_value);
-		}
-	}
+	cnum32_intersect_with(&reg->r32, cnum32_from_cnum64(reg->r64));
 }
 
 static void deduce_bounds_64_from_32(struct bpf_reg_state *reg)
 {
-	/* Try to tighten 64-bit bounds from 32-bit knowledge, using 32-bit
-	 * values on both sides of 64-bit range in hope to have tighter range.
-	 * E.g., if r1 is [0x1'00000000, 0x3'80000000], and we learn from
-	 * 32-bit signed > 0 operation that s32 bounds are now [1; 0x7fffffff].
-	 * With this, we can substitute 1 as low 32-bits of _low_ 64-bit bound
-	 * (0x100000000 -> 0x100000001) and 0x7fffffff as low 32-bits of
-	 * _high_ 64-bit bound (0x380000000 -> 0x37fffffff) and arrive at a
-	 * better overall bounds for r1 as [0x1'000000001; 0x3'7fffffff].
-	 * We just need to make sure that derived bounds we are intersecting
-	 * with are well-formed ranges in respective s64 or u64 domain, just
-	 * like we do with similar kinds of 32-to-64 or 64-to-32 adjustments.
-	 */
-	__u64 new_umin, new_umax;
-	__s64 new_smin, new_smax;
-
-	/* u32 -> u64 tightening, it's always well-formed */
-	new_umin = (reg->umin_value & ~0xffffffffULL) | reg->u32_min_value;
-	new_umax = (reg->umax_value & ~0xffffffffULL) | reg->u32_max_value;
-	reg->umin_value = max_t(u64, reg->umin_value, new_umin);
-	reg->umax_value = min_t(u64, reg->umax_value, new_umax);
-	/* u32 -> s64 tightening, u32 range embedded into s64 preserves range validity */
-	new_smin = (reg->smin_value & ~0xffffffffULL) | reg->u32_min_value;
-	new_smax = (reg->smax_value & ~0xffffffffULL) | reg->u32_max_value;
-	reg->smin_value = max_t(s64, reg->smin_value, new_smin);
-	reg->smax_value = min_t(s64, reg->smax_value, new_smax);
-
-	/* Here we would like to handle a special case after sign extending load,
-	 * when upper bits for a 64-bit range are all 1s or all 0s.
-	 *
-	 * Upper bits are all 1s when register is in a range:
-	 *   [0xffff_ffff_0000_0000, 0xffff_ffff_ffff_ffff]
-	 * Upper bits are all 0s when register is in a range:
-	 *   [0x0000_0000_0000_0000, 0x0000_0000_ffff_ffff]
-	 * Together this forms are continuous range:
-	 *   [0xffff_ffff_0000_0000, 0x0000_0000_ffff_ffff]
-	 *
-	 * Now, suppose that register range is in fact tighter:
-	 *   [0xffff_ffff_8000_0000, 0x0000_0000_ffff_ffff] (R)
-	 * Also suppose that it's 32-bit range is positive,
-	 * meaning that lower 32-bits of the full 64-bit register
-	 * are in the range:
-	 *   [0x0000_0000, 0x7fff_ffff] (W)
-	 *
-	 * If this happens, then any value in a range:
-	 *   [0xffff_ffff_0000_0000, 0xffff_ffff_7fff_ffff]
-	 * is smaller than a lowest bound of the range (R):
-	 *   0xffff_ffff_8000_0000
-	 * which means that upper bits of the full 64-bit register
-	 * can't be all 1s, when lower bits are in range (W).
-	 *
-	 * Note that:
-	 *  - 0xffff_ffff_8000_0000 == (s64)S32_MIN
-	 *  - 0x0000_0000_7fff_ffff == (s64)S32_MAX
-	 * These relations are used in the conditions below.
-	 */
-	if (reg->s32_min_value >= 0 && reg->smin_value >= S32_MIN && reg->smax_value <= S32_MAX) {
-		reg->smin_value = reg->s32_min_value;
-		reg->smax_value = reg->s32_max_value;
-		reg->umin_value = reg->s32_min_value;
-		reg->umax_value = reg->s32_max_value;
-		reg->var_off = tnum_intersect(reg->var_off,
-					      tnum_range(reg->smin_value, reg->smax_value));
-	}
+	reg->r64 = cnum64_cnum32_intersect(reg->r64, reg->r32);
 }
 
 static void __reg_deduce_bounds(struct bpf_reg_state *reg)
 {
-	deduce_bounds_64_from_64(reg);
 	deduce_bounds_32_from_64(reg);
-	deduce_bounds_32_from_32(reg);
 	deduce_bounds_64_from_32(reg);
 }
 
@@ -2336,11 +2080,11 @@ static void __reg_deduce_bounds(struct bpf_reg_state *reg)
 static void __reg_bound_offset(struct bpf_reg_state *reg)
 {
 	struct tnum var64_off = tnum_intersect(reg->var_off,
-					       tnum_range(reg->umin_value,
-							  reg->umax_value));
+					       tnum_range(reg_umin(reg),
+							  reg_umax(reg)));
 	struct tnum var32_off = tnum_intersect(tnum_subreg(var64_off),
-					       tnum_range(reg->u32_min_value,
-							  reg->u32_max_value));
+					       tnum_range(reg_u32_min(reg),
+							  reg_u32_max(reg)));
 
 	reg->var_off = tnum_or(tnum_clear_subreg(var64_off), var32_off);
 }
@@ -2366,35 +2110,25 @@ static void reg_bounds_sync(struct bpf_reg_state *reg)
 	__update_reg_bounds(reg);
 }
 
-static bool range_bounds_violation(struct bpf_reg_state *reg)
-{
-	return (reg->umin_value > reg->umax_value || reg->smin_value > reg->smax_value ||
-		reg->u32_min_value > reg->u32_max_value ||
-		reg->s32_min_value > reg->s32_max_value);
-}
-
 static bool const_tnum_range_mismatch(struct bpf_reg_state *reg)
 {
-	u64 uval = reg->var_off.value;
-	s64 sval = (s64)uval;
-
 	if (!tnum_is_const(reg->var_off))
 		return false;
 
-	return reg->umin_value != uval || reg->umax_value != uval ||
-	       reg->smin_value != sval || reg->smax_value != sval;
+	return !cnum64_is_const(reg->r64) || reg->r64.base != reg->var_off.value;
 }
 
 static bool const_tnum_range_mismatch_32(struct bpf_reg_state *reg)
 {
-	u32 uval32 = tnum_subreg(reg->var_off).value;
-	s32 sval32 = (s32)uval32;
-
 	if (!tnum_subreg_is_const(reg->var_off))
 		return false;
 
-	return reg->u32_min_value != uval32 || reg->u32_max_value != uval32 ||
-	       reg->s32_min_value != sval32 || reg->s32_max_value != sval32;
+	return !cnum32_is_const(reg->r32) || reg->r32.base != tnum_subreg(reg->var_off).value;
+}
+
+static bool range_bounds_violation(struct bpf_reg_state *reg)
+{
+	return cnum32_is_empty(reg->r32) || cnum64_is_empty(reg->r64);
 }
 
 static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
@@ -2419,12 +2153,11 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
 
 	return 0;
 out:
-	verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s u64=[%#llx, %#llx] "
-		     "s64=[%#llx, %#llx] u32=[%#x, %#x] s32=[%#x, %#x] var_off=(%#llx, %#llx)",
-		     ctx, msg, reg->umin_value, reg->umax_value,
-		     reg->smin_value, reg->smax_value,
-		     reg->u32_min_value, reg->u32_max_value,
-		     reg->s32_min_value, reg->s32_max_value,
+	verifier_bug(env, "REG INVARIANTS VIOLATION (%s): %s r64={.base=%#llx, .size=%#llx} "
+		     "r32={.base=%#x, .size=%#x} var_off=(%#llx, %#llx)",
+		     ctx, msg,
+		     reg->r64.base, reg->r64.size,
+		     reg->r32.base, reg->r32.size,
 		     reg->var_off.value, reg->var_off.mask);
 	if (env->test_reg_invariants)
 		return -EFAULT;
@@ -2432,30 +2165,6 @@ static int reg_bounds_sanity_check(struct bpf_verifier_env *env,
 	return 0;
 }
 
-static bool __reg32_bound_s64(s32 a)
-{
-	return a >= 0 && a <= S32_MAX;
-}
-
-static void __reg_assign_32_into_64(struct bpf_reg_state *reg)
-{
-	reg->umin_value = reg->u32_min_value;
-	reg->umax_value = reg->u32_max_value;
-
-	/* Attempt to pull 32-bit signed bounds into 64-bit bounds but must
-	 * be positive otherwise set to worse case bounds and refine later
-	 * from tnum.
-	 */
-	if (__reg32_bound_s64(reg->s32_min_value) &&
-	    __reg32_bound_s64(reg->s32_max_value)) {
-		reg->smin_value = reg->s32_min_value;
-		reg->smax_value = reg->s32_max_value;
-	} else {
-		reg->smin_value = 0;
-		reg->smax_value = U32_MAX;
-	}
-}
-
 /* Mark a register as having a completely unknown (scalar) value. */
 void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg)
 {
@@ -2497,11 +2206,12 @@ static int __mark_reg_s32_range(struct bpf_verifier_env *env,
 {
 	struct bpf_reg_state *reg = regs + regno;
 
-	reg->s32_min_value = max_t(s32, reg->s32_min_value, s32_min);
-	reg->s32_max_value = min_t(s32, reg->s32_max_value, s32_max);
-
-	reg->smin_value = max_t(s64, reg->smin_value, s32_min);
-	reg->smax_value = min_t(s64, reg->smax_value, s32_max);
+	reg_set_srange32(reg,
+			 max_t(s32, reg_s32_min(reg), s32_min),
+			 min_t(s32, reg_s32_max(reg), s32_max));
+	reg_set_srange64(reg,
+			 max_t(s64, reg_smin(reg), s32_min),
+			 min_t(s64, reg_smax(reg), s32_max));
 
 	reg_bounds_sync(reg);
 
@@ -3492,11 +3202,6 @@ static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
 	return __check_reg_arg(env, state->regs, regno, t);
 }
 
-static int insn_stack_access_flags(int frameno, int spi)
-{
-	return INSN_F_STACK_ACCESS | (spi << INSN_F_SPI_SHIFT) | frameno;
-}
-
 static void mark_indirect_target(struct bpf_verifier_env *env, int idx)
 {
 	env->insn_aux_data[idx].indirect_target = true;
@@ -3734,12 +3439,6 @@ static void assign_scalar_id_before_mov(struct bpf_verifier_env *env,
 		src_reg->id = ++env->id_gen;
 }
 
-/* Copy src state preserving dst->parent and dst->live fields */
-static void copy_register_state(struct bpf_reg_state *dst, const struct bpf_reg_state *src)
-{
-	*dst = *src;
-}
-
 static void save_register_state(struct bpf_verifier_env *env,
 				struct bpf_func_state *state,
 				int spi, struct bpf_reg_state *reg,
@@ -3747,7 +3446,7 @@ static void save_register_state(struct bpf_verifier_env *env,
 {
 	int i;
 
-	copy_register_state(&state->stack[spi].spilled_ptr, reg);
+	state->stack[spi].spilled_ptr = *reg;
 
 	for (i = BPF_REG_SIZE; i > BPF_REG_SIZE - size; i--)
 		state->stack[spi].slot_type[i - 1] = STACK_SPILL;
@@ -3764,7 +3463,7 @@ static bool is_bpf_st_mem(struct bpf_insn *insn)
 
 static int get_reg_width(struct bpf_reg_state *reg)
 {
-	return fls64(reg->umax_value);
+	return fls64(reg_umax(reg));
 }
 
 /* See comment for mark_fastcall_pattern_for_call() */
@@ -3817,7 +3516,8 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
 	struct bpf_reg_state *reg = NULL;
-	int insn_flags = insn_stack_access_flags(state->frameno, spi);
+	int insn_flags = INSN_F_STACK_ACCESS;
+	int hist_spi = spi, hist_frame = state->frameno;
 
 	/* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
 	 * so it's aligned access and [off, off + size) are within stack limits
@@ -3913,11 +3613,12 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 	}
 
 	if (insn_flags)
-		return bpf_push_jmp_history(env, env->cur_state, insn_flags, 0);
+		return bpf_push_jmp_history(env, env->cur_state, insn_flags,
+					    hist_spi, hist_frame, 0);
 	return 0;
 }
 
-/* Write the stack: 'stack[ptr_regno + off] = value_regno'. 'ptr_regno' is
+/* Write the stack: 'stack[ptr_reg + off] = value_regno'. 'ptr_reg' is
  * known to contain a variable offset.
  * This function checks whether the write is permitted and conservatively
  * tracks the effects of the write, considering that each stack slot in the
@@ -3938,13 +3639,13 @@ static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
 static int check_stack_write_var_off(struct bpf_verifier_env *env,
 				     /* func where register points to */
 				     struct bpf_func_state *state,
-				     int ptr_regno, int off, int size,
+				     struct bpf_reg_state *ptr_reg, int off, int size,
 				     int value_regno, int insn_idx)
 {
 	struct bpf_func_state *cur; /* state of the current function */
 	int min_off, max_off;
 	int i, err;
-	struct bpf_reg_state *ptr_reg = NULL, *value_reg = NULL;
+	struct bpf_reg_state *value_reg = NULL;
 	struct bpf_insn *insn = &env->prog->insnsi[insn_idx];
 	bool writing_zero = false;
 	/* set if the fact that we're writing a zero is used to let any
@@ -3953,9 +3654,8 @@ static int check_stack_write_var_off(struct bpf_verifier_env *env,
 	bool zero_used = false;
 
 	cur = env->cur_state->frame[env->cur_state->curframe];
-	ptr_reg = &cur->regs[ptr_regno];
-	min_off = ptr_reg->smin_value + off;
-	max_off = ptr_reg->smax_value + off + size;
+	min_off = reg_smin(ptr_reg) + off;
+	max_off = reg_smax(ptr_reg) + off + size;
 	if (value_regno >= 0)
 		value_reg = &cur->regs[value_regno];
 	if ((value_reg && bpf_register_is_null(value_reg)) ||
@@ -4110,7 +3810,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
 	struct bpf_reg_state *reg;
 	u8 *stype, type;
-	int insn_flags = insn_stack_access_flags(reg_state->frameno, spi);
+	int insn_flags = INSN_F_STACK_ACCESS;
+	int hist_spi = spi, hist_frame = reg_state->frameno;
 
 	stype = reg_state->stack[spi].slot_type;
 	reg = &reg_state->stack[spi].spilled_ptr;
@@ -4147,7 +3848,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 					 * with the destination register on fill.
 					 */
 					assign_scalar_id_before_mov(env, reg);
-				copy_register_state(&state->regs[dst_regno], reg);
+				state->regs[dst_regno] = *reg;
 				state->regs[dst_regno].subreg_def = subreg_def;
 
 				/* Break the relation on a narrowing fill.
@@ -4202,7 +3903,7 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 				 * with the destination register on fill.
 				 */
 				assign_scalar_id_before_mov(env, reg);
-			copy_register_state(&state->regs[dst_regno], reg);
+			state->regs[dst_regno] = *reg;
 			/* mark reg as written since spilled pointer state likely
 			 * has its liveness marks cleared by is_state_visited()
 			 * which resets stack/reg liveness for state transitions
@@ -4241,7 +3942,8 @@ static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
 		insn_flags = 0; /* we are not restoring spilled register */
 	}
 	if (insn_flags)
-		return bpf_push_jmp_history(env, env->cur_state, insn_flags, 0);
+		return bpf_push_jmp_history(env, env->cur_state, insn_flags,
+					    hist_spi, hist_frame, 0);
 	return 0;
 }
 
@@ -4250,8 +3952,8 @@ enum bpf_access_src {
 	ACCESS_HELPER = 2,  /* the access is performed by a helper */
 };
 
-static int check_stack_range_initialized(struct bpf_verifier_env *env,
-					 int regno, int off, int access_size,
+static int check_stack_range_initialized(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+					 argno_t argno, int off, int access_size,
 					 bool zero_size_allowed,
 					 enum bpf_access_type type,
 					 struct bpf_call_arg_meta *meta);
@@ -4261,37 +3963,35 @@ static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
 	return cur_regs(env) + regno;
 }
 
-/* Read the stack at 'ptr_regno + off' and put the result into the register
+/* Read the stack at 'reg + off' and put the result into the register
  * 'dst_regno'.
- * 'off' includes the pointer register's fixed offset(i.e. 'ptr_regno.off'),
+ * 'off' includes the pointer register's fixed offset(i.e. 'reg->off'),
  * but not its variable offset.
  * 'size' is assumed to be <= reg size and the access is assumed to be aligned.
  *
  * As opposed to check_stack_read_fixed_off, this function doesn't deal with
  * filling registers (i.e. reads of spilled register cannot be detected when
  * the offset is not fixed). We conservatively mark 'dst_regno' as containing
- * SCALAR_VALUE. That's why we assert that the 'ptr_regno' has a variable
+ * SCALAR_VALUE. That's why we assert that the 'reg' has a variable
  * offset; for a fixed offset check_stack_read_fixed_off should be used
  * instead.
  */
-static int check_stack_read_var_off(struct bpf_verifier_env *env,
-				    int ptr_regno, int off, int size, int dst_regno)
+static int check_stack_read_var_off(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+				    argno_t ptr_argno, int off, int size, int dst_regno)
 {
-	/* The state of the source register. */
-	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
 	struct bpf_func_state *ptr_state = bpf_func(env, reg);
 	int err;
 	int min_off, max_off;
 
 	/* Note that we pass a NULL meta, so raw access will not be permitted.
 	 */
-	err = check_stack_range_initialized(env, ptr_regno, off, size,
+	err = check_stack_range_initialized(env, reg, ptr_argno, off, size,
 					    false, BPF_READ, NULL);
 	if (err)
 		return err;
 
-	min_off = reg->smin_value + off;
-	max_off = reg->smax_value + off;
+	min_off = reg_smin(reg) + off;
+	max_off = reg_smax(reg) + off;
 	mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
 	check_fastcall_stack_contract(env, ptr_state, env->insn_idx, min_off);
 	return 0;
@@ -4307,10 +4007,9 @@ static int check_stack_read_var_off(struct bpf_verifier_env *env,
  * can be -1, meaning that the read value is not going to a register.
  */
 static int check_stack_read(struct bpf_verifier_env *env,
-			    int ptr_regno, int off, int size,
+			    struct bpf_reg_state *reg, argno_t ptr_argno, int off, int size,
 			    int dst_regno)
 {
-	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
 	struct bpf_func_state *state = bpf_func(env, reg);
 	int err;
 	/* Some accesses are only permitted with a static offset. */
@@ -4346,7 +4045,7 @@ static int check_stack_read(struct bpf_verifier_env *env,
 		 * than fixed offset ones. Note that dst_regno >= 0 on this
 		 * branch.
 		 */
-		err = check_stack_read_var_off(env, ptr_regno, off, size,
+		err = check_stack_read_var_off(env, reg, ptr_argno, off, size,
 					       dst_regno);
 	}
 	return err;
@@ -4356,17 +4055,16 @@ static int check_stack_read(struct bpf_verifier_env *env,
 /* check_stack_write dispatches to check_stack_write_fixed_off or
  * check_stack_write_var_off.
  *
- * 'ptr_regno' is the register used as a pointer into the stack.
+ * 'reg' is the register used as a pointer into the stack.
  * 'value_regno' is the register whose value we're writing to the stack. It can
  * be -1, meaning that we're not writing from a register.
  *
  * The caller must ensure that the offset falls within the maximum stack size.
  */
 static int check_stack_write(struct bpf_verifier_env *env,
-			     int ptr_regno, int off, int size,
+			     struct bpf_reg_state *reg, int off, int size,
 			     int value_regno, int insn_idx)
 {
-	struct bpf_reg_state *reg = reg_state(env, ptr_regno);
 	struct bpf_func_state *state = bpf_func(env, reg);
 	int err;
 
@@ -4379,28 +4077,135 @@ static int check_stack_write(struct bpf_verifier_env *env,
 		 * than fixed offset ones.
 		 */
 		err = check_stack_write_var_off(env, state,
-						ptr_regno, off, size,
+						reg, off, size,
 						value_regno, insn_idx);
 	}
 	return err;
 }
 
-static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
+/*
+ * Write a value to the outgoing stack arg area.
+ * off is a negative offset from r11 (e.g. -8 for arg6, -16 for arg7).
+ */
+static int check_stack_arg_write(struct bpf_verifier_env *env, struct bpf_func_state *state,
+				 int off, struct bpf_reg_state *value_reg)
+{
+	int max_stack_arg_regs = MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS;
+	struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
+	int spi = -off / BPF_REG_SIZE - 1;
+	struct bpf_reg_state *arg;
+	int err;
+
+	if (spi >= max_stack_arg_regs) {
+		verbose(env, "stack arg write offset %d exceeds max %d stack args\n",
+			off, max_stack_arg_regs);
+		return -EINVAL;
+	}
+
+	err = grow_stack_arg_slots(env, state, spi + 1);
+	if (err)
+		return err;
+
+	/* Track the max outgoing stack arg slot count. */
+	if (spi + 1 > subprog->max_out_stack_arg_cnt)
+		subprog->max_out_stack_arg_cnt = spi + 1;
+
+	if (value_reg) {
+		state->stack_arg_regs[spi] = *value_reg;
+	} else {
+		/* BPF_ST: store immediate, treat as scalar */
+		arg = &state->stack_arg_regs[spi];
+		arg->type = SCALAR_VALUE;
+		__mark_reg_known(arg, env->prog->insnsi[env->insn_idx].imm);
+	}
+	state->no_stack_arg_load = true;
+	return bpf_push_jmp_history(env, env->cur_state,
+				    INSN_F_STACK_ARG_ACCESS, spi, 0, 0);
+}
+
+/*
+ * Read a value from the incoming stack arg area.
+ * off is a positive offset from r11 (e.g. +8 for arg6, +16 for arg7).
+ */
+static int check_stack_arg_read(struct bpf_verifier_env *env, struct bpf_func_state *state,
+				int off, int dst_regno)
+{
+	struct bpf_subprog_info *subprog = &env->subprog_info[state->subprogno];
+	struct bpf_verifier_state *vstate = env->cur_state;
+	int spi = off / BPF_REG_SIZE - 1;
+	struct bpf_func_state *caller, *cur;
+	struct bpf_reg_state *arg;
+
+	if (state->no_stack_arg_load) {
+		verbose(env, "r11 load must be before any r11 store or call insn\n");
+		return -EINVAL;
+	}
+
+	if (spi + 1 > bpf_in_stack_arg_cnt(subprog)) {
+		verbose(env, "invalid read from stack arg off %d depth %d\n",
+			off, bpf_in_stack_arg_cnt(subprog) * BPF_REG_SIZE);
+		return -EACCES;
+	}
+
+	caller = vstate->frame[vstate->curframe - 1];
+	arg = &caller->stack_arg_regs[spi];
+	cur = vstate->frame[vstate->curframe];
+	cur->regs[dst_regno] = *arg;
+	return bpf_push_jmp_history(env, env->cur_state,
+				    INSN_F_STACK_ARG_ACCESS, spi, 0, 0);
+}
+
+static int mark_stack_arg_precision(struct bpf_verifier_env *env, int arg_idx)
+{
+	struct bpf_func_state *caller = cur_func(env);
+	int spi = arg_idx - MAX_BPF_FUNC_REG_ARGS;
+
+	bt_set_frame_stack_arg_slot(&env->bt, caller->frameno, spi);
+	return mark_chain_precision_batch(env, env->cur_state);
+}
+
+static int check_outgoing_stack_args(struct bpf_verifier_env *env, struct bpf_func_state *caller,
+				     int nargs)
+{
+	int i, spi;
+
+	for (i = MAX_BPF_FUNC_REG_ARGS; i < nargs; i++) {
+		spi = i - MAX_BPF_FUNC_REG_ARGS;
+		if (spi >= caller->out_stack_arg_cnt ||
+		    caller->stack_arg_regs[spi].type == NOT_INIT) {
+			verbose(env, "callee expects %d args, stack arg%d is not initialized\n",
+				nargs, spi + 1);
+			return -EFAULT;
+		}
+	}
+
+	return 0;
+}
+
+static struct bpf_reg_state *get_func_arg_reg(struct bpf_func_state *caller,
+					      struct bpf_reg_state *regs, int arg)
+{
+	if (arg < MAX_BPF_FUNC_REG_ARGS)
+		return &regs[arg + 1];
+
+	return &caller->stack_arg_regs[arg - MAX_BPF_FUNC_REG_ARGS];
+}
+
+static int check_map_access_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
 				 int off, int size, enum bpf_access_type type)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	struct bpf_map *map = reg->map_ptr;
 	u32 cap = bpf_map_flags_to_cap(map);
 
 	if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) {
 		verbose(env, "write into map forbidden, value_size=%d off=%lld size=%d\n",
-			map->value_size, reg->smin_value + off, size);
+			map->value_size, reg_smin(reg) + off, size);
 		return -EACCES;
 	}
 
 	if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) {
 		verbose(env, "read from map forbidden, value_size=%d off=%lld size=%d\n",
-			map->value_size, reg->smin_value + off, size);
+			map->value_size, reg_smin(reg) + off, size);
 		return -EACCES;
 	}
 
@@ -4408,17 +4213,15 @@ static int check_map_access_type(struct bpf_verifier_env *env, u32 regno,
 }
 
 /* check read/write into memory region (e.g., map value, ringbuf sample, etc) */
-static int __check_mem_access(struct bpf_verifier_env *env, int regno,
+static int __check_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 			      int off, int size, u32 mem_size,
 			      bool zero_size_allowed)
 {
 	bool size_ok = size > 0 || (size == 0 && zero_size_allowed);
-	struct bpf_reg_state *reg;
 
 	if (off >= 0 && size_ok && (u64)off + size <= mem_size)
 		return 0;
 
-	reg = &cur_regs(env)[regno];
 	switch (reg->type) {
 	case PTR_TO_MAP_KEY:
 		verbose(env, "invalid access to map key, key_size=%d off=%d size=%d\n",
@@ -4431,8 +4234,8 @@ static int __check_mem_access(struct bpf_verifier_env *env, int regno,
 	case PTR_TO_PACKET:
 	case PTR_TO_PACKET_META:
 	case PTR_TO_PACKET_END:
-		verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
-			off, size, regno, reg->id, off, mem_size);
+		verbose(env, "invalid access to packet, off=%d size=%d, %s(id=%d,off=%d,r=%d)\n",
+			off, size, reg_arg_name(env, argno), reg->id, off, mem_size);
 		break;
 	case PTR_TO_CTX:
 		verbose(env, "invalid access to context, ctx_size=%d off=%d size=%d\n",
@@ -4448,13 +4251,10 @@ static int __check_mem_access(struct bpf_verifier_env *env, int regno,
 }
 
 /* check read/write into a memory region with possible variable offset */
-static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
+static int check_mem_region_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 				   int off, int size, u32 mem_size,
 				   bool zero_size_allowed)
 {
-	struct bpf_verifier_state *vstate = env->cur_state;
-	struct bpf_func_state *state = vstate->frame[vstate->curframe];
-	struct bpf_reg_state *reg = &state->regs[regno];
 	int err;
 
 	/* We may have adjusted the register pointing to memory region, so we
@@ -4467,36 +4267,36 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
 	 * index'es we need to make sure that whatever we use
 	 * will have a set floor within our range.
 	 */
-	if (reg->smin_value < 0 &&
-	    (reg->smin_value == S64_MIN ||
-	     (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
-	      reg->smin_value + off < 0)) {
-		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
-			regno);
+	if (reg_smin(reg) < 0 &&
+	    (reg_smin(reg) == S64_MIN ||
+	     (off + reg_smin(reg) != (s64)(s32)(off + reg_smin(reg))) ||
+	      reg_smin(reg) + off < 0)) {
+		verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n",
+			reg_arg_name(env, argno));
 		return -EACCES;
 	}
-	err = __check_mem_access(env, regno, reg->smin_value + off, size,
+	err = __check_mem_access(env, reg, argno, reg_smin(reg) + off, size,
 				 mem_size, zero_size_allowed);
 	if (err) {
-		verbose(env, "R%d min value is outside of the allowed memory range\n",
-			regno);
+		verbose(env, "%s min value is outside of the allowed memory range\n",
+			reg_arg_name(env, argno));
 		return err;
 	}
 
 	/* If we haven't set a max value then we need to bail since we can't be
 	 * sure we won't do bad things.
-	 * If reg->umax_value + off could overflow, treat that as unbounded too.
+	 * If reg_umax(reg) + off could overflow, treat that as unbounded too.
 	 */
-	if (reg->umax_value >= BPF_MAX_VAR_OFF) {
-		verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n",
-			regno);
+	if (reg_umax(reg) >= BPF_MAX_VAR_OFF) {
+		verbose(env, "%s unbounded memory access, make sure to bounds check any such access\n",
+			reg_arg_name(env, argno));
 		return -EACCES;
 	}
-	err = __check_mem_access(env, regno, reg->umax_value + off, size,
+	err = __check_mem_access(env, reg, argno, reg_umax(reg) + off, size,
 				 mem_size, zero_size_allowed);
 	if (err) {
-		verbose(env, "R%d max value is outside of the allowed memory range\n",
-			regno);
+		verbose(env, "%s max value is outside of the allowed memory range\n",
+			reg_arg_name(env, argno));
 		return err;
 	}
 
@@ -4504,7 +4304,7 @@ static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno,
 }
 
 static int __check_ptr_off_reg(struct bpf_verifier_env *env,
-			       const struct bpf_reg_state *reg, int regno,
+			       const struct bpf_reg_state *reg, argno_t argno,
 			       bool fixed_off_ok)
 {
 	/* Access to this pointer-typed register or passing it to a helper
@@ -4520,15 +4320,15 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env,
 		return -EACCES;
 	}
 
-	if (reg->smin_value < 0) {
-		verbose(env, "negative offset %s ptr R%d off=%lld disallowed\n",
-			reg_type_str(env, reg->type), regno, reg->var_off.value);
+	if (reg_smin(reg) < 0) {
+		verbose(env, "negative offset %s ptr %s off=%lld disallowed\n",
+			reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value);
 		return -EACCES;
 	}
 
 	if (!fixed_off_ok && reg->var_off.value != 0) {
-		verbose(env, "dereference of modified %s ptr R%d off=%lld disallowed\n",
-			reg_type_str(env, reg->type), regno, reg->var_off.value);
+		verbose(env, "dereference of modified %s ptr %s off=%lld disallowed\n",
+			reg_type_str(env, reg->type), reg_arg_name(env, argno), reg->var_off.value);
 		return -EACCES;
 	}
 
@@ -4538,7 +4338,7 @@ static int __check_ptr_off_reg(struct bpf_verifier_env *env,
 static int check_ptr_off_reg(struct bpf_verifier_env *env,
 		             const struct bpf_reg_state *reg, int regno)
 {
-	return __check_ptr_off_reg(env, reg, regno, false);
+	return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false);
 }
 
 static int map_kptr_match_type(struct bpf_verifier_env *env,
@@ -4576,7 +4376,7 @@ static int map_kptr_match_type(struct bpf_verifier_env *env,
 	 * Since ref_ptr cannot be accessed directly by BPF insns, check for
 	 * reg->ref_obj_id is not needed here.
 	 */
-	if (__check_ptr_off_reg(env, reg, regno, true))
+	if (__check_ptr_off_reg(env, reg, argno_from_reg(regno), true))
 		return -EACCES;
 
 	/* A full type match is needed, as BTF can be vmlinux, module or prog BTF, and
@@ -4719,7 +4519,7 @@ static int mark_uptr_ld_reg(struct bpf_verifier_env *env, u32 regno,
 	return 0;
 }
 
-static int check_map_kptr_access(struct bpf_verifier_env *env, u32 regno,
+static int check_map_kptr_access(struct bpf_verifier_env *env,
 				 int value_regno, int insn_idx,
 				 struct btf_field *kptr_field)
 {
@@ -4796,19 +4596,16 @@ static u32 map_mem_size(const struct bpf_map *map)
 }
 
 /* check read/write into a map element with possible variable offset */
-static int check_map_access(struct bpf_verifier_env *env, u32 regno,
+static int check_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 			    int off, int size, bool zero_size_allowed,
 			    enum bpf_access_src src)
 {
-	struct bpf_verifier_state *vstate = env->cur_state;
-	struct bpf_func_state *state = vstate->frame[vstate->curframe];
-	struct bpf_reg_state *reg = &state->regs[regno];
 	struct bpf_map *map = reg->map_ptr;
 	u32 mem_size = map_mem_size(map);
 	struct btf_record *rec;
 	int err, i;
 
-	err = check_mem_region_access(env, regno, off, size, mem_size, zero_size_allowed);
+	err = check_mem_region_access(env, reg, argno, off, size, mem_size, zero_size_allowed);
 	if (err)
 		return err;
 
@@ -4823,8 +4620,8 @@ static int check_map_access(struct bpf_verifier_env *env, u32 regno,
 		 * this program. To check that [x1, x2) overlaps with [y1, y2),
 		 * it is sufficient to check x1 < y2 && y1 < x2.
 		 */
-		if (reg->smin_value + off < p + field->size &&
-		    p < reg->umax_value + off + size) {
+		if (reg_smin(reg) + off < p + field->size &&
+		    p < reg_umax(reg) + off + size) {
 			switch (field->type) {
 			case BPF_KPTR_UNREF:
 			case BPF_KPTR_REF:
@@ -4904,30 +4701,29 @@ static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
 	}
 }
 
-static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
+static int check_packet_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off,
 			       int size, bool zero_size_allowed)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	int err;
 
 	if (reg->range < 0) {
-		verbose(env, "R%d offset is outside of the packet\n", regno);
+		verbose(env, "%s offset is outside of the packet\n", reg_arg_name(env, argno));
 		return -EINVAL;
 	}
 
-	err = check_mem_region_access(env, regno, off, size, reg->range, zero_size_allowed);
+	err = check_mem_region_access(env, reg, argno, off, size, reg->range, zero_size_allowed);
 	if (err)
 		return err;
 
 	/* __check_mem_access has made sure "off + size - 1" is within u16.
-	 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
+	 * reg_umax(reg) can't be bigger than MAX_PACKET_OFF which is 0xffff,
 	 * otherwise find_good_pkt_pointers would have refused to set range info
 	 * that __check_mem_access would have rejected this pkt access.
-	 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
+	 * Therefore, "off + reg_umax(reg) + size - 1" won't overflow u32.
 	 */
 	env->prog->aux->max_pkt_offset =
 		max_t(u32, env->prog->aux->max_pkt_offset,
-		      off + reg->umax_value + size - 1);
+		      off + reg_umax(reg) + size - 1);
 
 	return 0;
 }
@@ -4970,7 +4766,7 @@ static int __check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int of
 	return -EACCES;
 }
 
-static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
+static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno,
 			    int off, int access_size, enum bpf_access_type t,
 			    struct bpf_insn_access_aux *info)
 {
@@ -4980,17 +4776,15 @@ static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 	 */
 	bool var_off_ok = is_var_ctx_off_allowed(env->prog);
 	bool fixed_off_ok = !env->ops->convert_ctx_access;
-	struct bpf_reg_state *regs = cur_regs(env);
-	struct bpf_reg_state *reg = regs + regno;
 	int err;
 
 	if (var_off_ok)
-		err = check_mem_region_access(env, regno, off, access_size, U16_MAX, false);
+		err = check_mem_region_access(env, reg, argno, off, access_size, U16_MAX, false);
 	else
-		err = __check_ptr_off_reg(env, reg, regno, fixed_off_ok);
+		err = __check_ptr_off_reg(env, reg, argno, fixed_off_ok);
 	if (err)
 		return err;
-	off += reg->umax_value;
+	off += reg_umax(reg);
 
 	err = __check_ctx_access(env, insn_idx, off, access_size, t, info);
 	if (err)
@@ -5011,16 +4805,15 @@ static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
 }
 
 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
-			     u32 regno, int off, int size,
+			     struct bpf_reg_state *reg, argno_t argno, int off, int size,
 			     enum bpf_access_type t)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	struct bpf_insn_access_aux info = {};
 	bool valid;
 
-	if (reg->smin_value < 0) {
-		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
-			regno);
+	if (reg_smin(reg) < 0) {
+		verbose(env, "%s min value is negative, either use unsigned index or do a if (index >=0) check.\n",
+			reg_arg_name(env, argno));
 		return -EACCES;
 	}
 
@@ -5048,8 +4841,8 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
 		return 0;
 	}
 
-	verbose(env, "R%d invalid %s access off=%d size=%d\n",
-		regno, reg_type_str(env, reg->type), off, size);
+	verbose(env, "%s invalid %s access off=%d size=%d\n",
+		reg_arg_name(env, argno), reg_type_str(env, reg->type), off, size);
 
 	return -EACCES;
 }
@@ -5369,7 +5162,10 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 	}
 
 	subprog_depth = round_up_stack_depth(env, subprog[idx].stack_depth);
-	if (priv_stack_supported) {
+	if (IS_ENABLED(CONFIG_X86_64) && subprog[idx].stack_arg_cnt) {
+		/* x86-64 uses R9 for both private stack frame pointer and arg6. */
+		subprog[idx].priv_stack_mode = NO_PRIV_STACK;
+	} else if (priv_stack_supported) {
 		/* Request private stack support only if the subprog stack
 		 * depth is no less than BPF_PRIV_STACK_MIN_SIZE. This is to
 		 * avoid jit penalty if the stack usage is small.
@@ -5380,6 +5176,8 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 	}
 
 	if (subprog[idx].priv_stack_mode == PRIV_STACK_ADAPTIVE) {
+		if (subprog_depth > env->max_stack_depth)
+			env->max_stack_depth = subprog_depth;
 		if (subprog_depth > MAX_BPF_STACK) {
 			verbose(env, "stack size of subprog %d is %d. Too large\n",
 				idx, subprog_depth);
@@ -5387,6 +5185,8 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 		}
 	} else {
 		depth += subprog_depth;
+		if (depth > env->max_stack_depth)
+			env->max_stack_depth = depth;
 		if (depth > MAX_BPF_STACK) {
 			total = 0;
 			for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller)
@@ -5405,7 +5205,7 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 		if (bpf_pseudo_kfunc_call(insn + i) && !insn[i].off) {
 			bool err = false;
 
-			if (!is_bpf_throw_kfunc(insn + i))
+			if (!bpf_is_throw_kfunc(insn + i))
 				continue;
 			for (tmp = idx; tmp >= 0 && !err; tmp = dinfo[tmp].caller) {
 				if (subprog[tmp].is_cb) {
@@ -5473,14 +5273,23 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 	 * this info will be utilized by JIT so that we will be preserving the
 	 * tail call counter throughout bpf2bpf calls combined with tailcalls
 	 */
-	if (tail_call_reachable)
+	if (tail_call_reachable) {
 		for (tmp = idx; tmp >= 0; tmp = dinfo[tmp].caller) {
 			if (subprog[tmp].is_exception_cb) {
 				verbose(env, "cannot tail call within exception cb\n");
 				return -EINVAL;
 			}
+			if (subprog[tmp].stack_arg_cnt) {
+				verbose(env, "tail_calls are not allowed in programs with stack args\n");
+				return -EINVAL;
+			}
 			subprog[tmp].tail_call_reachable = true;
 		}
+	} else if (!idx && subprog[0].has_tail_call && subprog[0].stack_arg_cnt) {
+		verbose(env, "tail_calls are not allowed in programs with stack args\n");
+		return -EINVAL;
+	}
+
 	if (subprog[0].tail_call_reachable)
 		env->prog->aux->tail_call_reachable = true;
 
@@ -5499,6 +5308,9 @@ static int check_max_stack_depth_subprog(struct bpf_verifier_env *env, int idx,
 	frame = dinfo[idx].frame;
 	i = dinfo[idx].ret_insn;
 
+	/* reset tail_call_reachable to the parent's actual state */
+	tail_call_reachable = subprog[idx].tail_call_reachable;
+
 	goto continue_func;
 }
 
@@ -5559,12 +5371,12 @@ static int check_max_stack_depth(struct bpf_verifier_env *env)
 static int __check_buffer_access(struct bpf_verifier_env *env,
 				 const char *buf_info,
 				 const struct bpf_reg_state *reg,
-				 int regno, int off, int size)
+				 argno_t argno, int off, int size)
 {
 	if (off < 0) {
 		verbose(env,
-			"R%d invalid %s buffer access: off=%d, size=%d\n",
-			regno, buf_info, off, size);
+			"%s invalid %s buffer access: off=%d, size=%d\n",
+			reg_arg_name(env, argno), buf_info, off, size);
 		return -EACCES;
 	}
 	if (!tnum_is_const(reg->var_off)) {
@@ -5572,8 +5384,8 @@ static int __check_buffer_access(struct bpf_verifier_env *env,
 
 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 		verbose(env,
-			"R%d invalid variable buffer offset: off=%d, var_off=%s\n",
-			regno, off, tn_buf);
+			"%s invalid variable buffer offset: off=%d, var_off=%s\n",
+			reg_arg_name(env, argno), off, tn_buf);
 		return -EACCES;
 	}
 
@@ -5582,11 +5394,11 @@ static int __check_buffer_access(struct bpf_verifier_env *env,
 
 static int check_tp_buffer_access(struct bpf_verifier_env *env,
 				  const struct bpf_reg_state *reg,
-				  int regno, int off, int size)
+				  argno_t argno, int off, int size)
 {
 	int err;
 
-	err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
+	err = __check_buffer_access(env, "tracepoint", reg, argno, off, size);
 	if (err)
 		return err;
 
@@ -5598,14 +5410,14 @@ static int check_tp_buffer_access(struct bpf_verifier_env *env,
 
 static int check_buffer_access(struct bpf_verifier_env *env,
 			       const struct bpf_reg_state *reg,
-			       int regno, int off, int size,
+			       argno_t argno, int off, int size,
 			       bool zero_size_allowed,
 			       u32 *max_access)
 {
 	const char *buf_info = type_is_rdonly_mem(reg->type) ? "rdonly" : "rdwr";
 	int err;
 
-	err = __check_buffer_access(env, buf_info, reg, regno, off, size);
+	err = __check_buffer_access(env, buf_info, reg, argno, off, size);
 	if (err)
 		return err;
 
@@ -5618,7 +5430,7 @@ static int check_buffer_access(struct bpf_verifier_env *env,
 static void zext_32_to_64(struct bpf_reg_state *reg)
 {
 	reg->var_off = tnum_subreg(reg->var_off);
-	__reg_assign_32_into_64(reg);
+	reg_set_urange64(reg, reg_u32_min(reg), reg_u32_max(reg));
 }
 
 /* truncate register to smaller size (in bytes)
@@ -5633,15 +5445,10 @@ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
 
 	/* fix arithmetic bounds */
 	mask = ((u64)1 << (size * 8)) - 1;
-	if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
-		reg->umin_value &= mask;
-		reg->umax_value &= mask;
-	} else {
-		reg->umin_value = 0;
-		reg->umax_value = mask;
-	}
-	reg->smin_value = reg->umin_value;
-	reg->smax_value = reg->umax_value;
+	if ((reg_umin(reg) & ~mask) == (reg_umax(reg) & ~mask))
+		reg_set_urange64(reg, reg_umin(reg) & mask, reg_umax(reg) & mask);
+	else
+		reg_set_urange64(reg, 0, mask);
 
 	/* If size is smaller than 32bit register the 32bit register
 	 * values are also truncated so we push 64-bit bounds into
@@ -5656,19 +5463,16 @@ static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
 static void set_sext64_default_val(struct bpf_reg_state *reg, int size)
 {
 	if (size == 1) {
-		reg->smin_value = reg->s32_min_value = S8_MIN;
-		reg->smax_value = reg->s32_max_value = S8_MAX;
+		reg_set_srange64(reg, S8_MIN, S8_MAX);
+		reg_set_srange32(reg, S8_MIN, S8_MAX);
 	} else if (size == 2) {
-		reg->smin_value = reg->s32_min_value = S16_MIN;
-		reg->smax_value = reg->s32_max_value = S16_MAX;
+		reg_set_srange64(reg, S16_MIN, S16_MAX);
+		reg_set_srange32(reg, S16_MIN, S16_MAX);
 	} else {
 		/* size == 4 */
-		reg->smin_value = reg->s32_min_value = S32_MIN;
-		reg->smax_value = reg->s32_max_value = S32_MAX;
+		reg_set_srange64(reg, S32_MIN, S32_MAX);
+		reg_set_srange32(reg, S32_MIN, S32_MAX);
 	}
-	reg->umin_value = reg->u32_min_value = 0;
-	reg->umax_value = U64_MAX;
-	reg->u32_max_value = U32_MAX;
 	reg->var_off = tnum_unknown;
 }
 
@@ -5689,29 +5493,27 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size)
 			reg->var_off = tnum_const((s32)u64_cval);
 
 		u64_cval = reg->var_off.value;
-		reg->smax_value = reg->smin_value = u64_cval;
-		reg->umax_value = reg->umin_value = u64_cval;
-		reg->s32_max_value = reg->s32_min_value = u64_cval;
-		reg->u32_max_value = reg->u32_min_value = u64_cval;
+		reg->r64 = cnum64_from_urange(u64_cval, u64_cval);
+		reg->r32 = cnum32_from_urange((u32)u64_cval, (u32)u64_cval);
 		return;
 	}
 
-	top_smax_value = ((u64)reg->smax_value >> num_bits) << num_bits;
-	top_smin_value = ((u64)reg->smin_value >> num_bits) << num_bits;
+	top_smax_value = ((u64)reg_smax(reg) >> num_bits) << num_bits;
+	top_smin_value = ((u64)reg_smin(reg) >> num_bits) << num_bits;
 
 	if (top_smax_value != top_smin_value)
 		goto out;
 
 	/* find the s64_min and s64_min after sign extension */
 	if (size == 1) {
-		init_s64_max = (s8)reg->smax_value;
-		init_s64_min = (s8)reg->smin_value;
+		init_s64_max = (s8)reg_smax(reg);
+		init_s64_min = (s8)reg_smin(reg);
 	} else if (size == 2) {
-		init_s64_max = (s16)reg->smax_value;
-		init_s64_min = (s16)reg->smin_value;
+		init_s64_max = (s16)reg_smax(reg);
+		init_s64_min = (s16)reg_smin(reg);
 	} else {
-		init_s64_max = (s32)reg->smax_value;
-		init_s64_min = (s32)reg->smin_value;
+		init_s64_max = (s32)reg_smax(reg);
+		init_s64_min = (s32)reg_smin(reg);
 	}
 
 	s64_max = max(init_s64_max, init_s64_min);
@@ -5719,10 +5521,8 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size)
 
 	/* both of s64_max/s64_min positive or negative */
 	if ((s64_max >= 0) == (s64_min >= 0)) {
-		reg->s32_min_value = reg->smin_value = s64_min;
-		reg->s32_max_value = reg->smax_value = s64_max;
-		reg->u32_min_value = reg->umin_value = s64_min;
-		reg->u32_max_value = reg->umax_value = s64_max;
+		reg_set_srange64(reg, s64_min, s64_max);
+		reg_set_srange32(reg, s64_min, s64_max);
 		reg->var_off = tnum_range(s64_min, s64_max);
 		return;
 	}
@@ -5733,16 +5533,11 @@ static void coerce_reg_to_size_sx(struct bpf_reg_state *reg, int size)
 
 static void set_sext32_default_val(struct bpf_reg_state *reg, int size)
 {
-	if (size == 1) {
-		reg->s32_min_value = S8_MIN;
-		reg->s32_max_value = S8_MAX;
-	} else {
+	if (size == 1)
+		reg_set_srange32(reg, S8_MIN, S8_MAX);
+	else
 		/* size == 2 */
-		reg->s32_min_value = S16_MIN;
-		reg->s32_max_value = S16_MAX;
-	}
-	reg->u32_min_value = 0;
-	reg->u32_max_value = U32_MAX;
+		reg_set_srange32(reg, S16_MIN, S16_MAX);
 	reg->var_off = tnum_subreg(tnum_unknown);
 }
 
@@ -5760,34 +5555,30 @@ static void coerce_subreg_to_size_sx(struct bpf_reg_state *reg, int size)
 			reg->var_off = tnum_const((s16)u32_val);
 
 		u32_val = reg->var_off.value;
-		reg->s32_min_value = reg->s32_max_value = u32_val;
-		reg->u32_min_value = reg->u32_max_value = u32_val;
+		reg_set_srange32(reg, u32_val, u32_val);
 		return;
 	}
 
-	top_smax_value = ((u32)reg->s32_max_value >> num_bits) << num_bits;
-	top_smin_value = ((u32)reg->s32_min_value >> num_bits) << num_bits;
+	top_smax_value = ((u32)reg_s32_max(reg) >> num_bits) << num_bits;
+	top_smin_value = ((u32)reg_s32_min(reg) >> num_bits) << num_bits;
 
 	if (top_smax_value != top_smin_value)
 		goto out;
 
 	/* find the s32_min and s32_min after sign extension */
 	if (size == 1) {
-		init_s32_max = (s8)reg->s32_max_value;
-		init_s32_min = (s8)reg->s32_min_value;
+		init_s32_max = (s8)reg_s32_max(reg);
+		init_s32_min = (s8)reg_s32_min(reg);
 	} else {
 		/* size == 2 */
-		init_s32_max = (s16)reg->s32_max_value;
-		init_s32_min = (s16)reg->s32_min_value;
+		init_s32_max = (s16)reg_s32_max(reg);
+		init_s32_min = (s16)reg_s32_min(reg);
 	}
 	s32_max = max(init_s32_max, init_s32_min);
 	s32_min = min(init_s32_max, init_s32_min);
 
 	if ((s32_min >= 0) == (s32_max >= 0)) {
-		reg->s32_min_value = s32_min;
-		reg->s32_max_value = s32_max;
-		reg->u32_min_value = (u32)s32_min;
-		reg->u32_max_value = (u32)s32_max;
+		reg_set_srange32(reg, s32_min, s32_max);
 		reg->var_off = tnum_subreg(tnum_range(s32_min, s32_max));
 		return;
 	}
@@ -5977,12 +5768,11 @@ static bool type_is_trusted_or_null(struct bpf_verifier_env *env,
 }
 
 static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
-				   struct bpf_reg_state *regs,
-				   int regno, int off, int size,
+				   struct bpf_reg_state *regs, struct bpf_reg_state *reg,
+				   argno_t argno, int off, int size,
 				   enum bpf_access_type atype,
 				   int value_regno)
 {
-	struct bpf_reg_state *reg = regs + regno;
 	const struct btf_type *t = btf_type_by_id(reg->btf, reg->btf_id);
 	const char *tname = btf_name_by_offset(reg->btf, t->name_off);
 	const char *field_name = NULL;
@@ -6008,8 +5798,8 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
 
 		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
 		verbose(env,
-			"R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
-			regno, tname, off, tn_buf);
+			"%s is ptr_%s invalid variable offset: off=%d, var_off=%s\n",
+			reg_arg_name(env, argno), tname, off, tn_buf);
 		return -EACCES;
 	}
 
@@ -6017,22 +5807,22 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
 
 	if (off < 0) {
 		verbose(env,
-			"R%d is ptr_%s invalid negative access: off=%d\n",
-			regno, tname, off);
+			"%s is ptr_%s invalid negative access: off=%d\n",
+			reg_arg_name(env, argno), tname, off);
 		return -EACCES;
 	}
 
 	if (reg->type & MEM_USER) {
 		verbose(env,
-			"R%d is ptr_%s access user memory: off=%d\n",
-			regno, tname, off);
+			"%s is ptr_%s access user memory: off=%d\n",
+			reg_arg_name(env, argno), tname, off);
 		return -EACCES;
 	}
 
 	if (reg->type & MEM_PERCPU) {
 		verbose(env,
-			"R%d is ptr_%s access percpu memory: off=%d\n",
-			regno, tname, off);
+			"%s is ptr_%s access percpu memory: off=%d\n",
+			reg_arg_name(env, argno), tname, off);
 		return -EACCES;
 	}
 
@@ -6134,12 +5924,11 @@ static int check_ptr_to_btf_access(struct bpf_verifier_env *env,
 }
 
 static int check_ptr_to_map_access(struct bpf_verifier_env *env,
-				   struct bpf_reg_state *regs,
-				   int regno, int off, int size,
+				   struct bpf_reg_state *regs, struct bpf_reg_state *reg,
+				   argno_t argno, int off, int size,
 				   enum bpf_access_type atype,
 				   int value_regno)
 {
-	struct bpf_reg_state *reg = regs + regno;
 	struct bpf_map *map = reg->map_ptr;
 	struct bpf_reg_state map_reg;
 	enum bpf_type_flag flag = 0;
@@ -6170,8 +5959,8 @@ static int check_ptr_to_map_access(struct bpf_verifier_env *env,
 	}
 
 	if (off < 0) {
-		verbose(env, "R%d is %s invalid negative access: off=%d\n",
-			regno, tname, off);
+		verbose(env, "%s is %s invalid negative access: off=%d\n",
+			reg_arg_name(env, argno), tname, off);
 		return -EACCES;
 	}
 
@@ -6228,11 +6017,10 @@ static int check_stack_slot_within_bounds(struct bpf_verifier_env *env,
  * 'off' includes `regno->offset`, but not its dynamic part (if any).
  */
 static int check_stack_access_within_bounds(
-		struct bpf_verifier_env *env,
-		int regno, int off, int access_size,
+		struct bpf_verifier_env *env, struct bpf_reg_state *reg,
+		argno_t argno, int off, int access_size,
 		enum bpf_access_type type)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	struct bpf_func_state *state = bpf_func(env, reg);
 	s64 min_off, max_off;
 	int err;
@@ -6247,14 +6035,14 @@ static int check_stack_access_within_bounds(
 		min_off = (s64)reg->var_off.value + off;
 		max_off = min_off + access_size;
 	} else {
-		if (reg->smax_value >= BPF_MAX_VAR_OFF ||
-		    reg->smin_value <= -BPF_MAX_VAR_OFF) {
-			verbose(env, "invalid unbounded variable-offset%s stack R%d\n",
-				err_extra, regno);
+		if (reg_smax(reg) >= BPF_MAX_VAR_OFF ||
+		    reg_smin(reg) <= -BPF_MAX_VAR_OFF) {
+			verbose(env, "invalid unbounded variable-offset%s stack %s\n",
+				err_extra, reg_arg_name(env, argno));
 			return -EACCES;
 		}
-		min_off = reg->smin_value + off;
-		max_off = reg->smax_value + off + access_size;
+		min_off = reg_smin(reg) + off;
+		max_off = reg_smax(reg) + off + access_size;
 	}
 
 	err = check_stack_slot_within_bounds(env, min_off, state, type);
@@ -6268,14 +6056,14 @@ static int check_stack_access_within_bounds(
 
 	if (err) {
 		if (tnum_is_const(reg->var_off)) {
-			verbose(env, "invalid%s stack R%d off=%lld size=%d\n",
-				err_extra, regno, min_off, access_size);
+			verbose(env, "invalid%s stack %s off=%lld size=%d\n",
+				err_extra, reg_arg_name(env, argno), min_off, access_size);
 		} else {
 			char tn_buf[48];
 
 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
-			verbose(env, "invalid variable-offset%s stack R%d var_off=%s off=%d size=%d\n",
-				err_extra, regno, tn_buf, off, access_size);
+			verbose(env, "invalid variable-offset%s stack %s var_off=%s off=%d size=%d\n",
+				err_extra, reg_arg_name(env, argno), tn_buf, off, access_size);
 		}
 		return err;
 	}
@@ -6320,12 +6108,11 @@ static void add_scalar_to_reg(struct bpf_reg_state *dst_reg, s64 val)
  * if t==write && value_regno==-1, some unknown value is stored into memory
  * if t==read && value_regno==-1, don't care what we read from memory
  */
-static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
+static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, struct bpf_reg_state *reg, argno_t argno,
 			    int off, int bpf_size, enum bpf_access_type t,
 			    int value_regno, bool strict_alignment_once, bool is_ldsx)
 {
 	struct bpf_reg_state *regs = cur_regs(env);
-	struct bpf_reg_state *reg = regs + regno;
 	int size, err = 0;
 
 	size = bpf_size_to_bytes(bpf_size);
@@ -6338,11 +6125,12 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 
 	if (reg->type == PTR_TO_MAP_KEY) {
 		if (t == BPF_WRITE) {
-			verbose(env, "write to change key R%d not allowed\n", regno);
+			verbose(env, "write to change key %s not allowed\n",
+				reg_arg_name(env, argno));
 			return -EACCES;
 		}
 
-		err = check_mem_region_access(env, regno, off, size,
+		err = check_mem_region_access(env, reg, argno, off, size,
 					      reg->map_ptr->key_size, false);
 		if (err)
 			return err;
@@ -6356,17 +6144,17 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 			verbose(env, "R%d leaks addr into map\n", value_regno);
 			return -EACCES;
 		}
-		err = check_map_access_type(env, regno, off, size, t);
+		err = check_map_access_type(env, reg, off, size, t);
 		if (err)
 			return err;
-		err = check_map_access(env, regno, off, size, false, ACCESS_DIRECT);
+		err = check_map_access(env, reg, argno, off, size, false, ACCESS_DIRECT);
 		if (err)
 			return err;
 		if (tnum_is_const(reg->var_off))
 			kptr_field = btf_record_find(reg->map_ptr->record,
 						     off + reg->var_off.value, BPF_KPTR | BPF_UPTR);
 		if (kptr_field) {
-			err = check_map_kptr_access(env, regno, value_regno, insn_idx, kptr_field);
+			err = check_map_kptr_access(env, value_regno, insn_idx, kptr_field);
 		} else if (t == BPF_READ && value_regno >= 0) {
 			struct bpf_map *map = reg->map_ptr;
 
@@ -6394,7 +6182,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 						     size);
 					return -EACCES;
 				}
-				copy_register_state(&regs[value_regno], reg);
+				regs[value_regno] = *reg;
 				add_scalar_to_reg(&regs[value_regno], off);
 				regs[value_regno].type = PTR_TO_INSN;
 			} else {
@@ -6406,14 +6194,14 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 		bool rdonly_untrusted = rdonly_mem && (reg->type & PTR_UNTRUSTED);
 
 		if (type_may_be_null(reg->type)) {
-			verbose(env, "R%d invalid mem access '%s'\n", regno,
+			verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno),
 				reg_type_str(env, reg->type));
 			return -EACCES;
 		}
 
 		if (t == BPF_WRITE && rdonly_mem) {
-			verbose(env, "R%d cannot write into %s\n",
-				regno, reg_type_str(env, reg->type));
+			verbose(env, "%s cannot write into %s\n",
+				reg_arg_name(env, argno), reg_type_str(env, reg->type));
 			return -EACCES;
 		}
 
@@ -6428,7 +6216,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 		 * instructions, hence no need to check bounds in that case.
 		 */
 		if (!rdonly_untrusted)
-			err = check_mem_region_access(env, regno, off, size,
+			err = check_mem_region_access(env, reg, argno, off, size,
 						      reg->mem_size, false);
 		if (!err && value_regno >= 0 && (t == BPF_READ || rdonly_mem))
 			mark_reg_unknown(env, regs, value_regno);
@@ -6446,7 +6234,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 			return -EACCES;
 		}
 
-		err = check_ctx_access(env, insn_idx, regno, off, size, t, &info);
+		err = check_ctx_access(env, insn_idx, reg, argno, off, size, t, &info);
 		if (!err && t == BPF_READ && value_regno >= 0) {
 			/* ctx access returns either a scalar, or a
 			 * PTR_TO_PACKET[_META,_END]. In the latter
@@ -6483,15 +6271,15 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 
 	} else if (reg->type == PTR_TO_STACK) {
 		/* Basic bounds checks. */
-		err = check_stack_access_within_bounds(env, regno, off, size, t);
+		err = check_stack_access_within_bounds(env, reg, argno, off, size, t);
 		if (err)
 			return err;
 
 		if (t == BPF_READ)
-			err = check_stack_read(env, regno, off, size,
+			err = check_stack_read(env, reg, argno, off, size,
 					       value_regno);
 		else
-			err = check_stack_write(env, regno, off, size,
+			err = check_stack_write(env, reg, off, size,
 						value_regno, insn_idx);
 	} else if (reg_is_pkt_pointer(reg)) {
 		if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
@@ -6504,7 +6292,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 				value_regno);
 			return -EACCES;
 		}
-		err = check_packet_access(env, regno, off, size, false);
+		err = check_packet_access(env, reg, argno, off, size, false);
 		if (!err && t == BPF_READ && value_regno >= 0)
 			mark_reg_unknown(env, regs, value_regno);
 	} else if (reg->type == PTR_TO_FLOW_KEYS) {
@@ -6520,23 +6308,23 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 			mark_reg_unknown(env, regs, value_regno);
 	} else if (type_is_sk_pointer(reg->type)) {
 		if (t == BPF_WRITE) {
-			verbose(env, "R%d cannot write into %s\n",
-				regno, reg_type_str(env, reg->type));
+			verbose(env, "%s cannot write into %s\n",
+				reg_arg_name(env, argno), reg_type_str(env, reg->type));
 			return -EACCES;
 		}
-		err = check_sock_access(env, insn_idx, regno, off, size, t);
+		err = check_sock_access(env, insn_idx, reg, argno, off, size, t);
 		if (!err && value_regno >= 0)
 			mark_reg_unknown(env, regs, value_regno);
 	} else if (reg->type == PTR_TO_TP_BUFFER) {
-		err = check_tp_buffer_access(env, reg, regno, off, size);
+		err = check_tp_buffer_access(env, reg, argno, off, size);
 		if (!err && t == BPF_READ && value_regno >= 0)
 			mark_reg_unknown(env, regs, value_regno);
 	} else if (base_type(reg->type) == PTR_TO_BTF_ID &&
 		   !type_may_be_null(reg->type)) {
-		err = check_ptr_to_btf_access(env, regs, regno, off, size, t,
+		err = check_ptr_to_btf_access(env, regs, reg, argno, off, size, t,
 					      value_regno);
 	} else if (reg->type == CONST_PTR_TO_MAP) {
-		err = check_ptr_to_map_access(env, regs, regno, off, size, t,
+		err = check_ptr_to_map_access(env, regs, reg, argno, off, size, t,
 					      value_regno);
 	} else if (base_type(reg->type) == PTR_TO_BUF &&
 		   !type_may_be_null(reg->type)) {
@@ -6545,8 +6333,8 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 
 		if (rdonly_mem) {
 			if (t == BPF_WRITE) {
-				verbose(env, "R%d cannot write into %s\n",
-					regno, reg_type_str(env, reg->type));
+				verbose(env, "%s cannot write into %s\n",
+					reg_arg_name(env, argno), reg_type_str(env, reg->type));
 				return -EACCES;
 			}
 			max_access = &env->prog->aux->max_rdonly_access;
@@ -6554,7 +6342,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 			max_access = &env->prog->aux->max_rdwr_access;
 		}
 
-		err = check_buffer_access(env, reg, regno, off, size, false,
+		err = check_buffer_access(env, reg, argno, off, size, false,
 					  max_access);
 
 		if (!err && value_regno >= 0 && (rdonly_mem || t == BPF_READ))
@@ -6563,7 +6351,7 @@ static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regn
 		if (t == BPF_READ && value_regno >= 0)
 			mark_reg_unknown(env, regs, value_regno);
 	} else {
-		verbose(env, "R%d invalid mem access '%s'\n", regno,
+		verbose(env, "%s invalid mem access '%s'\n", reg_arg_name(env, argno),
 			reg_type_str(env, reg->type));
 		return -EACCES;
 	}
@@ -6586,10 +6374,20 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			  bool strict_alignment_once, bool is_ldsx,
 			  bool allow_trust_mismatch, const char *ctx)
 {
+	struct bpf_verifier_state *vstate = env->cur_state;
+	struct bpf_func_state *state = vstate->frame[vstate->curframe];
 	struct bpf_reg_state *regs = cur_regs(env);
 	enum bpf_reg_type src_reg_type;
 	int err;
 
+	/* Handle stack arg read */
+	if (is_stack_arg_ldx(insn)) {
+		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
+		if (err)
+			return err;
+		return check_stack_arg_read(env, state, insn->off, insn->dst_reg);
+	}
+
 	/* check src operand */
 	err = check_reg_arg(env, insn->src_reg, SRC_OP);
 	if (err)
@@ -6605,7 +6403,7 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	/* Check if (src_reg + off) is readable. The state of dst_reg will be
 	 * updated by this call.
 	 */
-	err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off,
+	err = check_mem_access(env, env->insn_idx, regs + insn->src_reg, argno_from_reg(insn->src_reg), insn->off,
 			       BPF_SIZE(insn->code), BPF_READ, insn->dst_reg,
 			       strict_alignment_once, is_ldsx);
 	err = err ?: save_aux_ptr_type(env, src_reg_type,
@@ -6618,10 +6416,20 @@ static int check_load_mem(struct bpf_verifier_env *env, struct bpf_insn *insn,
 static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			   bool strict_alignment_once)
 {
+	struct bpf_verifier_state *vstate = env->cur_state;
+	struct bpf_func_state *state = vstate->frame[vstate->curframe];
 	struct bpf_reg_state *regs = cur_regs(env);
 	enum bpf_reg_type dst_reg_type;
 	int err;
 
+	/* Handle stack arg write */
+	if (is_stack_arg_stx(insn)) {
+		err = check_reg_arg(env, insn->src_reg, SRC_OP);
+		if (err)
+			return err;
+		return check_stack_arg_write(env, state, insn->off, regs + insn->src_reg);
+	}
+
 	/* check src1 operand */
 	err = check_reg_arg(env, insn->src_reg, SRC_OP);
 	if (err)
@@ -6635,7 +6443,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	dst_reg_type = regs[insn->dst_reg].type;
 
 	/* Check if (dst_reg + off) is writeable. */
-	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
+	err = check_mem_access(env, env->insn_idx, regs + insn->dst_reg, argno_from_reg(insn->dst_reg), insn->off,
 			       BPF_SIZE(insn->code), BPF_WRITE, insn->src_reg,
 			       strict_alignment_once, false);
 	err = err ?: save_aux_ptr_type(env, dst_reg_type, false);
@@ -6646,6 +6454,7 @@ static int check_store_reg(struct bpf_verifier_env *env, struct bpf_insn *insn,
 static int check_atomic_rmw(struct bpf_verifier_env *env,
 			    struct bpf_insn *insn)
 {
+	struct bpf_reg_state *dst_reg;
 	int load_reg;
 	int err;
 
@@ -6707,13 +6516,15 @@ static int check_atomic_rmw(struct bpf_verifier_env *env,
 		load_reg = -1;
 	}
 
+	dst_reg = cur_regs(env) + insn->dst_reg;
+
 	/* Check whether we can read the memory, with second call for fetch
 	 * case to simulate the register fill.
 	 */
-	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
+	err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off,
 			       BPF_SIZE(insn->code), BPF_READ, -1, true, false);
 	if (!err && load_reg >= 0)
-		err = check_mem_access(env, env->insn_idx, insn->dst_reg,
+		err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg),
 				       insn->off, BPF_SIZE(insn->code),
 				       BPF_READ, load_reg, true, false);
 	if (err)
@@ -6725,7 +6536,7 @@ static int check_atomic_rmw(struct bpf_verifier_env *env,
 			return err;
 	}
 	/* Check whether we can write into the same memory. */
-	err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off,
+	err = check_mem_access(env, env->insn_idx, dst_reg, argno_from_reg(insn->dst_reg), insn->off,
 			       BPF_SIZE(insn->code), BPF_WRITE, -1, true, false);
 	if (err)
 		return err;
@@ -6814,11 +6625,10 @@ static int check_atomic(struct bpf_verifier_env *env, struct bpf_insn *insn)
  * read offsets are marked as read.
  */
 static int check_stack_range_initialized(
-		struct bpf_verifier_env *env, int regno, int off,
+		struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int off,
 		int access_size, bool zero_size_allowed,
 		enum bpf_access_type type, struct bpf_call_arg_meta *meta)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	struct bpf_func_state *state = bpf_func(env, reg);
 	int err, min_off, max_off, i, j, slot, spi;
 	/* Some accesses can write anything into the stack, others are
@@ -6840,7 +6650,7 @@ static int check_stack_range_initialized(
 		return -EACCES;
 	}
 
-	err = check_stack_access_within_bounds(env, regno, off, access_size, type);
+	err = check_stack_access_within_bounds(env, reg, argno, off, access_size, type);
 	if (err)
 		return err;
 
@@ -6857,8 +6667,8 @@ static int check_stack_range_initialized(
 			char tn_buf[48];
 
 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
-			verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n",
-				regno, tn_buf);
+			verbose(env, "%s variable offset stack access prohibited for !root, var_off=%s\n",
+				reg_arg_name(env, argno), tn_buf);
 			return -EACCES;
 		}
 		/* Only initialized buffer on stack is allowed to be accessed
@@ -6870,8 +6680,8 @@ static int check_stack_range_initialized(
 		if (meta && meta->raw_mode)
 			meta = NULL;
 
-		min_off = reg->smin_value + off;
-		max_off = reg->smax_value + off;
+		min_off = reg_smin(reg) + off;
+		max_off = reg_smax(reg) + off;
 	}
 
 	if (meta && meta->raw_mode) {
@@ -6901,7 +6711,7 @@ static int check_stack_range_initialized(
 			}
 		}
 		meta->access_size = access_size;
-		meta->regno = regno;
+		meta->regno = reg_from_argno(argno);
 		return 0;
 	}
 
@@ -6941,17 +6751,17 @@ static int check_stack_range_initialized(
 		if (*stype == STACK_POISON) {
 			if (allow_poison)
 				goto mark;
-			verbose(env, "reading from stack R%d off %d+%d size %d, slot poisoned by dead code elimination\n",
-				regno, min_off, i - min_off, access_size);
+			verbose(env, "reading from stack %s off %d+%d size %d, slot poisoned by dead code elimination\n",
+				reg_arg_name(env, argno), min_off, i - min_off, access_size);
 		} else if (tnum_is_const(reg->var_off)) {
-			verbose(env, "invalid read from stack R%d off %d+%d size %d\n",
-				regno, min_off, i - min_off, access_size);
+			verbose(env, "invalid read from stack %s off %d+%d size %d\n",
+				reg_arg_name(env, argno), min_off, i - min_off, access_size);
 		} else {
 			char tn_buf[48];
 
 			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
-			verbose(env, "invalid read from stack R%d var_off %s+%d size %d\n",
-				regno, tn_buf, i - min_off, access_size);
+			verbose(env, "invalid read from stack %s var_off %s+%d size %d\n",
+				reg_arg_name(env, argno), tn_buf, i - min_off, access_size);
 		}
 		return -EACCES;
 mark:
@@ -6960,48 +6770,48 @@ static int check_stack_range_initialized(
 	return 0;
 }
 
-static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
+static int check_helper_mem_access(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 				   int access_size, enum bpf_access_type access_type,
 				   bool zero_size_allowed,
 				   struct bpf_call_arg_meta *meta)
 {
-	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
+	struct bpf_reg_state *regs = cur_regs(env);
 	u32 *max_access;
 
 	switch (base_type(reg->type)) {
 	case PTR_TO_PACKET:
 	case PTR_TO_PACKET_META:
-		return check_packet_access(env, regno, 0, access_size,
+		return check_packet_access(env, reg, argno, 0, access_size,
 					   zero_size_allowed);
 	case PTR_TO_MAP_KEY:
 		if (access_type == BPF_WRITE) {
-			verbose(env, "R%d cannot write into %s\n", regno,
-				reg_type_str(env, reg->type));
+			verbose(env, "%s cannot write into %s\n",
+				reg_arg_name(env, argno), reg_type_str(env, reg->type));
 			return -EACCES;
 		}
-		return check_mem_region_access(env, regno, 0, access_size,
+		return check_mem_region_access(env, reg, argno, 0, access_size,
 					       reg->map_ptr->key_size, false);
 	case PTR_TO_MAP_VALUE:
-		if (check_map_access_type(env, regno, 0, access_size, access_type))
+		if (check_map_access_type(env, reg, 0, access_size, access_type))
 			return -EACCES;
-		return check_map_access(env, regno, 0, access_size,
+		return check_map_access(env, reg, argno, 0, access_size,
 					zero_size_allowed, ACCESS_HELPER);
 	case PTR_TO_MEM:
 		if (type_is_rdonly_mem(reg->type)) {
 			if (access_type == BPF_WRITE) {
-				verbose(env, "R%d cannot write into %s\n", regno,
-					reg_type_str(env, reg->type));
+				verbose(env, "%s cannot write into %s\n",
+					reg_arg_name(env, argno), reg_type_str(env, reg->type));
 				return -EACCES;
 			}
 		}
-		return check_mem_region_access(env, regno, 0,
+		return check_mem_region_access(env, reg, argno, 0,
 					       access_size, reg->mem_size,
 					       zero_size_allowed);
 	case PTR_TO_BUF:
 		if (type_is_rdonly_mem(reg->type)) {
 			if (access_type == BPF_WRITE) {
-				verbose(env, "R%d cannot write into %s\n", regno,
-					reg_type_str(env, reg->type));
+				verbose(env, "%s cannot write into %s\n",
+					reg_arg_name(env, argno), reg_type_str(env, reg->type));
 				return -EACCES;
 			}
 
@@ -7009,26 +6819,26 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
 		} else {
 			max_access = &env->prog->aux->max_rdwr_access;
 		}
-		return check_buffer_access(env, reg, regno, 0,
+		return check_buffer_access(env, reg, argno, 0,
 					   access_size, zero_size_allowed,
 					   max_access);
 	case PTR_TO_STACK:
 		return check_stack_range_initialized(
-				env,
-				regno, 0, access_size,
+				env, reg,
+				argno, 0, access_size,
 				zero_size_allowed, access_type, meta);
 	case PTR_TO_BTF_ID:
-		return check_ptr_to_btf_access(env, regs, regno, 0,
+		return check_ptr_to_btf_access(env, regs, reg, argno, 0,
 					       access_size, BPF_READ, -1);
 	case PTR_TO_CTX:
 		/* Only permit reading or writing syscall context using helper calls. */
 		if (is_var_ctx_off_allowed(env->prog)) {
-			int err = check_mem_region_access(env, regno, 0, access_size, U16_MAX,
+			int err = check_mem_region_access(env, reg, argno, 0, access_size, U16_MAX,
 							  zero_size_allowed);
 			if (err)
 				return err;
-			if (env->prog->aux->max_ctx_offset < reg->umax_value + access_size)
-				env->prog->aux->max_ctx_offset = reg->umax_value + access_size;
+			if (env->prog->aux->max_ctx_offset < reg_umax(reg) + access_size)
+				env->prog->aux->max_ctx_offset = reg_umax(reg) + access_size;
 			return 0;
 		}
 		fallthrough;
@@ -7038,7 +6848,7 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
 		    bpf_register_is_null(reg))
 			return 0;
 
-		verbose(env, "R%d type=%s ", regno,
+		verbose(env, "%s type=%s ", reg_arg_name(env, argno),
 			reg_type_str(env, reg->type));
 		verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
 		return -EACCES;
@@ -7048,12 +6858,12 @@ static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
 /* verify arguments to helpers or kfuncs consisting of a pointer and an access
  * size.
  *
- * @regno is the register containing the access size. regno-1 is the register
- * containing the pointer.
+ * @mem_reg contains the pointer, @size_reg contains the access size.
  */
 static int check_mem_size_reg(struct bpf_verifier_env *env,
-			      struct bpf_reg_state *reg, u32 regno,
-			      enum bpf_access_type access_type,
+			      struct bpf_reg_state *mem_reg,
+			      struct bpf_reg_state *size_reg, argno_t mem_argno,
+			      argno_t size_argno, enum bpf_access_type access_type,
 			      bool zero_size_allowed,
 			      struct bpf_call_arg_meta *meta)
 {
@@ -7067,42 +6877,48 @@ static int check_mem_size_reg(struct bpf_verifier_env *env,
 	 * out. Only upper bounds can be learned because retval is an
 	 * int type and negative retvals are allowed.
 	 */
-	meta->msize_max_value = reg->umax_value;
+	meta->msize_max_value = reg_umax(size_reg);
 
 	/* The register is SCALAR_VALUE; the access check happens using
 	 * its boundaries. For unprivileged variable accesses, disable
 	 * raw mode so that the program is required to initialize all
 	 * the memory that the helper could just partially fill up.
 	 */
-	if (!tnum_is_const(reg->var_off))
+	if (!tnum_is_const(size_reg->var_off))
 		meta = NULL;
 
-	if (reg->smin_value < 0) {
-		verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
-			regno);
+	if (reg_smin(size_reg) < 0) {
+		verbose(env, "%s min value is negative, either use unsigned or 'var &= const'\n",
+			reg_arg_name(env, size_argno));
 		return -EACCES;
 	}
 
-	if (reg->umin_value == 0 && !zero_size_allowed) {
-		verbose(env, "R%d invalid zero-sized read: u64=[%lld,%lld]\n",
-			regno, reg->umin_value, reg->umax_value);
+	if (reg_umin(size_reg) == 0 && !zero_size_allowed) {
+		verbose(env, "%s invalid zero-sized read: u64=[%lld,%lld]\n",
+			reg_arg_name(env, size_argno), reg_umin(size_reg), reg_umax(size_reg));
 		return -EACCES;
 	}
 
-	if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
-		verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
-			regno);
+	if (reg_umax(size_reg) >= BPF_MAX_VAR_SIZ) {
+		verbose(env, "%s unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
+			reg_arg_name(env, size_argno));
 		return -EACCES;
 	}
-	err = check_helper_mem_access(env, regno - 1, reg->umax_value,
+	err = check_helper_mem_access(env, mem_reg, mem_argno, reg_umax(size_reg),
 				      access_type, zero_size_allowed, meta);
-	if (!err)
-		err = mark_chain_precision(env, regno);
+	if (!err) {
+		int regno = reg_from_argno(size_argno);
+
+		if (regno >= 0)
+			err = mark_chain_precision(env, regno);
+		else
+			err = mark_stack_arg_precision(env, arg_idx_from_argno(size_argno));
+	}
 	return err;
 }
 
 static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
-			 u32 regno, u32 mem_size)
+			 argno_t argno, u32 mem_size)
 {
 	bool may_be_null = type_may_be_null(reg->type);
 	struct bpf_reg_state saved_reg;
@@ -7122,8 +6938,8 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg
 
 	int size = base_type(reg->type) == PTR_TO_STACK ? -(int)mem_size : mem_size;
 
-	err = check_helper_mem_access(env, regno, size, BPF_READ, true, NULL);
-	err = err ?: check_helper_mem_access(env, regno, size, BPF_WRITE, true, NULL);
+	err = check_helper_mem_access(env, reg, argno, size, BPF_READ, true, NULL);
+	err = err ?: check_helper_mem_access(env, reg, argno, size, BPF_WRITE, true, NULL);
 
 	if (may_be_null)
 		*reg = saved_reg;
@@ -7131,17 +6947,14 @@ static int check_mem_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg
 	return err;
 }
 
-static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
-				    u32 regno)
+static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg_state *mem_reg,
+				    struct bpf_reg_state *size_reg, argno_t mem_argno, argno_t size_argno)
 {
-	struct bpf_reg_state *mem_reg = &cur_regs(env)[regno - 1];
 	bool may_be_null = type_may_be_null(mem_reg->type);
 	struct bpf_reg_state saved_reg;
 	struct bpf_call_arg_meta meta;
 	int err;
 
-	WARN_ON_ONCE(regno < BPF_REG_2 || regno > BPF_REG_5);
-
 	memset(&meta, 0, sizeof(meta));
 
 	if (may_be_null) {
@@ -7149,8 +6962,8 @@ static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg
 		mark_ptr_not_null_reg(mem_reg);
 	}
 
-	err = check_mem_size_reg(env, reg, regno, BPF_READ, true, &meta);
-	err = err ?: check_mem_size_reg(env, reg, regno, BPF_WRITE, true, &meta);
+	err = check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_READ, true, &meta);
+	err = err ?: check_mem_size_reg(env, mem_reg, size_reg, mem_argno, size_argno, BPF_WRITE, true, &meta);
 
 	if (may_be_null)
 		*mem_reg = saved_reg;
@@ -7186,11 +6999,10 @@ enum {
  * env->cur_state->active_locks remembers which map value element or allocated
  * object got locked and clears it after bpf_spin_unlock.
  */
-static int process_spin_lock(struct bpf_verifier_env *env, int regno, int flags)
+static int process_spin_lock(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int flags)
 {
 	bool is_lock = flags & PROCESS_SPIN_LOCK, is_res_lock = flags & PROCESS_RES_LOCK;
 	const char *lock_str = is_res_lock ? "bpf_res_spin" : "bpf_spin";
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	struct bpf_verifier_state *cur = env->cur_state;
 	bool is_const = tnum_is_const(reg->var_off);
 	bool is_irq = flags & PROCESS_LOCK_IRQ;
@@ -7203,8 +7015,8 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, int flags)
 
 	if (!is_const) {
 		verbose(env,
-			"R%d doesn't have constant offset. %s_lock has to be at the constant offset\n",
-			regno, lock_str);
+			"%s doesn't have constant offset. %s_lock has to be at the constant offset\n",
+			reg_arg_name(env, argno), lock_str);
 		return -EINVAL;
 	}
 	if (reg->type == PTR_TO_MAP_VALUE) {
@@ -7303,11 +7115,10 @@ static int process_spin_lock(struct bpf_verifier_env *env, int regno, int flags)
 }
 
 /* Check if @regno is a pointer to a specific field in a map value */
-static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno,
+static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 				   enum btf_field_type field_type,
 				   struct bpf_map_desc *map_desc)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	bool is_const = tnum_is_const(reg->var_off);
 	struct bpf_map *map = reg->map_ptr;
 	u64 val = reg->var_off.value;
@@ -7316,8 +7127,8 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno,
 
 	if (!is_const) {
 		verbose(env,
-			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
-			regno, struct_name);
+			"%s doesn't have constant offset. %s has to be at the constant offset\n",
+			reg_arg_name(env, argno), struct_name);
 		return -EINVAL;
 	}
 	if (!map->btf) {
@@ -7357,26 +7168,26 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, u32 regno,
 	return 0;
 }
 
-static int process_timer_func(struct bpf_verifier_env *env, int regno,
+static int process_timer_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 			      struct bpf_map_desc *map)
 {
 	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
 		verbose(env, "bpf_timer cannot be used for PREEMPT_RT.\n");
 		return -EOPNOTSUPP;
 	}
-	return check_map_field_pointer(env, regno, BPF_TIMER, map);
+	return check_map_field_pointer(env, reg, argno, BPF_TIMER, map);
 }
 
-static int process_timer_helper(struct bpf_verifier_env *env, int regno,
+static int process_timer_helper(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 				struct bpf_call_arg_meta *meta)
 {
-	return process_timer_func(env, regno, &meta->map);
+	return process_timer_func(env, reg, argno, &meta->map);
 }
 
-static int process_timer_kfunc(struct bpf_verifier_env *env, int regno,
+static int process_timer_kfunc(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 			       struct bpf_kfunc_call_arg_meta *meta)
 {
-	return process_timer_func(env, regno, &meta->map);
+	return process_timer_func(env, reg, argno, &meta->map);
 }
 
 static int process_kptr_func(struct bpf_verifier_env *env, int regno,
@@ -7434,45 +7245,25 @@ static int process_kptr_func(struct bpf_verifier_env *env, int regno,
  * bytes as STACK_DYNPTR in case of PTR_TO_STACK. In case of
  * CONST_PTR_TO_DYNPTR, we are guaranteed to get the beginning of the object.
  *
- * Mutability of bpf_dynptr is at two levels, one is at the level of struct
- * bpf_dynptr itself, i.e. whether the helper is receiving a pointer to struct
- * bpf_dynptr or pointer to const struct bpf_dynptr. In the former case, it can
- * mutate the view of the dynptr and also possibly destroy it. In the latter
- * case, it cannot mutate the bpf_dynptr itself but it can still mutate the
- * memory that dynptr points to.
- *
- * The verifier will keep track both levels of mutation (bpf_dynptr's in
- * reg->type and the memory's in reg->dynptr.type), but there is no support for
- * readonly dynptr view yet, hence only the first case is tracked and checked.
- *
- * This is consistent with how C applies the const modifier to a struct object,
- * where the pointer itself inside bpf_dynptr becomes const but not what it
- * points to.
- *
- * Helpers which do not mutate the bpf_dynptr set MEM_RDONLY in their argument
- * type, and declare it as 'const struct bpf_dynptr *' in their prototype.
+ * Mutability of bpf_dynptr is at two levels: the dynptr and the memory the
+ * dynptr points to. At the first level, the verifier will make sure a
+ * CONST_PTR_TO_DYNPTR cannot be reinitialized or destroyed. The mutability of
+ * a dynptr's view (i.e., start and offset) is not tracked as there is not such
+ * use case. The second level is tracked using the upper bit of bpf_dynptr->size
+ * and checked dynamically during runtime.
  */
-static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn_idx,
+static int process_dynptr_func(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx,
 			       enum bpf_arg_type arg_type, int clone_ref_obj_id)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	int err;
 
 	if (reg->type != PTR_TO_STACK && reg->type != CONST_PTR_TO_DYNPTR) {
 		verbose(env,
-			"arg#%d expected pointer to stack or const struct bpf_dynptr\n",
-			regno - 1);
+			"%s expected pointer to stack or const struct bpf_dynptr\n",
+			reg_arg_name(env, argno));
 		return -EINVAL;
 	}
 
-	/* MEM_UNINIT and MEM_RDONLY are exclusive, when applied to an
-	 * ARG_PTR_TO_DYNPTR (or ARG_PTR_TO_DYNPTR | DYNPTR_TYPE_*):
-	 */
-	if ((arg_type & (MEM_UNINIT | MEM_RDONLY)) == (MEM_UNINIT | MEM_RDONLY)) {
-		verifier_bug(env, "misconfigured dynptr helper type flags");
-		return -EFAULT;
-	}
-
 	/*  MEM_UNINIT - Points to memory that is an appropriate candidate for
 	 *		 constructing a mutable bpf_dynptr object.
 	 *
@@ -7480,13 +7271,12 @@ static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn
 	 *		 pointing to a region of at least 16 bytes which doesn't
 	 *		 contain an existing bpf_dynptr.
 	 *
-	 *  MEM_RDONLY - Points to a initialized bpf_dynptr that will not be
-	 *		 mutated or destroyed. However, the memory it points to
-	 *		 may be mutated.
+	 *  OBJ_RELEASE - Points to a initialized bpf_dynptr that will be
+	 *		  destroyed.
 	 *
-	 *  None       - Points to a initialized dynptr that can be mutated and
-	 *		 destroyed, including mutation of the memory it points
-	 *		 to.
+	 *  None       - Points to a initialized dynptr that cannot be
+	 *		 reinitialized or destroyed. However, the view of the
+	 *		 dynptr and the memory it points to may be mutated.
 	 */
 	if (arg_type & MEM_UNINIT) {
 		int i;
@@ -7498,32 +7288,32 @@ static int process_dynptr_func(struct bpf_verifier_env *env, int regno, int insn
 
 		/* we write BPF_DW bits (8 bytes) at a time */
 		for (i = 0; i < BPF_DYNPTR_SIZE; i += 8) {
-			err = check_mem_access(env, insn_idx, regno,
+			err = check_mem_access(env, insn_idx, reg, argno,
 					       i, BPF_DW, BPF_WRITE, -1, false, false);
 			if (err)
 				return err;
 		}
 
 		err = mark_stack_slots_dynptr(env, reg, arg_type, insn_idx, clone_ref_obj_id);
-	} else /* MEM_RDONLY and None case from above */ {
+	} else /* OBJ_RELEASE and None case from above */ {
 		/* For the reg->type == PTR_TO_STACK case, bpf_dynptr is never const */
-		if (reg->type == CONST_PTR_TO_DYNPTR && !(arg_type & MEM_RDONLY)) {
-			verbose(env, "cannot pass pointer to const bpf_dynptr, the helper mutates it\n");
+		if (reg->type == CONST_PTR_TO_DYNPTR && (arg_type & OBJ_RELEASE)) {
+			verbose(env, "CONST_PTR_TO_DYNPTR cannot be released\n");
 			return -EINVAL;
 		}
 
 		if (!is_dynptr_reg_valid_init(env, reg)) {
-			verbose(env,
-				"Expected an initialized dynptr as arg #%d\n",
-				regno - 1);
+			verbose(env, "Expected an initialized dynptr as %s\n",
+				reg_arg_name(env, argno));
 			return -EINVAL;
 		}
 
-		/* Fold modifiers (in this case, MEM_RDONLY) when checking expected type */
-		if (!is_dynptr_type_expected(env, reg, arg_type & ~MEM_RDONLY)) {
+		/* Fold modifiers (in this case, OBJ_RELEASE) when checking expected type */
+		if (!is_dynptr_type_expected(env, reg, arg_type & ~OBJ_RELEASE)) {
 			verbose(env,
-				"Expected a dynptr of type %s as arg #%d\n",
-				dynptr_type_str(arg_to_dynptr_type(arg_type)), regno - 1);
+				"Expected a dynptr of type %s as %s\n",
+				dynptr_type_str(arg_to_dynptr_type(arg_type)),
+				reg_arg_name(env, argno));
 			return -EINVAL;
 		}
 
@@ -7568,15 +7358,16 @@ static bool is_kfunc_arg_iter(struct bpf_kfunc_call_arg_meta *meta, int arg_idx,
 	return btf_param_match_suffix(meta->btf, arg, "__iter");
 }
 
-static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_idx,
+static int process_iter_arg(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno, int insn_idx,
 			    struct bpf_kfunc_call_arg_meta *meta)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	const struct btf_type *t;
+	u32 arg_idx = arg_idx_from_argno(argno);
 	int spi, err, i, nr_slots, btf_id;
 
 	if (reg->type != PTR_TO_STACK) {
-		verbose(env, "arg#%d expected pointer to an iterator on stack\n", regno - 1);
+		verbose(env, "%s expected pointer to an iterator on stack\n",
+			reg_arg_name(env, argno));
 		return -EINVAL;
 	}
 
@@ -7586,9 +7377,10 @@ static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_id
 	 * to any kfunc, if arg has "__iter" suffix, we need to be a bit more
 	 * conservative here.
 	 */
-	btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, regno - 1);
+	btf_id = btf_check_iter_arg(meta->btf, meta->func_proto, arg_idx);
 	if (btf_id < 0) {
-		verbose(env, "expected valid iter pointer as arg #%d\n", regno - 1);
+		verbose(env, "expected valid iter pointer as %s\n",
+			reg_arg_name(env, argno));
 		return -EINVAL;
 	}
 	t = btf_type_by_id(meta->btf, btf_id);
@@ -7597,13 +7389,13 @@ static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_id
 	if (is_iter_new_kfunc(meta)) {
 		/* bpf_iter_<type>_new() expects pointer to uninit iter state */
 		if (!is_iter_reg_valid_uninit(env, reg, nr_slots)) {
-			verbose(env, "expected uninitialized iter_%s as arg #%d\n",
-				iter_type_str(meta->btf, btf_id), regno - 1);
+			verbose(env, "expected uninitialized iter_%s as %s\n",
+				iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno));
 			return -EINVAL;
 		}
 
 		for (i = 0; i < nr_slots * 8; i += BPF_REG_SIZE) {
-			err = check_mem_access(env, insn_idx, regno,
+			err = check_mem_access(env, insn_idx, reg, argno,
 					       i, BPF_DW, BPF_WRITE, -1, false, false);
 			if (err)
 				return err;
@@ -7621,8 +7413,8 @@ static int process_iter_arg(struct bpf_verifier_env *env, int regno, int insn_id
 		case 0:
 			break;
 		case -EINVAL:
-			verbose(env, "expected an initialized iter_%s as arg #%d\n",
-				iter_type_str(meta->btf, btf_id), regno - 1);
+			verbose(env, "expected an initialized iter_%s as %s\n",
+				iter_type_str(meta->btf, btf_id), reg_arg_name(env, argno));
 			return err;
 		case -EPROTO:
 			verbose(env, "expected an RCU CS when using %s\n", meta->func_name);
@@ -8042,12 +7834,11 @@ static const struct bpf_reg_types *compatible_reg_types[__BPF_ARG_TYPE_MAX] = {
 	[ARG_PTR_TO_DYNPTR]		= &dynptr_types,
 };
 
-static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
+static int check_reg_type(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 			  enum bpf_arg_type arg_type,
 			  const u32 *arg_btf_id,
 			  struct bpf_call_arg_meta *meta)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	enum bpf_reg_type expected, type = reg->type;
 	const struct bpf_reg_types *compatible;
 	int i, j, err;
@@ -8078,7 +7869,7 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 		type &= ~DYNPTR_TYPE_FLAG_MASK;
 
 	/* Local kptr types are allowed as the source argument of bpf_kptr_xchg */
-	if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && regno == BPF_REG_2) {
+	if (meta->func_id == BPF_FUNC_kptr_xchg && type_is_alloc(type) && reg_from_argno(argno) == BPF_REG_2) {
 		type &= ~MEM_ALLOC;
 		type &= ~MEM_PERCPU;
 	}
@@ -8092,7 +7883,7 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 			goto found;
 	}
 
-	verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
+	verbose(env, "%s type=%s expected=", reg_arg_name(env, argno), reg_type_str(env, reg->type));
 	for (j = 0; j + 1 < i; j++)
 		verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
 	verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
@@ -8105,9 +7896,9 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 	if (compatible == &mem_types) {
 		if (!(arg_type & MEM_RDONLY)) {
 			verbose(env,
-				"%s() may write into memory pointed by R%d type=%s\n",
+				"%s() may write into memory pointed by %s type=%s\n",
 				func_id_name(meta->func_id),
-				regno, reg_type_str(env, reg->type));
+				reg_arg_name(env, argno), reg_type_str(env, reg->type));
 			return -EACCES;
 		}
 		return 0;
@@ -8130,7 +7921,8 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 
 		if (type_may_be_null(reg->type) &&
 		    (!type_may_be_null(arg_type) || arg_type_is_release(arg_type))) {
-			verbose(env, "Possibly NULL pointer passed to helper arg%d\n", regno);
+			verbose(env, "Possibly NULL pointer passed to helper %s\n",
+				reg_arg_name(env, argno));
 			return -EACCES;
 		}
 
@@ -8143,25 +7935,26 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 		}
 
 		if (meta->func_id == BPF_FUNC_kptr_xchg) {
-			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
+			if (map_kptr_match_type(env, meta->kptr_field, reg, reg_from_argno(argno)))
 				return -EACCES;
 		} else {
 			if (arg_btf_id == BPF_PTR_POISON) {
 				verbose(env, "verifier internal error:");
-				verbose(env, "R%d has non-overwritten BPF_PTR_POISON type\n",
-					regno);
+				verbose(env, "%s has non-overwritten BPF_PTR_POISON type\n",
+					reg_arg_name(env, argno));
 				return -EACCES;
 			}
 
-			err = __check_ptr_off_reg(env, reg, regno, true);
+			err = __check_ptr_off_reg(env, reg, argno, true);
 			if (err)
 				return err;
 
 			if (!btf_struct_ids_match(&env->log, reg->btf, reg->btf_id,
 						  reg->var_off.value, btf_vmlinux, *arg_btf_id,
 						  strict_type_match)) {
-				verbose(env, "R%d is of type %s but %s is expected\n",
-					regno, btf_type_name(reg->btf, reg->btf_id),
+				verbose(env, "%s is of type %s but %s is expected\n",
+					reg_arg_name(env, argno),
+					btf_type_name(reg->btf, reg->btf_id),
 					btf_type_name(btf_vmlinux, *arg_btf_id));
 				return -EACCES;
 			}
@@ -8178,8 +7971,11 @@ static int check_reg_type(struct bpf_verifier_env *env, u32 regno,
 			return -EFAULT;
 		}
 		/* Check if local kptr in src arg matches kptr in dst arg */
-		if (meta->func_id == BPF_FUNC_kptr_xchg && regno == BPF_REG_2) {
-			if (map_kptr_match_type(env, meta->kptr_field, reg, regno))
+		if (meta->func_id == BPF_FUNC_kptr_xchg) {
+			int regno = reg_from_argno(argno);
+
+			if (regno == BPF_REG_2 &&
+			    map_kptr_match_type(env, meta->kptr_field, reg, regno))
 				return -EACCES;
 		}
 		break;
@@ -8213,7 +8009,7 @@ reg_find_field_offset(const struct bpf_reg_state *reg, s32 off, u32 fields)
 }
 
 static int check_func_arg_reg_off(struct bpf_verifier_env *env,
-				  const struct bpf_reg_state *reg, int regno,
+				  const struct bpf_reg_state *reg, argno_t argno,
 				  enum bpf_arg_type arg_type)
 {
 	u32 type = reg->type;
@@ -8239,8 +8035,8 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env,
 		 * to give the user a better error message.
 		 */
 		if (!tnum_is_const(reg->var_off) || reg->var_off.value != 0) {
-			verbose(env, "R%d must have zero offset when passed to release func or trusted arg to kfunc\n",
-				regno);
+			verbose(env, "%s must have zero offset when passed to release func or trusted arg to kfunc\n",
+				reg_arg_name(env, argno));
 			return -EINVAL;
 		}
 	}
@@ -8276,7 +8072,7 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env,
 		 * cases. var_off always must be 0 for PTR_TO_BTF_ID, hence we
 		 * still need to do checks instead of returning.
 		 */
-		return __check_ptr_off_reg(env, reg, regno, true);
+		return __check_ptr_off_reg(env, reg, argno, true);
 	case PTR_TO_CTX:
 		/*
 		 * Allow fixed and variable offsets for syscall context, but
@@ -8288,7 +8084,7 @@ static int check_func_arg_reg_off(struct bpf_verifier_env *env,
 			return 0;
 		fallthrough;
 	default:
-		return __check_ptr_off_reg(env, reg, regno, false);
+		return __check_ptr_off_reg(env, reg, argno, false);
 	}
 }
 
@@ -8358,8 +8154,8 @@ static enum bpf_dynptr_type dynptr_get_type(struct bpf_verifier_env *env,
 	return state->stack[spi].spilled_ptr.dynptr.type;
 }
 
-static int check_reg_const_str(struct bpf_verifier_env *env,
-			       struct bpf_reg_state *reg, u32 regno)
+static int check_arg_const_str(struct bpf_verifier_env *env,
+			       struct bpf_reg_state *reg, argno_t argno)
 {
 	struct bpf_map *map = reg->map_ptr;
 	int err;
@@ -8371,17 +8167,18 @@ static int check_reg_const_str(struct bpf_verifier_env *env,
 		return -EINVAL;
 
 	if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY) {
-		verbose(env, "R%d points to insn_array map which cannot be used as const string\n", regno);
+		verbose(env, "%s points to insn_array map which cannot be used as const string\n",
+			reg_arg_name(env, argno));
 		return -EACCES;
 	}
 
 	if (!bpf_map_is_rdonly(map)) {
-		verbose(env, "R%d does not point to a readonly map'\n", regno);
+		verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno));
 		return -EACCES;
 	}
 
 	if (!tnum_is_const(reg->var_off)) {
-		verbose(env, "R%d is not a constant address'\n", regno);
+		verbose(env, "%s is not a constant address'\n", reg_arg_name(env, argno));
 		return -EACCES;
 	}
 
@@ -8390,7 +8187,7 @@ static int check_reg_const_str(struct bpf_verifier_env *env,
 		return -EACCES;
 	}
 
-	err = check_map_access(env, regno, 0,
+	err = check_map_access(env, reg, argno, 0,
 			       map->value_size - reg->var_off.value, false,
 			       ACCESS_HELPER);
 	if (err)
@@ -8526,11 +8323,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 	    base_type(arg_type) == ARG_PTR_TO_SPIN_LOCK)
 		arg_btf_id = fn->arg_btf_id[arg];
 
-	err = check_reg_type(env, regno, arg_type, arg_btf_id, meta);
+	err = check_reg_type(env, reg, argno_from_reg(regno), arg_type, arg_btf_id, meta);
 	if (err)
 		return err;
 
-	err = check_func_arg_reg_off(env, reg, regno, arg_type);
+	err = check_func_arg_reg_off(env, reg, argno_from_reg(regno), arg_type);
 	if (err)
 		return err;
 
@@ -8619,7 +8416,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 			return -EFAULT;
 		}
 		key_size = meta->map.ptr->key_size;
-		err = check_helper_mem_access(env, regno, key_size, BPF_READ, false, NULL);
+		err = check_helper_mem_access(env, reg, argno_from_reg(regno), key_size, BPF_READ, false, NULL);
 		if (err)
 			return err;
 		if (can_elide_value_nullness(meta->map.ptr->map_type)) {
@@ -8646,7 +8443,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 			return -EFAULT;
 		}
 		meta->raw_mode = arg_type & MEM_UNINIT;
-		err = check_helper_mem_access(env, regno, meta->map.ptr->value_size,
+		err = check_helper_mem_access(env, reg, argno_from_reg(regno), meta->map.ptr->value_size,
 					      arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
 					      false, meta);
 		break;
@@ -8664,11 +8461,11 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 			return -EACCES;
 		}
 		if (meta->func_id == BPF_FUNC_spin_lock) {
-			err = process_spin_lock(env, regno, PROCESS_SPIN_LOCK);
+			err = process_spin_lock(env, reg, argno_from_reg(regno), PROCESS_SPIN_LOCK);
 			if (err)
 				return err;
 		} else if (meta->func_id == BPF_FUNC_spin_unlock) {
-			err = process_spin_lock(env, regno, 0);
+			err = process_spin_lock(env, reg, argno_from_reg(regno), 0);
 			if (err)
 				return err;
 		} else {
@@ -8677,7 +8474,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 		}
 		break;
 	case ARG_PTR_TO_TIMER:
-		err = process_timer_helper(env, regno, meta);
+		err = process_timer_helper(env, reg, argno_from_reg(regno), meta);
 		if (err)
 			return err;
 		break;
@@ -8690,7 +8487,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 		 */
 		meta->raw_mode = arg_type & MEM_UNINIT;
 		if (arg_type & MEM_FIXED_SIZE) {
-			err = check_helper_mem_access(env, regno, fn->arg_size[arg],
+			err = check_helper_mem_access(env, reg, argno_from_reg(regno), fn->arg_size[arg],
 						      arg_type & MEM_WRITE ? BPF_WRITE : BPF_READ,
 						      false, meta);
 			if (err)
@@ -8700,19 +8497,21 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 		}
 		break;
 	case ARG_CONST_SIZE:
-		err = check_mem_size_reg(env, reg, regno,
+		err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1),
+					 argno_from_reg(regno),
 					 fn->arg_type[arg - 1] & MEM_WRITE ?
 					 BPF_WRITE : BPF_READ,
 					 false, meta);
 		break;
 	case ARG_CONST_SIZE_OR_ZERO:
-		err = check_mem_size_reg(env, reg, regno,
+		err = check_mem_size_reg(env, reg_state(env, regno - 1), reg, argno_from_reg(regno - 1),
+					 argno_from_reg(regno),
 					 fn->arg_type[arg - 1] & MEM_WRITE ?
 					 BPF_WRITE : BPF_READ,
 					 true, meta);
 		break;
 	case ARG_PTR_TO_DYNPTR:
-		err = process_dynptr_func(env, regno, insn_idx, arg_type, 0);
+		err = process_dynptr_func(env, reg, argno_from_reg(regno), insn_idx, arg_type, 0);
 		if (err)
 			return err;
 		break;
@@ -8729,7 +8528,7 @@ static int check_func_arg(struct bpf_verifier_env *env, u32 arg,
 		break;
 	case ARG_PTR_TO_CONST_STR:
 	{
-		err = check_reg_const_str(env, reg, regno);
+		err = check_arg_const_str(env, reg, argno_from_reg(regno));
 		if (err)
 			return err;
 		break;
@@ -9244,6 +9043,15 @@ static void clear_caller_saved_regs(struct bpf_verifier_env *env,
 	}
 }
 
+static void invalidate_outgoing_stack_args(const struct bpf_verifier_env *env,
+					   struct bpf_func_state *state)
+{
+	int i, nslots = state->out_stack_arg_cnt;
+
+	for (i = 0; i < nslots; i++)
+		bpf_mark_reg_not_init(env, &state->stack_arg_regs[i]);
+}
+
 typedef int (*set_callee_state_fn)(struct bpf_verifier_env *env,
 				   struct bpf_func_state *caller,
 				   struct bpf_func_state *callee,
@@ -9306,11 +9114,22 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
 				    struct bpf_reg_state *regs)
 {
 	struct bpf_subprog_info *sub = subprog_info(env, subprog);
+	struct bpf_func_state *caller = cur_func(env);
 	struct bpf_verifier_log *log = &env->log;
 	u32 i;
-	int ret;
+	int ret, err;
 
 	ret = btf_prepare_func_args(env, subprog);
+	if (ret) {
+		if (bpf_in_stack_arg_cnt(sub) > 0) {
+			err = check_outgoing_stack_args(env, caller, sub->arg_cnt);
+			if (err)
+				return err;
+		}
+		return ret;
+	}
+
+	ret = check_outgoing_stack_args(env, caller, sub->arg_cnt);
 	if (ret)
 		return ret;
 
@@ -9318,13 +9137,13 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
 	 * verifier sees.
 	 */
 	for (i = 0; i < sub->arg_cnt; i++) {
-		u32 regno = i + 1;
-		struct bpf_reg_state *reg = &regs[regno];
+		argno_t argno = argno_from_arg(i + 1);
+		struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i);
 		struct bpf_subprog_arg_info *arg = &sub->args[i];
 
 		if (arg->arg_type == ARG_ANYTHING) {
 			if (reg->type != SCALAR_VALUE) {
-				bpf_log(log, "R%d is not a scalar\n", regno);
+				bpf_log(log, "%s is not a scalar\n", reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 		} else if (arg->arg_type & PTR_UNTRUSTED) {
@@ -9334,24 +9153,26 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
 			 * invalid memory access.
 			 */
 		} else if (arg->arg_type == ARG_PTR_TO_CTX) {
-			ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_CTX);
+			ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_CTX);
 			if (ret < 0)
 				return ret;
 			/* If function expects ctx type in BTF check that caller
 			 * is passing PTR_TO_CTX.
 			 */
 			if (reg->type != PTR_TO_CTX) {
-				bpf_log(log, "arg#%d expects pointer to ctx\n", i);
+				bpf_log(log, "%s expects pointer to ctx\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 		} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
-			ret = check_func_arg_reg_off(env, reg, regno, ARG_DONTCARE);
+			ret = check_func_arg_reg_off(env, reg, argno, ARG_DONTCARE);
 			if (ret < 0)
 				return ret;
-			if (check_mem_reg(env, reg, regno, arg->mem_size))
+			if (check_mem_reg(env, reg, argno, arg->mem_size))
 				return -EINVAL;
 			if (!(arg->arg_type & PTR_MAYBE_NULL) && (reg->type & PTR_MAYBE_NULL)) {
-				bpf_log(log, "arg#%d is expected to be non-NULL\n", i);
+				bpf_log(log, "%s is expected to be non-NULL\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 		} else if (base_type(arg->arg_type) == ARG_PTR_TO_ARENA) {
@@ -9363,15 +9184,16 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
 			 * run-time debug nightmare.
 			 */
 			if (reg->type != PTR_TO_ARENA && reg->type != SCALAR_VALUE) {
-				bpf_log(log, "R%d is not a pointer to arena or scalar.\n", regno);
+				bpf_log(log, "%s is not a pointer to arena or scalar.\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
-		} else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) {
-			ret = check_func_arg_reg_off(env, reg, regno, ARG_PTR_TO_DYNPTR);
+		} else if (arg->arg_type == ARG_PTR_TO_DYNPTR) {
+			ret = check_func_arg_reg_off(env, reg, argno, ARG_PTR_TO_DYNPTR);
 			if (ret)
 				return ret;
 
-			ret = process_dynptr_func(env, regno, -1, arg->arg_type, 0);
+			ret = process_dynptr_func(env, reg, argno, -1, arg->arg_type, 0);
 			if (ret)
 				return ret;
 		} else if (base_type(arg->arg_type) == ARG_PTR_TO_BTF_ID) {
@@ -9382,12 +9204,13 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog,
 				continue;
 
 			memset(&meta, 0, sizeof(meta)); /* leave func_id as zero */
-			err = check_reg_type(env, regno, arg->arg_type, &arg->btf_id, &meta);
-			err = err ?: check_func_arg_reg_off(env, reg, regno, arg->arg_type);
+			err = check_reg_type(env, reg, argno, arg->arg_type, &arg->btf_id, &meta);
+			err = err ?: check_func_arg_reg_off(env, reg, argno, arg->arg_type);
 			if (err)
 				return err;
 		} else {
-			verifier_bug(env, "unrecognized arg#%d type %d", i, arg->arg_type);
+			verifier_bug(env, "unrecognized %s type %d",
+				     reg_arg_name(env, argno), arg->arg_type);
 			return -EFAULT;
 		}
 	}
@@ -9499,10 +9322,15 @@ static int push_callback_call(struct bpf_verifier_env *env, struct bpf_insn *ins
 	return 0;
 }
 
+static int process_bpf_exit_full(struct bpf_verifier_env *env,
+				 bool *do_print_state, bool exception_exit);
+
 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			   int *insn_idx)
 {
 	struct bpf_verifier_state *state = env->cur_state;
+	struct bpf_subprog_info *caller_info;
+	u16 callee_incoming, stack_arg_cnt;
 	struct bpf_func_state *caller;
 	int err, subprog, target_insn;
 
@@ -9545,6 +9373,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		/* mark global subprog for verifying after main prog */
 		subprog_aux(env, subprog)->called = true;
 		clear_caller_saved_regs(env, caller->regs);
+		invalidate_outgoing_stack_args(env, cur_func(env));
 
 		/* All non-void global functions return a 64-bit SCALAR_VALUE. */
 		if (!subprog_returns_void(env, subprog)) {
@@ -9552,10 +9381,31 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			caller->regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
 		}
 
+		if (env->subprog_info[subprog].might_throw) {
+			struct bpf_verifier_state *branch;
+
+			branch = push_stack(env, *insn_idx + 1, *insn_idx, false);
+			if (IS_ERR(branch)) {
+				verbose(env, "failed to push state for global subprog exception path\n");
+				return PTR_ERR(branch);
+			}
+			return process_bpf_exit_full(env, NULL, true);
+		}
+
 		/* continue with next insn after call */
 		return 0;
 	}
 
+	/*
+	 * Track caller's total stack arg count (incoming + max outgoing).
+	 * This is needed so the JIT knows how much stack arg space to allocate.
+	 */
+	caller_info = &env->subprog_info[caller->subprogno];
+	callee_incoming = bpf_in_stack_arg_cnt(&env->subprog_info[subprog]);
+	stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + callee_incoming;
+	if (stack_arg_cnt > caller_info->stack_arg_cnt)
+		caller_info->stack_arg_cnt = stack_arg_cnt;
+
 	/* for regular function entry setup new frame and continue
 	 * from that frame.
 	 */
@@ -9839,9 +9689,9 @@ static bool in_rbtree_lock_required_cb(struct bpf_verifier_env *env)
 static bool retval_range_within(struct bpf_retval_range range, const struct bpf_reg_state *reg)
 {
 	if (range.return_32bit)
-		return range.minval <= reg->s32_min_value && reg->s32_max_value <= range.maxval;
+		return range.minval <= reg_s32_min(reg) && reg_s32_max(reg) <= range.maxval;
 	else
-		return range.minval <= reg->smin_value && reg->smax_value <= range.maxval;
+		return range.minval <= reg_smin(reg) && reg_smax(reg) <= range.maxval;
 }
 
 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
@@ -9913,6 +9763,7 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
 	 * bpf_throw, this will be done by copy_verifier_state for extra frames. */
 	free_func_state(callee);
 	state->frame[state->curframe--] = NULL;
+	invalidate_outgoing_stack_args(env, caller);
 
 	/* for callbacks widen imprecise scalars to make programs like below verify:
 	 *
@@ -9950,21 +9801,13 @@ static int do_refine_retval_range(struct bpf_verifier_env *env,
 	case BPF_FUNC_probe_read_str:
 	case BPF_FUNC_probe_read_kernel_str:
 	case BPF_FUNC_probe_read_user_str:
-		ret_reg->smax_value = meta->msize_max_value;
-		ret_reg->s32_max_value = meta->msize_max_value;
-		ret_reg->smin_value = -MAX_ERRNO;
-		ret_reg->s32_min_value = -MAX_ERRNO;
+		reg_set_srange64(ret_reg, -MAX_ERRNO, meta->msize_max_value);
+		reg_set_srange32(ret_reg, -MAX_ERRNO, meta->msize_max_value);
 		reg_bounds_sync(ret_reg);
 		break;
 	case BPF_FUNC_get_smp_processor_id:
-		ret_reg->umax_value = nr_cpu_ids - 1;
-		ret_reg->u32_max_value = nr_cpu_ids - 1;
-		ret_reg->smax_value = nr_cpu_ids - 1;
-		ret_reg->s32_max_value = nr_cpu_ids - 1;
-		ret_reg->umin_value = 0;
-		ret_reg->u32_min_value = 0;
-		ret_reg->smin_value = 0;
-		ret_reg->s32_min_value = 0;
+		reg_set_urange64(ret_reg, 0, nr_cpu_ids - 1);
+		reg_set_urange32(ret_reg, 0, nr_cpu_ids - 1);
 		reg_bounds_sync(ret_reg);
 		break;
 	}
@@ -10340,18 +10183,18 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 	if (err)
 		return err;
 
+	regs = cur_regs(env);
+
 	/* Mark slots with STACK_MISC in case of raw mode, stack offset
 	 * is inferred from register state.
 	 */
 	for (i = 0; i < meta.access_size; i++) {
-		err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
+		err = check_mem_access(env, insn_idx, regs + meta.regno, argno_from_reg(meta.regno), i, BPF_B,
 				       BPF_WRITE, -1, false, false);
 		if (err)
 			return err;
 	}
 
-	regs = cur_regs(env);
-
 	if (meta.release_regno) {
 		err = -EINVAL;
 		if (arg_type_is_dynptr(fn->arg_type[meta.release_regno - BPF_REG_1])) {
@@ -10429,7 +10272,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		err = mark_chain_precision(env, BPF_REG_1);
 		if (err)
 			return err;
-		if (cur_func(env)->callback_depth < regs[BPF_REG_1].umax_value) {
+		if (cur_func(env)->callback_depth < reg_umax(&regs[BPF_REG_1])) {
 			err = push_callback_call(env, insn, insn_idx, meta.subprogno,
 						 set_loop_callback_state);
 		} else {
@@ -10547,6 +10390,7 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn
 		bpf_mark_reg_not_init(env, &regs[caller_saved[i]]);
 		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
 	}
+	invalidate_outgoing_stack_args(env, cur_func(env));
 
 	/* helper call returns 64-bit value. */
 	regs[BPF_REG_0].subreg_def = DEF_NOT_SUBREG;
@@ -11263,7 +11107,11 @@ BTF_ID(func, bpf_task_work_schedule_resume)
 BTF_ID(func, bpf_arena_alloc_pages)
 BTF_ID(func, bpf_arena_free_pages)
 BTF_ID(func, bpf_arena_reserve_pages)
+#ifdef CONFIG_BPF_EVENTS
 BTF_ID(func, bpf_session_is_return)
+#else
+BTF_ID_UNUSED
+#endif
 BTF_ID(func, bpf_stream_vprintk)
 BTF_ID(func, bpf_stream_print_stack)
 
@@ -11351,15 +11199,12 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta)
 }
 
 static enum kfunc_ptr_arg_type
-get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
-		       struct bpf_kfunc_call_arg_meta *meta,
+get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_func_state *caller,
+		       struct bpf_reg_state *regs, struct bpf_kfunc_call_arg_meta *meta,
 		       const struct btf_type *t, const struct btf_type *ref_t,
 		       const char *ref_tname, const struct btf_param *args,
-		       int argno, int nargs)
+		       int arg, int nargs, argno_t argno, struct bpf_reg_state *reg)
 {
-	u32 regno = argno + 1;
-	struct bpf_reg_state *regs = cur_regs(env);
-	struct bpf_reg_state *reg = &regs[regno];
 	bool arg_mem_size = false;
 
 	if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] ||
@@ -11367,9 +11212,9 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
 	    meta->func_id == special_kfunc_list[KF_bpf_session_cookie])
 		return KF_ARG_PTR_TO_CTX;
 
-	if (argno + 1 < nargs &&
-	    (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1]) ||
-	     is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], &regs[regno + 1])))
+	if (arg + 1 < nargs &&
+	    (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1)) ||
+	     is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1], get_func_arg_reg(caller, regs, arg + 1))))
 		arg_mem_size = true;
 
 	/* In this function, we verify the kfunc's BTF as per the argument type,
@@ -11377,68 +11222,69 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
 	 * type to our caller. When a set of conditions hold in the BTF type of
 	 * arguments, we resolve it to a known kfunc_ptr_arg_type.
 	 */
-	if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), argno))
+	if (btf_is_prog_ctx_type(&env->log, meta->btf, t, resolve_prog_type(env->prog), arg))
 		return KF_ARG_PTR_TO_CTX;
 
-	if (is_kfunc_arg_nullable(meta->btf, &args[argno]) && bpf_register_is_null(reg) &&
+	if (is_kfunc_arg_nullable(meta->btf, &args[arg]) && bpf_register_is_null(reg) &&
 	    !arg_mem_size)
 		return KF_ARG_PTR_TO_NULL;
 
-	if (is_kfunc_arg_alloc_obj(meta->btf, &args[argno]))
+	if (is_kfunc_arg_alloc_obj(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_ALLOC_BTF_ID;
 
-	if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[argno]))
+	if (is_kfunc_arg_refcounted_kptr(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_REFCOUNTED_KPTR;
 
-	if (is_kfunc_arg_dynptr(meta->btf, &args[argno]))
+	if (is_kfunc_arg_dynptr(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_DYNPTR;
 
-	if (is_kfunc_arg_iter(meta, argno, &args[argno]))
+	if (is_kfunc_arg_iter(meta, arg, &args[arg]))
 		return KF_ARG_PTR_TO_ITER;
 
-	if (is_kfunc_arg_list_head(meta->btf, &args[argno]))
+	if (is_kfunc_arg_list_head(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_LIST_HEAD;
 
-	if (is_kfunc_arg_list_node(meta->btf, &args[argno]))
+	if (is_kfunc_arg_list_node(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_LIST_NODE;
 
-	if (is_kfunc_arg_rbtree_root(meta->btf, &args[argno]))
+	if (is_kfunc_arg_rbtree_root(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_RB_ROOT;
 
-	if (is_kfunc_arg_rbtree_node(meta->btf, &args[argno]))
+	if (is_kfunc_arg_rbtree_node(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_RB_NODE;
 
-	if (is_kfunc_arg_const_str(meta->btf, &args[argno]))
+	if (is_kfunc_arg_const_str(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_CONST_STR;
 
-	if (is_kfunc_arg_map(meta->btf, &args[argno]))
+	if (is_kfunc_arg_map(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_MAP;
 
-	if (is_kfunc_arg_wq(meta->btf, &args[argno]))
+	if (is_kfunc_arg_wq(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_WORKQUEUE;
 
-	if (is_kfunc_arg_timer(meta->btf, &args[argno]))
+	if (is_kfunc_arg_timer(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_TIMER;
 
-	if (is_kfunc_arg_task_work(meta->btf, &args[argno]))
+	if (is_kfunc_arg_task_work(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_TASK_WORK;
 
-	if (is_kfunc_arg_irq_flag(meta->btf, &args[argno]))
+	if (is_kfunc_arg_irq_flag(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_IRQ_FLAG;
 
-	if (is_kfunc_arg_res_spin_lock(meta->btf, &args[argno]))
+	if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_RES_SPIN_LOCK;
 
 	if ((base_type(reg->type) == PTR_TO_BTF_ID || reg2btf_ids[base_type(reg->type)])) {
 		if (!btf_type_is_struct(ref_t)) {
-			verbose(env, "kernel function %s args#%d pointer type %s %s is not supported\n",
-				meta->func_name, argno, btf_type_str(ref_t), ref_tname);
+			verbose(env, "kernel function %s %s pointer type %s %s is not supported\n",
+				meta->func_name, reg_arg_name(env, argno),
+				btf_type_str(ref_t), ref_tname);
 			return -EINVAL;
 		}
 		return KF_ARG_PTR_TO_BTF_ID;
 	}
 
-	if (is_kfunc_arg_callback(env, meta->btf, &args[argno]))
+	if (is_kfunc_arg_callback(env, meta->btf, &args[arg]))
 		return KF_ARG_PTR_TO_CALLBACK;
 
 	/* This is the catch all argument type of register types supported by
@@ -11448,8 +11294,9 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env,
 	 */
 	if (!btf_type_is_scalar(ref_t) && !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0) &&
 	    (arg_mem_size ? !btf_type_is_void(ref_t) : 1)) {
-		verbose(env, "arg#%d pointer type %s %s must point to %sscalar, or struct with scalar\n",
-			argno, btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : "");
+		verbose(env, "%s pointer type %s %s must point to %sscalar, or struct with scalar\n",
+			reg_arg_name(env, argno),
+			btf_type_str(ref_t), ref_tname, arg_mem_size ? "void, " : "");
 		return -EINVAL;
 	}
 	return arg_mem_size ? KF_ARG_PTR_TO_MEM_SIZE : KF_ARG_PTR_TO_MEM;
@@ -11460,7 +11307,7 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
 					const struct btf_type *ref_t,
 					const char *ref_tname, u32 ref_id,
 					struct bpf_kfunc_call_arg_meta *meta,
-					int argno)
+					int arg, argno_t argno)
 {
 	const struct btf_type *reg_ref_t;
 	bool strict_type_match = false;
@@ -11518,18 +11365,18 @@ static int process_kf_arg_ptr_to_btf_id(struct bpf_verifier_env *env,
 	 */
 	taking_projection = btf_is_projection_of(ref_tname, reg_ref_tname);
 	if (!taking_projection && !struct_same) {
-		verbose(env, "kernel function %s args#%d expected pointer to %s %s but R%d has a pointer to %s %s\n",
-			meta->func_name, argno, btf_type_str(ref_t), ref_tname, argno + 1,
+		verbose(env, "kernel function %s %s expected pointer to %s %s but %s has a pointer to %s %s\n",
+			meta->func_name, reg_arg_name(env, argno),
+			btf_type_str(ref_t), ref_tname, reg_arg_name(env, argno),
 			btf_type_str(reg_ref_t), reg_ref_tname);
 		return -EINVAL;
 	}
 	return 0;
 }
 
-static int process_irq_flag(struct bpf_verifier_env *env, int regno,
+static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state *reg, argno_t argno,
 			     struct bpf_kfunc_call_arg_meta *meta)
 {
-	struct bpf_reg_state *reg = reg_state(env, regno);
 	int err, kfunc_class = IRQ_NATIVE_KFUNC;
 	bool irq_save;
 
@@ -11550,11 +11397,13 @@ static int process_irq_flag(struct bpf_verifier_env *env, int regno,
 
 	if (irq_save) {
 		if (!is_irq_flag_reg_valid_uninit(env, reg)) {
-			verbose(env, "expected uninitialized irq flag as arg#%d\n", regno - 1);
+			verbose(env, "expected uninitialized irq flag as %s\n",
+				reg_arg_name(env, argno));
 			return -EINVAL;
 		}
 
-		err = check_mem_access(env, env->insn_idx, regno, 0, BPF_DW, BPF_WRITE, -1, false, false);
+		err = check_mem_access(env, env->insn_idx, reg, argno, 0, BPF_DW,
+				       BPF_WRITE, -1, false, false);
 		if (err)
 			return err;
 
@@ -11564,7 +11413,8 @@ static int process_irq_flag(struct bpf_verifier_env *env, int regno,
 	} else {
 		err = is_irq_flag_reg_valid_init(env, reg);
 		if (err) {
-			verbose(env, "expected an initialized irq flag as arg#%d\n", regno - 1);
+			verbose(env, "expected an initialized irq flag as %s\n",
+				reg_arg_name(env, argno));
 			return err;
 		}
 
@@ -11778,7 +11628,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id)
 	       is_task_work_add_kfunc(btf_id);
 }
 
-static bool is_bpf_throw_kfunc(struct bpf_insn *insn)
+bool bpf_is_throw_kfunc(struct bpf_insn *insn)
 {
 	return bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
 	       insn->imm == special_kfunc_list[KF_bpf_throw];
@@ -11855,7 +11705,7 @@ static bool check_kfunc_is_graph_node_api(struct bpf_verifier_env *env,
 
 static int
 __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
-				   struct bpf_reg_state *reg, u32 regno,
+				   struct bpf_reg_state *reg, argno_t argno,
 				   struct bpf_kfunc_call_arg_meta *meta,
 				   enum btf_field_type head_field_type,
 				   struct btf_field **head_field)
@@ -11876,8 +11726,8 @@ __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
 	head_type_name = btf_field_type_name(head_field_type);
 	if (!tnum_is_const(reg->var_off)) {
 		verbose(env,
-			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
-			regno, head_type_name);
+			"%s doesn't have constant offset. %s has to be at the constant offset\n",
+			reg_arg_name(env, argno), head_type_name);
 		return -EINVAL;
 	}
 
@@ -11905,24 +11755,24 @@ __process_kf_arg_ptr_to_graph_root(struct bpf_verifier_env *env,
 }
 
 static int process_kf_arg_ptr_to_list_head(struct bpf_verifier_env *env,
-					   struct bpf_reg_state *reg, u32 regno,
+					   struct bpf_reg_state *reg, argno_t argno,
 					   struct bpf_kfunc_call_arg_meta *meta)
 {
-	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_LIST_HEAD,
+	return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_LIST_HEAD,
 							  &meta->arg_list_head.field);
 }
 
 static int process_kf_arg_ptr_to_rbtree_root(struct bpf_verifier_env *env,
-					     struct bpf_reg_state *reg, u32 regno,
+					     struct bpf_reg_state *reg, argno_t argno,
 					     struct bpf_kfunc_call_arg_meta *meta)
 {
-	return __process_kf_arg_ptr_to_graph_root(env, reg, regno, meta, BPF_RB_ROOT,
+	return __process_kf_arg_ptr_to_graph_root(env, reg, argno, meta, BPF_RB_ROOT,
 							  &meta->arg_rbtree_root.field);
 }
 
 static int
 __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
-				   struct bpf_reg_state *reg, u32 regno,
+				   struct bpf_reg_state *reg, argno_t argno,
 				   struct bpf_kfunc_call_arg_meta *meta,
 				   enum btf_field_type head_field_type,
 				   enum btf_field_type node_field_type,
@@ -11944,8 +11794,8 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
 	node_type_name = btf_field_type_name(node_field_type);
 	if (!tnum_is_const(reg->var_off)) {
 		verbose(env,
-			"R%d doesn't have constant offset. %s has to be at the constant offset\n",
-			regno, node_type_name);
+			"%s doesn't have constant offset. %s has to be at the constant offset\n",
+			reg_arg_name(env, argno), node_type_name);
 		return -EINVAL;
 	}
 
@@ -11986,19 +11836,19 @@ __process_kf_arg_ptr_to_graph_node(struct bpf_verifier_env *env,
 }
 
 static int process_kf_arg_ptr_to_list_node(struct bpf_verifier_env *env,
-					   struct bpf_reg_state *reg, u32 regno,
+					   struct bpf_reg_state *reg, argno_t argno,
 					   struct bpf_kfunc_call_arg_meta *meta)
 {
-	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
+	return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
 						  BPF_LIST_HEAD, BPF_LIST_NODE,
 						  &meta->arg_list_head.field);
 }
 
 static int process_kf_arg_ptr_to_rbtree_node(struct bpf_verifier_env *env,
-					     struct bpf_reg_state *reg, u32 regno,
+					     struct bpf_reg_state *reg, argno_t argno,
 					     struct bpf_kfunc_call_arg_meta *meta)
 {
-	return __process_kf_arg_ptr_to_graph_node(env, reg, regno, meta,
+	return __process_kf_arg_ptr_to_graph_node(env, reg, argno, meta,
 						  BPF_RB_ROOT, BPF_RB_NODE,
 						  &meta->arg_rbtree_root.field);
 }
@@ -12029,6 +11879,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			    int insn_idx)
 {
 	const char *func_name = meta->func_name, *ref_tname;
+	struct bpf_func_state *caller = cur_func(env);
+	struct bpf_reg_state *regs = cur_regs(env);
 	const struct btf *btf = meta->btf;
 	const struct btf_param *args;
 	struct btf_record *rec;
@@ -12037,20 +11889,31 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 
 	args = (const struct btf_param *)(meta->func_proto + 1);
 	nargs = btf_type_vlen(meta->func_proto);
-	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
+	if (nargs > MAX_BPF_FUNC_ARGS) {
 		verbose(env, "Function %s has %d > %d args\n", func_name, nargs,
-			MAX_BPF_FUNC_REG_ARGS);
+			MAX_BPF_FUNC_ARGS);
 		return -EINVAL;
 	}
+	if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) {
+		verbose(env, "JIT does not support kfunc %s() with %d args\n",
+			func_name, nargs);
+		return -ENOTSUPP;
+	}
+
+	ret = check_outgoing_stack_args(env, caller, nargs);
+	if (ret)
+		return ret;
 
 	/* Check that BTF function arguments match actual types that the
 	 * verifier sees.
 	 */
 	for (i = 0; i < nargs; i++) {
-		struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[i + 1];
+		struct bpf_reg_state *reg = get_func_arg_reg(caller, regs, i);
 		const struct btf_type *t, *ref_t, *resolve_ret;
 		enum bpf_arg_type arg_type = ARG_DONTCARE;
-		u32 regno = i + 1, ref_id, type_size;
+		argno_t argno = argno_from_arg(i + 1);
+		int regno = reg_from_argno(argno);
+		u32 ref_id, type_size;
 		bool is_ret_buf_sz = false;
 		int kf_arg_type;
 
@@ -12060,6 +11923,11 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 				verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc");
 				return -EFAULT;
 			}
+			if (regno < 0) {
+				verbose(env, "%s prog->aux cannot be a stack argument\n",
+					reg_arg_name(env, argno));
+				return -EINVAL;
+			}
 			meta->arg_prog = true;
 			cur_aux(env)->arg_prog = regno;
 			continue;
@@ -12072,7 +11940,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 
 		if (btf_type_is_scalar(t)) {
 			if (reg->type != SCALAR_VALUE) {
-				verbose(env, "R%d is not a scalar\n", regno);
+				verbose(env, "%s is not a scalar\n", reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 
@@ -12082,10 +11950,14 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 					return -EFAULT;
 				}
 				if (!tnum_is_const(reg->var_off)) {
-					verbose(env, "R%d must be a known constant\n", regno);
+					verbose(env, "%s must be a known constant\n",
+						reg_arg_name(env, argno));
 					return -EINVAL;
 				}
-				ret = mark_chain_precision(env, regno);
+				if (regno >= 0)
+					ret = mark_chain_precision(env, regno);
+				else
+					ret = mark_stack_arg_precision(env, i);
 				if (ret < 0)
 					return ret;
 				meta->arg_constant.found = true;
@@ -12104,12 +11976,16 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 				}
 
 				if (!tnum_is_const(reg->var_off)) {
-					verbose(env, "R%d is not a const\n", regno);
+					verbose(env, "%s is not a const\n",
+						reg_arg_name(env, argno));
 					return -EINVAL;
 				}
 
 				meta->r0_size = reg->var_off.value;
-				ret = mark_chain_precision(env, regno);
+				if (regno >= 0)
+					ret = mark_chain_precision(env, regno);
+				else
+					ret = mark_stack_arg_precision(env, i);
 				if (ret)
 					return ret;
 			}
@@ -12117,32 +11993,41 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 		}
 
 		if (!btf_type_is_ptr(t)) {
-			verbose(env, "Unrecognized arg#%d type %s\n", i, btf_type_str(t));
+			verbose(env, "Unrecognized %s type %s\n",
+				reg_arg_name(env, argno), btf_type_str(t));
 			return -EINVAL;
 		}
 
 		if ((bpf_register_is_null(reg) || type_may_be_null(reg->type)) &&
 		    !is_kfunc_arg_nullable(meta->btf, &args[i])) {
-			verbose(env, "Possibly NULL pointer passed to trusted arg%d\n", i);
+			verbose(env, "Possibly NULL pointer passed to trusted %s\n",
+				reg_arg_name(env, argno));
 			return -EACCES;
 		}
 
 		if (reg->ref_obj_id) {
 			if (is_kfunc_release(meta) && meta->ref_obj_id) {
-				verifier_bug(env, "more than one arg with ref_obj_id R%d %u %u",
-					     regno, reg->ref_obj_id,
+				verifier_bug(env, "more than one arg with ref_obj_id %s %u %u",
+					     reg_arg_name(env, argno), reg->ref_obj_id,
 					     meta->ref_obj_id);
 				return -EFAULT;
 			}
 			meta->ref_obj_id = reg->ref_obj_id;
-			if (is_kfunc_release(meta))
+			if (is_kfunc_release(meta)) {
+				if (regno < 0) {
+					verbose(env, "%s release arg cannot be a stack argument\n",
+						reg_arg_name(env, argno));
+					return -EINVAL;
+				}
 				meta->release_regno = regno;
+			}
 		}
 
 		ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id);
 		ref_tname = btf_name_by_offset(btf, ref_t->name_off);
 
-		kf_arg_type = get_kfunc_ptr_arg_type(env, meta, t, ref_t, ref_tname, args, i, nargs);
+		kf_arg_type = get_kfunc_ptr_arg_type(env, caller, regs, meta, t, ref_t, ref_tname,
+						     args, i, nargs, argno, reg);
 		if (kf_arg_type < 0)
 			return kf_arg_type;
 
@@ -12151,7 +12036,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			continue;
 		case KF_ARG_PTR_TO_MAP:
 			if (!reg->map_ptr) {
-				verbose(env, "pointer in R%d isn't map pointer\n", regno);
+				verbose(env, "pointer in %s isn't map pointer\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 			if (meta->map.ptr && (reg->map_ptr->record->wq_off >= 0 ||
@@ -12189,11 +12075,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 		case KF_ARG_PTR_TO_BTF_ID:
 			if (!is_trusted_reg(reg)) {
 				if (!is_kfunc_rcu(meta)) {
-					verbose(env, "R%d must be referenced or trusted\n", regno);
+					verbose(env, "%s must be referenced or trusted\n",
+						reg_arg_name(env, argno));
 					return -EINVAL;
 				}
 				if (!is_rcu_reg(reg)) {
-					verbose(env, "R%d must be a rcu pointer\n", regno);
+					verbose(env, "%s must be a rcu pointer\n",
+						reg_arg_name(env, argno));
 					return -EINVAL;
 				}
 			}
@@ -12225,15 +12113,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 
 		if (is_kfunc_release(meta) && reg->ref_obj_id)
 			arg_type |= OBJ_RELEASE;
-		ret = check_func_arg_reg_off(env, reg, regno, arg_type);
+		ret = check_func_arg_reg_off(env, reg, argno, arg_type);
 		if (ret < 0)
 			return ret;
 
 		switch (kf_arg_type) {
 		case KF_ARG_PTR_TO_CTX:
 			if (reg->type != PTR_TO_CTX) {
-				verbose(env, "arg#%d expected pointer to ctx, but got %s\n",
-					i, reg_type_str(env, reg->type));
+				verbose(env, "%s expected pointer to ctx, but got %s\n",
+					reg_arg_name(env, argno), reg_type_str(env, reg->type));
 				return -EINVAL;
 			}
 
@@ -12247,16 +12135,19 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 		case KF_ARG_PTR_TO_ALLOC_BTF_ID:
 			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC)) {
 				if (!is_bpf_obj_drop_kfunc(meta->func_id)) {
-					verbose(env, "arg#%d expected for bpf_obj_drop()\n", i);
+					verbose(env, "%s expected for bpf_obj_drop()\n",
+						reg_arg_name(env, argno));
 					return -EINVAL;
 				}
 			} else if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC | MEM_PERCPU)) {
 				if (!is_bpf_percpu_obj_drop_kfunc(meta->func_id)) {
-					verbose(env, "arg#%d expected for bpf_percpu_obj_drop()\n", i);
+					verbose(env, "%s expected for bpf_percpu_obj_drop()\n",
+						reg_arg_name(env, argno));
 					return -EINVAL;
 				}
 			} else {
-				verbose(env, "arg#%d expected pointer to allocated object\n", i);
+				verbose(env, "%s expected pointer to allocated object\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 			if (!reg->ref_obj_id) {
@@ -12273,9 +12164,6 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			enum bpf_arg_type dynptr_arg_type = ARG_PTR_TO_DYNPTR;
 			int clone_ref_obj_id = 0;
 
-			if (reg->type == CONST_PTR_TO_DYNPTR)
-				dynptr_arg_type |= MEM_RDONLY;
-
 			if (is_kfunc_arg_uninit(btf, &args[i]))
 				dynptr_arg_type |= MEM_UNINIT;
 
@@ -12288,7 +12176,12 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_from_file]) {
 				dynptr_arg_type |= DYNPTR_TYPE_FILE;
 			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) {
-				dynptr_arg_type |= DYNPTR_TYPE_FILE;
+				dynptr_arg_type |= DYNPTR_TYPE_FILE | OBJ_RELEASE;
+				if (regno < 0) {
+					verbose(env, "%s release arg cannot be a stack argument\n",
+						reg_arg_name(env, argno));
+					return -EINVAL;
+				}
 				meta->release_regno = regno;
 			} else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] &&
 				   (dynptr_arg_type & MEM_UNINIT)) {
@@ -12307,7 +12200,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 				}
 			}
 
-			ret = process_dynptr_func(env, regno, insn_idx, dynptr_arg_type, clone_ref_obj_id);
+			ret = process_dynptr_func(env, reg, argno, insn_idx,
+						  dynptr_arg_type, clone_ref_obj_id);
 			if (ret < 0)
 				return ret;
 
@@ -12332,55 +12226,59 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 					return -EINVAL;
 				}
 			}
-			ret = process_iter_arg(env, regno, insn_idx, meta);
+			ret = process_iter_arg(env, reg, argno, insn_idx, meta);
 			if (ret < 0)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_LIST_HEAD:
 			if (reg->type != PTR_TO_MAP_VALUE &&
 			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
+				verbose(env, "%s expected pointer to map value or allocated object\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
 				verbose(env, "allocated object must be referenced\n");
 				return -EINVAL;
 			}
-			ret = process_kf_arg_ptr_to_list_head(env, reg, regno, meta);
+			ret = process_kf_arg_ptr_to_list_head(env, reg, argno, meta);
 			if (ret < 0)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_RB_ROOT:
 			if (reg->type != PTR_TO_MAP_VALUE &&
 			    reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-				verbose(env, "arg#%d expected pointer to map value or allocated object\n", i);
+				verbose(env, "%s expected pointer to map value or allocated object\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 			if (reg->type == (PTR_TO_BTF_ID | MEM_ALLOC) && !reg->ref_obj_id) {
 				verbose(env, "allocated object must be referenced\n");
 				return -EINVAL;
 			}
-			ret = process_kf_arg_ptr_to_rbtree_root(env, reg, regno, meta);
+			ret = process_kf_arg_ptr_to_rbtree_root(env, reg, argno, meta);
 			if (ret < 0)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_LIST_NODE:
 			if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-				verbose(env, "arg#%d expected pointer to allocated object\n", i);
+				verbose(env, "%s expected pointer to allocated object\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 			if (!reg->ref_obj_id) {
 				verbose(env, "allocated object must be referenced\n");
 				return -EINVAL;
 			}
-			ret = process_kf_arg_ptr_to_list_node(env, reg, regno, meta);
+			ret = process_kf_arg_ptr_to_list_node(env, reg, argno, meta);
 			if (ret < 0)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_RB_NODE:
 			if (is_bpf_rbtree_add_kfunc(meta->func_id)) {
 				if (reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-					verbose(env, "arg#%d expected pointer to allocated object\n", i);
+					verbose(env, "%s expected pointer to allocated object\n",
+						reg_arg_name(env, argno));
 					return -EINVAL;
 				}
 				if (!reg->ref_obj_id) {
@@ -12398,7 +12296,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 				}
 			}
 
-			ret = process_kf_arg_ptr_to_rbtree_node(env, reg, regno, meta);
+			ret = process_kf_arg_ptr_to_rbtree_node(env, reg, argno, meta);
 			if (ret < 0)
 				return ret;
 			break;
@@ -12413,38 +12311,44 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			if ((base_type(reg->type) != PTR_TO_BTF_ID ||
 			     (bpf_type_has_unsafe_modifiers(reg->type) && !is_rcu_reg(reg))) &&
 			    !reg2btf_ids[base_type(reg->type)]) {
-				verbose(env, "arg#%d is %s ", i, reg_type_str(env, reg->type));
+				verbose(env, "%s is %s ", reg_arg_name(env, argno),
+					reg_type_str(env, reg->type));
 				verbose(env, "expected %s or socket\n",
 					reg_type_str(env, base_type(reg->type) |
 							  (type_flag(reg->type) & BPF_REG_TRUSTED_MODIFIERS)));
 				return -EINVAL;
 			}
-			ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i);
+			ret = process_kf_arg_ptr_to_btf_id(env, reg, ref_t, ref_tname, ref_id, meta, i, argno);
 			if (ret < 0)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_MEM:
 			resolve_ret = btf_resolve_size(btf, ref_t, &type_size);
 			if (IS_ERR(resolve_ret)) {
-				verbose(env, "arg#%d reference type('%s %s') size cannot be determined: %ld\n",
-					i, btf_type_str(ref_t), ref_tname, PTR_ERR(resolve_ret));
+				verbose(env, "%s reference type('%s %s') size cannot be determined: %ld\n",
+					reg_arg_name(env, argno), btf_type_str(ref_t),
+					ref_tname, PTR_ERR(resolve_ret));
 				return -EINVAL;
 			}
-			ret = check_mem_reg(env, reg, regno, type_size);
+			ret = check_mem_reg(env, reg, argno, type_size);
 			if (ret < 0)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_MEM_SIZE:
 		{
-			struct bpf_reg_state *buff_reg = &regs[regno];
+			struct bpf_reg_state *buff_reg = reg;
 			const struct btf_param *buff_arg = &args[i];
-			struct bpf_reg_state *size_reg = &regs[regno + 1];
+			struct bpf_reg_state *size_reg = get_func_arg_reg(caller, regs, i + 1);
 			const struct btf_param *size_arg = &args[i + 1];
+			argno_t next_argno = argno_from_arg(i + 2);
 
 			if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) {
-				ret = check_kfunc_mem_size_reg(env, size_reg, regno + 1);
+				ret = check_kfunc_mem_size_reg(env, buff_reg, size_reg,
+							       argno, next_argno);
 				if (ret < 0) {
-					verbose(env, "arg#%d arg#%d memory, len pair leads to invalid memory access\n", i, i + 1);
+					verbose(env, "%s and ", reg_arg_name(env, argno));
+					verbose(env, "%s memory, len pair leads to invalid memory access\n",
+						reg_arg_name(env, next_argno));
 					return ret;
 				}
 			}
@@ -12455,7 +12359,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 					return -EFAULT;
 				}
 				if (!tnum_is_const(size_reg->var_off)) {
-					verbose(env, "R%d must be a known constant\n", regno + 1);
+					verbose(env, "%s must be a known constant\n",
+						reg_arg_name(env, next_argno));
 					return -EINVAL;
 				}
 				meta->arg_constant.found = true;
@@ -12468,14 +12373,15 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 		}
 		case KF_ARG_PTR_TO_CALLBACK:
 			if (reg->type != PTR_TO_FUNC) {
-				verbose(env, "arg%d expected pointer to func\n", i);
+				verbose(env, "%s expected pointer to func\n", reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 			meta->subprogno = reg->subprogno;
 			break;
 		case KF_ARG_PTR_TO_REFCOUNTED_KPTR:
 			if (!type_is_ptr_alloc_obj(reg->type)) {
-				verbose(env, "arg#%d is neither owning or non-owning ref\n", i);
+				verbose(env, "%s is neither owning or non-owning ref\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 			if (!type_is_non_owning_ref(reg->type))
@@ -12488,7 +12394,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			}
 
 			if (rec->refcount_off < 0) {
-				verbose(env, "arg#%d doesn't point to a type with bpf_refcount field\n", i);
+				verbose(env, "%s doesn't point to a type with bpf_refcount field\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 
@@ -12497,46 +12404,51 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			break;
 		case KF_ARG_PTR_TO_CONST_STR:
 			if (reg->type != PTR_TO_MAP_VALUE) {
-				verbose(env, "arg#%d doesn't point to a const string\n", i);
+				verbose(env, "%s doesn't point to a const string\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
-			ret = check_reg_const_str(env, reg, regno);
+			ret = check_arg_const_str(env, reg, argno);
 			if (ret)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_WORKQUEUE:
 			if (reg->type != PTR_TO_MAP_VALUE) {
-				verbose(env, "arg#%d doesn't point to a map value\n", i);
+				verbose(env, "%s doesn't point to a map value\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
-			ret = check_map_field_pointer(env, regno, BPF_WORKQUEUE, &meta->map);
+			ret = check_map_field_pointer(env, reg, argno, BPF_WORKQUEUE, &meta->map);
 			if (ret < 0)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_TIMER:
 			if (reg->type != PTR_TO_MAP_VALUE) {
-				verbose(env, "arg#%d doesn't point to a map value\n", i);
+				verbose(env, "%s doesn't point to a map value\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
-			ret = process_timer_kfunc(env, regno, meta);
+			ret = process_timer_kfunc(env, reg, argno, meta);
 			if (ret < 0)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_TASK_WORK:
 			if (reg->type != PTR_TO_MAP_VALUE) {
-				verbose(env, "arg#%d doesn't point to a map value\n", i);
+				verbose(env, "%s doesn't point to a map value\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
-			ret = check_map_field_pointer(env, regno, BPF_TASK_WORK, &meta->map);
+			ret = check_map_field_pointer(env, reg, argno, BPF_TASK_WORK, &meta->map);
 			if (ret < 0)
 				return ret;
 			break;
 		case KF_ARG_PTR_TO_IRQ_FLAG:
 			if (reg->type != PTR_TO_STACK) {
-				verbose(env, "arg#%d doesn't point to an irq flag on stack\n", i);
+				verbose(env, "%s doesn't point to an irq flag on stack\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
-			ret = process_irq_flag(env, regno, meta);
+			ret = process_irq_flag(env, reg, argno, meta);
 			if (ret < 0)
 				return ret;
 			break;
@@ -12545,7 +12457,8 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			int flags = PROCESS_RES_LOCK;
 
 			if (reg->type != PTR_TO_MAP_VALUE && reg->type != (PTR_TO_BTF_ID | MEM_ALLOC)) {
-				verbose(env, "arg#%d doesn't point to map value or allocated object\n", i);
+				verbose(env, "%s doesn't point to map value or allocated object\n",
+					reg_arg_name(env, argno));
 				return -EINVAL;
 			}
 
@@ -12557,7 +12470,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_
 			if (meta->func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave] ||
 			    meta->func_id == special_kfunc_list[KF_bpf_res_spin_unlock_irqrestore])
 				flags |= PROCESS_LOCK_IRQ;
-			ret = process_spin_lock(env, regno, flags);
+			ret = process_spin_lock(env, reg, argno, flags);
 			if (ret < 0)
 				return ret;
 			break;
@@ -12968,8 +12881,6 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_kfunc_ca
 }
 
 static int check_return_code(struct bpf_verifier_env *env, int regno, const char *reg_name);
-static int process_bpf_exit_full(struct bpf_verifier_env *env,
-				 bool *do_print_state, bool exception_exit);
 
 static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 			    int *insn_idx_p)
@@ -13197,6 +13108,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		bpf_mark_reg_not_init(env, &regs[regno]);
 		regs[regno].subreg_def = DEF_NOT_SUBREG;
 	}
+	invalidate_outgoing_stack_args(env, cur_func(env));
 
 	/* Check return type */
 	t = btf_type_skip_modifiers(desc_btf, meta.func_proto->type, NULL);
@@ -13329,8 +13241,18 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		clear_all_pkt_pointers(env);
 
 	nargs = btf_type_vlen(meta.func_proto);
+	if (nargs > MAX_BPF_FUNC_REG_ARGS) {
+		struct bpf_func_state *caller = cur_func(env);
+		struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno];
+		u16 out_stack_arg_cnt = nargs - MAX_BPF_FUNC_REG_ARGS;
+		u16 stack_arg_cnt = bpf_in_stack_arg_cnt(caller_info) + out_stack_arg_cnt;
+
+		if (stack_arg_cnt > caller_info->stack_arg_cnt)
+			caller_info->stack_arg_cnt = stack_arg_cnt;
+	}
+
 	args = (const struct btf_param *)(meta.func_proto + 1);
-	for (i = 0; i < nargs; i++) {
+	for (i = 0; i < min_t(int, nargs, MAX_BPF_FUNC_REG_ARGS); i++) {
 		u32 regno = i + 1;
 
 		t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL);
@@ -13350,7 +13272,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	if (meta.func_id == special_kfunc_list[KF_bpf_session_cookie])
 		env->prog->call_session_cookie = true;
 
-	if (is_bpf_throw_kfunc(insn))
+	if (bpf_is_throw_kfunc(insn))
 		return process_bpf_exit_full(env, NULL, true);
 
 	return 0;
@@ -13362,7 +13284,7 @@ static bool check_reg_sane_offset_scalar(struct bpf_verifier_env *env,
 {
 	bool known = tnum_is_const(reg->var_off);
 	s64 val = reg->var_off.value;
-	s64 smin = reg->smin_value;
+	s64 smin = reg_smin(reg);
 
 	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
 		verbose(env, "math between %s pointer and %lld is not allowed\n",
@@ -13391,7 +13313,7 @@ static bool check_reg_sane_offset_ptr(struct bpf_verifier_env *env,
 {
 	bool known = tnum_is_const(reg->var_off);
 	s64 val = reg->var_off.value;
-	s64 smin = reg->smin_value;
+	s64 smin = reg_smin(reg);
 
 	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
 		verbose(env, "%s pointer offset %lld is not allowed\n",
@@ -13433,7 +13355,7 @@ static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
 		break;
 	case PTR_TO_MAP_VALUE:
 		max = ptr_reg->map_ptr->value_size;
-		ptr_limit = mask_to_left ? ptr_reg->smin_value : ptr_reg->umax_value;
+		ptr_limit = mask_to_left ? reg_smin(ptr_reg) : reg_umax(ptr_reg);
 		break;
 	default:
 		return REASON_TYPE;
@@ -13522,7 +13444,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
 	struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
 	struct bpf_verifier_state *vstate = env->cur_state;
 	bool off_is_imm = tnum_is_const(off_reg->var_off);
-	bool off_is_neg = off_reg->smin_value < 0;
+	bool off_is_neg = reg_smin(off_reg) < 0;
 	bool ptr_is_dst_reg = ptr_reg == dst_reg;
 	u8 opcode = BPF_OP(insn->code);
 	u32 alu_state, alu_limit;
@@ -13541,7 +13463,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
 
 	if (!commit_window) {
 		if (!tnum_is_const(off_reg->var_off) &&
-		    (off_reg->smin_value < 0) != (off_reg->smax_value < 0))
+		    (reg_smin(off_reg) < 0) != (reg_smax(off_reg) < 0))
 			return REASON_BOUNDS;
 
 		info->mask_to_left = (opcode == BPF_ADD &&  off_is_neg) ||
@@ -13597,7 +13519,7 @@ static int sanitize_ptr_alu(struct bpf_verifier_env *env,
 	 */
 	if (!ptr_is_dst_reg) {
 		tmp = *dst_reg;
-		copy_register_state(dst_reg, ptr_reg);
+		*dst_reg = *ptr_reg;
 	}
 	err = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx);
 	if (err < 0)
@@ -13691,7 +13613,7 @@ static int check_stack_access_for_ptr_arithmetic(
 
 static int sanitize_check_bounds(struct bpf_verifier_env *env,
 				 const struct bpf_insn *insn,
-				 const struct bpf_reg_state *dst_reg)
+				 struct bpf_reg_state *dst_reg)
 {
 	u32 dst = insn->dst_reg;
 
@@ -13708,7 +13630,7 @@ static int sanitize_check_bounds(struct bpf_verifier_env *env,
 			return -EACCES;
 		break;
 	case PTR_TO_MAP_VALUE:
-		if (check_map_access(env, dst, 0, 1, false, ACCESS_HELPER)) {
+		if (check_map_access(env, dst_reg, argno_from_reg(dst), 0, 1, false, ACCESS_HELPER)) {
 			verbose(env, "R%d pointer arithmetic of map value goes out of range, "
 				"prohibited for !root\n", dst);
 			return -EACCES;
@@ -13735,10 +13657,8 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 	struct bpf_func_state *state = vstate->frame[vstate->curframe];
 	struct bpf_reg_state *regs = state->regs, *dst_reg;
 	bool known = tnum_is_const(off_reg->var_off);
-	s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
-	    smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
-	u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
-	    umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
+	s64 smin_val = reg_smin(off_reg), smax_val = reg_smax(off_reg);
+	u64 umin_val = reg_umin(off_reg), umax_val = reg_umax(off_reg);
 	struct bpf_sanitize_info info = {};
 	u8 opcode = BPF_OP(insn->code);
 	u32 dst = insn->dst_reg;
@@ -13840,16 +13760,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 		 * added into the variable offset, and we copy the fixed offset
 		 * from ptr_reg.
 		 */
-		if (check_add_overflow(smin_ptr, smin_val, &dst_reg->smin_value) ||
-		    check_add_overflow(smax_ptr, smax_val, &dst_reg->smax_value)) {
-			dst_reg->smin_value = S64_MIN;
-			dst_reg->smax_value = S64_MAX;
-		}
-		if (check_add_overflow(umin_ptr, umin_val, &dst_reg->umin_value) ||
-		    check_add_overflow(umax_ptr, umax_val, &dst_reg->umax_value)) {
-			dst_reg->umin_value = 0;
-			dst_reg->umax_value = U64_MAX;
-		}
+		dst_reg->r64 = cnum64_add(ptr_reg->r64, off_reg->r64);
 		dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
 		dst_reg->raw = ptr_reg->raw;
 		if (reg_is_pkt_pointer(ptr_reg)) {
@@ -13881,24 +13792,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 				dst);
 			return -EACCES;
 		}
-		/* A new variable offset is created.  If the subtrahend is known
-		 * nonnegative, then any reg->range we had before is still good.
-		 */
-		if (check_sub_overflow(smin_ptr, smax_val, &dst_reg->smin_value) ||
-		    check_sub_overflow(smax_ptr, smin_val, &dst_reg->smax_value)) {
-			/* Overflow possible, we know nothing */
-			dst_reg->smin_value = S64_MIN;
-			dst_reg->smax_value = S64_MAX;
-		}
-		if (umin_ptr < umax_val) {
-			/* Overflow possible, we know nothing */
-			dst_reg->umin_value = 0;
-			dst_reg->umax_value = U64_MAX;
-		} else {
-			/* Cannot overflow (as long as bounds are consistent) */
-			dst_reg->umin_value = umin_ptr - umax_val;
-			dst_reg->umax_value = umax_ptr - umin_val;
-		}
+		dst_reg->r64 = cnum64_add(ptr_reg->r64, cnum64_negate(off_reg->r64));
 		dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
 		dst_reg->raw = ptr_reg->raw;
 		if (reg_is_pkt_pointer(ptr_reg)) {
@@ -13955,227 +13849,123 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
 static void scalar32_min_max_add(struct bpf_reg_state *dst_reg,
 				 struct bpf_reg_state *src_reg)
 {
-	s32 *dst_smin = &dst_reg->s32_min_value;
-	s32 *dst_smax = &dst_reg->s32_max_value;
-	u32 *dst_umin = &dst_reg->u32_min_value;
-	u32 *dst_umax = &dst_reg->u32_max_value;
-	u32 umin_val = src_reg->u32_min_value;
-	u32 umax_val = src_reg->u32_max_value;
-	bool min_overflow, max_overflow;
-
-	if (check_add_overflow(*dst_smin, src_reg->s32_min_value, dst_smin) ||
-	    check_add_overflow(*dst_smax, src_reg->s32_max_value, dst_smax)) {
-		*dst_smin = S32_MIN;
-		*dst_smax = S32_MAX;
-	}
-
-	/* If either all additions overflow or no additions overflow, then
-	 * it is okay to set: dst_umin = dst_umin + src_umin, dst_umax =
-	 * dst_umax + src_umax. Otherwise (some additions overflow), set
-	 * the output bounds to unbounded.
-	 */
-	min_overflow = check_add_overflow(*dst_umin, umin_val, dst_umin);
-	max_overflow = check_add_overflow(*dst_umax, umax_val, dst_umax);
-
-	if (!min_overflow && max_overflow) {
-		*dst_umin = 0;
-		*dst_umax = U32_MAX;
-	}
+	dst_reg->r32 = cnum32_add(dst_reg->r32, src_reg->r32);
 }
 
 static void scalar_min_max_add(struct bpf_reg_state *dst_reg,
 			       struct bpf_reg_state *src_reg)
 {
-	s64 *dst_smin = &dst_reg->smin_value;
-	s64 *dst_smax = &dst_reg->smax_value;
-	u64 *dst_umin = &dst_reg->umin_value;
-	u64 *dst_umax = &dst_reg->umax_value;
-	u64 umin_val = src_reg->umin_value;
-	u64 umax_val = src_reg->umax_value;
-	bool min_overflow, max_overflow;
-
-	if (check_add_overflow(*dst_smin, src_reg->smin_value, dst_smin) ||
-	    check_add_overflow(*dst_smax, src_reg->smax_value, dst_smax)) {
-		*dst_smin = S64_MIN;
-		*dst_smax = S64_MAX;
-	}
-
-	/* If either all additions overflow or no additions overflow, then
-	 * it is okay to set: dst_umin = dst_umin + src_umin, dst_umax =
-	 * dst_umax + src_umax. Otherwise (some additions overflow), set
-	 * the output bounds to unbounded.
-	 */
-	min_overflow = check_add_overflow(*dst_umin, umin_val, dst_umin);
-	max_overflow = check_add_overflow(*dst_umax, umax_val, dst_umax);
-
-	if (!min_overflow && max_overflow) {
-		*dst_umin = 0;
-		*dst_umax = U64_MAX;
-	}
+	dst_reg->r64 = cnum64_add(dst_reg->r64, src_reg->r64);
 }
 
 static void scalar32_min_max_sub(struct bpf_reg_state *dst_reg,
 				 struct bpf_reg_state *src_reg)
 {
-	s32 *dst_smin = &dst_reg->s32_min_value;
-	s32 *dst_smax = &dst_reg->s32_max_value;
-	u32 *dst_umin = &dst_reg->u32_min_value;
-	u32 *dst_umax = &dst_reg->u32_max_value;
-	u32 umin_val = src_reg->u32_min_value;
-	u32 umax_val = src_reg->u32_max_value;
-	bool min_underflow, max_underflow;
-
-	if (check_sub_overflow(*dst_smin, src_reg->s32_max_value, dst_smin) ||
-	    check_sub_overflow(*dst_smax, src_reg->s32_min_value, dst_smax)) {
-		/* Overflow possible, we know nothing */
-		*dst_smin = S32_MIN;
-		*dst_smax = S32_MAX;
-	}
-
-	/* If either all subtractions underflow or no subtractions
-	 * underflow, it is okay to set: dst_umin = dst_umin - src_umax,
-	 * dst_umax = dst_umax - src_umin. Otherwise (some subtractions
-	 * underflow), set the output bounds to unbounded.
-	 */
-	min_underflow = check_sub_overflow(*dst_umin, umax_val, dst_umin);
-	max_underflow = check_sub_overflow(*dst_umax, umin_val, dst_umax);
-
-	if (min_underflow && !max_underflow) {
-		*dst_umin = 0;
-		*dst_umax = U32_MAX;
-	}
+	dst_reg->r32 = cnum32_add(dst_reg->r32, cnum32_negate(src_reg->r32));
 }
 
 static void scalar_min_max_sub(struct bpf_reg_state *dst_reg,
 			       struct bpf_reg_state *src_reg)
 {
-	s64 *dst_smin = &dst_reg->smin_value;
-	s64 *dst_smax = &dst_reg->smax_value;
-	u64 *dst_umin = &dst_reg->umin_value;
-	u64 *dst_umax = &dst_reg->umax_value;
-	u64 umin_val = src_reg->umin_value;
-	u64 umax_val = src_reg->umax_value;
-	bool min_underflow, max_underflow;
-
-	if (check_sub_overflow(*dst_smin, src_reg->smax_value, dst_smin) ||
-	    check_sub_overflow(*dst_smax, src_reg->smin_value, dst_smax)) {
-		/* Overflow possible, we know nothing */
-		*dst_smin = S64_MIN;
-		*dst_smax = S64_MAX;
-	}
-
-	/* If either all subtractions underflow or no subtractions
-	 * underflow, it is okay to set: dst_umin = dst_umin - src_umax,
-	 * dst_umax = dst_umax - src_umin. Otherwise (some subtractions
-	 * underflow), set the output bounds to unbounded.
-	 */
-	min_underflow = check_sub_overflow(*dst_umin, umax_val, dst_umin);
-	max_underflow = check_sub_overflow(*dst_umax, umin_val, dst_umax);
-
-	if (min_underflow && !max_underflow) {
-		*dst_umin = 0;
-		*dst_umax = U64_MAX;
-	}
+	dst_reg->r64 = cnum64_add(dst_reg->r64, cnum64_negate(src_reg->r64));
 }
 
 static void scalar32_min_max_mul(struct bpf_reg_state *dst_reg,
 				 struct bpf_reg_state *src_reg)
 {
-	s32 *dst_smin = &dst_reg->s32_min_value;
-	s32 *dst_smax = &dst_reg->s32_max_value;
-	u32 *dst_umin = &dst_reg->u32_min_value;
-	u32 *dst_umax = &dst_reg->u32_max_value;
+	s32 smin = reg_s32_min(dst_reg);
+	s32 smax = reg_s32_max(dst_reg);
+	u32 umin = reg_u32_min(dst_reg);
+	u32 umax = reg_u32_max(dst_reg);
 	s32 tmp_prod[4];
 
-	if (check_mul_overflow(*dst_umax, src_reg->u32_max_value, dst_umax) ||
-	    check_mul_overflow(*dst_umin, src_reg->u32_min_value, dst_umin)) {
+	if (check_mul_overflow(umax, reg_u32_max(src_reg), &umax) ||
+	    check_mul_overflow(umin, reg_u32_min(src_reg), &umin)) {
 		/* Overflow possible, we know nothing */
-		*dst_umin = 0;
-		*dst_umax = U32_MAX;
+		umin = 0;
+		umax = U32_MAX;
 	}
-	if (check_mul_overflow(*dst_smin, src_reg->s32_min_value, &tmp_prod[0]) ||
-	    check_mul_overflow(*dst_smin, src_reg->s32_max_value, &tmp_prod[1]) ||
-	    check_mul_overflow(*dst_smax, src_reg->s32_min_value, &tmp_prod[2]) ||
-	    check_mul_overflow(*dst_smax, src_reg->s32_max_value, &tmp_prod[3])) {
+	if (check_mul_overflow(smin, reg_s32_min(src_reg), &tmp_prod[0]) ||
+	    check_mul_overflow(smin, reg_s32_max(src_reg), &tmp_prod[1]) ||
+	    check_mul_overflow(smax, reg_s32_min(src_reg), &tmp_prod[2]) ||
+	    check_mul_overflow(smax, reg_s32_max(src_reg), &tmp_prod[3])) {
 		/* Overflow possible, we know nothing */
-		*dst_smin = S32_MIN;
-		*dst_smax = S32_MAX;
+		smin = S32_MIN;
+		smax = S32_MAX;
 	} else {
-		*dst_smin = min_array(tmp_prod, 4);
-		*dst_smax = max_array(tmp_prod, 4);
+		smin = min_array(tmp_prod, 4);
+		smax = max_array(tmp_prod, 4);
 	}
+
+	dst_reg->r32 = cnum32_intersect(cnum32_from_urange(umin, umax),
+					cnum32_from_srange(smin, smax));
 }
 
 static void scalar_min_max_mul(struct bpf_reg_state *dst_reg,
 			       struct bpf_reg_state *src_reg)
 {
-	s64 *dst_smin = &dst_reg->smin_value;
-	s64 *dst_smax = &dst_reg->smax_value;
-	u64 *dst_umin = &dst_reg->umin_value;
-	u64 *dst_umax = &dst_reg->umax_value;
+	s64 smin = reg_smin(dst_reg);
+	s64 smax = reg_smax(dst_reg);
+	u64 umin = reg_umin(dst_reg);
+	u64 umax = reg_umax(dst_reg);
 	s64 tmp_prod[4];
 
-	if (check_mul_overflow(*dst_umax, src_reg->umax_value, dst_umax) ||
-	    check_mul_overflow(*dst_umin, src_reg->umin_value, dst_umin)) {
+	if (check_mul_overflow(umax, reg_umax(src_reg), &umax) ||
+	    check_mul_overflow(umin, reg_umin(src_reg), &umin)) {
 		/* Overflow possible, we know nothing */
-		*dst_umin = 0;
-		*dst_umax = U64_MAX;
+		umin = 0;
+		umax = U64_MAX;
 	}
-	if (check_mul_overflow(*dst_smin, src_reg->smin_value, &tmp_prod[0]) ||
-	    check_mul_overflow(*dst_smin, src_reg->smax_value, &tmp_prod[1]) ||
-	    check_mul_overflow(*dst_smax, src_reg->smin_value, &tmp_prod[2]) ||
-	    check_mul_overflow(*dst_smax, src_reg->smax_value, &tmp_prod[3])) {
+	if (check_mul_overflow(smin, reg_smin(src_reg), &tmp_prod[0]) ||
+	    check_mul_overflow(smin, reg_smax(src_reg), &tmp_prod[1]) ||
+	    check_mul_overflow(smax, reg_smin(src_reg), &tmp_prod[2]) ||
+	    check_mul_overflow(smax, reg_smax(src_reg), &tmp_prod[3])) {
 		/* Overflow possible, we know nothing */
-		*dst_smin = S64_MIN;
-		*dst_smax = S64_MAX;
+		smin = S64_MIN;
+		smax = S64_MAX;
 	} else {
-		*dst_smin = min_array(tmp_prod, 4);
-		*dst_smax = max_array(tmp_prod, 4);
+		smin = min_array(tmp_prod, 4);
+		smax = max_array(tmp_prod, 4);
 	}
+
+	dst_reg->r64 = cnum64_intersect(cnum64_from_urange(umin, umax),
+					cnum64_from_srange(smin, smax));
 }
 
 static void scalar32_min_max_udiv(struct bpf_reg_state *dst_reg,
 				  struct bpf_reg_state *src_reg)
 {
-	u32 *dst_umin = &dst_reg->u32_min_value;
-	u32 *dst_umax = &dst_reg->u32_max_value;
-	u32 src_val = src_reg->u32_min_value; /* non-zero, const divisor */
+	u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */
 
-	*dst_umin = *dst_umin / src_val;
-	*dst_umax = *dst_umax / src_val;
+	reg_set_urange32(dst_reg, reg_u32_min(dst_reg) / src_val,
+			 reg_u32_max(dst_reg) / src_val);
 
 	/* Reset other ranges/tnum to unbounded/unknown. */
-	dst_reg->s32_min_value = S32_MIN;
-	dst_reg->s32_max_value = S32_MAX;
 	reset_reg64_and_tnum(dst_reg);
 }
 
 static void scalar_min_max_udiv(struct bpf_reg_state *dst_reg,
 				struct bpf_reg_state *src_reg)
 {
-	u64 *dst_umin = &dst_reg->umin_value;
-	u64 *dst_umax = &dst_reg->umax_value;
-	u64 src_val = src_reg->umin_value; /* non-zero, const divisor */
+	u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */
 
-	*dst_umin = div64_u64(*dst_umin, src_val);
-	*dst_umax = div64_u64(*dst_umax, src_val);
+	reg_set_urange64(dst_reg, div64_u64(reg_umin(dst_reg), src_val),
+			 div64_u64(reg_umax(dst_reg), src_val));
 
 	/* Reset other ranges/tnum to unbounded/unknown. */
-	dst_reg->smin_value = S64_MIN;
-	dst_reg->smax_value = S64_MAX;
 	reset_reg32_and_tnum(dst_reg);
 }
 
 static void scalar32_min_max_sdiv(struct bpf_reg_state *dst_reg,
 				  struct bpf_reg_state *src_reg)
 {
-	s32 *dst_smin = &dst_reg->s32_min_value;
-	s32 *dst_smax = &dst_reg->s32_max_value;
-	s32 src_val = src_reg->s32_min_value; /* non-zero, const divisor */
+	s32 smin = reg_s32_min(dst_reg);
+	s32 smax = reg_s32_max(dst_reg);
+	s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */
 	s32 res1, res2;
 
 	/* BPF div specification: S32_MIN / -1 = S32_MIN */
-	if (*dst_smin == S32_MIN && src_val == -1) {
+	if (smin == S32_MIN && src_val == -1) {
 		/*
 		 * If the dividend range contains more than just S32_MIN,
 		 * we cannot precisely track the result, so it becomes unbounded.
@@ -14184,35 +13974,34 @@ static void scalar32_min_max_sdiv(struct bpf_reg_state *dst_reg,
 		 *     = {S32_MIN} U [S32_MAX-9, S32_MAX] = [S32_MIN, S32_MAX]
 		 * Otherwise (if dividend is exactly S32_MIN), result remains S32_MIN.
 		 */
-		if (*dst_smax != S32_MIN) {
-			*dst_smin = S32_MIN;
-			*dst_smax = S32_MAX;
+		if (smax != S32_MIN) {
+			smin = S32_MIN;
+			smax = S32_MAX;
 		}
 		goto reset;
 	}
 
-	res1 = *dst_smin / src_val;
-	res2 = *dst_smax / src_val;
-	*dst_smin = min(res1, res2);
-	*dst_smax = max(res1, res2);
+	res1 = smin / src_val;
+	res2 = smax / src_val;
+	smin = min(res1, res2);
+	smax = max(res1, res2);
 
 reset:
+	reg_set_srange32(dst_reg, smin, smax);
 	/* Reset other ranges/tnum to unbounded/unknown. */
-	dst_reg->u32_min_value = 0;
-	dst_reg->u32_max_value = U32_MAX;
 	reset_reg64_and_tnum(dst_reg);
 }
 
 static void scalar_min_max_sdiv(struct bpf_reg_state *dst_reg,
 				struct bpf_reg_state *src_reg)
 {
-	s64 *dst_smin = &dst_reg->smin_value;
-	s64 *dst_smax = &dst_reg->smax_value;
-	s64 src_val = src_reg->smin_value; /* non-zero, const divisor */
+	s64 smin = reg_smin(dst_reg);
+	s64 smax = reg_smax(dst_reg);
+	s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */
 	s64 res1, res2;
 
 	/* BPF div specification: S64_MIN / -1 = S64_MIN */
-	if (*dst_smin == S64_MIN && src_val == -1) {
+	if (smin == S64_MIN && src_val == -1) {
 		/*
 		 * If the dividend range contains more than just S64_MIN,
 		 * we cannot precisely track the result, so it becomes unbounded.
@@ -14221,79 +14010,66 @@ static void scalar_min_max_sdiv(struct bpf_reg_state *dst_reg,
 		 *     = {S64_MIN} U [S64_MAX-9, S64_MAX] = [S64_MIN, S64_MAX]
 		 * Otherwise (if dividend is exactly S64_MIN), result remains S64_MIN.
 		 */
-		if (*dst_smax != S64_MIN) {
-			*dst_smin = S64_MIN;
-			*dst_smax = S64_MAX;
+		if (smax != S64_MIN) {
+			smin = S64_MIN;
+			smax = S64_MAX;
 		}
 		goto reset;
 	}
 
-	res1 = div64_s64(*dst_smin, src_val);
-	res2 = div64_s64(*dst_smax, src_val);
-	*dst_smin = min(res1, res2);
-	*dst_smax = max(res1, res2);
+	res1 = div64_s64(smin, src_val);
+	res2 = div64_s64(smax, src_val);
+	smin = min(res1, res2);
+	smax = max(res1, res2);
 
 reset:
+	reg_set_srange64(dst_reg, smin, smax);
 	/* Reset other ranges/tnum to unbounded/unknown. */
-	dst_reg->umin_value = 0;
-	dst_reg->umax_value = U64_MAX;
 	reset_reg32_and_tnum(dst_reg);
 }
 
 static void scalar32_min_max_umod(struct bpf_reg_state *dst_reg,
 				  struct bpf_reg_state *src_reg)
 {
-	u32 *dst_umin = &dst_reg->u32_min_value;
-	u32 *dst_umax = &dst_reg->u32_max_value;
-	u32 src_val = src_reg->u32_min_value; /* non-zero, const divisor */
+	u32 src_val = reg_u32_min(src_reg); /* non-zero, const divisor */
 	u32 res_max = src_val - 1;
 
 	/*
 	 * If dst_umax <= res_max, the result remains unchanged.
 	 * e.g., [2, 5] % 10 = [2, 5].
 	 */
-	if (*dst_umax <= res_max)
+	if (reg_u32_max(dst_reg) <= res_max)
 		return;
 
-	*dst_umin = 0;
-	*dst_umax = min(*dst_umax, res_max);
+	reg_set_urange32(dst_reg, 0, min(reg_u32_max(dst_reg), res_max));
 
 	/* Reset other ranges/tnum to unbounded/unknown. */
-	dst_reg->s32_min_value = S32_MIN;
-	dst_reg->s32_max_value = S32_MAX;
 	reset_reg64_and_tnum(dst_reg);
 }
 
 static void scalar_min_max_umod(struct bpf_reg_state *dst_reg,
 				struct bpf_reg_state *src_reg)
 {
-	u64 *dst_umin = &dst_reg->umin_value;
-	u64 *dst_umax = &dst_reg->umax_value;
-	u64 src_val = src_reg->umin_value; /* non-zero, const divisor */
+	u64 src_val = reg_umin(src_reg); /* non-zero, const divisor */
 	u64 res_max = src_val - 1;
 
 	/*
 	 * If dst_umax <= res_max, the result remains unchanged.
 	 * e.g., [2, 5] % 10 = [2, 5].
 	 */
-	if (*dst_umax <= res_max)
+	if (reg_umax(dst_reg) <= res_max)
 		return;
 
-	*dst_umin = 0;
-	*dst_umax = min(*dst_umax, res_max);
+	reg_set_urange64(dst_reg, 0, min(reg_umax(dst_reg), res_max));
 
 	/* Reset other ranges/tnum to unbounded/unknown. */
-	dst_reg->smin_value = S64_MIN;
-	dst_reg->smax_value = S64_MAX;
 	reset_reg32_and_tnum(dst_reg);
 }
 
 static void scalar32_min_max_smod(struct bpf_reg_state *dst_reg,
 				  struct bpf_reg_state *src_reg)
 {
-	s32 *dst_smin = &dst_reg->s32_min_value;
-	s32 *dst_smax = &dst_reg->s32_max_value;
-	s32 src_val = src_reg->s32_min_value; /* non-zero, const divisor */
+	s32 src_val = reg_s32_min(src_reg); /* non-zero, const divisor */
 
 	/*
 	 * Safe absolute value calculation:
@@ -14313,33 +14089,26 @@ static void scalar32_min_max_smod(struct bpf_reg_state *dst_reg,
 	 * If the dividend is already within the result range,
 	 * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5].
 	 */
-	if (*dst_smin >= -res_max_abs && *dst_smax <= res_max_abs)
+	if (reg_s32_min(dst_reg) >= -res_max_abs && reg_s32_max(dst_reg) <= res_max_abs)
 		return;
 
 	/* General case: result has the same sign as the dividend. */
-	if (*dst_smin >= 0) {
-		*dst_smin = 0;
-		*dst_smax = min(*dst_smax, res_max_abs);
-	} else if (*dst_smax <= 0) {
-		*dst_smax = 0;
-		*dst_smin = max(*dst_smin, -res_max_abs);
+	if (reg_s32_min(dst_reg) >= 0) {
+		reg_set_srange32(dst_reg, 0, min(reg_s32_max(dst_reg), res_max_abs));
+	} else if (reg_s32_max(dst_reg) <= 0) {
+		reg_set_srange32(dst_reg, max(reg_s32_min(dst_reg), -res_max_abs), 0);
 	} else {
-		*dst_smin = -res_max_abs;
-		*dst_smax = res_max_abs;
+		reg_set_srange32(dst_reg, -res_max_abs, res_max_abs);
 	}
 
 	/* Reset other ranges/tnum to unbounded/unknown. */
-	dst_reg->u32_min_value = 0;
-	dst_reg->u32_max_value = U32_MAX;
 	reset_reg64_and_tnum(dst_reg);
 }
 
 static void scalar_min_max_smod(struct bpf_reg_state *dst_reg,
 				struct bpf_reg_state *src_reg)
 {
-	s64 *dst_smin = &dst_reg->smin_value;
-	s64 *dst_smax = &dst_reg->smax_value;
-	s64 src_val = src_reg->smin_value; /* non-zero, const divisor */
+	s64 src_val = reg_smin(src_reg); /* non-zero, const divisor */
 
 	/*
 	 * Safe absolute value calculation:
@@ -14359,24 +14128,19 @@ static void scalar_min_max_smod(struct bpf_reg_state *dst_reg,
 	 * If the dividend is already within the result range,
 	 * the result remains unchanged. e.g., [-2, 5] % 10 = [-2, 5].
 	 */
-	if (*dst_smin >= -res_max_abs && *dst_smax <= res_max_abs)
+	if (reg_smin(dst_reg) >= -res_max_abs && reg_smax(dst_reg) <= res_max_abs)
 		return;
 
 	/* General case: result has the same sign as the dividend. */
-	if (*dst_smin >= 0) {
-		*dst_smin = 0;
-		*dst_smax = min(*dst_smax, res_max_abs);
-	} else if (*dst_smax <= 0) {
-		*dst_smax = 0;
-		*dst_smin = max(*dst_smin, -res_max_abs);
+	if (reg_smin(dst_reg) >= 0) {
+		reg_set_srange64(dst_reg, 0, min(reg_smax(dst_reg), res_max_abs));
+	} else if (reg_smax(dst_reg) <= 0) {
+		reg_set_srange64(dst_reg, max(reg_smin(dst_reg), -res_max_abs), 0);
 	} else {
-		*dst_smin = -res_max_abs;
-		*dst_smax = res_max_abs;
+		reg_set_srange64(dst_reg, -res_max_abs, res_max_abs);
 	}
 
 	/* Reset other ranges/tnum to unbounded/unknown. */
-	dst_reg->umin_value = 0;
-	dst_reg->umax_value = U64_MAX;
 	reset_reg32_and_tnum(dst_reg);
 }
 
@@ -14386,7 +14150,7 @@ static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
-	u32 umax_val = src_reg->u32_max_value;
+	u32 umax_val = reg_u32_max(src_reg);
 
 	if (src_known && dst_known) {
 		__mark_reg32_known(dst_reg, var32_off.value);
@@ -14396,19 +14160,9 @@ static void scalar32_min_max_and(struct bpf_reg_state *dst_reg,
 	/* We get our minimum from the var_off, since that's inherently
 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
 	 */
-	dst_reg->u32_min_value = var32_off.value;
-	dst_reg->u32_max_value = min(dst_reg->u32_max_value, umax_val);
-
-	/* Safe to set s32 bounds by casting u32 result into s32 when u32
-	 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
-	 */
-	if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
-		dst_reg->s32_min_value = dst_reg->u32_min_value;
-		dst_reg->s32_max_value = dst_reg->u32_max_value;
-	} else {
-		dst_reg->s32_min_value = S32_MIN;
-		dst_reg->s32_max_value = S32_MAX;
-	}
+	reg_set_urange32(dst_reg,
+			 var32_off.value,
+			 min(reg_u32_max(dst_reg), umax_val));
 }
 
 static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
@@ -14416,7 +14170,7 @@ static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
 {
 	bool src_known = tnum_is_const(src_reg->var_off);
 	bool dst_known = tnum_is_const(dst_reg->var_off);
-	u64 umax_val = src_reg->umax_value;
+	u64 umax_val = reg_umax(src_reg);
 
 	if (src_known && dst_known) {
 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
@@ -14426,19 +14180,10 @@ static void scalar_min_max_and(struct bpf_reg_state *dst_reg,
 	/* We get our minimum from the var_off, since that's inherently
 	 * bitwise.  Our maximum is the minimum of the operands' maxima.
 	 */
-	dst_reg->umin_value = dst_reg->var_off.value;
-	dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
+	reg_set_urange64(dst_reg,
+			 dst_reg->var_off.value,
+			 min(reg_umax(dst_reg), umax_val));
 
-	/* Safe to set s64 bounds by casting u64 result into s64 when u64
-	 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
-	 */
-	if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
-		dst_reg->smin_value = dst_reg->umin_value;
-		dst_reg->smax_value = dst_reg->umax_value;
-	} else {
-		dst_reg->smin_value = S64_MIN;
-		dst_reg->smax_value = S64_MAX;
-	}
 	/* We may learn something more from the var_off */
 	__update_reg_bounds(dst_reg);
 }
@@ -14449,7 +14194,7 @@ static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
 	bool src_known = tnum_subreg_is_const(src_reg->var_off);
 	bool dst_known = tnum_subreg_is_const(dst_reg->var_off);
 	struct tnum var32_off = tnum_subreg(dst_reg->var_off);
-	u32 umin_val = src_reg->u32_min_value;
+	u32 umin_val = reg_u32_min(src_reg);
 
 	if (src_known && dst_known) {
 		__mark_reg32_known(dst_reg, var32_off.value);
@@ -14459,19 +14204,9 @@ static void scalar32_min_max_or(struct bpf_reg_state *dst_reg,
 	/* We get our maximum from the var_off, and our minimum is the
 	 * maximum of the operands' minima
 	 */
-	dst_reg->u32_min_value = max(dst_reg->u32_min_value, umin_val);
-	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
-
-	/* Safe to set s32 bounds by casting u32 result into s32 when u32
-	 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
-	 */
-	if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
-		dst_reg->s32_min_value = dst_reg->u32_min_value;
-		dst_reg->s32_max_value = dst_reg->u32_max_value;
-	} else {
-		dst_reg->s32_min_value = S32_MIN;
-		dst_reg->s32_max_value = S32_MAX;
-	}
+	reg_set_urange32(dst_reg,
+			 max(reg_u32_min(dst_reg), umin_val),
+			 var32_off.value | var32_off.mask);
 }
 
 static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
@@ -14479,7 +14214,7 @@ static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
 {
 	bool src_known = tnum_is_const(src_reg->var_off);
 	bool dst_known = tnum_is_const(dst_reg->var_off);
-	u64 umin_val = src_reg->umin_value;
+	u64 umin_val = reg_umin(src_reg);
 
 	if (src_known && dst_known) {
 		__mark_reg_known(dst_reg, dst_reg->var_off.value);
@@ -14489,19 +14224,10 @@ static void scalar_min_max_or(struct bpf_reg_state *dst_reg,
 	/* We get our maximum from the var_off, and our minimum is the
 	 * maximum of the operands' minima
 	 */
-	dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
-	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
+	reg_set_urange64(dst_reg,
+			 max(reg_umin(dst_reg), umin_val),
+			 dst_reg->var_off.value | dst_reg->var_off.mask);
 
-	/* Safe to set s64 bounds by casting u64 result into s64 when u64
-	 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
-	 */
-	if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
-		dst_reg->smin_value = dst_reg->umin_value;
-		dst_reg->smax_value = dst_reg->umax_value;
-	} else {
-		dst_reg->smin_value = S64_MIN;
-		dst_reg->smax_value = S64_MAX;
-	}
 	/* We may learn something more from the var_off */
 	__update_reg_bounds(dst_reg);
 }
@@ -14519,19 +14245,7 @@ static void scalar32_min_max_xor(struct bpf_reg_state *dst_reg,
 	}
 
 	/* We get both minimum and maximum from the var32_off. */
-	dst_reg->u32_min_value = var32_off.value;
-	dst_reg->u32_max_value = var32_off.value | var32_off.mask;
-
-	/* Safe to set s32 bounds by casting u32 result into s32 when u32
-	 * doesn't cross sign boundary. Otherwise set s32 bounds to unbounded.
-	 */
-	if ((s32)dst_reg->u32_min_value <= (s32)dst_reg->u32_max_value) {
-		dst_reg->s32_min_value = dst_reg->u32_min_value;
-		dst_reg->s32_max_value = dst_reg->u32_max_value;
-	} else {
-		dst_reg->s32_min_value = S32_MIN;
-		dst_reg->s32_max_value = S32_MAX;
-	}
+	reg_set_urange32(dst_reg, var32_off.value, var32_off.value | var32_off.mask);
 }
 
 static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
@@ -14547,46 +14261,30 @@ static void scalar_min_max_xor(struct bpf_reg_state *dst_reg,
 	}
 
 	/* We get both minimum and maximum from the var_off. */
-	dst_reg->umin_value = dst_reg->var_off.value;
-	dst_reg->umax_value = dst_reg->var_off.value | dst_reg->var_off.mask;
-
-	/* Safe to set s64 bounds by casting u64 result into s64 when u64
-	 * doesn't cross sign boundary. Otherwise set s64 bounds to unbounded.
-	 */
-	if ((s64)dst_reg->umin_value <= (s64)dst_reg->umax_value) {
-		dst_reg->smin_value = dst_reg->umin_value;
-		dst_reg->smax_value = dst_reg->umax_value;
-	} else {
-		dst_reg->smin_value = S64_MIN;
-		dst_reg->smax_value = S64_MAX;
-	}
-
-	__update_reg_bounds(dst_reg);
+	reg_set_urange64(dst_reg,
+			 dst_reg->var_off.value,
+			 dst_reg->var_off.value | dst_reg->var_off.mask);
 }
 
 static void __scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
 				   u64 umin_val, u64 umax_val)
 {
-	/* We lose all sign bit information (except what we can pick
-	 * up from var_off)
-	 */
-	dst_reg->s32_min_value = S32_MIN;
-	dst_reg->s32_max_value = S32_MAX;
 	/* If we might shift our top bit out, then we know nothing */
-	if (umax_val > 31 || dst_reg->u32_max_value > 1ULL << (31 - umax_val)) {
-		dst_reg->u32_min_value = 0;
-		dst_reg->u32_max_value = U32_MAX;
-	} else {
-		dst_reg->u32_min_value <<= umin_val;
-		dst_reg->u32_max_value <<= umax_val;
-	}
+	if (umax_val > 31 || reg_u32_max(dst_reg) > 1ULL << (31 - umax_val))
+		reg_set_urange32(dst_reg, 0, U32_MAX);
+	else
+		/* We lose all sign bit information (except what we can pick
+		 * up from var_off)
+		 */
+		reg_set_urange32(dst_reg, reg_u32_min(dst_reg) << umin_val,
+				 reg_u32_max(dst_reg) << umax_val);
 }
 
 static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
 				 struct bpf_reg_state *src_reg)
 {
-	u32 umax_val = src_reg->u32_max_value;
-	u32 umin_val = src_reg->u32_min_value;
+	u32 umax_val = reg_u32_max(src_reg);
+	u32 umin_val = reg_u32_min(src_reg);
 	/* u32 alu operation will zext upper bits */
 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
 
@@ -14603,34 +14301,34 @@ static void scalar32_min_max_lsh(struct bpf_reg_state *dst_reg,
 static void __scalar64_min_max_lsh(struct bpf_reg_state *dst_reg,
 				   u64 umin_val, u64 umax_val)
 {
+	struct cnum64 u, s;
+
 	/* Special case <<32 because it is a common compiler pattern to sign
 	 * extend subreg by doing <<32 s>>32. smin/smax assignments are correct
 	 * because s32 bounds don't flip sign when shifting to the left by
 	 * 32bits.
 	 */
-	if (umin_val == 32 && umax_val == 32) {
-		dst_reg->smax_value = (s64)dst_reg->s32_max_value << 32;
-		dst_reg->smin_value = (s64)dst_reg->s32_min_value << 32;
-	} else {
-		dst_reg->smax_value = S64_MAX;
-		dst_reg->smin_value = S64_MIN;
-	}
+	if (umin_val == 32 && umax_val == 32)
+		s = cnum64_from_srange((s64)reg_s32_min(dst_reg) << 32,
+				       (s64)reg_s32_max(dst_reg) << 32);
+	else
+		s = CNUM64_UNBOUNDED;
 
 	/* If we might shift our top bit out, then we know nothing */
-	if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
-		dst_reg->umin_value = 0;
-		dst_reg->umax_value = U64_MAX;
-	} else {
-		dst_reg->umin_value <<= umin_val;
-		dst_reg->umax_value <<= umax_val;
-	}
+	if (reg_umax(dst_reg) > 1ULL << (63 - umax_val))
+		u = CNUM64_UNBOUNDED;
+	else
+		u = cnum64_from_urange(reg_umin(dst_reg) << umin_val,
+				       reg_umax(dst_reg) << umax_val);
+
+	dst_reg->r64 = cnum64_intersect(u, s);
 }
 
 static void scalar_min_max_lsh(struct bpf_reg_state *dst_reg,
 			       struct bpf_reg_state *src_reg)
 {
-	u64 umax_val = src_reg->umax_value;
-	u64 umin_val = src_reg->umin_value;
+	u64 umax_val = reg_umax(src_reg);
+	u64 umin_val = reg_umin(src_reg);
 
 	/* scalar64 calc uses 32bit unshifted bounds so must be called first */
 	__scalar64_min_max_lsh(dst_reg, umin_val, umax_val);
@@ -14645,8 +14343,8 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
 				 struct bpf_reg_state *src_reg)
 {
 	struct tnum subreg = tnum_subreg(dst_reg->var_off);
-	u32 umax_val = src_reg->u32_max_value;
-	u32 umin_val = src_reg->u32_min_value;
+	u32 umax_val = reg_u32_max(src_reg);
+	u32 umin_val = reg_u32_min(src_reg);
 
 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
 	 * be negative, then either:
@@ -14662,12 +14360,10 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
 	 * and rely on inferring new ones from the unsigned bounds and
 	 * var_off of the result.
 	 */
-	dst_reg->s32_min_value = S32_MIN;
-	dst_reg->s32_max_value = S32_MAX;
 
 	dst_reg->var_off = tnum_rshift(subreg, umin_val);
-	dst_reg->u32_min_value >>= umax_val;
-	dst_reg->u32_max_value >>= umin_val;
+	reg_set_urange32(dst_reg, reg_u32_min(dst_reg) >> umax_val,
+			 reg_u32_max(dst_reg) >> umin_val);
 
 	__mark_reg64_unbounded(dst_reg);
 	__update_reg32_bounds(dst_reg);
@@ -14676,8 +14372,8 @@ static void scalar32_min_max_rsh(struct bpf_reg_state *dst_reg,
 static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
 			       struct bpf_reg_state *src_reg)
 {
-	u64 umax_val = src_reg->umax_value;
-	u64 umin_val = src_reg->umin_value;
+	u64 umax_val = reg_umax(src_reg);
+	u64 umin_val = reg_umin(src_reg);
 
 	/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
 	 * be negative, then either:
@@ -14693,11 +14389,9 @@ static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
 	 * and rely on inferring new ones from the unsigned bounds and
 	 * var_off of the result.
 	 */
-	dst_reg->smin_value = S64_MIN;
-	dst_reg->smax_value = S64_MAX;
 	dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
-	dst_reg->umin_value >>= umax_val;
-	dst_reg->umax_value >>= umin_val;
+	reg_set_urange64(dst_reg, reg_umin(dst_reg) >> umax_val,
+			 reg_umax(dst_reg) >> umin_val);
 
 	/* Its not easy to operate on alu32 bounds here because it depends
 	 * on bits being shifted in. Take easy way out and mark unbounded
@@ -14710,22 +14404,19 @@ static void scalar_min_max_rsh(struct bpf_reg_state *dst_reg,
 static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
 				  struct bpf_reg_state *src_reg)
 {
-	u64 umin_val = src_reg->u32_min_value;
+	u64 umin_val = reg_u32_min(src_reg);
 
 	/* Upon reaching here, src_known is true and
 	 * umax_val is equal to umin_val.
+	 * Blow away the dst_reg umin_value/umax_value and rely on
+	 * dst_reg var_off to refine the result.
 	 */
-	dst_reg->s32_min_value = (u32)(((s32)dst_reg->s32_min_value) >> umin_val);
-	dst_reg->s32_max_value = (u32)(((s32)dst_reg->s32_max_value) >> umin_val);
+	reg_set_srange32(dst_reg,
+			 (u32)(((s32)reg_s32_min(dst_reg)) >> umin_val),
+			 (u32)(((s32)reg_s32_max(dst_reg)) >> umin_val));
 
 	dst_reg->var_off = tnum_arshift(tnum_subreg(dst_reg->var_off), umin_val, 32);
 
-	/* blow away the dst_reg umin_value/umax_value and rely on
-	 * dst_reg var_off to refine the result.
-	 */
-	dst_reg->u32_min_value = 0;
-	dst_reg->u32_max_value = U32_MAX;
-
 	__mark_reg64_unbounded(dst_reg);
 	__update_reg32_bounds(dst_reg);
 }
@@ -14733,22 +14424,16 @@ static void scalar32_min_max_arsh(struct bpf_reg_state *dst_reg,
 static void scalar_min_max_arsh(struct bpf_reg_state *dst_reg,
 				struct bpf_reg_state *src_reg)
 {
-	u64 umin_val = src_reg->umin_value;
+	u64 umin_val = reg_umin(src_reg);
 
 	/* Upon reaching here, src_known is true and umax_val is equal
 	 * to umin_val.
 	 */
-	dst_reg->smin_value >>= umin_val;
-	dst_reg->smax_value >>= umin_val;
+	reg_set_srange64(dst_reg, reg_smin(dst_reg) >> umin_val,
+			 reg_smax(dst_reg) >> umin_val);
 
 	dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val, 64);
 
-	/* blow away the dst_reg umin_value/umax_value and rely on
-	 * dst_reg var_off to refine the result.
-	 */
-	dst_reg->umin_value = 0;
-	dst_reg->umax_value = U64_MAX;
-
 	/* Its not easy to operate on alu32 bounds here because it depends
 	 * on bits being shifted in from upper 32-bits. Take easy way out
 	 * and mark unbounded so we can recalculate later from tnum.
@@ -14814,13 +14499,13 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn,
 
 	if (insn_bitness == 32) {
 		if (tnum_subreg_is_const(src_reg->var_off)
-		    && src_reg->s32_min_value == src_reg->s32_max_value
-		    && src_reg->u32_min_value == src_reg->u32_max_value)
+		    && reg_s32_min(src_reg) == reg_s32_max(src_reg)
+		    && reg_u32_min(src_reg) == reg_u32_max(src_reg))
 			src_is_const = true;
 	} else {
 		if (tnum_is_const(src_reg->var_off)
-		    && src_reg->smin_value == src_reg->smax_value
-		    && src_reg->umin_value == src_reg->umax_value)
+		    && reg_smin(src_reg) == reg_smax(src_reg)
+		    && reg_umin(src_reg) == reg_umax(src_reg))
 			src_is_const = true;
 	}
 
@@ -14850,7 +14535,7 @@ static bool is_safe_to_compute_dst_reg_range(struct bpf_insn *insn,
 	case BPF_LSH:
 	case BPF_RSH:
 	case BPF_ARSH:
-		return (src_is_const && src_reg->umax_value < insn_bitness);
+		return (src_is_const && reg_umax(src_reg) < insn_bitness);
 	default:
 		return false;
 	}
@@ -14863,9 +14548,9 @@ static int maybe_fork_scalars(struct bpf_verifier_env *env, struct bpf_insn *ins
 	struct bpf_reg_state *regs;
 	bool alu32;
 
-	if (dst_reg->smin_value == -1 && dst_reg->smax_value == 0)
+	if (reg_smin(dst_reg) == -1 && reg_smax(dst_reg) == 0)
 		alu32 = false;
-	else if (dst_reg->s32_min_value == -1 && dst_reg->s32_max_value == 0)
+	else if (reg_s32_min(dst_reg) == -1 && reg_s32_max(dst_reg) == 0)
 		alu32 = true;
 	else
 		return 0;
@@ -14949,7 +14634,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
 		break;
 	case BPF_DIV:
 		/* BPF div specification: x / 0 = 0 */
-		if ((alu32 && src_reg.u32_min_value == 0) || (!alu32 && src_reg.umin_value == 0)) {
+		if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0)) {
 			___mark_reg_known(dst_reg, 0);
 			break;
 		}
@@ -14966,7 +14651,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
 		break;
 	case BPF_MOD:
 		/* BPF mod specification: x % 0 = x */
-		if ((alu32 && src_reg.u32_min_value == 0) || (!alu32 && src_reg.umin_value == 0))
+		if ((alu32 && reg_u32_min(&src_reg) == 0) || (!alu32 && reg_umin(&src_reg) == 0))
 			break;
 		if (alu32)
 			if (off == 1)
@@ -15154,7 +14839,7 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
 	 * umax_value before the ALU operation. After adjust_scalar_min_max_vals(),
 	 * alu32 ops will have zero-extended the result, making umax_value <= U32_MAX.
 	 */
-	u64 dst_umax = dst_reg->umax_value;
+	u64 dst_umax = reg_umax(dst_reg);
 
 	err = adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
 	if (err)
@@ -15284,7 +14969,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
 					 * copy register state to dest reg
 					 */
 					assign_scalar_id_before_mov(env, src_reg);
-					copy_register_state(dst_reg, src_reg);
+					*dst_reg = *src_reg;
 					dst_reg->subreg_def = DEF_NOT_SUBREG;
 				} else {
 					/* case: R1 = (s8, s16 s32)R2 */
@@ -15296,10 +14981,10 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
 					} else if (src_reg->type == SCALAR_VALUE) {
 						bool no_sext;
 
-						no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));
+						no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1));
 						if (no_sext)
 							assign_scalar_id_before_mov(env, src_reg);
-						copy_register_state(dst_reg, src_reg);
+						*dst_reg = *src_reg;
 						if (!no_sext)
 							clear_scalar_id(dst_reg);
 						coerce_reg_to_size_sx(dst_reg, insn->off >> 3);
@@ -15321,7 +15006,7 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
 
 						if (is_src_reg_u32)
 							assign_scalar_id_before_mov(env, src_reg);
-						copy_register_state(dst_reg, src_reg);
+						*dst_reg = *src_reg;
 						/* Make sure ID is cleared if src_reg is not in u32
 						 * range otherwise dst_reg min/max could be incorrectly
 						 * propagated into src_reg by sync_linked_regs()
@@ -15331,11 +15016,11 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
 						dst_reg->subreg_def = env->insn_idx + 1;
 					} else {
 						/* case: W1 = (s8, s16)W2 */
-						bool no_sext = src_reg->umax_value < (1ULL << (insn->off - 1));
+						bool no_sext = reg_umax(src_reg) < (1ULL << (insn->off - 1));
 
 						if (no_sext)
 							assign_scalar_id_before_mov(env, src_reg);
-						copy_register_state(dst_reg, src_reg);
+						*dst_reg = *src_reg;
 						if (!no_sext)
 							clear_scalar_id(dst_reg);
 						dst_reg->subreg_def = env->insn_idx + 1;
@@ -15413,17 +15098,17 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
 	struct bpf_reg_state *reg;
 	int new_range;
 
-	if (dst_reg->umax_value == 0 && range_right_open)
+	if (reg_umax(dst_reg) == 0 && range_right_open)
 		/* This doesn't give us any range */
 		return;
 
-	if (dst_reg->umax_value > MAX_PACKET_OFF)
+	if (reg_umax(dst_reg) > MAX_PACKET_OFF)
 		/* Risk of overflow.  For instance, ptr + (1<<63) may be less
 		 * than pkt_end, but that's because it's also less than pkt.
 		 */
 		return;
 
-	new_range = dst_reg->umax_value;
+	new_range = reg_umax(dst_reg);
 	if (range_right_open)
 		new_range++;
 
@@ -15472,7 +15157,7 @@ static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
 	/* If our ids match, then we must have the same max_value.  And we
 	 * don't care about the other reg's fixed offset, since if it's too big
 	 * the range won't allow anything.
-	 * dst_reg->umax_value is known < MAX_PACKET_OFF, therefore it fits in a u16.
+	 * reg_umax(dst_reg) is known < MAX_PACKET_OFF, therefore it fits in a u16.
 	 */
 	bpf_for_each_reg_in_vstate(vstate, state, reg, ({
 		if (reg->type == type && reg->id == dst_reg->id)
@@ -15528,14 +15213,14 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s
 {
 	struct tnum t1 = is_jmp32 ? tnum_subreg(reg1->var_off) : reg1->var_off;
 	struct tnum t2 = is_jmp32 ? tnum_subreg(reg2->var_off) : reg2->var_off;
-	u64 umin1 = is_jmp32 ? (u64)reg1->u32_min_value : reg1->umin_value;
-	u64 umax1 = is_jmp32 ? (u64)reg1->u32_max_value : reg1->umax_value;
-	s64 smin1 = is_jmp32 ? (s64)reg1->s32_min_value : reg1->smin_value;
-	s64 smax1 = is_jmp32 ? (s64)reg1->s32_max_value : reg1->smax_value;
-	u64 umin2 = is_jmp32 ? (u64)reg2->u32_min_value : reg2->umin_value;
-	u64 umax2 = is_jmp32 ? (u64)reg2->u32_max_value : reg2->umax_value;
-	s64 smin2 = is_jmp32 ? (s64)reg2->s32_min_value : reg2->smin_value;
-	s64 smax2 = is_jmp32 ? (s64)reg2->s32_max_value : reg2->smax_value;
+	u64 umin1 = is_jmp32 ? (u64)reg_u32_min(reg1) : reg_umin(reg1);
+	u64 umax1 = is_jmp32 ? (u64)reg_u32_max(reg1) : reg_umax(reg1);
+	s64 smin1 = is_jmp32 ? (s64)reg_s32_min(reg1) : reg_smin(reg1);
+	s64 smax1 = is_jmp32 ? (s64)reg_s32_max(reg1) : reg_smax(reg1);
+	u64 umin2 = is_jmp32 ? (u64)reg_u32_min(reg2) : reg_umin(reg2);
+	u64 umax2 = is_jmp32 ? (u64)reg_u32_max(reg2) : reg_umax(reg2);
+	s64 smin2 = is_jmp32 ? (s64)reg_s32_min(reg2) : reg_smin(reg2);
+	s64 smax2 = is_jmp32 ? (s64)reg_s32_max(reg2) : reg_smax(reg2);
 
 	if (reg1 == reg2) {
 		switch (opcode) {
@@ -15580,11 +15265,11 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s
 			 * utilize 32-bit subrange knowledge to eliminate
 			 * branches that can't be taken a priori
 			 */
-			if (reg1->u32_min_value > reg2->u32_max_value ||
-			    reg1->u32_max_value < reg2->u32_min_value)
+			if (reg_u32_min(reg1) > reg_u32_max(reg2) ||
+			    reg_u32_max(reg1) < reg_u32_min(reg2))
 				return 0;
-			if (reg1->s32_min_value > reg2->s32_max_value ||
-			    reg1->s32_max_value < reg2->s32_min_value)
+			if (reg_s32_min(reg1) > reg_s32_max(reg2) ||
+			    reg_s32_max(reg1) < reg_s32_min(reg2))
 				return 0;
 		}
 		break;
@@ -15606,11 +15291,11 @@ static int is_scalar_branch_taken(struct bpf_verifier_env *env, struct bpf_reg_s
 			 * utilize 32-bit subrange knowledge to eliminate
 			 * branches that can't be taken a priori
 			 */
-			if (reg1->u32_min_value > reg2->u32_max_value ||
-			    reg1->u32_max_value < reg2->u32_min_value)
+			if (reg_u32_min(reg1) > reg_u32_max(reg2) ||
+			    reg_u32_max(reg1) < reg_u32_min(reg2))
 				return 1;
-			if (reg1->s32_min_value > reg2->s32_max_value ||
-			    reg1->s32_max_value < reg2->s32_min_value)
+			if (reg_s32_min(reg1) > reg_s32_max(reg2) ||
+			    reg_s32_max(reg1) < reg_s32_min(reg2))
 				return 1;
 		}
 		break;
@@ -15837,27 +15522,15 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state
 	switch (opcode) {
 	case BPF_JEQ:
 		if (is_jmp32) {
-			reg1->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
-			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
-			reg1->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
-			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
-			reg2->u32_min_value = reg1->u32_min_value;
-			reg2->u32_max_value = reg1->u32_max_value;
-			reg2->s32_min_value = reg1->s32_min_value;
-			reg2->s32_max_value = reg1->s32_max_value;
+			reg1->r32 = cnum32_intersect(reg1->r32, reg2->r32);
+			reg2->r32 = reg1->r32;
 
 			t = tnum_intersect(tnum_subreg(reg1->var_off), tnum_subreg(reg2->var_off));
 			reg1->var_off = tnum_with_subreg(reg1->var_off, t);
 			reg2->var_off = tnum_with_subreg(reg2->var_off, t);
 		} else {
-			reg1->umin_value = max(reg1->umin_value, reg2->umin_value);
-			reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
-			reg1->smin_value = max(reg1->smin_value, reg2->smin_value);
-			reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
-			reg2->umin_value = reg1->umin_value;
-			reg2->umax_value = reg1->umax_value;
-			reg2->smin_value = reg1->smin_value;
-			reg2->smax_value = reg1->smax_value;
+			reg1->r64 = cnum64_intersect(reg1->r64, reg2->r64);
+			reg2->r64 = reg1->r64;
 
 			reg1->var_off = tnum_intersect(reg1->var_off, reg2->var_off);
 			reg2->var_off = reg1->var_off;
@@ -15874,32 +15547,11 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state
 		 */
 		val = reg_const_value(reg2, is_jmp32);
 		if (is_jmp32) {
-			/* u32_min_value is not equal to 0xffffffff at this point,
-			 * because otherwise u32_max_value is 0xffffffff as well,
-			 * in such a case both reg1 and reg2 would be constants,
-			 * jump would be predicted and regs_refine_cond_op()
-			 * wouldn't be called.
-			 *
-			 * Same reasoning works for all {u,s}{min,max}{32,64} cases
-			 * below.
-			 */
-			if (reg1->u32_min_value == (u32)val)
-				reg1->u32_min_value++;
-			if (reg1->u32_max_value == (u32)val)
-				reg1->u32_max_value--;
-			if (reg1->s32_min_value == (s32)val)
-				reg1->s32_min_value++;
-			if (reg1->s32_max_value == (s32)val)
-				reg1->s32_max_value--;
+			/* Complement of the range [val, val] as cnum32. */
+			cnum32_intersect_with(&reg1->r32, (struct cnum32){ val + 1, U32_MAX - 1 });
 		} else {
-			if (reg1->umin_value == (u64)val)
-				reg1->umin_value++;
-			if (reg1->umax_value == (u64)val)
-				reg1->umax_value--;
-			if (reg1->smin_value == (s64)val)
-				reg1->smin_value++;
-			if (reg1->smax_value == (s64)val)
-				reg1->smax_value--;
+			/* Complement of the range [val, val] as cnum64. */
+			cnum64_intersect_with(&reg1->r64, (struct cnum64){ val + 1, U64_MAX - 1 });
 		}
 		break;
 	case BPF_JSET:
@@ -15946,38 +15598,38 @@ static void regs_refine_cond_op(struct bpf_reg_state *reg1, struct bpf_reg_state
 		break;
 	case BPF_JLE:
 		if (is_jmp32) {
-			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value);
-			reg2->u32_min_value = max(reg1->u32_min_value, reg2->u32_min_value);
+			cnum32_intersect_with_urange(&reg1->r32, 0, reg_u32_max(reg2));
+			cnum32_intersect_with_urange(&reg2->r32, reg_u32_min(reg1), U32_MAX);
 		} else {
-			reg1->umax_value = min(reg1->umax_value, reg2->umax_value);
-			reg2->umin_value = max(reg1->umin_value, reg2->umin_value);
+			cnum64_intersect_with_urange(&reg1->r64, 0, reg_umax(reg2));
+			cnum64_intersect_with_urange(&reg2->r64, reg_umin(reg1), U64_MAX);
 		}
 		break;
 	case BPF_JLT:
 		if (is_jmp32) {
-			reg1->u32_max_value = min(reg1->u32_max_value, reg2->u32_max_value - 1);
-			reg2->u32_min_value = max(reg1->u32_min_value + 1, reg2->u32_min_value);
+			cnum32_intersect_with_urange(&reg1->r32, 0, reg_u32_max(reg2) - 1);
+			cnum32_intersect_with_urange(&reg2->r32, reg_u32_min(reg1) + 1, U32_MAX);
 		} else {
-			reg1->umax_value = min(reg1->umax_value, reg2->umax_value - 1);
-			reg2->umin_value = max(reg1->umin_value + 1, reg2->umin_value);
+			cnum64_intersect_with_urange(&reg1->r64, 0, reg_umax(reg2) - 1);
+			cnum64_intersect_with_urange(&reg2->r64, reg_umin(reg1) + 1, U64_MAX);
 		}
 		break;
 	case BPF_JSLE:
 		if (is_jmp32) {
-			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value);
-			reg2->s32_min_value = max(reg1->s32_min_value, reg2->s32_min_value);
+			cnum32_intersect_with_srange(&reg1->r32, S32_MIN, reg_s32_max(reg2));
+			cnum32_intersect_with_srange(&reg2->r32, reg_s32_min(reg1), S32_MAX);
 		} else {
-			reg1->smax_value = min(reg1->smax_value, reg2->smax_value);
-			reg2->smin_value = max(reg1->smin_value, reg2->smin_value);
+			cnum64_intersect_with_srange(&reg1->r64, S64_MIN, reg_smax(reg2));
+			cnum64_intersect_with_srange(&reg2->r64, reg_smin(reg1), S64_MAX);
 		}
 		break;
 	case BPF_JSLT:
 		if (is_jmp32) {
-			reg1->s32_max_value = min(reg1->s32_max_value, reg2->s32_max_value - 1);
-			reg2->s32_min_value = max(reg1->s32_min_value + 1, reg2->s32_min_value);
+			cnum32_intersect_with_srange(&reg1->r32, S32_MIN, reg_s32_max(reg2) - 1);
+			cnum32_intersect_with_srange(&reg2->r32, reg_s32_min(reg1) + 1, S32_MAX);
 		} else {
-			reg1->smax_value = min(reg1->smax_value, reg2->smax_value - 1);
-			reg2->smin_value = max(reg1->smin_value + 1, reg2->smin_value);
+			cnum64_intersect_with_srange(&reg1->r64, S64_MIN, reg_smax(reg2) - 1);
+			cnum64_intersect_with_srange(&reg2->r64, reg_smin(reg1) + 1, S64_MAX);
 		}
 		break;
 	default:
@@ -16248,7 +15900,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s
 		    reg->delta == known_reg->delta) {
 			s32 saved_subreg_def = reg->subreg_def;
 
-			copy_register_state(reg, known_reg);
+			*reg = *known_reg;
 			reg->subreg_def = saved_subreg_def;
 		} else {
 			s32 saved_subreg_def = reg->subreg_def;
@@ -16259,7 +15911,7 @@ static void sync_linked_regs(struct bpf_verifier_env *env, struct bpf_verifier_s
 			__mark_reg_known(&fake_reg, (s64)reg->delta - (s64)known_reg->delta);
 
 			/* reg = known_reg; reg += delta */
-			copy_register_state(reg, known_reg);
+			*reg = *known_reg;
 			/*
 			 * Must preserve off, id and subreg_def flag,
 			 * otherwise another sync_linked_regs() will be incorrect.
@@ -16356,16 +16008,16 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 	}
 
 	if (insn_flags) {
-		err = bpf_push_jmp_history(env, this_branch, insn_flags, 0);
+		err = bpf_push_jmp_history(env, this_branch, insn_flags, 0, 0, 0);
 		if (err)
 			return err;
 	}
 
 	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
-	copy_register_state(&env->false_reg1, dst_reg);
-	copy_register_state(&env->false_reg2, src_reg);
-	copy_register_state(&env->true_reg1, dst_reg);
-	copy_register_state(&env->true_reg2, src_reg);
+	env->false_reg1 = *dst_reg;
+	env->false_reg2 = *src_reg;
+	env->true_reg1 = *dst_reg;
+	env->true_reg2 = *src_reg;
 	pred = is_branch_taken(env, dst_reg, src_reg, opcode, is_jmp32);
 	if (pred >= 0) {
 		/* If we get here with a dst_reg pointer type it is because
@@ -16420,7 +16072,7 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 	if (dst_reg->type == SCALAR_VALUE && dst_reg->id)
 		collect_linked_regs(env, this_branch, dst_reg->id, &linked_regs);
 	if (linked_regs.cnt > 1) {
-		err = bpf_push_jmp_history(env, this_branch, 0, linked_regs_pack(&linked_regs));
+		err = bpf_push_jmp_history(env, this_branch, 0, 0, 0, linked_regs_pack(&linked_regs));
 		if (err)
 			return err;
 	}
@@ -16434,11 +16086,11 @@ static int check_cond_jmp_op(struct bpf_verifier_env *env,
 	if (err)
 		return err;
 
-	copy_register_state(dst_reg, &env->false_reg1);
-	copy_register_state(src_reg, &env->false_reg2);
-	copy_register_state(&other_branch_regs[insn->dst_reg], &env->true_reg1);
+	*dst_reg = env->false_reg1;
+	*src_reg = env->false_reg2;
+	other_branch_regs[insn->dst_reg] = env->true_reg1;
 	if (BPF_SRC(insn->code) == BPF_X)
-		copy_register_state(&other_branch_regs[insn->src_reg], &env->true_reg2);
+		other_branch_regs[insn->src_reg] = env->true_reg2;
 
 	if (BPF_SRC(insn->code) == BPF_X &&
 	    src_reg->type == SCALAR_VALUE && src_reg->id &&
@@ -16890,7 +16542,7 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char
 						prog->aux->attach_func_proto->type,
 						NULL);
 		if (ret_type && ret_type == reg_type && reg->ref_obj_id)
-			return __check_ptr_off_reg(env, reg, regno, false);
+			return __check_ptr_off_reg(env, reg, argno_from_reg(regno), false);
 	}
 
 	/* eBPF calling convention is such that R0 is used
@@ -17478,16 +17130,16 @@ static int indirect_jump_min_max_index(struct bpf_verifier_env *env,
 				       u32 *pmin_index, u32 *pmax_index)
 {
 	struct bpf_reg_state *reg = reg_state(env, regno);
-	u64 min_index = reg->umin_value;
-	u64 max_index = reg->umax_value;
+	u64 min_index = reg_umin(reg);
+	u64 max_index = reg_umax(reg);
 	const u32 size = 8;
 
 	if (min_index > (u64) U32_MAX * size) {
-		verbose(env, "the sum of R%u umin_value %llu is too big\n", regno, reg->umin_value);
+		verbose(env, "the sum of R%u umin_value %llu is too big\n", regno, reg_umin(reg));
 		return -ERANGE;
 	}
 	if (max_index > (u64) U32_MAX * size) {
-		verbose(env, "the sum of R%u umax_value %llu is too big\n", regno, reg->umax_value);
+		verbose(env, "the sum of R%u umax_value %llu is too big\n", regno, reg_umax(reg));
 		return -ERANGE;
 	}
 
@@ -17586,6 +17238,14 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
 		return check_store_reg(env, insn, false);
 
 	case BPF_ST: {
+		/* Handle stack arg write (store immediate) */
+		if (is_stack_arg_st(insn)) {
+			struct bpf_verifier_state *vstate = env->cur_state;
+			struct bpf_func_state *state = vstate->frame[vstate->curframe];
+
+			return check_stack_arg_write(env, state, insn->off, NULL);
+		}
+
 		enum bpf_reg_type dst_reg_type;
 
 		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
@@ -17594,7 +17254,7 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
 
 		dst_reg_type = cur_regs(env)[insn->dst_reg].type;
 
-		err = check_mem_access(env, env->insn_idx, insn->dst_reg,
+		err = check_mem_access(env, env->insn_idx, cur_regs(env) + insn->dst_reg, argno_from_reg(insn->dst_reg),
 				       insn->off, BPF_SIZE(insn->code),
 				       BPF_WRITE, -1, false, false);
 		if (err)
@@ -17620,6 +17280,8 @@ static int do_check_insn(struct bpf_verifier_env *env, bool *do_print_state)
 				}
 			}
 			mark_reg_scratched(env, BPF_REG_0);
+			if (bpf_in_stack_arg_cnt(&env->subprog_info[cur_func(env)->subprogno]))
+				cur_func(env)->no_stack_arg_load = true;
 			if (insn->src_reg == BPF_PSEUDO_CALL)
 				return check_func_call(env, insn, &env->insn_idx);
 			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL)
@@ -17717,7 +17379,7 @@ static int do_check(struct bpf_verifier_env *env)
 		}
 
 		if (bpf_is_jmp_point(env, env->insn_idx)) {
-			err = bpf_push_jmp_history(env, state, 0, 0);
+			err = bpf_push_jmp_history(env, state, 0, 0, 0, 0);
 			if (err)
 				return err;
 		}
@@ -18424,11 +18086,12 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env)
 		return err;
 
 	for (i = 0; i < insn_cnt; i++, insn++) {
-		if (insn->dst_reg >= MAX_BPF_REG) {
+		if (insn->dst_reg >= MAX_BPF_REG &&
+		    !is_stack_arg_st(insn) && !is_stack_arg_stx(insn)) {
 			verbose(env, "R%d is invalid\n", insn->dst_reg);
 			return -EINVAL;
 		}
-		if (insn->src_reg >= MAX_BPF_REG) {
+		if (insn->src_reg >= MAX_BPF_REG && !is_stack_arg_ldx(insn)) {
 			verbose(env, "R%d is invalid\n", insn->src_reg);
 			return -EINVAL;
 		}
@@ -18735,7 +18398,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 				goto out;
 			}
 		}
-		for (i = BPF_REG_1; i <= sub->arg_cnt; i++) {
+		for (i = BPF_REG_1; i <= min_t(u32, sub->arg_cnt, MAX_BPF_FUNC_REG_ARGS); i++) {
 			arg = &sub->args[i - BPF_REG_1];
 			reg = &regs[i];
 
@@ -18745,7 +18408,7 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 			} else if (arg->arg_type == ARG_ANYTHING) {
 				reg->type = SCALAR_VALUE;
 				mark_reg_unknown(env, regs, i);
-			} else if (arg->arg_type == (ARG_PTR_TO_DYNPTR | MEM_RDONLY)) {
+			} else if (arg->arg_type == ARG_PTR_TO_DYNPTR) {
 				/* assume unspecial LOCAL dynptr type */
 				__mark_dynptr_reg(reg, BPF_DYNPTR_TYPE_LOCAL, true, ++env->id_gen);
 			} else if (base_type(arg->arg_type) == ARG_PTR_TO_MEM) {
@@ -18773,11 +18436,17 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 				mark_reg_unknown(env, regs, i);
 			} else {
 				verifier_bug(env, "unhandled arg#%d type %d",
-					     i - BPF_REG_1, arg->arg_type);
+					     i - BPF_REG_1 + 1, arg->arg_type);
 				ret = -EFAULT;
 				goto out;
 			}
 		}
+		if (env->prog->type == BPF_PROG_TYPE_EXT && sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) {
+			verbose(env, "freplace programs with >%d args not supported yet\n",
+				MAX_BPF_FUNC_REG_ARGS);
+			ret = -EINVAL;
+			goto out;
+		}
 	} else {
 		/* if main BPF program has associated BTF info, validate that
 		 * it's matching expected signature, and otherwise mark BTF
@@ -18785,8 +18454,11 @@ static int do_check_common(struct bpf_verifier_env *env, int subprog)
 		 */
 		if (env->prog->aux->func_info_aux) {
 			ret = btf_prepare_func_args(env, 0);
-			if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX)
+			if (ret || sub->arg_cnt != 1 || sub->args[0].arg_type != ARG_PTR_TO_CTX) {
 				env->prog->aux->func_info_aux[0].unreliable = true;
+				sub->arg_cnt = 1;
+				sub->stack_arg_cnt = 0;
+			}
 		}
 
 		/* 1st arg to a function */
@@ -18834,6 +18506,7 @@ static int do_check_subprogs(struct bpf_verifier_env *env)
 	struct bpf_prog_aux *aux = env->prog->aux;
 	struct bpf_func_info_aux *sub_aux;
 	int i, ret, new_cnt;
+	u32 insn_processed;
 
 	if (!aux->func_info)
 		return 0;
@@ -18848,6 +18521,8 @@ static int do_check_subprogs(struct bpf_verifier_env *env)
 		if (!bpf_subprog_is_global(env, i))
 			continue;
 
+		insn_processed = env->insn_processed;
+
 		sub_aux = subprog_aux(env, i);
 		if (!sub_aux->called || sub_aux->verified)
 			continue;
@@ -18855,6 +18530,7 @@ static int do_check_subprogs(struct bpf_verifier_env *env)
 		env->insn_idx = env->subprog_info[i].start;
 		WARN_ON_ONCE(env->insn_idx == 0);
 		ret = do_check_common(env, i);
+		env->subprog_info[i].insn_processed = env->insn_processed - insn_processed;
 		if (ret) {
 			return ret;
 		} else if (env->log.level & BPF_LOG_LEVEL) {
@@ -18881,10 +18557,12 @@ static int do_check_subprogs(struct bpf_verifier_env *env)
 
 static int do_check_main(struct bpf_verifier_env *env)
 {
+	u32 insn_processed = env->insn_processed;
 	int ret;
 
 	env->insn_idx = 0;
 	ret = do_check_common(env, 0);
+	env->subprog_info[0].insn_processed = env->insn_processed - insn_processed;
 	if (!ret)
 		env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
 	return ret;
@@ -18893,19 +18571,20 @@ static int do_check_main(struct bpf_verifier_env *env)
 
 static void print_verification_stats(struct bpf_verifier_env *env)
 {
-	int i;
+	/* Skip over hidden subprogs which are not verified. */
+	int i, subprog_cnt = env->subprog_cnt - env->hidden_subprog_cnt;
 
 	if (env->log.level & BPF_LOG_STATS) {
 		verbose(env, "verification time %lld usec\n",
 			div_u64(env->verification_time, 1000));
-		verbose(env, "stack depth ");
-		for (i = 0; i < env->subprog_cnt; i++) {
-			u32 depth = env->subprog_info[i].stack_depth;
-
-			verbose(env, "%d", depth);
-			if (i + 1 < env->subprog_cnt)
-				verbose(env, "+");
-		}
+		verbose(env, "stack depth %d", env->subprog_info[0].stack_depth);
+		for (i = 1; i < subprog_cnt; i++)
+			verbose(env, "+%d", env->subprog_info[i].stack_depth);
+		verbose(env, " max %d\n", env->max_stack_depth);
+		verbose(env, "insns processed %d", env->subprog_info[0].insn_processed);
+		for (i = 1; i < subprog_cnt; i++)
+			if (bpf_subprog_is_global(env, i))
+				verbose(env, "+%d", env->subprog_info[i].insn_processed);
 		verbose(env, "\n");
 	}
 	verbose(env, "processed %d insns (limit %d) max_states_per_insn %d "
@@ -19299,6 +18978,12 @@ int bpf_check_attach_target(struct bpf_verifier_log *log,
 		btp = bpf_get_raw_tracepoint(tname);
 		if (!btp)
 			return -EINVAL;
+		if (prog->sleepable && !tracepoint_is_faultable(btp->tp)) {
+			bpf_log(log, "Sleepable program cannot attach to non-faultable tracepoint %s\n",
+				tname);
+			bpf_put_raw_tracepoint(btp);
+			return -EINVAL;
+		}
 		fname = kallsyms_lookup((unsigned long)btp->bpf_func, NULL, NULL, NULL,
 					trace_symbol);
 		bpf_put_raw_tracepoint(btp);
@@ -19515,6 +19200,7 @@ static bool can_be_sleepable(struct bpf_prog *prog)
 		case BPF_MODIFY_RETURN:
 		case BPF_TRACE_ITER:
 		case BPF_TRACE_FSESSION:
+		case BPF_TRACE_RAW_TP:
 			return true;
 		default:
 			return false;
@@ -19522,7 +19208,9 @@ static bool can_be_sleepable(struct bpf_prog *prog)
 	}
 	return prog->type == BPF_PROG_TYPE_LSM ||
 	       prog->type == BPF_PROG_TYPE_KPROBE /* only for uprobes */ ||
-	       prog->type == BPF_PROG_TYPE_STRUCT_OPS;
+	       prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
+	       prog->type == BPF_PROG_TYPE_RAW_TRACEPOINT ||
+	       prog->type == BPF_PROG_TYPE_TRACEPOINT;
 }
 
 static int check_attach_btf_id(struct bpf_verifier_env *env)
@@ -19544,7 +19232,7 @@ static int check_attach_btf_id(struct bpf_verifier_env *env)
 	}
 
 	if (prog->sleepable && !can_be_sleepable(prog)) {
-		verbose(env, "Only fentry/fexit/fsession/fmod_ret, lsm, iter, uprobe, and struct_ops programs can be sleepable\n");
+		verbose(env, "Program of this type cannot be sleepable\n");
 		return -EINVAL;
 	}
 
@@ -19897,12 +19585,12 @@ int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 	return 0;
 }
 
-int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u32 uattr_size)
+int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
+	      struct bpf_log_attr *attr_log)
 {
 	u64 start_time = ktime_get_ns();
 	struct bpf_verifier_env *env;
 	int i, len, ret = -EINVAL, err;
-	u32 log_true_size;
 	bool is_priv;
 
 	BTF_TYPE_EMIT(enum bpf_features);
@@ -19949,9 +19637,7 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
 	/* user could have requested verbose verifier output
 	 * and supplied buffer to store the verification trace
 	 */
-	ret = bpf_vlog_init(&env->log, attr->log_level,
-			    (char __user *) (unsigned long) attr->log_buf,
-			    attr->log_size);
+	ret = bpf_vlog_init(&env->log, attr_log->level, attr_log->ubuf, attr_log->size);
 	if (ret)
 		goto err_unlock;
 
@@ -20113,17 +19799,10 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, __u3
 	env->prog->aux->verified_insns = env->insn_processed;
 
 	/* preserve original error even if log finalization is successful */
-	err = bpf_vlog_finalize(&env->log, &log_true_size);
+	err = bpf_log_attr_finalize(attr_log, &env->log);
 	if (err)
 		ret = err;
 
-	if (uattr_size >= offsetofend(union bpf_attr, log_true_size) &&
-	    copy_to_bpfptr_offset(uattr, offsetof(union bpf_attr, log_true_size),
-				  &log_true_size, sizeof(log_true_size))) {
-		ret = -EFAULT;
-		goto err_release_maps;
-	}
-
 	if (ret)
 		goto err_release_maps;
 
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 6152add0c5eb6..48af01f440300 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -264,7 +264,6 @@ static void cgroup_finalize_control(struct cgroup *cgrp, int ret);
 static void css_task_iter_skip(struct css_task_iter *it,
 			       struct task_struct *task);
 static int cgroup_destroy_locked(struct cgroup *cgrp);
-static void cgroup_finish_destroy(struct cgroup *cgrp);
 static void kill_css_sync(struct cgroup_subsys_state *css);
 static void kill_css_finish(struct cgroup_subsys_state *css);
 static struct cgroup_subsys_state *css_create(struct cgroup *cgrp,
@@ -376,11 +375,6 @@ static void cgroup_idr_remove(struct idr *idr, int id)
 	spin_unlock_bh(&cgroup_idr_lock);
 }
 
-static bool cgroup_has_tasks(struct cgroup *cgrp)
-{
-	return cgrp->nr_populated_csets;
-}
-
 static bool cgroup_is_threaded(struct cgroup *cgrp)
 {
 	return cgrp->dom_cgrp != cgrp;
@@ -409,7 +403,7 @@ static bool cgroup_can_be_thread_root(struct cgroup *cgrp)
 		return false;
 
 	/* can only have either domain or threaded children */
-	if (cgrp->nr_populated_domain_children)
+	if (READ_ONCE(cgrp->nr_populated_domain_children))
 		return false;
 
 	/* and no domain controllers can be enabled */
@@ -761,62 +755,76 @@ static bool css_set_populated(struct css_set *cset)
 }
 
 /**
- * cgroup_update_populated - update the populated count of a cgroup
- * @cgrp: the target cgroup
- * @populated: inc or dec populated count
- *
- * One of the css_sets associated with @cgrp is either getting its first
- * task or losing the last.  Update @cgrp->nr_populated_* accordingly.  The
- * count is propagated towards root so that a given cgroup's
- * nr_populated_children is zero iff none of its descendants contain any
- * tasks.
- *
- * @cgrp's interface file "cgroup.populated" is zero if both
- * @cgrp->nr_populated_csets and @cgrp->nr_populated_children are zero and
- * 1 otherwise.  When the sum changes from or to zero, userland is notified
- * that the content of the interface file has changed.  This can be used to
- * detect when @cgrp and its descendants become populated or empty.
+ * css_update_populated - update the populated state of a css and ancestors
+ * @css: leaf css whose own populated count is changing
+ * @populated: inc or dec
+ *
+ * One of the css_sets pinned by @css is getting its first task or losing the
+ * last. Propagate the transition up the parent chain so that a css's
+ * nr_populated_children is zero iff none of its descendants contain any tasks.
+ *
+ * For a cgroup->self walk, also runs cgroup-side bookkeeping at each level:
+ * domain/threaded child split, deferred-destroy trigger, and notification via
+ * "cgroup.populated" (zero iff cgrp->self has neither populated csets nor
+ * populated children; userland is notified on transitions).
  */
-static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
+static void css_update_populated(struct cgroup_subsys_state *css, bool populated)
 {
-	struct cgroup *child = NULL;
+	struct cgroup_subsys_state *child = NULL;
 	int adj = populated ? 1 : -1;
 
 	lockdep_assert_held(&css_set_lock);
 
 	do {
-		bool was_populated = cgroup_is_populated(cgrp);
+		/* non-NULL only on the cgroup->self walk */
+		struct cgroup *cgrp = css_is_self(css) ? css->cgroup : NULL;
+		bool was_populated = css_is_populated(css);
 
 		if (!child) {
-			cgrp->nr_populated_csets += adj;
+			WRITE_ONCE(css->nr_populated_csets,
+				   css->nr_populated_csets + adj);
 		} else {
-			if (cgroup_is_threaded(child))
-				cgrp->nr_populated_threaded_children += adj;
-			else
-				cgrp->nr_populated_domain_children += adj;
+			WRITE_ONCE(css->nr_populated_children,
+				   css->nr_populated_children + adj);
+			if (cgrp) {
+				if (cgroup_is_threaded(child->cgroup))
+					WRITE_ONCE(cgrp->nr_populated_threaded_children,
+						   cgrp->nr_populated_threaded_children + adj);
+				else
+					WRITE_ONCE(cgrp->nr_populated_domain_children,
+						   cgrp->nr_populated_domain_children + adj);
+			}
 		}
 
-		if (was_populated == cgroup_is_populated(cgrp))
+		if (was_populated == css_is_populated(css))
 			break;
 
 		/*
-		 * Subtree just emptied below an offlined cgrp. Fire deferred
-		 * destroy. The transition is one-shot.
+		 * Pair with smp_mb() in kill_css_sync(). Either we observe
+		 * CSS_DYING and queue, or the caller observes our decrement
+		 * and fires synchronously.
+		 */
+		smp_mb();
+
+		/*
+		 * Subtree just emptied below a dying css. Fire deferred kill.
+		 * The transition is one-shot for a dying css.
 		 */
-		if (was_populated && !css_is_online(&cgrp->self)) {
-			cgroup_get(cgrp);
-			WARN_ON_ONCE(!queue_work(cgroup_offline_wq,
-						 &cgrp->finish_destroy_work));
+		if (was_populated && css_is_dying(css)) {
+			css_get(css);
+			WARN_ON_ONCE(!queue_work(cgroup_offline_wq, &css->kill_finish_work));
 		}
 
-		cgroup1_check_for_release(cgrp);
-		TRACE_CGROUP_PATH(notify_populated, cgrp,
-				  cgroup_is_populated(cgrp));
-		cgroup_file_notify(&cgrp->events_file);
+		if (cgrp) {
+			cgroup1_check_for_release(cgrp);
+			TRACE_CGROUP_PATH(notify_populated, cgrp,
+					  cgroup_is_populated(cgrp));
+			cgroup_file_notify(&cgrp->events_file);
+		}
 
-		child = cgrp;
-		cgrp = cgroup_parent(cgrp);
-	} while (cgrp);
+		child = css;
+		css = css->parent;
+	} while (css);
 }
 
 /**
@@ -824,17 +832,27 @@ static void cgroup_update_populated(struct cgroup *cgrp, bool populated)
  * @cset: target css_set
  * @populated: whether @cset is populated or depopulated
  *
- * @cset is either getting the first task or losing the last.  Update the
- * populated counters of all associated cgroups accordingly.
+ * @cset is either getting the first task or losing the last. Update the
+ * populated counters along each linked cgroup's self chain and each
+ * subsystem css that @cset pins.
  */
 static void css_set_update_populated(struct css_set *cset, bool populated)
 {
 	struct cgrp_cset_link *link;
+	struct cgroup_subsys *ss;
+	int ssid;
 
 	lockdep_assert_held(&css_set_lock);
 
 	list_for_each_entry(link, &cset->cgrp_links, cgrp_link)
-		cgroup_update_populated(link->cgrp, populated);
+		css_update_populated(&link->cgrp->self, populated);
+
+	for_each_subsys(ss, ssid) {
+		struct cgroup_subsys_state *css = cset->subsys[ssid];
+
+		if (css)
+			css_update_populated(css, populated);
+	}
 }
 
 /*
@@ -2051,16 +2069,6 @@ static int cgroup_reconfigure(struct fs_context *fc)
 	return 0;
 }
 
-static void cgroup_finish_destroy_work_fn(struct work_struct *work)
-{
-	struct cgroup *cgrp = container_of(work, struct cgroup, finish_destroy_work);
-
-	cgroup_lock();
-	cgroup_finish_destroy(cgrp);
-	cgroup_unlock();
-	cgroup_put(cgrp);
-}
-
 static void init_cgroup_housekeeping(struct cgroup *cgrp)
 {
 	struct cgroup_subsys *ss;
@@ -2087,7 +2095,6 @@ static void init_cgroup_housekeeping(struct cgroup *cgrp)
 #endif
 
 	init_waitqueue_head(&cgrp->offline_waitq);
-	INIT_WORK(&cgrp->finish_destroy_work, cgroup_finish_destroy_work_fn);
 	INIT_WORK(&cgrp->release_agent_work, cgroup1_release_agent);
 }
 
@@ -2192,7 +2199,7 @@ int cgroup_setup_root(struct cgroup_root *root, u32 ss_mask)
 	hash_for_each(css_set_table, i, cset, hlist) {
 		link_css_set(&tmp_links, cset, root_cgrp);
 		if (css_set_populated(cset))
-			cgroup_update_populated(root_cgrp, true);
+			css_update_populated(&root_cgrp->self, true);
 	}
 	spin_unlock_irq(&css_set_lock);
 
@@ -3230,7 +3237,7 @@ void cgroup_lock_and_drain_offline(struct cgroup *cgrp)
 			struct cgroup_subsys_state *css = cgroup_css(dsct, ss);
 			DEFINE_WAIT(wait);
 
-			if (!css || !percpu_ref_is_dying(&css->refcnt))
+			if (!css || !css_is_dying(css))
 				continue;
 
 			cgroup_get_live(dsct);
@@ -3398,7 +3405,8 @@ static void cgroup_apply_control_disable(struct cgroup *cgrp)
 			if (css->parent &&
 			    !(cgroup_ss_mask(dsct) & (1 << ss->id))) {
 				kill_css_sync(css);
-				kill_css_finish(css);
+				if (!css_is_populated(css))
+					kill_css_finish(css);
 			} else if (!css_visible(css)) {
 				css_clear_dir(css);
 				if (ss->css_reset)
@@ -5684,6 +5692,22 @@ static void css_release(struct percpu_ref *ref)
 	queue_work(cgroup_release_wq, &css->destroy_work);
 }
 
+/*
+ * Deferred kill_css_finish() fired from css_update_populated() once a dying
+ * css's hierarchical populated state drops to zero. Pinned by css_get() at the
+ * queue site; matched by css_put() here.
+ */
+static void kill_css_finish_work_fn(struct work_struct *work)
+{
+	struct cgroup_subsys_state *css =
+		container_of(work, struct cgroup_subsys_state, kill_finish_work);
+
+	cgroup_lock();
+	kill_css_finish(css);
+	cgroup_unlock();
+	css_put(css);
+}
+
 static void init_and_link_css(struct cgroup_subsys_state *css,
 			      struct cgroup_subsys *ss, struct cgroup *cgrp)
 {
@@ -5697,6 +5721,7 @@ static void init_and_link_css(struct cgroup_subsys_state *css,
 	css->id = -1;
 	INIT_LIST_HEAD(&css->sibling);
 	INIT_LIST_HEAD(&css->children);
+	INIT_WORK(&css->kill_finish_work, kill_css_finish_work_fn);
 	css->serial_nr = css_serial_nr_next++;
 	atomic_set(&css->online_cnt, 0);
 
@@ -6073,6 +6098,13 @@ static void kill_css_sync(struct cgroup_subsys_state *css)
 
 	css->flags |= CSS_DYING;
 
+	/*
+	 * Pair with smp_mb() in css_update_populated(). Either our
+	 * caller observes the walker's decrement and fires
+	 * synchronously, or the walker observes CSS_DYING and queues.
+	 */
+	smp_mb();
+
 	/*
 	 * This must happen before css is disassociated with its cgroup.
 	 * See seq_css() for details.
@@ -6148,9 +6180,9 @@ static void kill_css_finish(struct cgroup_subsys_state *css)
  * - This function: synchronous user-visible state teardown plus kill_css_sync()
  *   on each subsystem css.
  *
- * - cgroup_finish_destroy(): kicks the percpu_ref kill via kill_css_finish() on
- *   each subsystem css. Fires once @cgrp's subtree is fully drained, either
- *   inline here or from cgroup_update_populated().
+ * - For each subsys css: fire kill_css_finish() synchronously if the subtree is
+ *   already drained, otherwise rely on css_update_populated() to queue
+ *   kill_finish_work when the last populated cset under the css empties.
  *
  * - The percpu_ref kill chain: css_killed_ref_fn -> css_killed_work_fn ->
  *   ->css_offline() -> release/free.
@@ -6228,29 +6260,14 @@ static int cgroup_destroy_locked(struct cgroup *cgrp)
 	/* put the base reference */
 	percpu_ref_kill(&cgrp->self.refcnt);
 
-	if (!cgroup_is_populated(cgrp))
-		cgroup_finish_destroy(cgrp);
+	for_each_css(css, ssid, cgrp) {
+		if (!css_is_populated(css))
+			kill_css_finish(css);
+	}
 
 	return 0;
 };
 
-/**
- * cgroup_finish_destroy - deferred half of @cgrp destruction
- * @cgrp: cgroup whose subtree just became empty
- *
- * See cgroup_destroy_locked() for the rationale.
- */
-static void cgroup_finish_destroy(struct cgroup *cgrp)
-{
-	struct cgroup_subsys_state *css;
-	int ssid;
-
-	lockdep_assert_held(&cgroup_mutex);
-
-	for_each_css(css, ssid, cgrp)
-		kill_css_finish(css);
-}
-
 int cgroup_rmdir(struct kernfs_node *kn)
 {
 	struct cgroup *cgrp;
diff --git a/kernel/cgroup/cpuset-v1.c b/kernel/cgroup/cpuset-v1.c
index 7308e9b024951..3e9968dd91e9f 100644
--- a/kernel/cgroup/cpuset-v1.c
+++ b/kernel/cgroup/cpuset-v1.c
@@ -312,7 +312,7 @@ void cpuset1_hotplug_update_tasks(struct cpuset *cs,
 	 * This is full cgroup operation which will also call back into
 	 * cpuset. Execute it asynchronously using workqueue.
 	 */
-	if (is_empty && cs->css.cgroup->nr_populated_csets &&
+	if (is_empty && cgroup_has_tasks(cs->css.cgroup) &&
 	    css_tryget_online(&cs->css)) {
 		struct cpuset_remove_tasks_struct *s;
 
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 5c33ab20cc208..51327333980a4 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -432,7 +432,7 @@ static inline bool partition_is_populated(struct cpuset *cs,
 	 * nr_populated_domain_children may include populated
 	 * csets from descendants that are partitions.
 	 */
-	if (cs->css.cgroup->nr_populated_csets ||
+	if (cgroup_has_tasks(cs->css.cgroup) ||
 	    cs->attach_in_progress)
 		return true;
 
@@ -4235,6 +4235,9 @@ bool cpuset_current_node_allowed(int node, gfp_t gfp_mask)
 	if (gfp_mask & __GFP_HARDWALL)	/* If hardwall request, stop here */
 		return false;
 
+	if (cpuset_v2())
+		return true;
+
 	/* Not hardwall and node outside mems_allowed: scan up cpusets */
 	spin_lock_irqsave(&callback_lock, flags);
 
diff --git a/kernel/cgroup/rdma.c b/kernel/cgroup/rdma.c
index 4fdab4cf49e0f..5e82a03b3270a 100644
--- a/kernel/cgroup/rdma.c
+++ b/kernel/cgroup/rdma.c
@@ -9,6 +9,7 @@
  */
 
 #include <linux/bitops.h>
+#include <linux/limits.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
 #include <linux/cgroup.h>
@@ -17,6 +18,22 @@
 
 #define RDMACG_MAX_STR "max"
 
+enum rdmacg_limit_tokens {
+	RDMACG_HCA_HANDLE_VAL,
+	RDMACG_HCA_HANDLE_MAX,
+	RDMACG_HCA_OBJECT_VAL,
+	RDMACG_HCA_OBJECT_MAX,
+	NR_RDMACG_LIMIT_TOKENS,
+};
+
+static const match_table_t rdmacg_limit_tokens = {
+	{ RDMACG_HCA_HANDLE_VAL,	"hca_handle=%d"	},
+	{ RDMACG_HCA_HANDLE_MAX,	"hca_handle=max"	},
+	{ RDMACG_HCA_OBJECT_VAL,	"hca_object=%d"	},
+	{ RDMACG_HCA_OBJECT_MAX,	"hca_object=max"	},
+	{ NR_RDMACG_LIMIT_TOKENS,	NULL			},
+};
+
 /*
  * Protects list of resource pools maintained on per cgroup basis
  * and rdma device list.
@@ -27,6 +44,7 @@ static LIST_HEAD(rdmacg_devices);
 enum rdmacg_file_type {
 	RDMACG_RESOURCE_TYPE_MAX,
 	RDMACG_RESOURCE_TYPE_STAT,
+	RDMACG_RESOURCE_TYPE_PEAK,
 };
 
 /*
@@ -43,6 +61,7 @@ static char const *rdmacg_resource_names[] = {
 struct rdmacg_resource {
 	int max;
 	int usage;
+	int peak;
 };
 
 /*
@@ -62,6 +81,12 @@ struct rdmacg_resource_pool {
 	u64			usage_sum;
 	/* total number counts which are set to max */
 	int			num_max_cnt;
+
+	/* per-resource event counters */
+	u64			events_max[RDMACG_RESOURCE_MAX];
+	u64			events_alloc_fail[RDMACG_RESOURCE_MAX];
+	u64			events_local_max[RDMACG_RESOURCE_MAX];
+	u64			events_local_alloc_fail[RDMACG_RESOURCE_MAX];
 };
 
 static struct rdma_cgroup *css_rdmacg(struct cgroup_subsys_state *css)
@@ -109,6 +134,26 @@ static void free_cg_rpool_locked(struct rdmacg_resource_pool *rpool)
 	kfree(rpool);
 }
 
+static bool rpool_has_persistent_state(struct rdmacg_resource_pool *rpool)
+{
+	int i;
+
+	/*
+	 * Keep the rpool alive if any peak value is non-zero,
+	 * so that rdma.peak persists as a historical high-
+	 * watermark even after all resources are freed.
+	 */
+	for (i = 0; i < RDMACG_RESOURCE_MAX; i++) {
+		if (rpool->resources[i].peak ||
+		    rpool->events_max[i] ||
+		    rpool->events_local_max[i] ||
+		    rpool->events_alloc_fail[i] ||
+		    rpool->events_local_alloc_fail[i])
+			return true;
+	}
+	return false;
+}
+
 static struct rdmacg_resource_pool *
 find_cg_rpool_locked(struct rdma_cgroup *cg,
 		     struct rdmacg_device *device)
@@ -187,11 +232,67 @@ uncharge_cg_locked(struct rdma_cgroup *cg,
 	rpool->usage_sum--;
 	if (rpool->usage_sum == 0 &&
 	    rpool->num_max_cnt == RDMACG_RESOURCE_MAX) {
-		/*
-		 * No user of the rpool and all entries are set to max, so
-		 * safe to delete this rpool.
-		 */
-		free_cg_rpool_locked(rpool);
+		if (!rpool_has_persistent_state(rpool)) {
+			/*
+			 * No user of the rpool and all entries are set to max, so
+			 * safe to delete this rpool.
+			 */
+			free_cg_rpool_locked(rpool);
+		}
+	}
+}
+
+/**
+ * rdmacg_event_locked - fire event when resource allocation exceeds limit
+ * @cg: requesting cgroup
+ * @over_cg: cgroup whose limit was exceeded
+ * @device: rdma device
+ * @index: resource type index
+ *
+ * Must be called under rdmacg_mutex. Updates event counters in the
+ * resource pools of @cg and @over_cg, propagates hierarchical max
+ * events from @over_cg (including itself) upward, and notifies
+ * userspace via cgroup_file_notify().
+ */
+static void rdmacg_event_locked(struct rdma_cgroup *cg,
+				struct rdma_cgroup *over_cg,
+				struct rdmacg_device *device,
+				enum rdmacg_resource_type index)
+{
+	struct rdmacg_resource_pool *rpool;
+	struct rdma_cgroup *p;
+
+	lockdep_assert_held(&rdmacg_mutex);
+
+	/* Increment local alloc_fail in requesting cgroup */
+	rpool = find_cg_rpool_locked(cg, device);
+	if (rpool) {
+		rpool->events_local_alloc_fail[index]++;
+		cgroup_file_notify(&cg->events_local_file);
+	}
+
+	/* Increment local max in the over-limit cgroup */
+	rpool = find_cg_rpool_locked(over_cg, device);
+	if (rpool) {
+		rpool->events_local_max[index]++;
+		cgroup_file_notify(&over_cg->events_local_file);
+	}
+
+	/* Propagate hierarchical max events upward */
+	for (p = over_cg; parent_rdmacg(p); p = parent_rdmacg(p)) {
+		rpool = get_cg_rpool_locked(p, device);
+		if (!IS_ERR(rpool)) {
+			rpool->events_max[index]++;
+			cgroup_file_notify(&p->events_file);
+		}
+	}
+	/* Propagate hierarchical alloc_fail from requesting cgroup upward */
+	for (p = cg; parent_rdmacg(p); p = parent_rdmacg(p)) {
+		rpool = get_cg_rpool_locked(p, device);
+		if (!IS_ERR(rpool)) {
+			rpool->events_alloc_fail[index]++;
+			cgroup_file_notify(&p->events_file);
+		}
 	}
 }
 
@@ -293,12 +394,20 @@ int rdmacg_try_charge(struct rdma_cgroup **rdmacg,
 			}
 		}
 	}
+	/* Update peak only after all charges succeed */
+	for (p = cg; p; p = parent_rdmacg(p)) {
+		rpool = find_cg_rpool_locked(p, device);
+		if (rpool && rpool->resources[index].usage > rpool->resources[index].peak)
+			rpool->resources[index].peak = rpool->resources[index].usage;
+	}
 	mutex_unlock(&rdmacg_mutex);
 
 	*rdmacg = cg;
 	return 0;
 
 err:
+	if (ret == -EAGAIN)
+		rdmacg_event_locked(cg, p, device, index);
 	mutex_unlock(&rdmacg_mutex);
 	rdmacg_uncharge_hierarchy(cg, device, p, index);
 	return ret;
@@ -355,62 +464,6 @@ void rdmacg_unregister_device(struct rdmacg_device *device)
 }
 EXPORT_SYMBOL(rdmacg_unregister_device);
 
-static int parse_resource(char *c, int *intval)
-{
-	substring_t argstr;
-	char *name, *value = c;
-	size_t len;
-	int ret, i;
-
-	name = strsep(&value, "=");
-	if (!name || !value)
-		return -EINVAL;
-
-	i = match_string(rdmacg_resource_names, RDMACG_RESOURCE_MAX, name);
-	if (i < 0)
-		return i;
-
-	len = strlen(value);
-
-	argstr.from = value;
-	argstr.to = value + len;
-
-	ret = match_int(&argstr, intval);
-	if (ret >= 0) {
-		if (*intval < 0)
-			return -EINVAL;
-		return i;
-	}
-	if (strncmp(value, RDMACG_MAX_STR, len) == 0) {
-		*intval = S32_MAX;
-		return i;
-	}
-	return -EINVAL;
-}
-
-static int rdmacg_parse_limits(char *options,
-			       int *new_limits, unsigned long *enables)
-{
-	char *c;
-	int err = -EINVAL;
-
-	/* parse resource options */
-	while ((c = strsep(&options, " ")) != NULL) {
-		int index, intval;
-
-		index = parse_resource(c, &intval);
-		if (index < 0)
-			goto err;
-
-		new_limits[index] = intval;
-		*enables |= BIT(index);
-	}
-	return 0;
-
-err:
-	return err;
-}
-
 static struct rdmacg_device *rdmacg_get_device_locked(const char *name)
 {
 	struct rdmacg_device *device;
@@ -432,6 +485,7 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of,
 	struct rdmacg_resource_pool *rpool;
 	struct rdmacg_device *device;
 	char *options = strstrip(buf);
+	char *p;
 	int *new_limits;
 	unsigned long enables = 0;
 	int i = 0, ret = 0;
@@ -449,9 +503,45 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of,
 		goto err;
 	}
 
-	ret = rdmacg_parse_limits(options, new_limits, &enables);
-	if (ret)
-		goto parse_err;
+	/* parse resource limit tokens */
+	while ((p = strsep(&options, " \t\n"))) {
+		substring_t args[MAX_OPT_ARGS];
+		int tok, intval;
+
+		if (!*p)
+			continue;
+
+		tok = match_token(p, rdmacg_limit_tokens, args);
+		switch (tok) {
+		case RDMACG_HCA_HANDLE_VAL:
+			if (match_int(&args[0], &intval) || intval < 0) {
+				ret = -EINVAL;
+				goto parse_err;
+			}
+			new_limits[RDMACG_RESOURCE_HCA_HANDLE] = intval;
+			enables |= BIT(RDMACG_RESOURCE_HCA_HANDLE);
+			break;
+		case RDMACG_HCA_HANDLE_MAX:
+			new_limits[RDMACG_RESOURCE_HCA_HANDLE] = S32_MAX;
+			enables |= BIT(RDMACG_RESOURCE_HCA_HANDLE);
+			break;
+		case RDMACG_HCA_OBJECT_VAL:
+			if (match_int(&args[0], &intval) || intval < 0) {
+				ret = -EINVAL;
+				goto parse_err;
+			}
+			new_limits[RDMACG_RESOURCE_HCA_OBJECT] = intval;
+			enables |= BIT(RDMACG_RESOURCE_HCA_OBJECT);
+			break;
+		case RDMACG_HCA_OBJECT_MAX:
+			new_limits[RDMACG_RESOURCE_HCA_OBJECT] = S32_MAX;
+			enables |= BIT(RDMACG_RESOURCE_HCA_OBJECT);
+			break;
+		default:
+			ret = -EINVAL;
+			goto parse_err;
+		}
+	}
 
 	/* acquire lock to synchronize with hot plug devices */
 	mutex_lock(&rdmacg_mutex);
@@ -474,11 +564,13 @@ static ssize_t rdmacg_resource_set_max(struct kernfs_open_file *of,
 
 	if (rpool->usage_sum == 0 &&
 	    rpool->num_max_cnt == RDMACG_RESOURCE_MAX) {
-		/*
-		 * No user of the rpool and all entries are set to max, so
-		 * safe to delete this rpool.
-		 */
-		free_cg_rpool_locked(rpool);
+		if (!rpool_has_persistent_state(rpool)) {
+			/*
+			 * No user of the rpool and all entries are set to max, so
+			 * safe to delete this rpool.
+			 */
+			free_cg_rpool_locked(rpool);
+		}
 	}
 
 dev_err:
@@ -508,6 +600,8 @@ static void print_rpool_values(struct seq_file *sf,
 				value = rpool->resources[i].max;
 			else
 				value = S32_MAX;
+		} else if (sf_type == RDMACG_RESOURCE_TYPE_PEAK) {
+			value = rpool ? rpool->resources[i].peak : 0;
 		} else {
 			if (rpool)
 				value = rpool->resources[i].usage;
@@ -544,6 +638,64 @@ static int rdmacg_resource_read(struct seq_file *sf, void *v)
 	return 0;
 }
 
+static int rdmacg_events_show(struct seq_file *sf, void *v)
+{
+	struct rdma_cgroup *cg = css_rdmacg(seq_css(sf));
+	struct rdmacg_resource_pool *rpool;
+	struct rdmacg_device *device;
+	int i;
+
+	mutex_lock(&rdmacg_mutex);
+
+	list_for_each_entry(device, &rdmacg_devices, dev_node) {
+		rpool = find_cg_rpool_locked(cg, device);
+
+		seq_printf(sf, "%s ", device->name);
+		for (i = 0; i < RDMACG_RESOURCE_MAX; i++) {
+			seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu",
+				   rdmacg_resource_names[i],
+				   rpool ? rpool->events_max[i] : 0ULL,
+				   rdmacg_resource_names[i],
+				   rpool ? rpool->events_alloc_fail[i] : 0ULL);
+			if (i < RDMACG_RESOURCE_MAX - 1)
+				seq_putc(sf, ' ');
+		}
+		seq_putc(sf, '\n');
+	}
+
+	mutex_unlock(&rdmacg_mutex);
+	return 0;
+}
+
+static int rdmacg_events_local_show(struct seq_file *sf, void *v)
+{
+	struct rdma_cgroup *cg = css_rdmacg(seq_css(sf));
+	struct rdmacg_resource_pool *rpool;
+	struct rdmacg_device *device;
+	int i;
+
+	mutex_lock(&rdmacg_mutex);
+
+	list_for_each_entry(device, &rdmacg_devices, dev_node) {
+		rpool = find_cg_rpool_locked(cg, device);
+
+		seq_printf(sf, "%s ", device->name);
+		for (i = 0; i < RDMACG_RESOURCE_MAX; i++) {
+			seq_printf(sf, "%s.max=%llu %s.alloc_fail=%llu",
+				   rdmacg_resource_names[i],
+				   rpool ? rpool->events_local_max[i] : 0ULL,
+				   rdmacg_resource_names[i],
+				   rpool ? rpool->events_local_alloc_fail[i] : 0ULL);
+			if (i < RDMACG_RESOURCE_MAX - 1)
+				seq_putc(sf, ' ');
+		}
+		seq_putc(sf, '\n');
+	}
+
+	mutex_unlock(&rdmacg_mutex);
+	return 0;
+}
+
 static struct cftype rdmacg_files[] = {
 	{
 		.name = "max",
@@ -558,6 +710,24 @@ static struct cftype rdmacg_files[] = {
 		.private = RDMACG_RESOURCE_TYPE_STAT,
 		.flags = CFTYPE_NOT_ON_ROOT,
 	},
+	{
+		.name = "peak",
+		.seq_show = rdmacg_resource_read,
+		.private = RDMACG_RESOURCE_TYPE_PEAK,
+		.flags = CFTYPE_NOT_ON_ROOT,
+	},
+	{
+		.name = "events",
+		.seq_show = rdmacg_events_show,
+		.file_offset = offsetof(struct rdma_cgroup, events_file),
+		.flags = CFTYPE_NOT_ON_ROOT,
+	},
+	{
+		.name = "events.local",
+		.seq_show = rdmacg_events_local_show,
+		.file_offset = offsetof(struct rdma_cgroup, events_local_file),
+		.flags = CFTYPE_NOT_ON_ROOT,
+	},
 	{ }	/* terminate */
 };
 
@@ -577,6 +747,13 @@ rdmacg_css_alloc(struct cgroup_subsys_state *parent)
 static void rdmacg_css_free(struct cgroup_subsys_state *css)
 {
 	struct rdma_cgroup *cg = css_rdmacg(css);
+	struct rdmacg_resource_pool *rpool, *tmp;
+
+	/* Clean up rpools kept alive by non-zero peak values */
+	mutex_lock(&rdmacg_mutex);
+	list_for_each_entry_safe(rpool, tmp, &cg->rpools, cg_node)
+		free_cg_rpool_locked(rpool);
+	mutex_unlock(&rdmacg_mutex);
 
 	kfree(cg);
 }
diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index 150e5871e66f2..de816a43db9f0 100644
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0-only
 #include "cgroup-internal.h"
 
+#include <linux/cpumask.h>
 #include <linux/sched/cputime.h>
 
 #include <linux/bpf.h>
@@ -53,7 +54,7 @@ static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu)
 }
 
 /**
- * css_rstat_updated - keep track of updated rstat_cpu
+ * __css_rstat_updated - keep track of updated rstat_cpu
  * @css: target cgroup subsystem state
  * @cpu: cpu on which rstat_cpu was updated
  *
@@ -63,31 +64,27 @@ static inline struct llist_head *ss_lhead_cpu(struct cgroup_subsys *ss, int cpu)
  *
  * NOTE: if the user needs the guarantee that the updater either add itself in
  * the lockless list or the concurrent flusher flushes its updated stats, a
- * memory barrier is needed before the call to css_rstat_updated() i.e. a
+ * memory barrier is needed before the call to __css_rstat_updated() i.e. a
  * barrier after updating the per-cpu stats and before calling
- * css_rstat_updated().
+ * __css_rstat_updated().
  */
-__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
+void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
 {
 	struct llist_head *lhead;
 	struct css_rstat_cpu *rstatc;
 	struct llist_node *self;
 
-	/*
-	 * Since bpf programs can call this function, prevent access to
-	 * uninitialized rstat pointers.
-	 */
+	/* Prevent access to uninitialized rstat pointers. */
 	if (!css_uses_rstat(css))
 		return;
 
 	lockdep_assert_preemption_disabled();
 
 	/*
-	 * For archs withnot nmi safe cmpxchg or percpu ops support, ignore
-	 * the requests from nmi context.
+	 * The lockless insertion below relies on NMI-safe cmpxchg;
+	 * bail out in NMI on archs that don't provide it.
 	 */
-	if ((!IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG) ||
-	     !IS_ENABLED(CONFIG_ARCH_HAS_NMI_SAFE_THIS_CPU_OPS)) && in_nmi())
+	if (!IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG) && in_nmi())
 		return;
 
 	rstatc = css_rstat_cpu(css, cpu);
@@ -125,6 +122,18 @@ __bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
 	llist_add(&rstatc->lnode, lhead);
 }
 
+/*
+ * BPF-facing wrapper for __css_rstat_updated(). Validate the caller-provided
+ * CPU before passing it to the internal rstat updater.
+ */
+__bpf_kfunc void css_rstat_updated(struct cgroup_subsys_state *css, int cpu)
+{
+	if (unlikely(cpu < 0 || cpu >= nr_cpu_ids || !cpu_possible(cpu)))
+		return;
+
+	__css_rstat_updated(css, cpu);
+}
+
 static void __css_process_update_tree(struct cgroup_subsys_state *css, int cpu)
 {
 	/* put @css and all ancestors on the corresponding updated lists */
@@ -170,7 +179,7 @@ static void css_process_update_tree(struct cgroup_subsys *ss, int cpu)
 		 * flusher flush the stats updated by the updater who have
 		 * observed that they are already on the list. The
 		 * corresponding barrier pair for this one should be before
-		 * css_rstat_updated() by the user.
+		 * __css_rstat_updated() by the user.
 		 *
 		 * For now, there aren't any such user, so not adding the
 		 * barrier here but if such a use-case arise, please add
@@ -614,7 +623,7 @@ static void cgroup_base_stat_cputime_account_end(struct cgroup *cgrp,
 						 unsigned long flags)
 {
 	u64_stats_update_end_irqrestore(&rstatbc->bsync, flags);
-	css_rstat_updated(&cgrp->self, smp_processor_id());
+	__css_rstat_updated(&cgrp->self, smp_processor_id());
 	put_cpu_ptr(rstatbc);
 }
 
diff --git a/kernel/configs/hardening.config b/kernel/configs/hardening.config
index 7c3924614e01d..26831a2a57392 100644
--- a/kernel/configs/hardening.config
+++ b/kernel/configs/hardening.config
@@ -22,7 +22,7 @@ CONFIG_SLAB_FREELIST_RANDOM=y
 CONFIG_SLAB_FREELIST_HARDENED=y
 CONFIG_SLAB_BUCKETS=y
 CONFIG_SHUFFLE_PAGE_ALLOCATOR=y
-CONFIG_RANDOM_KMALLOC_CACHES=y
+CONFIG_KMALLOC_PARTITION_CACHES=y
 
 # Sanity check userspace page table mappings.
 CONFIG_PAGE_TABLE_CHECK=y
diff --git a/kernel/cpu.c b/kernel/cpu.c
index bc4f7a9ba64e6..f975bb34915b0 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -2639,7 +2639,7 @@ static void cpuhp_offline_cpu_device(unsigned int cpu)
 {
 	struct device *dev = get_cpu_device(cpu);
 
-	dev->offline = true;
+	dev_set_offline(dev);
 	/* Tell user space about the state change */
 	kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
 }
@@ -2648,7 +2648,7 @@ static void cpuhp_online_cpu_device(unsigned int cpu)
 {
 	struct device *dev = get_cpu_device(cpu);
 
-	dev->offline = false;
+	dev_clear_offline(dev);
 	/* Tell user space about the state change */
 	kobject_uevent(&dev->kobj, KOBJ_ONLINE);
 }
diff --git a/kernel/dma/Kconfig b/kernel/dma/Kconfig
index bfef21b4a9ae7..c9fa0a922cba9 100644
--- a/kernel/dma/Kconfig
+++ b/kernel/dma/Kconfig
@@ -181,6 +181,16 @@ config DMA_NUMA_CMA
 	  or set the node id and its size of CMA by specifying "numa_cma=
 	  <node>:size[,<node>:size]" on the kernel's command line.
 
+config CMA_SIZE_PERNUMA
+	bool "Default CMA area per NUMA node"
+	depends on DMA_NUMA_CMA
+	default y
+	help
+	  On systems with more than one NUMA node, the selected CMA
+	  area size will be also allocated on each additional node,
+	  so that most devices may have benefit from better DMA
+	  locality without an explicit command-line opt-in.
+
 comment "Default contiguous memory area size:"
 
 config CMA_SIZE_MBYTES
diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c
index 03f52bd171206..f286d49bf068c 100644
--- a/kernel/dma/contiguous.c
+++ b/kernel/dma/contiguous.c
@@ -136,6 +136,7 @@ static struct cma *dma_contiguous_numa_area[MAX_NUMNODES];
 static phys_addr_t numa_cma_size[MAX_NUMNODES] __initdata;
 static struct cma *dma_contiguous_pernuma_area[MAX_NUMNODES];
 static phys_addr_t pernuma_size_bytes __initdata;
+static bool __maybe_unused numa_cma_configured;
 
 static int __init early_numa_cma(char *p)
 {
@@ -164,6 +165,7 @@ static int __init early_numa_cma(char *p)
 			break;
 	}
 
+	numa_cma_configured = true;
 	return 0;
 }
 early_param("numa_cma", early_numa_cma);
@@ -171,6 +173,7 @@ early_param("numa_cma", early_numa_cma);
 static int __init early_cma_pernuma(char *p)
 {
 	pernuma_size_bytes = memparse(p, &p);
+	numa_cma_configured = true;
 	return 0;
 }
 early_param("cma_pernuma", early_cma_pernuma);
@@ -199,6 +202,12 @@ static void __init dma_numa_cma_reserve(void)
 {
 	int nid;
 
+#ifdef CONFIG_CMA_SIZE_PERNUMA
+	if (!numa_cma_configured && dma_contiguous_default_area
+	    && nr_online_nodes > 1)
+		pernuma_size_bytes = cma_get_size(dma_contiguous_default_area);
+#endif
+
 	for_each_node(nid) {
 		int ret;
 		char name[CMA_MAX_NAME];
@@ -255,8 +264,6 @@ void __init dma_contiguous_reserve(phys_addr_t limit)
 	phys_addr_t selected_limit = limit;
 	bool fixed = false;
 
-	dma_numa_cma_reserve();
-
 	pr_debug("%s(limit %08lx)\n", __func__, (unsigned long)limit);
 
 	if (size_cmdline != -1) {
@@ -312,6 +319,8 @@ void __init dma_contiguous_reserve(phys_addr_t limit)
 		if (ret)
 			pr_warn("Couldn't queue default CMA region for heap creation.");
 	}
+
+	dma_numa_cma_reserve();
 }
 
 void __weak
diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c
index 1a725edbbbf6a..fc177dda611e2 100644
--- a/kernel/dma/debug.c
+++ b/kernel/dma/debug.c
@@ -63,7 +63,7 @@ enum map_err_types {
  * @sg_mapped_ents: 'mapped_ents' from dma_map_sg
  * @paddr: physical start address of the mapping
  * @map_err_type: track whether dma_mapping_error() was checked
- * @is_cache_clean: driver promises not to write to buffer while mapped
+ * @attrs: dma attributes
  * @stack_len: number of backtrace entries in @stack_entries
  * @stack_entries: stack of backtrace history
  */
@@ -78,7 +78,7 @@ struct dma_debug_entry {
 	int		 sg_mapped_ents;
 	phys_addr_t	 paddr;
 	enum map_err_types map_err_type;
-	bool		 is_cache_clean;
+	unsigned long	 attrs;
 #ifdef CONFIG_STACKTRACE
 	unsigned int	stack_len;
 	unsigned long	stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
@@ -478,6 +478,9 @@ static int active_cacheline_insert(struct dma_debug_entry *entry,
 				   bool *overlap_cache_clean)
 {
 	phys_addr_t cln = to_cacheline_number(entry);
+	bool is_cache_clean = entry->attrs &
+			      (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |
+			       DMA_ATTR_REQUIRE_COHERENT);
 	unsigned long flags;
 	int rc;
 
@@ -495,12 +498,15 @@ static int active_cacheline_insert(struct dma_debug_entry *entry,
 	if (rc == -EEXIST) {
 		struct dma_debug_entry *existing;
 
-		active_cacheline_inc_overlap(cln, entry->is_cache_clean);
+		active_cacheline_inc_overlap(cln, is_cache_clean);
 		existing = radix_tree_lookup(&dma_active_cacheline, cln);
 		/* A lookup failure here after we got -EEXIST is unexpected. */
 		WARN_ON(!existing);
 		if (existing)
-			*overlap_cache_clean = existing->is_cache_clean;
+			*overlap_cache_clean =
+				existing->attrs &
+				(DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |
+				 DMA_ATTR_REQUIRE_COHERENT);
 	}
 	spin_unlock_irqrestore(&radix_lock, flags);
 
@@ -544,12 +550,13 @@ void debug_dma_dump_mappings(struct device *dev)
 			if (!dev || dev == entry->dev) {
 				cln = to_cacheline_number(entry);
 				dev_info(entry->dev,
-					 "%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n",
+					 "%s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n",
 					 type2name[entry->type], idx,
 					 &entry->paddr, entry->dev_addr,
 					 entry->size, &cln,
 					 dir2name[entry->direction],
-					 maperr2str[entry->map_err_type]);
+					 maperr2str[entry->map_err_type],
+					 entry->attrs);
 			}
 		}
 		spin_unlock_irqrestore(&bucket->lock, flags);
@@ -575,14 +582,15 @@ static int dump_show(struct seq_file *seq, void *v)
 		list_for_each_entry(entry, &bucket->list, list) {
 			cln = to_cacheline_number(entry);
 			seq_printf(seq,
-				   "%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s\n",
+				   "%s %s %s idx %d P=%pa D=%llx L=%llx cln=%pa %s %s attrs=0x%lx\n",
 				   dev_driver_string(entry->dev),
 				   dev_name(entry->dev),
 				   type2name[entry->type], idx,
 				   &entry->paddr, entry->dev_addr,
 				   entry->size, &cln,
 				   dir2name[entry->direction],
-				   maperr2str[entry->map_err_type]);
+				   maperr2str[entry->map_err_type],
+				   entry->attrs);
 		}
 		spin_unlock_irqrestore(&bucket->lock, flags);
 	}
@@ -594,16 +602,14 @@ DEFINE_SHOW_ATTRIBUTE(dump);
  * Wrapper function for adding an entry to the hash.
  * This function takes care of locking itself.
  */
-static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs)
+static void add_dma_entry(struct dma_debug_entry *entry)
 {
+	unsigned long attrs = entry->attrs;
 	bool overlap_cache_clean;
 	struct hash_bucket *bucket;
 	unsigned long flags;
 	int rc;
 
-	entry->is_cache_clean = attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |
-					 DMA_ATTR_REQUIRE_COHERENT);
-
 	bucket = get_hash_bucket(entry, &flags);
 	hash_bucket_add(bucket, entry);
 	put_hash_bucket(bucket, flags);
@@ -612,9 +618,10 @@ static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs)
 	if (rc == -ENOMEM) {
 		pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n");
 		global_disable = true;
-	} else if (rc == -EEXIST &&
-		   !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
-		   !(entry->is_cache_clean && overlap_cache_clean) &&
+	} else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
+		   !(attrs & (DMA_ATTR_DEBUGGING_IGNORE_CACHELINES |
+			      DMA_ATTR_REQUIRE_COHERENT) &&
+		     overlap_cache_clean) &&
 		   dma_get_cache_alignment() >= L1_CACHE_BYTES &&
 		   !(IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) &&
 		     is_swiotlb_active(entry->dev))) {
@@ -1067,6 +1074,29 @@ static void check_unmap(struct dma_debug_entry *ref)
 			   type2name[entry->type]);
 	}
 
+	/*
+	 * This may be no bug in reality - but DMA API still expects
+	 * that entry is unmapped with same attributes as it was mapped.
+	 *
+	 * DMA_ATTR_UNMAP_VALID lists the attributes that must be identical
+	 * between map and unmap. Any attribute outside this set (e.g.
+	 * DMA_ATTR_NO_WARN, DMA_ATTR_SKIP_CPU_SYNC) is allowed to differ.
+	 */
+#define DMA_ATTR_UNMAP_VALID                                               \
+	(DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_FORCE_CONTIGUOUS |          \
+	 DMA_ATTR_MMIO | DMA_ATTR_REQUIRE_COHERENT | DMA_ATTR_PRIVILEGED | \
+	 DMA_ATTR_CC_SHARED)
+	if ((ref->attrs & DMA_ATTR_UNMAP_VALID) !=
+	    (entry->attrs & DMA_ATTR_UNMAP_VALID)) {
+		err_printk(ref->dev, entry,
+			   "device driver frees "
+			   "DMA memory with different attributes "
+			   "[device address=0x%016llx] [size=%llu bytes] "
+			   "[mapped with 0x%lx] [unmapped with 0x%lx]\n",
+			   ref->dev_addr, ref->size, entry->attrs, ref->attrs);
+	}
+#undef DMA_ATTR_UNMAP_VALID
+
 	hash_bucket_del(entry);
 	put_hash_bucket(bucket, flags);
 
@@ -1250,15 +1280,23 @@ void debug_dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,
 	entry->size      = size;
 	entry->direction = direction;
 	entry->map_err_type = MAP_ERR_NOT_CHECKED;
+	entry->attrs     = attrs;
 
-	if (!(attrs & DMA_ATTR_MMIO)) {
+	if (attrs & DMA_ATTR_MMIO) {
+		unsigned long pfn = PHYS_PFN(phys);
+
+		if (pfn_valid(pfn) && !PageReserved(pfn_to_page(pfn)))
+			err_printk(dev, entry,
+				   "dma_map_resource called for RAM address %pa\n",
+				   &phys);
+	} else {
 		check_for_stack(dev, phys);
 
 		if (!PhysHighMem(phys))
 			check_for_illegal_area(dev, phys_to_virt(phys), size);
 	}
 
-	add_dma_entry(entry, attrs);
+	add_dma_entry(entry);
 }
 
 void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
@@ -1299,8 +1337,8 @@ void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 }
 EXPORT_SYMBOL(debug_dma_mapping_error);
 
-void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr,
-			  size_t size, int direction)
+void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size,
+			  int direction, unsigned long attrs)
 {
 	struct dma_debug_entry ref = {
 		.type           = dma_debug_phy,
@@ -1308,6 +1346,7 @@ void debug_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr,
 		.dev_addr       = dma_addr,
 		.size           = size,
 		.direction      = direction,
+		.attrs          = attrs,
 	};
 
 	if (unlikely(dma_debug_disabled()))
@@ -1345,10 +1384,11 @@ void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 		entry->direction      = direction;
 		entry->sg_call_ents   = nents;
 		entry->sg_mapped_ents = mapped_ents;
+		entry->attrs          = attrs;
 
 		check_sg_segment(dev, s);
 
-		add_dma_entry(entry, attrs);
+		add_dma_entry(entry);
 	}
 }
 
@@ -1372,7 +1412,7 @@ static int get_nr_mapped_entries(struct device *dev,
 }
 
 void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
-			int nelems, int dir)
+			int nelems, int dir, unsigned long attrs)
 {
 	struct scatterlist *s;
 	int mapped_ents = 0, i;
@@ -1390,6 +1430,7 @@ void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
 			.size           = sg_dma_len(s),
 			.direction      = dir,
 			.sg_call_ents   = nelems,
+			.attrs          = attrs,
 		};
 
 		if (mapped_ents && i >= mapped_ents)
@@ -1440,12 +1481,13 @@ void debug_dma_alloc_coherent(struct device *dev, size_t size,
 	entry->size      = size;
 	entry->dev_addr  = dma_addr;
 	entry->direction = DMA_BIDIRECTIONAL;
+	entry->attrs     = attrs;
 
-	add_dma_entry(entry, attrs);
+	add_dma_entry(entry);
 }
 
-void debug_dma_free_coherent(struct device *dev, size_t size,
-			 void *virt, dma_addr_t dma_addr)
+void debug_dma_free_coherent(struct device *dev, size_t size, void *virt,
+			     dma_addr_t dma_addr, unsigned long attrs)
 {
 	struct dma_debug_entry ref = {
 		.type           = dma_debug_coherent,
@@ -1453,6 +1495,7 @@ void debug_dma_free_coherent(struct device *dev, size_t size,
 		.dev_addr       = dma_addr,
 		.size           = size,
 		.direction      = DMA_BIDIRECTIONAL,
+		.attrs          = attrs,
 	};
 
 	/* handle vmalloc and linear addresses */
@@ -1567,8 +1610,7 @@ void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 
 void debug_dma_alloc_pages(struct device *dev, struct page *page,
 			   size_t size, int direction,
-			   dma_addr_t dma_addr,
-			   unsigned long attrs)
+			   dma_addr_t dma_addr)
 {
 	struct dma_debug_entry *entry;
 
@@ -1586,7 +1628,7 @@ void debug_dma_alloc_pages(struct device *dev, struct page *page,
 	entry->dev_addr  = dma_addr;
 	entry->direction = direction;
 
-	add_dma_entry(entry, attrs);
+	add_dma_entry(entry);
 }
 
 void debug_dma_free_pages(struct device *dev, struct page *page,
diff --git a/kernel/dma/debug.h b/kernel/dma/debug.h
index da7be0bddcf67..13e384633c32a 100644
--- a/kernel/dma/debug.h
+++ b/kernel/dma/debug.h
@@ -14,21 +14,22 @@ extern void debug_dma_map_phys(struct device *dev, phys_addr_t phys,
 			       unsigned long attrs);
 
 extern void debug_dma_unmap_phys(struct device *dev, dma_addr_t addr,
-				 size_t size, int direction);
+				 size_t size, int direction,
+				 unsigned long attrs);
 
 extern void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 			     int nents, int mapped_ents, int direction,
 			     unsigned long attrs);
 
 extern void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
-			       int nelems, int dir);
+			       int nelems, int dir, unsigned long attrs);
 
 extern void debug_dma_alloc_coherent(struct device *dev, size_t size,
 				     dma_addr_t dma_addr, void *virt,
 				     unsigned long attrs);
 
-extern void debug_dma_free_coherent(struct device *dev, size_t size,
-				    void *virt, dma_addr_t addr);
+extern void debug_dma_free_coherent(struct device *dev, size_t size, void *virt,
+				    dma_addr_t addr, unsigned long attrs);
 
 extern void debug_dma_sync_single_for_cpu(struct device *dev,
 					  dma_addr_t dma_handle, size_t size,
@@ -47,8 +48,7 @@ extern void debug_dma_sync_sg_for_device(struct device *dev,
 					 int nelems, int direction);
 extern void debug_dma_alloc_pages(struct device *dev, struct page *page,
 				  size_t size, int direction,
-				  dma_addr_t dma_addr,
-				  unsigned long attrs);
+				  dma_addr_t dma_addr);
 extern void debug_dma_free_pages(struct device *dev, struct page *page,
 				 size_t size, int direction,
 				 dma_addr_t dma_addr);
@@ -60,7 +60,8 @@ static inline void debug_dma_map_phys(struct device *dev, phys_addr_t phys,
 }
 
 static inline void debug_dma_unmap_phys(struct device *dev, dma_addr_t addr,
-					size_t size, int direction)
+					size_t size, int direction,
+					unsigned long attrs)
 {
 }
 
@@ -71,8 +72,8 @@ static inline void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
 }
 
 static inline void debug_dma_unmap_sg(struct device *dev,
-				      struct scatterlist *sglist,
-				      int nelems, int dir)
+				      struct scatterlist *sglist, int nelems,
+				      int dir, unsigned long attrs)
 {
 }
 
@@ -83,7 +84,8 @@ static inline void debug_dma_alloc_coherent(struct device *dev, size_t size,
 }
 
 static inline void debug_dma_free_coherent(struct device *dev, size_t size,
-					   void *virt, dma_addr_t addr)
+					   void *virt, dma_addr_t addr,
+					   unsigned long attrs)
 {
 }
 
@@ -113,8 +115,7 @@ static inline void debug_dma_sync_sg_for_device(struct device *dev,
 
 static inline void debug_dma_alloc_pages(struct device *dev, struct page *page,
 					 size_t size, int direction,
-					 dma_addr_t dma_addr,
-					 unsigned long attrs)
+					 dma_addr_t dma_addr)
 {
 }
 
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index ec887f4437418..583c5922bca2d 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -39,7 +39,7 @@ static inline struct page *dma_direct_to_page(struct device *dev,
 
 u64 dma_direct_get_required_mask(struct device *dev)
 {
-	phys_addr_t phys = (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT;
+	phys_addr_t phys = ((phys_addr_t)max_pfn << PAGE_SHIFT) - 1;
 	u64 max_dma = phys_to_dma_direct(dev, phys);
 
 	return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;
@@ -553,7 +553,7 @@ int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma,
 
 int dma_direct_supported(struct device *dev, u64 mask)
 {
-	u64 min_mask = (max_pfn - 1) << PAGE_SHIFT;
+	u64 min_mask = ((u64)max_pfn << PAGE_SHIFT) - 1;
 
 	/*
 	 * Because 32-bit DMA masks are so common we expect every architecture
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index 23ed8eb9233e5..4fe04669e5e66 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -126,11 +126,9 @@ static bool dma_go_direct(struct device *dev, dma_addr_t mask,
 	if (likely(!ops))
 		return true;
 
-#ifdef CONFIG_DMA_OPS_BYPASS
-	if (dev->dma_ops_bypass)
+	if (IS_ENABLED(CONFIG_DMA_OPS_BYPASS) && dev_dma_ops_bypass(dev))
 		return min_not_zero(mask, dev->bus_dma_limit) >=
 			    dma_direct_get_required_mask(dev);
-#endif
 	return false;
 }
 
@@ -225,7 +223,7 @@ void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size,
 	else if (ops->unmap_phys)
 		ops->unmap_phys(dev, addr, size, dir, attrs);
 	trace_dma_unmap_phys(dev, addr, size, dir, attrs);
-	debug_dma_unmap_phys(dev, addr, size, dir);
+	debug_dma_unmap_phys(dev, addr, size, dir, attrs);
 }
 EXPORT_SYMBOL_GPL(dma_unmap_phys);
 
@@ -351,7 +349,7 @@ void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
 
 	BUG_ON(!valid_dma_direction(dir));
 	trace_dma_unmap_sg(dev, sg, nents, dir, attrs);
-	debug_dma_unmap_sg(dev, sg, nents, dir);
+	debug_dma_unmap_sg(dev, sg, nents, dir, attrs);
 	if (dma_map_direct(dev, ops) ||
 	    arch_dma_unmap_sg_direct(dev, sg, nents))
 		dma_direct_unmap_sg(dev, sg, nents, dir, attrs);
@@ -365,10 +363,6 @@ EXPORT_SYMBOL(dma_unmap_sg_attrs);
 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
 		size_t size, enum dma_data_direction dir, unsigned long attrs)
 {
-	if (IS_ENABLED(CONFIG_DMA_API_DEBUG) &&
-	    WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr))))
-		return DMA_MAPPING_ERROR;
-
 	return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO);
 }
 EXPORT_SYMBOL(dma_map_resource);
@@ -476,7 +470,7 @@ bool dma_need_unmap(struct device *dev)
 {
 	if (!dma_map_direct(dev, get_dma_ops(dev)))
 		return true;
-	if (!dev->dma_skip_sync)
+	if (!dev_dma_skip_sync(dev))
 		return true;
 	return IS_ENABLED(CONFIG_DMA_API_DEBUG);
 }
@@ -492,16 +486,16 @@ static void dma_setup_need_sync(struct device *dev)
 		 * mapping, if any. During the device initialization, it's
 		 * enough to check only for the DMA coherence.
 		 */
-		dev->dma_skip_sync = dev_is_dma_coherent(dev);
+		dev_assign_dma_skip_sync(dev, dev_is_dma_coherent(dev));
 	else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu &&
 		 !ops->sync_sg_for_device && !ops->sync_sg_for_cpu)
 		/*
 		 * Synchronization is not possible when none of DMA sync ops
 		 * is set.
 		 */
-		dev->dma_skip_sync = true;
+		dev_set_dma_skip_sync(dev);
 	else
-		dev->dma_skip_sync = false;
+		dev_clear_dma_skip_sync(dev);
 }
 #else /* !CONFIG_DMA_NEED_SYNC */
 static inline void dma_setup_need_sync(struct device *dev) { }
@@ -693,7 +687,7 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
 	if (!cpu_addr)
 		return;
 
-	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);
+	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle, attrs);
 	if (dma_alloc_direct(dev, ops) || arch_dma_free_direct(dev, dma_handle))
 		dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);
 	else if (use_dma_iommu(dev))
@@ -733,7 +727,7 @@ struct page *dma_alloc_pages(struct device *dev, size_t size,
 	if (page) {
 		trace_dma_alloc_pages(dev, page_to_virt(page), *dma_handle,
 				      size, dir, gfp, 0);
-		debug_dma_alloc_pages(dev, page, size, dir, *dma_handle, 0);
+		debug_dma_alloc_pages(dev, page, size, dir, *dma_handle);
 	} else {
 		trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0);
 	}
@@ -840,7 +834,7 @@ void dma_free_noncontiguous(struct device *dev, size_t size,
 		struct sg_table *sgt, enum dma_data_direction dir)
 {
 	trace_dma_free_sgt(dev, sgt, size, dir);
-	debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir);
+	debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir, 0);
 
 	if (use_dma_iommu(dev))
 		iommu_dma_free_noncontiguous(dev, size, sgt, dir);
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 7935d5663944e..bcaf175f3ded2 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -11683,6 +11683,15 @@ static int __perf_event_set_bpf_prog(struct perf_event *event,
 		/* only uprobe programs are allowed to be sleepable */
 		return -EINVAL;
 
+	if (prog->type == BPF_PROG_TYPE_TRACEPOINT && prog->sleepable) {
+		/*
+		 * Sleepable tracepoint programs can only attach to faultable
+		 * tracepoints. Currently only syscall tracepoints are faultable.
+		 */
+		if (!is_syscall_tp)
+			return -EINVAL;
+	}
+
 	/* Kprobe override only works for kprobes, not uprobes. */
 	if (prog->kprobe_override && !is_kprobe)
 		return -EINVAL;
diff --git a/kernel/exit.c b/kernel/exit.c
index f50d73c272d6e..2187541ee8da9 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -543,6 +543,32 @@ void mm_update_next_owner(struct mm_struct *mm)
 }
 #endif /* CONFIG_MEMCG */
 
+#if defined(CONFIG_SCHED_CACHE) && defined(CONFIG_NUMA_BALANCING)
+/*
+ * Subtract the memory footprint of the current task from
+ * mm.
+ */
+static void exit_mm_sched_cache(struct mm_struct *mm)
+{
+	unsigned long fp, sub;
+
+	if (!current->total_numa_faults)
+		return;
+	/*
+	 * No lock protection due to performance considerations.
+	 * Make sure mm->sc_stat.footprint does not become
+	 * negative.
+	 */
+	fp = READ_ONCE(mm->sc_stat.footprint);
+	sub = min(fp, current->total_numa_faults);
+	WRITE_ONCE(mm->sc_stat.footprint, fp - sub);
+}
+#else
+static inline void exit_mm_sched_cache(struct mm_struct *mm)
+{
+}
+#endif /* CONFIG_SCHED_CACHE CONFIG_NUMA_BALANCING */
+
 /*
  * Turn us into a lazy TLB process if we
  * aren't already..
@@ -554,6 +580,9 @@ static void exit_mm(void)
 	exit_mm_release(current, mm);
 	if (!mm)
 		return;
+
+	exit_mm_sched_cache(mm);
+
 	mmap_read_lock(mm);
 	mmgrab_lazy_tlb(mm);
 	BUG_ON(mm != current->active_mm);
diff --git a/kernel/fork.c b/kernel/fork.c
index 5f3fdfdb14c7c..e2ec2ac3d7d79 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -204,7 +204,7 @@ static DEFINE_PER_CPU(struct vm_struct *, cached_stacks[NR_CACHED_STACKS]);
  * accounting is performed by the code assigning/releasing stacks to tasks.
  * We need a zeroed memory without __GFP_ACCOUNT.
  */
-#define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO)
+#define GFP_VMAP_STACK (GFP_KERNEL | __GFP_ZERO | __GFP_SKIP_KASAN)
 
 struct vm_stack {
 	struct rcu_head rcu;
@@ -342,7 +342,8 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node)
 		}
 
 		/* Reset stack metadata. */
-		kasan_unpoison_range(vm_area->addr, THREAD_SIZE);
+		if (!kasan_hw_tags_enabled())
+			kasan_unpoison_range(vm_area->addr, THREAD_SIZE);
 
 		stack = kasan_reset_tag(vm_area->addr);
 
@@ -726,6 +727,7 @@ void __mmdrop(struct mm_struct *mm)
 	cleanup_lazy_tlbs(mm);
 
 	WARN_ON_ONCE(mm == current->active_mm);
+	mm_destroy_sched(mm);
 	mm_free_pgd(mm);
 	mm_free_id(mm);
 	destroy_context(mm);
@@ -1128,6 +1130,9 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 	if (mm_alloc_cid(mm, p))
 		goto fail_cid;
 
+	if (mm_alloc_sched(mm))
+		goto fail_sched;
+
 	if (percpu_counter_init_many(mm->rss_stat, 0, GFP_KERNEL_ACCOUNT,
 				     NR_MM_COUNTERS))
 		goto fail_pcpu;
@@ -1137,6 +1142,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
 	return mm;
 
 fail_pcpu:
+	mm_destroy_sched(mm);
+fail_sched:
 	mm_destroy_cid(mm);
 fail_cid:
 	destroy_context(mm);
@@ -2664,8 +2671,6 @@ struct task_struct *create_io_thread(int (*fn)(void *), void *arg, int node)
  *
  * It copies the process, and if successful kick-starts
  * it and waits for it to finish using the VM if required.
- *
- * args->exit_signal is expected to be checked for sanity by the caller.
  */
 pid_t kernel_clone(struct kernel_clone_args *args)
 {
@@ -2700,6 +2705,9 @@ pid_t kernel_clone(struct kernel_clone_args *args)
 	    (args->pidfd == args->parent_tid))
 		return -EINVAL;
 
+	if (!valid_signal(args->exit_signal))
+		return -EINVAL;
+
 	/*
 	 * Determine whether and which event to report to ptracer.  When
 	 * called from kernel_thread or CLONE_UNTRACED is explicitly
@@ -2898,11 +2906,9 @@ static noinline int copy_clone_args_from_user(struct kernel_clone_args *kargs,
 		return -EINVAL;
 
 	/*
-	 * Verify that higher 32bits of exit_signal are unset and that
-	 * it is a valid signal
+	 * Verify that higher 32bits of exit_signal are unset
 	 */
-	if (unlikely((args.exit_signal & ~((u64)CSIGNAL)) ||
-		     !valid_signal(args.exit_signal)))
+	if (unlikely(args.exit_signal & ~((u64)CSIGNAL)))
 		return -EINVAL;
 
 	if ((args.flags & CLONE_INTO_CGROUP) &&
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index b0999a4f1f688..dfa0b07236429 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -10,6 +10,7 @@
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
 #include <linux/interrupt.h>
+#include <linux/kernel.h>
 #include <linux/kernel_stat.h>
 #include <linux/mutex.h>
 #include <linux/string.h>
@@ -326,7 +327,7 @@ void register_handler_proc(unsigned int irq, struct irqaction *action)
 
 #undef MAX_NAMELEN
 
-#define MAX_NAMELEN 10
+#define MAX_NAMELEN 11
 
 void register_irq_proc(unsigned int irq, struct irq_desc *desc)
 {
@@ -348,7 +349,7 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc)
 		return;
 
 	/* create /proc/irq/1234 */
-	sprintf(name, "%u", irq);
+	snprintf(name, MAX_NAMELEN, "%u", irq);
 	desc->dir = proc_mkdir(name, root_irq_dir);
 	if (!desc->dir)
 		return;
@@ -401,7 +402,7 @@ void unregister_irq_proc(unsigned int irq, struct irq_desc *desc)
 #endif
 	remove_proc_entry("spurious", desc->dir);
 
-	sprintf(name, "%u", irq);
+	snprintf(name, MAX_NAMELEN, "%u", irq);
 	remove_proc_entry(name, root_irq_dir);
 }
 
diff --git a/kernel/kcov.c b/kernel/kcov.c
index 0b369e88c7c9b..fd25030307299 100644
--- a/kernel/kcov.c
+++ b/kernel/kcov.c
@@ -368,6 +368,7 @@ static void kcov_start(struct task_struct *t, struct kcov *kcov,
 	WRITE_ONCE(t->kcov_mode, mode);
 }
 
+/* operates on coverage-generator-owned fields */
 static void kcov_stop(struct task_struct *t)
 {
 	WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED);
@@ -377,16 +378,17 @@ static void kcov_stop(struct task_struct *t)
 	t->kcov_area = NULL;
 }
 
+/* operates on coverage-generator-owned fields */
 static void kcov_task_reset(struct task_struct *t)
 {
 	kcov_stop(t);
 	t->kcov_sequence = 0;
-	t->kcov_handle = 0;
 }
 
 void kcov_task_init(struct task_struct *t)
 {
 	kcov_task_reset(t);
+	t->kcov_remote = NULL;
 	t->kcov_handle = current->kcov_handle;
 }
 
@@ -423,11 +425,14 @@ static void kcov_remote_reset(struct kcov *kcov)
 static void kcov_disable(struct task_struct *t, struct kcov *kcov)
 	__must_hold(&kcov->lock)
 {
-	kcov_task_reset(t);
-	if (kcov->remote)
+	if (kcov->remote) {
+		t->kcov_handle = 0;
+		t->kcov_remote = NULL;
 		kcov_remote_reset(kcov);
-	else
+	} else {
+		kcov_task_reset(t);
 		kcov_reset(kcov);
+	}
 }
 
 static void kcov_get(struct kcov *kcov)
@@ -453,41 +458,47 @@ void kcov_task_exit(struct task_struct *t)
 	unsigned long flags;
 
 	kcov = t->kcov;
-	if (kcov == NULL)
-		return;
-
-	spin_lock_irqsave(&kcov->lock, flags);
-	kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t);
-	/*
-	 * For KCOV_ENABLE devices we want to make sure that t->kcov->t == t,
-	 * which comes down to:
-	 *        WARN_ON(!kcov->remote && kcov->t != t);
-	 *
-	 * For KCOV_REMOTE_ENABLE devices, the exiting task is either:
-	 *
-	 * 1. A remote task between kcov_remote_start() and kcov_remote_stop().
-	 *    In this case we should print a warning right away, since a task
-	 *    shouldn't be exiting when it's in a kcov coverage collection
-	 *    section. Here t points to the task that is collecting remote
-	 *    coverage, and t->kcov->t points to the thread that created the
-	 *    kcov device. Which means that to detect this case we need to
-	 *    check that t != t->kcov->t, and this gives us the following:
-	 *        WARN_ON(kcov->remote && kcov->t != t);
-	 *
-	 * 2. The task that created kcov exiting without calling KCOV_DISABLE,
-	 *    and then again we make sure that t->kcov->t == t:
-	 *        WARN_ON(kcov->remote && kcov->t != t);
-	 *
-	 * By combining all three checks into one we get:
-	 */
-	if (WARN_ON(kcov->t != t)) {
+	if (kcov) {
+		spin_lock_irqsave(&kcov->lock, flags);
+		kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t);
+		/*
+		 * This could be a remote task between kcov_remote_start() and
+		 * kcov_remote_stop().
+		 * In this case we should print a warning right away, since a
+		 * task shouldn't be exiting when it's in a kcov coverage
+		 * collection section.
+		 *
+		 * Otherwise, this should be a task that created a local
+		 * kcov instance and hasn't called KCOV_DISABLE.
+		 * Make sure that t->kcov->t is consistent.
+		 */
+		if (WARN_ON(kcov->remote) || WARN_ON(kcov->t != t)) {
+			spin_unlock_irqrestore(&kcov->lock, flags);
+			return;
+		}
+		/* Just to not leave dangling references behind. */
+		kcov_disable(t, kcov);
 		spin_unlock_irqrestore(&kcov->lock, flags);
-		return;
+		kcov_put(kcov);
+	}
+	kcov = t->kcov_remote;
+	if (kcov) {
+		spin_lock_irqsave(&kcov->lock, flags);
+		kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t);
+		/*
+		 * This is a KCOV_REMOTE_ENABLE device, and the task is the
+		 * user task which has requested remote coverage collection.
+		 * Make sure that t->kcov->t is consistent.
+		 */
+		if (WARN_ON(!kcov->remote) || WARN_ON(kcov->t != t)) {
+			spin_unlock_irqrestore(&kcov->lock, flags);
+			return;
+		}
+		/* Just to not leave dangling references behind. */
+		kcov_disable(t, kcov);
+		spin_unlock_irqrestore(&kcov->lock, flags);
+		kcov_put(kcov);
 	}
-	/* Just to not leave dangling references behind. */
-	kcov_disable(t, kcov);
-	spin_unlock_irqrestore(&kcov->lock, flags);
-	kcov_put(kcov);
 }
 
 static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
@@ -629,9 +640,9 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
 	case KCOV_DISABLE:
 		/* Disable coverage for the current task. */
 		unused = arg;
-		if (unused != 0 || current->kcov != kcov)
-			return -EINVAL;
 		t = current;
+		if (unused != 0 || (kcov != t->kcov && kcov != t->kcov_remote))
+			return -EINVAL;
 		if (WARN_ON(kcov->t != t))
 			return -EINVAL;
 		kcov_disable(t, kcov);
@@ -641,7 +652,7 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
 		if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
 			return -EINVAL;
 		t = current;
-		if (kcov->t != NULL || t->kcov != NULL)
+		if (kcov->t != NULL || t->kcov_remote != NULL)
 			return -EBUSY;
 		remote_arg = (struct kcov_remote_arg *)arg;
 		mode = kcov_get_mode(remote_arg->trace_mode);
@@ -651,8 +662,7 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
 		    LONG_MAX / sizeof(unsigned long))
 			return -EINVAL;
 		kcov->mode = mode;
-		t->kcov = kcov;
-	        t->kcov_mode = KCOV_MODE_REMOTE;
+		t->kcov_remote = kcov;
 		kcov->t = t;
 		kcov->remote = true;
 		kcov->remote_size = remote_arg->area_size;
@@ -1083,11 +1093,11 @@ void kcov_remote_stop(void)
 EXPORT_SYMBOL(kcov_remote_stop);
 
 /* See the comment before kcov_remote_start() for usage details. */
-u64 kcov_common_handle(void)
+struct kcov_common_handle_id kcov_common_handle(void)
 {
 	if (!in_task())
-		return 0;
-	return current->kcov_handle;
+		return (struct kcov_common_handle_id){ .val = 0 };
+	return (struct kcov_common_handle_id){ .val = current->kcov_handle };
 }
 EXPORT_SYMBOL(kcov_common_handle);
 
diff --git a/kernel/liveupdate/Kconfig b/kernel/liveupdate/Kconfig
index 1a8513f16ef7c..c13af38ba23ab 100644
--- a/kernel/liveupdate/Kconfig
+++ b/kernel/liveupdate/Kconfig
@@ -1,12 +1,10 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
 menu "Live Update and Kexec HandOver"
-	depends on !DEFERRED_STRUCT_PAGE_INIT
 
 config KEXEC_HANDOVER
 	bool "kexec handover"
 	depends on ARCH_SUPPORTS_KEXEC_HANDOVER && ARCH_SUPPORTS_KEXEC_FILE
-	depends on !DEFERRED_STRUCT_PAGE_INIT
 	select MEMBLOCK_KHO_SCRATCH
 	select KEXEC_FILE
 	select LIBFDT
diff --git a/kernel/liveupdate/kexec_handover.c b/kernel/liveupdate/kexec_handover.c
index 2592f7ca16e2e..eb8abd04e7ffd 100644
--- a/kernel/liveupdate/kexec_handover.c
+++ b/kernel/liveupdate/kexec_handover.c
@@ -357,20 +357,6 @@ int kho_radix_walk_tree(struct kho_radix_tree *tree,
 }
 EXPORT_SYMBOL_GPL(kho_radix_walk_tree);
 
-static void __kho_unpreserve(struct kho_radix_tree *tree,
-			     unsigned long pfn, unsigned long end_pfn)
-{
-	unsigned int order;
-
-	while (pfn < end_pfn) {
-		order = min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
-
-		kho_radix_del_page(tree, pfn, order);
-
-		pfn += 1 << order;
-	}
-}
-
 /* For physically contiguous 0-order pages. */
 static void kho_init_pages(struct page *page, unsigned long nr_pages)
 {
@@ -473,6 +459,31 @@ struct page *kho_restore_pages(phys_addr_t phys, unsigned long nr_pages)
 }
 EXPORT_SYMBOL_GPL(kho_restore_pages);
 
+/*
+ * With CONFIG_DEFERRED_STRUCT_PAGE_INIT, struct pages in higher memory regions
+ * may not be initialized yet at the time KHO deserializes preserved memory.
+ * KHO uses the struct page to store metadata and a later initialization would
+ * overwrite it.
+ * Ensure all the struct pages in the preservation are
+ * initialized. kho_preserved_memory_reserve() marks the reservation as noinit
+ * to make sure they don't get re-initialized later.
+ */
+static struct page *__init kho_get_preserved_page(phys_addr_t phys,
+						  unsigned int order)
+{
+	unsigned long pfn = PHYS_PFN(phys);
+	int nid;
+
+	if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT))
+		return pfn_to_page(pfn);
+
+	nid = early_pfn_to_nid(pfn);
+	for (unsigned long i = 0; i < (1UL << order); i++)
+		init_deferred_page(pfn + i, nid);
+
+	return pfn_to_page(pfn);
+}
+
 static int __init kho_preserved_memory_reserve(phys_addr_t phys,
 					       unsigned int order)
 {
@@ -481,7 +492,7 @@ static int __init kho_preserved_memory_reserve(phys_addr_t phys,
 	u64 sz;
 
 	sz = 1 << (order + PAGE_SHIFT);
-	page = phys_to_page(phys);
+	page = kho_get_preserved_page(phys, order);
 
 	/* Reserve the memory preserved in KHO in memblock */
 	memblock_reserve(phys, sz);
@@ -860,6 +871,37 @@ void kho_unpreserve_folio(struct folio *folio)
 }
 EXPORT_SYMBOL_GPL(kho_unpreserve_folio);
 
+static unsigned int __kho_preserve_pages_order(unsigned long start_pfn,
+					       unsigned long end_pfn)
+{
+	unsigned int order = min(count_trailing_zeros(start_pfn),
+				 ilog2(end_pfn - start_pfn));
+
+	/*
+	 * Make sure all the pages in a single preservation are in the same NUMA
+	 * node. The restore machinery can not cope with a preservation spanning
+	 * multiple NUMA nodes.
+	 */
+	while (pfn_to_nid(start_pfn) != pfn_to_nid(start_pfn + (1UL << order) - 1))
+		order--;
+
+	return order;
+}
+
+static void __kho_unpreserve(struct kho_radix_tree *tree,
+			     unsigned long pfn, unsigned long end_pfn)
+{
+	unsigned int order;
+
+	while (pfn < end_pfn) {
+		order = __kho_preserve_pages_order(pfn, end_pfn);
+
+		kho_radix_del_page(tree, pfn, order);
+
+		pfn += 1 << order;
+	}
+}
+
 /**
  * kho_preserve_pages - preserve contiguous pages across kexec
  * @page: first page in the list.
@@ -885,16 +927,7 @@ int kho_preserve_pages(struct page *page, unsigned long nr_pages)
 	}
 
 	while (pfn < end_pfn) {
-		unsigned int order =
-			min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn));
-
-		/*
-		 * Make sure all the pages in a single preservation are in the
-		 * same NUMA node. The restore machinery can not cope with a
-		 * preservation spanning multiple NUMA nodes.
-		 */
-		while (pfn_to_nid(pfn) != pfn_to_nid(pfn + (1UL << order) - 1))
-			order--;
+		unsigned int order = __kho_preserve_pages_order(pfn, end_pfn);
 
 		err = kho_radix_add_page(tree, pfn, order);
 		if (err) {
@@ -1576,35 +1609,10 @@ static __init int kho_init(void)
 }
 fs_initcall(kho_init);
 
-static void __init kho_release_scratch(void)
-{
-	phys_addr_t start, end;
-	u64 i;
-
-	memmap_init_kho_scratch_pages();
-
-	/*
-	 * Mark scratch mem as CMA before we return it. That way we
-	 * ensure that no kernel allocations happen on it. That means
-	 * we can reuse it as scratch memory again later.
-	 */
-	__for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE,
-			     MEMBLOCK_KHO_SCRATCH, &start, &end, NULL) {
-		ulong start_pfn = pageblock_start_pfn(PFN_DOWN(start));
-		ulong end_pfn = pageblock_align(PFN_UP(end));
-		ulong pfn;
-
-		for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages)
-			init_pageblock_migratetype(pfn_to_page(pfn),
-						   MIGRATE_CMA, false);
-	}
-}
-
 void __init kho_memory_init(void)
 {
 	if (kho_in.scratch_phys) {
 		kho_scratch = phys_to_virt(kho_in.scratch_phys);
-		kho_release_scratch();
 
 		if (kho_mem_retrieve(kho_get_fdt()))
 			kho_in.fdt_phys = 0;
diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c
index 803f51c842756..5d5827ced73c8 100644
--- a/kernel/liveupdate/luo_core.c
+++ b/kernel/liveupdate/luo_core.c
@@ -36,6 +36,10 @@
  *
  * LUO uses Kexec Handover to transfer memory state from the current kernel to
  * the next kernel. For more details see Documentation/core-api/kho/index.rst.
+ *
+ * .. note::
+ *     To enable LUO, boot the kernel with the ``liveupdate=on`` command line
+ *     parameter.
  */
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
diff --git a/kernel/liveupdate/luo_flb.c b/kernel/liveupdate/luo_flb.c
index 00f5494812c4a..8f5c5dd01cd04 100644
--- a/kernel/liveupdate/luo_flb.c
+++ b/kernel/liveupdate/luo_flb.c
@@ -111,7 +111,7 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb)
 	struct luo_flb_private *private = luo_flb_get_private(flb);
 
 	scoped_guard(mutex, &private->outgoing.lock) {
-		if (!private->outgoing.count) {
+		if (!refcount_read(&private->outgoing.count)) {
 			struct liveupdate_flb_op_args args = {0};
 			int err;
 
@@ -126,8 +126,10 @@ static int luo_flb_file_preserve_one(struct liveupdate_flb *flb)
 			}
 			private->outgoing.data = args.data;
 			private->outgoing.obj = args.obj;
+			refcount_set(&private->outgoing.count, 1);
+		} else {
+			refcount_inc(&private->outgoing.count);
 		}
-		private->outgoing.count++;
 	}
 
 	return 0;
@@ -138,8 +140,7 @@ static void luo_flb_file_unpreserve_one(struct liveupdate_flb *flb)
 	struct luo_flb_private *private = luo_flb_get_private(flb);
 
 	scoped_guard(mutex, &private->outgoing.lock) {
-		private->outgoing.count--;
-		if (!private->outgoing.count) {
+		if (refcount_dec_and_test(&private->outgoing.count)) {
 			struct liveupdate_flb_op_args args = {0};
 
 			args.flb = flb;
@@ -164,7 +165,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb)
 	bool found = false;
 	int err;
 
-	guard(mutex)(&private->incoming.lock);
+	lockdep_assert_held(&private->incoming.lock);
 
 	if (private->incoming.finished)
 		return -ENODATA;
@@ -178,7 +179,7 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb)
 	for (int i = 0; i < fh->header_ser->count; i++) {
 		if (!strcmp(fh->ser[i].name, flb->compatible)) {
 			private->incoming.data = fh->ser[i].data;
-			private->incoming.count = fh->ser[i].count;
+			refcount_set(&private->incoming.count, fh->ser[i].count);
 			found = true;
 			break;
 		}
@@ -205,16 +206,14 @@ static int luo_flb_retrieve_one(struct liveupdate_flb *flb)
 	return 0;
 }
 
-static void luo_flb_file_finish_one(struct liveupdate_flb *flb)
+void liveupdate_flb_put_incoming(struct liveupdate_flb *flb)
 {
 	struct luo_flb_private *private = luo_flb_get_private(flb);
-	u64 count;
-
-	scoped_guard(mutex, &private->incoming.lock)
-		count = --private->incoming.count;
+	struct liveupdate_flb_op_args args = {0};
 
-	if (!count) {
-		struct liveupdate_flb_op_args args = {0};
+	scoped_guard(mutex, &private->incoming.lock) {
+		if (!refcount_dec_and_test(&private->incoming.count))
+			return;
 
 		if (!private->incoming.retrieved) {
 			int err = luo_flb_retrieve_one(flb);
@@ -223,16 +222,14 @@ static void luo_flb_file_finish_one(struct liveupdate_flb *flb)
 				return;
 		}
 
-		scoped_guard(mutex, &private->incoming.lock) {
-			args.flb = flb;
-			args.obj = private->incoming.obj;
-			flb->ops->finish(&args);
+		args.flb = flb;
+		args.obj = private->incoming.obj;
+		flb->ops->finish(&args);
 
-			private->incoming.data = 0;
-			private->incoming.obj = NULL;
-			private->incoming.finished = true;
-			module_put(flb->ops->owner);
-		}
+		private->incoming.data = 0;
+		private->incoming.obj = NULL;
+		private->incoming.finished = true;
+		module_put(flb->ops->owner);
 	}
 }
 
@@ -315,7 +312,7 @@ void luo_flb_file_finish(struct liveupdate_file_handler *fh)
 
 	guard(rwsem_read)(&luo_register_rwlock);
 	list_for_each_entry_reverse(iter, flb_list, list)
-		luo_flb_file_finish_one(iter->flb);
+		liveupdate_flb_put_incoming(iter->flb);
 }
 
 static void luo_flb_unregister_one(struct liveupdate_file_handler *fh,
@@ -512,6 +509,8 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp)
 	if (!liveupdate_enabled())
 		return -EOPNOTSUPP;
 
+	guard(mutex)(&private->incoming.lock);
+
 	if (!private->incoming.obj) {
 		int err = luo_flb_retrieve_one(flb);
 
@@ -519,7 +518,7 @@ int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp)
 			return err;
 	}
 
-	guard(mutex)(&private->incoming.lock);
+	refcount_inc(&private->incoming.count);
 	*objp = private->incoming.obj;
 
 	return 0;
@@ -652,12 +651,13 @@ void luo_flb_serialize(void)
 	guard(rwsem_read)(&luo_register_rwlock);
 	list_private_for_each_entry(gflb, &luo_flb_global.list, private.list) {
 		struct luo_flb_private *private = luo_flb_get_private(gflb);
+		long count = refcount_read(&private->outgoing.count);
 
-		if (private->outgoing.count > 0) {
+		if (count > 0) {
 			strscpy(fh->ser[i].name, gflb->compatible,
 				sizeof(fh->ser[i].name));
 			fh->ser[i].data = private->outgoing.data;
-			fh->ser[i].count = private->outgoing.count;
+			fh->ser[i].count = count;
 			i++;
 		}
 	}
diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c
index 7a42385dabe27..e7f27815c0517 100644
--- a/kernel/liveupdate/luo_session.c
+++ b/kernel/liveupdate/luo_session.c
@@ -289,10 +289,24 @@ static int luo_session_finish(struct luo_session *session,
 	return luo_ucmd_respond(ucmd, sizeof(*argp));
 }
 
+static int luo_session_get_name(struct luo_session *session,
+				struct luo_ucmd *ucmd)
+{
+	struct liveupdate_session_get_name *argp = ucmd->cmd;
+
+	if (argp->reserved != 0)
+		return -EINVAL;
+
+	strscpy((char *)argp->name, session->name, sizeof(argp->name));
+
+	return luo_ucmd_respond(ucmd, sizeof(*argp));
+}
+
 union ucmd_buffer {
 	struct liveupdate_session_finish finish;
 	struct liveupdate_session_preserve_fd preserve;
 	struct liveupdate_session_retrieve_fd retrieve;
+	struct liveupdate_session_get_name get_name;
 };
 
 struct luo_ioctl_op {
@@ -319,6 +333,8 @@ static const struct luo_ioctl_op luo_session_ioctl_ops[] = {
 		 struct liveupdate_session_preserve_fd, token),
 	IOCTL_OP(LIVEUPDATE_SESSION_RETRIEVE_FD, luo_session_retrieve_fd,
 		 struct liveupdate_session_retrieve_fd, token),
+	IOCTL_OP(LIVEUPDATE_SESSION_GET_NAME, luo_session_get_name,
+		 struct liveupdate_session_get_name, name),
 };
 
 static long luo_session_ioctl(struct file *filep, unsigned int cmd,
@@ -382,9 +398,13 @@ static int luo_session_getfile(struct luo_session *session, struct file **filep)
 
 int luo_session_create(const char *name, struct file **filep)
 {
+	size_t len = strnlen(name, LIVEUPDATE_SESSION_NAME_LENGTH);
 	struct luo_session *session;
 	int err;
 
+	if (len == 0 || len > LIVEUPDATE_SESSION_NAME_LENGTH - 1)
+		return -EINVAL;
+
 	session = luo_session_alloc(name);
 	if (IS_ERR(session))
 		return PTR_ERR(session);
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 2d4c5bab5af88..f13883162cfe1 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -796,8 +796,10 @@ static void lockdep_print_held_locks(struct task_struct *p)
 	 * It's not reliable to print a task's held locks if it's not sleeping
 	 * and it's not the current task.
 	 */
+#ifndef CONFIG_DEBUG_AID_FOR_SYZBOT
 	if (p != current && task_is_running(p))
 		return;
+#endif
 	for (i = 0; i < depth; i++) {
 		printk(" #%d: ", i);
 		print_lock(p->held_locks + i);
diff --git a/kernel/locking/rtmutex.c b/kernel/locking/rtmutex.c
index 4f386ea6c7928..9147d6a31b78c 100644
--- a/kernel/locking/rtmutex.c
+++ b/kernel/locking/rtmutex.c
@@ -484,6 +484,7 @@ static __always_inline bool __waiter_less(struct rb_node *a, const struct rb_nod
 
 static __always_inline void
 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
+	__must_hold(&lock->wait_lock)
 {
 	lockdep_assert_held(&lock->wait_lock);
 
@@ -492,6 +493,7 @@ rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
 
 static __always_inline void
 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
+	__must_hold(&lock->wait_lock)
 {
 	lockdep_assert_held(&lock->wait_lock);
 
@@ -1092,6 +1094,7 @@ static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task,
 static int __sched
 try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,
 		     struct rt_mutex_waiter *waiter)
+	__must_hold(&lock->wait_lock)
 {
 	lockdep_assert_held(&lock->wait_lock);
 
@@ -1319,6 +1322,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
  */
 static void __sched mark_wakeup_next_waiter(struct rt_wake_q_head *wqh,
 					    struct rt_mutex_base *lock)
+	__must_hold(&lock->wait_lock)
 {
 	struct rt_mutex_waiter *waiter;
 
diff --git a/kernel/locking/rtmutex_api.c b/kernel/locking/rtmutex_api.c
index 124219aea46e4..7c40b91422a06 100644
--- a/kernel/locking/rtmutex_api.c
+++ b/kernel/locking/rtmutex_api.c
@@ -41,6 +41,7 @@ static __always_inline int __rt_mutex_lock_common(struct rt_mutex *lock,
 						  unsigned int state,
 						  struct lockdep_map *nest_lock,
 						  unsigned int subclass)
+	__cond_acquires(0, lock)
 {
 	int ret;
 
@@ -67,13 +68,27 @@ EXPORT_SYMBOL(rt_mutex_base_init);
  */
 void __sched rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass)
 {
-	__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass);
+	if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, subclass) == 0)
+		return;
+	/*
+	 * The code below is never reached because __rt_mutex_lock_common() only
+	 * returns an error code if interrupted by a signal or upon a timeout.
+	 */
+	WARN_ON_ONCE(true);
+	__acquire(lock);
 }
 EXPORT_SYMBOL_GPL(rt_mutex_lock_nested);
 
 void __sched _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock)
 {
-	__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0);
+	if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, nest_lock, 0) == 0)
+		return;
+	/*
+	 * The code below is never reached because __rt_mutex_lock_common() only
+	 * returns an error code if interrupted by a signal or upon a timeout.
+	 */
+	WARN_ON_ONCE(true);
+	__acquire(lock);
 }
 EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock);
 
@@ -86,7 +101,14 @@ EXPORT_SYMBOL_GPL(_rt_mutex_lock_nest_lock);
  */
 void __sched rt_mutex_lock(struct rt_mutex *lock)
 {
-	__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0);
+	if (__rt_mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, NULL, 0) == 0)
+		return;
+	/*
+	 * The code below is never reached because __rt_mutex_lock_common() only
+	 * returns an error code if interrupted by a signal or upon a timeout.
+	 */
+	WARN_ON_ONCE(true);
+	__acquire(lock);
 }
 EXPORT_SYMBOL_GPL(rt_mutex_lock);
 #endif
@@ -157,6 +179,7 @@ void __sched rt_mutex_unlock(struct rt_mutex *lock)
 {
 	mutex_release(&lock->dep_map, _RET_IP_);
 	__rt_mutex_unlock(&lock->rtmutex);
+	__release(lock);
 }
 EXPORT_SYMBOL_GPL(rt_mutex_unlock);
 
@@ -182,6 +205,7 @@ int __sched __rt_mutex_futex_trylock(struct rt_mutex_base *lock)
  */
 bool __sched __rt_mutex_futex_unlock(struct rt_mutex_base *lock,
 				     struct rt_wake_q_head *wqh)
+	__must_hold(&lock->wait_lock)
 {
 	lockdep_assert_held(&lock->wait_lock);
 
@@ -312,6 +336,7 @@ int __sched __rt_mutex_start_proxy_lock(struct rt_mutex_base *lock,
 					struct rt_mutex_waiter *waiter,
 					struct task_struct *task,
 					struct wake_q_head *wake_q)
+	__must_hold(&lock->wait_lock)
 {
 	int ret;
 
diff --git a/kernel/panic.c b/kernel/panic.c
index 20feada5319d4..213725b612aa1 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -39,6 +39,7 @@
 #include <linux/sys_info.h>
 #include <trace/events/error_report.h>
 #include <asm/sections.h>
+#include <kunit/test-bug.h>
 
 #define PANIC_TIMER_STEP 100
 #define PANIC_BLINK_SPD 18
@@ -1124,6 +1125,11 @@ void warn_slowpath_fmt(const char *file, int line, unsigned taint,
 	bool rcu = warn_rcu_enter();
 	struct warn_args args;
 
+	if (kunit_is_suppressed_warning(true)) {
+		warn_rcu_exit(rcu);
+		return;
+	}
+
 	pr_warn(CUT_HERE);
 
 	if (!fmt) {
@@ -1146,6 +1152,11 @@ void __warn_printk(const char *fmt, ...)
 	bool rcu = warn_rcu_enter();
 	va_list args;
 
+	if (kunit_is_suppressed_warning(false)) {
+		warn_rcu_exit(rcu);
+		return;
+	}
+
 	pr_warn(CUT_HERE);
 
 	va_start(args, fmt);
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index a564650734dcd..d933b5b2c05d4 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -1244,8 +1244,9 @@ unsigned int snapshot_additional_pages(struct zone *zone)
 static void mark_free_pages(struct zone *zone)
 {
 	unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
+	struct list_head *free_list;
 	unsigned long flags;
-	unsigned int order, t;
+	unsigned int order;
 	struct page *page;
 
 	if (zone_is_empty(zone))
@@ -1269,9 +1270,8 @@ static void mark_free_pages(struct zone *zone)
 			swsusp_unset_page_free(page);
 	}
 
-	for_each_migratetype_order(order, t) {
-		list_for_each_entry(page,
-				&zone->free_area[order].free_list[t], buddy_list) {
+	for_each_free_list(free_list, zone, order) {
+		list_for_each_entry(page, free_list, buddy_list) {
 			unsigned long i;
 
 			pfn = page_to_pfn(page);
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 0d01cd8c4b4a7..7c2f7cc131f7a 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -897,11 +897,9 @@ static void srcu_schedule_cbs_snp(struct srcu_struct *ssp, struct srcu_node *snp
 {
 	int cpu;
 
-	for (cpu = snp->grplo; cpu <= snp->grphi; cpu++) {
-		if (!(mask & (1UL << (cpu - snp->grplo))))
-			continue;
-		srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);
-	}
+	for (cpu = snp->grplo; cpu <= snp->grphi; cpu++)
+		if ((mask & (1UL << (cpu - snp->grplo))) && rcu_cpu_beenfullyonline(cpu))
+			srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, cpu), delay);
 }
 
 /*
@@ -1322,7 +1320,9 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,
 	 */
 	idx = __srcu_read_lock_nmisafe(ssp);
 	ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);
-	if (ss_state < SRCU_SIZE_WAIT_CALL)
+	// If !rcu_cpu_beenfullyonline(), interrupts are still disabled,
+	// so no migration is possible in either direction from this CPU.
+	if (ss_state < SRCU_SIZE_WAIT_CALL || !rcu_cpu_beenfullyonline(raw_smp_processor_id()))
 		sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());
 	else
 		sdp = raw_cpu_ptr(ssp->sda);
diff --git a/kernel/resource.c b/kernel/resource.c
index d02a53fb95d85..3d17e3196a3ec 100644
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -1238,7 +1238,7 @@ reserve_region_with_split(struct resource *root, resource_size_t start,
  *
  * Returns alignment on success, 0 (invalid alignment) on failure.
  */
-resource_size_t resource_alignment(struct resource *res)
+resource_size_t resource_alignment(const struct resource *res)
 {
 	switch (res->flags & (IORESOURCE_SIZEALIGN | IORESOURCE_STARTALIGN)) {
 	case IORESOURCE_SIZEALIGN:
diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c
index 755883faf7518..5e76c9177d540 100644
--- a/kernel/sched/build_policy.c
+++ b/kernel/sched/build_policy.c
@@ -58,8 +58,13 @@
 #include "deadline.c"
 
 #ifdef CONFIG_SCHED_CLASS_EXT
+# include <linux/btf_ids.h>
+# include "ext_types.h"
 # include "ext_internal.h"
+# include "ext_cid.h"
+# include "ext_idle.h"
 # include "ext.c"
+# include "ext_cid.c"
 # include "ext_idle.c"
 #endif
 
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b8871449d3c69..7fb3f5f2d48c0 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -537,13 +537,22 @@ sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }
 /* need a wrapper since we may need to trace from modules */
 EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp);
 
-/* Call via the helper macro trace_set_current_state. */
+/*
+ * Call via the helper macro trace_set_current_state.
+ * Calls to this function MUST be guarded by a
+ * tracepoint_enabled(sched_set_state_tp)
+ */
 void __trace_set_current_state(int state_value)
 {
-	trace_sched_set_state_tp(current, state_value);
+	trace_call__sched_set_state_tp(current, state_value);
 }
 EXPORT_SYMBOL(__trace_set_current_state);
 
+int task_llc(const struct task_struct *p)
+{
+	return per_cpu(sd_llc_id, task_cpu(p));
+}
+
 /*
  * Serialization rules:
  *
@@ -1203,9 +1212,13 @@ static void __resched_curr(struct rq *rq, int tif)
 	}
 }
 
+/*
+ * Calls to this function MUST be guarded by a
+ * tracepoint_enabled(sched_set_need_resched_tp)
+ */
 void __trace_set_need_resched(struct task_struct *curr, int tif)
 {
-	trace_sched_set_need_resched_tp(curr, smp_processor_id(), tif);
+	trace_call__sched_set_need_resched_tp(curr, smp_processor_id(), tif);
 }
 EXPORT_SYMBOL_GPL(__trace_set_need_resched);
 
@@ -4498,6 +4511,7 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
 	init_numa_balancing(clone_flags, p);
 	p->wake_entry.u_flags = CSD_TYPE_TTWU;
 	p->migration_pending = NULL;
+	init_sched_mm(p);
 }
 
 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -6645,6 +6659,7 @@ static inline void proxy_set_task_cpu(struct task_struct *p, int cpu)
 static inline struct task_struct *proxy_resched_idle(struct rq *rq)
 {
 	put_prev_set_next_task(rq, rq->donor, rq->idle);
+	rq->next_class = &idle_sched_class;
 	rq_set_donor(rq, rq->idle);
 	set_tsk_need_resched(rq->idle);
 	return rq->idle;
@@ -8604,18 +8619,14 @@ static void cpuset_cpu_inactive(unsigned int cpu)
 
 static inline void sched_smt_present_inc(int cpu)
 {
-#ifdef CONFIG_SCHED_SMT
 	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
 		static_branch_inc_cpuslocked(&sched_smt_present);
-#endif
 }
 
 static inline void sched_smt_present_dec(int cpu)
 {
-#ifdef CONFIG_SCHED_SMT
 	if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
 		static_branch_dec_cpuslocked(&sched_smt_present);
-#endif
 }
 
 int sched_cpu_activate(unsigned int cpu)
@@ -8694,6 +8705,8 @@ int sched_cpu_deactivate(unsigned int cpu)
 	 */
 	synchronize_rcu();
 
+	sched_domains_free_llc_id(cpu);
+
 	sched_set_rq_offline(rq, cpu);
 
 	scx_rq_deactivate(rq);
@@ -8703,9 +8716,7 @@ int sched_cpu_deactivate(unsigned int cpu)
 	 */
 	sched_smt_present_dec(cpu);
 
-#ifdef CONFIG_SCHED_SMT
 	sched_core_cpu_deactivate(cpu);
-#endif
 
 	if (!sched_smp_initialized)
 		return 0;
@@ -9027,6 +9038,11 @@ void __init sched_init(void)
 
 		rq->core_cookie = 0UL;
 #endif
+#ifdef CONFIG_SCHED_CACHE
+		raw_spin_lock_init(&rq->cpu_epoch_lock);
+		rq->cpu_epoch_next = jiffies;
+#endif
+
 		zalloc_cpumask_var_node(&rq->scratch_mask, GFP_KERNEL, cpu_to_node(i));
 	}
 
diff --git a/kernel/sched/core_sched.c b/kernel/sched/core_sched.c
index 73b6b24269119..43e0bde3038e9 100644
--- a/kernel/sched/core_sched.c
+++ b/kernel/sched/core_sched.c
@@ -136,7 +136,7 @@ int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
 	struct pid *grp;
 	int err = 0;
 
-	if (!static_branch_likely(&sched_smt_present))
+	if (!sched_smt_active())
 		return -ENODEV;
 
 	BUILD_BUG_ON(PR_SCHED_CORE_SCOPE_THREAD != PIDTYPE_PID);
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index fbf31db0d2f3c..6e85023a81ff3 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -587,12 +587,6 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
 	}
 
 	stime = mul_u64_u64_div_u64(stime, rtime, stime + utime);
-	/*
-	 * Because mul_u64_u64_div_u64() can approximate on some
-	 * achitectures; enforce the constraint that: a*b/(b+c) <= a.
-	 */
-	if (unlikely(stime > rtime))
-		stime = rtime;
 
 update:
 	/*
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 7db4c87df83b0..ddfd6bc63ab1d 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -1515,8 +1515,12 @@ static void update_curr_dl_se(struct rq *rq, struct sched_dl_entity *dl_se, s64
 
 		if (unlikely(is_dl_boosted(dl_se) || !start_dl_timer(dl_se))) {
 			if (dl_server(dl_se)) {
-				replenish_dl_new_period(dl_se, rq);
-				start_dl_timer(dl_se);
+				if (dl_se->dl_defer) {
+					replenish_dl_new_period(dl_se, rq);
+					start_dl_timer(dl_se);
+				} else {
+					enqueue_dl_entity(dl_se, ENQUEUE_REPLENISH);
+				}
 			} else {
 				enqueue_task_dl(rq, dl_task_of(dl_se), ENQUEUE_REPLENISH);
 			}
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 74c1617cf6523..ed3a0d65da0ca 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -210,6 +210,48 @@ static const struct file_operations sched_scaling_fops = {
 	.release	= single_release,
 };
 
+#ifdef CONFIG_SCHED_CACHE
+static ssize_t
+sched_cache_enable_write(struct file *filp, const char __user *ubuf,
+			 size_t cnt, loff_t *ppos)
+{
+	bool val;
+	int ret;
+
+	ret = kstrtobool_from_user(ubuf, cnt, &val);
+	if (ret)
+		return ret;
+
+	sysctl_sched_cache_user = val;
+
+	sched_cache_active_set();
+
+	*ppos += cnt;
+
+	return cnt;
+}
+
+static int sched_cache_enable_show(struct seq_file *m, void *v)
+{
+	seq_printf(m, "%d\n", sysctl_sched_cache_user);
+	return 0;
+}
+
+static int sched_cache_enable_open(struct inode *inode,
+				   struct file *filp)
+{
+	return single_open(filp, sched_cache_enable_show, NULL);
+}
+
+static const struct file_operations sched_cache_enable_fops = {
+	.open           = sched_cache_enable_open,
+	.write          = sched_cache_enable_write,
+	.read           = seq_read,
+	.llseek         = seq_lseek,
+	.release        = single_release,
+};
+#endif
+
 #ifdef CONFIG_PREEMPT_DYNAMIC
 
 static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf,
@@ -593,7 +635,7 @@ static void debugfs_ext_server_init(void)
 
 static __init int sched_init_debug(void)
 {
-	struct dentry __maybe_unused *numa;
+	struct dentry __maybe_unused *numa, *llc;
 
 	debugfs_sched = debugfs_create_dir("sched", NULL);
 
@@ -626,6 +668,22 @@ static __init int sched_init_debug(void)
 	debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold);
 #endif /* CONFIG_NUMA_BALANCING */
 
+#ifdef CONFIG_SCHED_CACHE
+	llc = debugfs_create_dir("llc_balancing", debugfs_sched);
+	debugfs_create_file("enabled", 0644, llc, NULL,
+			    &sched_cache_enable_fops);
+	debugfs_create_u32("aggr_tolerance", 0644, llc,
+			   &llc_aggr_tolerance);
+	debugfs_create_u32("epoch_period", 0644, llc,
+			   &llc_epoch_period);
+	debugfs_create_u32("epoch_affinity_timeout", 0644, llc,
+			   &llc_epoch_affinity_timeout);
+	debugfs_create_u32("overaggr_pct", 0644, llc,
+			   &llc_overaggr_pct);
+	debugfs_create_u32("imb_pct", 0644, llc,
+			   &llc_imb_pct);
+#endif
+
 	debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops);
 
 	debugfs_fair_server_init();
diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
index d3ce708991c3e..295df262ee5a4 100644
--- a/kernel/sched/ext.c
+++ b/kernel/sched/ext.c
@@ -6,8 +6,6 @@
  * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
  * Copyright (c) 2022 David Vernet <dvernet@meta.com>
  */
-#include <linux/btf_ids.h>
-#include "ext_idle.h"
 
 static DEFINE_RAW_SPINLOCK(scx_sched_lock);
 
@@ -38,6 +36,15 @@ static const struct rhashtable_params scx_sched_hash_params = {
 static struct rhashtable scx_sched_hash;
 #endif
 
+/* see SCX_OPS_TID_TO_TASK */
+static const struct rhashtable_params scx_tid_hash_params = {
+	.key_len		= sizeof_field(struct sched_ext_entity, tid),
+	.key_offset		= offsetof(struct sched_ext_entity, tid),
+	.head_offset		= offsetof(struct sched_ext_entity, tid_hash_node),
+	.insecure_elasticity	= true,	/* inserted/removed under scx_tasks_lock */
+};
+static struct rhashtable scx_tid_hash;
+
 /*
  * During exit, a task may schedule after losing its PIDs. When disabling the
  * BPF scheduler, we need to be able to iterate tasks in every state to
@@ -56,10 +63,25 @@ static DEFINE_RAW_SPINLOCK(scx_bypass_lock);
 static bool scx_init_task_enabled;
 static bool scx_switching_all;
 DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
+static DEFINE_STATIC_KEY_FALSE(__scx_tid_to_task_enabled);
+
+/*
+ * True once SCX_OPS_TID_TO_TASK has been negotiated with the root scheduler
+ * and the tid->task table is live. Wraps the static key so callers don't
+ * take the address, and hints "likely enabled" for the common case where
+ * the feature is in use.
+ */
+static inline bool scx_tid_to_task_enabled(void)
+{
+	return static_branch_likely(&__scx_tid_to_task_enabled);
+}
 
 static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);
 static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
 
+/* Global cursor for the per-CPU tid allocator. Starts at 1; tid 0 is reserved. */
+static atomic64_t scx_tid_cursor = ATOMIC64_INIT(1);
+
 #ifdef CONFIG_EXT_SUB_SCHED
 /*
  * The sub sched being enabled. Used by scx_disable_and_exit_task() to exit
@@ -108,6 +130,17 @@ struct scx_kick_syncs {
 
 static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs);
 
+/*
+ * Per-CPU buffered allocator state for p->scx.tid. Each CPU pulls a chunk of
+ * SCX_TID_CHUNK ids from scx_tid_cursor and hands them out locally without
+ * further synchronization. See scx_alloc_tid().
+ */
+struct scx_tid_alloc {
+	u64	next;
+	u64	end;
+};
+static DEFINE_PER_CPU(struct scx_tid_alloc, scx_tid_alloc);
+
 /*
  * Direct dispatch marker.
  *
@@ -198,26 +231,21 @@ static void run_deferred(struct rq *rq);
 static bool task_dead_and_done(struct task_struct *p);
 static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
 static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind);
-static bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind,
-		      s64 exit_code, const char *fmt, va_list args);
 
-static __printf(4, 5) bool scx_exit(struct scx_sched *sch,
-				    enum scx_exit_kind kind, s64 exit_code,
-				    const char *fmt, ...)
+__printf(5, 6) bool __scx_exit(struct scx_sched *sch,
+			       enum scx_exit_kind kind, s64 exit_code,
+			       s32 exit_cpu, const char *fmt, ...)
 {
 	va_list args;
 	bool ret;
 
 	va_start(args, fmt);
-	ret = scx_vexit(sch, kind, exit_code, fmt, args);
+	ret = scx_vexit(sch, kind, exit_code, exit_cpu, fmt, args);
 	va_end(args);
 
 	return ret;
 }
 
-#define scx_error(sch, fmt, args...)	scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args)
-#define scx_verror(sch, fmt, args)	scx_vexit((sch), SCX_EXIT_ERROR, 0, fmt, args)
-
 #define SCX_HAS_OP(sch, op)	test_bit(SCX_OP_IDX(op), (sch)->has_op)
 
 static long jiffies_delta_msecs(unsigned long at, unsigned long now)
@@ -295,9 +323,9 @@ static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch)
 	rcu_assign_pointer(p->scx.sched, sch);
 }
 #else	/* CONFIG_EXT_SUB_SCHED */
-static struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
-static struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
-static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
+static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
+static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
+static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
 #endif	/* CONFIG_EXT_SUB_SCHED */
 
 /**
@@ -484,6 +512,33 @@ do {										\
 		update_locked_rq(__prev_locked_rq);				\
 } while (0)
 
+/*
+ * Flipped on enable per sch->is_cid_type. Declared in ext_internal.h so
+ * subsystem inlines can read it.
+ */
+DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type);
+
+/*
+ * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form
+ * schedulers it resolves to the matching cid; for cpu-form it passes @cpu
+ * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op
+ * (currently only ops.select_cpu); it validates the BPF-supplied cid and
+ * triggers scx_error() on @sch if invalid.
+ */
+static s32 scx_cpu_arg(s32 cpu)
+{
+	if (scx_is_cid_type())
+		return __scx_cpu_to_cid(cpu);
+	return cpu;
+}
+
+static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid)
+{
+	if (cpu_or_cid < 0 || !scx_is_cid_type())
+		return cpu_or_cid;
+	return scx_cid_to_cpu(sch, cpu_or_cid);
+}
+
 #define SCX_CALL_OP_RET(sch, op, locked_rq, args...)				\
 ({										\
 	struct rq *__prev_locked_rq;						\
@@ -545,6 +600,41 @@ do {										\
 	__ret;									\
 })
 
+/**
+ * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task
+ * @sch: scx_sched being invoked
+ * @rq: rq to update as the currently-locked rq, or NULL
+ * @task: task whose affinity is changing
+ * @cpumask: new cpumask
+ *
+ * For cid-form schedulers, translate @cpumask to a cmask via the per-cpu
+ * scratch in ext_cid.c and dispatch through the ops_cid union view. Caller
+ * must hold @rq's rq lock so this_cpu_ptr is stable across the call.
+ */
+static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq,
+					   struct task_struct *task,
+					   const struct cpumask *cpumask)
+{
+	WARN_ON_ONCE(current->scx.kf_tasks[0]);
+	current->scx.kf_tasks[0] = task;
+	if (rq)
+		update_locked_rq(rq);
+
+	if (scx_is_cid_type()) {
+		struct scx_cmask *cmask = this_cpu_ptr(scx_set_cmask_scratch);
+
+		lockdep_assert_irqs_disabled();
+		scx_cpumask_to_cmask(cpumask, cmask);
+		sch->ops_cid.set_cmask(task, cmask);
+	} else {
+		sch->ops.set_cpumask(task, cpumask);
+	}
+
+	if (rq)
+		update_locked_rq(NULL);
+	current->scx.kf_tasks[0] = NULL;
+}
+
 /* see SCX_CALL_OP_TASK() */
 static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch,
 							struct task_struct *p)
@@ -857,6 +947,24 @@ static void __scx_task_iter_maybe_relock(struct scx_task_iter *iter)
 	}
 }
 
+/**
+ * scx_task_iter_relock - Re-acquire scx_tasks_lock and, optionally, @p's rq
+ * @iter: iterator to relock
+ * @p: task whose rq to lock, or %NULL for scx_tasks_lock only
+ *
+ * Counterpart to scx_task_iter_unlock(). Locking @p's rq is optional. Once
+ * re-acquired, both locks are managed by the iterator from here on.
+ */
+static void scx_task_iter_relock(struct scx_task_iter *iter,
+				 struct task_struct *p)
+{
+	__scx_task_iter_maybe_relock(iter);
+	if (p) {
+		iter->rq = task_rq_lock(p, &iter->rf);
+		iter->locked_task = p;
+	}
+}
+
 /**
  * scx_task_iter_stop - Stop a task iteration and unlock scx_tasks_lock
  * @iter: iterator to exit
@@ -1086,7 +1194,7 @@ static inline bool __cpu_valid(s32 cpu)
 }
 
 /**
- * ops_cpu_valid - Verify a cpu number, to be used on ops input args
+ * scx_cpu_valid - Verify a cpu number, to be used on ops input args
  * @sch: scx_sched to abort on error
  * @cpu: cpu number which came from a BPF ops
  * @where: extra information reported on error
@@ -1095,7 +1203,7 @@ static inline bool __cpu_valid(s32 cpu)
  * Verify that it is in range and one of the possible cpus. If invalid, trigger
  * an ops error.
  */
-static bool ops_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where)
+bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where)
 {
 	if (__cpu_valid(cpu)) {
 		return true;
@@ -1742,9 +1850,9 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch,
 		return &rq->scx.local_dsq;
 
 	if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
-		s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
+		s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
 
-		if (!ops_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict"))
+		if (!scx_cpu_valid(sch, cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict"))
 			return find_global_dsq(sch, tcpu);
 
 		return &cpu_rq(cpu)->scx.local_dsq;
@@ -2824,11 +2932,13 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
 		dspc->nr_tasks = 0;
 
 		if (nested) {
-			SCX_CALL_OP(sch, dispatch, rq, cpu, prev_on_sch ? prev : NULL);
+			SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
+				    prev_on_sch ? prev : NULL);
 		} else {
 			/* stash @prev so that nested invocations can access it */
 			rq->scx.sub_dispatch_prev = prev;
-			SCX_CALL_OP(sch, dispatch, rq, cpu, prev_on_sch ? prev : NULL);
+			SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
+				    prev_on_sch ? prev : NULL);
 			rq->scx.sub_dispatch_prev = NULL;
 		}
 
@@ -2886,7 +2996,7 @@ static int balance_one(struct rq *rq, struct task_struct *prev)
 		 * core. This callback complements ->cpu_release(), which is
 		 * emitted in switch_class().
 		 */
-		if (SCX_HAS_OP(sch, cpu_acquire))
+		if (sch->ops.cpu_acquire)
 			SCX_CALL_OP(sch, cpu_acquire, rq, cpu, NULL);
 		rq->scx.cpu_released = false;
 	}
@@ -3032,7 +3142,7 @@ static void switch_class(struct rq *rq, struct task_struct *next)
 	 *  next time that balance_one() is invoked.
 	 */
 	if (!rq->scx.cpu_released) {
-		if (SCX_HAS_OP(sch, cpu_release)) {
+		if (sch->ops.cpu_release) {
 			struct scx_cpu_release_args args = {
 				.reason = preempt_reason_from_class(next_class),
 				.task = next,
@@ -3323,11 +3433,13 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag
 		*ddsp_taskp = p;
 
 		this_rq()->scx.in_select_cpu = true;
-		cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p, prev_cpu, wake_flags);
+		cpu = SCX_CALL_OP_TASK_RET(sch, select_cpu, NULL, p,
+					   scx_cpu_arg(prev_cpu), wake_flags);
+		cpu = scx_cpu_ret(sch, cpu);
 		this_rq()->scx.in_select_cpu = false;
 		p->scx.selected_cpu = cpu;
 		*ddsp_taskp = NULL;
-		if (ops_cpu_valid(sch, cpu, "from ops.select_cpu()"))
+		if (scx_cpu_valid(sch, cpu, "from ops.select_cpu()"))
 			return cpu;
 		else
 			return prev_cpu;
@@ -3373,7 +3485,7 @@ static void set_cpus_allowed_scx(struct task_struct *p,
 	 * designation pointless. Cast it away when calling the operation.
 	 */
 	if (SCX_HAS_OP(sch, set_cpumask))
-		SCX_CALL_OP_TASK(sch, set_cpumask, task_rq(p), p, (struct cpumask *)p->cpus_ptr);
+		scx_call_op_set_cpumask(sch, task_rq(p), p, (struct cpumask *)p->cpus_ptr);
 }
 
 static void handle_hotplug(struct rq *rq, bool online)
@@ -3395,9 +3507,9 @@ static void handle_hotplug(struct rq *rq, bool online)
 		scx_idle_update_selcpu_topology(&sch->ops);
 
 	if (online && SCX_HAS_OP(sch, cpu_online))
-		SCX_CALL_OP(sch, cpu_online, NULL, cpu);
+		SCX_CALL_OP(sch, cpu_online, NULL, scx_cpu_arg(cpu));
 	else if (!online && SCX_HAS_OP(sch, cpu_offline))
-		SCX_CALL_OP(sch, cpu_offline, NULL, cpu);
+		SCX_CALL_OP(sch, cpu_offline, NULL, scx_cpu_arg(cpu));
 	else
 		scx_exit(sch, SCX_EXIT_UNREG_KERN,
 			 SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG,
@@ -3445,9 +3557,10 @@ static bool check_rq_for_timeouts(struct rq *rq)
 					last_runnable + READ_ONCE(sch->watchdog_timeout)))) {
 			u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable);
 
-			scx_exit(sch, SCX_EXIT_ERROR_STALL, 0,
-				 "%s[%d] failed to run for %u.%03us",
-				 p->comm, p->pid, dur_ms / 1000, dur_ms % 1000);
+			__scx_exit(sch, SCX_EXIT_ERROR_STALL, 0, cpu_of(rq),
+				   "%s[%d] failed to run for %u.%03us",
+				   p->comm, p->pid, dur_ms / 1000,
+				   dur_ms % 1000);
 			timed_out = true;
 			break;
 		}
@@ -3735,6 +3848,33 @@ void init_scx_entity(struct sched_ext_entity *scx)
 	scx->slice = SCX_SLICE_DFL;
 }
 
+/* See scx_tid_alloc / scx_tid_cursor. */
+static u64 scx_alloc_tid(void)
+{
+	struct scx_tid_alloc *ta;
+
+	guard(preempt)();
+	ta = this_cpu_ptr(&scx_tid_alloc);
+
+	if (unlikely(ta->next >= ta->end)) {
+		ta->next = atomic64_fetch_add(SCX_TID_CHUNK, &scx_tid_cursor);
+		ta->end = ta->next + SCX_TID_CHUNK;
+	}
+	return ta->next++;
+}
+
+static void scx_tid_hash_insert(struct task_struct *p)
+{
+	int ret;
+
+	lockdep_assert_held(&scx_tasks_lock);
+
+	ret = rhashtable_lookup_insert_fast(&scx_tid_hash,
+					    &p->scx.tid_hash_node,
+					    scx_tid_hash_params);
+	WARN_ON_ONCE(ret);
+}
+
 void scx_pre_fork(struct task_struct *p)
 {
 	/*
@@ -3752,6 +3892,8 @@ int scx_fork(struct task_struct *p, struct kernel_clone_args *kargs)
 
 	percpu_rwsem_assert_held(&scx_fork_rwsem);
 
+	p->scx.tid = scx_alloc_tid();
+
 	if (scx_init_task_enabled) {
 #ifdef CONFIG_EXT_SUB_SCHED
 		struct scx_sched *sch = kargs->cset->dfl_cgrp->scx_sched;
@@ -3791,9 +3933,11 @@ void scx_post_fork(struct task_struct *p)
 		}
 	}
 
-	raw_spin_lock_irq(&scx_tasks_lock);
-	list_add_tail(&p->scx.tasks_node, &scx_tasks);
-	raw_spin_unlock_irq(&scx_tasks_lock);
+	scoped_guard(raw_spinlock_irq, &scx_tasks_lock) {
+		list_add_tail(&p->scx.tasks_node, &scx_tasks);
+		if (scx_tid_to_task_enabled())
+			scx_tid_hash_insert(p);
+	}
 
 	percpu_up_read(&scx_fork_rwsem);
 }
@@ -3844,17 +3988,19 @@ static bool task_dead_and_done(struct task_struct *p)
 
 void sched_ext_dead(struct task_struct *p)
 {
-	unsigned long flags;
-
 	/*
 	 * By the time control reaches here, @p has %TASK_DEAD set, switched out
 	 * for the last time and then dropped the rq lock - task_dead_and_done()
 	 * should be returning %true nullifying the straggling sched_class ops.
 	 * Remove from scx_tasks and exit @p.
 	 */
-	raw_spin_lock_irqsave(&scx_tasks_lock, flags);
-	list_del_init(&p->scx.tasks_node);
-	raw_spin_unlock_irqrestore(&scx_tasks_lock, flags);
+	scoped_guard(raw_spinlock_irqsave, &scx_tasks_lock) {
+		list_del_init(&p->scx.tasks_node);
+		if (scx_tid_to_task_enabled())
+			rhashtable_remove_fast(&scx_tid_hash,
+					       &p->scx.tid_hash_node,
+					       scx_tid_hash_params);
+	}
 
 	/*
 	 * @p is off scx_tasks and wholly ours. scx_root_enable()'s READY ->
@@ -3914,7 +4060,7 @@ static void switching_to_scx(struct rq *rq, struct task_struct *p)
 	 * different scheduler class. Keep the BPF scheduler up-to-date.
 	 */
 	if (SCX_HAS_OP(sch, set_cpumask))
-		SCX_CALL_OP_TASK(sch, set_cpumask, rq, p, (struct cpumask *)p->cpus_ptr);
+		scx_call_op_set_cpumask(sch, rq, p, (struct cpumask *)p->cpus_ptr);
 }
 
 static void switched_from_scx(struct rq *rq, struct task_struct *p)
@@ -4495,9 +4641,9 @@ static void scx_cgroup_unlock(void)
 #endif
 }
 #else	/* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
-static struct cgroup *root_cgroup(void) { return NULL; }
-static void scx_cgroup_lock(void) {}
-static void scx_cgroup_unlock(void) {}
+static inline struct cgroup *root_cgroup(void) { return NULL; }
+static inline void scx_cgroup_lock(void) {}
+static inline void scx_cgroup_unlock(void) {}
 #endif	/* CONFIG_EXT_GROUP_SCHED || CONFIG_EXT_SUB_SCHED */
 
 #ifdef CONFIG_EXT_SUB_SCHED
@@ -4516,8 +4662,8 @@ static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch)
 		rcu_assign_pointer(pos->scx_sched, sch);
 }
 #else	/* CONFIG_EXT_SUB_SCHED */
-static struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
-static void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
+static inline struct cgroup *sch_cgroup(struct scx_sched *sch) { return NULL; }
+static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
 #endif	/* CONFIG_EXT_SUB_SCHED */
 
 /*
@@ -4946,10 +5092,30 @@ static const struct kset_uevent_ops scx_uevent_ops = {
  */
 bool task_should_scx(int policy)
 {
-	if (!scx_enabled() || unlikely(scx_enable_state() == SCX_DISABLING))
+	/* if disabled, nothing should be on it */
+	if (!scx_enabled())
 		return false;
+
+	/* scx is taking over all SCHED_OTHER and SCHED_EXT tasks */
 	if (READ_ONCE(scx_switching_all))
 		return true;
+
+	/*
+	 * scx is tearing down - keep new SCHED_EXT tasks out.
+	 *
+	 * Must come after scx_switching_all test, which serves as a proxy
+	 * for __scx_switched_all. While __scx_switched_all is set, we must
+	 * return true via the branch above: a fork routed to fair would
+	 * stall because next_active_class() skips fair.
+	 *
+	 * This can develop into a deadlock - scx holds scx_enable_mutex across
+	 * kthread_create() in scx_alloc_and_add_sched(); if the new kthread is
+	 * the stalled task, the disable path can never grab the mutex to clear
+	 * scx_switching_all.
+	 */
+	if (unlikely(scx_enable_state() == SCX_DISABLING))
+		return false;
+
 	return policy == SCHED_EXT;
 }
 
@@ -5528,6 +5694,7 @@ static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len)
 	if (!ei)
 		return NULL;
 
+	ei->exit_cpu = -1;
 	ei->bt = kzalloc_objs(ei->bt[0], SCX_EXIT_BT_LEN);
 	ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL);
 	ei->dump = kvzalloc(exit_dump_len, GFP_KERNEL);
@@ -5674,6 +5841,26 @@ static void scx_disable_dump(struct scx_sched *sch)
 	sch->dump_disabled = true;
 }
 
+static void scx_log_sched_disable(struct scx_sched *sch)
+{
+	struct scx_exit_info *ei = sch->exit_info;
+	const char *type = scx_parent(sch) ? "sub-scheduler" : "scheduler";
+
+	if (ei->kind >= SCX_EXIT_ERROR) {
+		pr_err("sched_ext: BPF %s \"%s\" disabled (%s)\n", type,
+		       sch->ops.name, ei->reason);
+
+		if (ei->msg[0] != '\0')
+			pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg);
+#ifdef CONFIG_STACKTRACE
+		stack_trace_print(ei->bt, ei->bt_len, 2);
+#endif
+	} else {
+		pr_info("sched_ext: BPF %s \"%s\" disabled (%s)\n", type,
+			sch->ops.name, ei->reason);
+	}
+}
+
 #ifdef CONFIG_EXT_SUB_SCHED
 static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq);
 
@@ -5760,14 +5947,11 @@ static void scx_sub_disable(struct scx_sched *sch)
 		WARN_ON_ONCE(!scx_task_on_sched(sch, p));
 
 		/*
-		 * If $p is about to be freed, nothing prevents $sch from
-		 * unloading before $p reaches sched_ext_free(). Disable and
-		 * exit $p right away.
+		 * @p is pinned by the iter: css_task_iter_next() takes a
+		 * reference and holds it until the next iter_next() call, so
+		 * @p->usage is guaranteed > 0.
 		 */
-		if (!tryget_task_struct(p)) {
-			scx_disable_and_exit_task(sch, p);
-			continue;
-		}
+		get_task_struct(p);
 
 		scx_task_iter_unlock(&sti);
 
@@ -5860,6 +6044,8 @@ static void scx_sub_disable(struct scx_sched *sch)
 			    &sub_detach_args);
 	}
 
+	scx_log_sched_disable(sch);
+
 	if (sch->ops.exit)
 		SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
 	if (sch->sub_kset)
@@ -5867,13 +6053,12 @@ static void scx_sub_disable(struct scx_sched *sch)
 	kobject_del(&sch->kobj);
 }
 #else	/* CONFIG_EXT_SUB_SCHED */
-static void drain_descendants(struct scx_sched *sch) { }
-static void scx_sub_disable(struct scx_sched *sch) { }
+static inline void drain_descendants(struct scx_sched *sch) { }
+static inline void scx_sub_disable(struct scx_sched *sch) { }
 #endif	/* CONFIG_EXT_SUB_SCHED */
 
 static void scx_root_disable(struct scx_sched *sch)
 {
-	struct scx_exit_info *ei = sch->exit_info;
 	struct scx_task_iter sti;
 	struct task_struct *p;
 	int cpu;
@@ -5959,26 +6144,19 @@ static void scx_root_disable(struct scx_sched *sch)
 
 	/* no task is on scx, turn off all the switches and flush in-progress calls */
 	static_branch_disable(&__scx_enabled);
+	static_branch_disable(&__scx_is_cid_type);
+	if (sch->ops.flags & SCX_OPS_TID_TO_TASK)
+		static_branch_disable(&__scx_tid_to_task_enabled);
 	bitmap_zero(sch->has_op, SCX_OPI_END);
 	scx_idle_disable();
 	synchronize_rcu();
+	if (sch->ops.flags & SCX_OPS_TID_TO_TASK)
+		rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
 
-	if (ei->kind >= SCX_EXIT_ERROR) {
-		pr_err("sched_ext: BPF scheduler \"%s\" disabled (%s)\n",
-		       sch->ops.name, ei->reason);
-
-		if (ei->msg[0] != '\0')
-			pr_err("sched_ext: %s: %s\n", sch->ops.name, ei->msg);
-#ifdef CONFIG_STACKTRACE
-		stack_trace_print(ei->bt, ei->bt_len, 2);
-#endif
-	} else {
-		pr_info("sched_ext: BPF scheduler \"%s\" disabled (%s)\n",
-			sch->ops.name, ei->reason);
-	}
+	scx_log_sched_disable(sch);
 
 	if (sch->ops.exit)
-		SCX_CALL_OP(sch, exit, NULL, ei);
+		SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
 
 	scx_unlink_sched(sch);
 
@@ -6276,6 +6454,94 @@ static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_d
 	}
 }
 
+static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
+			 struct scx_dump_ctx *dctx, int cpu,
+			 bool dump_all_tasks)
+{
+	struct rq *rq = cpu_rq(cpu);
+	struct rq_flags rf;
+	struct task_struct *p;
+	struct seq_buf ns;
+	size_t avail, used;
+	char *buf;
+	bool idle;
+
+	rq_lock_irqsave(rq, &rf);
+
+	idle = list_empty(&rq->scx.runnable_list) &&
+		rq->curr->sched_class == &idle_sched_class;
+
+	if (idle && !SCX_HAS_OP(sch, dump_cpu))
+		goto next;
+
+	/*
+	 * We don't yet know whether ops.dump_cpu() will produce output
+	 * and we may want to skip the default CPU dump if it doesn't.
+	 * Use a nested seq_buf to generate the standard dump so that we
+	 * can decide whether to commit later.
+	 */
+	avail = seq_buf_get_buf(s, &buf);
+	seq_buf_init(&ns, buf, avail);
+
+	dump_newline(&ns);
+	dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu",
+		  cpu, rq->scx.nr_running, rq->scx.flags,
+		  rq->scx.cpu_released, rq->scx.ops_qseq,
+		  rq->scx.kick_sync);
+	dump_line(&ns, "          curr=%s[%d] class=%ps",
+		  rq->curr->comm, rq->curr->pid,
+		  rq->curr->sched_class);
+	if (!cpumask_empty(rq->scx.cpus_to_kick))
+		dump_line(&ns, "  cpus_to_kick   : %*pb",
+			  cpumask_pr_args(rq->scx.cpus_to_kick));
+	if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle))
+		dump_line(&ns, "  idle_to_kick   : %*pb",
+			  cpumask_pr_args(rq->scx.cpus_to_kick_if_idle));
+	if (!cpumask_empty(rq->scx.cpus_to_preempt))
+		dump_line(&ns, "  cpus_to_preempt: %*pb",
+			  cpumask_pr_args(rq->scx.cpus_to_preempt));
+	if (!cpumask_empty(rq->scx.cpus_to_wait))
+		dump_line(&ns, "  cpus_to_wait   : %*pb",
+			  cpumask_pr_args(rq->scx.cpus_to_wait));
+	if (!cpumask_empty(rq->scx.cpus_to_sync))
+		dump_line(&ns, "  cpus_to_sync   : %*pb",
+			  cpumask_pr_args(rq->scx.cpus_to_sync));
+
+	used = seq_buf_used(&ns);
+	if (SCX_HAS_OP(sch, dump_cpu)) {
+		ops_dump_init(&ns, "  ");
+		SCX_CALL_OP(sch, dump_cpu, rq, dctx, scx_cpu_arg(cpu), idle);
+		ops_dump_exit();
+	}
+
+	/*
+	 * If idle && nothing generated by ops.dump_cpu(), there's
+	 * nothing interesting. Skip.
+	 */
+	if (idle && used == seq_buf_used(&ns))
+		goto next;
+
+	/*
+	 * $s may already have overflowed when $ns was created. If so,
+	 * calling commit on it will trigger BUG.
+	 */
+	if (avail) {
+		seq_buf_commit(s, seq_buf_used(&ns));
+		if (seq_buf_has_overflowed(&ns))
+			seq_buf_set_overflow(s);
+	}
+
+	if (rq->curr->sched_class == &ext_sched_class &&
+	    (dump_all_tasks || scx_task_on_sched(sch, rq->curr)))
+		scx_dump_task(sch, s, dctx, rq, rq->curr, '*');
+
+	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
+		if (dump_all_tasks || scx_task_on_sched(sch, p))
+			scx_dump_task(sch, s, dctx, rq, p, ' ');
+next:
+	rq_unlock_irqrestore(rq, &rf);
+}
+
 /*
  * Dump scheduler state. If @dump_all_tasks is true, dump all tasks regardless
  * of which scheduler they belong to. If false, only dump tasks owned by @sch.
@@ -6296,7 +6562,6 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
 	};
 	struct seq_buf s;
 	struct scx_event_stats events;
-	char *buf;
 	int cpu;
 
 	guard(raw_spinlock_irqsave)(&scx_dump_lock);
@@ -6317,8 +6582,13 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
 	if (ei->kind == SCX_EXIT_NONE) {
 		dump_line(&s, "Debug dump triggered by %s", ei->reason);
 	} else {
-		dump_line(&s, "%s[%d] triggered exit kind %d:",
-			  current->comm, current->pid, ei->kind);
+		if (ei->exit_cpu >= 0)
+			dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:",
+				  current->comm, current->pid, ei->kind,
+				  ei->exit_cpu);
+		else
+			dump_line(&s, "%s[%d] triggered exit kind %d:",
+				  current->comm, current->pid, ei->kind);
 		dump_line(&s, "  %s (%s)", ei->reason, ei->msg);
 		dump_newline(&s);
 		dump_line(&s, "Backtrace:");
@@ -6335,88 +6605,15 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
 	dump_line(&s, "CPU states");
 	dump_line(&s, "----------");
 
+	/*
+	 * Dump the exit CPU first so it isn't lost to dump truncation, then
+	 * walk the rest in order, skipping the one already dumped.
+	 */
+	if (ei->exit_cpu >= 0)
+		scx_dump_cpu(sch, &s, &dctx, ei->exit_cpu, dump_all_tasks);
 	for_each_possible_cpu(cpu) {
-		struct rq *rq = cpu_rq(cpu);
-		struct rq_flags rf;
-		struct task_struct *p;
-		struct seq_buf ns;
-		size_t avail, used;
-		bool idle;
-
-		rq_lock_irqsave(rq, &rf);
-
-		idle = list_empty(&rq->scx.runnable_list) &&
-			rq->curr->sched_class == &idle_sched_class;
-
-		if (idle && !SCX_HAS_OP(sch, dump_cpu))
-			goto next;
-
-		/*
-		 * We don't yet know whether ops.dump_cpu() will produce output
-		 * and we may want to skip the default CPU dump if it doesn't.
-		 * Use a nested seq_buf to generate the standard dump so that we
-		 * can decide whether to commit later.
-		 */
-		avail = seq_buf_get_buf(&s, &buf);
-		seq_buf_init(&ns, buf, avail);
-
-		dump_newline(&ns);
-		dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu",
-			  cpu, rq->scx.nr_running, rq->scx.flags,
-			  rq->scx.cpu_released, rq->scx.ops_qseq,
-			  rq->scx.kick_sync);
-		dump_line(&ns, "          curr=%s[%d] class=%ps",
-			  rq->curr->comm, rq->curr->pid,
-			  rq->curr->sched_class);
-		if (!cpumask_empty(rq->scx.cpus_to_kick))
-			dump_line(&ns, "  cpus_to_kick   : %*pb",
-				  cpumask_pr_args(rq->scx.cpus_to_kick));
-		if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle))
-			dump_line(&ns, "  idle_to_kick   : %*pb",
-				  cpumask_pr_args(rq->scx.cpus_to_kick_if_idle));
-		if (!cpumask_empty(rq->scx.cpus_to_preempt))
-			dump_line(&ns, "  cpus_to_preempt: %*pb",
-				  cpumask_pr_args(rq->scx.cpus_to_preempt));
-		if (!cpumask_empty(rq->scx.cpus_to_wait))
-			dump_line(&ns, "  cpus_to_wait   : %*pb",
-				  cpumask_pr_args(rq->scx.cpus_to_wait));
-		if (!cpumask_empty(rq->scx.cpus_to_sync))
-			dump_line(&ns, "  cpus_to_sync   : %*pb",
-				  cpumask_pr_args(rq->scx.cpus_to_sync));
-
-		used = seq_buf_used(&ns);
-		if (SCX_HAS_OP(sch, dump_cpu)) {
-			ops_dump_init(&ns, "  ");
-			SCX_CALL_OP(sch, dump_cpu, rq, &dctx, cpu, idle);
-			ops_dump_exit();
-		}
-
-		/*
-		 * If idle && nothing generated by ops.dump_cpu(), there's
-		 * nothing interesting. Skip.
-		 */
-		if (idle && used == seq_buf_used(&ns))
-			goto next;
-
-		/*
-		 * $s may already have overflowed when $ns was created. If so,
-		 * calling commit on it will trigger BUG.
-		 */
-		if (avail) {
-			seq_buf_commit(&s, seq_buf_used(&ns));
-			if (seq_buf_has_overflowed(&ns))
-				seq_buf_set_overflow(&s);
-		}
-
-		if (rq->curr->sched_class == &ext_sched_class &&
-		    (dump_all_tasks || scx_task_on_sched(sch, rq->curr)))
-			scx_dump_task(sch, &s, &dctx, rq, rq->curr, '*');
-
-		list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
-			if (dump_all_tasks || scx_task_on_sched(sch, p))
-				scx_dump_task(sch, &s, &dctx, rq, p, ' ');
-	next:
-		rq_unlock_irqrestore(rq, &rf);
+		if (cpu != ei->exit_cpu)
+			scx_dump_cpu(sch, &s, &dctx, cpu, dump_all_tasks);
 	}
 
 	dump_newline(&s);
@@ -6454,9 +6651,9 @@ static void scx_disable_irq_workfn(struct irq_work *irq_work)
 	kthread_queue_work(sch->helper, &sch->disable_work);
 }
 
-static bool scx_vexit(struct scx_sched *sch,
-		      enum scx_exit_kind kind, s64 exit_code,
-		      const char *fmt, va_list args)
+bool scx_vexit(struct scx_sched *sch,
+	       enum scx_exit_kind kind, s64 exit_code, s32 exit_cpu,
+	       const char *fmt, va_list args)
 {
 	struct scx_exit_info *ei = sch->exit_info;
 
@@ -6478,6 +6675,7 @@ static bool scx_vexit(struct scx_sched *sch,
 	 */
 	ei->kind = kind;
 	ei->reason = scx_exit_reason(ei->kind);
+	ei->exit_cpu = exit_cpu;
 
 	irq_work_queue(&sch->disable_irq_work);
 	return true;
@@ -6534,14 +6732,32 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node)
 	return pnode;
 }
 
+/*
+ * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid
+ * starvation. During the READY -> ENABLED task switching loop, the calling
+ * thread's sched_class gets switched from fair to ext. As fair has higher
+ * priority than ext, the calling thread can be indefinitely starved under
+ * fair-class saturation, leading to a system hang.
+ */
+struct scx_enable_cmd {
+	struct kthread_work	work;
+	union {
+		struct sched_ext_ops		*ops;
+		struct sched_ext_ops_cid	*ops_cid;
+	};
+	bool			is_cid_type;
+	int			ret;
+};
+
 /*
  * Allocate and initialize a new scx_sched. @cgrp's reference is always
  * consumed whether the function succeeds or fails.
  */
-static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops,
+static struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
 						 struct cgroup *cgrp,
 						 struct scx_sched *parent)
 {
+	struct sched_ext_ops *ops = cmd->ops;
 	struct scx_sched *sch;
 	s32 level = parent ? parent->level + 1 : 0;
 	s32 node, cpu, ret, bypass_fail_cpu = nr_cpu_ids;
@@ -6633,7 +6849,18 @@ static struct scx_sched *scx_alloc_and_add_sched(struct sched_ext_ops *ops,
 		ret = -ENOMEM;
 		goto err_free_lb_cpumask;
 	}
-	sch->ops = *ops;
+	/*
+	 * Copy ops through the right union view. For cid-form the source is
+	 * struct sched_ext_ops_cid which lacks the trailing cpu_acquire/
+	 * cpu_release; those stay zero from kzalloc.
+	 */
+	if (cmd->is_cid_type) {
+		sch->ops_cid = *cmd->ops_cid;
+		sch->is_cid_type = true;
+	} else {
+		sch->ops = *cmd->ops;
+	}
+
 	rcu_assign_pointer(ops->priv, sch);
 
 	sch->kobj.kset = scx_kset;
@@ -6756,6 +6983,17 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
 		return -EINVAL;
 	}
 
+	/*
+	 * SCX_OPS_TID_TO_TASK is enabled by the root scheduler. A sub-sched
+	 * may set it to declare a dependency; reject if the root hasn't
+	 * enabled it.
+	 */
+	if ((ops->flags & SCX_OPS_TID_TO_TASK) && scx_parent(sch) &&
+	    !(scx_root->ops.flags & SCX_OPS_TID_TO_TASK)) {
+		scx_error(sch, "SCX_OPS_TID_TO_TASK requires root scheduler to enable it");
+		return -EINVAL;
+	}
+
 	/*
 	 * SCX_OPS_BUILTIN_IDLE_PER_NODE requires built-in CPU idle
 	 * selection policy to be enabled.
@@ -6766,25 +7004,34 @@ static int validate_ops(struct scx_sched *sch, const struct sched_ext_ops *ops)
 		return -EINVAL;
 	}
 
-	if (ops->cpu_acquire || ops->cpu_release)
+	/*
+	 * cid-form's struct is shorter and doesn't include the cpu_acquire /
+	 * cpu_release tail; reading those fields off a cid-form @ops would
+	 * run past the BPF allocation. Skip for cid-form.
+	 */
+	if (!sch->is_cid_type && (ops->cpu_acquire || ops->cpu_release))
 		pr_warn("ops->cpu_acquire/release() are deprecated, use sched_switch TP instead\n");
 
+	/*
+	 * Sub-scheduler support is tied to the cid-form struct_ops. A sub-sched
+	 * attaches through a cid-form-only interface (sub_attach/sub_detach),
+	 * and a root that accepts sub-scheds must expose cid-form state to
+	 * them. Reject cpu-form schedulers on either side.
+	 */
+	if (!sch->is_cid_type) {
+		if (scx_parent(sch)) {
+			scx_error(sch, "sub-sched requires cid-form struct_ops");
+			return -EINVAL;
+		}
+		if (ops->sub_attach || ops->sub_detach) {
+			scx_error(sch, "sub_attach/sub_detach requires cid-form struct_ops");
+			return -EINVAL;
+		}
+	}
+
 	return 0;
 }
 
-/*
- * scx_enable() is offloaded to a dedicated system-wide RT kthread to avoid
- * starvation. During the READY -> ENABLED task switching loop, the calling
- * thread's sched_class gets switched from fair to ext. As fair has higher
- * priority than ext, the calling thread can be indefinitely starved under
- * fair-class saturation, leading to a system hang.
- */
-struct scx_enable_cmd {
-	struct kthread_work	work;
-	struct sched_ext_ops	*ops;
-	int			ret;
-};
-
 static void scx_root_enable_workfn(struct kthread_work *work)
 {
 	struct scx_enable_cmd *cmd = container_of(work, struct scx_enable_cmd, work);
@@ -6819,15 +7066,24 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 	if (ret)
 		goto err_unlock;
 
+	if (ops->flags & SCX_OPS_TID_TO_TASK) {
+		ret = rhashtable_init(&scx_tid_hash, &scx_tid_hash_params);
+		if (ret)
+			goto err_free_ksyncs;
+	}
+
 #ifdef CONFIG_EXT_SUB_SCHED
 	cgroup_get(cgrp);
 #endif
-	sch = scx_alloc_and_add_sched(ops, cgrp, NULL);
+	sch = scx_alloc_and_add_sched(cmd, cgrp, NULL);
 	if (IS_ERR(sch)) {
 		ret = PTR_ERR(sch);
-		goto err_free_ksyncs;
+		goto err_free_tid_hash;
 	}
 
+	if (sch->is_cid_type)
+		static_branch_enable(&__scx_is_cid_type);
+
 	/*
 	 * Transition to ENABLING and clear exit info to arm the disable path.
 	 * Failure triggers full disabling from here on.
@@ -6850,6 +7106,18 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 	 */
 	cpus_read_lock();
 
+	/*
+	 * Build the cid mapping before publishing scx_root. The cid kfuncs
+	 * dereference the cid arrays unconditionally once scx_prog_sched()
+	 * returns non-NULL; the rcu_assign_pointer() below pairs with their
+	 * rcu_dereference() to make the populated arrays visible.
+	 */
+	ret = scx_cid_init(sch);
+	if (ret) {
+		cpus_read_unlock();
+		goto err_disable;
+	}
+
 	/*
 	 * Make the scheduler instance visible. Must be inside cpus_read_lock().
 	 * See handle_hotplug().
@@ -6916,6 +7184,10 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 	WARN_ON_ONCE(scx_init_task_enabled);
 	scx_init_task_enabled = true;
 
+	/* flip under fork_rwsem; the iter below covers existing tasks */
+	if (ops->flags & SCX_OPS_TID_TO_TASK)
+		static_branch_enable(&__scx_tid_to_task_enabled);
+
 	/*
 	 * Enable ops for every task. Fork is excluded by scx_fork_rwsem
 	 * preventing new tasks from being added. No need to exclude tasks
@@ -6937,16 +7209,14 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 
 	scx_task_iter_start(&sti, NULL);
 	while ((p = scx_task_iter_next_locked(&sti))) {
-		struct rq_flags rf;
-		struct rq *rq;
-
 		/*
-		 * @p may already be dead, have lost all its usages counts and
-		 * be waiting for RCU grace period before being freed. @p can't
-		 * be initialized for SCX in such cases and should be ignored.
+		 * @p is in scx_tasks under scx_tasks_lock, and SCX_TASK_DEAD
+		 * tasks are filtered by scx_task_iter_next_locked().
+		 * sched_ext_dead() removes @p from scx_tasks under the same
+		 * lock before put_task_struct_rcu_user() runs, so @p->usage
+		 * is guaranteed > 0 here.
 		 */
-		if (!tryget_task_struct(p))
-			continue;
+		get_task_struct(p);
 
 		/*
 		 * Set %INIT_BEGIN under the iter's rq lock so that a concurrent
@@ -6962,12 +7232,11 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 
 		ret = __scx_init_task(sch, p, false);
 
-		rq = task_rq_lock(p, &rf);
+		scx_task_iter_relock(&sti, p);
 
 		if (unlikely(ret)) {
 			if (scx_get_task_state(p) != SCX_TASK_DEAD)
 				scx_set_task_state(p, SCX_TASK_NONE);
-			task_rq_unlock(rq, p, &rf);
 			scx_task_iter_stop(&sti);
 			scx_error(sch, "ops.init_task() failed (%d) for %s[%d]",
 				  ret, p->comm, p->pid);
@@ -6988,7 +7257,14 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 			scx_set_task_state(p, SCX_TASK_READY);
 		}
 
-		task_rq_unlock(rq, p, &rf);
+		/*
+		 * Insert into the tid hash. scx_tasks_lock is held by the iter;
+		 * list_empty() guards against sched_ext_dead() having taken @p
+		 * off the list while init ran unlocked.
+		 */
+		if (scx_tid_to_task_enabled() && !list_empty(&p->scx.tasks_node))
+			scx_tid_hash_insert(p);
+
 		put_task_struct(p);
 	}
 	scx_task_iter_stop(&sti);
@@ -7048,6 +7324,9 @@ static void scx_root_enable_workfn(struct kthread_work *work)
 	cmd->ret = 0;
 	return;
 
+err_free_tid_hash:
+	if (ops->flags & SCX_OPS_TID_TO_TASK)
+		rhashtable_free_and_destroy(&scx_tid_hash, NULL, NULL);
 err_free_ksyncs:
 	free_kick_syncs();
 err_unlock:
@@ -7155,7 +7434,7 @@ static void scx_sub_enable_workfn(struct kthread_work *work)
 	raw_spin_unlock_irq(&scx_sched_lock);
 
 	/* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */
-	sch = scx_alloc_and_add_sched(ops, cgrp, parent);
+	sch = scx_alloc_and_add_sched(cmd, cgrp, parent);
 	kobject_put(&parent->kobj);
 	if (IS_ERR(sch)) {
 		ret = PTR_ERR(sch);
@@ -7244,9 +7523,8 @@ static void scx_sub_enable_workfn(struct kthread_work *work)
 		if (p->scx.flags & SCX_TASK_SUB_INIT)
 			continue;
 
-		/* see scx_root_enable() */
-		if (!tryget_task_struct(p))
-			continue;
+		/* @p is pinned by the iter; see scx_sub_disable() */
+		get_task_struct(p);
 
 		if (!assert_task_ready_or_enabled(p)) {
 			ret = -EINVAL;
@@ -7409,11 +7687,10 @@ static s32 __init scx_cgroup_lifetime_notifier_init(void)
 core_initcall(scx_cgroup_lifetime_notifier_init);
 #endif	/* CONFIG_EXT_SUB_SCHED */
 
-static s32 scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
+static s32 scx_enable(struct scx_enable_cmd *cmd, struct bpf_link *link)
 {
 	static struct kthread_worker *helper;
 	static DEFINE_MUTEX(helper_mutex);
-	struct scx_enable_cmd cmd;
 
 	if (housekeeping_enabled(HK_TYPE_DOMAIN_BOOT)) {
 		pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation\n");
@@ -7436,16 +7713,15 @@ static s32 scx_enable(struct sched_ext_ops *ops, struct bpf_link *link)
 	}
 
 #ifdef CONFIG_EXT_SUB_SCHED
-	if (ops->sub_cgroup_id > 1)
-		kthread_init_work(&cmd.work, scx_sub_enable_workfn);
+	if (cmd->ops->sub_cgroup_id > 1)
+		kthread_init_work(&cmd->work, scx_sub_enable_workfn);
 	else
 #endif	/* CONFIG_EXT_SUB_SCHED */
-		kthread_init_work(&cmd.work, scx_root_enable_workfn);
-	cmd.ops = ops;
+		kthread_init_work(&cmd->work, scx_root_enable_workfn);
 
-	kthread_queue_work(READ_ONCE(helper), &cmd.work);
-	kthread_flush_work(&cmd.work);
-	return cmd.ret;
+	kthread_queue_work(READ_ONCE(helper), &cmd->work);
+	kthread_flush_work(&cmd->work);
+	return cmd->ret;
 }
 
 
@@ -7617,7 +7893,16 @@ static int bpf_scx_check_member(const struct btf_type *t,
 
 static int bpf_scx_reg(void *kdata, struct bpf_link *link)
 {
-	return scx_enable(kdata, link);
+	struct scx_enable_cmd cmd = { .ops = kdata };
+
+	return scx_enable(&cmd, link);
+}
+
+static int bpf_scx_reg_cid(void *kdata, struct bpf_link *link)
+{
+	struct scx_enable_cmd cmd = { .ops_cid = kdata, .is_cid_type = true };
+
+	return scx_enable(&cmd, link);
 }
 
 static void bpf_scx_unreg(void *kdata, struct bpf_link *link)
@@ -7751,6 +8036,73 @@ static struct bpf_struct_ops bpf_sched_ext_ops = {
 	.cfi_stubs = &__bpf_ops_sched_ext_ops
 };
 
+/*
+ * cid-form cfi stubs. Stubs whose signatures match the cpu-form (param types
+ * identical, only param names differ across structs) are reused; only
+ * set_cmask needs a fresh stub since the second argument type differs.
+ */
+static void sched_ext_ops_cid__set_cmask(struct task_struct *p,
+					 const struct scx_cmask *cmask) {}
+
+static struct sched_ext_ops_cid __bpf_ops_sched_ext_ops_cid = {
+	.select_cid		= sched_ext_ops__select_cpu,
+	.enqueue		= sched_ext_ops__enqueue,
+	.dequeue		= sched_ext_ops__dequeue,
+	.dispatch		= sched_ext_ops__dispatch,
+	.tick			= sched_ext_ops__tick,
+	.runnable		= sched_ext_ops__runnable,
+	.running		= sched_ext_ops__running,
+	.stopping		= sched_ext_ops__stopping,
+	.quiescent		= sched_ext_ops__quiescent,
+	.yield			= sched_ext_ops__yield,
+	.core_sched_before	= sched_ext_ops__core_sched_before,
+	.set_weight		= sched_ext_ops__set_weight,
+	.set_cmask		= sched_ext_ops_cid__set_cmask,
+	.update_idle		= sched_ext_ops__update_idle,
+	.init_task		= sched_ext_ops__init_task,
+	.exit_task		= sched_ext_ops__exit_task,
+	.enable			= sched_ext_ops__enable,
+	.disable		= sched_ext_ops__disable,
+#ifdef CONFIG_EXT_GROUP_SCHED
+	.cgroup_init		= sched_ext_ops__cgroup_init,
+	.cgroup_exit		= sched_ext_ops__cgroup_exit,
+	.cgroup_prep_move	= sched_ext_ops__cgroup_prep_move,
+	.cgroup_move		= sched_ext_ops__cgroup_move,
+	.cgroup_cancel_move	= sched_ext_ops__cgroup_cancel_move,
+	.cgroup_set_weight	= sched_ext_ops__cgroup_set_weight,
+	.cgroup_set_bandwidth	= sched_ext_ops__cgroup_set_bandwidth,
+	.cgroup_set_idle	= sched_ext_ops__cgroup_set_idle,
+#endif
+	.sub_attach		= sched_ext_ops__sub_attach,
+	.sub_detach		= sched_ext_ops__sub_detach,
+	.cid_online		= sched_ext_ops__cpu_online,
+	.cid_offline		= sched_ext_ops__cpu_offline,
+	.init			= sched_ext_ops__init,
+	.exit			= sched_ext_ops__exit,
+	.dump			= sched_ext_ops__dump,
+	.dump_cid		= sched_ext_ops__dump_cpu,
+	.dump_task		= sched_ext_ops__dump_task,
+};
+
+/*
+ * The cid-form struct_ops shares all bpf_struct_ops hooks with the cpu form.
+ * init_member, check_member, reg, unreg, etc. process kdata as the byte block
+ * verified to match by the BUILD_BUG_ON checks in scx_init().
+ */
+static struct bpf_struct_ops bpf_sched_ext_ops_cid = {
+	.verifier_ops = &bpf_scx_verifier_ops,
+	.reg = bpf_scx_reg_cid,
+	.unreg = bpf_scx_unreg,
+	.check_member = bpf_scx_check_member,
+	.init_member = bpf_scx_init_member,
+	.init = bpf_scx_init,
+	.update = bpf_scx_update,
+	.validate = bpf_scx_validate,
+	.name = "sched_ext_ops_cid",
+	.owner = THIS_MODULE,
+	.cfi_stubs = &__bpf_ops_sched_ext_ops_cid
+};
+
 
 /********************************************************************************
  * System integration and init.
@@ -7760,13 +8112,11 @@ static void sysrq_handle_sched_ext_reset(u8 key)
 {
 	struct scx_sched *sch;
 
-	rcu_read_lock();
 	sch = rcu_dereference(scx_root);
 	if (likely(sch))
 		scx_disable(sch, SCX_EXIT_SYSRQ);
 	else
 		pr_info("sched_ext: BPF schedulers not loaded\n");
-	rcu_read_unlock();
 }
 
 static const struct sysrq_key_op sysrq_sched_ext_reset_op = {
@@ -7778,7 +8128,11 @@ static const struct sysrq_key_op sysrq_sched_ext_reset_op = {
 
 static void sysrq_handle_sched_ext_dump(u8 key)
 {
-	struct scx_exit_info ei = { .kind = SCX_EXIT_NONE, .reason = "SysRq-D" };
+	struct scx_exit_info ei = {
+		.kind		= SCX_EXIT_NONE,
+		.exit_cpu	= -1,
+		.reason		= "SysRq-D",
+	};
 	struct scx_sched *sch;
 
 	list_for_each_entry_rcu(sch, &scx_sched_all, all)
@@ -8848,9 +9202,6 @@ static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags)
 	struct rq *this_rq;
 	unsigned long irq_flags;
 
-	if (!ops_cpu_valid(sch, cpu, NULL))
-		return;
-
 	local_irq_save(irq_flags);
 
 	this_rq = this_rq();
@@ -8913,7 +9264,29 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux
 
 	guard(rcu)();
 	sch = scx_prog_sched(aux);
-	if (likely(sch))
+	if (likely(sch) && scx_cpu_valid(sch, cpu, NULL))
+		scx_kick_cpu(sch, cpu, flags);
+}
+
+/**
+ * scx_bpf_kick_cid - Trigger reschedule on the CPU mapped to @cid
+ * @cid: cid to kick
+ * @flags: %SCX_KICK_* flags
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * cid-addressed equivalent of scx_bpf_kick_cpu().
+ */
+__bpf_kfunc void scx_bpf_kick_cid(s32 cid, u64 flags, const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+	s32 cpu;
+
+	guard(rcu)();
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return;
+	cpu = scx_cid_to_cpu(sch, cid);
+	if (cpu >= 0)
 		scx_kick_cpu(sch, cpu, flags);
 }
 
@@ -8943,9 +9316,9 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux
 		ret = READ_ONCE(this_rq()->scx.local_dsq.nr);
 		goto out;
 	} else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
-		s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
+		s32 cpu = scx_cpu_ret(sch, dsq_id & SCX_DSQ_LOCAL_CPU_MASK);
 
-		if (ops_cpu_valid(sch, cpu, NULL)) {
+		if (scx_cpu_valid(sch, cpu, NULL)) {
 			ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr);
 			goto out;
 		}
@@ -9163,6 +9536,7 @@ __bpf_kfunc void scx_bpf_reenqueue_local___v2(const struct bpf_prog_aux *aux)
 
 __bpf_kfunc_end_defs();
 
+__printf(5, 0)
 static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf,
 			 size_t line_size, char *fmt, unsigned long long *data,
 			 u32 data__sz)
@@ -9200,6 +9574,7 @@ static s32 __bstr_format(struct scx_sched *sch, u64 *data_buf, char *line_buf,
 	return ret;
 }
 
+__printf(3, 0)
 static s32 bstr_format(struct scx_sched *sch, struct scx_bstr_buf *buf,
 		       char *fmt, unsigned long long *data, u32 data__sz)
 {
@@ -9220,6 +9595,7 @@ __bpf_kfunc_start_defs();
  * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops
  * disabling.
  */
+__printf(2, 0)
 __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt,
 				   unsigned long long *data, u32 data__sz,
 				   const struct bpf_prog_aux *aux)
@@ -9245,6 +9621,7 @@ __bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt,
  * Indicate that the BPF scheduler encountered a fatal error and initiate ops
  * disabling.
  */
+__printf(1, 0)
 __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data,
 				    u32 data__sz, const struct bpf_prog_aux *aux)
 {
@@ -9272,6 +9649,7 @@ __bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data,
  * The extra dump may be multiple lines. A single line may be split over
  * multiple calls. The last line is automatically terminated.
  */
+__printf(1, 0)
 __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data,
 				   u32 data__sz, const struct bpf_prog_aux *aux)
 {
@@ -9334,12 +9712,35 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu, const struct bpf_prog_aux *aux)
 	guard(rcu)();
 
 	sch = scx_prog_sched(aux);
-	if (likely(sch) && ops_cpu_valid(sch, cpu, NULL))
+	if (likely(sch) && scx_cpu_valid(sch, cpu, NULL))
 		return arch_scale_cpu_capacity(cpu);
 	else
 		return SCX_CPUPERF_ONE;
 }
 
+/**
+ * scx_bpf_cidperf_cap - Query the maximum relative capacity of the CPU at @cid
+ * @cid: cid of the CPU to query
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * cid-addressed equivalent of scx_bpf_cpuperf_cap().
+ */
+__bpf_kfunc u32 scx_bpf_cidperf_cap(s32 cid, const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+	s32 cpu;
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return SCX_CPUPERF_ONE;
+	cpu = scx_cid_to_cpu(sch, cid);
+	if (cpu < 0)
+		return SCX_CPUPERF_ONE;
+	return arch_scale_cpu_capacity(cpu);
+}
+
 /**
  * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU
  * @cpu: CPU of interest
@@ -9362,12 +9763,35 @@ __bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu, const struct bpf_prog_aux *aux)
 	guard(rcu)();
 
 	sch = scx_prog_sched(aux);
-	if (likely(sch) && ops_cpu_valid(sch, cpu, NULL))
+	if (likely(sch) && scx_cpu_valid(sch, cpu, NULL))
 		return arch_scale_freq_capacity(cpu);
 	else
 		return SCX_CPUPERF_ONE;
 }
 
+/**
+ * scx_bpf_cidperf_cur - Query the current performance of the CPU at @cid
+ * @cid: cid of the CPU to query
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * cid-addressed equivalent of scx_bpf_cpuperf_cur().
+ */
+__bpf_kfunc u32 scx_bpf_cidperf_cur(s32 cid, const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+	s32 cpu;
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return SCX_CPUPERF_ONE;
+	cpu = scx_cid_to_cpu(sch, cid);
+	if (cpu < 0)
+		return SCX_CPUPERF_ONE;
+	return arch_scale_freq_capacity(cpu);
+}
+
 /**
  * scx_bpf_cpuperf_set - Set the relative performance target of a CPU
  * @cpu: CPU of interest
@@ -9398,7 +9822,7 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au
 		return;
 	}
 
-	if (ops_cpu_valid(sch, cpu, NULL)) {
+	if (scx_cpu_valid(sch, cpu, NULL)) {
 		struct rq *rq = cpu_rq(cpu), *locked_rq = scx_locked_rq();
 		struct rq_flags rf;
 
@@ -9428,6 +9852,31 @@ __bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf, const struct bpf_prog_au
 	}
 }
 
+/**
+ * scx_bpf_cidperf_set - Set the performance target of the CPU at @cid
+ * @cid: cid of the CPU to target
+ * @perf: target performance level [0, %SCX_CPUPERF_ONE]
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * cid-addressed equivalent of scx_bpf_cpuperf_set().
+ */
+__bpf_kfunc void scx_bpf_cidperf_set(s32 cid, u32 perf,
+				     const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+	s32 cpu;
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return;
+	cpu = scx_cid_to_cpu(sch, cid);
+	if (cpu < 0)
+		return;
+	scx_bpf_cpuperf_set(cpu, perf, aux);
+}
+
 /**
  * scx_bpf_nr_node_ids - Return the number of possible node IDs
  *
@@ -9448,6 +9897,47 @@ __bpf_kfunc u32 scx_bpf_nr_cpu_ids(void)
 	return nr_cpu_ids;
 }
 
+/**
+ * scx_bpf_nr_cids - Return the size of the cid space
+ *
+ * Equals num_possible_cpus(). All valid cids are in [0, return value).
+ */
+__bpf_kfunc u32 scx_bpf_nr_cids(void)
+{
+	return num_possible_cpus();
+}
+
+/**
+ * scx_bpf_nr_online_cids - Return current count of online CPUs in cid space
+ *
+ * Return num_online_cpus(). The standard model restarts the scheduler on
+ * hotplug, which lets schedulers treat [0, nr_online_cids) as the online
+ * range. Schedulers that prefer to handle hotplug without a restart should
+ * install a custom mapping via scx_bpf_cid_override() and track onlining
+ * through the ops.cid_online / ops.cid_offline callbacks.
+ */
+__bpf_kfunc u32 scx_bpf_nr_online_cids(void)
+{
+	return num_online_cpus();
+}
+
+/**
+ * scx_bpf_this_cid - Return the cid of the CPU this program is running on
+ *
+ * cid-addressed equivalent of bpf_get_smp_processor_id() for scx programs.
+ * The current cpu is trivially valid, so this is just a table lookup. Return
+ * -EINVAL if called from a non-SCX program before any scheduler has ever
+ * been enabled (the cid table is still unallocated at that point).
+ */
+__bpf_kfunc s32 scx_bpf_this_cid(void)
+{
+	s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl);
+
+	if (!tbl)
+		return -EINVAL;
+	return tbl[raw_smp_processor_id()];
+}
+
 /**
  * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask
  */
@@ -9496,6 +9986,23 @@ __bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p)
 	return task_cpu(p);
 }
 
+/**
+ * scx_bpf_task_cid - cid a task is currently associated with
+ * @p: task of interest
+ *
+ * cid-addressed equivalent of scx_bpf_task_cpu(). task_cpu(p) is always a
+ * valid cpu, so this is just a table lookup. Return -EINVAL if called from
+ * a non-SCX program before any scheduler has ever been enabled.
+ */
+__bpf_kfunc s32 scx_bpf_task_cid(const struct task_struct *p)
+{
+	s16 *tbl = READ_ONCE(scx_cpu_to_cid_tbl);
+
+	if (!tbl)
+		return -EINVAL;
+	return tbl[task_cpu(p)];
+}
+
 /**
  * scx_bpf_cpu_rq - Fetch the rq of a CPU
  * @cpu: CPU of the rq
@@ -9511,7 +10018,7 @@ __bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu, const struct bpf_prog_aux *aux)
 	if (unlikely(!sch))
 		return NULL;
 
-	if (!ops_cpu_valid(sch, cpu, NULL))
+	if (!scx_cpu_valid(sch, cpu, NULL))
 		return NULL;
 
 	if (!sch->warned_deprecated_rq) {
@@ -9568,12 +10075,64 @@ __bpf_kfunc struct task_struct *scx_bpf_cpu_curr(s32 cpu, const struct bpf_prog_
 	if (unlikely(!sch))
 		return NULL;
 
-	if (!ops_cpu_valid(sch, cpu, NULL))
+	if (!scx_cpu_valid(sch, cpu, NULL))
 		return NULL;
 
 	return rcu_dereference(cpu_rq(cpu)->curr);
 }
 
+/**
+ * scx_bpf_cid_curr - Return the curr task on the CPU at @cid
+ * @cid: cid of interest
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * cid-addressed equivalent of scx_bpf_cpu_curr(). Callers must hold RCU
+ * read lock (KF_RCU).
+ */
+__bpf_kfunc struct task_struct *scx_bpf_cid_curr(s32 cid, const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+	s32 cpu;
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return NULL;
+	cpu = scx_cid_to_cpu(sch, cid);
+	if (cpu < 0)
+		return NULL;
+	return rcu_dereference(cpu_rq(cpu)->curr);
+}
+
+/**
+ * scx_bpf_tid_to_task - Look up a task by its scx tid
+ * @tid: task ID previously read from p->scx.tid
+ *
+ * Returns the task with the given tid, or NULL if no such task exists. The
+ * returned pointer is valid until the end of the current RCU read section
+ * (KF_RCU_PROTECTED). Requires SCX_OPS_TID_TO_TASK to be set on the root
+ * scheduler; otherwise an error is raised and NULL returned.
+ */
+__bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid)
+{
+	struct sched_ext_entity *scx;
+
+	if (!scx_tid_to_task_enabled()) {
+		struct scx_sched *sch = rcu_dereference(scx_root);
+
+		if (sch)
+			scx_error(sch, "scx_bpf_tid_to_task() called without SCX_OPS_TID_TO_TASK");
+		return NULL;
+	}
+
+	scx = rhashtable_lookup(&scx_tid_hash, &tid, scx_tid_hash_params);
+	if (!scx)
+		return NULL;
+
+	return container_of(scx, struct task_struct, scx);
+}
+
 /**
  * scx_bpf_now - Returns a high-performance monotonically non-decreasing
  * clock for the current CPU. The clock returned is in nanoseconds.
@@ -9733,6 +10292,7 @@ BTF_KFUNCS_START(scx_kfunc_ids_any)
 BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS | KF_RCU);
 BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS | KF_RCU);
 BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_kick_cid, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS | KF_RCU_PROTECTED | KF_RET_NULL)
@@ -9747,16 +10307,25 @@ BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cidperf_cap, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cidperf_cur, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cidperf_set, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_nr_node_ids)
 BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
+BTF_ID_FLAGS(func, scx_bpf_nr_cids)
+BTF_ID_FLAGS(func, scx_bpf_nr_online_cids)
+BTF_ID_FLAGS(func, scx_bpf_this_cid)
 BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
 BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
 BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
 BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_task_cid, KF_RCU)
 BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, scx_bpf_locked_rq, KF_IMPLICIT_ARGS | KF_RET_NULL)
 BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
+BTF_ID_FLAGS(func, scx_bpf_cid_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
+BTF_ID_FLAGS(func, scx_bpf_tid_to_task, KF_RET_NULL | KF_RCU_PROTECTED)
 BTF_ID_FLAGS(func, scx_bpf_now)
 BTF_ID_FLAGS(func, scx_bpf_events)
 #ifdef CONFIG_CGROUP_SCHED
@@ -9770,6 +10339,47 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = {
 	.filter			= scx_kfunc_context_filter,
 };
 
+/*
+ * cpu-form kfuncs that are forbidden from cid-form schedulers
+ * (bpf_sched_ext_ops_cid). Programs targeting the cid struct_ops type must
+ * use the cid-form alternative (cid/cmask kfuncs).
+ *
+ * Membership overlaps with scx_kfunc_ids_{any,idle,select_cpu}; the filter
+ * tests this set independently and rejects matches before the per-op
+ * allow-list check runs.
+ *
+ * pahole/resolve_btfids scans every BTF_ID_FLAGS() at build time and
+ * intersects flags across duplicate entries, so each entry must carry the
+ * same flags as the kfunc's primary declaration; otherwise the flags get
+ * dropped globally.
+ */
+BTF_KFUNCS_START(scx_kfunc_ids_cpu_only)
+BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_cpu_rq, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cpu_curr, KF_IMPLICIT_ARGS | KF_RET_NULL | KF_RCU_PROTECTED)
+BTF_ID_FLAGS(func, scx_bpf_cpu_node, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cpuperf_set, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
+BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, __scx_bpf_select_cpu_and, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_select_cpu_and, KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_IMPLICIT_ARGS | KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_IMPLICIT_ARGS | KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask_node, KF_IMPLICIT_ARGS | KF_ACQUIRE)
+BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE)
+BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu_node, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu_node, KF_IMPLICIT_ARGS | KF_RCU)
+BTF_KFUNCS_END(scx_kfunc_ids_cpu_only)
+
 /*
  * Per-op kfunc allow flags. Each bit corresponds to a context-sensitive kfunc
  * group; an op may permit zero or more groups, with the union expressed in
@@ -9779,10 +10389,11 @@ static const struct btf_kfunc_id_set scx_kfunc_set_any = {
  */
 enum scx_kf_allow_flags {
 	SCX_KF_ALLOW_UNLOCKED		= 1 << 0,
-	SCX_KF_ALLOW_CPU_RELEASE	= 1 << 1,
-	SCX_KF_ALLOW_DISPATCH		= 1 << 2,
-	SCX_KF_ALLOW_ENQUEUE		= 1 << 3,
-	SCX_KF_ALLOW_SELECT_CPU		= 1 << 4,
+	SCX_KF_ALLOW_INIT		= 1 << 1,
+	SCX_KF_ALLOW_CPU_RELEASE	= 1 << 2,
+	SCX_KF_ALLOW_DISPATCH		= 1 << 3,
+	SCX_KF_ALLOW_ENQUEUE		= 1 << 4,
+	SCX_KF_ALLOW_SELECT_CPU		= 1 << 5,
 };
 
 /*
@@ -9810,7 +10421,7 @@ static const u32 scx_kf_allow_flags[] = {
 	[SCX_OP_IDX(sub_detach)]	= SCX_KF_ALLOW_UNLOCKED,
 	[SCX_OP_IDX(cpu_online)]	= SCX_KF_ALLOW_UNLOCKED,
 	[SCX_OP_IDX(cpu_offline)]	= SCX_KF_ALLOW_UNLOCKED,
-	[SCX_OP_IDX(init)]		= SCX_KF_ALLOW_UNLOCKED,
+	[SCX_OP_IDX(init)]		= SCX_KF_ALLOW_UNLOCKED | SCX_KF_ALLOW_INIT,
 	[SCX_OP_IDX(exit)]		= SCX_KF_ALLOW_UNLOCKED,
 };
 
@@ -9825,16 +10436,18 @@ static const u32 scx_kf_allow_flags[] = {
 int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
 {
 	bool in_unlocked = btf_id_set8_contains(&scx_kfunc_ids_unlocked, kfunc_id);
+	bool in_init = btf_id_set8_contains(&scx_kfunc_ids_init, kfunc_id);
 	bool in_select_cpu = btf_id_set8_contains(&scx_kfunc_ids_select_cpu, kfunc_id);
 	bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id);
 	bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id);
 	bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id);
 	bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id);
 	bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id);
+	bool in_cpu_only = btf_id_set8_contains(&scx_kfunc_ids_cpu_only, kfunc_id);
 	u32 moff, flags;
 
 	/* Not an SCX kfunc - allow. */
-	if (!(in_unlocked || in_select_cpu || in_enqueue || in_dispatch ||
+	if (!(in_unlocked || in_init || in_select_cpu || in_enqueue || in_dispatch ||
 	      in_cpu_release || in_idle || in_any))
 		return 0;
 
@@ -9857,8 +10470,24 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
 
 	/*
 	 * Non-SCX struct_ops: SCX kfuncs are not permitted.
+	 *
+	 * Both bpf_sched_ext_ops (cpu-form) and bpf_sched_ext_ops_cid
+	 * (cid-form) are valid SCX struct_ops. Member offsets match between
+	 * the two (verified by BUILD_BUG_ON in scx_init()), so the shared
+	 * scx_kf_allow_flags[] table indexed by SCX_MOFF_IDX(moff) applies to
+	 * both.
+	 */
+	if (prog->aux->st_ops != &bpf_sched_ext_ops &&
+	    prog->aux->st_ops != &bpf_sched_ext_ops_cid)
+		return -EACCES;
+
+	/*
+	 * cid-form schedulers must use cid/cmask kfuncs. cid and cpu are both
+	 * small s32s and trivially confused, so cpu-only kfuncs are rejected at
+	 * load time. The reverse (cpu-form calling cid-form kfuncs) is
+	 * intentionally permissive to ease gradual cpumask -> cid migration.
 	 */
-	if (prog->aux->st_ops != &bpf_sched_ext_ops)
+	if (prog->aux->st_ops == &bpf_sched_ext_ops_cid && in_cpu_only)
 		return -EACCES;
 
 	/* SCX struct_ops: check the per-op allow list. */
@@ -9870,6 +10499,8 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
 
 	if ((flags & SCX_KF_ALLOW_UNLOCKED) && in_unlocked)
 		return 0;
+	if ((flags & SCX_KF_ALLOW_INIT) && in_init)
+		return 0;
 	if ((flags & SCX_KF_ALLOW_CPU_RELEASE) && in_cpu_release)
 		return 0;
 	if ((flags & SCX_KF_ALLOW_DISPATCH) && in_dispatch)
@@ -9886,6 +10517,73 @@ static int __init scx_init(void)
 {
 	int ret;
 
+	/*
+	 * sched_ext_ops_cid mirrors sched_ext_ops up to and including @priv.
+	 * Both bpf_scx_init_member() and bpf_scx_check_member() use offsets
+	 * from struct sched_ext_ops; sched_ext_ops_cid relies on those offsets
+	 * matching for the shared fields. Catch any drift at boot.
+	 */
+#define CID_OFFSET_MATCH(cpu_field, cid_field)					\
+	BUILD_BUG_ON(offsetof(struct sched_ext_ops, cpu_field) !=		\
+		     offsetof(struct sched_ext_ops_cid, cid_field))
+	/* data fields used by bpf_scx_init_member() */
+	CID_OFFSET_MATCH(dispatch_max_batch, dispatch_max_batch);
+	CID_OFFSET_MATCH(flags, flags);
+	CID_OFFSET_MATCH(name, name);
+	CID_OFFSET_MATCH(timeout_ms, timeout_ms);
+	CID_OFFSET_MATCH(exit_dump_len, exit_dump_len);
+	CID_OFFSET_MATCH(hotplug_seq, hotplug_seq);
+	CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id);
+	/* shared callbacks: the union view requires byte-for-byte offset match */
+	CID_OFFSET_MATCH(enqueue, enqueue);
+	CID_OFFSET_MATCH(dequeue, dequeue);
+	CID_OFFSET_MATCH(dispatch, dispatch);
+	CID_OFFSET_MATCH(tick, tick);
+	CID_OFFSET_MATCH(runnable, runnable);
+	CID_OFFSET_MATCH(running, running);
+	CID_OFFSET_MATCH(stopping, stopping);
+	CID_OFFSET_MATCH(quiescent, quiescent);
+	CID_OFFSET_MATCH(yield, yield);
+	CID_OFFSET_MATCH(core_sched_before, core_sched_before);
+	CID_OFFSET_MATCH(set_weight, set_weight);
+	CID_OFFSET_MATCH(update_idle, update_idle);
+	CID_OFFSET_MATCH(init_task, init_task);
+	CID_OFFSET_MATCH(exit_task, exit_task);
+	CID_OFFSET_MATCH(enable, enable);
+	CID_OFFSET_MATCH(disable, disable);
+	CID_OFFSET_MATCH(dump, dump);
+	CID_OFFSET_MATCH(dump_task, dump_task);
+	CID_OFFSET_MATCH(sub_attach, sub_attach);
+	CID_OFFSET_MATCH(sub_detach, sub_detach);
+	CID_OFFSET_MATCH(init, init);
+	CID_OFFSET_MATCH(exit, exit);
+#ifdef CONFIG_EXT_GROUP_SCHED
+	CID_OFFSET_MATCH(cgroup_init, cgroup_init);
+	CID_OFFSET_MATCH(cgroup_exit, cgroup_exit);
+	CID_OFFSET_MATCH(cgroup_prep_move, cgroup_prep_move);
+	CID_OFFSET_MATCH(cgroup_move, cgroup_move);
+	CID_OFFSET_MATCH(cgroup_cancel_move, cgroup_cancel_move);
+	CID_OFFSET_MATCH(cgroup_set_weight, cgroup_set_weight);
+	CID_OFFSET_MATCH(cgroup_set_bandwidth, cgroup_set_bandwidth);
+	CID_OFFSET_MATCH(cgroup_set_idle, cgroup_set_idle);
+#endif
+	/* renamed callbacks must occupy the same slot as their cpu-form sibling */
+	CID_OFFSET_MATCH(select_cpu, select_cid);
+	CID_OFFSET_MATCH(set_cpumask, set_cmask);
+	CID_OFFSET_MATCH(cpu_online, cid_online);
+	CID_OFFSET_MATCH(cpu_offline, cid_offline);
+	CID_OFFSET_MATCH(dump_cpu, dump_cid);
+	/* @priv tail must align since both share the same data block */
+	CID_OFFSET_MATCH(priv, priv);
+	/*
+	 * cid-form must end exactly at @priv - validate_ops() skips
+	 * cpu_acquire/cpu_release for cid-form because reading those fields
+	 * past the BPF allocation would be UB.
+	 */
+	BUILD_BUG_ON(offsetof(struct sched_ext_ops_cid, __end) !=
+		     offsetofend(struct sched_ext_ops, priv));
+#undef CID_OFFSET_MATCH
+
 	/*
 	 * kfunc registration can't be done from init_sched_ext_class() as
 	 * register_btf_kfunc_id_set() needs most of the system to be up.
@@ -9924,12 +10622,24 @@ static int __init scx_init(void)
 		return ret;
 	}
 
+	ret = scx_cid_kfunc_init();
+	if (ret) {
+		pr_err("sched_ext: Failed to register cid kfuncs (%d)\n", ret);
+		return ret;
+	}
+
 	ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops);
 	if (ret) {
 		pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret);
 		return ret;
 	}
 
+	ret = register_bpf_struct_ops(&bpf_sched_ext_ops_cid, sched_ext_ops_cid);
+	if (ret) {
+		pr_err("sched_ext: Failed to register cid struct_ops (%d)\n", ret);
+		return ret;
+	}
+
 	ret = register_pm_notifier(&scx_pm_notifier);
 	if (ret) {
 		pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret);
diff --git a/kernel/sched/ext_cid.c b/kernel/sched/ext_cid.c
new file mode 100644
index 0000000000000..0c91b951fd33c
--- /dev/null
+++ b/kernel/sched/ext_cid.c
@@ -0,0 +1,725 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2026 Tejun Heo <tj@kernel.org>
+ */
+#include <linux/cacheinfo.h>
+
+/*
+ * Per-cpu scratch cmask used by scx_call_op_set_cpumask() to synthesize a
+ * cmask from a cpumask. Allocated alongside the cid arrays on first enable
+ * and never freed. Sized to the full cid space. Caller holds rq lock so
+ * this_cpu_ptr is safe.
+ */
+struct scx_cmask __percpu *scx_set_cmask_scratch;
+
+/*
+ * cid tables.
+ *
+ * Pointers are published once on first enable and never revoked. The default
+ * mapping is populated before ops.init() runs; scx_bpf_cid_override() commits
+ * before it returns. As long as the BPF scheduler only uses the tables from
+ * those points onward, it sees a consistent view.
+ */
+s16 *scx_cid_to_cpu_tbl;
+s16 *scx_cpu_to_cid_tbl;
+struct scx_cid_topo *scx_cid_topo;
+
+#define SCX_CID_TOPO_NEG	(struct scx_cid_topo) {				\
+	.core_cid = -1, .core_idx = -1, .llc_cid = -1, .llc_idx = -1,		\
+	.node_cid = -1, .node_idx = -1,						\
+}
+
+/*
+ * Return @cpu's LLC shared_cpu_map. If cacheinfo isn't populated (offline or
+ * !present), record @cpu in @fallbacks and return its node mask instead - the
+ * worst that can happen is that the cpu's LLC becomes coarser than reality.
+ */
+static const struct cpumask *cpu_llc_mask(int cpu, struct cpumask *fallbacks)
+{
+	struct cpu_cacheinfo *ci = get_cpu_cacheinfo(cpu);
+
+	if (!ci || !ci->info_list || !ci->num_leaves) {
+		cpumask_set_cpu(cpu, fallbacks);
+		return cpumask_of_node(cpu_to_node(cpu));
+	}
+	return &ci->info_list[ci->num_leaves - 1].shared_cpu_map;
+}
+
+/* Allocate the cid tables once on first enable; never freed. */
+static s32 scx_cid_arrays_alloc(void)
+{
+	u32 npossible = num_possible_cpus();
+	s16 *cid_to_cpu, *cpu_to_cid;
+	struct scx_cid_topo *cid_topo;
+	struct scx_cmask __percpu *set_cmask_scratch;
+	s32 cpu;
+
+	if (scx_cid_to_cpu_tbl)
+		return 0;
+
+	cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL);
+	cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL);
+	cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL);
+	set_cmask_scratch = __alloc_percpu(struct_size(set_cmask_scratch, bits,
+						       SCX_CMASK_NR_WORDS(npossible)),
+					   sizeof(u64));
+
+	if (!cid_to_cpu || !cpu_to_cid || !cid_topo || !set_cmask_scratch) {
+		kfree(cid_to_cpu);
+		kfree(cpu_to_cid);
+		kfree(cid_topo);
+		free_percpu(set_cmask_scratch);
+		return -ENOMEM;
+	}
+
+	WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu);
+	WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid);
+	WRITE_ONCE(scx_cid_topo, cid_topo);
+	for_each_possible_cpu(cpu)
+		scx_cmask_init(per_cpu_ptr(set_cmask_scratch, cpu),
+			       0, npossible);
+	WRITE_ONCE(scx_set_cmask_scratch, set_cmask_scratch);
+	return 0;
+}
+
+/**
+ * scx_cid_init - build the cid mapping
+ * @sch: the scx_sched being initialized; used as the scx_error() target
+ *
+ * See "Topological CPU IDs" in ext_cid.h for the model. Walk online cpus by
+ * intersection at each level (parent_scratch & this_level_mask), which keeps
+ * containment correct by construction and naturally splits a physical LLC
+ * straddling two NUMA nodes into two LLC units. The caller must hold
+ * cpus_read_lock.
+ */
+s32 scx_cid_init(struct scx_sched *sch)
+{
+	cpumask_var_t to_walk __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+	cpumask_var_t node_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+	cpumask_var_t llc_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+	cpumask_var_t core_scratch __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+	cpumask_var_t llc_fallback __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+	cpumask_var_t online_no_topo __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+	u32 next_cid = 0;
+	s32 next_node_idx = 0, next_llc_idx = 0, next_core_idx = 0;
+	s32 cpu, ret;
+
+	/* CMASK_MAX_WORDS in cid.bpf.h covers NR_CPUS up to 8192 */
+	BUILD_BUG_ON(NR_CPUS > 8192);
+
+	lockdep_assert_cpus_held();
+
+	ret = scx_cid_arrays_alloc();
+	if (ret)
+		return ret;
+
+	if (!zalloc_cpumask_var(&to_walk, GFP_KERNEL) ||
+	    !zalloc_cpumask_var(&node_scratch, GFP_KERNEL) ||
+	    !zalloc_cpumask_var(&llc_scratch, GFP_KERNEL) ||
+	    !zalloc_cpumask_var(&core_scratch, GFP_KERNEL) ||
+	    !zalloc_cpumask_var(&llc_fallback, GFP_KERNEL) ||
+	    !zalloc_cpumask_var(&online_no_topo, GFP_KERNEL))
+		return -ENOMEM;
+
+	/* -1 sentinels for sparse-possible cpu id holes (0 is a valid cid) */
+	for (cpu = 0; cpu < nr_cpu_ids; cpu++)
+		scx_cpu_to_cid_tbl[cpu] = -1;
+
+	cpumask_copy(to_walk, cpu_online_mask);
+
+	while (!cpumask_empty(to_walk)) {
+		s32 next_cpu = cpumask_first(to_walk);
+		s32 nid = cpu_to_node(next_cpu);
+		s32 node_cid = next_cid;
+		s32 node_idx;
+
+		/*
+		 * No NUMA info: skip and let the tail loop assign a no-topo
+		 * cid. cpumask_of_node(-1) is undefined.
+		 */
+		if (nid < 0) {
+			cpumask_clear_cpu(next_cpu, to_walk);
+			continue;
+		}
+
+		node_idx = next_node_idx++;
+
+		/* node_scratch = to_walk & this node */
+		cpumask_and(node_scratch, to_walk, cpumask_of_node(nid));
+		if (WARN_ON_ONCE(!cpumask_test_cpu(next_cpu, node_scratch)))
+			return -EINVAL;
+
+		while (!cpumask_empty(node_scratch)) {
+			s32 ncpu = cpumask_first(node_scratch);
+			const struct cpumask *llc_mask = cpu_llc_mask(ncpu, llc_fallback);
+			s32 llc_cid = next_cid;
+			s32 llc_idx = next_llc_idx++;
+
+			/* llc_scratch = node_scratch & this llc */
+			cpumask_and(llc_scratch, node_scratch, llc_mask);
+			if (WARN_ON_ONCE(!cpumask_test_cpu(ncpu, llc_scratch)))
+				return -EINVAL;
+
+			while (!cpumask_empty(llc_scratch)) {
+				s32 lcpu = cpumask_first(llc_scratch);
+				const struct cpumask *sib = topology_sibling_cpumask(lcpu);
+				s32 core_cid = next_cid;
+				s32 core_idx = next_core_idx++;
+				s32 ccpu;
+
+				/* core_scratch = llc_scratch & this core */
+				cpumask_and(core_scratch, llc_scratch, sib);
+				if (WARN_ON_ONCE(!cpumask_test_cpu(lcpu, core_scratch)))
+					return -EINVAL;
+
+				for_each_cpu(ccpu, core_scratch) {
+					s32 cid = next_cid++;
+
+					scx_cid_to_cpu_tbl[cid] = ccpu;
+					scx_cpu_to_cid_tbl[ccpu] = cid;
+					scx_cid_topo[cid] = (struct scx_cid_topo){
+						.core_cid = core_cid,
+						.core_idx = core_idx,
+						.llc_cid = llc_cid,
+						.llc_idx = llc_idx,
+						.node_cid = node_cid,
+						.node_idx = node_idx,
+					};
+
+					cpumask_clear_cpu(ccpu, llc_scratch);
+					cpumask_clear_cpu(ccpu, node_scratch);
+					cpumask_clear_cpu(ccpu, to_walk);
+				}
+			}
+		}
+	}
+
+	/*
+	 * No-topo section: any possible cpu without a cid - normally just the
+	 * not-online ones. Collect any currently-online cpus that land here in
+	 * @online_no_topo so we can warn about them at the end.
+	 */
+	for_each_cpu(cpu, cpu_possible_mask) {
+		s32 cid;
+
+		if (__scx_cpu_to_cid(cpu) != -1)
+			continue;
+		if (cpu_online(cpu))
+			cpumask_set_cpu(cpu, online_no_topo);
+
+		cid = next_cid++;
+		scx_cid_to_cpu_tbl[cid] = cpu;
+		scx_cpu_to_cid_tbl[cpu] = cid;
+		scx_cid_topo[cid] = SCX_CID_TOPO_NEG;
+	}
+
+	if (!cpumask_empty(llc_fallback))
+		pr_warn("scx_cid: cpus without cacheinfo, using node mask as llc: %*pbl\n",
+			cpumask_pr_args(llc_fallback));
+	if (!cpumask_empty(online_no_topo))
+		pr_warn("scx_cid: online cpus with no usable topology: %*pbl\n",
+			cpumask_pr_args(online_no_topo));
+
+	return 0;
+}
+
+/**
+ * scx_cmask_clear - Zero every bit in @m's active range
+ * @m: cmask to clear
+ *
+ * Storage past the active range is left as is.
+ */
+void scx_cmask_clear(struct scx_cmask *m)
+{
+	u32 nr_words;
+
+	if (!m->nr_cids)
+		return;
+	nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
+	memset(m->bits, 0, nr_words * sizeof(u64));
+}
+
+/**
+ * scx_cmask_fill - Set every bit in @m's active range
+ * @m: cmask to fill
+ *
+ * Counterpart to scx_cmask_clear(). Storage past the active range is left as is.
+ */
+void scx_cmask_fill(struct scx_cmask *m)
+{
+	u32 nr_words, head_bits, tail_bits;
+
+	if (!m->nr_cids)
+		return;
+	nr_words = (m->base + m->nr_cids - 1) / 64 - m->base / 64 + 1;
+	memset(m->bits, 0xff, nr_words * sizeof(u64));
+
+	/* clear word-0 bits below base */
+	head_bits = m->base & 63;
+	if (head_bits)
+		m->bits[0] &= ~((1ULL << head_bits) - 1);
+
+	/* clear last-word bits at or past base + nr_cids */
+	tail_bits = (m->base + m->nr_cids) & 63;
+	if (tail_bits)
+		m->bits[nr_words - 1] &= (1ULL << tail_bits) - 1;
+}
+
+/**
+ * scx_cpumask_to_cmask - Translate a kernel cpumask into a cmask
+ * @src: source cpumask
+ * @dst: cmask to write
+ *
+ * Clear @dst's active range and set the bit for each cid whose cpu is in
+ * @src and lies within that range. Out-of-range cids are silently ignored.
+ */
+void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst)
+{
+	s32 cpu;
+
+	scx_cmask_clear(dst);
+	for_each_cpu(cpu, src) {
+		s32 cid = __scx_cpu_to_cid(cpu);
+
+		if (cid >= 0)
+			__scx_cmask_set(dst, cid);
+	}
+}
+
+__bpf_kfunc_start_defs();
+
+/**
+ * scx_bpf_cid_override - Install an explicit cpu->cid mapping
+ * @cpu_to_cid: array of nr_cpu_ids s32 entries (cid for each cpu)
+ * @cpu_to_cid__sz: must be nr_cpu_ids * sizeof(s32) bytes
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * May only be called from ops.init() of the root scheduler. Replace the
+ * topology-probed cid mapping with the caller-provided one. Each possible cpu
+ * must map to a unique cid in [0, num_possible_cpus()). Topo info is cleared.
+ * On invalid input, trigger scx_error() to abort the scheduler.
+ */
+__bpf_kfunc void scx_bpf_cid_override(const s32 *cpu_to_cid, u32 cpu_to_cid__sz,
+				      const struct bpf_prog_aux *aux)
+{
+	cpumask_var_t seen __free(free_cpumask_var) = CPUMASK_VAR_NULL;
+	struct scx_sched *sch;
+	bool alloced;
+	s32 cpu, cid;
+
+	/* GFP_KERNEL alloc must happen before the rcu read section */
+	alloced = zalloc_cpumask_var(&seen, GFP_KERNEL);
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return;
+
+	if (!alloced) {
+		scx_error(sch, "scx_bpf_cid_override: failed to allocate cpumask");
+		return;
+	}
+
+	if (scx_parent(sch)) {
+		scx_error(sch, "scx_bpf_cid_override() only allowed from root sched");
+		return;
+	}
+
+	if (cpu_to_cid__sz != nr_cpu_ids * sizeof(s32)) {
+		scx_error(sch, "scx_bpf_cid_override: expected %zu bytes, got %u",
+			  nr_cpu_ids * sizeof(s32), cpu_to_cid__sz);
+		return;
+	}
+
+	for_each_possible_cpu(cpu) {
+		s32 c = cpu_to_cid[cpu];
+
+		if (!cid_valid(sch, c))
+			return;
+		if (cpumask_test_and_set_cpu(c, seen)) {
+			scx_error(sch, "cid %d assigned to multiple cpus", c);
+			return;
+		}
+		scx_cpu_to_cid_tbl[cpu] = c;
+		scx_cid_to_cpu_tbl[c] = cpu;
+	}
+
+	/* Invalidate stale topo info - the override carries no topology. */
+	for (cid = 0; cid < num_possible_cpus(); cid++)
+		scx_cid_topo[cid] = SCX_CID_TOPO_NEG;
+}
+
+/**
+ * scx_bpf_cid_to_cpu - Return the raw CPU id for @cid
+ * @cid: cid to look up
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Return the raw CPU id for @cid. Trigger scx_error() and return -EINVAL if
+ * @cid is invalid. The cid<->cpu mapping is static for the lifetime of the
+ * loaded scheduler, so the BPF side can cache the result to avoid repeated
+ * kfunc invocations.
+ */
+__bpf_kfunc s32 scx_bpf_cid_to_cpu(s32 cid, const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return -EINVAL;
+	return scx_cid_to_cpu(sch, cid);
+}
+
+/**
+ * scx_bpf_cpu_to_cid - Return the cid for @cpu
+ * @cpu: cpu to look up
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Return the cid for @cpu. Trigger scx_error() and return -EINVAL if @cpu is
+ * invalid. The cid<->cpu mapping is static for the lifetime of the loaded
+ * scheduler, so the BPF side can cache the result to avoid repeated kfunc
+ * invocations.
+ */
+__bpf_kfunc s32 scx_bpf_cpu_to_cid(s32 cpu, const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch))
+		return -EINVAL;
+	return scx_cpu_to_cid(sch, cpu);
+}
+
+/*
+ * Set ops on cmasks. cmask_walk_op2() shares one walk across mutating
+ * (and/or/copy/andnot) and predicate (subset/intersects) two-cmask forms;
+ * cmask_walk_op1() does the same shape over a single cmask range. Every public
+ * entry passes a compile-time-constant @op; cmask_walk_op{1,2}() and
+ * cmask_word_op{1,2}() are __always_inline so the inner switch collapses to the
+ * selected op and cmask_op2_is_pred() folds the predicate early-exit out of
+ * mutating ops.
+ *
+ * Two-cmask ops only touch @dst bits inside the intersection of the two ranges;
+ * bits outside stay untouched. In particular, scx_cmask_copy() does NOT zero
+ * @dst bits that lie outside @src's range.
+ *
+ * The _RACY variants are otherwise identical to their non-racy counterpart but
+ * read @src word-by-word via data_race(). Memory ordering with concurrent
+ * writers is the caller's responsibility.
+ */
+enum cmask_op2 {
+	/* mutating */
+	CMASK_OP2_AND,
+	CMASK_OP2_OR,
+	CMASK_OP2_OR_RACY,
+	CMASK_OP2_COPY,
+	CMASK_OP2_COPY_RACY,
+	CMASK_OP2_ANDNOT,
+	/* predicates - short-circuit when the per-word result is true */
+	CMASK_OP2_SUBSET,
+	CMASK_OP2_INTERSECTS,
+};
+
+static __always_inline bool cmask_op2_is_pred(const enum cmask_op2 op)
+{
+	return op == CMASK_OP2_SUBSET || op == CMASK_OP2_INTERSECTS;
+}
+
+static __always_inline bool cmask_word_op2(u64 *av, const u64 *bp, u64 mask,
+					   const enum cmask_op2 op)
+{
+	switch (op) {
+	case CMASK_OP2_AND:
+		*av &= ~mask | *bp;
+		return false;
+	case CMASK_OP2_OR:
+		*av |= *bp & mask;
+		return false;
+	case CMASK_OP2_OR_RACY:
+		*av |= data_race(*bp) & mask;
+		return false;
+	case CMASK_OP2_COPY:
+		*av = (*av & ~mask) | (*bp & mask);
+		return false;
+	case CMASK_OP2_COPY_RACY:
+		*av = (*av & ~mask) | (data_race(*bp) & mask);
+		return false;
+	case CMASK_OP2_ANDNOT:
+		*av &= ~(*bp & mask);
+		return false;
+	case CMASK_OP2_SUBSET:
+		/* stop on the first bit in @sub not set in @super */
+		return (*bp & ~*av) & mask;
+	case CMASK_OP2_INTERSECTS:
+		return (*av & *bp) & mask;
+	}
+	unreachable();
+}
+
+/*
+ * Walk the intersection of [@a_base, @a_base + @a_nr_cids) with [@b_base,
+ * @b_base + @b_nr_cids) word by word, applying @op. Mutating ops walk all words
+ * and return false; predicates return true on the first word whose per-word
+ * test is true. Empty intersection returns false (matches "no bits to consider"
+ * for both mutate and predicate).
+ *
+ * Base/nr_cids are taken as parameters so callers with snapshotted bounds can
+ * drive the walk with values independent of the cmask's header.
+ */
+static __always_inline bool cmask_walk_op2(u64 *a_bits, u32 a_base, u32 a_nr_cids,
+					   const u64 *b_bits, u32 b_base, u32 b_nr_cids,
+					   const enum cmask_op2 op)
+{
+	u32 lo = max(a_base, b_base);
+	u32 hi = min(a_base + a_nr_cids, b_base + b_nr_cids);
+	u32 a_word_off = a_base / 64;
+	u32 b_word_off = b_base / 64;
+	u32 lo_word = lo / 64;
+	u32 hi_word = (hi - 1) / 64;
+	u64 head_mask = GENMASK_U64(63, lo & 63);
+	u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0);
+	u32 w;
+
+	if (lo >= hi)
+		return false;
+
+	if (lo_word == hi_word)
+		return cmask_word_op2(&a_bits[lo_word - a_word_off],
+				      &b_bits[lo_word - b_word_off],
+				      head_mask & tail_mask, op);
+
+	if (cmask_word_op2(&a_bits[lo_word - a_word_off],
+			   &b_bits[lo_word - b_word_off], head_mask, op) &&
+	    cmask_op2_is_pred(op))
+		return true;
+
+	for (w = lo_word + 1; w < hi_word; w++)
+		if (cmask_word_op2(&a_bits[w - a_word_off],
+				   &b_bits[w - b_word_off], ~0ULL, op) &&
+		    cmask_op2_is_pred(op))
+			return true;
+
+	return cmask_word_op2(&a_bits[hi_word - a_word_off],
+			      &b_bits[hi_word - b_word_off], tail_mask, op);
+}
+
+enum cmask_op1 {
+	CMASK_OP1_ANY_SET,
+};
+
+static __always_inline bool cmask_word_op1(const u64 *ap, u64 mask,
+					   const enum cmask_op1 op)
+{
+	switch (op) {
+	case CMASK_OP1_ANY_SET:
+		return *ap & mask;
+	}
+	unreachable();
+}
+
+/*
+ * Walk [@a_base, @a_base + @a_nr_cids) of @a_bits word by word, applying @op.
+ * Returns true on the first word whose per-word test is true; returns false if
+ * no word matches or the range is empty. All current op1s short-circuit on
+ * per-word true; if a non-predicate op1 lands here, add a cmask_op1_is_pred()
+ * guard analogous to cmask_op2_is_pred().
+ */
+static __always_inline bool cmask_walk_op1(const u64 *a_bits, u32 a_base,
+					   u32 a_nr_cids,
+					   const enum cmask_op1 op)
+{
+	u32 lo = a_base;
+	u32 hi = a_base + a_nr_cids;
+	u32 a_word_off = a_base / 64;
+	u32 lo_word = lo / 64;
+	u32 hi_word = (hi - 1) / 64;
+	u64 head_mask = GENMASK_U64(63, lo & 63);
+	u64 tail_mask = GENMASK_U64((hi - 1) & 63, 0);
+	u32 w;
+
+	if (lo >= hi)
+		return false;
+
+	if (lo_word == hi_word)
+		return cmask_word_op1(&a_bits[lo_word - a_word_off],
+				      head_mask & tail_mask, op);
+
+	if (cmask_word_op1(&a_bits[lo_word - a_word_off], head_mask, op))
+		return true;
+	for (w = lo_word + 1; w < hi_word; w++)
+		if (cmask_word_op1(&a_bits[w - a_word_off], ~0ULL, op))
+			return true;
+	return cmask_word_op1(&a_bits[hi_word - a_word_off], tail_mask, op);
+}
+
+void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src)
+{
+	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
+		       src->bits, src->base, src->nr_cids, CMASK_OP2_AND);
+}
+
+void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src)
+{
+	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
+		       src->bits, src->base, src->nr_cids, CMASK_OP2_OR);
+}
+
+/**
+ * scx_cmask_or_racy - OR @src into @dst, reading @src without locking
+ *
+ * @src is read word-by-word through data_race(). Same per-bit independence
+ * rationale as scx_cmask_copy_racy(). Memory ordering with writers is the
+ * caller's responsibility.
+ */
+void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src)
+{
+	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
+		       src->bits, src->base, src->nr_cids, CMASK_OP2_OR_RACY);
+}
+
+void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src)
+{
+	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
+		       src->bits, src->base, src->nr_cids, CMASK_OP2_COPY);
+}
+
+/**
+ * scx_cmask_copy_racy - Snapshot @src into @dst without locking
+ *
+ * @src is read word-by-word through data_race(). Head/tail masking matches
+ * scx_cmask_copy(). Each bit in a cmask is independent, so partial updates
+ * just leave some bits fresher than others. Memory ordering with writers is
+ * the caller's responsibility.
+ */
+void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src)
+{
+	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
+		       src->bits, src->base, src->nr_cids, CMASK_OP2_COPY_RACY);
+}
+
+void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src)
+{
+	cmask_walk_op2(dst->bits, dst->base, dst->nr_cids,
+		       src->bits, src->base, src->nr_cids, CMASK_OP2_ANDNOT);
+}
+
+/*
+ * Return true if @cm has any bit set in [@lo, @hi). Caller must ensure
+ * [@lo, @hi) is contained in @cm's range.
+ */
+static bool cmask_any_set_in_range(const struct scx_cmask *cm, u32 lo, u32 hi)
+{
+	if (lo >= hi)
+		return false;
+	return cmask_walk_op1(&cm->bits[lo / 64 - cm->base / 64], lo, hi - lo,
+			      CMASK_OP1_ANY_SET);
+}
+
+/**
+ * scx_cmask_subset - test whether @sub is a subset of @super
+ * @sub: cmask to test
+ * @super: cmask to test against
+ *
+ * Return true iff every set bit of @sub is also set in @super.
+ */
+bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super)
+{
+	u32 super_end = super->base + super->nr_cids;
+	u32 sub_end = sub->base + sub->nr_cids;
+
+	/*
+	 * Set bits in @sub outside @super's range can't be in @super, so any
+	 * such bit means not a subset. The walk below only visits words
+	 * common to both ranges, so these need a separate scan.
+	 */
+	if (sub->base < super->base &&
+	    cmask_any_set_in_range(sub, sub->base, min(super->base, sub_end)))
+		return false;
+	if (sub_end > super_end &&
+	    cmask_any_set_in_range(sub, max(sub->base, super_end), sub_end))
+		return false;
+
+	return !cmask_walk_op2((u64 *)super->bits, super->base, super->nr_cids,
+			       sub->bits, sub->base, sub->nr_cids, CMASK_OP2_SUBSET);
+}
+
+bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b)
+{
+	return cmask_walk_op2((u64 *)a->bits, a->base, a->nr_cids,
+			      b->bits, b->base, b->nr_cids, CMASK_OP2_INTERSECTS);
+}
+
+/**
+ * scx_cmask_empty - Test whether @m has no bits set
+ * @m: cmask to test
+ *
+ * Return true iff @m's active range has no bits set.
+ */
+bool scx_cmask_empty(const struct scx_cmask *m)
+{
+	return !cmask_any_set_in_range(m, m->base, m->base + m->nr_cids);
+}
+
+/**
+ * scx_bpf_cid_topo - Copy out per-cid topology info
+ * @cid: cid to look up
+ * @out__uninit: where to copy the topology info; fully written by this call
+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
+ *
+ * Fill @out__uninit with the topology info for @cid. Trigger scx_error() if
+ * @cid is out of range. If @cid is valid but in the no-topo section, all fields
+ * are set to -1.
+ */
+__bpf_kfunc void scx_bpf_cid_topo(s32 cid, struct scx_cid_topo *out__uninit,
+				  const struct bpf_prog_aux *aux)
+{
+	struct scx_sched *sch;
+
+	guard(rcu)();
+
+	sch = scx_prog_sched(aux);
+	if (unlikely(!sch) || !cid_valid(sch, cid)) {
+		*out__uninit = SCX_CID_TOPO_NEG;
+		return;
+	}
+
+	*out__uninit = READ_ONCE(scx_cid_topo)[cid];
+}
+
+__bpf_kfunc_end_defs();
+
+BTF_KFUNCS_START(scx_kfunc_ids_init)
+BTF_ID_FLAGS(func, scx_bpf_cid_override, KF_IMPLICIT_ARGS | KF_SLEEPABLE)
+BTF_KFUNCS_END(scx_kfunc_ids_init)
+
+static const struct btf_kfunc_id_set scx_kfunc_set_init = {
+	.owner	= THIS_MODULE,
+	.set	= &scx_kfunc_ids_init,
+	.filter	= scx_kfunc_context_filter,
+};
+
+BTF_KFUNCS_START(scx_kfunc_ids_cid)
+BTF_ID_FLAGS(func, scx_bpf_cid_to_cpu, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cpu_to_cid, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, scx_bpf_cid_topo, KF_IMPLICIT_ARGS)
+BTF_KFUNCS_END(scx_kfunc_ids_cid)
+
+static const struct btf_kfunc_id_set scx_kfunc_set_cid = {
+	.owner	= THIS_MODULE,
+	.set	= &scx_kfunc_ids_cid,
+};
+
+int scx_cid_kfunc_init(void)
+{
+	return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init) ?:
+		register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_cid) ?:
+		register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING, &scx_kfunc_set_cid) ?:
+		register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &scx_kfunc_set_cid);
+}
diff --git a/kernel/sched/ext_cid.h b/kernel/sched/ext_cid.h
new file mode 100644
index 0000000000000..abea22ba2cc25
--- /dev/null
+++ b/kernel/sched/ext_cid.h
@@ -0,0 +1,228 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Topological CPU IDs (cids)
+ * --------------------------
+ *
+ * Raw cpu numbers are clumsy for sharding work and communication across
+ * topology units, especially from BPF: the space can be sparse, numerical
+ * closeness doesn't imply topological closeness (x86 hyperthreading often puts
+ * SMT siblings far apart), and a range of cpu ids doesn't mean anything.
+ * Sub-scheds make this acute - cpu allocation, revocation and other state are
+ * constantly communicated across sub-scheds, and passing whole cpumasks scales
+ * poorly with cpu count. cpumasks are also awkward in BPF: a variable-length
+ * kernel type sized for the maximum NR_CPUS (4k), with verbose helper sequences
+ * for every op.
+ *
+ * cids give every cpu a dense, topology-ordered id. CPUs sharing a core, LLC or
+ * NUMA node get contiguous cid ranges, so a topology unit becomes a (start,
+ * length) slice of cid space. Communication can pass a slice instead of a
+ * cpumask, and BPF code can process, for example, a u64 word's worth of cids at
+ * a time.
+ *
+ * The mapping is built once at root scheduler enable time by walking the
+ * topology of online cpus only. Going by online cpus is out of necessity:
+ * depending on the arch, topology info isn't reliably available for offline
+ * cpus. The expected usage model is restarting the scheduler on hotplug events
+ * so the mapping is rebuilt against the new online set. A scheduler that wants
+ * to handle hotplug without a restart can provide its own cid and shard mapping
+ * through the override interface.
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2026 Tejun Heo <tj@kernel.org>
+ */
+#ifndef _KERNEL_SCHED_EXT_CID_H
+#define _KERNEL_SCHED_EXT_CID_H
+
+struct scx_sched;
+
+/*
+ * Cid space (total is always num_possible_cpus()) is laid out with
+ * topology-annotated cids first, then no-topo cids at the tail. The
+ * topology-annotated block covers the cpus that were online when scx_cid_init()
+ * ran and remains valid even after those cpus go offline. The tail block covers
+ * possible-but-not-online cpus and carries all-(-1) topo info (see
+ * scx_cid_topo); callers detect it via the -1 sentinels.
+ *
+ * See the comment above the table definitions in ext_cid.c for the
+ * memory-ordering and visibility contract.
+ */
+extern s16 *scx_cid_to_cpu_tbl;
+extern s16 *scx_cpu_to_cid_tbl;
+extern struct scx_cid_topo *scx_cid_topo;
+extern struct btf_id_set8 scx_kfunc_ids_init;
+
+void scx_cmask_clear(struct scx_cmask *m);
+void scx_cmask_fill(struct scx_cmask *m);
+void scx_cmask_and(struct scx_cmask *dst, const struct scx_cmask *src);
+void scx_cmask_or(struct scx_cmask *dst, const struct scx_cmask *src);
+void scx_cmask_or_racy(struct scx_cmask *dst, const struct scx_cmask *src);
+void scx_cmask_copy(struct scx_cmask *dst, const struct scx_cmask *src);
+void scx_cmask_copy_racy(struct scx_cmask *dst, const struct scx_cmask *src);
+void scx_cmask_andnot(struct scx_cmask *dst, const struct scx_cmask *src);
+bool scx_cmask_subset(const struct scx_cmask *sub, const struct scx_cmask *super);
+bool scx_cmask_intersects(const struct scx_cmask *a, const struct scx_cmask *b);
+bool scx_cmask_empty(const struct scx_cmask *m);
+s32 scx_cid_init(struct scx_sched *sch);
+int scx_cid_kfunc_init(void);
+void scx_cpumask_to_cmask(const struct cpumask *src, struct scx_cmask *dst);
+
+/**
+ * cid_valid - Verify a cid value, to be used on ops input args
+ * @sch: scx_sched to abort on error
+ * @cid: cid which came from a BPF ops
+ *
+ * Return true if @cid is in [0, num_possible_cpus()). On failure, trigger
+ * scx_error() and return false.
+ */
+static inline bool cid_valid(struct scx_sched *sch, s32 cid)
+{
+	if (likely(cid >= 0 && cid < num_possible_cpus()))
+		return true;
+	scx_error(sch, "invalid cid %d", cid);
+	return false;
+}
+
+/**
+ * __scx_cid_to_cpu - Unchecked cid->cpu table lookup
+ * @cid: cid to look up. Must be in [0, num_possible_cpus()).
+ *
+ * Intended for callsites that have already validated @cid and that hold a
+ * non-NULL @sch from scx_prog_sched() - a live sched implies the table has
+ * been allocated, so no NULL check is needed here.
+ */
+static inline s32 __scx_cid_to_cpu(s32 cid)
+{
+	/* READ_ONCE pairs with WRITE_ONCE in scx_cid_arrays_alloc() */
+	return READ_ONCE(scx_cid_to_cpu_tbl)[cid];
+}
+
+/**
+ * __scx_cpu_to_cid - Unchecked cpu->cid table lookup
+ * @cpu: cpu to look up. Must be a valid possible cpu id.
+ *
+ * Same usage constraints as __scx_cid_to_cpu().
+ */
+static inline s32 __scx_cpu_to_cid(s32 cpu)
+{
+	return READ_ONCE(scx_cpu_to_cid_tbl)[cpu];
+}
+
+/**
+ * scx_cid_to_cpu - Translate @cid to its cpu
+ * @sch: scx_sched for error reporting
+ * @cid: cid to look up
+ *
+ * Return the cpu for @cid or a negative errno on failure. Invalid cid triggers
+ * scx_error() on @sch. The cid arrays are allocated on first scheduler enable
+ * and never freed, so the returned cpu is stable for the lifetime of the loaded
+ * scheduler.
+ */
+static inline s32 scx_cid_to_cpu(struct scx_sched *sch, s32 cid)
+{
+	if (!cid_valid(sch, cid))
+		return -EINVAL;
+	return __scx_cid_to_cpu(cid);
+}
+
+/**
+ * scx_cpu_to_cid - Translate @cpu to its cid
+ * @sch: scx_sched for error reporting
+ * @cpu: cpu to look up
+ *
+ * Return the cid for @cpu or a negative errno on failure. Invalid cpu triggers
+ * scx_error() on @sch. Same lifetime guarantee as scx_cid_to_cpu().
+ */
+static inline s32 scx_cpu_to_cid(struct scx_sched *sch, s32 cpu)
+{
+	if (!scx_cpu_valid(sch, cpu, NULL))
+		return -EINVAL;
+	return __scx_cpu_to_cid(cpu);
+}
+
+/**
+ * scx_is_cid_type - Test whether the active scheduler hierarchy is cid-form
+ */
+static inline bool scx_is_cid_type(void)
+{
+	return static_branch_unlikely(&__scx_is_cid_type);
+}
+
+static inline bool __scx_cmask_contains(const struct scx_cmask *m, u32 cid)
+{
+	return likely(cid >= m->base && cid < m->base + m->nr_cids);
+}
+
+/* Word in bits[] covering @cid. @cid must satisfy __scx_cmask_contains(). */
+static inline u64 *__scx_cmask_word(const struct scx_cmask *m, u32 cid)
+{
+	return (u64 *)&m->bits[cid / 64 - m->base / 64];
+}
+
+/**
+ * __scx_cmask_init - Initialize @m with explicit storage capacity
+ * @m: cmask to initialize
+ * @base: first cid of the active range
+ * @nr_cids: number of cids in the active range
+ * @alloc_cids: storage capacity in cids, at least @nr_cids
+ *
+ * Use when storage is sized larger than the initial active range. All of
+ * bits[] is zeroed.
+ */
+static inline void __scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids,
+				    u32 alloc_cids)
+{
+	if (WARN_ON_ONCE(alloc_cids < nr_cids))
+		nr_cids = alloc_cids;
+
+	m->base = base;
+	m->nr_cids = nr_cids;
+	m->alloc_words = SCX_CMASK_NR_WORDS(alloc_cids);
+	memset(m->bits, 0, m->alloc_words * sizeof(u64));
+}
+
+/**
+ * scx_cmask_init - Initialize @m on tight storage
+ * @m: cmask to initialize
+ * @base: first cid of the active range
+ * @nr_cids: number of cids in the active range
+ *
+ * All of bits[] is zeroed.
+ */
+static inline void scx_cmask_init(struct scx_cmask *m, u32 base, u32 nr_cids)
+{
+	__scx_cmask_init(m, base, nr_cids, nr_cids);
+}
+
+/**
+ * scx_cmask_reframe - Reshape @m's active range without resizing storage
+ * @m: cmask to reframe
+ * @base: new active range base
+ * @nr_cids: new active range length, must fit within @m->alloc_words
+ *
+ * Body bits within the new range become garbage - only the head and tail
+ * words are zeroed to keep the padding invariant.
+ */
+static inline void scx_cmask_reframe(struct scx_cmask *m, u32 base, u32 nr_cids)
+{
+	if (WARN_ON_ONCE(SCX_CMASK_NR_WORDS(nr_cids) > m->alloc_words))
+		return;
+
+	if (nr_cids) {
+		u32 last_word = ((base & 63) + nr_cids - 1) / 64;
+
+		m->bits[0] = 0;
+		m->bits[last_word] = 0;
+	}
+
+	m->base = base;
+	m->nr_cids = nr_cids;
+}
+
+static inline void __scx_cmask_set(struct scx_cmask *m, u32 cid)
+{
+	if (!__scx_cmask_contains(m, cid))
+		return;
+	*__scx_cmask_word(m, cid) |= BIT_U64(cid & 63);
+}
+
+#endif /* _KERNEL_SCHED_EXT_CID_H */
diff --git a/kernel/sched/ext_idle.c b/kernel/sched/ext_idle.c
index 6e1980763270d..3032398ceb0d0 100644
--- a/kernel/sched/ext_idle.c
+++ b/kernel/sched/ext_idle.c
@@ -9,7 +9,6 @@
  * Copyright (c) 2022 David Vernet <dvernet@meta.com>
  * Copyright (c) 2024 Andrea Righi <arighi@nvidia.com>
  */
-#include "ext_idle.h"
 
 /* Enable/disable built-in idle CPU selection policy */
 static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled);
@@ -79,7 +78,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu)
 	int node = scx_cpu_node_if_enabled(cpu);
 	struct cpumask *idle_cpus = idle_cpumask(node)->cpu;
 
-#ifdef CONFIG_SCHED_SMT
 	/*
 	 * SMT mask should be cleared whether we can claim @cpu or not. The SMT
 	 * cluster is not wholly idle either way. This also prevents
@@ -104,7 +102,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu)
 		else if (cpumask_test_cpu(cpu, idle_smts))
 			__cpumask_clear_cpu(cpu, idle_smts);
 	}
-#endif
 
 	return cpumask_test_and_clear_cpu(cpu, idle_cpus);
 }
@@ -622,7 +619,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
 		goto out_unlock;
 	}
 
-#ifdef CONFIG_SCHED_SMT
 	/*
 	 * Use @prev_cpu's sibling if it's idle.
 	 */
@@ -634,7 +630,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
 				goto out_unlock;
 		}
 	}
-#endif
 
 	/*
 	 * Search for any idle CPU in the same LLC domain.
@@ -714,7 +709,6 @@ static void update_builtin_idle(int cpu, bool idle)
 
 	assign_cpu(cpu, idle_cpus, idle);
 
-#ifdef CONFIG_SCHED_SMT
 	if (sched_smt_active()) {
 		const struct cpumask *smt = cpu_smt_mask(cpu);
 		struct cpumask *idle_smts = idle_cpumask(node)->smt;
@@ -731,7 +725,6 @@ static void update_builtin_idle(int cpu, bool idle)
 			cpumask_andnot(idle_smts, idle_smts, smt);
 		}
 	}
-#endif
 }
 
 /*
@@ -789,7 +782,7 @@ void __scx_update_idle(struct rq *rq, bool idle, bool do_notify)
 	 */
 	if (SCX_HAS_OP(sch, update_idle) && do_notify &&
 	    !scx_bypassing(sch, cpu_of(rq)))
-		SCX_CALL_OP(sch, update_idle, rq, cpu_of(rq), idle);
+		SCX_CALL_OP(sch, update_idle, rq, scx_cpu_arg(cpu_of(rq)), idle);
 }
 
 static void reset_idle_masks(struct sched_ext_ops *ops)
@@ -917,7 +910,7 @@ static s32 select_cpu_from_kfunc(struct scx_sched *sch, struct task_struct *p,
 	bool we_locked = false;
 	s32 cpu;
 
-	if (!ops_cpu_valid(sch, prev_cpu, NULL))
+	if (!scx_cpu_valid(sch, prev_cpu, NULL))
 		return -EINVAL;
 
 	if (!check_builtin_idle_enabled(sch))
@@ -990,7 +983,7 @@ __bpf_kfunc s32 scx_bpf_cpu_node(s32 cpu, const struct bpf_prog_aux *aux)
 	guard(rcu)();
 
 	sch = scx_prog_sched(aux);
-	if (unlikely(!sch) || !ops_cpu_valid(sch, cpu, NULL))
+	if (unlikely(!sch) || !scx_cpu_valid(sch, cpu, NULL))
 		return NUMA_NO_NODE;
 	return cpu_to_node(cpu);
 }
@@ -1272,7 +1265,7 @@ __bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu, const struct bpf_prog_
 	if (!check_builtin_idle_enabled(sch))
 		return false;
 
-	if (!ops_cpu_valid(sch, cpu, NULL))
+	if (!scx_cpu_valid(sch, cpu, NULL))
 		return false;
 
 	return scx_idle_test_and_clear_cpu(cpu);
diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h
index a075732d4430d..7258aea94b9f8 100644
--- a/kernel/sched/ext_internal.h
+++ b/kernel/sched/ext_internal.h
@@ -8,35 +8,6 @@
 #define SCX_OP_IDX(op)		(offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void)))
 #define SCX_MOFF_IDX(moff)	((moff) / sizeof(void (*)(void)))
 
-enum scx_consts {
-	SCX_DSP_DFL_MAX_BATCH		= 32,
-	SCX_DSP_MAX_LOOPS		= 32,
-	SCX_WATCHDOG_MAX_TIMEOUT	= 30 * HZ,
-
-	SCX_EXIT_BT_LEN			= 64,
-	SCX_EXIT_MSG_LEN		= 1024,
-	SCX_EXIT_DUMP_DFL_LEN		= 32768,
-
-	SCX_CPUPERF_ONE			= SCHED_CAPACITY_SCALE,
-
-	/*
-	 * Iterating all tasks may take a while. Periodically drop
-	 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls.
-	 */
-	SCX_TASK_ITER_BATCH		= 32,
-
-	SCX_BYPASS_HOST_NTH		= 2,
-
-	SCX_BYPASS_LB_DFL_INTV_US	= 500 * USEC_PER_MSEC,
-	SCX_BYPASS_LB_DONOR_PCT		= 125,
-	SCX_BYPASS_LB_MIN_DELTA_DIV	= 4,
-	SCX_BYPASS_LB_BATCH		= 256,
-
-	SCX_REENQ_LOCAL_MAX_REPEAT	= 256,
-
-	SCX_SUB_MAX_DEPTH		= 4,
-};
-
 enum scx_exit_kind {
 	SCX_EXIT_NONE,
 	SCX_EXIT_DONE,
@@ -94,6 +65,12 @@ struct scx_exit_info {
 	/* %SCX_EXIT_* - broad category of the exit reason */
 	enum scx_exit_kind	kind;
 
+	/*
+	 * CPU that initiated the exit, valid once @kind has been set.
+	 * Negative if the exit path didn't identify a CPU.
+	 */
+	s32			exit_cpu;
+
 	/* exit code if gracefully exiting */
 	s64			exit_code;
 
@@ -138,7 +115,8 @@ enum scx_ops_flags {
 	 * To mask this problem, by default, unhashed tasks are automatically
 	 * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't
 	 * depend on pid lookups and wants to handle these tasks directly, the
-	 * following flag can be used.
+	 * following flag can be used. With %SCX_OPS_TID_TO_TASK,
+	 * scx_bpf_tid_to_task() can find exiting tasks reliably.
 	 */
 	SCX_OPS_ENQ_EXITING		= 1LLU << 2,
 
@@ -189,6 +167,17 @@ enum scx_ops_flags {
 	 */
 	SCX_OPS_ALWAYS_ENQ_IMMED	= 1LLU << 7,
 
+	/*
+	 * Maintain a mapping from p->scx.tid to task_struct so the BPF
+	 * scheduler can recover task pointers from stored tids via
+	 * scx_bpf_tid_to_task().
+	 *
+	 * Only the root scheduler turns this on. A sub-sched may set the flag
+	 * to declare a dependency on the lookup; if the root scheduler hasn't
+	 * enabled it, attaching the sub-sched is rejected.
+	 */
+	SCX_OPS_TID_TO_TASK		= 1LLU << 8,
+
 	SCX_OPS_ALL_FLAGS		= SCX_OPS_KEEP_BUILTIN_IDLE |
 					  SCX_OPS_ENQ_LAST |
 					  SCX_OPS_ENQ_EXITING |
@@ -196,7 +185,8 @@ enum scx_ops_flags {
 					  SCX_OPS_ALLOW_QUEUED_WAKEUP |
 					  SCX_OPS_SWITCH_PARTIAL |
 					  SCX_OPS_BUILTIN_IDLE_PER_NODE |
-					  SCX_OPS_ALWAYS_ENQ_IMMED,
+					  SCX_OPS_ALWAYS_ENQ_IMMED |
+					  SCX_OPS_TID_TO_TASK,
 
 	/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */
 	__SCX_OPS_INTERNAL_MASK		= 0xffLLU << 56,
@@ -539,28 +529,6 @@ struct sched_ext_ops {
 	 */
 	void (*update_idle)(s32 cpu, bool idle);
 
-	/**
-	 * @cpu_acquire: A CPU is becoming available to the BPF scheduler
-	 * @cpu: The CPU being acquired by the BPF scheduler.
-	 * @args: Acquire arguments, see the struct definition.
-	 *
-	 * A CPU that was previously released from the BPF scheduler is now once
-	 * again under its control.
-	 */
-	void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args);
-
-	/**
-	 * @cpu_release: A CPU is taken away from the BPF scheduler
-	 * @cpu: The CPU being released by the BPF scheduler.
-	 * @args: Release arguments, see the struct definition.
-	 *
-	 * The specified CPU is no longer under the control of the BPF
-	 * scheduler. This could be because it was preempted by a higher
-	 * priority sched_class, though there may be other reasons as well. The
-	 * caller should consult @args->reason to determine the cause.
-	 */
-	void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args);
-
 	/**
 	 * @init_task: Initialize a task to run in a BPF scheduler
 	 * @p: task to initialize for BPF scheduling
@@ -851,6 +819,128 @@ struct sched_ext_ops {
 
 	/* internal use only, must be NULL */
 	void __rcu *priv;
+
+	/*
+	 * Deprecated callbacks. Kept at the end of the struct so the cid-form
+	 * struct (sched_ext_ops_cid) can omit them without affecting the
+	 * shared field offsets. Use SCX_ENQ_IMMED instead. Sitting past
+	 * SCX_OPI_END means has_op doesn't cover them, so SCX_HAS_OP() cannot
+	 * be used; callers must test sch->ops.cpu_acquire / cpu_release
+	 * directly.
+	 */
+
+	/**
+	 * @cpu_acquire: A CPU is becoming available to the BPF scheduler
+	 * @cpu: The CPU being acquired by the BPF scheduler.
+	 * @args: Acquire arguments, see the struct definition.
+	 *
+	 * A CPU that was previously released from the BPF scheduler is now once
+	 * again under its control. Deprecated; use SCX_ENQ_IMMED instead.
+	 */
+	void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args);
+
+	/**
+	 * @cpu_release: A CPU is taken away from the BPF scheduler
+	 * @cpu: The CPU being released by the BPF scheduler.
+	 * @args: Release arguments, see the struct definition.
+	 *
+	 * The specified CPU is no longer under the control of the BPF
+	 * scheduler. This could be because it was preempted by a higher
+	 * priority sched_class, though there may be other reasons as well. The
+	 * caller should consult @args->reason to determine the cause.
+	 * Deprecated; use SCX_ENQ_IMMED instead.
+	 */
+	void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args);
+};
+
+/**
+ * struct sched_ext_ops_cid - cid-form alternative to struct sched_ext_ops
+ *
+ * Mirrors struct sched_ext_ops with cpu/cpumask substituted with cid/cmask
+ * where applicable. Layout up to and including @priv matches sched_ext_ops
+ * byte-for-byte (verified by BUILD_BUG_ON checks at scx_init() time) so
+ * shared field offsets work for both struct types in bpf_scx_init_member()
+ * and bpf_scx_check_member(). The deprecated cpu_acquire/cpu_release
+ * callbacks at the tail of sched_ext_ops are omitted here entirely.
+ *
+ * Differences from sched_ext_ops:
+ *   - select_cpu       -> select_cid (returns cid)
+ *   - dispatch         -> dispatch (cpu arg is now cid)
+ *   - update_idle      -> update_idle (cpu arg is now cid)
+ *   - set_cpumask      -> set_cmask (cmask instead of cpumask)
+ *   - cpu_online       -> cid_online
+ *   - cpu_offline      -> cid_offline
+ *   - dump_cpu         -> dump_cid
+ *   - cpu_acquire/cpu_release  -> not present (deprecated in sched_ext_ops)
+ *
+ * BPF schedulers using this type cannot call cpu-form scx_bpf_* kfuncs;
+ * use the cid-form variants instead. Enforced at BPF verifier time via
+ * scx_kfunc_context_filter() branching on prog->aux->st_ops.
+ *
+ * See sched_ext_ops for callback documentation.
+ */
+struct sched_ext_ops_cid {
+	s32 (*select_cid)(struct task_struct *p, s32 prev_cid, u64 wake_flags);
+	void (*enqueue)(struct task_struct *p, u64 enq_flags);
+	void (*dequeue)(struct task_struct *p, u64 deq_flags);
+	void (*dispatch)(s32 cid, struct task_struct *prev);
+	void (*tick)(struct task_struct *p);
+	void (*runnable)(struct task_struct *p, u64 enq_flags);
+	void (*running)(struct task_struct *p);
+	void (*stopping)(struct task_struct *p, bool runnable);
+	void (*quiescent)(struct task_struct *p, u64 deq_flags);
+	bool (*yield)(struct task_struct *from, struct task_struct *to);
+	bool (*core_sched_before)(struct task_struct *a,
+				   struct task_struct *b);
+	void (*set_weight)(struct task_struct *p, u32 weight);
+	void (*set_cmask)(struct task_struct *p,
+			   const struct scx_cmask *cmask);
+	void (*update_idle)(s32 cid, bool idle);
+	s32 (*init_task)(struct task_struct *p,
+			  struct scx_init_task_args *args);
+	void (*exit_task)(struct task_struct *p,
+			   struct scx_exit_task_args *args);
+	void (*enable)(struct task_struct *p);
+	void (*disable)(struct task_struct *p);
+	void (*dump)(struct scx_dump_ctx *ctx);
+	void (*dump_cid)(struct scx_dump_ctx *ctx, s32 cid, bool idle);
+	void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p);
+#ifdef CONFIG_EXT_GROUP_SCHED
+	s32 (*cgroup_init)(struct cgroup *cgrp,
+			    struct scx_cgroup_init_args *args);
+	void (*cgroup_exit)(struct cgroup *cgrp);
+	s32 (*cgroup_prep_move)(struct task_struct *p,
+				 struct cgroup *from, struct cgroup *to);
+	void (*cgroup_move)(struct task_struct *p,
+			     struct cgroup *from, struct cgroup *to);
+	void (*cgroup_cancel_move)(struct task_struct *p,
+				    struct cgroup *from, struct cgroup *to);
+	void (*cgroup_set_weight)(struct cgroup *cgrp, u32 weight);
+	void (*cgroup_set_bandwidth)(struct cgroup *cgrp,
+				      u64 period_us, u64 quota_us, u64 burst_us);
+	void (*cgroup_set_idle)(struct cgroup *cgrp, bool idle);
+#endif	/* CONFIG_EXT_GROUP_SCHED */
+	s32 (*sub_attach)(struct scx_sub_attach_args *args);
+	void (*sub_detach)(struct scx_sub_detach_args *args);
+	void (*cid_online)(s32 cid);
+	void (*cid_offline)(s32 cid);
+	s32 (*init)(void);
+	void (*exit)(struct scx_exit_info *info);
+
+	/* Data fields - must match sched_ext_ops layout exactly */
+	u32 dispatch_max_batch;
+	u64 flags;
+	u32 timeout_ms;
+	u32 exit_dump_len;
+	u64 hotplug_seq;
+	u64 sub_cgroup_id;
+	char name[SCX_OPS_NAME_LEN];
+
+	/* internal use only, must be NULL */
+	void __rcu *priv;
+
+	/* layout end anchor for the BUILD_BUG_ON in scx_init(); keep last */
+	char __end[0];
 };
 
 enum scx_opi {
@@ -1009,7 +1099,18 @@ struct scx_sched_pnode {
 };
 
 struct scx_sched {
-	struct sched_ext_ops	ops;
+	/*
+	 * cpu-form and cid-form ops share field offsets up to .priv (verified
+	 * by BUILD_BUG_ON in scx_init()). The anonymous union lets the kernel
+	 * access either view of the same storage without function-pointer
+	 * casts: use .ops for cpu-form and shared fields, .ops_cid for the
+	 * cid-renamed callbacks (set_cmask, select_cid, cid_online, ...).
+	 */
+	union {
+		struct sched_ext_ops		ops;
+		struct sched_ext_ops_cid	ops_cid;
+	};
+	bool			is_cid_type;	/* true if registered via bpf_sched_ext_ops_cid */
 	DECLARE_BITMAP(has_op, SCX_OPI_END);
 
 	/*
@@ -1366,8 +1467,32 @@ enum scx_ops_state {
 extern struct scx_sched __rcu *scx_root;
 DECLARE_PER_CPU(struct rq *, scx_locked_rq_state);
 
+extern struct scx_cmask __percpu *scx_set_cmask_scratch;
+
+/*
+ * True when the currently loaded scheduler hierarchy is cid-form. All scheds
+ * in a hierarchy share one form, so this single key tells callsites which
+ * view to use without per-sch dereferences. Use scx_is_cid_type() to test.
+ */
+DECLARE_STATIC_KEY_FALSE(__scx_is_cid_type);
+
 int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id);
 
+bool scx_cpu_valid(struct scx_sched *sch, s32 cpu, const char *where);
+
+__printf(5, 0) bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind,
+			      s64 exit_code, s32 exit_cpu, const char *fmt,
+			      va_list args);
+__printf(5, 6) bool __scx_exit(struct scx_sched *sch, enum scx_exit_kind kind,
+			       s64 exit_code, s32 exit_cpu, const char *fmt, ...);
+
+#define scx_exit(sch, kind, exit_code, fmt, args...)				\
+	__scx_exit(sch, kind, exit_code, raw_smp_processor_id(), fmt, ##args)
+#define scx_error(sch, fmt, args...)						\
+	scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args)
+#define scx_verror(sch, fmt, args)						\
+	scx_vexit((sch), SCX_EXIT_ERROR, 0, raw_smp_processor_id(), fmt, args)
+
 /*
  * Return the rq currently locked from an scx callback, or NULL if no rq is
  * locked.
@@ -1476,7 +1601,7 @@ static inline bool scx_task_on_sched(struct scx_sched *sch,
 	return true;
 }
 
-static struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux)
+static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux)
 {
 	return rcu_dereference_all(scx_root);
 }
diff --git a/kernel/sched/ext_types.h b/kernel/sched/ext_types.h
new file mode 100644
index 0000000000000..8b3527e21fca7
--- /dev/null
+++ b/kernel/sched/ext_types.h
@@ -0,0 +1,144 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Early sched_ext type definitions.
+ *
+ * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2026 Tejun Heo <tj@kernel.org>
+ */
+#ifndef _KERNEL_SCHED_EXT_TYPES_H
+#define _KERNEL_SCHED_EXT_TYPES_H
+
+enum scx_consts {
+	SCX_DSP_DFL_MAX_BATCH		= 32,
+	SCX_DSP_MAX_LOOPS		= 32,
+	SCX_WATCHDOG_MAX_TIMEOUT	= 30 * HZ,
+
+	/* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */
+	SCX_TID_CHUNK			= 1024,
+
+	SCX_EXIT_BT_LEN			= 64,
+	SCX_EXIT_MSG_LEN		= 1024,
+	SCX_EXIT_DUMP_DFL_LEN		= 32768,
+
+	SCX_CPUPERF_ONE			= SCHED_CAPACITY_SCALE,
+
+	/*
+	 * Iterating all tasks may take a while. Periodically drop
+	 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls.
+	 */
+	SCX_TASK_ITER_BATCH		= 32,
+
+	SCX_BYPASS_HOST_NTH		= 2,
+
+	SCX_BYPASS_LB_DFL_INTV_US	= 500 * USEC_PER_MSEC,
+	SCX_BYPASS_LB_DONOR_PCT		= 125,
+	SCX_BYPASS_LB_MIN_DELTA_DIV	= 4,
+	SCX_BYPASS_LB_BATCH		= 256,
+
+	SCX_REENQ_LOCAL_MAX_REPEAT	= 256,
+
+	SCX_SUB_MAX_DEPTH		= 4,
+};
+
+/*
+ * Per-cid topology info. For each topology level (core, LLC, node), records
+ * the first cid in the unit and its global index. Global indices are
+ * consecutive integers assigned in cid-walk order, so e.g. core_idx ranges
+ * over [0, nr_cores_at_init) with no gaps. No-topo cids have all fields set
+ * to -1.
+ *
+ * @core_cid: first cid of this cid's core (smt-sibling group)
+ * @core_idx: global index of that core, in [0, nr_cores_at_init)
+ * @llc_cid: first cid of this cid's LLC
+ * @llc_idx: global index of that LLC, in [0, nr_llcs_at_init)
+ * @node_cid: first cid of this cid's NUMA node
+ * @node_idx: global index of that node, in [0, nr_nodes_at_init)
+ */
+struct scx_cid_topo {
+	s32 core_cid;
+	s32 core_idx;
+	s32 llc_cid;
+	s32 llc_idx;
+	s32 node_cid;
+	s32 node_idx;
+};
+
+/*
+ * cmask: variable-length, base-windowed bitmap over cid space
+ * -----------------------------------------------------------
+ *
+ * A cmask covers the cid range [base, base + nr_cids). bits[] is aligned to the
+ * global 64-cid grid: bits[0] spans [base & ~63, (base & ~63) + 64), so the
+ * first (base & 63) bits of bits[0] are head padding and the trailing bits of
+ * the last active word past base + nr_cids are tail padding. Both stay zero;
+ * all mutating helpers preserve that. Words past the last active word are not
+ * read by any helper and have no constraint.
+ *
+ * Grid alignment means two cmasks always address bits[] against the same global
+ * 64-cid windows, so cross-cmask word ops (AND, OR, ...) reduce to
+ *
+ *	dst->bits[i] OP= src->bits[i - delta]
+ *
+ * with no bit-shifting, regardless of how the two bases relate mod 64.
+ */
+struct scx_cmask {
+	u32 base;
+	u32 nr_cids;
+	u32 alloc_words;
+	u64 bits[] __counted_by(alloc_words);
+};
+
+/*
+ * Number of u64 words of bits[] storage that covers @nr_cids regardless of base
+ * alignment. The +1 absorbs up to 63 bits of head padding when base is not
+ * 64-aligned - always allocating one extra word beats branching on base or
+ * splitting the compute. The u64 cast keeps the +63 from wrapping when @nr_cids
+ * is near U32_MAX, so callers bounds-checking the result against @alloc_words
+ * catch the overflow instead of seeing a small value.
+ */
+#define SCX_CMASK_NR_WORDS(nr_cids)	((u32)(((u64)(nr_cids) + 63) / 64 + 1))
+
+/**
+ * __SCX_CMASK_DEFINE - Define an on-stack cmask with explicit storage capacity
+ * @NAME: variable name to define
+ * @BASE: first cid of the active range
+ * @NR_CIDS: active range length
+ * @ALLOC_CIDS: storage capacity in cids, at least @NR_CIDS
+ *
+ * @NAME aliases zero-initialized storage with the active range set to
+ * [BASE, BASE + NR_CIDS). Use scx_cmask_reframe() to reshape later, up to
+ * @ALLOC_CIDS.
+ */
+#define __SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, ALLOC_CIDS)			\
+	_DEFINE_FLEX(struct scx_cmask, NAME, bits, SCX_CMASK_NR_WORDS(ALLOC_CIDS), \
+		     = { .base = (BASE),					\
+			 .nr_cids = (NR_CIDS),					\
+			 .alloc_words = SCX_CMASK_NR_WORDS(ALLOC_CIDS) })
+
+/**
+ * SCX_CMASK_DEFINE - Define an on-stack cmask on tight storage
+ * @NAME: variable name to define
+ * @BASE: first cid of the active range
+ * @NR_CIDS: active range length, also storage capacity
+ *
+ * @NAME aliases zero-initialized storage with the active range and storage
+ * both [BASE, BASE + NR_CIDS).
+ */
+#define SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS)					\
+	__SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, NR_CIDS)
+
+/**
+ * SCX_CMASK_DEFINE_SHARD - Define an on-stack cmask sized to one shard
+ * @NAME: variable name to define
+ * @BASE: first cid of the active range
+ * @NR_CIDS: active range length, must be <= SCX_CID_SHARD_MAX_CPUS
+ *
+ * Storage is fixed at SCX_CID_SHARD_MAX_CPUS, active range framed by
+ * (BASE, NR_CIDS). Passing NR_CIDS > SCX_CID_SHARD_MAX_CPUS leaves the
+ * cmask claiming more bits than storage holds and subsequent cmask
+ * operations will overrun.
+ */
+#define SCX_CMASK_DEFINE_SHARD(NAME, BASE, NR_CIDS)				\
+	__SCX_CMASK_DEFINE(NAME, BASE, NR_CIDS, SCX_CID_SHARD_MAX_CPUS)
+
+#endif /* _KERNEL_SCHED_EXT_TYPES_H */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3ebec186f9823..aac24cfddecdf 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1350,6 +1350,8 @@ void post_init_entity_util_avg(struct task_struct *p)
 	sa->runnable_avg = sa->util_avg;
 }
 
+static inline void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec);
+
 static s64 update_se(struct rq *rq, struct sched_entity *se)
 {
 	u64 now = rq_clock_task(rq);
@@ -1372,6 +1374,7 @@ static s64 update_se(struct rq *rq, struct sched_entity *se)
 
 		trace_sched_stat_runtime(running, delta_exec);
 		account_group_exec_runtime(running, delta_exec);
+		account_mm_sched(rq, running, delta_exec);
 
 		/* cgroup time is always accounted against the donor */
 		cgroup_account_cputime(donor, delta_exec);
@@ -1393,6 +1396,581 @@ static s64 update_se(struct rq *rq, struct sched_entity *se)
 
 static void set_next_buddy(struct sched_entity *se);
 
+#ifdef CONFIG_SCHED_CACHE
+
+/*
+ * XXX numbers come from a place the sun don't shine -- probably wants to be SD
+ * tunable or so.
+ */
+#define EPOCH_PERIOD	(HZ / 100)	/* 10 ms */
+#define EPOCH_LLC_AFFINITY_TIMEOUT	5	/* 50 ms */
+__read_mostly unsigned int llc_aggr_tolerance	= 1;
+__read_mostly unsigned int llc_epoch_period	= EPOCH_PERIOD;
+__read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT;
+__read_mostly unsigned int llc_imb_pct		= 20;
+__read_mostly unsigned int llc_overaggr_pct	= 50;
+
+static int llc_id(int cpu)
+{
+	if (cpu < 0)
+		return -1;
+
+	return per_cpu(sd_llc_id, cpu);
+}
+
+static inline int get_sched_cache_scale(int mul)
+{
+	unsigned int tol = READ_ONCE(llc_aggr_tolerance);
+
+	if (!tol)
+		return 0;
+
+	if (tol >= 100)
+		return INT_MAX;
+
+	return (1 + (tol - 1) * mul);
+}
+
+static bool exceed_llc_capacity(struct mm_struct *mm, int cpu)
+{
+#ifdef CONFIG_NUMA_BALANCING
+	unsigned long llc, footprint;
+	struct sched_domain *sd;
+	int scale;
+
+	guard(rcu)();
+
+	sd = rcu_dereference_sched_domain(cpu_rq(cpu)->sd);
+	if (!sd)
+		return true;
+
+	if (static_branch_likely(&sched_numa_balancing)) {
+		/*
+		 * TBD: RDT exclusive LLC ways reserved should be
+		 * excluded.
+		 */
+		llc = sd->llc_bytes;
+		footprint = READ_ONCE(mm->sc_stat.footprint);
+
+		/*
+		 * Scale the LLC size by 256*llc_aggr_tolerance
+		 * and compare it to the task's footprint.
+		 *
+		 * Suppose the L3 size is 32MB. If the
+		 * llc_aggr_tolerance is 1:
+		 * When the footprint is larger than 32MB, the
+		 * process is regarded as exceeding the LLC
+		 * capacity. If the llc_aggr_tolerance is 99:
+		 * When the footprint is larger than 784GB, the
+		 * process is regarded as exceeding the LLC
+		 * capacity:
+		 * 784GB = (1 + (99 - 1) * 256) * 32MB
+		 * If the llc_aggr_tolerance is 100:
+		 * ignore the footprint and do the aggregation
+		 * anyway.
+		 */
+		scale = get_sched_cache_scale(256);
+		if (scale == INT_MAX)
+			return false;
+
+		return ((llc * (u64)scale) < (footprint * PAGE_SIZE));
+	}
+#endif
+	return false;
+}
+
+static bool invalid_llc_nr(struct mm_struct *mm, struct task_struct *p,
+			   int cpu)
+{
+	int scale;
+
+	if (get_nr_threads(p) <= 1)
+		return true;
+
+	/*
+	 * Scale the number of 'cores' in a LLC by llc_aggr_tolerance
+	 * and compare it to the task's active threads.
+	 */
+	scale = get_sched_cache_scale(1);
+	if (scale == INT_MAX)
+		return false;
+
+	return !fits_capacity((mm->sc_stat.nr_running_avg * cpu_smt_num_threads),
+			(scale * per_cpu(sd_llc_size, cpu)));
+}
+
+static void account_llc_enqueue(struct rq *rq, struct task_struct *p)
+{
+	int pref_llc, pref_llc_queued;
+	struct sched_domain *sd;
+
+	pref_llc = p->preferred_llc;
+	if (pref_llc < 0)
+		return;
+
+	pref_llc_queued = (pref_llc == task_llc(p));
+	rq->nr_llc_running++;
+	rq->nr_pref_llc_running += pref_llc_queued;
+
+	/*
+	 * Record whether p is enqueued on its preferred
+	 * LLC, in order to pair with account_llc_dequeue()
+	 * to maintain a consistent nr_pref_llc_running per
+	 * runqueue.
+	 * This is necessary because a race condition exists:
+	 * after a task is enqueued on a runqueue, task_llc(p)
+	 * may change due to CPU hotplug. Therefore, checking
+	 * task_llc(p) to determine whether the task is being
+	 * dequeued from its preferred LLC is unreliable and
+	 * can cause inconsistent values - checking the
+	 * p->pref_llc_queued in account_llc_dequeue() would
+	 * be reliable.
+	 */
+	p->pref_llc_queued = pref_llc_queued;
+
+	sd = rcu_dereference_all(rq->sd);
+	if (sd && (unsigned int)pref_llc < sd->llc_max)
+		sd->llc_counts[pref_llc]++;
+}
+
+static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
+{
+	struct sched_domain *sd;
+	int pref_llc;
+
+	pref_llc = p->preferred_llc;
+	if (pref_llc < 0)
+		return;
+
+	rq->nr_llc_running--;
+	if (p->pref_llc_queued) {
+		rq->nr_pref_llc_running--;
+		/*
+		 * Update the status in case
+		 * other logic might query
+		 * this.
+		 */
+		p->pref_llc_queued = 0;
+	}
+
+	sd = rcu_dereference_all(rq->sd);
+	if (sd && (unsigned int)pref_llc < sd->llc_max) {
+		/*
+		 * There is a race condition between dequeue
+		 * and CPU hotplug. After a task has been enqueued
+		 * on CPUx, a CPU hotplug event occurs, and all online
+		 * CPUs (including CPUx) rebuild their sched_domains
+		 * and reset statistics to zero(including sd->llc_counts).
+		 * This can cause temporary undercount and we have to
+		 * check for such underflow in sd->llc_counts.
+		 *
+		 * This undercount is temporary and accurate accounting
+		 * will resume once the rq has a chance to be idle.
+		 */
+		if (sd->llc_counts[pref_llc])
+			sd->llc_counts[pref_llc]--;
+	}
+}
+
+void mm_init_sched(struct mm_struct *mm,
+		   struct sched_cache_time __percpu *_pcpu_sched)
+{
+	unsigned long epoch = 0;
+	int i;
+
+	for_each_possible_cpu(i) {
+		struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i);
+		struct rq *rq = cpu_rq(i);
+
+		pcpu_sched->runtime = 0;
+		/* a slightly stale cpu epoch is acceptible */
+		pcpu_sched->epoch = rq->cpu_epoch;
+		epoch = rq->cpu_epoch;
+	}
+
+	raw_spin_lock_init(&mm->sc_stat.lock);
+	mm->sc_stat.epoch = epoch;
+	mm->sc_stat.cpu = -1;
+	mm->sc_stat.next_scan = jiffies;
+	mm->sc_stat.nr_running_avg = 0;
+	mm->sc_stat.footprint = 0;
+	/*
+	 * The update to mm->sc_stat should not be reordered
+	 * before initialization to mm's other fields, in case
+	 * the readers may get invalid mm_sched_epoch, etc.
+	 */
+	smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched);
+}
+
+/* because why would C be fully specified */
+static __always_inline void __shr_u64(u64 *val, unsigned int n)
+{
+	if (n >= 64) {
+		*val = 0;
+		return;
+	}
+	*val >>= n;
+}
+
+static inline void __update_mm_sched(struct rq *rq,
+				     struct sched_cache_time *pcpu_sched)
+{
+	lockdep_assert_held(&rq->cpu_epoch_lock);
+
+	unsigned int period = max(READ_ONCE(llc_epoch_period), 1U);
+	unsigned long n, now = jiffies;
+	long delta = now - rq->cpu_epoch_next;
+
+	if (delta > 0) {
+		n = (delta + period - 1) / period;
+		rq->cpu_epoch += n;
+		rq->cpu_epoch_next += n * period;
+		__shr_u64(&rq->cpu_runtime, n);
+	}
+
+	n = rq->cpu_epoch - pcpu_sched->epoch;
+	if (n) {
+		pcpu_sched->epoch += n;
+		__shr_u64(&pcpu_sched->runtime, n);
+	}
+}
+
+static unsigned long fraction_mm_sched(struct rq *rq,
+				       struct sched_cache_time *pcpu_sched)
+{
+	guard(raw_spinlock_irqsave)(&rq->cpu_epoch_lock);
+
+	__update_mm_sched(rq, pcpu_sched);
+
+	/*
+	 * Runtime is a geometric series (r=0.5) and as such will sum to twice
+	 * the accumulation period, this means the multiplcation here should
+	 * not overflow.
+	 */
+	return div64_u64(NICE_0_LOAD * pcpu_sched->runtime, rq->cpu_runtime + 1);
+}
+
+static int get_pref_llc(struct task_struct *p, struct mm_struct *mm)
+{
+	int mm_sched_llc = -1, mm_sched_cpu;
+
+	if (!mm)
+		return -1;
+
+	mm_sched_cpu = READ_ONCE(mm->sc_stat.cpu);
+	if (mm_sched_cpu != -1) {
+		mm_sched_llc = llc_id(mm_sched_cpu);
+
+#ifdef CONFIG_NUMA_BALANCING
+		/*
+		 * Don't assign preferred LLC if it
+		 * conflicts with NUMA balancing.
+		 * This can happen when sched_setnuma() gets
+		 * called, however it is not much of an issue
+		 * because we expect account_mm_sched() to get
+		 * called fairly regularly -- at a higher rate
+		 * than sched_setnuma() at least -- and thus the
+		 * conflict only exists for a short period of time.
+		 */
+		if (static_branch_likely(&sched_numa_balancing) &&
+		    p->numa_preferred_nid >= 0 &&
+		    cpu_to_node(mm_sched_cpu) != p->numa_preferred_nid)
+			mm_sched_llc = -1;
+#endif
+	}
+
+	return mm_sched_llc;
+}
+
+static unsigned int task_running_on_cpu(int cpu, struct task_struct *p);
+
+static inline
+void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
+{
+	struct sched_cache_time *pcpu_sched;
+	struct mm_struct *mm = p->mm;
+	int mm_sched_llc = -1;
+	unsigned long epoch;
+
+	if (!sched_cache_enabled())
+		return;
+
+	if (p->sched_class != &fair_sched_class)
+		return;
+	/*
+	 * init_task, kthreads and user thread created
+	 * by user_mode_thread() don't have mm.
+	 */
+	if (!mm || !mm->sc_stat.pcpu_sched)
+		return;
+
+	pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, cpu_of(rq));
+
+	scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) {
+		__update_mm_sched(rq, pcpu_sched);
+		pcpu_sched->runtime += delta_exec;
+		rq->cpu_runtime += delta_exec;
+		epoch = rq->cpu_epoch;
+	}
+
+	/*
+	 * If this process hasn't hit task_cache_work() for a while invalidate
+	 * its preferred state.
+	 */
+	if ((long)(epoch - READ_ONCE(mm->sc_stat.epoch)) > llc_epoch_affinity_timeout ||
+	    invalid_llc_nr(mm, p, cpu_of(rq)) ||
+	    exceed_llc_capacity(mm, cpu_of(rq))) {
+		if (READ_ONCE(mm->sc_stat.cpu) != -1)
+			WRITE_ONCE(mm->sc_stat.cpu, -1);
+	}
+
+	mm_sched_llc = get_pref_llc(p, mm);
+
+	/* task not on rq accounted later in account_entity_enqueue() */
+	if (task_running_on_cpu(rq->cpu, p) &&
+	    READ_ONCE(p->preferred_llc) != mm_sched_llc) {
+		account_llc_dequeue(rq, p);
+		WRITE_ONCE(p->preferred_llc, mm_sched_llc);
+		account_llc_enqueue(rq, p);
+	}
+}
+
+static void task_tick_cache(struct rq *rq, struct task_struct *p)
+{
+	struct callback_head *work = &p->cache_work;
+	struct mm_struct *mm = p->mm;
+	unsigned long epoch;
+
+	if (!sched_cache_enabled())
+		return;
+
+	if (!mm || p->flags & PF_KTHREAD ||
+	    !mm->sc_stat.pcpu_sched)
+		return;
+
+	epoch = rq->cpu_epoch;
+	/* avoid moving backwards */
+	if (time_after_eq(mm->sc_stat.epoch, epoch))
+		return;
+
+	guard(raw_spinlock)(&mm->sc_stat.lock);
+
+	if (work->next == work) {
+		task_work_add(p, work, TWA_RESUME);
+		WRITE_ONCE(mm->sc_stat.epoch, epoch);
+	}
+}
+
+static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p)
+{
+#ifdef CONFIG_NUMA_BALANCING
+	int cpu, curr_cpu, nid, pref_nid;
+
+	if (!static_branch_likely(&sched_numa_balancing))
+		goto out;
+
+	cpu = READ_ONCE(p->mm->sc_stat.cpu);
+	if (cpu != -1)
+		nid = cpu_to_node(cpu);
+	curr_cpu = task_cpu(p);
+
+	/*
+	 * Scanning in the preferred NUMA node is ideal. However, the NUMA
+	 * preferred node is per-task rather than per-process. It is possible
+	 * for different threads of the process to have distinct preferred
+	 * nodes; consequently, the process-wide preferred LLC may bounce
+	 * between different nodes. As a workaround, maintain the scan
+	 * CPU mask to also cover the process's current preferred LLC and the
+	 * current running node to mitigate the bouncing risk.
+	 * TBD: numa_group should be considered during task aggregation.
+	 */
+	pref_nid = p->numa_preferred_nid;
+	/* honor the task's preferred node */
+	if (pref_nid == NUMA_NO_NODE)
+		goto out;
+
+	cpumask_or(cpus, cpus, cpumask_of_node(pref_nid));
+
+	/* honor the task's preferred LLC CPU */
+	if (cpu != -1 && !cpumask_test_cpu(cpu, cpus) && nid != NUMA_NO_NODE)
+		cpumask_or(cpus, cpus, cpumask_of_node(nid));
+
+	/* make sure the task's current running node is included */
+	if (!cpumask_test_cpu(curr_cpu, cpus))
+		cpumask_or(cpus, cpus, cpumask_of_node(cpu_to_node(curr_cpu)));
+
+	return;
+
+out:
+#endif
+	cpumask_copy(cpus, cpu_online_mask);
+}
+
+static inline void update_avg_scale(u64 *avg, u64 sample)
+{
+	int factor = per_cpu(sd_llc_size, raw_smp_processor_id());
+	s64 diff = sample - *avg;
+	u32 divisor;
+
+	/*
+	 * Scale the divisor based on the number of CPUs contained
+	 * in the LLC. This scaling ensures smaller LLC domains use
+	 * a smaller divisor to achieve more precise sensitivity to
+	 * changes in nr_running, while larger LLC domains are capped
+	 * at a maximum divisor of 8 which is the default smoothing
+	 * factor of EWMA in update_avg().
+	 */
+	divisor = clamp_t(u32, (factor >> 2), 2, 8);
+	*avg += div64_s64(diff, divisor);
+}
+
+static void task_cache_work(struct callback_head *work)
+{
+	int cpu, m_a_cpu = -1, nr_running = 0, curr_cpu;
+	unsigned long next_scan, now = jiffies;
+	struct task_struct *p = current, *cur;
+	unsigned long curr_m_a_occ = 0;
+	struct mm_struct *mm = p->mm;
+	unsigned long m_a_occ = 0;
+	cpumask_var_t cpus;
+
+	WARN_ON_ONCE(work != &p->cache_work);
+
+	work->next = work;
+
+	if (p->flags & PF_EXITING)
+		return;
+
+	next_scan = READ_ONCE(mm->sc_stat.next_scan);
+	if (time_before(now, next_scan))
+		return;
+
+	/* only 1 thread is allowed to scan */
+	if (!try_cmpxchg(&mm->sc_stat.next_scan, &next_scan,
+			 now + max_t(unsigned long,
+				     READ_ONCE(llc_epoch_period), 1)))
+		return;
+
+	curr_cpu = task_cpu(p);
+	if (invalid_llc_nr(mm, p, curr_cpu) ||
+	    exceed_llc_capacity(mm, curr_cpu)) {
+		if (READ_ONCE(mm->sc_stat.cpu) != -1)
+			WRITE_ONCE(mm->sc_stat.cpu, -1);
+
+		return;
+	}
+
+	if (!zalloc_cpumask_var(&cpus, GFP_KERNEL))
+		return;
+
+	scoped_guard (cpus_read_lock) {
+		guard(rcu)();
+
+		get_scan_cpumasks(cpus, p);
+
+		for_each_cpu(cpu, cpus) {
+			/* XXX sched_cluster_active */
+			struct sched_domain *sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
+			unsigned long occ, m_occ = 0, a_occ = 0;
+			int m_cpu = -1, i;
+
+			if (!sd)
+				continue;
+
+			for_each_cpu(i, sched_domain_span(sd)) {
+				occ = fraction_mm_sched(cpu_rq(i),
+							per_cpu_ptr(mm->sc_stat.pcpu_sched, i));
+				a_occ += occ;
+				if (occ > m_occ) {
+					m_occ = occ;
+					m_cpu = i;
+				}
+
+				cur = rcu_dereference_all(cpu_rq(i)->curr);
+				if (cur && !(cur->flags & (PF_EXITING | PF_KTHREAD)) &&
+				    cur->mm == mm)
+					nr_running++;
+			}
+
+			/*
+			 * Compare the accumulated occupancy of each LLC. The
+			 * reason for using accumulated occupancy rather than average
+			 * per CPU occupancy is that it works better in asymmetric LLC
+			 * scenarios.
+			 * For example, if there are 2 threads in a 4CPU LLC and 3
+			 * threads in an 8CPU LLC, it might be better to choose the one
+			 * with 3 threads. However, this would not be the case if the
+			 * occupancy is divided by the number of CPUs in an LLC (i.e.,
+			 * if average per CPU occupancy is used).
+			 * Besides, NUMA balancing fault statistics behave similarly:
+			 * the total number of faults per node is compared rather than
+			 * the average number of faults per CPU. This strategy is also
+			 * followed here.
+			 */
+			if (a_occ > m_a_occ) {
+				m_a_occ = a_occ;
+				m_a_cpu = m_cpu;
+			}
+
+			if (llc_id(cpu) == llc_id(READ_ONCE(mm->sc_stat.cpu)))
+				curr_m_a_occ = a_occ;
+
+			cpumask_andnot(cpus, cpus, sched_domain_span(sd));
+		}
+	}
+
+	if (m_a_occ > (2 * curr_m_a_occ)) {
+		/*
+		 * Avoid switching sc_stat.cpu too fast.
+		 * The reason to choose 2X is because:
+		 * 1. It is better to keep the preferred LLC stable,
+		 *    rather than changing it frequently and cause migrations
+		 * 2. 2X means the new preferred LLC has at least 1 more
+		 *    busy CPU than the old one(200% vs 100%, eg)
+		 * 3. 2X is chosen based on test results, as it delivers
+		 *    the optimal performance gain so far.
+		 */
+		WRITE_ONCE(mm->sc_stat.cpu, m_a_cpu);
+	}
+
+	update_avg_scale(&mm->sc_stat.nr_running_avg, nr_running);
+	free_cpumask_var(cpus);
+}
+
+void init_sched_mm(struct task_struct *p)
+{
+	struct callback_head *work = &p->cache_work;
+
+	init_task_work(work, task_cache_work);
+	work->next = work;
+	/*
+	 * Reset new task's preference to avoid
+	 * polluting account_llc_enqueue().
+	 */
+	p->preferred_llc = -1;
+}
+
+#else /* CONFIG_SCHED_CACHE */
+
+static inline void account_mm_sched(struct rq *rq, struct task_struct *p,
+				    s64 delta_exec) { }
+
+void init_sched_mm(struct task_struct *p) { }
+
+static void task_tick_cache(struct rq *rq, struct task_struct *p) { }
+
+static inline int get_pref_llc(struct task_struct *p,
+			       struct mm_struct *mm)
+{
+	return -1;
+}
+
+static void account_llc_enqueue(struct rq *rq, struct task_struct *p) {}
+
+static void account_llc_dequeue(struct rq *rq, struct task_struct *p) {}
+
+#endif /* CONFIG_SCHED_CACHE */
+
 /*
  * Used by other classes to account runtime.
  */
@@ -1578,13 +2156,9 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
 	se->exec_start = rq_clock_task(rq_of(cfs_rq));
 }
 
-/**************************************************
- * Scheduling class queueing methods:
- */
-
+/* Check sched_smt_active before calling this to avoid overheads in fastpaths */
 static inline bool is_core_idle(int cpu)
 {
-#ifdef CONFIG_SCHED_SMT
 	int sibling;
 
 	for_each_cpu(sibling, cpu_smt_mask(cpu)) {
@@ -1594,7 +2168,6 @@ static inline bool is_core_idle(int cpu)
 		if (!idle_cpu(sibling))
 			return false;
 	}
-#endif
 
 	return true;
 }
@@ -2277,12 +2850,11 @@ numa_type numa_classify(unsigned int imbalance_pct,
 	return node_fully_busy;
 }
 
-#ifdef CONFIG_SCHED_SMT
 /* Forward declarations of select_idle_sibling helpers */
 static inline bool test_idle_cores(int cpu);
 static inline int numa_idle_core(int idle_core, int cpu)
 {
-	if (!static_branch_likely(&sched_smt_present) ||
+	if (!sched_smt_active() ||
 	    idle_core >= 0 || !test_idle_cores(cpu))
 		return idle_core;
 
@@ -2295,12 +2867,6 @@ static inline int numa_idle_core(int idle_core, int cpu)
 
 	return idle_core;
 }
-#else /* !CONFIG_SCHED_SMT: */
-static inline int numa_idle_core(int idle_core, int cpu)
-{
-	return idle_core;
-}
-#endif /* !CONFIG_SCHED_SMT */
 
 /*
  * Gather all necessary information to make NUMA balancing placement
@@ -3079,6 +3645,7 @@ static void task_numa_placement(struct task_struct *p)
 	unsigned long total_faults;
 	u64 runtime, period;
 	spinlock_t *group_lock = NULL;
+	long __maybe_unused new_fp;
 	struct numa_group *ng;
 
 	/*
@@ -3153,6 +3720,31 @@ static void task_numa_placement(struct task_struct *p)
 				ng->total_faults += diff;
 				group_faults += ng->faults[mem_idx];
 			}
+#ifdef CONFIG_SCHED_CACHE
+			/*
+			 * Per task p->numa_faults[mem_idx] converges,
+			 * so the accumulation of each task's faults
+			 * converges too - Given the number of threads,
+			 * it cannot overflow an unsigned long.
+			 * Racy with concurrent updates from other threads
+			 * sharing this mm. Acceptable since footprint is a
+			 * heuristic and occasional lost updates are tolerable.
+			 *
+			 * If a task exits, its corresponding footprint must
+			 * be subtracted from the mm->sc_stat.footprint, otherwise
+			 * the mm->sc_stat.footprint will not converge:
+			 * the exiting thread's footprint remains unchanged/undecayed
+			 * in mm->sc_stat.footprint. See exit_mm().
+			 *
+			 * Lost updates and unsynchronized subtraction
+			 * in exit_mm() can cause footprint + diff to
+			 * go negative. Clamp to zero to prevent the
+			 * unsigned footprint from wrapping.
+			 */
+			new_fp = (long)READ_ONCE(p->mm->sc_stat.footprint) + diff;
+			WRITE_ONCE(p->mm->sc_stat.footprint,
+				   max(new_fp, 0L));
+#endif
 		}
 
 		if (!ng) {
@@ -3877,9 +4469,11 @@ account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	update_load_add(&cfs_rq->load, se->load.weight);
 	if (entity_is_task(se)) {
+		struct task_struct *p = task_of(se);
 		struct rq *rq = rq_of(cfs_rq);
 
-		account_numa_enqueue(rq, task_of(se));
+		account_numa_enqueue(rq, p);
+		account_llc_enqueue(rq, p);
 		list_add(&se->group_node, &rq->cfs_tasks);
 	}
 	cfs_rq->nr_queued++;
@@ -3890,7 +4484,11 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	update_load_sub(&cfs_rq->load, se->load.weight);
 	if (entity_is_task(se)) {
-		account_numa_dequeue(rq_of(cfs_rq), task_of(se));
+		struct task_struct *p = task_of(se);
+		struct rq *rq = rq_of(cfs_rq);
+
+		account_numa_dequeue(rq, p);
+		account_llc_dequeue(rq, p);
 		list_del_init(&se->group_node);
 	}
 	cfs_rq->nr_queued--;
@@ -4959,13 +5557,86 @@ static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
 	trace_pelt_cfs_tp(cfs_rq);
 }
 
+#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100)
+
+static inline void util_est_update(struct sched_entity *se)
+{
+	unsigned int ewma, dequeued, last_ewma_diff;
+
+	if (!sched_feat(UTIL_EST))
+		return;
+
+	/* Get current estimate of utilization */
+	ewma = READ_ONCE(se->avg.util_est);
+
+	/*
+	 * If the PELT values haven't changed since enqueue time,
+	 * skip the util_est update.
+	 */
+	if (ewma & UTIL_AVG_UNCHANGED)
+		return;
+
+	/* Get utilization at dequeue */
+	dequeued = READ_ONCE(se->avg.util_avg);
+
+	/*
+	 * Reset EWMA on utilization increases, the moving average is used only
+	 * to smooth utilization decreases.
+	 */
+	if (ewma <= dequeued) {
+		ewma = dequeued;
+		goto done;
+	}
+
+	/*
+	 * Skip update of task's estimated utilization when its members are
+	 * already ~1% close to its last activation value.
+	 */
+	last_ewma_diff = ewma - dequeued;
+	if (last_ewma_diff < UTIL_EST_MARGIN)
+		goto done;
+
+	/*
+	 * To avoid underestimate of task utilization, skip updates of EWMA if
+	 * we cannot grant that thread got all CPU time it wanted.
+	 */
+	if ((dequeued + UTIL_EST_MARGIN) < READ_ONCE(se->avg.runnable_avg))
+		goto done;
+
+	/*
+	 * Update Task's estimated utilization
+	 *
+	 * When *p completes an activation we can consolidate another sample
+	 * of the task size. This is done by using this value to update the
+	 * Exponential Weighted Moving Average (EWMA):
+	 *
+	 *  ewma(t) = w *  task_util(p) + (1-w) * ewma(t-1)
+	 *          = w *  task_util(p) +         ewma(t-1)  - w * ewma(t-1)
+	 *          = w * (task_util(p) -         ewma(t-1)) +     ewma(t-1)
+	 *          = w * (      -last_ewma_diff           ) +     ewma(t-1)
+	 *          = w * (-last_ewma_diff +  ewma(t-1) / w)
+	 *
+	 * Where 'w' is the weight of new samples, which is configured to be
+	 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
+	 */
+	ewma <<= UTIL_EST_WEIGHT_SHIFT;
+	ewma  -= last_ewma_diff;
+	ewma >>= UTIL_EST_WEIGHT_SHIFT;
+done:
+	ewma |= UTIL_AVG_UNCHANGED;
+	WRITE_ONCE(se->avg.util_est, ewma);
+
+	trace_sched_util_est_se_tp(se);
+}
+
 /*
  * Optional action to be done while updating the load average
  */
-#define UPDATE_TG	0x1
-#define SKIP_AGE_LOAD	0x2
-#define DO_ATTACH	0x4
-#define DO_DETACH	0x8
+#define UPDATE_TG	0x01
+#define SKIP_AGE_LOAD	0x02
+#define DO_ATTACH	0x04
+#define DO_DETACH	0x08
+#define UPDATE_UTIL_EST	0x10
 
 /* Update task and its cfs_rq load average */
 static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
@@ -5008,6 +5679,9 @@ static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s
 		if (flags & UPDATE_TG)
 			update_tg_load_avg(cfs_rq);
 	}
+
+	if (flags & UPDATE_UTIL_EST)
+		util_est_update(se);
 }
 
 /*
@@ -5066,11 +5740,6 @@ static inline unsigned long task_util(struct task_struct *p)
 	return READ_ONCE(p->se.avg.util_avg);
 }
 
-static inline unsigned long task_runnable(struct task_struct *p)
-{
-	return READ_ONCE(p->se.avg.runnable_avg);
-}
-
 static inline unsigned long _task_util_est(struct task_struct *p)
 {
 	return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED;
@@ -5113,88 +5782,6 @@ static inline void util_est_dequeue(struct cfs_rq *cfs_rq,
 	trace_sched_util_est_cfs_tp(cfs_rq);
 }
 
-#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100)
-
-static inline void util_est_update(struct cfs_rq *cfs_rq,
-				   struct task_struct *p,
-				   bool task_sleep)
-{
-	unsigned int ewma, dequeued, last_ewma_diff;
-
-	if (!sched_feat(UTIL_EST))
-		return;
-
-	/*
-	 * Skip update of task's estimated utilization when the task has not
-	 * yet completed an activation, e.g. being migrated.
-	 */
-	if (!task_sleep)
-		return;
-
-	/* Get current estimate of utilization */
-	ewma = READ_ONCE(p->se.avg.util_est);
-
-	/*
-	 * If the PELT values haven't changed since enqueue time,
-	 * skip the util_est update.
-	 */
-	if (ewma & UTIL_AVG_UNCHANGED)
-		return;
-
-	/* Get utilization at dequeue */
-	dequeued = task_util(p);
-
-	/*
-	 * Reset EWMA on utilization increases, the moving average is used only
-	 * to smooth utilization decreases.
-	 */
-	if (ewma <= dequeued) {
-		ewma = dequeued;
-		goto done;
-	}
-
-	/*
-	 * Skip update of task's estimated utilization when its members are
-	 * already ~1% close to its last activation value.
-	 */
-	last_ewma_diff = ewma - dequeued;
-	if (last_ewma_diff < UTIL_EST_MARGIN)
-		goto done;
-
-	/*
-	 * To avoid underestimate of task utilization, skip updates of EWMA if
-	 * we cannot grant that thread got all CPU time it wanted.
-	 */
-	if ((dequeued + UTIL_EST_MARGIN) < task_runnable(p))
-		goto done;
-
-
-	/*
-	 * Update Task's estimated utilization
-	 *
-	 * When *p completes an activation we can consolidate another sample
-	 * of the task size. This is done by using this value to update the
-	 * Exponential Weighted Moving Average (EWMA):
-	 *
-	 *  ewma(t) = w *  task_util(p) + (1-w) * ewma(t-1)
-	 *          = w *  task_util(p) +         ewma(t-1)  - w * ewma(t-1)
-	 *          = w * (task_util(p) -         ewma(t-1)) +     ewma(t-1)
-	 *          = w * (      -last_ewma_diff           ) +     ewma(t-1)
-	 *          = w * (-last_ewma_diff +  ewma(t-1) / w)
-	 *
-	 * Where 'w' is the weight of new samples, which is configured to be
-	 * 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
-	 */
-	ewma <<= UTIL_EST_WEIGHT_SHIFT;
-	ewma  -= last_ewma_diff;
-	ewma >>= UTIL_EST_WEIGHT_SHIFT;
-done:
-	ewma |= UTIL_AVG_UNCHANGED;
-	WRITE_ONCE(p->se.avg.util_est, ewma);
-
-	trace_sched_util_est_se_tp(&p->se);
-}
-
 static inline unsigned long get_actual_cpu_capacity(int cpu)
 {
 	unsigned long capacity = arch_scale_cpu_capacity(cpu);
@@ -5647,7 +6234,7 @@ static bool
 dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 {
 	bool sleep = flags & DEQUEUE_SLEEP;
-	int action = UPDATE_TG;
+	int action = 0;
 
 	update_curr(cfs_rq);
 	clear_buddies(cfs_rq, se);
@@ -5667,15 +6254,23 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 
 		if (sched_feat(DELAY_DEQUEUE) && delay &&
 		    !entity_eligible(cfs_rq, se)) {
-			update_load_avg(cfs_rq, se, 0);
+			if (entity_is_task(se))
+				action |= UPDATE_UTIL_EST;
+			update_load_avg(cfs_rq, se, action);
 			update_entity_lag(cfs_rq, se);
 			set_delayed(se);
 			return false;
 		}
 	}
 
-	if (entity_is_task(se) && task_on_rq_migrating(task_of(se)))
-		action |= DO_DETACH;
+	action = UPDATE_TG;
+	if (entity_is_task(se)) {
+		if (task_on_rq_migrating(task_of(se)))
+			action |= DO_DETACH;
+
+		if (sleep && !(flags & DEQUEUE_DELAYED))
+			action |= UPDATE_UTIL_EST;
+	}
 
 	/*
 	 * When dequeuing a sched_entity, we must:
@@ -7438,7 +8033,6 @@ static bool dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	if (!p->se.sched_delayed)
 		util_est_dequeue(&rq->cfs, p);
 
-	util_est_update(&rq->cfs, p, flags & DEQUEUE_SLEEP);
 	if (dequeue_entities(rq, &p->se, flags) < 0)
 		return false;
 
@@ -7811,7 +8405,6 @@ static inline int __select_idle_cpu(int cpu, struct task_struct *p)
 	return -1;
 }
 
-#ifdef CONFIG_SCHED_SMT
 DEFINE_STATIC_KEY_FALSE(sched_smt_present);
 EXPORT_SYMBOL_GPL(sched_smt_present);
 
@@ -7819,7 +8412,7 @@ static inline void set_idle_cores(int cpu, int val)
 {
 	struct sched_domain_shared *sds;
 
-	sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+	sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu));
 	if (sds)
 		WRITE_ONCE(sds->has_idle_cores, val);
 }
@@ -7828,7 +8421,7 @@ static inline bool test_idle_cores(int cpu)
 {
 	struct sched_domain_shared *sds;
 
-	sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+	sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu));
 	if (sds)
 		return READ_ONCE(sds->has_idle_cores);
 
@@ -7837,7 +8430,7 @@ static inline bool test_idle_cores(int cpu)
 
 /*
  * Scans the local SMT mask to see if the entire core is idle, and records this
- * information in sd_llc_shared->has_idle_cores.
+ * information in sd_balance_shared->has_idle_cores.
  *
  * Since SMT siblings share all cache levels, inspecting this limited remote
  * state should be fairly cheap.
@@ -7867,7 +8460,8 @@ void __update_idle_core(struct rq *rq)
 /*
  * Scan the entire LLC domain for idle cores; this dynamically switches off if
  * there are no idle cores left in the system; tracked through
- * sd_llc->shared->has_idle_cores and enabled through update_idle_core() above.
+ * sd_balance_shared->has_idle_cores and enabled through update_idle_core()
+ * above.
  */
 static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
 {
@@ -7921,29 +8515,6 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t
 	return -1;
 }
 
-#else /* !CONFIG_SCHED_SMT: */
-
-static inline void set_idle_cores(int cpu, int val)
-{
-}
-
-static inline bool test_idle_cores(int cpu)
-{
-	return false;
-}
-
-static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
-{
-	return __select_idle_cpu(core, p);
-}
-
-static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
-{
-	return -1;
-}
-
-#endif /* !CONFIG_SCHED_SMT */
-
 /*
  * Scan the LLC domain for idle CPUs; this is dynamically regulated by
  * comparing the average scan cost (tracked in sd->avg_scan_cost) against the
@@ -7954,7 +8525,7 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
 	struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
 	int i, cpu, idle_cpu = -1, nr = INT_MAX;
 
-	if (sched_feat(SIS_UTIL)) {
+	if (sched_feat(SIS_UTIL) && sd->shared) {
 		/*
 		 * Increment because !--nr is the condition to stop scan.
 		 *
@@ -8018,6 +8589,54 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
 	return idle_cpu;
 }
 
+/*
+ * Idle-capacity scan converts util_fits_cpu() outcomes into preference ranks,
+ * where lower values indicate a better fit - see select_idle_capacity().
+ *
+ * A CPU that both fits the task and sits on a fully-idle SMT core is returned
+ * immediately and is never assigned one of these ranks. On !SMT every CPU is
+ * its own "core", so the early return covers all fits-and-idle cases and the
+ * core-tier ranks below become unreachable.
+ *
+ *   Rank                            Val  Tier    Meaning
+ *   ------------------------------  ---  ------  ---------------------------
+ *   ASYM_IDLE_UCLAMP_MISFIT         -4   core    Idle core; capacity fits
+ *                                                util but uclamp_min misses.
+ *   ASYM_IDLE_COMPLETE_MISFIT       -3   core    Idle core; capacity does
+ *                                                not fit. Still beats every
+ *                                                thread-tier rank: a busy
+ *                                                sibling cuts effective
+ *                                                capacity more than a
+ *                                                misfit hurts a quiet core.
+ *   ASYM_IDLE_THREAD_FITS           -2   thread  Busy SMT sibling; capacity
+ *                                                fits util + uclamp.
+ *   ASYM_IDLE_THREAD_UCLAMP_MISFIT  -1   thread  Busy SMT sibling; capacity
+ *                                                fits but uclamp_min misses
+ *                                                (native util_fits_cpu()
+ *                                                return value).
+ *   ASYM_IDLE_THREAD_MISFIT          0   thread  Busy SMT sibling; capacity
+ *                                                does not fit.
+ *
+ * ASYM_IDLE_CORE_BIAS (-3) is an offset, not a state. On an idle core,
+ * fits += ASYM_IDLE_CORE_BIAS rebases thread-tier ranks into the core tier:
+ *
+ *   ASYM_IDLE_THREAD_UCLAMP_MISFIT (-1) + BIAS -> ASYM_IDLE_UCLAMP_MISFIT   (-4)
+ *   ASYM_IDLE_THREAD_MISFIT         (0) + BIAS -> ASYM_IDLE_COMPLETE_MISFIT (-3)
+ *
+ * ASYM_IDLE_THREAD_FITS (-2) is never rebased because a fully-fitting idle-core
+ * candidate early-returns from select_idle_capacity().
+ */
+enum asym_fits_state {
+	ASYM_IDLE_UCLAMP_MISFIT = -4,
+	ASYM_IDLE_COMPLETE_MISFIT,
+	ASYM_IDLE_THREAD_FITS,
+	ASYM_IDLE_THREAD_UCLAMP_MISFIT,
+	ASYM_IDLE_THREAD_MISFIT,
+
+	/* util_fits_cpu() bias for idle core */
+	ASYM_IDLE_CORE_BIAS = -3,
+};
+
 /*
  * Scan the asym_capacity domain for idle CPUs; pick the first idle one on which
  * the task fits. If no CPU is big enough, but there are idle ones, try to
@@ -8026,10 +8645,17 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool
 static int
 select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
 {
+	/*
+	 * On !SMT systems, has_idle_core is always false and preferred_core
+	 * is always true (CPU == core), so the SMT preference logic below
+	 * collapses to the plain capacity scan.
+	 */
+	bool has_idle_core = sched_smt_active() && test_idle_cores(target);
 	unsigned long task_util, util_min, util_max, best_cap = 0;
-	int fits, best_fits = 0;
+	int fits, best_fits = ASYM_IDLE_THREAD_MISFIT;
 	int cpu, best_cpu = -1;
 	struct cpumask *cpus;
+	int nr = INT_MAX;
 
 	cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
 	cpumask_and(cpus, sched_domain_span(sd), p->cpus_ptr);
@@ -8038,16 +8664,41 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
 	util_min = uclamp_eff_value(p, UCLAMP_MIN);
 	util_max = uclamp_eff_value(p, UCLAMP_MAX);
 
+	if (sched_feat(SIS_UTIL) && sd->shared) {
+		/*
+		 * Same nr_idle_scan hint as select_idle_cpu(), nr only limits
+		 * the scan when not preferring an idle core.
+		 */
+		nr = READ_ONCE(sd->shared->nr_idle_scan) + 1;
+		/* overloaded domain is unlikely to have idle cpu/core */
+		if (nr == 1)
+			return -1;
+	}
+
 	for_each_cpu_wrap(cpu, cpus, target) {
+		bool preferred_core = !has_idle_core || is_core_idle(cpu);
 		unsigned long cpu_cap = capacity_of(cpu);
 
+		/*
+		 * Stop when the nr_idle_scan is exhausted (mirrors
+		 * select_idle_cpu() logic).
+		 */
+		if (!has_idle_core && --nr <= 0)
+			return best_cpu;
+
 		if (!choose_idle_cpu(cpu, p))
 			continue;
 
 		fits = util_fits_cpu(task_util, util_min, util_max, cpu);
 
-		/* This CPU fits with all requirements */
-		if (fits > 0)
+		/*
+		 * Perfect fit: capacity satisfies util + uclamp and the CPU
+		 * sits on a fully-idle SMT core, this is a !SMT system, or
+		 * there is no idle core to find.
+		 * Short-circuit the rank-based selection and return
+		 * immediately.
+		 */
+		if (fits > 0 && preferred_core)
 			return cpu;
 		/*
 		 * Only the min performance hint (i.e. uclamp_min) doesn't fit.
@@ -8055,9 +8706,33 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
 		 */
 		else if (fits < 0)
 			cpu_cap = get_actual_cpu_capacity(cpu);
+		/*
+		 * fits > 0 implies we are not on a preferred core, but the util
+		 * fits CPU capacity. Set fits to ASYM_IDLE_THREAD_FITS
+		 * so the effective range becomes
+		 * [ASYM_IDLE_THREAD_FITS, ASYM_IDLE_THREAD_MISFIT], where:
+		 *    ASYM_IDLE_THREAD_MISFIT - does not fit
+		 *    ASYM_IDLE_THREAD_UCLAMP_MISFIT - fits with the exception of UCLAMP_MIN
+		 *    ASYM_IDLE_THREAD_FITS - fits with the exception of preferred_core
+		 */
+		else if (fits > 0)
+			fits = ASYM_IDLE_THREAD_FITS;
+
+		/*
+		 * If we are on a preferred core, translate the range of fits
+		 * of [ASYM_IDLE_THREAD_UCLAMP_MISFIT, ASYM_IDLE_THREAD_MISFIT] to
+		 * [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT].
+		 * This ensures that an idle core is always given priority over
+		 * (partially) busy core.
+		 *
+		 * A fully fitting idle core would have returned early and hence
+		 * fits > 0 for preferred_core need not be dealt with.
+		 */
+		if (preferred_core)
+			fits += ASYM_IDLE_CORE_BIAS;
 
 		/*
-		 * First, select CPU which fits better (-1 being better than 0).
+		 * First, select CPU which fits better (lower is more preferred).
 		 * Then, select the one with best capacity at same level.
 		 */
 		if ((fits < best_fits) ||
@@ -8068,6 +8743,19 @@ select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
 		}
 	}
 
+	/*
+	 * A value in the [ASYM_IDLE_UCLAMP_MISFIT, ASYM_IDLE_COMPLETE_MISFIT]
+	 * range means the chosen CPU is in a fully idle SMT core. Values above
+	 * ASYM_IDLE_COMPLETE_MISFIT mean we never ranked such a CPU best.
+	 *
+	 * The asym-capacity wakeup path returns from select_idle_sibling()
+	 * after this function and never runs select_idle_cpu(), so the usual
+	 * select_idle_cpu() tail that clears idle cores must live here when the
+	 * idle-core preference did not win.
+	 */
+	if (has_idle_core && best_fits > ASYM_IDLE_COMPLETE_MISFIT)
+		set_idle_cores(target, false);
+
 	return best_cpu;
 }
 
@@ -8076,12 +8764,22 @@ static inline bool asym_fits_cpu(unsigned long util,
 				 unsigned long util_max,
 				 int cpu)
 {
-	if (sched_asym_cpucap_active())
+	if (sched_asym_cpucap_active()) {
 		/*
 		 * Return true only if the cpu fully fits the task requirements
 		 * which include the utilization and the performance hints.
+		 *
+		 * When SMT is active, also require that the core has no busy
+		 * siblings.
+		 *
+		 * Note: gating on is_core_idle() also makes the early-bailout
+		 * candidates in select_idle_sibling() (target, prev,
+		 * recent_used_cpu) idle-core-aware on ASYM+SMT, which the
+		 * NO_ASYM path does not do.
 		 */
-		return (util_fits_cpu(util, util_min, util_max, cpu) > 0);
+		return (!sched_smt_active() || is_core_idle(cpu)) &&
+		       (util_fits_cpu(util, util_min, util_max, cpu) > 0);
+	}
 
 	return true;
 }
@@ -9558,6 +10256,16 @@ enum group_type {
 	 * from balancing the load across the system.
 	 */
 	group_imbalanced,
+	/*
+	 * There are tasks running on non-preferred LLC, possible to move
+	 * them to their preferred LLC without creating too much imbalance.
+	 * The priority of group_llc_balance is lower than that of
+	 * group_overloaded and higher than that of all other group types.
+	 * This is because group_llc_balance may exacerbate load imbalance.
+	 * If the LLC balancing attempt fails, the nr_balance_failed
+	 * mechanism will trigger other group types to rebalance the load.
+	 */
+	group_llc_balance,
 	/*
 	 * The CPU is overloaded and can't provide expected CPU cycles to all
 	 * tasks.
@@ -9569,7 +10277,8 @@ enum migration_type {
 	migrate_load = 0,
 	migrate_util,
 	migrate_task,
-	migrate_misfit
+	migrate_misfit,
+	migrate_llc_task
 };
 
 #define LBF_ALL_PINNED	0x01
@@ -9577,6 +10286,7 @@ enum migration_type {
 #define LBF_DST_PINNED  0x04
 #define LBF_SOME_PINNED	0x08
 #define LBF_ACTIVE_LB	0x10
+#define LBF_LLC_PINNED	0x20
 
 struct lb_env {
 	struct sched_domain	*sd;
@@ -9586,6 +10296,7 @@ struct lb_env {
 
 	int			dst_cpu;
 	struct rq		*dst_rq;
+	bool			dst_core_idle;
 
 	struct cpumask		*dst_grpmask;
 	int			new_dst_cpu;
@@ -9733,6 +10444,298 @@ static inline int task_is_ineligible_on_dst_cpu(struct task_struct *p, int dest_
 	return 0;
 }
 
+#ifdef CONFIG_SCHED_CACHE
+/*
+ * The margin used when comparing LLC utilization with CPU capacity.
+ * It determines the LLC load level where active LLC aggregation is
+ * done.
+ * Derived from fits_capacity().
+ *
+ * (default: ~50%, tunable via debugfs)
+ */
+static bool fits_llc_capacity(unsigned long util, unsigned long max)
+{
+	u32 aggr_pct = llc_overaggr_pct;
+
+	/*
+	 * For single core systems, raise the aggregation
+	 * threshold to accommodate more tasks.
+	 */
+	if (cpu_smt_num_threads == 1)
+		aggr_pct = (aggr_pct * 3 / 2);
+
+	return util * 100 < max * aggr_pct;
+}
+
+/*
+ * The margin used when comparing utilization.
+ * is 'util1' noticeably greater than 'util2'
+ * Derived from capacity_greater().
+ * Bias is in perentage.
+ */
+/* Allows dst util to be bigger than src util by up to bias percent */
+#define util_greater(util1, util2) \
+	((util1) * 100 > (util2) * (100 + llc_imb_pct))
+
+static __maybe_unused bool get_llc_stats(int cpu, unsigned long *util,
+					 unsigned long *cap)
+{
+	struct sched_domain_shared *sd_share;
+
+	sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+	if (!sd_share)
+		return false;
+
+	*util = READ_ONCE(sd_share->util_avg);
+	*cap = READ_ONCE(sd_share->capacity);
+
+	return true;
+}
+
+/*
+ * Decision matrix according to the LLC utilization. To
+ * decide whether we can do task aggregation across LLC.
+ *
+ * By default, 50% is the threshold for treating the LLC
+ * as busy. The reason for choosing 50% is to avoid saturation
+ * of SMT-2, and it is also a safe cutoff for other SMT-n
+ * platforms. SMT-1 has higher threshold because it is
+ * supposed to accommodate more tasks, see fits_llc_capacity().
+ *
+ * 20% is the utilization imbalance percentage to decide
+ * if the preferred LLC is busier than the non-preferred LLC.
+ * 20 is a little higher than the LLC domain's imbalance_pct
+ * 17. The hysteresis is used to avoid task bouncing between the
+ * preferred LLC and the non-preferred LLC, and it will
+ * be turned into tunable debugfs.
+ *
+ * 1. moving towards the preferred LLC, dst is the preferred
+ *    LLC, src is not.
+ *
+ * src \ dst      30%  40%  50%  60%
+ * 30%            Y    Y    Y    N
+ * 40%            Y    Y    Y    Y
+ * 50%            Y    Y    G    G
+ * 60%            Y    Y    G    G
+ *
+ * 2. moving out of the preferred LLC, src is the preferred
+ *    LLC, dst is not:
+ *
+ * src \ dst      30%  40%  50%  60%
+ * 30%            N    N    N    N
+ * 40%            N    N    N    N
+ * 50%            N    N    G    G
+ * 60%            Y    N    G    G
+ *
+ * src :      src_util
+ * dst :      dst_util
+ * Y :        Yes, migrate
+ * N :        No, do not migrate
+ * G :        let the Generic load balance to even the load.
+ *
+ * The intention is that if both LLCs are quite busy, cache aware
+ * load balance should not be performed, and generic load balance
+ * should take effect. However, if one is busy and the other is not,
+ * the preferred LLC capacity(50%) and imbalance criteria(20%) should
+ * be considered to determine whether LLC aggregation should be
+ * performed to bias the load towards the preferred LLC.
+ */
+
+/* migration decision, 3 states are orthogonal. */
+enum llc_mig {
+	mig_forbid = 0,		/* N: Don't migrate task, respect LLC preference */
+	mig_llc,		/* Y: Do LLC preference based migration */
+	mig_unrestricted	/* G: Don't restrict generic load balance migration */
+};
+
+/*
+ * Check if task can be moved from the source LLC to the
+ * destination LLC without breaking cache aware preferrence.
+ * src_cpu and dst_cpu are arbitrary CPUs within the source
+ * and destination LLCs, respectively.
+ */
+static enum llc_mig can_migrate_llc(int src_cpu, int dst_cpu,
+				    unsigned long tsk_util,
+				    bool to_pref)
+{
+	unsigned long src_util, dst_util, src_cap, dst_cap;
+
+	if (!get_llc_stats(src_cpu, &src_util, &src_cap) ||
+	    !get_llc_stats(dst_cpu, &dst_util, &dst_cap))
+		return mig_unrestricted;
+
+	src_util = src_util < tsk_util ? 0 : src_util - tsk_util;
+	dst_util = dst_util + tsk_util;
+
+	if (!fits_llc_capacity(dst_util, dst_cap) &&
+	    !fits_llc_capacity(src_util, src_cap))
+		return mig_unrestricted;
+
+	if (to_pref) {
+		/*
+		 * Don't migrate if we will get preferred LLC too
+		 * heavily loaded and if the dest is much busier
+		 * than the src, in which case migration will
+		 * increase the imbalance too much.
+		 */
+		if (!fits_llc_capacity(dst_util, dst_cap) &&
+		    util_greater(dst_util, src_util))
+			return mig_forbid;
+	} else {
+		/*
+		 * Don't migrate if we will leave preferred LLC
+		 * too idle, or if this migration leads to the
+		 * non-preferred LLC falls within sysctl_aggr_imb percent
+		 * of preferred LLC, leading to migration again
+		 * back to preferred LLC.
+		 */
+		if (fits_llc_capacity(src_util, src_cap) ||
+		    !util_greater(src_util, dst_util))
+			return mig_forbid;
+	}
+	return mig_llc;
+}
+
+/*
+ * Check if task p can migrate from source LLC to
+ * destination LLC in terms of cache aware load balance.
+ */
+static enum llc_mig can_migrate_llc_task(int src_cpu, int dst_cpu,
+					 struct task_struct *p)
+{
+	struct mm_struct *mm;
+	bool to_pref;
+	int cpu;
+
+	mm = p->mm;
+	if (!mm)
+		return mig_unrestricted;
+
+	cpu = READ_ONCE(mm->sc_stat.cpu);
+	if (cpu < 0 || cpus_share_cache(src_cpu, dst_cpu))
+		return mig_unrestricted;
+
+	/* skip cache aware load balance for too many threads */
+	if (invalid_llc_nr(mm, p, dst_cpu) ||
+	    exceed_llc_capacity(mm, dst_cpu)) {
+		if (READ_ONCE(mm->sc_stat.cpu) != -1)
+			WRITE_ONCE(mm->sc_stat.cpu, -1);
+		return mig_unrestricted;
+	}
+
+	if (cpus_share_cache(dst_cpu, cpu))
+		to_pref = true;
+	else if (cpus_share_cache(src_cpu, cpu))
+		to_pref = false;
+	else
+		return mig_unrestricted;
+
+	return can_migrate_llc(src_cpu, dst_cpu,
+			       task_util(p), to_pref);
+}
+
+/*
+ * Check if active load balance breaks LLC locality in
+ * terms of cache aware load balance. The load level and
+ * imbalance do not warrant breaking LLC preference per
+ * the can_migrate_llc() policy. Here, the benefit of
+ * LLC locality outweighs the power efficiency gained from
+ * migrating the only runnable task away.
+ */
+static inline bool
+alb_break_llc(struct lb_env *env)
+{
+	if (!sched_cache_enabled())
+		return false;
+
+	if (cpus_share_cache(env->src_cpu, env->dst_cpu))
+		return false;
+	/*
+	 * All tasks prefer to stay on their current CPU.
+	 * Do not pull a task from its preferred CPU if:
+	 * 1. It is the only task running and does not exceed
+	 *    imbalance allowance; OR
+	 * 2. Migrating it away from its preferred LLC would violate
+	 *    the cache-aware scheduling policy.
+	 */
+	if (env->src_rq->nr_pref_llc_running &&
+	    env->src_rq->nr_pref_llc_running == env->src_rq->cfs.h_nr_runnable) {
+		unsigned long util = 0;
+		struct task_struct *cur;
+
+		if (env->src_rq->nr_running <= 1)
+			return true;
+
+		cur = rcu_dereference_all(env->src_rq->curr);
+		if (cur && cur->sched_class == &fair_sched_class)
+			util = task_util(cur);
+
+		if (can_migrate_llc(env->src_cpu, env->dst_cpu,
+				    util, false) == mig_forbid)
+			return true;
+	}
+
+	return false;
+}
+
+/*
+ * Check if migrating task p from env->src_cpu to
+ * env->dst_cpu breaks LLC localiy.
+ */
+static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
+{
+	if (!sched_cache_enabled())
+		return false;
+
+	if (task_has_sched_core(p))
+		return false;
+	/*
+	 * Skip over tasks that would degrade LLC locality;
+	 * only when nr_balanced_failed is sufficiently high do we
+	 * ignore this constraint.
+	 *
+	 * Threshold of cache_nice_tries is set to 1 higher
+	 * than nr_balance_failed to avoid excessive task
+	 * migration at the same time.
+	 */
+	if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1)
+		return false;
+
+	/*
+	 * We know the env->src_cpu has some tasks prefer to
+	 * run on env->dst_cpu, skip the tasks do not prefer
+	 * env->dst_cpu, and find the one that prefers.
+	 */
+	if (env->migration_type == migrate_llc_task &&
+	    READ_ONCE(p->preferred_llc) != llc_id(env->dst_cpu))
+		return true;
+
+	if (can_migrate_llc_task(env->src_cpu,
+				 env->dst_cpu, p) != mig_forbid)
+		return false;
+
+	return true;
+}
+
+#else
+static inline bool get_llc_stats(int cpu, unsigned long *util,
+				 unsigned long *cap)
+{
+	return false;
+}
+
+static inline bool
+alb_break_llc(struct lb_env *env)
+{
+	return false;
+}
+
+static inline bool
+migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
+{
+	return false;
+}
+#endif
 /*
  * can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
  */
@@ -9829,10 +10832,29 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
 		return 1;
 
 	degrades = migrate_degrades_locality(p, env);
-	if (!degrades)
+	if (!degrades) {
+		/*
+		 * If the NUMA locality is not broken,
+		 * further check if migration would hurt
+		 * LLC locality.
+		 */
+		if (migrate_degrades_llc(p, env)) {
+			/*
+			 * If regular load balancing fails to pull a task
+			 * due to LLC locality, this is expected behavior
+			 * and we set LBF_LLC_PINNED so we don't increase
+			 * nr_balance_failed unecessarily.
+			 */
+			if (env->migration_type != migrate_llc_task)
+				env->flags |= LBF_LLC_PINNED;
+
+			return 0;
+		}
+
 		hot = task_hot(p, env);
-	else
+	} else {
 		hot = degrades > 0;
+	}
 
 	if (!hot || env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
 		if (hot)
@@ -9994,6 +11016,10 @@ static int detach_tasks(struct lb_env *env)
 
 			env->imbalance = 0;
 			break;
+
+		case migrate_llc_task:
+			env->imbalance--;
+			break;
 		}
 
 		detach_task(p, env);
@@ -10272,12 +11298,16 @@ struct sg_lb_stats {
 	enum group_type group_type;
 	unsigned int group_asym_packing;	/* Tasks should be moved to preferred CPU */
 	unsigned int group_smt_balance;		/* Task on busy SMT be moved */
+	unsigned int group_llc_balance;		/* Tasks should be moved to preferred LLC */
 	unsigned long group_misfit_task_load;	/* A CPU has a task too big for its capacity */
 	unsigned int group_overutilized;	/* At least one CPU is overutilized in the group */
 #ifdef CONFIG_NUMA_BALANCING
 	unsigned int nr_numa_running;
 	unsigned int nr_preferred_running;
 #endif
+#ifdef CONFIG_SCHED_CACHE
+	unsigned int nr_pref_dst_llc;
+#endif
 };
 
 /*
@@ -10535,6 +11565,9 @@ group_type group_classify(unsigned int imbalance_pct,
 	if (group_is_overloaded(imbalance_pct, sgs))
 		return group_overloaded;
 
+	if (sgs->group_llc_balance)
+		return group_llc_balance;
+
 	if (sg_imbalanced(group))
 		return group_imbalanced;
 
@@ -10689,6 +11722,105 @@ sched_reduced_capacity(struct rq *rq, struct sched_domain *sd)
 	return check_cpu_capacity(rq, sd);
 }
 
+#ifdef CONFIG_SCHED_CACHE
+/*
+ * Record the statistics for this scheduler group for later
+ * use. These values guide load balancing on aggregating tasks
+ * to a LLC.
+ */
+static void record_sg_llc_stats(struct lb_env *env,
+				struct sg_lb_stats *sgs,
+				struct sched_group *group)
+{
+	struct sched_domain_shared *sd_share;
+	int cpu;
+
+	if (!sched_cache_enabled() || env->idle == CPU_NEWLY_IDLE)
+		return;
+
+	/* Only care about sched domain spanning multiple LLCs */
+	if (env->sd->child != rcu_dereference_all(per_cpu(sd_llc, env->dst_cpu)))
+		return;
+
+	/*
+	 * At this point we know this group spans a LLC domain.
+	 * Record the statistic of this group in its corresponding
+	 * shared LLC domain.
+	 * Note: sd_share cannot be obtained via sd->child->shared,
+	 * because the latter refers to the domain that covers the
+	 * local group. Instead, sd_share should be located using
+	 * the first CPU of the LLC group.
+	 */
+	cpu = cpumask_first(sched_group_span(group));
+	sd_share = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+	if (!sd_share)
+		return;
+
+	if (READ_ONCE(sd_share->util_avg) != sgs->group_util)
+		WRITE_ONCE(sd_share->util_avg, sgs->group_util);
+
+	if (unlikely(READ_ONCE(sd_share->capacity) != sgs->group_capacity))
+		WRITE_ONCE(sd_share->capacity, sgs->group_capacity);
+}
+
+/*
+ * Do LLC balance on sched group that contains LLC, and have tasks preferring
+ * to run on LLC in idle dst_cpu.
+ */
+static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs,
+			       struct sched_group *group)
+{
+	if (!sched_cache_enabled())
+		return false;
+
+	if (env->sd->flags & SD_SHARE_LLC)
+		return false;
+
+	/*
+	 * Skip cache aware tagging if nr_balanced_failed is sufficiently high.
+	 * Threshold of cache_nice_tries is set to 1 higher than nr_balance_failed
+	 * to avoid excessive task migration at the same time.
+	 */
+	if (env->sd->nr_balance_failed >= env->sd->cache_nice_tries + 1)
+		return false;
+
+	if (sgs->nr_pref_dst_llc &&
+	    can_migrate_llc(cpumask_first(sched_group_span(group)),
+			    env->dst_cpu, 0, true) == mig_llc)
+		return true;
+
+	return false;
+}
+
+static bool update_llc_busiest(struct lb_env *env,
+			       struct sg_lb_stats *busiest,
+			       struct sg_lb_stats *sgs)
+{
+	/*
+	 * There are more tasks that want to run on dst_cpu's LLC.
+	 */
+	return sgs->nr_pref_dst_llc > busiest->nr_pref_dst_llc;
+}
+#else
+static inline void record_sg_llc_stats(struct lb_env *env, struct sg_lb_stats *sgs,
+				       struct sched_group *group)
+{
+}
+
+static inline bool llc_balance(struct lb_env *env, struct sg_lb_stats *sgs,
+			       struct sched_group *group)
+{
+	return false;
+}
+
+static bool update_llc_busiest(struct lb_env *env,
+			       struct sg_lb_stats *busiest,
+			       struct sg_lb_stats *sgs)
+{
+	return false;
+}
+#endif
+
 /**
  * update_sg_lb_stats - Update sched_group's statistics for load balancing.
  * @env: The load balancing environment.
@@ -10725,6 +11857,20 @@ static inline void update_sg_lb_stats(struct lb_env *env,
 		if (cpu_overutilized(i))
 			sgs->group_overutilized = 1;
 
+#ifdef CONFIG_SCHED_CACHE
+		if (sched_cache_enabled()) {
+			struct sched_domain *sd_tmp;
+			int dst_llc;
+
+			dst_llc = llc_id(env->dst_cpu);
+			if (llc_id(i) != dst_llc) {
+				sd_tmp = rcu_dereference_all(rq->sd);
+				if (sd_tmp && (unsigned int)dst_llc < sd_tmp->llc_max)
+					sgs->nr_pref_dst_llc += sd_tmp->llc_counts[dst_llc];
+			}
+		}
+#endif
+
 		/*
 		 * No need to call idle_cpu() if nr_running is not 0
 		 */
@@ -10765,17 +11911,24 @@ static inline void update_sg_lb_stats(struct lb_env *env,
 
 	sgs->group_weight = group->group_weight;
 
-	/* Check if dst CPU is idle and preferred to this group */
-	if (!local_group && env->idle && sgs->sum_h_nr_running &&
-	    sched_group_asym(env, sgs, group))
-		sgs->group_asym_packing = 1;
+	if (!local_group) {
+		/* Check if dst CPU is idle and preferred to this group */
+		if (env->idle && sgs->sum_h_nr_running &&
+		    sched_group_asym(env, sgs, group))
+			sgs->group_asym_packing = 1;
+
+		/* Check for loaded SMT group to be balanced to dst CPU */
+		if (smt_balance(env, sgs, group))
+			sgs->group_smt_balance = 1;
 
-	/* Check for loaded SMT group to be balanced to dst CPU */
-	if (!local_group && smt_balance(env, sgs, group))
-		sgs->group_smt_balance = 1;
+		/* Check for tasks in this group can be moved to their preferred LLC */
+		if (llc_balance(env, sgs, group))
+			sgs->group_llc_balance = 1;
+	}
 
 	sgs->group_type = group_classify(env->sd->imbalance_pct, group, sgs);
 
+	record_sg_llc_stats(env, sgs, group);
 	/* Computing avg_load makes sense only when group is overloaded */
 	if (sgs->group_type == group_overloaded)
 		sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
@@ -10811,10 +11964,16 @@ static bool update_sd_pick_busiest(struct lb_env *env,
 	 * We can use max_capacity here as reduction in capacity on some
 	 * CPUs in the group should either be possible to resolve
 	 * internally or be covered by avg_load imbalance (eventually).
+	 *
+	 * When SMT is active, only pull a misfit to dst_cpu if it is on a
+	 * fully idle core; otherwise the effective capacity of the core is
+	 * reduced and we may not actually provide more capacity than the
+	 * source.
 	 */
 	if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
 	    (sgs->group_type == group_misfit_task) &&
-	    (!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||
+	    (!env->dst_core_idle ||
+	     !capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||
 	     sds->local_stat.group_type != group_has_spare))
 		return false;
 
@@ -10834,6 +11993,10 @@ static bool update_sd_pick_busiest(struct lb_env *env,
 		/* Select the overloaded group with highest avg_load. */
 		return sgs->avg_load > busiest->avg_load;
 
+	case group_llc_balance:
+		/* Select the group with most tasks preferring dst LLC */
+		return update_llc_busiest(env, busiest, sgs);
+
 	case group_imbalanced:
 		/*
 		 * Select the 1st imbalanced group as we don't have any way to
@@ -11096,6 +12259,7 @@ static bool update_pick_idlest(struct sched_group *idlest,
 			return false;
 		break;
 
+	case group_llc_balance:
 	case group_imbalanced:
 	case group_asym_packing:
 	case group_smt_balance:
@@ -11228,6 +12392,7 @@ sched_balance_find_dst_group(struct sched_domain *sd, struct task_struct *p, int
 			return NULL;
 		break;
 
+	case group_llc_balance:
 	case group_imbalanced:
 	case group_asym_packing:
 	case group_smt_balance:
@@ -11378,6 +12543,8 @@ static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sd
 	unsigned long sum_util = 0;
 	bool sg_overloaded = 0, sg_overutilized = 0;
 
+	env->dst_core_idle = !sched_smt_active() || is_core_idle(env->dst_cpu);
+
 	do {
 		struct sg_lb_stats *sgs = &tmp_sgs;
 		int local_group;
@@ -11480,6 +12647,15 @@ static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *s
 		return;
 	}
 
+#ifdef CONFIG_SCHED_CACHE
+	if (busiest->group_type == group_llc_balance) {
+		/* Move a task that prefer local LLC */
+		env->migration_type = migrate_llc_task;
+		env->imbalance = 1;
+		return;
+	}
+#endif
+
 	if (busiest->group_type == group_imbalanced) {
 		/*
 		 * In the group_imb case we cannot rely on group-wide averages
@@ -11726,7 +12902,8 @@ static struct sched_group *sched_balance_find_src_group(struct lb_env *env)
 	 * group's child domain.
 	 */
 	if (sds.prefer_sibling && local->group_type == group_has_spare &&
-	    sibling_imbalance(env, &sds, busiest, local) > 1)
+	    (busiest->group_type == group_llc_balance ||
+	    sibling_imbalance(env, &sds, busiest, local) > 1))
 		goto force_balance;
 
 	if (busiest->group_type != group_overloaded) {
@@ -11785,7 +12962,10 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env,
 {
 	struct rq *busiest = NULL, *rq;
 	unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1;
+	unsigned int __maybe_unused busiest_pref_llc = 0;
+	struct sched_domain __maybe_unused *sd_tmp;
 	unsigned int busiest_nr = 0;
+	int __maybe_unused dst_llc;
 	int i;
 
 	for_each_cpu_and(i, sched_group_span(group), env->cpus) {
@@ -11913,6 +13093,23 @@ static struct rq *sched_balance_find_src_rq(struct lb_env *env,
 
 			break;
 
+		case migrate_llc_task:
+#ifdef CONFIG_SCHED_CACHE
+			sd_tmp = rcu_dereference_all(rq->sd);
+			dst_llc = llc_id(env->dst_cpu);
+
+			if (sd_tmp && (unsigned)dst_llc < sd_tmp->llc_max) {
+				unsigned int this_pref_llc =
+					sd_tmp->llc_counts[dst_llc];
+
+				if (busiest_pref_llc < this_pref_llc) {
+					busiest_pref_llc = this_pref_llc;
+					busiest = rq;
+				}
+			}
+#endif
+			break;
+
 		}
 	}
 
@@ -11964,6 +13161,9 @@ static int need_active_balance(struct lb_env *env)
 {
 	struct sched_domain *sd = env->sd;
 
+	if (alb_break_llc(env))
+		return 0;
+
 	if (asym_active_balance(env))
 		return 1;
 
@@ -11983,7 +13183,8 @@ static int need_active_balance(struct lb_env *env)
 			return 1;
 	}
 
-	if (env->migration_type == migrate_misfit)
+	if (env->migration_type == migrate_misfit ||
+	    env->migration_type == migrate_llc_task)
 		return 1;
 
 	return 0;
@@ -12028,7 +13229,9 @@ static int should_we_balance(struct lb_env *env)
 		 * balancing cores, but remember the first idle SMT CPU for
 		 * later consideration.  Find CPU on an idle core first.
 		 */
-		if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) {
+		if (sched_smt_active() &&
+		    !(env->sd->flags & SD_SHARE_CPUCAPACITY) &&
+		    !is_core_idle(cpu)) {
 			if (idle_smt == -1)
 				idle_smt = cpu;
 			/*
@@ -12036,9 +13239,7 @@ static int should_we_balance(struct lb_env *env)
 			 * idle has been found, then its not needed to check other
 			 * SMT siblings for idleness:
 			 */
-#ifdef CONFIG_SCHED_SMT
 			cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu));
-#endif
 			continue;
 		}
 
@@ -12076,6 +13277,8 @@ static void update_lb_imbalance_stat(struct lb_env *env, struct sched_domain *sd
 	case migrate_misfit:
 		__schedstat_add(sd->lb_imbalance_misfit[idle], env->imbalance);
 		break;
+	case migrate_llc_task:
+		break;
 	}
 }
 
@@ -12279,9 +13482,16 @@ static int sched_balance_rq(int this_cpu, struct rq *this_rq,
 		 *
 		 * Similarly for migration_misfit which is not related to
 		 * load/util migration, don't pollute nr_balance_failed.
+		 *
+		 * The same for cache aware scheduling's allowance for
+		 * load imbalance. If regular load balance does not
+		 * migrate task due to LLC locality, it is a expected
+		 * behavior and don't pollute nr_balance_failed.
+		 * See can_migrate_task().
 		 */
 		if (idle != CPU_NEWLY_IDLE &&
-		    env.migration_type != migrate_misfit)
+		    env.migration_type != migrate_misfit &&
+		    !(env.flags & LBF_LLC_PINNED))
 			sd->nr_balance_failed++;
 
 		if (need_active_balance(&env)) {
@@ -12785,8 +13995,6 @@ static void nohz_balancer_kick(struct rq *rq)
 		goto out;
 	}
 
-	rcu_read_lock();
-
 	sd = rcu_dereference_all(rq->sd);
 	if (sd) {
 		/*
@@ -12794,8 +14002,8 @@ static void nohz_balancer_kick(struct rq *rq)
 		 * capacity, kick the ILB to see if there's a better CPU to run on:
 		 */
 		if (rq->cfs.h_nr_runnable >= 1 && check_cpu_capacity(rq, sd)) {
-			flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
-			goto unlock;
+			flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
+			goto out;
 		}
 	}
 
@@ -12811,8 +14019,8 @@ static void nohz_balancer_kick(struct rq *rq)
 		 */
 		for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
 			if (sched_asym(sd, i, cpu)) {
-				flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
-				goto unlock;
+				flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
+				goto out;
 			}
 		}
 	}
@@ -12823,10 +14031,8 @@ static void nohz_balancer_kick(struct rq *rq)
 		 * When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
 		 * to run the misfit task on.
 		 */
-		if (check_misfit_status(rq)) {
-			flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
-			goto unlock;
-		}
+		if (check_misfit_status(rq))
+			flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
 
 		/*
 		 * For asymmetric systems, we do not want to nicely balance
@@ -12835,10 +14041,10 @@ static void nohz_balancer_kick(struct rq *rq)
 		 *
 		 * Skip the LLC logic because it's not relevant in that case.
 		 */
-		goto unlock;
+		goto out;
 	}
 
-	sds = rcu_dereference_all(per_cpu(sd_llc_shared, cpu));
+	sds = rcu_dereference_all(per_cpu(sd_balance_shared, cpu));
 	if (sds) {
 		/*
 		 * If there is an imbalance between LLC domains (IOW we could
@@ -12850,13 +14056,9 @@ static void nohz_balancer_kick(struct rq *rq)
 		 * like this LLC domain has tasks we could move.
 		 */
 		nr_busy = atomic_read(&sds->nr_busy_cpus);
-		if (nr_busy > 1) {
-			flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
-			goto unlock;
-		}
+		if (nr_busy > 1)
+			flags |= NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
 	}
-unlock:
-	rcu_read_unlock();
 out:
 	if (READ_ONCE(nohz.needs_update))
 		flags |= NOHZ_NEXT_KICK;
@@ -12868,17 +14070,17 @@ static void nohz_balancer_kick(struct rq *rq)
 static void set_cpu_sd_state_busy(int cpu)
 {
 	struct sched_domain *sd;
-
-	rcu_read_lock();
 	sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
 
-	if (!sd || !sd->nohz_idle)
-		goto unlock;
+	/*
+	 * sd->nohz_idle only pairs with nr_busy_cpus on sd->shared; if this
+	 * domain has no shared object there is nothing to clear or account.
+	 */
+	if (!sd || !sd->shared || !sd->nohz_idle)
+		return;
 	sd->nohz_idle = 0;
 
 	atomic_inc(&sd->shared->nr_busy_cpus);
-unlock:
-	rcu_read_unlock();
 }
 
 void nohz_balance_exit_idle(struct rq *rq)
@@ -12897,17 +14099,14 @@ void nohz_balance_exit_idle(struct rq *rq)
 static void set_cpu_sd_state_idle(int cpu)
 {
 	struct sched_domain *sd;
-
-	rcu_read_lock();
 	sd = rcu_dereference_all(per_cpu(sd_llc, cpu));
 
-	if (!sd || sd->nohz_idle)
-		goto unlock;
+	/* See set_cpu_sd_state_busy(): nohz_idle is only used with sd->shared. */
+	if (!sd || !sd->shared || sd->nohz_idle)
+		return;
 	sd->nohz_idle = 1;
 
 	atomic_dec(&sd->shared->nr_busy_cpus);
-unlock:
-	rcu_read_unlock();
 }
 
 /*
@@ -13700,6 +14899,8 @@ static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
 	if (static_branch_unlikely(&sched_numa_balancing))
 		task_tick_numa(rq, curr);
 
+	task_tick_cache(rq, curr);
+
 	update_misfit_status(curr, rq);
 	check_update_overutilized_status(task_rq(curr));
 
diff --git a/kernel/sched/features.h b/kernel/sched/features.h
index 84c4fe3abd743..8f0dee8fc4755 100644
--- a/kernel/sched/features.h
+++ b/kernel/sched/features.h
@@ -110,8 +110,16 @@ SCHED_FEAT(WARN_DOUBLE_CLOCK, false)
  * rq lock and possibly create a large contention, sending an
  * IPI to that CPU and let that CPU push the RT task to where
  * it should go may be a better scenario.
+ *
+ * This is best for PREEMPT_RT, but for non-RT it can cause issues
+ * when preemption is disabled for long periods of time. Have
+ * it only default enabled for PREEMPT_RT.
  */
+# ifdef CONFIG_PREEMPT_RT
 SCHED_FEAT(RT_PUSH_IPI, true)
+# else
+SCHED_FEAT(RT_PUSH_IPI, false)
+# endif
 #endif
 
 SCHED_FEAT(RT_RUNTIME_SHARE, false)
diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c
index 226a6329f3e92..cb957b8f1946a 100644
--- a/kernel/sched/membarrier.c
+++ b/kernel/sched/membarrier.c
@@ -164,8 +164,26 @@
 	| MEMBARRIER_PRIVATE_EXPEDITED_RSEQ_BITMASK			\
 	| MEMBARRIER_CMD_GET_REGISTRATIONS)
 
+/*
+ * Scoped guard for memory barriers on entry and exit.
+ * Matches memory barriers before & after rq->curr modification in scheduler.
+ */
+DEFINE_LOCK_GUARD_0(mb, smp_mb(), smp_mb())
 static DEFINE_MUTEX(membarrier_ipi_mutex);
+static DEFINE_PER_CPU(struct mutex, membarrier_cpu_mutexes);
+
 #define SERIALIZE_IPI() guard(mutex)(&membarrier_ipi_mutex)
+#define SERIALIZE_IPI_CPU(cpu_id) guard(mutex)(&per_cpu(membarrier_cpu_mutexes, cpu_id))
+
+static int __init membarrier_init(void)
+{
+	int i;
+
+	for_each_possible_cpu(i)
+		mutex_init(&per_cpu(membarrier_cpu_mutexes, i));
+	return 0;
+}
+core_initcall(membarrier_init);
 
 static void ipi_mb(void *info)
 {
@@ -258,23 +276,19 @@ void membarrier_update_current_mm(struct mm_struct *next_mm)
 
 static int membarrier_global_expedited(void)
 {
+	cpumask_var_t __free(free_cpumask_var) tmpmask = CPUMASK_VAR_NULL;
 	int cpu;
-	cpumask_var_t tmpmask;
 
 	if (num_online_cpus() == 1)
 		return 0;
 
-	/*
-	 * Matches memory barriers after rq->curr modification in
-	 * scheduler.
-	 */
-	smp_mb();	/* system call entry is not a mb. */
-
 	if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
 		return -ENOMEM;
 
+	guard(mb)();
 	SERIALIZE_IPI();
-	cpus_read_lock();
+	guard(cpus_read_lock)();
+
 	rcu_read_lock();
 	for_each_online_cpu(cpu) {
 		struct task_struct *p;
@@ -310,21 +324,11 @@ static int membarrier_global_expedited(void)
 	smp_call_function_many(tmpmask, ipi_mb, NULL, 1);
 	preempt_enable();
 
-	free_cpumask_var(tmpmask);
-	cpus_read_unlock();
-
-	/*
-	 * Memory barrier on the caller thread _after_ we finished
-	 * waiting for the last IPI. Matches memory barriers before
-	 * rq->curr modification in scheduler.
-	 */
-	smp_mb();	/* exit from system call is not a mb */
 	return 0;
 }
 
 static int membarrier_private_expedited(int flags, int cpu_id)
 {
-	cpumask_var_t tmpmask;
 	struct mm_struct *mm = current->mm;
 	smp_call_func_t ipi_func = ipi_mb;
 
@@ -361,30 +365,45 @@ static int membarrier_private_expedited(int flags, int cpu_id)
 	 * On RISC-V, this barrier pairing is also needed for the
 	 * SYNC_CORE command when switching between processes, cf.
 	 * the inline comments in membarrier_arch_switch_mm().
+	 *
+	 * Memory barrier on the caller thread _after_ we finished
+	 * waiting for the last IPI. Matches memory barriers before
+	 * rq->curr modification in scheduler.
 	 */
-	smp_mb();	/* system call entry is not a mb. */
-
-	if (cpu_id < 0 && !zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
-		return -ENOMEM;
-
-	SERIALIZE_IPI();
-	cpus_read_lock();
-
+	guard(mb)();
 	if (cpu_id >= 0) {
+		if (cpu_id >= nr_cpu_ids || !cpu_possible(cpu_id))
+			return 0;
+
+		SERIALIZE_IPI_CPU(cpu_id);
+		guard(cpus_read_lock)();
 		struct task_struct *p;
 
-		if (cpu_id >= nr_cpu_ids || !cpu_online(cpu_id))
-			goto out;
+		if (!cpu_online(cpu_id))
+			return 0;
+
 		rcu_read_lock();
 		p = rcu_dereference(cpu_rq(cpu_id)->curr);
 		if (!p || p->mm != mm) {
 			rcu_read_unlock();
-			goto out;
+			return 0;
 		}
 		rcu_read_unlock();
+		/*
+		 * smp_call_function_single() will call ipi_func() if cpu_id
+		 * is the calling CPU.
+		 */
+		smp_call_function_single(cpu_id, ipi_func, NULL, 1);
 	} else {
+		cpumask_var_t __free(free_cpumask_var) tmpmask = CPUMASK_VAR_NULL;
 		int cpu;
 
+		if (!zalloc_cpumask_var(&tmpmask, GFP_KERNEL))
+			return -ENOMEM;
+
+		SERIALIZE_IPI();
+		guard(cpus_read_lock)();
+
 		rcu_read_lock();
 		for_each_online_cpu(cpu) {
 			struct task_struct *p;
@@ -394,15 +413,6 @@ static int membarrier_private_expedited(int flags, int cpu_id)
 				__cpumask_set_cpu(cpu, tmpmask);
 		}
 		rcu_read_unlock();
-	}
-
-	if (cpu_id >= 0) {
-		/*
-		 * smp_call_function_single() will call ipi_func() if cpu_id
-		 * is the calling CPU.
-		 */
-		smp_call_function_single(cpu_id, ipi_func, NULL, 1);
-	} else {
 		/*
 		 * For regular membarrier, we can save a few cycles by
 		 * skipping the current cpu -- we're about to do smp_mb()
@@ -429,18 +439,6 @@ static int membarrier_private_expedited(int flags, int cpu_id)
 		}
 	}
 
-out:
-	if (cpu_id < 0)
-		free_cpumask_var(tmpmask);
-	cpus_read_unlock();
-
-	/*
-	 * Memory barrier on the caller thread _after_ we finished
-	 * waiting for the last IPI. Matches memory barriers before
-	 * rq->curr modification in scheduler.
-	 */
-	smp_mb();	/* exit from system call is not a mb */
-
 	return 0;
 }
 
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 4ee8faf01441a..e6ea728f519ee 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -19,9 +19,9 @@ int sysctl_sched_rt_period = 1000000;
 
 /*
  * part of the period that we allow rt tasks to run in us.
- * default: 0.95s
+ * default: 1s
  */
-int sysctl_sched_rt_runtime = 950000;
+int sysctl_sched_rt_runtime = 1000000;
 
 #ifdef CONFIG_SYSCTL
 static int sysctl_sched_rr_timeslice = (MSEC_PER_SEC * RR_TIMESLICE) / HZ;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 9f63b15d309d1..8eb8f83db6b04 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1187,6 +1187,12 @@ struct rq {
 	struct scx_rq		scx;
 	struct sched_dl_entity	ext_server;
 #endif
+#ifdef CONFIG_SCHED_CACHE
+	raw_spinlock_t		cpu_epoch_lock ____cacheline_aligned;
+	u64			cpu_runtime;
+	unsigned long		cpu_epoch;
+	unsigned long		cpu_epoch_next;
+#endif
 
 	struct sched_dl_entity	fair_server;
 
@@ -1199,6 +1205,12 @@ struct rq {
 #ifdef CONFIG_NUMA_BALANCING
 	unsigned int		numa_migrate_on;
 #endif
+
+#ifdef CONFIG_SCHED_CACHE
+	unsigned int		nr_pref_llc_running;
+	unsigned int		nr_llc_running;
+#endif
+
 	/*
 	 * This is part of a global counter where only the total sum
 	 * over all CPUs matters. A task can increase this counter on
@@ -1546,6 +1558,14 @@ extern void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags);
 extern void sched_core_get(void);
 extern void sched_core_put(void);
 
+static inline bool task_has_sched_core(struct task_struct *p)
+{
+	if (sched_core_disabled())
+		return false;
+
+	return !!p->core_cookie;
+}
+
 #else /* !CONFIG_SCHED_CORE: */
 
 static inline bool sched_core_enabled(struct rq *rq)
@@ -1586,6 +1606,11 @@ static inline bool sched_group_cookie_match(struct rq *rq,
 	return true;
 }
 
+static inline bool task_has_sched_core(struct task_struct *p)
+{
+	return false;
+}
+
 #endif /* !CONFIG_SCHED_CORE */
 
 #ifdef CONFIG_RT_GROUP_SCHED
@@ -1667,21 +1692,15 @@ do {						\
 	flags = _raw_spin_rq_lock_irqsave(rq);	\
 } while (0)
 
-#ifdef CONFIG_SCHED_SMT
 extern void __update_idle_core(struct rq *rq);
 
 static inline void update_idle_core(struct rq *rq)
 {
-	if (static_branch_unlikely(&sched_smt_present))
+	if (sched_smt_active())
 		__update_idle_core(rq);
 }
 
-#else /* !CONFIG_SCHED_SMT: */
-static inline void update_idle_core(struct rq *rq) { }
-#endif /* !CONFIG_SCHED_SMT */
-
 #ifdef CONFIG_FAIR_GROUP_SCHED
-
 static inline struct task_struct *task_of(struct sched_entity *se)
 {
 	WARN_ON_ONCE(!entity_is_task(se));
@@ -2082,6 +2101,8 @@ init_numa_balancing(u64 clone_flags, struct task_struct *p)
 
 #endif /* !CONFIG_NUMA_BALANCING */
 
+int task_llc(const struct task_struct *p);
+
 static inline void
 queue_balance_callback(struct rq *rq,
 		       struct balance_callback *head,
@@ -2171,6 +2192,7 @@ DECLARE_PER_CPU(int, sd_llc_size);
 DECLARE_PER_CPU(int, sd_llc_id);
 DECLARE_PER_CPU(int, sd_share_id);
 DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
+DECLARE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_numa);
 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
 DECLARE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
@@ -4037,6 +4059,29 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
 static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
 #endif /* !CONFIG_SCHED_MM_CID */
 
+#ifdef CONFIG_SCHED_CACHE
+DECLARE_STATIC_KEY_FALSE(sched_cache_present);
+DECLARE_STATIC_KEY_FALSE(sched_cache_active);
+extern int sysctl_sched_cache_user;
+extern unsigned int llc_aggr_tolerance;
+extern unsigned int llc_epoch_period;
+extern unsigned int llc_epoch_affinity_timeout;
+extern unsigned int llc_imb_pct;
+extern unsigned int llc_overaggr_pct;
+
+static inline bool sched_cache_enabled(void)
+{
+	return static_branch_unlikely(&sched_cache_active);
+}
+
+extern void sched_cache_active_set(void);
+
+#endif
+
+void sched_domains_free_llc_id(int cpu);
+
+extern void init_sched_mm(struct task_struct *p);
+
 extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
 extern int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se);
 static inline
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index 5847b83d9d552..df2ceb54c970e 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -19,8 +19,10 @@ void sched_domains_mutex_unlock(void)
 }
 
 /* Protected by sched_domains_mutex: */
+static cpumask_var_t sched_domains_llc_id_allocmask;
 static cpumask_var_t sched_domains_tmpmask;
 static cpumask_var_t sched_domains_tmpmask2;
+int max_lid;
 
 static int __init sched_debug_setup(char *str)
 {
@@ -621,6 +623,12 @@ static void free_sched_groups(struct sched_group *sg, int free_sgc)
 	} while (sg != first);
 }
 
+static void free_sched_domain_shared(struct sched_domain_shared *sds)
+{
+	if (sds && atomic_dec_and_test(&sds->ref))
+		kfree(sds);
+}
+
 static void destroy_sched_domain(struct sched_domain *sd)
 {
 	/*
@@ -629,9 +637,12 @@ static void destroy_sched_domain(struct sched_domain *sd)
 	 * dropping group/capacity references, freeing where none remain.
 	 */
 	free_sched_groups(sd->groups, 1);
+	free_sched_domain_shared(sd->shared);
 
-	if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
-		kfree(sd->shared);
+#ifdef CONFIG_SCHED_CACHE
+	/* only the bottom sd has llc_counts array */
+	kfree(sd->llc_counts);
+#endif
 	kfree(sd);
 }
 
@@ -663,9 +674,10 @@ static void destroy_sched_domains(struct sched_domain *sd)
  */
 DEFINE_PER_CPU(struct sched_domain __rcu *, sd_llc);
 DEFINE_PER_CPU(int, sd_llc_size);
-DEFINE_PER_CPU(int, sd_llc_id);
+DEFINE_PER_CPU(int, sd_llc_id) = -1;
 DEFINE_PER_CPU(int, sd_share_id);
 DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_llc_shared);
+DEFINE_PER_CPU(struct sched_domain_shared __rcu *, sd_balance_shared);
 DEFINE_PER_CPU(struct sched_domain __rcu *, sd_numa);
 DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_packing);
 DEFINE_PER_CPU(struct sched_domain __rcu *, sd_asym_cpucapacity);
@@ -692,7 +704,6 @@ static void update_top_cache_domain(int cpu)
 
 	rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
 	per_cpu(sd_llc_size, cpu) = size;
-	per_cpu(sd_llc_id, cpu) = id;
 	rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
 
 	sd = lowest_flag_domain(cpu, SD_CLUSTER);
@@ -713,7 +724,18 @@ static void update_top_cache_domain(int cpu)
 	rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd);
 
 	sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL);
+	/*
+	 * The shared object is attached to sd_asym_cpucapacity only when the
+	 * asym domain is non-overlapping (i.e., not built from SD_NUMA).
+	 * On overlapping (NUMA) asym domains we fall back to letting the
+	 * SD_SHARE_LLC path own the shared object, so sd->shared may be NULL
+	 * here.
+	 */
+	if (sd && sd->shared)
+		sds = sd->shared;
+
 	rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
+	rcu_assign_pointer(per_cpu(sd_balance_shared, cpu), sds);
 }
 
 /*
@@ -737,7 +759,14 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 
 			/* Pick reference to parent->shared. */
 			if (parent->shared) {
-				WARN_ON_ONCE(tmp->shared);
+				/*
+				 * It is safe to free a sd->shared that
+				 * has not been published yet. If a
+				 * sd->shared was published, the refcount
+				 * will end up being non-zero and it will
+				 * not be freed here.
+				 */
+				free_sched_domain_shared(tmp->shared);
 				tmp->shared = parent->shared;
 				parent->shared = NULL;
 			}
@@ -762,10 +791,20 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 	if (sd && sd_degenerate(sd)) {
 		tmp = sd;
 		sd = sd->parent;
-		destroy_sched_domain(tmp);
+
 		if (sd) {
 			struct sched_group *sg = sd->groups;
 
+#ifdef CONFIG_SCHED_CACHE
+			/* move buffer to parent as child is being destroyed */
+			sd->llc_counts = tmp->llc_counts;
+			sd->llc_max = tmp->llc_max;
+			sd->llc_bytes = tmp->llc_bytes;
+			/* make sure destroy_sched_domain() does not free it */
+			tmp->llc_counts = NULL;
+			tmp->llc_max = 0;
+			tmp->llc_bytes = 0;
+#endif
 			/*
 			 * sched groups hold the flags of the child sched
 			 * domain for convenience. Clear such flags since
@@ -777,6 +816,8 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 
 			sd->child = NULL;
 		}
+
+		destroy_sched_domain(tmp);
 	}
 
 	sched_domain_debug(sd, cpu);
@@ -804,6 +845,239 @@ enum s_alloc {
 	sa_none,
 };
 
+#ifdef CONFIG_SCHED_CACHE
+/* hardware support for cache aware scheduling */
+DEFINE_STATIC_KEY_FALSE(sched_cache_present);
+/*
+ * Indicator of whether cache aware scheduling
+ * is active, used by the scheduler.
+ */
+DEFINE_STATIC_KEY_FALSE(sched_cache_active);
+/* user wants cache aware scheduling [0 or 1] */
+int sysctl_sched_cache_user = 1;
+
+/*
+ * Get the effective LLC size in bytes that @cpu's bottom sched_domain
+ * can use. A CPU within a cpuset partition can only use a proportion
+ * of the physical LLC, scaled by the ratio of the partition's span
+ * weight to the hardware LLC sharing weight. @sd should be the
+ * topmost domain with SD_SHARE_LLC.
+ *
+ * Returns 0 if cacheinfo is not yet populated. This happens during
+ * early boot when build_sched_domains() runs before the generic
+ * cacheinfo framework has been initialized (cacheinfo_cpu_online()
+ * is a device_initcall cpuhp callback). In that case,
+ * cacheinfo_cpu_online() will later call sched_update_llc_bytes()
+ * to fill in the bottom domain's llc_bytes once the cache attributes
+ * are available.
+ */
+static unsigned long get_effective_llc_bytes(int cpu,
+					     struct sched_domain *sd)
+{
+	struct cacheinfo *ci;
+	unsigned int hw_weight;
+
+	ci = get_cpu_cacheinfo_llc(cpu);
+	if (!ci)
+		return 0;
+
+	hw_weight = cpumask_weight(&ci->shared_cpu_map);
+	if (!hw_weight)
+		return 0;
+
+	return div_u64((u64)ci->size * sd->span_weight, hw_weight);
+}
+
+static bool alloc_sd_llc(const struct cpumask *cpu_map,
+			 struct s_data *d)
+{
+	struct sched_domain *sd, *top_llc, *parent;
+	unsigned int *p;
+	int i;
+
+	for_each_cpu(i, cpu_map) {
+		sd = *per_cpu_ptr(d->sd, i);
+		if (!sd)
+			goto err;
+
+		p = kcalloc_node(max_lid + 1, sizeof(unsigned int),
+				 GFP_KERNEL, cpu_to_node(i));
+		if (!p)
+			goto err;
+
+		top_llc = sd;
+		/*
+		 * Find the topmost SD_SHARE_LLC domain.
+		 * Not yet attached to the CPU, so per_cpu(sd_llc, i)
+		 * can not be used.
+		 */
+		while ((parent = rcu_dereference_protected(top_llc->parent, true)) &&
+		       (parent->flags & SD_SHARE_LLC))
+			top_llc = parent;
+
+		if (top_llc->flags & SD_SHARE_LLC) {
+			sd->llc_max = max_lid + 1;
+			sd->llc_counts = p;
+			sd->llc_bytes = get_effective_llc_bytes(i, top_llc);
+		} else {
+			/* avoid memory leak */
+			kfree(p);
+		}
+	}
+
+	return true;
+err:
+	for_each_cpu(i, cpu_map) {
+		sd = *per_cpu_ptr(d->sd, i);
+		if (sd) {
+			kfree(sd->llc_counts);
+			sd->llc_counts = NULL;
+			sd->llc_max = 0;
+			sd->llc_bytes = 0;
+		}
+	}
+
+	return false;
+}
+
+/*
+ * Enable/disable cache aware scheduling according to
+ * user input and the presence of hardware support.
+ */
+static void _sched_cache_active_set(void)
+{
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&sched_domains_mutex);
+
+	/* hardware does not support */
+	if (!static_branch_likely(&sched_cache_present)) {
+		static_branch_disable_cpuslocked(&sched_cache_active);
+		if (sched_debug())
+			pr_info("%s: cache aware scheduling not supported on this platform\n", __func__);
+		return;
+	}
+
+	/*
+	 * user wants it or not ?
+	 * TBD: read before writing the static key.
+	 * It is not in the critical path, leave as-is
+	 * for now.
+	 */
+	if (sysctl_sched_cache_user) {
+		static_branch_enable_cpuslocked(&sched_cache_active);
+		if (sched_debug())
+			pr_info("%s: enabling cache aware scheduling\n", __func__);
+	} else {
+		static_branch_disable_cpuslocked(&sched_cache_active);
+		if (sched_debug())
+			pr_info("%s: disabling cache aware scheduling\n", __func__);
+	}
+}
+
+/* used by debugfs */
+void sched_cache_active_set(void)
+{
+	cpus_read_lock();
+	sched_domains_mutex_lock();
+	_sched_cache_active_set();
+	sched_domains_mutex_unlock();
+	cpus_read_unlock();
+}
+
+/*
+ * Update the bottom sched_domain's llc_bytes for @cpu and all its
+ * LLC siblings. Called from cacheinfo_cpu_online() or
+ * cacheinfo_cpu_pre_down() with cpu hotplug lock held.
+ *
+ * Note: get_effective_llc_bytes() returns 0 on PowerPC.
+ * thus cache aware scheduling is disabled on PowerPC for
+ * now. PowerPC does not use the generic cacheinfo framework --
+ * it has its own cacheinfo with a separate struct cache hierarchy
+ * and does not populates the per-CPU struct cpu_cacheinfo array
+ * that get_cpu_cacheinfo_llc() reads.
+ */
+void sched_update_llc_bytes(unsigned int cpu)
+{
+	struct sched_domain *sd, *sdp;
+	unsigned int i;
+
+	sched_domains_mutex_lock();
+
+	sdp = rcu_dereference_sched_domain(per_cpu(sd_llc, cpu));
+	if (!sdp)
+		goto unlock;
+
+	/*
+	 * ci->shared_cpu_map is built incrementally as CPUs come
+	 * online, so the first CPU in an LLC initially sees
+	 * hw_weight == 1 and computes an inflated llc_bytes in
+	 * get_effective_llc_bytes().  Re-evaluating every LLC
+	 * sibling on each online event corrects this once the full
+	 * shared_cpu_map is known.
+	 */
+	for_each_cpu(i, sched_domain_span(sdp)) {
+		sd = rcu_dereference_sched_domain(cpu_rq(i)->sd);
+		if (sd)
+			sd->llc_bytes = get_effective_llc_bytes(i, sdp);
+	}
+
+unlock:
+	sched_domains_mutex_unlock();
+}
+
+static void sched_cache_set(bool has_multi_llcs)
+{
+	/*
+	 * TBD: check before writing to it. sched domain rebuild
+	 * is not in the critical path, leave as-is for now.
+	 */
+	if (has_multi_llcs)
+		static_branch_enable_cpuslocked(&sched_cache_present);
+	else
+		static_branch_disable_cpuslocked(&sched_cache_present);
+
+	_sched_cache_active_set();
+}
+#else
+static bool alloc_sd_llc(const struct cpumask *cpu_map,
+			 struct s_data *d)
+{
+	return false;
+}
+static inline void sched_cache_set(bool has_multi_llcs) { }
+#endif
+
+/*
+ * Return true if @sd belongs to an LLC group whose enclosing
+ * partition spans more than one LLC. @sd must be the topmost
+ * SD_SHARE_LLC domain.
+ *
+ * Any duplicated parent domains with the same span as @sd are
+ * skipped: before cpu_attach_domain() degeneration these still
+ * exist, after degeneration the loop is a no-op. This makes the
+ * helper usable both during sched domain build and against an
+ * already-attached domain tree.
+ *
+ * Note: For systems with a single LLC per node, cache-aware
+ * scheduling is still enabled when multiple nodes exist.
+ * However, NUMA balancing decisions take precedence over
+ * cache-aware scheduling. Conversely, if there is only one
+ * LLC per partition, cache-aware scheduling should be disabled.
+ */
+static bool sd_in_multi_llcs(struct sched_domain *sd)
+{
+	struct sched_domain *sdp = sd->parent;
+
+	/* it does not make sense to aggregate to 1 CPU */
+	if (sd->span_weight == 1)
+		return false;
+
+	while (sdp && sdp->span_weight == sd->span_weight)
+		sdp = sdp->parent;
+
+	return !!sdp;
+}
+
 /*
  * Return the canonical balance CPU for this group, this is the first CPU
  * of this group that's also in the balance mask.
@@ -1310,9 +1584,7 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
 		cpumask_copy(mask, sched_group_span(sg));
 		for_each_cpu(cpu, mask) {
 			cores++;
-#ifdef CONFIG_SCHED_SMT
 			cpumask_andnot(mask, mask, cpu_smt_mask(cpu));
-#endif
 		}
 		sg->cores = cores;
 
@@ -1790,6 +2062,11 @@ const struct cpumask *tl_mc_mask(struct sched_domain_topology_level *tl, int cpu
 {
 	return cpu_coregroup_mask(cpu);
 }
+
+#define llc_mask(cpu) cpu_coregroup_mask(cpu)
+
+#else
+#define llc_mask(cpu) cpumask_of(cpu)
 #endif
 
 const struct cpumask *tl_pkg_mask(struct sched_domain_topology_level *tl, int cpu)
@@ -2650,14 +2927,153 @@ static void adjust_numa_imbalance(struct sched_domain *sd_llc)
 	}
 }
 
+static void
+init_sched_domain_shared(struct s_data *d, struct sched_domain *sd, int flags)
+{
+	struct sched_domain_shared *sds = NULL;
+	int cpu;
+
+	/*
+	 * Multiple domains can try to claim a shared object like
+	 * SD_ASYM_CPUCAPACITY and SD_SHARE_LLC which can alias to
+	 * same cpumask_first(sched_domain_span(sd)) CPU and can
+	 * cause "nr_idle_scan" to be populated incorrectly during
+	 * load balancing.
+	 *
+	 * Find the first CPU in sched_domain_span(sd) with an
+	 * unclaimed domain (!alloc_flags) or where the alloc_flag
+	 * matches the requested flag (SD_* flag)
+	 *
+	 * If the domain only has single CPU, allow temporary overlap
+	 * in allocation since the domains will be degenerated later.
+	 */
+	for_each_cpu(cpu, sched_domain_span(sd)) {
+		sds = *per_cpu_ptr(d->sds, cpu);
+
+		if (!sds->alloc_flags ||
+		    sd->span_weight == 1 ||
+		    sds->alloc_flags == flags) {
+			sds->alloc_flags = flags;
+			sd->shared = sds;
+			break;
+		}
+	}
+
+	/*
+	 * Use the sd_shared corresponding to the last
+	 * CPU in the span if none are avaialable.
+	 */
+	if (WARN_ON_ONCE(!sd->shared))
+		sd->shared = sds;
+
+	/*
+	 * nr_busy_cpus is consumed only by the NOHZ kick path via
+	 * sd_balance_shared; on the asym-capacity path it is initialized but
+	 * never read.
+	 */
+	atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight);
+	atomic_inc(&sd->shared->ref);
+}
+
+/*
+ * For asymmetric CPU capacity, attach sched_domain_shared on the innermost
+ * SD_ASYM_CPUCAPACITY_FULL ancestor of @cpu's base domain when that ancestor is
+ * not an overlapping NUMA-built domain (then LLC should claim shared).
+ *
+ * A CPU may lack any FULL ancestor (e.g., exclusive cpuset symmetric island),
+ * then LLC must claim shared instead.
+ *
+ * Note: SD_ASYM_CPUCAPACITY_FULL is only set when all CPU capacity values
+ * are present in the domain span, so the asym domain we attach to cannot
+ * degenerate into a single-capacity group. The relevant edge cases are instead
+ * covered by the caveats above.
+ *
+ * Return true if this CPU's asym path claimed sd->shared, false otherwise.
+ */
+static bool claim_asym_sched_domain_shared(struct s_data *d, int cpu)
+{
+	struct sched_domain *sd = *per_cpu_ptr(d->sd, cpu);
+	struct sched_domain *sd_asym;
+
+	if (!sd)
+		return false;
+
+	sd_asym = sd;
+	while (sd_asym && !(sd_asym->flags & SD_ASYM_CPUCAPACITY_FULL))
+		sd_asym = sd_asym->parent;
+
+	if (!sd_asym || (sd_asym->flags & SD_NUMA))
+		return false;
+
+	init_sched_domain_shared(d, sd_asym, SD_ASYM_CPUCAPACITY);
+	return true;
+}
+
+static int __sched_domains_alloc_llc_id(void)
+{
+	int lid, max;
+
+	lockdep_assert_held(&sched_domains_mutex);
+
+	lid = cpumask_first_zero(sched_domains_llc_id_allocmask);
+	/*
+	 * llc_id space should never grow larger than the
+	 * possible number of CPUs in the system.
+	 */
+	if (lid >= nr_cpu_ids)
+		return -1;
+
+	__cpumask_set_cpu(lid, sched_domains_llc_id_allocmask);
+	max = cpumask_last(sched_domains_llc_id_allocmask);
+	if (max > max_lid)
+		max_lid = max;
+
+	return lid;
+}
+
+static void __sched_domains_free_llc_id(int cpu)
+{
+	int i, lid, max;
+
+	lockdep_assert_held(&sched_domains_mutex);
+
+	lid = per_cpu(sd_llc_id, cpu);
+	if (lid == -1 || lid >= nr_cpu_ids)
+		return;
+
+	per_cpu(sd_llc_id, cpu) = -1;
+
+	for_each_cpu(i, llc_mask(cpu)) {
+		/* An online CPU owns the llc_id. */
+		if (per_cpu(sd_llc_id, i) == lid)
+			return;
+	}
+
+	__cpumask_clear_cpu(lid, sched_domains_llc_id_allocmask);
+
+	max = cpumask_last(sched_domains_llc_id_allocmask);
+	/* shrink max lid to save memory */
+	if (max < max_lid)
+		max_lid = max;
+}
+
+void sched_domains_free_llc_id(int cpu)
+{
+	sched_domains_mutex_lock();
+	__sched_domains_free_llc_id(cpu);
+	sched_domains_mutex_unlock();
+}
+
 /*
  * Build sched domains for a given set of CPUs and attach the sched domains
  * to the individual CPUs
  */
 static int
-build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr)
+build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *attr,
+		    bool *multi_llcs)
 {
 	enum s_alloc alloc_state = sa_none;
+	bool has_multi_llcs = false;
 	struct sched_domain *sd;
 	struct s_data d;
 	struct rq *rq = NULL;
@@ -2675,6 +3091,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 	/* Set up domains for CPUs specified by the cpu_map: */
 	for_each_cpu(i, cpu_map) {
 		struct sched_domain_topology_level *tl;
+		int lid;
 
 		sd = NULL;
 		for_each_sd_topology(tl) {
@@ -2688,6 +3105,29 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 			if (cpumask_equal(cpu_map, sched_domain_span(sd)))
 				break;
 		}
+
+		lid = per_cpu(sd_llc_id, i);
+		if (lid == -1) {
+			/* try to reuse the llc_id of its siblings */
+			for (int j = cpumask_first(llc_mask(i));
+			     j < nr_cpu_ids;
+			     j = cpumask_next(j, llc_mask(i))) {
+				if (i == j)
+					continue;
+
+				lid = per_cpu(sd_llc_id, j);
+
+				if (lid != -1) {
+					per_cpu(sd_llc_id, i) = lid;
+
+					break;
+				}
+			}
+
+			/* a new LLC is detected */
+			if (lid == -1)
+				per_cpu(sd_llc_id, i) = __sched_domains_alloc_llc_id();
+		}
 	}
 
 	if (WARN_ON(!topology_span_sane(cpu_map)))
@@ -2712,23 +3152,27 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 		if (!sd)
 			continue;
 
+		if (has_asym)
+			claim_asym_sched_domain_shared(&d, i);
+
 		/* First, find the topmost SD_SHARE_LLC domain */
 		while (sd->parent && (sd->parent->flags & SD_SHARE_LLC))
 			sd = sd->parent;
 
 		if (sd->flags & SD_SHARE_LLC) {
-			int sd_id = cpumask_first(sched_domain_span(sd));
-
-			sd->shared = *per_cpu_ptr(d.sds, sd_id);
-			atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight);
-			atomic_inc(&sd->shared->ref);
+			init_sched_domain_shared(&d, sd, SD_SHARE_LLC);
 
 			/*
 			 * In presence of higher domains, adjust the
 			 * NUMA imbalance stats for the hierarchy.
 			 */
-			if (IS_ENABLED(CONFIG_NUMA) && sd->parent)
-				adjust_numa_imbalance(sd);
+			if (sd->parent) {
+				if (IS_ENABLED(CONFIG_NUMA))
+					adjust_numa_imbalance(sd);
+
+				if (sd_in_multi_llcs(sd))
+					has_multi_llcs = true;
+			}
 		}
 	}
 
@@ -2743,6 +3187,8 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 			init_sched_groups_capacity(i, sd);
 	}
 
+	alloc_sd_llc(cpu_map, &d);
+
 	/* Attach the domains */
 	rcu_read_lock();
 	for_each_cpu(i, cpu_map) {
@@ -2767,6 +3213,7 @@ build_sched_domains(const struct cpumask *cpu_map, struct sched_domain_attr *att
 
 	ret = 0;
 error:
+	*multi_llcs = has_multi_llcs;
 	__free_domain_allocs(&d, alloc_state, cpu_map);
 
 	return ret;
@@ -2829,8 +3276,10 @@ void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms)
  */
 int __init sched_init_domains(const struct cpumask *cpu_map)
 {
+	bool multi_llcs;
 	int err;
 
+	zalloc_cpumask_var(&sched_domains_llc_id_allocmask, GFP_KERNEL);
 	zalloc_cpumask_var(&sched_domains_tmpmask, GFP_KERNEL);
 	zalloc_cpumask_var(&sched_domains_tmpmask2, GFP_KERNEL);
 	zalloc_cpumask_var(&fallback_doms, GFP_KERNEL);
@@ -2842,7 +3291,9 @@ int __init sched_init_domains(const struct cpumask *cpu_map)
 	if (!doms_cur)
 		doms_cur = &fallback_doms;
 	cpumask_and(doms_cur[0], cpu_map, housekeeping_cpumask(HK_TYPE_DOMAIN));
-	err = build_sched_domains(doms_cur[0], NULL);
+	err = build_sched_domains(doms_cur[0], NULL, &multi_llcs);
+	if (!err)
+		sched_cache_set(multi_llcs);
 
 	return err;
 }
@@ -2915,6 +3366,7 @@ static void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new
 				    struct sched_domain_attr *dattr_new)
 {
 	bool __maybe_unused has_eas = false;
+	bool has_multi_llcs = false, multi_llcs;
 	int i, j, n;
 	int new_topology;
 
@@ -2964,14 +3416,41 @@ static void partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new
 	for (i = 0; i < ndoms_new; i++) {
 		for (j = 0; j < n && !new_topology; j++) {
 			if (cpumask_equal(doms_new[i], doms_cur[j]) &&
-			    dattrs_equal(dattr_new, i, dattr_cur, j))
+			    dattrs_equal(dattr_new, i, dattr_cur, j)) {
+				/*
+				 * Reused partition has to be taken care
+				 * of here, because there could be a corner
+				 * case that if the reused partition is skipped
+				 * and only new partition is considered, an
+				 * incorrect has_multi_llcs would be set. For
+				 * example:
+				 * If the only multi-LLC partition is reused
+				 * and a new single-LLC partition is built,
+				 * sched_cache_set(false) disables cache-aware
+				 * scheduling globally despite the reused
+				 * multi-LLC partition still being active.
+				 */
+				struct sched_domain *sd;
+				int cpu = cpumask_first(doms_cur[j]);
+
+				guard(rcu)();
+				sd = rcu_dereference(cpu_rq(cpu)->sd);
+				while (sd && sd->parent && (sd->parent->flags & SD_SHARE_LLC))
+					sd = sd->parent;
+				if (sd && (sd->flags & SD_SHARE_LLC) && sd->parent &&
+				    sd_in_multi_llcs(sd))
+					has_multi_llcs = true;
 				goto match2;
+			}
 		}
 		/* No match - add a new doms_new */
-		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL);
+		build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL,
+				    &multi_llcs);
+		has_multi_llcs |= multi_llcs;
 match2:
 		;
 	}
+	sched_cache_set(has_multi_llcs);
 
 #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
 	/* Build perf domains: */
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 4425d8dce44b4..a3bd54de82595 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -576,6 +576,10 @@ static inline bool lockdep_softirq_start(void) { return false; }
 static inline void lockdep_softirq_end(bool in_hardirq) { }
 #endif
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+static noinline void handle_softirqs(bool ksirqd);
+#endif
+
 static void handle_softirqs(bool ksirqd)
 {
 	unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 3fe6b0c99f3d8..773d8e9ae30c5 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -633,6 +633,11 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
 EXPORT_SYMBOL_GPL(stop_machine);
 
 #ifdef CONFIG_SCHED_SMT
+/*
+ * INTEL_IFS is the only user of this API. That selftest can
+ * only be compiled if SMP=y. On x86 it selects SCHED_SMT.
+ * Keep the ifdefs for now.
+ */
 int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data)
 {
 	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index 73bd6a6a78935..2cd0172d05160 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -210,13 +210,39 @@ static int fill_stats_for_pid(pid_t pid, struct taskstats *stats)
 	return 0;
 }
 
+static void tgid_stats_add_task(struct taskstats *stats,
+				struct task_struct *tsk, u64 now_ns)
+{
+	u64 delta, utime, stime;
+
+	/*
+	 * Each accounting subsystem calls its functions here to
+	 * accumulate its per-task stats for tsk, into the per-tgid structure
+	 *
+	 *	per-task-foo(stats, tsk);
+	 */
+	delayacct_add_tsk(stats, tsk);
+
+	/* calculate task elapsed time in nsec */
+	delta = now_ns - tsk->start_time;
+	/* Convert to micro seconds */
+	do_div(delta, NSEC_PER_USEC);
+	stats->ac_etime += delta;
+
+	task_cputime(tsk, &utime, &stime);
+	stats->ac_utime += div_u64(utime, NSEC_PER_USEC);
+	stats->ac_stime += div_u64(stime, NSEC_PER_USEC);
+
+	stats->nvcsw += tsk->nvcsw;
+	stats->nivcsw += tsk->nivcsw;
+}
+
 static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats)
 {
 	struct task_struct *tsk, *first;
 	unsigned long flags;
 	int rc = -ESRCH;
-	u64 delta, utime, stime;
-	u64 start_time;
+	u64 now_ns;
 
 	/*
 	 * Add additional stats from live tasks except zombie thread group
@@ -233,30 +259,12 @@ static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats)
 	else
 		memset(stats, 0, sizeof(*stats));
 
-	start_time = ktime_get_ns();
+	now_ns = ktime_get_ns();
 	for_each_thread(first, tsk) {
 		if (tsk->exit_state)
 			continue;
-		/*
-		 * Accounting subsystem can call its functions here to
-		 * fill in relevant parts of struct taskstsats as follows
-		 *
-		 *	per-task-foo(stats, tsk);
-		 */
-		delayacct_add_tsk(stats, tsk);
-
-		/* calculate task elapsed time in nsec */
-		delta = start_time - tsk->start_time;
-		/* Convert to micro seconds */
-		do_div(delta, NSEC_PER_USEC);
-		stats->ac_etime += delta;
 
-		task_cputime(tsk, &utime, &stime);
-		stats->ac_utime += div_u64(utime, NSEC_PER_USEC);
-		stats->ac_stime += div_u64(stime, NSEC_PER_USEC);
-
-		stats->nvcsw += tsk->nvcsw;
-		stats->nivcsw += tsk->nivcsw;
+		tgid_stats_add_task(stats, tsk, now_ns);
 	}
 
 	unlock_task_sighand(first, &flags);
@@ -275,18 +283,14 @@ static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats)
 static void fill_tgid_exit(struct task_struct *tsk)
 {
 	unsigned long flags;
+	u64 now_ns;
 
 	spin_lock_irqsave(&tsk->sighand->siglock, flags);
 	if (!tsk->signal->stats)
 		goto ret;
 
-	/*
-	 * Each accounting subsystem calls its functions here to
-	 * accumalate its per-task stats for tsk, into the per-tgid structure
-	 *
-	 *	per-task-foo(tsk->signal->stats, tsk);
-	 */
-	delayacct_add_tsk(tsk->signal->stats, tsk);
+	now_ns = ktime_get_ns();
+	tgid_stats_add_task(tsk->signal->stats, tsk, now_ns);
 ret:
 	spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
 	return;
diff --git a/kernel/time/alarmtimer.c b/kernel/time/alarmtimer.c
index 6e173d70d8259..ea5be5870e515 100644
--- a/kernel/time/alarmtimer.c
+++ b/kernel/time/alarmtimer.c
@@ -337,48 +337,32 @@ void alarm_init(struct alarm *alarm, enum alarmtimer_type type,
 EXPORT_SYMBOL_GPL(alarm_init);
 
 /**
- * alarm_start - Sets an absolute alarm to fire
- * @alarm: ptr to alarm to set
- * @start: time to run the alarm
+ * alarm_start_timer - Sets an alarm to fire
+ * @alarm:	Pointer to alarm to set
+ * @expires:	Expiry time
+ * @relative:	True if @expires is relative
+ *
+ * Returns: True if the alarm was queued. False if it already expired
  */
-void alarm_start(struct alarm *alarm, ktime_t start)
+bool alarm_start_timer(struct alarm *alarm, ktime_t expires, bool relative)
 {
 	struct alarm_base *base = &alarm_bases[alarm->type];
 
-	scoped_guard(spinlock_irqsave, &base->lock) {
-		alarm->node.expires = start;
-		alarmtimer_enqueue(base, alarm);
-		hrtimer_start(&alarm->timer, alarm->node.expires, HRTIMER_MODE_ABS);
-	}
+	if (relative)
+		expires = ktime_add_safe(expires, base->get_ktime());
 
 	trace_alarmtimer_start(alarm, base->get_ktime());
-}
-EXPORT_SYMBOL_GPL(alarm_start);
-
-/**
- * alarm_start_relative - Sets a relative alarm to fire
- * @alarm: ptr to alarm to set
- * @start: time relative to now to run the alarm
- */
-void alarm_start_relative(struct alarm *alarm, ktime_t start)
-{
-	struct alarm_base *base = &alarm_bases[alarm->type];
-
-	start = ktime_add_safe(start, base->get_ktime());
-	alarm_start(alarm, start);
-}
-EXPORT_SYMBOL_GPL(alarm_start_relative);
-
-void alarm_restart(struct alarm *alarm)
-{
-	struct alarm_base *base = &alarm_bases[alarm->type];
 
 	guard(spinlock_irqsave)(&base->lock);
-	hrtimer_set_expires(&alarm->timer, alarm->node.expires);
-	hrtimer_restart(&alarm->timer);
+	alarm->node.expires = expires;
 	alarmtimer_enqueue(base, alarm);
+	if (!hrtimer_start_range_ns_user(&alarm->timer, expires, 0, HRTIMER_MODE_ABS)) {
+		alarmtimer_dequeue(base, alarm);
+		return false;
+	}
+	return true;
 }
-EXPORT_SYMBOL_GPL(alarm_restart);
+EXPORT_SYMBOL_GPL(alarm_start_timer);
 
 /**
  * alarm_try_to_cancel - Tries to cancel an alarm timer
@@ -512,8 +496,6 @@ static enum alarmtimer_type clock2alarm(clockid_t clockid)
  * @now: time at the timer expiration
  *
  * Posix timer callback for expired alarm timers.
- *
- * Return: whether the timer is to be restarted
  */
 static void alarm_handle_timer(struct alarm *alarm, ktime_t now)
 {
@@ -527,12 +509,12 @@ static void alarm_handle_timer(struct alarm *alarm, ktime_t now)
  * alarm_timer_rearm - Posix timer callback for rearming timer
  * @timr:	Pointer to the posixtimer data struct
  */
-static void alarm_timer_rearm(struct k_itimer *timr)
+static bool alarm_timer_rearm(struct k_itimer *timr)
 {
 	struct alarm *alarm = &timr->it.alarm.alarmtimer;
 
 	timr->it_overrun += alarm_forward_now(alarm, timr->it_interval);
-	alarm_start(alarm, alarm->node.expires);
+	return alarm_start_timer(alarm, alarm->node.expires, false);
 }
 
 /**
@@ -588,7 +570,7 @@ static void alarm_timer_wait_running(struct k_itimer *timr)
  * @absolute:	Expiry value is absolute time
  * @sigev_none:	Posix timer does not deliver signals
  */
-static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires,
+static bool alarm_timer_arm(struct k_itimer *timr, ktime_t expires,
 			    bool absolute, bool sigev_none)
 {
 	struct alarm *alarm = &timr->it.alarm.alarmtimer;
@@ -596,10 +578,16 @@ static void alarm_timer_arm(struct k_itimer *timr, ktime_t expires,
 
 	if (!absolute)
 		expires = ktime_add_safe(expires, base->get_ktime());
-	if (sigev_none)
+
+	/*
+	 * sigev_none needs to update the expires value and pretend
+	 * that the timer is queued
+	 */
+	if (sigev_none) {
 		alarm->node.expires = expires;
-	else
-		alarm_start(&timr->it.alarm.alarmtimer, expires);
+		return true;
+	}
+	return alarm_start_timer(&timr->it.alarm.alarmtimer, expires, false);
 }
 
 /**
@@ -706,7 +694,9 @@ static int alarmtimer_do_nsleep(struct alarm *alarm, ktime_t absexp,
 	alarm->data = (void *)current;
 	do {
 		set_current_state(TASK_INTERRUPTIBLE);
-		alarm_start(alarm, absexp);
+		if (!alarm_start_timer(alarm, absexp, false))
+			alarm->data = NULL;
+
 		if (likely(alarm->data))
 			schedule();
 
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index baee13a1f87f3..e48c4d379a7ce 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -1222,14 +1222,8 @@ static void clocksource_enqueue(struct clocksource *cs)
  * @cs:		clocksource to be registered
  * @scale:	Scale factor multiplied against freq to get clocksource hz
  * @freq:	clocksource frequency (cycles per second) divided by scale
- *
- * This should only be called from the clocksource->enable() method.
- *
- * This *SHOULD NOT* be called directly! Please use the
- * __clocksource_update_freq_hz() or __clocksource_update_freq_khz() helper
- * functions.
  */
-void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq)
+static void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq)
 {
 	u64 sec;
 
@@ -1287,7 +1281,6 @@ void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq
 	pr_info("%s: mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns\n",
 		cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns);
 }
-EXPORT_SYMBOL_GPL(__clocksource_update_freq_scale);
 
 /**
  * __clocksource_register_scale - Used to install new clocksources
@@ -1338,6 +1331,26 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq)
 }
 EXPORT_SYMBOL_GPL(__clocksource_register_scale);
 
+static void __devm_clocksource_unregister(void *data)
+{
+	struct clocksource *cs = data;
+
+	clocksource_unregister(cs);
+}
+
+int __devm_clocksource_register_scale(struct device *dev, struct clocksource *cs,
+				      u32 scale, u32 freq)
+{
+	int ret;
+
+	ret = __clocksource_register_scale(cs, scale, freq);
+	if (ret)
+		return ret;
+
+	return devm_add_action_or_reset(dev, __devm_clocksource_unregister, cs);
+}
+EXPORT_SYMBOL_GPL(__devm_clocksource_register_scale);
+
 /*
  * Unbind clocksource @cs. Called with clocksource_mutex held
  */
diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c
index 5bd6efe598f0f..638ce623c342f 100644
--- a/kernel/time/hrtimer.c
+++ b/kernel/time/hrtimer.c
@@ -1352,8 +1352,14 @@ static inline bool hrtimer_keep_base(struct hrtimer *timer, bool is_local, bool
 	return hrtimer_prefer_local(is_local, is_first, is_pinned);
 }
 
-static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
-				     const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
+enum {
+	HRTIMER_REPROGRAM_NONE,
+	HRTIMER_REPROGRAM,
+	HRTIMER_REPROGRAM_FORCE,
+};
+
+static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
+				    const enum hrtimer_mode mode, struct hrtimer_clock_base *base)
 {
 	struct hrtimer_cpu_base *this_cpu_base = this_cpu_ptr(&hrtimer_bases);
 	bool is_pinned, first, was_first, keep_base = false;
@@ -1410,7 +1416,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
 	/* If a deferred rearm is pending skip reprogramming the device */
 	if (cpu_base->deferred_rearm) {
 		cpu_base->deferred_needs_update = true;
-		return false;
+		return HRTIMER_REPROGRAM_NONE;
 	}
 
 	if (!was_first || cpu_base != this_cpu_base) {
@@ -1423,7 +1429,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
 		 * callbacks.
 		 */
 		if (likely(hrtimer_base_is_online(this_cpu_base)))
-			return first;
+			return first ? HRTIMER_REPROGRAM : HRTIMER_REPROGRAM_NONE;
 
 		/*
 		 * Timer was enqueued remote because the current base is
@@ -1432,7 +1438,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
 		 */
 		if (first)
 			smp_call_function_single_async(cpu_base->cpu, &cpu_base->csd);
-		return false;
+		return HRTIMER_REPROGRAM_NONE;
 	}
 
 	/*
@@ -1446,7 +1452,7 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
 	 */
 	if (timer->is_lazy) {
 		if (cpu_base->expires_next <= hrtimer_get_expires(timer))
-			return false;
+			return HRTIMER_REPROGRAM_NONE;
 	}
 
 	/*
@@ -1455,8 +1461,24 @@ static bool __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 del
 	 * reprogram the hardware by evaluating the new first expiring
 	 * timer.
 	 */
-	hrtimer_force_reprogram(cpu_base, /* skip_equal */ true);
-	return false;
+	return HRTIMER_REPROGRAM_FORCE;
+}
+
+static int hrtimer_start_range_ns_common(struct hrtimer *timer, ktime_t tim,
+					 u64 delta_ns, const enum hrtimer_mode mode,
+					 struct hrtimer_clock_base *base)
+{
+	/*
+	 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
+	 * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
+	 * expiry mode because unmarked timers are moved to softirq expiry.
+	 */
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
+		WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
+	else
+		WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
+
+	return __hrtimer_start_range_ns(timer, tim, delta_ns, mode, base);
 }
 
 /**
@@ -1476,24 +1498,104 @@ void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 delta_ns,
 
 	debug_hrtimer_assert_init(timer);
 
+	base = lock_hrtimer_base(timer, &flags);
+
+	switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) {
+	case HRTIMER_REPROGRAM:
+		hrtimer_reprogram(timer, true);
+		break;
+	case HRTIMER_REPROGRAM_FORCE:
+		hrtimer_force_reprogram(timer->base->cpu_base, 1);
+		break;
+	case HRTIMER_REPROGRAM_NONE:
+		break;
+	}
+
+	unlock_hrtimer_base(timer, &flags);
+}
+EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
+
+static inline bool hrtimer_check_user_timer(struct hrtimer *timer)
+{
+	struct hrtimer_cpu_base *cpu_base = timer->base->cpu_base;
+	ktime_t expires;
+
 	/*
-	 * Check whether the HRTIMER_MODE_SOFT bit and hrtimer.is_soft
-	 * match on CONFIG_PREEMPT_RT = n. With PREEMPT_RT check the hard
-	 * expiry mode because unmarked timers are moved to softirq expiry.
+	 * This uses soft expires because that's the user provided
+	 * expiry time, while expires can be further in the past
+	 * due to a slack value added to the user expiry time.
 	 */
-	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
-		WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
-	else
-		WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard);
+	expires = hrtimer_get_softexpires(timer);
+
+	/* Convert to monotonic */
+	expires = ktime_sub(expires, timer->base->offset);
+
+	/*
+	 * Check whether this timer will end up as the first expiring timer in
+	 * the CPU base. If not, no further checks required as it's then
+	 * guaranteed to expire in the future.
+	 */
+	if (expires >= cpu_base->expires_next)
+		return true;
+
+	/* Validate that the expiry time is in the future. */
+	if (expires > ktime_get())
+		return true;
+
+	debug_hrtimer_deactivate(timer);
+	__remove_hrtimer(timer, timer->base, HRTIMER_STATE_INACTIVE, false);
+	trace_hrtimer_start_expired(timer);
+	return false;
+}
+
+/**
+ * hrtimer_start_range_ns_user - (re)start an user controlled hrtimer
+ * @timer:	the timer to be added
+ * @tim:	expiry time
+ * @delta_ns:	"slack" range for the timer
+ * @mode:	timer mode: absolute (HRTIMER_MODE_ABS) or
+ *		relative (HRTIMER_MODE_REL), and pinned (HRTIMER_MODE_PINNED);
+ *		softirq based mode is considered for debug purpose only!
+ *
+ * Returns: True when the timer was queued, false if it was already expired
+ *
+ * This function cannot invoke the timer callback for expired timers as it might
+ * be called under a lock which the timer callback needs to acquire. So the
+ * caller has to handle that case.
+ */
+bool hrtimer_start_range_ns_user(struct hrtimer *timer, ktime_t tim,
+				 u64 delta_ns, const enum hrtimer_mode mode)
+{
+	struct hrtimer_clock_base *base;
+	unsigned long flags;
+	bool ret = true;
+
+	debug_hrtimer_assert_init(timer);
 
 	base = lock_hrtimer_base(timer, &flags);
 
-	if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
-		hrtimer_reprogram(timer, true);
+	switch (hrtimer_start_range_ns_common(timer, tim, delta_ns, mode, base)) {
+	case HRTIMER_REPROGRAM:
+		ret = hrtimer_check_user_timer(timer);
+		if (ret)
+			hrtimer_reprogram(timer, true);
+		break;
+	case HRTIMER_REPROGRAM_FORCE:
+		ret = hrtimer_check_user_timer(timer);
+		/*
+		 * The base must always be reevaluated, independent of the
+		 * result above because the timer was the first pending timer.
+		 */
+		hrtimer_force_reprogram(timer->base->cpu_base, 1);
+		break;
+	case HRTIMER_REPROGRAM_NONE:
+		break;
+	}
 
 	unlock_hrtimer_base(timer, &flags);
+	return ret;
 }
-EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
+EXPORT_SYMBOL_GPL(hrtimer_start_range_ns_user);
 
 /**
  * hrtimer_try_to_cancel - try to deactivate a timer
@@ -1681,10 +1783,10 @@ EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
  *
  * Returns the next expiry time or KTIME_MAX if no timer is pending.
  */
-u64 hrtimer_get_next_event(void)
+ktime_t hrtimer_get_next_event(void)
 {
 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
-	u64 expires = KTIME_MAX;
+	ktime_t expires = KTIME_MAX;
 
 	guard(raw_spinlock_irqsave)(&cpu_base->lock);
 	if (!hrtimer_hres_active(cpu_base))
@@ -1700,10 +1802,10 @@ u64 hrtimer_get_next_event(void)
  * Returns the next expiry time over all timers except for the @exclude one or
  * KTIME_MAX if none of them is pending.
  */
-u64 hrtimer_next_event_without(const struct hrtimer *exclude)
+ktime_t hrtimer_next_event_without(const struct hrtimer *exclude)
 {
 	struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
-	u64 expires = KTIME_MAX;
+	ktime_t expires = KTIME_MAX;
 	unsigned int active;
 
 	guard(raw_spinlock_irqsave)(&cpu_base->lock);
@@ -2213,7 +2315,11 @@ void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode
 	if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard)
 		mode |= HRTIMER_MODE_HARD;
 
-	hrtimer_start_expires(&sl->timer, mode);
+	/* If already expired, clear the task pointer and set current state to running */
+	if (!hrtimer_start_expires_user(&sl->timer, mode)) {
+		sl->task = NULL;
+		__set_current_state(TASK_RUNNING);
+	}
 }
 EXPORT_SYMBOL_GPL(hrtimer_sleeper_start_expires);
 
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 0de2bb7cbec01..395e297093f89 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -19,7 +19,7 @@
 
 #include "posix-timers.h"
 
-static void posix_cpu_timer_rearm(struct k_itimer *timer);
+static bool posix_cpu_timer_rearm(struct k_itimer *timer);
 
 void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit)
 {
@@ -1011,24 +1011,27 @@ static void check_process_timers(struct task_struct *tsk,
 /*
  * This is called from the signal code (via posixtimer_rearm)
  * when the last timer signal was delivered and we have to reload the timer.
+ *
+ * Return true unconditionally so the core code assumes the timer to be
+ * armed. Otherwise it would requeue the signal.
  */
-static void posix_cpu_timer_rearm(struct k_itimer *timer)
+static bool posix_cpu_timer_rearm(struct k_itimer *timer)
 {
 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
-	struct task_struct *p;
 	struct sighand_struct *sighand;
+	struct task_struct *p;
 	unsigned long flags;
 	u64 now;
 
-	rcu_read_lock();
+	guard(rcu)();
 	p = cpu_timer_task_rcu(timer);
 	if (!p)
-		goto out;
+		return true;
 
 	/* Protect timer list r/w in arm_timer() */
 	sighand = lock_task_sighand(p, &flags);
 	if (unlikely(sighand == NULL))
-		goto out;
+		return true;
 
 	/*
 	 * Fetch the current sample and update the timer's expiry time.
@@ -1045,8 +1048,7 @@ static void posix_cpu_timer_rearm(struct k_itimer *timer)
 	 */
 	arm_timer(timer, p);
 	unlock_task_sighand(p, &flags);
-out:
-	rcu_read_unlock();
+	return true;
 }
 
 /**
diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c
index 9331e16141246..436ba794cc0b5 100644
--- a/kernel/time/posix-timers.c
+++ b/kernel/time/posix-timers.c
@@ -288,16 +288,18 @@ static inline int timer_overrun_to_int(struct k_itimer *timr)
 	return (int)timr->it_overrun_last;
 }
 
-static void common_hrtimer_rearm(struct k_itimer *timr)
+static bool common_hrtimer_rearm(struct k_itimer *timr)
 {
 	struct hrtimer *timer = &timr->it.real.timer;
 
 	timr->it_overrun += hrtimer_forward_now(timer, timr->it_interval);
-	hrtimer_restart(timer);
+	return hrtimer_start_expires_user(timer, HRTIMER_MODE_ABS);
 }
 
 static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_itimer *timr)
 {
+	bool queued;
+
 	guard(spinlock)(&timr->it_lock);
 
 	/*
@@ -311,12 +313,18 @@ static bool __posixtimer_deliver_signal(struct kernel_siginfo *info, struct k_it
 	if (!timr->it_interval || WARN_ON_ONCE(timr->it_status != POSIX_TIMER_REQUEUE_PENDING))
 		return true;
 
-	timr->kclock->timer_rearm(timr);
-	timr->it_status = POSIX_TIMER_ARMED;
+	/* timer_rearm() updates timr::it_overrun */
+	queued = timr->kclock->timer_rearm(timr);
+
 	timr->it_overrun_last = timr->it_overrun;
 	timr->it_overrun = -1LL;
 	++timr->it_signal_seq;
 	info->si_overrun = timer_overrun_to_int(timr);
+
+	if (queued)
+		timr->it_status = POSIX_TIMER_ARMED;
+	else
+		posix_timer_queue_signal(timr);
 	return true;
 }
 
@@ -795,7 +803,7 @@ SYSCALL_DEFINE1(timer_getoverrun, timer_t, timer_id)
 		return timer_overrun_to_int(scoped_timer);
 }
 
-static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires,
+static bool common_hrtimer_arm(struct k_itimer *timr, ktime_t expires,
 			       bool absolute, bool sigev_none)
 {
 	struct hrtimer *timer = &timr->it.real.timer;
@@ -820,8 +828,11 @@ static void common_hrtimer_arm(struct k_itimer *timr, ktime_t expires,
 		expires = ktime_add_safe(expires, hrtimer_cb_get_time(timer));
 	hrtimer_set_expires(timer, expires);
 
-	if (!sigev_none)
-		hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
+	/* For sigev_none pretend that the timer is queued */
+	if (sigev_none)
+		return true;
+
+	return hrtimer_start_expires_user(timer, HRTIMER_MODE_ABS);
 }
 
 static int common_hrtimer_try_to_cancel(struct k_itimer *timr)
@@ -903,9 +914,13 @@ int common_timer_set(struct k_itimer *timr, int flags,
 		expires = timens_ktime_to_host(timr->it_clock, expires);
 	sigev_none = timr->it_sigev_notify == SIGEV_NONE;
 
-	kc->timer_arm(timr, expires, flags & TIMER_ABSTIME, sigev_none);
-	if (!sigev_none)
-		timr->it_status = POSIX_TIMER_ARMED;
+	if (kc->timer_arm(timr, expires, flags & TIMER_ABSTIME, sigev_none)) {
+		if (!sigev_none)
+			timr->it_status = POSIX_TIMER_ARMED;
+	} else {
+		/* Timer was already expired, queue the signal */
+		posix_timer_queue_signal(timr);
+	}
 	return 0;
 }
 
diff --git a/kernel/time/posix-timers.h b/kernel/time/posix-timers.h
index 7f259e845d240..4ea9611dd7165 100644
--- a/kernel/time/posix-timers.h
+++ b/kernel/time/posix-timers.h
@@ -27,11 +27,11 @@ struct k_clock {
 	int	(*timer_del)(struct k_itimer *timr);
 	void	(*timer_get)(struct k_itimer *timr,
 			     struct itimerspec64 *cur_setting);
-	void	(*timer_rearm)(struct k_itimer *timr);
+	bool	(*timer_rearm)(struct k_itimer *timr);
 	s64	(*timer_forward)(struct k_itimer *timr, ktime_t now);
 	ktime_t	(*timer_remaining)(struct k_itimer *timr, ktime_t now);
 	int	(*timer_try_to_cancel)(struct k_itimer *timr);
-	void	(*timer_arm)(struct k_itimer *timr, ktime_t expires,
+	bool	(*timer_arm)(struct k_itimer *timr, ktime_t expires,
 			     bool absolute, bool sigev_none);
 	void	(*timer_wait_running)(struct k_itimer *timr);
 };
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index cbbb87a0c6e7c..3026a301dff7e 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -1407,8 +1407,7 @@ ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
 	 * If the next highres timer to expire is earlier than 'next_event', the
 	 * idle governor needs to know that.
 	 */
-	next_event = min_t(u64, next_event,
-			   hrtimer_next_event_without(&ts->sched_timer));
+	next_event = min(next_event, hrtimer_next_event_without(&ts->sched_timer));
 
 	return ktime_sub(next_event, now);
 }
diff --git a/kernel/time/timer.c b/kernel/time/timer.c
index 04d928c21aba5..655a8c6cd84d7 100644
--- a/kernel/time/timer.c
+++ b/kernel/time/timer.c
@@ -1932,7 +1932,7 @@ static void timer_recalc_next_expiry(struct timer_base *base)
  */
 static u64 cmp_next_hrtimer_event(u64 basem, u64 expires)
 {
-	u64 nextevt = hrtimer_get_next_event();
+	u64 nextevt = ktime_to_ns(hrtimer_get_next_event());
 
 	/*
 	 * If high resolution timers are enabled
diff --git a/kernel/time/timer_migration.c b/kernel/time/timer_migration.c
index 1d0d3a4058d59..25e3c563eb748 100644
--- a/kernel/time/timer_migration.c
+++ b/kernel/time/timer_migration.c
@@ -102,7 +102,7 @@
  * active CPU/group information atomic_try_cmpxchg() is used instead and only
  * the per CPU tmigr_cpu->lock is held.
  *
- * During the setup of groups tmigr_level_list is required. It is protected by
+ * During the setup of groups, hier->level_list is required. It is protected by
  * @tmigr_mutex.
  *
  * When @timer_base->lock as well as tmigr related locks are required, the lock
@@ -416,13 +416,12 @@
  */
 
 static DEFINE_MUTEX(tmigr_mutex);
-static struct list_head *tmigr_level_list __read_mostly;
+
+static LIST_HEAD(tmigr_hierarchy_list);
 
 static unsigned int tmigr_hierarchy_levels __read_mostly;
 static unsigned int tmigr_crossnode_level __read_mostly;
 
-static struct tmigr_group *tmigr_root;
-
 static DEFINE_PER_CPU(struct tmigr_cpu, tmigr_cpu);
 
 /*
@@ -1653,14 +1652,14 @@ static void tmigr_init_group(struct tmigr_group *group, unsigned int lvl,
 	group->groupevt.ignore = true;
 }
 
-static struct tmigr_group *tmigr_get_group(int node, unsigned int lvl)
+static struct tmigr_group *tmigr_get_group(struct tmigr_hierarchy *hier, int node, unsigned int lvl)
 {
 	struct tmigr_group *tmp, *group = NULL;
 
 	lockdep_assert_held(&tmigr_mutex);
 
 	/* Try to attach to an existing group first */
-	list_for_each_entry(tmp, &tmigr_level_list[lvl], list) {
+	list_for_each_entry(tmp, &hier->level_list[lvl], list) {
 		/*
 		 * If @lvl is below the cross NUMA node level, check whether
 		 * this group belongs to the same NUMA node.
@@ -1694,14 +1693,14 @@ static struct tmigr_group *tmigr_get_group(int node, unsigned int lvl)
 	tmigr_init_group(group, lvl, node);
 
 	/* Setup successful. Add it to the hierarchy */
-	list_add(&group->list, &tmigr_level_list[lvl]);
+	list_add(&group->list, &hier->level_list[lvl]);
 	trace_tmigr_group_set(group);
 	return group;
 }
 
-static bool tmigr_init_root(struct tmigr_group *group, bool activate)
+static bool tmigr_init_root(struct tmigr_hierarchy *hier, struct tmigr_group *group, bool activate)
 {
-	if (!group->parent && group != tmigr_root) {
+	if (!group->parent && group != hier->root) {
 		/*
 		 * This is the new top-level, prepare its groupmask in advance
 		 * to avoid accidents where yet another new top-level is
@@ -1717,11 +1716,10 @@ static bool tmigr_init_root(struct tmigr_group *group, bool activate)
 
 }
 
-static void tmigr_connect_child_parent(struct tmigr_group *child,
-				       struct tmigr_group *parent,
-				       bool activate)
+static void tmigr_connect_child_parent(struct tmigr_hierarchy *hier, struct tmigr_group *child,
+				       struct tmigr_group *parent, bool activate)
 {
-	if (tmigr_init_root(parent, activate)) {
+	if (tmigr_init_root(hier, parent, activate)) {
 		/*
 		 * The previous top level had prepared its groupmask already,
 		 * simply account it in advance as the first child. If some groups
@@ -1754,13 +1752,13 @@ static void tmigr_connect_child_parent(struct tmigr_group *child,
 	 */
 	smp_store_release(&child->parent, parent);
 
-	trace_tmigr_connect_child_parent(child);
+	trace_tmigr_connect_child_parent(hier, child);
 }
 
-static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
-			      struct tmigr_group *start, bool activate)
+static int tmigr_setup_groups(struct tmigr_hierarchy *hier, unsigned int cpu,
+			      unsigned int node, struct tmigr_group *start, bool activate)
 {
-	struct tmigr_group *group, *child, **stack;
+	struct tmigr_group *root = hier->root, *group, *child, **stack;
 	int i, top = 0, err = 0, start_lvl = 0;
 	bool root_mismatch = false;
 
@@ -1773,11 +1771,11 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
 		start_lvl = start->level + 1;
 	}
 
-	if (tmigr_root)
-		root_mismatch = tmigr_root->numa_node != node;
+	if (root)
+		root_mismatch = root->numa_node != node;
 
 	for (i = start_lvl; i < tmigr_hierarchy_levels; i++) {
-		group = tmigr_get_group(node, i);
+		group = tmigr_get_group(hier, node, i);
 		if (IS_ERR(group)) {
 			err = PTR_ERR(group);
 			i--;
@@ -1799,7 +1797,7 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
 		if (group->parent)
 			break;
 		if ((!root_mismatch || i >= tmigr_crossnode_level) &&
-		    list_is_singular(&tmigr_level_list[i]))
+		    list_is_singular(&hier->level_list[i]))
 			break;
 	}
 
@@ -1827,15 +1825,15 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
 			tmc->tmgroup = group;
 			tmc->groupmask = BIT(group->num_children++);
 
-			tmigr_init_root(group, activate);
+			tmigr_init_root(hier, group, activate);
 
-			trace_tmigr_connect_cpu_parent(tmc);
+			trace_tmigr_connect_cpu_parent(hier, tmc);
 
 			/* There are no children that need to be connected */
 			continue;
 		} else {
 			child = stack[i - 1];
-			tmigr_connect_child_parent(child, group, activate);
+			tmigr_connect_child_parent(hier, child, group, activate);
 		}
 	}
 
@@ -1891,18 +1889,23 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
 			data.childmask = start->groupmask;
 			__walk_groups_from(tmigr_active_up, &data, start, start->parent);
 		}
+	} else if (start) {
+		union tmigr_state state;
+
+		/* Remote activation assumes the whole target's hierarchy is inactive */
+		state.state = atomic_read(&start->migr_state);
+		WARN_ON_ONCE(state.active);
 	}
 
 	/* Root update */
-	if (list_is_singular(&tmigr_level_list[top])) {
-		group = list_first_entry(&tmigr_level_list[top],
-					 typeof(*group), list);
+	if (list_is_singular(&hier->level_list[top])) {
+		group = list_first_entry(&hier->level_list[top], typeof(*group), list);
 		WARN_ON_ONCE(group->parent);
-		if (tmigr_root) {
+		if (root) {
 			/* Old root should be the same or below */
-			WARN_ON_ONCE(tmigr_root->level > top);
+			WARN_ON_ONCE(root->level > top);
 		}
-		tmigr_root = group;
+		hier->root = group;
 	}
 out:
 	kfree(stack);
@@ -1910,34 +1913,132 @@ static int tmigr_setup_groups(unsigned int cpu, unsigned int node,
 	return err;
 }
 
+static struct tmigr_hierarchy *tmigr_get_hierarchy(unsigned int capacity)
+{
+	struct tmigr_hierarchy *hier = NULL, *iter;
+
+	list_for_each_entry(iter, &tmigr_hierarchy_list, node) {
+		if (iter->capacity == capacity)
+			hier = iter;
+	}
+
+	if (hier)
+		return hier;
+
+	hier = kzalloc(sizeof(*hier), GFP_KERNEL);
+	if (!hier)
+		return ERR_PTR(-ENOMEM);
+
+	hier->cpumask = kzalloc(cpumask_size(), GFP_KERNEL);
+	if (!hier->cpumask)
+		goto err;
+
+	hier->level_list = kzalloc_objs(struct list_head, tmigr_hierarchy_levels);
+	if (!hier->level_list)
+		goto err;
+
+	for (int i = 0; i < tmigr_hierarchy_levels; i++)
+		INIT_LIST_HEAD(&hier->level_list[i]);
+
+	hier->capacity = capacity;
+	list_add_tail(&hier->node, &tmigr_hierarchy_list);
+
+	return hier;
+err:
+	kfree(hier->cpumask);
+	kfree(hier);
+	return ERR_PTR(-ENOMEM);
+}
+
+static int tmigr_connect_old_root(struct tmigr_hierarchy *hier, int cpu,
+				  struct tmigr_group *old_root,	bool activate)
+{
+	/*
+	 * The target CPU must never do the prepare work, except
+	 * on early boot when the boot CPU is the target. Otherwise
+	 * it may spuriously activate the old top level group inside
+	 * the new one (nevertheless whether old top level group is
+	 * active or not) and/or release an uninitialized childmask.
+	 */
+	WARN_ON_ONCE(cpu == smp_processor_id());
+	if (activate) {
+		/*
+		 * The current CPU is expected to be online in the hierarchy,
+		 * otherwise the old root may not be active as expected.
+		 */
+		WARN_ON_ONCE(!__this_cpu_read(tmigr_cpu.available));
+	}
+
+	return tmigr_setup_groups(hier, -1, old_root->numa_node, old_root, activate);
+}
+
+static long connect_old_root_work(void *arg)
+{
+	struct tmigr_group *old_root = arg;
+	struct tmigr_hierarchy *hier;
+	int cpu = smp_processor_id();
+
+	hier = tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu));
+	if (IS_ERR(hier))
+		return PTR_ERR(hier);
+
+	return tmigr_connect_old_root(hier, cpu, old_root, true);
+}
+
 static int tmigr_add_cpu(unsigned int cpu)
 {
-	struct tmigr_group *old_root = tmigr_root;
+	struct tmigr_hierarchy *hier;
+	struct tmigr_group *old_root;
 	int node = cpu_to_node(cpu);
 	int ret;
 
 	guard(mutex)(&tmigr_mutex);
 
-	ret = tmigr_setup_groups(cpu, node, NULL, false);
+	hier = tmigr_get_hierarchy(arch_scale_cpu_capacity(cpu));
+	if (IS_ERR(hier))
+		return PTR_ERR(hier);
+
+	old_root = hier->root;
+
+	ret = tmigr_setup_groups(hier, cpu, node, NULL, false);
+
+	if (ret < 0)
+		return ret;
 
 	/* Root has changed? Connect the old one to the new */
-	if (ret >= 0 && old_root && old_root != tmigr_root) {
-		/*
-		 * The target CPU must never do the prepare work, except
-		 * on early boot when the boot CPU is the target. Otherwise
-		 * it may spuriously activate the old top level group inside
-		 * the new one (nevertheless whether old top level group is
-		 * active or not) and/or release an uninitialized childmask.
-		 */
-		WARN_ON_ONCE(cpu == raw_smp_processor_id());
-		/*
-		 * The (likely) current CPU is expected to be online in the hierarchy,
-		 * otherwise the old root may not be active as expected.
-		 */
-		WARN_ON_ONCE(!per_cpu_ptr(&tmigr_cpu, raw_smp_processor_id())->available);
-		ret = tmigr_setup_groups(-1, old_root->numa_node, old_root, true);
+	if (old_root && old_root != hier->root) {
+		guard(migrate)();
+
+		if (cpumask_test_cpu(smp_processor_id(), hier->cpumask)) {
+			/*
+			 * If the target belong to the same hierarchy, the old root is expected
+			 * to be active. Link and propagate to the new root.
+			 */
+			ret = tmigr_connect_old_root(hier, cpu, old_root, true);
+		} else {
+			int target = cpumask_first_and(hier->cpumask, tmigr_available_cpumask);
+
+			if (target < nr_cpu_ids) {
+				/*
+				 * If the target doesn't belong to the same hierarchy as the current
+				 * CPU, activate from a relevant one to make sure the old root is
+				 * active.
+				 */
+				ret = work_on_cpu(target, connect_old_root_work, old_root);
+			} else {
+				/*
+				 * No other available CPUs in the remote hierarchy. Link the
+				 * old root remotely but don't propagate activation since the
+				 * old root is not expected to be active.
+				 */
+				ret = tmigr_connect_old_root(hier, cpu, old_root, false);
+			}
+		}
 	}
 
+	if (ret >= 0)
+		cpumask_set_cpu(cpu, hier->cpumask);
+
 	return ret;
 }
 
@@ -1970,7 +2071,7 @@ static int tmigr_cpu_prepare(unsigned int cpu)
 
 static int __init tmigr_init(void)
 {
-	unsigned int cpulvl, nodelvl, cpus_per_node, i;
+	unsigned int cpulvl, nodelvl, cpus_per_node;
 	unsigned int nnodes = num_possible_nodes();
 	unsigned int ncpus = num_possible_cpus();
 	int ret = -ENOMEM;
@@ -2017,14 +2118,6 @@ static int __init tmigr_init(void)
 	 */
 	tmigr_crossnode_level = cpulvl;
 
-	tmigr_level_list = kzalloc_objs(struct list_head,
-					tmigr_hierarchy_levels);
-	if (!tmigr_level_list)
-		goto err;
-
-	for (i = 0; i < tmigr_hierarchy_levels; i++)
-		INIT_LIST_HEAD(&tmigr_level_list[i]);
-
 	pr_info("Timer migration: %d hierarchy levels; %d children per group;"
 		" %d crossnode level\n",
 		tmigr_hierarchy_levels, TMIGR_CHILDREN_PER_GROUP,
diff --git a/kernel/time/timer_migration.h b/kernel/time/timer_migration.h
index 70879cde6fdd0..291bfb6adfc30 100644
--- a/kernel/time/timer_migration.h
+++ b/kernel/time/timer_migration.h
@@ -5,6 +5,25 @@
 /* Per group capacity. Must be a power of 2! */
 #define TMIGR_CHILDREN_PER_GROUP 8
 
+/**
+ * struct tmigr_hierarchy - a hierarchy associated to a given CPU capacity.
+ *                          Homogeneous systems have only one hierarchy.
+ *                          Heterogenous have one hierarchy per CPU capacity.
+ * @level_list:	Per level lists of tmigr groups
+ * @cpumask:	CPUs belonging to this hierarchy
+ * @root:	The current root of the hierarchy
+ * @capacity:	CPU capacity associated to this hierarchy
+ * @node:	Node in the global hierarchy list
+ */
+struct tmigr_hierarchy {
+	struct list_head	*level_list;
+	struct cpumask		*cpumask;
+	struct tmigr_group	*root;
+	unsigned long		capacity;
+	struct list_head	node;
+
+};
+
 /**
  * struct tmigr_event - a timer event associated to a CPU
  * @nextevt:	The node to enqueue an event in the parent group queue
diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
index a02bd258677ee..d853f97bd154f 100644
--- a/kernel/trace/bpf_trace.c
+++ b/kernel/trace/bpf_trace.c
@@ -152,6 +152,34 @@ unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
 	return ret;
 }
 
+/**
+ * trace_call_bpf_faultable - invoke BPF program in faultable context
+ * @call: tracepoint event
+ * @ctx: opaque context pointer
+ *
+ * Variant of trace_call_bpf() for faultable tracepoints (syscall
+ * tracepoints). Supports sleepable BPF programs by using rcu_tasks_trace
+ * for lifetime protection and bpf_prog_run_array_sleepable() for per-program
+ * RCU flavor selection, following the uprobe pattern.
+ *
+ * Per-program recursion protection is provided by
+ * bpf_prog_run_array_sleepable(). Global bpf_prog_active is not
+ * needed because syscall tracepoints cannot self-recurse.
+ *
+ * Must be called from a faultable/preemptible context.
+ */
+unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx)
+{
+	struct bpf_prog_array *prog_array;
+
+	might_fault();
+	guard(rcu_tasks_trace)();
+
+	prog_array = rcu_dereference_check(call->prog_array,
+					   rcu_read_lock_trace_held());
+	return bpf_prog_run_array_sleepable(prog_array, ctx, bpf_prog_run);
+}
+
 #ifdef CONFIG_BPF_KPROBE_OVERRIDE
 BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)
 {
@@ -2072,11 +2100,19 @@ void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
 static __always_inline
 void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args)
 {
+	struct srcu_ctr __percpu *scp = NULL;
 	struct bpf_prog *prog = link->link.prog;
+	bool sleepable = prog->sleepable;
 	struct bpf_run_ctx *old_run_ctx;
 	struct bpf_trace_run_ctx run_ctx;
 
-	rcu_read_lock_dont_migrate();
+	if (sleepable) {
+		scp = rcu_read_lock_tasks_trace();
+		migrate_disable();
+	} else {
+		rcu_read_lock_dont_migrate();
+	}
+
 	if (unlikely(!bpf_prog_get_recursion_context(prog))) {
 		bpf_prog_inc_misses_counter(prog);
 		goto out;
@@ -2085,12 +2121,18 @@ void __bpf_trace_run(struct bpf_raw_tp_link *link, u64 *args)
 	run_ctx.bpf_cookie = link->cookie;
 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
 
-	(void) bpf_prog_run(prog, args);
+	(void)bpf_prog_run(prog, args);
 
 	bpf_reset_run_ctx(old_run_ctx);
 out:
 	bpf_prog_put_recursion_context(prog);
-	rcu_read_unlock_migrate();
+
+	if (sleepable) {
+		migrate_enable();
+		rcu_read_unlock_tasks_trace(scp);
+	} else {
+		rcu_read_unlock_migrate();
+	}
 }
 
 #define UNPACK(...)			__VA_ARGS__
@@ -3398,12 +3440,12 @@ typedef int (*copy_fn_t)(void *dst, const void *src, u32 size, struct task_struc
  * direct calls into all the specific callback implementations
  * (copy_user_data_sleepable, copy_user_data_nofault, and so on)
  */
-static __always_inline int __bpf_dynptr_copy_str(struct bpf_dynptr *dptr, u64 doff, u64 size,
+static __always_inline int __bpf_dynptr_copy_str(const struct bpf_dynptr *dptr, u64 doff, u64 size,
 						 const void *unsafe_src,
 						 copy_fn_t str_copy_fn,
 						 struct task_struct *tsk)
 {
-	struct bpf_dynptr_kern *dst;
+	const struct bpf_dynptr_kern *dst;
 	u64 chunk_sz, off;
 	void *dst_slice;
 	int cnt, err;
@@ -3439,7 +3481,7 @@ static __always_inline int __bpf_dynptr_copy(const struct bpf_dynptr *dptr, u64
 					     u64 size, const void *unsafe_src,
 					     copy_fn_t copy_fn, struct task_struct *tsk)
 {
-	struct bpf_dynptr_kern *dst;
+	const struct bpf_dynptr_kern *dst;
 	void *dst_slice;
 	char buf[256];
 	u64 off, chunk_sz;
@@ -3540,49 +3582,49 @@ __bpf_kfunc int bpf_send_signal_task(struct task_struct *task, int sig, enum pid
 	return bpf_send_signal_common(sig, type, task, value);
 }
 
-__bpf_kfunc int bpf_probe_read_user_dynptr(struct bpf_dynptr *dptr, u64 off,
+__bpf_kfunc int bpf_probe_read_user_dynptr(const struct bpf_dynptr *dptr, u64 off,
 					   u64 size, const void __user *unsafe_ptr__ign)
 {
 	return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign,
 				 copy_user_data_nofault, NULL);
 }
 
-__bpf_kfunc int bpf_probe_read_kernel_dynptr(struct bpf_dynptr *dptr, u64 off,
+__bpf_kfunc int bpf_probe_read_kernel_dynptr(const struct bpf_dynptr *dptr, u64 off,
 					     u64 size, const void *unsafe_ptr__ign)
 {
 	return __bpf_dynptr_copy(dptr, off, size, unsafe_ptr__ign,
 				 copy_kernel_data_nofault, NULL);
 }
 
-__bpf_kfunc int bpf_probe_read_user_str_dynptr(struct bpf_dynptr *dptr, u64 off,
+__bpf_kfunc int bpf_probe_read_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off,
 					       u64 size, const void __user *unsafe_ptr__ign)
 {
 	return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign,
 				     copy_user_str_nofault, NULL);
 }
 
-__bpf_kfunc int bpf_probe_read_kernel_str_dynptr(struct bpf_dynptr *dptr, u64 off,
+__bpf_kfunc int bpf_probe_read_kernel_str_dynptr(const struct bpf_dynptr *dptr, u64 off,
 						 u64 size, const void *unsafe_ptr__ign)
 {
 	return __bpf_dynptr_copy_str(dptr, off, size, unsafe_ptr__ign,
 				     copy_kernel_str_nofault, NULL);
 }
 
-__bpf_kfunc int bpf_copy_from_user_dynptr(struct bpf_dynptr *dptr, u64 off,
+__bpf_kfunc int bpf_copy_from_user_dynptr(const struct bpf_dynptr *dptr, u64 off,
 					  u64 size, const void __user *unsafe_ptr__ign)
 {
 	return __bpf_dynptr_copy(dptr, off, size, (const void __force *)unsafe_ptr__ign,
 				 copy_user_data_sleepable, NULL);
 }
 
-__bpf_kfunc int bpf_copy_from_user_str_dynptr(struct bpf_dynptr *dptr, u64 off,
+__bpf_kfunc int bpf_copy_from_user_str_dynptr(const struct bpf_dynptr *dptr, u64 off,
 					      u64 size, const void __user *unsafe_ptr__ign)
 {
 	return __bpf_dynptr_copy_str(dptr, off, size, (const void __force *)unsafe_ptr__ign,
 				     copy_user_str_sleepable, NULL);
 }
 
-__bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u64 off,
+__bpf_kfunc int bpf_copy_from_user_task_dynptr(const struct bpf_dynptr *dptr, u64 off,
 					       u64 size, const void __user *unsafe_ptr__ign,
 					       struct task_struct *tsk)
 {
@@ -3590,7 +3632,7 @@ __bpf_kfunc int bpf_copy_from_user_task_dynptr(struct bpf_dynptr *dptr, u64 off,
 				 copy_user_data_sleepable, tsk);
 }
 
-__bpf_kfunc int bpf_copy_from_user_task_str_dynptr(struct bpf_dynptr *dptr, u64 off,
+__bpf_kfunc int bpf_copy_from_user_task_str_dynptr(const struct bpf_dynptr *dptr, u64 off,
 						   u64 size, const void __user *unsafe_ptr__ign,
 						   struct task_struct *tsk)
 {
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 5326924615a4e..02691c3c6dd63 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -3769,13 +3769,6 @@ rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
 	return skip_time_extend(event);
 }
 
-#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
-static inline bool sched_clock_stable(void)
-{
-	return true;
-}
-#endif
-
 static void
 rb_check_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
 		   struct rb_event_info *info)
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 8ad72e17d8eb9..e98ee7e1e66f4 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -1371,33 +1371,33 @@ static DECLARE_BITMAP(enabled_perf_exit_syscalls, NR_syscalls);
 static int sys_perf_refcount_enter;
 static int sys_perf_refcount_exit;
 
-static int perf_call_bpf_enter(struct trace_event_call *call, struct pt_regs *regs,
+static int perf_call_bpf_enter(struct trace_event_call *call,
 			       struct syscall_metadata *sys_data,
-			       struct syscall_trace_enter *rec)
+			       int syscall_nr, unsigned long *args)
 {
 	struct syscall_tp_t {
 		struct trace_entry ent;
 		int syscall_nr;
 		unsigned long args[SYSCALL_DEFINE_MAXARGS];
 	} __aligned(8) param;
+	struct pt_regs regs = {};
 	int i;
 
 	BUILD_BUG_ON(sizeof(param.ent) < sizeof(void *));
 
-	/* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */
-	perf_fetch_caller_regs(regs);
-	*(struct pt_regs **)&param = regs;
-	param.syscall_nr = rec->nr;
+	/* bpf prog requires 'regs' to be the first member in the ctx */
+	perf_fetch_caller_regs(&regs);
+	*(struct pt_regs **)&param = &regs;
+	param.syscall_nr = syscall_nr;
 	for (i = 0; i < sys_data->nb_args; i++)
-		param.args[i] = rec->args[i];
-	return trace_call_bpf(call, &param);
+		param.args[i] = args[i];
+	return trace_call_bpf_faultable(call, &param);
 }
 
 static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_enter *rec;
-	struct pt_regs *fake_regs;
 	struct hlist_head *head;
 	unsigned long args[6];
 	bool valid_prog_array;
@@ -1410,12 +1410,7 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
 	int size = 0;
 	int uargs = 0;
 
-	/*
-	 * Syscall probe called with preemption enabled, but the ring
-	 * buffer and per-cpu data require preemption to be disabled.
-	 */
 	might_fault();
-	guard(preempt_notrace)();
 
 	syscall_nr = trace_get_syscall_nr(current, regs);
 	if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
@@ -1429,6 +1424,26 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
 
 	syscall_get_arguments(current, regs, args);
 
+	/*
+	 * Run BPF program in faultable context before per-cpu buffer
+	 * allocation, allowing sleepable BPF programs to execute.
+	 */
+	valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
+	if (valid_prog_array &&
+	    !perf_call_bpf_enter(sys_data->enter_event, sys_data,
+				 syscall_nr, args))
+		return;
+
+	/*
+	 * Per-cpu ring buffer and perf event list operations require
+	 * preemption to be disabled.
+	 */
+	guard(preempt_notrace)();
+
+	head = this_cpu_ptr(sys_data->enter_event->perf_events);
+	if (hlist_empty(head))
+		return;
+
 	/* Check if this syscall event faults in user space memory */
 	mayfault = sys_data->user_mask != 0;
 
@@ -1438,17 +1453,12 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
 			return;
 	}
 
-	head = this_cpu_ptr(sys_data->enter_event->perf_events);
-	valid_prog_array = bpf_prog_array_valid(sys_data->enter_event);
-	if (!valid_prog_array && hlist_empty(head))
-		return;
-
 	/* get the size after alignment with the u32 buffer size field */
 	size += sizeof(unsigned long) * sys_data->nb_args + sizeof(*rec);
 	size = ALIGN(size + sizeof(u32), sizeof(u64));
 	size -= sizeof(u32);
 
-	rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);
+	rec = perf_trace_buf_alloc(size, NULL, &rctx);
 	if (!rec)
 		return;
 
@@ -1458,13 +1468,6 @@ static void perf_syscall_enter(void *ignore, struct pt_regs *regs, long id)
 	if (mayfault)
 		syscall_put_data(sys_data, rec, user_ptr, size, user_sizes, uargs);
 
-	if ((valid_prog_array &&
-	     !perf_call_bpf_enter(sys_data->enter_event, fake_regs, sys_data, rec)) ||
-	    hlist_empty(head)) {
-		perf_swevent_put_recursion_context(rctx);
-		return;
-	}
-
 	perf_trace_buf_submit(rec, size, rctx,
 			      sys_data->enter_event->event.type, 1, regs,
 			      head, NULL);
@@ -1514,40 +1517,35 @@ static void perf_sysenter_disable(struct trace_event_call *call)
 		syscall_fault_buffer_disable();
 }
 
-static int perf_call_bpf_exit(struct trace_event_call *call, struct pt_regs *regs,
-			      struct syscall_trace_exit *rec)
+static int perf_call_bpf_exit(struct trace_event_call *call,
+			      int syscall_nr, long ret_val)
 {
 	struct syscall_tp_t {
 		struct trace_entry ent;
 		int syscall_nr;
 		unsigned long ret;
 	} __aligned(8) param;
-
-	/* bpf prog requires 'regs' to be the first member in the ctx (a.k.a. &param) */
-	perf_fetch_caller_regs(regs);
-	*(struct pt_regs **)&param = regs;
-	param.syscall_nr = rec->nr;
-	param.ret = rec->ret;
-	return trace_call_bpf(call, &param);
+	struct pt_regs regs = {};
+
+	/* bpf prog requires 'regs' to be the first member in the ctx */
+	perf_fetch_caller_regs(&regs);
+	*(struct pt_regs **)&param = &regs;
+	param.syscall_nr = syscall_nr;
+	param.ret = ret_val;
+	return trace_call_bpf_faultable(call, &param);
 }
 
 static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
 {
 	struct syscall_metadata *sys_data;
 	struct syscall_trace_exit *rec;
-	struct pt_regs *fake_regs;
 	struct hlist_head *head;
 	bool valid_prog_array;
 	int syscall_nr;
 	int rctx;
 	int size;
 
-	/*
-	 * Syscall probe called with preemption enabled, but the ring
-	 * buffer and per-cpu data require preemption to be disabled.
-	 */
 	might_fault();
-	guard(preempt_notrace)();
 
 	syscall_nr = trace_get_syscall_nr(current, regs);
 	if (syscall_nr < 0 || syscall_nr >= NR_syscalls)
@@ -1559,29 +1557,37 @@ static void perf_syscall_exit(void *ignore, struct pt_regs *regs, long ret)
 	if (!sys_data)
 		return;
 
-	head = this_cpu_ptr(sys_data->exit_event->perf_events);
+	/*
+	 * Run BPF program in faultable context before per-cpu buffer
+	 * allocation, allowing sleepable BPF programs to execute.
+	 */
 	valid_prog_array = bpf_prog_array_valid(sys_data->exit_event);
-	if (!valid_prog_array && hlist_empty(head))
+	if (valid_prog_array &&
+	    !perf_call_bpf_exit(sys_data->exit_event, syscall_nr,
+				syscall_get_return_value(current, regs)))
+		return;
+
+	/*
+	 * Per-cpu ring buffer and perf event list operations require
+	 * preemption to be disabled.
+	 */
+	guard(preempt_notrace)();
+
+	head = this_cpu_ptr(sys_data->exit_event->perf_events);
+	if (hlist_empty(head))
 		return;
 
 	/* We can probably do that at build time */
 	size = ALIGN(sizeof(*rec) + sizeof(u32), sizeof(u64));
 	size -= sizeof(u32);
 
-	rec = perf_trace_buf_alloc(size, &fake_regs, &rctx);
+	rec = perf_trace_buf_alloc(size, NULL, &rctx);
 	if (!rec)
 		return;
 
 	rec->nr = syscall_nr;
 	rec->ret = syscall_get_return_value(current, regs);
 
-	if ((valid_prog_array &&
-	     !perf_call_bpf_exit(sys_data->exit_event, fake_regs, rec)) ||
-	    hlist_empty(head)) {
-		perf_swevent_put_recursion_context(rctx);
-		return;
-	}
-
 	perf_trace_buf_submit(rec, size, rctx, sys_data->exit_event->event.type,
 			      1, regs, head, NULL);
 }
diff --git a/kernel/trace/trace_uprobe.c b/kernel/trace/trace_uprobe.c
index 2cabf8a23ec5c..c5ee7920dec63 100644
--- a/kernel/trace/trace_uprobe.c
+++ b/kernel/trace/trace_uprobe.c
@@ -979,6 +979,7 @@ static struct uprobe_cpu_buffer *prepare_uprobe_buffer(struct trace_uprobe *tu,
 	ucb = uprobe_buffer_get();
 	ucb->dsize = tu->tp.size + dsize;
 
+	BUILD_BUG_ON(MAX_UCB_BUFFER_SIZE < MAX_PROBE_EVENT_SIZE);
 	if (WARN_ON_ONCE(ucb->dsize > MAX_UCB_BUFFER_SIZE)) {
 		ucb->dsize = MAX_UCB_BUFFER_SIZE;
 		dsize = MAX_UCB_BUFFER_SIZE - tu->tp.size;
diff --git a/kernel/umh.c b/kernel/umh.c
index cffda97d961cb..48117c569e1aa 100644
--- a/kernel/umh.c
+++ b/kernel/umh.c
@@ -430,7 +430,7 @@ int call_usermodehelper_exec(struct subprocess_info *sub_info, int wait)
 	sub_info->complete = (wait == UMH_NO_WAIT) ? NULL : &done;
 	sub_info->wait = wait;
 
-	queue_work(system_unbound_wq, &sub_info->work);
+	queue_work(system_dfl_wq, &sub_info->work);
 	if (wait == UMH_NO_WAIT)	/* task has freed sub_info */
 		goto unlock;
 
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 33b721a9af022..8df671066dd1a 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -3195,6 +3195,10 @@ static bool manage_workers(struct worker *worker)
 	return true;
 }
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+static noinline void process_one_work(struct worker *worker, struct work_struct *work);
+#endif
+
 /**
  * process_one_work - process single work
  * @worker: self
@@ -5312,16 +5316,6 @@ static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
 	return pwq;
 }
 
-static void apply_wqattrs_lock(void)
-{
-	mutex_lock(&wq_pool_mutex);
-}
-
-static void apply_wqattrs_unlock(void)
-{
-	mutex_unlock(&wq_pool_mutex);
-}
-
 /**
  * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod
  * @attrs: the wq_attrs of the default pwq of the target workqueue
@@ -5877,7 +5871,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
 	 * wq_pool_mutex protects the workqueues list, allocations of PWQs,
 	 * and the global freeze state.
 	 */
-	apply_wqattrs_lock();
+	mutex_lock(&wq_pool_mutex);
 
 	if (alloc_and_link_pwqs(wq) < 0)
 		goto err_unlock_free_node_nr_active;
@@ -5891,7 +5885,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
 	if (wq_online && init_rescuer(wq) < 0)
 		goto err_unlock_destroy;
 
-	apply_wqattrs_unlock();
+	mutex_unlock(&wq_pool_mutex);
 
 	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
 		goto err_destroy;
@@ -5899,7 +5893,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
 	return wq;
 
 err_unlock_free_node_nr_active:
-	apply_wqattrs_unlock();
+	mutex_unlock(&wq_pool_mutex);
 	/*
 	 * Failed alloc_and_link_pwqs() may leave pending pwq->release_work,
 	 * flushing the pwq_release_worker ensures that the pwq_release_workfn()
@@ -5914,7 +5908,7 @@ static struct workqueue_struct *__alloc_workqueue(const char *fmt,
 	kfree(wq);
 	return NULL;
 err_unlock_destroy:
-	apply_wqattrs_unlock();
+	mutex_unlock(&wq_pool_mutex);
 err_destroy:
 	destroy_workqueue(wq);
 	return NULL;
@@ -7315,7 +7309,7 @@ static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
 	struct workqueue_attrs *attrs;
 	int ret = -ENOMEM;
 
-	apply_wqattrs_lock();
+	mutex_lock(&wq_pool_mutex);
 
 	attrs = wq_sysfs_prep_attrs(wq);
 	if (!attrs)
@@ -7328,7 +7322,7 @@ static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
 		ret = -EINVAL;
 
 out_unlock:
-	apply_wqattrs_unlock();
+	mutex_unlock(&wq_pool_mutex);
 	free_workqueue_attrs(attrs);
 	return ret ?: count;
 }
@@ -7354,7 +7348,7 @@ static ssize_t wq_cpumask_store(struct device *dev,
 	struct workqueue_attrs *attrs;
 	int ret = -ENOMEM;
 
-	apply_wqattrs_lock();
+	mutex_lock(&wq_pool_mutex);
 
 	attrs = wq_sysfs_prep_attrs(wq);
 	if (!attrs)
@@ -7365,7 +7359,7 @@ static ssize_t wq_cpumask_store(struct device *dev,
 		ret = apply_workqueue_attrs_locked(wq, attrs);
 
 out_unlock:
-	apply_wqattrs_unlock();
+	mutex_unlock(&wq_pool_mutex);
 	free_workqueue_attrs(attrs);
 	return ret ?: count;
 }
@@ -7401,13 +7395,13 @@ static ssize_t wq_affn_scope_store(struct device *dev,
 	if (affn < 0)
 		return affn;
 
-	apply_wqattrs_lock();
+	mutex_lock(&wq_pool_mutex);
 	attrs = wq_sysfs_prep_attrs(wq);
 	if (attrs) {
 		attrs->affn_scope = affn;
 		ret = apply_workqueue_attrs_locked(wq, attrs);
 	}
-	apply_wqattrs_unlock();
+	mutex_unlock(&wq_pool_mutex);
 	free_workqueue_attrs(attrs);
 	return ret ?: count;
 }
@@ -7432,13 +7426,13 @@ static ssize_t wq_affinity_strict_store(struct device *dev,
 	if (sscanf(buf, "%d", &v) != 1)
 		return -EINVAL;
 
-	apply_wqattrs_lock();
+	mutex_lock(&wq_pool_mutex);
 	attrs = wq_sysfs_prep_attrs(wq);
 	if (attrs) {
 		attrs->affn_strict = (bool)v;
 		ret = apply_workqueue_attrs_locked(wq, attrs);
 	}
-	apply_wqattrs_unlock();
+	mutex_unlock(&wq_pool_mutex);
 	free_workqueue_attrs(attrs);
 	return ret ?: count;
 }
@@ -7479,12 +7473,12 @@ static int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
 	cpumask_and(cpumask, cpumask, cpu_possible_mask);
 	if (!cpumask_empty(cpumask)) {
 		ret = 0;
-		apply_wqattrs_lock();
+		mutex_lock(&wq_pool_mutex);
 		if (!cpumask_equal(cpumask, wq_unbound_cpumask))
 			ret = workqueue_apply_unbound_cpumask(cpumask);
 		if (!ret)
 			cpumask_copy(wq_requested_unbound_cpumask, cpumask);
-		apply_wqattrs_unlock();
+		mutex_unlock(&wq_pool_mutex);
 	}
 
 	return ret;
@@ -8212,11 +8206,7 @@ static bool __init cpus_dont_share(int cpu0, int cpu1)
 
 static bool __init cpus_share_smt(int cpu0, int cpu1)
 {
-#ifdef CONFIG_SCHED_SMT
 	return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));
-#else
-	return false;
-#endif
 }
 
 static bool __init cpus_share_numa(int cpu0, int cpu1)
diff --git a/lib/Kconfig b/lib/Kconfig
index 00a9509636c18..f01a33e521e12 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -8,17 +8,6 @@ config BINARY_PRINTF
 
 menu "Library routines"
 
-config RAID6_PQ
-	tristate
-
-config RAID6_PQ_BENCHMARK
-	bool "Automatically choose fastest RAID6 PQ functions"
-	depends on RAID6_PQ
-	default y
-	help
-	  Benchmark all available RAID6 PQ functions on init and choose the
-	  fastest one.
-
 config LINEAR_RANGES
 	tristate
 
@@ -590,7 +579,7 @@ config OBJAGG
 config LWQ_TEST
 	bool "Boot-time test for lwq queuing"
 	help
-          Run boot-time test of light-weight queuing.
+	  Run boot-time test of light-weight queuing.
 
 endmenu
 
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 8ff5adcfe1e0a..e2f976c3301b6 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -630,7 +630,7 @@ config DEBUG_FORCE_WEAK_PER_CPU
 
 config WARN_CONTEXT_ANALYSIS
 	bool "Compiler context-analysis warnings"
-	depends on CC_IS_CLANG && CLANG_VERSION >= 220100
+	depends on CC_IS_CLANG && CLANG_VERSION >= 230000
 	# Branch profiling re-defines "if", which messes with the compiler's
 	# ability to analyze __cond_acquires(..), resulting in false positives.
 	depends on !TRACE_BRANCH_PROFILING
@@ -641,7 +641,7 @@ config WARN_CONTEXT_ANALYSIS
 	  and releasing user-definable "context locks".
 
 	  Clang's name of the feature is "Thread Safety Analysis". Requires
-	  Clang 22.1.0 or later.
+	  Clang 23 or later.
 
 	  Produces warnings by default. Select CONFIG_WERROR if you wish to
 	  turn these warnings into errors.
@@ -2261,6 +2261,13 @@ config KCOV_SELFTEST
 	  On test failure, causes the kernel to panic. Recommended to be
 	  enabled, ensuring critical functionality works as intended.
 
+config DEBUG_AID_FOR_SYZBOT
+       bool "Additional debug code for syzbot"
+       default n
+       select PREEMPT_RT if EXPERT && ARCH_SUPPORTS_RT && !COMPILE_TEST
+       help
+         This option is intended for testing by syzbot.
+
 menuconfig RUNTIME_TESTING_MENU
 	bool "Runtime Testing"
 	default y
@@ -2649,12 +2656,17 @@ config TEST_VMALLOC
 
 config TEST_WORKQUEUE
 	tristate "Test module for stress/performance analysis of workqueue"
+	depends on m
 	default n
 	help
 	  This builds the "test_workqueue" module for benchmarking
 	  workqueue throughput under contention. Useful for evaluating
 	  affinity scope changes (e.g., cache_shard vs cache).
 
+	  The test drives sysfs to switch affinity scopes, so it must be
+	  loaded after userspace has mounted sysfs; building it in (=y)
+	  would run module_init before /sys is available.
+
 	  If unsure, say N.
 
 config TEST_BPF
@@ -2971,6 +2983,16 @@ config BITS_TEST
 
 	  If unsure, say N.
 
+config SHDI3_KUNIT_TEST
+	tristate "KUnit test for __ashldi3(), __ashrdi3(), and __lshrdi3()"
+	depends on KUNIT
+	depends on ARM || XTENSA || MICROBLAZE || ((RISCV || SPARC) && !64BIT)
+	help
+	  This builds the unit test for __ashldi3(), __ashrdi3(), and
+	  __lshrdi3() helper functions used to implement 64-bit arithmetic
+	  shift left, arithmetic shift right and logical shift right,
+	  respectively, on a 32-bit CPUs.
+
 config SLUB_KUNIT_TEST
 	tristate "KUnit test for SLUB cache error detection" if !KUNIT_ALL_TESTS
 	depends on SLUB_DEBUG && KUNIT
@@ -3070,7 +3092,7 @@ config FORTIFY_KUNIT_TEST
 config LONGEST_SYM_KUNIT_TEST
 	tristate "Test the longest symbol possible" if !KUNIT_ALL_TESTS
 	depends on KUNIT && KPROBES
-	depends on !PREFIX_SYMBOLS && !CFI && !GCOV_KERNEL
+	depends on !CALL_PADDING && !CFI && !GCOV_KERNEL
 	default KUNIT_ALL_TESTS
 	help
 	  Tests the longest symbol possible
diff --git a/lib/Makefile b/lib/Makefile
index f33a24bf1c19a..6e72d2c1cce71 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -167,7 +167,6 @@ obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
 obj-$(CONFIG_ZSTD_COMPRESS) += zstd/
 obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/
 obj-$(CONFIG_XZ_DEC) += xz/
-obj-$(CONFIG_RAID6_PQ) += raid6/
 
 lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o
 lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o
diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
index ed1bdcf1f8abf..af44bc8bff012 100644
--- a/lib/alloc_tag.c
+++ b/lib/alloc_tag.c
@@ -54,7 +54,7 @@ static void *allocinfo_start(struct seq_file *m, loff_t *pos)
 	loff_t node = *pos;
 
 	priv = (struct allocinfo_private *)m->private;
-	codetag_lock_module_list(alloc_tag_cttype, true);
+	codetag_lock_module_list(alloc_tag_cttype);
 	if (node == 0) {
 		priv->print_header = true;
 		priv->iter = codetag_get_ct_iter(alloc_tag_cttype);
@@ -77,7 +77,7 @@ static void *allocinfo_next(struct seq_file *m, void *arg, loff_t *pos)
 
 static void allocinfo_stop(struct seq_file *m, void *arg)
 {
-	codetag_lock_module_list(alloc_tag_cttype, false);
+	codetag_unlock_module_list(alloc_tag_cttype);
 }
 
 static void print_allocinfo_header(struct seq_buf *buf)
@@ -136,7 +136,7 @@ size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sl
 		return 0;
 
 	if (can_sleep)
-		codetag_lock_module_list(alloc_tag_cttype, true);
+		codetag_lock_module_list(alloc_tag_cttype);
 	else if (!codetag_trylock_module_list(alloc_tag_cttype))
 		return 0;
 
@@ -161,7 +161,7 @@ size_t alloc_tag_top_users(struct codetag_bytes *tags, size_t count, bool can_sl
 		}
 	}
 
-	codetag_lock_module_list(alloc_tag_cttype, false);
+	codetag_unlock_module_list(alloc_tag_cttype);
 
 	return nr;
 }
@@ -767,60 +767,95 @@ static __init bool need_page_alloc_tagging(void)
  * their codetag uninitialized. Track these early PFNs so we can clear
  * their codetag refs later to avoid warnings when they are freed.
  *
- * Early allocations include:
- *   - Base allocations independent of CPU count
- *   - Per-CPU allocations (e.g., CPU hotplug callbacks during smp_init,
- *     such as trace ring buffers, scheduler per-cpu data)
- *
- * For simplicity, we fix the size to 8192.
- * If insufficient, a warning will be triggered to alert the user.
+ * Each page is cast to a pfn_pool: the first few bytes hold metadata
+ * (next pointer and slot count), the remainder stores PFNs.
+ */
+struct pfn_pool {
+	struct pfn_pool *next;
+	atomic_t count;
+	unsigned long pfns[];
+};
+
+#define PFN_POOL_SIZE			((PAGE_SIZE - offsetof(struct pfn_pool, pfns)) / \
+					 sizeof(unsigned long))
+
+/*
+ * Skip early PFN recording for a page allocation.  Reuses the
+ * %__GFP_NO_OBJ_EXT bit.  Used by __alloc_tag_add_early_pfn() to avoid
+ * recursion when allocating pages for the early PFN tracking list
+ * itself.
  *
- * TODO: Replace fixed-size array with dynamic allocation using
- * a GFP flag similar to ___GFP_NO_OBJ_EXT to avoid recursion.
+ * Codetags of the pages allocated with __GFP_NO_CODETAG should be
+ * cleared (via clear_page_tag_ref()) before freeing the pages to prevent
+ * alloc_tag_sub_check() from triggering a warning.
  */
-#define EARLY_ALLOC_PFN_MAX		8192
+#define __GFP_NO_CODETAG		__GFP_NO_OBJ_EXT
 
-static unsigned long early_pfns[EARLY_ALLOC_PFN_MAX] __initdata;
-static atomic_t early_pfn_count __initdata = ATOMIC_INIT(0);
+static struct pfn_pool *current_pfn_pool __initdata;
 
-static void __init __alloc_tag_add_early_pfn(unsigned long pfn)
+static void __init __alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags)
 {
-	int old_idx, new_idx;
+	struct pfn_pool *pool;
+	int idx;
 
 	do {
-		old_idx = atomic_read(&early_pfn_count);
-		if (old_idx >= EARLY_ALLOC_PFN_MAX) {
-			pr_warn_once("Early page allocations before page_ext init exceeded EARLY_ALLOC_PFN_MAX (%d)\n",
-				      EARLY_ALLOC_PFN_MAX);
-			return;
+		pool = READ_ONCE(current_pfn_pool);
+		if (!pool || atomic_read(&pool->count) >= PFN_POOL_SIZE) {
+			gfp_t gfp = gfp_flags & ~(__GFP_DIRECT_RECLAIM | GFP_ZONEMASK);
+			struct page *new_page = alloc_page(gfp | __GFP_NO_CODETAG);
+			struct pfn_pool *new;
+
+			if (!new_page) {
+				pr_warn_once("early PFN tracking page allocation failed\n");
+				return;
+			}
+			new = page_address(new_page);
+			new->next = pool;
+			atomic_set(&new->count, 0);
+			if (cmpxchg(&current_pfn_pool, pool, new) != pool) {
+				clear_page_tag_ref(new_page);
+				__free_page(new_page);
+				continue;
+			}
+			pool = new;
 		}
-		new_idx = old_idx + 1;
-	} while (!atomic_try_cmpxchg(&early_pfn_count, &old_idx, new_idx));
+		idx = atomic_read(&pool->count);
+		if (idx >= PFN_POOL_SIZE)
+			continue;
+		if (atomic_cmpxchg(&pool->count, idx, idx + 1) == idx)
+			break;
+	} while (1);
 
-	early_pfns[old_idx] = pfn;
+	pool->pfns[idx] = pfn;
 }
 
-typedef void alloc_tag_add_func(unsigned long pfn);
+typedef void alloc_tag_add_func(unsigned long pfn, gfp_t gfp_flags);
 static alloc_tag_add_func __rcu *alloc_tag_add_early_pfn_ptr __refdata =
 	RCU_INITIALIZER(__alloc_tag_add_early_pfn);
 
-void alloc_tag_add_early_pfn(unsigned long pfn)
+void alloc_tag_add_early_pfn(unsigned long pfn, gfp_t gfp_flags)
 {
 	alloc_tag_add_func *alloc_tag_add;
 
 	if (static_key_enabled(&mem_profiling_compressed))
 		return;
 
+	/* Skip allocations for the tracking list itself to avoid recursion. */
+	if (gfp_flags & __GFP_NO_CODETAG)
+		return;
+
 	rcu_read_lock();
 	alloc_tag_add = rcu_dereference(alloc_tag_add_early_pfn_ptr);
 	if (alloc_tag_add)
-		alloc_tag_add(pfn);
+		alloc_tag_add(pfn, gfp_flags);
 	rcu_read_unlock();
 }
 
 static void __init clear_early_alloc_pfn_tag_refs(void)
 {
-	unsigned int i;
+	struct pfn_pool *pool, *next;
+	struct page *page;
+	int i;
 
 	if (static_key_enabled(&mem_profiling_compressed))
 		return;
@@ -829,37 +864,45 @@ static void __init clear_early_alloc_pfn_tag_refs(void)
 	/* Make sure we are not racing with __alloc_tag_add_early_pfn() */
 	synchronize_rcu();
 
-	for (i = 0; i < atomic_read(&early_pfn_count); i++) {
-		unsigned long pfn = early_pfns[i];
-
-		if (pfn_valid(pfn)) {
-			struct page *page = pfn_to_page(pfn);
-			union pgtag_ref_handle handle;
-			union codetag_ref ref;
-
-			if (get_page_tag_ref(page, &ref, &handle)) {
-				/*
-				 * An early-allocated page could be freed and reallocated
-				 * after its page_ext is initialized but before we clear it.
-				 * In that case, it already has a valid tag set.
-				 * We should not overwrite that valid tag with CODETAG_EMPTY.
-				 *
-				 * Note: there is still a small race window between checking
-				 * ref.ct and calling set_codetag_empty(). We accept this
-				 * race as it's unlikely and the extra complexity of atomic
-				 * cmpxchg is not worth it for this debug-only code path.
-				 */
-				if (ref.ct) {
+	for (pool = current_pfn_pool; pool; pool = next) {
+		int nr_pfns = atomic_read(&pool->count);
+
+		for (i = 0; i < nr_pfns; i++) {
+			unsigned long pfn = pool->pfns[i];
+
+			if (pfn_valid(pfn)) {
+				union pgtag_ref_handle handle;
+				union codetag_ref ref;
+
+				if (get_page_tag_ref(pfn_to_page(pfn), &ref, &handle)) {
+					/*
+					 * An early-allocated page could be freed and reallocated
+					 * after its page_ext is initialized but before we clear it.
+					 * In that case, it already has a valid tag set.
+					 * We should not overwrite that valid tag
+					 * with CODETAG_EMPTY.
+					 *
+					 * Note: there is still a small race window between checking
+					 * ref.ct and calling set_codetag_empty(). We accept this
+					 * race as it's unlikely and the extra complexity of atomic
+					 * cmpxchg is not worth it for this debug-only code path.
+					 */
+					if (ref.ct) {
+						put_page_tag_ref(handle);
+						continue;
+					}
+
+					set_codetag_empty(&ref);
+					update_page_tag_ref(handle, &ref);
 					put_page_tag_ref(handle);
-					continue;
 				}
-
-				set_codetag_empty(&ref);
-				update_page_tag_ref(handle, &ref);
-				put_page_tag_ref(handle);
 			}
 		}
 
+		next = pool->next;
+		page = virt_to_page(pool);
+		clear_page_tag_ref(page);
+		__free_page(page);
 	}
 }
 #else /* !CONFIG_MEM_ALLOC_PROFILING_DEBUG */
diff --git a/lib/base64.c b/lib/base64.c
index 41961a444028b..325c7332b0492 100644
--- a/lib/base64.c
+++ b/lib/base64.c
@@ -122,7 +122,7 @@ EXPORT_SYMBOL_GPL(base64_encode);
  * @src: the string to decode.  Doesn't need to be NUL-terminated.
  * @srclen: the length of @src in bytes
  * @dst: (output) the decoded binary data
- * @padding: whether to append '=' padding characters
+ * @padding: whether the input is expected to include '=' padding characters
  * @variant: which base64 variant to use
  *
  * Decodes a string using the selected Base64 variant.
@@ -168,15 +168,16 @@ int base64_decode(const char *src, int srclen, u8 *dst, bool padding, enum base6
 		return -1;
 
 	val = (base64_rev_tables[s[0]] << 12) | (base64_rev_tables[s[1]] << 6);
-	*bp++ = val >> 10;
 
 	if (srclen == 2) {
 		if (val & 0x800003ff)
 			return -1;
+		*bp++ = val >> 10;
 	} else {
 		val |= base64_rev_tables[s[2]];
 		if (val & 0x80000003)
 			return -1;
+		*bp++ = val >> 10;
 		*bp++ = val >> 2;
 	}
 	return bp - dst;
diff --git a/lib/bootconfig.c b/lib/bootconfig.c
index c470b93d5dbc2..f445b7703fdd9 100644
--- a/lib/bootconfig.c
+++ b/lib/bootconfig.c
@@ -408,6 +408,62 @@ const char * __init xbc_node_find_next_key_value(struct xbc_node *root,
 		return "";	/* No value key */
 }
 
+static char xbc_namebuf[XBC_KEYLEN_MAX] __initdata;
+
+#define rest(dst, end) ((end) > (dst) ? (end) - (dst) : 0)
+
+/**
+ * xbc_snprint_cmdline() - Render bootconfig keys under @root as a cmdline string
+ * @buf: Destination buffer (may be NULL when @size is 0 to query the length)
+ * @size: Size of @buf in bytes
+ * @root: Subtree root whose key=value pairs should be rendered
+ *
+ * Walk all key/value pairs under @root and emit them as a space-separated
+ * cmdline string into @buf. Values containing whitespace are quoted with
+ * double quotes. Returns the number of bytes that would be written if @buf
+ * were large enough (matching snprintf semantics), or a negative errno on
+ * failure.
+ */
+int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root)
+{
+	struct xbc_node *knode, *vnode;
+	char *end = buf + size;
+	const char *val, *q;
+	int ret;
+
+	xbc_node_for_each_key_value(root, knode, val) {
+		ret = xbc_node_compose_key_after(root, knode,
+					xbc_namebuf, XBC_KEYLEN_MAX);
+		if (ret < 0)
+			return ret;
+
+		vnode = xbc_node_get_child(knode);
+		if (!vnode) {
+			ret = snprintf(buf, rest(buf, end), "%s ", xbc_namebuf);
+			if (ret < 0)
+				return ret;
+			buf += ret;
+			continue;
+		}
+		xbc_array_for_each_value(vnode, val) {
+			/*
+			 * For prettier and more readable /proc/cmdline, only
+			 * quote the value when necessary, i.e. when it contains
+			 * whitespace.
+			 */
+			q = strpbrk(val, " \t\r\n") ? "\"" : "";
+			ret = snprintf(buf, rest(buf, end), "%s=%s%s%s ",
+				       xbc_namebuf, q, val, q);
+			if (ret < 0)
+				return ret;
+			buf += ret;
+		}
+	}
+
+	return buf - (end - size);
+}
+#undef rest
+
 /* XBC parse and tree build */
 
 static int __init xbc_init_node(struct xbc_node *node, char *data, uint16_t flag)
diff --git a/lib/bug.c b/lib/bug.c
index 224f4cfa4aa31..292420f45811a 100644
--- a/lib/bug.c
+++ b/lib/bug.c
@@ -1,41 +1,41 @@
 // SPDX-License-Identifier: GPL-2.0
 /*
-  Generic support for BUG()
-
-  This respects the following config options:
-
-  CONFIG_BUG - emit BUG traps.  Nothing happens without this.
-  CONFIG_GENERIC_BUG - enable this code.
-  CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file
-  CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG
-
-  CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable
-  (though they're generally always on).
-
-  CONFIG_GENERIC_BUG is set by each architecture using this code.
-
-  To use this, your architecture must:
-
-  1. Set up the config options:
-     - Enable CONFIG_GENERIC_BUG if CONFIG_BUG
-
-  2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON)
-     - Define HAVE_ARCH_BUG
-     - Implement BUG() to generate a faulting instruction
-     - NOTE: struct bug_entry does not have "file" or "line" entries
-       when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate
-       the values accordingly.
-
-  3. Implement the trap
-     - In the illegal instruction trap handler (typically), verify
-       that the fault was in kernel mode, and call report_bug()
-     - report_bug() will return whether it was a false alarm, a warning,
-       or an actual bug.
-     - You must implement the is_valid_bugaddr(bugaddr) callback which
-       returns true if the eip is a real kernel address, and it points
-       to the expected BUG trap instruction.
-
-    Jeremy Fitzhardinge <jeremy@goop.org> 2006
+ *  Generic support for BUG()
+ *
+ *  This respects the following config options:
+ *
+ *  CONFIG_BUG - emit BUG traps.  Nothing happens without this.
+ *  CONFIG_GENERIC_BUG - enable this code.
+ *  CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file
+ *  CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG
+ *
+ *  CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable
+ *  (though they're generally always on).
+ *
+ *  CONFIG_GENERIC_BUG is set by each architecture using this code.
+ *
+ *  To use this, your architecture must:
+ *
+ *  1. Set up the config options:
+ *     - Enable CONFIG_GENERIC_BUG if CONFIG_BUG
+ *
+ *  2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON)
+ *     - Define HAVE_ARCH_BUG
+ *     - Implement BUG() to generate a faulting instruction
+ *     - NOTE: struct bug_entry does not have "file" or "line" entries
+ *       when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate
+ *       the values accordingly.
+ *
+ *  3. Implement the trap
+ *     - In the illegal instruction trap handler (typically), verify
+ *       that the fault was in kernel mode, and call report_bug()
+ *     - report_bug() will return whether it was a false alarm, a warning,
+ *       or an actual bug.
+ *     - You must implement the is_valid_bugaddr(bugaddr) callback which
+ *       returns true if the eip is a real kernel address, and it points
+ *       to the expected BUG trap instruction.
+ *
+ *    Jeremy Fitzhardinge <jeremy@goop.org> 2006
  */
 
 #define pr_fmt(fmt) fmt
@@ -48,6 +48,7 @@
 #include <linux/rculist.h>
 #include <linux/ftrace.h>
 #include <linux/context_tracking.h>
+#include <kunit/test-bug.h>
 
 extern struct bug_entry __start___bug_table[], __stop___bug_table[];
 
@@ -71,7 +72,7 @@ static struct bug_entry *module_find_bug(unsigned long bugaddr)
 
 	guard(rcu)();
 	list_for_each_entry_rcu(mod, &module_bug_list, bug_list) {
-		unsigned i;
+		unsigned int i;
 
 		bug = mod->bug_table;
 		for (i = 0; i < mod->num_bugs; ++i, ++bug)
@@ -191,14 +192,14 @@ void __warn_printf(const char *fmt, struct pt_regs *regs)
 	}
 #endif
 
-	printk("%s", fmt);
+	pr_warn("%s", fmt);
 }
 
 static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long bugaddr, struct pt_regs *regs)
 {
 	bool warning, once, done, no_cut, has_args;
 	const char *file, *fmt;
-	unsigned line;
+	unsigned int line;
 
 	if (!bug) {
 		if (!is_valid_bugaddr(bugaddr))
@@ -209,8 +210,6 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga
 			return BUG_TRAP_TYPE_NONE;
 	}
 
-	disable_trace_on_warning();
-
 	bug_get_file_line(bug, &file, &line);
 	fmt = bug_get_format(bug);
 
@@ -220,6 +219,17 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga
 	no_cut   = bug->flags & BUGFLAG_NO_CUT_HERE;
 	has_args = bug->flags & BUGFLAG_ARGS;
 
+#ifdef CONFIG_KUNIT
+	/*
+	 * Before the once logic so suppressed warnings do not consume
+	 * the single-fire budget of WARN_ON_ONCE().
+	 */
+	if (warning && kunit_is_suppressed_warning(true))
+		return BUG_TRAP_TYPE_WARN;
+#endif
+
+	disable_trace_on_warning();
+
 	if (warning && once) {
 		if (done)
 			return BUG_TRAP_TYPE_WARN;
@@ -237,7 +247,7 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga
 	 * extra debugging message it writes before triggering the handler.
 	 */
 	if (!no_cut) {
-		printk(KERN_DEFAULT CUT_HERE);
+		pr_info(CUT_HERE);
 		__warn_printf(fmt, has_args ? regs : NULL);
 	}
 
diff --git a/lib/cmdline.c b/lib/cmdline.c
index 90ed997d95701..16cce6621cec1 100644
--- a/lib/cmdline.c
+++ b/lib/cmdline.c
@@ -43,7 +43,7 @@ static int get_range(char **str, int *pint, int n)
  *	When @pint is NULL the function can be used as a validator of
  *	the current option in the string.
  *
- *	Return values:
+ *	Return:
  *	0 - no int in string
  *	1 - int found, no subsequent comma
  *	2 - int found including a subsequent comma
@@ -145,44 +145,54 @@ EXPORT_SYMBOL(get_options);
  *
  *	Parses a string into a number.  The number stored at @ptr is
  *	potentially suffixed with K, M, G, T, P, E.
+ *
+ *	Return: The value as recognized by simple_strtoull() multiplied
+ *	by the value as specified by suffix, if any.
  */
 
 unsigned long long memparse(const char *ptr, char **retptr)
 {
 	char *endptr;	/* local pointer to end of parsed string */
-
 	unsigned long long ret = simple_strtoull(ptr, &endptr, 0);
+	unsigned int shl = 0;
 
+	/* Consume valid suffix even in case of overflow. */
 	switch (*endptr) {
 	case 'E':
 	case 'e':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'P':
 	case 'p':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'T':
 	case 't':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'G':
 	case 'g':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'M':
 	case 'm':
-		ret <<= 10;
+		shl += 10;
 		fallthrough;
 	case 'K':
 	case 'k':
-		ret <<= 10;
-		endptr++;
+		shl += 10;
 		fallthrough;
 	default:
 		break;
 	}
 
+	if (shl && likely(ptr != endptr)) {
+		/* Have valid suffix with preceding number. */
+		if (unlikely(check_shl_overflow(ret, shl, &ret)))
+			ret = ULLONG_MAX;
+		endptr++;
+	}
+
 	if (retptr)
 		*retptr = endptr;
 
@@ -198,7 +208,7 @@ EXPORT_SYMBOL(memparse);
  *	This function parses a string containing a comma-separated list of
  *	strings like a=b,c.
  *
- *	Return true if there's such option in the string, or return false.
+ *	Return: True if there's such option in the string or false otherwise.
  */
 bool parse_option_str(const char *str, const char *option)
 {
diff --git a/lib/codetag.c b/lib/codetag.c
index 304667897ad4a..4001a7ea66754 100644
--- a/lib/codetag.c
+++ b/lib/codetag.c
@@ -35,12 +35,9 @@ struct codetag_module {
 static DEFINE_MUTEX(codetag_lock);
 static LIST_HEAD(codetag_types);
 
-void codetag_lock_module_list(struct codetag_type *cttype, bool lock)
+void codetag_lock_module_list(struct codetag_type *cttype)
 {
-	if (lock)
-		down_read(&cttype->mod_lock);
-	else
-		up_read(&cttype->mod_lock);
+	down_read(&cttype->mod_lock);
 }
 
 bool codetag_trylock_module_list(struct codetag_type *cttype)
@@ -48,6 +45,11 @@ bool codetag_trylock_module_list(struct codetag_type *cttype)
 	return down_read_trylock(&cttype->mod_lock) != 0;
 }
 
+void codetag_unlock_module_list(struct codetag_type *cttype)
+{
+	up_read(&cttype->mod_lock);
+}
+
 struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype)
 {
 	struct codetag_iterator iter = {
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index d3904b72dae7a..591c1c2a7fb31 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -131,11 +131,6 @@ config CRYPTO_LIB_MD5
 	  The MD5 and HMAC-MD5 library functions.  Select this if your module
 	  uses any of the functions from <crypto/md5.h>.
 
-config CRYPTO_LIB_MD5_ARCH
-	bool
-	depends on CRYPTO_LIB_MD5 && !UML
-	default y if PPC
-
 config CRYPTO_LIB_MLDSA
 	tristate
 	select CRYPTO_LIB_SHA3
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 4ad91f3900380..f1e9bf89785ff 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -187,10 +187,6 @@ clean-files += powerpc/ghashp8-ppc.S
 
 obj-$(CONFIG_CRYPTO_LIB_MD5) += libmd5.o
 libmd5-y := md5.o
-ifeq ($(CONFIG_CRYPTO_LIB_MD5_ARCH),y)
-CFLAGS_md5.o += -I$(src)/$(SRCARCH)
-libmd5-$(CONFIG_PPC) += powerpc/md5-asm.o
-endif # CONFIG_CRYPTO_LIB_MD5_ARCH
 
 ################################################################################
 
diff --git a/lib/crypto/md5.c b/lib/crypto/md5.c
index c4af57db0ea83..6bf130cfbbf98 100644
--- a/lib/crypto/md5.c
+++ b/lib/crypto/md5.c
@@ -2,8 +2,8 @@
 /*
  * MD5 and HMAC-MD5 library functions
  *
- * md5_block_generic() is derived from cryptoapi implementation, originally
- * based on the public domain implementation written by Colin Plumb in 1993.
+ * md5_block() is derived from cryptoapi implementation, originally based on the
+ * public domain implementation written by Colin Plumb in 1993.
  *
  * Copyright (c) Cryptoapi developers.
  * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
@@ -31,8 +31,8 @@ static const struct md5_block_state md5_iv = {
 #define MD5STEP(f, w, x, y, z, in, s) \
 	(w += f(x, y, z) + in, w = rol32(w, s) + x)
 
-static void md5_block_generic(struct md5_block_state *state,
-			      const u8 data[MD5_BLOCK_SIZE])
+static void md5_block(struct md5_block_state *state,
+		      const u8 data[MD5_BLOCK_SIZE])
 {
 	u32 in[MD5_BLOCK_WORDS];
 	u32 a, b, c, d;
@@ -119,21 +119,15 @@ static void md5_block_generic(struct md5_block_state *state,
 	state->h[3] += d;
 }
 
-static void __maybe_unused md5_blocks_generic(struct md5_block_state *state,
-					      const u8 *data, size_t nblocks)
+static void md5_blocks(struct md5_block_state *state,
+		       const u8 *data, size_t nblocks)
 {
 	do {
-		md5_block_generic(state, data);
+		md5_block(state, data);
 		data += MD5_BLOCK_SIZE;
 	} while (--nblocks);
 }
 
-#ifdef CONFIG_CRYPTO_LIB_MD5_ARCH
-#include "md5.h" /* $(SRCARCH)/md5.h */
-#else
-#define md5_blocks md5_blocks_generic
-#endif
-
 void md5_init(struct md5_ctx *ctx)
 {
 	ctx->state = md5_iv;
diff --git a/lib/crypto/powerpc/md5-asm.S b/lib/crypto/powerpc/md5-asm.S
deleted file mode 100644
index fa6bc440cf4ac..0000000000000
--- a/lib/crypto/powerpc/md5-asm.S
+++ /dev/null
@@ -1,235 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Fast MD5 implementation for PPC
- *
- * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
- */
-#include <asm/ppc_asm.h>
-#include <asm/asm-offsets.h>
-#include <asm/asm-compat.h>
-
-#define rHP	r3
-#define rWP	r4
-
-#define rH0	r0
-#define rH1	r6
-#define rH2	r7
-#define rH3	r5
-
-#define rW00	r8
-#define rW01	r9
-#define rW02	r10
-#define rW03	r11
-#define rW04	r12
-#define rW05	r14
-#define rW06	r15
-#define rW07	r16
-#define rW08	r17
-#define rW09	r18
-#define rW10	r19
-#define rW11	r20
-#define rW12	r21
-#define rW13	r22
-#define rW14	r23
-#define rW15	r24
-
-#define rT0	r25
-#define rT1	r26
-
-#define INITIALIZE \
-	PPC_STLU r1,-INT_FRAME_SIZE(r1); \
-	SAVE_GPRS(14, 26, r1)		/* push registers onto stack	*/
-
-#define FINALIZE \
-	REST_GPRS(14, 26, r1);		/* pop registers from stack	*/ \
-	addi	r1,r1,INT_FRAME_SIZE
-
-#ifdef __BIG_ENDIAN__
-#define LOAD_DATA(reg, off) \
-	lwbrx		reg,0,rWP;	/* load data			*/
-#define INC_PTR \
-	addi		rWP,rWP,4;	/* increment per word		*/
-#define NEXT_BLOCK			/* nothing to do		*/
-#else
-#define LOAD_DATA(reg, off) \
-	lwz		reg,off(rWP);	/* load data			*/
-#define INC_PTR				/* nothing to do		*/
-#define NEXT_BLOCK \
-	addi		rWP,rWP,64;	/* increment per block		*/
-#endif
-
-#define R_00_15(a, b, c, d, w0, w1, p, q, off, k0h, k0l, k1h, k1l) \
-	LOAD_DATA(w0, off)		/*    W				*/ \
-	and		rT0,b,c;	/* 1: f = b and c		*/ \
-	INC_PTR				/*    ptr++			*/ \
-	andc		rT1,d,b;	/* 1: f' = ~b and d		*/ \
-	LOAD_DATA(w1, off+4)		/*    W				*/ \
-	or		rT0,rT0,rT1;	/* 1: f = f or f'		*/ \
-	addi		w0,w0,k0l;	/* 1: wk = w + k		*/ \
-	add		a,a,rT0;	/* 1: a = a + f			*/ \
-	addis		w0,w0,k0h;	/* 1: wk = w + k'		*/ \
-	addis		w1,w1,k1h;	/* 2: wk = w + k		*/ \
-	add		a,a,w0;		/* 1: a = a + wk		*/ \
-	addi		w1,w1,k1l;	/* 2: wk = w + k'		*/ \
-	rotrwi		a,a,p;		/* 1: a = a rotl x		*/ \
-	add		d,d,w1;		/* 2: a = a + wk		*/ \
-	add		a,a,b;		/* 1: a = a + b			*/ \
-	and		rT0,a,b;	/* 2: f = b and c		*/ \
-	andc		rT1,c,a;	/* 2: f' = ~b and d		*/ \
-	or		rT0,rT0,rT1;	/* 2: f = f or f'		*/ \
-	add		d,d,rT0;	/* 2: a = a + f			*/ \
-	INC_PTR				/*    ptr++			*/ \
-	rotrwi		d,d,q;		/* 2: a = a rotl x		*/ \
-	add		d,d,a;		/* 2: a = a + b			*/
-
-#define R_16_31(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \
-	andc		rT0,c,d;	/* 1: f = c and ~d		*/ \
-	and		rT1,b,d;	/* 1: f' = b and d		*/ \
-	addi		w0,w0,k0l;	/* 1: wk = w + k		*/ \
-	or		rT0,rT0,rT1;	/* 1: f = f or f'		*/ \
-	addis		w0,w0,k0h;	/* 1: wk = w + k'		*/ \
-	add		a,a,rT0;	/* 1: a = a + f			*/ \
-	addi		w1,w1,k1l;	/* 2: wk = w + k		*/ \
-	add		a,a,w0;		/* 1: a = a + wk		*/ \
-	addis		w1,w1,k1h;	/* 2: wk = w + k'		*/ \
-	andc		rT0,b,c;	/* 2: f = c and ~d		*/ \
-	rotrwi		a,a,p;		/* 1: a = a rotl x		*/ \
-	add		a,a,b;		/* 1: a = a + b			*/ \
-	add		d,d,w1;		/* 2: a = a + wk		*/ \
-	and		rT1,a,c;	/* 2: f' = b and d		*/ \
-	or		rT0,rT0,rT1;	/* 2: f = f or f'		*/ \
-	add		d,d,rT0;	/* 2: a = a + f			*/ \
-	rotrwi		d,d,q;		/* 2: a = a rotl x		*/ \
-	add		d,d,a;		/* 2: a = a +b			*/
-
-#define R_32_47(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \
-	xor		rT0,b,c;	/* 1: f' = b xor c		*/ \
-	addi		w0,w0,k0l;	/* 1: wk = w + k		*/ \
-	xor		rT1,rT0,d;	/* 1: f = f xor f'		*/ \
-	addis		w0,w0,k0h;	/* 1: wk = w + k'		*/ \
-	add		a,a,rT1;	/* 1: a = a + f			*/ \
-	addi		w1,w1,k1l;	/* 2: wk = w + k		*/ \
-	add		a,a,w0;		/* 1: a = a + wk		*/ \
-	addis		w1,w1,k1h;	/* 2: wk = w + k'		*/ \
-	rotrwi		a,a,p;		/* 1: a = a rotl x		*/ \
-	add		d,d,w1;		/* 2: a = a + wk		*/ \
-	add		a,a,b;		/* 1: a = a + b			*/ \
-	xor		rT1,rT0,a;	/* 2: f = b xor f'		*/ \
-	add		d,d,rT1;	/* 2: a = a + f			*/ \
-	rotrwi		d,d,q;		/* 2: a = a rotl x		*/ \
-	add		d,d,a;		/* 2: a = a + b			*/
-
-#define R_48_63(a, b, c, d, w0, w1, p, q, k0h, k0l, k1h, k1l) \
-	addi		w0,w0,k0l;	/* 1: w = w + k			*/ \
-	orc		rT0,b,d;	/* 1: f = b or ~d		*/ \
-	addis		w0,w0,k0h;	/* 1: w = w + k'		*/ \
-	xor		rT0,rT0,c;	/* 1: f = f xor c		*/ \
-	add		a,a,w0;		/* 1: a = a + wk		*/ \
-	addi		w1,w1,k1l;	/* 2: w = w + k			*/ \
-	add		a,a,rT0;	/* 1: a = a + f			*/ \
-	addis		w1,w1,k1h;	/* 2: w = w + k'		*/ \
-	rotrwi		a,a,p;		/* 1: a = a rotl x		*/ \
-	add		a,a,b;		/* 1: a = a + b			*/ \
-	orc		rT0,a,c;	/* 2: f = b or ~d		*/ \
-	add		d,d,w1;		/* 2: a = a + wk		*/ \
-	xor		rT0,rT0,b;	/* 2: f = f xor c		*/ \
-	add		d,d,rT0;	/* 2: a = a + f			*/ \
-	rotrwi		d,d,q;		/* 2: a = a rotl x		*/ \
-	add		d,d,a;		/* 2: a = a + b			*/
-
-_GLOBAL(ppc_md5_transform)
-	INITIALIZE
-
-	mtctr		r5
-	lwz		rH0,0(rHP)
-	lwz		rH1,4(rHP)
-	lwz		rH2,8(rHP)
-	lwz		rH3,12(rHP)
-
-ppc_md5_main:
-	R_00_15(rH0, rH1, rH2, rH3, rW00, rW01, 25, 20, 0,
-		0xd76b, -23432, 0xe8c8, -18602)
-	R_00_15(rH2, rH3, rH0, rH1, rW02, rW03, 15, 10, 8,
-		0x2420, 0x70db, 0xc1be, -12562)
-	R_00_15(rH0, rH1, rH2, rH3, rW04, rW05, 25, 20, 16,
-		0xf57c, 0x0faf, 0x4788, -14806)
-	R_00_15(rH2, rH3, rH0, rH1, rW06, rW07, 15, 10, 24,
-		0xa830, 0x4613, 0xfd47, -27391)
-	R_00_15(rH0, rH1, rH2, rH3, rW08, rW09, 25, 20, 32,
-		0x6981, -26408, 0x8b45,  -2129)
-	R_00_15(rH2, rH3, rH0, rH1, rW10, rW11, 15, 10, 40,
-		0xffff, 0x5bb1, 0x895d, -10306)
-	R_00_15(rH0, rH1, rH2, rH3, rW12, rW13, 25, 20, 48,
-		0x6b90, 0x1122, 0xfd98, 0x7193)
-	R_00_15(rH2, rH3, rH0, rH1, rW14, rW15, 15, 10, 56,
-		0xa679, 0x438e, 0x49b4, 0x0821)
-
-	R_16_31(rH0, rH1, rH2, rH3, rW01, rW06, 27, 23,
-		0x0d56, 0x6e0c, 0x1810, 0x6d2d)
-	R_16_31(rH2, rH3, rH0, rH1, rW11, rW00, 18, 12,
-		0x9d02, -32109, 0x124c, 0x2332)
-	R_16_31(rH0, rH1, rH2, rH3, rW05, rW10, 27, 23,
-		0x8ea7, 0x4a33, 0x0245, -18270)
-	R_16_31(rH2, rH3, rH0, rH1, rW15, rW04, 18, 12,
-		0x8eee,  -8608, 0xf258,  -5095)
-	R_16_31(rH0, rH1, rH2, rH3, rW09, rW14, 27, 23,
-		0x969d, -10697, 0x1cbe, -15288)
-	R_16_31(rH2, rH3, rH0, rH1, rW03, rW08, 18, 12,
-		0x3317, 0x3e99, 0xdbd9, 0x7c15)
-	R_16_31(rH0, rH1, rH2, rH3, rW13, rW02, 27, 23,
-		0xac4b, 0x7772, 0xd8cf, 0x331d)
-	R_16_31(rH2, rH3, rH0, rH1, rW07, rW12, 18, 12,
-		0x6a28, 0x6dd8, 0x219a, 0x3b68)
-
-	R_32_47(rH0, rH1, rH2, rH3, rW05, rW08, 28, 21,
-		0x29cb, 0x28e5, 0x4218,  -7788)
-	R_32_47(rH2, rH3, rH0, rH1, rW11, rW14, 16,  9,
-		0x473f, 0x06d1, 0x3aae, 0x3036)
-	R_32_47(rH0, rH1, rH2, rH3, rW01, rW04, 28, 21,
-		0xaea1, -15134, 0x640b, -11295)
-	R_32_47(rH2, rH3, rH0, rH1, rW07, rW10, 16,  9,
-		0x8f4c, 0x4887, 0xbc7c, -22499)
-	R_32_47(rH0, rH1, rH2, rH3, rW13, rW00, 28, 21,
-		0x7eb8, -27199, 0x00ea, 0x6050)
-	R_32_47(rH2, rH3, rH0, rH1, rW03, rW06, 16,  9,
-		0xe01a, 0x22fe, 0x4447, 0x69c5)
-	R_32_47(rH0, rH1, rH2, rH3, rW09, rW12, 28, 21,
-		0xb7f3, 0x0253, 0x59b1, 0x4d5b)
-	R_32_47(rH2, rH3, rH0, rH1, rW15, rW02, 16,  9,
-		0x4701, -27017, 0xc7bd, -19859)
-
-	R_48_63(rH0, rH1, rH2, rH3, rW00, rW07, 26, 22,
-		0x0988,  -1462, 0x4c70, -19401)
-	R_48_63(rH2, rH3, rH0, rH1, rW14, rW05, 17, 11,
-		0xadaf,  -5221, 0xfc99, 0x66f7)
-	R_48_63(rH0, rH1, rH2, rH3, rW12, rW03, 26, 22,
-		0x7e80, -16418, 0xba1e, -25587)
-	R_48_63(rH2, rH3, rH0, rH1, rW10, rW01, 17, 11,
-		0x4130, 0x380d, 0xe0c5, 0x738d)
-	lwz		rW00,0(rHP)
-	R_48_63(rH0, rH1, rH2, rH3, rW08, rW15, 26, 22,
-		0xe837, -30770, 0xde8a, 0x69e8)
-	lwz		rW14,4(rHP)
-	R_48_63(rH2, rH3, rH0, rH1, rW06, rW13, 17, 11,
-		0x9e79, 0x260f, 0x256d, -27941)
-	lwz		rW12,8(rHP)
-	R_48_63(rH0, rH1, rH2, rH3, rW04, rW11, 26, 22,
-		0xab75, -20775, 0x4f9e, -28397)
-	lwz		rW10,12(rHP)
-	R_48_63(rH2, rH3, rH0, rH1, rW02, rW09, 17, 11,
-		0x662b, 0x7c56, 0x11b2, 0x0358)
-
-	add		rH0,rH0,rW00
-	stw		rH0,0(rHP)
-	add		rH1,rH1,rW14
-	stw		rH1,4(rHP)
-	add		rH2,rH2,rW12
-	stw		rH2,8(rHP)
-	add		rH3,rH3,rW10
-	stw		rH3,12(rHP)
-	NEXT_BLOCK
-
-	bdnz		ppc_md5_main
-
-	FINALIZE
-	blr
diff --git a/lib/crypto/powerpc/md5.h b/lib/crypto/powerpc/md5.h
deleted file mode 100644
index 540b08e34d1d5..0000000000000
--- a/lib/crypto/powerpc/md5.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * MD5 optimized for PowerPC
- */
-
-void ppc_md5_transform(u32 *state, const u8 *data, size_t nblocks);
-
-static void md5_blocks(struct md5_block_state *state,
-		       const u8 *data, size_t nblocks)
-{
-	ppc_md5_transform(state->h, data, nblocks);
-}
diff --git a/lib/debugobjects.c b/lib/debugobjects.c
index 12e2e42e6a31a..772ddabcbe7d3 100644
--- a/lib/debugobjects.c
+++ b/lib/debugobjects.c
@@ -711,6 +711,15 @@ static struct debug_obj *lookup_object_or_alloc(void *addr, struct debug_bucket
 	return NULL;
 }
 
+static inline bool debug_objects_is_pi_blocked_on(void)
+{
+#ifdef CONFIG_RT_MUTEXES
+	return current->pi_blocked_on != NULL;
+#else
+	return false;
+#endif
+}
+
 static void debug_objects_fill_pool(void)
 {
 	if (!static_branch_likely(&obj_cache_enabled))
@@ -727,11 +736,12 @@ static void debug_objects_fill_pool(void)
 
 	/*
 	 * On RT enabled kernels the pool refill must happen in preemptible
-	 * context -- for !RT kernels we rely on the fact that spinlock_t and
-	 * raw_spinlock_t are basically the same type and this lock-type
-	 * inversion works just fine.
+	 * context and not enqueued on an rt_mutex -- for !RT kernels we rely
+	 * on the fact that spinlock_t and raw_spinlock_t are basically the
+	 * same type and this lock-type inversion works just fine.
 	 */
-	if (!IS_ENABLED(CONFIG_PREEMPT_RT) || preemptible() || system_state < SYSTEM_SCHEDULING) {
+	if (!IS_ENABLED(CONFIG_PREEMPT_RT) || system_state < SYSTEM_SCHEDULING ||
+	    (preemptible() && !debug_objects_is_pi_blocked_on())) {
 		/*
 		 * Annotate away the spinlock_t inside raw_spinlock_t warning
 		 * by temporarily raising the wait-type to LD_WAIT_CONFIG, matching
diff --git a/lib/error-inject.c b/lib/error-inject.c
index f3d1b70be605c..32f3d1ca9ea23 100644
--- a/lib/error-inject.c
+++ b/lib/error-inject.c
@@ -219,9 +219,9 @@ static int __init ei_debugfs_init(void)
 	dir = debugfs_create_dir("error_injection", NULL);
 
 	file = debugfs_create_file("list", 0444, dir, NULL, &ei_fops);
-	if (!file) {
+	if (IS_ERR(file)) {
 		debugfs_remove(dir);
-		return -ENOMEM;
+		return PTR_ERR(file);
 	}
 
 	return 0;
diff --git a/lib/idr.c b/lib/idr.c
index 69bee5369670c..1649f41016e7c 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -495,10 +495,9 @@ int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max)
 	unsigned long index = min / IDA_BITMAP_BITS;
 	unsigned int offset = min % IDA_BITMAP_BITS;
 	unsigned long *addr, size, bit;
-	unsigned long tmp = 0;
+	unsigned long tmp;
 	unsigned long flags;
 	void *entry;
-	int ret;
 
 	if ((int)min < 0)
 		return -EINVAL;
@@ -508,40 +507,34 @@ int ida_find_first_range(struct ida *ida, unsigned int min, unsigned int max)
 	xa_lock_irqsave(&ida->xa, flags);
 
 	entry = xa_find(&ida->xa, &index, max / IDA_BITMAP_BITS, XA_PRESENT);
-	if (!entry) {
-		ret = -ENOENT;
-		goto err_unlock;
-	}
-
-	if (index > min / IDA_BITMAP_BITS)
-		offset = 0;
-	if (index * IDA_BITMAP_BITS + offset > max) {
-		ret = -ENOENT;
-		goto err_unlock;
-	}
-
-	if (xa_is_value(entry)) {
-		tmp = xa_to_value(entry);
-		addr = &tmp;
-		size = BITS_PER_XA_VALUE;
-	} else {
-		addr = ((struct ida_bitmap *)entry)->bitmap;
-		size = IDA_BITMAP_BITS;
-	}
-
-	bit = find_next_bit(addr, size, offset);
+	while (entry) {
+		if (index > min / IDA_BITMAP_BITS)
+			offset = 0;
+		if (index * IDA_BITMAP_BITS + offset > max)
+			break;
 
-	xa_unlock_irqrestore(&ida->xa, flags);
+		if (xa_is_value(entry)) {
+			tmp = xa_to_value(entry);
+			addr = &tmp;
+			size = BITS_PER_XA_VALUE;
+		} else {
+			addr = ((struct ida_bitmap *)entry)->bitmap;
+			size = IDA_BITMAP_BITS;
+		}
 
-	if (bit == size ||
-	    index * IDA_BITMAP_BITS + bit > max)
-		return -ENOENT;
+		bit = find_next_bit(addr, size, offset);
+		if (bit < size &&
+		    index * IDA_BITMAP_BITS + bit <= max) {
+			xa_unlock_irqrestore(&ida->xa, flags);
+			return index * IDA_BITMAP_BITS + bit;
+		}
 
-	return index * IDA_BITMAP_BITS + bit;
+		entry = xa_find_after(&ida->xa, &index,
+				      max / IDA_BITMAP_BITS, XA_PRESENT);
+	}
 
-err_unlock:
 	xa_unlock_irqrestore(&ida->xa, flags);
-	return ret;
+	return -ENOENT;
 }
 EXPORT_SYMBOL(ida_find_first_range);
 
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 243662af1af73..68374087280e1 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -1224,13 +1224,13 @@ const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags)
 {
 	*new = *old;
 	if (iov_iter_is_bvec(new))
-		return new->bvec = kmemdup(new->bvec,
-				    new->nr_segs * sizeof(struct bio_vec),
+		return new->bvec = kmemdup_array(new->bvec,
+				    new->nr_segs, sizeof(struct bio_vec),
 				    flags);
 	else if (iov_iter_is_kvec(new) || iter_is_iovec(new))
 		/* iovec and kvec have identical layout */
-		return new->__iov = kmemdup(new->__iov,
-				   new->nr_segs * sizeof(struct iovec),
+		return new->__iov = kmemdup_array(new->__iov,
+				   new->nr_segs, sizeof(struct iovec),
 				   flags);
 	return NULL;
 }
@@ -1807,7 +1807,8 @@ static ssize_t iov_iter_extract_user_pages(struct iov_iter *i,
  *  (*) Use with ITER_DISCARD is not supported as that has no content.
  *
  * On success, the function sets *@pages to the new pagelist, if allocated, and
- * sets *offset0 to the offset into the first page.
+ * sets *offset0 to the offset into the first page. On error, new pagelist
+ * is freed if was allocated, and *@pages sets back to its original value.
  *
  * It may also return -ENOMEM and -EFAULT.
  */
@@ -1818,31 +1819,42 @@ ssize_t iov_iter_extract_pages(struct iov_iter *i,
 			       iov_iter_extraction_t extraction_flags,
 			       size_t *offset0)
 {
+	struct page **oldpages = *pages;
+	int ret;
+
 	maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT);
 	if (!maxsize)
 		return 0;
 
 	if (likely(user_backed_iter(i)))
-		return iov_iter_extract_user_pages(i, pages, maxsize,
-						   maxpages, extraction_flags,
-						   offset0);
-	if (iov_iter_is_kvec(i))
-		return iov_iter_extract_kvec_pages(i, pages, maxsize,
-						   maxpages, extraction_flags,
-						   offset0);
-	if (iov_iter_is_bvec(i))
-		return iov_iter_extract_bvec_pages(i, pages, maxsize,
-						   maxpages, extraction_flags,
-						   offset0);
-	if (iov_iter_is_folioq(i))
-		return iov_iter_extract_folioq_pages(i, pages, maxsize,
-						     maxpages, extraction_flags,
-						     offset0);
-	if (iov_iter_is_xarray(i))
-		return iov_iter_extract_xarray_pages(i, pages, maxsize,
-						     maxpages, extraction_flags,
-						     offset0);
-	return -EFAULT;
+		ret = iov_iter_extract_user_pages(i, pages, maxsize,
+						  maxpages, extraction_flags,
+						  offset0);
+	else if (iov_iter_is_kvec(i))
+		ret = iov_iter_extract_kvec_pages(i, pages, maxsize,
+						  maxpages, extraction_flags,
+						  offset0);
+	else if (iov_iter_is_bvec(i))
+		ret = iov_iter_extract_bvec_pages(i, pages, maxsize,
+						  maxpages, extraction_flags,
+						  offset0);
+	else if (iov_iter_is_folioq(i))
+		ret = iov_iter_extract_folioq_pages(i, pages, maxsize,
+						    maxpages, extraction_flags,
+						    offset0);
+	else if (iov_iter_is_xarray(i))
+		ret = iov_iter_extract_xarray_pages(i, pages, maxsize,
+						    maxpages, extraction_flags,
+						    offset0);
+	else
+		ret = -EFAULT;
+
+	if (unlikely(ret) && *pages && *pages != oldpages) {
+		kvfree(*pages);
+		*pages = oldpages;
+	}
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(iov_iter_extract_pages);
 
diff --git a/lib/kstrtox.c b/lib/kstrtox.c
index 97be2a39f5371..edc4eb7c1bca1 100644
--- a/lib/kstrtox.c
+++ b/lib/kstrtox.c
@@ -39,25 +39,30 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base)
 	return s;
 }
 
-/*
- * Convert non-negative integer string representation in explicitly given radix
- * to an integer. A maximum of max_chars characters will be converted.
+/**
+ * _parse_integer_limit - Convert integer string representation to an integer
+ * @s: Integer string representation
+ * @base: Radix
+ * @p: Where to store result
+ * @max_chars: Maximum amount of characters to convert
+ *
+ * Convert non-negative integer string representation in explicitly given
+ * radix to an integer. If overflow occurs, value at @p is set to ULLONG_MAX.
  *
- * Return number of characters consumed maybe or-ed with overflow bit.
- * If overflow occurs, result integer (incorrect) is still returned.
+ * This function is the workhorse of other string conversion functions and it
+ * is discouraged to use it explicitly. Consider kstrto*() family instead.
  *
- * Don't you dare use this function.
+ * Return: Number of characters consumed, maybe ORed with overflow bit
  */
 noinline
 unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p,
 				  size_t max_chars)
 {
+	unsigned int rv, overflow = 0;
 	unsigned long long res;
-	unsigned int rv;
 
 	res = 0;
-	rv = 0;
-	while (max_chars--) {
+	for (rv = 0; rv < max_chars; rv++, s++) {
 		unsigned int c = *s;
 		unsigned int lc = _tolower(c);
 		unsigned int val;
@@ -76,15 +81,17 @@ unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned lon
 		 * it in the max base we support (16)
 		 */
 		if (unlikely(res & (~0ull << 60))) {
-			if (res > div_u64(ULLONG_MAX - val, base))
-				rv |= KSTRTOX_OVERFLOW;
+			if (check_mul_overflow(res, base, &res) ||
+			    check_add_overflow(res, val, &res)) {
+				res = ULLONG_MAX;
+				overflow = KSTRTOX_OVERFLOW;
+			}
+		} else {
+			res = res * base + val;
 		}
-		res = res * base + val;
-		rv++;
-		s++;
 	}
 	*p = res;
-	return rv;
+	return rv | overflow;
 }
 
 noinline
diff --git a/lib/kunit/Makefile b/lib/kunit/Makefile
index 656f1fa35abcc..2e8a6b71a2ab0 100644
--- a/lib/kunit/Makefile
+++ b/lib/kunit/Makefile
@@ -10,7 +10,8 @@ kunit-objs +=				test.o \
 					executor.o \
 					attributes.o \
 					device.o \
-					platform.o
+					platform.o \
+					bug.o
 
 ifeq ($(CONFIG_KUNIT_DEBUGFS),y)
 kunit-objs +=				debugfs.o
@@ -21,6 +22,7 @@ obj-$(if $(CONFIG_KUNIT),y) +=		hooks.o
 
 obj-$(CONFIG_KUNIT_TEST) +=		kunit-test.o
 obj-$(CONFIG_KUNIT_TEST) +=		platform-test.o
+obj-$(CONFIG_KUNIT_TEST) +=		backtrace-suppression-test.o
 
 # string-stream-test compiles built-in only.
 ifeq ($(CONFIG_KUNIT_TEST),y)
diff --git a/lib/kunit/backtrace-suppression-test.c b/lib/kunit/backtrace-suppression-test.c
new file mode 100644
index 0000000000000..7a2a59c6a780d
--- /dev/null
+++ b/lib/kunit/backtrace-suppression-test.c
@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit test for suppressing warning tracebacks.
+ *
+ * Copyright (C) 2024, Guenter Roeck
+ * Author: Guenter Roeck <linux@roeck-us.net>
+ */
+
+#include <kunit/test.h>
+#include <linux/bug.h>
+#include <linux/completion.h>
+#include <linux/kthread.h>
+
+static void backtrace_suppression_test_warn_direct(struct kunit *test)
+{
+	if (!IS_ENABLED(CONFIG_BUG))
+		kunit_skip(test, "requires CONFIG_BUG");
+
+	kunit_warning_suppress(test) {
+		WARN(1, "This backtrace should be suppressed");
+		/*
+		 * Count must be checked inside the scope; the handle
+		 * is not accessible after the block exits.
+		 */
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+	}
+	KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning());
+}
+
+static noinline void trigger_backtrace_warn(void)
+{
+	WARN(1, "This backtrace should be suppressed");
+}
+
+static void backtrace_suppression_test_warn_indirect(struct kunit *test)
+{
+	if (!IS_ENABLED(CONFIG_BUG))
+		kunit_skip(test, "requires CONFIG_BUG");
+
+	kunit_warning_suppress(test) {
+		trigger_backtrace_warn();
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+	}
+}
+
+static void backtrace_suppression_test_warn_multi(struct kunit *test)
+{
+	if (!IS_ENABLED(CONFIG_BUG))
+		kunit_skip(test, "requires CONFIG_BUG");
+
+	kunit_warning_suppress(test) {
+		WARN(1, "This backtrace should be suppressed");
+		trigger_backtrace_warn();
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 2);
+	}
+}
+
+static void backtrace_suppression_test_warn_on_direct(struct kunit *test)
+{
+	if (!IS_ENABLED(CONFIG_BUG))
+		kunit_skip(test, "requires CONFIG_BUG");
+	if (!IS_ENABLED(CONFIG_DEBUG_BUGVERBOSE) && !IS_ENABLED(CONFIG_KALLSYMS))
+		kunit_skip(test, "requires CONFIG_DEBUG_BUGVERBOSE or CONFIG_KALLSYMS");
+
+	kunit_warning_suppress(test) {
+		WARN_ON(1);
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+	}
+}
+
+static noinline void trigger_backtrace_warn_on(void)
+{
+	WARN_ON(1);
+}
+
+static void backtrace_suppression_test_warn_on_indirect(struct kunit *test)
+{
+	if (!IS_ENABLED(CONFIG_BUG))
+		kunit_skip(test, "requires CONFIG_BUG");
+	if (!IS_ENABLED(CONFIG_DEBUG_BUGVERBOSE))
+		kunit_skip(test, "requires CONFIG_DEBUG_BUGVERBOSE");
+
+	kunit_warning_suppress(test) {
+		trigger_backtrace_warn_on();
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+	}
+}
+
+static void backtrace_suppression_test_count(struct kunit *test)
+{
+	if (!IS_ENABLED(CONFIG_BUG))
+		kunit_skip(test, "requires CONFIG_BUG");
+
+	kunit_warning_suppress(test) {
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 0);
+
+		WARN(1, "suppressed");
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 1);
+
+		WARN(1, "suppressed again");
+		KUNIT_EXPECT_SUPPRESSED_WARNING_COUNT(test, 2);
+	}
+}
+
+static void backtrace_suppression_test_active_state(struct kunit *test)
+{
+	KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning());
+
+	kunit_warning_suppress(test) {
+		KUNIT_EXPECT_TRUE(test, kunit_has_active_suppress_warning());
+	}
+
+	KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning());
+
+	kunit_warning_suppress(test) {
+		KUNIT_EXPECT_TRUE(test, kunit_has_active_suppress_warning());
+	}
+
+	KUNIT_EXPECT_FALSE(test, kunit_has_active_suppress_warning());
+}
+
+static void backtrace_suppression_test_multi_scope(struct kunit *test)
+{
+	struct kunit_suppressed_warning *sw1, *sw2;
+
+	if (!IS_ENABLED(CONFIG_BUG))
+		kunit_skip(test, "requires CONFIG_BUG");
+	if (!IS_ENABLED(CONFIG_DEBUG_BUGVERBOSE))
+		kunit_skip(test, "requires CONFIG_DEBUG_BUGVERBOSE");
+
+	sw1 = kunit_start_suppress_warning(test);
+	trigger_backtrace_warn_on();
+	WARN(1, "suppressed by sw1");
+	kunit_end_suppress_warning(test, sw1);
+
+	sw2 = kunit_start_suppress_warning(test);
+	WARN(1, "suppressed by sw2");
+	kunit_end_suppress_warning(test, sw2);
+
+	KUNIT_EXPECT_EQ(test, kunit_suppressed_warning_count(sw1), 2);
+	KUNIT_EXPECT_EQ(test, kunit_suppressed_warning_count(sw2), 1);
+}
+
+struct cross_kthread_data {
+	bool was_active;
+	struct completion done;
+};
+
+static int cross_kthread_fn(void *data)
+{
+	struct cross_kthread_data *d = data;
+
+	d->was_active = kunit_has_active_suppress_warning();
+	complete(&d->done);
+	while (!kthread_should_stop())
+		schedule();
+	return 0;
+}
+
+static void backtrace_suppression_test_cross_kthread(struct kunit *test)
+{
+	struct cross_kthread_data data;
+	struct task_struct *task;
+
+	data.was_active = false;
+	init_completion(&data.done);
+
+	kunit_warning_suppress(test) {
+		task = kthread_run(cross_kthread_fn, &data, "kunit-cross-test");
+		KUNIT_ASSERT_FALSE(test, IS_ERR(task));
+		wait_for_completion(&data.done);
+		kthread_stop(task);
+	}
+
+	KUNIT_EXPECT_FALSE(test, data.was_active);
+}
+
+static struct kunit_case backtrace_suppression_test_cases[] = {
+	KUNIT_CASE(backtrace_suppression_test_warn_direct),
+	KUNIT_CASE(backtrace_suppression_test_warn_indirect),
+	KUNIT_CASE(backtrace_suppression_test_warn_multi),
+	KUNIT_CASE(backtrace_suppression_test_warn_on_direct),
+	KUNIT_CASE(backtrace_suppression_test_warn_on_indirect),
+	KUNIT_CASE(backtrace_suppression_test_count),
+	KUNIT_CASE(backtrace_suppression_test_active_state),
+	KUNIT_CASE(backtrace_suppression_test_multi_scope),
+	KUNIT_CASE(backtrace_suppression_test_cross_kthread),
+	{}
+};
+
+static struct kunit_suite backtrace_suppression_test_suite = {
+	.name = "backtrace-suppression-test",
+	.test_cases = backtrace_suppression_test_cases,
+};
+kunit_test_suites(&backtrace_suppression_test_suite);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("KUnit test to verify warning backtrace suppression");
diff --git a/lib/kunit/bug.c b/lib/kunit/bug.c
new file mode 100644
index 0000000000000..8579235c9ca68
--- /dev/null
+++ b/lib/kunit/bug.c
@@ -0,0 +1,120 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit helpers for backtrace suppression
+ *
+ * Copyright (C) 2025 Alessandro Carminati <acarmina@redhat.com>
+ * Copyright (C) 2024 Guenter Roeck <linux@roeck-us.net>
+ */
+
+#include <kunit/resource.h>
+#include <linux/export.h>
+#include <linux/rculist.h>
+#include <linux/sched.h>
+#include <linux/sched/task.h>
+#include <linux/spinlock.h>
+
+#include "hooks-impl.h"
+
+struct kunit_suppressed_warning {
+	struct list_head node;
+	struct task_struct *task;
+	struct kunit *test;
+	atomic_t counter;
+};
+
+static LIST_HEAD(suppressed_warnings);
+static DEFINE_SPINLOCK(suppressed_warnings_lock);
+
+static void kunit_suppress_warning_remove(struct kunit_suppressed_warning *w)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&suppressed_warnings_lock, flags);
+	list_del_rcu(&w->node);
+	spin_unlock_irqrestore(&suppressed_warnings_lock, flags);
+	put_task_struct(w->task);
+}
+
+KUNIT_DEFINE_ACTION_WRAPPER(kunit_suppress_warning_cleanup,
+			    kunit_suppress_warning_remove,
+			    struct kunit_suppressed_warning *);
+
+bool kunit_has_active_suppress_warning(void)
+{
+	return __kunit_is_suppressed_warning_impl(false);
+}
+EXPORT_SYMBOL_GPL(kunit_has_active_suppress_warning);
+
+struct kunit_suppressed_warning *
+kunit_start_suppress_warning(struct kunit *test)
+{
+	struct kunit_suppressed_warning *w;
+	unsigned long flags;
+	int ret;
+
+	if (kunit_has_active_suppress_warning()) {
+		KUNIT_FAIL(test, "Another suppression block is already active");
+		return NULL;
+	}
+
+	w = kunit_kzalloc(test, sizeof(*w), GFP_KERNEL);
+	if (!w) {
+		KUNIT_FAIL(test, "Failed to allocate suppression handle.");
+		return NULL;
+	}
+
+	w->task = get_task_struct(current);
+	w->test = test;
+
+	spin_lock_irqsave(&suppressed_warnings_lock, flags);
+	list_add_rcu(&w->node, &suppressed_warnings);
+	spin_unlock_irqrestore(&suppressed_warnings_lock, flags);
+
+	ret = kunit_add_action_or_reset(test,
+					kunit_suppress_warning_cleanup, w);
+	if (ret) {
+		KUNIT_FAIL(test, "Failed to add suppression cleanup action.");
+		return NULL;
+	}
+
+	return w;
+}
+EXPORT_SYMBOL_GPL(kunit_start_suppress_warning);
+
+void kunit_end_suppress_warning(struct kunit *test,
+				struct kunit_suppressed_warning *w)
+{
+	if (!w)
+		return;
+	kunit_release_action(test, kunit_suppress_warning_cleanup, w);
+}
+EXPORT_SYMBOL_GPL(kunit_end_suppress_warning);
+
+void __kunit_suppress_auto_cleanup(struct kunit_suppressed_warning **wp)
+{
+	if (*wp)
+		kunit_end_suppress_warning((*wp)->test, *wp);
+}
+EXPORT_SYMBOL_GPL(__kunit_suppress_auto_cleanup);
+
+int kunit_suppressed_warning_count(struct kunit_suppressed_warning *w)
+{
+	return w ? atomic_read(&w->counter) : 0;
+}
+EXPORT_SYMBOL_GPL(kunit_suppressed_warning_count);
+
+bool __kunit_is_suppressed_warning_impl(bool count)
+{
+	struct kunit_suppressed_warning *w;
+
+	guard(rcu)();
+	list_for_each_entry_rcu(w, &suppressed_warnings, node) {
+		if (w->task == current) {
+			if (count)
+				atomic_inc(&w->counter);
+			return true;
+		}
+	}
+
+	return false;
+}
diff --git a/lib/kunit/hooks-impl.h b/lib/kunit/hooks-impl.h
index 4e71b2d0143ba..d8720f2616925 100644
--- a/lib/kunit/hooks-impl.h
+++ b/lib/kunit/hooks-impl.h
@@ -19,6 +19,7 @@ void __printf(3, 4) __kunit_fail_current_test_impl(const char *file,
 						   int line,
 						   const char *fmt, ...);
 void *__kunit_get_static_stub_address_impl(struct kunit *test, void *real_fn_addr);
+bool __kunit_is_suppressed_warning_impl(bool count);
 
 /* Code to set all of the function pointers. */
 static inline void kunit_install_hooks(void)
@@ -26,6 +27,7 @@ static inline void kunit_install_hooks(void)
 	/* Install the KUnit hook functions. */
 	kunit_hooks.fail_current_test = __kunit_fail_current_test_impl;
 	kunit_hooks.get_static_stub_address = __kunit_get_static_stub_address_impl;
+	kunit_hooks.is_suppressed_warning = __kunit_is_suppressed_warning_impl;
 }
 
 #endif /* _KUNIT_HOOKS_IMPL_H */
diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index 60ae5e6fc1ee6..e52876435b77f 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -5727,13 +5727,16 @@ int mtree_store(struct maple_tree *mt, unsigned long index, void *entry,
 EXPORT_SYMBOL(mtree_store);
 
 /**
- * mtree_insert_range() - Insert an entry at a given range if there is no value.
+ * mtree_insert_range() - Insert an entry from [first, last] at a given range
+ *                        if there is no value.
  * @mt: The maple tree
  * @first: The start of the range
- * @last: The end of the range
+ * @last: The end of the range (inclusive)
  * @entry: The entry to store
  * @gfp: The GFP_FLAGS to use for allocations.
  *
+ * Note that @last is inclusive. That is, @last = @first + length - 1;
+ *
  * Return: 0 on success, -EEXISTS if the range is occupied, -EINVAL on invalid
  * request, -ENOMEM if memory could not be allocated.
  */
diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig
index 5ab2b0a7be4c6..978cd6ba08ac6 100644
--- a/lib/raid/Kconfig
+++ b/lib/raid/Kconfig
@@ -28,3 +28,36 @@ config XOR_KUNIT_TEST
 
 	  This is intended to help people writing architecture-specific
 	  optimized versions.  If unsure, say N.
+
+config RAID6_PQ
+	tristate
+
+# selected by architectures that provide an optimized PQ implementation
+config RAID6_PQ_ARCH
+	depends on RAID6_PQ
+	default y if KERNEL_MODE_NEON		# arm32/arm64
+	default y if LOONGARCH
+	default y if ALTIVEC			# powerpc
+	default y if RISCV_ISA_V
+	default y if S390
+	default y if X86
+	bool
+
+config RAID6_PQ_KUNIT_TEST
+	tristate "KUnit tests for RAID6 PQ functions" if !KUNIT_ALL_TESTS
+	depends on KUNIT
+	depends on RAID6_PQ
+	default KUNIT_ALL_TESTS
+	help
+	  Unit tests for the RAID6 PQ library functions.
+
+	  This is intended to help people writing architecture-specific
+	  optimized versions.  If unsure, say N.
+
+config RAID6_PQ_BENCHMARK
+	bool "Automatically choose fastest RAID6 PQ functions"
+	depends on RAID6_PQ
+	default y
+	help
+	  Benchmark all available RAID6 PQ functions on init and choose the
+	  fastest one.
diff --git a/lib/raid/Makefile b/lib/raid/Makefile
index 3540fe846dc42..6fc5eeb53df0f 100644
--- a/lib/raid/Makefile
+++ b/lib/raid/Makefile
@@ -1,3 +1,3 @@
 # SPDX-License-Identifier: GPL-2.0
 
-obj-y				+= xor/
+obj-y				+= xor/ raid6/
diff --git a/lib/raid/raid6/.gitignore b/lib/raid/raid6/.gitignore
new file mode 100644
index 0000000000000..6be57745afd12
--- /dev/null
+++ b/lib/raid/raid6/.gitignore
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+mktables
+altivec*.c
+int*.c
+tables.c
+neon?.c
+s390vx?.c
+vpermxor*.c
diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile
new file mode 100644
index 0000000000000..1d79b76b320ea
--- /dev/null
+++ b/lib/raid/raid6/Makefile
@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: GPL-2.0
+
+ccflags-y			+= -I $(src)
+
+ccflags-y			+= -I $(src)
+
+ifeq ($(CONFIG_RAID6_PQ_ARCH),y)
+CFLAGS_algos.o			+= -I$(src)/$(SRCARCH)
+endif
+
+obj-$(CONFIG_RAID6_PQ)		+= raid6_pq.o tests/
+
+raid6_pq-y			+= algos.o tables.o
+
+# generic integer generation and recovery implementation
+raid6_pq-y			+= int1.o int2.o int4.o int8.o
+raid6_pq-y			+= recov.o
+
+# architecture-specific generation and recovery implementations:
+raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += arm/neon.o \
+				   arm/neon1.o \
+				   arm/neon2.o \
+				   arm/neon4.o \
+				   arm/neon8.o \
+				   arm/recov_neon.o \
+				   arm/recov_neon_inner.o
+raid6_pq-$(CONFIG_LOONGARCH)	+= loongarch/loongarch_simd.o \
+				   loongarch/recov_loongarch_simd.o
+raid6_pq-$(CONFIG_ALTIVEC)	+= powerpc/altivec1.o \
+				   powerpc/altivec2.o \
+				   powerpc/altivec4.o \
+				   powerpc/altivec8.o \
+				   powerpc/vpermxor1.o \
+				   powerpc/vpermxor2.o \
+				   powerpc/vpermxor4.o \
+				   powerpc/vpermxor8.o
+raid6_pq-$(CONFIG_RISCV_ISA_V)	+= riscv/rvv.o \
+				   riscv/recov_rvv.o
+raid6_pq-$(CONFIG_S390)		+= s390/s390vx8.o \
+				   s390/recov_s390xc.o
+ifeq ($(CONFIG_X86),y)
+raid6_pq-$(CONFIG_X86_32)	+= x86/mmx.o \
+				   x86/sse1.o
+endif
+raid6_pq-$(CONFIG_X86)		+= x86/sse2.o \
+				   x86/avx2.o \
+				   x86/avx512.o \
+				   x86/recov_ssse3.o \
+				   x86/recov_avx2.o \
+				   x86/recov_avx512.o
+
+hostprogs			+= mktables
+
+CFLAGS_arm/neon1.o += $(CC_FLAGS_FPU)
+CFLAGS_arm/neon2.o += $(CC_FLAGS_FPU)
+CFLAGS_arm/neon4.o += $(CC_FLAGS_FPU)
+CFLAGS_arm/neon8.o += $(CC_FLAGS_FPU)
+CFLAGS_arm/recov_neon_inner.o += $(CC_FLAGS_FPU)
+CFLAGS_REMOVE_arm/neon1.o += $(CC_FLAGS_NO_FPU)
+CFLAGS_REMOVE_arm/neon2.o += $(CC_FLAGS_NO_FPU)
+CFLAGS_REMOVE_arm/neon4.o += $(CC_FLAGS_NO_FPU)
+CFLAGS_REMOVE_arm/neon8.o += $(CC_FLAGS_NO_FPU)
+CFLAGS_REMOVE_arm/recov_neon_inner.o += $(CC_FLAGS_NO_FPU)
+
+ifeq ($(CONFIG_ALTIVEC),y)
+altivec_flags := -maltivec $(call cc-option,-mabi=altivec)
+# Enable <altivec.h>
+altivec_flags += -isystem $(shell $(CC) -print-file-name=include)
+
+CFLAGS_powerpc/altivec1.o += $(altivec_flags)
+CFLAGS_powerpc/altivec2.o += $(altivec_flags)
+CFLAGS_powerpc/altivec4.o += $(altivec_flags)
+CFLAGS_powerpc/altivec8.o += $(altivec_flags)
+CFLAGS_powerpc/vpermxor1.o += $(altivec_flags)
+CFLAGS_powerpc/vpermxor2.o += $(altivec_flags)
+CFLAGS_powerpc/vpermxor4.o += $(altivec_flags)
+CFLAGS_powerpc/vpermxor8.o += $(altivec_flags)
+
+ifdef CONFIG_CC_IS_CLANG
+# clang ppc port does not yet support -maltivec when -msoft-float is
+# enabled. A future release of clang will resolve this
+# https://llvm.org/pr31177
+CFLAGS_REMOVE_powerpc/altivec1.o  += -msoft-float
+CFLAGS_REMOVE_powerpc/altivec2.o  += -msoft-float
+CFLAGS_REMOVE_powerpc/altivec4.o  += -msoft-float
+CFLAGS_REMOVE_powerpc/altivec8.o  += -msoft-float
+CFLAGS_REMOVE_powerpc/vpermxor1.o += -msoft-float
+CFLAGS_REMOVE_powerpc/vpermxor2.o += -msoft-float
+CFLAGS_REMOVE_powerpc/vpermxor4.o += -msoft-float
+CFLAGS_REMOVE_powerpc/vpermxor8.o += -msoft-float
+endif # CONFIG_CC_IS_CLANG
+endif # CONFIG_ALTIVEC
+
+quiet_cmd_mktable = TABLE   $@
+      cmd_mktable = $(obj)/mktables > $@
+
+targets += tables.c
+$(obj)/tables.c: $(obj)/mktables FORCE
+	$(call if_changed,mktable)
+
+quiet_cmd_unroll = UNROLL  $@
+      cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@
+
+targets += int1.c int2.c int4.c int8.c
+$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE
+	$(call if_changed,unroll)
+
+targets += arm/neon1.c arm/neon2.c arm/neon4.c arm/neon8.c
+$(obj)/arm/neon%.c: $(src)/arm/neon.uc $(src)/unroll.awk FORCE
+	$(call if_changed,unroll)
+
+targets += powerpc/altivec1.c \
+	   powerpc/altivec2.c \
+	   powerpc/altivec4.c \
+	   powerpc/altivec8.c
+$(obj)/powerpc/altivec%.c: $(src)/powerpc/altivec.uc $(src)/unroll.awk FORCE
+	$(call if_changed,unroll)
+
+targets += powerpc/vpermxor1.c \
+	   powerpc/vpermxor2.c \
+	   powerpc/vpermxor4.c \
+	   powerpc/vpermxor8.c
+$(obj)/powerpc/vpermxor%.c: $(src)/powerpc/vpermxor.uc $(src)/unroll.awk FORCE
+	$(call if_changed,unroll)
+
+targets += s390/s390vx8.c
+$(obj)/s390/s390vx%.c: $(src)/s390/s390vx.uc $(src)/unroll.awk FORCE
+	$(call if_changed,unroll)
diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c
new file mode 100644
index 0000000000000..a600d58536729
--- /dev/null
+++ b/lib/raid/raid6/algos.c
@@ -0,0 +1,377 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ * Algorithm list and algorithm selection for RAID-6
+ */
+
+#include <linux/module.h>
+#include <linux/gfp.h>
+#include <linux/raid/pq.h>
+#include <linux/static_call.h>
+#include <kunit/visibility.h>
+#include "algos.h"
+
+#define RAID6_MAX_ALGOS		16
+static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS];
+static unsigned int raid6_nr_algos;
+static const struct raid6_recov_calls *raid6_recov_algo;
+
+/* Selected algorithm */
+DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome);
+DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome);
+DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2);
+DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap);
+
+/**
+ * raid6_gen_syndrome - generate RAID6 P/Q parity
+ * @disks:	number of "disks" to operate on including parity
+ * @bytes:	length in bytes of each vector
+ * @ptrs:	@disks size array of memory pointers
+ *
+ * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and
+ * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to
+ * @ptrs[@disks - 3].
+ *
+ * @disks must be at least 4, and the memory pointed to by each member of @ptrs
+ * must be at least 64-byte aligned.  @bytes must be non-zero and a multiple of
+ * 512.
+ *
+ * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying
+ * algorithm.
+ */
+void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());
+	WARN_ON_ONCE(bytes & 511);
+	WARN_ON_ONCE(disks < RAID6_MIN_DISKS);
+
+	static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs);
+}
+EXPORT_SYMBOL_GPL(raid6_gen_syndrome);
+
+/**
+ * raid6_xor_syndrome - update RAID6 P/Q parity
+ * @disks:	number of "disks" to operate on including parity
+ * @start:	first index into @disk to update
+ * @stop:	last index into @disk to update
+ * @bytes:	length in bytes of each vector
+ * @ptrs:	@disks size array of memory pointers
+ *
+ * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and
+ * @ptrs[@disks - 1] respectively for the memory pointed to by
+ * @ptrs[@start..@stop].
+ *
+ * This is used to update parity in place using the following sequence:
+ *
+ * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data.
+ * 2) update the the data in @ptrs[@start..@stop].
+ * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data.
+ *
+ * Data between @start and @stop that is not changed should be filled
+ * with a pointer to the kernel zero page.
+ *
+ * @disks must be at least 4, and the memory pointed to by each member of @ptrs
+ * must be at least 64-byte aligned.  @bytes must be non-zero and a multiple of
+ * 512.  @stop must be larger or equal to @start.
+ */
+void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes,
+		void **ptrs)
+{
+	WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());
+	WARN_ON_ONCE(bytes & 511);
+	WARN_ON_ONCE(disks < RAID6_MIN_DISKS);
+	WARN_ON_ONCE(stop < start);
+
+	static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs);
+}
+EXPORT_SYMBOL_GPL(raid6_xor_syndrome);
+
+/*
+ * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used
+ *
+ * Returns %true if raid6_can_xor_syndrome() can be used, else %false.
+ */
+bool raid6_can_xor_syndrome(void)
+{
+	return !!static_call_query(raid6_xor_syndrome_impl);
+}
+EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome);
+
+/**
+ * raid6_recov_2data - recover two missing data disks
+ * @disks:	number of "disks" to operate on including parity
+ * @bytes:	length in bytes of each vector
+ * @faila:	first failed data disk index
+ * @failb:	second failed data disk index
+ * @ptrs:	@disks size array of memory pointers
+ *
+ * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the
+ * data in the remaining disks and the two parities pointed to by the other
+ * indices between 0 and @disks - 1 in @ptrs.  @disks includes the data disks
+ * and the two parities.  @faila must be smaller than @failb.
+ *
+ * Memory pointed to by each pointer in @ptrs must be page aligned and is
+ * limited to %PAGE_SIZE.
+ */
+void raid6_recov_2data(int disks, size_t bytes, int faila, int failb,
+		void **ptrs)
+{
+	WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());
+	WARN_ON_ONCE(bytes & 511);
+	WARN_ON_ONCE(bytes > PAGE_SIZE);
+	WARN_ON_ONCE(failb <= faila);
+
+	static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs);
+}
+EXPORT_SYMBOL_GPL(raid6_recov_2data);
+
+/**
+ * raid6_recov_datap - recover a missing data disk and missing P-parity
+ * @disks:	number of "disks" to operate on including parity
+ * @bytes:	length in bytes of each vector
+ * @faila:	failed data disk index
+ * @ptrs:	@disks size array of memory pointers
+ *
+ * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in
+ * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity
+ * pointed to by the other indices between 0 and @disks - 1 in @ptrs.  @disks
+ * includes the data disks and the two parities.
+ *
+ * Memory pointed to by each pointer in @ptrs must be page aligned and is
+ * limited to %PAGE_SIZE.
+ */
+void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs)
+{
+	WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count());
+	WARN_ON_ONCE(bytes & 511);
+	WARN_ON_ONCE(bytes > PAGE_SIZE);
+
+	static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs);
+}
+EXPORT_SYMBOL_GPL(raid6_recov_datap);
+
+#define RAID6_TIME_JIFFIES_LG2	4
+#define RAID6_TEST_DISKS	8
+#define RAID6_TEST_DISKS_ORDER	3
+
+static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS],
+		const int disks)
+{
+	/* work on the second half of the disks */
+	int start = (disks >> 1) - 1, stop = disks - 3;
+	const struct raid6_calls *best = NULL;
+	unsigned long bestgenperf = 0;
+	unsigned int i;
+
+	for (i = 0; i < raid6_nr_algos; i++) {
+		const struct raid6_calls *algo = raid6_algos[i];
+		unsigned long perf = 0, j0, j1;
+
+		preempt_disable();
+		j0 = jiffies;
+		while ((j1 = jiffies) == j0)
+			cpu_relax();
+		while (time_before(jiffies,
+				    j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
+			algo->gen_syndrome(disks, PAGE_SIZE, *dptrs);
+			perf++;
+		}
+		preempt_enable();
+
+		if (perf > bestgenperf) {
+			bestgenperf = perf;
+			best = algo;
+		}
+		pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name,
+			(perf * HZ * (disks-2)) >>
+			(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
+	}
+
+	if (!best) {
+		pr_err("raid6: Yikes! No algorithm found!\n");
+		return -EINVAL;
+	}
+
+	static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome);
+	static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome);
+
+	pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
+		best->name,
+		(bestgenperf * HZ * (disks - 2)) >>
+		(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
+
+	if (best->xor_syndrome) {
+		unsigned long perf = 0, j0, j1;
+
+		preempt_disable();
+		j0 = jiffies;
+		while ((j1 = jiffies) == j0)
+			cpu_relax();
+		while (time_before(jiffies,
+				   j1 + (1 << RAID6_TIME_JIFFIES_LG2))) {
+			best->xor_syndrome(disks, start, stop,
+					   PAGE_SIZE, *dptrs);
+			perf++;
+		}
+		preempt_enable();
+
+		pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
+			(perf * HZ * (disks - 2)) >>
+			(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
+	}
+
+	return 0;
+}
+
+
+/* Try to pick the best algorithm */
+/* This code uses the gfmul table as convenient data set to abuse */
+
+static int __init raid6_select_algo(void)
+{
+	const int disks = RAID6_TEST_DISKS;
+	char *disk_ptr, *p;
+	void *dptrs[RAID6_TEST_DISKS];
+	int i, cycle;
+	int error;
+
+	if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) {
+		pr_info("raid6: skipped pq benchmark and selected %s\n",
+			raid6_algos[raid6_nr_algos - 1]->name);
+		static_call_update(raid6_gen_syndrome_impl,
+				raid6_algos[raid6_nr_algos - 1]->gen_syndrome);
+		static_call_update(raid6_xor_syndrome_impl,
+				raid6_algos[raid6_nr_algos - 1]->xor_syndrome);
+		return 0;
+	}
+
+	/* prepare the buffer and fill it circularly with gfmul table */
+	disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
+	if (!disk_ptr) {
+		pr_err("raid6: Yikes!  No memory available.\n");
+		return -ENOMEM;
+	}
+
+	p = disk_ptr;
+	for (i = 0; i < disks; i++)
+		dptrs[i] = p + PAGE_SIZE * i;
+
+	cycle = ((disks - 2) * PAGE_SIZE) / 65536;
+	for (i = 0; i < cycle; i++) {
+		memcpy(p, raid6_gfmul, 65536);
+		p += 65536;
+	}
+
+	if ((disks - 2) * PAGE_SIZE % 65536)
+		memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
+
+	/* select raid gen_syndrome function */
+	error = raid6_choose_gen(&dptrs, disks);
+
+	free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
+
+	return error;
+}
+
+/*
+ * Register a RAID6 P/Q generation algorithm.  The most optimized/unrolled
+ * implementation should be registered last so it will be selected when the
+ * boot-time benchmark is disabled.
+ */
+void __init raid6_algo_add(const struct raid6_calls *algo)
+{
+	if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS))
+		return;
+	raid6_algos[raid6_nr_algos++] = algo;
+}
+
+void __init raid6_algo_add_default(void)
+{
+	raid6_algo_add(&raid6_intx1);
+	raid6_algo_add(&raid6_intx2);
+	raid6_algo_add(&raid6_intx4);
+	raid6_algo_add(&raid6_intx8);
+}
+
+void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo)
+{
+	if (WARN_ON_ONCE(raid6_recov_algo))
+		return;
+	raid6_recov_algo = algo;
+}
+
+#ifdef CONFIG_RAID6_PQ_ARCH
+#include "pq_arch.h"
+#else
+static inline void arch_raid6_init(void)
+{
+	raid6_algo_add_default();
+}
+#endif /* CONFIG_RAID6_PQ_ARCH */
+
+static int __init raid6_init(void)
+{
+	/*
+	 * Architectures providing arch_raid6_init must add all PQ generation
+	 * algorithms they want to consider in arch_raid6_init(), including
+	 * the generic ones using raid6_algo_add_default() if wanted.
+	 */
+	arch_raid6_init();
+
+	/*
+	 * Architectures don't have to set a recovery algorithm, we'll just pick
+	 * the generic integer one if none was set.
+	 */
+	if (!raid6_recov_algo)
+		raid6_recov_algo = &raid6_recov_intx1;
+	static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2);
+	static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap);
+	pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name);
+
+	return raid6_select_algo();
+}
+
+static void __exit raid6_exit(void)
+{
+}
+
+subsys_initcall(raid6_init);
+module_exit(raid6_exit);
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("RAID6 Q-syndrome calculations");
+
+#if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST)
+const struct raid6_calls *raid6_algo_find(unsigned int idx)
+{
+	if (idx >= raid6_nr_algos) {
+		/*
+		 * Always include the simplest generic integer implementation in
+		 * the unit tests as a baseline.
+		 */
+		if (idx == raid6_nr_algos &&
+		    raid6_algos[0] != &raid6_intx1)
+			return &raid6_intx1;
+		return NULL;
+	}
+	return raid6_algos[idx];
+}
+EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find);
+
+const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx)
+{
+	switch (idx) {
+	case 0:
+		/* always test the generic integer implementation */
+		return &raid6_recov_intx1;
+	case 1:
+		/* test the optimized implementation if there is one */
+		if (raid6_recov_algo != &raid6_recov_intx1)
+			return raid6_recov_algo;
+		return NULL;
+	default:
+		return NULL;
+	}
+}
+EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find);
+#endif /* CONFIG_RAID6_PQ_KUNIT_TEST */
diff --git a/lib/raid/raid6/algos.h b/lib/raid/raid6/algos.h
new file mode 100644
index 0000000000000..43f636be183f6
--- /dev/null
+++ b/lib/raid/raid6/algos.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright 2003 H. Peter Anvin - All Rights Reserved
+ */
+#ifndef _PQ_IMPL_H
+#define _PQ_IMPL_H
+
+#include <linux/init.h>
+#include <linux/raid/pq_tables.h>
+
+/* Routine choices */
+struct raid6_calls {
+	const char *name;
+	void (*gen_syndrome)(int disks, size_t bytes, void **ptrs);
+	void (*xor_syndrome)(int disks, int start, int stop, size_t bytes,
+			void **ptrs);
+};
+
+struct raid6_recov_calls {
+	const char *name;
+	void (*data2)(int disks, size_t bytes, int faila, int failb,
+			void **ptrs);
+	void (*datap)(int disks, size_t bytes, int faila, void **ptrs);
+};
+
+void __init raid6_algo_add(const struct raid6_calls *algo);
+void __init raid6_algo_add_default(void);
+void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo);
+
+/* for the kunit test */
+const struct raid6_calls *raid6_algo_find(unsigned int idx);
+const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx);
+
+/* generic implementations */
+extern const struct raid6_calls raid6_intx1;
+extern const struct raid6_calls raid6_intx2;
+extern const struct raid6_calls raid6_intx4;
+extern const struct raid6_calls raid6_intx8;
+extern const struct raid6_recov_calls raid6_recov_intx1;
+
+#endif /* _PQ_IMPL_H */
diff --git a/lib/raid/raid6/arm/neon.c b/lib/raid/raid6/arm/neon.c
new file mode 100644
index 0000000000000..af90869aaffc1
--- /dev/null
+++ b/lib/raid/raid6/arm/neon.c
@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * RAID6 syndrome calculation using ARM NEON intrinsics
+ *
+ * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
+ */
+
+#include <asm/simd.h>
+#include "algos.h"
+
+/*
+ * There are 2 reasons these wrappers are kept in a separate compilation unit
+ * from the actual implementations in neonN.c (generated from neon.uc by
+ * unroll.awk):
+ * - the actual implementations use NEON intrinsics, and the GCC support header
+ *   (arm_neon.h) is not fully compatible (type wise) with the kernel;
+ * - the neonN.c files are compiled with -mfpu=neon and optimization enabled,
+ *   and we have to make sure that we never use *any* NEON/VFP instructions
+ *   outside a kernel_neon_begin()/kernel_neon_end() pair.
+ */
+
+#define RAID6_NEON_WRAPPER(_n)						\
+	static void raid6_neon ## _n ## _gen_syndrome(int disks,	\
+					size_t bytes, void **ptrs)	\
+	{								\
+		void raid6_neon ## _n  ## _gen_syndrome_real(int,	\
+						unsigned long, void**);	\
+		scoped_ksimd()						\
+			raid6_neon ## _n ## _gen_syndrome_real(disks,	\
+					(unsigned long)bytes, ptrs);	\
+	}								\
+	static void raid6_neon ## _n ## _xor_syndrome(int disks,	\
+					int start, int stop, 		\
+					size_t bytes, void **ptrs)	\
+	{								\
+		void raid6_neon ## _n  ## _xor_syndrome_real(int,	\
+				int, int, unsigned long, void**);	\
+		scoped_ksimd()						\
+			raid6_neon ## _n ## _xor_syndrome_real(disks,	\
+				start, stop, (unsigned long)bytes, ptrs);\
+	}								\
+	struct raid6_calls const raid6_neonx ## _n = {			\
+		.gen_syndrome	= raid6_neon ## _n ## _gen_syndrome,	\
+		.xor_syndrome	= raid6_neon ## _n ## _xor_syndrome,	\
+		.name		= "neonx" #_n,				\
+	}
+
+RAID6_NEON_WRAPPER(1);
+RAID6_NEON_WRAPPER(2);
+RAID6_NEON_WRAPPER(4);
+RAID6_NEON_WRAPPER(8);
diff --git a/lib/raid/raid6/arm/neon.h b/lib/raid/raid6/arm/neon.h
new file mode 100644
index 0000000000000..2ca41ee9b4996
--- /dev/null
+++ b/lib/raid/raid6/arm/neon.h
@@ -0,0 +1,22 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+void raid6_neon1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs);
+void raid6_neon1_xor_syndrome_real(int disks, int start, int stop,
+				    unsigned long bytes, void **ptrs);
+void raid6_neon2_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs);
+void raid6_neon2_xor_syndrome_real(int disks, int start, int stop,
+				    unsigned long bytes, void **ptrs);
+void raid6_neon4_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs);
+void raid6_neon4_xor_syndrome_real(int disks, int start, int stop,
+				    unsigned long bytes, void **ptrs);
+void raid6_neon8_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs);
+void raid6_neon8_xor_syndrome_real(int disks, int start, int stop,
+				    unsigned long bytes, void **ptrs);
+void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
+			      uint8_t *dq, const uint8_t *pbmul,
+			      const uint8_t *qmul);
+
+void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
+			      const uint8_t *qmul);
+
+
diff --git a/lib/raid/raid6/arm/neon.uc b/lib/raid/raid6/arm/neon.uc
new file mode 100644
index 0000000000000..14a9fc2c60fa2
--- /dev/null
+++ b/lib/raid/raid6/arm/neon.uc
@@ -0,0 +1,153 @@
+/* -----------------------------------------------------------------------
+ *
+ *   neon.uc - RAID-6 syndrome calculation using ARM NEON instructions
+ *
+ *   Copyright (C) 2012 Rob Herring
+ *   Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ *
+ *   Based on altivec.uc:
+ *     Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Boston MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * neon$#.c
+ *
+ * $#-way unrolled NEON intrinsics math RAID-6 instruction set
+ *
+ * This file is postprocessed using unroll.awk
+ */
+
+#include <arm_neon.h>
+#include "arm/neon.h"
+
+typedef uint8x16_t unative_t;
+
+#define NSIZE	sizeof(unative_t)
+
+/*
+ * The SHLBYTE() operation shifts each byte left by 1, *not*
+ * rolling over into the next byte
+ */
+static inline unative_t SHLBYTE(unative_t v)
+{
+	return vshlq_n_u8(v, 1);
+}
+
+/*
+ * The MASK() operation returns 0xFF in any byte for which the high
+ * bit is 1, 0x00 for any byte for which the high bit is 0.
+ */
+static inline unative_t MASK(unative_t v)
+{
+	return (unative_t)vshrq_n_s8((int8x16_t)v, 7);
+}
+
+static inline unative_t PMUL(unative_t v, unative_t u)
+{
+	return (unative_t)vmulq_p8((poly8x16_t)v, (poly8x16_t)u);
+}
+
+void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+	uint8_t **dptr = (uint8_t **)ptrs;
+	uint8_t *p, *q;
+	int d, z, z0;
+
+	register unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
+	const unative_t x1d = vdupq_n_u8(0x1d);
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
+		wq$$ = wp$$ = vld1q_u8(&dptr[z0][d+$$*NSIZE]);
+		for ( z = z0-1 ; z >= 0 ; z-- ) {
+			wd$$ = vld1q_u8(&dptr[z][d+$$*NSIZE]);
+			wp$$ = veorq_u8(wp$$, wd$$);
+			w2$$ = MASK(wq$$);
+			w1$$ = SHLBYTE(wq$$);
+
+			w2$$ = vandq_u8(w2$$, x1d);
+			w1$$ = veorq_u8(w1$$, w2$$);
+			wq$$ = veorq_u8(w1$$, wd$$);
+		}
+		vst1q_u8(&p[d+NSIZE*$$], wp$$);
+		vst1q_u8(&q[d+NSIZE*$$], wq$$);
+	}
+}
+
+void raid6_neon$#_xor_syndrome_real(int disks, int start, int stop,
+				    unsigned long bytes, void **ptrs)
+{
+	uint8_t **dptr = (uint8_t **)ptrs;
+	uint8_t *p, *q;
+	int d, z, z0;
+
+	register unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
+	const unative_t x1d = vdupq_n_u8(0x1d);
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
+		wq$$ = vld1q_u8(&dptr[z0][d+$$*NSIZE]);
+		wp$$ = veorq_u8(vld1q_u8(&p[d+$$*NSIZE]), wq$$);
+
+		/* P/Q data pages */
+		for ( z = z0-1 ; z >= start ; z-- ) {
+			wd$$ = vld1q_u8(&dptr[z][d+$$*NSIZE]);
+			wp$$ = veorq_u8(wp$$, wd$$);
+			w2$$ = MASK(wq$$);
+			w1$$ = SHLBYTE(wq$$);
+
+			w2$$ = vandq_u8(w2$$, x1d);
+			w1$$ = veorq_u8(w1$$, w2$$);
+			wq$$ = veorq_u8(w1$$, wd$$);
+		}
+		/* P/Q left side optimization */
+		for ( z = start-1 ; z >= 3 ; z -= 4 ) {
+			w2$$ = vshrq_n_u8(wq$$, 4);
+			w1$$ = vshlq_n_u8(wq$$, 4);
+
+			w2$$ = PMUL(w2$$, x1d);
+			wq$$ = veorq_u8(w1$$, w2$$);
+		}
+
+		switch (z) {
+		case 2:
+			w2$$ = vshrq_n_u8(wq$$, 5);
+			w1$$ = vshlq_n_u8(wq$$, 3);
+
+			w2$$ = PMUL(w2$$, x1d);
+			wq$$ = veorq_u8(w1$$, w2$$);
+			break;
+		case 1:
+			w2$$ = vshrq_n_u8(wq$$, 6);
+			w1$$ = vshlq_n_u8(wq$$, 2);
+
+			w2$$ = PMUL(w2$$, x1d);
+			wq$$ = veorq_u8(w1$$, w2$$);
+			break;
+		case 0:
+			w2$$ = MASK(wq$$);
+			w1$$ = SHLBYTE(wq$$);
+
+			w2$$ = vandq_u8(w2$$, x1d);
+			wq$$ = veorq_u8(w1$$, w2$$);
+		}
+		w1$$ = vld1q_u8(&q[d+NSIZE*$$]);
+		wq$$ = veorq_u8(wq$$, w1$$);
+
+		vst1q_u8(&p[d+NSIZE*$$], wp$$);
+		vst1q_u8(&q[d+NSIZE*$$], wq$$);
+	}
+}
diff --git a/lib/raid/raid6/arm/pq_arch.h b/lib/raid/raid6/arm/pq_arch.h
new file mode 100644
index 0000000000000..3f876ea6749cc
--- /dev/null
+++ b/lib/raid/raid6/arm/pq_arch.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/neon.h>
+
+extern const struct raid6_calls raid6_neonx1;
+extern const struct raid6_calls raid6_neonx2;
+extern const struct raid6_calls raid6_neonx4;
+extern const struct raid6_calls raid6_neonx8;
+extern const struct raid6_recov_calls raid6_recov_neon;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+	raid6_algo_add_default();
+	if (cpu_has_neon()) {
+		raid6_algo_add(&raid6_neonx1);
+		raid6_algo_add(&raid6_neonx2);
+		raid6_algo_add(&raid6_neonx4);
+		raid6_algo_add(&raid6_neonx8);
+		raid6_recov_algo_add(&raid6_recov_neon);
+	}
+}
diff --git a/lib/raid/raid6/arm/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c
new file mode 100644
index 0000000000000..1524050d09b7f
--- /dev/null
+++ b/lib/raid/raid6/arm/recov_neon.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Intel Corporation
+ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ */
+
+#include <linux/mm.h>
+#include <linux/raid/pq.h>
+#include <asm/simd.h>
+#include "algos.h"
+#include "arm/neon.h"
+
+static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
+		int failb, void **ptrs)
+{
+	u8 *p, *q, *dp, *dq;
+	const u8 *pbmul;	/* P multiplier table for B data */
+	const u8 *qmul;		/* Q multiplier table (for both) */
+
+	p = (u8 *)ptrs[disks - 2];
+	q = (u8 *)ptrs[disks - 1];
+
+	/*
+	 * Compute syndrome with zero for the missing data pages
+	 * Use the dead data pages as temporary storage for
+	 * delta p and delta q
+	 */
+	dp = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 2] = dp;
+	dq = (u8 *)ptrs[failb];
+	ptrs[failb] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]     = dp;
+	ptrs[failb]     = dq;
+	ptrs[disks - 2] = p;
+	ptrs[disks - 1] = q;
+
+	/* Now, pick the proper data tables */
+	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
+					 raid6_gfexp[failb]]];
+
+	scoped_ksimd()
+		__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
+}
+
+static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
+		void **ptrs)
+{
+	u8 *p, *q, *dq;
+	const u8 *qmul;		/* Q multiplier table */
+
+	p = (u8 *)ptrs[disks - 2];
+	q = (u8 *)ptrs[disks - 1];
+
+	/*
+	 * Compute syndrome with zero for the missing data page
+	 * Use the dead data page as temporary storage for delta q
+	 */
+	dq = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]     = dq;
+	ptrs[disks - 1] = q;
+
+	/* Now, pick the proper data tables */
+	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+	scoped_ksimd()
+		__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
+}
+
+const struct raid6_recov_calls raid6_recov_neon = {
+	.data2		= raid6_2data_recov_neon,
+	.datap		= raid6_datap_recov_neon,
+	.name		= "neon",
+};
diff --git a/lib/raid/raid6/arm/recov_neon_inner.c b/lib/raid/raid6/arm/recov_neon_inner.c
new file mode 100644
index 0000000000000..53c355efa7ff6
--- /dev/null
+++ b/lib/raid/raid6/arm/recov_neon_inner.c
@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Intel Corporation
+ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
+ */
+
+#include <arm_neon.h>
+#include "arm/neon.h"
+
+#ifdef CONFIG_ARM
+/*
+ * AArch32 does not provide this intrinsic natively because it does not
+ * implement the underlying instruction. AArch32 only provides a 64-bit
+ * wide vtbl.8 instruction, so use that instead.
+ */
+static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b)
+{
+	union {
+		uint8x16_t	val;
+		uint8x8x2_t	pair;
+	} __a = { a };
+
+	return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)),
+			   vtbl2_u8(__a.pair, vget_high_u8(b)));
+}
+#endif
+
+void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
+			      uint8_t *dq, const uint8_t *pbmul,
+			      const uint8_t *qmul)
+{
+	uint8x16_t pm0 = vld1q_u8(pbmul);
+	uint8x16_t pm1 = vld1q_u8(pbmul + 16);
+	uint8x16_t qm0 = vld1q_u8(qmul);
+	uint8x16_t qm1 = vld1q_u8(qmul + 16);
+	uint8x16_t x0f = vdupq_n_u8(0x0f);
+
+	/*
+	 * while ( bytes-- ) {
+	 *	uint8_t px, qx, db;
+	 *
+	 *	px    = *p ^ *dp;
+	 *	qx    = qmul[*q ^ *dq];
+	 *	*dq++ = db = pbmul[px] ^ qx;
+	 *	*dp++ = db ^ px;
+	 *	p++; q++;
+	 * }
+	 */
+
+	while (bytes) {
+		uint8x16_t vx, vy, px, qx, db;
+
+		px = veorq_u8(vld1q_u8(p), vld1q_u8(dp));
+		vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
+
+		vy = vshrq_n_u8(vx, 4);
+		vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
+		vy = vqtbl1q_u8(qm1, vy);
+		qx = veorq_u8(vx, vy);
+
+		vy = vshrq_n_u8(px, 4);
+		vx = vqtbl1q_u8(pm0, vandq_u8(px, x0f));
+		vy = vqtbl1q_u8(pm1, vy);
+		vx = veorq_u8(vx, vy);
+		db = veorq_u8(vx, qx);
+
+		vst1q_u8(dq, db);
+		vst1q_u8(dp, veorq_u8(db, px));
+
+		bytes -= 16;
+		p += 16;
+		q += 16;
+		dp += 16;
+		dq += 16;
+	}
+}
+
+void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
+			      const uint8_t *qmul)
+{
+	uint8x16_t qm0 = vld1q_u8(qmul);
+	uint8x16_t qm1 = vld1q_u8(qmul + 16);
+	uint8x16_t x0f = vdupq_n_u8(0x0f);
+
+	/*
+	 * while (bytes--) {
+	 *	*p++ ^= *dq = qmul[*q ^ *dq];
+	 *	q++; dq++;
+	 * }
+	 */
+
+	while (bytes) {
+		uint8x16_t vx, vy;
+
+		vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
+
+		vy = vshrq_n_u8(vx, 4);
+		vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
+		vy = vqtbl1q_u8(qm1, vy);
+		vx = veorq_u8(vx, vy);
+		vy = veorq_u8(vx, vld1q_u8(p));
+
+		vst1q_u8(dq, vx);
+		vst1q_u8(p, vy);
+
+		bytes -= 16;
+		p += 16;
+		q += 16;
+		dq += 16;
+	}
+}
diff --git a/lib/raid/raid6/arm64/pq_arch.h b/lib/raid/raid6/arm64/pq_arch.h
new file mode 100644
index 0000000000000..27ff564d7594d
--- /dev/null
+++ b/lib/raid/raid6/arm64/pq_arch.h
@@ -0,0 +1 @@
+#include "arm/pq_arch.h"
diff --git a/lib/raid/raid6/int.uc b/lib/raid/raid6/int.uc
new file mode 100644
index 0000000000000..e63bd5a9c2ed1
--- /dev/null
+++ b/lib/raid/raid6/int.uc
@@ -0,0 +1,145 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Boston MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * int$#.c
+ *
+ * $#-way unrolled portable integer math RAID-6 instruction set
+ *
+ * This file is postprocessed using unroll.awk
+ */
+
+#include "algos.h"
+
+/*
+ * This is the C data type to use
+ */
+
+/* Change this from BITS_PER_LONG if there is something better... */
+#if BITS_PER_LONG == 64
+# define NBYTES(x) ((x) * 0x0101010101010101UL)
+# define NSIZE  8
+# define NSHIFT 3
+# define NSTRING "64"
+typedef u64 unative_t;
+#else
+# define NBYTES(x) ((x) * 0x01010101U)
+# define NSIZE  4
+# define NSHIFT 2
+# define NSTRING "32"
+typedef u32 unative_t;
+#endif
+
+
+
+/*
+ * These sub-operations are separate inlines since they can sometimes be
+ * specially optimized using architecture-specific hacks.
+ */
+
+/*
+ * The SHLBYTE() operation shifts each byte left by 1, *not*
+ * rolling over into the next byte
+ */
+static inline __attribute_const__ unative_t SHLBYTE(unative_t v)
+{
+	unative_t vv;
+
+	vv = (v << 1) & NBYTES(0xfe);
+	return vv;
+}
+
+/*
+ * The MASK() operation returns 0xFF in any byte for which the high
+ * bit is 1, 0x00 for any byte for which the high bit is 0.
+ */
+static inline __attribute_const__ unative_t MASK(unative_t v)
+{
+	unative_t vv;
+
+	vv = v & NBYTES(0x80);
+	vv = (vv << 1) - (vv >> 7); /* Overflow on the top bit is OK */
+	return vv;
+}
+
+
+static void raid6_int$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
+		wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
+		for ( z = z0-1 ; z >= 0 ; z-- ) {
+			wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			wp$$ ^= wd$$;
+			w2$$ = MASK(wq$$);
+			w1$$ = SHLBYTE(wq$$);
+			w2$$ &= NBYTES(0x1d);
+			w1$$ ^= w2$$;
+			wq$$ = w1$$ ^ wd$$;
+		}
+		*(unative_t *)&p[d+NSIZE*$$] = wp$$;
+		*(unative_t *)&q[d+NSIZE*$$] = wq$$;
+	}
+}
+
+static void raid6_int$#_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
+		/* P/Q data pages */
+		wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
+		for ( z = z0-1 ; z >= start ; z-- ) {
+			wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			wp$$ ^= wd$$;
+			w2$$ = MASK(wq$$);
+			w1$$ = SHLBYTE(wq$$);
+			w2$$ &= NBYTES(0x1d);
+			w1$$ ^= w2$$;
+			wq$$ = w1$$ ^ wd$$;
+		}
+		/* P/Q left side optimization */
+		for ( z = start-1 ; z >= 0 ; z-- ) {
+			w2$$ = MASK(wq$$);
+			w1$$ = SHLBYTE(wq$$);
+			w2$$ &= NBYTES(0x1d);
+			wq$$ = w1$$ ^ w2$$;
+		}
+		*(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
+		*(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
+	}
+
+}
+
+const struct raid6_calls raid6_intx$# = {
+	.gen_syndrome	= raid6_int$#_gen_syndrome,
+	.xor_syndrome	= raid6_int$#_xor_syndrome,
+	.name		= "int" NSTRING "x$#",
+};
diff --git a/lib/raid/raid6/loongarch/loongarch_simd.c b/lib/raid/raid6/loongarch/loongarch_simd.c
new file mode 100644
index 0000000000000..c1eb53fafd27e
--- /dev/null
+++ b/lib/raid/raid6/loongarch/loongarch_simd.c
@@ -0,0 +1,409 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RAID6 syndrome calculations in LoongArch SIMD (LSX & LASX)
+ *
+ * Copyright 2023 WANG Xuerui <git@xen0n.name>
+ *
+ * Based on the generic RAID-6 code (int.uc):
+ *
+ * Copyright 2002-2004 H. Peter Anvin
+ */
+
+#include <asm/cpu-features.h>
+#include <asm/fpu.h>
+#include "algos.h"
+
+/*
+ * The vector algorithms are currently priority 0, which means the generic
+ * scalar algorithms are not being disabled if vector support is present.
+ * This is like the similar LoongArch RAID5 XOR code, with the main reason
+ * repeated here: it cannot be ruled out at this point of time, that some
+ * future (maybe reduced) models could run the vector algorithms slower than
+ * the scalar ones, maybe for errata or micro-op reasons. It may be
+ * appropriate to revisit this after one or two more uarch generations.
+ */
+
+#ifdef CONFIG_CPU_HAS_LSX
+#define NSIZE 16
+
+static void raid6_lsx_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	/*
+	 * $vr0, $vr1, $vr2, $vr3: wp
+	 * $vr4, $vr5, $vr6, $vr7: wq
+	 * $vr8, $vr9, $vr10, $vr11: wd
+	 * $vr12, $vr13, $vr14, $vr15: w2
+	 * $vr16, $vr17, $vr18, $vr19: w1
+	 */
+	for (d = 0; d < bytes; d += NSIZE*4) {
+		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile("vld $vr0, %0" : : "m"(dptr[z0][d+0*NSIZE]));
+		asm volatile("vld $vr1, %0" : : "m"(dptr[z0][d+1*NSIZE]));
+		asm volatile("vld $vr2, %0" : : "m"(dptr[z0][d+2*NSIZE]));
+		asm volatile("vld $vr3, %0" : : "m"(dptr[z0][d+3*NSIZE]));
+		asm volatile("vori.b $vr4, $vr0, 0");
+		asm volatile("vori.b $vr5, $vr1, 0");
+		asm volatile("vori.b $vr6, $vr2, 0");
+		asm volatile("vori.b $vr7, $vr3, 0");
+		for (z = z0-1; z >= 0; z--) {
+			/* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */
+			asm volatile("vld $vr8, %0" : : "m"(dptr[z][d+0*NSIZE]));
+			asm volatile("vld $vr9, %0" : : "m"(dptr[z][d+1*NSIZE]));
+			asm volatile("vld $vr10, %0" : : "m"(dptr[z][d+2*NSIZE]));
+			asm volatile("vld $vr11, %0" : : "m"(dptr[z][d+3*NSIZE]));
+			/* wp$$ ^= wd$$; */
+			asm volatile("vxor.v $vr0, $vr0, $vr8");
+			asm volatile("vxor.v $vr1, $vr1, $vr9");
+			asm volatile("vxor.v $vr2, $vr2, $vr10");
+			asm volatile("vxor.v $vr3, $vr3, $vr11");
+			/* w2$$ = MASK(wq$$); */
+			asm volatile("vslti.b $vr12, $vr4, 0");
+			asm volatile("vslti.b $vr13, $vr5, 0");
+			asm volatile("vslti.b $vr14, $vr6, 0");
+			asm volatile("vslti.b $vr15, $vr7, 0");
+			/* w1$$ = SHLBYTE(wq$$); */
+			asm volatile("vslli.b $vr16, $vr4, 1");
+			asm volatile("vslli.b $vr17, $vr5, 1");
+			asm volatile("vslli.b $vr18, $vr6, 1");
+			asm volatile("vslli.b $vr19, $vr7, 1");
+			/* w2$$ &= NBYTES(0x1d); */
+			asm volatile("vandi.b $vr12, $vr12, 0x1d");
+			asm volatile("vandi.b $vr13, $vr13, 0x1d");
+			asm volatile("vandi.b $vr14, $vr14, 0x1d");
+			asm volatile("vandi.b $vr15, $vr15, 0x1d");
+			/* w1$$ ^= w2$$; */
+			asm volatile("vxor.v $vr16, $vr16, $vr12");
+			asm volatile("vxor.v $vr17, $vr17, $vr13");
+			asm volatile("vxor.v $vr18, $vr18, $vr14");
+			asm volatile("vxor.v $vr19, $vr19, $vr15");
+			/* wq$$ = w1$$ ^ wd$$; */
+			asm volatile("vxor.v $vr4, $vr16, $vr8");
+			asm volatile("vxor.v $vr5, $vr17, $vr9");
+			asm volatile("vxor.v $vr6, $vr18, $vr10");
+			asm volatile("vxor.v $vr7, $vr19, $vr11");
+		}
+		/* *(unative_t *)&p[d+NSIZE*$$] = wp$$; */
+		asm volatile("vst $vr0, %0" : "=m"(p[d+NSIZE*0]));
+		asm volatile("vst $vr1, %0" : "=m"(p[d+NSIZE*1]));
+		asm volatile("vst $vr2, %0" : "=m"(p[d+NSIZE*2]));
+		asm volatile("vst $vr3, %0" : "=m"(p[d+NSIZE*3]));
+		/* *(unative_t *)&q[d+NSIZE*$$] = wq$$; */
+		asm volatile("vst $vr4, %0" : "=m"(q[d+NSIZE*0]));
+		asm volatile("vst $vr5, %0" : "=m"(q[d+NSIZE*1]));
+		asm volatile("vst $vr6, %0" : "=m"(q[d+NSIZE*2]));
+		asm volatile("vst $vr7, %0" : "=m"(q[d+NSIZE*3]));
+	}
+
+	kernel_fpu_end();
+}
+
+static void raid6_lsx_xor_syndrome(int disks, int start, int stop,
+				   size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	/*
+	 * $vr0, $vr1, $vr2, $vr3: wp
+	 * $vr4, $vr5, $vr6, $vr7: wq
+	 * $vr8, $vr9, $vr10, $vr11: wd
+	 * $vr12, $vr13, $vr14, $vr15: w2
+	 * $vr16, $vr17, $vr18, $vr19: w1
+	 */
+	for (d = 0; d < bytes; d += NSIZE*4) {
+		/* P/Q data pages */
+		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile("vld $vr0, %0" : : "m"(dptr[z0][d+0*NSIZE]));
+		asm volatile("vld $vr1, %0" : : "m"(dptr[z0][d+1*NSIZE]));
+		asm volatile("vld $vr2, %0" : : "m"(dptr[z0][d+2*NSIZE]));
+		asm volatile("vld $vr3, %0" : : "m"(dptr[z0][d+3*NSIZE]));
+		asm volatile("vori.b $vr4, $vr0, 0");
+		asm volatile("vori.b $vr5, $vr1, 0");
+		asm volatile("vori.b $vr6, $vr2, 0");
+		asm volatile("vori.b $vr7, $vr3, 0");
+		for (z = z0-1; z >= start; z--) {
+			/* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */
+			asm volatile("vld $vr8, %0" : : "m"(dptr[z][d+0*NSIZE]));
+			asm volatile("vld $vr9, %0" : : "m"(dptr[z][d+1*NSIZE]));
+			asm volatile("vld $vr10, %0" : : "m"(dptr[z][d+2*NSIZE]));
+			asm volatile("vld $vr11, %0" : : "m"(dptr[z][d+3*NSIZE]));
+			/* wp$$ ^= wd$$; */
+			asm volatile("vxor.v $vr0, $vr0, $vr8");
+			asm volatile("vxor.v $vr1, $vr1, $vr9");
+			asm volatile("vxor.v $vr2, $vr2, $vr10");
+			asm volatile("vxor.v $vr3, $vr3, $vr11");
+			/* w2$$ = MASK(wq$$); */
+			asm volatile("vslti.b $vr12, $vr4, 0");
+			asm volatile("vslti.b $vr13, $vr5, 0");
+			asm volatile("vslti.b $vr14, $vr6, 0");
+			asm volatile("vslti.b $vr15, $vr7, 0");
+			/* w1$$ = SHLBYTE(wq$$); */
+			asm volatile("vslli.b $vr16, $vr4, 1");
+			asm volatile("vslli.b $vr17, $vr5, 1");
+			asm volatile("vslli.b $vr18, $vr6, 1");
+			asm volatile("vslli.b $vr19, $vr7, 1");
+			/* w2$$ &= NBYTES(0x1d); */
+			asm volatile("vandi.b $vr12, $vr12, 0x1d");
+			asm volatile("vandi.b $vr13, $vr13, 0x1d");
+			asm volatile("vandi.b $vr14, $vr14, 0x1d");
+			asm volatile("vandi.b $vr15, $vr15, 0x1d");
+			/* w1$$ ^= w2$$; */
+			asm volatile("vxor.v $vr16, $vr16, $vr12");
+			asm volatile("vxor.v $vr17, $vr17, $vr13");
+			asm volatile("vxor.v $vr18, $vr18, $vr14");
+			asm volatile("vxor.v $vr19, $vr19, $vr15");
+			/* wq$$ = w1$$ ^ wd$$; */
+			asm volatile("vxor.v $vr4, $vr16, $vr8");
+			asm volatile("vxor.v $vr5, $vr17, $vr9");
+			asm volatile("vxor.v $vr6, $vr18, $vr10");
+			asm volatile("vxor.v $vr7, $vr19, $vr11");
+		}
+
+		/* P/Q left side optimization */
+		for (z = start-1; z >= 0; z--) {
+			/* w2$$ = MASK(wq$$); */
+			asm volatile("vslti.b $vr12, $vr4, 0");
+			asm volatile("vslti.b $vr13, $vr5, 0");
+			asm volatile("vslti.b $vr14, $vr6, 0");
+			asm volatile("vslti.b $vr15, $vr7, 0");
+			/* w1$$ = SHLBYTE(wq$$); */
+			asm volatile("vslli.b $vr16, $vr4, 1");
+			asm volatile("vslli.b $vr17, $vr5, 1");
+			asm volatile("vslli.b $vr18, $vr6, 1");
+			asm volatile("vslli.b $vr19, $vr7, 1");
+			/* w2$$ &= NBYTES(0x1d); */
+			asm volatile("vandi.b $vr12, $vr12, 0x1d");
+			asm volatile("vandi.b $vr13, $vr13, 0x1d");
+			asm volatile("vandi.b $vr14, $vr14, 0x1d");
+			asm volatile("vandi.b $vr15, $vr15, 0x1d");
+			/* wq$$ = w1$$ ^ w2$$; */
+			asm volatile("vxor.v $vr4, $vr16, $vr12");
+			asm volatile("vxor.v $vr5, $vr17, $vr13");
+			asm volatile("vxor.v $vr6, $vr18, $vr14");
+			asm volatile("vxor.v $vr7, $vr19, $vr15");
+		}
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
+		 */
+		asm volatile(
+			"vld $vr20, %0\n\t"
+			"vld $vr21, %1\n\t"
+			"vld $vr22, %2\n\t"
+			"vld $vr23, %3\n\t"
+			"vld $vr24, %4\n\t"
+			"vld $vr25, %5\n\t"
+			"vld $vr26, %6\n\t"
+			"vld $vr27, %7\n\t"
+			"vxor.v $vr20, $vr20, $vr0\n\t"
+			"vxor.v $vr21, $vr21, $vr1\n\t"
+			"vxor.v $vr22, $vr22, $vr2\n\t"
+			"vxor.v $vr23, $vr23, $vr3\n\t"
+			"vxor.v $vr24, $vr24, $vr4\n\t"
+			"vxor.v $vr25, $vr25, $vr5\n\t"
+			"vxor.v $vr26, $vr26, $vr6\n\t"
+			"vxor.v $vr27, $vr27, $vr7\n\t"
+			"vst $vr20, %0\n\t"
+			"vst $vr21, %1\n\t"
+			"vst $vr22, %2\n\t"
+			"vst $vr23, %3\n\t"
+			"vst $vr24, %4\n\t"
+			"vst $vr25, %5\n\t"
+			"vst $vr26, %6\n\t"
+			"vst $vr27, %7\n\t"
+			: "+m"(p[d+NSIZE*0]), "+m"(p[d+NSIZE*1]),
+			  "+m"(p[d+NSIZE*2]), "+m"(p[d+NSIZE*3]),
+			  "+m"(q[d+NSIZE*0]), "+m"(q[d+NSIZE*1]),
+			  "+m"(q[d+NSIZE*2]), "+m"(q[d+NSIZE*3])
+		);
+	}
+
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_lsx = {
+	.gen_syndrome	= raid6_lsx_gen_syndrome,
+	.xor_syndrome	= raid6_lsx_xor_syndrome,
+	.name		= "lsx",
+};
+
+#undef NSIZE
+#endif /* CONFIG_CPU_HAS_LSX */
+
+#ifdef CONFIG_CPU_HAS_LASX
+#define NSIZE 32
+
+static void raid6_lasx_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	/*
+	 * $xr0, $xr1: wp
+	 * $xr2, $xr3: wq
+	 * $xr4, $xr5: wd
+	 * $xr6, $xr7: w2
+	 * $xr8, $xr9: w1
+	 */
+	for (d = 0; d < bytes; d += NSIZE*2) {
+		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile("xvld $xr0, %0" : : "m"(dptr[z0][d+0*NSIZE]));
+		asm volatile("xvld $xr1, %0" : : "m"(dptr[z0][d+1*NSIZE]));
+		asm volatile("xvori.b $xr2, $xr0, 0");
+		asm volatile("xvori.b $xr3, $xr1, 0");
+		for (z = z0-1; z >= 0; z--) {
+			/* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */
+			asm volatile("xvld $xr4, %0" : : "m"(dptr[z][d+0*NSIZE]));
+			asm volatile("xvld $xr5, %0" : : "m"(dptr[z][d+1*NSIZE]));
+			/* wp$$ ^= wd$$; */
+			asm volatile("xvxor.v $xr0, $xr0, $xr4");
+			asm volatile("xvxor.v $xr1, $xr1, $xr5");
+			/* w2$$ = MASK(wq$$); */
+			asm volatile("xvslti.b $xr6, $xr2, 0");
+			asm volatile("xvslti.b $xr7, $xr3, 0");
+			/* w1$$ = SHLBYTE(wq$$); */
+			asm volatile("xvslli.b $xr8, $xr2, 1");
+			asm volatile("xvslli.b $xr9, $xr3, 1");
+			/* w2$$ &= NBYTES(0x1d); */
+			asm volatile("xvandi.b $xr6, $xr6, 0x1d");
+			asm volatile("xvandi.b $xr7, $xr7, 0x1d");
+			/* w1$$ ^= w2$$; */
+			asm volatile("xvxor.v $xr8, $xr8, $xr6");
+			asm volatile("xvxor.v $xr9, $xr9, $xr7");
+			/* wq$$ = w1$$ ^ wd$$; */
+			asm volatile("xvxor.v $xr2, $xr8, $xr4");
+			asm volatile("xvxor.v $xr3, $xr9, $xr5");
+		}
+		/* *(unative_t *)&p[d+NSIZE*$$] = wp$$; */
+		asm volatile("xvst $xr0, %0" : "=m"(p[d+NSIZE*0]));
+		asm volatile("xvst $xr1, %0" : "=m"(p[d+NSIZE*1]));
+		/* *(unative_t *)&q[d+NSIZE*$$] = wq$$; */
+		asm volatile("xvst $xr2, %0" : "=m"(q[d+NSIZE*0]));
+		asm volatile("xvst $xr3, %0" : "=m"(q[d+NSIZE*1]));
+	}
+
+	kernel_fpu_end();
+}
+
+static void raid6_lasx_xor_syndrome(int disks, int start, int stop,
+				    size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	/*
+	 * $xr0, $xr1: wp
+	 * $xr2, $xr3: wq
+	 * $xr4, $xr5: wd
+	 * $xr6, $xr7: w2
+	 * $xr8, $xr9: w1
+	 */
+	for (d = 0; d < bytes; d += NSIZE*2) {
+		/* P/Q data pages */
+		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile("xvld $xr0, %0" : : "m"(dptr[z0][d+0*NSIZE]));
+		asm volatile("xvld $xr1, %0" : : "m"(dptr[z0][d+1*NSIZE]));
+		asm volatile("xvori.b $xr2, $xr0, 0");
+		asm volatile("xvori.b $xr3, $xr1, 0");
+		for (z = z0-1; z >= start; z--) {
+			/* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */
+			asm volatile("xvld $xr4, %0" : : "m"(dptr[z][d+0*NSIZE]));
+			asm volatile("xvld $xr5, %0" : : "m"(dptr[z][d+1*NSIZE]));
+			/* wp$$ ^= wd$$; */
+			asm volatile("xvxor.v $xr0, $xr0, $xr4");
+			asm volatile("xvxor.v $xr1, $xr1, $xr5");
+			/* w2$$ = MASK(wq$$); */
+			asm volatile("xvslti.b $xr6, $xr2, 0");
+			asm volatile("xvslti.b $xr7, $xr3, 0");
+			/* w1$$ = SHLBYTE(wq$$); */
+			asm volatile("xvslli.b $xr8, $xr2, 1");
+			asm volatile("xvslli.b $xr9, $xr3, 1");
+			/* w2$$ &= NBYTES(0x1d); */
+			asm volatile("xvandi.b $xr6, $xr6, 0x1d");
+			asm volatile("xvandi.b $xr7, $xr7, 0x1d");
+			/* w1$$ ^= w2$$; */
+			asm volatile("xvxor.v $xr8, $xr8, $xr6");
+			asm volatile("xvxor.v $xr9, $xr9, $xr7");
+			/* wq$$ = w1$$ ^ wd$$; */
+			asm volatile("xvxor.v $xr2, $xr8, $xr4");
+			asm volatile("xvxor.v $xr3, $xr9, $xr5");
+		}
+
+		/* P/Q left side optimization */
+		for (z = start-1; z >= 0; z--) {
+			/* w2$$ = MASK(wq$$); */
+			asm volatile("xvslti.b $xr6, $xr2, 0");
+			asm volatile("xvslti.b $xr7, $xr3, 0");
+			/* w1$$ = SHLBYTE(wq$$); */
+			asm volatile("xvslli.b $xr8, $xr2, 1");
+			asm volatile("xvslli.b $xr9, $xr3, 1");
+			/* w2$$ &= NBYTES(0x1d); */
+			asm volatile("xvandi.b $xr6, $xr6, 0x1d");
+			asm volatile("xvandi.b $xr7, $xr7, 0x1d");
+			/* wq$$ = w1$$ ^ w2$$; */
+			asm volatile("xvxor.v $xr2, $xr8, $xr6");
+			asm volatile("xvxor.v $xr3, $xr9, $xr7");
+		}
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
+		 */
+		asm volatile(
+			"xvld $xr10, %0\n\t"
+			"xvld $xr11, %1\n\t"
+			"xvld $xr12, %2\n\t"
+			"xvld $xr13, %3\n\t"
+			"xvxor.v $xr10, $xr10, $xr0\n\t"
+			"xvxor.v $xr11, $xr11, $xr1\n\t"
+			"xvxor.v $xr12, $xr12, $xr2\n\t"
+			"xvxor.v $xr13, $xr13, $xr3\n\t"
+			"xvst $xr10, %0\n\t"
+			"xvst $xr11, %1\n\t"
+			"xvst $xr12, %2\n\t"
+			"xvst $xr13, %3\n\t"
+			: "+m"(p[d+NSIZE*0]), "+m"(p[d+NSIZE*1]),
+			  "+m"(q[d+NSIZE*0]), "+m"(q[d+NSIZE*1])
+		);
+	}
+
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_lasx = {
+	.gen_syndrome	= raid6_lasx_gen_syndrome,
+	.xor_syndrome	= raid6_lasx_xor_syndrome,
+	.name		= "lasx",
+};
+#undef NSIZE
+#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/lib/raid/raid6/loongarch/pq_arch.h b/lib/raid/raid6/loongarch/pq_arch.h
new file mode 100644
index 0000000000000..ae443a4d7b696
--- /dev/null
+++ b/lib/raid/raid6/loongarch/pq_arch.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/cpu-features.h>
+
+extern const struct raid6_calls raid6_lsx;
+extern const struct raid6_calls raid6_lasx;
+
+extern const struct raid6_recov_calls raid6_recov_lsx;
+extern const struct raid6_recov_calls raid6_recov_lasx;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+	raid6_algo_add_default();
+	if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx)
+		raid6_algo_add(&raid6_lsx);
+	if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx)
+		raid6_algo_add(&raid6_lasx);
+
+	if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx)
+		raid6_recov_algo_add(&raid6_recov_lasx);
+	else if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx)
+		raid6_recov_algo_add(&raid6_recov_lsx);
+}
diff --git a/lib/raid/raid6/loongarch/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
new file mode 100644
index 0000000000000..87a2313bbb4f7
--- /dev/null
+++ b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
@@ -0,0 +1,502 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * RAID6 recovery algorithms in LoongArch SIMD (LSX & LASX)
+ *
+ * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
+ *
+ * Originally based on recov_avx2.c and recov_ssse3.c:
+ *
+ * Copyright (C) 2012 Intel Corporation
+ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
+ */
+
+#include <linux/mm.h>
+#include <linux/raid/pq.h>
+#include <asm/cpu-features.h>
+#include <asm/fpu.h>
+#include "algos.h"
+
+/*
+ * Unlike with the syndrome calculation algorithms, there's no boot-time
+ * selection of recovery algorithms by benchmarking, so we have to specify
+ * the priorities and hope the future cores will all have decent vector
+ * support (i.e. no LASX slower than LSX, or even scalar code).
+ */
+
+#ifdef CONFIG_CPU_HAS_LSX
+static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila,
+				  int failb, void **ptrs)
+{
+	u8 *p, *q, *dp, *dq;
+	const u8 *pbmul;	/* P multiplier table for B data */
+	const u8 *qmul;		/* Q multiplier table (for both) */
+
+	p = (u8 *)ptrs[disks - 2];
+	q = (u8 *)ptrs[disks - 1];
+
+	/*
+	 * Compute syndrome with zero for the missing data pages
+	 * Use the dead data pages as temporary storage for
+	 * delta p and delta q
+	 */
+	dp = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 2] = dp;
+	dq = (u8 *)ptrs[failb];
+	ptrs[failb] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila] = dp;
+	ptrs[failb] = dq;
+	ptrs[disks - 2] = p;
+	ptrs[disks - 1] = q;
+
+	/* Now, pick the proper data tables */
+	pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]];
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]];
+
+	kernel_fpu_begin();
+
+	/*
+	 * vr20, vr21: qmul
+	 * vr22, vr23: pbmul
+	 */
+	asm volatile("vld $vr20, %0" : : "m" (qmul[0]));
+	asm volatile("vld $vr21, %0" : : "m" (qmul[16]));
+	asm volatile("vld $vr22, %0" : : "m" (pbmul[0]));
+	asm volatile("vld $vr23, %0" : : "m" (pbmul[16]));
+
+	while (bytes) {
+		/* vr4 - vr7: Q */
+		asm volatile("vld $vr4, %0" : : "m" (q[0]));
+		asm volatile("vld $vr5, %0" : : "m" (q[16]));
+		asm volatile("vld $vr6, %0" : : "m" (q[32]));
+		asm volatile("vld $vr7, %0" : : "m" (q[48]));
+		/*  vr4 - vr7: Q + Qxy */
+		asm volatile("vld $vr8, %0" : : "m" (dq[0]));
+		asm volatile("vld $vr9, %0" : : "m" (dq[16]));
+		asm volatile("vld $vr10, %0" : : "m" (dq[32]));
+		asm volatile("vld $vr11, %0" : : "m" (dq[48]));
+		asm volatile("vxor.v $vr4, $vr4, $vr8");
+		asm volatile("vxor.v $vr5, $vr5, $vr9");
+		asm volatile("vxor.v $vr6, $vr6, $vr10");
+		asm volatile("vxor.v $vr7, $vr7, $vr11");
+		/* vr0 - vr3: P */
+		asm volatile("vld $vr0, %0" : : "m" (p[0]));
+		asm volatile("vld $vr1, %0" : : "m" (p[16]));
+		asm volatile("vld $vr2, %0" : : "m" (p[32]));
+		asm volatile("vld $vr3, %0" : : "m" (p[48]));
+		/* vr0 - vr3: P + Pxy */
+		asm volatile("vld $vr8, %0" : : "m" (dp[0]));
+		asm volatile("vld $vr9, %0" : : "m" (dp[16]));
+		asm volatile("vld $vr10, %0" : : "m" (dp[32]));
+		asm volatile("vld $vr11, %0" : : "m" (dp[48]));
+		asm volatile("vxor.v $vr0, $vr0, $vr8");
+		asm volatile("vxor.v $vr1, $vr1, $vr9");
+		asm volatile("vxor.v $vr2, $vr2, $vr10");
+		asm volatile("vxor.v $vr3, $vr3, $vr11");
+
+		/* vr8 - vr11: higher 4 bits of each byte of (Q + Qxy) */
+		asm volatile("vsrli.b $vr8, $vr4, 4");
+		asm volatile("vsrli.b $vr9, $vr5, 4");
+		asm volatile("vsrli.b $vr10, $vr6, 4");
+		asm volatile("vsrli.b $vr11, $vr7, 4");
+		/* vr4 - vr7: lower 4 bits of each byte of (Q + Qxy) */
+		asm volatile("vandi.b $vr4, $vr4, 0x0f");
+		asm volatile("vandi.b $vr5, $vr5, 0x0f");
+		asm volatile("vandi.b $vr6, $vr6, 0x0f");
+		asm volatile("vandi.b $vr7, $vr7, 0x0f");
+		/* lookup from qmul[0] */
+		asm volatile("vshuf.b $vr4, $vr20, $vr20, $vr4");
+		asm volatile("vshuf.b $vr5, $vr20, $vr20, $vr5");
+		asm volatile("vshuf.b $vr6, $vr20, $vr20, $vr6");
+		asm volatile("vshuf.b $vr7, $vr20, $vr20, $vr7");
+		/* lookup from qmul[16] */
+		asm volatile("vshuf.b $vr8, $vr21, $vr21, $vr8");
+		asm volatile("vshuf.b $vr9, $vr21, $vr21, $vr9");
+		asm volatile("vshuf.b $vr10, $vr21, $vr21, $vr10");
+		asm volatile("vshuf.b $vr11, $vr21, $vr21, $vr11");
+		/* vr16 - vr19: B(Q + Qxy) */
+		asm volatile("vxor.v $vr16, $vr8, $vr4");
+		asm volatile("vxor.v $vr17, $vr9, $vr5");
+		asm volatile("vxor.v $vr18, $vr10, $vr6");
+		asm volatile("vxor.v $vr19, $vr11, $vr7");
+
+		/* vr4 - vr7: higher 4 bits of each byte of (P + Pxy) */
+		asm volatile("vsrli.b $vr4, $vr0, 4");
+		asm volatile("vsrli.b $vr5, $vr1, 4");
+		asm volatile("vsrli.b $vr6, $vr2, 4");
+		asm volatile("vsrli.b $vr7, $vr3, 4");
+		/* vr12 - vr15: lower 4 bits of each byte of (P + Pxy) */
+		asm volatile("vandi.b $vr12, $vr0, 0x0f");
+		asm volatile("vandi.b $vr13, $vr1, 0x0f");
+		asm volatile("vandi.b $vr14, $vr2, 0x0f");
+		asm volatile("vandi.b $vr15, $vr3, 0x0f");
+		/* lookup from pbmul[0] */
+		asm volatile("vshuf.b $vr12, $vr22, $vr22, $vr12");
+		asm volatile("vshuf.b $vr13, $vr22, $vr22, $vr13");
+		asm volatile("vshuf.b $vr14, $vr22, $vr22, $vr14");
+		asm volatile("vshuf.b $vr15, $vr22, $vr22, $vr15");
+		/* lookup from pbmul[16] */
+		asm volatile("vshuf.b $vr4, $vr23, $vr23, $vr4");
+		asm volatile("vshuf.b $vr5, $vr23, $vr23, $vr5");
+		asm volatile("vshuf.b $vr6, $vr23, $vr23, $vr6");
+		asm volatile("vshuf.b $vr7, $vr23, $vr23, $vr7");
+		/* vr4 - vr7: A(P + Pxy) */
+		asm volatile("vxor.v $vr4, $vr4, $vr12");
+		asm volatile("vxor.v $vr5, $vr5, $vr13");
+		asm volatile("vxor.v $vr6, $vr6, $vr14");
+		asm volatile("vxor.v $vr7, $vr7, $vr15");
+
+		/* vr4 - vr7: A(P + Pxy) + B(Q + Qxy) = Dx */
+		asm volatile("vxor.v $vr4, $vr4, $vr16");
+		asm volatile("vxor.v $vr5, $vr5, $vr17");
+		asm volatile("vxor.v $vr6, $vr6, $vr18");
+		asm volatile("vxor.v $vr7, $vr7, $vr19");
+		asm volatile("vst $vr4, %0" : "=m" (dq[0]));
+		asm volatile("vst $vr5, %0" : "=m" (dq[16]));
+		asm volatile("vst $vr6, %0" : "=m" (dq[32]));
+		asm volatile("vst $vr7, %0" : "=m" (dq[48]));
+
+		/* vr0 - vr3: P + Pxy + Dx = Dy */
+		asm volatile("vxor.v $vr0, $vr0, $vr4");
+		asm volatile("vxor.v $vr1, $vr1, $vr5");
+		asm volatile("vxor.v $vr2, $vr2, $vr6");
+		asm volatile("vxor.v $vr3, $vr3, $vr7");
+		asm volatile("vst $vr0, %0" : "=m" (dp[0]));
+		asm volatile("vst $vr1, %0" : "=m" (dp[16]));
+		asm volatile("vst $vr2, %0" : "=m" (dp[32]));
+		asm volatile("vst $vr3, %0" : "=m" (dp[48]));
+
+		bytes -= 64;
+		p += 64;
+		q += 64;
+		dp += 64;
+		dq += 64;
+	}
+
+	kernel_fpu_end();
+}
+
+static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila,
+				  void **ptrs)
+{
+	u8 *p, *q, *dq;
+	const u8 *qmul;		/* Q multiplier table */
+
+	p = (u8 *)ptrs[disks - 2];
+	q = (u8 *)ptrs[disks - 1];
+
+	/*
+	 * Compute syndrome with zero for the missing data page
+	 * Use the dead data page as temporary storage for delta q
+	 */
+	dq = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila] = dq;
+	ptrs[disks - 1] = q;
+
+	/* Now, pick the proper data tables */
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+	kernel_fpu_begin();
+
+	/* vr22, vr23: qmul */
+	asm volatile("vld $vr22, %0" : : "m" (qmul[0]));
+	asm volatile("vld $vr23, %0" : : "m" (qmul[16]));
+
+	while (bytes) {
+		/* vr0 - vr3: P + Dx */
+		asm volatile("vld $vr0, %0" : : "m" (p[0]));
+		asm volatile("vld $vr1, %0" : : "m" (p[16]));
+		asm volatile("vld $vr2, %0" : : "m" (p[32]));
+		asm volatile("vld $vr3, %0" : : "m" (p[48]));
+		/* vr4 - vr7: Qx */
+		asm volatile("vld $vr4, %0" : : "m" (dq[0]));
+		asm volatile("vld $vr5, %0" : : "m" (dq[16]));
+		asm volatile("vld $vr6, %0" : : "m" (dq[32]));
+		asm volatile("vld $vr7, %0" : : "m" (dq[48]));
+		/* vr4 - vr7: Q + Qx */
+		asm volatile("vld $vr8, %0" : : "m" (q[0]));
+		asm volatile("vld $vr9, %0" : : "m" (q[16]));
+		asm volatile("vld $vr10, %0" : : "m" (q[32]));
+		asm volatile("vld $vr11, %0" : : "m" (q[48]));
+		asm volatile("vxor.v $vr4, $vr4, $vr8");
+		asm volatile("vxor.v $vr5, $vr5, $vr9");
+		asm volatile("vxor.v $vr6, $vr6, $vr10");
+		asm volatile("vxor.v $vr7, $vr7, $vr11");
+
+		/* vr8 - vr11: higher 4 bits of each byte of (Q + Qx) */
+		asm volatile("vsrli.b $vr8, $vr4, 4");
+		asm volatile("vsrli.b $vr9, $vr5, 4");
+		asm volatile("vsrli.b $vr10, $vr6, 4");
+		asm volatile("vsrli.b $vr11, $vr7, 4");
+		/* vr4 - vr7: lower 4 bits of each byte of (Q + Qx) */
+		asm volatile("vandi.b $vr4, $vr4, 0x0f");
+		asm volatile("vandi.b $vr5, $vr5, 0x0f");
+		asm volatile("vandi.b $vr6, $vr6, 0x0f");
+		asm volatile("vandi.b $vr7, $vr7, 0x0f");
+		/* lookup from qmul[0] */
+		asm volatile("vshuf.b $vr4, $vr22, $vr22, $vr4");
+		asm volatile("vshuf.b $vr5, $vr22, $vr22, $vr5");
+		asm volatile("vshuf.b $vr6, $vr22, $vr22, $vr6");
+		asm volatile("vshuf.b $vr7, $vr22, $vr22, $vr7");
+		/* lookup from qmul[16] */
+		asm volatile("vshuf.b $vr8, $vr23, $vr23, $vr8");
+		asm volatile("vshuf.b $vr9, $vr23, $vr23, $vr9");
+		asm volatile("vshuf.b $vr10, $vr23, $vr23, $vr10");
+		asm volatile("vshuf.b $vr11, $vr23, $vr23, $vr11");
+		/* vr4 - vr7: qmul(Q + Qx) = Dx */
+		asm volatile("vxor.v $vr4, $vr4, $vr8");
+		asm volatile("vxor.v $vr5, $vr5, $vr9");
+		asm volatile("vxor.v $vr6, $vr6, $vr10");
+		asm volatile("vxor.v $vr7, $vr7, $vr11");
+		asm volatile("vst $vr4, %0" : "=m" (dq[0]));
+		asm volatile("vst $vr5, %0" : "=m" (dq[16]));
+		asm volatile("vst $vr6, %0" : "=m" (dq[32]));
+		asm volatile("vst $vr7, %0" : "=m" (dq[48]));
+
+		/* vr0 - vr3: P + Dx + Dx = P */
+		asm volatile("vxor.v $vr0, $vr0, $vr4");
+		asm volatile("vxor.v $vr1, $vr1, $vr5");
+		asm volatile("vxor.v $vr2, $vr2, $vr6");
+		asm volatile("vxor.v $vr3, $vr3, $vr7");
+		asm volatile("vst $vr0, %0" : "=m" (p[0]));
+		asm volatile("vst $vr1, %0" : "=m" (p[16]));
+		asm volatile("vst $vr2, %0" : "=m" (p[32]));
+		asm volatile("vst $vr3, %0" : "=m" (p[48]));
+
+		bytes -= 64;
+		p += 64;
+		q += 64;
+		dq += 64;
+	}
+
+	kernel_fpu_end();
+}
+
+const struct raid6_recov_calls raid6_recov_lsx = {
+	.data2 = raid6_2data_recov_lsx,
+	.datap = raid6_datap_recov_lsx,
+	.name = "lsx",
+};
+#endif /* CONFIG_CPU_HAS_LSX */
+
+#ifdef CONFIG_CPU_HAS_LASX
+static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila,
+				   int failb, void **ptrs)
+{
+	u8 *p, *q, *dp, *dq;
+	const u8 *pbmul;	/* P multiplier table for B data */
+	const u8 *qmul;		/* Q multiplier table (for both) */
+
+	p = (u8 *)ptrs[disks - 2];
+	q = (u8 *)ptrs[disks - 1];
+
+	/*
+	 * Compute syndrome with zero for the missing data pages
+	 * Use the dead data pages as temporary storage for
+	 * delta p and delta q
+	 */
+	dp = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 2] = dp;
+	dq = (u8 *)ptrs[failb];
+	ptrs[failb] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila] = dp;
+	ptrs[failb] = dq;
+	ptrs[disks - 2] = p;
+	ptrs[disks - 1] = q;
+
+	/* Now, pick the proper data tables */
+	pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]];
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]];
+
+	kernel_fpu_begin();
+
+	/*
+	 * xr20, xr21: qmul
+	 * xr22, xr23: pbmul
+	 */
+	asm volatile("vld $vr20, %0" : : "m" (qmul[0]));
+	asm volatile("vld $vr21, %0" : : "m" (qmul[16]));
+	asm volatile("vld $vr22, %0" : : "m" (pbmul[0]));
+	asm volatile("vld $vr23, %0" : : "m" (pbmul[16]));
+	asm volatile("xvreplve0.q $xr20, $xr20");
+	asm volatile("xvreplve0.q $xr21, $xr21");
+	asm volatile("xvreplve0.q $xr22, $xr22");
+	asm volatile("xvreplve0.q $xr23, $xr23");
+
+	while (bytes) {
+		/* xr0, xr1: Q */
+		asm volatile("xvld $xr0, %0" : : "m" (q[0]));
+		asm volatile("xvld $xr1, %0" : : "m" (q[32]));
+		/* xr0, xr1: Q + Qxy */
+		asm volatile("xvld $xr4, %0" : : "m" (dq[0]));
+		asm volatile("xvld $xr5, %0" : : "m" (dq[32]));
+		asm volatile("xvxor.v $xr0, $xr0, $xr4");
+		asm volatile("xvxor.v $xr1, $xr1, $xr5");
+		/* xr2, xr3: P */
+		asm volatile("xvld $xr2, %0" : : "m" (p[0]));
+		asm volatile("xvld $xr3, %0" : : "m" (p[32]));
+		/* xr2, xr3: P + Pxy */
+		asm volatile("xvld $xr4, %0" : : "m" (dp[0]));
+		asm volatile("xvld $xr5, %0" : : "m" (dp[32]));
+		asm volatile("xvxor.v $xr2, $xr2, $xr4");
+		asm volatile("xvxor.v $xr3, $xr3, $xr5");
+
+		/* xr4, xr5: higher 4 bits of each byte of (Q + Qxy) */
+		asm volatile("xvsrli.b $xr4, $xr0, 4");
+		asm volatile("xvsrli.b $xr5, $xr1, 4");
+		/* xr0, xr1: lower 4 bits of each byte of (Q + Qxy) */
+		asm volatile("xvandi.b $xr0, $xr0, 0x0f");
+		asm volatile("xvandi.b $xr1, $xr1, 0x0f");
+		/* lookup from qmul[0] */
+		asm volatile("xvshuf.b $xr0, $xr20, $xr20, $xr0");
+		asm volatile("xvshuf.b $xr1, $xr20, $xr20, $xr1");
+		/* lookup from qmul[16] */
+		asm volatile("xvshuf.b $xr4, $xr21, $xr21, $xr4");
+		asm volatile("xvshuf.b $xr5, $xr21, $xr21, $xr5");
+		/* xr6, xr7: B(Q + Qxy) */
+		asm volatile("xvxor.v $xr6, $xr4, $xr0");
+		asm volatile("xvxor.v $xr7, $xr5, $xr1");
+
+		/* xr4, xr5: higher 4 bits of each byte of (P + Pxy) */
+		asm volatile("xvsrli.b $xr4, $xr2, 4");
+		asm volatile("xvsrli.b $xr5, $xr3, 4");
+		/* xr0, xr1: lower 4 bits of each byte of (P + Pxy) */
+		asm volatile("xvandi.b $xr0, $xr2, 0x0f");
+		asm volatile("xvandi.b $xr1, $xr3, 0x0f");
+		/* lookup from pbmul[0] */
+		asm volatile("xvshuf.b $xr0, $xr22, $xr22, $xr0");
+		asm volatile("xvshuf.b $xr1, $xr22, $xr22, $xr1");
+		/* lookup from pbmul[16] */
+		asm volatile("xvshuf.b $xr4, $xr23, $xr23, $xr4");
+		asm volatile("xvshuf.b $xr5, $xr23, $xr23, $xr5");
+		/* xr0, xr1: A(P + Pxy) */
+		asm volatile("xvxor.v $xr0, $xr0, $xr4");
+		asm volatile("xvxor.v $xr1, $xr1, $xr5");
+
+		/* xr0, xr1: A(P + Pxy) + B(Q + Qxy) = Dx */
+		asm volatile("xvxor.v $xr0, $xr0, $xr6");
+		asm volatile("xvxor.v $xr1, $xr1, $xr7");
+
+		/* xr2, xr3: P + Pxy + Dx = Dy */
+		asm volatile("xvxor.v $xr2, $xr2, $xr0");
+		asm volatile("xvxor.v $xr3, $xr3, $xr1");
+
+		asm volatile("xvst $xr0, %0" : "=m" (dq[0]));
+		asm volatile("xvst $xr1, %0" : "=m" (dq[32]));
+		asm volatile("xvst $xr2, %0" : "=m" (dp[0]));
+		asm volatile("xvst $xr3, %0" : "=m" (dp[32]));
+
+		bytes -= 64;
+		p += 64;
+		q += 64;
+		dp += 64;
+		dq += 64;
+	}
+
+	kernel_fpu_end();
+}
+
+static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila,
+				   void **ptrs)
+{
+	u8 *p, *q, *dq;
+	const u8 *qmul;		/* Q multiplier table */
+
+	p = (u8 *)ptrs[disks - 2];
+	q = (u8 *)ptrs[disks - 1];
+
+	/*
+	 * Compute syndrome with zero for the missing data page
+	 * Use the dead data page as temporary storage for delta q
+	 */
+	dq = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila] = dq;
+	ptrs[disks - 1] = q;
+
+	/* Now, pick the proper data tables */
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+	kernel_fpu_begin();
+
+	/* xr22, xr23: qmul */
+	asm volatile("vld $vr22, %0" : : "m" (qmul[0]));
+	asm volatile("xvreplve0.q $xr22, $xr22");
+	asm volatile("vld $vr23, %0" : : "m" (qmul[16]));
+	asm volatile("xvreplve0.q $xr23, $xr23");
+
+	while (bytes) {
+		/* xr0, xr1: P + Dx */
+		asm volatile("xvld $xr0, %0" : : "m" (p[0]));
+		asm volatile("xvld $xr1, %0" : : "m" (p[32]));
+		/* xr2, xr3: Qx */
+		asm volatile("xvld $xr2, %0" : : "m" (dq[0]));
+		asm volatile("xvld $xr3, %0" : : "m" (dq[32]));
+		/* xr2, xr3: Q + Qx */
+		asm volatile("xvld $xr4, %0" : : "m" (q[0]));
+		asm volatile("xvld $xr5, %0" : : "m" (q[32]));
+		asm volatile("xvxor.v $xr2, $xr2, $xr4");
+		asm volatile("xvxor.v $xr3, $xr3, $xr5");
+
+		/* xr4, xr5: higher 4 bits of each byte of (Q + Qx) */
+		asm volatile("xvsrli.b $xr4, $xr2, 4");
+		asm volatile("xvsrli.b $xr5, $xr3, 4");
+		/* xr2, xr3: lower 4 bits of each byte of (Q + Qx) */
+		asm volatile("xvandi.b $xr2, $xr2, 0x0f");
+		asm volatile("xvandi.b $xr3, $xr3, 0x0f");
+		/* lookup from qmul[0] */
+		asm volatile("xvshuf.b $xr2, $xr22, $xr22, $xr2");
+		asm volatile("xvshuf.b $xr3, $xr22, $xr22, $xr3");
+		/* lookup from qmul[16] */
+		asm volatile("xvshuf.b $xr4, $xr23, $xr23, $xr4");
+		asm volatile("xvshuf.b $xr5, $xr23, $xr23, $xr5");
+		/* xr2, xr3: qmul(Q + Qx) = Dx */
+		asm volatile("xvxor.v $xr2, $xr2, $xr4");
+		asm volatile("xvxor.v $xr3, $xr3, $xr5");
+
+		/* xr0, xr1: P + Dx + Dx = P */
+		asm volatile("xvxor.v $xr0, $xr0, $xr2");
+		asm volatile("xvxor.v $xr1, $xr1, $xr3");
+
+		asm volatile("xvst $xr2, %0" : "=m" (dq[0]));
+		asm volatile("xvst $xr3, %0" : "=m" (dq[32]));
+		asm volatile("xvst $xr0, %0" : "=m" (p[0]));
+		asm volatile("xvst $xr1, %0" : "=m" (p[32]));
+
+		bytes -= 64;
+		p += 64;
+		q += 64;
+		dq += 64;
+	}
+
+	kernel_fpu_end();
+}
+
+const struct raid6_recov_calls raid6_recov_lasx = {
+	.data2 = raid6_2data_recov_lasx,
+	.datap = raid6_datap_recov_lasx,
+	.name = "lasx",
+};
+#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/lib/raid/raid6/mktables.c b/lib/raid/raid6/mktables.c
new file mode 100644
index 0000000000000..b6327b562fdb4
--- /dev/null
+++ b/lib/raid/raid6/mktables.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved
+ *
+ * Make RAID-6 tables.  This is a host user space program to be run at compile
+ * time.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <stdlib.h>
+#include <time.h>
+
+static uint8_t gfmul(uint8_t a, uint8_t b)
+{
+	uint8_t v = 0;
+
+	while (b) {
+		if (b & 1)
+			v ^= a;
+		a = (a << 1) ^ (a & 0x80 ? 0x1d : 0);
+		b >>= 1;
+	}
+
+	return v;
+}
+
+static uint8_t gfpow(uint8_t a, int b)
+{
+	uint8_t v = 1;
+
+	b %= 255;
+	if (b < 0)
+		b += 255;
+
+	while (b) {
+		if (b & 1)
+			v = gfmul(v, a);
+		a = gfmul(a, a);
+		b >>= 1;
+	}
+
+	return v;
+}
+
+int main(int argc, char *argv[])
+{
+	int i, j, k;
+	uint8_t v;
+	uint8_t exptbl[256], invtbl[256];
+
+	printf("#include <linux/export.h>\n");
+	printf("#include \"algos.h\"\n");
+
+	/* Compute multiplication table */
+	printf("\nconst u8  __attribute__((aligned(256)))\n"
+		"raid6_gfmul[256][256] =\n"
+		"{\n");
+	for (i = 0; i < 256; i++) {
+		printf("\t{\n");
+		for (j = 0; j < 256; j += 8) {
+			printf("\t\t");
+			for (k = 0; k < 8; k++)
+				printf("0x%02x,%c", gfmul(i, j + k),
+				       (k == 7) ? '\n' : ' ');
+		}
+		printf("\t},\n");
+	}
+	printf("};\n");
+	printf("EXPORT_SYMBOL(raid6_gfmul);\n");
+
+	/* Compute vector multiplication table */
+	printf("\nconst u8  __attribute__((aligned(256)))\n"
+		"raid6_vgfmul[256][32] =\n"
+		"{\n");
+	for (i = 0; i < 256; i++) {
+		printf("\t{\n");
+		for (j = 0; j < 16; j += 8) {
+			printf("\t\t");
+			for (k = 0; k < 8; k++)
+				printf("0x%02x,%c", gfmul(i, j + k),
+				       (k == 7) ? '\n' : ' ');
+		}
+		for (j = 0; j < 16; j += 8) {
+			printf("\t\t");
+			for (k = 0; k < 8; k++)
+				printf("0x%02x,%c", gfmul(i, (j + k) << 4),
+				       (k == 7) ? '\n' : ' ');
+		}
+		printf("\t},\n");
+	}
+	printf("};\n");
+	printf("EXPORT_SYMBOL(raid6_vgfmul);\n");
+
+	/* Compute power-of-2 table (exponent) */
+	v = 1;
+	printf("\nconst u8 __attribute__((aligned(256)))\n"
+	       "raid6_gfexp[256] =\n" "{\n");
+	for (i = 0; i < 256; i += 8) {
+		printf("\t");
+		for (j = 0; j < 8; j++) {
+			exptbl[i + j] = v;
+			printf("0x%02x,%c", v, (j == 7) ? '\n' : ' ');
+			v = gfmul(v, 2);
+			if (v == 1)
+				v = 0;	/* For entry 255, not a real entry */
+		}
+	}
+	printf("};\n");
+	printf("EXPORT_SYMBOL(raid6_gfexp);\n");
+
+	/* Compute log-of-2 table */
+	printf("\nconst u8 __attribute__((aligned(256)))\n"
+	       "raid6_gflog[256] =\n" "{\n");
+	for (i = 0; i < 256; i += 8) {
+		printf("\t");
+		for (j = 0; j < 8; j++) {
+			v = 255;
+			for (k = 0; k < 256; k++)
+				if (exptbl[k] == (i + j)) {
+					v = k;
+					break;
+				}
+			printf("0x%02x,%c", v, (j == 7) ? '\n' : ' ');
+		}
+	}
+	printf("};\n");
+	printf("EXPORT_SYMBOL(raid6_gflog);\n");
+
+	/* Compute inverse table x^-1 == x^254 */
+	printf("\nconst u8 __attribute__((aligned(256)))\n"
+	       "raid6_gfinv[256] =\n" "{\n");
+	for (i = 0; i < 256; i += 8) {
+		printf("\t");
+		for (j = 0; j < 8; j++) {
+			invtbl[i + j] = v = gfpow(i + j, 254);
+			printf("0x%02x,%c", v, (j == 7) ? '\n' : ' ');
+		}
+	}
+	printf("};\n");
+	printf("EXPORT_SYMBOL(raid6_gfinv);\n");
+
+	/* Compute inv(2^x + 1) (exponent-xor-inverse) table */
+	printf("\nconst u8 __attribute__((aligned(256)))\n"
+	       "raid6_gfexi[256] =\n" "{\n");
+	for (i = 0; i < 256; i += 8) {
+		printf("\t");
+		for (j = 0; j < 8; j++)
+			printf("0x%02x,%c", invtbl[exptbl[i + j] ^ 1],
+			       (j == 7) ? '\n' : ' ');
+	}
+	printf("};\n");
+	printf("EXPORT_SYMBOL(raid6_gfexi);\n");
+
+	return 0;
+}
diff --git a/lib/raid/raid6/powerpc/altivec.uc b/lib/raid/raid6/powerpc/altivec.uc
new file mode 100644
index 0000000000000..c5429fb71dd6f
--- /dev/null
+++ b/lib/raid/raid6/powerpc/altivec.uc
@@ -0,0 +1,110 @@
+/* -*- linux-c -*- ------------------------------------------------------- *
+ *
+ *   Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
+ *
+ *   This program is free software; you can redistribute it and/or modify
+ *   it under the terms of the GNU General Public License as published by
+ *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
+ *   Boston MA 02111-1307, USA; either version 2 of the License, or
+ *   (at your option) any later version; incorporated herein by reference.
+ *
+ * ----------------------------------------------------------------------- */
+
+/*
+ * raid6altivec$#.c
+ *
+ * $#-way unrolled portable integer math RAID-6 instruction set
+ *
+ * This file is postprocessed using unroll.awk
+ *
+ * <benh> hpa: in process,
+ * you can just "steal" the vec unit with enable_kernel_altivec() (but
+ * bracked this with preempt_disable/enable or in a lock)
+ */
+
+#include "algos.h"
+
+#include <altivec.h>
+#include <asm/cputable.h>
+#include <asm/switch_to.h>
+
+/*
+ * This is the C data type to use.  We use a vector of
+ * signed char so vec_cmpgt() will generate the right
+ * instruction.
+ */
+
+typedef vector signed char unative_t;
+
+#define NBYTES(x) ((vector signed char) {x,x,x,x, x,x,x,x, x,x,x,x, x,x,x,x})
+#define NSIZE	sizeof(unative_t)
+
+/*
+ * The SHLBYTE() operation shifts each byte left by 1, *not*
+ * rolling over into the next byte
+ */
+static inline __attribute_const__ unative_t SHLBYTE(unative_t v)
+{
+	return vec_add(v,v);
+}
+
+/*
+ * The MASK() operation returns 0xFF in any byte for which the high
+ * bit is 1, 0x00 for any byte for which the high bit is 0.
+ */
+static inline __attribute_const__ unative_t MASK(unative_t v)
+{
+	unative_t zv = NBYTES(0);
+
+	/* vec_cmpgt returns a vector bool char; thus the need for the cast */
+	return (unative_t)vec_cmpgt(zv, v);
+}
+
+
+/* This is noinline to make damned sure that gcc doesn't move any of the
+   Altivec code around the enable/disable code */
+static void noinline
+raid6_altivec$#_gen_syndrome_real(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
+	unative_t x1d = NBYTES(0x1d);
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
+		wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
+		for ( z = z0-1 ; z >= 0 ; z-- ) {
+			wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			wp$$ = vec_xor(wp$$, wd$$);
+			w2$$ = MASK(wq$$);
+			w1$$ = SHLBYTE(wq$$);
+			w2$$ = vec_and(w2$$, x1d);
+			w1$$ = vec_xor(w1$$, w2$$);
+			wq$$ = vec_xor(w1$$, wd$$);
+		}
+		*(unative_t *)&p[d+NSIZE*$$] = wp$$;
+		*(unative_t *)&q[d+NSIZE*$$] = wq$$;
+	}
+}
+
+static void raid6_altivec$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	preempt_disable();
+	enable_kernel_altivec();
+
+	raid6_altivec$#_gen_syndrome_real(disks, bytes, ptrs);
+
+	disable_kernel_altivec();
+	preempt_enable();
+}
+
+const struct raid6_calls raid6_altivec$# = {
+	.gen_syndrome	= raid6_altivec$#_gen_syndrome,
+	.name		= "altivecx$#",
+};
diff --git a/lib/raid/raid6/powerpc/pq_arch.h b/lib/raid/raid6/powerpc/pq_arch.h
new file mode 100644
index 0000000000000..ea1878777ff29
--- /dev/null
+++ b/lib/raid/raid6/powerpc/pq_arch.h
@@ -0,0 +1,32 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/cputable.h>
+
+extern const struct raid6_calls raid6_altivec1;
+extern const struct raid6_calls raid6_altivec2;
+extern const struct raid6_calls raid6_altivec4;
+extern const struct raid6_calls raid6_altivec8;
+extern const struct raid6_calls raid6_vpermxor1;
+extern const struct raid6_calls raid6_vpermxor2;
+extern const struct raid6_calls raid6_vpermxor4;
+extern const struct raid6_calls raid6_vpermxor8;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+	raid6_algo_add_default();
+
+	/* This assumes either all CPUs have Altivec or none does */
+	if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
+		raid6_algo_add(&raid6_altivec1);
+		raid6_algo_add(&raid6_altivec2);
+		raid6_algo_add(&raid6_altivec4);
+		raid6_algo_add(&raid6_altivec8);
+	}
+	if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) &&
+	    cpu_has_feature(CPU_FTR_ARCH_207S)) {
+		raid6_algo_add(&raid6_vpermxor1);
+		raid6_algo_add(&raid6_vpermxor2);
+		raid6_algo_add(&raid6_vpermxor4);
+		raid6_algo_add(&raid6_vpermxor8);
+	}
+}
diff --git a/lib/raid/raid6/powerpc/vpermxor.uc b/lib/raid/raid6/powerpc/vpermxor.uc
new file mode 100644
index 0000000000000..e8964361aaefb
--- /dev/null
+++ b/lib/raid/raid6/powerpc/vpermxor.uc
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2017, Matt Brown, IBM Corp.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * vpermxor$#.c
+ *
+ * Based on H. Peter Anvin's paper - The mathematics of RAID-6
+ *
+ * $#-way unrolled portable integer math RAID-6 instruction set
+ * This file is postprocessed using unroll.awk
+ *
+ * vpermxor$#.c makes use of the vpermxor instruction to optimise the RAID6 Q
+ * syndrome calculations.
+ * This can be run on systems which have both Altivec and vpermxor instruction.
+ *
+ * This instruction was introduced in POWER8 - ISA v2.07.
+ */
+
+#include <altivec.h>
+#include <asm/ppc-opcode.h>
+#include <asm/cputable.h>
+#include <asm/switch_to.h>
+#include "algos.h"
+
+typedef vector unsigned char unative_t;
+#define NSIZE sizeof(unative_t)
+
+static const vector unsigned char gf_low = {0x1e, 0x1c, 0x1a, 0x18, 0x16, 0x14,
+					    0x12, 0x10, 0x0e, 0x0c, 0x0a, 0x08,
+					    0x06, 0x04, 0x02,0x00};
+static const vector unsigned char gf_high = {0xfd, 0xdd, 0xbd, 0x9d, 0x7d, 0x5d,
+					     0x3d, 0x1d, 0xe0, 0xc0, 0xa0, 0x80,
+					     0x60, 0x40, 0x20, 0x00};
+
+static void noinline raid6_vpermxor$#_gen_syndrome_real(int disks, size_t bytes,
+							void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+	unative_t wp$$, wq$$, wd$$;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	for (d = 0; d < bytes; d += NSIZE*$#) {
+		wp$$ = wq$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
+
+		for (z = z0-1; z>=0; z--) {
+			wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			/* P syndrome */
+			wp$$ = vec_xor(wp$$, wd$$);
+
+			/* Q syndrome */
+			asm(VPERMXOR(%0,%1,%2,%3):"=v"(wq$$):"v"(gf_high), "v"(gf_low), "v"(wq$$));
+			wq$$ = vec_xor(wq$$, wd$$);
+		}
+		*(unative_t *)&p[d+NSIZE*$$] = wp$$;
+		*(unative_t *)&q[d+NSIZE*$$] = wq$$;
+	}
+}
+
+static void raid6_vpermxor$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	preempt_disable();
+	enable_kernel_altivec();
+
+	raid6_vpermxor$#_gen_syndrome_real(disks, bytes, ptrs);
+
+	disable_kernel_altivec();
+	preempt_enable();
+}
+
+const struct raid6_calls raid6_vpermxor$# = {
+	.gen_syndrome	= raid6_vpermxor$#_gen_syndrome,
+	.name		= "vpermxor$#",
+};
diff --git a/lib/raid/raid6/recov.c b/lib/raid/raid6/recov.c
new file mode 100644
index 0000000000000..3fa53bc3fde4d
--- /dev/null
+++ b/lib/raid/raid6/recov.c
@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ * RAID-6 data recovery in dual failure mode.  In single failure mode, use the
+ * RAID-5 algorithm (or, in the case of Q failure, just reconstruct the
+ * syndrome.)
+ */
+
+#include <linux/mm.h>
+#include <linux/raid/pq.h>
+#include "algos.h"
+
+/* Recover two failed data blocks. */
+static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila,
+		int failb, void **ptrs)
+{
+	u8 *p, *q, *dp, *dq;
+	u8 px, qx, db;
+	const u8 *pbmul;	/* P multiplier table for B data */
+	const u8 *qmul;		/* Q multiplier table (for both) */
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/* Compute syndrome with zero for the missing data pages
+	   Use the dead data pages as temporary storage for
+	   delta p and delta q */
+	dp = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-2] = dp;
+	dq = (u8 *)ptrs[failb];
+	ptrs[failb] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dp;
+	ptrs[failb]   = dq;
+	ptrs[disks-2] = p;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	pbmul = raid6_gfmul[raid6_gfexi[failb-faila]];
+	qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]];
+
+	/* Now do it... */
+	while ( bytes-- ) {
+		px    = *p ^ *dp;
+		qx    = qmul[*q ^ *dq];
+		*dq++ = db = pbmul[px] ^ qx; /* Reconstructed B */
+		*dp++ = db ^ px; /* Reconstructed A */
+		p++; q++;
+	}
+}
+
+/* Recover failure of one data block plus the P block */
+static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila,
+		void **ptrs)
+{
+	u8 *p, *q, *dq;
+	const u8 *qmul;		/* Q multiplier table */
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/* Compute syndrome with zero for the missing data page
+	   Use the dead data page as temporary storage for delta q */
+	dq = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dq;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+	/* Now do it... */
+	while ( bytes-- ) {
+		*p++ ^= *dq = qmul[*q ^ *dq];
+		q++; dq++;
+	}
+}
+
+
+const struct raid6_recov_calls raid6_recov_intx1 = {
+	.data2 = raid6_2data_recov_intx1,
+	.datap = raid6_datap_recov_intx1,
+	.name = "intx1",
+};
diff --git a/lib/raid/raid6/riscv/pq_arch.h b/lib/raid/raid6/riscv/pq_arch.h
new file mode 100644
index 0000000000000..82f1a188f8c4f
--- /dev/null
+++ b/lib/raid/raid6/riscv/pq_arch.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/vector.h>
+
+extern const struct raid6_calls raid6_rvvx1;
+extern const struct raid6_calls raid6_rvvx2;
+extern const struct raid6_calls raid6_rvvx4;
+extern const struct raid6_calls raid6_rvvx8;
+extern const struct raid6_recov_calls raid6_recov_rvv;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+	raid6_algo_add_default();
+	if (has_vector()) {
+		raid6_algo_add(&raid6_rvvx1);
+		raid6_algo_add(&raid6_rvvx2);
+		raid6_algo_add(&raid6_rvvx4);
+		raid6_algo_add(&raid6_rvvx8);
+		raid6_recov_algo_add(&raid6_recov_rvv);
+	}
+}
diff --git a/lib/raid/raid6/riscv/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c
new file mode 100644
index 0000000000000..2305940276ddf
--- /dev/null
+++ b/lib/raid/raid6/riscv/recov_rvv.c
@@ -0,0 +1,222 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright 2024 Institute of Software, CAS.
+ * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
+ */
+
+#include <linux/mm.h>
+#include <linux/raid/pq.h>
+#include "algos.h"
+#include "rvv.h"
+
+static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp,
+				    u8 *dq, const u8 *pbmul,
+				    const u8 *qmul)
+{
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	x0, %[avl], e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : :
+		      [avl]"r"(16)
+	);
+
+	/*
+	 * while ( bytes-- ) {
+	 *	uint8_t px, qx, db;
+	 *
+	 *	px	  = *p ^ *dp;
+	 *	qx	  = qmul[*q ^ *dq];
+	 *	*dq++ = db = pbmul[px] ^ qx;
+	 *	*dp++ = db ^ px;
+	 *	p++; q++;
+	 * }
+	 */
+	while (bytes) {
+		/*
+		 * v0:px, v1:dp,
+		 * v2:qx, v3:dq,
+		 * v4:vx, v5:vy,
+		 * v6:qm0, v7:qm1,
+		 * v8:pm0, v9:pm1,
+		 * v14:p/qm[vx], v15:p/qm[vy]
+		 */
+		asm volatile (".option		push\n"
+			      ".option		arch,+v\n"
+			      "vle8.v		v0, (%[px])\n"
+			      "vle8.v		v1, (%[dp])\n"
+			      "vxor.vv		v0, v0, v1\n"
+			      "vle8.v		v2, (%[qx])\n"
+			      "vle8.v		v3, (%[dq])\n"
+			      "vxor.vv		v4, v2, v3\n"
+			      "vsrl.vi		v5, v4, 4\n"
+			      "vand.vi		v4, v4, 0xf\n"
+			      "vle8.v		v6, (%[qm0])\n"
+			      "vle8.v		v7, (%[qm1])\n"
+			      "vrgather.vv	v14, v6, v4\n" /* v14 = qm[vx] */
+			      "vrgather.vv	v15, v7, v5\n" /* v15 = qm[vy] */
+			      "vxor.vv		v2, v14, v15\n" /* v2 = qmul[*q ^ *dq] */
+
+			      "vsrl.vi		v5, v0, 4\n"
+			      "vand.vi		v4, v0, 0xf\n"
+			      "vle8.v		v8, (%[pm0])\n"
+			      "vle8.v		v9, (%[pm1])\n"
+			      "vrgather.vv	v14, v8, v4\n" /* v14 = pm[vx] */
+			      "vrgather.vv	v15, v9, v5\n" /* v15 = pm[vy] */
+			      "vxor.vv		v4, v14, v15\n" /* v4 = pbmul[px] */
+			      "vxor.vv		v3, v4, v2\n" /* v3 = db = pbmul[px] ^ qx */
+			      "vxor.vv		v1, v3, v0\n" /* v1 = db ^ px; */
+			      "vse8.v		v3, (%[dq])\n"
+			      "vse8.v		v1, (%[dp])\n"
+			      ".option		pop\n"
+			      : :
+			      [px]"r"(p),
+			      [dp]"r"(dp),
+			      [qx]"r"(q),
+			      [dq]"r"(dq),
+			      [qm0]"r"(qmul),
+			      [qm1]"r"(qmul + 16),
+			      [pm0]"r"(pbmul),
+			      [pm1]"r"(pbmul + 16)
+			      :);
+
+		bytes -= 16;
+		p += 16;
+		q += 16;
+		dp += 16;
+		dq += 16;
+	}
+}
+
+static void __raid6_datap_recov_rvv(int bytes, u8 *p, u8 *q,
+				    u8 *dq, const u8 *qmul)
+{
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	x0, %[avl], e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : :
+		      [avl]"r"(16)
+	);
+
+	/*
+	 * while (bytes--) {
+	 *  *p++ ^= *dq = qmul[*q ^ *dq];
+	 *  q++; dq++;
+	 * }
+	 */
+	while (bytes) {
+		/*
+		 * v0:vx, v1:vy,
+		 * v2:dq, v3:p,
+		 * v4:qm0, v5:qm1,
+		 * v10:m[vx], v11:m[vy]
+		 */
+		asm volatile (".option		push\n"
+			      ".option		arch,+v\n"
+			      "vle8.v		v0, (%[vx])\n"
+			      "vle8.v		v2, (%[dq])\n"
+			      "vxor.vv		v0, v0, v2\n"
+			      "vsrl.vi		v1, v0, 4\n"
+			      "vand.vi		v0, v0, 0xf\n"
+			      "vle8.v		v4, (%[qm0])\n"
+			      "vle8.v		v5, (%[qm1])\n"
+			      "vrgather.vv	v10, v4, v0\n"
+			      "vrgather.vv	v11, v5, v1\n"
+			      "vxor.vv		v0, v10, v11\n"
+			      "vle8.v		v1, (%[vy])\n"
+			      "vxor.vv		v1, v0, v1\n"
+			      "vse8.v		v0, (%[dq])\n"
+			      "vse8.v		v1, (%[vy])\n"
+			      ".option		pop\n"
+			      : :
+			      [vx]"r"(q),
+			      [vy]"r"(p),
+			      [dq]"r"(dq),
+			      [qm0]"r"(qmul),
+			      [qm1]"r"(qmul + 16)
+			      :);
+
+		bytes -= 16;
+		p += 16;
+		q += 16;
+		dq += 16;
+	}
+}
+
+static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila,
+				  int failb, void **ptrs)
+{
+	u8 *p, *q, *dp, *dq;
+	const u8 *pbmul;	/* P multiplier table for B data */
+	const u8 *qmul;		/* Q multiplier table (for both) */
+
+	p = (u8 *)ptrs[disks - 2];
+	q = (u8 *)ptrs[disks - 1];
+
+	/*
+	 * Compute syndrome with zero for the missing data pages
+	 * Use the dead data pages as temporary storage for
+	 * delta p and delta q
+	 */
+	dp = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 2] = dp;
+	dq = (u8 *)ptrs[failb];
+	ptrs[failb] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]     = dp;
+	ptrs[failb]     = dq;
+	ptrs[disks - 2] = p;
+	ptrs[disks - 1] = q;
+
+	/* Now, pick the proper data tables */
+	pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]];
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
+					 raid6_gfexp[failb]]];
+
+	kernel_vector_begin();
+	__raid6_2data_recov_rvv(bytes, p, q, dp, dq, pbmul, qmul);
+	kernel_vector_end();
+}
+
+static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila,
+				  void **ptrs)
+{
+	u8 *p, *q, *dq;
+	const u8 *qmul;		/* Q multiplier table */
+
+	p = (u8 *)ptrs[disks - 2];
+	q = (u8 *)ptrs[disks - 1];
+
+	/*
+	 * Compute syndrome with zero for the missing data page
+	 * Use the dead data page as temporary storage for delta q
+	 */
+	dq = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks - 1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]     = dq;
+	ptrs[disks - 1] = q;
+
+	/* Now, pick the proper data tables */
+	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+	kernel_vector_begin();
+	__raid6_datap_recov_rvv(bytes, p, q, dq, qmul);
+	kernel_vector_end();
+}
+
+const struct raid6_recov_calls raid6_recov_rvv = {
+	.data2		= raid6_2data_recov_rvv,
+	.datap		= raid6_datap_recov_rvv,
+	.name		= "rvv",
+};
diff --git a/lib/raid/raid6/riscv/rvv.c b/lib/raid/raid6/riscv/rvv.c
new file mode 100644
index 0000000000000..75c9dafedb284
--- /dev/null
+++ b/lib/raid/raid6/riscv/rvv.c
@@ -0,0 +1,1228 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RAID-6 syndrome calculation using RISC-V vector instructions
+ *
+ * Copyright 2024 Institute of Software, CAS.
+ * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
+ *
+ * Based on neon.uc:
+ *	Copyright 2002-2004 H. Peter Anvin
+ */
+
+#include "rvv.h"
+
+#ifdef __riscv_vector
+#error "This code must be built without compiler support for vector"
+#endif
+
+static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	unsigned long vl, d, nsize;
+	int z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0 + 1];		/* XOR parity */
+	q = dptr[z0 + 2];		/* RS syndrome */
+
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : "=&r" (vl)
+	);
+
+	nsize = vl;
+
+	 /* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
+	for (d = 0; d < bytes; d += nsize * 1) {
+		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v0, (%[wp0])\n"
+			      "vmv.v.v	v1, v0\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize])
+		);
+
+		for (z = z0 - 1 ; z >= 0 ; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * w1$$ ^= w2$$;
+			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			 * wq$$ = w1$$ ^ wd$$;
+			 * wp$$ ^= wd$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v3, v3, v2\n"
+				      "vle8.v	v2, (%[wd0])\n"
+				      "vxor.vv	v1, v3, v2\n"
+				      "vxor.vv	v0, v0, v2\n"
+				      ".option	pop\n"
+				      : :
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
+		 */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vse8.v	v0, (%[wp0])\n"
+			      "vse8.v	v1, (%[wq0])\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0])
+		);
+	}
+}
+
+static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
+					 unsigned long bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	unsigned long vl, d, nsize;
+	int z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks - 2];	/* XOR parity */
+	q = dptr[disks - 1];	/* RS syndrome */
+
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : "=&r" (vl)
+	);
+
+	nsize = vl;
+
+	/* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
+	for (d = 0 ; d < bytes ; d += nsize * 1) {
+		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v0, (%[wp0])\n"
+			      "vmv.v.v	v1, v0\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize])
+		);
+
+		/* P/Q data pages */
+		for (z = z0 - 1; z >= start; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * w1$$ ^= w2$$;
+			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			 * wq$$ = w1$$ ^ wd$$;
+			 * wp$$ ^= wd$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v3, v3, v2\n"
+				      "vle8.v	v2, (%[wd0])\n"
+				      "vxor.vv	v1, v3, v2\n"
+				      "vxor.vv	v0, v0, v2\n"
+				      ".option	pop\n"
+				      : :
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/* P/Q left side optimization */
+		for (z = start - 1; z >= 0; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * wq$$ = w1$$ ^ w2$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v1, v3, v2\n"
+				      ".option	pop\n"
+				      : :
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
+		 * v0:wp0, v1:wq0, v2:p0, v3:q0
+		 */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v2, (%[wp0])\n"
+			      "vle8.v	v3, (%[wq0])\n"
+			      "vxor.vv	v2, v2, v0\n"
+			      "vxor.vv	v3, v3, v1\n"
+			      "vse8.v	v2, (%[wp0])\n"
+			      "vse8.v	v3, (%[wq0])\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0])
+		);
+	}
+}
+
+static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	unsigned long vl, d, nsize;
+	int z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0 + 1];		/* XOR parity */
+	q = dptr[z0 + 2];		/* RS syndrome */
+
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : "=&r" (vl)
+	);
+
+	nsize = vl;
+
+	/*
+	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
+	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
+	 */
+	for (d = 0; d < bytes; d += nsize * 2) {
+		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v0, (%[wp0])\n"
+			      "vmv.v.v	v1, v0\n"
+			      "vle8.v	v4, (%[wp1])\n"
+			      "vmv.v.v	v5, v4\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize])
+		);
+
+		for (z = z0 - 1; z >= 0; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * w1$$ ^= w2$$;
+			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			 * wq$$ = w1$$ ^ wd$$;
+			 * wp$$ ^= wd$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v3, v3, v2\n"
+				      "vle8.v	v2, (%[wd0])\n"
+				      "vxor.vv	v1, v3, v2\n"
+				      "vxor.vv	v0, v0, v2\n"
+
+				      "vsra.vi	v6, v5, 7\n"
+				      "vsll.vi	v7, v5, 1\n"
+				      "vand.vx	v6, v6, %[x1d]\n"
+				      "vxor.vv	v7, v7, v6\n"
+				      "vle8.v	v6, (%[wd1])\n"
+				      "vxor.vv	v5, v7, v6\n"
+				      "vxor.vv	v4, v4, v6\n"
+				      ".option	pop\n"
+				      : :
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
+		 */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vse8.v	v0, (%[wp0])\n"
+			      "vse8.v	v1, (%[wq0])\n"
+			      "vse8.v	v4, (%[wp1])\n"
+			      "vse8.v	v5, (%[wq1])\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1])
+		);
+	}
+}
+
+static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
+					 unsigned long bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	unsigned long vl, d, nsize;
+	int z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks - 2];	/* XOR parity */
+	q = dptr[disks - 1];	/* RS syndrome */
+
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : "=&r" (vl)
+	);
+
+	nsize = vl;
+
+	/*
+	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
+	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
+	 */
+	for (d = 0; d < bytes; d += nsize * 2) {
+		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v0, (%[wp0])\n"
+			      "vmv.v.v	v1, v0\n"
+			      "vle8.v	v4, (%[wp1])\n"
+			      "vmv.v.v	v5, v4\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize])
+		);
+
+		/* P/Q data pages */
+		for (z = z0 - 1; z >= start; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * w1$$ ^= w2$$;
+			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			 * wq$$ = w1$$ ^ wd$$;
+			 * wp$$ ^= wd$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v3, v3, v2\n"
+				      "vle8.v	v2, (%[wd0])\n"
+				      "vxor.vv	v1, v3, v2\n"
+				      "vxor.vv	v0, v0, v2\n"
+
+				      "vsra.vi	v6, v5, 7\n"
+				      "vsll.vi	v7, v5, 1\n"
+				      "vand.vx	v6, v6, %[x1d]\n"
+				      "vxor.vv	v7, v7, v6\n"
+				      "vle8.v	v6, (%[wd1])\n"
+				      "vxor.vv	v5, v7, v6\n"
+				      "vxor.vv	v4, v4, v6\n"
+				      ".option	pop\n"
+				      : :
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/* P/Q left side optimization */
+		for (z = start - 1; z >= 0; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * wq$$ = w1$$ ^ w2$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v1, v3, v2\n"
+
+				      "vsra.vi	v6, v5, 7\n"
+				      "vsll.vi	v7, v5, 1\n"
+				      "vand.vx	v6, v6, %[x1d]\n"
+				      "vxor.vv	v5, v7, v6\n"
+				      ".option	pop\n"
+				      : :
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
+		 * v0:wp0, v1:wq0, v2:p0, v3:q0
+		 * v4:wp1, v5:wq1, v6:p1, v7:q1
+		 */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v2, (%[wp0])\n"
+			      "vle8.v	v3, (%[wq0])\n"
+			      "vxor.vv	v2, v2, v0\n"
+			      "vxor.vv	v3, v3, v1\n"
+			      "vse8.v	v2, (%[wp0])\n"
+			      "vse8.v	v3, (%[wq0])\n"
+
+			      "vle8.v	v6, (%[wp1])\n"
+			      "vle8.v	v7, (%[wq1])\n"
+			      "vxor.vv	v6, v6, v4\n"
+			      "vxor.vv	v7, v7, v5\n"
+			      "vse8.v	v6, (%[wp1])\n"
+			      "vse8.v	v7, (%[wq1])\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1])
+		);
+	}
+}
+
+static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	unsigned long vl, d, nsize;
+	int z, z0;
+
+	z0 = disks - 3;	/* Highest data disk */
+	p = dptr[z0 + 1];	/* XOR parity */
+	q = dptr[z0 + 2];	/* RS syndrome */
+
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : "=&r" (vl)
+	);
+
+	nsize = vl;
+
+	/*
+	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
+	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
+	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
+	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
+	 */
+	for (d = 0; d < bytes; d += nsize * 4) {
+		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v0, (%[wp0])\n"
+			      "vmv.v.v	v1, v0\n"
+			      "vle8.v	v4, (%[wp1])\n"
+			      "vmv.v.v	v5, v4\n"
+			      "vle8.v	v8, (%[wp2])\n"
+			      "vmv.v.v	v9, v8\n"
+			      "vle8.v	v12, (%[wp3])\n"
+			      "vmv.v.v	v13, v12\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+			      [wp3]"r"(&dptr[z0][d + 3 * nsize])
+		);
+
+		for (z = z0 - 1; z >= 0; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * w1$$ ^= w2$$;
+			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			 * wq$$ = w1$$ ^ wd$$;
+			 * wp$$ ^= wd$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v3, v3, v2\n"
+				      "vle8.v	v2, (%[wd0])\n"
+				      "vxor.vv	v1, v3, v2\n"
+				      "vxor.vv	v0, v0, v2\n"
+
+				      "vsra.vi	v6, v5, 7\n"
+				      "vsll.vi	v7, v5, 1\n"
+				      "vand.vx	v6, v6, %[x1d]\n"
+				      "vxor.vv	v7, v7, v6\n"
+				      "vle8.v	v6, (%[wd1])\n"
+				      "vxor.vv	v5, v7, v6\n"
+				      "vxor.vv	v4, v4, v6\n"
+
+				      "vsra.vi	v10, v9, 7\n"
+				      "vsll.vi	v11, v9, 1\n"
+				      "vand.vx	v10, v10, %[x1d]\n"
+				      "vxor.vv	v11, v11, v10\n"
+				      "vle8.v	v10, (%[wd2])\n"
+				      "vxor.vv	v9, v11, v10\n"
+				      "vxor.vv	v8, v8, v10\n"
+
+				      "vsra.vi	v14, v13, 7\n"
+				      "vsll.vi	v15, v13, 1\n"
+				      "vand.vx	v14, v14, %[x1d]\n"
+				      "vxor.vv	v15, v15, v14\n"
+				      "vle8.v	v14, (%[wd3])\n"
+				      "vxor.vv	v13, v15, v14\n"
+				      "vxor.vv	v12, v12, v14\n"
+				      ".option	pop\n"
+				      : :
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
+				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
+		 */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vse8.v	v0, (%[wp0])\n"
+			      "vse8.v	v1, (%[wq0])\n"
+			      "vse8.v	v4, (%[wp1])\n"
+			      "vse8.v	v5, (%[wq1])\n"
+			      "vse8.v	v8, (%[wp2])\n"
+			      "vse8.v	v9, (%[wq2])\n"
+			      "vse8.v	v12, (%[wp3])\n"
+			      "vse8.v	v13, (%[wq3])\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1]),
+			      [wp2]"r"(&p[d + nsize * 2]),
+			      [wq2]"r"(&q[d + nsize * 2]),
+			      [wp3]"r"(&p[d + nsize * 3]),
+			      [wq3]"r"(&q[d + nsize * 3])
+		);
+	}
+}
+
+static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
+					 unsigned long bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	unsigned long vl, d, nsize;
+	int z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks - 2];	/* XOR parity */
+	q = dptr[disks - 1];	/* RS syndrome */
+
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : "=&r" (vl)
+	);
+
+	nsize = vl;
+
+	/*
+	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
+	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
+	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
+	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
+	 */
+	for (d = 0; d < bytes; d += nsize * 4) {
+		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v0, (%[wp0])\n"
+			      "vmv.v.v	v1, v0\n"
+			      "vle8.v	v4, (%[wp1])\n"
+			      "vmv.v.v	v5, v4\n"
+			      "vle8.v	v8, (%[wp2])\n"
+			      "vmv.v.v	v9, v8\n"
+			      "vle8.v	v12, (%[wp3])\n"
+			      "vmv.v.v	v13, v12\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+			      [wp3]"r"(&dptr[z0][d + 3 * nsize])
+		);
+
+		/* P/Q data pages */
+		for (z = z0 - 1; z >= start; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * w1$$ ^= w2$$;
+			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			 * wq$$ = w1$$ ^ wd$$;
+			 * wp$$ ^= wd$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v3, v3, v2\n"
+				      "vle8.v	v2, (%[wd0])\n"
+				      "vxor.vv	v1, v3, v2\n"
+				      "vxor.vv	v0, v0, v2\n"
+
+				      "vsra.vi	v6, v5, 7\n"
+				      "vsll.vi	v7, v5, 1\n"
+				      "vand.vx	v6, v6, %[x1d]\n"
+				      "vxor.vv	v7, v7, v6\n"
+				      "vle8.v	v6, (%[wd1])\n"
+				      "vxor.vv	v5, v7, v6\n"
+				      "vxor.vv	v4, v4, v6\n"
+
+				      "vsra.vi	v10, v9, 7\n"
+				      "vsll.vi	v11, v9, 1\n"
+				      "vand.vx	v10, v10, %[x1d]\n"
+				      "vxor.vv	v11, v11, v10\n"
+				      "vle8.v	v10, (%[wd2])\n"
+				      "vxor.vv	v9, v11, v10\n"
+				      "vxor.vv	v8, v8, v10\n"
+
+				      "vsra.vi	v14, v13, 7\n"
+				      "vsll.vi	v15, v13, 1\n"
+				      "vand.vx	v14, v14, %[x1d]\n"
+				      "vxor.vv	v15, v15, v14\n"
+				      "vle8.v	v14, (%[wd3])\n"
+				      "vxor.vv	v13, v15, v14\n"
+				      "vxor.vv	v12, v12, v14\n"
+				      ".option	pop\n"
+				      : :
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
+				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/* P/Q left side optimization */
+		for (z = start - 1; z >= 0; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * wq$$ = w1$$ ^ w2$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v1, v3, v2\n"
+
+				      "vsra.vi	v6, v5, 7\n"
+				      "vsll.vi	v7, v5, 1\n"
+				      "vand.vx	v6, v6, %[x1d]\n"
+				      "vxor.vv	v5, v7, v6\n"
+
+				      "vsra.vi	v10, v9, 7\n"
+				      "vsll.vi	v11, v9, 1\n"
+				      "vand.vx	v10, v10, %[x1d]\n"
+				      "vxor.vv	v9, v11, v10\n"
+
+				      "vsra.vi	v14, v13, 7\n"
+				      "vsll.vi	v15, v13, 1\n"
+				      "vand.vx	v14, v14, %[x1d]\n"
+				      "vxor.vv	v13, v15, v14\n"
+				      ".option	pop\n"
+				      : :
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
+		 * v0:wp0, v1:wq0, v2:p0, v3:q0
+		 * v4:wp1, v5:wq1, v6:p1, v7:q1
+		 * v8:wp2, v9:wq2, v10:p2, v11:q2
+		 * v12:wp3, v13:wq3, v14:p3, v15:q3
+		 */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v2, (%[wp0])\n"
+			      "vle8.v	v3, (%[wq0])\n"
+			      "vxor.vv	v2, v2, v0\n"
+			      "vxor.vv	v3, v3, v1\n"
+			      "vse8.v	v2, (%[wp0])\n"
+			      "vse8.v	v3, (%[wq0])\n"
+
+			      "vle8.v	v6, (%[wp1])\n"
+			      "vle8.v	v7, (%[wq1])\n"
+			      "vxor.vv	v6, v6, v4\n"
+			      "vxor.vv	v7, v7, v5\n"
+			      "vse8.v	v6, (%[wp1])\n"
+			      "vse8.v	v7, (%[wq1])\n"
+
+			      "vle8.v	v10, (%[wp2])\n"
+			      "vle8.v	v11, (%[wq2])\n"
+			      "vxor.vv	v10, v10, v8\n"
+			      "vxor.vv	v11, v11, v9\n"
+			      "vse8.v	v10, (%[wp2])\n"
+			      "vse8.v	v11, (%[wq2])\n"
+
+			      "vle8.v	v14, (%[wp3])\n"
+			      "vle8.v	v15, (%[wq3])\n"
+			      "vxor.vv	v14, v14, v12\n"
+			      "vxor.vv	v15, v15, v13\n"
+			      "vse8.v	v14, (%[wp3])\n"
+			      "vse8.v	v15, (%[wq3])\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1]),
+			      [wp2]"r"(&p[d + nsize * 2]),
+			      [wq2]"r"(&q[d + nsize * 2]),
+			      [wp3]"r"(&p[d + nsize * 3]),
+			      [wq3]"r"(&q[d + nsize * 3])
+		);
+	}
+}
+
+static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	unsigned long vl, d, nsize;
+	int z, z0;
+
+	z0 = disks - 3;	/* Highest data disk */
+	p = dptr[z0 + 1];	/* XOR parity */
+	q = dptr[z0 + 2];	/* RS syndrome */
+
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : "=&r" (vl)
+	);
+
+	nsize = vl;
+
+	/*
+	 * v0:wp0,   v1:wq0,  v2:wd0/w20,  v3:w10
+	 * v4:wp1,   v5:wq1,  v6:wd1/w21,  v7:w11
+	 * v8:wp2,   v9:wq2, v10:wd2/w22, v11:w12
+	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
+	 * v16:wp4, v17:wq4, v18:wd4/w24, v19:w14
+	 * v20:wp5, v21:wq5, v22:wd5/w25, v23:w15
+	 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
+	 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
+	 */
+	for (d = 0; d < bytes; d += nsize * 8) {
+		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v0, (%[wp0])\n"
+			      "vmv.v.v	v1, v0\n"
+			      "vle8.v	v4, (%[wp1])\n"
+			      "vmv.v.v	v5, v4\n"
+			      "vle8.v	v8, (%[wp2])\n"
+			      "vmv.v.v	v9, v8\n"
+			      "vle8.v	v12, (%[wp3])\n"
+			      "vmv.v.v	v13, v12\n"
+			      "vle8.v	v16, (%[wp4])\n"
+			      "vmv.v.v	v17, v16\n"
+			      "vle8.v	v20, (%[wp5])\n"
+			      "vmv.v.v	v21, v20\n"
+			      "vle8.v	v24, (%[wp6])\n"
+			      "vmv.v.v	v25, v24\n"
+			      "vle8.v	v28, (%[wp7])\n"
+			      "vmv.v.v	v29, v28\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+			      [wp3]"r"(&dptr[z0][d + 3 * nsize]),
+			      [wp4]"r"(&dptr[z0][d + 4 * nsize]),
+			      [wp5]"r"(&dptr[z0][d + 5 * nsize]),
+			      [wp6]"r"(&dptr[z0][d + 6 * nsize]),
+			      [wp7]"r"(&dptr[z0][d + 7 * nsize])
+		);
+
+		for (z = z0 - 1; z >= 0; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * w1$$ ^= w2$$;
+			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			 * wq$$ = w1$$ ^ wd$$;
+			 * wp$$ ^= wd$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v3, v3, v2\n"
+				      "vle8.v	v2, (%[wd0])\n"
+				      "vxor.vv	v1, v3, v2\n"
+				      "vxor.vv	v0, v0, v2\n"
+
+				      "vsra.vi	v6, v5, 7\n"
+				      "vsll.vi	v7, v5, 1\n"
+				      "vand.vx	v6, v6, %[x1d]\n"
+				      "vxor.vv	v7, v7, v6\n"
+				      "vle8.v	v6, (%[wd1])\n"
+				      "vxor.vv	v5, v7, v6\n"
+				      "vxor.vv	v4, v4, v6\n"
+
+				      "vsra.vi	v10, v9, 7\n"
+				      "vsll.vi	v11, v9, 1\n"
+				      "vand.vx	v10, v10, %[x1d]\n"
+				      "vxor.vv	v11, v11, v10\n"
+				      "vle8.v	v10, (%[wd2])\n"
+				      "vxor.vv	v9, v11, v10\n"
+				      "vxor.vv	v8, v8, v10\n"
+
+				      "vsra.vi	v14, v13, 7\n"
+				      "vsll.vi	v15, v13, 1\n"
+				      "vand.vx	v14, v14, %[x1d]\n"
+				      "vxor.vv	v15, v15, v14\n"
+				      "vle8.v	v14, (%[wd3])\n"
+				      "vxor.vv	v13, v15, v14\n"
+				      "vxor.vv	v12, v12, v14\n"
+
+				      "vsra.vi	v18, v17, 7\n"
+				      "vsll.vi	v19, v17, 1\n"
+				      "vand.vx	v18, v18, %[x1d]\n"
+				      "vxor.vv	v19, v19, v18\n"
+				      "vle8.v	v18, (%[wd4])\n"
+				      "vxor.vv	v17, v19, v18\n"
+				      "vxor.vv	v16, v16, v18\n"
+
+				      "vsra.vi	v22, v21, 7\n"
+				      "vsll.vi	v23, v21, 1\n"
+				      "vand.vx	v22, v22, %[x1d]\n"
+				      "vxor.vv	v23, v23, v22\n"
+				      "vle8.v	v22, (%[wd5])\n"
+				      "vxor.vv	v21, v23, v22\n"
+				      "vxor.vv	v20, v20, v22\n"
+
+				      "vsra.vi	v26, v25, 7\n"
+				      "vsll.vi	v27, v25, 1\n"
+				      "vand.vx	v26, v26, %[x1d]\n"
+				      "vxor.vv	v27, v27, v26\n"
+				      "vle8.v	v26, (%[wd6])\n"
+				      "vxor.vv	v25, v27, v26\n"
+				      "vxor.vv	v24, v24, v26\n"
+
+				      "vsra.vi	v30, v29, 7\n"
+				      "vsll.vi	v31, v29, 1\n"
+				      "vand.vx	v30, v30, %[x1d]\n"
+				      "vxor.vv	v31, v31, v30\n"
+				      "vle8.v	v30, (%[wd7])\n"
+				      "vxor.vv	v29, v31, v30\n"
+				      "vxor.vv	v28, v28, v30\n"
+				      ".option	pop\n"
+				      : :
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
+				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
+				      [wd4]"r"(&dptr[z][d + 4 * nsize]),
+				      [wd5]"r"(&dptr[z][d + 5 * nsize]),
+				      [wd6]"r"(&dptr[z][d + 6 * nsize]),
+				      [wd7]"r"(&dptr[z][d + 7 * nsize]),
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
+		 */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vse8.v	v0, (%[wp0])\n"
+			      "vse8.v	v1, (%[wq0])\n"
+			      "vse8.v	v4, (%[wp1])\n"
+			      "vse8.v	v5, (%[wq1])\n"
+			      "vse8.v	v8, (%[wp2])\n"
+			      "vse8.v	v9, (%[wq2])\n"
+			      "vse8.v	v12, (%[wp3])\n"
+			      "vse8.v	v13, (%[wq3])\n"
+			      "vse8.v	v16, (%[wp4])\n"
+			      "vse8.v	v17, (%[wq4])\n"
+			      "vse8.v	v20, (%[wp5])\n"
+			      "vse8.v	v21, (%[wq5])\n"
+			      "vse8.v	v24, (%[wp6])\n"
+			      "vse8.v	v25, (%[wq6])\n"
+			      "vse8.v	v28, (%[wp7])\n"
+			      "vse8.v	v29, (%[wq7])\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1]),
+			      [wp2]"r"(&p[d + nsize * 2]),
+			      [wq2]"r"(&q[d + nsize * 2]),
+			      [wp3]"r"(&p[d + nsize * 3]),
+			      [wq3]"r"(&q[d + nsize * 3]),
+			      [wp4]"r"(&p[d + nsize * 4]),
+			      [wq4]"r"(&q[d + nsize * 4]),
+			      [wp5]"r"(&p[d + nsize * 5]),
+			      [wq5]"r"(&q[d + nsize * 5]),
+			      [wp6]"r"(&p[d + nsize * 6]),
+			      [wq6]"r"(&q[d + nsize * 6]),
+			      [wp7]"r"(&p[d + nsize * 7]),
+			      [wq7]"r"(&q[d + nsize * 7])
+		);
+	}
+}
+
+static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
+					 unsigned long bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	unsigned long vl, d, nsize;
+	int z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks - 2];	/* XOR parity */
+	q = dptr[disks - 1];	/* RS syndrome */
+
+	asm volatile (".option	push\n"
+		      ".option	arch,+v\n"
+		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
+		      ".option	pop\n"
+		      : "=&r" (vl)
+	);
+
+	nsize = vl;
+
+	/*
+	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
+	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
+	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
+	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
+	 * v16:wp4, v17:wq4, v18:wd4/w24, v19:w14
+	 * v20:wp5, v21:wq5, v22:wd5/w25, v23:w15
+	 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
+	 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
+	 */
+	for (d = 0; d < bytes; d += nsize * 8) {
+		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v0, (%[wp0])\n"
+			      "vmv.v.v	v1, v0\n"
+			      "vle8.v	v4, (%[wp1])\n"
+			      "vmv.v.v	v5, v4\n"
+			      "vle8.v	v8, (%[wp2])\n"
+			      "vmv.v.v	v9, v8\n"
+			      "vle8.v	v12, (%[wp3])\n"
+			      "vmv.v.v	v13, v12\n"
+			      "vle8.v	v16, (%[wp4])\n"
+			      "vmv.v.v	v17, v16\n"
+			      "vle8.v	v20, (%[wp5])\n"
+			      "vmv.v.v	v21, v20\n"
+			      "vle8.v	v24, (%[wp6])\n"
+			      "vmv.v.v	v25, v24\n"
+			      "vle8.v	v28, (%[wp7])\n"
+			      "vmv.v.v	v29, v28\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+			      [wp3]"r"(&dptr[z0][d + 3 * nsize]),
+			      [wp4]"r"(&dptr[z0][d + 4 * nsize]),
+			      [wp5]"r"(&dptr[z0][d + 5 * nsize]),
+			      [wp6]"r"(&dptr[z0][d + 6 * nsize]),
+			      [wp7]"r"(&dptr[z0][d + 7 * nsize])
+		);
+
+		/* P/Q data pages */
+		for (z = z0 - 1; z >= start; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * w1$$ ^= w2$$;
+			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
+			 * wq$$ = w1$$ ^ wd$$;
+			 * wp$$ ^= wd$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v3, v3, v2\n"
+				      "vle8.v	v2, (%[wd0])\n"
+				      "vxor.vv	v1, v3, v2\n"
+				      "vxor.vv	v0, v0, v2\n"
+
+				      "vsra.vi	v6, v5, 7\n"
+				      "vsll.vi	v7, v5, 1\n"
+				      "vand.vx	v6, v6, %[x1d]\n"
+				      "vxor.vv	v7, v7, v6\n"
+				      "vle8.v	v6, (%[wd1])\n"
+				      "vxor.vv	v5, v7, v6\n"
+				      "vxor.vv	v4, v4, v6\n"
+
+				      "vsra.vi	v10, v9, 7\n"
+				      "vsll.vi	v11, v9, 1\n"
+				      "vand.vx	v10, v10, %[x1d]\n"
+				      "vxor.vv	v11, v11, v10\n"
+				      "vle8.v	v10, (%[wd2])\n"
+				      "vxor.vv	v9, v11, v10\n"
+				      "vxor.vv	v8, v8, v10\n"
+
+				      "vsra.vi	v14, v13, 7\n"
+				      "vsll.vi	v15, v13, 1\n"
+				      "vand.vx	v14, v14, %[x1d]\n"
+				      "vxor.vv	v15, v15, v14\n"
+				      "vle8.v	v14, (%[wd3])\n"
+				      "vxor.vv	v13, v15, v14\n"
+				      "vxor.vv	v12, v12, v14\n"
+
+				      "vsra.vi	v18, v17, 7\n"
+				      "vsll.vi	v19, v17, 1\n"
+				      "vand.vx	v18, v18, %[x1d]\n"
+				      "vxor.vv	v19, v19, v18\n"
+				      "vle8.v	v18, (%[wd4])\n"
+				      "vxor.vv	v17, v19, v18\n"
+				      "vxor.vv	v16, v16, v18\n"
+
+				      "vsra.vi	v22, v21, 7\n"
+				      "vsll.vi	v23, v21, 1\n"
+				      "vand.vx	v22, v22, %[x1d]\n"
+				      "vxor.vv	v23, v23, v22\n"
+				      "vle8.v	v22, (%[wd5])\n"
+				      "vxor.vv	v21, v23, v22\n"
+				      "vxor.vv	v20, v20, v22\n"
+
+				      "vsra.vi	v26, v25, 7\n"
+				      "vsll.vi	v27, v25, 1\n"
+				      "vand.vx	v26, v26, %[x1d]\n"
+				      "vxor.vv	v27, v27, v26\n"
+				      "vle8.v	v26, (%[wd6])\n"
+				      "vxor.vv	v25, v27, v26\n"
+				      "vxor.vv	v24, v24, v26\n"
+
+				      "vsra.vi	v30, v29, 7\n"
+				      "vsll.vi	v31, v29, 1\n"
+				      "vand.vx	v30, v30, %[x1d]\n"
+				      "vxor.vv	v31, v31, v30\n"
+				      "vle8.v	v30, (%[wd7])\n"
+				      "vxor.vv	v29, v31, v30\n"
+				      "vxor.vv	v28, v28, v30\n"
+				      ".option	pop\n"
+				      : :
+				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
+				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
+				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
+				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
+				      [wd4]"r"(&dptr[z][d + 4 * nsize]),
+				      [wd5]"r"(&dptr[z][d + 5 * nsize]),
+				      [wd6]"r"(&dptr[z][d + 6 * nsize]),
+				      [wd7]"r"(&dptr[z][d + 7 * nsize]),
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/* P/Q left side optimization */
+		for (z = start - 1; z >= 0; z--) {
+			/*
+			 * w2$$ = MASK(wq$$);
+			 * w1$$ = SHLBYTE(wq$$);
+			 * w2$$ &= NBYTES(0x1d);
+			 * wq$$ = w1$$ ^ w2$$;
+			 */
+			asm volatile (".option	push\n"
+				      ".option	arch,+v\n"
+				      "vsra.vi	v2, v1, 7\n"
+				      "vsll.vi	v3, v1, 1\n"
+				      "vand.vx	v2, v2, %[x1d]\n"
+				      "vxor.vv	v1, v3, v2\n"
+
+				      "vsra.vi	v6, v5, 7\n"
+				      "vsll.vi	v7, v5, 1\n"
+				      "vand.vx	v6, v6, %[x1d]\n"
+				      "vxor.vv	v5, v7, v6\n"
+
+				      "vsra.vi	v10, v9, 7\n"
+				      "vsll.vi	v11, v9, 1\n"
+				      "vand.vx	v10, v10, %[x1d]\n"
+				      "vxor.vv	v9, v11, v10\n"
+
+				      "vsra.vi	v14, v13, 7\n"
+				      "vsll.vi	v15, v13, 1\n"
+				      "vand.vx	v14, v14, %[x1d]\n"
+				      "vxor.vv	v13, v15, v14\n"
+
+				      "vsra.vi	v18, v17, 7\n"
+				      "vsll.vi	v19, v17, 1\n"
+				      "vand.vx	v18, v18, %[x1d]\n"
+				      "vxor.vv	v17, v19, v18\n"
+
+				      "vsra.vi	v22, v21, 7\n"
+				      "vsll.vi	v23, v21, 1\n"
+				      "vand.vx	v22, v22, %[x1d]\n"
+				      "vxor.vv	v21, v23, v22\n"
+
+				      "vsra.vi	v26, v25, 7\n"
+				      "vsll.vi	v27, v25, 1\n"
+				      "vand.vx	v26, v26, %[x1d]\n"
+				      "vxor.vv	v25, v27, v26\n"
+
+				      "vsra.vi	v30, v29, 7\n"
+				      "vsll.vi	v31, v29, 1\n"
+				      "vand.vx	v30, v30, %[x1d]\n"
+				      "vxor.vv	v29, v31, v30\n"
+				      ".option	pop\n"
+				      : :
+				      [x1d]"r"(0x1d)
+			);
+		}
+
+		/*
+		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
+		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
+		 * v0:wp0, v1:wq0, v2:p0, v3:q0
+		 * v4:wp1, v5:wq1, v6:p1, v7:q1
+		 * v8:wp2, v9:wq2, v10:p2, v11:q2
+		 * v12:wp3, v13:wq3, v14:p3, v15:q3
+		 * v16:wp4, v17:wq4, v18:p4, v19:q4
+		 * v20:wp5, v21:wq5, v22:p5, v23:q5
+		 * v24:wp6, v25:wq6, v26:p6, v27:q6
+		 * v28:wp7, v29:wq7, v30:p7, v31:q7
+		 */
+		asm volatile (".option	push\n"
+			      ".option	arch,+v\n"
+			      "vle8.v	v2, (%[wp0])\n"
+			      "vle8.v	v3, (%[wq0])\n"
+			      "vxor.vv	v2, v2, v0\n"
+			      "vxor.vv	v3, v3, v1\n"
+			      "vse8.v	v2, (%[wp0])\n"
+			      "vse8.v	v3, (%[wq0])\n"
+
+			      "vle8.v	v6, (%[wp1])\n"
+			      "vle8.v	v7, (%[wq1])\n"
+			      "vxor.vv	v6, v6, v4\n"
+			      "vxor.vv	v7, v7, v5\n"
+			      "vse8.v	v6, (%[wp1])\n"
+			      "vse8.v	v7, (%[wq1])\n"
+
+			      "vle8.v	v10, (%[wp2])\n"
+			      "vle8.v	v11, (%[wq2])\n"
+			      "vxor.vv	v10, v10, v8\n"
+			      "vxor.vv	v11, v11, v9\n"
+			      "vse8.v	v10, (%[wp2])\n"
+			      "vse8.v	v11, (%[wq2])\n"
+
+			      "vle8.v	v14, (%[wp3])\n"
+			      "vle8.v	v15, (%[wq3])\n"
+			      "vxor.vv	v14, v14, v12\n"
+			      "vxor.vv	v15, v15, v13\n"
+			      "vse8.v	v14, (%[wp3])\n"
+			      "vse8.v	v15, (%[wq3])\n"
+
+			      "vle8.v	v18, (%[wp4])\n"
+			      "vle8.v	v19, (%[wq4])\n"
+			      "vxor.vv	v18, v18, v16\n"
+			      "vxor.vv	v19, v19, v17\n"
+			      "vse8.v	v18, (%[wp4])\n"
+			      "vse8.v	v19, (%[wq4])\n"
+
+			      "vle8.v	v22, (%[wp5])\n"
+			      "vle8.v	v23, (%[wq5])\n"
+			      "vxor.vv	v22, v22, v20\n"
+			      "vxor.vv	v23, v23, v21\n"
+			      "vse8.v	v22, (%[wp5])\n"
+			      "vse8.v	v23, (%[wq5])\n"
+
+			      "vle8.v	v26, (%[wp6])\n"
+			      "vle8.v	v27, (%[wq6])\n"
+			      "vxor.vv	v26, v26, v24\n"
+			      "vxor.vv	v27, v27, v25\n"
+			      "vse8.v	v26, (%[wp6])\n"
+			      "vse8.v	v27, (%[wq6])\n"
+
+			      "vle8.v	v30, (%[wp7])\n"
+			      "vle8.v	v31, (%[wq7])\n"
+			      "vxor.vv	v30, v30, v28\n"
+			      "vxor.vv	v31, v31, v29\n"
+			      "vse8.v	v30, (%[wp7])\n"
+			      "vse8.v	v31, (%[wq7])\n"
+			      ".option	pop\n"
+			      : :
+			      [wp0]"r"(&p[d + nsize * 0]),
+			      [wq0]"r"(&q[d + nsize * 0]),
+			      [wp1]"r"(&p[d + nsize * 1]),
+			      [wq1]"r"(&q[d + nsize * 1]),
+			      [wp2]"r"(&p[d + nsize * 2]),
+			      [wq2]"r"(&q[d + nsize * 2]),
+			      [wp3]"r"(&p[d + nsize * 3]),
+			      [wq3]"r"(&q[d + nsize * 3]),
+			      [wp4]"r"(&p[d + nsize * 4]),
+			      [wq4]"r"(&q[d + nsize * 4]),
+			      [wp5]"r"(&p[d + nsize * 5]),
+			      [wq5]"r"(&q[d + nsize * 5]),
+			      [wp6]"r"(&p[d + nsize * 6]),
+			      [wq6]"r"(&q[d + nsize * 6]),
+			      [wp7]"r"(&p[d + nsize * 7]),
+			      [wq7]"r"(&q[d + nsize * 7])
+		);
+	}
+}
+
+RAID6_RVV_WRAPPER(1);
+RAID6_RVV_WRAPPER(2);
+RAID6_RVV_WRAPPER(4);
+RAID6_RVV_WRAPPER(8);
diff --git a/lib/raid/raid6/riscv/rvv.h b/lib/raid/raid6/riscv/rvv.h
new file mode 100644
index 0000000000000..df0e3637cae8b
--- /dev/null
+++ b/lib/raid/raid6/riscv/rvv.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright 2024 Institute of Software, CAS.
+ *
+ * Definitions for RISC-V RAID-6 code
+ */
+
+#include <asm/vector.h>
+#include "algos.h"
+
+#define RAID6_RVV_WRAPPER(_n)						\
+	static void raid6_rvv ## _n ## _gen_syndrome(int disks,		\
+					size_t bytes, void **ptrs)	\
+	{								\
+		void raid6_rvv ## _n  ## _gen_syndrome_real(int d,	\
+					unsigned long b, void **p);	\
+		kernel_vector_begin();					\
+		raid6_rvv ## _n ## _gen_syndrome_real(disks,		\
+				(unsigned long)bytes, ptrs);		\
+		kernel_vector_end();					\
+	}								\
+	static void raid6_rvv ## _n ## _xor_syndrome(int disks,		\
+					int start, int stop,		\
+					size_t bytes, void **ptrs)	\
+	{								\
+		void raid6_rvv ## _n  ## _xor_syndrome_real(int d,	\
+					int s1, int s2,			\
+					unsigned long b, void **p);	\
+		kernel_vector_begin();					\
+		raid6_rvv ## _n ## _xor_syndrome_real(disks,		\
+			start, stop, (unsigned long)bytes, ptrs);	\
+		kernel_vector_end();					\
+	}								\
+	struct raid6_calls const raid6_rvvx ## _n = {			\
+		.gen_syndrome	= raid6_rvv ## _n ## _gen_syndrome,	\
+		.xor_syndrome	= raid6_rvv ## _n ## _xor_syndrome,	\
+		.name		= "rvvx" #_n,				\
+	}
diff --git a/lib/raid/raid6/s390/pq_arch.h b/lib/raid/raid6/s390/pq_arch.h
new file mode 100644
index 0000000000000..95d14c3423068
--- /dev/null
+++ b/lib/raid/raid6/s390/pq_arch.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <linux/cpufeature.h>
+
+extern const struct raid6_calls raid6_s390vx8;
+extern const struct raid6_recov_calls raid6_recov_s390xc;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+	if (cpu_has_vx())
+		raid6_algo_add(&raid6_s390vx8);
+	else
+		raid6_algo_add_default();
+	raid6_recov_algo_add(&raid6_recov_s390xc);
+}
diff --git a/lib/raid/raid6/s390/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c
new file mode 100644
index 0000000000000..08d56896e5eab
--- /dev/null
+++ b/lib/raid/raid6/s390/recov_s390xc.c
@@ -0,0 +1,116 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * RAID-6 data recovery in dual failure mode based on the XC instruction.
+ *
+ * Copyright IBM Corp. 2016
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ */
+
+#include <linux/mm.h>
+#include <linux/raid/pq.h>
+#include "algos.h"
+
+static inline void xor_block(u8 *p1, u8 *p2)
+{
+	typedef struct { u8 _[256]; } addrtype;
+
+	asm volatile(
+		"	xc	0(256,%[p1]),0(%[p2])\n"
+		: "+m" (*(addrtype *) p1) : "m" (*(addrtype *) p2),
+		  [p1] "a" (p1), [p2] "a" (p2) : "cc");
+}
+
+/* Recover two failed data blocks. */
+static void raid6_2data_recov_s390xc(int disks, size_t bytes, int faila,
+		int failb, void **ptrs)
+{
+	u8 *p, *q, *dp, *dq;
+	const u8 *pbmul;	/* P multiplier table for B data */
+	const u8 *qmul;		/* Q multiplier table (for both) */
+	int i;
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/* Compute syndrome with zero for the missing data pages
+	   Use the dead data pages as temporary storage for
+	   delta p and delta q */
+	dp = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-2] = dp;
+	dq = (u8 *)ptrs[failb];
+	ptrs[failb] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dp;
+	ptrs[failb]   = dq;
+	ptrs[disks-2] = p;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	pbmul = raid6_gfmul[raid6_gfexi[failb-faila]];
+	qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]];
+
+	/* Now do it... */
+	while (bytes) {
+		xor_block(dp, p);
+		xor_block(dq, q);
+		for (i = 0; i < 256; i++)
+			dq[i] = pbmul[dp[i]] ^ qmul[dq[i]];
+		xor_block(dp, dq);
+		p += 256;
+		q += 256;
+		dp += 256;
+		dq += 256;
+		bytes -= 256;
+	}
+}
+
+/* Recover failure of one data block plus the P block */
+static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila,
+		void **ptrs)
+{
+	u8 *p, *q, *dq;
+	const u8 *qmul;		/* Q multiplier table */
+	int i;
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/* Compute syndrome with zero for the missing data page
+	   Use the dead data page as temporary storage for delta q */
+	dq = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dq;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+	/* Now do it... */
+	while (bytes) {
+		xor_block(dq, q);
+		for (i = 0; i < 256; i++)
+			dq[i] = qmul[dq[i]];
+		xor_block(p, dq);
+		p += 256;
+		q += 256;
+		dq += 256;
+		bytes -= 256;
+	}
+}
+
+
+const struct raid6_recov_calls raid6_recov_s390xc = {
+	.data2 = raid6_2data_recov_s390xc,
+	.datap = raid6_datap_recov_s390xc,
+	.name = "s390xc",
+};
diff --git a/lib/raid/raid6/s390/s390vx.uc b/lib/raid/raid6/s390/s390vx.uc
new file mode 100644
index 0000000000000..e5cf9054be2a2
--- /dev/null
+++ b/lib/raid/raid6/s390/s390vx.uc
@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * raid6_vx$#.c
+ *
+ * $#-way unrolled RAID6 gen/xor functions for s390
+ * based on the vector facility
+ *
+ * Copyright IBM Corp. 2016
+ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * This file is postprocessed using unroll.awk.
+ */
+
+#include <linux/cpufeature.h>
+#include <asm/fpu.h>
+#include "algos.h"
+
+#define NSIZE 16
+
+static __always_inline void LOAD_CONST(void)
+{
+	fpu_vrepib(24, 0x07);
+	fpu_vrepib(25, 0x1d);
+}
+
+/*
+ * The SHLBYTE() operation shifts each of the 16 bytes in
+ * vector register y left by 1 bit and stores the result in
+ * vector register x.
+ */
+#define SHLBYTE(x, y)		fpu_vab(x, y, y)
+
+/*
+ * For each of the 16 bytes in the vector register y the MASK()
+ * operation returns 0xFF if the high bit of the byte is 1,
+ * or 0x00 if the high bit is 0. The result is stored in vector
+ * register x.
+ */
+#define MASK(x, y)		fpu_vesravb(x, y, 24)
+
+#define AND(x, y, z)		fpu_vn(x, y, z)
+#define XOR(x, y, z)		fpu_vx(x, y, z)
+#define LOAD_DATA(x, ptr)	fpu_vlm(x, x + $# - 1, ptr)
+#define STORE_DATA(x, ptr)	fpu_vstm(x, x + $# - 1, ptr)
+#define COPY_VEC(x, y)		fpu_vlr(x, y)
+
+static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	DECLARE_KERNEL_FPU_ONSTACK32(vxstate);
+	u8 **dptr, *p, *q;
+	int d, z, z0;
+
+	kernel_fpu_begin(&vxstate, KERNEL_VXR);
+	LOAD_CONST();
+
+	dptr = (u8 **) ptrs;
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0 + 1];	/* XOR parity */
+	q = dptr[z0 + 2];	/* RS syndrome */
+
+	for (d = 0; d < bytes; d += $#*NSIZE) {
+		LOAD_DATA(0,&dptr[z0][d]);
+		COPY_VEC(8+$$,0+$$);
+		for (z = z0 - 1; z >= 0; z--) {
+			MASK(16+$$,8+$$);
+			AND(16+$$,16+$$,25);
+			SHLBYTE(8+$$,8+$$);
+			XOR(8+$$,8+$$,16+$$);
+			LOAD_DATA(16,&dptr[z][d]);
+			XOR(0+$$,0+$$,16+$$);
+			XOR(8+$$,8+$$,16+$$);
+		}
+		STORE_DATA(0,&p[d]);
+		STORE_DATA(8,&q[d]);
+	}
+	kernel_fpu_end(&vxstate, KERNEL_VXR);
+}
+
+static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop,
+					size_t bytes, void **ptrs)
+{
+	DECLARE_KERNEL_FPU_ONSTACK32(vxstate);
+	u8 **dptr, *p, *q;
+	int d, z, z0;
+
+	dptr = (u8 **) ptrs;
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks - 2];	/* XOR parity */
+	q = dptr[disks - 1];	/* RS syndrome */
+
+	kernel_fpu_begin(&vxstate, KERNEL_VXR);
+	LOAD_CONST();
+
+	for (d = 0; d < bytes; d += $#*NSIZE) {
+		/* P/Q data pages */
+		LOAD_DATA(0,&dptr[z0][d]);
+		COPY_VEC(8+$$,0+$$);
+		for (z = z0 - 1; z >= start; z--) {
+			MASK(16+$$,8+$$);
+			AND(16+$$,16+$$,25);
+			SHLBYTE(8+$$,8+$$);
+			XOR(8+$$,8+$$,16+$$);
+			LOAD_DATA(16,&dptr[z][d]);
+			XOR(0+$$,0+$$,16+$$);
+			XOR(8+$$,8+$$,16+$$);
+		}
+		/* P/Q left side optimization */
+		for (z = start - 1; z >= 0; z--) {
+			MASK(16+$$,8+$$);
+			AND(16+$$,16+$$,25);
+			SHLBYTE(8+$$,8+$$);
+			XOR(8+$$,8+$$,16+$$);
+		}
+		LOAD_DATA(16,&p[d]);
+		XOR(16+$$,16+$$,0+$$);
+		STORE_DATA(16,&p[d]);
+		LOAD_DATA(16,&q[d]);
+		XOR(16+$$,16+$$,8+$$);
+		STORE_DATA(16,&q[d]);
+	}
+	kernel_fpu_end(&vxstate, KERNEL_VXR);
+}
+
+const struct raid6_calls raid6_s390vx$# = {
+	.gen_syndrome	= raid6_s390vx$#_gen_syndrome,
+	.xor_syndrome	= raid6_s390vx$#_xor_syndrome,
+	.name		= "vx128x$#",
+};
diff --git a/lib/raid/raid6/tests/Makefile b/lib/raid/raid6/tests/Makefile
new file mode 100644
index 0000000000000..87a001b228474
--- /dev/null
+++ b/lib/raid/raid6/tests/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_RAID6_PQ_KUNIT_TEST)	+= raid6_kunit.o
diff --git a/lib/raid/raid6/tests/raid6_kunit.c b/lib/raid/raid6/tests/raid6_kunit.c
new file mode 100644
index 0000000000000..9f3e671a12241
--- /dev/null
+++ b/lib/raid/raid6/tests/raid6_kunit.c
@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved
+ *
+ * Test RAID-6 recovery algorithms.
+ */
+
+#include <kunit/test.h>
+#include <linux/prandom.h>
+#include <linux/vmalloc.h>
+#include <linux/raid/pq.h>
+#include "../algos.h"
+
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+
+#define RAID6_KUNIT_SEED		42
+#define RAID6_KUNIT_NUM_TEST_ITERS	10
+#define RAID6_KUNIT_MAX_BUFFERS		64 /* Including P and Q */
+#define RAID6_KUNIT_MAX_FAILURES	2
+#define RAID6_KUNIT_MAX_BYTES		PAGE_SIZE
+
+static struct rnd_state rng;
+static void *test_buffers[RAID6_KUNIT_MAX_BUFFERS];
+static void *aligned_buffers[RAID6_KUNIT_MAX_BUFFERS];
+static void *test_recov_buffers[RAID6_KUNIT_MAX_FAILURES];
+static size_t test_buflen;
+
+struct test_args {
+	unsigned int recov_idx;
+	const struct raid6_recov_calls *recov;
+	unsigned int gen_idx;
+	const struct raid6_calls *gen;
+};
+
+static struct test_args args;
+
+static u32 rand32(void)
+{
+	return prandom_u32_state(&rng);
+}
+
+/* Generate a random length that is a multiple of 512. */
+static unsigned int random_length(unsigned int max_length)
+{
+	return round_up((rand32() % max_length) + 1, 512);
+}
+
+static unsigned int random_nr_buffers(void)
+{
+	return (rand32() % (RAID6_KUNIT_MAX_BUFFERS - (RAID6_MIN_DISKS - 1))) +
+			RAID6_MIN_DISKS;
+}
+
+/* Generate a random alignment that is a multiple of 64. */
+static unsigned int random_alignment(unsigned int max_alignment)
+{
+	if (max_alignment == 0)
+		return 0;
+	return (rand32() % (max_alignment + 1)) & ~63;
+}
+
+static void makedata(int start, int stop)
+{
+	int i;
+
+	for (i = start; i <= stop; i++)
+		prandom_bytes_state(&rng, test_buffers[i], test_buflen);
+}
+
+static char member_type(unsigned int nr_buffers, int d)
+{
+	if (d == nr_buffers - 2)
+		return 'P';
+	if (d == nr_buffers - 1)
+		return 'Q';
+	return 'D';
+}
+
+static void test_recover_one(struct kunit *test, unsigned int nr_buffers,
+		unsigned int len, int faila, int failb)
+{
+	const struct test_args *ta = test->param_value;
+	void *dataptrs[RAID6_KUNIT_MAX_BUFFERS];
+	int i;
+
+	if (faila > failb)
+		swap(faila, failb);
+
+	for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++)
+		memset(test_recov_buffers[i], 0xf0, test_buflen);
+
+	memcpy(dataptrs, aligned_buffers, sizeof(dataptrs));
+	dataptrs[faila] = test_recov_buffers[0];
+	dataptrs[failb] = test_recov_buffers[1];
+
+	if (failb == nr_buffers - 1) {
+		/*
+		 * We don't implement the data+Q failure scenario, since it
+		 * is equivalent to a RAID-5 failure (XOR, then recompute Q).
+		 */
+		if (WARN_ON_ONCE(faila != nr_buffers - 2))
+			return;
+
+		/* P+Q failure.  Just rebuild the syndrome. */
+		ta->gen->gen_syndrome(nr_buffers, len, dataptrs);
+	} else if (failb == nr_buffers - 2) {
+		/* data+P failure. */
+		ta->recov->datap(nr_buffers, len, faila, dataptrs);
+	} else {
+		/* data+data failure. */
+		ta->recov->data2(nr_buffers, len, faila, failb, dataptrs);
+	}
+
+	KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[faila], dataptrs[faila],
+			len,
+			"faila miscompared: %3d[%c] buffers %u len %u (failb=%3d[%c])\n",
+			faila, member_type(nr_buffers, faila),
+			nr_buffers, len,
+			failb, member_type(nr_buffers, failb));
+	KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[failb], dataptrs[failb],
+			len,
+			"failb miscompared: %3d[%c] buffers %u len %u (faila=%3d[%c])\n",
+			failb, member_type(nr_buffers, failb),
+			nr_buffers, len,
+			faila, member_type(nr_buffers, faila));
+}
+
+static void test_recover(struct kunit *test, unsigned int nr_buffers,
+		unsigned int len)
+{
+	unsigned int nr_data = nr_buffers - 2;
+	int iterations, i;
+
+	/* Test P+Q recovery */
+	test_recover_one(test, nr_buffers, len, nr_data, nr_buffers - 1);
+
+	/* Test data+P recovery */
+	for (i = 0; i < nr_buffers - 2; i++)
+		test_recover_one(test, nr_buffers, len, i, nr_data);
+
+	/* Double data failure is impossible with a single data disk */
+	if (nr_data == 1)
+		return;
+
+	/* Test data+data recovery using random sampling */
+	iterations = nr_buffers * 2; /* should provide good enough coverage */
+	for (i = 0; i < iterations; i++) {
+		int faila = rand32() % nr_data, failb;
+
+		do {
+			failb = rand32() % nr_data;
+		} while (failb == faila);
+
+		test_recover_one(test, nr_buffers, len, faila, failb);
+	}
+}
+
+/* Simulate rmw run */
+static void test_rmw_one(struct kunit *test, unsigned int nr_buffers,
+		unsigned int len, int p1, int p2)
+{
+	const struct test_args *ta = test->param_value;
+
+	ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers);
+	makedata(p1, p2);
+	ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers);
+	test_recover(test, nr_buffers, len);
+}
+
+static void test_rmw(struct kunit *test, unsigned int nr_buffers,
+		unsigned int len)
+{
+	int iterations = nr_buffers / 2, i;
+
+	for (i = 0; i < iterations; i++) {
+		int p1 = rand32() % (nr_buffers - 2);
+		int p2 = rand32() % (nr_buffers - 2);
+
+		if (p2 < p1)
+			swap(p1, p2);
+		test_rmw_one(test, nr_buffers, len, p1, p2);
+	}
+}
+
+static void raid6_test_one(struct kunit *test)
+{
+	const struct test_args *ta = test->param_value;
+	unsigned int nr_buffers = random_nr_buffers();
+	unsigned int len = random_length(RAID6_KUNIT_MAX_BYTES);
+	unsigned int max_alignment;
+	int i;
+
+	/* Nuke syndromes */
+	memset(test_buffers[nr_buffers - 2], 0xee, test_buflen);
+	memset(test_buffers[nr_buffers - 1], 0xee, test_buflen);
+
+	/*
+	 * If we're not using the entire buffer size, inject randomize alignment
+	 * into the buffer.
+	 */
+	max_alignment = RAID6_KUNIT_MAX_BYTES - len;
+	if (rand32() % 2 == 0) {
+		/* Use random alignments mod 64 */
+		for (i = 0; i < nr_buffers; i++)
+			aligned_buffers[i] = test_buffers[i] +
+				random_alignment(max_alignment);
+	} else {
+		/* Go up to the guard page, to catch buffer overreads */
+		unsigned int align = test_buflen - len;
+
+		for (i = 0; i < nr_buffers; i++)
+			aligned_buffers[i] = test_buffers[i] + align;
+	}
+
+	/* Generate assumed good syndrome */
+	ta->gen->gen_syndrome(nr_buffers, len, aligned_buffers);
+
+	test_recover(test, nr_buffers, len);
+
+	if (ta->gen->xor_syndrome)
+		test_rmw(test, nr_buffers, len);
+}
+
+static void raid6_test(struct kunit *test)
+{
+	int i;
+
+	for (i = 0; i < RAID6_KUNIT_NUM_TEST_ITERS; i++)
+		raid6_test_one(test);
+}
+
+static const void *raid6_gen_params(struct kunit *test, const void *prev,
+		char *desc)
+{
+	if (!prev) {
+		memset(&args, 0, sizeof(args));
+next_algo:
+		args.recov_idx = 0;
+		args.gen = raid6_algo_find(args.gen_idx);
+		if (!args.gen)
+			return NULL;
+	}
+
+	if (args.recov)
+		args.recov_idx++;
+	args.recov = raid6_recov_algo_find(args.recov_idx);
+	if (!args.recov) {
+		args.gen_idx++;
+		goto next_algo;
+	}
+
+	snprintf(desc, KUNIT_PARAM_DESC_SIZE, "gen=%s recov=%s",
+			args.gen->name, args.recov->name);
+	return &args;
+}
+
+static struct kunit_case raid6_test_cases[] = {
+	KUNIT_CASE_PARAM(raid6_test, raid6_gen_params),
+	{},
+};
+
+static int raid6_suite_init(struct kunit_suite *suite)
+{
+	int i;
+
+	prandom_seed_state(&rng, RAID6_KUNIT_SEED);
+
+	/*
+	 * Allocate the test buffer using vmalloc() with a page-aligned length
+	 * so that it is immediately followed by a guard page.  This allows
+	 * buffer overreads to be detected, even in assembly code.
+	 */
+	test_buflen = round_up(RAID6_KUNIT_MAX_BYTES, PAGE_SIZE);
+	for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) {
+		test_recov_buffers[i] = vmalloc(test_buflen);
+		if (!test_recov_buffers[i])
+			goto out_free_recov_buffers;
+	}
+	for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) {
+		test_buffers[i] = vmalloc(test_buflen);
+		if (!test_buffers[i])
+			goto out_free_buffers;
+	}
+
+	makedata(0, RAID6_KUNIT_MAX_BUFFERS - 1);
+
+	return 0;
+
+out_free_buffers:
+	for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++)
+		vfree(test_buffers[i]);
+	memset(test_buffers, 0, sizeof(test_buffers));
+out_free_recov_buffers:
+	for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++)
+		vfree(test_recov_buffers[i]);
+	memset(test_recov_buffers, 0, sizeof(test_recov_buffers));
+	return -ENOMEM;
+}
+
+static void raid6_suite_exit(struct kunit_suite *suite)
+{
+	int i;
+
+	for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++)
+		vfree(test_buffers[i]);
+	memset(test_buffers, 0, sizeof(test_buffers));
+	for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++)
+		vfree(test_recov_buffers[i]);
+	memset(test_recov_buffers, 0, sizeof(test_recov_buffers));
+}
+
+static struct kunit_suite raid6_test_suite = {
+	.name		= "raid6",
+	.test_cases	= raid6_test_cases,
+	.suite_init	= raid6_suite_init,
+	.suite_exit	= raid6_suite_exit,
+};
+kunit_test_suite(raid6_test_suite);
+
+MODULE_DESCRIPTION("Unit test for the RAID P/Q library functions");
+MODULE_LICENSE("GPL");
diff --git a/lib/raid/raid6/unroll.awk b/lib/raid/raid6/unroll.awk
new file mode 100644
index 0000000000000..0809805a7e231
--- /dev/null
+++ b/lib/raid/raid6/unroll.awk
@@ -0,0 +1,20 @@
+
+# This filter requires one command line option of form -vN=n
+# where n must be a decimal number.
+#
+# Repeat each input line containing $$ n times, replacing $$ with 0...n-1.
+# Replace each $# with n, and each $* with a single $.
+
+BEGIN {
+	n = N + 0
+}
+{
+	if (/\$\$/) { rep = n } else { rep = 1 }
+	for (i = 0; i < rep; ++i) {
+		tmp = $0
+		gsub(/\$\$/, i, tmp)
+		gsub(/\$#/, n, tmp)
+		gsub(/\$\*/, "$", tmp)
+		print tmp
+	}
+}
diff --git a/lib/raid/raid6/x86/avx2.c b/lib/raid/raid6/x86/avx2.c
new file mode 100644
index 0000000000000..7d829c669ea79
--- /dev/null
+++ b/lib/raid/raid6/x86/avx2.c
@@ -0,0 +1,455 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2012 Intel Corporation
+ * Author: Yuanhan Liu <yuanhan.liu@linux.intel.com>
+ *
+ * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ * AVX2 implementation of RAID-6 syndrome functions
+ */
+
+#include <asm/cpufeature.h>
+#include <asm/fpu/api.h>
+#include "algos.h"
+
+static const struct raid6_avx2_constants {
+	u64 x1d[4];
+} raid6_avx2_constants __aligned(32) = {
+	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
+	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
+};
+
+/*
+ * Plain AVX2 implementation
+ */
+static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
+	asm volatile("vpxor %ymm3,%ymm3,%ymm3");	/* Zero temp */
+
+	for (d = 0; d < bytes; d += 32) {
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+		asm volatile("vmovdqa %0,%%ymm2" : : "m" (dptr[z0][d]));/* P[0] */
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
+		asm volatile("vmovdqa %ymm2,%ymm4");/* Q[0] */
+		asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z0-1][d]));
+		for (z = z0-2; z >= 0; z--) {
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+			asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm6,%ymm2,%ymm2");
+			asm volatile("vpxor %ymm6,%ymm4,%ymm4");
+			asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z][d]));
+		}
+		asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");
+		asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+		asm volatile("vpand %ymm0,%ymm5,%ymm5");
+		asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+		asm volatile("vpxor %ymm6,%ymm2,%ymm2");
+		asm volatile("vpxor %ymm6,%ymm4,%ymm4");
+
+		asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
+		asm volatile("vpxor %ymm2,%ymm2,%ymm2");
+		asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
+		asm volatile("vpxor %ymm4,%ymm4,%ymm4");
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+static void raid6_avx21_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
+
+	for (d = 0 ; d < bytes ; d += 32) {
+		asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
+		asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
+		asm volatile("vpxor %ymm4,%ymm2,%ymm2");
+		/* P/Q data pages */
+		for (z = z0-1 ; z >= start ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
+			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+		}
+		/* P/Q left side optimization */
+		for (z = start-1 ; z >= 0 ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+		}
+		asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
+		/* Don't use movntdq for r/w memory area < cache line */
+		asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));
+		asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_avx2x1 = {
+	.gen_syndrome	= raid6_avx21_gen_syndrome,
+	.xor_syndrome	= raid6_avx21_xor_syndrome,
+	.name		= "avx2x1",
+};
+
+/*
+ * Unrolled-by-2 AVX2 implementation
+ */
+static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
+	asm volatile("vpxor %ymm1,%ymm1,%ymm1"); /* Zero temp */
+
+	/* We uniformly assume a single prefetch covers at least 32 bytes */
+	for (d = 0; d < bytes; d += 64) {
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d+32]));
+		asm volatile("vmovdqa %0,%%ymm2" : : "m" (dptr[z0][d]));/* P[0] */
+		asm volatile("vmovdqa %0,%%ymm3" : : "m" (dptr[z0][d+32]));/* P[1] */
+		asm volatile("vmovdqa %ymm2,%ymm4"); /* Q[0] */
+		asm volatile("vmovdqa %ymm3,%ymm6"); /* Q[1] */
+		for (z = z0-1; z >= 0; z--) {
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d+32]));
+			asm volatile("vpcmpgtb %ymm4,%ymm1,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm1,%ymm7");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vmovdqa %0,%%ymm5" : : "m" (dptr[z][d]));
+			asm volatile("vmovdqa %0,%%ymm7" : : "m" (dptr[z][d+32]));
+			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
+			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+		}
+		asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
+		asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
+		asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
+		asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+static void raid6_avx22_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
+
+	for (d = 0 ; d < bytes ; d += 64) {
+		asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
+		asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));
+		asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
+		asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));
+		asm volatile("vpxor %ymm4,%ymm2,%ymm2");
+		asm volatile("vpxor %ymm6,%ymm3,%ymm3");
+		/* P/Q data pages */
+		for (z = z0-1 ; z >= start ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
+			asm volatile("vmovdqa %0,%%ymm7"
+				     :: "m" (dptr[z][d+32]));
+			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
+			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+		}
+		/* P/Q left side optimization */
+		for (z = start-1 ; z >= 0 ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+		}
+		asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
+		asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));
+		/* Don't use movntdq for r/w memory area < cache line */
+		asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));
+		asm volatile("vmovdqa %%ymm6,%0" : "=m" (q[d+32]));
+		asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));
+		asm volatile("vmovdqa %%ymm3,%0" : "=m" (p[d+32]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_avx2x2 = {
+	.gen_syndrome	= raid6_avx22_gen_syndrome,
+	.xor_syndrome	= raid6_avx22_xor_syndrome,
+	.name		= "avx2x2",
+};
+
+#ifdef CONFIG_X86_64
+
+/*
+ * Unrolled-by-4 AVX2 implementation
+ */
+static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
+	asm volatile("vpxor %ymm1,%ymm1,%ymm1");	/* Zero temp */
+	asm volatile("vpxor %ymm2,%ymm2,%ymm2");	/* P[0] */
+	asm volatile("vpxor %ymm3,%ymm3,%ymm3");	/* P[1] */
+	asm volatile("vpxor %ymm4,%ymm4,%ymm4");	/* Q[0] */
+	asm volatile("vpxor %ymm6,%ymm6,%ymm6");	/* Q[1] */
+	asm volatile("vpxor %ymm10,%ymm10,%ymm10");	/* P[2] */
+	asm volatile("vpxor %ymm11,%ymm11,%ymm11");	/* P[3] */
+	asm volatile("vpxor %ymm12,%ymm12,%ymm12");	/* Q[2] */
+	asm volatile("vpxor %ymm14,%ymm14,%ymm14");	/* Q[3] */
+
+	for (d = 0; d < bytes; d += 128) {
+		for (z = z0; z >= 0; z--) {
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d+32]));
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d+64]));
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d+96]));
+			asm volatile("vpcmpgtb %ymm4,%ymm1,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm1,%ymm7");
+			asm volatile("vpcmpgtb %ymm12,%ymm1,%ymm13");
+			asm volatile("vpcmpgtb %ymm14,%ymm1,%ymm15");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
+			asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpand %ymm0,%ymm13,%ymm13");
+			asm volatile("vpand %ymm0,%ymm15,%ymm15");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
+			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
+			asm volatile("vmovdqa %0,%%ymm5" : : "m" (dptr[z][d]));
+			asm volatile("vmovdqa %0,%%ymm7" : : "m" (dptr[z][d+32]));
+			asm volatile("vmovdqa %0,%%ymm13" : : "m" (dptr[z][d+64]));
+			asm volatile("vmovdqa %0,%%ymm15" : : "m" (dptr[z][d+96]));
+			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
+			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
+			asm volatile("vpxor %ymm13,%ymm10,%ymm10");
+			asm volatile("vpxor %ymm15,%ymm11,%ymm11");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
+			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
+		}
+		asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
+		asm volatile("vpxor %ymm2,%ymm2,%ymm2");
+		asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
+		asm volatile("vpxor %ymm3,%ymm3,%ymm3");
+		asm volatile("vmovntdq %%ymm10,%0" : "=m" (p[d+64]));
+		asm volatile("vpxor %ymm10,%ymm10,%ymm10");
+		asm volatile("vmovntdq %%ymm11,%0" : "=m" (p[d+96]));
+		asm volatile("vpxor %ymm11,%ymm11,%ymm11");
+		asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
+		asm volatile("vpxor %ymm4,%ymm4,%ymm4");
+		asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
+		asm volatile("vpxor %ymm6,%ymm6,%ymm6");
+		asm volatile("vmovntdq %%ymm12,%0" : "=m" (q[d+64]));
+		asm volatile("vpxor %ymm12,%ymm12,%ymm12");
+		asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));
+		asm volatile("vpxor %ymm14,%ymm14,%ymm14");
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+static void raid6_avx24_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa %0,%%ymm0" :: "m" (raid6_avx2_constants.x1d[0]));
+
+	for (d = 0 ; d < bytes ; d += 128) {
+		asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
+		asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));
+		asm volatile("vmovdqa %0,%%ymm12" :: "m" (dptr[z0][d+64]));
+		asm volatile("vmovdqa %0,%%ymm14" :: "m" (dptr[z0][d+96]));
+		asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
+		asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));
+		asm volatile("vmovdqa %0,%%ymm10" : : "m" (p[d+64]));
+		asm volatile("vmovdqa %0,%%ymm11" : : "m" (p[d+96]));
+		asm volatile("vpxor %ymm4,%ymm2,%ymm2");
+		asm volatile("vpxor %ymm6,%ymm3,%ymm3");
+		asm volatile("vpxor %ymm12,%ymm10,%ymm10");
+		asm volatile("vpxor %ymm14,%ymm11,%ymm11");
+		/* P/Q data pages */
+		for (z = z0-1 ; z >= start ; z--) {
+			asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
+			asm volatile("prefetchnta %0" :: "m" (dptr[z][d+64]));
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
+			asm volatile("vpxor %ymm13,%ymm13,%ymm13");
+			asm volatile("vpxor %ymm15,%ymm15,%ymm15");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
+			asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");
+			asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
+			asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpand %ymm0,%ymm13,%ymm13");
+			asm volatile("vpand %ymm0,%ymm15,%ymm15");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
+			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
+			asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
+			asm volatile("vmovdqa %0,%%ymm7"
+				     :: "m" (dptr[z][d+32]));
+			asm volatile("vmovdqa %0,%%ymm13"
+				     :: "m" (dptr[z][d+64]));
+			asm volatile("vmovdqa %0,%%ymm15"
+				     :: "m" (dptr[z][d+96]));
+			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
+			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
+			asm volatile("vpxor %ymm13,%ymm10,%ymm10");
+			asm volatile("vpxor %ymm15,%ymm11,%ymm11");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
+			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
+		}
+		asm volatile("prefetchnta %0" :: "m" (q[d]));
+		asm volatile("prefetchnta %0" :: "m" (q[d+64]));
+		/* P/Q left side optimization */
+		for (z = start-1 ; z >= 0 ; z--) {
+			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
+			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
+			asm volatile("vpxor %ymm13,%ymm13,%ymm13");
+			asm volatile("vpxor %ymm15,%ymm15,%ymm15");
+			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
+			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
+			asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");
+			asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");
+			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
+			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
+			asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
+			asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
+			asm volatile("vpand %ymm0,%ymm5,%ymm5");
+			asm volatile("vpand %ymm0,%ymm7,%ymm7");
+			asm volatile("vpand %ymm0,%ymm13,%ymm13");
+			asm volatile("vpand %ymm0,%ymm15,%ymm15");
+			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
+			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
+			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
+			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
+		}
+		asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
+		asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
+		asm volatile("vmovntdq %%ymm10,%0" : "=m" (p[d+64]));
+		asm volatile("vmovntdq %%ymm11,%0" : "=m" (p[d+96]));
+		asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
+		asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));
+		asm volatile("vpxor %0,%%ymm12,%%ymm12" : : "m" (q[d+64]));
+		asm volatile("vpxor %0,%%ymm14,%%ymm14" : : "m" (q[d+96]));
+		asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
+		asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
+		asm volatile("vmovntdq %%ymm12,%0" : "=m" (q[d+64]));
+		asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));
+	}
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_avx2x4 = {
+	.gen_syndrome	= raid6_avx24_gen_syndrome,
+	.xor_syndrome	= raid6_avx24_xor_syndrome,
+	.name		= "avx2x4",
+};
+#endif /* CONFIG_X86_64 */
diff --git a/lib/raid/raid6/x86/avx512.c b/lib/raid/raid6/x86/avx512.c
new file mode 100644
index 0000000000000..e671eb5bde63e
--- /dev/null
+++ b/lib/raid/raid6/x86/avx512.c
@@ -0,0 +1,539 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2016 Intel Corporation
+ *
+ * Author: Gayatri Kammela <gayatri.kammela@intel.com>
+ * Author: Megha Dey <megha.dey@linux.intel.com>
+ *
+ * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved
+ * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ * AVX512 implementation of RAID-6 syndrome functions
+ */
+
+#include <asm/cpufeature.h>
+#include <asm/fpu/api.h>
+#include "algos.h"
+
+static const struct raid6_avx512_constants {
+	u64 x1d[8];
+} raid6_avx512_constants __aligned(512/8) = {
+	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
+	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
+	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
+	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
+};
+
+static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;         /* Highest data disk */
+	p = dptr[z0+1];         /* XOR parity */
+	q = dptr[z0+2];         /* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
+		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
+		     :
+		     : "m" (raid6_avx512_constants.x1d[0]));
+
+	for (d = 0; d < bytes; d += 64) {
+		asm volatile("prefetchnta %0\n\t"
+			     "vmovdqa64 %0,%%zmm2\n\t"     /* P[0] */
+			     "prefetchnta %1\n\t"
+			     "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
+			     "vmovdqa64 %1,%%zmm6"
+			     :
+			     : "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
+		for (z = z0-2; z >= 0; z--) {
+			asm volatile("prefetchnta %0\n\t"
+				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
+				     "vpmovm2b %%k1,%%zmm5\n\t"
+				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
+				     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
+				     "vmovdqa64 %0,%%zmm6"
+				     :
+				     : "m" (dptr[z][d]));
+		}
+		asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
+			     "vpmovm2b %%k1,%%zmm5\n\t"
+			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+			     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
+			     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
+			     "vmovntdq %%zmm2,%0\n\t"
+			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
+			     "vmovntdq %%zmm4,%1\n\t"
+			     "vpxorq %%zmm4,%%zmm4,%%zmm4"
+			     :
+			     : "m" (p[d]), "m" (q[d]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
+				       size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa64 %0,%%zmm0"
+		     : : "m" (raid6_avx512_constants.x1d[0]));
+
+	for (d = 0 ; d < bytes ; d += 64) {
+		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
+			     "vmovdqa64 %1,%%zmm2\n\t"
+			     "vpxorq %%zmm4,%%zmm2,%%zmm2"
+			     :
+			     : "m" (dptr[z0][d]),  "m" (p[d]));
+		/* P/Q data pages */
+		for (z = z0-1 ; z >= start ; z--) {
+			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
+				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
+				     "vpmovm2b %%k1,%%zmm5\n\t"
+				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vmovdqa64 %0,%%zmm5\n\t"
+				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
+				     :
+				     : "m" (dptr[z][d]));
+		}
+		/* P/Q left side optimization */
+		for (z = start-1 ; z >= 0 ; z--) {
+			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
+				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
+				     "vpmovm2b %%k1,%%zmm5\n\t"
+				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
+				     :
+				     : );
+		}
+		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
+		/* Don't use movntdq for r/w memory area < cache line */
+			     "vmovdqa64 %%zmm4,%0\n\t"
+			     "vmovdqa64 %%zmm2,%1"
+			     :
+			     : "m" (q[d]), "m" (p[d]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_avx512x1 = {
+	.gen_syndrome	= raid6_avx5121_gen_syndrome,
+	.xor_syndrome	= raid6_avx5121_xor_syndrome,
+	.name		= "avx512x1",
+};
+
+/*
+ * Unrolled-by-2 AVX512 implementation
+ */
+static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;         /* Highest data disk */
+	p = dptr[z0+1];         /* XOR parity */
+	q = dptr[z0+2];         /* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
+		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
+		     :
+		     : "m" (raid6_avx512_constants.x1d[0]));
+
+	/* We uniformly assume a single prefetch covers at least 64 bytes */
+	for (d = 0; d < bytes; d += 128) {
+		asm volatile("prefetchnta %0\n\t"
+			     "prefetchnta %1\n\t"
+			     "vmovdqa64 %0,%%zmm2\n\t"      /* P[0] */
+			     "vmovdqa64 %1,%%zmm3\n\t"      /* P[1] */
+			     "vmovdqa64 %%zmm2,%%zmm4\n\t"  /* Q[0] */
+			     "vmovdqa64 %%zmm3,%%zmm6"      /* Q[1] */
+			     :
+			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]));
+		for (z = z0-1; z >= 0; z--) {
+			asm volatile("prefetchnta %0\n\t"
+				     "prefetchnta %1\n\t"
+				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
+				     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
+				     "vpmovm2b %%k1,%%zmm5\n\t"
+				     "vpmovm2b %%k2,%%zmm7\n\t"
+				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
+				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
+				     "vmovdqa64 %0,%%zmm5\n\t"
+				     "vmovdqa64 %1,%%zmm7\n\t"
+				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
+				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
+				     :
+				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]));
+		}
+		asm volatile("vmovntdq %%zmm2,%0\n\t"
+			     "vmovntdq %%zmm3,%1\n\t"
+			     "vmovntdq %%zmm4,%2\n\t"
+			     "vmovntdq %%zmm6,%3"
+			     :
+			     : "m" (p[d]), "m" (p[d+64]), "m" (q[d]),
+			       "m" (q[d+64]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
+				       size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa64 %0,%%zmm0"
+		     : : "m" (raid6_avx512_constants.x1d[0]));
+
+	for (d = 0 ; d < bytes ; d += 128) {
+		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
+			     "vmovdqa64 %1,%%zmm6\n\t"
+			     "vmovdqa64 %2,%%zmm2\n\t"
+			     "vmovdqa64 %3,%%zmm3\n\t"
+			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
+			     "vpxorq %%zmm6,%%zmm3,%%zmm3"
+			     :
+			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
+			       "m" (p[d]), "m" (p[d+64]));
+		/* P/Q data pages */
+		for (z = z0-1 ; z >= start ; z--) {
+			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
+				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
+				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
+				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
+				     "vpmovm2b %%k1,%%zmm5\n\t"
+				     "vpmovm2b %%k2,%%zmm7\n\t"
+				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
+				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
+				     "vmovdqa64 %0,%%zmm5\n\t"
+				     "vmovdqa64 %1,%%zmm7\n\t"
+				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
+				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
+				     :
+				     : "m" (dptr[z][d]),  "m" (dptr[z][d+64]));
+		}
+		/* P/Q left side optimization */
+		for (z = start-1 ; z >= 0 ; z--) {
+			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
+				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
+				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
+				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
+				     "vpmovm2b %%k1,%%zmm5\n\t"
+				     "vpmovm2b %%k2,%%zmm7\n\t"
+				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
+				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
+				     :
+				     : );
+		}
+		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
+			     "vpxorq %1,%%zmm6,%%zmm6\n\t"
+			     /* Don't use movntdq for r/w
+			      * memory area < cache line
+			      */
+			     "vmovdqa64 %%zmm4,%0\n\t"
+			     "vmovdqa64 %%zmm6,%1\n\t"
+			     "vmovdqa64 %%zmm2,%2\n\t"
+			     "vmovdqa64 %%zmm3,%3"
+			     :
+			     : "m" (q[d]), "m" (q[d+64]), "m" (p[d]),
+			       "m" (p[d+64]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_avx512x2 = {
+	.gen_syndrome	= raid6_avx5122_gen_syndrome,
+	.xor_syndrome	= raid6_avx5122_xor_syndrome,
+	.name		= "avx512x2",
+};
+
+#ifdef CONFIG_X86_64
+
+/*
+ * Unrolled-by-4 AVX2 implementation
+ */
+static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;         /* Highest data disk */
+	p = dptr[z0+1];         /* XOR parity */
+	q = dptr[z0+2];         /* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
+		     "vpxorq %%zmm1,%%zmm1,%%zmm1\n\t"       /* Zero temp */
+		     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"       /* P[0] */
+		     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"       /* P[1] */
+		     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"       /* Q[0] */
+		     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"       /* Q[1] */
+		     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"    /* P[2] */
+		     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"    /* P[3] */
+		     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"    /* Q[2] */
+		     "vpxorq %%zmm14,%%zmm14,%%zmm14"        /* Q[3] */
+		     :
+		     : "m" (raid6_avx512_constants.x1d[0]));
+
+	for (d = 0; d < bytes; d += 256) {
+		for (z = z0; z >= 0; z--) {
+		asm volatile("prefetchnta %0\n\t"
+			     "prefetchnta %1\n\t"
+			     "prefetchnta %2\n\t"
+			     "prefetchnta %3\n\t"
+			     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
+			     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
+			     "vpcmpgtb %%zmm12,%%zmm1,%%k3\n\t"
+			     "vpcmpgtb %%zmm14,%%zmm1,%%k4\n\t"
+			     "vpmovm2b %%k1,%%zmm5\n\t"
+			     "vpmovm2b %%k2,%%zmm7\n\t"
+			     "vpmovm2b %%k3,%%zmm13\n\t"
+			     "vpmovm2b %%k4,%%zmm15\n\t"
+			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+			     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
+			     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
+			     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
+			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+			     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
+			     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
+			     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
+			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
+			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
+			     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
+			     "vmovdqa64 %0,%%zmm5\n\t"
+			     "vmovdqa64 %1,%%zmm7\n\t"
+			     "vmovdqa64 %2,%%zmm13\n\t"
+			     "vmovdqa64 %3,%%zmm15\n\t"
+			     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
+			     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
+			     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
+			     "vpxorq %%zmm15,%%zmm11,%%zmm11\n"
+			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
+			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
+			     "vpxorq %%zmm15,%%zmm14,%%zmm14"
+			     :
+			     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
+			       "m" (dptr[z][d+128]), "m" (dptr[z][d+192]));
+		}
+		asm volatile("vmovntdq %%zmm2,%0\n\t"
+			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
+			     "vmovntdq %%zmm3,%1\n\t"
+			     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"
+			     "vmovntdq %%zmm10,%2\n\t"
+			     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"
+			     "vmovntdq %%zmm11,%3\n\t"
+			     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"
+			     "vmovntdq %%zmm4,%4\n\t"
+			     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"
+			     "vmovntdq %%zmm6,%5\n\t"
+			     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"
+			     "vmovntdq %%zmm12,%6\n\t"
+			     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"
+			     "vmovntdq %%zmm14,%7\n\t"
+			     "vpxorq %%zmm14,%%zmm14,%%zmm14"
+			     :
+			     : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
+			       "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
+			       "m" (q[d+128]), "m" (q[d+192]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
+				       size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("vmovdqa64 %0,%%zmm0"
+		     :: "m" (raid6_avx512_constants.x1d[0]));
+
+	for (d = 0 ; d < bytes ; d += 256) {
+		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
+			     "vmovdqa64 %1,%%zmm6\n\t"
+			     "vmovdqa64 %2,%%zmm12\n\t"
+			     "vmovdqa64 %3,%%zmm14\n\t"
+			     "vmovdqa64 %4,%%zmm2\n\t"
+			     "vmovdqa64 %5,%%zmm3\n\t"
+			     "vmovdqa64 %6,%%zmm10\n\t"
+			     "vmovdqa64 %7,%%zmm11\n\t"
+			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
+			     "vpxorq %%zmm6,%%zmm3,%%zmm3\n\t"
+			     "vpxorq %%zmm12,%%zmm10,%%zmm10\n\t"
+			     "vpxorq %%zmm14,%%zmm11,%%zmm11"
+			     :
+			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
+			       "m" (dptr[z0][d+128]), "m" (dptr[z0][d+192]),
+			       "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
+			       "m" (p[d+192]));
+		/* P/Q data pages */
+		for (z = z0-1 ; z >= start ; z--) {
+			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
+				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
+				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
+				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
+				     "prefetchnta %0\n\t"
+				     "prefetchnta %2\n\t"
+				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
+				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
+				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
+				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
+				     "vpmovm2b %%k1,%%zmm5\n\t"
+				     "vpmovm2b %%k2,%%zmm7\n\t"
+				     "vpmovm2b %%k3,%%zmm13\n\t"
+				     "vpmovm2b %%k4,%%zmm15\n\t"
+				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
+				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
+				     "vpaddb %%Zmm14,%%zmm14,%%zmm14\n\t"
+				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
+				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
+				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
+				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
+				     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
+				     "vmovdqa64 %0,%%zmm5\n\t"
+				     "vmovdqa64 %1,%%zmm7\n\t"
+				     "vmovdqa64 %2,%%zmm13\n\t"
+				     "vmovdqa64 %3,%%zmm15\n\t"
+				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
+				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
+				     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
+				     "vpxorq %%zmm15,%%zmm11,%%zmm11\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
+				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
+				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
+				     :
+				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
+				       "m" (dptr[z][d+128]),
+				       "m" (dptr[z][d+192]));
+		}
+		asm volatile("prefetchnta %0\n\t"
+			     "prefetchnta %1\n\t"
+			     :
+			     : "m" (q[d]), "m" (q[d+128]));
+		/* P/Q left side optimization */
+		for (z = start-1 ; z >= 0 ; z--) {
+			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
+				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
+				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
+				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
+				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
+				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
+				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
+				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
+				     "vpmovm2b %%k1,%%zmm5\n\t"
+				     "vpmovm2b %%k2,%%zmm7\n\t"
+				     "vpmovm2b %%k3,%%zmm13\n\t"
+				     "vpmovm2b %%k4,%%zmm15\n\t"
+				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
+				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
+				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
+				     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
+				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
+				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
+				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
+				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
+				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
+				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
+				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
+				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
+				     :
+				     : );
+		}
+		asm volatile("vmovntdq %%zmm2,%0\n\t"
+			     "vmovntdq %%zmm3,%1\n\t"
+			     "vmovntdq %%zmm10,%2\n\t"
+			     "vmovntdq %%zmm11,%3\n\t"
+			     "vpxorq %4,%%zmm4,%%zmm4\n\t"
+			     "vpxorq %5,%%zmm6,%%zmm6\n\t"
+			     "vpxorq %6,%%zmm12,%%zmm12\n\t"
+			     "vpxorq %7,%%zmm14,%%zmm14\n\t"
+			     "vmovntdq %%zmm4,%4\n\t"
+			     "vmovntdq %%zmm6,%5\n\t"
+			     "vmovntdq %%zmm12,%6\n\t"
+			     "vmovntdq %%zmm14,%7"
+			     :
+			     : "m" (p[d]),  "m" (p[d+64]), "m" (p[d+128]),
+			       "m" (p[d+192]), "m" (q[d]),  "m" (q[d+64]),
+			       "m" (q[d+128]), "m" (q[d+192]));
+	}
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+const struct raid6_calls raid6_avx512x4 = {
+	.gen_syndrome	= raid6_avx5124_gen_syndrome,
+	.xor_syndrome	= raid6_avx5124_xor_syndrome,
+	.name		= "avx512x4",
+};
+#endif
diff --git a/lib/raid/raid6/x86/mmx.c b/lib/raid/raid6/x86/mmx.c
new file mode 100644
index 0000000000000..afa82536142da
--- /dev/null
+++ b/lib/raid/raid6/x86/mmx.c
@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ * MMX implementation of RAID-6 syndrome functions.
+ */
+
+#include <asm/cpufeature.h>
+#include <asm/fpu/api.h>
+#include "algos.h"
+
+/* Shared with raid6/sse1.c */
+const struct raid6_mmx_constants {
+	u64 x1d;
+} raid6_mmx_constants = {
+	0x1d1d1d1d1d1d1d1dULL,
+};
+
+/*
+ * Plain MMX implementation
+ */
+static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
+	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
+
+	for ( d = 0 ; d < bytes ; d += 8 ) {
+		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
+		asm volatile("movq %mm2,%mm4");	/* Q[0] */
+		for ( z = z0-1 ; z >= 0 ; z-- ) {
+			asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
+			asm volatile("pcmpgtb %mm4,%mm5");
+			asm volatile("paddb %mm4,%mm4");
+			asm volatile("pand %mm0,%mm5");
+			asm volatile("pxor %mm5,%mm4");
+			asm volatile("pxor %mm5,%mm5");
+			asm volatile("pxor %mm6,%mm2");
+			asm volatile("pxor %mm6,%mm4");
+		}
+		asm volatile("movq %%mm2,%0" : "=m" (p[d]));
+		asm volatile("pxor %mm2,%mm2");
+		asm volatile("movq %%mm4,%0" : "=m" (q[d]));
+		asm volatile("pxor %mm4,%mm4");
+	}
+
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_mmxx1 = {
+	.gen_syndrome	= raid6_mmx1_gen_syndrome,
+	.name		= "mmxx1",
+};
+
+/*
+ * Unrolled-by-2 MMX implementation
+ */
+static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
+	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
+	asm volatile("pxor %mm7,%mm7"); /* Zero temp */
+
+	for ( d = 0 ; d < bytes ; d += 16 ) {
+		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
+		asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
+		asm volatile("movq %mm2,%mm4"); /* Q[0] */
+		asm volatile("movq %mm3,%mm6"); /* Q[1] */
+		for ( z = z0-1 ; z >= 0 ; z-- ) {
+			asm volatile("pcmpgtb %mm4,%mm5");
+			asm volatile("pcmpgtb %mm6,%mm7");
+			asm volatile("paddb %mm4,%mm4");
+			asm volatile("paddb %mm6,%mm6");
+			asm volatile("pand %mm0,%mm5");
+			asm volatile("pand %mm0,%mm7");
+			asm volatile("pxor %mm5,%mm4");
+			asm volatile("pxor %mm7,%mm6");
+			asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
+			asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
+			asm volatile("pxor %mm5,%mm2");
+			asm volatile("pxor %mm7,%mm3");
+			asm volatile("pxor %mm5,%mm4");
+			asm volatile("pxor %mm7,%mm6");
+			asm volatile("pxor %mm5,%mm5");
+			asm volatile("pxor %mm7,%mm7");
+		}
+		asm volatile("movq %%mm2,%0" : "=m" (p[d]));
+		asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
+		asm volatile("movq %%mm4,%0" : "=m" (q[d]));
+		asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
+	}
+
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_mmxx2 = {
+	.gen_syndrome	= raid6_mmx2_gen_syndrome,
+	.name		= "mmxx2",
+};
diff --git a/lib/raid/raid6/x86/pq_arch.h b/lib/raid/raid6/x86/pq_arch.h
new file mode 100644
index 0000000000000..28e395eb04d1a
--- /dev/null
+++ b/lib/raid/raid6/x86/pq_arch.h
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/cpufeature.h>
+
+extern const struct raid6_calls raid6_mmxx1;
+extern const struct raid6_calls raid6_mmxx2;
+extern const struct raid6_calls raid6_sse1x1;
+extern const struct raid6_calls raid6_sse1x2;
+extern const struct raid6_calls raid6_sse2x1;
+extern const struct raid6_calls raid6_sse2x2;
+extern const struct raid6_calls raid6_sse2x4;
+extern const struct raid6_calls raid6_avx2x1;
+extern const struct raid6_calls raid6_avx2x2;
+extern const struct raid6_calls raid6_avx2x4;
+extern const struct raid6_calls raid6_avx512x1;
+extern const struct raid6_calls raid6_avx512x2;
+extern const struct raid6_calls raid6_avx512x4;
+
+extern const struct raid6_recov_calls raid6_recov_ssse3;
+extern const struct raid6_recov_calls raid6_recov_avx2;
+extern const struct raid6_recov_calls raid6_recov_avx512;
+
+static inline int raid6_has_avx512(void)
+{
+	return boot_cpu_has(X86_FEATURE_AVX2) &&
+		boot_cpu_has(X86_FEATURE_AVX) &&
+		boot_cpu_has(X86_FEATURE_AVX512F) &&
+		boot_cpu_has(X86_FEATURE_AVX512BW) &&
+		boot_cpu_has(X86_FEATURE_AVX512VL) &&
+		boot_cpu_has(X86_FEATURE_AVX512DQ);
+}
+
+static inline bool raid6_has_avx2(void)
+{
+	return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX);
+}
+
+static inline bool raid6_has_ssse3(void)
+{
+	return boot_cpu_has(X86_FEATURE_XMM) &&
+		boot_cpu_has(X86_FEATURE_XMM2) &&
+		boot_cpu_has(X86_FEATURE_SSSE3);
+}
+
+static inline bool raid6_has_sse2(void)
+{
+	return boot_cpu_has(X86_FEATURE_MMX) &&
+		    boot_cpu_has(X86_FEATURE_FXSR) &&
+		    boot_cpu_has(X86_FEATURE_XMM) &&
+		    boot_cpu_has(X86_FEATURE_XMM2);
+}
+
+static inline bool raid6_has_sse1_or_mmxext(void)
+{
+	return boot_cpu_has(X86_FEATURE_MMX) &&
+		(boot_cpu_has(X86_FEATURE_XMM) ||
+		 boot_cpu_has(X86_FEATURE_MMXEXT));
+}
+
+static __always_inline void __init arch_raid6_init(void)
+{
+	if (raid6_has_avx2()) {
+		if (raid6_has_avx512()) {
+			raid6_algo_add(&raid6_avx512x1);
+			raid6_algo_add(&raid6_avx512x2);
+			if (IS_ENABLED(CONFIG_X86_64))
+				raid6_algo_add(&raid6_avx512x4);
+		}
+		raid6_algo_add(&raid6_avx2x1);
+		raid6_algo_add(&raid6_avx2x2);
+		if (IS_ENABLED(CONFIG_X86_64))
+			raid6_algo_add(&raid6_avx2x4);
+	} else if (IS_ENABLED(CONFIG_X86_64) || raid6_has_sse2()) {
+		/* x86_64 can assume SSE2 as baseline */
+		raid6_algo_add(&raid6_sse2x1);
+		raid6_algo_add(&raid6_sse2x2);
+		if (IS_ENABLED(CONFIG_X86_64))
+			raid6_algo_add(&raid6_sse2x4);
+	} else {
+		raid6_algo_add_default();
+		if (raid6_has_sse1_or_mmxext()) {
+			raid6_algo_add(&raid6_sse1x1);
+			raid6_algo_add(&raid6_sse1x2);
+		} else if (boot_cpu_has(X86_FEATURE_MMX)) {
+			raid6_algo_add(&raid6_mmxx1);
+			raid6_algo_add(&raid6_mmxx2);
+		}
+	}
+
+	if (raid6_has_avx512())
+		raid6_recov_algo_add(&raid6_recov_avx512);
+	else if (raid6_has_avx2())
+		raid6_recov_algo_add(&raid6_recov_avx2);
+	else if (raid6_has_ssse3())
+		raid6_recov_algo_add(&raid6_recov_ssse3);
+}
diff --git a/lib/raid/raid6/x86/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c
new file mode 100644
index 0000000000000..a714a780a2d8f
--- /dev/null
+++ b/lib/raid/raid6/x86/recov_avx2.c
@@ -0,0 +1,307 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Intel Corporation
+ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
+ */
+
+#include <linux/mm.h>
+#include <linux/raid/pq.h>
+#include <asm/fpu/api.h>
+#include "algos.h"
+
+static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila,
+		int failb, void **ptrs)
+{
+	u8 *p, *q, *dp, *dq;
+	const u8 *pbmul;	/* P multiplier table for B data */
+	const u8 *qmul;		/* Q multiplier table (for both) */
+	const u8 x0f = 0x0f;
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/* Compute syndrome with zero for the missing data pages
+	   Use the dead data pages as temporary storage for
+	   delta p and delta q */
+	dp = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-2] = dp;
+	dq = (u8 *)ptrs[failb];
+	ptrs[failb] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dp;
+	ptrs[failb]   = dq;
+	ptrs[disks-2] = p;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
+		raid6_gfexp[failb]]];
+
+	kernel_fpu_begin();
+
+	/* ymm0 = x0f[16] */
+	asm volatile("vpbroadcastb %0, %%ymm7" : : "m" (x0f));
+
+	while (bytes) {
+#ifdef CONFIG_X86_64
+		asm volatile("vmovdqa %0, %%ymm1" : : "m" (q[0]));
+		asm volatile("vmovdqa %0, %%ymm9" : : "m" (q[32]));
+		asm volatile("vmovdqa %0, %%ymm0" : : "m" (p[0]));
+		asm volatile("vmovdqa %0, %%ymm8" : : "m" (p[32]));
+		asm volatile("vpxor %0, %%ymm1, %%ymm1" : : "m" (dq[0]));
+		asm volatile("vpxor %0, %%ymm9, %%ymm9" : : "m" (dq[32]));
+		asm volatile("vpxor %0, %%ymm0, %%ymm0" : : "m" (dp[0]));
+		asm volatile("vpxor %0, %%ymm8, %%ymm8" : : "m" (dp[32]));
+
+		/*
+		 * 1 = dq[0]  ^ q[0]
+		 * 9 = dq[32] ^ q[32]
+		 * 0 = dp[0]  ^ p[0]
+		 * 8 = dp[32] ^ p[32]
+		 */
+
+		asm volatile("vbroadcasti128 %0, %%ymm4" : : "m" (qmul[0]));
+		asm volatile("vbroadcasti128 %0, %%ymm5" : : "m" (qmul[16]));
+
+		asm volatile("vpsraw $4, %ymm1, %ymm3");
+		asm volatile("vpsraw $4, %ymm9, %ymm12");
+		asm volatile("vpand %ymm7, %ymm1, %ymm1");
+		asm volatile("vpand %ymm7, %ymm9, %ymm9");
+		asm volatile("vpand %ymm7, %ymm3, %ymm3");
+		asm volatile("vpand %ymm7, %ymm12, %ymm12");
+		asm volatile("vpshufb %ymm9, %ymm4, %ymm14");
+		asm volatile("vpshufb %ymm1, %ymm4, %ymm4");
+		asm volatile("vpshufb %ymm12, %ymm5, %ymm15");
+		asm volatile("vpshufb %ymm3, %ymm5, %ymm5");
+		asm volatile("vpxor %ymm14, %ymm15, %ymm15");
+		asm volatile("vpxor %ymm4, %ymm5, %ymm5");
+
+		/*
+		 * 5 = qx[0]
+		 * 15 = qx[32]
+		 */
+
+		asm volatile("vbroadcasti128 %0, %%ymm4" : : "m" (pbmul[0]));
+		asm volatile("vbroadcasti128 %0, %%ymm1" : : "m" (pbmul[16]));
+		asm volatile("vpsraw $4, %ymm0, %ymm2");
+		asm volatile("vpsraw $4, %ymm8, %ymm6");
+		asm volatile("vpand %ymm7, %ymm0, %ymm3");
+		asm volatile("vpand %ymm7, %ymm8, %ymm14");
+		asm volatile("vpand %ymm7, %ymm2, %ymm2");
+		asm volatile("vpand %ymm7, %ymm6, %ymm6");
+		asm volatile("vpshufb %ymm14, %ymm4, %ymm12");
+		asm volatile("vpshufb %ymm3, %ymm4, %ymm4");
+		asm volatile("vpshufb %ymm6, %ymm1, %ymm13");
+		asm volatile("vpshufb %ymm2, %ymm1, %ymm1");
+		asm volatile("vpxor %ymm4, %ymm1, %ymm1");
+		asm volatile("vpxor %ymm12, %ymm13, %ymm13");
+
+		/*
+		 * 1  = pbmul[px[0]]
+		 * 13 = pbmul[px[32]]
+		 */
+		asm volatile("vpxor %ymm5, %ymm1, %ymm1");
+		asm volatile("vpxor %ymm15, %ymm13, %ymm13");
+
+		/*
+		 * 1 = db = DQ
+		 * 13 = db[32] = DQ[32]
+		 */
+		asm volatile("vmovdqa %%ymm1, %0" : "=m" (dq[0]));
+		asm volatile("vmovdqa %%ymm13,%0" : "=m" (dq[32]));
+		asm volatile("vpxor %ymm1, %ymm0, %ymm0");
+		asm volatile("vpxor %ymm13, %ymm8, %ymm8");
+
+		asm volatile("vmovdqa %%ymm0, %0" : "=m" (dp[0]));
+		asm volatile("vmovdqa %%ymm8, %0" : "=m" (dp[32]));
+
+		bytes -= 64;
+		p += 64;
+		q += 64;
+		dp += 64;
+		dq += 64;
+#else
+		asm volatile("vmovdqa %0, %%ymm1" : : "m" (*q));
+		asm volatile("vmovdqa %0, %%ymm0" : : "m" (*p));
+		asm volatile("vpxor %0, %%ymm1, %%ymm1" : : "m" (*dq));
+		asm volatile("vpxor %0, %%ymm0, %%ymm0" : : "m" (*dp));
+
+		/* 1 = dq ^ q;  0 = dp ^ p */
+
+		asm volatile("vbroadcasti128 %0, %%ymm4" : : "m" (qmul[0]));
+		asm volatile("vbroadcasti128 %0, %%ymm5" : : "m" (qmul[16]));
+
+		/*
+		 * 1 = dq ^ q
+		 * 3 = dq ^ p >> 4
+		 */
+		asm volatile("vpsraw $4, %ymm1, %ymm3");
+		asm volatile("vpand %ymm7, %ymm1, %ymm1");
+		asm volatile("vpand %ymm7, %ymm3, %ymm3");
+		asm volatile("vpshufb %ymm1, %ymm4, %ymm4");
+		asm volatile("vpshufb %ymm3, %ymm5, %ymm5");
+		asm volatile("vpxor %ymm4, %ymm5, %ymm5");
+
+		/* 5 = qx */
+
+		asm volatile("vbroadcasti128 %0, %%ymm4" : : "m" (pbmul[0]));
+		asm volatile("vbroadcasti128 %0, %%ymm1" : : "m" (pbmul[16]));
+
+		asm volatile("vpsraw $4, %ymm0, %ymm2");
+		asm volatile("vpand %ymm7, %ymm0, %ymm3");
+		asm volatile("vpand %ymm7, %ymm2, %ymm2");
+		asm volatile("vpshufb %ymm3, %ymm4, %ymm4");
+		asm volatile("vpshufb %ymm2, %ymm1, %ymm1");
+		asm volatile("vpxor %ymm4, %ymm1, %ymm1");
+
+		/* 1 = pbmul[px] */
+		asm volatile("vpxor %ymm5, %ymm1, %ymm1");
+		/* 1 = db = DQ */
+		asm volatile("vmovdqa %%ymm1, %0" : "=m" (dq[0]));
+
+		asm volatile("vpxor %ymm1, %ymm0, %ymm0");
+		asm volatile("vmovdqa %%ymm0, %0" : "=m" (dp[0]));
+
+		bytes -= 32;
+		p += 32;
+		q += 32;
+		dp += 32;
+		dq += 32;
+#endif
+	}
+
+	kernel_fpu_end();
+}
+
+static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila,
+		void **ptrs)
+{
+	u8 *p, *q, *dq;
+	const u8 *qmul;		/* Q multiplier table */
+	const u8 x0f = 0x0f;
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/* Compute syndrome with zero for the missing data page
+	   Use the dead data page as temporary storage for delta q */
+	dq = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dq;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+	kernel_fpu_begin();
+
+	asm volatile("vpbroadcastb %0, %%ymm7" : : "m" (x0f));
+
+	while (bytes) {
+#ifdef CONFIG_X86_64
+		asm volatile("vmovdqa %0, %%ymm3" : : "m" (dq[0]));
+		asm volatile("vmovdqa %0, %%ymm8" : : "m" (dq[32]));
+		asm volatile("vpxor %0, %%ymm3, %%ymm3" : : "m" (q[0]));
+		asm volatile("vpxor %0, %%ymm8, %%ymm8" : : "m" (q[32]));
+
+		/*
+		 * 3 = q[0] ^ dq[0]
+		 * 8 = q[32] ^ dq[32]
+		 */
+		asm volatile("vbroadcasti128 %0, %%ymm0" : : "m" (qmul[0]));
+		asm volatile("vmovapd %ymm0, %ymm13");
+		asm volatile("vbroadcasti128 %0, %%ymm1" : : "m" (qmul[16]));
+		asm volatile("vmovapd %ymm1, %ymm14");
+
+		asm volatile("vpsraw $4, %ymm3, %ymm6");
+		asm volatile("vpsraw $4, %ymm8, %ymm12");
+		asm volatile("vpand %ymm7, %ymm3, %ymm3");
+		asm volatile("vpand %ymm7, %ymm8, %ymm8");
+		asm volatile("vpand %ymm7, %ymm6, %ymm6");
+		asm volatile("vpand %ymm7, %ymm12, %ymm12");
+		asm volatile("vpshufb %ymm3, %ymm0, %ymm0");
+		asm volatile("vpshufb %ymm8, %ymm13, %ymm13");
+		asm volatile("vpshufb %ymm6, %ymm1, %ymm1");
+		asm volatile("vpshufb %ymm12, %ymm14, %ymm14");
+		asm volatile("vpxor %ymm0, %ymm1, %ymm1");
+		asm volatile("vpxor %ymm13, %ymm14, %ymm14");
+
+		/*
+		 * 1  = qmul[q[0]  ^ dq[0]]
+		 * 14 = qmul[q[32] ^ dq[32]]
+		 */
+		asm volatile("vmovdqa %0, %%ymm2" : : "m" (p[0]));
+		asm volatile("vmovdqa %0, %%ymm12" : : "m" (p[32]));
+		asm volatile("vpxor %ymm1, %ymm2, %ymm2");
+		asm volatile("vpxor %ymm14, %ymm12, %ymm12");
+
+		/*
+		 * 2  = p[0]  ^ qmul[q[0]  ^ dq[0]]
+		 * 12 = p[32] ^ qmul[q[32] ^ dq[32]]
+		 */
+
+		asm volatile("vmovdqa %%ymm1, %0" : "=m" (dq[0]));
+		asm volatile("vmovdqa %%ymm14, %0" : "=m" (dq[32]));
+		asm volatile("vmovdqa %%ymm2, %0" : "=m" (p[0]));
+		asm volatile("vmovdqa %%ymm12,%0" : "=m" (p[32]));
+
+		bytes -= 64;
+		p += 64;
+		q += 64;
+		dq += 64;
+#else
+		asm volatile("vmovdqa %0, %%ymm3" : : "m" (dq[0]));
+		asm volatile("vpxor %0, %%ymm3, %%ymm3" : : "m" (q[0]));
+
+		/* 3 = q ^ dq */
+
+		asm volatile("vbroadcasti128 %0, %%ymm0" : : "m" (qmul[0]));
+		asm volatile("vbroadcasti128 %0, %%ymm1" : : "m" (qmul[16]));
+
+		asm volatile("vpsraw $4, %ymm3, %ymm6");
+		asm volatile("vpand %ymm7, %ymm3, %ymm3");
+		asm volatile("vpand %ymm7, %ymm6, %ymm6");
+		asm volatile("vpshufb %ymm3, %ymm0, %ymm0");
+		asm volatile("vpshufb %ymm6, %ymm1, %ymm1");
+		asm volatile("vpxor %ymm0, %ymm1, %ymm1");
+
+		/* 1 = qmul[q ^ dq] */
+
+		asm volatile("vmovdqa %0, %%ymm2" : : "m" (p[0]));
+		asm volatile("vpxor %ymm1, %ymm2, %ymm2");
+
+		/* 2 = p ^ qmul[q ^ dq] */
+
+		asm volatile("vmovdqa %%ymm1, %0" : "=m" (dq[0]));
+		asm volatile("vmovdqa %%ymm2, %0" : "=m" (p[0]));
+
+		bytes -= 32;
+		p += 32;
+		q += 32;
+		dq += 32;
+#endif
+	}
+
+	kernel_fpu_end();
+}
+
+const struct raid6_recov_calls raid6_recov_avx2 = {
+	.data2 = raid6_2data_recov_avx2,
+	.datap = raid6_datap_recov_avx2,
+#ifdef CONFIG_X86_64
+	.name = "avx2x2",
+#else
+	.name = "avx2x1",
+#endif
+};
diff --git a/lib/raid/raid6/x86/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c
new file mode 100644
index 0000000000000..ec72d5a30c01e
--- /dev/null
+++ b/lib/raid/raid6/x86/recov_avx512.c
@@ -0,0 +1,367 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2016 Intel Corporation
+ *
+ * Author: Gayatri Kammela <gayatri.kammela@intel.com>
+ * Author: Megha Dey <megha.dey@linux.intel.com>
+ */
+
+#include <linux/mm.h>
+#include <linux/raid/pq.h>
+#include <asm/fpu/api.h>
+#include "algos.h"
+
+static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila,
+				     int failb, void **ptrs)
+{
+	u8 *p, *q, *dp, *dq;
+	const u8 *pbmul;	/* P multiplier table for B data */
+	const u8 *qmul;		/* Q multiplier table (for both) */
+	const u8 x0f = 0x0f;
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/*
+	 * Compute syndrome with zero for the missing data pages
+	 * Use the dead data pages as temporary storage for
+	 * delta p and delta q
+	 */
+
+	dp = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-2] = dp;
+	dq = (u8 *)ptrs[failb];
+	ptrs[failb] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dp;
+	ptrs[failb]   = dq;
+	ptrs[disks-2] = p;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
+		raid6_gfexp[failb]]];
+
+	kernel_fpu_begin();
+
+	/* zmm0 = x0f[16] */
+	asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));
+
+	while (bytes) {
+#ifdef CONFIG_X86_64
+		asm volatile("vmovdqa64 %0, %%zmm1\n\t"
+			     "vmovdqa64 %1, %%zmm9\n\t"
+			     "vmovdqa64 %2, %%zmm0\n\t"
+			     "vmovdqa64 %3, %%zmm8\n\t"
+			     "vpxorq %4, %%zmm1, %%zmm1\n\t"
+			     "vpxorq %5, %%zmm9, %%zmm9\n\t"
+			     "vpxorq %6, %%zmm0, %%zmm0\n\t"
+			     "vpxorq %7, %%zmm8, %%zmm8"
+			     :
+			     : "m" (q[0]), "m" (q[64]), "m" (p[0]),
+			       "m" (p[64]), "m" (dq[0]), "m" (dq[64]),
+			       "m" (dp[0]), "m" (dp[64]));
+
+		/*
+		 * 1 = dq[0]  ^ q[0]
+		 * 9 = dq[64] ^ q[64]
+		 * 0 = dp[0]  ^ p[0]
+		 * 8 = dp[64] ^ p[64]
+		 */
+
+		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
+			     "vbroadcasti64x2 %1, %%zmm5"
+			     :
+			     : "m" (qmul[0]), "m" (qmul[16]));
+
+		asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t"
+			     "vpsraw $4, %%zmm9, %%zmm12\n\t"
+			     "vpandq %%zmm7, %%zmm1, %%zmm1\n\t"
+			     "vpandq %%zmm7, %%zmm9, %%zmm9\n\t"
+			     "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
+			     "vpandq %%zmm7, %%zmm12, %%zmm12\n\t"
+			     "vpshufb %%zmm9, %%zmm4, %%zmm14\n\t"
+			     "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t"
+			     "vpshufb %%zmm12, %%zmm5, %%zmm15\n\t"
+			     "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t"
+			     "vpxorq %%zmm14, %%zmm15, %%zmm15\n\t"
+			     "vpxorq %%zmm4, %%zmm5, %%zmm5"
+			     :
+			     : );
+
+		/*
+		 * 5 = qx[0]
+		 * 15 = qx[64]
+		 */
+
+		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
+			     "vbroadcasti64x2 %1, %%zmm1\n\t"
+			     "vpsraw $4, %%zmm0, %%zmm2\n\t"
+			     "vpsraw $4, %%zmm8, %%zmm6\n\t"
+			     "vpandq %%zmm7, %%zmm0, %%zmm3\n\t"
+			     "vpandq %%zmm7, %%zmm8, %%zmm14\n\t"
+			     "vpandq %%zmm7, %%zmm2, %%zmm2\n\t"
+			     "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
+			     "vpshufb %%zmm14, %%zmm4, %%zmm12\n\t"
+			     "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t"
+			     "vpshufb %%zmm6, %%zmm1, %%zmm13\n\t"
+			     "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t"
+			     "vpxorq %%zmm4, %%zmm1, %%zmm1\n\t"
+			     "vpxorq %%zmm12, %%zmm13, %%zmm13"
+			     :
+			     : "m" (pbmul[0]), "m" (pbmul[16]));
+
+		/*
+		 * 1  = pbmul[px[0]]
+		 * 13 = pbmul[px[64]]
+		 */
+		asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t"
+			     "vpxorq %%zmm15, %%zmm13, %%zmm13"
+			     :
+			     : );
+
+		/*
+		 * 1 = db = DQ
+		 * 13 = db[64] = DQ[64]
+		 */
+		asm volatile("vmovdqa64 %%zmm1, %0\n\t"
+			     "vmovdqa64 %%zmm13,%1\n\t"
+			     "vpxorq %%zmm1, %%zmm0, %%zmm0\n\t"
+			     "vpxorq %%zmm13, %%zmm8, %%zmm8"
+			     :
+			     : "m" (dq[0]), "m" (dq[64]));
+
+		asm volatile("vmovdqa64 %%zmm0, %0\n\t"
+			     "vmovdqa64 %%zmm8, %1"
+			     :
+			     : "m" (dp[0]), "m" (dp[64]));
+
+		bytes -= 128;
+		p += 128;
+		q += 128;
+		dp += 128;
+		dq += 128;
+#else
+		asm volatile("vmovdqa64 %0, %%zmm1\n\t"
+			     "vmovdqa64 %1, %%zmm0\n\t"
+			     "vpxorq %2, %%zmm1, %%zmm1\n\t"
+			     "vpxorq %3, %%zmm0, %%zmm0"
+			     :
+			     : "m" (*q), "m" (*p), "m"(*dq), "m" (*dp));
+
+		/* 1 = dq ^ q;  0 = dp ^ p */
+
+		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
+			     "vbroadcasti64x2 %1, %%zmm5"
+			     :
+			     : "m" (qmul[0]), "m" (qmul[16]));
+
+		/*
+		 * 1 = dq ^ q
+		 * 3 = dq ^ p >> 4
+		 */
+		asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t"
+			     "vpandq %%zmm7, %%zmm1, %%zmm1\n\t"
+			     "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
+			     "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t"
+			     "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t"
+			     "vpxorq %%zmm4, %%zmm5, %%zmm5"
+			     :
+			     : );
+
+		/* 5 = qx */
+
+		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
+			     "vbroadcasti64x2 %1, %%zmm1"
+			     :
+			     : "m" (pbmul[0]), "m" (pbmul[16]));
+
+		asm volatile("vpsraw $4, %%zmm0, %%zmm2\n\t"
+			     "vpandq %%zmm7, %%zmm0, %%zmm3\n\t"
+			     "vpandq %%zmm7, %%zmm2, %%zmm2\n\t"
+			     "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t"
+			     "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t"
+			     "vpxorq %%zmm4, %%zmm1, %%zmm1"
+			     :
+			     : );
+
+		/* 1 = pbmul[px] */
+		asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t"
+			     /* 1 = db = DQ */
+			     "vmovdqa64 %%zmm1, %0\n\t"
+			     :
+			     : "m" (dq[0]));
+
+		asm volatile("vpxorq %%zmm1, %%zmm0, %%zmm0\n\t"
+			     "vmovdqa64 %%zmm0, %0"
+			     :
+			     : "m" (dp[0]));
+
+		bytes -= 64;
+		p += 64;
+		q += 64;
+		dp += 64;
+		dq += 64;
+#endif
+	}
+
+	kernel_fpu_end();
+}
+
+static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,
+				     void **ptrs)
+{
+	u8 *p, *q, *dq;
+	const u8 *qmul;		/* Q multiplier table */
+	const u8 x0f = 0x0f;
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/*
+	 * Compute syndrome with zero for the missing data page
+	 * Use the dead data page as temporary storage for delta q
+	 */
+
+	dq = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dq;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+	kernel_fpu_begin();
+
+	asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));
+
+	while (bytes) {
+#ifdef CONFIG_X86_64
+		asm volatile("vmovdqa64 %0, %%zmm3\n\t"
+			     "vmovdqa64 %1, %%zmm8\n\t"
+			     "vpxorq %2, %%zmm3, %%zmm3\n\t"
+			     "vpxorq %3, %%zmm8, %%zmm8"
+			     :
+			     : "m" (dq[0]), "m" (dq[64]), "m" (q[0]),
+			       "m" (q[64]));
+
+		/*
+		 * 3 = q[0] ^ dq[0]
+		 * 8 = q[64] ^ dq[64]
+		 */
+		asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"
+			     "vmovapd %%zmm0, %%zmm13\n\t"
+			     "vbroadcasti64x2 %1, %%zmm1\n\t"
+			     "vmovapd %%zmm1, %%zmm14"
+			     :
+			     : "m" (qmul[0]), "m" (qmul[16]));
+
+		asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t"
+			     "vpsraw $4, %%zmm8, %%zmm12\n\t"
+			     "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
+			     "vpandq %%zmm7, %%zmm8, %%zmm8\n\t"
+			     "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
+			     "vpandq %%zmm7, %%zmm12, %%zmm12\n\t"
+			     "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t"
+			     "vpshufb %%zmm8, %%zmm13, %%zmm13\n\t"
+			     "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t"
+			     "vpshufb %%zmm12, %%zmm14, %%zmm14\n\t"
+			     "vpxorq %%zmm0, %%zmm1, %%zmm1\n\t"
+			     "vpxorq %%zmm13, %%zmm14, %%zmm14"
+			     :
+			     : );
+
+		/*
+		 * 1  = qmul[q[0]  ^ dq[0]]
+		 * 14 = qmul[q[64] ^ dq[64]]
+		 */
+		asm volatile("vmovdqa64 %0, %%zmm2\n\t"
+			     "vmovdqa64 %1, %%zmm12\n\t"
+			     "vpxorq %%zmm1, %%zmm2, %%zmm2\n\t"
+			     "vpxorq %%zmm14, %%zmm12, %%zmm12"
+			     :
+			     : "m" (p[0]), "m" (p[64]));
+
+		/*
+		 * 2  = p[0]  ^ qmul[q[0]  ^ dq[0]]
+		 * 12 = p[64] ^ qmul[q[64] ^ dq[64]]
+		 */
+
+		asm volatile("vmovdqa64 %%zmm1, %0\n\t"
+			     "vmovdqa64 %%zmm14, %1\n\t"
+			     "vmovdqa64 %%zmm2, %2\n\t"
+			     "vmovdqa64 %%zmm12,%3"
+			     :
+			     : "m" (dq[0]), "m" (dq[64]), "m" (p[0]),
+			       "m" (p[64]));
+
+		bytes -= 128;
+		p += 128;
+		q += 128;
+		dq += 128;
+#else
+		asm volatile("vmovdqa64 %0, %%zmm3\n\t"
+			     "vpxorq %1, %%zmm3, %%zmm3"
+			     :
+			     : "m" (dq[0]), "m" (q[0]));
+
+		/* 3 = q ^ dq */
+
+		asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"
+			     "vbroadcasti64x2 %1, %%zmm1"
+			     :
+			     : "m" (qmul[0]), "m" (qmul[16]));
+
+		asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t"
+			     "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
+			     "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
+			     "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t"
+			     "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t"
+			     "vpxorq %%zmm0, %%zmm1, %%zmm1"
+			     :
+			     : );
+
+		/* 1 = qmul[q ^ dq] */
+
+		asm volatile("vmovdqa64 %0, %%zmm2\n\t"
+			     "vpxorq %%zmm1, %%zmm2, %%zmm2"
+			     :
+			     : "m" (p[0]));
+
+		/* 2 = p ^ qmul[q ^ dq] */
+
+		asm volatile("vmovdqa64 %%zmm1, %0\n\t"
+			     "vmovdqa64 %%zmm2, %1"
+			     :
+			     : "m" (dq[0]), "m" (p[0]));
+
+		bytes -= 64;
+		p += 64;
+		q += 64;
+		dq += 64;
+#endif
+	}
+
+	kernel_fpu_end();
+}
+
+const struct raid6_recov_calls raid6_recov_avx512 = {
+	.data2 = raid6_2data_recov_avx512,
+	.datap = raid6_datap_recov_avx512,
+#ifdef CONFIG_X86_64
+	.name = "avx512x2",
+#else
+	.name = "avx512x1",
+#endif
+};
diff --git a/lib/raid/raid6/x86/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c
new file mode 100644
index 0000000000000..700bd2c865ece
--- /dev/null
+++ b/lib/raid/raid6/x86/recov_ssse3.c
@@ -0,0 +1,321 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2012 Intel Corporation
+ */
+
+#include <linux/mm.h>
+#include <linux/raid/pq.h>
+#include <asm/fpu/api.h>
+#include "algos.h"
+
+static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila,
+		int failb, void **ptrs)
+{
+	u8 *p, *q, *dp, *dq;
+	const u8 *pbmul;	/* P multiplier table for B data */
+	const u8 *qmul;		/* Q multiplier table (for both) */
+	static const u8 __aligned(16) x0f[16] = {
+		 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
+		 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/* Compute syndrome with zero for the missing data pages
+	   Use the dead data pages as temporary storage for
+	   delta p and delta q */
+	dp = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-2] = dp;
+	dq = (u8 *)ptrs[failb];
+	ptrs[failb] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dp;
+	ptrs[failb]   = dq;
+	ptrs[disks-2] = p;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
+		raid6_gfexp[failb]]];
+
+	kernel_fpu_begin();
+
+	asm volatile("movdqa %0,%%xmm7" : : "m" (x0f[0]));
+
+#ifdef CONFIG_X86_64
+	asm volatile("movdqa %0,%%xmm6" : : "m" (qmul[0]));
+	asm volatile("movdqa %0,%%xmm14" : : "m" (pbmul[0]));
+	asm volatile("movdqa %0,%%xmm15" : : "m" (pbmul[16]));
+#endif
+
+	/* Now do it... */
+	while (bytes) {
+#ifdef CONFIG_X86_64
+		/* xmm6, xmm14, xmm15 */
+
+		asm volatile("movdqa %0,%%xmm1" : : "m" (q[0]));
+		asm volatile("movdqa %0,%%xmm9" : : "m" (q[16]));
+		asm volatile("movdqa %0,%%xmm0" : : "m" (p[0]));
+		asm volatile("movdqa %0,%%xmm8" : : "m" (p[16]));
+		asm volatile("pxor   %0,%%xmm1" : : "m" (dq[0]));
+		asm volatile("pxor   %0,%%xmm9" : : "m" (dq[16]));
+		asm volatile("pxor   %0,%%xmm0" : : "m" (dp[0]));
+		asm volatile("pxor   %0,%%xmm8" : : "m" (dp[16]));
+
+		/* xmm0/8 = px */
+
+		asm volatile("movdqa %xmm6,%xmm4");
+		asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
+		asm volatile("movdqa %xmm6,%xmm12");
+		asm volatile("movdqa %xmm5,%xmm13");
+		asm volatile("movdqa %xmm1,%xmm3");
+		asm volatile("movdqa %xmm9,%xmm11");
+		asm volatile("movdqa %xmm0,%xmm2"); /* xmm2/10 = px */
+		asm volatile("movdqa %xmm8,%xmm10");
+		asm volatile("psraw  $4,%xmm1");
+		asm volatile("psraw  $4,%xmm9");
+		asm volatile("pand   %xmm7,%xmm3");
+		asm volatile("pand   %xmm7,%xmm11");
+		asm volatile("pand   %xmm7,%xmm1");
+		asm volatile("pand   %xmm7,%xmm9");
+		asm volatile("pshufb %xmm3,%xmm4");
+		asm volatile("pshufb %xmm11,%xmm12");
+		asm volatile("pshufb %xmm1,%xmm5");
+		asm volatile("pshufb %xmm9,%xmm13");
+		asm volatile("pxor   %xmm4,%xmm5");
+		asm volatile("pxor   %xmm12,%xmm13");
+
+		/* xmm5/13 = qx */
+
+		asm volatile("movdqa %xmm14,%xmm4");
+		asm volatile("movdqa %xmm15,%xmm1");
+		asm volatile("movdqa %xmm14,%xmm12");
+		asm volatile("movdqa %xmm15,%xmm9");
+		asm volatile("movdqa %xmm2,%xmm3");
+		asm volatile("movdqa %xmm10,%xmm11");
+		asm volatile("psraw  $4,%xmm2");
+		asm volatile("psraw  $4,%xmm10");
+		asm volatile("pand   %xmm7,%xmm3");
+		asm volatile("pand   %xmm7,%xmm11");
+		asm volatile("pand   %xmm7,%xmm2");
+		asm volatile("pand   %xmm7,%xmm10");
+		asm volatile("pshufb %xmm3,%xmm4");
+		asm volatile("pshufb %xmm11,%xmm12");
+		asm volatile("pshufb %xmm2,%xmm1");
+		asm volatile("pshufb %xmm10,%xmm9");
+		asm volatile("pxor   %xmm4,%xmm1");
+		asm volatile("pxor   %xmm12,%xmm9");
+
+		/* xmm1/9 = pbmul[px] */
+		asm volatile("pxor   %xmm5,%xmm1");
+		asm volatile("pxor   %xmm13,%xmm9");
+		/* xmm1/9 = db = DQ */
+		asm volatile("movdqa %%xmm1,%0" : "=m" (dq[0]));
+		asm volatile("movdqa %%xmm9,%0" : "=m" (dq[16]));
+
+		asm volatile("pxor   %xmm1,%xmm0");
+		asm volatile("pxor   %xmm9,%xmm8");
+		asm volatile("movdqa %%xmm0,%0" : "=m" (dp[0]));
+		asm volatile("movdqa %%xmm8,%0" : "=m" (dp[16]));
+
+		bytes -= 32;
+		p += 32;
+		q += 32;
+		dp += 32;
+		dq += 32;
+#else
+		asm volatile("movdqa %0,%%xmm1" : : "m" (*q));
+		asm volatile("movdqa %0,%%xmm0" : : "m" (*p));
+		asm volatile("pxor   %0,%%xmm1" : : "m" (*dq));
+		asm volatile("pxor   %0,%%xmm0" : : "m" (*dp));
+
+		/* 1 = dq ^ q
+		 * 0 = dp ^ p
+		 */
+		asm volatile("movdqa %0,%%xmm4" : : "m" (qmul[0]));
+		asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
+
+		asm volatile("movdqa %xmm1,%xmm3");
+		asm volatile("psraw  $4,%xmm1");
+		asm volatile("pand   %xmm7,%xmm3");
+		asm volatile("pand   %xmm7,%xmm1");
+		asm volatile("pshufb %xmm3,%xmm4");
+		asm volatile("pshufb %xmm1,%xmm5");
+		asm volatile("pxor   %xmm4,%xmm5");
+
+		asm volatile("movdqa %xmm0,%xmm2"); /* xmm2 = px */
+
+		/* xmm5 = qx */
+
+		asm volatile("movdqa %0,%%xmm4" : : "m" (pbmul[0]));
+		asm volatile("movdqa %0,%%xmm1" : : "m" (pbmul[16]));
+		asm volatile("movdqa %xmm2,%xmm3");
+		asm volatile("psraw  $4,%xmm2");
+		asm volatile("pand   %xmm7,%xmm3");
+		asm volatile("pand   %xmm7,%xmm2");
+		asm volatile("pshufb %xmm3,%xmm4");
+		asm volatile("pshufb %xmm2,%xmm1");
+		asm volatile("pxor   %xmm4,%xmm1");
+
+		/* xmm1 = pbmul[px] */
+		asm volatile("pxor   %xmm5,%xmm1");
+		/* xmm1 = db = DQ */
+		asm volatile("movdqa %%xmm1,%0" : "=m" (*dq));
+
+		asm volatile("pxor   %xmm1,%xmm0");
+		asm volatile("movdqa %%xmm0,%0" : "=m" (*dp));
+
+		bytes -= 16;
+		p += 16;
+		q += 16;
+		dp += 16;
+		dq += 16;
+#endif
+	}
+
+	kernel_fpu_end();
+}
+
+
+static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila,
+		void **ptrs)
+{
+	u8 *p, *q, *dq;
+	const u8 *qmul;		/* Q multiplier table */
+	static const u8 __aligned(16) x0f[16] = {
+		 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
+		 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
+
+	p = (u8 *)ptrs[disks-2];
+	q = (u8 *)ptrs[disks-1];
+
+	/* Compute syndrome with zero for the missing data page
+	   Use the dead data page as temporary storage for delta q */
+	dq = (u8 *)ptrs[faila];
+	ptrs[faila] = page_address(ZERO_PAGE(0));
+	ptrs[disks-1] = dq;
+
+	raid6_gen_syndrome(disks, bytes, ptrs);
+
+	/* Restore pointer table */
+	ptrs[faila]   = dq;
+	ptrs[disks-1] = q;
+
+	/* Now, pick the proper data tables */
+	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
+
+	kernel_fpu_begin();
+
+	asm volatile("movdqa %0, %%xmm7" : : "m" (x0f[0]));
+
+	while (bytes) {
+#ifdef CONFIG_X86_64
+		asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
+		asm volatile("movdqa %0, %%xmm4" : : "m" (dq[16]));
+		asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
+		asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
+
+		/* xmm3 = q[0] ^ dq[0] */
+
+		asm volatile("pxor %0, %%xmm4" : : "m" (q[16]));
+		asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
+
+		/* xmm4 = q[16] ^ dq[16] */
+
+		asm volatile("movdqa %xmm3, %xmm6");
+		asm volatile("movdqa %xmm4, %xmm8");
+
+		/* xmm4 = xmm8 = q[16] ^ dq[16] */
+
+		asm volatile("psraw $4, %xmm3");
+		asm volatile("pand %xmm7, %xmm6");
+		asm volatile("pand %xmm7, %xmm3");
+		asm volatile("pshufb %xmm6, %xmm0");
+		asm volatile("pshufb %xmm3, %xmm1");
+		asm volatile("movdqa %0, %%xmm10" : : "m" (qmul[0]));
+		asm volatile("pxor %xmm0, %xmm1");
+		asm volatile("movdqa %0, %%xmm11" : : "m" (qmul[16]));
+
+		/* xmm1 = qmul[q[0] ^ dq[0]] */
+
+		asm volatile("psraw $4, %xmm4");
+		asm volatile("pand %xmm7, %xmm8");
+		asm volatile("pand %xmm7, %xmm4");
+		asm volatile("pshufb %xmm8, %xmm10");
+		asm volatile("pshufb %xmm4, %xmm11");
+		asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
+		asm volatile("pxor %xmm10, %xmm11");
+		asm volatile("movdqa %0, %%xmm12" : : "m" (p[16]));
+
+		/* xmm11 = qmul[q[16] ^ dq[16]] */
+
+		asm volatile("pxor %xmm1, %xmm2");
+
+		/* xmm2 = p[0] ^ qmul[q[0] ^ dq[0]] */
+
+		asm volatile("pxor %xmm11, %xmm12");
+
+		/* xmm12 = p[16] ^ qmul[q[16] ^ dq[16]] */
+
+		asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
+		asm volatile("movdqa %%xmm11, %0" : "=m" (dq[16]));
+
+		asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
+		asm volatile("movdqa %%xmm12, %0" : "=m" (p[16]));
+
+		bytes -= 32;
+		p += 32;
+		q += 32;
+		dq += 32;
+
+#else
+		asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
+		asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
+		asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
+		asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
+
+		/* xmm3 = *q ^ *dq */
+
+		asm volatile("movdqa %xmm3, %xmm6");
+		asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
+		asm volatile("psraw $4, %xmm3");
+		asm volatile("pand %xmm7, %xmm6");
+		asm volatile("pand %xmm7, %xmm3");
+		asm volatile("pshufb %xmm6, %xmm0");
+		asm volatile("pshufb %xmm3, %xmm1");
+		asm volatile("pxor %xmm0, %xmm1");
+
+		/* xmm1 = qmul[*q ^ *dq */
+
+		asm volatile("pxor %xmm1, %xmm2");
+
+		/* xmm2 = *p ^ qmul[*q ^ *dq] */
+
+		asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
+		asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
+
+		bytes -= 16;
+		p += 16;
+		q += 16;
+		dq += 16;
+#endif
+	}
+
+	kernel_fpu_end();
+}
+
+const struct raid6_recov_calls raid6_recov_ssse3 = {
+	.data2 = raid6_2data_recov_ssse3,
+	.datap = raid6_datap_recov_ssse3,
+#ifdef CONFIG_X86_64
+	.name = "ssse3x2",
+#else
+	.name = "ssse3x1",
+#endif
+};
diff --git a/lib/raid/raid6/x86/sse1.c b/lib/raid/raid6/x86/sse1.c
new file mode 100644
index 0000000000000..f4b260df522a3
--- /dev/null
+++ b/lib/raid/raid6/x86/sse1.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ * SSE-1/MMXEXT implementation of RAID-6 syndrome functions.
+ *
+ * This is really an MMX implementation, but it requires SSE-1 or AMD MMXEXT for
+ * prefetch support and a few other features.  The support for nontemporal
+ * memory accesses is enough to make this worthwhile as a separate
+ * implementation.
+ */
+
+#include <asm/cpufeature.h>
+#include <asm/fpu/api.h>
+#include "algos.h"
+
+/* Defined in raid6/mmx.c */
+extern const struct raid6_mmx_constants {
+	u64 x1d;
+} raid6_mmx_constants;
+
+/*
+ * Plain SSE1 implementation
+ */
+static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
+	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
+
+	for ( d = 0 ; d < bytes ; d += 8 ) {
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
+		asm volatile("movq %mm2,%mm4");	/* Q[0] */
+		asm volatile("movq %0,%%mm6" : : "m" (dptr[z0-1][d]));
+		for ( z = z0-2 ; z >= 0 ; z-- ) {
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+			asm volatile("pcmpgtb %mm4,%mm5");
+			asm volatile("paddb %mm4,%mm4");
+			asm volatile("pand %mm0,%mm5");
+			asm volatile("pxor %mm5,%mm4");
+			asm volatile("pxor %mm5,%mm5");
+			asm volatile("pxor %mm6,%mm2");
+			asm volatile("pxor %mm6,%mm4");
+			asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
+		}
+		asm volatile("pcmpgtb %mm4,%mm5");
+		asm volatile("paddb %mm4,%mm4");
+		asm volatile("pand %mm0,%mm5");
+		asm volatile("pxor %mm5,%mm4");
+		asm volatile("pxor %mm5,%mm5");
+		asm volatile("pxor %mm6,%mm2");
+		asm volatile("pxor %mm6,%mm4");
+
+		asm volatile("movntq %%mm2,%0" : "=m" (p[d]));
+		asm volatile("movntq %%mm4,%0" : "=m" (q[d]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_sse1x1 = {
+	.gen_syndrome	= raid6_sse11_gen_syndrome,
+	.name		= "sse1x1",
+};
+
+/*
+ * Unrolled-by-2 SSE1 implementation
+ */
+static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
+	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
+	asm volatile("pxor %mm7,%mm7"); /* Zero temp */
+
+	/* We uniformly assume a single prefetch covers at least 16 bytes */
+	for ( d = 0 ; d < bytes ; d += 16 ) {
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
+		asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8])); /* P[1] */
+		asm volatile("movq %mm2,%mm4");	/* Q[0] */
+		asm volatile("movq %mm3,%mm6"); /* Q[1] */
+		for ( z = z0-1 ; z >= 0 ; z-- ) {
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+			asm volatile("pcmpgtb %mm4,%mm5");
+			asm volatile("pcmpgtb %mm6,%mm7");
+			asm volatile("paddb %mm4,%mm4");
+			asm volatile("paddb %mm6,%mm6");
+			asm volatile("pand %mm0,%mm5");
+			asm volatile("pand %mm0,%mm7");
+			asm volatile("pxor %mm5,%mm4");
+			asm volatile("pxor %mm7,%mm6");
+			asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
+			asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
+			asm volatile("pxor %mm5,%mm2");
+			asm volatile("pxor %mm7,%mm3");
+			asm volatile("pxor %mm5,%mm4");
+			asm volatile("pxor %mm7,%mm6");
+			asm volatile("pxor %mm5,%mm5");
+			asm volatile("pxor %mm7,%mm7");
+		}
+		asm volatile("movntq %%mm2,%0" : "=m" (p[d]));
+		asm volatile("movntq %%mm3,%0" : "=m" (p[d+8]));
+		asm volatile("movntq %%mm4,%0" : "=m" (q[d]));
+		asm volatile("movntq %%mm6,%0" : "=m" (q[d+8]));
+	}
+
+	asm volatile("sfence" : :: "memory");
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_sse1x2 = {
+	.gen_syndrome	= raid6_sse12_gen_syndrome,
+	.name		= "sse1x2",
+};
diff --git a/lib/raid/raid6/x86/sse2.c b/lib/raid/raid6/x86/sse2.c
new file mode 100644
index 0000000000000..43b09ce58270a
--- /dev/null
+++ b/lib/raid/raid6/x86/sse2.c
@@ -0,0 +1,459 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright 2002 H. Peter Anvin - All Rights Reserved
+ *
+ * SSE-2 implementation of RAID-6 syndrome functions
+ */
+
+#include <asm/cpufeature.h>
+#include <asm/fpu/api.h>
+#include "algos.h"
+
+static const struct raid6_sse_constants {
+	u64 x1d[2];
+} raid6_sse_constants  __attribute__((aligned(16))) = {
+	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
+};
+
+/*
+ * Plain SSE2 implementation
+ */
+static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
+	asm volatile("pxor %xmm5,%xmm5");	/* Zero temp */
+
+	for ( d = 0 ; d < bytes ; d += 16 ) {
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+		asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
+		asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
+		asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d]));
+		for ( z = z0-2 ; z >= 0 ; z-- ) {
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+			asm volatile("pcmpgtb %xmm4,%xmm5");
+			asm volatile("paddb %xmm4,%xmm4");
+			asm volatile("pand %xmm0,%xmm5");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm5,%xmm5");
+			asm volatile("pxor %xmm6,%xmm2");
+			asm volatile("pxor %xmm6,%xmm4");
+			asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
+		}
+		asm volatile("pcmpgtb %xmm4,%xmm5");
+		asm volatile("paddb %xmm4,%xmm4");
+		asm volatile("pand %xmm0,%xmm5");
+		asm volatile("pxor %xmm5,%xmm4");
+		asm volatile("pxor %xmm5,%xmm5");
+		asm volatile("pxor %xmm6,%xmm2");
+		asm volatile("pxor %xmm6,%xmm4");
+
+		asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
+		asm volatile("pxor %xmm2,%xmm2");
+		asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
+		asm volatile("pxor %xmm4,%xmm4");
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+
+static void raid6_sse21_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
+
+	for ( d = 0 ; d < bytes ; d += 16 ) {
+		asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
+		asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
+		asm volatile("pxor %xmm4,%xmm2");
+		/* P/Q data pages */
+		for ( z = z0-1 ; z >= start ; z-- ) {
+			asm volatile("pxor %xmm5,%xmm5");
+			asm volatile("pcmpgtb %xmm4,%xmm5");
+			asm volatile("paddb %xmm4,%xmm4");
+			asm volatile("pand %xmm0,%xmm5");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
+			asm volatile("pxor %xmm5,%xmm2");
+			asm volatile("pxor %xmm5,%xmm4");
+		}
+		/* P/Q left side optimization */
+		for ( z = start-1 ; z >= 0 ; z-- ) {
+			asm volatile("pxor %xmm5,%xmm5");
+			asm volatile("pcmpgtb %xmm4,%xmm5");
+			asm volatile("paddb %xmm4,%xmm4");
+			asm volatile("pand %xmm0,%xmm5");
+			asm volatile("pxor %xmm5,%xmm4");
+		}
+		asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
+		/* Don't use movntdq for r/w memory area < cache line */
+		asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
+		asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_sse2x1 = {
+	.gen_syndrome	= raid6_sse21_gen_syndrome,
+	.xor_syndrome	= raid6_sse21_xor_syndrome,
+	.name		= "sse2x1",
+};
+
+/*
+ * Unrolled-by-2 SSE2 implementation
+ */
+static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
+	asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
+	asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
+
+	/* We uniformly assume a single prefetch covers at least 32 bytes */
+	for ( d = 0 ; d < bytes ; d += 32 ) {
+		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
+		asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d]));    /* P[0] */
+		asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16])); /* P[1] */
+		asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
+		asm volatile("movdqa %xmm3,%xmm6"); /* Q[1] */
+		for ( z = z0-1 ; z >= 0 ; z-- ) {
+			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
+			asm volatile("pcmpgtb %xmm4,%xmm5");
+			asm volatile("pcmpgtb %xmm6,%xmm7");
+			asm volatile("paddb %xmm4,%xmm4");
+			asm volatile("paddb %xmm6,%xmm6");
+			asm volatile("pand %xmm0,%xmm5");
+			asm volatile("pand %xmm0,%xmm7");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+			asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
+			asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
+			asm volatile("pxor %xmm5,%xmm2");
+			asm volatile("pxor %xmm7,%xmm3");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+			asm volatile("pxor %xmm5,%xmm5");
+			asm volatile("pxor %xmm7,%xmm7");
+		}
+		asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
+		asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
+		asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
+		asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+static void raid6_sse22_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
+
+	for ( d = 0 ; d < bytes ; d += 32 ) {
+		asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
+		asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
+		asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
+		asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
+		asm volatile("pxor %xmm4,%xmm2");
+		asm volatile("pxor %xmm6,%xmm3");
+		/* P/Q data pages */
+		for ( z = z0-1 ; z >= start ; z-- ) {
+			asm volatile("pxor %xmm5,%xmm5");
+			asm volatile("pxor %xmm7,%xmm7");
+			asm volatile("pcmpgtb %xmm4,%xmm5");
+			asm volatile("pcmpgtb %xmm6,%xmm7");
+			asm volatile("paddb %xmm4,%xmm4");
+			asm volatile("paddb %xmm6,%xmm6");
+			asm volatile("pand %xmm0,%xmm5");
+			asm volatile("pand %xmm0,%xmm7");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+			asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
+			asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
+			asm volatile("pxor %xmm5,%xmm2");
+			asm volatile("pxor %xmm7,%xmm3");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+		}
+		/* P/Q left side optimization */
+		for ( z = start-1 ; z >= 0 ; z-- ) {
+			asm volatile("pxor %xmm5,%xmm5");
+			asm volatile("pxor %xmm7,%xmm7");
+			asm volatile("pcmpgtb %xmm4,%xmm5");
+			asm volatile("pcmpgtb %xmm6,%xmm7");
+			asm volatile("paddb %xmm4,%xmm4");
+			asm volatile("paddb %xmm6,%xmm6");
+			asm volatile("pand %xmm0,%xmm5");
+			asm volatile("pand %xmm0,%xmm7");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+		}
+		asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
+		asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
+		/* Don't use movntdq for r/w memory area < cache line */
+		asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
+		asm volatile("movdqa %%xmm6,%0" : "=m" (q[d+16]));
+		asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
+		asm volatile("movdqa %%xmm3,%0" : "=m" (p[d+16]));
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+const struct raid6_calls raid6_sse2x2 = {
+	.gen_syndrome	= raid6_sse22_gen_syndrome,
+	.xor_syndrome	= raid6_sse22_xor_syndrome,
+	.name		= "sse2x2",
+};
+
+#ifdef CONFIG_X86_64
+
+/*
+ * Unrolled-by-4 SSE2 implementation
+ */
+static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = disks - 3;		/* Highest data disk */
+	p = dptr[z0+1];		/* XOR parity */
+	q = dptr[z0+2];		/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
+	asm volatile("pxor %xmm2,%xmm2");	/* P[0] */
+	asm volatile("pxor %xmm3,%xmm3");	/* P[1] */
+	asm volatile("pxor %xmm4,%xmm4"); 	/* Q[0] */
+	asm volatile("pxor %xmm5,%xmm5");	/* Zero temp */
+	asm volatile("pxor %xmm6,%xmm6"); 	/* Q[1] */
+	asm volatile("pxor %xmm7,%xmm7"); 	/* Zero temp */
+	asm volatile("pxor %xmm10,%xmm10");	/* P[2] */
+	asm volatile("pxor %xmm11,%xmm11");	/* P[3] */
+	asm volatile("pxor %xmm12,%xmm12"); 	/* Q[2] */
+	asm volatile("pxor %xmm13,%xmm13");	/* Zero temp */
+	asm volatile("pxor %xmm14,%xmm14"); 	/* Q[3] */
+	asm volatile("pxor %xmm15,%xmm15"); 	/* Zero temp */
+
+	for ( d = 0 ; d < bytes ; d += 64 ) {
+		for ( z = z0 ; z >= 0 ; z-- ) {
+			/* The second prefetch seems to improve performance... */
+			asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
+			asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
+			asm volatile("pcmpgtb %xmm4,%xmm5");
+			asm volatile("pcmpgtb %xmm6,%xmm7");
+			asm volatile("pcmpgtb %xmm12,%xmm13");
+			asm volatile("pcmpgtb %xmm14,%xmm15");
+			asm volatile("paddb %xmm4,%xmm4");
+			asm volatile("paddb %xmm6,%xmm6");
+			asm volatile("paddb %xmm12,%xmm12");
+			asm volatile("paddb %xmm14,%xmm14");
+			asm volatile("pand %xmm0,%xmm5");
+			asm volatile("pand %xmm0,%xmm7");
+			asm volatile("pand %xmm0,%xmm13");
+			asm volatile("pand %xmm0,%xmm15");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+			asm volatile("pxor %xmm13,%xmm12");
+			asm volatile("pxor %xmm15,%xmm14");
+			asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
+			asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
+			asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
+			asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
+			asm volatile("pxor %xmm5,%xmm2");
+			asm volatile("pxor %xmm7,%xmm3");
+			asm volatile("pxor %xmm13,%xmm10");
+			asm volatile("pxor %xmm15,%xmm11");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+			asm volatile("pxor %xmm13,%xmm12");
+			asm volatile("pxor %xmm15,%xmm14");
+			asm volatile("pxor %xmm5,%xmm5");
+			asm volatile("pxor %xmm7,%xmm7");
+			asm volatile("pxor %xmm13,%xmm13");
+			asm volatile("pxor %xmm15,%xmm15");
+		}
+		asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
+		asm volatile("pxor %xmm2,%xmm2");
+		asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
+		asm volatile("pxor %xmm3,%xmm3");
+		asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
+		asm volatile("pxor %xmm10,%xmm10");
+		asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
+		asm volatile("pxor %xmm11,%xmm11");
+		asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
+		asm volatile("pxor %xmm4,%xmm4");
+		asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
+		asm volatile("pxor %xmm6,%xmm6");
+		asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
+		asm volatile("pxor %xmm12,%xmm12");
+		asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
+		asm volatile("pxor %xmm14,%xmm14");
+	}
+
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+static void raid6_sse24_xor_syndrome(int disks, int start, int stop,
+				     size_t bytes, void **ptrs)
+{
+	u8 **dptr = (u8 **)ptrs;
+	u8 *p, *q;
+	int d, z, z0;
+
+	z0 = stop;		/* P/Q right side optimization */
+	p = dptr[disks-2];	/* XOR parity */
+	q = dptr[disks-1];	/* RS syndrome */
+
+	kernel_fpu_begin();
+
+	asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
+
+	for ( d = 0 ; d < bytes ; d += 64 ) {
+		asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
+		asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
+		asm volatile("movdqa %0,%%xmm12" :: "m" (dptr[z0][d+32]));
+		asm volatile("movdqa %0,%%xmm14" :: "m" (dptr[z0][d+48]));
+		asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
+		asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
+		asm volatile("movdqa %0,%%xmm10" : : "m" (p[d+32]));
+		asm volatile("movdqa %0,%%xmm11" : : "m" (p[d+48]));
+		asm volatile("pxor %xmm4,%xmm2");
+		asm volatile("pxor %xmm6,%xmm3");
+		asm volatile("pxor %xmm12,%xmm10");
+		asm volatile("pxor %xmm14,%xmm11");
+		/* P/Q data pages */
+		for ( z = z0-1 ; z >= start ; z-- ) {
+			asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
+			asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
+			asm volatile("pxor %xmm5,%xmm5");
+			asm volatile("pxor %xmm7,%xmm7");
+			asm volatile("pxor %xmm13,%xmm13");
+			asm volatile("pxor %xmm15,%xmm15");
+			asm volatile("pcmpgtb %xmm4,%xmm5");
+			asm volatile("pcmpgtb %xmm6,%xmm7");
+			asm volatile("pcmpgtb %xmm12,%xmm13");
+			asm volatile("pcmpgtb %xmm14,%xmm15");
+			asm volatile("paddb %xmm4,%xmm4");
+			asm volatile("paddb %xmm6,%xmm6");
+			asm volatile("paddb %xmm12,%xmm12");
+			asm volatile("paddb %xmm14,%xmm14");
+			asm volatile("pand %xmm0,%xmm5");
+			asm volatile("pand %xmm0,%xmm7");
+			asm volatile("pand %xmm0,%xmm13");
+			asm volatile("pand %xmm0,%xmm15");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+			asm volatile("pxor %xmm13,%xmm12");
+			asm volatile("pxor %xmm15,%xmm14");
+			asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
+			asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
+			asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
+			asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
+			asm volatile("pxor %xmm5,%xmm2");
+			asm volatile("pxor %xmm7,%xmm3");
+			asm volatile("pxor %xmm13,%xmm10");
+			asm volatile("pxor %xmm15,%xmm11");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+			asm volatile("pxor %xmm13,%xmm12");
+			asm volatile("pxor %xmm15,%xmm14");
+		}
+		asm volatile("prefetchnta %0" :: "m" (q[d]));
+		asm volatile("prefetchnta %0" :: "m" (q[d+32]));
+		/* P/Q left side optimization */
+		for ( z = start-1 ; z >= 0 ; z-- ) {
+			asm volatile("pxor %xmm5,%xmm5");
+			asm volatile("pxor %xmm7,%xmm7");
+			asm volatile("pxor %xmm13,%xmm13");
+			asm volatile("pxor %xmm15,%xmm15");
+			asm volatile("pcmpgtb %xmm4,%xmm5");
+			asm volatile("pcmpgtb %xmm6,%xmm7");
+			asm volatile("pcmpgtb %xmm12,%xmm13");
+			asm volatile("pcmpgtb %xmm14,%xmm15");
+			asm volatile("paddb %xmm4,%xmm4");
+			asm volatile("paddb %xmm6,%xmm6");
+			asm volatile("paddb %xmm12,%xmm12");
+			asm volatile("paddb %xmm14,%xmm14");
+			asm volatile("pand %xmm0,%xmm5");
+			asm volatile("pand %xmm0,%xmm7");
+			asm volatile("pand %xmm0,%xmm13");
+			asm volatile("pand %xmm0,%xmm15");
+			asm volatile("pxor %xmm5,%xmm4");
+			asm volatile("pxor %xmm7,%xmm6");
+			asm volatile("pxor %xmm13,%xmm12");
+			asm volatile("pxor %xmm15,%xmm14");
+		}
+		asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
+		asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
+		asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
+		asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
+		asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
+		asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
+		asm volatile("pxor %0,%%xmm12" : : "m" (q[d+32]));
+		asm volatile("pxor %0,%%xmm14" : : "m" (q[d+48]));
+		asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
+		asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
+		asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
+		asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
+	}
+	asm volatile("sfence" : : : "memory");
+	kernel_fpu_end();
+}
+
+
+const struct raid6_calls raid6_sse2x4 = {
+	.gen_syndrome	= raid6_sse24_gen_syndrome,
+	.xor_syndrome	= raid6_sse24_xor_syndrome,
+	.name		= "sse2x4",
+};
+
+#endif /* CONFIG_X86_64 */
diff --git a/lib/raid6/.gitignore b/lib/raid6/.gitignore
deleted file mode 100644
index 6be57745afd12..0000000000000
--- a/lib/raid6/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-mktables
-altivec*.c
-int*.c
-tables.c
-neon?.c
-s390vx?.c
-vpermxor*.c
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
deleted file mode 100644
index 5be0a4e60ab1e..0000000000000
--- a/lib/raid6/Makefile
+++ /dev/null
@@ -1,83 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_RAID6_PQ)	+= raid6_pq.o
-
-raid6_pq-y	+= algos.o recov.o tables.o int1.o int2.o int4.o \
-		   int8.o
-
-raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o
-raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \
-                              vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o
-raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
-raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o
-raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o
-raid6_pq-$(CONFIG_RISCV_ISA_V) += rvv.o recov_rvv.o
-
-hostprogs	+= mktables
-
-ifeq ($(CONFIG_ALTIVEC),y)
-altivec_flags := -maltivec $(call cc-option,-mabi=altivec)
-# Enable <altivec.h>
-altivec_flags += -isystem $(shell $(CC) -print-file-name=include)
-
-ifdef CONFIG_CC_IS_CLANG
-# clang ppc port does not yet support -maltivec when -msoft-float is
-# enabled. A future release of clang will resolve this
-# https://llvm.org/pr31177
-CFLAGS_REMOVE_altivec1.o  += -msoft-float
-CFLAGS_REMOVE_altivec2.o  += -msoft-float
-CFLAGS_REMOVE_altivec4.o  += -msoft-float
-CFLAGS_REMOVE_altivec8.o  += -msoft-float
-CFLAGS_REMOVE_vpermxor1.o += -msoft-float
-CFLAGS_REMOVE_vpermxor2.o += -msoft-float
-CFLAGS_REMOVE_vpermxor4.o += -msoft-float
-CFLAGS_REMOVE_vpermxor8.o += -msoft-float
-endif
-endif
-
-quiet_cmd_unroll = UNROLL  $@
-      cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@
-
-targets += int1.c int2.c int4.c int8.c
-$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE
-	$(call if_changed,unroll)
-
-CFLAGS_altivec1.o += $(altivec_flags)
-CFLAGS_altivec2.o += $(altivec_flags)
-CFLAGS_altivec4.o += $(altivec_flags)
-CFLAGS_altivec8.o += $(altivec_flags)
-targets += altivec1.c altivec2.c altivec4.c altivec8.c
-$(obj)/altivec%.c: $(src)/altivec.uc $(src)/unroll.awk FORCE
-	$(call if_changed,unroll)
-
-CFLAGS_vpermxor1.o += $(altivec_flags)
-CFLAGS_vpermxor2.o += $(altivec_flags)
-CFLAGS_vpermxor4.o += $(altivec_flags)
-CFLAGS_vpermxor8.o += $(altivec_flags)
-targets += vpermxor1.c vpermxor2.c vpermxor4.c vpermxor8.c
-$(obj)/vpermxor%.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE
-	$(call if_changed,unroll)
-
-CFLAGS_neon1.o += $(CC_FLAGS_FPU)
-CFLAGS_neon2.o += $(CC_FLAGS_FPU)
-CFLAGS_neon4.o += $(CC_FLAGS_FPU)
-CFLAGS_neon8.o += $(CC_FLAGS_FPU)
-CFLAGS_recov_neon_inner.o += $(CC_FLAGS_FPU)
-CFLAGS_REMOVE_neon1.o += $(CC_FLAGS_NO_FPU)
-CFLAGS_REMOVE_neon2.o += $(CC_FLAGS_NO_FPU)
-CFLAGS_REMOVE_neon4.o += $(CC_FLAGS_NO_FPU)
-CFLAGS_REMOVE_neon8.o += $(CC_FLAGS_NO_FPU)
-CFLAGS_REMOVE_recov_neon_inner.o += $(CC_FLAGS_NO_FPU)
-targets += neon1.c neon2.c neon4.c neon8.c
-$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE
-	$(call if_changed,unroll)
-
-targets += s390vx8.c
-$(obj)/s390vx%.c: $(src)/s390vx.uc $(src)/unroll.awk FORCE
-	$(call if_changed,unroll)
-
-quiet_cmd_mktable = TABLE   $@
-      cmd_mktable = $(obj)/mktables > $@
-
-targets += tables.c
-$(obj)/tables.c: $(obj)/mktables FORCE
-	$(call if_changed,mktable)
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
deleted file mode 100644
index 799e0e5eac26d..0000000000000
--- a/lib/raid6/algos.c
+++ /dev/null
@@ -1,291 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright 2002 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * raid6/algos.c
- *
- * Algorithm list and algorithm selection for RAID-6
- */
-
-#include <linux/raid/pq.h>
-#ifndef __KERNEL__
-#include <sys/mman.h>
-#include <stdio.h>
-#else
-#include <linux/module.h>
-#include <linux/gfp.h>
-#endif
-
-struct raid6_calls raid6_call;
-EXPORT_SYMBOL_GPL(raid6_call);
-
-const struct raid6_calls * const raid6_algos[] = {
-#if defined(__i386__) && !defined(__arch_um__)
-	&raid6_avx512x2,
-	&raid6_avx512x1,
-	&raid6_avx2x2,
-	&raid6_avx2x1,
-	&raid6_sse2x2,
-	&raid6_sse2x1,
-	&raid6_sse1x2,
-	&raid6_sse1x1,
-	&raid6_mmxx2,
-	&raid6_mmxx1,
-#endif
-#if defined(__x86_64__) && !defined(__arch_um__)
-	&raid6_avx512x4,
-	&raid6_avx512x2,
-	&raid6_avx512x1,
-	&raid6_avx2x4,
-	&raid6_avx2x2,
-	&raid6_avx2x1,
-	&raid6_sse2x4,
-	&raid6_sse2x2,
-	&raid6_sse2x1,
-#endif
-#ifdef CONFIG_ALTIVEC
-	&raid6_vpermxor8,
-	&raid6_vpermxor4,
-	&raid6_vpermxor2,
-	&raid6_vpermxor1,
-	&raid6_altivec8,
-	&raid6_altivec4,
-	&raid6_altivec2,
-	&raid6_altivec1,
-#endif
-#if defined(CONFIG_S390)
-	&raid6_s390vx8,
-#endif
-#ifdef CONFIG_KERNEL_MODE_NEON
-	&raid6_neonx8,
-	&raid6_neonx4,
-	&raid6_neonx2,
-	&raid6_neonx1,
-#endif
-#ifdef CONFIG_LOONGARCH
-#ifdef CONFIG_CPU_HAS_LASX
-	&raid6_lasx,
-#endif
-#ifdef CONFIG_CPU_HAS_LSX
-	&raid6_lsx,
-#endif
-#endif
-#ifdef CONFIG_RISCV_ISA_V
-	&raid6_rvvx1,
-	&raid6_rvvx2,
-	&raid6_rvvx4,
-	&raid6_rvvx8,
-#endif
-	&raid6_intx8,
-	&raid6_intx4,
-	&raid6_intx2,
-	&raid6_intx1,
-	NULL
-};
-
-void (*raid6_2data_recov)(int, size_t, int, int, void **);
-EXPORT_SYMBOL_GPL(raid6_2data_recov);
-
-void (*raid6_datap_recov)(int, size_t, int, void **);
-EXPORT_SYMBOL_GPL(raid6_datap_recov);
-
-const struct raid6_recov_calls *const raid6_recov_algos[] = {
-#ifdef CONFIG_X86
-	&raid6_recov_avx512,
-	&raid6_recov_avx2,
-	&raid6_recov_ssse3,
-#endif
-#ifdef CONFIG_S390
-	&raid6_recov_s390xc,
-#endif
-#if defined(CONFIG_KERNEL_MODE_NEON)
-	&raid6_recov_neon,
-#endif
-#ifdef CONFIG_LOONGARCH
-#ifdef CONFIG_CPU_HAS_LASX
-	&raid6_recov_lasx,
-#endif
-#ifdef CONFIG_CPU_HAS_LSX
-	&raid6_recov_lsx,
-#endif
-#endif
-#ifdef CONFIG_RISCV_ISA_V
-	&raid6_recov_rvv,
-#endif
-	&raid6_recov_intx1,
-	NULL
-};
-
-#ifdef __KERNEL__
-#define RAID6_TIME_JIFFIES_LG2	4
-#else
-/* Need more time to be stable in userspace */
-#define RAID6_TIME_JIFFIES_LG2	9
-#define time_before(x, y) ((x) < (y))
-#endif
-
-#define RAID6_TEST_DISKS	8
-#define RAID6_TEST_DISKS_ORDER	3
-
-static inline const struct raid6_recov_calls *raid6_choose_recov(void)
-{
-	const struct raid6_recov_calls *const *algo;
-	const struct raid6_recov_calls *best;
-
-	for (best = NULL, algo = raid6_recov_algos; *algo; algo++)
-		if (!best || (*algo)->priority > best->priority)
-			if (!(*algo)->valid || (*algo)->valid())
-				best = *algo;
-
-	if (best) {
-		raid6_2data_recov = best->data2;
-		raid6_datap_recov = best->datap;
-
-		pr_info("raid6: using %s recovery algorithm\n", best->name);
-	} else
-		pr_err("raid6: Yikes! No recovery algorithm found!\n");
-
-	return best;
-}
-
-static inline const struct raid6_calls *raid6_choose_gen(
-	void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
-{
-	unsigned long perf, bestgenperf, j0, j1;
-	int start = (disks>>1)-1, stop = disks-3;	/* work on the second half of the disks */
-	const struct raid6_calls *const *algo;
-	const struct raid6_calls *best;
-
-	for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
-		if (!best || (*algo)->priority >= best->priority) {
-			if ((*algo)->valid && !(*algo)->valid())
-				continue;
-
-			if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
-				best = *algo;
-				break;
-			}
-
-			perf = 0;
-
-			preempt_disable();
-			j0 = jiffies;
-			while ((j1 = jiffies) == j0)
-				cpu_relax();
-			while (time_before(jiffies,
-					    j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
-				(*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs);
-				perf++;
-			}
-			preempt_enable();
-
-			if (perf > bestgenperf) {
-				bestgenperf = perf;
-				best = *algo;
-			}
-			pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
-				(perf * HZ * (disks-2)) >>
-				(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
-		}
-	}
-
-	if (!best) {
-		pr_err("raid6: Yikes! No algorithm found!\n");
-		goto out;
-	}
-
-	raid6_call = *best;
-
-	if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
-		pr_info("raid6: skipped pq benchmark and selected %s\n",
-			best->name);
-		goto out;
-	}
-
-	pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
-		best->name,
-		(bestgenperf * HZ * (disks - 2)) >>
-		(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
-
-	if (best->xor_syndrome) {
-		perf = 0;
-
-		preempt_disable();
-		j0 = jiffies;
-		while ((j1 = jiffies) == j0)
-			cpu_relax();
-		while (time_before(jiffies,
-				   j1 + (1 << RAID6_TIME_JIFFIES_LG2))) {
-			best->xor_syndrome(disks, start, stop,
-					   PAGE_SIZE, *dptrs);
-			perf++;
-		}
-		preempt_enable();
-
-		pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n",
-			(perf * HZ * (disks - 2)) >>
-			(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
-	}
-
-out:
-	return best;
-}
-
-
-/* Try to pick the best algorithm */
-/* This code uses the gfmul table as convenient data set to abuse */
-
-int __init raid6_select_algo(void)
-{
-	const int disks = RAID6_TEST_DISKS;
-
-	const struct raid6_calls *gen_best;
-	const struct raid6_recov_calls *rec_best;
-	char *disk_ptr, *p;
-	void *dptrs[RAID6_TEST_DISKS];
-	int i, cycle;
-
-	/* prepare the buffer and fill it circularly with gfmul table */
-	disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
-	if (!disk_ptr) {
-		pr_err("raid6: Yikes!  No memory available.\n");
-		return -ENOMEM;
-	}
-
-	p = disk_ptr;
-	for (i = 0; i < disks; i++)
-		dptrs[i] = p + PAGE_SIZE * i;
-
-	cycle = ((disks - 2) * PAGE_SIZE) / 65536;
-	for (i = 0; i < cycle; i++) {
-		memcpy(p, raid6_gfmul, 65536);
-		p += 65536;
-	}
-
-	if ((disks - 2) * PAGE_SIZE % 65536)
-		memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
-
-	/* select raid gen_syndrome function */
-	gen_best = raid6_choose_gen(&dptrs, disks);
-
-	/* select raid recover functions */
-	rec_best = raid6_choose_recov();
-
-	free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
-
-	return gen_best && rec_best ? 0 : -EINVAL;
-}
-
-static void raid6_exit(void)
-{
-	do { } while (0);
-}
-
-subsys_initcall(raid6_select_algo);
-module_exit(raid6_exit);
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("RAID6 Q-syndrome calculations");
diff --git a/lib/raid6/altivec.uc b/lib/raid6/altivec.uc
deleted file mode 100644
index d20ed0d114111..0000000000000
--- a/lib/raid6/altivec.uc
+++ /dev/null
@@ -1,132 +0,0 @@
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
- *   Boston MA 02111-1307, USA; either version 2 of the License, or
- *   (at your option) any later version; incorporated herein by reference.
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * raid6altivec$#.c
- *
- * $#-way unrolled portable integer math RAID-6 instruction set
- *
- * This file is postprocessed using unroll.awk
- *
- * <benh> hpa: in process,
- * you can just "steal" the vec unit with enable_kernel_altivec() (but
- * bracked this with preempt_disable/enable or in a lock)
- */
-
-#include <linux/raid/pq.h>
-
-#ifdef CONFIG_ALTIVEC
-
-#include <altivec.h>
-#ifdef __KERNEL__
-# include <asm/cputable.h>
-# include <asm/switch_to.h>
-#endif /* __KERNEL__ */
-
-/*
- * This is the C data type to use.  We use a vector of
- * signed char so vec_cmpgt() will generate the right
- * instruction.
- */
-
-typedef vector signed char unative_t;
-
-#define NBYTES(x) ((vector signed char) {x,x,x,x, x,x,x,x, x,x,x,x, x,x,x,x})
-#define NSIZE	sizeof(unative_t)
-
-/*
- * The SHLBYTE() operation shifts each byte left by 1, *not*
- * rolling over into the next byte
- */
-static inline __attribute_const__ unative_t SHLBYTE(unative_t v)
-{
-	return vec_add(v,v);
-}
-
-/*
- * The MASK() operation returns 0xFF in any byte for which the high
- * bit is 1, 0x00 for any byte for which the high bit is 0.
- */
-static inline __attribute_const__ unative_t MASK(unative_t v)
-{
-	unative_t zv = NBYTES(0);
-
-	/* vec_cmpgt returns a vector bool char; thus the need for the cast */
-	return (unative_t)vec_cmpgt(zv, v);
-}
-
-
-/* This is noinline to make damned sure that gcc doesn't move any of the
-   Altivec code around the enable/disable code */
-static void noinline
-raid6_altivec$#_gen_syndrome_real(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
-	unative_t x1d = NBYTES(0x1d);
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
-		wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
-		for ( z = z0-1 ; z >= 0 ; z-- ) {
-			wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			wp$$ = vec_xor(wp$$, wd$$);
-			w2$$ = MASK(wq$$);
-			w1$$ = SHLBYTE(wq$$);
-			w2$$ = vec_and(w2$$, x1d);
-			w1$$ = vec_xor(w1$$, w2$$);
-			wq$$ = vec_xor(w1$$, wd$$);
-		}
-		*(unative_t *)&p[d+NSIZE*$$] = wp$$;
-		*(unative_t *)&q[d+NSIZE*$$] = wq$$;
-	}
-}
-
-static void raid6_altivec$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	preempt_disable();
-	enable_kernel_altivec();
-
-	raid6_altivec$#_gen_syndrome_real(disks, bytes, ptrs);
-
-	disable_kernel_altivec();
-	preempt_enable();
-}
-
-int raid6_have_altivec(void);
-#if $# == 1
-int raid6_have_altivec(void)
-{
-	/* This assumes either all CPUs have Altivec or none does */
-# ifdef __KERNEL__
-	return cpu_has_feature(CPU_FTR_ALTIVEC);
-# else
-	return 1;
-# endif
-}
-#endif
-
-const struct raid6_calls raid6_altivec$# = {
-	raid6_altivec$#_gen_syndrome,
-	NULL,			/* XOR not yet implemented */
-	raid6_have_altivec,
-	"altivecx$#",
-	0
-};
-
-#endif /* CONFIG_ALTIVEC */
diff --git a/lib/raid6/avx2.c b/lib/raid6/avx2.c
deleted file mode 100644
index 059024234dce1..0000000000000
--- a/lib/raid6/avx2.c
+++ /dev/null
@@ -1,470 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright (C) 2012 Intel Corporation
- *   Author: Yuanhan Liu <yuanhan.liu@linux.intel.com>
- *
- *   Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * AVX2 implementation of RAID-6 syndrome functions
- *
- */
-
-#include <linux/raid/pq.h>
-#include "x86.h"
-
-static const struct raid6_avx2_constants {
-	u64 x1d[4];
-} raid6_avx2_constants __aligned(32) = {
-	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
-	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
-};
-
-static int raid6_have_avx2(void)
-{
-	return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX);
-}
-
-/*
- * Plain AVX2 implementation
- */
-static void raid6_avx21_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
-	asm volatile("vpxor %ymm3,%ymm3,%ymm3");	/* Zero temp */
-
-	for (d = 0; d < bytes; d += 32) {
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
-		asm volatile("vmovdqa %0,%%ymm2" : : "m" (dptr[z0][d]));/* P[0] */
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
-		asm volatile("vmovdqa %ymm2,%ymm4");/* Q[0] */
-		asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z0-1][d]));
-		for (z = z0-2; z >= 0; z--) {
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
-			asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");
-			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-			asm volatile("vpand %ymm0,%ymm5,%ymm5");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm6,%ymm2,%ymm2");
-			asm volatile("vpxor %ymm6,%ymm4,%ymm4");
-			asm volatile("vmovdqa %0,%%ymm6" : : "m" (dptr[z][d]));
-		}
-		asm volatile("vpcmpgtb %ymm4,%ymm3,%ymm5");
-		asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-		asm volatile("vpand %ymm0,%ymm5,%ymm5");
-		asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-		asm volatile("vpxor %ymm6,%ymm2,%ymm2");
-		asm volatile("vpxor %ymm6,%ymm4,%ymm4");
-
-		asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
-		asm volatile("vpxor %ymm2,%ymm2,%ymm2");
-		asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
-		asm volatile("vpxor %ymm4,%ymm4,%ymm4");
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-static void raid6_avx21_xor_syndrome(int disks, int start, int stop,
-				     size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
-
-	for (d = 0 ; d < bytes ; d += 32) {
-		asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
-		asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
-		asm volatile("vpxor %ymm4,%ymm2,%ymm2");
-		/* P/Q data pages */
-		for (z = z0-1 ; z >= start ; z--) {
-			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
-			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
-			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-			asm volatile("vpand %ymm0,%ymm5,%ymm5");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
-			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-		}
-		/* P/Q left side optimization */
-		for (z = start-1 ; z >= 0 ; z--) {
-			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
-			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
-			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-			asm volatile("vpand %ymm0,%ymm5,%ymm5");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-		}
-		asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
-		/* Don't use movntdq for r/w memory area < cache line */
-		asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));
-		asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_avx2x1 = {
-	raid6_avx21_gen_syndrome,
-	raid6_avx21_xor_syndrome,
-	raid6_have_avx2,
-	"avx2x1",
-	.priority = 2		/* Prefer AVX2 over priority 1 (SSE2 and others) */
-};
-
-/*
- * Unrolled-by-2 AVX2 implementation
- */
-static void raid6_avx22_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
-	asm volatile("vpxor %ymm1,%ymm1,%ymm1"); /* Zero temp */
-
-	/* We uniformly assume a single prefetch covers at least 32 bytes */
-	for (d = 0; d < bytes; d += 64) {
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d+32]));
-		asm volatile("vmovdqa %0,%%ymm2" : : "m" (dptr[z0][d]));/* P[0] */
-		asm volatile("vmovdqa %0,%%ymm3" : : "m" (dptr[z0][d+32]));/* P[1] */
-		asm volatile("vmovdqa %ymm2,%ymm4"); /* Q[0] */
-		asm volatile("vmovdqa %ymm3,%ymm6"); /* Q[1] */
-		for (z = z0-1; z >= 0; z--) {
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d+32]));
-			asm volatile("vpcmpgtb %ymm4,%ymm1,%ymm5");
-			asm volatile("vpcmpgtb %ymm6,%ymm1,%ymm7");
-			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
-			asm volatile("vpand %ymm0,%ymm5,%ymm5");
-			asm volatile("vpand %ymm0,%ymm7,%ymm7");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-			asm volatile("vmovdqa %0,%%ymm5" : : "m" (dptr[z][d]));
-			asm volatile("vmovdqa %0,%%ymm7" : : "m" (dptr[z][d+32]));
-			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
-			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-		}
-		asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
-		asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
-		asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
-		asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-static void raid6_avx22_xor_syndrome(int disks, int start, int stop,
-				     size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
-
-	for (d = 0 ; d < bytes ; d += 64) {
-		asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
-		asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));
-		asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
-		asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));
-		asm volatile("vpxor %ymm4,%ymm2,%ymm2");
-		asm volatile("vpxor %ymm6,%ymm3,%ymm3");
-		/* P/Q data pages */
-		for (z = z0-1 ; z >= start ; z--) {
-			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
-			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
-			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
-			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
-			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
-			asm volatile("vpand %ymm0,%ymm5,%ymm5");
-			asm volatile("vpand %ymm0,%ymm7,%ymm7");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-			asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
-			asm volatile("vmovdqa %0,%%ymm7"
-				     :: "m" (dptr[z][d+32]));
-			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
-			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-		}
-		/* P/Q left side optimization */
-		for (z = start-1 ; z >= 0 ; z--) {
-			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
-			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
-			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
-			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
-			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
-			asm volatile("vpand %ymm0,%ymm5,%ymm5");
-			asm volatile("vpand %ymm0,%ymm7,%ymm7");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-		}
-		asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
-		asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));
-		/* Don't use movntdq for r/w memory area < cache line */
-		asm volatile("vmovdqa %%ymm4,%0" : "=m" (q[d]));
-		asm volatile("vmovdqa %%ymm6,%0" : "=m" (q[d+32]));
-		asm volatile("vmovdqa %%ymm2,%0" : "=m" (p[d]));
-		asm volatile("vmovdqa %%ymm3,%0" : "=m" (p[d+32]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_avx2x2 = {
-	raid6_avx22_gen_syndrome,
-	raid6_avx22_xor_syndrome,
-	raid6_have_avx2,
-	"avx2x2",
-	.priority = 2		/* Prefer AVX2 over priority 1 (SSE2 and others) */
-};
-
-#ifdef CONFIG_X86_64
-
-/*
- * Unrolled-by-4 AVX2 implementation
- */
-static void raid6_avx24_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa %0,%%ymm0" : : "m" (raid6_avx2_constants.x1d[0]));
-	asm volatile("vpxor %ymm1,%ymm1,%ymm1");	/* Zero temp */
-	asm volatile("vpxor %ymm2,%ymm2,%ymm2");	/* P[0] */
-	asm volatile("vpxor %ymm3,%ymm3,%ymm3");	/* P[1] */
-	asm volatile("vpxor %ymm4,%ymm4,%ymm4");	/* Q[0] */
-	asm volatile("vpxor %ymm6,%ymm6,%ymm6");	/* Q[1] */
-	asm volatile("vpxor %ymm10,%ymm10,%ymm10");	/* P[2] */
-	asm volatile("vpxor %ymm11,%ymm11,%ymm11");	/* P[3] */
-	asm volatile("vpxor %ymm12,%ymm12,%ymm12");	/* Q[2] */
-	asm volatile("vpxor %ymm14,%ymm14,%ymm14");	/* Q[3] */
-
-	for (d = 0; d < bytes; d += 128) {
-		for (z = z0; z >= 0; z--) {
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d+32]));
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d+64]));
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d+96]));
-			asm volatile("vpcmpgtb %ymm4,%ymm1,%ymm5");
-			asm volatile("vpcmpgtb %ymm6,%ymm1,%ymm7");
-			asm volatile("vpcmpgtb %ymm12,%ymm1,%ymm13");
-			asm volatile("vpcmpgtb %ymm14,%ymm1,%ymm15");
-			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
-			asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
-			asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
-			asm volatile("vpand %ymm0,%ymm5,%ymm5");
-			asm volatile("vpand %ymm0,%ymm7,%ymm7");
-			asm volatile("vpand %ymm0,%ymm13,%ymm13");
-			asm volatile("vpand %ymm0,%ymm15,%ymm15");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
-			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
-			asm volatile("vmovdqa %0,%%ymm5" : : "m" (dptr[z][d]));
-			asm volatile("vmovdqa %0,%%ymm7" : : "m" (dptr[z][d+32]));
-			asm volatile("vmovdqa %0,%%ymm13" : : "m" (dptr[z][d+64]));
-			asm volatile("vmovdqa %0,%%ymm15" : : "m" (dptr[z][d+96]));
-			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
-			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
-			asm volatile("vpxor %ymm13,%ymm10,%ymm10");
-			asm volatile("vpxor %ymm15,%ymm11,%ymm11");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
-			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
-		}
-		asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
-		asm volatile("vpxor %ymm2,%ymm2,%ymm2");
-		asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
-		asm volatile("vpxor %ymm3,%ymm3,%ymm3");
-		asm volatile("vmovntdq %%ymm10,%0" : "=m" (p[d+64]));
-		asm volatile("vpxor %ymm10,%ymm10,%ymm10");
-		asm volatile("vmovntdq %%ymm11,%0" : "=m" (p[d+96]));
-		asm volatile("vpxor %ymm11,%ymm11,%ymm11");
-		asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
-		asm volatile("vpxor %ymm4,%ymm4,%ymm4");
-		asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
-		asm volatile("vpxor %ymm6,%ymm6,%ymm6");
-		asm volatile("vmovntdq %%ymm12,%0" : "=m" (q[d+64]));
-		asm volatile("vpxor %ymm12,%ymm12,%ymm12");
-		asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));
-		asm volatile("vpxor %ymm14,%ymm14,%ymm14");
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-static void raid6_avx24_xor_syndrome(int disks, int start, int stop,
-				     size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa %0,%%ymm0" :: "m" (raid6_avx2_constants.x1d[0]));
-
-	for (d = 0 ; d < bytes ; d += 128) {
-		asm volatile("vmovdqa %0,%%ymm4" :: "m" (dptr[z0][d]));
-		asm volatile("vmovdqa %0,%%ymm6" :: "m" (dptr[z0][d+32]));
-		asm volatile("vmovdqa %0,%%ymm12" :: "m" (dptr[z0][d+64]));
-		asm volatile("vmovdqa %0,%%ymm14" :: "m" (dptr[z0][d+96]));
-		asm volatile("vmovdqa %0,%%ymm2" : : "m" (p[d]));
-		asm volatile("vmovdqa %0,%%ymm3" : : "m" (p[d+32]));
-		asm volatile("vmovdqa %0,%%ymm10" : : "m" (p[d+64]));
-		asm volatile("vmovdqa %0,%%ymm11" : : "m" (p[d+96]));
-		asm volatile("vpxor %ymm4,%ymm2,%ymm2");
-		asm volatile("vpxor %ymm6,%ymm3,%ymm3");
-		asm volatile("vpxor %ymm12,%ymm10,%ymm10");
-		asm volatile("vpxor %ymm14,%ymm11,%ymm11");
-		/* P/Q data pages */
-		for (z = z0-1 ; z >= start ; z--) {
-			asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
-			asm volatile("prefetchnta %0" :: "m" (dptr[z][d+64]));
-			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
-			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
-			asm volatile("vpxor %ymm13,%ymm13,%ymm13");
-			asm volatile("vpxor %ymm15,%ymm15,%ymm15");
-			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
-			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
-			asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");
-			asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");
-			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
-			asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
-			asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
-			asm volatile("vpand %ymm0,%ymm5,%ymm5");
-			asm volatile("vpand %ymm0,%ymm7,%ymm7");
-			asm volatile("vpand %ymm0,%ymm13,%ymm13");
-			asm volatile("vpand %ymm0,%ymm15,%ymm15");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
-			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
-			asm volatile("vmovdqa %0,%%ymm5" :: "m" (dptr[z][d]));
-			asm volatile("vmovdqa %0,%%ymm7"
-				     :: "m" (dptr[z][d+32]));
-			asm volatile("vmovdqa %0,%%ymm13"
-				     :: "m" (dptr[z][d+64]));
-			asm volatile("vmovdqa %0,%%ymm15"
-				     :: "m" (dptr[z][d+96]));
-			asm volatile("vpxor %ymm5,%ymm2,%ymm2");
-			asm volatile("vpxor %ymm7,%ymm3,%ymm3");
-			asm volatile("vpxor %ymm13,%ymm10,%ymm10");
-			asm volatile("vpxor %ymm15,%ymm11,%ymm11");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
-			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
-		}
-		asm volatile("prefetchnta %0" :: "m" (q[d]));
-		asm volatile("prefetchnta %0" :: "m" (q[d+64]));
-		/* P/Q left side optimization */
-		for (z = start-1 ; z >= 0 ; z--) {
-			asm volatile("vpxor %ymm5,%ymm5,%ymm5");
-			asm volatile("vpxor %ymm7,%ymm7,%ymm7");
-			asm volatile("vpxor %ymm13,%ymm13,%ymm13");
-			asm volatile("vpxor %ymm15,%ymm15,%ymm15");
-			asm volatile("vpcmpgtb %ymm4,%ymm5,%ymm5");
-			asm volatile("vpcmpgtb %ymm6,%ymm7,%ymm7");
-			asm volatile("vpcmpgtb %ymm12,%ymm13,%ymm13");
-			asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm15");
-			asm volatile("vpaddb %ymm4,%ymm4,%ymm4");
-			asm volatile("vpaddb %ymm6,%ymm6,%ymm6");
-			asm volatile("vpaddb %ymm12,%ymm12,%ymm12");
-			asm volatile("vpaddb %ymm14,%ymm14,%ymm14");
-			asm volatile("vpand %ymm0,%ymm5,%ymm5");
-			asm volatile("vpand %ymm0,%ymm7,%ymm7");
-			asm volatile("vpand %ymm0,%ymm13,%ymm13");
-			asm volatile("vpand %ymm0,%ymm15,%ymm15");
-			asm volatile("vpxor %ymm5,%ymm4,%ymm4");
-			asm volatile("vpxor %ymm7,%ymm6,%ymm6");
-			asm volatile("vpxor %ymm13,%ymm12,%ymm12");
-			asm volatile("vpxor %ymm15,%ymm14,%ymm14");
-		}
-		asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[d]));
-		asm volatile("vmovntdq %%ymm3,%0" : "=m" (p[d+32]));
-		asm volatile("vmovntdq %%ymm10,%0" : "=m" (p[d+64]));
-		asm volatile("vmovntdq %%ymm11,%0" : "=m" (p[d+96]));
-		asm volatile("vpxor %0,%%ymm4,%%ymm4" : : "m" (q[d]));
-		asm volatile("vpxor %0,%%ymm6,%%ymm6" : : "m" (q[d+32]));
-		asm volatile("vpxor %0,%%ymm12,%%ymm12" : : "m" (q[d+64]));
-		asm volatile("vpxor %0,%%ymm14,%%ymm14" : : "m" (q[d+96]));
-		asm volatile("vmovntdq %%ymm4,%0" : "=m" (q[d]));
-		asm volatile("vmovntdq %%ymm6,%0" : "=m" (q[d+32]));
-		asm volatile("vmovntdq %%ymm12,%0" : "=m" (q[d+64]));
-		asm volatile("vmovntdq %%ymm14,%0" : "=m" (q[d+96]));
-	}
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_avx2x4 = {
-	raid6_avx24_gen_syndrome,
-	raid6_avx24_xor_syndrome,
-	raid6_have_avx2,
-	"avx2x4",
-	.priority = 2		/* Prefer AVX2 over priority 1 (SSE2 and others) */
-};
-#endif /* CONFIG_X86_64 */
diff --git a/lib/raid6/avx512.c b/lib/raid6/avx512.c
deleted file mode 100644
index 009bd0adeebf0..0000000000000
--- a/lib/raid6/avx512.c
+++ /dev/null
@@ -1,560 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- --------------------------------------------------------
- *
- *   Copyright (C) 2016 Intel Corporation
- *
- *   Author: Gayatri Kammela <gayatri.kammela@intel.com>
- *   Author: Megha Dey <megha.dey@linux.intel.com>
- *
- *   Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved
- *   Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
- *
- * -----------------------------------------------------------------------
- */
-
-/*
- * AVX512 implementation of RAID-6 syndrome functions
- *
- */
-
-#include <linux/raid/pq.h>
-#include "x86.h"
-
-static const struct raid6_avx512_constants {
-	u64 x1d[8];
-} raid6_avx512_constants __aligned(512/8) = {
-	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
-	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
-	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
-	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
-};
-
-static int raid6_have_avx512(void)
-{
-	return boot_cpu_has(X86_FEATURE_AVX2) &&
-		boot_cpu_has(X86_FEATURE_AVX) &&
-		boot_cpu_has(X86_FEATURE_AVX512F) &&
-		boot_cpu_has(X86_FEATURE_AVX512BW) &&
-		boot_cpu_has(X86_FEATURE_AVX512VL) &&
-		boot_cpu_has(X86_FEATURE_AVX512DQ);
-}
-
-static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;         /* Highest data disk */
-	p = dptr[z0+1];         /* XOR parity */
-	q = dptr[z0+2];         /* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
-		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
-		     :
-		     : "m" (raid6_avx512_constants.x1d[0]));
-
-	for (d = 0; d < bytes; d += 64) {
-		asm volatile("prefetchnta %0\n\t"
-			     "vmovdqa64 %0,%%zmm2\n\t"     /* P[0] */
-			     "prefetchnta %1\n\t"
-			     "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
-			     "vmovdqa64 %1,%%zmm6"
-			     :
-			     : "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
-		for (z = z0-2; z >= 0; z--) {
-			asm volatile("prefetchnta %0\n\t"
-				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
-				     "vpmovm2b %%k1,%%zmm5\n\t"
-				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
-				     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
-				     "vmovdqa64 %0,%%zmm6"
-				     :
-				     : "m" (dptr[z][d]));
-		}
-		asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
-			     "vpmovm2b %%k1,%%zmm5\n\t"
-			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-			     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
-			     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
-			     "vmovntdq %%zmm2,%0\n\t"
-			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
-			     "vmovntdq %%zmm4,%1\n\t"
-			     "vpxorq %%zmm4,%%zmm4,%%zmm4"
-			     :
-			     : "m" (p[d]), "m" (q[d]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
-				       size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa64 %0,%%zmm0"
-		     : : "m" (raid6_avx512_constants.x1d[0]));
-
-	for (d = 0 ; d < bytes ; d += 64) {
-		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
-			     "vmovdqa64 %1,%%zmm2\n\t"
-			     "vpxorq %%zmm4,%%zmm2,%%zmm2"
-			     :
-			     : "m" (dptr[z0][d]),  "m" (p[d]));
-		/* P/Q data pages */
-		for (z = z0-1 ; z >= start ; z--) {
-			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
-				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
-				     "vpmovm2b %%k1,%%zmm5\n\t"
-				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vmovdqa64 %0,%%zmm5\n\t"
-				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
-				     :
-				     : "m" (dptr[z][d]));
-		}
-		/* P/Q left side optimization */
-		for (z = start-1 ; z >= 0 ; z--) {
-			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
-				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
-				     "vpmovm2b %%k1,%%zmm5\n\t"
-				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4"
-				     :
-				     : );
-		}
-		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
-		/* Don't use movntdq for r/w memory area < cache line */
-			     "vmovdqa64 %%zmm4,%0\n\t"
-			     "vmovdqa64 %%zmm2,%1"
-			     :
-			     : "m" (q[d]), "m" (p[d]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_avx512x1 = {
-	raid6_avx5121_gen_syndrome,
-	raid6_avx5121_xor_syndrome,
-	raid6_have_avx512,
-	"avx512x1",
-	.priority = 2		/* Prefer AVX512 over priority 1 (SSE2 and others) */
-};
-
-/*
- * Unrolled-by-2 AVX512 implementation
- */
-static void raid6_avx5122_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;         /* Highest data disk */
-	p = dptr[z0+1];         /* XOR parity */
-	q = dptr[z0+2];         /* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
-		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
-		     :
-		     : "m" (raid6_avx512_constants.x1d[0]));
-
-	/* We uniformly assume a single prefetch covers at least 64 bytes */
-	for (d = 0; d < bytes; d += 128) {
-		asm volatile("prefetchnta %0\n\t"
-			     "prefetchnta %1\n\t"
-			     "vmovdqa64 %0,%%zmm2\n\t"      /* P[0] */
-			     "vmovdqa64 %1,%%zmm3\n\t"      /* P[1] */
-			     "vmovdqa64 %%zmm2,%%zmm4\n\t"  /* Q[0] */
-			     "vmovdqa64 %%zmm3,%%zmm6"      /* Q[1] */
-			     :
-			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]));
-		for (z = z0-1; z >= 0; z--) {
-			asm volatile("prefetchnta %0\n\t"
-				     "prefetchnta %1\n\t"
-				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
-				     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
-				     "vpmovm2b %%k1,%%zmm5\n\t"
-				     "vpmovm2b %%k2,%%zmm7\n\t"
-				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
-				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
-				     "vmovdqa64 %0,%%zmm5\n\t"
-				     "vmovdqa64 %1,%%zmm7\n\t"
-				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
-				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
-				     :
-				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]));
-		}
-		asm volatile("vmovntdq %%zmm2,%0\n\t"
-			     "vmovntdq %%zmm3,%1\n\t"
-			     "vmovntdq %%zmm4,%2\n\t"
-			     "vmovntdq %%zmm6,%3"
-			     :
-			     : "m" (p[d]), "m" (p[d+64]), "m" (q[d]),
-			       "m" (q[d+64]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
-				       size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa64 %0,%%zmm0"
-		     : : "m" (raid6_avx512_constants.x1d[0]));
-
-	for (d = 0 ; d < bytes ; d += 128) {
-		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
-			     "vmovdqa64 %1,%%zmm6\n\t"
-			     "vmovdqa64 %2,%%zmm2\n\t"
-			     "vmovdqa64 %3,%%zmm3\n\t"
-			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
-			     "vpxorq %%zmm6,%%zmm3,%%zmm3"
-			     :
-			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
-			       "m" (p[d]), "m" (p[d+64]));
-		/* P/Q data pages */
-		for (z = z0-1 ; z >= start ; z--) {
-			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
-				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
-				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
-				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
-				     "vpmovm2b %%k1,%%zmm5\n\t"
-				     "vpmovm2b %%k2,%%zmm7\n\t"
-				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
-				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
-				     "vmovdqa64 %0,%%zmm5\n\t"
-				     "vmovdqa64 %1,%%zmm7\n\t"
-				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
-				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
-				     :
-				     : "m" (dptr[z][d]),  "m" (dptr[z][d+64]));
-		}
-		/* P/Q left side optimization */
-		for (z = start-1 ; z >= 0 ; z--) {
-			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
-				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
-				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
-				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
-				     "vpmovm2b %%k1,%%zmm5\n\t"
-				     "vpmovm2b %%k2,%%zmm7\n\t"
-				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
-				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vpxorq %%zmm7,%%zmm6,%%zmm6"
-				     :
-				     : );
-		}
-		asm volatile("vpxorq %0,%%zmm4,%%zmm4\n\t"
-			     "vpxorq %1,%%zmm6,%%zmm6\n\t"
-			     /* Don't use movntdq for r/w
-			      * memory area < cache line
-			      */
-			     "vmovdqa64 %%zmm4,%0\n\t"
-			     "vmovdqa64 %%zmm6,%1\n\t"
-			     "vmovdqa64 %%zmm2,%2\n\t"
-			     "vmovdqa64 %%zmm3,%3"
-			     :
-			     : "m" (q[d]), "m" (q[d+64]), "m" (p[d]),
-			       "m" (p[d+64]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_avx512x2 = {
-	raid6_avx5122_gen_syndrome,
-	raid6_avx5122_xor_syndrome,
-	raid6_have_avx512,
-	"avx512x2",
-	.priority = 2		/* Prefer AVX512 over priority 1 (SSE2 and others) */
-};
-
-#ifdef CONFIG_X86_64
-
-/*
- * Unrolled-by-4 AVX2 implementation
- */
-static void raid6_avx5124_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;         /* Highest data disk */
-	p = dptr[z0+1];         /* XOR parity */
-	q = dptr[z0+2];         /* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
-		     "vpxorq %%zmm1,%%zmm1,%%zmm1\n\t"       /* Zero temp */
-		     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"       /* P[0] */
-		     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"       /* P[1] */
-		     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"       /* Q[0] */
-		     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"       /* Q[1] */
-		     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"    /* P[2] */
-		     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"    /* P[3] */
-		     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"    /* Q[2] */
-		     "vpxorq %%zmm14,%%zmm14,%%zmm14"        /* Q[3] */
-		     :
-		     : "m" (raid6_avx512_constants.x1d[0]));
-
-	for (d = 0; d < bytes; d += 256) {
-		for (z = z0; z >= 0; z--) {
-		asm volatile("prefetchnta %0\n\t"
-			     "prefetchnta %1\n\t"
-			     "prefetchnta %2\n\t"
-			     "prefetchnta %3\n\t"
-			     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
-			     "vpcmpgtb %%zmm6,%%zmm1,%%k2\n\t"
-			     "vpcmpgtb %%zmm12,%%zmm1,%%k3\n\t"
-			     "vpcmpgtb %%zmm14,%%zmm1,%%k4\n\t"
-			     "vpmovm2b %%k1,%%zmm5\n\t"
-			     "vpmovm2b %%k2,%%zmm7\n\t"
-			     "vpmovm2b %%k3,%%zmm13\n\t"
-			     "vpmovm2b %%k4,%%zmm15\n\t"
-			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-			     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
-			     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
-			     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
-			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-			     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
-			     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
-			     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
-			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
-			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
-			     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
-			     "vmovdqa64 %0,%%zmm5\n\t"
-			     "vmovdqa64 %1,%%zmm7\n\t"
-			     "vmovdqa64 %2,%%zmm13\n\t"
-			     "vmovdqa64 %3,%%zmm15\n\t"
-			     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
-			     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
-			     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
-			     "vpxorq %%zmm15,%%zmm11,%%zmm11\n"
-			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-			     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
-			     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
-			     "vpxorq %%zmm15,%%zmm14,%%zmm14"
-			     :
-			     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
-			       "m" (dptr[z][d+128]), "m" (dptr[z][d+192]));
-		}
-		asm volatile("vmovntdq %%zmm2,%0\n\t"
-			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
-			     "vmovntdq %%zmm3,%1\n\t"
-			     "vpxorq %%zmm3,%%zmm3,%%zmm3\n\t"
-			     "vmovntdq %%zmm10,%2\n\t"
-			     "vpxorq %%zmm10,%%zmm10,%%zmm10\n\t"
-			     "vmovntdq %%zmm11,%3\n\t"
-			     "vpxorq %%zmm11,%%zmm11,%%zmm11\n\t"
-			     "vmovntdq %%zmm4,%4\n\t"
-			     "vpxorq %%zmm4,%%zmm4,%%zmm4\n\t"
-			     "vmovntdq %%zmm6,%5\n\t"
-			     "vpxorq %%zmm6,%%zmm6,%%zmm6\n\t"
-			     "vmovntdq %%zmm12,%6\n\t"
-			     "vpxorq %%zmm12,%%zmm12,%%zmm12\n\t"
-			     "vmovntdq %%zmm14,%7\n\t"
-			     "vpxorq %%zmm14,%%zmm14,%%zmm14"
-			     :
-			     : "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
-			       "m" (p[d+192]), "m" (q[d]), "m" (q[d+64]),
-			       "m" (q[d+128]), "m" (q[d+192]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
-				       size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("vmovdqa64 %0,%%zmm0"
-		     :: "m" (raid6_avx512_constants.x1d[0]));
-
-	for (d = 0 ; d < bytes ; d += 256) {
-		asm volatile("vmovdqa64 %0,%%zmm4\n\t"
-			     "vmovdqa64 %1,%%zmm6\n\t"
-			     "vmovdqa64 %2,%%zmm12\n\t"
-			     "vmovdqa64 %3,%%zmm14\n\t"
-			     "vmovdqa64 %4,%%zmm2\n\t"
-			     "vmovdqa64 %5,%%zmm3\n\t"
-			     "vmovdqa64 %6,%%zmm10\n\t"
-			     "vmovdqa64 %7,%%zmm11\n\t"
-			     "vpxorq %%zmm4,%%zmm2,%%zmm2\n\t"
-			     "vpxorq %%zmm6,%%zmm3,%%zmm3\n\t"
-			     "vpxorq %%zmm12,%%zmm10,%%zmm10\n\t"
-			     "vpxorq %%zmm14,%%zmm11,%%zmm11"
-			     :
-			     : "m" (dptr[z0][d]), "m" (dptr[z0][d+64]),
-			       "m" (dptr[z0][d+128]), "m" (dptr[z0][d+192]),
-			       "m" (p[d]), "m" (p[d+64]), "m" (p[d+128]),
-			       "m" (p[d+192]));
-		/* P/Q data pages */
-		for (z = z0-1 ; z >= start ; z--) {
-			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
-				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
-				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
-				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
-				     "prefetchnta %0\n\t"
-				     "prefetchnta %2\n\t"
-				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
-				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
-				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
-				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
-				     "vpmovm2b %%k1,%%zmm5\n\t"
-				     "vpmovm2b %%k2,%%zmm7\n\t"
-				     "vpmovm2b %%k3,%%zmm13\n\t"
-				     "vpmovm2b %%k4,%%zmm15\n\t"
-				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
-				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
-				     "vpaddb %%Zmm14,%%zmm14,%%zmm14\n\t"
-				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
-				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
-				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
-				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
-				     "vpxorq %%zmm15,%%zmm14,%%zmm14\n\t"
-				     "vmovdqa64 %0,%%zmm5\n\t"
-				     "vmovdqa64 %1,%%zmm7\n\t"
-				     "vmovdqa64 %2,%%zmm13\n\t"
-				     "vmovdqa64 %3,%%zmm15\n\t"
-				     "vpxorq %%zmm5,%%zmm2,%%zmm2\n\t"
-				     "vpxorq %%zmm7,%%zmm3,%%zmm3\n\t"
-				     "vpxorq %%zmm13,%%zmm10,%%zmm10\n\t"
-				     "vpxorq %%zmm15,%%zmm11,%%zmm11\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
-				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
-				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
-				     :
-				     : "m" (dptr[z][d]), "m" (dptr[z][d+64]),
-				       "m" (dptr[z][d+128]),
-				       "m" (dptr[z][d+192]));
-		}
-		asm volatile("prefetchnta %0\n\t"
-			     "prefetchnta %1\n\t"
-			     :
-			     : "m" (q[d]), "m" (q[d+128]));
-		/* P/Q left side optimization */
-		for (z = start-1 ; z >= 0 ; z--) {
-			asm volatile("vpxorq %%zmm5,%%zmm5,%%zmm5\n\t"
-				     "vpxorq %%zmm7,%%zmm7,%%zmm7\n\t"
-				     "vpxorq %%zmm13,%%zmm13,%%zmm13\n\t"
-				     "vpxorq %%zmm15,%%zmm15,%%zmm15\n\t"
-				     "vpcmpgtb %%zmm4,%%zmm5,%%k1\n\t"
-				     "vpcmpgtb %%zmm6,%%zmm7,%%k2\n\t"
-				     "vpcmpgtb %%zmm12,%%zmm13,%%k3\n\t"
-				     "vpcmpgtb %%zmm14,%%zmm15,%%k4\n\t"
-				     "vpmovm2b %%k1,%%zmm5\n\t"
-				     "vpmovm2b %%k2,%%zmm7\n\t"
-				     "vpmovm2b %%k3,%%zmm13\n\t"
-				     "vpmovm2b %%k4,%%zmm15\n\t"
-				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
-				     "vpaddb %%zmm6,%%zmm6,%%zmm6\n\t"
-				     "vpaddb %%zmm12,%%zmm12,%%zmm12\n\t"
-				     "vpaddb %%zmm14,%%zmm14,%%zmm14\n\t"
-				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
-				     "vpandq %%zmm0,%%zmm7,%%zmm7\n\t"
-				     "vpandq %%zmm0,%%zmm13,%%zmm13\n\t"
-				     "vpandq %%zmm0,%%zmm15,%%zmm15\n\t"
-				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
-				     "vpxorq %%zmm7,%%zmm6,%%zmm6\n\t"
-				     "vpxorq %%zmm13,%%zmm12,%%zmm12\n\t"
-				     "vpxorq %%zmm15,%%zmm14,%%zmm14"
-				     :
-				     : );
-		}
-		asm volatile("vmovntdq %%zmm2,%0\n\t"
-			     "vmovntdq %%zmm3,%1\n\t"
-			     "vmovntdq %%zmm10,%2\n\t"
-			     "vmovntdq %%zmm11,%3\n\t"
-			     "vpxorq %4,%%zmm4,%%zmm4\n\t"
-			     "vpxorq %5,%%zmm6,%%zmm6\n\t"
-			     "vpxorq %6,%%zmm12,%%zmm12\n\t"
-			     "vpxorq %7,%%zmm14,%%zmm14\n\t"
-			     "vmovntdq %%zmm4,%4\n\t"
-			     "vmovntdq %%zmm6,%5\n\t"
-			     "vmovntdq %%zmm12,%6\n\t"
-			     "vmovntdq %%zmm14,%7"
-			     :
-			     : "m" (p[d]),  "m" (p[d+64]), "m" (p[d+128]),
-			       "m" (p[d+192]), "m" (q[d]),  "m" (q[d+64]),
-			       "m" (q[d+128]), "m" (q[d+192]));
-	}
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-const struct raid6_calls raid6_avx512x4 = {
-	raid6_avx5124_gen_syndrome,
-	raid6_avx5124_xor_syndrome,
-	raid6_have_avx512,
-	"avx512x4",
-	.priority = 2		/* Prefer AVX512 over priority 1 (SSE2 and others) */
-};
-#endif
diff --git a/lib/raid6/int.uc b/lib/raid6/int.uc
deleted file mode 100644
index 1ba56c3fa4825..0000000000000
--- a/lib/raid6/int.uc
+++ /dev/null
@@ -1,147 +0,0 @@
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
- *   Boston MA 02111-1307, USA; either version 2 of the License, or
- *   (at your option) any later version; incorporated herein by reference.
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * int$#.c
- *
- * $#-way unrolled portable integer math RAID-6 instruction set
- *
- * This file is postprocessed using unroll.awk
- */
-
-#include <linux/raid/pq.h>
-
-/*
- * This is the C data type to use
- */
-
-/* Change this from BITS_PER_LONG if there is something better... */
-#if BITS_PER_LONG == 64
-# define NBYTES(x) ((x) * 0x0101010101010101UL)
-# define NSIZE  8
-# define NSHIFT 3
-# define NSTRING "64"
-typedef u64 unative_t;
-#else
-# define NBYTES(x) ((x) * 0x01010101U)
-# define NSIZE  4
-# define NSHIFT 2
-# define NSTRING "32"
-typedef u32 unative_t;
-#endif
-
-
-
-/*
- * These sub-operations are separate inlines since they can sometimes be
- * specially optimized using architecture-specific hacks.
- */
-
-/*
- * The SHLBYTE() operation shifts each byte left by 1, *not*
- * rolling over into the next byte
- */
-static inline __attribute_const__ unative_t SHLBYTE(unative_t v)
-{
-	unative_t vv;
-
-	vv = (v << 1) & NBYTES(0xfe);
-	return vv;
-}
-
-/*
- * The MASK() operation returns 0xFF in any byte for which the high
- * bit is 1, 0x00 for any byte for which the high bit is 0.
- */
-static inline __attribute_const__ unative_t MASK(unative_t v)
-{
-	unative_t vv;
-
-	vv = v & NBYTES(0x80);
-	vv = (vv << 1) - (vv >> 7); /* Overflow on the top bit is OK */
-	return vv;
-}
-
-
-static void raid6_int$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
-		wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
-		for ( z = z0-1 ; z >= 0 ; z-- ) {
-			wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			wp$$ ^= wd$$;
-			w2$$ = MASK(wq$$);
-			w1$$ = SHLBYTE(wq$$);
-			w2$$ &= NBYTES(0x1d);
-			w1$$ ^= w2$$;
-			wq$$ = w1$$ ^ wd$$;
-		}
-		*(unative_t *)&p[d+NSIZE*$$] = wp$$;
-		*(unative_t *)&q[d+NSIZE*$$] = wq$$;
-	}
-}
-
-static void raid6_int$#_xor_syndrome(int disks, int start, int stop,
-				     size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
-		/* P/Q data pages */
-		wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
-		for ( z = z0-1 ; z >= start ; z-- ) {
-			wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			wp$$ ^= wd$$;
-			w2$$ = MASK(wq$$);
-			w1$$ = SHLBYTE(wq$$);
-			w2$$ &= NBYTES(0x1d);
-			w1$$ ^= w2$$;
-			wq$$ = w1$$ ^ wd$$;
-		}
-		/* P/Q left side optimization */
-		for ( z = start-1 ; z >= 0 ; z-- ) {
-			w2$$ = MASK(wq$$);
-			w1$$ = SHLBYTE(wq$$);
-			w2$$ &= NBYTES(0x1d);
-			wq$$ = w1$$ ^ w2$$;
-		}
-		*(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
-		*(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
-	}
-
-}
-
-const struct raid6_calls raid6_intx$# = {
-	raid6_int$#_gen_syndrome,
-	raid6_int$#_xor_syndrome,
-	NULL,			/* always valid */
-	"int" NSTRING "x$#",
-	0
-};
diff --git a/lib/raid6/loongarch.h b/lib/raid6/loongarch.h
deleted file mode 100644
index acfc33ce70562..0000000000000
--- a/lib/raid6/loongarch.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- *
- * raid6/loongarch.h
- *
- * Definitions common to LoongArch RAID-6 code only
- */
-
-#ifndef _LIB_RAID6_LOONGARCH_H
-#define _LIB_RAID6_LOONGARCH_H
-
-#ifdef __KERNEL__
-
-#include <asm/cpu-features.h>
-#include <asm/fpu.h>
-
-#else /* for user-space testing */
-
-#include <sys/auxv.h>
-
-/* have to supply these defines for glibc 2.37- and musl */
-#ifndef HWCAP_LOONGARCH_LSX
-#define HWCAP_LOONGARCH_LSX	(1 << 4)
-#endif
-#ifndef HWCAP_LOONGARCH_LASX
-#define HWCAP_LOONGARCH_LASX	(1 << 5)
-#endif
-
-#define kernel_fpu_begin()
-#define kernel_fpu_end()
-
-#define cpu_has_lsx	(getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LSX)
-#define cpu_has_lasx	(getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LASX)
-
-#endif /* __KERNEL__ */
-
-#endif /* _LIB_RAID6_LOONGARCH_H */
diff --git a/lib/raid6/loongarch_simd.c b/lib/raid6/loongarch_simd.c
deleted file mode 100644
index aa5d9f924ca39..0000000000000
--- a/lib/raid6/loongarch_simd.c
+++ /dev/null
@@ -1,422 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * RAID6 syndrome calculations in LoongArch SIMD (LSX & LASX)
- *
- * Copyright 2023 WANG Xuerui <git@xen0n.name>
- *
- * Based on the generic RAID-6 code (int.uc):
- *
- * Copyright 2002-2004 H. Peter Anvin
- */
-
-#include <linux/raid/pq.h>
-#include "loongarch.h"
-
-/*
- * The vector algorithms are currently priority 0, which means the generic
- * scalar algorithms are not being disabled if vector support is present.
- * This is like the similar LoongArch RAID5 XOR code, with the main reason
- * repeated here: it cannot be ruled out at this point of time, that some
- * future (maybe reduced) models could run the vector algorithms slower than
- * the scalar ones, maybe for errata or micro-op reasons. It may be
- * appropriate to revisit this after one or two more uarch generations.
- */
-
-#ifdef CONFIG_CPU_HAS_LSX
-#define NSIZE 16
-
-static int raid6_has_lsx(void)
-{
-	return cpu_has_lsx;
-}
-
-static void raid6_lsx_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	/*
-	 * $vr0, $vr1, $vr2, $vr3: wp
-	 * $vr4, $vr5, $vr6, $vr7: wq
-	 * $vr8, $vr9, $vr10, $vr11: wd
-	 * $vr12, $vr13, $vr14, $vr15: w2
-	 * $vr16, $vr17, $vr18, $vr19: w1
-	 */
-	for (d = 0; d < bytes; d += NSIZE*4) {
-		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile("vld $vr0, %0" : : "m"(dptr[z0][d+0*NSIZE]));
-		asm volatile("vld $vr1, %0" : : "m"(dptr[z0][d+1*NSIZE]));
-		asm volatile("vld $vr2, %0" : : "m"(dptr[z0][d+2*NSIZE]));
-		asm volatile("vld $vr3, %0" : : "m"(dptr[z0][d+3*NSIZE]));
-		asm volatile("vori.b $vr4, $vr0, 0");
-		asm volatile("vori.b $vr5, $vr1, 0");
-		asm volatile("vori.b $vr6, $vr2, 0");
-		asm volatile("vori.b $vr7, $vr3, 0");
-		for (z = z0-1; z >= 0; z--) {
-			/* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */
-			asm volatile("vld $vr8, %0" : : "m"(dptr[z][d+0*NSIZE]));
-			asm volatile("vld $vr9, %0" : : "m"(dptr[z][d+1*NSIZE]));
-			asm volatile("vld $vr10, %0" : : "m"(dptr[z][d+2*NSIZE]));
-			asm volatile("vld $vr11, %0" : : "m"(dptr[z][d+3*NSIZE]));
-			/* wp$$ ^= wd$$; */
-			asm volatile("vxor.v $vr0, $vr0, $vr8");
-			asm volatile("vxor.v $vr1, $vr1, $vr9");
-			asm volatile("vxor.v $vr2, $vr2, $vr10");
-			asm volatile("vxor.v $vr3, $vr3, $vr11");
-			/* w2$$ = MASK(wq$$); */
-			asm volatile("vslti.b $vr12, $vr4, 0");
-			asm volatile("vslti.b $vr13, $vr5, 0");
-			asm volatile("vslti.b $vr14, $vr6, 0");
-			asm volatile("vslti.b $vr15, $vr7, 0");
-			/* w1$$ = SHLBYTE(wq$$); */
-			asm volatile("vslli.b $vr16, $vr4, 1");
-			asm volatile("vslli.b $vr17, $vr5, 1");
-			asm volatile("vslli.b $vr18, $vr6, 1");
-			asm volatile("vslli.b $vr19, $vr7, 1");
-			/* w2$$ &= NBYTES(0x1d); */
-			asm volatile("vandi.b $vr12, $vr12, 0x1d");
-			asm volatile("vandi.b $vr13, $vr13, 0x1d");
-			asm volatile("vandi.b $vr14, $vr14, 0x1d");
-			asm volatile("vandi.b $vr15, $vr15, 0x1d");
-			/* w1$$ ^= w2$$; */
-			asm volatile("vxor.v $vr16, $vr16, $vr12");
-			asm volatile("vxor.v $vr17, $vr17, $vr13");
-			asm volatile("vxor.v $vr18, $vr18, $vr14");
-			asm volatile("vxor.v $vr19, $vr19, $vr15");
-			/* wq$$ = w1$$ ^ wd$$; */
-			asm volatile("vxor.v $vr4, $vr16, $vr8");
-			asm volatile("vxor.v $vr5, $vr17, $vr9");
-			asm volatile("vxor.v $vr6, $vr18, $vr10");
-			asm volatile("vxor.v $vr7, $vr19, $vr11");
-		}
-		/* *(unative_t *)&p[d+NSIZE*$$] = wp$$; */
-		asm volatile("vst $vr0, %0" : "=m"(p[d+NSIZE*0]));
-		asm volatile("vst $vr1, %0" : "=m"(p[d+NSIZE*1]));
-		asm volatile("vst $vr2, %0" : "=m"(p[d+NSIZE*2]));
-		asm volatile("vst $vr3, %0" : "=m"(p[d+NSIZE*3]));
-		/* *(unative_t *)&q[d+NSIZE*$$] = wq$$; */
-		asm volatile("vst $vr4, %0" : "=m"(q[d+NSIZE*0]));
-		asm volatile("vst $vr5, %0" : "=m"(q[d+NSIZE*1]));
-		asm volatile("vst $vr6, %0" : "=m"(q[d+NSIZE*2]));
-		asm volatile("vst $vr7, %0" : "=m"(q[d+NSIZE*3]));
-	}
-
-	kernel_fpu_end();
-}
-
-static void raid6_lsx_xor_syndrome(int disks, int start, int stop,
-				   size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	/*
-	 * $vr0, $vr1, $vr2, $vr3: wp
-	 * $vr4, $vr5, $vr6, $vr7: wq
-	 * $vr8, $vr9, $vr10, $vr11: wd
-	 * $vr12, $vr13, $vr14, $vr15: w2
-	 * $vr16, $vr17, $vr18, $vr19: w1
-	 */
-	for (d = 0; d < bytes; d += NSIZE*4) {
-		/* P/Q data pages */
-		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile("vld $vr0, %0" : : "m"(dptr[z0][d+0*NSIZE]));
-		asm volatile("vld $vr1, %0" : : "m"(dptr[z0][d+1*NSIZE]));
-		asm volatile("vld $vr2, %0" : : "m"(dptr[z0][d+2*NSIZE]));
-		asm volatile("vld $vr3, %0" : : "m"(dptr[z0][d+3*NSIZE]));
-		asm volatile("vori.b $vr4, $vr0, 0");
-		asm volatile("vori.b $vr5, $vr1, 0");
-		asm volatile("vori.b $vr6, $vr2, 0");
-		asm volatile("vori.b $vr7, $vr3, 0");
-		for (z = z0-1; z >= start; z--) {
-			/* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */
-			asm volatile("vld $vr8, %0" : : "m"(dptr[z][d+0*NSIZE]));
-			asm volatile("vld $vr9, %0" : : "m"(dptr[z][d+1*NSIZE]));
-			asm volatile("vld $vr10, %0" : : "m"(dptr[z][d+2*NSIZE]));
-			asm volatile("vld $vr11, %0" : : "m"(dptr[z][d+3*NSIZE]));
-			/* wp$$ ^= wd$$; */
-			asm volatile("vxor.v $vr0, $vr0, $vr8");
-			asm volatile("vxor.v $vr1, $vr1, $vr9");
-			asm volatile("vxor.v $vr2, $vr2, $vr10");
-			asm volatile("vxor.v $vr3, $vr3, $vr11");
-			/* w2$$ = MASK(wq$$); */
-			asm volatile("vslti.b $vr12, $vr4, 0");
-			asm volatile("vslti.b $vr13, $vr5, 0");
-			asm volatile("vslti.b $vr14, $vr6, 0");
-			asm volatile("vslti.b $vr15, $vr7, 0");
-			/* w1$$ = SHLBYTE(wq$$); */
-			asm volatile("vslli.b $vr16, $vr4, 1");
-			asm volatile("vslli.b $vr17, $vr5, 1");
-			asm volatile("vslli.b $vr18, $vr6, 1");
-			asm volatile("vslli.b $vr19, $vr7, 1");
-			/* w2$$ &= NBYTES(0x1d); */
-			asm volatile("vandi.b $vr12, $vr12, 0x1d");
-			asm volatile("vandi.b $vr13, $vr13, 0x1d");
-			asm volatile("vandi.b $vr14, $vr14, 0x1d");
-			asm volatile("vandi.b $vr15, $vr15, 0x1d");
-			/* w1$$ ^= w2$$; */
-			asm volatile("vxor.v $vr16, $vr16, $vr12");
-			asm volatile("vxor.v $vr17, $vr17, $vr13");
-			asm volatile("vxor.v $vr18, $vr18, $vr14");
-			asm volatile("vxor.v $vr19, $vr19, $vr15");
-			/* wq$$ = w1$$ ^ wd$$; */
-			asm volatile("vxor.v $vr4, $vr16, $vr8");
-			asm volatile("vxor.v $vr5, $vr17, $vr9");
-			asm volatile("vxor.v $vr6, $vr18, $vr10");
-			asm volatile("vxor.v $vr7, $vr19, $vr11");
-		}
-
-		/* P/Q left side optimization */
-		for (z = start-1; z >= 0; z--) {
-			/* w2$$ = MASK(wq$$); */
-			asm volatile("vslti.b $vr12, $vr4, 0");
-			asm volatile("vslti.b $vr13, $vr5, 0");
-			asm volatile("vslti.b $vr14, $vr6, 0");
-			asm volatile("vslti.b $vr15, $vr7, 0");
-			/* w1$$ = SHLBYTE(wq$$); */
-			asm volatile("vslli.b $vr16, $vr4, 1");
-			asm volatile("vslli.b $vr17, $vr5, 1");
-			asm volatile("vslli.b $vr18, $vr6, 1");
-			asm volatile("vslli.b $vr19, $vr7, 1");
-			/* w2$$ &= NBYTES(0x1d); */
-			asm volatile("vandi.b $vr12, $vr12, 0x1d");
-			asm volatile("vandi.b $vr13, $vr13, 0x1d");
-			asm volatile("vandi.b $vr14, $vr14, 0x1d");
-			asm volatile("vandi.b $vr15, $vr15, 0x1d");
-			/* wq$$ = w1$$ ^ w2$$; */
-			asm volatile("vxor.v $vr4, $vr16, $vr12");
-			asm volatile("vxor.v $vr5, $vr17, $vr13");
-			asm volatile("vxor.v $vr6, $vr18, $vr14");
-			asm volatile("vxor.v $vr7, $vr19, $vr15");
-		}
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
-		 */
-		asm volatile(
-			"vld $vr20, %0\n\t"
-			"vld $vr21, %1\n\t"
-			"vld $vr22, %2\n\t"
-			"vld $vr23, %3\n\t"
-			"vld $vr24, %4\n\t"
-			"vld $vr25, %5\n\t"
-			"vld $vr26, %6\n\t"
-			"vld $vr27, %7\n\t"
-			"vxor.v $vr20, $vr20, $vr0\n\t"
-			"vxor.v $vr21, $vr21, $vr1\n\t"
-			"vxor.v $vr22, $vr22, $vr2\n\t"
-			"vxor.v $vr23, $vr23, $vr3\n\t"
-			"vxor.v $vr24, $vr24, $vr4\n\t"
-			"vxor.v $vr25, $vr25, $vr5\n\t"
-			"vxor.v $vr26, $vr26, $vr6\n\t"
-			"vxor.v $vr27, $vr27, $vr7\n\t"
-			"vst $vr20, %0\n\t"
-			"vst $vr21, %1\n\t"
-			"vst $vr22, %2\n\t"
-			"vst $vr23, %3\n\t"
-			"vst $vr24, %4\n\t"
-			"vst $vr25, %5\n\t"
-			"vst $vr26, %6\n\t"
-			"vst $vr27, %7\n\t"
-			: "+m"(p[d+NSIZE*0]), "+m"(p[d+NSIZE*1]),
-			  "+m"(p[d+NSIZE*2]), "+m"(p[d+NSIZE*3]),
-			  "+m"(q[d+NSIZE*0]), "+m"(q[d+NSIZE*1]),
-			  "+m"(q[d+NSIZE*2]), "+m"(q[d+NSIZE*3])
-		);
-	}
-
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_lsx = {
-	raid6_lsx_gen_syndrome,
-	raid6_lsx_xor_syndrome,
-	raid6_has_lsx,
-	"lsx",
-	.priority = 0 /* see the comment near the top of the file for reason */
-};
-
-#undef NSIZE
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-#define NSIZE 32
-
-static int raid6_has_lasx(void)
-{
-	return cpu_has_lasx;
-}
-
-static void raid6_lasx_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	/*
-	 * $xr0, $xr1: wp
-	 * $xr2, $xr3: wq
-	 * $xr4, $xr5: wd
-	 * $xr6, $xr7: w2
-	 * $xr8, $xr9: w1
-	 */
-	for (d = 0; d < bytes; d += NSIZE*2) {
-		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile("xvld $xr0, %0" : : "m"(dptr[z0][d+0*NSIZE]));
-		asm volatile("xvld $xr1, %0" : : "m"(dptr[z0][d+1*NSIZE]));
-		asm volatile("xvori.b $xr2, $xr0, 0");
-		asm volatile("xvori.b $xr3, $xr1, 0");
-		for (z = z0-1; z >= 0; z--) {
-			/* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */
-			asm volatile("xvld $xr4, %0" : : "m"(dptr[z][d+0*NSIZE]));
-			asm volatile("xvld $xr5, %0" : : "m"(dptr[z][d+1*NSIZE]));
-			/* wp$$ ^= wd$$; */
-			asm volatile("xvxor.v $xr0, $xr0, $xr4");
-			asm volatile("xvxor.v $xr1, $xr1, $xr5");
-			/* w2$$ = MASK(wq$$); */
-			asm volatile("xvslti.b $xr6, $xr2, 0");
-			asm volatile("xvslti.b $xr7, $xr3, 0");
-			/* w1$$ = SHLBYTE(wq$$); */
-			asm volatile("xvslli.b $xr8, $xr2, 1");
-			asm volatile("xvslli.b $xr9, $xr3, 1");
-			/* w2$$ &= NBYTES(0x1d); */
-			asm volatile("xvandi.b $xr6, $xr6, 0x1d");
-			asm volatile("xvandi.b $xr7, $xr7, 0x1d");
-			/* w1$$ ^= w2$$; */
-			asm volatile("xvxor.v $xr8, $xr8, $xr6");
-			asm volatile("xvxor.v $xr9, $xr9, $xr7");
-			/* wq$$ = w1$$ ^ wd$$; */
-			asm volatile("xvxor.v $xr2, $xr8, $xr4");
-			asm volatile("xvxor.v $xr3, $xr9, $xr5");
-		}
-		/* *(unative_t *)&p[d+NSIZE*$$] = wp$$; */
-		asm volatile("xvst $xr0, %0" : "=m"(p[d+NSIZE*0]));
-		asm volatile("xvst $xr1, %0" : "=m"(p[d+NSIZE*1]));
-		/* *(unative_t *)&q[d+NSIZE*$$] = wq$$; */
-		asm volatile("xvst $xr2, %0" : "=m"(q[d+NSIZE*0]));
-		asm volatile("xvst $xr3, %0" : "=m"(q[d+NSIZE*1]));
-	}
-
-	kernel_fpu_end();
-}
-
-static void raid6_lasx_xor_syndrome(int disks, int start, int stop,
-				    size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	/*
-	 * $xr0, $xr1: wp
-	 * $xr2, $xr3: wq
-	 * $xr4, $xr5: wd
-	 * $xr6, $xr7: w2
-	 * $xr8, $xr9: w1
-	 */
-	for (d = 0; d < bytes; d += NSIZE*2) {
-		/* P/Q data pages */
-		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile("xvld $xr0, %0" : : "m"(dptr[z0][d+0*NSIZE]));
-		asm volatile("xvld $xr1, %0" : : "m"(dptr[z0][d+1*NSIZE]));
-		asm volatile("xvori.b $xr2, $xr0, 0");
-		asm volatile("xvori.b $xr3, $xr1, 0");
-		for (z = z0-1; z >= start; z--) {
-			/* wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE]; */
-			asm volatile("xvld $xr4, %0" : : "m"(dptr[z][d+0*NSIZE]));
-			asm volatile("xvld $xr5, %0" : : "m"(dptr[z][d+1*NSIZE]));
-			/* wp$$ ^= wd$$; */
-			asm volatile("xvxor.v $xr0, $xr0, $xr4");
-			asm volatile("xvxor.v $xr1, $xr1, $xr5");
-			/* w2$$ = MASK(wq$$); */
-			asm volatile("xvslti.b $xr6, $xr2, 0");
-			asm volatile("xvslti.b $xr7, $xr3, 0");
-			/* w1$$ = SHLBYTE(wq$$); */
-			asm volatile("xvslli.b $xr8, $xr2, 1");
-			asm volatile("xvslli.b $xr9, $xr3, 1");
-			/* w2$$ &= NBYTES(0x1d); */
-			asm volatile("xvandi.b $xr6, $xr6, 0x1d");
-			asm volatile("xvandi.b $xr7, $xr7, 0x1d");
-			/* w1$$ ^= w2$$; */
-			asm volatile("xvxor.v $xr8, $xr8, $xr6");
-			asm volatile("xvxor.v $xr9, $xr9, $xr7");
-			/* wq$$ = w1$$ ^ wd$$; */
-			asm volatile("xvxor.v $xr2, $xr8, $xr4");
-			asm volatile("xvxor.v $xr3, $xr9, $xr5");
-		}
-
-		/* P/Q left side optimization */
-		for (z = start-1; z >= 0; z--) {
-			/* w2$$ = MASK(wq$$); */
-			asm volatile("xvslti.b $xr6, $xr2, 0");
-			asm volatile("xvslti.b $xr7, $xr3, 0");
-			/* w1$$ = SHLBYTE(wq$$); */
-			asm volatile("xvslli.b $xr8, $xr2, 1");
-			asm volatile("xvslli.b $xr9, $xr3, 1");
-			/* w2$$ &= NBYTES(0x1d); */
-			asm volatile("xvandi.b $xr6, $xr6, 0x1d");
-			asm volatile("xvandi.b $xr7, $xr7, 0x1d");
-			/* wq$$ = w1$$ ^ w2$$; */
-			asm volatile("xvxor.v $xr2, $xr8, $xr6");
-			asm volatile("xvxor.v $xr3, $xr9, $xr7");
-		}
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
-		 */
-		asm volatile(
-			"xvld $xr10, %0\n\t"
-			"xvld $xr11, %1\n\t"
-			"xvld $xr12, %2\n\t"
-			"xvld $xr13, %3\n\t"
-			"xvxor.v $xr10, $xr10, $xr0\n\t"
-			"xvxor.v $xr11, $xr11, $xr1\n\t"
-			"xvxor.v $xr12, $xr12, $xr2\n\t"
-			"xvxor.v $xr13, $xr13, $xr3\n\t"
-			"xvst $xr10, %0\n\t"
-			"xvst $xr11, %1\n\t"
-			"xvst $xr12, %2\n\t"
-			"xvst $xr13, %3\n\t"
-			: "+m"(p[d+NSIZE*0]), "+m"(p[d+NSIZE*1]),
-			  "+m"(q[d+NSIZE*0]), "+m"(q[d+NSIZE*1])
-		);
-	}
-
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_lasx = {
-	raid6_lasx_gen_syndrome,
-	raid6_lasx_xor_syndrome,
-	raid6_has_lasx,
-	"lasx",
-	.priority = 0 /* see the comment near the top of the file for reason */
-};
-#undef NSIZE
-#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/lib/raid6/mktables.c b/lib/raid6/mktables.c
deleted file mode 100644
index 3be03793237c2..0000000000000
--- a/lib/raid6/mktables.c
+++ /dev/null
@@ -1,177 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright 2002-2007 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * mktables.c
- *
- * Make RAID-6 tables.  This is a host user space program to be run at
- * compile time.
- */
-
-#include <stdio.h>
-#include <string.h>
-#include <inttypes.h>
-#include <stdlib.h>
-#include <time.h>
-
-static uint8_t gfmul(uint8_t a, uint8_t b)
-{
-	uint8_t v = 0;
-
-	while (b) {
-		if (b & 1)
-			v ^= a;
-		a = (a << 1) ^ (a & 0x80 ? 0x1d : 0);
-		b >>= 1;
-	}
-
-	return v;
-}
-
-static uint8_t gfpow(uint8_t a, int b)
-{
-	uint8_t v = 1;
-
-	b %= 255;
-	if (b < 0)
-		b += 255;
-
-	while (b) {
-		if (b & 1)
-			v = gfmul(v, a);
-		a = gfmul(a, a);
-		b >>= 1;
-	}
-
-	return v;
-}
-
-int main(int argc, char *argv[])
-{
-	int i, j, k;
-	uint8_t v;
-	uint8_t exptbl[256], invtbl[256];
-
-	printf("#ifdef __KERNEL__\n");
-	printf("#include <linux/export.h>\n");
-	printf("#endif\n");
-	printf("#include <linux/raid/pq.h>\n");
-
-	/* Compute multiplication table */
-	printf("\nconst u8  __attribute__((aligned(256)))\n"
-		"raid6_gfmul[256][256] =\n"
-		"{\n");
-	for (i = 0; i < 256; i++) {
-		printf("\t{\n");
-		for (j = 0; j < 256; j += 8) {
-			printf("\t\t");
-			for (k = 0; k < 8; k++)
-				printf("0x%02x,%c", gfmul(i, j + k),
-				       (k == 7) ? '\n' : ' ');
-		}
-		printf("\t},\n");
-	}
-	printf("};\n");
-	printf("#ifdef __KERNEL__\n");
-	printf("EXPORT_SYMBOL(raid6_gfmul);\n");
-	printf("#endif\n");
-
-	/* Compute vector multiplication table */
-	printf("\nconst u8  __attribute__((aligned(256)))\n"
-		"raid6_vgfmul[256][32] =\n"
-		"{\n");
-	for (i = 0; i < 256; i++) {
-		printf("\t{\n");
-		for (j = 0; j < 16; j += 8) {
-			printf("\t\t");
-			for (k = 0; k < 8; k++)
-				printf("0x%02x,%c", gfmul(i, j + k),
-				       (k == 7) ? '\n' : ' ');
-		}
-		for (j = 0; j < 16; j += 8) {
-			printf("\t\t");
-			for (k = 0; k < 8; k++)
-				printf("0x%02x,%c", gfmul(i, (j + k) << 4),
-				       (k == 7) ? '\n' : ' ');
-		}
-		printf("\t},\n");
-	}
-	printf("};\n");
-	printf("#ifdef __KERNEL__\n");
-	printf("EXPORT_SYMBOL(raid6_vgfmul);\n");
-	printf("#endif\n");
-
-	/* Compute power-of-2 table (exponent) */
-	v = 1;
-	printf("\nconst u8 __attribute__((aligned(256)))\n"
-	       "raid6_gfexp[256] =\n" "{\n");
-	for (i = 0; i < 256; i += 8) {
-		printf("\t");
-		for (j = 0; j < 8; j++) {
-			exptbl[i + j] = v;
-			printf("0x%02x,%c", v, (j == 7) ? '\n' : ' ');
-			v = gfmul(v, 2);
-			if (v == 1)
-				v = 0;	/* For entry 255, not a real entry */
-		}
-	}
-	printf("};\n");
-	printf("#ifdef __KERNEL__\n");
-	printf("EXPORT_SYMBOL(raid6_gfexp);\n");
-	printf("#endif\n");
-
-	/* Compute log-of-2 table */
-	printf("\nconst u8 __attribute__((aligned(256)))\n"
-	       "raid6_gflog[256] =\n" "{\n");
-	for (i = 0; i < 256; i += 8) {
-		printf("\t");
-		for (j = 0; j < 8; j++) {
-			v = 255;
-			for (k = 0; k < 256; k++)
-				if (exptbl[k] == (i + j)) {
-					v = k;
-					break;
-				}
-			printf("0x%02x,%c", v, (j == 7) ? '\n' : ' ');
-		}
-	}
-	printf("};\n");
-	printf("#ifdef __KERNEL__\n");
-	printf("EXPORT_SYMBOL(raid6_gflog);\n");
-	printf("#endif\n");
-
-	/* Compute inverse table x^-1 == x^254 */
-	printf("\nconst u8 __attribute__((aligned(256)))\n"
-	       "raid6_gfinv[256] =\n" "{\n");
-	for (i = 0; i < 256; i += 8) {
-		printf("\t");
-		for (j = 0; j < 8; j++) {
-			invtbl[i + j] = v = gfpow(i + j, 254);
-			printf("0x%02x,%c", v, (j == 7) ? '\n' : ' ');
-		}
-	}
-	printf("};\n");
-	printf("#ifdef __KERNEL__\n");
-	printf("EXPORT_SYMBOL(raid6_gfinv);\n");
-	printf("#endif\n");
-
-	/* Compute inv(2^x + 1) (exponent-xor-inverse) table */
-	printf("\nconst u8 __attribute__((aligned(256)))\n"
-	       "raid6_gfexi[256] =\n" "{\n");
-	for (i = 0; i < 256; i += 8) {
-		printf("\t");
-		for (j = 0; j < 8; j++)
-			printf("0x%02x,%c", invtbl[exptbl[i + j] ^ 1],
-			       (j == 7) ? '\n' : ' ');
-	}
-	printf("};\n");
-	printf("#ifdef __KERNEL__\n");
-	printf("EXPORT_SYMBOL(raid6_gfexi);\n");
-	printf("#endif\n");
-
-	return 0;
-}
diff --git a/lib/raid6/mmx.c b/lib/raid6/mmx.c
deleted file mode 100644
index 3a5bf53a297b4..0000000000000
--- a/lib/raid6/mmx.c
+++ /dev/null
@@ -1,139 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright 2002 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * raid6/mmx.c
- *
- * MMX implementation of RAID-6 syndrome functions
- */
-
-#ifdef CONFIG_X86_32
-
-#include <linux/raid/pq.h>
-#include "x86.h"
-
-/* Shared with raid6/sse1.c */
-const struct raid6_mmx_constants {
-	u64 x1d;
-} raid6_mmx_constants = {
-	0x1d1d1d1d1d1d1d1dULL,
-};
-
-static int raid6_have_mmx(void)
-{
-	/* Not really "boot_cpu" but "all_cpus" */
-	return boot_cpu_has(X86_FEATURE_MMX);
-}
-
-/*
- * Plain MMX implementation
- */
-static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
-	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
-
-	for ( d = 0 ; d < bytes ; d += 8 ) {
-		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
-		asm volatile("movq %mm2,%mm4");	/* Q[0] */
-		for ( z = z0-1 ; z >= 0 ; z-- ) {
-			asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
-			asm volatile("pcmpgtb %mm4,%mm5");
-			asm volatile("paddb %mm4,%mm4");
-			asm volatile("pand %mm0,%mm5");
-			asm volatile("pxor %mm5,%mm4");
-			asm volatile("pxor %mm5,%mm5");
-			asm volatile("pxor %mm6,%mm2");
-			asm volatile("pxor %mm6,%mm4");
-		}
-		asm volatile("movq %%mm2,%0" : "=m" (p[d]));
-		asm volatile("pxor %mm2,%mm2");
-		asm volatile("movq %%mm4,%0" : "=m" (q[d]));
-		asm volatile("pxor %mm4,%mm4");
-	}
-
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_mmxx1 = {
-	raid6_mmx1_gen_syndrome,
-	NULL,			/* XOR not yet implemented */
-	raid6_have_mmx,
-	"mmxx1",
-	0
-};
-
-/*
- * Unrolled-by-2 MMX implementation
- */
-static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
-	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
-	asm volatile("pxor %mm7,%mm7"); /* Zero temp */
-
-	for ( d = 0 ; d < bytes ; d += 16 ) {
-		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
-		asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
-		asm volatile("movq %mm2,%mm4"); /* Q[0] */
-		asm volatile("movq %mm3,%mm6"); /* Q[1] */
-		for ( z = z0-1 ; z >= 0 ; z-- ) {
-			asm volatile("pcmpgtb %mm4,%mm5");
-			asm volatile("pcmpgtb %mm6,%mm7");
-			asm volatile("paddb %mm4,%mm4");
-			asm volatile("paddb %mm6,%mm6");
-			asm volatile("pand %mm0,%mm5");
-			asm volatile("pand %mm0,%mm7");
-			asm volatile("pxor %mm5,%mm4");
-			asm volatile("pxor %mm7,%mm6");
-			asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
-			asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
-			asm volatile("pxor %mm5,%mm2");
-			asm volatile("pxor %mm7,%mm3");
-			asm volatile("pxor %mm5,%mm4");
-			asm volatile("pxor %mm7,%mm6");
-			asm volatile("pxor %mm5,%mm5");
-			asm volatile("pxor %mm7,%mm7");
-		}
-		asm volatile("movq %%mm2,%0" : "=m" (p[d]));
-		asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
-		asm volatile("movq %%mm4,%0" : "=m" (q[d]));
-		asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
-	}
-
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_mmxx2 = {
-	raid6_mmx2_gen_syndrome,
-	NULL,			/* XOR not yet implemented */
-	raid6_have_mmx,
-	"mmxx2",
-	0
-};
-
-#endif
diff --git a/lib/raid6/neon.c b/lib/raid6/neon.c
deleted file mode 100644
index 6d9474ce6da91..0000000000000
--- a/lib/raid6/neon.c
+++ /dev/null
@@ -1,64 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics
- *
- * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
- */
-
-#include <linux/raid/pq.h>
-
-#ifdef __KERNEL__
-#include <asm/simd.h>
-#else
-#define scoped_ksimd()
-#define cpu_has_neon()		(1)
-#endif
-
-/*
- * There are 2 reasons these wrappers are kept in a separate compilation unit
- * from the actual implementations in neonN.c (generated from neon.uc by
- * unroll.awk):
- * - the actual implementations use NEON intrinsics, and the GCC support header
- *   (arm_neon.h) is not fully compatible (type wise) with the kernel;
- * - the neonN.c files are compiled with -mfpu=neon and optimization enabled,
- *   and we have to make sure that we never use *any* NEON/VFP instructions
- *   outside a kernel_neon_begin()/kernel_neon_end() pair.
- */
-
-#define RAID6_NEON_WRAPPER(_n)						\
-	static void raid6_neon ## _n ## _gen_syndrome(int disks,	\
-					size_t bytes, void **ptrs)	\
-	{								\
-		void raid6_neon ## _n  ## _gen_syndrome_real(int,	\
-						unsigned long, void**);	\
-		scoped_ksimd()						\
-			raid6_neon ## _n ## _gen_syndrome_real(disks,	\
-					(unsigned long)bytes, ptrs);	\
-	}								\
-	static void raid6_neon ## _n ## _xor_syndrome(int disks,	\
-					int start, int stop, 		\
-					size_t bytes, void **ptrs)	\
-	{								\
-		void raid6_neon ## _n  ## _xor_syndrome_real(int,	\
-				int, int, unsigned long, void**);	\
-		scoped_ksimd()						\
-			raid6_neon ## _n ## _xor_syndrome_real(disks,	\
-				start, stop, (unsigned long)bytes, ptrs);\
-	}								\
-	struct raid6_calls const raid6_neonx ## _n = {			\
-		raid6_neon ## _n ## _gen_syndrome,			\
-		raid6_neon ## _n ## _xor_syndrome,			\
-		raid6_have_neon,					\
-		"neonx" #_n,						\
-		0							\
-	}
-
-static int raid6_have_neon(void)
-{
-	return cpu_has_neon();
-}
-
-RAID6_NEON_WRAPPER(1);
-RAID6_NEON_WRAPPER(2);
-RAID6_NEON_WRAPPER(4);
-RAID6_NEON_WRAPPER(8);
diff --git a/lib/raid6/neon.h b/lib/raid6/neon.h
deleted file mode 100644
index 2ca41ee9b4996..0000000000000
--- a/lib/raid6/neon.h
+++ /dev/null
@@ -1,22 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-void raid6_neon1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs);
-void raid6_neon1_xor_syndrome_real(int disks, int start, int stop,
-				    unsigned long bytes, void **ptrs);
-void raid6_neon2_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs);
-void raid6_neon2_xor_syndrome_real(int disks, int start, int stop,
-				    unsigned long bytes, void **ptrs);
-void raid6_neon4_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs);
-void raid6_neon4_xor_syndrome_real(int disks, int start, int stop,
-				    unsigned long bytes, void **ptrs);
-void raid6_neon8_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs);
-void raid6_neon8_xor_syndrome_real(int disks, int start, int stop,
-				    unsigned long bytes, void **ptrs);
-void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
-			      uint8_t *dq, const uint8_t *pbmul,
-			      const uint8_t *qmul);
-
-void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
-			      const uint8_t *qmul);
-
-
diff --git a/lib/raid6/neon.uc b/lib/raid6/neon.uc
deleted file mode 100644
index 355270af0cd61..0000000000000
--- a/lib/raid6/neon.uc
+++ /dev/null
@@ -1,153 +0,0 @@
-/* -----------------------------------------------------------------------
- *
- *   neon.uc - RAID-6 syndrome calculation using ARM NEON instructions
- *
- *   Copyright (C) 2012 Rob Herring
- *   Copyright (C) 2015 Linaro Ltd. <ard.biesheuvel@linaro.org>
- *
- *   Based on altivec.uc:
- *     Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
- *
- *   This program is free software; you can redistribute it and/or modify
- *   it under the terms of the GNU General Public License as published by
- *   the Free Software Foundation, Inc., 53 Temple Place Ste 330,
- *   Boston MA 02111-1307, USA; either version 2 of the License, or
- *   (at your option) any later version; incorporated herein by reference.
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * neon$#.c
- *
- * $#-way unrolled NEON intrinsics math RAID-6 instruction set
- *
- * This file is postprocessed using unroll.awk
- */
-
-#include <arm_neon.h>
-#include "neon.h"
-
-typedef uint8x16_t unative_t;
-
-#define NSIZE	sizeof(unative_t)
-
-/*
- * The SHLBYTE() operation shifts each byte left by 1, *not*
- * rolling over into the next byte
- */
-static inline unative_t SHLBYTE(unative_t v)
-{
-	return vshlq_n_u8(v, 1);
-}
-
-/*
- * The MASK() operation returns 0xFF in any byte for which the high
- * bit is 1, 0x00 for any byte for which the high bit is 0.
- */
-static inline unative_t MASK(unative_t v)
-{
-	return (unative_t)vshrq_n_s8((int8x16_t)v, 7);
-}
-
-static inline unative_t PMUL(unative_t v, unative_t u)
-{
-	return (unative_t)vmulq_p8((poly8x16_t)v, (poly8x16_t)u);
-}
-
-void raid6_neon$#_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
-{
-	uint8_t **dptr = (uint8_t **)ptrs;
-	uint8_t *p, *q;
-	int d, z, z0;
-
-	register unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
-	const unative_t x1d = vdupq_n_u8(0x1d);
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
-		wq$$ = wp$$ = vld1q_u8(&dptr[z0][d+$$*NSIZE]);
-		for ( z = z0-1 ; z >= 0 ; z-- ) {
-			wd$$ = vld1q_u8(&dptr[z][d+$$*NSIZE]);
-			wp$$ = veorq_u8(wp$$, wd$$);
-			w2$$ = MASK(wq$$);
-			w1$$ = SHLBYTE(wq$$);
-
-			w2$$ = vandq_u8(w2$$, x1d);
-			w1$$ = veorq_u8(w1$$, w2$$);
-			wq$$ = veorq_u8(w1$$, wd$$);
-		}
-		vst1q_u8(&p[d+NSIZE*$$], wp$$);
-		vst1q_u8(&q[d+NSIZE*$$], wq$$);
-	}
-}
-
-void raid6_neon$#_xor_syndrome_real(int disks, int start, int stop,
-				    unsigned long bytes, void **ptrs)
-{
-	uint8_t **dptr = (uint8_t **)ptrs;
-	uint8_t *p, *q;
-	int d, z, z0;
-
-	register unative_t wd$$, wq$$, wp$$, w1$$, w2$$;
-	const unative_t x1d = vdupq_n_u8(0x1d);
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	for ( d = 0 ; d < bytes ; d += NSIZE*$# ) {
-		wq$$ = vld1q_u8(&dptr[z0][d+$$*NSIZE]);
-		wp$$ = veorq_u8(vld1q_u8(&p[d+$$*NSIZE]), wq$$);
-
-		/* P/Q data pages */
-		for ( z = z0-1 ; z >= start ; z-- ) {
-			wd$$ = vld1q_u8(&dptr[z][d+$$*NSIZE]);
-			wp$$ = veorq_u8(wp$$, wd$$);
-			w2$$ = MASK(wq$$);
-			w1$$ = SHLBYTE(wq$$);
-
-			w2$$ = vandq_u8(w2$$, x1d);
-			w1$$ = veorq_u8(w1$$, w2$$);
-			wq$$ = veorq_u8(w1$$, wd$$);
-		}
-		/* P/Q left side optimization */
-		for ( z = start-1 ; z >= 3 ; z -= 4 ) {
-			w2$$ = vshrq_n_u8(wq$$, 4);
-			w1$$ = vshlq_n_u8(wq$$, 4);
-
-			w2$$ = PMUL(w2$$, x1d);
-			wq$$ = veorq_u8(w1$$, w2$$);
-		}
-
-		switch (z) {
-		case 2:
-			w2$$ = vshrq_n_u8(wq$$, 5);
-			w1$$ = vshlq_n_u8(wq$$, 3);
-
-			w2$$ = PMUL(w2$$, x1d);
-			wq$$ = veorq_u8(w1$$, w2$$);
-			break;
-		case 1:
-			w2$$ = vshrq_n_u8(wq$$, 6);
-			w1$$ = vshlq_n_u8(wq$$, 2);
-
-			w2$$ = PMUL(w2$$, x1d);
-			wq$$ = veorq_u8(w1$$, w2$$);
-			break;
-		case 0:
-			w2$$ = MASK(wq$$);
-			w1$$ = SHLBYTE(wq$$);
-
-			w2$$ = vandq_u8(w2$$, x1d);
-			wq$$ = veorq_u8(w1$$, w2$$);
-		}
-		w1$$ = vld1q_u8(&q[d+NSIZE*$$]);
-		wq$$ = veorq_u8(wq$$, w1$$);
-
-		vst1q_u8(&p[d+NSIZE*$$], wp$$);
-		vst1q_u8(&q[d+NSIZE*$$], wq$$);
-	}
-}
diff --git a/lib/raid6/recov.c b/lib/raid6/recov.c
deleted file mode 100644
index b5e47c008b41b..0000000000000
--- a/lib/raid6/recov.c
+++ /dev/null
@@ -1,135 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright 2002 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * raid6/recov.c
- *
- * RAID-6 data recovery in dual failure mode.  In single failure mode,
- * use the RAID-5 algorithm (or, in the case of Q failure, just reconstruct
- * the syndrome.)
- */
-
-#include <linux/raid/pq.h>
-
-/* Recover two failed data blocks. */
-static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila,
-		int failb, void **ptrs)
-{
-	u8 *p, *q, *dp, *dq;
-	u8 px, qx, db;
-	const u8 *pbmul;	/* P multiplier table for B data */
-	const u8 *qmul;		/* Q multiplier table (for both) */
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/* Compute syndrome with zero for the missing data pages
-	   Use the dead data pages as temporary storage for
-	   delta p and delta q */
-	dp = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-2] = dp;
-	dq = (u8 *)ptrs[failb];
-	ptrs[failb] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dp;
-	ptrs[failb]   = dq;
-	ptrs[disks-2] = p;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	pbmul = raid6_gfmul[raid6_gfexi[failb-faila]];
-	qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]];
-
-	/* Now do it... */
-	while ( bytes-- ) {
-		px    = *p ^ *dp;
-		qx    = qmul[*q ^ *dq];
-		*dq++ = db = pbmul[px] ^ qx; /* Reconstructed B */
-		*dp++ = db ^ px; /* Reconstructed A */
-		p++; q++;
-	}
-}
-
-/* Recover failure of one data block plus the P block */
-static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila,
-		void **ptrs)
-{
-	u8 *p, *q, *dq;
-	const u8 *qmul;		/* Q multiplier table */
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/* Compute syndrome with zero for the missing data page
-	   Use the dead data page as temporary storage for delta q */
-	dq = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dq;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]]];
-
-	/* Now do it... */
-	while ( bytes-- ) {
-		*p++ ^= *dq = qmul[*q ^ *dq];
-		q++; dq++;
-	}
-}
-
-
-const struct raid6_recov_calls raid6_recov_intx1 = {
-	.data2 = raid6_2data_recov_intx1,
-	.datap = raid6_datap_recov_intx1,
-	.valid = NULL,
-	.name = "intx1",
-	.priority = 0,
-};
-
-#ifndef __KERNEL__
-/* Testing only */
-
-/* Recover two failed blocks. */
-void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, void **ptrs)
-{
-	if ( faila > failb ) {
-		int tmp = faila;
-		faila = failb;
-		failb = tmp;
-	}
-
-	if ( failb == disks-1 ) {
-		if ( faila == disks-2 ) {
-			/* P+Q failure.  Just rebuild the syndrome. */
-			raid6_call.gen_syndrome(disks, bytes, ptrs);
-		} else {
-			/* data+Q failure.  Reconstruct data from P,
-			   then rebuild syndrome. */
-			/* NOT IMPLEMENTED - equivalent to RAID-5 */
-		}
-	} else {
-		if ( failb == disks-2 ) {
-			/* data+P failure. */
-			raid6_datap_recov(disks, bytes, faila, ptrs);
-		} else {
-			/* data+data failure. */
-			raid6_2data_recov(disks, bytes, faila, failb, ptrs);
-		}
-	}
-}
-
-#endif
diff --git a/lib/raid6/recov_avx2.c b/lib/raid6/recov_avx2.c
deleted file mode 100644
index 97d598d2535ca..0000000000000
--- a/lib/raid6/recov_avx2.c
+++ /dev/null
@@ -1,313 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2012 Intel Corporation
- * Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
- */
-
-#include <linux/raid/pq.h>
-#include "x86.h"
-
-static int raid6_has_avx2(void)
-{
-	return boot_cpu_has(X86_FEATURE_AVX2) &&
-		boot_cpu_has(X86_FEATURE_AVX);
-}
-
-static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila,
-		int failb, void **ptrs)
-{
-	u8 *p, *q, *dp, *dq;
-	const u8 *pbmul;	/* P multiplier table for B data */
-	const u8 *qmul;		/* Q multiplier table (for both) */
-	const u8 x0f = 0x0f;
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/* Compute syndrome with zero for the missing data pages
-	   Use the dead data pages as temporary storage for
-	   delta p and delta q */
-	dp = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-2] = dp;
-	dq = (u8 *)ptrs[failb];
-	ptrs[failb] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dp;
-	ptrs[failb]   = dq;
-	ptrs[disks-2] = p;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
-		raid6_gfexp[failb]]];
-
-	kernel_fpu_begin();
-
-	/* ymm0 = x0f[16] */
-	asm volatile("vpbroadcastb %0, %%ymm7" : : "m" (x0f));
-
-	while (bytes) {
-#ifdef CONFIG_X86_64
-		asm volatile("vmovdqa %0, %%ymm1" : : "m" (q[0]));
-		asm volatile("vmovdqa %0, %%ymm9" : : "m" (q[32]));
-		asm volatile("vmovdqa %0, %%ymm0" : : "m" (p[0]));
-		asm volatile("vmovdqa %0, %%ymm8" : : "m" (p[32]));
-		asm volatile("vpxor %0, %%ymm1, %%ymm1" : : "m" (dq[0]));
-		asm volatile("vpxor %0, %%ymm9, %%ymm9" : : "m" (dq[32]));
-		asm volatile("vpxor %0, %%ymm0, %%ymm0" : : "m" (dp[0]));
-		asm volatile("vpxor %0, %%ymm8, %%ymm8" : : "m" (dp[32]));
-
-		/*
-		 * 1 = dq[0]  ^ q[0]
-		 * 9 = dq[32] ^ q[32]
-		 * 0 = dp[0]  ^ p[0]
-		 * 8 = dp[32] ^ p[32]
-		 */
-
-		asm volatile("vbroadcasti128 %0, %%ymm4" : : "m" (qmul[0]));
-		asm volatile("vbroadcasti128 %0, %%ymm5" : : "m" (qmul[16]));
-
-		asm volatile("vpsraw $4, %ymm1, %ymm3");
-		asm volatile("vpsraw $4, %ymm9, %ymm12");
-		asm volatile("vpand %ymm7, %ymm1, %ymm1");
-		asm volatile("vpand %ymm7, %ymm9, %ymm9");
-		asm volatile("vpand %ymm7, %ymm3, %ymm3");
-		asm volatile("vpand %ymm7, %ymm12, %ymm12");
-		asm volatile("vpshufb %ymm9, %ymm4, %ymm14");
-		asm volatile("vpshufb %ymm1, %ymm4, %ymm4");
-		asm volatile("vpshufb %ymm12, %ymm5, %ymm15");
-		asm volatile("vpshufb %ymm3, %ymm5, %ymm5");
-		asm volatile("vpxor %ymm14, %ymm15, %ymm15");
-		asm volatile("vpxor %ymm4, %ymm5, %ymm5");
-
-		/*
-		 * 5 = qx[0]
-		 * 15 = qx[32]
-		 */
-
-		asm volatile("vbroadcasti128 %0, %%ymm4" : : "m" (pbmul[0]));
-		asm volatile("vbroadcasti128 %0, %%ymm1" : : "m" (pbmul[16]));
-		asm volatile("vpsraw $4, %ymm0, %ymm2");
-		asm volatile("vpsraw $4, %ymm8, %ymm6");
-		asm volatile("vpand %ymm7, %ymm0, %ymm3");
-		asm volatile("vpand %ymm7, %ymm8, %ymm14");
-		asm volatile("vpand %ymm7, %ymm2, %ymm2");
-		asm volatile("vpand %ymm7, %ymm6, %ymm6");
-		asm volatile("vpshufb %ymm14, %ymm4, %ymm12");
-		asm volatile("vpshufb %ymm3, %ymm4, %ymm4");
-		asm volatile("vpshufb %ymm6, %ymm1, %ymm13");
-		asm volatile("vpshufb %ymm2, %ymm1, %ymm1");
-		asm volatile("vpxor %ymm4, %ymm1, %ymm1");
-		asm volatile("vpxor %ymm12, %ymm13, %ymm13");
-
-		/*
-		 * 1  = pbmul[px[0]]
-		 * 13 = pbmul[px[32]]
-		 */
-		asm volatile("vpxor %ymm5, %ymm1, %ymm1");
-		asm volatile("vpxor %ymm15, %ymm13, %ymm13");
-
-		/*
-		 * 1 = db = DQ
-		 * 13 = db[32] = DQ[32]
-		 */
-		asm volatile("vmovdqa %%ymm1, %0" : "=m" (dq[0]));
-		asm volatile("vmovdqa %%ymm13,%0" : "=m" (dq[32]));
-		asm volatile("vpxor %ymm1, %ymm0, %ymm0");
-		asm volatile("vpxor %ymm13, %ymm8, %ymm8");
-
-		asm volatile("vmovdqa %%ymm0, %0" : "=m" (dp[0]));
-		asm volatile("vmovdqa %%ymm8, %0" : "=m" (dp[32]));
-
-		bytes -= 64;
-		p += 64;
-		q += 64;
-		dp += 64;
-		dq += 64;
-#else
-		asm volatile("vmovdqa %0, %%ymm1" : : "m" (*q));
-		asm volatile("vmovdqa %0, %%ymm0" : : "m" (*p));
-		asm volatile("vpxor %0, %%ymm1, %%ymm1" : : "m" (*dq));
-		asm volatile("vpxor %0, %%ymm0, %%ymm0" : : "m" (*dp));
-
-		/* 1 = dq ^ q;  0 = dp ^ p */
-
-		asm volatile("vbroadcasti128 %0, %%ymm4" : : "m" (qmul[0]));
-		asm volatile("vbroadcasti128 %0, %%ymm5" : : "m" (qmul[16]));
-
-		/*
-		 * 1 = dq ^ q
-		 * 3 = dq ^ p >> 4
-		 */
-		asm volatile("vpsraw $4, %ymm1, %ymm3");
-		asm volatile("vpand %ymm7, %ymm1, %ymm1");
-		asm volatile("vpand %ymm7, %ymm3, %ymm3");
-		asm volatile("vpshufb %ymm1, %ymm4, %ymm4");
-		asm volatile("vpshufb %ymm3, %ymm5, %ymm5");
-		asm volatile("vpxor %ymm4, %ymm5, %ymm5");
-
-		/* 5 = qx */
-
-		asm volatile("vbroadcasti128 %0, %%ymm4" : : "m" (pbmul[0]));
-		asm volatile("vbroadcasti128 %0, %%ymm1" : : "m" (pbmul[16]));
-
-		asm volatile("vpsraw $4, %ymm0, %ymm2");
-		asm volatile("vpand %ymm7, %ymm0, %ymm3");
-		asm volatile("vpand %ymm7, %ymm2, %ymm2");
-		asm volatile("vpshufb %ymm3, %ymm4, %ymm4");
-		asm volatile("vpshufb %ymm2, %ymm1, %ymm1");
-		asm volatile("vpxor %ymm4, %ymm1, %ymm1");
-
-		/* 1 = pbmul[px] */
-		asm volatile("vpxor %ymm5, %ymm1, %ymm1");
-		/* 1 = db = DQ */
-		asm volatile("vmovdqa %%ymm1, %0" : "=m" (dq[0]));
-
-		asm volatile("vpxor %ymm1, %ymm0, %ymm0");
-		asm volatile("vmovdqa %%ymm0, %0" : "=m" (dp[0]));
-
-		bytes -= 32;
-		p += 32;
-		q += 32;
-		dp += 32;
-		dq += 32;
-#endif
-	}
-
-	kernel_fpu_end();
-}
-
-static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila,
-		void **ptrs)
-{
-	u8 *p, *q, *dq;
-	const u8 *qmul;		/* Q multiplier table */
-	const u8 x0f = 0x0f;
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/* Compute syndrome with zero for the missing data page
-	   Use the dead data page as temporary storage for delta q */
-	dq = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dq;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
-
-	kernel_fpu_begin();
-
-	asm volatile("vpbroadcastb %0, %%ymm7" : : "m" (x0f));
-
-	while (bytes) {
-#ifdef CONFIG_X86_64
-		asm volatile("vmovdqa %0, %%ymm3" : : "m" (dq[0]));
-		asm volatile("vmovdqa %0, %%ymm8" : : "m" (dq[32]));
-		asm volatile("vpxor %0, %%ymm3, %%ymm3" : : "m" (q[0]));
-		asm volatile("vpxor %0, %%ymm8, %%ymm8" : : "m" (q[32]));
-
-		/*
-		 * 3 = q[0] ^ dq[0]
-		 * 8 = q[32] ^ dq[32]
-		 */
-		asm volatile("vbroadcasti128 %0, %%ymm0" : : "m" (qmul[0]));
-		asm volatile("vmovapd %ymm0, %ymm13");
-		asm volatile("vbroadcasti128 %0, %%ymm1" : : "m" (qmul[16]));
-		asm volatile("vmovapd %ymm1, %ymm14");
-
-		asm volatile("vpsraw $4, %ymm3, %ymm6");
-		asm volatile("vpsraw $4, %ymm8, %ymm12");
-		asm volatile("vpand %ymm7, %ymm3, %ymm3");
-		asm volatile("vpand %ymm7, %ymm8, %ymm8");
-		asm volatile("vpand %ymm7, %ymm6, %ymm6");
-		asm volatile("vpand %ymm7, %ymm12, %ymm12");
-		asm volatile("vpshufb %ymm3, %ymm0, %ymm0");
-		asm volatile("vpshufb %ymm8, %ymm13, %ymm13");
-		asm volatile("vpshufb %ymm6, %ymm1, %ymm1");
-		asm volatile("vpshufb %ymm12, %ymm14, %ymm14");
-		asm volatile("vpxor %ymm0, %ymm1, %ymm1");
-		asm volatile("vpxor %ymm13, %ymm14, %ymm14");
-
-		/*
-		 * 1  = qmul[q[0]  ^ dq[0]]
-		 * 14 = qmul[q[32] ^ dq[32]]
-		 */
-		asm volatile("vmovdqa %0, %%ymm2" : : "m" (p[0]));
-		asm volatile("vmovdqa %0, %%ymm12" : : "m" (p[32]));
-		asm volatile("vpxor %ymm1, %ymm2, %ymm2");
-		asm volatile("vpxor %ymm14, %ymm12, %ymm12");
-
-		/*
-		 * 2  = p[0]  ^ qmul[q[0]  ^ dq[0]]
-		 * 12 = p[32] ^ qmul[q[32] ^ dq[32]]
-		 */
-
-		asm volatile("vmovdqa %%ymm1, %0" : "=m" (dq[0]));
-		asm volatile("vmovdqa %%ymm14, %0" : "=m" (dq[32]));
-		asm volatile("vmovdqa %%ymm2, %0" : "=m" (p[0]));
-		asm volatile("vmovdqa %%ymm12,%0" : "=m" (p[32]));
-
-		bytes -= 64;
-		p += 64;
-		q += 64;
-		dq += 64;
-#else
-		asm volatile("vmovdqa %0, %%ymm3" : : "m" (dq[0]));
-		asm volatile("vpxor %0, %%ymm3, %%ymm3" : : "m" (q[0]));
-
-		/* 3 = q ^ dq */
-
-		asm volatile("vbroadcasti128 %0, %%ymm0" : : "m" (qmul[0]));
-		asm volatile("vbroadcasti128 %0, %%ymm1" : : "m" (qmul[16]));
-
-		asm volatile("vpsraw $4, %ymm3, %ymm6");
-		asm volatile("vpand %ymm7, %ymm3, %ymm3");
-		asm volatile("vpand %ymm7, %ymm6, %ymm6");
-		asm volatile("vpshufb %ymm3, %ymm0, %ymm0");
-		asm volatile("vpshufb %ymm6, %ymm1, %ymm1");
-		asm volatile("vpxor %ymm0, %ymm1, %ymm1");
-
-		/* 1 = qmul[q ^ dq] */
-
-		asm volatile("vmovdqa %0, %%ymm2" : : "m" (p[0]));
-		asm volatile("vpxor %ymm1, %ymm2, %ymm2");
-
-		/* 2 = p ^ qmul[q ^ dq] */
-
-		asm volatile("vmovdqa %%ymm1, %0" : "=m" (dq[0]));
-		asm volatile("vmovdqa %%ymm2, %0" : "=m" (p[0]));
-
-		bytes -= 32;
-		p += 32;
-		q += 32;
-		dq += 32;
-#endif
-	}
-
-	kernel_fpu_end();
-}
-
-const struct raid6_recov_calls raid6_recov_avx2 = {
-	.data2 = raid6_2data_recov_avx2,
-	.datap = raid6_datap_recov_avx2,
-	.valid = raid6_has_avx2,
-#ifdef CONFIG_X86_64
-	.name = "avx2x2",
-#else
-	.name = "avx2x1",
-#endif
-	.priority = 2,
-};
diff --git a/lib/raid6/recov_avx512.c b/lib/raid6/recov_avx512.c
deleted file mode 100644
index 7986120ca4442..0000000000000
--- a/lib/raid6/recov_avx512.c
+++ /dev/null
@@ -1,377 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2016 Intel Corporation
- *
- * Author: Gayatri Kammela <gayatri.kammela@intel.com>
- * Author: Megha Dey <megha.dey@linux.intel.com>
- */
-
-#include <linux/raid/pq.h>
-#include "x86.h"
-
-static int raid6_has_avx512(void)
-{
-	return boot_cpu_has(X86_FEATURE_AVX2) &&
-		boot_cpu_has(X86_FEATURE_AVX) &&
-		boot_cpu_has(X86_FEATURE_AVX512F) &&
-		boot_cpu_has(X86_FEATURE_AVX512BW) &&
-		boot_cpu_has(X86_FEATURE_AVX512VL) &&
-		boot_cpu_has(X86_FEATURE_AVX512DQ);
-}
-
-static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila,
-				     int failb, void **ptrs)
-{
-	u8 *p, *q, *dp, *dq;
-	const u8 *pbmul;	/* P multiplier table for B data */
-	const u8 *qmul;		/* Q multiplier table (for both) */
-	const u8 x0f = 0x0f;
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/*
-	 * Compute syndrome with zero for the missing data pages
-	 * Use the dead data pages as temporary storage for
-	 * delta p and delta q
-	 */
-
-	dp = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-2] = dp;
-	dq = (u8 *)ptrs[failb];
-	ptrs[failb] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dp;
-	ptrs[failb]   = dq;
-	ptrs[disks-2] = p;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
-		raid6_gfexp[failb]]];
-
-	kernel_fpu_begin();
-
-	/* zmm0 = x0f[16] */
-	asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));
-
-	while (bytes) {
-#ifdef CONFIG_X86_64
-		asm volatile("vmovdqa64 %0, %%zmm1\n\t"
-			     "vmovdqa64 %1, %%zmm9\n\t"
-			     "vmovdqa64 %2, %%zmm0\n\t"
-			     "vmovdqa64 %3, %%zmm8\n\t"
-			     "vpxorq %4, %%zmm1, %%zmm1\n\t"
-			     "vpxorq %5, %%zmm9, %%zmm9\n\t"
-			     "vpxorq %6, %%zmm0, %%zmm0\n\t"
-			     "vpxorq %7, %%zmm8, %%zmm8"
-			     :
-			     : "m" (q[0]), "m" (q[64]), "m" (p[0]),
-			       "m" (p[64]), "m" (dq[0]), "m" (dq[64]),
-			       "m" (dp[0]), "m" (dp[64]));
-
-		/*
-		 * 1 = dq[0]  ^ q[0]
-		 * 9 = dq[64] ^ q[64]
-		 * 0 = dp[0]  ^ p[0]
-		 * 8 = dp[64] ^ p[64]
-		 */
-
-		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
-			     "vbroadcasti64x2 %1, %%zmm5"
-			     :
-			     : "m" (qmul[0]), "m" (qmul[16]));
-
-		asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t"
-			     "vpsraw $4, %%zmm9, %%zmm12\n\t"
-			     "vpandq %%zmm7, %%zmm1, %%zmm1\n\t"
-			     "vpandq %%zmm7, %%zmm9, %%zmm9\n\t"
-			     "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
-			     "vpandq %%zmm7, %%zmm12, %%zmm12\n\t"
-			     "vpshufb %%zmm9, %%zmm4, %%zmm14\n\t"
-			     "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t"
-			     "vpshufb %%zmm12, %%zmm5, %%zmm15\n\t"
-			     "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t"
-			     "vpxorq %%zmm14, %%zmm15, %%zmm15\n\t"
-			     "vpxorq %%zmm4, %%zmm5, %%zmm5"
-			     :
-			     : );
-
-		/*
-		 * 5 = qx[0]
-		 * 15 = qx[64]
-		 */
-
-		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
-			     "vbroadcasti64x2 %1, %%zmm1\n\t"
-			     "vpsraw $4, %%zmm0, %%zmm2\n\t"
-			     "vpsraw $4, %%zmm8, %%zmm6\n\t"
-			     "vpandq %%zmm7, %%zmm0, %%zmm3\n\t"
-			     "vpandq %%zmm7, %%zmm8, %%zmm14\n\t"
-			     "vpandq %%zmm7, %%zmm2, %%zmm2\n\t"
-			     "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
-			     "vpshufb %%zmm14, %%zmm4, %%zmm12\n\t"
-			     "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t"
-			     "vpshufb %%zmm6, %%zmm1, %%zmm13\n\t"
-			     "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t"
-			     "vpxorq %%zmm4, %%zmm1, %%zmm1\n\t"
-			     "vpxorq %%zmm12, %%zmm13, %%zmm13"
-			     :
-			     : "m" (pbmul[0]), "m" (pbmul[16]));
-
-		/*
-		 * 1  = pbmul[px[0]]
-		 * 13 = pbmul[px[64]]
-		 */
-		asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t"
-			     "vpxorq %%zmm15, %%zmm13, %%zmm13"
-			     :
-			     : );
-
-		/*
-		 * 1 = db = DQ
-		 * 13 = db[64] = DQ[64]
-		 */
-		asm volatile("vmovdqa64 %%zmm1, %0\n\t"
-			     "vmovdqa64 %%zmm13,%1\n\t"
-			     "vpxorq %%zmm1, %%zmm0, %%zmm0\n\t"
-			     "vpxorq %%zmm13, %%zmm8, %%zmm8"
-			     :
-			     : "m" (dq[0]), "m" (dq[64]));
-
-		asm volatile("vmovdqa64 %%zmm0, %0\n\t"
-			     "vmovdqa64 %%zmm8, %1"
-			     :
-			     : "m" (dp[0]), "m" (dp[64]));
-
-		bytes -= 128;
-		p += 128;
-		q += 128;
-		dp += 128;
-		dq += 128;
-#else
-		asm volatile("vmovdqa64 %0, %%zmm1\n\t"
-			     "vmovdqa64 %1, %%zmm0\n\t"
-			     "vpxorq %2, %%zmm1, %%zmm1\n\t"
-			     "vpxorq %3, %%zmm0, %%zmm0"
-			     :
-			     : "m" (*q), "m" (*p), "m"(*dq), "m" (*dp));
-
-		/* 1 = dq ^ q;  0 = dp ^ p */
-
-		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
-			     "vbroadcasti64x2 %1, %%zmm5"
-			     :
-			     : "m" (qmul[0]), "m" (qmul[16]));
-
-		/*
-		 * 1 = dq ^ q
-		 * 3 = dq ^ p >> 4
-		 */
-		asm volatile("vpsraw $4, %%zmm1, %%zmm3\n\t"
-			     "vpandq %%zmm7, %%zmm1, %%zmm1\n\t"
-			     "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
-			     "vpshufb %%zmm1, %%zmm4, %%zmm4\n\t"
-			     "vpshufb %%zmm3, %%zmm5, %%zmm5\n\t"
-			     "vpxorq %%zmm4, %%zmm5, %%zmm5"
-			     :
-			     : );
-
-		/* 5 = qx */
-
-		asm volatile("vbroadcasti64x2 %0, %%zmm4\n\t"
-			     "vbroadcasti64x2 %1, %%zmm1"
-			     :
-			     : "m" (pbmul[0]), "m" (pbmul[16]));
-
-		asm volatile("vpsraw $4, %%zmm0, %%zmm2\n\t"
-			     "vpandq %%zmm7, %%zmm0, %%zmm3\n\t"
-			     "vpandq %%zmm7, %%zmm2, %%zmm2\n\t"
-			     "vpshufb %%zmm3, %%zmm4, %%zmm4\n\t"
-			     "vpshufb %%zmm2, %%zmm1, %%zmm1\n\t"
-			     "vpxorq %%zmm4, %%zmm1, %%zmm1"
-			     :
-			     : );
-
-		/* 1 = pbmul[px] */
-		asm volatile("vpxorq %%zmm5, %%zmm1, %%zmm1\n\t"
-			     /* 1 = db = DQ */
-			     "vmovdqa64 %%zmm1, %0\n\t"
-			     :
-			     : "m" (dq[0]));
-
-		asm volatile("vpxorq %%zmm1, %%zmm0, %%zmm0\n\t"
-			     "vmovdqa64 %%zmm0, %0"
-			     :
-			     : "m" (dp[0]));
-
-		bytes -= 64;
-		p += 64;
-		q += 64;
-		dp += 64;
-		dq += 64;
-#endif
-	}
-
-	kernel_fpu_end();
-}
-
-static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,
-				     void **ptrs)
-{
-	u8 *p, *q, *dq;
-	const u8 *qmul;		/* Q multiplier table */
-	const u8 x0f = 0x0f;
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/*
-	 * Compute syndrome with zero for the missing data page
-	 * Use the dead data page as temporary storage for delta q
-	 */
-
-	dq = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dq;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
-
-	kernel_fpu_begin();
-
-	asm volatile("vpbroadcastb %0, %%zmm7" : : "m" (x0f));
-
-	while (bytes) {
-#ifdef CONFIG_X86_64
-		asm volatile("vmovdqa64 %0, %%zmm3\n\t"
-			     "vmovdqa64 %1, %%zmm8\n\t"
-			     "vpxorq %2, %%zmm3, %%zmm3\n\t"
-			     "vpxorq %3, %%zmm8, %%zmm8"
-			     :
-			     : "m" (dq[0]), "m" (dq[64]), "m" (q[0]),
-			       "m" (q[64]));
-
-		/*
-		 * 3 = q[0] ^ dq[0]
-		 * 8 = q[64] ^ dq[64]
-		 */
-		asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"
-			     "vmovapd %%zmm0, %%zmm13\n\t"
-			     "vbroadcasti64x2 %1, %%zmm1\n\t"
-			     "vmovapd %%zmm1, %%zmm14"
-			     :
-			     : "m" (qmul[0]), "m" (qmul[16]));
-
-		asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t"
-			     "vpsraw $4, %%zmm8, %%zmm12\n\t"
-			     "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
-			     "vpandq %%zmm7, %%zmm8, %%zmm8\n\t"
-			     "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
-			     "vpandq %%zmm7, %%zmm12, %%zmm12\n\t"
-			     "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t"
-			     "vpshufb %%zmm8, %%zmm13, %%zmm13\n\t"
-			     "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t"
-			     "vpshufb %%zmm12, %%zmm14, %%zmm14\n\t"
-			     "vpxorq %%zmm0, %%zmm1, %%zmm1\n\t"
-			     "vpxorq %%zmm13, %%zmm14, %%zmm14"
-			     :
-			     : );
-
-		/*
-		 * 1  = qmul[q[0]  ^ dq[0]]
-		 * 14 = qmul[q[64] ^ dq[64]]
-		 */
-		asm volatile("vmovdqa64 %0, %%zmm2\n\t"
-			     "vmovdqa64 %1, %%zmm12\n\t"
-			     "vpxorq %%zmm1, %%zmm2, %%zmm2\n\t"
-			     "vpxorq %%zmm14, %%zmm12, %%zmm12"
-			     :
-			     : "m" (p[0]), "m" (p[64]));
-
-		/*
-		 * 2  = p[0]  ^ qmul[q[0]  ^ dq[0]]
-		 * 12 = p[64] ^ qmul[q[64] ^ dq[64]]
-		 */
-
-		asm volatile("vmovdqa64 %%zmm1, %0\n\t"
-			     "vmovdqa64 %%zmm14, %1\n\t"
-			     "vmovdqa64 %%zmm2, %2\n\t"
-			     "vmovdqa64 %%zmm12,%3"
-			     :
-			     : "m" (dq[0]), "m" (dq[64]), "m" (p[0]),
-			       "m" (p[64]));
-
-		bytes -= 128;
-		p += 128;
-		q += 128;
-		dq += 128;
-#else
-		asm volatile("vmovdqa64 %0, %%zmm3\n\t"
-			     "vpxorq %1, %%zmm3, %%zmm3"
-			     :
-			     : "m" (dq[0]), "m" (q[0]));
-
-		/* 3 = q ^ dq */
-
-		asm volatile("vbroadcasti64x2 %0, %%zmm0\n\t"
-			     "vbroadcasti64x2 %1, %%zmm1"
-			     :
-			     : "m" (qmul[0]), "m" (qmul[16]));
-
-		asm volatile("vpsraw $4, %%zmm3, %%zmm6\n\t"
-			     "vpandq %%zmm7, %%zmm3, %%zmm3\n\t"
-			     "vpandq %%zmm7, %%zmm6, %%zmm6\n\t"
-			     "vpshufb %%zmm3, %%zmm0, %%zmm0\n\t"
-			     "vpshufb %%zmm6, %%zmm1, %%zmm1\n\t"
-			     "vpxorq %%zmm0, %%zmm1, %%zmm1"
-			     :
-			     : );
-
-		/* 1 = qmul[q ^ dq] */
-
-		asm volatile("vmovdqa64 %0, %%zmm2\n\t"
-			     "vpxorq %%zmm1, %%zmm2, %%zmm2"
-			     :
-			     : "m" (p[0]));
-
-		/* 2 = p ^ qmul[q ^ dq] */
-
-		asm volatile("vmovdqa64 %%zmm1, %0\n\t"
-			     "vmovdqa64 %%zmm2, %1"
-			     :
-			     : "m" (dq[0]), "m" (p[0]));
-
-		bytes -= 64;
-		p += 64;
-		q += 64;
-		dq += 64;
-#endif
-	}
-
-	kernel_fpu_end();
-}
-
-const struct raid6_recov_calls raid6_recov_avx512 = {
-	.data2 = raid6_2data_recov_avx512,
-	.datap = raid6_datap_recov_avx512,
-	.valid = raid6_has_avx512,
-#ifdef CONFIG_X86_64
-	.name = "avx512x2",
-#else
-	.name = "avx512x1",
-#endif
-	.priority = 3,
-};
diff --git a/lib/raid6/recov_loongarch_simd.c b/lib/raid6/recov_loongarch_simd.c
deleted file mode 100644
index 93dc515997a14..0000000000000
--- a/lib/raid6/recov_loongarch_simd.c
+++ /dev/null
@@ -1,513 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * RAID6 recovery algorithms in LoongArch SIMD (LSX & LASX)
- *
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- *
- * Originally based on recov_avx2.c and recov_ssse3.c:
- *
- * Copyright (C) 2012 Intel Corporation
- * Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
- */
-
-#include <linux/raid/pq.h>
-#include "loongarch.h"
-
-/*
- * Unlike with the syndrome calculation algorithms, there's no boot-time
- * selection of recovery algorithms by benchmarking, so we have to specify
- * the priorities and hope the future cores will all have decent vector
- * support (i.e. no LASX slower than LSX, or even scalar code).
- */
-
-#ifdef CONFIG_CPU_HAS_LSX
-static int raid6_has_lsx(void)
-{
-	return cpu_has_lsx;
-}
-
-static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila,
-				  int failb, void **ptrs)
-{
-	u8 *p, *q, *dp, *dq;
-	const u8 *pbmul;	/* P multiplier table for B data */
-	const u8 *qmul;		/* Q multiplier table (for both) */
-
-	p = (u8 *)ptrs[disks - 2];
-	q = (u8 *)ptrs[disks - 1];
-
-	/*
-	 * Compute syndrome with zero for the missing data pages
-	 * Use the dead data pages as temporary storage for
-	 * delta p and delta q
-	 */
-	dp = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks - 2] = dp;
-	dq = (u8 *)ptrs[failb];
-	ptrs[failb] = raid6_get_zero_page();
-	ptrs[disks - 1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila] = dp;
-	ptrs[failb] = dq;
-	ptrs[disks - 2] = p;
-	ptrs[disks - 1] = q;
-
-	/* Now, pick the proper data tables */
-	pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]];
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]];
-
-	kernel_fpu_begin();
-
-	/*
-	 * vr20, vr21: qmul
-	 * vr22, vr23: pbmul
-	 */
-	asm volatile("vld $vr20, %0" : : "m" (qmul[0]));
-	asm volatile("vld $vr21, %0" : : "m" (qmul[16]));
-	asm volatile("vld $vr22, %0" : : "m" (pbmul[0]));
-	asm volatile("vld $vr23, %0" : : "m" (pbmul[16]));
-
-	while (bytes) {
-		/* vr4 - vr7: Q */
-		asm volatile("vld $vr4, %0" : : "m" (q[0]));
-		asm volatile("vld $vr5, %0" : : "m" (q[16]));
-		asm volatile("vld $vr6, %0" : : "m" (q[32]));
-		asm volatile("vld $vr7, %0" : : "m" (q[48]));
-		/*  vr4 - vr7: Q + Qxy */
-		asm volatile("vld $vr8, %0" : : "m" (dq[0]));
-		asm volatile("vld $vr9, %0" : : "m" (dq[16]));
-		asm volatile("vld $vr10, %0" : : "m" (dq[32]));
-		asm volatile("vld $vr11, %0" : : "m" (dq[48]));
-		asm volatile("vxor.v $vr4, $vr4, $vr8");
-		asm volatile("vxor.v $vr5, $vr5, $vr9");
-		asm volatile("vxor.v $vr6, $vr6, $vr10");
-		asm volatile("vxor.v $vr7, $vr7, $vr11");
-		/* vr0 - vr3: P */
-		asm volatile("vld $vr0, %0" : : "m" (p[0]));
-		asm volatile("vld $vr1, %0" : : "m" (p[16]));
-		asm volatile("vld $vr2, %0" : : "m" (p[32]));
-		asm volatile("vld $vr3, %0" : : "m" (p[48]));
-		/* vr0 - vr3: P + Pxy */
-		asm volatile("vld $vr8, %0" : : "m" (dp[0]));
-		asm volatile("vld $vr9, %0" : : "m" (dp[16]));
-		asm volatile("vld $vr10, %0" : : "m" (dp[32]));
-		asm volatile("vld $vr11, %0" : : "m" (dp[48]));
-		asm volatile("vxor.v $vr0, $vr0, $vr8");
-		asm volatile("vxor.v $vr1, $vr1, $vr9");
-		asm volatile("vxor.v $vr2, $vr2, $vr10");
-		asm volatile("vxor.v $vr3, $vr3, $vr11");
-
-		/* vr8 - vr11: higher 4 bits of each byte of (Q + Qxy) */
-		asm volatile("vsrli.b $vr8, $vr4, 4");
-		asm volatile("vsrli.b $vr9, $vr5, 4");
-		asm volatile("vsrli.b $vr10, $vr6, 4");
-		asm volatile("vsrli.b $vr11, $vr7, 4");
-		/* vr4 - vr7: lower 4 bits of each byte of (Q + Qxy) */
-		asm volatile("vandi.b $vr4, $vr4, 0x0f");
-		asm volatile("vandi.b $vr5, $vr5, 0x0f");
-		asm volatile("vandi.b $vr6, $vr6, 0x0f");
-		asm volatile("vandi.b $vr7, $vr7, 0x0f");
-		/* lookup from qmul[0] */
-		asm volatile("vshuf.b $vr4, $vr20, $vr20, $vr4");
-		asm volatile("vshuf.b $vr5, $vr20, $vr20, $vr5");
-		asm volatile("vshuf.b $vr6, $vr20, $vr20, $vr6");
-		asm volatile("vshuf.b $vr7, $vr20, $vr20, $vr7");
-		/* lookup from qmul[16] */
-		asm volatile("vshuf.b $vr8, $vr21, $vr21, $vr8");
-		asm volatile("vshuf.b $vr9, $vr21, $vr21, $vr9");
-		asm volatile("vshuf.b $vr10, $vr21, $vr21, $vr10");
-		asm volatile("vshuf.b $vr11, $vr21, $vr21, $vr11");
-		/* vr16 - vr19: B(Q + Qxy) */
-		asm volatile("vxor.v $vr16, $vr8, $vr4");
-		asm volatile("vxor.v $vr17, $vr9, $vr5");
-		asm volatile("vxor.v $vr18, $vr10, $vr6");
-		asm volatile("vxor.v $vr19, $vr11, $vr7");
-
-		/* vr4 - vr7: higher 4 bits of each byte of (P + Pxy) */
-		asm volatile("vsrli.b $vr4, $vr0, 4");
-		asm volatile("vsrli.b $vr5, $vr1, 4");
-		asm volatile("vsrli.b $vr6, $vr2, 4");
-		asm volatile("vsrli.b $vr7, $vr3, 4");
-		/* vr12 - vr15: lower 4 bits of each byte of (P + Pxy) */
-		asm volatile("vandi.b $vr12, $vr0, 0x0f");
-		asm volatile("vandi.b $vr13, $vr1, 0x0f");
-		asm volatile("vandi.b $vr14, $vr2, 0x0f");
-		asm volatile("vandi.b $vr15, $vr3, 0x0f");
-		/* lookup from pbmul[0] */
-		asm volatile("vshuf.b $vr12, $vr22, $vr22, $vr12");
-		asm volatile("vshuf.b $vr13, $vr22, $vr22, $vr13");
-		asm volatile("vshuf.b $vr14, $vr22, $vr22, $vr14");
-		asm volatile("vshuf.b $vr15, $vr22, $vr22, $vr15");
-		/* lookup from pbmul[16] */
-		asm volatile("vshuf.b $vr4, $vr23, $vr23, $vr4");
-		asm volatile("vshuf.b $vr5, $vr23, $vr23, $vr5");
-		asm volatile("vshuf.b $vr6, $vr23, $vr23, $vr6");
-		asm volatile("vshuf.b $vr7, $vr23, $vr23, $vr7");
-		/* vr4 - vr7: A(P + Pxy) */
-		asm volatile("vxor.v $vr4, $vr4, $vr12");
-		asm volatile("vxor.v $vr5, $vr5, $vr13");
-		asm volatile("vxor.v $vr6, $vr6, $vr14");
-		asm volatile("vxor.v $vr7, $vr7, $vr15");
-
-		/* vr4 - vr7: A(P + Pxy) + B(Q + Qxy) = Dx */
-		asm volatile("vxor.v $vr4, $vr4, $vr16");
-		asm volatile("vxor.v $vr5, $vr5, $vr17");
-		asm volatile("vxor.v $vr6, $vr6, $vr18");
-		asm volatile("vxor.v $vr7, $vr7, $vr19");
-		asm volatile("vst $vr4, %0" : "=m" (dq[0]));
-		asm volatile("vst $vr5, %0" : "=m" (dq[16]));
-		asm volatile("vst $vr6, %0" : "=m" (dq[32]));
-		asm volatile("vst $vr7, %0" : "=m" (dq[48]));
-
-		/* vr0 - vr3: P + Pxy + Dx = Dy */
-		asm volatile("vxor.v $vr0, $vr0, $vr4");
-		asm volatile("vxor.v $vr1, $vr1, $vr5");
-		asm volatile("vxor.v $vr2, $vr2, $vr6");
-		asm volatile("vxor.v $vr3, $vr3, $vr7");
-		asm volatile("vst $vr0, %0" : "=m" (dp[0]));
-		asm volatile("vst $vr1, %0" : "=m" (dp[16]));
-		asm volatile("vst $vr2, %0" : "=m" (dp[32]));
-		asm volatile("vst $vr3, %0" : "=m" (dp[48]));
-
-		bytes -= 64;
-		p += 64;
-		q += 64;
-		dp += 64;
-		dq += 64;
-	}
-
-	kernel_fpu_end();
-}
-
-static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila,
-				  void **ptrs)
-{
-	u8 *p, *q, *dq;
-	const u8 *qmul;		/* Q multiplier table */
-
-	p = (u8 *)ptrs[disks - 2];
-	q = (u8 *)ptrs[disks - 1];
-
-	/*
-	 * Compute syndrome with zero for the missing data page
-	 * Use the dead data page as temporary storage for delta q
-	 */
-	dq = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks - 1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila] = dq;
-	ptrs[disks - 1] = q;
-
-	/* Now, pick the proper data tables */
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
-
-	kernel_fpu_begin();
-
-	/* vr22, vr23: qmul */
-	asm volatile("vld $vr22, %0" : : "m" (qmul[0]));
-	asm volatile("vld $vr23, %0" : : "m" (qmul[16]));
-
-	while (bytes) {
-		/* vr0 - vr3: P + Dx */
-		asm volatile("vld $vr0, %0" : : "m" (p[0]));
-		asm volatile("vld $vr1, %0" : : "m" (p[16]));
-		asm volatile("vld $vr2, %0" : : "m" (p[32]));
-		asm volatile("vld $vr3, %0" : : "m" (p[48]));
-		/* vr4 - vr7: Qx */
-		asm volatile("vld $vr4, %0" : : "m" (dq[0]));
-		asm volatile("vld $vr5, %0" : : "m" (dq[16]));
-		asm volatile("vld $vr6, %0" : : "m" (dq[32]));
-		asm volatile("vld $vr7, %0" : : "m" (dq[48]));
-		/* vr4 - vr7: Q + Qx */
-		asm volatile("vld $vr8, %0" : : "m" (q[0]));
-		asm volatile("vld $vr9, %0" : : "m" (q[16]));
-		asm volatile("vld $vr10, %0" : : "m" (q[32]));
-		asm volatile("vld $vr11, %0" : : "m" (q[48]));
-		asm volatile("vxor.v $vr4, $vr4, $vr8");
-		asm volatile("vxor.v $vr5, $vr5, $vr9");
-		asm volatile("vxor.v $vr6, $vr6, $vr10");
-		asm volatile("vxor.v $vr7, $vr7, $vr11");
-
-		/* vr8 - vr11: higher 4 bits of each byte of (Q + Qx) */
-		asm volatile("vsrli.b $vr8, $vr4, 4");
-		asm volatile("vsrli.b $vr9, $vr5, 4");
-		asm volatile("vsrli.b $vr10, $vr6, 4");
-		asm volatile("vsrli.b $vr11, $vr7, 4");
-		/* vr4 - vr7: lower 4 bits of each byte of (Q + Qx) */
-		asm volatile("vandi.b $vr4, $vr4, 0x0f");
-		asm volatile("vandi.b $vr5, $vr5, 0x0f");
-		asm volatile("vandi.b $vr6, $vr6, 0x0f");
-		asm volatile("vandi.b $vr7, $vr7, 0x0f");
-		/* lookup from qmul[0] */
-		asm volatile("vshuf.b $vr4, $vr22, $vr22, $vr4");
-		asm volatile("vshuf.b $vr5, $vr22, $vr22, $vr5");
-		asm volatile("vshuf.b $vr6, $vr22, $vr22, $vr6");
-		asm volatile("vshuf.b $vr7, $vr22, $vr22, $vr7");
-		/* lookup from qmul[16] */
-		asm volatile("vshuf.b $vr8, $vr23, $vr23, $vr8");
-		asm volatile("vshuf.b $vr9, $vr23, $vr23, $vr9");
-		asm volatile("vshuf.b $vr10, $vr23, $vr23, $vr10");
-		asm volatile("vshuf.b $vr11, $vr23, $vr23, $vr11");
-		/* vr4 - vr7: qmul(Q + Qx) = Dx */
-		asm volatile("vxor.v $vr4, $vr4, $vr8");
-		asm volatile("vxor.v $vr5, $vr5, $vr9");
-		asm volatile("vxor.v $vr6, $vr6, $vr10");
-		asm volatile("vxor.v $vr7, $vr7, $vr11");
-		asm volatile("vst $vr4, %0" : "=m" (dq[0]));
-		asm volatile("vst $vr5, %0" : "=m" (dq[16]));
-		asm volatile("vst $vr6, %0" : "=m" (dq[32]));
-		asm volatile("vst $vr7, %0" : "=m" (dq[48]));
-
-		/* vr0 - vr3: P + Dx + Dx = P */
-		asm volatile("vxor.v $vr0, $vr0, $vr4");
-		asm volatile("vxor.v $vr1, $vr1, $vr5");
-		asm volatile("vxor.v $vr2, $vr2, $vr6");
-		asm volatile("vxor.v $vr3, $vr3, $vr7");
-		asm volatile("vst $vr0, %0" : "=m" (p[0]));
-		asm volatile("vst $vr1, %0" : "=m" (p[16]));
-		asm volatile("vst $vr2, %0" : "=m" (p[32]));
-		asm volatile("vst $vr3, %0" : "=m" (p[48]));
-
-		bytes -= 64;
-		p += 64;
-		q += 64;
-		dq += 64;
-	}
-
-	kernel_fpu_end();
-}
-
-const struct raid6_recov_calls raid6_recov_lsx = {
-	.data2 = raid6_2data_recov_lsx,
-	.datap = raid6_datap_recov_lsx,
-	.valid = raid6_has_lsx,
-	.name = "lsx",
-	.priority = 1,
-};
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-static int raid6_has_lasx(void)
-{
-	return cpu_has_lasx;
-}
-
-static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila,
-				   int failb, void **ptrs)
-{
-	u8 *p, *q, *dp, *dq;
-	const u8 *pbmul;	/* P multiplier table for B data */
-	const u8 *qmul;		/* Q multiplier table (for both) */
-
-	p = (u8 *)ptrs[disks - 2];
-	q = (u8 *)ptrs[disks - 1];
-
-	/*
-	 * Compute syndrome with zero for the missing data pages
-	 * Use the dead data pages as temporary storage for
-	 * delta p and delta q
-	 */
-	dp = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks - 2] = dp;
-	dq = (u8 *)ptrs[failb];
-	ptrs[failb] = raid6_get_zero_page();
-	ptrs[disks - 1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila] = dp;
-	ptrs[failb] = dq;
-	ptrs[disks - 2] = p;
-	ptrs[disks - 1] = q;
-
-	/* Now, pick the proper data tables */
-	pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]];
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ raid6_gfexp[failb]]];
-
-	kernel_fpu_begin();
-
-	/*
-	 * xr20, xr21: qmul
-	 * xr22, xr23: pbmul
-	 */
-	asm volatile("vld $vr20, %0" : : "m" (qmul[0]));
-	asm volatile("vld $vr21, %0" : : "m" (qmul[16]));
-	asm volatile("vld $vr22, %0" : : "m" (pbmul[0]));
-	asm volatile("vld $vr23, %0" : : "m" (pbmul[16]));
-	asm volatile("xvreplve0.q $xr20, $xr20");
-	asm volatile("xvreplve0.q $xr21, $xr21");
-	asm volatile("xvreplve0.q $xr22, $xr22");
-	asm volatile("xvreplve0.q $xr23, $xr23");
-
-	while (bytes) {
-		/* xr0, xr1: Q */
-		asm volatile("xvld $xr0, %0" : : "m" (q[0]));
-		asm volatile("xvld $xr1, %0" : : "m" (q[32]));
-		/* xr0, xr1: Q + Qxy */
-		asm volatile("xvld $xr4, %0" : : "m" (dq[0]));
-		asm volatile("xvld $xr5, %0" : : "m" (dq[32]));
-		asm volatile("xvxor.v $xr0, $xr0, $xr4");
-		asm volatile("xvxor.v $xr1, $xr1, $xr5");
-		/* xr2, xr3: P */
-		asm volatile("xvld $xr2, %0" : : "m" (p[0]));
-		asm volatile("xvld $xr3, %0" : : "m" (p[32]));
-		/* xr2, xr3: P + Pxy */
-		asm volatile("xvld $xr4, %0" : : "m" (dp[0]));
-		asm volatile("xvld $xr5, %0" : : "m" (dp[32]));
-		asm volatile("xvxor.v $xr2, $xr2, $xr4");
-		asm volatile("xvxor.v $xr3, $xr3, $xr5");
-
-		/* xr4, xr5: higher 4 bits of each byte of (Q + Qxy) */
-		asm volatile("xvsrli.b $xr4, $xr0, 4");
-		asm volatile("xvsrli.b $xr5, $xr1, 4");
-		/* xr0, xr1: lower 4 bits of each byte of (Q + Qxy) */
-		asm volatile("xvandi.b $xr0, $xr0, 0x0f");
-		asm volatile("xvandi.b $xr1, $xr1, 0x0f");
-		/* lookup from qmul[0] */
-		asm volatile("xvshuf.b $xr0, $xr20, $xr20, $xr0");
-		asm volatile("xvshuf.b $xr1, $xr20, $xr20, $xr1");
-		/* lookup from qmul[16] */
-		asm volatile("xvshuf.b $xr4, $xr21, $xr21, $xr4");
-		asm volatile("xvshuf.b $xr5, $xr21, $xr21, $xr5");
-		/* xr6, xr7: B(Q + Qxy) */
-		asm volatile("xvxor.v $xr6, $xr4, $xr0");
-		asm volatile("xvxor.v $xr7, $xr5, $xr1");
-
-		/* xr4, xr5: higher 4 bits of each byte of (P + Pxy) */
-		asm volatile("xvsrli.b $xr4, $xr2, 4");
-		asm volatile("xvsrli.b $xr5, $xr3, 4");
-		/* xr0, xr1: lower 4 bits of each byte of (P + Pxy) */
-		asm volatile("xvandi.b $xr0, $xr2, 0x0f");
-		asm volatile("xvandi.b $xr1, $xr3, 0x0f");
-		/* lookup from pbmul[0] */
-		asm volatile("xvshuf.b $xr0, $xr22, $xr22, $xr0");
-		asm volatile("xvshuf.b $xr1, $xr22, $xr22, $xr1");
-		/* lookup from pbmul[16] */
-		asm volatile("xvshuf.b $xr4, $xr23, $xr23, $xr4");
-		asm volatile("xvshuf.b $xr5, $xr23, $xr23, $xr5");
-		/* xr0, xr1: A(P + Pxy) */
-		asm volatile("xvxor.v $xr0, $xr0, $xr4");
-		asm volatile("xvxor.v $xr1, $xr1, $xr5");
-
-		/* xr0, xr1: A(P + Pxy) + B(Q + Qxy) = Dx */
-		asm volatile("xvxor.v $xr0, $xr0, $xr6");
-		asm volatile("xvxor.v $xr1, $xr1, $xr7");
-
-		/* xr2, xr3: P + Pxy + Dx = Dy */
-		asm volatile("xvxor.v $xr2, $xr2, $xr0");
-		asm volatile("xvxor.v $xr3, $xr3, $xr1");
-
-		asm volatile("xvst $xr0, %0" : "=m" (dq[0]));
-		asm volatile("xvst $xr1, %0" : "=m" (dq[32]));
-		asm volatile("xvst $xr2, %0" : "=m" (dp[0]));
-		asm volatile("xvst $xr3, %0" : "=m" (dp[32]));
-
-		bytes -= 64;
-		p += 64;
-		q += 64;
-		dp += 64;
-		dq += 64;
-	}
-
-	kernel_fpu_end();
-}
-
-static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila,
-				   void **ptrs)
-{
-	u8 *p, *q, *dq;
-	const u8 *qmul;		/* Q multiplier table */
-
-	p = (u8 *)ptrs[disks - 2];
-	q = (u8 *)ptrs[disks - 1];
-
-	/*
-	 * Compute syndrome with zero for the missing data page
-	 * Use the dead data page as temporary storage for delta q
-	 */
-	dq = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks - 1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila] = dq;
-	ptrs[disks - 1] = q;
-
-	/* Now, pick the proper data tables */
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
-
-	kernel_fpu_begin();
-
-	/* xr22, xr23: qmul */
-	asm volatile("vld $vr22, %0" : : "m" (qmul[0]));
-	asm volatile("xvreplve0.q $xr22, $xr22");
-	asm volatile("vld $vr23, %0" : : "m" (qmul[16]));
-	asm volatile("xvreplve0.q $xr23, $xr23");
-
-	while (bytes) {
-		/* xr0, xr1: P + Dx */
-		asm volatile("xvld $xr0, %0" : : "m" (p[0]));
-		asm volatile("xvld $xr1, %0" : : "m" (p[32]));
-		/* xr2, xr3: Qx */
-		asm volatile("xvld $xr2, %0" : : "m" (dq[0]));
-		asm volatile("xvld $xr3, %0" : : "m" (dq[32]));
-		/* xr2, xr3: Q + Qx */
-		asm volatile("xvld $xr4, %0" : : "m" (q[0]));
-		asm volatile("xvld $xr5, %0" : : "m" (q[32]));
-		asm volatile("xvxor.v $xr2, $xr2, $xr4");
-		asm volatile("xvxor.v $xr3, $xr3, $xr5");
-
-		/* xr4, xr5: higher 4 bits of each byte of (Q + Qx) */
-		asm volatile("xvsrli.b $xr4, $xr2, 4");
-		asm volatile("xvsrli.b $xr5, $xr3, 4");
-		/* xr2, xr3: lower 4 bits of each byte of (Q + Qx) */
-		asm volatile("xvandi.b $xr2, $xr2, 0x0f");
-		asm volatile("xvandi.b $xr3, $xr3, 0x0f");
-		/* lookup from qmul[0] */
-		asm volatile("xvshuf.b $xr2, $xr22, $xr22, $xr2");
-		asm volatile("xvshuf.b $xr3, $xr22, $xr22, $xr3");
-		/* lookup from qmul[16] */
-		asm volatile("xvshuf.b $xr4, $xr23, $xr23, $xr4");
-		asm volatile("xvshuf.b $xr5, $xr23, $xr23, $xr5");
-		/* xr2, xr3: qmul(Q + Qx) = Dx */
-		asm volatile("xvxor.v $xr2, $xr2, $xr4");
-		asm volatile("xvxor.v $xr3, $xr3, $xr5");
-
-		/* xr0, xr1: P + Dx + Dx = P */
-		asm volatile("xvxor.v $xr0, $xr0, $xr2");
-		asm volatile("xvxor.v $xr1, $xr1, $xr3");
-
-		asm volatile("xvst $xr2, %0" : "=m" (dq[0]));
-		asm volatile("xvst $xr3, %0" : "=m" (dq[32]));
-		asm volatile("xvst $xr0, %0" : "=m" (p[0]));
-		asm volatile("xvst $xr1, %0" : "=m" (p[32]));
-
-		bytes -= 64;
-		p += 64;
-		q += 64;
-		dq += 64;
-	}
-
-	kernel_fpu_end();
-}
-
-const struct raid6_recov_calls raid6_recov_lasx = {
-	.data2 = raid6_2data_recov_lasx,
-	.datap = raid6_datap_recov_lasx,
-	.valid = raid6_has_lasx,
-	.name = "lasx",
-	.priority = 2,
-};
-#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/lib/raid6/recov_neon.c b/lib/raid6/recov_neon.c
deleted file mode 100644
index 9d99aeabd31a9..0000000000000
--- a/lib/raid6/recov_neon.c
+++ /dev/null
@@ -1,97 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2012 Intel Corporation
- * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
- */
-
-#include <linux/raid/pq.h>
-
-#ifdef __KERNEL__
-#include <asm/simd.h>
-#include "neon.h"
-#else
-#define scoped_ksimd()
-#define cpu_has_neon()		(1)
-#endif
-
-static int raid6_has_neon(void)
-{
-	return cpu_has_neon();
-}
-
-static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
-		int failb, void **ptrs)
-{
-	u8 *p, *q, *dp, *dq;
-	const u8 *pbmul;	/* P multiplier table for B data */
-	const u8 *qmul;		/* Q multiplier table (for both) */
-
-	p = (u8 *)ptrs[disks - 2];
-	q = (u8 *)ptrs[disks - 1];
-
-	/*
-	 * Compute syndrome with zero for the missing data pages
-	 * Use the dead data pages as temporary storage for
-	 * delta p and delta q
-	 */
-	dp = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks - 2] = dp;
-	dq = (u8 *)ptrs[failb];
-	ptrs[failb] = raid6_get_zero_page();
-	ptrs[disks - 1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]     = dp;
-	ptrs[failb]     = dq;
-	ptrs[disks - 2] = p;
-	ptrs[disks - 1] = q;
-
-	/* Now, pick the proper data tables */
-	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
-					 raid6_gfexp[failb]]];
-
-	scoped_ksimd()
-		__raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul);
-}
-
-static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
-		void **ptrs)
-{
-	u8 *p, *q, *dq;
-	const u8 *qmul;		/* Q multiplier table */
-
-	p = (u8 *)ptrs[disks - 2];
-	q = (u8 *)ptrs[disks - 1];
-
-	/*
-	 * Compute syndrome with zero for the missing data page
-	 * Use the dead data page as temporary storage for delta q
-	 */
-	dq = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks - 1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]     = dq;
-	ptrs[disks - 1] = q;
-
-	/* Now, pick the proper data tables */
-	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
-
-	scoped_ksimd()
-		__raid6_datap_recov_neon(bytes, p, q, dq, qmul);
-}
-
-const struct raid6_recov_calls raid6_recov_neon = {
-	.data2		= raid6_2data_recov_neon,
-	.datap		= raid6_datap_recov_neon,
-	.valid		= raid6_has_neon,
-	.name		= "neon",
-	.priority	= 10,
-};
diff --git a/lib/raid6/recov_neon_inner.c b/lib/raid6/recov_neon_inner.c
deleted file mode 100644
index f9e7e8f5a1510..0000000000000
--- a/lib/raid6/recov_neon_inner.c
+++ /dev/null
@@ -1,111 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2012 Intel Corporation
- * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
- */
-
-#include <arm_neon.h>
-#include "neon.h"
-
-#ifdef CONFIG_ARM
-/*
- * AArch32 does not provide this intrinsic natively because it does not
- * implement the underlying instruction. AArch32 only provides a 64-bit
- * wide vtbl.8 instruction, so use that instead.
- */
-static uint8x16_t vqtbl1q_u8(uint8x16_t a, uint8x16_t b)
-{
-	union {
-		uint8x16_t	val;
-		uint8x8x2_t	pair;
-	} __a = { a };
-
-	return vcombine_u8(vtbl2_u8(__a.pair, vget_low_u8(b)),
-			   vtbl2_u8(__a.pair, vget_high_u8(b)));
-}
-#endif
-
-void __raid6_2data_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dp,
-			      uint8_t *dq, const uint8_t *pbmul,
-			      const uint8_t *qmul)
-{
-	uint8x16_t pm0 = vld1q_u8(pbmul);
-	uint8x16_t pm1 = vld1q_u8(pbmul + 16);
-	uint8x16_t qm0 = vld1q_u8(qmul);
-	uint8x16_t qm1 = vld1q_u8(qmul + 16);
-	uint8x16_t x0f = vdupq_n_u8(0x0f);
-
-	/*
-	 * while ( bytes-- ) {
-	 *	uint8_t px, qx, db;
-	 *
-	 *	px    = *p ^ *dp;
-	 *	qx    = qmul[*q ^ *dq];
-	 *	*dq++ = db = pbmul[px] ^ qx;
-	 *	*dp++ = db ^ px;
-	 *	p++; q++;
-	 * }
-	 */
-
-	while (bytes) {
-		uint8x16_t vx, vy, px, qx, db;
-
-		px = veorq_u8(vld1q_u8(p), vld1q_u8(dp));
-		vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
-
-		vy = vshrq_n_u8(vx, 4);
-		vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
-		vy = vqtbl1q_u8(qm1, vy);
-		qx = veorq_u8(vx, vy);
-
-		vy = vshrq_n_u8(px, 4);
-		vx = vqtbl1q_u8(pm0, vandq_u8(px, x0f));
-		vy = vqtbl1q_u8(pm1, vy);
-		vx = veorq_u8(vx, vy);
-		db = veorq_u8(vx, qx);
-
-		vst1q_u8(dq, db);
-		vst1q_u8(dp, veorq_u8(db, px));
-
-		bytes -= 16;
-		p += 16;
-		q += 16;
-		dp += 16;
-		dq += 16;
-	}
-}
-
-void __raid6_datap_recov_neon(int bytes, uint8_t *p, uint8_t *q, uint8_t *dq,
-			      const uint8_t *qmul)
-{
-	uint8x16_t qm0 = vld1q_u8(qmul);
-	uint8x16_t qm1 = vld1q_u8(qmul + 16);
-	uint8x16_t x0f = vdupq_n_u8(0x0f);
-
-	/*
-	 * while (bytes--) {
-	 *	*p++ ^= *dq = qmul[*q ^ *dq];
-	 *	q++; dq++;
-	 * }
-	 */
-
-	while (bytes) {
-		uint8x16_t vx, vy;
-
-		vx = veorq_u8(vld1q_u8(q), vld1q_u8(dq));
-
-		vy = vshrq_n_u8(vx, 4);
-		vx = vqtbl1q_u8(qm0, vandq_u8(vx, x0f));
-		vy = vqtbl1q_u8(qm1, vy);
-		vx = veorq_u8(vx, vy);
-		vy = veorq_u8(vx, vld1q_u8(p));
-
-		vst1q_u8(dq, vx);
-		vst1q_u8(p, vy);
-
-		bytes -= 16;
-		p += 16;
-		q += 16;
-		dq += 16;
-	}
-}
diff --git a/lib/raid6/recov_rvv.c b/lib/raid6/recov_rvv.c
deleted file mode 100644
index 40c393206b6a1..0000000000000
--- a/lib/raid6/recov_rvv.c
+++ /dev/null
@@ -1,222 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright 2024 Institute of Software, CAS.
- * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
- */
-
-#include <linux/raid/pq.h>
-#include "rvv.h"
-
-static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp,
-				    u8 *dq, const u8 *pbmul,
-				    const u8 *qmul)
-{
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	x0, %[avl], e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : :
-		      [avl]"r"(16)
-	);
-
-	/*
-	 * while ( bytes-- ) {
-	 *	uint8_t px, qx, db;
-	 *
-	 *	px	  = *p ^ *dp;
-	 *	qx	  = qmul[*q ^ *dq];
-	 *	*dq++ = db = pbmul[px] ^ qx;
-	 *	*dp++ = db ^ px;
-	 *	p++; q++;
-	 * }
-	 */
-	while (bytes) {
-		/*
-		 * v0:px, v1:dp,
-		 * v2:qx, v3:dq,
-		 * v4:vx, v5:vy,
-		 * v6:qm0, v7:qm1,
-		 * v8:pm0, v9:pm1,
-		 * v14:p/qm[vx], v15:p/qm[vy]
-		 */
-		asm volatile (".option		push\n"
-			      ".option		arch,+v\n"
-			      "vle8.v		v0, (%[px])\n"
-			      "vle8.v		v1, (%[dp])\n"
-			      "vxor.vv		v0, v0, v1\n"
-			      "vle8.v		v2, (%[qx])\n"
-			      "vle8.v		v3, (%[dq])\n"
-			      "vxor.vv		v4, v2, v3\n"
-			      "vsrl.vi		v5, v4, 4\n"
-			      "vand.vi		v4, v4, 0xf\n"
-			      "vle8.v		v6, (%[qm0])\n"
-			      "vle8.v		v7, (%[qm1])\n"
-			      "vrgather.vv	v14, v6, v4\n" /* v14 = qm[vx] */
-			      "vrgather.vv	v15, v7, v5\n" /* v15 = qm[vy] */
-			      "vxor.vv		v2, v14, v15\n" /* v2 = qmul[*q ^ *dq] */
-
-			      "vsrl.vi		v5, v0, 4\n"
-			      "vand.vi		v4, v0, 0xf\n"
-			      "vle8.v		v8, (%[pm0])\n"
-			      "vle8.v		v9, (%[pm1])\n"
-			      "vrgather.vv	v14, v8, v4\n" /* v14 = pm[vx] */
-			      "vrgather.vv	v15, v9, v5\n" /* v15 = pm[vy] */
-			      "vxor.vv		v4, v14, v15\n" /* v4 = pbmul[px] */
-			      "vxor.vv		v3, v4, v2\n" /* v3 = db = pbmul[px] ^ qx */
-			      "vxor.vv		v1, v3, v0\n" /* v1 = db ^ px; */
-			      "vse8.v		v3, (%[dq])\n"
-			      "vse8.v		v1, (%[dp])\n"
-			      ".option		pop\n"
-			      : :
-			      [px]"r"(p),
-			      [dp]"r"(dp),
-			      [qx]"r"(q),
-			      [dq]"r"(dq),
-			      [qm0]"r"(qmul),
-			      [qm1]"r"(qmul + 16),
-			      [pm0]"r"(pbmul),
-			      [pm1]"r"(pbmul + 16)
-			      :);
-
-		bytes -= 16;
-		p += 16;
-		q += 16;
-		dp += 16;
-		dq += 16;
-	}
-}
-
-static void __raid6_datap_recov_rvv(int bytes, u8 *p, u8 *q,
-				    u8 *dq, const u8 *qmul)
-{
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	x0, %[avl], e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : :
-		      [avl]"r"(16)
-	);
-
-	/*
-	 * while (bytes--) {
-	 *  *p++ ^= *dq = qmul[*q ^ *dq];
-	 *  q++; dq++;
-	 * }
-	 */
-	while (bytes) {
-		/*
-		 * v0:vx, v1:vy,
-		 * v2:dq, v3:p,
-		 * v4:qm0, v5:qm1,
-		 * v10:m[vx], v11:m[vy]
-		 */
-		asm volatile (".option		push\n"
-			      ".option		arch,+v\n"
-			      "vle8.v		v0, (%[vx])\n"
-			      "vle8.v		v2, (%[dq])\n"
-			      "vxor.vv		v0, v0, v2\n"
-			      "vsrl.vi		v1, v0, 4\n"
-			      "vand.vi		v0, v0, 0xf\n"
-			      "vle8.v		v4, (%[qm0])\n"
-			      "vle8.v		v5, (%[qm1])\n"
-			      "vrgather.vv	v10, v4, v0\n"
-			      "vrgather.vv	v11, v5, v1\n"
-			      "vxor.vv		v0, v10, v11\n"
-			      "vle8.v		v1, (%[vy])\n"
-			      "vxor.vv		v1, v0, v1\n"
-			      "vse8.v		v0, (%[dq])\n"
-			      "vse8.v		v1, (%[vy])\n"
-			      ".option		pop\n"
-			      : :
-			      [vx]"r"(q),
-			      [vy]"r"(p),
-			      [dq]"r"(dq),
-			      [qm0]"r"(qmul),
-			      [qm1]"r"(qmul + 16)
-			      :);
-
-		bytes -= 16;
-		p += 16;
-		q += 16;
-		dq += 16;
-	}
-}
-
-static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila,
-				  int failb, void **ptrs)
-{
-	u8 *p, *q, *dp, *dq;
-	const u8 *pbmul;	/* P multiplier table for B data */
-	const u8 *qmul;		/* Q multiplier table (for both) */
-
-	p = (u8 *)ptrs[disks - 2];
-	q = (u8 *)ptrs[disks - 1];
-
-	/*
-	 * Compute syndrome with zero for the missing data pages
-	 * Use the dead data pages as temporary storage for
-	 * delta p and delta q
-	 */
-	dp = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks - 2] = dp;
-	dq = (u8 *)ptrs[failb];
-	ptrs[failb] = raid6_get_zero_page();
-	ptrs[disks - 1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]     = dp;
-	ptrs[failb]     = dq;
-	ptrs[disks - 2] = p;
-	ptrs[disks - 1] = q;
-
-	/* Now, pick the proper data tables */
-	pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]];
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
-					 raid6_gfexp[failb]]];
-
-	kernel_vector_begin();
-	__raid6_2data_recov_rvv(bytes, p, q, dp, dq, pbmul, qmul);
-	kernel_vector_end();
-}
-
-static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila,
-				  void **ptrs)
-{
-	u8 *p, *q, *dq;
-	const u8 *qmul;		/* Q multiplier table */
-
-	p = (u8 *)ptrs[disks - 2];
-	q = (u8 *)ptrs[disks - 1];
-
-	/*
-	 * Compute syndrome with zero for the missing data page
-	 * Use the dead data page as temporary storage for delta q
-	 */
-	dq = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks - 1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]     = dq;
-	ptrs[disks - 1] = q;
-
-	/* Now, pick the proper data tables */
-	qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
-
-	kernel_vector_begin();
-	__raid6_datap_recov_rvv(bytes, p, q, dq, qmul);
-	kernel_vector_end();
-}
-
-const struct raid6_recov_calls raid6_recov_rvv = {
-	.data2		= raid6_2data_recov_rvv,
-	.datap		= raid6_datap_recov_rvv,
-	.valid		= rvv_has_vector,
-	.name		= "rvv",
-	.priority	= 1,
-};
diff --git a/lib/raid6/recov_s390xc.c b/lib/raid6/recov_s390xc.c
deleted file mode 100644
index 487018f811924..0000000000000
--- a/lib/raid6/recov_s390xc.c
+++ /dev/null
@@ -1,116 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * RAID-6 data recovery in dual failure mode based on the XC instruction.
- *
- * Copyright IBM Corp. 2016
- * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
- */
-
-#include <linux/raid/pq.h>
-
-static inline void xor_block(u8 *p1, u8 *p2)
-{
-	typedef struct { u8 _[256]; } addrtype;
-
-	asm volatile(
-		"	xc	0(256,%[p1]),0(%[p2])\n"
-		: "+m" (*(addrtype *) p1) : "m" (*(addrtype *) p2),
-		  [p1] "a" (p1), [p2] "a" (p2) : "cc");
-}
-
-/* Recover two failed data blocks. */
-static void raid6_2data_recov_s390xc(int disks, size_t bytes, int faila,
-		int failb, void **ptrs)
-{
-	u8 *p, *q, *dp, *dq;
-	const u8 *pbmul;	/* P multiplier table for B data */
-	const u8 *qmul;		/* Q multiplier table (for both) */
-	int i;
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/* Compute syndrome with zero for the missing data pages
-	   Use the dead data pages as temporary storage for
-	   delta p and delta q */
-	dp = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-2] = dp;
-	dq = (u8 *)ptrs[failb];
-	ptrs[failb] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dp;
-	ptrs[failb]   = dq;
-	ptrs[disks-2] = p;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	pbmul = raid6_gfmul[raid6_gfexi[failb-faila]];
-	qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]];
-
-	/* Now do it... */
-	while (bytes) {
-		xor_block(dp, p);
-		xor_block(dq, q);
-		for (i = 0; i < 256; i++)
-			dq[i] = pbmul[dp[i]] ^ qmul[dq[i]];
-		xor_block(dp, dq);
-		p += 256;
-		q += 256;
-		dp += 256;
-		dq += 256;
-		bytes -= 256;
-	}
-}
-
-/* Recover failure of one data block plus the P block */
-static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila,
-		void **ptrs)
-{
-	u8 *p, *q, *dq;
-	const u8 *qmul;		/* Q multiplier table */
-	int i;
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/* Compute syndrome with zero for the missing data page
-	   Use the dead data page as temporary storage for delta q */
-	dq = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dq;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	qmul  = raid6_gfmul[raid6_gfinv[raid6_gfexp[faila]]];
-
-	/* Now do it... */
-	while (bytes) {
-		xor_block(dq, q);
-		for (i = 0; i < 256; i++)
-			dq[i] = qmul[dq[i]];
-		xor_block(p, dq);
-		p += 256;
-		q += 256;
-		dq += 256;
-		bytes -= 256;
-	}
-}
-
-
-const struct raid6_recov_calls raid6_recov_s390xc = {
-	.data2 = raid6_2data_recov_s390xc,
-	.datap = raid6_datap_recov_s390xc,
-	.valid = NULL,
-	.name = "s390xc",
-	.priority = 1,
-};
diff --git a/lib/raid6/recov_ssse3.c b/lib/raid6/recov_ssse3.c
deleted file mode 100644
index 2e849185c32b3..0000000000000
--- a/lib/raid6/recov_ssse3.c
+++ /dev/null
@@ -1,328 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Copyright (C) 2012 Intel Corporation
- */
-
-#include <linux/raid/pq.h>
-#include "x86.h"
-
-static int raid6_has_ssse3(void)
-{
-	return boot_cpu_has(X86_FEATURE_XMM) &&
-		boot_cpu_has(X86_FEATURE_XMM2) &&
-		boot_cpu_has(X86_FEATURE_SSSE3);
-}
-
-static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila,
-		int failb, void **ptrs)
-{
-	u8 *p, *q, *dp, *dq;
-	const u8 *pbmul;	/* P multiplier table for B data */
-	const u8 *qmul;		/* Q multiplier table (for both) */
-	static const u8 __aligned(16) x0f[16] = {
-		 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
-		 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/* Compute syndrome with zero for the missing data pages
-	   Use the dead data pages as temporary storage for
-	   delta p and delta q */
-	dp = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-2] = dp;
-	dq = (u8 *)ptrs[failb];
-	ptrs[failb] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dp;
-	ptrs[failb]   = dq;
-	ptrs[disks-2] = p;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]];
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^
-		raid6_gfexp[failb]]];
-
-	kernel_fpu_begin();
-
-	asm volatile("movdqa %0,%%xmm7" : : "m" (x0f[0]));
-
-#ifdef CONFIG_X86_64
-	asm volatile("movdqa %0,%%xmm6" : : "m" (qmul[0]));
-	asm volatile("movdqa %0,%%xmm14" : : "m" (pbmul[0]));
-	asm volatile("movdqa %0,%%xmm15" : : "m" (pbmul[16]));
-#endif
-
-	/* Now do it... */
-	while (bytes) {
-#ifdef CONFIG_X86_64
-		/* xmm6, xmm14, xmm15 */
-
-		asm volatile("movdqa %0,%%xmm1" : : "m" (q[0]));
-		asm volatile("movdqa %0,%%xmm9" : : "m" (q[16]));
-		asm volatile("movdqa %0,%%xmm0" : : "m" (p[0]));
-		asm volatile("movdqa %0,%%xmm8" : : "m" (p[16]));
-		asm volatile("pxor   %0,%%xmm1" : : "m" (dq[0]));
-		asm volatile("pxor   %0,%%xmm9" : : "m" (dq[16]));
-		asm volatile("pxor   %0,%%xmm0" : : "m" (dp[0]));
-		asm volatile("pxor   %0,%%xmm8" : : "m" (dp[16]));
-
-		/* xmm0/8 = px */
-
-		asm volatile("movdqa %xmm6,%xmm4");
-		asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
-		asm volatile("movdqa %xmm6,%xmm12");
-		asm volatile("movdqa %xmm5,%xmm13");
-		asm volatile("movdqa %xmm1,%xmm3");
-		asm volatile("movdqa %xmm9,%xmm11");
-		asm volatile("movdqa %xmm0,%xmm2"); /* xmm2/10 = px */
-		asm volatile("movdqa %xmm8,%xmm10");
-		asm volatile("psraw  $4,%xmm1");
-		asm volatile("psraw  $4,%xmm9");
-		asm volatile("pand   %xmm7,%xmm3");
-		asm volatile("pand   %xmm7,%xmm11");
-		asm volatile("pand   %xmm7,%xmm1");
-		asm volatile("pand   %xmm7,%xmm9");
-		asm volatile("pshufb %xmm3,%xmm4");
-		asm volatile("pshufb %xmm11,%xmm12");
-		asm volatile("pshufb %xmm1,%xmm5");
-		asm volatile("pshufb %xmm9,%xmm13");
-		asm volatile("pxor   %xmm4,%xmm5");
-		asm volatile("pxor   %xmm12,%xmm13");
-
-		/* xmm5/13 = qx */
-
-		asm volatile("movdqa %xmm14,%xmm4");
-		asm volatile("movdqa %xmm15,%xmm1");
-		asm volatile("movdqa %xmm14,%xmm12");
-		asm volatile("movdqa %xmm15,%xmm9");
-		asm volatile("movdqa %xmm2,%xmm3");
-		asm volatile("movdqa %xmm10,%xmm11");
-		asm volatile("psraw  $4,%xmm2");
-		asm volatile("psraw  $4,%xmm10");
-		asm volatile("pand   %xmm7,%xmm3");
-		asm volatile("pand   %xmm7,%xmm11");
-		asm volatile("pand   %xmm7,%xmm2");
-		asm volatile("pand   %xmm7,%xmm10");
-		asm volatile("pshufb %xmm3,%xmm4");
-		asm volatile("pshufb %xmm11,%xmm12");
-		asm volatile("pshufb %xmm2,%xmm1");
-		asm volatile("pshufb %xmm10,%xmm9");
-		asm volatile("pxor   %xmm4,%xmm1");
-		asm volatile("pxor   %xmm12,%xmm9");
-
-		/* xmm1/9 = pbmul[px] */
-		asm volatile("pxor   %xmm5,%xmm1");
-		asm volatile("pxor   %xmm13,%xmm9");
-		/* xmm1/9 = db = DQ */
-		asm volatile("movdqa %%xmm1,%0" : "=m" (dq[0]));
-		asm volatile("movdqa %%xmm9,%0" : "=m" (dq[16]));
-
-		asm volatile("pxor   %xmm1,%xmm0");
-		asm volatile("pxor   %xmm9,%xmm8");
-		asm volatile("movdqa %%xmm0,%0" : "=m" (dp[0]));
-		asm volatile("movdqa %%xmm8,%0" : "=m" (dp[16]));
-
-		bytes -= 32;
-		p += 32;
-		q += 32;
-		dp += 32;
-		dq += 32;
-#else
-		asm volatile("movdqa %0,%%xmm1" : : "m" (*q));
-		asm volatile("movdqa %0,%%xmm0" : : "m" (*p));
-		asm volatile("pxor   %0,%%xmm1" : : "m" (*dq));
-		asm volatile("pxor   %0,%%xmm0" : : "m" (*dp));
-
-		/* 1 = dq ^ q
-		 * 0 = dp ^ p
-		 */
-		asm volatile("movdqa %0,%%xmm4" : : "m" (qmul[0]));
-		asm volatile("movdqa %0,%%xmm5" : : "m" (qmul[16]));
-
-		asm volatile("movdqa %xmm1,%xmm3");
-		asm volatile("psraw  $4,%xmm1");
-		asm volatile("pand   %xmm7,%xmm3");
-		asm volatile("pand   %xmm7,%xmm1");
-		asm volatile("pshufb %xmm3,%xmm4");
-		asm volatile("pshufb %xmm1,%xmm5");
-		asm volatile("pxor   %xmm4,%xmm5");
-
-		asm volatile("movdqa %xmm0,%xmm2"); /* xmm2 = px */
-
-		/* xmm5 = qx */
-
-		asm volatile("movdqa %0,%%xmm4" : : "m" (pbmul[0]));
-		asm volatile("movdqa %0,%%xmm1" : : "m" (pbmul[16]));
-		asm volatile("movdqa %xmm2,%xmm3");
-		asm volatile("psraw  $4,%xmm2");
-		asm volatile("pand   %xmm7,%xmm3");
-		asm volatile("pand   %xmm7,%xmm2");
-		asm volatile("pshufb %xmm3,%xmm4");
-		asm volatile("pshufb %xmm2,%xmm1");
-		asm volatile("pxor   %xmm4,%xmm1");
-
-		/* xmm1 = pbmul[px] */
-		asm volatile("pxor   %xmm5,%xmm1");
-		/* xmm1 = db = DQ */
-		asm volatile("movdqa %%xmm1,%0" : "=m" (*dq));
-
-		asm volatile("pxor   %xmm1,%xmm0");
-		asm volatile("movdqa %%xmm0,%0" : "=m" (*dp));
-
-		bytes -= 16;
-		p += 16;
-		q += 16;
-		dp += 16;
-		dq += 16;
-#endif
-	}
-
-	kernel_fpu_end();
-}
-
-
-static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila,
-		void **ptrs)
-{
-	u8 *p, *q, *dq;
-	const u8 *qmul;		/* Q multiplier table */
-	static const u8 __aligned(16) x0f[16] = {
-		 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
-		 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f};
-
-	p = (u8 *)ptrs[disks-2];
-	q = (u8 *)ptrs[disks-1];
-
-	/* Compute syndrome with zero for the missing data page
-	   Use the dead data page as temporary storage for delta q */
-	dq = (u8 *)ptrs[faila];
-	ptrs[faila] = raid6_get_zero_page();
-	ptrs[disks-1] = dq;
-
-	raid6_call.gen_syndrome(disks, bytes, ptrs);
-
-	/* Restore pointer table */
-	ptrs[faila]   = dq;
-	ptrs[disks-1] = q;
-
-	/* Now, pick the proper data tables */
-	qmul  = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]];
-
-	kernel_fpu_begin();
-
-	asm volatile("movdqa %0, %%xmm7" : : "m" (x0f[0]));
-
-	while (bytes) {
-#ifdef CONFIG_X86_64
-		asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
-		asm volatile("movdqa %0, %%xmm4" : : "m" (dq[16]));
-		asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
-		asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
-
-		/* xmm3 = q[0] ^ dq[0] */
-
-		asm volatile("pxor %0, %%xmm4" : : "m" (q[16]));
-		asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
-
-		/* xmm4 = q[16] ^ dq[16] */
-
-		asm volatile("movdqa %xmm3, %xmm6");
-		asm volatile("movdqa %xmm4, %xmm8");
-
-		/* xmm4 = xmm8 = q[16] ^ dq[16] */
-
-		asm volatile("psraw $4, %xmm3");
-		asm volatile("pand %xmm7, %xmm6");
-		asm volatile("pand %xmm7, %xmm3");
-		asm volatile("pshufb %xmm6, %xmm0");
-		asm volatile("pshufb %xmm3, %xmm1");
-		asm volatile("movdqa %0, %%xmm10" : : "m" (qmul[0]));
-		asm volatile("pxor %xmm0, %xmm1");
-		asm volatile("movdqa %0, %%xmm11" : : "m" (qmul[16]));
-
-		/* xmm1 = qmul[q[0] ^ dq[0]] */
-
-		asm volatile("psraw $4, %xmm4");
-		asm volatile("pand %xmm7, %xmm8");
-		asm volatile("pand %xmm7, %xmm4");
-		asm volatile("pshufb %xmm8, %xmm10");
-		asm volatile("pshufb %xmm4, %xmm11");
-		asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
-		asm volatile("pxor %xmm10, %xmm11");
-		asm volatile("movdqa %0, %%xmm12" : : "m" (p[16]));
-
-		/* xmm11 = qmul[q[16] ^ dq[16]] */
-
-		asm volatile("pxor %xmm1, %xmm2");
-
-		/* xmm2 = p[0] ^ qmul[q[0] ^ dq[0]] */
-
-		asm volatile("pxor %xmm11, %xmm12");
-
-		/* xmm12 = p[16] ^ qmul[q[16] ^ dq[16]] */
-
-		asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
-		asm volatile("movdqa %%xmm11, %0" : "=m" (dq[16]));
-
-		asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
-		asm volatile("movdqa %%xmm12, %0" : "=m" (p[16]));
-
-		bytes -= 32;
-		p += 32;
-		q += 32;
-		dq += 32;
-
-#else
-		asm volatile("movdqa %0, %%xmm3" : : "m" (dq[0]));
-		asm volatile("movdqa %0, %%xmm0" : : "m" (qmul[0]));
-		asm volatile("pxor %0, %%xmm3" : : "m" (q[0]));
-		asm volatile("movdqa %0, %%xmm1" : : "m" (qmul[16]));
-
-		/* xmm3 = *q ^ *dq */
-
-		asm volatile("movdqa %xmm3, %xmm6");
-		asm volatile("movdqa %0, %%xmm2" : : "m" (p[0]));
-		asm volatile("psraw $4, %xmm3");
-		asm volatile("pand %xmm7, %xmm6");
-		asm volatile("pand %xmm7, %xmm3");
-		asm volatile("pshufb %xmm6, %xmm0");
-		asm volatile("pshufb %xmm3, %xmm1");
-		asm volatile("pxor %xmm0, %xmm1");
-
-		/* xmm1 = qmul[*q ^ *dq */
-
-		asm volatile("pxor %xmm1, %xmm2");
-
-		/* xmm2 = *p ^ qmul[*q ^ *dq] */
-
-		asm volatile("movdqa %%xmm1, %0" : "=m" (dq[0]));
-		asm volatile("movdqa %%xmm2, %0" : "=m" (p[0]));
-
-		bytes -= 16;
-		p += 16;
-		q += 16;
-		dq += 16;
-#endif
-	}
-
-	kernel_fpu_end();
-}
-
-const struct raid6_recov_calls raid6_recov_ssse3 = {
-	.data2 = raid6_2data_recov_ssse3,
-	.datap = raid6_datap_recov_ssse3,
-	.valid = raid6_has_ssse3,
-#ifdef CONFIG_X86_64
-	.name = "ssse3x2",
-#else
-	.name = "ssse3x1",
-#endif
-	.priority = 1,
-};
diff --git a/lib/raid6/rvv.c b/lib/raid6/rvv.c
deleted file mode 100644
index 75c9dafedb284..0000000000000
--- a/lib/raid6/rvv.c
+++ /dev/null
@@ -1,1228 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * RAID-6 syndrome calculation using RISC-V vector instructions
- *
- * Copyright 2024 Institute of Software, CAS.
- * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
- *
- * Based on neon.uc:
- *	Copyright 2002-2004 H. Peter Anvin
- */
-
-#include "rvv.h"
-
-#ifdef __riscv_vector
-#error "This code must be built without compiler support for vector"
-#endif
-
-static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	unsigned long vl, d, nsize;
-	int z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0 + 1];		/* XOR parity */
-	q = dptr[z0 + 2];		/* RS syndrome */
-
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : "=&r" (vl)
-	);
-
-	nsize = vl;
-
-	 /* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
-	for (d = 0; d < bytes; d += nsize * 1) {
-		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v0, (%[wp0])\n"
-			      "vmv.v.v	v1, v0\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * nsize])
-		);
-
-		for (z = z0 - 1 ; z >= 0 ; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * w1$$ ^= w2$$;
-			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			 * wq$$ = w1$$ ^ wd$$;
-			 * wp$$ ^= wd$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v3, v3, v2\n"
-				      "vle8.v	v2, (%[wd0])\n"
-				      "vxor.vv	v1, v3, v2\n"
-				      "vxor.vv	v0, v0, v2\n"
-				      ".option	pop\n"
-				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
-		 */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vse8.v	v0, (%[wp0])\n"
-			      "vse8.v	v1, (%[wq0])\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&p[d + nsize * 0]),
-			      [wq0]"r"(&q[d + nsize * 0])
-		);
-	}
-}
-
-static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
-					 unsigned long bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	unsigned long vl, d, nsize;
-	int z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks - 2];	/* XOR parity */
-	q = dptr[disks - 1];	/* RS syndrome */
-
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : "=&r" (vl)
-	);
-
-	nsize = vl;
-
-	/* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
-	for (d = 0 ; d < bytes ; d += nsize * 1) {
-		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v0, (%[wp0])\n"
-			      "vmv.v.v	v1, v0\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * nsize])
-		);
-
-		/* P/Q data pages */
-		for (z = z0 - 1; z >= start; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * w1$$ ^= w2$$;
-			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			 * wq$$ = w1$$ ^ wd$$;
-			 * wp$$ ^= wd$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v3, v3, v2\n"
-				      "vle8.v	v2, (%[wd0])\n"
-				      "vxor.vv	v1, v3, v2\n"
-				      "vxor.vv	v0, v0, v2\n"
-				      ".option	pop\n"
-				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/* P/Q left side optimization */
-		for (z = start - 1; z >= 0; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * wq$$ = w1$$ ^ w2$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v1, v3, v2\n"
-				      ".option	pop\n"
-				      : :
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
-		 * v0:wp0, v1:wq0, v2:p0, v3:q0
-		 */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v2, (%[wp0])\n"
-			      "vle8.v	v3, (%[wq0])\n"
-			      "vxor.vv	v2, v2, v0\n"
-			      "vxor.vv	v3, v3, v1\n"
-			      "vse8.v	v2, (%[wp0])\n"
-			      "vse8.v	v3, (%[wq0])\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&p[d + nsize * 0]),
-			      [wq0]"r"(&q[d + nsize * 0])
-		);
-	}
-}
-
-static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	unsigned long vl, d, nsize;
-	int z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0 + 1];		/* XOR parity */
-	q = dptr[z0 + 2];		/* RS syndrome */
-
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : "=&r" (vl)
-	);
-
-	nsize = vl;
-
-	/*
-	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
-	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
-	 */
-	for (d = 0; d < bytes; d += nsize * 2) {
-		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v0, (%[wp0])\n"
-			      "vmv.v.v	v1, v0\n"
-			      "vle8.v	v4, (%[wp1])\n"
-			      "vmv.v.v	v5, v4\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
-			      [wp1]"r"(&dptr[z0][d + 1 * nsize])
-		);
-
-		for (z = z0 - 1; z >= 0; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * w1$$ ^= w2$$;
-			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			 * wq$$ = w1$$ ^ wd$$;
-			 * wp$$ ^= wd$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v3, v3, v2\n"
-				      "vle8.v	v2, (%[wd0])\n"
-				      "vxor.vv	v1, v3, v2\n"
-				      "vxor.vv	v0, v0, v2\n"
-
-				      "vsra.vi	v6, v5, 7\n"
-				      "vsll.vi	v7, v5, 1\n"
-				      "vand.vx	v6, v6, %[x1d]\n"
-				      "vxor.vv	v7, v7, v6\n"
-				      "vle8.v	v6, (%[wd1])\n"
-				      "vxor.vv	v5, v7, v6\n"
-				      "vxor.vv	v4, v4, v6\n"
-				      ".option	pop\n"
-				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
-				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
-		 */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vse8.v	v0, (%[wp0])\n"
-			      "vse8.v	v1, (%[wq0])\n"
-			      "vse8.v	v4, (%[wp1])\n"
-			      "vse8.v	v5, (%[wq1])\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&p[d + nsize * 0]),
-			      [wq0]"r"(&q[d + nsize * 0]),
-			      [wp1]"r"(&p[d + nsize * 1]),
-			      [wq1]"r"(&q[d + nsize * 1])
-		);
-	}
-}
-
-static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
-					 unsigned long bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	unsigned long vl, d, nsize;
-	int z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks - 2];	/* XOR parity */
-	q = dptr[disks - 1];	/* RS syndrome */
-
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : "=&r" (vl)
-	);
-
-	nsize = vl;
-
-	/*
-	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
-	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
-	 */
-	for (d = 0; d < bytes; d += nsize * 2) {
-		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v0, (%[wp0])\n"
-			      "vmv.v.v	v1, v0\n"
-			      "vle8.v	v4, (%[wp1])\n"
-			      "vmv.v.v	v5, v4\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
-			      [wp1]"r"(&dptr[z0][d + 1 * nsize])
-		);
-
-		/* P/Q data pages */
-		for (z = z0 - 1; z >= start; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * w1$$ ^= w2$$;
-			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			 * wq$$ = w1$$ ^ wd$$;
-			 * wp$$ ^= wd$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v3, v3, v2\n"
-				      "vle8.v	v2, (%[wd0])\n"
-				      "vxor.vv	v1, v3, v2\n"
-				      "vxor.vv	v0, v0, v2\n"
-
-				      "vsra.vi	v6, v5, 7\n"
-				      "vsll.vi	v7, v5, 1\n"
-				      "vand.vx	v6, v6, %[x1d]\n"
-				      "vxor.vv	v7, v7, v6\n"
-				      "vle8.v	v6, (%[wd1])\n"
-				      "vxor.vv	v5, v7, v6\n"
-				      "vxor.vv	v4, v4, v6\n"
-				      ".option	pop\n"
-				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
-				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/* P/Q left side optimization */
-		for (z = start - 1; z >= 0; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * wq$$ = w1$$ ^ w2$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v1, v3, v2\n"
-
-				      "vsra.vi	v6, v5, 7\n"
-				      "vsll.vi	v7, v5, 1\n"
-				      "vand.vx	v6, v6, %[x1d]\n"
-				      "vxor.vv	v5, v7, v6\n"
-				      ".option	pop\n"
-				      : :
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
-		 * v0:wp0, v1:wq0, v2:p0, v3:q0
-		 * v4:wp1, v5:wq1, v6:p1, v7:q1
-		 */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v2, (%[wp0])\n"
-			      "vle8.v	v3, (%[wq0])\n"
-			      "vxor.vv	v2, v2, v0\n"
-			      "vxor.vv	v3, v3, v1\n"
-			      "vse8.v	v2, (%[wp0])\n"
-			      "vse8.v	v3, (%[wq0])\n"
-
-			      "vle8.v	v6, (%[wp1])\n"
-			      "vle8.v	v7, (%[wq1])\n"
-			      "vxor.vv	v6, v6, v4\n"
-			      "vxor.vv	v7, v7, v5\n"
-			      "vse8.v	v6, (%[wp1])\n"
-			      "vse8.v	v7, (%[wq1])\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&p[d + nsize * 0]),
-			      [wq0]"r"(&q[d + nsize * 0]),
-			      [wp1]"r"(&p[d + nsize * 1]),
-			      [wq1]"r"(&q[d + nsize * 1])
-		);
-	}
-}
-
-static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	unsigned long vl, d, nsize;
-	int z, z0;
-
-	z0 = disks - 3;	/* Highest data disk */
-	p = dptr[z0 + 1];	/* XOR parity */
-	q = dptr[z0 + 2];	/* RS syndrome */
-
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : "=&r" (vl)
-	);
-
-	nsize = vl;
-
-	/*
-	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
-	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
-	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
-	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
-	 */
-	for (d = 0; d < bytes; d += nsize * 4) {
-		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v0, (%[wp0])\n"
-			      "vmv.v.v	v1, v0\n"
-			      "vle8.v	v4, (%[wp1])\n"
-			      "vmv.v.v	v5, v4\n"
-			      "vle8.v	v8, (%[wp2])\n"
-			      "vmv.v.v	v9, v8\n"
-			      "vle8.v	v12, (%[wp3])\n"
-			      "vmv.v.v	v13, v12\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
-			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
-			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
-			      [wp3]"r"(&dptr[z0][d + 3 * nsize])
-		);
-
-		for (z = z0 - 1; z >= 0; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * w1$$ ^= w2$$;
-			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			 * wq$$ = w1$$ ^ wd$$;
-			 * wp$$ ^= wd$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v3, v3, v2\n"
-				      "vle8.v	v2, (%[wd0])\n"
-				      "vxor.vv	v1, v3, v2\n"
-				      "vxor.vv	v0, v0, v2\n"
-
-				      "vsra.vi	v6, v5, 7\n"
-				      "vsll.vi	v7, v5, 1\n"
-				      "vand.vx	v6, v6, %[x1d]\n"
-				      "vxor.vv	v7, v7, v6\n"
-				      "vle8.v	v6, (%[wd1])\n"
-				      "vxor.vv	v5, v7, v6\n"
-				      "vxor.vv	v4, v4, v6\n"
-
-				      "vsra.vi	v10, v9, 7\n"
-				      "vsll.vi	v11, v9, 1\n"
-				      "vand.vx	v10, v10, %[x1d]\n"
-				      "vxor.vv	v11, v11, v10\n"
-				      "vle8.v	v10, (%[wd2])\n"
-				      "vxor.vv	v9, v11, v10\n"
-				      "vxor.vv	v8, v8, v10\n"
-
-				      "vsra.vi	v14, v13, 7\n"
-				      "vsll.vi	v15, v13, 1\n"
-				      "vand.vx	v14, v14, %[x1d]\n"
-				      "vxor.vv	v15, v15, v14\n"
-				      "vle8.v	v14, (%[wd3])\n"
-				      "vxor.vv	v13, v15, v14\n"
-				      "vxor.vv	v12, v12, v14\n"
-				      ".option	pop\n"
-				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
-				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
-				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
-				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
-		 */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vse8.v	v0, (%[wp0])\n"
-			      "vse8.v	v1, (%[wq0])\n"
-			      "vse8.v	v4, (%[wp1])\n"
-			      "vse8.v	v5, (%[wq1])\n"
-			      "vse8.v	v8, (%[wp2])\n"
-			      "vse8.v	v9, (%[wq2])\n"
-			      "vse8.v	v12, (%[wp3])\n"
-			      "vse8.v	v13, (%[wq3])\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&p[d + nsize * 0]),
-			      [wq0]"r"(&q[d + nsize * 0]),
-			      [wp1]"r"(&p[d + nsize * 1]),
-			      [wq1]"r"(&q[d + nsize * 1]),
-			      [wp2]"r"(&p[d + nsize * 2]),
-			      [wq2]"r"(&q[d + nsize * 2]),
-			      [wp3]"r"(&p[d + nsize * 3]),
-			      [wq3]"r"(&q[d + nsize * 3])
-		);
-	}
-}
-
-static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
-					 unsigned long bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	unsigned long vl, d, nsize;
-	int z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks - 2];	/* XOR parity */
-	q = dptr[disks - 1];	/* RS syndrome */
-
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : "=&r" (vl)
-	);
-
-	nsize = vl;
-
-	/*
-	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
-	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
-	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
-	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
-	 */
-	for (d = 0; d < bytes; d += nsize * 4) {
-		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v0, (%[wp0])\n"
-			      "vmv.v.v	v1, v0\n"
-			      "vle8.v	v4, (%[wp1])\n"
-			      "vmv.v.v	v5, v4\n"
-			      "vle8.v	v8, (%[wp2])\n"
-			      "vmv.v.v	v9, v8\n"
-			      "vle8.v	v12, (%[wp3])\n"
-			      "vmv.v.v	v13, v12\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
-			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
-			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
-			      [wp3]"r"(&dptr[z0][d + 3 * nsize])
-		);
-
-		/* P/Q data pages */
-		for (z = z0 - 1; z >= start; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * w1$$ ^= w2$$;
-			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			 * wq$$ = w1$$ ^ wd$$;
-			 * wp$$ ^= wd$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v3, v3, v2\n"
-				      "vle8.v	v2, (%[wd0])\n"
-				      "vxor.vv	v1, v3, v2\n"
-				      "vxor.vv	v0, v0, v2\n"
-
-				      "vsra.vi	v6, v5, 7\n"
-				      "vsll.vi	v7, v5, 1\n"
-				      "vand.vx	v6, v6, %[x1d]\n"
-				      "vxor.vv	v7, v7, v6\n"
-				      "vle8.v	v6, (%[wd1])\n"
-				      "vxor.vv	v5, v7, v6\n"
-				      "vxor.vv	v4, v4, v6\n"
-
-				      "vsra.vi	v10, v9, 7\n"
-				      "vsll.vi	v11, v9, 1\n"
-				      "vand.vx	v10, v10, %[x1d]\n"
-				      "vxor.vv	v11, v11, v10\n"
-				      "vle8.v	v10, (%[wd2])\n"
-				      "vxor.vv	v9, v11, v10\n"
-				      "vxor.vv	v8, v8, v10\n"
-
-				      "vsra.vi	v14, v13, 7\n"
-				      "vsll.vi	v15, v13, 1\n"
-				      "vand.vx	v14, v14, %[x1d]\n"
-				      "vxor.vv	v15, v15, v14\n"
-				      "vle8.v	v14, (%[wd3])\n"
-				      "vxor.vv	v13, v15, v14\n"
-				      "vxor.vv	v12, v12, v14\n"
-				      ".option	pop\n"
-				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
-				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
-				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
-				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/* P/Q left side optimization */
-		for (z = start - 1; z >= 0; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * wq$$ = w1$$ ^ w2$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v1, v3, v2\n"
-
-				      "vsra.vi	v6, v5, 7\n"
-				      "vsll.vi	v7, v5, 1\n"
-				      "vand.vx	v6, v6, %[x1d]\n"
-				      "vxor.vv	v5, v7, v6\n"
-
-				      "vsra.vi	v10, v9, 7\n"
-				      "vsll.vi	v11, v9, 1\n"
-				      "vand.vx	v10, v10, %[x1d]\n"
-				      "vxor.vv	v9, v11, v10\n"
-
-				      "vsra.vi	v14, v13, 7\n"
-				      "vsll.vi	v15, v13, 1\n"
-				      "vand.vx	v14, v14, %[x1d]\n"
-				      "vxor.vv	v13, v15, v14\n"
-				      ".option	pop\n"
-				      : :
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
-		 * v0:wp0, v1:wq0, v2:p0, v3:q0
-		 * v4:wp1, v5:wq1, v6:p1, v7:q1
-		 * v8:wp2, v9:wq2, v10:p2, v11:q2
-		 * v12:wp3, v13:wq3, v14:p3, v15:q3
-		 */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v2, (%[wp0])\n"
-			      "vle8.v	v3, (%[wq0])\n"
-			      "vxor.vv	v2, v2, v0\n"
-			      "vxor.vv	v3, v3, v1\n"
-			      "vse8.v	v2, (%[wp0])\n"
-			      "vse8.v	v3, (%[wq0])\n"
-
-			      "vle8.v	v6, (%[wp1])\n"
-			      "vle8.v	v7, (%[wq1])\n"
-			      "vxor.vv	v6, v6, v4\n"
-			      "vxor.vv	v7, v7, v5\n"
-			      "vse8.v	v6, (%[wp1])\n"
-			      "vse8.v	v7, (%[wq1])\n"
-
-			      "vle8.v	v10, (%[wp2])\n"
-			      "vle8.v	v11, (%[wq2])\n"
-			      "vxor.vv	v10, v10, v8\n"
-			      "vxor.vv	v11, v11, v9\n"
-			      "vse8.v	v10, (%[wp2])\n"
-			      "vse8.v	v11, (%[wq2])\n"
-
-			      "vle8.v	v14, (%[wp3])\n"
-			      "vle8.v	v15, (%[wq3])\n"
-			      "vxor.vv	v14, v14, v12\n"
-			      "vxor.vv	v15, v15, v13\n"
-			      "vse8.v	v14, (%[wp3])\n"
-			      "vse8.v	v15, (%[wq3])\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&p[d + nsize * 0]),
-			      [wq0]"r"(&q[d + nsize * 0]),
-			      [wp1]"r"(&p[d + nsize * 1]),
-			      [wq1]"r"(&q[d + nsize * 1]),
-			      [wp2]"r"(&p[d + nsize * 2]),
-			      [wq2]"r"(&q[d + nsize * 2]),
-			      [wp3]"r"(&p[d + nsize * 3]),
-			      [wq3]"r"(&q[d + nsize * 3])
-		);
-	}
-}
-
-static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	unsigned long vl, d, nsize;
-	int z, z0;
-
-	z0 = disks - 3;	/* Highest data disk */
-	p = dptr[z0 + 1];	/* XOR parity */
-	q = dptr[z0 + 2];	/* RS syndrome */
-
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : "=&r" (vl)
-	);
-
-	nsize = vl;
-
-	/*
-	 * v0:wp0,   v1:wq0,  v2:wd0/w20,  v3:w10
-	 * v4:wp1,   v5:wq1,  v6:wd1/w21,  v7:w11
-	 * v8:wp2,   v9:wq2, v10:wd2/w22, v11:w12
-	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
-	 * v16:wp4, v17:wq4, v18:wd4/w24, v19:w14
-	 * v20:wp5, v21:wq5, v22:wd5/w25, v23:w15
-	 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
-	 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
-	 */
-	for (d = 0; d < bytes; d += nsize * 8) {
-		/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v0, (%[wp0])\n"
-			      "vmv.v.v	v1, v0\n"
-			      "vle8.v	v4, (%[wp1])\n"
-			      "vmv.v.v	v5, v4\n"
-			      "vle8.v	v8, (%[wp2])\n"
-			      "vmv.v.v	v9, v8\n"
-			      "vle8.v	v12, (%[wp3])\n"
-			      "vmv.v.v	v13, v12\n"
-			      "vle8.v	v16, (%[wp4])\n"
-			      "vmv.v.v	v17, v16\n"
-			      "vle8.v	v20, (%[wp5])\n"
-			      "vmv.v.v	v21, v20\n"
-			      "vle8.v	v24, (%[wp6])\n"
-			      "vmv.v.v	v25, v24\n"
-			      "vle8.v	v28, (%[wp7])\n"
-			      "vmv.v.v	v29, v28\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
-			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
-			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
-			      [wp3]"r"(&dptr[z0][d + 3 * nsize]),
-			      [wp4]"r"(&dptr[z0][d + 4 * nsize]),
-			      [wp5]"r"(&dptr[z0][d + 5 * nsize]),
-			      [wp6]"r"(&dptr[z0][d + 6 * nsize]),
-			      [wp7]"r"(&dptr[z0][d + 7 * nsize])
-		);
-
-		for (z = z0 - 1; z >= 0; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * w1$$ ^= w2$$;
-			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			 * wq$$ = w1$$ ^ wd$$;
-			 * wp$$ ^= wd$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v3, v3, v2\n"
-				      "vle8.v	v2, (%[wd0])\n"
-				      "vxor.vv	v1, v3, v2\n"
-				      "vxor.vv	v0, v0, v2\n"
-
-				      "vsra.vi	v6, v5, 7\n"
-				      "vsll.vi	v7, v5, 1\n"
-				      "vand.vx	v6, v6, %[x1d]\n"
-				      "vxor.vv	v7, v7, v6\n"
-				      "vle8.v	v6, (%[wd1])\n"
-				      "vxor.vv	v5, v7, v6\n"
-				      "vxor.vv	v4, v4, v6\n"
-
-				      "vsra.vi	v10, v9, 7\n"
-				      "vsll.vi	v11, v9, 1\n"
-				      "vand.vx	v10, v10, %[x1d]\n"
-				      "vxor.vv	v11, v11, v10\n"
-				      "vle8.v	v10, (%[wd2])\n"
-				      "vxor.vv	v9, v11, v10\n"
-				      "vxor.vv	v8, v8, v10\n"
-
-				      "vsra.vi	v14, v13, 7\n"
-				      "vsll.vi	v15, v13, 1\n"
-				      "vand.vx	v14, v14, %[x1d]\n"
-				      "vxor.vv	v15, v15, v14\n"
-				      "vle8.v	v14, (%[wd3])\n"
-				      "vxor.vv	v13, v15, v14\n"
-				      "vxor.vv	v12, v12, v14\n"
-
-				      "vsra.vi	v18, v17, 7\n"
-				      "vsll.vi	v19, v17, 1\n"
-				      "vand.vx	v18, v18, %[x1d]\n"
-				      "vxor.vv	v19, v19, v18\n"
-				      "vle8.v	v18, (%[wd4])\n"
-				      "vxor.vv	v17, v19, v18\n"
-				      "vxor.vv	v16, v16, v18\n"
-
-				      "vsra.vi	v22, v21, 7\n"
-				      "vsll.vi	v23, v21, 1\n"
-				      "vand.vx	v22, v22, %[x1d]\n"
-				      "vxor.vv	v23, v23, v22\n"
-				      "vle8.v	v22, (%[wd5])\n"
-				      "vxor.vv	v21, v23, v22\n"
-				      "vxor.vv	v20, v20, v22\n"
-
-				      "vsra.vi	v26, v25, 7\n"
-				      "vsll.vi	v27, v25, 1\n"
-				      "vand.vx	v26, v26, %[x1d]\n"
-				      "vxor.vv	v27, v27, v26\n"
-				      "vle8.v	v26, (%[wd6])\n"
-				      "vxor.vv	v25, v27, v26\n"
-				      "vxor.vv	v24, v24, v26\n"
-
-				      "vsra.vi	v30, v29, 7\n"
-				      "vsll.vi	v31, v29, 1\n"
-				      "vand.vx	v30, v30, %[x1d]\n"
-				      "vxor.vv	v31, v31, v30\n"
-				      "vle8.v	v30, (%[wd7])\n"
-				      "vxor.vv	v29, v31, v30\n"
-				      "vxor.vv	v28, v28, v30\n"
-				      ".option	pop\n"
-				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
-				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
-				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
-				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
-				      [wd4]"r"(&dptr[z][d + 4 * nsize]),
-				      [wd5]"r"(&dptr[z][d + 5 * nsize]),
-				      [wd6]"r"(&dptr[z][d + 6 * nsize]),
-				      [wd7]"r"(&dptr[z][d + 7 * nsize]),
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] = wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] = wq$$;
-		 */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vse8.v	v0, (%[wp0])\n"
-			      "vse8.v	v1, (%[wq0])\n"
-			      "vse8.v	v4, (%[wp1])\n"
-			      "vse8.v	v5, (%[wq1])\n"
-			      "vse8.v	v8, (%[wp2])\n"
-			      "vse8.v	v9, (%[wq2])\n"
-			      "vse8.v	v12, (%[wp3])\n"
-			      "vse8.v	v13, (%[wq3])\n"
-			      "vse8.v	v16, (%[wp4])\n"
-			      "vse8.v	v17, (%[wq4])\n"
-			      "vse8.v	v20, (%[wp5])\n"
-			      "vse8.v	v21, (%[wq5])\n"
-			      "vse8.v	v24, (%[wp6])\n"
-			      "vse8.v	v25, (%[wq6])\n"
-			      "vse8.v	v28, (%[wp7])\n"
-			      "vse8.v	v29, (%[wq7])\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&p[d + nsize * 0]),
-			      [wq0]"r"(&q[d + nsize * 0]),
-			      [wp1]"r"(&p[d + nsize * 1]),
-			      [wq1]"r"(&q[d + nsize * 1]),
-			      [wp2]"r"(&p[d + nsize * 2]),
-			      [wq2]"r"(&q[d + nsize * 2]),
-			      [wp3]"r"(&p[d + nsize * 3]),
-			      [wq3]"r"(&q[d + nsize * 3]),
-			      [wp4]"r"(&p[d + nsize * 4]),
-			      [wq4]"r"(&q[d + nsize * 4]),
-			      [wp5]"r"(&p[d + nsize * 5]),
-			      [wq5]"r"(&q[d + nsize * 5]),
-			      [wp6]"r"(&p[d + nsize * 6]),
-			      [wq6]"r"(&q[d + nsize * 6]),
-			      [wp7]"r"(&p[d + nsize * 7]),
-			      [wq7]"r"(&q[d + nsize * 7])
-		);
-	}
-}
-
-static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
-					 unsigned long bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	unsigned long vl, d, nsize;
-	int z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks - 2];	/* XOR parity */
-	q = dptr[disks - 1];	/* RS syndrome */
-
-	asm volatile (".option	push\n"
-		      ".option	arch,+v\n"
-		      "vsetvli	%0, x0, e8, m1, ta, ma\n"
-		      ".option	pop\n"
-		      : "=&r" (vl)
-	);
-
-	nsize = vl;
-
-	/*
-	 * v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
-	 * v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
-	 * v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
-	 * v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
-	 * v16:wp4, v17:wq4, v18:wd4/w24, v19:w14
-	 * v20:wp5, v21:wq5, v22:wd5/w25, v23:w15
-	 * v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
-	 * v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
-	 */
-	for (d = 0; d < bytes; d += nsize * 8) {
-		 /* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v0, (%[wp0])\n"
-			      "vmv.v.v	v1, v0\n"
-			      "vle8.v	v4, (%[wp1])\n"
-			      "vmv.v.v	v5, v4\n"
-			      "vle8.v	v8, (%[wp2])\n"
-			      "vmv.v.v	v9, v8\n"
-			      "vle8.v	v12, (%[wp3])\n"
-			      "vmv.v.v	v13, v12\n"
-			      "vle8.v	v16, (%[wp4])\n"
-			      "vmv.v.v	v17, v16\n"
-			      "vle8.v	v20, (%[wp5])\n"
-			      "vmv.v.v	v21, v20\n"
-			      "vle8.v	v24, (%[wp6])\n"
-			      "vmv.v.v	v25, v24\n"
-			      "vle8.v	v28, (%[wp7])\n"
-			      "vmv.v.v	v29, v28\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&dptr[z0][d + 0 * nsize]),
-			      [wp1]"r"(&dptr[z0][d + 1 * nsize]),
-			      [wp2]"r"(&dptr[z0][d + 2 * nsize]),
-			      [wp3]"r"(&dptr[z0][d + 3 * nsize]),
-			      [wp4]"r"(&dptr[z0][d + 4 * nsize]),
-			      [wp5]"r"(&dptr[z0][d + 5 * nsize]),
-			      [wp6]"r"(&dptr[z0][d + 6 * nsize]),
-			      [wp7]"r"(&dptr[z0][d + 7 * nsize])
-		);
-
-		/* P/Q data pages */
-		for (z = z0 - 1; z >= start; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * w1$$ ^= w2$$;
-			 * wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			 * wq$$ = w1$$ ^ wd$$;
-			 * wp$$ ^= wd$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v3, v3, v2\n"
-				      "vle8.v	v2, (%[wd0])\n"
-				      "vxor.vv	v1, v3, v2\n"
-				      "vxor.vv	v0, v0, v2\n"
-
-				      "vsra.vi	v6, v5, 7\n"
-				      "vsll.vi	v7, v5, 1\n"
-				      "vand.vx	v6, v6, %[x1d]\n"
-				      "vxor.vv	v7, v7, v6\n"
-				      "vle8.v	v6, (%[wd1])\n"
-				      "vxor.vv	v5, v7, v6\n"
-				      "vxor.vv	v4, v4, v6\n"
-
-				      "vsra.vi	v10, v9, 7\n"
-				      "vsll.vi	v11, v9, 1\n"
-				      "vand.vx	v10, v10, %[x1d]\n"
-				      "vxor.vv	v11, v11, v10\n"
-				      "vle8.v	v10, (%[wd2])\n"
-				      "vxor.vv	v9, v11, v10\n"
-				      "vxor.vv	v8, v8, v10\n"
-
-				      "vsra.vi	v14, v13, 7\n"
-				      "vsll.vi	v15, v13, 1\n"
-				      "vand.vx	v14, v14, %[x1d]\n"
-				      "vxor.vv	v15, v15, v14\n"
-				      "vle8.v	v14, (%[wd3])\n"
-				      "vxor.vv	v13, v15, v14\n"
-				      "vxor.vv	v12, v12, v14\n"
-
-				      "vsra.vi	v18, v17, 7\n"
-				      "vsll.vi	v19, v17, 1\n"
-				      "vand.vx	v18, v18, %[x1d]\n"
-				      "vxor.vv	v19, v19, v18\n"
-				      "vle8.v	v18, (%[wd4])\n"
-				      "vxor.vv	v17, v19, v18\n"
-				      "vxor.vv	v16, v16, v18\n"
-
-				      "vsra.vi	v22, v21, 7\n"
-				      "vsll.vi	v23, v21, 1\n"
-				      "vand.vx	v22, v22, %[x1d]\n"
-				      "vxor.vv	v23, v23, v22\n"
-				      "vle8.v	v22, (%[wd5])\n"
-				      "vxor.vv	v21, v23, v22\n"
-				      "vxor.vv	v20, v20, v22\n"
-
-				      "vsra.vi	v26, v25, 7\n"
-				      "vsll.vi	v27, v25, 1\n"
-				      "vand.vx	v26, v26, %[x1d]\n"
-				      "vxor.vv	v27, v27, v26\n"
-				      "vle8.v	v26, (%[wd6])\n"
-				      "vxor.vv	v25, v27, v26\n"
-				      "vxor.vv	v24, v24, v26\n"
-
-				      "vsra.vi	v30, v29, 7\n"
-				      "vsll.vi	v31, v29, 1\n"
-				      "vand.vx	v30, v30, %[x1d]\n"
-				      "vxor.vv	v31, v31, v30\n"
-				      "vle8.v	v30, (%[wd7])\n"
-				      "vxor.vv	v29, v31, v30\n"
-				      "vxor.vv	v28, v28, v30\n"
-				      ".option	pop\n"
-				      : :
-				      [wd0]"r"(&dptr[z][d + 0 * nsize]),
-				      [wd1]"r"(&dptr[z][d + 1 * nsize]),
-				      [wd2]"r"(&dptr[z][d + 2 * nsize]),
-				      [wd3]"r"(&dptr[z][d + 3 * nsize]),
-				      [wd4]"r"(&dptr[z][d + 4 * nsize]),
-				      [wd5]"r"(&dptr[z][d + 5 * nsize]),
-				      [wd6]"r"(&dptr[z][d + 6 * nsize]),
-				      [wd7]"r"(&dptr[z][d + 7 * nsize]),
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/* P/Q left side optimization */
-		for (z = start - 1; z >= 0; z--) {
-			/*
-			 * w2$$ = MASK(wq$$);
-			 * w1$$ = SHLBYTE(wq$$);
-			 * w2$$ &= NBYTES(0x1d);
-			 * wq$$ = w1$$ ^ w2$$;
-			 */
-			asm volatile (".option	push\n"
-				      ".option	arch,+v\n"
-				      "vsra.vi	v2, v1, 7\n"
-				      "vsll.vi	v3, v1, 1\n"
-				      "vand.vx	v2, v2, %[x1d]\n"
-				      "vxor.vv	v1, v3, v2\n"
-
-				      "vsra.vi	v6, v5, 7\n"
-				      "vsll.vi	v7, v5, 1\n"
-				      "vand.vx	v6, v6, %[x1d]\n"
-				      "vxor.vv	v5, v7, v6\n"
-
-				      "vsra.vi	v10, v9, 7\n"
-				      "vsll.vi	v11, v9, 1\n"
-				      "vand.vx	v10, v10, %[x1d]\n"
-				      "vxor.vv	v9, v11, v10\n"
-
-				      "vsra.vi	v14, v13, 7\n"
-				      "vsll.vi	v15, v13, 1\n"
-				      "vand.vx	v14, v14, %[x1d]\n"
-				      "vxor.vv	v13, v15, v14\n"
-
-				      "vsra.vi	v18, v17, 7\n"
-				      "vsll.vi	v19, v17, 1\n"
-				      "vand.vx	v18, v18, %[x1d]\n"
-				      "vxor.vv	v17, v19, v18\n"
-
-				      "vsra.vi	v22, v21, 7\n"
-				      "vsll.vi	v23, v21, 1\n"
-				      "vand.vx	v22, v22, %[x1d]\n"
-				      "vxor.vv	v21, v23, v22\n"
-
-				      "vsra.vi	v26, v25, 7\n"
-				      "vsll.vi	v27, v25, 1\n"
-				      "vand.vx	v26, v26, %[x1d]\n"
-				      "vxor.vv	v25, v27, v26\n"
-
-				      "vsra.vi	v30, v29, 7\n"
-				      "vsll.vi	v31, v29, 1\n"
-				      "vand.vx	v30, v30, %[x1d]\n"
-				      "vxor.vv	v29, v31, v30\n"
-				      ".option	pop\n"
-				      : :
-				      [x1d]"r"(0x1d)
-			);
-		}
-
-		/*
-		 * *(unative_t *)&p[d+NSIZE*$$] ^= wp$$;
-		 * *(unative_t *)&q[d+NSIZE*$$] ^= wq$$;
-		 * v0:wp0, v1:wq0, v2:p0, v3:q0
-		 * v4:wp1, v5:wq1, v6:p1, v7:q1
-		 * v8:wp2, v9:wq2, v10:p2, v11:q2
-		 * v12:wp3, v13:wq3, v14:p3, v15:q3
-		 * v16:wp4, v17:wq4, v18:p4, v19:q4
-		 * v20:wp5, v21:wq5, v22:p5, v23:q5
-		 * v24:wp6, v25:wq6, v26:p6, v27:q6
-		 * v28:wp7, v29:wq7, v30:p7, v31:q7
-		 */
-		asm volatile (".option	push\n"
-			      ".option	arch,+v\n"
-			      "vle8.v	v2, (%[wp0])\n"
-			      "vle8.v	v3, (%[wq0])\n"
-			      "vxor.vv	v2, v2, v0\n"
-			      "vxor.vv	v3, v3, v1\n"
-			      "vse8.v	v2, (%[wp0])\n"
-			      "vse8.v	v3, (%[wq0])\n"
-
-			      "vle8.v	v6, (%[wp1])\n"
-			      "vle8.v	v7, (%[wq1])\n"
-			      "vxor.vv	v6, v6, v4\n"
-			      "vxor.vv	v7, v7, v5\n"
-			      "vse8.v	v6, (%[wp1])\n"
-			      "vse8.v	v7, (%[wq1])\n"
-
-			      "vle8.v	v10, (%[wp2])\n"
-			      "vle8.v	v11, (%[wq2])\n"
-			      "vxor.vv	v10, v10, v8\n"
-			      "vxor.vv	v11, v11, v9\n"
-			      "vse8.v	v10, (%[wp2])\n"
-			      "vse8.v	v11, (%[wq2])\n"
-
-			      "vle8.v	v14, (%[wp3])\n"
-			      "vle8.v	v15, (%[wq3])\n"
-			      "vxor.vv	v14, v14, v12\n"
-			      "vxor.vv	v15, v15, v13\n"
-			      "vse8.v	v14, (%[wp3])\n"
-			      "vse8.v	v15, (%[wq3])\n"
-
-			      "vle8.v	v18, (%[wp4])\n"
-			      "vle8.v	v19, (%[wq4])\n"
-			      "vxor.vv	v18, v18, v16\n"
-			      "vxor.vv	v19, v19, v17\n"
-			      "vse8.v	v18, (%[wp4])\n"
-			      "vse8.v	v19, (%[wq4])\n"
-
-			      "vle8.v	v22, (%[wp5])\n"
-			      "vle8.v	v23, (%[wq5])\n"
-			      "vxor.vv	v22, v22, v20\n"
-			      "vxor.vv	v23, v23, v21\n"
-			      "vse8.v	v22, (%[wp5])\n"
-			      "vse8.v	v23, (%[wq5])\n"
-
-			      "vle8.v	v26, (%[wp6])\n"
-			      "vle8.v	v27, (%[wq6])\n"
-			      "vxor.vv	v26, v26, v24\n"
-			      "vxor.vv	v27, v27, v25\n"
-			      "vse8.v	v26, (%[wp6])\n"
-			      "vse8.v	v27, (%[wq6])\n"
-
-			      "vle8.v	v30, (%[wp7])\n"
-			      "vle8.v	v31, (%[wq7])\n"
-			      "vxor.vv	v30, v30, v28\n"
-			      "vxor.vv	v31, v31, v29\n"
-			      "vse8.v	v30, (%[wp7])\n"
-			      "vse8.v	v31, (%[wq7])\n"
-			      ".option	pop\n"
-			      : :
-			      [wp0]"r"(&p[d + nsize * 0]),
-			      [wq0]"r"(&q[d + nsize * 0]),
-			      [wp1]"r"(&p[d + nsize * 1]),
-			      [wq1]"r"(&q[d + nsize * 1]),
-			      [wp2]"r"(&p[d + nsize * 2]),
-			      [wq2]"r"(&q[d + nsize * 2]),
-			      [wp3]"r"(&p[d + nsize * 3]),
-			      [wq3]"r"(&q[d + nsize * 3]),
-			      [wp4]"r"(&p[d + nsize * 4]),
-			      [wq4]"r"(&q[d + nsize * 4]),
-			      [wp5]"r"(&p[d + nsize * 5]),
-			      [wq5]"r"(&q[d + nsize * 5]),
-			      [wp6]"r"(&p[d + nsize * 6]),
-			      [wq6]"r"(&q[d + nsize * 6]),
-			      [wp7]"r"(&p[d + nsize * 7]),
-			      [wq7]"r"(&q[d + nsize * 7])
-		);
-	}
-}
-
-RAID6_RVV_WRAPPER(1);
-RAID6_RVV_WRAPPER(2);
-RAID6_RVV_WRAPPER(4);
-RAID6_RVV_WRAPPER(8);
diff --git a/lib/raid6/rvv.h b/lib/raid6/rvv.h
deleted file mode 100644
index 6d0708a2c8a4b..0000000000000
--- a/lib/raid6/rvv.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright 2024 Institute of Software, CAS.
- *
- * raid6/rvv.h
- *
- * Definitions for RISC-V RAID-6 code
- */
-
-#ifdef __KERNEL__
-#include <asm/vector.h>
-#else
-#define kernel_vector_begin()
-#define kernel_vector_end()
-#include <sys/auxv.h>
-#include <asm/hwcap.h>
-#define has_vector() (getauxval(AT_HWCAP) & COMPAT_HWCAP_ISA_V)
-#endif
-
-#include <linux/raid/pq.h>
-
-static int rvv_has_vector(void)
-{
-	return has_vector();
-}
-
-#define RAID6_RVV_WRAPPER(_n)						\
-	static void raid6_rvv ## _n ## _gen_syndrome(int disks,		\
-					size_t bytes, void **ptrs)	\
-	{								\
-		void raid6_rvv ## _n  ## _gen_syndrome_real(int d,	\
-					unsigned long b, void **p);	\
-		kernel_vector_begin();					\
-		raid6_rvv ## _n ## _gen_syndrome_real(disks,		\
-				(unsigned long)bytes, ptrs);		\
-		kernel_vector_end();					\
-	}								\
-	static void raid6_rvv ## _n ## _xor_syndrome(int disks,		\
-					int start, int stop,		\
-					size_t bytes, void **ptrs)	\
-	{								\
-		void raid6_rvv ## _n  ## _xor_syndrome_real(int d,	\
-					int s1, int s2,			\
-					unsigned long b, void **p);	\
-		kernel_vector_begin();					\
-		raid6_rvv ## _n ## _xor_syndrome_real(disks,		\
-			start, stop, (unsigned long)bytes, ptrs);	\
-		kernel_vector_end();					\
-	}								\
-	struct raid6_calls const raid6_rvvx ## _n = {			\
-		raid6_rvv ## _n ## _gen_syndrome,			\
-		raid6_rvv ## _n ## _xor_syndrome,			\
-		rvv_has_vector,						\
-		"rvvx" #_n,						\
-		0							\
-	}
diff --git a/lib/raid6/s390vx.uc b/lib/raid6/s390vx.uc
deleted file mode 100644
index 8aa53eb2f3958..0000000000000
--- a/lib/raid6/s390vx.uc
+++ /dev/null
@@ -1,135 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * raid6_vx$#.c
- *
- * $#-way unrolled RAID6 gen/xor functions for s390
- * based on the vector facility
- *
- * Copyright IBM Corp. 2016
- * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
- *
- * This file is postprocessed using unroll.awk.
- */
-
-#include <linux/cpufeature.h>
-#include <linux/raid/pq.h>
-#include <asm/fpu.h>
-
-#define NSIZE 16
-
-static __always_inline void LOAD_CONST(void)
-{
-	fpu_vrepib(24, 0x07);
-	fpu_vrepib(25, 0x1d);
-}
-
-/*
- * The SHLBYTE() operation shifts each of the 16 bytes in
- * vector register y left by 1 bit and stores the result in
- * vector register x.
- */
-#define SHLBYTE(x, y)		fpu_vab(x, y, y)
-
-/*
- * For each of the 16 bytes in the vector register y the MASK()
- * operation returns 0xFF if the high bit of the byte is 1,
- * or 0x00 if the high bit is 0. The result is stored in vector
- * register x.
- */
-#define MASK(x, y)		fpu_vesravb(x, y, 24)
-
-#define AND(x, y, z)		fpu_vn(x, y, z)
-#define XOR(x, y, z)		fpu_vx(x, y, z)
-#define LOAD_DATA(x, ptr)	fpu_vlm(x, x + $# - 1, ptr)
-#define STORE_DATA(x, ptr)	fpu_vstm(x, x + $# - 1, ptr)
-#define COPY_VEC(x, y)		fpu_vlr(x, y)
-
-static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	DECLARE_KERNEL_FPU_ONSTACK32(vxstate);
-	u8 **dptr, *p, *q;
-	int d, z, z0;
-
-	kernel_fpu_begin(&vxstate, KERNEL_VXR);
-	LOAD_CONST();
-
-	dptr = (u8 **) ptrs;
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0 + 1];	/* XOR parity */
-	q = dptr[z0 + 2];	/* RS syndrome */
-
-	for (d = 0; d < bytes; d += $#*NSIZE) {
-		LOAD_DATA(0,&dptr[z0][d]);
-		COPY_VEC(8+$$,0+$$);
-		for (z = z0 - 1; z >= 0; z--) {
-			MASK(16+$$,8+$$);
-			AND(16+$$,16+$$,25);
-			SHLBYTE(8+$$,8+$$);
-			XOR(8+$$,8+$$,16+$$);
-			LOAD_DATA(16,&dptr[z][d]);
-			XOR(0+$$,0+$$,16+$$);
-			XOR(8+$$,8+$$,16+$$);
-		}
-		STORE_DATA(0,&p[d]);
-		STORE_DATA(8,&q[d]);
-	}
-	kernel_fpu_end(&vxstate, KERNEL_VXR);
-}
-
-static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop,
-					size_t bytes, void **ptrs)
-{
-	DECLARE_KERNEL_FPU_ONSTACK32(vxstate);
-	u8 **dptr, *p, *q;
-	int d, z, z0;
-
-	dptr = (u8 **) ptrs;
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks - 2];	/* XOR parity */
-	q = dptr[disks - 1];	/* RS syndrome */
-
-	kernel_fpu_begin(&vxstate, KERNEL_VXR);
-	LOAD_CONST();
-
-	for (d = 0; d < bytes; d += $#*NSIZE) {
-		/* P/Q data pages */
-		LOAD_DATA(0,&dptr[z0][d]);
-		COPY_VEC(8+$$,0+$$);
-		for (z = z0 - 1; z >= start; z--) {
-			MASK(16+$$,8+$$);
-			AND(16+$$,16+$$,25);
-			SHLBYTE(8+$$,8+$$);
-			XOR(8+$$,8+$$,16+$$);
-			LOAD_DATA(16,&dptr[z][d]);
-			XOR(0+$$,0+$$,16+$$);
-			XOR(8+$$,8+$$,16+$$);
-		}
-		/* P/Q left side optimization */
-		for (z = start - 1; z >= 0; z--) {
-			MASK(16+$$,8+$$);
-			AND(16+$$,16+$$,25);
-			SHLBYTE(8+$$,8+$$);
-			XOR(8+$$,8+$$,16+$$);
-		}
-		LOAD_DATA(16,&p[d]);
-		XOR(16+$$,16+$$,0+$$);
-		STORE_DATA(16,&p[d]);
-		LOAD_DATA(16,&q[d]);
-		XOR(16+$$,16+$$,8+$$);
-		STORE_DATA(16,&q[d]);
-	}
-	kernel_fpu_end(&vxstate, KERNEL_VXR);
-}
-
-static int raid6_s390vx$#_valid(void)
-{
-	return cpu_has_vx();
-}
-
-const struct raid6_calls raid6_s390vx$# = {
-	raid6_s390vx$#_gen_syndrome,
-	raid6_s390vx$#_xor_syndrome,
-	raid6_s390vx$#_valid,
-	"vx128x$#",
-	1
-};
diff --git a/lib/raid6/sse1.c b/lib/raid6/sse1.c
deleted file mode 100644
index 692fa3a93bf0b..0000000000000
--- a/lib/raid6/sse1.c
+++ /dev/null
@@ -1,159 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright 2002 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * raid6/sse1.c
- *
- * SSE-1/MMXEXT implementation of RAID-6 syndrome functions
- *
- * This is really an MMX implementation, but it requires SSE-1 or
- * AMD MMXEXT for prefetch support and a few other features.  The
- * support for nontemporal memory accesses is enough to make this
- * worthwhile as a separate implementation.
- */
-
-#ifdef CONFIG_X86_32
-
-#include <linux/raid/pq.h>
-#include "x86.h"
-
-/* Defined in raid6/mmx.c */
-extern const struct raid6_mmx_constants {
-	u64 x1d;
-} raid6_mmx_constants;
-
-static int raid6_have_sse1_or_mmxext(void)
-{
-	/* Not really boot_cpu but "all_cpus" */
-	return boot_cpu_has(X86_FEATURE_MMX) &&
-		(boot_cpu_has(X86_FEATURE_XMM) ||
-		 boot_cpu_has(X86_FEATURE_MMXEXT));
-}
-
-/*
- * Plain SSE1 implementation
- */
-static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
-	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
-
-	for ( d = 0 ; d < bytes ; d += 8 ) {
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
-		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
-		asm volatile("movq %mm2,%mm4");	/* Q[0] */
-		asm volatile("movq %0,%%mm6" : : "m" (dptr[z0-1][d]));
-		for ( z = z0-2 ; z >= 0 ; z-- ) {
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
-			asm volatile("pcmpgtb %mm4,%mm5");
-			asm volatile("paddb %mm4,%mm4");
-			asm volatile("pand %mm0,%mm5");
-			asm volatile("pxor %mm5,%mm4");
-			asm volatile("pxor %mm5,%mm5");
-			asm volatile("pxor %mm6,%mm2");
-			asm volatile("pxor %mm6,%mm4");
-			asm volatile("movq %0,%%mm6" : : "m" (dptr[z][d]));
-		}
-		asm volatile("pcmpgtb %mm4,%mm5");
-		asm volatile("paddb %mm4,%mm4");
-		asm volatile("pand %mm0,%mm5");
-		asm volatile("pxor %mm5,%mm4");
-		asm volatile("pxor %mm5,%mm5");
-		asm volatile("pxor %mm6,%mm2");
-		asm volatile("pxor %mm6,%mm4");
-
-		asm volatile("movntq %%mm2,%0" : "=m" (p[d]));
-		asm volatile("movntq %%mm4,%0" : "=m" (q[d]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_sse1x1 = {
-	raid6_sse11_gen_syndrome,
-	NULL,			/* XOR not yet implemented */
-	raid6_have_sse1_or_mmxext,
-	"sse1x1",
-	1			/* Has cache hints */
-};
-
-/*
- * Unrolled-by-2 SSE1 implementation
- */
-static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
-	asm volatile("pxor %mm5,%mm5");	/* Zero temp */
-	asm volatile("pxor %mm7,%mm7"); /* Zero temp */
-
-	/* We uniformly assume a single prefetch covers at least 16 bytes */
-	for ( d = 0 ; d < bytes ; d += 16 ) {
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
-		asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d])); /* P[0] */
-		asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8])); /* P[1] */
-		asm volatile("movq %mm2,%mm4");	/* Q[0] */
-		asm volatile("movq %mm3,%mm6"); /* Q[1] */
-		for ( z = z0-1 ; z >= 0 ; z-- ) {
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
-			asm volatile("pcmpgtb %mm4,%mm5");
-			asm volatile("pcmpgtb %mm6,%mm7");
-			asm volatile("paddb %mm4,%mm4");
-			asm volatile("paddb %mm6,%mm6");
-			asm volatile("pand %mm0,%mm5");
-			asm volatile("pand %mm0,%mm7");
-			asm volatile("pxor %mm5,%mm4");
-			asm volatile("pxor %mm7,%mm6");
-			asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
-			asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
-			asm volatile("pxor %mm5,%mm2");
-			asm volatile("pxor %mm7,%mm3");
-			asm volatile("pxor %mm5,%mm4");
-			asm volatile("pxor %mm7,%mm6");
-			asm volatile("pxor %mm5,%mm5");
-			asm volatile("pxor %mm7,%mm7");
-		}
-		asm volatile("movntq %%mm2,%0" : "=m" (p[d]));
-		asm volatile("movntq %%mm3,%0" : "=m" (p[d+8]));
-		asm volatile("movntq %%mm4,%0" : "=m" (q[d]));
-		asm volatile("movntq %%mm6,%0" : "=m" (q[d+8]));
-	}
-
-	asm volatile("sfence" : :: "memory");
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_sse1x2 = {
-	raid6_sse12_gen_syndrome,
-	NULL,			/* XOR not yet implemented */
-	raid6_have_sse1_or_mmxext,
-	"sse1x2",
-	1			/* Has cache hints */
-};
-
-#endif
diff --git a/lib/raid6/sse2.c b/lib/raid6/sse2.c
deleted file mode 100644
index 2930220249c90..0000000000000
--- a/lib/raid6/sse2.c
+++ /dev/null
@@ -1,480 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright 2002 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * raid6/sse2.c
- *
- * SSE-2 implementation of RAID-6 syndrome functions
- *
- */
-
-#include <linux/raid/pq.h>
-#include "x86.h"
-
-static const struct raid6_sse_constants {
-	u64 x1d[2];
-} raid6_sse_constants  __attribute__((aligned(16))) = {
-	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
-};
-
-static int raid6_have_sse2(void)
-{
-	/* Not really boot_cpu but "all_cpus" */
-	return boot_cpu_has(X86_FEATURE_MMX) &&
-		boot_cpu_has(X86_FEATURE_FXSR) &&
-		boot_cpu_has(X86_FEATURE_XMM) &&
-		boot_cpu_has(X86_FEATURE_XMM2);
-}
-
-/*
- * Plain SSE2 implementation
- */
-static void raid6_sse21_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
-	asm volatile("pxor %xmm5,%xmm5");	/* Zero temp */
-
-	for ( d = 0 ; d < bytes ; d += 16 ) {
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
-		asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0-1][d]));
-		asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
-		asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z0-1][d]));
-		for ( z = z0-2 ; z >= 0 ; z-- ) {
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
-			asm volatile("pcmpgtb %xmm4,%xmm5");
-			asm volatile("paddb %xmm4,%xmm4");
-			asm volatile("pand %xmm0,%xmm5");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm5,%xmm5");
-			asm volatile("pxor %xmm6,%xmm2");
-			asm volatile("pxor %xmm6,%xmm4");
-			asm volatile("movdqa %0,%%xmm6" : : "m" (dptr[z][d]));
-		}
-		asm volatile("pcmpgtb %xmm4,%xmm5");
-		asm volatile("paddb %xmm4,%xmm4");
-		asm volatile("pand %xmm0,%xmm5");
-		asm volatile("pxor %xmm5,%xmm4");
-		asm volatile("pxor %xmm5,%xmm5");
-		asm volatile("pxor %xmm6,%xmm2");
-		asm volatile("pxor %xmm6,%xmm4");
-
-		asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
-		asm volatile("pxor %xmm2,%xmm2");
-		asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
-		asm volatile("pxor %xmm4,%xmm4");
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-
-static void raid6_sse21_xor_syndrome(int disks, int start, int stop,
-				     size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
-
-	for ( d = 0 ; d < bytes ; d += 16 ) {
-		asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
-		asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
-		asm volatile("pxor %xmm4,%xmm2");
-		/* P/Q data pages */
-		for ( z = z0-1 ; z >= start ; z-- ) {
-			asm volatile("pxor %xmm5,%xmm5");
-			asm volatile("pcmpgtb %xmm4,%xmm5");
-			asm volatile("paddb %xmm4,%xmm4");
-			asm volatile("pand %xmm0,%xmm5");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
-			asm volatile("pxor %xmm5,%xmm2");
-			asm volatile("pxor %xmm5,%xmm4");
-		}
-		/* P/Q left side optimization */
-		for ( z = start-1 ; z >= 0 ; z-- ) {
-			asm volatile("pxor %xmm5,%xmm5");
-			asm volatile("pcmpgtb %xmm4,%xmm5");
-			asm volatile("paddb %xmm4,%xmm4");
-			asm volatile("pand %xmm0,%xmm5");
-			asm volatile("pxor %xmm5,%xmm4");
-		}
-		asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
-		/* Don't use movntdq for r/w memory area < cache line */
-		asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
-		asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_sse2x1 = {
-	raid6_sse21_gen_syndrome,
-	raid6_sse21_xor_syndrome,
-	raid6_have_sse2,
-	"sse2x1",
-	1			/* Has cache hints */
-};
-
-/*
- * Unrolled-by-2 SSE2 implementation
- */
-static void raid6_sse22_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
-	asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
-	asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
-
-	/* We uniformly assume a single prefetch covers at least 32 bytes */
-	for ( d = 0 ; d < bytes ; d += 32 ) {
-		asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
-		asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d]));    /* P[0] */
-		asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16])); /* P[1] */
-		asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
-		asm volatile("movdqa %xmm3,%xmm6"); /* Q[1] */
-		for ( z = z0-1 ; z >= 0 ; z-- ) {
-			asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
-			asm volatile("pcmpgtb %xmm4,%xmm5");
-			asm volatile("pcmpgtb %xmm6,%xmm7");
-			asm volatile("paddb %xmm4,%xmm4");
-			asm volatile("paddb %xmm6,%xmm6");
-			asm volatile("pand %xmm0,%xmm5");
-			asm volatile("pand %xmm0,%xmm7");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-			asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
-			asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
-			asm volatile("pxor %xmm5,%xmm2");
-			asm volatile("pxor %xmm7,%xmm3");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-			asm volatile("pxor %xmm5,%xmm5");
-			asm volatile("pxor %xmm7,%xmm7");
-		}
-		asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
-		asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
-		asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
-		asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-static void raid6_sse22_xor_syndrome(int disks, int start, int stop,
-				     size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
-
-	for ( d = 0 ; d < bytes ; d += 32 ) {
-		asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
-		asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
-		asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
-		asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
-		asm volatile("pxor %xmm4,%xmm2");
-		asm volatile("pxor %xmm6,%xmm3");
-		/* P/Q data pages */
-		for ( z = z0-1 ; z >= start ; z-- ) {
-			asm volatile("pxor %xmm5,%xmm5");
-			asm volatile("pxor %xmm7,%xmm7");
-			asm volatile("pcmpgtb %xmm4,%xmm5");
-			asm volatile("pcmpgtb %xmm6,%xmm7");
-			asm volatile("paddb %xmm4,%xmm4");
-			asm volatile("paddb %xmm6,%xmm6");
-			asm volatile("pand %xmm0,%xmm5");
-			asm volatile("pand %xmm0,%xmm7");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-			asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
-			asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
-			asm volatile("pxor %xmm5,%xmm2");
-			asm volatile("pxor %xmm7,%xmm3");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-		}
-		/* P/Q left side optimization */
-		for ( z = start-1 ; z >= 0 ; z-- ) {
-			asm volatile("pxor %xmm5,%xmm5");
-			asm volatile("pxor %xmm7,%xmm7");
-			asm volatile("pcmpgtb %xmm4,%xmm5");
-			asm volatile("pcmpgtb %xmm6,%xmm7");
-			asm volatile("paddb %xmm4,%xmm4");
-			asm volatile("paddb %xmm6,%xmm6");
-			asm volatile("pand %xmm0,%xmm5");
-			asm volatile("pand %xmm0,%xmm7");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-		}
-		asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
-		asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
-		/* Don't use movntdq for r/w memory area < cache line */
-		asm volatile("movdqa %%xmm4,%0" : "=m" (q[d]));
-		asm volatile("movdqa %%xmm6,%0" : "=m" (q[d+16]));
-		asm volatile("movdqa %%xmm2,%0" : "=m" (p[d]));
-		asm volatile("movdqa %%xmm3,%0" : "=m" (p[d+16]));
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-const struct raid6_calls raid6_sse2x2 = {
-	raid6_sse22_gen_syndrome,
-	raid6_sse22_xor_syndrome,
-	raid6_have_sse2,
-	"sse2x2",
-	1			/* Has cache hints */
-};
-
-#ifdef CONFIG_X86_64
-
-/*
- * Unrolled-by-4 SSE2 implementation
- */
-static void raid6_sse24_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
-	asm volatile("pxor %xmm2,%xmm2");	/* P[0] */
-	asm volatile("pxor %xmm3,%xmm3");	/* P[1] */
-	asm volatile("pxor %xmm4,%xmm4"); 	/* Q[0] */
-	asm volatile("pxor %xmm5,%xmm5");	/* Zero temp */
-	asm volatile("pxor %xmm6,%xmm6"); 	/* Q[1] */
-	asm volatile("pxor %xmm7,%xmm7"); 	/* Zero temp */
-	asm volatile("pxor %xmm10,%xmm10");	/* P[2] */
-	asm volatile("pxor %xmm11,%xmm11");	/* P[3] */
-	asm volatile("pxor %xmm12,%xmm12"); 	/* Q[2] */
-	asm volatile("pxor %xmm13,%xmm13");	/* Zero temp */
-	asm volatile("pxor %xmm14,%xmm14"); 	/* Q[3] */
-	asm volatile("pxor %xmm15,%xmm15"); 	/* Zero temp */
-
-	for ( d = 0 ; d < bytes ; d += 64 ) {
-		for ( z = z0 ; z >= 0 ; z-- ) {
-			/* The second prefetch seems to improve performance... */
-			asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
-			asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
-			asm volatile("pcmpgtb %xmm4,%xmm5");
-			asm volatile("pcmpgtb %xmm6,%xmm7");
-			asm volatile("pcmpgtb %xmm12,%xmm13");
-			asm volatile("pcmpgtb %xmm14,%xmm15");
-			asm volatile("paddb %xmm4,%xmm4");
-			asm volatile("paddb %xmm6,%xmm6");
-			asm volatile("paddb %xmm12,%xmm12");
-			asm volatile("paddb %xmm14,%xmm14");
-			asm volatile("pand %xmm0,%xmm5");
-			asm volatile("pand %xmm0,%xmm7");
-			asm volatile("pand %xmm0,%xmm13");
-			asm volatile("pand %xmm0,%xmm15");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-			asm volatile("pxor %xmm13,%xmm12");
-			asm volatile("pxor %xmm15,%xmm14");
-			asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
-			asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
-			asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
-			asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
-			asm volatile("pxor %xmm5,%xmm2");
-			asm volatile("pxor %xmm7,%xmm3");
-			asm volatile("pxor %xmm13,%xmm10");
-			asm volatile("pxor %xmm15,%xmm11");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-			asm volatile("pxor %xmm13,%xmm12");
-			asm volatile("pxor %xmm15,%xmm14");
-			asm volatile("pxor %xmm5,%xmm5");
-			asm volatile("pxor %xmm7,%xmm7");
-			asm volatile("pxor %xmm13,%xmm13");
-			asm volatile("pxor %xmm15,%xmm15");
-		}
-		asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
-		asm volatile("pxor %xmm2,%xmm2");
-		asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
-		asm volatile("pxor %xmm3,%xmm3");
-		asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
-		asm volatile("pxor %xmm10,%xmm10");
-		asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
-		asm volatile("pxor %xmm11,%xmm11");
-		asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
-		asm volatile("pxor %xmm4,%xmm4");
-		asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
-		asm volatile("pxor %xmm6,%xmm6");
-		asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
-		asm volatile("pxor %xmm12,%xmm12");
-		asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
-		asm volatile("pxor %xmm14,%xmm14");
-	}
-
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-static void raid6_sse24_xor_syndrome(int disks, int start, int stop,
-				     size_t bytes, void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-
-	z0 = stop;		/* P/Q right side optimization */
-	p = dptr[disks-2];	/* XOR parity */
-	q = dptr[disks-1];	/* RS syndrome */
-
-	kernel_fpu_begin();
-
-	asm volatile("movdqa %0,%%xmm0" :: "m" (raid6_sse_constants.x1d[0]));
-
-	for ( d = 0 ; d < bytes ; d += 64 ) {
-		asm volatile("movdqa %0,%%xmm4" :: "m" (dptr[z0][d]));
-		asm volatile("movdqa %0,%%xmm6" :: "m" (dptr[z0][d+16]));
-		asm volatile("movdqa %0,%%xmm12" :: "m" (dptr[z0][d+32]));
-		asm volatile("movdqa %0,%%xmm14" :: "m" (dptr[z0][d+48]));
-		asm volatile("movdqa %0,%%xmm2" : : "m" (p[d]));
-		asm volatile("movdqa %0,%%xmm3" : : "m" (p[d+16]));
-		asm volatile("movdqa %0,%%xmm10" : : "m" (p[d+32]));
-		asm volatile("movdqa %0,%%xmm11" : : "m" (p[d+48]));
-		asm volatile("pxor %xmm4,%xmm2");
-		asm volatile("pxor %xmm6,%xmm3");
-		asm volatile("pxor %xmm12,%xmm10");
-		asm volatile("pxor %xmm14,%xmm11");
-		/* P/Q data pages */
-		for ( z = z0-1 ; z >= start ; z-- ) {
-			asm volatile("prefetchnta %0" :: "m" (dptr[z][d]));
-			asm volatile("prefetchnta %0" :: "m" (dptr[z][d+32]));
-			asm volatile("pxor %xmm5,%xmm5");
-			asm volatile("pxor %xmm7,%xmm7");
-			asm volatile("pxor %xmm13,%xmm13");
-			asm volatile("pxor %xmm15,%xmm15");
-			asm volatile("pcmpgtb %xmm4,%xmm5");
-			asm volatile("pcmpgtb %xmm6,%xmm7");
-			asm volatile("pcmpgtb %xmm12,%xmm13");
-			asm volatile("pcmpgtb %xmm14,%xmm15");
-			asm volatile("paddb %xmm4,%xmm4");
-			asm volatile("paddb %xmm6,%xmm6");
-			asm volatile("paddb %xmm12,%xmm12");
-			asm volatile("paddb %xmm14,%xmm14");
-			asm volatile("pand %xmm0,%xmm5");
-			asm volatile("pand %xmm0,%xmm7");
-			asm volatile("pand %xmm0,%xmm13");
-			asm volatile("pand %xmm0,%xmm15");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-			asm volatile("pxor %xmm13,%xmm12");
-			asm volatile("pxor %xmm15,%xmm14");
-			asm volatile("movdqa %0,%%xmm5" :: "m" (dptr[z][d]));
-			asm volatile("movdqa %0,%%xmm7" :: "m" (dptr[z][d+16]));
-			asm volatile("movdqa %0,%%xmm13" :: "m" (dptr[z][d+32]));
-			asm volatile("movdqa %0,%%xmm15" :: "m" (dptr[z][d+48]));
-			asm volatile("pxor %xmm5,%xmm2");
-			asm volatile("pxor %xmm7,%xmm3");
-			asm volatile("pxor %xmm13,%xmm10");
-			asm volatile("pxor %xmm15,%xmm11");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-			asm volatile("pxor %xmm13,%xmm12");
-			asm volatile("pxor %xmm15,%xmm14");
-		}
-		asm volatile("prefetchnta %0" :: "m" (q[d]));
-		asm volatile("prefetchnta %0" :: "m" (q[d+32]));
-		/* P/Q left side optimization */
-		for ( z = start-1 ; z >= 0 ; z-- ) {
-			asm volatile("pxor %xmm5,%xmm5");
-			asm volatile("pxor %xmm7,%xmm7");
-			asm volatile("pxor %xmm13,%xmm13");
-			asm volatile("pxor %xmm15,%xmm15");
-			asm volatile("pcmpgtb %xmm4,%xmm5");
-			asm volatile("pcmpgtb %xmm6,%xmm7");
-			asm volatile("pcmpgtb %xmm12,%xmm13");
-			asm volatile("pcmpgtb %xmm14,%xmm15");
-			asm volatile("paddb %xmm4,%xmm4");
-			asm volatile("paddb %xmm6,%xmm6");
-			asm volatile("paddb %xmm12,%xmm12");
-			asm volatile("paddb %xmm14,%xmm14");
-			asm volatile("pand %xmm0,%xmm5");
-			asm volatile("pand %xmm0,%xmm7");
-			asm volatile("pand %xmm0,%xmm13");
-			asm volatile("pand %xmm0,%xmm15");
-			asm volatile("pxor %xmm5,%xmm4");
-			asm volatile("pxor %xmm7,%xmm6");
-			asm volatile("pxor %xmm13,%xmm12");
-			asm volatile("pxor %xmm15,%xmm14");
-		}
-		asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
-		asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
-		asm volatile("movntdq %%xmm10,%0" : "=m" (p[d+32]));
-		asm volatile("movntdq %%xmm11,%0" : "=m" (p[d+48]));
-		asm volatile("pxor %0,%%xmm4" : : "m" (q[d]));
-		asm volatile("pxor %0,%%xmm6" : : "m" (q[d+16]));
-		asm volatile("pxor %0,%%xmm12" : : "m" (q[d+32]));
-		asm volatile("pxor %0,%%xmm14" : : "m" (q[d+48]));
-		asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
-		asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
-		asm volatile("movntdq %%xmm12,%0" : "=m" (q[d+32]));
-		asm volatile("movntdq %%xmm14,%0" : "=m" (q[d+48]));
-	}
-	asm volatile("sfence" : : : "memory");
-	kernel_fpu_end();
-}
-
-
-const struct raid6_calls raid6_sse2x4 = {
-	raid6_sse24_gen_syndrome,
-	raid6_sse24_xor_syndrome,
-	raid6_have_sse2,
-	"sse2x4",
-	1			/* Has cache hints */
-};
-
-#endif /* CONFIG_X86_64 */
diff --git a/lib/raid6/test/.gitignore b/lib/raid6/test/.gitignore
deleted file mode 100644
index 1b68a77f348f6..0000000000000
--- a/lib/raid6/test/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-/int.uc
-/neon.uc
-/raid6test
diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
deleted file mode 100644
index 09bbe2b14cceb..0000000000000
--- a/lib/raid6/test/Makefile
+++ /dev/null
@@ -1,156 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-#
-# This is a simple Makefile to test some of the RAID-6 code
-# from userspace.
-#
-
-pound := \#
-
-# Adjust as desired
-CC       = gcc
-OPTFLAGS = -O2
-CFLAGS   = -I.. -I ../../../include -g $(OPTFLAGS)
-LD       = ld
-AWK      = awk -f
-AR       = ar
-RANLIB   = ranlib
-OBJS     = int1.o int2.o int4.o int8.o int16.o int32.o recov.o algos.o tables.o
-
-ARCH := $(shell uname -m 2>/dev/null | sed -e /s/i.86/i386/)
-ifeq ($(ARCH),i386)
-        CFLAGS += -DCONFIG_X86_32
-        IS_X86 = yes
-endif
-ifeq ($(ARCH),x86_64)
-        CFLAGS += -DCONFIG_X86_64
-        IS_X86 = yes
-endif
-
-ifeq ($(ARCH),arm)
-        CFLAGS += -I../../../arch/arm/include -mfpu=neon
-        HAS_NEON = yes
-endif
-ifeq ($(ARCH),aarch64)
-        CFLAGS += -I../../../arch/arm64/include
-        HAS_NEON = yes
-endif
-
-ifeq ($(findstring riscv,$(ARCH)),riscv)
-        CFLAGS += -I../../../arch/riscv/include -DCONFIG_RISCV=1
-        HAS_RVV = yes
-endif
-
-ifeq ($(findstring ppc,$(ARCH)),ppc)
-        CFLAGS += -I../../../arch/powerpc/include
-        HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\
-                         gcc -c -x c - >/dev/null && rm ./-.o && echo yes)
-endif
-
-ifeq ($(ARCH),loongarch64)
-        CFLAGS += -I../../../arch/loongarch/include -DCONFIG_LOONGARCH=1
-        CFLAGS += $(shell echo 'vld $$vr0, $$zero, 0' |         \
-                    gcc -c -x assembler - >/dev/null 2>&1 &&    \
-                    rm ./-.o && echo -DCONFIG_CPU_HAS_LSX=1)
-        CFLAGS += $(shell echo 'xvld $$xr0, $$zero, 0' |        \
-                    gcc -c -x assembler - >/dev/null 2>&1 &&    \
-                    rm ./-.o && echo -DCONFIG_CPU_HAS_LASX=1)
-endif
-
-ifeq ($(IS_X86),yes)
-        OBJS   += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o
-        CFLAGS += -DCONFIG_X86
-else ifeq ($(HAS_NEON),yes)
-        OBJS   += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
-        CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1
-else ifeq ($(HAS_ALTIVEC),yes)
-        CFLAGS += -DCONFIG_ALTIVEC
-        OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \
-                vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o
-else ifeq ($(ARCH),loongarch64)
-        OBJS += loongarch_simd.o recov_loongarch_simd.o
-else ifeq ($(HAS_RVV),yes)
-        OBJS   += rvv.o recov_rvv.o
-        CFLAGS += -DCONFIG_RISCV_ISA_V=1
-endif
-
-.c.o:
-	$(CC) $(CFLAGS) -c -o $@ $<
-
-%.c: ../%.c
-	cp -f $< $@
-
-%.uc: ../%.uc
-	cp -f $< $@
-
-all: raid6.a raid6test
-
-raid6.a: $(OBJS)
-	rm -f $@
-	$(AR) cq $@ $^
-	$(RANLIB) $@
-
-raid6test: test.c raid6.a
-	$(CC) $(CFLAGS) -o raid6test $^
-
-neon1.c: neon.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=1 < neon.uc > $@
-
-neon2.c: neon.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=2 < neon.uc > $@
-
-neon4.c: neon.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=4 < neon.uc > $@
-
-neon8.c: neon.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=8 < neon.uc > $@
-
-altivec1.c: altivec.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=1 < altivec.uc > $@
-
-altivec2.c: altivec.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=2 < altivec.uc > $@
-
-altivec4.c: altivec.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=4 < altivec.uc > $@
-
-altivec8.c: altivec.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=8 < altivec.uc > $@
-
-vpermxor1.c: vpermxor.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=1 < vpermxor.uc > $@
-
-vpermxor2.c: vpermxor.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=2 < vpermxor.uc > $@
-
-vpermxor4.c: vpermxor.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=4 < vpermxor.uc > $@
-
-vpermxor8.c: vpermxor.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=8 < vpermxor.uc > $@
-
-int1.c: int.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=1 < int.uc > $@
-
-int2.c: int.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=2 < int.uc > $@
-
-int4.c: int.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=4 < int.uc > $@
-
-int8.c: int.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=8 < int.uc > $@
-
-int16.c: int.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=16 < int.uc > $@
-
-int32.c: int.uc ../unroll.awk
-	$(AWK) ../unroll.awk -vN=32 < int.uc > $@
-
-tables.c: mktables
-	./mktables > tables.c
-
-clean:
-	rm -f *.o *.a mktables mktables.c *.uc int*.c altivec*.c vpermxor*.c neon*.c tables.c raid6test
-
-spotless: clean
-	rm -f *~
diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c
deleted file mode 100644
index 841a55242abaa..0000000000000
--- a/lib/raid6/test/test.c
+++ /dev/null
@@ -1,152 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- *   Copyright 2002-2007 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * raid6test.c
- *
- * Test RAID-6 recovery with various algorithms
- */
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
-#include <linux/raid/pq.h>
-
-#define NDISKS		16	/* Including P and Q */
-
-const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
-
-char *dataptrs[NDISKS];
-char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
-char recovi[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
-char recovj[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
-
-static void makedata(int start, int stop)
-{
-	int i, j;
-
-	for (i = start; i <= stop; i++) {
-		for (j = 0; j < PAGE_SIZE; j++)
-			data[i][j] = rand();
-
-		dataptrs[i] = data[i];
-	}
-}
-
-static char disk_type(int d)
-{
-	switch (d) {
-	case NDISKS-2:
-		return 'P';
-	case NDISKS-1:
-		return 'Q';
-	default:
-		return 'D';
-	}
-}
-
-static int test_disks(int i, int j)
-{
-	int erra, errb;
-
-	memset(recovi, 0xf0, PAGE_SIZE);
-	memset(recovj, 0xba, PAGE_SIZE);
-
-	dataptrs[i] = recovi;
-	dataptrs[j] = recovj;
-
-	raid6_dual_recov(NDISKS, PAGE_SIZE, i, j, (void **)&dataptrs);
-
-	erra = memcmp(data[i], recovi, PAGE_SIZE);
-	errb = memcmp(data[j], recovj, PAGE_SIZE);
-
-	if (i < NDISKS-2 && j == NDISKS-1) {
-		/* We don't implement the DQ failure scenario, since it's
-		   equivalent to a RAID-5 failure (XOR, then recompute Q) */
-		erra = errb = 0;
-	} else {
-		printf("algo=%-8s  faila=%3d(%c)  failb=%3d(%c)  %s\n",
-		       raid6_call.name,
-		       i, disk_type(i),
-		       j, disk_type(j),
-		       (!erra && !errb) ? "OK" :
-		       !erra ? "ERRB" :
-		       !errb ? "ERRA" : "ERRAB");
-	}
-
-	dataptrs[i] = data[i];
-	dataptrs[j] = data[j];
-
-	return erra || errb;
-}
-
-int main(int argc, char *argv[])
-{
-	const struct raid6_calls *const *algo;
-	const struct raid6_recov_calls *const *ra;
-	int i, j, p1, p2;
-	int err = 0;
-
-	makedata(0, NDISKS-1);
-
-	for (ra = raid6_recov_algos; *ra; ra++) {
-		if ((*ra)->valid  && !(*ra)->valid())
-			continue;
-
-		raid6_2data_recov = (*ra)->data2;
-		raid6_datap_recov = (*ra)->datap;
-
-		printf("using recovery %s\n", (*ra)->name);
-
-		for (algo = raid6_algos; *algo; algo++) {
-			if ((*algo)->valid && !(*algo)->valid())
-				continue;
-
-			raid6_call = **algo;
-
-			/* Nuke syndromes */
-			memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE);
-
-			/* Generate assumed good syndrome */
-			raid6_call.gen_syndrome(NDISKS, PAGE_SIZE,
-						(void **)&dataptrs);
-
-			for (i = 0; i < NDISKS-1; i++)
-				for (j = i+1; j < NDISKS; j++)
-					err += test_disks(i, j);
-
-			if (!raid6_call.xor_syndrome)
-				continue;
-
-			for (p1 = 0; p1 < NDISKS-2; p1++)
-				for (p2 = p1; p2 < NDISKS-2; p2++) {
-
-					/* Simulate rmw run */
-					raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE,
-								(void **)&dataptrs);
-					makedata(p1, p2);
-					raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE,
-                                                                (void **)&dataptrs);
-
-					for (i = 0; i < NDISKS-1; i++)
-						for (j = i+1; j < NDISKS; j++)
-							err += test_disks(i, j);
-				}
-
-		}
-		printf("\n");
-	}
-
-	printf("\n");
-	/* Pick the best algorithm test */
-	raid6_select_algo();
-
-	if (err)
-		printf("\n*** ERRORS FOUND ***\n");
-
-	return err;
-}
diff --git a/lib/raid6/unroll.awk b/lib/raid6/unroll.awk
deleted file mode 100644
index 0809805a7e231..0000000000000
--- a/lib/raid6/unroll.awk
+++ /dev/null
@@ -1,20 +0,0 @@
-
-# This filter requires one command line option of form -vN=n
-# where n must be a decimal number.
-#
-# Repeat each input line containing $$ n times, replacing $$ with 0...n-1.
-# Replace each $# with n, and each $* with a single $.
-
-BEGIN {
-	n = N + 0
-}
-{
-	if (/\$\$/) { rep = n } else { rep = 1 }
-	for (i = 0; i < rep; ++i) {
-		tmp = $0
-		gsub(/\$\$/, i, tmp)
-		gsub(/\$#/, n, tmp)
-		gsub(/\$\*/, "$", tmp)
-		print tmp
-	}
-}
diff --git a/lib/raid6/vpermxor.uc b/lib/raid6/vpermxor.uc
deleted file mode 100644
index 1bfb127fbfe81..0000000000000
--- a/lib/raid6/vpermxor.uc
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2017, Matt Brown, IBM Corp.
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- *
- * vpermxor$#.c
- *
- * Based on H. Peter Anvin's paper - The mathematics of RAID-6
- *
- * $#-way unrolled portable integer math RAID-6 instruction set
- * This file is postprocessed using unroll.awk
- *
- * vpermxor$#.c makes use of the vpermxor instruction to optimise the RAID6 Q
- * syndrome calculations.
- * This can be run on systems which have both Altivec and vpermxor instruction.
- *
- * This instruction was introduced in POWER8 - ISA v2.07.
- */
-
-#include <linux/raid/pq.h>
-#ifdef CONFIG_ALTIVEC
-
-#include <altivec.h>
-#include <asm/ppc-opcode.h>
-#ifdef __KERNEL__
-#include <asm/cputable.h>
-#include <asm/switch_to.h>
-#endif
-
-typedef vector unsigned char unative_t;
-#define NSIZE sizeof(unative_t)
-
-static const vector unsigned char gf_low = {0x1e, 0x1c, 0x1a, 0x18, 0x16, 0x14,
-					    0x12, 0x10, 0x0e, 0x0c, 0x0a, 0x08,
-					    0x06, 0x04, 0x02,0x00};
-static const vector unsigned char gf_high = {0xfd, 0xdd, 0xbd, 0x9d, 0x7d, 0x5d,
-					     0x3d, 0x1d, 0xe0, 0xc0, 0xa0, 0x80,
-					     0x60, 0x40, 0x20, 0x00};
-
-static void noinline raid6_vpermxor$#_gen_syndrome_real(int disks, size_t bytes,
-							void **ptrs)
-{
-	u8 **dptr = (u8 **)ptrs;
-	u8 *p, *q;
-	int d, z, z0;
-	unative_t wp$$, wq$$, wd$$;
-
-	z0 = disks - 3;		/* Highest data disk */
-	p = dptr[z0+1];		/* XOR parity */
-	q = dptr[z0+2];		/* RS syndrome */
-
-	for (d = 0; d < bytes; d += NSIZE*$#) {
-		wp$$ = wq$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE];
-
-		for (z = z0-1; z>=0; z--) {
-			wd$$ = *(unative_t *)&dptr[z][d+$$*NSIZE];
-			/* P syndrome */
-			wp$$ = vec_xor(wp$$, wd$$);
-
-			/* Q syndrome */
-			asm(VPERMXOR(%0,%1,%2,%3):"=v"(wq$$):"v"(gf_high), "v"(gf_low), "v"(wq$$));
-			wq$$ = vec_xor(wq$$, wd$$);
-		}
-		*(unative_t *)&p[d+NSIZE*$$] = wp$$;
-		*(unative_t *)&q[d+NSIZE*$$] = wq$$;
-	}
-}
-
-static void raid6_vpermxor$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
-{
-	preempt_disable();
-	enable_kernel_altivec();
-
-	raid6_vpermxor$#_gen_syndrome_real(disks, bytes, ptrs);
-
-	disable_kernel_altivec();
-	preempt_enable();
-}
-
-int raid6_have_altivec_vpermxor(void);
-#if $# == 1
-int raid6_have_altivec_vpermxor(void)
-{
-	/* Check if arch has both altivec and the vpermxor instructions */
-# ifdef __KERNEL__
-	return (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) &&
-		cpu_has_feature(CPU_FTR_ARCH_207S));
-# else
-	return 1;
-#endif
-
-}
-#endif
-
-const struct raid6_calls raid6_vpermxor$# = {
-	raid6_vpermxor$#_gen_syndrome,
-	NULL,
-	raid6_have_altivec_vpermxor,
-	"vpermxor$#",
-	0
-};
-#endif
diff --git a/lib/raid6/x86.h b/lib/raid6/x86.h
deleted file mode 100644
index 9a6ff37115e71..0000000000000
--- a/lib/raid6/x86.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* ----------------------------------------------------------------------- *
- *
- *   Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * raid6/x86.h
- *
- * Definitions common to x86 and x86-64 RAID-6 code only
- */
-
-#ifndef LINUX_RAID_RAID6X86_H
-#define LINUX_RAID_RAID6X86_H
-
-#if (defined(__i386__) || defined(__x86_64__)) && !defined(__arch_um__)
-
-#ifdef __KERNEL__ /* Real code */
-
-#include <asm/fpu/api.h>
-
-#else /* Dummy code for user space testing */
-
-static inline void kernel_fpu_begin(void)
-{
-}
-
-static inline void kernel_fpu_end(void)
-{
-}
-
-#define __aligned(x) __attribute__((aligned(x)))
-
-#define X86_FEATURE_MMX		(0*32+23) /* Multimedia Extensions */
-#define X86_FEATURE_FXSR	(0*32+24) /* FXSAVE and FXRSTOR instructions
-					   * (fast save and restore) */
-#define X86_FEATURE_XMM		(0*32+25) /* Streaming SIMD Extensions */
-#define X86_FEATURE_XMM2	(0*32+26) /* Streaming SIMD Extensions-2 */
-#define X86_FEATURE_XMM3	(4*32+ 0) /* "pni" SSE-3 */
-#define X86_FEATURE_SSSE3	(4*32+ 9) /* Supplemental SSE-3 */
-#define X86_FEATURE_AVX	(4*32+28) /* Advanced Vector Extensions */
-#define X86_FEATURE_AVX2        (9*32+ 5) /* AVX2 instructions */
-#define X86_FEATURE_AVX512F     (9*32+16) /* AVX-512 Foundation */
-#define X86_FEATURE_AVX512DQ    (9*32+17) /* AVX-512 DQ (Double/Quad granular)
-					   * Instructions
-					   */
-#define X86_FEATURE_AVX512BW    (9*32+30) /* AVX-512 BW (Byte/Word granular)
-					   * Instructions
-					   */
-#define X86_FEATURE_AVX512VL    (9*32+31) /* AVX-512 VL (128/256 Vector Length)
-					   * Extensions
-					   */
-#define X86_FEATURE_MMXEXT	(1*32+22) /* AMD MMX extensions */
-
-/* Should work well enough on modern CPUs for testing */
-static inline int boot_cpu_has(int flag)
-{
-	u32 eax, ebx, ecx, edx;
-
-	eax = (flag & 0x100) ? 7 :
-		(flag & 0x20) ? 0x80000001 : 1;
-	ecx = 0;
-
-	asm volatile("cpuid"
-		     : "+a" (eax), "=b" (ebx), "=d" (edx), "+c" (ecx));
-
-	return ((flag & 0x100 ? ebx :
-		(flag & 0x80) ? ecx : edx) >> (flag & 31)) & 1;
-}
-
-#endif /* ndef __KERNEL__ */
-
-#endif
-#endif
diff --git a/lib/rhashtable.c b/lib/rhashtable.c
index 04b3a808fca9f..c0ba34eadb397 100644
--- a/lib/rhashtable.c
+++ b/lib/rhashtable.c
@@ -1057,8 +1057,9 @@ static u32 rhashtable_jhash2(const void *key, u32 length, u32 seed)
  *	.obj_hashfn = my_hash_fn,
  * };
  */
-int rhashtable_init_noprof(struct rhashtable *ht,
-		    const struct rhashtable_params *params)
+int __rhashtable_init_noprof(struct rhashtable *ht,
+		    const struct rhashtable_params *params,
+		    struct lock_class_key *key)
 {
 	struct bucket_table *tbl;
 	size_t size;
@@ -1068,7 +1069,7 @@ int rhashtable_init_noprof(struct rhashtable *ht,
 		return -EINVAL;
 
 	memset(ht, 0, sizeof(*ht));
-	mutex_init(&ht->mutex);
+	mutex_init_with_key(&ht->mutex, key);
 	spin_lock_init(&ht->lock);
 	memcpy(&ht->p, params, sizeof(*params));
 
@@ -1120,7 +1121,7 @@ int rhashtable_init_noprof(struct rhashtable *ht,
 
 	return 0;
 }
-EXPORT_SYMBOL_GPL(rhashtable_init_noprof);
+EXPORT_SYMBOL_GPL(__rhashtable_init_noprof);
 
 /**
  * rhltable_init - initialize a new hash list table
@@ -1131,15 +1132,17 @@ EXPORT_SYMBOL_GPL(rhashtable_init_noprof);
  *
  * See documentation for rhashtable_init.
  */
-int rhltable_init_noprof(struct rhltable *hlt, const struct rhashtable_params *params)
+int __rhltable_init_noprof(struct rhltable *hlt,
+			   const struct rhashtable_params *params,
+			   struct lock_class_key *key)
 {
 	int err;
 
-	err = rhashtable_init_noprof(&hlt->ht, params);
+	err = __rhashtable_init_noprof(&hlt->ht, params, key);
 	hlt->ht.rhlist = true;
 	return err;
 }
-EXPORT_SYMBOL_GPL(rhltable_init_noprof);
+EXPORT_SYMBOL_GPL(__rhltable_init_noprof);
 
 static void rhashtable_free_one(struct rhashtable *ht, struct rhash_head *obj,
 				void (*free_fn)(void *ptr, void *arg),
diff --git a/lib/seq_buf.c b/lib/seq_buf.c
index f3f3436d60a94..b59488fa8135c 100644
--- a/lib/seq_buf.c
+++ b/lib/seq_buf.c
@@ -298,6 +298,7 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem,
 	}
 	return 0;
 }
+EXPORT_SYMBOL_GPL(seq_buf_putmem_hex);
 
 /**
  * seq_buf_path - copy a path into the sequence buffer
diff --git a/lib/string.c b/lib/string.c
index b632c71df1a50..1f9297e9776a9 100644
--- a/lib/string.c
+++ b/lib/string.c
@@ -21,6 +21,7 @@
 #include <linux/errno.h>
 #include <linux/limits.h>
 #include <linux/linkage.h>
+#include <linux/minmax.h>
 #include <linux/stddef.h>
 #include <linux/string.h>
 #include <linux/types.h>
@@ -125,11 +126,8 @@ ssize_t sized_strscpy(char *dest, const char *src, size_t count)
 	 * If src is unaligned, don't cross a page boundary,
 	 * since we don't know if the next page is mapped.
 	 */
-	if ((long)src & (sizeof(long) - 1)) {
-		size_t limit = PAGE_SIZE - ((long)src & (PAGE_SIZE - 1));
-		if (limit < max)
-			max = limit;
-	}
+	if ((long)src & (sizeof(long) - 1))
+		max = min(PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)), max);
 #else
 	/* If src or dest is unaligned, don't do word-at-a-time. */
 	if (((long) dest | (long) src) & (sizeof(long) - 1))
diff --git a/lib/string_helpers.c b/lib/string_helpers.c
index 169eaf5834949..98d6ed0eaab7e 100644
--- a/lib/string_helpers.c
+++ b/lib/string_helpers.c
@@ -752,7 +752,7 @@ EXPORT_SYMBOL_GPL(kstrdup_and_replace);
  * kasprintf_strarray - allocate and fill array of sequential strings
  * @gfp: flags for the slab allocator
  * @prefix: prefix to be used
- * @n: amount of lines to be allocated and filled
+ * @n: number of strings to be allocated and filled
  *
  * Allocates and fills @n strings using pattern "%s-%zu", where prefix
  * is provided by caller. The caller is responsible to free them with
@@ -765,7 +765,7 @@ char **kasprintf_strarray(gfp_t gfp, const char *prefix, size_t n)
 	char **names;
 	size_t i;
 
-	names = kcalloc(n + 1, sizeof(char *), gfp);
+	names = kcalloc(n, sizeof(char *), gfp);
 	if (!names)
 		return NULL;
 
@@ -805,7 +805,7 @@ void kfree_strarray(char **array, size_t n)
 EXPORT_SYMBOL_GPL(kfree_strarray);
 
 struct strarray {
-	char **array;
+	char **array __counted_by_ptr(n);
 	size_t n;
 };
 
@@ -824,13 +824,13 @@ char **devm_kasprintf_strarray(struct device *dev, const char *prefix, size_t n)
 	if (!ptr)
 		return ERR_PTR(-ENOMEM);
 
+	ptr->n = n;
 	ptr->array = kasprintf_strarray(GFP_KERNEL, prefix, n);
 	if (!ptr->array) {
 		devres_free(ptr);
 		return ERR_PTR(-ENOMEM);
 	}
 
-	ptr->n = n;
 	devres_add(dev, ptr);
 
 	return ptr->array;
diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c
index ee87fef66cb58..811128d0df16f 100644
--- a/lib/test-kstrtox.c
+++ b/lib/test-kstrtox.c
@@ -198,6 +198,7 @@ static void __init test_kstrtoull_fail(void)
 		{"10000000000000000000000000000000000000000000000000000000000000000",	2},
 		{"2000000000000000000000",	8},
 		{"18446744073709551616",	10},
+		{"569202370375329612767",	10},
 		{"10000000000000000",	16},
 		/* negative */
 		{"-0", 0},
@@ -275,9 +276,11 @@ static void __init test_kstrtoll_fail(void)
 		{"9223372036854775809",	10},
 		{"18446744073709551614",	10},
 		{"18446744073709551615",	10},
+		{"569202370375329612767",	10},
 		{"-9223372036854775809",	10},
 		{"-18446744073709551614",	10},
 		{"-18446744073709551615",	10},
+		{"-569202370375329612767",	10},
 		/* sign is first character if any */
 		{"-+1", 0},
 		{"-+1", 8},
@@ -334,6 +337,7 @@ static void __init test_kstrtou64_fail(void)
 		{"-1",	10},
 		{"18446744073709551616",	10},
 		{"18446744073709551617",	10},
+		{"569202370375329612767",	10},
 	};
 	TEST_FAIL(kstrtou64, u64, "%llu", test_u64_fail);
 }
@@ -386,6 +390,8 @@ static void __init test_kstrtos64_fail(void)
 		{"18446744073709551615",	10},
 		{"18446744073709551616",	10},
 		{"18446744073709551617",	10},
+		{"569202370375329612767",	10},
+		{"-569202370375329612767",	10},
 	};
 	TEST_FAIL(kstrtos64, s64, "%lld", test_s64_fail);
 }
diff --git a/lib/test_bpf.c b/lib/test_bpf.c
index 5892c0f17ddc4..af6f3340c034f 100644
--- a/lib/test_bpf.c
+++ b/lib/test_bpf.c
@@ -560,8 +560,23 @@ static int bpf_fill_max_jmp_never_taken(struct bpf_test *self)
 }
 
 /* ALU result computation used in tests */
-static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op)
+enum { F_ALU32 = 1, F_SIGNED = 2 };
+
+static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op, u32 flags)
 {
+	bool is_signed = flags & F_SIGNED;
+
+	/* Narrow operands for ALU32 */
+	if (flags & F_ALU32) {
+		if (is_signed) {
+			v1 = (u64)(s32)v1;
+			v2 = (u64)(s32)v2;
+		} else {
+			v1 = (u32)v1;
+			v2 = (u32)v2;
+		}
+	}
+
 	*res = 0;
 	switch (op) {
 	case BPF_MOV:
@@ -599,12 +614,28 @@ static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op)
 	case BPF_DIV:
 		if (v2 == 0)
 			return false;
-		*res = div64_u64(v1, v2);
+		if (!is_signed) {
+			*res = div64_u64(v1, v2);
+		} else {
+			if ((s64)v2 == -1) /* Handled by verifier */
+				return false;
+			*res = (u64)div64_s64(v1, v2);
+		}
 		break;
 	case BPF_MOD:
 		if (v2 == 0)
 			return false;
-		div64_u64_rem(v1, v2, res);
+		if (!is_signed) {
+			div64_u64_rem(v1, v2, res);
+		} else {
+			if ((s64)v2 == -1)
+				return false;
+			/*
+			 * Avoid s64 % s64 which generates __moddi3 on
+			 * 32-bit architectures. Use div64_s64 instead.
+			 */
+			*res = (u64)((s64)v1 - div64_s64(v1, v2) * (s64)v2);
+		}
 		break;
 	}
 	return true;
@@ -612,7 +643,7 @@ static bool __bpf_alu_result(u64 *res, u64 v1, u64 v2, u8 op)
 
 /* Test an ALU shift operation for all valid shift values */
 static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op,
-				u8 mode, bool alu32)
+				u8 mode, u32 flags)
 {
 	static const s64 regs[] = {
 		0x0123456789abcdefLL, /* dword > 0, word < 0 */
@@ -620,7 +651,7 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op,
 		0xfedcba0198765432LL, /* dword < 0, word < 0 */
 		0x0123458967abcdefLL, /* dword > 0, word > 0 */
 	};
-	int bits = alu32 ? 32 : 64;
+	int bits = (flags & F_ALU32) ? 32 : 64;
 	int len = (2 + 7 * bits) * ARRAY_SIZE(regs) + 3;
 	struct bpf_insn *insn;
 	int imm, k;
@@ -643,7 +674,7 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op,
 			/* Perform operation */
 			insn[i++] = BPF_ALU64_REG(BPF_MOV, R1, R3);
 			insn[i++] = BPF_ALU64_IMM(BPF_MOV, R2, imm);
-			if (alu32) {
+			if (flags & F_ALU32) {
 				if (mode == BPF_K)
 					insn[i++] = BPF_ALU32_IMM(op, R1, imm);
 				else
@@ -653,14 +684,14 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op,
 					reg = (s32)reg;
 				else
 					reg = (u32)reg;
-				__bpf_alu_result(&val, reg, imm, op);
+				__bpf_alu_result(&val, reg, imm, op, 0);
 				val = (u32)val;
 			} else {
 				if (mode == BPF_K)
 					insn[i++] = BPF_ALU64_IMM(op, R1, imm);
 				else
 					insn[i++] = BPF_ALU64_REG(op, R1, R2);
-				__bpf_alu_result(&val, reg, imm, op);
+				__bpf_alu_result(&val, reg, imm, op, 0);
 			}
 
 			/*
@@ -688,62 +719,62 @@ static int __bpf_fill_alu_shift(struct bpf_test *self, u8 op,
 
 static int bpf_fill_alu64_lsh_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, false);
+	return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, 0);
 }
 
 static int bpf_fill_alu64_rsh_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, false);
+	return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, 0);
 }
 
 static int bpf_fill_alu64_arsh_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, false);
+	return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, 0);
 }
 
 static int bpf_fill_alu64_lsh_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, false);
+	return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, 0);
 }
 
 static int bpf_fill_alu64_rsh_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, false);
+	return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, 0);
 }
 
 static int bpf_fill_alu64_arsh_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, false);
+	return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, 0);
 }
 
 static int bpf_fill_alu32_lsh_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, true);
+	return __bpf_fill_alu_shift(self, BPF_LSH, BPF_K, F_ALU32);
 }
 
 static int bpf_fill_alu32_rsh_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, true);
+	return __bpf_fill_alu_shift(self, BPF_RSH, BPF_K, F_ALU32);
 }
 
 static int bpf_fill_alu32_arsh_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, true);
+	return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_K, F_ALU32);
 }
 
 static int bpf_fill_alu32_lsh_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, true);
+	return __bpf_fill_alu_shift(self, BPF_LSH, BPF_X, F_ALU32);
 }
 
 static int bpf_fill_alu32_rsh_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, true);
+	return __bpf_fill_alu_shift(self, BPF_RSH, BPF_X, F_ALU32);
 }
 
 static int bpf_fill_alu32_arsh_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, true);
+	return __bpf_fill_alu_shift(self, BPF_ARSH, BPF_X, F_ALU32);
 }
 
 /*
@@ -751,9 +782,9 @@ static int bpf_fill_alu32_arsh_reg(struct bpf_test *self)
  * for the case when the source and destination are the same.
  */
 static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op,
-					 bool alu32)
+					 u32 flags)
 {
-	int bits = alu32 ? 32 : 64;
+	int bits = (flags & F_ALU32) ? 32 : 64;
 	int len = 3 + 6 * bits;
 	struct bpf_insn *insn;
 	int i = 0;
@@ -770,14 +801,14 @@ static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op,
 
 		/* Perform operation */
 		insn[i++] = BPF_ALU64_IMM(BPF_MOV, R1, val);
-		if (alu32)
+		if (flags & F_ALU32)
 			insn[i++] = BPF_ALU32_REG(op, R1, R1);
 		else
 			insn[i++] = BPF_ALU64_REG(op, R1, R1);
 
 		/* Compute the reference result */
-		__bpf_alu_result(&res, val, val, op);
-		if (alu32)
+		__bpf_alu_result(&res, val, val, op, 0);
+		if (flags & F_ALU32)
 			res = (u32)res;
 		i += __bpf_ld_imm64(&insn[i], R2, res);
 
@@ -798,32 +829,32 @@ static int __bpf_fill_alu_shift_same_reg(struct bpf_test *self, u8 op,
 
 static int bpf_fill_alu64_lsh_same_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, false);
+	return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, 0);
 }
 
 static int bpf_fill_alu64_rsh_same_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, false);
+	return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, 0);
 }
 
 static int bpf_fill_alu64_arsh_same_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, false);
+	return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, 0);
 }
 
 static int bpf_fill_alu32_lsh_same_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, true);
+	return __bpf_fill_alu_shift_same_reg(self, BPF_LSH, F_ALU32);
 }
 
 static int bpf_fill_alu32_rsh_same_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, true);
+	return __bpf_fill_alu_shift_same_reg(self, BPF_RSH, F_ALU32);
 }
 
 static int bpf_fill_alu32_arsh_same_reg(struct bpf_test *self)
 {
-	return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, true);
+	return __bpf_fill_alu_shift_same_reg(self, BPF_ARSH, F_ALU32);
 }
 
 /*
@@ -936,17 +967,20 @@ static int __bpf_fill_pattern(struct bpf_test *self, void *arg,
 static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg,
 				struct bpf_insn *insns, s64 dst, s64 imm)
 {
-	int op = *(int *)arg;
+	int *a = arg;
+	int op = a[0];
+	u32 flags = a[1];
+	s16 off = (flags & F_SIGNED) ? 1 : 0;
 	int i = 0;
 	u64 res;
 
 	if (!insns)
 		return 7;
 
-	if (__bpf_alu_result(&res, dst, (s32)imm, op)) {
+	if (__bpf_alu_result(&res, dst, (s32)imm, op, flags)) {
 		i += __bpf_ld_imm64(&insns[i], R1, dst);
 		i += __bpf_ld_imm64(&insns[i], R3, res);
-		insns[i++] = BPF_ALU64_IMM(op, R1, imm);
+		insns[i++] = BPF_ALU64_IMM_OFF(op, R1, imm, off);
 		insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1);
 		insns[i++] = BPF_EXIT_INSN();
 	}
@@ -957,17 +991,20 @@ static int __bpf_emit_alu64_imm(struct bpf_test *self, void *arg,
 static int __bpf_emit_alu32_imm(struct bpf_test *self, void *arg,
 				struct bpf_insn *insns, s64 dst, s64 imm)
 {
-	int op = *(int *)arg;
+	int *a = arg;
+	int op = a[0];
+	u32 flags = a[1];
+	s16 off = (flags & F_SIGNED) ? 1 : 0;
 	int i = 0;
 	u64 res;
 
 	if (!insns)
 		return 7;
 
-	if (__bpf_alu_result(&res, (u32)dst, (u32)imm, op)) {
+	if (__bpf_alu_result(&res, dst, (s32)imm, op, flags | F_ALU32)) {
 		i += __bpf_ld_imm64(&insns[i], R1, dst);
 		i += __bpf_ld_imm64(&insns[i], R3, (u32)res);
-		insns[i++] = BPF_ALU32_IMM(op, R1, imm);
+		insns[i++] = BPF_ALU32_IMM_OFF(op, R1, imm, off);
 		insns[i++] = BPF_JMP_REG(BPF_JEQ, R1, R3, 1);
 		insns[i++] = BPF_EXIT_INSN();
 	}
@@ -985,7 +1022,7 @@ static int __bpf_emit_alu64_reg(struct bpf_test *self, void *arg,
 	if (!insns)
 		return 9;
 
-	if (__bpf_alu_result(&res, dst, src, op)) {
+	if (__bpf_alu_result(&res, dst, src, op, 0)) {
 		i += __bpf_ld_imm64(&insns[i], R1, dst);
 		i += __bpf_ld_imm64(&insns[i], R2, src);
 		i += __bpf_ld_imm64(&insns[i], R3, res);
@@ -1007,7 +1044,7 @@ static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg,
 	if (!insns)
 		return 9;
 
-	if (__bpf_alu_result(&res, (u32)dst, (u32)src, op)) {
+	if (__bpf_alu_result(&res, (u32)dst, (u32)src, op, 0)) {
 		i += __bpf_ld_imm64(&insns[i], R1, dst);
 		i += __bpf_ld_imm64(&insns[i], R2, src);
 		i += __bpf_ld_imm64(&insns[i], R3, (u32)res);
@@ -1019,16 +1056,20 @@ static int __bpf_emit_alu32_reg(struct bpf_test *self, void *arg,
 	return i;
 }
 
-static int __bpf_fill_alu64_imm(struct bpf_test *self, int op)
+static int __bpf_fill_alu64_imm(struct bpf_test *self, int op, u32 flags)
 {
-	return __bpf_fill_pattern(self, &op, 64, 32,
+	int arg[2] = {op, flags};
+
+	return __bpf_fill_pattern(self, &arg, 64, 32,
 				  PATTERN_BLOCK1, PATTERN_BLOCK2,
 				  &__bpf_emit_alu64_imm);
 }
 
-static int __bpf_fill_alu32_imm(struct bpf_test *self, int op)
+static int __bpf_fill_alu32_imm(struct bpf_test *self, int op, u32 flags)
 {
-	return __bpf_fill_pattern(self, &op, 64, 32,
+	int arg[2] = {op, flags};
+
+	return __bpf_fill_pattern(self, &arg, 64, 32,
 				  PATTERN_BLOCK1, PATTERN_BLOCK2,
 				  &__bpf_emit_alu32_imm);
 }
@@ -1050,93 +1091,115 @@ static int __bpf_fill_alu32_reg(struct bpf_test *self, int op)
 /* ALU64 immediate operations */
 static int bpf_fill_alu64_mov_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu64_imm(self, BPF_MOV);
+	return __bpf_fill_alu64_imm(self, BPF_MOV, 0);
 }
 
 static int bpf_fill_alu64_and_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu64_imm(self, BPF_AND);
+	return __bpf_fill_alu64_imm(self, BPF_AND, 0);
 }
 
 static int bpf_fill_alu64_or_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu64_imm(self, BPF_OR);
+	return __bpf_fill_alu64_imm(self, BPF_OR, 0);
 }
 
 static int bpf_fill_alu64_xor_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu64_imm(self, BPF_XOR);
+	return __bpf_fill_alu64_imm(self, BPF_XOR, 0);
 }
 
 static int bpf_fill_alu64_add_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu64_imm(self, BPF_ADD);
+	return __bpf_fill_alu64_imm(self, BPF_ADD, 0);
 }
 
 static int bpf_fill_alu64_sub_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu64_imm(self, BPF_SUB);
+	return __bpf_fill_alu64_imm(self, BPF_SUB, 0);
 }
 
 static int bpf_fill_alu64_mul_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu64_imm(self, BPF_MUL);
+	return __bpf_fill_alu64_imm(self, BPF_MUL, 0);
 }
 
 static int bpf_fill_alu64_div_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu64_imm(self, BPF_DIV);
+	return __bpf_fill_alu64_imm(self, BPF_DIV, 0);
 }
 
 static int bpf_fill_alu64_mod_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu64_imm(self, BPF_MOD);
+	return __bpf_fill_alu64_imm(self, BPF_MOD, 0);
+}
+
+/* Signed ALU64 immediate operations */
+static int bpf_fill_alu64_sdiv_imm(struct bpf_test *self)
+{
+	return __bpf_fill_alu64_imm(self, BPF_DIV, F_SIGNED);
+}
+
+static int bpf_fill_alu64_smod_imm(struct bpf_test *self)
+{
+	return __bpf_fill_alu64_imm(self, BPF_MOD, F_SIGNED);
+}
+
+/* Signed ALU32 immediate operations */
+static int bpf_fill_alu32_sdiv_imm(struct bpf_test *self)
+{
+	return __bpf_fill_alu32_imm(self, BPF_DIV, F_SIGNED);
+}
+
+static int bpf_fill_alu32_smod_imm(struct bpf_test *self)
+{
+	return __bpf_fill_alu32_imm(self, BPF_MOD, F_SIGNED);
 }
 
 /* ALU32 immediate operations */
 static int bpf_fill_alu32_mov_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu32_imm(self, BPF_MOV);
+	return __bpf_fill_alu32_imm(self, BPF_MOV, 0);
 }
 
 static int bpf_fill_alu32_and_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu32_imm(self, BPF_AND);
+	return __bpf_fill_alu32_imm(self, BPF_AND, 0);
 }
 
 static int bpf_fill_alu32_or_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu32_imm(self, BPF_OR);
+	return __bpf_fill_alu32_imm(self, BPF_OR, 0);
 }
 
 static int bpf_fill_alu32_xor_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu32_imm(self, BPF_XOR);
+	return __bpf_fill_alu32_imm(self, BPF_XOR, 0);
 }
 
 static int bpf_fill_alu32_add_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu32_imm(self, BPF_ADD);
+	return __bpf_fill_alu32_imm(self, BPF_ADD, 0);
 }
 
 static int bpf_fill_alu32_sub_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu32_imm(self, BPF_SUB);
+	return __bpf_fill_alu32_imm(self, BPF_SUB, 0);
 }
 
 static int bpf_fill_alu32_mul_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu32_imm(self, BPF_MUL);
+	return __bpf_fill_alu32_imm(self, BPF_MUL, 0);
 }
 
 static int bpf_fill_alu32_div_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu32_imm(self, BPF_DIV);
+	return __bpf_fill_alu32_imm(self, BPF_DIV, 0);
 }
 
 static int bpf_fill_alu32_mod_imm(struct bpf_test *self)
 {
-	return __bpf_fill_alu32_imm(self, BPF_MOD);
+	return __bpf_fill_alu32_imm(self, BPF_MOD, 0);
 }
 
 /* ALU64 register operations */
@@ -1235,7 +1298,8 @@ static int bpf_fill_alu32_mod_reg(struct bpf_test *self)
  * Test JITs that implement complex ALU operations as function
  * calls, and must re-arrange operands for argument passing.
  */
-static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32)
+static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op,
+				    u32 flags)
 {
 	int len = 2 + 10 * 10;
 	struct bpf_insn *insns;
@@ -1249,28 +1313,37 @@ static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32)
 		return -ENOMEM;
 
 	/* Operand and result values according to operation */
-	if (alu32)
-		dst = 0x76543210U;
-	else
-		dst = 0x7edcba9876543210ULL;
+	if (flags & F_SIGNED) {
+		if (flags & F_ALU32)
+			dst = -76543210;
+		else
+			dst = -7654321076543210LL;
+	} else {
+		if (flags & F_ALU32)
+			dst = 0x76543210U;
+		else
+			dst = 0x7edcba9876543210ULL;
+	}
 	imm = 0x01234567U;
 
 	if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH)
 		imm &= 31;
 
-	__bpf_alu_result(&res, dst, imm, op);
+	__bpf_alu_result(&res, dst, imm, op, flags);
 
-	if (alu32)
+	if (flags & F_ALU32)
 		res = (u32)res;
 
 	/* Check all operand registers */
 	for (rd = R0; rd <= R9; rd++) {
 		i += __bpf_ld_imm64(&insns[i], rd, dst);
 
-		if (alu32)
-			insns[i++] = BPF_ALU32_IMM(op, rd, imm);
+		s16 off = (flags & F_SIGNED) ? 1 : 0;
+
+		if (flags & F_ALU32)
+			insns[i++] = BPF_ALU32_IMM_OFF(op, rd, imm, off);
 		else
-			insns[i++] = BPF_ALU64_IMM(op, rd, imm);
+			insns[i++] = BPF_ALU64_IMM_OFF(op, rd, imm, off);
 
 		insns[i++] = BPF_JMP32_IMM(BPF_JEQ, rd, res, 2);
 		insns[i++] = BPF_MOV64_IMM(R0, __LINE__);
@@ -1295,123 +1368,145 @@ static int __bpf_fill_alu_imm_regs(struct bpf_test *self, u8 op, bool alu32)
 /* ALU64 K registers */
 static int bpf_fill_alu64_mov_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_MOV, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_MOV, 0);
 }
 
 static int bpf_fill_alu64_and_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_AND, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_AND, 0);
 }
 
 static int bpf_fill_alu64_or_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_OR, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_OR, 0);
 }
 
 static int bpf_fill_alu64_xor_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_XOR, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_XOR, 0);
 }
 
 static int bpf_fill_alu64_lsh_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_LSH, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_LSH, 0);
 }
 
 static int bpf_fill_alu64_rsh_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_RSH, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_RSH, 0);
 }
 
 static int bpf_fill_alu64_arsh_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_ARSH, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_ARSH, 0);
 }
 
 static int bpf_fill_alu64_add_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_ADD, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_ADD, 0);
 }
 
 static int bpf_fill_alu64_sub_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_SUB, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_SUB, 0);
 }
 
 static int bpf_fill_alu64_mul_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_MUL, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_MUL, 0);
 }
 
 static int bpf_fill_alu64_div_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_DIV, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_DIV, 0);
 }
 
 static int bpf_fill_alu64_mod_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_MOD, false);
+	return __bpf_fill_alu_imm_regs(self, BPF_MOD, 0);
+}
+
+/* Signed ALU64 K registers */
+static int bpf_fill_alu64_sdiv_imm_regs(struct bpf_test *self)
+{
+	return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_SIGNED);
+}
+
+static int bpf_fill_alu64_smod_imm_regs(struct bpf_test *self)
+{
+	return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_SIGNED);
 }
 
 /* ALU32 K registers */
 static int bpf_fill_alu32_mov_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_MOV, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_MOV, F_ALU32);
 }
 
 static int bpf_fill_alu32_and_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_AND, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_AND, F_ALU32);
 }
 
 static int bpf_fill_alu32_or_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_OR, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_OR, F_ALU32);
 }
 
 static int bpf_fill_alu32_xor_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_XOR, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_XOR, F_ALU32);
 }
 
 static int bpf_fill_alu32_lsh_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_LSH, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_LSH, F_ALU32);
 }
 
 static int bpf_fill_alu32_rsh_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_RSH, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_RSH, F_ALU32);
 }
 
 static int bpf_fill_alu32_arsh_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_ARSH, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_ARSH, F_ALU32);
 }
 
 static int bpf_fill_alu32_add_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_ADD, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_ADD, F_ALU32);
 }
 
 static int bpf_fill_alu32_sub_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_SUB, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_SUB, F_ALU32);
 }
 
 static int bpf_fill_alu32_mul_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_MUL, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_MUL, F_ALU32);
 }
 
 static int bpf_fill_alu32_div_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_DIV, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32);
 }
 
 static int bpf_fill_alu32_mod_imm_regs(struct bpf_test *self)
 {
-	return __bpf_fill_alu_imm_regs(self, BPF_MOD, true);
+	return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32);
+}
+
+/* Signed ALU32 K registers */
+static int bpf_fill_alu32_sdiv_imm_regs(struct bpf_test *self)
+{
+	return __bpf_fill_alu_imm_regs(self, BPF_DIV, F_ALU32 | F_SIGNED);
+}
+
+static int bpf_fill_alu32_smod_imm_regs(struct bpf_test *self)
+{
+	return __bpf_fill_alu_imm_regs(self, BPF_MOD, F_ALU32 | F_SIGNED);
 }
 
 /*
@@ -1442,8 +1537,8 @@ static int __bpf_fill_alu_reg_pairs(struct bpf_test *self, u8 op, bool alu32)
 	if (op == BPF_LSH || op == BPF_RSH || op == BPF_ARSH)
 		src &= 31;
 
-	__bpf_alu_result(&res, dst, src, op);
-	__bpf_alu_result(&same, src, src, op);
+	__bpf_alu_result(&res, dst, src, op, 0);
+	__bpf_alu_result(&same, src, src, op, 0);
 
 	if (alu32) {
 		res = (u32)res;
@@ -1626,7 +1721,7 @@ static int __bpf_emit_atomic64(struct bpf_test *self, void *arg,
 		res = src;
 		break;
 	default:
-		__bpf_alu_result(&res, dst, src, BPF_OP(op));
+		__bpf_alu_result(&res, dst, src, BPF_OP(op), 0);
 	}
 
 	keep = 0x0123456789abcdefULL;
@@ -1673,7 +1768,7 @@ static int __bpf_emit_atomic32(struct bpf_test *self, void *arg,
 		res = src;
 		break;
 	default:
-		__bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op));
+		__bpf_alu_result(&res, (u32)dst, (u32)src, BPF_OP(op), 0);
 	}
 
 	keep = 0x0123456789abcdefULL;
@@ -1939,7 +2034,7 @@ static int __bpf_fill_atomic_reg_pairs(struct bpf_test *self, u8 width, u8 op)
 		res = mem;
 		break;
 	default:
-		__bpf_alu_result(&res, mem, upd, BPF_OP(op));
+		__bpf_alu_result(&res, mem, upd, BPF_OP(op), 0);
 	}
 
 	/* Test all operand registers */
@@ -12354,6 +12449,22 @@ static struct bpf_test tests[] = {
 		{ { 0, 1 } },
 		.fill_helper = bpf_fill_alu64_mod_imm_regs,
 	},
+	{
+		"ALU64_SDIV_K: registers",
+		{ },
+		INTERNAL,
+		{ },
+		{ { 0, 1 } },
+		.fill_helper = bpf_fill_alu64_sdiv_imm_regs,
+	},
+	{
+		"ALU64_SMOD_K: registers",
+		{ },
+		INTERNAL,
+		{ },
+		{ { 0, 1 } },
+		.fill_helper = bpf_fill_alu64_smod_imm_regs,
+	},
 	/* ALU32 K registers */
 	{
 		"ALU32_MOV_K: registers",
@@ -12451,6 +12562,22 @@ static struct bpf_test tests[] = {
 		{ { 0, 1 } },
 		.fill_helper = bpf_fill_alu32_mod_imm_regs,
 	},
+	{
+		"ALU32_SDIV_K: registers",
+		{ },
+		INTERNAL,
+		{ },
+		{ { 0, 1 } },
+		.fill_helper = bpf_fill_alu32_sdiv_imm_regs,
+	},
+	{
+		"ALU32_SMOD_K: registers",
+		{ },
+		INTERNAL,
+		{ },
+		{ { 0, 1 } },
+		.fill_helper = bpf_fill_alu32_smod_imm_regs,
+	},
 	/* ALU64 X register combinations */
 	{
 		"ALU64_MOV_X: register combinations",
@@ -12881,6 +13008,24 @@ static struct bpf_test tests[] = {
 		.fill_helper = bpf_fill_alu64_mod_imm,
 		.nr_testruns = NR_PATTERN_RUNS,
 	},
+	{
+		"ALU64_SDIV_K: all immediate value magnitudes",
+		{ },
+		INTERNAL | FLAG_NO_DATA,
+		{ },
+		{ { 0, 1 } },
+		.fill_helper = bpf_fill_alu64_sdiv_imm,
+		.nr_testruns = NR_PATTERN_RUNS,
+	},
+	{
+		"ALU64_SMOD_K: all immediate value magnitudes",
+		{ },
+		INTERNAL | FLAG_NO_DATA,
+		{ },
+		{ { 0, 1 } },
+		.fill_helper = bpf_fill_alu64_smod_imm,
+		.nr_testruns = NR_PATTERN_RUNS,
+	},
 	/* ALU32 immediate magnitudes */
 	{
 		"ALU32_MOV_K: all immediate value magnitudes",
@@ -12963,6 +13108,24 @@ static struct bpf_test tests[] = {
 		.fill_helper = bpf_fill_alu32_mod_imm,
 		.nr_testruns = NR_PATTERN_RUNS,
 	},
+	{
+		"ALU32_SDIV_K: all immediate value magnitudes",
+		{ },
+		INTERNAL | FLAG_NO_DATA,
+		{ },
+		{ { 0, 1 } },
+		.fill_helper = bpf_fill_alu32_sdiv_imm,
+		.nr_testruns = NR_PATTERN_RUNS,
+	},
+	{
+		"ALU32_SMOD_K: all immediate value magnitudes",
+		{ },
+		INTERNAL | FLAG_NO_DATA,
+		{ },
+		{ { 0, 1 } },
+		.fill_helper = bpf_fill_alu32_smod_imm,
+		.nr_testruns = NR_PATTERN_RUNS,
+	},
 	/* ALU64 register magnitudes */
 	{
 		"ALU64_MOV_X: all register value magnitudes",
diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c
index 06b4a6a028e0e..316f4dfcda651 100644
--- a/lib/test_context-analysis.c
+++ b/lib/test_context-analysis.c
@@ -159,6 +159,10 @@ TEST_SPINLOCK_COMMON(read_lock,
 struct test_mutex_data {
 	struct mutex mtx;
 	int counter __guarded_by(&mtx);
+
+	struct mutex mtx2;
+	int anyread __guarded_by(&mtx, &mtx2);
+	int *anyptr __pt_guarded_by(&mtx, &mtx2);
 };
 
 static void __used test_mutex_init(struct test_mutex_data *d)
@@ -219,6 +223,26 @@ static void __used test_mutex_cond_guard(struct test_mutex_data *d)
 	}
 }
 
+static void __used test_mutex_multiguard(struct test_mutex_data *d)
+{
+	mutex_lock(&d->mtx);
+	(void)d->anyread;
+	(void)*d->anyptr;
+	mutex_unlock(&d->mtx);
+
+	mutex_lock(&d->mtx2);
+	(void)d->anyread;
+	(void)*d->anyptr;
+	mutex_unlock(&d->mtx2);
+
+	mutex_lock(&d->mtx);
+	mutex_lock(&d->mtx2);
+	d->anyread++;
+	(*d->anyptr)++;
+	mutex_unlock(&d->mtx2);
+	mutex_unlock(&d->mtx);
+}
+
 struct test_seqlock_data {
 	seqlock_t sl;
 	int counter __guarded_by(&sl);
diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index 2135049157372..38996c4baa408 100644
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -1253,8 +1253,8 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
 	mmap_read_unlock(mm);
 	mmput(mm);
 free_mem:
-	kfree(src_pfns);
-	kfree(dst_pfns);
+	kvfree(src_pfns);
+	kvfree(dst_pfns);
 	return ret;
 }
 
diff --git a/lib/test_ida.c b/lib/test_ida.c
index 63078f8dc13f5..c400b24f88b61 100644
--- a/lib/test_ida.c
+++ b/lib/test_ida.c
@@ -256,6 +256,20 @@ static void ida_check_find_first(struct ida *ida)
 	ida_free(ida, (1 << 20) - 1);
 
 	IDA_BUG_ON(ida, !ida_is_empty(ida));
+
+	/*
+	 * Test cross-chunk search.
+	 * Allocate ID in chunk 0 and ID in chunk 1.
+	 * Search for ID >= 1. min=1 maps to chunk 0. Chunk 0 has no IDs >= 1.
+	 * It should continue to chunk 1 and return 1024.
+	 */
+	IDA_BUG_ON(ida, ida_alloc_min(ida, 0, GFP_KERNEL) != 0);
+	IDA_BUG_ON(ida, ida_alloc_min(ida, 1024, GFP_KERNEL) != 1024);
+	IDA_BUG_ON(ida, ida_find_first_range(ida, 1, INT_MAX) != 1024);
+	ida_free(ida, 0);
+	ida_free(ida, 1024);
+
+	IDA_BUG_ON(ida, !ida_is_empty(ida));
 }
 
 static DEFINE_IDA(ida);
diff --git a/lib/test_meminit.c b/lib/test_meminit.c
index 6298f66c964bb..d028a6552cd61 100644
--- a/lib/test_meminit.c
+++ b/lib/test_meminit.c
@@ -387,7 +387,7 @@ static int __init test_kmemcache(int *total_failures)
 			ctor = flags & 1;
 			rcu = flags & 2;
 			zero = flags & 4;
-			if (ctor & zero)
+			if (ctor && zero)
 				continue;
 			num_tests += do_kmem_cache_size(size, ctor, rcu, zero,
 							&failures);
diff --git a/lib/tests/Makefile b/lib/tests/Makefile
index 7e9c2fa52e35a..4ead57602eac4 100644
--- a/lib/tests/Makefile
+++ b/lib/tests/Makefile
@@ -8,6 +8,7 @@ obj-$(CONFIG_BASE64_KUNIT) += base64_kunit.o
 obj-$(CONFIG_BITOPS_KUNIT) += bitops_kunit.o
 obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o
 obj-$(CONFIG_BITS_TEST) += test_bits.o
+obj-$(CONFIG_SHDI3_KUNIT_TEST) += shdi3_kunit.o
 obj-$(CONFIG_BLACKHOLE_DEV_KUNIT_TEST) += blackhole_dev_kunit.o
 obj-$(CONFIG_CHECKSUM_KUNIT) += checksum_kunit.o
 obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o
diff --git a/lib/tests/cmdline_kunit.c b/lib/tests/cmdline_kunit.c
index c1602f797637b..3f61ff8d31782 100644
--- a/lib/tests/cmdline_kunit.c
+++ b/lib/tests/cmdline_kunit.c
@@ -6,6 +6,7 @@
 #include <kunit/test.h>
 #include <linux/kernel.h>
 #include <linux/random.h>
+#include <linux/sizes.h>
 #include <linux/string.h>
 
 static const char *cmdline_test_strings[] = {
@@ -139,11 +140,128 @@ static void cmdline_test_range(struct kunit *test)
 	} while (++i < ARRAY_SIZE(cmdline_test_range_strings));
 }
 
+static void cmdline_test_next_arg_quoted_value(struct kunit *test)
+{
+	char in[] = "foo=\"bar baz\" qux=1";
+	char *next, *param, *val;
+
+	next = next_arg(in, &param, &val);
+	KUNIT_EXPECT_STREQ(test, param, "foo");
+	KUNIT_ASSERT_NOT_NULL(test, val);
+	KUNIT_EXPECT_STREQ(test, val, "bar baz");
+	KUNIT_EXPECT_STREQ(test, next, "qux=1");
+
+	next = next_arg(next, &param, &val);
+	KUNIT_EXPECT_STREQ(test, param, "qux");
+	KUNIT_ASSERT_NOT_NULL(test, val);
+	KUNIT_EXPECT_STREQ(test, val, "1");
+	KUNIT_EXPECT_STREQ(test, next, "");
+}
+
+static void cmdline_test_next_arg_bare_quote_regression(struct kunit *test)
+{
+	char in[] = "foo=bar \"";
+	char *next, *param, *val;
+
+	next = next_arg(in, &param, &val);
+	KUNIT_EXPECT_STREQ(test, param, "foo");
+	KUNIT_ASSERT_NOT_NULL(test, val);
+	KUNIT_EXPECT_STREQ(test, val, "bar");
+	KUNIT_EXPECT_STREQ(test, next, "\"");
+
+	/* This hits the i == 0 quoted-token case fixed by 9847f21225c4. */
+	next = next_arg(next, &param, &val);
+	KUNIT_EXPECT_STREQ(test, param, "");
+	KUNIT_EXPECT_PTR_EQ(test, val, NULL);
+	KUNIT_EXPECT_STREQ(test, next, "");
+}
+
+static void cmdline_test_next_arg_mixed_tokens(struct kunit *test)
+{
+	char in[] = "bbb= jjj kkk=\"a=b\"";
+	char *next, *param, *val;
+
+	next = next_arg(in, &param, &val);
+	KUNIT_EXPECT_STREQ(test, param, "bbb");
+	KUNIT_ASSERT_NOT_NULL(test, val);
+	KUNIT_EXPECT_STREQ(test, val, "");
+	KUNIT_EXPECT_STREQ(test, next, "jjj kkk=\"a=b\"");
+
+	next = next_arg(next, &param, &val);
+	KUNIT_EXPECT_STREQ(test, param, "jjj");
+	KUNIT_EXPECT_NULL(test, val);
+	KUNIT_EXPECT_STREQ(test, next, "kkk=\"a=b\"");
+
+	next = next_arg(next, &param, &val);
+	KUNIT_EXPECT_STREQ(test, param, "kkk");
+	KUNIT_ASSERT_NOT_NULL(test, val);
+	KUNIT_EXPECT_STREQ(test, val, "a=b");
+	KUNIT_EXPECT_STREQ(test, next, "");
+}
+
+struct cmdline_test_memparse_entry {
+	const char *input;
+	const char *unrecognized;
+	unsigned long long result;
+};
+
+static const struct cmdline_test_memparse_entry testdata[] = {
+	{ "0",				"",	0ULL },
+	{ "1",				"",	1ULL },
+	{ "a",				"a",	0ULL },
+	{ "k",				"k",	0ULL },
+	{ "E",				"E",	0ULL },
+	{ "0xb",			"",	11ULL },
+	{ "0xz",			"x",	0ULL },
+	{ "1234",			"",	1234ULL },
+	{ "04567",			"",	2423ULL },
+	{ "0x9876",			"",	39030LL },
+	{ "05678",			"8",	375ULL },
+	{ "0xabcdefz",			"z",	11259375ULL },
+	{ "0cdba",			"c",	0ULL },
+	{ "4K",				"",	SZ_4K },
+	{ "0x10k@0xaaaabbbb",		"@",	SZ_16K },
+	{ "32M",			"",	SZ_32M },
+	{ "067m:foo",			":",	55 * SZ_1M },
+	{ "2G;bar=baz",			";",	SZ_2G },
+	{ "07gz",			"z",	7ULL * SZ_1G },
+	{ "3T+data",			"+",	3 * SZ_1T },
+	{ "04t,ro",			",",	SZ_4T },
+	{ "012p",			"",	11258999068426240ULL },
+	{ "7P,sync",			",",	7881299347898368ULL },
+	{ "0x2e",			"",	46ULL },
+	{ "2E and more",		" ",	2305843009213693952ULL },
+	{ "18446744073709551615",	"",	ULLONG_MAX },
+	{ "0xffffffffffffffff0",	"",	ULLONG_MAX },
+	{ "1111111111111111111T",	"",	ULLONG_MAX },
+	{ "222222222222222222222G",	"",	ULLONG_MAX },
+	{ "3333333333333333333333M",	"",	ULLONG_MAX },
+};
+
+static void cmdline_test_memparse(struct kunit *test)
+{
+	const struct cmdline_test_memparse_entry *e;
+	unsigned long long ret;
+	char *retptr;
+
+	for (e = testdata; e < testdata + ARRAY_SIZE(testdata); e++) {
+		ret = memparse(e->input, &retptr);
+		KUNIT_EXPECT_EQ_MSG(test, ret, e->result,
+				    "    when parsing '%s'", e->input);
+		KUNIT_EXPECT_EQ_MSG(test, *retptr, *e->unrecognized,
+				    "    when parsing '%s'", e->input);
+	}
+}
+
 static struct kunit_case cmdline_test_cases[] = {
 	KUNIT_CASE(cmdline_test_noint),
 	KUNIT_CASE(cmdline_test_lead_int),
 	KUNIT_CASE(cmdline_test_tail_int),
 	KUNIT_CASE(cmdline_test_range),
+	KUNIT_CASE(cmdline_test_next_arg_quoted_value),
+	KUNIT_CASE(cmdline_test_next_arg_bare_quote_regression),
+	KUNIT_CASE(cmdline_test_next_arg_mixed_tokens),
+	KUNIT_CASE(cmdline_test_memparse),
 	{}
 };
 
diff --git a/lib/tests/kunit_iov_iter.c b/lib/tests/kunit_iov_iter.c
index 37bd6eb258960..f02f7b7aa7960 100644
--- a/lib/tests/kunit_iov_iter.c
+++ b/lib/tests/kunit_iov_iter.c
@@ -1128,7 +1128,7 @@ static void __init iov_kunit_iter_to_sg_kvec(struct kunit *test)
 	struct kvec kvec;
 	size_t bufsize;
 
-	bufsize = 0x100000;
+	bufsize = 0x200000;
 	iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
 
 	kvec.iov_base = data.buffer;
@@ -1146,7 +1146,7 @@ static void __init iov_kunit_iter_to_sg_bvec(struct kunit *test)
 	struct bio_vec *bvec;
 	struct iov_iter iter;
 
-	bufsize = 0x100000;
+	bufsize = 0x200000;
 	iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
 
 	bvec = kunit_kmalloc_array(test, data.npages, sizeof(*bvec),
@@ -1173,7 +1173,7 @@ static void __init iov_kunit_iter_to_sg_folioq(struct kunit *test)
 	struct iov_iter iter;
 	size_t bufsize;
 
-	bufsize = 0x100000;
+	bufsize = 0x200000;
 	iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
 
 	folioq = iov_kunit_create_folioq(test);
@@ -1190,7 +1190,7 @@ static void __init iov_kunit_iter_to_sg_xarray(struct kunit *test)
 	struct iov_iter iter;
 	size_t bufsize;
 
-	bufsize = 0x100000;
+	bufsize = 0x200000;
 	iov_kunit_iter_to_sg_init(test, bufsize, false, &data);
 
 	xarray = iov_kunit_create_xarray(test);
@@ -1206,7 +1206,7 @@ static void __init iov_kunit_iter_to_sg_ubuf(struct kunit *test)
 	struct iov_iter iter;
 	size_t bufsize;
 
-	bufsize = 0x100000;
+	bufsize = 0x200000;
 	iov_kunit_iter_to_sg_init(test, bufsize, true, &data);
 
 	iov_iter_ubuf(&iter, READ, data.ubuf, bufsize);
diff --git a/lib/tests/liveupdate.c b/lib/tests/liveupdate.c
index e4b0ecbee32fe..4c08a7c6fb788 100644
--- a/lib/tests/liveupdate.c
+++ b/lib/tests/liveupdate.c
@@ -105,6 +105,9 @@ static void liveupdate_test_init(void)
 			pr_err("liveupdate_flb_get_incoming for %s failed: %pe\n",
 			       flb->compatible, ERR_PTR(err));
 		}
+
+		if (!err)
+			liveupdate_flb_put_incoming(flb);
 	}
 	initialized = true;
 }
diff --git a/lib/tests/seq_buf_kunit.c b/lib/tests/seq_buf_kunit.c
index 8a01579a978e6..eb466386bbefb 100644
--- a/lib/tests/seq_buf_kunit.c
+++ b/lib/tests/seq_buf_kunit.c
@@ -184,6 +184,38 @@ static void seq_buf_get_buf_commit_test(struct kunit *test)
 	KUNIT_EXPECT_TRUE(test, seq_buf_has_overflowed(&s));
 }
 
+static void seq_buf_putmem_hex_test(struct kunit *test)
+{
+	DECLARE_SEQ_BUF(s, 24);
+	const u8 data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+#ifdef __BIG_ENDIAN
+	const char *expected = "0001020304050607 0809 ";
+#else
+	const char *expected = "0706050403020100 0908 ";
+#endif
+
+	KUNIT_EXPECT_EQ(test, seq_buf_putmem_hex(&s, data, sizeof(data)), 0);
+	KUNIT_EXPECT_FALSE(test, seq_buf_has_overflowed(&s));
+	KUNIT_EXPECT_EQ(test, seq_buf_used(&s), strlen(expected));
+	KUNIT_EXPECT_STREQ(test, seq_buf_str(&s), expected);
+}
+
+static void seq_buf_putmem_hex_overflow_test(struct kunit *test)
+{
+	DECLARE_SEQ_BUF(s, 20);
+	const u8 data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
+#ifdef __BIG_ENDIAN
+	const char *expected = "0001020304050607 ";
+#else
+	const char *expected = "0706050403020100 ";
+#endif
+
+	KUNIT_EXPECT_EQ(test, seq_buf_putmem_hex(&s, data, sizeof(data)), -1);
+	KUNIT_EXPECT_TRUE(test, seq_buf_has_overflowed(&s));
+	KUNIT_EXPECT_EQ(test, seq_buf_used(&s), 20);
+	KUNIT_EXPECT_STREQ(test, seq_buf_str(&s), expected);
+}
+
 static struct kunit_case seq_buf_test_cases[] = {
 	KUNIT_CASE(seq_buf_init_test),
 	KUNIT_CASE(seq_buf_declare_test),
@@ -194,6 +226,8 @@ static struct kunit_case seq_buf_test_cases[] = {
 	KUNIT_CASE(seq_buf_printf_test),
 	KUNIT_CASE(seq_buf_printf_overflow_test),
 	KUNIT_CASE(seq_buf_get_buf_commit_test),
+	KUNIT_CASE(seq_buf_putmem_hex_test),
+	KUNIT_CASE(seq_buf_putmem_hex_overflow_test),
 	{}
 };
 
diff --git a/lib/tests/shdi3_kunit.c b/lib/tests/shdi3_kunit.c
new file mode 100644
index 0000000000000..44f65e66512b5
--- /dev/null
+++ b/lib/tests/shdi3_kunit.c
@@ -0,0 +1,175 @@
+// SPDX-License-Identifier: GPL-2.0-or-later OR Apache-2.0
+/*
+ * Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3().
+ */
+
+#include <linux/array_size.h>
+#include <linux/module.h>
+#include <linux/libgcc.h>
+#include <kunit/test.h>
+
+struct shdi3_test_entry {
+	long long input;
+	int shift;
+	long long result;
+};
+
+static const struct shdi3_test_entry ashldi3_testdata[] = {
+	/* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashldi3_test.c */
+	{ 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL },
+	{ 0x0123456789ABCDEFLL, 1, 0x2468ACF13579BDELL },
+	{ 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BCLL },
+	{ 0x0123456789ABCDEFLL, 3, 0x91A2B3C4D5E6F78LL },
+	{ 0x0123456789ABCDEFLL, 4, 0x123456789ABCDEF0LL },
+	{ 0x0123456789ABCDEFLL, 28, 0x789ABCDEF0000000LL },
+	{ 0x0123456789ABCDEFLL, 29, 0xF13579BDE0000000LL },
+	{ 0x0123456789ABCDEFLL, 30, 0xE26AF37BC0000000LL },
+	{ 0x0123456789ABCDEFLL, 31, 0xC4D5E6F780000000LL },
+	{ 0x0123456789ABCDEFLL, 32, 0x89ABCDEF00000000LL },
+	{ 0x0123456789ABCDEFLL, 33, 0x13579BDE00000000LL },
+	{ 0x0123456789ABCDEFLL, 34, 0x26AF37BC00000000LL },
+	{ 0x0123456789ABCDEFLL, 35, 0x4D5E6F7800000000LL },
+	{ 0x0123456789ABCDEFLL, 36, 0x9ABCDEF000000000LL },
+	{ 0x0123456789ABCDEFLL, 60, 0xF000000000000000LL },
+	{ 0x0123456789ABCDEFLL, 61, 0xE000000000000000LL },
+	{ 0x0123456789ABCDEFLL, 62, 0xC000000000000000LL },
+	{ 0x0123456789ABCDEFLL, 63, 0x8000000000000000LL },
+};
+
+static void shdi3_test_ashldi3(struct kunit *test)
+{
+	const struct shdi3_test_entry *e;
+	long long ret;
+
+	for (e = ashldi3_testdata;
+	     e < ashldi3_testdata + ARRAY_SIZE(ashldi3_testdata); e++) {
+		ret = __ashldi3(e->input, e->shift);
+		KUNIT_EXPECT_EQ_MSG(test, ret, e->result,
+				    "    when evaluating __ashldi3(%lld, %d)",
+				    e->input, e->shift);
+	}
+}
+
+static const struct shdi3_test_entry ashrdi3_testdata[] = {
+	/* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashrdi3_test.c */
+	{ 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL },
+	{ 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL },
+	{ 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL },
+	{ 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL },
+	{ 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL },
+	{ 0x0123456789ABCDEFLL, 28, 0x12345678LL },
+	{ 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL },
+	{ 0x0123456789ABCDEFLL, 30, 0x48D159ELL },
+	{ 0x0123456789ABCDEFLL, 31, 0x2468ACFLL },
+	{ 0x0123456789ABCDEFLL, 32, 0x1234567LL },
+	{ 0x0123456789ABCDEFLL, 33, 0x91A2B3LL },
+	{ 0x0123456789ABCDEFLL, 34, 0x48D159LL },
+	{ 0x0123456789ABCDEFLL, 35, 0x2468ACLL },
+	{ 0x0123456789ABCDEFLL, 36, 0x123456LL },
+	{ 0x0123456789ABCDEFLL, 60, 0 },
+	{ 0x0123456789ABCDEFLL, 61, 0 },
+	{ 0x0123456789ABCDEFLL, 62, 0 },
+	{ 0x0123456789ABCDEFLL, 63, 0 },
+	{ 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL },
+	{ 0xFEDCBA9876543210LL, 1, 0xFF6E5D4C3B2A1908LL },
+	{ 0xFEDCBA9876543210LL, 2, 0xFFB72EA61D950C84LL },
+	{ 0xFEDCBA9876543210LL, 3, 0xFFDB97530ECA8642LL },
+	{ 0xFEDCBA9876543210LL, 4, 0xFFEDCBA987654321LL },
+	{ 0xFEDCBA9876543210LL, 28, 0xFFFFFFFFEDCBA987LL },
+	{ 0xFEDCBA9876543210LL, 29, 0xFFFFFFFFF6E5D4C3LL },
+	{ 0xFEDCBA9876543210LL, 30, 0xFFFFFFFFFB72EA61LL },
+	{ 0xFEDCBA9876543210LL, 31, 0xFFFFFFFFFDB97530LL },
+	{ 0xFEDCBA9876543210LL, 32, 0xFFFFFFFFFEDCBA98LL },
+	{ 0xFEDCBA9876543210LL, 33, 0xFFFFFFFFFF6E5D4CLL },
+	{ 0xFEDCBA9876543210LL, 34, 0xFFFFFFFFFFB72EA6LL },
+	{ 0xFEDCBA9876543210LL, 35, 0xFFFFFFFFFFDB9753LL },
+	{ 0xFEDCBA9876543210LL, 36, 0xFFFFFFFFFFEDCBA9LL },
+	{ 0xAEDCBA9876543210LL, 60, 0xFFFFFFFFFFFFFFFALL },
+	{ 0xAEDCBA9876543210LL, 61, 0xFFFFFFFFFFFFFFFDLL },
+	{ 0xAEDCBA9876543210LL, 62, 0xFFFFFFFFFFFFFFFELL },
+	{ 0xAEDCBA9876543210LL, 63, 0xFFFFFFFFFFFFFFFFLL },
+};
+
+static void shdi3_test_ashrdi3(struct kunit *test)
+{
+	const struct shdi3_test_entry *e;
+	long long ret;
+
+	for (e = ashrdi3_testdata;
+	     e < ashrdi3_testdata + ARRAY_SIZE(ashrdi3_testdata); e++) {
+		ret = __ashrdi3(e->input, e->shift);
+		KUNIT_EXPECT_EQ_MSG(test, ret, e->result,
+				    "    when evaluating __ashrdi3(%lld, %d)",
+				    e->input, e->shift);
+	}
+}
+
+static const struct shdi3_test_entry lshrdi3_testdata[] = {
+	/* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/lshrdi3_test.c */
+	{ 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL },
+	{ 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL },
+	{ 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL },
+	{ 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL },
+	{ 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL },
+	{ 0x0123456789ABCDEFLL, 28, 0x12345678LL },
+	{ 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL },
+	{ 0x0123456789ABCDEFLL, 30, 0x48D159ELL },
+	{ 0x0123456789ABCDEFLL, 31, 0x2468ACFLL },
+	{ 0x0123456789ABCDEFLL, 32, 0x1234567LL },
+	{ 0x0123456789ABCDEFLL, 33, 0x91A2B3LL },
+	{ 0x0123456789ABCDEFLL, 34, 0x48D159LL },
+	{ 0x0123456789ABCDEFLL, 35, 0x2468ACLL },
+	{ 0x0123456789ABCDEFLL, 36, 0x123456LL },
+	{ 0x0123456789ABCDEFLL, 60, 0 },
+	{ 0x0123456789ABCDEFLL, 61, 0 },
+	{ 0x0123456789ABCDEFLL, 62, 0 },
+	{ 0x0123456789ABCDEFLL, 63, 0 },
+	{ 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL },
+	{ 0xFEDCBA9876543210LL, 1, 0x7F6E5D4C3B2A1908LL },
+	{ 0xFEDCBA9876543210LL, 2, 0x3FB72EA61D950C84LL },
+	{ 0xFEDCBA9876543210LL, 3, 0x1FDB97530ECA8642LL },
+	{ 0xFEDCBA9876543210LL, 4, 0xFEDCBA987654321LL },
+	{ 0xFEDCBA9876543210LL, 28, 0xFEDCBA987LL },
+	{ 0xFEDCBA9876543210LL, 29, 0x7F6E5D4C3LL },
+	{ 0xFEDCBA9876543210LL, 30, 0x3FB72EA61LL },
+	{ 0xFEDCBA9876543210LL, 31, 0x1FDB97530LL },
+	{ 0xFEDCBA9876543210LL, 32, 0xFEDCBA98LL },
+	{ 0xFEDCBA9876543210LL, 33, 0x7F6E5D4CLL },
+	{ 0xFEDCBA9876543210LL, 34, 0x3FB72EA6LL },
+	{ 0xFEDCBA9876543210LL, 35, 0x1FDB9753LL },
+	{ 0xFEDCBA9876543210LL, 36, 0xFEDCBA9LL },
+	{ 0xAEDCBA9876543210LL, 60, 0xALL },
+	{ 0xAEDCBA9876543210LL, 61, 0x5LL },
+	{ 0xAEDCBA9876543210LL, 62, 0x2LL },
+	{ 0xAEDCBA9876543210LL, 63, 0x1LL },
+};
+
+static void shdi3_test_lshrdi3(struct kunit *test)
+{
+	const struct shdi3_test_entry *e;
+	long long ret;
+
+	for (e = lshrdi3_testdata;
+	     e < lshrdi3_testdata + ARRAY_SIZE(lshrdi3_testdata); e++) {
+		ret = __lshrdi3(e->input, e->shift);
+		KUNIT_EXPECT_EQ_MSG(test, ret, e->result,
+				    "    when evaluating __lshrdi3(%lld, %d)",
+				    e->input, e->shift);
+	}
+}
+
+static struct kunit_case shdi3_test_cases[] = {
+	KUNIT_CASE(shdi3_test_ashldi3),
+	KUNIT_CASE(shdi3_test_ashrdi3),
+	KUNIT_CASE(shdi3_test_lshrdi3),
+	{}
+};
+
+static struct kunit_suite shdi3_test_suite = {
+	.name = "shdi3",
+	.test_cases = shdi3_test_cases,
+};
+kunit_test_suite(shdi3_test_suite);
+
+MODULE_DESCRIPTION("Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3()");
+MODULE_LICENSE("GPL");
diff --git a/lib/tests/string_helpers_kunit.c b/lib/tests/string_helpers_kunit.c
index c853046183d24..9fbe91079c7ed 100644
--- a/lib/tests/string_helpers_kunit.c
+++ b/lib/tests/string_helpers_kunit.c
@@ -5,11 +5,16 @@
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
 #include <kunit/test.h>
+
 #include <linux/array_size.h>
-#include <linux/kernel.h>
+#include <linux/bug.h>
+#include <linux/limits.h>
+#include <linux/module.h>
 #include <linux/random.h>
-#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/sprintf.h>
 #include <linux/string_helpers.h>
+#include <linux/types.h>
 
 static void test_string_check_buf(struct kunit *test,
 				  const char *name, unsigned int flags,
@@ -601,6 +606,11 @@ static void test_unescape(struct kunit *test)
 		test_string_unescape(test, "unescape", i, false);
 	test_string_unescape(test, "unescape inplace",
 			     get_random_u32_below(UNESCAPE_ALL_MASK + 1), true);
+}
+
+static void test_escape(struct kunit *test)
+{
+	unsigned int i;
 
 	/* Without dictionary */
 	for (i = 0; i < ESCAPE_ALL_MASK + 1; i++)
@@ -615,6 +625,7 @@ static struct kunit_case string_helpers_test_cases[] = {
 	KUNIT_CASE(test_get_size),
 	KUNIT_CASE(test_upper_lower),
 	KUNIT_CASE(test_unescape),
+	KUNIT_CASE(test_escape),
 	{}
 };
 
diff --git a/lib/tests/uuid_kunit.c b/lib/tests/uuid_kunit.c
index de71b2649dac6..2ef64fbe67d6f 100644
--- a/lib/tests/uuid_kunit.c
+++ b/lib/tests/uuid_kunit.c
@@ -86,11 +86,67 @@ static void uuid_test_uuid_invalid(struct kunit *test)
 	}
 }
 
+/*
+ * RFC 4122 section 4.4 says random UUIDs/GUIDs (version 4) must have:
+ *   - version 4 in the high nibble of the version byte,
+ *   - variant DCE 1.1 (binary 10x) in the high bits of byte 8.
+ *
+ * The version byte is byte 6 in the "wire" uuid_t layout and byte 7 in
+ * the byte-swapped guid_t layout.
+ */
+static void uuid_test_uuid_gen(struct kunit *test)
+{
+	uuid_t u;
+
+	for (unsigned int i = 0; i < 8; i++) {
+		uuid_gen(&u);
+		KUNIT_EXPECT_EQ(test, u.b[6] & 0xf0, 0x40);
+		KUNIT_EXPECT_EQ(test, u.b[8] & 0xc0, 0x80);
+	}
+}
+
+static void uuid_test_guid_gen(struct kunit *test)
+{
+	guid_t g;
+
+	for (unsigned int i = 0; i < 8; i++) {
+		guid_gen(&g);
+		KUNIT_EXPECT_EQ(test, g.b[7] & 0xf0, 0x40);
+		KUNIT_EXPECT_EQ(test, g.b[8] & 0xc0, 0x80);
+	}
+}
+
+static void uuid_test_generate_random_uuid(struct kunit *test)
+{
+	unsigned char buf[16];
+
+	for (unsigned int i = 0; i < 8; i++) {
+		generate_random_uuid(buf);
+		KUNIT_EXPECT_EQ(test, buf[6] & 0xf0, 0x40);
+		KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80);
+	}
+}
+
+static void uuid_test_generate_random_guid(struct kunit *test)
+{
+	unsigned char buf[16];
+
+	for (unsigned int i = 0; i < 8; i++) {
+		generate_random_guid(buf);
+		KUNIT_EXPECT_EQ(test, buf[7] & 0xf0, 0x40);
+		KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80);
+	}
+}
+
 static struct kunit_case uuid_test_cases[] = {
 	KUNIT_CASE(uuid_test_guid_valid),
 	KUNIT_CASE(uuid_test_uuid_valid),
 	KUNIT_CASE(uuid_test_guid_invalid),
 	KUNIT_CASE(uuid_test_uuid_invalid),
+	KUNIT_CASE(uuid_test_uuid_gen),
+	KUNIT_CASE(uuid_test_guid_gen),
+	KUNIT_CASE(uuid_test_generate_random_uuid),
+	KUNIT_CASE(uuid_test_generate_random_guid),
 	{},
 };
 
diff --git a/lib/usercopy.c b/lib/usercopy.c
index b00a3a957de6b..e2f0bf104a591 100644
--- a/lib/usercopy.c
+++ b/lib/usercopy.c
@@ -12,15 +12,13 @@
 
 /* out-of-line parts */
 
-#if !defined(INLINE_COPY_FROM_USER)
+#if !defined(INLINE_COPY_USER)
 unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n)
 {
 	return _inline_copy_from_user(to, from, n);
 }
 EXPORT_SYMBOL(_copy_from_user);
-#endif
 
-#if !defined(INLINE_COPY_TO_USER)
 unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n)
 {
 	return _inline_copy_to_user(to, from, n);
diff --git a/lib/vdso/datastore.c b/lib/vdso/datastore.c
index cf5d784a4a5a1..17d37b82ebc68 100644
--- a/lib/vdso/datastore.c
+++ b/lib/vdso/datastore.c
@@ -11,21 +11,21 @@
 static u8 vdso_initdata[VDSO_NR_PAGES * PAGE_SIZE] __aligned(PAGE_SIZE) __initdata = {};
 
 #ifdef CONFIG_GENERIC_GETTIMEOFDAY
-struct vdso_time_data *vdso_k_time_data __refdata =
+struct vdso_time_data *vdso_k_time_data __ro_after_init =
 	(void *)&vdso_initdata[VDSO_TIME_PAGE_OFFSET * PAGE_SIZE];
 
 static_assert(sizeof(struct vdso_time_data) <= PAGE_SIZE);
 #endif /* CONFIG_GENERIC_GETTIMEOFDAY */
 
 #ifdef CONFIG_VDSO_GETRANDOM
-struct vdso_rng_data *vdso_k_rng_data __refdata =
+struct vdso_rng_data *vdso_k_rng_data __ro_after_init =
 	(void *)&vdso_initdata[VDSO_RNG_PAGE_OFFSET * PAGE_SIZE];
 
 static_assert(sizeof(struct vdso_rng_data) <= PAGE_SIZE);
 #endif /* CONFIG_VDSO_GETRANDOM */
 
 #ifdef CONFIG_ARCH_HAS_VDSO_ARCH_DATA
-struct vdso_arch_data *vdso_k_arch_data __refdata =
+struct vdso_arch_data *vdso_k_arch_data __ro_after_init =
 	(void *)&vdso_initdata[VDSO_ARCH_PAGES_START * PAGE_SIZE];
 #endif /* CONFIG_ARCH_HAS_VDSO_ARCH_DATA */
 
diff --git a/lib/vsprintf.c b/lib/vsprintf.c
index 9f359b31c8d1c..9ca676feb8eb9 100644
--- a/lib/vsprintf.c
+++ b/lib/vsprintf.c
@@ -129,13 +129,6 @@ unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base)
 }
 EXPORT_SYMBOL(simple_strtoul);
 
-unsigned long simple_strntoul(const char *cp, char **endp, unsigned int base,
-			      size_t max_chars)
-{
-	return simple_strntoull(cp, endp, base, max_chars);
-}
-EXPORT_SYMBOL(simple_strntoul);
-
 /**
  * simple_strtol - convert a string to a signed long
  * @cp: The start of the string
@@ -2640,6 +2633,18 @@ static unsigned char spec_flag(unsigned char c)
 	return (c < sizeof(spec_flag_array)) ? spec_flag_array[c] : 0;
 }
 
+static void set_field_width(struct printf_spec *spec, int width)
+{
+	spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX);
+	WARN_ONCE(spec->field_width != width, "field width %d out of range", width);
+}
+
+static void set_precision(struct printf_spec *spec, int prec)
+{
+	spec->precision = clamp(prec, 0, PRECISION_MAX);
+	WARN_ONCE(spec->precision < prec, "precision %d too large", prec);
+}
+
 /*
  * Helper function to decode printf style format.
  * Each call decode a token from the format and return the
@@ -2710,7 +2715,7 @@ struct fmt format_decode(struct fmt fmt, struct printf_spec *spec)
 	spec->field_width = -1;
 
 	if (isdigit(*fmt.str))
-		spec->field_width = skip_atoi(&fmt.str);
+		set_field_width(spec, skip_atoi(&fmt.str));
 	else if (unlikely(*fmt.str == '*')) {
 		/* it's the next argument */
 		fmt.state = FORMAT_STATE_WIDTH;
@@ -2724,9 +2729,7 @@ struct fmt format_decode(struct fmt fmt, struct printf_spec *spec)
 	if (unlikely(*fmt.str == '.')) {
 		fmt.str++;
 		if (isdigit(*fmt.str)) {
-			spec->precision = skip_atoi(&fmt.str);
-			if (spec->precision < 0)
-				spec->precision = 0;
+			set_precision(spec, skip_atoi(&fmt.str));
 		} else if (*fmt.str == '*') {
 			/* it's the next argument */
 			fmt.state = FORMAT_STATE_PRECISION;
@@ -2799,24 +2802,6 @@ struct fmt format_decode(struct fmt fmt, struct printf_spec *spec)
 	return fmt;
 }
 
-static void
-set_field_width(struct printf_spec *spec, int width)
-{
-	spec->field_width = width;
-	if (WARN_ONCE(spec->field_width != width, "field width %d too large", width)) {
-		spec->field_width = clamp(width, -FIELD_WIDTH_MAX, FIELD_WIDTH_MAX);
-	}
-}
-
-static void
-set_precision(struct printf_spec *spec, int prec)
-{
-	spec->precision = prec;
-	if (WARN_ONCE(spec->precision != prec, "precision %d too large", prec)) {
-		spec->precision = clamp(prec, 0, PRECISION_MAX);
-	}
-}
-
 /*
  * Turn a 1/2/4-byte value into a 64-bit one for printing: truncate
  * as necessary and deal with signedness.
@@ -2864,6 +2849,7 @@ static unsigned long long convert_num_spec(unsigned int val, int size, struct pr
 int vsnprintf(char *buf, size_t size, const char *fmt_str, va_list args)
 {
 	char *str, *end;
+	size_t ret_size;
 	struct printf_spec spec = {0};
 	struct fmt fmt = {
 		.str = fmt_str,
@@ -2983,8 +2969,12 @@ int vsnprintf(char *buf, size_t size, const char *fmt_str, va_list args)
 	}
 
 	/* the trailing null byte doesn't count towards the total */
-	return str-buf;
+	ret_size = str - buf;
 
+	/* Make sure the return value is within the positive integer range */
+	if (WARN_ON_ONCE(ret_size > INT_MAX))
+		ret_size = INT_MAX;
+	return ret_size;
 }
 EXPORT_SYMBOL(vsnprintf);
 
@@ -3288,6 +3278,7 @@ int bstr_printf(char *buf, size_t size, const char *fmt_str, const u32 *bin_buf)
 	struct printf_spec spec = {0};
 	char *str, *end;
 	const char *args = (const char *)bin_buf;
+	size_t ret_size;
 
 	if (WARN_ON_ONCE(size > INT_MAX))
 		return 0;
@@ -3436,7 +3427,12 @@ int bstr_printf(char *buf, size_t size, const char *fmt_str, const u32 *bin_buf)
 #undef get_arg
 
 	/* the trailing null byte doesn't count towards the total */
-	return str - buf;
+	ret_size =  str - buf;
+
+	/* Make sure the return value is within the positive integer range */
+	if (WARN_ON_ONCE(ret_size > INT_MAX))
+		ret_size = INT_MAX;
+	return ret_size;
 }
 EXPORT_SYMBOL_GPL(bstr_printf);
 
diff --git a/localversion-next b/localversion-next
new file mode 100644
index 0000000000000..f1bb51f0e5f2e
--- /dev/null
+++ b/localversion-next
@@ -0,0 +1 @@
+-next-20260521
diff --git a/mm/Kconfig b/mm/Kconfig
index e8bf1e9e6ad90..9e0ca48249054 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -248,22 +248,75 @@ config SLUB_STATS
 	  out which slabs are relevant to a particular load.
 	  Try running: slabinfo -DA
 
-config RANDOM_KMALLOC_CACHES
-	default n
+config KMALLOC_PARTITION_CACHES
 	depends on !SLUB_TINY
-	bool "Randomize slab caches for normal kmalloc"
+	bool "Partitioned slab caches for normal kmalloc"
+	default RANDOM_KMALLOC_CACHES
 	help
-	  A hardening feature that creates multiple copies of slab caches for
-	  normal kmalloc allocation and makes kmalloc randomly pick one based
-	  on code address, which makes the attackers more difficult to spray
-	  vulnerable memory objects on the heap for the purpose of exploiting
-	  memory vulnerabilities.
+	  A hardening feature that creates multiple isolated copies of slab
+	  caches for normal kmalloc allocations. This makes it more difficult
+	  to exploit memory-safety vulnerabilities by attacking vulnerable
+	  co-located memory objects. Several modes are provided.
 
 	  Currently the number of copies is set to 16, a reasonably large value
 	  that effectively diverges the memory objects allocated for different
 	  subsystems or modules into different caches, at the expense of a
-	  limited degree of memory and CPU overhead that relates to hardware and
-	  system workload.
+	  limited degree of memory and CPU overhead that relates to hardware
+	  and system workload.
+
+choice
+	prompt "Partitioned slab cache mode"
+	depends on KMALLOC_PARTITION_CACHES
+	default KMALLOC_PARTITION_TYPED if CC_HAS_ALLOC_TOKEN
+	default KMALLOC_PARTITION_RANDOM
+	help
+	  Selects the slab cache partitioning mode.
+
+config KMALLOC_PARTITION_RANDOM
+	bool "Randomize slab caches for normal kmalloc"
+	help
+	  Randomly pick a slab cache based on code address and a per-boot
+	  random seed.
+
+	  This makes it harder for attackers to predict object co-location.
+	  The placement is random: while attackers don't know which kmalloc
+	  cache an object will be allocated from, they might circumvent
+	  the randomization by retrying attacks across multiple machines until
+	  the target objects are co-located.
+
+config KMALLOC_PARTITION_TYPED
+	bool "Type based slab cache selection for normal kmalloc"
+	depends on CC_HAS_ALLOC_TOKEN
+	help
+	  Rely on Clang's allocation tokens to choose a slab cache, where token
+	  IDs are derived from the allocated type.
+
+	  Unlike KMALLOC_PARTITION_RANDOM, cache assignment is deterministic based
+	  on type, which guarantees that objects of certain types are not
+	  placed in the same cache. This effectively mitigates certain classes
+	  of exploits that probabilistic defenses like KMALLOC_PARTITION_RANDOM
+	  only make harder but not impossible. However, this also means the
+	  cache assignment is predictable.
+
+	  Clang's default token ID calculation returns a bounded hash with
+	  disjoint ranges for pointer-containing and pointerless objects: when
+	  used as the slab cache index, this prevents buffer overflows on
+	  primitive buffers from directly corrupting pointer-containing
+	  objects.
+
+	  The current effectiveness of Clang's type inference can be judged by
+	  -Rpass=alloc-token, which provides diagnostics where (after dead-code
+	  elimination) type inference failed.
+
+	  Requires Clang 22 or later.
+
+endchoice
+
+config RANDOM_KMALLOC_CACHES
+	bool
+	transitional
+	help
+	  Transitional config for migration to KMALLOC_PARTITION_CACHES.
 
 endmenu # Slab allocator options
 
@@ -537,7 +590,7 @@ endchoice
 
 config MEMORY_HOTREMOVE
 	bool "Allow for memory hot remove"
-	select HAVE_BOOTMEM_INFO_NODE if (X86_64 || PPC64)
+	select HAVE_BOOTMEM_INFO_NODE if X86_64
 	depends on MEMORY_HOTPLUG
 	select MIGRATION
 
@@ -810,7 +863,6 @@ if TRANSPARENT_HUGEPAGE
 
 choice
 	prompt "Transparent Hugepage Support sysfs defaults"
-	depends on TRANSPARENT_HUGEPAGE
 	default TRANSPARENT_HUGEPAGE_ALWAYS
 	help
 	  Selects the sysfs defaults for Transparent Hugepage Support.
@@ -840,7 +892,6 @@ endchoice
 
 choice
 	prompt "Shmem hugepage allocation defaults"
-	depends on TRANSPARENT_HUGEPAGE
 	default TRANSPARENT_HUGEPAGE_SHMEM_HUGE_NEVER
 	help
 	  Selects the hugepage allocation policy defaults for
@@ -886,7 +937,6 @@ endchoice
 
 choice
 	prompt "Tmpfs hugepage allocation defaults"
-	depends on TRANSPARENT_HUGEPAGE
 	default TRANSPARENT_HUGEPAGE_TMPFS_HUGE_NEVER
 	help
 	  Selects the hugepage allocation policy defaults for
@@ -931,7 +981,7 @@ endchoice
 
 config THP_SWAP
 	def_bool y
-	depends on TRANSPARENT_HUGEPAGE && ARCH_WANTS_THP_SWAP && SWAP && 64BIT
+	depends on ARCH_WANTS_THP_SWAP && SWAP && 64BIT
 	help
 	  Swap transparent huge pages in one piece, without splitting.
 	  XXX: For now, swap cluster backing transparent huge page
@@ -939,17 +989,6 @@ config THP_SWAP
 
 	  For selection by architectures with reasonable THP sizes.
 
-config READ_ONLY_THP_FOR_FS
-	bool "Read-only THP for filesystems (EXPERIMENTAL)"
-	depends on TRANSPARENT_HUGEPAGE
-
-	help
-	  Allow khugepaged to put read-only file-backed pages in THP.
-
-	  This is marked experimental because it is a new feature. Write
-	  support of file THPs will be developed in the next few release
-	  cycles.
-
 config NO_PAGE_MAPCOUNT
 	bool "No per-page mapcount (EXPERIMENTAL)"
 	help
@@ -1303,7 +1342,7 @@ config ARCH_HAS_PTE_SPECIAL
 	bool
 
 config MAPPING_DIRTY_HELPERS
-        bool
+	bool
 
 config KMAP_LOCAL
 	bool
@@ -1434,7 +1473,7 @@ config ARCH_HAS_USER_SHADOW_STACK
 	bool
 	help
 	  The architecture has hardware support for userspace shadow call
-          stacks (eg, x86 CET, arm64 GCS or RISC-V Zicfiss).
+	  stacks (eg, x86 CET, arm64 GCS or RISC-V Zicfiss).
 
 config HAVE_ARCH_TLB_REMOVE_TABLE
 	def_bool n
diff --git a/mm/bootmem_info.c b/mm/bootmem_info.c
index 3d7675a3ae048..0fa78db7fbc0b 100644
--- a/mm/bootmem_info.c
+++ b/mm/bootmem_info.c
@@ -19,7 +19,6 @@ void get_page_bootmem(unsigned long info, struct page *page,
 {
 	BUG_ON(type > 0xf);
 	BUG_ON(info > (ULONG_MAX >> 4));
-	SetPagePrivate(page);
 	set_page_private(page, info << 4 | type);
 	page_ref_inc(page);
 }
@@ -32,20 +31,15 @@ void put_page_bootmem(struct page *page)
 	       type > MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE);
 
 	if (page_ref_dec_return(page) == 1) {
-		ClearPagePrivate(page);
 		set_page_private(page, 0);
-		INIT_LIST_HEAD(&page->lru);
-		kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE);
 		free_reserved_page(page);
 	}
 }
 
 static void __init register_page_bootmem_info_section(unsigned long start_pfn)
 {
-	unsigned long mapsize, section_nr, i;
+	unsigned long section_nr;
 	struct mem_section *ms;
-	struct mem_section_usage *usage;
-	struct page *page;
 
 	start_pfn = SECTION_ALIGN_DOWN(start_pfn);
 	section_nr = pfn_to_section_nr(start_pfn);
@@ -54,27 +48,12 @@ static void __init register_page_bootmem_info_section(unsigned long start_pfn)
 	if (!preinited_vmemmap_section(ms))
 		register_page_bootmem_memmap(section_nr, pfn_to_page(start_pfn),
 					     PAGES_PER_SECTION);
-
-	usage = ms->usage;
-	page = virt_to_page(usage);
-
-	mapsize = PAGE_ALIGN(mem_section_usage_size()) >> PAGE_SHIFT;
-
-	for (i = 0; i < mapsize; i++, page++)
-		get_page_bootmem(section_nr, page, MIX_SECTION_INFO);
 }
 
 void __init register_page_bootmem_info_node(struct pglist_data *pgdat)
 {
-	unsigned long i, pfn, end_pfn, nr_pages;
+	unsigned long pfn, end_pfn;
 	int node = pgdat->node_id;
-	struct page *page;
-
-	nr_pages = PAGE_ALIGN(sizeof(struct pglist_data)) >> PAGE_SHIFT;
-	page = virt_to_page(pgdat);
-
-	for (i = 0; i < nr_pages; i++, page++)
-		get_page_bootmem(node, page, NODE_INFO);
 
 	pfn = pgdat->node_start_pfn;
 	end_pfn = pgdat_end_pfn(pgdat);
diff --git a/mm/compaction.c b/mm/compaction.c
index 3648ce22c8072..168e63940b782 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -2340,7 +2340,8 @@ static enum compact_result __compact_finished(struct compact_control *cc)
 		 * Job done if allocation would steal freepages from
 		 * other migratetype buddy lists.
 		 */
-		if (find_suitable_fallback(area, order, migratetype, true) >= 0)
+		if (find_suitable_fallback(area, order, migratetype, true, NULL)
+		    == FALLBACK_FOUND)
 			/*
 			 * Movable pages are OK in any pageblock. If we are
 			 * stealing for a non-movable allocation, make sure
diff --git a/mm/damon/core.c b/mm/damon/core.c
index 3dbbbfdeff719..3a8725e400c6b 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -13,10 +13,14 @@
 #include <linux/memcontrol.h>
 #include <linux/mm.h>
 #include <linux/psi.h>
+#include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/string_choices.h>
 
+/* for damon_get_folio() used by node eligible memory metrics */
+#include "ops-common.h"
+
 #define CREATE_TRACE_POINTS
 #include <trace/events/damon.h>
 
@@ -607,6 +611,8 @@ struct damon_ctx *damon_new_ctx(void)
 	INIT_LIST_HEAD(&ctx->adaptive_targets);
 	INIT_LIST_HEAD(&ctx->schemes);
 
+	prandom_seed_state(&ctx->rnd_state, get_random_u64());
+
 	return ctx;
 }
 
@@ -918,6 +924,8 @@ static int damos_commit_quota(struct damos_quota *dst, struct damos_quota *src)
 	if (err)
 		return err;
 	dst->goal_tuner = src->goal_tuner;
+	dst->fail_charge_num = src->fail_charge_num;
+	dst->fail_charge_denom = src->fail_charge_denom;
 	dst->weight_sz = src->weight_sz;
 	dst->weight_nr_accesses = src->weight_nr_accesses;
 	dst->weight_age = src->weight_age;
@@ -1326,11 +1334,26 @@ static int damon_commit_targets(
 int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
 {
 	int err;
+	struct damos *scheme;
+	struct damos_quota_goal *goal;
 
 	dst->maybe_corrupted = true;
 	if (!is_power_of_2(src->min_region_sz))
 		return -EINVAL;
 
+	/* node_eligible_mem_bp metric requires PADDR ops */
+	if (src->ops.id != DAMON_OPS_PADDR) {
+		damon_for_each_scheme(scheme, src) {
+			struct damos_quota *quota = &scheme->quota;
+
+			damos_for_each_quota_goal(goal, quota) {
+				if (goal->metric ==
+						DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP)
+					return -EINVAL;
+			}
+		}
+	}
+
 	err = damon_commit_schemes(dst, src);
 	if (err)
 		return err;
@@ -1349,6 +1372,7 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
 		if (err)
 			return err;
 	}
+	dst->pause = src->pause;
 	dst->ops = src->ops;
 	dst->addr_unit = src->addr_unit;
 	dst->min_region_sz = src->min_region_sz;
@@ -2046,6 +2070,37 @@ static void damos_walk_cancel(struct damon_ctx *ctx)
 	mutex_unlock(&ctx->walk_control_lock);
 }
 
+static void damos_charge_quota(struct damos_quota *quota,
+		unsigned long sz_region, unsigned long sz_applied)
+{
+	/*
+	 * sz_applied could be bigger than sz_region, depending on ops
+	 * implementation of the action, e.g., damos_pa_pageout().  Charge only
+	 * the region size in the case.
+	 */
+	if (!quota->fail_charge_denom || sz_applied > sz_region)
+		quota->charged_sz += sz_region;
+	else
+		quota->charged_sz += sz_applied + mult_frac(
+				(sz_region - sz_applied),
+				quota->fail_charge_num,
+				quota->fail_charge_denom);
+}
+
+static bool damos_quota_is_full(struct damos_quota *quota,
+		unsigned long min_region_sz)
+{
+	if (!damos_quota_is_set(quota))
+		return false;
+	if (quota->charged_sz >= quota->esz)
+		return true;
+	/*
+	 * DAMOS action is applied per region, so <min_region_sz remaining
+	 * quota means the quota is effectively full.
+	 */
+	return quota->esz - quota->charged_sz < min_region_sz;
+}
+
 static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
 		struct damon_region *r, struct damos *s)
 {
@@ -2102,11 +2157,10 @@ static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
 		ktime_get_coarse_ts64(&end);
 		quota->total_charged_ns += timespec64_to_ns(&end) -
 			timespec64_to_ns(&begin);
-		quota->charged_sz += sz;
-		if (damos_quota_is_set(quota) &&
-				quota->charged_sz >= quota->esz) {
+		damos_charge_quota(quota, sz, sz_applied);
+		if (damos_quota_is_full(quota, c->min_region_sz)) {
 			quota->charge_target_from = t;
-			quota->charge_addr_from = r->ar.end + 1;
+			quota->charge_addr_from = r->ar.end;
 		}
 	}
 	if (s->action != DAMOS_STAT)
@@ -2132,8 +2186,7 @@ static void damon_do_apply_schemes(struct damon_ctx *c,
 			continue;
 
 		/* Check the quota */
-		if (damos_quota_is_set(quota) &&
-				quota->charged_sz >= quota->esz)
+		if (damos_quota_is_full(quota, c->min_region_sz))
 			continue;
 
 		if (damos_skip_charged_region(t, r, s, c->min_region_sz))
@@ -2151,6 +2204,58 @@ static void damon_do_apply_schemes(struct damon_ctx *c,
 	}
 }
 
+/*
+ * damos_apply_target() - Apply DAMOS schemes to a given target.
+ * @c:			monitoring context to apply its DAMOS schemes to..
+ * @t:			monitoring target to apply the schemes to.
+ * @max_region_sz:	maximum region size for @c.
+ *
+ * This function could split regions for keeping the quota.  To minimize
+ * overhead from the split operations increased number of regions, this
+ * function will also merge regions after the schemes applying attempt is done,
+ * for each region.  The merge operation is made only when it doesn't lose the
+ * monitoring information and not violating @max_region_sz.
+ *
+ * Hence, after this function is called, the total number of regions could
+ * be increased or reduced.  The increase could make max_nr_regions temporarily
+ * be violated, until the next per-aggregation interval regions merge operation
+ * is executed.  The decrease will not violate min_nr_regions though, since it
+ * keeps @max_region_sz.
+ */
+static void damos_apply_target(struct damon_ctx *c, struct damon_target *t,
+		unsigned long max_region_sz)
+{
+	struct damon_region *r;
+
+	damon_for_each_region(r, t) {
+		struct damon_region *prev_r;
+
+		damon_do_apply_schemes(c, t, r);
+		/*
+		 * damon_do_apply_scheems() could split the region for the
+		 * quota.  Keeping the new slices is an overhead.  Merge back
+		 * the slices into the previous region if it doesn't lose any
+		 * information and not violating the max_region_sz.
+		 */
+		if (damon_first_region(t) == r)
+			continue;
+		prev_r = damon_prev_region(r);
+		if (prev_r->ar.end != r->ar.start)
+			continue;
+		if (prev_r->age != r->age)
+			continue;
+		if (prev_r->last_nr_accesses != r->last_nr_accesses)
+			continue;
+		if (prev_r->nr_accesses != r->nr_accesses)
+			continue;
+		if (r->ar.end - prev_r->ar.start > max_region_sz)
+			continue;
+		prev_r->ar.end = r->ar.end;
+		damon_destroy_region(r, t);
+		r = prev_r;
+	}
+}
+
 /*
  * damon_feed_loop_next_input() - get next input to achieve a target score.
  * @last_input	The last input.
@@ -2287,7 +2392,115 @@ static unsigned long damos_get_node_memcg_used_bp(
 		numerator = i.totalram - used_pages;
 	return mult_frac(numerator, 10000, i.totalram);
 }
-#else
+
+#ifdef CONFIG_DAMON_PADDR
+/*
+ * damos_calc_eligible_bytes() - Calculate raw eligible bytes per node.
+ * @c:		The DAMON context.
+ * @s:		The scheme.
+ * @nid:	The target NUMA node id.
+ * @total:	Output for total eligible bytes across all nodes.
+ *
+ * Iterates through each folio in eligible regions to accurately determine
+ * which node the memory resides on. Returns eligible bytes on the specified
+ * node and sets *total to the sum across all nodes.
+ *
+ * Note: This function requires damon_get_folio() from ops-common.c, which is
+ * only available when CONFIG_DAMON_PADDR is enabled. It also requires the
+ * context to be using PADDR operations for meaningful results.
+ */
+static phys_addr_t damos_calc_eligible_bytes(struct damon_ctx *c,
+		struct damos *s, int nid, phys_addr_t *total)
+{
+	struct damon_target *t;
+	struct damon_region *r;
+	phys_addr_t total_eligible = 0;
+	phys_addr_t node_eligible = 0;
+
+	damon_for_each_target(t, c) {
+		damon_for_each_region(r, t) {
+			phys_addr_t addr, end_addr;
+
+			if (!__damos_valid_target(r, s))
+				continue;
+
+			/* Convert from core address units to physical bytes */
+			addr = (phys_addr_t)r->ar.start * c->addr_unit;
+			end_addr = (phys_addr_t)r->ar.end * c->addr_unit;
+			while (addr < end_addr) {
+				struct folio *folio;
+				phys_addr_t folio_start, folio_end;
+				phys_addr_t overlap_start, overlap_end;
+				phys_addr_t counted;
+
+				folio = damon_get_folio(PHYS_PFN(addr));
+				if (!folio) {
+					addr = PAGE_ALIGN_DOWN(addr +
+							PAGE_SIZE);
+					if (!addr)
+						break;
+					continue;
+				}
+
+				/*
+				 * Calculate exact overlap between the region
+				 * [addr, end_addr) and the folio range.
+				 * The folio may start before addr if addr is
+				 * in the middle of a large folio.
+				 */
+				folio_start = PFN_PHYS(folio_pfn(folio));
+				folio_end = folio_start + folio_size(folio);
+
+				overlap_start = max(addr, folio_start);
+				overlap_end = min(end_addr, folio_end);
+
+				if (overlap_end > overlap_start) {
+					counted = overlap_end - overlap_start;
+					total_eligible += counted;
+					if (folio_nid(folio) == nid)
+						node_eligible += counted;
+				}
+
+				/* Advance past the entire folio */
+				addr = folio_end;
+				folio_put(folio);
+			}
+			cond_resched();
+		}
+	}
+
+	*total = total_eligible;
+	return node_eligible;
+}
+
+static unsigned long damos_get_node_eligible_mem_bp(struct damon_ctx *c,
+		struct damos *s, int nid)
+{
+	phys_addr_t total_eligible = 0;
+	phys_addr_t node_eligible;
+
+	if (c->ops.id != DAMON_OPS_PADDR)
+		return 0;
+
+	if (nid < 0 || nid >= MAX_NUMNODES || !node_online(nid))
+		return 0;
+
+	node_eligible = damos_calc_eligible_bytes(c, s, nid, &total_eligible);
+
+	if (!(unsigned long)total_eligible)
+		return 0;
+
+	return mult_frac((unsigned long)node_eligible, 10000,
+			(unsigned long)total_eligible);
+}
+#else /* CONFIG_DAMON_PADDR */
+static unsigned long damos_get_node_eligible_mem_bp(struct damon_ctx *c,
+		struct damos *s, int nid)
+{
+	return 0;
+}
+#endif /* CONFIG_DAMON_PADDR */
+#else /* CONFIG_NUMA */
 static __kernel_ulong_t damos_get_node_mem_bp(
 		struct damos_quota_goal *goal)
 {
@@ -2299,7 +2512,13 @@ static unsigned long damos_get_node_memcg_used_bp(
 {
 	return 0;
 }
-#endif
+
+static unsigned long damos_get_node_eligible_mem_bp(struct damon_ctx *c,
+		struct damos *s, int nid)
+{
+	return 0;
+}
+#endif /* CONFIG_NUMA */
 
 /*
  * Returns LRU-active or inactive memory to total LRU memory size ratio.
@@ -2319,7 +2538,8 @@ static unsigned int damos_get_in_active_mem_bp(bool active_ratio)
 	return mult_frac(inactive, 10000, total);
 }
 
-static void damos_set_quota_goal_current_value(struct damos_quota_goal *goal)
+static void damos_set_quota_goal_current_value(struct damon_ctx *c,
+		struct damos *s, struct damos_quota_goal *goal)
 {
 	u64 now_psi_total;
 
@@ -2345,19 +2565,24 @@ static void damos_set_quota_goal_current_value(struct damos_quota_goal *goal)
 		goal->current_value = damos_get_in_active_mem_bp(
 				goal->metric == DAMOS_QUOTA_ACTIVE_MEM_BP);
 		break;
+	case DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP:
+		goal->current_value = damos_get_node_eligible_mem_bp(c, s,
+				goal->nid);
+		break;
 	default:
 		break;
 	}
 }
 
 /* Return the highest score since it makes schemes least aggressive */
-static unsigned long damos_quota_score(struct damos_quota *quota)
+static unsigned long damos_quota_score(struct damon_ctx *c, struct damos *s)
 {
 	struct damos_quota_goal *goal;
+	struct damos_quota *quota = &s->quota;
 	unsigned long highest_score = 0;
 
 	damos_for_each_quota_goal(goal, quota) {
-		damos_set_quota_goal_current_value(goal);
+		damos_set_quota_goal_current_value(c, s, goal);
 		highest_score = max(highest_score,
 				mult_frac(goal->current_value, 10000,
 					goal->target_value));
@@ -2366,17 +2591,20 @@ static unsigned long damos_quota_score(struct damos_quota *quota)
 	return highest_score;
 }
 
-static void damos_goal_tune_esz_bp_consist(struct damos_quota *quota)
+static void damos_goal_tune_esz_bp_consist(struct damon_ctx *c, struct damos *s)
 {
-	unsigned long score = damos_quota_score(quota);
+	struct damos_quota *quota = &s->quota;
+	unsigned long score = damos_quota_score(c, s);
 
 	quota->esz_bp = damon_feed_loop_next_input(
 			max(quota->esz_bp, 10000UL), score);
 }
 
-static void damos_goal_tune_esz_bp_temporal(struct damos_quota *quota)
+static void damos_goal_tune_esz_bp_temporal(struct damon_ctx *c,
+		struct damos *s)
 {
-	unsigned long score = damos_quota_score(quota);
+	struct damos_quota *quota = &s->quota;
+	unsigned long score = damos_quota_score(c, s);
 
 	if (score >= 10000)
 		quota->esz_bp = 0;
@@ -2389,9 +2617,9 @@ static void damos_goal_tune_esz_bp_temporal(struct damos_quota *quota)
 /*
  * Called only if quota->ms, or quota->sz are set, or quota->goals is not empty
  */
-static void damos_set_effective_quota(struct damos_quota *quota,
-		struct damon_ctx *ctx)
+static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s)
 {
+	struct damos_quota *quota = &s->quota;
 	unsigned long throughput;
 	unsigned long esz = ULONG_MAX;
 
@@ -2402,9 +2630,9 @@ static void damos_set_effective_quota(struct damos_quota *quota,
 
 	if (!list_empty(&quota->goals)) {
 		if (quota->goal_tuner == DAMOS_QUOTA_GOAL_TUNER_CONSIST)
-			damos_goal_tune_esz_bp_consist(quota);
+			damos_goal_tune_esz_bp_consist(ctx, s);
 		else if (quota->goal_tuner == DAMOS_QUOTA_GOAL_TUNER_TEMPORAL)
-			damos_goal_tune_esz_bp_temporal(quota);
+			damos_goal_tune_esz_bp_temporal(ctx, s);
 		esz = quota->esz_bp / 10000;
 	}
 
@@ -2452,22 +2680,21 @@ static void damos_adjust_quota(struct damon_ctx *c, struct damos *s)
 	/* First charge window */
 	if (!quota->total_charged_sz && !quota->charged_from) {
 		quota->charged_from = jiffies;
-		damos_set_effective_quota(quota, c);
+		damos_set_effective_quota(c, s);
 	}
 
 	/* New charge window starts */
 	if (!time_in_range_open(jiffies, quota->charged_from,
 				quota->charged_from +
 				msecs_to_jiffies(quota->reset_interval))) {
-		if (damos_quota_is_set(quota) &&
-				quota->charged_sz >= quota->esz)
+		if (damos_quota_is_full(quota, c->min_region_sz))
 			s->stat.qt_exceeds++;
 		quota->total_charged_sz += quota->charged_sz;
 		quota->charged_from = jiffies;
 		quota->charged_sz = 0;
 		if (trace_damos_esz_enabled())
 			cached_esz = quota->esz;
-		damos_set_effective_quota(quota, c);
+		damos_set_effective_quota(c, s);
 		if (trace_damos_esz_enabled() && quota->esz != cached_esz)
 			damos_trace_esz(c, s, quota);
 	}
@@ -2521,9 +2748,9 @@ static void damos_trace_stat(struct damon_ctx *c, struct damos *s)
 static void kdamond_apply_schemes(struct damon_ctx *c)
 {
 	struct damon_target *t;
-	struct damon_region *r;
 	struct damos *s;
 	bool has_schemes_to_apply = false;
+	unsigned long max_region_sz;
 
 	damon_for_each_scheme(s, c) {
 		if (time_before(c->passed_sample_intervals, s->next_apply_sis))
@@ -2540,13 +2767,12 @@ static void kdamond_apply_schemes(struct damon_ctx *c)
 	if (!has_schemes_to_apply)
 		return;
 
+	max_region_sz = damon_region_sz_limit(c);
 	mutex_lock(&c->walk_control_lock);
 	damon_for_each_target(t, c) {
 		if (c->ops.target_valid && c->ops.target_valid(t) == false)
 			continue;
-
-		damon_for_each_region(r, t)
-			damon_do_apply_schemes(c, t, r);
+		damos_apply_target(c, t, max_region_sz);
 	}
 
 	damon_for_each_scheme(s, c) {
@@ -2715,8 +2941,9 @@ static void damon_split_region_at(struct damon_target *t,
 }
 
 /* Split every region in the given target into 'nr_subs' regions */
-static void damon_split_regions_of(struct damon_target *t, int nr_subs,
-				  unsigned long min_region_sz)
+static void damon_split_regions_of(struct damon_ctx *ctx,
+				   struct damon_target *t, int nr_subs,
+				   unsigned long min_region_sz)
 {
 	struct damon_region *r, *next;
 	unsigned long sz_region, sz_sub = 0;
@@ -2731,7 +2958,7 @@ static void damon_split_regions_of(struct damon_target *t, int nr_subs,
 			 * Randomly select size of left sub-region to be at
 			 * least 10 percent and at most 90% of original region
 			 */
-			sz_sub = ALIGN_DOWN(damon_rand(1, 10) *
+			sz_sub = ALIGN_DOWN(damon_rand(ctx, 1, 10) *
 					sz_region / 10, min_region_sz);
 			/* Do not allow blank region */
 			if (sz_sub == 0 || sz_sub >= sz_region)
@@ -2772,7 +2999,8 @@ static void kdamond_split_regions(struct damon_ctx *ctx)
 		nr_subregions = 3;
 
 	damon_for_each_target(t, ctx)
-		damon_split_regions_of(t, nr_subregions, ctx->min_region_sz);
+		damon_split_regions_of(ctx, t, nr_subregions,
+				       ctx->min_region_sz);
 
 	last_nr_regions = nr_regions;
 }
@@ -3014,6 +3242,14 @@ static int kdamond_fn(void *data)
 		kdamond_call(ctx, false);
 		if (ctx->maybe_corrupted)
 			break;
+		while (ctx->pause) {
+			damos_walk_cancel(ctx);
+			kdamond_usleep(ctx->attrs.sample_interval);
+			/* allow caller unset pause via damon_call() */
+			kdamond_call(ctx, false);
+			if (kdamond_need_stop(ctx) || ctx->maybe_corrupted)
+				goto done;
+		}
 		if (!list_empty(&ctx->schemes))
 			kdamond_apply_schemes(ctx);
 		else
@@ -3096,14 +3332,20 @@ static int kdamond_fn(void *data)
 	return 0;
 }
 
-static int walk_system_ram(struct resource *res, void *arg)
+struct damon_system_ram_range_walk_arg {
+	bool walked;
+	struct resource res;
+};
+
+static int damon_system_ram_walk_fn(struct resource *res, void *arg)
 {
-	struct resource *a = arg;
+	struct damon_system_ram_range_walk_arg *a = arg;
 
-	if (resource_size(a) < resource_size(res)) {
-		a->start = res->start;
-		a->end = res->end;
+	if (!a->walked) {
+		a->walked = true;
+		a->res.start = res->start;
 	}
+	a->res.end = res->end;
 	return 0;
 }
 
@@ -3120,27 +3362,24 @@ static unsigned long damon_res_to_core_addr(resource_size_t ra,
 	return ra / addr_unit;
 }
 
-/*
- * Find biggest 'System RAM' resource and store its start and end address in
- * @start and @end, respectively.  If no System RAM is found, returns false.
- */
-static bool damon_find_biggest_system_ram(unsigned long *start,
+static bool damon_find_system_rams_range(unsigned long *start,
 		unsigned long *end, unsigned long addr_unit)
-
 {
-	struct resource res = {};
+	struct damon_system_ram_range_walk_arg arg = {};
 
-	walk_system_ram_res(0, -1, &res, walk_system_ram);
-	*start = damon_res_to_core_addr(res.start, addr_unit);
-	*end = damon_res_to_core_addr(res.end + 1, addr_unit);
+	walk_system_ram_res(0, -1, &arg, damon_system_ram_walk_fn);
+	if (!arg.walked)
+		return false;
+	*start = damon_res_to_core_addr(arg.res.start, addr_unit);
+	*end = damon_res_to_core_addr(arg.res.end + 1, addr_unit);
 	if (*end <= *start)
 		return false;
 	return true;
 }
 
 /**
- * damon_set_region_biggest_system_ram_default() - Set the region of the given
- * monitoring target as requested, or biggest 'System RAM'.
+ * damon_set_region_system_rams_default() - Set the region of the given
+ * monitoring target as requested, or to cover all 'System RAM' resources.
  * @t:		The monitoring target to set the region.
  * @start:	The pointer to the start address of the region.
  * @end:	The pointer to the end address of the region.
@@ -3148,14 +3387,14 @@ static bool damon_find_biggest_system_ram(unsigned long *start,
  * @min_region_sz:	Minimum region size.
  *
  * This function sets the region of @t as requested by @start and @end.  If the
- * values of @start and @end are zero, however, this function finds the biggest
- * 'System RAM' resource and sets the region to cover the resource.  In the
- * latter case, this function saves the start and end addresses of the resource
- * in @start and @end, respectively.
+ * values of @start and @end are zero, however, this function finds 'System
+ * RAM' resources and sets the region to cover all the resource.  In the latter
+ * case, this function saves the start and the end addresseses of the first and
+ * the last resources in @start and @end, respectively.
  *
  * Return: 0 on success, negative error code otherwise.
  */
-int damon_set_region_biggest_system_ram_default(struct damon_target *t,
+int damon_set_region_system_rams_default(struct damon_target *t,
 			unsigned long *start, unsigned long *end,
 			unsigned long addr_unit, unsigned long min_region_sz)
 {
@@ -3165,7 +3404,7 @@ int damon_set_region_biggest_system_ram_default(struct damon_target *t,
 		return -EINVAL;
 
 	if (!*start && !*end &&
-			!damon_find_biggest_system_ram(start, end, addr_unit))
+		!damon_find_system_rams_range(start, end, addr_unit))
 		return -EINVAL;
 
 	addr_range.start = *start;
diff --git a/mm/damon/lru_sort.c b/mm/damon/lru_sort.c
index 8494040b1ee48..eca88ed941b32 100644
--- a/mm/damon/lru_sort.c
+++ b/mm/damon/lru_sort.c
@@ -39,7 +39,6 @@ static bool enabled __read_mostly;
  * the re-reading, DAMON_LRU_SORT will be disabled.
  */
 static bool commit_inputs __read_mostly;
-module_param(commit_inputs, bool, 0600);
 
 /*
  * Desired active to [in]active memory ratio in bp (1/10,000).
@@ -140,7 +139,8 @@ DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_lru_sort_mon_attrs);
  * Start of the target memory region in physical address.
  *
  * The start physical address of memory region that DAMON_LRU_SORT will do work
- * against.  By default, biggest System RAM is used as the region.
+ * against.  By default, the system's entire physical memory is used as the
+ * region.
  */
 static unsigned long monitor_region_start __read_mostly;
 module_param(monitor_region_start, ulong, 0600);
@@ -149,7 +149,8 @@ module_param(monitor_region_start, ulong, 0600);
  * End of the target memory region in physical address.
  *
  * The end physical address of memory region that DAMON_LRU_SORT will do work
- * against.  By default, biggest System RAM is used as the region.
+ * against.  By default, the system's entire physical memory is used as the
+ * region.
  */
 static unsigned long monitor_region_end __read_mostly;
 module_param(monitor_region_end, ulong, 0600);
@@ -285,6 +286,11 @@ static int damon_lru_sort_apply_parameters(void)
 	param_ctx->addr_unit = addr_unit;
 	param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1);
 
+	if (!is_power_of_2(param_ctx->min_region_sz)) {
+		err = -EINVAL;
+		goto out;
+	}
+
 	if (!damon_lru_sort_mon_attrs.sample_interval) {
 		err = -EINVAL;
 		goto out;
@@ -327,7 +333,7 @@ static int damon_lru_sort_apply_parameters(void)
 	if (err)
 		goto out;
 
-	err = damon_set_region_biggest_system_ram_default(param_target,
+	err = damon_set_region_system_rams_default(param_target,
 					&monitor_region_start,
 					&monitor_region_end,
 					param_ctx->addr_unit,
@@ -340,18 +346,51 @@ static int damon_lru_sort_apply_parameters(void)
 	return err;
 }
 
-static int damon_lru_sort_handle_commit_inputs(void)
+static int damon_lru_sort_commit_inputs_fn(void *arg)
+{
+	return damon_lru_sort_apply_parameters();
+}
+
+static int damon_lru_sort_commit_inputs_store(const char *val,
+					      const struct kernel_param *kp)
 {
+	bool commit_inputs_request;
 	int err;
+	struct damon_call_control control = {
+		.fn = damon_lru_sort_commit_inputs_fn,
+	};
+
+	if (!val) {
+		commit_inputs_request = true;
+	} else {
+		err = kstrtobool(val, &commit_inputs_request);
+		if (err)
+			return err;
+	}
 
-	if (!commit_inputs)
+	if (!commit_inputs_request)
 		return 0;
 
-	err = damon_lru_sort_apply_parameters();
-	commit_inputs = false;
-	return err;
+	/*
+	 * Skip damon_call() if ctx is not initialized to avoid
+	 * NULL pointer dereference.
+	 */
+	if (!ctx)
+		return -EINVAL;
+
+	err = damon_call(ctx, &control);
+
+	return err ? err : control.return_code;
 }
 
+static const struct kernel_param_ops commit_inputs_param_ops = {
+	.flags = KERNEL_PARAM_OPS_FL_NOARG,
+	.set = damon_lru_sort_commit_inputs_store,
+	.get = param_get_bool,
+};
+
+module_param_cb(commit_inputs, &commit_inputs_param_ops, &commit_inputs, 0600);
+
 static int damon_lru_sort_damon_call_fn(void *arg)
 {
 	struct damon_ctx *c = arg;
@@ -365,7 +404,7 @@ static int damon_lru_sort_damon_call_fn(void *arg)
 			damon_lru_sort_cold_stat = s->stat;
 	}
 
-	return damon_lru_sort_handle_commit_inputs();
+	return 0;
 }
 
 static struct damon_call_control call_control = {
diff --git a/mm/damon/ops-common.c b/mm/damon/ops-common.c
index 8c6d613425c13..3a0ddc3ac7196 100644
--- a/mm/damon/ops-common.c
+++ b/mm/damon/ops-common.c
@@ -117,9 +117,12 @@ int damon_hot_score(struct damon_ctx *c, struct damon_region *r,
 		damon_max_nr_accesses(&c->attrs);
 
 	age_in_sec = (unsigned long)r->age * c->attrs.aggr_interval / 1000000;
-	for (age_in_log = 0; age_in_log < DAMON_MAX_AGE_IN_LOG && age_in_sec;
-			age_in_log++, age_in_sec >>= 1)
-		;
+	if (age_in_sec)
+		age_in_log = min_t(int, ilog2(age_in_sec) + 1,
+				DAMON_MAX_AGE_IN_LOG);
+	else
+		age_in_log = 0;
+
 
 	/* If frequency is 0, higher age means it's colder */
 	if (freq_subscore == 0)
diff --git a/mm/damon/paddr.c b/mm/damon/paddr.c
index 5cdcc5037cbc1..c4738cd5e221e 100644
--- a/mm/damon/paddr.c
+++ b/mm/damon/paddr.c
@@ -49,11 +49,11 @@ static void damon_pa_mkold(phys_addr_t paddr)
 }
 
 static void __damon_pa_prepare_access_check(struct damon_region *r,
-		unsigned long addr_unit)
+		struct damon_ctx *ctx)
 {
-	r->sampling_addr = damon_rand(r->ar.start, r->ar.end);
+	r->sampling_addr = damon_rand(ctx, r->ar.start, r->ar.end);
 
-	damon_pa_mkold(damon_pa_phys_addr(r->sampling_addr, addr_unit));
+	damon_pa_mkold(damon_pa_phys_addr(r->sampling_addr, ctx->addr_unit));
 }
 
 static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
@@ -63,7 +63,7 @@ static void damon_pa_prepare_access_checks(struct damon_ctx *ctx)
 
 	damon_for_each_target(t, ctx) {
 		damon_for_each_region(r, t)
-			__damon_pa_prepare_access_check(r, ctx->addr_unit);
+			__damon_pa_prepare_access_check(r, ctx);
 	}
 }
 
diff --git a/mm/damon/reclaim.c b/mm/damon/reclaim.c
index fe7fce26cf6ce..ed446d00ef1cf 100644
--- a/mm/damon/reclaim.c
+++ b/mm/damon/reclaim.c
@@ -39,7 +39,6 @@ static bool enabled __read_mostly;
  * re-reading, DAMON_RECLAIM will be disabled.
  */
 static bool commit_inputs __read_mostly;
-module_param(commit_inputs, bool, 0600);
 
 /*
  * Time threshold for cold memory regions identification in microseconds.
@@ -92,6 +91,20 @@ module_param(quota_mem_pressure_us, ulong, 0600);
 static unsigned long quota_autotune_feedback __read_mostly;
 module_param(quota_autotune_feedback, ulong, 0600);
 
+/*
+ * Auto-tune monitoring intervals.
+ *
+ * If this parameter is set as ``Y``, DAMON_RECLAIM automatically tunes DAMON's
+ * sampling and aggregation intervals.  The auto-tuning aims to capture
+ * meaningful amount of access events in each DAMON-snapshot, while keeping the
+ * sampling intervals 5 milliseconds in minimum, and 10 seconds in maximum.
+ * Setting this as ``N`` disables the auto-tuning.
+ *
+ * Disabled by default.
+ */
+static bool autotune_monitoring_intervals __read_mostly;
+module_param(autotune_monitoring_intervals, bool, 0600);
+
 static struct damos_watermarks damon_reclaim_wmarks = {
 	.metric = DAMOS_WMARK_FREE_MEM_RATE,
 	.interval = 5000000,	/* 5 seconds */
@@ -114,7 +127,8 @@ DEFINE_DAMON_MODULES_MON_ATTRS_PARAMS(damon_reclaim_mon_attrs);
  * Start of the target memory region in physical address.
  *
  * The start physical address of memory region that DAMON_RECLAIM will do work
- * against.  By default, biggest System RAM is used as the region.
+ * against.  By default, the system's entire physical memory is used as the
+ * region.
  */
 static unsigned long monitor_region_start __read_mostly;
 module_param(monitor_region_start, ulong, 0600);
@@ -123,7 +137,8 @@ module_param(monitor_region_start, ulong, 0600);
  * End of the target memory region in physical address.
  *
  * The end physical address of memory region that DAMON_RECLAIM will do work
- * against.  By default, biggest System RAM is used as the region.
+ * against.  By default, the system's entire physical memory is used as the
+ * region.
  */
 static unsigned long monitor_region_end __read_mostly;
 module_param(monitor_region_end, ulong, 0600);
@@ -151,7 +166,7 @@ DEFINE_DAMON_MODULES_DAMOS_STATS_PARAMS(damon_reclaim_stat,
 static struct damon_ctx *ctx;
 static struct damon_target *target;
 
-static struct damos *damon_reclaim_new_scheme(void)
+static struct damos *damon_reclaim_new_scheme(unsigned long aggr_interval)
 {
 	struct damos_access_pattern pattern = {
 		/* Find regions having PAGE_SIZE or larger size */
@@ -161,8 +176,7 @@ static struct damos *damon_reclaim_new_scheme(void)
 		.min_nr_accesses = 0,
 		.max_nr_accesses = 0,
 		/* for min_age or more micro-seconds */
-		.min_age_region = min_age /
-			damon_reclaim_mon_attrs.aggr_interval,
+		.min_age_region = min_age / aggr_interval,
 		.max_age_region = UINT_MAX,
 	};
 
@@ -183,6 +197,7 @@ static int damon_reclaim_apply_parameters(void)
 {
 	struct damon_ctx *param_ctx;
 	struct damon_target *param_target;
+	struct damon_attrs attrs;
 	struct damos *scheme;
 	struct damos_quota_goal *goal;
 	struct damos_filter *filter;
@@ -195,17 +210,31 @@ static int damon_reclaim_apply_parameters(void)
 	param_ctx->addr_unit = addr_unit;
 	param_ctx->min_region_sz = max(DAMON_MIN_REGION_SZ / addr_unit, 1);
 
+	if (!is_power_of_2(param_ctx->min_region_sz)) {
+		err = -EINVAL;
+		goto out;
+	}
+
 	if (!damon_reclaim_mon_attrs.aggr_interval) {
 		err = -EINVAL;
 		goto out;
 	}
 
-	err = damon_set_attrs(param_ctx, &damon_reclaim_mon_attrs);
+	attrs = damon_reclaim_mon_attrs;
+	if (autotune_monitoring_intervals) {
+		attrs.sample_interval = 5000;
+		attrs.aggr_interval = 100000;
+		attrs.intervals_goal.access_bp = 40;
+		attrs.intervals_goal.aggrs = 3;
+		attrs.intervals_goal.min_sample_us = 5000;
+		attrs.intervals_goal.max_sample_us = 10 * 1000 * 1000;
+	}
+	err = damon_set_attrs(param_ctx, &attrs);
 	if (err)
 		goto out;
 
 	err = -ENOMEM;
-	scheme = damon_reclaim_new_scheme();
+	scheme = damon_reclaim_new_scheme(attrs.aggr_interval);
 	if (!scheme)
 		goto out;
 	damon_set_schemes(param_ctx, &scheme, 1);
@@ -233,11 +262,9 @@ static int damon_reclaim_apply_parameters(void)
 		damos_add_filter(scheme, filter);
 	}
 
-	err = damon_set_region_biggest_system_ram_default(param_target,
-					&monitor_region_start,
-					&monitor_region_end,
-					param_ctx->addr_unit,
-					param_ctx->min_region_sz);
+	err = damon_set_region_system_rams_default(param_target,
+			&monitor_region_start, &monitor_region_end,
+			param_ctx->addr_unit, param_ctx->min_region_sz);
 	if (err)
 		goto out;
 	err = damon_commit_ctx(ctx, param_ctx);
@@ -246,18 +273,51 @@ static int damon_reclaim_apply_parameters(void)
 	return err;
 }
 
-static int damon_reclaim_handle_commit_inputs(void)
+static int damon_reclaim_commit_inputs_fn(void *arg)
 {
+	return damon_reclaim_apply_parameters();
+}
+
+static int damon_reclaim_commit_inputs_store(const char *val,
+					     const struct kernel_param *kp)
+{
+	bool commit_inputs_request;
 	int err;
+	struct damon_call_control control = {
+		.fn = damon_reclaim_commit_inputs_fn,
+	};
 
-	if (!commit_inputs)
+	if (!val) {
+		commit_inputs_request = true;
+	} else {
+		err = kstrtobool(val, &commit_inputs_request);
+		if (err)
+			return err;
+	}
+
+	if (!commit_inputs_request)
 		return 0;
 
-	err = damon_reclaim_apply_parameters();
-	commit_inputs = false;
-	return err;
+	/*
+	 * Skip damon_call() if ctx is not initialized to avoid
+	 * NULL pointer dereference.
+	 */
+	if (!ctx)
+		return -EINVAL;
+
+	err = damon_call(ctx, &control);
+
+	return err ? err : control.return_code;
 }
 
+static const struct kernel_param_ops commit_inputs_param_ops = {
+	.flags = KERNEL_PARAM_OPS_FL_NOARG,
+	.set = damon_reclaim_commit_inputs_store,
+	.get = param_get_bool,
+};
+
+module_param_cb(commit_inputs, &commit_inputs_param_ops, &commit_inputs, 0600);
+
 static int damon_reclaim_damon_call_fn(void *arg)
 {
 	struct damon_ctx *c = arg;
@@ -267,7 +327,7 @@ static int damon_reclaim_damon_call_fn(void *arg)
 	damon_for_each_scheme(s, c)
 		damon_reclaim_stat = s->stat;
 
-	return damon_reclaim_handle_commit_inputs();
+	return 0;
 }
 
 static struct damon_call_control call_control = {
diff --git a/mm/damon/stat.c b/mm/damon/stat.c
index 3951b762cbddf..0e14f5bb8f75d 100644
--- a/mm/damon/stat.c
+++ b/mm/damon/stat.c
@@ -148,59 +148,12 @@ static int damon_stat_damon_call_fn(void *data)
 	return 0;
 }
 
-struct damon_stat_system_ram_range_walk_arg {
-	bool walked;
-	struct resource res;
-};
-
-static int damon_stat_system_ram_walk_fn(struct resource *res, void *arg)
-{
-	struct damon_stat_system_ram_range_walk_arg *a = arg;
-
-	if (!a->walked) {
-		a->walked = true;
-		a->res.start = res->start;
-	}
-	a->res.end = res->end;
-	return 0;
-}
-
-static unsigned long damon_stat_res_to_core_addr(resource_size_t ra,
-		unsigned long addr_unit)
-{
-	/*
-	 * Use div_u64() for avoiding linking errors related with __udivdi3,
-	 * __aeabi_uldivmod, or similar problems.  This should also improve the
-	 * performance optimization (read div_u64() comment for the detail).
-	 */
-	if (sizeof(ra) == 8 && sizeof(addr_unit) == 4)
-		return div_u64(ra, addr_unit);
-	return ra / addr_unit;
-}
-
-static int damon_stat_set_monitoring_region(struct damon_target *t,
-		unsigned long addr_unit, unsigned long min_region_sz)
-{
-	struct damon_addr_range addr_range;
-	struct damon_stat_system_ram_range_walk_arg arg = {};
-
-	walk_system_ram_res(0, -1, &arg, damon_stat_system_ram_walk_fn);
-	if (!arg.walked)
-		return -EINVAL;
-	addr_range.start = damon_stat_res_to_core_addr(
-			arg.res.start, addr_unit);
-	addr_range.end = damon_stat_res_to_core_addr(
-			arg.res.end + 1, addr_unit);
-	if (addr_range.end <= addr_range.start)
-		return -EINVAL;
-	return damon_set_regions(t, &addr_range, 1, min_region_sz);
-}
-
 static struct damon_ctx *damon_stat_build_ctx(void)
 {
 	struct damon_ctx *ctx;
 	struct damon_attrs attrs;
 	struct damon_target *target;
+	unsigned long start = 0, end = 0;
 
 	ctx = damon_new_ctx();
 	if (!ctx)
@@ -230,8 +183,8 @@ static struct damon_ctx *damon_stat_build_ctx(void)
 	if (!target)
 		goto free_out;
 	damon_add_target(ctx, target);
-	if (damon_stat_set_monitoring_region(target, ctx->addr_unit,
-				ctx->min_region_sz))
+	if (damon_set_region_system_rams_default(target, &start, &end,
+				ctx->addr_unit, ctx->min_region_sz))
 		goto free_out;
 	return ctx;
 free_out:
@@ -313,6 +266,45 @@ static int damon_stat_enabled_load(char *buffer, const struct kernel_param *kp)
 	return sprintf(buffer, "%c\n", damon_stat_enabled() ? 'Y' : 'N');
 }
 
+static int damon_stat_kdamond_pid_store(
+		const char *val, const struct kernel_param *kp)
+{
+	/*
+	 * kdamond_pid is read-only, but kernel command line could write it.
+	 * Do nothing here.
+	 */
+	return 0;
+}
+
+static int damon_stat_kdamond_pid_load(
+		char *buffer, const struct kernel_param *kp)
+{
+	int pid;
+
+	if (!damon_stat_context) {
+		pid = -1;
+	} else {
+		pid = damon_kdamond_pid(damon_stat_context);
+		if (pid < 1)
+			pid = -1;
+	}
+	return sprintf(buffer, "%d\n", pid);
+}
+
+static const struct kernel_param_ops kdamond_pid_param_ops = {
+	.set = damon_stat_kdamond_pid_store,
+	.get = damon_stat_kdamond_pid_load,
+};
+
+/*
+ * PID of the DAMON thread
+ *
+ * If DAMON_STAT is enabled, this becomes the PID of the worker thread.
+ * Else, -1.
+ */
+module_param_cb(kdamond_pid, &kdamond_pid_param_ops, NULL, 0400);
+MODULE_PARM_DESC(kdamond_pid, "pid of the kdamond");
+
 static int __init damon_stat_init(void)
 {
 	int err = 0;
diff --git a/mm/damon/sysfs-schemes.c b/mm/damon/sysfs-schemes.c
index 245d63808411a..0d3021db0b99b 100644
--- a/mm/damon/sysfs-schemes.c
+++ b/mm/damon/sysfs-schemes.c
@@ -88,7 +88,6 @@ static void damon_sysfs_scheme_region_release(struct kobject *kobj)
 	struct damon_sysfs_scheme_region *region = container_of(kobj,
 			struct damon_sysfs_scheme_region, kobj);
 
-	list_del(&region->list);
 	kfree(region);
 }
 
@@ -164,7 +163,7 @@ static void damon_sysfs_scheme_regions_rm_dirs(
 	struct damon_sysfs_scheme_region *r, *next;
 
 	list_for_each_entry_safe(r, next, &regions->regions_list, list) {
-		/* release function deletes it from the list */
+		list_del(&r->list);
 		kobject_put(&r->kobj);
 		regions->nr_regions--;
 	}
@@ -1094,6 +1093,10 @@ struct damos_sysfs_qgoal_metric_name damos_sysfs_qgoal_metric_names[] = {
 		.metric = DAMOS_QUOTA_INACTIVE_MEM_BP,
 		.name = "inactive_mem_bp",
 	},
+	{
+		.metric = DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP,
+		.name = "node_eligible_mem_bp",
+	},
 };
 
 static ssize_t target_metric_show(struct kobject *kobj,
@@ -1509,6 +1512,8 @@ struct damon_sysfs_quotas {
 	unsigned long reset_interval_ms;
 	unsigned long effective_sz;	/* Effective size quota in bytes */
 	enum damos_quota_goal_tuner goal_tuner;
+	unsigned int fail_charge_num;
+	unsigned int fail_charge_denom;
 };
 
 static struct damon_sysfs_quotas *damon_sysfs_quotas_alloc(void)
@@ -1683,6 +1688,48 @@ static ssize_t goal_tuner_store(struct kobject *kobj,
 	return -EINVAL;
 }
 
+static ssize_t fail_charge_num_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	struct damon_sysfs_quotas *quotas = container_of(kobj,
+			struct damon_sysfs_quotas, kobj);
+
+	return sysfs_emit(buf, "%u\n", quotas->fail_charge_num);
+}
+
+static ssize_t fail_charge_num_store(struct kobject *kobj,
+		struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	struct damon_sysfs_quotas *quotas = container_of(kobj,
+			struct damon_sysfs_quotas, kobj);
+	int err = kstrtouint(buf, 0, &quotas->fail_charge_num);
+
+	if (err)
+		return -EINVAL;
+	return count;
+}
+
+static ssize_t fail_charge_denom_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	struct damon_sysfs_quotas *quotas = container_of(kobj,
+			struct damon_sysfs_quotas, kobj);
+
+	return sysfs_emit(buf, "%u\n", quotas->fail_charge_denom);
+}
+
+static ssize_t fail_charge_denom_store(struct kobject *kobj,
+		struct kobj_attribute *attr, const char *buf, size_t count)
+{
+	struct damon_sysfs_quotas *quotas = container_of(kobj,
+			struct damon_sysfs_quotas, kobj);
+	int err = kstrtouint(buf, 0, &quotas->fail_charge_denom);
+
+	if (err)
+		return -EINVAL;
+	return count;
+}
+
 static void damon_sysfs_quotas_release(struct kobject *kobj)
 {
 	kfree(container_of(kobj, struct damon_sysfs_quotas, kobj));
@@ -1703,12 +1750,20 @@ static struct kobj_attribute damon_sysfs_quotas_effective_bytes_attr =
 static struct kobj_attribute damon_sysfs_quotas_goal_tuner_attr =
 		__ATTR_RW_MODE(goal_tuner, 0600);
 
+static struct kobj_attribute damon_sysfs_quotas_fail_charge_num_attr =
+		__ATTR_RW_MODE(fail_charge_num, 0600);
+
+static struct kobj_attribute damon_sysfs_quotas_fail_charge_denom_attr =
+		__ATTR_RW_MODE(fail_charge_denom, 0600);
+
 static struct attribute *damon_sysfs_quotas_attrs[] = {
 	&damon_sysfs_quotas_ms_attr.attr,
 	&damon_sysfs_quotas_sz_attr.attr,
 	&damon_sysfs_quotas_reset_interval_ms_attr.attr,
 	&damon_sysfs_quotas_effective_bytes_attr.attr,
 	&damon_sysfs_quotas_goal_tuner_attr.attr,
+	&damon_sysfs_quotas_fail_charge_num_attr.attr,
+	&damon_sysfs_quotas_fail_charge_denom_attr.attr,
 	NULL,
 };
 ATTRIBUTE_GROUPS(damon_sysfs_quotas);
@@ -2061,6 +2116,10 @@ static struct damos_sysfs_action_name damos_sysfs_action_names[] = {
 		.action = DAMOS_NOHUGEPAGE,
 		.name = "nohugepage",
 	},
+	{
+		.action = DAMOS_COLLAPSE,
+		.name = "collapse",
+	},
 	{
 		.action = DAMOS_LRU_PRIO,
 		.name = "lru_prio",
@@ -2594,6 +2653,7 @@ static int damon_sysfs_memcg_path_to_id(char *memcg_path, u64 *id)
 		if (damon_sysfs_memcg_path_eq(memcg, path, memcg_path)) {
 			*id = mem_cgroup_id(memcg);
 			found = true;
+			mem_cgroup_iter_break(NULL, memcg);
 			break;
 		}
 	}
@@ -2685,6 +2745,9 @@ static int damos_sysfs_add_quota_score(
 			}
 			goal->nid = sysfs_goal->nid;
 			break;
+		case DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP:
+			goal->nid = sysfs_goal->nid;
+			break;
 		default:
 			break;
 		}
@@ -2796,6 +2859,8 @@ static struct damos *damon_sysfs_mk_scheme(
 		.weight_nr_accesses = sysfs_weights->nr_accesses,
 		.weight_age = sysfs_weights->age,
 		.goal_tuner = sysfs_quotas->goal_tuner,
+		.fail_charge_num = sysfs_quotas->fail_charge_num,
+		.fail_charge_denom = sysfs_quotas->fail_charge_denom,
 	};
 	struct damos_watermarks wmarks = {
 		.metric = sysfs_wmarks->metric,
@@ -2927,14 +2992,15 @@ void damos_sysfs_populate_region_dir(struct damon_sysfs_schemes *sysfs_schemes,
 	if (!region)
 		return;
 	region->sz_filter_passed = sz_filter_passed;
-	list_add_tail(&region->list, &sysfs_regions->regions_list);
-	sysfs_regions->nr_regions++;
 	if (kobject_init_and_add(&region->kobj,
 				&damon_sysfs_scheme_region_ktype,
 				&sysfs_regions->kobj, "%d",
-				sysfs_regions->nr_regions++)) {
+				sysfs_regions->nr_regions)) {
 		kobject_put(&region->kobj);
+		return;
 	}
+	list_add_tail(&region->list, &sysfs_regions->regions_list);
+	sysfs_regions->nr_regions++;
 }
 
 int damon_sysfs_schemes_clear_regions(
diff --git a/mm/damon/sysfs.c b/mm/damon/sysfs.c
index eefa959aa30ae..d5863cc33d230 100644
--- a/mm/damon/sysfs.c
+++ b/mm/damon/sysfs.c
@@ -866,6 +866,7 @@ struct damon_sysfs_context {
 	struct damon_sysfs_attrs *attrs;
 	struct damon_sysfs_targets *targets;
 	struct damon_sysfs_schemes *schemes;
+	bool pause;
 };
 
 static struct damon_sysfs_context *damon_sysfs_context_alloc(
@@ -878,6 +879,7 @@ static struct damon_sysfs_context *damon_sysfs_context_alloc(
 	context->kobj = (struct kobject){};
 	context->ops_id = ops_id;
 	context->addr_unit = 1;
+	context->pause = false;
 	return context;
 }
 
@@ -1053,6 +1055,30 @@ static ssize_t addr_unit_store(struct kobject *kobj,
 	return count;
 }
 
+static ssize_t pause_show(struct kobject *kobj, struct kobj_attribute *attr,
+		char *buf)
+{
+	struct damon_sysfs_context *context = container_of(kobj,
+			struct damon_sysfs_context, kobj);
+
+	return sysfs_emit(buf, "%c\n", context->pause ? 'Y' : 'N');
+}
+
+static ssize_t pause_store(struct kobject *kobj, struct kobj_attribute *attr,
+		const char *buf, size_t count)
+{
+	struct damon_sysfs_context *context = container_of(kobj,
+			struct damon_sysfs_context, kobj);
+	bool pause;
+	int err = kstrtobool(buf, &pause);
+
+	if (err)
+		return err;
+	context->pause = pause;
+	return count;
+}
+
+
 static void damon_sysfs_context_release(struct kobject *kobj)
 {
 	kfree(container_of(kobj, struct damon_sysfs_context, kobj));
@@ -1067,10 +1093,14 @@ static struct kobj_attribute damon_sysfs_context_operations_attr =
 static struct kobj_attribute damon_sysfs_context_addr_unit_attr =
 		__ATTR_RW_MODE(addr_unit, 0600);
 
+static struct kobj_attribute damon_sysfs_context_pause_attr =
+		__ATTR_RW_MODE(pause, 0600);
+
 static struct attribute *damon_sysfs_context_attrs[] = {
 	&damon_sysfs_context_avail_operations_attr.attr,
 	&damon_sysfs_context_operations_attr.attr,
 	&damon_sysfs_context_addr_unit_attr.attr,
+	&damon_sysfs_context_pause_attr.attr,
 	NULL,
 };
 ATTRIBUTE_GROUPS(damon_sysfs_context);
@@ -1470,6 +1500,7 @@ static int damon_sysfs_apply_inputs(struct damon_ctx *ctx,
 	if (sys_ctx->ops_id == DAMON_OPS_PADDR)
 		ctx->min_region_sz = max(
 				DAMON_MIN_REGION_SZ / sys_ctx->addr_unit, 1);
+	ctx->pause = sys_ctx->pause;
 	err = damon_sysfs_set_attrs(ctx, sys_ctx->attrs);
 	if (err)
 		return err;
diff --git a/mm/damon/tests/core-kunit.h b/mm/damon/tests/core-kunit.h
index 9e5904c2beeb2..866f716e5760d 100644
--- a/mm/damon/tests/core-kunit.h
+++ b/mm/damon/tests/core-kunit.h
@@ -273,54 +273,70 @@ static void damon_test_merge_regions_of(struct kunit *test)
 
 static void damon_test_split_regions_of(struct kunit *test)
 {
+	struct damon_ctx *c;
 	struct damon_target *t;
 	struct damon_region *r;
 	unsigned long sa[] = {0, 300, 500};
 	unsigned long ea[] = {220, 400, 700};
 	int i;
 
+	c = damon_new_ctx();
+	if (!c)
+		kunit_skip(test, "ctx alloc fail");
+
 	t = damon_new_target();
-	if (!t)
+	if (!t) {
+		damon_destroy_ctx(c);
 		kunit_skip(test, "target alloc fail");
+	}
 	r = damon_new_region(0, 22);
 	if (!r) {
 		damon_free_target(t);
+		damon_destroy_ctx(c);
 		kunit_skip(test, "region alloc fail");
 	}
 	damon_add_region(r, t);
-	damon_split_regions_of(t, 2, 1);
+	damon_split_regions_of(c, t, 2, 1);
 	KUNIT_EXPECT_LE(test, damon_nr_regions(t), 2u);
 	damon_free_target(t);
 
 	t = damon_new_target();
-	if (!t)
+	if (!t) {
+		damon_destroy_ctx(c);
 		kunit_skip(test, "second target alloc fail");
+	}
 	r = damon_new_region(0, 220);
 	if (!r) {
 		damon_free_target(t);
+		damon_destroy_ctx(c);
 		kunit_skip(test, "second region alloc fail");
 	}
 	damon_add_region(r, t);
-	damon_split_regions_of(t, 4, 1);
+	damon_split_regions_of(c, t, 4, 1);
 	KUNIT_EXPECT_LE(test, damon_nr_regions(t), 4u);
 	damon_free_target(t);
 
 	t = damon_new_target();
-	if (!t)
+	if (!t) {
+		damon_destroy_ctx(c);
 		kunit_skip(test, "third target alloc fail");
+	}
 	for (i = 0; i < ARRAY_SIZE(sa); i++) {
 		r = damon_new_region(sa[i], ea[i]);
 		if (!r) {
 			damon_free_target(t);
+			damon_destroy_ctx(c);
 			kunit_skip(test, "region alloc fail");
 		}
 		damon_add_region(r, t);
 	}
-	damon_split_regions_of(t, 4, 5);
+	damon_split_regions_of(c, t, 4, 5);
 	KUNIT_EXPECT_LE(test, damon_nr_regions(t), 12u);
 	damon_for_each_region(r, t)
 		KUNIT_EXPECT_GE(test, damon_sz_region(r) % 5ul, 0ul);
 	damon_free_target(t);
+
+	damon_destroy_ctx(c);
 }
 
 static void damon_test_ops_registration(struct kunit *test)
@@ -694,6 +710,8 @@ static void damos_test_commit_quota(struct kunit *test)
 		.ms = 2,
 		.sz = 3,
 		.goal_tuner = DAMOS_QUOTA_GOAL_TUNER_CONSIST,
+		.fail_charge_num = 2,
+		.fail_charge_denom = 3,
 		.weight_sz = 4,
 		.weight_nr_accesses = 5,
 		.weight_age = 6,
@@ -703,6 +721,8 @@ static void damos_test_commit_quota(struct kunit *test)
 		.ms = 8,
 		.sz = 9,
 		.goal_tuner = DAMOS_QUOTA_GOAL_TUNER_TEMPORAL,
+		.fail_charge_num = 1,
+		.fail_charge_denom = 1024,
 		.weight_sz = 10,
 		.weight_nr_accesses = 11,
 		.weight_age = 12,
@@ -717,6 +737,8 @@ static void damos_test_commit_quota(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, dst.ms, src.ms);
 	KUNIT_EXPECT_EQ(test, dst.sz, src.sz);
 	KUNIT_EXPECT_EQ(test, dst.goal_tuner, src.goal_tuner);
+	KUNIT_EXPECT_EQ(test, dst.fail_charge_num, src.fail_charge_num);
+	KUNIT_EXPECT_EQ(test, dst.fail_charge_denom, src.fail_charge_denom);
 	KUNIT_EXPECT_EQ(test, dst.weight_sz, src.weight_sz);
 	KUNIT_EXPECT_EQ(test, dst.weight_nr_accesses, src.weight_nr_accesses);
 	KUNIT_EXPECT_EQ(test, dst.weight_age, src.weight_age);
@@ -1077,6 +1099,10 @@ static void damon_test_commit_ctx(struct kunit *test)
 	KUNIT_EXPECT_EQ(test, damon_commit_ctx(dst, src), 0);
 	src->min_region_sz = 4095;
 	KUNIT_EXPECT_EQ(test, damon_commit_ctx(dst, src), -EINVAL);
+	src->min_region_sz = 4096;
+	src->pause = true;
+	KUNIT_EXPECT_EQ(test, damon_commit_ctx(dst, src), 0);
+	KUNIT_EXPECT_TRUE(test, dst->pause);
 	damon_destroy_ctx(src);
 	damon_destroy_ctx(dst);
 }
diff --git a/mm/damon/vaddr.c b/mm/damon/vaddr.c
index b069dbc7e3d25..1b0ebe3b6951e 100644
--- a/mm/damon/vaddr.c
+++ b/mm/damon/vaddr.c
@@ -333,9 +333,10 @@ static void damon_va_mkold(struct mm_struct *mm, unsigned long addr)
  */
 
 static void __damon_va_prepare_access_check(struct mm_struct *mm,
-					struct damon_region *r)
+					struct damon_region *r,
+					struct damon_ctx *ctx)
 {
-	r->sampling_addr = damon_rand(r->ar.start, r->ar.end);
+	r->sampling_addr = damon_rand(ctx, r->ar.start, r->ar.end);
 
 	damon_va_mkold(mm, r->sampling_addr);
 }
@@ -351,7 +352,7 @@ static void damon_va_prepare_access_checks(struct damon_ctx *ctx)
 		if (!mm)
 			continue;
 		damon_for_each_region(r, t)
-			__damon_va_prepare_access_check(mm, r);
+			__damon_va_prepare_access_check(mm, r, ctx);
 		mmput(mm);
 	}
 }
@@ -903,6 +904,9 @@ static unsigned long damon_va_apply_scheme(struct damon_ctx *ctx,
 	case DAMOS_NOHUGEPAGE:
 		madv_action = MADV_NOHUGEPAGE;
 		break;
+	case DAMOS_COLLAPSE:
+		madv_action = MADV_COLLAPSE;
+		break;
 	case DAMOS_MIGRATE_HOT:
 	case DAMOS_MIGRATE_COLD:
 		return damos_va_migrate(t, r, scheme, sz_filter_passed);
diff --git a/mm/filemap.c b/mm/filemap.c
index 4e636647100c1..5aaba0d3e81d6 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -189,7 +189,6 @@ static void filemap_unaccount_folio(struct address_space *mapping,
 			lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr);
 	} else if (folio_test_pmd_mappable(folio)) {
 		lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr);
-		filemap_nr_thps_dec(mapping);
 	}
 	if (test_bit(AS_KERNEL_FILE, &folio->mapping->flags))
 		mod_node_page_state(folio_pgdat(folio),
@@ -1808,9 +1807,8 @@ pgoff_t page_cache_next_miss(struct address_space *mapping,
 			     pgoff_t index, unsigned long max_scan)
 {
 	XA_STATE(xas, &mapping->i_pages, index);
-	unsigned long nr = max_scan;
 
-	while (nr--) {
+	while (max_scan--) {
 		void *entry = xas_next(&xas);
 		if (!entry || xa_is_value(entry))
 			return xas.xa_index;
@@ -1818,7 +1816,8 @@ pgoff_t page_cache_next_miss(struct address_space *mapping,
 			return 0;
 	}
 
-	return index + max_scan;
+	/* Return end of the range + 1 when no hole is found */
+	return xas.xa_index + 1;
 }
 EXPORT_SYMBOL(page_cache_next_miss);
 
@@ -1849,12 +1848,13 @@ pgoff_t page_cache_prev_miss(struct address_space *mapping,
 	while (max_scan--) {
 		void *entry = xas_prev(&xas);
 		if (!entry || xa_is_value(entry))
-			break;
+			return xas.xa_index;
 		if (xas.xa_index == ULONG_MAX)
-			break;
+			return ULONG_MAX;
 	}
 
-	return xas.xa_index;
+	/* Return start of the range - 1 when no hole is found */
+	return xas.xa_index - 1;
 }
 EXPORT_SYMBOL(page_cache_prev_miss);
 
@@ -3314,6 +3314,8 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
 	bool force_thp_readahead = false;
 	unsigned short mmap_miss;
 
+	ractl._max_index = vmf->vma->vm_pgoff + vma_pages(vmf->vma) - 1;
+
 	/* Use the readahead code, even if readahead is disabled */
 	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
 	    (vm_flags & VM_HUGEPAGE) && HPAGE_PMD_ORDER <= MAX_PAGECACHE_ORDER)
@@ -3396,6 +3398,7 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
 		 * mmap read-around
 		 */
 		ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2);
+		ra->start = max(ra->start, vmf->vma->vm_pgoff);
 		ra->size = ra->ra_pages;
 		ra->async_size = ra->ra_pages / 4;
 		ra->order = 0;
@@ -3438,6 +3441,7 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf,
 	}
 
 	if (folio_test_readahead(folio)) {
+		ractl._max_index = vmf->vma->vm_pgoff + vma_pages(vmf->vma) - 1;
 		fpin = maybe_unlock_mmap_for_io(vmf, fpin);
 		page_cache_async_ra(&ractl, folio, ra->ra_pages);
 	}
@@ -3747,8 +3751,7 @@ static struct folio *next_uptodate_folio(struct xa_state *xas,
 static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf,
 			struct folio *folio, unsigned long start,
 			unsigned long addr, unsigned int nr_pages,
-			unsigned long *rss, unsigned short *mmap_miss,
-			pgoff_t file_end)
+			unsigned long *rss, pgoff_t file_end)
 {
 	struct address_space *mapping = folio->mapping;
 	unsigned int ref_from_caller = 1;
@@ -3780,16 +3783,6 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf,
 		if (PageHWPoison(page + count))
 			goto skip;
 
-		/*
-		 * If there are too many folios that are recently evicted
-		 * in a file, they will probably continue to be evicted.
-		 * In such situation, read-ahead is only a waste of IO.
-		 * Don't decrease mmap_miss in this scenario to make sure
-		 * we can stop read-ahead.
-		 */
-		if (!folio_test_workingset(folio))
-			(*mmap_miss)++;
-
 		/*
 		 * NOTE: If there're PTE markers, we'll leave them to be
 		 * handled in the specific fault path, and it'll prohibit the
@@ -3836,7 +3829,7 @@ static vm_fault_t filemap_map_folio_range(struct vm_fault *vmf,
 
 static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf,
 		struct folio *folio, unsigned long addr,
-		unsigned long *rss, unsigned short *mmap_miss)
+		unsigned long *rss)
 {
 	vm_fault_t ret = 0;
 	struct page *page = &folio->page;
@@ -3844,10 +3837,6 @@ static vm_fault_t filemap_map_order0_folio(struct vm_fault *vmf,
 	if (PageHWPoison(page))
 		goto out;
 
-	/* See comment of filemap_map_folio_range() */
-	if (!folio_test_workingset(folio))
-		(*mmap_miss)++;
-
 	/*
 	 * NOTE: If there're PTE markers, we'll leave them to be
 	 * handled in the specific fault path, and it'll prohibit
@@ -3882,7 +3871,6 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf,
 	vm_fault_t ret = 0;
 	unsigned long rss = 0;
 	unsigned int nr_pages = 0, folio_type;
-	unsigned short mmap_miss = 0, mmap_miss_saved;
 
 	/*
 	 * Recalculate end_pgoff based on file_end before calling
@@ -3921,6 +3909,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf,
 	folio_type = mm_counter_file(folio);
 	do {
 		unsigned long end;
+		vm_fault_t map_ret;
 
 		addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT;
 		vmf->pte += xas.xa_index - last_pgoff;
@@ -3928,13 +3917,35 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf,
 		end = folio_next_index(folio) - 1;
 		nr_pages = min(end, end_pgoff) - xas.xa_index + 1;
 
-		if (!folio_test_large(folio))
-			ret |= filemap_map_order0_folio(vmf,
-					folio, addr, &rss, &mmap_miss);
-		else
-			ret |= filemap_map_folio_range(vmf, folio,
-					xas.xa_index - folio->index, addr,
-					nr_pages, &rss, &mmap_miss, file_end);
+		if (!folio_test_large(folio)) {
+			map_ret = filemap_map_order0_folio(vmf, folio, addr,
+							   &rss);
+		} else {
+			unsigned long start = xas.xa_index - folio->index;
+
+			map_ret = filemap_map_folio_range(vmf, folio, start,
+							  addr, nr_pages, &rss,
+							  file_end);
+		}
+		ret |= map_ret;
+
+		/*
+		 * If there are too many folios that are recently evicted
+		 * in a file, they will probably continue to be evicted.
+		 * In such situation, read-ahead is only a waste of IO.
+		 * Don't decrease mmap_miss in this scenario to make sure
+		 * we can stop read-ahead.
+		 */
+		if ((map_ret & VM_FAULT_NOPAGE) &&
+		    !(vmf->flags & FAULT_FLAG_TRIED) &&
+		    !folio_test_workingset(folio)) {
+			unsigned short mmap_miss;
+
+			mmap_miss = READ_ONCE(file->f_ra.mmap_miss);
+			if (mmap_miss)
+				WRITE_ONCE(file->f_ra.mmap_miss,
+					   mmap_miss - 1);
+		}
 
 		folio_unlock(folio);
 	} while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL);
@@ -3944,12 +3955,6 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf,
 out:
 	rcu_read_unlock();
 
-	mmap_miss_saved = READ_ONCE(file->f_ra.mmap_miss);
-	if (mmap_miss >= mmap_miss_saved)
-		WRITE_ONCE(file->f_ra.mmap_miss, 0);
-	else
-		WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss_saved - mmap_miss);
-
 	return ret;
 }
 EXPORT_SYMBOL(filemap_map_pages);
diff --git a/mm/gup.c b/mm/gup.c
index ad9ded39609cb..21ea90baefb18 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1983,6 +1983,7 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
 	struct vm_area_struct *vma;
 	bool must_unlock = false;
 	vm_flags_t vm_flags;
+	int ret, err = -EFAULT;
 	long i;
 
 	if (!nr_pages)
@@ -2019,8 +2020,14 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
 
 		if (pages) {
 			pages[i] = virt_to_page((void *)start);
-			if (pages[i])
-				get_page(pages[i]);
+			if (!pages[i])
+				break;
+			ret = try_grab_folio(page_folio(pages[i]), 1, foll_flags);
+			if (unlikely(ret)) {
+				pages[i] = NULL;
+				err = ret;
+				break;
+			}
 		}
 
 		start = (start + PAGE_SIZE) & PAGE_MASK;
@@ -2031,7 +2038,7 @@ static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
 		*locked = 0;
 	}
 
-	return i ? : -EFAULT;
+	return i ? : err;
 }
 #endif /* !CONFIG_MMU */
 
@@ -2865,8 +2872,8 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
 		if (!folio)
 			goto pte_unmap;
 
-		if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) ||
-		    unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) {
+		if (unlikely(pmd_val(pmd) != pmd_val(pmdp_get_lockless(pmdp))) ||
+		    unlikely(pte_val(pte) != pte_val(ptep_get_lockless(ptep)))) {
 			gup_put_folio(folio, 1, flags);
 			goto pte_unmap;
 		}
@@ -2942,7 +2949,7 @@ static int gup_fast_pmd_leaf(pmd_t orig, pmd_t *pmdp, unsigned long addr,
 	if (!folio)
 		return 0;
 
-	if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
+	if (unlikely(pmd_val(orig) != pmd_val(pmdp_get_lockless(pmdp)))) {
 		gup_put_folio(folio, refs, flags);
 		return 0;
 	}
@@ -2985,7 +2992,7 @@ static int gup_fast_pud_leaf(pud_t orig, pud_t *pudp, unsigned long addr,
 	if (!folio)
 		return 0;
 
-	if (unlikely(pud_val(orig) != pud_val(*pudp))) {
+	if (unlikely(pud_val(orig) != pud_val(pudp_get(pudp)))) {
 		gup_put_folio(folio, refs, flags);
 		return 0;
 	}
diff --git a/mm/hmm.c b/mm/hmm.c
index 5955f2f0c83db..c72c9ddfdb2f6 100644
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -709,7 +709,7 @@ int hmm_dma_map_alloc(struct device *dev, struct hmm_dma_map *map,
 	 * best approximation to ensure no swiotlb buffering happens.
 	 */
 #ifdef CONFIG_DMA_NEED_SYNC
-	dma_need_sync = !dev->dma_skip_sync;
+	dma_need_sync = !dev_dma_skip_sync(dev);
 #endif /* CONFIG_DMA_NEED_SYNC */
 	if (dma_need_sync || dma_addressing_limited(dev))
 		return -EOPNOTSUPP;
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 970e077019b75..c565b2a651e06 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -86,9 +86,6 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
 {
 	struct inode *inode;
 
-	if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS))
-		return false;
-
 	if (!vma->vm_file)
 		return false;
 
@@ -97,7 +94,10 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
 	if (IS_ANON_FILE(inode))
 		return false;
 
-	return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
+	if (!mapping_pmd_folio_support(vma->vm_file->f_mapping))
+		return false;
+
+	return S_ISREG(inode->i_mode);
 }
 
 /* If returns true, we are unable to access the VMA's folios. */
@@ -429,61 +429,75 @@ ssize_t single_hugepage_flag_store(struct kobject *kobj,
 	return count;
 }
 
+enum defrag_mode {
+	DEFRAG_ALWAYS = 0,
+	DEFRAG_DEFER,
+	DEFRAG_DEFER_MADVISE,
+	DEFRAG_MADVISE,
+	DEFRAG_NEVER,
+};
+
+static const char * const defrag_mode_strings[] = {
+	[DEFRAG_ALWAYS]		= "always",
+	[DEFRAG_DEFER]		= "defer",
+	[DEFRAG_DEFER_MADVISE]	= "defer+madvise",
+	[DEFRAG_MADVISE]	= "madvise",
+	[DEFRAG_NEVER]		= "never",
+};
+
+static const enum transparent_hugepage_flag defrag_flags[] = {
+	[DEFRAG_ALWAYS]		= TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
+	[DEFRAG_DEFER]		= TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
+	[DEFRAG_DEFER_MADVISE]	= TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
+	[DEFRAG_MADVISE]	= TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
+};
+
 static ssize_t defrag_show(struct kobject *kobj,
 			   struct kobj_attribute *attr, char *buf)
 {
-	const char *output;
+	int active = DEFRAG_NEVER;
+	int len = 0;
+	int i;
 
-	if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
-		     &transparent_hugepage_flags))
-		output = "[always] defer defer+madvise madvise never";
-	else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
-			  &transparent_hugepage_flags))
-		output = "always [defer] defer+madvise madvise never";
-	else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG,
-			  &transparent_hugepage_flags))
-		output = "always defer [defer+madvise] madvise never";
-	else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG,
-			  &transparent_hugepage_flags))
-		output = "always defer defer+madvise [madvise] never";
-	else
-		output = "always defer defer+madvise madvise [never]";
+	for (i = 0; i < ARRAY_SIZE(defrag_flags); i++) {
+		if (test_bit(defrag_flags[i], &transparent_hugepage_flags)) {
+			active = i;
+			break;
+		}
+	}
 
-	return sysfs_emit(buf, "%s\n", output);
+	for (i = 0; i < ARRAY_SIZE(defrag_mode_strings); i++) {
+		if (i == active)
+			len += sysfs_emit_at(buf, len, "[%s] ",
+					     defrag_mode_strings[i]);
+		else
+			len += sysfs_emit_at(buf, len, "%s ",
+					     defrag_mode_strings[i]);
+	}
+
+	/* Replace trailing space with newline */
+	buf[len - 1] = '\n';
+
+	return len;
 }
 
 static ssize_t defrag_store(struct kobject *kobj,
 			    struct kobj_attribute *attr,
 			    const char *buf, size_t count)
 {
-	if (sysfs_streq(buf, "always")) {
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
-		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
-	} else if (sysfs_streq(buf, "defer+madvise")) {
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
-		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
-	} else if (sysfs_streq(buf, "defer")) {
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
-		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
-	} else if (sysfs_streq(buf, "madvise")) {
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
-		set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
-	} else if (sysfs_streq(buf, "never")) {
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags);
-		clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags);
-	} else
+	int mode, m;
+
+	mode = sysfs_match_string(defrag_mode_strings, buf);
+	if (mode < 0)
 		return -EINVAL;
 
+	for (m = 0; m < ARRAY_SIZE(defrag_flags); m++) {
+		if (m == mode)
+			set_bit(defrag_flags[m], &transparent_hugepage_flags);
+		else
+			clear_bit(defrag_flags[m], &transparent_hugepage_flags);
+	}
+
 	return count;
 }
 static struct kobj_attribute defrag_attr = __ATTR_RW(defrag);
@@ -685,6 +699,8 @@ static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
 DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC);
 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK);
 DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
+DEFINE_MTHP_STAT_ATTR(collapse_alloc, MTHP_STAT_COLLAPSE_ALLOC);
+DEFINE_MTHP_STAT_ATTR(collapse_alloc_failed, MTHP_STAT_COLLAPSE_ALLOC_FAILED);
 DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT);
 DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN);
 DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK);
@@ -701,6 +717,10 @@ DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED);
 DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED);
 DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON);
 DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED);
+DEFINE_MTHP_STAT_ATTR(collapse_exceed_swap_pte, MTHP_STAT_COLLAPSE_EXCEED_SWAP);
+DEFINE_MTHP_STAT_ATTR(collapse_exceed_none_pte, MTHP_STAT_COLLAPSE_EXCEED_NONE);
+DEFINE_MTHP_STAT_ATTR(collapse_exceed_shared_pte, MTHP_STAT_COLLAPSE_EXCEED_SHARED);
+
 
 static struct attribute *anon_stats_attrs[] = {
 	&anon_fault_alloc_attr.attr,
@@ -717,6 +737,9 @@ static struct attribute *anon_stats_attrs[] = {
 	&split_deferred_attr.attr,
 	&nr_anon_attr.attr,
 	&nr_anon_partially_mapped_attr.attr,
+	&collapse_exceed_swap_pte_attr.attr,
+	&collapse_exceed_none_pte_attr.attr,
+	&collapse_exceed_shared_pte_attr.attr,
 	NULL,
 };
 
@@ -750,6 +773,8 @@ static struct attribute *any_stats_attrs[] = {
 #endif
 	&split_attr.attr,
 	&split_failed_attr.attr,
+	&collapse_alloc_attr.attr,
+	&collapse_alloc_failed_attr.attr,
 	NULL,
 };
 
@@ -3821,33 +3846,9 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order,
 	if (!folio->mapping && !folio_test_anon(folio))
 		return -EBUSY;
 
-	if (folio_test_anon(folio)) {
-		/* order-1 is not supported for anonymous THP. */
-		if (new_order == 1)
-			return -EINVAL;
-	} else if (split_type == SPLIT_TYPE_NON_UNIFORM || new_order) {
-		if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
-		    !mapping_large_folio_support(folio->mapping)) {
-			/*
-			 * We can always split a folio down to a single page
-			 * (new_order == 0) uniformly.
-			 *
-			 * For any other scenario
-			 *   a) uniform split targeting a large folio
-			 *      (new_order > 0)
-			 *   b) any non-uniform split
-			 * we must confirm that the file system supports large
-			 * folios.
-			 *
-			 * Note that we might still have THPs in such
-			 * mappings, which is created from khugepaged when
-			 * CONFIG_READ_ONLY_THP_FOR_FS is enabled. But in that
-			 * case, the mapping does not actually support large
-			 * folios properly.
-			 */
-			return -EINVAL;
-		}
-	}
+	/* order-1 is not supported for anonymous THP. */
+	if (folio_test_anon(folio) && new_order == 1)
+		return -EINVAL;
 
 	/*
 	 * swapcache folio could only be split to order 0
@@ -3926,7 +3927,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
 				} else {
 					lruvec_stat_mod_folio(folio,
 							NR_FILE_THPS, -nr);
-					filemap_nr_thps_dec(mapping);
 				}
 			}
 		}
@@ -4190,11 +4190,10 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
 
 		folio_unlock(new_folio);
 		/*
-		 * Subpages may be freed if there wasn't any mapping
-		 * like if add_to_swap() is running on a lru page that
-		 * had its mapping zapped. And freeing these pages
-		 * requires taking the lru_lock so we do the put_page
-		 * of the tail pages after the split is complete.
+		 * Subpages whose mapping has been zapped may be freed
+		 * earlier, but freeing them requires taking the
+		 * lru_lock, so we defer put_page() on tail pages until
+		 * after the split completes.
 		 */
 		free_folio_and_swap_cache(new_folio);
 	}
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index f24bf49be047e..4b3849d9a3f3b 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -116,7 +116,11 @@ struct mutex *hugetlb_fault_mutex_table __ro_after_init;
 /* Forward declaration */
 static int hugetlb_acct_memory(struct hstate *h, long delta);
 static void hugetlb_vma_lock_free(struct vm_area_struct *vma);
+static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma);
 static void __hugetlb_vma_unlock_write_free(struct vm_area_struct *vma);
+static int __huge_pmd_unshare(struct mmu_gather *tlb,
+		struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
+		bool check_locks);
 static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
 		unsigned long start, unsigned long end, bool take_locks);
 static struct resv_map *vma_resv_map(struct vm_area_struct *vma);
@@ -413,21 +417,17 @@ static void hugetlb_vma_lock_free(struct vm_area_struct *vma)
 	}
 }
 
-/*
- * vma specific semaphore used for pmd sharing and fault/truncation
- * synchronization
- */
-int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
+static void hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
 {
 	struct hugetlb_vma_lock *vma_lock;
 
 	/* Only establish in (flags) sharable vmas */
 	if (!vma || !(vma->vm_flags & VM_MAYSHARE))
-		return 0;
+		return;
 
 	/* Should never get here with non-NULL vm_private_data */
 	if (vma->vm_private_data)
-		return -EINVAL;
+		return;
 
 	vma_lock = kmalloc_obj(*vma_lock);
 	if (!vma_lock) {
@@ -442,15 +442,13 @@ int hugetlb_vma_lock_alloc(struct vm_area_struct *vma)
 		 * allocation failure.
 		 */
 		pr_warn_once("HugeTLB: unable to allocate vma specific lock\n");
-		return -EINVAL;
+		return;
 	}
 
 	kref_init(&vma_lock->refs);
 	init_rwsem(&vma_lock->rw_sema);
 	vma_lock->vma = vma;
 	vma->vm_private_data = vma_lock;
-
-	return 0;
 }
 
 /* Helper that removes a struct file_region from the resv_map cache and returns
@@ -1147,28 +1145,20 @@ static struct resv_map *vma_resv_map(struct vm_area_struct *vma)
 	}
 }
 
-static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
+static void set_vma_resv_map(struct vm_area_struct *vma, struct resv_map *map)
 {
 	VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
-	VM_WARN_ON_ONCE_VMA(vma->vm_flags & VM_MAYSHARE, vma);
+	VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
 
-	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
+	set_vma_private_data(vma, (unsigned long)map);
 }
 
-static void set_vma_desc_resv_map(struct vm_area_desc *desc, struct resv_map *map)
-{
-	VM_WARN_ON_ONCE(!is_vma_hugetlb_flags(&desc->vma_flags));
-	VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_MAYSHARE_BIT));
-
-	desc->private_data = map;
-}
-
-static void set_vma_desc_resv_flags(struct vm_area_desc *desc, unsigned long flags)
+static void set_vma_resv_flags(struct vm_area_struct *vma, unsigned long flags)
 {
-	VM_WARN_ON_ONCE(!is_vma_hugetlb_flags(&desc->vma_flags));
-	VM_WARN_ON_ONCE(vma_desc_test(desc, VMA_MAYSHARE_BIT));
+	VM_WARN_ON_ONCE_VMA(!is_vm_hugetlb_page(vma), vma);
+	VM_WARN_ON_ONCE_VMA(vma_test(vma, VMA_MAYSHARE_BIT), vma);
 
-	desc->private_data = (void *)((unsigned long)desc->private_data | flags);
+	set_vma_private_data(vma, get_vma_private_data(vma) | flags);
 }
 
 static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
@@ -1178,13 +1168,6 @@ static int is_vma_resv_set(struct vm_area_struct *vma, unsigned long flag)
 	return (get_vma_private_data(vma) & flag) != 0;
 }
 
-static bool is_vma_desc_resv_set(struct vm_area_desc *desc, unsigned long flag)
-{
-	VM_WARN_ON_ONCE(!is_vma_hugetlb_flags(&desc->vma_flags));
-
-	return ((unsigned long)desc->private_data) & flag;
-}
-
 bool __vma_private_lock(struct vm_area_struct *vma)
 {
 	return !(vma->vm_flags & VM_MAYSHARE) &&
@@ -2879,6 +2862,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 	map_chg_state map_chg;
 	int ret, idx;
 	struct hugetlb_cgroup *h_cg = NULL;
+	struct hugetlb_cgroup *h_cg_rsvd = NULL;
 	gfp_t gfp = htlb_alloc_mask(h) | __GFP_RETRY_MAYFAIL;
 
 	idx = hstate_index(h);
@@ -2929,7 +2913,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 	 */
 	if (map_chg) {
 		ret = hugetlb_cgroup_charge_cgroup_rsvd(
-			idx, pages_per_huge_page(h), &h_cg);
+			idx, pages_per_huge_page(h), &h_cg_rsvd);
 		if (ret)
 			goto out_subpool_put;
 	}
@@ -2971,7 +2955,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 	 */
 	if (map_chg) {
 		hugetlb_cgroup_commit_charge_rsvd(idx, pages_per_huge_page(h),
-						  h_cg, folio);
+						  h_cg_rsvd, folio);
 	}
 
 	spin_unlock_irq(&hugetlb_lock);
@@ -3023,7 +3007,7 @@ struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma,
 out_uncharge_cgroup_reservation:
 	if (map_chg)
 		hugetlb_cgroup_uncharge_cgroup_rsvd(idx, pages_per_huge_page(h),
-						    h_cg);
+						    h_cg_rsvd);
 out_subpool_put:
 	/*
 	 * put page to subpool iff the quota of subpool's rsv_hpages is used
@@ -6290,6 +6274,7 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
 		folio_put(*foliop);
 		*foliop = NULL;
 		if (ret) {
+			restore_reserve_on_error(h, dst_vma, dst_addr, folio);
 			folio_put(folio);
 			goto out;
 		}
@@ -6553,7 +6538,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma,
 
 long hugetlb_reserve_pages(struct inode *inode,
 		long from, long to,
-		struct vm_area_desc *desc,
+		struct vm_area_struct *vma,
 		vma_flags_t vma_flags)
 {
 	long chg = -1, add = -1, spool_resv, gbl_resv;
@@ -6570,6 +6555,12 @@ long hugetlb_reserve_pages(struct inode *inode,
 		return -EINVAL;
 	}
 
+	/*
+	 * vma specific semaphore used for pmd sharing and fault/truncation
+	 * synchronization
+	 */
+	hugetlb_vma_lock_alloc(vma);
+
 	/*
 	 * Only apply hugepage reservation if asked. At fault time, an
 	 * attempt will be made for VM_NORESERVE to allocate a page
@@ -6582,9 +6573,9 @@ long hugetlb_reserve_pages(struct inode *inode,
 	 * Shared mappings base their reservation on the number of pages that
 	 * are already allocated on behalf of the file. Private mappings need
 	 * to reserve the full area even if read-only as mprotect() may be
-	 * called to make the mapping read-write. Assume !desc is a shm mapping
+	 * called to make the mapping read-write. Assume !vma is a shm mapping
 	 */
-	if (!desc || vma_desc_test(desc, VMA_MAYSHARE_BIT)) {
+	if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
 		/*
 		 * resv_map can not be NULL as hugetlb_reserve_pages is only
 		 * called for inodes for which resv_maps were created (see
@@ -6603,8 +6594,8 @@ long hugetlb_reserve_pages(struct inode *inode,
 
 		chg = to - from;
 
-		set_vma_desc_resv_map(desc, resv_map);
-		set_vma_desc_resv_flags(desc, HPAGE_RESV_OWNER);
+		set_vma_resv_map(vma, resv_map);
+		set_vma_resv_flags(vma, HPAGE_RESV_OWNER);
 	}
 
 	if (chg < 0) {
@@ -6618,7 +6609,7 @@ long hugetlb_reserve_pages(struct inode *inode,
 	if (err < 0)
 		goto out_err;
 
-	if (desc && !vma_desc_test(desc, VMA_MAYSHARE_BIT) && h_cg) {
+	if (vma && !vma_test(vma, VMA_MAYSHARE_BIT) && h_cg) {
 		/* For private mappings, the hugetlb_cgroup uncharge info hangs
 		 * of the resv_map.
 		 */
@@ -6655,7 +6646,7 @@ long hugetlb_reserve_pages(struct inode *inode,
 	 * consumed reservations are stored in the map. Hence, nothing
 	 * else has to be done for private mappings here
 	 */
-	if (!desc || vma_desc_test(desc, VMA_MAYSHARE_BIT)) {
+	if (!vma || vma_test(vma, VMA_MAYSHARE_BIT)) {
 		add = region_add(resv_map, from, to, regions_needed, h, h_cg);
 
 		if (unlikely(add < 0)) {
@@ -6719,15 +6710,16 @@ long hugetlb_reserve_pages(struct inode *inode,
 	hugetlb_cgroup_uncharge_cgroup_rsvd(hstate_index(h),
 					    chg * pages_per_huge_page(h), h_cg);
 out_err:
-	if (!desc || vma_desc_test(desc, VMA_MAYSHARE_BIT))
+	hugetlb_vma_lock_free(vma);
+	if (!vma || vma_test(vma, VMA_MAYSHARE_BIT))
 		/* Only call region_abort if the region_chg succeeded but the
 		 * region_add failed or didn't run.
 		 */
 		if (chg >= 0 && add < 0)
 			region_abort(resv_map, from, to, regions_needed);
-	if (desc && is_vma_desc_resv_set(desc, HPAGE_RESV_OWNER)) {
+	if (vma && is_vma_resv_set(vma, HPAGE_RESV_OWNER)) {
 		kref_put(&resv_map->refs, resv_map_release);
-		set_vma_desc_resv_map(desc, NULL);
+		set_vma_resv_map(vma, NULL);
 	}
 	return err;
 }
@@ -6904,6 +6896,31 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
 	return pte;
 }
 
+static int __huge_pmd_unshare(struct mmu_gather *tlb,
+		struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
+		bool check_locks)
+{
+	unsigned long sz = huge_page_size(hstate_vma(vma));
+	struct mm_struct *mm = vma->vm_mm;
+	pgd_t *pgd = pgd_offset(mm, addr);
+	p4d_t *p4d = p4d_offset(pgd, addr);
+	pud_t *pud = pud_offset(p4d, addr);
+
+	if (sz != PMD_SIZE)
+		return 0;
+	if (!ptdesc_pmd_is_shared(virt_to_ptdesc(ptep)))
+		return 0;
+	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
+	if (check_locks)
+		hugetlb_vma_assert_locked(vma);
+	pud_clear(pud);
+
+	tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr);
+
+	mm_dec_nr_pmds(mm);
+	return 1;
+}
+
 /**
  * huge_pmd_unshare - Unmap a pmd table if it is shared by multiple users
  * @tlb: the current mmu_gather.
@@ -6923,24 +6940,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
 int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		unsigned long addr, pte_t *ptep)
 {
-	unsigned long sz = huge_page_size(hstate_vma(vma));
-	struct mm_struct *mm = vma->vm_mm;
-	pgd_t *pgd = pgd_offset(mm, addr);
-	p4d_t *p4d = p4d_offset(pgd, addr);
-	pud_t *pud = pud_offset(p4d, addr);
-
-	if (sz != PMD_SIZE)
-		return 0;
-	if (!ptdesc_pmd_is_shared(virt_to_ptdesc(ptep)))
-		return 0;
-	i_mmap_assert_write_locked(vma->vm_file->f_mapping);
-	hugetlb_vma_assert_locked(vma);
-	pud_clear(pud);
-
-	tlb_unshare_pmd_ptdesc(tlb, virt_to_ptdesc(ptep), addr);
-
-	mm_dec_nr_pmds(mm);
-	return 1;
+	return __huge_pmd_unshare(tlb, vma, addr, ptep, /*check_locks=*/true);
 }
 
 /*
@@ -6974,6 +6974,13 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma,
 	return NULL;
 }
 
+static int __huge_pmd_unshare(struct mmu_gather *tlb,
+		struct vm_area_struct *vma, unsigned long addr, pte_t *ptep,
+		bool check_locks)
+{
+	return 0;
+}
+
 int huge_pmd_unshare(struct mmu_gather *tlb, struct vm_area_struct *vma,
 		unsigned long addr, pte_t *ptep)
 {
@@ -7282,7 +7289,7 @@ static void hugetlb_unshare_pmds(struct vm_area_struct *vma,
 		if (!ptep)
 			continue;
 		ptl = huge_pte_lock(h, mm, ptep);
-		huge_pmd_unshare(&tlb, vma, address, ptep);
+		__huge_pmd_unshare(&tlb, vma, address, ptep, take_locks);
 		spin_unlock(ptl);
 	}
 	huge_pmd_unshare_flush(&tlb, vma);
diff --git a/mm/hugetlb_cma.c b/mm/hugetlb_cma.c
index 7693ccefd0c64..39344d6c78d84 100644
--- a/mm/hugetlb_cma.c
+++ b/mm/hugetlb_cma.c
@@ -142,7 +142,7 @@ unsigned int __weak arch_hugetlb_cma_order(void)
 
 void __init hugetlb_cma_reserve(void)
 {
-	unsigned long size, reserved, per_node, order;
+	unsigned long size, reserved, per_node, order, gigantic_page_size;
 	bool node_specific_cma_alloc = false;
 	int nid;
 
@@ -162,37 +162,36 @@ void __init hugetlb_cma_reserve(void)
 	 * breaking this assumption.
 	 */
 	VM_WARN_ON(order <= MAX_PAGE_ORDER);
+	gigantic_page_size = PAGE_SIZE << order;
 
 	hugetlb_bootmem_set_nodes();
 
 	for (nid = 0; nid < MAX_NUMNODES; nid++) {
-		if (hugetlb_cma_size_in_node[nid] == 0)
+		size = hugetlb_cma_size_in_node[nid];
+		if (size == 0)
 			continue;
 
 		if (!node_isset(nid, hugetlb_bootmem_nodes)) {
 			pr_warn("hugetlb_cma: invalid node %d specified\n", nid);
-			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
-			hugetlb_cma_size_in_node[nid] = 0;
-			continue;
-		}
-
-		if (hugetlb_cma_size_in_node[nid] < (PAGE_SIZE << order)) {
-			pr_warn("hugetlb_cma: cma area of node %d should be at least %lu MiB\n",
-				nid, (PAGE_SIZE << order) / SZ_1M);
-			hugetlb_cma_size -= hugetlb_cma_size_in_node[nid];
-			hugetlb_cma_size_in_node[nid] = 0;
+		} else if (!IS_ALIGNED(size, gigantic_page_size)) {
+			pr_warn("hugetlb_cma: cma area of node %d must be a multiple of %lu MiB\n",
+				nid, gigantic_page_size / SZ_1M);
 		} else {
 			node_specific_cma_alloc = true;
+			continue;
 		}
+
+		hugetlb_cma_size -= size;
+		hugetlb_cma_size_in_node[nid] = 0;
 	}
 
 	/* Validate the CMA size again in case some invalid nodes specified. */
 	if (!hugetlb_cma_size)
 		return;
 
-	if (hugetlb_cma_size < (PAGE_SIZE << order)) {
-		pr_warn("hugetlb_cma: cma area should be at least %lu MiB\n",
-			(PAGE_SIZE << order) / SZ_1M);
+	if (!IS_ALIGNED(hugetlb_cma_size, gigantic_page_size)) {
+		pr_warn("hugetlb_cma: cma area must be a multiple of %lu MiB\n",
+			gigantic_page_size / SZ_1M);
 		hugetlb_cma_size = 0;
 		return;
 	}
@@ -204,7 +203,7 @@ void __init hugetlb_cma_reserve(void)
 		 */
 		per_node = DIV_ROUND_UP(hugetlb_cma_size,
 					nodes_weight(hugetlb_bootmem_nodes));
-		per_node = round_up(per_node, PAGE_SIZE << order);
+		per_node = round_up(per_node, gigantic_page_size);
 		pr_info("hugetlb_cma: reserve %lu MiB, up to %lu MiB per node\n",
 			hugetlb_cma_size / SZ_1M, per_node / SZ_1M);
 	}
@@ -223,15 +222,13 @@ void __init hugetlb_cma_reserve(void)
 			size = min(per_node, hugetlb_cma_size - reserved);
 		}
 
-		size = round_up(size, PAGE_SIZE << order);
-
 		snprintf(name, sizeof(name), "hugetlb%d", nid);
 		/*
 		 * Note that 'order per bit' is based on smallest size that
 		 * may be returned to CMA allocator in the case of
 		 * huge page demotion.
 		 */
-		res = cma_declare_contiguous_multi(size, PAGE_SIZE << order,
+		res = cma_declare_contiguous_multi(size, gigantic_page_size,
 					HUGETLB_PAGE_ORDER, name,
 					&hugetlb_cma[nid], nid);
 		if (res) {
diff --git a/mm/internal.h b/mm/internal.h
index 5a2ddcf68e0b6..09931b1e535f3 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -1104,9 +1104,17 @@ static inline void init_cma_pageblock(struct page *page)
 }
 #endif
 
-
-int find_suitable_fallback(struct free_area *area, unsigned int order,
-			   int migratetype, bool claimable);
+enum fallback_result {
+	/* Found suitable migratetype, *mt_out is valid. */
+	FALLBACK_FOUND,
+	/* No fallback found in requested order. */
+	FALLBACK_EMPTY,
+	/* Passed @claimable, but claiming whole block is a bad idea. */
+	FALLBACK_NOCLAIM,
+};
+enum fallback_result
+find_suitable_fallback(struct free_area *area, unsigned int order,
+		       int migratetype, bool claimable, int *mt_out);
 
 static inline bool free_area_empty(struct free_area *area, int migratetype)
 {
diff --git a/mm/kfence/core.c b/mm/kfence/core.c
index 655dc5ce32409..ee6ae01de5aef 100644
--- a/mm/kfence/core.c
+++ b/mm/kfence/core.c
@@ -77,6 +77,11 @@ static int param_set_sample_interval(const char *val, const struct kernel_param
 		WRITE_ONCE(kfence_enabled, false);
 	}
 
+	if (num && kasan_hw_tags_enabled()) {
+		pr_info("disabled as KASAN HW tags are enabled\n");
+		return -EINVAL;
+	}
+
 	*((unsigned long *)kp->arg) = num;
 
 	if (num && !READ_ONCE(kfence_enabled) && system_state != SYSTEM_BOOTING)
diff --git a/mm/kfence/kfence_test.c b/mm/kfence/kfence_test.c
index 5725a367246d9..6ec8ee26f845c 100644
--- a/mm/kfence/kfence_test.c
+++ b/mm/kfence/kfence_test.c
@@ -214,7 +214,7 @@ static void test_cache_destroy(void)
 static inline size_t kmalloc_cache_alignment(size_t size)
 {
 	/* just to get ->align so no need to pass in the real caller */
-	enum kmalloc_cache_type type = kmalloc_type(GFP_KERNEL, 0);
+	enum kmalloc_cache_type type = kmalloc_type(GFP_KERNEL, __kmalloc_token(0));
 	return kmalloc_caches[type][__kmalloc_index(size, false)]->align;
 }
 
@@ -263,7 +263,7 @@ static void *test_alloc(struct kunit *test, size_t size, gfp_t gfp, enum allocat
 		break;
 	}
 
-	kunit_info(test, "%s: size=%zu, gfp=%x, policy=%s, cache=%i\n", __func__, size, gfp,
+	kunit_info(test, "%s: size=%zu, gfp=%pGg, policy=%s, cache=%i\n", __func__, size, &gfp,
 		   policy_name, !!test_cache);
 
 	/*
@@ -285,7 +285,7 @@ static void *test_alloc(struct kunit *test, size_t size, gfp_t gfp, enum allocat
 
 		if (is_kfence_address(alloc)) {
 			struct slab *slab = virt_to_slab(alloc);
-			enum kmalloc_cache_type type = kmalloc_type(GFP_KERNEL, _RET_IP_);
+			enum kmalloc_cache_type type = kmalloc_type(GFP_KERNEL, __kmalloc_token(size));
 			struct kmem_cache *s = test_cache ?:
 					kmalloc_caches[type][__kmalloc_index(size, false)];
 
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index b8452dbdb043f..1a25af3d6d0f8 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -61,6 +61,7 @@ enum scan_result {
 	SCAN_COPY_MC,
 	SCAN_PAGE_FILLED,
 	SCAN_PAGE_DIRTY_OR_WRITEBACK,
+	SCAN_INVALID_PTES_NONE,
 };
 
 #define CREATE_TRACE_POINTS
@@ -99,6 +100,30 @@ static DEFINE_READ_MOSTLY_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
 
 static struct kmem_cache *mm_slot_cache __ro_after_init;
 
+#define KHUGEPAGED_MIN_MTHP_ORDER	2
+/*
+ * mthp_collapse() does an iterative DFS over a binary tree, from
+ * HPAGE_PMD_ORDER down to KHUGEPAGED_MIN_MTHP_ORDER. The max stack
+ * size needed for a DFS on a binary tree is height + 1, where
+ * height = HPAGE_PMD_ORDER - KHUGEPAGED_MIN_MTHP_ORDER.
+ *
+ * ilog2 is used in place of HPAGE_PMD_ORDER because some architectures
+ * (e.g. ppc64le) do not define HPAGE_PMD_ORDER until after build time.
+ */
+#define MTHP_STACK_SIZE	(ilog2(MAX_PTRS_PER_PTE) - KHUGEPAGED_MIN_MTHP_ORDER + 1)
+
+/*
+ * Defines a range of PTE entries in a PTE page table which are being
+ * considered for mTHP collapse.
+ *
+ * @offset: the offset of the first PTE entry in a PMD range.
+ * @order: the order of the PTE entries being considered for collapse.
+ */
+struct mthp_range {
+	u16 offset;
+	u8 order;
+};
+
 struct collapse_control {
 	bool is_khugepaged;
 
@@ -110,6 +135,12 @@ struct collapse_control {
 
 	/* nodemask for allocation fallback */
 	nodemask_t alloc_nmask;
+
+	/* Each bit represents a single occupied (!none/zero) page. */
+	DECLARE_BITMAP(mthp_bitmap, MAX_PTRS_PER_PTE);
+	/* A mask of the current range being considered for mTHP collapse. */
+	DECLARE_BITMAP(mthp_bitmap_mask, MAX_PTRS_PER_PTE);
+	struct mthp_range mthp_bitmap_stack[MTHP_STACK_SIZE];
 };
 
 /**
@@ -348,6 +379,91 @@ static bool pte_none_or_zero(pte_t pte)
 	return pte_present(pte) && is_zero_pfn(pte_pfn(pte));
 }
 
+/**
+ * collapse_max_ptes_none - Calculate maximum allowed none-page or zero-page
+ * PTEs for the given collapse operation.
+ * @cc: The collapse control struct
+ * @vma: The vma to check for userfaultfd
+ * @order: The folio order being collapsed to
+ *
+ * Return: Maximum number of none-page or zero-page PTEs allowed for the
+ * collapse operation.
+ */
+static int collapse_max_ptes_none(struct collapse_control *cc,
+		struct vm_area_struct *vma, unsigned int order)
+{
+	unsigned int max_ptes_none = khugepaged_max_ptes_none;
+	// If the vma is userfaultfd-armed, allow no none-page or zero-page PTEs.
+	if (vma && userfaultfd_armed(vma))
+		return 0;
+	// for MADV_COLLAPSE, allow any none-page or zero-page PTEs.
+	if (!cc->is_khugepaged)
+		return HPAGE_PMD_NR;
+	// for PMD collapse, respect the user defined maximum.
+	if (is_pmd_order(order))
+		return max_ptes_none;
+	/* Zero/non-present collapse disabled. */
+	if (!max_ptes_none)
+		return 0;
+	// for mTHP collapse with the sysctl value set to KHUGEPAGED_MAX_PTES_LIMIT,
+	// scale the maximum number of PTEs to the order of the collapse.
+	if (max_ptes_none == KHUGEPAGED_MAX_PTES_LIMIT)
+		return (1 << order) - 1;
+
+	// We currently only support max_ptes_none values of 0 or KHUGEPAGED_MAX_PTES_LIMIT.
+	// Emit a warning and return -EINVAL.
+	pr_warn_once("mTHP collapse only supports max_ptes_none values of 0 or %u\n",
+		      KHUGEPAGED_MAX_PTES_LIMIT);
+	return -EINVAL;
+}
+
+/**
+ * collapse_max_ptes_shared - Calculate maximum allowed PTEs that map shared
+ * anonymous pages for the given collapse operation.
+ * @cc: The collapse control struct
+ * @order: The folio order being collapsed to
+ *
+ * Return: Maximum number of PTEs that map shared anonymous pages for the
+ * collapse operation
+ */
+static unsigned int collapse_max_ptes_shared(struct collapse_control *cc,
+		unsigned int order)
+{
+	// for MADV_COLLAPSE, do not restrict the number of PTEs that map shared
+	// anonymous pages.
+	if (!cc->is_khugepaged)
+		return HPAGE_PMD_NR;
+	// for mTHP collapse do not allow collapsing anonymous memory pages that
+	// are shared between processes.
+	if (!is_pmd_order(order))
+		return 0;
+	// for PMD collapse, respect the user defined maximum.
+	return khugepaged_max_ptes_shared;
+}
+
+/**
+ * collapse_max_ptes_swap - Calculate the maximum allowed non-present PTEs or the
+ * maximum allowed non-present pagecache entries for the given collapse operation.
+ * @cc: The collapse control struct
+ * @order: The folio order being collapsed to
+ *
+ * Return: Maximum number of non-present PTEs or the maximum allowed non-present
+ * pagecache entries for the collapse operation.
+ */
+static unsigned int collapse_max_ptes_swap(struct collapse_control *cc,
+		unsigned int order)
+{
+	// for MADV_COLLAPSE, do not restrict the number PTEs entries or
+	// pagecache entries that are non-present.
+	if (!cc->is_khugepaged)
+		return HPAGE_PMD_NR;
+	// for mTHP collapse do not allow any non-present PTEs or pagecache entries.
+	if (!is_pmd_order(order))
+		return 0;
+	// for PMD collapse, respect the user defined maximum.
+	return khugepaged_max_ptes_swap;
+}
+
 int hugepage_madvise(struct vm_area_struct *vma,
 		     vm_flags_t *vm_flags, int advice)
 {
@@ -406,26 +522,32 @@ static inline int collapse_test_exit_or_disable(struct mm_struct *mm)
 		mm_flags_test(MMF_DISABLE_THP_COMPLETELY, mm);
 }
 
-static bool hugepage_pmd_enabled(void)
+static inline bool anon_hpage_enabled(void)
+{
+	if (READ_ONCE(huge_anon_orders_always))
+		return true;
+	if (READ_ONCE(huge_anon_orders_madvise))
+		return true;
+	if (READ_ONCE(huge_anon_orders_inherit) &&
+	    hugepage_global_enabled())
+		return true;
+	return false;
+}
+
+static bool hugepage_enabled(void)
 {
 	/*
 	 * We cover the anon, shmem and the file-backed case here; file-backed
-	 * hugepages, when configured in, are determined by the global control.
-	 * Anon pmd-sized hugepages are determined by the pmd-size control.
+	 * hugepages are determined by the global control.
+	 * Anon hugepages are determined by its per-size mTHP control.
 	 * Shmem pmd-sized hugepages are also determined by its pmd-size control,
 	 * except when the global shmem_huge is set to SHMEM_HUGE_DENY.
 	 */
-	if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
-	    hugepage_global_enabled())
+	if (hugepage_global_enabled())
 		return true;
-	if (test_bit(PMD_ORDER, &huge_anon_orders_always))
+	if (anon_hpage_enabled())
 		return true;
-	if (test_bit(PMD_ORDER, &huge_anon_orders_madvise))
-		return true;
-	if (test_bit(PMD_ORDER, &huge_anon_orders_inherit) &&
-	    hugepage_global_enabled())
-		return true;
-	if (IS_ENABLED(CONFIG_SHMEM) && shmem_hpage_pmd_enabled())
+	if (shmem_hpage_pmd_enabled())
 		return true;
 	return false;
 }
@@ -437,13 +559,16 @@ void __khugepaged_enter(struct mm_struct *mm)
 
 	/* __khugepaged_exit() must not run from under us */
 	VM_BUG_ON_MM(collapse_test_exit(mm), mm);
-	if (unlikely(mm_flags_test_and_set(MMF_VM_HUGEPAGE, mm)))
-		return;
 
 	slot = mm_slot_alloc(mm_slot_cache);
 	if (!slot)
 		return;
 
+	if (unlikely(mm_flags_test_and_set(MMF_VM_HUGEPAGE, mm))) {
+		mm_slot_free(mm_slot_cache, slot);
+		return;
+	}
+
 	spin_lock(&khugepaged_mm_lock);
 	mm_slot_insert(mm_slots_hash, mm, slot);
 	/*
@@ -459,14 +584,28 @@ void __khugepaged_enter(struct mm_struct *mm)
 		wake_up_interruptible(&khugepaged_wait);
 }
 
+/* Check what orders are allowed based on the vma and collapse type */
+static unsigned long collapse_allowable_orders(struct vm_area_struct *vma,
+		vm_flags_t vm_flags, enum tva_type tva_flags)
+{
+	unsigned long orders;
+
+	/* If khugepaged is scanning an anonymous vma, allow mTHP collapse */
+	if ((tva_flags == TVA_KHUGEPAGED) && vma_is_anonymous(vma))
+		orders = THP_ORDERS_ALL_ANON;
+	else
+		orders = BIT(HPAGE_PMD_ORDER);
+
+	return thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders);
+}
+
 void khugepaged_enter_vma(struct vm_area_struct *vma,
 			  vm_flags_t vm_flags)
 {
 	if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) &&
-	    hugepage_pmd_enabled()) {
-		if (thp_vma_allowable_order(vma, vm_flags, TVA_KHUGEPAGED, PMD_ORDER))
-			__khugepaged_enter(vma->vm_mm);
-	}
+	    collapse_allowable_orders(vma, vm_flags, TVA_KHUGEPAGED) &&
+	    hugepage_enabled())
+		__khugepaged_enter(vma->vm_mm);
 }
 
 void __khugepaged_exit(struct mm_struct *mm)
@@ -538,29 +677,33 @@ static void release_pte_pages(pte_t *pte, pte_t *_pte,
 
 static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
 		unsigned long start_addr, pte_t *pte, struct collapse_control *cc,
-		struct list_head *compound_pagelist)
+		unsigned int order, struct list_head *compound_pagelist)
 {
+	const unsigned long nr_pages = 1UL << order;
 	struct page *page = NULL;
 	struct folio *folio = NULL;
 	unsigned long addr = start_addr;
 	pte_t *_pte;
 	int none_or_zero = 0, shared = 0, referenced = 0;
 	enum scan_result result = SCAN_FAIL;
+	int max_ptes_none = collapse_max_ptes_none(cc, vma, order);
+	unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, order);
+
+	if (max_ptes_none < 0)
+		return SCAN_INVALID_PTES_NONE;
 
-	for (_pte = pte; _pte < pte + HPAGE_PMD_NR;
+	for (_pte = pte; _pte < pte + nr_pages;
 	     _pte++, addr += PAGE_SIZE) {
 		pte_t pteval = ptep_get(_pte);
 		if (pte_none_or_zero(pteval)) {
-			++none_or_zero;
-			if (!userfaultfd_armed(vma) &&
-			    (!cc->is_khugepaged ||
-			     none_or_zero <= khugepaged_max_ptes_none)) {
-				continue;
-			} else {
+			if (++none_or_zero > max_ptes_none) {
 				result = SCAN_EXCEED_NONE_PTE;
-				count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
+				if (is_pmd_order(order))
+					count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
+				count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_NONE);
 				goto out;
 			}
+			continue;
 		}
 		if (!pte_present(pteval)) {
 			result = SCAN_PTE_NON_PRESENT;
@@ -591,14 +734,28 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
 
 		/* See collapse_scan_pmd(). */
 		if (folio_maybe_mapped_shared(folio)) {
-			++shared;
-			if (cc->is_khugepaged &&
-			    shared > khugepaged_max_ptes_shared) {
+			/*
+			 * TODO: Support shared pages without leading to further
+			 * mTHP collapses. Currently bringing in new pages via
+			 * shared may cause a future higher order collapse on a
+			 * rescan of the same range.
+			 */
+			if (++shared > max_ptes_shared) {
 				result = SCAN_EXCEED_SHARED_PTE;
-				count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+				if (is_pmd_order(order))
+					count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+				count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_SHARED);
 				goto out;
 			}
 		}
+		/*
+		 * TODO: In some cases of partially-mapped folios, we'd actually
+		 * want to collapse.
+		 */
+		if (!is_pmd_order(order) && folio_order(folio) >= order) {
+			result = SCAN_PTE_MAPPED_HUGEPAGE;
+			goto out;
+		}
 
 		if (folio_test_large(folio)) {
 			struct folio *f;
@@ -675,29 +832,29 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
 	} else {
 		result = SCAN_SUCCEED;
 		trace_mm_collapse_huge_page_isolate(folio, none_or_zero,
-						    referenced, result);
+						    referenced, result, order);
 		return result;
 	}
 out:
 	release_pte_pages(pte, _pte, compound_pagelist);
 	trace_mm_collapse_huge_page_isolate(folio, none_or_zero,
-					    referenced, result);
+					    referenced, result, order);
 	return result;
 }
 
 static void __collapse_huge_page_copy_succeeded(pte_t *pte,
-						struct vm_area_struct *vma,
-						unsigned long address,
-						spinlock_t *ptl,
-						struct list_head *compound_pagelist)
+		struct vm_area_struct *vma, unsigned long address,
+		spinlock_t *ptl, unsigned int order,
+		struct list_head *compound_pagelist)
 {
-	unsigned long end = address + HPAGE_PMD_SIZE;
+	const unsigned long nr_pages = 1UL << order;
+	unsigned long end = address + (PAGE_SIZE << order);
 	struct folio *src, *tmp;
 	pte_t pteval;
 	pte_t *_pte;
 	unsigned int nr_ptes;
 
-	for (_pte = pte; _pte < pte + HPAGE_PMD_NR; _pte += nr_ptes,
+	for (_pte = pte; _pte < pte + nr_pages; _pte += nr_ptes,
 	     address += nr_ptes * PAGE_SIZE) {
 		nr_ptes = 1;
 		pteval = ptep_get(_pte);
@@ -750,11 +907,10 @@ static void __collapse_huge_page_copy_succeeded(pte_t *pte,
 }
 
 static void __collapse_huge_page_copy_failed(pte_t *pte,
-					     pmd_t *pmd,
-					     pmd_t orig_pmd,
-					     struct vm_area_struct *vma,
-					     struct list_head *compound_pagelist)
+		pmd_t *pmd, pmd_t orig_pmd, struct vm_area_struct *vma,
+		unsigned int order, struct list_head *compound_pagelist)
 {
+	const unsigned long nr_pages = 1UL << order;
 	spinlock_t *pmd_ptl;
 
 	/*
@@ -770,7 +926,7 @@ static void __collapse_huge_page_copy_failed(pte_t *pte,
 	 * Release both raw and compound pages isolated
 	 * in __collapse_huge_page_isolate.
 	 */
-	release_pte_pages(pte, pte + HPAGE_PMD_NR, compound_pagelist);
+	release_pte_pages(pte, pte + nr_pages, compound_pagelist);
 }
 
 /*
@@ -790,16 +946,17 @@ static void __collapse_huge_page_copy_failed(pte_t *pte,
  */
 static enum scan_result __collapse_huge_page_copy(pte_t *pte, struct folio *folio,
 		pmd_t *pmd, pmd_t orig_pmd, struct vm_area_struct *vma,
-		unsigned long address, spinlock_t *ptl,
+		unsigned long address, spinlock_t *ptl, unsigned int order,
 		struct list_head *compound_pagelist)
 {
+	const unsigned long nr_pages = 1UL << order;
 	unsigned int i;
 	enum scan_result result = SCAN_SUCCEED;
 
 	/*
 	 * Copying pages' contents is subject to memory poison at any iteration.
 	 */
-	for (i = 0; i < HPAGE_PMD_NR; i++) {
+	for (i = 0; i < nr_pages; i++) {
 		pte_t pteval = ptep_get(pte + i);
 		struct page *page = folio_page(folio, i);
 		unsigned long src_addr = address + i * PAGE_SIZE;
@@ -818,10 +975,10 @@ static enum scan_result __collapse_huge_page_copy(pte_t *pte, struct folio *foli
 
 	if (likely(result == SCAN_SUCCEED))
 		__collapse_huge_page_copy_succeeded(pte, vma, address, ptl,
-						    compound_pagelist);
+						    order, compound_pagelist);
 	else
 		__collapse_huge_page_copy_failed(pte, pmd, orig_pmd, vma,
-						 compound_pagelist);
+						 order, compound_pagelist);
 
 	return result;
 }
@@ -902,12 +1059,13 @@ static int collapse_find_target_node(struct collapse_control *cc)
 
 /*
  * If mmap_lock temporarily dropped, revalidate vma
- * before taking mmap_lock.
+ * after taking the mmap_lock again.
  * Returns enum scan_result value.
  */
 
 static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
-		bool expect_anon, struct vm_area_struct **vmap, struct collapse_control *cc)
+		bool expect_anon, struct vm_area_struct **vmap,
+		struct collapse_control *cc, unsigned int order)
 {
 	struct vm_area_struct *vma;
 	enum tva_type type = cc->is_khugepaged ? TVA_KHUGEPAGED :
@@ -920,15 +1078,16 @@ static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned l
 	if (!vma)
 		return SCAN_VMA_NULL;
 
+	/* Always check the PMD order to ensure its not shared by another VMA */
 	if (!thp_vma_suitable_order(vma, address, PMD_ORDER))
 		return SCAN_ADDRESS_RANGE;
-	if (!thp_vma_allowable_order(vma, vma->vm_flags, type, PMD_ORDER))
+	if (!thp_vma_allowable_orders(vma, vma->vm_flags, type, BIT(order)))
 		return SCAN_VMA_CHECK;
 	/*
 	 * Anon VMA expected, the address may be unmapped then
 	 * remapped to file after khugepaged reaquired the mmap_lock.
 	 *
-	 * thp_vma_allowable_order may return true for qualified file
+	 * thp_vma_allowable_orders may return true for qualified file
 	 * vmas.
 	 */
 	if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap)))
@@ -990,12 +1149,12 @@ static enum scan_result check_pmd_still_valid(struct mm_struct *mm,
  * Returns result: if not SCAN_SUCCEED, mmap_lock has been released.
  */
 static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm,
-		struct vm_area_struct *vma, unsigned long start_addr, pmd_t *pmd,
-		int referenced)
+		struct vm_area_struct *vma, unsigned long start_addr,
+		pmd_t *pmd, int referenced, unsigned int order)
 {
 	int swapped_in = 0;
 	vm_fault_t ret = 0;
-	unsigned long addr, end = start_addr + (HPAGE_PMD_NR * PAGE_SIZE);
+	unsigned long addr, end = start_addr + (PAGE_SIZE << order);
 	enum scan_result result;
 	pte_t *pte = NULL;
 	spinlock_t *ptl;
@@ -1027,6 +1186,20 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm,
 		    pte_present(vmf.orig_pte))
 			continue;
 
+		/*
+		 * TODO: Support swapin without leading to further mTHP
+		 * collapses. Currently bringing in new pages via swapin may
+		 * cause a future higher order collapse on a rescan of the same
+		 * range.
+		 */
+		if (!is_pmd_order(order)) {
+			count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_SWAP);
+			pte_unmap(pte);
+			mmap_read_unlock(mm);
+			result = SCAN_EXCEED_SWAP_PTE;
+			goto out;
+		}
+
 		vmf.pte = pte;
 		vmf.ptl = ptl;
 		ret = do_swap_page(&vmf);
@@ -1061,26 +1234,32 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm,
 
 	result = SCAN_SUCCEED;
 out:
-	trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, result);
+	trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, result,
+					   order);
 	return result;
 }
 
 static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_struct *mm,
-		struct collapse_control *cc)
+		struct collapse_control *cc, unsigned int order)
 {
 	gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() :
 		     GFP_TRANSHUGE);
 	int node = collapse_find_target_node(cc);
 	struct folio *folio;
 
-	folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask);
+	folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask);
 	if (!folio) {
 		*foliop = NULL;
-		count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
+		if (is_pmd_order(order))
+			count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
+		count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC_FAILED);
 		return SCAN_ALLOC_HUGE_PAGE_FAIL;
 	}
 
-	count_vm_event(THP_COLLAPSE_ALLOC);
+	if (is_pmd_order(order))
+		count_vm_event(THP_COLLAPSE_ALLOC);
+	count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC);
+
 	if (unlikely(mem_cgroup_charge(folio, mm, gfp))) {
 		folio_put(folio);
 		*foliop = NULL;
@@ -1093,41 +1272,44 @@ static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_stru
 	return SCAN_SUCCEED;
 }
 
-static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long address,
-		int referenced, int unmapped, struct collapse_control *cc)
+/*
+ * collapse_huge_page expects the mmap_read_lock to be dropped before
+ * entering this function. The function will also always return with the lock
+ * dropped. The function starts by allocation a folio, which can potentially
+ * take a long time if it involves sync compaction, and we do not need to hold
+ * the mmap_lock during that. We must recheck the vma after taking it again in
+ * write mode.
+ */
+static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long start_addr,
+		int referenced, int unmapped, struct collapse_control *cc,
+		unsigned int order)
 {
+	const unsigned long pmd_addr = start_addr & HPAGE_PMD_MASK;
+	const unsigned long end_addr = start_addr + (PAGE_SIZE << order);
 	LIST_HEAD(compound_pagelist);
 	pmd_t *pmd, _pmd;
-	pte_t *pte;
+	pte_t *pte = NULL;
 	pgtable_t pgtable;
 	struct folio *folio;
 	spinlock_t *pmd_ptl, *pte_ptl;
 	enum scan_result result = SCAN_FAIL;
 	struct vm_area_struct *vma;
 	struct mmu_notifier_range range;
+	bool anon_vma_locked = false;
 
-	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
-
-	/*
-	 * Before allocating the hugepage, release the mmap_lock read lock.
-	 * The allocation can take potentially a long time if it involves
-	 * sync compaction, and we do not need to hold the mmap_lock during
-	 * that. We will recheck the vma after taking it again in write mode.
-	 */
-	mmap_read_unlock(mm);
-
-	result = alloc_charge_folio(&folio, mm, cc);
+	result = alloc_charge_folio(&folio, mm, cc, order);
 	if (result != SCAN_SUCCEED)
 		goto out_nolock;
 
 	mmap_read_lock(mm);
-	result = hugepage_vma_revalidate(mm, address, true, &vma, cc);
+	result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true,
+					 &vma, cc, order);
 	if (result != SCAN_SUCCEED) {
 		mmap_read_unlock(mm);
 		goto out_nolock;
 	}
 
-	result = find_pmd_or_thp_or_none(mm, address, &pmd);
+	result = find_pmd_or_thp_or_none(mm, pmd_addr, &pmd);
 	if (result != SCAN_SUCCEED) {
 		mmap_read_unlock(mm);
 		goto out_nolock;
@@ -1139,8 +1321,8 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
 		 * released when it fails. So we jump out_nolock directly in
 		 * that case.  Continuing to collapse causes inconsistency.
 		 */
-		result = __collapse_huge_page_swapin(mm, vma, address, pmd,
-						     referenced);
+		result = __collapse_huge_page_swapin(mm, vma, start_addr, pmd,
+						     referenced, order);
 		if (result != SCAN_SUCCEED)
 			goto out_nolock;
 	}
@@ -1155,19 +1337,21 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
 	 * mmap_lock.
 	 */
 	mmap_write_lock(mm);
-	result = hugepage_vma_revalidate(mm, address, true, &vma, cc);
+	result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true,
+					 &vma, cc, order);
 	if (result != SCAN_SUCCEED)
 		goto out_up_write;
 	/* check if the pmd is still valid */
 	vma_start_write(vma);
-	result = check_pmd_still_valid(mm, address, pmd);
+	result = check_pmd_still_valid(mm, pmd_addr, pmd);
 	if (result != SCAN_SUCCEED)
 		goto out_up_write;
 
 	anon_vma_lock_write(vma->anon_vma);
+	anon_vma_locked = true;
 
-	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, address,
-				address + HPAGE_PMD_SIZE);
+	mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, start_addr,
+				end_addr);
 	mmu_notifier_invalidate_range_start(&range);
 
 	pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
@@ -1179,25 +1363,23 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
 	 * Parallel GUP-fast is fine since GUP-fast will back off when
 	 * it detects PMD is changed.
 	 */
-	_pmd = pmdp_collapse_flush(vma, address, pmd);
+	_pmd = pmdp_collapse_flush(vma, pmd_addr, pmd);
 	spin_unlock(pmd_ptl);
 	mmu_notifier_invalidate_range_end(&range);
 	tlb_remove_table_sync_one();
 
-	pte = pte_offset_map_lock(mm, &_pmd, address, &pte_ptl);
+	pte = pte_offset_map_lock(mm, &_pmd, start_addr, &pte_ptl);
 	if (pte) {
-		result = __collapse_huge_page_isolate(vma, address, pte, cc,
-						      &compound_pagelist);
+		result = __collapse_huge_page_isolate(vma, start_addr, pte, cc,
+						      order, &compound_pagelist);
 		spin_unlock(pte_ptl);
 	} else {
 		result = SCAN_NO_PTE_TABLE;
 	}
 
 	if (unlikely(result != SCAN_SUCCEED)) {
-		if (pte)
-			pte_unmap(pte);
 		spin_lock(pmd_ptl);
-		BUG_ON(!pmd_none(*pmd));
+		WARN_ON_ONCE(!pmd_none(*pmd));
 		/*
 		 * We can only use set_pmd_at when establishing
 		 * hugepmds and never for establishing regular pmds that
@@ -1205,20 +1387,24 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
 		 */
 		pmd_populate(mm, pmd, pmd_pgtable(_pmd));
 		spin_unlock(pmd_ptl);
-		anon_vma_unlock_write(vma->anon_vma);
 		goto out_up_write;
 	}
 
 	/*
-	 * All pages are isolated and locked so anon_vma rmap
-	 * can't run anymore.
+	 * For PMD collapse all pages are isolated and locked so anon_vma
+	 * rmap can't run anymore. For mTHP collapse the PMD entry has been
+	 * removed and not all pages are isolated and locked, so we must hold
+	 * the lock to prevent neighboring folios from attempting to access
+	 * this PMD until its reinstalled.
 	 */
-	anon_vma_unlock_write(vma->anon_vma);
+	if (is_pmd_order(order)) {
+		anon_vma_unlock_write(vma->anon_vma);
+		anon_vma_locked = false;
+	}
 
 	result = __collapse_huge_page_copy(pte, folio, pmd, _pmd,
-					   vma, address, pte_ptl,
-					   &compound_pagelist);
-	pte_unmap(pte);
+					   vma, start_addr, pte_ptl,
+					   order, &compound_pagelist);
 	if (unlikely(result != SCAN_SUCCEED))
 		goto out_up_write;
 
@@ -1228,37 +1414,196 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
 	 * write.
 	 */
 	__folio_mark_uptodate(folio);
-	pgtable = pmd_pgtable(_pmd);
-
 	spin_lock(pmd_ptl);
-	BUG_ON(!pmd_none(*pmd));
-	pgtable_trans_huge_deposit(mm, pmd, pgtable);
-	map_anon_folio_pmd_nopf(folio, pmd, vma, address);
+	WARN_ON_ONCE(!pmd_none(*pmd));
+	if (is_pmd_order(order)) {
+		pgtable = pmd_pgtable(_pmd);
+		pgtable_trans_huge_deposit(mm, pmd, pgtable);
+		map_anon_folio_pmd_nopf(folio, pmd, vma, pmd_addr);
+	} else {
+		/*
+		 * set_ptes is called in map_anon_folio_pte_nopf with the
+		 * pmd_ptl lock still held; this is safe as the PMD is expected
+		 * to be none. The pmd entry is then repopulated below.
+		 */
+		map_anon_folio_pte_nopf(folio, pte, vma, start_addr, /*uffd_wp=*/ false);
+		smp_wmb(); /* make PTEs visible before PMD. See pmd_install() */
+		pmd_populate(mm, pmd, pmd_pgtable(_pmd));
+	}
 	spin_unlock(pmd_ptl);
 
 	folio = NULL;
 
 	result = SCAN_SUCCEED;
 out_up_write:
+	if (anon_vma_locked)
+		anon_vma_unlock_write(vma->anon_vma);
+	if (pte)
+		pte_unmap(pte);
 	mmap_write_unlock(mm);
 out_nolock:
 	if (folio)
 		folio_put(folio);
-	trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result);
+	trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result, order);
 	return result;
 }
 
+static void collapse_mthp_stack_push(struct collapse_control *cc, int *stack_size,
+				     u16 offset, u8 order)
+{
+	const int size = *stack_size;
+	struct mthp_range *stack = &cc->mthp_bitmap_stack[size];
+
+	VM_WARN_ON_ONCE(size >= MTHP_STACK_SIZE);
+	stack->order = order;
+	stack->offset = offset;
+	(*stack_size)++;
+}
+
+static struct mthp_range collapse_mthp_stack_pop(struct collapse_control *cc,
+						 int *stack_size)
+{
+	const int size = *stack_size;
+
+	VM_WARN_ON_ONCE(size <= 0);
+	(*stack_size)--;
+	return cc->mthp_bitmap_stack[size - 1];
+}
+
+static unsigned int collapse_mthp_count_present(struct collapse_control *cc,
+						u16 offset, unsigned int nr_ptes)
+{
+	bitmap_zero(cc->mthp_bitmap_mask, MAX_PTRS_PER_PTE);
+	bitmap_set(cc->mthp_bitmap_mask, offset, nr_ptes);
+	return bitmap_weight_and(cc->mthp_bitmap, cc->mthp_bitmap_mask, MAX_PTRS_PER_PTE);
+}
+
+/*
+ * mthp_collapse() consumes the bitmap that is generated during
+ * collapse_scan_pmd() to determine what regions and mTHP orders fit best.
+ *
+ * Each bit in cc->mthp_bitmap represents a single occupied (!none/zero) page.
+ * A stack structure cc->mthp_bitmap_stack is used to check different regions
+ * of the bitmap for collapse eligibility. The stack maintains a pair of
+ * variables (offset, order), indicating the number of PTEs from the start of
+ * the PMD, and the order of the potential collapse candidate respectively. We
+ * start at the PMD order and check if it is eligible for collapse; if not, we
+ * add two entries to the stack at a lower order to represent the left and right
+ * halves of the PTE page table we are examining.
+ *
+ *                         offset       mid_offset
+ *                         |         |
+ *                         |         |
+ *                         v         v
+ *      --------------------------------------
+ *      |          cc->mthp_bitmap            |
+ *      --------------------------------------
+ *                         <-------><------->
+ *                          order-1  order-1
+ *
+ * For each of these, we determine how many PTE entries are occupied in the
+ * range of PTE entries we propose to collapse, then we compare this to a
+ * threshold number of PTE entries which would need to be occupied for a
+ * collapse to be permitted at that order (accounting for max_ptes_none).
+ *
+ * If a collapse is permitted, we attempt to collapse the PTE range into a
+ * mTHP.
+ */
+static int mthp_collapse(struct mm_struct *mm, unsigned long address,
+		int referenced, int unmapped, struct collapse_control *cc,
+		unsigned long enabled_orders)
+{
+	unsigned int nr_occupied_ptes, nr_ptes;
+	int max_ptes_none, collapsed = 0, stack_size = 0;
+	unsigned long collapse_address;
+	struct mthp_range range;
+	u16 offset;
+	u8 order;
+
+	collapse_mthp_stack_push(cc, &stack_size, 0, HPAGE_PMD_ORDER);
+
+	while (stack_size) {
+		range = collapse_mthp_stack_pop(cc, &stack_size);
+		order = range.order;
+		offset = range.offset;
+		nr_ptes = 1UL << order;
+
+		if (!test_bit(order, &enabled_orders))
+			goto next_order;
+
+		max_ptes_none = collapse_max_ptes_none(cc, NULL, order);
+
+		if (max_ptes_none < 0)
+			return collapsed;
+
+		nr_occupied_ptes = collapse_mthp_count_present(cc, offset,
+							       nr_ptes);
+
+		if (nr_occupied_ptes >= nr_ptes - max_ptes_none) {
+			int ret;
+
+			collapse_address = address + offset * PAGE_SIZE;
+			ret = collapse_huge_page(mm, collapse_address, referenced,
+						 unmapped, cc, order);
+
+			switch (ret) {
+			/* Cases where we continue to next collapse candidate */
+			case SCAN_SUCCEED:
+				collapsed += nr_ptes;
+				fallthrough;
+			case SCAN_PTE_MAPPED_HUGEPAGE:
+				continue;
+			/* Cases where lower orders might still succeed */
+			case SCAN_LACK_REFERENCED_PAGE:
+			case SCAN_EXCEED_NONE_PTE:
+			case SCAN_EXCEED_SWAP_PTE:
+			case SCAN_EXCEED_SHARED_PTE:
+			case SCAN_PAGE_LOCK:
+			case SCAN_PAGE_COUNT:
+			case SCAN_PAGE_NULL:
+			case SCAN_DEL_PAGE_LRU:
+			case SCAN_PTE_NON_PRESENT:
+			case SCAN_PTE_UFFD_WP:
+			case SCAN_ALLOC_HUGE_PAGE_FAIL:
+			case SCAN_PAGE_LAZYFREE:
+				goto next_order;
+			/* Cases where no further collapse is possible */
+			default:
+				return collapsed;
+			}
+		}
+
+next_order:
+		if (order > KHUGEPAGED_MIN_MTHP_ORDER) {
+			const u8 next_order = order - 1;
+			const u16 mid_offset = offset + (nr_ptes / 2);
+
+			collapse_mthp_stack_push(cc, &stack_size, mid_offset,
+						 next_order);
+			collapse_mthp_stack_push(cc, &stack_size, offset,
+						 next_order);
+		}
+	}
+	return collapsed;
+}
+
 static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 		struct vm_area_struct *vma, unsigned long start_addr,
 		bool *lock_dropped, struct collapse_control *cc)
 {
+	int max_ptes_none = collapse_max_ptes_none(cc, vma, HPAGE_PMD_ORDER);
+	const unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, HPAGE_PMD_ORDER);
+	const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER);
+	enum tva_type tva_flags = cc->is_khugepaged ? TVA_KHUGEPAGED : TVA_FORCED_COLLAPSE;
 	pmd_t *pmd;
-	pte_t *pte, *_pte;
-	int none_or_zero = 0, shared = 0, referenced = 0;
+	pte_t *pte, *_pte, pteval;
+	int i;
+	int none_or_zero = 0, shared = 0, nr_collapsed = 0, referenced = 0;
 	enum scan_result result = SCAN_FAIL;
 	struct page *page = NULL;
 	struct folio *folio = NULL;
 	unsigned long addr;
+	unsigned long enabled_orders;
 	spinlock_t *ptl;
 	int node = NUMA_NO_NODE, unmapped = 0;
 
@@ -1270,8 +1615,19 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 		goto out;
 	}
 
+	bitmap_zero(cc->mthp_bitmap, MAX_PTRS_PER_PTE);
 	memset(cc->node_load, 0, sizeof(cc->node_load));
 	nodes_clear(cc->alloc_nmask);
+
+	enabled_orders = collapse_allowable_orders(vma, vma->vm_flags, tva_flags);
+
+	/*
+	 * If PMD is the only enabled order, enforce max_ptes_none, otherwise
+	 * scan all pages to populate the bitmap for mTHP collapse.
+	 */
+	if (enabled_orders != BIT(HPAGE_PMD_ORDER))
+		max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT;
+
 	pte = pte_offset_map_lock(mm, pmd, start_addr, &ptl);
 	if (!pte) {
 		cc->progress++;
@@ -1279,42 +1635,41 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 		goto out;
 	}
 
-	for (addr = start_addr, _pte = pte; _pte < pte + HPAGE_PMD_NR;
-	     _pte++, addr += PAGE_SIZE) {
+	for (i = 0; i < HPAGE_PMD_NR; i++) {
+		_pte = pte + i;
+		addr = start_addr + i * PAGE_SIZE;
+		pteval = ptep_get(_pte);
+
 		cc->progress++;
 
-		pte_t pteval = ptep_get(_pte);
 		if (pte_none_or_zero(pteval)) {
-			++none_or_zero;
-			if (!userfaultfd_armed(vma) &&
-			    (!cc->is_khugepaged ||
-			     none_or_zero <= khugepaged_max_ptes_none)) {
-				continue;
-			} else {
+			if (++none_or_zero > max_ptes_none) {
 				result = SCAN_EXCEED_NONE_PTE;
 				count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
+				count_mthp_stat(HPAGE_PMD_ORDER,
+						MTHP_STAT_COLLAPSE_EXCEED_NONE);
 				goto out_unmap;
 			}
+			continue;
 		}
 		if (!pte_present(pteval)) {
-			++unmapped;
-			if (!cc->is_khugepaged ||
-			    unmapped <= khugepaged_max_ptes_swap) {
-				/*
-				 * Always be strict with uffd-wp
-				 * enabled swap entries.  Please see
-				 * comment below for pte_uffd_wp().
-				 */
-				if (pte_swp_uffd_wp_any(pteval)) {
-					result = SCAN_PTE_UFFD_WP;
-					goto out_unmap;
-				}
-				continue;
-			} else {
+			if (++unmapped > max_ptes_swap) {
 				result = SCAN_EXCEED_SWAP_PTE;
 				count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);
+				count_mthp_stat(HPAGE_PMD_ORDER,
+						MTHP_STAT_COLLAPSE_EXCEED_SWAP);
+				goto out_unmap;
+			}
+			/*
+			 * Always be strict with uffd-wp
+			 * enabled swap entries.  Please see
+			 * comment below for pte_uffd_wp().
+			 */
+			if (pte_swp_uffd_wp_any(pteval)) {
+				result = SCAN_PTE_UFFD_WP;
 				goto out_unmap;
 			}
+			continue;
 		}
 		if (pte_uffd_wp(pteval)) {
 			/*
@@ -1357,15 +1712,17 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 		 * is shared.
 		 */
 		if (folio_maybe_mapped_shared(folio)) {
-			++shared;
-			if (cc->is_khugepaged &&
-			    shared > khugepaged_max_ptes_shared) {
+			if (++shared > max_ptes_shared) {
 				result = SCAN_EXCEED_SHARED_PTE;
 				count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+				count_mthp_stat(HPAGE_PMD_ORDER,
+						MTHP_STAT_COLLAPSE_EXCEED_SHARED);
 				goto out_unmap;
 			}
 		}
 
+		/* Set bit for occupied pages */
+		__set_bit(i, cc->mthp_bitmap);
 		/*
 		 * Record which node the original page is from and save this
 		 * information to cc->node_load[].
@@ -1422,10 +1779,13 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
 out_unmap:
 	pte_unmap_unlock(pte, ptl);
 	if (result == SCAN_SUCCEED) {
-		result = collapse_huge_page(mm, start_addr, referenced,
-					    unmapped, cc);
+		/* collapse_huge_page expects the lock to be dropped before calling */
+		mmap_read_unlock(mm);
+		nr_collapsed = mthp_collapse(mm, start_addr, referenced, unmapped,
+					      cc, enabled_orders);
 		/* collapse_huge_page will return with the mmap_lock released */
 		*lock_dropped = true;
+		result = nr_collapsed ? SCAN_SUCCEED : SCAN_FAIL;
 	}
 out:
 	trace_mm_khugepaged_scan_pmd(mm, folio, referenced,
@@ -1892,10 +2252,16 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 	int nr_none = 0;
 	bool is_shmem = shmem_file(file);
 
-	VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem);
-	VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
+	/*
+	 * MADV_COLLAPSE ignores shmem huge config, so do not check shmem
+	 *
+	 * TODO: once shmem always calls mapping_set_large_folios() on its
+	 * mapping, the shmem check can be removed.
+	 */
+	VM_WARN_ON_ONCE(!is_shmem && !mapping_pmd_folio_support(mapping));
+	VM_WARN_ON_ONCE(start & (HPAGE_PMD_NR - 1));
 
-	result = alloc_charge_folio(&new_folio, mm, cc);
+	result = alloc_charge_folio(&new_folio, mm, cc, HPAGE_PMD_ORDER);
 	if (result != SCAN_SUCCEED)
 		goto out;
 
@@ -1978,20 +2344,22 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 				}
 			} else if (folio_test_dirty(folio)) {
 				/*
-				 * khugepaged only works on read-only fd,
-				 * so this page is dirty because it hasn't
-				 * been flushed since first write. There
-				 * won't be new dirty pages.
+				 * This page is dirty because it hasn't
+				 * been flushed since first write.
 				 *
-				 * Trigger async flush here and hope the
-				 * writeback is done when khugepaged
-				 * revisits this page.
+				 * Trigger async flush for read-only files and
+				 * hope the writeback is done when khugepaged
+				 * revisits this page. Writable files can have
+				 * their folios dirty at any time; blindly
+				 * flushing them would cause undesirable
+				 * system-wide writeback.
 				 *
 				 * This is a one-off situation. We are not
 				 * forcing writeback in loop.
 				 */
 				xas_unlock_irq(&xas);
-				filemap_flush(mapping);
+				if (!inode_is_open_for_write(mapping->host))
+					filemap_flush(mapping);
 				result = SCAN_PAGE_DIRTY_OR_WRITEBACK;
 				goto xa_unlocked;
 			} else if (folio_test_writeback(folio)) {
@@ -2037,8 +2405,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 		if (!is_shmem && (folio_test_dirty(folio) ||
 				  folio_test_writeback(folio))) {
 			/*
-			 * khugepaged only works on read-only fd, so this
-			 * folio is dirty because it hasn't been flushed
+			 * khugepaged only works on clean file-backed folios,
+			 * so this folio is dirty because it hasn't been flushed
 			 * since first write.
 			 */
 			result = SCAN_PAGE_DIRTY_OR_WRITEBACK;
@@ -2082,6 +2450,27 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 			goto out_unlock;
 		}
 
+		/*
+		 * At this point, the folio is locked and unmapped. If the PTE
+		 * was dirty, try_to_unmap() has transferred the dirty bit to
+		 * the folio and we must not collapse it into a clean
+		 * file-backed folio.
+		 *
+		 * If the folio is clean here, no one can write it until we
+		 * drop the folio lock. A write through a stale TLB entry came
+		 * from a clean PTE and must fault because the PTE has been
+		 * cleared; the fault path has to take the folio lock before
+		 * installing a writable mapping. Buffered write paths also
+		 * have to take the folio lock before modifying file contents
+		 * without a mapping, typically via write_begin_get_folio().
+		 */
+		if (!is_shmem && folio_test_dirty(folio)) {
+			result = SCAN_PAGE_DIRTY_OR_WRITEBACK;
+			xas_unlock_irq(&xas);
+			folio_putback_lru(folio);
+			goto out_unlock;
+		}
+
 		/*
 		 * Accumulate the folios that are being collapsed.
 		 */
@@ -2094,21 +2483,6 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 		goto xa_unlocked;
 	}
 
-	if (!is_shmem) {
-		filemap_nr_thps_inc(mapping);
-		/*
-		 * Paired with the fence in do_dentry_open() -> get_write_access()
-		 * to ensure i_writecount is up to date and the update to nr_thps
-		 * is visible. Ensures the page cache will be truncated if the
-		 * file is opened writable.
-		 */
-		smp_mb();
-		if (inode_is_open_for_write(mapping->host)) {
-			result = SCAN_FAIL;
-			filemap_nr_thps_dec(mapping);
-		}
-	}
-
 xa_locked:
 	xas_unlock_irq(&xas);
 xa_unlocked:
@@ -2286,19 +2660,6 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
 		folio_putback_lru(folio);
 		folio_put(folio);
 	}
-	/*
-	 * Undo the updates of filemap_nr_thps_inc for non-SHMEM
-	 * file only. This undo is not needed unless failure is
-	 * due to SCAN_COPY_MC.
-	 */
-	if (!is_shmem && result == SCAN_COPY_MC) {
-		filemap_nr_thps_dec(mapping);
-		/*
-		 * Paired with the fence in do_dentry_open() -> get_write_access()
-		 * to ensure the update to nr_thps is visible.
-		 */
-		smp_mb();
-	}
 
 	new_folio->mapping = NULL;
 
@@ -2314,6 +2675,8 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
 		unsigned long addr, struct file *file, pgoff_t start,
 		struct collapse_control *cc)
 {
+	const int max_ptes_none = collapse_max_ptes_none(cc, NULL, HPAGE_PMD_ORDER);
+	const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER);
 	struct folio *folio = NULL;
 	struct address_space *mapping = file->f_mapping;
 	XA_STATE(xas, &mapping->i_pages, start);
@@ -2332,8 +2695,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
 
 		if (xa_is_value(folio)) {
 			swap += 1 << xas_get_order(&xas);
-			if (cc->is_khugepaged &&
-			    swap > khugepaged_max_ptes_swap) {
+			if (swap > max_ptes_swap) {
 				result = SCAN_EXCEED_SWAP_PTE;
 				count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);
 				break;
@@ -2404,8 +2766,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
 		cc->progress += HPAGE_PMD_NR;
 
 	if (result == SCAN_SUCCEED) {
-		if (cc->is_khugepaged &&
-		    present < HPAGE_PMD_NR - khugepaged_max_ptes_none) {
+		if (present < HPAGE_PMD_NR - max_ptes_none) {
 			result = SCAN_EXCEED_NONE_PTE;
 			count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
 		} else {
@@ -2524,12 +2885,12 @@ static void collapse_scan_mm_slot(unsigned int progress_max,
 			cc->progress++;
 			break;
 		}
-		if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_KHUGEPAGED, PMD_ORDER)) {
+		if (!collapse_allowable_orders(vma, vma->vm_flags, TVA_KHUGEPAGED)) {
 			cc->progress++;
 			continue;
 		}
-		hstart = round_up(vma->vm_start, HPAGE_PMD_SIZE);
-		hend = round_down(vma->vm_end, HPAGE_PMD_SIZE);
+		hstart = ALIGN(vma->vm_start, HPAGE_PMD_SIZE);
+		hend = ALIGN_DOWN(vma->vm_end, HPAGE_PMD_SIZE);
 		if (khugepaged_scan.address > hend) {
 			cc->progress++;
 			continue;
@@ -2599,7 +2960,7 @@ static void collapse_scan_mm_slot(unsigned int progress_max,
 
 static int khugepaged_has_work(void)
 {
-	return !list_empty(&khugepaged_scan.mm_head) && hugepage_pmd_enabled();
+	return !list_empty(&khugepaged_scan.mm_head) && hugepage_enabled();
 }
 
 static int khugepaged_wait_event(void)
@@ -2672,7 +3033,7 @@ static void khugepaged_wait_work(void)
 		return;
 	}
 
-	if (hugepage_pmd_enabled())
+	if (hugepage_enabled())
 		wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
 }
 
@@ -2703,7 +3064,7 @@ void set_recommended_min_free_kbytes(void)
 	int nr_zones = 0;
 	unsigned long recommended_min;
 
-	if (!hugepage_pmd_enabled()) {
+	if (!hugepage_enabled()) {
 		calculate_min_free_kbytes();
 		goto update_wmarks;
 	}
@@ -2753,7 +3114,7 @@ int start_stop_khugepaged(void)
 	int err = 0;
 
 	mutex_lock(&khugepaged_mutex);
-	if (hugepage_pmd_enabled()) {
+	if (hugepage_enabled()) {
 		if (!khugepaged_thread)
 			khugepaged_thread = kthread_run(khugepaged, NULL,
 							"khugepaged");
@@ -2779,7 +3140,7 @@ int start_stop_khugepaged(void)
 void khugepaged_min_free_kbytes_update(void)
 {
 	mutex_lock(&khugepaged_mutex);
-	if (hugepage_pmd_enabled() && khugepaged_thread)
+	if (hugepage_enabled() && khugepaged_thread)
 		set_recommended_min_free_kbytes();
 	mutex_unlock(&khugepaged_mutex);
 }
@@ -2808,6 +3169,7 @@ static int madvise_collapse_errno(enum scan_result r)
 	case SCAN_PAGE_LRU:
 	case SCAN_DEL_PAGE_LRU:
 	case SCAN_PAGE_FILLED:
+	case SCAN_PAGE_HAS_PRIVATE:
 	case SCAN_PAGE_DIRTY_OR_WRITEBACK:
 		return -EAGAIN;
 	/*
@@ -2833,9 +3195,15 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
 	BUG_ON(vma->vm_start > start);
 	BUG_ON(vma->vm_end < end);
 
-	if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_FORCED_COLLAPSE, PMD_ORDER))
+	if (!collapse_allowable_orders(vma, vma->vm_flags, TVA_FORCED_COLLAPSE))
 		return -EINVAL;
 
+	hstart = ALIGN(start, HPAGE_PMD_SIZE);
+	hend = ALIGN_DOWN(end, HPAGE_PMD_SIZE);
+
+	if (hstart >= hend)
+		return 0;
+
 	cc = kmalloc_obj(*cc);
 	if (!cc)
 		return -ENOMEM;
@@ -2845,9 +3213,6 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
 	mmgrab(mm);
 	lru_add_drain_all();
 
-	hstart = (start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
-	hend = end & HPAGE_PMD_MASK;
-
 	for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) {
 		enum scan_result result = SCAN_FAIL;
 
@@ -2857,8 +3222,8 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
 			mmap_unlocked = false;
 			*lock_dropped = true;
 			result = hugepage_vma_revalidate(mm, addr, false, &vma,
-							 cc);
-			if (result  != SCAN_SUCCEED) {
+							 cc, HPAGE_PMD_ORDER);
+			if (result != SCAN_SUCCEED) {
 				last_fail = result;
 				goto out_nolock;
 			}
diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index 2eff0d6b622b6..7c7ba17ce7af0 100644
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -92,6 +92,7 @@
 #include <linux/nodemask.h>
 #include <linux/mm.h>
 #include <linux/workqueue.h>
+#include <linux/xarray.h>
 #include <linux/crc32.h>
 
 #include <asm/sections.h>
@@ -157,6 +158,8 @@ struct kmemleak_object {
 	struct hlist_head area_list;
 	unsigned long jiffies;		/* creation timestamp */
 	pid_t pid;			/* pid of the current task */
+	/* per-scan dedup count, valid only while in scan-local dedup xarray */
+	unsigned int dup_count;
 	char comm[TASK_COMM_LEN];	/* executable name */
 };
 
@@ -360,8 +363,9 @@ static const char *__object_type_str(struct kmemleak_object *object)
  * Printing of the unreferenced objects information to the seq file. The
  * print_unreferenced function must be called with the object->lock held.
  */
-static void print_unreferenced(struct seq_file *seq,
-			       struct kmemleak_object *object)
+static void __print_unreferenced(struct seq_file *seq,
+				 struct kmemleak_object *object,
+				 bool hex_dump)
 {
 	int i;
 	unsigned long *entries;
@@ -373,7 +377,8 @@ static void print_unreferenced(struct seq_file *seq,
 			   object->pointer, object->size);
 	warn_or_seq_printf(seq, "  comm \"%s\", pid %d, jiffies %lu\n",
 			   object->comm, object->pid, object->jiffies);
-	hex_dump_object(seq, object);
+	if (hex_dump)
+		hex_dump_object(seq, object);
 	warn_or_seq_printf(seq, "  backtrace (crc %x):\n", object->checksum);
 
 	for (i = 0; i < nr_entries; i++) {
@@ -382,6 +387,12 @@ static void print_unreferenced(struct seq_file *seq,
 	}
 }
 
+static void print_unreferenced(struct seq_file *seq,
+			       struct kmemleak_object *object)
+{
+	__print_unreferenced(seq, object, true);
+}
+
 /*
  * Print the kmemleak_object information. This function is used mainly for
  * debugging special cases when kmemleak operations. It must be called with
@@ -1684,6 +1695,103 @@ static void kmemleak_cond_resched(struct kmemleak_object *object)
 	put_object(object);
 }
 
+/*
+ * Print one leak inline. The hex dump is gated on OBJECT_ALLOCATED so it
+ * does not touch user memory that was freed concurrently; the rest of the
+ * report (backtrace, comm, pid) is always emitted since the kmemleak_object
+ * metadata is pinned by the caller.
+ */
+static void print_leak_locked(struct kmemleak_object *object, bool hex_dump)
+{
+	raw_spin_lock_irq(&object->lock);
+	__print_unreferenced(NULL, object,
+			     hex_dump && (object->flags & OBJECT_ALLOCATED));
+	raw_spin_unlock_irq(&object->lock);
+}
+
+/*
+ * Per-scan dedup table for verbose leak printing. The xarray is keyed by
+ * stackdepot trace_handle and stores a pointer to the representative
+ * kmemleak_object. The per-scan repeat count lives in object->dup_count.
+ *
+ * dedup_record() must run outside object->lock: xa_store() may take
+ * mutexes (xa_node slab allocation) which lockdep would flag against the
+ * raw spinlock object->lock.
+ */
+static void dedup_record(struct xarray *dedup, struct kmemleak_object *object,
+			 depot_stack_handle_t trace_handle)
+{
+	struct kmemleak_object *rep;
+	void *old;
+
+	/*
+	 * No stack trace to dedup against: early-boot allocation tracked
+	 * before kmemleak_init() set up object_cache, or stack_depot_save()
+	 * failure under memory pressure.
+	 */
+	if (!trace_handle) {
+		print_leak_locked(object, true);
+		return;
+	}
+
+	/* stack is available, now we can de-dup */
+	rep = xa_load(dedup, trace_handle);
+	if (rep) {
+		rep->dup_count++;
+		return;
+	}
+
+	/*
+	 * Object is being torn down (use_count already hit zero); the
+	 * tracked memory at object->pointer is unsafe to read, so skip.
+	 */
+	if (!get_object(object))
+		return;
+
+	object->dup_count = 1;
+	old = xa_store(dedup, trace_handle, object, GFP_ATOMIC);
+	if (xa_is_err(old)) {
+		/* xa_node allocation failed; fall back to inline print. */
+		print_leak_locked(object, true);
+		put_object(object);
+		return;
+	}
+	/*
+	 * scan_mutex serialises all writers to the dedup xarray, so xa_store()
+	 * after a NULL xa_load() must always overwrite an empty slot.
+	 */
+	WARN_ON_ONCE(old);
+}
+
+/*
+ * Drain the dedup table. Re-acquires object->lock and re-checks
+ * OBJECT_ALLOCATED before printing: while get_object() pins the
+ * kmemleak_object metadata, the underlying tracked allocation may have
+ * been freed since the scan walked it (kmemleak_free clears
+ * OBJECT_ALLOCATED under object->lock before the user memory goes away).
+ * The hex dump is skipped for coalesced entries since the bytes would
+ * differ across objects anyway.
+ */
+static void dedup_flush(struct xarray *dedup)
+{
+	struct kmemleak_object *object;
+	unsigned long idx;
+	unsigned int dup;
+	bool coalesced;
+
+	xa_for_each(dedup, idx, object) {
+		dup = object->dup_count;
+		coalesced = dup > 1;
+
+		print_leak_locked(object, !coalesced);
+		if (coalesced)
+			pr_warn("  ... and %u more object(s) with the same backtrace\n",
+				dup - 1);
+		put_object(object);
+		xa_erase(dedup, idx);
+	}
+}
+
 /*
  * Scan data sections and all the referenced memory blocks allocated via the
  * kernel's standard allocators. This function must be called with the
@@ -1694,6 +1802,7 @@ static void kmemleak_scan(void)
 	struct kmemleak_object *object;
 	struct zone *zone;
 	int __maybe_unused i;
+	struct xarray dedup;
 	int new_leaks = 0;
 
 	jiffies_last_scan = jiffies;
@@ -1834,10 +1943,18 @@ static void kmemleak_scan(void)
 		return;
 
 	/*
-	 * Scanning result reporting.
+	 * Scanning result reporting. When verbose printing is enabled, dedupe
+	 * by stackdepot trace_handle so each unique backtrace is logged once
+	 * per scan, annotated with the number of objects that share it. The
+	 * per-leak count below still reflects every object, and
+	 * /sys/kernel/debug/kmemleak still lists them individually.
 	 */
+	xa_init(&dedup);
 	rcu_read_lock();
 	list_for_each_entry_rcu(object, &object_list, object_list) {
+		depot_stack_handle_t trace_handle;
+		bool dedup_print;
+
 		if (need_resched())
 			kmemleak_cond_resched(object);
 
@@ -1849,18 +1966,33 @@ static void kmemleak_scan(void)
 		if (!color_white(object))
 			continue;
 		raw_spin_lock_irq(&object->lock);
+		trace_handle = 0;
+		dedup_print = false;
 		if (unreferenced_object(object) &&
 		    !(object->flags & OBJECT_REPORTED)) {
 			object->flags |= OBJECT_REPORTED;
-
-			if (kmemleak_verbose)
-				print_unreferenced(NULL, object);
-
+			if (kmemleak_verbose) {
+				trace_handle = object->trace_handle;
+				dedup_print = true;
+			}
 			new_leaks++;
 		}
 		raw_spin_unlock_irq(&object->lock);
+
+		/*
+		 * Defer the verbose print outside object->lock: xa_store()
+		 * may take xa_node slab locks at a higher wait-context level
+		 * which lockdep would flag against the raw_spinlock_t
+		 * object->lock. rcu_read_lock() keeps the kmemleak_object
+		 * alive across the call.
+		 */
+		if (dedup_print)
+			dedup_record(&dedup, object, trace_handle);
 	}
 	rcu_read_unlock();
+	/* Flush'em all */
+	dedup_flush(&dedup);
+	xa_destroy(&dedup);
 
 	if (new_leaks) {
 		kmemleak_found_leaks = true;
diff --git a/mm/madvise.c b/mm/madvise.c
index 69708e953cf56..cd9bb077072cc 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -1834,50 +1834,29 @@ static void madvise_finish_tlb(struct madvise_behavior *madv_behavior)
 		tlb_finish_mmu(madv_behavior->tlb);
 }
 
-static bool is_valid_madvise(unsigned long start, size_t len_in, int behavior)
+/**
+ * check_input_range() - Check if the requested range is valid.
+ * @start:	Start address of madvise-requested address range.
+ * @len_in:	Length of madvise-requested address range.
+ *
+ * Returns: 0 if the input range is valid, otherwise an error code.
+ */
+static int check_input_range(unsigned long start, size_t len_in)
 {
 	size_t len;
 
-	if (!madvise_behavior_valid(behavior))
-		return false;
-
 	if (!PAGE_ALIGNED(start))
-		return false;
+		return -EINVAL;
 	len = PAGE_ALIGN(len_in);
 
 	/* Check to see whether len was rounded up from small -ve to zero */
 	if (len_in && !len)
-		return false;
+		return -EINVAL;
 
 	if (start + len < start)
-		return false;
-
-	return true;
-}
+		return -EINVAL;
 
-/*
- * madvise_should_skip() - Return if the request is invalid or nothing.
- * @start:	Start address of madvise-requested address range.
- * @len_in:	Length of madvise-requested address range.
- * @behavior:	Requested madvise behavior.
- * @err:	Pointer to store an error code from the check.
- *
- * If the specified behaviour is invalid or nothing would occur, we skip the
- * operation.  This function returns true in the cases, otherwise false.  In
- * the former case we store an error on @err.
- */
-static bool madvise_should_skip(unsigned long start, size_t len_in,
-		int behavior, int *err)
-{
-	if (!is_valid_madvise(start, len_in, behavior)) {
-		*err = -EINVAL;
-		return true;
-	}
-	if (start + PAGE_ALIGN(len_in) == start) {
-		*err = 0;
-		return true;
-	}
-	return false;
+	return 0;
 }
 
 static bool is_madvise_populate(struct madvise_behavior *madv_behavior)
@@ -2013,8 +1992,13 @@ int do_madvise(struct mm_struct *mm, unsigned long start, size_t len_in, int beh
 		.tlb = &tlb,
 	};
 
-	if (madvise_should_skip(start, len_in, behavior, &error))
+	if (!madvise_behavior_valid(behavior))
+		return -EINVAL;
+
+	error = check_input_range(start, len_in);
+	if (error || !len_in)
 		return error;
+
 	error = madvise_lock(&madv_behavior);
 	if (error)
 		return error;
@@ -2056,7 +2040,8 @@ static ssize_t vector_madvise(struct mm_struct *mm, struct iov_iter *iter,
 		size_t len_in = iter_iov_len(iter);
 		int error;
 
-		if (madvise_should_skip(start, len_in, behavior, &error))
+		error = check_input_range(start, len_in);
+		if (error || !len_in)
 			ret = error;
 		else
 			ret = madvise_do_behavior(start, len_in, &madv_behavior);
@@ -2131,6 +2116,11 @@ SYSCALL_DEFINE5(process_madvise, int, pidfd, const struct iovec __user *, vec,
 		goto release_task;
 	}
 
+	if (!madvise_behavior_valid(behavior)) {
+		ret = -EINVAL;
+		goto release_mm;
+	}
+
 	/*
 	 * We need only perform this check if we are attempting to manipulate a
 	 * remote process's address space.
diff --git a/mm/memblock.c b/mm/memblock.c
index a6a1c91e276d3..01a9626817263 100644
--- a/mm/memblock.c
+++ b/mm/memblock.c
@@ -1026,40 +1026,6 @@ int __init_memblock memblock_physmem_add(phys_addr_t base, phys_addr_t size)
 }
 #endif
 
-#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH
-__init void memblock_set_kho_scratch_only(void)
-{
-	kho_scratch_only = true;
-}
-
-__init void memblock_clear_kho_scratch_only(void)
-{
-	kho_scratch_only = false;
-}
-
-__init void memmap_init_kho_scratch_pages(void)
-{
-	phys_addr_t start, end;
-	unsigned long pfn;
-	int nid;
-	u64 i;
-
-	if (!IS_ENABLED(CONFIG_DEFERRED_STRUCT_PAGE_INIT))
-		return;
-
-	/*
-	 * Initialize struct pages for free scratch memory.
-	 * The struct pages for reserved scratch memory will be set up in
-	 * memmap_init_reserved_pages()
-	 */
-	__for_each_mem_range(i, &memblock.memory, NULL, NUMA_NO_NODE,
-			     MEMBLOCK_KHO_SCRATCH, &start, &end, &nid) {
-		for (pfn = PFN_UP(start); pfn < PFN_DOWN(end); pfn++)
-			init_deferred_page(pfn, nid);
-	}
-}
-#endif
-
 /**
  * memblock_setclr_flag - set or clear flag for a memory region
  * @type: memblock type to set/clear flag for
@@ -2533,6 +2499,28 @@ int reserve_mem_release_by_name(const char *name)
 	return 1;
 }
 
+#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH
+__init void memblock_set_kho_scratch_only(void)
+{
+	kho_scratch_only = true;
+}
+
+__init void memblock_clear_kho_scratch_only(void)
+{
+	kho_scratch_only = false;
+}
+
+bool __init_memblock memblock_is_kho_scratch_memory(phys_addr_t addr)
+{
+	int i = memblock_search(&memblock.memory, addr);
+
+	if (i == -1)
+		return false;
+
+	return memblock_is_kho_scratch(&memblock.memory.regions[i]);
+}
+#endif
+
 #ifdef CONFIG_KEXEC_HANDOVER
 
 static int __init reserved_mem_preserve(void)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index c03d4787d4668..0eb50e639f0af 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -142,14 +142,24 @@ static void obj_cgroup_release(struct percpu_ref *ref)
 	struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
 	unsigned int nr_bytes;
 	unsigned int nr_pages;
+	unsigned int sub_bytes;
 	unsigned long flags;
 
 	/*
-	 * At this point all allocated objects are freed, and
-	 * objcg->nr_charged_bytes can't have an arbitrary byte value.
-	 * However, it can be PAGE_SIZE or (x * PAGE_SIZE).
+	 * At this point all allocated objects are freed, but
+	 * objcg->nr_charged_bytes can still hold either
+	 *   - (x * PAGE_SIZE)  if a small-alloc/drain race left whole pages
+	 *     stranded (see the historical sequence below), or
+	 *   - any sub-page residue, now that the stock is keyed by memcg and
+	 *     sibling per-node objcgs share its reserve: bytes consumed by
+	 *     one sibling can spill into another sibling's nr_charged_bytes
+	 *     when the stock is drained.
 	 *
-	 * The following sequence can lead to it:
+	 * Uncharge the page-aligned portion from this objcg's (post-reparent)
+	 * memcg, and forward any sub-page residue into a per-node objcg of
+	 * the same memcg so it can be reconciled later instead of being lost.
+	 *
+	 * Historical race producing the (x * PAGE_SIZE) case:
 	 * 1) CPU0: objcg == stock->cached_objcg
 	 * 2) CPU1: we do a small allocation (e.g. 92 bytes),
 	 *          PAGE_SIZE bytes are charged
@@ -160,23 +170,33 @@ static void obj_cgroup_release(struct percpu_ref *ref)
 	 *          92 bytes are added to stock->nr_bytes
 	 * 6) CPU0: stock is flushed,
 	 *          92 bytes are added to objcg->nr_charged_bytes
-	 *
-	 * In the result, nr_charged_bytes == PAGE_SIZE.
-	 * This page will be uncharged in obj_cgroup_release().
 	 */
 	nr_bytes = atomic_read(&objcg->nr_charged_bytes);
-	WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
 	nr_pages = nr_bytes >> PAGE_SHIFT;
+	sub_bytes = nr_bytes & (PAGE_SIZE - 1);
 
-	if (nr_pages) {
+	if (nr_pages || sub_bytes) {
 		struct mem_cgroup *memcg;
 
-		memcg = get_mem_cgroup_from_objcg(objcg);
-		mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages);
-		memcg1_account_kmem(memcg, -nr_pages);
-		if (!mem_cgroup_is_root(memcg))
-			memcg_uncharge(memcg, nr_pages);
-		mem_cgroup_put(memcg);
+		rcu_read_lock();
+		memcg = obj_cgroup_memcg(objcg);
+
+		if (nr_pages) {
+			mod_memcg_state(memcg, MEMCG_KMEM, -nr_pages);
+			memcg1_account_kmem(memcg, -nr_pages);
+			if (!mem_cgroup_is_root(memcg))
+				memcg_uncharge(memcg, nr_pages);
+		}
+
+		if (sub_bytes && !mem_cgroup_is_root(memcg)) {
+			struct obj_cgroup *fwd;
+
+			fwd = rcu_dereference(
+				memcg->nodeinfo[numa_node_id()]->objcg);
+			if (fwd)
+				atomic_add(sub_bytes, &fwd->nr_charged_bytes);
+		}
+		rcu_read_unlock();
 	}
 
 	spin_lock_irqsave(&objcg_lock, flags);
@@ -679,7 +699,7 @@ static inline void memcg_rstat_updated(struct mem_cgroup *memcg, long val,
 	if (!val)
 		return;
 
-	css_rstat_updated(&memcg->css, cpu);
+	__css_rstat_updated(&memcg->css, cpu);
 	statc_pcpu = memcg->vmstats_percpu;
 	for (; statc_pcpu; statc_pcpu = statc->parent_pcpu) {
 		statc = this_cpu_ptr(statc_pcpu);
@@ -2796,7 +2816,7 @@ static inline void account_slab_nmi_safe(struct mem_cgroup *memcg,
 		struct mem_cgroup_per_node *pn = memcg->nodeinfo[pgdat->node_id];
 
 		/* preemption is disabled in_nmi(). */
-		css_rstat_updated(&memcg->css, smp_processor_id());
+		__css_rstat_updated(&memcg->css, smp_processor_id());
 		if (idx == NR_SLAB_RECLAIMABLE_B)
 			atomic_add(nr, &pn->slab_reclaimable);
 		else
@@ -3019,7 +3039,7 @@ static inline void account_kmem_nmi_safe(struct mem_cgroup *memcg, int val)
 		mod_memcg_state(memcg, MEMCG_KMEM, val);
 	} else {
 		/* preemption is disabled in_nmi(). */
-		css_rstat_updated(&memcg->css, smp_processor_id());
+		__css_rstat_updated(&memcg->css, smp_processor_id());
 		atomic_add(val, &memcg->kmem_stat);
 	}
 }
@@ -3152,7 +3172,12 @@ static void unlock_stock(struct obj_stock_pcp *stock)
 		local_unlock(&obj_stock.lock);
 }
 
-/* Call after __refill_obj_stock() to ensure stock->cached_objg == objcg */
+/*
+ * Call after __consume_obj_stock() / __refill_obj_stock(). The stock may be
+ * cached for a sibling per-node objcg of the same memcg; in that case the
+ * vmstat batching slot does not match objcg and we fallthrough to the
+ * direct path.
+ */
 static void __account_obj_stock(struct obj_cgroup *objcg,
 				struct obj_stock_pcp *stock, int nr,
 				struct pglist_data *pgdat, enum node_stat_item idx)
@@ -3210,7 +3235,11 @@ static bool __consume_obj_stock(struct obj_cgroup *objcg,
 				struct obj_stock_pcp *stock,
 				unsigned int nr_bytes)
 {
-	if (objcg == READ_ONCE(stock->cached_objcg) &&
+	struct obj_cgroup *cached = READ_ONCE(stock->cached_objcg);
+
+	/* Sibling per-node objcgs share the reserve. */
+	if ((cached == objcg ||
+	     (cached && READ_ONCE(cached->memcg) == READ_ONCE(objcg->memcg))) &&
 	    stock->nr_bytes >= nr_bytes) {
 		stock->nr_bytes -= nr_bytes;
 		return true;
@@ -3318,6 +3347,7 @@ static void __refill_obj_stock(struct obj_cgroup *objcg,
 			       unsigned int nr_bytes,
 			       bool allow_uncharge)
 {
+	struct obj_cgroup *cached;
 	unsigned int nr_pages = 0;
 
 	if (!stock) {
@@ -3327,7 +3357,10 @@ static void __refill_obj_stock(struct obj_cgroup *objcg,
 		goto out;
 	}
 
-	if (READ_ONCE(stock->cached_objcg) != objcg) { /* reset if necessary */
+	cached = READ_ONCE(stock->cached_objcg);
+	/* Direct READ_ONCE due to just pointer comparison. */
+	if (cached != objcg &&
+	    (!cached || READ_ONCE(cached->memcg) != READ_ONCE(objcg->memcg))) {
 		drain_obj_stock(stock);
 		obj_cgroup_get(objcg);
 		stock->nr_bytes = atomic_read(&objcg->nr_charged_bytes)
@@ -4002,11 +4035,10 @@ static struct mem_cgroup *mem_cgroup_alloc(struct mem_cgroup *parent)
 	if (!memcg1_alloc_events(memcg))
 		goto fail;
 
+	pstatc_pcpu = parent ? parent->vmstats_percpu : NULL;
 	for_each_possible_cpu(cpu) {
-		if (parent)
-			pstatc_pcpu = parent->vmstats_percpu;
 		statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
-		statc->parent_pcpu = parent ? pstatc_pcpu : NULL;
+		statc->parent_pcpu = pstatc_pcpu;
 		statc->vmstats = memcg->vmstats;
 	}
 
@@ -4352,6 +4384,9 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
 			lstats->state[index] += slab;
 			if (plstats)
 				plstats->state_pending[index] += slab;
+			memcg->vmstats->state[index] += slab;
+			if (parent)
+				parent->vmstats->state_pending[index] += slab;
 		}
 		if (atomic_read(&pn->slab_unreclaimable)) {
 			int slab = atomic_xchg(&pn->slab_unreclaimable, 0);
@@ -4360,6 +4395,9 @@ static void flush_nmi_stats(struct mem_cgroup *memcg, struct mem_cgroup *parent,
 			lstats->state[index] += slab;
 			if (plstats)
 				plstats->state_pending[index] += slab;
+			memcg->vmstats->state[index] += slab;
+			if (parent)
+				parent->vmstats->state_pending[index] += slab;
 		}
 	}
 }
diff --git a/mm/memfd.c b/mm/memfd.c
index fb425f4e315f4..abe13b291ddc8 100644
--- a/mm/memfd.c
+++ b/mm/memfd.c
@@ -283,6 +283,12 @@ int memfd_add_seals(struct file *file, unsigned int seals)
 		goto unlock;
 	}
 
+	/*
+	 * SEAL_EXEC implies SEAL_WRITE, making W^X from the start.
+	 */
+	if (seals & F_SEAL_EXEC && inode->i_mode & 0111)
+		seals |= F_SEAL_SHRINK|F_SEAL_GROW|F_SEAL_WRITE|F_SEAL_FUTURE_WRITE;
+
 	if ((seals & F_SEAL_WRITE) && !(*file_seals & F_SEAL_WRITE)) {
 		error = mapping_deny_writable(file->f_mapping);
 		if (error)
@@ -295,12 +301,6 @@ int memfd_add_seals(struct file *file, unsigned int seals)
 		}
 	}
 
-	/*
-	 * SEAL_EXEC implies SEAL_WRITE, making W^X from the start.
-	 */
-	if (seals & F_SEAL_EXEC && inode->i_mode & 0111)
-		seals |= F_SEAL_SHRINK|F_SEAL_GROW|F_SEAL_WRITE|F_SEAL_FUTURE_WRITE;
-
 	*file_seals |= seals;
 	error = 0;
 
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index ee42d43613097..866c4428ac7ef 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -459,7 +459,7 @@ void add_to_kill_ksm(struct task_struct *tsk, const struct page *p,
  * Only do anything when FORCEKILL is set, otherwise just free the
  * list (this is used for clean pages which do not need killing)
  */
-static void kill_procs(struct list_head *to_kill, int forcekill,
+static void kill_procs(struct list_head *to_kill, bool forcekill,
 		unsigned long pfn, int flags)
 {
 	struct to_kill *tk, *next;
@@ -1418,7 +1418,7 @@ static int get_any_page(struct page *p, unsigned long flags)
 			 * We raced with (possibly temporary) unhandlable
 			 * page, retry.
 			 */
-			if (pass++ < 3) {
+			if (pass++ < GET_PAGE_MAX_RETRY_NUM) {
 				shake_page(p);
 				goto try_again;
 			}
@@ -1582,7 +1582,7 @@ static bool hwpoison_user_mappings(struct folio *folio, struct page *p,
 {
 	LIST_HEAD(tokill);
 	bool unmap_success;
-	int forcekill;
+	bool forcekill;
 	bool mlocked = folio_test_mlocked(folio);
 
 	/*
@@ -1703,7 +1703,7 @@ static void unmap_and_kill(struct list_head *to_kill, unsigned long pfn,
 		unmap_mapping_range(mapping, start, size, 0);
 	}
 
-	kill_procs(to_kill, flags & MF_MUST_KILL, pfn, flags);
+	kill_procs(to_kill, !!(flags & MF_MUST_KILL), pfn, flags);
 }
 
 /*
diff --git a/mm/memory.c b/mm/memory.c
index ea65685711311..0c9d9c2cbf0e0 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -612,6 +612,21 @@ static void print_bad_page_map(struct vm_area_struct *vma,
 	dump_stack();
 	add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
 }
+
+static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
+{
+	switch (level) {
+	case PGTABLE_LEVEL_PTE:
+		return IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL);
+	case PGTABLE_LEVEL_PMD:
+		return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PMD_PFNMAP);
+	case PGTABLE_LEVEL_PUD:
+		return IS_ENABLED(CONFIG_ARCH_SUPPORTS_PUD_PFNMAP);
+	default:
+		return false;
+	}
+}
+
 #define print_bad_pte(vma, addr, pte, page) \
 	print_bad_page_map(vma, addr, pte_val(pte), page, PGTABLE_LEVEL_PTE)
 
@@ -684,7 +699,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
 		unsigned long addr, unsigned long pfn, bool special,
 		unsigned long long entry, enum pgtable_level level)
 {
-	if (IS_ENABLED(CONFIG_ARCH_HAS_PTE_SPECIAL)) {
+	if (pgtable_level_has_pxx_special(level)) {
 		if (unlikely(special)) {
 #ifdef CONFIG_FIND_NORMAL_PAGE
 			if (vma->vm_ops && vma->vm_ops->find_normal_page)
@@ -699,8 +714,9 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
 			return NULL;
 		}
 		/*
-		 * With CONFIG_ARCH_HAS_PTE_SPECIAL, any special page table
-		 * mappings (incl. shared zero folios) are marked accordingly.
+		 * With working pte_special()/pmd_special()..., any special page
+		 * table mappings (incl. shared zero folios) are marked
+		 * accordingly.
 		 */
 	} else {
 		if (unlikely(vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))) {
@@ -1739,7 +1755,7 @@ static inline int zap_nonpresent_ptes(struct mmu_gather *tlb,
 		 * consider uffd-wp bit when zap. For more information,
 		 * see zap_install_uffd_wp_if_needed().
 		 */
-		WARN_ON_ONCE(!vma_is_anonymous(vma));
+		WARN_ON_ONCE(!folio_test_anon(folio));
 		rss[mm_counter(folio)]--;
 		folio_remove_rmap_pte(folio, page, vma);
 		folio_put(folio);
@@ -3821,8 +3837,8 @@ vm_fault_t __vmf_anon_prepare(struct vm_fault *vmf)
  * Handle the case of a page which we actually need to copy to a new page,
  * either due to COW or unsharing.
  *
- * Called with mmap_lock locked and the old page referenced, but
- * without the ptl held.
+ * Called with either the VMA lock or the mmap_lock held (see FAULT_FLAG_VMA_LOCK)
+ * and the old page referenced, but without the ptl held.
  *
  * High level logic flow:
  *
@@ -4221,9 +4237,9 @@ static bool wp_can_reuse_anon_folio(struct folio *folio,
  * though the page will change only once the write actually happens. This
  * avoids a few races, and potentially makes it more efficient.
  *
- * We enter with non-exclusive mmap_lock (to exclude vma changes,
- * but allow concurrent faults), with pte both mapped and locked.
- * We return with mmap_lock still held, but pte unmapped and unlocked.
+ * We enter with either the VMA lock or the mmap_lock held (see
+ * FAULT_FLAG_VMA_LOCK) and pte both mapped and locked. We return with
+ * the same lock still held, but pte unmapped and unlocked.
  */
 static vm_fault_t do_wp_page(struct vm_fault *vmf)
 	__releases(vmf->ptl)
@@ -4769,12 +4785,12 @@ static void check_swap_exclusive(struct folio *folio, swp_entry_t entry,
 }
 
 /*
- * We enter with non-exclusive mmap_lock (to exclude vma changes,
- * but allow concurrent faults), and pte mapped but not yet locked.
+ * We enter with either the VMA lock or the mmap_lock held (see
+ * FAULT_FLAG_VMA_LOCK), and pte mapped but not yet locked.
  * We return with pte unmapped and unlocked.
  *
- * We return with the mmap_lock locked or unlocked in the same cases
- * as does filemap_fault().
+ * When returning, the lock may have been released in the same cases
+ * as done by filemap_fault().
  */
 vm_fault_t do_swap_page(struct vm_fault *vmf)
 {
@@ -5314,9 +5330,10 @@ static void map_anon_folio_pte_pf(struct folio *folio, pte_t *pte,
 }
 
 /*
- * We enter with non-exclusive mmap_lock (to exclude vma changes,
- * but allow concurrent faults), and pte mapped but not yet locked.
- * We return with mmap_lock still held, but pte unmapped and unlocked.
+ * We enter with either the VMA lock or the mmap_lock held (see
+ * FAULT_FLAG_VMA_LOCK), and pte unmapped and unlocked.
+ * We return with the lock still held, but pte unmapped and unlocked.
+ * If VM_FAULT_RETRY is returned, the lock may have been released.
  */
 static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
 {
@@ -5424,9 +5441,10 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
 }
 
 /*
- * The mmap_lock must have been held on entry, and may have been
- * released depending on flags and vma->vm_ops->fault() return value.
- * See filemap_fault() and __lock_page_retry().
+ * Either the VMA lock or the mmap_lock must have been held on entry
+ * (see FAULT_FLAG_VMA_LOCK) and may have been released depending on
+ * flags and vma->vm_ops->fault() return value.
+ * See filemap_fault() and __folio_lock_or_retry().
  */
 static vm_fault_t __do_fault(struct vm_fault *vmf)
 {
@@ -5464,7 +5482,7 @@ static vm_fault_t __do_fault(struct vm_fault *vmf)
 	if (unlikely(PageHWPoison(vmf->page))) {
 		vm_fault_t poisonret = VM_FAULT_HWPOISON;
 		if (ret & VM_FAULT_LOCKED) {
-			if (page_mapped(vmf->page))
+			if (folio_mapped(folio))
 				unmap_mapping_folio(folio);
 			/* Retry if a clean folio was removed from the cache. */
 			if (mapping_evict_folio(folio->mapping, folio))
@@ -5987,11 +6005,11 @@ static vm_fault_t do_shared_fault(struct vm_fault *vmf)
 }
 
 /*
- * We enter with non-exclusive mmap_lock (to exclude vma changes,
- * but allow concurrent faults).
- * The mmap_lock may have been released depending on flags and our
+ * We enter with either the VMA lock or the mmap_lock held (see
+ * FAULT_FLAG_VMA_LOCK).
+ * The lock may have been released depending on flags and our
  * return value.  See filemap_fault() and __folio_lock_or_retry().
- * If mmap_lock is released, vma may become invalid (for example
+ * If the lock is released, vma may become invalid (for example
  * by other thread calling munmap()).
  */
 static vm_fault_t do_fault(struct vm_fault *vmf)
@@ -6358,10 +6376,11 @@ static void fix_spurious_fault(struct vm_fault *vmf,
  * with external mmu caches can use to update those (ie the Sparc or
  * PowerPC hashed page tables that act as extended TLBs).
  *
- * We enter with non-exclusive mmap_lock (to exclude vma changes, but allow
- * concurrent faults).
+ * On entry, we hold either the VMA lock or the mmap_lock
+ * (see FAULT_FLAG_VMA_LOCK).
  *
- * The mmap_lock may have been released depending on flags and our return value.
+ * The mmap_lock or VMA lock may have been released depending on flags
+ * and our return value.
  * See filemap_fault() and __folio_lock_or_retry().
  */
 static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
@@ -6442,8 +6461,8 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
 
 /*
  * On entry, we hold either the VMA lock or the mmap_lock
- * (FAULT_FLAG_VMA_LOCK tells you which).  If VM_FAULT_RETRY is set in
- * the result, the mmap_lock is not held on exit.  See filemap_fault()
+ * (see FAULT_FLAG_VMA_LOCK).  If VM_FAULT_RETRY is set in
+ * the result, the lock is not held on exit.  See filemap_fault()
  * and __folio_lock_or_retry().
  */
 static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
@@ -6675,9 +6694,9 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
 
 /*
  * By the time we get here, we already hold either the VMA lock or the
- * mmap_lock (FAULT_FLAG_VMA_LOCK tells you which).
+ * mmap_lock (see FAULT_FLAG_VMA_LOCK).
  *
- * The mmap_lock may have been released depending on flags and our
+ * The lock may have been released depending on flags and our
  * return value.  See filemap_fault() and __folio_lock_or_retry().
  */
 vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 2a943ec57c85b..7ac19fab22632 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -576,6 +576,7 @@ void remove_pfn_range_from_zone(struct zone *zone,
  * @pfn: starting pageframe (must be aligned to start of a section)
  * @nr_pages: number of pages to remove (must be multiple of section size)
  * @altmap: alternative device page map or %NULL if default memmap is used
+ * @pgmap: device page map or %NULL if not ZONE_DEVICE
  *
  * Generic helper function to remove section mappings and sysfs entries
  * for the section of the memory we are removing. Caller needs to make
@@ -583,7 +584,7 @@ void remove_pfn_range_from_zone(struct zone *zone,
  * calling offline_pages().
  */
 void __remove_pages(unsigned long pfn, unsigned long nr_pages,
-		    struct vmem_altmap *altmap)
+		    struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
 {
 	const unsigned long end_pfn = pfn + nr_pages;
 	unsigned long cur_nr_pages;
@@ -598,7 +599,7 @@ void __remove_pages(unsigned long pfn, unsigned long nr_pages,
 		/* Select all remaining pages up to the next section boundary */
 		cur_nr_pages = min(end_pfn - pfn,
 				   SECTION_ALIGN_UP(pfn + 1) - pfn);
-		sparse_remove_section(pfn, cur_nr_pages, altmap);
+		sparse_remove_section(pfn, cur_nr_pages, altmap, pgmap);
 	}
 }
 
@@ -1402,6 +1403,12 @@ bool mhp_supports_memmap_on_memory(void)
 }
 EXPORT_SYMBOL_GPL(mhp_supports_memmap_on_memory);
 
+static void altmap_free(struct vmem_altmap *altmap)
+{
+	WARN_ONCE(altmap->alloc, "Altmap not fully unmapped");
+	kfree(altmap);
+}
+
 static void remove_memory_blocks_and_altmaps(u64 start, u64 size)
 {
 	unsigned long memblock_size = memory_block_size_bytes();
@@ -1416,20 +1423,17 @@ static void remove_memory_blocks_and_altmaps(u64 start, u64 size)
 		struct vmem_altmap *altmap = NULL;
 		struct memory_block *mem;
 
-		mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(cur_start)));
+		mem = memory_block_get(phys_to_block_id(cur_start));
 		if (WARN_ON_ONCE(!mem))
 			continue;
 
 		altmap = mem->altmap;
 		mem->altmap = NULL;
+		memory_block_put(mem);
 
 		remove_memory_block_devices(cur_start, memblock_size);
-
-		arch_remove_memory(cur_start, memblock_size, altmap);
-
-		/* Verify that all vmemmap pages have actually been freed. */
-		WARN(altmap->alloc, "Altmap not fully unmapped");
-		kfree(altmap);
+		arch_remove_memory(cur_start, memblock_size, altmap, NULL);
+		altmap_free(altmap);
 	}
 }
 
@@ -1460,7 +1464,7 @@ static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group,
 		/* call arch's memory hotadd */
 		ret = arch_add_memory(nid, cur_start, memblock_size, &params);
 		if (ret < 0) {
-			kfree(params.altmap);
+			altmap_free(params.altmap);
 			goto out;
 		}
 
@@ -1468,8 +1472,8 @@ static int create_altmaps_and_memory_blocks(int nid, struct memory_group *group,
 		ret = create_memory_block_devices(cur_start, memblock_size, nid,
 						  params.altmap, group);
 		if (ret) {
-			arch_remove_memory(cur_start, memblock_size, NULL);
-			kfree(params.altmap);
+			arch_remove_memory(cur_start, memblock_size, params.altmap, NULL);
+			altmap_free(params.altmap);
 			goto out;
 		}
 	}
@@ -1554,7 +1558,7 @@ int add_memory_resource(int nid, struct resource *res, mhp_t mhp_flags)
 		/* create memory block devices after memory was added */
 		ret = create_memory_block_devices(start, size, nid, NULL, group);
 		if (ret) {
-			arch_remove_memory(start, size, params.altmap);
+			arch_remove_memory(start, size, params.altmap, NULL);
 			goto error;
 		}
 	}
@@ -2266,7 +2270,7 @@ static int try_remove_memory(u64 start, u64 size)
 		 * No altmaps present, do the removal directly
 		 */
 		remove_memory_block_devices(start, size);
-		arch_remove_memory(start, size, NULL);
+		arch_remove_memory(start, size, NULL, NULL);
 	} else {
 		/* all memblocks in the range have altmaps */
 		remove_memory_blocks_and_altmaps(start, size);
diff --git a/mm/memremap.c b/mm/memremap.c
index 053842d45cb10..81766d8224009 100644
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -97,10 +97,10 @@ static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)
 				   PHYS_PFN(range_len(range)));
 	if (pgmap->type == MEMORY_DEVICE_PRIVATE) {
 		__remove_pages(PHYS_PFN(range->start),
-			       PHYS_PFN(range_len(range)), NULL);
+			       PHYS_PFN(range_len(range)), NULL, pgmap);
 	} else {
 		arch_remove_memory(range->start, range_len(range),
-				pgmap_altmap(pgmap));
+				pgmap_altmap(pgmap), pgmap);
 		kasan_remove_zero_shadow(__va(range->start), range_len(range));
 	}
 	mem_hotplug_done();
diff --git a/mm/migrate.c b/mm/migrate.c
index 8a64291ab5b44..0c6a0ab6eccef 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1135,26 +1135,24 @@ static int move_to_new_folio(struct folio *dst, struct folio *src,
  * This is safe because nobody is using it except us.
  */
 enum {
-	PAGE_WAS_MAPPED = BIT(0),
-	PAGE_WAS_MLOCKED = BIT(1),
-	PAGE_OLD_STATES = PAGE_WAS_MAPPED | PAGE_WAS_MLOCKED,
+	FOLIO_WAS_MAPPED = BIT(0),
+	FOLIO_WAS_MLOCKED = BIT(1),
+	FOLIO_OLD_STATES = FOLIO_WAS_MAPPED | FOLIO_WAS_MLOCKED,
 };
 
 static void __migrate_folio_record(struct folio *dst,
-				   int old_page_state,
-				   struct anon_vma *anon_vma)
+		int old_folio_state, struct anon_vma *anon_vma)
 {
-	dst->private = (void *)anon_vma + old_page_state;
+	dst->private = (void *)anon_vma + old_folio_state;
 }
 
 static void __migrate_folio_extract(struct folio *dst,
-				   int *old_page_state,
-				   struct anon_vma **anon_vmap)
+		int *old_folio_state, struct anon_vma **anon_vmap)
 {
 	unsigned long private = (unsigned long)dst->private;
 
-	*anon_vmap = (struct anon_vma *)(private & ~PAGE_OLD_STATES);
-	*old_page_state = private & PAGE_OLD_STATES;
+	*anon_vmap = (struct anon_vma *)(private & ~FOLIO_OLD_STATES);
+	*old_folio_state = private & FOLIO_OLD_STATES;
 	dst->private = NULL;
 }
 
@@ -1209,7 +1207,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 {
 	struct folio *dst;
 	int rc = -EAGAIN;
-	int old_page_state = 0;
+	int old_folio_state = 0;
 	struct anon_vma *anon_vma = NULL;
 	bool locked = false;
 	bool dst_locked = false;
@@ -1253,7 +1251,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 	}
 	locked = true;
 	if (folio_test_mlocked(src))
-		old_page_state |= PAGE_WAS_MLOCKED;
+		old_folio_state |= FOLIO_WAS_MLOCKED;
 
 	if (folio_test_writeback(src)) {
 		/*
@@ -1302,7 +1300,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 	dst_locked = true;
 
 	if (unlikely(page_has_movable_ops(&src->page))) {
-		__migrate_folio_record(dst, old_page_state, anon_vma);
+		__migrate_folio_record(dst, old_folio_state, anon_vma);
 		return 0;
 	}
 
@@ -1328,11 +1326,11 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 		VM_BUG_ON_FOLIO(folio_test_anon(src) &&
 			       !folio_test_ksm(src) && !anon_vma, src);
 		try_to_migrate(src, mode == MIGRATE_ASYNC ? TTU_BATCH_FLUSH : 0);
-		old_page_state |= PAGE_WAS_MAPPED;
+		old_folio_state |= FOLIO_WAS_MAPPED;
 	}
 
 	if (!folio_mapped(src)) {
-		__migrate_folio_record(dst, old_page_state, anon_vma);
+		__migrate_folio_record(dst, old_folio_state, anon_vma);
 		return 0;
 	}
 
@@ -1344,7 +1342,7 @@ static int migrate_folio_unmap(new_folio_t get_new_folio,
 	if (rc == -EAGAIN)
 		ret = NULL;
 
-	migrate_folio_undo_src(src, old_page_state & PAGE_WAS_MAPPED,
+	migrate_folio_undo_src(src, old_folio_state & FOLIO_WAS_MAPPED,
 			       anon_vma, locked, ret);
 	migrate_folio_undo_dst(dst, dst_locked, put_new_folio, private);
 
@@ -1358,13 +1356,13 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
 			      struct list_head *ret)
 {
 	int rc;
-	int old_page_state = 0;
+	int old_folio_state = 0;
 	struct anon_vma *anon_vma = NULL;
 	bool src_deferred_split = false;
 	bool src_partially_mapped = false;
 	struct list_head *prev;
 
-	__migrate_folio_extract(dst, &old_page_state, &anon_vma);
+	__migrate_folio_extract(dst, &old_folio_state, &anon_vma);
 	prev = dst->lru.prev;
 	list_del(&dst->lru);
 
@@ -1404,10 +1402,10 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
 	 * isolated from the unevictable LRU: but this case is the easiest.
 	 */
 	folio_add_lru(dst);
-	if (old_page_state & PAGE_WAS_MLOCKED)
+	if (old_folio_state & FOLIO_WAS_MLOCKED)
 		lru_add_drain();
 
-	if (old_page_state & PAGE_WAS_MAPPED)
+	if (old_folio_state & FOLIO_WAS_MAPPED)
 		remove_migration_ptes(src, dst, 0);
 
 out_unlock_both:
@@ -1439,11 +1437,11 @@ static int migrate_folio_move(free_folio_t put_new_folio, unsigned long private,
 	 */
 	if (rc == -EAGAIN) {
 		list_add(&dst->lru, prev);
-		__migrate_folio_record(dst, old_page_state, anon_vma);
+		__migrate_folio_record(dst, old_folio_state, anon_vma);
 		return rc;
 	}
 
-	migrate_folio_undo_src(src, old_page_state & PAGE_WAS_MAPPED,
+	migrate_folio_undo_src(src, old_folio_state & FOLIO_WAS_MAPPED,
 			       anon_vma, true, ret);
 	migrate_folio_undo_dst(dst, true, put_new_folio, private);
 
@@ -1777,11 +1775,11 @@ static void migrate_folios_undo(struct list_head *src_folios,
 	dst = list_first_entry(dst_folios, struct folio, lru);
 	dst2 = list_next_entry(dst, lru);
 	list_for_each_entry_safe(folio, folio2, src_folios, lru) {
-		int old_page_state = 0;
+		int old_folio_state = 0;
 		struct anon_vma *anon_vma = NULL;
 
-		__migrate_folio_extract(dst, &old_page_state, &anon_vma);
-		migrate_folio_undo_src(folio, old_page_state & PAGE_WAS_MAPPED,
+		__migrate_folio_extract(dst, &old_folio_state, &anon_vma);
+		migrate_folio_undo_src(folio, old_folio_state & FOLIO_WAS_MAPPED,
 				anon_vma, true, ret_folios);
 		list_del(&dst->lru);
 		migrate_folio_undo_dst(dst, true, put_new_folio, private);
diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index fbfe5715f6351..554754eb26ff2 100644
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -801,8 +801,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
 	bool flush = false;
 	unsigned long i;
 
-	VM_WARN_ON_FOLIO(!folio, folio);
-	VM_WARN_ON_ONCE(!pmd_none(*pmdp) && !is_huge_zero_pmd(*pmdp));
+	VM_WARN_ON_ONCE(!folio);
 
 	if (!thp_vma_suitable_order(vma, addr, HPAGE_PMD_ORDER))
 		return -EINVAL;
@@ -840,7 +839,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
 	} else {
 		if (folio_is_zone_device(folio) &&
 		    !folio_is_device_coherent(folio)) {
-			goto abort;
+			goto free_abort;
 		}
 		entry = folio_mk_pmd(folio, vma->vm_page_prot);
 		if (vma->vm_flags & VM_WRITE)
@@ -850,7 +849,7 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
 	ptl = pmd_lock(vma->vm_mm, pmdp);
 	csa_ret = check_stable_address_space(vma->vm_mm);
 	if (csa_ret)
-		goto abort;
+		goto unlock_abort;
 
 	/*
 	 * Check for userfaultfd but do not deliver the fault. Instead,
@@ -859,11 +858,9 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
 	if (userfaultfd_missing(vma))
 		goto unlock_abort;
 
-	if (!pmd_none(*pmdp)) {
-		if (!is_huge_zero_pmd(*pmdp))
-			goto unlock_abort;
+	if (is_huge_zero_pmd(*pmdp))
 		flush = true;
-	} else if (!pmd_none(*pmdp))
+	else if (!pmd_none(*pmdp))
 		goto unlock_abort;
 
 	add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
@@ -893,6 +890,8 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate,
 
 unlock_abort:
 	spin_unlock(ptl);
+free_abort:
+	pte_free(vma->vm_mm, pgtable);
 abort:
 	for (i = 0; i < HPAGE_PMD_NR; i++)
 		src[i] &= ~MIGRATE_PFN_MIGRATE;
diff --git a/mm/mm_init.c b/mm/mm_init.c
index f9f8e1af921cd..6de3a77eb9ae5 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -674,6 +674,20 @@ static inline void fixup_hashdist(void)
 static inline void fixup_hashdist(void) {}
 #endif /* CONFIG_NUMA */
 
+#ifdef CONFIG_ZONE_DEVICE
+static __meminit void pageblock_migratetype_init_range(unsigned long pfn,
+		unsigned long nr_pages, int migratetype)
+{
+	const unsigned long end = pfn + nr_pages;
+
+	for (pfn = pageblock_align(pfn); pfn < end; pfn += pageblock_nr_pages) {
+		init_pageblock_migratetype(pfn_to_page(pfn), migratetype, false);
+		if (IS_ALIGNED(pfn, PAGES_PER_SECTION))
+			cond_resched();
+	}
+}
+#endif
+
 /*
  * Initialize a reserved page unconditionally, finding its zone first.
  */
@@ -692,9 +706,11 @@ void __meminit __init_page_from_nid(unsigned long pfn, int nid)
 	}
 	__init_single_page(pfn_to_page(pfn), pfn, zid, nid);
 
-	if (pageblock_aligned(pfn))
-		init_pageblock_migratetype(pfn_to_page(pfn), MIGRATE_MOVABLE,
-				false);
+	if (pageblock_aligned(pfn)) {
+		enum migratetype mt =
+			kho_scratch_migratetype(pfn, MIGRATE_MOVABLE);
+		init_pageblock_migratetype(pfn_to_page(pfn), mt, false);
+	}
 }
 
 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
@@ -927,7 +943,8 @@ void __meminit memmap_init_range(unsigned long size, int nid, unsigned long zone
 static void __init memmap_init_zone_range(struct zone *zone,
 					  unsigned long start_pfn,
 					  unsigned long end_pfn,
-					  unsigned long *hole_pfn)
+					  unsigned long *hole_pfn,
+					  enum migratetype mt)
 {
 	unsigned long zone_start_pfn = zone->zone_start_pfn;
 	unsigned long zone_end_pfn = zone_start_pfn + zone->spanned_pages;
@@ -940,8 +957,7 @@ static void __init memmap_init_zone_range(struct zone *zone,
 		return;
 
 	memmap_init_range(end_pfn - start_pfn, nid, zone_id, start_pfn,
-			  zone_end_pfn, MEMINIT_EARLY, NULL, MIGRATE_MOVABLE,
-			  false);
+			  zone_end_pfn, MEMINIT_EARLY, NULL, mt, false);
 
 	if (*hole_pfn < start_pfn)
 		init_unavailable_range(*hole_pfn, start_pfn, zone_id, nid);
@@ -957,6 +973,8 @@ static void __init memmap_init(void)
 
 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
 		struct pglist_data *node = NODE_DATA(nid);
+		enum migratetype mt =
+			kho_scratch_migratetype(start_pfn, MIGRATE_MOVABLE);
 
 		for (j = 0; j < MAX_NR_ZONES; j++) {
 			struct zone *zone = node->node_zones + j;
@@ -965,7 +983,7 @@ static void __init memmap_init(void)
 				continue;
 
 			memmap_init_zone_range(zone, start_pfn, end_pfn,
-					       &hole_pfn);
+					       &hole_pfn, mt);
 			zone_id = j;
 		}
 	}
@@ -1011,21 +1029,6 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
 	page_folio(page)->pgmap = pgmap;
 	page->zone_device_data = NULL;
 
-	/*
-	 * Mark the block movable so that blocks are reserved for
-	 * movable at startup. This will force kernel allocations
-	 * to reserve their blocks rather than leaking throughout
-	 * the address space during boot when many long-lived
-	 * kernel allocations are made.
-	 *
-	 * Please note that MEMINIT_HOTPLUG path doesn't clear memmap
-	 * because this is done early in section_activate()
-	 */
-	if (pageblock_aligned(pfn)) {
-		init_pageblock_migratetype(page, MIGRATE_MOVABLE, false);
-		cond_resched();
-	}
-
 	/*
 	 * ZONE_DEVICE pages other than MEMORY_TYPE_GENERIC are released
 	 * directly to the driver page allocator which will set the page count
@@ -1056,10 +1059,17 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
  * of how the sparse_vmemmap internals handle compound pages in the lack
  * of an altmap. See vmemmap_populate_compound_pages().
  */
-static inline unsigned long compound_nr_pages(struct vmem_altmap *altmap,
+static inline unsigned long compound_nr_pages(unsigned long pfn,
+					      struct vmem_altmap *altmap,
 					      struct dev_pagemap *pgmap)
 {
-	if (!vmemmap_can_optimize(altmap, pgmap))
+	/*
+	 * If DAX memory is hot-plugged into an unoccupied subsection
+	 * of an early section, the unoptimized boot memmap is reused.
+	 * See section_activate().
+	 */
+	if (early_section(__pfn_to_section(pfn)) ||
+	    !vmemmap_can_optimize(altmap, pgmap))
 		return pgmap_vmemmap_nr(pgmap);
 
 	return VMEMMAP_RESERVE_NR * (PAGE_SIZE / sizeof(struct page));
@@ -1122,13 +1132,18 @@ void __ref memmap_init_zone_device(struct zone *zone,
 
 		__init_zone_device_page(page, pfn, zone_idx, nid, pgmap);
 
+		if (IS_ALIGNED(pfn, PAGES_PER_SECTION))
+			cond_resched();
+
 		if (pfns_per_compound == 1)
 			continue;
 
 		memmap_init_compound(page, pfn, zone_idx, nid, pgmap,
-				     compound_nr_pages(altmap, pgmap));
+				     compound_nr_pages(pfn, altmap, pgmap));
 	}
 
+	pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE);
+
 	pr_debug("%s initialised %lu pages in %ums\n", __func__,
 		nr_pages, jiffies_to_msecs(jiffies - start));
 }
@@ -1418,11 +1433,14 @@ static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx,
 
 static void __meminit zone_init_free_lists(struct zone *zone)
 {
-	unsigned int order, t;
-	for_each_migratetype_order(order, t) {
-		INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
+	struct list_head *list;
+	unsigned int order;
+
+	for_each_free_list(list, zone, order)
+		INIT_LIST_HEAD(list);
+
+	for (order = 0; order < NR_PAGE_ORDERS; order++)
 		zone->free_area[order].nr_free = 0;
-	}
 
 #ifdef CONFIG_UNACCEPTED_MEMORY
 	INIT_LIST_HEAD(&zone->unaccepted_pages);
@@ -1970,7 +1988,7 @@ unsigned long __init node_map_pfn_alignment(void)
 
 #ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT
 static void __init deferred_free_pages(unsigned long pfn,
-		unsigned long nr_pages)
+		unsigned long nr_pages, enum migratetype mt)
 {
 	struct page *page;
 	unsigned long i;
@@ -1983,8 +2001,7 @@ static void __init deferred_free_pages(unsigned long pfn,
 	/* Free a large naturally-aligned chunk if possible */
 	if (nr_pages == MAX_ORDER_NR_PAGES && IS_MAX_ORDER_ALIGNED(pfn)) {
 		for (i = 0; i < nr_pages; i += pageblock_nr_pages)
-			init_pageblock_migratetype(page + i, MIGRATE_MOVABLE,
-					false);
+			init_pageblock_migratetype(page + i, mt, false);
 		__free_pages_core(page, MAX_PAGE_ORDER, MEMINIT_EARLY);
 		return;
 	}
@@ -1994,8 +2011,7 @@ static void __init deferred_free_pages(unsigned long pfn,
 
 	for (i = 0; i < nr_pages; i++, page++, pfn++) {
 		if (pageblock_aligned(pfn))
-			init_pageblock_migratetype(page, MIGRATE_MOVABLE,
-					false);
+			init_pageblock_migratetype(page, mt, false);
 		__free_pages_core(page, 0, MEMINIT_EARLY);
 	}
 }
@@ -2053,6 +2069,8 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
 	for_each_free_mem_range(i, nid, 0, &start, &end, NULL) {
 		unsigned long spfn = PFN_UP(start);
 		unsigned long epfn = PFN_DOWN(end);
+		enum migratetype mt =
+			kho_scratch_migratetype(spfn, MIGRATE_MOVABLE);
 
 		if (spfn >= end_pfn)
 			break;
@@ -2065,7 +2083,7 @@ deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn,
 			unsigned long chunk_end = min(mo_pfn, epfn);
 
 			nr_pages += deferred_init_pages(zone, spfn, chunk_end);
-			deferred_free_pages(spfn, chunk_end - spfn);
+			deferred_free_pages(spfn, chunk_end - spfn, mt);
 
 			spfn = chunk_end;
 
diff --git a/mm/mmap.c b/mm/mmap.c
index 5754d1c364624..2311ae7c2ff45 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -504,7 +504,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr,
 			break;
 		case MAP_DROPPABLE:
 			if (VM_DROPPABLE == VM_NONE)
-				return -ENOTSUPP;
+				return -EOPNOTSUPP;
 			/*
 			 * A locked or stack area makes no sense to be droppable.
 			 *
diff --git a/mm/mmu_gather.c b/mm/mmu_gather.c
index 3985d856de7f9..37a6a711c37e7 100644
--- a/mm/mmu_gather.c
+++ b/mm/mmu_gather.c
@@ -283,6 +283,14 @@ void tlb_remove_table_sync_one(void)
 	 * It is however sufficient for software page-table walkers that rely on
 	 * IRQ disabling.
 	 */
+
+	/*
+	 * Skip IPI if the preceding TLB flush already synchronized with
+	 * all CPUs that could be doing software/lockless page table walks.
+	 */
+	if (tlb_table_flush_implies_ipi_broadcast())
+		return;
+
 	smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
 }
 
@@ -312,6 +320,13 @@ static void tlb_remove_table_free(struct mmu_table_batch *batch)
  */
 void tlb_remove_table_sync_rcu(void)
 {
+	/*
+	 * Skip RCU wait if the preceding TLB flush already synchronized
+	 * with all CPUs that could be doing software/lockless page table walks.
+	 */
+	if (tlb_table_flush_implies_ipi_broadcast())
+		return;
+
 	synchronize_rcu();
 }
 
diff --git a/mm/mseal.c b/mm/mseal.c
index e2093ae3d25cd..9781647483d19 100644
--- a/mm/mseal.c
+++ b/mm/mseal.c
@@ -8,6 +8,7 @@
  */
 
 #include <linux/mempolicy.h>
+#include <linux/minmax.h>
 #include <linux/mman.h>
 #include <linux/mm.h>
 #include <linux/mm_inline.h>
@@ -65,8 +66,8 @@ static int mseal_apply(struct mm_struct *mm,
 		prev = vma;
 
 	for_each_vma_range(vmi, vma, end) {
-		const unsigned long curr_start = MAX(vma->vm_start, start);
-		const unsigned long curr_end = MIN(vma->vm_end, end);
+		const unsigned long curr_start = max(vma->vm_start, start);
+		const unsigned long curr_end = min(vma->vm_end, end);
 
 		if (!vma_test(vma, VMA_SEALED_BIT)) {
 			vma_flags_t vma_flags = vma->flags;
diff --git a/mm/numa_emulation.c b/mm/numa_emulation.c
index 703c8fa050480..55f26b22bb0be 100644
--- a/mm/numa_emulation.c
+++ b/mm/numa_emulation.c
@@ -214,7 +214,7 @@ static u64 uniform_size(u64 max_addr, u64 base, u64 hole, int nr_nodes)
  * Sets up fake nodes of `size' interleaved over physical nodes ranging from
  * `addr' to `max_addr'.
  *
- * Returns zero on success or negative on error.
+ * Returns node ID of the next node on success or negative error code.
  */
 static int __init split_nodes_size_interleave_uniform(struct numa_meminfo *ei,
 					      struct numa_meminfo *pi,
@@ -398,7 +398,7 @@ void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt)
 	 */
 	if (strchr(emu_cmdline, 'U')) {
 		unsigned long n;
-		int nid = 0;
+		int nid = 0, nr_created;
 
 		n = simple_strtoul(emu_cmdline, &emu_cmdline, 0);
 		ret = -1;
@@ -416,9 +416,18 @@ void __init numa_emulation(struct numa_meminfo *numa_meminfo, int numa_dist_cnt)
 					n, &pi.blk[0], nid);
 			if (ret < 0)
 				break;
-			if (ret < n) {
+
+			/*
+			 * If no memory was found for this physical node,
+			 * skip the under-allocation check.
+			 */
+			if (ret == nid)
+				continue;
+
+			nr_created = ret - nid;
+			if (nr_created < n) {
 				pr_info("%s: phys: %d only got %d of %ld nodes, failing\n",
-						__func__, i, ret, n);
+						__func__, i, nr_created, n);
 				ret = -1;
 				break;
 			}
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 227d58dc3de6a..ef6c96e2b1ce4 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -90,6 +90,9 @@ typedef int __bitwise fpi_t;
 /* Free the page without taking locks. Rely on trylock only. */
 #define FPI_TRYLOCK		((__force fpi_t)BIT(2))
 
+/* free_pages_prepare() has already been called for page(s) being freed. */
+#define FPI_PREPARED		((__force fpi_t)BIT(3))
+
 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
 static DEFINE_MUTEX(pcp_batch_high_lock);
 #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8)
@@ -282,6 +285,14 @@ EXPORT_SYMBOL(nr_node_ids);
 EXPORT_SYMBOL(nr_online_nodes);
 #endif
 
+/*
+ * When page allocations stall for longer than a threshold,
+ * ALLOC_STALL_WARN_MSECS, leave a warning in the kernel log.  Only one warning
+ * will be printed during this duration for the entire system.
+ */
+#define ALLOC_STALL_WARN_MSECS (10 * 1000UL)
+static unsigned long alloc_stall_warn_jiffies = INITIAL_JIFFIES;
+
 static bool page_contains_unaccepted(struct page *page, unsigned int order);
 static bool cond_accept_memory(struct zone *zone, unsigned int order,
 			       int alloc_flags);
@@ -353,7 +364,7 @@ get_pfnblock_bitmap_bitidx(const struct page *page, unsigned long pfn,
 #else
 	BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
 #endif
-	BUILD_BUG_ON(__MIGRATE_TYPE_END > MIGRATETYPE_MASK);
+	BUILD_BUG_ON(__MIGRATE_TYPE_END > PAGEBLOCK_MIGRATETYPE_MASK);
 	VM_BUG_ON_PAGE(!zone_spans_pfn(page_zone(page), pfn), page);
 
 	bitmap = get_pageblock_bitmap(page, pfn);
@@ -423,10 +434,10 @@ bool get_pfnblock_bit(const struct page *page, unsigned long pfn,
  * Use get_pfnblock_migratetype() if caller already has both @page and @pfn
  * to save a call to page_to_pfn().
  */
-__always_inline enum migratetype
+enum migratetype
 get_pfnblock_migratetype(const struct page *page, unsigned long pfn)
 {
-	unsigned long mask = MIGRATETYPE_AND_ISO_MASK;
+	unsigned long mask = PAGEBLOCK_MIGRATETYPE_MASK | PAGEBLOCK_ISO_MASK;
 	unsigned long flags;
 
 	flags = __get_pfnblock_flags_mask(page, pfn, mask);
@@ -435,7 +446,7 @@ get_pfnblock_migratetype(const struct page *page, unsigned long pfn)
 	if (flags & BIT(PB_migrate_isolate))
 		return MIGRATE_ISOLATE;
 #endif
-	return flags & MIGRATETYPE_MASK;
+	return flags & PAGEBLOCK_MIGRATETYPE_MASK;
 }
 
 /**
@@ -523,11 +534,11 @@ static void set_pageblock_migratetype(struct page *page,
 	}
 	VM_WARN_ONCE(get_pageblock_isolate(page),
 		     "Use clear_pageblock_isolate() to unisolate pageblock");
-	/* MIGRATETYPE_AND_ISO_MASK clears PB_migrate_isolate if it is set */
+	/* PAGEBLOCK_ISO_MASK clears PB_migrate_isolate if it is set */
 #endif
 	__set_pfnblock_flags_mask(page, page_to_pfn(page),
 				  (unsigned long)migratetype,
-				  MIGRATETYPE_AND_ISO_MASK);
+				  PAGEBLOCK_MIGRATETYPE_MASK | PAGEBLOCK_ISO_MASK);
 }
 
 void __meminit init_pageblock_migratetype(struct page *page,
@@ -553,7 +564,7 @@ void __meminit init_pageblock_migratetype(struct page *page,
 		flags |= BIT(PB_migrate_isolate);
 #endif
 	__set_pfnblock_flags_mask(page, page_to_pfn(page), flags,
-				  MIGRATETYPE_AND_ISO_MASK);
+				  PAGEBLOCK_MIGRATETYPE_MASK | PAGEBLOCK_ISO_MASK);
 }
 
 #ifdef CONFIG_DEBUG_VM
@@ -639,19 +650,17 @@ static void bad_page(struct page *page, const char *reason)
 
 static inline unsigned int order_to_pindex(int migratetype, int order)
 {
+	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
+		bool movable = migratetype == MIGRATE_MOVABLE;
 
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-	bool movable;
-	if (order > PAGE_ALLOC_COSTLY_ORDER) {
-		VM_BUG_ON(!is_pmd_order(order));
+		if (order > PAGE_ALLOC_COSTLY_ORDER) {
+			VM_BUG_ON(!is_pmd_order(order));
 
-		movable = migratetype == MIGRATE_MOVABLE;
-
-		return NR_LOWORDER_PCP_LISTS + movable;
+			return NR_LOWORDER_PCP_LISTS + movable;
+		}
+	} else {
+		VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
 	}
-#else
-	VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
-#endif
 
 	return (MIGRATE_PCPTYPES * order) + migratetype;
 }
@@ -660,12 +669,12 @@ static inline int pindex_to_order(unsigned int pindex)
 {
 	int order = pindex / MIGRATE_PCPTYPES;
 
-#ifdef CONFIG_TRANSPARENT_HUGEPAGE
-	if (pindex >= NR_LOWORDER_PCP_LISTS)
-		order = HPAGE_PMD_ORDER;
-#else
-	VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
-#endif
+	if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
+		if (pindex >= NR_LOWORDER_PCP_LISTS)
+			order = HPAGE_PMD_ORDER;
+	} else {
+		VM_BUG_ON(order > PAGE_ALLOC_COSTLY_ORDER);
+	}
 
 	return order;
 }
@@ -1035,6 +1044,7 @@ static inline bool page_expected_state(struct page *page,
 #ifdef CONFIG_MEMCG
 			page->memcg_data |
 #endif
+			page_pool_page_is_pp(page) |
 			(page->flags.f & check_flags)))
 		return false;
 
@@ -1061,6 +1071,8 @@ static const char *page_bad_reason(struct page *page, unsigned long flags)
 	if (unlikely(page->memcg_data))
 		bad_reason = "page still charged to cgroup";
 #endif
+	if (unlikely(page_pool_page_is_pp(page)))
+		bad_reason = "page_pool leak";
 	return bad_reason;
 }
 
@@ -1208,14 +1220,18 @@ static inline bool should_skip_kasan_poison(struct page *page)
 	return page_kasan_tag(page) == KASAN_TAG_KERNEL;
 }
 
-static void kernel_init_pages(struct page *page, int numpages)
+static void clear_highpages_kasan_tagged(struct page *page, int numpages)
 {
-	int i;
-
 	/* s390's use of memset() could override KASAN redzones. */
 	kasan_disable_current();
-	for (i = 0; i < numpages; i++)
-		clear_highpage_kasan_tagged(page + i);
+	if (!IS_ENABLED(CONFIG_HIGHMEM)) {
+		clear_pages(kasan_reset_tag(page_address(page)), numpages);
+	} else {
+		int i;
+
+		for (i = 0; i < numpages; i++)
+			clear_highpage_kasan_tagged(page + i);
+	}
 	kasan_enable_current();
 }
 
@@ -1237,7 +1253,7 @@ void __clear_page_tag_ref(struct page *page)
 /* Should be called only if mem_alloc_profiling_enabled() */
 static noinline
 void __pgalloc_tag_add(struct page *page, struct task_struct *task,
-		       unsigned int nr)
+		       unsigned int nr, gfp_t gfp_flags)
 {
 	union pgtag_ref_handle handle;
 	union codetag_ref ref;
@@ -1251,17 +1267,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,
 		 * page_ext is not available yet, record the pfn so we can
 		 * clear the tag ref later when page_ext is initialized.
 		 */
-		alloc_tag_add_early_pfn(page_to_pfn(page));
+		alloc_tag_add_early_pfn(page_to_pfn(page), gfp_flags);
 		if (task->alloc_tag)
 			alloc_tag_set_inaccurate(task->alloc_tag);
 	}
 }
 
 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
-				   unsigned int nr)
+				   unsigned int nr, gfp_t gfp_flags)
 {
 	if (mem_alloc_profiling_enabled())
-		__pgalloc_tag_add(page, task, nr);
+		__pgalloc_tag_add(page, task, nr, gfp_flags);
 }
 
 /* Should be called only if mem_alloc_profiling_enabled() */
@@ -1294,14 +1310,14 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)
 #else /* CONFIG_MEM_ALLOC_PROFILING */
 
 static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
-				   unsigned int nr) {}
+				   unsigned int nr, gfp_t gfp_flags) {}
 static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
 static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
 
 #endif /* CONFIG_MEM_ALLOC_PROFILING */
 
-__always_inline bool __free_pages_prepare(struct page *page,
-					  unsigned int order, fpi_t fpi_flags)
+static __always_inline bool __free_pages_prepare(struct page *page,
+		unsigned int order, fpi_t fpi_flags)
 {
 	int bad = 0;
 	bool skip_kasan_poison = should_skip_kasan_poison(page);
@@ -1309,6 +1325,9 @@ __always_inline bool __free_pages_prepare(struct page *page,
 	bool compound = PageCompound(page);
 	struct folio *folio = page_folio(page);
 
+	if (fpi_flags & FPI_PREPARED)
+		return true;
+
 	VM_BUG_ON_PAGE(PageTail(page), page);
 
 	trace_mm_page_free(page, order);
@@ -1377,17 +1396,9 @@ __always_inline bool __free_pages_prepare(struct page *page,
 		mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1);
 		folio->mapping = NULL;
 	}
-	if (unlikely(page_has_type(page))) {
-		/* networking expects to clear its page type before releasing */
-		if (is_check_pages_enabled()) {
-			if (unlikely(PageNetpp(page))) {
-				bad_page(page, "page_pool leak");
-				return false;
-			}
-		}
+	if (unlikely(page_has_type(page)))
 		/* Reset the page_type (which overlays _mapcount) */
 		page->page_type = UINT_MAX;
-	}
 
 	if (is_check_pages_enabled()) {
 		if (free_page_is_bad(page))
@@ -1428,7 +1439,7 @@ __always_inline bool __free_pages_prepare(struct page *page,
 			init = false;
 	}
 	if (init)
-		kernel_init_pages(page, 1 << order);
+		clear_highpages_kasan_tagged(page, 1 << order);
 
 	/*
 	 * arch_free_page() can make the page's contents inaccessible.  s390
@@ -1456,7 +1467,6 @@ static void free_pcppages_bulk(struct zone *zone, int count,
 					struct per_cpu_pages *pcp,
 					int pindex)
 {
-	unsigned long flags;
 	unsigned int order;
 	struct page *page;
 
@@ -1469,7 +1479,7 @@ static void free_pcppages_bulk(struct zone *zone, int count,
 	/* Ensure requested pindex is drained first. */
 	pindex = pindex - 1;
 
-	spin_lock_irqsave(&zone->lock, flags);
+	guard(spinlock_irqsave)(&zone->lock);
 
 	while (count > 0) {
 		struct list_head *list;
@@ -1501,8 +1511,6 @@ static void free_pcppages_bulk(struct zone *zone, int count,
 			trace_mm_page_pcpu_drain(page, order, mt);
 		} while (count > 0 && !list_empty(list));
 	}
-
-	spin_unlock_irqrestore(&zone->lock, flags);
 }
 
 /* Split a multi-block free page into its individual pageblocks. */
@@ -1808,9 +1816,9 @@ static inline bool should_skip_init(gfp_t flags)
 inline void post_alloc_hook(struct page *page, unsigned int order,
 				gfp_t gfp_flags)
 {
+	const bool zero_tags = gfp_flags & __GFP_ZEROTAGS;
 	bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags) &&
 			!should_skip_init(gfp_flags);
-	bool zero_tags = init && (gfp_flags & __GFP_ZEROTAGS);
 	int i;
 
 	set_page_private(page, 0);
@@ -1832,11 +1840,11 @@ inline void post_alloc_hook(struct page *page, unsigned int order,
 	 */
 
 	/*
-	 * If memory tags should be zeroed
-	 * (which happens only when memory should be initialized as well).
+	 * Clearing tags can efficiently clear the memory for us as well, if
+	 * required.
 	 */
 	if (zero_tags)
-		init = !tag_clear_highpages(page, 1 << order);
+		init = tag_clear_highpages(page, 1 << order, /* clear_pages= */init);
 
 	if (!should_skip_kasan_unpoison(gfp_flags) &&
 	    kasan_unpoison_pages(page, order, init)) {
@@ -1853,11 +1861,11 @@ inline void post_alloc_hook(struct page *page, unsigned int order,
 	}
 	/* If memory is still not initialized, initialize it now. */
 	if (init)
-		kernel_init_pages(page, 1 << order);
+		clear_highpages_kasan_tagged(page, 1 << order);
 
 	set_page_owner(page, order, gfp_flags);
 	page_table_check_alloc(page, order);
-	pgalloc_tag_add(page, current, 1 << order);
+	pgalloc_tag_add(page, current, 1 << order, gfp_flags);
 }
 
 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
@@ -2130,15 +2138,15 @@ static bool __move_freepages_block_isolate(struct zone *zone,
 	}
 
 move:
-	/* Use MIGRATETYPE_MASK to get non-isolate migratetype */
+	/* Use PAGEBLOCK_MIGRATETYPE_MASK to get non-isolate migratetype */
 	if (isolate) {
 		from_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),
-						    MIGRATETYPE_MASK);
+						    PAGEBLOCK_MIGRATETYPE_MASK);
 		to_mt = MIGRATE_ISOLATE;
 	} else {
 		from_mt = MIGRATE_ISOLATE;
 		to_mt = __get_pfnblock_flags_mask(page, page_to_pfn(page),
-						  MIGRATETYPE_MASK);
+						  PAGEBLOCK_MIGRATETYPE_MASK);
 	}
 
 	__move_freepages_block(zone, start_pfn, from_mt, to_mt);
@@ -2249,25 +2257,29 @@ static bool should_try_claim_block(unsigned int order, int start_mt)
  * we would do this whole-block claiming. This would help to reduce
  * fragmentation due to mixed migratetype pages in one pageblock.
  */
-int find_suitable_fallback(struct free_area *area, unsigned int order,
-			   int migratetype, bool claimable)
+enum fallback_result
+find_suitable_fallback(struct free_area *area, unsigned int order,
+		       int migratetype, bool claimable, int *mt_out)
 {
 	int i;
 
 	if (claimable && !should_try_claim_block(order, migratetype))
-		return -2;
+		return FALLBACK_NOCLAIM;
 
 	if (area->nr_free == 0)
-		return -1;
+		return FALLBACK_EMPTY;
 
 	for (i = 0; i < MIGRATE_PCPTYPES - 1 ; i++) {
 		int fallback_mt = fallbacks[migratetype][i];
 
-		if (!free_area_empty(area, fallback_mt))
-			return fallback_mt;
+		if (!free_area_empty(area, fallback_mt)) {
+			if (mt_out)
+				*mt_out = fallback_mt;
+			return FALLBACK_FOUND;
+		}
 	}
 
-	return -1;
+	return FALLBACK_EMPTY;
 }
 
 /*
@@ -2377,16 +2389,16 @@ __rmqueue_claim(struct zone *zone, int order, int start_migratetype,
 	 */
 	for (current_order = MAX_PAGE_ORDER; current_order >= min_order;
 				--current_order) {
+		enum fallback_result result;
+
 		area = &(zone->free_area[current_order]);
-		fallback_mt = find_suitable_fallback(area, current_order,
-						     start_migratetype, true);
+		result = find_suitable_fallback(area, current_order,
+						start_migratetype, true, &fallback_mt);
 
-		/* No block in that order */
-		if (fallback_mt == -1)
+		if (result == FALLBACK_EMPTY)
 			continue;
 
-		/* Advanced into orders too low to claim, abort */
-		if (fallback_mt == -2)
+		if (result == FALLBACK_NOCLAIM)
 			break;
 
 		page = get_page_from_free_area(area, fallback_mt);
@@ -2416,10 +2428,12 @@ __rmqueue_steal(struct zone *zone, int order, int start_migratetype)
 	int fallback_mt;
 
 	for (current_order = order; current_order < NR_PAGE_ORDERS; current_order++) {
+		enum fallback_result result;
+
 		area = &(zone->free_area[current_order]);
-		fallback_mt = find_suitable_fallback(area, current_order,
-						     start_migratetype, false);
-		if (fallback_mt == -1)
+		result = find_suitable_fallback(area, current_order, start_migratetype,
+						false, &fallback_mt);
+		if (result == FALLBACK_EMPTY)
 			continue;
 
 		page = get_page_from_free_area(area, fallback_mt);
@@ -3429,7 +3443,7 @@ static void reserve_highatomic_pageblock(struct page *page, int order,
 					 struct zone *zone)
 {
 	int mt;
-	unsigned long max_managed, flags;
+	unsigned long max_managed;
 
 	/*
 	 * The number reserved as: minimum is 1 pageblock, maximum is
@@ -3443,29 +3457,26 @@ static void reserve_highatomic_pageblock(struct page *page, int order,
 	if (zone->nr_reserved_highatomic >= max_managed)
 		return;
 
-	spin_lock_irqsave(&zone->lock, flags);
+	guard(spinlock_irqsave)(&zone->lock);
 
 	/* Recheck the nr_reserved_highatomic limit under the lock */
 	if (zone->nr_reserved_highatomic >= max_managed)
-		goto out_unlock;
+		return;
 
 	/* Yoink! */
 	mt = get_pageblock_migratetype(page);
 	/* Only reserve normal pageblocks (i.e., they can merge with others) */
 	if (!migratetype_is_mergeable(mt))
-		goto out_unlock;
+		return;
 
 	if (order < pageblock_order) {
 		if (move_freepages_block(zone, page, mt, MIGRATE_HIGHATOMIC) == -1)
-			goto out_unlock;
+			return;
 		zone->nr_reserved_highatomic += pageblock_nr_pages;
 	} else {
 		change_pageblock_range(page, order, MIGRATE_HIGHATOMIC);
 		zone->nr_reserved_highatomic += 1 << order;
 	}
-
-out_unlock:
-	spin_unlock_irqrestore(&zone->lock, flags);
 }
 
 /*
@@ -3481,7 +3492,6 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
 						bool force)
 {
 	struct zonelist *zonelist = ac->zonelist;
-	unsigned long flags;
 	struct zoneref *z;
 	struct zone *zone;
 	struct page *page;
@@ -3498,7 +3508,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
 					pageblock_nr_pages)
 			continue;
 
-		spin_lock_irqsave(&zone->lock, flags);
+		guard(spinlock_irqsave)(&zone->lock);
 		for (order = 0; order < NR_PAGE_ORDERS; order++) {
 			struct free_area *area = &(zone->free_area[order]);
 			unsigned long size;
@@ -3545,12 +3555,9 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
 			 * so this should not fail on zone boundaries.
 			 */
 			WARN_ON_ONCE(ret == -1);
-			if (ret > 0) {
-				spin_unlock_irqrestore(&zone->lock, flags);
+			if (ret > 0)
 				return ret;
-			}
 		}
-		spin_unlock_irqrestore(&zone->lock, flags);
 	}
 
 	return false;
@@ -4683,6 +4690,40 @@ check_retry_cpuset(int cpuset_mems_cookie, struct alloc_context *ac)
 	return false;
 }
 
+static void check_alloc_stall_warn(gfp_t gfp_mask, nodemask_t *nodemask,
+				unsigned int order, unsigned long alloc_start_time)
+{
+	static DEFINE_SPINLOCK(alloc_stall_lock);
+	unsigned long stall_msecs = jiffies_to_msecs(jiffies - alloc_start_time);
+
+	if (likely(stall_msecs < ALLOC_STALL_WARN_MSECS))
+		return;
+	if (time_is_after_jiffies(READ_ONCE(alloc_stall_warn_jiffies)))
+		return;
+	if (gfp_mask & __GFP_NOWARN)
+		return;
+
+	if (!spin_trylock(&alloc_stall_lock))
+		return;
+
+	/* Check again, this time under the lock */
+	if (time_is_after_jiffies(alloc_stall_warn_jiffies)) {
+		spin_unlock(&alloc_stall_lock);
+		return;
+	}
+
+	WRITE_ONCE(alloc_stall_warn_jiffies, jiffies + msecs_to_jiffies(ALLOC_STALL_WARN_MSECS));
+	spin_unlock(&alloc_stall_lock);
+
+	pr_warn("%s: page allocation stall for %lu secs: order:%d, mode:%#x(%pGg) nodemask=%*pbl",
+		current->comm, stall_msecs / MSEC_PER_SEC, order, gfp_mask, &gfp_mask,
+		nodemask_pr_args(nodemask));
+	cpuset_print_current_mems_allowed();
+	pr_cont("\n");
+	dump_stack();
+	warn_alloc_show_mem(gfp_mask, nodemask);
+}
+
 static inline struct page *
 __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 						struct alloc_context *ac)
@@ -4703,6 +4744,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 	int reserve_flags;
 	bool compact_first = false;
 	bool can_retry_reserves = true;
+	unsigned long alloc_start_time = jiffies;
 
 	if (unlikely(nofail)) {
 		/*
@@ -4818,6 +4860,9 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
 	if (current->flags & PF_MEMALLOC)
 		goto nopage;
 
+	/* If allocation has taken excessively long, warn about it */
+	check_alloc_stall_warn(gfp_mask, ac->nodemask, order, alloc_start_time);
+
 	/* Try direct reclaim and then allocating */
 	if (!compact_first) {
 		page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags,
@@ -5049,7 +5094,6 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
 	struct per_cpu_pages *pcp;
 	struct list_head *pcp_list;
 	struct alloc_context ac;
-	gfp_t alloc_gfp;
 	unsigned int alloc_flags = ALLOC_WMARK_LOW;
 	int nr_populated = 0, nr_account = 0;
 
@@ -5090,10 +5134,8 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
 
 	/* May set ALLOC_NOFRAGMENT, fragmentation will return 1 page. */
 	gfp &= gfp_allowed_mask;
-	alloc_gfp = gfp;
-	if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &alloc_gfp, &alloc_flags))
+	if (!prepare_alloc_pages(gfp, 0, preferred_nid, nodemask, &ac, &gfp, &alloc_flags))
 		goto out;
-	gfp = alloc_gfp;
 
 	/* Find an allowed local zone that meets the low watermark. */
 	z = ac.preferred_zoneref;
@@ -5184,6 +5226,34 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid,
 }
 EXPORT_SYMBOL_GPL(alloc_pages_bulk_noprof);
 
+/*
+ * free_pages_bulk - Free an array of order-0 pages
+ * @page_array: Array of pages to free
+ * @nr_pages: The number of pages in the array
+ *
+ * Free the order-0 pages. Adjacent entries whose PFNs form a contiguous
+ * run are released with a single __free_contig_range() call.
+ *
+ * This assumes page_array is sorted in ascending PFN order. Without that,
+ * the function still frees all pages, but contiguous runs may not be
+ * detected and the freeing pattern can degrade to freeing one page at a
+ * time.
+ *
+ * Context: Sleepable process context only; calls cond_resched()
+ */
+void free_pages_bulk(struct page **page_array, unsigned long nr_pages)
+{
+	while (nr_pages) {
+		unsigned long nr_contig = num_pages_contiguous(page_array, nr_pages);
+
+		__free_contig_range(page_to_pfn(*page_array), nr_contig);
+
+		nr_pages -= nr_contig;
+		page_array += nr_contig;
+		cond_resched();
+	}
+}
+
 /*
  * This is the 'heart' of the zoned buddy allocator.
  */
@@ -6763,6 +6833,105 @@ void __init page_alloc_sysctl_init(void)
 	register_sysctl_init("vm", page_alloc_sysctl_table);
 }
 
+static void free_prepared_contig_range(struct page *page,
+		unsigned long nr_pages)
+{
+	unsigned long pfn = page_to_pfn(page);
+
+	while (nr_pages) {
+		unsigned int order;
+
+		/* We are limited by the largest buddy order. */
+		order = pfn ? __ffs(pfn) : MAX_PAGE_ORDER;
+		/* Don't exceed the number of pages to free. */
+		order = min_t(unsigned int, order, ilog2(nr_pages));
+		order = min_t(unsigned int, order, MAX_PAGE_ORDER);
+
+		/*
+		 * Free the chunk as a single block. Our caller has already
+		 * called free_pages_prepare() for each order-0 page.
+		 */
+		__free_frozen_pages(page, order, FPI_PREPARED);
+
+		pfn += 1UL << order;
+		page += 1UL << order;
+		nr_pages -= 1UL << order;
+	}
+}
+
+static void __free_contig_range_common(unsigned long pfn, unsigned long nr_pages,
+		bool is_frozen)
+{
+	struct page *page, *start = NULL;
+	unsigned long nr_start = 0;
+	unsigned long start_sec;
+	unsigned long i;
+
+	for (i = 0; i < nr_pages; i++) {
+		bool can_free = true;
+
+		/*
+		 * Contiguous PFNs might not have contiguous "struct pages"
+		 * in some kernel configs: page++ across a section boundary
+		 * is undefined. Use pfn_to_page() for each PFN.
+		 */
+		page = pfn_to_page(pfn + i);
+
+		VM_WARN_ON_ONCE(PageHead(page));
+		VM_WARN_ON_ONCE(PageTail(page));
+
+		if (!is_frozen)
+			can_free = put_page_testzero(page);
+
+		if (can_free)
+			can_free = free_pages_prepare(page, 0);
+
+		if (!can_free) {
+			if (start) {
+				free_prepared_contig_range(start, i - nr_start);
+				start = NULL;
+			}
+			continue;
+		}
+
+		if (start && memdesc_section(page->flags) != start_sec) {
+			free_prepared_contig_range(start, i - nr_start);
+			start = page;
+			nr_start = i;
+			start_sec = memdesc_section(page->flags);
+		} else if (!start) {
+			start = page;
+			nr_start = i;
+			start_sec = memdesc_section(page->flags);
+		}
+	}
+
+	if (start)
+		free_prepared_contig_range(start, nr_pages - nr_start);
+}
+
+/**
+ * __free_contig_range - Free contiguous range of order-0 pages.
+ * @pfn: Page frame number of the first page in the range.
+ * @nr_pages: Number of pages to free.
+ *
+ * For each order-0 struct page in the physically contiguous range, put a
+ * reference. Free any page who's reference count falls to zero. The
+ * implementation is functionally equivalent to, but significantly faster than
+ * calling __free_page() for each struct page in a loop.
+ *
+ * Memory allocated with alloc_pages(order>=1) then subsequently split to
+ * order-0 with split_page() is an example of appropriate contiguous pages that
+ * can be freed with this API.
+ *
+ * Context: May be called in interrupt context or while holding a normal
+ * spinlock, but not in NMI context or while holding a raw spinlock.
+ */
+void __free_contig_range(unsigned long pfn, unsigned long nr_pages)
+{
+	__free_contig_range_common(pfn, nr_pages, /* is_frozen= */ false);
+}
+
 #ifdef CONFIG_CONTIG_ALLOC
 /* Usage: See admin-guide/dynamic-debug-howto.rst */
 static void alloc_contig_dump_pages(struct list_head *page_list)
@@ -6900,8 +7069,7 @@ static int __alloc_contig_verify_gfp_mask(gfp_t gfp_mask, gfp_t *gfp_cc_mask)
 
 static void __free_contig_frozen_range(unsigned long pfn, unsigned long nr_pages)
 {
-	for (; nr_pages--; pfn++)
-		free_frozen_pages(pfn_to_page(pfn), 0);
+	__free_contig_range_common(pfn, nr_pages, /* is_frozen= */ true);
 }
 
 /**
@@ -7309,8 +7477,7 @@ void free_contig_range(unsigned long pfn, unsigned long nr_pages)
 	if (WARN_ON_ONCE(PageHead(pfn_to_page(pfn))))
 		return;
 
-	for (; nr_pages--; pfn++)
-		__free_page(pfn_to_page(pfn));
+	__free_contig_range(pfn, nr_pages);
 }
 EXPORT_SYMBOL(free_contig_range);
 #endif /* CONFIG_CONTIG_ALLOC */
@@ -7368,7 +7535,7 @@ void zone_pcp_reset(struct zone *zone)
 unsigned long __offline_isolated_pages(unsigned long start_pfn,
 		unsigned long end_pfn)
 {
-	unsigned long already_offline = 0, flags;
+	unsigned long already_offline = 0;
 	unsigned long pfn = start_pfn;
 	struct page *page;
 	struct zone *zone;
@@ -7376,7 +7543,7 @@ unsigned long __offline_isolated_pages(unsigned long start_pfn,
 
 	offline_mem_sections(pfn, end_pfn);
 	zone = page_zone(pfn_to_page(pfn));
-	spin_lock_irqsave(&zone->lock, flags);
+	guard(spinlock_irqsave)(&zone->lock);
 	while (pfn < end_pfn) {
 		page = pfn_to_page(pfn);
 		/*
@@ -7406,7 +7573,6 @@ unsigned long __offline_isolated_pages(unsigned long start_pfn,
 		del_page_from_free_list(page, zone, order, MIGRATE_ISOLATE);
 		pfn += (1 << order);
 	}
-	spin_unlock_irqrestore(&zone->lock, flags);
 
 	return end_pfn - start_pfn - already_offline;
 }
@@ -7478,11 +7644,9 @@ bool take_page_off_buddy(struct page *page)
 {
 	struct zone *zone = page_zone(page);
 	unsigned long pfn = page_to_pfn(page);
-	unsigned long flags;
 	unsigned int order;
-	bool ret = false;
 
-	spin_lock_irqsave(&zone->lock, flags);
+	guard(spinlock_irqsave)(&zone->lock);
 	for (order = 0; order < NR_PAGE_ORDERS; order++) {
 		struct page *page_head = page - (pfn & ((1 << order) - 1));
 		int page_order = buddy_order(page_head);
@@ -7497,14 +7661,12 @@ bool take_page_off_buddy(struct page *page)
 			break_down_buddy_pages(zone, page_head, page, 0,
 						page_order, migratetype);
 			SetPageHWPoisonTakenOff(page);
-			ret = true;
-			break;
+			return true;
 		}
 		if (page_count(page_head) > 0)
 			break;
 	}
-	spin_unlock_irqrestore(&zone->lock, flags);
-	return ret;
+	return false;
 }
 
 /*
@@ -7513,23 +7675,19 @@ bool take_page_off_buddy(struct page *page)
 bool put_page_back_buddy(struct page *page)
 {
 	struct zone *zone = page_zone(page);
-	unsigned long flags;
-	bool ret = false;
 
-	spin_lock_irqsave(&zone->lock, flags);
+	guard(spinlock_irqsave)(&zone->lock);
 	if (put_page_testzero(page)) {
 		unsigned long pfn = page_to_pfn(page);
 		int migratetype = get_pfnblock_migratetype(page, pfn);
 
 		ClearPageHWPoisonTakenOff(page);
 		__free_one_page(page, pfn, zone, 0, migratetype, FPI_NONE);
-		if (TestClearPageHWPoison(page)) {
-			ret = true;
-		}
+		if (TestClearPageHWPoison(page))
+			return true;
 	}
-	spin_unlock_irqrestore(&zone->lock, flags);
 
-	return ret;
+	return false;
 }
 #endif
 
@@ -7779,8 +7937,8 @@ struct page *alloc_frozen_pages_nolock_noprof(gfp_t gfp_flags, int nid, unsigned
  * @order: allocation order size
  *
  * Allocates pages of a given order from the given node. This is safe to
- * call from any context (from atomic, NMI, and also reentrant
- * allocator -> tracepoint -> alloc_pages_nolock_noprof).
+ * call from any context where RCU is watching (from atomic, NMI, and also
+ * reentrant allocator -> tracepoint -> alloc_pages_nolock_noprof).
  * Allocation is best effort and to be expected to fail easily so nobody should
  * rely on the success. Failures are not reported via warn_alloc().
  * See always fail conditions below.
diff --git a/mm/page_io.c b/mm/page_io.c
index 70cea9e24d2fd..7ed76592e20d8 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -326,8 +326,8 @@ static void bio_associate_blkg_from_page(struct bio *bio, struct folio *folio)
 
 struct swap_iocb {
 	struct kiocb		iocb;
-	struct bio_vec		bvec[SWAP_CLUSTER_MAX];
-	int			pages;
+	struct bio_vec		bvecs[SWAP_CLUSTER_MAX];
+	int			nr_bvecs;
 	int			len;
 };
 static mempool_t *sio_pool;
@@ -348,7 +348,7 @@ int sio_pool_init(void)
 static void sio_write_complete(struct kiocb *iocb, long ret)
 {
 	struct swap_iocb *sio = container_of(iocb, struct swap_iocb, iocb);
-	struct page *page = sio->bvec[0].bv_page;
+	struct page *page = sio->bvecs[0].bv_page;
 	int p;
 
 	if (ret != sio->len) {
@@ -362,15 +362,15 @@ static void sio_write_complete(struct kiocb *iocb, long ret)
 		 */
 		pr_err_ratelimited("Write error %ld on dio swapfile (%llu)\n",
 				   ret, swap_dev_pos(page_swap_entry(page)));
-		for (p = 0; p < sio->pages; p++) {
-			page = sio->bvec[p].bv_page;
+		for (p = 0; p < sio->nr_bvecs; p++) {
+			page = sio->bvecs[p].bv_page;
 			set_page_dirty(page);
 			ClearPageReclaim(page);
 		}
 	}
 
-	for (p = 0; p < sio->pages; p++)
-		end_page_writeback(sio->bvec[p].bv_page);
+	for (p = 0; p < sio->nr_bvecs; p++)
+		end_page_writeback(sio->bvecs[p].bv_page);
 
 	mempool_free(sio, sio_pool);
 }
@@ -397,13 +397,13 @@ static void swap_writepage_fs(struct folio *folio, struct swap_iocb **swap_plug)
 		init_sync_kiocb(&sio->iocb, swap_file);
 		sio->iocb.ki_complete = sio_write_complete;
 		sio->iocb.ki_pos = pos;
-		sio->pages = 0;
+		sio->nr_bvecs = 0;
 		sio->len = 0;
 	}
-	bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);
+	bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
 	sio->len += folio_size(folio);
-	sio->pages += 1;
-	if (sio->pages == ARRAY_SIZE(sio->bvec) || !swap_plug) {
+	sio->nr_bvecs += 1;
+	if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !swap_plug) {
 		swap_write_unplug(sio);
 		sio = NULL;
 	}
@@ -477,7 +477,7 @@ void swap_write_unplug(struct swap_iocb *sio)
 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
 	int ret;
 
-	iov_iter_bvec(&from, ITER_SOURCE, sio->bvec, sio->pages, sio->len);
+	iov_iter_bvec(&from, ITER_SOURCE, sio->bvecs, sio->nr_bvecs, sio->len);
 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
 	if (ret != -EIOCBQUEUED)
 		sio_write_complete(&sio->iocb, ret);
@@ -489,8 +489,8 @@ static void sio_read_complete(struct kiocb *iocb, long ret)
 	int p;
 
 	if (ret == sio->len) {
-		for (p = 0; p < sio->pages; p++) {
-			struct folio *folio = page_folio(sio->bvec[p].bv_page);
+		for (p = 0; p < sio->nr_bvecs; p++) {
+			struct folio *folio = page_folio(sio->bvecs[p].bv_page);
 
 			count_mthp_stat(folio_order(folio), MTHP_STAT_SWPIN);
 			count_memcg_folio_events(folio, PSWPIN, folio_nr_pages(folio));
@@ -499,8 +499,8 @@ static void sio_read_complete(struct kiocb *iocb, long ret)
 		}
 		count_vm_events(PSWPIN, sio->len >> PAGE_SHIFT);
 	} else {
-		for (p = 0; p < sio->pages; p++) {
-			struct folio *folio = page_folio(sio->bvec[p].bv_page);
+		for (p = 0; p < sio->nr_bvecs; p++) {
+			struct folio *folio = page_folio(sio->bvecs[p].bv_page);
 
 			folio_unlock(folio);
 		}
@@ -559,13 +559,13 @@ static void swap_read_folio_fs(struct folio *folio, struct swap_iocb **plug)
 		init_sync_kiocb(&sio->iocb, sis->swap_file);
 		sio->iocb.ki_pos = pos;
 		sio->iocb.ki_complete = sio_read_complete;
-		sio->pages = 0;
+		sio->nr_bvecs = 0;
 		sio->len = 0;
 	}
-	bvec_set_folio(&sio->bvec[sio->pages], folio, folio_size(folio), 0);
+	bvec_set_folio(&sio->bvecs[sio->nr_bvecs], folio, folio_size(folio), 0);
 	sio->len += folio_size(folio);
-	sio->pages += 1;
-	if (sio->pages == ARRAY_SIZE(sio->bvec) || !plug) {
+	sio->nr_bvecs += 1;
+	if (sio->nr_bvecs == ARRAY_SIZE(sio->bvecs) || !plug) {
 		swap_read_unplug(sio);
 		sio = NULL;
 	}
@@ -666,7 +666,7 @@ void __swap_read_unplug(struct swap_iocb *sio)
 	struct address_space *mapping = sio->iocb.ki_filp->f_mapping;
 	int ret;
 
-	iov_iter_bvec(&from, ITER_DEST, sio->bvec, sio->pages, sio->len);
+	iov_iter_bvec(&from, ITER_DEST, sio->bvecs, sio->nr_bvecs, sio->len);
 	ret = mapping->a_ops->swap_rw(&sio->iocb, &from);
 	if (ret != -EIOCBQUEUED)
 		sio_read_complete(&sio->iocb, ret);
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index c48ff5c002449..7a9d631945a34 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -167,48 +167,40 @@ static int set_migratetype_isolate(struct page *page, enum pb_isolate_mode mode,
 {
 	struct zone *zone = page_zone(page);
 	struct page *unmovable;
-	unsigned long flags;
 	unsigned long check_unmovable_start, check_unmovable_end;
 
 	if (PageUnaccepted(page))
 		accept_page(page);
 
-	spin_lock_irqsave(&zone->lock, flags);
-
-	/*
-	 * We assume the caller intended to SET migrate type to isolate.
-	 * If it is already set, then someone else must have raced and
-	 * set it before us.
-	 */
-	if (is_migrate_isolate_page(page)) {
-		spin_unlock_irqrestore(&zone->lock, flags);
-		return -EBUSY;
-	}
-
-	/*
-	 * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
-	 * We just check MOVABLE pages.
-	 *
-	 * Pass the intersection of [start_pfn, end_pfn) and the page's pageblock
-	 * to avoid redundant checks.
-	 */
-	check_unmovable_start = max(page_to_pfn(page), start_pfn);
-	check_unmovable_end = min(pageblock_end_pfn(page_to_pfn(page)),
-				  end_pfn);
-
-	unmovable = has_unmovable_pages(check_unmovable_start, check_unmovable_end,
-			mode);
-	if (!unmovable) {
-		if (!pageblock_isolate_and_move_free_pages(zone, page)) {
-			spin_unlock_irqrestore(&zone->lock, flags);
+	scoped_guard(spinlock_irqsave, &zone->lock) {
+		/*
+		 * We assume the caller intended to SET migrate type to
+		 * isolate. If it is already set, then someone else must have
+		 * raced and set it before us.
+		 */
+		if (is_migrate_isolate_page(page))
 			return -EBUSY;
+
+		/*
+		 * FIXME: Now, memory hotplug doesn't call shrink_slab() by
+		 * itself. We just check MOVABLE pages.
+		 *
+		 * Pass the intersection of [start_pfn, end_pfn) and the page's
+		 * pageblock to avoid redundant checks.
+		 */
+		check_unmovable_start = max(page_to_pfn(page), start_pfn);
+		check_unmovable_end = min(pageblock_end_pfn(page_to_pfn(page)),
+					  end_pfn);
+
+		unmovable = has_unmovable_pages(check_unmovable_start,
+				check_unmovable_end, mode);
+		if (!unmovable) {
+			if (!pageblock_isolate_and_move_free_pages(zone, page))
+				return -EBUSY;
+			zone->nr_isolate_pageblock++;
+			return 0;
 		}
-		zone->nr_isolate_pageblock++;
-		spin_unlock_irqrestore(&zone->lock, flags);
-		return 0;
 	}
-
-	spin_unlock_irqrestore(&zone->lock, flags);
 	if (mode == PB_ISOLATE_MODE_MEM_OFFLINE) {
 		/*
 		 * printk() with zone->lock held will likely trigger a
@@ -223,15 +215,14 @@ static int set_migratetype_isolate(struct page *page, enum pb_isolate_mode mode,
 static void unset_migratetype_isolate(struct page *page)
 {
 	struct zone *zone;
-	unsigned long flags;
 	bool isolated_page = false;
 	unsigned int order;
 	struct page *buddy;
 
 	zone = page_zone(page);
-	spin_lock_irqsave(&zone->lock, flags);
+	guard(spinlock_irqsave)(&zone->lock);
 	if (!is_migrate_isolate_page(page))
-		goto out;
+		return;
 
 	/*
 	 * Because freepage with more than pageblock_order on isolated
@@ -279,8 +270,6 @@ static void unset_migratetype_isolate(struct page *page)
 		__putback_isolated_page(page, order, get_pageblock_migratetype(page));
 	}
 	zone->nr_isolate_pageblock--;
-out:
-	spin_unlock_irqrestore(&zone->lock, flags);
 }
 
 static inline struct page *
diff --git a/mm/page_owner.c b/mm/page_owner.c
index 8178e0be557f8..2dddcb6510aa1 100644
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -573,7 +573,7 @@ print_page_owner(char __user *buf, size_t count, unsigned long pfn,
 			migratetype_names[page_mt],
 			pfn >> pageblock_order,
 			migratetype_names[pageblock_mt],
-			&page->flags);
+			&page->flags.f);
 
 	ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0);
 	if (ret >= count)
diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h
index 4b3d6ec43703e..6ea2499b3a55f 100644
--- a/mm/percpu-internal.h
+++ b/mm/percpu-internal.h
@@ -6,6 +6,15 @@
 #include <linux/percpu.h>
 #include <linux/memcontrol.h>
 
+/*
+ * pcpu region represents a hint region.
+ * A region is only valid when @size is greater than 0.
+ */
+struct pcpu_region {
+	int start;
+	int size;
+};
+
 /*
  * pcpu_block_md is the metadata block struct.
  * Each chunk's bitmap is split into a number of full blocks.
@@ -13,17 +22,14 @@
  *
  * The scan hint is the largest known contiguous area before the contig hint.
  * It is not necessarily the actual largest contig hint though.  There is an
- * invariant that the scan_hint_start > contig_hint_start iff
- * scan_hint == contig_hint.  This is necessary because when scanning forward,
- * we don't know if a new contig hint would be better than the current one.
+ * invariant that the scan_hint.start > contig_hint.start iff
+ * scan_hint.size == contig_hint.size.  This is necessary because when scanning
+ * forward, we don't know if a new contig hint would be better than the current
+ * one.
  */
 struct pcpu_block_md {
-	int			scan_hint;	/* scan hint for block */
-	int			scan_hint_start; /* block relative starting
-						    position of the scan hint */
-	int                     contig_hint;    /* contig hint for block */
-	int                     contig_hint_start; /* block relative starting
-						      position of the contig hint */
+	struct pcpu_region	scan_hint;	/* scan hint for block */
+	struct pcpu_region	contig_hint;	/* contig hint for block */
 	int                     left_free;      /* size of free space along
 						   the left side of the block */
 	int                     right_free;     /* size of free space along
@@ -77,13 +83,13 @@ struct pcpu_chunk {
 	int			end_offset;	/* additional area required to
 						   have the region end page
 						   aligned */
+	int			nr_pages;	/* # of pages served by this chunk */
+	int			nr_populated;	/* # of populated pages */
+	int                     nr_empty_pop_pages; /* # of empty populated pages */
 #ifdef NEED_PCPUOBJ_EXT
 	struct pcpuobj_ext	*obj_exts;	/* vector of object cgroups */
 #endif
 
-	int			nr_pages;	/* # of pages served by this chunk */
-	int			nr_populated;	/* # of populated pages */
-	int                     nr_empty_pop_pages; /* # of empty populated pages */
 	unsigned long		populated[];	/* populated bitmap */
 };
 
diff --git a/mm/percpu-stats.c b/mm/percpu-stats.c
index 9b9d5d6accaed..4a735d09adefc 100644
--- a/mm/percpu-stats.c
+++ b/mm/percpu-stats.c
@@ -122,7 +122,7 @@ static void chunk_map_stats(struct seq_file *m, struct pcpu_chunk *chunk,
 	P("empty_pop_pages", chunk->nr_empty_pop_pages);
 	P("first_bit", chunk_md->first_free);
 	P("free_bytes", chunk->free_bytes);
-	P("contig_bytes", chunk_md->contig_hint * PCPU_MIN_ALLOC_SIZE);
+	P("contig_bytes", chunk_md->contig_hint.size * PCPU_MIN_ALLOC_SIZE);
 	P("sum_frag", sum_frag);
 	P("max_frag", max_frag);
 	P("cur_min_alloc", cur_min_alloc);
diff --git a/mm/percpu.c b/mm/percpu.c
index b0676b8054ed0..71a85d7245c7b 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -244,10 +244,11 @@ static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)
 	const struct pcpu_block_md *chunk_md = &chunk->chunk_md;
 
 	if (chunk->free_bytes < PCPU_MIN_ALLOC_SIZE ||
-	    chunk_md->contig_hint == 0)
+	    chunk_md->contig_hint.size == 0)
 		return 0;
 
-	return pcpu_size_to_slot(chunk_md->contig_hint * PCPU_MIN_ALLOC_SIZE);
+	return pcpu_size_to_slot(chunk_md->contig_hint.size *
+				 PCPU_MIN_ALLOC_SIZE);
 }
 
 /* set the pointer to a chunk in a page struct */
@@ -317,10 +318,10 @@ static unsigned long pcpu_block_off_to_off(int index, int off)
 static bool pcpu_check_block_hint(struct pcpu_block_md *block, int bits,
 				  size_t align)
 {
-	int bit_off = ALIGN(block->contig_hint_start, align) -
-		block->contig_hint_start;
+	int bit_off = ALIGN(block->contig_hint.start, align) -
+		block->contig_hint.start;
 
-	return bit_off + bits <= block->contig_hint;
+	return bit_off + bits <= block->contig_hint.size;
 }
 
 /*
@@ -330,7 +331,7 @@ static bool pcpu_check_block_hint(struct pcpu_block_md *block, int bits,
  *
  * This determines if we should scan based on the scan_hint or first_free.
  * In general, we want to scan from first_free to fulfill allocations by
- * first fit.  However, if we know a scan_hint at position scan_hint_start
+ * first fit.  However, if we know a scan_hint at position scan_hint.start
  * cannot fulfill an allocation, we can begin scanning from there knowing
  * the contig_hint will be our fallback.
  */
@@ -340,13 +341,13 @@ static int pcpu_next_hint(struct pcpu_block_md *block, int alloc_bits)
 	 * The three conditions below determine if we can skip past the
 	 * scan_hint.  First, does the scan hint exist.  Second, is the
 	 * contig_hint after the scan_hint (possibly not true iff
-	 * contig_hint == scan_hint).  Third, is the allocation request
-	 * larger than the scan_hint.
+	 * contig_hint.size == scan_hint.size).  Third, is the allocation
+	 * request larger than the scan_hint.
 	 */
-	if (block->scan_hint &&
-	    block->contig_hint_start > block->scan_hint_start &&
-	    alloc_bits > block->scan_hint)
-		return block->scan_hint_start + block->scan_hint;
+	if (block->scan_hint.size &&
+	    block->contig_hint.start > block->scan_hint.start &&
+	    alloc_bits > block->scan_hint.size)
+		return block->scan_hint.start + block->scan_hint.size;
 
 	return block->first_free;
 }
@@ -388,11 +389,11 @@ static void pcpu_next_md_free_region(struct pcpu_chunk *chunk, int *bit_off,
 		 * the next block and should be handled by the contig area
 		 * across blocks code.
 		 */
-		*bits = block->contig_hint;
-		if (*bits && block->contig_hint_start >= block_off &&
-		    *bits + block->contig_hint_start < PCPU_BITMAP_BLOCK_BITS) {
+		*bits = block->contig_hint.size;
+		if (*bits && block->contig_hint.start >= block_off &&
+		    *bits + block->contig_hint.start < PCPU_BITMAP_BLOCK_BITS) {
 			*bit_off = pcpu_block_off_to_off(i,
-					block->contig_hint_start);
+					block->contig_hint.start);
 			return;
 		}
 		/* reset to satisfy the second predicate above */
@@ -437,19 +438,18 @@ static void pcpu_next_fit_region(struct pcpu_chunk *chunk, int alloc_bits,
 		}
 
 		/* check block->contig_hint */
-		*bits = ALIGN(block->contig_hint_start, align) -
-			block->contig_hint_start;
+		*bits = ALIGN(block->contig_hint.start, align) -
+			block->contig_hint.start;
 		/*
 		 * This uses the block offset to determine if this has been
 		 * checked in the prior iteration.
 		 */
-		if (block->contig_hint &&
-		    block->contig_hint_start >= block_off &&
-		    block->contig_hint >= *bits + alloc_bits) {
+		if (block->contig_hint.size &&
+		    block->contig_hint.start >= block_off &&
+		    block->contig_hint.size >= *bits + alloc_bits) {
 			int start = pcpu_next_hint(block, alloc_bits);
 
-			*bits += alloc_bits + block->contig_hint_start -
-				 start;
+			*bits += alloc_bits + block->contig_hint.start - start;
 			*bit_off = pcpu_block_off_to_off(i, start);
 			return;
 		}
@@ -604,17 +604,30 @@ static inline void pcpu_update_empty_pages(struct pcpu_chunk *chunk, int nr)
 
 /*
  * pcpu_region_overlap - determines if two regions overlap
- * @a: start of first region, inclusive
- * @b: end of first region, exclusive
- * @x: start of second region, inclusive
- * @y: end of second region, exclusive
+ * @a: first region
+ * @b: second region
  *
- * This is used to determine if the hint region [a, b) overlaps with the
- * allocated region [x, y).
+ * This is used to determine if the hint region [a.start, a.start + a.size)
+ * overlaps with the allocated region [b.start, b.start + b.size).
  */
-static inline bool pcpu_region_overlap(int a, int b, int x, int y)
+static inline bool pcpu_region_overlap(struct pcpu_region a,
+				       struct pcpu_region b)
 {
-	return (a < y) && (x < b);
+	return (a.start < b.start + b.size) && (b.start < a.start + a.size);
+}
+
+/*
+ * pcpu_region_concat - determines if two regions meet on the border
+ * @a: first region
+ * @b: second region
+ *
+ * This is used to determine if the hint region [a.start, a.start + a.size)
+ * meets with the allocated region [b.start, b.start + b.size) on the border.
+ */
+static inline bool pcpu_region_concat(struct pcpu_region a,
+				      struct pcpu_region b)
+{
+	return (a.start == b.start + b.size) || (b.start == a.start + a.size);
 }
 
 /**
@@ -629,53 +642,108 @@ static inline bool pcpu_region_overlap(int a, int b, int x, int y)
  */
 static void pcpu_block_update(struct pcpu_block_md *block, int start, int end)
 {
-	int contig = end - start;
+	struct pcpu_region free = { .start = start, .size = end - start };
+	bool overlap_with_contig_hint =
+		block->contig_hint.size &&
+		(pcpu_region_overlap(block->contig_hint, free) ||
+		 pcpu_region_concat(block->contig_hint, free));
+
+	if (block->scan_hint.size &&
+	    (pcpu_region_overlap(block->scan_hint, free) ||
+	     pcpu_region_concat(block->scan_hint, free))) {
+		start = min(start, block->scan_hint.start);
+		end = max(end, block->scan_hint.start + block->scan_hint.size);
+		free = (struct pcpu_region){
+			.start = start,
+			.size = end - start,
+		};
+
+		block->scan_hint.size = 0;
+	}
+
+	if (overlap_with_contig_hint) {
+		start = min(start, block->contig_hint.start);
+		end = max(end,
+			  block->contig_hint.start + block->contig_hint.size);
+		free = (struct pcpu_region){
+			.start = start,
+			.size = end - start,
+		};
+
+		if (block->scan_hint.size &&
+		    free.size > block->scan_hint.size &&
+		    block->scan_hint.start > free.start)
+			block->scan_hint.size = 0;
+
+		block->contig_hint = free;
+	}
 
-	block->first_free = min(block->first_free, start);
-	if (start == 0)
-		block->left_free = contig;
+	block->first_free = min(block->first_free, free.start);
+	if (free.start == 0)
+		block->left_free = free.size;
 
-	if (end == block->nr_bits)
-		block->right_free = contig;
+	if (free.start + free.size == block->nr_bits)
+		block->right_free = free.size;
 
-	if (contig > block->contig_hint) {
+	if (overlap_with_contig_hint)
+		return;
+
+	/*
+	 * At this point, it is guaranteed that the new contig does neither
+	 * overlap with contig_hint nor with scan_hint.
+	 */
+
+	if (free.size > block->contig_hint.size) {
 		/* promote the old contig_hint to be the new scan_hint */
-		if (start > block->contig_hint_start) {
-			if (block->contig_hint > block->scan_hint) {
-				block->scan_hint_start =
-					block->contig_hint_start;
+		if (block->contig_hint.size &&
+		    free.start > block->contig_hint.start) {
+			if (block->contig_hint.size > block->scan_hint.size ||
+			    free.start < block->scan_hint.start)
 				block->scan_hint = block->contig_hint;
-			} else if (start < block->scan_hint_start) {
+		} else if (!block->contig_hint.size ||
+			   (block->scan_hint.size &&
+			    free.start < block->scan_hint.start)) {
+			block->scan_hint.size = 0;
+		}
+		block->contig_hint = free;
+	} else if (free.size == block->contig_hint.size) {
+		if (block->contig_hint.start &&
+		    (!free.start ||
+		     __ffs(free.start) > __ffs(block->contig_hint.start))) {
+			if (block->contig_hint.size > block->scan_hint.size) {
+				if (free.start < block->contig_hint.start)
+					block->scan_hint = block->contig_hint;
+			} else if (free.start > block->scan_hint.start) {
 				/*
-				 * The old contig_hint == scan_hint.  But, the
-				 * new contig is larger so hold the invariant
-				 * scan_hint_start < contig_hint_start.
+				 * old contig_hint.size == old scan_hint.size
+				 * == new contig size. But, the new contig is
+				 * farther than the old scan_hint so hold the
+				 * invariant scan_hint.start > contig_hint.start
+				 * iff scan_hint.size == contig_hint.size.
 				 */
-				block->scan_hint = 0;
+				block->scan_hint.size = 0;
 			}
+
+			/* new start has a better alignment so use it */
+			block->contig_hint.start = free.start;
 		} else {
-			block->scan_hint = 0;
-		}
-		block->contig_hint_start = start;
-		block->contig_hint = contig;
-	} else if (contig == block->contig_hint) {
-		if (block->contig_hint_start &&
-		    (!start ||
-		     __ffs(start) > __ffs(block->contig_hint_start))) {
-			/* start has a better alignment so use it */
-			block->contig_hint_start = start;
-			if (start < block->scan_hint_start &&
-			    block->contig_hint > block->scan_hint)
-				block->scan_hint = 0;
-		} else if (start > block->scan_hint_start ||
-			   block->contig_hint > block->scan_hint) {
-			/*
-			 * Knowing contig == contig_hint, update the scan_hint
-			 * if it is farther than or larger than the current
-			 * scan_hint.
-			 */
-			block->scan_hint_start = start;
-			block->scan_hint = contig;
+			if (block->contig_hint.size > block->scan_hint.size) {
+				if (free.start > block->contig_hint.start) {
+					block->scan_hint = free;
+				} else if (block->scan_hint.size &&
+					   free.start < block->scan_hint.start) {
+					/*
+					 * old scan_hint.size < new contig size
+					 * == old contig_hint.size. But, the new
+					 * contig is before the old scan_hint
+					 * so invalidate the scan_hint to
+					 * protect the contig_hint.
+					 */
+					block->scan_hint.size = 0;
+				}
+			} else if (free.start > block->scan_hint.start) {
+				block->scan_hint = free;
+			}
 		}
 	} else {
 		/*
@@ -683,12 +751,11 @@ static void pcpu_block_update(struct pcpu_block_md *block, int start, int end)
 		 * the scan_hint if it is larger than or equal and farther than
 		 * the current scan_hint.
 		 */
-		if ((start < block->contig_hint_start &&
-		     (contig > block->scan_hint ||
-		      (contig == block->scan_hint &&
-		       start > block->scan_hint_start)))) {
-			block->scan_hint_start = start;
-			block->scan_hint = contig;
+		if ((free.start < block->contig_hint.start &&
+		     (free.size > block->scan_hint.size ||
+		      (free.size == block->scan_hint.size &&
+		       free.start > block->scan_hint.start)))) {
+			block->scan_hint = free;
 		}
 	}
 }
@@ -738,8 +805,8 @@ static void pcpu_block_update_scan(struct pcpu_chunk *chunk, int bit_off,
  * Iterates over the metadata blocks to find the largest contig area.
  * A full scan can be avoided on the allocation path as this is triggered
  * if we broke the contig_hint.  In doing so, the scan_hint will be before
- * the contig_hint or after if the scan_hint == contig_hint.  This cannot
- * be prevented on freeing as we want to find the largest area possibly
+ * the contig_hint or after if the scan_hint.size == contig_hint.size.  This
+ * cannot be prevented on freeing as we want to find the largest area possibly
  * spanning blocks.
  */
 static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan)
@@ -748,14 +815,13 @@ static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk, bool full_scan)
 	int bit_off, bits;
 
 	/* promote scan_hint to contig_hint */
-	if (!full_scan && chunk_md->scan_hint) {
-		bit_off = chunk_md->scan_hint_start + chunk_md->scan_hint;
-		chunk_md->contig_hint_start = chunk_md->scan_hint_start;
+	if (!full_scan && chunk_md->scan_hint.size) {
+		bit_off = chunk_md->scan_hint.start + chunk_md->scan_hint.size;
 		chunk_md->contig_hint = chunk_md->scan_hint;
-		chunk_md->scan_hint = 0;
+		chunk_md->scan_hint.size = 0;
 	} else {
 		bit_off = chunk_md->first_free;
-		chunk_md->contig_hint = 0;
+		chunk_md->contig_hint.size = 0;
 	}
 
 	bits = 0;
@@ -778,14 +844,13 @@ static void pcpu_block_refresh_hint(struct pcpu_chunk *chunk, int index)
 	unsigned int start, end;	/* region start, region end */
 
 	/* promote scan_hint to contig_hint */
-	if (block->scan_hint) {
-		start = block->scan_hint_start + block->scan_hint;
-		block->contig_hint_start = block->scan_hint_start;
+	if (block->scan_hint.size) {
+		start = block->scan_hint.start + block->scan_hint.size;
 		block->contig_hint = block->scan_hint;
-		block->scan_hint = 0;
+		block->scan_hint.size = 0;
 	} else {
 		start = block->first_free;
-		block->contig_hint = 0;
+		block->contig_hint.size = 0;
 	}
 
 	block->right_free = 0;
@@ -831,7 +896,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
 	/*
 	 * Update s_block.
 	 */
-	if (s_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
+	if (s_block->contig_hint.size == PCPU_BITMAP_BLOCK_BITS)
 		nr_empty_pages++;
 
 	/*
@@ -845,17 +910,13 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
 					PCPU_BITMAP_BLOCK_BITS,
 					s_off + bits);
 
-	if (pcpu_region_overlap(s_block->scan_hint_start,
-				s_block->scan_hint_start + s_block->scan_hint,
-				s_off,
-				s_off + bits))
-		s_block->scan_hint = 0;
-
-	if (pcpu_region_overlap(s_block->contig_hint_start,
-				s_block->contig_hint_start +
-				s_block->contig_hint,
-				s_off,
-				s_off + bits)) {
+	struct pcpu_region s_region = { .start = s_off, .size = bits };
+
+	if (s_block->scan_hint.size &&
+	    pcpu_region_overlap(s_block->scan_hint, s_region))
+		s_block->scan_hint.size = 0;
+
+	if (pcpu_region_overlap(s_block->contig_hint, s_region)) {
 		/* block contig hint is broken - scan to fix it */
 		if (!s_off)
 			s_block->left_free = 0;
@@ -874,7 +935,7 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
 	 * Update e_block.
 	 */
 	if (s_index != e_index) {
-		if (e_block->contig_hint == PCPU_BITMAP_BLOCK_BITS)
+		if (e_block->contig_hint.size == PCPU_BITMAP_BLOCK_BITS)
 			nr_empty_pages++;
 
 		/*
@@ -889,11 +950,12 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
 			/* reset the block */
 			e_block++;
 		} else {
-			if (e_off > e_block->scan_hint_start)
-				e_block->scan_hint = 0;
+			if (e_block->scan_hint.size &&
+			    e_off > e_block->scan_hint.start)
+				e_block->scan_hint.size = 0;
 
 			e_block->left_free = 0;
-			if (e_off > e_block->contig_hint_start) {
+			if (e_off > e_block->contig_hint.start) {
 				/* contig hint is broken - scan to fix it */
 				pcpu_block_refresh_hint(chunk, e_index);
 			} else {
@@ -906,8 +968,8 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
 		/* update in-between md_blocks */
 		nr_empty_pages += (e_index - s_index - 1);
 		for (block = s_block + 1; block < e_block; block++) {
-			block->scan_hint = 0;
-			block->contig_hint = 0;
+			block->scan_hint.size = 0;
+			block->contig_hint.size = 0;
 			block->left_free = 0;
 			block->right_free = 0;
 		}
@@ -922,23 +984,18 @@ static void pcpu_block_update_hint_alloc(struct pcpu_chunk *chunk, int bit_off,
 	if (nr_empty_pages)
 		pcpu_update_empty_pages(chunk, -nr_empty_pages);
 
-	if (pcpu_region_overlap(chunk_md->scan_hint_start,
-				chunk_md->scan_hint_start +
-				chunk_md->scan_hint,
-				bit_off,
-				bit_off + bits))
-		chunk_md->scan_hint = 0;
+	struct pcpu_region req_region = { .start = bit_off, .size = bits };
+
+	if (chunk_md->scan_hint.size &&
+	    pcpu_region_overlap(chunk_md->scan_hint, req_region))
+		chunk_md->scan_hint.size = 0;
 
 	/*
 	 * The only time a full chunk scan is required is if the chunk
 	 * contig hint is broken.  Otherwise, it means a smaller space
 	 * was used and therefore the chunk contig hint is still correct.
 	 */
-	if (pcpu_region_overlap(chunk_md->contig_hint_start,
-				chunk_md->contig_hint_start +
-				chunk_md->contig_hint,
-				bit_off,
-				bit_off + bits))
+	if (pcpu_region_overlap(chunk_md->contig_hint, req_region))
 		pcpu_chunk_refresh_hint(chunk, false);
 }
 
@@ -994,8 +1051,8 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
 	 * or end of the block.
 	 */
 	start = s_off;
-	if (s_off == s_block->contig_hint + s_block->contig_hint_start) {
-		start = s_block->contig_hint_start;
+	if (s_off == s_block->contig_hint.size + s_block->contig_hint.start) {
+		start = s_block->contig_hint.start;
 	} else {
 		/*
 		 * Scan backwards to find the extent of the free area.
@@ -1009,8 +1066,8 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
 	}
 
 	end = e_off;
-	if (e_off == e_block->contig_hint_start)
-		end = e_block->contig_hint_start + e_block->contig_hint;
+	if (e_off == e_block->contig_hint.start)
+		end = e_block->contig_hint.start + e_block->contig_hint.size;
 	else
 		end = find_next_bit(pcpu_index_alloc_map(chunk, e_index),
 				    PCPU_BITMAP_BLOCK_BITS, end);
@@ -1032,9 +1089,11 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
 		nr_empty_pages += (e_index - s_index - 1);
 		for (block = s_block + 1; block < e_block; block++) {
 			block->first_free = 0;
-			block->scan_hint = 0;
-			block->contig_hint_start = 0;
-			block->contig_hint = PCPU_BITMAP_BLOCK_BITS;
+			block->scan_hint.size = 0;
+			block->contig_hint = (struct pcpu_region){
+				.start = 0,
+				.size = PCPU_BITMAP_BLOCK_BITS,
+			};
 			block->left_free = PCPU_BITMAP_BLOCK_BITS;
 			block->right_free = PCPU_BITMAP_BLOCK_BITS;
 		}
@@ -1054,7 +1113,7 @@ static void pcpu_block_update_hint_free(struct pcpu_chunk *chunk, int bit_off,
 	else
 		pcpu_block_update(&chunk->chunk_md,
 				  pcpu_block_off_to_off(s_index, start),
-				  end);
+				  pcpu_block_off_to_off(e_index, end));
 }
 
 /**
@@ -1314,8 +1373,9 @@ static int pcpu_free_area(struct pcpu_chunk *chunk, int off)
 
 static void pcpu_init_md_block(struct pcpu_block_md *block, int nr_bits)
 {
-	block->scan_hint = 0;
-	block->contig_hint = nr_bits;
+	block->scan_hint = (struct pcpu_region){ .start = 0, .size = 0 };
+	block->contig_hint =
+		(struct pcpu_region){ .start = 0, .size = nr_bits };
 	block->left_free = nr_bits;
 	block->right_free = nr_bits;
 	block->first_free = 0;
@@ -2141,7 +2201,7 @@ static void pcpu_reclaim_populated(void)
 			 * (first) page in the chunk.
 			 */
 			block = chunk->md_blocks + i;
-			if (block->contig_hint == PCPU_BITMAP_BLOCK_BITS &&
+			if (block->contig_hint.size == PCPU_BITMAP_BLOCK_BITS &&
 			    test_bit(i, chunk->populated)) {
 				if (end == -1)
 					end = i;
diff --git a/mm/readahead.c b/mm/readahead.c
index 7b05082c89ea2..38ce16e3fcbdf 100644
--- a/mm/readahead.c
+++ b/mm/readahead.c
@@ -146,6 +146,17 @@ file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping)
 }
 EXPORT_SYMBOL_GPL(file_ra_state_init);
 
+/**
+ * read_pages() - Start IO for a contiguous range of allocated folios in the
+ *                page cache.
+ * @rac: Readahead control.
+ *
+ * When read_pages() returns, it is guaranteed that all of the folios will have
+ * been processed or removed so that ``readahead_count(rac) == 0``. However,
+ * that does not imply that ``readahead_index(rac)`` will be updated to point
+ * to the end of the originally requested range because, for example, the
+ * filesystem may expand the range upwards.
+ */
 static void read_pages(struct readahead_control *rac)
 {
 	const struct address_space_operations *aops = rac->mapping->a_ops;
@@ -270,7 +281,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl,
 			 */
 			read_pages(ractl);
 			ractl->_index += min_nrpages;
-			i = ractl->_index + ractl->_nr_pages - index;
+			i = ractl->_index - index;
 			continue;
 		}
 
@@ -286,7 +297,7 @@ void page_cache_ra_unbounded(struct readahead_control *ractl,
 				break;
 			read_pages(ractl);
 			ractl->_index += min_nrpages;
-			i = ractl->_index + ractl->_nr_pages - index;
+			i = ractl->_index - index;
 			continue;
 		}
 		if (i == mark)
@@ -324,11 +335,16 @@ static void do_page_cache_ra(struct readahead_control *ractl,
 		return;
 
 	end_index = (isize - 1) >> PAGE_SHIFT;
+	if (end_index > ractl->_max_index)
+		end_index = ractl->_max_index;
 	if (index > end_index)
 		return;
 	/* Don't read past the page containing the last byte of the file */
-	if (nr_to_read > end_index - index)
+	if (nr_to_read > end_index - index) {
 		nr_to_read = end_index - index + 1;
+		/* We've reached the end, so don't set a readahead marker. */
+		lookahead_size = 0;
+	}
 
 	filemap_invalidate_lock_shared(mapping);
 	page_cache_ra_unbounded(ractl, nr_to_read, lookahead_size);
@@ -471,8 +487,8 @@ void page_cache_ra_order(struct readahead_control *ractl,
 	pgoff_t start = readahead_index(ractl);
 	pgoff_t index = start;
 	unsigned int min_order = mapping_min_folio_order(mapping);
-	pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;
-	pgoff_t mark = index + ra->size - ra->async_size;
+	pgoff_t limit;
+	pgoff_t mark;
 	unsigned int nofs;
 	int err = 0;
 	gfp_t gfp = readahead_gfp_mask(mapping);
@@ -484,7 +500,15 @@ void page_cache_ra_order(struct readahead_control *ractl,
 		goto fallback;
 	}
 
-	limit = min(limit, index + ra->size - 1);
+	limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;
+	limit = min(limit, ractl->_max_index);
+	if (limit > index + ra->size - 1) {
+		limit = index + ra->size - 1;
+		mark = index + ra->size - ra->async_size;
+	} else {
+		/* We've reached the end, so don't set a readahead marker. */
+		mark = ULONG_MAX;
+	}
 
 	new_order = min(mapping_max_folio_order(mapping), new_order);
 	new_order = min_t(unsigned int, new_order, ilog2(ra->size));
diff --git a/mm/rmap.c b/mm/rmap.c
index 78b7fb5f367ce..1c77d5dc06e9f 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -571,7 +571,7 @@ void __init anon_vma_init(void)
  * In case it was remapped to a different anon_vma, the new anon_vma will be a
  * child of the old anon_vma, and the anon_vma lifetime rules will therefore
  * ensure that any anon_vma obtained from the page will still be valid for as
- * long as we observe page_mapped() [ hence all those page_mapped() tests ].
+ * long as we observe folio_mapped() [ hence all those folio_mapped() tests ].
  *
  * All users of this function must be very careful when walking the anon_vma
  * chain and verify that the page in question is indeed mapped in it
@@ -1999,7 +1999,7 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 	/*
 	 * When racing against e.g. zap_pte_range() on another cpu,
 	 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
-	 * try_to_unmap() may return before page_mapped() has become false,
+	 * try_to_unmap() may return before folio_mapped() has become false,
 	 * if page table locking is skipped: use TTU_SYNC to wait for that.
 	 */
 	if (flags & TTU_SYNC)
@@ -2030,6 +2030,8 @@ static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
 	mmu_notifier_invalidate_range_start(&range);
 
 	while (page_vma_mapped_walk(&pvmw)) {
+		nr_pages = 1;
+
 		/*
 		 * If the folio is in an mlock()d vma, we must not swap it out.
 		 */
@@ -2426,7 +2428,7 @@ static bool try_to_migrate_one(struct folio *folio, struct vm_area_struct *vma,
 	/*
 	 * When racing against e.g. zap_pte_range() on another cpu,
 	 * in between its ptep_get_and_clear_full() and folio_remove_rmap_*(),
-	 * try_to_migrate() may return before page_mapped() has become false,
+	 * try_to_migrate() may return before folio_mapped() has become false,
 	 * if page table locking is skipped: use TTU_SYNC to wait for that.
 	 */
 	if (flags & TTU_SYNC)
@@ -2927,7 +2929,7 @@ static struct anon_vma *rmap_walk_anon_lock(const struct folio *folio,
 
 	/*
 	 * Note: remove_migration_ptes() cannot use folio_lock_anon_vma_read()
-	 * because that depends on page_mapped(); but not all its usages
+	 * because that depends on folio_mapped(); but not all its usages
 	 * are holding mmap_lock. Users without mmap_lock are required to
 	 * take a reference count to prevent the anon_vma disappearing
 	 */
diff --git a/mm/shmem.c b/mm/shmem.c
index 3b5dc21b323c2..bab3529af23c5 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3101,10 +3101,7 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap,
 	cache_no_acl(inode);
 	if (sbinfo->noswap)
 		mapping_set_unevictable(inode->i_mapping);
-
-	/* Don't consider 'deny' for emergencies and 'force' for testing */
-	if (sbinfo->huge)
-		mapping_set_large_folios(inode->i_mapping);
+	mapping_set_large_folios(inode->i_mapping);
 
 	switch (mode & S_IFMT) {
 	default:
diff --git a/mm/shrinker.c b/mm/shrinker.c
index 76b3f750cf655..7082d01c8c9d0 100644
--- a/mm/shrinker.c
+++ b/mm/shrinker.c
@@ -197,12 +197,13 @@ void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
 {
 	if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
 		struct shrinker_info *info;
-		struct shrinker_info_unit *unit;
 
 		rcu_read_lock();
 		info = rcu_dereference(memcg->nodeinfo[nid]->shrinker_info);
-		unit = info->unit[shrinker_id_to_index(shrinker_id)];
 		if (!WARN_ON_ONCE(shrinker_id >= info->map_nr_max)) {
+			struct shrinker_info_unit *unit;
+
+			unit = info->unit[shrinker_id_to_index(shrinker_id)];
 			/* Pairs with smp mb in shrink_slab() */
 			smp_mb__before_atomic();
 			set_bit(shrinker_id_to_offset(shrinker_id), unit->map);
@@ -215,29 +216,26 @@ static DEFINE_IDR(shrinker_idr);
 
 static int shrinker_memcg_alloc(struct shrinker *shrinker)
 {
-	int id, ret = -ENOMEM;
+	int id;
 
 	if (mem_cgroup_disabled())
 		return -ENOSYS;
 	if (mem_cgroup_kmem_disabled() && !(shrinker->flags & SHRINKER_NONSLAB))
 		return -ENOSYS;
 
-	mutex_lock(&shrinker_mutex);
+	guard(mutex)(&shrinker_mutex);
 	id = idr_alloc(&shrinker_idr, shrinker, 0, 0, GFP_KERNEL);
 	if (id < 0)
-		goto unlock;
+		return id;
 
 	if (id >= shrinker_nr_max) {
 		if (expand_shrinker_info(id)) {
 			idr_remove(&shrinker_idr, id);
-			goto unlock;
+			return -ENOMEM;
 		}
 	}
 	shrinker->id = id;
-	ret = 0;
-unlock:
-	mutex_unlock(&shrinker_mutex);
-	return ret;
+	return 0;
 }
 
 static void shrinker_memcg_remove(struct shrinker *shrinker)
diff --git a/mm/slab.h b/mm/slab.h
index bf2f87acf5e3a..1bf9c3021ae3d 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -362,12 +362,12 @@ static inline unsigned int size_index_elem(unsigned int bytes)
  * KMALLOC_MAX_CACHE_SIZE and the caller must check that.
  */
 static inline struct kmem_cache *
-kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, unsigned long caller)
+kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, kmalloc_token_t token)
 {
 	unsigned int index;
 
 	if (!b)
-		b = &kmalloc_caches[kmalloc_type(flags, caller)];
+		b = &kmalloc_caches[kmalloc_type(flags, token)];
 	if (size <= 192)
 		index = kmalloc_size_index[size_index_elem(size)];
 	else
diff --git a/mm/slab_common.c b/mm/slab_common.c
index d5a70a831a2a5..b6426d7ceec92 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -742,7 +742,7 @@ kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES] __ro_after_init =
 { /* initialization for https://llvm.org/pr42570 */ };
 EXPORT_SYMBOL(kmalloc_caches);
 
-#ifdef CONFIG_RANDOM_KMALLOC_CACHES
+#ifdef CONFIG_KMALLOC_PARTITION_RANDOM
 unsigned long random_kmalloc_seed __ro_after_init;
 EXPORT_SYMBOL(random_kmalloc_seed);
 #endif
@@ -787,7 +787,7 @@ size_t kmalloc_size_roundup(size_t size)
 		 * The flags don't matter since size_index is common to all.
 		 * Neither does the caller for just getting ->object_size.
 		 */
-		return kmalloc_slab(size, NULL, GFP_KERNEL, 0)->object_size;
+		return kmalloc_slab(size, NULL, GFP_KERNEL, __kmalloc_token(0))->object_size;
 	}
 
 	/* Above the smaller buckets, size is a multiple of page size. */
@@ -821,26 +821,26 @@ EXPORT_SYMBOL(kmalloc_size_roundup);
 #define KMALLOC_RCL_NAME(sz)
 #endif
 
-#ifdef CONFIG_RANDOM_KMALLOC_CACHES
-#define __KMALLOC_RANDOM_CONCAT(a, b) a ## b
-#define KMALLOC_RANDOM_NAME(N, sz) __KMALLOC_RANDOM_CONCAT(KMA_RAND_, N)(sz)
-#define KMA_RAND_1(sz)                  .name[KMALLOC_RANDOM_START +  1] = "kmalloc-rnd-01-" #sz,
-#define KMA_RAND_2(sz)  KMA_RAND_1(sz)  .name[KMALLOC_RANDOM_START +  2] = "kmalloc-rnd-02-" #sz,
-#define KMA_RAND_3(sz)  KMA_RAND_2(sz)  .name[KMALLOC_RANDOM_START +  3] = "kmalloc-rnd-03-" #sz,
-#define KMA_RAND_4(sz)  KMA_RAND_3(sz)  .name[KMALLOC_RANDOM_START +  4] = "kmalloc-rnd-04-" #sz,
-#define KMA_RAND_5(sz)  KMA_RAND_4(sz)  .name[KMALLOC_RANDOM_START +  5] = "kmalloc-rnd-05-" #sz,
-#define KMA_RAND_6(sz)  KMA_RAND_5(sz)  .name[KMALLOC_RANDOM_START +  6] = "kmalloc-rnd-06-" #sz,
-#define KMA_RAND_7(sz)  KMA_RAND_6(sz)  .name[KMALLOC_RANDOM_START +  7] = "kmalloc-rnd-07-" #sz,
-#define KMA_RAND_8(sz)  KMA_RAND_7(sz)  .name[KMALLOC_RANDOM_START +  8] = "kmalloc-rnd-08-" #sz,
-#define KMA_RAND_9(sz)  KMA_RAND_8(sz)  .name[KMALLOC_RANDOM_START +  9] = "kmalloc-rnd-09-" #sz,
-#define KMA_RAND_10(sz) KMA_RAND_9(sz)  .name[KMALLOC_RANDOM_START + 10] = "kmalloc-rnd-10-" #sz,
-#define KMA_RAND_11(sz) KMA_RAND_10(sz) .name[KMALLOC_RANDOM_START + 11] = "kmalloc-rnd-11-" #sz,
-#define KMA_RAND_12(sz) KMA_RAND_11(sz) .name[KMALLOC_RANDOM_START + 12] = "kmalloc-rnd-12-" #sz,
-#define KMA_RAND_13(sz) KMA_RAND_12(sz) .name[KMALLOC_RANDOM_START + 13] = "kmalloc-rnd-13-" #sz,
-#define KMA_RAND_14(sz) KMA_RAND_13(sz) .name[KMALLOC_RANDOM_START + 14] = "kmalloc-rnd-14-" #sz,
-#define KMA_RAND_15(sz) KMA_RAND_14(sz) .name[KMALLOC_RANDOM_START + 15] = "kmalloc-rnd-15-" #sz,
-#else // CONFIG_RANDOM_KMALLOC_CACHES
-#define KMALLOC_RANDOM_NAME(N, sz)
+#ifdef CONFIG_KMALLOC_PARTITION_CACHES
+#define __KMALLOC_PARTITION_CONCAT(a, b) a ## b
+#define KMALLOC_PARTITION_NAME(N, sz) __KMALLOC_PARTITION_CONCAT(KMA_PART_, N)(sz)
+#define KMA_PART_1(sz)                  .name[KMALLOC_PARTITION_START +  1] = "kmalloc-part-01-" #sz,
+#define KMA_PART_2(sz)  KMA_PART_1(sz)  .name[KMALLOC_PARTITION_START +  2] = "kmalloc-part-02-" #sz,
+#define KMA_PART_3(sz)  KMA_PART_2(sz)  .name[KMALLOC_PARTITION_START +  3] = "kmalloc-part-03-" #sz,
+#define KMA_PART_4(sz)  KMA_PART_3(sz)  .name[KMALLOC_PARTITION_START +  4] = "kmalloc-part-04-" #sz,
+#define KMA_PART_5(sz)  KMA_PART_4(sz)  .name[KMALLOC_PARTITION_START +  5] = "kmalloc-part-05-" #sz,
+#define KMA_PART_6(sz)  KMA_PART_5(sz)  .name[KMALLOC_PARTITION_START +  6] = "kmalloc-part-06-" #sz,
+#define KMA_PART_7(sz)  KMA_PART_6(sz)  .name[KMALLOC_PARTITION_START +  7] = "kmalloc-part-07-" #sz,
+#define KMA_PART_8(sz)  KMA_PART_7(sz)  .name[KMALLOC_PARTITION_START +  8] = "kmalloc-part-08-" #sz,
+#define KMA_PART_9(sz)  KMA_PART_8(sz)  .name[KMALLOC_PARTITION_START +  9] = "kmalloc-part-09-" #sz,
+#define KMA_PART_10(sz) KMA_PART_9(sz)  .name[KMALLOC_PARTITION_START + 10] = "kmalloc-part-10-" #sz,
+#define KMA_PART_11(sz) KMA_PART_10(sz) .name[KMALLOC_PARTITION_START + 11] = "kmalloc-part-11-" #sz,
+#define KMA_PART_12(sz) KMA_PART_11(sz) .name[KMALLOC_PARTITION_START + 12] = "kmalloc-part-12-" #sz,
+#define KMA_PART_13(sz) KMA_PART_12(sz) .name[KMALLOC_PARTITION_START + 13] = "kmalloc-part-13-" #sz,
+#define KMA_PART_14(sz) KMA_PART_13(sz) .name[KMALLOC_PARTITION_START + 14] = "kmalloc-part-14-" #sz,
+#define KMA_PART_15(sz) KMA_PART_14(sz) .name[KMALLOC_PARTITION_START + 15] = "kmalloc-part-15-" #sz,
+#else // CONFIG_KMALLOC_PARTITION_CACHES
+#define KMALLOC_PARTITION_NAME(N, sz)
 #endif
 
 #define INIT_KMALLOC_INFO(__size, __short_size)			\
@@ -849,7 +849,7 @@ EXPORT_SYMBOL(kmalloc_size_roundup);
 	KMALLOC_RCL_NAME(__short_size)				\
 	KMALLOC_CGROUP_NAME(__short_size)			\
 	KMALLOC_DMA_NAME(__short_size)				\
-	KMALLOC_RANDOM_NAME(RANDOM_KMALLOC_CACHES_NR, __short_size)	\
+	KMALLOC_PARTITION_NAME(KMALLOC_PARTITION_CACHES_NR, __short_size)	\
 	.size = __size,						\
 }
 
@@ -961,8 +961,8 @@ new_kmalloc_cache(int idx, enum kmalloc_cache_type type)
 		flags |= SLAB_CACHE_DMA;
 	}
 
-#ifdef CONFIG_RANDOM_KMALLOC_CACHES
-	if (type >= KMALLOC_RANDOM_START && type <= KMALLOC_RANDOM_END)
+#ifdef CONFIG_KMALLOC_PARTITION_CACHES
+	if (type >= KMALLOC_PARTITION_START && type <= KMALLOC_PARTITION_END)
 		flags |= SLAB_NO_MERGE;
 #endif
 
@@ -1010,7 +1010,7 @@ void __init create_kmalloc_caches(void)
 		for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
 			new_kmalloc_cache(i, type);
 	}
-#ifdef CONFIG_RANDOM_KMALLOC_CACHES
+#ifdef CONFIG_KMALLOC_PARTITION_RANDOM
 	random_kmalloc_seed = get_random_u64();
 #endif
 
@@ -2110,7 +2110,9 @@ EXPORT_SYMBOL_GPL(kvfree_rcu_barrier);
 void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
 {
 	if (cache_has_sheaves(s)) {
+		cpus_read_lock();
 		flush_rcu_sheaves_on_cache(s);
+		cpus_read_unlock();
 		rcu_barrier();
 	}
 
diff --git a/mm/slub.c b/mm/slub.c
index 8f90045367296..04692a6f9128b 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -227,6 +227,17 @@ struct partial_bulk_context {
 	struct list_head slabs;
 };
 
+/* Structure used to iterate over objects within a slab */
+struct slab_obj_iter {
+	unsigned long pos;
+	void *start;
+#ifdef CONFIG_SLAB_FREELIST_RANDOM
+	unsigned long freelist_count;
+	unsigned long page_limit;
+	bool random;
+#endif
+};
+
 static inline bool kmem_cache_debug(struct kmem_cache *s)
 {
 	return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
@@ -2129,11 +2140,11 @@ static inline size_t obj_exts_alloc_size(struct kmem_cache *s,
 	if (!is_kmalloc_normal(s))
 		return sz;
 
-	obj_exts_cache = kmalloc_slab(sz, NULL, gfp, 0);
+	obj_exts_cache = kmalloc_slab(sz, NULL, gfp, __kmalloc_token(0));
 	/*
-	 * We can't simply compare s with obj_exts_cache, because random kmalloc
-	 * caches have multiple caches per size, selected by caller address.
-	 * Since caller address may differ between kmalloc_slab() and actual
+	 * We can't simply compare s with obj_exts_cache, because partitioned kmalloc
+	 * caches have multiple caches per size, selected by caller address or type.
+	 * Since caller address or type may differ between kmalloc_slab() and actual
 	 * allocation, bump size when sizes are equal.
 	 */
 	if (s->object_size == obj_exts_cache->object_size)
@@ -2733,7 +2744,7 @@ bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail,
 	return *head != NULL;
 }
 
-static void *setup_object(struct kmem_cache *s, void *object)
+static inline void *setup_object(struct kmem_cache *s, void *object)
 {
 	setup_object_debug(s, object);
 	object = kasan_init_slab_obj(s, object);
@@ -3329,87 +3340,14 @@ static void __init init_freelist_randomization(void)
 	mutex_unlock(&slab_mutex);
 }
 
-/* Get the next entry on the pre-computed freelist randomized */
-static void *next_freelist_entry(struct kmem_cache *s,
-				unsigned long *pos, void *start,
-				unsigned long page_limit,
-				unsigned long freelist_count)
-{
-	unsigned int idx;
-
-	/*
-	 * If the target page allocation failed, the number of objects on the
-	 * page might be smaller than the usual size defined by the cache.
-	 */
-	do {
-		idx = s->random_seq[*pos];
-		*pos += 1;
-		if (*pos >= freelist_count)
-			*pos = 0;
-	} while (unlikely(idx >= page_limit));
-
-	return (char *)start + idx;
-}
-
 static DEFINE_PER_CPU(struct rnd_state, slab_rnd_state);
 
-/* Shuffle the single linked freelist based on a random pre-computed sequence */
-static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab,
-			     bool allow_spin)
-{
-	void *start;
-	void *cur;
-	void *next;
-	unsigned long idx, pos, page_limit, freelist_count;
-
-	if (slab->objects < 2 || !s->random_seq)
-		return false;
-
-	freelist_count = oo_objects(s->oo);
-	if (allow_spin) {
-		pos = get_random_u32_below(freelist_count);
-	} else {
-		struct rnd_state *state;
-
-		/*
-		 * An interrupt or NMI handler might interrupt and change
-		 * the state in the middle, but that's safe.
-		 */
-		state = &get_cpu_var(slab_rnd_state);
-		pos = prandom_u32_state(state) % freelist_count;
-		put_cpu_var(slab_rnd_state);
-	}
-
-	page_limit = slab->objects * s->size;
-	start = fixup_red_left(s, slab_address(slab));
-
-	/* First entry is used as the base of the freelist */
-	cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count);
-	cur = setup_object(s, cur);
-	slab->freelist = cur;
-
-	for (idx = 1; idx < slab->objects; idx++) {
-		next = next_freelist_entry(s, &pos, start, page_limit,
-			freelist_count);
-		next = setup_object(s, next);
-		set_freepointer(s, cur, next);
-		cur = next;
-	}
-	set_freepointer(s, cur, NULL);
-
-	return true;
-}
 #else
 static inline int init_cache_random_seq(struct kmem_cache *s)
 {
 	return 0;
 }
 static inline void init_freelist_randomization(void) { }
-static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab,
-				    bool allow_spin)
-{
-	return false;
-}
 #endif /* CONFIG_SLAB_FREELIST_RANDOM */
 
 static __always_inline void account_slab(struct slab *slab, int order,
@@ -3438,15 +3376,14 @@ static __always_inline void unaccount_slab(struct slab *slab, int order,
 			    -(PAGE_SIZE << order));
 }
 
+/* Allocate and initialize a slab without building its freelist. */
 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 {
 	bool allow_spin = gfpflags_allow_spinning(flags);
 	struct slab *slab;
 	struct kmem_cache_order_objects oo = s->oo;
 	gfp_t alloc_gfp;
-	void *start, *p, *next;
-	int idx;
-	bool shuffle;
+	void *start;
 
 	flags &= gfp_allowed_mask;
 
@@ -3497,21 +3434,6 @@ static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 	alloc_slab_obj_exts_early(s, slab);
 	account_slab(slab, oo_order(oo), s, flags);
 
-	shuffle = shuffle_freelist(s, slab, allow_spin);
-
-	if (!shuffle) {
-		start = fixup_red_left(s, start);
-		start = setup_object(s, start);
-		slab->freelist = start;
-		for (idx = 0, p = start; idx < slab->objects - 1; idx++) {
-			next = p + s->size;
-			next = setup_object(s, next);
-			set_freepointer(s, p, next);
-			p = next;
-		}
-		set_freepointer(s, p, NULL);
-	}
-
 	return slab;
 }
 
@@ -3665,30 +3587,112 @@ static void *alloc_single_from_partial(struct kmem_cache *s,
 	return object;
 }
 
+/* Return the next free object in allocation order. */
+static inline void *next_slab_obj(struct kmem_cache *s,
+				  struct slab_obj_iter *iter)
+{
+#ifdef CONFIG_SLAB_FREELIST_RANDOM
+	if (iter->random) {
+		unsigned long idx;
+
+		/*
+		 * If the target page allocation failed, the number of objects on the
+		 * page might be smaller than the usual size defined by the cache.
+		 */
+		do {
+			idx = s->random_seq[iter->pos];
+			iter->pos++;
+			if (iter->pos >= iter->freelist_count)
+				iter->pos = 0;
+		} while (unlikely(idx >= iter->page_limit));
+
+		return setup_object(s, (char *)iter->start + idx);
+	}
+#endif
+	return setup_object(s, (char *)iter->start + iter->pos++ * s->size);
+}
+
+/* Build a freelist from the objects not yet allocated from a fresh slab. */
+static inline void build_slab_freelist(struct kmem_cache *s, struct slab *slab,
+				       struct slab_obj_iter *iter)
+{
+	unsigned int nr = slab->objects - slab->inuse;
+	unsigned int i;
+	void *cur, *next;
+
+	if (!nr) {
+		slab->freelist = NULL;
+		return;
+	}
+
+	cur = next_slab_obj(s, iter);
+	slab->freelist = cur;
+
+	for (i = 1; i < nr; i++) {
+		next = next_slab_obj(s, iter);
+		set_freepointer(s, cur, next);
+		cur = next;
+	}
+
+	set_freepointer(s, cur, NULL);
+}
+
+/* Initialize an iterator over free objects in allocation order. */
+static inline void init_slab_obj_iter(struct kmem_cache *s, struct slab *slab,
+				      struct slab_obj_iter *iter,
+				      bool allow_spin)
+{
+	iter->pos = 0;
+	iter->start = fixup_red_left(s, slab_address(slab));
+
+#ifdef CONFIG_SLAB_FREELIST_RANDOM
+	iter->random = (slab->objects >= 2 && s->random_seq);
+	if (!iter->random)
+		return;
+
+	iter->freelist_count = oo_objects(s->oo);
+	iter->page_limit = slab->objects * s->size;
+
+	if (allow_spin) {
+		iter->pos = get_random_u32_below(iter->freelist_count);
+	} else {
+		struct rnd_state *state;
+
+		/*
+		 * An interrupt or NMI handler might interrupt and change
+		 * the state in the middle, but that's safe.
+		 */
+		state = &get_cpu_var(slab_rnd_state);
+		iter->pos = prandom_u32_state(state) % iter->freelist_count;
+		put_cpu_var(slab_rnd_state);
+	}
+#endif
+}
+
 /*
  * Called only for kmem_cache_debug() caches to allocate from a freshly
  * allocated slab. Allocate a single object instead of whole freelist
  * and put the slab to the partial (or full) list.
  */
 static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
-					int orig_size, gfp_t gfpflags)
+					int orig_size, bool allow_spin)
 {
-	bool allow_spin = gfpflags_allow_spinning(gfpflags);
-	int nid = slab_nid(slab);
-	struct kmem_cache_node *n = get_node(s, nid);
+	struct kmem_cache_node *n;
+	struct slab_obj_iter iter;
+	bool needs_add_partial;
 	unsigned long flags;
 	void *object;
 
-	if (!allow_spin && !spin_trylock_irqsave(&n->list_lock, flags)) {
-		/* Unlucky, discard newly allocated slab. */
-		free_new_slab_nolock(s, slab);
-		return NULL;
-	}
-
-	object = slab->freelist;
-	slab->freelist = get_freepointer(s, object);
+	init_slab_obj_iter(s, slab, &iter, allow_spin);
+	object = next_slab_obj(s, &iter);
 	slab->inuse = 1;
 
+	needs_add_partial = (slab->objects > 1);
+	build_slab_freelist(s, slab, &iter);
+
+	/* alloc_debug_processing() always expects a valid freepointer */
+	set_freepointer(s, object, slab->freelist);
+
 	if (!alloc_debug_processing(s, slab, object, orig_size)) {
 		/*
 		 * It's not really expected that this would fail on a
@@ -3696,20 +3700,32 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
 		 * corruption in theory could cause that.
 		 * Leak memory of allocated slab.
 		 */
-		if (!allow_spin)
-			spin_unlock_irqrestore(&n->list_lock, flags);
 		return NULL;
 	}
 
-	if (allow_spin)
+	n = get_node(s, slab_nid(slab));
+	if (allow_spin) {
 		spin_lock_irqsave(&n->list_lock, flags);
+	} else if (!spin_trylock_irqsave(&n->list_lock, flags)) {
+		/*
+		 * Unlucky, discard newly allocated slab.
+		 * The slab is not fully free, but it's fine as
+		 * objects are not allocated to users.
+		 */
+		free_new_slab_nolock(s, slab);
+		return NULL;
+	}
 
-	if (slab->inuse == slab->objects)
-		add_full(s, n, slab);
-	else
+	if (needs_add_partial)
 		add_partial(n, slab, ADD_TO_HEAD);
+	else
+		add_full(s, n, slab);
 
-	inc_slabs_node(s, nid, slab->objects);
+	/*
+	 * Debug caches require nr_slabs updates under n->list_lock so validation
+	 * cannot race with slab (de)allocations and observe inconsistent state.
+	 */
+	inc_slabs_node(s, slab_nid(slab), slab->objects);
 	spin_unlock_irqrestore(&n->list_lock, flags);
 
 	return object;
@@ -4024,6 +4040,7 @@ void flush_rcu_sheaves_on_cache(struct kmem_cache *s)
 	struct slub_flush_work *sfw;
 	unsigned int cpu;
 
+	lockdep_assert_cpus_held();
 	mutex_lock(&flush_lock);
 
 	for_each_online_cpu(cpu) {
@@ -4310,7 +4327,8 @@ static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags)
  * Assumes this is performed only for caches without debugging so we
  * don't need to worry about adding the slab to the full list.
  */
-static inline void *get_freelist_nofreeze(struct kmem_cache *s, struct slab *slab)
+static inline void *get_freelist_nofreeze(struct kmem_cache *s, struct slab *slab,
+					  unsigned int *count)
 {
 	struct freelist_counters old, new;
 
@@ -4326,6 +4344,7 @@ static inline void *get_freelist_nofreeze(struct kmem_cache *s, struct slab *sla
 
 	} while (!slab_update_freelist(s, slab, &old, &new, "get_freelist_nofreeze"));
 
+	*count = old.objects - old.inuse;
 	return old.freelist;
 }
 
@@ -4348,44 +4367,41 @@ static unsigned int alloc_from_new_slab(struct kmem_cache *s, struct slab *slab,
 		void **p, unsigned int count, bool allow_spin)
 {
 	unsigned int allocated = 0;
-	struct kmem_cache_node *n;
-	bool needs_add_partial;
+	struct slab_obj_iter iter;
+	bool needs_add_partial = true;
 	unsigned long flags;
-	void *object;
 
 	/*
 	 * Are we going to put the slab on the partial list?
 	 * Note slab->inuse is 0 on a new slab.
 	 */
-	needs_add_partial = (slab->objects > count);
-
-	if (!allow_spin && needs_add_partial) {
-
-		n = get_node(s, slab_nid(slab));
-
-		if (!spin_trylock_irqsave(&n->list_lock, flags)) {
-			/* Unlucky, discard newly allocated slab */
-			free_new_slab_nolock(s, slab);
-			return 0;
-		}
+	if (count >= slab->objects) {
+		needs_add_partial = false;
+		count = slab->objects;
 	}
 
-	object = slab->freelist;
-	while (object && allocated < count) {
-		p[allocated] = object;
-		object = get_freepointer(s, object);
-		maybe_wipe_obj_freeptr(s, p[allocated]);
+	init_slab_obj_iter(s, slab, &iter, allow_spin);
 
-		slab->inuse++;
+	while (allocated < count) {
+		p[allocated] = next_slab_obj(s, &iter);
 		allocated++;
 	}
-	slab->freelist = object;
+	slab->inuse = count;
+	build_slab_freelist(s, slab, &iter);
 
 	if (needs_add_partial) {
+		struct kmem_cache_node *n = get_node(s, slab_nid(slab));
 
 		if (allow_spin) {
-			n = get_node(s, slab_nid(slab));
 			spin_lock_irqsave(&n->list_lock, flags);
+		} else if (!spin_trylock_irqsave(&n->list_lock, flags)) {
+			/*
+			 * Unlucky, discard newly allocated slab.
+			 * The slab is not fully free, but it's fine as
+			 * objects are not allocated to users.
+			 */
+			free_new_slab_nolock(s, slab);
+			return 0;
 		}
 		add_partial(n, slab, ADD_TO_HEAD);
 		spin_unlock_irqrestore(&n->list_lock, flags);
@@ -4456,15 +4472,13 @@ static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
 	stat(s, ALLOC_SLAB);
 
 	if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
-		object = alloc_single_from_new_slab(s, slab, orig_size, gfpflags);
+		object = alloc_single_from_new_slab(s, slab, orig_size, allow_spin);
 
 		if (likely(object))
 			goto success;
 	} else {
-		alloc_from_new_slab(s, slab, &object, 1, allow_spin);
-
 		/* we don't need to check SLAB_STORE_USER here */
-		if (likely(object))
+		if (alloc_from_new_slab(s, slab, &object, 1, allow_spin))
 			return object;
 	}
 
@@ -5274,7 +5288,7 @@ EXPORT_SYMBOL(__kmalloc_large_node_noprof);
 
 static __always_inline
 void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node,
-			unsigned long caller)
+			unsigned long caller, kmalloc_token_t token)
 {
 	struct kmem_cache *s;
 	void *ret;
@@ -5289,37 +5303,28 @@ void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node,
 	if (unlikely(!size))
 		return ZERO_SIZE_PTR;
 
-	s = kmalloc_slab(size, b, flags, caller);
+	s = kmalloc_slab(size, b, flags, token);
 
 	ret = slab_alloc_node(s, NULL, flags, node, caller, size);
 	ret = kasan_kmalloc(s, ret, size, flags);
 	trace_kmalloc(caller, ret, size, s->size, flags, node);
 	return ret;
 }
-void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node)
+void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node)
 {
-	return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, _RET_IP_);
+	return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node,
+				 _RET_IP_, PASS_TOKEN_PARAM(token));
 }
 EXPORT_SYMBOL(__kmalloc_node_noprof);
 
-void *__kmalloc_noprof(size_t size, gfp_t flags)
+void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags)
 {
-	return __do_kmalloc_node(size, NULL, flags, NUMA_NO_NODE, _RET_IP_);
+	return __do_kmalloc_node(size, NULL, flags,  NUMA_NO_NODE, _RET_IP_,
+				 PASS_TOKEN_PARAM(token));
 }
 EXPORT_SYMBOL(__kmalloc_noprof);
 
-/**
- * kmalloc_nolock - Allocate an object of given size from any context.
- * @size: size to allocate
- * @gfp_flags: GFP flags. Only __GFP_ACCOUNT, __GFP_ZERO, __GFP_NO_OBJ_EXT
- * allowed.
- * @node: node number of the target node.
- *
- * Return: pointer to the new object or NULL in case of error.
- * NULL does not mean EBUSY or EAGAIN. It means ENOMEM.
- * There is no reason to call it again and expect !NULL.
- */
-void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node)
+void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node)
 {
 	gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags;
 	struct kmem_cache *s;
@@ -5346,7 +5351,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node)
 retry:
 	if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
 		return NULL;
-	s = kmalloc_slab(size, NULL, alloc_gfp, _RET_IP_);
+	s = kmalloc_slab(size, NULL, alloc_gfp, PASS_TOKEN_PARAM(token));
 
 	if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s))
 		/*
@@ -5399,12 +5404,13 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node)
 	ret = kasan_kmalloc(s, ret, size, alloc_gfp);
 	return ret;
 }
-EXPORT_SYMBOL_GPL(kmalloc_nolock_noprof);
+EXPORT_SYMBOL_GPL(_kmalloc_nolock_noprof);
 
-void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags,
+void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags,
 					 int node, unsigned long caller)
 {
-	return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node, caller);
+	return __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), flags, node,
+				 caller, PASS_TOKEN_PARAM(token));
 
 }
 EXPORT_SYMBOL(__kmalloc_node_track_caller_noprof);
@@ -5498,6 +5504,34 @@ static noinline void free_to_partial_list(
 	}
 }
 
+/*
+ * Try returning (remainder of) the freelist that we just detached from the
+ * slab.  Optimistically assume the slab is still full, so we don't need to find
+ * the tail of the detached freelist.
+ *
+ * Fail if the slab isn't full anymore due to a cocurrent free.
+ */
+static bool __slab_try_return_freelist(struct kmem_cache *s, struct slab *slab,
+				       void *head, int cnt)
+{
+	struct freelist_counters old, new;
+
+	old.freelist = slab->freelist;
+	old.counters = slab->counters;
+
+	if (old.freelist)
+		return false;
+
+	new.freelist = head;
+	new.counters = old.counters;
+	new.inuse -= cnt;
+
+	if (!slab_update_freelist(s, slab, &old, &new, "__slab_try_return_freelist"))
+		return false;
+
+	return true;
+}
+
 /*
  * Slow path handling. This may still be called frequently since objects
  * have a longer lifetime than the cpu slabs in most processing loads.
@@ -6635,7 +6669,7 @@ void kfree_nolock(const void *object)
 EXPORT_SYMBOL_GPL(kfree_nolock);
 
 static __always_inline __realloc_size(2) void *
-__do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid)
+__do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid, kmalloc_token_t token)
 {
 	void *ret;
 	size_t ks = 0;
@@ -6707,7 +6741,7 @@ __do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags,
 	return (void *)p;
 
 alloc_new:
-	ret = kmalloc_node_track_caller_noprof(new_size, flags, nid, _RET_IP_);
+	ret = __kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(new_size, NULL, token), flags, nid, _RET_IP_);
 	if (ret && p) {
 		/* Disable KASAN checks as the object's redzone is accessed. */
 		kasan_disable_current();
@@ -6718,45 +6752,7 @@ __do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t flags,
 	return ret;
 }
 
-/**
- * krealloc_node_align - reallocate memory. The contents will remain unchanged.
- * @p: object to reallocate memory for.
- * @new_size: how many bytes of memory are required.
- * @align: desired alignment.
- * @flags: the type of memory to allocate.
- * @nid: NUMA node or NUMA_NO_NODE
- *
- * If @p is %NULL, krealloc() behaves exactly like kmalloc().  If @new_size
- * is 0 and @p is not a %NULL pointer, the object pointed to is freed.
- *
- * Only alignments up to those guaranteed by kmalloc() will be honored. Please see
- * Documentation/core-api/memory-allocation.rst for more details.
- *
- * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
- * initial memory allocation, every subsequent call to this API for the same
- * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
- * __GFP_ZERO is not fully honored by this API.
- *
- * When slub_debug_orig_size() is off, krealloc() only knows about the bucket
- * size of an allocation (but not the exact size it was allocated with) and
- * hence implements the following semantics for shrinking and growing buffers
- * with __GFP_ZERO::
- *
- *           new             bucket
- *   0       size             size
- *   |--------|----------------|
- *   |  keep  |      zero      |
- *
- * Otherwise, the original allocation size 'orig_size' could be used to
- * precisely clear the requested size, and the new size will also be stored
- * as the new 'orig_size'.
- *
- * In any case, the contents of the object pointed to are preserved up to the
- * lesser of the new and old sizes.
- *
- * Return: pointer to the allocated memory or %NULL in case of error
- */
-void *krealloc_node_align_noprof(const void *p, size_t new_size, unsigned long align,
+void *krealloc_node_align_noprof(const void *p, DECL_TOKEN_PARAMS(new_size, token), unsigned long align,
 				 gfp_t flags, int nid)
 {
 	void *ret;
@@ -6766,7 +6762,7 @@ void *krealloc_node_align_noprof(const void *p, size_t new_size, unsigned long a
 		return ZERO_SIZE_PTR;
 	}
 
-	ret = __do_krealloc(p, new_size, align, flags, nid);
+	ret = __do_krealloc(p, new_size, align, flags, nid, PASS_TOKEN_PARAM(token));
 	if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret))
 		kfree(p);
 
@@ -6798,28 +6794,7 @@ static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size)
 	return flags;
 }
 
-/**
- * __kvmalloc_node - attempt to allocate physically contiguous memory, but upon
- * failure, fall back to non-contiguous (vmalloc) allocation.
- * @size: size of the request.
- * @b: which set of kmalloc buckets to allocate from.
- * @align: desired alignment.
- * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
- * @node: numa node to allocate from
- *
- * Only alignments up to those guaranteed by kmalloc() will be honored. Please see
- * Documentation/core-api/memory-allocation.rst for more details.
- *
- * Uses kmalloc to get the memory but if the allocation fails then falls back
- * to the vmalloc allocator. Use kvfree for freeing the memory.
- *
- * GFP_NOWAIT and GFP_ATOMIC are supported, the __GFP_NORETRY modifier is not.
- * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is
- * preferable to the vmalloc fallback, due to visible performance drawbacks.
- *
- * Return: pointer to the allocated memory of %NULL in case of failure
- */
-void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align,
+void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long align,
 			     gfp_t flags, int node)
 {
 	bool allow_block;
@@ -6831,7 +6806,7 @@ void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align,
 	 */
 	ret = __do_kmalloc_node(size, PASS_BUCKET_PARAM(b),
 				kmalloc_gfp_adjust(flags, size),
-				node, _RET_IP_);
+				node, _RET_IP_, PASS_TOKEN_PARAM(token));
 	if (ret || size <= PAGE_SIZE)
 		return ret;
 
@@ -6916,34 +6891,7 @@ void kvfree_sensitive(const void *addr, size_t len)
 }
 EXPORT_SYMBOL(kvfree_sensitive);
 
-/**
- * kvrealloc_node_align - reallocate memory; contents remain unchanged
- * @p: object to reallocate memory for
- * @size: the size to reallocate
- * @align: desired alignment
- * @flags: the flags for the page level allocator
- * @nid: NUMA node id
- *
- * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0
- * and @p is not a %NULL pointer, the object pointed to is freed.
- *
- * Only alignments up to those guaranteed by kmalloc() will be honored. Please see
- * Documentation/core-api/memory-allocation.rst for more details.
- *
- * If __GFP_ZERO logic is requested, callers must ensure that, starting with the
- * initial memory allocation, every subsequent call to this API for the same
- * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that
- * __GFP_ZERO is not fully honored by this API.
- *
- * In any case, the contents of the object pointed to are preserved up to the
- * lesser of the new and old sizes.
- *
- * This function must not be called concurrently with itself or kvfree() for the
- * same memory allocation.
- *
- * Return: pointer to the allocated memory or %NULL in case of error
- */
-void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align,
+void *kvrealloc_node_align_noprof(const void *p, DECL_TOKEN_PARAMS(size, token), unsigned long align,
 				  gfp_t flags, int nid)
 {
 	void *n;
@@ -6951,10 +6899,10 @@ void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long alig
 	if (is_vmalloc_addr(p))
 		return vrealloc_node_align_noprof(p, size, align, flags, nid);
 
-	n = krealloc_node_align_noprof(p, size, align, kmalloc_gfp_adjust(flags, size), nid);
+	n = krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(size, token), align, kmalloc_gfp_adjust(flags, size), nid);
 	if (!n) {
 		/* We failed to krealloc(), fall back to kvmalloc(). */
-		n = kvmalloc_node_align_noprof(size, align, flags, nid);
+		n = __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(size, NULL, token), align, flags, nid);
 		if (!n)
 			return NULL;
 
@@ -7125,34 +7073,41 @@ __refill_objects_node(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int mi
 
 	list_for_each_entry_safe(slab, slab2, &pc.slabs, slab_list) {
 
+		unsigned int count;
+
 		list_del(&slab->slab_list);
 
-		object = get_freelist_nofreeze(s, slab);
+		object = get_freelist_nofreeze(s, slab, &count);
 
-		while (object && refilled < max) {
+		while (count && refilled < max) {
 			p[refilled] = object;
 			object = get_freepointer(s, object);
 			maybe_wipe_obj_freeptr(s, p[refilled]);
 
 			refilled++;
+			count--;
 		}
 
 		/*
 		 * Freelist had more objects than we can accommodate, we need to
-		 * free them back. We can treat it like a detached freelist, just
-		 * need to find the tail object.
+		 * free them back. First we try to be optimistic and assume the
+		 * slab is stil full since we just detached its freelist.
+		 * Otherwise we must need to find the tail object.
 		 */
-		if (unlikely(object)) {
+		if (unlikely(count)) {
 			void *head = object;
 			void *tail;
-			int cnt = 0;
+
+			if (__slab_try_return_freelist(s, slab, head, count)) {
+				list_add(&slab->slab_list, &pc.slabs);
+				break;
+			}
 
 			do {
 				tail = object;
-				cnt++;
 				object = get_freepointer(s, object);
 			} while (object);
-			__slab_free(s, slab, head, tail, cnt, _RET_IP_);
+			__slab_free(s, slab, head, tail, count, _RET_IP_);
 		}
 
 		if (refilled >= max)
@@ -7274,10 +7229,6 @@ refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
 
 	stat(s, ALLOC_SLAB);
 
-	/*
-	 * TODO: possible optimization - if we know we will consume the whole
-	 * slab we might skip creating the freelist?
-	 */
 	refilled += alloc_from_new_slab(s, slab, p + refilled, max - refilled,
 					/* allow_spin = */ true);
 
@@ -7608,6 +7559,7 @@ static void early_kmem_cache_node_alloc(int node)
 {
 	struct slab *slab;
 	struct kmem_cache_node *n;
+	struct slab_obj_iter iter;
 
 	BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
 
@@ -7619,14 +7571,18 @@ static void early_kmem_cache_node_alloc(int node)
 		pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
 	}
 
-	n = slab->freelist;
+	init_slab_obj_iter(kmem_cache_node, slab, &iter, true);
+
+	n = next_slab_obj(kmem_cache_node, &iter);
 	BUG_ON(!n);
+
+	slab->inuse = 1;
+	build_slab_freelist(kmem_cache_node, slab, &iter);
+
 #ifdef CONFIG_SLUB_DEBUG
 	init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
 #endif
 	n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
-	slab->freelist = get_freepointer(kmem_cache_node, n);
-	slab->inuse = 1;
 	kmem_cache_node->per_node[node].node = n;
 	init_kmem_cache_node(n);
 	inc_slabs_node(kmem_cache_node, node, slab->objects);
@@ -8469,7 +8425,7 @@ static void __init bootstrap_kmalloc_sheaves(void)
 {
 	enum kmalloc_cache_type type;
 
-	for (type = KMALLOC_NORMAL; type <= KMALLOC_RANDOM_END; type++) {
+	for (type = KMALLOC_NORMAL; type <= KMALLOC_PARTITION_END; type++) {
 		for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {
 			if (kmalloc_caches[type][idx])
 				bootstrap_cache_sheaves(kmalloc_caches[type][idx]);
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 6eadb9d116e43..112ccf9c71caf 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -87,15 +87,10 @@ static void * __meminit altmap_alloc_block_buf(unsigned long size,
 void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node,
 					 struct vmem_altmap *altmap)
 {
-	void *ptr;
-
 	if (altmap)
 		return altmap_alloc_block_buf(size, altmap);
 
-	ptr = sparse_buffer_alloc(size);
-	if (!ptr)
-		ptr = vmemmap_alloc_block(size, node);
-	return ptr;
+	return vmemmap_alloc_block(size, node);
 }
 
 static unsigned long __meminit vmem_altmap_next_pfn(struct vmem_altmap *altmap)
@@ -151,7 +146,7 @@ void __meminit vmemmap_verify(pte_t *pte, int node,
 			start, end - 1);
 }
 
-pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
+static pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
 				       struct vmem_altmap *altmap,
 				       unsigned long ptpfn, unsigned long flags)
 {
@@ -195,7 +190,7 @@ static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
 	return p;
 }
 
-pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
+static pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
 {
 	pmd_t *pmd = pmd_offset(pud, addr);
 	if (pmd_none(*pmd)) {
@@ -208,7 +203,7 @@ pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
 	return pmd;
 }
 
-pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
+static pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
 {
 	pud_t *pud = pud_offset(p4d, addr);
 	if (pud_none(*pud)) {
@@ -221,7 +216,7 @@ pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
 	return pud;
 }
 
-p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
+static p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
 {
 	p4d_t *p4d = p4d_offset(pgd, addr);
 	if (p4d_none(*p4d)) {
@@ -234,7 +229,7 @@ p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
 	return p4d;
 }
 
-pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
+static pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
 {
 	pgd_t *pgd = pgd_offset_k(addr);
 	if (pgd_none(*pgd)) {
@@ -652,26 +647,61 @@ void offline_mem_sections(unsigned long start_pfn, unsigned long end_pfn)
 	}
 }
 
+static int __meminit section_nr_vmemmap_pages(unsigned long pfn, unsigned long nr_pages,
+		struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
+{
+	const unsigned int order = pgmap ? pgmap->vmemmap_shift : 0;
+	const unsigned long pages_per_compound = 1UL << order;
+
+	VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SUBSECTION));
+	VM_WARN_ON_ONCE(nr_pages > PAGES_PER_SECTION);
+
+	if (!vmemmap_can_optimize(altmap, pgmap))
+		return DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE);
+
+	if (order < PFN_SECTION_SHIFT) {
+		VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, pages_per_compound));
+		return VMEMMAP_RESERVE_NR * nr_pages / pages_per_compound;
+	}
+
+	VM_WARN_ON_ONCE(!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION));
+
+	if (IS_ALIGNED(pfn, pages_per_compound))
+		return VMEMMAP_RESERVE_NR;
+
+	return 0;
+}
+
 static struct page * __meminit populate_section_memmap(unsigned long pfn,
 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap,
 		struct dev_pagemap *pgmap)
 {
-	return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
+	struct page *page = __populate_section_memmap(pfn, nr_pages, nid, altmap,
+						      pgmap);
+
+	memmap_pages_add(section_nr_vmemmap_pages(pfn, nr_pages, altmap, pgmap));
+
+	return page;
 }
 
 static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
-		struct vmem_altmap *altmap)
+		struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
 {
 	unsigned long start = (unsigned long) pfn_to_page(pfn);
 	unsigned long end = start + nr_pages * sizeof(struct page);
 
+	memmap_pages_add(-section_nr_vmemmap_pages(pfn, nr_pages, altmap, pgmap));
 	vmemmap_free(start, end, altmap);
 }
+
 static void free_map_bootmem(struct page *memmap)
 {
 	unsigned long start = (unsigned long)memmap;
 	unsigned long end = (unsigned long)(memmap + PAGES_PER_SECTION);
+	unsigned long pfn = page_to_pfn(memmap);
 
+	memmap_boot_pages_add(-section_nr_vmemmap_pages(pfn, PAGES_PER_SECTION,
+							NULL, NULL));
 	vmemmap_free(start, end, NULL);
 }
 
@@ -737,7 +767,7 @@ static int fill_subsection_map(unsigned long pfn, unsigned long nr_pages)
  * usage map, but still need to free the vmemmap range.
  */
 static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
-		struct vmem_altmap *altmap)
+		struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
 {
 	struct mem_section *ms = __pfn_to_section(pfn);
 	bool section_is_early = early_section(ms);
@@ -774,14 +804,10 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
 	 * The memmap of early sections is always fully populated. See
 	 * section_activate() and pfn_valid() .
 	 */
-	if (!section_is_early) {
-		memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)));
-		depopulate_section_memmap(pfn, nr_pages, altmap);
-	} else if (memmap) {
-		memmap_boot_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page),
-							  PAGE_SIZE)));
+	if (!section_is_early)
+		depopulate_section_memmap(pfn, nr_pages, altmap, pgmap);
+	else if (memmap)
 		free_map_bootmem(memmap);
-	}
 
 	if (empty)
 		ms->section_mem_map = (unsigned long)NULL;
@@ -823,10 +849,9 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn,
 
 	memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap);
 	if (!memmap) {
-		section_deactivate(pfn, nr_pages, altmap);
+		section_deactivate(pfn, nr_pages, altmap, pgmap);
 		return ERR_PTR(-ENOMEM);
 	}
-	memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE));
 
 	return memmap;
 }
@@ -885,13 +910,13 @@ int __meminit sparse_add_section(int nid, unsigned long start_pfn,
 }
 
 void sparse_remove_section(unsigned long pfn, unsigned long nr_pages,
-			   struct vmem_altmap *altmap)
+		struct vmem_altmap *altmap, struct dev_pagemap *pgmap)
 {
 	struct mem_section *ms = __pfn_to_section(pfn);
 
 	if (WARN_ON_ONCE(!valid_section(ms)))
 		return;
 
-	section_deactivate(pfn, nr_pages, altmap);
+	section_deactivate(pfn, nr_pages, altmap, pgmap);
 }
 #endif /* CONFIG_MEMORY_HOTPLUG */
diff --git a/mm/sparse.c b/mm/sparse.c
index effdac6b0ab13..16ac6df3c89fa 100644
--- a/mm/sparse.c
+++ b/mm/sparse.c
@@ -201,13 +201,11 @@ static void __init memblocks_present(void)
 	int i, nid;
 
 #ifdef CONFIG_SPARSEMEM_EXTREME
-	if (unlikely(!mem_section)) {
-		unsigned long size, align;
+	unsigned long size, align;
 
-		size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;
-		align = 1 << (INTERNODE_CACHE_SHIFT);
-		mem_section = memblock_alloc_or_panic(size, align);
-	}
+	size = sizeof(struct mem_section *) * NR_SECTION_ROOTS;
+	align = 1 << (INTERNODE_CACHE_SHIFT);
+	mem_section = memblock_alloc_or_panic(size, align);
 #endif
 
 	for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid)
@@ -241,12 +239,9 @@ struct page __init *__populate_section_memmap(unsigned long pfn,
 		struct dev_pagemap *pgmap)
 {
 	unsigned long size = section_map_size();
-	struct page *map = sparse_buffer_alloc(size);
+	struct page *map;
 	phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
 
-	if (map)
-		return map;
-
 	map = memmap_alloc(size, size, addr, nid, false);
 	if (!map)
 		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa\n",
@@ -256,55 +251,6 @@ struct page __init *__populate_section_memmap(unsigned long pfn,
 }
 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
 
-static void *sparsemap_buf __meminitdata;
-static void *sparsemap_buf_end __meminitdata;
-
-static inline void __meminit sparse_buffer_free(unsigned long size)
-{
-	WARN_ON(!sparsemap_buf || size == 0);
-	memblock_free(sparsemap_buf, size);
-}
-
-static void __init sparse_buffer_init(unsigned long size, int nid)
-{
-	phys_addr_t addr = __pa(MAX_DMA_ADDRESS);
-	WARN_ON(sparsemap_buf);	/* forgot to call sparse_buffer_fini()? */
-	/*
-	 * Pre-allocated buffer is mainly used by __populate_section_memmap
-	 * and we want it to be properly aligned to the section size - this is
-	 * especially the case for VMEMMAP which maps memmap to PMDs
-	 */
-	sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true);
-	sparsemap_buf_end = sparsemap_buf + size;
-}
-
-static void __init sparse_buffer_fini(void)
-{
-	unsigned long size = sparsemap_buf_end - sparsemap_buf;
-
-	if (sparsemap_buf && size > 0)
-		sparse_buffer_free(size);
-	sparsemap_buf = NULL;
-}
-
-void * __meminit sparse_buffer_alloc(unsigned long size)
-{
-	void *ptr = NULL;
-
-	if (sparsemap_buf) {
-		ptr = (void *) roundup((unsigned long)sparsemap_buf, size);
-		if (ptr + size > sparsemap_buf_end)
-			ptr = NULL;
-		else {
-			/* Free redundant aligned space */
-			if ((unsigned long)(ptr - sparsemap_buf) > 0)
-				sparse_buffer_free((unsigned long)(ptr - sparsemap_buf));
-			sparsemap_buf = ptr + size;
-		}
-	}
-	return ptr;
-}
-
 void __weak __meminit vmemmap_populate_print_last(void)
 {
 }
@@ -362,8 +308,6 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
 		goto failed;
 	}
 
-	sparse_buffer_init(map_count * section_map_size(), nid);
-
 	sparse_vmemmap_init_nid_early(nid);
 
 	for_each_present_section_nr(pnum_begin, pnum) {
@@ -381,7 +325,6 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
 				       __func__, nid);
 				pnum_begin = pnum;
 				sparse_usage_fini();
-				sparse_buffer_fini();
 				goto failed;
 			}
 			memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
@@ -390,7 +333,6 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
 		}
 	}
 	sparse_usage_fini();
-	sparse_buffer_fini();
 	return;
 failed:
 	/*
diff --git a/mm/swap.c b/mm/swap.c
index 5cc44f0de9877..70c90b007fd3b 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -160,14 +160,42 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
 	int i;
 	struct lruvec *lruvec = NULL;
 	unsigned long flags = 0;
+	struct folio_batch free_fbatch;
+	bool is_lru_add = (move_fn == lru_add);
+
+	/*
+	 * If we're adding to the LRU, preemptively filter dead folios. Use
+	 * this dedicated folio batch for temp storage and deferred cleanup.
+	 */
+	if (is_lru_add)
+		folio_batch_init(&free_fbatch);
 
 	for (i = 0; i < folio_batch_count(fbatch); i++) {
 		struct folio *folio = fbatch->folios[i];
 
 		/* block memcg migration while the folio moves between lru */
-		if (move_fn != lru_add && !folio_test_clear_lru(folio))
+		if (!is_lru_add && !folio_test_clear_lru(folio))
 			continue;
 
+		/*
+		 * Filter dead folios by moving them from the add batch to the temp
+		 * batch for freeing after this loop.
+		 *
+		 * We're bypassing normal cleanup. Clear flags that are not
+		 * applicable to dead folios.
+		 *
+		 * Since the folio may be part of a huge page, unqueue from
+		 * deferred split list to avoid a dangling list entry.
+		 */
+		if (is_lru_add && folio_ref_freeze(folio, 1)) {
+			__folio_clear_active(folio);
+			__folio_clear_unevictable(folio);
+			folio_unqueue_deferred_split(folio);
+			fbatch->folios[i] = NULL;
+			folio_batch_add(&free_fbatch, folio);
+			continue;
+		}
+
 		folio_lruvec_relock_irqsave(folio, &lruvec, &flags);
 		move_fn(lruvec, folio);
 
@@ -176,6 +204,13 @@ static void folio_batch_move_lru(struct folio_batch *fbatch, move_fn_t move_fn)
 
 	if (lruvec)
 		lruvec_unlock_irqrestore(lruvec, flags);
+
+	/* Cleanup filtered dead folios. */
+	if (is_lru_add) {
+		mem_cgroup_uncharge_folios(&free_fbatch);
+		free_unref_folios(&free_fbatch);
+	}
+
 	folios_put(fbatch);
 }
 
@@ -512,7 +547,7 @@ void folio_add_lru(struct folio *folio)
 	/* see the comment in lru_gen_folio_seq() */
 	if (lru_gen_enabled() && !folio_test_unevictable(folio) &&
 	    lru_gen_in_fault() && !(current->flags & PF_MEMALLOC))
-		folio_set_active(folio);
+		folio_mark_accessed(folio);
 
 	folio_batch_add_and_move(folio, lru_add);
 }
@@ -964,6 +999,10 @@ void folios_put_refs(struct folio_batch *folios, unsigned int *refs)
 		struct folio *folio = folios->folios[i];
 		unsigned int nr_refs = refs ? refs[i] : 1;
 
+		/* Folio batch entry may have been preemptively removed during drain. */
+		if (!folio)
+			continue;
+
 		if (is_huge_zero_folio(folio))
 			continue;
 
diff --git a/mm/swap_cgroup.c b/mm/swap_cgroup.c
index de779fed8c210..95c38e54dd587 100644
--- a/mm/swap_cgroup.c
+++ b/mm/swap_cgroup.c
@@ -124,6 +124,8 @@ unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
 		return 0;
 
 	ctrl = &swap_cgroup_ctrl[swp_type(ent)];
+	if (unlikely(!ctrl->map))
+		return 0;
 	return __swap_cgroup_id_lookup(ctrl->map, swp_offset(ent));
 }
 
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 9174f1eeffb09..ee515a6fbccd4 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -1054,6 +1054,7 @@ static void swap_reclaim_full_clusters(struct swap_info_struct *si, bool force)
 		swap_cluster_unlock(ci);
 		if (to_scan <= 0)
 			break;
+		cond_resched();
 	}
 }
 
@@ -1442,8 +1443,10 @@ static bool swap_sync_discard(void)
 }
 
 static int swap_extend_table_alloc(struct swap_info_struct *si,
-				   struct swap_cluster_info *ci, gfp_t gfp)
+				   struct swap_cluster_info *ci,
+				   unsigned int ci_off, gfp_t gfp)
 {
+	int count;
 	void *table;
 
 	table = kzalloc(sizeof(ci->extend_table[0]) * SWAPFILE_CLUSTER, gfp);
@@ -1451,12 +1454,28 @@ static int swap_extend_table_alloc(struct swap_info_struct *si,
 		return -ENOMEM;
 
 	spin_lock(&ci->lock);
-	if (!ci->extend_table)
-		ci->extend_table = table;
-	else
-		kfree(table);
+	/*
+	 * Extend table allocation requires releasing ci lock first so it's
+	 * possible that the slot has been freed, no longer overflowed, or
+	 * a concurrent extend table allocation has already succeeded, so
+	 * the allocation is no longer needed.
+	 */
+	if (!cluster_table_is_alloced(ci))
+		goto out_free;
+	count = swp_tb_get_count(__swap_table_get(ci, ci_off));
+	if (count < (SWP_TB_COUNT_MAX - 1))
+		goto out_free;
+	if (ci->extend_table)
+		goto out_free;
+
+	ci->extend_table = table;
 	spin_unlock(&ci->lock);
 	return 0;
+
+out_free:
+	spin_unlock(&ci->lock);
+	kfree(table);
+	return 0;
 }
 
 int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp)
@@ -1471,7 +1490,7 @@ int swap_retry_table_alloc(swp_entry_t entry, gfp_t gfp)
 		return 0;
 
 	ci = __swap_offset_to_cluster(si, offset);
-	ret = swap_extend_table_alloc(si, ci, gfp);
+	ret = swap_extend_table_alloc(si, ci, swp_cluster_offset(entry), gfp);
 
 	put_swap_device(si);
 	return ret;
@@ -1518,13 +1537,21 @@ static void __swap_cluster_put_entry(struct swap_cluster_info *ci,
 		if (count == (SWP_TB_COUNT_MAX - 1)) {
 			ci->extend_table[ci_off] = 0;
 			__swap_table_set(ci, ci_off, __swp_tb_mk_count(swp_tb, count));
-			swap_extend_table_try_free(ci);
 		} else {
 			ci->extend_table[ci_off] = count;
 		}
 	} else {
 		__swap_table_set(ci, ci_off, __swp_tb_mk_count(swp_tb, --count));
 	}
+
+	/*
+	 * `SWP_TB_COUNT_MAX - 1` triggers extend table allocation. If the
+	 * count was above that, then the extend table is no longer needed,
+	 * so free it. And if we just put the count value from MAX - 1, it's
+	 * also possible that a pending dup just attached an extend table.
+	 */
+	if (unlikely(count == SWP_TB_COUNT_MAX - 2 || count == SWP_TB_COUNT_MAX - 1))
+		swap_extend_table_try_free(ci);
 }
 
 /**
@@ -1664,7 +1691,7 @@ static int swap_dup_entries_cluster(struct swap_info_struct *si,
 		if (unlikely(err)) {
 			if (err == -ENOMEM) {
 				spin_unlock(&ci->lock);
-				err = swap_extend_table_alloc(si, ci, GFP_ATOMIC);
+				err = swap_extend_table_alloc(si, ci, ci_off, GFP_ATOMIC);
 				spin_lock(&ci->lock);
 				if (!err)
 					goto restart;
diff --git a/mm/truncate.c b/mm/truncate.c
index 12cc89f89afcf..b58ba940be474 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -177,7 +177,7 @@ int truncate_inode_folio(struct address_space *mapping, struct folio *folio)
 	return 0;
 }
 
-static int try_folio_split_or_unmap(struct folio *folio, struct page *split_at,
+static int folio_split_or_unmap(struct folio *folio, struct page *split_at,
 				    unsigned long min_order)
 {
 	enum ttu_flags ttu_flags =
@@ -186,7 +186,7 @@ static int try_folio_split_or_unmap(struct folio *folio, struct page *split_at,
 		TTU_IGNORE_MLOCK;
 	int ret;
 
-	ret = try_folio_split_to_order(folio, split_at, min_order);
+	ret = folio_split(folio, min_order, split_at, NULL);
 
 	/*
 	 * If the split fails, unmap the folio, so it will be refaulted
@@ -252,7 +252,7 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
 
 	min_order = mapping_min_folio_order(folio->mapping);
 	split_at = folio_page(folio, PAGE_ALIGN_DOWN(offset) / PAGE_SIZE);
-	if (!try_folio_split_or_unmap(folio, split_at, min_order)) {
+	if (!folio_split_or_unmap(folio, split_at, min_order)) {
 		/*
 		 * try to split at offset + length to make sure folios within
 		 * the range can be dropped, especially to avoid memory waste
@@ -279,7 +279,7 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
 		/* make sure folio2 is large and does not change its mapping */
 		if (folio_test_large(folio2) &&
 		    folio2->mapping == folio->mapping)
-			try_folio_split_or_unmap(folio2, split_at2, min_order);
+			folio_split_or_unmap(folio2, split_at2, min_order);
 
 		folio_unlock(folio2);
 out:
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 180bad42fc79d..b70b84776a79a 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -14,6 +14,8 @@
 #include <linux/userfaultfd_k.h>
 #include <linux/mmu_notifier.h>
 #include <linux/hugetlb.h>
+#include <linux/file.h>
+#include <linux/cleanup.h>
 #include <asm/tlbflush.h>
 #include <asm/tlb.h>
 #include "internal.h"
@@ -69,6 +71,24 @@ static const struct vm_uffd_ops *vma_uffd_ops(struct vm_area_struct *vma)
 	return vma->vm_ops ? vma->vm_ops->uffd_ops : NULL;
 }
 
+static const struct vm_uffd_ops *vma_uffd_copy_ops(struct vm_area_struct *vma)
+{
+	const struct vm_uffd_ops *ops = vma_uffd_ops(vma);
+
+	if (!ops)
+		return NULL;
+
+	/*
+	 * UFFDIO_COPY fills MAP_PRIVATE file-backed mappings as anonymous
+	 * memory. This is an effective ops override, so retry validation must
+	 * compare the override result, not just vma->vm_ops->uffd_ops.
+	 */
+	if (!(vma->vm_flags & VM_SHARED))
+		return &anon_uffd_ops;
+
+	return ops;
+}
+
 static __always_inline
 bool validate_dst_vma(struct vm_area_struct *dst_vma, unsigned long dst_end)
 {
@@ -443,14 +463,70 @@ static int mfill_copy_folio_locked(struct folio *folio, unsigned long src_addr)
 	return ret;
 }
 
+#define VMA_SNAPSHOT_FLAGS append_vma_flags(__VMA_UFFD_FLAGS, VMA_SHARED_BIT)
+
+struct vma_snapshot {
+	const struct vm_uffd_ops *copy_ops;
+	const struct vm_uffd_ops *ops;
+	struct file *file;
+	vma_flags_t flags;
+	pgoff_t pgoff;
+};
+
+static void vma_snapshot_get(struct vma_snapshot *s, struct vm_area_struct *vma)
+{
+	s->flags = vma_flags_and_mask(&vma->flags, VMA_SNAPSHOT_FLAGS);
+	s->copy_ops = vma_uffd_copy_ops(vma);
+	s->ops = vma_uffd_ops(vma);
+	s->pgoff = vma->vm_pgoff;
+
+	if (vma->vm_file)
+		s->file = get_file(vma->vm_file);
+}
+
+static bool vma_snapshot_changed(struct vma_snapshot *s,
+				 struct vm_area_struct *vma)
+{
+	vma_flags_t flags = vma_flags_and_mask(&vma->flags, VMA_SNAPSHOT_FLAGS);
+
+	if (!vma_flags_same_pair(&s->flags, &flags))
+		return true;
+
+	/* VMA type or effective uffd_ops changed while the lock was dropped */
+	if (s->ops != vma_uffd_ops(vma) || s->copy_ops != vma_uffd_copy_ops(vma))
+		return true;
+
+	/* VMA was anonymous before; changed only if it no longer is */
+	if (!s->file)
+		return !vma_is_anonymous(vma);
+
+	/* VMA was file backed, but inode or offset has changed */
+	if (!vma->vm_file || vma->vm_file->f_inode != s->file->f_inode ||
+	    vma->vm_pgoff != s->pgoff)
+		return true;
+
+	return false;
+}
+
+static void vma_snapshot_put(struct vma_snapshot *s)
+{
+	if (s->file)
+		fput(s->file);
+}
+
+DEFINE_FREE(snapshot_put, struct vma_snapshot *, if (_T) vma_snapshot_put(_T));
+
 static int mfill_copy_folio_retry(struct mfill_state *state,
 				  struct folio *folio)
 {
-	const struct vm_uffd_ops *orig_ops = vma_uffd_ops(state->vma);
+	struct vma_snapshot s = { 0 };
+	struct vma_snapshot *p __free(snapshot_put) = &s;
 	unsigned long src_addr = state->src_addr;
 	void *kaddr;
 	int err;
 
+	vma_snapshot_get(&s, state->vma);
+
 	/* retry copying with mm_lock dropped */
 	mfill_put_vma(state);
 
@@ -467,12 +543,7 @@ static int mfill_copy_folio_retry(struct mfill_state *state,
 	if (err)
 		return err;
 
-	/*
-	 * The VMA type may have changed while the lock was dropped
-	 * (e.g. replaced with a hugetlb mapping), making the caller's
-	 * ops pointer stale.
-	 */
-	if (vma_uffd_ops(state->vma) != orig_ops)
+	if (vma_snapshot_changed(&s, state->vma))
 		return -EAGAIN;
 
 	err = mfill_establish_pmd(state);
@@ -545,19 +616,7 @@ static int __mfill_atomic_pte(struct mfill_state *state,
 
 static int mfill_atomic_pte_copy(struct mfill_state *state)
 {
-	const struct vm_uffd_ops *ops = vma_uffd_ops(state->vma);
-
-	/*
-	 * The normal page fault path for a MAP_PRIVATE mapping in a
-	 * file-backed VMA will invoke the fault, fill the hole in the file and
-	 * COW it right away. The result generates plain anonymous memory.
-	 * So when we are asked to fill a hole in a MAP_PRIVATE mapping, we'll
-	 * generate anonymous memory directly without actually filling the
-	 * hole. For the MAP_PRIVATE case the robustness check only happens in
-	 * the pagetable (to verify it's still none) and not in the page cache.
-	 */
-	if (!(state->vma->vm_flags & VM_SHARED))
-		ops = &anon_uffd_ops;
+	const struct vm_uffd_ops *ops = vma_uffd_copy_ops(state->vma);
 
 	return __mfill_atomic_pte(state, ops);
 }
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index c31a8615a8328..eabb86b13b7e5 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3203,7 +3203,7 @@ struct vm_struct *__get_vm_area_node(unsigned long size,
 	struct vm_struct *area;
 	unsigned long requested_size = size;
 
-	BUG_ON(in_interrupt());
+	BUG_ON(in_nmi() || in_hardirq());
 	size = ALIGN(size, 1ul << shift);
 	if (unlikely(!size))
 		return NULL;
@@ -3459,19 +3459,13 @@ void vfree(const void *addr)
 
 	if (unlikely(vm->flags & VM_FLUSH_RESET_PERMS))
 		vm_reset_perms(vm);
-	for (i = 0; i < vm->nr_pages; i++) {
-		struct page *page = vm->pages[i];
 
-		BUG_ON(!page);
-		/*
-		 * High-order allocs for huge vmallocs are split, so
-		 * can be freed as an array of order-0 allocations
-		 */
-		if (!(vm->flags & VM_MAP_PUT_PAGES))
-			mod_lruvec_page_state(page, NR_VMALLOC, -1);
-		__free_page(page);
-		cond_resched();
+	if (!(vm->flags & VM_MAP_PUT_PAGES)) {
+		for (i = 0; i < vm->nr_pages; i++)
+			mod_lruvec_page_state(vm->pages[i], NR_VMALLOC, -1);
 	}
+	free_pages_bulk(vm->pages, vm->nr_pages);
+
 	kvfree(vm->pages);
 	kfree(vm);
 }
@@ -3939,7 +3933,7 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
 				__GFP_NOFAIL | __GFP_ZERO |\
 				__GFP_NORETRY | __GFP_RETRY_MAYFAIL |\
 				GFP_NOFS | GFP_NOIO | GFP_KERNEL_ACCOUNT |\
-				GFP_USER | __GFP_NOLOCKDEP)
+				GFP_USER | __GFP_NOLOCKDEP | __GFP_SKIP_KASAN)
 
 static gfp_t vmalloc_fix_flags(gfp_t flags)
 {
@@ -3980,6 +3974,9 @@ static gfp_t vmalloc_fix_flags(gfp_t flags)
  *
  * %__GFP_NOWARN can be used to suppress failure messages.
  *
+ * %__GFP_SKIP_KASAN can be used to skip unpoisoning of mapped pages
+ * (when prot=%PAGE_KERNEL).
+ *
  * Can not be called from interrupt nor NMI contexts.
  * Return: the address of the area or %NULL on failure
  */
@@ -3993,6 +3990,7 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
 	kasan_vmalloc_flags_t kasan_flags = KASAN_VMALLOC_NONE;
 	unsigned long original_align = align;
 	unsigned int shift = PAGE_SHIFT;
+	bool skip_vmalloc_kasan = kasan_hw_tags_enabled() && (gfp_mask & __GFP_SKIP_KASAN);
 
 	if (WARN_ON_ONCE(!size))
 		return NULL;
@@ -4023,7 +4021,7 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
 again:
 	area = __get_vm_area_node(size, align, shift, VM_ALLOC |
 				  VM_UNINITIALIZED | vm_flags, start, end, node,
-				  gfp_mask, caller);
+				  gfp_mask & ~__GFP_SKIP_KASAN, caller);
 	if (!area) {
 		bool nofail = gfp_mask & __GFP_NOFAIL;
 		warn_alloc(gfp_mask, NULL,
@@ -4041,7 +4039,7 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
 	 * kasan_unpoison_vmalloc().
 	 */
 	if (pgprot_val(prot) == pgprot_val(PAGE_KERNEL)) {
-		if (kasan_hw_tags_enabled()) {
+		if (kasan_hw_tags_enabled() && !skip_vmalloc_kasan) {
 			/*
 			 * Modify protection bits to allow tagging.
 			 * This must be done before mapping.
@@ -4078,7 +4076,8 @@ void *__vmalloc_node_range_noprof(unsigned long size, unsigned long align,
 	    (gfp_mask & __GFP_SKIP_ZERO))
 		kasan_flags |= KASAN_VMALLOC_INIT;
 	/* KASAN_VMALLOC_PROT_NORMAL already set if required. */
-	area->addr = kasan_unpoison_vmalloc(area->addr, size, kasan_flags);
+	if (!skip_vmalloc_kasan)
+		area->addr = kasan_unpoison_vmalloc(area->addr, size, kasan_flags);
 
 	/*
 	 * In this function, newly allocated vm_struct has VM_UNINITIALIZED
diff --git a/mm/vmpressure.c b/mm/vmpressure.c
index 3fbb86996c4d2..f053554e58264 100644
--- a/mm/vmpressure.c
+++ b/mm/vmpressure.c
@@ -218,6 +218,7 @@ static void vmpressure_work_fn(struct work_struct *work)
 /**
  * vmpressure() - Account memory pressure through scanned/reclaimed ratio
  * @gfp:	reclaimer's gfp mask
+ * @order:	allocation order being reclaimed for
  * @memcg:	cgroup memory controller handle
  * @tree:	legacy subtree mode
  * @scanned:	number of pages scanned
@@ -236,7 +237,7 @@ static void vmpressure_work_fn(struct work_struct *work)
  *
  * This function does not return any value.
  */
-void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
+void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree,
 		unsigned long scanned, unsigned long reclaimed)
 {
 	struct vmpressure *vmpr;
@@ -307,7 +308,15 @@ void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree,
 
 		level = vmpressure_calc_level(scanned, reclaimed);
 
-		if (level > VMPRESSURE_LOW) {
+		/*
+		 * Once we go above COSTLY_ORDER, reclaim relies heavily on
+		 * compaction to make progress. Reclaim efficiency was never a
+		 * great proxy for pressure to begin with, but it's outright
+		 * misleading with these high orders. Don't throttle sockets
+		 * because somebody is attempting something crazy like an order-7
+		 * and predictably struggling.
+		 */
+		if (level > VMPRESSURE_LOW && order <= PAGE_ALLOC_COSTLY_ORDER) {
 			/*
 			 * Let the socket buffer allocator know that
 			 * we are having trouble reclaiming LRU pages.
@@ -348,7 +357,7 @@ void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio)
 	 * to the vmpressure() basically means that we signal 'critical'
 	 * level.
 	 */
-	vmpressure(gfp, memcg, true, vmpressure_win, 0);
+	vmpressure(gfp, 0, memcg, true, vmpressure_win, 0);
 }
 
 #define MAX_VMPRESSURE_ARGS_LEN	(strlen("critical") + strlen("hierarchy") + 2)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index bd1b1aa125819..b3e555561417c 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -109,7 +109,7 @@ struct scan_control {
 	/* zone_reclaim_mode */
 	unsigned int may_unmap:1;
 
-	/* zome_reclaim_mode, boost reclaim, cgroup restrictions */
+	/* zone_reclaim_mode, boost reclaim, cgroup restrictions */
 	unsigned int may_swap:1;
 
 	/* Not allow cache_trim_mode to be turned on as part of reclaim? */
@@ -169,11 +169,9 @@ struct scan_control {
 
 	struct {
 		unsigned int dirty;
-		unsigned int unqueued_dirty;
 		unsigned int congested;
 		unsigned int writeback;
 		unsigned int immediate;
-		unsigned int file_taken;
 		unsigned int taken;
 	} nr;
 
@@ -1944,6 +1942,44 @@ static int current_may_throttle(void)
 	return !(current->flags & PF_LOCAL_THROTTLE);
 }
 
+static void handle_reclaim_writeback(unsigned long nr_taken,
+				     struct pglist_data *pgdat,
+				     struct scan_control *sc,
+				     struct reclaim_stat *stat)
+{
+	/*
+	 * If dirty folios are scanned that are not queued for IO, it
+	 * implies that flushers are not doing their job. This can
+	 * happen when memory pressure pushes dirty folios to the end of
+	 * the LRU before the dirty limits are breached and the dirty
+	 * data has expired. It can also happen when the proportion of
+	 * dirty folios grows not through writes but through memory
+	 * pressure reclaiming all the clean cache. And in some cases,
+	 * the flushers simply cannot keep up with the allocation
+	 * rate. Nudge the flusher threads in case they are asleep.
+	 */
+	if (stat->nr_unqueued_dirty == nr_taken) {
+		wakeup_flusher_threads(WB_REASON_VMSCAN);
+		/*
+		 * For cgroupv1 dirty throttling is achieved by waking up
+		 * the kernel flusher here and later waiting on folios
+		 * which are in writeback to finish (see shrink_folio_list()).
+		 *
+		 * Flusher may not be able to issue writeback quickly
+		 * enough for cgroupv1 writeback throttling to work
+		 * on a large system.
+		 */
+		if (!writeback_throttling_sane(sc))
+			reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
+	}
+
+	sc->nr.dirty += stat->nr_dirty;
+	sc->nr.congested += stat->nr_congested;
+	sc->nr.writeback += stat->nr_writeback;
+	sc->nr.immediate += stat->nr_immediate;
+	sc->nr.taken += nr_taken;
+}
+
 /*
  * shrink_inactive_list() is a helper for shrink_node().  It returns the number
  * of reclaimed pages
@@ -2007,42 +2043,7 @@ static unsigned long shrink_inactive_list(unsigned long nr_to_scan,
 	lruvec_lock_irq(lruvec);
 	lru_note_cost_unlock_irq(lruvec, file, stat.nr_pageout,
 					nr_scanned - nr_reclaimed);
-
-	/*
-	 * If dirty folios are scanned that are not queued for IO, it
-	 * implies that flushers are not doing their job. This can
-	 * happen when memory pressure pushes dirty folios to the end of
-	 * the LRU before the dirty limits are breached and the dirty
-	 * data has expired. It can also happen when the proportion of
-	 * dirty folios grows not through writes but through memory
-	 * pressure reclaiming all the clean cache. And in some cases,
-	 * the flushers simply cannot keep up with the allocation
-	 * rate. Nudge the flusher threads in case they are asleep.
-	 */
-	if (stat.nr_unqueued_dirty == nr_taken) {
-		wakeup_flusher_threads(WB_REASON_VMSCAN);
-		/*
-		 * For cgroupv1 dirty throttling is achieved by waking up
-		 * the kernel flusher here and later waiting on folios
-		 * which are in writeback to finish (see shrink_folio_list()).
-		 *
-		 * Flusher may not be able to issue writeback quickly
-		 * enough for cgroupv1 writeback throttling to work
-		 * on a large system.
-		 */
-		if (!writeback_throttling_sane(sc))
-			reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
-	}
-
-	sc->nr.dirty += stat.nr_dirty;
-	sc->nr.congested += stat.nr_congested;
-	sc->nr.unqueued_dirty += stat.nr_unqueued_dirty;
-	sc->nr.writeback += stat.nr_writeback;
-	sc->nr.immediate += stat.nr_immediate;
-	sc->nr.taken += nr_taken;
-	if (file)
-		sc->nr.file_taken += nr_taken;
-
+	handle_reclaim_writeback(nr_taken, pgdat, sc, &stat);
 	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
 			nr_scanned, nr_reclaimed, &stat, sc->priority, file);
 	return nr_reclaimed;
@@ -3220,7 +3221,7 @@ static int folio_update_gen(struct folio *folio, int gen)
 }
 
 /* protect pages accessed multiple times through file descriptors */
-static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclaiming)
+static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio)
 {
 	int type = folio_is_file_lru(folio);
 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
@@ -3239,9 +3240,6 @@ static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio, bool reclai
 
 		new_flags = old_flags & ~(LRU_GEN_MASK | LRU_REFS_FLAGS);
 		new_flags |= (new_gen + 1UL) << LRU_GEN_PGOFF;
-		/* for folio_end_writeback() */
-		if (reclaiming)
-			new_flags |= BIT(PG_reclaim);
 	} while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags));
 
 	lru_gen_update_size(lruvec, folio, old_gen, new_gen);
@@ -3855,7 +3853,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 			VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) != type, folio);
 			VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) != zone, folio);
 
-			new_gen = folio_inc_gen(lruvec, folio, false);
+			new_gen = folio_inc_gen(lruvec, folio);
 			list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]);
 
 			/* don't count the workingset being lazily promoted */
@@ -3878,10 +3876,9 @@ static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness)
 	return true;
 }
 
-static bool try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
+static void try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
 {
 	int gen, type, zone;
-	bool success = false;
 	bool seq_inc_flag = false;
 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
 	DEFINE_MIN_SEQ(lruvec);
@@ -3907,11 +3904,10 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
 
 	/*
 	 * If min_seq[type] of both anonymous and file is not increased,
-	 * we can directly return false to avoid unnecessary checking
-	 * overhead later.
+	 * return here to avoid unnecessary checking overhead later.
 	 */
 	if (!seq_inc_flag)
-		return success;
+		return;
 
 	/* see the comment on lru_gen_folio */
 	if (swappiness && swappiness <= MAX_SWAPPINESS) {
@@ -3929,10 +3925,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, int swappiness)
 
 		reset_ctrl_pos(lruvec, type, true);
 		WRITE_ONCE(lrugen->min_seq[type], min_seq[type]);
-		success = true;
 	}
-
-	return success;
 }
 
 static bool inc_max_seq(struct lruvec *lruvec, unsigned long seq, int swappiness)
@@ -4084,27 +4077,33 @@ static void set_initial_priority(struct pglist_data *pgdat, struct scan_control
 	sc->priority = clamp(priority, DEF_PRIORITY / 2, DEF_PRIORITY);
 }
 
-static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
+static unsigned long lruvec_evictable_size(struct lruvec *lruvec, int swappiness)
 {
 	int gen, type, zone;
-	unsigned long total = 0;
-	int swappiness = get_swappiness(lruvec, sc);
+	unsigned long seq, total = 0;
 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
-	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 	DEFINE_MAX_SEQ(lruvec);
 	DEFINE_MIN_SEQ(lruvec);
 
 	for_each_evictable_type(type, swappiness) {
-		unsigned long seq;
-
 		for (seq = min_seq[type]; seq <= max_seq; seq++) {
 			gen = lru_gen_from_seq(seq);
-
 			for (zone = 0; zone < MAX_NR_ZONES; zone++)
 				total += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
 		}
 	}
 
+	return total;
+}
+
+static bool lruvec_is_sizable(struct lruvec *lruvec, struct scan_control *sc)
+{
+	unsigned long total;
+	int swappiness = get_swappiness(lruvec, sc);
+	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
+
+	total = lruvec_evictable_size(lruvec, swappiness);
+
 	/* whether the size is big enough to be helpful */
 	return mem_cgroup_online(memcg) ? (total >> sc->priority) : total;
 }
@@ -4577,7 +4576,6 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
 		       int tier_idx)
 {
 	bool success;
-	bool dirty, writeback;
 	int gen = folio_lru_gen(folio);
 	int type = folio_is_file_lru(folio);
 	int zone = folio_zonenum(folio);
@@ -4607,7 +4605,7 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
 
 	/* protected */
 	if (tier > tier_idx || refs + workingset == BIT(LRU_REFS_WIDTH) + 1) {
-		gen = folio_inc_gen(lruvec, folio, false);
+		gen = folio_inc_gen(lruvec, folio);
 		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
 
 		/* don't count the workingset being lazily promoted */
@@ -4622,26 +4620,11 @@ static bool sort_folio(struct lruvec *lruvec, struct folio *folio, struct scan_c
 
 	/* ineligible */
 	if (zone > sc->reclaim_idx) {
-		gen = folio_inc_gen(lruvec, folio, false);
+		gen = folio_inc_gen(lruvec, folio);
 		list_move_tail(&folio->lru, &lrugen->folios[gen][type][zone]);
 		return true;
 	}
 
-	dirty = folio_test_dirty(folio);
-	writeback = folio_test_writeback(folio);
-	if (type == LRU_GEN_FILE && dirty) {
-		sc->nr.file_taken += delta;
-		if (!writeback)
-			sc->nr.unqueued_dirty += delta;
-	}
-
-	/* waiting for writeback */
-	if (writeback || (type == LRU_GEN_FILE && dirty)) {
-		gen = folio_inc_gen(lruvec, folio, true);
-		list_move(&folio->lru, &lrugen->folios[gen][type][zone]);
-		return true;
-	}
-
 	return false;
 }
 
@@ -4649,12 +4632,6 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
 {
 	bool success;
 
-	/* swap constrained */
-	if (!(sc->gfp_mask & __GFP_IO) &&
-	    (folio_test_dirty(folio) ||
-	     (folio_test_anon(folio) && !folio_test_swapcache(folio))))
-		return false;
-
 	/* raced with release_pages() */
 	if (!folio_try_get(folio))
 		return false;
@@ -4669,9 +4646,6 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
 	if (!folio_test_referenced(folio))
 		set_mask_bits(&folio->flags.f, LRU_REFS_MASK, 0);
 
-	/* for shrink_folio_list() */
-	folio_clear_reclaim(folio);
-
 	success = lru_gen_del_folio(lruvec, folio, true);
 	VM_WARN_ON_ONCE_FOLIO(!success, folio);
 
@@ -4680,7 +4654,7 @@ static bool isolate_folio(struct lruvec *lruvec, struct folio *folio, struct sca
 
 static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 		       struct scan_control *sc, int type, int tier,
-		       struct list_head *list)
+		       struct list_head *list, int *isolatedp)
 {
 	int i;
 	int gen;
@@ -4689,10 +4663,10 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 	int scanned = 0;
 	int isolated = 0;
 	int skipped = 0;
-	int scan_batch = min(nr_to_scan, MAX_LRU_BATCH);
-	int remaining = scan_batch;
+	unsigned long remaining = nr_to_scan;
 	struct lru_gen_folio *lrugen = &lruvec->lrugen;
 
+	VM_WARN_ON_ONCE(nr_to_scan > MAX_LRU_BATCH);
 	VM_WARN_ON_ONCE(!list_empty(list));
 
 	if (get_nr_gens(lruvec, type) == MIN_NR_GENS)
@@ -4745,16 +4719,12 @@ static int scan_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 	mod_lruvec_state(lruvec, item, isolated);
 	mod_lruvec_state(lruvec, PGREFILL, sorted);
 	mod_lruvec_state(lruvec, PGSCAN_ANON + type, isolated);
-	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, scan_batch,
+	trace_mm_vmscan_lru_isolate(sc->reclaim_idx, sc->order, nr_to_scan,
 				scanned, skipped, isolated,
 				type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
-	if (type == LRU_GEN_FILE)
-		sc->nr.file_taken += isolated;
-	/*
-	 * There might not be eligible folios due to reclaim_idx. Check the
-	 * remaining to prevent livelock if it's not making progress.
-	 */
-	return isolated || !remaining ? scanned : 0;
+
+	*isolatedp = isolated;
+	return scanned;
 }
 
 static int get_tier_idx(struct lruvec *lruvec, int type)
@@ -4798,33 +4768,41 @@ static int get_type_to_scan(struct lruvec *lruvec, int swappiness)
 
 static int isolate_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 			  struct scan_control *sc, int swappiness,
-			  int *type_scanned, struct list_head *list)
+			  struct list_head *list, int *isolated,
+			  int *isolate_type, int *isolate_scanned)
 {
 	int i;
+	int total_scanned = 0;
 	int type = get_type_to_scan(lruvec, swappiness);
 
 	for_each_evictable_type(i, swappiness) {
 		int scanned;
 		int tier = get_tier_idx(lruvec, type);
 
-		*type_scanned = type;
-
-		scanned = scan_folios(nr_to_scan, lruvec, sc, type, tier, list);
-		if (scanned)
-			return scanned;
+		scanned = scan_folios(nr_to_scan, lruvec, sc,
+				      type, tier, list, isolated);
 
-		type = !type;
+		total_scanned += scanned;
+		if (*isolated) {
+			*isolate_type = type;
+			*isolate_scanned = scanned;
+			break;
+		}
+		/*
+		 * If scanned > 0 and isolated == 0, avoid falling back to the
+		 * other type, as this type remains sufficient. Falling back
+		 * too readily can disrupt the positive_ctrl_err() bias.
+		 */
+		if (!scanned)
+			type = !type;
 	}
 
-	return 0;
+	return total_scanned;
 }
 
 static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 			struct scan_control *sc, int swappiness)
 {
-	int type;
-	int scanned;
-	int reclaimed;
 	LIST_HEAD(list);
 	LIST_HEAD(clean);
 	struct folio *folio;
@@ -4832,19 +4810,23 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 	enum node_stat_item item;
 	struct reclaim_stat stat;
 	struct lru_gen_mm_walk *walk;
+	int scanned, reclaimed;
+	int isolated = 0, type, type_scanned;
 	bool skip_retry = false;
-	struct lru_gen_folio *lrugen = &lruvec->lrugen;
 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
 
 	lruvec_lock_irq(lruvec);
 
-	scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness, &type, &list);
+	/* In case folio deletion left empty old gens, flush them */
+	try_to_inc_min_seq(lruvec, swappiness);
 
-	scanned += try_to_inc_min_seq(lruvec, swappiness);
+	scanned = isolate_folios(nr_to_scan, lruvec, sc, swappiness,
+				 &list, &isolated, &type, &type_scanned);
 
-	if (evictable_min_seq(lrugen->min_seq, swappiness) + MIN_NR_GENS > lrugen->max_seq)
-		scanned = 0;
+	/* Scanning may have emptied the oldest gen, flush it */
+	if (scanned)
+		try_to_inc_min_seq(lruvec, swappiness);
 
 	lruvec_unlock_irq(lruvec);
 
@@ -4852,10 +4834,12 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 		return scanned;
 retry:
 	reclaimed = shrink_folio_list(&list, pgdat, sc, &stat, false, memcg);
-	sc->nr.unqueued_dirty += stat.nr_unqueued_dirty;
 	sc->nr_reclaimed += reclaimed;
+	/* Retry pass is only meant for clean folios without new isolation */
+	if (isolated)
+		handle_reclaim_writeback(isolated, pgdat, sc, &stat);
 	trace_mm_vmscan_lru_shrink_inactive(pgdat->node_id,
-			scanned, reclaimed, &stat, sc->priority,
+			type_scanned, reclaimed, &stat, sc->priority,
 			type ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON);
 
 	list_for_each_entry_safe_reverse(folio, next, &list, lru) {
@@ -4900,6 +4884,7 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 
 	if (!list_empty(&list)) {
 		skip_retry = true;
+		isolated = 0;
 		goto retry;
 	}
 
@@ -4907,63 +4892,37 @@ static int evict_folios(unsigned long nr_to_scan, struct lruvec *lruvec,
 }
 
 static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq,
-			     int swappiness, unsigned long *nr_to_scan)
+			     struct scan_control *sc, int swappiness)
 {
-	int gen, type, zone;
-	unsigned long size = 0;
-	struct lru_gen_folio *lrugen = &lruvec->lrugen;
 	DEFINE_MIN_SEQ(lruvec);
 
-	*nr_to_scan = 0;
 	/* have to run aging, since eviction is not possible anymore */
 	if (evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS > max_seq)
 		return true;
 
-	for_each_evictable_type(type, swappiness) {
-		unsigned long seq;
-
-		for (seq = min_seq[type]; seq <= max_seq; seq++) {
-			gen = lru_gen_from_seq(seq);
-
-			for (zone = 0; zone < MAX_NR_ZONES; zone++)
-				size += max(READ_ONCE(lrugen->nr_pages[gen][type][zone]), 0L);
-		}
-	}
+	/* try to avoid aging, do gentle reclaim at the default priority */
+	if (sc->priority == DEF_PRIORITY)
+		return false;
 
-	*nr_to_scan = size;
 	/* better to run aging even though eviction is still possible */
 	return evictable_min_seq(min_seq, swappiness) + MIN_NR_GENS == max_seq;
 }
 
-/*
- * For future optimizations:
- * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
- *    reclaim.
- */
-static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, int swappiness)
+static long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc,
+			   struct mem_cgroup *memcg, int swappiness)
 {
-	bool success;
-	unsigned long nr_to_scan;
-	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
-	DEFINE_MAX_SEQ(lruvec);
+	unsigned long nr_to_scan, evictable;
 
-	if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg))
-		return -1;
-
-	success = should_run_aging(lruvec, max_seq, swappiness, &nr_to_scan);
+	evictable = lruvec_evictable_size(lruvec, swappiness);
 
 	/* try to scrape all its memory if this memcg was deleted */
-	if (nr_to_scan && !mem_cgroup_online(memcg))
-		return nr_to_scan;
-
-	nr_to_scan = apply_proportional_protection(memcg, sc, nr_to_scan);
+	if (!mem_cgroup_online(memcg))
+		return evictable;
 
-	/* try to get away with not aging at the default priority */
-	if (!success || sc->priority == DEF_PRIORITY)
-		return nr_to_scan >> sc->priority;
+	nr_to_scan = apply_proportional_protection(memcg, sc, evictable);
+	nr_to_scan >>= sc->priority;
 
-	/* stop scanning this lruvec as it's low on cold folios */
-	return try_to_inc_max_seq(lruvec, max_seq, swappiness, false) ? -1 : 0;
+	return nr_to_scan;
 }
 
 static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
@@ -4993,62 +4952,59 @@ static bool should_abort_scan(struct lruvec *lruvec, struct scan_control *sc)
 	return true;
 }
 
+/*
+ * For future optimizations:
+ * 1. Defer try_to_inc_max_seq() to workqueues to reduce latency for memcg
+ *    reclaim.
+ */
 static bool try_to_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
 {
-	long nr_to_scan;
-	unsigned long scanned = 0;
+	bool need_rotate = false, should_age = false;
+	long nr_batch, nr_to_scan;
 	int swappiness = get_swappiness(lruvec, sc);
+	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
 
-	while (true) {
+	nr_to_scan = get_nr_to_scan(lruvec, sc, memcg, swappiness);
+	while (nr_to_scan > 0) {
 		int delta;
+		DEFINE_MAX_SEQ(lruvec);
 
-		nr_to_scan = get_nr_to_scan(lruvec, sc, swappiness);
-		if (nr_to_scan <= 0)
+		if (mem_cgroup_below_min(sc->target_mem_cgroup, memcg)) {
+			need_rotate = true;
 			break;
+		}
 
-		delta = evict_folios(nr_to_scan, lruvec, sc, swappiness);
-		if (!delta)
-			break;
+		if (should_run_aging(lruvec, max_seq, sc, swappiness)) {
+			if (try_to_inc_max_seq(lruvec, max_seq, swappiness, false))
+				need_rotate = true;
+			should_age = true;
+		}
 
-		scanned += delta;
-		if (scanned >= nr_to_scan)
+		nr_batch = min(nr_to_scan, MIN_LRU_BATCH);
+		delta = evict_folios(nr_batch, lruvec, sc, swappiness);
+		if (!delta)
 			break;
 
 		if (should_abort_scan(lruvec, sc))
 			break;
 
-		cond_resched();
-	}
-
-	/*
-	 * If too many file cache in the coldest generation can't be evicted
-	 * due to being dirty, wake up the flusher.
-	 */
-	if (sc->nr.unqueued_dirty && sc->nr.unqueued_dirty == sc->nr.file_taken) {
-		struct pglist_data *pgdat = lruvec_pgdat(lruvec);
-
-		wakeup_flusher_threads(WB_REASON_VMSCAN);
-
 		/*
-		 * For cgroupv1 dirty throttling is achieved by waking up
-		 * the kernel flusher here and later waiting on folios
-		 * which are in writeback to finish (see shrink_folio_list()).
-		 *
-		 * Flusher may not be able to issue writeback quickly
-		 * enough for cgroupv1 writeback throttling to work
-		 * on a large system.
+		 * Root reclaim needs rotation when low on cold folio for better
+		 * fairness. Cgroup reclaim gets fairness from the iterator.
 		 */
-		if (!writeback_throttling_sane(sc))
-			reclaim_throttle(pgdat, VMSCAN_THROTTLE_WRITEBACK);
+		if (root_reclaim(sc) && should_age)
+			break;
+
+		nr_to_scan -= delta;
+		cond_resched();
 	}
 
-	/* whether this lruvec should be rotated */
-	return nr_to_scan < 0;
+	return need_rotate;
 }
 
 static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
 {
-	bool success;
+	bool need_rotate;
 	unsigned long scanned = sc->nr_scanned;
 	unsigned long reclaimed = sc->nr_reclaimed;
 	struct mem_cgroup *memcg = lruvec_memcg(lruvec);
@@ -5066,20 +5022,20 @@ static int shrink_one(struct lruvec *lruvec, struct scan_control *sc)
 		memcg_memory_event(memcg, MEMCG_LOW);
 	}
 
-	success = try_to_shrink_lruvec(lruvec, sc);
+	need_rotate = try_to_shrink_lruvec(lruvec, sc);
 
 	shrink_slab(sc->gfp_mask, pgdat->node_id, memcg, sc->priority);
 
 	if (!sc->proactive)
-		vmpressure(sc->gfp_mask, memcg, false, sc->nr_scanned - scanned,
-			   sc->nr_reclaimed - reclaimed);
+		vmpressure(sc->gfp_mask, sc->order, memcg, false,
+			   sc->nr_scanned - scanned, sc->nr_reclaimed - reclaimed);
 
 	flush_reclaim_state(sc);
 
-	if (success && mem_cgroup_online(memcg))
+	if (need_rotate && mem_cgroup_online(memcg))
 		return MEMCG_LRU_YOUNG;
 
-	if (!success && lruvec_is_sizable(lruvec, sc))
+	if (!need_rotate && lruvec_is_sizable(lruvec, sc))
 		return 0;
 
 	/* one retry if offlined or too small */
@@ -5631,6 +5587,7 @@ static int run_aging(struct lruvec *lruvec, unsigned long seq,
 static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_control *sc,
 			int swappiness, unsigned long nr_to_reclaim)
 {
+	int nr_batch;
 	DEFINE_MAX_SEQ(lruvec);
 
 	if (seq + MIN_NR_GENS > max_seq)
@@ -5647,8 +5604,8 @@ static int run_eviction(struct lruvec *lruvec, unsigned long seq, struct scan_co
 		if (sc->nr_reclaimed >= nr_to_reclaim)
 			return 0;
 
-		if (!evict_folios(nr_to_reclaim - sc->nr_reclaimed, lruvec, sc,
-				  swappiness))
+		nr_batch = min(nr_to_reclaim - sc->nr_reclaimed, MAX_LRU_BATCH);
+		if (!evict_folios(nr_batch, lruvec, sc, swappiness))
 			return 0;
 
 		cond_resched();
@@ -6175,7 +6132,7 @@ static void shrink_node_memcgs(pg_data_t *pgdat, struct scan_control *sc)
 
 		/* Record the group's reclaim efficiency */
 		if (!sc->proactive)
-			vmpressure(sc->gfp_mask, memcg, false,
+			vmpressure(sc->gfp_mask, sc->order, memcg, false,
 				   sc->nr_scanned - scanned,
 				   sc->nr_reclaimed - reclaimed);
 
@@ -6220,7 +6177,7 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
 
 	/* Record the subtree's reclaim efficiency */
 	if (!sc->proactive)
-		vmpressure(sc->gfp_mask, sc->target_mem_cgroup, true,
+		vmpressure(sc->gfp_mask, sc->order, sc->target_mem_cgroup, true,
 			   sc->nr_scanned - nr_scanned, nr_node_reclaimed);
 
 	if (nr_node_reclaimed)
@@ -6359,7 +6316,7 @@ static void consider_reclaim_throttle(pg_data_t *pgdat, struct scan_control *sc)
 	if (current_is_kswapd() || cgroup_reclaim(sc))
 		return;
 
-	/* Throttle if making no progress at high prioities. */
+	/* Throttle if making no progress at high priorities. */
 	if (sc->priority == 1 && !sc->nr_reclaimed)
 		reclaim_throttle(pgdat, VMSCAN_THROTTLE_NOPROGRESS);
 }
@@ -7121,6 +7078,8 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
 		.may_unmap = 1,
 	};
 
+	trace_mm_vmscan_balance_pgdat_begin(pgdat->node_id, order,
+					    highest_zoneidx);
 	set_task_reclaim_state(current, &sc.reclaim_state);
 	psi_memstall_enter(&pflags);
 	__fs_reclaim_acquire(_THIS_IP_);
@@ -7222,7 +7181,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
 
 		/*
 		 * There should be no need to raise the scanning priority if
-		 * enough pages are already being scanned that that high
+		 * enough pages are already being scanned that the high
 		 * watermark would be met at 100% efficiency.
 		 */
 		if (kswapd_shrink_node(pgdat, &sc))
@@ -7314,6 +7273,9 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
 	psi_memstall_leave(&pflags);
 	set_task_reclaim_state(current, NULL);
 
+	trace_mm_vmscan_balance_pgdat_end(pgdat->node_id, sc.order,
+					  highest_zoneidx, sc.nr_reclaimed);
+
 	/*
 	 * Return the order kswapd stopped reclaiming at as
 	 * prepare_kswapd_sleep() takes it into account. If another caller
diff --git a/net/appletalk/aarp.c b/net/appletalk/aarp.c
index e7315c01a299d..30493ea3c0107 100644
--- a/net/appletalk/aarp.c
+++ b/net/appletalk/aarp.c
@@ -542,6 +542,11 @@ int aarp_send_ddp(struct net_device *dev, struct sk_buff *skb,
 		struct ddpehdr *ddp = (struct ddpehdr *)skb->data;
 		int ft = 2;
 
+		if (!at) {
+			kfree_skb(skb);
+			return NET_XMIT_DROP;
+		}
+
 		/*
 		 * Compressible ?
 		 *
diff --git a/net/batman-adv/distributed-arp-table.c b/net/batman-adv/distributed-arp-table.c
index 3efc4cf50b469..0a8bd95e2f99e 100644
--- a/net/batman-adv/distributed-arp-table.c
+++ b/net/batman-adv/distributed-arp-table.c
@@ -696,6 +696,9 @@ static bool batadv_dat_forward_data(struct batadv_priv *bat_priv,
 			goto free_orig;
 
 		tmp_skb = pskb_copy_for_clone(skb, GFP_ATOMIC);
+		if (!tmp_skb)
+			goto free_neigh;
+
 		if (!batadv_send_skb_prepare_unicast_4addr(bat_priv, tmp_skb,
 							   cand[i].orig_node,
 							   packet_subtype)) {
diff --git a/net/batman-adv/fragmentation.c b/net/batman-adv/fragmentation.c
index f4e45cc258164..4a594aa2ebf66 100644
--- a/net/batman-adv/fragmentation.c
+++ b/net/batman-adv/fragmentation.c
@@ -17,6 +17,7 @@
 #include <linux/lockdep.h>
 #include <linux/minmax.h>
 #include <linux/netdevice.h>
+#include <linux/overflow.h>
 #include <linux/skbuff.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
@@ -80,9 +81,9 @@ void batadv_frag_purge_orig(struct batadv_orig_node *orig_node,
  *
  * Return: the maximum size of payload that can be fragmented.
  */
-static int batadv_frag_size_limit(void)
+static size_t batadv_frag_size_limit(void)
 {
-	int limit = BATADV_FRAG_MAX_FRAG_SIZE;
+	size_t limit = BATADV_FRAG_MAX_FRAG_SIZE;
 
 	limit -= sizeof(struct batadv_frag_packet);
 	limit *= BATADV_FRAG_MAX_FRAGMENTS;
@@ -143,7 +144,9 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
 	struct batadv_frag_packet *frag_packet;
 	u8 bucket;
 	u16 seqno, hdr_size = sizeof(struct batadv_frag_packet);
+	bool overflow = false;
 	bool ret = false;
+	size_t data_len;
 
 	/* Linearize packet to avoid linearizing 16 packets in a row when doing
 	 * the later merge. Non-linear merge should be added to remove this
@@ -153,6 +156,7 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
 		goto err;
 
 	frag_packet = (struct batadv_frag_packet *)skb->data;
+	data_len = skb->len - hdr_size;
 	seqno = ntohs(frag_packet->seqno);
 	bucket = seqno % BATADV_FRAG_BUFFER_COUNT;
 
@@ -171,7 +175,7 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
 	spin_lock_bh(&chain->lock);
 	if (batadv_frag_init_chain(chain, seqno)) {
 		hlist_add_head(&frag_entry_new->list, &chain->fragment_list);
-		chain->size = skb->len - hdr_size;
+		chain->size = data_len;
 		chain->timestamp = jiffies;
 		chain->total_size = ntohs(frag_packet->total_size);
 		ret = true;
@@ -188,7 +192,11 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
 		if (frag_entry_curr->no < frag_entry_new->no) {
 			hlist_add_before(&frag_entry_new->list,
 					 &frag_entry_curr->list);
-			chain->size += skb->len - hdr_size;
+
+			if (check_add_overflow(chain->size, data_len,
+					       &chain->size))
+				overflow = true;
+
 			chain->timestamp = jiffies;
 			ret = true;
 			goto out;
@@ -201,13 +209,16 @@ static bool batadv_frag_insert_packet(struct batadv_orig_node *orig_node,
 	/* Reached the end of the list, so insert after 'frag_entry_last'. */
 	if (likely(frag_entry_last)) {
 		hlist_add_behind(&frag_entry_new->list, &frag_entry_last->list);
-		chain->size += skb->len - hdr_size;
+
+		if (check_add_overflow(chain->size, data_len, &chain->size))
+			overflow = true;
+
 		chain->timestamp = jiffies;
 		ret = true;
 	}
 
 out:
-	if (chain->size > batadv_frag_size_limit() ||
+	if (overflow || chain->size > batadv_frag_size_limit() ||
 	    chain->total_size != ntohs(frag_packet->total_size) ||
 	    chain->total_size > batadv_frag_size_limit()) {
 		/* Clear chain if total size of either the list or the packet
@@ -293,6 +304,31 @@ batadv_frag_merge_packets(struct hlist_head *chain)
 	return skb_out;
 }
 
+/**
+ * batadv_skb_is_frag() - check if newly merged skb is gain a unicast packet
+ * @skb: newly merged skb
+ *
+ * Return: if newly skb is of type BATADV_UNICAST_FRAG
+ */
+static bool batadv_skb_is_frag(struct sk_buff *skb)
+{
+	struct batadv_ogm_packet *batadv_ogm_packet;
+
+	/* packet should hold at least type and version */
+	if (unlikely(!pskb_may_pull(skb, 2)))
+		return false;
+
+	batadv_ogm_packet = (struct batadv_ogm_packet *)skb->data;
+
+	if (batadv_ogm_packet->version != BATADV_COMPAT_VERSION)
+		return false;
+
+	if (batadv_ogm_packet->packet_type != BATADV_UNICAST_FRAG)
+		return false;
+
+	return true;
+}
+
 /**
  * batadv_frag_skb_buffer() - buffer fragment for later merge
  * @skb: skb to buffer
@@ -326,6 +362,16 @@ bool batadv_frag_skb_buffer(struct sk_buff **skb,
 	if (!skb_out)
 		goto out_err;
 
+	/* fragment in fragment is not allowed. otherwise it is possible
+	 * to exhaust the stack when receiving a matryoshka-style
+	 * "fragments in a fragment packet"
+	 */
+	if (batadv_skb_is_frag(skb_out)) {
+		kfree_skb(skb_out);
+		skb_out = NULL;
+		goto out_err;
+	}
+
 out:
 	ret = true;
 out_err:
diff --git a/net/batman-adv/gateway_client.c b/net/batman-adv/gateway_client.c
index 51e9c081a2a4e..a9d0346e8332e 100644
--- a/net/batman-adv/gateway_client.c
+++ b/net/batman-adv/gateway_client.c
@@ -478,10 +478,14 @@ void batadv_gw_node_delete(struct batadv_priv *bat_priv,
  */
 void batadv_gw_node_free(struct batadv_priv *bat_priv)
 {
+	struct batadv_gw_node *curr_gw;
 	struct batadv_gw_node *gw_node;
 	struct hlist_node *node_tmp;
 
 	spin_lock_bh(&bat_priv->gw.list_lock);
+	curr_gw = rcu_replace_pointer(bat_priv->gw.curr_gw, NULL, true);
+	batadv_gw_node_put(curr_gw);
+
 	hlist_for_each_entry_safe(gw_node, node_tmp,
 				  &bat_priv->gw.gateway_list, list) {
 		hlist_del_init_rcu(&gw_node->list);
diff --git a/net/batman-adv/tp_meter.c b/net/batman-adv/tp_meter.c
index 066c76113fc43..1fd1526059d8a 100644
--- a/net/batman-adv/tp_meter.c
+++ b/net/batman-adv/tp_meter.c
@@ -8,6 +8,7 @@
 #include "main.h"
 
 #include <linux/atomic.h>
+#include <linux/bug.h>
 #include <linux/build_bug.h>
 #include <linux/byteorder/generic.h>
 #include <linux/cache.h>
@@ -400,13 +401,7 @@ static void batadv_tp_sender_cleanup(struct batadv_tp_vars *tp_vars)
 	batadv_tp_list_detach(tp_vars);
 
 	/* kill the timer and remove its reference */
-	timer_delete_sync(&tp_vars->timer);
-	/* the worker might have rearmed itself therefore we kill it again. Note
-	 * that if the worker should run again before invoking the following
-	 * timer_delete(), it would not re-arm itself once again because the status
-	 * is OFF now
-	 */
-	timer_delete(&tp_vars->timer);
+	timer_shutdown_sync(&tp_vars->timer);
 	batadv_tp_vars_put(tp_vars);
 }
 
@@ -451,7 +446,7 @@ static void batadv_tp_sender_end(struct batadv_priv *bat_priv,
 static void batadv_tp_sender_shutdown(struct batadv_tp_vars *tp_vars,
 				      enum batadv_tp_meter_reason reason)
 {
-	if (!atomic_dec_and_test(&tp_vars->sending))
+	if (atomic_xchg(&tp_vars->sending, 0) != 1)
 		return;
 
 	tp_vars->reason = reason;
@@ -663,6 +658,9 @@ static void batadv_tp_recv_ack(struct batadv_priv *bat_priv,
 	if (unlikely(!tp_vars))
 		return;
 
+	if (unlikely(tp_vars->role != BATADV_TP_SENDER))
+		goto out;
+
 	if (unlikely(atomic_read(&tp_vars->sending) == 0))
 		goto out;
 
@@ -885,7 +883,7 @@ static int batadv_tp_send(void *arg)
 				   "Meter: %s() cannot send packets (%d)\n",
 				   __func__, err);
 			/* ensure nobody else tries to stop the thread now */
-			if (atomic_dec_and_test(&tp_vars->sending))
+			if (atomic_xchg(&tp_vars->sending, 0) == 1)
 				tp_vars->reason = err;
 			break;
 		}
@@ -1100,12 +1098,16 @@ void batadv_tp_stop(struct batadv_priv *bat_priv, const u8 *dst,
 	if (!tp_vars) {
 		batadv_dbg(BATADV_DBG_TP_METER, bat_priv,
 			   "Meter: trying to interrupt an already over connection\n");
-		goto out;
+		goto out_put_orig_node;
 	}
 
+	if (unlikely(tp_vars->role != BATADV_TP_SENDER))
+		goto out_put_tp_vars;
+
 	batadv_tp_sender_shutdown(tp_vars, return_value);
+out_put_tp_vars:
 	batadv_tp_vars_put(tp_vars);
-out:
+out_put_orig_node:
 	batadv_orig_node_put(orig_node);
 }
 
@@ -1156,6 +1158,9 @@ static void batadv_tp_receiver_shutdown(struct timer_list *t)
 	spin_unlock_bh(&tp_vars->unacked_lock);
 
 	/* drop reference of timer */
+	if (WARN_ON(atomic_xchg(&tp_vars->receiving, 0) != 1))
+		return;
+
 	batadv_tp_vars_put(tp_vars);
 }
 
@@ -1374,6 +1379,7 @@ batadv_tp_init_recv(struct batadv_priv *bat_priv,
 
 	ether_addr_copy(tp_vars->other_end, icmp->orig);
 	tp_vars->role = BATADV_TP_RECEIVER;
+	atomic_set(&tp_vars->receiving, 1);
 	memcpy(tp_vars->session, icmp->session, sizeof(tp_vars->session));
 	tp_vars->last_recv = BATADV_TP_FIRST_SEQ;
 	tp_vars->bat_priv = bat_priv;
@@ -1546,8 +1552,12 @@ void batadv_tp_stop_all(struct batadv_priv *bat_priv)
 			break;
 		case BATADV_TP_RECEIVER:
 			batadv_tp_list_detach(tp_var);
-			if (timer_shutdown_sync(&tp_var->timer))
-				batadv_tp_vars_put(tp_var);
+			timer_shutdown_sync(&tp_var->timer);
+
+			if (atomic_xchg(&tp_var->receiving, 0) != 1)
+				break;
+
+			batadv_tp_vars_put(tp_var);
 			break;
 		}
 
diff --git a/net/batman-adv/translation-table.c b/net/batman-adv/translation-table.c
index 05cddcf994f65..9f6e67771ffa8 100644
--- a/net/batman-adv/translation-table.c
+++ b/net/batman-adv/translation-table.c
@@ -797,24 +797,33 @@ batadv_tt_prepare_tvlv_global_data(struct batadv_orig_node *orig_node,
 				   s32 *tt_len)
 {
 	u16 num_vlan = 0;
-	u16 num_entries = 0;
 	u16 tvlv_len = 0;
 	unsigned int change_offset;
 	struct batadv_tvlv_tt_vlan_data *tt_vlan;
 	struct batadv_orig_node_vlan *vlan;
+	u16 total_entries = 0;
 	u8 *tt_change_ptr;
+	int vlan_entries;
+	u16 sum_entries;
 
 	spin_lock_bh(&orig_node->vlan_list_lock);
 	hlist_for_each_entry(vlan, &orig_node->vlan_list, list) {
+		vlan_entries = atomic_read(&vlan->tt.num_entries);
+
+		if (check_add_overflow(vlan_entries, total_entries, &sum_entries)) {
+			*tt_len = 0;
+			goto out;
+		}
+
+		total_entries = sum_entries;
 		num_vlan++;
-		num_entries += atomic_read(&vlan->tt.num_entries);
 	}
 
 	change_offset = struct_size(*tt_data, vlan_data, num_vlan);
 
 	/* if tt_len is negative, allocate the space needed by the full table */
 	if (*tt_len < 0)
-		*tt_len = batadv_tt_len(num_entries);
+		*tt_len = batadv_tt_len(total_entries);
 
 	if (change_offset > U16_MAX || *tt_len > U16_MAX - change_offset) {
 		*tt_len = 0;
@@ -835,14 +844,26 @@ batadv_tt_prepare_tvlv_global_data(struct batadv_orig_node *orig_node,
 	(*tt_data)->num_vlan = htons(num_vlan);
 
 	tt_vlan = (*tt_data)->vlan_data;
+	num_vlan = 0;
 	hlist_for_each_entry(vlan, &orig_node->vlan_list, list) {
+		vlan_entries = atomic_read(&vlan->tt.num_entries);
+		if (vlan_entries < 1)
+			continue;
+
 		tt_vlan->vid = htons(vlan->vid);
 		tt_vlan->crc = htonl(vlan->tt.crc);
 		tt_vlan->reserved = 0;
 
 		tt_vlan++;
+		num_vlan++;
 	}
 
+	/* recalculate in case number of VLANs reduced */
+	change_offset = struct_size(*tt_data, vlan_data, num_vlan);
+	tvlv_len = *tt_len + change_offset;
+
+	(*tt_data)->num_vlan = htons(num_vlan);
+
 	tt_change_ptr = (u8 *)*tt_data + change_offset;
 	*tt_change = (struct batadv_tvlv_tt_change *)tt_change_ptr;
 
@@ -877,21 +898,25 @@ batadv_tt_prepare_tvlv_local_data(struct batadv_priv *bat_priv,
 {
 	struct batadv_tvlv_tt_vlan_data *tt_vlan;
 	struct batadv_meshif_vlan *vlan;
+	size_t change_offset;
 	u16 num_vlan = 0;
-	u16 vlan_entries = 0;
 	u16 total_entries = 0;
 	u16 tvlv_len;
 	u8 *tt_change_ptr;
-	int change_offset;
+	int vlan_entries;
+	u16 sum_entries;
 
 	spin_lock_bh(&bat_priv->meshif_vlan_list_lock);
 	hlist_for_each_entry(vlan, &bat_priv->meshif_vlan_list, list) {
 		vlan_entries = atomic_read(&vlan->tt.num_entries);
-		if (vlan_entries < 1)
-			continue;
 
+		if (check_add_overflow(vlan_entries, total_entries, &sum_entries)) {
+			tvlv_len = 0;
+			goto out;
+		}
+
+		total_entries = sum_entries;
 		num_vlan++;
-		total_entries += vlan_entries;
 	}
 
 	change_offset = struct_size(*tt_data, vlan_data, num_vlan);
@@ -900,8 +925,10 @@ batadv_tt_prepare_tvlv_local_data(struct batadv_priv *bat_priv,
 	if (*tt_len < 0)
 		*tt_len = batadv_tt_len(total_entries);
 
-	tvlv_len = *tt_len;
-	tvlv_len += change_offset;
+	if (check_add_overflow(*tt_len, change_offset, &tvlv_len)) {
+		tvlv_len = 0;
+		goto out;
+	}
 
 	*tt_data = kmalloc(tvlv_len, GFP_ATOMIC);
 	if (!*tt_data) {
@@ -914,6 +941,7 @@ batadv_tt_prepare_tvlv_local_data(struct batadv_priv *bat_priv,
 	(*tt_data)->num_vlan = htons(num_vlan);
 
 	tt_vlan = (*tt_data)->vlan_data;
+	num_vlan = 0;
 	hlist_for_each_entry(vlan, &bat_priv->meshif_vlan_list, list) {
 		vlan_entries = atomic_read(&vlan->tt.num_entries);
 		if (vlan_entries < 1)
@@ -924,8 +952,15 @@ batadv_tt_prepare_tvlv_local_data(struct batadv_priv *bat_priv,
 		tt_vlan->reserved = 0;
 
 		tt_vlan++;
+		num_vlan++;
 	}
 
+	/* recalculate in case number of VLANs reduced */
+	change_offset = struct_size(*tt_data, vlan_data, num_vlan);
+	tvlv_len = *tt_len + change_offset;
+
+	(*tt_data)->num_vlan = htons(num_vlan);
+
 	tt_change_ptr = (u8 *)*tt_data + change_offset;
 	*tt_change = (struct batadv_tvlv_tt_change *)tt_change_ptr;
 
diff --git a/net/batman-adv/types.h b/net/batman-adv/types.h
index daa06f4211542..c8c3e8064f00c 100644
--- a/net/batman-adv/types.h
+++ b/net/batman-adv/types.h
@@ -301,7 +301,7 @@ struct batadv_frag_table_entry {
 	u16 seqno;
 
 	/** @size: accumulated size of packets in list */
-	u16 size;
+	size_t size;
 
 	/** @total_size: expected size of the assembled packet */
 	u16 total_size;
@@ -452,7 +452,7 @@ struct batadv_orig_node {
 	 * @tt_buff_len: length of the last tt changeset this node received
 	 *  from the orig node
 	 */
-	s16 tt_buff_len;
+	u16 tt_buff_len;
 
 	/** @tt_buff_lock: lock that protects tt_buff and tt_buff_len */
 	spinlock_t tt_buff_lock;
@@ -993,7 +993,7 @@ struct batadv_priv_tt {
 	 * @last_changeset_len: length of last tt changeset this host has
 	 *  generated
 	 */
-	s16 last_changeset_len;
+	u16 last_changeset_len;
 
 	/**
 	 * @last_changeset_lock: lock protecting last_changeset &
@@ -1323,6 +1323,9 @@ struct batadv_tp_vars {
 	/** @sending: sending binary semaphore: 1 if sending, 0 is not */
 	atomic_t sending;
 
+	/** @receiving: receiving binary semaphore: 1 if receiving, 0 is not */
+	atomic_t receiving;
+
 	/** @reason: reason for a stopped session */
 	enum batadv_tp_meter_reason reason;
 
diff --git a/net/bluetooth/6lowpan.c b/net/bluetooth/6lowpan.c
index 2f03b780b40d8..23a229ab6a33d 100644
--- a/net/bluetooth/6lowpan.c
+++ b/net/bluetooth/6lowpan.c
@@ -758,13 +758,33 @@ static inline struct l2cap_chan *chan_new_conn_cb(struct l2cap_chan *pchan)
 	return chan;
 }
 
+static void unregister_dev(struct lowpan_btle_dev *dev)
+{
+	struct hci_dev *hdev = READ_ONCE(dev->hdev);
+
+	/* If netdev holds last reference to hci_dev (its parent device), this
+	 * leads to theoretical cyclic locking on lowpan_unregister_netdev:
+	 *
+	 * rtnl_lock -> put_device(parent) -> hci_release_dev ->
+	 * destroy_workqueue -> hci_rx_work -> l2cap_recv_acldata ->
+	 * chan_ready_cb -> ifup -> rtnl_lock
+	 *
+	 * However, hci_rx_work is disabled in hci_unregister_dev, so this
+	 * should not occur. Make lockdep happy by postponing hdev release after
+	 * netdev put.
+	 */
+	hci_dev_hold(hdev);
+	lowpan_unregister_netdev(dev->netdev);
+	hci_dev_put(hdev);
+}
+
 static void delete_netdev(struct work_struct *work)
 {
 	struct lowpan_btle_dev *entry = container_of(work,
 						     struct lowpan_btle_dev,
 						     delete_netdev);
 
-	lowpan_unregister_netdev(entry->netdev);
+	unregister_dev(entry);
 
 	/* The entry pointer is deleted by the netdev destructor. */
 }
@@ -1250,6 +1270,7 @@ static void disconnect_devices(void)
 			break;
 
 		new_dev->netdev = entry->netdev;
+		new_dev->hdev = entry->hdev;
 		INIT_LIST_HEAD(&new_dev->list);
 
 		list_add_rcu(&new_dev->list, &devices);
@@ -1261,7 +1282,7 @@ static void disconnect_devices(void)
 		ifdown(entry->netdev);
 		BT_DBG("Unregistering netdev %s %p",
 		       entry->netdev->name, entry->netdev);
-		lowpan_unregister_netdev(entry->netdev);
+		unregister_dev(entry);
 		kfree(entry);
 	}
 }
diff --git a/net/bluetooth/Kconfig b/net/bluetooth/Kconfig
index ee6457d1a5eed..d250e94e90eba 100644
--- a/net/bluetooth/Kconfig
+++ b/net/bluetooth/Kconfig
@@ -9,12 +9,8 @@ menuconfig BT
 	depends on RFKILL || !RFKILL
 	select CRC16
 	select CRYPTO
-	select CRYPTO_SKCIPHER
 	select CRYPTO_LIB_AES
-	imply CRYPTO_AES
-	select CRYPTO_CMAC
-	select CRYPTO_ECB
-	select CRYPTO_SHA256
+	select CRYPTO_LIB_AES_CBC_MACS
 	select CRYPTO_ECDH
 	help
 	  Bluetooth is low-cost, low-power, short-range wireless technology.
diff --git a/net/bluetooth/af_bluetooth.c b/net/bluetooth/af_bluetooth.c
index 33d053d634072..1a6aa3f8d4d61 100644
--- a/net/bluetooth/af_bluetooth.c
+++ b/net/bluetooth/af_bluetooth.c
@@ -154,6 +154,7 @@ struct sock *bt_sock_alloc(struct net *net, struct socket *sock,
 
 	sock_init_data(sock, sk);
 	INIT_LIST_HEAD(&bt_sk(sk)->accept_q);
+	spin_lock_init(&bt_sk(sk)->accept_q_lock);
 
 	sock_reset_flag(sk, SOCK_ZAPPED);
 
@@ -214,6 +215,7 @@ void bt_accept_enqueue(struct sock *parent, struct sock *sk, bool bh)
 {
 	const struct cred *old_cred;
 	struct pid *old_pid;
+	struct bt_sock *par = bt_sk(parent);
 
 	BT_DBG("parent %p, sk %p", parent, sk);
 
@@ -224,9 +226,13 @@ void bt_accept_enqueue(struct sock *parent, struct sock *sk, bool bh)
 	else
 		lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
 
-	list_add_tail(&bt_sk(sk)->accept_q, &bt_sk(parent)->accept_q);
 	bt_sk(sk)->parent = parent;
 
+	spin_lock_bh(&par->accept_q_lock);
+	list_add_tail(&bt_sk(sk)->accept_q, &par->accept_q);
+	sk_acceptq_added(parent);
+	spin_unlock_bh(&par->accept_q_lock);
+
 	/* Copy credentials from parent since for incoming connections the
 	 * socket is allocated by the kernel.
 	 */
@@ -244,8 +250,6 @@ void bt_accept_enqueue(struct sock *parent, struct sock *sk, bool bh)
 		bh_unlock_sock(sk);
 	else
 		release_sock(sk);
-
-	sk_acceptq_added(parent);
 }
 EXPORT_SYMBOL(bt_accept_enqueue);
 
@@ -254,45 +258,72 @@ EXPORT_SYMBOL(bt_accept_enqueue);
  */
 void bt_accept_unlink(struct sock *sk)
 {
+	struct sock *parent = bt_sk(sk)->parent;
+
 	BT_DBG("sk %p state %d", sk, sk->sk_state);
 
+	spin_lock_bh(&bt_sk(parent)->accept_q_lock);
 	list_del_init(&bt_sk(sk)->accept_q);
-	sk_acceptq_removed(bt_sk(sk)->parent);
+	sk_acceptq_removed(parent);
+	spin_unlock_bh(&bt_sk(parent)->accept_q_lock);
 	bt_sk(sk)->parent = NULL;
 	sock_put(sk);
 }
 EXPORT_SYMBOL(bt_accept_unlink);
 
+static struct sock *bt_accept_get(struct sock *parent, struct sock *sk)
+{
+	struct bt_sock *bt = bt_sk(parent);
+	struct sock *next = NULL;
+
+	/* accept_q is modified from child teardown paths too, so take a
+	 * temporary reference before dropping the queue lock.
+	 */
+	spin_lock_bh(&bt->accept_q_lock);
+
+	if (sk) {
+		if (bt_sk(sk)->parent != parent)
+			goto out;
+
+		if (!list_is_last(&bt_sk(sk)->accept_q, &bt->accept_q)) {
+			next = &list_next_entry(bt_sk(sk), accept_q)->sk;
+			sock_hold(next);
+		}
+	} else if (!list_empty(&bt->accept_q)) {
+		next = &list_first_entry(&bt->accept_q,
+					 struct bt_sock, accept_q)->sk;
+		sock_hold(next);
+	}
+
+out:
+	spin_unlock_bh(&bt->accept_q_lock);
+	return next;
+}
+
 struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock)
 {
-	struct bt_sock *s, *n;
-	struct sock *sk;
+	struct sock *sk, *next;
 
 	BT_DBG("parent %p", parent);
 
 restart:
-	list_for_each_entry_safe(s, n, &bt_sk(parent)->accept_q, accept_q) {
-		sk = (struct sock *)s;
-
+	for (sk = bt_accept_get(parent, NULL); sk; sk = next) {
 		/* Prevent early freeing of sk due to unlink and sock_kill */
-		sock_hold(sk);
 		lock_sock(sk);
 
 		/* Check sk has not already been unlinked via
 		 * bt_accept_unlink() due to serialisation caused by sk locking
 		 */
-		if (!bt_sk(sk)->parent) {
+		if (bt_sk(sk)->parent != parent) {
 			BT_DBG("sk %p, already unlinked", sk);
 			release_sock(sk);
 			sock_put(sk);
 
-			/* Restart the loop as sk is no longer in the list
-			 * and also avoid a potential infinite loop because
-			 * list_for_each_entry_safe() is not thread safe.
-			 */
 			goto restart;
 		}
 
+		next = bt_accept_get(parent, sk);
+
 		/* sk is safely in the parent list so reduce reference count */
 		sock_put(sk);
 
@@ -309,7 +340,19 @@ struct sock *bt_accept_dequeue(struct sock *parent, struct socket *newsock)
 			if (newsock)
 				sock_graft(sk, newsock);
 
+			/* Hand the caller a reference taken while sk is
+			 * still locked.  bt_accept_unlink() just dropped
+			 * the accept-queue reference; without this hold a
+			 * concurrent teardown (e.g. l2cap_conn_del() ->
+			 * l2cap_sock_kill()) could free sk between
+			 * release_sock() and the caller using it.  Every
+			 * caller drops this with sock_put() when done.
+			 */
+			sock_hold(sk);
+
 			release_sock(sk);
+			if (next)
+				sock_put(next);
 			return sk;
 		}
 
@@ -518,18 +561,28 @@ EXPORT_SYMBOL(bt_sock_stream_recvmsg);
 
 static inline __poll_t bt_accept_poll(struct sock *parent)
 {
-	struct bt_sock *s, *n;
+	struct bt_sock *bt = bt_sk(parent);
+	struct bt_sock *s;
 	struct sock *sk;
+	__poll_t mask = 0;
+
+	spin_lock_bh(&bt->accept_q_lock);
+	list_for_each_entry(s, &bt->accept_q, accept_q) {
+		int state;
 
-	list_for_each_entry_safe(s, n, &bt_sk(parent)->accept_q, accept_q) {
 		sk = (struct sock *)s;
-		if (sk->sk_state == BT_CONNECTED ||
-		    (test_bit(BT_SK_DEFER_SETUP, &bt_sk(parent)->flags) &&
-		     sk->sk_state == BT_CONNECT2))
-			return EPOLLIN | EPOLLRDNORM;
+		state = READ_ONCE(sk->sk_state);
+
+		if (state == BT_CONNECTED ||
+		    (test_bit(BT_SK_DEFER_SETUP, &bt->flags) &&
+		     state == BT_CONNECT2)) {
+			mask = EPOLLIN | EPOLLRDNORM;
+			break;
+		}
 	}
+	spin_unlock_bh(&bt->accept_q_lock);
 
-	return 0;
+	return mask;
 }
 
 __poll_t bt_sock_poll(struct file *file, struct socket *sock,
diff --git a/net/bluetooth/bnep/core.c b/net/bluetooth/bnep/core.c
index 853c8d7644b55..0de5df690bd0b 100644
--- a/net/bluetooth/bnep/core.c
+++ b/net/bluetooth/bnep/core.c
@@ -645,8 +645,8 @@ int bnep_add_connection(struct bnep_connadd_req *req, struct socket *sock)
 		goto failed;
 	}
 
-	up_write(&bnep_session_sem);
 	strcpy(req->device, dev->name);
+	up_write(&bnep_session_sem);
 	return 0;
 
 failed:
diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c
index 0290dea081f62..61fec674a2078 100644
--- a/net/bluetooth/hci_sock.c
+++ b/net/bluetooth/hci_sock.c
@@ -27,6 +27,7 @@
 #include <linux/export.h>
 #include <linux/utsname.h>
 #include <linux/sched.h>
+#include <linux/uio.h>
 #include <linux/unaligned.h>
 
 #include <net/bluetooth/bluetooth.h>
@@ -2063,7 +2064,7 @@ static int hci_sock_setsockopt(struct socket *sock, int level, int optname,
 }
 
 static int hci_sock_getsockopt_old(struct socket *sock, int level, int optname,
-				   char __user *optval, int __user *optlen)
+				   sockopt_t *sopt)
 {
 	struct hci_ufilter uf;
 	struct sock *sk = sock->sk;
@@ -2071,8 +2072,7 @@ static int hci_sock_getsockopt_old(struct socket *sock, int level, int optname,
 
 	BT_DBG("sk %p, opt %d", sk, optname);
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = sopt->optlen;
 
 	lock_sock(sk);
 
@@ -2088,7 +2088,8 @@ static int hci_sock_getsockopt_old(struct socket *sock, int level, int optname,
 		else
 			opt = 0;
 
-		if (put_user(opt, optval))
+		if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) !=
+		    sizeof(opt))
 			err = -EFAULT;
 		break;
 
@@ -2098,7 +2099,8 @@ static int hci_sock_getsockopt_old(struct socket *sock, int level, int optname,
 		else
 			opt = 0;
 
-		if (put_user(opt, optval))
+		if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) !=
+		    sizeof(opt))
 			err = -EFAULT;
 		break;
 
@@ -2114,7 +2116,7 @@ static int hci_sock_getsockopt_old(struct socket *sock, int level, int optname,
 		}
 
 		len = min_t(unsigned int, len, sizeof(uf));
-		if (copy_to_user(optval, &uf, len))
+		if (copy_to_iter(&uf, len, &sopt->iter_out) != len)
 			err = -EFAULT;
 		break;
 
@@ -2129,16 +2131,16 @@ static int hci_sock_getsockopt_old(struct socket *sock, int level, int optname,
 }
 
 static int hci_sock_getsockopt(struct socket *sock, int level, int optname,
-			       char __user *optval, int __user *optlen)
+			       sockopt_t *sopt)
 {
 	struct sock *sk = sock->sk;
 	int err = 0;
+	u16 mtu;
 
 	BT_DBG("sk %p, opt %d", sk, optname);
 
 	if (level == SOL_HCI)
-		return hci_sock_getsockopt_old(sock, level, optname, optval,
-					       optlen);
+		return hci_sock_getsockopt_old(sock, level, optname, sopt);
 
 	if (level != SOL_BLUETOOTH)
 		return -ENOPROTOOPT;
@@ -2148,7 +2150,9 @@ static int hci_sock_getsockopt(struct socket *sock, int level, int optname,
 	switch (optname) {
 	case BT_SNDMTU:
 	case BT_RCVMTU:
-		if (put_user(hci_pi(sk)->mtu, (u16 __user *)optval))
+		mtu = hci_pi(sk)->mtu;
+		if (copy_to_iter(&mtu, sizeof(mtu), &sopt->iter_out) !=
+		    sizeof(mtu))
 			err = -EFAULT;
 		break;
 
@@ -2185,7 +2189,7 @@ static const struct proto_ops hci_sock_ops = {
 	.listen		= sock_no_listen,
 	.shutdown	= sock_no_shutdown,
 	.setsockopt	= hci_sock_setsockopt,
-	.getsockopt	= hci_sock_getsockopt,
+	.getsockopt_iter = hci_sock_getsockopt,
 	.connect	= sock_no_connect,
 	.socketpair	= sock_no_socketpair,
 	.accept		= sock_no_accept,
diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c
index fd3aacdea512a..aff8562a8690d 100644
--- a/net/bluetooth/hci_sync.c
+++ b/net/bluetooth/hci_sync.c
@@ -4438,6 +4438,9 @@ static int hci_le_set_event_mask_sync(struct hci_dev *hdev)
 		events[4] |= 0x02;	/* LE BIG Info Advertising Report */
 	}
 
+	if (ll_ext_feature_capable(hdev))
+		events[5] |= BIT(2);
+
 	if (le_cs_capable(hdev)) {
 		/* Channel Sounding events */
 		events[5] |= 0x08;	/* LE CS Read Remote Supported Cap Complete event */
@@ -7413,9 +7416,6 @@ static int hci_le_read_all_remote_features_sync(struct hci_dev *hdev,
 					sizeof(cp), &cp,
 					HCI_EVT_LE_ALL_REMOTE_FEATURES_COMPLETE,
 					HCI_CMD_TIMEOUT, NULL);
-
-	return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_ALL_REMOTE_FEATURES,
-				     sizeof(cp), &cp, HCI_CMD_TIMEOUT);
 }
 
 static int hci_le_read_remote_features_sync(struct hci_dev *hdev, void *data)
diff --git a/net/bluetooth/iso.c b/net/bluetooth/iso.c
index 7cb2864fe8724..1c90d0fbcb4ae 100644
--- a/net/bluetooth/iso.c
+++ b/net/bluetooth/iso.c
@@ -10,6 +10,7 @@
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
 #include <linux/sched/signal.h>
+#include <linux/uio.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
@@ -751,6 +752,8 @@ static void iso_sock_cleanup_listen(struct sock *parent)
 	while ((sk = bt_accept_dequeue(parent, NULL))) {
 		iso_sock_close(sk);
 		iso_sock_kill(sk);
+		/* Drop the reference handed back by bt_accept_dequeue(). */
+		sock_put(sk);
 	}
 
 	/* If listening socket has a hcon, properly disconnect it */
@@ -1356,8 +1359,13 @@ static int iso_sock_accept(struct socket *sock, struct socket *newsock,
 		}
 
 		ch = bt_accept_dequeue(sk, newsock);
-		if (ch)
+		if (ch) {
+			/* Drop the bridging ref from bt_accept_dequeue();
+			 * the grafted socket keeps ch alive from here.
+			 */
+			sock_put(ch);
 			break;
+		}
 
 		if (!timeo) {
 			err = -EAGAIN;
@@ -1842,18 +1850,17 @@ static int iso_sock_setsockopt(struct socket *sock, int level, int optname,
 }
 
 static int iso_sock_getsockopt(struct socket *sock, int level, int optname,
-			       char __user *optval, int __user *optlen)
+			       sockopt_t *opt)
 {
 	struct sock *sk = sock->sk;
-	int len, err = 0;
 	struct bt_iso_qos *qos;
+	int len, val, err = 0;
 	u8 base_len;
 	u8 *base;
 
 	BT_DBG("sk %p", sk);
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = opt->optlen;
 
 	lock_sock(sk);
 
@@ -1864,15 +1871,17 @@ static int iso_sock_getsockopt(struct socket *sock, int level, int optname,
 			break;
 		}
 
-		if (put_user(test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags),
-			     (u32 __user *)optval))
+		val = test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
+		if (copy_to_iter(&val, sizeof(val), &opt->iter_out) !=
+		    sizeof(val))
 			err = -EFAULT;
 
 		break;
 
 	case BT_PKT_STATUS:
-		if (put_user(test_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags),
-			     (int __user *)optval))
+		val = test_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags);
+		if (copy_to_iter(&val, sizeof(val), &opt->iter_out) !=
+		    sizeof(val))
 			err = -EFAULT;
 		break;
 
@@ -1880,7 +1889,7 @@ static int iso_sock_getsockopt(struct socket *sock, int level, int optname,
 		qos = iso_sock_get_qos(sk);
 
 		len = min_t(unsigned int, len, sizeof(*qos));
-		if (copy_to_user(optval, qos, len))
+		if (copy_to_iter(qos, len, &opt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -1896,9 +1905,8 @@ static int iso_sock_getsockopt(struct socket *sock, int level, int optname,
 		}
 
 		len = min_t(unsigned int, len, base_len);
-		if (copy_to_user(optval, base, len))
-			err = -EFAULT;
-		if (put_user(len, optlen))
+		opt->optlen = len;
+		if (copy_to_iter(base, len, &opt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -2593,6 +2601,11 @@ int iso_recv(struct hci_dev *hdev, u16 handle, struct sk_buff *skb, u16 flags)
 		break;
 
 	case ISO_END:
+		if (!conn->rx_len) {
+			BT_ERR("Unexpected end frame (len %d)", skb->len);
+			goto drop;
+		}
+
 		skb_copy_from_linear_data(skb, skb_put(conn->rx_skb, skb->len),
 					  skb->len);
 		conn->rx_len -= skb->len;
@@ -2660,7 +2673,7 @@ static const struct proto_ops iso_sock_ops = {
 	.socketpair	= sock_no_socketpair,
 	.shutdown	= iso_sock_shutdown,
 	.setsockopt	= iso_sock_setsockopt,
-	.getsockopt	= iso_sock_getsockopt
+	.getsockopt_iter = iso_sock_getsockopt
 };
 
 static const struct net_proto_family iso_sock_family_ops = {
diff --git a/net/bluetooth/l2cap_core.c b/net/bluetooth/l2cap_core.c
index 7701528f11677..fdccd62ccca8f 100644
--- a/net/bluetooth/l2cap_core.c
+++ b/net/bluetooth/l2cap_core.c
@@ -7274,7 +7274,7 @@ static void l2cap_ecred_reconfigure(struct l2cap_chan *chan)
 	chan->ident = l2cap_get_ident(conn);
 
 	l2cap_send_cmd(conn, chan->ident, L2CAP_ECRED_RECONF_REQ,
-		       sizeof(pdu), &pdu);
+		       struct_size(pdu, scid, 1), pdu);
 }
 
 int l2cap_chan_reconfigure(struct l2cap_chan *chan, __u16 mtu)
diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c
index cf590a67d3641..4ed745a9c2cfb 100644
--- a/net/bluetooth/l2cap_sock.c
+++ b/net/bluetooth/l2cap_sock.c
@@ -31,6 +31,7 @@
 #include <linux/export.h>
 #include <linux/filter.h>
 #include <linux/sched/signal.h>
+#include <linux/uio.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
@@ -349,8 +350,13 @@ static int l2cap_sock_accept(struct socket *sock, struct socket *newsock,
 		}
 
 		nsk = bt_accept_dequeue(sk, newsock);
-		if (nsk)
+		if (nsk) {
+			/* Drop the bridging ref from bt_accept_dequeue();
+			 * the grafted socket keeps nsk alive from here.
+			 */
+			sock_put(nsk);
 			break;
+		}
 
 		if (!timeo) {
 			err = -EAGAIN;
@@ -433,7 +439,7 @@ static int l2cap_get_mode(struct l2cap_chan *chan)
 }
 
 static int l2cap_sock_getsockopt_old(struct socket *sock, int optname,
-				     char __user *optval, int __user *optlen)
+				     sockopt_t *sopt)
 {
 	struct sock *sk = sock->sk;
 	struct l2cap_chan *chan = l2cap_pi(sk)->chan;
@@ -445,8 +451,7 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname,
 
 	BT_DBG("sk %p", sk);
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = sopt->optlen;
 
 	lock_sock(sk);
 
@@ -488,7 +493,7 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname,
 		BT_DBG("mode 0x%2.2x", chan->mode);
 
 		len = min(len, sizeof(opts));
-		if (copy_to_user(optval, (char *) &opts, len))
+		if (copy_to_iter(&opts, len, &sopt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -520,7 +525,8 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname,
 		if (test_bit(FLAG_FORCE_RELIABLE, &chan->flags))
 			opt |= L2CAP_LM_RELIABLE;
 
-		if (put_user(opt, (u32 __user *) optval))
+		if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) !=
+		    sizeof(opt))
 			err = -EFAULT;
 
 		break;
@@ -538,7 +544,7 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname,
 		memcpy(cinfo.dev_class, chan->conn->hcon->dev_class, 3);
 
 		len = min(len, sizeof(cinfo));
-		if (copy_to_user(optval, (char *) &cinfo, len))
+		if (copy_to_iter(&cinfo, len, &sopt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -553,25 +559,26 @@ static int l2cap_sock_getsockopt_old(struct socket *sock, int optname,
 }
 
 static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname,
-				 char __user *optval, int __user *optlen)
+				 sockopt_t *sopt)
 {
 	struct sock *sk = sock->sk;
 	struct l2cap_chan *chan = l2cap_pi(sk)->chan;
 	struct bt_security sec;
 	struct bt_power pwr;
-	u32 phys;
 	int len, mode, err = 0;
+	u32 opt;
+	u16 mtu;
+	u8 mval;
 
 	BT_DBG("sk %p", sk);
 
 	if (level == SOL_L2CAP)
-		return l2cap_sock_getsockopt_old(sock, optname, optval, optlen);
+		return l2cap_sock_getsockopt_old(sock, optname, sopt);
 
 	if (level != SOL_BLUETOOTH)
 		return -ENOPROTOOPT;
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = sopt->optlen;
 
 	lock_sock(sk);
 
@@ -595,7 +602,7 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname,
 		}
 
 		len = min_t(unsigned int, len, sizeof(sec));
-		if (copy_to_user(optval, (char *) &sec, len))
+		if (copy_to_iter(&sec, len, &sopt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -606,15 +613,17 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname,
 			break;
 		}
 
-		if (put_user(test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags),
-			     (u32 __user *) optval))
+		opt = test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
+		if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) !=
+		    sizeof(opt))
 			err = -EFAULT;
 
 		break;
 
 	case BT_FLUSHABLE:
-		if (put_user(test_bit(FLAG_FLUSHABLE, &chan->flags),
-			     (u32 __user *) optval))
+		opt = test_bit(FLAG_FLUSHABLE, &chan->flags);
+		if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) !=
+		    sizeof(opt))
 			err = -EFAULT;
 
 		break;
@@ -629,13 +638,15 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname,
 		pwr.force_active = test_bit(FLAG_FORCE_ACTIVE, &chan->flags);
 
 		len = min_t(unsigned int, len, sizeof(pwr));
-		if (copy_to_user(optval, (char *) &pwr, len))
+		if (copy_to_iter(&pwr, len, &sopt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
 
 	case BT_CHANNEL_POLICY:
-		if (put_user(chan->chan_policy, (u32 __user *) optval))
+		opt = chan->chan_policy;
+		if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) !=
+		    sizeof(opt))
 			err = -EFAULT;
 		break;
 
@@ -650,7 +661,9 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname,
 			break;
 		}
 
-		if (put_user(chan->omtu, (u16 __user *) optval))
+		mtu = chan->omtu;
+		if (copy_to_iter(&mtu, sizeof(mtu), &sopt->iter_out) !=
+		    sizeof(mtu))
 			err = -EFAULT;
 		break;
 
@@ -660,7 +673,9 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname,
 			break;
 		}
 
-		if (put_user(chan->imtu, (u16 __user *) optval))
+		mtu = chan->imtu;
+		if (copy_to_iter(&mtu, sizeof(mtu), &sopt->iter_out) !=
+		    sizeof(mtu))
 			err = -EFAULT;
 		break;
 
@@ -670,9 +685,10 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname,
 			break;
 		}
 
-		phys = hci_conn_get_phy(chan->conn->hcon);
+		opt = hci_conn_get_phy(chan->conn->hcon);
 
-		if (put_user(phys, (u32 __user *) optval))
+		if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) !=
+		    sizeof(opt))
 			err = -EFAULT;
 		break;
 
@@ -693,7 +709,9 @@ static int l2cap_sock_getsockopt(struct socket *sock, int level, int optname,
 			break;
 		}
 
-		if (put_user(mode, (u8 __user *) optval))
+		mval = mode;
+		if (copy_to_iter(&mval, sizeof(mval), &sopt->iter_out) !=
+		    sizeof(mval))
 			err = -EFAULT;
 		break;
 
@@ -1475,22 +1493,54 @@ static void l2cap_sock_cleanup_listen(struct sock *parent)
 	BT_DBG("parent %p state %s", parent,
 	       state_to_string(parent->sk_state));
 
-	/* Close not yet accepted channels */
+	/* Close not yet accepted channels.
+	 *
+	 * bt_accept_dequeue() now returns sk with an extra reference held
+	 * (taken while sk was still locked) so a concurrent l2cap_conn_del()
+	 * -> l2cap_sock_kill() cannot free sk under us.
+	 *
+	 * cleanup_listen() runs under the parent sk lock, so unlike
+	 * l2cap_sock_shutdown() we must NOT take conn->lock here: that would
+	 * establish sk_lock -> conn->lock and invert the established
+	 * conn->lock -> chan->lock -> sk_lock order (lockdep deadlock).
+	 *
+	 * Instead, briefly take the child sk lock to fetch and pin its chan.
+	 * l2cap_conn_del() reaches the chan free only via
+	 * l2cap_chan_del() -> l2cap_sock_teardown_cb(), which itself takes
+	 * the child sk lock; holding it across l2cap_chan_hold_unless_zero()
+	 * therefore guarantees the chan cannot be freed while we read and
+	 * pin it (hold_unless_zero() additionally skips a chan already past
+	 * its last reference).  We then drop the sk lock before taking
+	 * chan->lock, so sk and chan locks are never held together.
+	 */
 	while ((sk = bt_accept_dequeue(parent, NULL))) {
-		struct l2cap_chan *chan = l2cap_pi(sk)->chan;
+		struct l2cap_chan *chan;
+
+		lock_sock_nested(sk, L2CAP_NESTING_NORMAL);
+		chan = l2cap_chan_hold_unless_zero(l2cap_pi(sk)->chan);
+		release_sock(sk);
+		if (!chan) {
+			/* l2cap_conn_del() already tearing this child down */
+			sock_put(sk);
+			continue;
+		}
 
 		BT_DBG("child chan %p state %s", chan,
 		       state_to_string(chan->state));
 
-		l2cap_chan_hold(chan);
 		l2cap_chan_lock(chan);
-
 		__clear_chan_timer(chan);
 		l2cap_chan_close(chan, ECONNRESET);
-		l2cap_sock_kill(sk);
-
+		/* l2cap_conn_del() may already have killed this socket
+		 * (it sets SOCK_DEAD); skip the duplicate to avoid a
+		 * double sock_put()/l2cap_chan_put().
+		 */
+		if (!sock_flag(sk, SOCK_DEAD))
+			l2cap_sock_kill(sk);
 		l2cap_chan_unlock(chan);
+
 		l2cap_chan_put(chan);
+		sock_put(sk);
 	}
 }
 
@@ -2000,7 +2050,7 @@ static const struct proto_ops l2cap_sock_ops = {
 	.socketpair	= sock_no_socketpair,
 	.shutdown	= l2cap_sock_shutdown,
 	.setsockopt	= l2cap_sock_setsockopt,
-	.getsockopt	= l2cap_sock_getsockopt
+	.getsockopt_iter = l2cap_sock_getsockopt
 };
 
 static const struct net_proto_family l2cap_sock_family_ops = {
diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c
index b05bb380e5f87..de5bd6b637b20 100644
--- a/net/bluetooth/mgmt.c
+++ b/net/bluetooth/mgmt.c
@@ -9110,9 +9110,15 @@ static int add_ext_adv_data(struct sock *sk, struct hci_dev *hdev, void *data,
 	struct adv_info *adv_instance;
 	int err = 0;
 	struct mgmt_pending_cmd *cmd;
+	u16 expected_len;
 
 	BT_DBG("%s", hdev->name);
 
+	expected_len = struct_size(cp, data, cp->adv_data_len + cp->scan_rsp_len);
+	if (expected_len != data_len)
+		return mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_EXT_ADV_DATA,
+				       MGMT_STATUS_INVALID_PARAMS);
+
 	hci_dev_lock(hdev);
 
 	adv_instance = hci_find_adv_instance(hdev, cp->instance);
diff --git a/net/bluetooth/rfcomm/sock.c b/net/bluetooth/rfcomm/sock.c
index be6639cd6f590..38d2355114cff 100644
--- a/net/bluetooth/rfcomm/sock.c
+++ b/net/bluetooth/rfcomm/sock.c
@@ -28,6 +28,7 @@
 #include <linux/export.h>
 #include <linux/debugfs.h>
 #include <linux/sched/signal.h>
+#include <linux/uio.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
@@ -180,6 +181,8 @@ static void rfcomm_sock_cleanup_listen(struct sock *parent)
 	while ((sk = bt_accept_dequeue(parent, NULL))) {
 		rfcomm_sock_close(sk);
 		rfcomm_sock_kill(sk);
+		/* Drop the reference handed back by bt_accept_dequeue(). */
+		sock_put(sk);
 	}
 
 	parent->sk_state  = BT_CLOSED;
@@ -497,8 +500,13 @@ static int rfcomm_sock_accept(struct socket *sock, struct socket *newsock,
 		}
 
 		nsk = bt_accept_dequeue(sk, newsock);
-		if (nsk)
+		if (nsk) {
+			/* Drop the bridging ref from bt_accept_dequeue();
+			 * the grafted socket keeps nsk alive from here.
+			 */
+			sock_put(nsk);
 			break;
+		}
 
 		if (!timeo) {
 			err = -EAGAIN;
@@ -723,7 +731,8 @@ static int rfcomm_sock_setsockopt(struct socket *sock, int level, int optname,
 	return err;
 }
 
-static int rfcomm_sock_getsockopt_old(struct socket *sock, int optname, char __user *optval, int __user *optlen)
+static int rfcomm_sock_getsockopt_old(struct socket *sock, int optname,
+				      sockopt_t *sopt)
 {
 	struct sock *sk = sock->sk;
 	struct sock *l2cap_sk;
@@ -735,8 +744,7 @@ static int rfcomm_sock_getsockopt_old(struct socket *sock, int optname, char __u
 
 	BT_DBG("sk %p", sk);
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = sopt->optlen;
 
 	lock_sock(sk);
 
@@ -765,7 +773,8 @@ static int rfcomm_sock_getsockopt_old(struct socket *sock, int optname, char __u
 		if (rfcomm_pi(sk)->role_switch)
 			opt |= RFCOMM_LM_MASTER;
 
-		if (put_user(opt, (u32 __user *) optval))
+		if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) !=
+		    sizeof(opt))
 			err = -EFAULT;
 
 		break;
@@ -785,7 +794,7 @@ static int rfcomm_sock_getsockopt_old(struct socket *sock, int optname, char __u
 		memcpy(cinfo.dev_class, conn->hcon->dev_class, 3);
 
 		len = min(len, sizeof(cinfo));
-		if (copy_to_user(optval, (char *) &cinfo, len))
+		if (copy_to_iter(&cinfo, len, &sopt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -799,23 +808,24 @@ static int rfcomm_sock_getsockopt_old(struct socket *sock, int optname, char __u
 	return err;
 }
 
-static int rfcomm_sock_getsockopt(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen)
+static int rfcomm_sock_getsockopt(struct socket *sock, int level, int optname,
+				  sockopt_t *sopt)
 {
 	struct sock *sk = sock->sk;
 	struct bt_security sec;
 	int err = 0;
 	size_t len;
+	u32 opt;
 
 	BT_DBG("sk %p", sk);
 
 	if (level == SOL_RFCOMM)
-		return rfcomm_sock_getsockopt_old(sock, optname, optval, optlen);
+		return rfcomm_sock_getsockopt_old(sock, optname, sopt);
 
 	if (level != SOL_BLUETOOTH)
 		return -ENOPROTOOPT;
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = sopt->optlen;
 
 	lock_sock(sk);
 
@@ -830,7 +840,7 @@ static int rfcomm_sock_getsockopt(struct socket *sock, int level, int optname, c
 		sec.key_size = 0;
 
 		len = min(len, sizeof(sec));
-		if (copy_to_user(optval, (char *) &sec, len))
+		if (copy_to_iter(&sec, len, &sopt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -841,8 +851,9 @@ static int rfcomm_sock_getsockopt(struct socket *sock, int level, int optname, c
 			break;
 		}
 
-		if (put_user(test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags),
-			     (u32 __user *) optval))
+		opt = test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
+		if (copy_to_iter(&opt, sizeof(opt), &sopt->iter_out) !=
+		    sizeof(opt))
 			err = -EFAULT;
 
 		break;
@@ -1014,7 +1025,7 @@ static const struct proto_ops rfcomm_sock_ops = {
 	.recvmsg	= rfcomm_sock_recvmsg,
 	.shutdown	= rfcomm_sock_shutdown,
 	.setsockopt	= rfcomm_sock_setsockopt,
-	.getsockopt	= rfcomm_sock_getsockopt,
+	.getsockopt_iter = rfcomm_sock_getsockopt,
 	.ioctl		= rfcomm_sock_ioctl,
 	.gettstamp	= sock_gettstamp,
 	.poll		= bt_sock_poll,
diff --git a/net/bluetooth/sco.c b/net/bluetooth/sco.c
index eba44525d41d9..9fe8dbacec2d1 100644
--- a/net/bluetooth/sco.c
+++ b/net/bluetooth/sco.c
@@ -28,6 +28,7 @@
 #include <linux/debugfs.h>
 #include <linux/seq_file.h>
 #include <linux/sched/signal.h>
+#include <linux/uio.h>
 
 #include <net/bluetooth/bluetooth.h>
 #include <net/bluetooth/hci_core.h>
@@ -502,6 +503,8 @@ static void sco_sock_cleanup_listen(struct sock *parent)
 	while ((sk = bt_accept_dequeue(parent, NULL))) {
 		sco_sock_close(sk);
 		sco_sock_kill(sk);
+		/* Drop the reference handed back by bt_accept_dequeue(). */
+		sock_put(sk);
 	}
 
 	parent->sk_state  = BT_CLOSED;
@@ -765,8 +768,13 @@ static int sco_sock_accept(struct socket *sock, struct socket *newsock,
 		}
 
 		ch = bt_accept_dequeue(sk, newsock);
-		if (ch)
+		if (ch) {
+			/* Drop the bridging ref from bt_accept_dequeue();
+			 * the grafted socket keeps ch alive from here.
+			 */
+			sock_put(ch);
 			break;
+		}
 
 		if (!timeo) {
 			err = -EAGAIN;
@@ -1072,7 +1080,7 @@ static int sco_sock_setsockopt(struct socket *sock, int level, int optname,
 }
 
 static int sco_sock_getsockopt_old(struct socket *sock, int optname,
-				   char __user *optval, int __user *optlen)
+				   sockopt_t *opt)
 {
 	struct sock *sk = sock->sk;
 	struct sco_options opts;
@@ -1082,8 +1090,7 @@ static int sco_sock_getsockopt_old(struct socket *sock, int optname,
 
 	BT_DBG("sk %p", sk);
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = opt->optlen;
 
 	lock_sock(sk);
 
@@ -1101,7 +1108,7 @@ static int sco_sock_getsockopt_old(struct socket *sock, int optname,
 		BT_DBG("mtu %u", opts.mtu);
 
 		len = min(len, sizeof(opts));
-		if (copy_to_user(optval, (char *)&opts, len))
+		if (copy_to_iter(&opts, len, &opt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -1119,7 +1126,7 @@ static int sco_sock_getsockopt_old(struct socket *sock, int optname,
 		memcpy(cinfo.dev_class, sco_pi(sk)->conn->hcon->dev_class, 3);
 
 		len = min(len, sizeof(cinfo));
-		if (copy_to_user(optval, (char *)&cinfo, len))
+		if (copy_to_iter(&cinfo, len, &opt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -1134,15 +1141,15 @@ static int sco_sock_getsockopt_old(struct socket *sock, int optname,
 }
 
 static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
-			       char __user *optval, int __user *optlen)
+			       sockopt_t *opt)
 {
 	struct sock *sk = sock->sk;
-	int len, err = 0;
+	int len, val, err = 0;
 	struct bt_voice voice;
 	u32 phys;
 	int buf_len;
 	struct codec_list *c;
-	u8 num_codecs, i, __user *ptr;
+	u8 num_codecs, i;
 	struct hci_dev *hdev;
 	struct hci_codec_caps *caps;
 	struct bt_codec codec;
@@ -1150,10 +1157,9 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
 	BT_DBG("sk %p", sk);
 
 	if (level == SOL_SCO)
-		return sco_sock_getsockopt_old(sock, optname, optval, optlen);
+		return sco_sock_getsockopt_old(sock, optname, opt);
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = opt->optlen;
 
 	lock_sock(sk);
 
@@ -1165,8 +1171,9 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
 			break;
 		}
 
-		if (put_user(test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags),
-			     (u32 __user *)optval))
+		val = test_bit(BT_SK_DEFER_SETUP, &bt_sk(sk)->flags);
+		if (copy_to_iter(&val, sizeof(val), &opt->iter_out) !=
+		    sizeof(val))
 			err = -EFAULT;
 
 		break;
@@ -1175,7 +1182,7 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
 		voice.setting = sco_pi(sk)->setting;
 
 		len = min_t(unsigned int, len, sizeof(voice));
-		if (copy_to_user(optval, (char *)&voice, len))
+		if (copy_to_iter(&voice, len, &opt->iter_out) != len)
 			err = -EFAULT;
 
 		break;
@@ -1188,13 +1195,15 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
 
 		phys = hci_conn_get_phy(sco_pi(sk)->conn->hcon);
 
-		if (put_user(phys, (u32 __user *) optval))
+		if (copy_to_iter(&phys, sizeof(phys), &opt->iter_out) !=
+		    sizeof(phys))
 			err = -EFAULT;
 		break;
 
 	case BT_PKT_STATUS:
-		if (put_user(test_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags),
-			     (int __user *)optval))
+		val = test_bit(BT_SK_PKT_STATUS, &bt_sk(sk)->flags);
+		if (copy_to_iter(&val, sizeof(val), &opt->iter_out) !=
+		    sizeof(val))
 			err = -EFAULT;
 		break;
 
@@ -1205,7 +1214,9 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
 			break;
 		}
 
-		if (put_user(sco_pi(sk)->conn->mtu, (u32 __user *)optval))
+		val = sco_pi(sk)->conn->mtu;
+		if (copy_to_iter(&val, sizeof(val), &opt->iter_out) !=
+		    sizeof(val))
 			err = -EFAULT;
 		break;
 
@@ -1252,13 +1263,12 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
 			hci_dev_put(hdev);
 			return -ENOBUFS;
 		}
-		ptr = optval;
 
-		if (put_user(num_codecs, ptr)) {
+		if (copy_to_iter(&num_codecs, sizeof(num_codecs),
+				 &opt->iter_out) != sizeof(num_codecs)) {
 			hci_dev_put(hdev);
 			return -EFAULT;
 		}
-		ptr += sizeof(num_codecs);
 
 		/* Iterate all the codecs supported over SCO and populate
 		 * codec data
@@ -1275,11 +1285,11 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
 			if (err < 0)
 				break;
 			codec.num_caps = c->num_caps;
-			if (copy_to_user(ptr, &codec, sizeof(codec))) {
+			if (copy_to_iter(&codec, sizeof(codec), &opt->iter_out)
+			    != sizeof(codec)) {
 				err = -EFAULT;
 				break;
 			}
-			ptr += sizeof(codec);
 
 			/* find codec capabilities data length */
 			len = 0;
@@ -1289,11 +1299,11 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
 			}
 
 			/* copy codec capabilities data */
-			if (len && copy_to_user(ptr, c->caps, len)) {
+			if (len &&
+			    copy_to_iter(c->caps, len, &opt->iter_out) != len) {
 				err = -EFAULT;
 				break;
 			}
-			ptr += len;
 		}
 
 		hci_dev_unlock(hdev);
@@ -1301,8 +1311,8 @@ static int sco_sock_getsockopt(struct socket *sock, int level, int optname,
 
 		lock_sock(sk);
 
-		if (!err && put_user(buf_len, optlen))
-			err = -EFAULT;
+		if (!err)
+			opt->optlen = buf_len;
 
 		break;
 
@@ -1577,7 +1587,7 @@ static const struct proto_ops sco_sock_ops = {
 	.socketpair	= sock_no_socketpair,
 	.shutdown	= sco_sock_shutdown,
 	.setsockopt	= sco_sock_setsockopt,
-	.getsockopt	= sco_sock_getsockopt
+	.getsockopt_iter = sco_sock_getsockopt
 };
 
 static const struct net_proto_family sco_sock_family_ops = {
diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c
index 98f1da4f5f559..1739c1989dbd5 100644
--- a/net/bluetooth/smp.c
+++ b/net/bluetooth/smp.c
@@ -22,8 +22,8 @@
 
 #include <linux/debugfs.h>
 #include <linux/scatterlist.h>
+#include <crypto/aes-cbc-macs.h>
 #include <crypto/aes.h>
-#include <crypto/hash.h>
 #include <crypto/kpp.h>
 #include <crypto/utils.h>
 
@@ -63,7 +63,7 @@
 				 0x3f : 0x07)
 #define KEY_DIST_MASK		0x07
 
-/* Maximum message length that can be passed to aes_cmac */
+/* Maximum message length that can be passed to smp_aes_cmac */
 #define CMAC_MSG_MAX	80
 
 enum {
@@ -89,7 +89,6 @@ struct smp_dev {
 	u8			local_rand[16];
 	bool			debug_key;
 
-	struct crypto_shash	*tfm_cmac;
 	struct crypto_kpp	*tfm_ecdh;
 };
 
@@ -127,7 +126,6 @@ struct smp_chan {
 	u8			dhkey[32];
 	u8			mackey[16];
 
-	struct crypto_shash	*tfm_cmac;
 	struct crypto_kpp	*tfm_ecdh;
 };
 
@@ -166,20 +164,15 @@ static inline void swap_buf(const u8 *src, u8 *dst, size_t len)
  * AES-CMAC, f4, f5, f6, g2 and h6.
  */
 
-static int aes_cmac(struct crypto_shash *tfm, const u8 k[16], const u8 *m,
-		    size_t len, u8 mac[16])
+static int smp_aes_cmac(const u8 k[16], const u8 *m, size_t len, u8 mac[16])
 {
 	uint8_t tmp[16], mac_msb[16], msg_msb[CMAC_MSG_MAX];
+	struct aes_cmac_key key;
 	int err;
 
 	if (len > CMAC_MSG_MAX)
 		return -EFBIG;
 
-	if (!tfm) {
-		BT_ERR("tfm %p", tfm);
-		return -EINVAL;
-	}
-
 	/* Swap key and message from LSB to MSB */
 	swap_buf(k, tmp, 16);
 	swap_buf(m, msg_msb, len);
@@ -187,17 +180,10 @@ static int aes_cmac(struct crypto_shash *tfm, const u8 k[16], const u8 *m,
 	SMP_DBG("msg (len %zu) %*phN", len, (int) len, m);
 	SMP_DBG("key %16phN", k);
 
-	err = crypto_shash_setkey(tfm, tmp, 16);
-	if (err) {
-		BT_ERR("cipher setkey failed: %d", err);
+	err = aes_cmac_preparekey(&key, tmp, 16);
+	if (WARN_ON_ONCE(err)) /* Should never happen, as 16 is valid keylen */
 		return err;
-	}
-
-	err = crypto_shash_tfm_digest(tfm, msg_msb, len, mac_msb);
-	if (err) {
-		BT_ERR("Hash computation error %d", err);
-		return err;
-	}
+	aes_cmac(&key, msg_msb, len, mac_msb);
 
 	swap_buf(mac_msb, mac, 16);
 
@@ -206,8 +192,8 @@ static int aes_cmac(struct crypto_shash *tfm, const u8 k[16], const u8 *m,
 	return 0;
 }
 
-static int smp_f4(struct crypto_shash *tfm_cmac, const u8 u[32],
-		  const u8 v[32], const u8 x[16], u8 z, u8 res[16])
+static int smp_f4(const u8 u[32], const u8 v[32], const u8 x[16], u8 z,
+		  u8 res[16])
 {
 	u8 m[65];
 	int err;
@@ -220,7 +206,7 @@ static int smp_f4(struct crypto_shash *tfm_cmac, const u8 u[32],
 	memcpy(m + 1, v, 32);
 	memcpy(m + 33, u, 32);
 
-	err = aes_cmac(tfm_cmac, x, m, sizeof(m), res);
+	err = smp_aes_cmac(x, m, sizeof(m), res);
 	if (err)
 		return err;
 
@@ -229,9 +215,8 @@ static int smp_f4(struct crypto_shash *tfm_cmac, const u8 u[32],
 	return err;
 }
 
-static int smp_f5(struct crypto_shash *tfm_cmac, const u8 w[32],
-		  const u8 n1[16], const u8 n2[16], const u8 a1[7],
-		  const u8 a2[7], u8 mackey[16], u8 ltk[16])
+static int smp_f5(const u8 w[32], const u8 n1[16], const u8 n2[16],
+		  const u8 a1[7], const u8 a2[7], u8 mackey[16], u8 ltk[16])
 {
 	/* The btle, salt and length "magic" values are as defined in
 	 * the SMP section of the Bluetooth core specification. In ASCII
@@ -250,7 +235,7 @@ static int smp_f5(struct crypto_shash *tfm_cmac, const u8 w[32],
 	SMP_DBG("n1 %16phN n2 %16phN", n1, n2);
 	SMP_DBG("a1 %7phN a2 %7phN", a1, a2);
 
-	err = aes_cmac(tfm_cmac, salt, w, 32, t);
+	err = smp_aes_cmac(salt, w, 32, t);
 	if (err)
 		return err;
 
@@ -265,7 +250,7 @@ static int smp_f5(struct crypto_shash *tfm_cmac, const u8 w[32],
 
 	m[52] = 0; /* Counter */
 
-	err = aes_cmac(tfm_cmac, t, m, sizeof(m), mackey);
+	err = smp_aes_cmac(t, m, sizeof(m), mackey);
 	if (err)
 		return err;
 
@@ -273,7 +258,7 @@ static int smp_f5(struct crypto_shash *tfm_cmac, const u8 w[32],
 
 	m[52] = 1; /* Counter */
 
-	err = aes_cmac(tfm_cmac, t, m, sizeof(m), ltk);
+	err = smp_aes_cmac(t, m, sizeof(m), ltk);
 	if (err)
 		return err;
 
@@ -282,10 +267,9 @@ static int smp_f5(struct crypto_shash *tfm_cmac, const u8 w[32],
 	return 0;
 }
 
-static int smp_f6(struct crypto_shash *tfm_cmac, const u8 w[16],
-		  const u8 n1[16], const u8 n2[16], const u8 r[16],
-		  const u8 io_cap[3], const u8 a1[7], const u8 a2[7],
-		  u8 res[16])
+static int smp_f6(const u8 w[16], const u8 n1[16], const u8 n2[16],
+		  const u8 r[16], const u8 io_cap[3], const u8 a1[7],
+		  const u8 a2[7], u8 res[16])
 {
 	u8 m[65];
 	int err;
@@ -301,7 +285,7 @@ static int smp_f6(struct crypto_shash *tfm_cmac, const u8 w[16],
 	memcpy(m + 33, n2, 16);
 	memcpy(m + 49, n1, 16);
 
-	err = aes_cmac(tfm_cmac, w, m, sizeof(m), res);
+	err = smp_aes_cmac(w, m, sizeof(m), res);
 	if (err)
 		return err;
 
@@ -310,8 +294,8 @@ static int smp_f6(struct crypto_shash *tfm_cmac, const u8 w[16],
 	return err;
 }
 
-static int smp_g2(struct crypto_shash *tfm_cmac, const u8 u[32], const u8 v[32],
-		  const u8 x[16], const u8 y[16], u32 *val)
+static int smp_g2(const u8 u[32], const u8 v[32], const u8 x[16],
+		  const u8 y[16], u32 *val)
 {
 	u8 m[80], tmp[16];
 	int err;
@@ -324,7 +308,7 @@ static int smp_g2(struct crypto_shash *tfm_cmac, const u8 u[32], const u8 v[32],
 	memcpy(m + 16, v, 32);
 	memcpy(m + 48, u, 32);
 
-	err = aes_cmac(tfm_cmac, x, m, sizeof(m), tmp);
+	err = smp_aes_cmac(x, m, sizeof(m), tmp);
 	if (err)
 		return err;
 
@@ -336,14 +320,13 @@ static int smp_g2(struct crypto_shash *tfm_cmac, const u8 u[32], const u8 v[32],
 	return 0;
 }
 
-static int smp_h6(struct crypto_shash *tfm_cmac, const u8 w[16],
-		  const u8 key_id[4], u8 res[16])
+static int smp_h6(const u8 w[16], const u8 key_id[4], u8 res[16])
 {
 	int err;
 
 	SMP_DBG("w %16phN key_id %4phN", w, key_id);
 
-	err = aes_cmac(tfm_cmac, w, key_id, 4, res);
+	err = smp_aes_cmac(w, key_id, 4, res);
 	if (err)
 		return err;
 
@@ -352,14 +335,13 @@ static int smp_h6(struct crypto_shash *tfm_cmac, const u8 w[16],
 	return err;
 }
 
-static int smp_h7(struct crypto_shash *tfm_cmac, const u8 w[16],
-		  const u8 salt[16], u8 res[16])
+static int smp_h7(const u8 w[16], const u8 salt[16], u8 res[16])
 {
 	int err;
 
 	SMP_DBG("w %16phN salt %16phN", w, salt);
 
-	err = aes_cmac(tfm_cmac, salt, w, 16, res);
+	err = smp_aes_cmac(salt, w, 16, res);
 	if (err)
 		return err;
 
@@ -574,8 +556,7 @@ int smp_generate_oob(struct hci_dev *hdev, u8 hash[16], u8 rand[16])
 
 	get_random_bytes(smp->local_rand, 16);
 
-	err = smp_f4(smp->tfm_cmac, smp->local_pk, smp->local_pk,
-		     smp->local_rand, 0, hash);
+	err = smp_f4(smp->local_pk, smp->local_pk, smp->local_rand, 0, hash);
 	if (err < 0)
 		return err;
 
@@ -757,7 +738,6 @@ static void smp_chan_destroy(struct l2cap_conn *conn)
 	kfree_sensitive(smp->responder_csrk);
 	kfree_sensitive(smp->link_key);
 
-	crypto_free_shash(smp->tfm_cmac);
 	crypto_free_kpp(smp->tfm_ecdh);
 
 	/* Ensure that we don't leave any debug key around if debug key
@@ -1160,7 +1140,7 @@ static void sc_generate_link_key(struct smp_chan *smp)
 		/* SALT = 0x000000000000000000000000746D7031 */
 		const u8 salt[16] = { 0x31, 0x70, 0x6d, 0x74 };
 
-		if (smp_h7(smp->tfm_cmac, smp->tk, salt, smp->link_key)) {
+		if (smp_h7(smp->tk, salt, smp->link_key)) {
 			kfree_sensitive(smp->link_key);
 			smp->link_key = NULL;
 			return;
@@ -1169,14 +1149,14 @@ static void sc_generate_link_key(struct smp_chan *smp)
 		/* From core spec. Spells out in ASCII as 'tmp1'. */
 		const u8 tmp1[4] = { 0x31, 0x70, 0x6d, 0x74 };
 
-		if (smp_h6(smp->tfm_cmac, smp->tk, tmp1, smp->link_key)) {
+		if (smp_h6(smp->tk, tmp1, smp->link_key)) {
 			kfree_sensitive(smp->link_key);
 			smp->link_key = NULL;
 			return;
 		}
 	}
 
-	if (smp_h6(smp->tfm_cmac, smp->link_key, lebr, smp->link_key)) {
+	if (smp_h6(smp->link_key, lebr, smp->link_key)) {
 		kfree_sensitive(smp->link_key);
 		smp->link_key = NULL;
 		return;
@@ -1218,17 +1198,17 @@ static void sc_generate_ltk(struct smp_chan *smp)
 		/* SALT = 0x000000000000000000000000746D7032 */
 		const u8 salt[16] = { 0x32, 0x70, 0x6d, 0x74 };
 
-		if (smp_h7(smp->tfm_cmac, key->val, salt, smp->tk))
+		if (smp_h7(key->val, salt, smp->tk))
 			return;
 	} else {
 		/* From core spec. Spells out in ASCII as 'tmp2'. */
 		const u8 tmp2[4] = { 0x32, 0x70, 0x6d, 0x74 };
 
-		if (smp_h6(smp->tfm_cmac, key->val, tmp2, smp->tk))
+		if (smp_h6(key->val, tmp2, smp->tk))
 			return;
 	}
 
-	if (smp_h6(smp->tfm_cmac, smp->tk, brle, smp->tk))
+	if (smp_h6(smp->tk, brle, smp->tk))
 		return;
 
 	sc_add_ltk(smp);
@@ -1389,16 +1369,10 @@ static struct smp_chan *smp_chan_create(struct l2cap_conn *conn)
 	if (!smp)
 		return NULL;
 
-	smp->tfm_cmac = crypto_alloc_shash("cmac(aes)", 0, 0);
-	if (IS_ERR(smp->tfm_cmac)) {
-		bt_dev_err(hcon->hdev, "Unable to create CMAC crypto context");
-		goto zfree_smp;
-	}
-
 	smp->tfm_ecdh = crypto_alloc_kpp("ecdh-nist-p256", 0, 0);
 	if (IS_ERR(smp->tfm_ecdh)) {
 		bt_dev_err(hcon->hdev, "Unable to create ECDH crypto context");
-		goto free_shash;
+		goto zfree_smp;
 	}
 
 	smp->conn = conn;
@@ -1412,8 +1386,6 @@ static struct smp_chan *smp_chan_create(struct l2cap_conn *conn)
 
 	return smp;
 
-free_shash:
-	crypto_free_shash(smp->tfm_cmac);
 zfree_smp:
 	kfree_sensitive(smp);
 	return NULL;
@@ -1437,7 +1409,7 @@ static int sc_mackey_and_ltk(struct smp_chan *smp, u8 mackey[16], u8 ltk[16])
 	a[6] = hcon->init_addr_type;
 	b[6] = hcon->resp_addr_type;
 
-	return smp_f5(smp->tfm_cmac, smp->dhkey, na, nb, a, b, mackey, ltk);
+	return smp_f5(smp->dhkey, na, nb, a, b, mackey, ltk);
 }
 
 static void sc_dhkey_check(struct smp_chan *smp)
@@ -1470,8 +1442,8 @@ static void sc_dhkey_check(struct smp_chan *smp)
 	if (smp->method == REQ_OOB)
 		memcpy(r, smp->rr, 16);
 
-	smp_f6(smp->tfm_cmac, smp->mackey, smp->prnd, smp->rrnd, r, io_cap,
-	       local_addr, remote_addr, check.e);
+	smp_f6(smp->mackey, smp->prnd, smp->rrnd, r, io_cap, local_addr,
+	       remote_addr, check.e);
 
 	smp_send_cmd(smp->conn, SMP_CMD_DHKEY_CHECK, sizeof(check), &check);
 }
@@ -1488,7 +1460,7 @@ static u8 sc_passkey_send_confirm(struct smp_chan *smp)
 
 	get_random_bytes(smp->prnd, sizeof(smp->prnd));
 
-	if (smp_f4(smp->tfm_cmac, smp->local_pk, smp->remote_pk, smp->prnd, r,
+	if (smp_f4(smp->local_pk, smp->remote_pk, smp->prnd, r,
 		   cfm.confirm_val))
 		return SMP_UNSPECIFIED;
 
@@ -1513,8 +1485,7 @@ static u8 sc_passkey_round(struct smp_chan *smp, u8 smp_op)
 		r = ((hcon->passkey_notify >> smp->passkey_round) & 0x01);
 		r |= 0x80;
 
-		if (smp_f4(smp->tfm_cmac, smp->remote_pk, smp->local_pk,
-			   smp->rrnd, r, cfm))
+		if (smp_f4(smp->remote_pk, smp->local_pk, smp->rrnd, r, cfm))
 			return SMP_UNSPECIFIED;
 
 		if (crypto_memneq(smp->pcnf, cfm, 16))
@@ -2178,8 +2149,7 @@ static u8 smp_cmd_pairing_random(struct l2cap_conn *conn, struct sk_buff *skb)
 	if (test_bit(SMP_FLAG_INITIATOR, &smp->flags)) {
 		u8 cfm[16];
 
-		err = smp_f4(smp->tfm_cmac, smp->remote_pk, smp->local_pk,
-			     smp->rrnd, 0, cfm);
+		err = smp_f4(smp->remote_pk, smp->local_pk, smp->rrnd, 0, cfm);
 		if (err)
 			return SMP_UNSPECIFIED;
 
@@ -2205,7 +2175,7 @@ static u8 smp_cmd_pairing_random(struct l2cap_conn *conn, struct sk_buff *skb)
 		return 0;
 	}
 
-	err = smp_g2(smp->tfm_cmac, pkax, pkbx, na, nb, &passkey);
+	err = smp_g2(pkax, pkbx, na, nb, &passkey);
 	if (err)
 		return SMP_UNSPECIFIED;
 
@@ -2750,8 +2720,8 @@ static int smp_cmd_public_key(struct l2cap_conn *conn, struct sk_buff *skb)
 	memcpy(smp->remote_pk, key, 64);
 
 	if (test_bit(SMP_FLAG_REMOTE_OOB, &smp->flags)) {
-		err = smp_f4(smp->tfm_cmac, smp->remote_pk, smp->remote_pk,
-			     smp->rr, 0, cfm.confirm_val);
+		err = smp_f4(smp->remote_pk, smp->remote_pk, smp->rr, 0,
+			     cfm.confirm_val);
 		if (err)
 			return SMP_UNSPECIFIED;
 
@@ -2851,8 +2821,8 @@ static int smp_cmd_public_key(struct l2cap_conn *conn, struct sk_buff *skb)
 	if (test_bit(SMP_FLAG_INITIATOR, &smp->flags))
 		return 0;
 
-	err = smp_f4(smp->tfm_cmac, smp->local_pk, smp->remote_pk, smp->prnd,
-		     0, cfm.confirm_val);
+	err = smp_f4(smp->local_pk, smp->remote_pk, smp->prnd, 0,
+		     cfm.confirm_val);
 	if (err)
 		return SMP_UNSPECIFIED;
 
@@ -2899,8 +2869,8 @@ static int smp_cmd_dhkey_check(struct l2cap_conn *conn, struct sk_buff *skb)
 	else if (smp->method == REQ_OOB)
 		memcpy(r, smp->lr, 16);
 
-	err = smp_f6(smp->tfm_cmac, smp->mackey, smp->rrnd, smp->prnd, r,
-		     io_cap, remote_addr, local_addr, e);
+	err = smp_f6(smp->mackey, smp->rrnd, smp->prnd, r, io_cap, remote_addr,
+		     local_addr, e);
 	if (err)
 		return SMP_UNSPECIFIED;
 
@@ -3286,7 +3256,6 @@ static struct l2cap_chan *smp_add_cid(struct hci_dev *hdev, u16 cid)
 {
 	struct l2cap_chan *chan;
 	struct smp_dev *smp;
-	struct crypto_shash *tfm_cmac;
 	struct crypto_kpp *tfm_ecdh;
 
 	if (cid == L2CAP_CID_SMP_BREDR) {
@@ -3298,30 +3267,20 @@ static struct l2cap_chan *smp_add_cid(struct hci_dev *hdev, u16 cid)
 	if (!smp)
 		return ERR_PTR(-ENOMEM);
 
-	tfm_cmac = crypto_alloc_shash("cmac(aes)", 0, 0);
-	if (IS_ERR(tfm_cmac)) {
-		bt_dev_err(hdev, "Unable to create CMAC crypto context");
-		kfree_sensitive(smp);
-		return ERR_CAST(tfm_cmac);
-	}
-
 	tfm_ecdh = crypto_alloc_kpp("ecdh-nist-p256", 0, 0);
 	if (IS_ERR(tfm_ecdh)) {
 		bt_dev_err(hdev, "Unable to create ECDH crypto context");
-		crypto_free_shash(tfm_cmac);
 		kfree_sensitive(smp);
 		return ERR_CAST(tfm_ecdh);
 	}
 
 	smp->local_oob = false;
-	smp->tfm_cmac = tfm_cmac;
 	smp->tfm_ecdh = tfm_ecdh;
 
 create_chan:
 	chan = l2cap_chan_create();
 	if (!chan) {
 		if (smp) {
-			crypto_free_shash(smp->tfm_cmac);
 			crypto_free_kpp(smp->tfm_ecdh);
 			kfree_sensitive(smp);
 		}
@@ -3368,7 +3327,6 @@ static void smp_del_chan(struct l2cap_chan *chan)
 	smp = chan->data;
 	if (smp) {
 		chan->data = NULL;
-		crypto_free_shash(smp->tfm_cmac);
 		crypto_free_kpp(smp->tfm_ecdh);
 		kfree_sensitive(smp);
 	}
@@ -3565,7 +3523,7 @@ static int __init test_s1(void)
 	return 0;
 }
 
-static int __init test_f4(struct crypto_shash *tfm_cmac)
+static int __init test_f4(void)
 {
 	const u8 u[32] = {
 			0xe6, 0x9d, 0x35, 0x0e, 0x48, 0x01, 0x03, 0xcc,
@@ -3587,7 +3545,7 @@ static int __init test_f4(struct crypto_shash *tfm_cmac)
 	u8 res[16];
 	int err;
 
-	err = smp_f4(tfm_cmac, u, v, x, z, res);
+	err = smp_f4(u, v, x, z, res);
 	if (err)
 		return err;
 
@@ -3597,7 +3555,7 @@ static int __init test_f4(struct crypto_shash *tfm_cmac)
 	return 0;
 }
 
-static int __init test_f5(struct crypto_shash *tfm_cmac)
+static int __init test_f5(void)
 {
 	const u8 w[32] = {
 			0x98, 0xa6, 0xbf, 0x73, 0xf3, 0x34, 0x8d, 0x86,
@@ -3621,7 +3579,7 @@ static int __init test_f5(struct crypto_shash *tfm_cmac)
 	u8 mackey[16], ltk[16];
 	int err;
 
-	err = smp_f5(tfm_cmac, w, n1, n2, a1, a2, mackey, ltk);
+	err = smp_f5(w, n1, n2, a1, a2, mackey, ltk);
 	if (err)
 		return err;
 
@@ -3634,7 +3592,7 @@ static int __init test_f5(struct crypto_shash *tfm_cmac)
 	return 0;
 }
 
-static int __init test_f6(struct crypto_shash *tfm_cmac)
+static int __init test_f6(void)
 {
 	const u8 w[16] = {
 			0x20, 0x6e, 0x63, 0xce, 0x20, 0x6a, 0x3f, 0xfd,
@@ -3657,7 +3615,7 @@ static int __init test_f6(struct crypto_shash *tfm_cmac)
 	u8 res[16];
 	int err;
 
-	err = smp_f6(tfm_cmac, w, n1, n2, r, io_cap, a1, a2, res);
+	err = smp_f6(w, n1, n2, r, io_cap, a1, a2, res);
 	if (err)
 		return err;
 
@@ -3667,7 +3625,7 @@ static int __init test_f6(struct crypto_shash *tfm_cmac)
 	return 0;
 }
 
-static int __init test_g2(struct crypto_shash *tfm_cmac)
+static int __init test_g2(void)
 {
 	const u8 u[32] = {
 			0xe6, 0x9d, 0x35, 0x0e, 0x48, 0x01, 0x03, 0xcc,
@@ -3689,7 +3647,7 @@ static int __init test_g2(struct crypto_shash *tfm_cmac)
 	u32 val;
 	int err;
 
-	err = smp_g2(tfm_cmac, u, v, x, y, &val);
+	err = smp_g2(u, v, x, y, &val);
 	if (err)
 		return err;
 
@@ -3699,7 +3657,7 @@ static int __init test_g2(struct crypto_shash *tfm_cmac)
 	return 0;
 }
 
-static int __init test_h6(struct crypto_shash *tfm_cmac)
+static int __init test_h6(void)
 {
 	const u8 w[16] = {
 			0x9b, 0x7d, 0x39, 0x0a, 0xa6, 0x10, 0x10, 0x34,
@@ -3711,7 +3669,7 @@ static int __init test_h6(struct crypto_shash *tfm_cmac)
 	u8 res[16];
 	int err;
 
-	err = smp_h6(tfm_cmac, w, key_id, res);
+	err = smp_h6(w, key_id, res);
 	if (err)
 		return err;
 
@@ -3736,8 +3694,7 @@ static const struct file_operations test_smp_fops = {
 	.llseek		= default_llseek,
 };
 
-static int __init run_selftests(struct crypto_shash *tfm_cmac,
-				struct crypto_kpp *tfm_ecdh)
+static int __init run_selftests(struct crypto_kpp *tfm_ecdh)
 {
 	ktime_t calltime, delta, rettime;
 	unsigned long long duration;
@@ -3769,31 +3726,31 @@ static int __init run_selftests(struct crypto_shash *tfm_cmac,
 		goto done;
 	}
 
-	err = test_f4(tfm_cmac);
+	err = test_f4();
 	if (err) {
 		BT_ERR("smp_f4 test failed");
 		goto done;
 	}
 
-	err = test_f5(tfm_cmac);
+	err = test_f5();
 	if (err) {
 		BT_ERR("smp_f5 test failed");
 		goto done;
 	}
 
-	err = test_f6(tfm_cmac);
+	err = test_f6();
 	if (err) {
 		BT_ERR("smp_f6 test failed");
 		goto done;
 	}
 
-	err = test_g2(tfm_cmac);
+	err = test_g2();
 	if (err) {
 		BT_ERR("smp_g2 test failed");
 		goto done;
 	}
 
-	err = test_h6(tfm_cmac);
+	err = test_h6();
 	if (err) {
 		BT_ERR("smp_h6 test failed");
 		goto done;
@@ -3820,26 +3777,17 @@ static int __init run_selftests(struct crypto_shash *tfm_cmac,
 
 int __init bt_selftest_smp(void)
 {
-	struct crypto_shash *tfm_cmac;
 	struct crypto_kpp *tfm_ecdh;
 	int err;
 
-	tfm_cmac = crypto_alloc_shash("cmac(aes)", 0, 0);
-	if (IS_ERR(tfm_cmac)) {
-		BT_ERR("Unable to create CMAC crypto context");
-		return PTR_ERR(tfm_cmac);
-	}
-
 	tfm_ecdh = crypto_alloc_kpp("ecdh-nist-p256", 0, 0);
 	if (IS_ERR(tfm_ecdh)) {
 		BT_ERR("Unable to create ECDH crypto context");
-		crypto_free_shash(tfm_cmac);
 		return PTR_ERR(tfm_ecdh);
 	}
 
-	err = run_selftests(tfm_cmac, tfm_ecdh);
+	err = run_selftests(tfm_ecdh);
 
-	crypto_free_shash(tfm_cmac);
 	crypto_free_kpp(tfm_ecdh);
 
 	return err;
diff --git a/net/bpf/test_run.c b/net/bpf/test_run.c
index 2bc04feadfabe..c9aea7052ba72 100644
--- a/net/bpf/test_run.c
+++ b/net/bpf/test_run.c
@@ -748,14 +748,35 @@ static void
 __bpf_prog_test_run_raw_tp(void *data)
 {
 	struct bpf_raw_tp_test_run_info *info = data;
+	struct srcu_ctr __percpu *scp = NULL;
 	struct bpf_trace_run_ctx run_ctx = {};
 	struct bpf_run_ctx *old_run_ctx;
 
 	old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx);
 
-	rcu_read_lock();
+	if (info->prog->sleepable) {
+		scp = rcu_read_lock_tasks_trace();
+		migrate_disable();
+	} else {
+		rcu_read_lock();
+	}
+
+	if (unlikely(!bpf_prog_get_recursion_context(info->prog))) {
+		bpf_prog_inc_misses_counter(info->prog);
+		goto out;
+	}
+
 	info->retval = bpf_prog_run(info->prog, info->ctx);
-	rcu_read_unlock();
+
+out:
+	bpf_prog_put_recursion_context(info->prog);
+
+	if (info->prog->sleepable) {
+		migrate_enable();
+		rcu_read_unlock_tasks_trace(scp);
+	} else {
+		rcu_read_unlock();
+	}
 
 	bpf_reset_run_ctx(old_run_ctx);
 }
@@ -783,6 +804,13 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
 	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 && cpu != 0)
 		return -EINVAL;
 
+	/*
+	 * Sleepable programs cannot run with preemption disabled or in
+	 * hardirq context (smp_call_function_single), reject the flag.
+	 */
+	if (prog->sleepable && (kattr->test.flags & BPF_F_TEST_RUN_ON_CPU))
+		return -EINVAL;
+
 	if (ctx_size_in) {
 		info.ctx = memdup_user(ctx_in, ctx_size_in);
 		if (IS_ERR(info.ctx))
@@ -791,24 +819,31 @@ int bpf_prog_test_run_raw_tp(struct bpf_prog *prog,
 		info.ctx = NULL;
 	}
 
+	info.retval = 0;
 	info.prog = prog;
 
-	current_cpu = get_cpu();
-	if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
-	    cpu == current_cpu) {
+	if (prog->sleepable) {
 		__bpf_prog_test_run_raw_tp(&info);
-	} else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
-		/* smp_call_function_single() also checks cpu_online()
-		 * after csd_lock(). However, since cpu is from user
-		 * space, let's do an extra quick check to filter out
-		 * invalid value before smp_call_function_single().
-		 */
-		err = -ENXIO;
 	} else {
-		err = smp_call_function_single(cpu, __bpf_prog_test_run_raw_tp,
-					       &info, 1);
+		current_cpu = get_cpu();
+		if ((kattr->test.flags & BPF_F_TEST_RUN_ON_CPU) == 0 ||
+		    cpu == current_cpu) {
+			__bpf_prog_test_run_raw_tp(&info);
+		} else if (cpu >= nr_cpu_ids || !cpu_online(cpu)) {
+			/*
+			 * smp_call_function_single() also checks cpu_online()
+			 * after csd_lock(). However, since cpu is from user
+			 * space, let's do an extra quick check to filter out
+			 * invalid value before smp_call_function_single().
+			 */
+			err = -ENXIO;
+		} else {
+			err = smp_call_function_single(cpu,
+						       __bpf_prog_test_run_raw_tp,
+						       &info, 1);
+		}
+		put_cpu();
 	}
-	put_cpu();
 
 	if (!err &&
 	    copy_to_user(&uattr->test.retval, &info.retval, sizeof(u32)))
diff --git a/net/bridge/br_arp_nd_proxy.c b/net/bridge/br_arp_nd_proxy.c
index deb1ab1f24b05..5263232278b4f 100644
--- a/net/bridge/br_arp_nd_proxy.c
+++ b/net/bridge/br_arp_nd_proxy.c
@@ -132,6 +132,7 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br,
 	__be32 sip, tip;
 
 	BR_INPUT_SKB_CB(skb)->proxyarp_replied = 0;
+	BR_INPUT_SKB_CB(skb)->grat_arp = 0;
 
 	if ((dev->flags & IFF_NOARP) ||
 	    !pskb_may_pull(skb, arp_hdr_len(dev)))
@@ -164,9 +165,10 @@ void br_do_proxy_suppress_arp(struct sk_buff *skb, struct net_bridge *br,
 			return;
 		if (parp->ar_op != htons(ARPOP_RREQUEST) &&
 		    parp->ar_op != htons(ARPOP_RREPLY) &&
-		    (ipv4_is_zeronet(sip) || sip == tip)) {
+		    sip == tip) {
 			/* prevent flooding to neigh suppress ports */
 			BR_INPUT_SKB_CB(skb)->proxyarp_replied = 1;
+			BR_INPUT_SKB_CB(skb)->grat_arp = 1;
 			return;
 		}
 	}
@@ -262,6 +264,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p,
 	int ns_olen;
 	int i, len;
 	u8 *daddr;
+	bool dad;
 	u16 pvid;
 
 	if (!dev || skb_linearize(request))
@@ -300,8 +303,13 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p,
 		}
 	}
 
+	dad = ipv6_addr_any(&ipv6_hdr(request)->saddr);
+
 	/* Ethernet header */
-	ether_addr_copy(eth_hdr(reply)->h_dest, daddr);
+	if (dad)
+		ipv6_eth_mc_map(&in6addr_linklocal_allnodes, eth_hdr(reply)->h_dest);
+	else
+		ether_addr_copy(eth_hdr(reply)->h_dest, daddr);
 	ether_addr_copy(eth_hdr(reply)->h_source, n->ha);
 	eth_hdr(reply)->h_proto = htons(ETH_P_IPV6);
 	reply->protocol = htons(ETH_P_IPV6);
@@ -317,7 +325,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p,
 	pip6->priority = ipv6_hdr(request)->priority;
 	pip6->nexthdr = IPPROTO_ICMPV6;
 	pip6->hop_limit = 255;
-	pip6->daddr = ipv6_hdr(request)->saddr;
+	pip6->daddr = dad ? in6addr_linklocal_allnodes : ipv6_hdr(request)->saddr;
 	pip6->saddr = *(struct in6_addr *)n->primary_key;
 
 	skb_pull(reply, sizeof(struct ipv6hdr));
@@ -330,7 +338,7 @@ static void br_nd_send(struct net_bridge *br, struct net_bridge_port *p,
 	na->icmph.icmp6_type = NDISC_NEIGHBOUR_ADVERTISEMENT;
 	na->icmph.icmp6_router = (n->flags & NTF_ROUTER) ? 1 : 0;
 	na->icmph.icmp6_override = 1;
-	na->icmph.icmp6_solicited = 1;
+	na->icmph.icmp6_solicited = dad ? 0 : 1;
 	na->target = ns->target;
 	ether_addr_copy(&na->opt[2], n->ha);
 	na->opt[0] = ND_OPT_TARGET_LL_ADDR;
@@ -413,6 +421,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
 	struct neighbour *n;
 
 	BR_INPUT_SKB_CB(skb)->proxyarp_replied = 0;
+	BR_INPUT_SKB_CB(skb)->grat_arp = 0;
 
 	if (br_is_neigh_suppress_enabled(p, vid))
 		return;
@@ -425,6 +434,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
 	    !msg->icmph.icmp6_solicited) {
 		/* prevent flooding to neigh suppress ports */
 		BR_INPUT_SKB_CB(skb)->proxyarp_replied = 1;
+		BR_INPUT_SKB_CB(skb)->grat_arp = 1;
 		return;
 	}
 
@@ -435,7 +445,7 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
 	saddr = &iphdr->saddr;
 	daddr = &iphdr->daddr;
 
-	if (ipv6_addr_any(saddr) || !ipv6_addr_cmp(saddr, daddr)) {
+	if (!ipv6_addr_cmp(saddr, daddr)) {
 		/* prevent flooding to neigh suppress ports */
 		BR_INPUT_SKB_CB(skb)->proxyarp_replied = 1;
 		return;
@@ -516,3 +526,21 @@ bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p, u16 vid)
 		return !!(p->flags & BR_NEIGH_SUPPRESS);
 	}
 }
+
+bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p, u16 vid)
+{
+	if (!vid)
+		return !!(p->flags & BR_NEIGH_FORWARD_GRAT);
+
+	if (p->flags & BR_NEIGH_VLAN_SUPPRESS) {
+		struct net_bridge_vlan_group *vg = nbp_vlan_group_rcu(p);
+		struct net_bridge_vlan *v;
+
+		v = br_vlan_find(vg, vid);
+		if (!v)
+			return false;
+		return !!(v->priv_flags & BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED);
+	} else {
+		return !!(p->flags & BR_NEIGH_FORWARD_GRAT);
+	}
+}
diff --git a/net/bridge/br_device.c b/net/bridge/br_device.c
index a35ceae0a6f2c..e7f343ab22d37 100644
--- a/net/bridge/br_device.c
+++ b/net/bridge/br_device.c
@@ -107,7 +107,7 @@ netdev_tx_t br_dev_xmit(struct sk_buff *skb, struct net_device *dev)
 		else
 			br_flood(br, skb, BR_PKT_MULTICAST, false, true, vid);
 	} else if ((dst = br_fdb_find_rcu(br, dest, vid)) != NULL) {
-		br_forward(dst->dst, skb, false, true);
+		br_forward(READ_ONCE(dst->dst), skb, false, true);
 	} else {
 		br_flood(br, skb, BR_PKT_UNICAST, false, true, vid);
 	}
diff --git a/net/bridge/br_fdb.c b/net/bridge/br_fdb.c
index ac81e58d5f70c..a114373c98163 100644
--- a/net/bridge/br_fdb.c
+++ b/net/bridge/br_fdb.c
@@ -470,7 +470,8 @@ void br_fdb_changeaddr(struct net_bridge_port *p, const unsigned char *newaddr)
 	spin_lock_bh(&br->hash_lock);
 	vg = nbp_vlan_group(p);
 	hlist_for_each_entry(f, &br->fdb_list, fdb_node) {
-		if (f->dst == p && test_bit(BR_FDB_LOCAL, &f->flags) &&
+		if (READ_ONCE(f->dst) == p &&
+		    test_bit(BR_FDB_LOCAL, &f->flags) &&
 		    !test_bit(BR_FDB_ADDED_BY_USER, &f->flags)) {
 			/* delete old one */
 			fdb_delete_local(br, p, f);
@@ -878,7 +879,7 @@ void br_fdb_delete_by_port(struct net_bridge *br,
 
 	spin_lock_bh(&br->hash_lock);
 	hlist_for_each_entry_safe(f, tmp, &br->fdb_list, fdb_node) {
-		if (f->dst != p)
+		if (READ_ONCE(f->dst) != p)
 			continue;
 
 		if (!do_all)
@@ -1631,7 +1632,7 @@ void br_fdb_clear_offload(const struct net_device *dev, u16 vid)
 
 	spin_lock_bh(&p->br->hash_lock);
 	hlist_for_each_entry(f, &p->br->fdb_list, fdb_node) {
-		if (f->dst == p && f->key.vlan_id == vid)
+		if (READ_ONCE(f->dst) == p && f->key.vlan_id == vid)
 			clear_bit(BR_FDB_OFFLOADED, &f->flags);
 	}
 	spin_unlock_bh(&p->br->hash_lock);
diff --git a/net/bridge/br_forward.c b/net/bridge/br_forward.c
index dea09096ad0fb..4a77d07433745 100644
--- a/net/bridge/br_forward.c
+++ b/net/bridge/br_forward.c
@@ -230,10 +230,17 @@ void br_flood(struct net_bridge *br, struct sk_buff *skb,
 		/* Do not flood to ports that enable proxy ARP */
 		if (p->flags & BR_PROXYARP)
 			continue;
-		if (BR_INPUT_SKB_CB(skb)->proxyarp_replied &&
-		    ((p->flags & BR_PROXYARP_WIFI) ||
-		     br_is_neigh_suppress_enabled(p, vid)))
-			continue;
+		if (BR_INPUT_SKB_CB(skb)->proxyarp_replied) {
+			if (p->flags & BR_PROXYARP_WIFI)
+				continue;
+			/* For gratuitous ARPs/NAs, check neigh_forward_grat.
+			 * For regular ARPs/NDs, check only neigh_suppress.
+			 */
+			if (br_is_neigh_suppress_enabled(p, vid) &&
+			    (!BR_INPUT_SKB_CB(skb)->grat_arp ||
+			     !br_is_neigh_forward_grat_enabled(p, vid)))
+				continue;
+		}
 
 		prev = maybe_deliver(prev, p, skb, local_orig);
 		if (IS_ERR(prev)) {
diff --git a/net/bridge/br_input.c b/net/bridge/br_input.c
index 2cbae0f9ae1f0..470615675bdc0 100644
--- a/net/bridge/br_input.c
+++ b/net/bridge/br_input.c
@@ -223,7 +223,7 @@ int br_handle_frame_finish(struct net *net, struct sock *sk, struct sk_buff *skb
 
 		if (now != READ_ONCE(dst->used))
 			WRITE_ONCE(dst->used, now);
-		br_forward(dst->dst, skb, local_rcv, false);
+		br_forward(READ_ONCE(dst->dst), skb, local_rcv, false);
 	} else {
 		if (!mcast_hit)
 			br_flood(br, skb, pkt_type, local_rcv, false, vid);
diff --git a/net/bridge/br_multicast.c b/net/bridge/br_multicast.c
index 881d866d687a0..5d6fdfb43c046 100644
--- a/net/bridge/br_multicast.c
+++ b/net/bridge/br_multicast.c
@@ -934,12 +934,12 @@ static struct sk_buff *br_ip4_multicast_alloc_query(struct net_bridge_mcast *brm
 	size_t pkt_size, igmp_hdr_size;
 	unsigned long now = jiffies;
 	struct igmpv3_query *ihv3;
+	unsigned long lmqt, mrt;
 	void *csum_start = NULL;
 	__sum16 *csum = NULL;
 	struct sk_buff *skb;
 	struct igmphdr *ih;
 	struct ethhdr *eth;
-	unsigned long lmqt;
 	struct iphdr *iph;
 	u16 lmqt_srcs = 0;
 
@@ -1004,15 +1004,15 @@ static struct sk_buff *br_ip4_multicast_alloc_query(struct net_bridge_mcast *brm
 	skb_put(skb, 24);
 
 	skb_set_transport_header(skb, skb->len);
+	mrt = group ? brmctx->multicast_last_member_interval :
+		      brmctx->multicast_query_response_interval;
 	*igmp_type = IGMP_HOST_MEMBERSHIP_QUERY;
 
 	switch (brmctx->multicast_igmp_version) {
 	case 2:
 		ih = igmp_hdr(skb);
 		ih->type = IGMP_HOST_MEMBERSHIP_QUERY;
-		ih->code = (group ? brmctx->multicast_last_member_interval :
-				    brmctx->multicast_query_response_interval) /
-			   (HZ / IGMP_TIMER_SCALE);
+		ih->code = mrt / (HZ / IGMP_TIMER_SCALE);
 		ih->group = group;
 		ih->csum = 0;
 		csum = &ih->csum;
@@ -1021,11 +1021,9 @@ static struct sk_buff *br_ip4_multicast_alloc_query(struct net_bridge_mcast *brm
 	case 3:
 		ihv3 = igmpv3_query_hdr(skb);
 		ihv3->type = IGMP_HOST_MEMBERSHIP_QUERY;
-		ihv3->code = (group ? brmctx->multicast_last_member_interval :
-				      brmctx->multicast_query_response_interval) /
-			     (HZ / IGMP_TIMER_SCALE);
+		ihv3->code = igmpv3_mrc(mrt / (HZ / IGMP_TIMER_SCALE));
 		ihv3->group = group;
-		ihv3->qqic = brmctx->multicast_query_interval / HZ;
+		ihv3->qqic = igmpv3_qqic(brmctx->multicast_query_interval / HZ);
 		ihv3->nsrcs = htons(lmqt_srcs);
 		ihv3->resv = 0;
 		ihv3->suppress = sflag;
@@ -1183,7 +1181,7 @@ static struct sk_buff *br_ip6_multicast_alloc_query(struct net_bridge_mcast *brm
 		break;
 	case 2:
 		mld2q = (struct mld2_query *)icmp6_hdr(skb);
-		mld2q->mld2q_mrc = htons((u16)jiffies_to_msecs(interval));
+		mld2q->mld2q_mrc = htons(mldv2_mrc(jiffies_to_msecs(interval)));
 		mld2q->mld2q_type = ICMPV6_MGM_QUERY;
 		mld2q->mld2q_code = 0;
 		mld2q->mld2q_cksum = 0;
@@ -1192,7 +1190,7 @@ static struct sk_buff *br_ip6_multicast_alloc_query(struct net_bridge_mcast *brm
 		mld2q->mld2q_suppress = sflag;
 		mld2q->mld2q_qrv = 2;
 		mld2q->mld2q_nsrcs = htons(llqt_srcs);
-		mld2q->mld2q_qqic = brmctx->multicast_query_interval / HZ;
+		mld2q->mld2q_qqic = mldv2_qqic(brmctx->multicast_query_interval / HZ);
 		mld2q->mld2q_mca = *group;
 		csum = &mld2q->mld2q_cksum;
 		csum_start = (void *)mld2q;
@@ -3518,7 +3516,7 @@ static void br_ip4_multicast_query(struct net_bridge_mcast *brmctx,
 			goto out;
 
 		max_delay = ih3->code ?
-			    IGMPV3_MRC(ih3->code) * (HZ / IGMP_TIMER_SCALE) : 1;
+			    igmpv3_mrt(ih3) * (HZ / IGMP_TIMER_SCALE) : 1;
 	} else {
 		goto out;
 	}
@@ -3606,7 +3604,7 @@ static int br_ip6_multicast_query(struct net_bridge_mcast *brmctx,
 		    mld2q->mld2q_suppress)
 			goto out;
 
-		max_delay = max(msecs_to_jiffies(mldv2_mrc(mld2q)), 1UL);
+		max_delay = max(msecs_to_jiffies(mldv2_mrd(mld2q)), 1UL);
 	}
 
 	is_general_query = group && ipv6_addr_any(group);
@@ -4455,8 +4453,6 @@ void br_multicast_dev_del(struct net_bridge *br)
 	br_multicast_ctx_deinit(&br->multicast_ctx);
 	br_multicast_gc(&deleted_head);
 	cancel_work_sync(&br->mcast_gc_work);
-
-	rcu_barrier();
 }
 
 int br_multicast_set_router(struct net_bridge_mcast *brmctx, unsigned long val)
@@ -4640,10 +4636,24 @@ static void br_multicast_start_querier(struct net_bridge_mcast *brmctx,
 	rcu_read_unlock();
 }
 
-static void br_multicast_del_grps(struct net_bridge *br)
+static void br_multicast_enable_all_ports(struct net_bridge *br)
 {
 	struct net_bridge_port *port;
 
+	if (br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED))
+		return;
+
+	list_for_each_entry(port, &br->port_list, list)
+		__br_multicast_enable_port_ctx(&port->multicast_ctx);
+}
+
+static void br_multicast_disable_all_ports(struct net_bridge *br)
+{
+	struct net_bridge_port *port;
+
+	if (br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED))
+		return;
+
 	list_for_each_entry(port, &br->port_list, list)
 		__br_multicast_disable_port_ctx(&port->multicast_ctx);
 }
@@ -4651,7 +4661,6 @@ static void br_multicast_del_grps(struct net_bridge *br)
 int br_multicast_toggle(struct net_bridge *br, unsigned long val,
 			struct netlink_ext_ack *extack)
 {
-	struct net_bridge_port *port;
 	bool change_snoopers = false;
 	int err = 0;
 
@@ -4668,7 +4677,7 @@ int br_multicast_toggle(struct net_bridge *br, unsigned long val,
 	br_opt_toggle(br, BROPT_MULTICAST_ENABLED, !!val);
 	if (!br_opt_get(br, BROPT_MULTICAST_ENABLED)) {
 		change_snoopers = true;
-		br_multicast_del_grps(br);
+		br_multicast_disable_all_ports(br);
 		goto unlock;
 	}
 
@@ -4676,8 +4685,7 @@ int br_multicast_toggle(struct net_bridge *br, unsigned long val,
 		goto unlock;
 
 	br_multicast_open(br);
-	list_for_each_entry(port, &br->port_list, list)
-		__br_multicast_enable_port_ctx(&port->multicast_ctx);
+	br_multicast_enable_all_ports(br);
 
 	change_snoopers = true;
 
diff --git a/net/bridge/br_netfilter_hooks.c b/net/bridge/br_netfilter_hooks.c
index 0ab1c94db4b95..0a394e5f43916 100644
--- a/net/bridge/br_netfilter_hooks.c
+++ b/net/bridge/br_netfilter_hooks.c
@@ -297,7 +297,11 @@ int br_nf_pre_routing_finish_bridge(struct net *net, struct sock *sk, struct sk_
 				goto free_skb;
 			}
 
-			neigh_hh_bridge(&neigh->hh, skb);
+			if (neigh_hh_bridge(&neigh->hh, skb)) {
+				neigh_release(neigh);
+				goto free_skb;
+			}
+
 			skb->dev = br_indev;
 
 			ret = br_handle_frame_finish(net, sk, skb);
diff --git a/net/bridge/br_netlink.c b/net/bridge/br_netlink.c
index 6fd5386a1d646..898326c201ef9 100644
--- a/net/bridge/br_netlink.c
+++ b/net/bridge/br_netlink.c
@@ -190,6 +190,7 @@ static inline size_t br_port_info_size(void)
 		+ nla_total_size(1)	/* IFLA_BRPORT_LOCKED */
 		+ nla_total_size(1)	/* IFLA_BRPORT_MAB */
 		+ nla_total_size(1)	/* IFLA_BRPORT_NEIGH_VLAN_SUPPRESS */
+		+ nla_total_size(1)	/* IFLA_BRPORT_NEIGH_FORWARD_GRAT */
 		+ nla_total_size(sizeof(struct ifla_bridge_id))	/* IFLA_BRPORT_ROOT_ID */
 		+ nla_total_size(sizeof(struct ifla_bridge_id))	/* IFLA_BRPORT_BRIDGE_ID */
 		+ nla_total_size(sizeof(u16))	/* IFLA_BRPORT_DESIGNATED_PORT */
@@ -282,7 +283,9 @@ static int br_port_fill_attrs(struct sk_buff *skb,
 	    nla_put_u8(skb, IFLA_BRPORT_LOCKED, !!(p->flags & BR_PORT_LOCKED)) ||
 	    nla_put_u8(skb, IFLA_BRPORT_MAB, !!(p->flags & BR_PORT_MAB)) ||
 	    nla_put_u8(skb, IFLA_BRPORT_NEIGH_VLAN_SUPPRESS,
-		       !!(p->flags & BR_NEIGH_VLAN_SUPPRESS)))
+		       !!(p->flags & BR_NEIGH_VLAN_SUPPRESS)) ||
+	    nla_put_u8(skb, IFLA_BRPORT_NEIGH_FORWARD_GRAT,
+		       !!(p->flags & BR_NEIGH_FORWARD_GRAT)))
 		return -EMSGSIZE;
 
 	timerval = br_timer_value(&p->message_age_timer);
@@ -902,6 +905,7 @@ static const struct nla_policy br_port_policy[IFLA_BRPORT_MAX + 1] = {
 	[IFLA_BRPORT_MCAST_MAX_GROUPS] = { .type = NLA_U32 },
 	[IFLA_BRPORT_NEIGH_VLAN_SUPPRESS] = NLA_POLICY_MAX(NLA_U8, 1),
 	[IFLA_BRPORT_BACKUP_NHID] = { .type = NLA_U32 },
+	[IFLA_BRPORT_NEIGH_FORWARD_GRAT] = NLA_POLICY_MAX(NLA_U8, 1),
 };
 
 /* Change the state of the port and notify spanning tree */
@@ -970,6 +974,8 @@ static int br_setport(struct net_bridge_port *p, struct nlattr *tb[],
 	br_set_port_flag(p, tb, IFLA_BRPORT_MAB, BR_PORT_MAB);
 	br_set_port_flag(p, tb, IFLA_BRPORT_NEIGH_VLAN_SUPPRESS,
 			 BR_NEIGH_VLAN_SUPPRESS);
+	br_set_port_flag(p, tb, IFLA_BRPORT_NEIGH_FORWARD_GRAT,
+			 BR_NEIGH_FORWARD_GRAT);
 
 	if ((p->flags & BR_PORT_MAB) &&
 	    (!(p->flags & BR_PORT_LOCKED) || !(p->flags & BR_LEARNING))) {
diff --git a/net/bridge/br_nf_core.c b/net/bridge/br_nf_core.c
index a8c67035e23c0..5b3a85b4be8fd 100644
--- a/net/bridge/br_nf_core.c
+++ b/net/bridge/br_nf_core.c
@@ -70,6 +70,7 @@ void br_netfilter_rtable_init(struct net_bridge *br)
 	struct rtable *rt = &br->fake_rtable;
 
 	rcuref_init(&rt->dst.__rcuref, 1);
+	save_dst_trace_buffer(&rt->dst, 1);
 	rt->dst.dev = br->dev;
 	dst_init_metrics(&rt->dst, br->metrics, false);
 	dst_metric_set(&rt->dst, RTAX_MTU, br->dev->mtu);
diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h
index bed1b1d9b2823..02671e648dac7 100644
--- a/net/bridge/br_private.h
+++ b/net/bridge/br_private.h
@@ -183,6 +183,7 @@ enum {
 	BR_VLFLAG_GLOBAL_MCAST_ENABLED = BIT(3),
 	BR_VLFLAG_NEIGH_SUPPRESS_ENABLED = BIT(4),
 	BR_VLFLAG_TAGGING_BY_SWITCHDEV = BIT(5),
+	BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED = BIT(6),
 };
 
 /**
@@ -600,6 +601,7 @@ struct br_input_skb_cb {
 	u8 proxyarp_replied:1;
 	u8 src_port_isolated:1;
 	u8 promisc:1;
+	u8 grat_arp:1;
 #ifdef CONFIG_BRIDGE_VLAN_FILTERING
 	u8 vlan_filtered:1;
 #endif
@@ -2361,4 +2363,5 @@ void br_do_suppress_nd(struct sk_buff *skb, struct net_bridge *br,
 		       u16 vid, struct net_bridge_port *p, struct nd_msg *msg);
 struct nd_msg *br_is_nd_neigh_msg(const struct sk_buff *skb, struct nd_msg *m);
 bool br_is_neigh_suppress_enabled(const struct net_bridge_port *p, u16 vid);
+bool br_is_neigh_forward_grat_enabled(const struct net_bridge_port *p, u16 vid);
 #endif
diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c
index 84a180927eb73..5560afcaaca32 100644
--- a/net/bridge/br_vlan.c
+++ b/net/bridge/br_vlan.c
@@ -2176,6 +2176,7 @@ static const struct nla_policy br_vlan_db_policy[BRIDGE_VLANDB_ENTRY_MAX + 1] =
 	[BRIDGE_VLANDB_ENTRY_MCAST_N_GROUPS]	= { .type = NLA_REJECT },
 	[BRIDGE_VLANDB_ENTRY_MCAST_MAX_GROUPS]	= { .type = NLA_U32 },
 	[BRIDGE_VLANDB_ENTRY_NEIGH_SUPPRESS]	= NLA_POLICY_MAX(NLA_U8, 1),
+	[BRIDGE_VLANDB_ENTRY_NEIGH_FORWARD_GRAT] = NLA_POLICY_MAX(NLA_U8, 1),
 };
 
 static int br_vlan_rtm_process_one(struct net_device *dev,
diff --git a/net/bridge/br_vlan_options.c b/net/bridge/br_vlan_options.c
index 5514e1fc8d1fa..92af1e558fda8 100644
--- a/net/bridge/br_vlan_options.c
+++ b/net/bridge/br_vlan_options.c
@@ -54,7 +54,8 @@ bool br_vlan_opts_eq_range(const struct net_bridge_vlan *v_curr,
 
 	/* Check user-visible priv_flags that affect output */
 	if ((v_curr->priv_flags ^ range_end->priv_flags) &
-	    (BR_VLFLAG_NEIGH_SUPPRESS_ENABLED | BR_VLFLAG_MCAST_ENABLED))
+	    (BR_VLFLAG_NEIGH_SUPPRESS_ENABLED | BR_VLFLAG_MCAST_ENABLED |
+	     BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED))
 		return false;
 
 #ifdef CONFIG_BRIDGE_IGMP_SNOOPING
@@ -74,7 +75,9 @@ bool br_vlan_opts_fill(struct sk_buff *skb, const struct net_bridge_vlan *v,
 	if (nla_put_u8(skb, BRIDGE_VLANDB_ENTRY_STATE, br_vlan_get_state(v)) ||
 	    !__vlan_tun_put(skb, v) ||
 	    nla_put_u8(skb, BRIDGE_VLANDB_ENTRY_NEIGH_SUPPRESS,
-		       !!(v->priv_flags & BR_VLFLAG_NEIGH_SUPPRESS_ENABLED)))
+		       !!(v->priv_flags & BR_VLFLAG_NEIGH_SUPPRESS_ENABLED)) ||
+	    nla_put_u8(skb, BRIDGE_VLANDB_ENTRY_NEIGH_FORWARD_GRAT,
+		       !!(v->priv_flags & BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED)))
 		return false;
 
 #ifdef CONFIG_BRIDGE_IGMP_SNOOPING
@@ -103,6 +106,7 @@ size_t br_vlan_opts_nl_size(void)
 	       + nla_total_size(sizeof(u32)) /* BRIDGE_VLANDB_ENTRY_MCAST_MAX_GROUPS */
 #endif
 	       + nla_total_size(sizeof(u8)) /* BRIDGE_VLANDB_ENTRY_NEIGH_SUPPRESS */
+	       + nla_total_size(sizeof(u8)) /* BRIDGE_VLANDB_ENTRY_NEIGH_FORWARD_GRAT */
 	       + 0;
 }
 
@@ -277,6 +281,22 @@ static int br_vlan_process_one_opts(const struct net_bridge *br,
 		}
 	}
 
+	if (tb[BRIDGE_VLANDB_ENTRY_NEIGH_FORWARD_GRAT]) {
+		bool enabled = v->priv_flags & BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED;
+		bool val = nla_get_u8(tb[BRIDGE_VLANDB_ENTRY_NEIGH_FORWARD_GRAT]);
+
+		if (!p) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Can't set neigh_forward_grat for non-port vlans");
+			return -EINVAL;
+		}
+
+		if (val != enabled) {
+			v->priv_flags ^= BR_VLFLAG_NEIGH_FORWARD_GRAT_ENABLED;
+			*changed = true;
+		}
+	}
+
 	return 0;
 }
 
diff --git a/net/core/dev.c b/net/core/dev.c
index 0c6c270d9f7d1..0b9bde16ab3b2 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -1874,6 +1874,7 @@ const char *netdev_cmd_to_name(enum netdev_cmd cmd)
 	N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
 	N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
 	N(XDP_FEAT_CHANGE)
+	N(DEBUG_UNREGISTER)
 	}
 #undef N
 	return "UNKNOWN_NETDEV_EVENT";
@@ -3993,10 +3994,11 @@ static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb,
 	struct skb_shared_info *shinfo;
 	struct net_iov *niov;
 
-	if (likely(skb_frags_readable(skb)))
+	if (likely(skb_frags_readable(skb) ||
+		   dev->netmem_tx == NETMEM_TX_NO_DMA))
 		goto out;
 
-	if (!dev->netmem_tx)
+	if (dev->netmem_tx == NETMEM_TX_NONE)
 		goto out_free;
 
 	shinfo = skb_shinfo(skb);
@@ -10144,17 +10146,36 @@ bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
 }
 EXPORT_SYMBOL(netdev_port_same_parent_id);
 
+static struct net_device *dev_get_iflink_dev(struct net_device *dev)
+{
+	struct net *net;
+
+	ASSERT_RTNL();
+
+	if (!dev->netdev_ops->ndo_get_iflink || !dev->rtnl_link_ops ||
+	    !dev->rtnl_link_ops->get_link_net)
+		return dev;
+
+	net = dev->rtnl_link_ops->get_link_net(dev);
+	return __dev_get_by_index(net, dev_get_iflink(dev));
+}
+
 int netif_change_proto_down(struct net_device *dev, bool proto_down)
 {
+	struct net_device *iflink_dev;
+
 	if (!dev->change_proto_down)
 		return -EOPNOTSUPP;
 	if (!netif_device_present(dev))
 		return -ENODEV;
+	iflink_dev = dev_get_iflink_dev(dev);
+	if (!iflink_dev)
+		return -ENODEV;
+	WRITE_ONCE(dev->proto_down, proto_down);
 	if (proto_down)
 		netif_carrier_off(dev);
-	else
+	else if (dev == iflink_dev || netif_carrier_ok(iflink_dev))
 		netif_carrier_on(dev);
-	WRITE_ONCE(dev->proto_down, proto_down);
 	return 0;
 }
 
@@ -11558,6 +11579,14 @@ int netdev_refcnt_read(const struct net_device *dev)
 }
 EXPORT_SYMBOL(netdev_refcnt_read);
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+static void dump_netdev_trace_buffer(const struct net_device *dev);
+static void erase_netdev_trace_buffer(const struct net_device *dev);
+#else
+static inline void dump_netdev_trace_buffer(const struct net_device *dev) { }
+static inline void erase_netdev_trace_buffer(const struct net_device *dev) { }
+#endif
+
 int netdev_unregister_timeout_secs __read_mostly = 10;
 
 #define WAIT_REFS_MIN_MSECS 1
@@ -11631,11 +11660,16 @@ static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
 
 		if (time_after(jiffies, warning_time +
 			       READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
+			rtnl_lock();
 			list_for_each_entry(dev, list, todo_list) {
 				pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
 					 dev->name, netdev_refcnt_read(dev));
 				ref_tracker_dir_print(&dev->refcnt_tracker, 10);
+				call_netdevice_notifiers(NETDEV_DEBUG_UNREGISTER, dev);
+				dump_netdev_trace_buffer(dev);
 			}
+			__rtnl_unlock();
+			rcu_barrier();
 
 			warning_time = jiffies;
 		}
@@ -12033,6 +12067,9 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
 
 	dev->priv_len = sizeof_priv;
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+	INIT_LIST_HEAD(&dev->netdev_trace_buffer_list);
+#endif
 	ref_tracker_dir_init(&dev->refcnt_tracker, 128, "netdev");
 #ifdef CONFIG_PCPU_DEV_REFCNT
 	dev->pcpu_refcnt = alloc_percpu(int);
@@ -12209,6 +12246,8 @@ void free_netdev(struct net_device *dev)
 
 	mutex_destroy(&dev->lock);
 
+	erase_netdev_trace_buffer(dev);
+
 	/*  Compatibility with error handling in drivers */
 	if (dev->reg_state == NETREG_UNINITIALIZED ||
 	    dev->reg_state == NETREG_DUMMY) {
@@ -13315,3 +13354,172 @@ static int __init net_dev_init(void)
 }
 
 subsys_initcall(net_dev_init);
+
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+
+#define NETDEV_TRACE_BUFFER_SIZE 32768
+static struct netdev_trace_buffer {
+	struct list_head list;
+	int prev_count;
+	atomic_t count;
+	int nr_entries;
+	unsigned long entries[20];
+} netdev_trace_buffer[NETDEV_TRACE_BUFFER_SIZE];
+static LIST_HEAD(netdev_trace_buffer_list);
+static DEFINE_SPINLOCK(netdev_trace_buffer_lock);
+static bool netdev_trace_buffer_exhausted;
+
+static int netdev_trace_buffer_init(void)
+{
+	int i;
+
+	for (i = 0; i < NETDEV_TRACE_BUFFER_SIZE; i++)
+		list_add_tail(&netdev_trace_buffer[i].list, &netdev_trace_buffer_list);
+	return 0;
+}
+pure_initcall(netdev_trace_buffer_init);
+
+static void dump_netdev_trace_buffer(const struct net_device *dev)
+{
+	struct netdev_trace_buffer *ptr;
+	int count, balance = 0, pos = 0;
+
+	dump_dst_trace_buffer(dev);
+	list_for_each_entry_rcu(ptr, &dev->netdev_trace_buffer_list, list,
+				/* list elements can't go away. */ 1) {
+		pos++;
+		count = atomic_read(&ptr->count);
+		balance += count;
+		if (ptr->prev_count == count)
+			continue;
+		ptr->prev_count = count;
+		pr_info("Call trace for %s[%d] %+d at\n", dev->name, pos, count);
+		stack_trace_print(ptr->entries, ptr->nr_entries, 4);
+		cond_resched();
+	}
+	if (!netdev_trace_buffer_exhausted)
+		pr_info("balance as of %s[%d] is %d\n", dev->name, pos, balance);
+}
+
+static void erase_netdev_trace_buffer(const struct net_device *dev)
+{
+	struct netdev_trace_buffer *ptr;
+	unsigned long flags;
+
+	spin_lock_irqsave(&netdev_trace_buffer_lock, flags);
+	while (!list_empty(&dev->netdev_trace_buffer_list)) {
+		ptr = list_first_entry(&dev->netdev_trace_buffer_list, typeof(*ptr), list);
+		list_del(&ptr->list);
+		list_add_tail(&ptr->list, &netdev_trace_buffer_list);
+	}
+	spin_unlock_irqrestore(&netdev_trace_buffer_lock, flags);
+}
+
+int trim_netdev_trace(unsigned long *entries, int nr_entries)
+{
+#ifdef CONFIG_KALLSYMS
+	char buffer[32] = { };
+	char *cp;
+	int i;
+
+	if (in_softirq()) {
+		static unsigned long __data_racy caller;
+
+		if (!caller) {
+			for (i = 0; i < nr_entries; i++) {
+				snprintf(buffer, sizeof(buffer) - 1, "%ps", (void *)entries[i]);
+				cp = strchr(buffer, ' ');
+				if (cp)
+					*cp = '\0';
+				if (!strcmp(buffer, "handle_softirqs")) {
+					caller = entries[i];
+					break;
+				}
+			}
+		}
+		for (i = 0; i < nr_entries; i++)
+			if (entries[i] == caller)
+				return i + 1;
+	} else if (current->flags & PF_WQ_WORKER) {
+		static unsigned long __data_racy caller;
+
+		if (!caller) {
+			for (i = 0; i < nr_entries; i++) {
+				snprintf(buffer, sizeof(buffer) - 1, "%ps", (void *)entries[i]);
+				cp = strchr(buffer, ' ');
+				if (cp)
+					*cp = '\0';
+				if (!strcmp(buffer, "process_one_work")) {
+					caller = entries[i];
+					break;
+				}
+			}
+		}
+		for (i = 0; i < nr_entries; i++)
+			if (entries[i] == caller)
+				return i + 1;
+	} else {
+		for (i = 0; i < nr_entries; i++) {
+			snprintf(buffer, sizeof(buffer) - 1, "%ps", (void *)entries[i]);
+			cp = strchr(buffer, ' ');
+			if (cp)
+				*cp = '\0';
+			if (buffer[0] == 'k') {
+				if (!strcmp(buffer, "ksys_unshare"))
+					return i + 1;
+			} else if (buffer[0] == 's') {
+				if (!strcmp(buffer, "sock_sendmsg_nosec") ||
+				    !strcmp(buffer, "sock_recvmsg_nosec"))
+					return i + 1;
+			} else if (buffer[0] == '_') {
+				if (!strcmp(buffer, "__sys_bind") ||
+				    !strcmp(buffer, "__sock_release") ||
+				    !strcmp(buffer, "__sys_bpf"))
+					return i + 1;
+			} else {
+				if (!strcmp(buffer, "do_sock_setsockopt"))
+					return i + 1;
+			}
+		}
+	}
+#endif
+	return nr_entries;
+}
+EXPORT_SYMBOL(trim_netdev_trace);
+
+void save_netdev_trace_buffer(struct net_device *dev, int delta)
+{
+	struct netdev_trace_buffer *ptr;
+	unsigned long entries[ARRAY_SIZE(ptr->entries)];
+	unsigned long nr_entries;
+	unsigned long flags;
+
+	if (in_nmi())
+		return;
+	nr_entries = stack_trace_save(entries, ARRAY_SIZE(ptr->entries), 1);
+	nr_entries = trim_netdev_trace(entries, nr_entries);
+	list_for_each_entry_rcu(ptr, &dev->netdev_trace_buffer_list, list,
+				/* list elements can't go away. */ 1) {
+		if (ptr->nr_entries == nr_entries &&
+		    !memcmp(ptr->entries, entries, nr_entries * sizeof(unsigned long))) {
+			atomic_add(delta, &ptr->count);
+			return;
+		}
+	}
+	spin_lock_irqsave(&netdev_trace_buffer_lock, flags);
+	if (!list_empty(&netdev_trace_buffer_list)) {
+		ptr = list_first_entry(&netdev_trace_buffer_list, typeof(*ptr), list);
+		list_del(&ptr->list);
+		ptr->prev_count = 0;
+		atomic_set(&ptr->count, delta);
+		ptr->nr_entries = nr_entries;
+		memmove(ptr->entries, entries, nr_entries * sizeof(unsigned long));
+		list_add_tail_rcu(&ptr->list, &dev->netdev_trace_buffer_list);
+	} else {
+		netdev_trace_buffer_exhausted = true;
+	}
+	spin_unlock_irqrestore(&netdev_trace_buffer_lock, flags);
+}
+EXPORT_SYMBOL(save_netdev_trace_buffer);
+
+#endif
diff --git a/net/core/devmem.c b/net/core/devmem.c
index 468344739db29..957d6b96216b3 100644
--- a/net/core/devmem.c
+++ b/net/core/devmem.c
@@ -181,7 +181,7 @@ int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
 }
 
 struct net_devmem_dmabuf_binding *
-net_devmem_bind_dmabuf(struct net_device *dev,
+net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
 		       struct device *dma_dev,
 		       enum dma_data_direction direction,
 		       unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
@@ -212,6 +212,7 @@ net_devmem_bind_dmabuf(struct net_device *dev,
 	}
 
 	binding->dev = dev;
+	binding->vdev = vdev;
 	xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
 
 	err = percpu_ref_init(&binding->ref,
@@ -241,6 +242,11 @@ net_devmem_bind_dmabuf(struct net_device *dev,
 	}
 
 	if (direction == DMA_TO_DEVICE) {
+		if (!IS_ALIGNED(dmabuf->size, PAGE_SIZE)) {
+			err = -EINVAL;
+			NL_SET_ERR_MSG(extack, "TX dma-buf size must be a multiple of PAGE_SIZE");
+			goto err_unmap;
+		}
 		binding->tx_vec = kvmalloc_objs(struct net_iov *,
 						dmabuf->size / PAGE_SIZE);
 		if (!binding->tx_vec) {
@@ -267,6 +273,12 @@ net_devmem_bind_dmabuf(struct net_device *dev,
 		size_t len = sg_dma_len(sg);
 		struct net_iov *niov;
 
+		if (!IS_ALIGNED(len, PAGE_SIZE)) {
+			err = -EINVAL;
+			NL_SET_ERR_MSG(extack, "dma-buf SG length must be PAGE_SIZE aligned");
+			goto err_free_chunks;
+		}
+
 		owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
 				     dev_to_node(&dev->dev));
 		if (!owner) {
@@ -396,7 +408,8 @@ struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk,
 	 */
 	dst_dev = dst_dev_rcu(dst);
 	if (unlikely(!dst_dev) ||
-	    unlikely(dst_dev != READ_ONCE(binding->dev))) {
+	    unlikely(dst_dev != READ_ONCE(binding->dev) &&
+		     dst_dev != READ_ONCE(binding->vdev))) {
 		err = -ENODEV;
 		goto out_unlock;
 	}
diff --git a/net/core/devmem.h b/net/core/devmem.h
index 1c5c18581fcb1..3852a56036cb4 100644
--- a/net/core/devmem.h
+++ b/net/core/devmem.h
@@ -19,7 +19,13 @@ struct net_devmem_dmabuf_binding {
 	struct dma_buf *dmabuf;
 	struct dma_buf_attachment *attachment;
 	struct sg_table *sgt;
+	/* Physical NIC that does the actual DMA for this binding. */
 	struct net_device *dev;
+	/* Opaque cookie identifying the virtual device (e.g. netkit) the user
+	 * called bind-tx on. Used only for pointer comparison. Never
+	 * dereferenced.
+	 */
+	void *vdev;
 	struct gen_pool *chunk_pool;
 	/* Protect dev */
 	struct mutex lock;
@@ -84,7 +90,7 @@ struct dmabuf_genpool_chunk_owner {
 
 void __net_devmem_dmabuf_binding_free(struct work_struct *wq);
 struct net_devmem_dmabuf_binding *
-net_devmem_bind_dmabuf(struct net_device *dev,
+net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
 		       struct device *dma_dev,
 		       enum dma_data_direction direction,
 		       unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
@@ -165,7 +171,7 @@ static inline void net_devmem_put_net_iov(struct net_iov *niov)
 }
 
 static inline struct net_devmem_dmabuf_binding *
-net_devmem_bind_dmabuf(struct net_device *dev,
+net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
 		       struct device *dma_dev,
 		       enum dma_data_direction direction,
 		       unsigned int dmabuf_fd,
diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c
index f23cea9e1aafb..2bf3cab5e557a 100644
--- a/net/core/drop_monitor.c
+++ b/net/core/drop_monitor.c
@@ -110,7 +110,7 @@ struct net_dm_alert_ops {
 	void (*kfree_skb_probe)(void *ignore, struct sk_buff *skb,
 				void *location,
 				enum skb_drop_reason reason,
-				struct sock *rx_sk);
+				const struct sock *rx_sk);
 	void (*napi_poll_probe)(void *ignore, struct napi_struct *napi,
 				int work, int budget);
 	void (*work_item_func)(struct work_struct *work);
@@ -266,7 +266,7 @@ static void trace_drop_common(struct sk_buff *skb, void *location)
 static void trace_kfree_skb_hit(void *ignore, struct sk_buff *skb,
 				void *location,
 				enum skb_drop_reason reason,
-				struct sock *rx_sk)
+				const struct sock *rx_sk)
 {
 	trace_drop_common(skb, location);
 }
@@ -493,7 +493,7 @@ static void net_dm_packet_trace_kfree_skb_hit(void *ignore,
 					      struct sk_buff *skb,
 					      void *location,
 					      enum skb_drop_reason reason,
-					      struct sock *rx_sk)
+					      const struct sock *rx_sk)
 {
 	ktime_t tstamp = ktime_get_real();
 	struct per_cpu_dm_data *data;
diff --git a/net/core/dst.c b/net/core/dst.c
index 092861133023c..45452fa62e24f 100644
--- a/net/core/dst.c
+++ b/net/core/dst.c
@@ -44,6 +44,82 @@ const struct dst_metrics dst_default_metrics = {
 };
 EXPORT_SYMBOL(dst_default_metrics);
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+
+#define DST_TRACE_BUFFER_SIZE 8192
+static struct dst_trace_buffer {
+	struct dst_entry *dst; // no-ref
+	struct net_device *ndev; // no-ref
+	int seq;
+	int delta;
+	u32 caller_id;
+	int nr_entries;
+	unsigned long entries[20];
+} dst_trace_buffer[DST_TRACE_BUFFER_SIZE];
+static bool dst_trace_buffer_exhausted;
+
+void dump_dst_trace_buffer(const struct net_device *ndev)
+{
+	struct dst_trace_buffer *ptr;
+	int count, balance = 0;
+	int i;
+
+	for (i = 0; i < DST_TRACE_BUFFER_SIZE; i++) {
+		ptr = &dst_trace_buffer[i];
+		if (!ptr->dst || ptr->ndev != ndev)
+			continue;
+		count = ptr->delta;
+		balance += count;
+		pr_info("Call trace for %s@%p[%d] %+d %c%u at\n", ndev->name, ptr->dst, ptr->seq,
+			count, ptr->caller_id & 0x80000000 ? 'C' : 'T',
+			ptr->caller_id & ~0x80000000);
+		stack_trace_print(ptr->entries, ptr->nr_entries, 4);
+	}
+	if (!dst_trace_buffer_exhausted)
+		pr_info("balance for %s@dst_entry is %d\n", ndev->name, balance);
+	else
+		pr_info("balance for %s@dst_entry is unknown\n", ndev->name);
+}
+
+static void erase_dst_trace_buffer(struct dst_entry *dst)
+{
+	int i;
+
+	for (i = 0; i < DST_TRACE_BUFFER_SIZE; i++)
+		if (dst_trace_buffer[i].dst == dst)
+			dst_trace_buffer[i].dst = NULL;
+}
+
+void save_dst_trace_buffer(struct dst_entry *dst, int delta)
+{
+	struct dst_trace_buffer *ptr;
+	unsigned long entries[ARRAY_SIZE(ptr->entries)];
+	unsigned long nr_entries;
+	int i;
+
+	if (!dst->dev)
+		return;
+	nr_entries = stack_trace_save(entries, ARRAY_SIZE(ptr->entries), 1);
+	nr_entries = trim_netdev_trace(entries, nr_entries);
+	for (i = 0; i < DST_TRACE_BUFFER_SIZE; i++) {
+		ptr = &dst_trace_buffer[i];
+		if (!ptr->dst && !cmpxchg(&ptr->dst, NULL, dst)) {
+			ptr->ndev = dst->dev;
+			ptr->seq = atomic_inc_return(&dst->dst_trace_seq);
+			ptr->delta = delta;
+			ptr->caller_id = in_task() ? task_pid_nr(current) :
+				0x80000000 + raw_smp_processor_id();
+			ptr->nr_entries = nr_entries;
+			memmove(ptr->entries, entries, nr_entries * sizeof(unsigned long));
+			return;
+		}
+	}
+	dst_trace_buffer_exhausted = true;
+}
+EXPORT_SYMBOL(save_dst_trace_buffer);
+
+#endif
+
 void dst_init(struct dst_entry *dst, struct dst_ops *ops,
 	      struct net_device *dev, int initial_obsolete,
 	      unsigned short flags)
@@ -67,6 +143,7 @@ void dst_init(struct dst_entry *dst, struct dst_ops *ops,
 #endif
 	dst->lwtstate = NULL;
 	rcuref_init(&dst->__rcuref, 1);
+	save_dst_trace_buffer(dst, 1);
 	INIT_LIST_HEAD(&dst->rt_uncached);
 	dst->rt_uncached_list = NULL;
 	dst->__use = 0;
@@ -116,6 +193,10 @@ static void dst_destroy(struct dst_entry *dst)
 
 	lwtstate_put(dst->lwtstate);
 
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+	erase_dst_trace_buffer(dst);
+#endif
+
 	if (dst->flags & DST_METADATA)
 		metadata_dst_free((struct metadata_dst *)dst);
 	else
@@ -165,6 +246,8 @@ static void dst_count_dec(struct dst_entry *dst)
 
 void dst_release(struct dst_entry *dst)
 {
+	if (dst)
+		save_dst_trace_buffer(dst, -1);
 	if (dst && rcuref_put(&dst->__rcuref)) {
 #ifdef CONFIG_DST_CACHE
 		if (dst->flags & DST_METADATA) {
@@ -182,6 +265,8 @@ EXPORT_SYMBOL(dst_release);
 
 void dst_release_immediate(struct dst_entry *dst)
 {
+	if (dst)
+		save_dst_trace_buffer(dst, -1);
 	if (dst && rcuref_put(&dst->__rcuref)) {
 		dst_count_dec(dst);
 		dst_destroy(dst);
diff --git a/net/core/gen_stats.c b/net/core/gen_stats.c
index b71ccaec09914..9468bc5ca3f46 100644
--- a/net/core/gen_stats.c
+++ b/net/core/gen_stats.c
@@ -124,13 +124,13 @@ void gnet_stats_basic_sync_init(struct gnet_stats_basic_sync *b)
 EXPORT_SYMBOL(gnet_stats_basic_sync_init);
 
 static void gnet_stats_add_basic_cpu(struct gnet_stats_basic_sync *bstats,
-				     struct gnet_stats_basic_sync __percpu *cpu)
+				     const struct gnet_stats_basic_sync __percpu *cpu)
 {
 	u64 t_bytes = 0, t_packets = 0;
 	int i;
 
 	for_each_possible_cpu(i) {
-		struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i);
+		const struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i);
 		unsigned int start;
 		u64 bytes, packets;
 
@@ -147,8 +147,8 @@ static void gnet_stats_add_basic_cpu(struct gnet_stats_basic_sync *bstats,
 }
 
 void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats,
-			  struct gnet_stats_basic_sync __percpu *cpu,
-			  struct gnet_stats_basic_sync *b, bool running)
+			  const struct gnet_stats_basic_sync __percpu *cpu,
+			  const struct gnet_stats_basic_sync *b, bool running)
 {
 	unsigned int start;
 	u64 bytes = 0;
@@ -172,8 +172,8 @@ void gnet_stats_add_basic(struct gnet_stats_basic_sync *bstats,
 EXPORT_SYMBOL(gnet_stats_add_basic);
 
 static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets,
-				  struct gnet_stats_basic_sync __percpu *cpu,
-				  struct gnet_stats_basic_sync *b, bool running)
+				  const struct gnet_stats_basic_sync __percpu *cpu,
+				  const struct gnet_stats_basic_sync *b, bool running)
 {
 	unsigned int start;
 
@@ -182,8 +182,7 @@ static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets,
 		int i;
 
 		for_each_possible_cpu(i) {
-			struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i);
-			unsigned int start;
+			const struct gnet_stats_basic_sync *bcpu = per_cpu_ptr(cpu, i);
 			u64 bytes, packets;
 
 			do {
@@ -209,8 +208,8 @@ static void gnet_stats_read_basic(u64 *ret_bytes, u64 *ret_packets,
 
 static int
 ___gnet_stats_copy_basic(struct gnet_dump *d,
-			 struct gnet_stats_basic_sync __percpu *cpu,
-			 struct gnet_stats_basic_sync *b,
+			 const struct gnet_stats_basic_sync __percpu *cpu,
+			 const struct gnet_stats_basic_sync *b,
 			 int type, bool running)
 {
 	u64 bstats_bytes, bstats_packets;
@@ -258,8 +257,8 @@ ___gnet_stats_copy_basic(struct gnet_dump *d,
  */
 int
 gnet_stats_copy_basic(struct gnet_dump *d,
-		      struct gnet_stats_basic_sync __percpu *cpu,
-		      struct gnet_stats_basic_sync *b,
+		      const struct gnet_stats_basic_sync __percpu *cpu,
+		      const struct gnet_stats_basic_sync *b,
 		      bool running)
 {
 	return ___gnet_stats_copy_basic(d, cpu, b, TCA_STATS_BASIC, running);
@@ -285,8 +284,8 @@ EXPORT_SYMBOL(gnet_stats_copy_basic);
  */
 int
 gnet_stats_copy_basic_hw(struct gnet_dump *d,
-			 struct gnet_stats_basic_sync __percpu *cpu,
-			 struct gnet_stats_basic_sync *b,
+			 const struct gnet_stats_basic_sync __percpu *cpu,
+			 const struct gnet_stats_basic_sync *b,
 			 bool running)
 {
 	return ___gnet_stats_copy_basic(d, cpu, b, TCA_STATS_BASIC_HW, running);
@@ -345,11 +344,11 @@ static void gnet_stats_add_queue_cpu(struct gnet_stats_queue *qstats,
 	for_each_possible_cpu(i) {
 		const struct gnet_stats_queue *qcpu = per_cpu_ptr(q, i);
 
-		qstats->qlen += qcpu->qlen;
-		qstats->backlog += qcpu->backlog;
-		qstats->drops += qcpu->drops;
-		qstats->requeues += qcpu->requeues;
-		qstats->overlimits += qcpu->overlimits;
+		qstats->qlen += READ_ONCE(qcpu->qlen);
+		qstats->backlog += READ_ONCE(qcpu->backlog);
+		qstats->drops += READ_ONCE(qcpu->drops);
+		qstats->requeues += READ_ONCE(qcpu->requeues);
+		qstats->overlimits += READ_ONCE(qcpu->overlimits);
 	}
 }
 
@@ -360,11 +359,11 @@ void gnet_stats_add_queue(struct gnet_stats_queue *qstats,
 	if (cpu) {
 		gnet_stats_add_queue_cpu(qstats, cpu);
 	} else {
-		qstats->qlen += q->qlen;
-		qstats->backlog += q->backlog;
-		qstats->drops += q->drops;
-		qstats->requeues += q->requeues;
-		qstats->overlimits += q->overlimits;
+		qstats->qlen += READ_ONCE(q->qlen);
+		qstats->backlog += READ_ONCE(q->backlog);
+		qstats->drops += READ_ONCE(q->drops);
+		qstats->requeues += READ_ONCE(q->requeues);
+		qstats->overlimits += READ_ONCE(q->overlimits);
 	}
 }
 EXPORT_SYMBOL(gnet_stats_add_queue);
@@ -385,8 +384,8 @@ EXPORT_SYMBOL(gnet_stats_add_queue);
  */
 int
 gnet_stats_copy_queue(struct gnet_dump *d,
-		      struct gnet_stats_queue __percpu *cpu_q,
-		      struct gnet_stats_queue *q, __u32 qlen)
+		      const struct gnet_stats_queue __percpu *cpu_q,
+		      const struct gnet_stats_queue *q, __u32 qlen)
 {
 	struct gnet_stats_queue qstats = {0};
 
diff --git a/net/core/lock_debug.c b/net/core/lock_debug.c
index 9e9fb25314b9c..78d611bb6d1cd 100644
--- a/net/core/lock_debug.c
+++ b/net/core/lock_debug.c
@@ -29,6 +29,7 @@ int netdev_debug_event(struct notifier_block *nb, unsigned long event,
 	case NETDEV_DOWN:
 	case NETDEV_REBOOT:
 	case NETDEV_UNREGISTER:
+	case NETDEV_DEBUG_UNREGISTER:
 	case NETDEV_CHANGEMTU:
 	case NETDEV_CHANGEADDR:
 	case NETDEV_PRE_CHANGEADDR:
diff --git a/net/core/netdev-genl-gen.c b/net/core/netdev-genl-gen.c
index 81aecb5d3bc5d..c7e138bfe3457 100644
--- a/net/core/netdev-genl-gen.c
+++ b/net/core/netdev-genl-gen.c
@@ -51,14 +51,28 @@ static const struct nla_policy netdev_dev_get_nl_policy[NETDEV_A_DEV_IFINDEX + 1
 
 /* NETDEV_CMD_PAGE_POOL_GET - do */
 #ifdef CONFIG_PAGE_POOL
-static const struct nla_policy netdev_page_pool_get_nl_policy[NETDEV_A_PAGE_POOL_ID + 1] = {
+static const struct nla_policy netdev_page_pool_get_do_nl_policy[NETDEV_A_PAGE_POOL_ID + 1] = {
 	[NETDEV_A_PAGE_POOL_ID] = NLA_POLICY_FULL_RANGE(NLA_UINT, &netdev_a_page_pool_id_range),
 };
 #endif /* CONFIG_PAGE_POOL */
 
+/* NETDEV_CMD_PAGE_POOL_GET - dump */
+#ifdef CONFIG_PAGE_POOL
+static const struct nla_policy netdev_page_pool_get_dump_nl_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = {
+	[NETDEV_A_PAGE_POOL_IFINDEX] = NLA_POLICY_FULL_RANGE(NLA_U32, &netdev_a_page_pool_ifindex_range),
+};
+#endif /* CONFIG_PAGE_POOL */
+
 /* NETDEV_CMD_PAGE_POOL_STATS_GET - do */
 #ifdef CONFIG_PAGE_POOL_STATS
-static const struct nla_policy netdev_page_pool_stats_get_nl_policy[NETDEV_A_PAGE_POOL_STATS_INFO + 1] = {
+static const struct nla_policy netdev_page_pool_stats_get_do_nl_policy[NETDEV_A_PAGE_POOL_STATS_INFO + 1] = {
+	[NETDEV_A_PAGE_POOL_STATS_INFO] = NLA_POLICY_NESTED(netdev_page_pool_info_nl_policy),
+};
+#endif /* CONFIG_PAGE_POOL_STATS */
+
+/* NETDEV_CMD_PAGE_POOL_STATS_GET - dump */
+#ifdef CONFIG_PAGE_POOL_STATS
+static const struct nla_policy netdev_page_pool_stats_get_dump_nl_policy[NETDEV_A_PAGE_POOL_STATS_INFO + 1] = {
 	[NETDEV_A_PAGE_POOL_STATS_INFO] = NLA_POLICY_NESTED(netdev_page_pool_info_nl_policy),
 };
 #endif /* CONFIG_PAGE_POOL_STATS */
@@ -138,28 +152,32 @@ static const struct genl_split_ops netdev_nl_ops[] = {
 	{
 		.cmd		= NETDEV_CMD_PAGE_POOL_GET,
 		.doit		= netdev_nl_page_pool_get_doit,
-		.policy		= netdev_page_pool_get_nl_policy,
+		.policy		= netdev_page_pool_get_do_nl_policy,
 		.maxattr	= NETDEV_A_PAGE_POOL_ID,
 		.flags		= GENL_CMD_CAP_DO,
 	},
 	{
-		.cmd	= NETDEV_CMD_PAGE_POOL_GET,
-		.dumpit	= netdev_nl_page_pool_get_dumpit,
-		.flags	= GENL_CMD_CAP_DUMP,
+		.cmd		= NETDEV_CMD_PAGE_POOL_GET,
+		.dumpit		= netdev_nl_page_pool_get_dumpit,
+		.policy		= netdev_page_pool_get_dump_nl_policy,
+		.maxattr	= NETDEV_A_PAGE_POOL_IFINDEX,
+		.flags		= GENL_CMD_CAP_DUMP,
 	},
 #endif /* CONFIG_PAGE_POOL */
 #ifdef CONFIG_PAGE_POOL_STATS
 	{
 		.cmd		= NETDEV_CMD_PAGE_POOL_STATS_GET,
 		.doit		= netdev_nl_page_pool_stats_get_doit,
-		.policy		= netdev_page_pool_stats_get_nl_policy,
+		.policy		= netdev_page_pool_stats_get_do_nl_policy,
 		.maxattr	= NETDEV_A_PAGE_POOL_STATS_INFO,
 		.flags		= GENL_CMD_CAP_DO,
 	},
 	{
-		.cmd	= NETDEV_CMD_PAGE_POOL_STATS_GET,
-		.dumpit	= netdev_nl_page_pool_stats_get_dumpit,
-		.flags	= GENL_CMD_CAP_DUMP,
+		.cmd		= NETDEV_CMD_PAGE_POOL_STATS_GET,
+		.dumpit		= netdev_nl_page_pool_stats_get_dumpit,
+		.policy		= netdev_page_pool_stats_get_dump_nl_policy,
+		.maxattr	= NETDEV_A_PAGE_POOL_STATS_INFO,
+		.flags		= GENL_CMD_CAP_DUMP,
 	},
 #endif /* CONFIG_PAGE_POOL_STATS */
 	{
diff --git a/net/core/netdev-genl.c b/net/core/netdev-genl.c
index b8f6076d80072..b4d48f3672a5b 100644
--- a/net/core/netdev-genl.c
+++ b/net/core/netdev-genl.c
@@ -1077,7 +1077,7 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
 		goto err_rxq_bitmap;
 	}
 
-	binding = net_devmem_bind_dmabuf(netdev, dma_dev, DMA_FROM_DEVICE,
+	binding = net_devmem_bind_dmabuf(netdev, NULL, dma_dev, DMA_FROM_DEVICE,
 					 dmabuf_fd, priv, info->extack);
 	if (IS_ERR(binding)) {
 		err = PTR_ERR(binding);
@@ -1119,9 +1119,43 @@ int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
 	return err;
 }
 
+/* Find the DMA-capable device for a netmem TX binding.
+ *
+ * For NETMEM_TX_DMA devices, return the device itself.
+ * For NETMEM_TX_NO_DMA devices, walk leased RX queues to find the underlying
+ * physical device and return it.
+ */
+static struct net_device *
+netdev_find_netmem_tx_dev(struct net_device *dev)
+{
+	struct netdev_rx_queue *lease_rxq;
+	struct net_device *phys_dev;
+	int i;
+
+	if (dev->netmem_tx == NETMEM_TX_DMA)
+		return dev;
+
+	if (dev->netmem_tx != NETMEM_TX_NO_DMA)
+		return NULL;
+
+	for (i = 0; i < dev->real_num_rx_queues; i++) {
+		lease_rxq = READ_ONCE(__netif_get_rx_queue(dev, i)->lease);
+		if (!lease_rxq)
+			continue;
+
+		phys_dev = lease_rxq->dev;
+		if (netif_device_present(phys_dev) &&
+		    phys_dev->netmem_tx == NETMEM_TX_DMA)
+			return phys_dev;
+	}
+
+	return NULL;
+}
+
 int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info)
 {
 	struct net_devmem_dmabuf_binding *binding;
+	struct net_device *bind_dev;
 	struct netdev_nl_sock *priv;
 	struct net_device *netdev;
 	struct device *dma_dev;
@@ -1164,29 +1198,48 @@ int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info)
 		goto err_unlock_netdev;
 	}
 
-	if (!netdev->netmem_tx) {
+	if (netdev->netmem_tx == NETMEM_TX_NONE) {
 		err = -EOPNOTSUPP;
 		NL_SET_ERR_MSG(info->extack,
 			       "Driver does not support netmem TX");
 		goto err_unlock_netdev;
 	}
 
-	dma_dev = netdev_queue_get_dma_dev(netdev, 0, NETDEV_QUEUE_TYPE_TX);
-	binding = net_devmem_bind_dmabuf(netdev, dma_dev, DMA_TO_DEVICE,
-					 dmabuf_fd, priv, info->extack);
+	bind_dev = netdev_find_netmem_tx_dev(netdev);
+	if (!bind_dev) {
+		err = -EOPNOTSUPP;
+		NL_SET_ERR_MSG(info->extack,
+			       "No DMA-capable device found for netmem TX");
+		goto err_unlock_netdev;
+	}
+
+	if (bind_dev != netdev)
+		netdev_lock(bind_dev);
+
+	dma_dev = netdev_queue_get_dma_dev(bind_dev, 0, NETDEV_QUEUE_TYPE_TX);
+
+	binding = net_devmem_bind_dmabuf(bind_dev,
+					 bind_dev != netdev ? netdev : NULL,
+					 dma_dev, DMA_TO_DEVICE, dmabuf_fd,
+					 priv, info->extack);
 	if (IS_ERR(binding)) {
 		err = PTR_ERR(binding);
-		goto err_unlock_netdev;
+		goto err_unlock_bind_dev;
 	}
 
 	nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
 	genlmsg_end(rsp, hdr);
 
+	if (bind_dev != netdev)
+		netdev_unlock(bind_dev);
 	netdev_unlock(netdev);
 	mutex_unlock(&priv->lock);
 
 	return genlmsg_reply(rsp, info);
 
+err_unlock_bind_dev:
+	if (bind_dev != netdev)
+		netdev_unlock(bind_dev);
 err_unlock_netdev:
 	netdev_unlock(netdev);
 err_unlock_sock:
diff --git a/net/core/netmem_priv.h b/net/core/netmem_priv.h
index 3e6fde8f1726f..23175cb2bd866 100644
--- a/net/core/netmem_priv.h
+++ b/net/core/netmem_priv.h
@@ -8,18 +8,21 @@ static inline unsigned long netmem_get_pp_magic(netmem_ref netmem)
 	return netmem_to_nmdesc(netmem)->pp_magic & ~PP_DMA_INDEX_MASK;
 }
 
-static inline bool netmem_is_pp(netmem_ref netmem)
+static inline void netmem_or_pp_magic(netmem_ref netmem, unsigned long pp_magic)
+{
+	netmem_to_nmdesc(netmem)->pp_magic |= pp_magic;
+}
+
+static inline void netmem_clear_pp_magic(netmem_ref netmem)
 {
-	struct page *page;
+	WARN_ON_ONCE(netmem_to_nmdesc(netmem)->pp_magic & PP_DMA_INDEX_MASK);
 
-	/* XXX: Now that the offset of page_type is shared between
-	 * struct page and net_iov, just cast the netmem to struct page
-	 * unconditionally by clearing NET_IOV if any, no matter whether
-	 * it comes from struct net_iov or struct page.  This should be
-	 * adjusted once the offset is no longer shared.
-	 */
-	page = (struct page *)((__force unsigned long)netmem & ~NET_IOV);
-	return PageNetpp(page);
+	netmem_to_nmdesc(netmem)->pp_magic = 0;
+}
+
+static inline bool netmem_is_pp(netmem_ref netmem)
+{
+	return (netmem_get_pp_magic(netmem) & PP_MAGIC_MASK) == PP_SIGNATURE;
 }
 
 static inline void netmem_set_pp(netmem_ref netmem, struct page_pool *pool)
diff --git a/net/core/netpoll.c b/net/core/netpoll.c
index 84faace50ac28..33ab3d827a427 100644
--- a/net/core/netpoll.c
+++ b/net/core/netpoll.c
@@ -29,12 +29,10 @@
 #include <linux/slab.h>
 #include <linux/export.h>
 #include <linux/if_vlan.h>
+#include <linux/udp.h>
 #include <net/tcp.h>
-#include <net/udp.h>
 #include <net/addrconf.h>
 #include <net/ndisc.h>
-#include <net/ip6_checksum.h>
-#include <linux/unaligned.h>
 #include <trace/events/napi.h>
 #include <linux/kconfig.h>
 
@@ -53,8 +51,6 @@
 	 sizeof(struct udphdr) +					\
 	 MAX_UDP_CHUNK)
 
-static void zap_completion_queue(void);
-
 static unsigned int carrier_timeout = 4;
 module_param(carrier_timeout, uint, 0644);
 
@@ -201,7 +197,7 @@ void netpoll_poll_dev(struct net_device *dev)
 
 	up(&ni->dev_lock);
 
-	zap_completion_queue();
+	netpoll_zap_completion_queue();
 }
 EXPORT_SYMBOL(netpoll_poll_dev);
 
@@ -240,7 +236,7 @@ static void refill_skbs(struct netpoll *np)
 	}
 }
 
-static void zap_completion_queue(void)
+void netpoll_zap_completion_queue(void)
 {
 	unsigned long flags;
 	struct softnet_data *sd = &get_cpu_var(softnet_data);
@@ -267,33 +263,7 @@ static void zap_completion_queue(void)
 
 	put_cpu_var(softnet_data);
 }
-
-static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve)
-{
-	int count = 0;
-	struct sk_buff *skb;
-
-	zap_completion_queue();
-repeat:
-
-	skb = alloc_skb(len, GFP_ATOMIC);
-	if (!skb) {
-		skb = skb_dequeue(&np->skb_pool);
-		schedule_work(&np->refill_wq);
-	}
-
-	if (!skb) {
-		if (++count < 10) {
-			netpoll_poll_dev(np->dev);
-			goto repeat;
-		}
-		return NULL;
-	}
-
-	refcount_set(&skb->users, 1);
-	skb_reserve(skb, reserve);
-	return skb;
-}
+EXPORT_SYMBOL_NS_GPL(netpoll_zap_completion_queue, "NETDEV_INTERNAL");
 
 static int netpoll_owner_active(struct net_device *dev)
 {
@@ -369,31 +339,6 @@ static netdev_tx_t __netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
 	return ret;
 }
 
-static void netpoll_udp_checksum(struct netpoll *np, struct sk_buff *skb,
-				 int len)
-{
-	struct udphdr *udph;
-	int udp_len;
-
-	udp_len = len + sizeof(struct udphdr);
-	udph = udp_hdr(skb);
-
-	/* check needs to be set, since it will be consumed in csum_partial */
-	udph->check = 0;
-	if (np->ipv6)
-		udph->check = csum_ipv6_magic(&np->local_ip.in6,
-					      &np->remote_ip.in6,
-					      udp_len, IPPROTO_UDP,
-					      csum_partial(udph, udp_len, 0));
-	else
-		udph->check = csum_tcpudp_magic(np->local_ip.ip,
-						np->remote_ip.ip,
-						udp_len, IPPROTO_UDP,
-						csum_partial(udph, udp_len, 0));
-	if (udph->check == 0)
-		udph->check = CSUM_MANGLED_0;
-}
-
 netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
 {
 	unsigned long flags;
@@ -411,125 +356,6 @@ netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
 }
 EXPORT_SYMBOL(netpoll_send_skb);
 
-static void push_ipv6(struct netpoll *np, struct sk_buff *skb, int len)
-{
-	struct ipv6hdr *ip6h;
-
-	skb_push(skb, sizeof(struct ipv6hdr));
-	skb_reset_network_header(skb);
-	ip6h = ipv6_hdr(skb);
-
-	/* ip6h->version = 6; ip6h->priority = 0; */
-	*(unsigned char *)ip6h = 0x60;
-	ip6h->flow_lbl[0] = 0;
-	ip6h->flow_lbl[1] = 0;
-	ip6h->flow_lbl[2] = 0;
-
-	ip6h->payload_len = htons(sizeof(struct udphdr) + len);
-	ip6h->nexthdr = IPPROTO_UDP;
-	ip6h->hop_limit = 32;
-	ip6h->saddr = np->local_ip.in6;
-	ip6h->daddr = np->remote_ip.in6;
-
-	skb->protocol = htons(ETH_P_IPV6);
-}
-
-static void push_ipv4(struct netpoll *np, struct sk_buff *skb, int len)
-{
-	static atomic_t ip_ident;
-	struct iphdr *iph;
-	int ip_len;
-
-	ip_len = len + sizeof(struct udphdr) + sizeof(struct iphdr);
-
-	skb_push(skb, sizeof(struct iphdr));
-	skb_reset_network_header(skb);
-	iph = ip_hdr(skb);
-
-	/* iph->version = 4; iph->ihl = 5; */
-	*(unsigned char *)iph = 0x45;
-	iph->tos = 0;
-	put_unaligned(htons(ip_len), &iph->tot_len);
-	iph->id = htons(atomic_inc_return(&ip_ident));
-	iph->frag_off = 0;
-	iph->ttl = 64;
-	iph->protocol = IPPROTO_UDP;
-	iph->check = 0;
-	put_unaligned(np->local_ip.ip, &iph->saddr);
-	put_unaligned(np->remote_ip.ip, &iph->daddr);
-	iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
-	skb->protocol = htons(ETH_P_IP);
-}
-
-static void push_udp(struct netpoll *np, struct sk_buff *skb, int len)
-{
-	struct udphdr *udph;
-	int udp_len;
-
-	udp_len = len + sizeof(struct udphdr);
-
-	skb_push(skb, sizeof(struct udphdr));
-	skb_reset_transport_header(skb);
-
-	udph = udp_hdr(skb);
-	udph->source = htons(np->local_port);
-	udph->dest = htons(np->remote_port);
-	udph->len = htons(udp_len);
-
-	netpoll_udp_checksum(np, skb, len);
-}
-
-static void push_eth(struct netpoll *np, struct sk_buff *skb)
-{
-	struct ethhdr *eth;
-
-	eth = skb_push(skb, ETH_HLEN);
-	skb_reset_mac_header(skb);
-	ether_addr_copy(eth->h_source, np->dev->dev_addr);
-	ether_addr_copy(eth->h_dest, np->remote_mac);
-	if (np->ipv6)
-		eth->h_proto = htons(ETH_P_IPV6);
-	else
-		eth->h_proto = htons(ETH_P_IP);
-}
-
-int netpoll_send_udp(struct netpoll *np, const char *msg, int len)
-{
-	int total_len, ip_len, udp_len;
-	struct sk_buff *skb;
-
-	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
-		WARN_ON_ONCE(!irqs_disabled());
-
-	udp_len = len + sizeof(struct udphdr);
-	if (np->ipv6)
-		ip_len = udp_len + sizeof(struct ipv6hdr);
-	else
-		ip_len = udp_len + sizeof(struct iphdr);
-
-	total_len = ip_len + LL_RESERVED_SPACE(np->dev);
-
-	skb = find_skb(np, total_len + np->dev->needed_tailroom,
-		       total_len - len);
-	if (!skb)
-		return -ENOMEM;
-
-	skb_copy_to_linear_data(skb, msg, len);
-	skb_put(skb, len);
-
-	push_udp(np, skb, len);
-	if (np->ipv6)
-		push_ipv6(np, skb, len);
-	else
-		push_ipv4(np, skb, len);
-	push_eth(np, skb);
-	skb->dev = np->dev;
-
-	return (int)netpoll_send_skb(np, skb);
-}
-EXPORT_SYMBOL(netpoll_send_udp);
-
-
 static void skb_pool_flush(struct netpoll *np)
 {
 	struct sk_buff_head *skb_pool;
diff --git a/net/core/page_pool.c b/net/core/page_pool.c
index 6e576dec80db4..8171d1173221b 100644
--- a/net/core/page_pool.c
+++ b/net/core/page_pool.c
@@ -707,18 +707,8 @@ s32 page_pool_inflight(const struct page_pool *pool, bool strict)
 
 void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)
 {
-	struct page *page;
-
 	netmem_set_pp(netmem, pool);
-
-	/* XXX: Now that the offset of page_type is shared between
-	 * struct page and net_iov, just cast the netmem to struct page
-	 * unconditionally by clearing NET_IOV if any, no matter whether
-	 * it comes from struct net_iov or struct page.  This should be
-	 * adjusted once the offset is no longer shared.
-	 */
-	page = (struct page *)((__force unsigned long)netmem & ~NET_IOV);
-	__SetPageNetpp(page);
+	netmem_or_pp_magic(netmem, PP_SIGNATURE);
 
 	/* Ensuring all pages have been split into one fragment initially:
 	 * page_pool_set_pp_info() is only called once for every page when it
@@ -733,17 +723,7 @@ void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)
 
 void page_pool_clear_pp_info(netmem_ref netmem)
 {
-	struct page *page;
-
-	/* XXX: Now that the offset of page_type is shared between
-	 * struct page and net_iov, just cast the netmem to struct page
-	 * unconditionally by clearing NET_IOV if any, no matter whether
-	 * it comes from struct net_iov or struct page.  This should be
-	 * adjusted once the offset is no longer shared.
-	 */
-	page = (struct page *)((__force unsigned long)netmem & ~NET_IOV);
-	__ClearPageNetpp(page);
-
+	netmem_clear_pp_magic(netmem);
 	netmem_set_pp(netmem, NULL);
 }
 
diff --git a/net/core/page_pool_user.c b/net/core/page_pool_user.c
index ee5060d8eec0e..01509d1b3cbaa 100644
--- a/net/core/page_pool_user.c
+++ b/net/core/page_pool_user.c
@@ -79,7 +79,7 @@ struct page_pool_dump_cb {
 
 static int
 netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
-			     pp_nl_fill_cb fill)
+			     pp_nl_fill_cb fill, struct nlattr *ifindex_attr)
 {
 	struct page_pool_dump_cb *state = (void *)cb->ctx;
 	const struct genl_info *info = genl_info_dump(cb);
@@ -88,9 +88,17 @@ netdev_nl_page_pool_get_dump(struct sk_buff *skb, struct netlink_callback *cb,
 	struct page_pool *pool;
 	int err = 0;
 
+	if (ifindex_attr)
+		state->ifindex = nla_get_u32(ifindex_attr);
+
 	rtnl_lock();
 	mutex_lock(&page_pools_lock);
 	for_each_netdev_dump(net, netdev, state->ifindex) {
+		/* Either the provided ifindex doesn't exist or done dumping */
+		if (ifindex_attr &&
+		    netdev->ifindex != nla_get_u32(ifindex_attr))
+			break;
+
 		hlist_for_each_entry(pool, &netdev->page_pools, user.list) {
 			if (state->pp_id && state->pp_id < pool->user.id)
 				continue;
@@ -206,10 +214,40 @@ int netdev_nl_page_pool_stats_get_doit(struct sk_buff *skb,
 	return netdev_nl_page_pool_get_do(info, id, page_pool_nl_stats_fill);
 }
 
+static const struct netlink_range_validation page_pool_ifindex_range = {
+	.min	= 1ULL,
+	.max	= S32_MAX,
+};
+
+static const struct nla_policy
+page_pool_stat_info_policy[NETDEV_A_PAGE_POOL_IFINDEX + 1] = {
+	[NETDEV_A_PAGE_POOL_IFINDEX] =
+		NLA_POLICY_FULL_RANGE(NLA_U32, &page_pool_ifindex_range),
+};
+
 int netdev_nl_page_pool_stats_get_dumpit(struct sk_buff *skb,
 					 struct netlink_callback *cb)
 {
-	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill);
+	struct nlattr *tb[ARRAY_SIZE(page_pool_stat_info_policy)];
+	const struct genl_info *info = genl_info_dump(cb);
+	struct nlattr *ifindex_attr = NULL;
+
+	if (info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO]) {
+		struct nlattr *nest;
+		int err;
+
+		nest = info->attrs[NETDEV_A_PAGE_POOL_STATS_INFO];
+		err = nla_parse_nested(tb, ARRAY_SIZE(tb) - 1, nest,
+				       page_pool_stat_info_policy,
+				       info->extack);
+		if (err)
+			return err;
+
+		ifindex_attr = tb[NETDEV_A_PAGE_POOL_IFINDEX];
+	}
+
+	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_stats_fill,
+					    ifindex_attr);
 }
 
 static int
@@ -305,7 +343,10 @@ int netdev_nl_page_pool_get_doit(struct sk_buff *skb, struct genl_info *info)
 int netdev_nl_page_pool_get_dumpit(struct sk_buff *skb,
 				   struct netlink_callback *cb)
 {
-	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill);
+	const struct genl_info *info = genl_info_dump(cb);
+
+	return netdev_nl_page_pool_get_dump(skb, cb, page_pool_nl_fill,
+					    info->attrs[NETDEV_A_PAGE_POOL_IFINDEX]);
 }
 
 int page_pool_list(struct page_pool *pool)
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index df042da422ef3..6a5e9ace55a08 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -63,7 +63,7 @@
 #include "dev.h"
 
 #define RTNL_MAX_TYPE		50
-#define RTNL_SLAVE_MAX_TYPE	44
+#define RTNL_SLAVE_MAX_TYPE	45
 
 struct rtnl_link {
 	rtnl_doit_func		doit;
@@ -1295,7 +1295,12 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev,
 
 	size = NLMSG_ALIGN(sizeof(struct ifinfomsg))
 	       + nla_total_size(IFNAMSIZ) /* IFLA_IFNAME */
-	       + nla_total_size(IFALIASZ) /* IFLA_IFALIAS */
+	       + rtnl_prop_list_size(dev);
+
+	if (ext_filter_mask & RTEXT_FILTER_NAME_ONLY)
+		return size;
+
+	size += nla_total_size(IFALIASZ) /* IFLA_IFALIAS */
 	       + nla_total_size(IFNAMSIZ) /* IFLA_QDISC */
 	       + nla_total_size_64bit(sizeof(struct rtnl_link_ifmap))
 	       + nla_total_size(MAX_ADDR_LEN) /* IFLA_ADDRESS */
@@ -1342,7 +1347,6 @@ static noinline size_t if_nlmsg_size(const struct net_device *dev,
 	       + nla_total_size(4)  /* IFLA_CARRIER_DOWN_COUNT */
 	       + nla_total_size(4)  /* IFLA_MIN_MTU */
 	       + nla_total_size(4)  /* IFLA_MAX_MTU */
-	       + rtnl_prop_list_size(dev)
 	       + nla_total_size(MAX_ADDR_LEN) /* IFLA_PERM_ADDRESS */
 	       + rtnl_devlink_port_size(dev)
 	       + rtnl_dpll_pin_size(dev)
@@ -1941,15 +1945,18 @@ static int rtnl_fill_alt_ifnames(struct sk_buff *skb,
 	struct netdev_name_node *name_node;
 	int count = 0;
 
+	rcu_read_lock();
 	list_for_each_entry_rcu(name_node, &dev->name_node->list, list) {
-		if (nla_put_string(skb, IFLA_ALT_IFNAME, name_node->name))
+		if (nla_put_string(skb, IFLA_ALT_IFNAME, name_node->name)) {
+			rcu_read_unlock();
 			return -EMSGSIZE;
+		}
 		count++;
 	}
+	rcu_read_unlock();
 	return count;
 }
 
-/* RCU protected. */
 static int rtnl_fill_prop_list(struct sk_buff *skb,
 			       const struct net_device *dev)
 {
@@ -2072,13 +2079,20 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
 	ifm->ifi_flags = netif_get_flags(dev);
 	ifm->ifi_change = change;
 
-	if (tgt_netnsid >= 0 && nla_put_s32(skb, IFLA_TARGET_NETNSID, tgt_netnsid))
-		goto nla_put_failure;
-
 	netdev_copy_name(dev, devname);
 	if (nla_put_string(skb, IFLA_IFNAME, devname))
 		goto nla_put_failure;
 
+	if (rtnl_fill_prop_list(skb, dev))
+		goto nla_put_failure;
+
+	if (ext_filter_mask & RTEXT_FILTER_NAME_ONLY)
+		goto end;
+
+	if (tgt_netnsid >= 0 &&
+	    nla_put_s32(skb, IFLA_TARGET_NETNSID, tgt_netnsid))
+		goto nla_put_failure;
+
 	if (nla_put_u32(skb, IFLA_TXQLEN, READ_ONCE(dev->tx_queue_len)) ||
 	    nla_put_u8(skb, IFLA_OPERSTATE,
 		       netif_running(dev) ? READ_ONCE(dev->operstate) :
@@ -2191,8 +2205,6 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
 		goto nla_put_failure_rcu;
 	if (rtnl_fill_link_ifmap(skb, dev))
 		goto nla_put_failure_rcu;
-	if (rtnl_fill_prop_list(skb, dev))
-		goto nla_put_failure_rcu;
 	rcu_read_unlock();
 
 	if (dev->dev.parent &&
@@ -2211,6 +2223,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
 	if (rtnl_fill_dpll_pin(skb, dev))
 		goto nla_put_failure;
 
+end:
 	nlmsg_end(skb, nlh);
 	return 0;
 
diff --git a/net/core/skbuff.c b/net/core/skbuff.c
index 7dad68e3b5186..89d0654976052 100644
--- a/net/core/skbuff.c
+++ b/net/core/skbuff.c
@@ -78,6 +78,7 @@
 #include <net/mpls.h>
 #include <net/mptcp.h>
 #include <net/mctp.h>
+#include <net/tcp.h>
 #include <net/can.h>
 #include <net/page_pool/helpers.h>
 #include <net/psp/types.h>
@@ -96,7 +97,6 @@
 #include "devmem.h"
 #include "net-sysfs.h"
 #include "netmem_priv.h"
-#include "sock_destructor.h"
 
 #ifdef CONFIG_SKB_EXTENSIONS
 static struct kmem_cache *skbuff_ext_cache __ro_after_init;
@@ -1206,7 +1206,7 @@ void __kfree_skb(struct sk_buff *skb)
 EXPORT_SYMBOL(__kfree_skb);
 
 static __always_inline
-bool __sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb,
+bool __sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb,
 			  enum skb_drop_reason reason)
 {
 	if (unlikely(!skb_unref(skb)))
@@ -1235,7 +1235,8 @@ bool __sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb,
  *	'kfree_skb' tracepoint.
  */
 void __fix_address
-sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason reason)
+sk_skb_reason_drop(const struct sock *sk, struct sk_buff *skb,
+		   enum skb_drop_reason reason)
 {
 	if (__sk_skb_reason_drop(sk, skb, reason))
 		__kfree_skb(skb);
@@ -6200,6 +6201,8 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,
 	       from_shinfo->frags,
 	       from_shinfo->nr_frags * sizeof(skb_frag_t));
 	to_shinfo->nr_frags += from_shinfo->nr_frags;
+	if (from_shinfo->nr_frags)
+		to_shinfo->flags |= from_shinfo->flags & SKBFL_SHARED_FRAG;
 
 	if (!skb_cloned(from))
 		from_shinfo->nr_frags = 0;
diff --git a/net/core/skmsg.c b/net/core/skmsg.c
index 6187a83bd7411..e1850caf1a71a 100644
--- a/net/core/skmsg.c
+++ b/net/core/skmsg.c
@@ -1268,12 +1268,19 @@ static int sk_psock_verdict_recv(struct sock *sk, struct sk_buff *skb)
 static void sk_psock_verdict_data_ready(struct sock *sk)
 {
 	const struct proto_ops *ops = NULL;
+	struct sk_psock *psock;
 	struct socket *sock;
 	int copied;
 
 	trace_sk_data_ready(sk);
 
 	rcu_read_lock();
+	psock = sk_psock(sk);
+	if (psock && tls_sw_has_ctx_rx(sk)) {
+		psock->saved_data_ready(sk);
+		rcu_read_unlock();
+		return;
+	}
 	sock = READ_ONCE(sk->sk_socket);
 	if (likely(sock))
 		ops = READ_ONCE(sock->ops);
@@ -1283,8 +1290,6 @@ static void sk_psock_verdict_data_ready(struct sock *sk)
 
 	copied = ops->read_skb(sk, sk_psock_verdict_recv);
 	if (copied >= 0) {
-		struct sk_psock *psock;
-
 		rcu_read_lock();
 		psock = sk_psock(sk);
 		if (psock)
diff --git a/net/core/sock.c b/net/core/sock.c
index b37b664b6eb92..f362e3ce1efb6 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -2708,6 +2708,7 @@ EXPORT_SYMBOL(sock_wfree);
 /* This variant of sock_wfree() is used by TCP,
  * since it sets SOCK_USE_WRITE_QUEUE.
  */
+#ifdef CONFIG_INET
 void __sock_wfree(struct sk_buff *skb)
 {
 	struct sock *sk = skb->sk;
@@ -2715,6 +2716,8 @@ void __sock_wfree(struct sk_buff *skb)
 	if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
 		__sk_free(sk);
 }
+EXPORT_SYMBOL_GPL(__sock_wfree);
+#endif
 
 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
 {
diff --git a/net/core/sock_destructor.h b/net/core/sock_destructor.h
deleted file mode 100644
index 2f396e6bfba5a..0000000000000
--- a/net/core/sock_destructor.h
+++ /dev/null
@@ -1,12 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef _NET_CORE_SOCK_DESTRUCTOR_H
-#define _NET_CORE_SOCK_DESTRUCTOR_H
-#include <net/tcp.h>
-
-static inline bool is_skb_wmem(const struct sk_buff *skb)
-{
-	return skb->destructor == sock_wfree ||
-	       skb->destructor == __sock_wfree ||
-	       (IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree);
-}
-#endif
diff --git a/net/dsa/user.c b/net/dsa/user.c
index c4bd6fe90b455..8704c1a3a5b74 100644
--- a/net/dsa/user.c
+++ b/net/dsa/user.c
@@ -1499,7 +1499,7 @@ dsa_user_add_cls_matchall_police(struct net_device *dev,
 	policer = &mall_tc_entry->policer;
 	*policer = act->police;
 
-	err = ds->ops->port_policer_add(ds, dp->index, policer);
+	err = ds->ops->port_policer_add(ds, dp->index, policer, extack);
 	if (err) {
 		kfree(mall_tc_entry);
 		return err;
diff --git a/net/ethtool/rings.c b/net/ethtool/rings.c
index 0fd5dcc3729f8..9054c89c5d7bb 100644
--- a/net/ethtool/rings.c
+++ b/net/ethtool/rings.c
@@ -63,9 +63,9 @@ static int rings_reply_size(const struct ethnl_req_info *req_base,
 	       nla_total_size(sizeof(u32)) +	/* _RINGS_TX */
 	       nla_total_size(sizeof(u32)) +	/* _RINGS_RX_BUF_LEN */
 	       nla_total_size(sizeof(u8))  +	/* _RINGS_TCP_DATA_SPLIT */
-	       nla_total_size(sizeof(u32)  +	/* _RINGS_CQE_SIZE */
+	       nla_total_size(sizeof(u32)) +	/* _RINGS_CQE_SIZE */
 	       nla_total_size(sizeof(u8))  +	/* _RINGS_TX_PUSH */
-	       nla_total_size(sizeof(u8))) +	/* _RINGS_RX_PUSH */
+	       nla_total_size(sizeof(u8))  +	/* _RINGS_RX_PUSH */
 	       nla_total_size(sizeof(u32)) +	/* _RINGS_TX_PUSH_BUF_LEN */
 	       nla_total_size(sizeof(u32)) +	/* _RINGS_TX_PUSH_BUF_LEN_MAX */
 	       nla_total_size(sizeof(u32)) +	/* _RINGS_HDS_THRESH */
diff --git a/net/ethtool/tsconfig.c b/net/ethtool/tsconfig.c
index e4f518e49d4cb..239bbf2d826c3 100644
--- a/net/ethtool/tsconfig.c
+++ b/net/ethtool/tsconfig.c
@@ -302,10 +302,6 @@ static int ethnl_set_tsconfig(struct ethnl_req_info *req_base,
 	struct nlattr **tb = info->attrs;
 	int ret;
 
-	BUILD_BUG_ON(__HWTSTAMP_TX_CNT >= 32);
-	BUILD_BUG_ON(__HWTSTAMP_FILTER_CNT >= 32);
-	BUILD_BUG_ON(__HWTSTAMP_FLAG_CNT > 32);
-
 	if (!netif_device_present(dev))
 		return -ENODEV;
 
diff --git a/net/hsr/hsr_framereg.c b/net/hsr/hsr_framereg.c
index 124619920d386..b514e43766eff 100644
--- a/net/hsr/hsr_framereg.c
+++ b/net/hsr/hsr_framereg.c
@@ -163,8 +163,8 @@ void hsr_del_nodes(struct list_head *node_db)
 	struct hsr_node *tmp;
 
 	list_for_each_entry_safe(node, tmp, node_db, mac_list) {
-		list_del(&node->mac_list);
-		hsr_free_node(node);
+		list_del_rcu(&node->mac_list);
+		call_rcu(&node->rcu_head, hsr_free_node_rcu);
 	}
 }
 
diff --git a/net/hsr/hsr_netlink.c b/net/hsr/hsr_netlink.c
index db0b0af7a6920..f0ca23da3ab99 100644
--- a/net/hsr/hsr_netlink.c
+++ b/net/hsr/hsr_netlink.c
@@ -76,9 +76,14 @@ static int hsr_newlink(struct net_device *dev,
 		return -EINVAL;
 	}
 
-	if (data[IFLA_HSR_INTERLINK])
+	if (data[IFLA_HSR_INTERLINK]) {
 		interlink = __dev_get_by_index(link_net,
 					       nla_get_u32(data[IFLA_HSR_INTERLINK]));
+		if (!interlink) {
+			NL_SET_ERR_MSG_MOD(extack, "Interlink does not exist");
+			return -EINVAL;
+		}
+	}
 
 	if (interlink && interlink == link[0]) {
 		NL_SET_ERR_MSG_MOD(extack, "Interlink and Slave1 are the same");
diff --git a/net/ieee802154/core.c b/net/ieee802154/core.c
index 89b671b12600f..c0b8712018a16 100644
--- a/net/ieee802154/core.c
+++ b/net/ieee802154/core.c
@@ -228,36 +228,43 @@ int cfg802154_switch_netns(struct cfg802154_registered_device *rdev,
 			continue;
 		wpan_dev->netdev->netns_immutable = false;
 		err = dev_change_net_namespace(wpan_dev->netdev, net, "wpan%d");
-		if (err)
+		if (err) {
+			WARN_ON(err && err != -ENOMEM);
 			break;
+		}
 		wpan_dev->netdev->netns_immutable = true;
 	}
 
-	if (err) {
-		/* failed -- clean up to old netns */
-		net = wpan_phy_net(&rdev->wpan_phy);
-
-		list_for_each_entry_continue_reverse(wpan_dev,
-						     &rdev->wpan_dev_list,
-						     list) {
-			if (!wpan_dev->netdev)
-				continue;
-			wpan_dev->netdev->netns_immutable = false;
-			err = dev_change_net_namespace(wpan_dev->netdev, net,
-						       "wpan%d");
-			WARN_ON(err);
-			wpan_dev->netdev->netns_immutable = true;
-		}
+	if (err)
+		goto errout;
 
-		return err;
-	}
+	err = device_rename(&rdev->wpan_phy.dev, dev_name(&rdev->wpan_phy.dev));
+	WARN_ON(err && err != -ENOMEM);
 
-	wpan_phy_net_set(&rdev->wpan_phy, net);
+	if (err)
+		goto errout;
 
-	err = device_rename(&rdev->wpan_phy.dev, dev_name(&rdev->wpan_phy.dev));
-	WARN_ON(err);
+	wpan_phy_net_set(&rdev->wpan_phy, net);
 
 	return 0;
+
+errout:
+	/* failed -- clean up to old netns */
+	net = wpan_phy_net(&rdev->wpan_phy);
+
+	list_for_each_entry_continue_reverse(wpan_dev,
+					     &rdev->wpan_dev_list,
+					     list) {
+		if (!wpan_dev->netdev)
+			continue;
+		wpan_dev->netdev->netns_immutable = false;
+		err = dev_change_net_namespace(wpan_dev->netdev, net,
+					       "wpan%d");
+		WARN_ON(err && err != -ENOMEM);
+		wpan_dev->netdev->netns_immutable = true;
+	}
+
+	return err;
 }
 
 void cfg802154_dev_free(struct cfg802154_registered_device *rdev)
@@ -351,7 +358,7 @@ static void __net_exit cfg802154_pernet_exit(struct net *net)
 	rtnl_lock();
 	list_for_each_entry(rdev, &cfg802154_rdev_list, list) {
 		if (net_eq(wpan_phy_net(&rdev->wpan_phy), net))
-			WARN_ON(cfg802154_switch_netns(rdev, &init_net));
+			cfg802154_switch_netns(rdev, &init_net);
 	}
 	rtnl_unlock();
 }
diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig
index 21e5164e30db6..301b47660305b 100644
--- a/net/ipv4/Kconfig
+++ b/net/ipv4/Kconfig
@@ -741,14 +741,12 @@ config DEFAULT_TCP_CONG
 	default "bbr" if DEFAULT_BBR
 	default "cubic"
 
-config TCP_SIGPOOL
-	tristate
-
 config TCP_AO
 	bool "TCP: Authentication Option (RFC5925)"
-	select CRYPTO
+	select CRYPTO_LIB_AES_CBC_MACS
+	select CRYPTO_LIB_SHA1
+	select CRYPTO_LIB_SHA256
 	select CRYPTO_LIB_UTILS
-	select TCP_SIGPOOL
 	depends on 64BIT # seq-number extension needs WRITE_ONCE(u64)
 	help
 	  TCP-AO specifies the use of stronger Message Authentication Codes (MACs),
diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile
index 7f9f988139867..7964234f0d088 100644
--- a/net/ipv4/Makefile
+++ b/net/ipv4/Makefile
@@ -60,7 +60,6 @@ obj-$(CONFIG_TCP_CONG_SCALABLE) += tcp_scalable.o
 obj-$(CONFIG_TCP_CONG_LP) += tcp_lp.o
 obj-$(CONFIG_TCP_CONG_YEAH) += tcp_yeah.o
 obj-$(CONFIG_TCP_CONG_ILLINOIS) += tcp_illinois.o
-obj-$(CONFIG_TCP_SIGPOOL) += tcp_sigpool.o
 obj-$(CONFIG_NET_SOCK_MSG) += tcp_bpf.o
 obj-$(CONFIG_BPF_SYSCALL) += udp_bpf.o
 obj-$(CONFIG_NETLABEL) += cipso_ipv4.o
diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c
index 0e62032e76b19..cbac072633bb5 100644
--- a/net/ipv4/af_inet.c
+++ b/net/ipv4/af_inet.c
@@ -1540,6 +1540,7 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb)
 
 	return pp;
 }
+EXPORT_INDIRECT_CALLABLE(inet_gro_receive);
 
 static struct sk_buff *ipip_gro_receive(struct list_head *head,
 					struct sk_buff *skb)
@@ -1625,6 +1626,7 @@ int inet_gro_complete(struct sk_buff *skb, int nhoff)
 out:
 	return err;
 }
+EXPORT_INDIRECT_CALLABLE(inet_gro_complete);
 
 static int ipip_gro_complete(struct sk_buff *skb, int nhoff)
 {
diff --git a/net/ipv4/ah4.c b/net/ipv4/ah4.c
index 4366cbac3f06c..6fd642d2278d6 100644
--- a/net/ipv4/ah4.c
+++ b/net/ipv4/ah4.c
@@ -143,7 +143,7 @@ static void ah_output_done(void *data, int err)
 	}
 
 	kfree(AH_SKB_CB(skb)->tmp);
-	xfrm_output_resume(skb->sk, skb, err);
+	xfrm_output_resume(skb_to_full_sk(skb), skb, err);
 }
 
 static int ah_output(struct xfrm_state *x, struct sk_buff *skb)
diff --git a/net/ipv4/fou_core.c b/net/ipv4/fou_core.c
index 5bae3cf7fe769..865bd7205122d 100644
--- a/net/ipv4/fou_core.c
+++ b/net/ipv4/fou_core.c
@@ -22,7 +22,7 @@
 #include "fou_nl.h"
 
 struct fou {
-	struct socket *sock;
+	struct sock *sk;
 	u8 protocol;
 	u8 flags;
 	__be16 port;
@@ -508,8 +508,8 @@ static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
 
 static bool fou_cfg_cmp(struct fou *fou, struct fou_cfg *cfg)
 {
-	struct sock *sk = fou->sock->sk;
 	struct udp_port_cfg *udp_cfg = &cfg->udp_config;
+	struct sock *sk = fou->sk;
 
 	if (fou->family != udp_cfg->family ||
 	    fou->port != udp_cfg->local_udp_port ||
@@ -558,11 +558,8 @@ static int fou_add_to_port_list(struct net *net, struct fou *fou,
 
 static void fou_release(struct fou *fou)
 {
-	struct socket *sock = fou->sock;
-
 	list_del(&fou->list);
-	udp_tunnel_sock_release(sock);
-
+	udp_tunnel_sock_release(fou->sk);
 	kfree_rcu(fou, rcu);
 }
 
@@ -593,7 +590,7 @@ static int fou_create(struct net *net, struct fou_cfg *cfg,
 	fou->family = cfg->udp_config.family;
 	fou->flags = cfg->flags;
 	fou->type = cfg->type;
-	fou->sock = sock;
+	fou->sk = sk;
 
 	memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
 	tunnel_cfg.encap_type = 1;
@@ -618,7 +615,7 @@ static int fou_create(struct net *net, struct fou_cfg *cfg,
 		goto error;
 	}
 
-	setup_udp_tunnel_sock(net, sock, &tunnel_cfg);
+	setup_udp_tunnel_sock(net, sk, &tunnel_cfg);
 
 	sk->sk_allocation = GFP_ATOMIC;
 
@@ -634,7 +631,7 @@ static int fou_create(struct net *net, struct fou_cfg *cfg,
 error:
 	kfree(fou);
 	if (sock)
-		udp_tunnel_sock_release(sock);
+		udp_tunnel_sock_release(sock->sk);
 
 	return err;
 }
@@ -779,9 +776,9 @@ int fou_nl_del_doit(struct sk_buff *skb, struct genl_info *info)
 
 static int fou_fill_info(struct fou *fou, struct sk_buff *msg)
 {
-	struct sock *sk = fou->sock->sk;
+	struct sock *sk = fou->sk;
 
-	if (nla_put_u8(msg, FOU_ATTR_AF, fou->sock->sk->sk_family) ||
+	if (nla_put_u8(msg, FOU_ATTR_AF, sk->sk_family) ||
 	    nla_put_be16(msg, FOU_ATTR_PORT, fou->port) ||
 	    nla_put_be16(msg, FOU_ATTR_PEER_PORT, sk->sk_dport) ||
 	    nla_put_u8(msg, FOU_ATTR_IPPROTO, fou->protocol) ||
@@ -793,7 +790,7 @@ static int fou_fill_info(struct fou *fou, struct sk_buff *msg)
 		if (nla_put_flag(msg, FOU_ATTR_REMCSUM_NOPARTIAL))
 			return -1;
 
-	if (fou->sock->sk->sk_family == AF_INET) {
+	if (sk->sk_family == AF_INET) {
 		if (nla_put_in_addr(msg, FOU_ATTR_LOCAL_V4, sk->sk_rcv_saddr))
 			return -1;
 
diff --git a/net/ipv4/icmp.c b/net/ipv4/icmp.c
index 7eeff658b467a..23e921d313b36 100644
--- a/net/ipv4/icmp.c
+++ b/net/ipv4/icmp.c
@@ -961,6 +961,9 @@ void __icmp_send(struct sk_buff *skb_in, int type, int code, __be32 info,
 	if (IS_ERR(rt))
 		goto out_unlock;
 
+	if (rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
+		goto ende;
+
 	/* peer icmp_ratelimit */
 	if (!icmpv4_xrlim_allow(net, rt, &fl4, type, code, apply_ratelimit))
 		goto ende;
diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c
index a9ad39064f3bb..27d1201837793 100644
--- a/net/ipv4/igmp.c
+++ b/net/ipv4/igmp.c
@@ -1002,7 +1002,7 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb,
 		 * different encoding. We use the v3 encoding as more likely
 		 * to be intended in a v3 query.
 		 */
-		max_delay = IGMPV3_MRC(ih3->code)*(HZ/IGMP_TIMER_SCALE);
+		max_delay = igmpv3_mrt(ih3) * (HZ / IGMP_TIMER_SCALE);
 		if (!max_delay)
 			max_delay = 1;	/* can't mod w/ 0 */
 	} else { /* v3 */
@@ -1019,7 +1019,7 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb,
 			ih3 = igmpv3_query_hdr(skb);
 		}
 
-		max_delay = IGMPV3_MRC(ih3->code)*(HZ/IGMP_TIMER_SCALE);
+		max_delay = igmpv3_mrt(ih3) * (HZ / IGMP_TIMER_SCALE);
 		if (!max_delay)
 			max_delay = 1;	/* can't mod w/ 0 */
 		WRITE_ONCE(in_dev->mr_maxdelay, max_delay);
@@ -1030,7 +1030,7 @@ static bool igmp_heard_query(struct in_device *in_dev, struct sk_buff *skb,
 		 */
 		WRITE_ONCE(in_dev->mr_qrv,
 			   ih3->qrv ?: READ_ONCE(net->ipv4.sysctl_igmp_qrv));
-		mr_qi = IGMPV3_QQIC(ih3->qqic)*HZ ?: IGMP_QUERY_INTERVAL;
+		mr_qi = igmpv3_qqi(ih3) * HZ ? : IGMP_QUERY_INTERVAL;
 		WRITE_ONCE(in_dev->mr_qi, mr_qi);
 		/* RFC3376, 8.3. Query Response Interval:
 		 * The number of seconds represented by the [Query Response
diff --git a/net/ipv4/inet_fragment.c b/net/ipv4/inet_fragment.c
index 393770920abd7..86b100694659e 100644
--- a/net/ipv4/inet_fragment.c
+++ b/net/ipv4/inet_fragment.c
@@ -24,8 +24,6 @@
 #include <net/ip.h>
 #include <net/ipv6.h>
 
-#include "../core/sock_destructor.h"
-
 /* Use skb->cb to track consecutive/adjacent fragments coming at
  * the end of the queue. Nodes in the rb-tree queue will
  * contain "runs" of one or more adjacent fragments.
diff --git a/net/ipv4/ipmr_base.c b/net/ipv4/ipmr_base.c
index 3930d612c3dee..867b24beded11 100644
--- a/net/ipv4/ipmr_base.c
+++ b/net/ipv4/ipmr_base.c
@@ -39,7 +39,7 @@ static void __mr_free_table(struct work_struct *work)
 
 void mr_table_free(struct mr_table *mrt)
 {
-	queue_rcu_work(system_unbound_wq, &mrt->work);
+	queue_rcu_work(system_dfl_wq, &mrt->work);
 }
 
 struct mr_table *
diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c
index f92fcc39fc4c8..6205bd57aa852 100644
--- a/net/ipv4/nexthop.c
+++ b/net/ipv4/nexthop.c
@@ -20,8 +20,8 @@
 #define NH_RES_DEFAULT_IDLE_TIMER	(120 * HZ)
 #define NH_RES_DEFAULT_UNBALANCED_TIMER	0	/* No forced rebalancing. */
 
-static void remove_nexthop(struct net *net, struct nexthop *nh,
-			   struct nl_info *nlinfo);
+static bool __must_check remove_nexthop(struct net *net, struct nexthop *nh,
+					struct nl_info *nlinfo);
 
 #define NH_DEV_HASHBITS  8
 #define NH_DEV_HASHSIZE (1U << NH_DEV_HASHBITS)
@@ -2016,9 +2016,9 @@ static void nh_hthr_group_rebalance(struct nh_group *nhg)
 	}
 }
 
-static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
-				struct nl_info *nlinfo,
-				struct list_head *deferred_free)
+static bool __must_check
+remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
+		    struct nl_info *nlinfo, struct list_head *deferred_free)
 {
 	struct nh_grp_entry *nhges, *new_nhges;
 	struct nexthop *nhp = nhge->nh_parent;
@@ -2033,10 +2033,8 @@ static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
 	newg = nhg->spare;
 
 	/* last entry, keep it visible and remove the parent */
-	if (nhg->num_nh == 1) {
-		remove_nexthop(net, nhp, nlinfo);
-		return;
-	}
+	if (nhg->num_nh == 1)
+		return remove_nexthop(net, nhp, nlinfo);
 
 	newg->has_v4 = false;
 	newg->is_multipath = nhg->is_multipath;
@@ -2093,22 +2091,27 @@ static void remove_nh_grp_entry(struct net *net, struct nh_grp_entry *nhge,
 
 	if (nlinfo)
 		nexthop_notify(RTM_NEWNEXTHOP, nhp, nlinfo);
+
+	return false;
 }
 
-static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
-				       struct nl_info *nlinfo)
+static bool __must_check
+remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
+			   struct nl_info *nlinfo)
 {
 	struct nh_grp_entry *nhge, *tmp;
 	LIST_HEAD(deferred_free);
+	bool need_flush = false;
 
 	/* If there is nothing to do, let's avoid the costly call to
 	 * synchronize_net()
 	 */
 	if (list_empty(&nh->grp_list))
-		return;
+		return false;
 
 	list_for_each_entry_safe(nhge, tmp, &nh->grp_list, nh_list)
-		remove_nh_grp_entry(net, nhge, nlinfo, &deferred_free);
+		need_flush |= remove_nh_grp_entry(net, nhge, nlinfo,
+						  &deferred_free);
 
 	/* make sure all see the newly published array before releasing rtnl */
 	synchronize_net();
@@ -2118,6 +2121,8 @@ static void remove_nexthop_from_groups(struct net *net, struct nexthop *nh,
 		list_del(&nhge->nh_list);
 		free_percpu(nhge->stats);
 	}
+
+	return need_flush;
 }
 
 static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
@@ -2142,18 +2147,15 @@ static void remove_nexthop_group(struct nexthop *nh, struct nl_info *nlinfo)
 }
 
 /* not called for nexthop replace */
-static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)
+static bool __must_check __remove_nexthop_fib(struct net *net,
+					      struct nexthop *nh)
 {
+	bool need_flush = !list_empty(&nh->fi_list);
 	struct fib6_info *f6i;
-	bool do_flush = false;
 	struct fib_info *fi;
 
-	list_for_each_entry(fi, &nh->fi_list, nh_list) {
+	list_for_each_entry(fi, &nh->fi_list, nh_list)
 		fi->fib_flags |= RTNH_F_DEAD;
-		do_flush = true;
-	}
-	if (do_flush)
-		fib_flush(net);
 
 	spin_lock_bh(&nh->lock);
 
@@ -2173,12 +2175,14 @@ static void __remove_nexthop_fib(struct net *net, struct nexthop *nh)
 	}
 
 	spin_unlock_bh(&nh->lock);
+
+	return need_flush;
 }
 
-static void __remove_nexthop(struct net *net, struct nexthop *nh,
-			     struct nl_info *nlinfo)
+static bool __must_check __remove_nexthop(struct net *net, struct nexthop *nh,
+					  struct nl_info *nlinfo)
 {
-	__remove_nexthop_fib(net, nh);
+	bool need_flush = __remove_nexthop_fib(net, nh);
 
 	if (nh->is_group) {
 		remove_nexthop_group(nh, nlinfo);
@@ -2189,13 +2193,17 @@ static void __remove_nexthop(struct net *net, struct nexthop *nh,
 		if (nhi->fib_nhc.nhc_dev)
 			hlist_del(&nhi->dev_hash);
 
-		remove_nexthop_from_groups(net, nh, nlinfo);
+		need_flush |= remove_nexthop_from_groups(net, nh, nlinfo);
 	}
+
+	return need_flush;
 }
 
-static void remove_nexthop(struct net *net, struct nexthop *nh,
-			   struct nl_info *nlinfo)
+static bool __must_check remove_nexthop(struct net *net, struct nexthop *nh,
+					struct nl_info *nlinfo)
 {
+	bool need_flush;
+
 	call_nexthop_notifiers(net, NEXTHOP_EVENT_DEL, nh, NULL);
 
 	/* remove from the tree */
@@ -2204,10 +2212,19 @@ static void remove_nexthop(struct net *net, struct nexthop *nh,
 	if (nlinfo)
 		nexthop_notify(RTM_DELNEXTHOP, nh, nlinfo);
 
-	__remove_nexthop(net, nh, nlinfo);
+	need_flush = __remove_nexthop(net, nh, nlinfo);
 	nh_base_seq_inc(net);
 
 	nexthop_put(nh);
+
+	return need_flush;
+}
+
+static void remove_one_nexthop(struct net *net, struct nexthop *nh,
+			       struct nl_info *nlinfo)
+{
+	if (remove_nexthop(net, nh, nlinfo))
+		fib_flush(net);
 }
 
 /* if any FIB entries reference this nexthop, any dst entries
@@ -2592,7 +2609,7 @@ static int replace_nexthop(struct net *net, struct nexthop *old,
 	if (!err) {
 		nh_rt_cache_flush(net, old, new);
 
-		__remove_nexthop(net, new, NULL);
+		WARN_ON_ONCE(__remove_nexthop(net, new, NULL));
 		nexthop_put(new);
 	}
 
@@ -2699,6 +2716,7 @@ static void nexthop_flush_dev(struct net_device *dev, unsigned long event)
 	unsigned int hash = nh_dev_hashfn(dev->ifindex);
 	struct net *net = dev_net(dev);
 	struct hlist_head *head = &net->nexthop.devhash[hash];
+	bool need_flush = false;
 	struct hlist_node *n;
 	struct nh_info *nhi;
 
@@ -2710,22 +2728,28 @@ static void nexthop_flush_dev(struct net_device *dev, unsigned long event)
 		    (event == NETDEV_DOWN || event == NETDEV_CHANGE))
 			continue;
 
-		remove_nexthop(net, nhi->nh_parent, NULL);
+		need_flush |= remove_nexthop(net, nhi->nh_parent, NULL);
 	}
+
+	if (need_flush)
+		fib_flush(net);
 }
 
 /* rtnl; called when net namespace is deleted */
 static void flush_all_nexthops(struct net *net)
 {
 	struct rb_root *root = &net->nexthop.rb_root;
+	bool need_flush = false;
 	struct rb_node *node;
 	struct nexthop *nh;
 
 	while ((node = rb_first(root))) {
 		nh = rb_entry(node, struct nexthop, rb_node);
-		remove_nexthop(net, nh, NULL);
+		need_flush |= remove_nexthop(net, nh, NULL);
 		cond_resched();
 	}
+	if (need_flush)
+		fib_flush(net);
 }
 
 static struct nexthop *nexthop_create_group(struct net *net,
@@ -2994,7 +3018,7 @@ static struct nexthop *nexthop_add(struct net *net, struct nh_config *cfg,
 
 	err = insert_nexthop(net, nh, cfg, extack);
 	if (err) {
-		__remove_nexthop(net, nh, NULL);
+		WARN_ON_ONCE(__remove_nexthop(net, nh, NULL));
 		nexthop_put(nh);
 		nh = ERR_PTR(err);
 	}
@@ -3363,7 +3387,7 @@ static int rtm_del_nexthop(struct sk_buff *skb, struct nlmsghdr *nlh,
 
 	nh = nexthop_find_by_id(net, id);
 	if (nh)
-		remove_nexthop(net, nh, &nlinfo);
+		remove_one_nexthop(net, nh, &nlinfo);
 	else
 		err = -ENOENT;
 
diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c
index 5aaf9c62c8e1f..68e88cb3e55cb 100644
--- a/net/ipv4/raw.c
+++ b/net/ipv4/raw.c
@@ -391,7 +391,7 @@ static int raw_send_hdrinc(struct sock *sk, struct flowi4 *fl4,
 	 * in, reject the frame as invalid
 	 */
 	err = -EINVAL;
-	if (iphlen > length)
+	if (iphlen > length || iphlen < sizeof(*iph))
 		goto error_free;
 
 	if (iphlen >= sizeof(*iph)) {
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index bc1296f0ea69b..3d62d45d84bda 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -1272,7 +1272,7 @@ static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
 		 __func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
 		 skb->dev ? skb->dev->name : "?");
 	kfree_skb(skb);
-	WARN_ON(1);
+	WARN_ON_ONCE(1);
 	return 0;
 }
 
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index 432fa28e47d4c..890182a151e16 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -299,9 +299,6 @@ enum {
 DEFINE_PER_CPU(unsigned int, tcp_orphan_count);
 EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count);
 
-DEFINE_PER_CPU(u32, tcp_tw_isn);
-EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn);
-
 long sysctl_tcp_mem[3] __read_mostly;
 
 DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
@@ -5230,9 +5227,13 @@ static void __init tcp_struct_check(void)
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh);
 
 	/* TX read-write hotpath cache lines */
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, segs_out);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered_ce);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_acked);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, first_tx_mstamp);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, delivered_mstamp);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat);
@@ -5243,6 +5244,10 @@ static void __init tcp_struct_check(void)
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, accecn_opt_tstamp);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, max_packets_out);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, cwnd_usage_seq);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rate_delivered);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rate_interval_us);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags);
@@ -5258,8 +5263,6 @@ static void __init tcp_struct_check(void)
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered_ce);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ce);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, received_ecn_bytes);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited);
@@ -5267,22 +5270,16 @@ static void __init tcp_struct_check(void)
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_mwnd_seq);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_tstamp);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, segs_in);
+	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, segs_out);
 
 	/* RX read-write hotpath cache lines */
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, segs_in);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, max_packets_out);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, cwnd_usage_seq);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_delivered);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_interval_us);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_ecn_bytes);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, pkts_acked_ewma);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, first_tx_mstamp);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_mstamp);
-	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_acked);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est);
 	CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space);
 }
diff --git a/net/ipv4/tcp_ao.c b/net/ipv4/tcp_ao.c
index 0a4b38b315fed..2f69bcecae78a 100644
--- a/net/ipv4/tcp_ao.c
+++ b/net/ipv4/tcp_ao.c
@@ -9,7 +9,9 @@
  */
 #define pr_fmt(fmt) "TCP: " fmt
 
-#include <crypto/hash.h>
+#include <crypto/aes-cbc-macs.h>
+#include <crypto/sha1.h>
+#include <crypto/sha2.h>
 #include <crypto/utils.h>
 #include <linux/inetdevice.h>
 #include <linux/tcp.h>
@@ -21,32 +23,133 @@
 
 DEFINE_STATIC_KEY_DEFERRED_FALSE(tcp_ao_needed, HZ);
 
-int tcp_ao_calc_traffic_key(struct tcp_ao_key *mkt, u8 *key, void *ctx,
-			    unsigned int len, struct tcp_sigpool *hp)
-{
-	struct scatterlist sg;
-	int ret;
+static const struct tcp_ao_algo {
+	const char *name;
+	unsigned int digest_size;
+} tcp_ao_algos[] = {
+	[TCP_AO_ALGO_HMAC_SHA1] = {
+		.name = "hmac(sha1)",
+		.digest_size = SHA1_DIGEST_SIZE,
+	},
+	[TCP_AO_ALGO_HMAC_SHA256] = {
+		.name = "hmac(sha256)",
+		.digest_size = SHA256_DIGEST_SIZE,
+	},
+	[TCP_AO_ALGO_AES_128_CMAC] = {
+		.name = "cmac(aes128)",
+		.digest_size = AES_BLOCK_SIZE, /* same as AES_KEYSIZE_128 */
+	},
+};
+
+struct tcp_ao_mac_ctx {
+	enum tcp_ao_algo_id algo;
+	union {
+		struct hmac_sha1_ctx hmac_sha1;
+		struct hmac_sha256_ctx hmac_sha256;
+		struct {
+			struct aes_cmac_key key;
+			struct aes_cmac_ctx ctx;
+		} aes_cmac;
+	};
+};
+
+static const struct tcp_ao_algo *tcp_ao_find_algo(const char *name)
+{
+	for (size_t i = 0; i < ARRAY_SIZE(tcp_ao_algos); i++) {
+		const struct tcp_ao_algo *algo = &tcp_ao_algos[i];
+
+		if (!algo->name)
+			continue;
+		if (WARN_ON_ONCE(algo->digest_size > TCP_AO_MAX_MAC_LEN ||
+				 algo->digest_size >
+					 TCP_AO_MAX_TRAFFIC_KEY_LEN))
+			continue;
+		if (strcmp(name, algo->name) == 0)
+			return algo;
+	}
+	return NULL;
+}
 
-	if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp->req),
-				mkt->key, mkt->keylen))
-		goto clear_hash;
+static void tcp_ao_mac_init(struct tcp_ao_mac_ctx *mac_ctx,
+			    enum tcp_ao_algo_id algo, const u8 *traffic_key)
+{
+	mac_ctx->algo = algo;
+	switch (mac_ctx->algo) {
+	case TCP_AO_ALGO_HMAC_SHA1:
+		hmac_sha1_init_usingrawkey(&mac_ctx->hmac_sha1, traffic_key,
+					   SHA1_DIGEST_SIZE);
+		return;
+	case TCP_AO_ALGO_HMAC_SHA256:
+		hmac_sha256_init_usingrawkey(&mac_ctx->hmac_sha256, traffic_key,
+					     SHA256_DIGEST_SIZE);
+		return;
+	case TCP_AO_ALGO_AES_128_CMAC:
+		aes_cmac_preparekey(&mac_ctx->aes_cmac.key, traffic_key,
+				    AES_KEYSIZE_128);
+		aes_cmac_init(&mac_ctx->aes_cmac.ctx, &mac_ctx->aes_cmac.key);
+		return;
+	default:
+		WARN_ON_ONCE(1); /* algo was validated earlier. */
+	}
+}
 
-	ret = crypto_ahash_init(hp->req);
-	if (ret)
-		goto clear_hash;
+void tcp_ao_mac_update(struct tcp_ao_mac_ctx *mac_ctx, const void *data,
+		       size_t data_len)
+{
+	switch (mac_ctx->algo) {
+	case TCP_AO_ALGO_HMAC_SHA1:
+		hmac_sha1_update(&mac_ctx->hmac_sha1, data, data_len);
+		return;
+	case TCP_AO_ALGO_HMAC_SHA256:
+		hmac_sha256_update(&mac_ctx->hmac_sha256, data, data_len);
+		return;
+	case TCP_AO_ALGO_AES_128_CMAC:
+		aes_cmac_update(&mac_ctx->aes_cmac.ctx, data, data_len);
+		return;
+	default:
+		WARN_ON_ONCE(1); /* algo was validated earlier. */
+	}
+}
 
-	sg_init_one(&sg, ctx, len);
-	ahash_request_set_crypt(hp->req, &sg, key, len);
-	crypto_ahash_update(hp->req);
+static void tcp_ao_mac_final(struct tcp_ao_mac_ctx *mac_ctx, u8 *out)
+{
+	switch (mac_ctx->algo) {
+	case TCP_AO_ALGO_HMAC_SHA1:
+		hmac_sha1_final(&mac_ctx->hmac_sha1, out);
+		return;
+	case TCP_AO_ALGO_HMAC_SHA256:
+		hmac_sha256_final(&mac_ctx->hmac_sha256, out);
+		return;
+	case TCP_AO_ALGO_AES_128_CMAC:
+		aes_cmac_final(&mac_ctx->aes_cmac.ctx, out);
+		return;
+	default:
+		WARN_ON_ONCE(1); /* algo was validated earlier. */
+	}
+}
 
-	ret = crypto_ahash_final(hp->req);
-	if (ret)
-		goto clear_hash;
+void tcp_ao_calc_traffic_key(const struct tcp_ao_key *mkt, u8 *traffic_key,
+			     const void *input, unsigned int input_len)
+{
+	switch (mkt->algo) {
+	case TCP_AO_ALGO_HMAC_SHA1:
+		hmac_sha1_usingrawkey(mkt->key, mkt->keylen, input, input_len,
+				      traffic_key);
+		return;
+	case TCP_AO_ALGO_HMAC_SHA256:
+		hmac_sha256_usingrawkey(mkt->key, mkt->keylen, input, input_len,
+					traffic_key);
+		return;
+	case TCP_AO_ALGO_AES_128_CMAC: {
+		struct aes_cmac_key k;
 
-	return 0;
-clear_hash:
-	memset(key, 0, tcp_ao_digest_size(mkt));
-	return 1;
+		aes_cmac_preparekey(&k, mkt->key, AES_KEYSIZE_128);
+		aes_cmac(&k, input, input_len, traffic_key);
+		return;
+	}
+	default:
+		WARN_ON_ONCE(1); /* algo was validated earlier. */
+	}
 }
 
 bool tcp_ao_ignore_icmp(const struct sock *sk, int family, int type, int code)
@@ -255,7 +358,6 @@ static struct tcp_ao_key *tcp_ao_copy_key(struct sock *sk,
 
 	*new_key = *key;
 	INIT_HLIST_NODE(&new_key->node);
-	tcp_sigpool_get(new_key->tcp_sigpool_id);
 	atomic64_set(&new_key->pkt_good, 0);
 	atomic64_set(&new_key->pkt_bad, 0);
 
@@ -266,7 +368,6 @@ static void tcp_ao_key_free_rcu(struct rcu_head *head)
 {
 	struct tcp_ao_key *key = container_of(head, struct tcp_ao_key, rcu);
 
-	tcp_sigpool_release(key->tcp_sigpool_id);
 	kfree_sensitive(key);
 }
 
@@ -277,7 +378,6 @@ static void tcp_ao_info_free(struct tcp_ao_info *ao)
 
 	hlist_for_each_entry_safe(key, n, &ao->head, node) {
 		hlist_del(&key->node);
-		tcp_sigpool_release(key->tcp_sigpool_id);
 		kfree_sensitive(key);
 	}
 	kfree(ao);
@@ -336,10 +436,10 @@ void tcp_ao_time_wait(struct tcp_timewait_sock *tcptw, struct tcp_sock *tp)
 }
 
 /* 4 tuple and ISNs are expected in NBO */
-static int tcp_v4_ao_calc_key(struct tcp_ao_key *mkt, u8 *key,
-			      __be32 saddr, __be32 daddr,
-			      __be16 sport, __be16 dport,
-			      __be32 sisn,  __be32 disn)
+static void tcp_v4_ao_calc_key(struct tcp_ao_key *mkt, u8 *key,
+			       __be32 saddr, __be32 daddr,
+			       __be16 sport, __be16 dport,
+			       __be32 sisn, __be32 disn)
 {
 	/* See RFC5926 3.1.1 */
 	struct kdf_input_block {
@@ -347,148 +447,146 @@ static int tcp_v4_ao_calc_key(struct tcp_ao_key *mkt, u8 *key,
 		u8                      label[6];
 		struct tcp4_ao_context	ctx;
 		__be16                  outlen;
-	} __packed * tmp;
-	struct tcp_sigpool hp;
-	int err;
-
-	err = tcp_sigpool_start(mkt->tcp_sigpool_id, &hp);
-	if (err)
-		return err;
-
-	tmp = hp.scratch;
-	tmp->counter	= 1;
-	memcpy(tmp->label, "TCP-AO", 6);
-	tmp->ctx.saddr	= saddr;
-	tmp->ctx.daddr	= daddr;
-	tmp->ctx.sport	= sport;
-	tmp->ctx.dport	= dport;
-	tmp->ctx.sisn	= sisn;
-	tmp->ctx.disn	= disn;
-	tmp->outlen	= htons(tcp_ao_digest_size(mkt) * 8); /* in bits */
-
-	err = tcp_ao_calc_traffic_key(mkt, key, tmp, sizeof(*tmp), &hp);
-	tcp_sigpool_end(&hp);
-
-	return err;
-}
-
-int tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
-			  const struct sock *sk,
-			  __be32 sisn, __be32 disn, bool send)
+	} __packed input = {
+		.counter = 1,
+		.label = "TCP-AO",
+		.ctx = {
+			.saddr = saddr,
+			.daddr = daddr,
+			.sport = sport,
+			.dport = dport,
+			.sisn = sisn,
+			.disn = disn,
+		},
+		.outlen = htons(tcp_ao_digest_size(mkt) * 8), /* in bits */
+	};
+
+	tcp_ao_calc_traffic_key(mkt, key, &input, sizeof(input));
+}
+
+void tcp_v4_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
+			   const struct sock *sk,
+			   __be32 sisn, __be32 disn, bool send)
 {
 	if (send)
-		return tcp_v4_ao_calc_key(mkt, key, sk->sk_rcv_saddr,
-					  sk->sk_daddr, htons(sk->sk_num),
-					  sk->sk_dport, sisn, disn);
+		tcp_v4_ao_calc_key(mkt, key, sk->sk_rcv_saddr, sk->sk_daddr,
+				   htons(sk->sk_num), sk->sk_dport, sisn, disn);
 	else
-		return tcp_v4_ao_calc_key(mkt, key, sk->sk_daddr,
-					  sk->sk_rcv_saddr, sk->sk_dport,
-					  htons(sk->sk_num), disn, sisn);
+		tcp_v4_ao_calc_key(mkt, key, sk->sk_daddr, sk->sk_rcv_saddr,
+				   sk->sk_dport, htons(sk->sk_num), disn, sisn);
 }
 
 static int tcp_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
 			      const struct sock *sk,
 			      __be32 sisn, __be32 disn, bool send)
 {
-	if (mkt->family == AF_INET)
-		return tcp_v4_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
+	if (mkt->family == AF_INET) {
+		tcp_v4_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
+		return 0;
+	}
 #if IS_ENABLED(CONFIG_IPV6)
-	else if (mkt->family == AF_INET6)
-		return tcp_v6_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
+	if (mkt->family == AF_INET6) {
+		tcp_v6_ao_calc_key_sk(mkt, key, sk, sisn, disn, send);
+		return 0;
+	}
 #endif
-	else
-		return -EOPNOTSUPP;
+	return -EOPNOTSUPP;
 }
 
-int tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
-			   struct request_sock *req)
+void tcp_v4_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
+			    struct request_sock *req)
 {
 	struct inet_request_sock *ireq = inet_rsk(req);
 
-	return tcp_v4_ao_calc_key(mkt, key,
-				  ireq->ir_loc_addr, ireq->ir_rmt_addr,
-				  htons(ireq->ir_num), ireq->ir_rmt_port,
-				  htonl(tcp_rsk(req)->snt_isn),
-				  htonl(tcp_rsk(req)->rcv_isn));
+	tcp_v4_ao_calc_key(mkt, key, ireq->ir_loc_addr, ireq->ir_rmt_addr,
+			   htons(ireq->ir_num), ireq->ir_rmt_port,
+			   htonl(tcp_rsk(req)->snt_isn),
+			   htonl(tcp_rsk(req)->rcv_isn));
 }
 
-static int tcp_v4_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
-				  const struct sk_buff *skb,
-				  __be32 sisn, __be32 disn)
+static void tcp_v4_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
+				   const struct sk_buff *skb,
+				   __be32 sisn, __be32 disn)
 {
 	const struct iphdr *iph = ip_hdr(skb);
 	const struct tcphdr *th = tcp_hdr(skb);
 
-	return tcp_v4_ao_calc_key(mkt, key, iph->saddr, iph->daddr,
-				  th->source, th->dest, sisn, disn);
+	tcp_v4_ao_calc_key(mkt, key, iph->saddr, iph->daddr, th->source,
+			   th->dest, sisn, disn);
 }
 
 static int tcp_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
 			       const struct sk_buff *skb,
 			       __be32 sisn, __be32 disn, int family)
 {
-	if (family == AF_INET)
-		return tcp_v4_ao_calc_key_skb(mkt, key, skb, sisn, disn);
+	if (family == AF_INET) {
+		tcp_v4_ao_calc_key_skb(mkt, key, skb, sisn, disn);
+		return 0;
+	}
 #if IS_ENABLED(CONFIG_IPV6)
-	else if (family == AF_INET6)
-		return tcp_v6_ao_calc_key_skb(mkt, key, skb, sisn, disn);
+	if (family == AF_INET6) {
+		tcp_v6_ao_calc_key_skb(mkt, key, skb, sisn, disn);
+		return 0;
+	}
 #endif
 	return -EAFNOSUPPORT;
 }
 
-static int tcp_v4_ao_hash_pseudoheader(struct tcp_sigpool *hp,
-				       __be32 daddr, __be32 saddr,
-				       int nbytes)
+static void tcp_v4_ao_hash_pseudoheader(struct tcp_ao_mac_ctx *mac_ctx,
+					__be32 daddr, __be32 saddr, int nbytes)
 {
-	struct tcp4_pseudohdr *bp;
-	struct scatterlist sg;
+	struct tcp4_pseudohdr phdr = {
+		.saddr = saddr,
+		.daddr = daddr,
+		.pad = 0,
+		.protocol = IPPROTO_TCP,
+		.len = cpu_to_be16(nbytes),
+	};
 
-	bp = hp->scratch;
-	bp->saddr = saddr;
-	bp->daddr = daddr;
-	bp->pad = 0;
-	bp->protocol = IPPROTO_TCP;
-	bp->len = cpu_to_be16(nbytes);
-
-	sg_init_one(&sg, bp, sizeof(*bp));
-	ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp));
-	return crypto_ahash_update(hp->req);
+	tcp_ao_mac_update(mac_ctx, &phdr, sizeof(phdr));
 }
 
 static int tcp_ao_hash_pseudoheader(unsigned short int family,
 				    const struct sock *sk,
 				    const struct sk_buff *skb,
-				    struct tcp_sigpool *hp, int nbytes)
+				    struct tcp_ao_mac_ctx *mac_ctx, int nbytes)
 {
 	const struct tcphdr *th = tcp_hdr(skb);
 
 	/* TODO: Can we rely on checksum being zero to mean outbound pkt? */
 	if (!th->check) {
-		if (family == AF_INET)
-			return tcp_v4_ao_hash_pseudoheader(hp, sk->sk_daddr,
-					sk->sk_rcv_saddr, skb->len);
+		if (family == AF_INET) {
+			tcp_v4_ao_hash_pseudoheader(mac_ctx, sk->sk_daddr,
+						    sk->sk_rcv_saddr, skb->len);
+			return 0;
+		}
 #if IS_ENABLED(CONFIG_IPV6)
-		else if (family == AF_INET6)
-			return tcp_v6_ao_hash_pseudoheader(hp, &sk->sk_v6_daddr,
-					&sk->sk_v6_rcv_saddr, skb->len);
+		if (family == AF_INET6) {
+			tcp_v6_ao_hash_pseudoheader(mac_ctx, &sk->sk_v6_daddr,
+						    &sk->sk_v6_rcv_saddr,
+						    skb->len);
+			return 0;
+		}
 #endif
-		else
-			return -EAFNOSUPPORT;
+		return -EAFNOSUPPORT;
 	}
 
 	if (family == AF_INET) {
 		const struct iphdr *iph = ip_hdr(skb);
 
-		return tcp_v4_ao_hash_pseudoheader(hp, iph->daddr,
-				iph->saddr, skb->len);
+		tcp_v4_ao_hash_pseudoheader(mac_ctx, iph->daddr, iph->saddr,
+					    skb->len);
+		return 0;
+	}
 #if IS_ENABLED(CONFIG_IPV6)
-	} else if (family == AF_INET6) {
+	if (family == AF_INET6) {
 		const struct ipv6hdr *iph = ipv6_hdr(skb);
 
-		return tcp_v6_ao_hash_pseudoheader(hp, &iph->daddr,
-				&iph->saddr, skb->len);
-#endif
+		tcp_v6_ao_hash_pseudoheader(mac_ctx, &iph->daddr, &iph->saddr,
+					    skb->len);
+		return 0;
 	}
+#endif
 	return -EAFNOSUPPORT;
 }
 
@@ -507,31 +605,20 @@ u32 tcp_ao_compute_sne(u32 next_sne, u32 next_seq, u32 seq)
 	return sne;
 }
 
-/* tcp_ao_hash_sne(struct tcp_sigpool *hp)
- * @hp	- used for hashing
- * @sne - sne value
- */
-static int tcp_ao_hash_sne(struct tcp_sigpool *hp, u32 sne)
+static void tcp_ao_hash_sne(struct tcp_ao_mac_ctx *mac_ctx, u32 sne)
 {
-	struct scatterlist sg;
-	__be32 *bp;
+	__be32 sne_be32 = htonl(sne);
 
-	bp = (__be32 *)hp->scratch;
-	*bp = htonl(sne);
-
-	sg_init_one(&sg, bp, sizeof(*bp));
-	ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp));
-	return crypto_ahash_update(hp->req);
+	tcp_ao_mac_update(mac_ctx, &sne_be32, sizeof(sne_be32));
 }
 
-static int tcp_ao_hash_header(struct tcp_sigpool *hp,
-			      const struct tcphdr *th,
-			      bool exclude_options, u8 *hash,
-			      int hash_offset, int hash_len)
+static void tcp_ao_hash_header(struct tcp_ao_mac_ctx *mac_ctx,
+			       const struct tcphdr *th, bool exclude_options,
+			       u8 *hash, int hash_offset, int hash_len)
 {
-	struct scatterlist sg;
-	u8 *hdr = hp->scratch;
-	int err, len;
+	/* Full TCP header (th->doff << 2) should fit into scratch area. */
+	u8 hdr[60];
+	int len;
 
 	/* We are not allowed to change tcphdr, make a local copy */
 	if (exclude_options) {
@@ -551,11 +638,7 @@ static int tcp_ao_hash_header(struct tcp_sigpool *hp,
 		memset(hdr + hash_offset, 0, hash_len);
 	}
 
-	sg_init_one(&sg, hdr, len);
-	ahash_request_set_crypt(hp->req, &sg, NULL, len);
-	err = crypto_ahash_update(hp->req);
-	WARN_ON_ONCE(err != 0);
-	return err;
+	tcp_ao_mac_update(mac_ctx, hdr, len);
 }
 
 int tcp_ao_hash_hdr(unsigned short int family, char *ao_hash,
@@ -564,109 +647,92 @@ int tcp_ao_hash_hdr(unsigned short int family, char *ao_hash,
 		    const union tcp_ao_addr *saddr,
 		    const struct tcphdr *th, u32 sne)
 {
-	int tkey_len = tcp_ao_digest_size(key);
 	int hash_offset = ao_hash - (char *)th;
-	struct tcp_sigpool hp;
-	void *hash_buf = NULL;
-
-	hash_buf = kmalloc(tkey_len, GFP_ATOMIC);
-	if (!hash_buf)
-		goto clear_hash_noput;
-
-	if (tcp_sigpool_start(key->tcp_sigpool_id, &hp))
-		goto clear_hash_noput;
-
-	if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp.req), tkey, tkey_len))
-		goto clear_hash;
-
-	if (crypto_ahash_init(hp.req))
-		goto clear_hash;
+	struct tcp_ao_mac_ctx mac_ctx;
+	u8 hash_buf[TCP_AO_MAX_MAC_LEN];
 
-	if (tcp_ao_hash_sne(&hp, sne))
-		goto clear_hash;
+	tcp_ao_mac_init(&mac_ctx, key->algo, tkey);
+	tcp_ao_hash_sne(&mac_ctx, sne);
 	if (family == AF_INET) {
-		if (tcp_v4_ao_hash_pseudoheader(&hp, daddr->a4.s_addr,
-						saddr->a4.s_addr, th->doff * 4))
-			goto clear_hash;
+		tcp_v4_ao_hash_pseudoheader(&mac_ctx, daddr->a4.s_addr,
+					    saddr->a4.s_addr, th->doff * 4);
 #if IS_ENABLED(CONFIG_IPV6)
 	} else if (family == AF_INET6) {
-		if (tcp_v6_ao_hash_pseudoheader(&hp, &daddr->a6,
-						&saddr->a6, th->doff * 4))
-			goto clear_hash;
+		tcp_v6_ao_hash_pseudoheader(&mac_ctx, &daddr->a6,
+					    &saddr->a6, th->doff * 4);
 #endif
 	} else {
 		WARN_ON_ONCE(1);
 		goto clear_hash;
 	}
-	if (tcp_ao_hash_header(&hp, th,
-			       !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
-			       ao_hash, hash_offset, tcp_ao_maclen(key)))
-		goto clear_hash;
-	ahash_request_set_crypt(hp.req, NULL, hash_buf, 0);
-	if (crypto_ahash_final(hp.req))
-		goto clear_hash;
+	tcp_ao_hash_header(&mac_ctx, th,
+			   !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
+			   ao_hash, hash_offset, tcp_ao_maclen(key));
+	tcp_ao_mac_final(&mac_ctx, hash_buf);
 
 	memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
-	tcp_sigpool_end(&hp);
-	kfree(hash_buf);
 	return 0;
 
 clear_hash:
-	tcp_sigpool_end(&hp);
-clear_hash_noput:
 	memset(ao_hash, 0, tcp_ao_maclen(key));
-	kfree(hash_buf);
 	return 1;
 }
 
+static void tcp_ao_hash_skb_data(struct tcp_ao_mac_ctx *mac_ctx,
+				 const struct sk_buff *skb,
+				 unsigned int header_len)
+{
+	const unsigned int head_data_len = skb_headlen(skb) > header_len ?
+					   skb_headlen(skb) - header_len : 0;
+	const struct skb_shared_info *shi = skb_shinfo(skb);
+	struct sk_buff *frag_iter;
+	unsigned int i;
+
+	tcp_ao_mac_update(mac_ctx, (const u8 *)tcp_hdr(skb) + header_len,
+			  head_data_len);
+
+	for (i = 0; i < shi->nr_frags; ++i) {
+		const skb_frag_t *f = &shi->frags[i];
+		u32 p_off, p_len, copied;
+		const void *vaddr;
+		struct page *p;
+
+		skb_frag_foreach_page(f, skb_frag_off(f), skb_frag_size(f),
+				      p, p_off, p_len, copied) {
+			vaddr = kmap_local_page(p);
+			tcp_ao_mac_update(mac_ctx, vaddr + p_off, p_len);
+			kunmap_local(vaddr);
+		}
+	}
+
+	skb_walk_frags(skb, frag_iter)
+		tcp_ao_hash_skb_data(mac_ctx, frag_iter, 0);
+}
+
 int tcp_ao_hash_skb(unsigned short int family,
 		    char *ao_hash, struct tcp_ao_key *key,
 		    const struct sock *sk, const struct sk_buff *skb,
 		    const u8 *tkey, int hash_offset, u32 sne)
 {
 	const struct tcphdr *th = tcp_hdr(skb);
-	int tkey_len = tcp_ao_digest_size(key);
-	struct tcp_sigpool hp;
-	void *hash_buf = NULL;
+	struct tcp_ao_mac_ctx mac_ctx;
+	u8 hash_buf[TCP_AO_MAX_MAC_LEN];
 
-	hash_buf = kmalloc(tkey_len, GFP_ATOMIC);
-	if (!hash_buf)
-		goto clear_hash_noput;
-
-	if (tcp_sigpool_start(key->tcp_sigpool_id, &hp))
-		goto clear_hash_noput;
-
-	if (crypto_ahash_setkey(crypto_ahash_reqtfm(hp.req), tkey, tkey_len))
-		goto clear_hash;
-
-	/* For now use sha1 by default. Depends on alg in tcp_ao_key */
-	if (crypto_ahash_init(hp.req))
-		goto clear_hash;
-
-	if (tcp_ao_hash_sne(&hp, sne))
-		goto clear_hash;
-	if (tcp_ao_hash_pseudoheader(family, sk, skb, &hp, skb->len))
-		goto clear_hash;
-	if (tcp_ao_hash_header(&hp, th,
-			       !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
-			       ao_hash, hash_offset, tcp_ao_maclen(key)))
-		goto clear_hash;
-	if (tcp_sigpool_hash_skb_data(&hp, skb, th->doff << 2))
-		goto clear_hash;
-	ahash_request_set_crypt(hp.req, NULL, hash_buf, 0);
-	if (crypto_ahash_final(hp.req))
+	tcp_ao_mac_init(&mac_ctx, key->algo, tkey);
+	tcp_ao_hash_sne(&mac_ctx, sne);
+	if (tcp_ao_hash_pseudoheader(family, sk, skb, &mac_ctx, skb->len))
 		goto clear_hash;
+	tcp_ao_hash_header(&mac_ctx, th,
+			   !!(key->keyflags & TCP_AO_KEYF_EXCLUDE_OPT),
+			   ao_hash, hash_offset, tcp_ao_maclen(key));
+	tcp_ao_hash_skb_data(&mac_ctx, skb, th->doff << 2);
+	tcp_ao_mac_final(&mac_ctx, hash_buf);
 
 	memcpy(ao_hash, hash_buf, tcp_ao_maclen(key));
-	tcp_sigpool_end(&hp);
-	kfree(hash_buf);
 	return 0;
 
 clear_hash:
-	tcp_sigpool_end(&hp);
-clear_hash_noput:
 	memset(ao_hash, 0, tcp_ao_maclen(key));
-	kfree(hash_buf);
 	return 1;
 }
 
@@ -682,22 +748,12 @@ int tcp_v4_ao_synack_hash(char *ao_hash, struct tcp_ao_key *ao_key,
 			  struct request_sock *req, const struct sk_buff *skb,
 			  int hash_offset, u32 sne)
 {
-	void *hash_buf = NULL;
-	int err;
+	u8 tkey_buf[TCP_AO_MAX_TRAFFIC_KEY_LEN];
 
-	hash_buf = kmalloc(tcp_ao_digest_size(ao_key), GFP_ATOMIC);
-	if (!hash_buf)
-		return -ENOMEM;
+	tcp_v4_ao_calc_key_rsk(ao_key, tkey_buf, req);
 
-	err = tcp_v4_ao_calc_key_rsk(ao_key, hash_buf, req);
-	if (err)
-		goto out;
-
-	err = tcp_ao_hash_skb(AF_INET, ao_hash, ao_key, req_to_sk(req), skb,
-			      hash_buf, hash_offset, sne);
-out:
-	kfree(hash_buf);
-	return err;
+	return tcp_ao_hash_skb(AF_INET, ao_hash, ao_key, req_to_sk(req), skb,
+			       tkey_buf, hash_offset, sne);
 }
 
 struct tcp_ao_key *tcp_v4_ao_lookup_rsk(const struct sock *sk,
@@ -807,14 +863,14 @@ int tcp_ao_prepare_reset(const struct sock *sk, struct sk_buff *skb,
 	return 0;
 }
 
-int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
-			struct tcp_ao_key *key, struct tcphdr *th,
-			__u8 *hash_location)
+void tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
+			 struct tcp_ao_key *key, struct tcphdr *th,
+			 __u8 *hash_location)
 {
 	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
+	u8 tkey_buf[TCP_AO_MAX_TRAFFIC_KEY_LEN];
 	struct tcp_sock *tp = tcp_sk(sk);
 	struct tcp_ao_info *ao;
-	void *tkey_buf = NULL;
 	u8 *traffic_key;
 	u32 sne;
 
@@ -826,9 +882,6 @@ int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
 
 		if (!(tcb->tcp_flags & TCPHDR_ACK)) {
 			disn = 0;
-			tkey_buf = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
-			if (!tkey_buf)
-				return -ENOMEM;
 			traffic_key = tkey_buf;
 		} else {
 			disn = ao->risn;
@@ -840,8 +893,6 @@ int tcp_ao_transmit_skb(struct sock *sk, struct sk_buff *skb,
 				 ntohl(th->seq));
 	tp->af_specific->calc_ao_hash(hash_location, key, sk, skb, traffic_key,
 				      hash_location - (u8 *)th, sne);
-	kfree(tkey_buf);
-	return 0;
 }
 
 static struct tcp_ao_key *tcp_ao_inbound_lookup(unsigned short int family,
@@ -906,7 +957,7 @@ tcp_ao_verify_hash(const struct sock *sk, const struct sk_buff *skb,
 {
 	const struct tcphdr *th = tcp_hdr(skb);
 	u8 maclen = tcp_ao_hdr_maclen(aoh);
-	void *hash_buf = NULL;
+	u8 hash_buf[TCP_AO_MAX_MAC_LEN];
 
 	if (maclen != tcp_ao_maclen(key)) {
 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOBAD);
@@ -917,10 +968,6 @@ tcp_ao_verify_hash(const struct sock *sk, const struct sk_buff *skb,
 		return SKB_DROP_REASON_TCP_AOFAILURE;
 	}
 
-	hash_buf = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
-	if (!hash_buf)
-		return SKB_DROP_REASON_NOT_SPECIFIED;
-
 	/* XXX: make it per-AF callback? */
 	tcp_ao_hash_skb(family, hash_buf, key, sk, skb, traffic_key,
 			(phash - (u8 *)th), sne);
@@ -930,13 +977,11 @@ tcp_ao_verify_hash(const struct sock *sk, const struct sk_buff *skb,
 		atomic64_inc(&key->pkt_bad);
 		trace_tcp_ao_mismatch(sk, skb, aoh->keyid,
 				      aoh->rnext_keyid, maclen);
-		kfree(hash_buf);
 		return SKB_DROP_REASON_TCP_AOFAILURE;
 	}
 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOGOOD);
 	atomic64_inc(&info->counters.pkt_good);
 	atomic64_inc(&key->pkt_good);
-	kfree(hash_buf);
 	return SKB_NOT_DROPPED_YET;
 }
 
@@ -945,11 +990,11 @@ tcp_inbound_ao_hash(struct sock *sk, const struct sk_buff *skb,
 		    unsigned short int family, const struct request_sock *req,
 		    int l3index, const struct tcp_ao_hdr *aoh)
 {
+	u8 tkey_buf[TCP_AO_MAX_TRAFFIC_KEY_LEN];
 	const struct tcphdr *th = tcp_hdr(skb);
 	u8 maclen = tcp_ao_hdr_maclen(aoh);
 	u8 *phash = (u8 *)(aoh + 1); /* hash goes just after the header */
 	struct tcp_ao_info *info;
-	enum skb_drop_reason ret;
 	struct tcp_ao_key *key;
 	__be32 sisn, disn;
 	u8 *traffic_key;
@@ -1057,14 +1102,9 @@ tcp_inbound_ao_hash(struct sock *sk, const struct sk_buff *skb,
 		return SKB_DROP_REASON_TCP_AOFAILURE;
 	}
 verify_hash:
-	traffic_key = kmalloc(tcp_ao_digest_size(key), GFP_ATOMIC);
-	if (!traffic_key)
-		return SKB_DROP_REASON_NOT_SPECIFIED;
-	tcp_ao_calc_key_skb(key, traffic_key, skb, sisn, disn, family);
-	ret = tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
-				 traffic_key, phash, sne, l3index);
-	kfree(traffic_key);
-	return ret;
+	tcp_ao_calc_key_skb(key, tkey_buf, skb, sisn, disn, family);
+	return tcp_ao_verify_hash(sk, skb, family, info, aoh, key,
+				  tkey_buf, phash, sne, l3index);
 
 key_not_found:
 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOKEYNOTFOUND);
@@ -1281,7 +1321,6 @@ int tcp_ao_copy_all_matching(const struct sock *sk, struct sock *newsk,
 free_and_exit:
 	hlist_for_each_entry_safe(key, key_head, &new_ao->head, node) {
 		hlist_del(&key->node);
-		tcp_sigpool_release(key->tcp_sigpool_id);
 		atomic_sub(tcp_ao_sizeof_key(key), &newsk->sk_omem_alloc);
 		kfree_sensitive(key);
 	}
@@ -1337,23 +1376,10 @@ static int tcp_ao_verify_ipv4(struct sock *sk, struct tcp_ao_add *cmd,
 	return 0;
 }
 
-static int tcp_ao_parse_crypto(struct tcp_ao_add *cmd, struct tcp_ao_key *key)
+static int tcp_ao_parse_crypto(const struct tcp_ao_add *cmd,
+			       struct tcp_ao_key *key)
 {
 	unsigned int syn_tcp_option_space;
-	bool is_kdf_aes_128_cmac = false;
-	struct crypto_ahash *tfm;
-	struct tcp_sigpool hp;
-	void *tmp_key = NULL;
-	int err;
-
-	/* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
-	if (!strcmp("cmac(aes128)", cmd->alg_name)) {
-		strscpy(cmd->alg_name, "cmac(aes)", sizeof(cmd->alg_name));
-		is_kdf_aes_128_cmac = (cmd->keylen != 16);
-		tmp_key = kmalloc(cmd->keylen, GFP_KERNEL);
-		if (!tmp_key)
-			return -ENOMEM;
-	}
 
 	key->maclen = cmd->maclen ?: 12; /* 12 is the default in RFC5925 */
 
@@ -1389,64 +1415,27 @@ static int tcp_ao_parse_crypto(struct tcp_ao_add *cmd, struct tcp_ao_key *key)
 	syn_tcp_option_space -= TCPOLEN_MSS_ALIGNED;
 	syn_tcp_option_space -= TCPOLEN_TSTAMP_ALIGNED;
 	syn_tcp_option_space -= TCPOLEN_WSCALE_ALIGNED;
-	if (tcp_ao_len_aligned(key) > syn_tcp_option_space) {
-		err = -EMSGSIZE;
-		goto err_kfree;
-	}
-
-	key->keylen = cmd->keylen;
-	memcpy(key->key, cmd->key, cmd->keylen);
-
-	err = tcp_sigpool_start(key->tcp_sigpool_id, &hp);
-	if (err)
-		goto err_kfree;
-
-	tfm = crypto_ahash_reqtfm(hp.req);
-	if (is_kdf_aes_128_cmac) {
-		void *scratch = hp.scratch;
-		struct scatterlist sg;
-
-		memcpy(tmp_key, cmd->key, cmd->keylen);
-		sg_init_one(&sg, tmp_key, cmd->keylen);
-
-		/* Using zero-key of 16 bytes as described in RFC5926 */
-		memset(scratch, 0, 16);
-		err = crypto_ahash_setkey(tfm, scratch, 16);
-		if (err)
-			goto err_pool_end;
-
-		err = crypto_ahash_init(hp.req);
-		if (err)
-			goto err_pool_end;
-
-		ahash_request_set_crypt(hp.req, &sg, key->key, cmd->keylen);
-		err = crypto_ahash_update(hp.req);
-		if (err)
-			goto err_pool_end;
-
-		err |= crypto_ahash_final(hp.req);
-		if (err)
-			goto err_pool_end;
-		key->keylen = 16;
+	if (tcp_ao_len_aligned(key) > syn_tcp_option_space)
+		return -EMSGSIZE;
+
+	if (key->algo == TCP_AO_ALGO_AES_128_CMAC &&
+	    cmd->keylen != AES_KEYSIZE_128) {
+		/* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
+		static const u8 zeroes[AES_KEYSIZE_128];
+		struct aes_cmac_key extractor;
+
+		aes_cmac_preparekey(&extractor, zeroes, AES_KEYSIZE_128);
+		aes_cmac(&extractor, cmd->key, cmd->keylen, key->key);
+		key->keylen = AES_KEYSIZE_128;
+	} else {
+		memcpy(key->key, cmd->key, cmd->keylen);
+		key->keylen = cmd->keylen;
 	}
 
-	err = crypto_ahash_setkey(tfm, key->key, key->keylen);
-	if (err)
-		goto err_pool_end;
-
-	tcp_sigpool_end(&hp);
-	kfree_sensitive(tmp_key);
-
 	if (tcp_ao_maclen(key) > key->digest_size)
 		return -EINVAL;
 
 	return 0;
-
-err_pool_end:
-	tcp_sigpool_end(&hp);
-err_kfree:
-	kfree_sensitive(tmp_key);
-	return err;
 }
 
 #if IS_ENABLED(CONFIG_IPV6)
@@ -1550,50 +1539,33 @@ static struct tcp_ao_info *getsockopt_ao_info(struct sock *sk)
 static struct tcp_ao_key *tcp_ao_key_alloc(struct sock *sk,
 					   struct tcp_ao_add *cmd)
 {
-	const char *algo = cmd->alg_name;
-	unsigned int digest_size;
-	struct crypto_ahash *tfm;
+	const struct tcp_ao_algo *algo;
 	struct tcp_ao_key *key;
-	struct tcp_sigpool hp;
-	int err, pool_id;
 	size_t size;
 
 	/* Force null-termination of alg_name */
 	cmd->alg_name[ARRAY_SIZE(cmd->alg_name) - 1] = '\0';
 
-	/* RFC5926, 3.1.1.2. KDF_AES_128_CMAC */
-	if (!strcmp("cmac(aes128)", algo))
-		algo = "cmac(aes)";
-
-	/* Full TCP header (th->doff << 2) should fit into scratch area,
-	 * see tcp_ao_hash_header().
+	/*
+	 * For backwards compatibility, accept "cmac(aes)" as an alias for
+	 * "cmac(aes128)", provided that the key length is exactly 128 bits.
 	 */
-	pool_id = tcp_sigpool_alloc_ahash(algo, 60);
-	if (pool_id < 0)
-		return ERR_PTR(pool_id);
-
-	err = tcp_sigpool_start(pool_id, &hp);
-	if (err)
-		goto err_free_pool;
+	if (strcmp(cmd->alg_name, "cmac(aes)") == 0 &&
+	    cmd->keylen == AES_KEYSIZE_128)
+		strscpy(cmd->alg_name, "cmac(aes128)");
 
-	tfm = crypto_ahash_reqtfm(hp.req);
-	digest_size = crypto_ahash_digestsize(tfm);
-	tcp_sigpool_end(&hp);
+	algo = tcp_ao_find_algo(cmd->alg_name);
+	if (!algo)
+		return ERR_PTR(-ENOENT);
 
-	size = sizeof(struct tcp_ao_key) + (digest_size << 1);
+	size = sizeof(struct tcp_ao_key) + (algo->digest_size << 1);
 	key = sock_kmalloc(sk, size, GFP_KERNEL);
-	if (!key) {
-		err = -ENOMEM;
-		goto err_free_pool;
-	}
+	if (!key)
+		return ERR_PTR(-ENOMEM);
 
-	key->tcp_sigpool_id = pool_id;
-	key->digest_size = digest_size;
+	key->algo = algo - tcp_ao_algos;
+	key->digest_size = algo->digest_size;
 	return key;
-
-err_free_pool:
-	tcp_sigpool_release(pool_id);
-	return ERR_PTR(err);
 }
 
 static int tcp_ao_add_cmd(struct sock *sk, unsigned short int family,
@@ -1754,7 +1726,6 @@ static int tcp_ao_add_cmd(struct sock *sk, unsigned short int family,
 
 err_free_sock:
 	atomic_sub(tcp_ao_sizeof_key(key), &sk->sk_omem_alloc);
-	tcp_sigpool_release(key->tcp_sigpool_id);
 	kfree_sensitive(key);
 err_free_ao:
 	if (first)
@@ -2285,7 +2256,11 @@ static int tcp_ao_copy_mkts_to_user(const struct sock *sk,
 		opt_out.pkt_good = atomic64_read(&key->pkt_good);
 		opt_out.pkt_bad = atomic64_read(&key->pkt_bad);
 		memcpy(&opt_out.key, key->key, key->keylen);
-		tcp_sigpool_algo(key->tcp_sigpool_id, opt_out.alg_name, 64);
+		if (key->algo == TCP_AO_ALGO_AES_128_CMAC)
+			/* This is needed for backwards compatibility. */
+			strscpy(opt_out.alg_name, "cmac(aes)");
+		else
+			strscpy(opt_out.alg_name, tcp_ao_algos[key->algo].name);
 
 		/* Copy key to user */
 		if (copy_to_sockptr_offset(optval, out_offset,
diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
index d5c9e65d97606..ab7a4e5435a8a 100644
--- a/net/ipv4/tcp_input.c
+++ b/net/ipv4/tcp_input.c
@@ -6474,7 +6474,7 @@ void tcp_rcv_established(struct sock *sk, struct sk_buff *skb)
 	/* TCP congestion window tracking */
 	trace_tcp_probe(sk, skb);
 
-	tcp_mstamp_refresh(tp);
+	tcp_mstamp_refresh_inline(tp);
 	if (unlikely(!rcu_access_pointer(sk->sk_rx_dst)))
 		inet_csk(sk)->icsk_af_ops->sk_rx_dst_set(sk, skb);
 	/*
@@ -7589,6 +7589,7 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops,
 		     struct sock *sk, struct sk_buff *skb)
 {
 	struct tcp_fastopen_cookie foc = { .len = -1 };
+	u32 isn = TCP_SKB_CB(skb)->tcp_tw_isn;
 	struct tcp_options_received tmp_opt;
 	const struct tcp_sock *tp = tcp_sk(sk);
 	struct net *net = sock_net(sk);
@@ -7599,20 +7600,16 @@ int tcp_conn_request(struct request_sock_ops *rsk_ops,
 	struct dst_entry *dst;
 	struct flowi fl;
 	u8 syncookies;
-	u32 isn;
 
 #ifdef CONFIG_TCP_AO
 	const struct tcp_ao_hdr *aoh;
 #endif
 
-	isn = __this_cpu_read(tcp_tw_isn);
-	if (isn) {
-		/* TW buckets are converted to open requests without
-		 * limitations, they conserve resources and peer is
-		 * evidently real one.
-		 */
-		__this_cpu_write(tcp_tw_isn, 0);
-	} else {
+	/* If isn is non-zero, this SYN originally matched a TIME_WAIT socket.
+	 * TW sockets are converted to open requests without limitations,
+	 * we skip the queue limits and syncookie checks in the block below.
+	 */
+	if (!isn) {
 		syncookies = READ_ONCE(net->ipv4.sysctl_tcp_syncookies);
 
 		if (syncookies == 2 || inet_csk_reqsk_queue_is_full(sk)) {
diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c
index c0526cc039804..fdc81150ff6cf 100644
--- a/net/ipv4/tcp_ipv4.c
+++ b/net/ipv4/tcp_ipv4.c
@@ -2198,6 +2198,7 @@ int tcp_v4_rcv(struct sk_buff *skb)
 		}
 	}
 
+	isn = 0;
 process:
 	if (static_branch_unlikely(&ip4_min_ttl)) {
 		/* min_ttl can be changed concurrently from do_ip_setsockopt() */
@@ -2227,6 +2228,7 @@ int tcp_v4_rcv(struct sk_buff *skb)
 	th = (const struct tcphdr *)skb->data;
 	iph = ip_hdr(skb);
 	tcp_v4_fill_cb(skb, iph, th);
+	TCP_SKB_CB(skb)->tcp_tw_isn = isn;
 
 	skb->dev = NULL;
 
@@ -2313,7 +2315,6 @@ int tcp_v4_rcv(struct sk_buff *skb)
 			sk = sk2;
 			tcp_v4_restore_cb(skb);
 			refcounted = false;
-			__this_cpu_write(tcp_tw_isn, isn);
 			goto process;
 		}
 
diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
index f9d8755705f76..47647a99e3244 100644
--- a/net/ipv4/tcp_output.c
+++ b/net/ipv4/tcp_output.c
@@ -52,15 +52,9 @@
 
 #include <trace/events/tcp.h>
 
-/* Refresh clocks of a TCP socket,
- * ensuring monotically increasing values.
- */
-void tcp_mstamp_refresh(struct tcp_sock *tp)
+void noinline tcp_mstamp_refresh(struct tcp_sock *tp)
 {
-	u64 val = tcp_clock_ns();
-
-	tp->tcp_clock_cache = val;
-	tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC);
+	tcp_mstamp_refresh_inline(tp);
 }
 
 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
@@ -1421,6 +1415,7 @@ void tcp_wfree(struct sk_buff *skb)
 out:
 	sk_free(sk);
 }
+EXPORT_SYMBOL_GPL(tcp_wfree);
 
 /* Note: Called under soft irq.
  * We can call TCP stack right away, unless socket is owned by user.
@@ -1663,14 +1658,8 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
 					       key.md5_key, sk, skb);
 #endif
 	} else if (tcp_key_is_ao(&key)) {
-		int err;
-
-		err = tcp_ao_transmit_skb(sk, skb, key.ao_key, th,
-					  opts.hash_location);
-		if (err) {
-			sk_skb_reason_drop(sk, skb, SKB_DROP_REASON_NOT_SPECIFIED);
-			return -ENOMEM;
-		}
+		tcp_ao_transmit_skb(sk, skb, key.ao_key, th,
+				    opts.hash_location);
 	}
 
 	/* BPF prog is the last one writing header option */
@@ -2971,7 +2960,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
 
 	sent_pkts = 0;
 
-	tcp_mstamp_refresh(tp);
+	tcp_mstamp_refresh_inline(tp);
 
 	/* AccECN option beacon depends on mstamp, it may change mss */
 	if (tcp_ecn_mode_accecn(tp) && tcp_accecn_option_beacon_check(sk))
diff --git a/net/ipv4/tcp_sigpool.c b/net/ipv4/tcp_sigpool.c
deleted file mode 100644
index 10b2e5970c402..0000000000000
--- a/net/ipv4/tcp_sigpool.c
+++ /dev/null
@@ -1,366 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-
-#include <crypto/hash.h>
-#include <linux/cpu.h>
-#include <linux/kref.h>
-#include <linux/module.h>
-#include <linux/mutex.h>
-#include <linux/percpu.h>
-#include <linux/workqueue.h>
-#include <net/tcp.h>
-
-static size_t __scratch_size;
-struct sigpool_scratch {
-	local_lock_t bh_lock;
-	void __rcu *pad;
-};
-
-static DEFINE_PER_CPU(struct sigpool_scratch, sigpool_scratch) = {
-	.bh_lock = INIT_LOCAL_LOCK(bh_lock),
-};
-
-struct sigpool_entry {
-	struct crypto_ahash	*hash;
-	const char		*alg;
-	struct kref		kref;
-	uint16_t		needs_key:1,
-				reserved:15;
-};
-
-#define CPOOL_SIZE (PAGE_SIZE / sizeof(struct sigpool_entry))
-static struct sigpool_entry cpool[CPOOL_SIZE];
-static unsigned int cpool_populated;
-static DEFINE_MUTEX(cpool_mutex);
-
-/* Slow-path */
-struct scratches_to_free {
-	struct rcu_head rcu;
-	unsigned int cnt;
-	void *scratches[];
-};
-
-static void free_old_scratches(struct rcu_head *head)
-{
-	struct scratches_to_free *stf;
-
-	stf = container_of(head, struct scratches_to_free, rcu);
-	while (stf->cnt--)
-		kfree(stf->scratches[stf->cnt]);
-	kfree(stf);
-}
-
-/**
- * sigpool_reserve_scratch - re-allocates scratch buffer, slow-path
- * @size: request size for the scratch/temp buffer
- */
-static int sigpool_reserve_scratch(size_t size)
-{
-	struct scratches_to_free *stf;
-	size_t stf_sz = struct_size(stf, scratches, num_possible_cpus());
-	int cpu, err = 0;
-
-	lockdep_assert_held(&cpool_mutex);
-	if (__scratch_size >= size)
-		return 0;
-
-	stf = kmalloc(stf_sz, GFP_KERNEL);
-	if (!stf)
-		return -ENOMEM;
-	stf->cnt = 0;
-
-	size = max(size, __scratch_size);
-	cpus_read_lock();
-	for_each_possible_cpu(cpu) {
-		void *scratch, *old_scratch;
-
-		scratch = kmalloc_node(size, GFP_KERNEL, cpu_to_node(cpu));
-		if (!scratch) {
-			err = -ENOMEM;
-			break;
-		}
-
-		old_scratch = rcu_replace_pointer(per_cpu(sigpool_scratch.pad, cpu),
-					scratch, lockdep_is_held(&cpool_mutex));
-		if (!cpu_online(cpu) || !old_scratch) {
-			kfree(old_scratch);
-			continue;
-		}
-		stf->scratches[stf->cnt++] = old_scratch;
-	}
-	cpus_read_unlock();
-	if (!err)
-		__scratch_size = size;
-
-	call_rcu(&stf->rcu, free_old_scratches);
-	return err;
-}
-
-static void sigpool_scratch_free(void)
-{
-	int cpu;
-
-	for_each_possible_cpu(cpu)
-		kfree(rcu_replace_pointer(per_cpu(sigpool_scratch.pad, cpu),
-					  NULL, lockdep_is_held(&cpool_mutex)));
-	__scratch_size = 0;
-}
-
-static int __cpool_try_clone(struct crypto_ahash *hash)
-{
-	struct crypto_ahash *tmp;
-
-	tmp = crypto_clone_ahash(hash);
-	if (IS_ERR(tmp))
-		return PTR_ERR(tmp);
-
-	crypto_free_ahash(tmp);
-	return 0;
-}
-
-static int __cpool_alloc_ahash(struct sigpool_entry *e, const char *alg)
-{
-	struct crypto_ahash *cpu0_hash;
-	int ret;
-
-	e->alg = kstrdup(alg, GFP_KERNEL);
-	if (!e->alg)
-		return -ENOMEM;
-
-	cpu0_hash = crypto_alloc_ahash(alg, 0, CRYPTO_ALG_ASYNC);
-	if (IS_ERR(cpu0_hash)) {
-		ret = PTR_ERR(cpu0_hash);
-		goto out_free_alg;
-	}
-
-	e->needs_key = crypto_ahash_get_flags(cpu0_hash) & CRYPTO_TFM_NEED_KEY;
-
-	ret = __cpool_try_clone(cpu0_hash);
-	if (ret)
-		goto out_free_cpu0_hash;
-	e->hash = cpu0_hash;
-	kref_init(&e->kref);
-	return 0;
-
-out_free_cpu0_hash:
-	crypto_free_ahash(cpu0_hash);
-out_free_alg:
-	kfree(e->alg);
-	e->alg = NULL;
-	return ret;
-}
-
-/**
- * tcp_sigpool_alloc_ahash - allocates pool for ahash requests
- * @alg: name of async hash algorithm
- * @scratch_size: reserve a tcp_sigpool::scratch buffer of this size
- */
-int tcp_sigpool_alloc_ahash(const char *alg, size_t scratch_size)
-{
-	int i, ret;
-
-	/* slow-path */
-	mutex_lock(&cpool_mutex);
-	ret = sigpool_reserve_scratch(scratch_size);
-	if (ret)
-		goto out;
-	for (i = 0; i < cpool_populated; i++) {
-		if (!cpool[i].alg)
-			continue;
-		if (strcmp(cpool[i].alg, alg))
-			continue;
-
-		/* pairs with tcp_sigpool_release() */
-		if (!kref_get_unless_zero(&cpool[i].kref))
-			kref_init(&cpool[i].kref);
-		ret = i;
-		goto out;
-	}
-
-	for (i = 0; i < cpool_populated; i++) {
-		if (!cpool[i].alg)
-			break;
-	}
-	if (i >= CPOOL_SIZE) {
-		ret = -ENOSPC;
-		goto out;
-	}
-
-	ret = __cpool_alloc_ahash(&cpool[i], alg);
-	if (!ret) {
-		ret = i;
-		if (i == cpool_populated)
-			cpool_populated++;
-	}
-out:
-	mutex_unlock(&cpool_mutex);
-	return ret;
-}
-EXPORT_SYMBOL_GPL(tcp_sigpool_alloc_ahash);
-
-static void __cpool_free_entry(struct sigpool_entry *e)
-{
-	crypto_free_ahash(e->hash);
-	kfree(e->alg);
-	memset(e, 0, sizeof(*e));
-}
-
-static void cpool_cleanup_work_cb(struct work_struct *work)
-{
-	bool free_scratch = true;
-	unsigned int i;
-
-	mutex_lock(&cpool_mutex);
-	for (i = 0; i < cpool_populated; i++) {
-		if (kref_read(&cpool[i].kref) > 0) {
-			free_scratch = false;
-			continue;
-		}
-		if (!cpool[i].alg)
-			continue;
-		__cpool_free_entry(&cpool[i]);
-	}
-	if (free_scratch)
-		sigpool_scratch_free();
-	mutex_unlock(&cpool_mutex);
-}
-
-static DECLARE_WORK(cpool_cleanup_work, cpool_cleanup_work_cb);
-static void cpool_schedule_cleanup(struct kref *kref)
-{
-	schedule_work(&cpool_cleanup_work);
-}
-
-/**
- * tcp_sigpool_release - decreases number of users for a pool. If it was
- * the last user of the pool, releases any memory that was consumed.
- * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
- */
-void tcp_sigpool_release(unsigned int id)
-{
-	if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg))
-		return;
-
-	/* slow-path */
-	kref_put(&cpool[id].kref, cpool_schedule_cleanup);
-}
-EXPORT_SYMBOL_GPL(tcp_sigpool_release);
-
-/**
- * tcp_sigpool_get - increases number of users (refcounter) for a pool
- * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
- */
-void tcp_sigpool_get(unsigned int id)
-{
-	if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg))
-		return;
-	kref_get(&cpool[id].kref);
-}
-EXPORT_SYMBOL_GPL(tcp_sigpool_get);
-
-int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c) __cond_acquires(0, RCU_BH)
-{
-	struct crypto_ahash *hash;
-
-	rcu_read_lock_bh();
-	if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg)) {
-		rcu_read_unlock_bh();
-		return -EINVAL;
-	}
-
-	hash = crypto_clone_ahash(cpool[id].hash);
-	if (IS_ERR(hash)) {
-		rcu_read_unlock_bh();
-		return PTR_ERR(hash);
-	}
-
-	c->req = ahash_request_alloc(hash, GFP_ATOMIC);
-	if (!c->req) {
-		crypto_free_ahash(hash);
-		rcu_read_unlock_bh();
-		return -ENOMEM;
-	}
-	ahash_request_set_callback(c->req, 0, NULL, NULL);
-
-	/* Pairs with tcp_sigpool_reserve_scratch(), scratch area is
-	 * valid (allocated) until tcp_sigpool_end().
-	 */
-	local_lock_nested_bh(&sigpool_scratch.bh_lock);
-	c->scratch = rcu_dereference_bh(*this_cpu_ptr(&sigpool_scratch.pad));
-	return 0;
-}
-EXPORT_SYMBOL_GPL(tcp_sigpool_start);
-
-void tcp_sigpool_end(struct tcp_sigpool *c) __releases(RCU_BH)
-{
-	struct crypto_ahash *hash = crypto_ahash_reqtfm(c->req);
-
-	local_unlock_nested_bh(&sigpool_scratch.bh_lock);
-	rcu_read_unlock_bh();
-	ahash_request_free(c->req);
-	crypto_free_ahash(hash);
-}
-EXPORT_SYMBOL_GPL(tcp_sigpool_end);
-
-/**
- * tcp_sigpool_algo - return algorithm of tcp_sigpool
- * @id: tcp_sigpool that was previously allocated by tcp_sigpool_alloc_ahash()
- * @buf: buffer to return name of algorithm
- * @buf_len: size of @buf
- */
-size_t tcp_sigpool_algo(unsigned int id, char *buf, size_t buf_len)
-{
-	if (WARN_ON_ONCE(id >= cpool_populated || !cpool[id].alg))
-		return -EINVAL;
-
-	return strscpy(buf, cpool[id].alg, buf_len);
-}
-EXPORT_SYMBOL_GPL(tcp_sigpool_algo);
-
-/**
- * tcp_sigpool_hash_skb_data - hash data in skb with initialized tcp_sigpool
- * @hp: tcp_sigpool pointer
- * @skb: buffer to add sign for
- * @header_len: TCP header length for this segment
- */
-int tcp_sigpool_hash_skb_data(struct tcp_sigpool *hp,
-			      const struct sk_buff *skb,
-			      unsigned int header_len)
-{
-	const unsigned int head_data_len = skb_headlen(skb) > header_len ?
-					   skb_headlen(skb) - header_len : 0;
-	const struct skb_shared_info *shi = skb_shinfo(skb);
-	const struct tcphdr *tp = tcp_hdr(skb);
-	struct ahash_request *req = hp->req;
-	struct sk_buff *frag_iter;
-	struct scatterlist sg;
-	unsigned int i;
-
-	sg_init_table(&sg, 1);
-
-	sg_set_buf(&sg, ((u8 *)tp) + header_len, head_data_len);
-	ahash_request_set_crypt(req, &sg, NULL, head_data_len);
-	if (crypto_ahash_update(req))
-		return 1;
-
-	for (i = 0; i < shi->nr_frags; ++i) {
-		const skb_frag_t *f = &shi->frags[i];
-		unsigned int offset = skb_frag_off(f);
-		struct page *page;
-
-		page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
-		sg_set_page(&sg, page, skb_frag_size(f), offset_in_page(offset));
-		ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f));
-		if (crypto_ahash_update(req))
-			return 1;
-	}
-
-	skb_walk_frags(skb, frag_iter)
-		if (tcp_sigpool_hash_skb_data(hp, frag_iter, 0))
-			return 1;
-
-	return 0;
-}
-EXPORT_SYMBOL(tcp_sigpool_hash_skb_data);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Per-CPU pool of crypto requests");
diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c
index a0813d425b716..29651b1a0bc70 100644
--- a/net/ipv4/udp_offload.c
+++ b/net/ipv4/udp_offload.c
@@ -482,11 +482,11 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
 	struct sock *sk = gso_skb->sk;
 	unsigned int sum_truesize = 0;
 	struct sk_buff *segs, *seg;
-	__be16 newlen, msslen;
 	struct udphdr *uh;
 	unsigned int mss;
 	bool copy_dtor;
 	__sum16 check;
+	__be16 newlen;
 	int ret = 0;
 
 	mss = skb_shinfo(gso_skb)->gso_size;
@@ -555,15 +555,6 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
 		return segs;
 	}
 
-	msslen = htons(sizeof(*uh) + mss);
-
-	/* GSO partial and frag_list segmentation only requires splitting
-	 * the frame into an MSS multiple and possibly a remainder, both
-	 * cases return a GSO skb. So update the mss now.
-	 */
-	if (skb_is_gso(segs))
-		mss *= skb_shinfo(segs)->gso_segs;
-
 	seg = segs;
 	uh = udp_hdr(seg);
 
@@ -586,7 +577,7 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
 		if (!seg->next)
 			break;
 
-		uh->len = msslen;
+		uh->len = newlen;
 		uh->check = check;
 
 		if (seg->ip_summed == CHECKSUM_PARTIAL)
@@ -599,9 +590,12 @@ struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb,
 		uh = udp_hdr(seg);
 	}
 
-	/* last packet can be partial gso_size, account for that in checksum */
-	newlen = htons(skb_tail_pointer(seg) - skb_transport_header(seg) +
-		       seg->data_len);
+	/* Unless skb fits perfectly as GSO_PARTIAL, the trailing
+	 * segment may not be full MSS, account for that in the checksum
+	 */
+	if (!skb_is_gso(seg))
+		newlen = htons(skb_tail_pointer(seg) -
+			       skb_transport_header(seg) + seg->data_len);
 	check = csum16_add(csum16_sub(uh->check, uh->len), newlen);
 
 	uh->len = newlen;
diff --git a/net/ipv4/udp_tunnel_core.c b/net/ipv4/udp_tunnel_core.c
index b1f667c52cb21..9ab3728f9630b 100644
--- a/net/ipv4/udp_tunnel_core.c
+++ b/net/ipv4/udp_tunnel_core.c
@@ -68,11 +68,9 @@ static bool sk_saddr_any(struct sock *sk)
 #endif
 }
 
-void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
+void setup_udp_tunnel_sock(struct net *net, struct sock *sk,
 			   struct udp_tunnel_sock_cfg *cfg)
 {
-	struct sock *sk = sock->sk;
-
 	/* Disable multicast loopback */
 	inet_clear_bit(MC_LOOP, sk);
 
@@ -99,10 +97,9 @@ void setup_udp_tunnel_sock(struct net *net, struct socket *sock,
 }
 EXPORT_SYMBOL_GPL(setup_udp_tunnel_sock);
 
-void udp_tunnel_push_rx_port(struct net_device *dev, struct socket *sock,
+void udp_tunnel_push_rx_port(struct net_device *dev, struct sock *sk,
 			     unsigned short type)
 {
-	struct sock *sk = sock->sk;
 	struct udp_tunnel_info ti;
 
 	ti.type = type;
@@ -113,10 +110,9 @@ void udp_tunnel_push_rx_port(struct net_device *dev, struct socket *sock,
 }
 EXPORT_SYMBOL_GPL(udp_tunnel_push_rx_port);
 
-void udp_tunnel_drop_rx_port(struct net_device *dev, struct socket *sock,
+void udp_tunnel_drop_rx_port(struct net_device *dev, struct sock *sk,
 			     unsigned short type)
 {
-	struct sock *sk = sock->sk;
 	struct udp_tunnel_info ti;
 
 	ti.type = type;
@@ -128,9 +124,8 @@ void udp_tunnel_drop_rx_port(struct net_device *dev, struct socket *sock,
 EXPORT_SYMBOL_GPL(udp_tunnel_drop_rx_port);
 
 /* Notify netdevs that UDP port started listening */
-void udp_tunnel_notify_add_rx_port(struct socket *sock, unsigned short type)
+void udp_tunnel_notify_add_rx_port(struct sock *sk, unsigned short type)
 {
-	struct sock *sk = sock->sk;
 	struct net *net = sock_net(sk);
 	struct udp_tunnel_info ti;
 	struct net_device *dev;
@@ -150,9 +145,8 @@ void udp_tunnel_notify_add_rx_port(struct socket *sock, unsigned short type)
 EXPORT_SYMBOL_GPL(udp_tunnel_notify_add_rx_port);
 
 /* Notify netdevs that UDP port is no more listening */
-void udp_tunnel_notify_del_rx_port(struct socket *sock, unsigned short type)
+void udp_tunnel_notify_del_rx_port(struct sock *sk, unsigned short type)
 {
-	struct sock *sk = sock->sk;
 	struct net *net = sock_net(sk);
 	struct udp_tunnel_info ti;
 	struct net_device *dev;
@@ -195,10 +189,11 @@ void udp_tunnel_xmit_skb(struct rtable *rt, struct sock *sk, struct sk_buff *skb
 }
 EXPORT_SYMBOL_GPL(udp_tunnel_xmit_skb);
 
-void udp_tunnel_sock_release(struct socket *sock)
+void udp_tunnel_sock_release(struct sock *sk)
 {
-	rcu_assign_sk_user_data(sock->sk, NULL);
-	synchronize_rcu();
+	struct socket *sock = sk->sk_socket;
+
+	rcu_assign_sk_user_data(sk, NULL);
 	kernel_sock_shutdown(sock, SHUT_RDWR);
 	sock_release(sock);
 }
diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c
index 5476b6536eb76..da0f07e21e3fd 100644
--- a/net/ipv6/addrconf.c
+++ b/net/ipv6/addrconf.c
@@ -2166,16 +2166,18 @@ void addrconf_dad_failure(struct sk_buff *skb, struct inet6_ifaddr *ifp)
 	struct net *net = dev_net(idev->dev);
 	int max_addresses;
 
-	if (addrconf_dad_end(ifp)) {
+	spin_lock_bh(&ifp->lock);
+
+	if (ifp->state != INET6_IFADDR_STATE_DAD) {
+		spin_unlock_bh(&ifp->lock);
 		in6_ifa_put(ifp);
 		return;
 	}
+	ifp->state = INET6_IFADDR_STATE_POSTDAD;
 
 	net_info_ratelimited("%s: IPv6 duplicate address %pI6c used by %pM detected!\n",
 			     ifp->idev->dev->name, &ifp->addr, eth_hdr(skb)->h_source);
 
-	spin_lock_bh(&ifp->lock);
-
 	if (ifp->flags & IFA_F_STABLE_PRIVACY) {
 		struct in6_addr new_addr;
 		struct inet6_ifaddr *ifp2;
@@ -2223,6 +2225,11 @@ void addrconf_dad_failure(struct sk_buff *skb, struct inet6_ifaddr *ifp)
 		in6_ifa_put(ifp2);
 lock_errdad:
 		spin_lock_bh(&ifp->lock);
+		if (ifp->state != INET6_IFADDR_STATE_POSTDAD) {
+			spin_unlock_bh(&ifp->lock);
+			in6_ifa_put(ifp);
+			return;
+		}
 	}
 
 errdad:
diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c
index de1e68199a014..76f7a2de9108e 100644
--- a/net/ipv6/ah6.c
+++ b/net/ipv6/ah6.c
@@ -337,7 +337,7 @@ static void ah6_output_done(void *data, int err)
 	ah6_restore_hdrs(top_iph, iph_ext, extlen);
 
 	kfree(AH_SKB_CB(skb)->tmp);
-	xfrm_output_resume(skb->sk, skb, err);
+	xfrm_output_resume(skb_to_full_sk(skb), skb, err);
 }
 
 static int ah6_output(struct xfrm_state *x, struct sk_buff *skb)
diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c
index 03cbce842c1a7..47c5502a34a22 100644
--- a/net/ipv6/exthdrs.c
+++ b/net/ipv6/exthdrs.c
@@ -910,16 +910,27 @@ static bool ipv6_hop_ra(struct sk_buff *skb, int optoff)
 
 static bool ipv6_hop_ioam(struct sk_buff *skb, int optoff)
 {
+	enum skb_drop_reason drop_reason;
 	struct ioam6_trace_hdr *trace;
 	struct ioam6_namespace *ns;
+	struct inet6_dev *idev;
 	struct ioam6_hdr *hdr;
 
+	drop_reason = SKB_DROP_REASON_IP_INHDR;
+
 	/* Bad alignment (must be 4n-aligned) */
 	if (optoff & 3)
 		goto drop;
 
+	/* Does the device still have IPv6 configuration? */
+	idev = __in6_dev_get(skb->dev);
+	if (!idev) {
+		drop_reason = SKB_DROP_REASON_IPV6DISABLED;
+		goto drop;
+	}
+
 	/* Ignore if IOAM is not enabled on ingress */
-	if (!READ_ONCE(__in6_dev_get(skb->dev)->cnf.ioam6_enabled))
+	if (!READ_ONCE(idev->cnf.ioam6_enabled))
 		goto ignore;
 
 	/* Truncated Option header */
@@ -972,7 +983,7 @@ static bool ipv6_hop_ioam(struct sk_buff *skb, int optoff)
 	return true;
 
 drop:
-	kfree_skb_reason(skb, SKB_DROP_REASON_IP_INHDR);
+	kfree_skb_reason(skb, drop_reason);
 	return false;
 }
 
diff --git a/net/ipv6/inet6_connection_sock.c b/net/ipv6/inet6_connection_sock.c
index 37534e1168992..4665d84a7380d 100644
--- a/net/ipv6/inet6_connection_sock.c
+++ b/net/ipv6/inet6_connection_sock.c
@@ -102,7 +102,8 @@ int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl_unused
 		if (IS_ERR(dst)) {
 			WRITE_ONCE(sk->sk_err_soft, -PTR_ERR(dst));
 			sk->sk_route_caps = 0;
-			kfree_skb(skb);
+			sk_skb_reason_drop(sk, skb,
+					   SKB_DROP_REASON_IP_OUTNOROUTES);
 			return PTR_ERR(dst);
 		}
 		/* Restore final destination back after routing done */
diff --git a/net/ipv6/ioam6.c b/net/ipv6/ioam6.c
index e963a71858a78..943861a5581ee 100644
--- a/net/ipv6/ioam6.c
+++ b/net/ipv6/ioam6.c
@@ -800,7 +800,7 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb,
 	/* queue depth */
 	if (trace->type.bit6) {
 		struct netdev_queue *queue;
-		struct Qdisc *qdisc;
+		const struct Qdisc *qdisc;
 		__u32 qlen, backlog;
 
 		if (dev->flags & IFF_LOOPBACK ||
@@ -810,9 +810,7 @@ static void __ioam6_fill_trace_data(struct sk_buff *skb,
 			queue = skb_get_tx_queue(dev, skb);
 			qdisc = rcu_dereference(queue->qdisc);
 
-			spin_lock_bh(qdisc_lock(qdisc));
 			qdisc_qstats_qlen_backlog(qdisc, &qlen, &backlog);
-			spin_unlock_bh(qdisc_lock(qdisc));
 
 			*(__be32 *)data = cpu_to_be32(backlog);
 		}
diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c
index d8072ad6b8c4a..78f50c93c536c 100644
--- a/net/ipv6/ip6_offload.c
+++ b/net/ipv6/ip6_offload.c
@@ -297,6 +297,7 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
 
 	return pp;
 }
+EXPORT_INDIRECT_CALLABLE(ipv6_gro_receive);
 
 static struct sk_buff *sit_ip6ip6_gro_receive(struct list_head *head,
 					      struct sk_buff *skb)
@@ -359,6 +360,7 @@ INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
 out:
 	return err;
 }
+EXPORT_INDIRECT_CALLABLE(ipv6_gro_complete);
 
 static int sit_gro_complete(struct sk_buff *skb, int nhoff)
 {
diff --git a/net/ipv6/ip6_udp_tunnel.c b/net/ipv6/ip6_udp_tunnel.c
index 405ef1cb88644..9adb5775487fa 100644
--- a/net/ipv6/ip6_udp_tunnel.c
+++ b/net/ipv6/ip6_udp_tunnel.c
@@ -118,7 +118,7 @@ EXPORT_SYMBOL_GPL(udp_tunnel6_xmit_skb);
  *      @skb: Packet for which lookup is done
  *      @dev: Tunnel device
  *      @net: Network namespace of tunnel device
- *      @sock: Socket which provides route info
+ *      @sk: Socket which provides route info
  *      @oif: Index of the output interface
  *      @saddr: Memory to store the src ip address
  *      @key: Tunnel information
@@ -135,7 +135,7 @@ EXPORT_SYMBOL_GPL(udp_tunnel6_xmit_skb);
 struct dst_entry *udp_tunnel6_dst_lookup(struct sk_buff *skb,
 					 struct net_device *dev,
 					 struct net *net,
-					 struct socket *sock,
+					 struct sock *sk,
 					 int oif,
 					 struct in6_addr *saddr,
 					 const struct ip_tunnel_key *key,
@@ -162,7 +162,7 @@ struct dst_entry *udp_tunnel6_dst_lookup(struct sk_buff *skb,
 	fl6.fl6_dport = dport;
 	fl6.flowlabel = ip6_make_flowinfo(dsfield, key->label);
 
-	dst = ip6_dst_lookup_flow(net, sock->sk, &fl6, NULL);
+	dst = ip6_dst_lookup_flow(net, sk, &fl6, NULL);
 	if (IS_ERR(dst)) {
 		netdev_dbg(dev, "no route to %pI6\n", &fl6.daddr);
 		return ERR_PTR(-ENETUNREACH);
diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c
index 85010ff21c981..9d748f4c44e97 100644
--- a/net/ipv6/ip6mr.c
+++ b/net/ipv6/ip6mr.c
@@ -1063,10 +1063,14 @@ static void ip6mr_cache_resolve(struct net *net, struct mr_table *mrt,
 static int ip6mr_cache_report(const struct mr_table *mrt, struct sk_buff *pkt,
 			      mifi_t mifi, int assert)
 {
+	enum skb_drop_reason reason;
 	struct sock *mroute6_sk;
 	struct sk_buff *skb;
 	struct mrt6msg *msg;
-	int ret;
+
+	mroute6_sk = rcu_dereference(mrt->mroute_sk);
+	if (!mroute6_sk)
+		return -EINVAL;
 
 #ifdef CONFIG_IPV6_PIMSM_V2
 	if (assert == MRT6MSG_WHOLEPKT || assert == MRT6MSG_WRMIFWHOLE)
@@ -1136,23 +1140,17 @@ static int ip6mr_cache_report(const struct mr_table *mrt, struct sk_buff *pkt,
 	skb->ip_summed = CHECKSUM_UNNECESSARY;
 	}
 
-	mroute6_sk = rcu_dereference(mrt->mroute_sk);
-	if (!mroute6_sk) {
-		kfree_skb(skb);
-		return -EINVAL;
-	}
-
 	mrt6msg_netlink_event(mrt, skb);
 
 	/* Deliver to user space multicast routing algorithms */
-	ret = sock_queue_rcv_skb(mroute6_sk, skb);
+	reason = sock_queue_rcv_skb_reason(mroute6_sk, skb);
 
-	if (ret < 0) {
-		net_warn_ratelimited("mroute6: pending queue full, dropping entries\n");
-		kfree_skb(skb);
+	if (reason) {
+		sk_skb_reason_drop(mroute6_sk, skb, reason);
+		return -ENOMEM;
 	}
 
-	return ret;
+	return 0;
 }
 
 /* Queue a packet for resolution. It gets locked cache entry! */
diff --git a/net/ipv6/mcast.c b/net/ipv6/mcast.c
index 3330adcf26db2..6ddc18ac59b95 100644
--- a/net/ipv6/mcast.c
+++ b/net/ipv6/mcast.c
@@ -1315,20 +1315,7 @@ static void mld_update_qi(struct inet6_dev *idev,
 	 *  - 9.12. Older Version Querier Present Timeout
 	 *    (the [Query Interval] in the last Query received)
 	 */
-	unsigned long mc_qqi;
-
-	if (mlh2->mld2q_qqic < 128) {
-		mc_qqi = mlh2->mld2q_qqic;
-	} else {
-		unsigned long mc_man, mc_exp;
-
-		mc_exp = MLDV2_QQIC_EXP(mlh2->mld2q_qqic);
-		mc_man = MLDV2_QQIC_MAN(mlh2->mld2q_qqic);
-
-		mc_qqi = (mc_man | 0x10) << (mc_exp + 3);
-	}
-
-	idev->mc_qi = mc_qqi * HZ;
+	idev->mc_qi = mldv2_qqi(mlh2) * HZ;
 }
 
 static void mld_update_qri(struct inet6_dev *idev,
@@ -1338,7 +1325,7 @@ static void mld_update_qri(struct inet6_dev *idev,
 	 *  - 5.1.3. Maximum Response Code
 	 *  - 9.3. Query Response Interval
 	 */
-	idev->mc_qri = msecs_to_jiffies(mldv2_mrc(mlh2));
+	idev->mc_qri = msecs_to_jiffies(mldv2_mrd(mlh2));
 }
 
 static int mld_process_v1(struct inet6_dev *idev, struct mld_msg *mld,
@@ -1390,7 +1377,7 @@ static int mld_process_v1(struct inet6_dev *idev, struct mld_msg *mld,
 static void mld_process_v2(struct inet6_dev *idev, struct mld2_query *mld,
 			   unsigned long *max_delay)
 {
-	*max_delay = max(msecs_to_jiffies(mldv2_mrc(mld)), 1UL);
+	*max_delay = max(msecs_to_jiffies(mldv2_mrd(mld)), 1UL);
 
 	mld_update_qrv(idev, mld);
 	mld_update_qi(idev, mld);
diff --git a/net/ipv6/netfilter/ip6t_hbh.c b/net/ipv6/netfilter/ip6t_hbh.c
index e7a3fb9355ee3..450dd53846a2f 100644
--- a/net/ipv6/netfilter/ip6t_hbh.c
+++ b/net/ipv6/netfilter/ip6t_hbh.c
@@ -168,6 +168,10 @@ static int hbh_mt6_check(const struct xt_mtchk_param *par)
 		pr_debug("unknown flags %X\n", optsinfo->invflags);
 		return -EINVAL;
 	}
+	if (optsinfo->optsnr > IP6T_OPTS_OPTSNR) {
+		pr_debug("too many supported opts specified\n");
+		return -EINVAL;
+	}
 
 	if (optsinfo->flags & IP6T_OPTS_NSTRICT) {
 		pr_debug("Not strict - not implemented");
diff --git a/net/ipv6/tcp_ao.c b/net/ipv6/tcp_ao.c
index 3c09ac26206e3..01a8472805d1d 100644
--- a/net/ipv6/tcp_ao.c
+++ b/net/ipv6/tcp_ao.c
@@ -7,84 +7,73 @@
  *		Francesco Ruggeri <fruggeri@arista.com>
  *		Salam Noureddine <noureddine@arista.com>
  */
-#include <crypto/hash.h>
 #include <linux/tcp.h>
 
 #include <net/tcp.h>
 #include <net/ipv6.h>
 
-static int tcp_v6_ao_calc_key(struct tcp_ao_key *mkt, u8 *key,
-			      const struct in6_addr *saddr,
-			      const struct in6_addr *daddr,
-			      __be16 sport, __be16 dport,
-			      __be32 sisn, __be32 disn)
+static void tcp_v6_ao_calc_key(struct tcp_ao_key *mkt, u8 *key,
+			       const struct in6_addr *saddr,
+			       const struct in6_addr *daddr,
+			       __be16 sport, __be16 dport,
+			       __be32 sisn, __be32 disn)
 {
 	struct kdf_input_block {
 		u8			counter;
 		u8			label[6];
 		struct tcp6_ao_context	ctx;
 		__be16			outlen;
-	} __packed * tmp;
-	struct tcp_sigpool hp;
-	int err;
-
-	err = tcp_sigpool_start(mkt->tcp_sigpool_id, &hp);
-	if (err)
-		return err;
-
-	tmp = hp.scratch;
-	tmp->counter	= 1;
-	memcpy(tmp->label, "TCP-AO", 6);
-	tmp->ctx.saddr	= *saddr;
-	tmp->ctx.daddr	= *daddr;
-	tmp->ctx.sport	= sport;
-	tmp->ctx.dport	= dport;
-	tmp->ctx.sisn	= sisn;
-	tmp->ctx.disn	= disn;
-	tmp->outlen	= htons(tcp_ao_digest_size(mkt) * 8); /* in bits */
-
-	err = tcp_ao_calc_traffic_key(mkt, key, tmp, sizeof(*tmp), &hp);
-	tcp_sigpool_end(&hp);
-
-	return err;
+	} __packed input = {
+		.counter = 1,
+		.label = "TCP-AO",
+		.ctx = {
+			.saddr = *saddr,
+			.daddr = *daddr,
+			.sport = sport,
+			.dport = dport,
+			.sisn = sisn,
+			.disn = disn,
+		},
+		.outlen = htons(tcp_ao_digest_size(mkt) * 8), /* in bits */
+	};
+
+	tcp_ao_calc_traffic_key(mkt, key, &input, sizeof(input));
 }
 
-int tcp_v6_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
-			   const struct sk_buff *skb,
-			   __be32 sisn, __be32 disn)
+void tcp_v6_ao_calc_key_skb(struct tcp_ao_key *mkt, u8 *key,
+			    const struct sk_buff *skb, __be32 sisn, __be32 disn)
 {
 	const struct ipv6hdr *iph = ipv6_hdr(skb);
 	const struct tcphdr *th = tcp_hdr(skb);
 
-	return tcp_v6_ao_calc_key(mkt, key, &iph->saddr,
-				  &iph->daddr, th->source,
-				  th->dest, sisn, disn);
+	tcp_v6_ao_calc_key(mkt, key, &iph->saddr, &iph->daddr, th->source,
+			   th->dest, sisn, disn);
 }
 
-int tcp_v6_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
-			  const struct sock *sk, __be32 sisn,
-			  __be32 disn, bool send)
+void tcp_v6_ao_calc_key_sk(struct tcp_ao_key *mkt, u8 *key,
+			   const struct sock *sk, __be32 sisn,
+			   __be32 disn, bool send)
 {
 	if (send)
-		return tcp_v6_ao_calc_key(mkt, key, &sk->sk_v6_rcv_saddr,
-					  &sk->sk_v6_daddr, htons(sk->sk_num),
-					  sk->sk_dport, sisn, disn);
+		tcp_v6_ao_calc_key(mkt, key, &sk->sk_v6_rcv_saddr,
+				   &sk->sk_v6_daddr, htons(sk->sk_num),
+				   sk->sk_dport, sisn, disn);
 	else
-		return tcp_v6_ao_calc_key(mkt, key, &sk->sk_v6_daddr,
-					  &sk->sk_v6_rcv_saddr, sk->sk_dport,
-					  htons(sk->sk_num), disn, sisn);
+		tcp_v6_ao_calc_key(mkt, key, &sk->sk_v6_daddr,
+				   &sk->sk_v6_rcv_saddr, sk->sk_dport,
+				   htons(sk->sk_num), disn, sisn);
 }
 
-int tcp_v6_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
-			   struct request_sock *req)
+void tcp_v6_ao_calc_key_rsk(struct tcp_ao_key *mkt, u8 *key,
+			    struct request_sock *req)
 {
 	struct inet_request_sock *ireq = inet_rsk(req);
 
-	return tcp_v6_ao_calc_key(mkt, key,
-			&ireq->ir_v6_loc_addr, &ireq->ir_v6_rmt_addr,
-			htons(ireq->ir_num), ireq->ir_rmt_port,
-			htonl(tcp_rsk(req)->snt_isn),
-			htonl(tcp_rsk(req)->rcv_isn));
+	tcp_v6_ao_calc_key(mkt, key,
+			   &ireq->ir_v6_loc_addr, &ireq->ir_v6_rmt_addr,
+			   htons(ireq->ir_num), ireq->ir_rmt_port,
+			   htonl(tcp_rsk(req)->snt_isn),
+			   htonl(tcp_rsk(req)->rcv_isn));
 }
 
 struct tcp_ao_key *tcp_v6_ao_lookup(const struct sock *sk,
@@ -112,23 +101,19 @@ struct tcp_ao_key *tcp_v6_ao_lookup_rsk(const struct sock *sk,
 				AF_INET6, sndid, rcvid);
 }
 
-int tcp_v6_ao_hash_pseudoheader(struct tcp_sigpool *hp,
-				const struct in6_addr *daddr,
-				const struct in6_addr *saddr, int nbytes)
+void tcp_v6_ao_hash_pseudoheader(struct tcp_ao_mac_ctx *mac_ctx,
+				 const struct in6_addr *daddr,
+				 const struct in6_addr *saddr, int nbytes)
 {
-	struct tcp6_pseudohdr *bp;
-	struct scatterlist sg;
-
-	bp = hp->scratch;
 	/* 1. TCP pseudo-header (RFC2460) */
-	bp->saddr = *saddr;
-	bp->daddr = *daddr;
-	bp->len = cpu_to_be32(nbytes);
-	bp->protocol = cpu_to_be32(IPPROTO_TCP);
-
-	sg_init_one(&sg, bp, sizeof(*bp));
-	ahash_request_set_crypt(hp->req, &sg, NULL, sizeof(*bp));
-	return crypto_ahash_update(hp->req);
+	struct tcp6_pseudohdr phdr = {
+		.saddr = *saddr,
+		.daddr = *daddr,
+		.len = cpu_to_be32(nbytes),
+		.protocol = cpu_to_be32(IPPROTO_TCP),
+	};
+
+	tcp_ao_mac_update(mac_ctx, &phdr, sizeof(phdr));
 }
 
 int tcp_v6_ao_hash_skb(char *ao_hash, struct tcp_ao_key *key,
@@ -149,20 +134,10 @@ int tcp_v6_ao_synack_hash(char *ao_hash, struct tcp_ao_key *ao_key,
 			  struct request_sock *req, const struct sk_buff *skb,
 			  int hash_offset, u32 sne)
 {
-	void *hash_buf = NULL;
-	int err;
-
-	hash_buf = kmalloc(tcp_ao_digest_size(ao_key), GFP_ATOMIC);
-	if (!hash_buf)
-		return -ENOMEM;
-
-	err = tcp_v6_ao_calc_key_rsk(ao_key, hash_buf, req);
-	if (err)
-		goto out;
-
-	err = tcp_ao_hash_skb(AF_INET6, ao_hash, ao_key, req_to_sk(req), skb,
-			      hash_buf, hash_offset, sne);
-out:
-	kfree(hash_buf);
-	return err;
+	u8 tkey_buf[TCP_AO_MAX_TRAFFIC_KEY_LEN];
+
+	tcp_v6_ao_calc_key_rsk(ao_key, tkey_buf, req);
+
+	return tcp_ao_hash_skb(AF_INET6, ao_hash, ao_key, req_to_sk(req), skb,
+			       tkey_buf, hash_offset, sne);
 }
diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c
index d13d49bfef194..c219f33348353 100644
--- a/net/ipv6/tcp_ipv6.c
+++ b/net/ipv6/tcp_ipv6.c
@@ -982,7 +982,7 @@ static void tcp_v6_send_response(const struct sock *sk, struct sk_buff *skb, u32
 		return;
 	}
 
-	kfree_skb(buff);
+	sk_skb_reason_drop(sk, buff, SKB_DROP_REASON_IP_OUTNOROUTES);
 }
 
 static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb,
@@ -1839,6 +1839,7 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
 		}
 	}
 
+	isn = 0;
 process:
 	if (static_branch_unlikely(&ip6_min_hopcount)) {
 		/* min_hopcount can be changed concurrently from do_ipv6_setsockopt() */
@@ -1868,6 +1869,7 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
 	th = (const struct tcphdr *)skb->data;
 	hdr = ipv6_hdr(skb);
 	tcp_v6_fill_cb(skb, hdr, th);
+	TCP_SKB_CB(skb)->tcp_tw_isn = isn;
 
 	skb->dev = NULL;
 
@@ -1956,7 +1958,6 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
 			sk = sk2;
 			tcp_v6_restore_cb(skb);
 			refcounted = false;
-			__this_cpu_write(tcp_tw_isn, isn);
 			goto process;
 		}
 
diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c
index 3912e75079f5e..b273213cc68d1 100644
--- a/net/kcm/kcmsock.c
+++ b/net/kcm/kcmsock.c
@@ -24,6 +24,7 @@
 #include <linux/workqueue.h>
 #include <linux/syscalls.h>
 #include <linux/sched/signal.h>
+#include <linux/uio.h>
 
 #include <net/kcm.h>
 #include <net/netns/generic.h>
@@ -1167,7 +1168,7 @@ static int kcm_setsockopt(struct socket *sock, int level, int optname,
 }
 
 static int kcm_getsockopt(struct socket *sock, int level, int optname,
-			  char __user *optval, int __user *optlen)
+			  sockopt_t *opt)
 {
 	struct kcm_sock *kcm = kcm_sk(sock->sk);
 	int val, len;
@@ -1175,9 +1176,7 @@ static int kcm_getsockopt(struct socket *sock, int level, int optname,
 	if (level != SOL_KCM)
 		return -ENOPROTOOPT;
 
-	if (get_user(len, optlen))
-		return -EFAULT;
-
+	len = opt->optlen;
 	if (len < 0)
 		return -EINVAL;
 
@@ -1191,9 +1190,8 @@ static int kcm_getsockopt(struct socket *sock, int level, int optname,
 		return -ENOPROTOOPT;
 	}
 
-	if (put_user(len, optlen))
-		return -EFAULT;
-	if (copy_to_user(optval, &val, len))
+	opt->optlen = len;
+	if (copy_to_iter(&val, len, &opt->iter_out) != len)
 		return -EFAULT;
 	return 0;
 }
@@ -1755,7 +1753,7 @@ static const struct proto_ops kcm_dgram_ops = {
 	.listen =	sock_no_listen,
 	.shutdown =	sock_no_shutdown,
 	.setsockopt =	kcm_setsockopt,
-	.getsockopt =	kcm_getsockopt,
+	.getsockopt_iter = kcm_getsockopt,
 	.sendmsg =	kcm_sendmsg,
 	.recvmsg =	kcm_recvmsg,
 	.mmap =		sock_no_mmap,
@@ -1776,7 +1774,7 @@ static const struct proto_ops kcm_seqpacket_ops = {
 	.listen =	sock_no_listen,
 	.shutdown =	sock_no_shutdown,
 	.setsockopt =	kcm_setsockopt,
-	.getsockopt =	kcm_getsockopt,
+	.getsockopt_iter = kcm_getsockopt,
 	.sendmsg =	kcm_sendmsg,
 	.recvmsg =	kcm_recvmsg,
 	.mmap =		sock_no_mmap,
diff --git a/net/key/af_key.c b/net/key/af_key.c
index a166a88d87880..9cffeef18cd91 100644
--- a/net/key/af_key.c
+++ b/net/key/af_key.c
@@ -3564,7 +3564,7 @@ static int set_ipsecrequest(struct sk_buff *skb,
 #ifdef CONFIG_NET_KEY_MIGRATE
 static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 			      const struct xfrm_migrate *m, int num_bundles,
-			      const struct xfrm_kmaddress *k,
+			      const struct xfrm_kmaddress *k, struct net *net,
 			      const struct xfrm_encap_tmpl *encap)
 {
 	int i;
@@ -3669,7 +3669,7 @@ static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 	}
 
 	/* broadcast migrate message to sockets */
-	pfkey_broadcast(skb, GFP_ATOMIC, BROADCAST_ALL, NULL, &init_net);
+	pfkey_broadcast(skb, GFP_ATOMIC, BROADCAST_ALL, NULL, net);
 
 	return 0;
 
@@ -3680,7 +3680,7 @@ static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 #else
 static int pfkey_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 			      const struct xfrm_migrate *m, int num_bundles,
-			      const struct xfrm_kmaddress *k,
+			      const struct xfrm_kmaddress *k, struct net *net,
 			      const struct xfrm_encap_tmpl *encap)
 {
 	return -ENOPROTOOPT;
diff --git a/net/l2tp/l2tp_core.c b/net/l2tp/l2tp_core.c
index 157fc23ce4e14..cbc5a3e57b33c 100644
--- a/net/l2tp/l2tp_core.c
+++ b/net/l2tp/l2tp_core.c
@@ -1683,7 +1683,7 @@ int l2tp_tunnel_register(struct l2tp_tunnel *tunnel, struct net *net,
 			.encap_destroy = l2tp_udp_encap_destroy,
 		};
 
-		setup_udp_tunnel_sock(net, sock, &udp_cfg);
+		setup_udp_tunnel_sock(net, sock->sk, &udp_cfg);
 	}
 
 	sk->sk_allocation = GFP_ATOMIC;
diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c
index 1b210db3119e8..35278c519a305 100644
--- a/net/llc/af_llc.c
+++ b/net/llc/af_llc.c
@@ -27,6 +27,7 @@
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/sched/signal.h>
+#include <linux/uio.h>
 
 #include <net/llc.h>
 #include <net/llc_sap.h>
@@ -1166,25 +1167,21 @@ static int llc_ui_setsockopt(struct socket *sock, int level, int optname,
  *	@sock: Socket to get information from.
  *	@level: Socket level user is requesting operations on.
  *	@optname: Operation name.
- *	@optval: Variable to return operation data in.
- *	@optlen: Length of optval.
+ *	@opt: sockopt context with iterator and length for returning data.
  *
  *	Get connection specific socket information.
  */
 static int llc_ui_getsockopt(struct socket *sock, int level, int optname,
-			     char __user *optval, int __user *optlen)
+			     sockopt_t *opt)
 {
 	struct sock *sk = sock->sk;
 	struct llc_sock *llc = llc_sk(sk);
-	int val = 0, len = 0, rc = -EINVAL;
+	int val = 0, len, rc = -EINVAL;
 
 	lock_sock(sk);
 	if (unlikely(level != SOL_LLC))
 		goto out;
-	rc = get_user(len, optlen);
-	if (rc)
-		goto out;
-	rc = -EINVAL;
+	len = opt->optlen;
 	if (len != sizeof(int))
 		goto out;
 	switch (optname) {
@@ -1212,7 +1209,8 @@ static int llc_ui_getsockopt(struct socket *sock, int level, int optname,
 		goto out;
 	}
 	rc = 0;
-	if (put_user(len, optlen) || copy_to_user(optval, &val, len))
+	opt->optlen = len;
+	if (copy_to_iter(&val, len, &opt->iter_out) != len)
 		rc = -EFAULT;
 out:
 	release_sock(sk);
@@ -1239,7 +1237,7 @@ static const struct proto_ops llc_ui_ops = {
 	.listen      = llc_ui_listen,
 	.shutdown    = llc_ui_shutdown,
 	.setsockopt  = llc_ui_setsockopt,
-	.getsockopt  = llc_ui_getsockopt,
+	.getsockopt_iter = llc_ui_getsockopt,
 	.sendmsg     = llc_ui_sendmsg,
 	.recvmsg     = llc_ui_recvmsg,
 	.mmap	     = sock_no_mmap,
diff --git a/net/llc/llc_c_ac.c b/net/llc/llc_c_ac.c
index 0779daa8aa8f3..ab86c720b3ecf 100644
--- a/net/llc/llc_c_ac.c
+++ b/net/llc/llc_c_ac.c
@@ -1005,8 +1005,8 @@ static int llc_conn_ac_inc_npta_value(struct sock *sk, struct sk_buff *skb)
 		llc->dec_step = 0;
 		llc->dec_cntr = llc->inc_cntr = 2;
 		++llc->npta;
-		if (llc->npta > (u8) ~LLC_2_SEQ_NBR_MODULO)
-			llc->npta = (u8) ~LLC_2_SEQ_NBR_MODULO;
+		if (llc->npta > LLC_2_SEQ_NBR_MODULO - 1)
+			llc->npta = LLC_2_SEQ_NBR_MODULO - 1;
 	} else
 		--llc->inc_cntr;
 	return 0;
@@ -1096,8 +1096,8 @@ int llc_conn_ac_inc_tx_win_size(struct sock *sk, struct sk_buff *skb)
 	struct llc_sock *llc = llc_sk(sk);
 
 	llc->k += 1;
-	if (llc->k > (u8) ~LLC_2_SEQ_NBR_MODULO)
-		llc->k = (u8) ~LLC_2_SEQ_NBR_MODULO;
+	if (llc->k > LLC_2_SEQ_NBR_MODULO - 1)
+		llc->k = LLC_2_SEQ_NBR_MODULO - 1;
 	return 0;
 }
 
diff --git a/net/mac80211/Makefile b/net/mac80211/Makefile
index 20c3135b73ea6..36f1e3e2222b4 100644
--- a/net/mac80211/Makefile
+++ b/net/mac80211/Makefile
@@ -30,7 +30,7 @@ mac80211-y := \
 	key.o \
 	util.o parse.o \
 	wme.o \
-	chan.o \
+	chan.o ap.o \
 	trace.o mlme.o \
 	tdls.o \
 	ocb.o \
diff --git a/net/mac80211/ap.c b/net/mac80211/ap.c
new file mode 100644
index 0000000000000..6c7d2d51a372a
--- /dev/null
+++ b/net/mac80211/ap.c
@@ -0,0 +1,207 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * AP handling
+ *
+ * Partially
+ * Copyright (C) 2026 Intel Corporation
+ */
+
+#include "driver-ops.h"
+#include "ieee80211_i.h"
+
+static void
+ieee80211_send_eml_op_mode_notif(struct ieee80211_sub_if_data *sdata,
+				 struct ieee80211_mgmt *req, int opt_len)
+{
+	int len = IEEE80211_MIN_ACTION_SIZE(eml_omn);
+	struct ieee80211_local *local = sdata->local;
+	struct ieee80211_mgmt *mgmt;
+	struct sk_buff *skb;
+
+	len += opt_len; /* optional len */
+	skb = dev_alloc_skb(local->tx_headroom + len);
+	if (!skb)
+		return;
+
+	skb_reserve(skb, local->tx_headroom);
+	mgmt = skb_put_zero(skb, len);
+	mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
+					  IEEE80211_STYPE_ACTION);
+	memcpy(mgmt->da, req->sa, ETH_ALEN);
+	memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN);
+	memcpy(mgmt->bssid, sdata->vif.addr, ETH_ALEN);
+
+	mgmt->u.action.category = WLAN_CATEGORY_PROTECTED_EHT;
+	mgmt->u.action.action_code =
+		WLAN_PROTECTED_EHT_ACTION_EML_OP_MODE_NOTIF;
+	mgmt->u.action.eml_omn.dialog_token =
+		req->u.action.eml_omn.dialog_token;
+	mgmt->u.action.eml_omn.control = req->u.action.eml_omn.control &
+		~(IEEE80211_EML_CTRL_EMLSR_PARAM_UPDATE |
+		  IEEE80211_EML_CTRL_INDEV_COEX_ACT);
+	/* Copy optional fields from the received notification frame */
+	memcpy(mgmt->u.action.eml_omn.variable,
+	       req->u.action.eml_omn.variable, opt_len);
+
+	ieee80211_tx_skb(sdata, skb);
+}
+
+static void
+ieee80211_rx_eml_op_mode_notif(struct ieee80211_sub_if_data *sdata,
+			       struct sk_buff *skb)
+{
+	int len = IEEE80211_MIN_ACTION_SIZE(eml_omn);
+	enum nl80211_iftype type = ieee80211_vif_type_p2p(&sdata->vif);
+	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
+	const struct wiphy_iftype_ext_capab *ift_ext_capa;
+	struct ieee80211_mgmt *mgmt = (void *)skb->data;
+	struct ieee80211_local *local = sdata->local;
+	u8 control = mgmt->u.action.eml_omn.control;
+	u8 *ptr = mgmt->u.action.eml_omn.variable;
+	struct ieee80211_eml_params eml_params = {
+		.link_id = status->link_id,
+		.control = control,
+	};
+	struct sta_info *sta;
+	int opt_len = 0;
+
+	if (!ieee80211_vif_is_mld(&sdata->vif))
+		return;
+
+	/* eMLSR and eMLMR can't be enabled at the same time */
+	if ((control & IEEE80211_EML_CTRL_EMLSR_MODE) &&
+	    (control & IEEE80211_EML_CTRL_EMLMR_MODE))
+		return;
+
+	if ((control & IEEE80211_EML_CTRL_EMLMR_MODE) &&
+	    (control & IEEE80211_EML_CTRL_EMLSR_PARAM_UPDATE))
+		return;
+
+	ift_ext_capa = cfg80211_get_iftype_ext_capa(local->hw.wiphy, type);
+	if (!ift_ext_capa)
+		return;
+
+	if (!status->link_valid)
+		return;
+
+	sta = sta_info_get_bss(sdata, mgmt->sa);
+	if (!sta)
+		return;
+
+	if (control & IEEE80211_EML_CTRL_EMLSR_MODE) {
+		u8 emlsr_param_update_len;
+
+		if (!(ift_ext_capa->eml_capabilities &
+		      IEEE80211_EML_CAP_EMLSR_SUPP))
+			return;
+
+		opt_len += sizeof(__le16); /* eMLSR link_bitmap */
+		/*
+		 * eMLSR param update field is not part of Notification frame
+		 * sent by the AP to client so account it separately.
+		 */
+		emlsr_param_update_len =
+			!!(control & IEEE80211_EML_CTRL_EMLSR_PARAM_UPDATE);
+
+		if (skb->len < len + opt_len + emlsr_param_update_len)
+			return;
+
+		if (control & IEEE80211_EML_CTRL_EMLSR_PARAM_UPDATE) {
+			u8 pad_delay, trans_delay;
+
+			pad_delay = u8_get_bits(ptr[2],
+						IEEE80211_EML_EMLSR_PAD_DELAY);
+			if (pad_delay >
+			    IEEE80211_EML_CAP_EML_PADDING_DELAY_256US)
+				return;
+
+			trans_delay = u8_get_bits(ptr[2],
+					IEEE80211_EML_EMLSR_TRANS_DELAY);
+			if (trans_delay >
+			    IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US)
+				return;
+
+			/* Update sta padding and transition delay */
+			sta->sta.eml_cap =
+				u8_replace_bits(sta->sta.eml_cap,
+						pad_delay,
+						IEEE80211_EML_CAP_EML_PADDING_DELAY);
+			sta->sta.eml_cap =
+				u8_replace_bits(sta->sta.eml_cap,
+						trans_delay,
+						IEEE80211_EML_CAP_EML_TRANSITION_DELAY);
+		}
+	}
+
+	if (control & IEEE80211_EML_CTRL_EMLMR_MODE) {
+		u8 mcs_map_size;
+		int i;
+
+		if (!(ift_ext_capa->eml_capabilities &
+		      IEEE80211_EML_CAP_EMLMR_SUPPORT))
+			return;
+
+		opt_len += sizeof(__le16); /* eMLMR link_bitmap */
+		opt_len++; /* eMLMR mcs_map_count */
+		if (skb->len < len + opt_len)
+			return;
+
+		eml_params.emlmr_mcs_map_count = ptr[2];
+		if (eml_params.emlmr_mcs_map_count > 2)
+			return;
+
+		mcs_map_size = 3 * (1 + eml_params.emlmr_mcs_map_count);
+		opt_len += mcs_map_size;
+		if (skb->len < len + opt_len)
+			return;
+
+		for (i = 0; i < mcs_map_size; i++) {
+			u8 rx_mcs, tx_mcs;
+
+			rx_mcs = u8_get_bits(ptr[3 + i],
+					     IEEE80211_EML_EMLMR_RX_MCS_MAP);
+			if (rx_mcs > 8)
+				return;
+
+			tx_mcs = u8_get_bits(ptr[3 + i],
+					     IEEE80211_EML_EMLMR_TX_MCS_MAP);
+			if (tx_mcs > 8)
+				return;
+		}
+
+		memcpy(eml_params.emlmr_mcs_map_bw, &ptr[3], mcs_map_size);
+	}
+
+	if ((control & IEEE80211_EML_CTRL_EMLSR_MODE) ||
+	    (control & IEEE80211_EML_CTRL_EMLMR_MODE)) {
+		eml_params.link_bitmap = get_unaligned_le16(ptr);
+		if ((eml_params.link_bitmap & sdata->vif.active_links) !=
+		    eml_params.link_bitmap)
+			return;
+	}
+
+	if (drv_set_eml_op_mode(sdata, &sta->sta, &eml_params))
+		return;
+
+	ieee80211_send_eml_op_mode_notif(sdata, mgmt, opt_len);
+}
+
+void ieee80211_ap_rx_queued_frame(struct ieee80211_sub_if_data *sdata,
+				  struct sk_buff *skb)
+{
+	struct ieee80211_mgmt *mgmt = (void *)skb->data;
+
+	/* rx.c cannot queue any non-action frames to AP interfaces */
+	if (WARN_ON(!ieee80211_is_action(mgmt->frame_control)))
+		return;
+
+	switch (mgmt->u.action.category) {
+	case WLAN_CATEGORY_PROTECTED_EHT:
+		switch (mgmt->u.action.action_code) {
+		case WLAN_PROTECTED_EHT_ACTION_EML_OP_MODE_NOTIF:
+			ieee80211_rx_eml_op_mode_notif(sdata, skb);
+			break;
+		}
+		break;
+	}
+}
diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c
index 7b77d57c9f96f..fb4c1c2981596 100644
--- a/net/mac80211/cfg.c
+++ b/net/mac80211/cfg.c
@@ -716,6 +716,7 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct wireless_dev *wdev,
 	case NL80211_IFTYPE_WDS:
 	case NL80211_IFTYPE_MONITOR:
 	case NL80211_IFTYPE_P2P_DEVICE:
+	case NL80211_IFTYPE_PD:
 	case NL80211_IFTYPE_UNSPECIFIED:
 	case NUM_NL80211_IFTYPES:
 	case NL80211_IFTYPE_P2P_CLIENT:
@@ -1312,6 +1313,120 @@ ieee80211_copy_rnr_beacon(u8 *pos, struct cfg80211_rnr_elems *dst,
 	return offset;
 }
 
+static enum ieee80211_sta_rx_bandwidth
+ieee80211_calc_ap_he_and_lower(struct cfg80211_beacon_data *params)
+{
+	const struct ieee80211_vht_operation *vht_oper = params->vht_oper;
+	int ccfs0, ccfs1;
+
+	if (params->he_oper) {
+		const struct ieee80211_he_6ghz_oper *he_6ghz_oper;
+
+		if (params->he_oper->he_oper_params &
+				cpu_to_le32(IEEE80211_HE_OPERATION_VHT_OPER_INFO))
+			vht_oper = (void *)params->he_oper->optional;
+
+		he_6ghz_oper = ieee80211_he_6ghz_oper(params->he_oper);
+		if (he_6ghz_oper) {
+			switch (u8_get_bits(he_6ghz_oper->control,
+					    IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH)) {
+			case IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_20MHZ:
+				return IEEE80211_STA_RX_BW_20;
+			case IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_40MHZ:
+				return IEEE80211_STA_RX_BW_40;
+			case IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_80MHZ:
+				return IEEE80211_STA_RX_BW_80;
+			case IEEE80211_HE_6GHZ_OPER_CTRL_CHANWIDTH_160MHZ:
+				return IEEE80211_STA_RX_BW_160;
+			}
+		}
+	}
+
+	if (vht_oper) {
+		switch (vht_oper->chan_width) {
+		case IEEE80211_VHT_CHANWIDTH_USE_HT:
+			/* check for HT (or fall down to 20) below */
+			break;
+		case IEEE80211_VHT_CHANWIDTH_160MHZ:
+		case IEEE80211_VHT_CHANWIDTH_80P80MHZ:
+			/* deprecated encodings */
+			return IEEE80211_STA_RX_BW_160;
+		case IEEE80211_VHT_CHANWIDTH_80MHZ:
+			/*
+			 * See IEEE 802.11-2020 Table 9-352-BSS bandwidth
+			 * when the VHT Operation Information field Channel
+			 * Width subfield is 1
+			 *
+			 * (IEEE80211_VHT_CHANWIDTH_80MHZ == 1)
+			 */
+			ccfs0 = vht_oper->center_freq_seg0_idx;
+			ccfs1 = vht_oper->center_freq_seg1_idx;
+			if (!ccfs0)
+				return IEEE80211_STA_RX_BW_80;
+			if (ccfs1 && abs(ccfs1 - ccfs0) == 8)
+				return IEEE80211_STA_RX_BW_160;
+			/* 80+80 - RX BW doesn't cover that / uses 160 */
+			if (ccfs1 && abs(ccfs1 - ccfs0) > 16)
+				return IEEE80211_STA_RX_BW_160;
+			fallthrough;
+		default:
+			/* reserved encoding - assume 80 */
+			return IEEE80211_STA_RX_BW_80;
+		}
+	}
+
+	if (params->ht_oper) {
+		switch (u8_get_bits(params->ht_oper->ht_param,
+				    IEEE80211_HT_PARAM_CHA_SEC_OFFSET)) {
+		case IEEE80211_HT_PARAM_CHA_SEC_NONE:
+		default: /* invalid values */
+			return IEEE80211_STA_RX_BW_20;
+		case IEEE80211_HT_PARAM_CHA_SEC_ABOVE:
+		case IEEE80211_HT_PARAM_CHA_SEC_BELOW:
+			return IEEE80211_STA_RX_BW_40;
+		}
+	}
+
+	/* nothing found, must be 20 MHz */
+	return IEEE80211_STA_RX_BW_20;
+}
+
+static void ieee80211_update_ap_bandwidth(struct ieee80211_link_data *link,
+					  struct cfg80211_beacon_data *params)
+{
+	struct ieee80211_local *local = link->sdata->local;
+	struct ieee80211_chanctx_conf *chanctx_conf;
+	struct ieee80211_chanctx *chanctx;
+
+	/*
+	 * Updating the beacon might, without even changing the channel, cause
+	 * the usable bandwidth for some stations to be changed, for example
+	 * if the beacon configuration is EHT with 160 MHz, HE could change
+	 * between 20, 40, 80 and 160 MHz, and HE (or lower) clients need to
+	 * be handled accordingly.
+	 * Calculate the HE and lower bandwidth and apply that to all stations.
+	 *
+	 * In the future, this also needs to calculate EHT bandwidth and apply
+	 * it to all stations not using UHR DBE, since the chandef would then
+	 * include DBE.
+	 */
+
+	if (link->conf->chanreq.oper.chan->band == NL80211_BAND_S1GHZ)
+		return;
+
+	link->bss_bw.he_and_lower = ieee80211_calc_ap_he_and_lower(params);
+
+	chanctx_conf = sdata_dereference(link->conf->chanctx_conf, link->sdata);
+	chanctx = container_of(chanctx_conf, struct ieee80211_chanctx, conf);
+
+	/*
+	 * Note: this relies on ieee80211_recalc_chanctx_min_def() having
+	 * the side effect of updating all stations, if they changed; that
+	 * was normally for when the chandef changed but is used here too.
+	 */
+	ieee80211_recalc_chanctx_min_def(local, chanctx);
+}
+
 static int
 ieee80211_assign_beacon(struct ieee80211_sub_if_data *sdata,
 			struct ieee80211_link_data *link,
@@ -1450,6 +1565,8 @@ ieee80211_assign_beacon(struct ieee80211_sub_if_data *sdata,
 	if (old)
 		kfree_rcu(old, rcu_head);
 
+	ieee80211_update_ap_bandwidth(link, params);
+
 	*changed |= _changed;
 	return 0;
 }
@@ -1496,7 +1613,10 @@ static int ieee80211_start_ap(struct wiphy *wiphy, struct net_device *dev,
 	unsigned int link_id = params->beacon.link_id;
 	struct ieee80211_link_data *link;
 	struct ieee80211_bss_conf *link_conf;
-	struct ieee80211_chan_req chanreq = { .oper = params->chandef };
+	struct ieee80211_chan_req chanreq = {
+		.oper = params->chandef,
+		.require_npca = true,
+	};
 	u64 tsf;
 
 	lockdep_assert_wiphy(local->hw.wiphy);
@@ -1541,13 +1661,13 @@ static int ieee80211_start_ap(struct wiphy *wiphy, struct net_device *dev,
 				cpu_to_le32(IEEE80211_VHT_CAP_MU_BEAMFORMEE_CAPABLE);
 	}
 
-	if (params->he_cap && params->he_oper) {
+	if (params->he_cap && params->beacon.he_oper) {
 		link_conf->he_support = true;
 		link_conf->htc_trig_based_pkt_ext =
-			le32_get_bits(params->he_oper->he_oper_params,
+			le32_get_bits(params->beacon.he_oper->he_oper_params,
 			      IEEE80211_HE_OPERATION_DFLT_PE_DURATION_MASK);
 		link_conf->frame_time_rts_th =
-			le32_get_bits(params->he_oper->he_oper_params,
+			le32_get_bits(params->beacon.he_oper->he_oper_params,
 			      IEEE80211_HE_OPERATION_RTS_THRESHOLD_MASK);
 		changed |= BSS_CHANGED_HE_OBSS_PD;
 
@@ -1596,7 +1716,7 @@ static int ieee80211_start_ap(struct wiphy *wiphy, struct net_device *dev,
 				 IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_160MHZ |
 				 IEEE80211_EHT_PHY_CAP7_NON_OFDMA_UL_MU_MIMO_320MHZ);
 		link_conf->eht_disable_mcs15 =
-			u8_get_bits(params->eht_oper->params,
+			u8_get_bits(params->beacon.eht_oper->params,
 				    IEEE80211_EHT_OPER_MCS15_DISABLE);
 	} else {
 		link_conf->eht_su_beamformer = false;
@@ -1604,11 +1724,43 @@ static int ieee80211_start_ap(struct wiphy *wiphy, struct net_device *dev,
 		link_conf->eht_mu_beamformer = false;
 	}
 
-	if (params->uhr_oper) {
+	if (params->beacon.uhr_oper) {
+		const struct ieee80211_uhr_npca_info *npca;
+		struct ieee80211_bss_npca_params npca_params = {};
+
 		if (!link_conf->eht_support)
 			return -EOPNOTSUPP;
 
 		link_conf->uhr_support = true;
+
+		npca = ieee80211_uhr_npca_info(params->beacon.uhr_oper);
+		if (!npca) {
+			chanreq.oper.npca_chan = NULL;
+			chanreq.oper.npca_punctured = 0;
+		} else {
+			npca_params.min_dur_thresh =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_MIN_DUR_THRESH);
+			npca_params.switch_delay =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_SWITCH_DELAY);
+			npca_params.switch_back_delay =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_SWITCH_BACK_DELAY);
+			npca_params.init_qsrc =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_INIT_QSRC);
+			npca_params.moplen =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_MOPLEN);
+			npca_params.enabled = true;
+		}
+
+		if (memcmp(&npca_params, &link->conf->npca,
+			   sizeof(npca_params))) {
+			link->conf->npca = npca_params;
+			changed |= BSS_CHANGED_NPCA;
+		}
 	}
 
 	if (sdata->vif.type == NL80211_IFTYPE_AP &&
@@ -1838,6 +1990,7 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev,
 	struct ieee80211_link_data *link =
 		sdata_dereference(sdata->link[link_id], sdata);
 	struct ieee80211_bss_conf *link_conf = link->conf;
+	u64 changes = BSS_CHANGED_BEACON_ENABLED;
 	LIST_HEAD(keys);
 
 	lockdep_assert_wiphy(local->hw.wiphy);
@@ -1887,6 +2040,11 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev,
 	if (old_s1g_short_beacon)
 		kfree_rcu(old_s1g_short_beacon, rcu_head);
 
+	if (link_conf->ftm_responder) {
+		link_conf->ftm_responder = false;
+		changes |= BSS_CHANGED_FTM_RESPONDER;
+	}
+
 	kfree(link_conf->ftmr_params);
 	link_conf->ftmr_params = NULL;
 
@@ -1908,8 +2066,7 @@ static int ieee80211_stop_ap(struct wiphy *wiphy, struct net_device *dev,
 	sdata->vif.cfg.ssid_len = 0;
 	sdata->vif.cfg.s1g = false;
 	clear_bit(SDATA_STATE_OFFCHANNEL_BEACON_STOPPED, &sdata->state);
-	ieee80211_link_info_change_notify(sdata, link,
-					  BSS_CHANGED_BEACON_ENABLED);
+	ieee80211_link_info_change_notify(sdata, link, changes);
 
 	ieee80211_remove_link_keys(link, &keys);
 	if (!list_empty(&keys)) {
@@ -2221,7 +2378,15 @@ static int sta_link_apply_parameters(struct ieee80211_local *local,
 		ieee80211_s1g_cap_to_sta_s1g_cap(sdata, params->s1g_capa,
 						 link_sta);
 
-	ieee80211_sta_init_nss(link_sta);
+	switch (sdata->vif.type) {
+	case NL80211_IFTYPE_NAN:
+	case NL80211_IFTYPE_NAN_DATA:
+		/* not applicable - they don't use NSS/BW as capability */
+		break;
+	default:
+		ieee80211_sta_init_nss_bw_capa(link_sta, &link->conf->chanreq.oper);
+		break;
+	}
 
 	if (params->opmode_notif_used) {
 		enum nl80211_chan_width width = link->conf->chanreq.oper.width;
@@ -2344,8 +2509,9 @@ static int sta_apply_parameters(struct ieee80211_local *local,
 		sta->sta.max_sp = params->max_sp;
 	}
 
-	ieee80211_sta_set_max_amsdu_subframes(sta, params->ext_capab,
-					      params->ext_capab_len);
+	if (params->ext_capab)
+		ieee80211_sta_set_max_amsdu_subframes(sta, params->ext_capab,
+						      params->ext_capab_len);
 
 	/*
 	 * cfg80211 validates this (1-2007) and allows setting the AID
@@ -3335,6 +3501,7 @@ static int ieee80211_scan(struct wiphy *wiphy,
 		}
 		break;
 	case NL80211_IFTYPE_NAN:
+	case NL80211_IFTYPE_PD:
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -4501,7 +4668,10 @@ __ieee80211_channel_switch(struct wiphy *wiphy, struct net_device *dev,
 			   struct cfg80211_csa_settings *params)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
-	struct ieee80211_chan_req chanreq = { .oper = params->chandef };
+	struct ieee80211_chan_req chanreq = {
+		.oper = params->chandef,
+		.require_npca = true,
+	};
 	struct ieee80211_local *local = sdata->local;
 	struct ieee80211_channel_switch ch_switch = {
 		.link_id = params->link_id,
@@ -4943,7 +5113,10 @@ static int ieee80211_set_ap_chanwidth(struct wiphy *wiphy,
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_DEV_TO_SUB_IF(dev);
 	struct ieee80211_link_data *link;
-	struct ieee80211_chan_req chanreq = { .oper = *chandef };
+	struct ieee80211_chan_req chanreq = {
+		.oper = *chandef,
+		.require_npca = true,
+	};
 	int ret;
 	u64 changed = 0;
 
@@ -5091,6 +5264,25 @@ void ieee80211_nan_func_match(struct ieee80211_vif *vif,
 }
 EXPORT_SYMBOL(ieee80211_nan_func_match);
 
+void ieee80211_nan_cluster_joined(struct ieee80211_vif *vif,
+				  const u8 *cluster_id, bool new_cluster,
+				  gfp_t gfp)
+{
+	struct ieee80211_sub_if_data *sdata = vif_to_sdata(vif);
+
+	if (WARN_ON(vif->type != NL80211_IFTYPE_NAN))
+		return;
+
+	if (WARN_ON(!sdata->u.nan.started))
+		return;
+
+	ether_addr_copy(sdata->u.nan.conf.cluster_id, cluster_id);
+
+	cfg80211_nan_cluster_joined(ieee80211_vif_to_wdev(vif), cluster_id,
+				    new_cluster, gfp);
+}
+EXPORT_SYMBOL(ieee80211_nan_cluster_joined);
+
 static int ieee80211_set_multicast_to_unicast(struct wiphy *wiphy,
 					      struct net_device *dev,
 					      const bool enabled)
diff --git a/net/mac80211/chan.c b/net/mac80211/chan.c
index fda692316f081..c64a991319549 100644
--- a/net/mac80211/chan.c
+++ b/net/mac80211/chan.c
@@ -285,19 +285,41 @@ ieee80211_chanreq_compatible(const struct ieee80211_chan_req *a,
 			     const struct ieee80211_chan_req *b,
 			     struct ieee80211_chan_req *tmp)
 {
+	struct ieee80211_chan_req _a = *a, _b = *b;
 	const struct cfg80211_chan_def *compat;
 
 	if (a->ap.chan && b->ap.chan &&
 	    !cfg80211_chandef_identical(&a->ap, &b->ap))
 		return NULL;
 
-	compat = cfg80211_chandef_compatible(&a->oper, &b->oper);
+	/*
+	 * Remove NPCA if it's not required, so that interfaces
+	 * sharing a channel context will not use NPCA while the
+	 * channel context is shared.
+	 * If both sides are AP interfaces requiring NPC, there's
+	 * an assumption that userspace will set them up with
+	 * identical configurations and the same BSS color
+	 * (if the config is not identical, sharing will fail due
+	 * to cfg80211_chandef_compatible() failing below.)
+	 */
+	if (!_a.require_npca) {
+		_a.oper.npca_chan = NULL;
+		_a.oper.npca_punctured = 0;
+	}
+
+	if (!_b.require_npca) {
+		_b.oper.npca_chan = NULL;
+		_b.oper.npca_punctured = 0;
+	}
+
+	compat = cfg80211_chandef_compatible(&_a.oper, &_b.oper);
 	if (!compat)
 		return NULL;
 
 	/* Note: later code assumes this always fills & returns tmp if compat */
 	tmp->oper = *compat;
 	tmp->ap = a->ap.chan ? a->ap : b->ap;
+	tmp->require_npca = a->require_npca && b->require_npca;
 	return tmp;
 }
 
@@ -438,11 +460,12 @@ ieee80211_find_reservation_chanctx(struct ieee80211_local *local,
 	return NULL;
 }
 
-static enum nl80211_chan_width ieee80211_get_sta_bw(struct sta_info *sta,
-						    unsigned int link_id)
+static enum nl80211_chan_width
+ieee80211_get_sta_bw(struct sta_info *sta, struct ieee80211_link_data *link)
 {
 	enum ieee80211_sta_rx_bandwidth width;
 	struct link_sta_info *link_sta;
+	int link_id = link->link_id;
 
 	link_sta = wiphy_dereference(sta->local->hw.wiphy, sta->link[link_id]);
 
@@ -454,45 +477,28 @@ static enum nl80211_chan_width ieee80211_get_sta_bw(struct sta_info *sta,
 	 * We assume that TX/RX might be asymmetric (so e.g. VHT operating
 	 * mode notification changes what a STA wants to receive, but not
 	 * necessarily what it will transmit to us), and therefore use the
-	 * capabilities here. Calling it RX bandwidth capability is a bit
-	 * wrong though, since capabilities are in fact symmetric.
+	 * "from station" bandwidth here.
 	 */
-	width = ieee80211_sta_cap_rx_bw(link_sta);
+	width = ieee80211_sta_current_bw(link_sta, &link->conf->chanreq.oper,
+					 IEEE80211_STA_BW_RX_FROM_STA);
 
-	switch (width) {
-	case IEEE80211_STA_RX_BW_20:
-		if (link_sta->pub->ht_cap.ht_supported)
-			return NL80211_CHAN_WIDTH_20;
-		else
-			return NL80211_CHAN_WIDTH_20_NOHT;
-	case IEEE80211_STA_RX_BW_40:
-		return NL80211_CHAN_WIDTH_40;
-	case IEEE80211_STA_RX_BW_80:
-		return NL80211_CHAN_WIDTH_80;
-	case IEEE80211_STA_RX_BW_160:
-		/*
-		 * This applied for both 160 and 80+80. since we use
-		 * the returned value to consider degradation of
-		 * ctx->conf.min_def, we have to make sure to take
-		 * the bigger one (NL80211_CHAN_WIDTH_160).
-		 * Otherwise we might try degrading even when not
-		 * needed, as the max required sta_bw returned (80+80)
-		 * might be smaller than the configured bw (160).
-		 */
-		return NL80211_CHAN_WIDTH_160;
-	case IEEE80211_STA_RX_BW_320:
-		return NL80211_CHAN_WIDTH_320;
-	default:
-		WARN_ON(1);
-		return NL80211_CHAN_WIDTH_20;
-	}
+	if (width == IEEE80211_STA_RX_BW_20 &&
+	    !link_sta->pub->ht_cap.ht_supported &&
+	    !link_sta->pub->he_cap.has_he)
+		return NL80211_CHAN_WIDTH_20_NOHT;
+
+	/*
+	 * This returns 160 for both 160 and 80+80. Since we use
+	 * the returned value to consider narrowing for
+	 * ctx->conf.min_def, that's correct and necessary.
+	 */
+	return ieee80211_sta_rx_bw_to_chan_width(width);
 }
 
 static enum nl80211_chan_width
 ieee80211_get_max_required_bw(struct ieee80211_link_data *link)
 {
 	struct ieee80211_sub_if_data *sdata = link->sdata;
-	unsigned int link_id = link->link_id;
 	enum nl80211_chan_width max_bw = NL80211_CHAN_WIDTH_20_NOHT;
 	struct sta_info *sta;
 
@@ -503,7 +509,7 @@ ieee80211_get_max_required_bw(struct ieee80211_link_data *link)
 		    !(sta->sdata->bss && sta->sdata->bss == sdata->bss))
 			continue;
 
-		max_bw = max(max_bw, ieee80211_get_sta_bw(sta, link_id));
+		max_bw = max(max_bw, ieee80211_get_sta_bw(sta, link));
 	}
 
 	return max_bw;
@@ -551,6 +557,7 @@ ieee80211_get_width_of_link(struct ieee80211_link_data *link)
 	case NL80211_IFTYPE_P2P_GO:
 	case NL80211_IFTYPE_NAN:
 	case NL80211_IFTYPE_NAN_DATA:
+	case NL80211_IFTYPE_PD:
 		WARN_ON_ONCE(1);
 		break;
 	}
@@ -709,8 +716,9 @@ static void ieee80211_chan_bw_change(struct ieee80211_local *local,
 			else
 				new_chandef = &link_conf->chanreq.oper;
 
-			new_sta_bw = _ieee80211_sta_cur_vht_bw(link_sta,
-							       new_chandef);
+			new_sta_bw = ieee80211_sta_current_bw(link_sta,
+							      new_chandef,
+							      IEEE80211_STA_BW_TX_TO_STA);
 
 			/* nothing change */
 			if (new_sta_bw == link_sta->pub->bandwidth)
@@ -733,6 +741,9 @@ static void ieee80211_chan_bw_change(struct ieee80211_local *local,
  * recalc the min required chan width of the channel context, which is
  * the max of min required widths of all the interfaces bound to this
  * channel context.
+ *
+ * Note: ieee80211_update_ap_bandwidth() relies on this iterating all
+ *	 affected stations, even if min_def didn't change.
  */
 static void
 _ieee80211_recalc_chanctx_min_def(struct ieee80211_local *local,
@@ -740,16 +751,20 @@ _ieee80211_recalc_chanctx_min_def(struct ieee80211_local *local,
 				  struct ieee80211_link_data *rsvd_for,
 				  bool check_reserved)
 {
-	u32 changed = __ieee80211_recalc_chanctx_min_def(local, ctx, rsvd_for,
-							 check_reserved);
+	u32 changed;
 
-	if (!changed)
+	/* No recalc for S1G chan ctx's */
+	if (cfg80211_chandef_is_s1g(&ctx->conf.def))
 		return;
 
+	changed = __ieee80211_recalc_chanctx_min_def(local, ctx, rsvd_for,
+						     check_reserved);
+
 	/* check is BW narrowed */
 	ieee80211_chan_bw_change(local, ctx, false, true);
 
-	drv_change_chanctx(local, ctx, changed);
+	if (changed)
+		drv_change_chanctx(local, ctx, changed);
 
 	/* check is BW wider */
 	ieee80211_chan_bw_change(local, ctx, false, false);
@@ -761,13 +776,44 @@ void ieee80211_recalc_chanctx_min_def(struct ieee80211_local *local,
 	_ieee80211_recalc_chanctx_min_def(local, ctx, NULL, false);
 }
 
+static void
+ieee80211_chanctx_update_npca_links(struct ieee80211_local *local,
+				    struct ieee80211_chanctx *ctx,
+				    bool enable)
+{
+	struct ieee80211_chanctx_user_iter iter;
+
+	if (!!ctx->conf.def.npca_chan != enable)
+		return;
+
+	for_each_chanctx_user_assigned(local, ctx, &iter) {
+		if (!iter.link)
+			continue;
+		if (!iter.sdata->vif.cfg.assoc)
+			continue;
+
+		if (enable) {
+			if (!iter.link->conf->chanreq.oper.npca_chan)
+				continue;
+		} else {
+			if (!iter.link->conf->npca.enabled)
+				continue;
+		}
+
+		iter.link->conf->npca.enabled = enable;
+		drv_link_info_changed(local, iter.sdata,
+				      iter.link->conf,
+				      iter.link->link_id,
+				      BSS_CHANGED_NPCA);
+	}
+}
+
 static void _ieee80211_change_chanctx(struct ieee80211_local *local,
 				      struct ieee80211_chanctx *ctx,
 				      struct ieee80211_chanctx *old_ctx,
 				      const struct ieee80211_chan_req *chanreq,
 				      struct ieee80211_link_data *rsvd_for)
 {
-	const struct cfg80211_chan_def *chandef = &chanreq->oper;
 	struct ieee80211_chan_req ctx_req = {
 		.oper = ctx->conf.def,
 		.ap = ctx->conf.ap,
@@ -775,7 +821,7 @@ static void _ieee80211_change_chanctx(struct ieee80211_local *local,
 	u32 changed = 0;
 
 	/* 5/10 MHz not handled here */
-	switch (chandef->width) {
+	switch (chanreq->oper.width) {
 	case NL80211_CHAN_WIDTH_1:
 	case NL80211_CHAN_WIDTH_2:
 	case NL80211_CHAN_WIDTH_4:
@@ -820,10 +866,15 @@ static void _ieee80211_change_chanctx(struct ieee80211_local *local,
 			changed |= IEEE80211_CHANCTX_CHANGE_WIDTH;
 		if (ctx->conf.def.punctured != chanreq->oper.punctured)
 			changed |= IEEE80211_CHANCTX_CHANGE_PUNCTURING;
+		if (ctx->conf.def.npca_chan != chanreq->oper.npca_chan)
+			changed |= IEEE80211_CHANCTX_CHANGE_NPCA;
+		if (chanreq->oper.npca_chan &&
+		    ctx->conf.def.npca_punctured != chanreq->oper.npca_punctured)
+			changed |= IEEE80211_CHANCTX_CHANGE_NPCA_PUNCT;
 	}
 	if (!cfg80211_chandef_identical(&ctx->conf.ap, &chanreq->ap))
 		changed |= IEEE80211_CHANCTX_CHANGE_AP;
-	ctx->conf.def = *chandef;
+	ctx->conf.def = chanreq->oper;
 	ctx->conf.ap = chanreq->ap;
 
 	/* check if min chanctx also changed */
@@ -832,10 +883,16 @@ static void _ieee80211_change_chanctx(struct ieee80211_local *local,
 
 	ieee80211_add_wbrf(local, &ctx->conf.def);
 
+	/* disable NPCA on the link using it */
+	ieee80211_chanctx_update_npca_links(local, ctx, false);
+
 	drv_change_chanctx(local, ctx, changed);
 
 	/* check if BW is wider */
 	ieee80211_chan_bw_change(local, old_ctx, false, false);
+
+	/* enable NPCA on the link that requested it */
+	ieee80211_chanctx_update_npca_links(local, ctx, true);
 }
 
 static void ieee80211_change_chanctx(struct ieee80211_local *local,
@@ -1546,6 +1603,7 @@ ieee80211_link_chanctx_reservation_complete(struct ieee80211_link_data *link)
 	case NL80211_IFTYPE_P2P_DEVICE:
 	case NL80211_IFTYPE_NAN:
 	case NL80211_IFTYPE_NAN_DATA:
+	case NL80211_IFTYPE_PD:
 	case NUM_NL80211_IFTYPES:
 		WARN_ON(1);
 		break;
@@ -1875,16 +1933,21 @@ static int ieee80211_vif_use_reserved_switch(struct ieee80211_local *local)
 		}
 
 		if (n_assigned != n_reserved) {
-			if (n_ready == n_reserved) {
-				wiphy_info(local->hw.wiphy,
-					   "channel context reservation cannot be finalized because some interfaces aren't switching\n");
-				err = -EBUSY;
-				goto err;
-			}
+			if (n_ready != n_reserved)
+				return -EAGAIN;
 
-			return -EAGAIN;
+			if (n_assigned == n_reserved + 1 &&
+			    ieee80211_nan_try_evacuate(&local->hw,
+						       &ctx->replace_ctx->conf))
+				goto use_reserved;
+
+			wiphy_info(local->hw.wiphy,
+				   "channel context reservation cannot be finalized because some interfaces aren't switching\n");
+			err = -EBUSY;
+			goto err;
 		}
 
+use_reserved:
 		ctx->conf.radar_enabled = false;
 		for_each_chanctx_user_reserved(local, ctx, &iter) {
 			if (ieee80211_link_has_in_place_reservation(iter.link) &&
@@ -2192,6 +2255,15 @@ int _ieee80211_link_use_channel(struct ieee80211_link_data *link,
 
 	ctx = ieee80211_find_or_create_chanctx(sdata, chanreq, mode,
 					       assign_on_failure, &reused_ctx);
+	if (IS_ERR(ctx)) {
+		/* Try to evacuate a NAN channel to free up a chanctx */
+		if (ieee80211_nan_try_evacuate(&local->hw, NULL))
+			ctx = ieee80211_find_or_create_chanctx(sdata, chanreq,
+							       mode,
+							       assign_on_failure,
+							       &reused_ctx);
+	}
+
 	if (IS_ERR(ctx)) {
 		ret = PTR_ERR(ctx);
 		goto out;
diff --git a/net/mac80211/debugfs.c b/net/mac80211/debugfs.c
index 5a1831b08677b..a4d5461f6480f 100644
--- a/net/mac80211/debugfs.c
+++ b/net/mac80211/debugfs.c
@@ -320,26 +320,17 @@ static ssize_t aql_enable_read(struct file *file, char __user *user_buf,
 static ssize_t aql_enable_write(struct file *file, const char __user *user_buf,
 				size_t count, loff_t *ppos)
 {
-	char buf[3];
-	size_t len;
-
-	if (count > sizeof(buf))
-		return -EINVAL;
-
-	if (copy_from_user(buf, user_buf, count))
-		return -EFAULT;
+	bool val;
+	int ret;
 
-	buf[sizeof(buf) - 1] = '\0';
-	len = strlen(buf);
-	if (len > 0 && buf[len - 1] == '\n')
-		buf[len - 1] = 0;
+	ret = kstrtobool_from_user(user_buf, count, &val);
+	if (unlikely(ret))
+		return ret;
 
-	if (buf[0] == '0' && buf[1] == '\0')
-		static_branch_enable(&aql_disable);
-	else if (buf[0] == '1' && buf[1] == '\0')
+	if (val)
 		static_branch_disable(&aql_disable);
 	else
-		return -EINVAL;
+		static_branch_enable(&aql_disable);
 
 	return count;
 }
@@ -371,26 +362,14 @@ static ssize_t force_tx_status_write(struct file *file,
 				     loff_t *ppos)
 {
 	struct ieee80211_local *local = file->private_data;
-	char buf[3];
-
-	if (count >= sizeof(buf))
-		return -EINVAL;
-
-	if (copy_from_user(buf, user_buf, count))
-		return -EFAULT;
-
-	if (count && buf[count - 1] == '\n')
-		buf[count - 1] = '\0';
-	else
-		buf[count] = '\0';
+	bool val;
+	int ret;
 
-	if (buf[0] == '0' && buf[1] == '\0')
-		local->force_tx_status = 0;
-	else if (buf[0] == '1' && buf[1] == '\0')
-		local->force_tx_status = 1;
-	else
-		return -EINVAL;
+	ret = kstrtobool_from_user(user_buf, count, &val);
+	if (unlikely(ret))
+		return ret;
 
+	local->force_tx_status = val;
 	return count;
 }
 
diff --git a/net/mac80211/eht.c b/net/mac80211/eht.c
index 768bfc4e737d3..23fe82df6abfe 100644
--- a/net/mac80211/eht.c
+++ b/net/mac80211/eht.c
@@ -2,7 +2,7 @@
 /*
  * EHT handling
  *
- * Copyright(c) 2021-2025 Intel Corporation
+ * Copyright(c) 2021-2026 Intel Corporation
  */
 
 #include "driver-ops.h"
@@ -74,9 +74,6 @@ ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata,
 
 	eht_cap->has_eht = true;
 
-	link_sta->cur_max_bandwidth = ieee80211_sta_cap_rx_bw(link_sta);
-	link_sta->pub->bandwidth = ieee80211_sta_cur_vht_bw(link_sta);
-
 	/*
 	 * The MPDU length bits are reserved on all but 2.4 GHz and get set via
 	 * VHT (5 GHz) or HE (6 GHz) capabilities.
@@ -103,177 +100,3 @@ ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata,
 
 	ieee80211_sta_recalc_aggregates(&link_sta->sta->sta);
 }
-
-static void
-ieee80211_send_eml_op_mode_notif(struct ieee80211_sub_if_data *sdata,
-				 struct ieee80211_mgmt *req, int opt_len)
-{
-	int len = IEEE80211_MIN_ACTION_SIZE(eml_omn);
-	struct ieee80211_local *local = sdata->local;
-	struct ieee80211_mgmt *mgmt;
-	struct sk_buff *skb;
-
-	len += opt_len; /* optional len */
-	skb = dev_alloc_skb(local->tx_headroom + len);
-	if (!skb)
-		return;
-
-	skb_reserve(skb, local->tx_headroom);
-	mgmt = skb_put_zero(skb, len);
-	mgmt->frame_control = cpu_to_le16(IEEE80211_FTYPE_MGMT |
-					  IEEE80211_STYPE_ACTION);
-	memcpy(mgmt->da, req->sa, ETH_ALEN);
-	memcpy(mgmt->sa, sdata->vif.addr, ETH_ALEN);
-	memcpy(mgmt->bssid, sdata->vif.addr, ETH_ALEN);
-
-	mgmt->u.action.category = WLAN_CATEGORY_PROTECTED_EHT;
-	mgmt->u.action.action_code = WLAN_PROTECTED_EHT_ACTION_EML_OP_MODE_NOTIF;
-	mgmt->u.action.eml_omn.dialog_token =
-		req->u.action.eml_omn.dialog_token;
-	mgmt->u.action.eml_omn.control = req->u.action.eml_omn.control &
-		~(IEEE80211_EML_CTRL_EMLSR_PARAM_UPDATE |
-		  IEEE80211_EML_CTRL_INDEV_COEX_ACT);
-	/* Copy optional fields from the received notification frame */
-	memcpy(mgmt->u.action.eml_omn.variable,
-	       req->u.action.eml_omn.variable, opt_len);
-
-	ieee80211_tx_skb(sdata, skb);
-}
-
-void ieee80211_rx_eml_op_mode_notif(struct ieee80211_sub_if_data *sdata,
-				    struct sk_buff *skb)
-{
-	int len = IEEE80211_MIN_ACTION_SIZE(eml_omn);
-	enum nl80211_iftype type = ieee80211_vif_type_p2p(&sdata->vif);
-	struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb);
-	const struct wiphy_iftype_ext_capab *ift_ext_capa;
-	struct ieee80211_mgmt *mgmt = (void *)skb->data;
-	struct ieee80211_local *local = sdata->local;
-	u8 control = mgmt->u.action.eml_omn.control;
-	u8 *ptr = mgmt->u.action.eml_omn.variable;
-	struct ieee80211_eml_params eml_params = {
-		.link_id = status->link_id,
-		.control = control,
-	};
-	struct sta_info *sta;
-	int opt_len = 0;
-
-	if (!ieee80211_vif_is_mld(&sdata->vif))
-		return;
-
-	/* eMLSR and eMLMR can't be enabled at the same time */
-	if ((control & IEEE80211_EML_CTRL_EMLSR_MODE) &&
-	    (control & IEEE80211_EML_CTRL_EMLMR_MODE))
-		return;
-
-	if ((control & IEEE80211_EML_CTRL_EMLMR_MODE) &&
-	    (control & IEEE80211_EML_CTRL_EMLSR_PARAM_UPDATE))
-		return;
-
-	ift_ext_capa = cfg80211_get_iftype_ext_capa(local->hw.wiphy, type);
-	if (!ift_ext_capa)
-		return;
-
-	if (!status->link_valid)
-		return;
-
-	sta = sta_info_get_bss(sdata, mgmt->sa);
-	if (!sta)
-		return;
-
-	if (control & IEEE80211_EML_CTRL_EMLSR_MODE) {
-		u8 emlsr_param_update_len;
-
-		if (!(ift_ext_capa->eml_capabilities &
-		      IEEE80211_EML_CAP_EMLSR_SUPP))
-			return;
-
-		opt_len += sizeof(__le16); /* eMLSR link_bitmap */
-		/* eMLSR param update field is not part of Notfication frame
-		 * sent by the AP to client so account it separately.
-		 */
-		emlsr_param_update_len =
-			!!(control & IEEE80211_EML_CTRL_EMLSR_PARAM_UPDATE);
-
-		if (skb->len < len + opt_len + emlsr_param_update_len)
-			return;
-
-		if (control & IEEE80211_EML_CTRL_EMLSR_PARAM_UPDATE) {
-			u8 pad_delay, trans_delay;
-
-			pad_delay = u8_get_bits(ptr[2],
-						IEEE80211_EML_EMLSR_PAD_DELAY);
-			if (pad_delay >
-			    IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US)
-				return;
-
-			trans_delay = u8_get_bits(ptr[2],
-					IEEE80211_EML_EMLSR_TRANS_DELAY);
-			if (trans_delay >
-			    IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US)
-				return;
-
-			/* Update sta padding and transition delay */
-			sta->sta.eml_cap =
-				u8_replace_bits(sta->sta.eml_cap,
-						pad_delay,
-						IEEE80211_EML_CAP_EMLSR_PADDING_DELAY);
-			sta->sta.eml_cap =
-				u8_replace_bits(sta->sta.eml_cap,
-						trans_delay,
-						IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY);
-		}
-	}
-
-	if (control & IEEE80211_EML_CTRL_EMLMR_MODE) {
-		u8 mcs_map_size;
-		int i;
-
-		if (!(ift_ext_capa->eml_capabilities &
-		      IEEE80211_EML_CAP_EMLMR_SUPPORT))
-			return;
-
-		opt_len += sizeof(__le16); /* eMLMR link_bitmap */
-		opt_len++; /* eMLMR mcs_map_count */
-		if (skb->len < len + opt_len)
-			return;
-
-		eml_params.emlmr_mcs_map_count = ptr[2];
-		if (eml_params.emlmr_mcs_map_count > 2)
-			return;
-
-		mcs_map_size = 3 * (1 + eml_params.emlmr_mcs_map_count);
-		opt_len += mcs_map_size;
-		if (skb->len < len + opt_len)
-			return;
-
-		for (i = 0; i < mcs_map_size; i++) {
-			u8 rx_mcs, tx_mcs;
-
-			rx_mcs = u8_get_bits(ptr[3 + i],
-					     IEEE80211_EML_EMLMR_RX_MCS_MAP);
-			if (rx_mcs > 8)
-				return;
-
-			tx_mcs = u8_get_bits(ptr[3 + i],
-					     IEEE80211_EML_EMLMR_TX_MCS_MAP);
-			if (tx_mcs > 8)
-				return;
-		}
-
-		memcpy(eml_params.emlmr_mcs_map_bw, &ptr[3], mcs_map_size);
-	}
-
-	if ((control & IEEE80211_EML_CTRL_EMLSR_MODE) ||
-	    (control & IEEE80211_EML_CTRL_EMLMR_MODE)) {
-		eml_params.link_bitmap = get_unaligned_le16(ptr);
-		if ((eml_params.link_bitmap & sdata->vif.active_links) !=
-		    eml_params.link_bitmap)
-			return;
-	}
-
-	if (drv_set_eml_op_mode(sdata, &sta->sta, &eml_params))
-		return;
-
-	ieee80211_send_eml_op_mode_notif(sdata, mgmt, opt_len);
-}
diff --git a/net/mac80211/he.c b/net/mac80211/he.c
index a3e16a5bec224..b7d9e4cb6ba6e 100644
--- a/net/mac80211/he.c
+++ b/net/mac80211/he.c
@@ -3,7 +3,7 @@
  * HE handling
  *
  * Copyright(c) 2017 Intel Deutschland GmbH
- * Copyright(c) 2019-2025 Intel Corporation
+ * Copyright(c) 2019-2026 Intel Corporation
  */
 
 #include "ieee80211_i.h"
@@ -159,10 +159,6 @@ _ieee80211_he_cap_ie_to_sta_he_cap(struct ieee80211_sub_if_data *sdata,
 
 	he_cap->has_he = true;
 
-	link_sta->cur_max_bandwidth = ieee80211_sta_cap_rx_bw(link_sta);
-	if (sdata->vif.type != NL80211_IFTYPE_NAN)
-		link_sta->pub->bandwidth = ieee80211_sta_cur_vht_bw(link_sta);
-
 	if (he_6ghz_capa)
 		ieee80211_update_from_he_6ghz_capa(he_6ghz_capa, link_sta);
 
@@ -277,7 +273,8 @@ static void ieee80211_link_sta_rc_update_omi(struct ieee80211_link_data *link,
 	band = link->conf->chanreq.oper.chan->band;
 	sband = sdata->local->hw.wiphy->bands[band];
 
-	new_bw = ieee80211_sta_cur_vht_bw(link_sta);
+	new_bw = ieee80211_sta_current_bw(link_sta, &link->conf->chanreq.oper,
+					  IEEE80211_STA_BW_TX_TO_STA);
 	if (link_sta->pub->bandwidth == new_bw)
 		return;
 
diff --git a/net/mac80211/ht.c b/net/mac80211/ht.c
index 97719298e038a..6285ac15c16c4 100644
--- a/net/mac80211/ht.c
+++ b/net/mac80211/ht.c
@@ -140,14 +140,11 @@ bool ieee80211_ht_cap_ie_to_sta_ht_cap(struct ieee80211_sub_if_data *sdata,
 				       const struct ieee80211_ht_cap *ht_cap_ie,
 				       struct link_sta_info *link_sta)
 {
-	struct ieee80211_bss_conf *link_conf;
 	struct sta_info *sta = link_sta->sta;
 	struct ieee80211_sta_ht_cap ht_cap, own_cap;
 	u8 ampdu_info, tx_mcs_set_cap;
 	int i, max_tx_streams;
 	bool changed;
-	enum ieee80211_sta_rx_bandwidth bw;
-	enum nl80211_chan_width width;
 
 	memset(&ht_cap, 0, sizeof(ht_cap));
 
@@ -256,45 +253,6 @@ bool ieee80211_ht_cap_ie_to_sta_ht_cap(struct ieee80211_sub_if_data *sdata,
 
 	memcpy(&link_sta->pub->ht_cap, &ht_cap, sizeof(ht_cap));
 
-	rcu_read_lock();
-	link_conf = rcu_dereference(sdata->vif.link_conf[link_sta->link_id]);
-	if (WARN_ON(!link_conf)) {
-		width = NL80211_CHAN_WIDTH_20_NOHT;
-	} else if (sdata->vif.type == NL80211_IFTYPE_NAN ||
-		   sdata->vif.type == NL80211_IFTYPE_NAN_DATA) {
-		/* In NAN, link_sta->bandwidth is invalid since NAN operates on
-		 * multiple channels. Just take the maximum.
-		 */
-		width = NL80211_CHAN_WIDTH_320;
-	} else {
-		width = link_conf->chanreq.oper.width;
-	}
-
-	switch (width) {
-	default:
-		WARN_ON_ONCE(1);
-		fallthrough;
-	case NL80211_CHAN_WIDTH_20_NOHT:
-	case NL80211_CHAN_WIDTH_20:
-		bw = IEEE80211_STA_RX_BW_20;
-		break;
-	case NL80211_CHAN_WIDTH_40:
-	case NL80211_CHAN_WIDTH_80:
-	case NL80211_CHAN_WIDTH_80P80:
-	case NL80211_CHAN_WIDTH_160:
-	case NL80211_CHAN_WIDTH_320:
-		bw = ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 ?
-				IEEE80211_STA_RX_BW_40 : IEEE80211_STA_RX_BW_20;
-		break;
-	}
-	rcu_read_unlock();
-
-	link_sta->pub->bandwidth = bw;
-
-	link_sta->cur_max_bandwidth =
-		ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 ?
-				IEEE80211_STA_RX_BW_40 : IEEE80211_STA_RX_BW_20;
-
 	if (sta->sdata->vif.type == NL80211_IFTYPE_AP ||
 	    sta->sdata->vif.type == NL80211_IFTYPE_AP_VLAN ||
 	    sta->sdata->vif.type == NL80211_IFTYPE_NAN ||
@@ -629,25 +587,30 @@ void ieee80211_ht_handle_chanwidth_notif(struct ieee80211_local *local,
 	enum ieee80211_sta_rx_bandwidth max_bw, new_bw;
 	struct ieee80211_supported_band *sband;
 	struct sta_opmode_info sta_opmode = {};
+	struct ieee80211_link_data *link;
 
 	lockdep_assert_wiphy(local->hw.wiphy);
 
+	link = sdata_dereference(sdata->link[link_sta->link_id], sdata);
+	if (WARN_ON(!link))
+		return;
+
 	if (chanwidth == IEEE80211_HT_CHANWIDTH_20MHZ)
 		max_bw = IEEE80211_STA_RX_BW_20;
 	else
-		max_bw = ieee80211_sta_cap_rx_bw(link_sta);
+		max_bw = IEEE80211_STA_RX_BW_MAX;
 
-	/* set cur_max_bandwidth and recalc sta bw */
-	link_sta->cur_max_bandwidth = max_bw;
-	new_bw = ieee80211_sta_cur_vht_bw(link_sta);
+	/* set op_mode_bw and recalc sta bw */
+	link_sta->op_mode_bw = max_bw;
+	new_bw = ieee80211_sta_current_bw(link_sta, &link->conf->chanreq.oper,
+					  IEEE80211_STA_BW_TX_TO_STA);
 
 	if (link_sta->pub->bandwidth == new_bw)
 		return;
 
 	link_sta->pub->bandwidth = new_bw;
 	sband = local->hw.wiphy->bands[band];
-	sta_opmode.bw =
-		ieee80211_sta_rx_bw_to_chan_width(link_sta);
+	sta_opmode.bw = ieee80211_sta_rx_bw_to_chan_width(new_bw);
 	sta_opmode.changed = STA_OPMODE_MAX_BW_CHANGED;
 
 	rate_control_rate_update(local, sband, link_sta,
diff --git a/net/mac80211/ibss.c b/net/mac80211/ibss.c
index 97292ff51475f..08690342cfaa3 100644
--- a/net/mac80211/ibss.c
+++ b/net/mac80211/ibss.c
@@ -553,6 +553,9 @@ static struct sta_info *ieee80211_ibss_finish_sta(struct sta_info *sta)
 
 	memcpy(addr, sta->sta.addr, ETH_ALEN);
 
+	ieee80211_sta_init_nss_bw_capa(&sta->deflink,
+				       &sdata->deflink.conf->chanreq.oper);
+
 	ibss_dbg(sdata, "Adding new IBSS station %pM\n", addr);
 
 	sta_info_pre_move_state(sta, IEEE80211_STA_AUTH);
diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h
index 2a693406294bc..fc4424b125c13 100644
--- a/net/mac80211/ieee80211_i.h
+++ b/net/mac80211/ieee80211_i.h
@@ -1133,6 +1133,10 @@ struct ieee80211_link_data {
 
 	struct ieee80211_bss_conf *conf;
 
+	struct {
+		enum ieee80211_sta_rx_bandwidth he_and_lower;
+	} bss_bw;
+
 #ifdef CONFIG_MAC80211_DEBUGFS
 	struct dentry *debugfs_dir;
 #endif
@@ -1786,6 +1790,8 @@ struct ieee802_11_elems {
 	const u8 *ie_start;
 	size_t total_len;
 	u32 crc;
+	u8 frame_type;
+	bool from_ap;
 
 	/* pointers to IEs */
 	const struct ieee80211_tdls_lnkie *lnk_id;
@@ -1969,6 +1975,10 @@ void ieee80211_clear_fast_rx(struct sta_info *sta);
 bool ieee80211_is_our_addr(struct ieee80211_sub_if_data *sdata,
 			   const u8 *addr, int *out_link_id);
 
+/* AP code */
+void ieee80211_ap_rx_queued_frame(struct ieee80211_sub_if_data *sdata,
+				  struct sk_buff *skb);
+
 /* STA code */
 void ieee80211_sta_setup_sdata(struct ieee80211_sub_if_data *sdata);
 int ieee80211_mgd_auth(struct ieee80211_sub_if_data *sdata,
@@ -1984,10 +1994,8 @@ void ieee80211_send_pspoll(struct ieee80211_local *local,
 void ieee80211_recalc_ps(struct ieee80211_local *local);
 void ieee80211_recalc_ps_vif(struct ieee80211_sub_if_data *sdata);
 void ieee80211_sta_work(struct ieee80211_sub_if_data *sdata);
-void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata,
-				  struct sk_buff *skb);
-void ieee80211_sta_rx_queued_ext(struct ieee80211_sub_if_data *sdata,
-				 struct sk_buff *skb);
+void ieee80211_sta_rx_queued_frame(struct ieee80211_sub_if_data *sdata,
+				   struct sk_buff *skb);
 void ieee80211_sta_reset_beacon_monitor(struct ieee80211_sub_if_data *sdata);
 void ieee80211_sta_reset_conn_monitor(struct ieee80211_sub_if_data *sdata);
 void ieee80211_mgd_stop(struct ieee80211_sub_if_data *sdata);
@@ -2093,7 +2101,8 @@ void ieee80211_roc_purge(struct ieee80211_local *local,
 			 struct ieee80211_sub_if_data *sdata);
 int ieee80211_remain_on_channel(struct wiphy *wiphy, struct wireless_dev *wdev,
 				struct ieee80211_channel *chan,
-				unsigned int duration, u64 *cookie);
+				unsigned int duration, u64 *cookie,
+				const u8 *rx_addr);
 int ieee80211_cancel_remain_on_channel(struct wiphy *wiphy,
 				       struct wireless_dev *wdev, u64 cookie);
 int ieee80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
@@ -2299,25 +2308,6 @@ ieee80211_vht_cap_ie_to_sta_vht_cap(struct ieee80211_sub_if_data *sdata,
 				    const struct ieee80211_vht_cap *vht_cap_ie,
 				    const struct ieee80211_vht_cap *vht_cap_ie2,
 				    struct link_sta_info *link_sta);
-enum ieee80211_sta_rx_bandwidth
-_ieee80211_sta_cap_rx_bw(struct link_sta_info *link_sta,
-			 struct cfg80211_chan_def *chandef);
-static inline enum ieee80211_sta_rx_bandwidth
-ieee80211_sta_cap_rx_bw(struct link_sta_info *link_sta)
-{
-	return _ieee80211_sta_cap_rx_bw(link_sta, NULL);
-}
-enum ieee80211_sta_rx_bandwidth
-_ieee80211_sta_cur_vht_bw(struct link_sta_info *link_sta,
-			  struct cfg80211_chan_def *chandef);
-static inline enum ieee80211_sta_rx_bandwidth
-ieee80211_sta_cur_vht_bw(struct link_sta_info *link_sta)
-{
-	return _ieee80211_sta_cur_vht_bw(link_sta, NULL);
-}
-void ieee80211_sta_init_nss(struct link_sta_info *link_sta);
-enum nl80211_chan_width
-ieee80211_sta_cap_chan_bw(struct link_sta_info *link_sta);
 void ieee80211_process_mu_groups(struct ieee80211_sub_if_data *sdata,
 				 struct ieee80211_link_data *link,
 				 struct ieee80211_mgmt *mgmt);
@@ -2331,8 +2321,6 @@ void ieee80211_apply_vhtcap_overrides(struct ieee80211_sub_if_data *sdata,
 				      struct ieee80211_sta_vht_cap *vht_cap);
 void ieee80211_get_vht_mask_from_cap(__le16 vht_cap,
 				     u16 vht_mask[NL80211_VHT_NSS_MAX]);
-enum nl80211_chan_width
-ieee80211_sta_rx_bw_to_chan_width(struct link_sta_info *sta);
 
 /* HE */
 void
@@ -2718,6 +2706,9 @@ void ieee80211_add_s1g_capab_ie(struct ieee80211_sub_if_data *sdata,
 void ieee80211_add_aid_request_ie(struct ieee80211_sub_if_data *sdata,
 				  struct sk_buff *skb);
 
+enum nl80211_chan_width
+ieee80211_sta_rx_bw_to_chan_width(enum ieee80211_sta_rx_bandwidth bw);
+
 /* element building in SKBs */
 int ieee80211_put_srates_elem(struct sk_buff *skb,
 			      const struct ieee80211_supported_band *sband,
@@ -2891,8 +2882,6 @@ void ieee80211_destroy_frag_cache(struct ieee80211_fragment_cache *cache);
 
 u8 ieee80211_ie_len_eht_cap(struct ieee80211_sub_if_data *sdata);
 
-void ieee80211_rx_eml_op_mode_notif(struct ieee80211_sub_if_data *sdata,
-				    struct sk_buff *skb);
 void
 ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata,
 				    struct ieee80211_supported_band *sband,
@@ -2900,28 +2889,17 @@ ieee80211_eht_cap_ie_to_sta_eht_cap(struct ieee80211_sub_if_data *sdata,
 				    const struct ieee80211_eht_cap_elem *eht_cap_ie_elem,
 				    u8 eht_cap_len,
 				    struct link_sta_info *link_sta);
-void ieee80211_process_neg_ttlm_req(struct ieee80211_sub_if_data *sdata,
-				    struct ieee80211_mgmt *mgmt, size_t len);
-void ieee80211_process_neg_ttlm_res(struct ieee80211_sub_if_data *sdata,
-				    struct ieee80211_mgmt *mgmt, size_t len);
 int ieee80211_req_neg_ttlm(struct ieee80211_sub_if_data *sdata,
 			   struct cfg80211_ttlm_params *params);
-void ieee80211_process_ttlm_teardown(struct ieee80211_sub_if_data *sdata);
 
 void ieee80211_check_wbrf_support(struct ieee80211_local *local);
 void ieee80211_add_wbrf(struct ieee80211_local *local, struct cfg80211_chan_def *chandef);
 void ieee80211_remove_wbrf(struct ieee80211_local *local, struct cfg80211_chan_def *chandef);
 int ieee80211_mgd_set_epcs(struct ieee80211_sub_if_data *sdata, bool enable);
-void ieee80211_process_epcs_ena_resp(struct ieee80211_sub_if_data *sdata,
-				     struct ieee80211_mgmt *mgmt, size_t len);
-void ieee80211_process_epcs_teardown(struct ieee80211_sub_if_data *sdata,
-				     struct ieee80211_mgmt *mgmt, size_t len);
 
 int ieee80211_mgd_assoc_ml_reconf(struct ieee80211_sub_if_data *sdata,
 				  struct cfg80211_ml_reconf_req *req);
 
-void ieee80211_process_ml_reconf_resp(struct ieee80211_sub_if_data *sdata,
-				      struct ieee80211_mgmt *mgmt, size_t len);
 void ieee80211_stop_mbssid(struct ieee80211_sub_if_data *sdata);
 
 void
diff --git a/net/mac80211/iface.c b/net/mac80211/iface.c
index 95b779c4d627e..56400c22d8bcb 100644
--- a/net/mac80211/iface.c
+++ b/net/mac80211/iface.c
@@ -1400,6 +1400,7 @@ int ieee80211_do_open(struct wireless_dev *wdev, bool coming_up)
 	case NL80211_IFTYPE_P2P_DEVICE:
 	case NL80211_IFTYPE_OCB:
 	case NL80211_IFTYPE_NAN:
+	case NL80211_IFTYPE_PD:
 		/* no special treatment */
 		break;
 	case NL80211_IFTYPE_NAN_DATA:
@@ -1532,7 +1533,8 @@ int ieee80211_do_open(struct wireless_dev *wdev, bool coming_up)
 						FIF_PROBE_REQ);
 
 		if (sdata->vif.type != NL80211_IFTYPE_P2P_DEVICE &&
-		    sdata->vif.type != NL80211_IFTYPE_NAN)
+		    sdata->vif.type != NL80211_IFTYPE_NAN &&
+		    sdata->vif.type != NL80211_IFTYPE_PD)
 			changed |= ieee80211_reset_erp_info(sdata);
 		ieee80211_link_info_change_notify(sdata, &sdata->deflink,
 						  changed);
@@ -1548,6 +1550,7 @@ int ieee80211_do_open(struct wireless_dev *wdev, bool coming_up)
 			break;
 		case NL80211_IFTYPE_P2P_DEVICE:
 		case NL80211_IFTYPE_NAN:
+		case NL80211_IFTYPE_PD:
 			break;
 		default:
 			/* not reached */
@@ -1713,50 +1716,6 @@ static void ieee80211_iface_process_skb(struct ieee80211_local *local,
 		default:
 			break;
 		}
-	} else if (ieee80211_is_action(mgmt->frame_control) &&
-		   mgmt->u.action.category == WLAN_CATEGORY_PROTECTED_EHT) {
-		if (sdata->vif.type == NL80211_IFTYPE_AP) {
-			switch (mgmt->u.action.action_code) {
-			case WLAN_PROTECTED_EHT_ACTION_EML_OP_MODE_NOTIF:
-				ieee80211_rx_eml_op_mode_notif(sdata, skb);
-				break;
-			default:
-				break;
-			}
-		} else if (sdata->vif.type == NL80211_IFTYPE_STATION) {
-			switch (mgmt->u.action.action_code) {
-			case WLAN_PROTECTED_EHT_ACTION_TTLM_REQ:
-				ieee80211_process_neg_ttlm_req(sdata, mgmt,
-							       skb->len);
-				break;
-			case WLAN_PROTECTED_EHT_ACTION_TTLM_RES:
-				ieee80211_process_neg_ttlm_res(sdata, mgmt,
-							       skb->len);
-				break;
-			case WLAN_PROTECTED_EHT_ACTION_TTLM_TEARDOWN:
-				ieee80211_process_ttlm_teardown(sdata);
-				break;
-			case WLAN_PROTECTED_EHT_ACTION_LINK_RECONFIG_RESP:
-				ieee80211_process_ml_reconf_resp(sdata, mgmt,
-								 skb->len);
-				break;
-			case WLAN_PROTECTED_EHT_ACTION_EPCS_ENABLE_RESP:
-				ieee80211_process_epcs_ena_resp(sdata, mgmt,
-								skb->len);
-				break;
-			case WLAN_PROTECTED_EHT_ACTION_EPCS_ENABLE_TEARDOWN:
-				ieee80211_process_epcs_teardown(sdata, mgmt,
-								skb->len);
-				break;
-			default:
-				break;
-			}
-		}
-	} else if (ieee80211_is_ext(mgmt->frame_control)) {
-		if (sdata->vif.type == NL80211_IFTYPE_STATION)
-			ieee80211_sta_rx_queued_ext(sdata, skb);
-		else
-			WARN_ON(1);
 	} else if (ieee80211_is_data_qos(mgmt->frame_control)) {
 		struct ieee80211_hdr *hdr = (void *)mgmt;
 		struct sta_info *sta;
@@ -1788,8 +1747,11 @@ static void ieee80211_iface_process_skb(struct ieee80211_local *local,
 				true);
 		}
 	} else switch (sdata->vif.type) {
+	case NL80211_IFTYPE_AP:
+		ieee80211_ap_rx_queued_frame(sdata, skb);
+		break;
 	case NL80211_IFTYPE_STATION:
-		ieee80211_sta_rx_queued_mgmt(sdata, skb);
+		ieee80211_sta_rx_queued_frame(sdata, skb);
 		break;
 	case NL80211_IFTYPE_ADHOC:
 		ieee80211_ibss_rx_queued_mgmt(sdata, skb);
@@ -1988,6 +1950,7 @@ static void ieee80211_setup_sdata(struct ieee80211_sub_if_data *sdata,
 		sdata->vif.bss_conf.bssid = sdata->vif.addr;
 		break;
 	case NL80211_IFTYPE_NAN_DATA:
+	case NL80211_IFTYPE_PD:
 		break;
 	case NL80211_IFTYPE_UNSPECIFIED:
 	case NL80211_IFTYPE_WDS:
@@ -2280,7 +2243,12 @@ int ieee80211_if_add(struct ieee80211_local *local, const char *name,
 
 		sdata->dev = NULL;
 		strscpy(sdata->name, name, IFNAMSIZ);
-		ieee80211_assign_perm_addr(local, wdev->address, type);
+
+		if (is_valid_ether_addr(params->macaddr))
+			memcpy(wdev->address, params->macaddr, ETH_ALEN);
+		else
+			ieee80211_assign_perm_addr(local, wdev->address, type);
+
 		memcpy(sdata->vif.addr, wdev->address, ETH_ALEN);
 		ether_addr_copy(sdata->vif.bss_conf.addr, sdata->vif.addr);
 
diff --git a/net/mac80211/key.c b/net/mac80211/key.c
index 4b8965633df34..f45e792abede5 100644
--- a/net/mac80211/key.c
+++ b/net/mac80211/key.c
@@ -6,7 +6,7 @@
  * Copyright 2007-2008	Johannes Berg <johannes@sipsolutions.net>
  * Copyright 2013-2014  Intel Mobile Communications GmbH
  * Copyright 2015-2017	Intel Deutschland GmbH
- * Copyright 2018-2020, 2022-2025  Intel Corporation
+ * Copyright 2018-2020, 2022-2026  Intel Corporation
  */
 
 #include <crypto/utils.h>
@@ -150,11 +150,14 @@ static int ieee80211_key_enable_hw_accel(struct ieee80211_key *key)
 	sta = key->sta;
 
 	/*
-	 * If this is a per-STA GTK, check if it
-	 * is supported; if not, return.
+	 * Allow installation of a per-STA GTK if per-STA GTK is supported
+	 * by the driver or the interface is a NAN Data interface (as
+	 * per-station GTKs are required to be supported if secure NAN is
+	 * supported).
 	 */
 	if (sta && !(key->conf.flags & IEEE80211_KEY_FLAG_PAIRWISE) &&
-	    !ieee80211_hw_check(&key->local->hw, SUPPORTS_PER_STA_GTK))
+	    !(ieee80211_hw_check(&key->local->hw, SUPPORTS_PER_STA_GTK) ||
+	      sdata->vif.type == NL80211_IFTYPE_NAN_DATA))
 		goto out_unsupported;
 
 	if (sta && !sta->uploaded)
@@ -867,11 +870,16 @@ int ieee80211_key_link(struct ieee80211_key *key,
 		alt_key = wiphy_dereference(sdata->local->hw.wiphy,
 					    sta->ptk[idx ^ 1]);
 
-		/* The rekey code assumes that the old and new key are using
+		/*
+		 * The rekey code assumes that the old and new key are using
 		 * the same cipher. Enforce the assumption for pairwise keys.
+		 * NAN Data interfaces are exempt: Wi-Fi Aware v4.0 section 7.4
+		 * requires upgrading the ND-TKSA when a new NDP negotiates a
+		 * stronger cipher suite.
 		 */
-		if ((alt_key && alt_key->conf.cipher != key->conf.cipher) ||
-		    (old_key && old_key->conf.cipher != key->conf.cipher)) {
+		if (sdata->vif.type != NL80211_IFTYPE_NAN_DATA &&
+		    ((alt_key && alt_key->conf.cipher != key->conf.cipher) ||
+		     (old_key && old_key->conf.cipher != key->conf.cipher))) {
 			ret = -EOPNOTSUPP;
 			goto out;
 		}
diff --git a/net/mac80211/link.c b/net/mac80211/link.c
index 93e290dd783f2..e81dd02de12e0 100644
--- a/net/mac80211/link.c
+++ b/net/mac80211/link.c
@@ -2,7 +2,7 @@
 /*
  * MLO link handling
  *
- * Copyright (C) 2022-2025 Intel Corporation
+ * Copyright (C) 2022-2026 Intel Corporation
  */
 #include <linux/slab.h>
 #include <linux/kernel.h>
@@ -307,6 +307,9 @@ static int ieee80211_vif_update_links(struct ieee80211_sub_if_data *sdata,
 	if (old_links == new_links && dormant_links == sdata->vif.dormant_links)
 		return 0;
 
+	if (!old_links || !new_links)
+		WARN_ON(sta_info_flush(sdata, -1) > 0);
+
 	/* if there were no old links, need to clear the pointers to deflink */
 	if (!old_links)
 		rem |= BIT(0);
diff --git a/net/mac80211/main.c b/net/mac80211/main.c
index f47dd58770adb..90d295cc364fb 100644
--- a/net/mac80211/main.c
+++ b/net/mac80211/main.c
@@ -750,6 +750,10 @@ ieee80211_default_mgmt_stypes[NUM_NL80211_IFTYPES] = {
 		.rx = BIT(IEEE80211_STYPE_ACTION >> 4) |
 			BIT(IEEE80211_STYPE_AUTH >> 4),
 	},
+	[NL80211_IFTYPE_NAN_DATA] = {
+		.tx = 0xffff,
+		.rx = BIT(IEEE80211_STYPE_ACTION >> 4),
+	},
 };
 
 static const struct ieee80211_ht_cap mac80211_ht_capa_mod_mask = {
@@ -1459,8 +1463,7 @@ int ieee80211_register_hw(struct ieee80211_hw *hw)
 
 	if (supp_uhr)
 		local->scan_ies_len +=
-			3 + sizeof(struct ieee80211_uhr_cap) +
-			sizeof(struct ieee80211_uhr_cap_phy);
+			3 + sizeof(struct ieee80211_uhr_cap);
 
 	if (!local->ops->hw_scan) {
 		/* For hw_scan, driver needs to set these up. */
diff --git a/net/mac80211/mesh_plink.c b/net/mac80211/mesh_plink.c
index 7cbab90c8784f..37adb053213e6 100644
--- a/net/mac80211/mesh_plink.c
+++ b/net/mac80211/mesh_plink.c
@@ -470,6 +470,9 @@ static void mesh_sta_info_init(struct ieee80211_sub_if_data *sdata,
 					    elems->eht_cap, elems->eht_cap_len,
 					    &sta->deflink);
 
+	ieee80211_sta_init_nss_bw_capa(&sta->deflink,
+				       &sdata->deflink.conf->chanreq.oper);
+
 	if (bw != sta->sta.deflink.bandwidth)
 		changed |= IEEE80211_RC_BW_CHANGED;
 
diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c
index 0a0f27836d570..f60f59d1b37d9 100644
--- a/net/mac80211/mlme.c
+++ b/net/mac80211/mlme.c
@@ -149,20 +149,47 @@ static int ecw2cw(int ecw)
 	return (1 << ecw) - 1;
 }
 
+static bool ieee80211_chandef_usable(struct ieee80211_sub_if_data *sdata,
+				     const struct cfg80211_chan_def *chandef,
+				     u32 prohibited_flags)
+{
+	if (!cfg80211_chandef_usable(sdata->local->hw.wiphy,
+				     chandef, prohibited_flags))
+		return false;
+
+	if (chandef->punctured &&
+	    ieee80211_hw_check(&sdata->local->hw, DISALLOW_PUNCTURING))
+		return false;
+
+	return true;
+}
+
+struct ieee80211_determine_ap_chan_data {
+	/* input data */
+	struct ieee80211_channel *channel;
+	const struct ieee802_11_elems *elems;
+	const struct ieee80211_conn_settings *conn;
+	u32 vht_cap_info;
+	bool ignore_ht_channel_mismatch;
+
+	/* target chandef is filled in */
+	struct cfg80211_chan_def *chandef;
+};
+
 static enum ieee80211_conn_mode
 ieee80211_determine_ap_chan(struct ieee80211_sub_if_data *sdata,
-			    struct ieee80211_channel *channel,
-			    u32 vht_cap_info,
-			    const struct ieee802_11_elems *elems,
-			    bool ignore_ht_channel_mismatch,
-			    const struct ieee80211_conn_settings *conn,
-			    struct cfg80211_chan_def *chandef)
+			    struct ieee80211_determine_ap_chan_data *data)
 {
+	bool ignore_ht_channel_mismatch = data->ignore_ht_channel_mismatch;
+	const struct ieee802_11_elems *elems = data->elems;
 	const struct ieee80211_ht_operation *ht_oper = elems->ht_operation;
 	const struct ieee80211_vht_operation *vht_oper = elems->vht_operation;
 	const struct ieee80211_he_operation *he_oper = elems->he_operation;
 	const struct ieee80211_eht_operation *eht_oper = elems->eht_operation;
 	const struct ieee80211_uhr_operation *uhr_oper = elems->uhr_operation;
+	const struct ieee80211_conn_settings *conn = data->conn;
+	struct ieee80211_channel *channel = data->channel;
+	struct cfg80211_chan_def *chandef = data->chandef;
 	struct ieee80211_supported_band *sband =
 		sdata->local->hw.wiphy->bands[channel->band];
 	struct cfg80211_chan_def vht_chandef;
@@ -288,7 +315,7 @@ ieee80211_determine_ap_chan(struct ieee80211_sub_if_data *sdata,
 		memcpy(&he_oper_vht_cap, he_oper->optional, 3);
 		he_oper_vht_cap.basic_mcs_set = cpu_to_le16(0);
 
-		if (!ieee80211_chandef_vht_oper(&sdata->local->hw, vht_cap_info,
+		if (!ieee80211_chandef_vht_oper(&sdata->local->hw, data->vht_cap_info,
 						&he_oper_vht_cap, ht_oper,
 						&vht_chandef)) {
 			sdata_info(sdata,
@@ -303,7 +330,7 @@ ieee80211_determine_ap_chan(struct ieee80211_sub_if_data *sdata,
 	} else if (sband->band == NL80211_BAND_2GHZ) {
 		no_vht = true;
 	} else if (!ieee80211_chandef_vht_oper(&sdata->local->hw,
-					       vht_cap_info,
+					       data->vht_cap_info,
 					       vht_oper, ht_oper,
 					       &vht_chandef)) {
 		sdata_info(sdata,
@@ -365,56 +392,41 @@ ieee80211_determine_ap_chan(struct ieee80211_sub_if_data *sdata,
 	if (conn->mode < IEEE80211_CONN_MODE_UHR || !uhr_oper)
 		return IEEE80211_CONN_MODE_EHT;
 
-	/*
-	 * In beacons we don't have all the data - but we know the size was OK,
-	 * so if the size is valid as a non-beacon case, we have more data and
-	 * can validate the NPCA parameters.
-	 */
-	if (ieee80211_uhr_oper_size_ok((const void *)uhr_oper,
-				       elems->uhr_operation_len,
-				       false)) {
+	if (elems->frame_type != (IEEE80211_FTYPE_MGMT | IEEE80211_STYPE_BEACON)) {
 		struct cfg80211_chan_def npca_chandef = *chandef;
+		const struct ieee80211_sta_uhr_cap *uhr_cap;
 		const struct ieee80211_uhr_npca_info *npca;
-		const __le16 *dis_subch_bmap;
-		u16 punct = chandef->punctured, npca_punct;
 
-		npca = ieee80211_uhr_npca_info(uhr_oper);
-		if (npca) {
-			int width = cfg80211_chandef_get_width(chandef);
-			u8 offs = le32_get_bits(npca->params,
-						IEEE80211_UHR_NPCA_PARAMS_PRIMARY_CHAN_OFFS);
-			u32 cf1 = chandef->center_freq1;
-			bool pri_upper, npca_upper;
-
-			pri_upper = chandef->chan->center_freq > cf1;
-			npca_upper = 20 * offs >= width / 2;
-
-			if (20 * offs >= cfg80211_chandef_get_width(chandef) ||
-			    pri_upper == npca_upper) {
-				sdata_info(sdata,
-					   "AP UHR NPCA primary channel invalid, disabling UHR\n");
-				return IEEE80211_CONN_MODE_EHT;
-			}
-		}
+		/* frames other than beacons carry UHR capability too */
+		if (!elems->uhr_cap)
+			return IEEE80211_CONN_MODE_EHT;
 
-		dis_subch_bmap = ieee80211_uhr_npca_dis_subch_bitmap(uhr_oper);
+		npca = ieee80211_uhr_npca_info(uhr_oper);
 
-		if (dis_subch_bmap) {
-			npca_punct = get_unaligned_le16(dis_subch_bmap);
-			npca_chandef.punctured = npca_punct;
+		if (npca && !(elems->uhr_cap->mac.mac_cap[0] &
+				IEEE80211_UHR_MAC_CAP0_NPCA_SUPP)) {
+			sdata_info(sdata,
+				   "AP without UHR NPCA capability uses it, disabling UHR\n");
+			return IEEE80211_CONN_MODE_EHT;
 		}
 
-		/*
-		 * must be a valid puncturing pattern for this channel as
-		 * well as puncturing all subchannels that are already in
-		 * the disabled subchannel bitmap on the primary channel
-		 */
-		if (!cfg80211_chandef_valid(&npca_chandef) ||
-		    ((punct & npca_punct) != punct)) {
+		/* DBE is not considered yet, so this works */
+		if (!cfg80211_chandef_npca_valid(sdata->local->hw.wiphy,
+						 &npca_chandef, npca) ||
+		    cfg80211_chandef_add_npca(sdata->local->hw.wiphy,
+					      &npca_chandef, npca)) {
 			sdata_info(sdata,
-				   "AP UHR NPCA disabled subchannel bitmap invalid, disabling UHR\n");
+				   "AP UHR NPCA settings invalid, disabling UHR\n");
 			return IEEE80211_CONN_MODE_EHT;
 		}
+
+		uhr_cap = ieee80211_get_uhr_iftype_cap_vif(sband, &sdata->vif);
+		/* can't happen since we must have UHR to parse the elems */
+		if (WARN_ON(!uhr_cap))
+			return IEEE80211_CONN_MODE_EHT;
+
+		if (uhr_cap->mac.mac_cap[0] & IEEE80211_UHR_MAC_CAP0_NPCA_SUPP)
+			*chandef = npca_chandef;
 	}
 
 	return IEEE80211_CONN_MODE_UHR;
@@ -848,21 +860,6 @@ static void ieee80211_get_rates(struct ieee80211_supported_band *sband,
 	}
 }
 
-static bool ieee80211_chandef_usable(struct ieee80211_sub_if_data *sdata,
-				     const struct cfg80211_chan_def *chandef,
-				     u32 prohibited_flags)
-{
-	if (!cfg80211_chandef_usable(sdata->local->hw.wiphy,
-				     chandef, prohibited_flags))
-		return false;
-
-	if (chandef->punctured &&
-	    ieee80211_hw_check(&sdata->local->hw, DISALLOW_PUNCTURING))
-		return false;
-
-	return true;
-}
-
 static int ieee80211_chandef_num_subchans(const struct cfg80211_chan_def *c)
 {
 	if (c->width == NL80211_CHAN_WIDTH_80P80)
@@ -1094,6 +1091,13 @@ ieee80211_determine_chan_mode(struct ieee80211_sub_if_data *sdata,
 	enum ieee80211_conn_mode ap_mode;
 	unsigned long unknown_rates_selectors[BITS_TO_LONGS(128)] = {};
 	unsigned long sta_selectors[BITS_TO_LONGS(128)] = {};
+	struct ieee80211_determine_ap_chan_data ap_chan_data = {
+		.channel = channel,
+		.vht_cap_info = bss->vht_cap_info,
+		.ignore_ht_channel_mismatch = false,
+		.chandef = ap_chandef,
+		.conn = conn,
+	};
 	int ret;
 
 again:
@@ -1102,8 +1106,8 @@ ieee80211_determine_chan_mode(struct ieee80211_sub_if_data *sdata,
 	if (!elems)
 		return ERR_PTR(-ENOMEM);
 
-	ap_mode = ieee80211_determine_ap_chan(sdata, channel, bss->vht_cap_info,
-					      elems, false, conn, ap_chandef);
+	ap_chan_data.elems = elems;
+	ap_mode = ieee80211_determine_ap_chan(sdata, &ap_chan_data);
 
 	/* this should be impossible since parsing depends on our mode */
 	if (WARN_ON(ap_mode > conn->mode)) {
@@ -1331,12 +1335,20 @@ static int ieee80211_config_bw(struct ieee80211_link_data *link,
 			       bool update, u64 *changed, u16 stype)
 {
 	struct ieee80211_channel *channel = link->conf->chanreq.oper.chan;
+	struct cfg80211_chan_def ap_chandef;
+	struct ieee80211_determine_ap_chan_data ap_chan_data = {
+		.channel = channel,
+		.vht_cap_info = 0,
+		.ignore_ht_channel_mismatch = true,
+		.chandef = &ap_chandef,
+		.elems = elems,
+		.conn = &link->u.mgd.conn,
+	};
 	struct ieee80211_sub_if_data *sdata = link->sdata;
+	struct ieee80211_chanctx_conf *chanctx_conf;
 	struct ieee80211_chan_req chanreq = {};
-	struct cfg80211_chan_def ap_chandef;
 	enum ieee80211_conn_mode ap_mode;
 	const char *frame;
-	u32 vht_cap_info = 0;
 	u16 ht_opmode;
 	int ret;
 
@@ -1364,11 +1376,10 @@ static int ieee80211_config_bw(struct ieee80211_link_data *link,
 		return 0;
 
 	if (elems->vht_cap_elem)
-		vht_cap_info = le32_to_cpu(elems->vht_cap_elem->vht_cap_info);
+		ap_chan_data.vht_cap_info =
+			le32_to_cpu(elems->vht_cap_elem->vht_cap_info);
 
-	ap_mode = ieee80211_determine_ap_chan(sdata, channel, vht_cap_info,
-					      elems, true, &link->u.mgd.conn,
-					      &ap_chandef);
+	ap_mode = ieee80211_determine_ap_chan(sdata, &ap_chan_data);
 
 	if (ap_mode != link->u.mgd.conn.mode) {
 		link_info(link,
@@ -1418,8 +1429,55 @@ static int ieee80211_config_bw(struct ieee80211_link_data *link,
 		}
 	}
 
-	if (ieee80211_chanreq_identical(&chanreq, &link->conf->chanreq))
+	/*
+	 * Beacons don't have the full information - we need to track
+	 * critical updates for NPCA parameters etc. For now only handle
+	 * association and link reconfiguration response.
+	 */
+	if (stype != IEEE80211_STYPE_BEACON &&
+	    chanreq.oper.npca_chan && elems->uhr_operation &&
+	    ieee80211_uhr_oper_size_ok((const void *)elems->uhr_operation,
+				       elems->uhr_operation_len,
+				       false)) {
+		const struct ieee80211_uhr_npca_info *npca;
+		struct ieee80211_bss_npca_params params = {};
+
+		npca = ieee80211_uhr_npca_info(elems->uhr_operation);
+		if (!npca) {
+			chanreq.oper.npca_chan = NULL;
+			chanreq.oper.npca_punctured = 0;
+		} else {
+			params.min_dur_thresh =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_MIN_DUR_THRESH);
+			params.switch_delay =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_SWITCH_DELAY);
+			params.switch_back_delay =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_SWITCH_BACK_DELAY);
+			params.init_qsrc =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_INIT_QSRC);
+			params.moplen =
+				le32_get_bits(npca->params,
+					      IEEE80211_UHR_NPCA_PARAMS_MOPLEN);
+			/* don't change the enabled bit yet */
+			params.enabled = link->conf->npca.enabled;
+		}
+
+		if (memcmp(&params, &link->conf->npca, sizeof(params)) ||
+		    !update) {
+			link->conf->npca = params;
+			*changed |= BSS_CHANGED_NPCA;
+		}
+	}
+
+	if (ieee80211_chanreq_identical(&chanreq, &link->conf->chanreq)) {
+		if (update)
+			goto update_npca;
 		return 0;
+	}
 
 	link_info(link,
 		  "AP %pM changed bandwidth in %s, new used config is %d.%03d MHz, width %d (%d.%03d/%d MHz)\n",
@@ -1466,6 +1524,24 @@ static int ieee80211_config_bw(struct ieee80211_link_data *link,
 	}
 
 	cfg80211_schedule_channels_check(&sdata->wdev);
+
+update_npca:
+	chanctx_conf = sdata_dereference(link->conf->chanctx_conf, sdata);
+	/* must be non-NULL when update is true */
+	if (WARN_ON(!chanctx_conf))
+		return -EINVAL;
+
+	/*
+	 * If we're not associated yet (i.e. in the process associating)
+	 * then the chanctx code won't have enabled NPCA in the link, so
+	 * if the channel context was set up with NPCA for us, enable it.
+	 */
+	if (chanreq.oper.npca_chan && chanctx_conf->def.npca_chan &&
+	    !link->conf->npca.enabled && !sdata->vif.cfg.assoc) {
+		link->conf->npca.enabled = true;
+		*changed |= BSS_CHANGED_NPCA;
+	}
+
 	return 0;
 }
 
@@ -2234,8 +2310,7 @@ ieee80211_link_common_elems_size(struct ieee80211_sub_if_data *sdata,
 		sizeof(struct ieee80211_eht_mcs_nss_supp) +
 		IEEE80211_EHT_PPE_THRES_MAX_LEN;
 
-	size += 2 + 1 + sizeof(struct ieee80211_uhr_cap) +
-		sizeof(struct ieee80211_uhr_cap_phy);
+	size += 2 + 1 + sizeof(struct ieee80211_uhr_cap);
 
 	return size;
 }
@@ -2580,8 +2655,9 @@ static void ieee80211_csa_switch_work(struct wiphy *wiphy,
 			return;
 
 		link_sta->pub->bandwidth =
-			_ieee80211_sta_cur_vht_bw(link_sta,
-						  &link->csa.chanreq.oper);
+			ieee80211_sta_current_bw(link_sta,
+						 &link->csa.chanreq.oper,
+						 IEEE80211_STA_BW_TX_TO_STA);
 		return;
 	}
 
@@ -5757,20 +5833,13 @@ static bool ieee80211_assoc_config_link(struct ieee80211_link_data *link,
 	 * next beacon and update then.
 	 */
 
-	/*
-	 * If an operating mode notification IE is present, override the
-	 * NSS calculation (that would be done in rate_control_rate_init())
-	 * and use the # of streams from that element.
-	 */
-	if (elems->opmode_notif &&
-	    !(*elems->opmode_notif & IEEE80211_OPMODE_NOTIF_RX_NSS_TYPE_BF)) {
-		u8 nss;
+	ieee80211_sta_init_nss_bw_capa(link_sta, &bss_conf->chanreq.oper);
 
-		nss = *elems->opmode_notif & IEEE80211_OPMODE_NOTIF_RX_NSS_MASK;
-		nss >>= IEEE80211_OPMODE_NOTIF_RX_NSS_SHIFT;
-		nss += 1;
-		link_sta->pub->rx_nss = nss;
-	}
+	/* If an operating mode notification element is present, use it. */
+	if (elems->opmode_notif)
+		__ieee80211_vht_handle_opmode(sdata, link_sta,
+					      *elems->opmode_notif,
+					      sband->band);
 
 	/*
 	 * Always handle WMM once after association regardless
@@ -6477,6 +6546,12 @@ static bool ieee80211_assoc_success(struct ieee80211_sub_if_data *sdata,
 	sta->sta.spp_amsdu = assoc_data->spp_amsdu;
 
 	if (ieee80211_vif_is_mld(&sdata->vif)) {
+		if (!elems->ml_basic)
+			goto out_err;
+
+		sta->sta.ext_mld_capa_ops =
+			ieee80211_mle_get_ext_mld_capa_op((const void *)elems->ml_basic);
+
 		for (link_id = 0; link_id < IEEE80211_MLD_MAX_NUM_LINKS; link_id++) {
 			if (!assoc_data->link[link_id].bss)
 				continue;
@@ -6667,6 +6742,7 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata,
 		.type = le16_to_cpu(mgmt->frame_control) & IEEE80211_FCTL_TYPE,
 	};
 	struct ieee802_11_elems *elems;
+	struct sta_info *sta;
 	int ac;
 	const u8 *elem_start;
 	unsigned int elem_len;
@@ -6862,6 +6938,14 @@ static void ieee80211_rx_mgmt_assoc_resp(struct ieee80211_sub_if_data *sdata,
 		resp.ap_mld_addr = ap_mld_addr;
 	}
 
+	/*
+	 * If epp_peer set, unprotected (Re)Association Request/Response frames
+	 * are dropped, which ensures that the (re)association exchange is
+	 * encrypted over the air.
+	 */
+	sta = sta_info_get_bss(sdata, sdata->vif.cfg.ap_addr);
+	resp.assoc_encrypted = sta && sta->sta.epp_peer;
+
 	ieee80211_destroy_assoc_data(sdata,
 				     status_code == WLAN_STATUS_SUCCESS ?
 					ASSOC_SUCCESS :
@@ -8164,6 +8248,7 @@ ieee80211_parse_neg_ttlm(struct ieee80211_sub_if_data *sdata,
 					 "No active links for TID %d", tid);
 				return -EINVAL;
 			}
+			pos += map_size;
 		} else {
 			map = 0;
 		}
@@ -8182,13 +8267,13 @@ ieee80211_parse_neg_ttlm(struct ieee80211_sub_if_data *sdata,
 		default:
 			return -EINVAL;
 		}
-		pos += map_size;
 	}
 	return 0;
 }
 
-void ieee80211_process_neg_ttlm_req(struct ieee80211_sub_if_data *sdata,
-				    struct ieee80211_mgmt *mgmt, size_t len)
+static void ieee80211_process_neg_ttlm_req(struct ieee80211_sub_if_data *sdata,
+					   struct ieee80211_mgmt *mgmt,
+					   size_t len)
 {
 	u8 dialog_token, direction[IEEE80211_TTLM_MAX_CNT] = {}, i;
 	size_t ies_len;
@@ -8250,8 +8335,9 @@ void ieee80211_process_neg_ttlm_req(struct ieee80211_sub_if_data *sdata,
 	ieee80211_send_neg_ttlm_res(sdata, ttlm_res, dialog_token, &neg_ttlm);
 }
 
-void ieee80211_process_neg_ttlm_res(struct ieee80211_sub_if_data *sdata,
-				    struct ieee80211_mgmt *mgmt, size_t len)
+static void ieee80211_process_neg_ttlm_res(struct ieee80211_sub_if_data *sdata,
+					   struct ieee80211_mgmt *mgmt,
+					   size_t len)
 {
 	if (!ieee80211_vif_is_mld(&sdata->vif) ||
 	    mgmt->u.action.ttlm_res.dialog_token != sdata->u.mgd.dialog_token_alloc)
@@ -8272,7 +8358,7 @@ void ieee80211_process_neg_ttlm_res(struct ieee80211_sub_if_data *sdata,
 		__ieee80211_disconnect(sdata);
 }
 
-void ieee80211_process_ttlm_teardown(struct ieee80211_sub_if_data *sdata)
+static void ieee80211_process_ttlm_teardown(struct ieee80211_sub_if_data *sdata)
 {
 	u16 new_dormant_links;
 
@@ -8330,8 +8416,8 @@ void ieee80211_send_teardown_neg_ttlm(struct ieee80211_vif *vif)
 }
 EXPORT_SYMBOL(ieee80211_send_teardown_neg_ttlm);
 
-void ieee80211_sta_rx_queued_ext(struct ieee80211_sub_if_data *sdata,
-				 struct sk_buff *skb)
+static void ieee80211_sta_rx_queued_ext(struct ieee80211_sub_if_data *sdata,
+					struct sk_buff *skb)
 {
 	struct ieee80211_link_data *link = &sdata->deflink;
 	struct ieee80211_rx_status *rx_status;
@@ -8351,129 +8437,6 @@ void ieee80211_sta_rx_queued_ext(struct ieee80211_sub_if_data *sdata,
 	}
 }
 
-void ieee80211_sta_rx_queued_mgmt(struct ieee80211_sub_if_data *sdata,
-				  struct sk_buff *skb)
-{
-	struct ieee80211_link_data *link = &sdata->deflink;
-	struct ieee80211_rx_status *rx_status;
-	struct ieee802_11_elems *elems;
-	struct ieee80211_mgmt *mgmt;
-	u16 fc;
-	int ies_len;
-
-	lockdep_assert_wiphy(sdata->local->hw.wiphy);
-
-	rx_status = (struct ieee80211_rx_status *) skb->cb;
-	mgmt = (struct ieee80211_mgmt *) skb->data;
-	fc = le16_to_cpu(mgmt->frame_control);
-
-	if (rx_status->link_valid) {
-		link = sdata_dereference(sdata->link[rx_status->link_id],
-					 sdata);
-		if (!link)
-			return;
-	}
-
-	switch (fc & IEEE80211_FCTL_STYPE) {
-	case IEEE80211_STYPE_BEACON:
-		ieee80211_rx_mgmt_beacon(link, (void *)mgmt,
-					 skb->len, rx_status);
-		break;
-	case IEEE80211_STYPE_PROBE_RESP:
-		ieee80211_rx_mgmt_probe_resp(link, skb);
-		break;
-	case IEEE80211_STYPE_AUTH:
-		ieee80211_rx_mgmt_auth(sdata, mgmt, skb->len);
-		break;
-	case IEEE80211_STYPE_DEAUTH:
-		ieee80211_rx_mgmt_deauth(sdata, mgmt, skb->len);
-		break;
-	case IEEE80211_STYPE_DISASSOC:
-		ieee80211_rx_mgmt_disassoc(sdata, mgmt, skb->len);
-		break;
-	case IEEE80211_STYPE_ASSOC_RESP:
-	case IEEE80211_STYPE_REASSOC_RESP:
-		ieee80211_rx_mgmt_assoc_resp(sdata, mgmt, skb->len);
-		break;
-	case IEEE80211_STYPE_ACTION:
-		if (!sdata->u.mgd.associated ||
-		    !ether_addr_equal(mgmt->bssid, sdata->vif.cfg.ap_addr))
-			break;
-
-		switch (mgmt->u.action.category) {
-		case WLAN_CATEGORY_SPECTRUM_MGMT:
-			ies_len = skb->len -
-				  offsetof(struct ieee80211_mgmt,
-					   u.action.chan_switch.variable);
-
-			if (ies_len < 0)
-				break;
-
-			/* CSA IE cannot be overridden, no need for BSSID */
-			elems = ieee802_11_parse_elems(mgmt->u.action.chan_switch.variable,
-						       ies_len,
-						       IEEE80211_FTYPE_MGMT |
-						       IEEE80211_STYPE_ACTION,
-						       NULL);
-
-			if (elems && !elems->parse_error) {
-				enum ieee80211_csa_source src =
-					IEEE80211_CSA_SOURCE_PROT_ACTION;
-
-				ieee80211_sta_process_chanswitch(link,
-								 rx_status->mactime,
-								 rx_status->device_timestamp,
-								 elems, elems,
-								 src);
-			}
-			kfree(elems);
-			break;
-		case WLAN_CATEGORY_PUBLIC:
-		case WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION:
-			ies_len = skb->len -
-				  offsetof(struct ieee80211_mgmt,
-					   u.action.ext_chan_switch.variable);
-
-			if (ies_len < 0)
-				break;
-
-			/*
-			 * extended CSA IE can't be overridden, no need for
-			 * BSSID
-			 */
-			elems = ieee802_11_parse_elems(mgmt->u.action.ext_chan_switch.variable,
-						       ies_len,
-						       IEEE80211_FTYPE_MGMT |
-						       IEEE80211_STYPE_ACTION,
-						       NULL);
-
-			if (elems && !elems->parse_error) {
-				enum ieee80211_csa_source src;
-
-				if (mgmt->u.action.category ==
-						WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION)
-					src = IEEE80211_CSA_SOURCE_PROT_ACTION;
-				else
-					src = IEEE80211_CSA_SOURCE_UNPROT_ACTION;
-
-				/* for the handling code pretend it was an IE */
-				elems->ext_chansw_ie =
-					&mgmt->u.action.ext_chan_switch.data;
-
-				ieee80211_sta_process_chanswitch(link,
-								 rx_status->mactime,
-								 rx_status->device_timestamp,
-								 elems, elems,
-								 src);
-			}
-
-			kfree(elems);
-			break;
-		}
-		break;
-	}
-}
-
 static void ieee80211_sta_timer(struct timer_list *t)
 {
 	struct ieee80211_sub_if_data *sdata =
@@ -9857,7 +9820,9 @@ int ieee80211_mgd_assoc(struct ieee80211_sub_if_data *sdata,
 	struct ieee80211_mgd_assoc_data *assoc_data;
 	const struct element *ssid_elem;
 	struct ieee80211_vif_cfg *vif_cfg = &sdata->vif.cfg;
+	const struct wiphy_iftype_ext_capab *ift_ext_capa;
 	struct ieee80211_link_data *link;
+	u16 driver_ext_mld_capa_ops = 0;
 	struct cfg80211_bss *cbss;
 	bool override, uapsd_supported;
 	bool match_auth;
@@ -9896,17 +9861,26 @@ int ieee80211_mgd_assoc(struct ieee80211_sub_if_data *sdata,
 	else
 		memcpy(assoc_data->ap_addr, cbss->bssid, ETH_ALEN);
 
+	ift_ext_capa = cfg80211_get_iftype_ext_capa(local->hw.wiphy,
+						    ieee80211_vif_type_p2p(&sdata->vif));
+	if (ift_ext_capa)
+		driver_ext_mld_capa_ops = ift_ext_capa->ext_mld_capa_and_ops;
+
 	/*
 	 * Many APs have broken parsing of the extended MLD capa/ops field,
 	 * dropping (re-)association request frames or replying with association
 	 * response with a failure status if it's present.
 	 * Set our value from the userspace request only in strict mode or if
 	 * the AP also had that field present.
+	 * For UHR we may want to advertise ML-PM (per driver_ext_mld_capa_ops)
+	 * but if the AP doesn't have it then it's pointless, and if it does
+	 * then it has to have the extended MLD capa/ops field.
 	 */
 	if (ieee80211_hw_check(&local->hw, STRICT) ||
 	    ieee80211_mgd_assoc_bss_has_mld_ext_capa_ops(req))
 		assoc_data->ext_mld_capa_ops =
-			cpu_to_le16(req->ext_mld_capa_ops);
+			cpu_to_le16(req->ext_mld_capa_ops |
+				    driver_ext_mld_capa_ops);
 
 	if (ifmgd->associated) {
 		u8 frame_buf[IEEE80211_DEAUTH_FRAME_LEN];
@@ -10437,8 +10411,9 @@ void ieee80211_disable_rssi_reports(struct ieee80211_vif *vif)
 }
 EXPORT_SYMBOL(ieee80211_disable_rssi_reports);
 
-void ieee80211_process_ml_reconf_resp(struct ieee80211_sub_if_data *sdata,
-				      struct ieee80211_mgmt *mgmt, size_t len)
+static void
+ieee80211_process_ml_reconf_resp(struct ieee80211_sub_if_data *sdata,
+				 struct ieee80211_mgmt *mgmt, size_t len)
 {
 	struct ieee80211_local *local = sdata->local;
 	struct ieee80211_if_managed *ifmgd = &sdata->u.mgd;
@@ -10625,7 +10600,6 @@ void ieee80211_process_ml_reconf_resp(struct ieee80211_sub_if_data *sdata,
 		if (add_links_data->link[link_id].status != WLAN_STATUS_SUCCESS)
 			goto disconnect;
 
-		ieee80211_sta_init_nss(link_sta);
 		if (ieee80211_sta_activate_link(sta, link_id))
 			goto disconnect;
 
@@ -10901,11 +10875,13 @@ ieee80211_build_ml_reconf_req(struct ieee80211_sub_if_data *sdata,
 int ieee80211_mgd_assoc_ml_reconf(struct ieee80211_sub_if_data *sdata,
 				  struct cfg80211_ml_reconf_req *req)
 {
+	const struct wiphy_iftype_ext_capab *ift_ext_capa;
 	struct ieee80211_local *local = sdata->local;
 	struct ieee80211_mgd_assoc_data *data = NULL;
 	struct sta_info *sta;
 	struct sk_buff *skb;
 	u16 added_links, new_valid_links;
+	u16 driver_ext_mld_capa_ops = 0;
 	int link_id, err;
 
 	if (!ieee80211_vif_is_mld(&sdata->vif) ||
@@ -11063,6 +11039,11 @@ int ieee80211_mgd_assoc_ml_reconf(struct ieee80211_sub_if_data *sdata,
 		}
 	}
 
+	ift_ext_capa = cfg80211_get_iftype_ext_capa(local->hw.wiphy,
+						    ieee80211_vif_type_p2p(&sdata->vif));
+	if (ift_ext_capa)
+		driver_ext_mld_capa_ops = ift_ext_capa->ext_mld_capa_and_ops;
+
 	/* Build the SKB before the link removal as the construction of the
 	 * station info for removed links requires the local address.
 	 * Invalidate the removed links, so that the transmission of the ML
@@ -11071,7 +11052,8 @@ int ieee80211_mgd_assoc_ml_reconf(struct ieee80211_sub_if_data *sdata,
 	 * on which the request was received.
 	 */
 	skb = ieee80211_build_ml_reconf_req(sdata, data, req->rem_links,
-					    cpu_to_le16(req->ext_mld_capa_ops));
+					    cpu_to_le16(req->ext_mld_capa_ops |
+							driver_ext_mld_capa_ops));
 	if (!skb) {
 		err = -ENOMEM;
 		goto err_free;
@@ -11232,6 +11214,9 @@ static void ieee80211_ml_epcs(struct ieee80211_sub_if_data *sdata,
 		control = get_unaligned_le16(pos);
 		link_id = control & IEEE80211_MLE_STA_EPCS_CONTROL_LINK_ID;
 
+		if (link_id >= IEEE80211_MLD_MAX_NUM_LINKS)
+			continue;
+
 		link = sdata_dereference(sdata->link[link_id], sdata);
 		if (!link)
 			continue;
@@ -11262,8 +11247,9 @@ static void ieee80211_ml_epcs(struct ieee80211_sub_if_data *sdata,
 	}
 }
 
-void ieee80211_process_epcs_ena_resp(struct ieee80211_sub_if_data *sdata,
-				     struct ieee80211_mgmt *mgmt, size_t len)
+static void ieee80211_process_epcs_ena_resp(struct ieee80211_sub_if_data *sdata,
+					    struct ieee80211_mgmt *mgmt,
+					    size_t len)
 {
 	struct ieee802_11_elems *elems __free(kfree) = NULL;
 	size_t ies_len;
@@ -11310,8 +11296,9 @@ void ieee80211_process_epcs_ena_resp(struct ieee80211_sub_if_data *sdata,
 	ieee80211_epcs_changed(sdata, true);
 }
 
-void ieee80211_process_epcs_teardown(struct ieee80211_sub_if_data *sdata,
-				     struct ieee80211_mgmt *mgmt, size_t len)
+static void ieee80211_process_epcs_teardown(struct ieee80211_sub_if_data *sdata,
+					    struct ieee80211_mgmt *mgmt,
+					    size_t len)
 {
 	if (!ieee80211_vif_is_mld(&sdata->vif) ||
 	    !sdata->u.mgd.epcs.enabled)
@@ -11320,3 +11307,161 @@ void ieee80211_process_epcs_teardown(struct ieee80211_sub_if_data *sdata,
 	ieee80211_epcs_teardown(sdata);
 	ieee80211_epcs_changed(sdata, false);
 }
+
+void ieee80211_sta_rx_queued_frame(struct ieee80211_sub_if_data *sdata,
+				   struct sk_buff *skb)
+{
+	struct ieee80211_link_data *link = &sdata->deflink;
+	struct ieee80211_rx_status *rx_status;
+	struct ieee802_11_elems *elems;
+	struct ieee80211_mgmt *mgmt;
+	u16 fc;
+	int ies_len;
+
+	lockdep_assert_wiphy(sdata->local->hw.wiphy);
+
+	mgmt = (struct ieee80211_mgmt *) skb->data;
+
+	if (ieee80211_is_ext(mgmt->frame_control)) {
+		ieee80211_sta_rx_queued_ext(sdata, skb);
+		return;
+	}
+
+	rx_status = (struct ieee80211_rx_status *) skb->cb;
+	fc = le16_to_cpu(mgmt->frame_control);
+
+	if (rx_status->link_valid) {
+		link = sdata_dereference(sdata->link[rx_status->link_id],
+					 sdata);
+		if (!link)
+			return;
+	}
+
+	switch (fc & IEEE80211_FCTL_STYPE) {
+	case IEEE80211_STYPE_BEACON:
+		ieee80211_rx_mgmt_beacon(link, (void *)mgmt,
+					 skb->len, rx_status);
+		break;
+	case IEEE80211_STYPE_PROBE_RESP:
+		ieee80211_rx_mgmt_probe_resp(link, skb);
+		break;
+	case IEEE80211_STYPE_AUTH:
+		ieee80211_rx_mgmt_auth(sdata, mgmt, skb->len);
+		break;
+	case IEEE80211_STYPE_DEAUTH:
+		ieee80211_rx_mgmt_deauth(sdata, mgmt, skb->len);
+		break;
+	case IEEE80211_STYPE_DISASSOC:
+		ieee80211_rx_mgmt_disassoc(sdata, mgmt, skb->len);
+		break;
+	case IEEE80211_STYPE_ASSOC_RESP:
+	case IEEE80211_STYPE_REASSOC_RESP:
+		ieee80211_rx_mgmt_assoc_resp(sdata, mgmt, skb->len);
+		break;
+	case IEEE80211_STYPE_ACTION:
+		if (!sdata->u.mgd.associated ||
+		    !ether_addr_equal(mgmt->bssid, sdata->vif.cfg.ap_addr))
+			break;
+
+		switch (mgmt->u.action.category) {
+		case WLAN_CATEGORY_SPECTRUM_MGMT:
+			ies_len = skb->len -
+				  offsetof(struct ieee80211_mgmt,
+					   u.action.chan_switch.variable);
+
+			if (ies_len < 0)
+				break;
+
+			/* CSA IE cannot be overridden, no need for BSSID */
+			elems = ieee802_11_parse_elems(mgmt->u.action.chan_switch.variable,
+						       ies_len,
+						       IEEE80211_FTYPE_MGMT |
+						       IEEE80211_STYPE_ACTION,
+						       NULL);
+
+			if (elems && !elems->parse_error) {
+				enum ieee80211_csa_source src =
+					IEEE80211_CSA_SOURCE_PROT_ACTION;
+
+				ieee80211_sta_process_chanswitch(link,
+								 rx_status->mactime,
+								 rx_status->device_timestamp,
+								 elems, elems,
+								 src);
+			}
+			kfree(elems);
+			break;
+		case WLAN_CATEGORY_PUBLIC:
+		case WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION:
+			ies_len = skb->len -
+				  offsetof(struct ieee80211_mgmt,
+					   u.action.ext_chan_switch.variable);
+
+			if (ies_len < 0)
+				break;
+
+			/*
+			 * extended CSA IE can't be overridden, no need for
+			 * BSSID
+			 */
+			elems = ieee802_11_parse_elems(mgmt->u.action.ext_chan_switch.variable,
+						       ies_len,
+						       IEEE80211_FTYPE_MGMT |
+						       IEEE80211_STYPE_ACTION,
+						       NULL);
+
+			if (elems && !elems->parse_error) {
+				enum ieee80211_csa_source src;
+
+				if (mgmt->u.action.category ==
+						WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION)
+					src = IEEE80211_CSA_SOURCE_PROT_ACTION;
+				else
+					src = IEEE80211_CSA_SOURCE_UNPROT_ACTION;
+
+				/* for the handling code pretend it was an IE */
+				elems->ext_chansw_ie =
+					&mgmt->u.action.ext_chan_switch.data;
+
+				ieee80211_sta_process_chanswitch(link,
+								 rx_status->mactime,
+								 rx_status->device_timestamp,
+								 elems, elems,
+								 src);
+			}
+
+			kfree(elems);
+			break;
+		case WLAN_CATEGORY_PROTECTED_EHT:
+			switch (mgmt->u.action.action_code) {
+			case WLAN_PROTECTED_EHT_ACTION_TTLM_REQ:
+				ieee80211_process_neg_ttlm_req(sdata, mgmt,
+							       skb->len);
+				break;
+			case WLAN_PROTECTED_EHT_ACTION_TTLM_RES:
+				ieee80211_process_neg_ttlm_res(sdata, mgmt,
+							       skb->len);
+				break;
+			case WLAN_PROTECTED_EHT_ACTION_TTLM_TEARDOWN:
+				ieee80211_process_ttlm_teardown(sdata);
+				break;
+			case WLAN_PROTECTED_EHT_ACTION_LINK_RECONFIG_RESP:
+				ieee80211_process_ml_reconf_resp(sdata, mgmt,
+								 skb->len);
+				break;
+			case WLAN_PROTECTED_EHT_ACTION_EPCS_ENABLE_RESP:
+				ieee80211_process_epcs_ena_resp(sdata, mgmt,
+								skb->len);
+				break;
+			case WLAN_PROTECTED_EHT_ACTION_EPCS_ENABLE_TEARDOWN:
+				ieee80211_process_epcs_teardown(sdata, mgmt,
+								skb->len);
+				break;
+			default:
+				break;
+			}
+			break;
+		}
+		break;
+	}
+}
diff --git a/net/mac80211/nan.c b/net/mac80211/nan.c
index 4e262b624521d..cea620aaee6ad 100644
--- a/net/mac80211/nan.c
+++ b/net/mac80211/nan.c
@@ -334,7 +334,10 @@ int ieee80211_nan_set_local_sched(struct ieee80211_sub_if_data *sdata,
 			sched_idx_to_chan[i] = chan;
 			ieee80211_nan_init_channel(chan,
 						   &sched->nan_channels[i]);
+		}
 
+		/* Also a pre-existing channel might have been ULWed, so no chanctx */
+		if (!chan->chanctx_conf) {
 			ret = ieee80211_nan_use_chanctx(sdata, chan, false);
 			if (ret) {
 				memset(chan, 0, sizeof(*chan));
@@ -708,3 +711,126 @@ int ieee80211_nan_set_peer_sched(struct ieee80211_sub_if_data *sdata,
 	ieee80211_nan_free_peer_sched(to_free);
 	return ret;
 }
+
+static void
+ieee80211_nan_evacuate_channel(struct ieee80211_sub_if_data *sdata,
+			       struct ieee80211_nan_channel *nan_channel)
+{
+	struct ieee80211_chanctx_conf *conf;
+	struct ieee80211_chanctx *ctx;
+
+	lockdep_assert_wiphy(sdata->local->hw.wiphy);
+
+	if (WARN_ON(!nan_channel || !nan_channel->chanreq.oper.chan))
+		return;
+
+	conf = nan_channel->chanctx_conf;
+	if (WARN_ON(!conf))
+		return;
+
+	nan_channel->chanctx_conf = NULL;
+
+	/* Update all peer channels that reference this chanctx */
+	ieee80211_nan_update_peer_channels(sdata, conf);
+
+	drv_vif_cfg_changed(sdata->local, sdata, BSS_CHANGED_NAN_LOCAL_SCHED);
+
+	cfg80211_nan_channel_evac(&sdata->wdev, &nan_channel->chanreq.oper,
+				  GFP_KERNEL);
+
+	/* Update NDI carrier states */
+	ieee80211_nan_update_all_ndi_carriers(sdata->local);
+
+	/* Clean up the channel context if no longer used */
+	ctx = container_of(conf, struct ieee80211_chanctx, conf);
+
+	if (ieee80211_chanctx_num_assigned(sdata->local, ctx) > 0) {
+		ieee80211_recalc_chanctx_chantype(sdata->local, ctx);
+		ieee80211_recalc_smps_chanctx(sdata->local, ctx);
+		ieee80211_recalc_chanctx_min_def(sdata->local, ctx);
+	}
+
+	if (ieee80211_chanctx_refcount(sdata->local, ctx) == 0)
+		ieee80211_free_chanctx(sdata->local, ctx, false);
+}
+
+bool ieee80211_nan_try_evacuate(struct ieee80211_hw *hw,
+				struct ieee80211_chanctx_conf *conf)
+{
+	struct ieee80211_sub_if_data *sdata = NULL, *tmp;
+	struct ieee80211_local *local = hw_to_local(hw);
+	struct ieee80211_nan_channel *evac_chan = NULL;
+	struct ieee80211_nan_sched_cfg *sched_cfg;
+	struct ieee80211_chanctx *ctx = NULL;
+	int min_slot_count = INT_MAX;
+	int usable_channels = 0;
+
+	lockdep_assert_wiphy(local->hw.wiphy);
+
+	if (conf)
+		ctx = container_of(conf, struct ieee80211_chanctx, conf);
+
+	/* Find the NAN interface - there can only be one */
+	list_for_each_entry(tmp, &local->interfaces, list) {
+		if (ieee80211_sdata_running(tmp) &&
+		    tmp->vif.type == NL80211_IFTYPE_NAN) {
+			sdata = tmp;
+			break;
+		}
+	}
+
+	if (!sdata)
+		return false;
+
+	sched_cfg = &sdata->vif.cfg.nan_sched;
+
+	/* Find the channel to evacuate and count usable channels */
+	for (int i = 0; i < IEEE80211_NAN_MAX_CHANNELS; i++) {
+		struct ieee80211_nan_channel *chan =
+			&sched_cfg->channels[i];
+		struct ieee80211_chanctx *chan_ctx;
+		int slot_count = 0;
+
+		if (!chan->chanreq.oper.chan || !chan->chanctx_conf)
+			continue;
+
+		usable_channels++;
+
+		chan_ctx = container_of(chan->chanctx_conf,
+					struct ieee80211_chanctx, conf);
+
+		/* If ctx specified, only consider that specific chanctx */
+		if (ctx) {
+			if (chan_ctx == ctx)
+				evac_chan = chan;
+			continue;
+		}
+
+		/* Can only evacuate channels whose chanctx is NAN-only */
+		if (ieee80211_chanctx_refcount(local, chan_ctx) > 1)
+			continue;
+
+		/* Count how many time slots use this channel */
+		for (int s = 0; s < CFG80211_NAN_SCHED_NUM_TIME_SLOTS; s++)
+			if (sched_cfg->schedule[s] == chan)
+				slot_count++;
+
+		if (slot_count < min_slot_count) {
+			min_slot_count = slot_count;
+			evac_chan = chan;
+		}
+	}
+
+	/* No suitable NAN channel found */
+	if (!evac_chan)
+		return false;
+
+	/* NAN needs at least one remaining usable channel after evacuation */
+	if (usable_channels < 2)
+		return false;
+
+	ieee80211_nan_evacuate_channel(sdata, evac_chan);
+
+	return true;
+}
+EXPORT_SYMBOL(ieee80211_nan_try_evacuate);
diff --git a/net/mac80211/ocb.c b/net/mac80211/ocb.c
index ebb4f4d88c237..e9bf4ba3e60b7 100644
--- a/net/mac80211/ocb.c
+++ b/net/mac80211/ocb.c
@@ -4,7 +4,7 @@
  *
  * Copyright: (c) 2014 Czech Technical University in Prague
  *            (c) 2014 Volkswagen Group Research
- * Copyright (C) 2022 - 2024 Intel Corporation
+ * Copyright (C) 2022 - 2024, 2026 Intel Corporation
  * Author:    Rostislav Lisovy <rostislav.lisovy@fel.cvut.cz>
  * Funded by: Volkswagen Group Research
  */
@@ -92,6 +92,9 @@ static struct sta_info *ieee80211_ocb_finish_sta(struct sta_info *sta)
 
 	memcpy(addr, sta->sta.addr, ETH_ALEN);
 
+	ieee80211_sta_init_nss_bw_capa(&sta->deflink,
+				       &sdata->deflink.conf->chanreq.oper);
+
 	ocb_dbg(sdata, "Adding new IBSS station %pM (dev=%s)\n",
 		addr, sdata->name);
 
diff --git a/net/mac80211/offchannel.c b/net/mac80211/offchannel.c
index f60f6a58948b1..8bec39b099a09 100644
--- a/net/mac80211/offchannel.c
+++ b/net/mac80211/offchannel.c
@@ -8,7 +8,7 @@
  * Copyright 2006-2007	Jiri Benc <jbenc@suse.cz>
  * Copyright 2007, Michael Wu <flamingice@sourmilk.net>
  * Copyright 2009	Johannes Berg <johannes@sipsolutions.net>
- * Copyright (C) 2019, 2022-2025 Intel Corporation
+ * Copyright (C) 2019, 2022-2026 Intel Corporation
  */
 #include <linux/export.h>
 #include <net/mac80211.h>
@@ -706,7 +706,8 @@ static int ieee80211_start_roc_work(struct ieee80211_local *local,
 
 int ieee80211_remain_on_channel(struct wiphy *wiphy, struct wireless_dev *wdev,
 				struct ieee80211_channel *chan,
-				unsigned int duration, u64 *cookie)
+				unsigned int duration, u64 *cookie,
+				const u8 *rx_addr)
 {
 	struct ieee80211_sub_if_data *sdata = IEEE80211_WDEV_TO_SUB_IF(wdev);
 	struct ieee80211_local *local = sdata->local;
@@ -894,10 +895,15 @@ int ieee80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 		}
 		break;
 	case NL80211_IFTYPE_P2P_DEVICE:
+	case NL80211_IFTYPE_PD:
 		need_offchan = true;
 		break;
 	case NL80211_IFTYPE_NAN:
 		break;
+	case NL80211_IFTYPE_NAN_DATA:
+		if (is_multicast_ether_addr(mgmt->da))
+			return -EOPNOTSUPP;
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -911,7 +917,8 @@ int ieee80211_mgmt_tx(struct wiphy *wiphy, struct wireless_dev *wdev,
 	/* Check if the operating channel is the requested channel */
 	if (!params->chan && mlo_sta) {
 		need_offchan = false;
-	} else if (sdata->vif.type == NL80211_IFTYPE_NAN) {
+	} else if (sdata->vif.type == NL80211_IFTYPE_NAN ||
+		   sdata->vif.type == NL80211_IFTYPE_NAN_DATA) {
 		/* Frames can be sent during NAN schedule */
 	} else if (!need_offchan) {
 		struct ieee80211_chanctx_conf *chanctx_conf = NULL;
diff --git a/net/mac80211/parse.c b/net/mac80211/parse.c
index 2b3632c6008af..8b30e361b6229 100644
--- a/net/mac80211/parse.c
+++ b/net/mac80211/parse.c
@@ -34,6 +34,22 @@
 #include "led.h"
 #include "wep.h"
 
+static const u8 empty_non_inheritance[] = {
+	WLAN_EID_EXTENSION, 1, WLAN_EID_EXT_NON_INHERITANCE,
+	/*
+	 * cfg80211_is_element_inherited() hardcodes elements that
+	 * cannot be inherited, so we just need an empty one to be
+	 * calling it at all.
+	 */
+};
+
+struct ieee80211_elem_defrag {
+	const struct element *elem;
+	/* container start/len */
+	const u8 *start;
+	size_t len;
+};
+
 struct ieee80211_elems_parse {
 	/* must be first for kfree to work */
 	struct ieee802_11_elems elems;
@@ -41,11 +57,7 @@ struct ieee80211_elems_parse {
 	/* The basic Multi-Link element in the original elements */
 	const struct element *ml_basic_elem;
 
-	/* The reconfiguration Multi-Link element in the original elements */
-	const struct element *ml_reconf_elem;
-
-	/* The EPCS Multi-Link element in the original elements */
-	const struct element *ml_epcs_elem;
+	struct ieee80211_elem_defrag ml_reconf, ml_epcs;
 
 	bool multi_link_inner;
 	bool skip_vendor;
@@ -162,10 +174,14 @@ ieee80211_parse_extension_element(u32 *crc,
 				}
 				break;
 			case IEEE80211_ML_CONTROL_TYPE_RECONF:
-				elems_parse->ml_reconf_elem = elem;
+				elems_parse->ml_reconf.elem = elem;
+				elems_parse->ml_reconf.start = params->start;
+				elems_parse->ml_reconf.len = params->len;
 				break;
 			case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS:
-				elems_parse->ml_epcs_elem = elem;
+				elems_parse->ml_epcs.elem = elem;
+				elems_parse->ml_epcs.start = params->start;
+				elems_parse->ml_epcs.len = params->len;
 				break;
 			default:
 				break;
@@ -916,7 +932,7 @@ ieee80211_prep_mle_link_parse(struct ieee80211_elems_parse *elems_parse,
 {
 	struct ieee802_11_elems *elems = &elems_parse->elems;
 	struct ieee80211_mle_per_sta_profile *prof;
-	const struct element *tmp;
+	const struct element *tmp, *ret;
 	ssize_t ml_len;
 	const u8 *end;
 
@@ -986,50 +1002,40 @@ ieee80211_prep_mle_link_parse(struct ieee80211_elems_parse *elems_parse,
 	sub->from_ap = params->from_ap;
 	sub->link_id = -1;
 
-	return cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE,
-				      sub->start, sub->len);
-}
-
-static void
-ieee80211_mle_defrag_reconf(struct ieee80211_elems_parse *elems_parse)
-{
-	struct ieee802_11_elems *elems = &elems_parse->elems;
-	ssize_t ml_len;
+	ret = cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE,
+				     sub->start, sub->len);
+	if (ret)
+		return ret;
 
-	ml_len = cfg80211_defragment_element(elems_parse->ml_reconf_elem,
-					     elems->ie_start,
-					     elems->total_len,
-					     elems_parse->scratch_pos,
-					     elems_parse->scratch +
-						elems_parse->scratch_len -
-						elems_parse->scratch_pos,
-					     WLAN_EID_FRAGMENT);
-	if (ml_len < 0)
-		return;
-	elems->ml_reconf = (void *)elems_parse->scratch_pos;
-	elems->ml_reconf_len = ml_len;
-	elems_parse->scratch_pos += ml_len;
+	/*
+	 * Since we know we want and found a profile, apply an empty
+	 * non-inheritance if the profile didn't have one, so that any
+	 * element that shouldn't be inherited by spec isn't.
+	 */
+	return (const void *)empty_non_inheritance;
 }
 
-static void
-ieee80211_mle_defrag_epcs(struct ieee80211_elems_parse *elems_parse)
+static const void *
+ieee80211_mle_defrag(struct ieee80211_elems_parse *elems_parse,
+		     struct ieee80211_elem_defrag *defrag,
+		     size_t *out_len)
 {
-	struct ieee802_11_elems *elems = &elems_parse->elems;
+	const void *ret;
 	ssize_t ml_len;
 
-	ml_len = cfg80211_defragment_element(elems_parse->ml_epcs_elem,
-					     elems->ie_start,
-					     elems->total_len,
+	ml_len = cfg80211_defragment_element(defrag->elem,
+					     defrag->start, defrag->len,
 					     elems_parse->scratch_pos,
 					     elems_parse->scratch +
 						elems_parse->scratch_len -
 						elems_parse->scratch_pos,
 					     WLAN_EID_FRAGMENT);
 	if (ml_len < 0)
-		return;
-	elems->ml_epcs = (void *)elems_parse->scratch_pos;
-	elems->ml_epcs_len = ml_len;
+		return NULL;
+	ret = elems_parse->scratch_pos;
+	*out_len = ml_len;
 	elems_parse->scratch_pos += ml_len;
+	return ret;
 }
 
 struct ieee802_11_elems *
@@ -1042,6 +1048,7 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params)
 	size_t scratch_len = 3 * params->len;
 	bool multi_link_inner = false;
 
+	BUILD_BUG_ON(sizeof(empty_non_inheritance) != empty_non_inheritance[1] + 2);
 	BUILD_BUG_ON(offsetof(typeof(*elems_parse), elems) != 0);
 
 	/* cannot parse for both a specific link and non-transmitted BSS */
@@ -1053,6 +1060,9 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params)
 	if (!elems_parse)
 		return NULL;
 
+	elems_parse->elems.frame_type = params->type;
+	elems_parse->elems.from_ap = params->from_ap;
+
 	elems_parse->scratch_len = scratch_len;
 	elems_parse->scratch_pos = elems_parse->scratch;
 
@@ -1089,6 +1099,17 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params)
 
 		non_inherit = cfg80211_find_ext_elem(WLAN_EID_EXT_NON_INHERITANCE,
 						     sub.start, nontx_len);
+		/*
+		 * If it's a non-transmitted BSS, we shouldn't pick
+		 * any elements in the outer parsing that shouldn't
+		 * be inherited. If the profile has a non-inheritance
+		 * element this automatically happens, but if not then
+		 * provide an empty one so that the hard-coded elements
+		 * in cfg80211_is_element_inherited() are ignored, but
+		 * it must be called.
+		 */
+		if (params->bss->transmitted_bss && !non_inherit)
+			non_inherit = (const void *)empty_non_inheritance;
 	} else {
 		/* must always parse to get elems_parse->ml_basic_elem */
 		non_inherit = ieee80211_prep_mle_link_parse(elems_parse, params,
@@ -1109,9 +1130,12 @@ ieee802_11_parse_elems_full(struct ieee80211_elems_parse_params *params)
 		_ieee802_11_parse_elems_full(&sub, elems_parse, NULL);
 	}
 
-	ieee80211_mle_defrag_reconf(elems_parse);
-
-	ieee80211_mle_defrag_epcs(elems_parse);
+	elems->ml_reconf = ieee80211_mle_defrag(elems_parse,
+						&elems_parse->ml_reconf,
+						&elems->ml_reconf_len);
+	elems->ml_epcs = ieee80211_mle_defrag(elems_parse,
+					      &elems_parse->ml_epcs,
+					      &elems->ml_epcs_len);
 
 	if (elems->tim && !elems->parse_error) {
 		const struct ieee80211_tim_ie *tim_ie = elems->tim;
diff --git a/net/mac80211/rate.c b/net/mac80211/rate.c
index 31af7dd6aedc2..ba1a3aa3f5d4a 100644
--- a/net/mac80211/rate.c
+++ b/net/mac80211/rate.c
@@ -4,7 +4,7 @@
  * Copyright 2005-2006, Devicescape Software, Inc.
  * Copyright (c) 2006 Jiri Benc <jbenc@suse.cz>
  * Copyright 2017	Intel Deutschland GmbH
- * Copyright (C) 2019, 2022-2025 Intel Corporation
+ * Copyright (C) 2019, 2022-2026 Intel Corporation
  */
 
 #include <linux/kernel.h>
@@ -38,8 +38,6 @@ void rate_control_rate_init(struct link_sta_info *link_sta)
 	struct ieee80211_supported_band *sband;
 	struct ieee80211_chanctx_conf *chanctx_conf;
 
-	ieee80211_sta_init_nss(link_sta);
-
 	if (!ref)
 		return;
 
diff --git a/net/mac80211/rx.c b/net/mac80211/rx.c
index d18e962126ce4..ef6086b183f71 100644
--- a/net/mac80211/rx.c
+++ b/net/mac80211/rx.c
@@ -4624,16 +4624,24 @@ static bool ieee80211_accept_frame(struct ieee80211_rx_data *rx)
 		    ieee80211_has_fromds(hdr->frame_control))
 			return false;
 
-		/* Accept only frames that are addressed to the NAN cluster
+		/*
+		 * Accept only frames that are addressed to the NAN cluster
 		 * (based on the Cluster ID). From these frames, accept only
-		 * action frames or authentication frames that are addressed to
-		 * the local NAN interface.
+		 *  - public action frames,
+		 *  - authentication frames to the local address, and
+		 *  - robust management frames except disassoc.
 		 */
-		return memcmp(sdata->wdev.u.nan.cluster_id,
-			      hdr->addr3, ETH_ALEN) == 0 &&
-			(ieee80211_is_public_action(hdr, skb->len) ||
-			 (ieee80211_is_auth(hdr->frame_control) &&
-			  ether_addr_equal(sdata->vif.addr, hdr->addr1)));
+		if (!ether_addr_equal(sdata->u.nan.conf.cluster_id, hdr->addr3))
+			return false;
+		if (ieee80211_is_public_action(hdr, skb->len))
+			return true;
+		if (ieee80211_is_auth(hdr->frame_control) &&
+		    ether_addr_equal(sdata->vif.addr, hdr->addr1))
+			return true;
+		if (!ieee80211_is_disassoc(hdr->frame_control) &&
+		    ieee80211_is_robust_mgmt_frame(skb))
+			return true;
+		return false;
 	case NL80211_IFTYPE_NAN_DATA:
 		if (ieee80211_has_tods(hdr->frame_control) ||
 		    ieee80211_has_fromds(hdr->frame_control))
@@ -4646,7 +4654,7 @@ static bool ieee80211_accept_frame(struct ieee80211_rx_data *rx)
 			if (!nmi)
 				return false;
 
-			if (!ether_addr_equal(nmi->wdev.u.nan.cluster_id,
+			if (!ether_addr_equal(nmi->u.nan.conf.cluster_id,
 					      hdr->addr3))
 				return false;
 
@@ -4664,6 +4672,13 @@ static bool ieee80211_accept_frame(struct ieee80211_rx_data *rx)
 		/* Unicast secure management frames */
 		return ether_addr_equal(sdata->vif.addr, hdr->addr1) &&
 		       ieee80211_is_unicast_robust_mgmt_frame(skb);
+	case NL80211_IFTYPE_PD:
+		/*
+		 * Accept only authentication frames (PASN) addressed to
+		 * this interface.
+		 */
+		return ieee80211_is_auth(hdr->frame_control) &&
+		       ether_addr_equal(sdata->vif.addr, hdr->addr1);
 	default:
 		break;
 	}
@@ -4984,6 +4999,7 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx,
 		u8 sa[ETH_ALEN];
 	} addrs __aligned(2);
 	struct ieee80211_sta_rx_stats *stats;
+	u32 encoded_rate;
 
 	/* for parallel-rx, we need to have DUP_VALIDATED, otherwise we write
 	 * to a common data structure; drivers can implement that per queue
@@ -5091,11 +5107,14 @@ static bool ieee80211_invoke_fast_rx(struct ieee80211_rx_data *rx,
 	/* push the addresses in front */
 	memcpy(skb_push(skb, sizeof(addrs)), &addrs, sizeof(addrs));
 
+	/* capture before mesh forward may memset or free skb->cb */
+	encoded_rate = sta_stats_encode_rate(status);
+
 	res = ieee80211_rx_mesh_data(rx->sdata, rx->sta, rx->skb);
 	switch (res) {
 	case RX_QUEUED:
 		stats->last_rx = jiffies;
-		stats->last_rate = sta_stats_encode_rate(status);
+		stats->last_rate = encoded_rate;
 		return true;
 	case RX_CONTINUE:
 		break;
diff --git a/net/mac80211/sta_info.c b/net/mac80211/sta_info.c
index 4c31ef8817ce0..0ea37016cd4f7 100644
--- a/net/mac80211/sta_info.c
+++ b/net/mac80211/sta_info.c
@@ -572,11 +572,13 @@ static int sta_info_alloc_link(struct ieee80211_local *local,
 	link_info->rx_omi_bw_tx = IEEE80211_STA_RX_BW_MAX;
 	link_info->rx_omi_bw_staging = IEEE80211_STA_RX_BW_MAX;
 
+	link_info->op_mode_bw = IEEE80211_STA_RX_BW_MAX;
+
 	/*
 	 * Cause (a) warning(s) if IEEE80211_STA_RX_BW_MAX != 320
 	 * or if new values are added to the enum.
 	 */
-	switch (link_info->cur_max_bandwidth) {
+	switch (link_info->op_mode_bw) {
 	case IEEE80211_STA_RX_BW_20:
 	case IEEE80211_STA_RX_BW_40:
 	case IEEE80211_STA_RX_BW_80:
@@ -3424,6 +3426,121 @@ void ieee80211_sta_remove_link(struct sta_info *sta, unsigned int link_id)
 	sta_remove_link(sta, link_id, true);
 }
 
+static u8 ieee80211_sta_nss_capability(struct link_sta_info *link_sta)
+{
+	u8 ht_rx_nss = 0, vht_rx_nss = 0, he_rx_nss = 0, eht_rx_nss = 0, rx_nss;
+	bool support_160;
+
+	if (link_sta->pub->eht_cap.has_eht) {
+		int i;
+		const u8 *rx_nss_mcs = (void *)&link_sta->pub->eht_cap.eht_mcs_nss_supp;
+
+		/* get the max nss for EHT over all possible bandwidths and mcs */
+		for (i = 0; i < sizeof(struct ieee80211_eht_mcs_nss_supp); i++)
+			eht_rx_nss = max_t(u8, eht_rx_nss,
+					   u8_get_bits(rx_nss_mcs[i],
+						       IEEE80211_EHT_MCS_NSS_RX));
+	}
+
+	if (link_sta->pub->he_cap.has_he) {
+		int i;
+		u8 rx_mcs_80 = 0, rx_mcs_160 = 0;
+		const struct ieee80211_sta_he_cap *he_cap = &link_sta->pub->he_cap;
+		u16 mcs_160_map =
+			le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_160);
+		u16 mcs_80_map = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_80);
+
+		for (i = 7; i >= 0; i--) {
+			u8 mcs_160 = (mcs_160_map >> (2 * i)) & 3;
+
+			if (mcs_160 != IEEE80211_HE_MCS_NOT_SUPPORTED) {
+				rx_mcs_160 = i + 1;
+				break;
+			}
+		}
+		for (i = 7; i >= 0; i--) {
+			u8 mcs_80 = (mcs_80_map >> (2 * i)) & 3;
+
+			if (mcs_80 != IEEE80211_HE_MCS_NOT_SUPPORTED) {
+				rx_mcs_80 = i + 1;
+				break;
+			}
+		}
+
+		support_160 = he_cap->he_cap_elem.phy_cap_info[0] &
+			      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G;
+
+		if (support_160)
+			he_rx_nss = min(rx_mcs_80, rx_mcs_160);
+		else
+			he_rx_nss = rx_mcs_80;
+	}
+
+	if (link_sta->pub->ht_cap.ht_supported) {
+		if (link_sta->pub->ht_cap.mcs.rx_mask[0])
+			ht_rx_nss++;
+		if (link_sta->pub->ht_cap.mcs.rx_mask[1])
+			ht_rx_nss++;
+		if (link_sta->pub->ht_cap.mcs.rx_mask[2])
+			ht_rx_nss++;
+		if (link_sta->pub->ht_cap.mcs.rx_mask[3])
+			ht_rx_nss++;
+		/* FIXME: consider rx_highest? */
+	}
+
+	if (link_sta->pub->vht_cap.vht_supported) {
+		int i;
+		u16 rx_mcs_map;
+
+		rx_mcs_map = le16_to_cpu(link_sta->pub->vht_cap.vht_mcs.rx_mcs_map);
+
+		for (i = 7; i >= 0; i--) {
+			u8 mcs = (rx_mcs_map >> (2 * i)) & 3;
+
+			if (mcs != IEEE80211_VHT_MCS_NOT_SUPPORTED) {
+				vht_rx_nss = i + 1;
+				break;
+			}
+		}
+		/* FIXME: consider rx_highest? */
+	}
+
+	rx_nss = max(vht_rx_nss, ht_rx_nss);
+	rx_nss = max(he_rx_nss, rx_nss);
+	rx_nss = max(eht_rx_nss, rx_nss);
+	rx_nss = max_t(u8, 1, rx_nss);
+
+	return rx_nss;
+}
+
+void ieee80211_sta_init_nss_bw_capa(struct link_sta_info *link_sta,
+				    struct cfg80211_chan_def *chandef)
+{
+	/*
+	 * TODO: The entirety of the STA Tx/Rx bandwidth handling
+	 * assumes 20MHz based widths, so for now don't initialise
+	 * pubsta->bandwidth for S1G bands. Since enum
+	 * ieee80211_sta_rx_bandwidth is ordered, we will probably
+	 * need to introduce ieee80211_s1g_sta_rx_bandwidth with
+	 * S1G widths and associated S1G specific code. Additionally,
+	 * existing S1G hardware is all 1SS, in the future if hardware
+	 * starts supporting >1SS this should be implemented in
+	 * ieee80211_sta_nss_capability().
+	 */
+	if (cfg80211_chandef_is_s1g(chandef)) {
+		link_sta->capa_nss = 1;
+		link_sta->pub->rx_nss = 1;
+		return;
+	}
+
+	link_sta->capa_nss = ieee80211_sta_nss_capability(link_sta);
+	link_sta->pub->rx_nss = link_sta->capa_nss;
+
+	link_sta->pub->bandwidth =
+		ieee80211_sta_current_bw(link_sta, chandef,
+					 IEEE80211_STA_BW_TX_TO_STA);
+}
+
 void ieee80211_sta_set_max_amsdu_subframes(struct sta_info *sta,
 					   const u8 *ext_capab,
 					   unsigned int ext_capab_len)
@@ -3456,3 +3573,195 @@ bool lockdep_sta_mutex_held(struct ieee80211_sta *pubsta)
 }
 EXPORT_SYMBOL(lockdep_sta_mutex_held);
 #endif
+
+/**
+ * ieee80211_sta_bw_capability - get STA's bandwidth capability
+ * @link_sta: the (link) STA to get the capability for
+ * @band: the band to get the capability on
+ *
+ * Return: the maximum bandwidth supported by the STA
+ */
+static enum ieee80211_sta_rx_bandwidth
+ieee80211_sta_bw_capability(struct link_sta_info *link_sta,
+			    enum nl80211_band band)
+{
+	struct ieee80211_sta_vht_cap *vht_cap = &link_sta->pub->vht_cap;
+	struct ieee80211_sta_he_cap *he_cap = &link_sta->pub->he_cap;
+	struct ieee80211_sta_eht_cap *eht_cap = &link_sta->pub->eht_cap;
+	u32 cap_width;
+
+	if (he_cap->has_he) {
+		u8 info;
+
+		if (eht_cap->has_eht && band == NL80211_BAND_6GHZ) {
+			info = eht_cap->eht_cap_elem.phy_cap_info[0];
+
+			if (info & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ)
+				return IEEE80211_STA_RX_BW_320;
+		}
+
+		info = he_cap->he_cap_elem.phy_cap_info[0];
+
+		if (band == NL80211_BAND_2GHZ) {
+			if (info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G)
+				return IEEE80211_STA_RX_BW_40;
+			return IEEE80211_STA_RX_BW_20;
+		}
+
+		if (info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G ||
+		    info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G)
+			return IEEE80211_STA_RX_BW_160;
+
+		if (info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G)
+			return IEEE80211_STA_RX_BW_80;
+
+		return IEEE80211_STA_RX_BW_20;
+	}
+
+	if (!vht_cap->vht_supported)
+		return link_sta->pub->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 ?
+				IEEE80211_STA_RX_BW_40 :
+				IEEE80211_STA_RX_BW_20;
+
+	cap_width = vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;
+
+	if (cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ ||
+	    cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ)
+		return IEEE80211_STA_RX_BW_160;
+
+	/*
+	 * If this is non-zero, then it does support 160 MHz after all,
+	 * in one form or the other. We don't distinguish here (or even
+	 * above) between 160 and 80+80 yet.
+	 */
+	if (vht_cap->cap & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK)
+		return IEEE80211_STA_RX_BW_160;
+
+	return IEEE80211_STA_RX_BW_80;
+}
+
+/**
+ * ieee80211_sta_usable_bw - get STA's usable bandwidth capability
+ * @link_sta: the (link) STA to get the capability for
+ * @band: the band to get the capability on
+ *
+ * If the STA is on an AP interface, take into account the AP's
+ * bandwidth corresponding to this station's PHY capability
+ *
+ * Return: the maximum bandwidth supported by the STA on the
+ *	connection to the interface it's connected to
+ */
+static enum ieee80211_sta_rx_bandwidth
+ieee80211_sta_usable_bw(struct link_sta_info *link_sta,
+			enum nl80211_band band)
+{
+	struct ieee80211_sub_if_data *sdata = link_sta->sta->sdata;
+	enum ieee80211_sta_rx_bandwidth bw;
+	struct ieee80211_link_data *link;
+
+	bw = ieee80211_sta_bw_capability(link_sta, band);
+
+	if (sdata->vif.type == NL80211_IFTYPE_AP_VLAN) {
+		sdata = get_bss_sdata(sdata);
+
+		/* for a STA to exist on VLAN, it must have AP */
+		if (WARN_ON(!sdata))
+			return IEEE80211_STA_RX_BW_20;
+	}
+
+	if (sdata->vif.type != NL80211_IFTYPE_AP)
+		return bw;
+
+	/* for a link STA to exist, vif must have the link */
+	link = sdata_dereference(sdata->link[link_sta->link_id], sdata);
+	if (WARN_ON(!link))
+		return IEEE80211_STA_RX_BW_20;
+
+	if (link_sta->pub->eht_cap.has_eht)
+		return bw;
+
+	return min(bw, link->bss_bw.he_and_lower);
+}
+
+static enum ieee80211_sta_rx_bandwidth
+ieee80211_sta_current_bw_rx_from_sta(struct link_sta_info *link_sta,
+				     struct cfg80211_chan_def *chandef)
+{
+	/*
+	 * Take RX OMI into account. The value "rx_omi_bw_rx" is what
+	 * we've indicated to the STA we can currently receive.
+	 *
+	 * This is needed since the RX OMI is done by us to save power,
+	 * requiring changing both our TX (rate control) and RX (chanctx),
+	 * which in turn needs to be done in the right order (stop TX
+	 * at a higher bandwidth first while reducing bandwidth, and
+	 * change the chanctx only after the peer accepts, etc.)
+	 */
+	return min(ieee80211_sta_usable_bw(link_sta, chandef->chan->band),
+		   link_sta->rx_omi_bw_rx);
+}
+
+static enum ieee80211_sta_rx_bandwidth
+ieee80211_sta_current_bw_tx_to_sta(struct link_sta_info *link_sta,
+				   struct cfg80211_chan_def *chandef)
+{
+	struct sta_info *sta = link_sta->sta;
+	enum nl80211_chan_width bss_width;
+	enum ieee80211_sta_rx_bandwidth bw;
+	enum nl80211_band band;
+
+	bss_width = chandef->width;
+	band = chandef->chan->band;
+
+	bw = ieee80211_sta_usable_bw(link_sta, band);
+	bw = min(bw, link_sta->op_mode_bw);
+	/* also limit to RX OMI bandwidth we TX to the STA */
+	bw = min(bw, link_sta->rx_omi_bw_tx);
+
+	/* Don't consider AP's bandwidth for TDLS peers, section 11.23.1 of
+	 * IEEE80211-2016 specification makes higher bandwidth operation
+	 * possible on the TDLS link if the peers have wider bandwidth
+	 * capability.
+	 *
+	 * However, in this case, and only if the TDLS peer is authorized,
+	 * limit to the tdls_chandef so that the configuration here isn't
+	 * wider than what's actually requested on the channel context.
+	 */
+	if (test_sta_flag(sta, WLAN_STA_TDLS_PEER) &&
+	    test_sta_flag(sta, WLAN_STA_TDLS_WIDER_BW) &&
+	    test_sta_flag(sta, WLAN_STA_AUTHORIZED) &&
+	    sta->tdls_chandef.chan)
+		bw = min(bw, ieee80211_chan_width_to_rx_bw(sta->tdls_chandef.width));
+	else
+		bw = min(bw, ieee80211_chan_width_to_rx_bw(bss_width));
+
+	return bw;
+}
+
+/**
+ * ieee80211_sta_current_bw - get STA's current usable bandwidth
+ * @link_sta: the (link) STA to get the bandwidth for
+ * @chandef: the chandef for the channel the STA is on
+ * @direction: the direction (to or from STA)
+ *
+ * Return: the maximum bandwidth that the station can/may
+ *	(currently) use in the given direction
+ */
+enum ieee80211_sta_rx_bandwidth
+ieee80211_sta_current_bw(struct link_sta_info *link_sta,
+			 struct cfg80211_chan_def *chandef,
+			 enum ieee80211_sta_bw_direction direction)
+{
+	if (WARN_ON(!chandef))
+		return IEEE80211_STA_RX_BW_20;
+
+	switch (direction) {
+	case IEEE80211_STA_BW_RX_FROM_STA:
+		return ieee80211_sta_current_bw_rx_from_sta(link_sta, chandef);
+	case IEEE80211_STA_BW_TX_TO_STA:
+		return ieee80211_sta_current_bw_tx_to_sta(link_sta, chandef);
+	}
+
+	/* unreachable */
+	return IEEE80211_STA_RX_BW_20;
+}
diff --git a/net/mac80211/sta_info.h b/net/mac80211/sta_info.h
index 3e5d003bd31f1..39608a0abbb59 100644
--- a/net/mac80211/sta_info.h
+++ b/net/mac80211/sta_info.h
@@ -504,7 +504,7 @@ struct ieee80211_fragment_cache {
  * @status_stats.last_ack_signal: last ACK signal
  * @status_stats.ack_signal_filled: last ACK signal validity
  * @status_stats.avg_ack_signal: average ACK signal
- * @cur_max_bandwidth: maximum bandwidth to use for TX to the station,
+ * @op_mode_bw: dynamic bandwidth limit to transmit to the STA,
  *	taken from HT/VHT capabilities or VHT operating mode notification.
  *	Invalid for NAN since that is operating on multiple bands.
  * @rx_omi_bw_rx: RX OMI bandwidth restriction to apply for RX
@@ -558,7 +558,7 @@ struct link_sta_info {
 		u64 msdu[IEEE80211_NUM_TIDS + 1];
 	} tx_stats;
 
-	enum ieee80211_sta_rx_bandwidth cur_max_bandwidth;
+	enum ieee80211_sta_rx_bandwidth op_mode_bw;
 	enum ieee80211_sta_rx_bandwidth rx_omi_bw_rx,
 					rx_omi_bw_tx,
 					rx_omi_bw_staging;
@@ -997,12 +997,24 @@ void ieee80211_sta_ps_deliver_uapsd(struct sta_info *sta);
 
 unsigned long ieee80211_sta_last_active(struct sta_info *sta, int link_id);
 
+void ieee80211_sta_init_nss_bw_capa(struct link_sta_info *link_sta,
+				    struct cfg80211_chan_def *chandef);
 void ieee80211_sta_set_max_amsdu_subframes(struct sta_info *sta,
 					   const u8 *ext_capab,
 					   unsigned int ext_capab_len);
 
 void __ieee80211_sta_recalc_aggregates(struct sta_info *sta, u16 active_links);
 
+enum ieee80211_sta_bw_direction {
+	IEEE80211_STA_BW_RX_FROM_STA,
+	IEEE80211_STA_BW_TX_TO_STA,
+};
+
+enum ieee80211_sta_rx_bandwidth
+ieee80211_sta_current_bw(struct link_sta_info *link_sta,
+			 struct cfg80211_chan_def *chandef,
+			 enum ieee80211_sta_bw_direction direction);
+
 enum sta_stats_type {
 	STA_STATS_RATE_TYPE_INVALID = 0,
 	STA_STATS_RATE_TYPE_LEGACY,
diff --git a/net/mac80211/status.c b/net/mac80211/status.c
index 4b38aa0e902a8..8716eda8317d1 100644
--- a/net/mac80211/status.c
+++ b/net/mac80211/status.c
@@ -5,7 +5,7 @@
  * Copyright 2006-2007	Jiri Benc <jbenc@suse.cz>
  * Copyright 2008-2010	Johannes Berg <johannes@sipsolutions.net>
  * Copyright 2013-2014  Intel Mobile Communications GmbH
- * Copyright 2021-2025  Intel Corporation
+ * Copyright 2021-2026  Intel Corporation
  */
 
 #include <linux/export.h>
@@ -295,9 +295,10 @@ ieee80211_add_tx_radiotap_header(struct ieee80211_local *local,
 						 RATE_INFO_FLAGS_VHT_MCS |
 						 RATE_INFO_FLAGS_HE_MCS)))
 			legacy_rate = status_rate->rate_idx.legacy;
-	} else if (info->status.rates[0].idx >= 0 &&
-		 !(info->status.rates[0].flags & (IEEE80211_TX_RC_MCS |
-						  IEEE80211_TX_RC_VHT_MCS))) {
+	} else if (info->band < NUM_NL80211_BANDS &&
+		   info->status.rates[0].idx >= 0 &&
+		   !(info->status.rates[0].flags & (IEEE80211_TX_RC_MCS |
+						    IEEE80211_TX_RC_VHT_MCS))) {
 		struct ieee80211_supported_band *sband;
 
 		sband = local->hw.wiphy->bands[info->band];
diff --git a/net/mac80211/tdls.c b/net/mac80211/tdls.c
index 44bba7c3e6039..ffd575a8d188f 100644
--- a/net/mac80211/tdls.c
+++ b/net/mac80211/tdls.c
@@ -311,17 +311,21 @@ ieee80211_tdls_chandef_vht_upgrade(struct ieee80211_sub_if_data *sdata,
 	/* IEEE802.11ac-2013 Table E-4 */
 	static const u16 centers_80mhz[] = { 5210, 5290, 5530, 5610, 5690, 5775 };
 	struct cfg80211_chan_def uc = sta->tdls_chandef;
-	enum nl80211_chan_width max_width =
-		ieee80211_sta_cap_chan_bw(&sta->deflink);
+	enum nl80211_chan_width max_width;
 	int i;
 
-	/* only support upgrading non-narrow channels up to 80Mhz */
-	if (max_width == NL80211_CHAN_WIDTH_5 ||
-	    max_width == NL80211_CHAN_WIDTH_10)
-		return;
-
-	if (max_width > NL80211_CHAN_WIDTH_80)
+	switch (ieee80211_sta_current_bw(&sta->deflink, &uc,
+					 IEEE80211_STA_BW_RX_FROM_STA)) {
+	case IEEE80211_STA_RX_BW_20:
+		max_width = NL80211_CHAN_WIDTH_20;
+		break;
+	case IEEE80211_STA_RX_BW_40:
+		max_width = NL80211_CHAN_WIDTH_40;
+		break;
+	default: /* 80 or higher, only support upgrade to 80 */
 		max_width = NL80211_CHAN_WIDTH_80;
+		break;
+	}
 
 	if (uc.width >= max_width)
 		return;
@@ -1334,7 +1338,9 @@ static void iee80211_tdls_recalc_chanctx(struct ieee80211_sub_if_data *sdata,
 			enum ieee80211_sta_rx_bandwidth bw;
 
 			bw = ieee80211_chan_width_to_rx_bw(conf->def.width);
-			bw = min(bw, ieee80211_sta_cap_rx_bw(&sta->deflink));
+			bw = min(bw, ieee80211_sta_current_bw(&sta->deflink,
+							      &conf->def,
+							      IEEE80211_STA_BW_RX_FROM_STA));
 			if (bw != sta->sta.deflink.bandwidth) {
 				sta->sta.deflink.bandwidth = bw;
 				rate_control_rate_update(local, sband,
diff --git a/net/mac80211/trace.h b/net/mac80211/trace.h
index 71cf88039bd46..562a4964afa3a 100644
--- a/net/mac80211/trace.h
+++ b/net/mac80211/trace.h
@@ -44,7 +44,9 @@
 			__field(u32, n##center_freq1)					\
 			__field(u32, n##freq1_offset)					\
 			__field(u32, n##center_freq2)					\
-			__field(u16, n##punctured)
+			__field(u16, n##punctured)					\
+			__field(u32, n##npca_pri_freq)					\
+			__field(u16, n##npca_punctured)
 #define __CHANDEF_ASSIGN(n, c)								\
 			__entry->n##control_freq = (c) && (c)->chan ?			\
 				(c)->chan->center_freq : 0;				\
@@ -54,14 +56,18 @@
 			__entry->n##center_freq1 = (c) ? (c)->center_freq1 : 0;		\
 			__entry->n##freq1_offset = (c) ? (c)->freq1_offset : 0;		\
 			__entry->n##center_freq2 = (c) ? (c)->center_freq2 : 0;		\
-			__entry->n##punctured = (c) ? (c)->punctured : 0;
+			__entry->n##punctured = (c) ? (c)->punctured : 0;		\
+			__entry->n##npca_pri_freq = (c) && (c)->npca_chan ?		\
+				(c)->npca_chan->center_freq : 0;			\
+			__entry->n##npca_punctured = (c) ? (c)->npca_punctured : 0;
 #define __CHANDEF_PR_FMT(n)								\
-	" " #n "(%d.%03d MHz,width:%d,center: %d.%03d/%d MHz, punct:0x%x)"
+	" " #n "(%d.%03d MHz,width:%d,center: %d.%03d/%d MHz, punct:0x%x, npca:%u, npca_punct:0x%x)"
 #define __CHANDEF_PR_ARG(n)								\
 			__entry->n##control_freq, __entry->n##freq_offset,		\
 			__entry->n##chan_width, __entry->n##center_freq1,		\
 			__entry->n##freq1_offset, __entry->n##center_freq2,		\
-			__entry->n##punctured
+			__entry->n##punctured, __entry->n##npca_pri_freq,		\
+			__entry->n##npca_punctured
 
 #define CHANDEF_ENTRY		__CHANDEF_ENTRY()
 #define CHANDEF_ASSIGN(c)	__CHANDEF_ASSIGN(, c)
diff --git a/net/mac80211/tx.c b/net/mac80211/tx.c
index b487d2330f25f..933c86ca21c38 100644
--- a/net/mac80211/tx.c
+++ b/net/mac80211/tx.c
@@ -5,7 +5,7 @@
  * Copyright 2006-2007	Jiri Benc <jbenc@suse.cz>
  * Copyright 2007	Johannes Berg <johannes@sipsolutions.net>
  * Copyright 2013-2014  Intel Mobile Communications GmbH
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  *
  * Transmit and frame generation functions.
  */
@@ -2854,7 +2854,7 @@ static struct sk_buff *ieee80211_build_hdr(struct ieee80211_sub_if_data *sdata,
 			ret = -ENOTCONN;
 			goto free;
 		}
-		memcpy(hdr.addr3, nmi->wdev.u.nan.cluster_id, ETH_ALEN);
+		memcpy(hdr.addr3, nmi->u.nan.conf.cluster_id, ETH_ALEN);
 		hdrlen = 24;
 		break;
 	}
@@ -3916,7 +3916,7 @@ struct sk_buff *ieee80211_tx_dequeue(struct ieee80211_hw *hw,
 		 * injected frames or EAPOL frames from the local station.
 		 */
 		if (unlikely(!(info->flags & IEEE80211_TX_CTL_INJECTED) &&
-			     ieee80211_is_data(hdr->frame_control) &&
+			     ieee80211_is_data_present(hdr->frame_control) &&
 			     !ieee80211_vif_is_mesh(&tx.sdata->vif) &&
 			     tx.sdata->vif.type != NL80211_IFTYPE_OCB &&
 			     !is_multicast_ether_addr(hdr->addr1) &&
@@ -5355,7 +5355,7 @@ static int ieee80211_beacon_protect(struct sk_buff *skb,
 int ieee80211_encrypt_tx_skb(struct sk_buff *skb)
 {
 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
-	struct ieee80211_sub_if_data *sdata;
+	struct ieee80211_sub_if_data *sdata = NULL;
 	struct sk_buff *check_skb;
 	struct ieee80211_tx_data tx;
 	ieee80211_tx_result res;
@@ -5370,7 +5370,14 @@ int ieee80211_encrypt_tx_skb(struct sk_buff *skb)
 	__skb_queue_head_init(&tx.skbs);
 	__skb_queue_tail(&tx.skbs, skb);
 
-	sdata = IEEE80211_DEV_TO_SUB_IF(skb->dev);
+	if (skb->dev)
+		sdata = IEEE80211_DEV_TO_SUB_IF(skb->dev);
+	else if (info->control.vif)
+		sdata = vif_to_sdata(info->control.vif);
+
+	if (WARN_ON(!sdata))
+		return -EINVAL;
+
 	tx.sdata = sdata;
 	tx.local = sdata->local;
 	res = ieee80211_tx_h_encrypt(&tx);
@@ -6377,7 +6384,8 @@ void ieee80211_tx_skb_tid(struct ieee80211_sub_if_data *sdata,
 	enum nl80211_band band;
 
 	rcu_read_lock();
-	if (sdata->vif.type == NL80211_IFTYPE_NAN) {
+	if (sdata->vif.type == NL80211_IFTYPE_NAN ||
+	    sdata->vif.type == NL80211_IFTYPE_NAN_DATA) {
 		band = NUM_NL80211_BANDS;
 	} else if (!ieee80211_vif_is_mld(&sdata->vif)) {
 		WARN_ON(link_id >= 0);
diff --git a/net/mac80211/uhr.c b/net/mac80211/uhr.c
index 2d8f5e5480ef4..ca3b471b7981b 100644
--- a/net/mac80211/uhr.c
+++ b/net/mac80211/uhr.c
@@ -15,7 +15,6 @@ ieee80211_uhr_cap_ie_to_sta_uhr_cap(struct ieee80211_sub_if_data *sdata,
 				    struct link_sta_info *link_sta)
 {
 	struct ieee80211_sta_uhr_cap *sta_uhr_cap = &link_sta->pub->uhr_cap;
-	bool from_ap;
 
 	memset(sta_uhr_cap, 0, sizeof(*sta_uhr_cap));
 
@@ -23,8 +22,6 @@ ieee80211_uhr_cap_ie_to_sta_uhr_cap(struct ieee80211_sub_if_data *sdata,
 		return;
 
 	sta_uhr_cap->has_uhr = true;
-
 	sta_uhr_cap->mac = uhr_cap->mac;
-	from_ap = sdata->vif.type == NL80211_IFTYPE_STATION;
-	sta_uhr_cap->phy = *ieee80211_uhr_phy_cap(uhr_cap, from_ap);
+	sta_uhr_cap->phy = uhr_cap->phy;
 }
diff --git a/net/mac80211/util.c b/net/mac80211/util.c
index 2529b01e2cd55..255905f971c8a 100644
--- a/net/mac80211/util.c
+++ b/net/mac80211/util.c
@@ -2147,6 +2147,9 @@ int ieee80211_reconfig(struct ieee80211_local *local)
 				ieee80211_bss_info_change_notify(sdata,
 								 changed);
 			} else if (!WARN_ON(!link)) {
+				if (link->conf->npca.enabled)
+					changed |= BSS_CHANGED_NPCA;
+
 				ieee80211_link_info_change_notify(sdata, link,
 								  changed);
 				changed = BSS_CHANGED_ASSOC |
@@ -2200,16 +2203,13 @@ int ieee80211_reconfig(struct ieee80211_local *local)
 			}
 			break;
 		case NL80211_IFTYPE_NAN:
-			res = ieee80211_reconfig_nan(sdata);
-			if (res < 0) {
-				ieee80211_handle_reconfig_failure(local);
-				return res;
-			}
+			WARN_ON(ieee80211_reconfig_nan(sdata));
 			break;
 		case NL80211_IFTYPE_NAN_DATA:
 		case NL80211_IFTYPE_AP_VLAN:
 		case NL80211_IFTYPE_MONITOR:
 		case NL80211_IFTYPE_P2P_DEVICE:
+		case NL80211_IFTYPE_PD:
 			/* nothing to do */
 			break;
 		case NL80211_IFTYPE_UNSPECIFIED:
@@ -3149,7 +3149,9 @@ bool ieee80211_chandef_vht_oper(struct ieee80211_hw *hw, u32 vht_cap_info,
 		ext_nss_bw_supp = 0;
 
 	/*
-	 * Cf. IEEE 802.11 Table 9-250
+	 * Cf. IEEE 802.11-2020 Table 9-272 - Setting of the Supported Channel
+	 * Width Set subfield and Extended NSS BW Support subfield at a STA
+	 * transmitting the VHT Capabilities Information field
 	 *
 	 * We really just consider that because it's inefficient to connect
 	 * at a higher bandwidth than we'll actually be able to use.
@@ -3829,6 +3831,10 @@ void ieee80211_chandef_downgrade(struct cfg80211_chan_def *c,
 		c->width = new_primary_width;
 	}
 
+	/* whatever we do, downgrading removes NPCA */
+	c->npca_chan = NULL;
+	c->npca_punctured = 0;
+
 	/*
 	 * With an 80 MHz channel, we might have the puncturing in the primary
 	 * 40 Mhz channel, but that's not valid when downgraded to 40 MHz width.
@@ -3840,6 +3846,25 @@ void ieee80211_chandef_downgrade(struct cfg80211_chan_def *c,
 	WARN_ON_ONCE(!cfg80211_chandef_valid(c));
 }
 
+enum nl80211_chan_width
+ieee80211_sta_rx_bw_to_chan_width(enum ieee80211_sta_rx_bandwidth bw)
+{
+	switch (bw) {
+	case IEEE80211_STA_RX_BW_20:
+		return NL80211_CHAN_WIDTH_20;
+	case IEEE80211_STA_RX_BW_40:
+		return NL80211_CHAN_WIDTH_40;
+	case IEEE80211_STA_RX_BW_80:
+		return NL80211_CHAN_WIDTH_80;
+	case IEEE80211_STA_RX_BW_160:
+		return NL80211_CHAN_WIDTH_160;
+	case IEEE80211_STA_RX_BW_320:
+		return NL80211_CHAN_WIDTH_320;
+	default:
+		return NL80211_CHAN_WIDTH_20;
+	}
+}
+
 int ieee80211_send_action_csa(struct ieee80211_sub_if_data *sdata,
 			      struct cfg80211_csa_settings *csa_settings)
 {
@@ -4619,8 +4644,7 @@ int ieee80211_put_uhr_cap(struct sk_buff *skb,
 	if (!uhr_cap)
 		return 0;
 
-	len = 2 + 1 + sizeof(struct ieee80211_uhr_cap) +
-	      sizeof(struct ieee80211_uhr_cap_phy);
+	len = 2 + 1 + sizeof(struct ieee80211_uhr_cap);
 
 	if (skb_tailroom(skb) < len)
 		return -ENOBUFS;
diff --git a/net/mac80211/vht.c b/net/mac80211/vht.c
index f3bb5a561a38a..92a4e04505936 100644
--- a/net/mac80211/vht.c
+++ b/net/mac80211/vht.c
@@ -301,32 +301,6 @@ ieee80211_vht_cap_ie_to_sta_vht_cap(struct ieee80211_sub_if_data *sdata,
 		return;
 	}
 
-	/* finally set up the bandwidth */
-	switch (vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK) {
-	case IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ:
-	case IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ:
-		link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_160;
-		break;
-	default:
-		link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_80;
-
-		if (!(vht_cap->vht_mcs.tx_highest &
-				cpu_to_le16(IEEE80211_VHT_EXT_NSS_BW_CAPABLE)))
-			break;
-
-		/*
-		 * If this is non-zero, then it does support 160 MHz after all,
-		 * in one form or the other. We don't distinguish here (or even
-		 * above) between 160 and 80+80 yet.
-		 */
-		if (cap_info & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK)
-			link_sta->cur_max_bandwidth =
-				IEEE80211_STA_RX_BW_160;
-	}
-
-	if (sdata->vif.type != NL80211_IFTYPE_NAN)
-		link_sta->pub->bandwidth = ieee80211_sta_cur_vht_bw(link_sta);
-
 	/*
 	 * Work around the Cisco 9115 FW 17.3 bug by taking the min of
 	 * both reported MPDU lengths.
@@ -357,319 +331,20 @@ ieee80211_vht_cap_ie_to_sta_vht_cap(struct ieee80211_sub_if_data *sdata,
 	ieee80211_sta_recalc_aggregates(&link_sta->sta->sta);
 }
 
-/* FIXME: move this to some better location - parses HE/EHT now */
-static enum ieee80211_sta_rx_bandwidth
-__ieee80211_sta_cap_rx_bw(struct link_sta_info *link_sta,
-			  struct cfg80211_chan_def *chandef)
-{
-	unsigned int link_id = link_sta->link_id;
-	struct ieee80211_sub_if_data *sdata = link_sta->sta->sdata;
-	struct ieee80211_sta_vht_cap *vht_cap = &link_sta->pub->vht_cap;
-	struct ieee80211_sta_he_cap *he_cap = &link_sta->pub->he_cap;
-	struct ieee80211_sta_eht_cap *eht_cap = &link_sta->pub->eht_cap;
-	u32 cap_width;
-
-	if (he_cap->has_he) {
-		enum nl80211_band band;
-		u8 info;
-
-		if (chandef) {
-			band = chandef->chan->band;
-		} else {
-			struct ieee80211_bss_conf *link_conf;
-
-			if (WARN_ON_ONCE(sdata->vif.type == NL80211_IFTYPE_NAN_DATA ||
-					 sdata->vif.type == NL80211_IFTYPE_NAN))
-				return IEEE80211_STA_RX_BW_20;
-
-			rcu_read_lock();
-			link_conf = rcu_dereference(sdata->vif.link_conf[link_id]);
-			band = link_conf->chanreq.oper.chan->band;
-			rcu_read_unlock();
-		}
-
-		if (eht_cap->has_eht && band == NL80211_BAND_6GHZ) {
-			info = eht_cap->eht_cap_elem.phy_cap_info[0];
-
-			if (info & IEEE80211_EHT_PHY_CAP0_320MHZ_IN_6GHZ)
-				return IEEE80211_STA_RX_BW_320;
-		}
-
-		info = he_cap->he_cap_elem.phy_cap_info[0];
-
-		if (band == NL80211_BAND_2GHZ) {
-			if (info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_IN_2G)
-				return IEEE80211_STA_RX_BW_40;
-			return IEEE80211_STA_RX_BW_20;
-		}
-
-		if (info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G ||
-		    info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_80PLUS80_MHZ_IN_5G)
-			return IEEE80211_STA_RX_BW_160;
-
-		if (info & IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_40MHZ_80MHZ_IN_5G)
-			return IEEE80211_STA_RX_BW_80;
-
-		return IEEE80211_STA_RX_BW_20;
-	}
-
-	if (!vht_cap->vht_supported)
-		return link_sta->pub->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 ?
-				IEEE80211_STA_RX_BW_40 :
-				IEEE80211_STA_RX_BW_20;
-
-	cap_width = vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;
-
-	if (cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ ||
-	    cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ)
-		return IEEE80211_STA_RX_BW_160;
-
-	/*
-	 * If this is non-zero, then it does support 160 MHz after all,
-	 * in one form or the other. We don't distinguish here (or even
-	 * above) between 160 and 80+80 yet.
-	 */
-	if (vht_cap->cap & IEEE80211_VHT_CAP_EXT_NSS_BW_MASK)
-		return IEEE80211_STA_RX_BW_160;
-
-	return IEEE80211_STA_RX_BW_80;
-}
-
-enum ieee80211_sta_rx_bandwidth
-_ieee80211_sta_cap_rx_bw(struct link_sta_info *link_sta,
-			 struct cfg80211_chan_def *chandef)
-{
-	/*
-	 * With RX OMI, also pretend that the STA's capability changed.
-	 * Of course this isn't really true, it didn't change, only our
-	 * RX capability was changed by notifying RX OMI to the STA.
-	 * The purpose, however, is to save power, and that requires
-	 * changing also transmissions to the AP and the chanctx. The
-	 * transmissions depend on link_sta->bandwidth which is set in
-	 * _ieee80211_sta_cur_vht_bw() below, but the chanctx depends
-	 * on the result of this function which is also called by
-	 * _ieee80211_sta_cur_vht_bw(), so we need to do that here as
-	 * well. This is sufficient for the steady state, but during
-	 * the transition we already need to change TX/RX separately,
-	 * so _ieee80211_sta_cur_vht_bw() below applies the _tx one.
-	 */
-	return min(__ieee80211_sta_cap_rx_bw(link_sta, chandef),
-		   link_sta->rx_omi_bw_rx);
-}
-
-enum nl80211_chan_width
-ieee80211_sta_cap_chan_bw(struct link_sta_info *link_sta)
-{
-	struct ieee80211_sta_vht_cap *vht_cap = &link_sta->pub->vht_cap;
-	u32 cap_width;
-
-	if (!vht_cap->vht_supported) {
-		if (!link_sta->pub->ht_cap.ht_supported)
-			return NL80211_CHAN_WIDTH_20_NOHT;
-
-		return link_sta->pub->ht_cap.cap & IEEE80211_HT_CAP_SUP_WIDTH_20_40 ?
-				NL80211_CHAN_WIDTH_40 : NL80211_CHAN_WIDTH_20;
-	}
-
-	cap_width = vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;
-
-	if (cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ)
-		return NL80211_CHAN_WIDTH_160;
-	else if (cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160_80PLUS80MHZ)
-		return NL80211_CHAN_WIDTH_80P80;
-
-	return NL80211_CHAN_WIDTH_80;
-}
-
-enum nl80211_chan_width
-ieee80211_sta_rx_bw_to_chan_width(struct link_sta_info *link_sta)
-{
-	enum ieee80211_sta_rx_bandwidth cur_bw =
-		link_sta->pub->bandwidth;
-	struct ieee80211_sta_vht_cap *vht_cap =
-		&link_sta->pub->vht_cap;
-	u32 cap_width;
-
-	switch (cur_bw) {
-	case IEEE80211_STA_RX_BW_20:
-		if (!link_sta->pub->ht_cap.ht_supported)
-			return NL80211_CHAN_WIDTH_20_NOHT;
-		else
-			return NL80211_CHAN_WIDTH_20;
-	case IEEE80211_STA_RX_BW_40:
-		return NL80211_CHAN_WIDTH_40;
-	case IEEE80211_STA_RX_BW_80:
-		return NL80211_CHAN_WIDTH_80;
-	case IEEE80211_STA_RX_BW_160:
-		cap_width =
-			vht_cap->cap & IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_MASK;
-
-		if (cap_width == IEEE80211_VHT_CAP_SUPP_CHAN_WIDTH_160MHZ)
-			return NL80211_CHAN_WIDTH_160;
-
-		return NL80211_CHAN_WIDTH_80P80;
-	default:
-		return NL80211_CHAN_WIDTH_20;
-	}
-}
-
-/* FIXME: rename/move - this deals with everything not just VHT */
-enum ieee80211_sta_rx_bandwidth
-_ieee80211_sta_cur_vht_bw(struct link_sta_info *link_sta,
-			  struct cfg80211_chan_def *chandef)
-{
-	struct sta_info *sta = link_sta->sta;
-	enum nl80211_chan_width bss_width;
-	enum ieee80211_sta_rx_bandwidth bw;
-
-	if (chandef) {
-		bss_width = chandef->width;
-	} else {
-		struct ieee80211_bss_conf *link_conf;
-
-		/* NAN operates on multiple channels so a chandef must be given */
-		if (WARN_ON_ONCE(sta->sdata->vif.type == NL80211_IFTYPE_NAN ||
-				 sta->sdata->vif.type == NL80211_IFTYPE_NAN_DATA))
-			return IEEE80211_STA_RX_BW_20;
-
-		rcu_read_lock();
-		link_conf = rcu_dereference(sta->sdata->vif.link_conf[link_sta->link_id]);
-		if (WARN_ON_ONCE(!link_conf)) {
-			rcu_read_unlock();
-			return IEEE80211_STA_RX_BW_20;
-		}
-		bss_width = link_conf->chanreq.oper.width;
-		rcu_read_unlock();
-	}
-
-	/* intentionally do not take rx_bw_omi_rx into account */
-	bw = __ieee80211_sta_cap_rx_bw(link_sta, chandef);
-	bw = min(bw, link_sta->cur_max_bandwidth);
-	/* but do apply rx_omi_bw_tx */
-	bw = min(bw, link_sta->rx_omi_bw_tx);
-
-	/* Don't consider AP's bandwidth for TDLS peers, section 11.23.1 of
-	 * IEEE80211-2016 specification makes higher bandwidth operation
-	 * possible on the TDLS link if the peers have wider bandwidth
-	 * capability.
-	 *
-	 * However, in this case, and only if the TDLS peer is authorized,
-	 * limit to the tdls_chandef so that the configuration here isn't
-	 * wider than what's actually requested on the channel context.
-	 */
-	if (test_sta_flag(sta, WLAN_STA_TDLS_PEER) &&
-	    test_sta_flag(sta, WLAN_STA_TDLS_WIDER_BW) &&
-	    test_sta_flag(sta, WLAN_STA_AUTHORIZED) &&
-	    sta->tdls_chandef.chan)
-		bw = min(bw, ieee80211_chan_width_to_rx_bw(sta->tdls_chandef.width));
-	else
-		bw = min(bw, ieee80211_chan_width_to_rx_bw(bss_width));
-
-	return bw;
-}
-
-void ieee80211_sta_init_nss(struct link_sta_info *link_sta)
-{
-	u8 ht_rx_nss = 0, vht_rx_nss = 0, he_rx_nss = 0, eht_rx_nss = 0, rx_nss;
-	bool support_160;
-
-	if (link_sta->pub->eht_cap.has_eht) {
-		int i;
-		const u8 *rx_nss_mcs = (void *)&link_sta->pub->eht_cap.eht_mcs_nss_supp;
-
-		/* get the max nss for EHT over all possible bandwidths and mcs */
-		for (i = 0; i < sizeof(struct ieee80211_eht_mcs_nss_supp); i++)
-			eht_rx_nss = max_t(u8, eht_rx_nss,
-					   u8_get_bits(rx_nss_mcs[i],
-						       IEEE80211_EHT_MCS_NSS_RX));
-	}
-
-	if (link_sta->pub->he_cap.has_he) {
-		int i;
-		u8 rx_mcs_80 = 0, rx_mcs_160 = 0;
-		const struct ieee80211_sta_he_cap *he_cap = &link_sta->pub->he_cap;
-		u16 mcs_160_map =
-			le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_160);
-		u16 mcs_80_map = le16_to_cpu(he_cap->he_mcs_nss_supp.rx_mcs_80);
-
-		for (i = 7; i >= 0; i--) {
-			u8 mcs_160 = (mcs_160_map >> (2 * i)) & 3;
-
-			if (mcs_160 != IEEE80211_HE_MCS_NOT_SUPPORTED) {
-				rx_mcs_160 = i + 1;
-				break;
-			}
-		}
-		for (i = 7; i >= 0; i--) {
-			u8 mcs_80 = (mcs_80_map >> (2 * i)) & 3;
-
-			if (mcs_80 != IEEE80211_HE_MCS_NOT_SUPPORTED) {
-				rx_mcs_80 = i + 1;
-				break;
-			}
-		}
-
-		support_160 = he_cap->he_cap_elem.phy_cap_info[0] &
-			      IEEE80211_HE_PHY_CAP0_CHANNEL_WIDTH_SET_160MHZ_IN_5G;
-
-		if (support_160)
-			he_rx_nss = min(rx_mcs_80, rx_mcs_160);
-		else
-			he_rx_nss = rx_mcs_80;
-	}
-
-	if (link_sta->pub->ht_cap.ht_supported) {
-		if (link_sta->pub->ht_cap.mcs.rx_mask[0])
-			ht_rx_nss++;
-		if (link_sta->pub->ht_cap.mcs.rx_mask[1])
-			ht_rx_nss++;
-		if (link_sta->pub->ht_cap.mcs.rx_mask[2])
-			ht_rx_nss++;
-		if (link_sta->pub->ht_cap.mcs.rx_mask[3])
-			ht_rx_nss++;
-		/* FIXME: consider rx_highest? */
-	}
-
-	if (link_sta->pub->vht_cap.vht_supported) {
-		int i;
-		u16 rx_mcs_map;
-
-		rx_mcs_map = le16_to_cpu(link_sta->pub->vht_cap.vht_mcs.rx_mcs_map);
-
-		for (i = 7; i >= 0; i--) {
-			u8 mcs = (rx_mcs_map >> (2 * i)) & 3;
-
-			if (mcs != IEEE80211_VHT_MCS_NOT_SUPPORTED) {
-				vht_rx_nss = i + 1;
-				break;
-			}
-		}
-		/* FIXME: consider rx_highest? */
-	}
-
-	rx_nss = max(vht_rx_nss, ht_rx_nss);
-	rx_nss = max(he_rx_nss, rx_nss);
-	rx_nss = max(eht_rx_nss, rx_nss);
-	rx_nss = max_t(u8, 1, rx_nss);
-	link_sta->capa_nss = rx_nss;
-
-	/* that shouldn't be set yet, but we can handle it anyway */
-	if (link_sta->op_mode_nss)
-		link_sta->pub->rx_nss =
-			min_t(u8, rx_nss, link_sta->op_mode_nss);
-	else
-		link_sta->pub->rx_nss = rx_nss;
-}
-
 u32 __ieee80211_vht_handle_opmode(struct ieee80211_sub_if_data *sdata,
 				  struct link_sta_info *link_sta,
 				  u8 opmode, enum nl80211_band band)
 {
 	enum ieee80211_sta_rx_bandwidth new_bw;
 	struct sta_opmode_info sta_opmode = {};
+	struct ieee80211_link_data *link;
 	u32 changed = 0;
 	u8 nss;
 
+	link = sdata_dereference(sdata->link[link_sta->link_id], sdata);
+	if (WARN_ON(!link))
+		return 0;
+
 	/* ignore - no support for BF yet */
 	if (opmode & IEEE80211_OPMODE_NOTIF_RX_NSS_TYPE_BF)
 		return 0;
@@ -698,28 +373,29 @@ u32 __ieee80211_vht_handle_opmode(struct ieee80211_sub_if_data *sdata,
 	switch (opmode & IEEE80211_OPMODE_NOTIF_CHANWIDTH_MASK) {
 	case IEEE80211_OPMODE_NOTIF_CHANWIDTH_20MHZ:
 		/* ignore IEEE80211_OPMODE_NOTIF_BW_160_80P80 must not be set */
-		link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_20;
+		link_sta->op_mode_bw = IEEE80211_STA_RX_BW_20;
 		break;
 	case IEEE80211_OPMODE_NOTIF_CHANWIDTH_40MHZ:
 		/* ignore IEEE80211_OPMODE_NOTIF_BW_160_80P80 must not be set */
-		link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_40;
+		link_sta->op_mode_bw = IEEE80211_STA_RX_BW_40;
 		break;
 	case IEEE80211_OPMODE_NOTIF_CHANWIDTH_80MHZ:
 		if (opmode & IEEE80211_OPMODE_NOTIF_BW_160_80P80)
-			link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_160;
+			link_sta->op_mode_bw = IEEE80211_STA_RX_BW_160;
 		else
-			link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_80;
+			link_sta->op_mode_bw = IEEE80211_STA_RX_BW_80;
 		break;
 	case IEEE80211_OPMODE_NOTIF_CHANWIDTH_160MHZ:
 		/* legacy only, no longer used by newer spec */
-		link_sta->cur_max_bandwidth = IEEE80211_STA_RX_BW_160;
+		link_sta->op_mode_bw = IEEE80211_STA_RX_BW_160;
 		break;
 	}
 
-	new_bw = ieee80211_sta_cur_vht_bw(link_sta);
+	new_bw = ieee80211_sta_current_bw(link_sta, &link->conf->chanreq.oper,
+					  IEEE80211_STA_BW_TX_TO_STA);
 	if (new_bw != link_sta->pub->bandwidth) {
 		link_sta->pub->bandwidth = new_bw;
-		sta_opmode.bw = ieee80211_sta_rx_bw_to_chan_width(link_sta);
+		sta_opmode.bw = ieee80211_sta_rx_bw_to_chan_width(new_bw);
 		changed |= IEEE80211_RC_BW_CHANGED;
 		sta_opmode.changed |= STA_OPMODE_MAX_BW_CHANGED;
 	}
diff --git a/net/mctp/af_mctp.c b/net/mctp/af_mctp.c
index 209a963112e3a..8af5e2b3c8d12 100644
--- a/net/mctp/af_mctp.c
+++ b/net/mctp/af_mctp.c
@@ -12,6 +12,7 @@
 #include <linux/mctp.h>
 #include <linux/module.h>
 #include <linux/socket.h>
+#include <linux/uio.h>
 
 #include <net/mctp.h>
 #include <net/mctpdevice.h>
@@ -405,7 +406,7 @@ static int mctp_setsockopt(struct socket *sock, int level, int optname,
 }
 
 static int mctp_getsockopt(struct socket *sock, int level, int optname,
-			   char __user *optval, int __user *optlen)
+			   sockopt_t *opt)
 {
 	struct mctp_sock *msk = container_of(sock->sk, struct mctp_sock, sk);
 	int len, val;
@@ -413,14 +414,13 @@ static int mctp_getsockopt(struct socket *sock, int level, int optname,
 	if (level != SOL_MCTP)
 		return -EINVAL;
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = opt->optlen;
 
 	if (optname == MCTP_OPT_ADDR_EXT) {
 		if (len != sizeof(int))
 			return -EINVAL;
 		val = !!msk->addr_ext;
-		if (copy_to_user(optval, &val, len))
+		if (copy_to_iter(&val, len, &opt->iter_out) != len)
 			return -EFAULT;
 		return 0;
 	}
@@ -639,7 +639,7 @@ static const struct proto_ops mctp_dgram_ops = {
 	.listen		= sock_no_listen,
 	.shutdown	= sock_no_shutdown,
 	.setsockopt	= mctp_setsockopt,
-	.getsockopt	= mctp_getsockopt,
+	.getsockopt_iter = mctp_getsockopt,
 	.sendmsg	= mctp_sendmsg,
 	.recvmsg	= mctp_recvmsg,
 	.mmap		= sock_no_mmap,
diff --git a/net/mctp/test/route-test.c b/net/mctp/test/route-test.c
index e4b230ef60996..c92e3abb40d77 100644
--- a/net/mctp/test/route-test.c
+++ b/net/mctp/test/route-test.c
@@ -859,12 +859,6 @@ static void mctp_test_route_input_cloned_frag(struct kunit *test)
 	skb[0]->data[4] = 0;
 	skb[3]->data[4] = 0;
 
-	skb_dump("pkt1 ", skb[0], false);
-	skb_dump("pkt2 ", skb[1], false);
-	skb_dump("pkt3 ", skb[2], false);
-	skb_dump("pkt4 ", skb[3], false);
-	skb_dump("pkt5 ", skb[4], false);
-
 	for (int i = 0; i < 5; i++) {
 		KUNIT_EXPECT_EQ(test, refcount_read(&skb[i]->users), 1);
 		/* Take a reference so we can check refcounts at the end */
diff --git a/net/mptcp/pm.c b/net/mptcp/pm.c
index 3c152bf66cd5a..3e770c7407e1f 100644
--- a/net/mptcp/pm.c
+++ b/net/mptcp/pm.c
@@ -364,7 +364,13 @@ static void mptcp_pm_add_timer(struct timer_list *timer)
 
 	spin_lock_bh(&msk->pm.lock);
 
-	if (!mptcp_pm_should_add_signal_addr(msk)) {
+	/* The cancel path (mptcp_pm_del_add_timer()) can race with this
+	 * callback. Once cancel updates retrans_times to MAX, suppress further
+	 * retransmissions here. If this callback acquires pm.lock first, one
+	 * final transmit attempt is still possible.
+	 */
+	if (entry->retrans_times < ADD_ADDR_RETRANS_MAX &&
+	    !mptcp_pm_should_add_signal_addr(msk)) {
 		pr_debug("retransmit ADD_ADDR id=%d\n", entry->addr.id);
 		mptcp_pm_announce_addr(msk, &entry->addr, false);
 		mptcp_pm_add_addr_send_ack(msk);
@@ -414,8 +420,12 @@ mptcp_pm_del_add_timer(struct mptcp_sock *msk,
 	/* Note: entry might have been removed by another thread.
 	 * We hold rcu_read_lock() to ensure it is not freed under us.
 	 */
-	if (stop_timer)
-		sk_stop_timer_sync(sk, &entry->add_timer);
+	if (stop_timer) {
+		if (check_id)
+			sk_stop_timer(sk, &entry->add_timer);
+		else
+			sk_stop_timer_sync(sk, &entry->add_timer);
+	}
 
 	rcu_read_unlock();
 	return entry;
@@ -882,6 +892,7 @@ bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
 			      struct mptcp_addr_info *addr, bool *echo,
 			      bool *drop_other_suboptions)
 {
+	bool skip_add_addr = false;
 	int ret = false;
 	u8 add_addr;
 	u8 family;
@@ -903,24 +914,49 @@ bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
 	}
 
 	*echo = mptcp_pm_should_add_signal_echo(msk);
-	port = !!(*echo ? msk->pm.remote.port : msk->pm.local.port);
-
-	family = *echo ? msk->pm.remote.family : msk->pm.local.family;
-	if (remaining < mptcp_add_addr_len(family, *echo, port))
-		goto out_unlock;
-
 	if (*echo) {
 		*addr = msk->pm.remote;
 		add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO);
+		port = !!msk->pm.remote.port;
+		family = msk->pm.remote.family;
 	} else {
 		*addr = msk->pm.local;
 		add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL);
+		port = !!msk->pm.local.port;
+		family = msk->pm.local.family;
 	}
-	WRITE_ONCE(msk->pm.addr_signal, add_addr);
+
+	if (remaining < mptcp_add_addr_len(family, *echo, port)) {
+		struct net *net = sock_net((struct sock *)msk);
+
+		if (!*drop_other_suboptions)
+			goto out_unlock;
+
+		if (*echo) {
+			MPTCP_INC_STATS(net, MPTCP_MIB_ECHOADDTXDROP);
+		} else {
+			skip_add_addr = true;
+			MPTCP_INC_STATS(net, MPTCP_MIB_ADDADDRTXDROP);
+		}
+		goto drop_signal_mark;
+	}
+
 	ret = true;
 
+drop_signal_mark:
+	WRITE_ONCE(msk->pm.addr_signal, add_addr);
+
 out_unlock:
 	spin_unlock_bh(&msk->pm.lock);
+
+	/* On pure-ACK option-space exhaustion, stop retrying this ADD_ADDR:
+	 * clear the signal bit, cancel the matching retransmission timer, and
+	 * let the PM state machine progress.
+	 */
+	if (skip_add_addr) {
+		mptcp_pm_del_add_timer(msk, addr, true);
+		mptcp_pm_subflow_established(msk);
+	}
 	return ret;
 }
 
diff --git a/net/mptcp/pm_kernel.c b/net/mptcp/pm_kernel.c
index fc818b63752e3..4ba4346d7adc7 100644
--- a/net/mptcp/pm_kernel.c
+++ b/net/mptcp/pm_kernel.c
@@ -30,6 +30,7 @@ struct pm_nl_pernet {
 };
 
 #define MPTCP_PM_ADDR_MAX	8
+#define MPTCP_PM_SUBFLOWS_MAX	64
 
 static struct pm_nl_pernet *pm_nl_get_pernet(const struct net *net)
 {
@@ -201,7 +202,8 @@ fill_remote_addr(struct mptcp_sock *msk, struct mptcp_addr_info *local,
 static unsigned int
 fill_remote_addresses_fullmesh(struct mptcp_sock *msk,
 			       struct mptcp_addr_info *local,
-			       struct mptcp_addr_info *addrs)
+			       struct mptcp_addr_info *addrs,
+			       int addrs_size)
 {
 	u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
 	bool deny_id0 = READ_ONCE(msk->pm.remote_deny_join_id0);
@@ -236,7 +238,8 @@ fill_remote_addresses_fullmesh(struct mptcp_sock *msk,
 		msk->pm.extra_subflows++;
 		i++;
 
-		if (msk->pm.extra_subflows >= limit_extra_subflows)
+		if (msk->pm.extra_subflows >= limit_extra_subflows ||
+		    i == addrs_size)
 			break;
 	}
 
@@ -248,7 +251,8 @@ fill_remote_addresses_fullmesh(struct mptcp_sock *msk,
  */
 static unsigned int
 fill_remote_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *local,
-			  bool fullmesh, struct mptcp_addr_info *addrs)
+			  bool fullmesh, struct mptcp_addr_info *addrs,
+			  int addrs_size)
 {
 	/* Non-fullmesh: fill in the single entry corresponding to the primary
 	 * MPC subflow remote address, and return 1, corresponding to 1 entry.
@@ -257,7 +261,7 @@ fill_remote_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *local,
 		return fill_remote_addr(msk, local, addrs);
 
 	/* Fullmesh endpoint: fill all possible remote addresses */
-	return fill_remote_addresses_fullmesh(msk, local, addrs);
+	return fill_remote_addresses_fullmesh(msk, local, addrs, addrs_size);
 }
 
 static struct mptcp_pm_addr_entry *
@@ -410,7 +414,8 @@ static void mptcp_pm_create_subflow_or_signal_addr(struct mptcp_sock *msk)
 		else /* local_addr_used is not decr for ID 0 */
 			msk->pm.local_addr_used++;
 
-		nr = fill_remote_addresses_vec(msk, &local.addr, fullmesh, addrs);
+		nr = fill_remote_addresses_vec(msk, &local.addr, fullmesh,
+					       addrs, ARRAY_SIZE(addrs));
 		if (nr == 0)
 			continue;
 
@@ -447,6 +452,7 @@ static unsigned int
 fill_local_addresses_vec_fullmesh(struct mptcp_sock *msk,
 				  struct mptcp_addr_info *remote,
 				  struct mptcp_pm_local *locals,
+				  int locals_size,
 				  bool c_flag_case)
 {
 	u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
@@ -488,7 +494,8 @@ fill_local_addresses_vec_fullmesh(struct mptcp_sock *msk,
 		msk->pm.extra_subflows++;
 		i++;
 
-		if (msk->pm.extra_subflows >= limit_extra_subflows)
+		if (msk->pm.extra_subflows >= limit_extra_subflows ||
+		    i == locals_size)
 			break;
 	}
 	rcu_read_unlock();
@@ -559,7 +566,8 @@ fill_local_laminar_endp(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
 static unsigned int
 fill_local_addresses_vec_c_flag(struct mptcp_sock *msk,
 				struct mptcp_addr_info *remote,
-				struct mptcp_pm_local *locals)
+				struct mptcp_pm_local *locals,
+				int locals_size)
 {
 	u8 limit_extra_subflows = mptcp_pm_get_limit_extra_subflows(msk);
 	struct pm_nl_pernet *pernet = pm_nl_get_pernet_from_msk(msk);
@@ -586,7 +594,8 @@ fill_local_addresses_vec_c_flag(struct mptcp_sock *msk,
 		msk->pm.extra_subflows++;
 		i++;
 
-		if (msk->pm.extra_subflows >= limit_extra_subflows)
+		if (msk->pm.extra_subflows >= limit_extra_subflows ||
+		    i == locals_size)
 			break;
 	}
 
@@ -620,13 +629,14 @@ fill_local_address_any(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
  */
 static unsigned int
 fill_local_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
-			 struct mptcp_pm_local *locals)
+			 struct mptcp_pm_local *locals, int locals_size)
 {
 	bool c_flag_case = remote->id && mptcp_pm_add_addr_c_flag_case(msk);
 
 	/* If there is at least one MPTCP endpoint with a fullmesh flag */
 	if (mptcp_pm_get_endp_fullmesh_max(msk))
 		return fill_local_addresses_vec_fullmesh(msk, remote, locals,
+							 locals_size,
 							 c_flag_case);
 
 	/* If there is at least one MPTCP endpoint with a laminar flag */
@@ -637,7 +647,8 @@ fill_local_addresses_vec(struct mptcp_sock *msk, struct mptcp_addr_info *remote,
 	 * limits are used -- accepting no ADD_ADDR -- and use subflow endpoints
 	 */
 	if (c_flag_case)
-		return fill_local_addresses_vec_c_flag(msk, remote, locals);
+		return fill_local_addresses_vec_c_flag(msk, remote, locals,
+						       locals_size);
 
 	/* No special case: fill in the single 'IPADDRANY' local address */
 	return fill_local_address_any(msk, remote, &locals[0]);
@@ -672,7 +683,7 @@ static void mptcp_pm_nl_add_addr_received(struct mptcp_sock *msk)
 	/* connect to the specified remote address, using whatever
 	 * local address the routing configuration will pick.
 	 */
-	nr = fill_local_addresses_vec(msk, &remote, locals);
+	nr = fill_local_addresses_vec(msk, &remote, locals, ARRAY_SIZE(locals));
 	if (nr == 0)
 		return;
 
@@ -735,7 +746,7 @@ static int mptcp_pm_nl_append_new_local_addr(struct pm_nl_pernet *pernet,
 	 */
 	if (pernet->next_id == MPTCP_PM_MAX_ADDR_ID)
 		pernet->next_id = 1;
-	if (pernet->endpoints >= MPTCP_PM_ADDR_MAX) {
+	if (pernet->endpoints == MPTCP_PM_MAX_ADDR_ID) {
 		ret = -ERANGE;
 		goto out;
 	}
@@ -1212,19 +1223,30 @@ int mptcp_pm_nl_del_addr_doit(struct sk_buff *skb, struct genl_info *info)
 }
 
 static void mptcp_pm_flush_addrs_and_subflows(struct mptcp_sock *msk,
-					      struct list_head *rm_list)
+					      struct list_head *rm_list,
+					      struct mptcp_pm_addr_entry *entry)
 {
-	struct mptcp_rm_list alist = { .nr = 0 }, slist = { .nr = 0 };
-	struct mptcp_pm_addr_entry *entry;
+	struct mptcp_rm_list alist, slist;
+	bool more;
+
+again:
+	alist.nr = 0;
+	slist.nr = 0;
+	more = false;
 
-	list_for_each_entry(entry, rm_list, list) {
-		if (slist.nr < MPTCP_RM_IDS_MAX &&
-		    mptcp_lookup_subflow_by_saddr(&msk->conn_list, &entry->addr))
+	entry = list_prepare_entry(entry, rm_list, list);
+	list_for_each_entry_continue(entry, rm_list, list) {
+		if (mptcp_lookup_subflow_by_saddr(&msk->conn_list, &entry->addr))
 			slist.ids[slist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
 
-		if (alist.nr < MPTCP_RM_IDS_MAX &&
-		    mptcp_remove_anno_list_by_saddr(msk, &entry->addr))
+		if (mptcp_remove_anno_list_by_saddr(msk, &entry->addr))
 			alist.ids[alist.nr++] = mptcp_endp_get_local_id(msk, &entry->addr);
+
+		if (slist.nr == MPTCP_RM_IDS_MAX ||
+		    alist.nr == MPTCP_RM_IDS_MAX) {
+			more = !list_is_last(&entry->list, rm_list);
+			break;
+		}
 	}
 
 	spin_lock_bh(&msk->pm.lock);
@@ -1235,9 +1257,14 @@ static void mptcp_pm_flush_addrs_and_subflows(struct mptcp_sock *msk,
 	if (slist.nr)
 		mptcp_pm_rm_subflow(msk, &slist);
 	/* Reset counters: maybe some subflows have been removed before */
-	bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
-	msk->pm.local_addr_used = 0;
+	if (!more) {
+		bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
+		msk->pm.local_addr_used = 0;
+	}
 	spin_unlock_bh(&msk->pm.lock);
+
+	if (more)
+		goto again;
 }
 
 static void mptcp_nl_flush_addrs_list(struct net *net,
@@ -1254,7 +1281,7 @@ static void mptcp_nl_flush_addrs_list(struct net *net,
 
 		if (!mptcp_pm_is_userspace(msk)) {
 			lock_sock(sk);
-			mptcp_pm_flush_addrs_and_subflows(msk, rm_list);
+			mptcp_pm_flush_addrs_and_subflows(msk, rm_list, NULL);
 			release_sock(sk);
 		}
 
@@ -1371,10 +1398,10 @@ static int parse_limit(struct genl_info *info, int id, unsigned int *limit)
 		return 0;
 
 	*limit = nla_get_u32(attr);
-	if (*limit > MPTCP_PM_ADDR_MAX) {
+	if (*limit > MPTCP_PM_SUBFLOWS_MAX) {
 		NL_SET_ERR_MSG_ATTR_FMT(info->extack, attr,
 					"limit greater than maximum (%u)",
-					MPTCP_PM_ADDR_MAX);
+					MPTCP_PM_SUBFLOWS_MAX);
 		return -EINVAL;
 	}
 	return 0;
diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c
index 4546a8b09884a..a72a6ad6ee8b1 100644
--- a/net/mptcp/protocol.c
+++ b/net/mptcp/protocol.c
@@ -397,12 +397,26 @@ static bool __mptcp_move_skb(struct sock *sk, struct sk_buff *skb)
 		return false;
 	}
 
-	/* old data, keep it simple and drop the whole pkt, sender
-	 * will retransmit as needed, if needed.
+	/* Completely old data? */
+	if (!after64(MPTCP_SKB_CB(skb)->end_seq, msk->ack_seq)) {
+		MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
+		mptcp_drop(sk, skb);
+		return false;
+	}
+
+	/* Partial packet: map_seq < ack_seq < end_seq.
+	 * Skip the already-acked bytes and enqueue the new data.
 	 */
-	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
-	mptcp_drop(sk, skb);
-	return false;
+	copy_len = MPTCP_SKB_CB(skb)->end_seq - msk->ack_seq;
+	MPTCP_SKB_CB(skb)->offset += msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq;
+	MPTCP_SKB_CB(skb)->map_seq += msk->ack_seq -
+				      MPTCP_SKB_CB(skb)->map_seq;
+	msk->bytes_received += copy_len;
+	WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len);
+
+	skb_set_owner_r(skb, sk);
+	__skb_queue_tail(&sk->sk_receive_queue, skb);
+	return true;
 }
 
 static void mptcp_stop_rtx_timer(struct sock *sk)
@@ -3473,6 +3487,7 @@ static int mptcp_disconnect(struct sock *sk, int flags)
 
 	/* for fallback's sake */
 	WRITE_ONCE(msk->ack_seq, 0);
+	atomic64_set(&msk->rcv_wnd_sent, 0);
 
 	WRITE_ONCE(sk->sk_shutdown, 0);
 	sk_error_report(sk);
diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c
index 1cf608e7357bd..87b5796d01353 100644
--- a/net/mptcp/sockopt.c
+++ b/net/mptcp/sockopt.c
@@ -67,6 +67,12 @@ static int mptcp_get_int_option(struct mptcp_sock *msk, sockptr_t optval,
 	return 0;
 }
 
+static void __mptcp_subflow_set_rcvbuf(struct sock *ssk, int val)
+{
+	WRITE_ONCE(ssk->sk_rcvbuf, val);
+	tcp_set_rcvbuf(ssk, val);
+}
+
 static void mptcp_sol_socket_sync_intval(struct mptcp_sock *msk, int optname, int val)
 {
 	struct mptcp_subflow_context *subflow;
@@ -100,7 +106,7 @@ static void mptcp_sol_socket_sync_intval(struct mptcp_sock *msk, int optname, in
 		case SO_RCVBUF:
 		case SO_RCVBUFFORCE:
 			ssk->sk_userlocks |= SOCK_RCVBUF_LOCK;
-			WRITE_ONCE(ssk->sk_rcvbuf, sk->sk_rcvbuf);
+			__mptcp_subflow_set_rcvbuf(ssk, sk->sk_rcvbuf);
 			break;
 		case SO_MARK:
 			if (READ_ONCE(ssk->sk_mark) != sk->sk_mark) {
@@ -1560,7 +1566,7 @@ static void sync_socket_options(struct mptcp_sock *msk, struct sock *ssk)
 			mptcp_subflow_ctx(ssk)->cached_sndbuf = sk->sk_sndbuf;
 		}
 		if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
-			WRITE_ONCE(ssk->sk_rcvbuf, sk->sk_rcvbuf);
+			__mptcp_subflow_set_rcvbuf(ssk, sk->sk_rcvbuf);
 	}
 
 	if (sock_flag(sk, SOCK_LINGER)) {
diff --git a/net/netfilter/ipset/ip_set_core.c b/net/netfilter/ipset/ip_set_core.c
index c5a26236a0bb8..3706b4a85a0f1 100644
--- a/net/netfilter/ipset/ip_set_core.c
+++ b/net/netfilter/ipset/ip_set_core.c
@@ -1613,6 +1613,7 @@ ip_set_dump_do(struct sk_buff *skb, struct netlink_callback *cb)
 		    ((dump_type == DUMP_ALL) ==
 		     !!(set->type->features & IPSET_DUMP_LAST))) {
 			write_unlock_bh(&ip_set_ref_lock);
+			set = NULL;
 			continue;
 		}
 		pr_debug("List set: %s\n", set->name);
@@ -1648,13 +1649,13 @@ ip_set_dump_do(struct sk_buff *skb, struct netlink_callback *cb)
 			if (cb->args[IPSET_CB_PROTO] > IPSET_PROTOCOL_MIN &&
 			    nla_put_net16(skb, IPSET_ATTR_INDEX, htons(index)))
 				goto nla_put_failure;
+			if (set->variant->uref)
+				set->variant->uref(set, cb, true);
 			ret = set->variant->head(set, skb);
 			if (ret < 0)
 				goto release_refcount;
 			if (dump_flags & IPSET_FLAG_LIST_HEADER)
 				goto next_set;
-			if (set->variant->uref)
-				set->variant->uref(set, cb, true);
 			fallthrough;
 		default:
 			ret = set->variant->list(set, skb, cb);
diff --git a/net/netfilter/ipset/ip_set_hash_gen.h b/net/netfilter/ipset/ip_set_hash_gen.h
index b79e5dd2af036..04e4627ddfc10 100644
--- a/net/netfilter/ipset/ip_set_hash_gen.h
+++ b/net/netfilter/ipset/ip_set_hash_gen.h
@@ -386,8 +386,9 @@ static void
 mtype_ext_cleanup(struct ip_set *set, struct hbucket *n)
 {
 	int i;
+	u8 pos = smp_load_acquire(&n->pos);
 
-	for (i = 0; i < n->pos; i++)
+	for (i = 0; i < pos; i++)
 		if (test_bit(i, n->used))
 			ip_set_ext_destroy(set, ahash_data(n, i, set->dsize));
 }
@@ -490,7 +491,7 @@ mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r)
 #ifdef IP_SET_HASH_WITH_NETS
 	u8 k;
 #endif
-	u8 htable_bits = t->htable_bits;
+	u8 pos, htable_bits = t->htable_bits;
 
 	spin_lock_bh(&t->hregion[r].lock);
 	for (i = ahash_bucket_start(r, htable_bits);
@@ -498,7 +499,8 @@ mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r)
 		n = __ipset_dereference(hbucket(t, i));
 		if (!n)
 			continue;
-		for (j = 0, d = 0; j < n->pos; j++) {
+		pos = smp_load_acquire(&n->pos);
+		for (j = 0, d = 0; j < pos; j++) {
 			if (!test_bit(j, n->used)) {
 				d++;
 				continue;
@@ -534,7 +536,7 @@ mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r)
 				/* Still try to delete expired elements. */
 				continue;
 			tmp->size = n->size - AHASH_INIT_SIZE;
-			for (j = 0, d = 0; j < n->pos; j++) {
+			for (j = 0, d = 0; j < pos; j++) {
 				if (!test_bit(j, n->used))
 					continue;
 				data = ahash_data(n, j, dsize);
@@ -623,7 +625,7 @@ mtype_resize(struct ip_set *set, bool retried)
 {
 	struct htype *h = set->data;
 	struct htable *t, *orig;
-	u8 htable_bits;
+	u8 pos, htable_bits;
 	size_t hsize, dsize = set->dsize;
 #ifdef IP_SET_HASH_WITH_NETS
 	u8 flags;
@@ -685,7 +687,8 @@ mtype_resize(struct ip_set *set, bool retried)
 			n = __ipset_dereference(hbucket(orig, i));
 			if (!n)
 				continue;
-			for (j = 0; j < n->pos; j++) {
+			pos = smp_load_acquire(&n->pos);
+			for (j = 0; j < pos; j++) {
 				if (!test_bit(j, n->used))
 					continue;
 				data = ahash_data(n, j, dsize);
@@ -809,9 +812,10 @@ mtype_ext_size(struct ip_set *set, u32 *elements, size_t *ext_size)
 {
 	struct htype *h = set->data;
 	const struct htable *t;
-	u32 i, j, r;
 	struct hbucket *n;
 	struct mtype_elem *data;
+	u32 i, j, r;
+	u8 pos;
 
 	t = rcu_dereference_bh(h->table);
 	for (r = 0; r < ahash_numof_locks(t->htable_bits); r++) {
@@ -820,7 +824,8 @@ mtype_ext_size(struct ip_set *set, u32 *elements, size_t *ext_size)
 			n = rcu_dereference_bh(hbucket(t, i));
 			if (!n)
 				continue;
-			for (j = 0; j < n->pos; j++) {
+			pos = smp_load_acquire(&n->pos);
+			for (j = 0; j < pos; j++) {
 				if (!test_bit(j, n->used))
 					continue;
 				data = ahash_data(n, j, set->dsize);
@@ -848,6 +853,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	bool flag_exist = flags & IPSET_FLAG_EXIST;
 	bool deleted = false, forceadd = false, reuse = false;
 	u32 r, key, multi = 0, elements, maxelem;
+	u8 npos = 0;
 
 	rcu_read_lock_bh();
 	t = rcu_dereference_bh(h->table);
@@ -889,7 +895,8 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 			ext_size(AHASH_INIT_SIZE, set->dsize);
 		goto copy_elem;
 	}
-	for (i = 0; i < n->pos; i++) {
+	npos = smp_load_acquire(&n->pos);
+	for (i = 0; i < npos; i++) {
 		if (!test_bit(i, n->used)) {
 			/* Reuse first deleted entry */
 			if (j == -1) {
@@ -933,7 +940,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	if (elements >= maxelem)
 		goto set_full;
 	/* Create a new slot */
-	if (n->pos >= n->size) {
+	if (npos >= n->size) {
 #ifdef IP_SET_HASH_WITH_MULTI
 		if (h->bucketsize >= AHASH_MAX_TUNED)
 			goto set_full;
@@ -962,7 +969,7 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	}
 
 copy_elem:
-	j = n->pos++;
+	j = npos++;
 	data = ahash_data(n, j, set->dsize);
 copy_data:
 	t->hregion[r].elements++;
@@ -985,6 +992,8 @@ mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	if (SET_WITH_TIMEOUT(set))
 		ip_set_timeout_set(ext_timeout(data, set), ext->timeout);
 	smp_mb__before_atomic();
+	/* Ensure all data writes are visible before updating position */
+	smp_store_release(&n->pos, npos);
 	set_bit(j, n->used);
 	if (old != ERR_PTR(-ENOENT)) {
 		rcu_assign_pointer(hbucket(t, key), n);
@@ -1043,6 +1052,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	int i, j, k, r, ret = -IPSET_ERR_EXIST;
 	u32 key, multi = 0;
 	size_t dsize = set->dsize;
+	u8 pos;
 
 	/* Userspace add and resize is excluded by the mutex.
 	 * Kernespace add does not trigger resize.
@@ -1058,7 +1068,8 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	n = rcu_dereference_bh(hbucket(t, key));
 	if (!n)
 		goto out;
-	for (i = 0, k = 0; i < n->pos; i++) {
+	pos = smp_load_acquire(&n->pos);
+	for (i = 0, k = 0; i < pos; i++) {
 		if (!test_bit(i, n->used)) {
 			k++;
 			continue;
@@ -1072,8 +1083,8 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 		ret = 0;
 		clear_bit(i, n->used);
 		smp_mb__after_atomic();
-		if (i + 1 == n->pos)
-			n->pos--;
+		if (i + 1 == pos)
+			smp_store_release(&n->pos, --pos);
 		t->hregion[r].elements--;
 #ifdef IP_SET_HASH_WITH_NETS
 		for (j = 0; j < IPSET_NET_COUNT; j++)
@@ -1094,11 +1105,11 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 				x->flags = flags;
 			}
 		}
-		for (; i < n->pos; i++) {
+		for (; i < pos; i++) {
 			if (!test_bit(i, n->used))
 				k++;
 		}
-		if (k == n->pos) {
+		if (k == pos) {
 			t->hregion[r].ext_size -= ext_size(n->size, dsize);
 			rcu_assign_pointer(hbucket(t, key), NULL);
 			kfree_rcu(n, rcu);
@@ -1109,7 +1120,7 @@ mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 			if (!tmp)
 				goto out;
 			tmp->size = n->size - AHASH_INIT_SIZE;
-			for (j = 0, k = 0; j < n->pos; j++) {
+			for (j = 0, k = 0; j < pos; j++) {
 				if (!test_bit(j, n->used))
 					continue;
 				data = ahash_data(n, j, dsize);
@@ -1170,6 +1181,7 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d,
 	int ret, i, j = 0;
 #endif
 	u32 key, multi = 0;
+	u8 pos;
 
 	pr_debug("test by nets\n");
 	for (; j < NLEN && h->nets[j].cidr[0] && !multi; j++) {
@@ -1187,7 +1199,8 @@ mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d,
 		n = rcu_dereference_bh(hbucket(t, key));
 		if (!n)
 			continue;
-		for (i = 0; i < n->pos; i++) {
+		pos = smp_load_acquire(&n->pos);
+		for (i = 0; i < pos; i++) {
 			if (!test_bit(i, n->used))
 				continue;
 			data = ahash_data(n, i, set->dsize);
@@ -1221,6 +1234,7 @@ mtype_test(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 	struct mtype_elem *data;
 	int i, ret = 0;
 	u32 key, multi = 0;
+	u8 pos;
 
 	rcu_read_lock_bh();
 	t = rcu_dereference_bh(h->table);
@@ -1243,7 +1257,8 @@ mtype_test(struct ip_set *set, void *value, const struct ip_set_ext *ext,
 		ret = 0;
 		goto out;
 	}
-	for (i = 0; i < n->pos; i++) {
+	pos = smp_load_acquire(&n->pos);
+	for (i = 0; i < pos; i++) {
 		if (!test_bit(i, n->used))
 			continue;
 		data = ahash_data(n, i, set->dsize);
@@ -1360,6 +1375,7 @@ mtype_list(const struct ip_set *set,
 	/* We assume that one hash bucket fills into one page */
 	void *incomplete;
 	int i, ret = 0;
+	u8 pos;
 
 	atd = nla_nest_start(skb, IPSET_ATTR_ADT);
 	if (!atd)
@@ -1378,7 +1394,8 @@ mtype_list(const struct ip_set *set,
 			 cb->args[IPSET_CB_ARG0], t, n);
 		if (!n)
 			continue;
-		for (i = 0; i < n->pos; i++) {
+		pos = smp_load_acquire(&n->pos);
+		for (i = 0; i < pos; i++) {
 			if (!test_bit(i, n->used))
 				continue;
 			e = ahash_data(n, i, set->dsize);
diff --git a/net/netfilter/ipset/ip_set_hash_ipmark.c b/net/netfilter/ipset/ip_set_hash_ipmark.c
index a22ec1a6f6ec8..e26ca2a370e34 100644
--- a/net/netfilter/ipset/ip_set_hash_ipmark.c
+++ b/net/netfilter/ipset/ip_set_hash_ipmark.c
@@ -150,7 +150,7 @@ hash_ipmark4_uadt(struct ip_set *set, struct nlattr *tb[],
 
 	if (retried)
 		ip = ntohl(h->next.ip);
-	for (; ip <= ip_to; ip++, i++) {
+	for (; ip <= ip_to; i++) {
 		e.ip = htonl(ip);
 		if (i > IPSET_MAX_RANGE) {
 			hash_ipmark4_data_next(&h->next, &e);
@@ -162,6 +162,10 @@ hash_ipmark4_uadt(struct ip_set *set, struct nlattr *tb[],
 			return ret;
 
 		ret = 0;
+
+		if (ip == ip_to)
+			break;
+		ip++;
 	}
 	return ret;
 }
diff --git a/net/netfilter/ipset/ip_set_hash_ipport.c b/net/netfilter/ipset/ip_set_hash_ipport.c
index e977b5a9c48dc..41ca24a22a026 100644
--- a/net/netfilter/ipset/ip_set_hash_ipport.c
+++ b/net/netfilter/ipset/ip_set_hash_ipport.c
@@ -186,7 +186,7 @@ hash_ipport4_uadt(struct ip_set *set, struct nlattr *tb[],
 
 	if (retried)
 		ip = ntohl(h->next.ip);
-	for (; ip <= ip_to; ip++) {
+	for (; ip <= ip_to;) {
 		p = retried && ip == ntohl(h->next.ip) ? ntohs(h->next.port)
 						       : port;
 		for (; p <= port_to; p++, i++) {
@@ -203,6 +203,9 @@ hash_ipport4_uadt(struct ip_set *set, struct nlattr *tb[],
 
 			ret = 0;
 		}
+		if (ip == ip_to)
+			break;
+		ip++;
 	}
 	return ret;
 }
diff --git a/net/netfilter/ipset/ip_set_hash_ipportip.c b/net/netfilter/ipset/ip_set_hash_ipportip.c
index 39a01934b1536..b9ac2efaa15c7 100644
--- a/net/netfilter/ipset/ip_set_hash_ipportip.c
+++ b/net/netfilter/ipset/ip_set_hash_ipportip.c
@@ -182,7 +182,7 @@ hash_ipportip4_uadt(struct ip_set *set, struct nlattr *tb[],
 
 	if (retried)
 		ip = ntohl(h->next.ip);
-	for (; ip <= ip_to; ip++) {
+	for (; ip <= ip_to;) {
 		p = retried && ip == ntohl(h->next.ip) ? ntohs(h->next.port)
 						       : port;
 		for (; p <= port_to; p++, i++) {
@@ -199,6 +199,9 @@ hash_ipportip4_uadt(struct ip_set *set, struct nlattr *tb[],
 
 			ret = 0;
 		}
+		if (ip == ip_to)
+			break;
+		ip++;
 	}
 	return ret;
 }
diff --git a/net/netfilter/ipset/ip_set_hash_ipportnet.c b/net/netfilter/ipset/ip_set_hash_ipportnet.c
index 5c6de605a9fb7..2d6652d43199a 100644
--- a/net/netfilter/ipset/ip_set_hash_ipportnet.c
+++ b/net/netfilter/ipset/ip_set_hash_ipportnet.c
@@ -274,7 +274,7 @@ hash_ipportnet4_uadt(struct ip_set *set, struct nlattr *tb[],
 		p = port;
 		ip2 = ip2_from;
 	}
-	for (; ip <= ip_to; ip++) {
+	for (; ip <= ip_to;) {
 		e.ip = htonl(ip);
 		for (; p <= port_to; p++) {
 			e.port = htons(p);
@@ -298,6 +298,9 @@ hash_ipportnet4_uadt(struct ip_set *set, struct nlattr *tb[],
 			ip2 = ip2_from;
 		}
 		p = port;
+		if (ip == ip_to)
+			break;
+		ip++;
 	}
 	return ret;
 }
diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c
index c7c7f6a7a9f6d..bd9cae44d2149 100644
--- a/net/netfilter/ipvs/ip_vs_ctl.c
+++ b/net/netfilter/ipvs/ip_vs_ctl.c
@@ -327,18 +327,22 @@ ip_vs_use_count_dec(void)
 /* Service hashing:
  * Operation			Locking order
  * ---------------------------------------------------------------------------
- * add table			service_mutex, svc_resize_sem(W)
- * del table			service_mutex
- * move between tables		svc_resize_sem(W), seqcount_t(W), bit lock
- * add/del service		service_mutex, bit lock
+ * add first table		service_mutex
+ * attach new table		service_mutex
+ * add/del service		service_mutex, RCU, bit lock
+ * move between tables (rehash)	svc_resize_sem(W), seqcount_t(W), bit lock
+ * replace old with attached	svc_resize_sem(W), svc_replace_sem(W)
  * find service			RCU, seqcount_t(R)
  * walk services(blocking)	service_mutex, svc_resize_sem(R)
  * walk services(non-blocking)	RCU, seqcount_t(R)
+ * walk services(non-blocking)	svc_resize_sem(R), RCU, seqcount_t(R)
+ * walk services(non-blocking)	svc_replace_sem(R), RCU, seqcount_t(R)
+ * del table			service_mutex after stopped work
  *
- * - new tables are linked/unlinked under service_mutex and svc_resize_sem
- * - new table is linked on resizing and all operations can run in parallel
- * in 2 tables until the new table is registered as current one
- * - two contexts can modify buckets: config and table resize, both in
+ * - new table is attached on resizing under service_mutex and all operations
+ * can run in parallel in 2 tables until the new table is registered as current
+ * one
+ * - two contexts can modify buckets: config and table resize (work), both in
  * process context
  * - only table resizer can move entries, so we do not protect t->seqc[]
  * items with t->lock[]
@@ -346,9 +350,13 @@ ip_vs_use_count_dec(void)
  * services are moved to new table
  * - move operations may disturb readers: find operation will not miss entries
  * but walkers may see same entry twice if they are forced to retry chains
- * - walkers using cond_resched_rcu() on !PREEMPT_RCU may need to hold
- * service_mutex to disallow new tables to be installed or to check
+ * or to walk the newly attached second table
+ * - walkers using cond_resched_rcu() on !PREEMPT_RCU may need to check
  * svc_table_changes and repeat the RCU read section if new table is installed
+ * - walkers may serialize with the whole resizing process (svc_resize_sem)
+ * to prevent seeing same service twice or just with the svc_table
+ * replace (svc_replace_sem) when we can see entries twice but we
+ * prefer to run concurrently with the rehashing.
  */
 
 /*
@@ -387,9 +395,16 @@ static int ip_vs_svc_hash(struct ip_vs_service *svc)
 	/* increase its refcnt because it is referenced by the svc table */
 	atomic_inc(&svc->refcnt);
 
+	/* We know if new table is attached under service_mutex but rely on
+	 * RCU to hold the old table to be freed in resizer
+	 */
+	rcu_read_lock();
+
+	/* This can be the old or the new table */
+	t = rcu_dereference(ipvs->svc_table);
+
 	/* New entries go into recent table */
-	t = rcu_dereference_protected(ipvs->svc_table, 1);
-	t = rcu_dereference_protected(t->new_tbl, 1);
+	t = rcu_dereference(t->new_tbl);
 
 	if (svc->fwmark == 0) {
 		/*
@@ -410,6 +425,8 @@ static int ip_vs_svc_hash(struct ip_vs_service *svc)
 	hlist_bl_add_head_rcu(&svc->s_list, head);
 	hlist_bl_unlock(head);
 
+	rcu_read_unlock();
+
 	return 1;
 }
 
@@ -432,7 +449,13 @@ static int ip_vs_svc_unhash(struct ip_vs_service *svc)
 		return 0;
 	}
 
-	t = rcu_dereference_protected(ipvs->svc_table, 1);
+	/* We know if new table is attached under service_mutex but rely on
+	 * RCU to hold the old table to be freed in resizer
+	 */
+	rcu_read_lock();
+
+	/* This can be the old or the new table */
+	t = rcu_dereference(ipvs->svc_table);
 	hash_key = READ_ONCE(svc->hash_key);
 	/* We need to lock the bucket in the right table */
 	if (ip_vs_rht_same_table(t, hash_key)) {
@@ -443,13 +466,13 @@ static int ip_vs_svc_unhash(struct ip_vs_service *svc)
 		/* Moved to new table ? */
 		if (hash_key != hash_key2) {
 			hlist_bl_unlock(head);
-			t = rcu_dereference_protected(t->new_tbl, 1);
+			t = rcu_dereference(t->new_tbl);
 			head = t->buckets + (hash_key2 & t->mask);
 			hlist_bl_lock(head);
 		}
 	} else {
 		/* It is already moved to new table */
-		t = rcu_dereference_protected(t->new_tbl, 1);
+		t = rcu_dereference(t->new_tbl);
 		head = t->buckets + (hash_key & t->mask);
 		hlist_bl_lock(head);
 	}
@@ -459,6 +482,8 @@ static int ip_vs_svc_unhash(struct ip_vs_service *svc)
 	svc->flags &= ~IP_VS_SVC_F_HASHED;
 	atomic_dec(&svc->refcnt);
 	hlist_bl_unlock(head);
+
+	rcu_read_unlock();
 	return 1;
 }
 
@@ -666,15 +691,14 @@ static void svc_resize_work_handler(struct work_struct *work)
 		goto unlock_sem;
 	more_work = false;
 	clear_bit(IP_VS_WORK_SVC_RESIZE, &ipvs->work_flags);
-	if (!READ_ONCE(ipvs->enable) ||
-	    test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
+	if (!READ_ONCE(ipvs->enable))
 		goto unlock_m;
 	t = rcu_dereference_protected(ipvs->svc_table, 1);
 	/* Do nothing if table is removed */
 	if (!t)
 		goto unlock_m;
-	/* New table needs to be registered? BUG! */
-	if (t != rcu_dereference_protected(t->new_tbl, 1))
+	/* New table already attached? BUG! */
+	if (t != rcu_access_pointer(t->new_tbl))
 		goto unlock_m;
 
 	lfactor = sysctl_svc_lfactor(ipvs);
@@ -691,6 +715,7 @@ static void svc_resize_work_handler(struct work_struct *work)
 	/* Flip the table_id */
 	t_new->table_id = t->table_id ^ IP_VS_RHT_TABLE_ID_MASK;
 
+	/* Attach new table */
 	rcu_assign_pointer(t->new_tbl, t_new);
 	/* Allow add/del to new_tbl while moving from old table */
 	mutex_unlock(&ipvs->service_mutex);
@@ -698,8 +723,8 @@ static void svc_resize_work_handler(struct work_struct *work)
 	ip_vs_rht_for_each_bucket(t, bucket, head) {
 same_bucket:
 		if (++limit >= 16) {
-			if (!READ_ONCE(ipvs->enable) ||
-			    test_bit(IP_VS_WORK_SVC_NORESIZE,
+			/* Check if work is stopped */
+			if (test_bit(IP_VS_WORK_SVC_NORESIZE,
 				     &ipvs->work_flags))
 				goto unlock_sem;
 			if (resched_score >= 100) {
@@ -764,16 +789,12 @@ static void svc_resize_work_handler(struct work_struct *work)
 			goto same_bucket;
 	}
 
-	/* Tables can be switched only under service_mutex */
-	while (!mutex_trylock(&ipvs->service_mutex)) {
-		cond_resched();
-		if (!READ_ONCE(ipvs->enable) ||
-		    test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
-			goto unlock_sem;
-	}
-	if (!READ_ONCE(ipvs->enable) ||
-	    test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
-		goto unlock_m;
+	/* Serialize with readers that don't like svc_table changes */
+	down_write(&ipvs->svc_replace_sem);
+
+	/* Check if work is stopped to avoid synchronize_rcu() */
+	if (test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
+		goto unlock_repl;
 
 	rcu_assign_pointer(ipvs->svc_table, t_new);
 	/* Inform readers that new table is installed */
@@ -781,8 +802,8 @@ static void svc_resize_work_handler(struct work_struct *work)
 	atomic_inc(&ipvs->svc_table_changes);
 	t_free = t;
 
-unlock_m:
-	mutex_unlock(&ipvs->service_mutex);
+unlock_repl:
+	up_write(&ipvs->svc_replace_sem);
 
 unlock_sem:
 	up_write(&ipvs->svc_resize_sem);
@@ -801,6 +822,11 @@ static void svc_resize_work_handler(struct work_struct *work)
 	    test_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags))
 		return;
 	queue_delayed_work(system_unbound_wq, &ipvs->svc_resize_work, 1);
+	return;
+
+unlock_m:
+	mutex_unlock(&ipvs->service_mutex);
+	goto unlock_sem;
 }
 
 static inline void
@@ -1691,6 +1717,7 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u,
 	struct ip_vs_pe *pe = NULL;
 	int ret_hooks = -1;
 	int ret = 0;
+	bool grow;
 
 	/* increase the module use count */
 	if (!ip_vs_use_count_inc())
@@ -1732,16 +1759,25 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u,
 	}
 #endif
 
-	t = rcu_dereference_protected(ipvs->svc_table, 1);
+	/* The old table can be freed, protect it with RCU */
+	rcu_read_lock();
+	t = rcu_dereference(ipvs->svc_table);
 	if (!t) {
 		int lfactor = sysctl_svc_lfactor(ipvs);
 		int new_size = ip_vs_svc_desired_size(ipvs, NULL, lfactor);
 
+		rcu_read_unlock();
 		t_new = ip_vs_svc_table_alloc(ipvs, new_size, lfactor);
 		if (!t_new) {
 			ret = -ENOMEM;
 			goto out_err;
 		}
+		grow = false;
+	} else {
+		/* Even the currently attached new table may need to grow */
+		t = rcu_dereference(t->new_tbl);
+		grow = ip_vs_get_num_services(ipvs) + 1 > t->u_thresh;
+		rcu_read_unlock();
 	}
 
 	if (!rcu_dereference_protected(ipvs->conn_tab, 1)) {
@@ -1800,6 +1836,7 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u,
 		goto out_err;
 
 	if (t_new) {
+		/* Add table for first time */
 		clear_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags);
 		rcu_assign_pointer(ipvs->svc_table, t_new);
 		t_new = NULL;
@@ -1831,8 +1868,7 @@ ip_vs_add_service(struct netns_ipvs *ipvs, struct ip_vs_service_user_kern *u,
 	ip_vs_svc_hash(svc);
 
 	/* Schedule resize work */
-	if (t && ip_vs_get_num_services(ipvs) > t->u_thresh &&
-	    !test_and_set_bit(IP_VS_WORK_SVC_RESIZE, &ipvs->work_flags))
+	if (grow && !test_and_set_bit(IP_VS_WORK_SVC_RESIZE, &ipvs->work_flags))
 		queue_delayed_work(system_unbound_wq, &ipvs->svc_resize_work,
 				   1);
 
@@ -2054,7 +2090,6 @@ static int ip_vs_del_service(struct ip_vs_service *svc)
 		return -EEXIST;
 	ipvs = svc->ipvs;
 	ip_vs_unlink_service(svc, false);
-	t = rcu_dereference_protected(ipvs->svc_table, 1);
 
 	/* Drop the table if no more services */
 	ns = ip_vs_get_num_services(ipvs);
@@ -2062,6 +2097,7 @@ static int ip_vs_del_service(struct ip_vs_service *svc)
 		/* Stop the resizer and drop the tables */
 		set_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags);
 		cancel_delayed_work_sync(&ipvs->svc_resize_work);
+		t = rcu_dereference_protected(ipvs->svc_table, 1);
 		if (t) {
 			rcu_assign_pointer(ipvs->svc_table, NULL);
 			/* Inform readers that table is removed */
@@ -2075,11 +2111,19 @@ static int ip_vs_del_service(struct ip_vs_service *svc)
 				t = p;
 			}
 		}
-	} else if (ns <= t->l_thresh &&
-		   !test_and_set_bit(IP_VS_WORK_SVC_RESIZE,
-				     &ipvs->work_flags)) {
-		queue_delayed_work(system_unbound_wq, &ipvs->svc_resize_work,
-				   1);
+	} else {
+		bool shrink;
+
+		rcu_read_lock();
+		t = rcu_dereference(ipvs->svc_table);
+		/* Even the currently attached new table may need to shrink */
+		t = rcu_dereference(t->new_tbl);
+		shrink = ns <= t->l_thresh;
+		rcu_read_unlock();
+		if (shrink && !test_and_set_bit(IP_VS_WORK_SVC_RESIZE,
+						&ipvs->work_flags))
+			queue_delayed_work(system_unbound_wq,
+					   &ipvs->svc_resize_work, 1);
 	}
 	return 0;
 }
@@ -2184,17 +2228,21 @@ static int ip_vs_dst_event(struct notifier_block *this, unsigned long event,
 	struct ip_vs_service *svc;
 	struct hlist_bl_node *e;
 	struct ip_vs_dest *dest;
-	int old_gen, new_gen;
+	int old_gen;
 
 	if (event != NETDEV_DOWN || !ipvs)
 		return NOTIFY_DONE;
 	IP_VS_DBG(3, "%s() dev=%s\n", __func__, dev->name);
 
+	/* Allow concurrent rehashing on resize but to avoid loop
+	 * serialize with installing the new table.
+	 */
+	down_read(&ipvs->svc_replace_sem);
+
 	old_gen = atomic_read(&ipvs->svc_table_changes);
 
 	rcu_read_lock();
 
-repeat:
 	smp_rmb(); /* ipvs->svc_table and svc_table_changes */
 	ip_vs_rht_walk_buckets_rcu(ipvs->svc_table, head) {
 		hlist_bl_for_each_entry_rcu(svc, e, head, s_list) {
@@ -2207,17 +2255,17 @@ static int ip_vs_dst_event(struct notifier_block *this, unsigned long event,
 		}
 		resched_score++;
 		if (resched_score >= 100) {
-			resched_score = 0;
 			cond_resched_rcu();
-			new_gen = atomic_read(&ipvs->svc_table_changes);
-			/* New table installed ? */
-			if (old_gen != new_gen) {
-				old_gen = new_gen;
-				goto repeat;
-			}
+			/* Flushed? So no more dev refs */
+			if (atomic_read(&ipvs->svc_table_changes) != old_gen)
+				goto done;
+			resched_score = 0;
 		}
 	}
+
+done:
 	rcu_read_unlock();
+	up_read(&ipvs->svc_replace_sem);
 
 	return NOTIFY_DONE;
 }
@@ -2244,6 +2292,10 @@ static int ip_vs_zero_all(struct netns_ipvs *ipvs)
 	struct ip_vs_service *svc;
 	struct hlist_bl_node *e;
 
+	/* svc_table can not be replaced (svc_replace_sem) or
+	 * removed (service_mutex)
+	 */
+	down_read(&ipvs->svc_replace_sem);
 	rcu_read_lock();
 
 	ip_vs_rht_walk_buckets_rcu(ipvs->svc_table, head) {
@@ -2259,6 +2311,7 @@ static int ip_vs_zero_all(struct netns_ipvs *ipvs)
 	}
 
 	rcu_read_unlock();
+	up_read(&ipvs->svc_replace_sem);
 
 	ip_vs_zero_stats(&ipvs->tot_stats->s);
 	return 0;
@@ -3062,6 +3115,7 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
 	u32 sum;
 	int i;
 
+	/* Info for conns */
 	rcu_read_lock();
 
 	t = rcu_dereference(ipvs->conn_tab);
@@ -3123,6 +3177,12 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
 	}
 
 after_conns:
+	rcu_read_unlock();
+
+	/* Info for services */
+	down_read(&ipvs->svc_replace_sem);
+	rcu_read_lock();
+
 	t = rcu_dereference(ipvs->svc_table);
 
 	count = ip_vs_get_num_services(ipvs);
@@ -3133,9 +3193,7 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
 	if (!count)
 		goto after_svc;
 	old_gen = atomic_read(&ipvs->svc_table_changes);
-	loops = 0;
 
-repeat_svc:
 	smp_rmb(); /* ipvs->svc_table and svc_table_changes */
 	memset(counts, 0, sizeof(counts));
 	ip_vs_rht_for_each_table_rcu(ipvs->svc_table, t, pt) {
@@ -3157,15 +3215,10 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
 			if (resched_score >= 100) {
 				resched_score = 0;
 				cond_resched_rcu();
-				new_gen = atomic_read(&ipvs->svc_table_changes);
-				/* New table installed ? */
-				if (old_gen != new_gen) {
-					/* Too many changes? */
-					if (++loops >= 5)
-						goto after_svc;
-					old_gen = new_gen;
-					goto repeat_svc;
-				}
+				/* Flushed? */
+				if (atomic_read(&ipvs->svc_table_changes) !=
+				    old_gen)
+					goto after_svc;
 			}
 			counts[count]++;
 		}
@@ -3184,6 +3237,9 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
 	}
 
 after_svc:
+	rcu_read_unlock();
+	up_read(&ipvs->svc_replace_sem);
+
 	seq_printf(seq, "Stats thread slots:\t%d (max %lu)\n",
 		   ipvs->est_kt_count, ipvs->est_max_threads);
 	seq_printf(seq, "Stats chain max len:\t%d\n", ipvs->est_chain_max);
@@ -3191,7 +3247,6 @@ static int ip_vs_status_show(struct seq_file *seq, void *v)
 		   ipvs->est_chain_max * IPVS_EST_CHAIN_FACTOR *
 		   IPVS_EST_NTICKS);
 
-	rcu_read_unlock();
 	return 0;
 }
 
@@ -3503,7 +3558,7 @@ __ip_vs_get_service_entries(struct netns_ipvs *ipvs,
 	int ret = 0;
 
 	lockdep_assert_held(&ipvs->svc_resize_sem);
-	/* All service modifications are disabled, go ahead */
+	/* All svc_table modifications are disabled, go ahead */
 	ip_vs_rht_walk_buckets(ipvs->svc_table, head) {
 		hlist_bl_for_each_entry(svc, e, head, s_list) {
 			/* Only expose IPv4 entries to old interface */
@@ -3687,7 +3742,7 @@ do_ip_vs_get_ctl(struct sock *sk, int cmd, void __user *user, int *len)
 			pr_err("length: %u != %zu\n", *len, size);
 			return -EINVAL;
 		}
-		/* Protect against table resizer moving the entries.
+		/* Prevent modifications to the list with services.
 		 * Try reverse locking, so that we do not hold the mutex
 		 * while waiting for semaphore.
 		 */
@@ -4029,6 +4084,7 @@ static int ip_vs_genl_dump_services(struct sk_buff *skb,
 	int start = cb->args[0];
 	int idx = 0;
 
+	/* Make sure we do not see same service twice during resize */
 	down_read(&ipvs->svc_resize_sem);
 	rcu_read_lock();
 	ip_vs_rht_walk_buckets_safe_rcu(ipvs->svc_table, head) {
@@ -5072,6 +5128,7 @@ int __net_init ip_vs_control_net_init(struct netns_ipvs *ipvs)
 	/* Initialize service_mutex, svc_table per netns */
 	__mutex_init(&ipvs->service_mutex, "ipvs->service_mutex", &__ipvs_service_key);
 	init_rwsem(&ipvs->svc_resize_sem);
+	init_rwsem(&ipvs->svc_replace_sem);
 	INIT_DELAYED_WORK(&ipvs->svc_resize_work, svc_resize_work_handler);
 	atomic_set(&ipvs->svc_table_changes, 0);
 	RCU_INIT_POINTER(ipvs->svc_table, NULL);
diff --git a/net/netfilter/nf_conntrack_helper.c b/net/netfilter/nf_conntrack_helper.c
index b594cd244fe1d..17e971bd4c746 100644
--- a/net/netfilter/nf_conntrack_helper.c
+++ b/net/netfilter/nf_conntrack_helper.c
@@ -321,8 +321,8 @@ __printf(3, 4)
 void nf_ct_helper_log(struct sk_buff *skb, const struct nf_conn *ct,
 		      const char *fmt, ...)
 {
+	const char *helper_name = "(null)";
 	const struct nf_conn_help *help;
-	const struct nf_conntrack_helper *helper;
 	struct va_format vaf;
 	va_list args;
 
@@ -331,14 +331,17 @@ void nf_ct_helper_log(struct sk_buff *skb, const struct nf_conn *ct,
 	vaf.fmt = fmt;
 	vaf.va = &args;
 
-	/* Called from the helper function, this call never fails */
 	help = nfct_help(ct);
+	if (help) {
+		const struct nf_conntrack_helper *helper;
 
-	/* rcu_read_lock()ed by nf_hook_thresh */
-	helper = rcu_dereference(help->helper);
+		helper = rcu_dereference(help->helper);
+		if (helper)
+			helper_name = helper->name;
+	}
 
 	nf_log_packet(nf_ct_net(ct), nf_ct_l3num(ct), 0, skb, NULL, NULL, NULL,
-		      "nf_ct_%s: dropping packet: %pV ", helper->name, &vaf);
+		      "helper %s dropping packet: %pV ", helper_name, &vaf);
 
 	va_end(args);
 }
diff --git a/net/netfilter/nf_queue.c b/net/netfilter/nf_queue.c
index a6c81c04b3a52..57b450024a99e 100644
--- a/net/netfilter/nf_queue.c
+++ b/net/netfilter/nf_queue.c
@@ -61,6 +61,7 @@ static void nf_queue_entry_release_refs(struct nf_queue_entry *entry)
 	struct nf_hook_state *state = &entry->state;
 
 	/* Release those devices we held, or Alexey will kill me. */
+	dev_put(entry->skb_dev);
 	dev_put(state->in);
 	dev_put(state->out);
 	if (state->sk)
@@ -102,6 +103,7 @@ bool nf_queue_entry_get_refs(struct nf_queue_entry *entry)
 	if (state->sk && !refcount_inc_not_zero(&state->sk->sk_refcnt))
 		return false;
 
+	dev_hold(entry->skb_dev);
 	dev_hold(state->in);
 	dev_hold(state->out);
 
@@ -202,11 +204,11 @@ static int __nf_queue(struct sk_buff *skb, const struct nf_hook_state *state,
 
 	*entry = (struct nf_queue_entry) {
 		.skb	= skb,
+		.skb_dev = skb->dev,
 		.state	= *state,
 		.hook_index = index,
 		.size	= sizeof(*entry) + route_key_size,
 	};
-
 	__nf_queue_entry_init_physdevs(entry);
 
 	if (!nf_queue_entry_get_refs(entry)) {
diff --git a/net/netfilter/nfnetlink_queue.c b/net/netfilter/nfnetlink_queue.c
index 58304fd1f70ff..984a0eb9e1492 100644
--- a/net/netfilter/nfnetlink_queue.c
+++ b/net/netfilter/nfnetlink_queue.c
@@ -1212,6 +1212,8 @@ dev_cmp(struct nf_queue_entry *entry, unsigned long ifindex)
 	if (physinif == ifindex || physoutif == ifindex)
 		return 1;
 #endif
+	if (entry->skb_dev && entry->skb_dev->ifindex == ifindex)
+		return 1;
 	if (entry->state.in)
 		if (entry->state.in->ifindex == ifindex)
 			return 1;
diff --git a/net/netfilter/nft_inner.c b/net/netfilter/nft_inner.c
index 03ffb1159fc18..d14ca157910b5 100644
--- a/net/netfilter/nft_inner.c
+++ b/net/netfilter/nft_inner.c
@@ -163,7 +163,6 @@ static int nft_inner_parse_l2l3(const struct nft_inner *priv,
 			return -1;
 
 		if (fragoff == 0) {
-			thoff = nhoff + sizeof(_ip6h);
 			ctx->flags |= NFT_PAYLOAD_CTX_INNER_TH;
 			ctx->inner_thoff = thoff;
 			ctx->l4proto = l4proto;
@@ -247,8 +246,8 @@ static bool nft_inner_restore_tun_ctx(const struct nft_pktinfo *pkt,
 	local_lock_nested_bh(&nft_pcpu_tun_ctx.bh_lock);
 	this_cpu_tun_ctx = this_cpu_ptr(&nft_pcpu_tun_ctx.ctx);
 	if (this_cpu_tun_ctx->cookie != (unsigned long)pkt->skb) {
-		local_bh_enable();
 		local_unlock_nested_bh(&nft_pcpu_tun_ctx.bh_lock);
+		local_bh_enable();
 		return false;
 	}
 	*tun_ctx = *this_cpu_tun_ctx;
diff --git a/net/netfilter/xt_IDLETIMER.c b/net/netfilter/xt_IDLETIMER.c
index 517106165ad2a..bfcf2d44e93db 100644
--- a/net/netfilter/xt_IDLETIMER.c
+++ b/net/netfilter/xt_IDLETIMER.c
@@ -115,6 +115,21 @@ static void idletimer_tg_alarmproc(struct alarm *alarm, ktime_t now)
 	schedule_work(&timer->work);
 }
 
+static void idletimer_start_alarm_ktime(struct idletimer_tg *timer, ktime_t timeout)
+{
+	/*
+	 * The timer should always be queued as @tout it should be least one
+	 * second, but handle it correctly in any case. Virt will manage!
+	 */
+	if (!alarm_start_timer(&timer->alarm, timeout, true))
+		schedule_work(&timer->work);
+}
+
+static void idletimer_start_alarm_sec(struct idletimer_tg *timer, unsigned int seconds)
+{
+	idletimer_start_alarm_ktime(timer, ktime_set(seconds, 0));
+}
+
 static int idletimer_check_sysfs_name(const char *name, unsigned int size)
 {
 	int ret;
@@ -220,12 +235,10 @@ static int idletimer_tg_create_v1(struct idletimer_tg_info_v1 *info)
 	INIT_WORK(&info->timer->work, idletimer_tg_work);
 
 	if (info->timer->timer_type & XT_IDLETIMER_ALARM) {
-		ktime_t tout;
 		alarm_init(&info->timer->alarm, ALARM_BOOTTIME,
 			   idletimer_tg_alarmproc);
 		info->timer->alarm.data = info->timer;
-		tout = ktime_set(info->timeout, 0);
-		alarm_start_relative(&info->timer->alarm, tout);
+		idletimer_start_alarm_sec(info->timer, info->timeout);
 	} else {
 		timer_setup(&info->timer->timer, idletimer_tg_expired, 0);
 		mod_timer(&info->timer->timer,
@@ -271,8 +284,7 @@ static unsigned int idletimer_tg_target_v1(struct sk_buff *skb,
 		 info->label, info->timeout);
 
 	if (info->timer->timer_type & XT_IDLETIMER_ALARM) {
-		ktime_t tout = ktime_set(info->timeout, 0);
-		alarm_start_relative(&info->timer->alarm, tout);
+		idletimer_start_alarm_sec(info->timer, info->timeout);
 	} else {
 		mod_timer(&info->timer->timer,
 				secs_to_jiffies(info->timeout) + jiffies);
@@ -384,7 +396,7 @@ static int idletimer_tg_checkentry_v1(const struct xt_tgchk_param *par)
 			if (ktimespec.tv_sec > 0) {
 				pr_debug("time_expiry_remaining %lld\n",
 					 ktimespec.tv_sec);
-				alarm_start_relative(&info->timer->alarm, tout);
+				idletimer_start_alarm_ktime(info->timer, tout);
 			}
 		} else {
 				mod_timer(&info->timer->timer,
diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c
index 2aeb0680807d6..db3be485b4804 100644
--- a/net/netlink/af_netlink.c
+++ b/net/netlink/af_netlink.c
@@ -39,6 +39,7 @@
 #include <linux/fs.h>
 #include <linux/slab.h>
 #include <linux/uaccess.h>
+#include <linux/uio.h>
 #include <linux/skbuff.h>
 #include <linux/netdevice.h>
 #include <linux/rtnetlink.h>
@@ -1716,18 +1717,18 @@ static int netlink_setsockopt(struct socket *sock, int level, int optname,
 }
 
 static int netlink_getsockopt(struct socket *sock, int level, int optname,
-			      char __user *optval, int __user *optlen)
+			      sockopt_t *opt)
 {
 	struct sock *sk = sock->sk;
 	struct netlink_sock *nlk = nlk_sk(sk);
 	unsigned int flag;
 	int len, val;
+	u32 group;
 
 	if (level != SOL_NETLINK)
 		return -ENOPROTOOPT;
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = opt->optlen;
 	if (len < 0)
 		return -EINVAL;
 
@@ -1751,14 +1752,14 @@ static int netlink_getsockopt(struct socket *sock, int level, int optname,
 
 			idx = pos / sizeof(unsigned long);
 			shift = (pos % sizeof(unsigned long)) * 8;
-			if (put_user((u32)(nlk->groups[idx] >> shift),
-				     (u32 __user *)(optval + pos))) {
+			group = (u32)(nlk->groups[idx] >> shift);
+			if (copy_to_iter(&group, sizeof(u32),
+					 &opt->iter_out) != sizeof(u32)) {
 				err = -EFAULT;
 				break;
 			}
 		}
-		if (put_user(ALIGN(BITS_TO_BYTES(nlk->ngroups), sizeof(u32)), optlen))
-			err = -EFAULT;
+		opt->optlen = ALIGN(BITS_TO_BYTES(nlk->ngroups), sizeof(u32));
 		netlink_unlock_table();
 		return err;
 	}
@@ -1784,8 +1785,8 @@ static int netlink_getsockopt(struct socket *sock, int level, int optname,
 	len = sizeof(int);
 	val = test_bit(flag, &nlk->flags);
 
-	if (put_user(len, optlen) ||
-	    copy_to_user(optval, &val, len))
+	opt->optlen = len;
+	if (copy_to_iter(&val, len, &opt->iter_out) != len)
 		return -EFAULT;
 
 	return 0;
@@ -2813,7 +2814,7 @@ static const struct proto_ops netlink_ops = {
 	.listen =	sock_no_listen,
 	.shutdown =	sock_no_shutdown,
 	.setsockopt =	netlink_setsockopt,
-	.getsockopt =	netlink_getsockopt,
+	.getsockopt_iter = netlink_getsockopt,
 	.sendmsg =	netlink_sendmsg,
 	.recvmsg =	netlink_recvmsg,
 	.mmap =		sock_no_mmap,
diff --git a/net/nfc/hci/core.c b/net/nfc/hci/core.c
index 0d33c81a15fe1..ba6f0310ffd7c 100644
--- a/net/nfc/hci/core.c
+++ b/net/nfc/hci/core.c
@@ -861,6 +861,11 @@ static void nfc_hci_recv_from_llc(struct nfc_hci_dev *hdev, struct sk_buff *skb)
 	struct sk_buff *frag_skb;
 	int msg_len;
 
+	if (!pskb_may_pull(skb, NFC_HCI_HCP_PACKET_HEADER_LEN)) {
+		kfree_skb(skb);
+		return;
+	}
+
 	packet = (struct hcp_packet *)skb->data;
 	if ((packet->header & ~NFC_HCI_FRAGMENT) == 0) {
 		skb_queue_tail(&hdev->rx_hcp_frags, skb);
@@ -904,6 +909,11 @@ static void nfc_hci_recv_from_llc(struct nfc_hci_dev *hdev, struct sk_buff *skb)
 	 * unblock waiting cmd context. Otherwise, enqueue to dispatch
 	 * in separate context where handler can also execute command.
 	 */
+	if (!pskb_may_pull(hcp_skb, NFC_HCI_HCP_HEADER_LEN)) {
+		kfree_skb(hcp_skb);
+		return;
+	}
+
 	packet = (struct hcp_packet *)hcp_skb->data;
 	type = HCP_MSG_GET_TYPE(packet->message.header);
 	if (type == NFC_HCI_HCP_RESPONSE) {
diff --git a/net/nfc/llcp_core.c b/net/nfc/llcp_core.c
index db5bc6a878ddb..dc65c719f35f2 100644
--- a/net/nfc/llcp_core.c
+++ b/net/nfc/llcp_core.c
@@ -1218,6 +1218,15 @@ static void nfc_llcp_recv_cc(struct nfc_llcp_local *local,
 
 	sk = &llcp_sock->sk;
 
+	lock_sock(sk);
+
+	/* Check if socket was destroyed whilst waiting for the lock */
+	if (!sk_hashed(sk)) {
+		release_sock(sk);
+		nfc_llcp_sock_put(llcp_sock);
+		return;
+	}
+
 	/* Unlink from connecting and link to the client array */
 	nfc_llcp_sock_unlink(&local->connecting_sockets, sk);
 	nfc_llcp_sock_link(&local->sockets, sk);
@@ -1229,6 +1238,8 @@ static void nfc_llcp_recv_cc(struct nfc_llcp_local *local,
 	sk->sk_state = LLCP_CONNECTED;
 	sk->sk_state_change(sk);
 
+	release_sock(sk);
+
 	nfc_llcp_sock_put(llcp_sock);
 }
 
diff --git a/net/nfc/llcp_sock.c b/net/nfc/llcp_sock.c
index f1be1e84f6653..feab29fc62f44 100644
--- a/net/nfc/llcp_sock.c
+++ b/net/nfc/llcp_sock.c
@@ -633,6 +633,8 @@ static int llcp_sock_release(struct socket *sock)
 
 	if (sock->type == SOCK_RAW)
 		nfc_llcp_sock_unlink(&local->raw_sockets, sk);
+	else if (sk->sk_state == LLCP_CONNECTING)
+		nfc_llcp_sock_unlink(&local->connecting_sockets, sk);
 	else
 		nfc_llcp_sock_unlink(&local->sockets, sk);
 
diff --git a/net/nfc/nci/hci.c b/net/nfc/nci/hci.c
index 40ae8e5a7ec7a..c03e8a0bd3bd6 100644
--- a/net/nfc/nci/hci.c
+++ b/net/nfc/nci/hci.c
@@ -439,6 +439,11 @@ void nci_hci_data_received_cb(void *context,
 		return;
 	}
 
+	if (!pskb_may_pull(skb, NCI_HCI_HCP_PACKET_HEADER_LEN)) {
+		kfree_skb(skb);
+		return;
+	}
+
 	packet = (struct nci_hcp_packet *)skb->data;
 	if ((packet->header & ~NCI_HCI_FRAGMENT) == 0) {
 		skb_queue_tail(&ndev->hci_dev->rx_hcp_frags, skb);
@@ -482,6 +487,11 @@ void nci_hci_data_received_cb(void *context,
 	 * unblock waiting cmd context. Otherwise, enqueue to dispatch
 	 * in separate context where handler can also execute command.
 	 */
+	if (!pskb_may_pull(hcp_skb, NCI_HCI_HCP_HEADER_LEN)) {
+		kfree_skb(hcp_skb);
+		return;
+	}
+
 	packet = (struct nci_hcp_packet *)hcp_skb->data;
 	type = NCI_HCP_MSG_GET_TYPE(packet->message.header);
 	if (type == NCI_HCI_HCP_RESPONSE) {
diff --git a/net/nfc/nci/uart.c b/net/nfc/nci/uart.c
index 5a3aed7f84f5b..aa20e8603f327 100644
--- a/net/nfc/nci/uart.c
+++ b/net/nfc/nci/uart.c
@@ -421,7 +421,7 @@ void nci_uart_set_config(struct nci_uart *nu, int baudrate, int flow_ctrl)
 }
 EXPORT_SYMBOL_GPL(nci_uart_set_config);
 
-static struct tty_ldisc_ops nci_uart_ldisc = {
+static const struct tty_ldisc_ops nci_uart_ldisc = {
 	.owner		= THIS_MODULE,
 	.num		= N_NCI,
 	.name		= "n_nci",
diff --git a/net/openvswitch/vport-netdev.c b/net/openvswitch/vport-netdev.c
index c42642075685d..e7e8490a53d80 100644
--- a/net/openvswitch/vport-netdev.c
+++ b/net/openvswitch/vport-netdev.c
@@ -83,6 +83,14 @@ struct vport *ovs_netdev_link(struct vport *vport, bool tunnel)
 	}
 
 	rtnl_lock();
+	/* Do not link devices that are not registered to avoid a potential
+	 * race with the NETDEV_UNREGISTER notification in dp_device_event().
+	 */
+	if (vport->dev->reg_state != NETREG_REGISTERED) {
+		err = -ENODEV;
+		goto error_put_unlock;
+	}
+
 	err = netdev_master_upper_dev_link(vport->dev,
 					   get_dpdev(vport->dp),
 					   NULL, NULL, NULL);
diff --git a/net/psp/psp_main.c b/net/psp/psp_main.c
index e45549f08eef8..ccbbb2a5fa586 100644
--- a/net/psp/psp_main.c
+++ b/net/psp/psp_main.c
@@ -90,6 +90,10 @@ psp_dev_create(struct net_device *netdev,
 	mutex_lock(&psd->lock);
 	mutex_unlock(&psp_devs_lock);
 
+	/* notify before netdev assignment
+	 * There's no strong reason for it, but thinking is to avoid creating
+	 * implicit expectations about the PSP dev <> netdev relationship.
+	 */
 	psp_nl_notify_dev(psd, PSP_CMD_DEV_ADD_NTF);
 
 	rcu_assign_pointer(netdev->psp_dev, psd);
@@ -228,6 +232,10 @@ bool psp_dev_encapsulate(struct net *net, struct sk_buff *skb, __be32 spi,
 	u32 ethr_len = skb_mac_header_len(skb);
 	u32 bufflen = ethr_len + network_len;
 
+	if (skb->protocol != htons(ETH_P_IP) &&
+	    skb->protocol != htons(ETH_P_IPV6))
+		return false;
+
 	if (skb_cow_head(skb, PSP_ENCAP_HLEN))
 		return false;
 
@@ -243,11 +251,9 @@ bool psp_dev_encapsulate(struct net *net, struct sk_buff *skb, __be32 spi,
 		ip_hdr(skb)->check = 0;
 		ip_hdr(skb)->check =
 			ip_fast_csum((u8 *)ip_hdr(skb), ip_hdr(skb)->ihl);
-	} else if (skb->protocol == htons(ETH_P_IPV6)) {
+	} else {
 		ipv6_hdr(skb)->nexthdr = IPPROTO_UDP;
 		be16_add_cpu(&ipv6_hdr(skb)->payload_len, PSP_ENCAP_HLEN);
-	} else {
-		return false;
 	}
 
 	skb_set_inner_ipproto(skb, IPPROTO_TCP);
@@ -295,6 +301,9 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv)
 	if (proto == htons(ETH_P_IP)) {
 		struct iphdr *iph = (struct iphdr *)(skb->data + l2_hlen);
 
+		if (unlikely(iph->ihl < 5))
+			return -EINVAL;
+
 		is_udp = iph->protocol == IPPROTO_UDP;
 		l3_hlen = iph->ihl * 4;
 		if (l3_hlen != sizeof(struct iphdr) &&
@@ -348,6 +357,9 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv)
 	if (proto == htons(ETH_P_IP)) {
 		struct iphdr *iph = (struct iphdr *)(skb->data + l2_hlen);
 
+		if (unlikely(ntohs(iph->tot_len) < l3_hlen + encap))
+			return -EINVAL;
+
 		iph->protocol = psph->nexthdr;
 		iph->tot_len = htons(ntohs(iph->tot_len) - encap);
 		iph->check = 0;
@@ -355,6 +367,9 @@ int psp_dev_rcv(struct sk_buff *skb, u16 dev_id, u8 generation, bool strip_icv)
 	} else {
 		struct ipv6hdr *ipv6h = (struct ipv6hdr *)(skb->data + l2_hlen);
 
+		if (unlikely(ntohs(ipv6h->payload_len) < encap))
+			return -EINVAL;
+
 		ipv6h->nexthdr = psph->nexthdr;
 		ipv6h->payload_len = htons(ntohs(ipv6h->payload_len) - encap);
 	}
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 0c64c504f79db..6b40345ba44d1 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -1038,6 +1038,19 @@ int rds_ib_conn_path_connect(struct rds_conn_path *cp)
 	return ret;
 }
 
+static unsigned long rds_ib_conn_path_shutdown_check_wait(struct rds_conn_path *cp)
+{
+	struct rds_connection *conn = cp->cp_conn;
+	struct rds_ib_connection *ic = conn->c_transport_data;
+
+	return (!ic->i_cm_id ||
+		(rds_ib_ring_empty(&ic->i_recv_ring) &&
+		 (atomic_read(&ic->i_signaled_sends) == 0) &&
+		 (atomic_read(&ic->i_fastreg_inuse_count)) == 0 &&
+		 (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR))) ? 0
+		: msecs_to_jiffies(1000);
+}
+
 /*
  * This is so careful about only cleaning up resources that were built up
  * so that it can be called at any point during startup.  In fact it
@@ -1078,11 +1091,13 @@ void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
 		 * sends to complete we're ensured that there will be no
 		 * more tx processing.
 		 */
-		wait_event(rds_ib_ring_empty_wait,
-			   rds_ib_ring_empty(&ic->i_recv_ring) &&
-			   (atomic_read(&ic->i_signaled_sends) == 0) &&
-			   (atomic_read(&ic->i_fastreg_inuse_count) == 0) &&
-			   (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR));
+		while (!wait_event_timeout(rds_ib_ring_empty_wait,
+					   rds_ib_conn_path_shutdown_check_wait(cp) == 0,
+					   msecs_to_jiffies(1000))) {
+			tasklet_schedule(&ic->i_send_tasklet);
+			tasklet_schedule(&ic->i_recv_tasklet);
+		}
+
 		tasklet_kill(&ic->i_send_tasklet);
 		tasklet_kill(&ic->i_recv_tasklet);
 
diff --git a/net/rds/tcp.c b/net/rds/tcp.c
index 654e23d13e3df..5830b31a1f37b 100644
--- a/net/rds/tcp.c
+++ b/net/rds/tcp.c
@@ -198,8 +198,13 @@ void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp)
 	rdsdebug("setting sock %p callbacks to tc %p\n", sock, tc);
 	write_lock_bh(&sock->sk->sk_callback_lock);
 
-	/* done under the callback_lock to serialize with write_space */
+	/* done under the callback_lock to serialize with write_space.
+	 * Set t_sock inside rds_tcp_tc_list_lock so readers walking
+	 * rds_tcp_tc_list under the same lock cannot observe an
+	 * entry whose t_sock is NULL.
+	 */
 	spin_lock(&rds_tcp_tc_list_lock);
+	tc->t_sock = sock;
 	list_add_tail(&tc->t_list_item, &rds_tcp_tc_list);
 #if IS_ENABLED(CONFIG_IPV6)
 	rds6_tcp_tc_count++;
@@ -211,8 +216,6 @@ void rds_tcp_set_callbacks(struct socket *sock, struct rds_conn_path *cp)
 	/* accepted sockets need our listen data ready undone */
 	if (sock->sk->sk_data_ready == rds_tcp_listen_data_ready)
 		sock->sk->sk_data_ready = sock->sk->sk_user_data;
-
-	tc->t_sock = sock;
 	if (!tc->t_rtn)
 		tc->t_rtn = net_generic(sock_net(sock->sk), rds_tcp_netid);
 	tc->t_cpath = cp;
diff --git a/net/rds/tcp_listen.c b/net/rds/tcp_listen.c
index 08a506aa7ce7a..a3db9b057084d 100644
--- a/net/rds/tcp_listen.c
+++ b/net/rds/tcp_listen.c
@@ -69,7 +69,7 @@ rds_tcp_get_peer_sport(struct socket *sock)
 
 /* rds_tcp_accept_one_path(): if accepting on cp_index > 0, make sure the
  * client's ipaddr < server's ipaddr. Otherwise, close the accepted
- * socket and force a reconneect from smaller -> larger ip addr. The reason
+ * socket and force a reconnect from smaller -> larger ip addr. The reason
  * we special case cp_index 0 is to allow the rds probe ping itself to itself
  * get through efficiently.
  */
@@ -143,7 +143,7 @@ void rds_tcp_conn_slots_available(struct rds_connection *conn, bool fan_out)
 	 *
 	 * Doing so is necessary to address the case where an
 	 * incoming connection on "rds_tcp_listen_sock" is ready
-	 * to be acccepted prior to a free slot being available:
+	 * to be accepted prior to a free slot being available:
 	 * the -ENOBUFS case in "rds_tcp_accept_one".
 	 */
 	rds_tcp_accept_work(rtn);
diff --git a/net/rds/tcp_recv.c b/net/rds/tcp_recv.c
index 49f96ee0c40f6..ffe843ca219c7 100644
--- a/net/rds/tcp_recv.c
+++ b/net/rds/tcp_recv.c
@@ -275,7 +275,7 @@ static int rds_tcp_read_sock(struct rds_conn_path *cp, gfp_t gfp)
 	desc.count = 1; /* give more than one skb per call */
 
 	tcp_read_sock(sock->sk, &desc, rds_tcp_data_recv);
-	rdsdebug("tcp_read_sock for tc %p gfp 0x%x returned %d\n", tc, gfp,
+	rdsdebug("tcp_read_sock for tc %p gfp %pGg returned %d\n", tc, &gfp,
 		 desc.error);
 
 	if (skb_queue_empty_lockless(&sock->sk->sk_receive_queue) &&
diff --git a/net/rxrpc/ar-internal.h b/net/rxrpc/ar-internal.h
index 27c2aa2dd023c..98f2165159d72 100644
--- a/net/rxrpc/ar-internal.h
+++ b/net/rxrpc/ar-internal.h
@@ -213,8 +213,6 @@ struct rxrpc_skb_priv {
 		struct {
 			u16		offset;		/* Offset of data */
 			u16		len;		/* Length of data */
-			u8		flags;
-#define RXRPC_RX_VERIFIED	0x01
 		};
 		struct {
 			rxrpc_seq_t	first_ack;	/* First packet in acks table */
@@ -309,15 +307,16 @@ struct rxrpc_security {
 				    struct sk_buff *challenge);
 
 	/* verify a response */
-	int (*verify_response)(struct rxrpc_connection *,
-			       struct sk_buff *);
+	int (*verify_response)(struct rxrpc_connection *conn,
+			       struct sk_buff *response_skb,
+			       void *response, unsigned int len);
 
 	/* clear connection security */
 	void (*clear)(struct rxrpc_connection *);
 
 	/* Default ticket -> key decoder */
 	int (*default_decode_ticket)(struct rxrpc_connection *conn, struct sk_buff *skb,
-				     unsigned int ticket_offset, unsigned int ticket_len,
+				     void *ticket, unsigned int ticket_len,
 				     struct key **_key);
 };
 
@@ -774,6 +773,11 @@ struct rxrpc_call {
 	struct sk_buff_head	recvmsg_queue;	/* Queue of packets ready for recvmsg() */
 	struct sk_buff_head	rx_queue;	/* Queue of packets for this call to receive */
 	struct sk_buff_head	rx_oos_queue;	/* Queue of out of sequence packets */
+	void			*rx_dec_buffer;	/* Decryption buffer */
+	unsigned short		rx_dec_bsize;	/* rx_dec_buffer size */
+	unsigned short		rx_dec_offset;	/* Decrypted packet data offset */
+	unsigned short		rx_dec_len;	/* Decrypted packet data len */
+	rxrpc_seq_t		rx_dec_seq;	/* Packet in decryption buffer */
 
 	rxrpc_seq_t		rx_highest_seq;	/* Higest sequence number received */
 	rxrpc_seq_t		rx_consumed;	/* Highest packet consumed */
diff --git a/net/rxrpc/call_event.c b/net/rxrpc/call_event.c
index 2b19b252225e5..fec59d9338b9f 100644
--- a/net/rxrpc/call_event.c
+++ b/net/rxrpc/call_event.c
@@ -332,27 +332,7 @@ bool rxrpc_input_call_event(struct rxrpc_call *call)
 
 			saw_ack |= sp->hdr.type == RXRPC_PACKET_TYPE_ACK;
 
-			if (sp->hdr.type == RXRPC_PACKET_TYPE_DATA &&
-			    sp->hdr.securityIndex != 0 &&
-			    (skb_cloned(skb) ||
-			     skb_has_frag_list(skb) ||
-			     skb_has_shared_frag(skb))) {
-				/* Unshare the packet so that it can be
-				 * modified by in-place decryption.
-				 */
-				struct sk_buff *nskb = skb_copy(skb, GFP_ATOMIC);
-
-				if (nskb) {
-					rxrpc_new_skb(nskb, rxrpc_skb_new_unshared);
-					rxrpc_input_call_packet(call, nskb);
-					rxrpc_free_skb(nskb, rxrpc_skb_put_call_rx);
-				} else {
-					/* OOM - Drop the packet. */
-					rxrpc_see_skb(skb, rxrpc_skb_see_unshare_nomem);
-				}
-			} else {
-				rxrpc_input_call_packet(call, skb);
-			}
+			rxrpc_input_call_packet(call, skb);
 			rxrpc_free_skb(skb, rxrpc_skb_put_call_rx);
 			did_receive = true;
 		}
diff --git a/net/rxrpc/call_object.c b/net/rxrpc/call_object.c
index f035f486c1397..fcb9d38bb5214 100644
--- a/net/rxrpc/call_object.c
+++ b/net/rxrpc/call_object.c
@@ -152,6 +152,7 @@ struct rxrpc_call *rxrpc_alloc_call(struct rxrpc_sock *rx, gfp_t gfp,
 	spin_lock_init(&call->notify_lock);
 	refcount_set(&call->ref, 1);
 	call->debug_id		= debug_id;
+	call->rx_pkt_offset	= USHRT_MAX;
 	call->tx_total_len	= -1;
 	call->tx_jumbo_max	= 1;
 	call->next_rx_timo	= 20 * HZ;
@@ -553,6 +554,7 @@ static void rxrpc_cleanup_rx_buffers(struct rxrpc_call *call)
 	rxrpc_purge_queue(&call->recvmsg_queue);
 	rxrpc_purge_queue(&call->rx_queue);
 	rxrpc_purge_queue(&call->rx_oos_queue);
+	kfree(call->rx_dec_buffer);
 }
 
 /*
diff --git a/net/rxrpc/conn_event.c b/net/rxrpc/conn_event.c
index 442414d90ba1c..c96ca615b787c 100644
--- a/net/rxrpc/conn_event.c
+++ b/net/rxrpc/conn_event.c
@@ -243,28 +243,22 @@ static void rxrpc_call_is_secure(struct rxrpc_call *call)
 static int rxrpc_verify_response(struct rxrpc_connection *conn,
 				 struct sk_buff *skb)
 {
+	unsigned int len = skb->len - sizeof(struct rxrpc_wire_header);
+	void *buffer;
 	int ret;
 
-	if (skb_cloned(skb) || skb_has_frag_list(skb) ||
-	    skb_has_shared_frag(skb)) {
-		/* Copy the packet if shared so that we can do in-place
-		 * decryption.
-		 */
-		struct sk_buff *nskb = skb_copy(skb, GFP_NOFS);
-
-		if (nskb) {
-			rxrpc_new_skb(nskb, rxrpc_skb_new_unshared);
-			ret = conn->security->verify_response(conn, nskb);
-			rxrpc_free_skb(nskb, rxrpc_skb_put_response_copy);
-		} else {
-			/* OOM - Drop the packet. */
-			rxrpc_see_skb(skb, rxrpc_skb_see_unshare_nomem);
-			ret = -ENOMEM;
-		}
-	} else {
-		ret = conn->security->verify_response(conn, skb);
-	}
+	buffer = kmalloc(len, GFP_NOFS);
+	if (!buffer)
+		return -ENOMEM;
+
+	ret = skb_copy_bits(skb, sizeof(struct rxrpc_wire_header), buffer, len);
+	if (ret < 0)
+		goto out;
+
+	ret = conn->security->verify_response(conn, skb, buffer, len);
 
+out:
+	kfree(buffer);
 	return ret;
 }
 
diff --git a/net/rxrpc/insecure.c b/net/rxrpc/insecure.c
index 0a260df45d25a..0b39046bdc616 100644
--- a/net/rxrpc/insecure.c
+++ b/net/rxrpc/insecure.c
@@ -32,9 +32,6 @@ static int none_secure_packet(struct rxrpc_call *call, struct rxrpc_txbuf *txb)
 
 static int none_verify_packet(struct rxrpc_call *call, struct sk_buff *skb)
 {
-	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
-
-	sp->flags |= RXRPC_RX_VERIFIED;
 	return 0;
 }
 
@@ -57,9 +54,10 @@ static int none_sendmsg_respond_to_challenge(struct sk_buff *challenge,
 }
 
 static int none_verify_response(struct rxrpc_connection *conn,
-				struct sk_buff *skb)
+				struct sk_buff *response_skb,
+				void *response, unsigned int len)
 {
-	return rxrpc_abort_conn(conn, skb, RX_PROTOCOL_ERROR, -EPROTO,
+	return rxrpc_abort_conn(conn, response_skb, RX_PROTOCOL_ERROR, -EPROTO,
 				rxrpc_eproto_rxnull_response);
 }
 
diff --git a/net/rxrpc/local_object.c b/net/rxrpc/local_object.c
index 111f574fe667d..169f9dfdaa77f 100644
--- a/net/rxrpc/local_object.c
+++ b/net/rxrpc/local_object.c
@@ -194,7 +194,7 @@ static int rxrpc_open_socket(struct rxrpc_local *local, struct net *net)
 	tuncfg.encap_rcv = rxrpc_encap_rcv;
 	tuncfg.encap_err_rcv = rxrpc_encap_err_rcv;
 	tuncfg.sk_user_data = local;
-	setup_udp_tunnel_sock(net, local->socket, &tuncfg);
+	setup_udp_tunnel_sock(net, local->socket->sk, &tuncfg);
 
 	/* set the socket up */
 	usk = local->socket->sk;
diff --git a/net/rxrpc/recvmsg.c b/net/rxrpc/recvmsg.c
index e1f7513a46dbe..c940600117a48 100644
--- a/net/rxrpc/recvmsg.c
+++ b/net/rxrpc/recvmsg.c
@@ -147,15 +147,52 @@ static void rxrpc_rotate_rx_window(struct rxrpc_call *call)
 }
 
 /*
- * Decrypt and verify a DATA packet.
+ * Decrypt and verify a DATA packet.  The content of the packet is pulled out
+ * into a flat buffer rather than decrypting in place in the skbuff.  This also
+ * has the advantage of aligning the buffer correctly for the crypto routines.
+ *
+ * We keep track of the sequence number of the packet currently decrypted into
+ * the buffer in ->rx_dec_seq.  If MSG_PEEK is used and steps onto a new
+ * packet, subsequent recvmsg() calls will have to go back and re-decrypt the
+ * current packet.
  */
 static int rxrpc_verify_data(struct rxrpc_call *call, struct sk_buff *skb)
 {
 	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
+	int ret;
 
-	if (sp->flags & RXRPC_RX_VERIFIED)
-		return 0;
-	return call->security->verify_packet(call, skb);
+	if (sp->len > call->rx_dec_bsize) {
+		/* Make sure we can hold a 1412-byte jumbo subpacket and make
+		 * sure that the buffer size is aligned to a crypto blocksize.
+		 */
+		size_t size = clamp(round_up(sp->len, 32), 2048, 65535);
+		void *buffer = krealloc(call->rx_dec_buffer, size, GFP_NOFS);
+
+		if (!buffer)
+			return -ENOMEM;
+		call->rx_dec_buffer = buffer;
+		call->rx_dec_bsize = size;
+	}
+
+	ret = -EFAULT;
+	if (skb_copy_bits(skb, sp->offset, call->rx_dec_buffer, sp->len) < 0)
+		goto err;
+
+	call->rx_dec_offset = 0;
+	call->rx_dec_len = sp->len;
+	call->rx_dec_seq = sp->hdr.seq;
+	ret = call->security->verify_packet(call, skb);
+	if (ret < 0)
+		goto err;
+	return 0;
+
+err:
+	kfree(call->rx_dec_buffer);
+	call->rx_dec_buffer = NULL;
+	call->rx_dec_bsize = 0;
+	call->rx_dec_offset = 0;
+	call->rx_dec_len = 0;
+	return ret;
 }
 
 /*
@@ -283,16 +320,21 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
 		if (msg)
 			sock_recv_timestamp(msg, sock->sk, skb);
 
-		if (rx_pkt_offset == 0) {
+		if (call->rx_dec_seq != sp->hdr.seq ||
+		    !call->rx_dec_buffer) {
 			ret2 = rxrpc_verify_data(call, skb);
 			trace_rxrpc_recvdata(call, rxrpc_recvmsg_next, seq,
-					     sp->offset, sp->len, ret2);
+					     call->rx_dec_offset,
+					     call->rx_dec_len, ret2);
 			if (ret2 < 0) {
 				ret = ret2;
 				goto out;
 			}
-			rx_pkt_offset = sp->offset;
-			rx_pkt_len = sp->len;
+		}
+
+		if (rx_pkt_offset == USHRT_MAX) {
+			rx_pkt_offset = call->rx_dec_offset;
+			rx_pkt_len = call->rx_dec_len;
 		} else {
 			trace_rxrpc_recvdata(call, rxrpc_recvmsg_cont, seq,
 					     rx_pkt_offset, rx_pkt_len, 0);
@@ -304,10 +346,10 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
 		if (copy > remain)
 			copy = remain;
 		if (copy > 0) {
-			ret2 = skb_copy_datagram_iter(skb, rx_pkt_offset, iter,
-						      copy);
-			if (ret2 < 0) {
-				ret = ret2;
+			ret2 = copy_to_iter(call->rx_dec_buffer + rx_pkt_offset,
+					    copy, iter);
+			if (ret2 != copy) {
+				ret = -EFAULT;
 				goto out;
 			}
 
@@ -328,7 +370,7 @@ static int rxrpc_recvmsg_data(struct socket *sock, struct rxrpc_call *call,
 		/* The whole packet has been transferred. */
 		if (sp->hdr.flags & RXRPC_LAST_PACKET)
 			ret = 1;
-		rx_pkt_offset = 0;
+		rx_pkt_offset = USHRT_MAX;
 		rx_pkt_len = 0;
 
 		skb = skb_peek_next(skb, &call->recvmsg_queue);
diff --git a/net/rxrpc/rxgk.c b/net/rxrpc/rxgk.c
index 0d5e654da918f..a1ee102abae18 100644
--- a/net/rxrpc/rxgk.c
+++ b/net/rxrpc/rxgk.c
@@ -473,15 +473,20 @@ static int rxgk_verify_packet_integrity(struct rxrpc_call *call,
 	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 	struct rxgk_header *hdr;
 	struct krb5_buffer metadata;
-	unsigned int offset = sp->offset, len = sp->len;
+	unsigned int len = call->rx_dec_len;
 	size_t data_offset = 0, data_len = len;
+	void *data = call->rx_dec_buffer, *p = data;
 	u32 ac = 0;
 	int ret = -ENOMEM;
 
 	_enter("");
 
-	crypto_krb5_where_is_the_data(gk->krb5, KRB5_CHECKSUM_MODE,
-				      &data_offset, &data_len);
+	if (crypto_krb5_where_is_the_data(gk->krb5, KRB5_CHECKSUM_MODE,
+					  &data_offset, &data_len) < 0) {
+		ret = rxrpc_abort_eproto(call, skb, RXGK_PACKETSHORT,
+					 rxgk_abort_1_short_header);
+		goto put_gk;
+	}
 
 	hdr = kzalloc_obj(*hdr, GFP_NOFS);
 	if (!hdr)
@@ -496,16 +501,15 @@ static int rxgk_verify_packet_integrity(struct rxrpc_call *call,
 
 	metadata.len = sizeof(*hdr);
 	metadata.data = hdr;
-	ret = rxgk_verify_mic_skb(gk->krb5, gk->rx_Kc, &metadata,
-				  skb, &offset, &len, &ac);
+	ret = rxgk_verify_mic(gk->krb5, gk->rx_Kc, &metadata, &p, &len, &ac);
 	kfree(hdr);
 	if (ret < 0) {
 		if (ret != -ENOMEM)
 			rxrpc_abort_eproto(call, skb, ac,
 					   rxgk_abort_1_verify_mic_eproto);
 	} else {
-		sp->offset = offset;
-		sp->len = len;
+		call->rx_dec_offset = p - data;
+		call->rx_dec_len = len;
 	}
 
 put_gk:
@@ -522,49 +526,53 @@ static int rxgk_verify_packet_encrypted(struct rxrpc_call *call,
 					struct sk_buff *skb)
 {
 	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
-	struct rxgk_header hdr;
-	unsigned int offset = sp->offset, len = sp->len;
+	struct rxgk_header *hdr;
+	unsigned int offset = 0, len = call->rx_dec_len;
+	void *data = call->rx_dec_buffer, *p = data;
 	int ret;
 	u32 ac = 0;
 
 	_enter("");
 
-	ret = rxgk_decrypt_skb(gk->krb5, gk->rx_enc, skb, &offset, &len, &ac);
+	if (crypto_krb5_check_data_len(gk->krb5, KRB5_ENCRYPT_MODE,
+				       len, sizeof(*hdr)) < 0) {
+		ret = rxrpc_abort_eproto(call, skb, RXGK_PACKETSHORT,
+					 rxgk_abort_2_short_header);
+		goto error;
+	}
+
+	ret = rxgk_decrypt(gk->krb5, gk->rx_enc, &p, &len, &ac);
 	if (ret < 0) {
 		if (ret != -ENOMEM)
 			rxrpc_abort_eproto(call, skb, ac, rxgk_abort_2_decrypt_eproto);
 		goto error;
 	}
+	offset = p - data;
 
-	if (len < sizeof(hdr)) {
+	if (len < sizeof(*hdr)) {
 		ret = rxrpc_abort_eproto(call, skb, RXGK_PACKETSHORT,
 					 rxgk_abort_2_short_header);
 		goto error;
 	}
 
 	/* Extract the header from the skb */
-	ret = skb_copy_bits(skb, offset, &hdr, sizeof(hdr));
-	if (ret < 0) {
-		ret = rxrpc_abort_eproto(call, skb, RXGK_PACKETSHORT,
-					 rxgk_abort_2_short_encdata);
-		goto error;
-	}
-	offset += sizeof(hdr);
-	len -= sizeof(hdr);
-
-	if (ntohl(hdr.epoch)		!= call->conn->proto.epoch ||
-	    ntohl(hdr.cid)		!= call->cid ||
-	    ntohl(hdr.call_number)	!= call->call_id ||
-	    ntohl(hdr.seq)		!= sp->hdr.seq ||
-	    ntohl(hdr.sec_index)	!= call->security_ix ||
-	    ntohl(hdr.data_len)		> len) {
+	hdr = data + offset;
+	offset += sizeof(*hdr);
+	len -= sizeof(*hdr);
+
+	if (ntohl(hdr->epoch)		!= call->conn->proto.epoch ||
+	    ntohl(hdr->cid)		!= call->cid ||
+	    ntohl(hdr->call_number)	!= call->call_id ||
+	    ntohl(hdr->seq)		!= sp->hdr.seq ||
+	    ntohl(hdr->sec_index)	!= call->security_ix ||
+	    ntohl(hdr->data_len)	> len) {
 		ret = rxrpc_abort_eproto(call, skb, RXGK_SEALEDINCON,
 					 rxgk_abort_2_short_data);
 		goto error;
 	}
 
-	sp->offset = offset;
-	sp->len = ntohl(hdr.data_len);
+	call->rx_dec_offset = offset;
+	call->rx_dec_len = ntohl(hdr->data_len);
 	ret = 0;
 error:
 	rxgk_put(gk);
@@ -1076,11 +1084,12 @@ static int rxgk_sendmsg_respond_to_challenge(struct sk_buff *challenge,
  *	unsigned int call_numbers<>;
  * };
  */
-static int rxgk_do_verify_authenticator(struct rxrpc_connection *conn,
-					const struct krb5_enctype *krb5,
-					struct sk_buff *skb,
-					__be32 *p, __be32 *end)
+static int rxgk_verify_authenticator(struct rxrpc_connection *conn,
+				     const struct krb5_enctype *krb5,
+				     struct sk_buff *skb,
+				     void *auth, unsigned int auth_len)
 {
+	__be32 *p = auth, *end = auth + auth_len;
 	u32 app_len, call_count, level, epoch, cid, i;
 
 	_enter("");
@@ -1143,37 +1152,6 @@ static int rxgk_do_verify_authenticator(struct rxrpc_connection *conn,
 	return 0;
 }
 
-/*
- * Extract the authenticator and verify it.
- */
-static int rxgk_verify_authenticator(struct rxrpc_connection *conn,
-				     const struct krb5_enctype *krb5,
-				     struct sk_buff *skb,
-				     unsigned int auth_offset, unsigned int auth_len)
-{
-	void *auth;
-	__be32 *p;
-	int ret;
-
-	auth = kmalloc(auth_len, GFP_NOFS);
-	if (!auth)
-		return -ENOMEM;
-
-	ret = skb_copy_bits(skb, auth_offset, auth, auth_len);
-	if (ret < 0) {
-		ret = rxrpc_abort_conn(conn, skb, RXGK_NOTAUTH, -EPROTO,
-				       rxgk_abort_resp_short_auth);
-		goto error;
-	}
-
-	p = auth;
-	ret = rxgk_do_verify_authenticator(conn, krb5, skb, p,
-					   p + auth_len / sizeof(*p));
-error:
-	kfree(auth);
-	return ret;
-}
-
 /*
  * Verify a response.
  *
@@ -1184,49 +1162,45 @@ static int rxgk_verify_authenticator(struct rxrpc_connection *conn,
  * };
  */
 static int rxgk_verify_response(struct rxrpc_connection *conn,
-				struct sk_buff *skb)
+				struct sk_buff *skb,
+				void *buffer, unsigned int len)
 {
 	const struct krb5_enctype *krb5;
 	struct rxrpc_key_token *token;
 	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
-	struct rxgk_response rhdr;
+	struct rxgk_response *rhdr;
 	struct rxgk_context *gk;
 	struct key *key = NULL;
-	unsigned int offset = sizeof(struct rxrpc_wire_header);
-	unsigned int len = skb->len - sizeof(struct rxrpc_wire_header);
-	unsigned int token_offset, token_len;
-	unsigned int auth_offset, auth_len;
+	unsigned int resp_token_len, auth_len;
+	void *resp_token, *auth;
 	__be32 xauth_len;
 	int ret, ec;
 
 	_enter("{%d}", conn->debug_id);
 
 	/* Parse the RXGK_Response object */
-	if (sizeof(rhdr) + sizeof(__be32) > len)
+	if (len < sizeof(*rhdr) + sizeof(__be32))
 		goto short_packet;
-
-	if (skb_copy_bits(skb, offset, &rhdr, sizeof(rhdr)) < 0)
-		goto short_packet;
-	offset	+= sizeof(rhdr);
-	len	-= sizeof(rhdr);
-
-	token_offset	= offset;
-	token_len	= ntohl(rhdr.token_len);
-	if (token_len > len ||
-	    xdr_round_up(token_len) + sizeof(__be32) > len)
+	rhdr = buffer;
+	buffer	+= sizeof(*rhdr);
+	len	-= sizeof(*rhdr);
+
+	resp_token	= buffer;
+	resp_token_len	= ntohl(rhdr->token_len);
+	if (resp_token_len > len ||
+	    xdr_round_up(resp_token_len) + sizeof(__be32) > len)
 		goto short_packet;
 
-	trace_rxrpc_rx_response(conn, sp->hdr.serial, 0, sp->hdr.cksum, token_len);
+	trace_rxrpc_rx_response(conn, sp->hdr.serial, 0, sp->hdr.cksum, resp_token_len);
 
-	offset	+= xdr_round_up(token_len);
-	len	-= xdr_round_up(token_len);
+	buffer	+= xdr_round_up(resp_token_len);
+	len	-= xdr_round_up(resp_token_len);
 
-	if (skb_copy_bits(skb, offset, &xauth_len, sizeof(xauth_len)) < 0)
-		goto short_packet;
-	offset	+= sizeof(xauth_len);
+	xauth_len = *(__be32 *)buffer;
+	buffer	+= sizeof(xauth_len);
 	len	-= sizeof(xauth_len);
 
-	auth_offset	= offset;
+	auth		= buffer;
 	auth_len	= ntohl(xauth_len);
 	if (auth_len > len)
 		goto short_packet;
@@ -1241,7 +1215,7 @@ static int rxgk_verify_response(struct rxrpc_connection *conn,
 	 * to the app to deal with - which might mean a round trip to
 	 * userspace.
 	 */
-	ret = rxgk_extract_token(conn, skb, token_offset, token_len, &key);
+	ret = rxgk_extract_token(conn, skb, resp_token, resp_token_len, &key);
 	if (ret < 0)
 		goto out;
 
@@ -1255,7 +1229,7 @@ static int rxgk_verify_response(struct rxrpc_connection *conn,
 	 */
 	token = key->payload.data[0];
 	conn->security_level = token->rxgk->level;
-	conn->rxgk.start_time = __be64_to_cpu(rhdr.start_time);
+	conn->rxgk.start_time = __be64_to_cpu(rhdr->start_time);
 
 	gk = rxgk_generate_transport_key(conn, token->rxgk, sp->hdr.cksum, GFP_NOFS);
 	if (IS_ERR(gk)) {
@@ -1265,18 +1239,18 @@ static int rxgk_verify_response(struct rxrpc_connection *conn,
 
 	krb5 = gk->krb5;
 
-	trace_rxrpc_rx_response(conn, sp->hdr.serial, krb5->etype, sp->hdr.cksum, token_len);
+	trace_rxrpc_rx_response(conn, sp->hdr.serial, krb5->etype, sp->hdr.cksum,
+				resp_token_len);
 
 	/* Decrypt, parse and verify the authenticator. */
-	ret = rxgk_decrypt_skb(krb5, gk->resp_enc, skb,
-			       &auth_offset, &auth_len, &ec);
+	ret = rxgk_decrypt(krb5, gk->resp_enc, &auth, &auth_len, &ec);
 	if (ret < 0) {
 		rxrpc_abort_conn(conn, skb, RXGK_SEALEDINCON, ret,
 				 rxgk_abort_resp_auth_dec);
 		goto out_gk;
 	}
 
-	ret = rxgk_verify_authenticator(conn, krb5, skb, auth_offset, auth_len);
+	ret = rxgk_verify_authenticator(conn, krb5, skb, auth, auth_len);
 	if (ret < 0)
 		goto out_gk;
 
diff --git a/net/rxrpc/rxgk_app.c b/net/rxrpc/rxgk_app.c
index 0ef2a29eb6958..200a30064fae1 100644
--- a/net/rxrpc/rxgk_app.c
+++ b/net/rxrpc/rxgk_app.c
@@ -40,7 +40,7 @@
  * };
  */
 int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
-			   unsigned int ticket_offset, unsigned int ticket_len,
+			   void *buffer, unsigned int ticket_len,
 			   struct key **_key)
 {
 	struct rxrpc_key_token *token;
@@ -49,7 +49,7 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
 	size_t pre_ticket_len, payload_len;
 	unsigned int klen, enctype;
 	void *payload, *ticket;
-	__be32 *t, *p, *q, tmp[2];
+	__be32 *t, *p, *q, *tmp;
 	int ret;
 
 	_enter("");
@@ -59,10 +59,7 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
 					rxgk_abort_resp_short_yfs_tkt);
 
 	/* Get the session key length */
-	ret = skb_copy_bits(skb, ticket_offset, tmp, sizeof(tmp));
-	if (ret < 0)
-		return rxrpc_abort_conn(conn, skb, RXGK_INCONSISTENCY, -EPROTO,
-					rxgk_abort_resp_short_yfs_klen);
+	tmp = buffer;
 	enctype = ntohl(tmp[0]);
 	klen = ntohl(tmp[1]);
 
@@ -84,12 +81,7 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
 	 * it.
 	 */
 	ticket = payload + pre_ticket_len;
-	ret = skb_copy_bits(skb, ticket_offset, ticket, ticket_len);
-	if (ret < 0) {
-		ret = rxrpc_abort_conn(conn, skb, RXGK_INCONSISTENCY, -EPROTO,
-				       rxgk_abort_resp_short_yfs_tkt);
-		goto error;
-	}
+	memcpy(ticket, buffer, ticket_len);
 
 	/* Fill out the form header. */
 	p = payload;
@@ -131,7 +123,7 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
 		goto error;
 	}
 
-	/* Ticket read in with skb_copy_bits above */
+	/* Ticket appended above. */
 	q += xdr_round_up(ticket_len) / 4;
 	if (WARN_ON((unsigned long)q - (unsigned long)payload != payload_len)) {
 		ret = -EIO;
@@ -182,14 +174,15 @@ int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
  * [tools.ietf.org/html/draft-wilkinson-afs3-rxgk-afs-08 sec 6.1]
  */
 int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
-		       unsigned int token_offset, unsigned int token_len,
+		       void *token, unsigned int token_len,
 		       struct key **_key)
 {
 	const struct krb5_enctype *krb5;
 	const struct krb5_buffer *server_secret;
 	struct crypto_aead *token_enc = NULL;
 	struct key *server_key;
-	unsigned int ticket_offset, ticket_len;
+	unsigned int ticket_len;
+	void *ticket;
 	u32 kvno, enctype;
 	int ret, ec = 0;
 
@@ -197,24 +190,23 @@ int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
 		__be32 kvno;
 		__be32 enctype;
 		__be32 token_len;
-	} container;
+	} *container;
 
-	if (token_len < sizeof(container))
+	if (token_len < sizeof(*container))
 		goto short_packet;
 
 	/* Decode the RXGK_TokenContainer object.  This tells us which server
 	 * key we should be using.  We can then fetch the key, get the secret
 	 * and set up the crypto to extract the token.
 	 */
-	if (skb_copy_bits(skb, token_offset, &container, sizeof(container)) < 0)
-		goto short_packet;
+	container = token;
+	token += sizeof(*container);
 
-	kvno		= ntohl(container.kvno);
-	enctype		= ntohl(container.enctype);
-	ticket_len	= ntohl(container.token_len);
-	ticket_offset	= token_offset + sizeof(container);
+	kvno		= ntohl(container->kvno);
+	enctype		= ntohl(container->enctype);
+	ticket_len	= ntohl(container->token_len);
 
-	if (ticket_len > xdr_round_down(token_len - sizeof(container)))
+	if (ticket_len > xdr_round_down(token_len - sizeof(*container)))
 		goto short_packet;
 
 	_debug("KVNO %u", kvno);
@@ -237,8 +229,8 @@ int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
 	 * gain access to K0, from which we can derive the transport key and
 	 * thence decode the authenticator.
 	 */
-	ret = rxgk_decrypt_skb(krb5, token_enc, skb,
-			       &ticket_offset, &ticket_len, &ec);
+	ticket = token;
+	ret = rxgk_decrypt(krb5, token_enc, &ticket, &ticket_len, &ec);
 	crypto_free_aead(token_enc);
 	token_enc = NULL;
 	if (ret < 0) {
@@ -248,7 +240,7 @@ int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
 		return ret;
 	}
 
-	ret = conn->security->default_decode_ticket(conn, skb, ticket_offset,
+	ret = conn->security->default_decode_ticket(conn, skb, ticket,
 						    ticket_len, _key);
 	if (ret < 0)
 		goto cant_get_token;
diff --git a/net/rxrpc/rxgk_common.h b/net/rxrpc/rxgk_common.h
index 1e257d7ab8ec1..3deed5863f5aa 100644
--- a/net/rxrpc/rxgk_common.h
+++ b/net/rxrpc/rxgk_common.h
@@ -41,10 +41,10 @@ struct rxgk_context {
  * rxgk_app.c
  */
 int rxgk_yfs_decode_ticket(struct rxrpc_connection *conn, struct sk_buff *skb,
-			   unsigned int ticket_offset, unsigned int ticket_len,
+			   void *ticket, unsigned int ticket_len,
 			   struct key **_key);
 int rxgk_extract_token(struct rxrpc_connection *conn, struct sk_buff *skb,
-		       unsigned int token_offset, unsigned int token_len,
+		       void *token, unsigned int token_len,
 		       struct key **_key);
 
 /*
@@ -62,31 +62,30 @@ int rxgk_set_up_token_cipher(const struct krb5_buffer *server_key,
 			     gfp_t gfp);
 
 /*
- * Apply decryption and checksumming functions to part of an skbuff.  The
- * offset and length are updated to reflect the actual content of the encrypted
+ * Apply decryption and checksumming functions a flat data buffer.  The data
+ * point and length are updated to reflect the actual content of the encrypted
  * region.
  */
-static inline
-int rxgk_decrypt_skb(const struct krb5_enctype *krb5,
-		     struct crypto_aead *aead,
-		     struct sk_buff *skb,
-		     unsigned int *_offset, unsigned int *_len,
-		     int *_error_code)
+static inline int rxgk_decrypt(const struct krb5_enctype *krb5,
+			       struct crypto_aead *aead,
+			       void **_data, unsigned int *_len,
+			       int *_error_code)
 {
-	struct scatterlist sg[16];
+	struct scatterlist sg[1];
 	size_t offset = 0, len = *_len;
-	int nr_sg, ret;
+	int ret;
 
-	sg_init_table(sg, ARRAY_SIZE(sg));
-	nr_sg = skb_to_sgvec(skb, sg, *_offset, len);
-	if (unlikely(nr_sg < 0))
-		return nr_sg;
+	sg_init_one(sg, *_data, len);
 
-	ret = crypto_krb5_decrypt(krb5, aead, sg, nr_sg,
-				  &offset, &len);
+	ret = crypto_krb5_decrypt(krb5, aead, sg, 1, &offset, &len);
 	switch (ret) {
 	case 0:
-		*_offset += offset;
+		if (offset & 3) {
+			*_error_code = RXGK_INCONSISTENCY;
+			ret = -EPROTO;
+			break;
+		}
+		*_data += offset;
 		*_len = len;
 		break;
 	case -EBADMSG: /* Checksum mismatch. */
@@ -106,31 +105,26 @@ int rxgk_decrypt_skb(const struct krb5_enctype *krb5,
 }
 
 /*
- * Check the MIC on a region of an skbuff.  The offset and length are updated
- * to reflect the actual content of the secure region.
+ * Check the MIC on a flat buffer.  The data pointer and length are updated to
+ * reflect the actual content of the secure region.
  */
 static inline
-int rxgk_verify_mic_skb(const struct krb5_enctype *krb5,
-			struct crypto_shash *shash,
-			const struct krb5_buffer *metadata,
-			struct sk_buff *skb,
-			unsigned int *_offset, unsigned int *_len,
-			u32 *_error_code)
+int rxgk_verify_mic(const struct krb5_enctype *krb5,
+		    struct crypto_shash *shash,
+		    const struct krb5_buffer *metadata,
+		    void **_data, unsigned int *_len,
+		    u32 *_error_code)
 {
-	struct scatterlist sg[16];
+	struct scatterlist sg[1];
 	size_t offset = 0, len = *_len;
-	int nr_sg, ret;
+	int ret;
 
-	sg_init_table(sg, ARRAY_SIZE(sg));
-	nr_sg = skb_to_sgvec(skb, sg, *_offset, len);
-	if (unlikely(nr_sg < 0))
-		return nr_sg;
+	sg_init_one(sg, *_data, len);
 
-	ret = crypto_krb5_verify_mic(krb5, shash, metadata, sg, nr_sg,
-				     &offset, &len);
+	ret = crypto_krb5_verify_mic(krb5, shash, metadata, sg, 1, &offset, &len);
 	switch (ret) {
 	case 0:
-		*_offset += offset;
+		*_data += offset;
 		*_len = len;
 		break;
 	case -EBADMSG: /* Checksum mismatch */
diff --git a/net/rxrpc/rxkad.c b/net/rxrpc/rxkad.c
index cba7935977f0b..6fbd883401acd 100644
--- a/net/rxrpc/rxkad.c
+++ b/net/rxrpc/rxkad.c
@@ -430,27 +430,25 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb,
 				 rxrpc_seq_t seq,
 				 struct skcipher_request *req)
 {
-	struct rxkad_level1_hdr sechdr;
+	struct rxkad_level1_hdr *sechdr;
 	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 	struct rxrpc_crypt iv;
-	struct scatterlist sg[16];
-	u32 data_size, buf;
+	struct scatterlist sg[1];
+	void *data = call->rx_dec_buffer;
+	u32 len = sp->len, data_size, buf;
 	u16 check;
 	int ret;
 
 	_enter("");
 
-	if (sp->len < 8)
+	if (len < 8)
 		return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON,
 					  rxkad_abort_1_short_header);
 
 	/* Decrypt the skbuff in-place.  TODO: We really want to decrypt
 	 * directly into the target buffer.
 	 */
-	sg_init_table(sg, ARRAY_SIZE(sg));
-	ret = skb_to_sgvec(skb, sg, sp->offset, 8);
-	if (unlikely(ret < 0))
-		return ret;
+	sg_init_one(sg, data, len);
 
 	/* start the decryption afresh */
 	memset(&iv, 0, sizeof(iv));
@@ -464,13 +462,11 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb,
 		return ret;
 
 	/* Extract the decrypted packet length */
-	if (skb_copy_bits(skb, sp->offset, &sechdr, sizeof(sechdr)) < 0)
-		return rxrpc_abort_eproto(call, skb, RXKADDATALEN,
-					  rxkad_abort_1_short_encdata);
-	sp->offset += sizeof(sechdr);
-	sp->len    -= sizeof(sechdr);
+	sechdr = data;
+	call->rx_dec_offset = sizeof(*sechdr);
+	len -= sizeof(*sechdr);
 
-	buf = ntohl(sechdr.data_size);
+	buf = ntohl(sechdr->data_size);
 	data_size = buf & 0xffff;
 
 	check = buf >> 16;
@@ -479,10 +475,10 @@ static int rxkad_verify_packet_1(struct rxrpc_call *call, struct sk_buff *skb,
 	if (check != 0)
 		return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON,
 					  rxkad_abort_1_short_check);
-	if (data_size > sp->len)
+	if (data_size > len)
 		return rxrpc_abort_eproto(call, skb, RXKADDATALEN,
 					  rxkad_abort_1_short_data);
-	sp->len = data_size;
+	call->rx_dec_len = data_size;
 
 	_leave(" = 0 [dlen=%x]", data_size);
 	return 0;
@@ -496,43 +492,28 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb,
 				 struct skcipher_request *req)
 {
 	const struct rxrpc_key_token *token;
-	struct rxkad_level2_hdr sechdr;
+	struct rxkad_level2_hdr *sechdr;
 	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 	struct rxrpc_crypt iv;
-	struct scatterlist _sg[4], *sg;
-	u32 data_size, buf;
+	struct scatterlist sg[1];
+	void *data = call->rx_dec_buffer;
+	u32 len = sp->len, data_size, buf;
 	u16 check;
-	int nsg, ret;
+	int ret;
 
-	_enter(",{%d}", sp->len);
+	_enter(",{%d}", len);
 
-	if (sp->len < 8)
+	if (len < 8)
 		return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON,
 					  rxkad_abort_2_short_header);
 
 	/* Don't let the crypto algo see a misaligned length. */
-	sp->len = round_down(sp->len, 8);
+	len = round_down(len, 8);
 
-	/* Decrypt the skbuff in-place.  TODO: We really want to decrypt
-	 * directly into the target buffer.
+	/* Decrypt in place in the call's decryption buffer.  TODO: We really
+	 * want to decrypt directly into the target buffer.
 	 */
-	sg = _sg;
-	nsg = skb_shinfo(skb)->nr_frags + 1;
-	if (nsg <= 4) {
-		nsg = 4;
-	} else {
-		sg = kmalloc_objs(*sg, nsg, GFP_NOIO);
-		if (!sg)
-			return -ENOMEM;
-	}
-
-	sg_init_table(sg, nsg);
-	ret = skb_to_sgvec(skb, sg, sp->offset, sp->len);
-	if (unlikely(ret < 0)) {
-		if (sg != _sg)
-			kfree(sg);
-		return ret;
-	}
+	sg_init_one(sg, data, len);
 
 	/* decrypt from the session key */
 	token = call->conn->key->payload.data[0];
@@ -540,11 +521,9 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb,
 
 	skcipher_request_set_sync_tfm(req, call->conn->rxkad.cipher);
 	skcipher_request_set_callback(req, 0, NULL, NULL);
-	skcipher_request_set_crypt(req, sg, sg, sp->len, iv.x);
+	skcipher_request_set_crypt(req, sg, sg, len, iv.x);
 	ret = crypto_skcipher_decrypt(req);
 	skcipher_request_zero(req);
-	if (sg != _sg)
-		kfree(sg);
 	if (ret < 0) {
 		if (ret == -ENOMEM)
 			return ret;
@@ -553,13 +532,11 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb,
 	}
 
 	/* Extract the decrypted packet length */
-	if (skb_copy_bits(skb, sp->offset, &sechdr, sizeof(sechdr)) < 0)
-		return rxrpc_abort_eproto(call, skb, RXKADDATALEN,
-					  rxkad_abort_2_short_len);
-	sp->offset += sizeof(sechdr);
-	sp->len    -= sizeof(sechdr);
+	sechdr = data;
+	call->rx_dec_offset = sizeof(*sechdr);
+	len -= sizeof(*sechdr);
 
-	buf = ntohl(sechdr.data_size);
+	buf = ntohl(sechdr->data_size);
 	data_size = buf & 0xffff;
 
 	check = buf >> 16;
@@ -569,17 +546,18 @@ static int rxkad_verify_packet_2(struct rxrpc_call *call, struct sk_buff *skb,
 		return rxrpc_abort_eproto(call, skb, RXKADSEALEDINCON,
 					  rxkad_abort_2_short_check);
 
-	if (data_size > sp->len)
+	if (data_size > len)
 		return rxrpc_abort_eproto(call, skb, RXKADDATALEN,
 					  rxkad_abort_2_short_data);
 
-	sp->len = data_size;
+	call->rx_dec_len = data_size;
 	_leave(" = 0 [dlen=%x]", data_size);
 	return 0;
 }
 
 /*
- * Verify the security on a received packet and the subpackets therein.
+ * Verify the security on a received (sub)packet.  If the packet needs
+ * modifying (e.g. decrypting), it must be copied.
  */
 static int rxkad_verify_packet(struct rxrpc_call *call, struct sk_buff *skb)
 {
@@ -985,7 +963,6 @@ static int rxkad_decrypt_ticket(struct rxrpc_connection *conn,
 	*_expiry = 0;
 
 	ASSERT(server_key->payload.data[0] != NULL);
-	ASSERTCMP((unsigned long) ticket & 7UL, ==, 0);
 
 	memcpy(&iv, &server_key->payload.data[2], sizeof(iv));
 
@@ -1134,14 +1111,15 @@ static int rxkad_decrypt_response(struct rxrpc_connection *conn,
  * verify a response
  */
 static int rxkad_verify_response(struct rxrpc_connection *conn,
-				 struct sk_buff *skb)
+				 struct sk_buff *skb,
+				 void *buffer, unsigned int len)
 {
 	struct rxkad_response *response;
 	struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 	struct rxrpc_crypt session_key;
 	struct key *server_key;
 	time64_t expiry;
-	void *ticket = NULL;
+	void *ticket;
 	u32 version, kvno, ticket_len, level;
 	__be32 csum;
 	int ret, i;
@@ -1164,13 +1142,8 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
 		}
 	}
 
-	ret = -ENOMEM;
-	response = kzalloc_obj(struct rxkad_response, GFP_NOFS);
-	if (!response)
-		goto error;
-
-	if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header),
-			  response, sizeof(*response)) < 0) {
+	response = buffer;
+	if (len < sizeof(*response)) {
 		ret = rxrpc_abort_conn(conn, skb, RXKADPACKETSHORT, -EPROTO,
 				       rxkad_abort_resp_short);
 		goto error;
@@ -1182,6 +1155,9 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
 
 	trace_rxrpc_rx_response(conn, sp->hdr.serial, version, kvno, ticket_len);
 
+	buffer	+= sizeof(*response);
+	len	-= sizeof(*response);
+
 	if (version != RXKAD_VERSION) {
 		ret = rxrpc_abort_conn(conn, skb, RXKADINCONSISTENCY, -EPROTO,
 				       rxkad_abort_resp_version);
@@ -1201,13 +1177,8 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
 	}
 
 	/* extract the kerberos ticket and decrypt and decode it */
-	ret = -ENOMEM;
-	ticket = kmalloc(ticket_len, GFP_NOFS);
-	if (!ticket)
-		goto error;
-
-	if (skb_copy_bits(skb, sizeof(struct rxrpc_wire_header) + sizeof(*response),
-			  ticket, ticket_len) < 0) {
+	ticket = buffer;
+	if (ticket_len > len) {
 		ret = rxrpc_abort_conn(conn, skb, RXKADPACKETSHORT, -EPROTO,
 				       rxkad_abort_resp_short_tkt);
 		goto error;
@@ -1287,8 +1258,6 @@ static int rxkad_verify_response(struct rxrpc_connection *conn,
 	ret = rxrpc_get_server_data_key(conn, &session_key, expiry, kvno);
 
 error:
-	kfree(ticket);
-	kfree(response);
 	key_put(server_key);
 	_leave(" = %d", ret);
 	return ret;
diff --git a/net/sched/act_api.c b/net/sched/act_api.c
index 332fd9695e54a..551992683d9e6 100644
--- a/net/sched/act_api.c
+++ b/net/sched/act_api.c
@@ -1578,7 +1578,7 @@ void tcf_action_update_stats(struct tc_action *a, u64 bytes, u64 packets,
 	if (a->cpu_bstats) {
 		_bstats_update(this_cpu_ptr(a->cpu_bstats), bytes, packets);
 
-		this_cpu_ptr(a->cpu_qstats)->drops += drops;
+		this_cpu_add(a->cpu_qstats->drops, drops);
 
 		if (hw)
 			_bstats_update(this_cpu_ptr(a->cpu_bstats_hw),
diff --git a/net/sched/act_bpf.c b/net/sched/act_bpf.c
index c2b5bc19e0911..58a0746511767 100644
--- a/net/sched/act_bpf.c
+++ b/net/sched/act_bpf.c
@@ -76,7 +76,7 @@ TC_INDIRECT_SCOPE int tcf_bpf_act(struct sk_buff *skb,
 		break;
 	case TC_ACT_SHOT:
 		action = filter_res;
-		qstats_drop_inc(this_cpu_ptr(prog->common.cpu_qstats));
+		qstats_cpu_drop_inc(prog->common.cpu_qstats);
 		break;
 	case TC_ACT_UNSPEC:
 		action = prog->tcf_action;
diff --git a/net/sched/act_ife.c b/net/sched/act_ife.c
index d5e8a91bb4eb9..065228026c58e 100644
--- a/net/sched/act_ife.c
+++ b/net/sched/act_ife.c
@@ -727,7 +727,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a,
 
 	tlv_data = ife_decode(skb, &metalen);
 	if (unlikely(!tlv_data)) {
-		qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats));
+		qstats_cpu_drop_inc(ife->common.cpu_qstats);
 		return TC_ACT_SHOT;
 	}
 
@@ -740,7 +740,7 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a,
 		curr_data = ife_tlv_meta_decode(tlv_data, ifehdr_end, &mtype,
 						&dlen, NULL);
 		if (!curr_data) {
-			qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats));
+			qstats_cpu_drop_inc(ife->common.cpu_qstats);
 			return TC_ACT_SHOT;
 		}
 
@@ -750,12 +750,12 @@ static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a,
 			 */
 			pr_info_ratelimited("Unknown metaid %d dlen %d\n",
 					    mtype, dlen);
-			qstats_overlimit_inc(this_cpu_ptr(ife->common.cpu_qstats));
+			qstats_cpu_overlimit_inc(ife->common.cpu_qstats);
 		}
 	}
 
 	if (WARN_ON(tlv_data != ifehdr_end)) {
-		qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats));
+		qstats_cpu_drop_inc(ife->common.cpu_qstats);
 		return TC_ACT_SHOT;
 	}
 
@@ -814,14 +814,14 @@ static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a,
 		/* abuse overlimits to count when we allow packet
 		 * with no metadata
 		 */
-		qstats_overlimit_inc(this_cpu_ptr(ife->common.cpu_qstats));
+		qstats_cpu_overlimit_inc(ife->common.cpu_qstats);
 		return action;
 	}
 	/* could be stupid policy setup or mtu config
 	 * so lets be conservative.. */
 	if ((action == TC_ACT_SHOT) || exceed_mtu) {
 drop:
-		qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats));
+		qstats_cpu_drop_inc(ife->common.cpu_qstats);
 		return TC_ACT_SHOT;
 	}
 
diff --git a/net/sched/act_mpls.c b/net/sched/act_mpls.c
index 1abfaf9d99f1f..4ea8b2e08c3a4 100644
--- a/net/sched/act_mpls.c
+++ b/net/sched/act_mpls.c
@@ -123,7 +123,7 @@ TC_INDIRECT_SCOPE int tcf_mpls_act(struct sk_buff *skb,
 	return p->action;
 
 drop:
-	qstats_drop_inc(this_cpu_ptr(m->common.cpu_qstats));
+	qstats_cpu_drop_inc(m->common.cpu_qstats);
 	return TC_ACT_SHOT;
 }
 
diff --git a/net/sched/act_police.c b/net/sched/act_police.c
index 12ea9e5a60053..b16468a98c55e 100644
--- a/net/sched/act_police.c
+++ b/net/sched/act_police.c
@@ -307,10 +307,10 @@ TC_INDIRECT_SCOPE int tcf_police_act(struct sk_buff *skb,
 	}
 
 inc_overlimits:
-	qstats_overlimit_inc(this_cpu_ptr(police->common.cpu_qstats));
+	qstats_cpu_overlimit_inc(police->common.cpu_qstats);
 inc_drops:
 	if (ret == TC_ACT_SHOT)
-		qstats_drop_inc(this_cpu_ptr(police->common.cpu_qstats));
+		qstats_cpu_drop_inc(police->common.cpu_qstats);
 end:
 	return ret;
 }
diff --git a/net/sched/act_skbedit.c b/net/sched/act_skbedit.c
index a778cdba9258c..bfec6b6684103 100644
--- a/net/sched/act_skbedit.c
+++ b/net/sched/act_skbedit.c
@@ -86,7 +86,7 @@ TC_INDIRECT_SCOPE int tcf_skbedit_act(struct sk_buff *skb,
 	return params->action;
 
 err:
-	qstats_drop_inc(this_cpu_ptr(d->common.cpu_qstats));
+	qstats_cpu_drop_inc(d->common.cpu_qstats);
 	return TC_ACT_SHOT;
 }
 
diff --git a/net/sched/act_skbmod.c b/net/sched/act_skbmod.c
index 23ca46138f040..a464b0a3c1b81 100644
--- a/net/sched/act_skbmod.c
+++ b/net/sched/act_skbmod.c
@@ -87,7 +87,7 @@ TC_INDIRECT_SCOPE int tcf_skbmod_act(struct sk_buff *skb,
 	return p->action;
 
 drop:
-	qstats_overlimit_inc(this_cpu_ptr(d->common.cpu_qstats));
+	qstats_cpu_overlimit_inc(d->common.cpu_qstats);
 	return TC_ACT_SHOT;
 }
 
diff --git a/net/sched/sch_api.c b/net/sched/sch_api.c
index ed869a5ffc737..3c779e5098efd 100644
--- a/net/sched/sch_api.c
+++ b/net/sched/sch_api.c
@@ -805,8 +805,8 @@ void qdisc_tree_reduce_backlog(struct Qdisc *sch, int n, int len)
 			cl = cops->find(sch, parentid);
 			cops->qlen_notify(sch, cl);
 		}
-		sch->q.qlen -= n;
-		sch->qstats.backlog -= len;
+		WRITE_ONCE(sch->q.qlen, sch->q.qlen - n);
+		qstats_backlog_sub(sch, len);
 		__qdisc_qstats_drop(sch, drops);
 	}
 	rcu_read_unlock();
@@ -976,16 +976,20 @@ static int tc_fill_qdisc(struct sk_buff *skb, struct Qdisc *q, u32 clid,
 out_nlmsg_trim:
 nla_put_failure:
 	nlmsg_trim(skb, b);
-	return -1;
+	return -EMSGSIZE;
 }
 
-static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible)
+static bool tc_qdisc_dump_ignore(struct Qdisc *q, bool dump_invisible,
+				 const struct tcmsg *tcm)
 {
 	if (q->flags & TCQ_F_BUILTIN)
 		return true;
 	if ((q->flags & TCQ_F_INVISIBLE) && !dump_invisible)
 		return true;
-
+	if (tcm) {
+		if (tcm->tcm_handle && tcm->tcm_handle != q->handle)
+			return true;
+	}
 	return false;
 }
 
@@ -1000,7 +1004,7 @@ static int qdisc_get_notify(struct net *net, struct sk_buff *oskb,
 	if (!skb)
 		return -ENOBUFS;
 
-	if (!tc_qdisc_dump_ignore(q, false)) {
+	if (!tc_qdisc_dump_ignore(q, false, NULL)) {
 		if (tc_fill_qdisc(skb, q, clid, portid, n->nlmsg_seq, 0,
 				  RTM_NEWQDISC, extack) < 0)
 			goto err_out;
@@ -1030,12 +1034,12 @@ static int qdisc_notify(struct net *net, struct sk_buff *oskb,
 	if (!skb)
 		return -ENOBUFS;
 
-	if (old && !tc_qdisc_dump_ignore(old, false)) {
+	if (old && !tc_qdisc_dump_ignore(old, false, NULL)) {
 		if (tc_fill_qdisc(skb, old, clid, portid, n->nlmsg_seq,
 				  0, RTM_DELQDISC, extack) < 0)
 			goto err_out;
 	}
-	if (new && !tc_qdisc_dump_ignore(new, false)) {
+	if (new && !tc_qdisc_dump_ignore(new, false, NULL)) {
 		if (tc_fill_qdisc(skb, new, clid, portid, n->nlmsg_seq,
 				  old ? NLM_F_REPLACE : 0, RTM_NEWQDISC, extack) < 0)
 			goto err_out;
@@ -1825,22 +1829,27 @@ static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
 			      int *q_idx_p, int s_q_idx, bool recur,
 			      bool dump_invisible)
 {
+	const struct nlmsghdr *nlh = cb->nlh;
 	int ret = 0, q_idx = *q_idx_p;
+	const struct tcmsg *tcm;
 	struct Qdisc *q;
 	int b;
 
 	if (!root)
 		return 0;
 
+	tcm = nlmsg_data(nlh);
 	q = root;
 	if (q_idx < s_q_idx) {
 		q_idx++;
 	} else {
-		if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
-		    tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
-				  cb->nlh->nlmsg_seq, NLM_F_MULTI,
-				  RTM_NEWQDISC, NULL) <= 0)
-			goto done;
+		if (!tc_qdisc_dump_ignore(q, dump_invisible, tcm))
+		    ret = tc_fill_qdisc(skb, q, q->parent,
+					NETLINK_CB(cb->skb).portid,
+					nlh->nlmsg_seq, NLM_F_MULTI,
+					RTM_NEWQDISC, NULL);
+		if (ret < 0)
+			goto out;
 		q_idx++;
 	}
 
@@ -1858,79 +1867,81 @@ static int tc_dump_qdisc_root(struct Qdisc *root, struct sk_buff *skb,
 			q_idx++;
 			continue;
 		}
-		if (!tc_qdisc_dump_ignore(q, dump_invisible) &&
-		    tc_fill_qdisc(skb, q, q->parent, NETLINK_CB(cb->skb).portid,
-				  cb->nlh->nlmsg_seq, NLM_F_MULTI,
-				  RTM_NEWQDISC, NULL) <= 0)
-			goto done;
+		if (!tc_qdisc_dump_ignore(q, dump_invisible, tcm))
+			ret = tc_fill_qdisc(skb, q, q->parent,
+					    NETLINK_CB(cb->skb).portid,
+					    nlh->nlmsg_seq, NLM_F_MULTI,
+					    RTM_NEWQDISC, NULL);
+		if (ret < 0)
+			goto out;
 		q_idx++;
 	}
 
 out:
 	*q_idx_p = q_idx;
 	return ret;
-done:
-	ret = -1;
-	goto out;
 }
 
 static int tc_dump_qdisc(struct sk_buff *skb, struct netlink_callback *cb)
 {
-	struct net *net = sock_net(skb->sk);
-	int idx, q_idx;
-	int s_idx, s_q_idx;
-	struct net_device *dev;
 	const struct nlmsghdr *nlh = cb->nlh;
+	struct net *net = sock_net(skb->sk);
 	struct nlattr *tca[TCA_MAX + 1];
+	struct {
+		unsigned long ifindex;
+		int q_idx;
+	} *ctx = (void *)cb->ctx;
+	const struct tcmsg *tcm;
+	struct net_device *dev;
+	int s_q_idx, q_idx;
 	int err;
 
-	s_idx = cb->args[0];
-	s_q_idx = q_idx = cb->args[1];
-
-	idx = 0;
 	ASSERT_RTNL();
 
 	err = nlmsg_parse_deprecated(nlh, sizeof(struct tcmsg), tca, TCA_MAX,
 				     rtm_tca_policy, cb->extack);
 	if (err < 0)
 		return err;
+	tcm = nlmsg_data(nlh);
+	if (tcm->tcm_ifindex && !ctx->ifindex)
+		ctx->ifindex = tcm->tcm_ifindex;
 
-	for_each_netdev(net, dev) {
+	s_q_idx = ctx->q_idx;
+
+	for_each_netdev_dump(net, dev, ctx->ifindex) {
 		struct netdev_queue *dev_queue;
+		struct Qdisc *q;
+
+		if (tcm->tcm_ifindex && ctx->ifindex != tcm->tcm_ifindex)
+			break;
 
-		if (idx < s_idx)
-			goto cont;
-		if (idx > s_idx)
-			s_q_idx = 0;
 		q_idx = 0;
 
 		netdev_lock_ops(dev);
-		if (tc_dump_qdisc_root(rtnl_dereference(dev->qdisc),
-				       skb, cb, &q_idx, s_q_idx,
-				       true, tca[TCA_DUMP_INVISIBLE]) < 0) {
-			netdev_unlock_ops(dev);
-			goto done;
-		}
+		q = rtnl_dereference(dev->qdisc);
+		err = tc_dump_qdisc_root(q, skb, cb, &q_idx, s_q_idx,
+					 true, tca[TCA_DUMP_INVISIBLE]);
+		if (err < 0)
+			goto error_unlock;
 
 		dev_queue = dev_ingress_queue(dev);
-		if (dev_queue &&
-		    tc_dump_qdisc_root(rtnl_dereference(dev_queue->qdisc_sleeping),
-				       skb, cb, &q_idx, s_q_idx, false,
-				       tca[TCA_DUMP_INVISIBLE]) < 0) {
-			netdev_unlock_ops(dev);
-			goto done;
+		if (dev_queue) {
+			q = rtnl_dereference(dev_queue->qdisc_sleeping);
+			err = tc_dump_qdisc_root(q, skb, cb, &q_idx, s_q_idx,
+						 false, tca[TCA_DUMP_INVISIBLE]);
+			if (err < 0)
+				goto error_unlock;
 		}
 		netdev_unlock_ops(dev);
-
-cont:
-		idx++;
+		s_q_idx = 0;
 	}
+	return skb->len;
 
-done:
-	cb->args[0] = idx;
-	cb->args[1] = q_idx;
+error_unlock:
+	netdev_unlock_ops(dev);
+	ctx->q_idx = q_idx;
 
-	return skb->len;
+	return err;
 }
 
 
@@ -1987,15 +1998,16 @@ static int tc_fill_tclass(struct sk_buff *skb, struct Qdisc *q,
 out_nlmsg_trim:
 nla_put_failure:
 	nlmsg_trim(skb, b);
-	return -1;
+	return -EMSGSIZE;
 }
 
 static int tclass_notify(struct net *net, struct sk_buff *oskb,
 			 struct nlmsghdr *n, struct Qdisc *q,
 			 unsigned long cl, int event, struct netlink_ext_ack *extack)
 {
-	struct sk_buff *skb;
 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
+	struct sk_buff *skb;
+	int ret;
 
 	if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC))
 		return 0;
@@ -2004,9 +2016,10 @@ static int tclass_notify(struct net *net, struct sk_buff *oskb,
 	if (!skb)
 		return -ENOBUFS;
 
-	if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event, extack) < 0) {
+	ret = tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, event, extack);
+	if (ret < 0) {
 		kfree_skb(skb);
-		return -EINVAL;
+		return ret;
 	}
 
 	return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
@@ -2017,17 +2030,19 @@ static int tclass_get_notify(struct net *net, struct sk_buff *oskb,
 			     struct nlmsghdr *n, struct Qdisc *q,
 			     unsigned long cl, struct netlink_ext_ack *extack)
 {
-	struct sk_buff *skb;
 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
+	struct sk_buff *skb;
+	int ret;
 
 	skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
 	if (!skb)
 		return -ENOBUFS;
 
-	if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0, RTM_NEWTCLASS,
-			   extack) < 0) {
+	ret = tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,
+			     RTM_NEWTCLASS, extack);
+	if (ret < 0) {
 		kfree_skb(skb);
-		return -EINVAL;
+		return ret;
 	}
 
 	return rtnetlink_send(skb, net, portid, RTNLGRP_TC,
@@ -2041,7 +2056,7 @@ static int tclass_del_notify(struct net *net,
 			     struct netlink_ext_ack *extack)
 {
 	u32 portid = oskb ? NETLINK_CB(oskb).portid : 0;
-	struct sk_buff *skb;
+	struct sk_buff *skb = NULL;
 	int err = 0;
 
 	if (!cops->delete)
@@ -2052,13 +2067,12 @@ static int tclass_del_notify(struct net *net,
 		if (!skb)
 			return -ENOBUFS;
 
-		if (tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,
-				   RTM_DELTCLASS, extack) < 0) {
+		err = tc_fill_tclass(skb, q, cl, portid, n->nlmsg_seq, 0,
+				     RTM_DELTCLASS, extack);
+		if (err < 0) {
 			kfree_skb(skb);
-			return -EINVAL;
+			return err;
 		}
-	} else {
-		skb = NULL;
 	}
 
 	err = cops->delete(q, cl, extack);
@@ -2334,7 +2348,7 @@ static int tc_dump_tclass_qdisc(struct Qdisc *q, struct sk_buff *skb,
 {
 	struct qdisc_dump_args arg;
 
-	if (tc_qdisc_dump_ignore(q, false) ||
+	if (tc_qdisc_dump_ignore(q, false, NULL) ||
 	    *t_p < s_t || !q->ops->cl_ops ||
 	    (tcm->tcm_parent &&
 	     TC_H_MAJ(tcm->tcm_parent) != q->handle)) {
diff --git a/net/sched/sch_cake.c b/net/sched/sch_cake.c
index 5862933be8d74..a3c185505afce 100644
--- a/net/sched/sch_cake.c
+++ b/net/sched/sch_cake.c
@@ -1600,10 +1600,10 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free)
 			   b->unresponsive_flow_count + 1);
 
 	len = qdisc_pkt_len(skb);
-	q->buffer_used      -= skb->truesize;
+	qstats_backlog_sub(sch, len);
+	q->buffer_used -= skb->truesize;
 	WRITE_ONCE(b->tin_backlog, b->tin_backlog - len);
 	WRITE_ONCE(b->backlogs[idx], b->backlogs[idx] - len);
-	sch->qstats.backlog -= len;
 
 	WRITE_ONCE(flow->dropped, flow->dropped + 1);
 	WRITE_ONCE(b->tin_dropped, b->tin_dropped + 1);
@@ -1612,7 +1612,7 @@ static unsigned int cake_drop(struct Qdisc *sch, struct sk_buff **to_free)
 		cake_advance_shaper(q, b, skb, now, true);
 
 	qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT);
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 
 	cake_heapify(q, 0);
 
@@ -1822,7 +1822,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 									  segs);
 			flow_queue_add(flow, segs);
 
-			sch->q.qlen++;
+			qdisc_qlen_inc(sch);
 			numsegs++;
 			slen += segs->len;
 			q->buffer_used += segs->truesize;
@@ -1830,7 +1830,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		}
 
 		/* stats */
-		sch->qstats.backlog += slen;
+		qstats_backlog_add(sch, slen);
 		q->avg_window_bytes += slen;
 		WRITE_ONCE(b->bytes, b->bytes + slen);
 		WRITE_ONCE(b->tin_backlog, b->tin_backlog + slen);
@@ -1851,7 +1851,7 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 
 		if (ack) {
 			WRITE_ONCE(b->ack_drops, b->ack_drops + 1);
-			sch->qstats.drops++;
+			qdisc_qstats_drop(sch);
 			ack_pkt_len = qdisc_pkt_len(ack);
 			WRITE_ONCE(b->bytes, b->bytes + ack_pkt_len);
 			q->buffer_used += skb->truesize - ack->truesize;
@@ -1861,13 +1861,13 @@ static s32 cake_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 			qdisc_tree_reduce_backlog(sch, 1, ack_pkt_len);
 			consume_skb(ack);
 		} else {
-			sch->q.qlen++;
+			qdisc_qlen_inc(sch);
 			q->buffer_used      += skb->truesize;
 		}
 
 		/* stats */
 		WRITE_ONCE(b->packets, b->packets + 1);
-		sch->qstats.backlog += len - ack_pkt_len;
+		qstats_backlog_add(sch, len - ack_pkt_len);
 		q->avg_window_bytes += len - ack_pkt_len;
 		WRITE_ONCE(b->bytes, b->bytes + len - ack_pkt_len);
 		WRITE_ONCE(b->tin_backlog, b->tin_backlog + len - ack_pkt_len);
@@ -1985,9 +1985,9 @@ static struct sk_buff *cake_dequeue_one(struct Qdisc *sch)
 		len = qdisc_pkt_len(skb);
 		WRITE_ONCE(b->backlogs[q->cur_flow], b->backlogs[q->cur_flow] - len);
 		WRITE_ONCE(b->tin_backlog, b->tin_backlog - len);
-		sch->qstats.backlog      -= len;
+		qstats_backlog_sub(sch, len);
 		q->buffer_used		 -= skb->truesize;
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 
 		if (q->overflow_timeout)
 			cake_heapify(q, b->overflow_idx[q->cur_flow]);
diff --git a/net/sched/sch_cbs.c b/net/sched/sch_cbs.c
index 0f953bd46b581..b952353ec2a53 100644
--- a/net/sched/sch_cbs.c
+++ b/net/sched/sch_cbs.c
@@ -96,8 +96,8 @@ static int cbs_child_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	if (err != NET_XMIT_SUCCESS)
 		return err;
 
-	sch->qstats.backlog += len;
-	sch->q.qlen++;
+	qstats_backlog_add(sch, len);
+	qdisc_qlen_inc(sch);
 
 	return NET_XMIT_SUCCESS;
 }
@@ -168,7 +168,7 @@ static struct sk_buff *cbs_child_dequeue(struct Qdisc *sch, struct Qdisc *child)
 
 	qdisc_qstats_backlog_dec(sch, skb);
 	qdisc_bstats_update(sch, skb);
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 
 	return skb;
 }
diff --git a/net/sched/sch_choke.c b/net/sched/sch_choke.c
index 2875bcdb18a41..73d3e673dc7b1 100644
--- a/net/sched/sch_choke.c
+++ b/net/sched/sch_choke.c
@@ -123,7 +123,7 @@ static void choke_drop_by_idx(struct Qdisc *sch, unsigned int idx,
 	if (idx == q->tail)
 		choke_zap_tail_holes(q);
 
-	--sch->q.qlen;
+	qdisc_qlen_dec(sch);
 	qdisc_qstats_backlog_dec(sch, skb);
 	qdisc_tree_reduce_backlog(sch, 1, qdisc_pkt_len(skb));
 	qdisc_drop(skb, sch, to_free);
@@ -271,7 +271,7 @@ static int choke_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	if (sch->q.qlen < q->limit) {
 		q->tab[q->tail] = skb;
 		q->tail = (q->tail + 1) & q->tab_mask;
-		++sch->q.qlen;
+		qdisc_qlen_inc(sch);
 		qdisc_qstats_backlog_inc(sch, skb);
 		return NET_XMIT_SUCCESS;
 	}
@@ -298,7 +298,7 @@ static struct sk_buff *choke_dequeue(struct Qdisc *sch)
 	skb = q->tab[q->head];
 	q->tab[q->head] = NULL;
 	choke_zap_head_holes(q);
-	--sch->q.qlen;
+	qdisc_qlen_dec(sch);
 	qdisc_qstats_backlog_dec(sch, skb);
 	qdisc_bstats_update(sch, skb);
 
@@ -396,7 +396,7 @@ static int choke_change(struct Qdisc *sch, struct nlattr *opt,
 				}
 				dropped += qdisc_pkt_len(skb);
 				qdisc_qstats_backlog_dec(sch, skb);
-				--sch->q.qlen;
+				qdisc_qlen_dec(sch);
 				rtnl_qdisc_drop(skb, sch);
 			}
 			qdisc_tree_reduce_backlog(sch, oqlen - sch->q.qlen, dropped);
diff --git a/net/sched/sch_codel.c b/net/sched/sch_codel.c
index 317aae0ec7bd6..91dd2e629af8f 100644
--- a/net/sched/sch_codel.c
+++ b/net/sched/sch_codel.c
@@ -42,7 +42,7 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
 	struct sk_buff *skb = __qdisc_dequeue_head(&sch->q);
 
 	if (skb) {
-		sch->qstats.backlog -= qdisc_pkt_len(skb);
+		qstats_backlog_sub(sch, qdisc_pkt_len(skb));
 		prefetch(&skb->end); /* we'll need skb_shinfo() */
 	}
 	return skb;
diff --git a/net/sched/sch_drr.c b/net/sched/sch_drr.c
index 01335a49e0914..020657f959b57 100644
--- a/net/sched/sch_drr.c
+++ b/net/sched/sch_drr.c
@@ -97,10 +97,8 @@ static int drr_change_class(struct Qdisc *sch, u32 classid, u32 parentid,
 			}
 		}
 
-		sch_tree_lock(sch);
 		if (tb[TCA_DRR_QUANTUM])
-			cl->quantum = quantum;
-		sch_tree_unlock(sch);
+			WRITE_ONCE(cl->quantum, quantum);
 
 		return 0;
 	}
@@ -251,7 +249,7 @@ static int drr_dump_class(struct Qdisc *sch, unsigned long arg,
 	nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
 	if (nest == NULL)
 		goto nla_put_failure;
-	if (nla_put_u32(skb, TCA_DRR_QUANTUM, cl->quantum))
+	if (nla_put_u32(skb, TCA_DRR_QUANTUM, READ_ONCE(cl->quantum)))
 		goto nla_put_failure;
 	return nla_nest_end(skb, nest);
 
@@ -270,7 +268,7 @@ static int drr_dump_class_stats(struct Qdisc *sch, unsigned long arg,
 
 	memset(&xstats, 0, sizeof(xstats));
 	if (qlen)
-		xstats.deficit = cl->deficit;
+		xstats.deficit = READ_ONCE(cl->deficit);
 
 	if (gnet_stats_copy_basic(d, NULL, &cl->bstats, true) < 0 ||
 	    gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 ||
@@ -362,11 +360,11 @@ static int drr_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 
 	if (!cl_is_active(cl)) {
 		list_add_tail(&cl->alist, &q->active);
-		cl->deficit = cl->quantum;
+		WRITE_ONCE(cl->deficit, READ_ONCE(cl->quantum));
 	}
 
-	sch->qstats.backlog += len;
-	sch->q.qlen++;
+	qstats_backlog_add(sch, len);
+	qdisc_qlen_inc(sch);
 	return err;
 }
 
@@ -389,7 +387,7 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch)
 
 		len = qdisc_pkt_len(skb);
 		if (len <= cl->deficit) {
-			cl->deficit -= len;
+			WRITE_ONCE(cl->deficit, cl->deficit - len);
 			skb = qdisc_dequeue_peeked(cl->qdisc);
 			if (unlikely(skb == NULL))
 				goto out;
@@ -399,11 +397,11 @@ static struct sk_buff *drr_dequeue(struct Qdisc *sch)
 			bstats_update(&cl->bstats, skb);
 			qdisc_bstats_update(sch, skb);
 			qdisc_qstats_backlog_dec(sch, skb);
-			sch->q.qlen--;
+			qdisc_qlen_dec(sch);
 			return skb;
 		}
 
-		cl->deficit += cl->quantum;
+		WRITE_ONCE(cl->deficit, cl->deficit + READ_ONCE(cl->quantum));
 		list_move_tail(&cl->alist, &q->active);
 	}
 out:
diff --git a/net/sched/sch_dualpi2.c b/net/sched/sch_dualpi2.c
index a22489c14458e..100ef7e9f9f28 100644
--- a/net/sched/sch_dualpi2.c
+++ b/net/sched/sch_dualpi2.c
@@ -190,7 +190,7 @@ static bool skb_apply_step(struct sk_buff *skb, struct dualpi2_sched_data *q)
 static bool dualpi2_mark(struct dualpi2_sched_data *q, struct sk_buff *skb)
 {
 	if (INET_ECN_set_ce(skb)) {
-		q->ecn_mark++;
+		WRITE_ONCE(q->ecn_mark, q->ecn_mark + 1);
 		return true;
 	}
 	return false;
@@ -198,7 +198,7 @@ static bool dualpi2_mark(struct dualpi2_sched_data *q, struct sk_buff *skb)
 
 static void dualpi2_reset_c_protection(struct dualpi2_sched_data *q)
 {
-	q->c_protection_credit = q->c_protection_init;
+	WRITE_ONCE(q->c_protection_credit, q->c_protection_init);
 }
 
 /* This computes the initial credit value and WRR weight for the L queue (wl)
@@ -403,28 +403,28 @@ static int dualpi2_enqueue_skb(struct sk_buff *skb, struct Qdisc *sch,
 
 	cb = dualpi2_skb_cb(skb);
 	cb->ts = ktime_get_ns();
-	q->memory_used += skb->truesize;
+	WRITE_ONCE(q->memory_used, q->memory_used + skb->truesize);
 	if (q->memory_used > q->max_memory_used)
-		q->max_memory_used = q->memory_used;
+		WRITE_ONCE(q->max_memory_used, q->memory_used);
 
 	if (qdisc_qlen(sch) > q->maxq)
-		q->maxq = qdisc_qlen(sch);
+		WRITE_ONCE(q->maxq, qdisc_qlen(sch));
 
 	if (skb_in_l_queue(skb)) {
 		/* Apply step thresh if skb is L4S && L-queue len >= min_qlen */
 		dualpi2_skb_cb(skb)->apply_step = skb_apply_step(skb, q);
 
 		/* Keep the overall qdisc stats consistent */
-		++sch->q.qlen;
+		qdisc_qlen_inc(sch);
 		qdisc_qstats_backlog_inc(sch, skb);
-		++q->packets_in_l;
+		WRITE_ONCE(q->packets_in_l, q->packets_in_l + 1);
 		if (!q->l_head_ts)
-			q->l_head_ts = cb->ts;
+			WRITE_ONCE(q->l_head_ts, cb->ts);
 		return qdisc_enqueue_tail(skb, q->l_queue);
 	}
-	++q->packets_in_c;
+	WRITE_ONCE(q->packets_in_c, q->packets_in_c + 1);
 	if (!q->c_head_ts)
-		q->c_head_ts = cb->ts;
+		WRITE_ONCE(q->c_head_ts, cb->ts);
 	return qdisc_enqueue_tail(skb, sch);
 }
 
@@ -530,18 +530,17 @@ static struct sk_buff *dequeue_packet(struct Qdisc *sch,
 		qdisc_qstats_backlog_dec(q->l_queue, skb);
 
 		/* Keep the global queue size consistent */
-		--sch->q.qlen;
-		q->memory_used -= skb->truesize;
+		qdisc_qlen_dec(sch);
 	} else if (c_len) {
 		skb = __qdisc_dequeue_head(&sch->q);
 		WRITE_ONCE(q->c_head_ts, head_enqueue_time(sch));
 		if (qdisc_qlen(q->l_queue))
 			*credit_change = ~((s32)q->c_protection_wl) + 1;
-		q->memory_used -= skb->truesize;
 	} else {
 		dualpi2_reset_c_protection(q);
 		return NULL;
 	}
+	WRITE_ONCE(q->memory_used, q->memory_used - skb->truesize);
 	*credit_change *= qdisc_pkt_len(skb);
 	qdisc_qstats_backlog_dec(sch, skb);
 	return skb;
@@ -564,7 +563,7 @@ static int do_step_aqm(struct dualpi2_sched_data *q, struct sk_buff *skb,
 		}
 
 		if (dualpi2_mark(q, skb))
-			++q->step_marks;
+			WRITE_ONCE(q->step_marks, q->step_marks + 1);
 	}
 	qdisc_bstats_update(q->l_queue, skb);
 	return 0;
@@ -600,7 +599,8 @@ static struct sk_buff *dualpi2_qdisc_dequeue(struct Qdisc *sch)
 			continue;
 		}
 
-		q->c_protection_credit += credit_change;
+		WRITE_ONCE(q->c_protection_credit,
+			   q->c_protection_credit + credit_change);
 		qdisc_bstats_update(sch, skb);
 		break;
 	}
@@ -876,7 +876,7 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt,
 				WARN_ON_ONCE(1);
 				break;
 			}
-			q->memory_used -= skb->truesize;
+			WRITE_ONCE(q->memory_used, q->memory_used - skb->truesize);
 			rtnl_qdisc_drop(skb, sch);
 		} else if (qdisc_qlen(q->l_queue)) {
 			skb = qdisc_dequeue_internal(q->l_queue, true);
@@ -888,9 +888,9 @@ static int dualpi2_change(struct Qdisc *sch, struct nlattr *opt,
 			 * l_queue on enqueue; qdisc_dequeue_internal()
 			 * handled l_queue, so we further account for sch.
 			 */
-			--sch->q.qlen;
+			qdisc_qlen_dec(sch);
 			qdisc_qstats_backlog_dec(sch, skb);
-			q->memory_used -= skb->truesize;
+			WRITE_ONCE(q->memory_used, q->memory_used - skb->truesize);
 			rtnl_qdisc_drop(skb, q->l_queue);
 			qdisc_qstats_drop(sch);
 		} else {
@@ -1046,15 +1046,15 @@ static int dualpi2_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
 	struct dualpi2_sched_data *q = qdisc_priv(sch);
 	struct tc_dualpi2_xstats st = {
 		.prob			= READ_ONCE(q->pi2_prob),
-		.packets_in_c		= q->packets_in_c,
-		.packets_in_l		= q->packets_in_l,
-		.maxq			= q->maxq,
-		.ecn_mark		= q->ecn_mark,
-		.credit			= q->c_protection_credit,
-		.step_marks		= q->step_marks,
-		.memory_used		= q->memory_used,
-		.max_memory_used	= q->max_memory_used,
-		.memory_limit		= q->memory_limit,
+		.packets_in_c		= READ_ONCE(q->packets_in_c),
+		.packets_in_l		= READ_ONCE(q->packets_in_l),
+		.maxq			= READ_ONCE(q->maxq),
+		.ecn_mark		= READ_ONCE(q->ecn_mark),
+		.credit			= READ_ONCE(q->c_protection_credit),
+		.step_marks		= READ_ONCE(q->step_marks),
+		.memory_used		= READ_ONCE(q->memory_used),
+		.max_memory_used	= READ_ONCE(q->max_memory_used),
+		.memory_limit		= READ_ONCE(q->memory_limit),
 	};
 	u64 qc, ql;
 
@@ -1074,16 +1074,16 @@ static void dualpi2_reset(struct Qdisc *sch)
 
 	qdisc_reset_queue(sch);
 	qdisc_reset_queue(q->l_queue);
-	q->c_head_ts = 0;
-	q->l_head_ts = 0;
-	q->pi2_prob = 0;
-	q->packets_in_c = 0;
-	q->packets_in_l = 0;
-	q->maxq = 0;
-	q->ecn_mark = 0;
-	q->step_marks = 0;
-	q->memory_used = 0;
-	q->max_memory_used = 0;
+	WRITE_ONCE(q->c_head_ts, 0);
+	WRITE_ONCE(q->l_head_ts, 0);
+	WRITE_ONCE(q->pi2_prob, 0);
+	WRITE_ONCE(q->packets_in_c, 0);
+	WRITE_ONCE(q->packets_in_l, 0);
+	WRITE_ONCE(q->maxq, 0);
+	WRITE_ONCE(q->ecn_mark, 0);
+	WRITE_ONCE(q->step_marks, 0);
+	WRITE_ONCE(q->memory_used, 0);
+	WRITE_ONCE(q->max_memory_used, 0);
 	dualpi2_reset_c_protection(q);
 }
 
diff --git a/net/sched/sch_etf.c b/net/sched/sch_etf.c
index c74d778c32a1e..ada87a81da6ac 100644
--- a/net/sched/sch_etf.c
+++ b/net/sched/sch_etf.c
@@ -189,7 +189,7 @@ static int etf_enqueue_timesortedlist(struct sk_buff *nskb, struct Qdisc *sch,
 	rb_insert_color_cached(&nskb->rbnode, &q->head, leftmost);
 
 	qdisc_qstats_backlog_inc(sch, nskb);
-	sch->q.qlen++;
+	qdisc_qlen_inc(sch);
 
 	/* Now we may need to re-arm the qdisc watchdog for the next packet. */
 	reset_watchdog(sch);
@@ -222,7 +222,7 @@ static void timesortedlist_drop(struct Qdisc *sch, struct sk_buff *skb,
 		qdisc_qstats_backlog_dec(sch, skb);
 		qdisc_drop(skb, sch, &to_free);
 		qdisc_qstats_overlimit(sch);
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 	}
 
 	kfree_skb_list(to_free);
@@ -247,7 +247,7 @@ static void timesortedlist_remove(struct Qdisc *sch, struct sk_buff *skb)
 
 	q->last = skb->tstamp;
 
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 }
 
 static struct sk_buff *etf_dequeue_timesortedlist(struct Qdisc *sch)
@@ -426,7 +426,7 @@ static void timesortedlist_clear(struct Qdisc *sch)
 
 		rb_erase_cached(&skb->rbnode, &q->head);
 		rtnl_kfree_skbs(skb, skb);
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 	}
 }
 
diff --git a/net/sched/sch_ets.c b/net/sched/sch_ets.c
index a4b07b661b775..1cc559634ed27 100644
--- a/net/sched/sch_ets.c
+++ b/net/sched/sch_ets.c
@@ -448,8 +448,8 @@ static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		cl->deficit = cl->quantum;
 	}
 
-	sch->qstats.backlog += len;
-	sch->q.qlen++;
+	qstats_backlog_add(sch, len);
+	qdisc_qlen_inc(sch);
 	return err;
 }
 
@@ -458,7 +458,7 @@ ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb)
 {
 	qdisc_bstats_update(sch, skb);
 	qdisc_qstats_backlog_dec(sch, skb);
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 	return skb;
 }
 
diff --git a/net/sched/sch_fq.c b/net/sched/sch_fq.c
index f2edcf872981f..796cb8046a902 100644
--- a/net/sched/sch_fq.c
+++ b/net/sched/sch_fq.c
@@ -497,7 +497,7 @@ static void fq_dequeue_skb(struct Qdisc *sch, struct fq_flow *flow,
 	fq_erase_head(sch, flow, skb);
 	skb_mark_not_on_list(skb);
 	qdisc_qstats_backlog_dec(sch, skb);
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 	qdisc_bstats_update(sch, skb);
 }
 
@@ -597,7 +597,7 @@ static int fq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	flow_queue_add(f, skb);
 
 	qdisc_qstats_backlog_inc(sch, skb);
-	sch->q.qlen++;
+	qdisc_qlen_inc(sch);
 
 	return NET_XMIT_SUCCESS;
 }
@@ -801,8 +801,8 @@ static void fq_reset(struct Qdisc *sch)
 	struct fq_flow *f;
 	unsigned int idx;
 
-	sch->q.qlen = 0;
-	sch->qstats.backlog = 0;
+	WRITE_ONCE(sch->q.qlen, 0);
+	WRITE_ONCE(sch->qstats.backlog, 0);
 
 	fq_flow_purge(&q->internal);
 
diff --git a/net/sched/sch_fq_codel.c b/net/sched/sch_fq_codel.c
index 24db54684e8a5..71107dc52be79 100644
--- a/net/sched/sch_fq_codel.c
+++ b/net/sched/sch_fq_codel.c
@@ -176,9 +176,9 @@ static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets,
 	WRITE_ONCE(flow->cvars.count, flow->cvars.count + i);
 	WRITE_ONCE(q->backlogs[idx], q->backlogs[idx] - len);
 	q->memory_usage -= mem;
-	sch->qstats.drops += i;
-	sch->qstats.backlog -= len;
-	sch->q.qlen -= i;
+	__qdisc_qstats_drop(sch, i);
+	qstats_backlog_sub(sch, len);
+	WRITE_ONCE(sch->q.qlen, sch->q.qlen - i);
 	return idx;
 }
 
@@ -212,10 +212,11 @@ static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		q->new_flow_count++;
 		WRITE_ONCE(flow->deficit, q->quantum);
 	}
-	get_codel_cb(skb)->mem_usage = skb->truesize;
+	get_codel_cb(skb)->mem_usage = is_skb_wmem(skb) ? 0 : skb->truesize;
 	q->memory_usage += get_codel_cb(skb)->mem_usage;
 	memory_limited = q->memory_usage > q->memory_limit;
-	if (++sch->q.qlen <= sch->limit && !memory_limited)
+	qdisc_qlen_inc(sch);
+	if (sch->q.qlen <= sch->limit && !memory_limited)
 		return NET_XMIT_SUCCESS;
 
 	prev_backlog = sch->qstats.backlog;
@@ -266,8 +267,8 @@ static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx)
 		WRITE_ONCE(q->backlogs[flow - q->flows],
 			   q->backlogs[flow - q->flows] - qdisc_pkt_len(skb));
 		q->memory_usage -= get_codel_cb(skb)->mem_usage;
-		sch->q.qlen--;
-		sch->qstats.backlog -= qdisc_pkt_len(skb);
+		qdisc_qlen_dec(sch);
+		qdisc_qstats_backlog_dec(sch, skb);
 	}
 	return skb;
 }
diff --git a/net/sched/sch_fq_pie.c b/net/sched/sch_fq_pie.c
index 7becbf5362b31..72f48fa4010be 100644
--- a/net/sched/sch_fq_pie.c
+++ b/net/sched/sch_fq_pie.c
@@ -184,8 +184,8 @@ static int fq_pie_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		pkt_len = qdisc_pkt_len(skb);
 		q->stats.packets_in++;
 		q->memory_usage += skb->truesize;
-		sch->qstats.backlog += pkt_len;
-		sch->q.qlen++;
+		qstats_backlog_add(sch, pkt_len);
+		qdisc_qlen_inc(sch);
 		flow_queue_add(sel_flow, skb);
 		if (list_empty(&sel_flow->flowchain)) {
 			list_add_tail(&sel_flow->flowchain, &q->new_flows);
@@ -262,8 +262,8 @@ static struct sk_buff *fq_pie_qdisc_dequeue(struct Qdisc *sch)
 	if (flow->head) {
 		skb = dequeue_head(flow);
 		pkt_len = qdisc_pkt_len(skb);
-		sch->qstats.backlog -= pkt_len;
-		sch->q.qlen--;
+		qstats_backlog_sub(sch, pkt_len);
+		qdisc_qlen_dec(sch);
 		qdisc_bstats_update(sch, skb);
 	}
 
diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c
index a93321db8fd75..237ee1cd01367 100644
--- a/net/sched/sch_generic.c
+++ b/net/sched/sch_generic.c
@@ -118,7 +118,7 @@ static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
 				qdisc_qstats_cpu_qlen_dec(q);
 			} else {
 				qdisc_qstats_backlog_dec(q, skb);
-				q->q.qlen--;
+				qdisc_qlen_dec(q);
 			}
 		} else {
 			skb = SKB_XOFF_MAGIC;
@@ -159,7 +159,7 @@ static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
 		qdisc_qstats_cpu_qlen_inc(q);
 	} else {
 		qdisc_qstats_backlog_inc(q, skb);
-		q->q.qlen++;
+		qdisc_qlen_inc(q);
 	}
 
 	if (lock)
@@ -188,7 +188,7 @@ static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
 		} else {
 			q->qstats.requeues++;
 			qdisc_qstats_backlog_inc(q, skb);
-			q->q.qlen++;
+			qdisc_qlen_inc(q);
 		}
 
 		skb = next;
@@ -294,7 +294,7 @@ static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
 				qdisc_qstats_cpu_qlen_dec(q);
 			} else {
 				qdisc_qstats_backlog_dec(q, skb);
-				q->q.qlen--;
+				qdisc_qlen_dec(q);
 			}
 		} else {
 			skb = NULL;
@@ -609,6 +609,9 @@ static void netdev_watchdog_down(struct net_device *dev)
  */
 void netif_carrier_on(struct net_device *dev)
 {
+	if (READ_ONCE(dev->proto_down))
+		return;
+
 	if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
 		if (dev->reg_state == NETREG_UNINITIALIZED)
 			return;
@@ -1059,8 +1062,8 @@ void qdisc_reset(struct Qdisc *qdisc)
 	__skb_queue_purge(&qdisc->gso_skb);
 	__skb_queue_purge(&qdisc->skb_bad_txq);
 
-	qdisc->q.qlen = 0;
-	qdisc->qstats.backlog = 0;
+	WRITE_ONCE(qdisc->q.qlen, 0);
+	WRITE_ONCE(qdisc->qstats.backlog, 0);
 }
 EXPORT_SYMBOL(qdisc_reset);
 
diff --git a/net/sched/sch_gred.c b/net/sched/sch_gred.c
index 36d0cafac2063..fcc1a4c036362 100644
--- a/net/sched/sch_gred.c
+++ b/net/sched/sch_gred.c
@@ -388,8 +388,8 @@ static int gred_offload_dump_stats(struct Qdisc *sch)
 		bytes += u64_stats_read(&hw_stats->stats.bstats[i].bytes);
 		packets += u64_stats_read(&hw_stats->stats.bstats[i].packets);
 		sch->qstats.qlen += hw_stats->stats.qstats[i].qlen;
-		sch->qstats.backlog += hw_stats->stats.qstats[i].backlog;
-		sch->qstats.drops += hw_stats->stats.qstats[i].drops;
+		qstats_backlog_add(sch, hw_stats->stats.qstats[i].backlog);
+		__qdisc_qstats_drop(sch, hw_stats->stats.qstats[i].drops);
 		sch->qstats.requeues += hw_stats->stats.qstats[i].requeues;
 		sch->qstats.overlimits += hw_stats->stats.qstats[i].overlimits;
 	}
diff --git a/net/sched/sch_hfsc.c b/net/sched/sch_hfsc.c
index 83b2ca2e37fc8..c06d2761a1fb9 100644
--- a/net/sched/sch_hfsc.c
+++ b/net/sched/sch_hfsc.c
@@ -715,7 +715,7 @@ init_vf(struct hfsc_class *cl, unsigned int len)
 			rtsc_min(&cl->cl_virtual, &cl->cl_fsc, cl->cl_vt, cl->cl_total);
 			cl->cl_vtadj = 0;
 
-			cl->cl_vtperiod++;  /* increment vt period */
+			WRITE_ONCE(cl->cl_vtperiod, cl->cl_vtperiod + 1);  /* increment vt period */
 			cl->cl_parentperiod = cl->cl_parent->cl_vtperiod;
 			if (cl->cl_parent->cl_nactive == 0)
 				cl->cl_parentperiod++;
@@ -757,7 +757,7 @@ update_vf(struct hfsc_class *cl, unsigned int len, u64 cur_time)
 		go_passive = 1;
 
 	for (; cl->cl_parent != NULL; cl = cl->cl_parent) {
-		cl->cl_total += len;
+		WRITE_ONCE(cl->cl_total, cl->cl_total + len);
 
 		if (!(cl->cl_flags & HFSC_FSC) || cl->cl_nactive == 0)
 			continue;
@@ -847,7 +847,7 @@ hfsc_adjust_levels(struct hfsc_class *cl)
 			if (p->level >= level)
 				level = p->level + 1;
 		}
-		cl->level = level;
+		WRITE_ONCE(cl->level, level);
 	} while ((cl = cl->cl_parent) != NULL);
 }
 
@@ -1338,10 +1338,10 @@ hfsc_dump_class_stats(struct Qdisc *sch, unsigned long arg,
 	__u32 qlen;
 
 	qdisc_qstats_qlen_backlog(cl->qdisc, &qlen, &cl->qstats.backlog);
-	xstats.level   = cl->level;
-	xstats.period  = cl->cl_vtperiod;
-	xstats.work    = cl->cl_total;
-	xstats.rtwork  = cl->cl_cumul;
+	xstats.level   = READ_ONCE(cl->level);
+	xstats.period  = READ_ONCE(cl->cl_vtperiod);
+	xstats.work    = READ_ONCE(cl->cl_total);
+	xstats.rtwork  = READ_ONCE(cl->cl_cumul);
 
 	if (gnet_stats_copy_basic(d, NULL, &cl->bstats, true) < 0 ||
 	    gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 ||
@@ -1452,15 +1452,15 @@ hfsc_change_qdisc(struct Qdisc *sch, struct nlattr *opt,
 static void
 hfsc_reset_class(struct hfsc_class *cl)
 {
-	cl->cl_total        = 0;
-	cl->cl_cumul        = 0;
+	WRITE_ONCE(cl->cl_total, 0);
+	WRITE_ONCE(cl->cl_cumul, 0);
 	cl->cl_d            = 0;
 	cl->cl_e            = 0;
 	cl->cl_vt           = 0;
 	cl->cl_vtadj        = 0;
 	cl->cl_cvtmin       = 0;
 	cl->cl_cvtoff       = 0;
-	cl->cl_vtperiod     = 0;
+	WRITE_ONCE(cl->cl_vtperiod, 0);
 	cl->cl_parentperiod = 0;
 	cl->cl_f            = 0;
 	cl->cl_myf          = 0;
@@ -1560,8 +1560,8 @@ hfsc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
 		return err;
 	}
 
-	sch->qstats.backlog += len;
-	sch->q.qlen++;
+	qstats_backlog_add(sch, len);
+	qdisc_qlen_inc(sch);
 
 	if (first && !cl_in_el_or_vttree(cl)) {
 		if (cl->cl_flags & HFSC_RSC)
@@ -1626,7 +1626,7 @@ hfsc_dequeue(struct Qdisc *sch)
 	bstats_update(&cl->bstats, skb);
 	update_vf(cl, qdisc_pkt_len(skb), cur_time);
 	if (realtime)
-		cl->cl_cumul += qdisc_pkt_len(skb);
+		WRITE_ONCE(cl->cl_cumul, cl->cl_cumul + qdisc_pkt_len(skb));
 
 	if (cl->cl_flags & HFSC_RSC) {
 		if (cl->qdisc->q.qlen != 0) {
@@ -1650,7 +1650,7 @@ hfsc_dequeue(struct Qdisc *sch)
 
 	qdisc_bstats_update(sch, skb);
 	qdisc_qstats_backlog_dec(sch, skb);
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 
 	return skb;
 }
diff --git a/net/sched/sch_hhf.c b/net/sched/sch_hhf.c
index 96021f52d835b..1e25b75daae2e 100644
--- a/net/sched/sch_hhf.c
+++ b/net/sched/sch_hhf.c
@@ -360,7 +360,7 @@ static unsigned int hhf_drop(struct Qdisc *sch, struct sk_buff **to_free)
 	if (bucket->head) {
 		struct sk_buff *skb = dequeue_head(bucket);
 
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 		qdisc_qstats_backlog_dec(sch, skb);
 		qdisc_drop(skb, sch, to_free);
 	}
@@ -400,7 +400,8 @@ static int hhf_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		}
 		bucket->deficit = weight * q->quantum;
 	}
-	if (++sch->q.qlen <= sch->limit)
+	qdisc_qlen_inc(sch);
+	if (sch->q.qlen <= sch->limit)
 		return NET_XMIT_SUCCESS;
 
 	prev_backlog = sch->qstats.backlog;
@@ -443,7 +444,7 @@ static struct sk_buff *hhf_dequeue(struct Qdisc *sch)
 
 	if (bucket->head) {
 		skb = dequeue_head(bucket);
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 		qdisc_qstats_backlog_dec(sch, skb);
 	}
 
diff --git a/net/sched/sch_htb.c b/net/sched/sch_htb.c
index eb12381795ce1..908b9ba9ba2ef 100644
--- a/net/sched/sch_htb.c
+++ b/net/sched/sch_htb.c
@@ -114,7 +114,8 @@ struct htb_class {
 	 */
 	struct gnet_stats_basic_sync bstats;
 	struct gnet_stats_basic_sync bstats_bias;
-	struct tc_htb_xstats	xstats;	/* our special stats */
+	u32			xstats_lends;
+	u32			xstats_borrows;
 
 	/* token bucket parameters */
 	s64			tokens, ctokens;/* current number of tokens */
@@ -567,8 +568,8 @@ htb_change_class_mode(struct htb_sched *q, struct htb_class *cl, s64 *diff)
 		return;
 
 	if (new_mode == HTB_CANT_SEND) {
-		cl->overlimits++;
-		q->overlimits++;
+		WRITE_ONCE(cl->overlimits, cl->overlimits + 1);
+		WRITE_ONCE(q->overlimits, q->overlimits + 1);
 	}
 
 	if (cl->prio_activity) {	/* not necessary: speed optimization */
@@ -628,7 +629,7 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		/* enqueue to helper queue */
 		if (q->direct_queue.qlen < q->direct_qlen) {
 			__qdisc_enqueue_tail(skb, &q->direct_queue);
-			q->direct_pkts++;
+			WRITE_ONCE(q->direct_pkts, q->direct_pkts + 1);
 		} else {
 			return qdisc_drop(skb, sch, to_free);
 		}
@@ -643,15 +644,15 @@ static int htb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 					to_free)) != NET_XMIT_SUCCESS) {
 		if (net_xmit_drop_count(ret)) {
 			qdisc_qstats_drop(sch);
-			cl->drops++;
+			WRITE_ONCE(cl->drops, cl->drops + 1);
 		}
 		return ret;
 	} else {
 		htb_activate(q, cl);
 	}
 
-	sch->qstats.backlog += len;
-	sch->q.qlen++;
+	qstats_backlog_add(sch, len);
+	qdisc_qlen_inc(sch);
 	return NET_XMIT_SUCCESS;
 }
 
@@ -665,7 +666,7 @@ static inline void htb_accnt_tokens(struct htb_class *cl, int bytes, s64 diff)
 	if (toks <= -cl->mbuffer)
 		toks = 1 - cl->mbuffer;
 
-	cl->tokens = toks;
+	WRITE_ONCE(cl->tokens, toks);
 }
 
 static inline void htb_accnt_ctokens(struct htb_class *cl, int bytes, s64 diff)
@@ -678,7 +679,7 @@ static inline void htb_accnt_ctokens(struct htb_class *cl, int bytes, s64 diff)
 	if (toks <= -cl->mbuffer)
 		toks = 1 - cl->mbuffer;
 
-	cl->ctokens = toks;
+	WRITE_ONCE(cl->ctokens, toks);
 }
 
 /**
@@ -707,11 +708,12 @@ static void htb_charge_class(struct htb_sched *q, struct htb_class *cl,
 		diff = min_t(s64, q->now - cl->t_c, cl->mbuffer);
 		if (cl->level >= level) {
 			if (cl->level == level)
-				cl->xstats.lends++;
+				WRITE_ONCE(cl->xstats_lends, cl->xstats_lends + 1);
 			htb_accnt_tokens(cl, bytes, diff);
 		} else {
-			cl->xstats.borrows++;
-			cl->tokens += diff;	/* we moved t_c; update tokens */
+			WRITE_ONCE(cl->xstats_borrows, cl->xstats_borrows + 1);
+			/* we moved t_c; update tokens */
+			WRITE_ONCE(cl->tokens, cl->tokens + diff);
 		}
 		htb_accnt_ctokens(cl, bytes, diff);
 		cl->t_c = q->now;
@@ -951,7 +953,7 @@ static struct sk_buff *htb_dequeue(struct Qdisc *sch)
 ok:
 		qdisc_bstats_update(sch, skb);
 		qdisc_qstats_backlog_dec(sch, skb);
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 		return skb;
 	}
 
@@ -1147,6 +1149,7 @@ static int htb_init(struct Qdisc *sch, struct nlattr *opt,
 	 * parts (especially calling ndo_setup_tc) on errors.
 	 */
 	q->offload = true;
+	sch->flags |= TCQ_F_OFFLOADED;
 
 	return 0;
 }
@@ -1207,17 +1210,12 @@ static int htb_dump(struct Qdisc *sch, struct sk_buff *skb)
 	struct nlattr *nest;
 	struct tc_htb_glob gopt;
 
-	if (q->offload)
-		sch->flags |= TCQ_F_OFFLOADED;
-	else
-		sch->flags &= ~TCQ_F_OFFLOADED;
-
-	sch->qstats.overlimits = q->overlimits;
+	sch->qstats.overlimits = READ_ONCE(q->overlimits);
 	/* Its safe to not acquire qdisc lock. As we hold RTNL,
 	 * no change can happen on the qdisc parameters.
 	 */
 
-	gopt.direct_pkts = q->direct_pkts;
+	gopt.direct_pkts = READ_ONCE(q->direct_pkts);
 	gopt.version = HTB_VER;
 	gopt.rate2quantum = q->rate2quantum;
 	gopt.defcls = q->defcls;
@@ -1295,8 +1293,6 @@ static void htb_offload_aggregate_stats(struct htb_sched *q,
 	struct htb_class *c;
 	unsigned int i;
 
-	gnet_stats_basic_sync_init(&cl->bstats);
-
 	for (i = 0; i < q->clhash.hashsize; i++) {
 		hlist_for_each_entry(c, &q->clhash.hash[i], common.hnode) {
 			struct htb_class *p = c;
@@ -1315,7 +1311,7 @@ static void htb_offload_aggregate_stats(struct htb_sched *q,
 			}
 		}
 	}
-	_bstats_update(&cl->bstats, bytes, packets);
+	_bstats_set(&cl->bstats, bytes, packets);
 }
 
 static int
@@ -1323,32 +1319,40 @@ htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)
 {
 	struct htb_class *cl = (struct htb_class *)arg;
 	struct htb_sched *q = qdisc_priv(sch);
+	struct tc_htb_xstats xstats = {
+		.lends = READ_ONCE(cl->xstats_lends),
+		.borrows = READ_ONCE(cl->xstats_borrows),
+	};
 	struct gnet_stats_queue qs = {
-		.drops = cl->drops,
-		.overlimits = cl->overlimits,
+		.drops = READ_ONCE(cl->drops),
+		.overlimits = READ_ONCE(cl->overlimits),
 	};
 	__u32 qlen = 0;
 
 	if (!cl->level && cl->leaf.q)
 		qdisc_qstats_qlen_backlog(cl->leaf.q, &qlen, &qs.backlog);
 
-	cl->xstats.tokens = clamp_t(s64, PSCHED_NS2TICKS(cl->tokens),
-				    INT_MIN, INT_MAX);
-	cl->xstats.ctokens = clamp_t(s64, PSCHED_NS2TICKS(cl->ctokens),
-				     INT_MIN, INT_MAX);
+	xstats.tokens = clamp_t(s64, PSCHED_NS2TICKS(READ_ONCE(cl->tokens)),
+				INT_MIN, INT_MAX);
+	xstats.ctokens = clamp_t(s64, PSCHED_NS2TICKS(READ_ONCE(cl->ctokens)),
+				 INT_MIN, INT_MAX);
 
 	if (q->offload) {
+		spin_lock_bh(qdisc_lock(sch));
 		if (!cl->level) {
-			if (cl->leaf.q)
-				cl->bstats = cl->leaf.q->bstats;
-			else
-				gnet_stats_basic_sync_init(&cl->bstats);
-			_bstats_update(&cl->bstats,
-				       u64_stats_read(&cl->bstats_bias.bytes),
-				       u64_stats_read(&cl->bstats_bias.packets));
+			u64 bytes = 0, packets = 0;
+
+			if (cl->leaf.q) {
+				bytes = u64_stats_read(&cl->leaf.q->bstats.bytes);
+				packets = u64_stats_read(&cl->leaf.q->bstats.packets);
+			}
+			bytes += u64_stats_read(&cl->bstats_bias.bytes);
+			packets += u64_stats_read(&cl->bstats_bias.packets);
+			_bstats_set(&cl->bstats, bytes, packets);
 		} else {
 			htb_offload_aggregate_stats(q, cl);
 		}
+		spin_unlock_bh(qdisc_lock(sch));
 	}
 
 	if (gnet_stats_copy_basic(d, NULL, &cl->bstats, true) < 0 ||
@@ -1356,7 +1360,7 @@ htb_dump_class_stats(struct Qdisc *sch, unsigned long arg, struct gnet_dump *d)
 	    gnet_stats_copy_queue(d, NULL, &qs, qlen) < 0)
 		return -1;
 
-	return gnet_stats_copy_app(d, &cl->xstats, sizeof(cl->xstats));
+	return gnet_stats_copy_app(d, &xstats, sizeof(xstats));
 }
 
 static struct netdev_queue *
@@ -1516,8 +1520,8 @@ static void htb_parent_to_leaf(struct Qdisc *sch, struct htb_class *cl,
 	parent->level = 0;
 	memset(&parent->inner, 0, sizeof(parent->inner));
 	parent->leaf.q = new_q ? new_q : &noop_qdisc;
-	parent->tokens = parent->buffer;
-	parent->ctokens = parent->cbuffer;
+	WRITE_ONCE(parent->tokens, parent->buffer);
+	WRITE_ONCE(parent->ctokens, parent->cbuffer);
 	parent->t_c = ktime_get_ns();
 	parent->cmode = HTB_CAN_SEND;
 	if (q->offload)
diff --git a/net/sched/sch_mq.c b/net/sched/sch_mq.c
index a0133a7b9d3b0..5a6b14557d85f 100644
--- a/net/sched/sch_mq.c
+++ b/net/sched/sch_mq.c
@@ -143,30 +143,43 @@ EXPORT_SYMBOL_NS_GPL(mq_attach, "NET_SCHED_INTERNAL");
 void mq_dump_common(struct Qdisc *sch, struct sk_buff *skb)
 {
 	struct net_device *dev = qdisc_dev(sch);
-	struct Qdisc *qdisc;
+	struct gnet_stats_queue qstats = { 0 };
+	struct gnet_stats_basic_sync bstats;
+	const struct Qdisc *qdisc;
+	unsigned int qlen = 0;
 	unsigned int ntx;
 
-	sch->q.qlen = 0;
-	gnet_stats_basic_sync_init(&sch->bstats);
-	memset(&sch->qstats, 0, sizeof(sch->qstats));
+	gnet_stats_basic_sync_init(&bstats);
 
 	/* MQ supports lockless qdiscs. However, statistics accounting needs
 	 * to account for all, none, or a mix of locked and unlocked child
 	 * qdiscs. Percpu stats are added to counters in-band and locking
 	 * qdisc totals are added at end.
 	 */
+	rcu_read_lock();
 	for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
-		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, ntx)->qdisc_sleeping);
-		spin_lock_bh(qdisc_lock(qdisc));
+		qdisc = rcu_dereference(netdev_get_tx_queue(dev, ntx)->qdisc_sleeping);
 
-		gnet_stats_add_basic(&sch->bstats, qdisc->cpu_bstats,
-				     &qdisc->bstats, false);
-		gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats,
+		gnet_stats_add_basic(&bstats, qdisc->cpu_bstats,
+				     &qdisc->bstats, true);
+		gnet_stats_add_queue(&qstats, qdisc->cpu_qstats,
 				     &qdisc->qstats);
-		sch->q.qlen += qdisc_qlen(qdisc);
-
-		spin_unlock_bh(qdisc_lock(qdisc));
+		qlen += qdisc_qlen_lockless(qdisc);
 	}
+	rcu_read_unlock();
+
+	spin_lock_bh(qdisc_lock(sch));
+	_bstats_set(&sch->bstats, u64_stats_read(&bstats.bytes),
+		    u64_stats_read(&bstats.packets));
+	spin_unlock_bh(qdisc_lock(sch));
+
+	WRITE_ONCE(sch->qstats.qlen, qstats.qlen);
+	WRITE_ONCE(sch->qstats.backlog, qstats.backlog);
+	WRITE_ONCE(sch->qstats.drops, qstats.drops);
+	WRITE_ONCE(sch->qstats.requeues, qstats.requeues);
+	WRITE_ONCE(sch->qstats.overlimits, qstats.overlimits);
+
+	WRITE_ONCE(sch->q.qlen, qlen);
 }
 EXPORT_SYMBOL_NS_GPL(mq_dump_common, "NET_SCHED_INTERNAL");
 
diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c
index 002add5ce9e0a..ae991fc25b43f 100644
--- a/net/sched/sch_mqprio.c
+++ b/net/sched/sch_mqprio.c
@@ -551,34 +551,47 @@ static int mqprio_dump_tc_entries(struct mqprio_sched *priv,
 
 static int mqprio_dump(struct Qdisc *sch, struct sk_buff *skb)
 {
-	struct net_device *dev = qdisc_dev(sch);
-	struct mqprio_sched *priv = qdisc_priv(sch);
 	struct nlattr *nla = (struct nlattr *)skb_tail_pointer(skb);
+	struct mqprio_sched *priv = qdisc_priv(sch);
+	struct net_device *dev = qdisc_dev(sch);
+	struct gnet_stats_queue qstats = { 0 };
+	struct gnet_stats_basic_sync bstats;
 	struct tc_mqprio_qopt opt = { 0 };
-	struct Qdisc *qdisc;
+	const struct Qdisc *qdisc;
+	unsigned int qlen = 0;
 	unsigned int ntx;
 
-	sch->q.qlen = 0;
-	gnet_stats_basic_sync_init(&sch->bstats);
-	memset(&sch->qstats, 0, sizeof(sch->qstats));
+	gnet_stats_basic_sync_init(&bstats);
 
 	/* MQ supports lockless qdiscs. However, statistics accounting needs
 	 * to account for all, none, or a mix of locked and unlocked child
 	 * qdiscs. Percpu stats are added to counters in-band and locking
 	 * qdisc totals are added at end.
 	 */
+	rcu_read_lock();
 	for (ntx = 0; ntx < dev->num_tx_queues; ntx++) {
-		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, ntx)->qdisc_sleeping);
-		spin_lock_bh(qdisc_lock(qdisc));
+		qdisc = rcu_dereference(netdev_get_tx_queue(dev, ntx)->qdisc_sleeping);
 
-		gnet_stats_add_basic(&sch->bstats, qdisc->cpu_bstats,
-				     &qdisc->bstats, false);
-		gnet_stats_add_queue(&sch->qstats, qdisc->cpu_qstats,
+		gnet_stats_add_basic(&bstats, qdisc->cpu_bstats,
+				     &qdisc->bstats, true);
+		gnet_stats_add_queue(&qstats, qdisc->cpu_qstats,
 				     &qdisc->qstats);
-		sch->q.qlen += qdisc_qlen(qdisc);
-
-		spin_unlock_bh(qdisc_lock(qdisc));
+		qlen += qdisc_qlen_lockless(qdisc);
 	}
+	rcu_read_unlock();
+
+	spin_lock_bh(qdisc_lock(sch));
+	_bstats_set(&sch->bstats, u64_stats_read(&bstats.bytes),
+		    u64_stats_read(&bstats.packets));
+	spin_unlock_bh(qdisc_lock(sch));
+
+	WRITE_ONCE(sch->qstats.qlen, qstats.qlen);
+	WRITE_ONCE(sch->qstats.backlog, qstats.backlog);
+	WRITE_ONCE(sch->qstats.drops, qstats.drops);
+	WRITE_ONCE(sch->qstats.requeues, qstats.requeues);
+	WRITE_ONCE(sch->qstats.overlimits, qstats.overlimits);
+
+	WRITE_ONCE(sch->q.qlen, qlen);
 
 	mqprio_qopt_reconstruct(dev, &opt);
 	opt.hw = priv->hw_offload;
@@ -659,58 +672,48 @@ static int mqprio_dump_class(struct Qdisc *sch, unsigned long cl,
 
 static int mqprio_dump_class_stats(struct Qdisc *sch, unsigned long cl,
 				   struct gnet_dump *d)
-	__releases(d->lock)
-	__acquires(d->lock)
 {
+	const struct Qdisc *qdisc;
+	int res = 0;
+
+	rcu_read_lock();
 	if (cl >= TC_H_MIN_PRIORITY) {
-		int i;
-		__u32 qlen;
-		struct gnet_stats_queue qstats = {0};
-		struct gnet_stats_basic_sync bstats;
 		struct net_device *dev = qdisc_dev(sch);
 		struct netdev_tc_txq tc = dev->tc_to_txq[cl & TC_BITMASK];
+		struct gnet_stats_queue qstats = {0};
+		struct gnet_stats_basic_sync bstats;
+		u32 qlen = 0;
+		int i;
 
 		gnet_stats_basic_sync_init(&bstats);
-		/* Drop lock here it will be reclaimed before touching
-		 * statistics this is required because the d->lock we
-		 * hold here is the look on dev_queue->qdisc_sleeping
-		 * also acquired below.
-		 */
-		if (d->lock)
-			spin_unlock_bh(d->lock);
 
 		for (i = tc.offset; i < tc.offset + tc.count; i++) {
 			struct netdev_queue *q = netdev_get_tx_queue(dev, i);
-			struct Qdisc *qdisc = rtnl_dereference(q->qdisc);
-
-			spin_lock_bh(qdisc_lock(qdisc));
 
+			qdisc = rcu_dereference(q->qdisc);
 			gnet_stats_add_basic(&bstats, qdisc->cpu_bstats,
-					     &qdisc->bstats, false);
+					     &qdisc->bstats, true);
 			gnet_stats_add_queue(&qstats, qdisc->cpu_qstats,
 					     &qdisc->qstats);
-			sch->q.qlen += qdisc_qlen(qdisc);
-
-			spin_unlock_bh(qdisc_lock(qdisc));
+			qlen += qdisc_qlen_lockless(qdisc);
 		}
-		qlen = qdisc_qlen(sch) + qstats.qlen;
 
-		/* Reclaim root sleeping lock before completing stats */
-		if (d->lock)
-			spin_lock_bh(d->lock);
+		qlen = qlen + qstats.qlen;
+
 		if (gnet_stats_copy_basic(d, NULL, &bstats, false) < 0 ||
 		    gnet_stats_copy_queue(d, NULL, &qstats, qlen) < 0)
-			return -1;
+			res = -1;
 	} else {
 		struct netdev_queue *dev_queue = mqprio_queue_get(sch, cl);
 
-		sch = rtnl_dereference(dev_queue->qdisc_sleeping);
-		if (gnet_stats_copy_basic(d, sch->cpu_bstats,
-					  &sch->bstats, true) < 0 ||
-		    qdisc_qstats_copy(d, sch) < 0)
-			return -1;
+		qdisc = rcu_dereference(dev_queue->qdisc_sleeping);
+		if (gnet_stats_copy_basic(d, qdisc->cpu_bstats,
+					  &qdisc->bstats, true) < 0 ||
+		    qdisc_qstats_copy(d, qdisc) < 0)
+			res = -1;
 	}
-	return 0;
+	rcu_read_unlock();
+	return res;
 }
 
 static void mqprio_walk(struct Qdisc *sch, struct qdisc_walker *arg)
diff --git a/net/sched/sch_multiq.c b/net/sched/sch_multiq.c
index 9f822fee113df..4e465d11e3d75 100644
--- a/net/sched/sch_multiq.c
+++ b/net/sched/sch_multiq.c
@@ -76,7 +76,7 @@ multiq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 
 	ret = qdisc_enqueue(skb, qdisc, to_free);
 	if (ret == NET_XMIT_SUCCESS) {
-		sch->q.qlen++;
+		qdisc_qlen_inc(sch);
 		return NET_XMIT_SUCCESS;
 	}
 	if (net_xmit_drop_count(ret))
@@ -106,7 +106,7 @@ static struct sk_buff *multiq_dequeue(struct Qdisc *sch)
 			skb = qdisc->dequeue(qdisc);
 			if (skb) {
 				qdisc_bstats_update(sch, skb);
-				sch->q.qlen--;
+				qdisc_qlen_dec(sch);
 				return skb;
 			}
 		}
diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c
index bc18e1976b6e0..6cd1838e09e7d 100644
--- a/net/sched/sch_netem.c
+++ b/net/sched/sch_netem.c
@@ -27,8 +27,6 @@
 #include <net/pkt_sched.h>
 #include <net/inet_ecn.h>
 
-#define VERSION "1.3"
-
 /*	Network Emulation Queuing algorithm.
 	====================================
 
@@ -71,89 +69,101 @@ struct disttable {
 	s16 table[] __counted_by(size);
 };
 
-struct netem_sched_data {
-	/* internal t(ime)fifo qdisc uses t_root and sch->limit */
-	struct rb_root t_root;
-
-	/* a linear queue; reduces rbtree rebalancing when jitter is low */
-	struct sk_buff	*t_head;
-	struct sk_buff	*t_tail;
-
-	u32 t_len;
-
-	/* optional qdisc for classful handling (NULL at netem init) */
-	struct Qdisc	*qdisc;
-
-	struct qdisc_watchdog watchdog;
+/* Loss models */
+enum {
+	CLG_RANDOM,
+	CLG_4_STATES,
+	CLG_GILB_ELL,
+};
 
-	s64 latency;
-	s64 jitter;
+/* States in GE model */
+enum {
+	GOOD_STATE = 1,
+	BAD_STATE,
+};
 
-	u32 loss;
-	u32 ecn;
-	u32 limit;
-	u32 counter;
-	u32 gap;
-	u32 duplicate;
-	u32 reorder;
-	u32 corrupt;
-	u64 rate;
-	s32 packet_overhead;
-	u32 cell_size;
-	struct reciprocal_value cell_size_reciprocal;
-	s32 cell_overhead;
+/* States in 4 state model */
+enum {
+	TX_IN_GAP_PERIOD = 1,
+	TX_IN_BURST_PERIOD,
+	LOST_IN_GAP_PERIOD,
+	LOST_IN_BURST_PERIOD,
+};
 
+struct netem_sched_data {
+	/* Cacheline 0: tfifo state and per-packet enqueue/dequeue scalars. */
+	struct rb_root		t_root;
+	struct sk_buff		*t_head;
+	struct sk_buff		*t_tail;
+	u32			t_len;
+	u32			counter;
+	s64			latency;
+	s64			jitter;
+	u64			rate;
+	u32			gap;
+	u32			loss;
+
+	/* Cacheline 1: zero-check scalars and correlation states. */
+	u32			duplicate;
+	u32			reorder;
+	u32			corrupt;
+	u32			ecn;
 	struct crndstate {
 		u32 last;
 		u32 rho;
 	} delay_cor, loss_cor, dup_cor, reorder_cor, corrupt_cor;
+	u8			loss_model;
 
-	struct prng  {
+	/* Cacheline 2: PRNG, distribution tables, slot dequeue state etc. */
+	struct prng {
 		u64 seed;
 		struct rnd_state prng_state;
 	} prng;
+	struct disttable	*delay_dist;
+	struct slotstate {
+		u64 slot_next;
+		s32 packets_left;
+		s32 bytes_left;
+	} slot;
+	struct disttable	*slot_dist;
+	struct Qdisc		*qdisc;
 
-	struct disttable *delay_dist;
-
-	enum  {
-		CLG_RANDOM,
-		CLG_4_STATES,
-		CLG_GILB_ELL,
-	} loss_model;
-
-	enum {
-		TX_IN_GAP_PERIOD = 1,
-		TX_IN_BURST_PERIOD,
-		LOST_IN_GAP_PERIOD,
-		LOST_IN_BURST_PERIOD,
-	} _4_state_model;
-
-	enum {
-		GOOD_STATE = 1,
-		BAD_STATE,
-	} GE_state_model;
+	/*
+	 * Warm: rate-shaping parameters (only read when rate != 0) and
+	 * configuration-only fields.  The fast path reads sch->limit, not
+	 * q->limit.
+	 */
+	s32			packet_overhead;
+	u32			cell_size;
+	struct reciprocal_value	cell_size_reciprocal;
+	s32			cell_overhead;
+	u32			limit;
 
 	/* Correlated Loss Generation models */
 	struct clgstate {
-		/* state of the Markov chain */
-		u8 state;
-
 		/* 4-states and Gilbert-Elliot models */
 		u32 a1;	/* p13 for 4-states or p for GE */
 		u32 a2;	/* p31 for 4-states or r for GE */
 		u32 a3;	/* p32 for 4-states or h for GE */
 		u32 a4;	/* p14 for 4-states or 1-k for GE */
 		u32 a5; /* p23 used only in 4-states */
-	} clg;
 
-	struct tc_netem_slot slot_config;
-	struct slotstate {
-		u64 slot_next;
-		s32 packets_left;
-		s32 bytes_left;
-	} slot;
+		/* state of the Markov chain */
+		u8  state;
+	} clg;
 
-	struct disttable *slot_dist;
+	/* Impairment counters */
+	u64			delayed;
+	u64			dropped;
+	u64			corrupted;
+	u64			duplicated;
+	u64			ecn_marked;
+	u64			reordered;
+	u64			allocation_errors;
+
+	/* Cold tail: slot reschedule config and the watchdog timer. */
+	struct tc_netem_slot	slot_config;
+	struct qdisc_watchdog	watchdog;
 };
 
 /* Time stamp put into socket buffer control block
@@ -416,7 +426,7 @@ static void tfifo_enqueue(struct sk_buff *nskb, struct Qdisc *sch)
 		rb_insert_color(&nskb->rbnode, &q->t_root);
 	}
 	q->t_len++;
-	sch->q.qlen++;
+	qdisc_qlen_inc(sch);
 }
 
 /* netem can't properly corrupt a megapacket (like we get from GSO), so instead
@@ -461,16 +471,21 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	skb->prev = NULL;
 
 	/* Random duplication */
-	if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor, &q->prng))
+	if (q->duplicate && q->duplicate >= get_crandom(&q->dup_cor, &q->prng)) {
 		++count;
+		WRITE_ONCE(q->duplicated, q->duplicated + 1);
+	}
 
 	/* Drop packet? */
 	if (loss_event(q)) {
-		if (q->ecn && INET_ECN_set_ce(skb))
-			qdisc_qstats_drop(sch); /* mark packet */
-		else
+		if (q->ecn && INET_ECN_set_ce(skb)) {
+			WRITE_ONCE(q->ecn_marked, q->ecn_marked + 1);
+		} else {
+			WRITE_ONCE(q->dropped, q->dropped + 1);
 			--count;
+		}
 	}
+
 	if (count == 0) {
 		qdisc_qstats_drop(sch);
 		__qdisc_drop(skb, to_free);
@@ -487,8 +502,11 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	 * If we need to duplicate packet, then clone it before
 	 * original is modified.
 	 */
-	if (count > 1)
+	if (count > 1) {
 		skb2 = skb_clone(skb, GFP_ATOMIC);
+		if (!skb2)
+			WRITE_ONCE(q->allocation_errors, q->allocation_errors + 1);
+	}
 
 	/*
 	 * Randomized packet corruption.
@@ -499,8 +517,10 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	if (q->corrupt && q->corrupt >= get_crandom(&q->corrupt_cor, &q->prng)) {
 		if (skb_is_gso(skb)) {
 			skb = netem_segment(skb, sch, to_free);
-			if (!skb)
+			if (!skb) {
+				WRITE_ONCE(q->allocation_errors, q->allocation_errors + 1);
 				goto finish_segs;
+			}
 
 			segs = skb->next;
 			skb_mark_not_on_list(skb);
@@ -509,19 +529,23 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 
 		skb = skb_unshare(skb, GFP_ATOMIC);
 		if (unlikely(!skb)) {
+			WRITE_ONCE(q->allocation_errors, q->allocation_errors + 1);
 			qdisc_qstats_drop(sch);
 			goto finish_segs;
 		}
-		if (skb->ip_summed == CHECKSUM_PARTIAL &&
-		    skb_checksum_help(skb)) {
+		if (skb_linearize(skb) ||
+		    (skb->ip_summed == CHECKSUM_PARTIAL && skb_checksum_help(skb))) {
+			WRITE_ONCE(q->allocation_errors, q->allocation_errors + 1);
 			qdisc_drop(skb, sch, to_free);
 			skb = NULL;
 			goto finish_segs;
 		}
 
-		if (skb_headlen(skb))
-			skb->data[get_random_u32_below(skb_headlen(skb))] ^=
-				1 << get_random_u32_below(8);
+		if (skb->len) {
+			u32 offset = get_random_u32_below(skb->len);
+			skb->data[offset] ^= 1 << get_random_u32_below(8);
+			WRITE_ONCE(q->corrupted, q->corrupted + 1);
+		}
 	}
 
 	if (unlikely(sch->q.qlen >= sch->limit)) {
@@ -602,12 +626,16 @@ static int netem_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 
 		cb->time_to_send = now + delay;
 		++q->counter;
+		if (delay)
+			WRITE_ONCE(q->delayed, q->delayed + 1);
+
 		tfifo_enqueue(skb, sch);
 	} else {
 		/*
 		 * Do re-ordering by putting one out of N packets at the front
 		 * of the queue.
 		 */
+		WRITE_ONCE(q->reordered, q->reordered + 1);
 		cb->time_to_send = ktime_get_ns();
 		q->counter = 0;
 
@@ -750,20 +778,20 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch)
 				if (err != NET_XMIT_SUCCESS) {
 					if (net_xmit_drop_count(err))
 						qdisc_qstats_drop(sch);
-					sch->qstats.backlog -= pkt_len;
-					sch->q.qlen--;
+					qstats_backlog_sub(sch, pkt_len);
+					qdisc_qlen_dec(sch);
 					qdisc_tree_reduce_backlog(sch, 1, pkt_len);
 				}
 				goto tfifo_dequeue;
 			}
-			sch->q.qlen--;
+			qdisc_qlen_dec(sch);
 			goto deliver;
 		}
 
 		if (q->qdisc) {
 			skb = q->qdisc->ops->dequeue(q->qdisc);
 			if (skb) {
-				sch->q.qlen--;
+				qdisc_qlen_dec(sch);
 				goto deliver;
 			}
 		}
@@ -776,7 +804,7 @@ static struct sk_buff *netem_dequeue(struct Qdisc *sch)
 	if (q->qdisc) {
 		skb = q->qdisc->ops->dequeue(q->qdisc);
 		if (skb) {
-			sch->q.qlen--;
+			qdisc_qlen_dec(sch);
 			goto deliver;
 		}
 	}
@@ -920,7 +948,8 @@ static void get_rate(struct netem_sched_data *q, const struct nlattr *attr)
 		q->cell_size_reciprocal = (struct reciprocal_value) { 0 };
 }
 
-static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr)
+static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr,
+			struct netlink_ext_ack *extack)
 {
 	const struct nlattr *la;
 	int rem;
@@ -933,7 +962,8 @@ static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr)
 			const struct tc_netem_gimodel *gi = nla_data(la);
 
 			if (nla_len(la) < sizeof(struct tc_netem_gimodel)) {
-				pr_info("netem: incorrect gi model size\n");
+				NL_SET_ERR_MSG_ATTR(extack, la,
+						    "netem: incorrect gi model size");
 				return -EINVAL;
 			}
 
@@ -952,7 +982,8 @@ static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr)
 			const struct tc_netem_gemodel *ge = nla_data(la);
 
 			if (nla_len(la) < sizeof(struct tc_netem_gemodel)) {
-				pr_info("netem: incorrect ge model size\n");
+				NL_SET_ERR_MSG_ATTR(extack, la,
+						    "netem: incorrect ge model size");
 				return -EINVAL;
 			}
 
@@ -966,7 +997,8 @@ static int get_loss_clg(struct netem_sched_data *q, const struct nlattr *attr)
 		}
 
 		default:
-			pr_info("netem: unknown loss type %u\n", type);
+			NL_SET_ERR_MSG_ATTR_FMT(extack, la,
+						"netem: unknown loss type %u", type);
 			return -EINVAL;
 		}
 	}
@@ -989,19 +1021,21 @@ static const struct nla_policy netem_policy[TCA_NETEM_MAX + 1] = {
 };
 
 static int parse_attr(struct nlattr *tb[], int maxtype, struct nlattr *nla,
-		      const struct nla_policy *policy, int len)
+		      const struct nla_policy *policy, int len,
+		      struct netlink_ext_ack *extack)
 {
 	int nested_len = nla_len(nla) - NLA_ALIGN(len);
 
 	if (nested_len < 0) {
-		pr_info("netem: invalid attributes len %d\n", nested_len);
+		NL_SET_ERR_MSG_FMT(extack, "netem: invalid attributes len %d < %d",
+				   nla_len(nla), NLA_ALIGN(len));
 		return -EINVAL;
 	}
 
 	if (nested_len >= nla_attr_size(0))
 		return nla_parse_deprecated(tb, maxtype,
 					    nla_data(nla) + NLA_ALIGN(len),
-					    nested_len, policy, NULL);
+					    nested_len, policy, extack);
 
 	memset(tb, 0, sizeof(struct nlattr *) * (maxtype + 1));
 	return 0;
@@ -1056,7 +1090,7 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt,
 	int ret;
 
 	qopt = nla_data(opt);
-	ret = parse_attr(tb, TCA_NETEM_MAX, opt, netem_policy, sizeof(*qopt));
+	ret = parse_attr(tb, TCA_NETEM_MAX, opt, netem_policy, sizeof(*qopt), extack);
 	if (ret < 0)
 		return ret;
 
@@ -1096,7 +1130,7 @@ static int netem_change(struct Qdisc *sch, struct nlattr *opt,
 	old_loss_model = q->loss_model;
 
 	if (tb[TCA_NETEM_LOSS]) {
-		ret = get_loss_clg(q, tb[TCA_NETEM_LOSS]);
+		ret = get_loss_clg(q, tb[TCA_NETEM_LOSS], extack);
 		if (ret) {
 			q->loss_model = old_loss_model;
 			q->clg = old_clg;
@@ -1180,7 +1214,6 @@ static int netem_init(struct Qdisc *sch, struct nlattr *opt,
 		      struct netlink_ext_ack *extack)
 {
 	struct netem_sched_data *q = qdisc_priv(sch);
-	int ret;
 
 	qdisc_watchdog_init(&q->watchdog, sch);
 
@@ -1191,10 +1224,7 @@ static int netem_init(struct Qdisc *sch, struct nlattr *opt,
 	q->prng.seed = get_random_u64();
 	prandom_seed_state(&q->prng.prng_state, q->prng.seed);
 
-	ret = netem_change(sch, opt, extack);
-	if (ret)
-		pr_info("netem: change failed\n");
-	return ret;
+	return netem_change(sch, opt, extack);
 }
 
 static void netem_destroy(struct Qdisc *sch)
@@ -1344,6 +1374,22 @@ static int netem_dump(struct Qdisc *sch, struct sk_buff *skb)
 	return -1;
 }
 
+static int netem_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
+{
+	struct netem_sched_data *q = qdisc_priv(sch);
+	struct tc_netem_xstats st = {
+		.delayed    = READ_ONCE(q->delayed),
+		.dropped    = READ_ONCE(q->dropped),
+		.corrupted  = READ_ONCE(q->corrupted),
+		.duplicated = READ_ONCE(q->duplicated),
+		.reordered  = READ_ONCE(q->reordered),
+		.ecn_marked = READ_ONCE(q->ecn_marked),
+		.allocation_errors = READ_ONCE(q->allocation_errors),
+	};
+
+	return gnet_stats_copy_app(d, &st, sizeof(st));
+}
+
 static int netem_dump_class(struct Qdisc *sch, unsigned long cl,
 			  struct sk_buff *skb, struct tcmsg *tcm)
 {
@@ -1406,14 +1452,13 @@ static struct Qdisc_ops netem_qdisc_ops __read_mostly = {
 	.destroy	=	netem_destroy,
 	.change		=	netem_change,
 	.dump		=	netem_dump,
+	.dump_stats	=	netem_dump_stats,
 	.owner		=	THIS_MODULE,
 };
 MODULE_ALIAS_NET_SCH("netem");
 
-
 static int __init netem_module_init(void)
 {
-	pr_info("netem: version " VERSION "\n");
 	return register_qdisc(&netem_qdisc_ops);
 }
 static void __exit netem_module_exit(void)
diff --git a/net/sched/sch_prio.c b/net/sched/sch_prio.c
index 9e2b9a490db23..e4dd56a890725 100644
--- a/net/sched/sch_prio.c
+++ b/net/sched/sch_prio.c
@@ -85,8 +85,8 @@ prio_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
 
 	ret = qdisc_enqueue(skb, qdisc, to_free);
 	if (ret == NET_XMIT_SUCCESS) {
-		sch->qstats.backlog += len;
-		sch->q.qlen++;
+		qstats_backlog_add(sch, len);
+		qdisc_qlen_inc(sch);
 		return NET_XMIT_SUCCESS;
 	}
 	if (net_xmit_drop_count(ret))
@@ -119,7 +119,7 @@ static struct sk_buff *prio_dequeue(struct Qdisc *sch)
 		if (skb) {
 			qdisc_bstats_update(sch, skb);
 			qdisc_qstats_backlog_dec(sch, skb);
-			sch->q.qlen--;
+			qdisc_qlen_dec(sch);
 			return skb;
 		}
 	}
diff --git a/net/sched/sch_qfq.c b/net/sched/sch_qfq.c
index 699e45873f861..cb56787e1d258 100644
--- a/net/sched/sch_qfq.c
+++ b/net/sched/sch_qfq.c
@@ -1152,12 +1152,12 @@ static struct sk_buff *qfq_dequeue(struct Qdisc *sch)
 	if (!skb)
 		return NULL;
 
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 
 	skb = agg_dequeue(in_serv_agg, cl, len);
 
 	if (!skb) {
-		sch->q.qlen++;
+		qdisc_qlen_inc(sch);
 		return NULL;
 	}
 
@@ -1264,8 +1264,8 @@ static int qfq_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	}
 
 	_bstats_update(&cl->bstats, len, gso_segs);
-	sch->qstats.backlog += len;
-	++sch->q.qlen;
+	qstats_backlog_add(sch, len);
+	qdisc_qlen_inc(sch);
 
 	agg = cl->agg;
 	/* if the class is active, then done here */
diff --git a/net/sched/sch_red.c b/net/sched/sch_red.c
index 4d0e44a2e7c66..d7598214270b8 100644
--- a/net/sched/sch_red.c
+++ b/net/sched/sch_red.c
@@ -138,8 +138,8 @@ static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	len = qdisc_pkt_len(skb);
 	ret = qdisc_enqueue(skb, child, to_free);
 	if (likely(ret == NET_XMIT_SUCCESS)) {
-		sch->qstats.backlog += len;
-		sch->q.qlen++;
+		qstats_backlog_add(sch, len);
+		qdisc_qlen_inc(sch);
 	} else if (net_xmit_drop_count(ret)) {
 		WRITE_ONCE(q->stats.pdrop,
 			   q->stats.pdrop + 1);
@@ -166,7 +166,7 @@ static struct sk_buff *red_dequeue(struct Qdisc *sch)
 	if (skb) {
 		qdisc_bstats_update(sch, skb);
 		qdisc_qstats_backlog_dec(sch, skb);
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 	} else {
 		if (!red_is_idling(&q->vars))
 			red_start_of_idle_period(&q->vars);
diff --git a/net/sched/sch_sfb.c b/net/sched/sch_sfb.c
index d3ee8e5479b35..b1d4650942769 100644
--- a/net/sched/sch_sfb.c
+++ b/net/sched/sch_sfb.c
@@ -415,8 +415,8 @@ static int sfb_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 	memcpy(&cb, sfb_skb_cb(skb), sizeof(cb));
 	ret = qdisc_enqueue(skb, child, to_free);
 	if (likely(ret == NET_XMIT_SUCCESS)) {
-		sch->qstats.backlog += len;
-		sch->q.qlen++;
+		qstats_backlog_add(sch, len);
+		qdisc_qlen_inc(sch);
 		increment_qlen(&cb, q);
 	} else if (net_xmit_drop_count(ret)) {
 		WRITE_ONCE(q->stats.childdrop,
@@ -446,7 +446,7 @@ static struct sk_buff *sfb_dequeue(struct Qdisc *sch)
 	if (skb) {
 		qdisc_bstats_update(sch, skb);
 		qdisc_qstats_backlog_dec(sch, skb);
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 		decrement_qlen(skb, q);
 	}
 
@@ -592,7 +592,7 @@ static int sfb_dump(struct Qdisc *sch, struct sk_buff *skb)
 		.penalty_burst = q->penalty_burst,
 	};
 
-	sch->qstats.backlog = q->qdisc->qstats.backlog;
+	WRITE_ONCE(sch->qstats.backlog, READ_ONCE(q->qdisc->qstats.backlog));
 	opts = nla_nest_start_noflag(skb, TCA_OPTIONS);
 	if (opts == NULL)
 		goto nla_put_failure;
diff --git a/net/sched/sch_sfq.c b/net/sched/sch_sfq.c
index f39822babf88b..758b88f218652 100644
--- a/net/sched/sch_sfq.c
+++ b/net/sched/sch_sfq.c
@@ -302,7 +302,7 @@ static unsigned int sfq_drop(struct Qdisc *sch, struct sk_buff **to_free)
 		len = qdisc_pkt_len(skb);
 		WRITE_ONCE(slot->backlog, slot->backlog - len);
 		sfq_dec(q, x);
-		sch->q.qlen--;
+		qdisc_qlen_dec(sch);
 		qdisc_qstats_backlog_dec(sch, skb);
 		qdisc_drop_reason(skb, sch, to_free, QDISC_DROP_OVERLIMIT);
 		return len;
@@ -427,7 +427,7 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
 		/* We know we have at least one packet in queue */
 		head = slot_dequeue_head(slot);
 		delta = qdisc_pkt_len(head) - qdisc_pkt_len(skb);
-		sch->qstats.backlog -= delta;
+		qstats_backlog_sub(sch, delta);
 		WRITE_ONCE(slot->backlog, slot->backlog - delta);
 		qdisc_drop_reason(head, sch, to_free, QDISC_DROP_FLOW_LIMIT);
 
@@ -456,7 +456,8 @@ sfq_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free)
 		/* We could use a bigger initial quantum for new flows */
 		WRITE_ONCE(slot->allot, q->quantum);
 	}
-	if (++sch->q.qlen <= q->limit)
+	qdisc_qlen_inc(sch);
+	if (sch->q.qlen <= q->limit)
 		return NET_XMIT_SUCCESS;
 
 	qlen = slot->qlen;
@@ -497,7 +498,7 @@ sfq_dequeue(struct Qdisc *sch)
 	skb = slot_dequeue_head(slot);
 	sfq_dec(q, a);
 	qdisc_bstats_update(sch, skb);
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 	qdisc_qstats_backlog_dec(sch, skb);
 	WRITE_ONCE(slot->backlog, slot->backlog - qdisc_pkt_len(skb));
 	/* Is the slot empty? */
@@ -596,7 +597,7 @@ static void sfq_rehash(struct Qdisc *sch)
 			WRITE_ONCE(slot->allot, q->quantum);
 		}
 	}
-	sch->q.qlen -= dropped;
+	WRITE_ONCE(sch->q.qlen, sch->q.qlen - dropped);
 	qdisc_tree_reduce_backlog(sch, dropped, drop_len);
 }
 
diff --git a/net/sched/sch_skbprio.c b/net/sched/sch_skbprio.c
index f485f62ab721a..52abfb4015a36 100644
--- a/net/sched/sch_skbprio.c
+++ b/net/sched/sch_skbprio.c
@@ -93,7 +93,7 @@ static int skbprio_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		if (prio < q->lowest_prio)
 			q->lowest_prio = prio;
 
-		sch->q.qlen++;
+		qdisc_qlen_inc(sch);
 		return NET_XMIT_SUCCESS;
 	}
 
@@ -145,7 +145,7 @@ static struct sk_buff *skbprio_dequeue(struct Qdisc *sch)
 	if (unlikely(!skb))
 		return NULL;
 
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 	qdisc_qstats_backlog_dec(sch, skb);
 	qdisc_bstats_update(sch, skb);
 
diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c
index 45245157e00a6..d6b981e5df11c 100644
--- a/net/sched/sch_taprio.c
+++ b/net/sched/sch_taprio.c
@@ -308,10 +308,10 @@ static void taprio_update_queue_max_sdu(struct taprio_sched *q,
 
 		if (max_sdu != U32_MAX) {
 			sched->max_frm_len[tc] = max_sdu + dev->hard_header_len;
-			sched->max_sdu[tc] = max_sdu;
+			WRITE_ONCE(sched->max_sdu[tc], max_sdu);
 		} else {
 			sched->max_frm_len[tc] = U32_MAX; /* never oversized */
-			sched->max_sdu[tc] = 0;
+			WRITE_ONCE(sched->max_sdu[tc], 0);
 		}
 	}
 }
@@ -574,7 +574,7 @@ static int taprio_enqueue_one(struct sk_buff *skb, struct Qdisc *sch,
 	}
 
 	qdisc_qstats_backlog_inc(sch, skb);
-	sch->q.qlen++;
+	qdisc_qlen_inc(sch);
 
 	return qdisc_enqueue(skb, child, to_free);
 }
@@ -755,7 +755,7 @@ static struct sk_buff *taprio_dequeue_from_txq(struct Qdisc *sch, int txq,
 
 	qdisc_bstats_update(sch, skb);
 	qdisc_qstats_backlog_dec(sch, skb);
-	sch->q.qlen--;
+	qdisc_qlen_dec(sch);
 
 	return skb;
 }
@@ -1771,8 +1771,8 @@ static int taprio_parse_tc_entries(struct Qdisc *sch,
 	}
 
 	for (tc = 0; tc < TC_QOPT_MAX_QUEUE; tc++) {
-		q->max_sdu[tc] = max_sdu[tc];
-		q->fp[tc] = fp[tc];
+		WRITE_ONCE(q->max_sdu[tc], max_sdu[tc]);
+		WRITE_ONCE(q->fp[tc], fp[tc]);
 		if (fp[tc] != TC_FP_EXPRESS)
 			have_preemption = true;
 	}
@@ -1852,12 +1852,14 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt,
 		return -EINVAL;
 	}
 
-	if (q->flags != TAPRIO_FLAGS_INVALID && q->flags != taprio_flags) {
-		NL_SET_ERR_MSG_MOD(extack,
-				   "Changing 'flags' of a running schedule is not supported");
-		return -EOPNOTSUPP;
+	if (q->flags != taprio_flags) {
+		if (q->flags != TAPRIO_FLAGS_INVALID) {
+			NL_SET_ERR_MSG_MOD(extack,
+					   "Changing 'flags' of a running schedule is not supported");
+			return -EOPNOTSUPP;
+		}
+		WRITE_ONCE(q->flags, taprio_flags);
 	}
-	q->flags = taprio_flags;
 
 	/* Needed for length_to_duration() during netlink attribute parsing */
 	taprio_set_picos_per_byte(dev, q, extack);
@@ -1940,7 +1942,8 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt,
 			goto unlock;
 		}
 
-		q->txtime_delay = nla_get_u32(tb[TCA_TAPRIO_ATTR_TXTIME_DELAY]);
+		WRITE_ONCE(q->txtime_delay,
+			   nla_get_u32(tb[TCA_TAPRIO_ATTR_TXTIME_DELAY]));
 	}
 
 	if (!TXTIME_ASSIST_IS_ENABLED(q->flags) &&
@@ -2283,8 +2286,8 @@ static int dump_schedule(struct sk_buff *msg,
 }
 
 static int taprio_dump_tc_entries(struct sk_buff *skb,
-				  struct taprio_sched *q,
-				  struct sched_gate_list *sched)
+				  const struct taprio_sched *q,
+				  const struct sched_gate_list *sched)
 {
 	struct nlattr *n;
 	int tc;
@@ -2298,10 +2301,11 @@ static int taprio_dump_tc_entries(struct sk_buff *skb,
 			goto nla_put_failure;
 
 		if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_MAX_SDU,
-				sched->max_sdu[tc]))
+				READ_ONCE(sched->max_sdu[tc])))
 			goto nla_put_failure;
 
-		if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_FP, q->fp[tc]))
+		if (nla_put_u32(skb, TCA_TAPRIO_TC_ENTRY_FP,
+				READ_ONCE(q->fp[tc])))
 			goto nla_put_failure;
 
 		nla_nest_end(skb, n);
@@ -2387,6 +2391,7 @@ static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb)
 	struct sched_gate_list *oper, *admin;
 	struct tc_mqprio_qopt opt = { 0 };
 	struct nlattr *nest, *sched_nest;
+	u32 txtime_delay;
 
 	mqprio_qopt_reconstruct(dev, &opt);
 
@@ -2404,14 +2409,15 @@ static int taprio_dump(struct Qdisc *sch, struct sk_buff *skb)
 	if (q->flags && nla_put_u32(skb, TCA_TAPRIO_ATTR_FLAGS, q->flags))
 		goto options_error;
 
-	if (q->txtime_delay &&
-	    nla_put_u32(skb, TCA_TAPRIO_ATTR_TXTIME_DELAY, q->txtime_delay))
+	txtime_delay = READ_ONCE(q->txtime_delay);
+	if (txtime_delay &&
+	    nla_put_u32(skb, TCA_TAPRIO_ATTR_TXTIME_DELAY, txtime_delay))
 		goto options_error;
 
 	rcu_read_lock();
 
-	oper = rtnl_dereference(q->oper_sched);
-	admin = rtnl_dereference(q->admin_sched);
+	oper = rcu_dereference(q->oper_sched);
+	admin = rcu_dereference(q->admin_sched);
 
 	if (oper && taprio_dump_tc_entries(skb, q, oper))
 		goto options_error_rcu;
diff --git a/net/sched/sch_tbf.c b/net/sched/sch_tbf.c
index f2340164f579a..67c7aaaf8f607 100644
--- a/net/sched/sch_tbf.c
+++ b/net/sched/sch_tbf.c
@@ -231,8 +231,8 @@ static int tbf_segment(struct sk_buff *skb, struct Qdisc *sch,
 			len += seg_len;
 		}
 	}
-	sch->q.qlen += nb;
-	sch->qstats.backlog += len;
+	WRITE_ONCE(sch->q.qlen, sch->q.qlen + nb);
+	qstats_backlog_add(sch, len);
 	if (nb > 0) {
 		qdisc_tree_reduce_backlog(sch, 1 - nb, prev_len - len);
 		consume_skb(skb);
@@ -263,8 +263,8 @@ static int tbf_enqueue(struct sk_buff *skb, struct Qdisc *sch,
 		return ret;
 	}
 
-	sch->qstats.backlog += len;
-	sch->q.qlen++;
+	qstats_backlog_add(sch, len);
+	qdisc_qlen_inc(sch);
 	return NET_XMIT_SUCCESS;
 }
 
@@ -309,7 +309,7 @@ static struct sk_buff *tbf_dequeue(struct Qdisc *sch)
 			q->tokens = toks;
 			q->ptokens = ptoks;
 			qdisc_qstats_backlog_dec(sch, skb);
-			sch->q.qlen--;
+			qdisc_qlen_dec(sch);
 			qdisc_bstats_update(sch, skb);
 			return skb;
 		}
diff --git a/net/sched/sch_teql.c b/net/sched/sch_teql.c
index ec4039a201a2c..e7bbc9e5174d0 100644
--- a/net/sched/sch_teql.c
+++ b/net/sched/sch_teql.c
@@ -107,7 +107,7 @@ teql_dequeue(struct Qdisc *sch)
 	} else {
 		qdisc_bstats_update(sch, skb);
 	}
-	sch->q.qlen = dat->q.qlen + q->q.qlen;
+	WRITE_ONCE(sch->q.qlen, dat->q.qlen + READ_ONCE(q->q.qlen));
 	return skb;
 }
 
diff --git a/net/sctp/endpointola.c b/net/sctp/endpointola.c
index 8d342b514142f..dfb1719275dba 100644
--- a/net/sctp/endpointola.c
+++ b/net/sctp/endpointola.c
@@ -209,7 +209,7 @@ static void sctp_endpoint_destroy(struct sctp_endpoint *ep)
 		return;
 	}
 
-	/* SCTP-AUTH: Free up AUTH releated data such as shared keys
+	/* SCTP-AUTH: Free up AUTH related data such as shared keys
 	 * chunks and hmacs arrays that were allocated
 	 */
 	sctp_auth_destroy_keys(&ep->endpoint_shared_keys);
diff --git a/net/sctp/protocol.c b/net/sctp/protocol.c
index 5800e7ee7ea02..5c6fa8e8d34d3 100644
--- a/net/sctp/protocol.c
+++ b/net/sctp/protocol.c
@@ -864,7 +864,7 @@ int sctp_udp_sock_start(struct net *net)
 	tuncfg.encap_type = 1;
 	tuncfg.encap_rcv = sctp_udp_rcv;
 	tuncfg.encap_err_lookup = sctp_udp_v4_err;
-	setup_udp_tunnel_sock(net, sock, &tuncfg);
+	setup_udp_tunnel_sock(net, sock->sk, &tuncfg);
 	net->sctp.udp4_sock = sock->sk;
 
 #if IS_ENABLED(CONFIG_IPV6)
@@ -878,7 +878,7 @@ int sctp_udp_sock_start(struct net *net)
 	err = udp_sock_create(net, &udp_conf, &sock);
 	if (err) {
 		pr_err("Failed to create the SCTP UDP tunneling v6 sock\n");
-		udp_tunnel_sock_release(net->sctp.udp4_sock->sk_socket);
+		udp_tunnel_sock_release(net->sctp.udp4_sock);
 		net->sctp.udp4_sock = NULL;
 		return err;
 	}
@@ -886,7 +886,7 @@ int sctp_udp_sock_start(struct net *net)
 	tuncfg.encap_type = 1;
 	tuncfg.encap_rcv = sctp_udp_rcv;
 	tuncfg.encap_err_lookup = sctp_udp_v6_err;
-	setup_udp_tunnel_sock(net, sock, &tuncfg);
+	setup_udp_tunnel_sock(net, sock->sk, &tuncfg);
 	net->sctp.udp6_sock = sock->sk;
 #endif
 
@@ -896,11 +896,11 @@ int sctp_udp_sock_start(struct net *net)
 void sctp_udp_sock_stop(struct net *net)
 {
 	if (net->sctp.udp4_sock) {
-		udp_tunnel_sock_release(net->sctp.udp4_sock->sk_socket);
+		udp_tunnel_sock_release(net->sctp.udp4_sock);
 		net->sctp.udp4_sock = NULL;
 	}
 	if (net->sctp.udp6_sock) {
-		udp_tunnel_sock_release(net->sctp.udp6_sock->sk_socket);
+		udp_tunnel_sock_release(net->sctp.udp6_sock);
 		net->sctp.udp6_sock = NULL;
 	}
 }
diff --git a/net/shaper/shaper.c b/net/shaper/shaper.c
index b1c65110f04d3..dea9270f3e57d 100644
--- a/net/shaper/shaper.c
+++ b/net/shaper/shaper.c
@@ -138,35 +138,58 @@ static int net_shaper_fill_handle(struct sk_buff *msg,
 	return -EMSGSIZE;
 }
 
+static void net_shaper_copy(struct net_shaper *dst,
+			    const struct net_shaper *src)
+{
+	WRITE_ONCE(dst->parent.scope, READ_ONCE(src->parent.scope));
+	WRITE_ONCE(dst->parent.id, READ_ONCE(src->parent.id));
+	WRITE_ONCE(dst->handle.scope, READ_ONCE(src->handle.scope));
+	WRITE_ONCE(dst->handle.id, READ_ONCE(src->handle.id));
+
+	WRITE_ONCE(dst->metric, READ_ONCE(src->metric));
+	WRITE_ONCE(dst->bw_min, READ_ONCE(src->bw_min));
+	WRITE_ONCE(dst->bw_max, READ_ONCE(src->bw_max));
+	WRITE_ONCE(dst->burst, READ_ONCE(src->burst));
+	WRITE_ONCE(dst->priority, READ_ONCE(src->priority));
+	WRITE_ONCE(dst->weight, READ_ONCE(src->weight));
+
+	/* private fields are only used on the write path under the lock */
+	data_race(dst->leaves = src->leaves);
+}
+
 static int
 net_shaper_fill_one(struct sk_buff *msg,
 		    const struct net_shaper_binding *binding,
 		    const struct net_shaper *shaper,
 		    const struct genl_info *info)
 {
+	struct net_shaper cur;
 	void *hdr;
 
 	hdr = genlmsg_iput(msg, info);
 	if (!hdr)
 		return -EMSGSIZE;
 
+	/* Make a copy to avoid data races */
+	net_shaper_copy(&cur, shaper);
+
 	if (net_shaper_fill_binding(msg, binding, NET_SHAPER_A_IFINDEX) ||
-	    net_shaper_fill_handle(msg, &shaper->parent,
+	    net_shaper_fill_handle(msg, &cur.parent,
 				   NET_SHAPER_A_PARENT) ||
-	    net_shaper_fill_handle(msg, &shaper->handle,
+	    net_shaper_fill_handle(msg, &cur.handle,
 				   NET_SHAPER_A_HANDLE) ||
-	    ((shaper->bw_min || shaper->bw_max || shaper->burst) &&
-	     nla_put_u32(msg, NET_SHAPER_A_METRIC, shaper->metric)) ||
-	    (shaper->bw_min &&
-	     nla_put_uint(msg, NET_SHAPER_A_BW_MIN, shaper->bw_min)) ||
-	    (shaper->bw_max &&
-	     nla_put_uint(msg, NET_SHAPER_A_BW_MAX, shaper->bw_max)) ||
-	    (shaper->burst &&
-	     nla_put_uint(msg, NET_SHAPER_A_BURST, shaper->burst)) ||
-	    (shaper->priority &&
-	     nla_put_u32(msg, NET_SHAPER_A_PRIORITY, shaper->priority)) ||
-	    (shaper->weight &&
-	     nla_put_u32(msg, NET_SHAPER_A_WEIGHT, shaper->weight)))
+	    ((cur.bw_min || cur.bw_max || cur.burst) &&
+	     nla_put_u32(msg, NET_SHAPER_A_METRIC, cur.metric)) ||
+	    (cur.bw_min &&
+	     nla_put_uint(msg, NET_SHAPER_A_BW_MIN, cur.bw_min)) ||
+	    (cur.bw_max &&
+	     nla_put_uint(msg, NET_SHAPER_A_BW_MAX, cur.bw_max)) ||
+	    (cur.burst &&
+	     nla_put_uint(msg, NET_SHAPER_A_BURST, cur.burst)) ||
+	    (cur.priority &&
+	     nla_put_u32(msg, NET_SHAPER_A_PRIORITY, cur.priority)) ||
+	    (cur.weight &&
+	     nla_put_u32(msg, NET_SHAPER_A_WEIGHT, cur.weight)))
 		goto nla_put_failure;
 
 	genlmsg_end(msg, hdr);
@@ -283,31 +306,24 @@ static void net_shaper_default_parent(const struct net_shaper_handle *handle,
 	parent->id = 0;
 }
 
-/* MARK_0 is already in use due to XA_FLAGS_ALLOC. The VALID mark is set on
- * an entry only after the device-side configuration has completed
- * successfully (see net_shaper_commit()). Lookups and dumps must filter on
- * this mark to avoid exposing tentative entries inserted by
- * net_shaper_pre_insert() while the driver call is still in flight.
- */
-#define NET_SHAPER_VALID	XA_MARK_1
-
 static struct net_shaper *
 net_shaper_lookup(struct net_shaper_binding *binding,
 		  const struct net_shaper_handle *handle)
 {
 	u32 index = net_shaper_handle_to_index(handle);
 	struct net_shaper_hierarchy *hierarchy;
+	struct net_shaper *cur;
 
 	hierarchy = net_shaper_hierarchy_rcu(binding);
-	if (!hierarchy || !xa_get_mark(&hierarchy->shapers, index,
-				       NET_SHAPER_VALID))
+	if (!hierarchy)
 		return NULL;
 
-	/* Pairs with smp_wmb() in net_shaper_commit(): if the entry is
-	 * valid, its contents must be visible too.
-	 */
-	smp_rmb();
-	return xa_load(&hierarchy->shapers, index);
+	cur = xa_load(&hierarchy->shapers, index);
+	/* Check valid before reading fields */
+	if (!cur || !smp_load_acquire(&cur->valid))
+		return NULL;
+
+	return cur;
 }
 
 /* Allocate on demand the per device shaper's hierarchy container.
@@ -421,12 +437,10 @@ static void net_shaper_commit(struct net_shaper_binding *binding,
 		if (WARN_ON_ONCE(!cur))
 			continue;
 
-		/* Successful update: drop the tentative mark
-		 * and update the hierarchy container.
-		 */
-		*cur = shapers[i];
-		smp_wmb();
-		__xa_set_mark(&hierarchy->shapers, index, NET_SHAPER_VALID);
+		/* Successful update: update the hierarchy container... */
+		net_shaper_copy(cur, &shapers[i]);
+		/* ... publish to lockless readers. */
+		smp_store_release(&cur->valid, true);
 	}
 	xa_unlock(&hierarchy->shapers);
 }
@@ -443,10 +457,10 @@ static void net_shaper_rollback(struct net_shaper_binding *binding)
 
 	xa_lock(&hierarchy->shapers);
 	xa_for_each(&hierarchy->shapers, index, cur) {
-		if (xa_get_mark(&hierarchy->shapers, index, NET_SHAPER_VALID))
+		if (cur->valid)
 			continue;
 		__xa_erase(&hierarchy->shapers, index);
-		kfree(cur);
+		kfree_rcu(cur, rcu);
 	}
 	xa_unlock(&hierarchy->shapers);
 }
@@ -859,12 +873,12 @@ int net_shaper_nl_get_dumpit(struct sk_buff *skb,
 		goto out_unlock;
 
 	for (; (shaper = xa_find(&hierarchy->shapers, &ctx->start_index,
-				 U32_MAX, NET_SHAPER_VALID));
+				 U32_MAX, XA_PRESENT));
 	     ctx->start_index++) {
-		/* Pairs with smp_wmb() in net_shaper_commit(): the entry
-		 * is marked VALID, so its contents must be visible too.
-		 */
-		smp_rmb();
+		/* Check valid before reading fields */
+		if (!smp_load_acquire(&shaper->valid))
+			continue;
+
 		ret = net_shaper_fill_one(skb, binding, shaper, info);
 		if (ret)
 			break;
diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c
index e2d083daeb7e8..cf6b620fef05f 100644
--- a/net/smc/smc_core.c
+++ b/net/smc/smc_core.c
@@ -2440,6 +2440,10 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb)
 		/* use socket send buffer size (w/o overhead) as start value */
 		bufsize = smc->sk.sk_sndbuf / 2;
 
+	/* limit bufsize for physically contiguous buffers */
+	if (!is_smcd && lgr->buf_type == SMCR_PHYS_CONT_BUFS)
+		bufsize = min_t(int, bufsize, PAGE_SIZE << MAX_PAGE_ORDER);
+
 	for (bufsize_comp = smc_compress_bufsize(bufsize, is_smcd, is_rmb);
 	     bufsize_comp >= 0; bufsize_comp--) {
 		if (is_rmb) {
diff --git a/net/smc/smc_ism.c b/net/smc/smc_ism.c
index e0dba2c7b6e31..bde938c5eb397 100644
--- a/net/smc/smc_ism.c
+++ b/net/smc/smc_ism.c
@@ -467,17 +467,14 @@ static struct smcd_dev *smcd_alloc_dev(const char *name, int max_dmbs)
 {
 	struct smcd_dev *smcd;
 
-	smcd = kzalloc_obj(*smcd);
+	smcd = kzalloc_flex(*smcd, conn, max_dmbs);
 	if (!smcd)
 		return NULL;
-	smcd->conn = kzalloc_objs(struct smc_connection *, max_dmbs);
-	if (!smcd->conn)
-		goto free_smcd;
 
 	smcd->event_wq = alloc_ordered_workqueue("ism_evt_wq-%s)",
 						 WQ_MEM_RECLAIM, name);
 	if (!smcd->event_wq)
-		goto free_conn;
+		goto free_smcd;
 
 	spin_lock_init(&smcd->lock);
 	spin_lock_init(&smcd->lgr_lock);
@@ -486,8 +483,6 @@ static struct smcd_dev *smcd_alloc_dev(const char *name, int max_dmbs)
 	init_waitqueue_head(&smcd->lgrs_deleted);
 	return smcd;
 
-free_conn:
-	kfree(smcd->conn);
 free_smcd:
 	kfree(smcd);
 	return NULL;
@@ -557,7 +552,6 @@ static void smcd_unregister_dev(struct dibs_dev *dibs)
 	list_del_init(&smcd->list);
 	mutex_unlock(&smcd_dev_list.mutex);
 	destroy_workqueue(smcd->event_wq);
-	kfree(smcd->conn);
 	kfree(smcd);
 }
 
diff --git a/net/socket.c b/net/socket.c
index 22a412fdec079..0a10b63a72e97 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -710,7 +710,11 @@ struct socket *sock_alloc(void)
 }
 EXPORT_SYMBOL(sock_alloc);
 
-static void __sock_release(struct socket *sock, struct inode *inode)
+static
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+noinline
+#endif
+void __sock_release(struct socket *sock, struct inode *inode)
 {
 	const struct proto_ops *ops = READ_ONCE(sock->ops);
 
@@ -782,7 +786,13 @@ static noinline void call_trace_sock_send_length(struct sock *sk, int ret,
 	trace_sock_send_length(sk, ret, 0);
 }
 
-static inline int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg)
+static
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+noinline
+#else
+inline
+#endif
+int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg)
 {
 	int ret = INDIRECT_CALL_INET(READ_ONCE(sock->ops)->sendmsg, inet6_sendmsg,
 				     inet_sendmsg, sock, msg,
@@ -1131,8 +1141,13 @@ static noinline void call_trace_sock_recv_length(struct sock *sk, int ret, int f
 	trace_sock_recv_length(sk, ret, flags);
 }
 
-static inline int sock_recvmsg_nosec(struct socket *sock, struct msghdr *msg,
-				     int flags)
+static
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+noinline
+#else
+inline
+#endif
+int sock_recvmsg_nosec(struct socket *sock, struct msghdr *msg, int flags)
 {
 	int ret = INDIRECT_CALL_INET(READ_ONCE(sock->ops)->recvmsg,
 				     inet6_recvmsg,
@@ -2089,7 +2104,7 @@ static int __sys_accept4_file(struct file *file, struct sockaddr __user *upeer_s
  *	we open the socket then return an error.
  *
  *	1003.1g adds the ability to recvmsg() to query connection pending
- *	status to recvmsg. We need to add that support in a way thats
+ *	status. We need to add that support in a way that's
  *	clean when we restructure accept also.
  */
 
@@ -2638,9 +2653,12 @@ static int copy_msghdr_from_user(struct msghdr *kmsg,
 	return err < 0 ? err : 0;
 }
 
-static int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys,
-			   unsigned int flags, struct used_address *used_address,
-			   unsigned int allowed_msghdr_flags)
+static
+#ifdef CONFIG_NET_DEV_REFCNT_TRACKER
+noinline
+#endif
+int ____sys_sendmsg(struct socket *sock, struct msghdr *msg_sys, unsigned int flags,
+		    struct used_address *used_address, unsigned int allowed_msghdr_flags)
 {
 	unsigned char ctl[sizeof(struct cmsghdr) + 20]
 				__aligned(sizeof(__kernel_size_t));
diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile
index f89c10fe7e6ac..96727df3aa854 100644
--- a/net/sunrpc/Makefile
+++ b/net/sunrpc/Makefile
@@ -14,7 +14,7 @@ sunrpc-y := clnt.o xprt.o socklib.o xprtsock.o sched.o \
 	    addr.o rpcb_clnt.o timer.o xdr.o \
 	    sunrpc_syms.o cache.o rpc_pipe.o sysfs.o \
 	    svc_xprt.o \
-	    xprtmultipath.o
+	    xprtmultipath.o netlink.o
 sunrpc-$(CONFIG_SUNRPC_DEBUG) += debugfs.o
 sunrpc-$(CONFIG_SUNRPC_BACKCHANNEL) += backchannel_rqst.o
 sunrpc-$(CONFIG_PROC_FS) += stats.o
diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c
index 161d02cc1c2c9..d14209031e180 100644
--- a/net/sunrpc/auth_gss/svcauth_gss.c
+++ b/net/sunrpc/auth_gss/svcauth_gss.c
@@ -206,7 +206,7 @@ static struct cache_head *rsi_alloc(void)
 
 static int rsi_upcall(struct cache_detail *cd, struct cache_head *h)
 {
-	return sunrpc_cache_pipe_upcall_timeout(cd, h);
+	return sunrpc_cache_upcall_warn(cd, h);
 }
 
 static void rsi_request(struct cache_detail *cd,
diff --git a/net/sunrpc/cache.c b/net/sunrpc/cache.c
index b5474ce534fb9..305c6e67f0527 100644
--- a/net/sunrpc/cache.c
+++ b/net/sunrpc/cache.c
@@ -33,9 +33,11 @@
 #include <linux/sunrpc/cache.h>
 #include <linux/sunrpc/stats.h>
 #include <linux/sunrpc/rpc_pipe_fs.h>
+#include <net/genetlink.h>
 #include <trace/events/sunrpc.h>
 
 #include "netns.h"
+#include "netlink.h"
 #include "fail.h"
 
 #define	 RPCDBG_FACILITY RPCDBG_CACHE
@@ -1195,12 +1197,12 @@ static bool cache_listeners_exist(struct cache_detail *detail)
 }
 
 /*
- * register an upcall request to user-space and queue it up for read() by the
- * upcall daemon.
+ * register an upcall request to user-space and queue it up to be fetched by
+ * the upcall daemon.
  *
  * Each request is at most one page long.
  */
-static int cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h)
+static int cache_do_upcall(struct cache_detail *detail, struct cache_head *h)
 {
 	char *buf;
 	struct cache_request *crq;
@@ -1233,6 +1235,8 @@ static int cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h)
 		/* Lost a race, no longer PENDING, so don't enqueue */
 		ret = -EAGAIN;
 	spin_unlock(&detail->queue_lock);
+	if (ret != -EAGAIN && detail->cache_notify)
+		detail->cache_notify(detail, h);
 	wake_up(&detail->queue_wait);
 	if (ret == -EAGAIN) {
 		kfree(buf);
@@ -1241,25 +1245,25 @@ static int cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h)
 	return ret;
 }
 
-int sunrpc_cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h)
+int sunrpc_cache_upcall(struct cache_detail *detail, struct cache_head *h)
 {
 	if (test_and_set_bit(CACHE_PENDING, &h->flags))
 		return 0;
-	return cache_pipe_upcall(detail, h);
+	return cache_do_upcall(detail, h);
 }
-EXPORT_SYMBOL_GPL(sunrpc_cache_pipe_upcall);
+EXPORT_SYMBOL_GPL(sunrpc_cache_upcall);
 
-int sunrpc_cache_pipe_upcall_timeout(struct cache_detail *detail,
-				     struct cache_head *h)
+int sunrpc_cache_upcall_warn(struct cache_detail *detail,
+			     struct cache_head *h)
 {
 	if (!cache_listeners_exist(detail)) {
 		warn_no_listener(detail);
 		trace_cache_entry_no_listener(detail, h);
 		return -EINVAL;
 	}
-	return sunrpc_cache_pipe_upcall(detail, h);
+	return sunrpc_cache_upcall(detail, h);
 }
-EXPORT_SYMBOL_GPL(sunrpc_cache_pipe_upcall_timeout);
+EXPORT_SYMBOL_GPL(sunrpc_cache_upcall_warn);
 
 /*
  * parse a message from user-space and pass it
@@ -1897,3 +1901,106 @@ void sunrpc_cache_unhash(struct cache_detail *cd, struct cache_head *h)
 		spin_unlock(&cd->hash_lock);
 }
 EXPORT_SYMBOL_GPL(sunrpc_cache_unhash);
+
+/**
+ * sunrpc_cache_requests_count - count pending upcall requests
+ * @cd: cache_detail to query
+ *
+ * Returns the number of requests on the cache's request list that
+ * still have CACHE_PENDING set.
+ */
+int sunrpc_cache_requests_count(struct cache_detail *cd)
+{
+	struct cache_request *crq;
+	int cnt = 0;
+
+	spin_lock(&cd->queue_lock);
+	list_for_each_entry(crq, &cd->requests, list) {
+		if (test_bit(CACHE_PENDING, &crq->item->flags))
+			cnt++;
+	}
+	spin_unlock(&cd->queue_lock);
+	return cnt;
+}
+EXPORT_SYMBOL_GPL(sunrpc_cache_requests_count);
+
+/**
+ * sunrpc_cache_requests_snapshot - snapshot pending upcall requests
+ * @cd: cache_detail to query
+ * @items: array to fill with cache_head pointers (caller-allocated)
+ * @seqnos: array to fill with sequence numbers (caller-allocated)
+ * @max: size of the arrays
+ * @min_seqno: only include entries with seqno > min_seqno (0 for all)
+ *
+ * Only entries with CACHE_PENDING set are included. Takes a reference
+ * on each cache_head via cache_get(). Caller must call cache_put()
+ * on each returned item when done.
+ *
+ * Returns the number of entries filled.
+ */
+int sunrpc_cache_requests_snapshot(struct cache_detail *cd,
+				   struct cache_head **items,
+				   u64 *seqnos, int max,
+				   u64 min_seqno)
+{
+	struct cache_request *crq;
+	int i = 0;
+
+	spin_lock(&cd->queue_lock);
+	list_for_each_entry(crq, &cd->requests, list) {
+		if (i >= max)
+			break;
+		if (!test_bit(CACHE_PENDING, &crq->item->flags))
+			continue;
+		if (crq->seqno <= min_seqno)
+			continue;
+		items[i] = cache_get(crq->item);
+		seqnos[i] = crq->seqno;
+		i++;
+	}
+	spin_unlock(&cd->queue_lock);
+	return i;
+}
+EXPORT_SYMBOL_GPL(sunrpc_cache_requests_snapshot);
+
+/**
+ * sunrpc_cache_notify - send a netlink notification for a cache event
+ * @cd: cache_detail for the cache
+ * @h: cache_head entry (unused, reserved for future use)
+ * @cache_type: cache type identifier (e.g. SUNRPC_CACHE_TYPE_UNIX_GID)
+ *
+ * Sends a SUNRPC_CMD_CACHE_NOTIFY multicast message on the "exportd"
+ * group if any listeners are present. Returns 0 on success or a
+ * negative errno.
+ */
+int sunrpc_cache_notify(struct cache_detail *cd, struct cache_head *h,
+			u32 cache_type)
+{
+	struct genlmsghdr *hdr;
+	struct sk_buff *msg;
+
+	if (!genl_has_listeners(&sunrpc_nl_family, cd->net,
+				SUNRPC_NLGRP_EXPORTD))
+		return -ENOLINK;
+
+	msg = genlmsg_new(nla_total_size(sizeof(u32)), GFP_KERNEL);
+	if (!msg)
+		return -ENOMEM;
+
+	hdr = genlmsg_put(msg, 0, 0, &sunrpc_nl_family, 0,
+			  SUNRPC_CMD_CACHE_NOTIFY);
+	if (!hdr) {
+		nlmsg_free(msg);
+		return -ENOMEM;
+	}
+
+	if (nla_put_u32(msg, SUNRPC_A_CACHE_NOTIFY_CACHE_TYPE, cache_type)) {
+		nlmsg_free(msg);
+		return -ENOMEM;
+	}
+
+	genlmsg_end(msg, hdr);
+	return genlmsg_multicast_netns(&sunrpc_nl_family, cd->net, msg, 0,
+				       SUNRPC_NLGRP_EXPORTD, GFP_KERNEL);
+}
+EXPORT_SYMBOL_GPL(sunrpc_cache_notify);
diff --git a/net/sunrpc/netlink.c b/net/sunrpc/netlink.c
new file mode 100644
index 0000000000000..ce09ecc0faa2c
--- /dev/null
+++ b/net/sunrpc/netlink.c
@@ -0,0 +1,97 @@
+// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause)
+/* Do not edit directly, auto-generated from: */
+/*	Documentation/netlink/specs/sunrpc_cache.yaml */
+/* YNL-GEN kernel source */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include "netlink.h"
+
+#include <uapi/linux/sunrpc_netlink.h>
+
+/* Common nested types */
+const struct nla_policy sunrpc_ip_map_nl_policy[SUNRPC_A_IP_MAP_EXPIRY + 1] = {
+	[SUNRPC_A_IP_MAP_SEQNO] = { .type = NLA_U64, },
+	[SUNRPC_A_IP_MAP_CLASS] = { .type = NLA_NUL_STRING, },
+	[SUNRPC_A_IP_MAP_ADDR] = { .type = NLA_NUL_STRING, },
+	[SUNRPC_A_IP_MAP_DOMAIN] = { .type = NLA_NUL_STRING, },
+	[SUNRPC_A_IP_MAP_NEGATIVE] = { .type = NLA_FLAG, },
+	[SUNRPC_A_IP_MAP_EXPIRY] = { .type = NLA_U64, },
+};
+
+const struct nla_policy sunrpc_unix_gid_nl_policy[SUNRPC_A_UNIX_GID_EXPIRY + 1] = {
+	[SUNRPC_A_UNIX_GID_SEQNO] = { .type = NLA_U64, },
+	[SUNRPC_A_UNIX_GID_UID] = { .type = NLA_U32, },
+	[SUNRPC_A_UNIX_GID_GIDS] = { .type = NLA_U32, },
+	[SUNRPC_A_UNIX_GID_NEGATIVE] = { .type = NLA_FLAG, },
+	[SUNRPC_A_UNIX_GID_EXPIRY] = { .type = NLA_U64, },
+};
+
+/* SUNRPC_CMD_IP_MAP_SET_REQS - do */
+static const struct nla_policy sunrpc_ip_map_set_reqs_nl_policy[SUNRPC_A_IP_MAP_REQS_REQUESTS + 1] = {
+	[SUNRPC_A_IP_MAP_REQS_REQUESTS] = NLA_POLICY_NESTED(sunrpc_ip_map_nl_policy),
+};
+
+/* SUNRPC_CMD_UNIX_GID_SET_REQS - do */
+static const struct nla_policy sunrpc_unix_gid_set_reqs_nl_policy[SUNRPC_A_UNIX_GID_REQS_REQUESTS + 1] = {
+	[SUNRPC_A_UNIX_GID_REQS_REQUESTS] = NLA_POLICY_NESTED(sunrpc_unix_gid_nl_policy),
+};
+
+/* SUNRPC_CMD_CACHE_FLUSH - do */
+static const struct nla_policy sunrpc_cache_flush_nl_policy[SUNRPC_A_CACHE_FLUSH_MASK + 1] = {
+	[SUNRPC_A_CACHE_FLUSH_MASK] = NLA_POLICY_MASK(NLA_U32, 0x3),
+};
+
+/* Ops table for sunrpc */
+static const struct genl_split_ops sunrpc_nl_ops[] = {
+	{
+		.cmd	= SUNRPC_CMD_IP_MAP_GET_REQS,
+		.dumpit	= sunrpc_nl_ip_map_get_reqs_dumpit,
+		.flags	= GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= SUNRPC_CMD_IP_MAP_SET_REQS,
+		.doit		= sunrpc_nl_ip_map_set_reqs_doit,
+		.policy		= sunrpc_ip_map_set_reqs_nl_policy,
+		.maxattr	= SUNRPC_A_IP_MAP_REQS_REQUESTS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd	= SUNRPC_CMD_UNIX_GID_GET_REQS,
+		.dumpit	= sunrpc_nl_unix_gid_get_reqs_dumpit,
+		.flags	= GENL_ADMIN_PERM | GENL_CMD_CAP_DUMP,
+	},
+	{
+		.cmd		= SUNRPC_CMD_UNIX_GID_SET_REQS,
+		.doit		= sunrpc_nl_unix_gid_set_reqs_doit,
+		.policy		= sunrpc_unix_gid_set_reqs_nl_policy,
+		.maxattr	= SUNRPC_A_UNIX_GID_REQS_REQUESTS,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+	{
+		.cmd		= SUNRPC_CMD_CACHE_FLUSH,
+		.doit		= sunrpc_nl_cache_flush_doit,
+		.policy		= sunrpc_cache_flush_nl_policy,
+		.maxattr	= SUNRPC_A_CACHE_FLUSH_MASK,
+		.flags		= GENL_ADMIN_PERM | GENL_CMD_CAP_DO,
+	},
+};
+
+static const struct genl_multicast_group sunrpc_nl_mcgrps[] = {
+	[SUNRPC_NLGRP_NONE] = { "none", },
+	[SUNRPC_NLGRP_EXPORTD] = { "exportd", },
+};
+
+struct genl_family sunrpc_nl_family __ro_after_init = {
+	.name		= SUNRPC_FAMILY_NAME,
+	.version	= SUNRPC_FAMILY_VERSION,
+	.netnsok	= true,
+	.parallel_ops	= true,
+	.module		= THIS_MODULE,
+	.split_ops	= sunrpc_nl_ops,
+	.n_split_ops	= ARRAY_SIZE(sunrpc_nl_ops),
+	.mcgrps		= sunrpc_nl_mcgrps,
+	.n_mcgrps	= ARRAY_SIZE(sunrpc_nl_mcgrps),
+};
diff --git a/net/sunrpc/netlink.h b/net/sunrpc/netlink.h
new file mode 100644
index 0000000000000..2c1012303d48b
--- /dev/null
+++ b/net/sunrpc/netlink.h
@@ -0,0 +1,35 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+/* Do not edit directly, auto-generated from: */
+/*	Documentation/netlink/specs/sunrpc_cache.yaml */
+/* YNL-GEN kernel header */
+/* To regenerate run: tools/net/ynl/ynl-regen.sh */
+
+#ifndef _LINUX_SUNRPC_GEN_H
+#define _LINUX_SUNRPC_GEN_H
+
+#include <net/netlink.h>
+#include <net/genetlink.h>
+
+#include <uapi/linux/sunrpc_netlink.h>
+
+/* Common nested types */
+extern const struct nla_policy sunrpc_ip_map_nl_policy[SUNRPC_A_IP_MAP_EXPIRY + 1];
+extern const struct nla_policy sunrpc_unix_gid_nl_policy[SUNRPC_A_UNIX_GID_EXPIRY + 1];
+
+int sunrpc_nl_ip_map_get_reqs_dumpit(struct sk_buff *skb,
+				     struct netlink_callback *cb);
+int sunrpc_nl_ip_map_set_reqs_doit(struct sk_buff *skb, struct genl_info *info);
+int sunrpc_nl_unix_gid_get_reqs_dumpit(struct sk_buff *skb,
+				       struct netlink_callback *cb);
+int sunrpc_nl_unix_gid_set_reqs_doit(struct sk_buff *skb,
+				     struct genl_info *info);
+int sunrpc_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info);
+
+enum {
+	SUNRPC_NLGRP_NONE,
+	SUNRPC_NLGRP_EXPORTD,
+};
+
+extern struct genl_family sunrpc_nl_family;
+
+#endif /* _LINUX_SUNRPC_GEN_H */
diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c
index bab6cab294052..ab88ce46afb55 100644
--- a/net/sunrpc/sunrpc_syms.c
+++ b/net/sunrpc/sunrpc_syms.c
@@ -23,9 +23,12 @@
 #include <linux/sunrpc/rpc_pipe_fs.h>
 #include <linux/sunrpc/xprtsock.h>
 
+#include <net/genetlink.h>
+
 #include "sunrpc.h"
 #include "sysfs.h"
 #include "netns.h"
+#include "netlink.h"
 
 unsigned int sunrpc_net_id;
 EXPORT_SYMBOL_GPL(sunrpc_net_id);
@@ -108,6 +111,10 @@ init_sunrpc(void)
 	if (err)
 		goto out5;
 
+	err = genl_register_family(&sunrpc_nl_family);
+	if (err)
+		goto out6;
+
 	sunrpc_debugfs_init();
 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
 	rpc_register_sysctl();
@@ -116,6 +123,8 @@ init_sunrpc(void)
 	init_socket_xprt();	/* clnt sock transport */
 	return 0;
 
+out6:
+	rpc_sysfs_exit();
 out5:
 	unregister_rpc_pipefs();
 out4:
@@ -131,6 +140,7 @@ init_sunrpc(void)
 static void __exit
 cleanup_sunrpc(void)
 {
+	genl_unregister_family(&sunrpc_nl_family);
 	rpc_sysfs_exit();
 	rpc_cleanup_clids();
 	xprt_cleanup_ids();
diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
index b16e710926c1e..63d1002e63e74 100644
--- a/net/sunrpc/svc_xprt.c
+++ b/net/sunrpc/svc_xprt.c
@@ -234,7 +234,19 @@ void svc_xprt_received(struct svc_xprt *xprt)
 	svc_xprt_get(xprt);
 	smp_mb__before_atomic();
 	clear_bit(XPT_BUSY, &xprt->xpt_flags);
-	svc_xprt_enqueue(xprt);
+
+	/*
+	 * Skip the enqueue when no actionable flags are set.
+	 * Each producer both sets its flag (XPT_DATA, XPT_CLOSE,
+	 * etc.) and calls svc_xprt_enqueue(); if a set_bit races
+	 * with this check, the producer's own enqueue observes
+	 * !XPT_BUSY and dispatches the transport.
+	 */
+	if (READ_ONCE(xprt->xpt_flags) &
+	    (BIT(XPT_CONN) | BIT(XPT_CLOSE) | BIT(XPT_HANDSHAKE) |
+	     BIT(XPT_DATA) | BIT(XPT_DEFERRED)))
+		svc_xprt_enqueue(xprt);
+
 	svc_xprt_put(xprt);
 }
 EXPORT_SYMBOL_GPL(svc_xprt_received);
@@ -425,13 +437,35 @@ static bool svc_xprt_reserve_slot(struct svc_rqst *rqstp, struct svc_xprt *xprt)
 	return true;
 }
 
+/*
+ * After a caller releases write-space or a request slot,
+ * re-enqueue the transport only when there is pending
+ * work that a thread could act on.  The smp_mb() pairs
+ * with the smp_rmb() in svc_xprt_ready() and orders the
+ * preceding counter update before the flags read so a
+ * concurrent set_bit(XPT_DATA) is visible here.
+ *
+ * When the transport is BUSY, the thread holding it will
+ * call svc_xprt_received() upon completion, which checks
+ * for pending work and re-enqueues as needed.
+ */
+static void svc_xprt_resource_released(struct svc_xprt *xprt)
+{
+	unsigned long xpt_flags;
+
+	smp_mb();
+	xpt_flags = READ_ONCE(xprt->xpt_flags);
+	if (xpt_flags & (BIT(XPT_DATA) | BIT(XPT_DEFERRED)) &&
+	    !(xpt_flags & BIT(XPT_BUSY)))
+		svc_xprt_enqueue(xprt);
+}
+
 static void svc_xprt_release_slot(struct svc_rqst *rqstp)
 {
 	struct svc_xprt	*xprt = rqstp->rq_xprt;
 	if (test_and_clear_bit(RQ_DATA, &rqstp->rq_flags)) {
 		atomic_dec(&xprt->xpt_nr_rqsts);
-		smp_wmb(); /* See smp_rmb() in svc_xprt_ready() */
-		svc_xprt_enqueue(xprt);
+		svc_xprt_resource_released(xprt);
 	}
 }
 
@@ -525,10 +559,10 @@ void svc_reserve(struct svc_rqst *rqstp, int space)
 	space += rqstp->rq_res.head[0].iov_len;
 
 	if (xprt && space < rqstp->rq_reserved) {
-		atomic_sub((rqstp->rq_reserved - space), &xprt->xpt_reserved);
+		atomic_sub((rqstp->rq_reserved - space),
+			   &xprt->xpt_reserved);
 		rqstp->rq_reserved = space;
-		smp_wmb(); /* See smp_rmb() in svc_xprt_ready() */
-		svc_xprt_enqueue(xprt);
+		svc_xprt_resource_released(xprt);
 	}
 }
 EXPORT_SYMBOL_GPL(svc_reserve);
diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c
index 3be69c145d2a8..64a2658faddbe 100644
--- a/net/sunrpc/svcauth_unix.c
+++ b/net/sunrpc/svcauth_unix.c
@@ -17,11 +17,14 @@
 #include <net/ipv6.h>
 #include <linux/kernel.h>
 #include <linux/user_namespace.h>
+#include <net/genetlink.h>
+#include <uapi/linux/sunrpc_netlink.h>
 #include <trace/events/sunrpc.h>
 
 #define RPCDBG_FACILITY	RPCDBG_AUTH
 
 #include "netns.h"
+#include "netlink.h"
 
 /*
  * AUTHUNIX and AUTHNULL credentials are both handled here.
@@ -152,7 +155,7 @@ static struct cache_head *ip_map_alloc(void)
 
 static int ip_map_upcall(struct cache_detail *cd, struct cache_head *h)
 {
-	return sunrpc_cache_pipe_upcall(cd, h);
+	return sunrpc_cache_upcall(cd, h);
 }
 
 static void ip_map_request(struct cache_detail *cd,
@@ -467,7 +470,7 @@ static struct cache_head *unix_gid_alloc(void)
 
 static int unix_gid_upcall(struct cache_detail *cd, struct cache_head *h)
 {
-	return sunrpc_cache_pipe_upcall_timeout(cd, h);
+	return sunrpc_cache_upcall_warn(cd, h);
 }
 
 static void unix_gid_request(struct cache_detail *cd,
@@ -582,12 +585,278 @@ static int unix_gid_show(struct seq_file *m,
 	return 0;
 }
 
+static int unix_gid_notify(struct cache_detail *cd, struct cache_head *h)
+{
+	return sunrpc_cache_notify(cd, h, SUNRPC_CACHE_TYPE_UNIX_GID);
+}
+
+/**
+ * sunrpc_nl_unix_gid_get_reqs_dumpit - dump pending unix_gid requests
+ * @skb: reply buffer
+ * @cb: netlink metadata and command arguments
+ *
+ * Walk the unix_gid cache's pending request list and create a netlink
+ * message with a nested entry for each cache_request, containing the
+ * seqno and uid.
+ *
+ * Uses cb->args[0] as a seqno cursor for dump continuation across
+ * multiple netlink messages.
+ *
+ * Returns the size of the reply or a negative errno.
+ */
+int sunrpc_nl_unix_gid_get_reqs_dumpit(struct sk_buff *skb,
+					struct netlink_callback *cb)
+{
+	struct sunrpc_net *sn;
+	struct cache_detail *cd;
+	struct cache_head **items;
+	u64 *seqnos;
+	int cnt, i, emitted;
+	void *hdr;
+	int ret;
+
+	sn = net_generic(sock_net(skb->sk), sunrpc_net_id);
+
+	cd = sn->unix_gid_cache;
+	if (!cd)
+		return -ENODEV;
+
+	cnt = sunrpc_cache_requests_count(cd);
+	if (!cnt)
+		return 0;
+
+	items = kcalloc(cnt, sizeof(*items), GFP_KERNEL);
+	seqnos = kcalloc(cnt, sizeof(*seqnos), GFP_KERNEL);
+	if (!items || !seqnos) {
+		ret = -ENOMEM;
+		goto out_alloc;
+	}
+
+	cnt = sunrpc_cache_requests_snapshot(cd, items, seqnos, cnt,
+					     cb->args[0]);
+	if (!cnt) {
+		ret = 0;
+		goto out_alloc;
+	}
+
+	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			  cb->nlh->nlmsg_seq, &sunrpc_nl_family,
+			  NLM_F_MULTI, SUNRPC_CMD_UNIX_GID_GET_REQS);
+	if (!hdr) {
+		ret = -ENOBUFS;
+		goto out_put;
+	}
+
+	emitted = 0;
+	for (i = 0; i < cnt; i++) {
+		struct unix_gid *ug;
+		struct nlattr *nest;
+
+		ug = container_of(items[i], struct unix_gid, h);
+
+		nest = nla_nest_start(skb,
+				      SUNRPC_A_UNIX_GID_REQS_REQUESTS);
+		if (!nest)
+			break;
+
+		if (nla_put_u64_64bit(skb, SUNRPC_A_UNIX_GID_SEQNO,
+				      seqnos[i], 0) ||
+		    nla_put_u32(skb, SUNRPC_A_UNIX_GID_UID,
+				from_kuid(&init_user_ns, ug->uid))) {
+			nla_nest_cancel(skb, nest);
+			break;
+		}
+
+		nla_nest_end(skb, nest);
+		cb->args[0] = seqnos[i];
+		emitted++;
+	}
+
+	if (!emitted) {
+		genlmsg_cancel(skb, hdr);
+		ret = -EMSGSIZE;
+		goto out_put;
+	}
+
+	genlmsg_end(skb, hdr);
+	ret = skb->len;
+out_put:
+	for (i = 0; i < cnt; i++)
+		cache_put(items[i], cd);
+out_alloc:
+	kfree(seqnos);
+	kfree(items);
+	return ret;
+}
+
+/**
+ * sunrpc_nl_parse_one_unix_gid - parse one unix_gid entry from netlink
+ * @cd: cache_detail for the unix_gid cache
+ * @attr: nested attribute containing unix_gid fields
+ *
+ * Parses one unix_gid entry from a netlink message and updates the
+ * cache. Mirrors the logic in unix_gid_parse().
+ *
+ * Returns 0 on success or a negative errno.
+ */
+static int sunrpc_nl_parse_one_unix_gid(struct cache_detail *cd,
+					 struct nlattr *attr)
+{
+	struct nlattr *tb[SUNRPC_A_UNIX_GID_EXPIRY + 1];
+	struct unix_gid ug, *ugp;
+	struct timespec64 boot;
+	struct nlattr *gid_attr;
+	int err, rem, gids = 0;
+	kuid_t uid;
+
+	err = nla_parse_nested(tb, SUNRPC_A_UNIX_GID_EXPIRY, attr,
+			       sunrpc_unix_gid_nl_policy, NULL);
+	if (err)
+		return err;
+
+	/* uid (required) */
+	if (!tb[SUNRPC_A_UNIX_GID_UID])
+		return -EINVAL;
+	uid = make_kuid(current_user_ns(),
+			nla_get_u32(tb[SUNRPC_A_UNIX_GID_UID]));
+	ug.uid = uid;
+
+	/* expiry (required, wallclock seconds) */
+	if (!tb[SUNRPC_A_UNIX_GID_EXPIRY])
+		return -EINVAL;
+	getboottime64(&boot);
+	ug.h.flags = 0;
+	ug.h.expiry_time = nla_get_u64(tb[SUNRPC_A_UNIX_GID_EXPIRY]) -
+			   boot.tv_sec;
+
+	if (tb[SUNRPC_A_UNIX_GID_NEGATIVE]) {
+		ug.gi = groups_alloc(0);
+		if (!ug.gi)
+			return -ENOMEM;
+	} else {
+		/* Count gids */
+		nla_for_each_nested_type(gid_attr, SUNRPC_A_UNIX_GID_GIDS,
+					 attr, rem)
+			gids++;
+
+		if (gids > 8192)
+			return -EINVAL;
+
+		ug.gi = groups_alloc(gids);
+		if (!ug.gi)
+			return -ENOMEM;
+
+		gids = 0;
+		nla_for_each_nested_type(gid_attr, SUNRPC_A_UNIX_GID_GIDS,
+					 attr, rem) {
+			kgid_t kgid;
+
+			kgid = make_kgid(current_user_ns(),
+					 nla_get_u32(gid_attr));
+			if (!gid_valid(kgid)) {
+				err = -EINVAL;
+				goto out;
+			}
+			ug.gi->gid[gids++] = kgid;
+		}
+		groups_sort(ug.gi);
+	}
+
+	ugp = unix_gid_lookup(cd, uid);
+	if (ugp) {
+		struct cache_head *ch;
+
+		ch = sunrpc_cache_update(cd, &ug.h, &ugp->h,
+					 unix_gid_hash(uid));
+		if (!ch) {
+			err = -ENOMEM;
+		} else {
+			err = 0;
+			cache_put(ch, cd);
+		}
+	} else {
+		err = -ENOMEM;
+	}
+out:
+	if (ug.gi)
+		put_group_info(ug.gi);
+	return err;
+}
+
+/**
+ * sunrpc_nl_unix_gid_set_reqs_doit - respond to unix_gid requests
+ * @skb: reply buffer
+ * @info: netlink metadata and command arguments
+ *
+ * Parse one or more unix_gid cache responses from userspace and
+ * update the unix_gid cache accordingly.
+ *
+ * Returns 0 on success or a negative errno.
+ */
+int sunrpc_nl_unix_gid_set_reqs_doit(struct sk_buff *skb,
+				     struct genl_info *info)
+{
+	struct sunrpc_net *sn;
+	struct cache_detail *cd;
+	const struct nlattr *attr;
+	int rem, ret = 0;
+
+	sn = net_generic(genl_info_net(info), sunrpc_net_id);
+
+	cd = sn->unix_gid_cache;
+	if (!cd)
+		return -ENODEV;
+
+	nlmsg_for_each_attr_type(attr, SUNRPC_A_UNIX_GID_REQS_REQUESTS,
+				 info->nlhdr, GENL_HDRLEN, rem) {
+		ret = sunrpc_nl_parse_one_unix_gid(cd,
+						   (struct nlattr *)attr);
+		if (ret)
+			break;
+	}
+
+	return ret;
+}
+
+/**
+ * sunrpc_nl_cache_flush_doit - flush sunrpc caches via netlink
+ * @skb: reply buffer
+ * @info: netlink metadata and command arguments
+ *
+ * Flush the ip_map and/or unix_gid caches. If SUNRPC_A_CACHE_FLUSH_MASK
+ * is provided, only flush the caches indicated by the bitmask (bit 1 =
+ * ip_map, bit 2 = unix_gid). If omitted, flush both.
+ *
+ * Return 0 on success or a negative errno.
+ */
+int sunrpc_nl_cache_flush_doit(struct sk_buff *skb, struct genl_info *info)
+{
+	struct sunrpc_net *sn;
+	u32 mask = ~0U;
+
+	sn = net_generic(genl_info_net(info), sunrpc_net_id);
+
+	if (info->attrs[SUNRPC_A_CACHE_FLUSH_MASK])
+		mask = nla_get_u32(info->attrs[SUNRPC_A_CACHE_FLUSH_MASK]);
+
+	if ((mask & SUNRPC_CACHE_TYPE_IP_MAP) &&
+	    sn->ip_map_cache)
+		cache_purge(sn->ip_map_cache);
+
+	if ((mask & SUNRPC_CACHE_TYPE_UNIX_GID) &&
+	    sn->unix_gid_cache)
+		cache_purge(sn->unix_gid_cache);
+
+	return 0;
+}
+
 static const struct cache_detail unix_gid_cache_template = {
 	.owner		= THIS_MODULE,
 	.hash_size	= GID_HASHMAX,
 	.name		= "auth.unix.gid",
 	.cache_put	= unix_gid_put,
 	.cache_upcall	= unix_gid_upcall,
+	.cache_notify	= unix_gid_notify,
 	.cache_request	= unix_gid_request,
 	.cache_parse	= unix_gid_parse,
 	.cache_show	= unix_gid_show,
@@ -1017,12 +1286,255 @@ struct auth_ops svcauth_unix = {
 	.set_client	= svcauth_unix_set_client,
 };
 
+static int ip_map_notify(struct cache_detail *cd, struct cache_head *h)
+{
+	return sunrpc_cache_notify(cd, h, SUNRPC_CACHE_TYPE_IP_MAP);
+}
+
+/**
+ * sunrpc_nl_ip_map_get_reqs_dumpit - dump pending ip_map requests
+ * @skb: reply buffer
+ * @cb: netlink metadata and command arguments
+ *
+ * Walk the ip_map cache's pending request list and create a netlink
+ * message with a nested entry for each cache_request, containing the
+ * seqno, class and addr.
+ *
+ * Uses cb->args[0] as a seqno cursor for dump continuation across
+ * multiple netlink messages.
+ *
+ * Returns the size of the reply or a negative errno.
+ */
+int sunrpc_nl_ip_map_get_reqs_dumpit(struct sk_buff *skb,
+				     struct netlink_callback *cb)
+{
+	struct sunrpc_net *sn;
+	struct cache_detail *cd;
+	struct cache_head **items;
+	u64 *seqnos;
+	int cnt, i, emitted;
+	void *hdr;
+	int ret;
+
+	sn = net_generic(sock_net(skb->sk), sunrpc_net_id);
+
+	cd = sn->ip_map_cache;
+	if (!cd)
+		return -ENODEV;
+
+	cnt = sunrpc_cache_requests_count(cd);
+	if (!cnt)
+		return 0;
+
+	items = kcalloc(cnt, sizeof(*items), GFP_KERNEL);
+	seqnos = kcalloc(cnt, sizeof(*seqnos), GFP_KERNEL);
+	if (!items || !seqnos) {
+		ret = -ENOMEM;
+		goto out_alloc;
+	}
+
+	cnt = sunrpc_cache_requests_snapshot(cd, items, seqnos, cnt,
+					     cb->args[0]);
+	if (!cnt) {
+		ret = 0;
+		goto out_alloc;
+	}
+
+	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid,
+			  cb->nlh->nlmsg_seq, &sunrpc_nl_family,
+			  NLM_F_MULTI, SUNRPC_CMD_IP_MAP_GET_REQS);
+	if (!hdr) {
+		ret = -ENOBUFS;
+		goto out_put;
+	}
+
+	emitted = 0;
+	for (i = 0; i < cnt; i++) {
+		struct ip_map *im;
+		struct nlattr *nest;
+		char text_addr[40];
+
+		im = container_of(items[i], struct ip_map, h);
+
+		if (ipv6_addr_v4mapped(&im->m_addr))
+			snprintf(text_addr, 20, "%pI4",
+				 &im->m_addr.s6_addr32[3]);
+		else
+			snprintf(text_addr, 40, "%pI6", &im->m_addr);
+
+		nest = nla_nest_start(skb, SUNRPC_A_IP_MAP_REQS_REQUESTS);
+		if (!nest)
+			break;
+
+		if (nla_put_u64_64bit(skb, SUNRPC_A_IP_MAP_SEQNO,
+				      seqnos[i], 0) ||
+		    nla_put_string(skb, SUNRPC_A_IP_MAP_CLASS,
+				   im->m_class) ||
+		    nla_put_string(skb, SUNRPC_A_IP_MAP_ADDR, text_addr)) {
+			nla_nest_cancel(skb, nest);
+			break;
+		}
+
+		nla_nest_end(skb, nest);
+		cb->args[0] = seqnos[i];
+		emitted++;
+	}
+
+	if (!emitted) {
+		genlmsg_cancel(skb, hdr);
+		ret = -EMSGSIZE;
+		goto out_put;
+	}
+
+	genlmsg_end(skb, hdr);
+	ret = skb->len;
+out_put:
+	for (i = 0; i < cnt; i++)
+		cache_put(items[i], cd);
+out_alloc:
+	kfree(seqnos);
+	kfree(items);
+	return ret;
+}
+
+/**
+ * sunrpc_nl_parse_one_ip_map - parse one ip_map entry from netlink
+ * @cd: cache_detail for the ip_map cache
+ * @attr: nested attribute containing ip_map fields
+ *
+ * Parses one ip_map entry from a netlink message and updates the
+ * cache. Mirrors the logic in ip_map_parse().
+ *
+ * Returns 0 on success or a negative errno.
+ */
+static int sunrpc_nl_parse_one_ip_map(struct cache_detail *cd,
+				      struct nlattr *attr)
+{
+	struct nlattr *tb[SUNRPC_A_IP_MAP_EXPIRY + 1];
+	union {
+		struct sockaddr		sa;
+		struct sockaddr_in	s4;
+		struct sockaddr_in6	s6;
+	} address;
+	struct sockaddr_in6 sin6;
+	struct ip_map *ipmp;
+	struct auth_domain *dom = NULL;
+	struct unix_domain *udom = NULL;
+	struct timespec64 boot;
+	time64_t expiry;
+	char class[8];
+	int err;
+	int len;
+
+	err = nla_parse_nested(tb, SUNRPC_A_IP_MAP_EXPIRY, attr,
+			       sunrpc_ip_map_nl_policy, NULL);
+	if (err)
+		return err;
+
+	/* class (required) */
+	if (!tb[SUNRPC_A_IP_MAP_CLASS])
+		return -EINVAL;
+	len = nla_len(tb[SUNRPC_A_IP_MAP_CLASS]);
+	if (len <= 0 || len > sizeof(class))
+		return -EINVAL;
+	nla_strscpy(class, tb[SUNRPC_A_IP_MAP_CLASS], sizeof(class));
+
+	/* addr (required) */
+	if (!tb[SUNRPC_A_IP_MAP_ADDR])
+		return -EINVAL;
+	if (rpc_pton(cd->net, nla_data(tb[SUNRPC_A_IP_MAP_ADDR]),
+		     nla_len(tb[SUNRPC_A_IP_MAP_ADDR]) - 1,
+		     &address.sa, sizeof(address)) == 0)
+		return -EINVAL;
+
+	switch (address.sa.sa_family) {
+	case AF_INET:
+		sin6.sin6_family = AF_INET6;
+		ipv6_addr_set_v4mapped(address.s4.sin_addr.s_addr,
+				       &sin6.sin6_addr);
+		break;
+#if IS_ENABLED(CONFIG_IPV6)
+	case AF_INET6:
+		memcpy(&sin6, &address.s6, sizeof(sin6));
+		break;
+#endif
+	default:
+		return -EINVAL;
+	}
+
+	/* expiry (required, wallclock seconds) */
+	if (!tb[SUNRPC_A_IP_MAP_EXPIRY])
+		return -EINVAL;
+	getboottime64(&boot);
+	expiry = nla_get_u64(tb[SUNRPC_A_IP_MAP_EXPIRY]) - boot.tv_sec;
+
+	/* domain name or negative */
+	if (tb[SUNRPC_A_IP_MAP_NEGATIVE]) {
+		udom = NULL;
+	} else if (tb[SUNRPC_A_IP_MAP_DOMAIN]) {
+		dom = unix_domain_find(nla_data(tb[SUNRPC_A_IP_MAP_DOMAIN]));
+		if (!dom)
+			return -ENOENT;
+		udom = container_of(dom, struct unix_domain, h);
+	} else {
+		return -EINVAL;
+	}
+
+	ipmp = __ip_map_lookup(cd, class, &sin6.sin6_addr);
+	if (ipmp)
+		err = __ip_map_update(cd, ipmp, udom, expiry);
+	else
+		err = -ENOMEM;
+
+	if (dom)
+		auth_domain_put(dom);
+
+	cache_flush();
+	return err;
+}
+
+/**
+ * sunrpc_nl_ip_map_set_reqs_doit - respond to ip_map requests
+ * @skb: reply buffer
+ * @info: netlink metadata and command arguments
+ *
+ * Parse one or more ip_map cache responses from userspace and
+ * update the ip_map cache accordingly.
+ *
+ * Returns 0 on success or a negative errno.
+ */
+int sunrpc_nl_ip_map_set_reqs_doit(struct sk_buff *skb,
+				   struct genl_info *info)
+{
+	struct sunrpc_net *sn;
+	struct cache_detail *cd;
+	const struct nlattr *attr;
+	int rem, ret = 0;
+
+	sn = net_generic(genl_info_net(info), sunrpc_net_id);
+
+	cd = sn->ip_map_cache;
+	if (!cd)
+		return -ENODEV;
+
+	nlmsg_for_each_attr_type(attr, SUNRPC_A_IP_MAP_REQS_REQUESTS,
+				 info->nlhdr, GENL_HDRLEN, rem) {
+		ret = sunrpc_nl_parse_one_ip_map(cd,
+						 (struct nlattr *)attr);
+		if (ret)
+			break;
+	}
+
+	return ret;
+}
+
 static const struct cache_detail ip_map_cache_template = {
 	.owner		= THIS_MODULE,
 	.hash_size	= IP_HASHMAX,
 	.name		= "auth.unix.ip",
 	.cache_put	= ip_map_put,
 	.cache_upcall	= ip_map_upcall,
+	.cache_notify	= ip_map_notify,
 	.cache_request	= ip_map_request,
 	.cache_parse	= ip_map_parse,
 	.cache_show	= ip_map_show,
diff --git a/net/tipc/udp_media.c b/net/tipc/udp_media.c
index 2c66b356025a7..988b8a7f953ad 100644
--- a/net/tipc/udp_media.c
+++ b/net/tipc/udp_media.c
@@ -89,14 +89,14 @@ struct udp_replicast {
 /**
  * struct udp_bearer - ip/udp bearer data structure
  * @bearer:	associated generic tipc bearer
- * @ubsock:	bearer associated socket
+ * @sk:		bearer associated socket
  * @ifindex:	local address scope
  * @work:	used to schedule deferred work on a bearer
  * @rcast:	associated udp_replicast container
  */
 struct udp_bearer {
 	struct tipc_bearer __rcu *bearer;
-	struct socket *ubsock;
+	struct sock *sk;
 	u32 ifindex;
 	struct work_struct work;
 	struct udp_replicast rcast;
@@ -194,7 +194,7 @@ static int tipc_udp_xmit(struct net *net, struct sk_buff *skb,
 		}
 
 		ttl = ip4_dst_hoplimit(&rt->dst);
-		udp_tunnel_xmit_skb(rt, ub->ubsock->sk, skb, src->ipv4.s_addr,
+		udp_tunnel_xmit_skb(rt, ub->sk, skb, src->ipv4.s_addr,
 				    dst->ipv4.s_addr, 0, ttl, 0, src->port,
 				    dst->port, false, true, 0);
 #if IS_ENABLED(CONFIG_IPV6)
@@ -206,7 +206,7 @@ static int tipc_udp_xmit(struct net *net, struct sk_buff *skb,
 				.saddr = src->ipv6,
 				.flowi6_proto = IPPROTO_UDP
 			};
-			ndst = ip6_dst_lookup_flow(net, ub->ubsock->sk,
+			ndst = ip6_dst_lookup_flow(net, ub->sk,
 						   &fl6, NULL);
 			if (IS_ERR(ndst)) {
 				err = PTR_ERR(ndst);
@@ -215,7 +215,7 @@ static int tipc_udp_xmit(struct net *net, struct sk_buff *skb,
 			dst_cache_set_ip6(cache, ndst, &fl6.saddr);
 		}
 		ttl = ip6_dst_hoplimit(ndst);
-		udp_tunnel6_xmit_skb(ndst, ub->ubsock->sk, skb, NULL,
+		udp_tunnel6_xmit_skb(ndst, ub->sk, skb, NULL,
 				     &src->ipv6, &dst->ipv6, 0, ttl, 0,
 				     src->port, dst->port, false, 0);
 #endif
@@ -405,9 +405,9 @@ static int tipc_udp_recv(struct sock *sk, struct sk_buff *skb)
 
 static int enable_mcast(struct udp_bearer *ub, struct udp_media_addr *remote)
 {
-	int err = 0;
+	struct sock *sk = ub->sk;
 	struct ip_mreqn mreqn;
-	struct sock *sk = ub->ubsock->sk;
+	int err = 0;
 
 	if (ntohs(remote->proto) == ETH_P_IP) {
 		mreqn.imr_multiaddr = remote->ipv4;
@@ -670,6 +670,7 @@ static int tipc_udp_enable(struct net *net, struct tipc_bearer *b,
 	struct nlattr *opts[TIPC_NLA_UDP_MAX + 1];
 	u8 node_id[NODE_ID_LEN] = {0,};
 	struct net_device *dev;
+	struct socket *sock;
 	int rmcast = 0;
 
 	ub = kzalloc_obj(*ub, GFP_ATOMIC);
@@ -764,14 +765,16 @@ static int tipc_udp_enable(struct net *net, struct tipc_bearer *b,
 		goto err;
 	}
 	udp_conf.local_udp_port = local.port;
-	err = udp_sock_create(net, &udp_conf, &ub->ubsock);
+	err = udp_sock_create(net, &udp_conf, &sock);
 	if (err)
 		goto err;
+
+	ub->sk = sock->sk;
 	tuncfg.sk_user_data = ub;
 	tuncfg.encap_type = 1;
 	tuncfg.encap_rcv = tipc_udp_recv;
 	tuncfg.encap_destroy = NULL;
-	setup_udp_tunnel_sock(net, ub->ubsock, &tuncfg);
+	setup_udp_tunnel_sock(net, ub->sk, &tuncfg);
 
 	err = dst_cache_init(&ub->rcast.dst_cache, GFP_ATOMIC);
 	if (err)
@@ -793,7 +796,8 @@ static int tipc_udp_enable(struct net *net, struct tipc_bearer *b,
 
 free:
 	dst_cache_destroy(&ub->rcast.dst_cache);
-	udp_tunnel_sock_release(ub->ubsock);
+	udp_tunnel_sock_release(ub->sk);
+	synchronize_rcu();
 err:
 	kfree(ub);
 	return err;
@@ -812,10 +816,10 @@ static void cleanup_bearer(struct work_struct *work)
 		kfree_rcu(rcast, rcu);
 	}
 
-	tn = tipc_net(sock_net(ub->ubsock->sk));
+	tn = tipc_net(sock_net(ub->sk));
 
 	dst_cache_destroy(&ub->rcast.dst_cache);
-	udp_tunnel_sock_release(ub->ubsock);
+	udp_tunnel_sock_release(ub->sk);
 
 	/* Note: could use a call_rcu() to avoid another synchronize_net() */
 	synchronize_net();
@@ -833,11 +837,11 @@ static void tipc_udp_disable(struct tipc_bearer *b)
 		pr_err("UDP bearer instance not found\n");
 		return;
 	}
-	sock_set_flag(ub->ubsock->sk, SOCK_DEAD);
+	sock_set_flag(ub->sk, SOCK_DEAD);
 	RCU_INIT_POINTER(ub->bearer, NULL);
 
 	/* sock_release need to be done outside of rtnl lock */
-	atomic_inc(&tipc_net(sock_net(ub->ubsock->sk))->wq_count);
+	atomic_inc(&tipc_net(sock_net(ub->sk))->wq_count);
 	INIT_WORK(&ub->work, cleanup_bearer);
 	schedule_work(&ub->work);
 }
diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c
index 99c8eff9783e6..741aef09bfd35 100644
--- a/net/tls/tls_device.c
+++ b/net/tls/tls_device.c
@@ -1387,16 +1387,15 @@ static int tls_dev_event(struct notifier_block *this, unsigned long event,
 	case NETDEV_FEAT_CHANGE:
 		if (netif_is_bond_master(dev))
 			return NOTIFY_DONE;
+		if  (!dev->tlsdev_ops ||
+		     !dev->tlsdev_ops->tls_dev_add ||
+		     !dev->tlsdev_ops->tls_dev_del)
+			return NOTIFY_BAD;
 		if ((dev->features & NETIF_F_HW_TLS_RX) &&
 		    !dev->tlsdev_ops->tls_dev_resync)
 			return NOTIFY_BAD;
 
-		if  (dev->tlsdev_ops &&
-		     dev->tlsdev_ops->tls_dev_add &&
-		     dev->tlsdev_ops->tls_dev_del)
-			return NOTIFY_DONE;
-		else
-			return NOTIFY_BAD;
+		return NOTIFY_DONE;
 	case NETDEV_DOWN:
 		return tls_device_down(dev);
 	}
diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c
index 3bfdaf5e64f50..964ebc268ee46 100644
--- a/net/tls/tls_sw.c
+++ b/net/tls/tls_sw.c
@@ -1366,9 +1366,14 @@ void tls_sw_splice_eof(struct socket *sock)
 	mutex_unlock(&tls_ctx->tx_lock);
 }
 
+/* When has_copied is true the caller has already moved bytes to
+ * userspace. Report sk_err but leave it set so the next read
+ * surfaces it instead of a spurious EOF, otherwise sk_err is
+ * consumed via sock_error().
+ */
 static int
 tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock,
-		bool released)
+		bool released, bool has_copied)
 {
 	struct tls_context *tls_ctx = tls_get_ctx(sk);
 	struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
@@ -1386,8 +1391,11 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock,
 		if (!sk_psock_queue_empty(psock))
 			return 0;
 
-		if (sk->sk_err)
+		if (sk->sk_err) {
+			if (has_copied)
+				return -READ_ONCE(sk->sk_err);
 			return sock_error(sk);
+		}
 
 		if (ret < 0)
 			return ret;
@@ -1423,7 +1431,7 @@ tls_rx_rec_wait(struct sock *sk, struct sk_psock *psock, bool nonblock,
 	}
 
 	if (unlikely(!tls_strp_msg_load(&ctx->strp, released)))
-		return tls_rx_rec_wait(sk, psock, nonblock, false);
+		return tls_rx_rec_wait(sk, psock, nonblock, false, has_copied);
 
 	return 1;
 }
@@ -2110,7 +2118,7 @@ int tls_sw_recvmsg(struct sock *sk,
 		int to_decrypt, chunk;
 
 		err = tls_rx_rec_wait(sk, psock, flags & MSG_DONTWAIT,
-				      released);
+				      released, !!(decrypted + copied));
 		if (err <= 0) {
 			if (psock) {
 				chunk = sk_msg_recvmsg(sk, psock, msg, len,
@@ -2297,7 +2305,7 @@ ssize_t tls_sw_splice_read(struct socket *sock,  loff_t *ppos,
 		struct tls_decrypt_arg darg;
 
 		err = tls_rx_rec_wait(sk, NULL, flags & SPLICE_F_NONBLOCK,
-				      true);
+				      true, false);
 		if (err <= 0)
 			goto splice_read_end;
 
@@ -2383,7 +2391,7 @@ int tls_sw_read_sock(struct sock *sk, read_descriptor_t *desc,
 		} else {
 			struct tls_decrypt_arg darg;
 
-			err = tls_rx_rec_wait(sk, NULL, true, released);
+			err = tls_rx_rec_wait(sk, NULL, true, released, !!copied);
 			if (err <= 0)
 				goto read_sock_end;
 
diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index 1cbf36ea043bd..dc71ed79be4aa 100644
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -2711,8 +2711,7 @@ static int unix_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
  *	Sleep until more data has arrived. But check for races..
  */
 static long unix_stream_data_wait(struct sock *sk, long timeo,
-				  struct sk_buff *last, unsigned int last_len,
-				  bool freezable)
+				  struct sk_buff *last, bool freezable)
 {
 	unsigned int state = TASK_INTERRUPTIBLE | freezable * TASK_FREEZABLE;
 	struct sk_buff *tail;
@@ -2725,7 +2724,6 @@ static long unix_stream_data_wait(struct sock *sk, long timeo,
 
 		tail = skb_peek_tail(&sk->sk_receive_queue);
 		if (tail != last ||
-		    (tail && tail->len != last_len) ||
 		    sk->sk_err ||
 		    (sk->sk_shutdown & RCV_SHUTDOWN) ||
 		    signal_pending(current) ||
@@ -2921,7 +2919,6 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
 	int flags = state->flags;
 	bool check_creds = false;
 	struct scm_cookie scm;
-	unsigned int last_len;
 	struct unix_sock *u;
 	int copied = 0;
 	int err = 0;
@@ -2967,7 +2964,6 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
 			goto unlock;
 		}
 		last = skb = skb_peek(&sk->sk_receive_queue);
-		last_len = last ? last->len : 0;
 
 again:
 #if IS_ENABLED(CONFIG_AF_UNIX_OOB)
@@ -3001,8 +2997,7 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
 
 			mutex_unlock(&u->iolock);
 
-			timeo = unix_stream_data_wait(sk, timeo, last,
-						      last_len, freezable);
+			timeo = unix_stream_data_wait(sk, timeo, last, freezable);
 
 			if (signal_pending(current)) {
 				err = sock_intr_errno(timeo);
@@ -3019,7 +3014,6 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
 		while (skip >= unix_skb_len(skb)) {
 			skip -= unix_skb_len(skb);
 			last = skb;
-			last_len = skb->len;
 			skb = skb_peek_next(skb, &sk->sk_receive_queue);
 			if (!skb)
 				goto again;
@@ -3094,7 +3088,6 @@ static int unix_stream_read_generic(struct unix_stream_read_state *state,
 
 			skip = 0;
 			last = skb;
-			last_len = skb->len;
 			unix_state_lock(sk);
 			skb = skb_peek_next(skb, &sk->sk_receive_queue);
 			if (skb)
diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
index 44037b066a5ff..d4a97eeb596e6 100644
--- a/net/vmw_vsock/af_vsock.c
+++ b/net/vmw_vsock/af_vsock.c
@@ -155,6 +155,7 @@
 #include <linux/random.h>
 #include <linux/skbuff.h>
 #include <linux/smp.h>
+#include <linux/uio.h>
 #include <linux/socket.h>
 #include <linux/stddef.h>
 #include <linux/sysctl.h>
@@ -2091,8 +2092,7 @@ static int vsock_connectible_setsockopt(struct socket *sock,
 
 static int vsock_connectible_getsockopt(struct socket *sock,
 					int level, int optname,
-					char __user *optval,
-					int __user *optlen)
+					sockopt_t *opt)
 {
 	struct sock *sk = sock->sk;
 	struct vsock_sock *vsk = vsock_sk(sk);
@@ -2110,8 +2110,7 @@ static int vsock_connectible_getsockopt(struct socket *sock,
 	if (level != AF_VSOCK)
 		return -ENOPROTOOPT;
 
-	if (get_user(len, optlen))
-		return -EFAULT;
+	len = opt->optlen;
 
 	memset(&v, 0, sizeof(v));
 
@@ -2142,11 +2141,10 @@ static int vsock_connectible_getsockopt(struct socket *sock,
 		return -EINVAL;
 	if (len > lv)
 		len = lv;
-	if (copy_to_user(optval, &v, len))
+	if (copy_to_iter(&v, len, &opt->iter_out) != len)
 		return -EFAULT;
 
-	if (put_user(len, optlen))
-		return -EFAULT;
+	opt->optlen = len;
 
 	return 0;
 }
@@ -2631,7 +2629,7 @@ static const struct proto_ops vsock_stream_ops = {
 	.listen = vsock_listen,
 	.shutdown = vsock_shutdown,
 	.setsockopt = vsock_connectible_setsockopt,
-	.getsockopt = vsock_connectible_getsockopt,
+	.getsockopt_iter = vsock_connectible_getsockopt,
 	.sendmsg = vsock_connectible_sendmsg,
 	.recvmsg = vsock_connectible_recvmsg,
 	.mmap = sock_no_mmap,
@@ -2653,7 +2651,7 @@ static const struct proto_ops vsock_seqpacket_ops = {
 	.listen = vsock_listen,
 	.shutdown = vsock_shutdown,
 	.setsockopt = vsock_connectible_setsockopt,
-	.getsockopt = vsock_connectible_getsockopt,
+	.getsockopt_iter = vsock_connectible_getsockopt,
 	.sendmsg = vsock_connectible_sendmsg,
 	.recvmsg = vsock_connectible_recvmsg,
 	.mmap = sock_no_mmap,
diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
index 989cc252d3d34..1e3409d28164e 100644
--- a/net/vmw_vsock/virtio_transport_common.c
+++ b/net/vmw_vsock/virtio_transport_common.c
@@ -70,34 +70,6 @@ static bool virtio_transport_can_zcopy(const struct virtio_transport *t_ops,
 	return true;
 }
 
-static int virtio_transport_init_zcopy_skb(struct vsock_sock *vsk,
-					   struct sk_buff *skb,
-					   struct msghdr *msg,
-					   size_t pkt_len,
-					   bool zerocopy)
-{
-	struct ubuf_info *uarg;
-
-	if (msg->msg_ubuf) {
-		uarg = msg->msg_ubuf;
-		net_zcopy_get(uarg);
-	} else {
-		struct ubuf_info_msgzc *uarg_zc;
-
-		uarg = msg_zerocopy_realloc(sk_vsock(vsk),
-					    pkt_len, NULL, false);
-		if (!uarg)
-			return -1;
-
-		uarg_zc = uarg_to_msgzc(uarg);
-		uarg_zc->zerocopy = zerocopy ? 1 : 0;
-	}
-
-	skb_zcopy_init(skb, uarg);
-
-	return 0;
-}
-
 static int virtio_transport_fill_skb(struct sk_buff *skb,
 				     struct virtio_vsock_pkt_info *info,
 				     size_t len,
@@ -317,8 +289,10 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
 	u32 src_cid, src_port, dst_cid, dst_port;
 	const struct virtio_transport *t_ops;
 	struct virtio_vsock_sock *vvs;
+	struct ubuf_info *uarg = NULL;
 	u32 pkt_len = info->pkt_len;
 	bool can_zcopy = false;
+	bool have_uref = false;
 	u32 rest_len;
 	int ret;
 
@@ -360,6 +334,25 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
 		if (can_zcopy)
 			max_skb_len = min_t(u32, VIRTIO_VSOCK_MAX_PKT_BUF_SIZE,
 					    (MAX_SKB_FRAGS * PAGE_SIZE));
+
+		if (info->msg->msg_flags & MSG_ZEROCOPY &&
+		    info->op == VIRTIO_VSOCK_OP_RW) {
+			uarg = info->msg->msg_ubuf;
+
+			if (!uarg) {
+				uarg = msg_zerocopy_realloc(sk_vsock(vsk),
+							    pkt_len, NULL, false);
+				if (!uarg) {
+					virtio_transport_put_credit(vvs, pkt_len);
+					return -ENOMEM;
+				}
+
+				if (!can_zcopy)
+					uarg_to_msgzc(uarg)->zerocopy = 0;
+
+				have_uref = true;
+			}
+		}
 	}
 
 	rest_len = pkt_len;
@@ -378,27 +371,7 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
 			break;
 		}
 
-		/* We process buffer part by part, allocating skb on
-		 * each iteration. If this is last skb for this buffer
-		 * and MSG_ZEROCOPY mode is in use - we must allocate
-		 * completion for the current syscall.
-		 *
-		 * Pass pkt_len because msg iter is already consumed
-		 * by virtio_transport_fill_skb(), so iter->count
-		 * can not be used for RLIMIT_MEMLOCK pinned-pages
-		 * accounting done by msg_zerocopy_realloc().
-		 */
-		if (info->msg && info->msg->msg_flags & MSG_ZEROCOPY &&
-		    skb_len == rest_len && info->op == VIRTIO_VSOCK_OP_RW) {
-			if (virtio_transport_init_zcopy_skb(vsk, skb,
-							    info->msg,
-							    pkt_len,
-							    can_zcopy)) {
-				kfree_skb(skb);
-				ret = -ENOMEM;
-				break;
-			}
-		}
+		skb_zcopy_set(skb, uarg, NULL);
 
 		virtio_transport_inc_tx_pkt(vvs, skb);
 
@@ -422,6 +395,18 @@ static int virtio_transport_send_pkt_info(struct vsock_sock *vsk,
 
 	virtio_transport_put_credit(vvs, rest_len);
 
+	/* msg_zerocopy_realloc() initializes the ubuf_info refcnt to 1.
+	 * skb_zcopy_set() increases it for each skb, so we can drop that
+	 * initial reference to keep it balanced.
+	 */
+	if (have_uref) {
+		if (rest_len == pkt_len)
+			/* No data sent, abort the notification. */
+			net_zcopy_put_abort(uarg, true);
+		else
+			net_zcopy_put(uarg);
+	}
+
 	/* Return number of bytes, if any data has been sent. */
 	if (rest_len != pkt_len)
 		ret = pkt_len - rest_len;
diff --git a/net/vmw_vsock/vmci_transport.c b/net/vmw_vsock/vmci_transport.c
index 4296ca1183f11..d2579380f51e5 100644
--- a/net/vmw_vsock/vmci_transport.c
+++ b/net/vmw_vsock/vmci_transport.c
@@ -1164,7 +1164,7 @@ vmci_transport_recv_connecting_server(struct sock *listener,
 		/* Close and cleanup the connection. */
 		vmci_transport_send_reset(pending, pkt);
 		skerr = EPROTO;
-		err = pkt->type == VMCI_TRANSPORT_PACKET_TYPE_RST ? 0 : -EINVAL;
+		err = -EINVAL;
 		goto destroy;
 	}
 
diff --git a/net/wireless/chan.c b/net/wireless/chan.c
index 8b94c0de80ade..f0811efb5d0fb 100644
--- a/net/wireless/chan.c
+++ b/net/wireless/chan.c
@@ -138,9 +138,10 @@ static const struct cfg80211_per_bw_puncturing_values per_bw_puncturing[] = {
 	CFG80211_PER_BW_VALID_PUNCTURING_VALUES(320)
 };
 
-static bool valid_puncturing_bitmap(const struct cfg80211_chan_def *chandef)
+static bool valid_puncturing_bitmap(const struct cfg80211_chan_def *chandef,
+				    u32 primary_center, u32 punctured)
 {
-	u32 idx, i, start_freq, primary_center = chandef->chan->center_freq;
+	u32 idx, i, start_freq;
 
 	switch (chandef->width) {
 	case NL80211_CHAN_WIDTH_80:
@@ -156,18 +157,18 @@ static bool valid_puncturing_bitmap(const struct cfg80211_chan_def *chandef)
 		start_freq = chandef->center_freq1 - 160;
 		break;
 	default:
-		return chandef->punctured == 0;
+		return punctured == 0;
 	}
 
-	if (!chandef->punctured)
+	if (!punctured)
 		return true;
 
 	/* check if primary channel is punctured */
-	if (chandef->punctured & (u16)BIT((primary_center - start_freq) / 20))
+	if (punctured & (u16)BIT((primary_center - start_freq) / 20))
 		return false;
 
 	for (i = 0; i < per_bw_puncturing[idx].len; i++) {
-		if (per_bw_puncturing[idx].valid_values[i] == chandef->punctured)
+		if (per_bw_puncturing[idx].valid_values[i] == punctured)
 			return true;
 	}
 
@@ -458,6 +459,19 @@ bool cfg80211_chandef_valid(const struct cfg80211_chan_def *chandef)
 	if (!cfg80211_chandef_valid_control_freq(chandef, control_freq))
 		return false;
 
+	if (chandef->npca_chan) {
+		switch (chandef->width) {
+		case NL80211_CHAN_WIDTH_80:
+		case NL80211_CHAN_WIDTH_160:
+		case NL80211_CHAN_WIDTH_320:
+			break;
+		default:
+			return false;
+		}
+	} else if (chandef->npca_punctured) {
+		return false;
+	}
+
 	if (!cfg80211_valid_center_freq(chandef->center_freq1, chandef->width))
 		return false;
 
@@ -477,7 +491,8 @@ bool cfg80211_chandef_valid(const struct cfg80211_chan_def *chandef)
 	if (!cfg80211_chandef_is_s1g(chandef) && chandef->s1g_primary_2mhz)
 		return false;
 
-	return valid_puncturing_bitmap(chandef);
+	return valid_puncturing_bitmap(chandef, control_freq,
+				       chandef->punctured);
 }
 EXPORT_SYMBOL(cfg80211_chandef_valid);
 
@@ -520,6 +535,220 @@ int cfg80211_chandef_primary(const struct cfg80211_chan_def *c,
 }
 EXPORT_SYMBOL(cfg80211_chandef_primary);
 
+bool cfg80211_chandef_npca_valid(struct wiphy *wiphy,
+				 const struct cfg80211_chan_def *chandef,
+				 const struct ieee80211_uhr_npca_info *npca)
+{
+	struct cfg80211_chan_def tmp = *chandef;
+	bool pri_upper, npca_upper;
+	u32 cf1;
+
+	if (chandef->npca_chan || chandef->npca_punctured)
+		return false;
+
+	if (!npca)
+		return true;
+
+	if (cfg80211_chandef_add_npca(wiphy, &tmp, npca))
+		return false;
+
+	if (!cfg80211_chandef_valid_control_freq(&tmp,
+						 tmp.npca_chan->center_freq))
+		return false;
+
+	cf1 = tmp.center_freq1;
+	pri_upper = tmp.chan->center_freq > cf1;
+	npca_upper = tmp.npca_chan->center_freq > cf1;
+
+	if (pri_upper == npca_upper)
+		return false;
+
+	if (!valid_puncturing_bitmap(&tmp,
+				     tmp.npca_chan->center_freq,
+				     tmp.npca_punctured) ||
+	    (tmp.punctured & tmp.npca_punctured) != tmp.punctured)
+		return false;
+
+	return true;
+}
+EXPORT_SYMBOL(cfg80211_chandef_npca_valid);
+
+int cfg80211_chandef_add_npca(struct wiphy *wiphy,
+			      struct cfg80211_chan_def *chandef,
+			      const struct ieee80211_uhr_npca_info *npca)
+{
+	struct cfg80211_chan_def new_chandef = *chandef;
+	u32 width, npca_freq;
+	u8 offs;
+
+	if (chandef->npca_chan || chandef->npca_punctured)
+		return -EINVAL;
+
+	if (WARN_ON(!cfg80211_chandef_valid(chandef)))
+		return -EINVAL;
+
+	if (!npca)
+		return 0;
+
+	switch (chandef->width) {
+	case NL80211_CHAN_WIDTH_80:
+	case NL80211_CHAN_WIDTH_160:
+	case NL80211_CHAN_WIDTH_320:
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	offs = le32_get_bits(npca->params,
+			     IEEE80211_UHR_NPCA_PARAMS_PRIMARY_CHAN_OFFS);
+
+	width = cfg80211_chandef_get_width(chandef);
+	npca_freq = chandef->center_freq1 - width / 2 + 10 + 20 * offs;
+	new_chandef.npca_chan = ieee80211_get_channel(wiphy, npca_freq);
+	if (!new_chandef.npca_chan)
+		return -EINVAL;
+
+	if (npca->params & cpu_to_le32(IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES))
+		new_chandef.npca_punctured = le16_to_cpu(npca->dis_subch_bmap[0]);
+
+	if (!cfg80211_chandef_valid(&new_chandef))
+		return -EINVAL;
+
+	*chandef = new_chandef;
+	return 0;
+}
+EXPORT_SYMBOL(cfg80211_chandef_add_npca);
+
+int cfg80211_chandef_add_dbe(struct cfg80211_chan_def *chandef,
+			     const struct ieee80211_uhr_dbe_info *dbe)
+{
+	struct cfg80211_chan_def new_chandef = *chandef;
+	u16 starting_freq, bw_mhz, start_old, start_new;
+	u8 bw, punct_shift;
+	int offset, index;
+
+	if (!dbe)
+		return 0;
+
+	if (!cfg80211_chandef_valid(chandef))
+		return -EINVAL;
+
+	if (chandef->width == NL80211_CHAN_WIDTH_20_NOHT)
+		return -EINVAL;
+
+	bw = u8_get_bits(dbe->params, IEEE80211_UHR_DBE_OPER_BANDWIDTH);
+
+	switch (chandef->chan->band) {
+	case NL80211_BAND_5GHZ:
+		if (bw > IEEE80211_UHR_DBE_OPER_BW_160)
+			return -EINVAL;
+		if (chandef->chan->center_freq < 5745)
+			starting_freq = 5180; /* channel 36 */
+		else
+			starting_freq = 5745; /* channel 149 */
+		break;
+	case NL80211_BAND_6GHZ:
+		starting_freq = 5955; /* channel 1 center */
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (bw) {
+	case IEEE80211_UHR_DBE_OPER_BW_320_2:
+	case IEEE80211_UHR_DBE_OPER_BW_320_1:
+		if (chandef->width == NL80211_CHAN_WIDTH_160)
+			break;
+		fallthrough;
+	case IEEE80211_UHR_DBE_OPER_BW_160:
+		if (chandef->width == NL80211_CHAN_WIDTH_80)
+			break;
+		fallthrough;
+	case IEEE80211_UHR_DBE_OPER_BW_80:
+		if (chandef->width == NL80211_CHAN_WIDTH_40)
+			break;
+		fallthrough;
+	case IEEE80211_UHR_DBE_OPER_BW_40:
+		if (chandef->width == NL80211_CHAN_WIDTH_20)
+			break;
+		fallthrough;
+	default:
+		return -EINVAL;
+	}
+
+	switch (bw) {
+	case IEEE80211_UHR_DBE_OPER_BW_320_2:
+		/* 320-2 starts shifted by 160 */
+		starting_freq += 160;
+		fallthrough;
+	case IEEE80211_UHR_DBE_OPER_BW_320_1:
+		new_chandef.width = NL80211_CHAN_WIDTH_320;
+		bw_mhz = 320;
+		break;
+	case IEEE80211_UHR_DBE_OPER_BW_160:
+		new_chandef.width = NL80211_CHAN_WIDTH_160;
+		bw_mhz = 160;
+		break;
+	case IEEE80211_UHR_DBE_OPER_BW_80:
+		new_chandef.width = NL80211_CHAN_WIDTH_80;
+		bw_mhz = 80;
+		break;
+	case IEEE80211_UHR_DBE_OPER_BW_40:
+		new_chandef.width = NL80211_CHAN_WIDTH_40;
+		bw_mhz = 40;
+		break;
+	}
+
+	/* this should only happen for 320-2 and misconfigured AP */
+	if (chandef->chan->center_freq < starting_freq)
+		return -EINVAL;
+
+	offset = chandef->chan->center_freq - starting_freq;
+	index = offset / bw_mhz;
+	start_new = starting_freq - 10 + index * bw_mhz;
+	new_chandef.center_freq1 = start_new + bw_mhz / 2;
+
+	start_old = chandef->center_freq1 -
+		    cfg80211_chandef_get_width(chandef) / 2;
+
+	/*
+	 * If the DBE channel extends downward below the lower
+	 * edge of the BSS channel, we need to shift puncturing
+	 * bitmaps up to adjust for that.
+	 */
+	if (start_new < start_old)
+		punct_shift = (start_old - start_new) / 20;
+	else
+		punct_shift = 0;
+
+	new_chandef.punctured <<= punct_shift;
+	new_chandef.npca_punctured <<= punct_shift;
+
+	if (dbe->params & IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES) {
+		u16 punct_mask = ((1 << (bw_mhz / 40)) - 1) << punct_shift;
+		u16 punctured = le16_to_cpu(dbe->dis_subch_bmap[0]);
+
+		if ((punctured & punct_mask) != (new_chandef.punctured & punct_mask))
+			return -EINVAL;
+
+		new_chandef.punctured = punctured;
+	}
+
+	if (!cfg80211_chandef_valid(&new_chandef))
+		return -EINVAL;
+
+	/*
+	 * If e.g. a 40 MHz BSS channel (erroneously) occupies the center of the
+	 * DBE 80 MHz channel, they would be incompatible; check and reject.
+	 */
+	if (!cfg80211_chandef_compatible(&new_chandef, chandef))
+		return -EINVAL;
+
+	*chandef = new_chandef;
+	return 0;
+}
+EXPORT_SYMBOL(cfg80211_chandef_add_dbe);
+
 static const struct cfg80211_chan_def *
 check_chandef_primary_compat(const struct cfg80211_chan_def *c1,
 			     const struct cfg80211_chan_def *c2,
@@ -564,6 +793,15 @@ _cfg80211_chandef_compatible(const struct cfg80211_chan_def *c1,
 	if (c1->width == c2->width)
 		return NULL;
 
+	/*
+	 * We need NPCA to be compatible for some scenarios such as
+	 * multiple APs, but in this case userspace should configure
+	 * identical chandefs including NPCA, even if perhaps one of
+	 * the AP interfaces doesn't even advertise it.
+	 */
+	if (c1->npca_chan || c2->npca_chan)
+		return NULL;
+
 	/*
 	 * can't be compatible if one of them is 5/10 MHz or S1G
 	 * but they don't have the same width.
@@ -817,6 +1055,7 @@ int cfg80211_chandef_dfs_required(struct wiphy *wiphy,
 	case NL80211_IFTYPE_AP_VLAN:
 	case NL80211_IFTYPE_P2P_DEVICE:
 	case NL80211_IFTYPE_NAN_DATA:
+	case NL80211_IFTYPE_PD:
 		break;
 	case NL80211_IFTYPE_WDS:
 	case NL80211_IFTYPE_UNSPECIFIED:
@@ -941,6 +1180,7 @@ bool cfg80211_beaconing_iface_active(struct wireless_dev *wdev)
 	/* Can NAN type be considered as beaconing interface? */
 	case NL80211_IFTYPE_NAN:
 	case NL80211_IFTYPE_NAN_DATA:
+	case NL80211_IFTYPE_PD:
 		break;
 	case NL80211_IFTYPE_UNSPECIFIED:
 	case NL80211_IFTYPE_WDS:
diff --git a/net/wireless/core.c b/net/wireless/core.c
index 6783e0672dcb7..62ab5e4639be1 100644
--- a/net/wireless/core.c
+++ b/net/wireless/core.c
@@ -322,6 +322,28 @@ int cfg80211_nan_set_local_schedule(struct cfg80211_registered_device *rdev,
 	return 0;
 }
 
+void cfg80211_stop_pd(struct cfg80211_registered_device *rdev,
+		      struct wireless_dev *wdev)
+{
+	lockdep_assert_held(&rdev->wiphy.mtx);
+
+	if (WARN_ON(wdev->iftype != NL80211_IFTYPE_PD))
+		return;
+
+	if (!rdev->ops->stop_pd)
+		return;
+
+	if (!wdev_running(wdev))
+		return;
+
+	cfg80211_pmsr_wdev_down(wdev);
+
+	rdev_stop_pd(rdev, wdev);
+	wdev->is_running = false;
+
+	rdev->opencount--;
+}
+
 void cfg80211_shutdown_all_interfaces(struct wiphy *wiphy)
 {
 	struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
@@ -351,6 +373,9 @@ void cfg80211_shutdown_all_interfaces(struct wiphy *wiphy)
 		case NL80211_IFTYPE_NAN:
 			cfg80211_stop_nan(rdev, wdev);
 			break;
+		case NL80211_IFTYPE_PD:
+			cfg80211_stop_pd(rdev, wdev);
+			break;
 		default:
 			break;
 		}
@@ -804,6 +829,24 @@ static int wiphy_verify_combinations(struct wiphy *wiphy)
 	return ret;
 }
 
+static bool wiphy_cipher_suites_valid(const struct wiphy *wiphy)
+{
+	int i, j;
+
+	if (wiphy->n_cipher_suites && !wiphy->cipher_suites)
+		return false;
+
+	for (i = 0; i < wiphy->n_cipher_suites; i++) {
+		for (j = 0; j < i; j++) {
+			if (wiphy->cipher_suites[i] ==
+			    wiphy->cipher_suites[j])
+				return false;
+		}
+	}
+
+	return true;
+}
+
 int wiphy_register(struct wiphy *wiphy)
 {
 	struct cfg80211_registered_device *rdev = wiphy_to_rdev(wiphy);
@@ -828,6 +871,9 @@ int wiphy_register(struct wiphy *wiphy)
 		    (!rdev->ops->tdls_channel_switch ||
 		     !rdev->ops->tdls_cancel_channel_switch)))
 		return -EINVAL;
+	if (WARN_ON((wiphy->interface_modes & BIT(NL80211_IFTYPE_PD)) &&
+		    (!rdev->ops->start_pd || !rdev->ops->stop_pd)))
+		return -EINVAL;
 
 	if (WARN_ON((wiphy->interface_modes & BIT(NL80211_IFTYPE_NAN)) &&
 		    (!rdev->ops->start_nan || !rdev->ops->stop_nan ||
@@ -836,7 +882,7 @@ int wiphy_register(struct wiphy *wiphy)
 		return -EINVAL;
 
 	if (WARN_ON((wiphy->interface_modes & BIT(NL80211_IFTYPE_NAN_DATA)) &&
-		    !wiphy->nan_capa.phy.ht.ht_supported))
+		    (!wiphy->nan_capa.phy.ht.ht_supported || wiphy->n_radio > 1)))
 		return -EINVAL;
 
 	if (WARN_ON(wiphy->interface_modes & BIT(NL80211_IFTYPE_WDS)))
@@ -940,6 +986,9 @@ int wiphy_register(struct wiphy *wiphy)
 	if (res)
 		return res;
 
+	if (!wiphy_cipher_suites_valid(wiphy))
+		return -EINVAL;
+
 	/* sanity check supported bands/channels */
 	for (band = 0; band < NUM_NL80211_BANDS; band++) {
 		const struct ieee80211_sband_iftype_data *iftd;
@@ -1387,6 +1436,9 @@ static void _cfg80211_unregister_wdev(struct wireless_dev *wdev,
 	case NL80211_IFTYPE_NAN:
 		cfg80211_stop_nan(rdev, wdev);
 		break;
+	case NL80211_IFTYPE_PD:
+		cfg80211_stop_pd(rdev, wdev);
+		break;
 	default:
 		break;
 	}
@@ -1495,6 +1547,9 @@ void cfg80211_leave_locked(struct cfg80211_registered_device *rdev,
 	case NL80211_IFTYPE_NAN:
 		cfg80211_stop_nan(rdev, wdev);
 		break;
+	case NL80211_IFTYPE_PD:
+		cfg80211_stop_pd(rdev, wdev);
+		break;
 	case NL80211_IFTYPE_AP_VLAN:
 	case NL80211_IFTYPE_MONITOR:
 	case NL80211_IFTYPE_NAN_DATA:
diff --git a/net/wireless/core.h b/net/wireless/core.h
index ae2d56d3ad908..df47ed6208a50 100644
--- a/net/wireless/core.h
+++ b/net/wireless/core.h
@@ -3,7 +3,7 @@
  * Wireless configuration interface internals.
  *
  * Copyright 2006-2010	Johannes Berg <johannes@sipsolutions.net>
- * Copyright (C) 2018-2025 Intel Corporation
+ * Copyright (C) 2018-2026 Intel Corporation
  */
 #ifndef __NET_WIRELESS_CORE_H
 #define __NET_WIRELESS_CORE_H
@@ -446,6 +446,7 @@ bool cfg80211_supported_cipher_suite(struct wiphy *wiphy, u32 cipher);
 bool cfg80211_valid_key_idx(struct cfg80211_registered_device *rdev,
 			    int key_idx, bool pairwise);
 int cfg80211_validate_key_settings(struct cfg80211_registered_device *rdev,
+				   struct wireless_dev *wdev,
 				   struct key_params *params, int key_idx,
 				   bool pairwise, const u8 *mac_addr);
 void __cfg80211_scan_done(struct wiphy *wiphy, struct wiphy_work *wk);
@@ -556,6 +557,8 @@ void cfg80211_stop_p2p_device(struct cfg80211_registered_device *rdev,
 
 void cfg80211_stop_nan(struct cfg80211_registered_device *rdev,
 		       struct wireless_dev *wdev);
+void cfg80211_stop_pd(struct cfg80211_registered_device *rdev,
+		      struct wireless_dev *wdev);
 
 int cfg80211_nan_set_local_schedule(struct cfg80211_registered_device *rdev,
 				    struct wireless_dev *wdev,
diff --git a/net/wireless/mlme.c b/net/wireless/mlme.c
index bd72317c4964e..2a2c173058bac 100644
--- a/net/wireless/mlme.c
+++ b/net/wireless/mlme.c
@@ -38,6 +38,7 @@ void cfg80211_rx_assoc_resp(struct net_device *dev,
 					u.assoc_resp.variable),
 		.status = le16_to_cpu(mgmt->u.assoc_resp.status_code),
 		.ap_mld_addr = data->ap_mld_addr,
+		.assoc_encrypted = data->assoc_encrypted,
 	};
 	unsigned int link_id;
 
@@ -360,17 +361,18 @@ cfg80211_mlme_check_mlo_compat(const struct ieee80211_multi_link_elem *mle_a,
 	 * reserved when included in a unicast Probe Response frame and may
 	 * also change when the AP adds/removes links. The BTM MLD
 	 * Recommendation For Multiple APs Support subfield is reserved when
-	 * transmitted by an AP. All other bits are currently reserved.
-	 * See IEEE P802.11be/D7.0, Table 9-417o.
+	 * transmitted by an AP.
 	 */
 	if ((ieee80211_mle_get_ext_mld_capa_op((const u8 *)mle_a) &
 	     (IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_PARAM_UPDATE |
 	      IEEE80211_EHT_ML_EXT_MLD_CAPA_NSTR_UPDATE |
-	      IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK)) !=
+	      IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK |
+	      IEEE80211_UHR_ML_EXT_MLD_CAPA_ML_PM)) !=
 	    (ieee80211_mle_get_ext_mld_capa_op((const u8 *)mle_b) &
 	     (IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_PARAM_UPDATE |
 	      IEEE80211_EHT_ML_EXT_MLD_CAPA_NSTR_UPDATE |
-	      IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK))) {
+	      IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK |
+	      IEEE80211_UHR_ML_EXT_MLD_CAPA_ML_PM))) {
 		NL_SET_ERR_MSG(extack,
 			       "extended link MLD capabilities/ops mismatch");
 		return -EINVAL;
@@ -944,6 +946,7 @@ int cfg80211_mlme_mgmt_tx(struct cfg80211_registered_device *rdev,
 			 * fall through, P2P device only supports
 			 * public action frames
 			 */
+		case NL80211_IFTYPE_PD:
 		default:
 			err = -EOPNOTSUPP;
 			break;
diff --git a/net/wireless/nl80211.c b/net/wireless/nl80211.c
index 7db9cd4338011..c272a2fbad03f 100644
--- a/net/wireless/nl80211.c
+++ b/net/wireless/nl80211.c
@@ -476,6 +476,18 @@ nl80211_pmsr_ftm_req_attr_policy[NL80211_PMSR_FTM_REQ_ATTR_MAX + 1] = {
 	[NL80211_PMSR_FTM_REQ_ATTR_LMR_FEEDBACK] = { .type = NLA_FLAG },
 	[NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR] = { .type = NLA_U8 },
 	[NL80211_PMSR_FTM_REQ_ATTR_RSTA] = { .type = NLA_FLAG },
+	[NL80211_PMSR_FTM_REQ_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS] = {
+		.type = NLA_U32
+	},
+	[NL80211_PMSR_FTM_REQ_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS] = {
+		.type = NLA_U32
+	},
+	[NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_REQ_ATTR_AW_DURATION] = NLA_POLICY_MAX(NLA_U32, 255),
+	[NL80211_PMSR_FTM_REQ_ATTR_NUM_MEASUREMENTS] = { .type = NLA_U32 },
+	[NL80211_PMSR_FTM_REQ_ATTR_INGRESS] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_REQ_ATTR_EGRESS] = { .type = NLA_U64 },
+	[NL80211_PMSR_FTM_REQ_ATTR_PD_SUPPRESS_RESULTS] = { .type = NLA_FLAG },
 };
 
 static const struct nla_policy
@@ -498,6 +510,8 @@ nl80211_pmsr_peer_attr_policy[NL80211_PMSR_PEER_ATTR_MAX + 1] = {
 	[NL80211_PMSR_PEER_ATTR_REQ] =
 		NLA_POLICY_NESTED(nl80211_pmsr_req_attr_policy),
 	[NL80211_PMSR_PEER_ATTR_RESP] = { .type = NLA_REJECT },
+	[NL80211_PMSR_PEER_ATTR_REQ_TYPE] =
+		NLA_POLICY_MAX(NLA_U32, NL80211_PMSR_FTM_REQ_TYPE_MAX),
 };
 
 static const struct nla_policy
@@ -1047,7 +1061,7 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = {
 				       NL80211_MAX_SUPP_SELECTORS),
 	[NL80211_ATTR_MLO_RECONF_REM_LINKS] = { .type = NLA_U16 },
 	[NL80211_ATTR_EPCS] = { .type = NLA_FLAG },
-	[NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS] = { .type = NLA_U16 },
+	[NL80211_ATTR_EXT_MLD_CAPA_AND_OPS] = { .type = NLA_U16 },
 	[NL80211_ATTR_WIPHY_RADIO_INDEX] = { .type = NLA_U8 },
 	[NL80211_ATTR_S1G_LONG_BEACON_PERIOD] = NLA_POLICY_MIN(NLA_U8, 2),
 	[NL80211_ATTR_S1G_SHORT_BEACON] =
@@ -1076,6 +1090,9 @@ static const struct nla_policy nl80211_policy[NUM_NL80211_ATTR] = {
 	[NL80211_ATTR_NAN_MAX_CHAN_SWITCH_TIME] = { .type = NLA_U16 },
 	[NL80211_ATTR_NAN_PEER_MAPS] =
 		NLA_POLICY_NESTED_ARRAY(nl80211_nan_peer_map_policy),
+	[NL80211_ATTR_NPCA_PRIMARY_FREQ] = { .type = NLA_U32 },
+	[NL80211_ATTR_NPCA_PUNCT_BITMAP] =
+		NLA_POLICY_FULL_RANGE(NLA_U32, &nl80211_punct_bitmap_range),
 };
 
 /* policy for the key attributes */
@@ -1089,6 +1106,10 @@ static const struct nla_policy nl80211_key_policy[NL80211_KEY_MAX + 1] = {
 	[NL80211_KEY_TYPE] = NLA_POLICY_MAX(NLA_U32, NUM_NL80211_KEYTYPES - 1),
 	[NL80211_KEY_DEFAULT_TYPES] = { .type = NLA_NESTED },
 	[NL80211_KEY_MODE] = NLA_POLICY_RANGE(NLA_U8, 0, NL80211_KEY_SET_TX),
+	[NL80211_KEY_LTF_SEED] = {
+		.type = NLA_BINARY,
+		.len = WLAN_MAX_SECURE_LTF_KEYSEED_LEN,
+	},
 };
 
 /* policy for the key default flags */
@@ -1632,6 +1653,11 @@ static int nl80211_parse_key_new(struct genl_info *info, struct nlattr *key,
 	if (tb[NL80211_KEY_MODE])
 		k->p.mode = nla_get_u8(tb[NL80211_KEY_MODE]);
 
+	if (tb[NL80211_KEY_LTF_SEED]) {
+		k->p.ltf_keyseed = nla_data(tb[NL80211_KEY_LTF_SEED]);
+		k->p.ltf_keyseed_len = nla_len(tb[NL80211_KEY_LTF_SEED]);
+	}
+
 	return 0;
 }
 
@@ -1745,6 +1771,7 @@ static int nl80211_parse_key(struct genl_info *info, struct key_parse *k)
 
 static struct cfg80211_cached_keys *
 nl80211_parse_connkeys(struct cfg80211_registered_device *rdev,
+		       struct wireless_dev *wdev,
 		       struct genl_info *info, bool *no_ht)
 {
 	struct nlattr *keys = info->attrs[NL80211_ATTR_KEYS];
@@ -1794,7 +1821,7 @@ nl80211_parse_connkeys(struct cfg80211_registered_device *rdev,
 				goto error;
 		} else if (parse.defmgmt)
 			goto error;
-		err = cfg80211_validate_key_settings(rdev, &parse.p,
+		err = cfg80211_validate_key_settings(rdev, wdev, &parse.p,
 						     parse.idx, false, NULL);
 		if (err)
 			goto error;
@@ -1853,6 +1880,11 @@ static int nl80211_key_allowed(struct wireless_dev *wdev)
 					    NL80211_EXT_FEATURE_SECURE_NAN))
 			return 0;
 		return -EINVAL;
+	case NL80211_IFTYPE_PD:
+		if (wiphy_ext_feature_isset(wdev->wiphy,
+					    NL80211_EXT_FEATURE_SECURE_RTT))
+			return 0;
+		return -EINVAL;
 	case NL80211_IFTYPE_UNSPECIFIED:
 	case NL80211_IFTYPE_OCB:
 	case NL80211_IFTYPE_MONITOR:
@@ -2509,10 +2541,117 @@ nl80211_send_pmsr_ftm_capa(const struct cfg80211_pmsr_capabilities *cap,
 	    nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_TOTAL_LTF_RX,
 			cap->ftm.max_total_ltf_rx))
 		return -ENOBUFS;
-	if (cap->ftm.support_rsta &&
-	    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_RSTA_SUPPORT))
+
+	if (cap->ftm.ista.support_ntb || cap->ftm.ista.support_tb ||
+	    cap->ftm.ista.support_edca) {
+		struct nlattr *ista_caps;
+
+		ista_caps = nla_nest_start_noflag(msg,
+						  NL80211_PMSR_FTM_CAPA_ATTR_ISTA_CAPS);
+		if (!ista_caps)
+			return -ENOBUFS;
+		if (cap->ftm.ista.support_ntb &&
+		    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_NTB))
+			return -ENOBUFS;
+		if (cap->ftm.ista.support_tb &&
+		    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_TB))
+			return -ENOBUFS;
+		if (cap->ftm.ista.support_edca &&
+		    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_EDCA))
+			return -ENOBUFS;
+		if (cap->ftm.ista.max_peers &&
+		    nla_put_u32(msg, NL80211_PMSR_ATTR_MAX_PEER_ISTA_ROLE,
+				cap->ftm.ista.max_peers))
+			return -ENOBUFS;
+		nla_nest_end(msg, ista_caps);
+	}
+
+	if (cap->ftm.rsta.support_ntb || cap->ftm.rsta.support_tb ||
+	    cap->ftm.rsta.support_edca) {
+		struct nlattr *rsta_caps;
+
+		/*
+		 * Set the generic RSTA_SUPPORT flag if any of the specific
+		 * ranging modes is supported to maintain the backward
+		 * compatibility.
+		 */
+		if (nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_RSTA_SUPPORT))
+			return -ENOBUFS;
+
+		rsta_caps = nla_nest_start_noflag(msg,
+						  NL80211_PMSR_FTM_CAPA_ATTR_RSTA_CAPS);
+		if (!rsta_caps)
+			return -ENOBUFS;
+		if (cap->ftm.rsta.support_ntb &&
+		    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_NTB))
+			return -ENOBUFS;
+		if (cap->ftm.rsta.support_tb &&
+		    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_TB))
+			return -ENOBUFS;
+		if (cap->ftm.rsta.support_edca &&
+		    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_SUPPORT_EDCA))
+			return -ENOBUFS;
+		if (cap->ftm.rsta.max_peers &&
+		    nla_put_u32(msg, NL80211_PMSR_ATTR_MAX_PEER_RSTA_ROLE,
+				cap->ftm.rsta.max_peers))
+			return -ENOBUFS;
+		nla_nest_end(msg, rsta_caps);
+	}
+
+	if (cap->ftm.max_no_of_tx_antennas &&
+	    nla_put_u8(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_TX_ANTENNAS,
+		       cap->ftm.max_no_of_tx_antennas))
+		return -ENOBUFS;
+
+	if (cap->ftm.max_no_of_rx_antennas &&
+	    nla_put_u8(msg, NL80211_PMSR_FTM_CAPA_ATTR_MAX_NUM_RX_ANTENNAS,
+		       cap->ftm.max_no_of_rx_antennas))
+		return -ENOBUFS;
+
+	if (cap->ftm.min_allowed_ranging_interval_edca &&
+	    nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_EDCA,
+			cap->ftm.min_allowed_ranging_interval_edca))
 		return -ENOBUFS;
 
+	if (cap->ftm.min_allowed_ranging_interval_ntb &&
+	    nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_MIN_INTERVAL_NTB,
+			cap->ftm.min_allowed_ranging_interval_ntb))
+		return -ENOBUFS;
+
+	if (cap->ftm.type.infra_support || cap->ftm.type.pd_support) {
+		struct nlattr *pd_caps;
+
+		pd_caps = nla_nest_start_noflag(msg,
+						NL80211_PMSR_FTM_CAPA_ATTR_TYPE_CAPS);
+		if (!pd_caps)
+			return -ENOBUFS;
+
+		if (cap->ftm.type.infra_support &&
+		    nla_put_flag(msg, NL80211_PMSR_FTM_TYPE_CAPA_ATTR_INFRA_SUPPORT))
+			return -ENOBUFS;
+
+		if (cap->ftm.type.pd_support &&
+		    nla_put_flag(msg, NL80211_PMSR_FTM_TYPE_CAPA_ATTR_PD_SUPPORT))
+			return -ENOBUFS;
+
+		nla_nest_end(msg, pd_caps);
+	}
+
+	if (cap->ftm.concurrent_ista_rsta_support &&
+	    nla_put_flag(msg, NL80211_PMSR_FTM_CAPA_ATTR_CONCURRENT_ISTA_RSTA_SUPPORT))
+		return -ENOBUFS;
+
+	if (cap->ftm.type.pd_support) {
+		if (cap->ftm.pd_preambles &&
+		    nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_PD_PREAMBLES,
+				cap->ftm.pd_preambles))
+			return -ENOBUFS;
+		if (cap->ftm.pd_bandwidths &&
+		    nla_put_u32(msg, NL80211_PMSR_FTM_CAPA_ATTR_PD_BANDWIDTHS,
+				cap->ftm.pd_bandwidths))
+			return -ENOBUFS;
+	}
+
 	nla_nest_end(msg, ftm);
 	return 0;
 }
@@ -3472,6 +3611,12 @@ static int nl80211_send_wiphy(struct cfg80211_registered_device *rdev,
 						 NL80211_ATTR_MLD_CAPA_AND_OPS,
 						 capab->mld_capa_and_ops)))
 					goto nla_put_failure;
+				if (rdev->wiphy.flags & WIPHY_FLAG_SUPPORTS_MLO &&
+				    capab->ext_mld_capa_and_ops &&
+				    nla_put_u16(msg,
+						NL80211_ATTR_EXT_MLD_CAPA_AND_OPS,
+						capab->ext_mld_capa_and_ops))
+					goto nla_put_failure;
 
 				nla_nest_end(msg, nested_ext_capab);
 				if (state->split)
@@ -3783,7 +3928,8 @@ static bool nl80211_can_set_dev_channel(struct wireless_dev *wdev)
 static int _nl80211_parse_chandef(struct cfg80211_registered_device *rdev,
 				  struct netlink_ext_ack *extack,
 				  struct nlattr **attrs, bool monitor,
-				  struct cfg80211_chan_def *chandef)
+				  struct cfg80211_chan_def *chandef,
+				  bool permit_npca)
 {
 	u32 control_freq;
 
@@ -3897,6 +4043,34 @@ static int _nl80211_parse_chandef(struct cfg80211_registered_device *rdev,
 		}
 	}
 
+	if (attrs[NL80211_ATTR_NPCA_PRIMARY_FREQ]) {
+		if (!permit_npca) {
+			NL_SET_ERR_MSG_ATTR(extack,
+					    attrs[NL80211_ATTR_NPCA_PRIMARY_FREQ],
+					    "NPCA not supported");
+			return -EINVAL;
+		}
+
+		chandef->npca_chan =
+			ieee80211_get_channel(&rdev->wiphy,
+					      nla_get_u32(attrs[NL80211_ATTR_NPCA_PRIMARY_FREQ]));
+		if (!chandef->npca_chan) {
+			NL_SET_ERR_MSG_ATTR(extack,
+					    attrs[NL80211_ATTR_NPCA_PRIMARY_FREQ],
+					    "invalid NPCA primary channel");
+			return -EINVAL;
+		}
+
+		chandef->npca_punctured =
+			nla_get_u32_default(attrs[NL80211_ATTR_NPCA_PUNCT_BITMAP],
+					    chandef->punctured);
+	} else if (attrs[NL80211_ATTR_NPCA_PUNCT_BITMAP]) {
+		NL_SET_ERR_MSG_ATTR(extack,
+				    attrs[NL80211_ATTR_NPCA_PUNCT_BITMAP],
+				    "NPCA puncturing only valid with NPCA");
+		return -EINVAL;
+	}
+
 	if (!cfg80211_chandef_valid(chandef)) {
 		NL_SET_ERR_MSG_ATTR(extack, attrs[NL80211_ATTR_WIPHY_FREQ],
 				    "invalid channel definition");
@@ -3924,9 +4098,11 @@ static int _nl80211_parse_chandef(struct cfg80211_registered_device *rdev,
 int nl80211_parse_chandef(struct cfg80211_registered_device *rdev,
 			  struct netlink_ext_ack *extack,
 			  struct nlattr **attrs,
-			  struct cfg80211_chan_def *chandef)
+			  struct cfg80211_chan_def *chandef,
+			  bool permit_npca)
 {
-	return _nl80211_parse_chandef(rdev, extack, attrs, false, chandef);
+	return _nl80211_parse_chandef(rdev, extack, attrs, false, chandef,
+				      permit_npca);
 }
 
 static int __nl80211_set_channel(struct cfg80211_registered_device *rdev,
@@ -3939,6 +4115,7 @@ static int __nl80211_set_channel(struct cfg80211_registered_device *rdev,
 	enum nl80211_iftype iftype = NL80211_IFTYPE_MONITOR;
 	struct wireless_dev *wdev = NULL;
 	int link_id = _link_id;
+	bool permit_npca;
 
 	if (dev)
 		wdev = dev->ieee80211_ptr;
@@ -3953,9 +4130,13 @@ static int __nl80211_set_channel(struct cfg80211_registered_device *rdev,
 		link_id = 0;
 	}
 
+	/* allow parsing it - will check on start_ap or below */
+	permit_npca = iftype == NL80211_IFTYPE_AP ||
+		      iftype == NL80211_IFTYPE_P2P_GO;
+
 	result = _nl80211_parse_chandef(rdev, info->extack, info->attrs,
 					iftype == NL80211_IFTYPE_MONITOR,
-					&chandef);
+					&chandef, permit_npca);
 	if (result)
 		return result;
 
@@ -3974,6 +4155,9 @@ static int __nl80211_set_channel(struct cfg80211_registered_device *rdev,
 				return -EBUSY;
 
 			/* Only allow dynamic channel width changes */
+			cur_chan = wdev->links[link_id].ap.chandef.npca_chan;
+			if (chandef.npca_chan != cur_chan)
+				return -EBUSY;
 			cur_chan = wdev->links[link_id].ap.chandef.chan;
 			if (chandef.chan != cur_chan)
 				return -EBUSY;
@@ -4458,6 +4642,15 @@ int nl80211_send_chandef(struct sk_buff *msg, const struct cfg80211_chan_def *ch
 	    nla_put_flag(msg, NL80211_ATTR_S1G_PRIMARY_2MHZ))
 		return -ENOBUFS;
 
+	if (chandef->npca_chan &&
+	    nla_put_u32(msg, NL80211_ATTR_NPCA_PRIMARY_FREQ,
+			chandef->npca_chan->center_freq))
+		return -ENOBUFS;
+	if (chandef->npca_punctured &&
+	    nla_put_u32(msg, NL80211_ATTR_NPCA_PUNCT_BITMAP,
+			chandef->npca_punctured))
+		return -ENOBUFS;
+
 	return 0;
 }
 EXPORT_SYMBOL(nl80211_send_chandef);
@@ -4949,6 +5142,7 @@ static int _nl80211_new_interface(struct sk_buff *skb, struct genl_info *info)
 		return -EOPNOTSUPP;
 
 	if ((type == NL80211_IFTYPE_P2P_DEVICE || type == NL80211_IFTYPE_NAN ||
+	     type == NL80211_IFTYPE_PD ||
 	     rdev->wiphy.features & NL80211_FEATURE_MAC_ON_CREATE) &&
 	    info->attrs[NL80211_ATTR_MAC]) {
 		nla_memcpy(params.macaddr, info->attrs[NL80211_ATTR_MAC],
@@ -5005,8 +5199,9 @@ static int _nl80211_new_interface(struct sk_buff *skb, struct genl_info *info)
 		break;
 	case NL80211_IFTYPE_NAN:
 	case NL80211_IFTYPE_P2P_DEVICE:
+	case NL80211_IFTYPE_PD:
 		/*
-		 * P2P Device and NAN do not have a netdev, so don't go
+		 * P2P Device, NAN and PD do not have a netdev, so don't go
 		 * through the netdev notifier and must be added here
 		 */
 		cfg80211_init_wdev(wdev);
@@ -5112,7 +5307,7 @@ static int nl80211_validate_key_link_id(struct genl_info *info,
 	if (wdev->valid_links) {
 		if (link_id == -1) {
 			GENL_SET_ERR_MSG(info,
-					 "link ID must for MLO group key");
+					 "link ID must be set for MLO group key");
 			return -EINVAL;
 		}
 		if (!(wdev->valid_links & BIT(link_id))) {
@@ -5419,7 +5614,7 @@ static int nl80211_new_key(struct sk_buff *skb, struct genl_info *info)
 	if (!rdev->ops->add_key)
 		return -EOPNOTSUPP;
 
-	if (cfg80211_validate_key_settings(rdev, &key.p, key.idx,
+	if (cfg80211_validate_key_settings(rdev, wdev, &key.p, key.idx,
 					   key.type == NL80211_KEYTYPE_PAIRWISE,
 					   mac_addr)) {
 		GENL_SET_ERR_MSG(info, "key setting validation failed");
@@ -6406,9 +6601,104 @@ static int nl80211_parse_he_bss_color(struct nlattr *attrs,
 	return 0;
 }
 
+static void nl80211_check_ap_rate_selectors(struct cfg80211_beacon_data *bcn,
+					    const struct element *rates)
+{
+	int i;
+
+	if (!rates)
+		return;
+
+	for (i = 0; i < rates->datalen; i++) {
+		if (rates->data[i] == BSS_MEMBERSHIP_SELECTOR_HT_PHY)
+			bcn->ht_required = true;
+		if (rates->data[i] == BSS_MEMBERSHIP_SELECTOR_VHT_PHY)
+			bcn->vht_required = true;
+	}
+}
+
+/*
+ * Since the nl80211 API didn't include, from the beginning, attributes about
+ * HT/VHT/... operation, we parse them out of the elements and check for
+ * validity for use by drivers/mac80211.
+ */
+static int nl80211_calculate_ap_operation(struct nlattr *attrs[],
+					  struct cfg80211_beacon_data *bcn,
+					  struct netlink_ext_ack *extack)
+{
+	size_t ies_len = bcn->tail_len;
+	const u8 *ies = bcn->tail;
+	const struct element *rates;
+	const struct element *op;
+
+	rates = cfg80211_find_elem(WLAN_EID_SUPP_RATES, ies, ies_len);
+	nl80211_check_ap_rate_selectors(bcn, rates);
+
+	rates = cfg80211_find_elem(WLAN_EID_EXT_SUPP_RATES, ies, ies_len);
+	nl80211_check_ap_rate_selectors(bcn, rates);
+
+	op = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ies, ies_len);
+	if (op) {
+		if (op->datalen < sizeof(*bcn->he_oper) + 1) {
+			NL_SET_ERR_MSG(extack, "bad HE operation in beacon");
+			return -EINVAL;
+		}
+		bcn->he_oper = (void *)(op->data + 1);
+		/* takes extension ID into account */
+		if (op->datalen < ieee80211_he_oper_size((void *)bcn->he_oper)) {
+			NL_SET_ERR_MSG(extack, "bad HE operation in beacon");
+			return -EINVAL;
+		}
+	}
+
+	op = cfg80211_find_elem(WLAN_EID_HT_OPERATION, ies, ies_len);
+	if (op) {
+		if (op->datalen < sizeof(*bcn->ht_oper)) {
+			NL_SET_ERR_MSG(extack, "bad HT operation in beacon");
+			return -EINVAL;
+		}
+		bcn->ht_oper = (void *)op->data;
+	}
+
+	op = cfg80211_find_elem(WLAN_EID_VHT_OPERATION, ies, ies_len);
+	if (op) {
+		if (op->datalen < sizeof(*bcn->vht_oper)) {
+			NL_SET_ERR_MSG(extack, "bad VHT operation in beacon");
+			return -EINVAL;
+		}
+		bcn->vht_oper = (void *)op->data;
+	}
+
+	op = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_OPERATION, ies, ies_len);
+	if (op) {
+		if (!ieee80211_eht_oper_size_ok(op->data + 1,
+						op->datalen - 1)) {
+			NL_SET_ERR_MSG(extack, "bad EHT operation in beacon");
+			return -EINVAL;
+		}
+		bcn->eht_oper = (void *)(op->data + 1);
+	}
+
+	op = cfg80211_find_ext_elem(WLAN_EID_EXT_UHR_OPER, ies, ies_len);
+	if (op) {
+		/* need full UHR operation separately */
+		if (!attrs[NL80211_ATTR_UHR_OPERATION]) {
+			NL_SET_ERR_MSG(extack, "missing UHR operation");
+			return -EINVAL;
+		}
+		bcn->uhr_oper = nla_data(attrs[NL80211_ATTR_UHR_OPERATION]);
+	} else if (attrs[NL80211_ATTR_UHR_OPERATION]) {
+		NL_SET_ERR_MSG(extack, "unexpected UHR operation");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev,
 				struct nlattr *attrs[],
 				struct cfg80211_beacon_data *bcn,
+				struct ieee80211_channel *chan,
 				struct netlink_ext_ack *extack)
 {
 	bool haveinfo = false;
@@ -6523,6 +6813,19 @@ static int nl80211_parse_beacon(struct cfg80211_registered_device *rdev,
 		}
 	}
 
+	err = nl80211_calculate_ap_operation(attrs, bcn, extack);
+	if (err)
+		return err;
+
+	if (bcn->he_oper && (chan->flags & IEEE80211_CHAN_NO_HE))
+		return -EOPNOTSUPP;
+
+	if (bcn->eht_oper && (chan->flags & IEEE80211_CHAN_NO_EHT))
+		return -EOPNOTSUPP;
+
+	if (bcn->uhr_oper && (chan->flags & IEEE80211_CHAN_NO_UHR))
+		return -EOPNOTSUPP;
+
 	return 0;
 }
 
@@ -6640,75 +6943,54 @@ nl80211_parse_unsol_bcast_probe_resp(struct cfg80211_registered_device *rdev,
 	return 0;
 }
 
-static void nl80211_check_ap_rate_selectors(struct cfg80211_ap_settings *params,
-					    const struct element *rates)
-{
-	int i;
-
-	if (!rates)
-		return;
-
-	for (i = 0; i < rates->datalen; i++) {
-		if (rates->data[i] == BSS_MEMBERSHIP_SELECTOR_HT_PHY)
-			params->ht_required = true;
-		if (rates->data[i] == BSS_MEMBERSHIP_SELECTOR_VHT_PHY)
-			params->vht_required = true;
-		if (rates->data[i] == BSS_MEMBERSHIP_SELECTOR_HE_PHY)
-			params->he_required = true;
-		if (rates->data[i] == BSS_MEMBERSHIP_SELECTOR_SAE_H2E)
-			params->sae_h2e_required = true;
-	}
-}
-
 /*
  * Since the nl80211 API didn't include, from the beginning, attributes about
- * HT/VHT requirements/capabilities, we parse them out of the IEs for the
- * benefit of drivers that rebuild IEs in the firmware.
+ * HT/VHT/... capabilities, we parse them out of the elements and check for
+ * validity for use by drivers/mac80211.
  */
-static int nl80211_calculate_ap_params(struct cfg80211_ap_settings *params)
+static int nl80211_calculate_ap_capabilities(struct genl_info *info,
+					     struct cfg80211_ap_settings *params)
 {
-	const struct cfg80211_beacon_data *bcn = &params->beacon;
-	size_t ies_len = bcn->tail_len;
-	const u8 *ies = bcn->tail;
-	const struct element *rates;
+	size_t ies_len = params->beacon.tail_len;
+	const u8 *ies = params->beacon.tail;
 	const struct element *cap;
 
-	rates = cfg80211_find_elem(WLAN_EID_SUPP_RATES, ies, ies_len);
-	nl80211_check_ap_rate_selectors(params, rates);
-
-	rates = cfg80211_find_elem(WLAN_EID_EXT_SUPP_RATES, ies, ies_len);
-	nl80211_check_ap_rate_selectors(params, rates);
-
 	cap = cfg80211_find_elem(WLAN_EID_HT_CAPABILITY, ies, ies_len);
-	if (cap && cap->datalen >= sizeof(*params->ht_cap))
+	if (cap) {
+		if (cap->datalen < sizeof(*params->ht_cap)) {
+			GENL_SET_ERR_MSG(info, "bad HT capability in beacon");
+			return -EINVAL;
+		}
 		params->ht_cap = (void *)cap->data;
+	}
+
 	cap = cfg80211_find_elem(WLAN_EID_VHT_CAPABILITY, ies, ies_len);
-	if (cap && cap->datalen >= sizeof(*params->vht_cap))
+	if (cap) {
+		if (cap->datalen < sizeof(*params->vht_cap)) {
+			GENL_SET_ERR_MSG(info, "bad VHT capability in beacon");
+			return -EINVAL;
+		}
 		params->vht_cap = (void *)cap->data;
+	}
+
 	cap = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_CAPABILITY, ies, ies_len);
-	if (cap && cap->datalen >= sizeof(*params->he_cap) + 1)
+	if (cap) {
+		if (cap->datalen < sizeof(*params->he_cap) + 1) {
+			GENL_SET_ERR_MSG(info, "bad HE capability in beacon");
+			return -EINVAL;
+		}
 		params->he_cap = (void *)(cap->data + 1);
-	cap = cfg80211_find_ext_elem(WLAN_EID_EXT_HE_OPERATION, ies, ies_len);
-	if (cap && cap->datalen >= sizeof(*params->he_oper) + 1)
-		params->he_oper = (void *)(cap->data + 1);
+	}
+
 	cap = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_CAPABILITY, ies, ies_len);
 	if (cap) {
-		if (!cap->datalen)
-			return -EINVAL;
 		params->eht_cap = (void *)(cap->data + 1);
 		if (!ieee80211_eht_capa_size_ok((const u8 *)params->he_cap,
 						(const u8 *)params->eht_cap,
-						cap->datalen - 1, true))
-			return -EINVAL;
-	}
-	cap = cfg80211_find_ext_elem(WLAN_EID_EXT_EHT_OPERATION, ies, ies_len);
-	if (cap) {
-		if (!cap->datalen)
-			return -EINVAL;
-		params->eht_oper = (void *)(cap->data + 1);
-		if (!ieee80211_eht_oper_size_ok((const u8 *)params->eht_oper,
-						cap->datalen - 1))
+						cap->datalen - 1, true)) {
+			GENL_SET_ERR_MSG(info, "bad EHT capability in beacon");
 			return -EINVAL;
+		}
 	}
 
 	return 0;
@@ -6838,24 +7120,6 @@ static void nl80211_send_ap_started(struct wireless_dev *wdev,
 	nlmsg_free(msg);
 }
 
-static int nl80211_validate_ap_phy_operation(struct cfg80211_ap_settings *params)
-{
-	struct ieee80211_channel *channel = params->chandef.chan;
-
-	if ((params->he_cap ||  params->he_oper) &&
-	    (channel->flags & IEEE80211_CHAN_NO_HE))
-		return -EOPNOTSUPP;
-
-	if ((params->eht_cap || params->eht_oper) &&
-	    (channel->flags & IEEE80211_CHAN_NO_EHT))
-		return -EOPNOTSUPP;
-
-	if (params->uhr_oper && (channel->flags & IEEE80211_CHAN_NO_UHR))
-		return -EOPNOTSUPP;
-
-	return 0;
-}
-
 static int
 nl80211_parse_s1g_short_beacon(struct cfg80211_registered_device *rdev,
 			       struct nlattr *attrs,
@@ -6891,6 +7155,28 @@ nl80211_parse_s1g_short_beacon(struct cfg80211_registered_device *rdev,
 	return 0;
 }
 
+static int nl80211_check_npca(struct cfg80211_registered_device *rdev,
+			      const struct cfg80211_chan_def *chandef,
+			      enum nl80211_iftype iftype,
+			      struct netlink_ext_ack *extack)
+{
+	const struct ieee80211_supported_band *sband;
+	const struct ieee80211_sta_uhr_cap *uhr_cap;
+
+	if (!chandef->npca_chan)
+		return 0;
+
+	sband = rdev->wiphy.bands[chandef->chan->band];
+	uhr_cap = ieee80211_get_uhr_iftype_cap(sband, iftype);
+
+	if (uhr_cap &&
+	    (uhr_cap->mac.mac_cap[0] & IEEE80211_UHR_MAC_CAP0_NPCA_SUPP))
+		return 0;
+
+	NL_SET_ERR_MSG(extack, "NPCA not supported");
+	return -EINVAL;
+}
+
 static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info)
 {
 	struct cfg80211_registered_device *rdev = info->user_ptr[0];
@@ -6928,11 +7214,6 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info)
 	if (!params)
 		return -ENOMEM;
 
-	err = nl80211_parse_beacon(rdev, info->attrs, &params->beacon,
-				   info->extack);
-	if (err)
-		goto out;
-
 	params->beacon_interval =
 		nla_get_u32(info->attrs[NL80211_ATTR_BEACON_INTERVAL]);
 	params->dtim_period =
@@ -7035,7 +7316,7 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info)
 
 	if (info->attrs[NL80211_ATTR_WIPHY_FREQ]) {
 		err = nl80211_parse_chandef(rdev, info->extack, info->attrs,
-					    &params->chandef);
+					    &params->chandef, true);
 		if (err)
 			goto out;
 	} else if (wdev->valid_links) {
@@ -7049,6 +7330,16 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info)
 		goto out;
 	}
 
+	err = nl80211_parse_beacon(rdev, info->attrs, &params->beacon,
+				   params->chandef.chan, info->extack);
+	if (err)
+		goto out;
+
+	err = nl80211_check_npca(rdev, &params->chandef, wdev->iftype,
+				 info->extack);
+	if (err)
+		goto out;
+
 	beacon_check.iftype = wdev->iftype;
 	beacon_check.relax = true;
 	beacon_check.reg_power =
@@ -7148,14 +7439,7 @@ static int nl80211_start_ap(struct sk_buff *skb, struct genl_info *info)
 			goto out;
 	}
 
-	err = nl80211_calculate_ap_params(params);
-	if (err)
-		goto out;
-
-	if (info->attrs[NL80211_ATTR_UHR_OPERATION])
-		params->uhr_oper = nla_data(info->attrs[NL80211_ATTR_UHR_OPERATION]);
-
-	err = nl80211_validate_ap_phy_operation(params);
+	err = nl80211_calculate_ap_capabilities(info, params);
 	if (err)
 		goto out;
 
@@ -7226,6 +7510,7 @@ static int nl80211_set_beacon(struct sk_buff *skb, struct genl_info *info)
 		return -ENOMEM;
 
 	err = nl80211_parse_beacon(rdev, info->attrs, &params->beacon,
+				   wdev->links[link_id].ap.chandef.chan,
 				   info->extack);
 	if (err)
 		goto out;
@@ -10817,7 +11102,8 @@ static int nl80211_trigger_scan(struct sk_buff *skb, struct genl_info *info)
 
 	wiphy = &rdev->wiphy;
 
-	if (wdev->iftype == NL80211_IFTYPE_NAN)
+	if (wdev->iftype == NL80211_IFTYPE_NAN ||
+	    wdev->iftype == NL80211_IFTYPE_PD)
 		return -EOPNOTSUPP;
 
 	if (!rdev->ops->scan)
@@ -11605,7 +11891,8 @@ static int nl80211_start_radar_detection(struct sk_buff *skb,
 	if (dfs_region == NL80211_DFS_UNSET)
 		return -EINVAL;
 
-	err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef);
+	err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef,
+				    false);
 	if (err)
 		return err;
 
@@ -11694,7 +11981,8 @@ static int nl80211_notify_radar_detection(struct sk_buff *skb,
 		return -EINVAL;
 	}
 
-	err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef);
+	err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef,
+				    false);
 	if (err) {
 		GENL_SET_ERR_MSG(info, "Unable to extract chandef info");
 		return err;
@@ -11777,6 +12065,7 @@ static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info)
 	int err;
 	bool need_new_beacon = false;
 	bool need_handle_dfs_flag = true;
+	bool permit_npca = false;
 	u32 cs_count;
 
 	if (!rdev->ops->channel_switch ||
@@ -11794,6 +12083,8 @@ static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info)
 		 */
 		need_handle_dfs_flag = false;
 
+		permit_npca = true;
+
 		/* useless if AP is not running */
 		if (!wdev->links[link_id].ap.beacon_interval)
 			return -ENOTCONN;
@@ -11830,11 +12121,21 @@ static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info)
 
 	params.count = cs_count;
 
+	err = nl80211_parse_chandef(rdev, info->extack, info->attrs,
+				    &params.chandef, permit_npca);
+	if (err)
+		goto free;
+
+	err = nl80211_check_npca(rdev, &params.chandef, wdev->iftype,
+				 info->extack);
+	if (err)
+		goto free;
+
 	if (!need_new_beacon)
 		goto skip_beacons;
 
 	err = nl80211_parse_beacon(rdev, info->attrs, &params.beacon_after,
-				   info->extack);
+				   params.chandef.chan, info->extack);
 	if (err)
 		goto free;
 
@@ -11851,6 +12152,7 @@ static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info)
 		goto free;
 
 	err = nl80211_parse_beacon(rdev, csa_attrs, &params.beacon_csa,
+				   wdev->links[link_id].ap.chandef.chan,
 				   info->extack);
 	if (err)
 		goto free;
@@ -11879,11 +12181,6 @@ static int nl80211_channel_switch(struct sk_buff *skb, struct genl_info *info)
 		goto free;
 
 skip_beacons:
-	err = nl80211_parse_chandef(rdev, info->extack, info->attrs,
-				    &params.chandef);
-	if (err)
-		goto free;
-
 	if (!cfg80211_reg_can_beacon_relax(&rdev->wiphy, &params.chandef,
 					   wdev->iftype)) {
 		err = -EINVAL;
@@ -12868,9 +13165,9 @@ static int nl80211_associate(struct sk_buff *skb, struct genl_info *info)
 			goto free;
 		}
 
-		if (info->attrs[NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS])
+		if (info->attrs[NL80211_ATTR_EXT_MLD_CAPA_AND_OPS])
 			req.ext_mld_capa_ops =
-				nla_get_u16(info->attrs[NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS]);
+				nla_get_u16(info->attrs[NL80211_ATTR_EXT_MLD_CAPA_AND_OPS]);
 	} else {
 		if (req.link_id >= 0)
 			return -EINVAL;
@@ -12881,7 +13178,7 @@ static int nl80211_associate(struct sk_buff *skb, struct genl_info *info)
 			return PTR_ERR(req.bss);
 		ap_addr = req.bss->bssid;
 
-		if (info->attrs[NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS])
+		if (info->attrs[NL80211_ATTR_EXT_MLD_CAPA_AND_OPS])
 			return -EINVAL;
 	}
 
@@ -13104,7 +13401,7 @@ static int nl80211_join_ibss(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	err = nl80211_parse_chandef(rdev, info->extack, info->attrs,
-				    &ibss.chandef);
+				    &ibss.chandef, false);
 	if (err)
 		return err;
 
@@ -13175,7 +13472,8 @@ static int nl80211_join_ibss(struct sk_buff *skb, struct genl_info *info)
 	if (ibss.privacy && info->attrs[NL80211_ATTR_KEYS]) {
 		bool no_ht = false;
 
-		connkeys = nl80211_parse_connkeys(rdev, info, &no_ht);
+		connkeys = nl80211_parse_connkeys(rdev, dev->ieee80211_ptr,
+						  info, &no_ht);
 		if (IS_ERR(connkeys))
 			return PTR_ERR(connkeys);
 
@@ -13617,7 +13915,8 @@ static int nl80211_connect(struct sk_buff *skb, struct genl_info *info)
 	}
 
 	if (connect.privacy && info->attrs[NL80211_ATTR_KEYS]) {
-		connkeys = nl80211_parse_connkeys(rdev, info, NULL);
+		connkeys = nl80211_parse_connkeys(rdev, dev->ieee80211_ptr,
+						  info, NULL);
 		if (IS_ERR(connkeys))
 			return PTR_ERR(connkeys);
 	}
@@ -14091,6 +14390,7 @@ static int nl80211_remain_on_channel(struct sk_buff *skb,
 	unsigned int link_id = nl80211_link_id(info->attrs);
 	struct wireless_dev *wdev = info->user_ptr[1];
 	struct cfg80211_chan_def chandef;
+	const u8 *rx_addr = NULL;
 	struct sk_buff *msg;
 	void *hdr;
 	u64 cookie;
@@ -14103,6 +14403,14 @@ static int nl80211_remain_on_channel(struct sk_buff *skb,
 
 	duration = nla_get_u32(info->attrs[NL80211_ATTR_DURATION]);
 
+	if (info->attrs[NL80211_ATTR_MAC])
+		rx_addr = nla_data(info->attrs[NL80211_ATTR_MAC]);
+
+	if (rx_addr &&
+	    !wiphy_ext_feature_isset(wdev->wiphy,
+				     NL80211_EXT_FEATURE_ROC_ADDR_FILTER))
+		return -EOPNOTSUPP;
+
 	if (!rdev->ops->remain_on_channel ||
 	    !(rdev->wiphy.flags & WIPHY_FLAG_HAS_REMAIN_ON_CHANNEL))
 		return -EOPNOTSUPP;
@@ -14115,7 +14423,8 @@ static int nl80211_remain_on_channel(struct sk_buff *skb,
 	    duration > rdev->wiphy.max_remain_on_channel_duration)
 		return -EINVAL;
 
-	err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef);
+	err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef,
+				    false);
 	if (err)
 		return err;
 
@@ -14150,7 +14459,7 @@ static int nl80211_remain_on_channel(struct sk_buff *skb,
 	}
 
 	err = rdev_remain_on_channel(rdev, wdev, chandef.chan,
-				     duration, &cookie);
+				     duration, &cookie, rx_addr);
 
 	if (err)
 		goto free_msg;
@@ -14239,6 +14548,11 @@ static int nl80211_register_mgmt(struct sk_buff *skb, struct genl_info *info)
 		      WIPHY_NAN_FLAGS_USERSPACE_DE))
 			return -EOPNOTSUPP;
 		break;
+	case NL80211_IFTYPE_PD:
+		if (!wiphy_ext_feature_isset(wdev->wiphy,
+					     NL80211_EXT_FEATURE_SECURE_RTT))
+			return -EOPNOTSUPP;
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -14303,6 +14617,11 @@ static int nl80211_tx_mgmt(struct sk_buff *skb, struct genl_info *info)
 		      WIPHY_NAN_FLAGS_USERSPACE_DE))
 			return -EOPNOTSUPP;
 		break;
+	case NL80211_IFTYPE_PD:
+		if (!wiphy_ext_feature_isset(wdev->wiphy,
+					     NL80211_EXT_FEATURE_SECURE_RTT))
+			return -EOPNOTSUPP;
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -14334,7 +14653,7 @@ static int nl80211_tx_mgmt(struct sk_buff *skb, struct genl_info *info)
 	chandef.chan = NULL;
 	if (info->attrs[NL80211_ATTR_WIPHY_FREQ]) {
 		err = nl80211_parse_chandef(rdev, info->extack, info->attrs,
-					    &chandef);
+					    &chandef, false);
 		if (err)
 			return err;
 	}
@@ -14428,6 +14747,11 @@ static int nl80211_tx_mgmt_cancel_wait(struct sk_buff *skb, struct genl_info *in
 					     NL80211_EXT_FEATURE_SECURE_NAN))
 			return -EOPNOTSUPP;
 		break;
+	case NL80211_IFTYPE_PD:
+		if (!wiphy_ext_feature_isset(wdev->wiphy,
+					     NL80211_EXT_FEATURE_SECURE_RTT))
+			return -EOPNOTSUPP;
+		break;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -14737,7 +15061,7 @@ static int nl80211_join_ocb(struct sk_buff *skb, struct genl_info *info)
 	int err;
 
 	err = nl80211_parse_chandef(rdev, info->extack, info->attrs,
-				    &setup.chandef);
+				    &setup.chandef, false);
 	if (err)
 		return err;
 
@@ -14813,7 +15137,7 @@ static int nl80211_join_mesh(struct sk_buff *skb, struct genl_info *info)
 
 	if (info->attrs[NL80211_ATTR_WIPHY_FREQ]) {
 		err = nl80211_parse_chandef(rdev, info->extack, info->attrs,
-					    &setup.chandef);
+					    &setup.chandef, false);
 		if (err)
 			return err;
 	} else {
@@ -16564,6 +16888,46 @@ static int nl80211_nan_change_config(struct sk_buff *skb,
 	return rdev_nan_change_conf(rdev, wdev, &conf, changed);
 }
 
+static int nl80211_start_pd(struct sk_buff *skb, struct genl_info *info)
+{
+	struct cfg80211_registered_device *rdev = info->user_ptr[0];
+	struct wireless_dev *wdev = info->user_ptr[1];
+	int err;
+
+	if (wdev->iftype != NL80211_IFTYPE_PD)
+		return -EOPNOTSUPP;
+
+	if (wdev_running(wdev))
+		return -EEXIST;
+
+	if (rfkill_blocked(rdev->wiphy.rfkill))
+		return -ERFKILL;
+
+	if (!rdev->ops->start_pd)
+		return -EOPNOTSUPP;
+
+	err = rdev_start_pd(rdev, wdev);
+	if (err)
+		return err;
+	wdev->is_running = true;
+	rdev->opencount++;
+
+	return 0;
+}
+
+static int nl80211_stop_pd(struct sk_buff *skb, struct genl_info *info)
+{
+	struct cfg80211_registered_device *rdev = info->user_ptr[0];
+	struct wireless_dev *wdev = info->user_ptr[1];
+
+	if (wdev->iftype != NL80211_IFTYPE_PD)
+		return -EOPNOTSUPP;
+
+	cfg80211_stop_pd(rdev, wdev);
+
+	return 0;
+}
+
 void cfg80211_nan_match(struct wireless_dev *wdev,
 			struct cfg80211_nan_match_params *match, gfp_t gfp)
 {
@@ -16777,7 +17141,7 @@ static int nl80211_parse_nan_channel(struct cfg80211_registered_device *rdev,
 		return ret;
 
 	ret = nl80211_parse_chandef(rdev, info->extack, channel_parsed,
-				    &chandef);
+				    &chandef, false);
 	if (ret)
 		return ret;
 
@@ -17776,7 +18140,8 @@ static int nl80211_tdls_channel_switch(struct sk_buff *skb,
 	    !info->attrs[NL80211_ATTR_OPER_CLASS])
 		return -EINVAL;
 
-	err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef);
+	err = nl80211_parse_chandef(rdev, info->extack, info->attrs, &chandef,
+				    false);
 	if (err)
 		return err;
 
@@ -18349,7 +18714,12 @@ static int nl80211_color_change(struct sk_buff *skb, struct genl_info *info)
 	params.count = nla_get_u8(info->attrs[NL80211_ATTR_COLOR_CHANGE_COUNT]);
 	params.color = nla_get_u8(info->attrs[NL80211_ATTR_COLOR_CHANGE_COLOR]);
 
+	params.link_id = nl80211_link_id(info->attrs);
+	if (!wdev->links[params.link_id].ap.beacon_interval)
+		return -EINVAL;
+
 	err = nl80211_parse_beacon(rdev, info->attrs, &params.beacon_next,
+				   wdev->links[params.link_id].ap.chandef.chan,
 				   info->extack);
 	if (err)
 		return err;
@@ -18365,6 +18735,7 @@ static int nl80211_color_change(struct sk_buff *skb, struct genl_info *info)
 		goto out;
 
 	err = nl80211_parse_beacon(rdev, tb, &params.beacon_color_change,
+				   wdev->links[params.link_id].ap.chandef.chan,
 				   info->extack);
 	if (err)
 		goto out;
@@ -18422,7 +18793,6 @@ static int nl80211_color_change(struct sk_buff *skb, struct genl_info *info)
 			goto out;
 	}
 
-	params.link_id = nl80211_link_id(info->attrs);
 	err = rdev_color_change(rdev, dev, &params);
 
 out:
@@ -18751,9 +19121,9 @@ static int nl80211_assoc_ml_reconf(struct sk_buff *skb, struct genl_info *info)
 		goto out;
 	}
 
-	if (info->attrs[NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS])
+	if (info->attrs[NL80211_ATTR_EXT_MLD_CAPA_AND_OPS])
 		req.ext_mld_capa_ops =
-			nla_get_u16(info->attrs[NL80211_ATTR_ASSOC_MLD_EXT_CAPA_OPS]);
+			nla_get_u16(info->attrs[NL80211_ATTR_EXT_MLD_CAPA_AND_OPS]);
 
 	err = cfg80211_assoc_ml_reconf(rdev, dev, &req);
 
@@ -19726,6 +20096,20 @@ static const struct genl_small_ops nl80211_small_ops[] = {
 		.flags = GENL_ADMIN_PERM,
 		.internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV_UP),
 	},
+	{
+		.cmd = NL80211_CMD_START_PD,
+		.doit = nl80211_start_pd,
+		.flags = GENL_ADMIN_PERM,
+		.internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV |
+					 NL80211_FLAG_NEED_RTNL),
+	},
+	{
+		.cmd = NL80211_CMD_STOP_PD,
+		.doit = nl80211_stop_pd,
+		.flags = GENL_ADMIN_PERM,
+		.internal_flags = IFLAGS(NL80211_FLAG_NEED_WDEV_UP |
+					 NL80211_FLAG_NEED_RTNL),
+	},
 	{
 		.cmd = NL80211_CMD_SET_MCAST_RATE,
 		.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
@@ -20615,7 +20999,9 @@ void nl80211_send_connect_result(struct cfg80211_registered_device *rdev,
 	      (cr->fils.pmk &&
 	       nla_put(msg, NL80211_ATTR_PMK, cr->fils.pmk_len, cr->fils.pmk)) ||
 	      (cr->fils.pmkid &&
-	       nla_put(msg, NL80211_ATTR_PMKID, WLAN_PMKID_LEN, cr->fils.pmkid)))))
+	       nla_put(msg, NL80211_ATTR_PMKID, WLAN_PMKID_LEN, cr->fils.pmkid)))) ||
+	    (cr->assoc_encrypted &&
+	     nla_put_flag(msg, NL80211_ATTR_ASSOC_ENCRYPTED)))
 		goto nla_put_failure;
 
 	if (cr->valid_links) {
diff --git a/net/wireless/nl80211.h b/net/wireless/nl80211.h
index 048ba92c3e429..bdb065d140549 100644
--- a/net/wireless/nl80211.h
+++ b/net/wireless/nl80211.h
@@ -1,7 +1,7 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
  * Portions of this file
- * Copyright (C) 2018, 2020-2025 Intel Corporation
+ * Copyright (C) 2018, 2020-2026 Intel Corporation
  */
 #ifndef __NET_WIRELESS_NL80211_H
 #define __NET_WIRELESS_NL80211_H
@@ -25,7 +25,8 @@ static inline u64 wdev_id(struct wireless_dev *wdev)
 int nl80211_parse_chandef(struct cfg80211_registered_device *rdev,
 			  struct netlink_ext_ack *extack,
 			  struct nlattr **attrs,
-			  struct cfg80211_chan_def *chandef);
+			  struct cfg80211_chan_def *chandef,
+			  bool npca_permitted);
 int nl80211_parse_random_mac(struct nlattr **attrs,
 			     u8 *mac_addr, u8 *mac_addr_mask);
 
diff --git a/net/wireless/pmsr.c b/net/wireless/pmsr.c
index d6cd0de64d1f8..c8447448f3a50 100644
--- a/net/wireless/pmsr.c
+++ b/net/wireless/pmsr.c
@@ -17,11 +17,19 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev,
 	u32 preamble = NL80211_PREAMBLE_DMG; /* only optional in DMG */
 
 	/* validate existing data */
-	if (!(rdev->wiphy.pmsr_capa->ftm.bandwidths & BIT(out->chandef.width))) {
+	if (out->ftm.request_type == NL80211_PMSR_FTM_REQ_TYPE_INFRA &&
+	    !(capa->ftm.bandwidths & BIT(out->chandef.width))) {
 		NL_SET_ERR_MSG(info->extack, "FTM: unsupported bandwidth");
 		return -EINVAL;
 	}
 
+	if (out->ftm.request_type == NL80211_PMSR_FTM_REQ_TYPE_PD &&
+	    !(capa->ftm.pd_bandwidths & BIT(out->chandef.width))) {
+		NL_SET_ERR_MSG(info->extack,
+			       "FTM: unsupported bandwidth for PD request");
+		return -EINVAL;
+	}
+
 	/* no validation needed - was already done via nested policy */
 	nla_parse_nested_deprecated(tb, NL80211_PMSR_FTM_REQ_ATTR_MAX, ftmreq,
 				    NULL, NULL);
@@ -44,13 +52,22 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev,
 		}
 	}
 
-	if (!(capa->ftm.preambles & BIT(preamble))) {
+	if (out->ftm.request_type == NL80211_PMSR_FTM_REQ_TYPE_INFRA &&
+	    !(capa->ftm.preambles & BIT(preamble))) {
 		NL_SET_ERR_MSG_ATTR(info->extack,
 				    tb[NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE],
 				    "FTM: invalid preamble");
 		return -EINVAL;
 	}
 
+	if (out->ftm.request_type == NL80211_PMSR_FTM_REQ_TYPE_PD &&
+	    !(capa->ftm.pd_preambles & BIT(preamble))) {
+		NL_SET_ERR_MSG_ATTR(info->extack,
+				    tb[NL80211_PMSR_FTM_REQ_ATTR_PREAMBLE],
+				    "FTM: invalid preamble for PD request");
+		return -EINVAL;
+	}
+
 	out->ftm.preamble = preamble;
 
 	out->ftm.burst_period = 0;
@@ -91,11 +108,10 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev,
 			nla_get_u8(tb[NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST]);
 
 	if (capa->ftm.max_ftms_per_burst &&
-	    (out->ftm.ftms_per_burst > capa->ftm.max_ftms_per_burst ||
-	     out->ftm.ftms_per_burst == 0)) {
+	    out->ftm.ftms_per_burst > capa->ftm.max_ftms_per_burst) {
 		NL_SET_ERR_MSG_ATTR(info->extack,
 				    tb[NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST],
-				    "FTM: FTMs per burst must be set lower than the device limit but non-zero");
+				    "FTM: FTMs per burst must be set lower than the device limit");
 		return -EINVAL;
 	}
 
@@ -128,6 +144,14 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev,
 		return -EINVAL;
 	}
 
+	if (out->ftm.request_type == NL80211_PMSR_FTM_REQ_TYPE_PD &&
+	    out->ftm.trigger_based) {
+		NL_SET_ERR_MSG_ATTR(info->extack,
+				    ftmreq,
+				    "FTM: TB ranging is not supported for PD request type");
+		return -EINVAL;
+	}
+
 	out->ftm.non_trigger_based =
 		!!tb[NL80211_PMSR_FTM_REQ_ATTR_NON_TRIGGER_BASED];
 	if (out->ftm.non_trigger_based && !capa->ftm.non_trigger_based) {
@@ -143,6 +167,14 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev,
 		return -EINVAL;
 	}
 
+	if (out->ftm.request_type == NL80211_PMSR_FTM_REQ_TYPE_PD &&
+	    out->ftm.non_trigger_based && out->ftm.ftms_per_burst > 4) {
+		NL_SET_ERR_MSG_ATTR(info->extack,
+				    tb[NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST],
+				    "FTM: FTMs per burst must not exceed 4 for PD NTB ranging");
+		return -ERANGE;
+	}
+
 	if (out->ftm.ftms_per_burst > 31 && !out->ftm.non_trigger_based &&
 	    !out->ftm.trigger_based) {
 		NL_SET_ERR_MSG_ATTR(info->extack,
@@ -188,20 +220,86 @@ static int pmsr_parse_ftm(struct cfg80211_registered_device *rdev,
 	}
 
 	out->ftm.rsta = !!tb[NL80211_PMSR_FTM_REQ_ATTR_RSTA];
-	if (out->ftm.rsta && !capa->ftm.support_rsta) {
+	if (out->ftm.rsta && out->ftm.non_trigger_based &&
+	    !capa->ftm.rsta.support_ntb) {
+		NL_SET_ERR_MSG_ATTR(info->extack,
+				    tb[NL80211_PMSR_FTM_REQ_ATTR_RSTA],
+				    "FTM: NTB RSTA not supported by device");
+		return -EOPNOTSUPP;
+	}
+
+	if (out->ftm.rsta && out->ftm.trigger_based &&
+	    !capa->ftm.rsta.support_tb) {
 		NL_SET_ERR_MSG_ATTR(info->extack,
 				    tb[NL80211_PMSR_FTM_REQ_ATTR_RSTA],
-				    "FTM: RSTA not supported by device");
+				    "FTM: TB RSTA not supported by device");
 		return -EOPNOTSUPP;
 	}
 
-	if (out->ftm.rsta && !out->ftm.lmr_feedback) {
+	if (out->ftm.rsta && !out->ftm.non_trigger_based &&
+	    !out->ftm.trigger_based &&
+	    !capa->ftm.rsta.support_edca) {
+		NL_SET_ERR_MSG_ATTR(info->extack,
+				    tb[NL80211_PMSR_FTM_REQ_ATTR_RSTA],
+				    "FTM: EDCA RSTA not supported by device");
+		return -EOPNOTSUPP;
+	}
+
+	if (out->ftm.rsta &&
+	    (out->ftm.non_trigger_based || out->ftm.trigger_based) &&
+	    !out->ftm.lmr_feedback) {
 		NL_SET_ERR_MSG_ATTR(info->extack,
 				    tb[NL80211_PMSR_FTM_REQ_ATTR_RSTA],
 				    "FTM: RSTA set without LMR feedback");
 		return -EINVAL;
 	}
 
+	if (out->ftm.non_trigger_based) {
+		if (out->ftm.request_type == NL80211_PMSR_FTM_REQ_TYPE_PD &&
+		    !tb[NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME]) {
+			NL_SET_ERR_MSG(info->extack,
+				       "FTM: nominal time is required for PD NTB ranging");
+			return -EINVAL;
+		}
+		out->ftm.nominal_time =
+			nla_get_u32(tb[NL80211_PMSR_FTM_REQ_ATTR_NOMINAL_TIME]);
+
+		if (tb[NL80211_PMSR_FTM_REQ_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS])
+			out->ftm.min_time_between_measurements =
+			nla_get_u32(tb[NL80211_PMSR_FTM_REQ_ATTR_MIN_TIME_BETWEEN_MEASUREMENTS]);
+
+		if (tb[NL80211_PMSR_FTM_REQ_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS])
+			out->ftm.max_time_between_measurements =
+			nla_get_u32(tb[NL80211_PMSR_FTM_REQ_ATTR_MAX_TIME_BETWEEN_MEASUREMENTS]);
+
+		if (tb[NL80211_PMSR_FTM_REQ_ATTR_AW_DURATION])
+			out->ftm.availability_window =
+				nla_get_u8(tb[NL80211_PMSR_FTM_REQ_ATTR_AW_DURATION]);
+
+		if (tb[NL80211_PMSR_FTM_REQ_ATTR_NUM_MEASUREMENTS])
+			out->ftm.num_measurements =
+				nla_get_u32(tb[NL80211_PMSR_FTM_REQ_ATTR_NUM_MEASUREMENTS]);
+	}
+
+	if (tb[NL80211_PMSR_FTM_REQ_ATTR_INGRESS])
+		out->ftm.ingress_distance =
+			nla_get_u64(tb[NL80211_PMSR_FTM_REQ_ATTR_INGRESS]);
+
+	if (tb[NL80211_PMSR_FTM_REQ_ATTR_EGRESS])
+		out->ftm.egress_distance =
+			nla_get_u64(tb[NL80211_PMSR_FTM_REQ_ATTR_EGRESS]);
+
+	out->ftm.pd_suppress_range_results =
+		nla_get_flag(tb[NL80211_PMSR_FTM_REQ_ATTR_PD_SUPPRESS_RESULTS]);
+
+	if (out->ftm.request_type != NL80211_PMSR_FTM_REQ_TYPE_PD &&
+	    out->ftm.pd_suppress_range_results) {
+		NL_SET_ERR_MSG_ATTR(info->extack,
+				    tb[NL80211_PMSR_FTM_REQ_ATTR_PD_SUPPRESS_RESULTS],
+				    "FTM: suppress range result flag only valid for PD requests");
+		return -EINVAL;
+	}
+
 	return 0;
 }
 
@@ -229,6 +327,19 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev,
 
 	memcpy(out->addr, nla_data(tb[NL80211_PMSR_PEER_ATTR_ADDR]), ETH_ALEN);
 
+	if (tb[NL80211_PMSR_PEER_ATTR_REQ_TYPE])
+		out->ftm.request_type =
+			nla_get_u32(tb[NL80211_PMSR_PEER_ATTR_REQ_TYPE]);
+	else
+		out->ftm.request_type = NL80211_PMSR_FTM_REQ_TYPE_INFRA;
+
+	if (out->ftm.request_type == NL80211_PMSR_FTM_REQ_TYPE_PD &&
+	    !rdev->wiphy.pmsr_capa->ftm.type.pd_support) {
+		NL_SET_ERR_MSG_ATTR(info->extack,
+				    tb[NL80211_PMSR_PEER_ATTR_REQ_TYPE],
+				    "FTM: PD request type not supported by device");
+		return -EINVAL;
+	}
 	/* reuse info->attrs */
 	memset(info->attrs, 0, sizeof(*info->attrs) * (NL80211_ATTR_MAX + 1));
 	err = nla_parse_nested_deprecated(info->attrs, NL80211_ATTR_MAX,
@@ -238,7 +349,7 @@ static int pmsr_parse_peer(struct cfg80211_registered_device *rdev,
 		return err;
 
 	err = nl80211_parse_chandef(rdev, info->extack, info->attrs,
-				    &out->chandef);
+				    &out->chandef, false);
 	if (err)
 		return err;
 
@@ -286,12 +397,15 @@ int nl80211_pmsr_start(struct sk_buff *skb, struct genl_info *info)
 {
 	struct nlattr *reqattr = info->attrs[NL80211_ATTR_PEER_MEASUREMENTS];
 	struct cfg80211_registered_device *rdev = info->user_ptr[0];
+	int count, rem, err, idx, peer_count;
 	struct wireless_dev *wdev = info->user_ptr[1];
+	const struct cfg80211_pmsr_capabilities *capa;
 	struct cfg80211_pmsr_request *req;
 	struct nlattr *peers, *peer;
-	int count, rem, err, idx;
 
-	if (!rdev->wiphy.pmsr_capa)
+	capa = rdev->wiphy.pmsr_capa;
+
+	if (!capa)
 		return -EOPNOTSUPP;
 
 	if (!reqattr)
@@ -306,7 +420,7 @@ int nl80211_pmsr_start(struct sk_buff *skb, struct genl_info *info)
 	nla_for_each_nested(peer, peers, rem) {
 		count++;
 
-		if (count > rdev->wiphy.pmsr_capa->max_peers) {
+		if (count > capa->max_peers) {
 			NL_SET_ERR_MSG_ATTR(info->extack, peer,
 					    "Too many peers used");
 			return -EINVAL;
@@ -322,7 +436,7 @@ int nl80211_pmsr_start(struct sk_buff *skb, struct genl_info *info)
 		req->timeout = nla_get_u32(info->attrs[NL80211_ATTR_TIMEOUT]);
 
 	if (info->attrs[NL80211_ATTR_MAC]) {
-		if (!rdev->wiphy.pmsr_capa->randomize_mac_addr) {
+		if (!capa->randomize_mac_addr) {
 			NL_SET_ERR_MSG_ATTR(info->extack,
 					    info->attrs[NL80211_ATTR_MAC],
 					    "device cannot randomize MAC address");
@@ -347,6 +461,41 @@ int nl80211_pmsr_start(struct sk_buff *skb, struct genl_info *info)
 			goto out_err;
 		idx++;
 	}
+
+	/* Validate per-role peer limits if advertised */
+	if (capa->ftm.ista.max_peers) {
+		peer_count = 0;
+
+		for (idx = 0; idx < req->n_peers; idx++) {
+			if (!req->peers[idx].ftm.rsta) {
+				peer_count++;
+
+				if (peer_count > capa->ftm.ista.max_peers) {
+					NL_SET_ERR_MSG(info->extack,
+						       "Too many ISTA peers for device limit");
+					err = -EINVAL;
+					goto out_err;
+				}
+			}
+		}
+	}
+
+	if (capa->ftm.rsta.max_peers) {
+		peer_count = 0;
+
+		for (idx = 0; idx < req->n_peers; idx++) {
+			if (req->peers[idx].ftm.rsta) {
+				peer_count++;
+
+				if (peer_count > capa->ftm.rsta.max_peers) {
+					NL_SET_ERR_MSG(info->extack,
+						       "Too many RSTA peers for device limit");
+					err = -EINVAL;
+					goto out_err;
+				}
+			}
+		}
+	}
 	req->cookie = cfg80211_assign_cookie(rdev);
 	req->nl_portid = info->snd_portid;
 
@@ -487,6 +636,21 @@ static int nl80211_pmsr_send_ftm_res(struct sk_buff *msg,
 	PUTOPT_U64(DIST_AVG, dist_avg);
 	PUTOPT_U64(DIST_VARIANCE, dist_variance);
 	PUTOPT_U64(DIST_SPREAD, dist_spread);
+	PUTOPT(u32, TX_LTF_REPETITION_COUNT, tx_ltf_repetition_count);
+	PUTOPT(u32, RX_LTF_REPETITION_COUNT, rx_ltf_repetition_count);
+	PUTOPT(u32, MAX_TIME_BETWEEN_MEASUREMENTS,
+	       max_time_between_measurements);
+	PUTOPT(u32, MIN_TIME_BETWEEN_MEASUREMENTS,
+	       min_time_between_measurements);
+	PUTOPT(u8, NUM_TX_SPATIAL_STREAMS, num_tx_spatial_streams);
+	PUTOPT(u8, NUM_RX_SPATIAL_STREAMS, num_rx_spatial_streams);
+	PUTOPT(u32, NOMINAL_TIME, nominal_time);
+	PUTOPT(u8, AVAILABILITY_WINDOW, availability_window);
+	PUTOPT(u32, CHANNEL_WIDTH, chan_width);
+	PUTOPT(u32, PREAMBLE, preamble);
+	if (res->ftm.is_delayed_lmr &&
+	    nla_put_flag(msg, NL80211_PMSR_FTM_RESP_ATTR_IS_DELAYED_LMR))
+		goto error;
 	if (res->ftm.lci && res->ftm.lci_len &&
 	    nla_put(msg, NL80211_PMSR_FTM_RESP_ATTR_LCI,
 		    res->ftm.lci_len, res->ftm.lci))
diff --git a/net/wireless/rdev-ops.h b/net/wireless/rdev-ops.h
index bba239a068f63..63c26e8b11395 100644
--- a/net/wireless/rdev-ops.h
+++ b/net/wireless/rdev-ops.h
@@ -736,12 +736,13 @@ static inline int
 rdev_remain_on_channel(struct cfg80211_registered_device *rdev,
 		       struct wireless_dev *wdev,
 		       struct ieee80211_channel *chan,
-		       unsigned int duration, u64 *cookie)
+		       unsigned int duration, u64 *cookie, const u8 *rx_addr)
 {
 	int ret;
-	trace_rdev_remain_on_channel(&rdev->wiphy, wdev, chan, duration);
+	trace_rdev_remain_on_channel(&rdev->wiphy, wdev, chan, duration,
+				     rx_addr);
 	ret = rdev->ops->remain_on_channel(&rdev->wiphy, wdev, chan,
-					   duration, cookie);
+					   duration, cookie, rx_addr);
 	trace_rdev_return_int_cookie(&rdev->wiphy, ret, *cookie);
 	return ret;
 }
@@ -1092,6 +1093,25 @@ rdev_nan_set_peer_sched(struct cfg80211_registered_device *rdev,
 	return ret;
 }
 
+static inline int rdev_start_pd(struct cfg80211_registered_device *rdev,
+				struct wireless_dev *wdev)
+{
+	int ret;
+
+	trace_rdev_start_pd(&rdev->wiphy, wdev);
+	ret = rdev->ops->start_pd(&rdev->wiphy, wdev);
+	trace_rdev_return_int(&rdev->wiphy, ret);
+	return ret;
+}
+
+static inline void rdev_stop_pd(struct cfg80211_registered_device *rdev,
+				struct wireless_dev *wdev)
+{
+	trace_rdev_stop_pd(&rdev->wiphy, wdev);
+	rdev->ops->stop_pd(&rdev->wiphy, wdev);
+	trace_rdev_return_void(&rdev->wiphy);
+}
+
 static inline int rdev_set_mac_acl(struct cfg80211_registered_device *rdev,
 				   struct net_device *dev,
 				   struct cfg80211_acl_data *params)
diff --git a/net/wireless/reg.c b/net/wireless/reg.c
index 5db2121c0b570..1e8214d6b6d88 100644
--- a/net/wireless/reg.c
+++ b/net/wireless/reg.c
@@ -2412,6 +2412,9 @@ static bool reg_wdev_chan_valid(struct wiphy *wiphy, struct wireless_dev *wdev)
 		case NL80211_IFTYPE_NAN_DATA:
 			/* NAN channels are checked in NL80211_IFTYPE_NAN interface */
 			break;
+		case NL80211_IFTYPE_PD:
+			/* we have no info, but PD is also pretty universal */
+			continue;
 		default:
 			/* others not implemented for now */
 			WARN_ON_ONCE(1);
diff --git a/net/wireless/scan.c b/net/wireless/scan.c
index 328af43ef8325..358cbc9e43d85 100644
--- a/net/wireless/scan.c
+++ b/net/wireless/scan.c
@@ -2462,6 +2462,9 @@ size_t cfg80211_merge_profile(const u8 *ie, size_t ielen,
 		memcpy(merged_ie + copied_len, next_sub->data,
 		       next_sub->datalen);
 		copied_len += next_sub->datalen;
+
+		mbssid_elem = next_mbssid;
+		sub_elem = next_sub;
 	}
 
 	return copied_len;
diff --git a/net/wireless/sme.c b/net/wireless/sme.c
index 86e2ccaa678ce..b451df3096dd1 100644
--- a/net/wireless/sme.c
+++ b/net/wireless/sme.c
@@ -1066,6 +1066,7 @@ void cfg80211_connect_done(struct net_device *dev,
 	}
 	ev->cr.status = params->status;
 	ev->cr.timeout_reason = params->timeout_reason;
+	ev->cr.assoc_encrypted = params->assoc_encrypted;
 
 	spin_lock_irqsave(&wdev->event_lock, flags);
 	list_add_tail(&ev->list, &wdev->event_list);
diff --git a/net/wireless/tests/chan.c b/net/wireless/tests/chan.c
index 7b97b731993ca..65eb18c498deb 100644
--- a/net/wireless/tests/chan.c
+++ b/net/wireless/tests/chan.c
@@ -2,13 +2,18 @@
 /*
  * KUnit tests for channel helper functions
  *
- * Copyright (C) 2023-2024 Intel Corporation
+ * Copyright (C) 2023-2024, 2026 Intel Corporation
  */
 #include <net/cfg80211.h>
 #include <kunit/test.h>
 
 MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
 
+static struct ieee80211_channel chan_2ghz_1 = {
+	.band = NL80211_BAND_2GHZ,
+	.center_freq = 2412,
+};
+
 static struct ieee80211_channel chan_6ghz_1 = {
 	.band = NL80211_BAND_6GHZ,
 	.center_freq = 5955,
@@ -215,14 +220,189 @@ static void test_chandef_compat(struct kunit *test)
 	KUNIT_EXPECT_PTR_EQ(test, ret, expect);
 }
 
-static struct kunit_case chandef_compat_test_cases[] = {
+static const struct chandef_dbe_case {
+	const char *desc;
+	struct cfg80211_chan_def c;
+	u8 dbe[3];
+	bool fails;
+	u16 cf1;
+} chandef_dbe_cases[] = {
+	{
+		.desc = "non-HT failure",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_20_NOHT,
+			.chan = &chan_6ghz_1,
+			.center_freq1 = 5955,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_40,
+		.fails = true,
+	},
+	{
+		.desc = "2.4 GHz fails",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_20,
+			.chan = &chan_2ghz_1,
+			.center_freq1 = 2412,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_40,
+		.fails = true,
+	},
+	{
+		.desc = "DBE narrower",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_320,
+			.chan = &chan_6ghz_1,
+			.center_freq1 = 5955 + 10 + 20 + 40 + 80,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_160,
+		.fails = true,
+	},
+	{
+		.desc = "DBE to 320-1",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_160,
+			.chan = &chan_6ghz_105,
+			.center_freq1 = 6475 + 30,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_320_1,
+		.cf1 = 6425,
+	},
+	{
+		.desc = "DBE to 320-2",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_160,
+			.chan = &chan_6ghz_105,
+			.center_freq1 = 6475 + 30,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_320_2,
+		.cf1 = 6585,
+	},
+	{
+		.desc = "bad disabled subchannel bitmap - not enough in BSS (1)",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_160,
+			.chan = &chan_6ghz_105,
+			.center_freq1 = 6475 + 30,
+			.punctured = 0x0001,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_320_1 |
+			  IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES,
+		/* DBE disabled subchannel bitmap == 0 */
+		.fails = true,
+	},
+	{
+		.desc = "bad disabled subchannel bitmap - too much in BSS (1)",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_160,
+			.chan = &chan_6ghz_105,
+			.center_freq1 = 6475 + 30,
+			.punctured = 0x0001,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_320_1 |
+			  IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES,
+		/* DBE disabled subchannel bitmap == 0x0300 */
+		.dbe[2] = 0x03,
+		.fails = true,
+	},
+	{
+		.desc = "bad disabled subchannel bitmap - not enough in BSS (2)",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_160,
+			.chan = &chan_6ghz_105,
+			.center_freq1 = 6475 + 30,
+			.punctured = 0x0001,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_320_2 |
+			  IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES,
+		/* DBE disabled subchannel bitmap == 0 */
+		.fails = true,
+	},
+	{
+		.desc = "bad disabled subchannel bitmap - too much in BSS (2)",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_160,
+			.chan = &chan_6ghz_105,
+			.center_freq1 = 6475 + 30,
+			.punctured = 0x0001,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_320_2 |
+			  IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES,
+		/* DBE disabled subchannel bitmap == 0x03 */
+		.dbe[1] = 0x03,
+		.fails = true,
+	},
+	{
+		.desc = "bad disabled subchannel bitmap - bad bitmap",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_160,
+			.chan = &chan_6ghz_105,
+			.center_freq1 = 6475 + 30,
+			.punctured = 0x0001,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_320_1 |
+			  IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES,
+		/* DBE disabled subchannel bitmap == 0x1100 */
+		.dbe[2] = 0x11,
+		.fails = true,
+	},
+	{
+		.desc = "good disabled subchannel bitmap (1)",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_160,
+			.chan = &chan_6ghz_105,
+			.center_freq1 = 6475 + 30,
+			.punctured = 0x0003,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_320_1 |
+			  IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES,
+		/* DBE disabled subchannel bitmap == 0x0300 */
+		.dbe[2] = 0x03,
+		.cf1 = 6425,
+	},
+	{
+		.desc = "good disabled subchannel bitmap (2)",
+		.c = {
+			.width = NL80211_CHAN_WIDTH_160,
+			.chan = &chan_6ghz_105,
+			.center_freq1 = 6475 + 30,
+			.punctured = 0x0003,
+		},
+		.dbe[0] = IEEE80211_UHR_DBE_OPER_BW_320_2 |
+			  IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES,
+		/* DBE disabled subchannel bitmap == 0x0003 */
+		.dbe[1] = 0x03,
+		.cf1 = 6585,
+	},
+};
+
+KUNIT_ARRAY_PARAM_DESC(chandef_dbe, chandef_dbe_cases, desc)
+
+static void test_chandef_dbe(struct kunit *test)
+{
+	const struct chandef_dbe_case *params = test->param_value;
+	struct cfg80211_chan_def c = params->c;
+	int ret;
+
+	KUNIT_EXPECT_EQ(test, cfg80211_chandef_valid(&params->c), true);
+
+	ret = cfg80211_chandef_add_dbe(&c, (void *)params->dbe);
+	KUNIT_EXPECT_EQ(test, ret != 0, params->fails);
+
+	if (params->fails)
+		return;
+
+	KUNIT_EXPECT_EQ(test, c.center_freq1, params->cf1);
+}
+
+static struct kunit_case chandef_test_cases[] = {
 	KUNIT_CASE_PARAM(test_chandef_compat, chandef_compat_gen_params),
+	KUNIT_CASE_PARAM(test_chandef_dbe, chandef_dbe_gen_params),
 	{}
 };
 
-static struct kunit_suite chandef_compat = {
-	.name = "cfg80211-chandef-compat",
-	.test_cases = chandef_compat_test_cases,
+static struct kunit_suite chandef = {
+	.name = "cfg80211-chandef",
+	.test_cases = chandef_test_cases,
 };
 
-kunit_test_suite(chandef_compat);
+kunit_test_suite(chandef);
diff --git a/net/wireless/trace.h b/net/wireless/trace.h
index eb5bedf9c92a4..94944f2a39a4d 100644
--- a/net/wireless/trace.h
+++ b/net/wireless/trace.h
@@ -141,7 +141,9 @@
 		       __field(u32, center_freq1)			\
 		       __field(u32, freq1_offset)			\
 		       __field(u32, center_freq2)			\
-		       __field(u16, punctured)
+		       __field(u16, punctured)				\
+		       __field(u32, npca_pri_freq)			\
+		       __field(u16, npca_punctured)
 #define CHAN_DEF_ASSIGN(chandef)					\
 	do {								\
 		if ((chandef) && (chandef)->chan) {			\
@@ -155,6 +157,11 @@
 			__entry->freq1_offset = (chandef)->freq1_offset;\
 			__entry->center_freq2 = (chandef)->center_freq2;\
 			__entry->punctured = (chandef)->punctured;	\
+			__entry->npca_pri_freq =			\
+				(chandef)->npca_chan ?			\
+				(chandef)->npca_chan->center_freq : 0;	\
+			__entry->npca_punctured =			\
+				(chandef)->npca_punctured;		\
 		} else {						\
 			__entry->band = 0;				\
 			__entry->control_freq = 0;			\
@@ -164,14 +171,17 @@
 			__entry->freq1_offset = 0;			\
 			__entry->center_freq2 = 0;			\
 			__entry->punctured = 0;				\
+			__entry->npca_pri_freq = 0;			\
+			__entry->npca_punctured = 0;			\
 		}							\
 	} while (0)
 #define CHAN_DEF_PR_FMT							\
-	"band: %d, control freq: %u.%03u, width: %d, cf1: %u.%03u, cf2: %u, punct: 0x%x"
+	"band: %d, control freq: %u.%03u, width: %d, cf1: %u.%03u, cf2: %u, punct: 0x%x, npca:%u, npca_punct:0x%x"
 #define CHAN_DEF_PR_ARG __entry->band, __entry->control_freq,		\
 			__entry->freq_offset, __entry->width,		\
 			__entry->center_freq1, __entry->freq1_offset,	\
-			__entry->center_freq2, __entry->punctured
+			__entry->center_freq2, __entry->punctured,	\
+			__entry->npca_pri_freq, __entry->npca_punctured
 
 #define FILS_AAD_ASSIGN(fa)						\
 	do {								\
@@ -2155,22 +2165,26 @@ DEFINE_EVENT(rdev_pmksa, rdev_del_pmksa,
 TRACE_EVENT(rdev_remain_on_channel,
 	TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev,
 		 struct ieee80211_channel *chan,
-		 unsigned int duration),
-	TP_ARGS(wiphy, wdev, chan, duration),
+		 unsigned int duration, const u8 *rx_addr),
+	TP_ARGS(wiphy, wdev, chan, duration, rx_addr),
 	TP_STRUCT__entry(
 		WIPHY_ENTRY
 		WDEV_ENTRY
 		CHAN_ENTRY
 		__field(unsigned int, duration)
+		MAC_ENTRY(rx_addr)
 	),
 	TP_fast_assign(
 		WIPHY_ASSIGN;
 		WDEV_ASSIGN;
 		CHAN_ASSIGN(chan);
 		__entry->duration = duration;
+		MAC_ASSIGN(rx_addr, rx_addr);
 	),
-	TP_printk(WIPHY_PR_FMT ", " WDEV_PR_FMT ", " CHAN_PR_FMT ", duration: %u",
-		  WIPHY_PR_ARG, WDEV_PR_ARG, CHAN_PR_ARG, __entry->duration)
+	TP_printk(WIPHY_PR_FMT ", " WDEV_PR_FMT ", " CHAN_PR_FMT
+		  ", duration: %u, %pM",
+		  WIPHY_PR_ARG, WDEV_PR_ARG, CHAN_PR_ARG, __entry->duration,
+		  __entry->rx_addr)
 );
 
 TRACE_EVENT(rdev_return_int_cookie,
@@ -2371,6 +2385,16 @@ DEFINE_EVENT(wiphy_wdev_evt, rdev_stop_nan,
 	TP_ARGS(wiphy, wdev)
 );
 
+DEFINE_EVENT(wiphy_wdev_evt, rdev_start_pd,
+	TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev),
+	TP_ARGS(wiphy, wdev)
+);
+
+DEFINE_EVENT(wiphy_wdev_evt, rdev_stop_pd,
+	TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev),
+	TP_ARGS(wiphy, wdev)
+);
+
 TRACE_EVENT(rdev_add_nan_func,
 	TP_PROTO(struct wiphy *wiphy, struct wireless_dev *wdev,
 		 const struct cfg80211_nan_func *func),
diff --git a/net/wireless/util.c b/net/wireless/util.c
index cff5a1bd95cc4..8dd7545b90977 100644
--- a/net/wireless/util.c
+++ b/net/wireless/util.c
@@ -284,6 +284,7 @@ bool cfg80211_valid_key_idx(struct cfg80211_registered_device *rdev,
 }
 
 int cfg80211_validate_key_settings(struct cfg80211_registered_device *rdev,
+				   struct wireless_dev *wdev,
 				   struct key_params *params, int key_idx,
 				   bool pairwise, const u8 *mac_addr)
 {
@@ -344,6 +345,15 @@ int cfg80211_validate_key_settings(struct cfg80211_registered_device *rdev,
 		break;
 	}
 
+	/*
+	 * Per Wi-Fi Aware v4.0 section 7.1.2, NAN Data interfaces
+	 * shall only use CCMP-128 or GCMP-256.
+	 */
+	if (wdev->iftype == NL80211_IFTYPE_NAN_DATA &&
+	    params->cipher != WLAN_CIPHER_SUITE_CCMP &&
+	    params->cipher != WLAN_CIPHER_SUITE_GCMP_256)
+		return -EINVAL;
+
 	switch (params->cipher) {
 	case WLAN_CIPHER_SUITE_WEP40:
 		if (params->key_len != WLAN_KEY_LEN_WEP40)
@@ -424,6 +434,21 @@ int cfg80211_validate_key_settings(struct cfg80211_registered_device *rdev,
 	if (!cfg80211_supported_cipher_suite(&rdev->wiphy, params->cipher))
 		return -EINVAL;
 
+	if (params->ltf_keyseed) {
+		if (!wiphy_ext_feature_isset(&rdev->wiphy,
+					     NL80211_EXT_FEATURE_SECURE_LTF) ||
+		    !wiphy_ext_feature_isset(&rdev->wiphy,
+					     NL80211_EXT_FEATURE_SET_KEY_LTF_SEED))
+			return -EOPNOTSUPP;
+
+		/*
+		 * LTF key seed is pairwise key material and must only be
+		 * used with a pairwise key
+		 */
+		if (!pairwise)
+			return -EINVAL;
+	}
+
 	return 0;
 }
 
@@ -1201,7 +1226,8 @@ int cfg80211_change_iface(struct cfg80211_registered_device *rdev,
 
 	/* cannot change into P2P device or NAN */
 	if (ntype == NL80211_IFTYPE_P2P_DEVICE ||
-	    ntype == NL80211_IFTYPE_NAN)
+	    ntype == NL80211_IFTYPE_NAN ||
+	    ntype == NL80211_IFTYPE_PD)
 		return -EOPNOTSUPP;
 
 	if (!rdev->ops->change_virtual_intf ||
@@ -1266,6 +1292,7 @@ int cfg80211_change_iface(struct cfg80211_registered_device *rdev,
 		case NL80211_IFTYPE_P2P_DEVICE:
 		case NL80211_IFTYPE_WDS:
 		case NL80211_IFTYPE_NAN:
+		case NL80211_IFTYPE_PD:
 			WARN_ON(1);
 			break;
 		}
diff --git a/net/wireless/wext-compat.c b/net/wireless/wext-compat.c
index 22d9d9bae8f58..c3aa569772437 100644
--- a/net/wireless/wext-compat.c
+++ b/net/wireless/wext-compat.c
@@ -489,7 +489,8 @@ static int cfg80211_set_encryption(struct cfg80211_registered_device *rdev,
 	if (addr)
 		tx_key = false;
 
-	if (cfg80211_validate_key_settings(rdev, params, idx, pairwise, addr))
+	if (cfg80211_validate_key_settings(rdev, wdev, params, idx,
+					   pairwise, addr))
 		return -EINVAL;
 
 	err = 0;
@@ -789,6 +790,8 @@ static int cfg80211_wext_siwfreq(struct net_device *dev,
 		chandef.chan = ieee80211_get_channel(&rdev->wiphy, freq);
 		if (!chandef.chan)
 			return -EINVAL;
+		if (!cfg80211_chandef_valid(&chandef))
+			return -EINVAL;
 		return cfg80211_set_monitor_channel(rdev, dev, &chandef);
 	case NL80211_IFTYPE_MESH_POINT:
 		freq = cfg80211_wext_freq(wextfreq);
diff --git a/net/x25/af_x25.c b/net/x25/af_x25.c
index af8762b24039d..c31d2af5dd223 100644
--- a/net/x25/af_x25.c
+++ b/net/x25/af_x25.c
@@ -53,6 +53,7 @@
 #include <linux/init.h>
 #include <linux/compat.h>
 #include <linux/ctype.h>
+#include <linux/uio.h>
 
 #include <net/x25.h>
 #include <net/compat.h>
@@ -448,7 +449,7 @@ static int x25_setsockopt(struct socket *sock, int level, int optname,
 }
 
 static int x25_getsockopt(struct socket *sock, int level, int optname,
-			  char __user *optval, int __user *optlen)
+			  sockopt_t *opt)
 {
 	struct sock *sk = sock->sk;
 	int val, len, rc = -ENOPROTOOPT;
@@ -456,22 +457,17 @@ static int x25_getsockopt(struct socket *sock, int level, int optname,
 	if (level != SOL_X25 || optname != X25_QBITINCL)
 		goto out;
 
-	rc = -EFAULT;
-	if (get_user(len, optlen))
-		goto out;
+	len = opt->optlen;
 
 	rc = -EINVAL;
 	if (len < 0)
 		goto out;
 
 	len = min_t(unsigned int, len, sizeof(int));
-
-	rc = -EFAULT;
-	if (put_user(len, optlen))
-		goto out;
+	opt->optlen = len;
 
 	val = test_bit(X25_Q_BIT_FLAG, &x25_sk(sk)->flags);
-	rc = copy_to_user(optval, &val, len) ? -EFAULT : 0;
+	rc = copy_to_iter(&val, len, &opt->iter_out) != len ? -EFAULT : 0;
 out:
 	return rc;
 }
@@ -1753,7 +1749,7 @@ static const struct proto_ops x25_proto_ops = {
 	.listen =	x25_listen,
 	.shutdown =	sock_no_shutdown,
 	.setsockopt =	x25_setsockopt,
-	.getsockopt =	x25_getsockopt,
+	.getsockopt_iter = x25_getsockopt,
 	.sendmsg =	x25_sendmsg,
 	.recvmsg =	x25_recvmsg,
 	.mmap =		sock_no_mmap,
diff --git a/net/xfrm/xfrm_ipcomp.c b/net/xfrm/xfrm_ipcomp.c
index 5f38dff16177c..671d48f8c9374 100644
--- a/net/xfrm/xfrm_ipcomp.c
+++ b/net/xfrm/xfrm_ipcomp.c
@@ -51,11 +51,15 @@ static int ipcomp_post_acomp(struct sk_buff *skb, int err, int hlen)
 	struct scatterlist *dsg;
 	int len, dlen;
 
-	if (unlikely(err))
-		goto out_free_req;
+	if (unlikely(!req))
+		return err;
 
 	extra = acomp_request_extra(req);
 	dsg = extra->sg;
+
+	if (unlikely(err))
+		goto out_free_req;
+
 	dlen = req->dlen;
 
 	pskb_trim_unique(skb, 0);
@@ -84,10 +88,10 @@ static int ipcomp_post_acomp(struct sk_buff *skb, int err, int hlen)
 		skb_shinfo(skb)->nr_frags++;
 	} while ((dlen -= len));
 
-	for (; dsg; dsg = sg_next(dsg))
+out_free_req:
+	for (; dsg && sg_page(dsg); dsg = sg_next(dsg))
 		__free_page(sg_page(dsg));
 
-out_free_req:
 	acomp_request_free(req);
 	return err;
 }
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index c944327ce66c0..59968dcbafe1f 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -4703,7 +4703,7 @@ int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 	}
 
 	/* Stage 5 - announce */
-	km_migrate(sel, dir, type, m, num_migrate, k, encap);
+	km_migrate(sel, dir, type, m, num_migrate, k, net, encap);
 
 	xfrm_pol_put(pol);
 
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 686014d394298..589c3b6e46791 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -2837,7 +2837,7 @@ EXPORT_SYMBOL(km_policy_expired);
 #ifdef CONFIG_XFRM_MIGRATE
 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 	       const struct xfrm_migrate *m, int num_migrate,
-	       const struct xfrm_kmaddress *k,
+	       const struct xfrm_kmaddress *k, struct net *net,
 	       const struct xfrm_encap_tmpl *encap)
 {
 	int err = -EINVAL;
@@ -2848,7 +2848,7 @@ int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 	list_for_each_entry_rcu(km, &xfrm_km_list, list) {
 		if (km->migrate) {
 			ret = km->migrate(sel, dir, type, m, num_migrate, k,
-					  encap);
+					  net, encap);
 			if (!ret)
 				err = ret;
 		}
@@ -3114,10 +3114,14 @@ u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
 	const struct xfrm_type *type = READ_ONCE(x->type);
 	struct crypto_aead *aead;
 	u32 blksize, net_adj = 0;
+	u32 overhead, payload_mtu;
 
 	if (x->km.state != XFRM_STATE_VALID ||
-	    !type || type->proto != IPPROTO_ESP)
+	    !type || type->proto != IPPROTO_ESP) {
+		if (mtu <= x->props.header_len)
+			return 1;
 		return mtu - x->props.header_len;
+	}
 
 	aead = x->data;
 	blksize = ALIGN(crypto_aead_blocksize(aead), 4);
@@ -3140,8 +3144,17 @@ u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
 		break;
 	}
 
-	return ((mtu - x->props.header_len - crypto_aead_authsize(aead) -
-		 net_adj) & ~(blksize - 1)) + net_adj - 2;
+	overhead = x->props.header_len + crypto_aead_authsize(aead) + net_adj;
+	if (mtu <= overhead)
+		return 1;
+
+	payload_mtu = mtu - overhead;
+	payload_mtu &= ~(blksize - 1);
+	if (payload_mtu <= 2)
+		return 1;
+
+	return payload_mtu + net_adj - 2;
+
 }
 EXPORT_SYMBOL_GPL(xfrm_state_mtu);
 
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index 38a90e5ee3d93..2a725ed2ad371 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -2267,9 +2267,8 @@ static int xfrm_add_policy(struct sk_buff *skb, struct nlmsghdr *nlh,
 
 	if (err) {
 		xfrm_dev_policy_delete(xp);
-		xfrm_dev_policy_free(xp);
-		security_xfrm_policy_free(xp->security);
-		kfree(xp);
+		xp->walk.dead = 1;
+		xfrm_policy_destroy(xp);
 		return err;
 	}
 
@@ -3271,10 +3270,9 @@ static int build_migrate(struct sk_buff *skb, const struct xfrm_migrate *m,
 
 static int xfrm_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 			     const struct xfrm_migrate *m, int num_migrate,
-			     const struct xfrm_kmaddress *k,
+			     const struct xfrm_kmaddress *k, struct net *net,
 			     const struct xfrm_encap_tmpl *encap)
 {
-	struct net *net = &init_net;
 	struct sk_buff *skb;
 	int err;
 
@@ -3292,7 +3290,7 @@ static int xfrm_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 #else
 static int xfrm_send_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
 			     const struct xfrm_migrate *m, int num_migrate,
-			     const struct xfrm_kmaddress *k,
+			     const struct xfrm_kmaddress *k, struct net *net,
 			     const struct xfrm_encap_tmpl *encap)
 {
 	return -ENOPROTOOPT;
diff --git a/rust/bindings/bindings_helper.h b/rust/bindings/bindings_helper.h
index 446dbeaf08668..1124785e210b3 100644
--- a/rust/bindings/bindings_helper.h
+++ b/rust/bindings/bindings_helper.h
@@ -35,6 +35,7 @@
 #include <drm/drm_file.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_gem_shmem_helper.h>
+#include <drm/drm_gpuvm.h>
 #include <drm/drm_ioctl.h>
 #include <kunit/test.h>
 #include <linux/auxiliary_bus.h>
diff --git a/rust/helpers/acpi.c b/rust/helpers/acpi.c
new file mode 100644
index 0000000000000..e75c9807bbadc
--- /dev/null
+++ b/rust/helpers/acpi.c
@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/acpi.h>
+#include <acpi/acpi_bus.h>
+
+__rust_helper bool rust_helper_acpi_of_match_device(const struct acpi_device *adev,
+						    const struct of_device_id *of_match_table,
+						    const struct of_device_id **of_id)
+{
+	return acpi_of_match_device(adev, of_match_table, of_id);
+}
+
+__rust_helper struct acpi_device *rust_helper_to_acpi_device_node(struct fwnode_handle *fwnode)
+{
+	return to_acpi_device_node(fwnode);
+}
diff --git a/rust/helpers/drm_gpuvm.c b/rust/helpers/drm_gpuvm.c
new file mode 100644
index 0000000000000..4130b63252138
--- /dev/null
+++ b/rust/helpers/drm_gpuvm.c
@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: GPL-2.0 or MIT
+
+#ifdef CONFIG_RUST_DRM_GPUVM
+
+#include <drm/drm_gpuvm.h>
+
+__rust_helper
+struct drm_gpuvm_bo *rust_helper_drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo)
+{
+	return drm_gpuvm_bo_get(vm_bo);
+}
+
+__rust_helper
+struct drm_gpuvm *rust_helper_drm_gpuvm_get(struct drm_gpuvm *obj)
+{
+	return drm_gpuvm_get(obj);
+}
+
+__rust_helper
+bool rust_helper_drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm,
+				     struct drm_gem_object *obj)
+{
+	return drm_gpuvm_is_extobj(gpuvm, obj);
+}
+
+#endif // CONFIG_RUST_DRM_GPUVM
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index 625921e27dfbc..337da1ba94628 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -38,6 +38,7 @@
 #define __rust_helper __always_inline
 #endif
 
+#include "acpi.c"
 #include "atomic.c"
 #include "atomic_ext.c"
 #include "auxiliary.c"
@@ -59,6 +60,7 @@
 #include "dma.c"
 #include "dma-resv.c"
 #include "drm.c"
+#include "drm_gpuvm.c"
 #include "err.c"
 #include "irq.c"
 #include "fs.c"
diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c
index d9625b9ee0466..6e59cc9c665cc 100644
--- a/rust/helpers/uaccess.c
+++ b/rust/helpers/uaccess.c
@@ -14,7 +14,7 @@ rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n)
 	return copy_to_user(to, from, n);
 }
 
-#ifdef INLINE_COPY_FROM_USER
+#ifdef INLINE_COPY_USER
 __rust_helper
 unsigned long rust_helper__copy_from_user(void *to, const void __user *from, unsigned long n)
 {
diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs
index e38720349dcf7..21067bde68607 100644
--- a/rust/kernel/alloc.rs
+++ b/rust/kernel/alloc.rs
@@ -22,8 +22,12 @@
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub struct AllocError;
 
-use crate::error::{code::EINVAL, Result};
-use core::{alloc::Layout, ptr::NonNull};
+use crate::prelude::*;
+
+use core::{
+    alloc::Layout,
+    ptr::NonNull, //
+};
 
 /// Flags to be used when allocating memory.
 ///
diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs
index 63bfb91b36712..562e41925ada5 100644
--- a/rust/kernel/alloc/allocator.rs
+++ b/rust/kernel/alloc/allocator.rs
@@ -8,14 +8,25 @@
 //!
 //! Reference: <https://docs.kernel.org/core-api/memory-allocation.html>
 
-use super::Flags;
-use core::alloc::Layout;
-use core::ptr;
-use core::ptr::NonNull;
-
-use crate::alloc::{AllocError, Allocator, NumaNode};
-use crate::bindings;
-use crate::page;
+use super::{
+    AllocError,
+    Allocator,
+    Flags,
+    NumaNode, //
+};
+
+use crate::{
+    bindings,
+    page, //
+};
+
+use core::{
+    alloc::Layout,
+    ptr::{
+        self,
+        NonNull, //
+    }, //
+};
 
 const ARCH_KMALLOC_MINALIGN: usize = bindings::ARCH_KMALLOC_MINALIGN;
 
@@ -163,8 +174,11 @@ impl Vmalloc {
     /// # Examples
     ///
     /// ```
-    /// # use core::ptr::{NonNull, from_mut};
-    /// # use kernel::{page, prelude::*};
+    /// # use core::ptr::{
+    /// #     from_mut,
+    /// #     NonNull, //
+    /// # };
+    /// # use kernel::page;
     /// use kernel::alloc::allocator::Vmalloc;
     ///
     /// let mut vbox = VBox::<[u8; page::PAGE_SIZE]>::new_uninit(GFP_KERNEL)?;
diff --git a/rust/kernel/alloc/allocator/iter.rs b/rust/kernel/alloc/allocator/iter.rs
index e0a70b7a744af..02fda3ea5cae6 100644
--- a/rust/kernel/alloc/allocator/iter.rs
+++ b/rust/kernel/alloc/allocator/iter.rs
@@ -1,9 +1,13 @@
 // SPDX-License-Identifier: GPL-2.0
 
 use super::Vmalloc;
+
 use crate::page;
-use core::marker::PhantomData;
-use core::ptr::NonNull;
+
+use core::{
+    marker::PhantomData,
+    ptr::NonNull, //
+};
 
 /// An [`Iterator`] of [`page::BorrowedPage`] items owned by a [`Vmalloc`] allocation.
 ///
diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs
index bd6da02c7ab8f..76b340e361bb3 100644
--- a/rust/kernel/alloc/kbox.rs
+++ b/rust/kernel/alloc/kbox.rs
@@ -3,24 +3,47 @@
 //! Implementation of [`Box`].
 
 #[allow(unused_imports)] // Used in doc comments.
-use super::allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter};
-use super::{AllocError, Allocator, Flags, NumaNode};
-use core::alloc::Layout;
-use core::borrow::{Borrow, BorrowMut};
-use core::marker::PhantomData;
-use core::mem::ManuallyDrop;
-use core::mem::MaybeUninit;
-use core::ops::{Deref, DerefMut};
-use core::pin::Pin;
-use core::ptr::NonNull;
-use core::result::Result;
-
-use crate::ffi::c_void;
-use crate::fmt;
-use crate::init::InPlaceInit;
-use crate::page::AsPageIter;
-use crate::types::ForeignOwnable;
-use pin_init::{InPlaceWrite, Init, PinInit, ZeroableOption};
+use super::allocator::{
+    KVmalloc,
+    Kmalloc,
+    Vmalloc,
+    VmallocPageIter, //
+};
+
+use super::{
+    AllocError,
+    Allocator,
+    Flags,
+    NumaNode, //
+};
+
+use crate::{
+    fmt,
+    page::AsPageIter,
+    prelude::*,
+    types::ForeignOwnable, //
+};
+
+use core::{
+    alloc::Layout,
+    borrow::{
+        Borrow,
+        BorrowMut, //
+    },
+    marker::PhantomData,
+    mem::{
+        ManuallyDrop,
+        MaybeUninit, //
+    },
+    ops::{
+        Deref,
+        DerefMut, //
+    },
+    ptr::NonNull,
+    result::Result, //
+};
+
+use pin_init::ZeroableOption;
 
 /// The kernel's [`Box`] type -- a heap allocation for a single value of type `T`.
 ///
@@ -256,6 +279,27 @@ pub fn new_uninit(flags: Flags) -> Result<Box<MaybeUninit<T>, A>, AllocError> {
         Ok(Box(ptr.cast(), PhantomData))
     }
 
+    /// Creates a new zero-initialized `Box<T, A>`.
+    ///
+    /// New memory is allocated with `A` and the [`__GFP_ZERO`] flag. The allocation may fail, in
+    /// which case an error is returned. For ZSTs no memory is allocated.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let b = KBox::<[u8; 128]>::zeroed(GFP_KERNEL)?;
+    /// assert_eq!(*b, [0; 128]);
+    /// # Ok::<(), Error>(())
+    /// ```
+    pub fn zeroed(flags: Flags) -> Result<Self, AllocError>
+    where
+        T: Zeroable,
+    {
+        // SAFETY: `__GFP_ZERO` guarantees the memory is zeroed; `T: Zeroable` guarantees that
+        // all-zeroes is a valid bit pattern for `T`.
+        Ok(unsafe { Self::new_uninit(flags | __GFP_ZERO)?.assume_init() })
+    }
+
     /// Constructs a new `Pin<Box<T, A>>`. If `T` does not implement [`Unpin`], then `x` will be
     /// pinned in memory and can't be moved.
     #[inline]
@@ -274,7 +318,10 @@ pub fn pin(x: T, flags: Flags) -> Result<Pin<Box<T, A>>, AllocError>
     /// # Examples
     ///
     /// ```
-    /// use kernel::sync::{new_spinlock, SpinLock};
+    /// use kernel::sync::{
+    ///     new_spinlock,
+    ///     SpinLock, //
+    /// };
     ///
     /// struct Inner {
     ///     a: u32,
@@ -567,7 +614,6 @@ fn deref_mut(&mut self) -> &mut T {
 ///
 /// ```
 /// # use core::borrow::Borrow;
-/// # use kernel::alloc::KBox;
 /// struct Foo<B: Borrow<u32>>(B);
 ///
 /// // Owned instance.
@@ -595,7 +641,6 @@ fn borrow(&self) -> &T {
 ///
 /// ```
 /// # use core::borrow::BorrowMut;
-/// # use kernel::alloc::KBox;
 /// struct Foo<B: BorrowMut<u32>>(B);
 ///
 /// // Owned instance.
@@ -660,9 +705,13 @@ fn drop(&mut self) {
 /// # Examples
 ///
 /// ```
-/// # use kernel::prelude::*;
-/// use kernel::alloc::allocator::VmallocPageIter;
-/// use kernel::page::{AsPageIter, PAGE_SIZE};
+/// use kernel::{
+///     alloc::allocator::VmallocPageIter,
+///     page::{
+///         AsPageIter,
+///         PAGE_SIZE, //
+///     }, //
+/// };
 ///
 /// let mut vbox = VBox::new((), GFP_KERNEL)?;
 ///
diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs
index 6438385e4322e..6f35484387de9 100644
--- a/rust/kernel/alloc/kvec.rs
+++ b/rust/kernel/alloc/kvec.rs
@@ -3,29 +3,52 @@
 //! Implementation of [`Vec`].
 
 use super::{
-    allocator::{KVmalloc, Kmalloc, Vmalloc, VmallocPageIter},
+    allocator::{
+        KVmalloc,
+        Kmalloc,
+        Vmalloc,
+        VmallocPageIter, //
+    },
     layout::ArrayLayout,
-    AllocError, Allocator, Box, Flags, NumaNode,
+    AllocError,
+    Allocator,
+    Box,
+    Flags,
+    NumaNode, //
 };
+
 use crate::{
     fmt,
     page::{
         AsPageIter,
         PAGE_SIZE, //
-    },
+    }, //
 };
+
 use core::{
-    borrow::{Borrow, BorrowMut},
+    borrow::{
+        Borrow,
+        BorrowMut, //
+    },
     marker::PhantomData,
-    mem::{ManuallyDrop, MaybeUninit},
-    ops::Deref,
-    ops::DerefMut,
-    ops::Index,
-    ops::IndexMut,
-    ptr,
-    ptr::NonNull,
-    slice,
-    slice::SliceIndex,
+    mem::{
+        ManuallyDrop,
+        MaybeUninit, //
+    },
+    ops::{
+        Deref,
+        DerefMut,
+        Index,
+        IndexMut, //
+    },
+    ptr::{
+        self,
+        NonNull, //
+    },
+    slice::{
+        self,
+        SliceIndex, //
+    }, //
 };
 
 mod errors;
@@ -614,7 +637,7 @@ pub fn clear(&mut self) {
     ///
     /// v.reserve(10, GFP_KERNEL)?;
     /// let cap = v.capacity();
-    /// assert!(cap >= 10);
+    /// assert!(cap >= v.len() + 10);
     ///
     /// v.reserve(10, GFP_KERNEL)?;
     /// let new_cap = v.capacity();
@@ -849,6 +872,24 @@ pub fn shrink_to(&mut self, min_capacity: usize, flags: Flags) -> Result<(), All
 
 impl<T: Clone, A: Allocator> Vec<T, A> {
     /// Extend the vector by `n` clones of `value`.
+    ///
+    /// # Examples
+    ///
+    /// ```
+    /// let mut v = KVec::new();
+    /// v.push(1, GFP_KERNEL)?;
+    ///
+    /// v.extend_with(3, 5, GFP_KERNEL)?;
+    /// assert_eq!(&v, &[1, 5, 5, 5]);
+    ///
+    /// v.extend_with(2, 8, GFP_KERNEL)?;
+    /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]);
+    ///
+    /// v.extend_with(0, 3, GFP_KERNEL)?;
+    /// assert_eq!(&v, &[1, 5, 5, 5, 8, 8]);
+    ///
+    /// # Ok::<(), Error>(())
+    /// ```
     pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), AllocError> {
         if n == 0 {
             return Ok(());
@@ -866,7 +907,7 @@ pub fn extend_with(&mut self, n: usize, value: T, flags: Flags) -> Result<(), Al
         spare[n - 1].write(value);
 
         // SAFETY:
-        // - `self.len() + n < self.capacity()` due to the call to reserve above,
+        // - `self.len() + n <= self.capacity()` due to the call to reserve above,
         // - the loop and the line above initialized the next `n` elements.
         unsafe { self.inc_len(n) };
 
@@ -1146,9 +1187,13 @@ fn into_iter(self) -> Self::IntoIter {
 /// # Examples
 ///
 /// ```
-/// # use kernel::prelude::*;
-/// use kernel::alloc::allocator::VmallocPageIter;
-/// use kernel::page::{AsPageIter, PAGE_SIZE};
+/// use kernel::{
+///     alloc::allocator::VmallocPageIter,
+///     page::{
+///         AsPageIter,
+///         PAGE_SIZE, //
+///     }, //
+/// };
 ///
 /// let mut vec = VVec::<u8>::new();
 ///
diff --git a/rust/kernel/alloc/kvec/errors.rs b/rust/kernel/alloc/kvec/errors.rs
index 985c5f2c3962f..aaca6446516af 100644
--- a/rust/kernel/alloc/kvec/errors.rs
+++ b/rust/kernel/alloc/kvec/errors.rs
@@ -2,8 +2,10 @@
 
 //! Errors for the [`Vec`] type.
 
-use kernel::fmt;
-use kernel::prelude::*;
+use crate::{
+    fmt,
+    prelude::*, //
+};
 
 /// Error type for [`Vec::push_within_capacity`].
 pub struct PushError<T>(pub T);
diff --git a/rust/kernel/alloc/layout.rs b/rust/kernel/alloc/layout.rs
index 9f8be72feb7a8..62a459c66bafe 100644
--- a/rust/kernel/alloc/layout.rs
+++ b/rust/kernel/alloc/layout.rs
@@ -4,7 +4,10 @@
 //!
 //! Custom layout types extending or improving [`Layout`].
 
-use core::{alloc::Layout, marker::PhantomData};
+use core::{
+    alloc::Layout,
+    marker::PhantomData, //
+};
 
 /// Error when constructing an [`ArrayLayout`].
 pub struct LayoutError;
@@ -47,7 +50,10 @@ pub const fn empty() -> Self {
     /// # Examples
     ///
     /// ```
-    /// # use kernel::alloc::layout::{ArrayLayout, LayoutError};
+    /// # use kernel::alloc::layout::{
+    /// #     ArrayLayout,
+    /// #     LayoutError, //
+    /// # };
     /// let layout = ArrayLayout::<i32>::new(15)?;
     /// assert_eq!(layout.len(), 15);
     ///
diff --git a/rust/kernel/auxiliary.rs b/rust/kernel/auxiliary.rs
index 93c0db1f66555..19aec94aa95b7 100644
--- a/rust/kernel/auxiliary.rs
+++ b/rust/kernel/auxiliary.rs
@@ -19,12 +19,17 @@
         to_result, //
     },
     prelude::*,
-    types::Opaque,
+    types::{
+        ForeignOwnable,
+        Opaque, //
+    },
     ThisModule, //
 };
 use core::{
+    any::TypeId,
     marker::PhantomData,
     mem::offset_of,
+    pin::Pin,
     ptr::{
         addr_of_mut,
         NonNull, //
@@ -257,6 +262,40 @@ pub fn parent(&self) -> &device::Device<device::Bound> {
         // SAFETY: A bound auxiliary device always has a bound parent device.
         unsafe { parent.as_bound() }
     }
+
+    /// Returns a pinned reference to the registration data set by the registering (parent) driver.
+    ///
+    /// Returns [`EINVAL`] if `T` does not match the type used by the parent driver when calling
+    /// [`Registration::new()`].
+    ///
+    /// Returns [`ENOENT`] if no registration data has been set, e.g. when the device was
+    /// registered by a C driver.
+    pub fn registration_data<T: 'static>(&self) -> Result<Pin<&T>> {
+        // SAFETY: By the type invariant, `self.as_raw()` is a valid `struct auxiliary_device`.
+        let ptr = unsafe { (*self.as_raw()).registration_data_rust };
+        if ptr.is_null() {
+            dev_warn!(
+                self.as_ref(),
+                "No registration data set; parent is not a Rust driver.\n"
+            );
+            return Err(ENOENT);
+        }
+
+        // SAFETY: `ptr` is non-null and was set via `into_foreign()` in `Registration::new()`;
+        // `RegistrationData` is `#[repr(C)]` with `type_id` at offset 0, so reading a `TypeId`
+        // at the start of the allocation is valid regardless of `T`.
+        let type_id = unsafe { ptr.cast::<TypeId>().read() };
+        if type_id != TypeId::of::<T>() {
+            return Err(EINVAL);
+        }
+
+        // SAFETY: The `TypeId` check above confirms that the stored type is `T`; `ptr` remains
+        // valid until `Registration::drop()` calls `from_foreign()`.
+        let wrapper = unsafe { Pin::<KBox<RegistrationData<T>>>::borrow(ptr) };
+
+        // SAFETY: `data` is a structurally pinned field of `RegistrationData`.
+        Ok(unsafe { wrapper.map_unchecked(|w| &w.data) })
+    }
 }
 
 impl Device {
@@ -326,87 +365,132 @@ unsafe impl Send for Device {}
 // (i.e. `Device<Normal>) are thread safe.
 unsafe impl Sync for Device {}
 
+/// Wrapper that stores a [`TypeId`] alongside the registration data for runtime type checking.
+#[repr(C)]
+#[pin_data]
+struct RegistrationData<T> {
+    type_id: TypeId,
+    #[pin]
+    data: T,
+}
+
 /// The registration of an auxiliary device.
 ///
 /// This type represents the registration of a [`struct auxiliary_device`]. When its parent device
 /// is unbound, the corresponding auxiliary device will be unregistered from the system.
 ///
+/// The type parameter `T` is the type of the registration data owned by the registering (parent)
+/// driver. It can be accessed by the auxiliary driver through
+/// [`Device::registration_data()`].
+///
 /// # Invariants
 ///
-/// `self.0` always holds a valid pointer to an initialized and registered
-/// [`struct auxiliary_device`].
-pub struct Registration(NonNull<bindings::auxiliary_device>);
-
-impl Registration {
-    /// Create and register a new auxiliary device.
-    pub fn new<'a>(
-        parent: &'a device::Device<device::Bound>,
-        name: &'a CStr,
+/// `self.adev` always holds a valid pointer to an initialized and registered
+/// [`struct auxiliary_device`] whose `registration_data_rust` field points to a
+/// valid `Pin<KBox<RegistrationData<T>>>`.
+pub struct Registration<T: 'static> {
+    adev: NonNull<bindings::auxiliary_device>,
+    _data: PhantomData<T>,
+}
+
+impl<T: Send + Sync + 'static> Registration<T> {
+    /// Create and register a new auxiliary device with the given registration data.
+    ///
+    /// The `data` is owned by the registration and can be accessed through the auxiliary device
+    /// via [`Device::registration_data()`].
+    pub fn new<E>(
+        parent: &device::Device<device::Bound>,
+        name: &CStr,
         id: u32,
-        modname: &'a CStr,
-    ) -> impl PinInit<Devres<Self>, Error> + 'a {
-        pin_init::pin_init_scope(move || {
-            let boxed = KBox::new(Opaque::<bindings::auxiliary_device>::zeroed(), GFP_KERNEL)?;
-            let adev = boxed.get();
-
-            // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization.
-            unsafe {
-                (*adev).dev.parent = parent.as_raw();
-                (*adev).dev.release = Some(Device::release);
-                (*adev).name = name.as_char_ptr();
-                (*adev).id = id;
-            }
-
-            // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`,
-            // which has not been initialized yet.
-            unsafe { bindings::auxiliary_device_init(adev) };
-
-            // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be
-            // freed by `Device::release` when the last reference to the `struct auxiliary_device`
-            // is dropped.
-            let _ = KBox::into_raw(boxed);
-
-            // SAFETY:
-            // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which
-            //   has been initialized,
-            // - `modname.as_char_ptr()` is a NULL terminated string.
-            let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) };
-            if ret != 0 {
-                // SAFETY: `adev` is guaranteed to be a valid pointer to a
-                // `struct auxiliary_device`, which has been initialized.
-                unsafe { bindings::auxiliary_device_uninit(adev) };
-
-                return Err(Error::from_errno(ret));
-            }
-
-            // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is
-            // called, which happens in `Self::drop()`.
-            Ok(Devres::new(
-                parent,
-                // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated
-                // successfully.
-                Self(unsafe { NonNull::new_unchecked(adev) }),
-            ))
-        })
+        modname: &CStr,
+        data: impl PinInit<T, E>,
+    ) -> Result<Devres<Self>>
+    where
+        Error: From<E>,
+    {
+        let data = KBox::pin_init::<Error>(
+            try_pin_init!(RegistrationData {
+                type_id: TypeId::of::<T>(),
+                data <- data,
+            }),
+            GFP_KERNEL,
+        )?;
+
+        let boxed: KBox<Opaque<bindings::auxiliary_device>> = KBox::zeroed(GFP_KERNEL)?;
+        let adev = boxed.get();
+
+        // SAFETY: It's safe to set the fields of `struct auxiliary_device` on initialization.
+        unsafe {
+            (*adev).dev.parent = parent.as_raw();
+            (*adev).dev.release = Some(Device::release);
+            (*adev).name = name.as_char_ptr();
+            (*adev).id = id;
+            (*adev).registration_data_rust = data.into_foreign();
+        }
+
+        // SAFETY: `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`,
+        // which has not been initialized yet.
+        unsafe { bindings::auxiliary_device_init(adev) };
+
+        // Now that `adev` is initialized, leak the `Box`; the corresponding memory will be
+        // freed by `Device::release` when the last reference to the `struct auxiliary_device`
+        // is dropped.
+        let _ = KBox::into_raw(boxed);
+
+        // SAFETY:
+        // - `adev` is guaranteed to be a valid pointer to a `struct auxiliary_device`, which
+        //   has been initialized,
+        // - `modname.as_char_ptr()` is a NULL terminated string.
+        let ret = unsafe { bindings::__auxiliary_device_add(adev, modname.as_char_ptr()) };
+        if ret != 0 {
+            // SAFETY: `registration_data` was set above via `into_foreign()`.
+            drop(unsafe {
+                Pin::<KBox<RegistrationData<T>>>::from_foreign((*adev).registration_data_rust)
+            });
+
+            // SAFETY: `adev` is guaranteed to be a valid pointer to a
+            // `struct auxiliary_device`, which has been initialized.
+            unsafe { bindings::auxiliary_device_uninit(adev) };
+
+            return Err(Error::from_errno(ret));
+        }
+
+        // INVARIANT: The device will remain registered until `auxiliary_device_delete()` is
+        // called, which happens in `Self::drop()`.
+        let reg = Self {
+            // SAFETY: `adev` is guaranteed to be non-null, since the `KBox` was allocated
+            // successfully.
+            adev: unsafe { NonNull::new_unchecked(adev) },
+            _data: PhantomData,
+        };
+
+        Devres::new::<core::convert::Infallible>(parent, reg)
     }
 }
 
-impl Drop for Registration {
+impl<T: 'static> Drop for Registration<T> {
     fn drop(&mut self) {
-        // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered
+        // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered
         // `struct auxiliary_device`.
-        unsafe { bindings::auxiliary_device_delete(self.0.as_ptr()) };
+        unsafe { bindings::auxiliary_device_delete(self.adev.as_ptr()) };
+
+        // SAFETY: `registration_data` was set in `new()` via `into_foreign()`.
+        drop(unsafe {
+            Pin::<KBox<RegistrationData<T>>>::from_foreign(
+                (*self.adev.as_ptr()).registration_data_rust,
+            )
+        });
 
         // This drops the reference we acquired through `auxiliary_device_init()`.
         //
-        // SAFETY: By the type invariant of `Self`, `self.0.as_ptr()` is a valid registered
+        // SAFETY: By the type invariant of `Self`, `self.adev.as_ptr()` is a valid registered
         // `struct auxiliary_device`.
-        unsafe { bindings::auxiliary_device_uninit(self.0.as_ptr()) };
+        unsafe { bindings::auxiliary_device_uninit(self.adev.as_ptr()) };
     }
 }
 
 // SAFETY: A `Registration` of a `struct auxiliary_device` can be released from any thread.
-unsafe impl Send for Registration {}
+unsafe impl<T: Send + Sync> Send for Registration<T> {}
 
 // SAFETY: `Registration` does not expose any methods or fields that need synchronization.
-unsafe impl Sync for Registration {}
+unsafe impl<T: Send + Sync> Sync for Registration<T> {}
diff --git a/rust/kernel/device.rs b/rust/kernel/device.rs
index 6d5396a43ebea..fd50399aadeab 100644
--- a/rust/kernel/device.rs
+++ b/rust/kernel/device.rs
@@ -15,16 +15,12 @@
     }, //
 };
 use core::{
-    any::TypeId,
     marker::PhantomData,
     ptr, //
 };
 
 pub mod property;
 
-// Assert that we can `read()` / `write()` a `TypeId` instance from / into `struct driver_type`.
-static_assert!(core::mem::size_of::<bindings::driver_type>() >= core::mem::size_of::<TypeId>());
-
 /// The core representation of a device in the kernel's driver model.
 ///
 /// This structure represents the Rust abstraction for a C `struct device`. A [`Device`] can either
@@ -206,29 +202,12 @@ pub unsafe fn as_bound(&self) -> &Device<Bound> {
 }
 
 impl Device<CoreInternal> {
-    fn set_type_id<T: 'static>(&self) {
-        // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`.
-        let private = unsafe { (*self.as_raw()).p };
-
-        // SAFETY: For a bound device (implied by the `CoreInternal` device context), `private` is
-        // guaranteed to be a valid pointer to a `struct device_private`.
-        let driver_type = unsafe { &raw mut (*private).driver_type };
-
-        // SAFETY: `driver_type` is valid for (unaligned) writes of a `TypeId`.
-        unsafe {
-            driver_type
-                .cast::<TypeId>()
-                .write_unaligned(TypeId::of::<T>())
-        };
-    }
-
     /// Store a pointer to the bound driver's private data.
     pub fn set_drvdata<T: 'static>(&self, data: impl PinInit<T, Error>) -> Result {
         let data = KBox::pin_init(data, GFP_KERNEL)?;
 
         // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`.
         unsafe { bindings::dev_set_drvdata(self.as_raw(), data.into_foreign().cast()) };
-        self.set_type_id::<T>();
 
         Ok(())
     }
@@ -292,45 +271,6 @@ unsafe fn drvdata_unchecked<T: 'static>(&self) -> Pin<&T> {
         //   in `into_foreign()`.
         unsafe { Pin::<KBox<T>>::borrow(ptr.cast()) }
     }
-
-    fn match_type_id<T: 'static>(&self) -> Result {
-        // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`.
-        let private = unsafe { (*self.as_raw()).p };
-
-        // SAFETY: For a bound device, `private` is guaranteed to be a valid pointer to a
-        // `struct device_private`.
-        let driver_type = unsafe { &raw mut (*private).driver_type };
-
-        // SAFETY:
-        // - `driver_type` is valid for (unaligned) reads of a `TypeId`.
-        // - A bound device guarantees that `driver_type` contains a valid `TypeId` value.
-        let type_id = unsafe { driver_type.cast::<TypeId>().read_unaligned() };
-
-        if type_id != TypeId::of::<T>() {
-            return Err(EINVAL);
-        }
-
-        Ok(())
-    }
-
-    /// Access a driver's private data.
-    ///
-    /// Returns a pinned reference to the driver's private data or [`EINVAL`] if it doesn't match
-    /// the asserted type `T`.
-    pub fn drvdata<T: 'static>(&self) -> Result<Pin<&T>> {
-        // SAFETY: By the type invariants, `self.as_raw()` is a valid pointer to a `struct device`.
-        if unsafe { bindings::dev_get_drvdata(self.as_raw()) }.is_null() {
-            return Err(ENOENT);
-        }
-
-        self.match_type_id::<T>()?;
-
-        // SAFETY:
-        // - The above check of `dev_get_drvdata()` guarantees that we are called after
-        //   `set_drvdata()`.
-        // - We've just checked that the type of the driver's private data is in fact `T`.
-        Ok(unsafe { self.drvdata_unchecked() })
-    }
 }
 
 impl<Ctx: DeviceContext> Device<Ctx> {
diff --git a/rust/kernel/driver.rs b/rust/kernel/driver.rs
index 36de8098754d0..93e5dd6ae3715 100644
--- a/rust/kernel/driver.rs
+++ b/rust/kernel/driver.rs
@@ -278,6 +278,26 @@ fn init(
     }
 }
 
+// Calling the FFI function directly from the `Adapter` impl may result in it being called
+// directly from driver modules. This happens since the Rust compiler will use monomorphisation, so
+// it might happen that functions are instantiated within the calling driver module. For now, work
+// around this with `#[inline(never)]` helpers.
+//
+// TODO: Remove once a more generic solution has been implemented. For instance, we may be able to
+// leverage `bindgen` to take care of this depending on whether a symbol is (already) exported.
+#[inline(never)]
+#[allow(clippy::missing_safety_doc)]
+#[allow(dead_code)]
+#[must_use]
+unsafe fn acpi_of_match_device(
+    adev: *const bindings::acpi_device,
+    of_match_table: *const bindings::of_device_id,
+    of_id: *mut *const bindings::of_device_id,
+) -> bool {
+    // SAFETY: Safety requirements are the same as `bindings::acpi_of_match_device`.
+    unsafe { bindings::acpi_of_match_device(adev, of_match_table, of_id) }
+}
+
 /// The bus independent adapter to match a drivers and a devices.
 ///
 /// This trait should be implemented by the bus specific adapter, which represents the connection
@@ -329,35 +349,63 @@ fn acpi_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> {
     ///
     /// If this returns `None`, it means there is no match with an entry in the [`of::IdTable`].
     fn of_id_info(dev: &device::Device) -> Option<&'static Self::IdInfo> {
-        #[cfg(not(CONFIG_OF))]
+        let table = Self::of_id_table()?;
+
+        #[cfg(not(any(CONFIG_OF, CONFIG_ACPI)))]
         {
-            let _ = dev;
-            None
+            let _ = (dev, table);
         }
 
         #[cfg(CONFIG_OF)]
         {
-            let table = Self::of_id_table()?;
-
             // SAFETY:
             // - `table` has static lifetime, hence it's valid for read,
             // - `dev` is guaranteed to be valid while it's alive, and so is `dev.as_raw()`.
             let raw_id = unsafe { bindings::of_match_device(table.as_ptr(), dev.as_raw()) };
 
-            if raw_id.is_null() {
-                None
-            } else {
+            if !raw_id.is_null() {
                 // SAFETY: `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id`
                 // and does not add additional invariants, so it's safe to transmute.
                 let id = unsafe { &*raw_id.cast::<of::DeviceId>() };
 
-                Some(
-                    table.info(<of::DeviceId as crate::device_id::RawDeviceIdIndex>::index(
-                        id,
-                    )),
-                )
+                return Some(table.info(
+                    <of::DeviceId as crate::device_id::RawDeviceIdIndex>::index(id),
+                ));
             }
         }
+
+        #[cfg(CONFIG_ACPI)]
+        {
+            use core::ptr;
+            use device::property::FwNode;
+
+            let mut raw_id = ptr::null();
+
+            let fwnode = dev.fwnode().map_or(ptr::null_mut(), FwNode::as_raw);
+
+            // SAFETY: `fwnode` is a pointer to a valid `fwnode_handle`. A null pointer will be
+            // passed through the function.
+            let adev = unsafe { bindings::to_acpi_device_node(fwnode) };
+
+            // SAFETY:
+            // - `adev` is a valid pointer to `acpi_device` or is null. It is guaranteed to be
+            //   valid as long as `dev` is alive.
+            // - `table` has static lifetime, hence it's valid for read.
+            if unsafe { acpi_of_match_device(adev, table.as_ptr(), &raw mut raw_id) } {
+                // SAFETY:
+                // - the function returns true, therefore `raw_id` has been set to a pointer to a
+                //   valid `of_device_id`.
+                // - `DeviceId` is a `#[repr(transparent)]` wrapper of `struct of_device_id`
+                //   and does not add additional invariants, so it's safe to transmute.
+                let id = unsafe { &*raw_id.cast::<of::DeviceId>() };
+
+                return Some(table.info(
+                    <of::DeviceId as crate::device_id::RawDeviceIdIndex>::index(id),
+                ));
+            }
+        }
+
+        None
     }
 
     /// Returns the driver's private data from the matching entry of any of the ID tables, if any.
diff --git a/rust/kernel/drm/gpuvm/mod.rs b/rust/kernel/drm/gpuvm/mod.rs
new file mode 100644
index 0000000000000..ae58f6f667c1f
--- /dev/null
+++ b/rust/kernel/drm/gpuvm/mod.rs
@@ -0,0 +1,328 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+#![cfg(CONFIG_RUST_DRM_GPUVM)]
+
+//! DRM GPUVM in immediate mode
+//!
+//! Rust abstractions for using GPUVM in immediate mode. This is when the GPUVM state is updated
+//! during `run_job()`, i.e., in the DMA fence signalling critical path, to ensure that the GPUVM
+//! and the GPU's virtual address space has the same state at all times.
+//!
+//! C header: [`include/drm/drm_gpuvm.h`](srctree/include/drm/drm_gpuvm.h)
+
+use kernel::{
+    alloc::{
+        AllocError,
+        Flags as AllocFlags, //
+    },
+    bindings,
+    drm,
+    drm::gem::IntoGEMObject,
+    error::to_result,
+    prelude::*,
+    sync::aref::{
+        ARef,
+        AlwaysRefCounted, //
+    },
+    types::Opaque, //
+};
+
+use core::{
+    cell::UnsafeCell,
+    marker::PhantomData,
+    mem::{
+        ManuallyDrop,
+        MaybeUninit, //
+    },
+    ops::{
+        Deref,
+        DerefMut,
+        Range, //
+    },
+    ptr::{
+        self,
+        NonNull, //
+    }, //
+};
+
+mod sm_ops;
+pub use self::sm_ops::*;
+
+mod vm_bo;
+pub use self::vm_bo::*;
+
+mod va;
+pub use self::va::*;
+
+/// A DRM GPU VA manager.
+///
+/// This object is refcounted, but the locations of mapped ranges may only be accessed or changed
+/// via the special unique handle [`UniqueRefGpuVm`].
+///
+/// # Invariants
+///
+/// * Stored in an allocation managed by the refcount in `self.vm`.
+/// * Access to `data` and the gpuvm interval tree is controlled via the [`UniqueRefGpuVm`] type.
+/// * Does not contain any sparse [`GpuVa<T>`] instances.
+#[pin_data]
+pub struct GpuVm<T: DriverGpuVm> {
+    #[pin]
+    vm: Opaque<bindings::drm_gpuvm>,
+    /// Accessed only through the [`UniqueRefGpuVm`] reference.
+    data: UnsafeCell<T>,
+}
+
+// SAFETY: The GPUVM api does not assume that it is tied to a specific thread. The destructor will
+// drop the `data` field, which is okay because it is guaranteed `Send` by the `DriverGpuVm` trait.
+unsafe impl<T: DriverGpuVm> Send for GpuVm<T> {}
+// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel.
+unsafe impl<T: DriverGpuVm> Sync for GpuVm<T> {}
+
+// SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
+unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVm<T> {
+    fn inc_ref(&self) {
+        // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
+        unsafe { bindings::drm_gpuvm_get(self.vm.get()) };
+    }
+
+    unsafe fn dec_ref(obj: NonNull<Self>) {
+        // SAFETY: By type invariants, the allocation is managed by the refcount in `self.vm`.
+        unsafe { bindings::drm_gpuvm_put((*obj.as_ptr()).vm.get()) };
+    }
+}
+
+impl<T: DriverGpuVm> PartialEq for GpuVm<T> {
+    #[inline]
+    fn eq(&self, other: &Self) -> bool {
+        core::ptr::eq(self.as_raw(), other.as_raw())
+    }
+}
+impl<T: DriverGpuVm> Eq for GpuVm<T> {}
+
+impl<T: DriverGpuVm> GpuVm<T> {
+    const fn vtable() -> &'static bindings::drm_gpuvm_ops {
+        &bindings::drm_gpuvm_ops {
+            vm_free: Some(Self::vm_free),
+            op_alloc: None,
+            op_free: None,
+            vm_bo_alloc: GpuVmBo::<T>::ALLOC_FN,
+            vm_bo_free: GpuVmBo::<T>::FREE_FN,
+            vm_bo_validate: None,
+            sm_step_map: Some(Self::sm_step_map),
+            sm_step_unmap: Some(Self::sm_step_unmap),
+            sm_step_remap: Some(Self::sm_step_remap),
+        }
+    }
+
+    /// Creates a GPUVM instance.
+    #[expect(clippy::new_ret_no_self)]
+    pub fn new<E>(
+        name: &'static CStr,
+        dev: &drm::Device<T::Driver>,
+        r_obj: &T::Object,
+        range: Range<u64>,
+        reserve_range: Range<u64>,
+        data: T,
+    ) -> Result<UniqueRefGpuVm<T>, E>
+    where
+        E: From<AllocError>,
+        E: From<core::convert::Infallible>,
+    {
+        let obj = KBox::try_pin_init::<E>(
+            try_pin_init!(Self {
+                data: UnsafeCell::new(data),
+                vm <- Opaque::ffi_init(|vm| {
+                    // SAFETY: These arguments are valid. `vm` is valid until refcount drops to
+                    // zero. The `vm` is zeroed before calling this method by `__GFP_ZERO` flag
+                    // below.
+                    unsafe {
+                        bindings::drm_gpuvm_init(
+                            vm,
+                            name.as_char_ptr(),
+                            bindings::drm_gpuvm_flags_DRM_GPUVM_IMMEDIATE_MODE
+                                | bindings::drm_gpuvm_flags_DRM_GPUVM_RESV_PROTECTED,
+                            dev.as_raw(),
+                            r_obj.as_raw(),
+                            range.start,
+                            range.end - range.start,
+                            reserve_range.start,
+                            reserve_range.end - reserve_range.start,
+                            const { Self::vtable() },
+                        )
+                    }
+                }),
+            }? E),
+            GFP_KERNEL | __GFP_ZERO,
+        )?;
+        // SAFETY: This transfers the initial refcount to the ARef.
+        let aref = unsafe {
+            ARef::from_raw(NonNull::new_unchecked(KBox::into_raw(
+                Pin::into_inner_unchecked(obj),
+            )))
+        };
+        // INVARIANT: This reference is unique.
+        Ok(UniqueRefGpuVm(aref))
+    }
+
+    /// Access this [`GpuVm`] from a raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// The pointer must reference the `struct drm_gpuvm` in a valid [`GpuVm<T>`] that remains
+    /// valid for at least `'a`.
+    #[inline]
+    pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm) -> &'a Self {
+        // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`. Caller ensures the
+        // pointer is valid for 'a.
+        unsafe { &*kernel::container_of!(Opaque::cast_from(ptr), Self, vm) }
+    }
+
+    /// Returns a raw pointer to the embedded `struct drm_gpuvm`.
+    #[inline]
+    pub fn as_raw(&self) -> *mut bindings::drm_gpuvm {
+        self.vm.get()
+    }
+
+    /// The start of the VA space.
+    #[inline]
+    pub fn va_start(&self) -> u64 {
+        // SAFETY: The `mm_start` field is immutable.
+        unsafe { (*self.as_raw()).mm_start }
+    }
+
+    /// The length of the GPU's virtual address space.
+    #[inline]
+    pub fn va_length(&self) -> u64 {
+        // SAFETY: The `mm_range` field is immutable.
+        unsafe { (*self.as_raw()).mm_range }
+    }
+
+    /// Returns the range of the GPU virtual address space.
+    #[inline]
+    pub fn va_range(&self) -> Range<u64> {
+        let start = self.va_start();
+        // OVERFLOW: This reconstructs the Range<u64> passed to the constructor, so it won't fail.
+        let end = start + self.va_length();
+        Range { start, end }
+    }
+
+    /// Get or create the [`GpuVmBo`] for this gem object.
+    #[inline]
+    pub fn obtain(
+        &self,
+        obj: &T::Object,
+        data: impl PinInit<T::VmBoData>,
+    ) -> Result<ARef<GpuVmBo<T>>, AllocError> {
+        Ok(GpuVmBoAlloc::new(self, obj, data)?.obtain())
+    }
+
+    /// Clean up buffer objects that are no longer used.
+    #[inline]
+    pub fn deferred_cleanup(&self) {
+        // SAFETY: This GPUVM uses immediate mode.
+        unsafe { bindings::drm_gpuvm_bo_deferred_cleanup(self.as_raw()) }
+    }
+
+    /// Check if this GEM object is an external object for this GPUVM.
+    #[inline]
+    pub fn is_extobj(&self, obj: &T::Object) -> bool {
+        // SAFETY: We may call this with any GPUVM and GEM object.
+        unsafe { bindings::drm_gpuvm_is_extobj(self.as_raw(), obj.as_raw()) }
+    }
+
+    /// Free this GPUVM.
+    ///
+    /// # Safety
+    ///
+    /// Called when refcount hits zero.
+    unsafe extern "C" fn vm_free(me: *mut bindings::drm_gpuvm) {
+        // SAFETY: Caller passes a pointer to the `drm_gpuvm` in a `GpuVm<T>`.
+        let me = unsafe { kernel::container_of!(Opaque::cast_from(me), Self, vm).cast_mut() };
+        // SAFETY: By type invariants we can free it when refcount hits zero.
+        drop(unsafe { KBox::from_raw(me) })
+    }
+
+    #[inline]
+    fn raw_resv(&self) -> *mut bindings::dma_resv {
+        // SAFETY: `r_obj` is immutable and valid for duration of GPUVM.
+        unsafe { (*(*self.as_raw()).r_obj).resv }
+    }
+}
+
+/// The manager for a GPUVM.
+pub trait DriverGpuVm: Sized + Send {
+    /// Parent `Driver` for this object.
+    type Driver: drm::Driver<Object = Self::Object>;
+
+    /// The kind of GEM object stored in this GPUVM.
+    type Object: IntoGEMObject;
+
+    /// Data stored with each [`struct drm_gpuva`](struct@GpuVa).
+    type VaData;
+
+    /// Data stored with each [`struct drm_gpuvm_bo`](struct@GpuVmBo).
+    type VmBoData;
+
+    /// The private data passed to callbacks.
+    type SmContext<'ctx>;
+
+    /// Indicates that a new mapping should be created.
+    fn sm_step_map<'op, 'ctx>(
+        &mut self,
+        op: OpMap<'op, Self>,
+        context: &mut Self::SmContext<'ctx>,
+    ) -> Result<OpMapped<'op, Self>, Error>;
+
+    /// Indicates that an existing mapping should be removed.
+    fn sm_step_unmap<'op, 'ctx>(
+        &mut self,
+        op: OpUnmap<'op, Self>,
+        context: &mut Self::SmContext<'ctx>,
+    ) -> Result<OpUnmapped<'op, Self>, Error>;
+
+    /// Indicates that an existing mapping should be split up.
+    fn sm_step_remap<'op, 'ctx>(
+        &mut self,
+        op: OpRemap<'op, Self>,
+        context: &mut Self::SmContext<'ctx>,
+    ) -> Result<OpRemapped<'op, Self>, Error>;
+}
+
+/// The core of the DRM GPU VA manager.
+///
+/// This object is a unique reference to the VM that can access the interval tree and the Rust
+/// `data` field.
+///
+/// # Invariants
+///
+/// Each `GpuVm` instance has at most one `UniqueRefGpuVm` reference.
+pub struct UniqueRefGpuVm<T: DriverGpuVm>(ARef<GpuVm<T>>);
+
+// SAFETY: The GPUVM api is designed to allow &self methods to be called in parallel, and
+// concurrent access to `data` is safe due to the `T: Sync` requirement.
+unsafe impl<T: DriverGpuVm + Sync> Sync for UniqueRefGpuVm<T> {}
+
+impl<T: DriverGpuVm> UniqueRefGpuVm<T> {
+    /// Access the data owned by this `UniqueRefGpuVm` immutably.
+    #[inline]
+    pub fn data_ref(&self) -> &T {
+        // SAFETY: By the type invariants we may access `data`.
+        unsafe { &*self.0.data.get() }
+    }
+
+    /// Access the data owned by this `UniqueRefGpuVm` mutably.
+    #[inline]
+    pub fn data(&mut self) -> &mut T {
+        // SAFETY: By the type invariants we may access `data`.
+        unsafe { &mut *self.0.data.get() }
+    }
+}
+
+impl<T: DriverGpuVm> Deref for UniqueRefGpuVm<T> {
+    type Target = GpuVm<T>;
+
+    #[inline]
+    fn deref(&self) -> &GpuVm<T> {
+        &self.0
+    }
+}
diff --git a/rust/kernel/drm/gpuvm/sm_ops.rs b/rust/kernel/drm/gpuvm/sm_ops.rs
new file mode 100644
index 0000000000000..69a8e5ab2821f
--- /dev/null
+++ b/rust/kernel/drm/gpuvm/sm_ops.rs
@@ -0,0 +1,429 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+use super::*;
+
+/// The actual data that gets threaded through the callbacks.
+struct SmData<'a, 'ctx, T: DriverGpuVm> {
+    gpuvm: &'a mut UniqueRefGpuVm<T>,
+    user_context: &'a mut T::SmContext<'ctx>,
+}
+
+/// Adds an extra field to `SmData` for `sm_map()` callbacks.
+///
+/// # Invariants
+///
+/// `self.vm_bo.gpuvm() == self.sm_data.gpuvm`.
+#[repr(C)]
+struct SmMapData<'a, 'ctx, T: DriverGpuVm> {
+    sm_data: SmData<'a, 'ctx, T>,
+    vm_bo: &'a GpuVmBo<T>,
+}
+
+/// The argument for [`UniqueRefGpuVm::sm_map`].
+pub struct OpMapRequest<'a, 'ctx, T: DriverGpuVm> {
+    /// Address in GPU virtual address space.
+    pub addr: u64,
+    /// Length of mapping to create.
+    pub range: u64,
+    /// Offset in GEM object.
+    pub gem_offset: u64,
+    /// The GEM object to map.
+    pub vm_bo: &'a GpuVmBo<T>,
+    /// The user-provided context type.
+    pub context: &'a mut T::SmContext<'ctx>,
+}
+
+impl<'a, 'ctx, T: DriverGpuVm> OpMapRequest<'a, 'ctx, T> {
+    fn raw_request(&self) -> bindings::drm_gpuvm_map_req {
+        bindings::drm_gpuvm_map_req {
+            map: bindings::drm_gpuva_op_map {
+                va: bindings::drm_gpuva_op_map__bindgen_ty_1 {
+                    addr: self.addr,
+                    range: self.range,
+                },
+                gem: bindings::drm_gpuva_op_map__bindgen_ty_2 {
+                    offset: self.gem_offset,
+                    obj: self.vm_bo.obj().as_raw(),
+                },
+            },
+        }
+    }
+}
+
+/// Represents an `sm_step_map` operation that has not yet been completed.
+pub struct OpMap<'op, T: DriverGpuVm> {
+    op: &'op bindings::drm_gpuva_op_map,
+    // Since these abstractions are designed for immediate mode, the VM BO needs to be
+    // pre-allocated, so we always have it available when we reach this point.
+    vm_bo: &'op GpuVmBo<T>,
+    // This ensures that 'op is invariant, so that `OpMap<'long, T>` does not
+    // coerce to `OpMap<'short, T>`. This ensures that the user can't return
+    // the wrong `OpMapped` value.
+    _invariant: PhantomData<*mut &'op mut T>,
+}
+
+impl<'op, T: DriverGpuVm> OpMap<'op, T> {
+    /// The base address of the new mapping.
+    pub fn addr(&self) -> u64 {
+        self.op.va.addr
+    }
+
+    /// The length of the new mapping.
+    pub fn length(&self) -> u64 {
+        self.op.va.range
+    }
+
+    /// The offset within the [`drm_gem_object`](DriverGpuVm::Object).
+    pub fn gem_offset(&self) -> u64 {
+        self.op.gem.offset
+    }
+
+    /// The [`drm_gem_object`](DriverGpuVm::Object) to map.
+    pub fn obj(&self) -> &T::Object {
+        // SAFETY: The `obj` pointer is guaranteed to be valid.
+        unsafe { <T::Object as IntoGEMObject>::from_raw(self.op.gem.obj) }
+    }
+
+    /// The [`GpuVmBo`] that the new VA will be associated with.
+    pub fn vm_bo(&self) -> &GpuVmBo<T> {
+        self.vm_bo
+    }
+
+    /// Use the pre-allocated VA to carry out this map operation.
+    pub fn insert(self, va: GpuVaAlloc<T>, va_data: impl PinInit<T::VaData>) -> OpMapped<'op, T> {
+        let va = va.prepare(va_data);
+        // SAFETY: By the type invariants we may access the interval tree.
+        unsafe { bindings::drm_gpuva_map(self.vm_bo.gpuvm().as_raw(), va, self.op) };
+
+        let _gpuva_guard = self.vm_bo().lock_gpuva();
+        // SAFETY: The va is prepared for insertion, and we hold the GEM lock.
+        unsafe { bindings::drm_gpuva_link(va, self.vm_bo.as_raw()) };
+
+        OpMapped {
+            _invariant: self._invariant,
+        }
+    }
+}
+
+/// Represents a completed [`OpMap`] operation.
+pub struct OpMapped<'op, T> {
+    _invariant: PhantomData<*mut &'op mut T>,
+}
+
+/// Represents an `sm_step_unmap` operation that has not yet been completed.
+pub struct OpUnmap<'op, T: DriverGpuVm> {
+    op: &'op bindings::drm_gpuva_op_unmap,
+    // This ensures that 'op is invariant, so that `OpUnmap<'long, T>` does not
+    // coerce to `OpUnmap<'short, T>`. This ensures that the user can't return the
+    // wrong`OpUnmapped` value.
+    _invariant: PhantomData<*mut &'op mut T>,
+}
+
+impl<'op, T: DriverGpuVm> OpUnmap<'op, T> {
+    /// Indicates whether this [`GpuVa`] is physically contiguous with the
+    /// original mapping request.
+    ///
+    /// Optionally, if `keep` is set, drivers may keep the actual page table
+    /// mappings for this `drm_gpuva`, adding the missing page table entries
+    /// only and update the `drm_gpuvm` accordingly.
+    pub fn keep(&self) -> bool {
+        self.op.keep
+    }
+
+    /// The range being unmapped.
+    pub fn va(&self) -> &GpuVa<T> {
+        // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it,
+        // and it's not sparse by the `GpuVm<T>` type invariants.
+        unsafe { GpuVa::<T>::from_raw(self.op.va) }
+    }
+
+    /// Remove the VA.
+    pub fn remove(self) -> (OpUnmapped<'op, T>, GpuVaRemoved<T>) {
+        // SAFETY: The op references a valid drm_gpuva in the GPUVM.
+        unsafe { bindings::drm_gpuva_unmap(self.op) };
+        // SAFETY: The va is no longer in the interval tree so we may unlink it.
+        unsafe { bindings::drm_gpuva_unlink_defer(self.op.va) };
+
+        // SAFETY: We just removed this va from the `GpuVm<T>`.
+        let va = unsafe { GpuVaRemoved::from_raw(self.op.va) };
+
+        (
+            OpUnmapped {
+                _invariant: self._invariant,
+            },
+            va,
+        )
+    }
+}
+
+/// Represents a completed [`OpUnmap`] operation.
+pub struct OpUnmapped<'op, T> {
+    _invariant: PhantomData<*mut &'op mut T>,
+}
+
+/// Represents an `sm_step_remap` operation that has not yet been completed.
+pub struct OpRemap<'op, T: DriverGpuVm> {
+    op: &'op bindings::drm_gpuva_op_remap,
+    // This ensures that 'op is invariant, so that `OpRemap<'long, T>` does not
+    // coerce to `OpRemap<'short, T>`. This ensures that the user can't return the
+    // wrong`OpRemapped` value.
+    _invariant: PhantomData<*mut &'op mut T>,
+}
+
+impl<'op, T: DriverGpuVm> OpRemap<'op, T> {
+    /// The preceding part of a split mapping.
+    #[inline]
+    pub fn prev(&self) -> Option<&OpRemapMapData> {
+        // SAFETY: We checked for null, so the pointer must be valid.
+        NonNull::new(self.op.prev).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) })
+    }
+
+    /// The subsequent part of a split mapping.
+    #[inline]
+    pub fn next(&self) -> Option<&OpRemapMapData> {
+        // SAFETY: We checked for null, so the pointer must be valid.
+        NonNull::new(self.op.next).map(|ptr| unsafe { OpRemapMapData::from_raw(ptr) })
+    }
+
+    /// Indicates whether the `drm_gpuva` being removed is physically contiguous with the original
+    /// mapping request.
+    ///
+    /// Optionally, if `keep` is set, drivers may keep the actual page table mappings for this
+    /// `drm_gpuva`, adding the missing page table entries only and update the `drm_gpuvm`
+    /// accordingly.
+    #[inline]
+    pub fn keep(&self) -> bool {
+        // SAFETY: The unmap pointer is always valid.
+        unsafe { (*self.op.unmap).keep }
+    }
+
+    /// The range being unmapped.
+    #[inline]
+    pub fn va_to_unmap(&self) -> &GpuVa<T> {
+        // SAFETY: This is a valid va. It's not the `kernel_alloc_node` because you can't unmap it,
+        // and it's not sparse by the `GpuVm<T>` type invariants.
+        unsafe { GpuVa::<T>::from_raw((*self.op.unmap).va) }
+    }
+
+    /// The [`drm_gem_object`](DriverGpuVm::Object) whose VA is being remapped.
+    #[inline]
+    pub fn obj(&self) -> &T::Object {
+        self.va_to_unmap().obj()
+    }
+
+    /// The [`GpuVmBo`] that is being remapped.
+    #[inline]
+    pub fn vm_bo(&self) -> &GpuVmBo<T> {
+        self.va_to_unmap().vm_bo()
+    }
+
+    /// Update the GPUVM to perform the remapping.
+    pub fn remap(
+        self,
+        va_alloc: [GpuVaAlloc<T>; 2],
+        prev_data: impl PinInit<T::VaData>,
+        next_data: impl PinInit<T::VaData>,
+    ) -> (OpRemapped<'op, T>, OpRemapRet<T>) {
+        let [va1, va2] = va_alloc;
+
+        let mut unused_va = None;
+        let mut prev_ptr = ptr::null_mut();
+        let mut next_ptr = ptr::null_mut();
+        if self.prev().is_some() {
+            prev_ptr = va1.prepare(prev_data);
+        } else {
+            unused_va = Some(va1);
+        }
+        if self.next().is_some() {
+            next_ptr = va2.prepare(next_data);
+        } else {
+            unused_va = Some(va2);
+        }
+
+        // SAFETY: the pointers are non-null when required
+        unsafe { bindings::drm_gpuva_remap(prev_ptr, next_ptr, self.op) };
+
+        let gpuva_guard = self.vm_bo().lock_gpuva();
+        if !prev_ptr.is_null() {
+            // SAFETY: The prev_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still
+            // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo.
+            unsafe { bindings::drm_gpuva_link(prev_ptr, self.vm_bo().as_raw()) };
+        }
+        if !next_ptr.is_null() {
+            // SAFETY: The next_ptr is a valid drm_gpuva prepared for insertion. The vm_bo is still
+            // valid as the not-yet-unlinked gpuva holds a refcount on the vm_bo.
+            unsafe { bindings::drm_gpuva_link(next_ptr, self.vm_bo().as_raw()) };
+        }
+        drop(gpuva_guard);
+
+        // SAFETY: The va is no longer in the interval tree so we may unlink it.
+        unsafe { bindings::drm_gpuva_unlink_defer((*self.op.unmap).va) };
+
+        (
+            OpRemapped {
+                _invariant: self._invariant,
+            },
+            OpRemapRet {
+                // SAFETY: We just removed this va from the `GpuVm<T>`.
+                unmapped_va: unsafe { GpuVaRemoved::from_raw((*self.op.unmap).va) },
+                unused_va,
+            },
+        )
+    }
+}
+
+/// Part of an [`OpRemap`] that represents a new mapping.
+#[repr(transparent)]
+pub struct OpRemapMapData(bindings::drm_gpuva_op_map);
+
+impl OpRemapMapData {
+    /// # Safety
+    /// Must reference a valid `drm_gpuva_op_map` for duration of `'a`.
+    unsafe fn from_raw<'a>(ptr: NonNull<bindings::drm_gpuva_op_map>) -> &'a Self {
+        // SAFETY: ok per safety requirements
+        unsafe { ptr.cast().as_ref() }
+    }
+
+    /// The base address of the new mapping.
+    pub fn addr(&self) -> u64 {
+        self.0.va.addr
+    }
+
+    /// The length of the new mapping.
+    pub fn length(&self) -> u64 {
+        self.0.va.range
+    }
+
+    /// The offset within the [`drm_gem_object`](DriverGpuVm::Object).
+    pub fn gem_offset(&self) -> u64 {
+        self.0.gem.offset
+    }
+}
+
+/// Struct containing objects removed or not used by [`OpRemap::remap`].
+pub struct OpRemapRet<T: DriverGpuVm> {
+    /// The `drm_gpuva` that was removed.
+    pub unmapped_va: GpuVaRemoved<T>,
+    /// If the remap did not split the region into two pieces, then the unused `drm_gpuva` is
+    /// returned here.
+    pub unused_va: Option<GpuVaAlloc<T>>,
+}
+
+/// Represents a completed [`OpRemap`] operation.
+pub struct OpRemapped<'op, T> {
+    _invariant: PhantomData<*mut &'op mut T>,
+}
+
+impl<T: DriverGpuVm> UniqueRefGpuVm<T> {
+    /// Create a mapping, removing or remapping anything that overlaps.
+    ///
+    /// Internally calls the [`DriverGpuVm`] callbacks similar to [`Self::sm_unmap`], except that
+    /// the [`DriverGpuVm::sm_step_map`] is called once to create the requested mapping.
+    #[inline]
+    pub fn sm_map(&mut self, req: OpMapRequest<'_, '_, T>) -> Result {
+        if req.vm_bo.gpuvm() != &**self {
+            return Err(EINVAL);
+        }
+
+        let gpuvm = self.as_raw();
+        let raw_req = req.raw_request();
+        // INVARIANT: Checked above that `vm_bo.gpuvm() == self`.
+        let mut p = SmMapData {
+            sm_data: SmData {
+                gpuvm: self,
+                user_context: req.context,
+            },
+            vm_bo: req.vm_bo,
+        };
+        // SAFETY:
+        // * raw_request() creates a valid request.
+        // * The private data is valid to be interpreted as both SmData and SmMapData since the
+        //   first field of SmMapData is SmData.
+        to_result(unsafe {
+            bindings::drm_gpuvm_sm_map(gpuvm, (&raw mut p).cast(), &raw const raw_req)
+        })
+    }
+
+    /// Remove any mappings in the given region.
+    ///
+    /// Internally calls [`DriverGpuVm::sm_step_unmap`] for ranges entirely contained within the
+    /// given range, and [`DriverGpuVm::sm_step_remap`] for ranges that overlap with the range.
+    #[inline]
+    pub fn sm_unmap(&mut self, addr: u64, length: u64, context: &mut T::SmContext<'_>) -> Result {
+        let gpuvm = self.as_raw();
+        let mut p = SmData {
+            gpuvm: self,
+            user_context: context,
+        };
+        // SAFETY:
+        // * raw_request() creates a valid request.
+        // * The private data is a valid SmData.
+        to_result(unsafe { bindings::drm_gpuvm_sm_unmap(gpuvm, (&raw mut p).cast(), addr, length) })
+    }
+}
+
+impl<T: DriverGpuVm> GpuVm<T> {
+    /// # Safety
+    /// Must be called from `sm_map` with a pointer to `SmMapData`.
+    pub(super) unsafe extern "C" fn sm_step_map(
+        op: *mut bindings::drm_gpuva_op,
+        p: *mut c_void,
+    ) -> c_int {
+        // SAFETY: If we reach `sm_step_map` then we were called from `sm_map` which always passes
+        // an `SmMapData` as private data.
+        let p = unsafe { &mut *p.cast::<SmMapData<'_, '_, T>>() };
+        let op = OpMap {
+            // SAFETY: sm_step_map is called with a map operation.
+            op: unsafe { &(*op).__bindgen_anon_1.map },
+            vm_bo: p.vm_bo,
+            _invariant: PhantomData,
+        };
+        match p
+            .sm_data
+            .gpuvm
+            .data()
+            .sm_step_map(op, p.sm_data.user_context)
+        {
+            Ok(OpMapped { .. }) => 0,
+            Err(err) => err.to_errno(),
+        }
+    }
+
+    /// # Safety
+    /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`.
+    pub(super) unsafe extern "C" fn sm_step_unmap(
+        op: *mut bindings::drm_gpuva_op,
+        p: *mut c_void,
+    ) -> c_int {
+        // SAFETY: The caller provides a pointer that can be treated as `SmData`.
+        let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() };
+        let op = OpUnmap {
+            // SAFETY: sm_step_unmap is called with an unmap operation.
+            op: unsafe { &(*op).__bindgen_anon_1.unmap },
+            _invariant: PhantomData,
+        };
+        match p.gpuvm.data().sm_step_unmap(op, p.user_context) {
+            Ok(OpUnmapped { .. }) => 0,
+            Err(err) => err.to_errno(),
+        }
+    }
+
+    /// # Safety
+    /// Must be called from `sm_map` or `sm_unmap` with a pointer to `SmMapData` or `SmData`.
+    pub(super) unsafe extern "C" fn sm_step_remap(
+        op: *mut bindings::drm_gpuva_op,
+        p: *mut c_void,
+    ) -> c_int {
+        // SAFETY: The caller provides a pointer that can be treated as `SmData`.
+        let p = unsafe { &mut *p.cast::<SmData<'_, '_, T>>() };
+        let op = OpRemap {
+            // SAFETY: sm_step_remap is called with a remap operation.
+            op: unsafe { &(*op).__bindgen_anon_1.remap },
+            _invariant: PhantomData,
+        };
+        match p.gpuvm.data().sm_step_remap(op, p.user_context) {
+            Ok(OpRemapped { .. }) => 0,
+            Err(err) => err.to_errno(),
+        }
+    }
+}
diff --git a/rust/kernel/drm/gpuvm/va.rs b/rust/kernel/drm/gpuvm/va.rs
new file mode 100644
index 0000000000000..0b09fe44ab397
--- /dev/null
+++ b/rust/kernel/drm/gpuvm/va.rs
@@ -0,0 +1,168 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+use super::*;
+
+/// Represents that a range of a GEM object is mapped in this [`GpuVm`] instance.
+///
+/// Does not assume that GEM lock is held.
+///
+/// # Invariants
+///
+/// * This is a valid `drm_gpuva` object that is resident in a [`GpuVm<T>`] instance.
+/// * It is associated with a [`GpuVmBo<T>`]. Or in other words, it's not an
+///   `gpuvm->kernel_alloc_node` and `DRM_GPUVA_SPARSE` is not set.
+/// * The associated [`GpuVmBo<T>`] is part of the GEM list.
+#[repr(C)]
+#[pin_data]
+pub struct GpuVa<T: DriverGpuVm> {
+    #[pin]
+    inner: Opaque<bindings::drm_gpuva>,
+    #[pin]
+    data: T::VaData,
+}
+
+impl<T: DriverGpuVm> PartialEq for GpuVa<T> {
+    #[inline]
+    fn eq(&self, other: &Self) -> bool {
+        core::ptr::eq(self.as_raw(), other.as_raw())
+    }
+}
+impl<T: DriverGpuVm> Eq for GpuVa<T> {}
+
+impl<T: DriverGpuVm> GpuVa<T> {
+    /// Access this [`GpuVa`] from a raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// * For the duration of `'a`, the pointer must reference a valid `drm_gpuva` associated with
+    ///   a [`GpuVm<T>`].
+    /// * It must be associated with a [`GpuVmBo<T>`].
+    /// * The associated [`GpuVmBo<T>`] is part of the GEM list.
+    #[inline]
+    pub unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuva) -> &'a Self {
+        // CAST: `drm_gpuva` is first field and `repr(C)`.
+        // SAFETY: The safety requirements match the invariants of `GpuVa`.
+        unsafe { &*ptr.cast() }
+    }
+
+    /// Returns a raw pointer to underlying C value.
+    #[inline]
+    pub fn as_raw(&self) -> *mut bindings::drm_gpuva {
+        self.inner.get()
+    }
+
+    /// Returns the address of this mapping in the GPU virtual address space.
+    #[inline]
+    pub fn addr(&self) -> u64 {
+        // SAFETY: The `va.addr` field of `drm_gpuva` is immutable.
+        unsafe { (*self.as_raw()).va.addr }
+    }
+
+    /// Returns the length of this mapping.
+    #[inline]
+    pub fn length(&self) -> u64 {
+        // SAFETY: The `va.range` field of `drm_gpuva` is immutable.
+        unsafe { (*self.as_raw()).va.range }
+    }
+
+    /// Returns `addr..addr+length`.
+    #[inline]
+    pub fn range(&self) -> Range<u64> {
+        let addr = self.addr();
+        addr..addr + self.length()
+    }
+
+    /// Returns the offset within the GEM object.
+    #[inline]
+    pub fn gem_offset(&self) -> u64 {
+        // SAFETY: The `gem.offset` field of `drm_gpuva` is immutable.
+        unsafe { (*self.as_raw()).gem.offset }
+    }
+
+    /// Returns the GEM object.
+    #[inline]
+    pub fn obj(&self) -> &T::Object {
+        // SAFETY: The `gem.obj` field of `drm_gpuva` is immutable. We know that it's not null
+        // because this VA is associated with a `GpuVmBo<T>`.
+        unsafe { <T::Object as IntoGEMObject>::from_raw((*self.as_raw()).gem.obj) }
+    }
+
+    /// Returns the underlying [`GpuVmBo`] object that backs this [`GpuVa`].
+    #[inline]
+    pub fn vm_bo(&self) -> &GpuVmBo<T> {
+        // SAFETY: The `vm_bo` field of `drm_gpuva` is immutable. We know that it's not null
+        // because this VA is associated with a `GpuVmBo<T>`. The BO is in the GEM list by the type
+        // invariants.
+        unsafe { GpuVmBo::from_raw((*self.as_raw()).vm_bo) }
+    }
+}
+
+/// A pre-allocated [`GpuVa`] object.
+///
+/// # Invariants
+///
+/// The memory is zeroed.
+pub struct GpuVaAlloc<T: DriverGpuVm>(KBox<MaybeUninit<GpuVa<T>>>);
+
+impl<T: DriverGpuVm> GpuVaAlloc<T> {
+    /// Pre-allocate a [`GpuVa`] object.
+    pub fn new(flags: AllocFlags) -> Result<GpuVaAlloc<T>, AllocError> {
+        // INVARIANTS: Memory allocated with __GFP_ZERO.
+        Ok(GpuVaAlloc(KBox::new_uninit(flags | __GFP_ZERO)?))
+    }
+
+    /// Prepare this `drm_gpuva` for insertion into the GPUVM.
+    #[must_use]
+    pub(super) fn prepare(mut self, va_data: impl PinInit<T::VaData>) -> *mut bindings::drm_gpuva {
+        let va_ptr = MaybeUninit::as_mut_ptr(&mut self.0);
+        // SAFETY: The `data` field is pinned.
+        let Ok(()) = unsafe { va_data.__pinned_init(&raw mut (*va_ptr).data) };
+        KBox::into_raw(self.0).cast()
+    }
+}
+
+/// A [`GpuVa`] object that has been removed.
+///
+/// # Invariants
+///
+/// The `drm_gpuva` is not resident in the [`GpuVm`].
+pub struct GpuVaRemoved<T: DriverGpuVm>(KBox<GpuVa<T>>);
+
+impl<T: DriverGpuVm> GpuVaRemoved<T> {
+    /// Convert a raw pointer into a [`GpuVaRemoved`].
+    ///
+    /// # Safety
+    ///
+    /// * Must have been removed from a [`GpuVm<T>`].
+    /// * It must not be a `gpuvm->kernel_alloc_node` va.
+    pub(super) unsafe fn from_raw(ptr: *mut bindings::drm_gpuva) -> Self {
+        // SAFETY: Since it used to be a VA in a `GpuVm<T>` and it's not a kernel_alloc_node, this
+        // pointer references a `GpuVa<T>` with a valid `T::VaData`. Since it has been removed, we
+        // can take ownership of the allocation.
+        GpuVaRemoved(unsafe { KBox::from_raw(ptr.cast()) })
+    }
+
+    /// Take ownership of the VA data.
+    pub fn into_inner(self) -> T::VaData
+    where
+        T::VaData: Unpin,
+    {
+        KBox::into_inner(self.0).data
+    }
+}
+
+impl<T: DriverGpuVm> Deref for GpuVaRemoved<T> {
+    type Target = T::VaData;
+    fn deref(&self) -> &T::VaData {
+        &self.0.data
+    }
+}
+
+impl<T: DriverGpuVm> DerefMut for GpuVaRemoved<T>
+where
+    T::VaData: Unpin,
+{
+    fn deref_mut(&mut self) -> &mut T::VaData {
+        &mut self.0.data
+    }
+}
diff --git a/rust/kernel/drm/gpuvm/vm_bo.rs b/rust/kernel/drm/gpuvm/vm_bo.rs
new file mode 100644
index 0000000000000..c064ac63897be
--- /dev/null
+++ b/rust/kernel/drm/gpuvm/vm_bo.rs
@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: GPL-2.0 OR MIT
+
+use super::*;
+
+/// Represents that a given GEM object has at least one mapping on this [`GpuVm`] instance.
+///
+/// Does not assume that GEM lock is held.
+///
+/// # Invariants
+///
+/// * Allocated with `kmalloc` and refcounted via `inner`.
+/// * Is present in the gem list.
+#[repr(C)]
+#[pin_data]
+pub struct GpuVmBo<T: DriverGpuVm> {
+    #[pin]
+    inner: Opaque<bindings::drm_gpuvm_bo>,
+    #[pin]
+    data: T::VmBoData,
+}
+
+// SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`.
+unsafe impl<T: DriverGpuVm> AlwaysRefCounted for GpuVmBo<T> {
+    fn inc_ref(&self) {
+        // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`.
+        unsafe { bindings::drm_gpuvm_bo_get(self.inner.get()) };
+    }
+
+    unsafe fn dec_ref(obj: NonNull<Self>) {
+        // CAST: `drm_gpuvm_bo` is first field of repr(C) struct.
+        // SAFETY: By type invariants, the allocation is managed by the refcount in `self.inner`.
+        // This GPUVM instance uses immediate mode, so we may put the refcount using the deferred
+        // mechanism.
+        unsafe { bindings::drm_gpuvm_bo_put_deferred(obj.as_ptr().cast()) };
+    }
+}
+
+impl<T: DriverGpuVm> PartialEq for GpuVmBo<T> {
+    #[inline]
+    fn eq(&self, other: &Self) -> bool {
+        core::ptr::eq(self.as_raw(), other.as_raw())
+    }
+}
+impl<T: DriverGpuVm> Eq for GpuVmBo<T> {}
+
+impl<T: DriverGpuVm> GpuVmBo<T> {
+    /// The function pointer for allocating a GpuVmBo stored in the gpuvm vtable.
+    ///
+    /// Allocation is always implemented according to [`Self::vm_bo_alloc`], but it is set to
+    /// `None` if the default gpuvm behavior is the same as `vm_bo_alloc`.
+    ///
+    /// This may be `Some` even if `FREE_FN` is `None`, or vice-versa.
+    pub(super) const ALLOC_FN: Option<unsafe extern "C" fn() -> *mut bindings::drm_gpuvm_bo> = {
+        use core::alloc::Layout;
+        let base = Layout::new::<bindings::drm_gpuvm_bo>();
+        let rust = Layout::new::<Self>();
+        assert!(base.size() <= rust.size());
+        if base.size() != rust.size() || base.align() != rust.align() {
+            Some(Self::vm_bo_alloc)
+        } else {
+            // This causes GPUVM to allocate a `GpuVmBo<T>` with `kzalloc(sizeof(drm_gpuvm_bo))`.
+            None
+        }
+    };
+
+    /// The function pointer for freeing a GpuVmBo stored in the gpuvm vtable.
+    ///
+    /// Freeing is always implemented according to [`Self::vm_bo_free`], but it is set to `None` if
+    /// the default gpuvm behavior is the same as `vm_bo_free`.
+    ///
+    /// This may be `Some` even if `ALLOC_FN` is `None`, or vice-versa.
+    pub(super) const FREE_FN: Option<unsafe extern "C" fn(*mut bindings::drm_gpuvm_bo)> = {
+        if core::mem::needs_drop::<Self>() {
+            Some(Self::vm_bo_free)
+        } else {
+            // This causes GPUVM to free a `GpuVmBo<T>` with `kfree`.
+            None
+        }
+    };
+
+    /// Custom function for allocating a `drm_gpuvm_bo`.
+    ///
+    /// # Safety
+    ///
+    /// Always safe to call.
+    unsafe extern "C" fn vm_bo_alloc() -> *mut bindings::drm_gpuvm_bo {
+        let raw_ptr = KBox::<Self>::new_uninit(GFP_KERNEL | __GFP_ZERO)
+            .map(KBox::into_raw)
+            .unwrap_or(ptr::null_mut());
+
+        // CAST: `drm_gpuvm_bo` is first field of `Self`.
+        raw_ptr.cast()
+    }
+
+    /// Custom function for freeing a `drm_gpuvm_bo`.
+    ///
+    /// # Safety
+    ///
+    /// The pointer must have been allocated with [`GpuVmBo::ALLOC_FN`], and must not be used after
+    /// this call.
+    unsafe extern "C" fn vm_bo_free(ptr: *mut bindings::drm_gpuvm_bo) {
+        // CAST: `drm_gpuvm_bo` is first field of `Self`.
+        // SAFETY:
+        // * The ptr was allocated from kmalloc with the layout of `GpuVmBo<T>`.
+        // * `ptr->inner` has no destructor.
+        // * `ptr->data` contains a valid `T::VmBoData` that we can drop.
+        drop(unsafe { KBox::<Self>::from_raw(ptr.cast()) });
+    }
+
+    /// Access this [`GpuVmBo`] from a raw pointer.
+    ///
+    /// # Safety
+    ///
+    /// For the duration of `'a`, the pointer must reference a valid `drm_gpuvm_bo` associated with
+    /// a [`GpuVm<T>`]. The BO must also be present in the GEM list.
+    #[inline]
+    pub(crate) unsafe fn from_raw<'a>(ptr: *mut bindings::drm_gpuvm_bo) -> &'a Self {
+        // SAFETY: `drm_gpuvm_bo` is first field and `repr(C)`.
+        unsafe { &*ptr.cast() }
+    }
+
+    /// Returns a raw pointer to underlying C value.
+    #[inline]
+    pub fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo {
+        self.inner.get()
+    }
+
+    /// The [`GpuVm`] that this GEM object is mapped in.
+    #[inline]
+    pub fn gpuvm(&self) -> &GpuVm<T> {
+        // SAFETY: The `obj` pointer is guaranteed to be valid.
+        unsafe { GpuVm::<T>::from_raw((*self.inner.get()).vm) }
+    }
+
+    /// The [`drm_gem_object`](DriverGpuVm::Object) for these mappings.
+    #[inline]
+    pub fn obj(&self) -> &T::Object {
+        // SAFETY: The `obj` pointer is guaranteed to be valid.
+        unsafe { <T::Object as IntoGEMObject>::from_raw((*self.inner.get()).obj) }
+    }
+
+    /// The driver data with this buffer object.
+    #[inline]
+    pub fn data(&self) -> &T::VmBoData {
+        &self.data
+    }
+
+    pub(super) fn lock_gpuva(&self) -> crate::sync::MutexGuard<'_, ()> {
+        // SAFETY: The GEM object is valid.
+        let ptr = unsafe { &raw mut (*self.obj().as_raw()).gpuva.lock };
+        // SAFETY: The GEM object is valid, so the mutex is properly initialized.
+        let mutex = unsafe { crate::sync::Mutex::from_raw(ptr) };
+        mutex.lock()
+    }
+}
+
+/// A pre-allocated [`GpuVmBo`] object.
+///
+/// # Invariants
+///
+/// Points at a `drm_gpuvm_bo` that contains a valid `T::VmBoData`, has a refcount of one, and is
+/// absent from any gem, extobj, or evict lists.
+pub(super) struct GpuVmBoAlloc<T: DriverGpuVm>(NonNull<GpuVmBo<T>>);
+
+impl<T: DriverGpuVm> GpuVmBoAlloc<T> {
+    /// Create a new pre-allocated [`GpuVmBo`].
+    ///
+    /// It's intentional that the initializer is infallible because `drm_gpuvm_bo_put` will call
+    /// drop on the data, so we don't have a way to free it when the data is missing.
+    #[inline]
+    pub(super) fn new(
+        gpuvm: &GpuVm<T>,
+        gem: &T::Object,
+        value: impl PinInit<T::VmBoData>,
+    ) -> Result<GpuVmBoAlloc<T>, AllocError> {
+        // CAST: `GpuVmBoAlloc::vm_bo_alloc` ensures that this memory was allocated with the layout
+        // of `GpuVmBo<T>`. The type is repr(C), so `container_of` is not required.
+        // SAFETY: The provided gpuvm and gem ptrs are valid for the duration of this call.
+        let raw_ptr = unsafe {
+            bindings::drm_gpuvm_bo_create(gpuvm.as_raw(), gem.as_raw()).cast::<GpuVmBo<T>>()
+        };
+        let ptr = NonNull::new(raw_ptr).ok_or(AllocError)?;
+        // SAFETY: `ptr->data` is a valid pinned location.
+        let Ok(()) = unsafe { value.__pinned_init(&raw mut (*raw_ptr).data) };
+        // INVARIANTS: We just created the vm_bo so it's absent from lists, and the data is valid
+        // as we just initialized it.
+        Ok(GpuVmBoAlloc(ptr))
+    }
+
+    /// Returns a raw pointer to underlying C value.
+    #[inline]
+    pub(super) fn as_raw(&self) -> *mut bindings::drm_gpuvm_bo {
+        // SAFETY: The pointer references a valid `drm_gpuvm_bo`.
+        unsafe { (*self.0.as_ptr()).inner.get() }
+    }
+
+    /// Look up whether there is an existing [`GpuVmBo`] for this gem object.
+    ///
+    /// The caller should not hold the GEM mutex or DMA resv lock.
+    #[inline]
+    pub(super) fn obtain(self) -> ARef<GpuVmBo<T>> {
+        let me = ManuallyDrop::new(self);
+        // SAFETY: Valid `drm_gpuvm_bo` not already in the lists. We do not access `me` after this
+        // call.
+        let ptr = unsafe { bindings::drm_gpuvm_bo_obtain_prealloc(me.as_raw()) };
+
+        // SAFETY: `drm_gpuvm_bo_obtain_prealloc` always returns a non-null ptr
+        let nonnull = unsafe { NonNull::new_unchecked(ptr.cast()) };
+
+        // INVARIANTS: `drm_gpuvm_bo_obtain_prealloc` ensures that the bo is in the GEM list.
+        // SAFETY: We received one refcount from `drm_gpuvm_bo_obtain_prealloc`.
+        let ret = unsafe { ARef::<GpuVmBo<T>>::from_raw(nonnull) };
+
+        // Ensure that external objects are in the extobj list.
+        //
+        // Note that we must call `extobj_add` even if `ptr != me` to avoid a race condition where
+        // we could end up using the extobj before the thread with `ptr == me` calls extobj_add.
+        if ret.gpuvm().is_extobj(ret.obj()) {
+            let resv_lock = ret.gpuvm().raw_resv();
+            // TODO: Use a proper lock guard here once a dma_resv lock abstraction exists.
+            // SAFETY: The GPUVM is still alive, so its resv lock is too.
+            unsafe { bindings::dma_resv_lock(resv_lock, ptr::null_mut()) };
+            // SAFETY: We hold the GPUVMs resv lock.
+            unsafe { bindings::drm_gpuvm_bo_extobj_add(ptr) };
+            // SAFETY: We took the lock, so we can unlock it.
+            unsafe { bindings::dma_resv_unlock(resv_lock) };
+        }
+
+        ret
+    }
+}
+
+impl<T: DriverGpuVm> Deref for GpuVmBoAlloc<T> {
+    type Target = GpuVmBo<T>;
+    #[inline]
+    fn deref(&self) -> &GpuVmBo<T> {
+        // SAFETY: By the type invariants we may deref while `Self` exists.
+        unsafe { self.0.as_ref() }
+    }
+}
+
+impl<T: DriverGpuVm> Drop for GpuVmBoAlloc<T> {
+    #[inline]
+    fn drop(&mut self) {
+        // TODO: Call drm_gpuvm_bo_destroy_not_in_lists() directly.
+        // SAFETY: It's safe to perform a deferred put in any context.
+        unsafe { bindings::drm_gpuvm_bo_put_deferred(self.as_raw()) };
+    }
+}
diff --git a/rust/kernel/drm/mod.rs b/rust/kernel/drm/mod.rs
index 1b82b6945edf2..a4b6c54301985 100644
--- a/rust/kernel/drm/mod.rs
+++ b/rust/kernel/drm/mod.rs
@@ -6,6 +6,7 @@
 pub mod driver;
 pub mod file;
 pub mod gem;
+pub mod gpuvm;
 pub mod ioctl;
 
 pub use self::device::Device;
diff --git a/rust/kernel/fmt.rs b/rust/kernel/fmt.rs
index 1e8725eb44ed7..73afbc51ba33a 100644
--- a/rust/kernel/fmt.rs
+++ b/rust/kernel/fmt.rs
@@ -4,7 +4,14 @@
 //!
 //! This module is intended to be used in place of `core::fmt` in kernel code.
 
-pub use core::fmt::{Arguments, Debug, Error, Formatter, Result, Write};
+pub use core::fmt::{
+    Arguments,
+    Debug,
+    Error,
+    Formatter,
+    Result,
+    Write, //
+};
 
 /// Internal adapter used to route and allow implementations of formatting traits for foreign types.
 ///
@@ -27,7 +34,15 @@ fn fmt(&self, f: &mut Formatter<'_>) -> Result {
     };
 }
 
-use core::fmt::{Binary, LowerExp, LowerHex, Octal, Pointer, UpperExp, UpperHex};
+use core::fmt::{
+    Binary,
+    LowerExp,
+    LowerHex,
+    Octal,
+    Pointer,
+    UpperExp,
+    UpperHex, //
+};
 impl_fmt_adapter_forward!(Debug, LowerHex, UpperHex, Octal, Binary, Pointer, LowerExp, UpperExp);
 
 /// A copy of [`core::fmt::Display`] that allows us to implement it for foreign types.
diff --git a/rust/kernel/prelude.rs b/rust/kernel/prelude.rs
index 44edf72a4a24e..bcd4e7f90bc72 100644
--- a/rust/kernel/prelude.rs
+++ b/rust/kernel/prelude.rs
@@ -22,6 +22,7 @@
     pin::Pin, //
 };
 
+#[doc(no_inline)]
 pub use ::ffi::{
     c_char,
     c_int,
@@ -47,6 +48,7 @@
     vtable, //
 };
 
+#[doc(no_inline)]
 pub use pin_init::{
     init,
     pin_data,
@@ -58,6 +60,7 @@
     Zeroable, //
 };
 
+#[doc(no_inline)]
 pub use super::{
     alloc::{
         flags::*,
diff --git a/rust/pin-init/README.md b/rust/pin-init/README.md
index 9095d6661ff60..2312c9e75f8c4 100644
--- a/rust/pin-init/README.md
+++ b/rust/pin-init/README.md
@@ -3,7 +3,7 @@
 [![Dependency status](https://deps.rs/repo/github/Rust-for-Linux/pin-init/status.svg)](https://deps.rs/repo/github/Rust-for-Linux/pin-init)
 ![License](https://img.shields.io/crates/l/pin-init)
 [![Toolchain](https://img.shields.io/badge/toolchain-nightly-red)](#nightly-only)
-![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/Rust-for-Linux/pin-init/test.yml)
+![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/Rust-for-Linux/pin-init/ci.yml)
 # `pin-init`
 
 > [!NOTE]
diff --git a/rust/pin-init/examples/big_struct_in_place.rs b/rust/pin-init/examples/big_struct_in_place.rs
index 80f89b5f8fd60..c05139927486d 100644
--- a/rust/pin-init/examples/big_struct_in_place.rs
+++ b/rust/pin-init/examples/big_struct_in_place.rs
@@ -1,8 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
-#![cfg_attr(USE_RUSTC_FEATURES, feature(lint_reasons))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(raw_ref_op))]
-
 use pin_init::*;
 
 // Struct with size over 1GiB
diff --git a/rust/pin-init/examples/error.rs b/rust/pin-init/examples/error.rs
index 8f4e135eb8ba5..96f095398e8d5 100644
--- a/rust/pin-init/examples/error.rs
+++ b/rust/pin-init/examples/error.rs
@@ -11,6 +11,7 @@
 pub struct Error;
 
 impl From<Infallible> for Error {
+    #[inline]
     fn from(e: Infallible) -> Self {
         match e {}
     }
@@ -18,6 +19,7 @@ fn from(e: Infallible) -> Self {
 
 #[cfg(feature = "alloc")]
 impl From<AllocError> for Error {
+    #[inline]
     fn from(_: AllocError) -> Self {
         Self
     }
diff --git a/rust/pin-init/examples/linked_list.rs b/rust/pin-init/examples/linked_list.rs
index 119169e4dc41c..424585fe226d2 100644
--- a/rust/pin-init/examples/linked_list.rs
+++ b/rust/pin-init/examples/linked_list.rs
@@ -2,8 +2,6 @@
 
 #![allow(clippy::undocumented_unsafe_blocks)]
 #![cfg_attr(feature = "alloc", feature(allocator_api))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(lint_reasons))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(raw_ref_op))]
 
 use core::{
     cell::Cell,
diff --git a/rust/pin-init/examples/mutex.rs b/rust/pin-init/examples/mutex.rs
index d53671f0edb89..35ecb5f68dc31 100644
--- a/rust/pin-init/examples/mutex.rs
+++ b/rust/pin-init/examples/mutex.rs
@@ -2,8 +2,6 @@
 
 #![allow(clippy::undocumented_unsafe_blocks)]
 #![cfg_attr(feature = "alloc", feature(allocator_api))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(lint_reasons))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(raw_ref_op))]
 #![allow(clippy::missing_safety_doc)]
 
 use core::{
@@ -220,7 +218,7 @@ fn main() {
         for h in handles {
             h.join().expect("thread panicked");
         }
-        println!("{:?}", &*mtx.lock());
+        println!("{:?}", *mtx.lock());
         assert_eq!(*mtx.lock(), workload * thread_count * 2);
     }
 }
diff --git a/rust/pin-init/examples/pthread_mutex.rs b/rust/pin-init/examples/pthread_mutex.rs
index f3b5cc9b71348..00f457e688272 100644
--- a/rust/pin-init/examples/pthread_mutex.rs
+++ b/rust/pin-init/examples/pthread_mutex.rs
@@ -3,8 +3,6 @@
 // inspired by <https://github.com/nbdd0121/pin-init/blob/trunk/examples/pthread_mutex.rs>
 #![allow(clippy::undocumented_unsafe_blocks)]
 #![cfg_attr(feature = "alloc", feature(allocator_api))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(lint_reasons))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(raw_ref_op))]
 
 #[cfg(not(windows))]
 mod pthread_mtx {
@@ -179,7 +177,7 @@ fn main() {
         for h in handles {
             h.join().expect("thread panicked");
         }
-        println!("{:?}", &*mtx.lock());
+        println!("{:?}", *mtx.lock());
         assert_eq!(*mtx.lock(), workload * thread_count * 2);
     }
 }
diff --git a/rust/pin-init/examples/static_init.rs b/rust/pin-init/examples/static_init.rs
index f7e53d1a5ae63..58cd4241b78ce 100644
--- a/rust/pin-init/examples/static_init.rs
+++ b/rust/pin-init/examples/static_init.rs
@@ -2,8 +2,6 @@
 
 #![allow(clippy::undocumented_unsafe_blocks)]
 #![cfg_attr(feature = "alloc", feature(allocator_api))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(lint_reasons))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(raw_ref_op))]
 #![allow(unused_imports)]
 
 use core::{
@@ -119,7 +117,7 @@ fn main() {
         for h in handles {
             h.join().expect("thread panicked");
         }
-        println!("{:?}, {:?}", &*mtx.lock(), &*COUNT.lock());
+        println!("{:?}, {:?}", *mtx.lock(), *COUNT.lock());
         assert_eq!(*mtx.lock(), workload * thread_count * 2);
     }
 }
diff --git a/rust/pin-init/internal/src/diagnostics.rs b/rust/pin-init/internal/src/diagnostics.rs
index 3bdb477c2f2b8..c7d9b3e624fc8 100644
--- a/rust/pin-init/internal/src/diagnostics.rs
+++ b/rust/pin-init/internal/src/diagnostics.rs
@@ -3,6 +3,7 @@
 use std::fmt::Display;
 
 use proc_macro2::TokenStream;
+use quote::quote_spanned;
 use syn::{spanned::Spanned, Error};
 
 pub(crate) struct DiagCtxt(TokenStream);
@@ -15,6 +16,19 @@ pub(crate) fn error(&mut self, span: impl Spanned, msg: impl Display) -> ErrorGu
         ErrorGuaranteed(())
     }
 
+    pub(crate) fn warn(&mut self, span: impl Spanned, msg: impl Display) {
+        // Have the message start on a new line for visual clarity.
+        let msg = format!("\n{}", msg);
+        self.0.extend(quote_spanned!(span.span() =>
+            // Approximate using deprecated warning while `proc_macro_diagnostic` is unstable.
+            const _: () = {
+                #[deprecated = #msg]
+                const fn warn() {}
+                warn();
+            };
+        ));
+    }
+
     pub(crate) fn with(
         fun: impl FnOnce(&mut DiagCtxt) -> Result<TokenStream, ErrorGuaranteed>,
     ) -> TokenStream {
diff --git a/rust/pin-init/internal/src/init.rs b/rust/pin-init/internal/src/init.rs
index 487ee0013fafe..699b105570a54 100644
--- a/rust/pin-init/internal/src/init.rs
+++ b/rust/pin-init/internal/src/init.rs
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
 use proc_macro2::{Span, TokenStream};
-use quote::{format_ident, quote, quote_spanned};
+use quote::{format_ident, quote};
 use syn::{
     braced,
     parse::{End, Parse},
@@ -103,17 +103,15 @@ pub(crate) fn expand(
         |(_, err)| Box::new(err),
     );
     let slot = format_ident!("slot");
-    let (has_data_trait, data_trait, get_data, init_from_closure) = if pinned {
+    let (has_data_trait, get_data, init_from_closure) = if pinned {
         (
             format_ident!("HasPinData"),
-            format_ident!("PinData"),
             format_ident!("__pin_data"),
             format_ident!("pin_init_from_closure"),
         )
     } else {
         (
             format_ident!("HasInitData"),
-            format_ident!("InitData"),
             format_ident!("__init_data"),
             format_ident!("init_from_closure"),
         )
@@ -157,8 +155,7 @@ fn assert_zeroable<T: ?::core::marker::Sized>(_: *mut T)
             #path::#get_data()
         };
         // Ensure that `#data` really is of type `#data` and help with type inference:
-        let init = ::pin_init::__internal::#data_trait::make_closure::<_, #error>(
-            #data,
+        let init = #data.__make_closure::<_, #error>(
             move |slot| {
                 #zeroable_check
                 #this
@@ -172,14 +169,7 @@ fn assert_zeroable<T: ?::core::marker::Sized>(_: *mut T)
             init(slot).map(|__InitOk| ())
         };
         // SAFETY: TODO
-        let init = unsafe { ::pin_init::#init_from_closure::<_, #error>(init) };
-        // FIXME: this let binding is required to avoid a compiler error (cycle when computing the
-        // opaque type returned by this function) before Rust 1.81. Remove after MSRV bump.
-        #[allow(
-            clippy::let_and_return,
-            reason = "some clippy versions warn about the let binding"
-        )]
-        init
+        unsafe { ::pin_init::#init_from_closure::<_, #error>(init) }
     }})
 }
 
@@ -238,107 +228,80 @@ fn init_fields(
             cfgs.retain(|attr| attr.path().is_ident("cfg"));
             cfgs
         };
+
+        let ident = match kind {
+            InitializerKind::Value { ident, .. } => ident,
+            InitializerKind::Init { ident, .. } => ident,
+            InitializerKind::Code { block, .. } => {
+                res.extend(quote! {
+                    #(#attrs)*
+                    #[allow(unused_braces)]
+                    #block
+                });
+                continue;
+            }
+        };
+
+        let slot = if pinned {
+            quote! {
+                // SAFETY:
+                // - `slot` is valid and properly aligned.
+                // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned.
+                // - `make_field_check` prevents `#ident` from being used twice, therefore
+                //   `(*slot).#ident` is exclusively accessed and has not been initialized.
+                (unsafe { #data.#ident(#slot) })
+            }
+        } else {
+            quote! {
+                // For `init!()` macro, everything is unpinned.
+                // SAFETY:
+                // - `&raw mut (*slot).#ident` is valid.
+                // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned.
+                // - `make_field_check` prevents `#ident` from being used twice, therefore
+                //   `(*slot).#ident` is exclusively accessed and has not been initialized.
+                (unsafe {
+                    ::pin_init::__internal::Slot::<::pin_init::__internal::Unpinned, _>::new(
+                        &raw mut (*#slot).#ident
+                    )
+                })
+            }
+        };
+
+        // `mixed_site` ensures that the guard is not accessible to the user-controlled code.
+        let guard = format_ident!("__{ident}_guard", span = Span::mixed_site());
+
         let init = match kind {
             InitializerKind::Value { ident, value } => {
-                let mut value_ident = ident.clone();
-                let value_prep = value.as_ref().map(|value| &value.1).map(|value| {
-                    // Setting the span of `value_ident` to `value`'s span improves error messages
-                    // when the type of `value` is wrong.
-                    value_ident.set_span(value.span());
-                    quote!(let #value_ident = #value;)
-                });
-                // Again span for better diagnostics
-                let write = quote_spanned!(ident.span()=> ::core::ptr::write);
+                let value = value
+                    .as_ref()
+                    .map(|(_, value)| quote!(#value))
+                    .unwrap_or_else(|| quote!(#ident));
+
                 quote! {
                     #(#attrs)*
-                    {
-                        #value_prep
-                        // SAFETY: TODO
-                        unsafe { #write(&raw mut (*#slot).#ident, #value_ident) };
-                    }
+                    let mut #guard = #slot.write(#value);
+
                 }
             }
-            InitializerKind::Init { ident, value, .. } => {
-                // Again span for better diagnostics
-                let init = format_ident!("init", span = value.span());
-                let value_init = if pinned {
-                    quote! {
-                        // SAFETY:
-                        // - `slot` is valid, because we are inside of an initializer closure, we
-                        //   return when an error/panic occurs.
-                        // - We also use `#data` to require the correct trait (`Init` or `PinInit`)
-                        //   for `#ident`.
-                        unsafe { #data.#ident(&raw mut (*#slot).#ident, #init)? };
-                    }
-                } else {
-                    quote! {
-                        // SAFETY: `slot` is valid, because we are inside of an initializer
-                        // closure, we return when an error/panic occurs.
-                        unsafe {
-                            ::pin_init::Init::__init(
-                                #init,
-                                &raw mut (*#slot).#ident,
-                            )?
-                        };
-                    }
-                };
+            InitializerKind::Init { value, .. } => {
                 quote! {
                     #(#attrs)*
-                    {
-                        let #init = #value;
-                        #value_init
-                    }
+                    let mut #guard = #slot.init(#value)?;
                 }
             }
-            InitializerKind::Code { block: value, .. } => quote! {
-                #(#attrs)*
-                #[allow(unused_braces)]
-                #value
-            },
+            InitializerKind::Code { .. } => unreachable!(),
         };
-        res.extend(init);
-        if let Some(ident) = kind.ident() {
-            // `mixed_site` ensures that the guard is not accessible to the user-controlled code.
-            let guard = format_ident!("__{ident}_guard", span = Span::mixed_site());
 
-            // NOTE: The reference is derived from the guard so that it only lives as long as the
-            // guard does and cannot escape the scope. If it's created via `&mut (*#slot).#ident`
-            // like the unaligned field guard, it will become effectively `'static`.
-            let accessor = if pinned {
-                let project_ident = format_ident!("__project_{ident}");
-                quote! {
-                    // SAFETY: the initialization is pinned.
-                    unsafe { #data.#project_ident(#guard.let_binding()) }
-                }
-            } else {
-                quote! {
-                    #guard.let_binding()
-                }
-            };
+        res.extend(quote! {
+            #init
 
-            res.extend(quote! {
-                #(#cfgs)*
-                // Create the drop guard.
-                //
-                // SAFETY:
-                // - `&raw mut (*slot).#ident` is valid.
-                // - `make_field_check` checks that `&raw mut (*slot).#ident` is properly aligned.
-                // - `(*slot).#ident` has been initialized above.
-                // - We only need the ownership to the pointee back when initialization has
-                //   succeeded, where we `forget` the guard.
-                let mut #guard = unsafe {
-                    ::pin_init::__internal::DropGuard::new(
-                        &raw mut (*slot).#ident
-                    )
-                };
+            #(#cfgs)*
+            #[allow(unused_variables)]
+            let #ident = #guard.let_binding();
+        });
 
-                #(#cfgs)*
-                #[allow(unused_variables)]
-                let #ident = #accessor;
-            });
-            guards.push(guard);
-            guard_attrs.push(cfgs);
-        }
+        guards.push(guard);
+        guard_attrs.push(cfgs);
     }
     quote! {
         #res
diff --git a/rust/pin-init/internal/src/lib.rs b/rust/pin-init/internal/src/lib.rs
index b08dfe0030317..60d5093f3128e 100644
--- a/rust/pin-init/internal/src/lib.rs
+++ b/rust/pin-init/internal/src/lib.rs
@@ -6,7 +6,6 @@
 
 //! `pin-init` proc macros.
 
-#![cfg_attr(USE_RUSTC_FEATURES, feature(lint_reasons))]
 // Documentation is done in the pin-init crate instead.
 #![allow(missing_docs)]
 
diff --git a/rust/pin-init/internal/src/pin_data.rs b/rust/pin-init/internal/src/pin_data.rs
index 7d871236b49c7..2284256a62207 100644
--- a/rust/pin-init/internal/src/pin_data.rs
+++ b/rust/pin-init/internal/src/pin_data.rs
@@ -35,6 +35,11 @@ fn parse(input: syn::parse::ParseStream<'_>) -> syn::Result<Self> {
     }
 }
 
+struct FieldInfo<'a> {
+    field: &'a Field,
+    pinned: bool,
+}
+
 pub(crate) fn pin_data(
     args: Args,
     input: Item,
@@ -73,24 +78,30 @@ pub(crate) fn pin_data(
     replacer.visit_generics_mut(&mut struct_.generics);
     replacer.visit_fields_mut(&mut struct_.fields);
 
-    let fields: Vec<(bool, &Field)> = struct_
+    let fields: Vec<FieldInfo<'_>> = struct_
         .fields
         .iter_mut()
         .map(|field| {
             let len = field.attrs.len();
             field.attrs.retain(|a| !a.path().is_ident("pin"));
-            (len != field.attrs.len(), &*field)
+            let pinned = len != field.attrs.len();
+
+            FieldInfo {
+                field: &*field,
+                pinned,
+            }
         })
         .collect();
 
-    for (pinned, field) in &fields {
-        if !pinned && is_phantom_pinned(&field.ty) {
-            dcx.error(
-                field,
+    for field in &fields {
+        let ident = field.field.ident.as_ref().unwrap();
+
+        if !field.pinned && is_phantom_pinned(&field.field.ty) {
+            dcx.warn(
+                field.field,
                 format!(
-                    "The field `{}` of type `PhantomPinned` only has an effect \
+                    "The field `{ident}` of type `PhantomPinned` only has an effect \
                     if it has the `#[pin]` attribute",
-                    field.ident.as_ref().unwrap(),
                 ),
             );
         }
@@ -143,7 +154,7 @@ fn is_phantom_pinned(ty: &Type) -> bool {
 fn generate_unpin_impl(
     ident: &Ident,
     generics: &Generics,
-    fields: &[(bool, &Field)],
+    fields: &[FieldInfo<'_>],
 ) -> TokenStream {
     let (_, ty_generics, _) = generics.split_for_impl();
     let mut generics_with_pin_lt = generics.clone();
@@ -160,7 +171,7 @@ fn generate_unpin_impl(
     else {
         unreachable!()
     };
-    let pinned_fields = fields.iter().filter_map(|(b, f)| b.then_some(f));
+    let pinned_fields = fields.iter().filter(|f| f.pinned).map(|f| f.field);
     quote! {
         // This struct will be used for the unpin analysis. It is needed, because only structurally
         // pinned fields are relevant whether the struct should implement `Unpin`.
@@ -169,10 +180,8 @@ struct __Unpin #generics_with_pin_lt
         #where_token
             #predicates
         {
-            __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
-            __phantom: ::core::marker::PhantomData<
-                fn(#ident #ty_generics) -> #ident #ty_generics
-            >,
+            __phantom_pin: ::pin_init::__internal::PhantomInvariantLifetime<'__pin>,
+            __phantom: ::pin_init::__internal::PhantomInvariant<#ident #ty_generics>,
             #(#pinned_fields),*
         }
 
@@ -238,7 +247,7 @@ fn generate_projections(
     vis: &Visibility,
     ident: &Ident,
     generics: &Generics,
-    fields: &[(bool, &Field)],
+    fields: &[FieldInfo<'_>],
 ) -> TokenStream {
     let (impl_generics, ty_generics, _) = generics.split_for_impl();
     let mut generics_with_pin_lt = generics.clone();
@@ -247,25 +256,23 @@ fn generate_projections(
     let projection = format_ident!("{ident}Projection");
     let this = format_ident!("this");
 
-    let (fields_decl, fields_proj) = collect_tuple(fields.iter().map(
-        |(
-            pinned,
-            Field {
+    let (fields_decl, fields_proj): (Vec<_>, Vec<_>) = fields
+        .iter()
+        .map(|field| {
+            let Field {
                 vis,
                 ident,
                 ty,
                 attrs,
                 ..
-            },
-        )| {
-            let mut attrs = attrs.clone();
-            attrs.retain(|a| !a.path().is_ident("pin"));
+            } = &field.field;
+
             let mut no_doc_attrs = attrs.clone();
             no_doc_attrs.retain(|a| !a.path().is_ident("doc"));
             let ident = ident
                 .as_ref()
                 .expect("only structs with named fields are supported");
-            if *pinned {
+            if field.pinned {
                 (
                     quote!(
                         #(#attrs)*
@@ -289,22 +296,24 @@ fn generate_projections(
                     ),
                 )
             }
-        },
-    ));
+        })
+        .collect();
     let structurally_pinned_fields_docs = fields
         .iter()
-        .filter_map(|(pinned, field)| pinned.then_some(field))
-        .map(|Field { ident, .. }| format!(" - `{}`", ident.as_ref().unwrap()));
+        .filter(|f| f.pinned)
+        .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap()));
     let not_structurally_pinned_fields_docs = fields
         .iter()
-        .filter_map(|(pinned, field)| (!pinned).then_some(field))
-        .map(|Field { ident, .. }| format!(" - `{}`", ident.as_ref().unwrap()));
+        .filter(|f| !f.pinned)
+        .map(|f| format!(" - `{}`", f.field.ident.as_ref().unwrap()));
     let docs = format!(" Pin-projections of [`{ident}`]");
     quote! {
         #[doc = #docs]
         #[allow(dead_code)]
         #[doc(hidden)]
-        #vis struct #projection #generics_with_pin_lt {
+        #vis struct #projection #generics_with_pin_lt
+            #whr
+        {
             #(#fields_decl)*
             ___pin_phantom_data: ::core::marker::PhantomData<&'__pin mut ()>,
         }
@@ -336,91 +345,56 @@ impl #impl_generics #ident #ty_generics
 
 fn generate_the_pin_data(
     vis: &Visibility,
-    ident: &Ident,
+    struct_name: &Ident,
     generics: &Generics,
-    fields: &[(bool, &Field)],
+    fields: &[FieldInfo<'_>],
 ) -> TokenStream {
     let (impl_generics, ty_generics, whr) = generics.split_for_impl();
 
     // For every field, we create an initializing projection function according to its projection
-    // type. If a field is structurally pinned, then it must be initialized via `PinInit`, if it is
-    // not structurally pinned, then it can be initialized via `Init`.
-    //
-    // The functions are `unsafe` to prevent accidentally calling them.
-    fn handle_field(
-        Field {
-            vis,
-            ident,
-            ty,
-            attrs,
-            ..
-        }: &Field,
-        struct_ident: &Ident,
-        pinned: bool,
-    ) -> TokenStream {
-        let mut attrs = attrs.clone();
-        attrs.retain(|a| !a.path().is_ident("pin"));
-        let ident = ident
-            .as_ref()
-            .expect("only structs with named fields are supported");
-        let project_ident = format_ident!("__project_{ident}");
-        let (init_ty, init_fn, project_ty, project_body, pin_safety) = if pinned {
-            (
-                quote!(PinInit),
-                quote!(__pinned_init),
-                quote!(::core::pin::Pin<&'__slot mut #ty>),
-                // SAFETY: this field is structurally pinned.
-                quote!(unsafe { ::core::pin::Pin::new_unchecked(slot) }),
-                quote!(
-                    /// - `slot` will not move until it is dropped, i.e. it will be pinned.
-                ),
-            )
-        } else {
-            (
-                quote!(Init),
-                quote!(__init),
-                quote!(&'__slot mut #ty),
-                quote!(slot),
-                quote!(),
-            )
-        };
-        let slot_safety = format!(
-            " `slot` points at the field `{ident}` inside of `{struct_ident}`, which is pinned.",
-        );
-        quote! {
-            /// # Safety
-            ///
-            /// - `slot` is a valid pointer to uninitialized memory.
-            /// - the caller does not touch `slot` when `Err` is returned, they are only permitted
-            ///   to deallocate.
-            #pin_safety
-            #(#attrs)*
-            #vis unsafe fn #ident<E>(
-                self,
-                slot: *mut #ty,
-                init: impl ::pin_init::#init_ty<#ty, E>,
-            ) -> ::core::result::Result<(), E> {
-                // SAFETY: this function has the same safety requirements as the __init function
-                // called below.
-                unsafe { ::pin_init::#init_ty::#init_fn(init, slot) }
-            }
-
-            /// # Safety
-            ///
-            #[doc = #slot_safety]
-            #(#attrs)*
-            #vis unsafe fn #project_ident<'__slot>(
-                self,
-                slot: &'__slot mut #ty,
-            ) -> #project_ty {
-                #project_body
-            }
-        }
-    }
-
+    // type. If a field is structurally pinned, we create a `Slot` with `Pinned` which must be
+    // initialized via `PinInit`; if it is not structurally pinned, then we create a `Slot` with
+    // `Unpinned` which allows initialization via `Init`.
     let field_accessors = fields
         .iter()
-        .map(|(pinned, field)| handle_field(field, ident, *pinned))
+        .map(|f| {
+            let Field {
+                vis,
+                ident,
+                ty,
+                attrs,
+                ..
+            } = f.field;
+
+            let field_name = ident
+                .as_ref()
+                .expect("only structs with named fields are supported");
+            let pin_marker = if f.pinned {
+                quote!(Pinned)
+            } else {
+                quote!(Unpinned)
+            };
+            quote! {
+                /// # Safety
+                ///
+                /// - `slot` is valid and properly aligned.
+                /// - `(*slot).#field_name` is properly aligned.
+                /// - `(*slot).#field_name` points to uninitialized and exclusively accessed
+                ///   memory.
+                #(#attrs)*
+                #[inline(always)]
+                #vis unsafe fn #field_name(
+                    self,
+                    slot: *mut #struct_name #ty_generics,
+                ) -> ::pin_init::__internal::Slot<::pin_init::__internal::#pin_marker, #ty> {
+                    // SAFETY:
+                    // - If `#pin_marker` is `Pinned`, the corresponding field is structurally
+                    //   pinned.
+                    // - Other safety requirements follows the safety requirement.
+                    unsafe { ::pin_init::__internal::Slot::new(&raw mut (*slot).#field_name) }
+                }
+            }
+        })
         .collect::<TokenStream>();
     quote! {
         // We declare this struct which will host all of the projection function for our type. It
@@ -429,9 +403,7 @@ fn handle_field(
         #vis struct __ThePinData #generics
             #whr
         {
-            __phantom: ::core::marker::PhantomData<
-                fn(#ident #ty_generics) -> #ident #ty_generics
-            >,
+            __phantom: ::pin_init::__internal::PhantomInvariant<#struct_name #ty_generics>,
         }
 
         impl #impl_generics ::core::clone::Clone for __ThePinData #ty_generics
@@ -449,27 +421,30 @@ impl #impl_generics ::core::marker::Copy for __ThePinData #ty_generics
         impl #impl_generics __ThePinData #ty_generics
             #whr
         {
+            /// Type inference helper function.
+            #[inline(always)]
+            #vis fn __make_closure<__F, __E>(self, f: __F) -> __F
+            where
+                __F: FnOnce(*mut #struct_name #ty_generics) ->
+                    ::core::result::Result<::pin_init::__internal::InitOk, __E>,
+            {
+                f
+            }
+
             #field_accessors
         }
 
         // SAFETY: We have added the correct projection functions above to `__ThePinData` and
         // we also use the least restrictive generics possible.
-        unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #ident #ty_generics
+        unsafe impl #impl_generics ::pin_init::__internal::HasPinData for #struct_name #ty_generics
             #whr
         {
             type PinData = __ThePinData #ty_generics;
 
             unsafe fn __pin_data() -> Self::PinData {
-                __ThePinData { __phantom: ::core::marker::PhantomData }
+                __ThePinData { __phantom: ::pin_init::__internal::PhantomInvariant::new() }
             }
         }
-
-        // SAFETY: TODO
-        unsafe impl #impl_generics ::pin_init::__internal::PinData for __ThePinData #ty_generics
-            #whr
-        {
-            type Datee = #ident #ty_generics;
-        }
     }
 }
 
@@ -500,14 +475,3 @@ fn visit_item_mut(&mut self, _: &mut Item) {
         // Do not descend into items, since items reset/change what `Self` refers to.
     }
 }
-
-// replace with `.collect()` once MSRV is above 1.79
-fn collect_tuple<A, B>(iter: impl Iterator<Item = (A, B)>) -> (Vec<A>, Vec<B>) {
-    let mut res_a = vec![];
-    let mut res_b = vec![];
-    for (a, b) in iter {
-        res_a.push(a);
-        res_b.push(b);
-    }
-    (res_a, res_b)
-}
diff --git a/rust/pin-init/internal/src/zeroable.rs b/rust/pin-init/internal/src/zeroable.rs
index 05683319b0f7b..b11feaeb1ca6e 100644
--- a/rust/pin-init/internal/src/zeroable.rs
+++ b/rust/pin-init/internal/src/zeroable.rs
@@ -1,4 +1,4 @@
-// SPDX-License-Identifier: GPL-2.0
+// SPDX-License-Identifier: Apache-2.0 OR MIT
 
 use proc_macro2::TokenStream;
 use quote::quote;
diff --git a/rust/pin-init/src/__internal.rs b/rust/pin-init/src/__internal.rs
index 5720a621aed74..540add6cee8a5 100644
--- a/rust/pin-init/src/__internal.rs
+++ b/rust/pin-init/src/__internal.rs
@@ -7,20 +7,62 @@
 
 use super::*;
 
-/// See the [nomicon] for what subtyping is. See also [this table].
+/// Zero-sized type used to mark a type as invariant.
+///
+/// This is a polyfill for the [unstable type] in the standard library of the same name.
 ///
-/// The reason for not using `PhantomData<*mut T>` is that that type never implements [`Send`] and
-/// [`Sync`]. Hence `fn(*mut T) -> *mut T` is used, as that type always implements them.
+/// See the [nomicon] for what subtyping is. See also [this table].
 ///
+/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariant.html
 /// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html
 /// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns
-pub(crate) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>;
+#[repr(transparent)]
+pub struct PhantomInvariant<T: ?Sized>(PhantomData<fn(T) -> T>);
+
+impl<T: ?Sized> Clone for PhantomInvariant<T> {
+    #[inline(always)]
+    fn clone(&self) -> Self {
+        *self
+    }
+}
+
+impl<T: ?Sized> Copy for PhantomInvariant<T> {}
+
+impl<T: ?Sized> Default for PhantomInvariant<T> {
+    #[inline(always)]
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl<T: ?Sized> PhantomInvariant<T> {
+    #[inline(always)]
+    pub const fn new() -> Self {
+        Self(PhantomData)
+    }
+}
+
+/// Zero-sized type used to mark a lifetime as invariant.
+///
+/// This is a polyfill for the [unstable type] in the standard library of the same name.
+///
+/// [unstable type]: https://doc.rust-lang.org/nightly/std/marker/struct.PhantomInvariantLifetime.html
+#[repr(transparent)]
+#[derive(Clone, Copy, Default)]
+pub struct PhantomInvariantLifetime<'a>(PhantomInvariant<&'a ()>);
+
+impl PhantomInvariantLifetime<'_> {
+    #[inline(always)]
+    pub const fn new() -> Self {
+        Self(PhantomInvariant::new())
+    }
+}
 
 /// Module-internal type implementing `PinInit` and `Init`.
 ///
 /// It is unsafe to create this type, since the closure needs to fulfill the same safety
 /// requirement as the `__pinned_init`/`__init` functions.
-pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>);
+pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) PhantomInvariant<(E, T)>);
 
 // SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
 // `__init` invariants.
@@ -71,30 +113,12 @@ pub unsafe fn new() -> Self {
 ///
 /// Only the `init` module is allowed to use this trait.
 pub unsafe trait HasPinData {
-    type PinData: PinData;
+    type PinData;
 
     #[expect(clippy::missing_safety_doc)]
     unsafe fn __pin_data() -> Self::PinData;
 }
 
-/// Marker trait for pinning data of structs.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait PinData: Copy {
-    type Datee: ?Sized + HasPinData;
-
-    /// Type inference helper function.
-    #[inline(always)]
-    fn make_closure<F, E>(self, f: F) -> F
-    where
-        F: FnOnce(*mut Self::Datee) -> Result<InitOk, E>,
-    {
-        f
-    }
-}
-
 /// This trait is automatically implemented for every type. It aims to provide the same type
 /// inference help as `HasPinData`.
 ///
@@ -102,31 +126,13 @@ fn make_closure<F, E>(self, f: F) -> F
 ///
 /// Only the `init` module is allowed to use this trait.
 pub unsafe trait HasInitData {
-    type InitData: InitData;
+    type InitData;
 
     #[expect(clippy::missing_safety_doc)]
     unsafe fn __init_data() -> Self::InitData;
 }
 
-/// Same function as `PinData`, but for arbitrary data.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait InitData: Copy {
-    type Datee: ?Sized + HasInitData;
-
-    /// Type inference helper function.
-    #[inline(always)]
-    fn make_closure<F, E>(self, f: F) -> F
-    where
-        F: FnOnce(*mut Self::Datee) -> Result<InitOk, E>,
-    {
-        f
-    }
-}
-
-pub struct AllData<T: ?Sized>(Invariant<T>);
+pub struct AllData<T: ?Sized>(PhantomInvariant<T>);
 
 impl<T: ?Sized> Clone for AllData<T> {
     fn clone(&self) -> Self {
@@ -136,9 +142,15 @@ fn clone(&self) -> Self {
 
 impl<T: ?Sized> Copy for AllData<T> {}
 
-// SAFETY: TODO.
-unsafe impl<T: ?Sized> InitData for AllData<T> {
-    type Datee = T;
+impl<T: ?Sized> AllData<T> {
+    /// Type inference helper function.
+    #[inline(always)]
+    pub fn __make_closure<F, E>(self, f: F) -> F
+    where
+        F: FnOnce(*mut T) -> Result<InitOk, E>,
+    {
+        f
+    }
 }
 
 // SAFETY: TODO.
@@ -146,7 +158,7 @@ unsafe impl<T: ?Sized> HasInitData for T {
     type InitData = AllData<T>;
 
     unsafe fn __init_data() -> Self::InitData {
-        AllData(PhantomData)
+        AllData(PhantomInvariant::new())
     }
 }
 
@@ -235,6 +247,87 @@ struct Foo {
     println!("{value:?}");
 }
 
+// Marker types that determines type of `DropGuard`'s let bindings.
+pub struct Pinned;
+pub struct Unpinned;
+
+/// Represent an uninitialized field.
+///
+/// # Invariants
+///
+/// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed memory.
+/// - If `P` is `Pinned`, then `ptr` is structurally pinned.
+pub struct Slot<P, T: ?Sized> {
+    ptr: *mut T,
+    _phantom: PhantomData<P>,
+}
+
+impl<P, T: ?Sized> Slot<P, T> {
+    /// # Safety
+    ///
+    /// - `ptr` is valid, properly aligned and points to uninitialized and exclusively accessed
+    ///   memory.
+    /// - If `P` is `Pinned`, then `ptr` is structurally pinned.
+    #[inline(always)]
+    pub unsafe fn new(ptr: *mut T) -> Self {
+        // INVARIANT: Per safety requirement.
+        Self {
+            ptr,
+            _phantom: PhantomData,
+        }
+    }
+
+    /// Initialize the field by value.
+    #[inline(always)]
+    pub fn write(self, value: T) -> DropGuard<P, T>
+    where
+        T: Sized,
+    {
+        // SAFETY: `self.ptr` is a valid and aligned pointer for write.
+        unsafe { self.ptr.write(value) }
+        // SAFETY:
+        // - `self.ptr` is valid and properly aligned per type invariant.
+        // - `*self.ptr` is initialized above and the ownership is transferred to the guard.
+        // - If `P` is `Pinned`, `self.ptr` is pinned.
+        unsafe { DropGuard::new(self.ptr) }
+    }
+}
+
+impl<T: ?Sized> Slot<Unpinned, T> {
+    /// Initialize the field.
+    #[inline(always)]
+    pub fn init<E>(self, init: impl Init<T, E>) -> Result<DropGuard<Unpinned, T>, E> {
+        // SAFETY:
+        // - `self.ptr` is valid and properly aligned.
+        // - when `Err` is returned, we also propagate the error without touching `slot`;
+        //   also `self` is consumed so it cannot be touched further.
+        unsafe { init.__init(self.ptr)? };
+
+        // SAFETY:
+        // - `self.ptr` is valid and properly aligned per type invariant.
+        // - `*self.ptr` is initialized above and the ownership is transferred to the guard.
+        Ok(unsafe { DropGuard::new(self.ptr) })
+    }
+}
+
+impl<T: ?Sized> Slot<Pinned, T> {
+    /// Initialize the field.
+    #[inline(always)]
+    pub fn init<E>(self, init: impl PinInit<T, E>) -> Result<DropGuard<Pinned, T>, E> {
+        // SAFETY:
+        // - `self.ptr` is valid and properly aligned.
+        // - when `Err` is returned, we also propagate the error without touching `ptr`;
+        //   also `self` is consumed so it cannot be touched further.
+        // - the drop guard will not hand out `&mut` (only `Pin<&mut T>`).
+        unsafe { init.__pinned_init(self.ptr)? };
+
+        // SAFETY:
+        // - `self.ptr` is valid, properly aligned and pinned per type invariant.
+        // - `*self.ptr` is initialized above and the ownership is transferred to the guard.
+        Ok(unsafe { DropGuard::new(self.ptr) })
+    }
+}
+
 /// When a value of this type is dropped, it drops a `T`.
 ///
 /// Can be forgotten to prevent the drop.
@@ -243,11 +336,13 @@ struct Foo {
 ///
 /// - `ptr` is valid and properly aligned.
 /// - `*ptr` is initialized and owned by this guard.
-pub struct DropGuard<T: ?Sized> {
+/// - if `P` is `Pinned`, `ptr` is pinned.
+pub struct DropGuard<P, T: ?Sized> {
     ptr: *mut T,
+    phantom: PhantomData<P>,
 }
 
-impl<T: ?Sized> DropGuard<T> {
+impl<P, T: ?Sized> DropGuard<P, T> {
     /// Creates a drop guard and transfer the ownership of the pointer content.
     ///
     /// The ownership is only relinguished if the guard is forgotten via [`core::mem::forget`].
@@ -256,12 +351,18 @@ impl<T: ?Sized> DropGuard<T> {
     ///
     /// - `ptr` is valid and properly aligned.
     /// - `*ptr` is initialized, and the ownership is transferred to this guard.
+    /// - if `P` is `Pinned`, `ptr` is pinned.
     #[inline]
     pub unsafe fn new(ptr: *mut T) -> Self {
         // INVARIANT: By safety requirement.
-        Self { ptr }
+        Self {
+            ptr,
+            phantom: PhantomData,
+        }
     }
+}
 
+impl<T: ?Sized> DropGuard<Unpinned, T> {
     /// Create a let binding for accessor use.
     #[inline]
     pub fn let_binding(&mut self) -> &mut T {
@@ -270,7 +371,17 @@ pub fn let_binding(&mut self) -> &mut T {
     }
 }
 
-impl<T: ?Sized> Drop for DropGuard<T> {
+impl<T: ?Sized> DropGuard<Pinned, T> {
+    /// Create a let binding for accessor use.
+    #[inline]
+    pub fn let_binding(&mut self) -> Pin<&mut T> {
+        // SAFETY: `self.ptr` is valid, properly aligned, initialized, exclusively accessible and
+        // pinned per type invariant.
+        unsafe { Pin::new_unchecked(&mut *self.ptr) }
+    }
+}
+
+impl<P, T: ?Sized> Drop for DropGuard<P, T> {
     #[inline]
     fn drop(&mut self) {
         // SAFETY: `self.ptr` is valid, properly aligned and `*self.ptr` is owned by this guard.
diff --git a/rust/pin-init/src/lib.rs b/rust/pin-init/src/lib.rs
index 64eec095c859c..c9e2cbe279157 100644
--- a/rust/pin-init/src/lib.rs
+++ b/rust/pin-init/src/lib.rs
@@ -263,12 +263,6 @@
 //! [`impl Init<T, E>`]: crate::Init
 //! [Rust-for-Linux]: https://rust-for-linux.com/
 
-#![cfg_attr(USE_RUSTC_FEATURES, feature(lint_reasons))]
-#![cfg_attr(USE_RUSTC_FEATURES, feature(raw_ref_op))]
-#![cfg_attr(
-    all(any(feature = "alloc", feature = "std"), USE_RUSTC_FEATURES),
-    feature(new_uninit)
-)]
 #![forbid(missing_docs, unsafe_op_in_unsafe_fn)]
 #![cfg_attr(not(feature = "std"), no_std)]
 #![cfg_attr(feature = "alloc", feature(allocator_api))]
@@ -873,12 +867,12 @@ macro_rules! stack_try_pin_init {
 #[macro_export]
 macro_rules! assert_pinned {
     ($ty:ty, $field:ident, $field_ty:ty, inline) => {
-        let _ = move |ptr: *mut $field_ty| {
-            // SAFETY: This code is unreachable.
-            let data = unsafe { <$ty as $crate::__internal::HasPinData>::__pin_data() };
-            let init = $crate::__internal::AlwaysFail::<$field_ty>::new();
-            // SAFETY: This code is unreachable.
-            unsafe { data.$field(ptr, init) }.ok();
+        // SAFETY: This code is unreachable.
+        let _ = move |ptr: *mut $ty| unsafe {
+            let data = <$ty as $crate::__internal::HasPinData>::__pin_data();
+            _ = data
+                .$field(ptr)
+                .init($crate::__internal::AlwaysFail::<$field_ty>::new());
         };
     };
 
@@ -953,12 +947,12 @@ fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
     where
         F: FnOnce(Pin<&mut T>) -> Result<(), E>,
     {
-        ChainPinInit(self, f, PhantomData)
+        ChainPinInit(self, f, __internal::PhantomInvariant::new())
     }
 }
 
 /// An initializer returned by [`PinInit::pin_chain`].
-pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>);
+pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>);
 
 // SAFETY: The `__pinned_init` function is implemented such that it
 // - returns `Ok(())` on successful initialization,
@@ -1061,12 +1055,12 @@ fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
     where
         F: FnOnce(&mut T) -> Result<(), E>,
     {
-        ChainInit(self, f, PhantomData)
+        ChainInit(self, f, __internal::PhantomInvariant::new())
     }
 }
 
 /// An initializer returned by [`Init::chain`].
-pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, T)>);
+pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::PhantomInvariant<(E, T)>);
 
 // SAFETY: The `__init` function is implemented such that it
 // - returns `Ok(())` on successful initialization,
@@ -1114,7 +1108,7 @@ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
 pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
     f: impl FnOnce(*mut T) -> Result<(), E>,
 ) -> impl PinInit<T, E> {
-    __internal::InitClosure(f, PhantomData)
+    __internal::InitClosure(f, __internal::PhantomInvariant::new())
 }
 
 /// Creates a new [`Init<T, E>`] from the given closure.
@@ -1133,7 +1127,7 @@ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
 pub const unsafe fn init_from_closure<T: ?Sized, E>(
     f: impl FnOnce(*mut T) -> Result<(), E>,
 ) -> impl Init<T, E> {
-    __internal::InitClosure(f, PhantomData)
+    __internal::InitClosure(f, __internal::PhantomInvariant::new())
 }
 
 /// Changes the to be initialized type.
@@ -1145,14 +1139,7 @@ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
 pub const unsafe fn cast_pin_init<T, U, E>(init: impl PinInit<T, E>) -> impl PinInit<U, E> {
     // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
     // requirements.
-    let res = unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) };
-    // FIXME: this let binding is required to avoid a compiler error (cycle when computing the opaque
-    // type returned by this function) before Rust 1.81. Remove after MSRV bump.
-    #[allow(
-        clippy::let_and_return,
-        reason = "some clippy versions warn about the let binding"
-    )]
-    res
+    unsafe { pin_init_from_closure(|ptr: *mut U| init.__pinned_init(ptr.cast::<T>())) }
 }
 
 /// Changes the to be initialized type.
@@ -1164,14 +1151,7 @@ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
 pub const unsafe fn cast_init<T, U, E>(init: impl Init<T, E>) -> impl Init<U, E> {
     // SAFETY: initialization delegated to a valid initializer. Cast is valid by function safety
     // requirements.
-    let res = unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) };
-    // FIXME: this let binding is required to avoid a compiler error (cycle when computing the opaque
-    // type returned by this function) before Rust 1.81. Remove after MSRV bump.
-    #[allow(
-        clippy::let_and_return,
-        reason = "some clippy versions warn about the let binding"
-    )]
-    res
+    unsafe { init_from_closure(|ptr: *mut U| init.__init(ptr.cast::<T>())) }
 }
 
 /// An initializer that leaves the memory uninitialized.
@@ -1523,27 +1503,6 @@ fn zeroed() -> Self
     }
 }
 
-/// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write
-/// `None` to that location.
-///
-/// # Safety
-///
-/// The implementer needs to ensure that `unsafe impl Zeroable for Option<Self> {}` is sound.
-pub unsafe trait ZeroableOption {}
-
-// SAFETY: by the safety requirement of `ZeroableOption`, this is valid.
-unsafe impl<T: ZeroableOption> Zeroable for Option<T> {}
-
-// SAFETY: `Option<&T>` is part of the option layout optimization guarantee:
-// <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
-unsafe impl<T> ZeroableOption for &T {}
-// SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee:
-// <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
-unsafe impl<T> ZeroableOption for &mut T {}
-// SAFETY: `Option<NonNull<T>>` is part of the option layout optimization guarantee:
-// <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
-unsafe impl<T> ZeroableOption for NonNull<T> {}
-
 /// Create an initializer for a zeroed `T`.
 ///
 /// The returned initializer will write `0x00` to every byte of the given `slot`.
@@ -1649,6 +1608,17 @@ unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*)
 
 impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
 
+/// Marker trait for types that allow `Option<Self>` to be set to all zeroes in order to write
+/// `None` to that location.
+///
+/// # Safety
+///
+/// The implementer needs to ensure that `unsafe impl Zeroable for Option<Self> {}` is sound.
+pub unsafe trait ZeroableOption {}
+
+// SAFETY: by the safety requirement of `ZeroableOption`, this is valid.
+unsafe impl<T: ZeroableOption> Zeroable for Option<T> {}
+
 macro_rules! impl_fn_zeroable_option {
     ([$($abi:literal),* $(,)?] $args:tt) => {
         $(impl_fn_zeroable_option!({extern $abi} $args);)*
@@ -1674,18 +1644,27 @@ unsafe impl<$ret, $($rest),*> ZeroableOption for $($prefix)* fn($($rest),*) -> $
 
 impl_fn_zeroable_option!(["Rust", "C"] { A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U });
 
-macro_rules! impl_non_zero_int_zeroable_option {
-    ($($int:ty),* $(,)?) => {
-        // SAFETY: Safety comment written in the macro invocation.
-        $(unsafe impl ZeroableOption for $int {})*
+macro_rules! impl_zeroable_option {
+    ($($({$($generics:tt)*})? $t:ty, )*) => {
+        // SAFETY: Safety comments written in the macro invocation.
+        $(unsafe impl$($($generics)*)? ZeroableOption for $t {})*
     };
 }
 
-impl_non_zero_int_zeroable_option! {
+impl_zeroable_option! {
+    // SAFETY: `Option<&T>` is part of the option layout optimization guarantee:
+    // <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
+    {<T: ?Sized>} &T,
+    // SAFETY: `Option<&mut T>` is part of the option layout optimization guarantee:
+    // <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
+    {<T: ?Sized>} &mut T,
+    // SAFETY: `Option<NonNull<T>>` is part of the option layout optimization guarantee:
+    // <https://doc.rust-lang.org/stable/std/option/index.html#representation>.
+    {<T: ?Sized>} NonNull<T>,
     // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee:
     // <https://doc.rust-lang.org/stable/std/option/index.html#representation>).
-    NonZeroU8, NonZeroU16, NonZeroU32, NonZeroU64, NonZeroU128, NonZeroUsize,
-    NonZeroI8, NonZeroI16, NonZeroI32, NonZeroI64, NonZeroI128, NonZeroIsize,
+    NonZero<u8>, NonZero<u16>, NonZero<u32>, NonZero<u64>, NonZero<u128>, NonZero<usize>,
+    NonZero<i8>, NonZero<i16>, NonZero<i32>, NonZero<i64>, NonZero<i128>, NonZero<isize>,
 }
 
 /// This trait allows creating an instance of `Self` which contains exactly one
diff --git a/samples/rust/rust_driver_auxiliary.rs b/samples/rust/rust_driver_auxiliary.rs
index 5c5a5105a3fff..319ef734c02bb 100644
--- a/samples/rust/rust_driver_auxiliary.rs
+++ b/samples/rust/rust_driver_auxiliary.rs
@@ -17,8 +17,6 @@
     InPlaceModule, //
 };
 
-use core::any::TypeId;
-
 const MODULE_NAME: &CStr = <LocalModule as kernel::ModuleMetadata>::NAME;
 const AUXILIARY_NAME: &CStr = c"auxiliary";
 
@@ -49,13 +47,13 @@ fn probe(adev: &auxiliary::Device<Core>, _info: &Self::IdInfo) -> impl PinInit<S
     }
 }
 
-#[pin_data]
+struct Data {
+    index: u32,
+}
+
 struct ParentDriver {
-    private: TypeId,
-    #[pin]
-    _reg0: Devres<auxiliary::Registration>,
-    #[pin]
-    _reg1: Devres<auxiliary::Registration>,
+    _reg0: Devres<auxiliary::Registration<Data>>,
+    _reg1: Devres<auxiliary::Registration<Data>>,
 }
 
 kernel::pci_device_table!(
@@ -71,10 +69,21 @@ impl pci::Driver for ParentDriver {
     const ID_TABLE: pci::IdTable<Self::IdInfo> = &PCI_TABLE;
 
     fn probe(pdev: &pci::Device<Core>, _info: &Self::IdInfo) -> impl PinInit<Self, Error> {
-        try_pin_init!(Self {
-            private: TypeId::of::<Self>(),
-            _reg0 <- auxiliary::Registration::new(pdev.as_ref(), AUXILIARY_NAME, 0, MODULE_NAME),
-            _reg1 <- auxiliary::Registration::new(pdev.as_ref(), AUXILIARY_NAME, 1, MODULE_NAME),
+        Ok(Self {
+            _reg0: auxiliary::Registration::new(
+                pdev.as_ref(),
+                AUXILIARY_NAME,
+                0,
+                MODULE_NAME,
+                Data { index: 0 },
+            )?,
+            _reg1: auxiliary::Registration::new(
+                pdev.as_ref(),
+                AUXILIARY_NAME,
+                1,
+                MODULE_NAME,
+                Data { index: 1 },
+            )?,
         })
     }
 }
@@ -83,7 +92,8 @@ impl ParentDriver {
     fn connect(adev: &auxiliary::Device<Bound>) -> Result {
         let dev = adev.parent();
         let pdev: &pci::Device<Bound> = dev.try_into()?;
-        let drvdata = dev.drvdata::<Self>()?;
+
+        let data = adev.registration_data::<Data>()?;
 
         dev_info!(
             dev,
@@ -95,8 +105,8 @@ fn connect(adev: &auxiliary::Device<Bound>) -> Result {
 
         dev_info!(
             dev,
-            "We have access to the private data of {:?}.\n",
-            drvdata.private
+            "Connected to auxiliary device with index {}.\n",
+            data.index
         );
 
         Ok(())
diff --git a/scripts/Makefile b/scripts/Makefile
index 3434a82a119f0..b028214fe767b 100644
--- a/scripts/Makefile
+++ b/scripts/Makefile
@@ -64,6 +64,7 @@ subdir-$(CONFIG_GENKSYMS) += genksyms
 subdir-$(CONFIG_GENDWARFKSYMS) += gendwarfksyms
 subdir-$(CONFIG_SECURITY_SELINUX) += selinux
 subdir-$(CONFIG_SECURITY_IPE) += ipe
+subdir-$(CONFIG_SECURITY_HORNET) += hornet
 
 # Let clean descend into subdirs
 subdir-	+= basic dtc gdb kconfig mod
diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
index 0718e39cedda2..7e216d82e9887 100644
--- a/scripts/Makefile.lib
+++ b/scripts/Makefile.lib
@@ -187,7 +187,11 @@ objtool-args-$(CONFIG_HAVE_JUMP_LABEL_HACK)		+= --hacks=jump_label
 objtool-args-$(CONFIG_HAVE_NOINSTR_HACK)		+= --hacks=noinstr
 objtool-args-$(CONFIG_MITIGATION_CALL_DEPTH_TRACKING)	+= --hacks=skylake
 objtool-args-$(CONFIG_X86_KERNEL_IBT)			+= --ibt
-objtool-args-$(CONFIG_FINEIBT)				+= --cfi
+objtool-args-$(CONFIG_CALL_PADDING)			+= --prefix=$(CONFIG_FUNCTION_PADDING_BYTES)
+ifdef CONFIG_CALL_PADDING
+objtool-args-$(CONFIG_CFI)				+= --cfi
+objtool-args-$(CONFIG_FINEIBT)				+= --fineibt
+endif
 objtool-args-$(CONFIG_FTRACE_MCOUNT_USE_OBJTOOL)	+= --mcount
 ifdef CONFIG_FTRACE_MCOUNT_USE_OBJTOOL
 objtool-args-$(CONFIG_HAVE_OBJTOOL_NOP_MCOUNT)		+= --mnop
@@ -200,7 +204,6 @@ objtool-args-$(CONFIG_STACK_VALIDATION)			+= --stackval
 objtool-args-$(CONFIG_HAVE_STATIC_CALL_INLINE)		+= --static-call
 objtool-args-$(CONFIG_HAVE_UACCESS_VALIDATION)		+= --uaccess
 objtool-args-$(or $(CONFIG_GCOV_KERNEL),$(CONFIG_KCOV))	+= --no-unreachable
-objtool-args-$(CONFIG_PREFIX_SYMBOLS)			+= --prefix=$(CONFIG_FUNCTION_PADDING_BYTES)
 objtool-args-$(CONFIG_OBJTOOL_WERROR)			+= --werror
 
 objtool-args = $(objtool-args-y)					\
diff --git a/scripts/Makefile.modfinal b/scripts/Makefile.modfinal
index adcbcde16a071..01a37ec872b90 100644
--- a/scripts/Makefile.modfinal
+++ b/scripts/Makefile.modfinal
@@ -46,17 +46,9 @@ quiet_cmd_btf_ko = BTF [M] $@
 		$(CONFIG_SHELL) $(srctree)/scripts/gen-btf.sh --btf_base $(objtree)/vmlinux $@; \
 	fi;
 
-# Same as newer-prereqs, but allows to exclude specified extra dependencies
-newer_prereqs_except = $(filter-out $(PHONY) $(1),$?)
-
-# Same as if_changed, but allows to exclude specified extra dependencies
-if_changed_except = $(if $(call newer_prereqs_except,$(2))$(cmd-check),      \
-	$(cmd);                                                              \
-	printf '%s\n' 'savedcmd_$@ := $(make-cmd)' > $(dot-target).cmd, @:)
-
 # Re-generate module BTFs if either module's .ko or vmlinux changed
 %.ko: %.o %.mod.o .module-common.o $(objtree)/scripts/module.lds $(and $(CONFIG_DEBUG_INFO_BTF_MODULES),$(KBUILD_BUILTIN),$(objtree)/vmlinux) FORCE
-	+$(call if_changed_except,ld_ko_o,$(objtree)/vmlinux)
+	+$(call if_changed,ld_ko_o)
 ifdef CONFIG_DEBUG_INFO_BTF_MODULES
 	+$(if $(newer-prereqs),$(call cmd,btf_ko))
 endif
diff --git a/scripts/bloat-o-meter b/scripts/bloat-o-meter
index 9b4fb996d95bd..5868a8b11b0f9 100755
--- a/scripts/bloat-o-meter
+++ b/scripts/bloat-o-meter
@@ -43,6 +43,7 @@ def getsizes(file, format):
                 if name.startswith("__se_compat_sys"): continue
                 if name.startswith("__addressable_"): continue
                 if name.startswith("__noinstr_text_start"): continue
+                if name.startswith("_sdata"): continue
                 if name == "linux_banner": continue
                 if name == "vermagic": continue
                 # statics and some other optimizations adds random .NUMBER
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 0492d6afc9a1f..3727156e4ccad 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -57,8 +57,12 @@ my %ignore_type = ();
 my @ignore = ();
 my $help = 0;
 my $configuration_file = ".checkpatch.conf";
+my $def_configuration_dirs_help = '.:$HOME:.scripts';
+(my $def_configuration_dirs = $def_configuration_dirs_help) =~ s/\$(\w+)/$ENV{$1}/g;
+my $env_config_dir = 'CHECKPATCH_CONFIG_DIR';
 my $max_line_length = 100;
 my $ignore_perl_version = 0;
+my $spdx_cxx_comments = 0;
 my $minimum_perl_version = 5.10.0;
 my $min_conf_desc_length = 4;
 my $spelling_file = "$D/spelling.txt";
@@ -135,6 +139,10 @@ Options:
                              file.  It's your fault if there's no backup or git
   --ignore-perl-version      override checking of perl version.  expect
                              runtime errors.
+  --spdx-cxx-comments        don't force C comments (/* */) for SPDX license
+                             (required by old toolchains), allow also C++
+                             comments (//).
+                             NOTE: it should *not* be used for Linux mainline.
   --codespell                Use the codespell dictionary for spelling/typos
                              (default:$codespellfile)
   --codespellfile            Use this codespell dictionary
@@ -146,6 +154,11 @@ Options:
   -h, --help, --version      display this help and exit
 
 When FILE is - read standard input.
+
+CONFIGURATION FILE
+Default configuration options can be stored in $configuration_file,
+search path: '$def_configuration_dirs_help' or in a directory specified by
+\$$env_config_dir environment variable (fallback to the default search path).
 EOM
 
 	exit($exitcode);
@@ -237,7 +250,7 @@ sub list_types {
 	exit($exitcode);
 }
 
-my $conf = which_conf($configuration_file);
+my $conf = which_conf($configuration_file, $env_config_dir, $def_configuration_dirs);
 if (-f $conf) {
 	my @conf_args;
 	open(my $conffile, '<', "$conf")
@@ -339,6 +352,7 @@ GetOptions(
 	'fix!'		=> \$fix,
 	'fix-inplace!'	=> \$fix_inplace,
 	'ignore-perl-version!' => \$ignore_perl_version,
+	'spdx-cxx-comments!' => \$spdx_cxx_comments,
 	'debug=s'	=> \%debug,
 	'test-only=s'	=> \$tst_only,
 	'codespell!'	=> \$codespell,
@@ -1531,9 +1545,15 @@ sub which {
 }
 
 sub which_conf {
-	my ($conf) = @_;
+	my ($conf, $env_key, $paths) = @_;
+	my $env_dir = $ENV{$env_key};
+
+	if (defined($env_dir) && $env_dir ne "") {
+		return "$env_dir/$conf" if (-e "$env_dir/$conf");
+		warn "$P: Can't find a readable $conf in '$env_dir', falling back to default search paths\n";
+	}
 
-	foreach my $path (split(/:/, ".:$ENV{HOME}:.scripts")) {
+	foreach my $path (split(/:/, $paths)) {
 		if (-e "$path/$conf") {
 			return "$path/$conf";
 		}
@@ -3801,26 +3821,33 @@ sub process {
 				$checklicenseline = 2;
 			} elsif ($rawline =~ /^\+/) {
 				my $comment = "";
-				if ($realfile =~ /\.(h|s|S)$/) {
-					$comment = '/*';
-				} elsif ($realfile =~ /\.(c|rs|dts|dtsi)$/) {
+				if ($realfile =~ /\.(c|rs|dts|dtsi)$/) {
 					$comment = '//';
 				} elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc|yaml)$/) {
 					$comment = '#';
 				} elsif ($realfile =~ /\.rst$/) {
 					$comment = '..';
 				}
+				my $pattern = qr{\Q$comment\E};
+				if ($realfile =~ /\.(h|s|S)$/) {
+					$comment = '/*';
+					$pattern = qr{/\*};
+					if ($spdx_cxx_comments) {
+						$comment = '// or /*';
+						$pattern = qr{//|/\*};
+					}
+				}
 
 # check SPDX comment style for .[chsS] files
 				if ($realfile =~ /\.[chsS]$/ &&
 				    $rawline =~ /SPDX-License-Identifier:/ &&
-				    $rawline !~ m@^\+\s*\Q$comment\E\s*@) {
+				    $rawline !~ m@^\+\s*$pattern\s*@) {
 					WARN("SPDX_LICENSE_TAG",
 					     "Improper SPDX comment style for '$realfile', please use '$comment' instead\n" . $herecurr);
 				}
 
 				if ($comment !~ /^$/ &&
-				    $rawline !~ m@^\+\Q$comment\E SPDX-License-Identifier: @) {
+				    $rawline !~ m@^\+$pattern SPDX-License-Identifier: @) {
 					WARN("SPDX_LICENSE_TAG",
 					     "Missing or malformed SPDX-License-Identifier tag in line $checklicenseline\n" . $herecurr);
 				} elsif ($rawline =~ /(SPDX-License-Identifier: .*)/) {
@@ -4156,7 +4183,7 @@ sub process {
 			$pl =~ s/\b(?:$Attribute|$Sparse)\b//g;
 			if (($pl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ ||
 			# function pointer declarations
-			     $pl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ ||
+			     $pl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident(?:\s*\[\s*(?:$Ident|$Constant)?\s*\])?\s*\)\s*[=,;:\[\(]/ ||
 			# foo bar; where foo is some local typedef or #define
 			     $pl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ ||
 			# known declaration macros
@@ -4170,7 +4197,7 @@ sub process {
 			# looks like a declaration
 			    !($sl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ ||
 			# function pointer declarations
-			      $sl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ ||
+			      $sl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident(?:\s*\[\s*(?:$Ident|$Constant)?\s*\])?\s*\)\s*[=,;:\[\(]/ ||
 			# foo bar; where foo is some local typedef or #define
 			      $sl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ ||
 			# known declaration macros
diff --git a/scripts/clang-tools/gen_compile_commands.py b/scripts/clang-tools/gen_compile_commands.py
index 96e6e46ad1a70..8d14b81efd737 100755
--- a/scripts/clang-tools/gen_compile_commands.py
+++ b/scripts/clang-tools/gen_compile_commands.py
@@ -201,6 +201,8 @@ def main():
         # Modules are listed in modules.order.
         if os.path.isdir(path):
             cmdfiles = cmdfiles_in_dir(path)
+        elif os.path.basename(path) == 'libgcc.a':
+            cmdfiles = []
         elif path.endswith('.a'):
             cmdfiles = cmdfiles_for_a(path, ar)
         elif path.endswith('modules.order'):
diff --git a/scripts/gdb/linux/mm.py b/scripts/gdb/linux/mm.py
index d78908f6664d5..dffadccbb01d2 100644
--- a/scripts/gdb/linux/mm.py
+++ b/scripts/gdb/linux/mm.py
@@ -40,11 +40,11 @@ class x86_page_ops():
 
         self.PAGE_OFFSET = int(gdb.parse_and_eval("page_offset_base"))
         self.VMEMMAP_START = int(gdb.parse_and_eval("vmemmap_base"))
-        self.PHYS_BASE = int(gdb.parse_and_eval("phys_base"))
+        self.PHYS_BASE = int(gdb.parse_and_eval("(unsigned long) phys_base"))
         self.START_KERNEL_map = 0xffffffff80000000
 
-        self.KERNEL_START = gdb.parse_and_eval("_text")
-        self.KERNEL_END = gdb.parse_and_eval("_end")
+        self.KERNEL_START = gdb.parse_and_eval("(unsigned long) &_text")
+        self.KERNEL_END = gdb.parse_and_eval("(unsigned long) &_end")
 
         self.VMALLOC_START = int(gdb.parse_and_eval("vmalloc_base"))
         if self.VMALLOC_START == 0xffffc90000000000:
diff --git a/scripts/gdb/linux/slab.py b/scripts/gdb/linux/slab.py
index 0e2d93867fe22..ddde25aeca8de 100644
--- a/scripts/gdb/linux/slab.py
+++ b/scripts/gdb/linux/slab.py
@@ -196,7 +196,7 @@ def slabtrace(alloc, cache_name):
 
     if target_cache['flags'] & SLAB_STORE_USER:
         for i in range(0, nr_node_ids):
-            cache_node = target_cache['node'][i]
+            cache_node = target_cache['per_node']['node'][i]
             if cache_node['nr_slabs']['counter'] == 0:
                 continue
             process_slab(loc_track, cache_node['partial'], alloc, target_cache)
@@ -300,7 +300,7 @@ def slabinfo():
         nr_free = 0
         nr_slabs = 0
         for i in range(0, nr_node_ids):
-            cache_node = cache['node'][i]
+            cache_node = cache['per_node']['node'][i]
             try:
                 nr_slabs += cache_node['nr_slabs']['counter']
                 nr_objs = int(cache_node['total_objects']['counter'])
diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl
index f0ca0db6ddc27..16b80a700d4ac 100755
--- a/scripts/get_maintainer.pl
+++ b/scripts/get_maintainer.pl
@@ -21,6 +21,7 @@ use Cwd;
 use File::Find;
 use File::Spec::Functions;
 use open qw(:std :encoding(UTF-8));
+use JSON::PP;
 
 my $cur_path = fastgetcwd() . '/';
 my $lk_path = "./";
@@ -68,6 +69,7 @@ my $pattern_depth = 0;
 my $self_test = undef;
 my $version = 0;
 my $help = 0;
+my $json = 0;
 my $find_maintainer_files = 0;
 my $maintainer_path;
 my $vcs_used = 0;
@@ -285,6 +287,7 @@ if (!GetOptions(
 		'find-maintainer-files' => \$find_maintainer_files,
 		'mpath|maintainer-path=s' => \$maintainer_path,
 		'self-test:s' => \$self_test,
+		'json!' => \$json,
 		'v|version' => \$version,
 		'h|help|usage' => \$help,
 		)) {
@@ -650,39 +653,48 @@ my %deduplicate_name_hash = ();
 my %deduplicate_address_hash = ();
 
 my @maintainers = get_maintainers();
-if (@maintainers) {
-    @maintainers = merge_email(@maintainers);
-    output(@maintainers);
-}
-
-if ($scm) {
-    @scm = uniq(@scm);
-    output(@scm);
-}
-
-if ($output_substatus) {
-    @substatus = uniq(@substatus);
-    output(@substatus);
-}
-
-if ($status) {
-    @status = uniq(@status);
-    output(@status);
-}
 
-if ($subsystem) {
-    @subsystem = uniq(@subsystem);
-    output(@subsystem);
-}
+@maintainers = merge_email(@maintainers) if (@maintainers);
+@scm = uniq(@scm) if ($scm);
+@substatus = uniq(@substatus) if ($output_substatus);
+@status = uniq(@status) if ($status);
+@subsystem = uniq(@subsystem) if ($subsystem);
+@web = uniq(@web) if ($web);
+@bug = uniq(@bug) if ($bug);
+
+if ($json) {
+    my @json_maintainers;
+    for my $m (@maintainers) {
+	my ($addr, $role);
+	if ($output_roles && $m =~ /^(.*?)\s+\((.+)\)\s*$/) {
+	    $addr = $1;
+	    $role = $2;
+	} else {
+	    $addr = $m;
+	}
+	my ($name, $email_addr) = parse_email($addr);
+	my %entry = (name => $name, email => $email_addr);
+	$entry{role} = $role if (defined $role && $role ne '');
+	push(@json_maintainers, \%entry);
+    }
 
-if ($web) {
-    @web = uniq(@web);
-    output(@web);
-}
+    my %result = (maintainers => \@json_maintainers);
+    $result{scm} = \@scm if ($scm);
+    $result{status} = \@status if ($status);
+    $result{subsystem} = \@subsystem if ($subsystem);
+    $result{web} = \@web if ($web);
+    $result{bug} = \@bug if ($bug);
 
-if ($bug) {
-    @bug = uniq(@bug);
-    output(@bug);
+    my $json_encoder = JSON::PP->new->canonical->utf8;
+    print($json_encoder->encode(\%result) . "\n");
+} else {
+    output(@maintainers) if (@maintainers);
+    output(@scm) if ($scm);
+    output(@substatus) if ($output_substatus);
+    output(@status) if ($status);
+    output(@subsystem) if ($subsystem);
+    output(@web) if ($web);
+    output(@bug) if ($bug);
 }
 
 exit($exit);
@@ -1104,6 +1116,7 @@ Output type options:
   --separator [, ] => separator for multiple entries on 1 line
     using --separator also sets --nomultiline if --separator is not [, ]
   --multiline => print 1 entry per line
+  --json => output results as JSON
 
 Other options:
   --pattern-depth => Number of pattern directory traversals (default: 0 (all))
diff --git a/scripts/hornet/.gitignore b/scripts/hornet/.gitignore
new file mode 100644
index 0000000000000..ce40d0acdb1bf
--- /dev/null
+++ b/scripts/hornet/.gitignore
@@ -0,0 +1 @@
+gen_sig
diff --git a/scripts/hornet/Makefile b/scripts/hornet/Makefile
new file mode 100644
index 0000000000000..3ee41e5e9a9ff
--- /dev/null
+++ b/scripts/hornet/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+hostprogs-always-y	:= gen_sig
+
+HOSTCFLAGS_gen_sig.o = $(shell $(HOSTPKG_CONFIG) --cflags libcrypto 2> /dev/null)
+HOSTLDLIBS_gen_sig = $(shell $(HOSTPKG_CONFIG) --libs libcrypto 2> /dev/null || echo -lcrypto)
diff --git a/scripts/hornet/extract-insn.sh b/scripts/hornet/extract-insn.sh
new file mode 100755
index 0000000000000..52338f057ff6b
--- /dev/null
+++ b/scripts/hornet/extract-insn.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2025 Microsoft Corporation
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 2 of the GNU General Public
+# License as published by the Free Software Foundation.
+
+function usage() {
+    echo "Sample script for extracting instructions"
+    echo "autogenerated eBPF lskel headers"
+    echo ""
+    echo "USAGE: header_file"
+    exit
+}
+
+ARGC=$#
+
+EXPECTED_ARGS=1
+
+if [ $ARGC -ne $EXPECTED_ARGS ] ; then
+    usage
+else
+    printf $(gcc -E $1 | grep "opts_insn" | \
+		 awk -F"=" '{print $2}' | sed 's/;\+$//' | sed 's/\"//g')
+fi
diff --git a/scripts/hornet/extract-map.sh b/scripts/hornet/extract-map.sh
new file mode 100755
index 0000000000000..c309f505c6238
--- /dev/null
+++ b/scripts/hornet/extract-map.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2025 Microsoft Corporation
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 2 of the GNU General Public
+# License as published by the Free Software Foundation.
+
+function usage() {
+    echo "Sample script for extracting instructions"
+    echo "autogenerated eBPF lskel headers"
+    echo ""
+    echo "USAGE: header_file"
+    exit
+}
+
+ARGC=$#
+
+EXPECTED_ARGS=1
+
+if [ $ARGC -ne $EXPECTED_ARGS ] ; then
+    usage
+else
+    printf $(gcc -E $1 | grep "opts_data" | \
+		 awk -F"=" '{print $2}' | sed 's/;\+$//' | sed 's/\"//g')
+fi
diff --git a/scripts/hornet/extract-skel.sh b/scripts/hornet/extract-skel.sh
new file mode 100755
index 0000000000000..6550a86b89917
--- /dev/null
+++ b/scripts/hornet/extract-skel.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2025 Microsoft Corporation
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 2 of the GNU General Public
+# License as published by the Free Software Foundation.
+
+function usage() {
+    echo "Sample script for extracting instructions and map data out of"
+    echo "autogenerated eBPF lskel headers"
+    echo ""
+    echo "USAGE: header_file field"
+    exit
+}
+
+ARGC=$#
+
+EXPECTED_ARGS=2
+
+if [ $ARGC -ne $EXPECTED_ARGS ] ; then
+    usage
+else
+    printf $(gcc -E $1 | grep "static const char opts_$2" | \
+		 awk -F"=" '{print $2}' | sed 's/;\+$//' | sed 's/\"//g')
+fi
diff --git a/scripts/hornet/gen_sig.c b/scripts/hornet/gen_sig.c
new file mode 100644
index 0000000000000..8dd9ed66346a2
--- /dev/null
+++ b/scripts/hornet/gen_sig.c
@@ -0,0 +1,402 @@
+// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
+/*
+ *
+ * Generate a signature for an eBPF program along with appending
+ * map hashes as signed attributes
+ *
+ * Copyright © 2025      Microsoft Corporation.
+ *
+ * Authors: Blaise Boscaccy <bboscaccy@linux.microsoft.com>
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2.1
+ * of the licence, or (at your option) any later version.
+ */
+
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <err.h>
+#include <getopt.h>
+
+#include <openssl/cms.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/pkcs7.h>
+#include <openssl/x509.h>
+#include <openssl/pem.h>
+#include <openssl/objects.h>
+#include <openssl/asn1.h>
+#include <openssl/asn1t.h>
+#include <openssl/opensslv.h>
+#include <openssl/bio.h>
+#include <openssl/stack.h>
+
+#if OPENSSL_VERSION_MAJOR >= 3
+# define USE_PKCS11_PROVIDER
+# include <openssl/provider.h>
+# include <openssl/store.h>
+#else
+# if !defined(OPENSSL_NO_ENGINE) && !defined(OPENSSL_NO_DEPRECATED_3_0)
+#  define USE_PKCS11_ENGINE
+#  include <openssl/engine.h>
+# endif
+#endif
+#include "../ssl-common.h"
+
+#define SHA256_LEN 32
+#define BUF_SIZE   (1 << 15) // 32 KiB
+#define MAX_HASHES 64
+
+struct hash_spec {
+	char *file;
+	int index;
+};
+
+typedef struct {
+	ASN1_INTEGER *index;
+	ASN1_OCTET_STRING *hash;
+
+} HORNET_MAP;
+
+DECLARE_ASN1_FUNCTIONS(HORNET_MAP)
+ASN1_SEQUENCE(HORNET_MAP) = {
+	ASN1_SIMPLE(HORNET_MAP, index, ASN1_INTEGER),
+	ASN1_SIMPLE(HORNET_MAP, hash, ASN1_OCTET_STRING)
+} ASN1_SEQUENCE_END(HORNET_MAP);
+
+IMPLEMENT_ASN1_FUNCTIONS(HORNET_MAP)
+
+DEFINE_STACK_OF(HORNET_MAP)
+
+typedef struct {
+	STACK_OF(HORNET_MAP) * maps;
+} MAP_SET;
+
+DECLARE_ASN1_FUNCTIONS(MAP_SET)
+ASN1_SEQUENCE(MAP_SET) = {
+	ASN1_SET_OF(MAP_SET, maps, HORNET_MAP)
+} ASN1_SEQUENCE_END(MAP_SET);
+
+IMPLEMENT_ASN1_FUNCTIONS(MAP_SET)
+
+#define DIE(...) do { fprintf(stderr, __VA_ARGS__); fputc('\n', stderr); \
+		exit(EXIT_FAILURE); } while (0)
+
+static BIO *bio_open_wr(const char *path)
+{
+	BIO *b = BIO_new_file(path, "wb");
+
+	if (!b) {
+		perror(path);
+		ERR_print_errors_fp(stderr);
+		exit(EXIT_FAILURE);
+	}
+	return b;
+}
+
+static void usage(const char *prog)
+{
+	fprintf(stderr,
+		"Usage:\n"
+		"  %s --data content.bin --cert signer.crt --key signer.key [-pass pass]\n"
+		"     --out newsig.p7b \n"
+		"     --add FILE:index [--add FILE:index ...]\n",
+		prog);
+}
+
+static const char *key_pass;
+
+static int pem_pw_cb(char *buf, int len, int w, void *v)
+{
+	int pwlen;
+
+	if (!key_pass)
+		return -1;
+
+	pwlen = strlen(key_pass);
+	if (pwlen >= len)
+		return -1;
+
+	strcpy(buf, key_pass);
+
+	key_pass = NULL;
+
+	return pwlen;
+}
+
+static EVP_PKEY *read_private_key(const char *private_key_name)
+{
+	EVP_PKEY *private_key;
+	BIO *b;
+
+	b = BIO_new_file(private_key_name, "rb");
+	ERR(!b, "%s", private_key_name);
+	private_key = PEM_read_bio_PrivateKey(b, NULL, pem_pw_cb,
+					      NULL);
+	ERR(!private_key, "%s", private_key_name);
+	BIO_free(b);
+
+	return private_key;
+}
+
+static X509 *read_x509(const char *x509_name)
+{
+	unsigned char buf[2];
+	X509 *x509;
+	BIO *b;
+	int n;
+
+	b = BIO_new_file(x509_name, "rb");
+	ERR(!b, "%s", x509_name);
+
+	/* Look at the first two bytes of the file to determine the encoding */
+	n = BIO_read(b, buf, 2);
+	if (n != 2) {
+		if (BIO_should_retry(b)) {
+			fprintf(stderr, "%s: Read wanted retry\n", x509_name);
+			exit(1);
+		}
+		if (n >= 0) {
+			fprintf(stderr, "%s: Short read\n", x509_name);
+			exit(1);
+		}
+		ERR(1, "%s", x509_name);
+	}
+
+	ERR(BIO_reset(b) != 0, "%s", x509_name);
+
+	if (buf[0] == 0x30 && buf[1] >= 0x81 && buf[1] <= 0x84)
+		/* Assume raw DER encoded X.509 */
+		x509 = d2i_X509_bio(b, NULL);
+	else
+		/* Assume PEM encoded X.509 */
+		x509 = PEM_read_bio_X509(b, NULL, NULL, NULL);
+
+	BIO_free(b);
+	ERR(!x509, "%s", x509_name);
+
+	return x509;
+}
+
+static int sha256(const char *path, unsigned char out[SHA256_LEN], unsigned int *out_len)
+{
+	FILE *f;
+	int rc;
+	EVP_MD_CTX *ctx;
+	unsigned char buf[BUF_SIZE];
+	size_t n;
+	unsigned int mdlen = 0;
+
+	if (!path || !out)
+		return -1;
+
+	f = fopen(path, "rb");
+	if (!f) {
+		perror("fopen");
+		return -2;
+	}
+
+	ERR_load_crypto_strings();
+
+	rc = -3;
+	ctx = EVP_MD_CTX_new();
+	if (!ctx) {
+		rc = -4;
+		goto done;
+	}
+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+	if (EVP_DigestInit_ex2(ctx, EVP_sha256(), NULL) != 1) {
+		rc = -5;
+		goto done;
+	}
+#else
+	if (EVP_DigestInit_ex(ctx, EVP_sha256(), NULL) != 1) {
+		rc = -5;
+		goto done;
+	}
+#endif
+	while ((n = fread(buf, 1, sizeof(buf), f)) > 0) {
+		if (EVP_DigestUpdate(ctx, buf, n) != 1) {
+			rc = -6;
+			goto done;
+		}
+	}
+	if (ferror(f)) {
+		rc = -7;
+		goto done;
+	}
+
+	if (EVP_DigestFinal_ex(ctx, out, &mdlen) != 1) {
+		rc = -8;
+		goto done;
+	}
+	if (mdlen != SHA256_LEN) {
+		rc = -9;
+		goto done;
+	}
+
+	if (out_len)
+		*out_len = mdlen;
+	rc = 0;
+
+done:
+	EVP_MD_CTX_free(ctx);
+	fclose(f);
+	ERR_free_strings();
+	return rc;
+}
+
+static void add_hash(MAP_SET *set, unsigned char *buffer, int buffer_len, int index)
+{
+	HORNET_MAP *map = NULL;
+
+	map = HORNET_MAP_new();
+	ASN1_INTEGER_set(map->index, index);
+	ASN1_OCTET_STRING_set(map->hash, buffer, buffer_len);
+	sk_HORNET_MAP_push(set->maps, map);
+}
+
+int main(int argc, char **argv)
+{
+	const char *cert_path = NULL;
+	const char *key_path = NULL;
+	const char *data_path = NULL;
+	const char *out_path = NULL;
+
+	X509 *signer;
+	EVP_PKEY *pkey;
+	BIO *data_in;
+	CMS_ContentInfo *cms_out;
+	struct hash_spec hashes[MAX_HASHES];
+	int hash_count = 0;
+	int flags;
+	CMS_SignerInfo *si;
+	MAP_SET *set;
+	unsigned char hash_buffer[SHA256_LEN];
+	unsigned int hash_len;
+	ASN1_OBJECT *oid;
+	unsigned char *der = NULL;
+	int der_len;
+	int err;
+	BIO *b_out;
+	int i;
+	int opt;
+
+	const char *short_opts = "C:K:P:O:A:Sh";
+
+	static const struct option long_opts[] = {
+		{"cert", required_argument, 0, 'C'},
+		{"key",  required_argument, 0, 'K'},
+		{"pass",  required_argument, 0, 'P'},
+		{"out",  required_argument, 0, 'O'},
+		{"data",  required_argument, 0, 'D'},
+		{"add",  required_argument, 0, 'A'},
+		{"help",    no_argument,       0, 'h'},
+		{0, 0, 0, 0}
+	};
+
+	while ((opt = getopt_long_only(argc, argv, short_opts, long_opts, NULL)) != -1) {
+		switch (opt) {
+		case 'C':
+			cert_path = optarg;
+			break;
+		case 'K':
+			key_path = optarg;
+			break;
+		case 'P':
+			key_pass = optarg;
+			break;
+		case 'O':
+			out_path = optarg;
+			break;
+		case 'D':
+			data_path = optarg;
+			break;
+		case 'A':
+			if (strchr(optarg, ':')) {
+				hashes[hash_count].file = strsep(&optarg, ":");
+				hashes[hash_count].index = atoi(optarg);
+				if (++hash_count >= MAX_HASHES) {
+					usage(argv[0]);
+					return EXIT_FAILURE;
+				}
+			} else {
+				usage(argv[0]);
+				return EXIT_FAILURE;
+			}
+			break;
+		default:
+			usage(argv[0]);
+			return EXIT_FAILURE;
+		}
+	}
+
+	if (!cert_path || !key_path || !out_path || !data_path) {
+		usage(argv[0]);
+		return EXIT_FAILURE;
+	}
+
+	OpenSSL_add_all_algorithms();
+	ERR_load_crypto_strings();
+
+	signer = read_x509(cert_path);
+	ERR(!signer, "Load cert failed");
+
+	pkey = read_private_key(key_path);
+	ERR(!pkey, "Load key failed");
+
+	data_in = BIO_new_file(data_path, "rb");
+	ERR(!data_in, "Load data failed");
+
+	cms_out = CMS_sign(NULL, NULL, NULL, NULL,
+			   CMS_NOCERTS | CMS_PARTIAL | CMS_BINARY | CMS_DETACHED);
+	ERR(!cms_out, "create cms failed");
+
+	flags = CMS_NOCERTS | CMS_PARTIAL | CMS_BINARY | CMS_NOSMIMECAP | CMS_DETACHED;
+
+	si = CMS_add1_signer(cms_out, signer, pkey, EVP_sha256(), flags);
+	ERR(!si, "add signer failed");
+
+	set = MAP_SET_new();
+	set->maps = sk_HORNET_MAP_new_null();
+
+	for (i = 0; i < hash_count; i++) {
+		if (sha256(hashes[i].file, hash_buffer, &hash_len) != 0) {
+			DIE("failed to hash input");
+		}
+		add_hash(set, hash_buffer, hash_len, hashes[i].index);
+	}
+
+	oid = OBJ_txt2obj("2.25.316487325684022475439036912669789383960", 1);
+	if (!oid) {
+		ERR_print_errors_fp(stderr);
+		DIE("create oid failed");
+	}
+
+	der_len = ASN1_item_i2d((ASN1_VALUE *)set, &der, ASN1_ITEM_rptr(MAP_SET));
+	CMS_signed_add1_attr_by_OBJ(si, oid, V_ASN1_SEQUENCE, der, der_len);
+
+	err = CMS_final(cms_out, data_in, NULL, CMS_NOCERTS | CMS_BINARY);
+	ERR(!err, "cms final failed");
+
+	OPENSSL_free(der);
+	MAP_SET_free(set);
+
+	b_out = bio_open_wr(out_path);
+	ERR(!b_out, "opening output path failed");
+
+	i2d_CMS_bio_stream(b_out, cms_out, NULL, 0);
+
+	BIO_free(data_in);
+	BIO_free(b_out);
+	EVP_cleanup();
+	ERR_free_strings();
+	return 0;
+}
diff --git a/scripts/hornet/write-sig.sh b/scripts/hornet/write-sig.sh
new file mode 100755
index 0000000000000..7eaabe3bab9aa
--- /dev/null
+++ b/scripts/hornet/write-sig.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Copyright (c) 2025 Microsoft Corporation
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of version 2 of the GNU General Public
+# License as published by the Free Software Foundation.
+
+function usage() {
+    echo "Sample for rewriting an autogenerated eBPF lskel headers"
+    echo "with a new signature"
+    echo ""
+    echo "USAGE: header_file sig"
+    exit
+}
+
+ARGC=$#
+
+EXPECTED_ARGS=2
+
+if [ $ARGC -ne $EXPECTED_ARGS ] ; then
+    usage
+else
+    SIG=$(xxd -p $2 | tr -d '\n' | sed 's/\(..\)/\\\\x\1/g')
+    sed '/const char opts_sig/,/;/c\\tstatic const char opts_sig[] __attribute__((__aligned__(8))) = "\\\n'"$(printf '%s\n' "$SIG")"'\";' $1
+fi
diff --git a/scripts/kconfig/conf.c b/scripts/kconfig/conf.c
index a7b44cd8ae140..c368bec5ab601 100644
--- a/scripts/kconfig/conf.c
+++ b/scripts/kconfig/conf.c
@@ -297,9 +297,7 @@ static int conf_askvalue(struct symbol *sym, const char *def)
 	line[1] = 0;
 
 	if (!sym_is_changeable(sym)) {
-		printf("%s\n", def);
-		line[0] = '\n';
-		line[1] = 0;
+		printf("%s\n", def ?: "");
 		return 0;
 	}
 
@@ -307,7 +305,7 @@ static int conf_askvalue(struct symbol *sym, const char *def)
 	case oldconfig:
 	case syncconfig:
 		if (sym_has_value(sym)) {
-			printf("%s\n", def);
+			printf("%s\n", def ?: "");
 			return 0;
 		}
 		/* fall through */
diff --git a/scripts/livepatch/klp-build b/scripts/livepatch/klp-build
index 0ad7e66313142..c4a7acf8edc3f 100755
--- a/scripts/livepatch/klp-build
+++ b/scripts/livepatch/klp-build
@@ -3,7 +3,7 @@
 #
 # Build a livepatch module
 
-# shellcheck disable=SC1090,SC2155
+# shellcheck disable=SC1090,SC2155,SC2164
 
 if (( BASH_VERSINFO[0]  < 4 || \
      (BASH_VERSINFO[0] == 4 && BASH_VERSINFO[1] < 4) )); then
@@ -11,21 +11,19 @@ if (( BASH_VERSINFO[0]  < 4 || \
 	exit 1
 fi
 
-set -o errexit
 set -o errtrace
 set -o pipefail
 set -o nounset
 
 # Allow doing 'cmd | mapfile -t array' instead of 'mapfile -t array < <(cmd)'.
-# This helps keep execution in pipes so pipefail+errexit can catch errors.
+# This helps keep execution in pipes so pipefail+ERR trap can catch errors.
 shopt -s lastpipe
 
-unset DEBUG_CLONE DIFF_CHECKSUM SKIP_CLEANUP XTRACE
+unset DEBUG_CLONE DIFF_CHECKSUM SKIP_CLEANUP VERBOSE XTRACE
 
 REPLACE=1
 SHORT_CIRCUIT=0
 JOBS="$(getconf _NPROCESSORS_ONLN)"
-VERBOSE="-s"
 shopt -o xtrace | grep -q 'on' && XTRACE=1
 
 # Avoid removing the previous $TMP_DIR until args have been fully processed.
@@ -35,16 +33,16 @@ SCRIPT="$(basename "$0")"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 FIX_PATCH_LINES="$SCRIPT_DIR/fix-patch-lines"
 
-SRC="$(pwd)"
-OBJ="$(pwd)"
+OBJTOOL="$PWD/tools/objtool/objtool"
+CONFIG="$PWD/.config"
+TMP_DIR="$PWD/klp-tmp"
 
-CONFIG="$OBJ/.config"
-TMP_DIR="$OBJ/klp-tmp"
-
-ORIG_DIR="$TMP_DIR/orig"
-PATCHED_DIR="$TMP_DIR/patched"
-DIFF_DIR="$TMP_DIR/diff"
-KMOD_DIR="$TMP_DIR/kmod"
+ORIG_DIR="$TMP_DIR/1-orig"
+PATCHED_DIR="$TMP_DIR/2-patched"
+ORIG_CSUM_DIR="$TMP_DIR/3-checksum-orig"
+PATCHED_CSUM_DIR="$TMP_DIR/3-checksum-patched"
+DIFF_DIR="$TMP_DIR/4-diff"
+KMOD_DIR="$TMP_DIR/5-kmod"
 
 STASH_DIR="$TMP_DIR/stash"
 TIMESTAMP="$TMP_DIR/timestamp"
@@ -90,7 +88,7 @@ declare -a STASHED_FILES
 
 stash_file() {
 	local file="$1"
-	local rel_file="${file#"$SRC"/}"
+	local rel_file="${file#"$PWD"/}"
 
 	[[ ! -e "$file" ]] && die "no file to stash: $file"
 
@@ -104,7 +102,7 @@ restore_files() {
 	local file
 
 	for file in "${STASHED_FILES[@]}"; do
-		mv -f "$STASH_DIR/$file" "$SRC/$file" || warn "can't restore file: $file"
+		mv -f "$STASH_DIR/$file" "$PWD/$file" || warn "can't restore file: $file"
 	done
 
 	STASHED_FILES=()
@@ -140,10 +138,11 @@ Options:
 Advanced Options:
    -d, --debug			Show symbol/reloc cloning decisions
    -S, --short-circuit=STEP	Start at build step (requires prior --keep-tmp)
-				   1|orig	Build original kernel (default)
-				   2|patched	Build patched kernel
-				   3|diff	Diff objects
-				   4|kmod	Build patch module
+				   1|orig		Build original kernel (default)
+				   2|patched		Build patched kernel
+				   3|checksum		Generate checksums
+				   4|diff		Diff objects
+				   5|kmod		Build patch module
    -T, --keep-tmp		Preserve tmp dir on exit
 
 EOF
@@ -158,6 +157,7 @@ process_args() {
 	local short
 	local long
 	local args
+	local patch
 
 	short="hfj:o:vdS:T"
 	long="help,show-first-changed,jobs:,output:,no-replace,verbose,debug,short-circuit:,keep-tmp"
@@ -194,7 +194,7 @@ process_args() {
 				shift
 				;;
 			-v | --verbose)
-				VERBOSE="V=1"
+				VERBOSE=1
 				shift
 				;;
 			-d | --debug)
@@ -206,10 +206,11 @@ process_args() {
 				[[ ! -d "$TMP_DIR" ]] && die "--short-circuit requires preserved klp-tmp dir"
 				keep_tmp=1
 				case "$2" in
-					1 | orig)	SHORT_CIRCUIT=1; ;;
-					2 | patched)	SHORT_CIRCUIT=2; ;;
-					3 | diff)	SHORT_CIRCUIT=3; ;;
-					4 | mod)	SHORT_CIRCUIT=4; ;;
+					1 | orig)		SHORT_CIRCUIT=1; ;;
+					2 | patched)		SHORT_CIRCUIT=2; ;;
+					3 | checksum)		SHORT_CIRCUIT=3; ;;
+					4 | diff)		SHORT_CIRCUIT=4; ;;
+					5 | kmod)		SHORT_CIRCUIT=5; ;;
 					*)		die "invalid short-circuit step '$2'" ;;
 				esac
 				shift 2
@@ -236,6 +237,10 @@ process_args() {
 
 	KEEP_TMP="$keep_tmp"
 	PATCHES=("$@")
+
+	for patch in "${PATCHES[@]}"; do
+		[[ -f "$patch" ]] || die "$patch doesn't exist"
+	done
 }
 
 # temporarily disable xtrace for especially verbose code
@@ -270,6 +275,9 @@ validate_config() {
 		[[ "$CONFIG_AS_VERSION" -lt 200000 ]] &&	\
 		die "Clang assembler version < 20 not supported"
 
+	[[ -x "$OBJTOOL" ]] && "$OBJTOOL" klp 2>&1 | command grep -q "not implemented" && \
+		die "objtool not built with KLP support; install xxhash-devel/libxxhash-dev (version >= 0.8) and recompile"
+
 	return 0
 }
 
@@ -301,12 +309,17 @@ set_module_name() {
 # Hardcode the value printed by the localversion script to prevent patch
 # application from appending it with '+' due to a dirty working tree.
 set_kernelversion() {
-	local file="$SRC/scripts/setlocalversion"
+	local file="$PWD/scripts/setlocalversion"
 	local kernelrelease
 
 	stash_file "$file"
 
-	kernelrelease="$(cd "$SRC" && make syncconfig &>/dev/null && make -s kernelrelease)"
+	if [[ -n "$(make -s listnewconfig 2>/dev/null)" ]]; then
+		die ".config mismatch, check your .config or run 'make olddefconfig'"
+	fi
+	make syncconfig &>/dev/null || die "make syncconfig failed"
+
+	kernelrelease="$(make -s kernelrelease)"
 	[[ -z "$kernelrelease" ]] && die "failed to get kernel version"
 
 	sed -i "2i echo $kernelrelease; exit 0" scripts/setlocalversion
@@ -349,7 +362,7 @@ check_unsupported_patches() {
 
 		for file in "${files[@]}"; do
 			case "$file" in
-				lib/*|*.S)
+				lib/*|*/vdso/*|*/realmode/rm/*|*.S)
 					die "${patch}: unsupported patch to $file"
 					;;
 			esac
@@ -367,24 +380,24 @@ apply_patch() {
 
 	[[ ! -f "$patch" ]] && die "$patch doesn't exist"
 	status=0
-	output=$(patch -d "$SRC" -p1 --dry-run --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" < "$patch" 2>&1) || status=$?
+	output=$(patch -p1 --dry-run --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" < "$patch" 2>&1) || status=$?
 	if [[ "$status" -ne 0 ]]; then
 		echo "$output" >&2
 		die "$patch did not apply"
 	elif [[ "$output" =~ $drift_regex ]]; then
-		echo "$output" >&2
+		[[ -v VERBOSE ]] && echo "$output" >&2
 		warn "${patch} applied with fuzz"
 	fi
 
-	patch -d "$SRC" -p1 --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" --silent < "$patch"
 	APPLIED_PATCHES+=("$patch")
+	patch -p1 --no-backup-if-mismatch -r /dev/null "${extra_args[@]}" --silent < "$patch"
 }
 
 revert_patch() {
 	local patch="$1"
 	local tmp=()
 
-	patch -d "$SRC" -p1 -R --silent --no-backup-if-mismatch -r /dev/null < "$patch"
+	patch -p1 -R --force --no-backup-if-mismatch -r /dev/null &> /dev/null < "$patch" || true
 
 	for p in "${APPLIED_PATCHES[@]}"; do
 		[[ "$p" == "$patch" ]] && continue
@@ -422,8 +435,21 @@ validate_patches() {
 do_init() {
 	# We're not yet smart enough to handle anything other than in-tree
 	# builds in pwd.
-	[[ ! "$SRC" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory"
-	[[ ! "$OBJ" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory"
+	[[ ! "$PWD" -ef "$SCRIPT_DIR/../.." ]] && die "please run from the kernel root directory"
+
+	if (( SHORT_CIRCUIT >= 2 )); then
+		[[ -f "$ORIG_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $ORIG_DIR"
+	fi
+	if (( SHORT_CIRCUIT >= 3 )); then
+		[[ -f "$PATCHED_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $PATCHED_DIR"
+	fi
+	if (( SHORT_CIRCUIT >= 4 )); then
+		[[ -f "$ORIG_CSUM_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $ORIG_CSUM_DIR"
+		[[ -f "$PATCHED_CSUM_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $PATCHED_CSUM_DIR"
+	fi
+	if (( SHORT_CIRCUIT >= 5 )); then
+		[[ -f "$DIFF_DIR/.complete" ]] || die "-S $SHORT_CIRCUIT requires completed $DIFF_DIR"
+	fi
 
 	(( SHORT_CIRCUIT <= 1 )) && rm -rf "$TMP_DIR"
 	mkdir -p "$TMP_DIR"
@@ -454,11 +480,11 @@ refresh_patch() {
 	get_patch_output_files "$patch" | mapfile -t output_files
 
 	# Copy orig source files to 'a'
-	( cd "$SRC" && echo "${input_files[@]}" | xargs cp --parents --target-directory="$tmpdir/a" )
+	echo "${input_files[@]}" | xargs cp --parents --target-directory="$tmpdir/a"
 
 	# Copy patched source files to 'b'
 	apply_patch "$patch" "--silent"
-	( cd "$SRC" && echo "${output_files[@]}" | xargs cp --parents --target-directory="$tmpdir/b" )
+	echo "${output_files[@]}" | xargs cp --parents --target-directory="$tmpdir/b"
 	revert_patch "$patch"
 
 	# Diff 'a' and 'b' to make a clean patch
@@ -502,20 +528,14 @@ clean_kernel() {
 	cmd+=("-j$JOBS")
 	cmd+=("clean")
 
-	(
-		cd "$SRC"
-		"${cmd[@]}"
-	)
+	"${cmd[@]}"
 }
 
 build_kernel() {
 	local build="$1"
 	local log="$TMP_DIR/build.log"
-	local objtool_args=()
 	local cmd=()
 
-	objtool_args=("--checksum")
-
 	cmd=("make")
 
 	# When a patch to a kernel module references a newly created unexported
@@ -535,19 +555,20 @@ build_kernel() {
 	#
 	cmd+=("KBUILD_MODPOST_WARN=1")
 
-	cmd+=("$VERBOSE")
+	if [[ -v VERBOSE ]]; then
+		cmd+=("V=1")
+	else
+		cmd+=("-s")
+	fi
 	cmd+=("-j$JOBS")
 	cmd+=("KCFLAGS=-ffunction-sections -fdata-sections")
-	cmd+=("OBJTOOL_ARGS=${objtool_args[*]}")
 	cmd+=("vmlinux")
 	cmd+=("modules")
 
-	(
-		cd "$SRC"
-		"${cmd[@]}"							\
-			1> >(tee -a "$log")					\
-			2> >(tee -a "$log" | grep0 -v "modpost.*undefined!" >&2)
-	) || die "$build kernel build failed"
+	"${cmd[@]}"							\
+		1> >(tee -a "$log")					\
+		2> >(tee -a "$log" | grep0 -v "modpost.*undefined!" >&2) \
+		|| die "$build kernel build failed"
 }
 
 find_objects() {
@@ -555,9 +576,9 @@ find_objects() {
 
 	# Find root-level vmlinux.o and non-root-level .ko files,
 	# excluding klp-tmp/ and .git/
-	find "$OBJ" \( -path "$TMP_DIR" -o -path "$OBJ/.git" -o	-regex "$OBJ/[^/][^/]*\.ko" \) -prune -o \
+	find "$PWD" \( -path "$TMP_DIR" -o -path "$PWD/.git" -o -regex "$PWD/[^/][^/]*\.ko" \) -prune -o \
 		    -type f "${opts[@]}"				\
-		    \( -name "*.ko" -o -path "$OBJ/vmlinux.o" \)	\
+		    \( -name "*.ko" -o -path "$PWD/vmlinux.o" \)	\
 		    -printf '%P\n'
 }
 
@@ -570,10 +591,10 @@ copy_orig_objects() {
 
 	find_objects | mapfile -t files
 
-	xtrace_save "copying orig objects"
+	xtrace_save "copying original objects"
 	for _file in "${files[@]}"; do
 		local rel_file="${_file/.ko/.o}"
-		local file="$OBJ/$rel_file"
+		local file="$PWD/$rel_file"
 		local orig_file="$ORIG_DIR/$rel_file"
 		local orig_dir="$(dirname "$orig_file")"
 
@@ -586,6 +607,7 @@ copy_orig_objects() {
 
 	mv -f "$TMP_DIR/build.log" "$ORIG_DIR"
 	touch "$TIMESTAMP"
+	touch "$ORIG_DIR/.complete"
 }
 
 # Copy all changed objects to $PATCHED_DIR
@@ -606,7 +628,7 @@ copy_patched_objects() {
 	xtrace_save "copying changed objects"
 	for _file in "${files[@]}"; do
 		local rel_file="${_file/.ko/.o}"
-		local file="$OBJ/$rel_file"
+		local file="$PWD/$rel_file"
 		local orig_file="$ORIG_DIR/$rel_file"
 		local patched_file="$PATCHED_DIR/$rel_file"
 		local patched_dir="$(dirname "$patched_file")"
@@ -624,6 +646,36 @@ copy_patched_objects() {
 	(( found == 0 )) && die "no changes detected"
 
 	mv -f "$TMP_DIR/build.log" "$PATCHED_DIR"
+	touch "$PATCHED_DIR/.complete"
+}
+
+# Copy .o files to a separate directory and run "objtool klp checksum" on each
+# copy.  The checksums are written to a .discard.sym_checksum section.
+#
+# If match_dir is given, only process files which also exist there.
+generate_checksums() {
+	local src_dir="$1"
+	local dest_dir="$2"
+	local match_dir="${3:-}"
+	local files=()
+	local file
+
+	rm -rf "$dest_dir"
+	mkdir -p "$dest_dir"
+
+	find "$src_dir" -type f -name "*.o" | mapfile -t files
+	for file in "${files[@]}"; do
+		local rel="${file#"$src_dir"/}"
+		local dest="$dest_dir/$rel"
+
+		[[ -n "$match_dir" && ! -f "$match_dir/$rel" ]] && continue
+
+		mkdir -p "$(dirname "$dest")"
+		cp -f "$file" "$dest"
+		"$OBJTOOL" klp checksum "$dest"
+	done
+
+	touch "$dest_dir/.complete"
 }
 
 # Diff changed objects, writing output object to $DIFF_DIR
@@ -635,23 +687,23 @@ diff_objects() {
 	rm -rf "$DIFF_DIR"
 	mkdir -p "$DIFF_DIR"
 
-	find "$PATCHED_DIR" -type f -name "*.o" | mapfile -t files
+	find "$PATCHED_CSUM_DIR" -type f -name "*.o" | mapfile -t files
 	[[ ${#files[@]} -eq 0 ]] && die "no changes detected"
 
 	[[ -v DEBUG_CLONE ]] && opts=("--debug")
 
 	# Diff all changed objects
 	for file in "${files[@]}"; do
-		local rel_file="${file#"$PATCHED_DIR"/}"
+		local rel_file="${file#"$PATCHED_CSUM_DIR"/}"
 		local orig_file="$rel_file"
-		local patched_file="$PATCHED_DIR/$rel_file"
+		local patched_file="$PATCHED_CSUM_DIR/$rel_file"
 		local out_file="$DIFF_DIR/$rel_file"
 		local filter=()
 		local cmd=()
 
 		mkdir -p "$(dirname "$out_file")"
 
-		cmd=("$SRC/tools/objtool/objtool")
+		cmd=("$OBJTOOL")
 		cmd+=("klp")
 		cmd+=("diff")
 		(( ${#opts[@]} > 0 )) && cmd+=("${opts[@]}")
@@ -668,18 +720,21 @@ diff_objects() {
 		fi
 
 		(
-			cd "$ORIG_DIR"
+			cd "$ORIG_CSUM_DIR"
+			[[ -v VERBOSE ]] && echo "cd $ORIG_CSUM_DIR && ${cmd[*]}"
 			"${cmd[@]}"							\
 				1> >(tee -a "$log")					\
 				2> >(tee -a "$log" | "${filter[@]}" >&2) ||		\
 				die "objtool klp diff failed"
 		)
 	done
+
+	touch "$DIFF_DIR/.complete"
 }
 
-# For each changed object, run objtool with --debug-checksum to get the
-# per-instruction checksums, and then diff those to find the first changed
-# instruction for each function.
+# For each changed object, run "objtool klp checksum" with --debug-checksum to
+# get the per-instruction checksums, and then diff those to find the first
+# changed instruction for each function.
 diff_checksums() {
 	local orig_log="$ORIG_DIR/checksum.log"
 	local patched_log="$PATCHED_DIR/checksum.log"
@@ -703,9 +758,8 @@ diff_checksums() {
 		fi
 	done
 
-	cmd=("$SRC/tools/objtool/objtool")
-	cmd+=("--checksum")
-	cmd+=("--link")
+	cmd=("$OBJTOOL")
+	cmd+=("klp" "checksum")
 	cmd+=("--dry-run")
 
 	for file in "${!funcs[@]}"; do
@@ -714,21 +768,37 @@ diff_checksums() {
 		(
 			cd "$ORIG_DIR"
 			"${cmd[@]}" "$opt" "$file" &> "$orig_log" || \
-				( cat "$orig_log" >&2; die "objtool --debug-checksum failed" )
+				( cat "$orig_log" >&2; die "objtool klp checksum failed" )
 
 			cd "$PATCHED_DIR"
 			"${cmd[@]}" "$opt" "$file" &> "$patched_log" ||	\
-				( cat "$patched_log" >&2; die "objtool --debug-checksum failed" )
+				( cat "$patched_log" >&2; die "objtool klp checksum failed" )
 		)
 
 		for func in ${funcs[$file]}; do
-			diff <( grep0 -E "^DEBUG: .*checksum: $func " "$orig_log"    | sed "s|$ORIG_DIR/||")	\
-			     <( grep0 -E "^DEBUG: .*checksum: $func " "$patched_log" | sed "s|$PATCHED_DIR/||")	\
-				| gawk '/^< DEBUG: / {
-					gsub(/:/, "")
-					printf "%s: %s: %s\n", $3, $5, $6
-					exit
-			}' || true
+			local -a orig patched
+			paste <(grep0 -E "^DEBUG: .*checksum: $func " "$orig_log") \
+			      <(grep0 -E "^DEBUG: .*checksum: $func " "$patched_log") |
+			while IFS= read -r line; do
+				read -ra orig <<< "${line%%$'\t'*}"
+				read -ra patched <<< "${line#*$'\t'}"
+
+				if [[ ${#patched[@]} -eq 0 ]]; then
+					printf "%s: %s: %s (removed)\n" "${orig[1]%:}" "${orig[3]}" "${orig[-2]}"
+					break
+				elif [[ ${#orig[@]} -eq 0 ]]; then
+					printf "%s: %s: %s (added)\n" "${patched[1]%:}" "${patched[3]}" "${patched[-2]}"
+					break
+				fi
+
+				[[ "${orig[-1]}" == "${patched[-1]}" ]] && continue
+
+				printf "%s: %s: %s" "${orig[1]%:}" "${orig[3]}" "${orig[-2]}"
+				[[ "${orig[-2]}" != "${patched[-2]}" ]] && \
+					printf " (patched: %s)" "${patched[-2]}"
+				printf "\n"
+				break
+			done || true
 		done
 	done
 }
@@ -745,7 +815,7 @@ build_patch_module() {
 	rm -rf "$KMOD_DIR"
 	mkdir -p "$KMOD_DIR"
 
-	cp -f "$SRC/scripts/livepatch/init.c" "$KMOD_DIR"
+	cp -f "$SCRIPT_DIR/init.c" "$KMOD_DIR"
 
 	echo "obj-m := $NAME.o" > "$makefile"
 	echo -n "$NAME-y := init.o" >> "$makefile"
@@ -780,19 +850,20 @@ build_patch_module() {
 	[[ $REPLACE -eq 0 ]] && cflags+=("-DKLP_NO_REPLACE")
 
 	cmd=("make")
-	cmd+=("$VERBOSE")
+	if [[ -v VERBOSE ]]; then
+		cmd+=("V=1")
+	else
+		cmd+=("-s")
+	fi
 	cmd+=("-j$JOBS")
 	cmd+=("--directory=.")
 	cmd+=("M=$KMOD_DIR")
 	cmd+=("KCFLAGS=${cflags[*]}")
 
 	# Build a "normal" kernel module with init.c and the diffed objects
-	(
-		cd "$SRC"
-		"${cmd[@]}"							\
-			1> >(tee -a "$log")					\
-			2> >(tee -a "$log" >&2)
-	)
+	"${cmd[@]}"							\
+		1> >(tee -a "$log")					\
+		2> >(tee -a "$log" >&2)
 
 	kmod_file="$KMOD_DIR/$NAME.ko"
 
@@ -803,7 +874,7 @@ build_patch_module() {
 	objcopy --remove-section=.BTF "$kmod_file"
 
 	# Fix (and work around) linker wreckage for klp syms / relocs
-	"$SRC/tools/objtool/objtool" klp post-link "$kmod_file" || die "objtool klp post-link failed"
+	"$OBJTOOL" klp post-link "$kmod_file" || die "objtool klp post-link failed"
 
 	cp -f "$kmod_file" "$OUTFILE"
 }
@@ -839,6 +910,13 @@ if (( SHORT_CIRCUIT <= 2 )); then
 fi
 
 if (( SHORT_CIRCUIT <= 3 )); then
+	status "Generating original checksums"
+	generate_checksums "$ORIG_DIR" "$ORIG_CSUM_DIR" "$PATCHED_DIR"
+	status "Generating patched checksums"
+	generate_checksums "$PATCHED_DIR" "$PATCHED_CSUM_DIR"
+fi
+
+if (( SHORT_CIRCUIT <= 4 )); then
 	status "Diffing objects"
 	diff_objects
 	if [[ -v DIFF_CHECKSUM ]]; then
@@ -847,7 +925,7 @@ if (( SHORT_CIRCUIT <= 3 )); then
 	fi
 fi
 
-if (( SHORT_CIRCUIT <= 4 )); then
+if (( SHORT_CIRCUIT <= 5 )); then
 	status "Building patch module: $OUTFILE"
 	build_patch_module
 fi
diff --git a/scripts/mod/file2alias.c b/scripts/mod/file2alias.c
index 4e99393a35f15..2ad87a74bb033 100644
--- a/scripts/mod/file2alias.c
+++ b/scripts/mod/file2alias.c
@@ -651,7 +651,26 @@ static void do_vio_entry(struct module *mod, void *symval)
 	module_alias_printf(mod, true, "%s", alias);
 }
 
-static void do_input(char *alias,
+static void __attribute__((format(printf, 3, 4)))
+alias_append(char *alias, size_t size, const char *fmt, ...)
+{
+	size_t len = strlen(alias);
+	va_list args;
+	int n;
+
+	if (len >= size)
+		fatal("alias buffer (%zu) overflow before append\n", size);
+
+	va_start(args, fmt);
+	n = vsnprintf(alias + len, size - len, fmt, args);
+	va_end(args);
+
+	if (n < 0 || (size_t)n >= size - len)
+		fatal("alias buffer (%zu) overflow on append (need %d, have %zu)\n",
+		      size, n, size - len);
+}
+
+static void do_input(char *alias, size_t size,
 		     kernel_ulong_t *arr, unsigned int min, unsigned int max)
 {
 	unsigned int i;
@@ -659,13 +678,14 @@ static void do_input(char *alias,
 	for (i = min; i <= max; i++)
 		if (get_unaligned_native(arr + i / BITS_PER_LONG) &
 		    (1ULL << (i % BITS_PER_LONG)))
-			sprintf(alias + strlen(alias), "%X,*", i);
+			alias_append(alias, size, "%X,*", i);
 }
 
 /* input:b0v0p0e0-eXkXrXaXmXlXsXfXwX where X is comma-separated %02X. */
 static void do_input_entry(struct module *mod, void *symval)
 {
 	char alias[256] = {};
+	const size_t sizeof_alias = sizeof(alias);
 
 	DEF_FIELD(symval, input_device_id, flags);
 	DEF_FIELD(symval, input_device_id, bustype);
@@ -687,35 +707,35 @@ static void do_input_entry(struct module *mod, void *symval)
 	ADD(alias, "p", flags & INPUT_DEVICE_ID_MATCH_PRODUCT, product);
 	ADD(alias, "e", flags & INPUT_DEVICE_ID_MATCH_VERSION, version);
 
-	sprintf(alias + strlen(alias), "-e*");
+	alias_append(alias, sizeof_alias, "-e*");
 	if (flags & INPUT_DEVICE_ID_MATCH_EVBIT)
-		do_input(alias, *evbit, 0, INPUT_DEVICE_ID_EV_MAX);
-	sprintf(alias + strlen(alias), "k*");
+		do_input(alias, sizeof_alias, *evbit, 0, INPUT_DEVICE_ID_EV_MAX);
+	alias_append(alias, sizeof_alias, "k*");
 	if (flags & INPUT_DEVICE_ID_MATCH_KEYBIT)
-		do_input(alias, *keybit,
+		do_input(alias, sizeof_alias, *keybit,
 			 INPUT_DEVICE_ID_KEY_MIN_INTERESTING,
 			 INPUT_DEVICE_ID_KEY_MAX);
-	sprintf(alias + strlen(alias), "r*");
+	alias_append(alias, sizeof_alias, "r*");
 	if (flags & INPUT_DEVICE_ID_MATCH_RELBIT)
-		do_input(alias, *relbit, 0, INPUT_DEVICE_ID_REL_MAX);
-	sprintf(alias + strlen(alias), "a*");
+		do_input(alias, sizeof_alias, *relbit, 0, INPUT_DEVICE_ID_REL_MAX);
+	alias_append(alias, sizeof_alias, "a*");
 	if (flags & INPUT_DEVICE_ID_MATCH_ABSBIT)
-		do_input(alias, *absbit, 0, INPUT_DEVICE_ID_ABS_MAX);
-	sprintf(alias + strlen(alias), "m*");
+		do_input(alias, sizeof_alias, *absbit, 0, INPUT_DEVICE_ID_ABS_MAX);
+	alias_append(alias, sizeof_alias, "m*");
 	if (flags & INPUT_DEVICE_ID_MATCH_MSCIT)
-		do_input(alias, *mscbit, 0, INPUT_DEVICE_ID_MSC_MAX);
-	sprintf(alias + strlen(alias), "l*");
+		do_input(alias, sizeof_alias, *mscbit, 0, INPUT_DEVICE_ID_MSC_MAX);
+	alias_append(alias, sizeof_alias, "l*");
 	if (flags & INPUT_DEVICE_ID_MATCH_LEDBIT)
-		do_input(alias, *ledbit, 0, INPUT_DEVICE_ID_LED_MAX);
-	sprintf(alias + strlen(alias), "s*");
+		do_input(alias, sizeof_alias, *ledbit, 0, INPUT_DEVICE_ID_LED_MAX);
+	alias_append(alias, sizeof_alias, "s*");
 	if (flags & INPUT_DEVICE_ID_MATCH_SNDBIT)
-		do_input(alias, *sndbit, 0, INPUT_DEVICE_ID_SND_MAX);
-	sprintf(alias + strlen(alias), "f*");
+		do_input(alias, sizeof_alias, *sndbit, 0, INPUT_DEVICE_ID_SND_MAX);
+	alias_append(alias, sizeof_alias, "f*");
 	if (flags & INPUT_DEVICE_ID_MATCH_FFBIT)
-		do_input(alias, *ffbit, 0, INPUT_DEVICE_ID_FF_MAX);
-	sprintf(alias + strlen(alias), "w*");
+		do_input(alias, sizeof_alias, *ffbit, 0, INPUT_DEVICE_ID_FF_MAX);
+	alias_append(alias, sizeof_alias, "w*");
 	if (flags & INPUT_DEVICE_ID_MATCH_SWBIT)
-		do_input(alias, *swbit, 0, INPUT_DEVICE_ID_SW_MAX);
+		do_input(alias, sizeof_alias, *swbit, 0, INPUT_DEVICE_ID_SW_MAX);
 
 	module_alias_printf(mod, false, "input:%s", alias);
 }
@@ -895,12 +915,16 @@ static const struct dmifield {
 	{ NULL,  DMI_NONE }
 };
 
-static void dmi_ascii_filter(char *d, const char *s)
+static void dmi_ascii_filter(char *d, size_t avail, const char *s)
 {
 	/* Filter out characters we don't want to see in the modalias string */
 	for (; *s; s++)
-		if (*s > ' ' && *s < 127 && *s != ':')
+		if (*s > ' ' && *s < 127 && *s != ':') {
+			if (avail <= 1)
+				fatal("%s: alias buffer overflow\n", __func__);
 			*(d++) = *s;
+			avail--;
+		}
 
 	*d = 0;
 }
@@ -909,6 +933,8 @@ static void dmi_ascii_filter(char *d, const char *s)
 static void do_dmi_entry(struct module *mod, void *symval)
 {
 	char alias[256] = {};
+	const size_t sizeof_alias = sizeof(alias);
+	size_t len;
 	int i, j;
 	DEF_FIELD_ADDR(symval, dmi_system_id, matches);
 
@@ -916,11 +942,12 @@ static void do_dmi_entry(struct module *mod, void *symval)
 		for (j = 0; j < 4; j++) {
 			if ((*matches)[j].slot &&
 			    (*matches)[j].slot == dmi_fields[i].field) {
-				sprintf(alias + strlen(alias), ":%s*",
-					dmi_fields[i].prefix);
-				dmi_ascii_filter(alias + strlen(alias),
+				alias_append(alias, sizeof_alias, ":%s*",
+					     dmi_fields[i].prefix);
+				len = strlen(alias);
+				dmi_ascii_filter(alias + len, sizeof_alias - len,
 						 (*matches)[j].substr);
-				strcat(alias, "*");
+				alias_append(alias, sizeof_alias, "*");
 			}
 		}
 	}
diff --git a/scripts/package/PKGBUILD b/scripts/package/PKGBUILD
index 452374d63c244..66e4b6a37783c 100644
--- a/scripts/package/PKGBUILD
+++ b/scripts/package/PKGBUILD
@@ -10,7 +10,7 @@ for pkg in $_extrapackages; do
 	pkgname+=("${pkgbase}-${pkg}")
 done
 
-pkgver="${KERNELRELEASE//-/_}"
+pkgver="$(echo "${KERNELRELEASE}" | sed 's/-\(rc[0-9]\+\)/\1/;s/-/_/g')"
 # The PKGBUILD is evaluated multiple times.
 # Running scripts/build-version from here would introduce inconsistencies.
 pkgrel="${KBUILD_REVISION}"
@@ -121,6 +121,9 @@ _package-debug(){
 	install -Dt "${debugdir}" -m644 vmlinux
 	mkdir -p "${builddir}"
 	ln -sr "${debugdir}/vmlinux" "${builddir}/vmlinux"
+
+	echo "Installing unstripped vDSO(s)..."
+	${MAKE} INSTALL_MOD_PATH="${pkgdir}/usr" vdso_install
 }
 
 for _p in "${pkgname[@]}"; do
diff --git a/scripts/timer_migration_tree.py b/scripts/timer_migration_tree.py
new file mode 100755
index 0000000000000..faac9de854bd4
--- /dev/null
+++ b/scripts/timer_migration_tree.py
@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+
+"""
+Draw the timer migration tree.
+
+1) Boot with trace_event==tmigr_connect_cpu_parent,tmigr_connect_child_parent
+2) ./timer_migration_tree.py < /sys/kernel/tracing/trace
+"""
+
+import re, sys
+from ete3 import Tree
+
+class Node:
+	def __init__(self, group):
+		self.group = group
+		self.children = []
+		self.parent = None
+		self.num_children = 0
+		self.groupmask = 0
+		self.lvl = -1
+
+	def set_groupmask(self, groupmask):
+		self.groupmask = groupmask
+
+	def set_parent(self, parent):
+		self.parent = parent
+
+	def add_child(self, child):
+		self.children.append(child)
+
+	def set_lvl(self, lvl):
+		self.lvl = lvl
+
+	def set_numa(self, numa):
+		self.numa = numa
+
+	def set_num_children(self, num_children):
+		self.num_children = num_children
+
+	def __repr__(self):
+		if self.parent:
+			parent_grp = self.parent.group
+		else:
+			parent_grp = "-"
+		return "Group: %s mask: %s parent: %s lvl: %d numa: %d num_children: %d" % (self.group, self.groupmask, parent_grp, self.lvl, self.numa, self.num_children)
+
+hierarchies = { }
+
+def get_hierarchy(capacity):
+	if capacity not in hierarchies:
+		hierarchies[capacity] = {}
+	return hierarchies[capacity]
+
+def get_node(capacity, group):
+	hier = get_hierarchy(capacity)
+	if group in hier:
+		return hier[group]
+	else:
+		n = Node(group)
+		hier[group] = n
+		return n
+
+def tmigr_connect_cpu_parent(ts, line):
+	s = re.search("tmigr_connect_cpu_parent: cpu=([0-9]+) groupmask=([0-9a-zA-Z]+) parent=([0-9a-zA-Z]+) lvl=([0-9]+) numa=([-]?[0-9]+) capacity=([-]?[0-9]+) num_children=([0-9]+)", line)
+	if s is None:
+		return False
+	(cpu, groupmask, parent, lvl, numa, capacity, num_children) = (int(s.group(1)), s.group(2), s.group(3), int(s.group(4)), int(s.group(5)), int(s.group(6)), int(s.group(7)))
+	n = get_node(capacity, cpu)
+	p = get_node(capacity, parent)
+	n.set_parent(p)
+	n.set_groupmask(groupmask)
+	n.set_lvl(-1)
+	p.set_lvl(lvl)
+	p.set_numa(numa)
+	n.set_numa(numa)
+	p.set_num_children(num_children)
+	p.add_child(n)
+
+def tmigr_connect_child_parent(ts, line):
+	s = re.search("tmigr_connect_child_parent: group=([0-9a-zA-Z]+) groupmask=([0-9a-zA-Z]+) parent=([0-9a-zA-Z]+) lvl=([0-9]+) numa=([-]?[0-9]+) capacity=([-]?[0-9]+) num_children=([0-9]+)", line)
+	if s is None:
+		return False
+	(group, groupmask, parent, lvl, numa, capacity, num_children) = (s.group(1), s.group(2), s.group(3), int(s.group(4)), int(s.group(5)), int(s.group(6)), int(s.group(7)))
+	n = get_node(capacity, group)
+	p = get_node(capacity, parent)
+	n.set_parent(p)
+	n.set_groupmask(groupmask)
+	p.set_lvl(lvl)
+	p.set_numa(numa)
+	p.set_num_children(num_children)
+	p.add_child(n)
+
+def populate(enode, node):
+	enode = enode.add_child(name = node.group)
+	enode.add_feature("groupmask", "m:%s" % node.groupmask)
+	enode.add_feature("lvl", "lvl:%d" % node.lvl)
+	enode.add_feature("numa", "node %d" % node.numa)
+	enode.add_feature("num_children", "c=%d" % node.num_children)
+	for child in node.children:
+		populate(enode, child)
+
+if __name__ == "__main__":
+	for line in sys.stdin:
+		s = re.search("([0-9]+[.][0-9]{6}): (.+?)$", line, re.S)
+		if s is not None:
+			if tmigr_connect_cpu_parent(float(s.group(1)), s.group(2)):
+				continue
+			if tmigr_connect_child_parent(float(s.group(1)), s.group(2)):
+				continue
+
+	for cap in hierarchies:
+		h = hierarchies[cap]
+		print("Tree for capacity %d" % cap)
+		for k in h:
+			n = h[k]
+			while n.parent != None:
+				n = n.parent
+			root = Tree()
+			populate(root, n)
+			print(root.get_ascii(show_internal=True, attributes=["name", "numa", "lvl"]))
+			break
diff --git a/scripts/update-intel-ucode-defs.py b/scripts/update-intel-ucode-defs.py
new file mode 100755
index 0000000000000..9d6cc2c6075fe
--- /dev/null
+++ b/scripts/update-intel-ucode-defs.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0
+import argparse
+import re
+import shutil
+import subprocess
+import sys
+import os
+
+script = os.path.relpath(__file__)
+
+DESCRIPTION = f"""
+For Intel CPUs, update the microcode revisions that determine
+X86_BUG_OLD_MICROCODE.
+
+This script is intended to be run in response to releases of the
+official Intel microcode GitHub repository:
+https://github.com/intel/Intel-Linux-Processor-Microcode-Data-Files.git
+
+It takes the Intel microcode files as input and uses iucode-tool to
+extract the revision information. It prints the output in the format
+expected by intel-ucode-defs.h.
+
+Usage:
+    ./{script} /path/to/microcode/files > /path/to/intel-ucode-defs.h
+
+Typically, someone at Intel would see a new public release, wait for at
+least three months to ensure the update is stable, run this script to
+refresh the intel-ucode-defs.h file, and send a patch upstream to update
+the mainline and stable versions.
+
+Any exception to this process should be supported with an appropriate
+justification.
+"""
+
+SIG_RE = re.compile(r'sig (0x[0-9a-fA-F]+)')
+PFM_RE = re.compile(r'pf_mask (0x[0-9a-fA-F]+)')
+REV_RE = re.compile(r'rev (0x[0-9a-fA-F]+)')
+
+# Functions to extract family, model, and stepping
+def bits(val, bottom, top):
+    mask = (1 << (top + 1 - bottom)) - 1
+    return (val >> bottom) & mask
+
+def family(sig):
+    if bits(sig, 8, 11) == 0xf:
+        return bits(sig, 8, 11) + bits(sig, 20, 27)
+    return bits(sig, 8, 11)
+
+def model(sig):
+    return bits(sig, 4, 7) | (bits(sig, 16, 19) << 4)
+
+def step(sig):
+    return bits(sig, 0, 3)
+
+class Ucode:
+    def __init__(self, sig, pfm, rev):
+        self.family = family(sig)
+        self.model = model(sig)
+        self.steppings = 1 << step(sig)
+        self.platforms = pfm
+        self.rev = rev
+
+        self.key = (self.family, self.model, self.steppings, self.platforms)
+
+    def __eq__(self, other):
+        return self.key == other.key
+
+    def __hash__(self):
+        return hash(self.key)
+
+    def __str__(self):
+        return "{ .flags = X86_CPU_ID_FLAG_ENTRY_VALID, .vendor = X86_VENDOR_INTEL, .family = 0x%x, .model = 0x%02x, .steppings = 0x%04x, .platform_mask = 0x%02x, .driver_data = 0x%x }," % \
+                (self.family, self.model, self.steppings, self.platforms, self.rev)
+
+def main():
+    parser = argparse.ArgumentParser(description=DESCRIPTION,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument('ucode_files', nargs='+', help='Path(s) to the microcode files')
+
+    args = parser.parse_args()
+
+    # Process the microcode files using iucode-tool
+    iucode_tool = shutil.which("iucode-tool") or shutil.which("iucode_tool")
+    if iucode_tool is None:
+        print("Error: iucode-tool not found, please install it", file=sys.stderr)
+        sys.exit(1)
+
+    cmd = [iucode_tool, '--list-all'] + args.ucode_files
+
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0:
+        print("Error: iucode-tool ran into an error, exiting", file=sys.stderr)
+        if result.stderr:
+            print(result.stderr, file=sys.stderr, end='')
+        sys.exit(1)
+
+    ucodes = set()
+
+    # Parse the output of iucode-tool
+    for line in result.stdout.splitlines():
+        sig_match = SIG_RE.search(line)
+        pfm_match = PFM_RE.search(line)
+        rev_match = REV_RE.search(line)
+
+        if not (sig_match and pfm_match and rev_match):
+            continue
+
+        sig = int(sig_match.group(1), 16)
+        pfm = int(pfm_match.group(1), 16)
+        rev = int(rev_match.group(1), 16)
+        debug_rev = bits(rev, 31, 31)
+        if debug_rev != 0:
+            print("Error: Debug ucode file found, exiting", file=sys.stderr)
+            sys.exit(1)
+
+        ucodes.add(Ucode(sig, pfm, rev))
+
+    if not ucodes:
+        print("Error: No valid microcode files found, exiting", file=sys.stderr)
+        sys.exit(1)
+
+    # Sort and print the microcode entries
+    print("/* SPDX-License-Identifier: GPL-2.0 */")
+    print("/* Auto-generated by scripts/update-intel-ucode-defs.py */")
+    for u in sorted(ucodes, key=lambda x: x.key):
+        print(u)
+
+if __name__ == "__main__":
+    main()
diff --git a/security/Kconfig b/security/Kconfig
index f7bf6cdc6229e..ab7aa622455a8 100644
--- a/security/Kconfig
+++ b/security/Kconfig
@@ -228,6 +228,7 @@ source "security/safesetid/Kconfig"
 source "security/lockdown/Kconfig"
 source "security/landlock/Kconfig"
 source "security/ipe/Kconfig"
+source "security/hornet/Kconfig"
 
 source "security/integrity/Kconfig"
 
@@ -272,7 +273,7 @@ config LSM
 	default "landlock,lockdown,yama,loadpin,safesetid,apparmor,selinux,smack,tomoyo,ipe,bpf" if DEFAULT_SECURITY_APPARMOR
 	default "landlock,lockdown,yama,loadpin,safesetid,tomoyo,ipe,bpf" if DEFAULT_SECURITY_TOMOYO
 	default "landlock,lockdown,yama,loadpin,safesetid,ipe,bpf" if DEFAULT_SECURITY_DAC
-	default "landlock,lockdown,yama,loadpin,safesetid,selinux,smack,tomoyo,apparmor,ipe,bpf"
+	default "landlock,lockdown,yama,loadpin,safesetid,selinux,smack,tomoyo,apparmor,ipe,hornet,bpf"
 	help
 	  A comma-separated list of LSMs, in initialization order.
 	  Any LSMs left off this list, except for those with order
diff --git a/security/Makefile b/security/Makefile
index 4601230ba442a..b68cb56e419bc 100644
--- a/security/Makefile
+++ b/security/Makefile
@@ -26,6 +26,7 @@ obj-$(CONFIG_CGROUPS)			+= device_cgroup.o
 obj-$(CONFIG_BPF_LSM)			+= bpf/
 obj-$(CONFIG_SECURITY_LANDLOCK)		+= landlock/
 obj-$(CONFIG_SECURITY_IPE)		+= ipe/
+obj-$(CONFIG_SECURITY_HORNET)		+= hornet/
 
 # Object integrity file lists
 obj-$(CONFIG_INTEGRITY)			+= integrity/
diff --git a/security/hornet/Kconfig b/security/hornet/Kconfig
new file mode 100644
index 0000000000000..537ad015958cd
--- /dev/null
+++ b/security/hornet/Kconfig
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0-only
+config SECURITY_HORNET
+	bool "Hornet support"
+	depends on SECURITY && BPF_SYSCALL
+	select CRYPTO_LIB_SHA256
+	select PKCS7_MESSAGE_PARSER
+	select SYSTEM_DATA_VERIFICATION
+	default n
+	help
+	  This selects Hornet.
+	  Further information can be found in
+	  Documentation/admin-guide/LSM/Hornet.rst.
+
+	  If you are unsure how to answer this question, answer N.
diff --git a/security/hornet/Makefile b/security/hornet/Makefile
new file mode 100644
index 0000000000000..26b6f954f762e
--- /dev/null
+++ b/security/hornet/Makefile
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-$(CONFIG_SECURITY_HORNET) := hornet.o
+
+hornet-y := hornet.asn1.o \
+	hornet_lsm.o \
+
+$(obj)/hornet.asn1.o: $(obj)/hornet.asn1.c $(obj)/hornet.asn1.h
diff --git a/security/hornet/hornet.asn1 b/security/hornet/hornet.asn1
new file mode 100644
index 0000000000000..e60abf451ae23
--- /dev/null
+++ b/security/hornet/hornet.asn1
@@ -0,0 +1,12 @@
+-- SPDX-License-Identifier: BSD-3-Clause
+--
+-- Copyright (C) 2026 Microsoft
+--
+-- https://www.rfc-editor.org/rfc/rfc5652#section-3
+
+HornetData ::= SET OF Map
+
+Map ::= SEQUENCE {
+	index			INTEGER ({ hornet_map_index }),
+	sha			OCTET STRING ({ hornet_map_hash })
+} ({ hornet_next_map })
diff --git a/security/hornet/hornet_lsm.c b/security/hornet/hornet_lsm.c
new file mode 100644
index 0000000000000..a4d11fa5b0889
--- /dev/null
+++ b/security/hornet/hornet_lsm.c
@@ -0,0 +1,352 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Hornet Linux Security Module
+ *
+ * Author: Blaise Boscaccy <bboscaccy@linux.microsoft.com>
+ *
+ * Copyright (C) 2026 Microsoft Corporation
+ */
+
+#include <linux/lsm_hooks.h>
+#include <uapi/linux/lsm.h>
+#include <linux/bpf.h>
+#include <linux/verification.h>
+#include <crypto/public_key.h>
+#include <linux/module_signature.h>
+#include <crypto/pkcs7.h>
+#include <linux/sort.h>
+#include <linux/asn1_decoder.h>
+#include <linux/oid_registry.h>
+#include "hornet.asn1.h"
+
+#define MAX_USED_MAPS 64
+
+struct hornet_maps {
+	bpfptr_t fd_array;
+};
+
+/* The only hashing algorithm available is SHA256 due to it be hardcoded
+ * in the bpf subsystem.
+ */
+
+struct hornet_parse_context {
+	int indexes[MAX_USED_MAPS];
+	bool skips[MAX_USED_MAPS];
+	unsigned char hashes[SHA256_DIGEST_SIZE * MAX_USED_MAPS];
+	int hash_count;
+};
+
+struct hornet_prog_security_struct {
+	int signed_hash_count;
+	unsigned char signed_hashes[SHA256_DIGEST_SIZE * MAX_USED_MAPS];
+};
+
+struct lsm_blob_sizes hornet_blob_sizes __ro_after_init = {
+	.lbs_bpf_prog = sizeof(struct hornet_prog_security_struct),
+};
+
+static inline struct hornet_prog_security_struct *
+hornet_bpf_prog_security(struct bpf_prog *prog)
+{
+	return prog->aux->security + hornet_blob_sizes.lbs_bpf_prog;
+}
+
+static int hornet_verify_hashes(struct hornet_maps *maps,
+				struct hornet_parse_context *ctx,
+				struct bpf_prog *prog)
+{
+	int map_fd;
+	u32 i;
+	struct bpf_map *map;
+	int err = 0;
+	unsigned char hash[SHA256_DIGEST_SIZE];
+	struct hornet_prog_security_struct *security = hornet_bpf_prog_security(prog);
+
+	for (i = 0; i < ctx->hash_count; i++) {
+		if (ctx->skips[i])
+			continue;
+
+		err = copy_from_bpfptr_offset(&map_fd, maps->fd_array,
+					      ctx->indexes[i] * sizeof(map_fd),
+					      sizeof(map_fd));
+		if (err != 0)
+			return LSM_INT_VERDICT_FAULT;
+
+		CLASS(fd, f)(map_fd);
+		if (fd_empty(f))
+			return LSM_INT_VERDICT_FAULT;
+		if (unlikely(fd_file(f)->f_op != &bpf_map_fops))
+			return LSM_INT_VERDICT_FAULT;
+
+		map = fd_file(f)->private_data;
+		if (!READ_ONCE(map->frozen))
+			return LSM_INT_VERDICT_FAULT;
+
+		if (!map->ops->map_get_hash)
+			return LSM_INT_VERDICT_FAULT;
+
+		if (map->ops->map_get_hash(map, SHA256_DIGEST_SIZE, hash))
+			return LSM_INT_VERDICT_FAULT;
+
+		err = memcmp(hash, &ctx->hashes[i * SHA256_DIGEST_SIZE],
+			      SHA256_DIGEST_SIZE);
+		if (err)
+			return LSM_INT_VERDICT_UNEXPECTED;
+
+		memcpy(&security->signed_hashes[security->signed_hash_count * SHA256_DIGEST_SIZE],
+		       &ctx->hashes[i * SHA256_DIGEST_SIZE], SHA256_DIGEST_SIZE);
+		security->signed_hash_count++;
+	}
+	return LSM_INT_VERDICT_OK;
+}
+
+int hornet_next_map(void *context, size_t hdrlen,
+		     unsigned char tag,
+		     const void *value, size_t vlen)
+{
+	struct hornet_parse_context *ctx = (struct hornet_parse_context *)context;
+
+	if (++ctx->hash_count >= MAX_USED_MAPS)
+		return -EINVAL;
+	return 0;
+}
+
+int hornet_map_index(void *context, size_t hdrlen,
+		     unsigned char tag,
+		     const void *value, size_t vlen)
+{
+	struct hornet_parse_context *ctx = (struct hornet_parse_context *)context;
+
+	if (vlen != 1)
+		return -EINVAL;
+
+	ctx->indexes[ctx->hash_count] = *(u8 *)value;
+	return 0;
+}
+
+int hornet_map_hash(void *context, size_t hdrlen,
+		    unsigned char tag,
+		    const void *value, size_t vlen)
+
+{
+	struct hornet_parse_context *ctx = (struct hornet_parse_context *)context;
+
+	if (vlen != SHA256_DIGEST_SIZE && vlen != 0)
+		return -EINVAL;
+
+	if (vlen) {
+		ctx->skips[ctx->hash_count] = false;
+		memcpy(&ctx->hashes[ctx->hash_count * SHA256_DIGEST_SIZE], value, vlen);
+	} else
+		ctx->skips[ctx->hash_count] = true;
+
+	return 0;
+}
+
+static int hornet_check_program(struct bpf_prog *prog, union bpf_attr *attr,
+				struct bpf_token *token, bool is_kernel,
+				enum lsm_integrity_verdict *verdict)
+{
+	struct hornet_maps maps = {0};
+	bpfptr_t usig = make_bpfptr(attr->signature, is_kernel);
+	struct pkcs7_message *msg;
+	struct hornet_parse_context *ctx;
+	void *sig;
+	int err;
+	const void *authattrs;
+	size_t authattrs_len;
+	struct key *key;
+	key_ref_t user_key = ERR_PTR(-ENOKEY);
+
+	if (!attr->signature) {
+		*verdict = LSM_INT_VERDICT_UNSIGNED;
+		return 0;
+	}
+
+	if (!attr->signature_size) {
+		*verdict = LSM_INT_VERDICT_BADSIG;
+		return 0;
+	}
+
+	ctx = kzalloc(sizeof(struct hornet_parse_context), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+
+	maps.fd_array = make_bpfptr(attr->fd_array, is_kernel);
+	sig = kzalloc(attr->signature_size, GFP_KERNEL);
+	if (!sig) {
+		err = -ENOMEM;
+		goto out;
+	}
+	err = copy_from_bpfptr(sig, usig, attr->signature_size);
+	if (err != 0) {
+		err = -EFAULT;
+		goto cleanup_sig;
+	}
+
+	msg = pkcs7_parse_message(sig, attr->signature_size);
+	if (IS_ERR(msg)) {
+		*verdict = LSM_INT_VERDICT_BADSIG;
+		err = 0;
+		goto cleanup_sig;
+	}
+
+	if (system_keyring_id_check(attr->keyring_id) == 0)
+		key = (struct key *)(unsigned long)attr->keyring_id;
+	else {
+		user_key = lookup_user_key(attr->keyring_id, 0, KEY_DEFER_PERM_CHECK);
+		if (IS_ERR(user_key)) {
+			*verdict = LSM_INT_VERDICT_UNKNOWNKEY;
+			goto cleanup_msg;
+		}
+		key = key_ref_to_ptr(user_key);
+	}
+
+	if (verify_pkcs7_message_sig(prog->insnsi, prog->len * sizeof(struct bpf_insn), msg,
+				     key,
+				     VERIFYING_BPF_SIGNATURE,
+				     NULL, NULL)) {
+		*verdict = LSM_INT_VERDICT_BADSIG;
+		err = 0;
+		goto cleanup_msg;
+	}
+
+	if (pkcs7_get_authattr(msg, OID_hornet_data,
+			       &authattrs, &authattrs_len) == -ENODATA) {
+		*verdict = LSM_INT_VERDICT_PARTIALSIG;
+		err = 0;
+		goto cleanup_msg;
+	}
+
+	err = asn1_ber_decoder(&hornet_decoder, ctx, authattrs, authattrs_len);
+	if (err < 0 || authattrs == NULL) {
+		*verdict = LSM_INT_VERDICT_BADSIG;
+		err = 0;
+		goto cleanup_msg;
+	}
+
+	*verdict = hornet_verify_hashes(&maps, ctx, prog);
+	err = 0;
+
+cleanup_msg:
+	pkcs7_free_message(msg);
+	if (!IS_ERR(user_key))
+		key_put(key);
+cleanup_sig:
+	kfree(sig);
+out:
+	kfree(ctx);
+	return err;
+}
+
+static const struct lsm_id hornet_lsmid = {
+	.name = "hornet",
+	.id = LSM_ID_HORNET,
+};
+
+static int hornet_bpf_prog_load_integrity(struct bpf_prog *prog, union bpf_attr *attr,
+					  struct bpf_token *token, bool is_kernel)
+{
+	enum lsm_integrity_verdict verdict;
+	int result = hornet_check_program(prog, attr, token, is_kernel, &verdict);
+
+	if (result < 0)
+		return result;
+
+	return security_bpf_prog_load_post_integrity(prog, attr, token, is_kernel,
+						     &hornet_lsmid, verdict);
+}
+
+static int hornet_check_prog_maps(u32 ufd)
+{
+	CLASS(fd, f)(ufd);
+	struct bpf_prog *prog;
+	struct hornet_prog_security_struct *security;
+	unsigned char hash[SHA256_DIGEST_SIZE];
+	struct bpf_map *map;
+	int i, j;
+	bool found;
+	int covered_count = 0;
+
+	if (fd_empty(f))
+		return -EBADF;
+	if (fd_file(f)->f_op != &bpf_prog_fops)
+		return -EINVAL;
+
+	prog = fd_file(f)->private_data;
+	security = hornet_bpf_prog_security(prog);
+
+	if (!security->signed_hash_count)
+		return 0;
+
+	mutex_lock(&prog->aux->used_maps_mutex);
+
+	/* Verify every used_map has a matching signed hash */
+	for (j = 0; j < prog->aux->used_map_cnt; j++) {
+		map = prog->aux->used_maps[j];
+
+		if (!READ_ONCE(map->frozen) || !map->ops->map_get_hash)
+			continue;
+
+		if (map->ops->map_get_hash(map, SHA256_DIGEST_SIZE, hash))
+			continue;
+
+		found = false;
+		for (i = 0; i < security->signed_hash_count; i++) {
+			if (memcmp(hash,
+				   &security->signed_hashes[i * SHA256_DIGEST_SIZE],
+				   SHA256_DIGEST_SIZE) == 0) {
+				found = true;
+				break;
+			}
+		}
+		if (!found) {
+			mutex_unlock(&prog->aux->used_maps_mutex);
+			return -EPERM;
+		}
+		covered_count++;
+	}
+
+	mutex_unlock(&prog->aux->used_maps_mutex);
+
+	/* Ensure all signed hashes were accounted for */
+	if (covered_count != security->signed_hash_count)
+		return -EPERM;
+
+	return 0;
+}
+
+static int hornet_bpf(int cmd, union bpf_attr *attr, unsigned int size, bool kernel)
+{
+	/* in horent_bpf(), anything that had originated from kernel space we assume
+	 * has already been checked, in some form or another, so we don't bother
+	 * checking the intergity of any maps. In hornet_bpf_prog_load_integrity(),
+	 * hornet doesn't make any opinion on that and delegates that to the downstream
+	 * policy enforcement.
+	 */
+
+	if (cmd != BPF_PROG_RUN)
+		return 0;
+	if (kernel)
+		return 0;
+
+	return hornet_check_prog_maps(attr->test.prog_fd);
+}
+
+static struct security_hook_list hornet_hooks[] __ro_after_init = {
+	LSM_HOOK_INIT(bpf_prog_load_integrity, hornet_bpf_prog_load_integrity),
+	LSM_HOOK_INIT(bpf, hornet_bpf),
+};
+
+static int __init hornet_init(void)
+{
+	pr_info("Hornet: eBPF signature verification enabled\n");
+	security_add_hooks(hornet_hooks, ARRAY_SIZE(hornet_hooks), &hornet_lsmid);
+	return 0;
+}
+
+DEFINE_LSM(hornet) = {
+	.id = &hornet_lsmid,
+	.blobs = &hornet_blob_sizes,
+	.init = hornet_init,
+};
diff --git a/security/integrity/digsig_asymmetric.c b/security/integrity/digsig_asymmetric.c
index 6e68ec3becbd1..b4c23a0ed68f2 100644
--- a/security/integrity/digsig_asymmetric.c
+++ b/security/integrity/digsig_asymmetric.c
@@ -79,18 +79,25 @@ static struct key *request_asymmetric_key(struct key *keyring, uint32_t keyid)
 	return key;
 }
 
-int asymmetric_verify(struct key *keyring, const char *sig,
-		      int siglen, const char *data, int datalen)
+/**
+ * asymmetric_verify_common -- sigv2 and sigv3 common verify function
+ * @key: The key to use for signature verification; caller must free it
+ * @pk: The associated public key; must not be NULL
+ * @sig: The xattr signature
+ * @siglen: The length of the xattr signature; must be at least
+ *          sizeof(struct signature_v2_hdr)
+ * @data: The data to verify the signature on
+ * @datalen: Length of @data
+ */
+static int asymmetric_verify_common(const struct key *key,
+				    const struct public_key *pk,
+				    const char *sig, int siglen,
+				    const char *data, int datalen)
 {
-	struct public_key_signature pks;
 	struct signature_v2_hdr *hdr = (struct signature_v2_hdr *)sig;
-	const struct public_key *pk;
-	struct key *key;
+	struct public_key_signature pks;
 	int ret;
 
-	if (siglen <= sizeof(*hdr))
-		return -EBADMSG;
-
 	siglen -= sizeof(*hdr);
 
 	if (siglen != be16_to_cpu(hdr->sig_size))
@@ -99,15 +106,9 @@ int asymmetric_verify(struct key *keyring, const char *sig,
 	if (hdr->hash_algo >= HASH_ALGO__LAST)
 		return -ENOPKG;
 
-	key = request_asymmetric_key(keyring, be32_to_cpu(hdr->keyid));
-	if (IS_ERR(key))
-		return PTR_ERR(key);
-
 	memset(&pks, 0, sizeof(pks));
 
 	pks.hash_algo = hash_algo_name[hdr->hash_algo];
-
-	pk = asymmetric_key_public_key(key);
 	pks.pkey_algo = pk->pkey_algo;
 	if (!strcmp(pk->pkey_algo, "rsa")) {
 		pks.encoding = "pkcs1";
@@ -127,15 +128,42 @@ int asymmetric_verify(struct key *keyring, const char *sig,
 	pks.s_size = siglen;
 	ret = verify_signature(key, &pks);
 out:
-	key_put(key);
 	pr_debug("%s() = %d\n", __func__, ret);
 	return ret;
 }
 
+int asymmetric_verify(struct key *keyring, const char *sig,
+		      int siglen, const char *data, int datalen)
+{
+	struct signature_v2_hdr *hdr = (struct signature_v2_hdr *)sig;
+	const struct public_key *pk;
+	struct key *key;
+	int ret;
+
+	if (siglen <= sizeof(*hdr))
+		return -EBADMSG;
+
+	key = request_asymmetric_key(keyring, be32_to_cpu(hdr->keyid));
+	if (IS_ERR(key))
+		return PTR_ERR(key);
+	pk = asymmetric_key_public_key(key);
+	if (!pk) {
+		ret = -ENOKEY;
+		goto out;
+	}
+
+	ret = asymmetric_verify_common(key, pk, sig, siglen, data, datalen);
+
+out:
+	key_put(key);
+
+	return ret;
+}
+
 /*
  * calc_file_id_hash - calculate the hash of the ima_file_id struct data
  * @type: xattr type [enum evm_ima_xattr_type]
- * @algo: hash algorithm [enum hash_algo]
+ * @algo: hash algorithm [enum hash_algo]; caller must ensure valid value
  * @digest: pointer to the digest to be hashed
  * @hash: (out) pointer to the hash
  *
@@ -176,17 +204,99 @@ static int calc_file_id_hash(enum evm_ima_xattr_type type,
 	return rc;
 }
 
+/**
+ * asymmetric_verify_v3_hashless - Use hashless signature verification on sigv3
+ * @key: The key to use for signature verification; caller must free it
+ * @pk: The associated public key; must not be NULL
+ * @encoding: The encoding the key type uses
+ * @sig: The xattr signature
+ * @siglen: The length of the xattr signature; must be at least
+ *          sizeof(struct signature_v2_hdr)
+ * @algo: hash algorithm [enum hash_algo]; caller must ensure valid value
+ * @digest: The file digest
+ *
+ * Create an ima_file_id structure and use it for signature verification
+ * directly. This can be used for ML-DSA in pure mode for example.
+ */
+static int asymmetric_verify_v3_hashless(struct key *key,
+					 const struct public_key *pk,
+					 const char *encoding,
+					 const char *sig, int siglen,
+					 u8 algo,
+					 const u8 *digest)
+{
+	struct signature_v2_hdr *hdr = (struct signature_v2_hdr *)sig;
+	struct ima_file_id file_id = {
+		.hash_type = hdr->type,
+		.hash_algorithm = algo,
+	};
+	size_t digest_size = hash_digest_size[algo];
+	struct public_key_signature pks = {
+		.m = (u8 *)&file_id,
+		.m_size = sizeof(file_id) - (HASH_MAX_DIGESTSIZE - digest_size),
+		.s = hdr->sig,
+		.s_size = siglen - sizeof(*hdr),
+		.pkey_algo = pk->pkey_algo,
+		.hash_algo = "none",
+		.encoding = encoding,
+	};
+	int ret;
+
+	if (hdr->type != IMA_VERITY_DIGSIG &&
+	    hdr->type != EVM_IMA_XATTR_DIGSIG &&
+	    hdr->type != EVM_XATTR_PORTABLE_DIGSIG)
+		return -EINVAL;
+
+	if (pks.s_size != be16_to_cpu(hdr->sig_size))
+		return -EBADMSG;
+
+	memcpy(file_id.hash, digest, digest_size);
+
+	ret = verify_signature(key, &pks);
+	pr_debug("%s() = %d\n", __func__, ret);
+	return ret;
+}
+
 int asymmetric_verify_v3(struct key *keyring, const char *sig, int siglen,
 			 const char *data, int datalen, u8 algo)
 {
 	struct signature_v2_hdr *hdr = (struct signature_v2_hdr *)sig;
 	struct ima_max_digest_data hash;
+	const struct public_key *pk;
+	struct key *key;
 	int rc;
 
-	rc = calc_file_id_hash(hdr->type, algo, data, &hash);
-	if (rc)
-		return -EINVAL;
+	if (algo >= HASH_ALGO__LAST)
+		return -ENOPKG;
+
+	if (siglen <= sizeof(*hdr))
+		return -EBADMSG;
+
+	key = request_asymmetric_key(keyring, be32_to_cpu(hdr->keyid));
+	if (IS_ERR(key))
+		return PTR_ERR(key);
+
+	pk = asymmetric_key_public_key(key);
+	if (!pk) {
+		rc = -ENOKEY;
+		goto out;
+	}
+	if (!strncmp(pk->pkey_algo, "mldsa", 5)) {
+		rc = asymmetric_verify_v3_hashless(key, pk, "raw",
+						   sig, siglen, algo, data);
+	} else {
+		rc = calc_file_id_hash(hdr->type, algo, data, &hash);
+		if (rc) {
+			rc = -EINVAL;
+			goto out;
+		}
+
+		rc = asymmetric_verify_common(key, pk, sig, siglen, hash.digest,
+					      hash.hdr.length);
+	}
 
-	return asymmetric_verify(keyring, sig, siglen, hash.digest,
-				 hash.hdr.length);
+out:
+	key_put(key);
+
+	return rc;
 }
diff --git a/security/integrity/evm/evm_secfs.c b/security/integrity/evm/evm_secfs.c
index acd840461902f..4baf5e23bc97e 100644
--- a/security/integrity/evm/evm_secfs.c
+++ b/security/integrity/evm/evm_secfs.c
@@ -127,8 +127,8 @@ static ssize_t evm_read_xattrs(struct file *filp, char __user *buf,
 			       size_t count, loff_t *ppos)
 {
 	char *temp;
-	int offset = 0;
-	ssize_t rc, size = 0;
+	size_t offset = 0, size = 0;
+	ssize_t rc;
 	struct xattr_list *xattr;
 
 	if (*ppos != 0)
@@ -151,16 +151,22 @@ static ssize_t evm_read_xattrs(struct file *filp, char __user *buf,
 		return -ENOMEM;
 	}
 
+	temp[size] = '\0';
+
+	/*
+	 * No truncation possible: size is computed over the same enabled
+	 * xattrs under xattr_list_mutex, so offset never exceeds size.
+	 */
 	list_for_each_entry(xattr, &evm_config_xattrnames, list) {
 		if (!xattr->enabled)
 			continue;
 
-		sprintf(temp + offset, "%s\n", xattr->name);
-		offset += strlen(xattr->name) + 1;
+		offset += snprintf(temp + offset, size + 1 - offset, "%s\n",
+				   xattr->name);
 	}
 
 	mutex_unlock(&xattr_list_mutex);
-	rc = simple_read_from_buffer(buf, count, ppos, temp, strlen(temp));
+	rc = simple_read_from_buffer(buf, count, ppos, temp, offset);
 
 	kfree(temp);
 
diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c
index de963b9f36344..18d0d9154317b 100644
--- a/security/integrity/ima/ima_appraise.c
+++ b/security/integrity/ima/ima_appraise.c
@@ -90,6 +90,11 @@ static int ima_fix_xattr(struct dentry *dentry, struct ima_iint_cache *iint)
 	int rc, offset;
 	u8 algo = iint->ima_hash->algo;
 
+	if (IS_RDONLY(d_inode(dentry)))
+		return -EROFS;
+	if (IS_IMMUTABLE(d_inode(dentry)))
+		return -EPERM;
+
 	if (algo <= HASH_ALGO_SHA1) {
 		offset = 1;
 		iint->ima_hash->xattr.sha1.type = IMA_XATTR_DIGEST;
@@ -195,8 +200,9 @@ enum hash_algo ima_get_hash_algo(const struct evm_ima_xattr_data *xattr_value,
 		return sig->hash_algo;
 	case EVM_IMA_XATTR_DIGSIG:
 		sig = (typeof(sig))xattr_value;
-		if (sig->version != 2 || xattr_len <= sizeof(*sig)
-		    || sig->hash_algo >= HASH_ALGO__LAST)
+		if ((sig->version != 2 && sig->version != 3) ||
+		    xattr_len <= sizeof(*sig) ||
+		    sig->hash_algo >= HASH_ALGO__LAST)
 			return ima_hash_algo;
 		return sig->hash_algo;
 	case IMA_XATTR_DIGEST_NG:
diff --git a/security/integrity/ima/ima_policy.c b/security/integrity/ima/ima_policy.c
index f7f940a769223..b1c010e8eb138 100644
--- a/security/integrity/ima/ima_policy.c
+++ b/security/integrity/ima/ima_policy.c
@@ -1313,7 +1313,8 @@ static bool ima_validate_rule(struct ima_rule_entry *entry)
 				     IMA_GID | IMA_EGID |
 				     IMA_FGROUP | IMA_DIGSIG_REQUIRED |
 				     IMA_PERMIT_DIRECTIO | IMA_MODSIG_ALLOWED |
-				     IMA_CHECK_BLACKLIST | IMA_VALIDATE_ALGOS))
+				     IMA_CHECK_BLACKLIST | IMA_VALIDATE_ALGOS |
+				     IMA_SIGV3_REQUIRED))
 			return false;
 
 		break;
diff --git a/security/ipe/Kconfig b/security/ipe/Kconfig
index a110a6cd848b7..95775139612df 100644
--- a/security/ipe/Kconfig
+++ b/security/ipe/Kconfig
@@ -95,6 +95,21 @@ config IPE_PROP_FS_VERITY_BUILTIN_SIG
 
 	  if unsure, answer Y.
 
+config IPE_PROP_BPF_SIGNATURE
+	bool "Enable support for Hornet BPF program signature verification"
+	depends on SECURITY_HORNET
+	help
+	  This option enables the 'bpf_signature', 'bpf_kernel' and
+	  'bpf_keyring' properties within IPE policies. The
+	  'bpf_signature' property allows IPE to make policy decisions
+	  based on the integrity verdict provided by the Hornet LSM
+	  when a BPF program is loaded.  Verdicts include OK,
+	  UNSIGNED, PARTIALSIG, BADSIG, and others.  The 'bpf_keyring'
+	  property allows policies to match against the keyring
+	  specified in bpf_attr (BUILTIN, SECONDARY, PLATFORM).
+
+	  If unsure, answer Y.
+
 endmenu
 
 config SECURITY_IPE_KUNIT_TEST
diff --git a/security/ipe/audit.c b/security/ipe/audit.c
index 93fb59fbddd60..77bbf04d950bd 100644
--- a/security/ipe/audit.c
+++ b/security/ipe/audit.c
@@ -41,6 +41,7 @@ static const char *const audit_op_names[__IPE_OP_MAX + 1] = {
 	"KEXEC_INITRAMFS",
 	"POLICY",
 	"X509_CERT",
+	"BPF_PROG_LOAD",
 	"UNKNOWN",
 };
 
@@ -51,6 +52,7 @@ static const char *const audit_hook_names[__IPE_HOOK_MAX] = {
 	"MPROTECT",
 	"KERNEL_READ",
 	"KERNEL_LOAD",
+	"BPF_PROG_LOAD",
 };
 
 static const char *const audit_prop_names[__IPE_PROP_MAX] = {
@@ -62,6 +64,19 @@ static const char *const audit_prop_names[__IPE_PROP_MAX] = {
 	"fsverity_digest=",
 	"fsverity_signature=FALSE",
 	"fsverity_signature=TRUE",
+	"bpf_signature=NONE",
+	"bpf_signature=OK",
+	"bpf_signature=UNSIGNED",
+	"bpf_signature=PARTIALSIG",
+	"bpf_signature=UNKNOWNKEY",
+	"bpf_signature=UNEXPECTED",
+	"bpf_signature=FAULT",
+	"bpf_signature=BADSIG",
+	"bpf_keyring=BUILTIN",
+	"bpf_keyring=SECONDARY",
+	"bpf_keyring=PLATFORM",
+	"bpf_kernel=FALSE",
+	"bpf_kernel=TRUE",
 };
 
 /**
diff --git a/security/ipe/eval.c b/security/ipe/eval.c
index 21439c5be3364..23ae1edf896b0 100644
--- a/security/ipe/eval.c
+++ b/security/ipe/eval.c
@@ -11,6 +11,7 @@
 #include <linux/rcupdate.h>
 #include <linux/moduleparam.h>
 #include <linux/fsverity.h>
+#include <linux/verification.h>
 
 #include "ipe.h"
 #include "eval.h"
@@ -265,6 +266,72 @@ static bool evaluate_fsv_sig_true(const struct ipe_eval_ctx *const ctx)
 }
 #endif /* CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG */
 
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+/**
+ * evaluate_bpf_sig() - Evaluate @ctx against a bpf_signature property.
+ * @ctx: Supplies a pointer to the context being evaluated.
+ * @expected: The expected lsm_integrity_verdict to match against.
+ *
+ * Return:
+ * * %true	- The current @ctx matches the expected verdict
+ * * %false	- The current @ctx doesn't match the expected verdict
+ */
+static bool evaluate_bpf_sig(const struct ipe_eval_ctx *const ctx,
+			     enum lsm_integrity_verdict expected)
+{
+	return ctx->bpf_verdict == expected;
+}
+#else
+static bool evaluate_bpf_sig(const struct ipe_eval_ctx *const ctx,
+			     enum lsm_integrity_verdict expected)
+{
+	return false;
+}
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
+
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+/**
+ * evaluate_bpf_keyring() - Evaluate @ctx against a bpf_keyring property.
+ * @ctx: Supplies a pointer to the context being evaluated.
+ * @expected: The expected keyring_id to match against.
+ *
+ * Return:
+ * * %true	- The current @ctx matches the expected keyring
+ * * %false	- The current @ctx doesn't match the expected keyring
+ */
+static bool evaluate_bpf_keyring(const struct ipe_eval_ctx *const ctx,
+				 s32 expected)
+{
+	return ctx->bpf_keyring_id == expected;
+}
+#else
+static bool evaluate_bpf_keyring(const struct ipe_eval_ctx *const ctx,
+				 s32 expected)
+{
+	return false;
+}
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
+
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+/**
+ * evaluate_bpf_kernel() - Evaluate @ctx against the bpf_kernel property.
+ * @ctx: Supplies a pointer to the context being evaluated.
+ *
+ * Return:
+ * * %true	- The current @ctx bpf_kernel property is true
+ * * %false	- The current @ctx bpf_kernel property is false
+ */
+static bool evaluate_bpf_kernel(const struct ipe_eval_ctx *const ctx)
+{
+	return ctx->bpf_kernel;
+}
+#else
+static bool evaluate_bpf_kernel(const struct ipe_eval_ctx *const ctx)
+{
+	return false;
+}
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
+
 /**
  * evaluate_property() - Analyze @ctx against a rule property.
  * @ctx: Supplies a pointer to the context to be evaluated.
@@ -297,6 +364,32 @@ static bool evaluate_property(const struct ipe_eval_ctx *const ctx,
 		return evaluate_fsv_sig_false(ctx);
 	case IPE_PROP_FSV_SIG_TRUE:
 		return evaluate_fsv_sig_true(ctx);
+	case IPE_PROP_BPF_SIG_NONE:
+		return evaluate_bpf_sig(ctx, LSM_INT_VERDICT_NONE);
+	case IPE_PROP_BPF_SIG_OK:
+		return evaluate_bpf_sig(ctx, LSM_INT_VERDICT_OK);
+	case IPE_PROP_BPF_SIG_UNSIGNED:
+		return evaluate_bpf_sig(ctx, LSM_INT_VERDICT_UNSIGNED);
+	case IPE_PROP_BPF_SIG_PARTIALSIG:
+		return evaluate_bpf_sig(ctx, LSM_INT_VERDICT_PARTIALSIG);
+	case IPE_PROP_BPF_SIG_UNKNOWNKEY:
+		return evaluate_bpf_sig(ctx, LSM_INT_VERDICT_UNKNOWNKEY);
+	case IPE_PROP_BPF_SIG_UNEXPECTED:
+		return evaluate_bpf_sig(ctx, LSM_INT_VERDICT_UNEXPECTED);
+	case IPE_PROP_BPF_SIG_FAULT:
+		return evaluate_bpf_sig(ctx, LSM_INT_VERDICT_FAULT);
+	case IPE_PROP_BPF_SIG_BADSIG:
+		return evaluate_bpf_sig(ctx, LSM_INT_VERDICT_BADSIG);
+	case IPE_PROP_BPF_KEYRING_BUILTIN:
+		return evaluate_bpf_keyring(ctx, 0);
+	case IPE_PROP_BPF_KEYRING_SECONDARY:
+		return evaluate_bpf_keyring(ctx, (s32)(unsigned long)VERIFY_USE_SECONDARY_KEYRING);
+	case IPE_PROP_BPF_KEYRING_PLATFORM:
+		return evaluate_bpf_keyring(ctx, (s32)(unsigned long)VERIFY_USE_PLATFORM_KEYRING);
+	case IPE_PROP_BPF_KERNEL_FALSE:
+		return !evaluate_bpf_kernel(ctx);
+	case IPE_PROP_BPF_KERNEL_TRUE:
+		return evaluate_bpf_kernel(ctx);
 	default:
 		return false;
 	}
diff --git a/security/ipe/eval.h b/security/ipe/eval.h
index fef65a36468cb..b061cb5ade27e 100644
--- a/security/ipe/eval.h
+++ b/security/ipe/eval.h
@@ -37,6 +37,12 @@ struct ipe_inode {
 };
 #endif /* CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG */
 
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+struct ipe_bpf_prog {
+	enum lsm_integrity_verdict verdict;
+};
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
+
 struct ipe_eval_ctx {
 	enum ipe_op_type op;
 	enum ipe_hook_type hook;
@@ -52,6 +58,11 @@ struct ipe_eval_ctx {
 #ifdef CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG
 	const struct ipe_inode *ipe_inode;
 #endif /* CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG */
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+	enum lsm_integrity_verdict bpf_verdict;
+	s32 bpf_keyring_id;
+	bool bpf_kernel;
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
 };
 
 enum ipe_match {
diff --git a/security/ipe/hooks.c b/security/ipe/hooks.c
index 0ae54a880405a..9271e129a2cf2 100644
--- a/security/ipe/hooks.c
+++ b/security/ipe/hooks.c
@@ -340,3 +340,66 @@ int ipe_inode_setintegrity(const struct inode *inode,
 	return -EINVAL;
 }
 #endif /* CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG */
+
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+/**
+ * ipe_bpf_prog_load_post_integrity() - Store integrity verdict in per-prog blob.
+ * @prog: Supplies the BPF program being loaded.
+ * @attr: Supplies the bpf syscall attributes.
+ * @token: Supplies the BPF token, if any.
+ * @kernel: Whether the call originated from the kernel.
+ * @lsmid: Supplies the LSM ID of the integrity provider.
+ * @verdict: Supplies the integrity verdict from the provider (e.g. Hornet).
+ *
+ * This hook stores the integrity verdict in IPE's per-prog security blob
+ * so that ipe_bpf_prog_load() can later read it for policy evaluation.
+ *
+ * Return:
+ * * %0		- Always succeeds (policy is evaluated in bpf_prog_load)
+ */
+int ipe_bpf_prog_load_post_integrity(struct bpf_prog *prog,
+				     union bpf_attr *attr,
+				     struct bpf_token *token,
+				     bool kernel,
+				     const struct lsm_id *lsmid,
+				     enum lsm_integrity_verdict verdict)
+{
+	struct ipe_bpf_prog *blob = ipe_bpf_prog(prog);
+
+	blob->verdict = verdict;
+
+	return 0;
+}
+
+/**
+ * ipe_bpf_prog_load() - IPE policy evaluation for BPF program load.
+ * @prog: Supplies the BPF program being loaded.
+ * @attr: Supplies the bpf syscall attributes.
+ * @token: Supplies the BPF token, if any.
+ * @kernel: Whether the call originated from the kernel.
+ *
+ * Reads the integrity verdict previously stored by post_integrity (if any)
+ * and evaluates IPE policy. If no integrity provider ran, the verdict
+ * defaults to LSM_INT_VERDICT_NONE.
+ *
+ * Return:
+ * * %0		- Success
+ * * %-EACCES	- Did not pass IPE policy
+ */
+int ipe_bpf_prog_load(struct bpf_prog *prog,
+		      union bpf_attr *attr,
+		      struct bpf_token *token,
+		      bool kernel)
+{
+	struct ipe_bpf_prog *blob = ipe_bpf_prog(prog);
+	struct ipe_eval_ctx ctx = IPE_EVAL_CTX_INIT;
+
+	ctx.op = IPE_OP_BPF_PROG_LOAD;
+	ctx.hook = IPE_HOOK_BPF_PROG_LOAD;
+	ctx.bpf_verdict = blob->verdict;
+	ctx.bpf_keyring_id = attr->keyring_id;
+	ctx.bpf_kernel = kernel;
+
+	return ipe_evaluate_event(&ctx);
+}
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
diff --git a/security/ipe/hooks.h b/security/ipe/hooks.h
index 07db373327402..8a6d1a459e00c 100644
--- a/security/ipe/hooks.h
+++ b/security/ipe/hooks.h
@@ -10,6 +10,7 @@
 #include <linux/security.h>
 #include <linux/blk_types.h>
 #include <linux/fsverity.h>
+#include <linux/bpf.h>
 
 enum ipe_hook_type {
 	IPE_HOOK_BPRM_CHECK = 0,
@@ -18,6 +19,7 @@ enum ipe_hook_type {
 	IPE_HOOK_MPROTECT,
 	IPE_HOOK_KERNEL_READ,
 	IPE_HOOK_KERNEL_LOAD,
+	IPE_HOOK_BPF_PROG_LOAD,
 	__IPE_HOOK_MAX
 };
 
@@ -52,4 +54,17 @@ int ipe_inode_setintegrity(const struct inode *inode, enum lsm_integrity_type ty
 			   const void *value, size_t size);
 #endif /* CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG */
 
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+int ipe_bpf_prog_load_post_integrity(struct bpf_prog *prog,
+				     union bpf_attr *attr,
+				     struct bpf_token *token,
+				     bool kernel,
+				     const struct lsm_id *lsmid,
+				     enum lsm_integrity_verdict verdict);
+int ipe_bpf_prog_load(struct bpf_prog *prog,
+		      union bpf_attr *attr,
+		      struct bpf_token *token,
+		      bool kernel);
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
+
 #endif /* _IPE_HOOKS_H */
diff --git a/security/ipe/ipe.c b/security/ipe/ipe.c
index 495bb765de1b8..5af13903287fe 100644
--- a/security/ipe/ipe.c
+++ b/security/ipe/ipe.c
@@ -19,6 +19,9 @@ static struct lsm_blob_sizes ipe_blobs __ro_after_init = {
 #ifdef CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG
 	.lbs_inode = sizeof(struct ipe_inode),
 #endif /* CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG */
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+	.lbs_bpf_prog = sizeof(struct ipe_bpf_prog),
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
 };
 
 static const struct lsm_id ipe_lsmid = {
@@ -45,6 +48,13 @@ struct ipe_inode *ipe_inode(const struct inode *inode)
 }
 #endif /* CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG */
 
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+struct ipe_bpf_prog *ipe_bpf_prog(const struct bpf_prog *prog)
+{
+	return prog->aux->security + ipe_blobs.lbs_bpf_prog;
+}
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
+
 static struct security_hook_list ipe_hooks[] __ro_after_init = {
 	LSM_HOOK_INIT(bprm_check_security, ipe_bprm_check_security),
 	LSM_HOOK_INIT(bprm_creds_for_exec, ipe_bprm_creds_for_exec),
@@ -60,6 +70,10 @@ static struct security_hook_list ipe_hooks[] __ro_after_init = {
 #ifdef CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG
 	LSM_HOOK_INIT(inode_setintegrity, ipe_inode_setintegrity),
 #endif /* CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG */
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+	LSM_HOOK_INIT(bpf_prog_load_post_integrity, ipe_bpf_prog_load_post_integrity),
+	LSM_HOOK_INIT(bpf_prog_load, ipe_bpf_prog_load),
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
 };
 
 /**
diff --git a/security/ipe/ipe.h b/security/ipe/ipe.h
index 25cfdb8f0c20a..47de32b5bc938 100644
--- a/security/ipe/ipe.h
+++ b/security/ipe/ipe.h
@@ -22,6 +22,9 @@ struct ipe_bdev *ipe_bdev(struct block_device *b);
 #ifdef CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG
 struct ipe_inode *ipe_inode(const struct inode *inode);
 #endif /* CONFIG_IPE_PROP_FS_VERITY_BUILTIN_SIG */
+#ifdef CONFIG_IPE_PROP_BPF_SIGNATURE
+struct ipe_bpf_prog *ipe_bpf_prog(const struct bpf_prog *prog);
+#endif /* CONFIG_IPE_PROP_BPF_SIGNATURE */
 
 int ipe_init_securityfs(void);
 
diff --git a/security/ipe/policy.h b/security/ipe/policy.h
index 5bfbdbddeef86..748bea92beb19 100644
--- a/security/ipe/policy.h
+++ b/security/ipe/policy.h
@@ -17,6 +17,7 @@ enum ipe_op_type {
 	IPE_OP_KEXEC_INITRAMFS,
 	IPE_OP_POLICY,
 	IPE_OP_X509,
+	IPE_OP_BPF_PROG_LOAD,
 	__IPE_OP_MAX,
 };
 
@@ -39,6 +40,19 @@ enum ipe_prop_type {
 	IPE_PROP_FSV_DIGEST,
 	IPE_PROP_FSV_SIG_FALSE,
 	IPE_PROP_FSV_SIG_TRUE,
+	IPE_PROP_BPF_SIG_NONE,
+	IPE_PROP_BPF_SIG_OK,
+	IPE_PROP_BPF_SIG_UNSIGNED,
+	IPE_PROP_BPF_SIG_PARTIALSIG,
+	IPE_PROP_BPF_SIG_UNKNOWNKEY,
+	IPE_PROP_BPF_SIG_UNEXPECTED,
+	IPE_PROP_BPF_SIG_FAULT,
+	IPE_PROP_BPF_SIG_BADSIG,
+	IPE_PROP_BPF_KEYRING_BUILTIN,
+	IPE_PROP_BPF_KEYRING_SECONDARY,
+	IPE_PROP_BPF_KEYRING_PLATFORM,
+	IPE_PROP_BPF_KERNEL_FALSE,
+	IPE_PROP_BPF_KERNEL_TRUE,
 	__IPE_PROP_MAX
 };
 
diff --git a/security/ipe/policy_parser.c b/security/ipe/policy_parser.c
index 6fa5bebf84714..71f63de56616b 100644
--- a/security/ipe/policy_parser.c
+++ b/security/ipe/policy_parser.c
@@ -237,6 +237,7 @@ static const match_table_t operation_tokens = {
 	{IPE_OP_KEXEC_INITRAMFS,	"op=KEXEC_INITRAMFS"},
 	{IPE_OP_POLICY,			"op=POLICY"},
 	{IPE_OP_X509,			"op=X509_CERT"},
+	{IPE_OP_BPF_PROG_LOAD,		"op=BPF_PROG_LOAD"},
 	{IPE_OP_INVALID,		NULL}
 };
 
@@ -281,6 +282,19 @@ static const match_table_t property_tokens = {
 	{IPE_PROP_FSV_DIGEST,		"fsverity_digest=%s"},
 	{IPE_PROP_FSV_SIG_FALSE,	"fsverity_signature=FALSE"},
 	{IPE_PROP_FSV_SIG_TRUE,		"fsverity_signature=TRUE"},
+	{IPE_PROP_BPF_SIG_NONE,		"bpf_signature=NONE"},
+	{IPE_PROP_BPF_SIG_OK,		"bpf_signature=OK"},
+	{IPE_PROP_BPF_SIG_UNSIGNED,	"bpf_signature=UNSIGNED"},
+	{IPE_PROP_BPF_SIG_PARTIALSIG,	"bpf_signature=PARTIALSIG"},
+	{IPE_PROP_BPF_SIG_UNKNOWNKEY,	"bpf_signature=UNKNOWNKEY"},
+	{IPE_PROP_BPF_SIG_UNEXPECTED,	"bpf_signature=UNEXPECTED"},
+	{IPE_PROP_BPF_SIG_FAULT,	"bpf_signature=FAULT"},
+	{IPE_PROP_BPF_SIG_BADSIG,	"bpf_signature=BADSIG"},
+	{IPE_PROP_BPF_KEYRING_BUILTIN,	"bpf_keyring=BUILTIN"},
+	{IPE_PROP_BPF_KEYRING_SECONDARY,	"bpf_keyring=SECONDARY"},
+	{IPE_PROP_BPF_KEYRING_PLATFORM,	"bpf_keyring=PLATFORM"},
+	{IPE_PROP_BPF_KERNEL_FALSE,	"bpf_kernel=FALSE"},
+	{IPE_PROP_BPF_KERNEL_TRUE,	"bpf_kernel=TRUE"},
 	{IPE_PROP_INVALID,		NULL}
 };
 
@@ -331,6 +345,19 @@ static int parse_property(char *t, struct ipe_rule *r)
 	case IPE_PROP_DMV_SIG_TRUE:
 	case IPE_PROP_FSV_SIG_FALSE:
 	case IPE_PROP_FSV_SIG_TRUE:
+	case IPE_PROP_BPF_SIG_NONE:
+	case IPE_PROP_BPF_SIG_OK:
+	case IPE_PROP_BPF_SIG_UNSIGNED:
+	case IPE_PROP_BPF_SIG_PARTIALSIG:
+	case IPE_PROP_BPF_SIG_UNKNOWNKEY:
+	case IPE_PROP_BPF_SIG_UNEXPECTED:
+	case IPE_PROP_BPF_SIG_FAULT:
+	case IPE_PROP_BPF_SIG_BADSIG:
+	case IPE_PROP_BPF_KEYRING_BUILTIN:
+	case IPE_PROP_BPF_KEYRING_SECONDARY:
+	case IPE_PROP_BPF_KEYRING_PLATFORM:
+	case IPE_PROP_BPF_KERNEL_FALSE:
+	case IPE_PROP_BPF_KERNEL_TRUE:
 		p->type = token;
 		break;
 	default:
diff --git a/security/keys/Kconfig b/security/keys/Kconfig
index 84f39e50ca36a..f4510d8cb485f 100644
--- a/security/keys/Kconfig
+++ b/security/keys/Kconfig
@@ -87,7 +87,6 @@ config ENCRYPTED_KEYS
 	select CRYPTO_AES
 	select CRYPTO_CBC
 	select CRYPTO_LIB_SHA256
-	select CRYPTO_RNG
 	help
 	  This option provides support for create/encrypting/decrypting keys
 	  in the kernel.  Encrypted keys are instantiated using kernel
diff --git a/security/keys/trusted-keys/Kconfig b/security/keys/trusted-keys/Kconfig
index 9e00482d886a5..e5a4a53aeab2f 100644
--- a/security/keys/trusted-keys/Kconfig
+++ b/security/keys/trusted-keys/Kconfig
@@ -1,10 +1,29 @@
 config HAVE_TRUSTED_KEYS
 	bool
 
+config HAVE_TRUSTED_KEYS_DEBUG
+	bool
+
+config TRUSTED_KEYS_DEBUG
+	bool "Debug trusted keys"
+	depends on HAVE_TRUSTED_KEYS_DEBUG
+	default n
+	help
+	  Trusted key backends and core code that support debug traces can
+	  opt-in that feature here. Traces must only use debug level output, as
+	  sensitive data may pass by. In the kernel-command line traces can be
+	  enabled via trusted.dyndbg='+p'.
+
+	  SAFETY: Debug dumps are inactive at runtime until trusted.debug is set
+	  to a true value on the kernel command-line. Use at your utmost
+	  consideration when enabling this feature on a production build. The
+	  general advice is not to do this.
+
 config TRUSTED_KEYS_TPM
 	bool "TPM-based trusted keys"
 	depends on TCG_TPM >= TRUSTED_KEYS
 	default y
+	select HAVE_TRUSTED_KEYS_DEBUG
 	select CRYPTO_HASH_INFO
 	select CRYPTO_LIB_SHA1
 	select CRYPTO_LIB_UTILS
@@ -23,6 +42,7 @@ config TRUSTED_KEYS_TEE
 	bool "TEE-based trusted keys"
 	depends on TEE >= TRUSTED_KEYS
 	default y
+	select HAVE_TRUSTED_KEYS_DEBUG
 	select HAVE_TRUSTED_KEYS
 	help
 	  Enable use of the Trusted Execution Environment (TEE) as trusted
@@ -33,6 +53,7 @@ config TRUSTED_KEYS_CAAM
 	depends on CRYPTO_DEV_FSL_CAAM_JR >= TRUSTED_KEYS
 	select CRYPTO_DEV_FSL_CAAM_BLOB_GEN
 	default y
+	select HAVE_TRUSTED_KEYS_DEBUG
 	select HAVE_TRUSTED_KEYS
 	help
 	  Enable use of NXP's Cryptographic Accelerator and Assurance Module
@@ -42,6 +63,7 @@ config TRUSTED_KEYS_DCP
 	bool "DCP-based trusted keys"
 	depends on CRYPTO_DEV_MXS_DCP >= TRUSTED_KEYS
 	default y
+	select HAVE_TRUSTED_KEYS_DEBUG
 	select HAVE_TRUSTED_KEYS
 	help
 	  Enable use of NXP's DCP (Data Co-Processor) as trusted key backend.
@@ -50,6 +72,7 @@ config TRUSTED_KEYS_PKWM
 	bool "PKWM-based trusted keys"
 	depends on PSERIES_PLPKS >= TRUSTED_KEYS
 	default y
+	select HAVE_TRUSTED_KEYS_DEBUG
 	select HAVE_TRUSTED_KEYS
 	help
 	  Enable use of IBM PowerVM Key Wrapping Module (PKWM) as a trusted key backend.
diff --git a/security/keys/trusted-keys/trusted_caam.c b/security/keys/trusted-keys/trusted_caam.c
index 601943ce0d60f..6a33dbf2a7f55 100644
--- a/security/keys/trusted-keys/trusted_caam.c
+++ b/security/keys/trusted-keys/trusted_caam.c
@@ -28,10 +28,13 @@ static const match_table_t key_tokens = {
 	{opt_err, NULL}
 };
 
-#ifdef CAAM_DEBUG
+#ifdef CONFIG_TRUSTED_KEYS_DEBUG
 static inline void dump_options(const struct caam_pkey_info *pkey_info)
 {
-	pr_info("key encryption algo %d\n", pkey_info->key_enc_algo);
+	if (!trusted_debug)
+		return;
+
+	pr_debug("key encryption algo %d\n", pkey_info->key_enc_algo);
 }
 #else
 static inline void dump_options(const struct caam_pkey_info *pkey_info)
diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c
index 0b142d941cd2e..6aed17bee09d8 100644
--- a/security/keys/trusted-keys/trusted_core.c
+++ b/security/keys/trusted-keys/trusted_core.c
@@ -31,6 +31,12 @@ static char *trusted_rng = "default";
 module_param_named(rng, trusted_rng, charp, 0);
 MODULE_PARM_DESC(rng, "Select trusted key RNG");
 
+#ifdef CONFIG_TRUSTED_KEYS_DEBUG
+bool trusted_debug;
+module_param_named(debug, trusted_debug, bool, 0);
+MODULE_PARM_DESC(debug, "Enable trusted keys debug traces (default: 0)");
+#endif
+
 static char *trusted_key_source;
 module_param_named(source, trusted_key_source, charp, 0);
 MODULE_PARM_DESC(source, "Select trusted keys source (tpm, tee, caam, dcp or pkwm)");
diff --git a/security/keys/trusted-keys/trusted_tpm1.c b/security/keys/trusted-keys/trusted_tpm1.c
index 6ea728f1eae6f..13513819991e7 100644
--- a/security/keys/trusted-keys/trusted_tpm1.c
+++ b/security/keys/trusted-keys/trusted_tpm1.c
@@ -46,38 +46,44 @@ enum {
 	SRK_keytype = 4
 };
 
-#define TPM_DEBUG 0
-
-#if TPM_DEBUG
+#ifdef CONFIG_TRUSTED_KEYS_DEBUG
 static inline void dump_options(struct trusted_key_options *o)
 {
-	pr_info("sealing key type %d\n", o->keytype);
-	pr_info("sealing key handle %0X\n", o->keyhandle);
-	pr_info("pcrlock %d\n", o->pcrlock);
-	pr_info("pcrinfo %d\n", o->pcrinfo_len);
-	print_hex_dump(KERN_INFO, "pcrinfo ", DUMP_PREFIX_NONE,
-		       16, 1, o->pcrinfo, o->pcrinfo_len, 0);
+	if (!trusted_debug)
+		return;
+
+	pr_debug("sealing key type %d\n", o->keytype);
+	pr_debug("sealing key handle %0X\n", o->keyhandle);
+	pr_debug("pcrlock %d\n", o->pcrlock);
+	pr_debug("pcrinfo %d\n", o->pcrinfo_len);
+	print_hex_dump_debug("pcrinfo ", DUMP_PREFIX_NONE,
+			     16, 1, o->pcrinfo, o->pcrinfo_len, 0);
 }
 
 static inline void dump_sess(struct osapsess *s)
 {
-	print_hex_dump(KERN_INFO, "trusted-key: handle ", DUMP_PREFIX_NONE,
-		       16, 1, &s->handle, 4, 0);
-	pr_info("secret:\n");
-	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_NONE,
-		       16, 1, &s->secret, SHA1_DIGEST_SIZE, 0);
-	pr_info("trusted-key: enonce:\n");
-	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_NONE,
-		       16, 1, &s->enonce, SHA1_DIGEST_SIZE, 0);
+	if (!trusted_debug)
+		return;
+
+	print_hex_dump_debug("trusted-key: handle ", DUMP_PREFIX_NONE,
+			     16, 1, &s->handle, 4, 0);
+	pr_debug("secret:\n");
+	print_hex_dump_debug("", DUMP_PREFIX_NONE,
+			     16, 1, &s->secret, SHA1_DIGEST_SIZE, 0);
+	pr_debug("trusted-key: enonce:\n");
+	print_hex_dump_debug("", DUMP_PREFIX_NONE,
+			     16, 1, &s->enonce, SHA1_DIGEST_SIZE, 0);
 }
 
 static inline void dump_tpm_buf(unsigned char *buf)
 {
 	int len;
 
-	pr_info("\ntpm buffer\n");
+	if (!trusted_debug)
+		return;
+	pr_debug("\ntpm buffer\n");
 	len = LOAD32(buf, TPM_SIZE_OFFSET);
-	print_hex_dump(KERN_INFO, "", DUMP_PREFIX_NONE, 16, 1, buf, len, 0);
+	print_hex_dump_debug("", DUMP_PREFIX_NONE, 16, 1, buf, len, 0);
 }
 #else
 static inline void dump_options(struct trusted_key_options *o)
diff --git a/security/keys/user_defined.c b/security/keys/user_defined.c
index 686d56e4cc850..6f88b507f9279 100644
--- a/security/keys/user_defined.c
+++ b/security/keys/user_defined.c
@@ -64,7 +64,7 @@ int user_preparse(struct key_preparsed_payload *prep)
 	if (datalen == 0 || datalen > 32767 || !prep->data)
 		return -EINVAL;
 
-	upayload = kmalloc(sizeof(*upayload) + datalen, GFP_KERNEL);
+	upayload = kmalloc_flex(*upayload, data, datalen);
 	if (!upayload)
 		return -ENOMEM;
 
diff --git a/security/landlock/domain.c b/security/landlock/domain.c
index 06b6bd845060f..5dd06f7c2312f 100644
--- a/security/landlock/domain.c
+++ b/security/landlock/domain.c
@@ -90,11 +90,12 @@ static struct landlock_details *get_current_details(void)
 		return ERR_CAST(buffer);
 
 	/*
-	 * Create the new details according to the path's length.  Do not
-	 * allocate with GFP_KERNEL_ACCOUNT because it is independent from the
-	 * caller.
+	 * Create the new details according to the path's length.  Account
+	 * to the calling task's memcg, like the other Landlock per-domain
+	 * allocations, even if it may not control the related size.
 	 */
-	details = kzalloc_flex(*details, exe_path, path_size);
+	details =
+		kzalloc_flex(*details, exe_path, path_size, GFP_KERNEL_ACCOUNT);
 	if (!details)
 		return ERR_PTR(-ENOMEM);
 
diff --git a/security/landlock/domain.h b/security/landlock/domain.h
index a9d57db0120dc..35cac8f6daee7 100644
--- a/security/landlock/domain.h
+++ b/security/landlock/domain.h
@@ -33,10 +33,7 @@ enum landlock_log_status {
  * Rarely accessed, mainly when logging the first domain's denial.
  *
  * The contained pointers are initialized at the domain creation time and never
- * changed again.  Contrary to most other Landlock object types, this one is
- * not allocated with GFP_KERNEL_ACCOUNT because its size may not be under the
- * caller's control (e.g. unknown exe_path) and the data is not explicitly
- * requested nor used by tasks.
+ * changed again.
  */
 struct landlock_details {
 	/**
diff --git a/security/landlock/fs.c b/security/landlock/fs.c
index c1ecfe2390326..32d560f12dbd3 100644
--- a/security/landlock/fs.c
+++ b/security/landlock/fs.c
@@ -1278,7 +1278,7 @@ static void hook_sb_delete(struct super_block *const sb)
 		struct landlock_object *object;
 
 		/* Only handles referenced inodes. */
-		if (!icount_read(inode))
+		if (!icount_read_once(inode))
 			continue;
 
 		/*
diff --git a/security/lsm_syscalls.c b/security/lsm_syscalls.c
index 5648b1f0ce9c1..08a017669c025 100644
--- a/security/lsm_syscalls.c
+++ b/security/lsm_syscalls.c
@@ -57,7 +57,14 @@ u64 lsm_name_to_attr(const char *name)
 SYSCALL_DEFINE4(lsm_set_self_attr, unsigned int, attr, struct lsm_ctx __user *,
 		ctx, u32, size, u32, flags)
 {
-	return security_setselfattr(attr, ctx, size, flags);
+	int rc;
+
+	rc = mutex_lock_interruptible(&current->signal->cred_guard_mutex);
+	if (rc < 0)
+		return rc;
+	rc = security_setselfattr(attr, ctx, size, flags);
+	mutex_unlock(&current->signal->cred_guard_mutex);
+	return rc;
 }
 
 /**
diff --git a/security/security.c b/security/security.c
index 4e999f0236516..55ab49c84003b 100644
--- a/security/security.c
+++ b/security/security.c
@@ -2258,22 +2258,22 @@ int security_inode_setsecurity(struct inode *inode, const char *name,
 /**
  * security_inode_listsecurity() - List the xattr security label names
  * @inode: inode
- * @buffer: buffer
- * @buffer_size: size of buffer
+ * @buffer: pointer to buffer
+ * @remaining_size: pointer to remaining size of buffer
  *
  * Copy the extended attribute names for the security labels associated with
- * @inode into @buffer.  The maximum size of @buffer is specified by
- * @buffer_size.  @buffer may be NULL to request the size of the buffer
- * required.
+ * @inode into *(@buffer).  The remaining size of @buffer is specified by
+ * *(@remaining_size).  *(@buffer) may be NULL to request the size of the
+ * buffer required. Updates *(@buffer) and *(@remaining_size).
  *
- * Return: Returns number of bytes used/required on success.
+ * Return: Returns 0 on success, or -errno on failure.
  */
 int security_inode_listsecurity(struct inode *inode,
-				char *buffer, size_t buffer_size)
+				char **buffer, ssize_t *remaining_size)
 {
 	if (unlikely(IS_PRIVATE(inode)))
 		return 0;
-	return call_int_hook(inode_listsecurity, inode, buffer, buffer_size);
+	return call_int_hook(inode_listsecurity, inode, buffer, remaining_size);
 }
 EXPORT_SYMBOL(security_inode_listsecurity);
 
@@ -5355,6 +5355,50 @@ int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
 	return rc;
 }
 
+/**
+ * security_bpf_prog_load_post_integrity() - Check if the BPF prog is allowed
+ * @prog: BPF program object
+ * @attr: BPF syscall attributes used to create BPF program
+ * @token: BPF token used to grant user access to BPF subsystem
+ * @kernel: whether or not call originated from kernel
+ * @lsmid: LSM ID of the LSM providing @verdict
+ * @verdict: result of the integrity verification
+ *
+ * See the comment block for the security_bpf_prog_load() LSM hook.
+ *
+ * This LSM hook is intended to be called from within the
+ * bpf_prog_load_integrity() callback that is part of the
+ * security_bpf_prog_load() hook; kernel subsystems outside the scope of the
+ * LSM framework should not call this hook directly.
+ *
+ * If the LSM calling into this hook receives a non-zero error code, it should
+ * return the same error code back to its caller.  If this hook returns a zero,
+ * it does not necessarily mean that all of the enabled LSMs have authorized
+ * the BPF program load, as there may be other LSMs implementing BPF integrity
+ * checks which have yet to execute.  However, if a zero is returned, the LSM
+ * calling into this hook should continue and return zero back to its caller.
+ *
+ * LSMs which implement the bpf_prog_load_post_integrity() callback and
+ * determine that a particular BPF program load is not authorized may choose to
+ * either return an error code for immediate rejection, or store their decision
+ * in their own LSM state attached to @prog, later returning an error code in
+ * the bpf_prog_load() callback.  An immediate error code return is in keeping
+ * with the "fail fast" practice, but waiting until the bpf_prog_load()
+ * callback allows the LSM to consider multiple different integrity verdicts.
+ *
+ * Return: Returns 0 on success, error on failure.
+ */
+int security_bpf_prog_load_post_integrity(struct bpf_prog *prog,
+					  union bpf_attr *attr,
+					  struct bpf_token *token,
+					  bool kernel,
+					  const struct lsm_id *lsmid,
+					  enum lsm_integrity_verdict verdict)
+{
+	return call_int_hook(bpf_prog_load_post_integrity, prog, attr, token,
+			     kernel, lsmid, verdict);
+}
+
 /**
  * security_bpf_prog_load() - Check if loading of BPF program is allowed
  * @prog: BPF program object
@@ -5363,8 +5407,24 @@ int security_bpf_map_create(struct bpf_map *map, union bpf_attr *attr,
  * @kernel: whether or not call originated from kernel
  *
  * Perform an access control check when the kernel loads a BPF program and
- * allocates associated BPF program object. This hook is also responsible for
- * allocating any required LSM state for the BPF program.
+ * allocates the associated BPF program object. This hook is also responsible
+ * for allocating any required LSM state for the BPF program.
+ *
+ * This hook calls two LSM callbacks: bpf_prog_load_integrity() and
+ * bpf_prog_load().  The bpf_prog_load_integrity() callback is for those LSMs
+ * that wish to implement integrity verifications of BPF programs, e.g.
+ * signature verification, while the bpf_prog_load() callback is for general
+ * authorization of the BPF program load.  Performing both verification and
+ * authorization in a single callback, with arbitrary LSM ordering, would be
+ * a challenge.
+ *
+ * LSMs which implement the bpf_prog_load_integrity() callback should call into
+ * the security_bpf_prog_load_post_integrity() hook with their integrity
+ * verdict.  LSMs which implement BPF program integrity policy can register a
+ * callback for the security_bpf_prog_load_post_integrity() hook and
+ * either update their own internal state based on the verdict, or immediately
+ * reject the BPF program load with an error code.  See the comment block for
+ * security_bpf_prog_load_post_integrity() for more information.
  *
  * Return: Returns 0 on success, error on failure.
  */
@@ -5377,9 +5437,18 @@ int security_bpf_prog_load(struct bpf_prog *prog, union bpf_attr *attr,
 	if (unlikely(rc))
 		return rc;
 
+	rc = call_int_hook(bpf_prog_load_integrity, prog, attr, token, kernel);
+	if (unlikely(rc))
+		goto err;
+
 	rc = call_int_hook(bpf_prog_load, prog, attr, token, kernel);
 	if (unlikely(rc))
-		security_bpf_prog_free(prog);
+		goto err;
+
+	return rc;
+
+err:
+	security_bpf_prog_free(prog);
 	return rc;
 }
 
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index 0f704380a8c81..1a713d96206f5 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -3680,16 +3680,12 @@ static int selinux_inode_setsecurity(struct inode *inode, const char *name,
 	return 0;
 }
 
-static int selinux_inode_listsecurity(struct inode *inode, char *buffer, size_t buffer_size)
+static int selinux_inode_listsecurity(struct inode *inode, char **buffer,
+				ssize_t *remaining_size)
 {
-	const int len = sizeof(XATTR_NAME_SELINUX);
-
 	if (!selinux_initialized())
 		return 0;
-
-	if (buffer && len <= buffer_size)
-		memcpy(buffer, XATTR_NAME_SELINUX, len);
-	return len;
+	return xattr_list_one(buffer, remaining_size, XATTR_NAME_SELINUX);
 }
 
 static void selinux_inode_getlsmprop(struct inode *inode, struct lsm_prop *prop)
diff --git a/security/selinux/include/security.h b/security/selinux/include/security.h
index 0babb89921816..a74415e3afd3a 100644
--- a/security/selinux/include/security.h
+++ b/security/selinux/include/security.h
@@ -236,6 +236,7 @@ int security_read_policy(void **data, size_t *len);
 int security_read_state_kernel(void **data, size_t *len);
 int security_policycap_supported(unsigned int req_cap);
 
+/* Maximum supported number of permissions per class */
 #define SEL_VEC_MAX 32
 struct av_decision {
 	u32 allowed;
diff --git a/security/selinux/selinuxfs.c b/security/selinux/selinuxfs.c
index 25ca7d7140144..5f8f357fe7ca8 100644
--- a/security/selinux/selinuxfs.c
+++ b/security/selinux/selinuxfs.c
@@ -94,9 +94,8 @@ static int selinux_fs_info_create(struct super_block *sb)
 	return 0;
 }
 
-static void selinux_fs_info_free(struct super_block *sb)
+static void selinux_fs_info_free(struct selinux_fs_info *fsi)
 {
-	struct selinux_fs_info *fsi = sb->s_fs_info;
 	unsigned int i;
 
 	if (fsi) {
@@ -105,8 +104,7 @@ static void selinux_fs_info_free(struct super_block *sb)
 		kfree(fsi->bool_pending_names);
 		kfree(fsi->bool_pending_values);
 	}
-	kfree(sb->s_fs_info);
-	sb->s_fs_info = NULL;
+	kfree(fsi);
 }
 
 #define SEL_INITCON_INO_OFFSET		0x01000000
@@ -1959,8 +1957,10 @@ static int sel_init_fs_context(struct fs_context *fc)
 
 static void sel_kill_sb(struct super_block *sb)
 {
-	selinux_fs_info_free(sb);
+	struct selinux_fs_info *fsi = sb->s_fs_info;
+
 	kill_anon_super(sb);
+	selinux_fs_info_free(fsi);
 }
 
 static struct file_system_type sel_fs_type = {
@@ -1973,8 +1973,7 @@ struct path selinux_null __ro_after_init;
 
 int __init init_sel_fs(void)
 {
-	struct qstr null_name = QSTR_INIT(NULL_FILE_NAME,
-					  sizeof(NULL_FILE_NAME)-1);
+	struct qstr null_name = QSTR(NULL_FILE_NAME);
 	int err;
 
 	if (!selinux_enabled_boot)
diff --git a/security/selinux/ss/avtab.c b/security/selinux/ss/avtab.c
index d12ca337e6498..0f94edd01f696 100644
--- a/security/selinux/ss/avtab.c
+++ b/security/selinux/ss/avtab.c
@@ -316,7 +316,7 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po
 	struct avtab_extended_perms xperms;
 	__le32 buf32[ARRAY_SIZE(xperms.perms.p)];
 	int rc;
-	unsigned int set, vers = pol->policyvers;
+	unsigned int vers = pol->policyvers;
 
 	memset(&key, 0, sizeof(struct avtab_key));
 	memset(&datum, 0, sizeof(struct avtab_datum));
@@ -327,9 +327,12 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po
 			pr_err("SELinux: avtab: truncated entry\n");
 			return rc;
 		}
+		/* Read five or more items: source type, target type,
+		 * target class, AV type, and at least one datum.
+		 */
 		items2 = le32_to_cpu(buf32[0]);
-		if (items2 > ARRAY_SIZE(buf32)) {
-			pr_err("SELinux: avtab: entry overflow\n");
+		if (items2 < 5 || items2 > ARRAY_SIZE(buf32)) {
+			pr_err("SELinux: avtab: invalid item count\n");
 			return -EINVAL;
 		}
 		rc = next_entry(buf32, fp, sizeof(u32) * items2);
@@ -358,6 +361,13 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po
 			return -EINVAL;
 		}
 
+		if (!policydb_type_isvalid(pol, key.source_type) ||
+		    !policydb_type_isvalid(pol, key.target_type) ||
+		    !policydb_class_isvalid(pol, key.target_class)) {
+			pr_err("SELinux: avtab: invalid type or class\n");
+			return -EINVAL;
+		}
+
 		val = le32_to_cpu(buf32[items++]);
 		enabled = (val & AVTAB_ENABLED_OLD) ? AVTAB_ENABLED : 0;
 
@@ -376,8 +386,20 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po
 
 		for (i = 0; i < ARRAY_SIZE(spec_order); i++) {
 			if (val & spec_order[i]) {
+				if (items >= items2) {
+					pr_err("SELinux: avtab: entry has too many items (%d/%d)\n",
+					       items + 1, items2);
+					return -EINVAL;
+				}
 				key.specified = spec_order[i] | enabled;
 				datum.u.data = le32_to_cpu(buf32[items++]);
+
+				if ((key.specified & AVTAB_TYPE) &&
+				    !policydb_simpletype_isvalid(pol, datum.u.data)) {
+					pr_err("SELinux: avtab: invalid type\n");
+					return -EINVAL;
+				}
+
 				rc = insertf(a, &key, &datum, p);
 				if (rc)
 					return rc;
@@ -411,9 +433,13 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po
 		return -EINVAL;
 	}
 
-	set = hweight16(key.specified & (AVTAB_XPERMS | AVTAB_TYPE | AVTAB_AV));
-	if (!set || set > 1) {
-		pr_err("SELinux:  avtab:  more than one specifier\n");
+	if (hweight16(key.specified & ~AVTAB_ENABLED) != 1) {
+		pr_err("SELinux:  avtab:  not exactly one specifier\n");
+		return -EINVAL;
+	}
+
+	if (key.specified & ~AVTAB_SPECIFIER_MASK) {
+		pr_err("SELinux:  avtab:  invalid specifier\n");
 		return -EINVAL;
 	}
 
@@ -438,6 +464,10 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po
 			pr_err("SELinux: avtab: truncated entry\n");
 			return rc;
 		}
+		if (!avtab_is_valid_xperm_specified(xperms.specified))
+			pr_warn_once_policyload(pol,
+						"SELinux: avtab: unsupported xperm specifier %#x\n",
+						xperms.specified);
 		rc = next_entry(&xperms.driver, fp, sizeof(u8));
 		if (rc) {
 			pr_err("SELinux: avtab: truncated entry\n");
@@ -461,7 +491,7 @@ int avtab_read_item(struct avtab *a, struct policy_file *fp, struct policydb *po
 		datum.u.data = le32_to_cpu(*buf32);
 	}
 	if ((key.specified & AVTAB_TYPE) &&
-	    !policydb_type_isvalid(pol, datum.u.data)) {
+	    !policydb_simpletype_isvalid(pol, datum.u.data)) {
 		pr_err("SELinux: avtab: invalid type\n");
 		return -EINVAL;
 	}
@@ -492,6 +522,11 @@ int avtab_read(struct avtab *a, struct policy_file *fp, struct policydb *pol)
 		goto bad;
 	}
 
+	/* avtab_read_item() reads at least 96 bytes for any valid entry */
+	rc = size_check(3 * sizeof(u32), nel, fp);
+	if (rc)
+		goto bad;
+
 	rc = avtab_alloc(a, nel);
 	if (rc)
 		goto bad;
diff --git a/security/selinux/ss/avtab.h b/security/selinux/ss/avtab.h
index 850b3453f2598..1de4cce288a76 100644
--- a/security/selinux/ss/avtab.h
+++ b/security/selinux/ss/avtab.h
@@ -44,6 +44,7 @@ struct avtab_key {
 	 AVTAB_XPERMS_DONTAUDIT)
 #define AVTAB_ENABLED_OLD 0x80000000 /* reserved for used in cond_avtab */
 #define AVTAB_ENABLED	  0x8000 /* reserved for used in cond_avtab */
+#define AVTAB_SPECIFIER_MASK (AVTAB_AV | AVTAB_TYPE | AVTAB_XPERMS | AVTAB_ENABLED)
 	u16 specified; /* what field is specified */
 };
 
@@ -68,6 +69,18 @@ struct avtab_extended_perms {
 	struct extended_perms_data perms;
 };
 
+static inline bool avtab_is_valid_xperm_specified(u8 specified)
+{
+	switch (specified) {
+	case AVTAB_XPERMS_IOCTLFUNCTION:
+	case AVTAB_XPERMS_IOCTLDRIVER:
+	case AVTAB_XPERMS_NLMSG:
+		return true;
+	default:
+		return false;
+	}
+}
+
 struct avtab_datum {
 	union {
 		u32 data; /* access vector or type value */
diff --git a/security/selinux/ss/conditional.c b/security/selinux/ss/conditional.c
index 824c3f8961518..2956dd7edea78 100644
--- a/security/selinux/ss/conditional.c
+++ b/security/selinux/ss/conditional.c
@@ -12,6 +12,7 @@
 
 #include "security.h"
 #include "conditional.h"
+#include "policydb.h"
 #include "services.h"
 
 /*
@@ -165,7 +166,7 @@ void cond_policydb_destroy(struct policydb *p)
 int cond_init_bool_indexes(struct policydb *p)
 {
 	kfree(p->bool_val_to_struct);
-	p->bool_val_to_struct = kmalloc_objs(*p->bool_val_to_struct,
+	p->bool_val_to_struct = kzalloc_objs(*p->bool_val_to_struct,
 					     p->p_bools.nprim);
 	if (!p->bool_val_to_struct)
 		return -ENOMEM;
@@ -199,19 +200,12 @@ int cond_index_bool(void *key, void *datum, void *datap)
 	return 0;
 }
 
-static int bool_isvalid(struct cond_bool_datum *b)
-{
-	if (!(b->state == 0 || b->state == 1))
-		return 0;
-	return 1;
-}
-
 int cond_read_bool(struct policydb *p, struct symtab *s, struct policy_file *fp)
 {
 	char *key = NULL;
 	struct cond_bool_datum *booldatum;
 	__le32 buf[3];
-	u32 len;
+	u32 len, val;
 	int rc;
 
 	booldatum = kzalloc_obj(*booldatum);
@@ -223,11 +217,12 @@ int cond_read_bool(struct policydb *p, struct symtab *s, struct policy_file *fp)
 		goto err;
 
 	booldatum->value = le32_to_cpu(buf[0]);
-	booldatum->state = le32_to_cpu(buf[1]);
+	val = le32_to_cpu(buf[1]);
 
 	rc = -EINVAL;
-	if (!bool_isvalid(booldatum))
+	if (!val_is_boolean(val))
 		goto err;
+	booldatum->state = (int)val;
 
 	len = le32_to_cpu(buf[2]);
 
@@ -241,6 +236,7 @@ int cond_read_bool(struct policydb *p, struct symtab *s, struct policy_file *fp)
 
 	return 0;
 err:
+	pr_err("SELinux: conditional: failed to read boolean\n");
 	cond_destroy_bool(key, booldatum, NULL);
 	return rc;
 }
@@ -334,6 +330,11 @@ static int cond_read_av_list(struct policydb *p, struct policy_file *fp,
 	if (len == 0)
 		return 0;
 
+	/* avtab_read_item() reads at least 96 bytes for any valid entry */
+	rc = size_check(3 * sizeof(u32), len, fp);
+	if (rc)
+		return rc;
+
 	list->nodes = kzalloc_objs(*list->nodes, len);
 	if (!list->nodes)
 		return -ENOMEM;
@@ -362,7 +363,8 @@ static int expr_node_isvalid(struct policydb *p, struct cond_expr_node *expr)
 		return 0;
 	}
 
-	if (expr->boolean > p->p_bools.nprim) {
+	if (expr->expr_type == COND_BOOL &&
+	    (expr->boolean == 0 || expr->boolean > p->p_bools.nprim)) {
 		pr_err("SELinux: conditional expressions uses unknown bool.\n");
 		return 0;
 	}
@@ -383,6 +385,12 @@ static int cond_read_node(struct policydb *p, struct cond_node *node, struct pol
 
 	/* expr */
 	len = le32_to_cpu(buf[1]);
+
+	/* we will read 64 bytes per node */
+	rc = size_check(2 * sizeof(u32), len, fp);
+	if (rc)
+		return rc;
+
 	node->expr.nodes = kzalloc_objs(*node->expr.nodes, len);
 	if (!node->expr.nodes)
 		return -ENOMEM;
@@ -421,6 +429,11 @@ int cond_read_list(struct policydb *p, struct policy_file *fp)
 
 	len = le32_to_cpu(buf[0]);
 
+	/* cond_read_node() reads at least 128 bytes for any valid node */
+	rc = size_check(4 * sizeof(u32), len, fp);
+	if (rc)
+		return rc;
+
 	p->cond_list = kzalloc_objs(*p->cond_list, len);
 	if (!p->cond_list)
 		return -ENOMEM;
@@ -709,7 +722,7 @@ static int duplicate_policydb_bools(struct policydb *newdb,
 	struct cond_bool_datum **cond_bool_array;
 	int rc;
 
-	cond_bool_array = kmalloc_objs(*orig->bool_val_to_struct,
+	cond_bool_array = kzalloc_objs(*orig->bool_val_to_struct,
 				       orig->p_bools.nprim);
 	if (!cond_bool_array)
 		return -ENOMEM;
diff --git a/security/selinux/ss/constraint.h b/security/selinux/ss/constraint.h
index 203033cfad672..1d75a8a044df9 100644
--- a/security/selinux/ss/constraint.h
+++ b/security/selinux/ss/constraint.h
@@ -50,6 +50,7 @@ struct constraint_expr {
 	u32 op; /* operator */
 
 	struct ebitmap names; /* names */
+	/* internally unused, only forwarded via policydb_write() */
 	struct type_set *type_names;
 
 	struct constraint_expr *next; /* next expression */
diff --git a/security/selinux/ss/ebitmap.c b/security/selinux/ss/ebitmap.c
index 43bc19e219609..ea894361fed64 100644
--- a/security/selinux/ss/ebitmap.c
+++ b/security/selinux/ss/ebitmap.c
@@ -257,6 +257,33 @@ int ebitmap_contains(const struct ebitmap *e1, const struct ebitmap *e2,
 	return 1;
 }
 
+u32 ebitmap_get_highest_set_bit(const struct ebitmap *e)
+{
+	const struct ebitmap_node *n;
+	unsigned long unit;
+	u32 pos = 0;
+
+	n = e->node;
+	if (!n)
+		return 0;
+
+	while (n->next)
+		n = n->next;
+
+	for (unsigned int i = EBITMAP_UNIT_NUMS; i > 0; i--) {
+		unit = n->maps[i - 1];
+		if (unit == 0)
+			continue;
+
+		pos = (i - 1) * EBITMAP_UNIT_SIZE;
+		while (unit >>= 1)
+			pos++;
+		break;
+	}
+
+	return n->startbit + pos;
+}
+
 int ebitmap_get_bit(const struct ebitmap *e, u32 bit)
 {
 	const struct ebitmap_node *n;
diff --git a/security/selinux/ss/ebitmap.h b/security/selinux/ss/ebitmap.h
index c9569998f287d..ae05cc8e46721 100644
--- a/security/selinux/ss/ebitmap.h
+++ b/security/selinux/ss/ebitmap.h
@@ -126,6 +126,7 @@ int ebitmap_and(struct ebitmap *dst, const struct ebitmap *e1,
 		const struct ebitmap *e2);
 int ebitmap_contains(const struct ebitmap *e1, const struct ebitmap *e2,
 		     u32 last_e2bit);
+u32 ebitmap_get_highest_set_bit(const struct ebitmap *e);
 int ebitmap_get_bit(const struct ebitmap *e, u32 bit);
 int ebitmap_set_bit(struct ebitmap *e, u32 bit, int value);
 void ebitmap_destroy(struct ebitmap *e);
diff --git a/security/selinux/ss/hashtab.h b/security/selinux/ss/hashtab.h
index deba82d78c3ae..c641fb12916b0 100644
--- a/security/selinux/ss/hashtab.h
+++ b/security/selinux/ss/hashtab.h
@@ -94,11 +94,11 @@ static inline int hashtab_insert(struct hashtab *h, void *key, void *datum,
  * Returns NULL if no entry has the specified key or
  * the datum of the entry otherwise.
  */
-static inline void *hashtab_search(struct hashtab *h, const void *key,
+static inline void *hashtab_search(const struct hashtab *h, const void *key,
 				   struct hashtab_key_params key_params)
 {
 	u32 hvalue;
-	struct hashtab_node *cur;
+	const struct hashtab_node *cur;
 
 	if (!h->size)
 		return NULL;
diff --git a/security/selinux/ss/mls.c b/security/selinux/ss/mls.c
index a6e49269f5351..3cd36e2015fa2 100644
--- a/security/selinux/ss/mls.c
+++ b/security/selinux/ss/mls.c
@@ -32,7 +32,7 @@
 int mls_compute_context_len(struct policydb *p, struct context *context)
 {
 	int i, l, len, head, prev;
-	char *nm;
+	const char *nm;
 	struct ebitmap *e;
 	struct ebitmap_node *node;
 
@@ -86,7 +86,8 @@ int mls_compute_context_len(struct policydb *p, struct context *context)
 void mls_sid_to_context(struct policydb *p, struct context *context,
 			char **scontext)
 {
-	char *scontextp, *nm;
+	const char *nm;
+	char *scontextp;
 	int i, l, head, prev;
 	struct ebitmap *e;
 	struct ebitmap_node *node;
@@ -155,27 +156,44 @@ void mls_sid_to_context(struct policydb *p, struct context *context,
 	*scontext = scontextp;
 }
 
-int mls_level_isvalid(struct policydb *p, struct mls_level *l)
+bool mls_level_isvalid(const struct policydb *p, const struct mls_level *l)
 {
-	struct level_datum *levdatum;
+	const char *name;
+	const struct level_datum *levdatum;
+	struct ebitmap_node *node;
+	u32 bit;
+	int rc;
 
 	if (!l->sens || l->sens > p->p_levels.nprim)
-		return 0;
-	levdatum = symtab_search(&p->p_levels,
-				 sym_name(p, SYM_LEVELS, l->sens - 1));
+		return false;
+
+	name = sym_name(p, SYM_LEVELS, l->sens - 1);
+	if (!name)
+		return false;
+
+	levdatum = symtab_search(&p->p_levels, name);
 	if (!levdatum)
-		return 0;
+		return false;
 
 	/*
-	 * Return 1 iff all the bits set in l->cat are also be set in
+	 * Validate that all bits set in l->cat are also be set in
 	 * levdatum->level->cat and no bit in l->cat is larger than
 	 * p->p_cats.nprim.
 	 */
-	return ebitmap_contains(&levdatum->level.cat, &l->cat,
-				p->p_cats.nprim);
+	rc = ebitmap_contains(&levdatum->level.cat, &l->cat,
+			      p->p_cats.nprim);
+	if (!rc)
+		return false;
+
+	ebitmap_for_each_positive_bit(&levdatum->level.cat, node, bit) {
+		if (!sym_name(p, SYM_CATS, bit))
+			return false;
+	}
+
+	return true;
 }
 
-int mls_range_isvalid(struct policydb *p, struct mls_range *r)
+bool mls_range_isvalid(const struct policydb *p, const struct mls_range *r)
 {
 	return (mls_level_isvalid(p, &r->level[0]) &&
 		mls_level_isvalid(p, &r->level[1]) &&
@@ -183,32 +201,32 @@ int mls_range_isvalid(struct policydb *p, struct mls_range *r)
 }
 
 /*
- * Return 1 if the MLS fields in the security context
+ * Return true if the MLS fields in the security context
  * structure `c' are valid.  Return 0 otherwise.
  */
-int mls_context_isvalid(struct policydb *p, struct context *c)
+bool mls_context_isvalid(const struct policydb *p, const struct context *c)
 {
-	struct user_datum *usrdatum;
+	const struct user_datum *usrdatum;
 
 	if (!p->mls_enabled)
-		return 1;
+		return true;
 
 	if (!mls_range_isvalid(p, &c->range))
-		return 0;
+		return false;
 
 	if (c->role == OBJECT_R_VAL)
-		return 1;
+		return true;
 
 	/*
 	 * User must be authorized for the MLS range.
 	 */
 	if (!c->user || c->user > p->p_users.nprim)
-		return 0;
+		return false;
 	usrdatum = p->user_val_to_struct[c->user - 1];
-	if (!mls_range_contains(usrdatum->range, c->range))
-		return 0; /* user may not be associated with range */
+	if (!usrdatum || !mls_range_contains(usrdatum->range, c->range))
+		return false; /* user may not be associated with range */
 
-	return 1;
+	return true;
 }
 
 /*
@@ -449,8 +467,8 @@ int mls_convert_context(struct policydb *oldp, struct policydb *newp,
 		return 0;
 
 	for (l = 0; l < 2; l++) {
-		char *name = sym_name(oldp, SYM_LEVELS,
-				      oldc->range.level[l].sens - 1);
+		const char *name = sym_name(oldp, SYM_LEVELS,
+					    oldc->range.level[l].sens - 1);
 
 		levdatum = symtab_search(&newp->p_levels, name);
 
diff --git a/security/selinux/ss/mls.h b/security/selinux/ss/mls.h
index 07980636751fa..93cde1b22992c 100644
--- a/security/selinux/ss/mls.h
+++ b/security/selinux/ss/mls.h
@@ -27,9 +27,9 @@
 int mls_compute_context_len(struct policydb *p, struct context *context);
 void mls_sid_to_context(struct policydb *p, struct context *context,
 			char **scontext);
-int mls_context_isvalid(struct policydb *p, struct context *c);
-int mls_range_isvalid(struct policydb *p, struct mls_range *r);
-int mls_level_isvalid(struct policydb *p, struct mls_level *l);
+bool mls_context_isvalid(const struct policydb *p, const struct context *c);
+bool mls_range_isvalid(const struct policydb *p, const struct mls_range *r);
+bool mls_level_isvalid(const struct policydb *p, const struct mls_level *l);
 
 int mls_context_to_sid(struct policydb *p, char oldc, char *scontext,
 		       struct context *context, struct sidtab *s, u32 def_sid);
diff --git a/security/selinux/ss/policydb.c b/security/selinux/ss/policydb.c
index 738fd47f33e68..ead504a639e36 100644
--- a/security/selinux/ss/policydb.c
+++ b/security/selinux/ss/policydb.c
@@ -30,6 +30,7 @@
 #include <linux/string.h>
 #include <linux/errno.h>
 #include <linux/audit.h>
+#include <linux/sort.h>
 #include "security.h"
 
 #include "policydb.h"
@@ -429,11 +430,11 @@ static int filenametr_cmp(const void *k1, const void *k2)
 	const struct filename_trans_key *ft2 = k2;
 	int v;
 
-	v = ft1->ttype - ft2->ttype;
+	v = cmp_int(ft1->ttype, ft2->ttype);
 	if (v)
 		return v;
 
-	v = ft1->tclass - ft2->tclass;
+	v = cmp_int(ft1->tclass, ft2->tclass);
 	if (v)
 		return v;
 
@@ -464,15 +465,15 @@ static int rangetr_cmp(const void *k1, const void *k2)
 	const struct range_trans *key1 = k1, *key2 = k2;
 	int v;
 
-	v = key1->source_type - key2->source_type;
+	v = cmp_int(key1->source_type, key2->source_type);
 	if (v)
 		return v;
 
-	v = key1->target_type - key2->target_type;
+	v = cmp_int(key1->target_type, key2->target_type);
 	if (v)
 		return v;
 
-	v = key1->target_class - key2->target_class;
+	v = cmp_int(key1->target_class, key2->target_class);
 
 	return v;
 }
@@ -501,15 +502,15 @@ static int role_trans_cmp(const void *k1, const void *k2)
 	const struct role_trans_key *key1 = k1, *key2 = k2;
 	int v;
 
-	v = key1->role - key2->role;
+	v = cmp_int(key1->role, key2->role);
 	if (v)
 		return v;
 
-	v = key1->type - key2->type;
+	v = cmp_int(key1->type, key2->type);
 	if (v)
 		return v;
 
-	return key1->tclass - key2->tclass;
+	return cmp_int(key1->tclass, key2->tclass);
 }
 
 static const struct hashtab_key_params roletr_key_params = {
@@ -638,13 +639,11 @@ static int sens_index(void *key, void *datum, void *datap)
 	levdatum = datum;
 	p = datap;
 
-	if (!levdatum->isalias) {
-		if (!levdatum->level.sens ||
-		    levdatum->level.sens > p->p_levels.nprim)
-			return -EINVAL;
+	if (!levdatum->level.sens || levdatum->level.sens > p->p_levels.nprim)
+		return -EINVAL;
 
+	if (!levdatum->isalias)
 		p->sym_val_to_name[SYM_LEVELS][levdatum->level.sens - 1] = key;
-	}
 
 	return 0;
 }
@@ -657,12 +656,11 @@ static int cat_index(void *key, void *datum, void *datap)
 	catdatum = datum;
 	p = datap;
 
-	if (!catdatum->isalias) {
-		if (!catdatum->value || catdatum->value > p->p_cats.nprim)
-			return -EINVAL;
+	if (!catdatum->value || catdatum->value > p->p_cats.nprim)
+		return -EINVAL;
 
+	if (!catdatum->isalias)
 		p->sym_val_to_name[SYM_CATS][catdatum->value - 1] = key;
-	}
 
 	return 0;
 }
@@ -735,7 +733,6 @@ static int policydb_index(struct policydb *p)
 	pr_debug("SELinux:  %d classes, %d rules\n", p->p_classes.nprim,
 		 p->te_avtab.nel);
 
-	avtab_hash_eval(&p->te_avtab, "rules");
 	symtab_hash_eval(p->symtab);
 
 	p->class_val_to_struct = kzalloc_objs(*p->class_val_to_struct,
@@ -931,44 +928,76 @@ int policydb_load_isids(struct policydb *p, struct sidtab *s)
 	return 0;
 }
 
-int policydb_class_isvalid(struct policydb *p, unsigned int class)
+bool policydb_class_isvalid(const struct policydb *p, u16 class)
 {
 	if (!class || class > p->p_classes.nprim)
-		return 0;
-	return 1;
+		return false;
+	if (!p->sym_val_to_name[SYM_CLASSES][class - 1])
+		return false;
+	return true;
+}
+
+bool policydb_user_isvalid(const struct policydb *p, u32 user)
+{
+	if (!user || user > p->p_users.nprim)
+		return false;
+	if (!p->sym_val_to_name[SYM_USERS][user - 1])
+		return false;
+	return true;
 }
 
-int policydb_role_isvalid(struct policydb *p, unsigned int role)
+bool policydb_role_isvalid(const struct policydb *p, u32 role)
 {
 	if (!role || role > p->p_roles.nprim)
-		return 0;
-	return 1;
+		return false;
+	if (!p->sym_val_to_name[SYM_ROLES][role - 1])
+		return false;
+	return true;
 }
 
-int policydb_type_isvalid(struct policydb *p, unsigned int type)
+bool policydb_type_isvalid(const struct policydb *p, u32 type)
 {
 	if (!type || type > p->p_types.nprim)
-		return 0;
-	return 1;
+		return false;
+	if (!p->sym_val_to_name[SYM_TYPES][type - 1])
+		return false;
+	return true;
+}
+
+bool policydb_simpletype_isvalid(const struct policydb *p, u32 type)
+{
+	const struct type_datum *datum;
+
+	if (!type || type > p->p_types.nprim)
+		return false;
+
+	datum = p->type_val_to_struct[type - 1];
+	if (!datum)
+		return false;
+
+	if (datum->attribute)
+		return false;
+
+	return true;
 }
 
 /*
- * Return 1 if the fields in the security context
+ * Return true if the fields in the security context
  * structure `c' are valid.  Return 0 otherwise.
  */
-int policydb_context_isvalid(struct policydb *p, struct context *c)
+bool policydb_context_isvalid(const struct policydb *p, const struct context *c)
 {
-	struct role_datum *role;
-	struct user_datum *usrdatum;
+	const struct role_datum *role;
+	const struct user_datum *usrdatum;
 
 	if (!c->role || c->role > p->p_roles.nprim)
-		return 0;
+		return false;
 
 	if (!c->user || c->user > p->p_users.nprim)
-		return 0;
+		return false;
 
 	if (!c->type || c->type > p->p_types.nprim)
-		return 0;
+		return false;
 
 	if (c->role != OBJECT_R_VAL) {
 		/*
@@ -977,24 +1006,24 @@ int policydb_context_isvalid(struct policydb *p, struct context *c)
 		role = p->role_val_to_struct[c->role - 1];
 		if (!role || !ebitmap_get_bit(&role->types, c->type - 1))
 			/* role may not be associated with type */
-			return 0;
+			return false;
 
 		/*
 		 * User must be authorized for the role.
 		 */
 		usrdatum = p->user_val_to_struct[c->user - 1];
 		if (!usrdatum)
-			return 0;
+			return false;
 
 		if (!ebitmap_get_bit(&usrdatum->roles, c->role - 1))
 			/* user may not be associated with role */
-			return 0;
+			return false;
 	}
 
 	if (!mls_context_isvalid(p, c))
-		return 0;
+		return false;
 
-	return 1;
+	return true;
 }
 
 /*
@@ -1107,6 +1136,9 @@ int str_read(char **strp, gfp_t flags, struct policy_file *fp, u32 len)
 	if ((len == 0) || (len == (u32)-1))
 		return -EINVAL;
 
+	if (size_check(sizeof(char), len, fp))
+		return -EINVAL;
+
 	str = kmalloc(len + 1, flags | __GFP_NOWARN);
 	if (!str)
 		return -ENOMEM;
@@ -1140,6 +1172,9 @@ static int perm_read(struct policydb *p, struct symtab *s, struct policy_file *f
 
 	len = le32_to_cpu(buf[0]);
 	perdatum->value = le32_to_cpu(buf[1]);
+	rc = -EINVAL;
+	if (perdatum->value < 1 || perdatum->value > SEL_VEC_MAX)
+		goto bad;
 
 	rc = str_read(&key, GFP_KERNEL, fp, len);
 	if (rc)
@@ -1174,6 +1209,14 @@ static int common_read(struct policydb *p, struct symtab *s, struct policy_file
 	len = le32_to_cpu(buf[0]);
 	comdatum->value = le32_to_cpu(buf[1]);
 	nel = le32_to_cpu(buf[3]);
+	rc = -EINVAL;
+	if (nel > SEL_VEC_MAX)
+		goto bad;
+
+	/* perm_read() reads at least 64 bytes for any valid permission */
+	rc = size_check(2 * sizeof(u32), nel, fp);
+	if (rc)
+		goto bad;
 
 	rc = symtab_init(&comdatum->permissions, nel);
 	if (rc)
@@ -1323,7 +1366,7 @@ static int class_read(struct policydb *p, struct symtab *s, struct policy_file *
 	char *key = NULL;
 	struct class_datum *cladatum;
 	__le32 buf[6];
-	u32 i, len, len2, ncons, nel;
+	u32 i, len, len2, ncons, nel, val;
 	int rc;
 
 	cladatum = kzalloc_obj(*cladatum);
@@ -1336,8 +1379,21 @@ static int class_read(struct policydb *p, struct symtab *s, struct policy_file *
 
 	len = le32_to_cpu(buf[0]);
 	len2 = le32_to_cpu(buf[1]);
-	cladatum->value = le32_to_cpu(buf[2]);
 	nel = le32_to_cpu(buf[4]);
+	rc = -EINVAL;
+	if (nel > SEL_VEC_MAX)
+		goto bad;
+
+	val = le32_to_cpu(buf[2]);
+	rc = -EINVAL;
+	if (val > U16_MAX)
+		goto bad;
+	cladatum->value = val;
+
+	/* perm_read() reads at least 64 bytes for any valid permission */
+	rc = size_check(2 * sizeof(u32), nel, fp);
+	if (rc)
+		goto bad;
 
 	rc = symtab_init(&cladatum->permissions, nel);
 	if (rc)
@@ -1393,16 +1449,59 @@ static int class_read(struct policydb *p, struct symtab *s, struct policy_file *
 		if (rc)
 			goto bad;
 
-		cladatum->default_user = le32_to_cpu(buf[0]);
-		cladatum->default_role = le32_to_cpu(buf[1]);
-		cladatum->default_range = le32_to_cpu(buf[2]);
+		rc = -EINVAL;
+		val = le32_to_cpu(buf[0]);
+		switch (val) {
+		case 0:
+		case DEFAULT_SOURCE:
+		case DEFAULT_TARGET:
+			cladatum->default_user = val;
+			break;
+		default:
+			goto bad;
+		}
+		val = le32_to_cpu(buf[1]);
+		switch (val) {
+		case 0:
+		case DEFAULT_SOURCE:
+		case DEFAULT_TARGET:
+			cladatum->default_role = val;
+			break;
+		default:
+			goto bad;
+		}
+		val = le32_to_cpu(buf[2]);
+		switch (val) {
+		case 0:
+		case DEFAULT_SOURCE_LOW:
+		case DEFAULT_SOURCE_HIGH:
+		case DEFAULT_SOURCE_LOW_HIGH:
+		case DEFAULT_TARGET_LOW:
+		case DEFAULT_TARGET_HIGH:
+		case DEFAULT_TARGET_LOW_HIGH:
+		case DEFAULT_GLBLUB:
+			cladatum->default_range = val;
+			break;
+		default:
+			goto bad;
+		}
 	}
 
 	if (p->policyvers >= POLICYDB_VERSION_DEFAULT_TYPE) {
 		rc = next_entry(buf, fp, sizeof(u32) * 1);
 		if (rc)
 			goto bad;
-		cladatum->default_type = le32_to_cpu(buf[0]);
+		rc = -EINVAL;
+		val = le32_to_cpu(buf[0]);
+		switch (val) {
+		case 0:
+		case DEFAULT_TARGET:
+		case DEFAULT_SOURCE:
+			cladatum->default_type = val;
+			break;
+		default:
+			goto bad;
+		}
 	}
 
 	rc = symtab_insert(s, key, cladatum);
@@ -1412,6 +1511,8 @@ static int class_read(struct policydb *p, struct symtab *s, struct policy_file *
 	return 0;
 bad:
 	cls_destroy(key, cladatum, NULL);
+	if (rc)
+		pr_err("SELinux:  invalid class\n");
 	return rc;
 }
 
@@ -1603,7 +1704,7 @@ static int sens_read(struct policydb *p, struct symtab *s, struct policy_file *f
 	struct level_datum *levdatum;
 	int rc;
 	__le32 buf[2];
-	u32 len;
+	u32 len, val;
 
 	levdatum = kzalloc_obj(*levdatum);
 	if (!levdatum)
@@ -1614,7 +1715,11 @@ static int sens_read(struct policydb *p, struct symtab *s, struct policy_file *f
 		goto bad;
 
 	len = le32_to_cpu(buf[0]);
-	levdatum->isalias = le32_to_cpu(buf[1]);
+	val = le32_to_cpu(buf[1]);
+	rc = -EINVAL;
+	if (!val_is_boolean(val))
+		goto bad;
+	levdatum->isalias = val;
 
 	rc = str_read(&key, GFP_KERNEL, fp, len);
 	if (rc)
@@ -1630,6 +1735,8 @@ static int sens_read(struct policydb *p, struct symtab *s, struct policy_file *f
 	return 0;
 bad:
 	sens_destroy(key, levdatum, NULL);
+	if (rc)
+		pr_err("SELinux:  invalid sensitivity\n");
 	return rc;
 }
 
@@ -1639,7 +1746,7 @@ static int cat_read(struct policydb *p, struct symtab *s, struct policy_file *fp
 	struct cat_datum *catdatum;
 	int rc;
 	__le32 buf[3];
-	u32 len;
+	u32 len, val;
 
 	catdatum = kzalloc_obj(*catdatum);
 	if (!catdatum)
@@ -1651,7 +1758,11 @@ static int cat_read(struct policydb *p, struct symtab *s, struct policy_file *fp
 
 	len = le32_to_cpu(buf[0]);
 	catdatum->value = le32_to_cpu(buf[1]);
-	catdatum->isalias = le32_to_cpu(buf[2]);
+	val = le32_to_cpu(buf[2]);
+	rc = -EINVAL;
+	if (!val_is_boolean(val))
+		goto bad;
+	catdatum->isalias = val;
 
 	rc = str_read(&key, GFP_KERNEL, fp, len);
 	if (rc)
@@ -1663,6 +1774,8 @@ static int cat_read(struct policydb *p, struct symtab *s, struct policy_file *fp
 	return 0;
 bad:
 	cat_destroy(key, catdatum, NULL);
+	if (rc)
+		pr_err("SELinux:  invalid category\n");
 	return rc;
 }
 
@@ -1698,6 +1811,12 @@ static int user_bounds_sanity_check(void *key, void *datum, void *datap)
 			return -EINVAL;
 		}
 
+		if (!policydb_user_isvalid(p, upper->bounds)) {
+			pr_err("SELinux: user %s: invalid boundary id %d\n",
+			       (char *) key, upper->bounds);
+			return -EINVAL;
+		}
+
 		upper = p->user_val_to_struct[upper->bounds - 1];
 		ebitmap_for_each_positive_bit(&user->roles, node, bit)
 		{
@@ -1735,6 +1854,12 @@ static int role_bounds_sanity_check(void *key, void *datum, void *datap)
 			return -EINVAL;
 		}
 
+		if (!policydb_role_isvalid(p, upper->bounds)) {
+			pr_err("SELinux: role %s: invalid boundary id %d\n",
+			       (char *) key, upper->bounds);
+			return -EINVAL;
+		}
+
 		upper = p->role_val_to_struct[upper->bounds - 1];
 		ebitmap_for_each_positive_bit(&role->types, node, bit)
 		{
@@ -1769,9 +1894,13 @@ static int type_bounds_sanity_check(void *key, void *datum, void *datap)
 			return -EINVAL;
 		}
 
-		upper = p->type_val_to_struct[upper->bounds - 1];
-		BUG_ON(!upper);
+		if (!policydb_type_isvalid(p, upper->bounds)) {
+			pr_err("SELinux: type %s: invalid boundary id %d\n",
+			       (char *) key, upper->bounds);
+			return -EINVAL;
+		}
 
+		upper = p->type_val_to_struct[upper->bounds - 1];
 		if (upper->attribute) {
 			pr_err("SELinux: type %s: "
 			       "bounded by attribute %s\n",
@@ -1844,7 +1973,7 @@ static int range_read(struct policydb *p, struct policy_file *fp)
 	struct mls_range *r = NULL;
 	int rc;
 	__le32 buf[2];
-	u32 i, nel;
+	u32 i, nel, val;
 
 	if (p->policyvers < POLICYDB_VERSION_MLS)
 		return 0;
@@ -1855,6 +1984,13 @@ static int range_read(struct policydb *p, struct policy_file *fp)
 
 	nel = le32_to_cpu(buf[0]);
 
+	/* we read at least 64 bytes and mls_read_range_helper() 32 bytes
+	 * for any valid range-transition
+	 */
+	rc = size_check(3 * sizeof(u32), nel, fp);
+	if (rc)
+		return rc;
+
 	rc = hashtab_init(&p->range_tr, nel);
 	if (rc)
 		return rc;
@@ -1875,7 +2011,11 @@ static int range_read(struct policydb *p, struct policy_file *fp)
 			rc = next_entry(buf, fp, sizeof(u32));
 			if (rc)
 				goto out;
-			rt->target_class = le32_to_cpu(buf[0]);
+			rc = -EINVAL;
+			val = le32_to_cpu(buf[0]);
+			if (val > U16_MAX)
+				goto out;
+			rt->target_class = val;
 		} else
 			rt->target_class = p->process_class;
 
@@ -1912,6 +2052,8 @@ static int range_read(struct policydb *p, struct policy_file *fp)
 out:
 	kfree(rt);
 	kfree(r);
+	if (rc)
+		pr_err("SELinux:  invalid range\n");
 	return rc;
 }
 
@@ -1920,7 +2062,7 @@ static int filename_trans_read_helper_compat(struct policydb *p, struct policy_f
 	struct filename_trans_key key, *ft = NULL;
 	struct filename_trans_datum *last, *datum = NULL;
 	char *name = NULL;
-	u32 len, stype, otype;
+	u32 len, stype, otype, val;
 	__le32 buf[4];
 	int rc;
 
@@ -1939,12 +2081,22 @@ static int filename_trans_read_helper_compat(struct policydb *p, struct policy_f
 	if (rc)
 		goto out;
 
+	rc = -EINVAL;
 	stype = le32_to_cpu(buf[0]);
+	if (!policydb_type_isvalid(p, stype))
+		goto out;
 	key.ttype = le32_to_cpu(buf[1]);
-	key.tclass = le32_to_cpu(buf[2]);
+	if (!policydb_type_isvalid(p, key.ttype))
+		goto out;
+	val = le32_to_cpu(buf[2]);
+	if (val > U16_MAX || !policydb_class_isvalid(p, val))
+		goto out;
+	key.tclass = val;
 	key.name = name;
 
 	otype = le32_to_cpu(buf[3]);
+	if (!policydb_simpletype_isvalid(p, otype))
+		goto out;
 
 	last = NULL;
 	datum = policydb_filenametr_search(p, &key);
@@ -1997,6 +2149,9 @@ static int filename_trans_read_helper_compat(struct policydb *p, struct policy_f
 	kfree(ft);
 	kfree(name);
 	kfree(datum);
+
+	if (rc)
+		pr_err("SELinux:  invalid compat filename transition\n");
 	return rc;
 }
 
@@ -2005,7 +2160,8 @@ static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp
 	struct filename_trans_key *ft = NULL;
 	struct filename_trans_datum **dst, *datum, *first = NULL;
 	char *name = NULL;
-	u32 len, ttype, tclass, ndatum, i;
+	u32 len, ttype, ndatum, i, val;
+	u16 tclass;
 	__le32 buf[3];
 	int rc;
 
@@ -2024,8 +2180,15 @@ static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp
 	if (rc)
 		goto out;
 
+	rc = -EINVAL;
 	ttype = le32_to_cpu(buf[0]);
-	tclass = le32_to_cpu(buf[1]);
+	if (!policydb_type_isvalid(p, ttype))
+		goto out;
+	val = le32_to_cpu(buf[1]);
+	rc = -EINVAL;
+	if (val > U16_MAX || !policydb_class_isvalid(p, val))
+		goto out;
+	tclass = val;
 
 	ndatum = le32_to_cpu(buf[2]);
 	if (ndatum == 0) {
@@ -2055,6 +2218,10 @@ static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp
 
 		datum->otype = le32_to_cpu(buf[0]);
 
+		rc = -EINVAL;
+		if (!policydb_simpletype_isvalid(p, datum->otype))
+			goto out;
+
 		dst = &datum->next;
 	}
 
@@ -2086,6 +2253,9 @@ static int filename_trans_read_helper(struct policydb *p, struct policy_file *fp
 		ebitmap_destroy(&datum->stypes);
 		kfree(datum);
 	}
+
+	if (rc)
+		pr_err("SELinux:  invalid filename transition\n");
 	return rc;
 }
 
@@ -2133,7 +2303,7 @@ static int filename_trans_read(struct policydb *p, struct policy_file *fp)
 static int genfs_read(struct policydb *p, struct policy_file *fp)
 {
 	int rc;
-	u32 i, j, nel, nel2, len, len2;
+	u32 i, j, nel, nel2, len, len2, val;
 	__le32 buf[1];
 	struct ocontext *l, *c;
 	struct ocontext *newc = NULL;
@@ -2203,7 +2373,11 @@ static int genfs_read(struct policydb *p, struct policy_file *fp)
 			if (rc)
 				goto out;
 
-			newc->v.sclass = le32_to_cpu(buf[0]);
+			rc = -EINVAL;
+			val = le32_to_cpu(buf[0]);
+			if (val > U16_MAX || (val != 0 && !policydb_class_isvalid(p, val)))
+				goto out;
+			newc->v.sclass = val;
 			rc = context_read_and_validate(&newc->context[0], p,
 						       fp);
 			if (rc)
@@ -2240,6 +2414,9 @@ static int genfs_read(struct policydb *p, struct policy_file *fp)
 	}
 	ocontext_destroy(newc, OCON_FSUSE);
 
+	if (rc)
+		pr_err("SELinux:  invalid genfs\n");
+
 	return rc;
 }
 
@@ -2248,7 +2425,7 @@ static int ocontext_read(struct policydb *p,
 {
 	int rc;
 	unsigned int i;
-	u32 j, nel, len;
+	u32 j, nel, len, val;
 	__be64 prefixbuf[1];
 	__le32 buf[3];
 	struct ocontext *l, *c;
@@ -2312,11 +2489,25 @@ static int ocontext_read(struct policydb *p,
 				rc = next_entry(buf, fp, sizeof(u32) * 3);
 				if (rc)
 					goto out;
-				c->u.port.protocol = le32_to_cpu(buf[0]);
-				c->u.port.low_port = le32_to_cpu(buf[1]);
-				c->u.port.high_port = le32_to_cpu(buf[2]);
-				rc = context_read_and_validate(&c->context[0],
-							       p, fp);
+
+				rc = -EINVAL;
+				val = le32_to_cpu(buf[0]);
+				if (val > U8_MAX)
+					goto out;
+				c->u.port.protocol = val;
+				val = le32_to_cpu(buf[1]);
+				if (val > U16_MAX)
+					goto out;
+				c->u.port.low_port = val;
+				val = le32_to_cpu(buf[2]);
+				if (val > U16_MAX)
+					goto out;
+				c->u.port.high_port = val;
+				if (c->u.port.low_port == 0 ||
+				    c->u.port.low_port > c->u.port.high_port)
+					goto out;
+
+				rc = context_read_and_validate(&c->context[0], p, fp);
 				if (rc)
 					goto out;
 				break;
@@ -2434,6 +2625,8 @@ static int ocontext_read(struct policydb *p,
 	}
 	rc = 0;
 out:
+	if (rc)
+		pr_err("SELinux:  invalid ocon\n");
 	return rc;
 }
 
@@ -2448,7 +2641,7 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 	struct role_trans_datum *rtd = NULL;
 	int rc;
 	__le32 buf[4];
-	u32 i, j, len, nprim, nel, perm;
+	u32 i, j, len, nprim, nel, perm, val;
 
 	char *policydb_str;
 	const struct policydb_compat_info *info;
@@ -2574,6 +2767,13 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 		nprim = le32_to_cpu(buf[0]);
 		nel = le32_to_cpu(buf[1]);
 
+		/* every read_f() implementation reads at least 128 bytes
+		 * for any valid entry
+		 */
+		rc = size_check(4 * sizeof(u32), nel, fp);
+		if (rc)
+			goto out;
+
 		rc = symtab_init(&p->symtab[i], nel);
 		if (rc)
 			goto out;
@@ -2593,6 +2793,10 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 		p->symtab[i].nprim = nprim;
 	}
 
+	rc = policydb_index(p);
+	if (rc)
+		goto bad;
+
 	rc = -EINVAL;
 	p->process_class = string_to_security_class(p, "process");
 	if (!p->process_class) {
@@ -2604,6 +2808,8 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 	if (rc)
 		goto bad;
 
+	avtab_hash_eval(&p->te_avtab, "rules");
+
 	if (p->policyvers >= POLICYDB_VERSION_BOOL) {
 		rc = cond_read_list(p, fp);
 		if (rc)
@@ -2615,6 +2821,11 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 		goto bad;
 	nel = le32_to_cpu(buf[0]);
 
+	/* we read at least 96 bytes for any valid role-transition */
+	rc = size_check(3 * sizeof(u32), nel, fp);
+	if (rc)
+		goto bad;
+
 	rc = hashtab_init(&p->role_tr, nel);
 	if (rc)
 		goto bad;
@@ -2640,7 +2851,11 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 			rc = next_entry(buf, fp, sizeof(u32));
 			if (rc)
 				goto bad;
-			rtk->tclass = le32_to_cpu(buf[0]);
+			rc = -EINVAL;
+			val = le32_to_cpu(buf[0]);
+			if (val > U16_MAX)
+				goto bad;
+			rtk->tclass = val;
 		} else
 			rtk->tclass = p->process_class;
 
@@ -2692,10 +2907,6 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 	if (rc)
 		goto bad;
 
-	rc = policydb_index(p);
-	if (rc)
-		goto bad;
-
 	rc = -EINVAL;
 	perm = string_to_av_perm(p, p->process_class, "transition");
 	if (!perm) {
@@ -2740,6 +2951,11 @@ int policydb_read(struct policydb *p, struct policy_file *fp)
 			if (rc)
 				goto bad;
 		}
+
+		rc = -EINVAL;
+		if (ebitmap_get_highest_set_bit(e) >= p->p_types.nprim)
+			goto bad;
+
 		/* add the type itself as the degenerate case */
 		rc = ebitmap_set_bit(e, i, 1);
 		if (rc)
diff --git a/security/selinux/ss/policydb.h b/security/selinux/ss/policydb.h
index 89a180b1742fb..974180b2e3a35 100644
--- a/security/selinux/ss/policydb.h
+++ b/security/selinux/ss/policydb.h
@@ -48,7 +48,7 @@ struct common_datum {
 
 /* Class attributes */
 struct class_datum {
-	u32 value; /* class value */
+	u16 value; /* class value */
 	char *comkey; /* common name */
 	struct common_datum *comdatum; /* common datum */
 	struct symtab permissions; /* class-specific permission symbol table */
@@ -74,7 +74,7 @@ struct class_datum {
 /* Role attributes */
 struct role_datum {
 	u32 value; /* internal role value */
-	u32 bounds; /* boundary of role */
+	u32 bounds; /* boundary of role, 0 for none */
 	struct ebitmap dominates; /* set of roles dominated by this role */
 	struct ebitmap types; /* set of authorized types for role */
 };
@@ -82,7 +82,7 @@ struct role_datum {
 struct role_trans_key {
 	u32 role; /* current role */
 	u32 type; /* program executable type, or new object type */
-	u32 tclass; /* process class, or new object class */
+	u16 tclass; /* process class, or new object class */
 };
 
 struct role_trans_datum {
@@ -110,7 +110,8 @@ struct role_allow {
 /* Type attributes */
 struct type_datum {
 	u32 value; /* internal type value */
-	u32 bounds; /* boundary of type */
+	u32 bounds; /* boundary of type, 0 for none */
+	/* internally unused, only forwarded via policydb_write() */
 	unsigned char primary; /* primary name? */
 	unsigned char attribute; /* attribute ?*/
 };
@@ -118,7 +119,7 @@ struct type_datum {
 /* User attributes */
 struct user_datum {
 	u32 value; /* internal user value */
-	u32 bounds; /* bounds of user */
+	u32 bounds; /* bounds of user, 0 for none */
 	struct ebitmap roles; /* set of authorized roles for user */
 	struct mls_range range; /* MLS range (min - max) for user */
 	struct mls_level dfltlevel; /* default login MLS level for user */
@@ -139,7 +140,7 @@ struct cat_datum {
 struct range_trans {
 	u32 source_type;
 	u32 target_type;
-	u32 target_class;
+	u16 target_class;
 };
 
 /* Boolean data type */
@@ -195,7 +196,7 @@ struct ocontext {
 		} ibendport;
 	} u;
 	union {
-		u32 sclass; /* security class for genfs */
+		u16 sclass; /* security class for genfs (can be 0 for wildcard) */
 		u32 behavior; /* labeling behavior for fs_use */
 	} v;
 	struct context context[2]; /* security context(s) */
@@ -321,10 +322,13 @@ struct policy_file {
 
 extern void policydb_destroy(struct policydb *p);
 extern int policydb_load_isids(struct policydb *p, struct sidtab *s);
-extern int policydb_context_isvalid(struct policydb *p, struct context *c);
-extern int policydb_class_isvalid(struct policydb *p, unsigned int class);
-extern int policydb_type_isvalid(struct policydb *p, unsigned int type);
-extern int policydb_role_isvalid(struct policydb *p, unsigned int role);
+extern bool policydb_context_isvalid(const struct policydb *p,
+				     const struct context *c);
+extern bool policydb_class_isvalid(const struct policydb *p, u16 class);
+extern bool policydb_type_isvalid(const struct policydb *p, u32 type);
+extern bool policydb_simpletype_isvalid(const struct policydb *p, u32 type);
+extern bool policydb_role_isvalid(const struct policydb *p, u32 role);
+extern bool policydb_user_isvalid(const struct policydb *p, u32 user);
 extern int policydb_read(struct policydb *p, struct policy_file *fp);
 extern int policydb_write(struct policydb *p, struct policy_file *fp);
 
@@ -354,6 +358,20 @@ struct policy_data {
 	struct policy_file *fp;
 };
 
+static inline int size_check(size_t bytes, size_t num,
+			     const struct policy_file *fp)
+{
+	size_t len;
+
+	if (unlikely(check_mul_overflow(bytes, num, &len)))
+		return -EINVAL;
+
+	if (unlikely(len > fp->len))
+		return -EINVAL;
+
+	return 0;
+}
+
 static inline int next_entry(void *buf, struct policy_file *fp, size_t bytes)
 {
 	if (bytes > fp->len)
@@ -382,15 +400,29 @@ static inline int put_entry(const void *buf, size_t bytes, size_t num,
 	return 0;
 }
 
-static inline char *sym_name(struct policydb *p, unsigned int sym_num,
+static inline const char *sym_name(const struct policydb *p, unsigned int sym_num,
 			     unsigned int element_nr)
 {
 	return p->sym_val_to_name[sym_num][element_nr];
 }
 
+static inline bool val_is_boolean(u32 value)
+{
+	return value == 0 || value == 1;
+}
+
 extern int str_read(char **strp, gfp_t flags, struct policy_file *fp, u32 len);
 
 extern u16 string_to_security_class(struct policydb *p, const char *name);
 extern u32 string_to_av_perm(struct policydb *p, u16 tclass, const char *name);
 
+#define pr_warn_once_policyload(policy, fmt, ...)    \
+	do {                                         \
+		static const void *prev_policy__;    \
+		if (prev_policy__ != policy) {       \
+			pr_warn(fmt, ##__VA_ARGS__); \
+			prev_policy__ = policy;      \
+		}                                    \
+	} while (0)
+
 #endif /* _SS_POLICYDB_H_ */
diff --git a/security/selinux/ss/services.c b/security/selinux/ss/services.c
index 143021c5e326d..2d828548f3db6 100644
--- a/security/selinux/ss/services.c
+++ b/security/selinux/ss/services.c
@@ -446,8 +446,6 @@ static int dump_masked_av_helper(void *k, void *d, void *args)
 	struct perm_datum *pdatum = d;
 	char **permission_names = args;
 
-	BUG_ON(pdatum->value < 1 || pdatum->value > 32);
-
 	permission_names[pdatum->value - 1] = (char *)k;
 
 	return 0;
@@ -463,10 +461,10 @@ static void security_dump_masked_av(struct policydb *policydb,
 	struct common_datum *common_dat;
 	struct class_datum *tclass_dat;
 	struct audit_buffer *ab;
-	char *tclass_name;
+	const char *tclass_name;
 	char *scontext_name = NULL;
 	char *tcontext_name = NULL;
-	char *permission_names[32];
+	char *permission_names[SEL_VEC_MAX];
 	int index;
 	u32 length;
 	bool need_comma = false;
@@ -507,7 +505,7 @@ static void security_dump_masked_av(struct policydb *policydb,
 			 "scontext=%s tcontext=%s tclass=%s perms=",
 			 reason, scontext_name, tcontext_name, tclass_name);
 
-	for (index = 0; index < 32; index++) {
+	for (index = 0; index < SEL_VEC_MAX; index++) {
 		u32 mask = (1 << index);
 
 		if ((mask & permissions) == 0)
@@ -717,6 +715,9 @@ static void context_struct_compute_av(struct policydb *policydb,
 	 * If the given source and target types have boundary
 	 * constraint, lazy checks have to mask any violated
 	 * permission and notice it to userspace via audit.
+	 *
+	 * Infinite recursion is avoided via a depth pre-check in
+	 * type_bounds_sanity_check().
 	 */
 	type_attribute_bounds_av(policydb, scontext, tcontext,
 				 tclass, avd);
@@ -3288,7 +3289,7 @@ static int get_classes_callback(void *k, void *d, void *args)
 {
 	struct class_datum *datum = d;
 	char *name = k, **classes = args;
-	u32 value = datum->value - 1;
+	u16 value = datum->value - 1;
 
 	classes[value] = kstrdup(name, GFP_ATOMIC);
 	if (!classes[value])
diff --git a/security/selinux/ss/symtab.c b/security/selinux/ss/symtab.c
index 832660fd84a96..a756554e7f1d3 100644
--- a/security/selinux/ss/symtab.c
+++ b/security/selinux/ss/symtab.c
@@ -50,7 +50,7 @@ int symtab_insert(struct symtab *s, char *name, void *datum)
 	return hashtab_insert(&s->table, name, datum, symtab_key_params);
 }
 
-void *symtab_search(struct symtab *s, const char *name)
+void *symtab_search(const struct symtab *s, const char *name)
 {
 	return hashtab_search(&s->table, name, symtab_key_params);
 }
diff --git a/security/selinux/ss/symtab.h b/security/selinux/ss/symtab.h
index 8e667cdbf38fc..7cfa3b44953ab 100644
--- a/security/selinux/ss/symtab.h
+++ b/security/selinux/ss/symtab.h
@@ -21,6 +21,6 @@ struct symtab {
 int symtab_init(struct symtab *s, u32 size);
 
 int symtab_insert(struct symtab *s, char *name, void *datum);
-void *symtab_search(struct symtab *s, const char *name);
+void *symtab_search(const struct symtab *s, const char *name);
 
 #endif /* _SS_SYMTAB_H_ */
diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c
index 3f9ae05039a28..f2143e6757686 100644
--- a/security/smack/smack_lsm.c
+++ b/security/smack/smack_lsm.c
@@ -1312,7 +1312,7 @@ static int smack_inode_getattr(const struct path *path)
  */
 static int smack_inode_xattr_skipcap(const char *name)
 {
-	if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX)))
+	if (strncmp(name, XATTR_SMACK_SUFFIX, strlen(XATTR_SMACK_SUFFIX)) == 0)
 		return 0;
 
 	if (strcmp(name, XATTR_NAME_SMACK) == 0 ||
@@ -1665,17 +1665,12 @@ static int smack_inode_getsecurity(struct mnt_idmap *idmap,
  * smack_inode_listsecurity - list the Smack attributes
  * @inode: the object
  * @buffer: where they go
- * @buffer_size: size of buffer
+ * @remaining_size: size of buffer
  */
-static int smack_inode_listsecurity(struct inode *inode, char *buffer,
-				    size_t buffer_size)
+static int smack_inode_listsecurity(struct inode *inode, char **buffer,
+				    ssize_t *remaining_size)
 {
-	int len = sizeof(XATTR_NAME_SMACK);
-
-	if (buffer != NULL && len <= buffer_size)
-		memcpy(buffer, XATTR_NAME_SMACK, len);
-
-	return len;
+	return xattr_list_one(buffer, remaining_size, XATTR_NAME_SMACK);
 }
 
 /**
diff --git a/security/smack/smackfs.c b/security/smack/smackfs.c
index 6e62dcb36f74f..f60d5469043ed 100644
--- a/security/smack/smackfs.c
+++ b/security/smack/smackfs.c
@@ -1598,24 +1598,17 @@ static ssize_t smk_read_doi(struct file *filp, char __user *buf,
 static ssize_t smk_write_doi(struct file *file, const char __user *buf,
 			     size_t count, loff_t *ppos)
 {
-	char temp[80];
-	unsigned long u;
+	int ret;
+	u32 u;
 
 	if (!smack_privileged(CAP_MAC_ADMIN))
 		return -EPERM;
 
-	if (count >= sizeof(temp) || count == 0)
-		return -EINVAL;
-
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
-
-	temp[count] = '\0';
+	ret = kstrtou32_from_user(buf, count, 10, &u);
+	if (unlikely(ret))
+		return ret;
 
-	if (kstrtoul(temp, 10, &u))
-		return -EINVAL;
-
-	if (u == CIPSO_V4_DOI_UNKNOWN || u > U32_MAX)
+	if (u == CIPSO_V4_DOI_UNKNOWN)
 		return -EINVAL;
 
 	return smk_cipso_doi(u, GFP_KERNEL) ? : count;
@@ -1664,22 +1657,14 @@ static ssize_t smk_write_direct(struct file *file, const char __user *buf,
 				size_t count, loff_t *ppos)
 {
 	struct smack_known *skp;
-	char temp[80];
-	int i;
+	int i, ret;
 
 	if (!smack_privileged(CAP_MAC_ADMIN))
 		return -EPERM;
 
-	if (count >= sizeof(temp) || count == 0)
-		return -EINVAL;
-
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
-
-	temp[count] = '\0';
-
-	if (sscanf(temp, "%d", &i) != 1)
-		return -EINVAL;
+	ret = kstrtos32_from_user(buf, count, 10, &i);
+	if (unlikely(ret))
+		return ret;
 
 	/*
 	 * Don't do anything if the value hasn't actually changed.
@@ -1742,22 +1727,14 @@ static ssize_t smk_write_mapped(struct file *file, const char __user *buf,
 				size_t count, loff_t *ppos)
 {
 	struct smack_known *skp;
-	char temp[80];
-	int i;
+	int i, ret;
 
 	if (!smack_privileged(CAP_MAC_ADMIN))
 		return -EPERM;
 
-	if (count >= sizeof(temp) || count == 0)
-		return -EINVAL;
-
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
-
-	temp[count] = '\0';
-
-	if (sscanf(temp, "%d", &i) != 1)
-		return -EINVAL;
+	ret = kstrtos32_from_user(buf, count, 10, &i);
+	if (unlikely(ret))
+		return ret;
 
 	/*
 	 * Don't do anything if the value hasn't actually changed.
@@ -2179,22 +2156,15 @@ static ssize_t smk_read_logging(struct file *filp, char __user *buf,
 static ssize_t smk_write_logging(struct file *file, const char __user *buf,
 				size_t count, loff_t *ppos)
 {
-	char temp[32];
-	int i;
+	int i, ret;
 
 	if (!smack_privileged(CAP_MAC_ADMIN))
 		return -EPERM;
 
-	if (count >= sizeof(temp) || count == 0)
-		return -EINVAL;
-
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
+	ret = kstrtos32_from_user(buf, count, 10, &i);
+	if (unlikely(ret))
+		return ret;
 
-	temp[count] = '\0';
-
-	if (sscanf(temp, "%d", &i) != 1)
-		return -EINVAL;
 	if (i < 0 || i > 3)
 		return -EINVAL;
 	log_policy = i;
@@ -2838,22 +2808,15 @@ static ssize_t smk_read_ptrace(struct file *filp, char __user *buf,
 static ssize_t smk_write_ptrace(struct file *file, const char __user *buf,
 				size_t count, loff_t *ppos)
 {
-	char temp[32];
-	int i;
+	int i, ret;
 
 	if (!smack_privileged(CAP_MAC_ADMIN))
 		return -EPERM;
 
-	if (*ppos != 0 || count >= sizeof(temp) || count == 0)
-		return -EINVAL;
-
-	if (copy_from_user(temp, buf, count) != 0)
-		return -EFAULT;
+	ret = kstrtos32_from_user(buf, count, 10, &i);
+	if (unlikely(ret))
+		return ret;
 
-	temp[count] = '\0';
-
-	if (sscanf(temp, "%d", &i) != 1)
-		return -EINVAL;
 	if (i < SMACK_PTRACE_DEFAULT || i > SMACK_PTRACE_MAX)
 		return -EINVAL;
 	smack_ptrace_rule = i;
diff --git a/sound/ac97/bus.c b/sound/ac97/bus.c
index 15487837e8946..a4d230a19c563 100644
--- a/sound/ac97/bus.c
+++ b/sound/ac97/bus.c
@@ -206,24 +206,6 @@ void snd_ac97_codec_driver_unregister(struct ac97_codec_driver *drv)
 }
 EXPORT_SYMBOL_GPL(snd_ac97_codec_driver_unregister);
 
-/**
- * snd_ac97_codec_get_platdata - get platform_data
- * @adev: the ac97 codec device
- *
- * For legacy platforms, in order to have platform_data in codec drivers
- * available, while ac97 device are auto-created upon probe, this retrieves the
- * platdata which was setup on ac97 controller registration.
- *
- * Returns the platform data pointer
- */
-void *snd_ac97_codec_get_platdata(const struct ac97_codec_device *adev)
-{
-	struct ac97_controller *ac97_ctrl = adev->ac97_ctrl;
-
-	return ac97_ctrl->codecs_pdata[adev->num];
-}
-EXPORT_SYMBOL_GPL(snd_ac97_codec_get_platdata);
-
 static void ac97_ctrl_codecs_unregister(struct ac97_controller *ac97_ctrl)
 {
 	int i;
@@ -337,7 +319,6 @@ static int ac97_add_adapter(struct ac97_controller *ac97_ctrl)
  * @dev: the device providing the ac97 DC function
  * @slots_available: mask of the ac97 codecs that can be scanned and probed
  *                   bit0 => codec 0, bit1 => codec 1 ... bit 3 => codec 3
- * @codecs_pdata: codec platform data
  *
  * Register a digital controller which can control up to 4 ac97 codecs. This is
  * the controller side of the AC97 AC-link, while the slave side are the codecs.
@@ -346,18 +327,15 @@ static int ac97_add_adapter(struct ac97_controller *ac97_ctrl)
  */
 struct ac97_controller *snd_ac97_controller_register(
 	const struct ac97_controller_ops *ops, struct device *dev,
-	unsigned short slots_available, void **codecs_pdata)
+	unsigned short slots_available)
 {
 	struct ac97_controller *ac97_ctrl;
-	int ret, i;
+	int ret;
 
 	ac97_ctrl = kzalloc_obj(*ac97_ctrl);
 	if (!ac97_ctrl)
 		return ERR_PTR(-ENOMEM);
 
-	for (i = 0; i < AC97_BUS_MAX_CODECS && codecs_pdata; i++)
-		ac97_ctrl->codecs_pdata[i] = codecs_pdata[i];
-
 	ac97_ctrl->ops = ops;
 	ac97_ctrl->slots_available = slots_available;
 	ac97_ctrl->parent = dev;
diff --git a/sound/ac97_bus.c b/sound/ac97_bus.c
index 8a44297964f50..ad7fb6b0c2c00 100644
--- a/sound/ac97_bus.c
+++ b/sound/ac97_bus.c
@@ -73,7 +73,6 @@ int snd_ac97_reset(struct snd_ac97 *ac97, bool try_warm, unsigned int id,
 
 	if (snd_ac97_check_id(ac97, id, id_mask))
 		return 0;
-
 	return -ENODEV;
 }
 EXPORT_SYMBOL_GPL(snd_ac97_reset);
@@ -81,6 +80,7 @@ EXPORT_SYMBOL_GPL(snd_ac97_reset);
 const struct bus_type ac97_bus_type = {
 	.name		= "ac97",
 };
+EXPORT_SYMBOL(ac97_bus_type);
 
 static int __init ac97_bus_init(void)
 {
@@ -96,7 +96,5 @@ static void __exit ac97_bus_exit(void)
 
 module_exit(ac97_bus_exit);
 
-EXPORT_SYMBOL(ac97_bus_type);
-
 MODULE_DESCRIPTION("Legacy AC97 bus interface");
 MODULE_LICENSE("GPL");
diff --git a/sound/arm/Kconfig b/sound/arm/Kconfig
index e4d7288d1e1e3..09054ce8074f0 100644
--- a/sound/arm/Kconfig
+++ b/sound/arm/Kconfig
@@ -19,10 +19,3 @@ config SND_ARMAACI
 	select SND_AC97_CODEC
 
 endif	# SND_ARM
-
-config SND_PXA2XX_LIB
-	tristate
-	select SND_DMAENGINE_PCM
-
-config SND_PXA2XX_LIB_AC97
-	bool
diff --git a/sound/arm/Makefile b/sound/arm/Makefile
index 99325a66cf771..6b91eb796b9b2 100644
--- a/sound/arm/Makefile
+++ b/sound/arm/Makefile
@@ -5,7 +5,3 @@
 
 obj-$(CONFIG_SND_ARMAACI)	+= snd-aaci.o
 snd-aaci-y			:= aaci.o
-
-obj-$(CONFIG_SND_PXA2XX_LIB)	+= snd-pxa2xx-lib.o
-snd-pxa2xx-lib-y		:= pxa2xx-pcm-lib.o
-snd-pxa2xx-lib-$(CONFIG_SND_PXA2XX_LIB_AC97)	+= pxa2xx-ac97-lib.o
diff --git a/sound/arm/pxa2xx-ac97-lib.c b/sound/arm/pxa2xx-ac97-lib.c
deleted file mode 100644
index 79eb557d4942f..0000000000000
--- a/sound/arm/pxa2xx-ac97-lib.c
+++ /dev/null
@@ -1,439 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/*
- * Based on sound/arm/pxa2xx-ac97.c and sound/soc/pxa/pxa2xx-ac97.c
- * which contain:
- *
- * Author:	Nicolas Pitre
- * Created:	Dec 02, 2004
- * Copyright:	MontaVista Software Inc.
- */
-
-#include <linux/kernel.h>
-#include <linux/platform_device.h>
-#include <linux/interrupt.h>
-#include <linux/clk.h>
-#include <linux/delay.h>
-#include <linux/gpio/consumer.h>
-#include <linux/module.h>
-#include <linux/io.h>
-#include <linux/soc/pxa/cpu.h>
-
-#include <sound/pxa2xx-lib.h>
-
-#include <linux/platform_data/asoc-pxa.h>
-
-#include "pxa2xx-ac97-regs.h"
-
-static DEFINE_MUTEX(car_mutex);
-static DECLARE_WAIT_QUEUE_HEAD(gsr_wq);
-static volatile long gsr_bits;
-static struct clk *ac97_clk;
-static struct clk *ac97conf_clk;
-static int reset_gpio;
-struct gpio_desc *rst_gpio;
-static void __iomem *ac97_reg_base;
-
-/*
- * Beware PXA27x bugs:
- *
- *   o Slot 12 read from modem space will hang controller.
- *   o CDONE, SDONE interrupt fails after any slot 12 IO.
- *
- * We therefore have an hybrid approach for waiting on SDONE (interrupt or
- * 1 jiffy timeout if interrupt never comes).
- */
-
-int pxa2xx_ac97_read(int slot, unsigned short reg)
-{
-	int val = -ENODEV;
-	u32 __iomem *reg_addr;
-
-	if (slot > 0)
-		return -ENODEV;
-
-	guard(mutex)(&car_mutex);
-
-	/* set up primary or secondary codec space */
-	if (cpu_is_pxa25x() && reg == AC97_GPIO_STATUS)
-		reg_addr = ac97_reg_base +
-			   (slot ? SMC_REG_BASE : PMC_REG_BASE);
-	else
-		reg_addr = ac97_reg_base +
-			   (slot ? SAC_REG_BASE : PAC_REG_BASE);
-	reg_addr += (reg >> 1);
-
-	/* start read access across the ac97 link */
-	writel(GSR_CDONE | GSR_SDONE, ac97_reg_base + GSR);
-	gsr_bits = 0;
-	val = (readl(reg_addr) & 0xffff);
-	if (reg == AC97_GPIO_STATUS)
-		return val;
-	if (wait_event_timeout(gsr_wq, (readl(ac97_reg_base + GSR) | gsr_bits) & GSR_SDONE, 1) <= 0 &&
-	    !((readl(ac97_reg_base + GSR) | gsr_bits) & GSR_SDONE)) {
-		printk(KERN_ERR "%s: read error (ac97_reg=%d GSR=%#lx)\n",
-				__func__, reg, readl(ac97_reg_base + GSR) | gsr_bits);
-		return -ETIMEDOUT;
-	}
-
-	/* valid data now */
-	writel(GSR_CDONE | GSR_SDONE, ac97_reg_base + GSR);
-	gsr_bits = 0;
-	val = (readl(reg_addr) & 0xffff);
-	/* but we've just started another cycle... */
-	wait_event_timeout(gsr_wq, (readl(ac97_reg_base + GSR) | gsr_bits) & GSR_SDONE, 1);
-	return val;
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_read);
-
-int pxa2xx_ac97_write(int slot, unsigned short reg, unsigned short val)
-{
-	u32 __iomem *reg_addr;
-	int ret = 0;
-
-	guard(mutex)(&car_mutex);
-
-	/* set up primary or secondary codec space */
-	if (cpu_is_pxa25x() && reg == AC97_GPIO_STATUS)
-		reg_addr = ac97_reg_base +
-			   (slot ? SMC_REG_BASE : PMC_REG_BASE);
-	else
-		reg_addr = ac97_reg_base +
-			   (slot ? SAC_REG_BASE : PAC_REG_BASE);
-	reg_addr += (reg >> 1);
-
-	writel(GSR_CDONE | GSR_SDONE, ac97_reg_base + GSR);
-	gsr_bits = 0;
-	writel(val, reg_addr);
-	if (wait_event_timeout(gsr_wq, (readl(ac97_reg_base + GSR) | gsr_bits) & GSR_CDONE, 1) <= 0 &&
-	    !((readl(ac97_reg_base + GSR) | gsr_bits) & GSR_CDONE)) {
-		printk(KERN_ERR "%s: write error (ac97_reg=%d GSR=%#lx)\n",
-				__func__, reg, readl(ac97_reg_base + GSR) | gsr_bits);
-		ret = -EIO;
-	}
-
-	return ret;
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_write);
-
-#ifdef CONFIG_PXA25x
-static inline void pxa_ac97_warm_pxa25x(void)
-{
-	gsr_bits = 0;
-
-	writel(readl(ac97_reg_base + GCR) | (GCR_WARM_RST), ac97_reg_base + GCR);
-}
-
-static inline void pxa_ac97_cold_pxa25x(void)
-{
-	writel(readl(ac97_reg_base + GCR) & ( GCR_COLD_RST), ac97_reg_base + GCR);  /* clear everything but nCRST */
-	writel(readl(ac97_reg_base + GCR) & (~GCR_COLD_RST), ac97_reg_base + GCR);  /* then assert nCRST */
-
-	gsr_bits = 0;
-
-	writel(GCR_COLD_RST, ac97_reg_base + GCR);
-}
-#endif
-
-#ifdef CONFIG_PXA27x
-static inline void pxa_ac97_warm_pxa27x(void)
-{
-	gsr_bits = 0;
-
-	/* warm reset broken on Bulverde, so manually keep AC97 reset high */
-	pxa27x_configure_ac97reset(reset_gpio, true);
-	udelay(10);
-	writel(readl(ac97_reg_base + GCR) | (GCR_WARM_RST), ac97_reg_base + GCR);
-	pxa27x_configure_ac97reset(reset_gpio, false);
-	udelay(500);
-}
-
-static inline void pxa_ac97_cold_pxa27x(void)
-{
-	writel(readl(ac97_reg_base + GCR) & ( GCR_COLD_RST), ac97_reg_base + GCR);  /* clear everything but nCRST */
-	writel(readl(ac97_reg_base + GCR) & (~GCR_COLD_RST), ac97_reg_base + GCR);  /* then assert nCRST */
-
-	gsr_bits = 0;
-
-	/* PXA27x Developers Manual section 13.5.2.2.1 */
-	clk_prepare_enable(ac97conf_clk);
-	udelay(5);
-	clk_disable_unprepare(ac97conf_clk);
-	writel(GCR_COLD_RST | GCR_WARM_RST, ac97_reg_base + GCR);
-}
-#endif
-
-#ifdef CONFIG_PXA3xx
-static inline void pxa_ac97_warm_pxa3xx(void)
-{
-	gsr_bits = 0;
-
-	/* Can't use interrupts */
-	writel(readl(ac97_reg_base + GCR) | (GCR_WARM_RST), ac97_reg_base + GCR);
-}
-
-static inline void pxa_ac97_cold_pxa3xx(void)
-{
-	/* Hold CLKBPB for 100us */
-	writel(0, ac97_reg_base + GCR);
-	writel(GCR_CLKBPB, ac97_reg_base + GCR);
-	udelay(100);
-	writel(0, ac97_reg_base + GCR);
-
-	writel(readl(ac97_reg_base + GCR) & ( GCR_COLD_RST), ac97_reg_base + GCR);  /* clear everything but nCRST */
-	writel(readl(ac97_reg_base + GCR) & (~GCR_COLD_RST), ac97_reg_base + GCR);  /* then assert nCRST */
-
-	gsr_bits = 0;
-
-	/* Can't use interrupts on PXA3xx */
-	writel(readl(ac97_reg_base + GCR) & (~(GCR_PRIRDY_IEN|GCR_SECRDY_IEN)), ac97_reg_base + GCR);
-
-	writel(GCR_WARM_RST | GCR_COLD_RST, ac97_reg_base + GCR);
-}
-#endif
-
-bool pxa2xx_ac97_try_warm_reset(void)
-{
-	unsigned long gsr;
-	unsigned int timeout = 100;
-
-#ifdef CONFIG_PXA25x
-	if (cpu_is_pxa25x())
-		pxa_ac97_warm_pxa25x();
-	else
-#endif
-#ifdef CONFIG_PXA27x
-	if (cpu_is_pxa27x())
-		pxa_ac97_warm_pxa27x();
-	else
-#endif
-#ifdef CONFIG_PXA3xx
-	if (cpu_is_pxa3xx())
-		pxa_ac97_warm_pxa3xx();
-	else
-#endif
-		snd_BUG();
-
-	while (!((readl(ac97_reg_base + GSR) | gsr_bits) & (GSR_PCR | GSR_SCR)) && timeout--)
-		mdelay(1);
-
-	gsr = readl(ac97_reg_base + GSR) | gsr_bits;
-	if (!(gsr & (GSR_PCR | GSR_SCR))) {
-		printk(KERN_INFO "%s: warm reset timeout (GSR=%#lx)\n",
-				 __func__, gsr);
-
-		return false;
-	}
-
-	return true;
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_try_warm_reset);
-
-bool pxa2xx_ac97_try_cold_reset(void)
-{
-	unsigned long gsr;
-	unsigned int timeout = 1000;
-
-#ifdef CONFIG_PXA25x
-	if (cpu_is_pxa25x())
-		pxa_ac97_cold_pxa25x();
-	else
-#endif
-#ifdef CONFIG_PXA27x
-	if (cpu_is_pxa27x())
-		pxa_ac97_cold_pxa27x();
-	else
-#endif
-#ifdef CONFIG_PXA3xx
-	if (cpu_is_pxa3xx())
-		pxa_ac97_cold_pxa3xx();
-	else
-#endif
-		snd_BUG();
-
-	while (!((readl(ac97_reg_base + GSR) | gsr_bits) & (GSR_PCR | GSR_SCR)) && timeout--)
-		mdelay(1);
-
-	gsr = readl(ac97_reg_base + GSR) | gsr_bits;
-	if (!(gsr & (GSR_PCR | GSR_SCR))) {
-		printk(KERN_INFO "%s: cold reset timeout (GSR=%#lx)\n",
-				 __func__, gsr);
-
-		return false;
-	}
-
-	return true;
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_try_cold_reset);
-
-
-void pxa2xx_ac97_finish_reset(void)
-{
-	u32 gcr = readl(ac97_reg_base + GCR);
-	gcr &= ~(GCR_PRIRDY_IEN|GCR_SECRDY_IEN);
-	gcr |= GCR_SDONE_IE|GCR_CDONE_IE;
-	writel(gcr, ac97_reg_base + GCR);
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_finish_reset);
-
-static irqreturn_t pxa2xx_ac97_irq(int irq, void *dev_id)
-{
-	long status;
-
-	status = readl(ac97_reg_base + GSR);
-	if (status) {
-		writel(status, ac97_reg_base + GSR);
-		gsr_bits |= status;
-		wake_up(&gsr_wq);
-
-		/* Although we don't use those we still need to clear them
-		   since they tend to spuriously trigger when MMC is used
-		   (hardware bug? go figure)... */
-		if (cpu_is_pxa27x()) {
-			writel(MISR_EOC, ac97_reg_base + MISR);
-			writel(PISR_EOC, ac97_reg_base + PISR);
-			writel(MCSR_EOC, ac97_reg_base + MCSR);
-		}
-
-		return IRQ_HANDLED;
-	}
-
-	return IRQ_NONE;
-}
-
-#ifdef CONFIG_PM
-int pxa2xx_ac97_hw_suspend(void)
-{
-	writel(readl(ac97_reg_base + GCR) | (GCR_ACLINK_OFF), ac97_reg_base + GCR);
-	clk_disable_unprepare(ac97_clk);
-	return 0;
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_hw_suspend);
-
-int pxa2xx_ac97_hw_resume(void)
-{
-	clk_prepare_enable(ac97_clk);
-	return 0;
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_hw_resume);
-#endif
-
-int pxa2xx_ac97_hw_probe(struct platform_device *dev)
-{
-	int ret;
-	int irq;
-
-	ac97_reg_base = devm_platform_ioremap_resource(dev, 0);
-	if (IS_ERR(ac97_reg_base)) {
-		dev_err(&dev->dev, "Missing MMIO resource\n");
-		return PTR_ERR(ac97_reg_base);
-	}
-
-	if (dev->dev.of_node) {
-		/* Assert reset using GPIOD_OUT_HIGH, because reset is GPIO_ACTIVE_LOW */
-		rst_gpio = devm_gpiod_get(&dev->dev, "reset", GPIOD_OUT_HIGH);
-		if (IS_ERR(rst_gpio)) {
-			ret = PTR_ERR(rst_gpio);
-			if (ret == -ENOENT)
-				reset_gpio = -1;
-			else if (ret)
-				return ret;
-		} else {
-			reset_gpio = desc_to_gpio(rst_gpio);
-		}
-	} else {
-		if (cpu_is_pxa27x())
-			reset_gpio = 113;
-	}
-
-	if (cpu_is_pxa27x()) {
-		/*
-		 * This gpio is needed for a work-around to a bug in the ac97
-		 * controller during warm reset.  The direction and level is set
-		 * here so that it is an output driven high when switching from
-		 * AC97_nRESET alt function to generic gpio.
-		 */
-		gpiod_set_consumer_name(rst_gpio, "pxa27x ac97 reset");
-		pxa27x_configure_ac97reset(reset_gpio, false);
-
-		ac97conf_clk = clk_get(&dev->dev, "AC97CONFCLK");
-		if (IS_ERR(ac97conf_clk)) {
-			ret = PTR_ERR(ac97conf_clk);
-			ac97conf_clk = NULL;
-			goto err_conf;
-		}
-	}
-
-	ac97_clk = clk_get(&dev->dev, "AC97CLK");
-	if (IS_ERR(ac97_clk)) {
-		ret = PTR_ERR(ac97_clk);
-		ac97_clk = NULL;
-		goto err_clk;
-	}
-
-	ret = clk_prepare_enable(ac97_clk);
-	if (ret)
-		goto err_clk2;
-
-	irq = platform_get_irq(dev, 0);
-	if (irq < 0) {
-		ret = irq;
-		goto err_irq;
-	}
-
-	ret = request_irq(irq, pxa2xx_ac97_irq, 0, "AC97", NULL);
-	if (ret < 0)
-		goto err_irq;
-
-	return 0;
-
-err_irq:
-	writel(readl(ac97_reg_base + GCR) | (GCR_ACLINK_OFF), ac97_reg_base + GCR);
-err_clk2:
-	clk_put(ac97_clk);
-	ac97_clk = NULL;
-err_clk:
-	if (ac97conf_clk) {
-		clk_put(ac97conf_clk);
-		ac97conf_clk = NULL;
-	}
-err_conf:
-	return ret;
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_hw_probe);
-
-void pxa2xx_ac97_hw_remove(struct platform_device *dev)
-{
-	writel(readl(ac97_reg_base + GCR) | (GCR_ACLINK_OFF), ac97_reg_base + GCR);
-	free_irq(platform_get_irq(dev, 0), NULL);
-	if (ac97conf_clk) {
-		clk_put(ac97conf_clk);
-		ac97conf_clk = NULL;
-	}
-	clk_disable_unprepare(ac97_clk);
-	clk_put(ac97_clk);
-	ac97_clk = NULL;
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_hw_remove);
-
-u32 pxa2xx_ac97_read_modr(void)
-{
-	if (!ac97_reg_base)
-		return 0;
-
-	return readl(ac97_reg_base + MODR);
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_read_modr);
-
-u32 pxa2xx_ac97_read_misr(void)
-{
-	if (!ac97_reg_base)
-		return 0;
-
-	return readl(ac97_reg_base + MISR);
-}
-EXPORT_SYMBOL_GPL(pxa2xx_ac97_read_misr);
-
-MODULE_AUTHOR("Nicolas Pitre");
-MODULE_DESCRIPTION("Intel/Marvell PXA sound library");
-MODULE_LICENSE("GPL");
-
diff --git a/sound/arm/pxa2xx-ac97-regs.h b/sound/arm/pxa2xx-ac97-regs.h
deleted file mode 100644
index ae638a1b919b7..0000000000000
--- a/sound/arm/pxa2xx-ac97-regs.h
+++ /dev/null
@@ -1,100 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef __ASM_ARCH_REGS_AC97_H
-#define __ASM_ARCH_REGS_AC97_H
-
-/*
- * AC97 Controller registers
- */
-
-#define POCR		(0x0000)  	/* PCM Out Control Register */
-#define POCR_FEIE	(1 << 3)	/* FIFO Error Interrupt Enable */
-#define POCR_FSRIE	(1 << 1)	/* FIFO Service Request Interrupt Enable */
-
-#define PICR		(0x0004) 	/* PCM In Control Register */
-#define PICR_FEIE	(1 << 3)	/* FIFO Error Interrupt Enable */
-#define PICR_FSRIE	(1 << 1)	/* FIFO Service Request Interrupt Enable */
-
-#define MCCR		(0x0008)  	/* Mic In Control Register */
-#define MCCR_FEIE	(1 << 3)	/* FIFO Error Interrupt Enable */
-#define MCCR_FSRIE	(1 << 1)	/* FIFO Service Request Interrupt Enable */
-
-#define GCR		(0x000C) 	 /* Global Control Register */
-#ifdef CONFIG_PXA3xx
-#define GCR_CLKBPB	(1 << 31)	/* Internal clock enable */
-#endif
-#define GCR_nDMAEN	(1 << 24)	/* non DMA Enable */
-#define GCR_CDONE_IE	(1 << 19)	/* Command Done Interrupt Enable */
-#define GCR_SDONE_IE	(1 << 18)	/* Status Done Interrupt Enable */
-#define GCR_SECRDY_IEN	(1 << 9)	/* Secondary Ready Interrupt Enable */
-#define GCR_PRIRDY_IEN	(1 << 8)	/* Primary Ready Interrupt Enable */
-#define GCR_SECRES_IEN	(1 << 5)	/* Secondary Resume Interrupt Enable */
-#define GCR_PRIRES_IEN	(1 << 4)	/* Primary Resume Interrupt Enable */
-#define GCR_ACLINK_OFF	(1 << 3)	/* AC-link Shut Off */
-#define GCR_WARM_RST	(1 << 2)	/* AC97 Warm Reset */
-#define GCR_COLD_RST	(1 << 1)	/* AC'97 Cold Reset (0 = active) */
-#define GCR_GIE		(1 << 0)	/* Codec GPI Interrupt Enable */
-
-#define POSR		(0x0010)  	/* PCM Out Status Register */
-#define POSR_FIFOE	(1 << 4)	/* FIFO error */
-#define POSR_FSR	(1 << 2)	/* FIFO Service Request */
-
-#define PISR		(0x0014)  	/* PCM In Status Register */
-#define PISR_FIFOE	(1 << 4)	/* FIFO error */
-#define PISR_EOC	(1 << 3)	/* DMA End-of-Chain (exclusive clear) */
-#define PISR_FSR	(1 << 2)	/* FIFO Service Request */
-
-#define MCSR		(0x0018)  	/* Mic In Status Register */
-#define MCSR_FIFOE	(1 << 4)	/* FIFO error */
-#define MCSR_EOC	(1 << 3)	/* DMA End-of-Chain (exclusive clear) */
-#define MCSR_FSR	(1 << 2)	/* FIFO Service Request */
-
-#define GSR		(0x001C)  	/* Global Status Register */
-#define GSR_CDONE	(1 << 19)	/* Command Done */
-#define GSR_SDONE	(1 << 18)	/* Status Done */
-#define GSR_RDCS	(1 << 15)	/* Read Completion Status */
-#define GSR_BIT3SLT12	(1 << 14)	/* Bit 3 of slot 12 */
-#define GSR_BIT2SLT12	(1 << 13)	/* Bit 2 of slot 12 */
-#define GSR_BIT1SLT12	(1 << 12)	/* Bit 1 of slot 12 */
-#define GSR_SECRES	(1 << 11)	/* Secondary Resume Interrupt */
-#define GSR_PRIRES	(1 << 10)	/* Primary Resume Interrupt */
-#define GSR_SCR		(1 << 9)	/* Secondary Codec Ready */
-#define GSR_PCR		(1 << 8)	/*  Primary Codec Ready */
-#define GSR_MCINT	(1 << 7)	/* Mic In Interrupt */
-#define GSR_POINT	(1 << 6)	/* PCM Out Interrupt */
-#define GSR_PIINT	(1 << 5)	/* PCM In Interrupt */
-#define GSR_ACOFFD	(1 << 3)	/* AC-link Shut Off Done */
-#define GSR_MOINT	(1 << 2)	/* Modem Out Interrupt */
-#define GSR_MIINT	(1 << 1)	/* Modem In Interrupt */
-#define GSR_GSCI	(1 << 0)	/* Codec GPI Status Change Interrupt */
-
-#define CAR		(0x0020)  	/* CODEC Access Register */
-#define CAR_CAIP	(1 << 0)	/* Codec Access In Progress */
-
-#define PCDR		(0x0040)  	/* PCM FIFO Data Register */
-#define MCDR		(0x0060)  	/* Mic-in FIFO Data Register */
-
-#define MOCR		(0x0100)  	/* Modem Out Control Register */
-#define MOCR_FEIE	(1 << 3)	/* FIFO Error */
-#define MOCR_FSRIE	(1 << 1)	/* FIFO Service Request Interrupt Enable */
-
-#define MICR		(0x0108)  	/* Modem In Control Register */
-#define MICR_FEIE	(1 << 3)	/* FIFO Error */
-#define MICR_FSRIE	(1 << 1)	/* FIFO Service Request Interrupt Enable */
-
-#define MOSR		(0x0110)  	/* Modem Out Status Register */
-#define MOSR_FIFOE	(1 << 4)	/* FIFO error */
-#define MOSR_FSR	(1 << 2)	/* FIFO Service Request */
-
-#define MISR		(0x0118)  	/* Modem In Status Register */
-#define MISR_FIFOE	(1 << 4)	/* FIFO error */
-#define MISR_EOC	(1 << 3)	/* DMA End-of-Chain (exclusive clear) */
-#define MISR_FSR	(1 << 2)	/* FIFO Service Request */
-
-#define MODR		(0x0140)  	/* Modem FIFO Data Register */
-
-#define PAC_REG_BASE	(0x0200)  	/* Primary Audio Codec */
-#define SAC_REG_BASE	(0x0300)  	/* Secondary Audio Codec */
-#define PMC_REG_BASE	(0x0400)  	/* Primary Modem Codec */
-#define SMC_REG_BASE	(0x0500)  	/* Secondary Modem Codec */
-
-#endif /* __ASM_ARCH_REGS_AC97_H */
diff --git a/sound/arm/pxa2xx-pcm-lib.c b/sound/arm/pxa2xx-pcm-lib.c
deleted file mode 100644
index 571e9d909cdf0..0000000000000
--- a/sound/arm/pxa2xx-pcm-lib.c
+++ /dev/null
@@ -1,195 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-
-#include <linux/slab.h>
-#include <linux/module.h>
-#include <linux/dma-mapping.h>
-#include <linux/dmaengine.h>
-#include <linux/dma/pxa-dma.h>
-
-#include <sound/core.h>
-#include <sound/pcm.h>
-#include <sound/pcm_params.h>
-#include <sound/pxa2xx-lib.h>
-#include <sound/dmaengine_pcm.h>
-
-static const struct snd_pcm_hardware pxa2xx_pcm_hardware = {
-	.info			= SNDRV_PCM_INFO_MMAP |
-				  SNDRV_PCM_INFO_MMAP_VALID |
-				  SNDRV_PCM_INFO_INTERLEAVED |
-				  SNDRV_PCM_INFO_PAUSE |
-				  SNDRV_PCM_INFO_RESUME,
-	.formats		= SNDRV_PCM_FMTBIT_S16_LE |
-				  SNDRV_PCM_FMTBIT_S24_LE |
-				  SNDRV_PCM_FMTBIT_S32_LE,
-	.period_bytes_min	= 32,
-	.period_bytes_max	= 8192 - 32,
-	.periods_min		= 1,
-	.periods_max		= 256,
-	.buffer_bytes_max	= 128 * 1024,
-	.fifo_size		= 32,
-};
-
-int pxa2xx_pcm_hw_params(struct snd_pcm_substream *substream,
-			 struct snd_pcm_hw_params *params)
-{
-	struct dma_chan *chan = snd_dmaengine_pcm_get_chan(substream);
-	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
-	struct snd_dmaengine_dai_dma_data *dma_params;
-	struct dma_slave_config config;
-	int ret;
-
-	dma_params = snd_soc_dai_get_dma_data(snd_soc_rtd_to_cpu(rtd, 0), substream);
-	if (!dma_params)
-		return 0;
-
-	ret = snd_hwparams_to_dma_slave_config(substream, params, &config);
-	if (ret)
-		return ret;
-
-	snd_dmaengine_pcm_set_config_from_dai_data(substream,
-			snd_soc_dai_get_dma_data(snd_soc_rtd_to_cpu(rtd, 0), substream),
-			&config);
-
-	ret = dmaengine_slave_config(chan, &config);
-	if (ret)
-		return ret;
-
-	return 0;
-}
-EXPORT_SYMBOL(pxa2xx_pcm_hw_params);
-
-int pxa2xx_pcm_trigger(struct snd_pcm_substream *substream, int cmd)
-{
-	return snd_dmaengine_pcm_trigger(substream, cmd);
-}
-EXPORT_SYMBOL(pxa2xx_pcm_trigger);
-
-snd_pcm_uframes_t
-pxa2xx_pcm_pointer(struct snd_pcm_substream *substream)
-{
-	return snd_dmaengine_pcm_pointer(substream);
-}
-EXPORT_SYMBOL(pxa2xx_pcm_pointer);
-
-int pxa2xx_pcm_prepare(struct snd_pcm_substream *substream)
-{
-	return 0;
-}
-EXPORT_SYMBOL(pxa2xx_pcm_prepare);
-
-int pxa2xx_pcm_open(struct snd_pcm_substream *substream)
-{
-	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
-	struct snd_pcm_runtime *runtime = substream->runtime;
-	struct snd_dmaengine_dai_dma_data *dma_params;
-	int ret;
-
-	runtime->hw = pxa2xx_pcm_hardware;
-
-	dma_params = snd_soc_dai_get_dma_data(snd_soc_rtd_to_cpu(rtd, 0), substream);
-	if (!dma_params)
-		return 0;
-
-	/*
-	 * For mysterious reasons (and despite what the manual says)
-	 * playback samples are lost if the DMA count is not a multiple
-	 * of the DMA burst size.  Let's add a rule to enforce that.
-	 */
-	ret = snd_pcm_hw_constraint_step(runtime, 0,
-		SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 32);
-	if (ret)
-		return ret;
-
-	ret = snd_pcm_hw_constraint_step(runtime, 0,
-		SNDRV_PCM_HW_PARAM_BUFFER_BYTES, 32);
-	if (ret)
-		return ret;
-
-	ret = snd_pcm_hw_constraint_integer(runtime,
-					    SNDRV_PCM_HW_PARAM_PERIODS);
-	if (ret < 0)
-		return ret;
-
-	return snd_dmaengine_pcm_open(
-		substream, dma_request_slave_channel(snd_soc_rtd_to_cpu(rtd, 0)->dev,
-						     dma_params->chan_name));
-}
-EXPORT_SYMBOL(pxa2xx_pcm_open);
-
-int pxa2xx_pcm_close(struct snd_pcm_substream *substream)
-{
-	return snd_dmaengine_pcm_close_release_chan(substream);
-}
-EXPORT_SYMBOL(pxa2xx_pcm_close);
-
-int pxa2xx_pcm_preallocate_dma_buffer(struct snd_pcm *pcm)
-{
-	size_t size = pxa2xx_pcm_hardware.buffer_bytes_max;
-
-	return snd_pcm_set_fixed_buffer_all(pcm, SNDRV_DMA_TYPE_DEV_WC,
-					    pcm->card->dev, size);
-}
-EXPORT_SYMBOL(pxa2xx_pcm_preallocate_dma_buffer);
-
-int pxa2xx_soc_pcm_new(struct snd_soc_component *component,
-		       struct snd_soc_pcm_runtime *rtd)
-{
-	struct snd_card *card = rtd->card->snd_card;
-	struct snd_pcm *pcm = rtd->pcm;
-	int ret;
-
-	ret = dma_coerce_mask_and_coherent(card->dev, DMA_BIT_MASK(32));
-	if (ret)
-		return ret;
-
-	return pxa2xx_pcm_preallocate_dma_buffer(pcm);
-}
-EXPORT_SYMBOL(pxa2xx_soc_pcm_new);
-
-int pxa2xx_soc_pcm_open(struct snd_soc_component *component,
-			struct snd_pcm_substream *substream)
-{
-	return pxa2xx_pcm_open(substream);
-}
-EXPORT_SYMBOL(pxa2xx_soc_pcm_open);
-
-int pxa2xx_soc_pcm_close(struct snd_soc_component *component,
-			 struct snd_pcm_substream *substream)
-{
-	return pxa2xx_pcm_close(substream);
-}
-EXPORT_SYMBOL(pxa2xx_soc_pcm_close);
-
-int pxa2xx_soc_pcm_hw_params(struct snd_soc_component *component,
-			     struct snd_pcm_substream *substream,
-			     struct snd_pcm_hw_params *params)
-{
-	return pxa2xx_pcm_hw_params(substream, params);
-}
-EXPORT_SYMBOL(pxa2xx_soc_pcm_hw_params);
-
-int pxa2xx_soc_pcm_prepare(struct snd_soc_component *component,
-			   struct snd_pcm_substream *substream)
-{
-	return pxa2xx_pcm_prepare(substream);
-}
-EXPORT_SYMBOL(pxa2xx_soc_pcm_prepare);
-
-int pxa2xx_soc_pcm_trigger(struct snd_soc_component *component,
-			   struct snd_pcm_substream *substream, int cmd)
-{
-	return pxa2xx_pcm_trigger(substream, cmd);
-}
-EXPORT_SYMBOL(pxa2xx_soc_pcm_trigger);
-
-snd_pcm_uframes_t
-pxa2xx_soc_pcm_pointer(struct snd_soc_component *component,
-		       struct snd_pcm_substream *substream)
-{
-	return pxa2xx_pcm_pointer(substream);
-}
-EXPORT_SYMBOL(pxa2xx_soc_pcm_pointer);
-
-MODULE_AUTHOR("Nicolas Pitre");
-MODULE_DESCRIPTION("Intel PXA2xx sound library");
-MODULE_LICENSE("GPL");
diff --git a/sound/core/jack.c b/sound/core/jack.c
index 5e8a2f3f41966..96e0733ede778 100644
--- a/sound/core/jack.c
+++ b/sound/core/jack.c
@@ -250,18 +250,15 @@ static const char * const jack_events_name[] = {
 /* the recommended buffer size is 256 */
 static int parse_mask_bits(unsigned int mask_bits, char *buf, size_t buf_size)
 {
+	int len = scnprintf(buf, buf_size, "0x%04x", mask_bits);
 	int i;
 
-	scnprintf(buf, buf_size, "0x%04x", mask_bits);
-
 	for (i = 0; i < ARRAY_SIZE(jack_events_name); i++)
-		if (mask_bits & (1 << i)) {
-			strlcat(buf, " ", buf_size);
-			strlcat(buf, jack_events_name[i], buf_size);
-		}
-	strlcat(buf, "\n", buf_size);
+		if (mask_bits & (1 << i))
+			len += scnprintf(buf + len, buf_size - len, " %s", jack_events_name[i]);
+	len += scnprintf(buf + len, buf_size - len, "\n");
 
-	return strlen(buf);
+	return len;
 }
 
 static ssize_t jack_kctl_mask_bits_read(struct file *file,
diff --git a/sound/core/oss/pcm_plugin.c b/sound/core/oss/pcm_plugin.c
index 14b4a390a2192..5f4d6945a7df6 100644
--- a/sound/core/oss/pcm_plugin.c
+++ b/sound/core/oss/pcm_plugin.c
@@ -146,7 +146,7 @@ int snd_pcm_plugin_build(struct snd_pcm_substream *plug,
 		return -ENXIO;
 	if (snd_BUG_ON(!src_format || !dst_format))
 		return -ENXIO;
-	plugin = kzalloc(sizeof(*plugin) + extra, GFP_KERNEL);
+	plugin = kzalloc_flex(*plugin, extra_data, extra);
 	if (plugin == NULL)
 		return -ENOMEM;
 	plugin->name = name;
diff --git a/sound/core/pcm_drm_eld.c b/sound/core/pcm_drm_eld.c
index cb2eebaac85fd..1941ee5200639 100644
--- a/sound/core/pcm_drm_eld.c
+++ b/sound/core/pcm_drm_eld.c
@@ -334,7 +334,7 @@ int snd_parse_eld(struct device *dev, struct snd_parsed_hdmi_eld *e,
 	e->eld_ver = GRAB_BITS(buf, 0, 3, 5);
 	if (e->eld_ver != ELD_VER_CEA_861D &&
 	    e->eld_ver != ELD_VER_PARTIAL) {
-		dev_info(dev, "HDMI: Unknown ELD version %d\n", e->eld_ver);
+		dev_info_ratelimited(dev, "HDMI: Unknown ELD version %d\n", e->eld_ver);
 		goto out_fail;
 	}
 
@@ -357,7 +357,7 @@ int snd_parse_eld(struct device *dev, struct snd_parsed_hdmi_eld *e,
 	e->product_id	  = get_unaligned_le16(buf + 18);
 
 	if (mnl > ELD_MAX_MNL) {
-		dev_info(dev, "HDMI: MNL is reserved value %d\n", mnl);
+		dev_info_ratelimited(dev, "HDMI: MNL is reserved value %d\n", mnl);
 		goto out_fail;
 	} else if (ELD_FIXED_BYTES + mnl > size) {
 		dev_info(dev, "HDMI: out of range MNL %d\n", mnl);
diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c
index 09c421cd9319e..fe597f7d522de 100644
--- a/sound/core/pcm_lib.c
+++ b/sound/core/pcm_lib.c
@@ -2138,6 +2138,9 @@ static int interleaved_copy(struct snd_pcm_substream *substream,
 	off = frames_to_bytes(runtime, off);
 	frames = frames_to_bytes(runtime, frames);
 
+	if (!data)
+		return fill_silence(substream, 0, hwoff, NULL, frames);
+
 	return do_transfer(substream, 0, hwoff, data + off, frames, transfer,
 			   in_kernel);
 }
diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c
index a541bb235cfa1..50da38b141cb7 100644
--- a/sound/core/pcm_native.c
+++ b/sound/core/pcm_native.c
@@ -1973,13 +1973,15 @@ static int snd_pcm_reset(struct snd_pcm_substream *substream)
 static int snd_pcm_pre_prepare(struct snd_pcm_substream *substream,
 			       snd_pcm_state_t state)
 {
-	struct snd_pcm_runtime *runtime = substream->runtime;
+	snd_pcm_state_t cur_state = snd_pcm_get_state(substream);
 	int f_flags = (__force int)state;
 
-	if (runtime->state == SNDRV_PCM_STATE_OPEN ||
-	    runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (cur_state == SNDRV_PCM_STATE_OPEN ||
+	    cur_state == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
-	if (snd_pcm_running(substream))
+	if (cur_state == SNDRV_PCM_STATE_RUNNING ||
+	    (cur_state == SNDRV_PCM_STATE_DRAINING &&
+	     substream->stream == SNDRV_PCM_STREAM_PLAYBACK))
 		return -EBUSY;
 	substream->f_flags = f_flags;
 	return 0;
@@ -2139,7 +2141,7 @@ static int snd_pcm_drain(struct snd_pcm_substream *substream,
 	card = substream->pcm->card;
 	runtime = substream->runtime;
 
-	if (runtime->state == SNDRV_PCM_STATE_OPEN)
+	if (snd_pcm_get_state(substream) == SNDRV_PCM_STATE_OPEN)
 		return -EBADFD;
 
 	if (file) {
@@ -3524,7 +3526,7 @@ int snd_pcm_kernel_ioctl(struct snd_pcm_substream *substream,
 	snd_pcm_uframes_t *frames = arg;
 	snd_pcm_sframes_t result;
 	
-	if (substream->runtime->state == SNDRV_PCM_STATE_DISCONNECTED)
+	if (snd_pcm_get_state(substream) == SNDRV_PCM_STATE_DISCONNECTED)
 		return -EBADFD;
 
 	switch (cmd) {
diff --git a/sound/core/seq/oss/seq_oss_readq.c b/sound/core/seq/oss/seq_oss_readq.c
index c880d47711698..6b474615ced89 100644
--- a/sound/core/seq/oss/seq_oss_readq.c
+++ b/sound/core/seq/oss/seq_oss_readq.c
@@ -34,16 +34,10 @@ snd_seq_oss_readq_new(struct seq_oss_devinfo *dp, int maxlen)
 {
 	struct seq_oss_readq *q;
 
-	q = kzalloc_obj(*q);
+	q = kzalloc_flex(*q, q, maxlen);
 	if (!q)
 		return NULL;
 
-	q->q = kzalloc_objs(union evrec, maxlen);
-	if (!q->q) {
-		kfree(q);
-		return NULL;
-	}
-
 	q->maxlen = maxlen;
 	q->qlen = 0;
 	q->head = q->tail = 0;
@@ -61,10 +55,7 @@ snd_seq_oss_readq_new(struct seq_oss_devinfo *dp, int maxlen)
 void
 snd_seq_oss_readq_delete(struct seq_oss_readq *q)
 {
-	if (q) {
-		kfree(q->q);
-		kfree(q);
-	}
+	kfree(q);
 }
 
 /*
diff --git a/sound/core/seq/oss/seq_oss_readq.h b/sound/core/seq/oss/seq_oss_readq.h
index 38d0c4682b298..d8e1e7504d8fb 100644
--- a/sound/core/seq/oss/seq_oss_readq.h
+++ b/sound/core/seq/oss/seq_oss_readq.h
@@ -10,13 +10,13 @@
 #define __SEQ_OSS_READQ_H
 
 #include "seq_oss_device.h"
+#include "seq_oss_event.h"
 
 
 /*
  * definition of read queue
  */
 struct seq_oss_readq {
-	union evrec *q;
 	int qlen;
 	int maxlen;
 	int head, tail;
@@ -24,6 +24,7 @@ struct seq_oss_readq {
 	unsigned long input_time;
 	wait_queue_head_t midi_sleep;
 	spinlock_t lock;
+	union evrec q[] __counted_by(maxlen);
 };
 
 struct seq_oss_readq *snd_seq_oss_readq_new(struct seq_oss_devinfo *dp, int maxlen);
diff --git a/sound/core/seq/seq_clientmgr.c b/sound/core/seq/seq_clientmgr.c
index 5719637575a91..19d6fea012f6a 100644
--- a/sound/core/seq/seq_clientmgr.c
+++ b/sound/core/seq/seq_clientmgr.c
@@ -1287,7 +1287,7 @@ static int snd_seq_ioctl_create_port(struct snd_seq_client *client, void *arg)
 		port_idx = -1;
 	if (port_idx >= SNDRV_SEQ_ADDRESS_UNKNOWN)
 		return -EINVAL;
-	err = snd_seq_create_port(client, port_idx, &port);
+	err = snd_seq_create_port(client, &port);
 	if (err < 0)
 		return err;
 
@@ -1309,6 +1309,11 @@ static int snd_seq_ioctl_create_port(struct snd_seq_client *client, void *arg)
 	info->addr = port->addr;
 
 	snd_seq_set_port_info(port, info);
+	err = snd_seq_insert_port(client, port_idx, port);
+	if (err < 0) {
+		kfree(port);
+		return err;
+	}
 	if (info->capability & SNDRV_SEQ_PORT_CAP_UMP_ENDPOINT)
 		client->ump_endpoint_port = port->addr.port;
 	snd_seq_system_client_ev_port_start(port->addr.client, port->addr.port);
diff --git a/sound/core/seq/seq_midi_emul.c b/sound/core/seq/seq_midi_emul.c
index fd067c85c524f..a90ebc7b3811b 100644
--- a/sound/core/seq/seq_midi_emul.c
+++ b/sound/core/seq/seq_midi_emul.c
@@ -81,9 +81,6 @@ snd_midi_process_event(const struct snd_midi_op *ops,
 		pr_debug("ALSA: seq_midi_emul: ev or chanbase NULL (snd_midi_process_event)\n");
 		return;
 	}
-	if (chanset->channels == NULL)
-		return;
-
 	if (snd_seq_ev_is_channel_type(ev)) {
 		dest_channel = ev->data.note.channel;
 		if (dest_channel >= chanset->max_channels) {
@@ -642,23 +639,6 @@ static void snd_midi_channel_init(struct snd_midi_channel *p, int n)
 		p->drum_channel = 1;	/* Default ch 10 as drums */
 }
 
-/*
- * Allocate and initialise a set of midi channel control blocks.
- */
-static struct snd_midi_channel *snd_midi_channel_init_set(int n)
-{
-	struct snd_midi_channel *chan;
-	int  i;
-
-	chan = kmalloc_objs(struct snd_midi_channel, n);
-	if (chan) {
-		for (i = 0; i < n; i++)
-			snd_midi_channel_init(chan+i, i);
-	}
-
-	return chan;
-}
-
 /*
  * reset all midi channels
  */
@@ -687,13 +667,18 @@ reset_all_channels(struct snd_midi_channel_set *chset)
 struct snd_midi_channel_set *snd_midi_channel_alloc_set(int n)
 {
 	struct snd_midi_channel_set *chset;
+	int i;
+
+	chset = kmalloc_flex(*chset, channels, n);
+	if (!chset)
+		return NULL;
+
+	chset->max_channels = n;
+	chset->private_data = NULL;
+
+	for (i = 0; i < n; i++)
+		snd_midi_channel_init(&chset->channels[i], i);
 
-	chset = kmalloc_obj(*chset);
-	if (chset) {
-		chset->channels = snd_midi_channel_init_set(n);
-		chset->private_data = NULL;
-		chset->max_channels = n;
-	}
 	return chset;
 }
 EXPORT_SYMBOL(snd_midi_channel_alloc_set);
@@ -717,7 +702,6 @@ void snd_midi_channel_free_set(struct snd_midi_channel_set *chset)
 {
 	if (chset == NULL)
 		return;
-	kfree(chset->channels);
 	kfree(chset);
 }
 EXPORT_SYMBOL(snd_midi_channel_free_set);
diff --git a/sound/core/seq/seq_ports.c b/sound/core/seq/seq_ports.c
index da8d358958f15..17daacd4476aa 100644
--- a/sound/core/seq/seq_ports.c
+++ b/sound/core/seq/seq_ports.c
@@ -108,14 +108,13 @@ static void port_subs_info_init(struct snd_seq_port_subs_info *grp)
 }
 
 
-/* create a port, port number or a negative error code is returned
+/* create a port, 0 on success or a negative error code is returned
  * the caller needs to unref the port via snd_seq_port_unlock() appropriately
  */
-int snd_seq_create_port(struct snd_seq_client *client, int port,
+int snd_seq_create_port(struct snd_seq_client *client,
 			struct snd_seq_client_port **port_ret)
 {
-	struct snd_seq_client_port *new_port, *p;
-	int num;
+	struct snd_seq_client_port *new_port;
 	
 	*port_ret = NULL;
 
@@ -141,25 +140,37 @@ int snd_seq_create_port(struct snd_seq_client *client, int port,
 	port_subs_info_init(&new_port->c_dest);
 	snd_use_lock_use(&new_port->use_lock);
 
+	*port_ret = new_port;
+
+	return 0;
+}
+
+/* insert the port; return the port address or a negative error code */
+int snd_seq_insert_port(struct snd_seq_client *client, int port,
+			struct snd_seq_client_port *new_port)
+{
+	struct snd_seq_client_port *p;
+	int num;
+
 	num = max(port, 0);
 	guard(mutex)(&client->ports_mutex);
 	guard(write_lock_irq)(&client->ports_lock);
+	struct list_head *insert_before = &client->ports_list_head;
 	list_for_each_entry(p, &client->ports_list_head, list) {
-		if (p->addr.port == port) {
-			kfree(new_port);
+		if (p->addr.port == port)
 			return -EBUSY;
-		}
-		if (p->addr.port > num)
+		if (p->addr.port > num) {
+			insert_before = &p->list;
 			break;
+		}
 		if (port < 0) /* auto-probe mode */
 			num = p->addr.port + 1;
 	}
 	/* insert the new port */
-	list_add_tail(&new_port->list, &p->list);
+	list_add_tail(&new_port->list, insert_before);
 	client->num_ports++;
 	new_port->addr.port = num;	/* store the port number in the port */
 	sprintf(new_port->name, "port-%d", num);
-	*port_ret = new_port;
 
 	return num;
 }
diff --git a/sound/core/seq/seq_ports.h b/sound/core/seq/seq_ports.h
index 40ed6cf7cb90b..b689c0f4867c4 100644
--- a/sound/core/seq/seq_ports.h
+++ b/sound/core/seq/seq_ports.h
@@ -98,10 +98,14 @@ struct snd_seq_client_port *snd_seq_port_query_nearest(struct snd_seq_client *cl
 
 DEFINE_FREE(snd_seq_port, struct snd_seq_client_port *, if (!IS_ERR_OR_NULL(_T)) snd_seq_port_unlock(_T))
 
-/* create a port, port number or a negative error code is returned */
-int snd_seq_create_port(struct snd_seq_client *client, int port_index,
+/* create a port, 0 on success or a negative error code is returned */
+int snd_seq_create_port(struct snd_seq_client *client,
 			struct snd_seq_client_port **port_ret);
 
+/* insert the port; return the port address or a negative error code */
+int snd_seq_insert_port(struct snd_seq_client *client, int port,
+			struct snd_seq_client_port *new_port);
+
 /* delete a port */
 int snd_seq_delete_port(struct snd_seq_client *client, int port);
 
diff --git a/sound/core/seq/seq_ump_client.c b/sound/core/seq/seq_ump_client.c
index 9079ccfdc8666..ccd93599b493b 100644
--- a/sound/core/seq/seq_ump_client.c
+++ b/sound/core/seq/seq_ump_client.c
@@ -37,6 +37,7 @@ struct seq_ump_client {
 	struct snd_ump_endpoint *ump;	/* assigned endpoint */
 	int seq_client;			/* sequencer client id */
 	int opened[2];			/* current opens for each direction */
+	rwlock_t output_lock;		/* protects out_rfile output access */
 	struct snd_rawmidi_file out_rfile; /* rawmidi for output */
 	struct seq_ump_input_buffer input; /* input parser context */
 	void *ump_info[SNDRV_UMP_MAX_BLOCKS + 1]; /* shadow of seq client ump_info */
@@ -88,6 +89,7 @@ static int seq_ump_process_event(struct snd_seq_event *ev, int direct,
 	unsigned char type;
 	int len;
 
+	guard(read_lock_irqsave)(&client->output_lock);
 	substream = client->out_rfile.output;
 	if (!substream)
 		return -ENODEV;
@@ -106,6 +108,7 @@ static int seq_ump_process_event(struct snd_seq_event *ev, int direct,
 static int seq_ump_client_open(struct seq_ump_client *client, int dir)
 {
 	struct snd_ump_endpoint *ump = client->ump;
+	struct snd_rawmidi_file rfile = {};
 	int err;
 
 	guard(mutex)(&ump->open_mutex);
@@ -113,9 +116,11 @@ static int seq_ump_client_open(struct seq_ump_client *client, int dir)
 		err = snd_rawmidi_kernel_open(&ump->core, 0,
 					      SNDRV_RAWMIDI_LFLG_OUTPUT |
 					      SNDRV_RAWMIDI_LFLG_APPEND,
-					      &client->out_rfile);
+					      &rfile);
 		if (err < 0)
 			return err;
+		scoped_guard(write_lock_irqsave, &client->output_lock)
+			client->out_rfile = rfile;
 	}
 	client->opened[dir]++;
 	return 0;
@@ -125,11 +130,19 @@ static int seq_ump_client_open(struct seq_ump_client *client, int dir)
 static int seq_ump_client_close(struct seq_ump_client *client, int dir)
 {
 	struct snd_ump_endpoint *ump = client->ump;
+	struct snd_rawmidi_file rfile = {};
 
 	guard(mutex)(&ump->open_mutex);
-	if (!--client->opened[dir])
-		if (dir == STR_OUT)
-			snd_rawmidi_kernel_release(&client->out_rfile);
+	if (!--client->opened[dir]) {
+		if (dir == STR_OUT) {
+			scoped_guard(write_lock_irqsave, &client->output_lock) {
+				rfile = client->out_rfile;
+				client->out_rfile = (struct snd_rawmidi_file){};
+			}
+			if (rfile.rmidi)
+				snd_rawmidi_kernel_release(&rfile);
+		}
+	}
 	return 0;
 }
 
@@ -467,6 +480,7 @@ static int snd_seq_ump_probe(struct snd_seq_device *dev)
 
 	INIT_WORK(&client->group_notify_work, handle_group_notify);
 	client->ump = ump;
+	rwlock_init(&client->output_lock);
 
 	client->seq_client =
 		snd_seq_create_kernel_client(card, ump->core.device,
diff --git a/sound/core/timer.c b/sound/core/timer.c
index 820901d503af3..57583dec39748 100644
--- a/sound/core/timer.c
+++ b/sound/core/timer.c
@@ -1007,6 +1007,7 @@ static int snd_timer_dev_register(struct snd_device *dev)
 {
 	struct snd_timer *timer = dev->device_data;
 	struct snd_timer *timer1;
+	struct list_head *insert_before = &snd_timer_list;
 
 	if (snd_BUG_ON(!timer || !timer->hw.start || !timer->hw.stop))
 		return -ENXIO;
@@ -1016,28 +1017,36 @@ static int snd_timer_dev_register(struct snd_device *dev)
 
 	guard(mutex)(&register_mutex);
 	list_for_each_entry(timer1, &snd_timer_list, device_list) {
-		if (timer1->tmr_class > timer->tmr_class)
+		if (timer1->tmr_class > timer->tmr_class) {
+			insert_before = &timer1->device_list;
 			break;
+		}
 		if (timer1->tmr_class < timer->tmr_class)
 			continue;
 		if (timer1->card && timer->card) {
-			if (timer1->card->number > timer->card->number)
+			if (timer1->card->number > timer->card->number) {
+				insert_before = &timer1->device_list;
 				break;
+			}
 			if (timer1->card->number < timer->card->number)
 				continue;
 		}
-		if (timer1->tmr_device > timer->tmr_device)
+		if (timer1->tmr_device > timer->tmr_device) {
+			insert_before = &timer1->device_list;
 			break;
+		}
 		if (timer1->tmr_device < timer->tmr_device)
 			continue;
-		if (timer1->tmr_subdevice > timer->tmr_subdevice)
+		if (timer1->tmr_subdevice > timer->tmr_subdevice) {
+			insert_before = &timer1->device_list;
 			break;
+		}
 		if (timer1->tmr_subdevice < timer->tmr_subdevice)
 			continue;
 		/* conflicts.. */
 		return -EBUSY;
 	}
-	list_add_tail(&timer->device_list, &timer1->device_list);
+	list_add_tail(&timer->device_list, insert_before);
 	return 0;
 }
 
diff --git a/sound/firewire/dice/dice.c b/sound/firewire/dice/dice.c
index f7a50bae4b559..58f14aadc73de 100644
--- a/sound/firewire/dice/dice.c
+++ b/sound/firewire/dice/dice.c
@@ -148,7 +148,7 @@ static int dice_probe(struct fw_unit *unit, const struct ieee1394_device_id *ent
 	snd_dice_detect_formats_t detect_formats;
 	int err;
 
-	if (!entry->driver_data && entry->vendor_id != OUI_SSL) {
+	if (!entry->driver_data_ptr && entry->vendor_id != OUI_SSL) {
 		err = check_dice_category(unit);
 		if (err < 0)
 			return -ENODEV;
@@ -164,10 +164,10 @@ static int dice_probe(struct fw_unit *unit, const struct ieee1394_device_id *ent
 	dev_set_drvdata(&unit->device, dice);
 	dice->card = card;
 
-	if (!entry->driver_data)
+	if (!entry->driver_data_ptr)
 		detect_formats = snd_dice_stream_detect_current_formats;
 	else
-		detect_formats = (snd_dice_detect_formats_t)entry->driver_data;
+		detect_formats = entry->driver_data_ptr;
 
 	// Below models are compliant to IEC 61883-1/6 and have no quirk at high sampling transfer
 	// frequency.
@@ -255,7 +255,7 @@ static void dice_bus_reset(struct fw_unit *unit)
 		.model_id	= (model), \
 		.specifier_id	= (vendor), \
 		.version	= DICE_INTERFACE, \
-		.driver_data = (kernel_ulong_t)(data), \
+		.driver_data_ptr = (data), \
 	}
 
 static const struct ieee1394_device_id dice_id_table[] = {
@@ -267,7 +267,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_MAUDIO,
 		.model_id	= 0x000010,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_extension_formats,
+		.driver_data_ptr = snd_dice_detect_extension_formats,
 	},
 	/* M-Audio Profire 610 has a different value in version field. */
 	{
@@ -275,7 +275,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_MAUDIO,
 		.model_id	= 0x000011,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_extension_formats,
+		.driver_data_ptr = snd_dice_detect_extension_formats,
 	},
 	/* TC Electronic Konnekt 24D. */
 	{
@@ -283,7 +283,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_TCELECTRONIC,
 		.model_id	= 0x000020,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats,
+		.driver_data_ptr = snd_dice_detect_tcelectronic_formats,
 	},
 	/* TC Electronic Konnekt 8. */
 	{
@@ -291,7 +291,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_TCELECTRONIC,
 		.model_id	= 0x000021,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats,
+		.driver_data_ptr = snd_dice_detect_tcelectronic_formats,
 	},
 	/* TC Electronic Studio Konnekt 48. */
 	{
@@ -299,7 +299,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_TCELECTRONIC,
 		.model_id	= 0x000022,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats,
+		.driver_data_ptr = snd_dice_detect_tcelectronic_formats,
 	},
 	/* TC Electronic Konnekt Live. */
 	{
@@ -307,7 +307,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_TCELECTRONIC,
 		.model_id	= 0x000023,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats,
+		.driver_data_ptr = snd_dice_detect_tcelectronic_formats,
 	},
 	/* TC Electronic Desktop Konnekt 6. */
 	{
@@ -315,7 +315,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_TCELECTRONIC,
 		.model_id	= 0x000024,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats,
+		.driver_data_ptr = snd_dice_detect_tcelectronic_formats,
 	},
 	/* TC Electronic Impact Twin. */
 	{
@@ -323,7 +323,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_TCELECTRONIC,
 		.model_id	= 0x000027,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats,
+		.driver_data_ptr = snd_dice_detect_tcelectronic_formats,
 	},
 	/* TC Electronic Digital Konnekt x32. */
 	{
@@ -331,7 +331,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_TCELECTRONIC,
 		.model_id	= 0x000030,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_tcelectronic_formats,
+		.driver_data_ptr = snd_dice_detect_tcelectronic_formats,
 	},
 	/* Alesis iO14/iO26. */
 	{
@@ -339,7 +339,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_ALESIS,
 		.model_id	= MODEL_ALESIS_IO_BOTH,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_alesis_formats,
+		.driver_data_ptr = snd_dice_detect_alesis_formats,
 	},
 	// Alesis MasterControl.
 	{
@@ -347,7 +347,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_ALESIS,
 		.model_id	= 0x000002,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_alesis_mastercontrol_formats,
+		.driver_data_ptr = snd_dice_detect_alesis_mastercontrol_formats,
 	},
 	/* Mytek Stereo 192 DSD-DAC. */
 	{
@@ -355,7 +355,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 				  IEEE1394_MATCH_MODEL_ID,
 		.vendor_id	= OUI_MYTEK,
 		.model_id	= 0x000002,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_mytek_formats,
+		.driver_data_ptr = snd_dice_detect_mytek_formats,
 	},
 	// Solid State Logic, Duende Classic and Mini.
 	// NOTE: each field of GUID in config ROM is not compliant to standard
@@ -469,7 +469,7 @@ static const struct ieee1394_device_id dice_id_table[] = {
 		.model_id	= OUI_TEAC,
 		.specifier_id	= OUI_TEAC,
 		.version	= 0x800006,
-		.driver_data = (kernel_ulong_t)snd_dice_detect_teac_formats,
+		.driver_data_ptr = snd_dice_detect_teac_formats,
 	},
 	{ }
 };
diff --git a/sound/firewire/fireface/ff.c b/sound/firewire/fireface/ff.c
index 5d2c4fbf44341..13472822d2bec 100644
--- a/sound/firewire/fireface/ff.c
+++ b/sound/firewire/fireface/ff.c
@@ -70,7 +70,7 @@ static int snd_ff_probe(struct fw_unit *unit, const struct ieee1394_device_id *e
 	init_waitqueue_head(&ff->hwdep_wait);
 
 	ff->unit_version = entry->version;
-	ff->spec = (const struct snd_ff_spec *)entry->driver_data;
+	ff->spec = entry->driver_data_ptr;
 
 	err = snd_ff_transaction_register(ff);
 	if (err < 0)
@@ -186,7 +186,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = {
 		.specifier_id	= OUI_RME,
 		.version	= SND_FF_UNIT_VERSION_FF800,
 		.model_id	= 0x101800,
-		.driver_data	= (kernel_ulong_t)&spec_ff800,
+		.driver_data_ptr = &spec_ff800,
 	},
 	/* Fireface 400 */
 	{
@@ -198,7 +198,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = {
 		.specifier_id	= OUI_RME,
 		.version	= SND_FF_UNIT_VERSION_FF400,
 		.model_id	= 0x101800,
-		.driver_data	= (kernel_ulong_t)&spec_ff400,
+		.driver_data_ptr = &spec_ff400,
 	},
 	// Fireface UFX.
 	{
@@ -210,7 +210,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = {
 		.specifier_id	= OUI_RME,
 		.version	= SND_FF_UNIT_VERSION_UFX,
 		.model_id	= 0x101800,
-		.driver_data	= (kernel_ulong_t)&spec_ufx_802,
+		.driver_data_ptr = &spec_ufx_802,
 	},
 	// Fireface UCX.
 	{
@@ -222,7 +222,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = {
 		.specifier_id	= OUI_RME,
 		.version	= SND_FF_UNIT_VERSION_UCX,
 		.model_id	= 0x101800,
-		.driver_data	= (kernel_ulong_t)&spec_ucx,
+		.driver_data_ptr = &spec_ucx,
 	},
 	// Fireface 802.
 	{
@@ -234,7 +234,7 @@ static const struct ieee1394_device_id snd_ff_id_table[] = {
 		.specifier_id	= OUI_RME,
 		.version	= SND_FF_UNIT_VERSION_802,
 		.model_id	= 0x101800,
-		.driver_data	= (kernel_ulong_t)&spec_ufx_802,
+		.driver_data_ptr = &spec_ufx_802,
 	},
 	{}
 };
diff --git a/sound/firewire/motu/motu.c b/sound/firewire/motu/motu.c
index fd2a9dddbfa6f..1fec6c8cdf6c4 100644
--- a/sound/firewire/motu/motu.c
+++ b/sound/firewire/motu/motu.c
@@ -78,7 +78,7 @@ static int motu_probe(struct fw_unit *unit, const struct ieee1394_device_id *ent
 	dev_set_drvdata(&unit->device, motu);
 	motu->card = card;
 
-	motu->spec = (const struct snd_motu_spec *)entry->driver_data;
+	motu->spec = entry->driver_data_ptr;
 	mutex_init(&motu->mutex);
 	spin_lock_init(&motu->lock);
 	init_waitqueue_head(&motu->hwdep_wait);
@@ -148,7 +148,7 @@ static void motu_bus_update(struct fw_unit *unit)
 	snd_motu_transaction_reregister(motu);
 }
 
-#define SND_MOTU_DEV_ENTRY(model, data)			\
+#define SND_MOTU_DEV_ENTRY(model, data_ptr)		\
 {							\
 	.match_flags	= IEEE1394_MATCH_VENDOR_ID |	\
 			  IEEE1394_MATCH_SPECIFIER_ID |	\
@@ -156,7 +156,7 @@ static void motu_bus_update(struct fw_unit *unit)
 	.vendor_id	= OUI_MOTU,			\
 	.specifier_id	= OUI_MOTU,			\
 	.version	= model,			\
-	.driver_data	= (kernel_ulong_t)data,		\
+	.driver_data_ptr = data_ptr,			\
 }
 
 static const struct ieee1394_device_id motu_id_table[] = {
diff --git a/sound/firewire/oxfw/oxfw.c b/sound/firewire/oxfw/oxfw.c
index 5039bd79b18ed..38a3c3b150dfb 100644
--- a/sound/firewire/oxfw/oxfw.c
+++ b/sound/firewire/oxfw/oxfw.c
@@ -95,7 +95,7 @@ static int name_card(struct snd_oxfw *oxfw, const struct ieee1394_device_id *ent
 
 	/* to apply card definitions */
 	if (entry->vendor_id == VENDOR_GRIFFIN || entry->vendor_id == VENDOR_LACIE) {
-		info = (const struct compat_info *)entry->driver_data;
+		info = entry->driver_data_ptr;
 		d = info->driver_name;
 		v = info->vendor_name;
 		m = info->model_name;
@@ -321,7 +321,7 @@ static const struct compat_info lacie_speakers = {
 	.model_id     = model, \
 	.specifier_id = SPECIFIER_1394TA, \
 	.version      = VERSION_AVC, \
-	.driver_data  = (kernel_ulong_t)data, \
+	.driver_data_ptr = data, \
 }
 
 static const struct ieee1394_device_id oxfw_id_table[] = {
diff --git a/sound/hda/codecs/Kconfig b/sound/hda/codecs/Kconfig
index addbc94243365..dcf340e5a0c1a 100644
--- a/sound/hda/codecs/Kconfig
+++ b/sound/hda/codecs/Kconfig
@@ -69,6 +69,7 @@ comment "Set to Y if you want auto-loading the codec driver"
 
 config SND_HDA_CODEC_CA0132
 	tristate "Build Creative CA0132 codec support"
+	select SND_HDA_GENERIC
 	help
 	  Say Y or M here to include Creative CA0132 codec support in
 	  snd-hda-intel driver.
diff --git a/sound/hda/codecs/ca0132.c b/sound/hda/codecs/ca0132.c
index be565ffaade0a..3fe11983d6ca1 100644
--- a/sound/hda/codecs/ca0132.c
+++ b/sound/hda/codecs/ca0132.c
@@ -24,6 +24,7 @@
 #include "hda_local.h"
 #include "hda_auto_parser.h"
 #include "hda_jack.h"
+#include "generic.h"
 
 #include "ca0132_regs.h"
 
@@ -1060,6 +1061,8 @@ enum dsp_download_state {
  */
 
 struct ca0132_spec {
+	struct hda_gen_spec gen;
+
 	const struct snd_kcontrol_new *mixers[5];
 	unsigned int num_mixers;
 	const struct hda_verb *base_init_verbs;
@@ -1174,6 +1177,7 @@ enum {
 	QUIRK_R3D,
 	QUIRK_AE5,
 	QUIRK_AE7,
+	QUIRK_GENERIC,
 	QUIRK_NONE = HDA_FIXUP_ID_NOT_SET,
 };
 
@@ -1292,6 +1296,20 @@ static const struct hda_pintbl ae7_pincfgs[] = {
 	{}
 };
 
+static const struct hda_pintbl ca0132_generic_pincfgs[] = {
+	{ 0x0b, 0x41014111 },
+	{ 0x0c, 0x414520f0 }, /* SPDIF out */
+	{ 0x0d, 0x01014010 }, /* lineout */
+	{ 0x0e, 0x41c501f0 },
+	{ 0x0f, 0x411111f0 }, /* disabled */
+	{ 0x10, 0x411111f0 }, /* disabled */
+	{ 0x11, 0x41012014 },
+	{ 0x12, 0x37a790f0 }, /* mic */
+	{ 0x13, 0x77a701f0 },
+	{ 0x18, 0x500000f0 },
+	{}
+};
+
 static const struct hda_quirk ca0132_quirks[] = {
 	SND_PCI_QUIRK(0x1028, 0x057b, "Alienware M17x R4", QUIRK_ALIENWARE_M17XR4),
 	SND_PCI_QUIRK(0x1028, 0x0685, "Alienware 15 2015", QUIRK_ALIENWARE),
@@ -1304,6 +1322,7 @@ static const struct hda_quirk ca0132_quirks[] = {
 	SND_PCI_QUIRK(0x1458, 0xA016, "Recon3Di", QUIRK_R3DI),
 	SND_PCI_QUIRK(0x1458, 0xA026, "Gigabyte G1.Sniper Z97", QUIRK_R3DI),
 	SND_PCI_QUIRK(0x1458, 0xA036, "Gigabyte GA-Z170X-Gaming 7", QUIRK_R3DI),
+	SND_PCI_QUIRK(0x1458, 0xA046, "Gigabyte GA-Z170X-Gaming G1", QUIRK_GENERIC),
 	SND_PCI_QUIRK(0x3842, 0x1038, "EVGA X99 Classified", QUIRK_R3DI),
 	SND_PCI_QUIRK(0x3842, 0x104b, "EVGA X299 Dark", QUIRK_R3DI),
 	SND_PCI_QUIRK(0x3842, 0x1055, "EVGA Z390 DARK", QUIRK_R3DI),
@@ -1325,6 +1344,7 @@ static const struct hda_model_fixup ca0132_quirk_models[] = {
 	{ .id = QUIRK_R3D, .name = "r3d" },
 	{ .id = QUIRK_AE5, .name = "ae5" },
 	{ .id = QUIRK_AE7, .name = "ae7" },
+	{ .id = QUIRK_GENERIC, .name = "generic" },
 	{}
 };
 
@@ -9897,14 +9917,57 @@ static void sbz_detect_quirk(struct hda_codec *codec)
 	}
 }
 
+static void ca0132_generic_init_hook(struct hda_codec *codec)
+{
+	struct ca0132_spec *spec = codec->spec;
+
+	snd_hda_sequence_write(codec, spec->spec_init_verbs);
+}
+
+static int ca0132_generic_probe(struct hda_codec *codec)
+{
+	struct ca0132_spec *spec = codec->spec;
+	struct auto_pin_cfg *cfg = &spec->gen.autocfg;
+	int err;
+
+	snd_hda_gen_spec_init(&spec->gen);
+
+	snd_hda_apply_pincfgs(codec, ca0132_generic_pincfgs);
+
+	ca0132_init_chip(codec);
+
+	err = ca0132_prepare_verbs(codec);
+	if (err < 0)
+		return err;
+
+	err = snd_hda_parse_pin_def_config(codec, cfg, NULL);
+	if (err < 0)
+		return err;
+	err = snd_hda_gen_parse_auto_config(codec, cfg);
+	if (err < 0)
+		return err;
+
+	spec->gen.init_hook = ca0132_generic_init_hook;
+	spec->gen.automute_speaker = 0;
+	spec->gen.automute_lo = 0;
+
+	snd_hda_sequence_write(codec, spec->spec_init_verbs);
+	return 0;
+}
+
 static void ca0132_codec_remove(struct hda_codec *codec)
 {
 	struct ca0132_spec *spec = codec->spec;
 
-	if (ca0132_quirk(spec) == QUIRK_ZXR_DBPRO)
+	switch (ca0132_quirk(spec)) {
+	case QUIRK_GENERIC:
+		snd_hda_gen_remove(codec);
+		return;
+	case QUIRK_ZXR_DBPRO:
 		return dbpro_free(codec);
-	else
+	default:
 		return ca0132_free(codec);
+	}
 }
 
 static int ca0132_codec_probe(struct hda_codec *codec,
@@ -9921,14 +9984,21 @@ static int ca0132_codec_probe(struct hda_codec *codec,
 	codec->spec = spec;
 	spec->codec = codec;
 
-	/* Detect codec quirk */
-	snd_hda_pick_fixup(codec, ca0132_quirk_models, ca0132_quirks, NULL);
-	if (ca0132_quirk(spec) == QUIRK_SBZ)
-		sbz_detect_quirk(codec);
-
+	/* These must be set before any path is taken */
 	codec->pcm_format_first = 1;
 	codec->no_sticky_stream = 1;
 
+	/* Detect codec quirk */
+	snd_hda_pick_fixup(codec, ca0132_quirk_models, ca0132_quirks, NULL);
+	switch (ca0132_quirk(spec)) {
+	case QUIRK_SBZ:
+		sbz_detect_quirk(codec);
+		break;
+	case QUIRK_GENERIC:
+		return ca0132_generic_probe(codec);
+	default:
+		break;
+	}
 
 	spec->dsp_state = DSP_DOWNLOAD_INIT;
 	spec->num_mixers = 1;
@@ -10029,36 +10099,51 @@ static int ca0132_codec_build_controls(struct hda_codec *codec)
 {
 	struct ca0132_spec *spec = codec->spec;
 
-	if (ca0132_quirk(spec) == QUIRK_ZXR_DBPRO)
+	switch (ca0132_quirk(spec)) {
+	case QUIRK_GENERIC:
+		return snd_hda_gen_build_controls(codec);
+	case QUIRK_ZXR_DBPRO:
 		return dbpro_build_controls(codec);
-	else
+	default:
 		return ca0132_build_controls(codec);
+	}
 }
 
 static int ca0132_codec_build_pcms(struct hda_codec *codec)
 {
 	struct ca0132_spec *spec = codec->spec;
 
-	if (ca0132_quirk(spec) == QUIRK_ZXR_DBPRO)
+	switch (ca0132_quirk(spec)) {
+	case QUIRK_GENERIC:
+		return snd_hda_gen_build_pcms(codec);
+	case QUIRK_ZXR_DBPRO:
 		return dbpro_build_pcms(codec);
-	else
+	default:
 		return ca0132_build_pcms(codec);
+	}
 }
 
 static int ca0132_codec_init(struct hda_codec *codec)
 {
 	struct ca0132_spec *spec = codec->spec;
 
-	if (ca0132_quirk(spec) == QUIRK_ZXR_DBPRO)
+	switch (ca0132_quirk(spec)) {
+	case QUIRK_GENERIC:
+		return snd_hda_gen_init(codec);
+	case QUIRK_ZXR_DBPRO:
 		return dbpro_init(codec);
-	else
+	default:
 		return ca0132_init(codec);
+	}
 }
 
 static int ca0132_codec_suspend(struct hda_codec *codec)
 {
 	struct ca0132_spec *spec = codec->spec;
 
+	if (ca0132_quirk(spec) == QUIRK_GENERIC)
+		return 0;
+
 	cancel_delayed_work_sync(&spec->unsol_hp_work);
 	return 0;
 }
diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c
index 2f5d13032fd7a..2b150bb5c7a86 100644
--- a/sound/hda/codecs/realtek/alc269.c
+++ b/sound/hda/codecs/realtek/alc269.c
@@ -3338,6 +3338,18 @@ static void alc256_fixup_acer_sfg16_micmute_led(struct hda_codec *codec,
 	alc_fixup_hp_gpio_led(codec, action, 0, 0x04);
 }
 
+static void alc287_fixup_acer_micmute_led(struct hda_codec *codec,
+	const struct hda_fixup *fix, int action)
+{
+	struct alc_spec *spec = codec->spec;
+
+	alc_fixup_hp_gpio_led(codec, action, 0, 0x10);
+	if (action == HDA_FIXUP_ACT_PRE_PROBE) {
+		spec->micmute_led_polarity = 1;
+		spec->gpio_mask |= 0x10;
+		spec->gpio_dir |= 0x10;
+	}
+}
 
 /* for alc295_fixup_hp_top_speakers */
 #include "../helpers/hp_x360.c"
@@ -4155,6 +4167,7 @@ enum {
 	ALC236_FIXUP_HP_DMIC,
 	ALC256_FIXUP_HONOR_MRB_XXX_M1020_AUDIO,
 	ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX,
+	ALC287_FIXUP_ACER_MICMUTE_LED,
 };
 
 /* A special fixup for Lenovo C940 and Yoga Duet 7;
@@ -6727,7 +6740,13 @@ static const struct hda_fixup alc269_fixups[] = {
 		.v.func = cs35l41_fixup_i2c_two,
 		.chained = true,
 		.chain_id = ALC245_FIXUP_HP_X360_MUTE_LEDS
-	}
+	},
+	[ALC287_FIXUP_ACER_MICMUTE_LED] = {
+		.type = HDA_FIXUP_FUNC,
+		.v.func = alc287_fixup_acer_micmute_led,
+		.chained = true,
+		.chain_id = ALC2XX_FIXUP_HEADSET_MIC,
+	},
 };
 
 static const struct hda_quirk alc269_fixup_tbl[] = {
@@ -6777,7 +6796,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1025, 0x1466, "Acer Aspire A515-56", ALC255_FIXUP_ACER_HEADPHONE_AND_MIC),
 	SND_PCI_QUIRK(0x1025, 0x1534, "Acer Predator PH315-54", ALC255_FIXUP_ACER_MIC_NO_PRESENCE),
 	SND_PCI_QUIRK(0x1025, 0x1539, "Acer Nitro 5 AN515-57", ALC2XX_FIXUP_HEADSET_MIC),
-	SND_PCI_QUIRK(0x1025, 0x159c, "Acer Nitro 5 AN515-58", ALC2XX_FIXUP_HEADSET_MIC),
+	SND_PCI_QUIRK(0x1025, 0x159c, "Acer Nitro 5 AN515-58", ALC287_FIXUP_ACER_MICMUTE_LED),
 	SND_PCI_QUIRK(0x1025, 0x1597, "Acer Nitro 5 AN517-55", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1025, 0x160e, "Acer PT316-51S", ALC2XX_FIXUP_HEADSET_MIC),
 	SND_PCI_QUIRK(0x1025, 0x1640, "Acer Aspire A315-44P", ALC256_FIXUP_ACER_SFG16_MICMUTE_LED),
@@ -6955,6 +6974,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x84da, "HP OMEN dc0019-ur", ALC295_FIXUP_HP_OMEN),
 	SND_PCI_QUIRK(0x103c, 0x84e7, "HP Pavilion 15", ALC269_FIXUP_HP_MUTE_LED_MIC3),
 	SND_PCI_QUIRK(0x103c, 0x8519, "HP Spectre x360 15-df0xxx", ALC285_FIXUP_HP_SPECTRE_X360),
+	SND_PCI_QUIRK(0x103c, 0x8536, "HP ProBook 430 G6", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
 	SND_PCI_QUIRK(0x103c, 0x8537, "HP ProBook 440 G6", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
 	SND_PCI_QUIRK(0x103c, 0x8548, "HP EliteBook x360 830 G6", ALC285_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x854a, "HP EliteBook 830 G6", ALC285_FIXUP_HP_GPIO_LED),
@@ -7084,6 +7104,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8a30, "HP Envy 17", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8a31, "HP Envy 15", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8a34, "HP Pavilion x360 2-in-1 Laptop 14-ek0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
+	SND_PCI_QUIRK(0x103c, 0x8a36, "HP Pavilion Plus 14-eh0xxx", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8a3d, "HP Victus 15-fb0xxx (MB 8A3D)", ALC245_FIXUP_HP_MUTE_LED_V2_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8a4f, "HP Victus 15-fa0xxx (MB 8A4F)", ALC245_FIXUP_HP_MUTE_LED_COEFBIT),
 	SND_PCI_QUIRK(0x103c, 0x8a6e, "HP EDNA 360", ALC287_FIXUP_CS35L41_I2C_4),
@@ -7103,6 +7124,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8ad8, "HP 800 G9", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8b0f, "HP Elite mt645 G7 Mobile Thin Client U81", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
 	SND_PCI_QUIRK(0x103c, 0x8b2f, "HP 255 15.6 inch G10 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
+	SND_PCI_QUIRK(0x103c, 0x8b34, "HP 250 15.6 inch G10 Notebook PC", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
 	SND_PCI_QUIRK(0x103c, 0x8b3a, "HP Envy 15", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8b3f, "HP mt440 Mobile Thin Client U91", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8b42, "HP", ALC245_FIXUP_CS35L41_SPI_2_HP_GPIO_LED),
@@ -7229,7 +7251,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8da0, "HP 16 Clipper OmniBook 7(X360)", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8da1, "HP 16 Clipper OmniBook X", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x103c, 0x8da7, "HP 14 Enstrom OmniBook X", ALC287_FIXUP_CS35L41_I2C_2),
-	SND_PCI_QUIRK(0x103c, 0x8da8, "HP 16 Piston OmniBook X", ALC287_FIXUP_CS35L41_I2C_2),
+	SND_PCI_QUIRK(0x103c, 0x8da8, "HP 16 Piston OmniBook X", ALC245_FIXUP_HP_ENVY_X360_15_FH0XXX),
 	SND_PCI_QUIRK(0x103c, 0x8dc9, "HP Laptop 15-fc0xxx", ALC236_FIXUP_HP_DMIC),
 	SND_PCI_QUIRK(0x103c, 0x8dd4, "HP EliteStudio 8 AIO", ALC274_FIXUP_HP_AIO_BIND_DACS),
 	SND_PCI_QUIRK(0x103c, 0x8dd7, "HP Laptop 15-fd0xxx", ALC236_FIXUP_HP_MUTE_LED_COEFBIT2),
@@ -7241,6 +7263,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x103c, 0x8def, "HP EliteBook 660 G12", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8df0, "HP EliteBook 630 G12", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8df1, "HP EliteBook 630 G12", ALC236_FIXUP_HP_GPIO_LED),
+	SND_PCI_QUIRK(0x103c, 0x8df7, "HP Z66 G6", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8dfb, "HP EliteBook 6 G1a 14", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
 	SND_PCI_QUIRK(0x103c, 0x8dfc, "HP EliteBook 645 G12", ALC236_FIXUP_HP_GPIO_LED),
 	SND_PCI_QUIRK(0x103c, 0x8dfd, "HP EliteBook 6 G1a 16", ALC236_FIXUP_HP_MUTE_LED_MICMUTE_VREF),
@@ -7453,12 +7476,12 @@ static const struct hda_quirk alc269_fixup_tbl[] = {
 	SND_PCI_QUIRK(0x1043, 0x3e00, "ASUS G814FH/FM/FP", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x1043, 0x3e20, "ASUS G814PH/PM/PP", ALC287_FIXUP_CS35L41_I2C_2),
 	SND_PCI_QUIRK(0x1043, 0x3e30, "ASUS TP3607SA", ALC287_FIXUP_TAS2781_I2C),
-	SND_PCI_QUIRK(0x1043, 0x3ee0, "ASUS Strix G815_JHR_JMR_JPR", ALC287_FIXUP_TAS2781_I2C),
-	SND_PCI_QUIRK(0x1043, 0x3ef0, "ASUS Strix G635LR_LW_LX", ALC287_FIXUP_TAS2781_I2C),
-	SND_PCI_QUIRK(0x1043, 0x3f00, "ASUS Strix G815LH_LM_LP", ALC287_FIXUP_TAS2781_I2C),
-	SND_PCI_QUIRK(0x1043, 0x3f10, "ASUS Strix G835LR_LW_LX", ALC287_FIXUP_TAS2781_I2C),
-	SND_PCI_QUIRK(0x1043, 0x3f20, "ASUS Strix G615LR_LW", ALC287_FIXUP_TAS2781_I2C),
-	SND_PCI_QUIRK(0x1043, 0x3f30, "ASUS Strix G815LR_LW", ALC287_FIXUP_TAS2781_I2C),
+	SND_PCI_QUIRK(0x1043, 0x3ee0, "ASUS Strix G815_JHR_JMR_JPR", ALC287_FIXUP_TXNW2781_I2C),
+	SND_PCI_QUIRK(0x1043, 0x3ef0, "ASUS Strix G635LR_LW_LX", ALC287_FIXUP_TXNW2781_I2C),
+	SND_PCI_QUIRK(0x1043, 0x3f00, "ASUS Strix G815LH_LM_LP", ALC287_FIXUP_TXNW2781_I2C),
+	SND_PCI_QUIRK(0x1043, 0x3f10, "ASUS Strix G835LR_LW_LX", ALC287_FIXUP_TXNW2781_I2C),
+	SND_PCI_QUIRK(0x1043, 0x3f20, "ASUS Strix G615LR_LW", ALC287_FIXUP_TXNW2781_I2C),
+	SND_PCI_QUIRK(0x1043, 0x3f30, "ASUS Strix G815LR_LW", ALC287_FIXUP_TXNW2781_I2C),
 	SND_PCI_QUIRK(0x1043, 0x3fd0, "ASUS B3605CVA", ALC245_FIXUP_CS35L41_SPI_2),
 	SND_PCI_QUIRK(0x1043, 0x3ff0, "ASUS B5405CVA", ALC245_FIXUP_CS35L41_SPI_2),
 	SND_PCI_QUIRK(0x1043, 0x831a, "ASUS P901", ALC269_FIXUP_STEREO_DMIC),
diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda.c b/sound/hda/codecs/side-codecs/cs35l41_hda.c
index acfccc848f82d..64a5bd895fd1e 100644
--- a/sound/hda/codecs/side-codecs/cs35l41_hda.c
+++ b/sound/hda/codecs/side-codecs/cs35l41_hda.c
@@ -1325,6 +1325,43 @@ static int cs35l41_fw_type_ctl_info(struct snd_kcontrol *kcontrol, struct snd_ct
 	return snd_ctl_enum_info(uinfo, 1, ARRAY_SIZE(cs35l41_hda_fw_ids), cs35l41_hda_fw_ids);
 }
 
+static void cs35l41_remove_controls(struct cs35l41_hda *cs35l41)
+{
+	if (!cs35l41->codec)
+		return;
+
+	snd_ctl_remove(cs35l41->codec->card, cs35l41->mute_override_ctl);
+	cs35l41->mute_override_ctl = NULL;
+
+	snd_ctl_remove(cs35l41->codec->card, cs35l41->fw_load_ctl);
+	cs35l41->fw_load_ctl = NULL;
+
+	snd_ctl_remove(cs35l41->codec->card, cs35l41->fw_type_ctl);
+	cs35l41->fw_type_ctl = NULL;
+}
+
+static int cs35l41_add_control(struct cs35l41_hda *cs35l41,
+			       struct snd_kcontrol_new *ctl,
+			       struct snd_kcontrol **kctl)
+{
+	int ret;
+
+	*kctl = snd_ctl_new1(ctl, cs35l41);
+	if (!*kctl)
+		return -ENOMEM;
+
+	ret = snd_ctl_add(cs35l41->codec->card, *kctl);
+	if (ret) {
+		dev_err(cs35l41->dev, "Failed to add KControl %s = %d\n", ctl->name, ret);
+		*kctl = NULL;
+		return ret;
+	}
+
+	dev_dbg(cs35l41->dev, "Added Control %s\n", ctl->name);
+
+	return 0;
+}
+
 static int cs35l41_create_controls(struct cs35l41_hda *cs35l41)
 {
 	char fw_type_ctl_name[SNDRV_CTL_ELEM_ID_NAME_MAXLEN];
@@ -1360,32 +1397,23 @@ static int cs35l41_create_controls(struct cs35l41_hda *cs35l41)
 	scnprintf(mute_override_ctl_name, SNDRV_CTL_ELEM_ID_NAME_MAXLEN, "%s Forced Mute Status",
 		  cs35l41->amp_name);
 
-	ret = snd_ctl_add(cs35l41->codec->card, snd_ctl_new1(&fw_type_ctl, cs35l41));
-	if (ret) {
-		dev_err(cs35l41->dev, "Failed to add KControl %s = %d\n", fw_type_ctl.name, ret);
-		return ret;
-	}
-
-	dev_dbg(cs35l41->dev, "Added Control %s\n", fw_type_ctl.name);
-
-	ret = snd_ctl_add(cs35l41->codec->card, snd_ctl_new1(&fw_load_ctl, cs35l41));
-	if (ret) {
-		dev_err(cs35l41->dev, "Failed to add KControl %s = %d\n", fw_load_ctl.name, ret);
-		return ret;
-	}
-
-	dev_dbg(cs35l41->dev, "Added Control %s\n", fw_load_ctl.name);
+	ret = cs35l41_add_control(cs35l41, &fw_type_ctl, &cs35l41->fw_type_ctl);
+	if (ret)
+		goto err;
 
-	ret = snd_ctl_add(cs35l41->codec->card, snd_ctl_new1(&mute_override_ctl, cs35l41));
-	if (ret) {
-		dev_err(cs35l41->dev, "Failed to add KControl %s = %d\n", mute_override_ctl.name,
-			ret);
-		return ret;
-	}
+	ret = cs35l41_add_control(cs35l41, &fw_load_ctl, &cs35l41->fw_load_ctl);
+	if (ret)
+		goto err;
 
-	dev_dbg(cs35l41->dev, "Added Control %s\n", mute_override_ctl.name);
+	ret = cs35l41_add_control(cs35l41, &mute_override_ctl, &cs35l41->mute_override_ctl);
+	if (ret)
+		goto err;
 
 	return 0;
+
+err:
+	cs35l41_remove_controls(cs35l41);
+	return ret;
 }
 
 static bool cs35l41_dsm_supported(acpi_handle handle, unsigned int commands)
@@ -1522,6 +1550,10 @@ static void cs35l41_hda_unbind(struct device *dev, struct device *master, void *
 		device_link_remove(&cs35l41->codec->core.dev, cs35l41->dev);
 		unlock_system_sleep(sleep_flags);
 		memset(comp, 0, sizeof(*comp));
+
+		cs35l41_remove_controls(cs35l41);
+		cancel_work_sync(&cs35l41->fw_load_work);
+		cs35l41->codec = NULL;
 	}
 }
 
@@ -2060,6 +2092,7 @@ void cs35l41_hda_remove(struct device *dev)
 	struct cs35l41_hda *cs35l41 = dev_get_drvdata(dev);
 
 	component_del(cs35l41->dev, &cs35l41_hda_comp_ops);
+	cancel_work_sync(&cs35l41->fw_load_work);
 
 	pm_runtime_get_sync(cs35l41->dev);
 	pm_runtime_dont_use_autosuspend(cs35l41->dev);
diff --git a/sound/hda/codecs/side-codecs/cs35l41_hda.h b/sound/hda/codecs/side-codecs/cs35l41_hda.h
index 7d003c598e93e..56ec07c0bb745 100644
--- a/sound/hda/codecs/side-codecs/cs35l41_hda.h
+++ b/sound/hda/codecs/side-codecs/cs35l41_hda.h
@@ -57,6 +57,8 @@ enum control_bus {
 	SPI
 };
 
+struct snd_kcontrol;
+
 struct cs35l41_hda {
 	struct device *dev;
 	struct regmap *regmap;
@@ -75,6 +77,9 @@ struct cs35l41_hda {
 	int speaker_id;
 	struct mutex fw_mutex;
 	struct work_struct fw_load_work;
+	struct snd_kcontrol *fw_type_ctl;
+	struct snd_kcontrol *fw_load_ctl;
+	struct snd_kcontrol *mute_override_ctl;
 
 	struct regmap_irq_chip_data *irq_data;
 	bool firmware_running;
diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c b/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c
index 67240ce184e1a..dd1b0cc63ad6c 100644
--- a/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c
+++ b/sound/hda/codecs/side-codecs/tas2781_hda_i2c.c
@@ -588,6 +588,9 @@ static void tas2781_hda_unbind(struct device *dev,
 		comp->playback_hook = NULL;
 	}
 
+	request_firmware_nowait_cancel(tas_hda->priv->dev, tas_hda->priv,
+				       tasdev_fw_ready);
+
 	tas2781_hda_remove_controls(tas_hda);
 
 	tasdevice_config_info_remove(tas_hda->priv);
diff --git a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c
index 0e4f3553f2738..d243baff95a72 100644
--- a/sound/hda/codecs/side-codecs/tas2781_hda_spi.c
+++ b/sound/hda/codecs/side-codecs/tas2781_hda_spi.c
@@ -750,6 +750,9 @@ static void tas2781_hda_unbind(struct device *dev, struct device *master,
 		comp->playback_hook = NULL;
 	}
 
+	request_firmware_nowait_cancel(tas_priv->dev, tas_priv,
+				       tasdev_fw_ready);
+
 	tas2781_hda_remove_controls(tas_hda);
 
 	tasdevice_config_info_remove(tas_priv);
diff --git a/sound/hda/common/auto_parser.c b/sound/hda/common/auto_parser.c
index 8923813ce4247..5bc95d3116ff0 100644
--- a/sound/hda/common/auto_parser.c
+++ b/sound/hda/common/auto_parser.c
@@ -1013,7 +1013,7 @@ void snd_hda_pick_fixup(struct hda_codec *codec,
 	const char *name = NULL;
 	const char *type = NULL;
 	unsigned int vendor, device;
-	u16 pci_vendor, pci_device;
+	u16 pci_vendor = 0, pci_device = 0;
 	u16 codec_vendor, codec_device;
 
 	if (codec->fixup_id != HDA_FIXUP_ID_NOT_SET)
@@ -1066,7 +1066,7 @@ void snd_hda_pick_fixup(struct hda_codec *codec,
 	/* match primarily with the PCI SSID */
 	for (q = quirk; q->subvendor || q->subdevice; q++) {
 		/* if the entry is specific to codec SSID, check with it */
-		if (!codec->bus->pci || q->match_codec_ssid) {
+		if (!pci_vendor || !pci_device || q->match_codec_ssid) {
 			if (hda_quirk_match(codec_vendor, codec_device, q)) {
 				type = "codec SSID";
 				goto found_device;
diff --git a/sound/hda/common/controller.c b/sound/hda/common/controller.c
index 5934e5cdfdfd8..a847546753db9 100644
--- a/sound/hda/common/controller.c
+++ b/sound/hda/common/controller.c
@@ -97,6 +97,8 @@ static int azx_pcm_close(struct snd_pcm_substream *substream)
 
 	trace_azx_pcm_close(chip, azx_dev);
 	scoped_guard(mutex, &chip->open_mutex) {
+		if (chip->ops->pcm_close)
+			chip->ops->pcm_close(chip, azx_dev);
 		azx_release_device(azx_dev);
 		if (hinfo->ops.close)
 			hinfo->ops.close(hinfo, apcm->codec, substream);
@@ -1264,19 +1266,17 @@ int azx_codec_configure(struct azx *chip)
 }
 EXPORT_SYMBOL_GPL(azx_codec_configure);
 
-static int stream_direction(struct azx *chip, unsigned char index)
+void azx_add_stream(struct azx *chip, struct azx_dev *azx_dev, int idx, int tag)
 {
-	if (index >= chip->capture_index_offset &&
-	    index < chip->capture_index_offset + chip->capture_streams)
-		return SNDRV_PCM_STREAM_CAPTURE;
-	return SNDRV_PCM_STREAM_PLAYBACK;
+	snd_hdac_stream_init(azx_bus(chip), azx_stream(azx_dev), idx,
+			     azx_stream_direction(chip, idx), tag);
 }
+EXPORT_SYMBOL_GPL(azx_add_stream);
 
 /* initialize SD streams */
 int azx_init_streams(struct azx *chip)
 {
 	int i;
-	int stream_tags[2] = { 0, 0 };
 
 	/* initialize each stream (aka device)
 	 * assign the starting bdl address to each stream (device)
@@ -1284,24 +1284,10 @@ int azx_init_streams(struct azx *chip)
 	 */
 	for (i = 0; i < chip->num_streams; i++) {
 		struct azx_dev *azx_dev = kzalloc_obj(*azx_dev);
-		int dir, tag;
 
 		if (!azx_dev)
 			return -ENOMEM;
-
-		dir = stream_direction(chip, i);
-		/* stream tag must be unique throughout
-		 * the stream direction group,
-		 * valid values 1...15
-		 * use separate stream tag if the flag
-		 * AZX_DCAPS_SEPARATE_STREAM_TAG is used
-		 */
-		if (chip->driver_caps & AZX_DCAPS_SEPARATE_STREAM_TAG)
-			tag = ++stream_tags[dir];
-		else
-			tag = i + 1;
-		snd_hdac_stream_init(azx_bus(chip), azx_stream(azx_dev),
-				     i, dir, tag);
+		azx_add_stream(chip, azx_dev, i, i + 1);
 	}
 
 	return 0;
diff --git a/sound/hda/common/hda_controller.h b/sound/hda/common/hda_controller.h
index 7434f38038a0b..38227f82e7040 100644
--- a/sound/hda/common/hda_controller.h
+++ b/sound/hda/common/hda_controller.h
@@ -57,13 +57,12 @@ enum {
 struct azx_dev {
 	struct hdac_stream core;
 
-	unsigned int irq_pending:1;
 	/*
 	 * For VIA:
 	 *  A flag to ensure DMA position is 0
 	 *  when link position is not greater than FIFO size
 	 */
-	unsigned int insufficient:1;
+	bool insufficient;
 };
 
 #define azx_stream(dev)		(&(dev)->core)
@@ -79,6 +78,8 @@ struct hda_controller_ops {
 	int (*position_check)(struct azx *chip, struct azx_dev *azx_dev);
 	/* enable/disable the link power */
 	int (*link_power)(struct azx *chip, bool enable);
+	/* additional hook for PCM */
+	void (*pcm_close)(struct azx *chip, struct azx_dev *azx_dev);
 };
 
 struct azx_pcm {
@@ -206,6 +207,15 @@ int azx_bus_init(struct azx *chip, const char *model);
 int azx_probe_codecs(struct azx *chip, unsigned int max_slots);
 int azx_codec_configure(struct azx *chip);
 int azx_init_streams(struct azx *chip);
+void azx_add_stream(struct azx *chip, struct azx_dev *s, int idx, int tag);
 void azx_free_streams(struct azx *chip);
 
+static inline int azx_stream_direction(struct azx *chip, unsigned char index)
+{
+	if (index >= chip->capture_index_offset &&
+	    index < chip->capture_index_offset + chip->capture_streams)
+		return SNDRV_PCM_STREAM_CAPTURE;
+	return SNDRV_PCM_STREAM_PLAYBACK;
+}
+
 #endif /* __SOUND_HDA_CONTROLLER_H */
diff --git a/sound/hda/controllers/intel.c b/sound/hda/controllers/intel.c
index c87d75dbd8aa4..4b03c64e72ab4 100644
--- a/sound/hda/controllers/intel.c
+++ b/sound/hda/controllers/intel.c
@@ -615,17 +615,17 @@ static int azx_position_ok(struct azx *chip, struct azx_dev *azx_dev);
 /* called from IRQ */
 static int azx_position_check(struct azx *chip, struct azx_dev *azx_dev)
 {
-	struct hda_intel *hda = container_of(chip, struct hda_intel, chip);
+	struct hda_intel_stream *istream = azx_dev_to_istream(azx_dev);
 	int ok;
 
 	ok = azx_position_ok(chip, azx_dev);
 	if (ok == 1) {
-		azx_dev->irq_pending = 0;
+		istream->irq_pending = false;
 		return ok;
 	} else if (ok == 0) {
 		/* bogus IRQ, process it later */
-		azx_dev->irq_pending = 1;
-		schedule_work(&hda->irq_pending_work);
+		istream->irq_pending = true;
+		schedule_work(&istream->irq_pending_work);
 	}
 	return 0;
 }
@@ -721,11 +721,13 @@ static int azx_position_ok(struct azx *chip, struct azx_dev *azx_dev)
  */
 static void azx_irq_pending_work(struct work_struct *work)
 {
-	struct hda_intel *hda = container_of(work, struct hda_intel, irq_pending_work);
+	struct hda_intel_stream *istream =
+		container_of(work, struct hda_intel_stream, irq_pending_work);
+	struct azx_dev *azx_dev = &istream->azx_dev;
+	struct hda_intel *hda = istream->hda;
 	struct azx *chip = &hda->chip;
 	struct hdac_bus *bus = azx_bus(chip);
-	struct hdac_stream *s;
-	int pending, ok;
+	int ok;
 
 	if (!hda->irq_pending_warned) {
 		dev_info(chip->card->dev,
@@ -735,42 +737,51 @@ static void azx_irq_pending_work(struct work_struct *work)
 	}
 
 	for (;;) {
-		pending = 0;
-		spin_lock_irq(&bus->reg_lock);
-		list_for_each_entry(s, &bus->stream_list, list) {
-			struct azx_dev *azx_dev = stream_to_azx_dev(s);
-			if (!azx_dev->irq_pending ||
-			    !s->substream ||
-			    !s->running)
-				continue;
+		scoped_guard(spinlock_irq, &bus->reg_lock) {
+			if (!istream->irq_pending ||
+			    !azx_dev->core.substream ||
+			    !azx_dev->core.running) {
+				return;
+			}
+
 			ok = azx_position_ok(chip, azx_dev);
-			if (ok > 0) {
-				azx_dev->irq_pending = 0;
-				spin_unlock(&bus->reg_lock);
-				snd_pcm_period_elapsed(s->substream);
-				spin_lock(&bus->reg_lock);
-			} else if (ok < 0) {
-				pending = 0;	/* too early */
-			} else
-				pending++;
+			if (ok < 0)
+				return; /* too early */
+			if (ok > 0)
+				istream->irq_pending = false;
 		}
-		spin_unlock_irq(&bus->reg_lock);
-		if (!pending)
+
+		if (ok) {
+			snd_pcm_period_elapsed(azx_dev->core.substream);
 			return;
+		}
+
 		msleep(1);
 	}
 }
 
 /* clear irq_pending flags and assure no on-going workq */
+static void hda_intel_stream_clear_irq_pending(struct azx_dev *azx_dev)
+{
+	struct hda_intel_stream *istream = azx_dev_to_istream(azx_dev);
+
+	istream->irq_pending = false;
+	cancel_work_sync(&istream->irq_pending_work);
+}
+
+/* called at PCM close */
+static void hda_intel_pcm_close(struct azx *chip, struct azx_dev *azx_dev)
+{
+	hda_intel_stream_clear_irq_pending(azx_dev);
+}
+
 static void azx_clear_irq_pending(struct azx *chip)
 {
 	struct hdac_bus *bus = azx_bus(chip);
 	struct hdac_stream *s;
 
-	guard(spinlock_irq)(&bus->reg_lock);
 	list_for_each_entry(s, &bus->stream_list, list) {
-		struct azx_dev *azx_dev = stream_to_azx_dev(s);
-		azx_dev->irq_pending = 0;
+		hda_intel_stream_clear_irq_pending(stream_to_azx_dev(s));
 	}
 }
 
@@ -1797,7 +1808,6 @@ static int azx_create(struct snd_card *card, struct pci_dev *pci,
 	if (jackpoll_ms[dev] >= 50 && jackpoll_ms[dev] <= 60000)
 		chip->jackpoll_interval = msecs_to_jiffies(jackpoll_ms[dev]);
 	INIT_LIST_HEAD(&chip->pcm_list);
-	INIT_WORK(&hda->irq_pending_work, azx_irq_pending_work);
 	INIT_LIST_HEAD(&hda->list);
 	init_vga_switcheroo(chip);
 	init_completion(&hda->probe_wait);
@@ -1846,6 +1856,39 @@ static int azx_create(struct snd_card *card, struct pci_dev *pci,
 	return 0;
 }
 
+/* create and assign streams */
+static int hda_init_streams(struct azx *chip)
+{
+	int i;
+	int stream_tags[2] = { 0, 0 };
+
+	for (i = 0; i < chip->num_streams; i++) {
+		struct hda_intel_stream *s = kzalloc_obj(*s);
+		int tag, dir;
+
+		if (!s)
+			return -ENOMEM;
+
+		s->hda = container_of(chip, struct hda_intel, chip);
+		INIT_WORK(&s->irq_pending_work, azx_irq_pending_work);
+
+		/* stream tag must be unique throughout
+		 * the stream direction group,
+		 * valid values 1...15
+		 * use separate stream tag if the flag
+		 * AZX_DCAPS_SEPARATE_STREAM_TAG is used
+		 */
+		dir = azx_stream_direction(chip, i);
+		if (chip->driver_caps & AZX_DCAPS_SEPARATE_STREAM_TAG)
+			tag = ++stream_tags[dir];
+		else
+			tag = i + 1;
+		azx_add_stream(chip, &s->azx_dev, i, tag);
+	}
+
+	return 0;
+}
+
 static int azx_first_init(struct azx *chip)
 {
 	int dev = chip->dev_index;
@@ -2000,7 +2043,7 @@ static int azx_first_init(struct azx *chip)
 	}
 
 	/* initialize streams */
-	err = azx_init_streams(chip);
+	err = hda_init_streams(chip);
 	if (err < 0)
 		return err;
 
@@ -2099,6 +2142,7 @@ static const struct dmi_system_id driver_denylist_dmi[] = {
 static const struct hda_controller_ops pci_hda_ops = {
 	.disable_msi_reset_irq = disable_msi_reset_irq,
 	.position_check = azx_position_check,
+	.pcm_close = hda_intel_pcm_close,
 };
 
 static DECLARE_BITMAP(probed_devs, SNDRV_CARDS);
diff --git a/sound/hda/controllers/intel.h b/sound/hda/controllers/intel.h
index 2d1725f86ef17..4efb3b0fc2d81 100644
--- a/sound/hda/controllers/intel.h
+++ b/sound/hda/controllers/intel.h
@@ -9,9 +9,6 @@
 struct hda_intel {
 	struct azx chip;
 
-	/* for pending irqs */
-	struct work_struct irq_pending_work;
-
 	/* sync probing */
 	struct completion probe_wait;
 	struct delayed_work probe_work;
@@ -35,4 +32,16 @@ struct hda_intel {
 	int probe_retry;	/* being probe-retry */
 };
 
+struct hda_intel_stream {
+	struct azx_dev azx_dev;
+
+	/* for pending irqs */
+	struct hda_intel *hda;
+	struct work_struct irq_pending_work;
+	bool irq_pending;
+};
+
+#define azx_dev_to_istream(azx_dev) \
+	container_of(azx_dev, struct hda_intel_stream, azx_dev)
+
 #endif
diff --git a/sound/hda/core/i915.c b/sound/hda/core/i915.c
index 44438c799f957..6c068b135b7dc 100644
--- a/sound/hda/core/i915.c
+++ b/sound/hda/core/i915.c
@@ -130,22 +130,22 @@ static int i915_gfx_present(struct pci_dev *hdac_pci)
 	/* List of known platforms with no i915 support. */
 	static const struct pci_device_id denylist[] = {
 		/* CNL */
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a40), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a41), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a42), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a44), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a49), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a4a), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a4c), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a50), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a51), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a52), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a54), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a59), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a5a), 0x030000, 0xff0000 },
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a5c), 0x030000, 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a40), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a41), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a42), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a44), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a49), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a4a), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a4c), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a50), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a51), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a52), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a54), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a59), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a5a), .class = 0x030000, .class_mask = 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x5a5c), .class = 0x030000, .class_mask = 0xff0000 },
 		/* LKF */
-		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x9840), 0x030000, 0xff0000 },
+		{ PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0x9840), .class = 0x030000, .class_mask = 0xff0000 },
 		{}
 	};
 	struct pci_dev *display_dev = NULL;
diff --git a/sound/isa/opti9xx/miro.c b/sound/isa/opti9xx/miro.c
index c320af3e9a05c..e5870e5e07db8 100644
--- a/sound/isa/opti9xx/miro.c
+++ b/sound/isa/opti9xx/miro.c
@@ -88,6 +88,25 @@ MODULE_PARM_DESC(isapnp, "Enable ISA PnP detection for specified soundcard.");
 
 #define OPTi9XX_MC_REG(n)	n
 
+enum {
+	MIRO_ACI_MASTER,
+	MIRO_ACI_MIC,
+	MIRO_ACI_LINE,
+	MIRO_ACI_CD,
+	MIRO_ACI_SYNTH,
+	MIRO_ACI_PCM,
+	MIRO_ACI_LINE1,
+	MIRO_ACI_LINE2,
+	MIRO_ACI_EQ1,
+	MIRO_ACI_EQ2,
+	MIRO_ACI_EQ3,
+	MIRO_ACI_EQ4,
+	MIRO_ACI_EQ5,
+	MIRO_ACI_EQ6,
+	MIRO_ACI_EQ7,
+	MIRO_ACI_COUNT,
+};
+
 struct snd_miro {
 	unsigned short hardware;
 	unsigned char password;
@@ -102,6 +121,7 @@ struct snd_miro {
 
 	spinlock_t lock;
 	struct snd_pcm *pcm;
+	struct snd_wss *codec;
 
 	long wss_base;
 	int irq;
@@ -113,6 +133,12 @@ struct snd_miro {
 
 	struct snd_card *card;
 	struct snd_miro_aci *aci;
+#ifdef CONFIG_PM
+	unsigned char aci_saved[MIRO_ACI_COUNT][2];
+	unsigned char aci_saved_amp;
+	unsigned char aci_saved_preamp;
+	unsigned char aci_saved_solomode;
+#endif
 };
 
 static struct snd_miro_aci aci_device;
@@ -664,6 +690,44 @@ static const unsigned char aci_init_values[][2] = {
 	{ ACI_SET_MASTER + 1, 0x20 },
 };
 
+#ifdef CONFIG_PM
+static const unsigned char snd_miro_saved_get_regs[MIRO_ACI_COUNT] = {
+	[MIRO_ACI_MASTER] = ACI_GET_MASTER,
+	[MIRO_ACI_MIC] = ACI_GET_MIC,
+	[MIRO_ACI_LINE] = ACI_GET_LINE,
+	[MIRO_ACI_CD] = ACI_GET_CD,
+	[MIRO_ACI_SYNTH] = ACI_GET_SYNTH,
+	[MIRO_ACI_PCM] = ACI_GET_PCM,
+	[MIRO_ACI_LINE1] = ACI_GET_LINE1,
+	[MIRO_ACI_LINE2] = ACI_GET_LINE2,
+	[MIRO_ACI_EQ1] = ACI_GET_EQ1,
+	[MIRO_ACI_EQ2] = ACI_GET_EQ2,
+	[MIRO_ACI_EQ3] = ACI_GET_EQ3,
+	[MIRO_ACI_EQ4] = ACI_GET_EQ4,
+	[MIRO_ACI_EQ5] = ACI_GET_EQ5,
+	[MIRO_ACI_EQ6] = ACI_GET_EQ6,
+	[MIRO_ACI_EQ7] = ACI_GET_EQ7,
+};
+
+static const unsigned char snd_miro_saved_set_regs[MIRO_ACI_COUNT] = {
+	[MIRO_ACI_MASTER] = ACI_SET_MASTER,
+	[MIRO_ACI_MIC] = ACI_SET_MIC,
+	[MIRO_ACI_LINE] = ACI_SET_LINE,
+	[MIRO_ACI_CD] = ACI_SET_CD,
+	[MIRO_ACI_SYNTH] = ACI_SET_SYNTH,
+	[MIRO_ACI_PCM] = ACI_SET_PCM,
+	[MIRO_ACI_LINE1] = ACI_SET_LINE1,
+	[MIRO_ACI_LINE2] = ACI_SET_LINE2,
+	[MIRO_ACI_EQ1] = ACI_SET_EQ1,
+	[MIRO_ACI_EQ2] = ACI_SET_EQ2,
+	[MIRO_ACI_EQ3] = ACI_SET_EQ3,
+	[MIRO_ACI_EQ4] = ACI_SET_EQ4,
+	[MIRO_ACI_EQ5] = ACI_SET_EQ5,
+	[MIRO_ACI_EQ6] = ACI_SET_EQ6,
+	[MIRO_ACI_EQ7] = ACI_SET_EQ7,
+};
+#endif
+
 static int snd_set_aci_init_values(struct snd_miro *miro)
 {
 	int idx, error;
@@ -702,11 +766,116 @@ static int snd_set_aci_init_values(struct snd_miro *miro)
 	}
 	aci->aci_amp = 0;
 	aci->aci_preamp = 0;
-	aci->aci_solomode = 1;
+	aci->aci_solomode = 0;
+
+	return 0;
+}
+
+static int snd_miro_aci_force_known_state(struct snd_miro_aci *aci)
+{
+	int i, err;
+
+	for (i = 0; i < 3; i++) {
+		err = snd_aci_cmd(aci, ACI_ERROR_OP, -1, -1);
+		if (err < 0)
+			return err;
+	}
 
 	return 0;
 }
 
+static int snd_miro_aci_initialize(struct snd_miro_aci *aci)
+{
+	int err;
+
+	err = snd_aci_cmd(aci, ACI_INIT, -1, -1);
+	if (err < 0)
+		return err;
+	err = snd_aci_cmd(aci, ACI_ERROR_OP, ACI_ERROR_OP, ACI_ERROR_OP);
+	if (err < 0)
+		return err;
+
+	return snd_aci_cmd(aci, ACI_ERROR_OP, ACI_ERROR_OP, ACI_ERROR_OP);
+}
+
+#ifdef CONFIG_PM
+static int snd_miro_save_aci_state(struct snd_miro *miro)
+{
+	struct snd_miro_aci *aci = miro->aci;
+	int i, limit, value;
+
+	limit = aci->aci_product == 'C' ? MIRO_ACI_COUNT : MIRO_ACI_LINE2 + 1;
+	for (i = 0; i < limit; i++) {
+		value = aci_getvalue(aci, snd_miro_saved_get_regs[i]);
+		if (value < 0)
+			return value;
+		miro->aci_saved[i][1] = value;
+
+		value = aci_getvalue(aci, snd_miro_saved_get_regs[i] + 1);
+		if (value < 0)
+			return value;
+		miro->aci_saved[i][0] = value;
+	}
+
+	miro->aci_saved_amp = aci->aci_amp;
+	if (aci->aci_version <= 176) {
+		miro->aci_saved_preamp = aci->aci_preamp;
+	} else {
+		value = aci_getvalue(aci, ACI_GET_PREAMP);
+		if (value < 0)
+			return value;
+		miro->aci_saved_preamp = value;
+	}
+
+	value = aci_getvalue(aci, ACI_S_GENERAL);
+	if (value < 0)
+		return value;
+	miro->aci_saved_solomode = !(value & 0x20);
+
+	return 0;
+}
+
+static int snd_miro_restore_aci_state(struct snd_miro *miro)
+{
+	struct snd_miro_aci *aci = miro->aci;
+	int i, limit, err, left_reg;
+
+	err = snd_set_aci_init_values(miro);
+	if (err < 0)
+		return err;
+
+	limit = aci->aci_product == 'C' ? MIRO_ACI_COUNT : MIRO_ACI_LINE2 + 1;
+	for (i = 0; i < limit; i++) {
+		left_reg = snd_miro_saved_set_regs[i] == ACI_SET_MASTER ?
+			snd_miro_saved_set_regs[i] + 1 :
+			snd_miro_saved_set_regs[i] + 8;
+		err = aci_setvalue(aci, left_reg, miro->aci_saved[i][0]);
+		if (err < 0)
+			return err;
+		err = aci_setvalue(aci, snd_miro_saved_set_regs[i],
+				   miro->aci_saved[i][1]);
+		if (err < 0)
+			return err;
+	}
+
+	err = aci_setvalue(aci, ACI_SET_POWERAMP, miro->aci_saved_amp);
+	if (err < 0)
+		return err;
+	err = aci_setvalue(aci, ACI_SET_PREAMP, miro->aci_saved_preamp);
+	if (err < 0)
+		return err;
+	err = aci_setvalue(aci, ACI_SET_SOLOMODE, miro->aci_saved_solomode);
+	if (err < 0)
+		return err;
+
+	aci->aci_amp = miro->aci_saved_amp;
+	aci->aci_preamp = miro->aci_saved_preamp;
+	aci->aci_solomode = miro->aci_saved_solomode;
+
+	return 0;
+}
+#endif
+
 static int snd_miro_mixer(struct snd_card *card,
 			  struct snd_miro *miro)
 {
@@ -1203,7 +1372,7 @@ static int snd_card_miro_aci_detect(struct snd_card *card,
 				    struct snd_miro *miro)
 {
 	unsigned char regval;
-	int i;
+	int err;
 	struct snd_miro_aci *aci = &aci_device;
 
 	miro->aci = aci;
@@ -1224,12 +1393,12 @@ static int snd_card_miro_aci_detect(struct snd_card *card,
 		return -ENOMEM;
 	}
 
-        /* force ACI into a known state */
-	for (i = 0; i < 3; i++)
-		if (snd_aci_cmd(aci, ACI_ERROR_OP, -1, -1) < 0) {
-			dev_err(card->dev, "can't force aci into known state.\n");
-			return -ENXIO;
-		}
+	/* force ACI into a known state */
+	err = snd_miro_aci_force_known_state(aci);
+	if (err < 0) {
+		dev_err(card->dev, "can't force aci into known state.\n");
+		return -ENXIO;
+	}
 
 	aci->aci_vendor = snd_aci_cmd(aci, ACI_READ_IDCODE, -1, -1);
 	aci->aci_product = snd_aci_cmd(aci, ACI_READ_IDCODE, -1, -1);
@@ -1246,9 +1415,8 @@ static int snd_card_miro_aci_detect(struct snd_card *card,
 		return -ENXIO;
 	}
 
-	if (snd_aci_cmd(aci, ACI_INIT, -1, -1) < 0 ||
-	    snd_aci_cmd(aci, ACI_ERROR_OP, ACI_ERROR_OP, ACI_ERROR_OP) < 0 ||
-	    snd_aci_cmd(aci, ACI_ERROR_OP, ACI_ERROR_OP, ACI_ERROR_OP) < 0) {
+	err = snd_miro_aci_initialize(aci);
+	if (err < 0) {
 		dev_err(card->dev, "can't initialize aci.\n");
 		return -ENXIO;
 	}
@@ -1299,6 +1467,7 @@ static int snd_miro_probe(struct snd_card *card)
 			       WSS_HW_DETECT, 0, &codec);
 	if (error < 0)
 		return error;
+	miro->codec = codec;
 
 	error = snd_wss_pcm(codec, 0);
 	if (error < 0)
@@ -1408,6 +1577,7 @@ static int snd_miro_isa_probe(struct device *devptr, unsigned int n)
 		return error;
 
 	miro = card->private_data;
+	miro->card = card;
 
 	error = snd_card_miro_detect(card, miro);
 	if (error < 0) {
@@ -1470,12 +1640,69 @@ static int snd_miro_isa_probe(struct device *devptr, unsigned int n)
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int snd_miro_suspend(struct snd_card *card)
+{
+	struct snd_miro *miro = card->private_data;
+	int error;
+
+	error = snd_miro_save_aci_state(miro);
+	if (error < 0)
+		return error;
+
+	snd_power_change_state(card, SNDRV_CTL_POWER_D3hot);
+	miro->codec->suspend(miro->codec);
+	return 0;
+}
+
+static int snd_miro_resume(struct snd_card *card)
+{
+	struct snd_miro *miro = card->private_data;
+	int error;
+
+	error = snd_miro_configure(miro);
+	if (error < 0)
+		return error;
+	error = snd_miro_aci_force_known_state(miro->aci);
+	if (error < 0) {
+		dev_err(card->dev, "can't force aci into known state\n");
+		return error;
+	}
+	error = snd_miro_aci_initialize(miro->aci);
+	if (error < 0) {
+		dev_err(card->dev, "can't initialize aci\n");
+		return error;
+	}
+	error = snd_miro_restore_aci_state(miro);
+	if (error < 0)
+		return error;
+
+	miro->codec->resume(miro->codec);
+	snd_power_change_state(card, SNDRV_CTL_POWER_D0);
+	return 0;
+}
+
+static int snd_miro_isa_suspend(struct device *dev, unsigned int n,
+				pm_message_t state)
+{
+	return snd_miro_suspend(dev_get_drvdata(dev));
+}
+
+static int snd_miro_isa_resume(struct device *dev, unsigned int n)
+{
+	return snd_miro_resume(dev_get_drvdata(dev));
+}
+#endif
+
 #define DEV_NAME "miro"
 
 static struct isa_driver snd_miro_driver = {
 	.match		= snd_miro_isa_match,
 	.probe		= snd_miro_isa_probe,
-	/* FIXME: suspend/resume */
+#ifdef CONFIG_PM
+	.suspend	= snd_miro_isa_suspend,
+	.resume		= snd_miro_isa_resume,
+#endif
 	.driver		= {
 		.name	= DEV_NAME
 	},
@@ -1591,12 +1818,29 @@ static void snd_miro_pnp_remove(struct pnp_card_link *pcard)
 	snd_miro_pnp_is_probed = 0;
 }
 
+#ifdef CONFIG_PM
+static int snd_miro_pnp_suspend(struct pnp_card_link *pcard,
+				pm_message_t state)
+{
+	return snd_miro_suspend(pnp_get_card_drvdata(pcard));
+}
+
+static int snd_miro_pnp_resume(struct pnp_card_link *pcard)
+{
+	return snd_miro_resume(pnp_get_card_drvdata(pcard));
+}
+#endif
+
 static struct pnp_card_driver miro_pnpc_driver = {
 	.flags		= PNP_DRIVER_RES_DISABLE,
 	.name		= "miro",
 	.id_table	= snd_miro_pnpids,
 	.probe		= snd_miro_pnp_probe,
 	.remove		= snd_miro_pnp_remove,
+#ifdef CONFIG_PM
+	.suspend	= snd_miro_pnp_suspend,
+	.resume		= snd_miro_pnp_resume,
+#endif
 };
 #endif
 
diff --git a/sound/isa/wavefront/wavefront.c b/sound/isa/wavefront/wavefront.c
index 07c68568091d1..f4a0c21c905e9 100644
--- a/sound/isa/wavefront/wavefront.c
+++ b/sound/isa/wavefront/wavefront.c
@@ -353,6 +353,7 @@ snd_wavefront_probe (struct snd_card *card, int dev)
 		dev_err(card->dev, "can't allocate WSS device\n");
 		return err;
 	}
+	acard->chip = chip;
 
 	err = snd_wss_pcm(chip, 0);
 	if (err < 0)
@@ -400,6 +401,7 @@ snd_wavefront_probe (struct snd_card *card, int dev)
 	acard->wavefront.irq = ics2115_irq[dev];
 	card->sync_irq = acard->wavefront.irq;
 	acard->wavefront.base = ics2115_port[dev];
+	snd_wavefront_cache_firmware(&acard->wavefront);
 
 	wavefront_synth = snd_wavefront_new_synth(card, hw_dev, acard);
 	if (wavefront_synth == NULL) {
@@ -553,12 +555,51 @@ static int snd_wavefront_isa_probe(struct device *pdev,
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int snd_wavefront_suspend(struct snd_card *card)
+{
+	snd_wavefront_card_t *acard = card->private_data;
+
+	snd_wavefront_midi_suspend(acard);
+	snd_power_change_state(card, SNDRV_CTL_POWER_D3hot);
+	acard->chip->suspend(acard->chip);
+	return 0;
+}
+
+static int snd_wavefront_resume(struct snd_card *card)
+{
+	snd_wavefront_card_t *acard = card->private_data;
+	int err;
+
+	acard->chip->resume(acard->chip);
+	err = snd_wavefront_resume_synth(acard);
+	if (err < 0)
+		return err;
+	snd_power_change_state(card, SNDRV_CTL_POWER_D0);
+	return 0;
+}
+
+static int snd_wavefront_isa_suspend(struct device *dev, unsigned int id,
+				     pm_message_t state)
+{
+	return snd_wavefront_suspend(dev_get_drvdata(dev));
+}
+
+static int snd_wavefront_isa_resume(struct device *dev, unsigned int id)
+{
+	return snd_wavefront_resume(dev_get_drvdata(dev));
+}
+#endif
+
 #define DEV_NAME "wavefront"
 
 static struct isa_driver snd_wavefront_driver = {
 	.match		= snd_wavefront_isa_match,
 	.probe		= snd_wavefront_isa_probe,
-	/* FIXME: suspend, resume */
+#ifdef CONFIG_PM
+	.suspend	= snd_wavefront_isa_suspend,
+	.resume		= snd_wavefront_isa_resume,
+#endif
 	.driver		= {
 		.name	= DEV_NAME
 	},
@@ -600,12 +641,28 @@ static int snd_wavefront_pnp_detect(struct pnp_card_link *pcard,
 	return 0;
 }
 
+#ifdef CONFIG_PM
+static int snd_wavefront_pnpc_suspend(struct pnp_card_link *pcard,
+				      pm_message_t state)
+{
+	return snd_wavefront_suspend(pnp_get_card_drvdata(pcard));
+}
+
+static int snd_wavefront_pnpc_resume(struct pnp_card_link *pcard)
+{
+	return snd_wavefront_resume(pnp_get_card_drvdata(pcard));
+}
+#endif
+
 static struct pnp_card_driver wavefront_pnpc_driver = {
 	.flags		= PNP_DRIVER_RES_DISABLE,
 	.name		= "wavefront",
 	.id_table	= snd_wavefront_pnpids,
 	.probe		= snd_wavefront_pnp_detect,
-	/* FIXME: suspend,resume */
+#ifdef CONFIG_PM
+	.suspend	= snd_wavefront_pnpc_suspend,
+	.resume		= snd_wavefront_pnpc_resume,
+#endif
 };
 
 #endif /* CONFIG_PNP */
diff --git a/sound/isa/wavefront/wavefront_midi.c b/sound/isa/wavefront/wavefront_midi.c
index 69d87c4cafaed..fb184d9ef284a 100644
--- a/sound/isa/wavefront/wavefront_midi.c
+++ b/sound/isa/wavefront/wavefront_midi.c
@@ -455,6 +455,49 @@ snd_wavefront_midi_disable_virtual (snd_wavefront_card_t *card)
 	card->wavefront.midi.isvirtual = 0;
 }
 
+void
+snd_wavefront_midi_suspend(snd_wavefront_card_t *card)
+
+{
+	snd_wavefront_midi_t *midi = &card->wavefront.midi;
+
+	if (!midi->istimer)
+		return;
+
+	timer_delete_sync(&midi->timer);
+
+	guard(spinlock_irqsave)(&midi->virtual);
+	midi->istimer = 0;
+}
+
+void
+snd_wavefront_midi_resume(snd_wavefront_card_t *card)
+
+{
+	snd_wavefront_midi_t *midi = &card->wavefront.midi;
+	int istimer = 0;
+	bool pending_output = false;
+
+	midi->timer_card = card;
+
+	scoped_guard(spinlock_irqsave, &midi->virtual) {
+		if (midi->mode[internal_mpu] & MPU401_MODE_OUTPUT_TRIGGER)
+			istimer++;
+		if (midi->mode[external_mpu] & MPU401_MODE_OUTPUT_TRIGGER)
+			istimer++;
+		if (!istimer)
+			return;
+
+		midi->istimer = istimer;
+		timer_setup(&midi->timer, snd_wavefront_midi_output_timer, 0);
+		mod_timer(&midi->timer, 1 + jiffies);
+		pending_output = true;
+	}
+
+	if (pending_output)
+		snd_wavefront_midi_output_write(card);
+}
+
 int
 snd_wavefront_midi_start (snd_wavefront_card_t *card)
 
@@ -466,6 +509,7 @@ snd_wavefront_midi_start (snd_wavefront_card_t *card)
 
 	dev = &card->wavefront;
 	midi = &dev->midi;
+	midi->timer_card = card;
 
 	/* The ICS2115 MPU-401 interface doesn't do anything
 	   until its set into UART mode.
@@ -511,6 +555,8 @@ snd_wavefront_midi_start (snd_wavefront_card_t *card)
 		dev_warn(card->wavefront.card->dev,
 			 "can't enable MIDI-IN-2-synth routing.\n");
 		/* XXX error ? */
+	} else {
+		dev->midi_in_to_synth = 1;
 	}
 
 	/* Turn on Virtual MIDI, but first *always* turn it off,
@@ -553,4 +599,3 @@ const struct snd_rawmidi_ops snd_wavefront_midi_input =
 	.close =	snd_wavefront_midi_input_close,
 	.trigger =	snd_wavefront_midi_input_trigger,
 };
-
diff --git a/sound/isa/wavefront/wavefront_synth.c b/sound/isa/wavefront/wavefront_synth.c
index 33b563707a58d..2f57a6795d22b 100644
--- a/sound/isa/wavefront/wavefront_synth.c
+++ b/sound/isa/wavefront/wavefront_synth.c
@@ -1626,6 +1626,14 @@ wavefront_synth_control (snd_wavefront_card_t *acard,
 				"support for sample aliases still being considered.\n");
 			break;
 
+		case WFC_MISYNTH_OFF:
+			dev->midi_in_to_synth = 0;
+			break;
+
+		case WFC_MISYNTH_ON:
+			dev->midi_in_to_synth = 1;
+			break;
+
 		case WFC_VMIDI_OFF:
 			snd_wavefront_midi_disable_virtual (acard);
 			break;
@@ -1639,6 +1647,83 @@ wavefront_synth_control (snd_wavefront_card_t *acard,
 	return 0;
 }
 
+static int
+wavefront_restore_midi_state(snd_wavefront_card_t *acard, char isvirtual,
+			     char midi_in_to_synth)
+{
+	snd_wavefront_t *dev = &acard->wavefront;
+	unsigned char rbuf[4], wbuf[4];
+
+	if (dev->midi_in_to_synth != midi_in_to_synth) {
+		if (snd_wavefront_cmd(dev, midi_in_to_synth ?
+				      WFC_MISYNTH_ON : WFC_MISYNTH_OFF,
+				      rbuf, wbuf)) {
+			dev_err(dev->card->dev,
+				"cannot restore MIDI-IN routing after resume\n");
+			return -EIO;
+		}
+		dev->midi_in_to_synth = midi_in_to_synth;
+	}
+
+	if (dev->midi.isvirtual != isvirtual) {
+		if (snd_wavefront_cmd(dev, isvirtual ?
+				      WFC_VMIDI_ON : WFC_VMIDI_OFF,
+				      rbuf, wbuf)) {
+			dev_err(dev->card->dev,
+				"cannot restore virtual MIDI mode after resume\n");
+			return -EIO;
+		}
+		if (isvirtual)
+			snd_wavefront_midi_enable_virtual(acard);
+		else
+			snd_wavefront_midi_disable_virtual(acard);
+	}
+
+	return 0;
+}
+
+int snd_wavefront_resume_synth(snd_wavefront_card_t *acard)
+{
+	snd_wavefront_t *dev = &acard->wavefront;
+	char was_virtual = dev->midi.isvirtual;
+	char midi_in_to_synth = dev->midi_in_to_synth;
+	char rom_samples_rdonly = dev->rom_samples_rdonly;
+	int err;
+
+	err = snd_wavefront_detect(acard);
+	if (err < 0)
+		dev->israw = 1;
+
+	if (dev->israw) {
+		dev->fx_initialized = 0;
+		err = snd_wavefront_start(dev);
+		if (err < 0)
+			return err;
+	} else {
+		dev->has_fx = (snd_wavefront_fx_detect(dev) == 0);
+		wavefront_get_sample_status(dev, 0);
+		wavefront_get_program_status(dev);
+		wavefront_get_patch_status(dev);
+		outb(0x80 | 0x40 | 0x20, dev->control_port);
+	}
+
+	dev->rom_samples_rdonly = rom_samples_rdonly;
+	dev->midi.base = dev->base;
+
+	err = snd_wavefront_midi_start(acard);
+	if (err < 0)
+		return err;
+
+	err = wavefront_restore_midi_state(acard, was_virtual,
+					   midi_in_to_synth);
+	if (err < 0)
+		return err;
+
+	snd_wavefront_midi_resume(acard);
+
+	return 0;
+}
+
 int 
 snd_wavefront_synth_open (struct snd_hwdep *hw, struct file *file)
 
@@ -2032,6 +2117,17 @@ wavefront_download_firmware (snd_wavefront_t *dev, char *path)
 	return 1;
 }
 
+void snd_wavefront_cache_firmware(snd_wavefront_t *dev)
+{
+	int err;
+
+	err = firmware_request_cache(dev->card->dev, ospath);
+	if (err < 0)
+		dev_warn(dev->card->dev,
+			 "unable to cache firmware %s for resume: %d\n",
+			 ospath, err);
+}
+
 
 static int
 wavefront_do_reset (snd_wavefront_t *dev)
diff --git a/sound/oss/dmasound/dmasound_core.c b/sound/oss/dmasound/dmasound_core.c
index a718b75bb6a09..e80f730d08038 100644
--- a/sound/oss/dmasound/dmasound_core.c
+++ b/sound/oss/dmasound/dmasound_core.c
@@ -574,11 +574,6 @@ static ssize_t sq_write(struct file *file, const char __user *src, size_t uLeft,
 		uWritten = 0 ;
 	}
 
-/* FIXME: I think that this may be the wrong behaviour when we get strapped
-	for time and the cpu is close to being (or actually) behind in sending data.
-	- because we've lost the time that the N samples, already in the buffer,
-	would have given us to get here with the next lot from the user.
-*/
 	/* The interrupt doesn't start to play the last, incomplete frame.
 	 * Thus we can append to it without disabling the interrupts! (Note
 	 * also that write_sq.rear isn't affected by the interrupt.)
@@ -598,6 +593,11 @@ static ssize_t sq_write(struct file *file, const char __user *src, size_t uLeft,
 	write_sq.syncing &= ~2 ; /* take out POST status */
 	spin_unlock_irqrestore(&dmasound.lock, flags);
 
+	/* Start any already-complete fragments before we spend
+	 * more time extending the incomplete tail fragment.
+	 */
+	sq_play();
+
 	if (write_sq.count > 0 &&
 	    (bLeft = write_sq.block_size-write_sq.rear_size) > 0) {
 		dest = write_sq.buffers[write_sq.rear];
diff --git a/sound/pci/ali5451/ali5451.c b/sound/pci/ali5451/ali5451.c
index 571d89a6a8da2..f07c5f4c23801 100644
--- a/sound/pci/ali5451/ali5451.c
+++ b/sound/pci/ali5451/ali5451.c
@@ -247,8 +247,8 @@ struct snd_ali {
 };
 
 static const struct pci_device_id snd_ali_ids[] = {
-	{PCI_DEVICE(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M5451), 0, 0, 0},
-	{0, }
+	{ PCI_DEVICE(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M5451) },
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, snd_ali_ids);
 
diff --git a/sound/pci/als300.c b/sound/pci/als300.c
index a73893a2cbd66..00f0720169c3a 100644
--- a/sound/pci/als300.c
+++ b/sound/pci/als300.c
@@ -127,9 +127,9 @@ struct snd_als300_substream_data {
 };
 
 static const struct pci_device_id snd_als300_ids[] = {
-	{ 0x4005, 0x0300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, DEVICE_ALS300 },
-	{ 0x4005, 0x0308, PCI_ANY_ID, PCI_ANY_ID, 0, 0, DEVICE_ALS300_PLUS },
-	{ 0, }
+	{ PCI_DEVICE(0x4005, 0x0300), .driver_data = DEVICE_ALS300 },
+	{ PCI_DEVICE(0x4005, 0x0308), .driver_data = DEVICE_ALS300_PLUS },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_als300_ids);
diff --git a/sound/pci/als4000.c b/sound/pci/als4000.c
index 636f309c94240..6396aa6c3bf07 100644
--- a/sound/pci/als4000.c
+++ b/sound/pci/als4000.c
@@ -102,8 +102,8 @@ struct snd_card_als4000 {
 };
 
 static const struct pci_device_id snd_als4000_ids[] = {
-	{ 0x4005, 0x4000, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0, },   /* ALS4000 */
-	{ 0, }
+	{ PCI_DEVICE(0x4005, 0x4000) },   /* ALS4000 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_als4000_ids);
diff --git a/sound/pci/asihpi/asihpi.c b/sound/pci/asihpi/asihpi.c
index b1c7ed7f1604e..4dbc79899c091 100644
--- a/sound/pci/asihpi/asihpi.c
+++ b/sound/pci/asihpi/asihpi.c
@@ -2933,13 +2933,16 @@ static void snd_asihpi_remove(struct pci_dev *pci_dev)
 }
 
 static const struct pci_device_id asihpi_pci_tbl[] = {
-	{HPI_PCI_VENDOR_ID_TI, HPI_PCI_DEV_ID_DSP6205,
-		HPI_PCI_VENDOR_ID_AUDIOSCIENCE, PCI_ANY_ID, 0, 0,
-		(kernel_ulong_t)HPI_6205},
-	{HPI_PCI_VENDOR_ID_TI, HPI_PCI_DEV_ID_PCI2040,
-		HPI_PCI_VENDOR_ID_AUDIOSCIENCE, PCI_ANY_ID, 0, 0,
-		(kernel_ulong_t)HPI_6000},
-	{0,}
+	{
+		PCI_DEVICE_SUB(HPI_PCI_VENDOR_ID_TI, HPI_PCI_DEV_ID_DSP6205,
+			       HPI_PCI_VENDOR_ID_AUDIOSCIENCE, PCI_ANY_ID),
+		.driver_data = (kernel_ulong_t)HPI_6205,
+	}, {
+		PCI_DEVICE_SUB(HPI_PCI_VENDOR_ID_TI, HPI_PCI_DEV_ID_PCI2040,
+			       HPI_PCI_VENDOR_ID_AUDIOSCIENCE, PCI_ANY_ID),
+		.driver_data = (kernel_ulong_t)HPI_6000,
+	},
+	{}
 };
 MODULE_DEVICE_TABLE(pci, asihpi_pci_tbl);
 
diff --git a/sound/pci/asihpi/hpicmn.c b/sound/pci/asihpi/hpicmn.c
index d846777e7462b..5f69c04c672aa 100644
--- a/sound/pci/asihpi/hpicmn.c
+++ b/sound/pci/asihpi/hpicmn.c
@@ -276,6 +276,12 @@ static short find_control(u16 control_index,
 		return 0;
 	}
 
+	if (control_index >= p_cache->control_count) {
+		HPI_DEBUG_LOG(VERBOSE, "control_index out of bounce %d\n",
+			control_index);
+		return 0;
+	}
+
 	*pI = p_cache->p_info[control_index];
 	if (!*pI) {
 		HPI_DEBUG_LOG(VERBOSE, "Uncached Control %d\n",
@@ -641,16 +647,11 @@ void hpi_cmn_control_cache_sync_to_msg(struct hpi_control_cache *p_cache,
 struct hpi_control_cache *hpi_alloc_control_cache(const u32 control_count,
 	const u32 size_in_bytes, u8 *p_dsp_control_buffer)
 {
-	struct hpi_control_cache *p_cache = kmalloc_obj(*p_cache);
-	if (!p_cache)
-		return NULL;
+	struct hpi_control_cache *p_cache;
 
-	p_cache->p_info =
-		kzalloc_objs(*p_cache->p_info, control_count);
-	if (!p_cache->p_info) {
-		kfree(p_cache);
+	p_cache = kzalloc_flex(*p_cache, p_info, control_count);
+	if (!p_cache)
 		return NULL;
-	}
 
 	p_cache->cache_size_in_bytes = size_in_bytes;
 	p_cache->control_count = control_count;
@@ -661,10 +662,7 @@ struct hpi_control_cache *hpi_alloc_control_cache(const u32 control_count,
 
 void hpi_free_control_cache(struct hpi_control_cache *p_cache)
 {
-	if (p_cache) {
-		kfree(p_cache->p_info);
-		kfree(p_cache);
-	}
+	kfree(p_cache);
 }
 
 static void subsys_message(struct hpi_message *phm, struct hpi_response *phr)
diff --git a/sound/pci/asihpi/hpicmn.h b/sound/pci/asihpi/hpicmn.h
index 8ec656cf8848c..40af7329587a1 100644
--- a/sound/pci/asihpi/hpicmn.h
+++ b/sound/pci/asihpi/hpicmn.h
@@ -37,10 +37,10 @@ struct hpi_control_cache {
 	u16 adap_idx;
 	u32 control_count;
 	u32 cache_size_in_bytes;
-	/** pointer to allocated memory of lookup pointers. */
-	struct hpi_control_cache_info **p_info;
 	/** pointer to DSP's control cache. */
 	u8 *p_cache;
+	/** pointer to allocated memory of lookup pointers. */
+	struct hpi_control_cache_info *p_info[] __counted_by(control_count);
 };
 
 struct hpi_adapter_obj *hpi_find_adapter(u16 adapter_index);
diff --git a/sound/pci/asihpi/hpipcida.h b/sound/pci/asihpi/hpipcida.h
index 0673e82780707..6dceff2b47419 100644
--- a/sound/pci/asihpi/hpipcida.h
+++ b/sound/pci/asihpi/hpipcida.h
@@ -15,12 +15,12 @@
 */
 
 {
-HPI_PCI_VENDOR_ID_TI, HPI_PCI_DEV_ID_DSP6205,
-		HPI_PCI_VENDOR_ID_AUDIOSCIENCE, PCI_ANY_ID, 0, 0,
-		(kernel_ulong_t) HPI_6205}
-, {
-HPI_PCI_VENDOR_ID_TI, HPI_PCI_DEV_ID_PCI2040,
-		HPI_PCI_VENDOR_ID_AUDIOSCIENCE, PCI_ANY_ID, 0, 0,
-		(kernel_ulong_t) HPI_6000}
-, {
-0}
+	PCI_DEVICE_SUB(HPI_PCI_VENDOR_ID_TI, HPI_PCI_DEV_ID_DSP6205,
+		       HPI_PCI_VENDOR_ID_AUDIOSCIENCE, PCI_ANY_ID),
+	.driver_data = (kernel_ulong_t) HPI_6205,
+}, {
+	PCI_DEVICE_SUB(HPI_PCI_VENDOR_ID_TI, HPI_PCI_DEV_ID_PCI2040,
+		       HPI_PCI_VENDOR_ID_AUDIOSCIENCE, PCI_ANY_ID),
+	.driver_data = (kernel_ulong_t) HPI_6000,
+},
+{ }
diff --git a/sound/pci/atiixp.c b/sound/pci/atiixp.c
index 2a0c59d5afa58..b738295b41e5d 100644
--- a/sound/pci/atiixp.c
+++ b/sound/pci/atiixp.c
@@ -272,11 +272,11 @@ struct atiixp {
 /*
  */
 static const struct pci_device_id snd_atiixp_ids[] = {
-	{ PCI_VDEVICE(ATI, 0x4341), 0 }, /* SB200 */
-	{ PCI_VDEVICE(ATI, 0x4361), 0 }, /* SB300 */
-	{ PCI_VDEVICE(ATI, 0x4370), 0 }, /* SB400 */
-	{ PCI_VDEVICE(ATI, 0x4382), 0 }, /* SB600 */
-	{ 0, }
+	{ PCI_VDEVICE(ATI, 0x4341) }, /* SB200 */
+	{ PCI_VDEVICE(ATI, 0x4361) }, /* SB300 */
+	{ PCI_VDEVICE(ATI, 0x4370) }, /* SB400 */
+	{ PCI_VDEVICE(ATI, 0x4382) }, /* SB600 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_atiixp_ids);
diff --git a/sound/pci/atiixp_modem.c b/sound/pci/atiixp_modem.c
index 91f31e2ad3d3b..8aaeb197ce458 100644
--- a/sound/pci/atiixp_modem.c
+++ b/sound/pci/atiixp_modem.c
@@ -247,9 +247,9 @@ struct atiixp_modem {
 /*
  */
 static const struct pci_device_id snd_atiixp_ids[] = {
-	{ PCI_VDEVICE(ATI, 0x434d), 0 }, /* SB200 */
-	{ PCI_VDEVICE(ATI, 0x4378), 0 }, /* SB400 */
-	{ 0, }
+	{ PCI_VDEVICE(ATI, 0x434d) }, /* SB200 */
+	{ PCI_VDEVICE(ATI, 0x4378) }, /* SB400 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_atiixp_ids);
diff --git a/sound/pci/au88x0/au8810.c b/sound/pci/au88x0/au8810.c
index b2bfa50bfe30e..f712e32d33700 100644
--- a/sound/pci/au88x0/au8810.c
+++ b/sound/pci/au88x0/au8810.c
@@ -2,8 +2,8 @@
 #include "au8810.h"
 #include "au88x0.h"
 static const struct pci_device_id snd_vortex_ids[] = {
-	{PCI_VDEVICE(AUREAL, PCI_DEVICE_ID_AUREAL_ADVANTAGE), 1,},
-	{0,}
+	{ PCI_VDEVICE(AUREAL, PCI_DEVICE_ID_AUREAL_ADVANTAGE), .driver_data = 1 },
+	{ }
 };
 
 #include "au88x0_core.c"
diff --git a/sound/pci/au88x0/au8820.c b/sound/pci/au88x0/au8820.c
index dbc2263b49c6e..aa841b615182c 100644
--- a/sound/pci/au88x0/au8820.c
+++ b/sound/pci/au88x0/au8820.c
@@ -2,8 +2,8 @@
 #include "au8820.h"
 #include "au88x0.h"
 static const struct pci_device_id snd_vortex_ids[] = {
-	{PCI_VDEVICE(AUREAL, PCI_DEVICE_ID_AUREAL_VORTEX_1), 0,},
-	{0,}
+	{ PCI_VDEVICE(AUREAL, PCI_DEVICE_ID_AUREAL_VORTEX_1), .driver_data = 0 },
+	{ }
 };
 
 #include "au88x0_synth.c"
diff --git a/sound/pci/au88x0/au8830.c b/sound/pci/au88x0/au8830.c
index e963c4e2f026c..aeb8d458c6299 100644
--- a/sound/pci/au88x0/au8830.c
+++ b/sound/pci/au88x0/au8830.c
@@ -2,8 +2,8 @@
 #include "au8830.h"
 #include "au88x0.h"
 static const struct pci_device_id snd_vortex_ids[] = {
-	{PCI_VDEVICE(AUREAL, PCI_DEVICE_ID_AUREAL_VORTEX_2), 0,},
-	{0,}
+	{ PCI_VDEVICE(AUREAL, PCI_DEVICE_ID_AUREAL_VORTEX_2), .driver_data = 0 },
+	{ }
 };
 
 #include "au88x0_synth.c"
diff --git a/sound/pci/aw2/aw2-alsa.c b/sound/pci/aw2/aw2-alsa.c
index e2c501f4394c1..60a87322eae49 100644
--- a/sound/pci/aw2/aw2-alsa.c
+++ b/sound/pci/aw2/aw2-alsa.c
@@ -142,9 +142,8 @@ module_param_array(enable, bool, NULL, 0444);
 MODULE_PARM_DESC(enable, "Enable Audiowerk2 soundcard.");
 
 static const struct pci_device_id snd_aw2_ids[] = {
-	{PCI_VENDOR_ID_PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7146, 0, 0,
-	 0, 0, 0},
-	{0}
+	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_PHILIPS, PCI_DEVICE_ID_PHILIPS_SAA7146, 0, 0) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_aw2_ids);
diff --git a/sound/pci/azt3328.c b/sound/pci/azt3328.c
index 6cdf76e2b7d2e..ccca82417657b 100644
--- a/sound/pci/azt3328.c
+++ b/sound/pci/azt3328.c
@@ -305,9 +305,9 @@ struct snd_azf3328 {
 };
 
 static const struct pci_device_id snd_azf3328_ids[] = {
-	{ 0x122D, 0x50DC, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },   /* PCI168/3328 */
-	{ 0x122D, 0x80DA, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0 },   /* 3328 */
-	{ 0, }
+	{ PCI_DEVICE(0x122D, 0x50DC) },   /* PCI168/3328 */
+	{ PCI_DEVICE(0x122D, 0x80DA) },   /* 3328 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_azf3328_ids);
diff --git a/sound/pci/ca0106/ca0106_main.c b/sound/pci/ca0106/ca0106_main.c
index 35392f6525b3b..cf923f551211e 100644
--- a/sound/pci/ca0106/ca0106_main.c
+++ b/sound/pci/ca0106/ca0106_main.c
@@ -1820,8 +1820,8 @@ static SIMPLE_DEV_PM_OPS(snd_ca0106_pm, snd_ca0106_suspend, snd_ca0106_resume);
 
 // PCI IDs
 static const struct pci_device_id snd_ca0106_ids[] = {
-	{ PCI_VDEVICE(CREATIVE, 0x0007), 0 },	/* Audigy LS or Live 24bit */
-	{ 0, }
+	{ PCI_VDEVICE(CREATIVE, 0x0007) },	/* Audigy LS or Live 24bit */
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, snd_ca0106_ids);
 
diff --git a/sound/pci/cmipci.c b/sound/pci/cmipci.c
index cd73b68336394..f5382b10865a3 100644
--- a/sound/pci/cmipci.c
+++ b/sound/pci/cmipci.c
@@ -2721,12 +2721,12 @@ static void snd_cmipci_proc_init(struct cmipci *cm)
 }
 
 static const struct pci_device_id snd_cmipci_ids[] = {
-	{PCI_VDEVICE(CMEDIA, PCI_DEVICE_ID_CMEDIA_CM8338A), 0},
-	{PCI_VDEVICE(CMEDIA, PCI_DEVICE_ID_CMEDIA_CM8338B), 0},
-	{PCI_VDEVICE(CMEDIA, PCI_DEVICE_ID_CMEDIA_CM8738), 0},
-	{PCI_VDEVICE(CMEDIA, PCI_DEVICE_ID_CMEDIA_CM8738B), 0},
-	{PCI_VDEVICE(AL, PCI_DEVICE_ID_CMEDIA_CM8738), 0},
-	{0,},
+	{PCI_VDEVICE(CMEDIA, PCI_DEVICE_ID_CMEDIA_CM8338A) },
+	{PCI_VDEVICE(CMEDIA, PCI_DEVICE_ID_CMEDIA_CM8338B) },
+	{PCI_VDEVICE(CMEDIA, PCI_DEVICE_ID_CMEDIA_CM8738) },
+	{PCI_VDEVICE(CMEDIA, PCI_DEVICE_ID_CMEDIA_CM8738B) },
+	{PCI_VDEVICE(AL, PCI_DEVICE_ID_CMEDIA_CM8738) },
+	{ }
 };
 
 
diff --git a/sound/pci/cs4281.c b/sound/pci/cs4281.c
index d00b2c9fb1e38..f51f4bb637664 100644
--- a/sound/pci/cs4281.c
+++ b/sound/pci/cs4281.c
@@ -476,8 +476,8 @@ struct cs4281 {
 static irqreturn_t snd_cs4281_interrupt(int irq, void *dev_id);
 
 static const struct pci_device_id snd_cs4281_ids[] = {
-	{ PCI_VDEVICE(CIRRUS, 0x6005), 0, },	/* CS4281 */
-	{ 0, }
+	{ PCI_VDEVICE(CIRRUS, 0x6005) },	/* CS4281 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_cs4281_ids);
diff --git a/sound/pci/cs46xx/cs46xx.c b/sound/pci/cs46xx/cs46xx.c
index 9c1995737eb7f..0cb7c2a2929fe 100644
--- a/sound/pci/cs46xx/cs46xx.c
+++ b/sound/pci/cs46xx/cs46xx.c
@@ -43,10 +43,10 @@ module_param_array(mmap_valid, bool, NULL, 0444);
 MODULE_PARM_DESC(mmap_valid, "Support OSS mmap.");
 
 static const struct pci_device_id snd_cs46xx_ids[] = {
-	{ PCI_VDEVICE(CIRRUS, 0x6001), 0, },   /* CS4280 */
-	{ PCI_VDEVICE(CIRRUS, 0x6003), 0, },   /* CS4612 */
-	{ PCI_VDEVICE(CIRRUS, 0x6004), 0, },   /* CS4615 */
-	{ 0, }
+	{ PCI_VDEVICE(CIRRUS, 0x6001) },   /* CS4280 */
+	{ PCI_VDEVICE(CIRRUS, 0x6003) },   /* CS4612 */
+	{ PCI_VDEVICE(CIRRUS, 0x6004) },   /* CS4615 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_cs46xx_ids);
diff --git a/sound/pci/ctxfi/ctmixer.c b/sound/pci/ctxfi/ctmixer.c
index e3ee76bd84824..9abe1d53c3c5e 100644
--- a/sound/pci/ctxfi/ctmixer.c
+++ b/sound/pci/ctxfi/ctmixer.c
@@ -221,10 +221,6 @@ ct_mixer_recording_select(struct ct_mixer *mixer, enum CT_AMIXER_CTL type);
 static void
 ct_mixer_recording_unselect(struct ct_mixer *mixer, enum CT_AMIXER_CTL type);
 
-/* FIXME: this static looks like it would fail if more than one card was */
-/* installed. */
-static struct snd_kcontrol *kctls[2] = {NULL};
-
 static enum CT_AMIXER_CTL get_amixer_index(enum CTALSA_MIXER_CTL alsa_index)
 {
 	switch (alsa_index) {
@@ -481,17 +477,18 @@ static struct snd_kcontrol_new mic_source_ctl = {
 static void
 do_line_mic_switch(struct ct_atc *atc, enum CTALSA_MIXER_CTL type)
 {
+	struct ct_mixer *mixer = atc->mixer;
 
 	if (MIXER_LINEIN_C_S == type) {
 		atc->select_line_in(atc);
-		set_switch_state(atc->mixer, MIXER_MIC_C_S, 0);
+		set_switch_state(mixer, MIXER_MIC_C_S, 0);
 		snd_ctl_notify(atc->card, SNDRV_CTL_EVENT_MASK_VALUE,
-							&kctls[1]->id);
+			       &mixer->line_mic_kctls[1]->id);
 	} else if (MIXER_MIC_C_S == type) {
 		atc->select_mic_in(atc);
-		set_switch_state(atc->mixer, MIXER_LINEIN_C_S, 0);
+		set_switch_state(mixer, MIXER_LINEIN_C_S, 0);
 		snd_ctl_notify(atc->card, SNDRV_CTL_EVENT_MASK_VALUE,
-							&kctls[0]->id);
+			       &mixer->line_mic_kctls[0]->id);
 	}
 }
 
@@ -778,9 +775,9 @@ ct_mixer_kcontrol_new(struct ct_mixer *mixer, struct snd_kcontrol_new *new)
 
 	switch (new->private_value) {
 	case MIXER_LINEIN_C_S:
-		kctls[0] = kctl; break;
+		mixer->line_mic_kctls[0] = kctl; break;
 	case MIXER_MIC_C_S:
-		kctls[1] = kctl; break;
+		mixer->line_mic_kctls[1] = kctl; break;
 	default:
 		break;
 	}
@@ -965,35 +962,20 @@ static int ct_mixer_get_resources(struct ct_mixer *mixer)
 static int ct_mixer_get_mem(struct ct_mixer **rmixer)
 {
 	struct ct_mixer *mixer;
-	int err;
+	size_t alloc_size;
 
 	*rmixer = NULL;
 	/* Allocate mem for mixer obj */
-	mixer = kzalloc_obj(*mixer);
+	alloc_size = struct_size(mixer, amixers, NUM_CT_AMIXERS * CHN_NUM);
+	alloc_size += sizeof(*mixer->sums) * NUM_CT_SUMS * CHN_NUM;
+	mixer = kzalloc(alloc_size, GFP_KERNEL);
 	if (!mixer)
 		return -ENOMEM;
 
-	mixer->amixers = kcalloc(NUM_CT_AMIXERS * CHN_NUM, sizeof(void *),
-				 GFP_KERNEL);
-	if (!mixer->amixers) {
-		err = -ENOMEM;
-		goto error1;
-	}
-	mixer->sums = kcalloc(NUM_CT_SUMS * CHN_NUM, sizeof(void *),
-			      GFP_KERNEL);
-	if (!mixer->sums) {
-		err = -ENOMEM;
-		goto error2;
-	}
+	mixer->sums = (struct sum **)(mixer->amixers + (NUM_CT_AMIXERS * CHN_NUM));
 
 	*rmixer = mixer;
 	return 0;
-
-error2:
-	kfree(mixer->amixers);
-error1:
-	kfree(mixer);
-	return err;
 }
 
 static int ct_mixer_topology_build(struct ct_mixer *mixer)
@@ -1228,8 +1210,6 @@ int ct_mixer_destroy(struct ct_mixer *mixer)
 	}
 
 	/* Release mem assigned to mixer object */
-	kfree(mixer->sums);
-	kfree(mixer->amixers);
 	kfree(mixer);
 
 	return 0;
diff --git a/sound/pci/ctxfi/ctmixer.h b/sound/pci/ctxfi/ctmixer.h
index e812f6c93b41c..0c0963bba99c8 100644
--- a/sound/pci/ctxfi/ctmixer.h
+++ b/sound/pci/ctxfi/ctmixer.h
@@ -17,6 +17,8 @@
 #include "ctatc.h"
 #include "ctresource.h"
 
+struct snd_kcontrol;
+
 #define INIT_VOL	0x1c00
 
 enum MIXER_PORT_T {
@@ -41,8 +43,8 @@ enum MIXER_PORT_T {
 struct ct_mixer {
 	struct ct_atc *atc;
 
-	void **amixers;		/* amixer resources for volume control */
-	void **sums;		/* sum resources for signal collection */
+	struct sum **sums;		/* sum resources for signal collection */
+	struct snd_kcontrol *line_mic_kctls[2]; /* line/mic capture switch controls */
 	unsigned int switch_state; /* A bit-map to indicate state of switches */
 
 	int (*get_output_ports)(struct ct_mixer *mixer, enum MIXER_PORT_T type,
@@ -55,6 +57,7 @@ struct ct_mixer {
 #ifdef CONFIG_PM_SLEEP
 	int (*resume)(struct ct_mixer *mixer);
 #endif
+	struct amixer *amixers[];		/* amixer resources for volume control */
 };
 
 int ct_alsa_mix_create(struct ct_atc *atc,
diff --git a/sound/pci/ctxfi/ctsrc.c b/sound/pci/ctxfi/ctsrc.c
index 326997bb885df..46dc1f509234f 100644
--- a/sound/pci/ctxfi/ctsrc.c
+++ b/sound/pci/ctxfi/ctsrc.c
@@ -668,13 +668,6 @@ static int srcimp_rsc_init(struct srcimp *srcimp,
 	if (err)
 		return err;
 
-	/* Reserve memory for imapper nodes */
-	srcimp->imappers = kzalloc_objs(struct imapper, desc->msr);
-	if (!srcimp->imappers) {
-		err = -ENOMEM;
-		goto error1;
-	}
-
 	/* Set srcimp specific operations */
 	srcimp->rsc.ops = &srcimp_basic_rsc_ops;
 	srcimp->ops = &srcimp_ops;
@@ -683,16 +676,10 @@ static int srcimp_rsc_init(struct srcimp *srcimp,
 	srcimp->rsc.ops->master(&srcimp->rsc);
 
 	return 0;
-
-error1:
-	rsc_uninit(&srcimp->rsc);
-	return err;
 }
 
 static int srcimp_rsc_uninit(struct srcimp *srcimp)
 {
-	kfree(srcimp->imappers);
-	srcimp->imappers = NULL;
 	srcimp->ops = NULL;
 	srcimp->mgr = NULL;
 	rsc_uninit(&srcimp->rsc);
@@ -711,7 +698,7 @@ static int get_srcimp_rsc(struct srcimp_mgr *mgr,
 	*rsrcimp = NULL;
 
 	/* Allocate mem for SRCIMP resource */
-	srcimp = kzalloc(sizeof(*srcimp), GFP_KERNEL);
+	srcimp = kzalloc_flex(*srcimp, imappers, desc->msr);
 	if (!srcimp)
 		return -ENOMEM;
 
diff --git a/sound/pci/ctxfi/ctsrc.h b/sound/pci/ctxfi/ctsrc.h
index e6366cc6a7ae6..7ba94adde9208 100644
--- a/sound/pci/ctxfi/ctsrc.h
+++ b/sound/pci/ctxfi/ctsrc.h
@@ -103,10 +103,10 @@ struct srcimp_rsc_ops;
 struct srcimp {
 	struct rsc rsc;
 	unsigned char idx[8];
-	struct imapper *imappers;
 	unsigned int mapped; /* A bit-map indicating which conj rsc is mapped */
 	struct srcimp_mgr *mgr;
 	const struct srcimp_rsc_ops *ops;
+	struct imapper imappers[];
 };
 
 struct srcimp_rsc_ops {
diff --git a/sound/pci/echoaudio/darla20.c b/sound/pci/echoaudio/darla20.c
index e295c71c7a390..48c25bd3f401e 100644
--- a/sound/pci/echoaudio/darla20.c
+++ b/sound/pci/echoaudio/darla20.c
@@ -52,8 +52,8 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x1801, 0xECC0, 0x0010, 0, 0, 0},	/* DSP 56301 Darla20 rev.0 */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0010) },	/* DSP 56301 Darla20 rev.0 */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/darla24.c b/sound/pci/echoaudio/darla24.c
index ae816e78f5991..c8cdd1b052df6 100644
--- a/sound/pci/echoaudio/darla24.c
+++ b/sound/pci/echoaudio/darla24.c
@@ -56,9 +56,9 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x1801, 0xECC0, 0x0040, 0, 0, 0},	/* DSP 56301 Darla24 rev.0 */
-	{0x1057, 0x1801, 0xECC0, 0x0041, 0, 0, 0},	/* DSP 56301 Darla24 rev.1 */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0040) },	/* DSP 56301 Darla24 rev.0 */
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0041) },	/* DSP 56301 Darla24 rev.1 */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/echo3g.c b/sound/pci/echoaudio/echo3g.c
index 3d37bb4030ec2..e8b476e073073 100644
--- a/sound/pci/echoaudio/echo3g.c
+++ b/sound/pci/echoaudio/echo3g.c
@@ -70,8 +70,8 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x3410, 0xECC0, 0x0100, 0, 0, 0},	/* Echo 3G */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0100) },	/* Echo 3G */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/gina20.c b/sound/pci/echoaudio/gina20.c
index 4f864ddc95309..b5f88922e19a2 100644
--- a/sound/pci/echoaudio/gina20.c
+++ b/sound/pci/echoaudio/gina20.c
@@ -56,8 +56,8 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x1801, 0xECC0, 0x0020, 0, 0, 0},	/* DSP 56301 Gina20 rev.0 */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0020) },	/* DSP 56301 Gina20 rev.0 */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/gina24.c b/sound/pci/echoaudio/gina24.c
index eff69e83ca0a1..fe0e510209ed3 100644
--- a/sound/pci/echoaudio/gina24.c
+++ b/sound/pci/echoaudio/gina24.c
@@ -74,11 +74,11 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x1801, 0xECC0, 0x0050, 0, 0, 0},	/* DSP 56301 Gina24 rev.0 */
-	{0x1057, 0x1801, 0xECC0, 0x0051, 0, 0, 0},	/* DSP 56301 Gina24 rev.1 */
-	{0x1057, 0x3410, 0xECC0, 0x0050, 0, 0, 0},	/* DSP 56361 Gina24 rev.0 */
-	{0x1057, 0x3410, 0xECC0, 0x0051, 0, 0, 0},	/* DSP 56361 Gina24 rev.1 */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0050) },	/* DSP 56301 Gina24 rev.0 */
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0051) },	/* DSP 56301 Gina24 rev.1 */
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0050) },	/* DSP 56361 Gina24 rev.0 */
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0051) },	/* DSP 56361 Gina24 rev.1 */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/indigo.c b/sound/pci/echoaudio/indigo.c
index a9f2efc58f6e0..496cc5aa9516d 100644
--- a/sound/pci/echoaudio/indigo.c
+++ b/sound/pci/echoaudio/indigo.c
@@ -57,8 +57,8 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x3410, 0xECC0, 0x0090, 0, 0, 0},	/* Indigo */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0090) },	/* Indigo */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/indigodj.c b/sound/pci/echoaudio/indigodj.c
index 14e9769ceba14..45ad92e8f5311 100644
--- a/sound/pci/echoaudio/indigodj.c
+++ b/sound/pci/echoaudio/indigodj.c
@@ -57,8 +57,8 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x3410, 0xECC0, 0x00B0, 0, 0, 0},	/* Indigo DJ*/
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x00B0) },	/* Indigo DJ*/
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/indigodjx.c b/sound/pci/echoaudio/indigodjx.c
index a14a7dc8c87db..b1878ecb83753 100644
--- a/sound/pci/echoaudio/indigodjx.c
+++ b/sound/pci/echoaudio/indigodjx.c
@@ -57,8 +57,8 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x3410, 0xECC0, 0x00E0, 0, 0, 0},	/* Indigo DJx*/
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x00E0) },	/* Indigo DJx*/
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/indigoio.c b/sound/pci/echoaudio/indigoio.c
index 97e024450d19c..2c6a6f74b0bdc 100644
--- a/sound/pci/echoaudio/indigoio.c
+++ b/sound/pci/echoaudio/indigoio.c
@@ -58,8 +58,8 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x3410, 0xECC0, 0x00A0, 0, 0, 0},	/* Indigo IO*/
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x00A0) },	/* Indigo IO*/
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/indigoiox.c b/sound/pci/echoaudio/indigoiox.c
index a017c966b4dce..dfaa563317ad1 100644
--- a/sound/pci/echoaudio/indigoiox.c
+++ b/sound/pci/echoaudio/indigoiox.c
@@ -58,8 +58,8 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x3410, 0xECC0, 0x00D0, 0, 0, 0},	/* Indigo IOx */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x00D0) },	/* Indigo IOx */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/layla20.c b/sound/pci/echoaudio/layla20.c
index 7e38bc9c025d5..82c089f7b452b 100644
--- a/sound/pci/echoaudio/layla20.c
+++ b/sound/pci/echoaudio/layla20.c
@@ -65,9 +65,9 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x1801, 0xECC0, 0x0030, 0, 0, 0},	/* DSP 56301 Layla20 rev.0 */
-	{0x1057, 0x1801, 0xECC0, 0x0031, 0, 0, 0},	/* DSP 56301 Layla20 rev.1 */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0030) },	/* DSP 56301 Layla20 rev.0 */
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0031) },	/* DSP 56301 Layla20 rev.1 */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/layla24.c b/sound/pci/echoaudio/layla24.c
index 95c52210fb654..efd676467a374 100644
--- a/sound/pci/echoaudio/layla24.c
+++ b/sound/pci/echoaudio/layla24.c
@@ -76,8 +76,8 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x3410, 0xECC0, 0x0060, 0, 0, 0},	/* DSP 56361 Layla24 rev.0 */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0060) },	/* DSP 56361 Layla24 rev.0 */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/mia.c b/sound/pci/echoaudio/mia.c
index a2d4b0003b570..950aaec74a439 100644
--- a/sound/pci/echoaudio/mia.c
+++ b/sound/pci/echoaudio/mia.c
@@ -66,9 +66,9 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x3410, 0xECC0, 0x0080, 0, 0, 0},	/* DSP 56361 Mia rev.0 */
-	{0x1057, 0x3410, 0xECC0, 0x0081, 0, 0, 0},	/* DSP 56361 Mia rev.1 */
-	{0,}
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0080) },	/* DSP 56361 Mia rev.0 */
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0081) },	/* DSP 56361 Mia rev.1 */
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/echoaudio/mona.c b/sound/pci/echoaudio/mona.c
index 1b45a2b5066fe..4f999324f388d 100644
--- a/sound/pci/echoaudio/mona.c
+++ b/sound/pci/echoaudio/mona.c
@@ -81,13 +81,19 @@ static const struct firmware card_fw[] = {
 };
 
 static const struct pci_device_id snd_echo_ids[] = {
-	{0x1057, 0x1801, 0xECC0, 0x0070, 0, 0, 0},	/* DSP 56301 Mona rev.0 */
-	{0x1057, 0x1801, 0xECC0, 0x0071, 0, 0, 0},	/* DSP 56301 Mona rev.1 */
-	{0x1057, 0x1801, 0xECC0, 0x0072, 0, 0, 0},	/* DSP 56301 Mona rev.2 */
-	{0x1057, 0x3410, 0xECC0, 0x0070, 0, 0, 0},	/* DSP 56361 Mona rev.0 */
-	{0x1057, 0x3410, 0xECC0, 0x0071, 0, 0, 0},	/* DSP 56361 Mona rev.1 */
-	{0x1057, 0x3410, 0xECC0, 0x0072, 0, 0, 0},	/* DSP 56361 Mona rev.2 */
-	{0,}
+	/* DSP 56301 Mona rev.0 */
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0070) },
+	/* DSP 56301 Mona rev.1 */
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0071) },
+	/* DSP 56301 Mona rev.2 */
+	{ PCI_DEVICE_SUB(0x1057, 0x1801, 0xECC0, 0x0072) },
+	/* DSP 56361 Mona rev.0 */
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0070) },
+	/* DSP 56361 Mona rev.1 */
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0071) },
+	/* DSP 56361 Mona rev.2 */
+	{ PCI_DEVICE_SUB(0x1057, 0x3410, 0xECC0, 0x0072) },
+	{ }
 };
 
 static const struct snd_pcm_hardware pcm_hardware_skel = {
diff --git a/sound/pci/emu10k1/emu10k1.c b/sound/pci/emu10k1/emu10k1.c
index 548e7d0499013..3b21bd2883b64 100644
--- a/sound/pci/emu10k1/emu10k1.c
+++ b/sound/pci/emu10k1/emu10k1.c
@@ -58,10 +58,10 @@ MODULE_PARM_DESC(subsystem, "Force card subsystem model.");
  * Class 0401: 1102:0008 (rev 00) Subsystem: 1102:1001 -> Audigy2 Value  Model:SB0400
  */
 static const struct pci_device_id snd_emu10k1_ids[] = {
-	{ PCI_VDEVICE(CREATIVE, 0x0002), 0 },	/* EMU10K1 */
-	{ PCI_VDEVICE(CREATIVE, 0x0004), 1 },	/* Audigy */
-	{ PCI_VDEVICE(CREATIVE, 0x0008), 1 },	/* Audigy 2 Value SB0400 */
-	{ 0, }
+	{ PCI_VDEVICE(CREATIVE, 0x0002), .driver_data = 0 },	/* EMU10K1 */
+	{ PCI_VDEVICE(CREATIVE, 0x0004), .driver_data = 1 },	/* Audigy */
+	{ PCI_VDEVICE(CREATIVE, 0x0008), .driver_data = 1 },	/* Audigy 2 Value SB0400 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_emu10k1_ids);
diff --git a/sound/pci/emu10k1/emu10k1x.c b/sound/pci/emu10k1/emu10k1x.c
index 1b207ca25814e..ed4630c8342b4 100644
--- a/sound/pci/emu10k1/emu10k1x.c
+++ b/sound/pci/emu10k1/emu10k1x.c
@@ -1518,8 +1518,8 @@ static int snd_emu10k1x_probe(struct pci_dev *pci,
 
 // PCI IDs
 static const struct pci_device_id snd_emu10k1x_ids[] = {
-	{ PCI_VDEVICE(CREATIVE, 0x0006), 0 },	/* Dell OEM version (EMU10K1) */
-	{ 0, }
+	{ PCI_VDEVICE(CREATIVE, 0x0006) },	/* Dell OEM version (EMU10K1) */
+	{ }
 };
 MODULE_DEVICE_TABLE(pci, snd_emu10k1x_ids);
 
diff --git a/sound/pci/ens1370.c b/sound/pci/ens1370.c
index 657056a59175b..0b9eb55a21218 100644
--- a/sound/pci/ens1370.c
+++ b/sound/pci/ens1370.c
@@ -426,14 +426,14 @@ static irqreturn_t snd_audiopci_interrupt(int irq, void *dev_id);
 
 static const struct pci_device_id snd_audiopci_ids[] = {
 #ifdef CHIP1370
-	{ PCI_VDEVICE(ENSONIQ, 0x5000), 0, },	/* ES1370 */
+	{ PCI_VDEVICE(ENSONIQ, 0x5000) },	/* ES1370 */
 #endif
 #ifdef CHIP1371
-	{ PCI_VDEVICE(ENSONIQ, 0x1371), 0, },	/* ES1371 */
-	{ PCI_VDEVICE(ENSONIQ, 0x5880), 0, },	/* ES1373 - CT5880 */
-	{ PCI_VDEVICE(ECTIVA, 0x8938), 0, },	/* Ectiva EV1938 */
+	{ PCI_VDEVICE(ENSONIQ, 0x1371) },	/* ES1371 */
+	{ PCI_VDEVICE(ENSONIQ, 0x5880) },	/* ES1373 - CT5880 */
+	{ PCI_VDEVICE(ECTIVA, 0x8938) },	/* Ectiva EV1938 */
 #endif
-	{ 0, }
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_audiopci_ids);
diff --git a/sound/pci/es1938.c b/sound/pci/es1938.c
index 280125eff3624..f4c7a9532f4a6 100644
--- a/sound/pci/es1938.c
+++ b/sound/pci/es1938.c
@@ -222,8 +222,8 @@ struct es1938 {
 static irqreturn_t snd_es1938_interrupt(int irq, void *dev_id);
 
 static const struct pci_device_id snd_es1938_ids[] = {
-	{ PCI_VDEVICE(ESS, 0x1969), 0, },   /* Solo-1 */
-	{ 0, }
+	{ PCI_VDEVICE(ESS, 0x1969) },   /* Solo-1 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_es1938_ids);
diff --git a/sound/pci/es1968.c b/sound/pci/es1968.c
index b7282b3fa1b16..f04628c8cbb72 100644
--- a/sound/pci/es1968.c
+++ b/sound/pci/es1968.c
@@ -549,13 +549,26 @@ struct es1968 {
 static irqreturn_t snd_es1968_interrupt(int irq, void *dev_id);
 
 static const struct pci_device_id snd_es1968_ids[] = {
-	/* Maestro 1 */
-        { 0x1285, 0x0100, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, TYPE_MAESTRO },
-	/* Maestro 2 */
-	{ 0x125d, 0x1968, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, TYPE_MAESTRO2 },
-	/* Maestro 2E */
-        { 0x125d, 0x1978, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, TYPE_MAESTRO2E },
-	{ 0, }
+	{
+		/* Maestro 1 */
+		PCI_DEVICE(0x1285, 0x0100),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+		.driver_data = TYPE_MAESTRO,
+	}, {
+		/* Maestro 2 */
+		PCI_DEVICE(0x125d, 0x1968),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+		.driver_data = TYPE_MAESTRO2,
+	}, {
+		/* Maestro 2E */
+		PCI_DEVICE(0x125d, 0x1978),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+		.driver_data = TYPE_MAESTRO2E,
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_es1968_ids);
diff --git a/sound/pci/fm801.c b/sound/pci/fm801.c
index 4ca992449ea30..9cc96b807dd79 100644
--- a/sound/pci/fm801.c
+++ b/sound/pci/fm801.c
@@ -240,9 +240,18 @@ static inline u16 fm801_ioread16(struct fm801 *chip, unsigned short offset)
 }
 
 static const struct pci_device_id snd_fm801_ids[] = {
-	{ 0x1319, 0x0801, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0, },   /* FM801 */
-	{ 0x5213, 0x0510, PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0, },   /* Gallant Odyssey Sound 4 */
-	{ 0, }
+	{
+		/* FM801 */
+		PCI_DEVICE(0x1319, 0x0801),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	}, {
+		/* Gallant Odyssey Sound 4 */
+		PCI_DEVICE(0x5213, 0x0510),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_fm801_ids);
diff --git a/sound/pci/ice1712/ice1712.c b/sound/pci/ice1712/ice1712.c
index 1e39b985bef26..0e27e84d2ce45 100644
--- a/sound/pci/ice1712/ice1712.c
+++ b/sound/pci/ice1712/ice1712.c
@@ -86,8 +86,8 @@ MODULE_PARM_DESC(dxr_enable, "Enable DXR support for Terratec DMX6FIRE.");
 
 
 static const struct pci_device_id snd_ice1712_ids[] = {
-	{ PCI_VDEVICE(ICE, PCI_DEVICE_ID_ICE_1712), 0 },   /* ICE1712 */
-	{ 0, }
+	{ PCI_VDEVICE(ICE, PCI_DEVICE_ID_ICE_1712) },   /* ICE1712 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_ice1712_ids);
diff --git a/sound/pci/ice1712/ice1724.c b/sound/pci/ice1712/ice1724.c
index 65bf48647d089..79d57938a1c8c 100644
--- a/sound/pci/ice1712/ice1724.c
+++ b/sound/pci/ice1712/ice1724.c
@@ -62,8 +62,8 @@ MODULE_PARM_DESC(model, "Use the given board model.");
 
 /* Both VT1720 and VT1724 have the same PCI IDs */
 static const struct pci_device_id snd_vt1724_ids[] = {
-	{ PCI_VDEVICE(ICE, PCI_DEVICE_ID_VT1724), 0 },
-	{ 0, }
+	{ PCI_VDEVICE(ICE, PCI_DEVICE_ID_VT1724) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_vt1724_ids);
@@ -730,13 +730,22 @@ static int snd_vt1724_pcm_hw_params(struct snd_pcm_substream *substream,
 static int snd_vt1724_pcm_hw_free(struct snd_pcm_substream *substream)
 {
 	struct snd_ice1712 *ice = snd_pcm_substream_chip(substream);
+	bool released = false;
 	int i;
 
-	guard(mutex)(&ice->open_mutex);
-	/* unmark surround channels */
-	for (i = 0; i < 3; i++)
-		if (ice->pcm_reserved[i] == substream)
+	scoped_guard(mutex, &ice->open_mutex) {
+		/* unmark surround channels */
+		for (i = 0; i < 3; i++) {
+			if (ice->pcm_reserved[i] != substream)
+				continue;
 			ice->pcm_reserved[i] = NULL;
+			released = true;
+		}
+	}
+
+	if (released && ice->pcm_ds)
+		wake_up(&ice->pcm_ds->open_wait);
+
 	return 0;
 }
 
@@ -1364,7 +1373,7 @@ static int snd_vt1724_playback_indep_open(struct snd_pcm_substream *substream)
 	scoped_guard(mutex, &ice->open_mutex) {
 		/* already used by PDMA0? */
 		if (ice->pcm_reserved[substream->number])
-			return -EBUSY; /* FIXME: should handle blocking mode properly */
+			return -EAGAIN;
 	}
 	runtime->private_data = (void *)&vt1724_playback_dma_regs[substream->number];
 	ice->playback_con_substream_ds[substream->number] = substream;
diff --git a/sound/pci/intel8x0.c b/sound/pci/intel8x0.c
index 3b53c5e63c298..e2ea9016a73e4 100644
--- a/sound/pci/intel8x0.c
+++ b/sound/pci/intel8x0.c
@@ -384,30 +384,100 @@ struct intel8x0 {
 };
 
 static const struct pci_device_id snd_intel8x0_ids[] = {
-	{ PCI_VDEVICE(INTEL, 0x2415), DEVICE_INTEL },	/* 82801AA */
-	{ PCI_VDEVICE(INTEL, 0x2425), DEVICE_INTEL },	/* 82901AB */
-	{ PCI_VDEVICE(INTEL, 0x2445), DEVICE_INTEL },	/* 82801BA */
-	{ PCI_VDEVICE(INTEL, 0x2485), DEVICE_INTEL },	/* ICH3 */
-	{ PCI_VDEVICE(INTEL, 0x24c5), DEVICE_INTEL_ICH4 }, /* ICH4 */
-	{ PCI_VDEVICE(INTEL, 0x24d5), DEVICE_INTEL_ICH4 }, /* ICH5 */
-	{ PCI_VDEVICE(INTEL, 0x25a6), DEVICE_INTEL_ICH4 }, /* ESB */
-	{ PCI_VDEVICE(INTEL, 0x266e), DEVICE_INTEL_ICH4 }, /* ICH6 */
-	{ PCI_VDEVICE(INTEL, 0x27de), DEVICE_INTEL_ICH4 }, /* ICH7 */
-	{ PCI_VDEVICE(INTEL, 0x2698), DEVICE_INTEL_ICH4 }, /* ESB2 */
-	{ PCI_VDEVICE(INTEL, 0x7195), DEVICE_INTEL },	/* 440MX */
-	{ PCI_VDEVICE(SI, 0x7012), DEVICE_SIS },	/* SI7012 */
-	{ PCI_VDEVICE(NVIDIA, 0x01b1), DEVICE_NFORCE },	/* NFORCE */
-	{ PCI_VDEVICE(NVIDIA, 0x003a), DEVICE_NFORCE },	/* MCP04 */
-	{ PCI_VDEVICE(NVIDIA, 0x006a), DEVICE_NFORCE },	/* NFORCE2 */
-	{ PCI_VDEVICE(NVIDIA, 0x0059), DEVICE_NFORCE },	/* CK804 */
-	{ PCI_VDEVICE(NVIDIA, 0x008a), DEVICE_NFORCE },	/* CK8 */
-	{ PCI_VDEVICE(NVIDIA, 0x00da), DEVICE_NFORCE },	/* NFORCE3 */
-	{ PCI_VDEVICE(NVIDIA, 0x00ea), DEVICE_NFORCE },	/* CK8S */
-	{ PCI_VDEVICE(NVIDIA, 0x026b), DEVICE_NFORCE },	/* MCP51 */
-	{ PCI_VDEVICE(AMD, 0x746d), DEVICE_INTEL },	/* AMD8111 */
-	{ PCI_VDEVICE(AMD, 0x7445), DEVICE_INTEL },	/* AMD768 */
-	{ PCI_VDEVICE(AL, 0x5455), DEVICE_ALI },   /* Ali5455 */
-	{ 0, }
+	{
+		/* 82801AA */
+		PCI_VDEVICE(INTEL, 0x2415),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* 82901AB */
+		PCI_VDEVICE(INTEL, 0x2425),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* 82801BA */
+		PCI_VDEVICE(INTEL, 0x2445),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* ICH3 */
+		PCI_VDEVICE(INTEL, 0x2485),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* ICH4 */
+		PCI_VDEVICE(INTEL, 0x24c5),
+		.driver_data = DEVICE_INTEL_ICH4,
+	}, {
+		/* ICH5 */
+		PCI_VDEVICE(INTEL, 0x24d5),
+		.driver_data = DEVICE_INTEL_ICH4,
+	}, {
+		/* ESB */
+		PCI_VDEVICE(INTEL, 0x25a6),
+		.driver_data = DEVICE_INTEL_ICH4,
+	}, {
+		/* ICH6 */
+		PCI_VDEVICE(INTEL, 0x266e),
+		.driver_data = DEVICE_INTEL_ICH4,
+	}, {
+		/* ICH7 */
+		PCI_VDEVICE(INTEL, 0x27de),
+		.driver_data = DEVICE_INTEL_ICH4,
+	}, {
+		/* ESB2 */
+		PCI_VDEVICE(INTEL, 0x2698),
+		.driver_data = DEVICE_INTEL_ICH4,
+	}, {
+		/* 440MX */
+		PCI_VDEVICE(INTEL, 0x7195),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* SI7012 */
+		PCI_VDEVICE(SI, 0x7012),
+		.driver_data = DEVICE_SIS,
+	}, {
+		/* NFORCE */
+		PCI_VDEVICE(NVIDIA, 0x01b1),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* MCP04 */
+		PCI_VDEVICE(NVIDIA, 0x003a),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* NFORCE2 */
+		PCI_VDEVICE(NVIDIA, 0x006a),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* CK804 */
+		PCI_VDEVICE(NVIDIA, 0x0059),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* CK8 */
+		PCI_VDEVICE(NVIDIA, 0x008a),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* NFORCE3 */
+		PCI_VDEVICE(NVIDIA, 0x00da),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* CK8S */
+		PCI_VDEVICE(NVIDIA, 0x00ea),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* MCP51 */
+		PCI_VDEVICE(NVIDIA, 0x026b),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* AMD8111 */
+		PCI_VDEVICE(AMD, 0x746d),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* AMD768 */
+		PCI_VDEVICE(AMD, 0x7445),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* Ali5455 */
+		PCI_VDEVICE(AL, 0x5455),
+		.driver_data = DEVICE_ALI,
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_intel8x0_ids);
diff --git a/sound/pci/intel8x0m.c b/sound/pci/intel8x0m.c
index 84e1b7ea34e2a..27dbf61254278 100644
--- a/sound/pci/intel8x0m.c
+++ b/sound/pci/intel8x0m.c
@@ -190,26 +190,78 @@ struct intel8x0m {
 };
 
 static const struct pci_device_id snd_intel8x0m_ids[] = {
-	{ PCI_VDEVICE(INTEL, 0x2416), DEVICE_INTEL },	/* 82801AA */
-	{ PCI_VDEVICE(INTEL, 0x2426), DEVICE_INTEL },	/* 82901AB */
-	{ PCI_VDEVICE(INTEL, 0x2446), DEVICE_INTEL },	/* 82801BA */
-	{ PCI_VDEVICE(INTEL, 0x2486), DEVICE_INTEL },	/* ICH3 */
-	{ PCI_VDEVICE(INTEL, 0x24c6), DEVICE_INTEL }, /* ICH4 */
-	{ PCI_VDEVICE(INTEL, 0x24d6), DEVICE_INTEL }, /* ICH5 */
-	{ PCI_VDEVICE(INTEL, 0x266d), DEVICE_INTEL },	/* ICH6 */
-	{ PCI_VDEVICE(INTEL, 0x27dd), DEVICE_INTEL },	/* ICH7 */
-	{ PCI_VDEVICE(INTEL, 0x7196), DEVICE_INTEL },	/* 440MX */
-	{ PCI_VDEVICE(AMD, 0x7446), DEVICE_INTEL },	/* AMD768 */
-	{ PCI_VDEVICE(SI, 0x7013), DEVICE_SIS },	/* SI7013 */
-	{ PCI_VDEVICE(NVIDIA, 0x01c1), DEVICE_NFORCE }, /* NFORCE */
-	{ PCI_VDEVICE(NVIDIA, 0x0069), DEVICE_NFORCE }, /* NFORCE2 */
-	{ PCI_VDEVICE(NVIDIA, 0x0089), DEVICE_NFORCE }, /* NFORCE2s */
-	{ PCI_VDEVICE(NVIDIA, 0x00d9), DEVICE_NFORCE }, /* NFORCE3 */
-	{ PCI_VDEVICE(AMD, 0x746e), DEVICE_INTEL },	/* AMD8111 */
+	{
+		/* 82801AA */
+		PCI_VDEVICE(INTEL, 0x2416),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* 82901AB */
+		PCI_VDEVICE(INTEL, 0x2426),
+		.driver_data = DEVICE_INTEL
+	}, {
+		/* 82801BA */
+		PCI_VDEVICE(INTEL, 0x2446),
+		.driver_data = DEVICE_INTEL
+	}, {
+		/* ICH3 */
+		PCI_VDEVICE(INTEL, 0x2486),
+		.driver_data = DEVICE_INTEL
+	}, {
+		/* ICH4 */
+		PCI_VDEVICE(INTEL, 0x24c6),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* ICH5 */
+		PCI_VDEVICE(INTEL, 0x24d6),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* ICH6 */
+		PCI_VDEVICE(INTEL, 0x266d),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* ICH7 */
+		PCI_VDEVICE(INTEL, 0x27dd),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* 440MX */
+		PCI_VDEVICE(INTEL, 0x7196),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* AMD768 */
+		PCI_VDEVICE(AMD, 0x7446),
+		.driver_data = DEVICE_INTEL,
+	}, {
+		/* SI7013 */
+		PCI_VDEVICE(SI, 0x7013),
+		.driver_data = DEVICE_SIS,
+	}, {
+		/* NFORCE */
+		PCI_VDEVICE(NVIDIA, 0x01c1),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* NFORCE2 */
+		PCI_VDEVICE(NVIDIA, 0x0069),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* NFORCE2s */
+		PCI_VDEVICE(NVIDIA, 0x0089),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* NFORCE3 */
+		PCI_VDEVICE(NVIDIA, 0x00d9),
+		.driver_data = DEVICE_NFORCE,
+	}, {
+		/* AMD8111 */
+		PCI_VDEVICE(AMD, 0x746e),
+		.driver_data = DEVICE_INTEL
 #if 0
-	{ PCI_VDEVICE(AL, 0x5455), DEVICE_ALI },   /* Ali5455 */
+	}, {
+		/* Ali5455 */
+		PCI_VDEVICE(AL, 0x5455),
+		.driver_data = DEVICE_ALI,
 #endif
-	{ 0, }
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_intel8x0m_ids);
diff --git a/sound/pci/maestro3.c b/sound/pci/maestro3.c
index 3353980d5cd8a..dd45ffa171e18 100644
--- a/sound/pci/maestro3.c
+++ b/sound/pci/maestro3.c
@@ -779,23 +779,40 @@ struct snd_m3 {
  * pci ids
  */
 static const struct pci_device_id snd_m3_ids[] = {
-	{PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_ALLEGRO_1, PCI_ANY_ID, PCI_ANY_ID,
-	 PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0},
-	{PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_ALLEGRO, PCI_ANY_ID, PCI_ANY_ID,
-	 PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0},
-	{PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_CANYON3D_2LE, PCI_ANY_ID, PCI_ANY_ID,
-	 PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0},
-	{PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_CANYON3D_2, PCI_ANY_ID, PCI_ANY_ID,
-	 PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0},
-	{PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_MAESTRO3, PCI_ANY_ID, PCI_ANY_ID,
-	 PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0},
-	{PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_MAESTRO3_1, PCI_ANY_ID, PCI_ANY_ID,
-	 PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0},
-	{PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_MAESTRO3_HW, PCI_ANY_ID, PCI_ANY_ID,
-	 PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0},
-	{PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_MAESTRO3_2, PCI_ANY_ID, PCI_ANY_ID,
-	 PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0},
-	{0,},
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_ALLEGRO_1),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_ALLEGRO),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_CANYON3D_2LE),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_CANYON3D_2),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_MAESTRO3),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_MAESTRO3_1),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_MAESTRO3_HW),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_ESS, PCI_DEVICE_ID_ESS_MAESTRO3_2),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	},
+	{ },
 };
 
 MODULE_DEVICE_TABLE(pci, snd_m3_ids);
diff --git a/sound/pci/mixart/mixart.c b/sound/pci/mixart/mixart.c
index a7760a23bfe90..f451554cff5f1 100644
--- a/sound/pci/mixart/mixart.c
+++ b/sound/pci/mixart/mixart.c
@@ -48,8 +48,8 @@ MODULE_PARM_DESC(enable, "Enable Digigram " CARD_NAME " soundcard.");
  */
 
 static const struct pci_device_id snd_mixart_ids[] = {
-	{ PCI_VDEVICE(MOTOROLA, 0x0003), 0, }, /* MC8240 */
-	{ 0, }
+	{ PCI_VDEVICE(MOTOROLA, 0x0003) }, /* MC8240 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_mixart_ids);
diff --git a/sound/pci/nm256/nm256.c b/sound/pci/nm256/nm256.c
index da74b923bc88f..a7da55d9c025a 100644
--- a/sound/pci/nm256/nm256.c
+++ b/sound/pci/nm256/nm256.c
@@ -245,10 +245,10 @@ struct nm256 {
  * PCI ids
  */
 static const struct pci_device_id snd_nm256_ids[] = {
-	{PCI_VDEVICE(NEOMAGIC, PCI_DEVICE_ID_NEOMAGIC_NM256AV_AUDIO), 0},
-	{PCI_VDEVICE(NEOMAGIC, PCI_DEVICE_ID_NEOMAGIC_NM256ZX_AUDIO), 0},
-	{PCI_VDEVICE(NEOMAGIC, PCI_DEVICE_ID_NEOMAGIC_NM256XL_PLUS_AUDIO), 0},
-	{0,},
+	{ PCI_VDEVICE(NEOMAGIC, PCI_DEVICE_ID_NEOMAGIC_NM256AV_AUDIO) },
+	{ PCI_VDEVICE(NEOMAGIC, PCI_DEVICE_ID_NEOMAGIC_NM256ZX_AUDIO) },
+	{ PCI_VDEVICE(NEOMAGIC, PCI_DEVICE_ID_NEOMAGIC_NM256XL_PLUS_AUDIO) },
+	{ },
 };
 
 MODULE_DEVICE_TABLE(pci, snd_nm256_ids);
diff --git a/sound/pci/pcxhr/pcxhr.c b/sound/pci/pcxhr/pcxhr.c
index e7d63972c2caf..1eea40e94c43c 100644
--- a/sound/pci/pcxhr/pcxhr.c
+++ b/sound/pci/pcxhr/pcxhr.c
@@ -89,41 +89,41 @@ enum {
 };
 
 static const struct pci_device_id pcxhr_ids[] = {
-	{ 0x10b5, 0x9656, 0x1369, 0xb001, 0, 0, PCI_ID_VX882HR, },
-	{ 0x10b5, 0x9656, 0x1369, 0xb101, 0, 0, PCI_ID_PCX882HR, },
-	{ 0x10b5, 0x9656, 0x1369, 0xb201, 0, 0, PCI_ID_VX881HR, },
-	{ 0x10b5, 0x9656, 0x1369, 0xb301, 0, 0, PCI_ID_PCX881HR, },
-	{ 0x10b5, 0x9056, 0x1369, 0xb021, 0, 0, PCI_ID_VX882E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xb121, 0, 0, PCI_ID_PCX882E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xb221, 0, 0, PCI_ID_VX881E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xb321, 0, 0, PCI_ID_PCX881E, },
-	{ 0x10b5, 0x9656, 0x1369, 0xb401, 0, 0, PCI_ID_VX1222HR, },
-	{ 0x10b5, 0x9656, 0x1369, 0xb501, 0, 0, PCI_ID_PCX1222HR, },
-	{ 0x10b5, 0x9656, 0x1369, 0xb601, 0, 0, PCI_ID_VX1221HR, },
-	{ 0x10b5, 0x9656, 0x1369, 0xb701, 0, 0, PCI_ID_PCX1221HR, },
-	{ 0x10b5, 0x9056, 0x1369, 0xb421, 0, 0, PCI_ID_VX1222E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xb521, 0, 0, PCI_ID_PCX1222E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xb621, 0, 0, PCI_ID_VX1221E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xb721, 0, 0, PCI_ID_PCX1221E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xba01, 0, 0, PCI_ID_VX222HR, },
-	{ 0x10b5, 0x9056, 0x1369, 0xba21, 0, 0, PCI_ID_VX222E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xbd01, 0, 0, PCI_ID_PCX22HR, },
-	{ 0x10b5, 0x9056, 0x1369, 0xbd21, 0, 0, PCI_ID_PCX22E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xbc01, 0, 0, PCI_ID_VX222HRMIC, },
-	{ 0x10b5, 0x9056, 0x1369, 0xbc21, 0, 0, PCI_ID_VX222E_MIC, },
-	{ 0x10b5, 0x9056, 0x1369, 0xbb01, 0, 0, PCI_ID_PCX924HR, },
-	{ 0x10b5, 0x9056, 0x1369, 0xbb21, 0, 0, PCI_ID_PCX924E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xbf01, 0, 0, PCI_ID_PCX924HRMIC, },
-	{ 0x10b5, 0x9056, 0x1369, 0xbf21, 0, 0, PCI_ID_PCX924E_MIC, },
-	{ 0x10b5, 0x9656, 0x1369, 0xd001, 0, 0, PCI_ID_VX442HR, },
-	{ 0x10b5, 0x9656, 0x1369, 0xd101, 0, 0, PCI_ID_PCX442HR, },
-	{ 0x10b5, 0x9056, 0x1369, 0xd021, 0, 0, PCI_ID_VX442E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xd121, 0, 0, PCI_ID_PCX442E, },
-	{ 0x10b5, 0x9656, 0x1369, 0xd201, 0, 0, PCI_ID_VX822HR, },
-	{ 0x10b5, 0x9656, 0x1369, 0xd301, 0, 0, PCI_ID_PCX822HR, },
-	{ 0x10b5, 0x9056, 0x1369, 0xd221, 0, 0, PCI_ID_VX822E, },
-	{ 0x10b5, 0x9056, 0x1369, 0xd321, 0, 0, PCI_ID_PCX822E, },
-	{ 0, }
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xb001), .driver_data = PCI_ID_VX882HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xb101), .driver_data = PCI_ID_PCX882HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xb201), .driver_data = PCI_ID_VX881HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xb301), .driver_data = PCI_ID_PCX881HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xb021), .driver_data = PCI_ID_VX882E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xb121), .driver_data = PCI_ID_PCX882E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xb221), .driver_data = PCI_ID_VX881E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xb321), .driver_data = PCI_ID_PCX881E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xb401), .driver_data = PCI_ID_VX1222HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xb501), .driver_data = PCI_ID_PCX1222HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xb601), .driver_data = PCI_ID_VX1221HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xb701), .driver_data = PCI_ID_PCX1221HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xb421), .driver_data = PCI_ID_VX1222E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xb521), .driver_data = PCI_ID_PCX1222E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xb621), .driver_data = PCI_ID_VX1221E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xb721), .driver_data = PCI_ID_PCX1221E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xba01), .driver_data = PCI_ID_VX222HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xba21), .driver_data = PCI_ID_VX222E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xbd01), .driver_data = PCI_ID_PCX22HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xbd21), .driver_data = PCI_ID_PCX22E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xbc01), .driver_data = PCI_ID_VX222HRMIC },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xbc21), .driver_data = PCI_ID_VX222E_MIC },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xbb01), .driver_data = PCI_ID_PCX924HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xbb21), .driver_data = PCI_ID_PCX924E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xbf01), .driver_data = PCI_ID_PCX924HRMIC },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xbf21), .driver_data = PCI_ID_PCX924E_MIC },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xd001), .driver_data = PCI_ID_VX442HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xd101), .driver_data = PCI_ID_PCX442HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xd021), .driver_data = PCI_ID_VX442E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xd121), .driver_data = PCI_ID_PCX442E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xd201), .driver_data = PCI_ID_VX822HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9656, 0x1369, 0xd301), .driver_data = PCI_ID_PCX822HR },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xd221), .driver_data = PCI_ID_VX822E },
+	{ PCI_DEVICE_SUB(0x10b5, 0x9056, 0x1369, 0xd321), .driver_data = PCI_ID_PCX822E },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, pcxhr_ids);
diff --git a/sound/pci/rme32.c b/sound/pci/rme32.c
index ca9bbf5546509..454a30a2c07e1 100644
--- a/sound/pci/rme32.c
+++ b/sound/pci/rme32.c
@@ -211,10 +211,10 @@ struct rme32 {
 };
 
 static const struct pci_device_id snd_rme32_ids[] = {
-	{PCI_VDEVICE(XILINX_RME, PCI_DEVICE_ID_RME_DIGI32), 0,},
-	{PCI_VDEVICE(XILINX_RME, PCI_DEVICE_ID_RME_DIGI32_8), 0,},
-	{PCI_VDEVICE(XILINX_RME, PCI_DEVICE_ID_RME_DIGI32_PRO), 0,},
-	{0,}
+	{ PCI_VDEVICE(XILINX_RME, PCI_DEVICE_ID_RME_DIGI32) },
+	{ PCI_VDEVICE(XILINX_RME, PCI_DEVICE_ID_RME_DIGI32_8) },
+	{ PCI_VDEVICE(XILINX_RME, PCI_DEVICE_ID_RME_DIGI32_PRO) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_rme32_ids);
diff --git a/sound/pci/rme96.c b/sound/pci/rme96.c
index 58b8ebf1a24e7..892fcc5985575 100644
--- a/sound/pci/rme96.c
+++ b/sound/pci/rme96.c
@@ -242,11 +242,11 @@ struct rme96 {
 };
 
 static const struct pci_device_id snd_rme96_ids[] = {
-	{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_RME_DIGI96), 0, },
-	{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_RME_DIGI96_8), 0, },
-	{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_RME_DIGI96_8_PRO), 0, },
-	{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_RME_DIGI96_8_PAD_OR_PST), 0, },
-	{ 0, }
+	{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_RME_DIGI96) },
+	{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_RME_DIGI96_8) },
+	{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_RME_DIGI96_8_PRO) },
+	{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_RME_DIGI96_8_PAD_OR_PST) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_rme96_ids);
diff --git a/sound/pci/sonicvibes.c b/sound/pci/sonicvibes.c
index a4c72799d0348..a885d544acd65 100644
--- a/sound/pci/sonicvibes.c
+++ b/sound/pci/sonicvibes.c
@@ -227,8 +227,8 @@ struct sonicvibes {
 };
 
 static const struct pci_device_id snd_sonic_ids[] = {
-	{ PCI_VDEVICE(S3, 0xca00), 0, },
-        { 0, }
+	{ PCI_VDEVICE(S3, 0xca00) },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_sonic_ids);
diff --git a/sound/pci/trident/trident.c b/sound/pci/trident/trident.c
index ddb6ccc72e443..8dcbd022ca1fc 100644
--- a/sound/pci/trident/trident.c
+++ b/sound/pci/trident/trident.c
@@ -36,12 +36,16 @@ module_param_array(wavetable_size, int, NULL, 0444);
 MODULE_PARM_DESC(wavetable_size, "Maximum memory size in kB for wavetable synth.");
 
 static const struct pci_device_id snd_trident_ids[] = {
-	{PCI_DEVICE(PCI_VENDOR_ID_TRIDENT, PCI_DEVICE_ID_TRIDENT_4DWAVE_DX), 
-		PCI_CLASS_MULTIMEDIA_AUDIO << 8, 0xffff00, 0},
-	{PCI_DEVICE(PCI_VENDOR_ID_TRIDENT, PCI_DEVICE_ID_TRIDENT_4DWAVE_NX), 
-		0, 0, 0},
-	{PCI_DEVICE(PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_7018), 0, 0, 0},
-	{ 0, }
+	{
+		PCI_DEVICE(PCI_VENDOR_ID_TRIDENT, PCI_DEVICE_ID_TRIDENT_4DWAVE_DX),
+		.class = PCI_CLASS_MULTIMEDIA_AUDIO << 8,
+		.class_mask = 0xffff00,
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_TRIDENT, PCI_DEVICE_ID_TRIDENT_4DWAVE_NX),
+	}, {
+		PCI_DEVICE(PCI_VENDOR_ID_SI, PCI_DEVICE_ID_SI_7018)
+	},
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_trident_ids);
diff --git a/sound/pci/via82xx.c b/sound/pci/via82xx.c
index 41b322fbd9efd..24ee1302f27d3 100644
--- a/sound/pci/via82xx.c
+++ b/sound/pci/via82xx.c
@@ -389,10 +389,10 @@ struct via82xx {
 
 static const struct pci_device_id snd_via82xx_ids[] = {
 	/* 0x1106, 0x3058 */
-	{ PCI_VDEVICE(VIA, PCI_DEVICE_ID_VIA_82C686_5), TYPE_CARD_VIA686, },	/* 686A */
+	{ PCI_VDEVICE(VIA, PCI_DEVICE_ID_VIA_82C686_5), .driver_data = TYPE_CARD_VIA686 },	/* 686A */
 	/* 0x1106, 0x3059 */
-	{ PCI_VDEVICE(VIA, PCI_DEVICE_ID_VIA_8233_5), TYPE_CARD_VIA8233, },	/* VT8233 */
-	{ 0, }
+	{ PCI_VDEVICE(VIA, PCI_DEVICE_ID_VIA_8233_5), .driver_data = TYPE_CARD_VIA8233 },	/* VT8233 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_via82xx_ids);
diff --git a/sound/pci/via82xx_modem.c b/sound/pci/via82xx_modem.c
index a6f176d612e3a..9b84d3fb9eaf5 100644
--- a/sound/pci/via82xx_modem.c
+++ b/sound/pci/via82xx_modem.c
@@ -246,8 +246,8 @@ struct via82xx_modem {
 };
 
 static const struct pci_device_id snd_via82xx_modem_ids[] = {
-	{ PCI_VDEVICE(VIA, 0x3068), TYPE_CARD_VIA82XX_MODEM, },
-	{ 0, }
+	{ PCI_VDEVICE(VIA, 0x3068), .driver_data = TYPE_CARD_VIA82XX_MODEM },
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_via82xx_modem_ids);
diff --git a/sound/pci/vx222/vx222.c b/sound/pci/vx222/vx222.c
index 693a4e471cf7f..0b89ca859e667 100644
--- a/sound/pci/vx222/vx222.c
+++ b/sound/pci/vx222/vx222.c
@@ -47,9 +47,9 @@ enum {
 };
 
 static const struct pci_device_id snd_vx222_ids[] = {
-	{ 0x10b5, 0x9050, 0x1369, PCI_ANY_ID, 0, 0, VX_PCI_VX222_OLD, },   /* PLX */
-	{ 0x10b5, 0x9030, 0x1369, PCI_ANY_ID, 0, 0, VX_PCI_VX222_NEW, },   /* PLX */
-	{ 0, }
+	{ PCI_DEVICE_SUB(0x10b5, 0x9050, 0x1369, PCI_ANY_ID), .driver_data = VX_PCI_VX222_OLD },   /* PLX */
+	{ PCI_DEVICE_SUB(0x10b5, 0x9030, 0x1369, PCI_ANY_ID), .driver_data = VX_PCI_VX222_NEW },   /* PLX */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_vx222_ids);
diff --git a/sound/pci/ymfpci/ymfpci.c b/sound/pci/ymfpci/ymfpci.c
index 764ca59e98d1d..d3fb047c9a450 100644
--- a/sound/pci/ymfpci/ymfpci.c
+++ b/sound/pci/ymfpci/ymfpci.c
@@ -46,13 +46,13 @@ module_param_array(rear_switch, bool, NULL, 0444);
 MODULE_PARM_DESC(rear_switch, "Enable shared rear/line-in switch");
 
 static const struct pci_device_id snd_ymfpci_ids[] = {
-	{ PCI_VDEVICE(YAMAHA, 0x0004), 0, },   /* YMF724 */
-	{ PCI_VDEVICE(YAMAHA, 0x000d), 0, },   /* YMF724F */
-	{ PCI_VDEVICE(YAMAHA, 0x000a), 0, },   /* YMF740 */
-	{ PCI_VDEVICE(YAMAHA, 0x000c), 0, },   /* YMF740C */
-	{ PCI_VDEVICE(YAMAHA, 0x0010), 0, },   /* YMF744 */
-	{ PCI_VDEVICE(YAMAHA, 0x0012), 0, },   /* YMF754 */
-	{ 0, }
+	{ PCI_VDEVICE(YAMAHA, 0x0004) },   /* YMF724 */
+	{ PCI_VDEVICE(YAMAHA, 0x000d) },   /* YMF724F */
+	{ PCI_VDEVICE(YAMAHA, 0x000a) },   /* YMF740 */
+	{ PCI_VDEVICE(YAMAHA, 0x000c) },   /* YMF740C */
+	{ PCI_VDEVICE(YAMAHA, 0x0010) },   /* YMF744 */
+	{ PCI_VDEVICE(YAMAHA, 0x0012) },   /* YMF754 */
+	{ }
 };
 
 MODULE_DEVICE_TABLE(pci, snd_ymfpci_ids);
diff --git a/sound/soc/amd/acp-config.c b/sound/soc/amd/acp-config.c
index 1604ed679224b..309dc9ed6e0d2 100644
--- a/sound/soc/amd/acp-config.c
+++ b/sound/soc/amd/acp-config.c
@@ -30,6 +30,13 @@ static const struct dmi_system_id acp70_acpi_flag_override_table[] = {
 			DMI_MATCH(DMI_PRODUCT_NAME, "HN7306EA"),
 		},
 	},
+	{
+		/* ASUS Zenbook S16 UM5606GA (Strix Point, ACP 7.0) */
+		.matches = {
+			DMI_MATCH(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "Zenbook S16 UM5606GA"),
+		},
+	},
 	{}
 };
 
diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c
index 0f21e5f64531a..09b475c83c496 100644
--- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c
+++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c
@@ -260,9 +260,9 @@ static int create_sdw_dailink(struct snd_soc_card *card,
 			cpus->dai_name = devm_kasprintf(dev, GFP_KERNEL,
 							"SDW%d Pin%d",
 							link_num, cpu_pin_id);
-			dev_dbg(dev, "cpu->dai_name:%s\n", cpus->dai_name);
 			if (!cpus->dai_name)
 				return -ENOMEM;
+			dev_dbg(dev, "cpu->dai_name:%s\n", cpus->dai_name);
 
 			codec_maps[j].cpu = 0;
 			codec_maps[j].codec = j;
diff --git a/sound/soc/apple/mca.c b/sound/soc/apple/mca.c
index 39269cc7def62..492165c0e1ea9 100644
--- a/sound/soc/apple/mca.c
+++ b/sound/soc/apple/mca.c
@@ -307,7 +307,7 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl)
 	struct mca_cluster *be_cl;
 	int stream, i;
 
-	mutex_lock(&mca->port_mutex);
+	guard(mutex)(&mca->port_mutex);
 	for (i = 0; i < mca->nclusters; i++) {
 		be_cl = &mca->clusters[i];
 
@@ -316,12 +316,10 @@ static bool mca_fe_clocks_in_use(struct mca_cluster *cl)
 
 		for_each_pcm_streams(stream) {
 			if (be_cl->clocks_in_use[stream]) {
-				mutex_unlock(&mca->port_mutex);
 				return true;
 			}
 		}
 	}
-	mutex_unlock(&mca->port_mutex);
 	return false;
 }
 
@@ -765,9 +763,8 @@ static int mca_be_startup(struct snd_pcm_substream *substream,
 		       cl->base + REG_PORT_CLOCK_SEL);
 	writel_relaxed(PORT_DATA_SEL_TXA(fe_cl->no),
 		       cl->base + REG_PORT_DATA_SEL);
-	mutex_lock(&mca->port_mutex);
-	cl->port_driver = fe_cl->no;
-	mutex_unlock(&mca->port_mutex);
+	scoped_guard(mutex, &mca->port_mutex)
+		cl->port_driver = fe_cl->no;
 	cl->port_started[substream->stream] = true;
 
 	return 0;
@@ -788,9 +785,8 @@ static void mca_be_shutdown(struct snd_pcm_substream *substream,
 		 */
 		writel_relaxed(0, cl->base + REG_PORT_ENABLES);
 		writel_relaxed(0, cl->base + REG_PORT_DATA_SEL);
-		mutex_lock(&mca->port_mutex);
-		cl->port_driver = -1;
-		mutex_unlock(&mca->port_mutex);
+		scoped_guard(mutex, &mca->port_mutex)
+			cl->port_driver = -1;
 	}
 }
 
diff --git a/sound/soc/atmel/atmel-classd.c b/sound/soc/atmel/atmel-classd.c
index 1f8c60d2de823..6693bdcbb8f58 100644
--- a/sound/soc/atmel/atmel-classd.c
+++ b/sound/soc/atmel/atmel-classd.c
@@ -11,6 +11,7 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/regmap.h>
+#include <linux/bitfield.h>
 #include <linux/string_choices.h>
 #include <sound/core.h>
 #include <sound/dmaengine_pcm.h>
@@ -236,34 +237,34 @@ static int atmel_classd_component_probe(struct snd_soc_component *component)
 	u32 mask, val;
 
 	mask = CLASSD_MR_PWMTYP_MASK;
-	val = pdata->pwm_type << CLASSD_MR_PWMTYP_SHIFT;
+	val = FIELD_PREP(CLASSD_MR_PWMTYP_MASK, pdata->pwm_type);
 
 	mask |= CLASSD_MR_NON_OVERLAP_MASK;
 	if (pdata->non_overlap_enable) {
-		val |= (CLASSD_MR_NON_OVERLAP_EN
-			<< CLASSD_MR_NON_OVERLAP_SHIFT);
+		val |= FIELD_PREP(CLASSD_MR_NON_OVERLAP_MASK,
+				CLASSD_MR_NON_OVERLAP_EN);
 
 		mask |= CLASSD_MR_NOVR_VAL_MASK;
 		switch (pdata->non_overlap_time) {
 		case 5:
-			val |= (CLASSD_MR_NOVR_VAL_5NS
-				<< CLASSD_MR_NOVR_VAL_SHIFT);
+			val |= FIELD_PREP(CLASSD_MR_NOVR_VAL_MASK,
+					CLASSD_MR_NOVR_VAL_5NS);
 			break;
 		case 10:
-			val |= (CLASSD_MR_NOVR_VAL_10NS
-				<< CLASSD_MR_NOVR_VAL_SHIFT);
+			val |= FIELD_PREP(CLASSD_MR_NOVR_VAL_MASK,
+					CLASSD_MR_NOVR_VAL_10NS);
 			break;
 		case 15:
-			val |= (CLASSD_MR_NOVR_VAL_15NS
-				<< CLASSD_MR_NOVR_VAL_SHIFT);
+			val |= FIELD_PREP(CLASSD_MR_NOVR_VAL_MASK,
+					CLASSD_MR_NOVR_VAL_15NS);
 			break;
 		case 20:
-			val |= (CLASSD_MR_NOVR_VAL_20NS
-				<< CLASSD_MR_NOVR_VAL_SHIFT);
+			val |= FIELD_PREP(CLASSD_MR_NOVR_VAL_MASK,
+					CLASSD_MR_NOVR_VAL_20NS);
 			break;
 		default:
-			val |= (CLASSD_MR_NOVR_VAL_10NS
-				<< CLASSD_MR_NOVR_VAL_SHIFT);
+			val |= FIELD_PREP(CLASSD_MR_NOVR_VAL_MASK,
+					CLASSD_MR_NOVR_VAL_10NS);
 			dev_warn(component->dev,
 				"non-overlapping value %d is invalid, the default value 10 is specified\n",
 				pdata->non_overlap_time);
@@ -370,8 +371,10 @@ atmel_classd_cpu_dai_hw_params(struct snd_pcm_substream *substream,
 		return ret;
 
 	mask = CLASSD_INTPMR_DSP_CLK_FREQ_MASK | CLASSD_INTPMR_FRAME_MASK;
-	val = (sample_rates[best].dsp_clk << CLASSD_INTPMR_DSP_CLK_FREQ_SHIFT)
-	| (sample_rates[best].sample_rate << CLASSD_INTPMR_FRAME_SHIFT);
+	val = FIELD_PREP(CLASSD_INTPMR_DSP_CLK_FREQ_MASK,
+			sample_rates[best].dsp_clk) |
+		FIELD_PREP(CLASSD_INTPMR_FRAME_MASK,
+			sample_rates[best].sample_rate);
 
 	snd_soc_component_update_bits(component, CLASSD_INTPMR, mask, val);
 
@@ -395,8 +398,8 @@ static int atmel_classd_cpu_dai_prepare(struct snd_pcm_substream *substream,
 
 	snd_soc_component_update_bits(component, CLASSD_MR,
 				CLASSD_MR_LEN_MASK | CLASSD_MR_REN_MASK,
-				(CLASSD_MR_LEN_DIS << CLASSD_MR_LEN_SHIFT)
-				|(CLASSD_MR_REN_DIS << CLASSD_MR_REN_SHIFT));
+				FIELD_PREP(CLASSD_MR_LEN_MASK, CLASSD_MR_LEN_DIS) |
+				FIELD_PREP(CLASSD_MR_REN_MASK, CLASSD_MR_REN_DIS));
 
 	return 0;
 }
@@ -418,8 +421,8 @@ static int atmel_classd_cpu_dai_trigger(struct snd_pcm_substream *substream,
 	case SNDRV_PCM_TRIGGER_STOP:
 	case SNDRV_PCM_TRIGGER_SUSPEND:
 	case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
-		val = (CLASSD_MR_LEN_DIS << CLASSD_MR_LEN_SHIFT)
-			| (CLASSD_MR_REN_DIS << CLASSD_MR_REN_SHIFT);
+		val = FIELD_PREP(CLASSD_MR_LEN_MASK, CLASSD_MR_LEN_DIS) |
+			FIELD_PREP(CLASSD_MR_REN_MASK, CLASSD_MR_REN_DIS);
 		break;
 	default:
 		return -EINVAL;
diff --git a/sound/soc/atmel/mchp-spdifrx.c b/sound/soc/atmel/mchp-spdifrx.c
index 521bee4998f8d..2c47aabdca95d 100644
--- a/sound/soc/atmel/mchp-spdifrx.c
+++ b/sound/soc/atmel/mchp-spdifrx.c
@@ -6,6 +6,7 @@
 //
 // Author: Codrin Ciubotariu <codrin.ciubotariu@microchip.com>
 
+#include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/io.h>
 #include <linux/module.h>
@@ -41,6 +42,13 @@
 
 #define SPDIFRX_VERSION			0xFC	/* Version Register */
 
+
+/* 32-bit word byte masks */
+#define SPDIFRX_BYTE_0_MASK         GENMASK(7, 0)
+#define SPDIFRX_BYTE_1_MASK         GENMASK(15, 8)
+#define SPDIFRX_BYTE_2_MASK         GENMASK(23, 16)
+#define SPDIFRX_BYTE_3_MASK         GENMASK(31, 24)
+
 /*
  * ---- Control Register (Write-only) ----
  */
@@ -55,7 +63,7 @@
 #define SPDIFRX_MR_RXEN_ENABLE		(1 << 0)	/* SPDIF Receiver Enabled */
 
 /* Validity Bit Mode */
-#define SPDIFRX_MR_VBMODE_MASK		GENAMSK(1, 1)
+#define SPDIFRX_MR_VBMODE_MASK		GENMASK(1, 1)
 #define SPDIFRX_MR_VBMODE_ALWAYS_LOAD \
 	(0 << 1)	/* Load sample regardless of validity bit value */
 #define SPDIFRX_MR_VBMODE_DISCARD_IF_VB1 \
@@ -74,7 +82,7 @@
 /* Sample Data Width */
 #define SPDIFRX_MR_DATAWIDTH_MASK	GENMASK(5, 4)
 #define SPDIFRX_MR_DATAWIDTH(width) \
-	(((6 - (width) / 4) << 4) & SPDIFRX_MR_DATAWIDTH_MASK)
+	FIELD_PREP(SPDIFRX_MR_DATAWIDTH_MASK, 6 - ((width) / 4))
 
 /* Packed Data Mode in Receive Holding Register */
 #define SPDIFRX_MR_PACK_MASK		GENMASK(7, 7)
@@ -118,15 +126,14 @@
 #define SPDIFRX_RSR_LOWF			BIT(2)
 #define SPDIFRX_RSR_NOSIGNAL			BIT(3)
 #define SPDIFRX_RSR_IFS_MASK			GENMASK(27, 16)
-#define SPDIFRX_RSR_IFS(reg)			\
-	(((reg) & SPDIFRX_RSR_IFS_MASK) >> 16)
+#define SPDIFRX_RSR_IFS(reg)			FIELD_GET(SPDIFRX_RSR_IFS_MASK, reg)
 
 /*
  *  ---- Version Register (Read-only) ----
  */
 #define SPDIFRX_VERSION_MASK		GENMASK(11, 0)
 #define SPDIFRX_VERSION_MFN_MASK	GENMASK(18, 16)
-#define SPDIFRX_VERSION_MFN(reg)	(((reg) & SPDIFRX_VERSION_MFN_MASK) >> 16)
+#define SPDIFRX_VERSION_MFN(reg)	FIELD_GET(SPDIFRX_VERSION_MFN_MASK, reg)
 
 static bool mchp_spdifrx_readable_reg(struct device *dev, unsigned int reg)
 {
@@ -317,10 +324,10 @@ static void mchp_spdifrx_channel_status_read(struct mchp_spdifrx_dev *dev,
 
 	for (i = 0; i < ARRAY_SIZE(ctrl->ch_stat[channel].data) / 4; i++) {
 		regmap_read(dev->regmap, SPDIFRX_CHSR(channel, i), &val);
-		*ch_stat++ = val & 0xFF;
-		*ch_stat++ = (val >> 8) & 0xFF;
-		*ch_stat++ = (val >> 16) & 0xFF;
-		*ch_stat++ = (val >> 24) & 0xFF;
+		*ch_stat++ = FIELD_GET(SPDIFRX_BYTE_0_MASK, val);
+		*ch_stat++ = FIELD_GET(SPDIFRX_BYTE_1_MASK, val);
+		*ch_stat++ = FIELD_GET(SPDIFRX_BYTE_2_MASK, val);
+		*ch_stat++ = FIELD_GET(SPDIFRX_BYTE_3_MASK, val);
 	}
 }
 
@@ -334,10 +341,10 @@ static void mchp_spdifrx_channel_user_data_read(struct mchp_spdifrx_dev *dev,
 
 	for (i = 0; i < ARRAY_SIZE(ctrl->user_data[channel].data) / 4; i++) {
 		regmap_read(dev->regmap, SPDIFRX_CHUD(channel, i), &val);
-		*user_data++ = val & 0xFF;
-		*user_data++ = (val >> 8) & 0xFF;
-		*user_data++ = (val >> 16) & 0xFF;
-		*user_data++ = (val >> 24) & 0xFF;
+		*user_data++ = FIELD_GET(SPDIFRX_BYTE_0_MASK, val);
+		*user_data++ = FIELD_GET(SPDIFRX_BYTE_1_MASK, val);
+		*user_data++ = FIELD_GET(SPDIFRX_BYTE_2_MASK, val);
+		*user_data++ = FIELD_GET(SPDIFRX_BYTE_3_MASK, val);
 	}
 }
 
diff --git a/sound/soc/codecs/Kconfig b/sound/soc/codecs/Kconfig
index cf94a1c756e09..a7c61f7c7f4c5 100644
--- a/sound/soc/codecs/Kconfig
+++ b/sound/soc/codecs/Kconfig
@@ -122,6 +122,7 @@ config SND_SOC_ALL_CODECS
 	imply SND_SOC_ES8328_I2C
 	imply SND_SOC_ES8375
 	imply SND_SOC_ES8389
+	imply SND_SOC_ES9356
 	imply SND_SOC_ES7134
 	imply SND_SOC_ES7241
 	imply SND_SOC_FRAMER
@@ -264,6 +265,7 @@ config SND_SOC_ALL_CODECS
 	imply SND_SOC_STA529
 	imply SND_SOC_STAC9766
 	imply SND_SOC_STI_SAS
+	imply SND_SOC_TAC5XX2_SDW
 	imply SND_SOC_TAS2552
 	imply SND_SOC_TAS2562
 	imply SND_SOC_TAS2764
@@ -275,6 +277,7 @@ config SND_SOC_ALL_CODECS
 	imply SND_SOC_TAS571X
 	imply SND_SOC_TAS5720
 	imply SND_SOC_TAS6424
+	imply SND_SOC_TAS675X
 	imply SND_SOC_TDA7419
 	imply SND_SOC_TFA9879
 	imply SND_SOC_TFA989X
@@ -884,7 +887,7 @@ config SND_SOC_CS35L56_SPI
 config SND_SOC_CS35L56_SDW
 	tristate "Cirrus Logic CS35L56 CODEC (SDW)"
 	depends on SOUNDWIRE
-	select REGMAP
+	select REGMAP_SOUNDWIRE
 	select SND_SOC_CS35L56
 	select SND_SOC_CS35L56_SHARED
 	help
@@ -898,7 +901,6 @@ menu "CS35L56 driver options"
 
 config SND_SOC_CS35L56_CAL_DEBUGFS
 	bool "CS35L56 create debugfs for factory calibration"
-	default N
 	depends on DEBUG_FS
 	select SND_SOC_CS35L56_CAL_DEBUGFS_COMMON
 	help
@@ -909,7 +911,6 @@ config SND_SOC_CS35L56_CAL_DEBUGFS
 
 config SND_SOC_CS35L56_CAL_SET_CTRL
 	bool "CS35L56 ALSA control to restore factory calibration"
-	default N
 	select SND_SOC_CS35L56_CAL_DEBUGFS_COMMON
 	help
 	  Allow restoring factory calibration data through an ALSA
@@ -923,7 +924,6 @@ config SND_SOC_CS35L56_CAL_SET_CTRL
 
 config SND_SOC_CS35L56_CAL_PERFORM_CTRL
 	bool "CS35L56 ALSA control to perform factory calibration"
-	default N
 	select SND_SOC_CS35L56_CAL_DEBUGFS_COMMON
 	help
 	  Allow performing factory calibration data through an ALSA
@@ -1300,6 +1300,12 @@ config SND_SOC_ES8389
 	tristate "Everest Semi ES8389 CODEC"
 	depends on I2C
 
+config SND_SOC_ES9356
+        tristate "Everest Semi ES9356 CODEC SDW"
+        depends on SOUNDWIRE
+        select REGMAP_SOUNDWIRE
+        select REGMAP_SOUNDWIRE_MBQ
+
 config SND_SOC_FRAMER
 	tristate "Framer codec"
 	depends on GENERIC_FRAMER
@@ -2139,6 +2145,16 @@ config SND_SOC_STAC9766
 config SND_SOC_STI_SAS
 	tristate "codec Audio support for STI SAS codec"
 
+config SND_SOC_TAC5XX2_SDW
+	tristate "Texas Instruments TAC5XX2 SoundWire Smart Amplifier"
+	depends on SOUNDWIRE
+	depends on SND_SOC_SDCA
+	help
+	  This option enables support for Texas Instruments TAC5XX2 family
+	  of SoundWire Smart Amplifiers. This includes TAC5572, TAC5672,
+	  TAC5682 and TAS2883. To compile this driver as a module, choose
+	  M here: the module will be called snd-soc-tac5xx2.
+
 config SND_SOC_TAS2552
 	tristate "Texas Instruments TAS2552 Mono Audio amplifier"
 	depends on I2C
@@ -2237,6 +2253,17 @@ config SND_SOC_TAS6424
 	  Enable support for Texas Instruments TAS6424 high-efficiency
 	  digital input quad-channel Class-D audio power amplifiers.
 
+config SND_SOC_TAS675X
+	tristate "Texas Instruments TAS675x Quad-Channel Audio Amplifier"
+	depends on I2C
+	select REGMAP_I2C
+	help
+	  Enable support for Texas Instruments TAS675x quad-channel Class-D
+	  audio power amplifiers (TAS6754, TAS67524). The devices support I2S
+	  and TDM interfaces with real-time voltage and current sense feedback
+	  via SDOUT, and provide DC/AC/real-time load diagnostics for speaker
+	  monitoring.
+
 config SND_SOC_TDA7419
 	tristate "ST TDA7419 audio processor"
 	depends on I2C
diff --git a/sound/soc/codecs/Makefile b/sound/soc/codecs/Makefile
index 172861d17cfd0..73315d017c57b 100644
--- a/sound/soc/codecs/Makefile
+++ b/sound/soc/codecs/Makefile
@@ -138,6 +138,7 @@ snd-soc-es8328-i2c-y := es8328-i2c.o
 snd-soc-es8328-spi-y := es8328-spi.o
 snd-soc-es8375-y := es8375.o
 snd-soc-es8389-y := es8389.o
+snd-soc-es9356-y := es9356.o
 snd-soc-framer-y := framer-codec.o
 snd-soc-fs-amp-lib-y := fs-amp-lib.o
 snd-soc-fs210x-y := fs210x.o
@@ -313,11 +314,13 @@ snd-soc-sta350-y := sta350.o
 snd-soc-sta529-y := sta529.o
 snd-soc-stac9766-y := stac9766.o
 snd-soc-sti-sas-y := sti-sas.o
+snd-soc-tac5xx2-sdw-y := tac5xx2-sdw.o
 snd-soc-tas5086-y := tas5086.o
 snd-soc-tas571x-y := tas571x.o
 snd-soc-tas5720-y := tas5720.o
 snd-soc-tas5805m-y := tas5805m.o
 snd-soc-tas6424-y := tas6424.o
+snd-soc-tas675x-y := tas675x.o
 snd-soc-tda7419-y := tda7419.o
 snd-soc-tas2770-y := tas2770.o
 snd-soc-tas2781-comlib-y := tas2781-comlib.o
@@ -575,6 +578,7 @@ obj-$(CONFIG_SND_SOC_ES8328_I2C)+= snd-soc-es8328-i2c.o
 obj-$(CONFIG_SND_SOC_ES8328_SPI)+= snd-soc-es8328-spi.o
 obj-$(CONFIG_SND_SOC_ES8375)    += snd-soc-es8375.o
 obj-$(CONFIG_SND_SOC_ES8389)    += snd-soc-es8389.o
+obj-$(CONFIG_SND_SOC_ES9356)    += snd-soc-es9356.o
 obj-$(CONFIG_SND_SOC_FRAMER)	+= snd-soc-framer.o
 obj-$(CONFIG_SND_SOC_FS_AMP_LIB)+= snd-soc-fs-amp-lib.o
 obj-$(CONFIG_SND_SOC_FS210X)	+= snd-soc-fs210x.o
@@ -746,6 +750,7 @@ obj-$(CONFIG_SND_SOC_STA350)   += snd-soc-sta350.o
 obj-$(CONFIG_SND_SOC_STA529)   += snd-soc-sta529.o
 obj-$(CONFIG_SND_SOC_STAC9766)	+= snd-soc-stac9766.o
 obj-$(CONFIG_SND_SOC_STI_SAS)	+= snd-soc-sti-sas.o
+obj-$(CONFIG_SND_SOC_TAC5XX2_SDW)	+= snd-soc-tac5xx2-sdw.o
 obj-$(CONFIG_SND_SOC_TAS2552)	+= snd-soc-tas2552.o
 obj-$(CONFIG_SND_SOC_TAS2562)	+= snd-soc-tas2562.o
 obj-$(CONFIG_SND_SOC_TAS2764)	+= snd-soc-tas2764.o
@@ -760,6 +765,7 @@ obj-$(CONFIG_SND_SOC_TAS571X)	+= snd-soc-tas571x.o
 obj-$(CONFIG_SND_SOC_TAS5720)	+= snd-soc-tas5720.o
 obj-$(CONFIG_SND_SOC_TAS5805M)	+= snd-soc-tas5805m.o
 obj-$(CONFIG_SND_SOC_TAS6424)	+= snd-soc-tas6424.o
+obj-$(CONFIG_SND_SOC_TAS675X)	+= snd-soc-tas675x.o
 obj-$(CONFIG_SND_SOC_TDA7419)	+= snd-soc-tda7419.o
 obj-$(CONFIG_SND_SOC_TAS2770) += snd-soc-tas2770.o
 obj-$(CONFIG_SND_SOC_TFA9879)	+= snd-soc-tfa9879.o
diff --git a/sound/soc/codecs/cs35l56-sdw.c b/sound/soc/codecs/cs35l56-sdw.c
index 9dc47fec1ea04..d2b82a846ae8f 100644
--- a/sound/soc/codecs/cs35l56-sdw.c
+++ b/sound/soc/codecs/cs35l56-sdw.c
@@ -17,6 +17,7 @@
 #include <linux/string_choices.h>
 #include <linux/swab.h>
 #include <linux/types.h>
+#include <linux/unaligned.h>
 #include <linux/workqueue.h>
 
 #include "cs35l56.h"
@@ -59,8 +60,6 @@ static int cs35l56_sdw_slow_read(struct sdw_slave *peripheral, unsigned int reg,
 {
 	int ret, i;
 
-	reg += CS35L56_SDW_ADDR_OFFSET;
-
 	for (i = 0; i < val_size; i += sizeof(u32)) {
 		/* Poll for bus bridge idle */
 		ret = cs35l56_sdw_poll_mem_status(peripheral,
@@ -97,57 +96,23 @@ static int cs35l56_sdw_slow_read(struct sdw_slave *peripheral, unsigned int reg,
 	return 0;
 }
 
-static int cs35l56_sdw_read_one(struct sdw_slave *peripheral, unsigned int reg, void *buf)
-{
-	int ret;
-
-	ret = sdw_nread_no_pm(peripheral, reg, 4, (u8 *)buf);
-	if (ret != 0) {
-		dev_err(&peripheral->dev, "Read failed @%#x:%d\n", reg, ret);
-		return ret;
-	}
-
-	swab32s((u32 *)buf);
-
-	return 0;
-}
-
 static int cs35l56_sdw_read(void *context, const void *reg_buf,
 			    const size_t reg_size, void *val_buf,
 			    size_t val_size)
 {
 	struct sdw_slave *peripheral = context;
-	u8 *buf8 = val_buf;
-	unsigned int reg, bytes;
+	struct cs35l56_private *cs35l56 = dev_get_drvdata(&peripheral->dev);
+	unsigned int reg_addr = get_unaligned_le32(reg_buf);
 	int ret;
 
-	reg = le32_to_cpu(*(const __le32 *)reg_buf);
-
-	if (cs35l56_is_otp_register(reg))
-		return cs35l56_sdw_slow_read(peripheral, reg, buf8, val_size);
+	if (cs35l56_is_otp_register(reg_addr - CS35L56_SDW_ADDR_OFFSET))
+		return cs35l56_sdw_slow_read(peripheral, reg_addr, (u8 *)val_buf, val_size);
 
-	reg += CS35L56_SDW_ADDR_OFFSET;
-
-	if (val_size == 4)
-		return cs35l56_sdw_read_one(peripheral, reg, val_buf);
-
-	while (val_size) {
-		bytes = SDW_REG_NO_PAGE - (reg & SDW_REGADDR); /* to end of page */
-		if (bytes > val_size)
-			bytes = val_size;
-
-		ret = sdw_nread_no_pm(peripheral, reg, bytes, buf8);
-		if (ret != 0) {
-			dev_err(&peripheral->dev, "Read failed @%#x..%#x:%d\n",
-				reg, reg + bytes - 1, ret);
-			return ret;
-		}
+	ret = regmap_raw_read(cs35l56->sdw_bus_regmap, reg_addr, val_buf, val_size);
+	if (ret)
+		return ret;
 
-		swab32_array((u32 *)buf8, bytes / 4);
-		val_size -= bytes;
-		reg += bytes;
-		buf8 += bytes;
-	}
+	swab32_array((u32 *)val_buf, val_size / sizeof(u32));
 
 	return 0;
 }
@@ -161,58 +126,34 @@ static inline void cs35l56_swab_copy(void *dest, const void *src, size_t nbytes)
 		*dest32++ = swab32(*src32++);
 }
 
-static int cs35l56_sdw_write_one(struct sdw_slave *peripheral, unsigned int reg, const void *buf)
-{
-	u32 val_le = swab32(*(u32 *)buf);
-	int ret;
-
-	ret = sdw_nwrite_no_pm(peripheral, reg, 4, (u8 *)&val_le);
-	if (ret != 0) {
-		dev_err(&peripheral->dev, "Write failed @%#x:%d\n", reg, ret);
-		return ret;
-	}
-
-	return 0;
-}
-
 static int cs35l56_sdw_gather_write(void *context,
 				    const void *reg_buf, size_t reg_size,
 				    const void *val_buf, size_t val_size)
 {
 	struct sdw_slave *peripheral = context;
-	const u8 *src_be = val_buf;
-	u32 val_le_buf[64];	/* Define u32 so it is 32-bit aligned */
-	unsigned int reg, bytes;
+	struct cs35l56_private *cs35l56 = dev_get_drvdata(&peripheral->dev);
+	unsigned int reg_addr = get_unaligned_le32(reg_buf);
+	u32 swab_buf[64];	/* Define u32 so it is 32-bit aligned */
 	int ret;
 
-	reg = le32_to_cpu(*(const __le32 *)reg_buf);
-	reg += CS35L56_SDW_ADDR_OFFSET;
-
-	if (val_size == 4)
-		return cs35l56_sdw_write_one(peripheral, reg, src_be);
-
-	while (val_size) {
-		bytes = SDW_REG_NO_PAGE - (reg & SDW_REGADDR); /* to end of page */
-		if (bytes > val_size)
-			bytes = val_size;
-		if (bytes > sizeof(val_le_buf))
-			bytes = sizeof(val_le_buf);
-
-		cs35l56_swab_copy(val_le_buf, src_be, bytes);
-
-		ret = sdw_nwrite_no_pm(peripheral, reg, bytes, (u8 *)val_le_buf);
-		if (ret != 0) {
-			dev_err(&peripheral->dev, "Write failed @%#x..%#x:%d\n",
-				reg, reg + bytes - 1, ret);
+	while (val_size > sizeof(swab_buf)) {
+		cs35l56_swab_copy(swab_buf, val_buf, sizeof(swab_buf));
+		ret = regmap_raw_write(cs35l56->sdw_bus_regmap, reg_addr,
+				       swab_buf, sizeof(swab_buf));
+		if (ret)
 			return ret;
-		}
 
-		val_size -= bytes;
-		reg += bytes;
-		src_be += bytes;
+		val_size -= sizeof(swab_buf);
+		reg_addr += sizeof(swab_buf);
+		val_buf += sizeof(swab_buf);
 	}
 
-	return 0;
+	if (val_size == 0)
+		return 0;
+
+	cs35l56_swab_copy(swab_buf, val_buf, val_size);
+
+	return regmap_raw_write(cs35l56->sdw_bus_regmap, reg_addr, swab_buf, val_size);
 }
 
 static int cs35l56_sdw_write(void *context, const void *val_buf, size_t val_size)
@@ -231,7 +172,7 @@ static int cs35l56_sdw_write(void *context, const void *val_buf, size_t val_size
  * byte controls always have the same byte order, and firmware file blobs
  * can be written verbatim.
  */
-static const struct regmap_bus cs35l56_regmap_bus_sdw = {
+static const struct regmap_bus cs35l56_regmap_swab_bus_sdw = {
 	.read = cs35l56_sdw_read,
 	.write = cs35l56_sdw_write,
 	.gather_write = cs35l56_sdw_gather_write,
@@ -239,6 +180,18 @@ static const struct regmap_bus cs35l56_regmap_bus_sdw = {
 	.val_format_endian_default = REGMAP_ENDIAN_BIG,
 };
 
+/* Low-level SoundWire regmap to transfer the data over the bus */
+static const struct regmap_config cs35l56_sdw_bus_regmap = {
+	.name = "sdw-le32",
+	.reg_bits = 32,
+	.val_bits = 32,
+	.reg_stride = 4,
+	.reg_format_endian = REGMAP_ENDIAN_LITTLE,
+	.val_format_endian = REGMAP_ENDIAN_LITTLE,
+	.max_register = CS35L56_DSP1_PMEM_5114 + 0x8000,
+	.cache_type = REGCACHE_NONE,
+};
+
 static int cs35l56_sdw_get_unique_id(struct cs35l56_private *cs35l56)
 {
 	int ret;
@@ -385,18 +338,19 @@ static int cs35l56_sdw_update_status(struct sdw_slave *peripheral,
 
 	switch (status) {
 	case SDW_SLAVE_ATTACHED:
-		dev_dbg(cs35l56->base.dev, "%s: ATTACHED\n", __func__);
 		cs35l56->sdw_in_clock_stop_1 = false;
 		if (cs35l56->sdw_attached)
 			break;
 
+		dev_dbg(cs35l56->base.dev, "%s: ATTACHED\n", __func__);
 		if (!cs35l56->base.init_done || cs35l56->soft_resetting)
 			cs35l56_sdw_init(peripheral);
 
 		cs35l56->sdw_attached = true;
 		break;
 	case SDW_SLAVE_UNATTACHED:
-		dev_dbg(cs35l56->base.dev, "%s: UNATTACHED\n", __func__);
+		if (cs35l56->sdw_attached)
+			dev_dbg(cs35l56->base.dev, "%s: UNATTACHED\n", __func__);
 		cs35l56->sdw_attached = false;
 		break;
 	default:
@@ -436,6 +390,7 @@ static const struct sdw_slave_ops cs35l56_sdw_ops = {
 static int __maybe_unused cs35l56_sdw_handle_unattach(struct cs35l56_private *cs35l56)
 {
 	struct sdw_slave *peripheral = cs35l56->sdw_peripheral;
+	int ret;
 
 	dev_dbg(cs35l56->base.dev, "attached:%u unattach_request:%u in_clock_stop_1:%u\n",
 		cs35l56->sdw_attached, peripheral->unattach_request, cs35l56->sdw_in_clock_stop_1);
@@ -443,13 +398,10 @@ static int __maybe_unused cs35l56_sdw_handle_unattach(struct cs35l56_private *cs
 	if (cs35l56->sdw_in_clock_stop_1 || peripheral->unattach_request) {
 		/* Cannot access registers until bus is re-initialized. */
 		dev_dbg(cs35l56->base.dev, "Wait for initialization_complete\n");
-		if (!wait_for_completion_timeout(&peripheral->initialization_complete,
-						 msecs_to_jiffies(5000))) {
-			dev_err(cs35l56->base.dev, "initialization_complete timed out\n");
-			return -ETIMEDOUT;
-		}
+		ret = sdw_slave_wait_for_init(peripheral, 5000);
+		if (ret)
+			return ret;
 
-		peripheral->unattach_request = 0;
 		cs35l56->sdw_in_clock_stop_1 = false;
 
 		/*
@@ -561,8 +513,16 @@ static int cs35l56_sdw_probe(struct sdw_slave *peripheral, const struct sdw_devi
 
 	cs35l56->base.type = ((unsigned int)id->driver_data) & 0xff;
 
-	cs35l56->base.regmap = devm_regmap_init(dev, &cs35l56_regmap_bus_sdw,
-					   peripheral, regmap_config);
+	/* Low-level regmap to transfer read/writes over SoundWire bus */
+	cs35l56->sdw_bus_regmap = devm_regmap_init_sdw(peripheral, &cs35l56_sdw_bus_regmap);
+	if (IS_ERR(cs35l56->sdw_bus_regmap)) {
+		ret = PTR_ERR(cs35l56->sdw_bus_regmap);
+		return dev_err_probe(dev, ret, "Failed to allocate bus register map\n");
+	}
+
+	/* Wrapper regmap to simulate big-endian ordering */
+	cs35l56->base.regmap = devm_regmap_init(dev, &cs35l56_regmap_swab_bus_sdw,
+						peripheral, regmap_config);
 	if (IS_ERR(cs35l56->base.regmap)) {
 		ret = PTR_ERR(cs35l56->base.regmap);
 		return dev_err_probe(dev, ret, "Failed to allocate register map\n");
diff --git a/sound/soc/codecs/cs35l56-shared-test.c b/sound/soc/codecs/cs35l56-shared-test.c
index cfe7938065f90..8060a275d32bd 100644
--- a/sound/soc/codecs/cs35l56-shared-test.c
+++ b/sound/soc/codecs/cs35l56-shared-test.c
@@ -29,6 +29,7 @@ struct cs35l56_shared_test_priv {
 	struct faux_device *gpio_dev;
 	struct cs35l56_shared_test_mock_gpio *gpio_priv;
 	struct regmap *registers;
+	unsigned int reg_offset;
 	struct cs35l56_base *cs35l56_base;
 	u8 applied_pad_pull_state[CS35L56_MAX_GPIO];
 };
@@ -194,6 +195,8 @@ static int cs35l56_shared_test_reg_read(void *context, unsigned int reg, unsigne
 {
 	struct cs35l56_shared_test_priv *priv = context;
 
+	reg -= priv->reg_offset;
+
 	switch (reg) {
 	case CS35L56_SYNC_GPIO1_CFG ... CS35L56_ASP2_DIO_GPIO13_CFG:
 	case CS35L56_GPIO1_CTRL1 ... CS35L56_GPIO13_CTRL1:
@@ -214,6 +217,8 @@ static int cs35l56_shared_test_reg_write(void *context, unsigned int reg, unsign
 {
 	struct cs35l56_shared_test_priv *priv = context;
 
+	reg -= priv->reg_offset;
+
 	switch (reg) {
 	case CS35L56_UPDATE_REGS:
 		return cs35l56_shared_test_updt_gpio_pres(priv, reg, val);
@@ -657,6 +662,7 @@ static int cs35l56_shared_test_case_base_init(struct kunit *test, u8 type, u8 re
 
 	test->priv = priv;
 	priv->test = test;
+	priv->reg_offset = regmap_config->reg_base;
 
 	/* Create dummy amp driver dev */
 	priv->amp_dev = faux_device_create("cs35l56_shared_test_drv", NULL, NULL);
diff --git a/sound/soc/codecs/cs35l56-shared.c b/sound/soc/codecs/cs35l56-shared.c
index 795e2764d67ec..8e3538e28fade 100644
--- a/sound/soc/codecs/cs35l56-shared.c
+++ b/sound/soc/codecs/cs35l56-shared.c
@@ -1880,6 +1880,7 @@ EXPORT_SYMBOL_NS_GPL(cs35l56_regmap_spi, "SND_SOC_CS35L56_SHARED");
 
 const struct regmap_config cs35l56_regmap_sdw = {
 	.reg_bits = 32,
+	.reg_base = 0x8000,
 	.val_bits = 32,
 	.reg_stride = 4,
 	.reg_format_endian = REGMAP_ENDIAN_LITTLE,
@@ -1915,6 +1916,7 @@ const struct regmap_config cs35l63_regmap_sdw = {
 	.reg_bits = 32,
 	.val_bits = 32,
 	.reg_stride = 4,
+	.reg_base = 0x8000,
 	.reg_format_endian = REGMAP_ENDIAN_LITTLE,
 	.val_format_endian = REGMAP_ENDIAN_BIG,
 	.max_register = CS35L56_DSP1_PMEM_5114,
diff --git a/sound/soc/codecs/cs35l56.c b/sound/soc/codecs/cs35l56.c
index 849d70ca23d6f..80158913a60e0 100644
--- a/sound/soc/codecs/cs35l56.c
+++ b/sound/soc/codecs/cs35l56.c
@@ -426,6 +426,8 @@ static unsigned int cs35l56_make_tdm_config_word(unsigned int reg_val, unsigned
 		reg_val &= ~(0x3f << channel_shift);
 		reg_val |= bit_num << channel_shift;
 		channel_shift += 8;
+		if (channel_shift > 24)
+			break;
 	}
 
 	return reg_val;
@@ -867,11 +869,16 @@ static void cs35l56_dsp_work(struct work_struct *work)
 	if (!cs35l56->base.init_done)
 		return;
 
-	pm_runtime_get_sync(cs35l56->base.dev);
+	PM_RUNTIME_ACQUIRE(cs35l56->base.dev, pm);
+	ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
+	if (ret) {
+		dev_err(cs35l56->base.dev, "dsp_work failed to runtime-resume: %d\n", ret);
+		return;
+	}
 
 	ret = cs35l56_read_prot_status(&cs35l56->base, &firmware_missing, &firmware_version);
 	if (ret)
-		goto err;
+		return;
 
 	/* Populate fw file qualifier with the revision and security state */
 	kfree(cs35l56->dsp.fwf_name);
@@ -887,7 +894,7 @@ static void cs35l56_dsp_work(struct work_struct *work)
 	}
 
 	if (!cs35l56->dsp.fwf_name)
-		goto err;
+		return;
 
 	dev_dbg(cs35l56->base.dev, "DSP fwf name: '%s' system name: '%s'\n",
 		cs35l56->dsp.fwf_name, cs35l56->dsp.system_name);
@@ -905,8 +912,6 @@ static void cs35l56_dsp_work(struct work_struct *work)
 		cs35l56_patch(cs35l56, firmware_missing);
 
 	cs35l56_log_tuning(&cs35l56->base, &cs35l56->dsp.cs_dsp);
-err:
-	pm_runtime_put_autosuspend(cs35l56->base.dev);
 }
 
 static struct snd_soc_dapm_context *cs35l56_power_up_for_cal(struct cs35l56_private *cs35l56)
diff --git a/sound/soc/codecs/cs35l56.h b/sound/soc/codecs/cs35l56.h
index cd71b23b2a3ac..d029fa3f86565 100644
--- a/sound/soc/codecs/cs35l56.h
+++ b/sound/soc/codecs/cs35l56.h
@@ -37,6 +37,7 @@ struct cs35l56_private {
 	struct snd_soc_component *component;
 	struct regulator_bulk_data supplies[CS35L56_NUM_BULK_SUPPLIES];
 	struct sdw_slave *sdw_peripheral;
+	struct regmap *sdw_bus_regmap;
 	const char *fallback_fw_suffix;
 	struct work_struct sdw_irq_work;
 	bool sdw_irq_no_unmask;
diff --git a/sound/soc/codecs/cs42l42-sdw.c b/sound/soc/codecs/cs42l42-sdw.c
index d5999ad9ff9b4..b8256ce0b8fbe 100644
--- a/sound/soc/codecs/cs42l42-sdw.c
+++ b/sound/soc/codecs/cs42l42-sdw.c
@@ -433,19 +433,16 @@ static const struct reg_sequence cs42l42_soft_reboot_seq[] = {
 static int cs42l42_sdw_handle_unattach(struct cs42l42_private *cs42l42)
 {
 	struct sdw_slave *peripheral = cs42l42->sdw_peripheral;
+	int ret;
 
 	if (!peripheral->unattach_request)
 		return 0;
 
 	/* Cannot access registers until master re-attaches. */
 	dev_dbg(&peripheral->dev, "Wait for initialization_complete\n");
-	if (!wait_for_completion_timeout(&peripheral->initialization_complete,
-					 msecs_to_jiffies(5000))) {
-		dev_err(&peripheral->dev, "initialization_complete timed out\n");
-		return -ETIMEDOUT;
-	}
-
-	peripheral->unattach_request = 0;
+	ret = sdw_slave_wait_for_init(peripheral, 5000);
+	if (ret)
+		return ret;
 
 	/*
 	 * After a bus reset there must be a reconfiguration reset to
diff --git a/sound/soc/codecs/cs42l43-jack.c b/sound/soc/codecs/cs42l43-jack.c
index 3e04e6897b142..934666295ee30 100644
--- a/sound/soc/codecs/cs42l43-jack.c
+++ b/sound/soc/codecs/cs42l43-jack.c
@@ -805,7 +805,7 @@ irqreturn_t cs42l43_tip_sense(int irq, void *data)
 	if (priv->suspend_jack_debounce)
 		db_delay += priv->tip_fall_db_ms + priv->tip_rise_db_ms;
 
-	queue_delayed_work(system_long_wq, &priv->tip_sense_work,
+	queue_delayed_work(system_dfl_long_wq, &priv->tip_sense_work,
 			   msecs_to_jiffies(db_delay));
 
 	return IRQ_HANDLED;
@@ -932,7 +932,8 @@ int cs42l43_jack_put(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *u
 	snd_soc_jack_report(priv->jack_hp, 0, 0xFFFF);
 
 	if (!override) {
-		queue_delayed_work(system_long_wq, &priv->tip_sense_work, 0);
+		queue_delayed_work(system_dfl_long_wq, &priv->tip_sense_work,
+				   0);
 	} else {
 		override--;
 
diff --git a/sound/soc/codecs/cx20442.c b/sound/soc/codecs/cx20442.c
index c0dc69ce4d0e7..0dea84cd8ef2d 100644
--- a/sound/soc/codecs/cx20442.c
+++ b/sound/soc/codecs/cx20442.c
@@ -236,7 +236,8 @@ static int v253_open(struct tty_struct *tty)
 /* Line discipline .close() */
 static void v253_close(struct tty_struct *tty)
 {
-	struct snd_soc_component *component = tty->disc_data;
+	struct cx20442_codec *codec = tty->disc_data;
+	struct snd_soc_component *component = codec->component;
 	struct cx20442_priv *cx20442;
 
 	tty->disc_data = NULL;
@@ -248,7 +249,7 @@ static void v253_close(struct tty_struct *tty)
 
 	/* Prevent the codec driver from further accessing the modem */
 	cx20442->tty = NULL;
-	component->card->pop_time = 0;
+	codec->ready = false;
 }
 
 /* Line discipline .hangup() */
@@ -261,7 +262,8 @@ static void v253_hangup(struct tty_struct *tty)
 static void v253_receive(struct tty_struct *tty, const u8 *cp, const u8 *fp,
 			 size_t count)
 {
-	struct snd_soc_component *component = tty->disc_data;
+	struct cx20442_codec *codec = tty->disc_data;
+	struct snd_soc_component *component = codec->component;
 	struct cx20442_priv *cx20442;
 
 	if (!component)
@@ -274,7 +276,7 @@ static void v253_receive(struct tty_struct *tty, const u8 *cp, const u8 *fp,
 
 		/* Set up codec driver access to modem controls */
 		cx20442->tty = tty;
-		component->card->pop_time = 1;
+		codec->ready = true;
 	}
 }
 
@@ -375,7 +377,6 @@ static int cx20442_component_probe(struct snd_soc_component *component)
 	cx20442->tty = NULL;
 
 	snd_soc_component_set_drvdata(component, cx20442);
-	component->card->pop_time = 0;
 
 	return 0;
 }
diff --git a/sound/soc/codecs/cx20442.h b/sound/soc/codecs/cx20442.h
index bb897bcb24869..78ab1eab083e7 100644
--- a/sound/soc/codecs/cx20442.h
+++ b/sound/soc/codecs/cx20442.h
@@ -8,6 +8,11 @@
 #ifndef _CX20442_CODEC_H
 #define _CX20442_CODEC_H
 
+struct cx20442_codec {
+	struct snd_soc_component *component;
+	bool ready;
+};
+
 extern struct tty_ldisc_ops v253_ops;
 
 #endif
diff --git a/sound/soc/codecs/es9356.c b/sound/soc/codecs/es9356.c
new file mode 100644
index 0000000000000..78fddd9d01711
--- /dev/null
+++ b/sound/soc/codecs/es9356.c
@@ -0,0 +1,1151 @@
+// SPDX-License-Identifier: GPL-2.0-only
+//
+// es9356.c -- SoundWire codec driver
+//
+// Copyright(c) 2025 Everest Semiconductor Co., Ltd
+//
+//
+
+#include <linux/device.h>
+#include <linux/mod_devicetable.h>
+#include <linux/module.h>
+#include <linux/soundwire/sdw.h>
+#include <linux/soundwire/sdw_type.h>
+#include <linux/soundwire/sdw_registers.h>
+#include <sound/core.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
+#include <linux/pm_runtime.h>
+#include <sound/sdw.h>
+#include <sound/soc.h>
+#include <sound/soc-dapm.h>
+#include <sound/tlv.h>
+#include <sound/sdca_function.h>
+#include <sound/sdca_regmap.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <sound/jack.h>
+#include <sound/sdca_asoc.h>
+#include "es9356.h"
+
+struct  es9356_sdw_priv {
+	struct sdw_slave *slave;
+	struct device *dev;
+	struct regmap *regmap;
+	struct snd_soc_component *component;
+	struct snd_soc_jack *hs_jack;
+
+	/* lock for irq*/
+	struct mutex disable_irq_lock;
+
+	/* lock for pde*/
+	struct mutex pde_lock;
+
+	bool hw_init;
+	bool first_hw_init;
+	int jack_type;
+	bool disable_irq;
+
+	struct delayed_work interrupt_handle_work;
+	struct delayed_work button_detect_work;
+	unsigned int sdca_status;
+};
+
+static int es9356_sdw_component_probe(struct snd_soc_component *component)
+{
+	struct es9356_sdw_priv *es9356 = snd_soc_component_get_drvdata(component);
+
+	es9356->component = component;
+
+	return 0;
+}
+
+static const DECLARE_TLV_DB_SCALE(out_vol_tlv, -9600, 12, 0);
+static const DECLARE_TLV_DB_SCALE(amic_gain_tlv, 0, 3, 0);
+static const DECLARE_TLV_DB_SCALE(dmic_gain_tlv, 0, 6, 0);
+
+static const struct snd_kcontrol_new es9356_sdca_controls[] = {
+	SDCA_SINGLE_Q78_TLV("FU41 Left Playback Volume",
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU41, ES9356_SDCA_CTL_FU_VOLUME, CH_L),
+		ES9356_VOLUME_MIN, ES9356_VOLUME_MAX, ES9356_VOLUME_STEP, out_vol_tlv),
+	SDCA_SINGLE_Q78_TLV("FU41 Right Playback Volume",
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU41, ES9356_SDCA_CTL_FU_VOLUME, CH_R),
+		ES9356_VOLUME_MIN, ES9356_VOLUME_MAX, ES9356_VOLUME_STEP, out_vol_tlv),
+	SDCA_SINGLE_Q78_TLV("FU36 Left Capture Volume",
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU36, ES9356_SDCA_CTL_FU_VOLUME, CH_L),
+		ES9356_VOLUME_MIN, ES9356_VOLUME_MAX, ES9356_VOLUME_STEP, out_vol_tlv),
+	SDCA_SINGLE_Q78_TLV("FU36 Right Capture Volume",
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU36, ES9356_SDCA_CTL_FU_VOLUME, CH_R),
+		ES9356_VOLUME_MIN, ES9356_VOLUME_MAX, ES9356_VOLUME_STEP, out_vol_tlv),
+	SDCA_SINGLE_Q78_TLV("FU33 Capture Volume",
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU33, ES9356_SDCA_CTL_FU_CH_GAIN, 0),
+		ES9356_GAIN_MIN, ES9356_AMIC_GAIN_MAX, ES9356_AMIC_GAIN_STEP, amic_gain_tlv),
+	SDCA_SINGLE_Q78_TLV("FU21 Left Playback Volume",
+		SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_FU21, ES9356_SDCA_CTL_FU_VOLUME, CH_L),
+		ES9356_VOLUME_MIN, ES9356_VOLUME_MAX, ES9356_VOLUME_STEP, out_vol_tlv),
+	SDCA_SINGLE_Q78_TLV("FU21 Right Playback Volume",
+		SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_FU21, ES9356_SDCA_CTL_FU_VOLUME, CH_R),
+		ES9356_VOLUME_MIN, ES9356_VOLUME_MAX, ES9356_VOLUME_STEP, out_vol_tlv),
+	SDCA_SINGLE_Q78_TLV("FU113 Left Capture Volume",
+		SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU113, ES9356_SDCA_CTL_FU_VOLUME, CH_L),
+		ES9356_VOLUME_MIN, ES9356_VOLUME_MAX, ES9356_VOLUME_STEP, out_vol_tlv),
+	SDCA_SINGLE_Q78_TLV("FU113 Right Capture Volume",
+		SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU113, ES9356_SDCA_CTL_FU_VOLUME, CH_R),
+		ES9356_VOLUME_MIN, ES9356_VOLUME_MAX, ES9356_VOLUME_STEP, out_vol_tlv),
+	SDCA_SINGLE_Q78_TLV("FU11 Capture Volume",
+		SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU11, ES9356_SDCA_CTL_FU_CH_GAIN, 0),
+		ES9356_GAIN_MIN, ES9356_DMIC_GAIN_MAX, ES9356_DMIC_GAIN_STEP, dmic_gain_tlv),
+};
+
+static const char *const es9356_left_mux_txt[] = {
+	"Left",
+	"Right",
+};
+
+static const char *const es9356_right_mux_txt[] = {
+	"Right",
+	"Left",
+};
+
+static const struct soc_enum es9356_left_mux_enum =
+	SOC_ENUM_SINGLE(ES9356_DAC_SWAP, 1,
+			ARRAY_SIZE(es9356_left_mux_txt), es9356_left_mux_txt);
+static const struct soc_enum es9356_right_mux_enum =
+	SOC_ENUM_SINGLE(ES9356_DAC_SWAP, 0,
+			ARRAY_SIZE(es9356_right_mux_txt), es9356_right_mux_txt);
+
+static const struct snd_kcontrol_new es9356_left_mux_controls =
+	SOC_DAPM_ENUM("Channel MUX", es9356_left_mux_enum);
+static const struct snd_kcontrol_new es9356_right_mux_controls =
+	SOC_DAPM_ENUM("Channel MUX", es9356_right_mux_enum);
+
+static const struct snd_soc_dapm_widget es9356_dapm_widgets[] = {
+	SND_SOC_DAPM_OUTPUT("HP"),
+	SND_SOC_DAPM_OUTPUT("SPK"),
+	SND_SOC_DAPM_INPUT("MIC1"),
+	SND_SOC_DAPM_INPUT("PDM_DIN"),
+
+	SND_SOC_DAPM_SUPPLY("DMIC Clock", ES9356_DMIC_GPIO, 1, 1, NULL, 0),
+
+	SND_SOC_DAPM_AIF_IN("DP4RX", "DP4 Playback", 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_IN("DP3RX", "DP3 Playback", 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("DP1TX", "DP1 Capture", 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_AIF_OUT("DP2TX", "DP2 Capture", 0, SND_SOC_NOPM, 0, 0),
+
+	SND_SOC_DAPM_PGA("IF DP3RXL", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("IF DP3RXR", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	SND_SOC_DAPM_MUX("Left Channel MUX", SND_SOC_NOPM, 0, 0, &es9356_left_mux_controls),
+	SND_SOC_DAPM_MUX("Right Channel MUX", SND_SOC_NOPM, 0, 0, &es9356_right_mux_controls),
+
+	SND_SOC_DAPM_DAC("FU 21 Left", NULL,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_FU21, ES9356_SDCA_CTL_FU_MUTE, CH_L), 0, 1),
+	SND_SOC_DAPM_DAC("FU 21 Right", NULL,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_FU21, ES9356_SDCA_CTL_FU_MUTE, CH_R), 0, 1),
+	SND_SOC_DAPM_DAC("FU 41 Left", NULL,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU41, ES9356_SDCA_CTL_FU_MUTE, CH_L), 0, 1),
+	SND_SOC_DAPM_DAC("FU 41 Right", NULL,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU41, ES9356_SDCA_CTL_FU_MUTE, CH_R), 0, 1),
+	SND_SOC_DAPM_DAC("FU 113 Left", NULL,
+		SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU113, ES9356_SDCA_CTL_FU_MUTE, CH_L), 0, 1),
+	SND_SOC_DAPM_DAC("FU 113 Right", NULL,
+		SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU113, ES9356_SDCA_CTL_FU_MUTE, CH_R), 0, 1),
+	SND_SOC_DAPM_DAC("FU 36 Left", NULL,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU36, ES9356_SDCA_CTL_FU_MUTE, CH_L), 0, 1),
+	SND_SOC_DAPM_DAC("FU 36 Right", NULL,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU36, ES9356_SDCA_CTL_FU_MUTE, CH_R), 0, 1),
+};
+
+static const struct snd_soc_dapm_route es9356_audio_map[] = {
+	{"FU 36 Left", NULL, "MIC1"},
+	{"FU 36 Right", NULL, "MIC1"},
+	{"DP2TX", NULL, "FU 36 Left"},
+	{"DP2TX", NULL, "FU 36 Right"},
+
+	{"PDM_DIN", NULL, "DMIC Clock"},
+	{"FU 113 Left", NULL, "PDM_DIN"},
+	{"FU 113 Right", NULL, "PDM_DIN"},
+	{"DP1TX", NULL, "FU 113 Left"},
+	{"DP1TX", NULL, "FU 113 Right"},
+
+	{"FU 41 Left", NULL, "DP4RX"},
+	{"FU 41 Right", NULL, "DP4RX"},
+
+	{"IF DP3RXL", NULL, "DP3RX"},
+	{"IF DP3RXR", NULL, "DP3RX"},
+
+	{"Left Channel MUX", "Left", "IF DP3RXL"},
+	{"Left Channel MUX", "Right", "IF DP3RXR"},
+	{"Right Channel MUX", "Left", "IF DP3RXL"},
+	{"Right Channel MUX", "Right", "IF DP3RXR"},
+
+	{"FU 21 Left", NULL, "Left Channel MUX"},
+	{"FU 21 Right", NULL, "Right Channel MUX"},
+
+	{"SPK", NULL, "FU 21 Left"},
+	{"SPK", NULL, "FU 21 Right"},
+
+	{"HP", NULL, "FU 41 Left"},
+	{"HP", NULL, "FU 41 Right"},
+};
+
+static int es9356_set_jack_detect(struct snd_soc_component *component,
+	struct snd_soc_jack *hs_jack, void *data)
+{
+	struct es9356_sdw_priv *es9356 = snd_soc_component_get_drvdata(component);
+	int ret;
+
+	es9356->hs_jack = hs_jack;
+
+	/* we can only resume if the device was initialized at least once */
+	if (!es9356->first_hw_init)
+		return 0;
+
+	ret = pm_runtime_resume_and_get(component->dev);
+	if (ret < 0) {
+		if (ret != -EACCES) {
+			dev_err(component->dev, "%s: failed to resume %d\n", __func__, ret);
+			return ret;
+		}
+		/* pm_runtime not enabled yet */
+		dev_info(component->dev, "%s: skipping jack init for now\n", __func__);
+		return 0;
+	}
+
+	if (es9356->hs_jack)
+		sdw_write_no_pm(es9356->slave, SDW_SCP_SDCA_INTMASK1,
+			(SDW_SCP_SDCA_INTMASK_SDCA_7 | SDW_SCP_SDCA_INTMASK_SDCA_5 | SDW_SCP_SDCA_INTMASK_SDCA_1));
+
+	pm_runtime_mark_last_busy(component->dev);
+	pm_runtime_put_autosuspend(component->dev);
+
+	return 0;
+}
+static const struct snd_soc_component_driver snd_soc_es9356_sdw_component = {
+	.probe = es9356_sdw_component_probe,
+	.controls = es9356_sdca_controls,
+	.num_controls = ARRAY_SIZE(es9356_sdca_controls),
+	.dapm_widgets = es9356_dapm_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(es9356_dapm_widgets),
+	.dapm_routes = es9356_audio_map,
+	.num_dapm_routes = ARRAY_SIZE(es9356_audio_map),
+	.set_jack = es9356_set_jack_detect,
+	.endianness = 1,
+};
+
+static int es9356_sdw_set_sdw_stream(struct snd_soc_dai *dai, void *sdw_stream,
+				     int direction)
+{
+	snd_soc_dai_dma_data_set(dai, direction, sdw_stream);
+
+	return 0;
+}
+
+static void es9356_sdw_shutdown(struct snd_pcm_substream *substream,
+				struct snd_soc_dai *dai)
+{
+	snd_soc_dai_set_dma_data(dai, substream, NULL);
+}
+
+static int es9356_sdca_button(unsigned int *buffer)
+{
+	int cur_button = -1;
+
+	if (*(buffer + 1) | *(buffer + 2))
+		return -EINVAL;
+	switch (*buffer) {
+	case 0x00:
+		cur_button = 0;
+		break;
+	case 0x20:
+		cur_button = SND_JACK_BTN_4;
+		break;
+	case 0x10:
+		cur_button = SND_JACK_BTN_2;
+		break;
+	case 0x08:
+		cur_button = SND_JACK_BTN_1;
+		break;
+	case 0x02:
+		cur_button = SND_JACK_BTN_3;
+		break;
+	case 0x01:
+		cur_button = SND_JACK_BTN_0;
+		break;
+	default:
+		break;
+	}
+
+	return cur_button;
+}
+
+static int es9356_sdca_button_detect(struct es9356_sdw_priv *es9356)
+{
+	unsigned int btn_type = 0, offset, idx, val, owner;
+	unsigned int button[3];
+	int ret;
+
+	ret = regmap_read(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_HID, ES9356_SDCA_ENT_HID01, ES9356_SDCA_CTL_HIDTX_CURRENT_OWNER, 0), &owner);
+	if (ret < 0 || owner == 0x01)
+		return 0;
+
+	ret = regmap_read(es9356->regmap, ES9356_BUF_ADDR_HID, &offset);
+	if (ret < 0)
+		goto button_det_end;
+
+	for (idx = 0; idx < ARRAY_SIZE(button); idx++) {
+		ret = regmap_read(es9356->regmap, ES9356_BUF_ADDR_HID + offset + idx, &val);
+		if (ret < 0)
+			goto button_det_end;
+		button[idx] = val;
+	}
+
+	btn_type = es9356_sdca_button(&button[0]);
+
+button_det_end:
+	if (owner == 0x00)
+		regmap_write(es9356->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_HID, ES9356_SDCA_ENT_HID01, ES9356_SDCA_CTL_HIDTX_CURRENT_OWNER, 0), 0x01);
+
+	return btn_type;
+}
+
+static int es9356_sdca_headset_detect(struct es9356_sdw_priv *es9356)
+{
+	unsigned int reg;
+	int ret;
+
+	ret = regmap_read(es9356->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_GE35, ES9356_SDCA_CTL_DETECTED_MODE, 0), &reg);
+
+	if (ret < 0)
+		goto io_error;
+
+	switch (reg) {
+	case 0x00:
+		es9356->jack_type = 0;
+		break;
+	case 0x03:
+		es9356->jack_type = SND_JACK_HEADPHONE;
+		break;
+	case 0x04:
+		es9356->jack_type = SND_JACK_HEADSET;
+		break;
+	default:
+		es9356->jack_type = 0;
+		return -1;
+	}
+
+	if (reg) {
+		regmap_write(es9356->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_GE35, ES9356_SDCA_CTL_SELECTED_MODE, 0), reg);
+		regmap_write(es9356->regmap, ES9356_HP_DETECTTIME, 0x75);
+	} else {
+		regmap_write(es9356->regmap, ES9356_HP_DETECTTIME, 0xa4);
+	}
+
+	return 0;
+
+io_error:
+	pr_err_ratelimited("IO error in %s, ret %d\n", __func__, ret);
+	return ret;
+}
+
+static void es9356_interrupt_handler(struct work_struct *work)
+{
+	struct es9356_sdw_priv *es9356 =
+		container_of(work, struct es9356_sdw_priv, interrupt_handle_work.work);
+	int ret, btn_type = 0;
+
+	if (!es9356->hs_jack)
+		return;
+
+	if (!es9356->component->card || !es9356->component->card->instantiated)
+		return;
+
+	/* Handling different types of interrupts based on the mask bit */
+	if (es9356->sdca_status & SDW_SCP_SDCA_INT_SDCA_7) {
+		btn_type = es9356_sdca_button_detect(es9356);
+		if (btn_type < 0)
+			return;
+	} else {
+		ret = es9356_sdca_headset_detect(es9356);
+		if (ret < 0)
+			return;
+	}
+
+	if (es9356->jack_type != SND_JACK_HEADSET)
+		btn_type = 0;
+
+	snd_soc_jack_report(es9356->hs_jack, es9356->jack_type | btn_type,
+			SND_JACK_HEADSET |
+			SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+			SND_JACK_BTN_2 | SND_JACK_BTN_3 |
+			SND_JACK_BTN_4);
+
+	if (btn_type) {
+		snd_soc_jack_report(es9356->hs_jack, es9356->jack_type,
+			SND_JACK_HEADSET |
+			SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+			SND_JACK_BTN_2 | SND_JACK_BTN_3 |
+			SND_JACK_BTN_4);
+		mod_delayed_work(system_power_efficient_wq,
+			&es9356->button_detect_work, msecs_to_jiffies(280));
+	}
+}
+
+static void es9356_button_detect_handler(struct work_struct *work)
+{
+	struct es9356_sdw_priv *es9356 =
+		container_of(work, struct es9356_sdw_priv, button_detect_work.work);
+	int ret, idx, btn_type = 0;
+	unsigned int reg, offset;
+	unsigned int button[3];
+
+	/* Check headset */
+	ret = regmap_read(es9356->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_GE35, ES9356_SDCA_CTL_DETECTED_MODE, 0), &reg);
+
+	if (ret < 0)
+		goto io_error;
+
+	if (reg == 0x04) {
+		ret = regmap_read(es9356->regmap, ES9356_BUF_ADDR_HID, &offset);
+		if (ret < 0)
+			goto io_error;
+		for (idx = 0; idx < ARRAY_SIZE(button); idx++) {
+			ret = regmap_read(es9356->regmap, ES9356_BUF_ADDR_HID + offset + idx, &reg);
+			if (ret < 0)
+				goto io_error;
+			button[idx] = reg;
+		}
+		btn_type = es9356_sdca_button(&button[0]);
+		if (btn_type < 0)
+			return;
+	}
+
+	snd_soc_jack_report(es9356->hs_jack, es9356->jack_type | btn_type,
+			SND_JACK_HEADSET |
+			SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+			SND_JACK_BTN_2 | SND_JACK_BTN_3 |
+			SND_JACK_BTN_4);
+
+	if (btn_type) {
+		snd_soc_jack_report(es9356->hs_jack, es9356->jack_type,
+			SND_JACK_HEADSET |
+			SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+			SND_JACK_BTN_2 | SND_JACK_BTN_3 |
+			SND_JACK_BTN_4);
+		mod_delayed_work(system_power_efficient_wq,
+			&es9356->button_detect_work, msecs_to_jiffies(280));
+	}
+
+	return;
+io_error:
+	pr_err_ratelimited("IO error in %s, ret %d\n", __func__, ret);
+}
+
+static int es9356_pde_transition_delay(struct es9356_sdw_priv *es9356, unsigned char func,
+	unsigned char entity, unsigned char ps)
+{
+	unsigned int retries = 10, val;
+
+	/* waiting for Actual PDE becomes to PS0/PS3 */
+	while (retries) {
+		regmap_read(es9356->regmap,
+			SDW_SDCA_CTL(func, entity, ES9356_SDCA_CTL_ACTUAL_POWER_STATE, 0), &val);
+		if (val == ps)
+			return 1;
+
+		usleep_range(1000, 1500);
+		retries--;
+	}
+	if (!retries) {
+		dev_dbg(&es9356->slave->dev, "%s PDE is NOT %s", __func__, ps?"PS3":"PS0");
+	}
+	return 0;
+}
+
+static int es9356_power_state(struct snd_soc_dai *dai, unsigned char ps, unsigned int *rate)
+{
+	struct snd_soc_component *component = dai->component;
+	struct es9356_sdw_priv *es9356 = snd_soc_component_get_drvdata(component);
+	unsigned char ps0 = 0x0, ps3 = 0x3;
+	unsigned char func, cs_entity, pde_entity;
+	int ret;
+
+	switch (dai->id) {
+	case ES9356_DMIC:
+		func = FUNC_NUM_MIC;
+		cs_entity = ES9356_SDCA_ENT_CS113;
+		pde_entity = ES9356_SDCA_ENT_PDE11;
+		break;
+	case ES9356_AMP:
+		func = FUNC_NUM_AMP;
+		cs_entity = ES9356_SDCA_ENT_CS21;
+		pde_entity = ES9356_SDCA_ENT_PDE23;
+		break;
+	case ES9356_JACK_IN:
+		func = FUNC_NUM_UAJ;
+		cs_entity = ES9356_SDCA_ENT_CS36;
+		pde_entity = ES9356_SDCA_ENT_PDE34;
+		break;
+	case ES9356_JACK_OUT:
+		func = FUNC_NUM_UAJ;
+		cs_entity = ES9356_SDCA_ENT_CS41;
+		pde_entity = ES9356_SDCA_ENT_PDE47;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* power state changes are not independent across functions */
+	mutex_lock(&es9356->pde_lock);
+	ret = es9356_pde_transition_delay(es9356, func, pde_entity, ps?ps0:ps3);
+	if (ret) {
+		regmap_write(es9356->regmap,
+			SDW_SDCA_CTL(func, pde_entity, ES9356_SDCA_CTL_REQ_POWER_STATE, 0), ps?ps3:ps0);
+		es9356_pde_transition_delay(es9356, func, pde_entity, ps?ps3:ps0);
+	} else
+		dev_dbg(component->dev, "%s PDE is already %d\n", __func__, ps?ps0:ps3);
+
+	mutex_unlock(&es9356->pde_lock);
+
+	if (rate)
+		regmap_write(es9356->regmap,
+			SDW_SDCA_CTL(func, cs_entity, ES9356_SDCA_CTL_SAMPLE_FREQ_INDEX, 0), *rate);
+
+	return 0;
+}
+
+static int es9356_sdw_pcm_hw_params(struct snd_pcm_substream *substream,
+				    struct snd_pcm_hw_params *params,
+				    struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct es9356_sdw_priv *es9356 = snd_soc_component_get_drvdata(component);
+	struct sdw_stream_config stream_config = {0};
+	struct sdw_port_config port_config = {0};
+	struct sdw_stream_runtime *sdw_stream = snd_soc_dai_get_dma_data(dai, substream);
+	unsigned char ps0 = 0x0;
+	unsigned int rate;
+	int ret;
+
+	if (!sdw_stream)
+		return -EINVAL;
+
+	if (!es9356->slave)
+		return -EINVAL;
+
+	/* SoundWire specific configuration */
+	snd_sdw_params_to_config(substream, params, &stream_config, &port_config);
+
+	port_config.num = dai->id;
+
+	ret = sdw_stream_add_slave(es9356->slave, &stream_config,
+				   &port_config, 1, sdw_stream);
+	if (ret) {
+		dev_err(dai->dev, "Unable to configure port\n");
+		return -EINVAL;
+	}
+
+	switch (params_rate(params)) {
+	case 16000:
+		rate = ES9356_SDCA_RATE_16000HZ;
+		break;
+	case 44100:
+		rate = ES9356_SDCA_RATE_44100HZ;
+		break;
+	case 48000:
+		rate = ES9356_SDCA_RATE_48000HZ;
+		break;
+	case 96000:
+		rate = ES9356_SDCA_RATE_96000HZ;
+		break;
+	default:
+		dev_err(component->dev, "%s: Rate %d is not supported\n",
+			__func__, params_rate(params));
+		return -EINVAL;
+	}
+
+	ret = es9356_power_state(dai, ps0, &rate);
+	if (ret) {
+		dev_err(component->dev, "%s: Invalid dai id: %d\n",
+			__func__, dai->id);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int es9356_sdw_pcm_hw_free(struct snd_pcm_substream *substream,
+				  struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct es9356_sdw_priv *es9356 = snd_soc_component_get_drvdata(component);
+	struct sdw_stream_runtime *sdw_stream = snd_soc_dai_get_dma_data(dai, substream);
+	unsigned char ps3 = 0x3;
+	int ret;
+
+	if (!es9356->slave)
+		return -EINVAL;
+
+	sdw_stream_remove_slave(es9356->slave, sdw_stream);
+
+	ret = es9356_power_state(dai, ps3, NULL);
+	if (ret) {
+		dev_err(component->dev, "%s: Invalid dai id: %d\n",
+			__func__, dai->id);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dai_ops es9356_sdw_ops = {
+	.hw_params	= es9356_sdw_pcm_hw_params,
+	.hw_free	= es9356_sdw_pcm_hw_free,
+	.set_stream	= es9356_sdw_set_sdw_stream,
+	.shutdown	= es9356_sdw_shutdown,
+};
+
+static struct snd_soc_dai_driver es9356_sdw_dai[] = {
+	{
+		.name = "es9356-sdp-aif4",
+		.id = ES9356_DMIC,
+		.capture = {
+			.stream_name = "DP1 Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+		},
+		.ops = &es9356_sdw_ops,
+	},
+	{
+		.name = "es9356-sdp-aif2",
+		.id = ES9356_JACK_IN,
+		.capture = {
+			.stream_name = "DP2 Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+		},
+		.ops = &es9356_sdw_ops,
+	},
+	{
+		.name = "es9356-sdp-aif3",
+		.id = ES9356_AMP,
+		.playback = {
+			.stream_name = "DP3 Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+		},
+		.ops = &es9356_sdw_ops,
+	},
+	{
+		.name = "es9356-sdp-aif1",
+		.id = ES9356_JACK_OUT,
+		.playback = {
+			.stream_name = "DP4 Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+		},
+		.ops = &es9356_sdw_ops,
+	},
+};
+
+static int es9356_sdca_mbq_size(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU113, ES9356_SDCA_CTL_FU_VOLUME, CH_L):
+	case SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU113, ES9356_SDCA_CTL_FU_VOLUME, CH_R):
+	case SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_FU21, ES9356_SDCA_CTL_FU_VOLUME, CH_L):
+	case SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_FU21, ES9356_SDCA_CTL_FU_VOLUME, CH_R):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU41, ES9356_SDCA_CTL_FU_VOLUME, CH_L):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU41, ES9356_SDCA_CTL_FU_VOLUME, CH_R):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU36, ES9356_SDCA_CTL_FU_VOLUME, CH_L):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU36, ES9356_SDCA_CTL_FU_VOLUME, CH_R):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU33, ES9356_SDCA_CTL_FU_CH_GAIN, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU11, ES9356_SDCA_CTL_FU_CH_GAIN, 0):
+		return 2;
+	default:
+		return 1;
+	}
+}
+
+static struct regmap_sdw_mbq_cfg es9356_mbq_config = {
+	.mbq_size = es9356_sdca_mbq_size,
+};
+
+static bool es9356_sdca_volatile_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case ES9356_BUF_ADDR_HID:
+	case ES9356_HID_BYTE2:
+	case ES9356_HID_BYTE3:
+	case ES9356_HID_BYTE4:
+	case SDW_SDCA_CTL(FUNC_NUM_HID, ES9356_SDCA_ENT_HID01, ES9356_SDCA_CTL_HIDTX_CURRENT_OWNER, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_GE35, ES9356_SDCA_CTL_DETECTED_MODE, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_GE35, ES9356_SDCA_CTL_SELECTED_MODE, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_PDE23, ES9356_SDCA_CTL_REQ_POWER_STATE, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_PDE23, ES9356_SDCA_CTL_ACTUAL_POWER_STATE, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_PDE11, ES9356_SDCA_CTL_REQ_POWER_STATE, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_PDE11, ES9356_SDCA_CTL_ACTUAL_POWER_STATE, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_PDE47, ES9356_SDCA_CTL_REQ_POWER_STATE, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_PDE47, ES9356_SDCA_CTL_ACTUAL_POWER_STATE, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_PDE34, ES9356_SDCA_CTL_REQ_POWER_STATE, 0):
+	case SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_PDE34, ES9356_SDCA_CTL_ACTUAL_POWER_STATE, 0):
+	case ES9356_FLAGS_HP:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_config es9356_sdca_regmap = {
+	.reg_bits = 32,
+	.val_bits = 16,
+	.volatile_reg = es9356_sdca_volatile_register,
+	.max_register = 0x45ffffff,
+	.cache_type = REGCACHE_MAPLE,
+	.use_single_read = true,
+	.use_single_write = true,
+};
+
+static void es9356_register_init(struct es9356_sdw_priv *es9356)
+{
+	regmap_write(es9356->regmap, ES9356_STATE, 0x02);
+	regmap_write(es9356->regmap, ES9356_ENDPOINT_MODE, 0x24);
+	regmap_write(es9356->regmap, ES9356_PRE_DIV_CTL, 0x00);
+	regmap_write(es9356->regmap, ES9356_ADC_OSR, 0x18);
+	regmap_write(es9356->regmap, ES9356_ADC_OSRGAIN, 0x13);
+	regmap_write(es9356->regmap, ES9356_DAC_OSR, 0x16);
+	regmap_write(es9356->regmap, ES9356_CLK_CTL, 0x0f);
+	regmap_write(es9356->regmap, ES9356_CSM_RESET, 0x01);
+	regmap_write(es9356->regmap, ES9356_CLK_SEL, 0x30);
+
+	regmap_write(es9356->regmap, ES9356_DETCLK_CTL, 0x51);
+	regmap_write(es9356->regmap, ES9356_HP_TYPE, 0x10);
+	regmap_write(es9356->regmap, ES9356_MICBIAS_CTL, 0x10);
+	regmap_write(es9356->regmap, ES9356_HPDETECT_CTL, 0x07);
+	regmap_write(es9356->regmap, ES9356_ADC_ANA, 0x30);
+	regmap_write(es9356->regmap, ES9356_PGA_CTL, 0xa8);
+	regmap_write(es9356->regmap, ES9356_ADC_INT, 0xaa);
+	regmap_write(es9356->regmap, ES9356_ADC_LP, 0x19);
+	regmap_write(es9356->regmap, ES9356_VMID1SEL, 0xbc);
+	regmap_write(es9356->regmap, ES9356_VMID_TIME, 0x0b);
+	regmap_write(es9356->regmap, ES9356_STATE_TIME, 0xbb);
+	regmap_write(es9356->regmap, ES9356_HP_SPK_TIME, 0x77);
+	regmap_write(es9356->regmap, ES9356_HP_DETECTTIME, 0xa4);
+	regmap_write(es9356->regmap, ES9356_MICBIAS_SEL, 0x15);
+	regmap_write(es9356->regmap, ES9356_KEY_PRESS_TIME, 0xff);
+	regmap_write(es9356->regmap, ES9356_KEY_RELEASE_TIME, 0xff);
+	regmap_write(es9356->regmap, ES9356_KEY_HOLD_TIME, 0x0f);
+	regmap_write(es9356->regmap, ES9356_BTSEL_REF, 0x00);
+	regmap_write(es9356->regmap, ES9356_KEYD_DETECT, 0x18);
+	regmap_write(es9356->regmap, ES9356_MICBIAS_RES, 0x03);
+	regmap_write(es9356->regmap, ES9356_BUTTON_CHARGE, 0x00);
+	regmap_write(es9356->regmap, ES9356_CALIBRATION_TIME, 0x13);
+	regmap_write(es9356->regmap, ES9356_CALIBRATION_SETTING, 0xf4);
+
+	regmap_write(es9356->regmap, ES9356_SPK_VOLUME, 0x33);
+	regmap_write(es9356->regmap, ES9356_DAC_VROI, 0x01);
+	regmap_write(es9356->regmap, ES9356_DAC_LP, 0x00);
+	regmap_write(es9356->regmap, ES9356_HP_IBIAS, 0x04);
+	regmap_write(es9356->regmap, ES9356_HP_LP, 0x03);
+	regmap_write(es9356->regmap, ES9356_SPKLDO_CTL, 0x65);
+	regmap_write(es9356->regmap, ES9356_SPKBIAS_COMP, 0x09);
+	regmap_write(es9356->regmap, ES9356_VMID1STL, 0x00);
+	regmap_write(es9356->regmap, ES9356_VMID2STL, 0x00);
+	regmap_write(es9356->regmap, ES9356_VSEL, 0xfc);
+
+	regmap_write(es9356->regmap, ES9356_IBIASGEN, 0x10);
+	regmap_write(es9356->regmap, ES9356_ADC_AMIC_CTL, 0x0d);
+	regmap_write(es9356->regmap, ES9356_STATE, 0x0e);
+	regmap_write(es9356->regmap, ES9356_CSM_RESET, 0x00);
+	regmap_write(es9356->regmap, ES9356_HP_TYPE, 0x08);
+
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU113, ES9356_SDCA_CTL_FU_VOLUME, CH_L), ES9356_DEFAULT_VOLUME);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_FU113, ES9356_SDCA_CTL_FU_VOLUME, CH_R), ES9356_DEFAULT_VOLUME);
+
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_FU21, ES9356_SDCA_CTL_FU_VOLUME, CH_L), ES9356_DEFAULT_VOLUME);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_FU21, ES9356_SDCA_CTL_FU_VOLUME, CH_R), ES9356_DEFAULT_VOLUME);
+
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU41, ES9356_SDCA_CTL_FU_VOLUME, CH_L), ES9356_DEFAULT_VOLUME);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU41, ES9356_SDCA_CTL_FU_VOLUME, CH_R), ES9356_DEFAULT_VOLUME);
+
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU36, ES9356_SDCA_CTL_FU_VOLUME, CH_L), ES9356_DEFAULT_VOLUME);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_FU36, ES9356_SDCA_CTL_FU_VOLUME, CH_R), ES9356_DEFAULT_VOLUME);
+}
+
+static int es9356_sdca_io_init(struct device *dev, struct sdw_slave *slave)
+{
+	struct es9356_sdw_priv *es9356 = dev_get_drvdata(&slave->dev);
+
+	if (es9356->hw_init)
+		return 0;
+
+	es9356->disable_irq = false;
+
+	regcache_cache_only(es9356->regmap, false);
+
+	if (es9356->first_hw_init) {
+		regcache_cache_bypass(es9356->regmap, true);
+	} else {
+		/* update count of parent 'active' children */
+		pm_runtime_set_active(&slave->dev);
+
+		es9356_register_init(es9356);
+	}
+	pm_runtime_get_noresume(&slave->dev);
+
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT_XU12, ES9356_SDCA_CTL_SELECTED_MODE, 0), 0x01);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_MIC, ES9356_SDCA_ENT0, ES9356_SDCA_CTL_FUNC_STATUS, 0), 0x40);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT_XU22, ES9356_SDCA_CTL_SELECTED_MODE, 0), 0x01);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_AMP, ES9356_SDCA_ENT0, ES9356_SDCA_CTL_FUNC_STATUS, 0), 0x40);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_XU42, ES9356_SDCA_CTL_SELECTED_MODE, 0), 0x01);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT_XU36, ES9356_SDCA_CTL_SELECTED_MODE, 0), 0x01);
+	regmap_write(es9356->regmap,
+		SDW_SDCA_CTL(FUNC_NUM_UAJ, ES9356_SDCA_ENT0, ES9356_SDCA_CTL_FUNC_STATUS, 0), 0x40);
+
+	if (es9356->first_hw_init) {
+		regcache_cache_bypass(es9356->regmap, false);
+		regcache_mark_dirty(es9356->regmap);
+	} else
+		es9356->first_hw_init = true;
+
+	es9356->hw_init = true;
+
+	pm_runtime_mark_last_busy(&slave->dev);
+	pm_runtime_put_autosuspend(&slave->dev);
+
+	return 0;
+}
+
+static int es9356_sdw_update_status(struct sdw_slave *slave,
+				    enum sdw_slave_status status)
+{
+	struct es9356_sdw_priv *es9356 = dev_get_drvdata(&slave->dev);
+
+	if (status == SDW_SLAVE_UNATTACHED) {
+		es9356->hw_init = false;
+		cancel_delayed_work_sync(&es9356->interrupt_handle_work);
+		cancel_delayed_work_sync(&es9356->button_detect_work);
+		regcache_cache_only(es9356->regmap, true);
+	}
+
+	if (status == SDW_SLAVE_ATTACHED) {
+		if (es9356->hs_jack)
+			sdw_write_no_pm(es9356->slave, SDW_SCP_SDCA_INTMASK1,
+			(SDW_SCP_SDCA_INTMASK_SDCA_7 | SDW_SCP_SDCA_INTMASK_SDCA_5 | SDW_SCP_SDCA_INTMASK_SDCA_1));
+	}
+
+	if (es9356->hw_init || status != SDW_SLAVE_ATTACHED)
+		return 0;
+
+	return es9356_sdca_io_init(&slave->dev, slave);
+}
+
+static int es9356_sdw_read_prop(struct sdw_slave *slave)
+{
+	struct sdw_slave_prop *prop = &slave->prop;
+	int nval;
+	int i, j;
+	u32 bit;
+	unsigned long addr;
+	struct sdw_dpn_prop *dpn;
+
+	prop->paging_support = true;
+
+	/*
+	 * first we need to allocate memory for set bits in port lists
+	 * the port allocation is completely arbitrary:
+	 * DP0 is not supported
+	 * DP3 and DP4 is sink
+	 * DP1 and DP2 is source
+	 */
+	prop->source_ports = BIT(1) | BIT(2);
+	prop->sink_ports = BIT(3) | BIT(4);
+
+	nval = hweight32(prop->source_ports);
+	prop->src_dpn_prop = devm_kcalloc(&slave->dev, nval,
+					  sizeof(*prop->src_dpn_prop),
+					  GFP_KERNEL);
+	if (!prop->src_dpn_prop)
+		return -ENOMEM;
+
+	i = 0;
+	dpn = prop->src_dpn_prop;
+	addr = prop->source_ports;
+	for_each_set_bit(bit, &addr, 32) {
+		dpn[i].num = bit;
+		dpn[i].type = SDW_DPN_FULL;
+		dpn[i].simple_ch_prep_sm = true;
+		i++;
+	}
+
+	/* do this again for sink now */
+	nval = hweight32(prop->sink_ports);
+	prop->sink_dpn_prop = devm_kcalloc(&slave->dev, nval,
+					   sizeof(*prop->sink_dpn_prop),
+					   GFP_KERNEL);
+	if (!prop->sink_dpn_prop)
+		return -ENOMEM;
+
+	j = 0;
+	dpn = prop->sink_dpn_prop;
+	addr = prop->sink_ports;
+	for_each_set_bit(bit, &addr, 32) {
+		dpn[j].num = bit;
+		dpn[j].type = SDW_DPN_FULL;
+		dpn[j].simple_ch_prep_sm = true;
+		j++;
+	}
+
+	/* wake-up event */
+	prop->wake_capable = 1;
+
+	return 0;
+}
+
+static int es9356_sdw_interrupt_callback(struct sdw_slave *slave,
+					 struct sdw_slave_intr_status *status)
+{
+	struct es9356_sdw_priv *es9356 = dev_get_drvdata(&slave->dev);
+	unsigned int sdca_cascade, scp_sdca_stat1 = 0;
+	int count = 0, retry = 3;
+	int ret, stat, reg;
+
+	mutex_lock(&es9356->disable_irq_lock);
+
+	ret = sdw_read_no_pm(es9356->slave, SDW_SCP_SDCA_INT1);
+	if (ret < 0)
+		goto io_error;
+	es9356->sdca_status = ret;
+
+	do {
+		reg = sdw_read_no_pm(es9356->slave, SDW_SCP_SDCA_INT1);
+		if (reg < 0)
+			goto io_error;
+		if (reg & SDW_SCP_SDCA_INTMASK_SDCA_1) {
+			ret = sdw_update_no_pm(es9356->slave, SDW_SCP_SDCA_INT1,
+				SDW_SCP_SDCA_INT_SDCA_1, SDW_SCP_SDCA_INT_SDCA_1);
+			if (ret < 0)
+				goto io_error;
+		}
+
+		if (reg & SDW_SCP_SDCA_INTMASK_SDCA_5) {
+			ret = sdw_update_no_pm(es9356->slave, SDW_SCP_SDCA_INT1,
+				SDW_SCP_SDCA_INT_SDCA_5, SDW_SCP_SDCA_INT_SDCA_5);
+			if (ret < 0)
+				goto io_error;
+		}
+
+		if (reg & SDW_SCP_SDCA_INTMASK_SDCA_7) {
+			ret = sdw_update_no_pm(es9356->slave, SDW_SCP_SDCA_INT1,
+				SDW_SCP_SDCA_INT_SDCA_7, SDW_SCP_SDCA_INT_SDCA_7);
+			if (ret < 0)
+				goto io_error;
+		}
+
+		ret = sdw_read_no_pm(es9356->slave, SDW_DP0_INT);
+		if (ret < 0)
+			goto io_error;
+		sdca_cascade = ret & SDW_DP0_SDCA_CASCADE;
+
+		ret = sdw_read_no_pm(es9356->slave, SDW_SCP_SDCA_INT1);
+		if (ret < 0)
+			goto io_error;
+		scp_sdca_stat1 = ret &
+			(SDW_SCP_SDCA_INTMASK_SDCA_1 | SDW_SCP_SDCA_INTMASK_SDCA_5 | SDW_SCP_SDCA_INTMASK_SDCA_7);
+
+		stat = scp_sdca_stat1 || sdca_cascade;
+
+		count++;
+	} while (stat != 0 && count < retry);
+
+	/* The 280 ms figure was determined through testing */
+	if (status->sdca_cascade && !es9356->disable_irq)
+		mod_delayed_work(system_power_efficient_wq,
+			&es9356->interrupt_handle_work, msecs_to_jiffies(280));
+
+	mutex_unlock(&es9356->disable_irq_lock);
+	return 0;
+
+io_error:
+	mutex_unlock(&es9356->disable_irq_lock);
+	pr_err_ratelimited("IO error in %s, ret %d\n", __func__, ret);
+	return ret;
+}
+
+static const struct sdw_slave_ops es9356_sdw_slave_ops = {
+	.read_prop = es9356_sdw_read_prop,
+	.interrupt_callback = es9356_sdw_interrupt_callback,
+	.update_status = es9356_sdw_update_status,
+};
+
+static int es9356_sdca_init(struct device *dev, struct regmap *regmap, struct sdw_slave *slave)
+{
+	struct es9356_sdw_priv *es9356;
+	int ret;
+
+	es9356 = devm_kzalloc(dev, sizeof(*es9356), GFP_KERNEL);
+	if (!es9356)
+		return -ENOMEM;
+
+	dev_set_drvdata(dev, es9356);
+
+	es9356->slave = slave;
+	es9356->regmap = regmap;
+	mutex_init(&es9356->disable_irq_lock);
+	mutex_init(&es9356->pde_lock);
+
+	regcache_cache_only(es9356->regmap, true);
+
+	es9356->hw_init = false;
+	es9356->first_hw_init = false;
+
+	INIT_DELAYED_WORK(&es9356->interrupt_handle_work,
+			  es9356_interrupt_handler);
+	INIT_DELAYED_WORK(&es9356->button_detect_work,
+			  es9356_button_detect_handler);
+
+	ret = devm_snd_soc_register_component(dev,
+					       &snd_soc_es9356_sdw_component,
+					       es9356_sdw_dai,
+					       ARRAY_SIZE(es9356_sdw_dai));
+	if (ret) {
+		dev_err_probe(dev, ret, "Failed to register component\n");
+		return ret;
+	}
+	/* set autosuspend parameters */
+	pm_runtime_set_autosuspend_delay(dev, 3000);
+	pm_runtime_use_autosuspend(dev);
+	/* make sure the device does not suspend immediately */
+	pm_runtime_mark_last_busy(dev);
+	pm_runtime_enable(dev);
+
+	return 0;
+}
+
+static int es9356_sdw_probe(struct sdw_slave *slave,
+				const struct sdw_device_id *id)
+{
+	struct regmap *regmap;
+
+	regmap = devm_regmap_init_sdw_mbq_cfg(&slave->dev, slave, &es9356_sdca_regmap, &es9356_mbq_config);
+	if (IS_ERR(regmap))
+		return PTR_ERR(regmap);
+
+	return es9356_sdca_init(&slave->dev, regmap, slave);
+}
+
+static void es9356_sdw_remove(struct sdw_slave *slave)
+{
+	struct es9356_sdw_priv *es9356 = dev_get_drvdata(&slave->dev);
+
+	if (es9356->hw_init) {
+		cancel_delayed_work_sync(&es9356->interrupt_handle_work);
+		cancel_delayed_work_sync(&es9356->button_detect_work);
+	}
+
+	if (es9356->first_hw_init)
+		pm_runtime_disable(&slave->dev);
+
+	mutex_destroy(&es9356->disable_irq_lock);
+	mutex_destroy(&es9356->pde_lock);
+}
+
+static const struct sdw_device_id es9356_sdw_id[] = {
+	SDW_SLAVE_ENTRY_EXT(0x04b3, 0x9356, 0x02, 0, 0),
+	SDW_SLAVE_ENTRY_EXT(0x04b3, 0x9356, 0x03, 0, 0),
+	{},
+};
+MODULE_DEVICE_TABLE(sdw, es9356_sdw_id);
+
+static int es9356_sdca_dev_suspend(struct device *dev)
+{
+	struct es9356_sdw_priv *es9356 = dev_get_drvdata(dev);
+
+	cancel_delayed_work_sync(&es9356->interrupt_handle_work);
+	cancel_delayed_work_sync(&es9356->button_detect_work);
+
+	regcache_cache_only(es9356->regmap, true);
+
+	return 0;
+}
+
+static int es9356_sdca_dev_system_suspend(struct device *dev)
+{
+	struct es9356_sdw_priv *es9356 = dev_get_drvdata(dev);
+
+	mutex_lock(&es9356->disable_irq_lock);
+	es9356->disable_irq = true;
+	mutex_unlock(&es9356->disable_irq_lock);
+
+	return es9356_sdca_dev_suspend(dev);
+}
+
+#define es9356_PROBE_TIMEOUT 2000
+
+static int es9356_sdca_dev_resume(struct device *dev)
+{
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);
+	struct es9356_sdw_priv *es9356 = dev_get_drvdata(dev);
+	unsigned long time;
+
+	if (!slave->unattach_request) {
+		es9356->disable_irq = false;
+		goto regmap_sync;
+	}
+
+	time = wait_for_completion_timeout(&slave->initialization_complete,
+				msecs_to_jiffies(es9356_PROBE_TIMEOUT));
+	if (!time) {
+		dev_err(&slave->dev, "Initialization not complete, timed out\n");
+		sdw_show_ping_status(slave->bus, true);
+
+		return -ETIMEDOUT;
+	}
+
+regmap_sync:
+	slave->unattach_request = 0;
+	regcache_cache_only(es9356->regmap, false);
+	regcache_sync(es9356->regmap);
+	return 0;
+}
+
+static const struct dev_pm_ops es9356_sdca_pm = {
+	SYSTEM_SLEEP_PM_OPS(es9356_sdca_dev_system_suspend, es9356_sdca_dev_resume)
+	RUNTIME_PM_OPS(es9356_sdca_dev_suspend, es9356_sdca_dev_resume, NULL)
+};
+
+static struct sdw_driver es9356_sdw_driver = {
+	.driver = {
+		.name = "es9356",
+		.pm = pm_ptr(&es9356_sdca_pm),
+	},
+	.probe = es9356_sdw_probe,
+	.remove = es9356_sdw_remove,
+	.ops = &es9356_sdw_slave_ops,
+	.id_table = es9356_sdw_id,
+};
+module_sdw_driver(es9356_sdw_driver);
+
+MODULE_IMPORT_NS("SND_SOC_SDCA");
+MODULE_DESCRIPTION("ASoC ES9356 SDCA SDW codec driver");
+MODULE_AUTHOR("Michael Zhang <zhangyi@everest-semi.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/es9356.h b/sound/soc/codecs/es9356.h
new file mode 100644
index 0000000000000..2a676f36590f6
--- /dev/null
+++ b/sound/soc/codecs/es9356.h
@@ -0,0 +1,208 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+#ifndef __ES9356_H__
+#define __ES9356_H__
+
+/*ES9356 Implementation-define*/
+#define ES9356_FLAGS_HP                         0x2003
+#define ES9356_CSM_RESET                        0x2020
+#define ES9356_FUC_RESET                        0x2021
+#define ES9356_STATE                            0x2022
+#define ES9356_VMID_TIME                        0x2023
+#define ES9356_STATE_TIME                       0x2024
+#define ES9356_HP_SPK_TIME                      0x2025
+#define ES9356_WP_ENABLE                        0x2026
+#define ES9356_DMIC_GPIO                        0x2027
+#define ES9356_ENDPOINT_MODE                    0x2028
+
+/*HP DETECT*/
+#define ES9356_HP_TYPE                          0x2029
+#define ES9356_HP_DETECTTIME                    0x202A
+#define ES9356_MICBIAS_SEL                      0x202B
+#define ES9356_KEY_PRESS_TIME                   0x202C
+#define ES9356_KEY_RELEASE_TIME                 0x202D
+#define ES9356_KEY_HOLD_TIME                    0x202E
+#define ES9356_BTSEL_REF                        0x202F
+#define ES9356_BUTTON_CHARGE                    0x2030
+
+#define ES9356_KEYD_DETECT                      0x2031
+#define ES9356_DPEN_TIME                        0x2032
+#define ES9356_TIMER_CHECK                      0x2033
+#define ES9356_IBIASGEN                         0x2041
+#define ES9356_VMID1SEL                         0x2042
+#define ES9356_VMID1STL                         0x2043
+#define ES9356_VMID2SEL                         0x2044
+#define ES9356_VMID2STL                         0x2045
+#define ES9356_VSEL                             0x2046
+#define ES9356_MICBIAS_CTL                      0x2047
+#define ES9356_HPDETECT_CTL                     0x2048
+#define ES9356_MICBIAS_RES                      0x2049
+
+/*CLK*/
+#define ES9356_CLK_SEL                          0x2050
+#define ES9356_CLK_CTL                          0x2051
+#define ES9356_DETCLK_CTL                       0x2052
+#define ES9356_CPCLK_CTL                        0x2053
+#define ES9356_SPKCLK_CTL                       0x2054
+#define ES9356_PRE_DIV_CTL                      0x2055
+#define ES9356_DLL_MODE                         0x2056
+#define ES9356_ANACLK_SEL                       0x2057
+#define ES9356_OSRCLK_SEL                       0x2058
+#define ES9356_DSPCLK_SEL                       0x2059
+#define ES9356_SPK9M_MODE                       0x205a
+
+/*ADC DIG CTL*/
+#define ES9356_DMIC_POL                         0x2061
+#define ES9356_ADC_SWAP                         0x2062
+#define ES9356_ADC_OSR                          0x2063
+#define ES9356_ADC_OSRGAIN                      0x2064
+#define ES9356_ADC_CLEARRAM                     0x2065
+#define ES9356_ADC_RAMP                         0x2066
+#define ES9356_ADC_HPF1                         0x2067
+#define ES9356_ADC_HPF2                         0x2068
+#define ES9356_ADC_ALC                          0x206C
+#define ES9356_ALC_LEVEL                        0x206D
+#define ES9356_ALC_RAMP_WINSIZE                 0x206E
+
+/*ADC ANA CTL*/
+#define ES9356_ADC_REF_EN                       0x2080
+#define ES9356_ADC_AMIC_CTL                     0x2081
+#define ES9356_ADC_ANA                          0x2082
+#define ES9356_PGA_CTL                          0x2083
+#define ES9356_ADC_INT                          0x2084
+#define ES9356_ADC_VCM                          0x2085
+#define ES9356_ADC_VRPBIAS                      0x2086
+#define ES9356_ADC_LP                           0x2087
+
+/*DAC DIG CTL*/
+#define ES9356_DAC_FSMODE                       0x2090
+#define ES9356_DAC_OSR                          0x2091
+#define ES9356_DAC_INV                          0x2092
+#define ES9356_DAC_RAMP                         0x2093
+#define ES9356_DAC_VPPSCALE                     0x2094
+#define ES9356_DAC_SWAP                         0x2097
+#define ES9356_SPKCMP_VPPSC                     0x20A0
+#define ES9356_CALIBRATION_TIME                 0x20A1
+#define ES9356_CALIBRATION_SETTING              0x20A2
+#define ES9356_DAC_OFFSET_LH                    0x20A3
+#define ES9356_DAC_OFFSET_LL                    0x20A4
+#define ES9356_DAC_OFFSET_RH                    0x20A5
+#define ES9356_DAC_OFFSET_RL                    0x20A6
+
+/*DAC ANA CTL*/
+#define ES9356_DAC_REF_EN                       0x20B0
+#define ES9356_DAC_ENABLE                       0x20B1
+#define ES9356_DAC_VROI                         0x20B2
+#define ES9356_DAC_LP                           0x20B3
+
+/*HP CTL*/
+#define ES9356_CHARGEPUMP_CTL                   0x20C0
+#define ES9356_CPLDO_CTL                        0x20C1
+#define ES9356_HP_REF_CTL                       0x20C2
+#define ES9356_HP_IBIAS                         0x20C3
+#define ES9356_HP_EN                            0x20C4
+#define ES9356_HP_VOLUME                        0x20C5
+#define ES9356_HP_LP                            0x20C6
+
+/*SPK CTL*/
+#define ES9356_SPKLDO_CTL                       0x20D0
+#define ES9356_CLASSD_CTL                       0x20D1
+#define ES9356_SPK_HBDG                         0x20D5
+#define ES9356_SPK_VOLUME                       0x20D7
+#define ES9356_SPK_SCP                          0x20D8
+#define ES9356_SPK_DT                           0x20D9
+#define ES9356_SPK_OTP                          0x20DA
+#define ES9356_SPKBIAS_COMP                     0x20DB
+
+/* ES9356 SDCA Control - function number */
+#define FUNC_NUM_UAJ                            0x01
+#define FUNC_NUM_MIC                            0x02
+#define FUNC_NUM_AMP                            0x03
+#define FUNC_NUM_HID                            0x04
+
+/* ES9356 SDCA entity */
+#define ES9356_SDCA_ENT0                        0x00
+#define ES9356_SDCA_ENT_PDE11                   0x03
+#define ES9356_SDCA_ENT_FU11                    0x04
+#define ES9356_SDCA_ENT_XU12                    0x05
+#define ES9356_SDCA_ENT_FU113                   0x07
+#define ES9356_SDCA_ENT_CS113                   0x09
+#define ES9356_SDCA_ENT_PPU11                   0x0C
+
+#define ES9356_SDCA_ENT_CS21                    0x02
+#define ES9356_SDCA_ENT_PPU21                   0x03
+#define ES9356_SDCA_ENT_FU21                    0X04
+#define ES9356_SDCA_ENT_XU22                    0x06
+#define ES9356_SDCA_ENT_SAPU29                  0x03
+#define ES9356_SDCA_ENT_PDE23                   0x0B
+#define ES9356_SDCA_ENT_HID01                   0x01
+
+#define ES9356_SDCA_ENT_CS41                    0x02
+#define ES9356_SDCA_ENT_FU35                    0x04
+#define ES9356_SDCA_ENT_XU42                    0x06
+#define ES9356_SDCA_ENT_FU41                    0x07
+#define ES9356_SDCA_ENT_PDE47                   0x0E
+#define ES9356_SDCA_ENT_IT33                    0x0F
+#define ES9356_SDCA_ENT_PDE34                   0x10
+#define ES9356_SDCA_ENT_FU33                    0x11
+#define ES9356_SDCA_ENT_XU36                    0x13
+#define ES9356_SDCA_ENT_FU36                    0x15
+#define ES9356_SDCA_ENT_CS36                    0x17
+#define ES9356_SDCA_ENT_GE35                    0x18
+
+/* ES9356 SDCA control */
+#define ES9356_SDCA_CTL_SAMPLE_FREQ_INDEX       0x10
+#define ES9356_SDCA_CTL_FU_MUTE                 0x01
+#define ES9356_SDCA_CTL_FU_VOLUME               0x02
+#define ES9356_SDCA_CTL_HIDTX_CURRENT_OWNER     0x10
+#define ES9356_SDCA_CTL_SELECTED_MODE           0x01
+#define ES9356_SDCA_CTL_DETECTED_MODE           0x02
+#define ES9356_SDCA_CTL_REQ_POWER_STATE         0x01
+#define ES9356_SDCA_CTL_FU_CH_GAIN              0x0b
+#define ES9356_SDCA_CTL_FUNC_STATUS             0x10
+#define ES9356_SDCA_CTL_ACTUAL_POWER_STATE      0x10
+#define ES9356_SDCA_CTL_POSTURE_NUMBER          0x00
+
+/* ES9356 SDCA channel */
+#define CH_L	0x01
+#define CH_R	0x02
+#define MBQ	0x2000
+
+/* ES9356 HID*/
+#define ES9356_BUF_ADDR_HID     0x44000000
+#define ES9356_HID_BYTE2        0x44000001
+#define ES9356_HID_BYTE3        0x44000002
+#define ES9356_HID_BYTE4        0x44000003
+
+/* ES9356 Volume Setting*/
+#define ES9356_VU_BASE          768
+#define ES9356_OFFSET_HIGH      0x07F8
+#define ES9356_OFFSET_LOW       0x0007
+#define ES9356_DEFAULT_VOLUME   0x00
+#define	ES9356_VOLUME_STEP      32
+#define ES9356_VOLUME_MIN       -768
+#define	ES9356_VOLUME_MAX       285
+#define	ES9356_AMIC_GAIN_STEP   768
+#define	ES9356_DMIC_GAIN_STEP   1536
+#define ES9356_GAIN_MIN         0
+#define	ES9356_AMIC_GAIN_MAX    10
+#define	ES9356_DMIC_GAIN_MAX    3
+
+enum {
+	ES9356_DMIC = 1, /* For dmic */
+	ES9356_JACK_IN, /* For headset mic */
+	ES9356_AMP, /* For speaker */
+	ES9356_JACK_OUT, /* For headphone */
+};
+
+enum {
+	ES9356_SDCA_RATE_16000HZ,
+	ES9356_SDCA_RATE_24000HZ,
+	ES9356_SDCA_RATE_32000HZ,
+	ES9356_SDCA_RATE_44100HZ,
+	ES9356_SDCA_RATE_48000HZ,
+	ES9356_SDCA_RATE_88200HZ,
+	ES9356_SDCA_RATE_96000HZ,
+};
+
+#endif
diff --git a/sound/soc/codecs/fs210x.c b/sound/soc/codecs/fs210x.c
index e6195b71adadc..eda716f817b58 100644
--- a/sound/soc/codecs/fs210x.c
+++ b/sound/soc/codecs/fs210x.c
@@ -968,7 +968,7 @@ static int fs210x_effect_scene_info(struct snd_kcontrol *kcontrol,
 	if (scene->name)
 		name = scene->name;
 
-	strscpy(uinfo->value.enumerated.name, name, strlen(name) + 1);
+	strscpy(uinfo->value.enumerated.name, name);
 
 	return 0;
 }
diff --git a/sound/soc/codecs/max98363.c b/sound/soc/codecs/max98363.c
index 25af78ab30d5c..099dc5bf6195f 100644
--- a/sound/soc/codecs/max98363.c
+++ b/sound/soc/codecs/max98363.c
@@ -90,24 +90,15 @@ static int max98363_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct max98363_priv *max98363 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!max98363->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-					   msecs_to_jiffies(MAX98363_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(dev, "Initialization not complete, timed out\n");
-		return -ETIMEDOUT;
-	}
-
-regmap_sync:
+	ret = sdw_slave_wait_for_init(slave, MAX98363_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-	slave->unattach_request = 0;
 	regcache_cache_only(max98363->regmap, false);
 	regcache_sync(max98363->regmap);
 
diff --git a/sound/soc/codecs/max98373-sdw.c b/sound/soc/codecs/max98373-sdw.c
index 16673440218cb..6829fa07c9ecb 100644
--- a/sound/soc/codecs/max98373-sdw.c
+++ b/sound/soc/codecs/max98373-sdw.c
@@ -266,25 +266,15 @@ static int max98373_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct max98373_priv *max98373 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!max98373->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-					   msecs_to_jiffies(MAX98373_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(dev, "Initialization not complete, timed out\n");
-		sdw_show_ping_status(slave->bus, true);
-
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, MAX98373_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(max98373->regmap, false);
 	regcache_sync(max98373->regmap);
 
diff --git a/sound/soc/codecs/nau8825.c b/sound/soc/codecs/nau8825.c
index dd3528537ae42..80964d58bb87d 100644
--- a/sound/soc/codecs/nau8825.c
+++ b/sound/soc/codecs/nau8825.c
@@ -712,8 +712,8 @@ static void nau8825_xtalk_measure(struct nau8825 *nau8825)
 		/* In left headphone IMM state, read out left headphone
 		 * impedance measure result, and delay some time to wait
 		 * detection sine wave output finish. Then, we can calculate
-		 * the cross talk suppresstion side tone according to the L/R
-		 * headphone imedance.
+		 * the cross talk suppression side tone according to the L/R
+		 * headphone impedance.
 		 */
 		regmap_read(nau8825->regmap, NAU8825_REG_IMM_RMS_L,
 			&nau8825->imp_rms[NAU8825_XTALK_HPL_R2L]);
@@ -2237,7 +2237,7 @@ static void nau8825_component_remove(struct snd_soc_component *component)
 {
 	struct nau8825 *nau8825 = snd_soc_component_get_drvdata(component);
 
-	/* Cancel and reset cross tak suppresstion detection funciton */
+	/* Cancel and reset cross talk suppression detection function */
 	nau8825_xtalk_cancel(nau8825);
 }
 
@@ -2651,7 +2651,7 @@ static int nau8825_set_bias_level(struct snd_soc_component *component,
 		/* ground HPL/HPR, MICGRND1/2 */
 		regmap_update_bits(nau8825->regmap,
 			NAU8825_REG_HSD_CTRL, 0xf, 0xf);
-		/* Cancel and reset cross talk detection funciton */
+		/* Cancel and reset cross talk detection function */
 		nau8825_xtalk_cancel(nau8825);
 		/* Turn off all interruptions before system shutdown. Keep the
 		 * interruption quiet before resume setup completes.
diff --git a/sound/soc/codecs/pcm6240.c b/sound/soc/codecs/pcm6240.c
index 78b21fbfad50e..667596fc1781b 100644
--- a/sound/soc/codecs/pcm6240.c
+++ b/sound/soc/codecs/pcm6240.c
@@ -1230,15 +1230,11 @@ static struct pcmdevice_config_info *pcmdevice_add_config(void *ctxt,
 	int *status)
 {
 	struct pcmdevice_priv *pcm_dev = (struct pcmdevice_priv *)ctxt;
-	struct pcmdevice_config_info *cfg_info;
+	struct pcmdevice_config_info *cfg_info = NULL;
 	struct pcmdevice_block_data **bk_da;
+	char cfg_name[64] = {};
 	unsigned int config_offset = 0, i;
-
-	cfg_info = kzalloc_obj(struct pcmdevice_config_info);
-	if (!cfg_info) {
-		*status = -ENOMEM;
-		goto out;
-	}
+	unsigned int nblocks;
 
 	if (pcm_dev->regbin.fw_hdr.binary_version_num >= 0x105) {
 		if (config_offset + 64 > (int)config_size) {
@@ -1247,7 +1243,7 @@ static struct pcmdevice_config_info *pcmdevice_add_config(void *ctxt,
 				"%s: cfg_name out of boundary\n", __func__);
 			goto out;
 		}
-		memcpy(cfg_info->cfg_name, &config_data[config_offset], 64);
+		memcpy(cfg_name, &config_data[config_offset], 64);
 		config_offset += 64;
 	}
 
@@ -1257,16 +1253,17 @@ static struct pcmdevice_config_info *pcmdevice_add_config(void *ctxt,
 			__func__);
 		goto out;
 	}
-	cfg_info->nblocks =
-		get_unaligned_be32(&config_data[config_offset]);
+	nblocks = get_unaligned_be32(&config_data[config_offset]);
 	config_offset += 4;
 
-	bk_da = cfg_info->blk_data = kzalloc_objs(struct pcmdevice_block_data *,
-						  cfg_info->nblocks);
-	if (!bk_da) {
+	cfg_info = kzalloc_flex(*cfg_info, blk_data, nblocks);
+	if (!cfg_info) {
 		*status = -ENOMEM;
 		goto out;
 	}
+	cfg_info->nblocks = nblocks;
+	memcpy(cfg_info->cfg_name, cfg_name, sizeof(cfg_info->cfg_name));
+	bk_da = cfg_info->blk_data;
 	cfg_info->real_nblocks = 0;
 	for (i = 0; i < cfg_info->nblocks; i++) {
 		if (config_offset + 12 > config_size) {
@@ -1449,14 +1446,11 @@ static void pcmdevice_config_info_remove(void *ctxt)
 	for (i = 0; i < regbin->ncfgs; i++) {
 		if (!cfg_info[i])
 			continue;
-		if (cfg_info[i]->blk_data) {
-			for (j = 0; j < (int)cfg_info[i]->real_nblocks; j++) {
-				if (!cfg_info[i]->blk_data[j])
-					continue;
-				kfree(cfg_info[i]->blk_data[j]->regdata);
-				kfree(cfg_info[i]->blk_data[j]);
-			}
-			kfree(cfg_info[i]->blk_data);
+		for (j = 0; j < (int)cfg_info[i]->real_nblocks; j++) {
+			if (!cfg_info[i]->blk_data[j])
+				continue;
+			kfree(cfg_info[i]->blk_data[j]->regdata);
+			kfree(cfg_info[i]->blk_data[j]);
 		}
 		kfree(cfg_info[i]);
 	}
diff --git a/sound/soc/codecs/pcm6240.h b/sound/soc/codecs/pcm6240.h
index 2d8f9e798139a..e27afd33910c4 100644
--- a/sound/soc/codecs/pcm6240.h
+++ b/sound/soc/codecs/pcm6240.h
@@ -199,7 +199,7 @@ struct pcmdevice_config_info {
 	unsigned int nblocks;
 	unsigned int real_nblocks;
 	unsigned char active_dev;
-	struct pcmdevice_block_data **blk_data;
+	struct pcmdevice_block_data *blk_data[] __counted_by(nblocks);
 };
 
 struct pcmdevice_regbin {
diff --git a/sound/soc/codecs/rt1017-sdca-sdw.c b/sound/soc/codecs/rt1017-sdca-sdw.c
index 148b36173a257..d62e8a2536767 100644
--- a/sound/soc/codecs/rt1017-sdca-sdw.c
+++ b/sound/soc/codecs/rt1017-sdca-sdw.c
@@ -773,25 +773,15 @@ static int rt1017_sdca_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt1017_sdca_priv *rt1017 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt1017->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT1017_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "Initialization not complete, timed out\n");
-		sdw_show_ping_status(slave->bus, true);
-
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT1017_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt1017->regmap, false);
 	regcache_sync(rt1017->regmap);
 
diff --git a/sound/soc/codecs/rt1308-sdw.c b/sound/soc/codecs/rt1308-sdw.c
index e077d096bc239..39e06a3a75609 100644
--- a/sound/soc/codecs/rt1308-sdw.c
+++ b/sound/soc/codecs/rt1308-sdw.c
@@ -768,25 +768,15 @@ static int rt1308_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt1308_sdw_priv *rt1308 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt1308->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT1308_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "Initialization not complete, timed out\n");
-		sdw_show_ping_status(slave->bus, true);
-
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT1308_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt1308->regmap, false);
 	regcache_sync_region(rt1308->regmap, 0xc000, 0xcfff);
 
diff --git a/sound/soc/codecs/rt1316-sdw.c b/sound/soc/codecs/rt1316-sdw.c
index 20fc1579eb9cf..1828fd9d5af6a 100644
--- a/sound/soc/codecs/rt1316-sdw.c
+++ b/sound/soc/codecs/rt1316-sdw.c
@@ -745,25 +745,15 @@ static int rt1316_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt1316_sdw_priv *rt1316 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt1316->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT1316_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n", __func__);
-		sdw_show_ping_status(slave->bus, true);
-
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT1316_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt1316->regmap, false);
 	regcache_sync(rt1316->regmap);
 
diff --git a/sound/soc/codecs/rt1318-sdw.c b/sound/soc/codecs/rt1318-sdw.c
index d28f1afe68f18..51bd11b92a554 100644
--- a/sound/soc/codecs/rt1318-sdw.c
+++ b/sound/soc/codecs/rt1318-sdw.c
@@ -821,23 +821,15 @@ static int rt1318_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt1318_sdw_priv *rt1318 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt1318->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT1318_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n", __func__);
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT1318_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt1318->regmap, false);
 	regcache_sync(rt1318->regmap);
 
diff --git a/sound/soc/codecs/rt1320-sdw.c b/sound/soc/codecs/rt1320-sdw.c
index 192faa431b5e9..13493b85f3c95 100644
--- a/sound/soc/codecs/rt1320-sdw.c
+++ b/sound/soc/codecs/rt1320-sdw.c
@@ -3053,23 +3053,15 @@ static int rt1320_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt1320_sdw_priv *rt1320 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt1320->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT1320_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n", __func__);
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT1320_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt1320->regmap, false);
 	regcache_sync(rt1320->regmap);
 	regcache_cache_only(rt1320->mbq_regmap, false);
diff --git a/sound/soc/codecs/rt5640.c b/sound/soc/codecs/rt5640.c
index f6c6294e15880..f9ee8cc64ee5e 100644
--- a/sound/soc/codecs/rt5640.c
+++ b/sound/soc/codecs/rt5640.c
@@ -2397,7 +2397,7 @@ static void rt5640_jack_work(struct work_struct *work)
 		 * disabled the OVCD IRQ, the IRQ pin will stay high and as
 		 * we react to edges, we miss the unplug event -> recheck.
 		 */
-		queue_delayed_work(system_long_wq, &rt5640->jack_work, 0);
+		queue_delayed_work(system_dfl_long_wq, &rt5640->jack_work, 0);
 	}
 }
 
@@ -2410,7 +2410,8 @@ static irqreturn_t rt5640_irq(int irq, void *data)
 		delay = 100;
 
 	if (rt5640->jack)
-		mod_delayed_work(system_long_wq, &rt5640->jack_work, delay);
+		mod_delayed_work(system_dfl_long_wq, &rt5640->jack_work,
+				 delay);
 
 	return IRQ_HANDLED;
 }
@@ -2419,7 +2420,7 @@ static irqreturn_t rt5640_jd_gpio_irq(int irq, void *data)
 {
 	struct rt5640_priv *rt5640 = data;
 
-	queue_delayed_work(system_long_wq, &rt5640->jack_work,
+	queue_delayed_work(system_dfl_long_wq, &rt5640->jack_work,
 			   msecs_to_jiffies(JACK_SETTLE_TIME));
 
 	return IRQ_HANDLED;
@@ -2553,10 +2554,12 @@ static void rt5640_enable_jack_detect(struct snd_soc_component *component,
 		rt5640->jd_gpio = jack_data->jd_gpio;
 		rt5640->jd_gpio_irq = gpiod_to_irq(rt5640->jd_gpio);
 
-		ret = request_irq(rt5640->jd_gpio_irq, rt5640_jd_gpio_irq,
-				  IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING,
-				  "rt5640-jd-gpio", rt5640);
-		if (ret) {
+		ret = request_any_context_irq(rt5640->jd_gpio_irq,
+					      rt5640_jd_gpio_irq,
+					      IRQF_TRIGGER_RISING |
+					      IRQF_TRIGGER_FALLING,
+					      "rt5640-jd-gpio", rt5640);
+		if (ret < 0) {
 			dev_warn(component->dev, "Failed to request jd GPIO IRQ %d: %d\n",
 				 rt5640->jd_gpio_irq, ret);
 			rt5640_disable_jack_detect(component);
@@ -2568,10 +2571,10 @@ static void rt5640_enable_jack_detect(struct snd_soc_component *component,
 	if (jack_data && jack_data->use_platform_clock)
 		rt5640->use_platform_clock = jack_data->use_platform_clock;
 
-	ret = request_irq(rt5640->irq, rt5640_irq,
-			  IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING,
-			  "rt5640", rt5640);
-	if (ret) {
+	ret = request_any_context_irq(rt5640->irq, rt5640_irq,
+				      IRQF_TRIGGER_RISING | IRQF_TRIGGER_FALLING,
+				      "rt5640", rt5640);
+	if (ret < 0) {
 		dev_warn(component->dev, "Failed to request IRQ %d: %d\n", rt5640->irq, ret);
 		rt5640_disable_jack_detect(component);
 		return;
@@ -2579,7 +2582,7 @@ static void rt5640_enable_jack_detect(struct snd_soc_component *component,
 	rt5640->irq_requested = true;
 
 	/* sync initial jack state */
-	queue_delayed_work(system_long_wq, &rt5640->jack_work, 0);
+	queue_delayed_work(system_dfl_long_wq, &rt5640->jack_work, 0);
 }
 
 static const struct snd_soc_dapm_route rt5640_hda_jack_dapm_routes[] = {
@@ -2622,9 +2625,9 @@ static void rt5640_enable_hda_jack_detect(
 
 	rt5640->jack = jack;
 
-	ret = request_irq(rt5640->irq, rt5640_irq,
-			  IRQF_TRIGGER_RISING, "rt5640", rt5640);
-	if (ret) {
+	ret = request_any_context_irq(rt5640->irq, rt5640_irq,
+				      IRQF_TRIGGER_RISING, "rt5640", rt5640);
+	if (ret < 0) {
 		dev_warn(component->dev, "Failed to request IRQ %d: %d\n", rt5640->irq, ret);
 		rt5640->jack = NULL;
 		return;
@@ -2632,7 +2635,7 @@ static void rt5640_enable_hda_jack_detect(
 	rt5640->irq_requested = true;
 
 	/* sync initial jack state */
-	queue_delayed_work(system_long_wq, &rt5640->jack_work, 0);
+	queue_delayed_work(system_dfl_long_wq, &rt5640->jack_work, 0);
 
 	snd_soc_dapm_add_routes(dapm, rt5640_hda_jack_dapm_routes,
 		ARRAY_SIZE(rt5640_hda_jack_dapm_routes));
@@ -2860,7 +2863,7 @@ static int rt5640_resume(struct snd_soc_component *component)
 		}
 
 		enable_irq(rt5640->irq);
-		queue_delayed_work(system_long_wq, &rt5640->jack_work, 0);
+		queue_delayed_work(system_dfl_long_wq, &rt5640->jack_work, 0);
 	}
 
 	return 0;
diff --git a/sound/soc/codecs/rt5682-sdw.c b/sound/soc/codecs/rt5682-sdw.c
index fc464538ceffb..ec2a35a0cacde 100644
--- a/sound/soc/codecs/rt5682-sdw.c
+++ b/sound/soc/codecs/rt5682-sdw.c
@@ -754,7 +754,7 @@ static int rt5682_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt5682_priv *rt5682 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt5682->first_hw_init)
 		return 0;
@@ -766,20 +766,12 @@ static int rt5682_dev_resume(struct device *dev)
 			rt5682->disable_irq = false;
 		}
 		mutex_unlock(&rt5682->disable_irq_lock);
-		goto regmap_sync;
 	}
 
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT5682_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n", __func__);
-		sdw_show_ping_status(slave->bus, true);
-
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT5682_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt5682->sdw_regmap, false);
 	regcache_cache_only(rt5682->regmap, false);
 	regcache_sync(rt5682->regmap);
diff --git a/sound/soc/codecs/rt700-sdw.c b/sound/soc/codecs/rt700-sdw.c
index 9ce36a66fae1d..30fcca210f051 100644
--- a/sound/soc/codecs/rt700-sdw.c
+++ b/sound/soc/codecs/rt700-sdw.c
@@ -522,25 +522,15 @@ static int rt700_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt700_priv *rt700 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt700->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT700_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "Initialization not complete, timed out\n");
-		sdw_show_ping_status(slave->bus, true);
-
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT700_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt700->regmap, false);
 	regcache_sync_region(rt700->regmap, 0x3000, 0x8fff);
 	regcache_sync_region(rt700->regmap, 0x752010, 0x75206b);
diff --git a/sound/soc/codecs/rt711-sdca-sdw.c b/sound/soc/codecs/rt711-sdca-sdw.c
index 49dacceddf815..a8164fc3979ab 100644
--- a/sound/soc/codecs/rt711-sdca-sdw.c
+++ b/sound/soc/codecs/rt711-sdca-sdw.c
@@ -438,7 +438,7 @@ static int rt711_sdca_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt711_sdca_priv *rt711 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt711->first_hw_init)
 		return 0;
@@ -451,20 +451,12 @@ static int rt711_sdca_dev_resume(struct device *dev)
 			rt711->disable_irq = false;
 		}
 		mutex_unlock(&rt711->disable_irq_lock);
-		goto regmap_sync;
 	}
 
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT711_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n", __func__);
-		sdw_show_ping_status(slave->bus, true);
+	ret = sdw_slave_wait_for_init(slave, RT711_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-		return -ETIMEDOUT;
-	}
-
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt711->regmap, false);
 	regcache_sync(rt711->regmap);
 	regcache_cache_only(rt711->mbq_regmap, false);
diff --git a/sound/soc/codecs/rt711-sdw.c b/sound/soc/codecs/rt711-sdw.c
index 72ddf4cebdf36..df3c43f2ab6b8 100644
--- a/sound/soc/codecs/rt711-sdw.c
+++ b/sound/soc/codecs/rt711-sdw.c
@@ -530,7 +530,7 @@ static int rt711_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt711_priv *rt711 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt711->first_hw_init)
 		return 0;
@@ -542,18 +542,12 @@ static int rt711_dev_resume(struct device *dev)
 			rt711->disable_irq = false;
 		}
 		mutex_unlock(&rt711->disable_irq_lock);
-		goto regmap_sync;
 	}
 
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT711_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n", __func__);
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT711_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt711->regmap, false);
 	regcache_sync_region(rt711->regmap, 0x3000, 0x8fff);
 	regcache_sync_region(rt711->regmap, 0x752009, 0x752091);
diff --git a/sound/soc/codecs/rt712-sdca-dmic.c b/sound/soc/codecs/rt712-sdca-dmic.c
index 4d83544ef2049..4c5c2f5ba5edf 100644
--- a/sound/soc/codecs/rt712-sdca-dmic.c
+++ b/sound/soc/codecs/rt712-sdca-dmic.c
@@ -905,26 +905,15 @@ static int rt712_sdca_dmic_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt712_sdca_dmic_priv *rt712 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt712->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT712_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n",
-			__func__);
-		sdw_show_ping_status(slave->bus, true);
-
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT712_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt712->regmap, false);
 	regcache_sync(rt712->regmap);
 	regcache_cache_only(rt712->mbq_regmap, false);
diff --git a/sound/soc/codecs/rt712-sdca-sdw.c b/sound/soc/codecs/rt712-sdca-sdw.c
index 8c82887174db2..5817321804736 100644
--- a/sound/soc/codecs/rt712-sdca-sdw.c
+++ b/sound/soc/codecs/rt712-sdca-sdw.c
@@ -450,7 +450,7 @@ static int rt712_sdca_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt712_sdca_priv *rt712 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt712->first_hw_init)
 		return 0;
@@ -464,20 +464,12 @@ static int rt712_sdca_dev_resume(struct device *dev)
 			rt712->disable_irq = false;
 		}
 		mutex_unlock(&rt712->disable_irq_lock);
-		goto regmap_sync;
 	}
 
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT712_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n", __func__);
-		sdw_show_ping_status(slave->bus, true);
+	ret = sdw_slave_wait_for_init(slave, RT712_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-		return -ETIMEDOUT;
-	}
-
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt712->regmap, false);
 	regcache_sync(rt712->regmap);
 	regcache_cache_only(rt712->mbq_regmap, false);
diff --git a/sound/soc/codecs/rt715-sdca-sdw.c b/sound/soc/codecs/rt715-sdca-sdw.c
index 968bc183b8d8c..4b9815b5628db 100644
--- a/sound/soc/codecs/rt715-sdca-sdw.c
+++ b/sound/soc/codecs/rt715-sdca-sdw.c
@@ -224,25 +224,15 @@ static int rt715_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt715_sdca_priv *rt715 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt715->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
+	ret = sdw_slave_wait_for_init(slave, RT715_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-					   msecs_to_jiffies(RT715_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n", __func__);
-		sdw_show_ping_status(slave->bus, true);
-
-		return -ETIMEDOUT;
-	}
-
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt715->regmap, false);
 	regcache_sync_region(rt715->regmap,
 		SDW_SDCA_CTL(FUN_JACK_CODEC, RT715_SDCA_ST_EN, RT715_SDCA_ST_CTRL,
diff --git a/sound/soc/codecs/rt715-sdw.c b/sound/soc/codecs/rt715-sdw.c
index 49c91d015be4d..7f83a8f1a06e9 100644
--- a/sound/soc/codecs/rt715-sdw.c
+++ b/sound/soc/codecs/rt715-sdw.c
@@ -501,25 +501,15 @@ static int rt715_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt715_priv *rt715 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt715->first_hw_init)
 		return 0;
 
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-					   msecs_to_jiffies(RT715_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "%s: Initialization not complete, timed out\n", __func__);
-		sdw_show_ping_status(slave->bus, true);
-
-		return -ETIMEDOUT;
-	}
+	ret = sdw_slave_wait_for_init(slave, RT715_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt715->regmap, false);
 	regcache_sync_region(rt715->regmap, 0x3000, 0x8fff);
 	regcache_sync_region(rt715->regmap, 0x752039, 0x752039);
diff --git a/sound/soc/codecs/rt721-sdca-sdw.c b/sound/soc/codecs/rt721-sdca-sdw.c
index 6eb8512975b85..58606209316a4 100644
--- a/sound/soc/codecs/rt721-sdca-sdw.c
+++ b/sound/soc/codecs/rt721-sdca-sdw.c
@@ -489,7 +489,7 @@ static int rt721_sdca_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt721_sdca_priv *rt721 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt721->first_hw_init)
 		return 0;
@@ -502,20 +502,12 @@ static int rt721_sdca_dev_resume(struct device *dev)
 			rt721->disable_irq = false;
 		}
 		mutex_unlock(&rt721->disable_irq_lock);
-		goto regmap_sync;
 	}
 
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT721_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "Initialization not complete, timed out\n");
-		sdw_show_ping_status(slave->bus, true);
+	ret = sdw_slave_wait_for_init(slave, RT721_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-		return -ETIMEDOUT;
-	}
-
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt721->regmap, false);
 	regcache_sync(rt721->regmap);
 	regcache_cache_only(rt721->mbq_regmap, false);
diff --git a/sound/soc/codecs/rt722-sdca-sdw.c b/sound/soc/codecs/rt722-sdca-sdw.c
index 0a5b3ffa90daf..0f76492ff915c 100644
--- a/sound/soc/codecs/rt722-sdca-sdw.c
+++ b/sound/soc/codecs/rt722-sdca-sdw.c
@@ -19,7 +19,9 @@
 static int rt722_sdca_mbq_size(struct device *dev, unsigned int reg)
 {
 	switch (reg) {
-	case 0x2f01 ... 0x2f0a:
+	case 0x22f0 ... 0x22f1:
+	case 0x2f01 ... 0x2f0c:
+	case 0x2f21 ... 0x2f24:
 	case 0x2f35 ... 0x2f36:
 	case 0x2f50 ... 0x2f52:
 	case 0x2f54:
@@ -84,6 +86,21 @@ static int rt722_sdca_mbq_size(struct device *dev, unsigned int reg)
 	case SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_CS31,
 			  RT722_SDCA_CTL_SAMPLE_FREQ_INDEX, 0):
 	case RT722_BUF_ADDR_HID1 ... RT722_BUF_ADDR_HID2:
+	case 0x44011000 ... 0x440115ff:
+	case 0x44012000:
+	case 0x44012021:
+	case 0x44012022:
+	case 0x44012025:
+	case 0x44021000 ... 0x440211ff:
+	case 0x44022000:
+	case 0x44022019:
+	case 0x4402201a:
+	case 0x4402201d:
+	case 0x44041000 ... 0x440415ff:
+	case 0x44042000:
+	case 0x44042019:
+	case 0x4404201a:
+	case 0x4404201d:
 		return 1;
 	case 0x2000000 ... 0x2000024:
 	case 0x2000029 ... 0x200004a:
@@ -95,8 +112,10 @@ static int rt722_sdca_mbq_size(struct device *dev, unsigned int reg)
 	case 0x200007f:
 	case 0x2000082 ... 0x200008e:
 	case 0x2000090 ... 0x2000094:
+	case 0x20000b1:
+	case 0x20000b4:
 	case 0x3110000:
-	case 0x5300000 ... 0x5300002:
+	case 0x5300000 ... 0x5300300:
 	case 0x5400002:
 	case 0x5600000 ... 0x5600007:
 	case 0x5700000 ... 0x5700004:
@@ -175,6 +194,7 @@ static bool rt722_sdca_volatile_register(struct device *dev, unsigned int reg)
 	case SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_PDE23, RT722_SDCA_CTL_ACTUAL_POWER_STATE, 0):
 	case RT722_BUF_ADDR_HID1 ... RT722_BUF_ADDR_HID2:
 	case 0x2000000:
+	case 0x2000007:
 	case 0x200000d:
 	case 0x2000019:
 	case 0x2000020:
@@ -186,6 +206,21 @@ static bool rt722_sdca_volatile_register(struct device *dev, unsigned int reg)
 	case 0x3110000:
 	case 0x5800003:
 	case 0x5810000:
+	case 0x44011000 ... 0x440115ff:
+	case 0x44012000:
+	case 0x44012021:
+	case 0x44012022:
+	case 0x44012025:
+	case 0x44021000 ... 0x440211ff:
+	case 0x44022000:
+	case 0x44022019:
+	case 0x4402201a:
+	case 0x4402201d:
+	case 0x44041000 ... 0x440415ff:
+	case 0x44042000:
+	case 0x44042019:
+	case 0x4404201a:
+	case 0x4404201d:
 		return true;
 	default:
 		return false;
@@ -501,7 +536,7 @@ static int rt722_sdca_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct rt722_sdca_priv *rt722 = dev_get_drvdata(dev);
-	unsigned long time;
+	int ret;
 
 	if (!rt722->first_hw_init)
 		return 0;
@@ -514,20 +549,12 @@ static int rt722_sdca_dev_resume(struct device *dev)
 			rt722->disable_irq = false;
 		}
 		mutex_unlock(&rt722->disable_irq_lock);
-		goto regmap_sync;
 	}
 
-	time = wait_for_completion_timeout(&slave->initialization_complete,
-				msecs_to_jiffies(RT722_PROBE_TIMEOUT));
-	if (!time) {
-		dev_err(&slave->dev, "Initialization not complete, timed out\n");
-		sdw_show_ping_status(slave->bus, true);
+	ret = sdw_slave_wait_for_init(slave, RT722_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-		return -ETIMEDOUT;
-	}
-
-regmap_sync:
-	slave->unattach_request = 0;
 	regcache_cache_only(rt722->regmap, false);
 	regcache_sync(rt722->regmap);
 	return 0;
diff --git a/sound/soc/codecs/rt722-sdca.c b/sound/soc/codecs/rt722-sdca.c
index 79b8b7e70a334..1b6729f363fc4 100644
--- a/sound/soc/codecs/rt722-sdca.c
+++ b/sound/soc/codecs/rt722-sdca.c
@@ -7,19 +7,21 @@
 //
 
 #include <linux/bitops.h>
-#include <sound/core.h>
 #include <linux/delay.h>
+#include <linux/dmi.h>
+#include <linux/firmware.h>
 #include <linux/init.h>
-#include <sound/initval.h>
-#include <sound/jack.h>
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
-#include <sound/pcm.h>
 #include <linux/pm_runtime.h>
-#include <sound/pcm_params.h>
-#include <linux/soundwire/sdw_registers.h>
 #include <linux/slab.h>
+#include <linux/soundwire/sdw_registers.h>
+#include <sound/core.h>
+#include <sound/initval.h>
+#include <sound/jack.h>
+#include <sound/pcm.h>
+#include <sound/pcm_params.h>
 #include <sound/soc-dapm.h>
 #include <sound/tlv.h>
 
@@ -344,6 +346,280 @@ static int rt722_sdca_set_jack_detect(struct snd_soc_component *component,
 	return 0;
 }
 
+static int rt722_cae_load(struct rt722_sdca_priv *rt722)
+{
+	struct device *dev = &rt722->slave->dev;
+	static const char func_tag[] = "FUNC";
+	static const char xu_tag[] = "XU";
+	const char *dmi_vendor, *dmi_product, *dmi_sku;
+	char *cae_filename;
+	const struct firmware *cae_fw = NULL;
+	unsigned int cae_st_spk, cae_st_hp, cae_st_mic;
+	unsigned int func, value;
+	unsigned int combined_val;
+	unsigned int addr, size;
+	unsigned int fw_offset;
+	unsigned char mbq_high_val = 0;
+	unsigned char *param_data;
+	unsigned char *fw_data;
+	char tag[5];
+	char *space;
+	int v_len, p_len, s_len;
+	int ret = 0, i;
+	int retry = 50;
+
+	dmi_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
+	dmi_product = dmi_get_system_info(DMI_PRODUCT_NAME);
+	dmi_sku = dmi_get_system_info(DMI_PRODUCT_SKU);
+
+	if (!dmi_vendor || !dmi_product || !dmi_sku) {
+		dev_warn(dev, "%s: Incomplete DMI info\n", __func__);
+		return -EINVAL;
+	}
+	space = strchr(dmi_vendor, ' ');
+	v_len = space ? space - dmi_vendor : strlen(dmi_vendor);
+
+	space = strchr(dmi_product, ' ');
+	p_len = space ? space - dmi_product : strlen(dmi_product);
+
+	space = strchr(dmi_sku, ' ');
+	s_len = space ? space - dmi_sku : strlen(dmi_sku);
+
+	cae_filename = kasprintf(GFP_KERNEL,
+				 "realtek/rt722/rt722_RAE_%.*s_%.*s_%.*s.dat",
+				 v_len, dmi_vendor,
+				 p_len, dmi_product,
+				 s_len, dmi_sku);
+	if (!cae_filename)
+		return -ENOMEM;
+	dev_dbg(dev, "%s: try to load CAE file %s\n", __func__, cae_filename);
+
+	regmap_write(rt722->regmap, RT722_SPK_CAE_PARAM1, 0x5f);
+	regmap_write(rt722->regmap, RT722_HP_CAE_PARAM39, 0x5f);
+	regmap_write(rt722->regmap, RT722_MIC_CAE_PARAM39, 0x5f);
+	usleep_range(50000, 60000);
+
+	request_firmware(&cae_fw, cae_filename, dev);
+	kfree(cae_filename);
+	if (!cae_fw) {
+		dev_err(dev, "%s: Failed to load CAE firmware\n", __func__);
+		return -ENOENT;
+	}
+
+	regmap_read(rt722->regmap, RT722_SPK_CAE_PARAM38, &cae_st_spk);
+	regmap_read(rt722->regmap, RT722_HP_CAE_PARAM68, &cae_st_hp);
+	regmap_read(rt722->regmap, RT722_MIC_CAE_PARAM99, &cae_st_mic);
+	cae_st_spk &= 0x80;
+	cae_st_hp &= 0x80;
+	cae_st_mic &= 0x80;
+
+	dev_dbg(dev, "%s(%d) spk_crc:%x, hp_crc:%x, mic_crc:%x\n",
+		__func__, __LINE__, cae_st_spk, cae_st_hp, cae_st_mic);
+
+	if (cae_st_spk)
+		rt722_sdca_index_update_bits(rt722, RT722_VENDOR_EQ_CAE,
+			RT722_EQ_CTRL_SPK, 0x0008, 0x0008);
+	else if (cae_st_hp)
+		rt722_sdca_index_update_bits(rt722, RT722_VENDOR_EQ_CAE,
+			RT722_EQ_CTRL_HP, 0x0008, 0x0008);
+	else if (cae_st_mic)
+		rt722_sdca_index_update_bits(rt722, RT722_VENDOR_EQ_CAE,
+			RT722_EQ_CTRL_DMIC, 0x0008, 0x0008);
+
+	rt722_sdca_index_update_bits(rt722, RT722_VENDOR_REG,
+			RT722_MISC_CTRL1, 0x8000, 0x8000);
+
+	regmap_update_bits(rt722->regmap, RT722_SPK_CAE_PARAM34, 0x1, 0x0);
+	regmap_update_bits(rt722->regmap, RT722_HP_CAE_PARAM64, 0x1, 0x0);
+	regmap_update_bits(rt722->regmap, RT722_MIC_CAE_PARAM95, 0x1, 0x0);
+
+	while (--retry) {
+		regmap_read(rt722->regmap, RT722_SPK_CAE_PARAM35, &cae_st_spk);
+		regmap_read(rt722->regmap, RT722_HP_CAE_PARAM65, &cae_st_hp);
+		regmap_read(rt722->regmap, RT722_MIC_CAE_PARAM96, &cae_st_mic);
+		dev_dbg(dev, "%s(%d) cae_st_spk:%x, cae_st_hp:%x, cae_st_mic:%x\n",
+			__func__, __LINE__, cae_st_spk, cae_st_hp, cae_st_mic);
+		if ((cae_st_spk & 0x40) && (cae_st_hp & 0x40) && (cae_st_mic & 0x40))
+			break;
+		usleep_range(1000, 1100);
+	}
+
+	if (!retry && !((cae_st_spk & 0x40) && (cae_st_hp & 0x40)
+		&& (cae_st_mic & 0x40))) {
+		dev_err(dev, "%s: CAE is not ready to be loaded.\n", __func__);
+		ret = -ETIMEDOUT;
+		goto out_release;
+	}
+
+	dev_dbg(dev, "%s, cae_fw size=0x%zx, start\n", __func__, cae_fw->size);
+
+	rt722_sdca_index_write(rt722, RT722_VENDOR_EQ_CAE,
+		RT722_EQ_CTRL_AMIC, 0x8000);
+	rt722_sdca_index_write(rt722, RT722_VENDOR_EQ_CAE,
+		RT722_EQ_CTRL_DMIC, 0x8004);
+	rt722_sdca_index_write(rt722, RT722_VENDOR_EQ_CAE,
+		RT722_EQ_CTRL_HP, 0x8074);
+	rt722_sdca_index_write(rt722, RT722_VENDOR_EQ_CAE,
+		RT722_EQ_CTRL_SPK, 0xa074);
+
+	regcache_cache_bypass(rt722->regmap, true);
+	for (fw_offset = 0; fw_offset < cae_fw->size;) {
+
+		if (fw_offset + 12 > cae_fw->size) {
+			dev_err(dev, "%s: Unexpected end of firmware\n", __func__);
+			ret = -EINVAL;
+			goto verify_abort;
+		}
+
+		fw_data = (unsigned char *)&cae_fw->data[fw_offset];
+		memcpy(tag, fw_data, 4);
+		tag[4] = '\0';
+
+		if (strcmp(tag, xu_tag) == 0) {
+			dev_dbg(dev, "%s: This is a XU tag", __func__);
+			memcpy(&addr, (fw_data + 4), 4);
+			memcpy(&size, (fw_data + 8), 4);
+
+			if (size == 0 || size > cae_fw->size - fw_offset - 12) {
+				dev_err(dev, "%s: Invalid payload size: %u\n", __func__, size);
+				ret = -EINVAL;
+				goto verify_abort;
+			}
+
+			param_data = (unsigned char *)(fw_data + 12);
+
+			dev_dbg(dev, "%s: addr=0x%x, size=0x%x\n", __func__, addr, size);
+
+			if ((addr <= 0x05302300 && addr >= 0x05300000) ||
+				(addr <= 0x020020b4 && addr >= 0x020000b1)) {
+				if (addr & BIT(13)) {
+					mbq_high_val = param_data[0];
+					dev_dbg(dev, "MBQ: High Byte 0x%02x\n", mbq_high_val);
+					fw_offset += (size + 12);
+
+					continue;
+				} else {
+					regcache_cache_bypass(rt722->regmap, false);
+					combined_val = (mbq_high_val << 8) | param_data[0];
+					if (addr == 0x20000b1 || addr == 0x20000b4)
+						combined_val |= (0x2 << 8);
+					ret = regmap_write(rt722->regmap, addr, combined_val);
+					if (ret) {
+						dev_err(dev,
+							"MBQ fail: addr=0x%x ret=%d\n", addr, ret);
+						regcache_cache_bypass(rt722->regmap, true);
+						goto verify_abort;
+					}
+
+					dev_dbg(dev, "MBQ-reg=0x%x, value=0x%x\n",
+						addr, combined_val);
+
+					fw_offset += (size + 12);
+					regcache_cache_bypass(rt722->regmap, true);
+					continue;
+				}
+			}
+
+			for (i = 0; i < size; i++) {
+				ret = regmap_write(rt722->regmap, addr + i, param_data[i]);
+				if (ret) {
+					dev_err(dev,
+						"CAE write fail: addr=0x%x ret=%d\n",
+						addr + i, ret);
+					goto verify_abort;
+				}
+			}
+			fw_offset += (size + 12);
+		} else if (strcmp(tag, func_tag) == 0) {
+			dev_dbg(dev, "%s: This is a FUNC tag", __func__);
+
+			memcpy(&func, (fw_data + 4), 4);
+			memcpy(&value, (fw_data + 8), 4);
+			dev_dbg(dev, "%s: func=0x%x, value=0x%x\n",
+				__func__, func, value);
+
+			if (func == 1)
+				msleep(value);
+
+			fw_offset += 12;
+		} else {
+			dev_err(dev, "%s: No XU/FUNC tag at fw_offset=0x%x\n",
+				__func__, fw_offset);
+			ret = -EINVAL;
+			goto verify_abort;
+		}
+	}
+
+	rt722_sdca_index_update_bits(rt722, RT722_VENDOR_REG,
+			RT722_MISC_CTRL1, 0x8000, 0x0000);
+	regcache_cache_bypass(rt722->regmap, false);
+	rt722->cae_update_done = 1;
+	dev_dbg(dev, "%s: CAE FW update done.\n", __func__);
+	release_firmware(cae_fw);
+	return 0;
+
+verify_abort:
+	regcache_cache_bypass(rt722->regmap, false);
+	if (!ret)
+		ret = -EIO;
+out_release:
+	rt722_sdca_index_update_bits(rt722, RT722_VENDOR_REG,
+			RT722_MISC_CTRL1, 0x8000, 0x0000);
+	release_firmware(cae_fw);
+	dev_err(dev, "%s: CAE FW update aborted (ret=%d).\n", __func__, ret);
+	return ret;
+}
+
+static int rt722_cae_update_put(struct snd_kcontrol *kcontrol,
+				 struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt722_sdca_priv *rt722 = snd_soc_component_get_drvdata(component);
+	struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component);
+	int ret, changed = 0;
+
+	if (!rt722->hw_init)
+		return 0;
+
+	ret = pm_runtime_resume_and_get(component->dev);
+	if (ret < 0 && ret != -EACCES)
+		return ret;
+
+	if (ucontrol->value.integer.value[0]) {
+		if (snd_soc_dapm_get_bias_level(dapm) == SND_SOC_BIAS_OFF) {
+			ret = rt722_cae_load(rt722);
+			if (ret) {
+				dev_err(component->dev, "CAE load failed: %d\n", ret);
+				goto out;
+			} else
+				changed = 1;
+		}
+	} else {
+		if (rt722->cae_update_done) {
+			rt722->cae_update_done = 0;
+			changed = 1;
+		}
+	}
+
+out:
+	pm_runtime_mark_last_busy(component->dev);
+	pm_runtime_put_autosuspend(component->dev);
+
+	return ret < 0 ? ret : changed;
+}
+
+static int rt722_cae_update_get(struct snd_kcontrol *kcontrol,
+				 struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt722_sdca_priv *rt722 = snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = rt722->cae_update_done;
+
+	return 0;
+}
+
+
 /* For SDCA control DAC/ADC Gain */
 static int rt722_sdca_set_gain_put(struct snd_kcontrol *kcontrol,
 		struct snd_ctl_elem_value *ucontrol)
@@ -517,6 +793,61 @@ static int rt722_sdca_fu1e_capture_put(struct snd_kcontrol *kcontrol,
 	return changed;
 }
 
+static int rt722_sdca_set_fu06_playback_ctl(struct rt722_sdca_priv *rt722)
+{
+	int err;
+	unsigned int ch_l, ch_r;
+
+	ch_l = (rt722->fu06_dapm_mute || rt722->fu06_mixer_l_mute) ? 0x01 : 0x00;
+	ch_r = (rt722->fu06_dapm_mute || rt722->fu06_mixer_r_mute) ? 0x01 : 0x00;
+
+	err = regmap_write(rt722->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_USER_FU06,
+			RT722_SDCA_CTL_FU_MUTE, CH_L), ch_l);
+	if (err < 0)
+		return err;
+
+	err = regmap_write(rt722->regmap,
+			SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_USER_FU06,
+			RT722_SDCA_CTL_FU_MUTE, CH_R), ch_r);
+	if (err < 0)
+		return err;
+
+	return 0;
+}
+
+static int rt722_sdca_fu06_playback_get(struct snd_kcontrol *kcontrol,
+			struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt722_sdca_priv *rt722 = snd_soc_component_get_drvdata(component);
+
+	ucontrol->value.integer.value[0] = !rt722->fu06_mixer_l_mute;
+	ucontrol->value.integer.value[1] = !rt722->fu06_mixer_r_mute;
+	return 0;
+}
+
+static int rt722_sdca_fu06_playback_put(struct snd_kcontrol *kcontrol,
+			struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *component = snd_kcontrol_chip(kcontrol);
+	struct rt722_sdca_priv *rt722 = snd_soc_component_get_drvdata(component);
+	int err, changed = 0;
+
+	if (rt722->fu06_mixer_l_mute != !ucontrol->value.integer.value[0] ||
+		rt722->fu06_mixer_r_mute != !ucontrol->value.integer.value[1])
+		changed = 1;
+
+	rt722->fu06_mixer_l_mute = !ucontrol->value.integer.value[0];
+	rt722->fu06_mixer_r_mute = !ucontrol->value.integer.value[1];
+
+	err = rt722_sdca_set_fu06_playback_ctl(rt722);
+	if (err < 0)
+		return err;
+
+	return changed;
+}
+
 static int rt722_sdca_set_fu0f_capture_ctl(struct rt722_sdca_priv *rt722)
 {
 	int err;
@@ -718,6 +1049,8 @@ static const struct snd_kcontrol_new rt722_sdca_controls[] = {
 			RT722_SDCA_CTL_FU_CH_GAIN, CH_R), 8, 3, 0,
 		rt722_sdca_set_gain_get, rt722_sdca_set_gain_put, boost_vol_tlv),
 	/* AMP playback settings */
+	SOC_DOUBLE_EXT("FU06 Playback Switch", SND_SOC_NOPM, 0, 1, 1, 0,
+		rt722_sdca_fu06_playback_get, rt722_sdca_fu06_playback_put),
 	SOC_DOUBLE_R_EXT_TLV("FU06 Playback Volume",
 		SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_USER_FU06,
 			RT722_SDCA_CTL_FU_VOLUME, CH_L),
@@ -738,6 +1071,9 @@ static const struct snd_kcontrol_new rt722_sdca_controls[] = {
 			RT722_SDCA_CTL_FU_CH_GAIN, CH_01),
 		rt722_sdca_dmic_set_gain_get, rt722_sdca_dmic_set_gain_put,
 			4, 3, boost_vol_tlv),
+	/* CAE firmware update */
+	SOC_SINGLE_EXT("CAE Update", SND_SOC_NOPM, 0, 1, 0,
+		rt722_cae_update_get, rt722_cae_update_put),
 };
 
 static const char * const adc22_mux_text[] = {
@@ -807,27 +1143,17 @@ static int rt722_sdca_fu21_event(struct snd_soc_dapm_widget *w,
 	struct snd_soc_component *component =
 		snd_soc_dapm_to_component(w->dapm);
 	struct rt722_sdca_priv *rt722 = snd_soc_component_get_drvdata(component);
-	unsigned char unmute = 0x0, mute = 0x1;
 
 	switch (event) {
 	case SND_SOC_DAPM_POST_PMU:
-		regmap_write(rt722->regmap,
-			SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_USER_FU06,
-				RT722_SDCA_CTL_FU_MUTE, CH_L), unmute);
-		regmap_write(rt722->regmap,
-			SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_USER_FU06,
-				RT722_SDCA_CTL_FU_MUTE, CH_R), unmute);
+		rt722->fu06_dapm_mute = false;
 		break;
 	case SND_SOC_DAPM_PRE_PMD:
-		regmap_write(rt722->regmap,
-			SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_USER_FU06,
-				RT722_SDCA_CTL_FU_MUTE, CH_L), mute);
-		regmap_write(rt722->regmap,
-			SDW_SDCA_CTL(FUNC_NUM_AMP, RT722_SDCA_ENT_USER_FU06,
-				RT722_SDCA_CTL_FU_MUTE, CH_R), mute);
+		rt722->fu06_dapm_mute = true;
 		break;
 	}
-	return 0;
+
+	return rt722_sdca_set_fu06_playback_ctl(rt722);
 }
 
 static int rt722_sdca_fu113_event(struct snd_soc_dapm_widget *w,
@@ -1324,9 +1650,12 @@ int rt722_sdca_init(struct device *dev, struct regmap *regmap, struct sdw_slave
 	rt722->first_hw_init = false;
 	rt722->fu1e_dapm_mute = true;
 	rt722->fu0f_dapm_mute = true;
+	rt722->fu06_dapm_mute = true;
+	rt722->fu06_mixer_l_mute = rt722->fu06_mixer_r_mute = false;
 	rt722->fu0f_mixer_l_mute = rt722->fu0f_mixer_r_mute = true;
 	rt722->fu1e_mixer_mute[0] = rt722->fu1e_mixer_mute[1] =
 		rt722->fu1e_mixer_mute[2] = rt722->fu1e_mixer_mute[3] = true;
+	rt722->cae_update_done = 0;
 
 	return devm_snd_soc_register_component(dev,
 			&soc_sdca_dev_rt722, rt722_sdca_dai, ARRAY_SIZE(rt722_sdca_dai));
diff --git a/sound/soc/codecs/rt722-sdca.h b/sound/soc/codecs/rt722-sdca.h
index 823abee9ab76c..b654b14334d43 100644
--- a/sound/soc/codecs/rt722-sdca.h
+++ b/sound/soc/codecs/rt722-sdca.h
@@ -36,10 +36,15 @@ struct  rt722_sdca_priv {
 	bool fu0f_dapm_mute;
 	bool fu0f_mixer_l_mute;
 	bool fu0f_mixer_r_mute;
+	/* For AMP */
+	bool fu06_dapm_mute;
+	bool fu06_mixer_l_mute;
+	bool fu06_mixer_r_mute;
 	/* For DMIC */
 	bool fu1e_dapm_mute;
 	bool fu1e_mixer_mute[4];
 	int hw_vid;
+	int cae_update_done;
 };
 
 struct rt722_sdca_dmic_kctrl_priv {
@@ -51,6 +56,7 @@ struct rt722_sdca_dmic_kctrl_priv {
 
 /* NID */
 #define RT722_VENDOR_REG			0x20
+#define RT722_VENDOR_EQ_CAE			0x53
 #define RT722_VENDOR_CALI			0x58
 #define RT722_VENDOR_SPK_EFUSE			0x5c
 #define RT722_VENDOR_IMS_DRE			0x5b
@@ -60,6 +66,7 @@ struct rt722_sdca_dmic_kctrl_priv {
 /* Index (NID:20h) */
 #define RT722_JD_PRODUCT_NUM			0x00
 #define RT722_ANALOG_BIAS_CTL3			0x04
+#define RT722_MISC_CTRL1			0x07
 #define RT722_JD_CTRL1				0x09
 #define RT722_LDO2_3_CTL1			0x0e
 #define RT722_LDO1_CTL				0x1a
@@ -75,6 +82,12 @@ struct rt722_sdca_dmic_kctrl_priv {
 #define RT722_SW_CONFIG1			0x8a
 #define RT722_SW_CONFIG2			0x8b
 
+/* Index (NID:53h) */
+#define RT722_EQ_CTRL_SPK			0x00
+#define RT722_EQ_CTRL_HP			0x100
+#define RT722_EQ_CTRL_DMIC			0x200
+#define RT722_EQ_CTRL_AMIC			0x300
+
 /* Index (NID:58h) */
 #define RT722_DAC_DC_CALI_CTL0			0x00
 #define RT722_DAC_DC_CALI_CTL1			0x01
@@ -152,6 +165,20 @@ struct rt722_sdca_dmic_kctrl_priv {
 #define RT722_BUF_ADDR_HID1			0x44030000
 #define RT722_BUF_ADDR_HID2			0x44030020
 
+/* RT722 CAE parameter settings */
+#define RT722_SPK_CAE_PARAM1			0x44012000
+#define RT722_SPK_CAE_PARAM34			0x44012021
+#define RT722_SPK_CAE_PARAM35			0x44012022
+#define RT722_SPK_CAE_PARAM38			0x44012025
+#define RT722_HP_CAE_PARAM39			0x44022000
+#define RT722_HP_CAE_PARAM64			0x44022019
+#define RT722_HP_CAE_PARAM65			0x4402201a
+#define RT722_HP_CAE_PARAM68			0x4402201d
+#define RT722_MIC_CAE_PARAM39			0x44042000
+#define RT722_MIC_CAE_PARAM95			0x44042019
+#define RT722_MIC_CAE_PARAM96			0x4404201a
+#define RT722_MIC_CAE_PARAM99			0x4404201d
+
 /* RT722 SDCA Control - function number */
 #define FUNC_NUM_JACK_CODEC			0x01
 #define FUNC_NUM_MIC_ARRAY			0x02
diff --git a/sound/soc/codecs/sigmadsp.c b/sound/soc/codecs/sigmadsp.c
index 3343dbb63d2b7..2e08fde3989c2 100644
--- a/sound/soc/codecs/sigmadsp.c
+++ b/sound/soc/codecs/sigmadsp.c
@@ -35,7 +35,7 @@ struct sigmadsp_control {
 	struct snd_kcontrol *kcontrol;
 	bool is_readback;
 	bool cached;
-	uint8_t cache[];
+	u8 cache[] __counted_by(num_bytes);
 };
 
 struct sigmadsp_data {
@@ -223,7 +223,7 @@ static int sigma_fw_load_control(struct sigmadsp *sigmadsp,
 		return -EINVAL;
 
 	num_bytes = le16_to_cpu(ctrl_chunk->num_bytes);
-	ctrl = kzalloc(sizeof(*ctrl) + num_bytes, GFP_KERNEL);
+	ctrl = kzalloc_flex(*ctrl, cache, num_bytes);
 	if (!ctrl)
 		return -ENOMEM;
 
diff --git a/sound/soc/codecs/simple-amplifier.c b/sound/soc/codecs/simple-amplifier.c
index d306c585b52b0..ca0e6ce8cc379 100644
--- a/sound/soc/codecs/simple-amplifier.c
+++ b/sound/soc/codecs/simple-amplifier.c
@@ -1,25 +1,101 @@
 // SPDX-License-Identifier: GPL-2.0-only
 /*
- * Copyright (c) 2017 BayLibre, SAS.
- * Author: Jerome Brunet <jbrunet@baylibre.com>
+ * Support for gpio amplifier
+ *   Copyright 2026 CS GROUP France
+ *   Author: Herve Codina <herve.codina@bootlin.com>
+ *
+ * Basic simple amplifier driver
+ *   Copyright (c) 2017 BayLibre, SAS.
+ *   Author: Jerome Brunet <jbrunet@baylibre.com>
  */
 
+#include <linux/bitmap.h>
+#include <linux/bits.h>
 #include <linux/gpio/consumer.h>
+#include <linux/math.h>
+#include <linux/minmax.h>
+#include <linux/mod_devicetable.h>
 #include <linux/module.h>
+#include <linux/platform_device.h>
 #include <linux/regulator/consumer.h>
+#include <linux/slab.h>
 #include <sound/soc.h>
+#include <linux/sort.h>
+#include <sound/tlv.h>
 
-#define DRV_NAME "simple-amplifier"
+struct simple_amp_single {
+	struct gpio_desc *gpio;
+	bool is_inverted;
+	int kctrl_val;
+	const char *control_name;
+};
+
+struct simple_amp_point {
+	u32 gpio_val;
+	int gain_db;
+};
+
+struct simple_amp_range {
+	unsigned int nb_points;
+	struct simple_amp_point min;
+	struct simple_amp_point max;
+};
+
+struct simple_amp_ranges {
+	unsigned int nb_ranges;
+	struct simple_amp_range *tab_ranges;
+};
+
+struct simple_amp_labels {
+	unsigned int nb_labels;
+	const char **tab_labels;
+};
+
+enum simple_amp_mode {
+	SIMPLE_AMP_MODE_NONE,
+	SIMPLE_AMP_MODE_RANGES,
+	SIMPLE_AMP_MODE_LABELS,
+};
+
+struct simple_amp_multi {
+	struct gpio_descs *gpios;
+	u32 kctrl_val;
+	u32 kctrl_max;
+	const char *control_name;
+	unsigned int *tlv_array;
+	enum simple_amp_mode mode;
+	union {
+		struct simple_amp_ranges ranges;
+		struct simple_amp_labels labels;
+	};
+};
+
+struct simple_amp_data {
+	unsigned int supports;
+#define SIMPLE_AUDIO_SUPPORT_PGA		BIT(0)
+#define SIMPLE_AUDIO_SUPPORT_POWER_SUPPLIES	BIT(1)
+#define SIMPLE_AUDIO_SUPPORT_MUTE		BIT(2)
+#define SIMPLE_AUDIO_SUPPORT_BYPASS		BIT(3)
+
+	const struct snd_soc_dapm_widget *dapm_widgets;
+	unsigned int num_dapm_widgets;
+	const struct snd_soc_dapm_route *dapm_routes;
+	unsigned int num_dapm_routes;
+};
 
 struct simple_amp {
+	const struct simple_amp_data *data;
 	struct gpio_desc *gpiod_enable;
+	struct simple_amp_single mute;
+	struct simple_amp_single bypass;
+	struct simple_amp_multi gain;
 };
 
-static int drv_event(struct snd_soc_dapm_widget *w,
-		     struct snd_kcontrol *control, int event)
+static int simple_amp_power_event(struct snd_soc_dapm_widget *w,
+				  struct snd_kcontrol *control, int event)
 {
 	struct snd_soc_component *c = snd_soc_dapm_to_component(w->dapm);
-	struct simple_amp *priv = snd_soc_component_get_drvdata(c);
+	struct simple_amp *simple_amp = snd_soc_component_get_drvdata(c);
 	int val;
 
 	switch (event) {
@@ -34,7 +110,7 @@ static int drv_event(struct snd_soc_dapm_widget *w,
 		return -EINVAL;
 	}
 
-	gpiod_set_value_cansleep(priv->gpiod_enable, val);
+	gpiod_set_value_cansleep(simple_amp->gpiod_enable, val);
 
 	return 0;
 }
@@ -42,7 +118,7 @@ static int drv_event(struct snd_soc_dapm_widget *w,
 static const struct snd_soc_dapm_widget simple_amp_dapm_widgets[] = {
 	SND_SOC_DAPM_INPUT("INL"),
 	SND_SOC_DAPM_INPUT("INR"),
-	SND_SOC_DAPM_OUT_DRV_E("DRV", SND_SOC_NOPM, 0, 0, NULL, 0, drv_event,
+	SND_SOC_DAPM_OUT_DRV_E("DRV", SND_SOC_NOPM, 0, 0, NULL, 0, simple_amp_power_event,
 			       (SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD)),
 	SND_SOC_DAPM_OUTPUT("OUTL"),
 	SND_SOC_DAPM_OUTPUT("OUTR"),
@@ -58,47 +134,836 @@ static const struct snd_soc_dapm_route simple_amp_dapm_routes[] = {
 	{ "OUTR", NULL, "DRV" },
 };
 
+static const struct snd_soc_dapm_widget simple_amp_mono_pga_dapm_widgets[] = {
+	SND_SOC_DAPM_INPUT("IN"),
+	SND_SOC_DAPM_OUTPUT("OUT"),
+	SND_SOC_DAPM_PGA_E("PGA", SND_SOC_NOPM, 0, 0, NULL, 0, simple_amp_power_event,
+			   (SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD)),
+	SND_SOC_DAPM_REGULATOR_SUPPLY("vdd", 0, 0),
+};
+
+static const struct snd_soc_dapm_route simple_amp_mono_pga_dapm_routes[] = {
+	{ "PGA", NULL, "IN" },
+	{ "PGA", NULL, "vdd" },
+	{ "OUT", NULL, "PGA" },
+};
+
+static const struct snd_soc_dapm_widget simple_amp_stereo_pga_dapm_widgets[] = {
+	SND_SOC_DAPM_INPUT("INL"),
+	SND_SOC_DAPM_INPUT("INR"),
+	SND_SOC_DAPM_OUTPUT("OUTL"),
+	SND_SOC_DAPM_OUTPUT("OUTR"),
+	SND_SOC_DAPM_PGA_E("PGA", SND_SOC_NOPM, 0, 0, NULL, 0, simple_amp_power_event,
+			   (SND_SOC_DAPM_POST_PMU | SND_SOC_DAPM_PRE_PMD)),
+	SND_SOC_DAPM_REGULATOR_SUPPLY("vdd", 0, 0),
+};
+
+static const struct snd_soc_dapm_route simple_amp_stereo_pga_dapm_routes[] = {
+	{ "PGA", NULL, "INL" },
+	{ "PGA", NULL, "INR" },
+	{ "PGA", NULL, "vdd" },
+	{ "OUTL", NULL, "PGA" },
+	{ "OUTR", NULL, "PGA" },
+};
+
+static int simple_amp_single_kctrl_write_gpio(struct simple_amp_single *single,
+					      int kctrl_val)
+{
+	int gpio_val;
+
+	gpio_val = single->is_inverted ? !kctrl_val : kctrl_val;
+
+	return gpiod_set_value_cansleep(single->gpio, gpio_val);
+}
+
+static int simple_amp_single_kctrl_info(struct snd_kcontrol *kcontrol,
+					struct snd_ctl_elem_info *uinfo)
+{
+	uinfo->count = 1;
+	uinfo->value.integer.min = 0;
+	uinfo->value.integer.max = 1;
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_BOOLEAN;
+	return 0;
+}
+
+static int simple_amp_single_kctrl_get(struct snd_kcontrol *kcontrol,
+				       struct snd_ctl_elem_value *ucontrol)
+{
+	struct simple_amp_single *single = (struct simple_amp_single *)kcontrol->private_value;
+
+	ucontrol->value.integer.value[0] = single->kctrl_val;
+
+	return 0;
+}
+
+static int simple_amp_single_kctrl_put(struct snd_kcontrol *kcontrol,
+				       struct snd_ctl_elem_value *ucontrol)
+{
+	struct simple_amp_single *single = (struct simple_amp_single *)kcontrol->private_value;
+	int kctrl_val;
+	int err;
+
+	kctrl_val = ucontrol->value.integer.value[0] ? 1 : 0;
+
+	if (kctrl_val == single->kctrl_val)
+		return 0;
+
+	err = simple_amp_single_kctrl_write_gpio(single, kctrl_val);
+	if (err)
+		return err;
+
+	single->kctrl_val = kctrl_val;
+
+	return 1; /* The value changed */
+}
+
+static int simple_amp_single_add_kcontrol(struct snd_soc_component *component,
+					  struct simple_amp_single *single)
+{
+	struct snd_kcontrol_new control = {
+		.iface = SNDRV_CTL_ELEM_IFACE_MIXER,
+		.name = single->control_name,
+		.info = simple_amp_single_kctrl_info,
+		.get = simple_amp_single_kctrl_get,
+		.put = simple_amp_single_kctrl_put,
+		.private_value = (unsigned long)single,
+	};
+	int ret;
+
+	/* Be consistent between single->kctrl_val value and the GPIO value */
+	ret = simple_amp_single_kctrl_write_gpio(single, single->kctrl_val);
+	if (ret)
+		return ret;
+
+	return snd_soc_add_component_controls(component, &control, 1);
+}
+
+static u32 simple_amp_multi_ranges_kctrl_to_gpio(u32 kctrl_val,
+						 struct simple_amp_ranges *ranges)
+{
+	struct simple_amp_range *range;
+	u32 index = kctrl_val;
+	unsigned int i;
+
+	for (i = 0; i < ranges->nb_ranges; i++) {
+		range = &ranges->tab_ranges[i];
+
+		if (index < range->nb_points)
+			return (range->max.gpio_val >= range->min.gpio_val) ?
+				range->min.gpio_val + index :
+				range->min.gpio_val - index;
+
+		index -= range->nb_points;
+	}
+
+	/*
+	 * Given index out of possible ranges. This is shouldn't happen.
+	 * Signal the issue and return the maximum value
+	 */
+	WARN(1, "kctrl_val %u out of ranges\n", kctrl_val);
+	return ranges->tab_ranges[ranges->nb_ranges - 1].max.gpio_val;
+}
+
+static int simple_amp_multi_kctrl_write_gpios(struct simple_amp_multi *multi,
+					      u32 kctrl_val)
+{
+	DECLARE_BITMAP(bm, 32);
+	u32 gpio_val;
+
+	if (kctrl_val > multi->kctrl_max)
+		return -EINVAL;
+
+	if (multi->mode == SIMPLE_AMP_MODE_RANGES)
+		gpio_val = simple_amp_multi_ranges_kctrl_to_gpio(kctrl_val,
+								 &multi->ranges);
+	else
+		gpio_val = kctrl_val;
+
+	bitmap_from_arr32(bm, &gpio_val, multi->gpios->ndescs);
+
+	return gpiod_multi_set_value_cansleep(multi->gpios, bm);
+}
+
+static int simple_amp_multi_kctrl_int_info(struct snd_kcontrol *kcontrol,
+					   struct snd_ctl_elem_info *uinfo)
+{
+	struct simple_amp_multi *multi = (struct simple_amp_multi *)kcontrol->private_value;
+
+	uinfo->count = 1;
+	uinfo->value.integer.min = 0;
+	uinfo->value.integer.max = multi->kctrl_max;
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+	return 0;
+}
+
+static int simple_amp_multi_kctrl_int_get(struct snd_kcontrol *kcontrol,
+					  struct snd_ctl_elem_value *ucontrol)
+{
+	struct simple_amp_multi *multi = (struct simple_amp_multi *)kcontrol->private_value;
+
+	ucontrol->value.integer.value[0] = multi->kctrl_val;
+	return 0;
+}
+
+static int simple_amp_multi_kctrl_int_put(struct snd_kcontrol *kcontrol,
+					  struct snd_ctl_elem_value *ucontrol)
+{
+	struct simple_amp_multi *multi = (struct simple_amp_multi *)kcontrol->private_value;
+	u32 kctrl_val;
+	int ret;
+
+	kctrl_val = ucontrol->value.integer.value[0];
+
+	if (kctrl_val == multi->kctrl_val)
+		return 0;
+
+	ret = simple_amp_multi_kctrl_write_gpios(multi, kctrl_val);
+	if (ret)
+		return ret;
+
+	multi->kctrl_val = kctrl_val;
+
+	return 1; /* The value changed */
+}
+
+static int simple_amp_multi_kctrl_enum_info(struct snd_kcontrol *kcontrol,
+					    struct snd_ctl_elem_info *uinfo)
+{
+	struct simple_amp_multi *multi = (struct simple_amp_multi *)kcontrol->private_value;
+
+	return snd_ctl_enum_info(uinfo, 1, multi->labels.nb_labels,
+				 multi->labels.tab_labels);
+}
+
+static int simple_amp_multi_kctrl_enum_get(struct snd_kcontrol *kcontrol,
+					   struct snd_ctl_elem_value *ucontrol)
+{
+	struct simple_amp_multi *multi = (struct simple_amp_multi *)kcontrol->private_value;
+
+	ucontrol->value.enumerated.item[0] = multi->kctrl_val;
+	return 0;
+}
+
+static int simple_amp_multi_kctrl_enum_put(struct snd_kcontrol *kcontrol,
+					   struct snd_ctl_elem_value *ucontrol)
+{
+	struct simple_amp_multi *multi = (struct simple_amp_multi *)kcontrol->private_value;
+	u32 kctrl_val;
+	int ret;
+
+	kctrl_val = ucontrol->value.enumerated.item[0];
+
+	if (kctrl_val == multi->kctrl_val)
+		return 0;
+
+	ret = simple_amp_multi_kctrl_write_gpios(multi, kctrl_val);
+	if (ret)
+		return ret;
+
+	multi->kctrl_val = kctrl_val;
+
+	return 1; /* The value changed */
+}
+
+static unsigned int *simple_amp_alloc_tlv_ranges(const struct simple_amp_ranges *ranges)
+{
+	unsigned int index;
+	unsigned int *tlv;
+	unsigned int *t;
+	unsigned int i;
+
+	tlv = kzalloc_objs(*tlv, 2 + ranges->nb_ranges * 6, GFP_KERNEL);
+	if (!tlv)
+		return NULL;
+
+	t = tlv;
+
+	/* Fill first TLV */
+	*t++ = SNDRV_CTL_TLVT_DB_RANGE; /* Tag */
+	*t++ = ranges->nb_ranges * 6 * sizeof(*tlv); /* Len */
+	/* Ranges are sorted from lower to higher value */
+	index = 0;
+	for (i = 0; i < ranges->nb_ranges; i++) {
+		/* Fill range item i */
+		*t++ = index;  /* min */
+		index += ranges->tab_ranges[i].nb_points;
+		*t++ = index - 1;  /* max */
+		*t++ = SNDRV_CTL_TLVT_DB_MINMAX; /* Tag */
+		*t++ = 2 * sizeof(*tlv); /* Len */
+		*t++ = ranges->tab_ranges[i].min.gain_db; /* min_dB */
+		*t++ = ranges->tab_ranges[i].max.gain_db; /* max_dB */
+	}
+
+	return tlv;
+}
+
+static int simple_amp_multi_add_kcontrol(struct snd_soc_component *component,
+					 struct simple_amp_multi *multi)
+{
+	struct snd_kcontrol_new control = {
+		.iface = SNDRV_CTL_ELEM_IFACE_MIXER,
+		.name = multi->control_name,
+		.info = simple_amp_multi_kctrl_int_info,
+		.get = simple_amp_multi_kctrl_int_get,
+		.put = simple_amp_multi_kctrl_int_put,
+		.private_value = (unsigned long)multi,
+	};
+	int ret;
+
+	switch (multi->mode) {
+	case SIMPLE_AMP_MODE_RANGES:
+		multi->tlv_array = simple_amp_alloc_tlv_ranges(&multi->ranges);
+		if (!multi->tlv_array)
+			return -ENOMEM;
+
+		control.access = SNDRV_CTL_ELEM_ACCESS_TLV_READ |
+				 SNDRV_CTL_ELEM_ACCESS_READWRITE;
+		control.tlv.p = multi->tlv_array;
+		break;
+
+	case SIMPLE_AMP_MODE_LABELS:
+		/* Use enumerated values */
+		control.info = simple_amp_multi_kctrl_enum_info;
+		control.get = simple_amp_multi_kctrl_enum_get;
+		control.put = simple_amp_multi_kctrl_enum_put;
+		break;
+
+	case SIMPLE_AMP_MODE_NONE:
+		/* Already set control configuration is enough */
+		break;
+
+	default:
+		return -EINVAL;
+	}
+
+	/* Be consistent between multi->kctrl_val value and the GPIOs value */
+	ret = simple_amp_multi_kctrl_write_gpios(multi, multi->kctrl_val);
+	if (ret)
+		goto err_free_tlv_array;
+
+	ret = snd_soc_add_component_controls(component, &control, 1);
+	if (ret)
+		goto err_free_tlv_array;
+
+	return 0;
+
+err_free_tlv_array:
+	kfree(multi->tlv_array);
+	return ret;
+}
+
+static int simple_amp_add_basic_dapm(struct snd_soc_component *component)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component);
+	struct simple_amp *simple_amp = snd_soc_component_get_drvdata(component);
+	struct device *dev = component->dev;
+	int ret;
+
+	/* Add basic dapm widgets and routes */
+	ret = snd_soc_dapm_new_controls(dapm, simple_amp->data->dapm_widgets,
+					simple_amp->data->num_dapm_widgets);
+	if (ret) {
+		dev_err(dev, "Failed to add basic dapm widgets (%d)\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_dapm_add_routes(dapm, simple_amp->data->dapm_routes,
+				      simple_amp->data->num_dapm_routes);
+	if (ret) {
+		dev_err(dev, "Failed to add basic dapm routes (%d)\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+struct simple_amp_supply {
+	const char *prop_name;
+	const struct snd_soc_dapm_widget dapm_widget;
+	const struct snd_soc_dapm_route dapm_route;
+};
+
+static const struct simple_amp_supply simple_amp_supplies[] = {
+	{
+		.prop_name = "vddio-supply",
+		.dapm_widget = SND_SOC_DAPM_REGULATOR_SUPPLY("vddio", 0, 0),
+		.dapm_route = { "PGA", NULL, "vddio" },
+	}, {
+		.prop_name = "vdda1-supply",
+		.dapm_widget = SND_SOC_DAPM_REGULATOR_SUPPLY("vdda1", 0, 0),
+		.dapm_route = { "PGA", NULL, "vdda1" },
+	}, {
+		.prop_name = "vdda2-supply",
+		.dapm_widget = SND_SOC_DAPM_REGULATOR_SUPPLY("vdda2", 0, 0),
+		.dapm_route = { "PGA", NULL, "vdda2" },
+	},
+	{ /* End of list */}
+};
+
+static int simple_amp_add_power_supplies(struct snd_soc_component *component)
+{
+	struct snd_soc_dapm_context *dapm = snd_soc_component_to_dapm(component);
+	struct simple_amp *simple_amp = snd_soc_component_get_drvdata(component);
+	const struct simple_amp_supply *supply;
+	struct device *dev = component->dev;
+	int ret;
+
+	/*
+	 * Those additional power supplies are attached to the PGA.
+	 * If PGA is not supported, simply skipped them.
+	 */
+	if (!(simple_amp->data->supports & SIMPLE_AUDIO_SUPPORT_PGA)) {
+		dev_err(dev, "Extra power supplied need PGA\n");
+		return -EINVAL;
+	}
+
+	supply = simple_amp_supplies;
+	do {
+		if (!of_property_present(dev->of_node, supply->prop_name))
+			continue;
+
+		ret = snd_soc_dapm_new_controls(dapm, &supply->dapm_widget, 1);
+		if (ret) {
+			dev_err(dev, "Failed to add control for '%s' (%d)\n",
+				supply->prop_name, ret);
+			return ret;
+		}
+		ret = snd_soc_dapm_add_routes(dapm, &supply->dapm_route, 1);
+		if (ret) {
+			dev_err(dev, "Failed to add route for '%s' (%d)\n",
+				supply->prop_name, ret);
+			return ret;
+		}
+	} while ((++supply)->prop_name);
+
+	return 0;
+}
+
+static int simple_amp_component_probe(struct snd_soc_component *component)
+{
+	struct simple_amp *simple_amp = snd_soc_component_get_drvdata(component);
+	int ret;
+
+	/* Add basic dapm widgets and routes */
+	ret = simple_amp_add_basic_dapm(component);
+	if (ret)
+		return ret;
+
+	/* Add additional power supplies */
+	if (simple_amp->data->supports & SIMPLE_AUDIO_SUPPORT_POWER_SUPPLIES) {
+		ret = simple_amp_add_power_supplies(component);
+		if (ret)
+			return ret;
+	}
+
+	if (simple_amp->mute.gpio) {
+		/*
+		 * The name of the GPIO used is mute. According to this name, 1
+		 * means muted and 0 means un-muted.
+		 *
+		 * An inversion is expected by ALSA. Indeed from ALSA point of
+		 * view, 1 means 'on' (un-muted) and 0 means 'off' (muted).
+		 */
+		simple_amp->mute.is_inverted = true;
+		simple_amp->mute.kctrl_val = 1; /* Un-muted */
+		ret = simple_amp_single_add_kcontrol(component, &simple_amp->mute);
+		if (ret)
+			return ret;
+	}
+
+	if (simple_amp->bypass.gpio) {
+		ret = simple_amp_single_add_kcontrol(component, &simple_amp->bypass);
+		if (ret)
+			return ret;
+	}
+
+	if (simple_amp->gain.gpios) {
+		ret = simple_amp_multi_add_kcontrol(component, &simple_amp->gain);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static void simple_amp_component_remove(struct snd_soc_component *component)
+{
+	struct simple_amp *simple_amp = snd_soc_component_get_drvdata(component);
+
+	kfree(simple_amp->gain.tlv_array);
+	simple_amp->gain.tlv_array = NULL;
+}
+
 static const struct snd_soc_component_driver simple_amp_component_driver = {
-	.dapm_widgets		= simple_amp_dapm_widgets,
-	.num_dapm_widgets	= ARRAY_SIZE(simple_amp_dapm_widgets),
-	.dapm_routes		= simple_amp_dapm_routes,
-	.num_dapm_routes	= ARRAY_SIZE(simple_amp_dapm_routes),
+	.probe = simple_amp_component_probe,
+	.remove = simple_amp_component_remove,
 };
 
+static int simple_amp_parse_single_gpio(struct device *dev,
+					struct simple_amp_single *single,
+					const char *gpio_property)
+{
+	/* Start with the inactive value */
+	single->is_inverted = false;
+	single->kctrl_val = 0;
+	single->gpio = devm_gpiod_get_optional(dev, gpio_property, GPIOD_OUT_LOW);
+	if (IS_ERR(single->gpio))
+		return dev_err_probe(dev, PTR_ERR(single->gpio),
+				     "Failed to get '%s' gpio\n",
+				     gpio_property);
+	return 0;
+}
+
+static int simple_amp_cmp_ranges(const void *a, const void *b)
+{
+	const struct simple_amp_range *a_range = a;
+	const struct simple_amp_range *b_range = b;
+
+	/* Ranges a and b don't overlap. This has been already checked */
+
+	return a_range->min.gain_db - b_range->max.gain_db;
+}
+
+static int simple_amp_check_new_range(const struct simple_amp_range *new_range,
+				      const struct simple_amp_range *tab_ranges,
+				      unsigned int nb_ranges)
+{
+	unsigned int i;
+
+	for (i = 0; i < nb_ranges; i++) {
+		/* Check for range overlaps */
+		if (new_range->min.gain_db >= tab_ranges[i].min.gain_db &&
+		    new_range->min.gain_db <= tab_ranges[i].max.gain_db)
+			return -EINVAL;
+
+		if (new_range->max.gain_db >= tab_ranges[i].min.gain_db &&
+		    new_range->max.gain_db <= tab_ranges[i].max.gain_db)
+			return -EINVAL;
+
+		if (new_range->min.gain_db <= tab_ranges[i].min.gain_db &&
+		    new_range->max.gain_db >= tab_ranges[i].max.gain_db)
+			return -EINVAL;
+	}
+	return 0;
+}
+
+static int simple_amp_parse_ranges(struct device *dev,
+				   struct simple_amp_multi *multi,
+				   const char *ranges_property)
+{
+	struct simple_amp_ranges *ranges = &multi->ranges;
+	struct simple_amp_range *range;
+	struct device_node *np = dev->of_node;
+	struct simple_amp_point first_point;
+	unsigned int max_gpio_val;
+	unsigned int i;
+	int ret;
+	u32 u;
+	s32 s;
+
+	max_gpio_val = (1 << multi->gpios->ndescs) - 1;
+
+	ret = of_property_count_u32_elems(np, ranges_property);
+	if (ret < 0)
+		return ret;
+
+	/* The ranges array cannot be empty */
+	if (ret == 0)
+		return -EINVAL;
+	/*
+	 * One range item is composed of 2 points and each point is composed of
+	 * 2 values.
+	 */
+	if (ret % 4)
+		return -EINVAL;
+
+	ranges->nb_ranges = ret / 4;
+
+	/* The worst case is one range per possible gpio value */
+	if (ranges->nb_ranges > max_gpio_val + 1)
+		return -EINVAL;
+
+	ranges->tab_ranges = devm_kcalloc(dev, ranges->nb_ranges,
+					  sizeof(*ranges->tab_ranges),
+					  GFP_KERNEL);
+	if (!ranges->tab_ranges)
+		return -ENOMEM;
+
+	multi->kctrl_max = 0;
+	for (i = 0; i < ranges->nb_ranges; i++) {
+		range = &ranges->tab_ranges[i];
+
+		/* First gpios value */
+		ret = of_property_read_u32_index(np, ranges_property, i * 4, &u);
+		if (ret)
+			return ret;
+		if (u > max_gpio_val)
+			return -EINVAL;
+
+		range->min.gpio_val = u;
+
+		/* First Gain value */
+		ret = of_property_read_s32_index(np, ranges_property, i * 4 + 1, &s);
+		if (ret)
+			return ret;
+
+		range->min.gain_db = s;
+
+		/* Second gpios value */
+		ret = of_property_read_u32_index(np, ranges_property, i * 4 + 2, &u);
+		if (ret)
+			return ret;
+		if (u > max_gpio_val)
+			return -EINVAL;
+
+		range->max.gpio_val = u;
+
+		/* Second Gain value */
+		ret = of_property_read_s32_index(np, ranges_property, i * 4 + 3, &s);
+		if (ret)
+			return ret;
+
+		range->max.gain_db = s;
+
+		/* Save the first point for later usage */
+		if (i == 0)
+			first_point = range->min;
+
+		/* Fix min and max if needed */
+		if (range->min.gain_db > range->max.gain_db)
+			swap(range->min, range->max);
+
+		ret = simple_amp_check_new_range(range, ranges->tab_ranges, i);
+		if (ret)
+			return ret;
+
+		range->nb_points = abs_diff(range->min.gpio_val,
+					    range->max.gpio_val) + 1;
+
+		multi->kctrl_max += range->nb_points;
+	}
+
+	multi->kctrl_max -= 1;
+
+	/* Sort the tab_range array by gain_db value */
+	sort(ranges->tab_ranges, ranges->nb_ranges, sizeof(*ranges->tab_ranges),
+	     simple_amp_cmp_ranges, NULL);
+
+	/*
+	 * multi->kctrl_val is the index in tab_ranges.
+	 *
+	 * Choose to have the initial amplification value set to the first point
+	 * available in the first range available in the tab_ranges array before
+	 * sorting.
+	 *
+	 * This first point has been identified before sorting. Search for it in
+	 * the sorted array in order to set the multi->kctrl_val initial value.
+	 */
+	multi->kctrl_val = 0;
+	for (i = 0; i < ranges->nb_ranges; i++) {
+		range = &ranges->tab_ranges[i];
+
+		if (range->min.gpio_val == first_point.gpio_val &&
+		    range->min.gain_db == first_point.gain_db)
+			break;
+
+		multi->kctrl_val += range->nb_points;
+
+		if (range->max.gpio_val == first_point.gpio_val &&
+		    range->max.gain_db == first_point.gain_db) {
+			multi->kctrl_val--;
+			break;
+		}
+	}
+
+	return 0;
+}
+
+static int simple_amp_parse_labels(struct device *dev,
+				   struct simple_amp_multi *multi,
+				   const char *labels_property)
+{
+	struct simple_amp_labels *labels = &multi->labels;
+	struct device_node *np = dev->of_node;
+	int ret;
+
+	ret = of_property_count_strings(np, labels_property);
+	if (ret < 0)
+		return ret;
+
+	/* The labels array cannot be empty */
+	if (ret == 0)
+		return -EINVAL;
+
+	labels->nb_labels = ret;
+	if (labels->nb_labels > (1 << multi->gpios->ndescs))
+		return -EINVAL;
+
+	labels->tab_labels = devm_kcalloc(dev, labels->nb_labels,
+					  sizeof(*labels->tab_labels),
+					  GFP_KERNEL);
+	if (!labels->tab_labels)
+		return -ENOMEM;
+
+	multi->kctrl_max = labels->nb_labels - 1;
+	multi->kctrl_val = 0;
+
+	return of_property_read_string_array(np, labels_property, labels->tab_labels,
+					     labels->nb_labels);
+}
+
+static int simple_amp_parse_multi_gpio(struct device *dev,
+				       struct simple_amp_multi *multi,
+				       const char *gpios_property,
+				       const char *ranges_property,
+				       const char *labels_property)
+{
+	struct device_node *np = dev->of_node;
+	int ret;
+
+	/* Start with the value 0 (GPIO inactive). Can be changed later */
+	multi->kctrl_val = 0;
+	multi->gpios = devm_gpiod_get_array_optional(dev, gpios_property, GPIOD_OUT_LOW);
+	if (IS_ERR(multi->gpios))
+		return dev_err_probe(dev, PTR_ERR(multi->gpios),
+				     "Failed to get '%s' gpios\n",
+				     gpios_property);
+	if (!multi->gpios)
+		return 0;
+
+	if (multi->gpios->ndescs > 16)
+		return dev_err_probe(dev, -EINVAL,
+				     "Number of '%s' gpios limited to 16\n",
+				     gpios_property);
+
+	/* Set default value for the kctrl_max. Can be changed later */
+	multi->kctrl_max = (1 << multi->gpios->ndescs) - 1;
+
+	multi->mode = SIMPLE_AMP_MODE_NONE;
+	if (of_property_present(np, ranges_property)) {
+		ret = simple_amp_parse_ranges(dev, multi, ranges_property);
+		if (ret < 0)
+			return dev_err_probe(dev, ret, "Failed to parse '%s'\n",
+					     ranges_property);
+		multi->mode = SIMPLE_AMP_MODE_RANGES;
+	} else if (of_property_present(np, labels_property)) {
+		ret = simple_amp_parse_labels(dev, multi, labels_property);
+		if (ret < 0)
+			return dev_err_probe(dev, ret, "Failed to parse '%s'\n",
+					     labels_property);
+
+		multi->mode = SIMPLE_AMP_MODE_LABELS;
+	}
+
+	return 0;
+}
+
 static int simple_amp_probe(struct platform_device *pdev)
 {
 	struct device *dev = &pdev->dev;
-	struct simple_amp *priv;
+	struct simple_amp *simple_amp;
+	int ret;
 
-	priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
-	if (priv == NULL)
+	simple_amp = devm_kzalloc(dev, sizeof(*simple_amp), GFP_KERNEL);
+	if (!simple_amp)
 		return -ENOMEM;
-	platform_set_drvdata(pdev, priv);
+	platform_set_drvdata(pdev, simple_amp);
 
-	priv->gpiod_enable = devm_gpiod_get_optional(dev, "enable",
-						     GPIOD_OUT_LOW);
-	if (IS_ERR(priv->gpiod_enable))
-		return dev_err_probe(dev, PTR_ERR(priv->gpiod_enable),
+	simple_amp->data = of_device_get_match_data(dev);
+	if (!simple_amp->data)
+		return -EINVAL;
+
+	simple_amp->gpiod_enable = devm_gpiod_get_optional(dev, "enable",
+							   GPIOD_OUT_LOW);
+	if (IS_ERR(simple_amp->gpiod_enable))
+		return dev_err_probe(dev, PTR_ERR(simple_amp->gpiod_enable),
 				     "Failed to get 'enable' gpio");
 
+	if (simple_amp->data->supports & SIMPLE_AUDIO_SUPPORT_MUTE) {
+		ret = simple_amp_parse_single_gpio(dev, &simple_amp->mute, "mute");
+		if (ret)
+			return ret;
+	}
+
+	if (simple_amp->data->supports & SIMPLE_AUDIO_SUPPORT_BYPASS) {
+		ret = simple_amp_parse_single_gpio(dev, &simple_amp->bypass, "bypass");
+		if (ret)
+			return ret;
+	}
+
+	if (simple_amp->data->supports & SIMPLE_AUDIO_SUPPORT_PGA) {
+		ret = simple_amp_parse_multi_gpio(dev, &simple_amp->gain, "gain",
+						  "gain-ranges", "gain-labels");
+		if (ret)
+			return ret;
+	}
+
+	/* Set controls name */
+	simple_amp->gain.control_name = "Volume";
+	simple_amp->mute.control_name = "Switch";
+	simple_amp->bypass.control_name = "Bypass Switch";
+
+	if (simple_amp->gain.mode == SIMPLE_AMP_MODE_LABELS) {
+		/*
+		 * The gain widget control will use enumerated values.
+		 *
+		 * Having just "Voltage" and "Switch" widget names with
+		 * enumerated values and boolean value can confuse ALSA in terms
+		 * of possible values (strings).
+		 *
+		 * Make things clear and avoid the just "Switch" name in that
+		 * case.
+		 */
+		simple_amp->mute.control_name = "Out Switch";
+	}
+
 	return devm_snd_soc_register_component(dev,
 					       &simple_amp_component_driver,
 					       NULL, 0);
 }
 
-#ifdef CONFIG_OF
+static const struct simple_amp_data simple_audio_amplifier_data = {
+	.dapm_widgets		= simple_amp_dapm_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(simple_amp_dapm_widgets),
+	.dapm_routes		= simple_amp_dapm_routes,
+	.num_dapm_routes	= ARRAY_SIZE(simple_amp_dapm_routes),
+};
+
+static const struct simple_amp_data simple_audio_mono_pga_data = {
+	.supports		= SIMPLE_AUDIO_SUPPORT_PGA |
+				  SIMPLE_AUDIO_SUPPORT_POWER_SUPPLIES |
+				  SIMPLE_AUDIO_SUPPORT_MUTE |
+				  SIMPLE_AUDIO_SUPPORT_BYPASS,
+	.dapm_widgets		= simple_amp_mono_pga_dapm_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(simple_amp_mono_pga_dapm_widgets),
+	.dapm_routes		= simple_amp_mono_pga_dapm_routes,
+	.num_dapm_routes	= ARRAY_SIZE(simple_amp_mono_pga_dapm_routes),
+};
+
+static const struct simple_amp_data simple_audio_stereo_pga_data = {
+	.supports		= SIMPLE_AUDIO_SUPPORT_PGA |
+				  SIMPLE_AUDIO_SUPPORT_POWER_SUPPLIES |
+				  SIMPLE_AUDIO_SUPPORT_MUTE |
+				  SIMPLE_AUDIO_SUPPORT_BYPASS,
+	.dapm_widgets		= simple_amp_stereo_pga_dapm_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(simple_amp_stereo_pga_dapm_widgets),
+	.dapm_routes		= simple_amp_stereo_pga_dapm_routes,
+	.num_dapm_routes	= ARRAY_SIZE(simple_amp_stereo_pga_dapm_routes),
+};
+
 static const struct of_device_id simple_amp_ids[] = {
-	{ .compatible = "dioo,dio2125", },
-	{ .compatible = "simple-audio-amplifier", },
+	{ .compatible = "dioo,dio2125",		  .data = &simple_audio_amplifier_data},
+	{ .compatible = "simple-audio-amplifier", .data = &simple_audio_amplifier_data},
+	{ .compatible = "gpio-audio-amp-mono",	  .data = &simple_audio_mono_pga_data},
+	{ .compatible = "gpio-audio-amp-stereo",  .data = &simple_audio_stereo_pga_data},
 	{ }
 };
 MODULE_DEVICE_TABLE(of, simple_amp_ids);
-#endif
 
 static struct platform_driver simple_amp_driver = {
 	.driver = {
-		.name = DRV_NAME,
-		.of_match_table = of_match_ptr(simple_amp_ids),
+		.name = "simple-amplifier",
+		.of_match_table = simple_amp_ids,
 	},
 	.probe = simple_amp_probe,
 };
@@ -107,4 +972,5 @@ module_platform_driver(simple_amp_driver);
 
 MODULE_DESCRIPTION("ASoC Simple Audio Amplifier driver");
 MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>");
+MODULE_AUTHOR("Herve Codina <herve.codina@bootlin.com>");
 MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/tac5xx2-sdw.c b/sound/soc/codecs/tac5xx2-sdw.c
new file mode 100644
index 0000000000000..bb12cfb6da12b
--- /dev/null
+++ b/sound/soc/codecs/tac5xx2-sdw.c
@@ -0,0 +1,2037 @@
+// SPDX-License-Identifier: GPL-2.0
+//
+// ALSA SoC Texas Instruments TAC5XX2 Audio Smart Amplifier
+//
+// Copyright (C) 2025 Texas Instruments Incorporated
+// https://www.ti.com
+//
+// Author: Niranjan H Y <niranjan.hy@ti.com>
+
+#include <linux/err.h>
+#include <linux/firmware.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pm.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+#include <linux/soundwire/sdw.h>
+#include <linux/soundwire/sdw_registers.h>
+#include <linux/soundwire/sdw_type.h>
+#include <linux/time.h>
+#include <linux/unaligned.h>
+#include <sound/pcm_params.h>
+#include <sound/sdw.h>
+#include <sound/soc.h>
+#include <sound/tlv.h>
+#include <sound/sdca_asoc.h>
+#include <sound/sdca_function.h>
+#include <sound/sdca_regmap.h>
+#include <sound/jack.h>
+
+#include "tac5xx2.h"
+
+#define TAC5XX2_PROBE_TIMEOUT_MS 3000
+#define TAC5XX2_FW_CACHE_TIMEOUT_MS 300
+
+#define TAC5XX2_DEVICE_RATES (SNDRV_PCM_RATE_44100 | \
+			      SNDRV_PCM_RATE_48000 | \
+			      SNDRV_PCM_RATE_96000 | \
+			      SNDRV_PCM_RATE_88200)
+#define TAC5XX2_DEVICE_FORMATS (SNDRV_PCM_FMTBIT_S16_LE | \
+				SNDRV_PCM_FMTBIT_S24_LE | \
+				SNDRV_PCM_FMTBIT_S32_LE)
+/* Define channel constants */
+#define TAC_CHANNEL_LEFT	1
+#define TAC_CHANNEL_RIGHT	2
+#define TAC_JACK_MONO_CS	2
+
+#define TAC_MUTE_REG(func, fu, ch) \
+	SDW_SDCA_CTL(TAC_FUNCTION_ID_##func, TAC_SDCA_ENT_##fu, \
+		     TAC_SDCA_CHANNEL_MUTE, TAC_CHANNEL_##ch)
+#define TAC_USAGE_REG(func, ent) \
+	SDW_SDCA_CTL(TAC_FUNCTION_ID_##func, TAC_SDCA_ENT_##ent, \
+		     TAC_SDCA_CTL_USAGE, 0)
+#define TAC_XU_BYPASS_REG(func, xu)                        \
+	SDW_SDCA_CTL(TAC_FUNCTION_ID_##func, TAC_SDCA_ENT_##xu, \
+			TAC_SDCA_CTL_XU_BYPASS, 0)
+
+/* mute registers */
+#define FU21_L_MUTE_REG  TAC_MUTE_REG(SA, FU21, LEFT)
+#define FU21_R_MUTE_REG  TAC_MUTE_REG(SA, FU21, RIGHT)
+#define FU23_L_MUTE_REG  TAC_MUTE_REG(SA, FU23, LEFT)
+#define FU23_R_MUTE_REG  TAC_MUTE_REG(SA, FU23, RIGHT)
+#define FU26_MUTE_REG    TAC_MUTE_REG(SA, FU26, LEFT)
+#define FU11_L_MUTE_REG  TAC_MUTE_REG(SM, FU11, LEFT)
+#define FU11_R_MUTE_REG  TAC_MUTE_REG(SM, FU11, RIGHT)
+#define FU113_L_MUTE_REG TAC_MUTE_REG(SM, FU113, LEFT)
+#define FU113_R_MUTE_REG TAC_MUTE_REG(SM, FU113, RIGHT)
+#define FU41_L_MUTE_REG  TAC_MUTE_REG(UAJ, FU41, LEFT)
+#define FU41_R_MUTE_REG  TAC_MUTE_REG(UAJ, FU41, RIGHT)
+#define FU36_MUTE_REG    TAC_MUTE_REG(UAJ, FU36, RIGHT)
+
+/* it/ot usage */
+#define IT11_USAGE_REG  TAC_USAGE_REG(SM, IT11)
+#define IT41_USAGE_REG  TAC_USAGE_REG(UAJ, IT41)
+#define IT33_USAGE_REG  TAC_USAGE_REG(UAJ, IT33)
+#define OT113_USAGE_REG TAC_USAGE_REG(SM, OT113)
+#define OT45_USAGE_REG TAC_USAGE_REG(UAJ, OT45)
+#define OT36_USAGE_REG TAC_USAGE_REG(UAJ, OT36)
+
+/* xu bypass */
+#define XU12_BYPASS_REG TAC_XU_BYPASS_REG(SM, XU12)
+#define XU42_BYPASS_REG TAC_XU_BYPASS_REG(UAJ, XU42)
+
+#define TAC_DSP_ALGO_STATUS		TAC_REG_SDW(0, 3, 12)
+#define TAC_DSP_ALGO_STATUS_RUNNING	0x20
+#define TAC_FW_HDR_SIZE		88
+#define TAC_FW_FILE_HDR	 20
+#define TAC_MAX_FW_CHUNKS 512
+
+struct tac_fw_hdr {
+	u32 size;
+	u32 version_offset;
+	u32 plt_id;
+	u32 ppc3_ver;
+	u64 timestamp;
+	u8 ddc_name[64];
+};
+
+/* Firmware file/chunk structure */
+struct tac_fw_file {
+	u32 vendor_id;
+	u32 file_id;
+	u32 version;
+	u32 length;
+	u32 dest_addr;
+	u8 *fw_data;
+};
+
+/* TLV for volume control */
+static const DECLARE_TLV_DB_SCALE(tac5xx2_amp_tlv, 0, 50, 0);
+static const DECLARE_TLV_DB_SCALE(tac5xx2_dvc_tlv, -7200, 50, 0);
+
+/* Q7.8 volume control parameters: range -72dB to +6dB, step 0.5dB */
+#define TAC_DVC_STEP		128	/* 0.5 dB in Q7.8 format */
+#define TAC_DVC_MIN		(-144)	/* -72 dB / 0.5 dB step */
+#define TAC_DVC_MAX		12	/* +6 dB / 0.5 dB step */
+
+/* TAC-specific stereo volume control macro using SDW_SDCA_CTL (single control for L/R) */
+#define TAC_DOUBLE_Q78_TLV(name, func_id, ent_id) \
+	SDCA_DOUBLE_Q78_TLV(name, \
+			    SDW_SDCA_CTL(TAC_FUNCTION_ID_##func_id, TAC_SDCA_ENT_##ent_id, \
+					 TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT), \
+			    SDW_SDCA_CTL(TAC_FUNCTION_ID_##func_id, TAC_SDCA_ENT_##ent_id, \
+					 TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT), \
+			    TAC_DVC_MIN, TAC_DVC_MAX, TAC_DVC_STEP, tac5xx2_dvc_tlv)
+
+struct tac5xx2_prv {
+	struct snd_soc_component *component;
+	struct sdw_slave *sdw_peripheral;
+	struct sdca_function_data *sa_func_data;
+	struct sdca_function_data *sm_func_data;
+	struct sdca_function_data *uaj_func_data;
+	struct sdca_function_data *hid_func_data;
+	enum sdw_slave_status status;
+	struct regmap *regmap;
+	struct device *dev;
+	bool hw_init;
+	bool first_hw_init_done;
+	u32 part_id;
+	struct snd_soc_jack *hs_jack;
+	int jack_type;
+	 /* Custom fw binary. UMP File Download is not used. */
+	unsigned int fw_file_cnt;
+	struct tac_fw_file *fw_files;
+	struct completion fw_caching_complete;
+	bool fw_dl_success;
+	u8 fw_binaryname[64];
+};
+
+static const struct reg_default tac_reg_default[] = {
+	{TAC_SW_RESET, 0x0},
+	{TAC_SLEEP_MODEZ, 0x0},
+	{TAC_FEATURE_PDZ, 0x0},
+	{TAC_TX_CH_EN, 0xf0},
+	{TAC_REG_SDW(0, 0, 0x5), 0xcf},
+	{TAC_REG_SDW(0, 0, 0x6), 0xa},
+	{TAC_REG_SDW(0, 0, 0x7), 0x0},
+	{TAC_REG_SDW(0, 0, 0x8), 0xfe},
+	{TAC_REG_SDW(0, 0, 0x9), 0x9},
+	{TAC_REG_SDW(0, 0, 0xa), 0x28},
+	{TAC_REG_SDW(0, 0, 0xb), 0x1},
+	{TAC_REG_SDW(0, 0, 0xc), 0x11},
+	{TAC_REG_SDW(0, 0, 0xd), 0x11},
+	{TAC_REG_SDW(0, 0, 0xe), 0x61},
+	{TAC_REG_SDW(0, 0, 0xf), 0x0},
+	{TAC_REG_SDW(0, 0, 0x10), 0x50},
+	{TAC_REG_SDW(0, 0, 0x11), 0x70},
+	{TAC_REG_SDW(0, 0, 0x12), 0x60},
+	{TAC_REG_SDW(0, 0, 0x13), 0x28},
+	{TAC_REG_SDW(0, 0, 0x14), 0x0},
+	{TAC_REG_SDW(0, 0, 0x15), 0x18},
+	{TAC_REG_SDW(0, 0, 0x16), 0x20},
+	{TAC_REG_SDW(0, 0, 0x17), 0x0},
+	{TAC_REG_SDW(0, 0, 0x18), 0x18},
+	{TAC_REG_SDW(0, 0, 0x19), 0x54},
+	{TAC_REG_SDW(0, 0, 0x1a), 0x8},
+	{TAC_REG_SDW(0, 0, 0x1b), 0x0},
+	{TAC_REG_SDW(0, 0, 0x1c), 0x30},
+	{TAC_REG_SDW(0, 0, 0x1d), 0x0},
+	{TAC_REG_SDW(0, 0, 0x1e), 0x0},
+	{TAC_REG_SDW(0, 0, 0x1f), 0x0},
+	{TAC_REG_SDW(0, 0, 0x20), 0x0},
+	{TAC_REG_SDW(0, 0, 0x21), 0x20},
+	{TAC_REG_SDW(0, 0, 0x22), 0x21},
+	{TAC_REG_SDW(0, 0, 0x23), 0x22},
+	{TAC_REG_SDW(0, 0, 0x24), 0x23},
+	{TAC_REG_SDW(0, 0, 0x25), 0x4},
+	{TAC_REG_SDW(0, 0, 0x26), 0x5},
+	{TAC_REG_SDW(0, 0, 0x27), 0x6},
+	{TAC_REG_SDW(0, 0, 0x28), 0x7},
+	{TAC_REG_SDW(0, 0, 0x29), 0x0},
+	{TAC_REG_SDW(0, 0, 0x2a), 0x0},
+	{TAC_REG_SDW(0, 0, 0x2b), 0x0},
+	{TAC_REG_SDW(0, 0, 0x2c), 0x20},
+	{TAC_REG_SDW(0, 0, 0x2d), 0x21},
+	{TAC_REG_SDW(0, 0, 0x2e), 0x2},
+	{TAC_REG_SDW(0, 0, 0x2f), 0x3},
+	{TAC_REG_SDW(0, 0, 0x30), 0x4},
+	{TAC_REG_SDW(0, 0, 0x31), 0x5},
+	{TAC_REG_SDW(0, 0, 0x32), 0x6},
+	{TAC_REG_SDW(0, 0, 0x33), 0x7},
+	{TAC_REG_SDW(0, 0, 0x34), 0x0},
+	{TAC_REG_SDW(0, 0, 0x35), 0x90},
+	{TAC_REG_SDW(0, 0, 0x36), 0x80},
+	{TAC_REG_SDW(0, 0, 0x37), 0x0},
+	{TAC_REG_SDW(0, 0, 0x39), 0x0},
+	{TAC_REG_SDW(0, 0, 0x3a), 0x90},
+	{TAC_REG_SDW(0, 0, 0x3b), 0x80},
+	{TAC_REG_SDW(0, 0, 0x3c), 0x0},
+	{TAC_REG_SDW(0, 0, 0x3e), 0x0},
+	{TAC_REG_SDW(0, 0, 0x3f), 0x90},
+	{TAC_REG_SDW(0, 0, 0x40), 0x80},
+	{TAC_REG_SDW(0, 0, 0x41), 0x0},
+	{TAC_REG_SDW(0, 0, 0x43), 0x90},
+	{TAC_REG_SDW(0, 0, 0x44), 0x80},
+	{TAC_REG_SDW(0, 0, 0x45), 0x0},
+	{TAC_REG_SDW(0, 0, 0x47), 0x90},
+	{TAC_REG_SDW(0, 0, 0x48), 0x80},
+	{TAC_REG_SDW(0, 0, 0x49), 0x0},
+	{TAC_REG_SDW(0, 0, 0x4b), 0x90},
+	{TAC_REG_SDW(0, 0, 0x4c), 0x80},
+	{TAC_REG_SDW(0, 0, 0x4d), 0x0},
+	{TAC_REG_SDW(0, 0, 0x4f), 0x31},
+	{TAC_REG_SDW(0, 0, 0x50), 0x0},
+	{TAC_REG_SDW(0, 0, 0x51), 0x0},
+	{TAC_REG_SDW(0, 0, 0x52), 0x90},
+	{TAC_REG_SDW(0, 0, 0x53), 0x80},
+	{TAC_REG_SDW(0, 0, 0x55), 0x90},
+	{TAC_REG_SDW(0, 0, 0x56), 0x80},
+	{TAC_REG_SDW(0, 0, 0x58), 0x90},
+	{TAC_REG_SDW(0, 0, 0x59), 0x80},
+	{TAC_REG_SDW(0, 0, 0x5b), 0x90},
+	{TAC_REG_SDW(0, 0, 0x5c), 0x80},
+	{TAC_REG_SDW(0, 0, 0x5e), 0x8},
+	{TAC_REG_SDW(0, 0, 0x5f), 0x8},
+	{TAC_REG_SDW(0, 0, 0x60), 0x0},
+	{TAC_REG_SDW(0, 0, 0x61), 0x0},
+	{TAC_REG_SDW(0, 0, 0x62), 0xff},
+	{TAC_REG_SDW(0, 0, 0x63), 0xc0},
+	{TAC_REG_SDW(0, 0, 0x64), 0x5},
+	{TAC_REG_SDW(0, 0, 0x65), 0x3},
+	{TAC_REG_SDW(0, 0, 0x66), 0x0},
+	{TAC_REG_SDW(0, 0, 0x67), 0x0},
+	{TAC_REG_SDW(0, 0, 0x68), 0x0},
+	{TAC_REG_SDW(0, 0, 0x69), 0x8},
+	{TAC_REG_SDW(0, 0, 0x6a), 0x0},
+	{TAC_REG_SDW(0, 0, 0x6b), 0xa0},
+	{TAC_REG_SDW(0, 0, 0x6c), 0x18},
+	{TAC_REG_SDW(0, 0, 0x6d), 0x18},
+	{TAC_REG_SDW(0, 0, 0x6e), 0x18},
+	{TAC_REG_SDW(0, 0, 0x6f), 0x18},
+	{TAC_REG_SDW(0, 0, 0x70), 0x88},
+	{TAC_REG_SDW(0, 0, 0x71), 0xff},
+	{TAC_REG_SDW(0, 0, 0x72), 0x0},
+	{TAC_REG_SDW(0, 0, 0x73), 0x31},
+	{TAC_REG_SDW(0, 0, 0x74), 0xc0},
+	{TAC_REG_SDW(0, 0, 0x75), 0x0},
+	{TAC_REG_SDW(0, 0, 0x76), 0x0},
+	{TAC_REG_SDW(0, 0, 0x77), 0x0},
+	{TAC_REG_SDW(0, 0, 0x78), 0x0},
+	{TAC_REG_SDW(0, 0, 0x7b), 0x0},
+	{TAC_REG_SDW(0, 0, 0x7c), 0xd0},
+	{TAC_REG_SDW(0, 0, 0x7d), 0x0},
+	{TAC_REG_SDW(0, 0, 0x7e), 0x0},
+	{TAC_REG_SDW(0, 1, 0x1), 0x0},
+	{TAC_REG_SDW(0, 1, 0x2), 0x0},
+	{TAC_REG_SDW(0, 1, 0x3), 0x0},
+	{TAC_REG_SDW(0, 1, 0x4), 0x4},
+	{TAC_REG_SDW(0, 1, 0x5), 0x0},
+	{TAC_REG_SDW(0, 1, 0x6), 0x0},
+	{TAC_REG_SDW(0, 1, 0x7), 0x0},
+	{TAC_REG_SDW(0, 1, 0x8), 0x0},
+	{TAC_REG_SDW(0, 1, 0x9), 0x0},
+	{TAC_REG_SDW(0, 1, 0xa), 0x0},
+	{TAC_REG_SDW(0, 1, 0xb), 0x1},
+	{TAC_REG_SDW(0, 1, 0xc), 0x0},
+	{TAC_REG_SDW(0, 1, 0xd), 0x0},
+	{TAC_REG_SDW(0, 1, 0xe), 0x0},
+	{TAC_REG_SDW(0, 1, 0xf), 0x8},
+	{TAC_REG_SDW(0, 1, 0x10), 0x0},
+	{TAC_REG_SDW(0, 1, 0x11), 0x0},
+	{TAC_REG_SDW(0, 1, 0x12), 0x1},
+	{TAC_REG_SDW(0, 1, 0x13), 0x0},
+	{TAC_REG_SDW(0, 1, 0x14), 0x0},
+	{TAC_REG_SDW(0, 1, 0x15), 0x0},
+	{TAC_REG_SDW(0, 1, 0x16), 0x0},
+	{TAC_REG_SDW(0, 1, 0x17), 0x0},
+	{TAC_REG_SDW(0, 1, 0x18), 0x0},
+	{TAC_REG_SDW(0, 1, 0x19), 0x0},
+	{TAC_REG_SDW(0, 1, 0x1a), 0x0},
+	{TAC_REG_SDW(0, 1, 0x1b), 0x0},
+	{TAC_REG_SDW(0, 1, 0x1c), 0x0},
+	{TAC_REG_SDW(0, 1, 0x1d), 0x0},
+	{TAC_REG_SDW(0, 1, 0x1e), 0x2},
+	{TAC_REG_SDW(0, 1, 0x1f), 0x8},
+	{TAC_REG_SDW(0, 1, 0x20), 0x9},
+	{TAC_REG_SDW(0, 1, 0x21), 0xa},
+	{TAC_REG_SDW(0, 1, 0x22), 0xb},
+	{TAC_REG_SDW(0, 1, 0x23), 0xc},
+	{TAC_REG_SDW(0, 1, 0x24), 0xd},
+	{TAC_REG_SDW(0, 1, 0x25), 0xe},
+	{TAC_REG_SDW(0, 1, 0x26), 0xf},
+	{TAC_REG_SDW(0, 1, 0x27), 0x8},
+	{TAC_REG_SDW(0, 1, 0x28), 0x9},
+	{TAC_REG_SDW(0, 1, 0x29), 0xa},
+	{TAC_REG_SDW(0, 1, 0x2a), 0xb},
+	{TAC_REG_SDW(0, 1, 0x2b), 0xc},
+	{TAC_REG_SDW(0, 1, 0x2c), 0xd},
+	{TAC_REG_SDW(0, 1, 0x2d), 0xe},
+	{TAC_REG_SDW(0, 1, 0x2e), 0xf},
+	{TAC_REG_SDW(0, 1, 0x2f), 0x0},
+	{TAC_REG_SDW(0, 1, 0x30), 0x0},
+	{TAC_REG_SDW(0, 1, 0x31), 0x0},
+	{TAC_REG_SDW(0, 1, 0x32), 0x0},
+	{TAC_REG_SDW(0, 1, 0x33), 0x0},
+	{TAC_REG_SDW(0, 1, 0x34), 0x0},
+	{TAC_REG_SDW(0, 1, 0x35), 0x0},
+	{TAC_REG_SDW(0, 1, 0x36), 0x0},
+	{TAC_REG_SDW(0, 1, 0x37), 0x0},
+	{TAC_REG_SDW(0, 1, 0x38), 0x98},
+	{TAC_REG_SDW(0, 1, 0x39), 0x0},
+	{TAC_REG_SDW(0, 1, 0x3a), 0x0},
+	{TAC_REG_SDW(0, 1, 0x3b), 0x0},
+	{TAC_REG_SDW(0, 1, 0x3c), 0x1},
+	{TAC_REG_SDW(0, 1, 0x3d), 0x2},
+	{TAC_REG_SDW(0, 1, 0x3e), 0x3},
+	{TAC_REG_SDW(0, 1, 0x3f), 0x4},
+	{TAC_REG_SDW(0, 1, 0x40), 0x5},
+	{TAC_REG_SDW(0, 1, 0x41), 0x6},
+	{TAC_REG_SDW(0, 1, 0x42), 0x7},
+	{TAC_REG_SDW(0, 1, 0x43), 0x0},
+	{TAC_REG_SDW(0, 1, 0x44), 0x0},
+	{TAC_REG_SDW(0, 1, 0x45), 0x1},
+	{TAC_REG_SDW(0, 1, 0x46), 0x2},
+	{TAC_REG_SDW(0, 1, 0x47), 0x3},
+	{TAC_REG_SDW(0, 1, 0x48), 0x4},
+	{TAC_REG_SDW(0, 1, 0x49), 0x5},
+	{TAC_REG_SDW(0, 1, 0x4a), 0x6},
+	{TAC_REG_SDW(0, 1, 0x4b), 0x7},
+	{TAC_REG_SDW(0, 1, 0x4c), 0x98},
+	{TAC_REG_SDW(0, 1, 0x4d), 0x0},
+	{TAC_REG_SDW(0, 1, 0x4e), 0x0},
+	{TAC_REG_SDW(0, 1, 0x4f), 0x0},
+	{TAC_REG_SDW(0, 1, 0x50), 0x1},
+	{TAC_REG_SDW(0, 1, 0x51), 0x2},
+	{TAC_REG_SDW(0, 1, 0x52), 0x3},
+	{TAC_REG_SDW(0, 1, 0x53), 0x4},
+	{TAC_REG_SDW(0, 1, 0x54), 0x5},
+	{TAC_REG_SDW(0, 1, 0x55), 0x6},
+	{TAC_REG_SDW(0, 1, 0x56), 0x7},
+	{TAC_REG_SDW(0, 1, 0x57), 0x0},
+	{TAC_REG_SDW(0, 1, 0x58), 0x0},
+	{TAC_REG_SDW(0, 1, 0x59), 0x1},
+	{TAC_REG_SDW(0, 1, 0x5a), 0x2},
+	{TAC_REG_SDW(0, 1, 0x5b), 0x3},
+	{TAC_REG_SDW(0, 1, 0x5c), 0x4},
+	{TAC_REG_SDW(0, 1, 0x5d), 0x5},
+	{TAC_REG_SDW(0, 1, 0x5e), 0x6},
+	{TAC_REG_SDW(0, 1, 0x5f), 0x7},
+	{TAC_REG_SDW(0, 1, 0x60), 0x98},
+	{TAC_REG_SDW(0, 1, 0x61), 0x0},
+	{TAC_REG_SDW(0, 1, 0x62), 0x0},
+	{TAC_REG_SDW(0, 1, 0x63), 0x0},
+	{TAC_REG_SDW(0, 1, 0x64), 0x1},
+	{TAC_REG_SDW(0, 1, 0x65), 0x2},
+	{TAC_REG_SDW(0, 1, 0x66), 0x3},
+	{TAC_REG_SDW(0, 1, 0x67), 0x4},
+	{TAC_REG_SDW(0, 1, 0x68), 0x5},
+	{TAC_REG_SDW(0, 1, 0x69), 0x6},
+	{TAC_REG_SDW(0, 1, 0x6a), 0x7},
+	{TAC_REG_SDW(0, 1, 0x6b), 0x0},
+	{TAC_REG_SDW(0, 1, 0x6c), 0x0},
+	{TAC_REG_SDW(0, 1, 0x6d), 0x1},
+	{TAC_REG_SDW(0, 1, 0x6e), 0x2},
+	{TAC_REG_SDW(0, 1, 0x6f), 0x3},
+	{TAC_REG_SDW(0, 1, 0x70), 0x4},
+	{TAC_REG_SDW(0, 1, 0x71), 0x5},
+	{TAC_REG_SDW(0, 1, 0x72), 0x6},
+	{TAC_REG_SDW(0, 1, 0x73), 0x7},
+};
+
+static const struct reg_sequence tac_spk_seq[] = {
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_FU21,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT), 0),
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_FU21,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT), 0),
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_FU23,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT), 0),
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_FU23,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT), 0),
+};
+
+static const struct reg_sequence tac_sm_seq[] = {
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_FU113,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT), 0),
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_FU113,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT), 0),
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_FU11,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT), 0),
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_FU11,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT), 0),
+};
+
+static const struct reg_sequence tac_uaj_seq[] = {
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_FU41,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT), 0),
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_FU41,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT), 0),
+	REG_SEQ0(SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_FU36,
+			      TAC_SDCA_CHANNEL_GAIN, TAC_JACK_MONO_CS), 0),
+};
+
+static bool tac_volatile_reg(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case TAC_REG_SDW(0, 0, 1) ... TAC_REG_SDW(0, 0, 5):
+	case TAC_REG_SDW(0, 2, 1) ... TAC_REG_SDW(0, 2, 6):
+	case TAC_REG_SDW(0, 2, 24) ... TAC_REG_SDW(0, 2, 55):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_HID, TAC_SDCA_ENT_HID1,
+			  TAC_SDCA_CTL_HIDTX_CURRENT_OWNER, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_HID, TAC_SDCA_ENT_HID1,
+			  TAC_SDCA_CTL_HIDTX_MESSAGE_OFFSET, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_GE35,
+			  TAC_SDCA_CTL_DET_MODE, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_PDE23,
+			  TAC_SDCA_REQUESTED_PS, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_PDE11,
+			  TAC_SDCA_REQUESTED_PS, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_PDE47,
+			  TAC_SDCA_REQUESTED_PS, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_PDE34,
+			  TAC_SDCA_REQUESTED_PS, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_PDE23,
+			  TAC_SDCA_ACTUAL_PS, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_PDE11,
+			  TAC_SDCA_ACTUAL_PS, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_PDE47,
+			  TAC_SDCA_ACTUAL_PS, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_PDE34,
+			  TAC_SDCA_ACTUAL_PS, 0):
+	case SDW_SCP_SDCA_INT1:
+	case SDW_SCP_SDCA_INT2:
+	case SDW_SCP_SDCA_INT3:
+	case SDW_SCP_SDCA_INT4:
+	case SDW_SDCA_CTL(1, 0, 0x10, 0):
+	case SDW_SDCA_CTL(2, 0, 0x10, 0):
+	case SDW_SDCA_CTL(3, 0, 0x10, 0):
+	case SDW_SDCA_CTL(4, 0, 0x1, 0):
+	case 0x44007F80 ... 0x44007F87:
+	case TAC_DSP_ALGO_STATUS:	/* DSP algo status - always read from HW */
+		return true;
+	default:
+		break;
+	}
+
+	return false;
+}
+
+static int tac_sdca_mbq_size(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_FU21,
+			  TAC_SDCA_CHANNEL_VOLUME, TAC_CHANNEL_LEFT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_FU21,
+			  TAC_SDCA_CHANNEL_VOLUME, TAC_CHANNEL_RIGHT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_FU23,
+			  TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_FU23,
+			  TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SA, TAC_SDCA_ENT_FU23,
+			  TAC_SDCA_MASTER_GAIN, 0):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_FU113,
+			  TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_FU113,
+			  TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_FU11,
+			  TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_SM, TAC_SDCA_ENT_FU11,
+			  TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_FU41,
+			  TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_LEFT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_FU41,
+			  TAC_SDCA_CHANNEL_GAIN, TAC_CHANNEL_RIGHT):
+	case SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_FU36,
+			  TAC_SDCA_CHANNEL_GAIN, TAC_JACK_MONO_CS):
+		return 2;
+
+	default:
+		return 1;
+	}
+}
+
+static const struct regmap_sdw_mbq_cfg tac_mbq_cfg = {
+	.mbq_size = tac_sdca_mbq_size,
+};
+
+static const struct regmap_config tac_regmap = {
+	.reg_bits = 32,
+	.val_bits = 16, /* mbq support */
+	.reg_defaults = tac_reg_default,
+	.num_reg_defaults = ARRAY_SIZE(tac_reg_default),
+	.max_register = 0x47FFFFFF,
+	.cache_type = REGCACHE_MAPLE,
+	.volatile_reg = tac_volatile_reg,
+	.use_single_read = true,
+	.use_single_write = true,
+};
+
+/* Check if device has DSP algo that needs status monitoring */
+static bool tac_has_dsp_algo(struct tac5xx2_prv *tac_dev)
+{
+	switch (tac_dev->part_id) {
+	case 0x5682:
+	case 0x2883:
+		return true;
+	default:
+		return false;
+	}
+}
+
+/* Check if device has UAJ (Universal Audio Jack) support */
+static bool tac_has_uaj_support(struct tac5xx2_prv *tac_dev)
+{
+	return tac_dev->uaj_func_data;
+}
+
+/* Forward declaration for headset detection */
+static int tac5xx2_sdca_headset_detect(struct tac5xx2_prv *tac_dev);
+
+/* Volume controls for mic, hp and mic cap */
+static const struct snd_kcontrol_new tac5xx2_snd_controls[] = {
+	SOC_DOUBLE_R_RANGE_TLV("Amp Volume", TAC_AMP_LVL_CFG0, TAC_AMP_LVL_CFG1,
+			       2, 0, 44, 1, tac5xx2_amp_tlv),
+	TAC_DOUBLE_Q78_TLV("DMIC Capture Volume", SM, FU113),
+	TAC_DOUBLE_Q78_TLV("Speaker Volume", SA, FU21),
+};
+
+static const struct snd_kcontrol_new tac_uaj_controls[] = {
+	TAC_DOUBLE_Q78_TLV("UAJ Playback Volume", UAJ, FU41),
+	SDCA_SINGLE_Q78_TLV("UAJ Capture Volume",
+			    SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_FU36,
+					 TAC_SDCA_CHANNEL_GAIN, TAC_JACK_MONO_CS),
+			   TAC_DVC_MIN, TAC_DVC_MAX, TAC_DVC_STEP, tac5xx2_dvc_tlv),
+};
+
+static const struct snd_soc_dapm_widget tac5xx2_common_widgets[] = {
+	/* Port 1: Speaker Playback Path */
+	SND_SOC_DAPM_AIF_IN("AIF1 Playback", "DP1 Speaker Playback", 0,
+			    SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_PGA("FU21_L", FU21_L_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("FU21_R", FU21_R_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("FU23_L", FU23_L_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("FU23_R", FU23_R_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_OUTPUT("SPK_L"),
+	SND_SOC_DAPM_OUTPUT("SPK_R"),
+
+	/* Port 3: Smart Mic (DMIC) Capture Path */
+	SND_SOC_DAPM_AIF_OUT("AIF3 Capture", "DP3 Mic Capture", 0, SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_INPUT("DMIC_L"),
+	SND_SOC_DAPM_INPUT("DMIC_R"),
+	SND_SOC_DAPM_PGA("IT11", IT11_USAGE_REG, 0, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("CS113", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("FU11_L", FU11_L_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("FU11_R", FU11_R_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("PPU11", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("XU12", XU12_BYPASS_REG, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("FU113_L", FU113_L_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("FU113_R", FU113_R_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("OT113", OT113_USAGE_REG, 0, 0, NULL, 0),
+};
+
+static const struct snd_soc_dapm_widget tac_uaj_widgets[] = {
+	/* Port 4: UAJ (Headphone) Playback Path */
+	SND_SOC_DAPM_AIF_IN("AIF4 Playback", "DP4 UAJ Speaker Playback", 0,
+			    SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_PGA("IT41", IT41_USAGE_REG, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("FU41_L", FU41_L_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("FU41_R", FU41_R_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_PGA("XU42", XU42_BYPASS_REG, 0, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("CS41", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_DAC("OT45", "DP4 UAJ Speaker Playback", OT45_USAGE_REG, 0, 0),
+	SND_SOC_DAPM_OUTPUT("HP_L"),
+	SND_SOC_DAPM_OUTPUT("HP_R"),
+
+	/* Port 7: UAJ (Headset Mic) Capture Path */
+	SND_SOC_DAPM_AIF_OUT("AIF7 Capture", "DP7 UAJ Mic Capture", 0,
+			     SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_INPUT("UAJ_MIC"),
+	SND_SOC_DAPM_ADC("IT33", "DP7 UAJ Mic Capture", IT33_USAGE_REG, 0, 0),
+	SND_SOC_DAPM_PGA("FU36", FU36_MUTE_REG, 0, 1, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("CS36", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_PGA("OT36", OT36_USAGE_REG, 0, 0, NULL, 0),
+};
+
+static const struct snd_soc_dapm_route tac5xx2_common_routes[] = {
+	/* Speaker Playback Path */
+	{"FU21_L", NULL, "AIF1 Playback"},
+	{"FU21_R", NULL, "AIF1 Playback"},
+
+	{"FU23_L", NULL, "FU21_L"},
+	{"FU23_R", NULL, "FU21_R"},
+
+	{"SPK_L", NULL, "FU23_L"},
+	{"SPK_R", NULL, "FU23_R"},
+
+	/* Smart Mic DAPM Routes */
+	{"IT11", NULL, "DMIC_L"},
+	{"IT11", NULL, "DMIC_R"},
+	{"FU11_L", NULL, "IT11"},
+	{"FU11_R", NULL, "IT11"},
+	{"PPU11", NULL, "FU11_L"},
+	{"PPU11", NULL, "FU11_R"},
+	{"XU12", NULL, "PPU11"},
+	{"FU113_L", NULL, "XU12"},
+	{"FU113_R", NULL, "XU12"},
+	{"FU113_L", NULL, "CS113"},
+	{"FU113_R", NULL, "CS113"},
+	{"OT113", NULL, "FU113_L"},
+	{"OT113", NULL, "FU113_R"},
+	{"OT113", NULL, "CS113"},
+	{"AIF3 Capture", NULL, "OT113"},
+};
+
+static const struct snd_soc_dapm_route tac_uaj_routes[] = {
+	/* UAJ Playback routes */
+	{"IT41", NULL, "AIF4 Playback"},
+	{"IT41", NULL, "CS41"},
+	{"FU41_L", NULL, "IT41"},
+	{"FU41_R", NULL, "IT41"},
+	{"XU42", NULL, "FU41_L"},
+	{"XU42", NULL, "FU41_R"},
+	{"OT45", NULL, "XU42"},
+	{"OT45", NULL, "CS41"},
+	{"HP_L", NULL, "OT45"},
+	{"HP_R", NULL, "OT45"},
+
+	/* UAJ Capture routes */
+	{"IT33", NULL, "UAJ_MIC"},
+	{"IT33", NULL, "CS36"},
+	{"FU36", NULL, "IT33"},
+	{"OT36", NULL, "FU36"},
+	{"OT36", NULL, "CS36"},
+	{"AIF7 Capture", NULL, "OT36"},
+};
+
+static s32 tac_set_sdw_stream(struct snd_soc_dai *dai,
+			      void *sdw_stream, s32 direction)
+{
+	if (sdw_stream)
+		snd_soc_dai_dma_data_set(dai, direction, sdw_stream);
+
+	return 0;
+}
+
+static void tac_sdw_shutdown(struct snd_pcm_substream *substream,
+			     struct snd_soc_dai *dai)
+{
+	snd_soc_dai_set_dma_data(dai, substream, NULL);
+}
+
+static int tac_clear_latch(struct tac5xx2_prv *priv)
+{
+	/* CLR_REG is a self-clearing bit */
+	return regmap_update_bits(priv->regmap, TAC_INT_CFG,
+				  TAC_INT_CFG_CLR_REG, TAC_INT_CFG_CLR_REG);
+}
+
+static int tac_sdw_hw_params(struct snd_pcm_substream *substream,
+			     struct snd_pcm_hw_params *params,
+			     struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct tac5xx2_prv *tac_dev = snd_soc_component_get_drvdata(component);
+	struct sdw_slave *sdw_peripheral = tac_dev->sdw_peripheral;
+	struct sdw_stream_runtime *sdw_stream;
+	struct sdw_stream_config stream_config = {0};
+	struct sdw_port_config port_config = {0};
+	u8 sample_rate_idx = 0;
+	int function_id;
+	int pde_entity;
+	int port_num;
+	int ret;
+
+	if (!tac_dev->hw_init) {
+		dev_err(tac_dev->dev,
+			"error: operation without hw initialization");
+		return -EINVAL;
+	}
+
+	sdw_stream = snd_soc_dai_get_dma_data(dai, substream);
+	if (!sdw_stream) {
+		dev_err(tac_dev->dev, "failed to get dma data");
+		return -EINVAL;
+	}
+
+	ret = tac_clear_latch(tac_dev);
+	if (ret)
+		dev_warn(tac_dev->dev, "clear latch failed, err=%d", ret);
+
+	switch (dai->id) {
+	case TAC5XX2_DMIC:
+		function_id = TAC_FUNCTION_ID_SM;
+		pde_entity = TAC_SDCA_ENT_PDE11;
+		port_num = TAC_SDW_PORT_NUM_DMIC;
+		break;
+	case TAC5XX2_UAJ:
+		function_id = TAC_FUNCTION_ID_UAJ;
+		pde_entity = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ?
+				TAC_SDCA_ENT_PDE47 : TAC_SDCA_ENT_PDE34;
+		port_num = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ?
+				TAC_SDW_PORT_NUM_UAJ_PLAYBACK :
+				TAC_SDW_PORT_NUM_UAJ_CAPTURE;
+		break;
+	case TAC5XX2_SPK:
+		function_id = TAC_FUNCTION_ID_SA;
+		pde_entity = TAC_SDCA_ENT_PDE23;
+		port_num = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ?
+				TAC_SDW_PORT_NUM_SPK_PLAYBACK :
+				TAC_SDW_PORT_NUM_SPK_CAPTURE;
+		break;
+	default:
+		dev_err(tac_dev->dev, "Invalid dai id: %d for power up\n", dai->id);
+		return -EINVAL;
+	}
+
+	snd_sdw_params_to_config(substream, params, &stream_config, &port_config);
+	port_config.num = port_num;
+	ret = sdw_stream_add_slave(sdw_peripheral, &stream_config,
+				   &port_config, 1, sdw_stream);
+	if (ret) {
+		dev_err(dai->dev,
+			"Unable to configure port %d: %d\n", port_num, ret);
+		return ret;
+	}
+
+	switch (params_rate(params)) {
+	case 48000:
+		sample_rate_idx = 0x01;
+		break;
+	case 44100:
+		sample_rate_idx = 0x02;
+		break;
+	case 96000:
+		sample_rate_idx = 0x03;
+		break;
+	case 88200:
+		sample_rate_idx = 0x04;
+		break;
+	default:
+		dev_dbg(tac_dev->dev, "Unsupported sample rate: %d Hz",
+			params_rate(params));
+		return -EINVAL;
+	}
+
+	switch (function_id) {
+	case TAC_FUNCTION_ID_SM:
+		ret = regmap_write(tac_dev->regmap,
+				   SDW_SDCA_CTL(function_id, TAC_SDCA_ENT_CS113,
+						TAC_SDCA_CTL_CS_SAMP_RATE_IDX, 0),
+			sample_rate_idx);
+		if (ret) {
+			dev_err(tac_dev->dev, "Failed to set CS113 sample rate: %d", ret);
+			return ret;
+		}
+
+		break;
+	case TAC_FUNCTION_ID_UAJ:
+		if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+			ret = regmap_write(tac_dev->regmap,
+					   SDW_SDCA_CTL(function_id, TAC_SDCA_ENT_CS41,
+							TAC_SDCA_CTL_CS_SAMP_RATE_IDX, 0),
+					sample_rate_idx);
+			if (ret) {
+				dev_err(tac_dev->dev, "Failed to set CS41 sample rate: %d", ret);
+				return ret;
+			}
+		} else {
+			ret = regmap_write(tac_dev->regmap,
+					   SDW_SDCA_CTL(function_id, TAC_SDCA_ENT_CS36,
+							TAC_SDCA_CTL_CS_SAMP_RATE_IDX, 0),
+					sample_rate_idx);
+			if (ret) {
+				dev_err(tac_dev->dev, "Failed to set CS36 sample rate: %d", ret);
+				return ret;
+			}
+		}
+		break;
+	case TAC_FUNCTION_ID_SA:
+		/* SmartAmp: no additional sample rate configuration needed */
+		break;
+	}
+
+	ret = regmap_write(tac_dev->regmap, SDW_SDCA_CTL(function_id, pde_entity,
+							 TAC_SDCA_REQUESTED_PS, 0), 0);
+	if (ret) {
+		dev_err(tac_dev->dev,
+			"failed to set func %d, entity %d's requested PS to 0: %d\n",
+			function_id, pde_entity, ret);
+		return ret;
+	}
+
+	ret = sdca_asoc_pde_poll_actual_ps(tac_dev->dev, tac_dev->regmap, function_id, pde_entity,
+					   SDCA_PDE_PS3, SDCA_PDE_PS0, NULL, 0);
+	if (ret)
+		dev_err(tac_dev->dev, "failed to transition func %d, pde %d from PS3 -> PS0, err=%d\n",
+			function_id, pde_entity, ret);
+	return ret;
+}
+
+static int tac_sdw_pcm_hw_free(struct snd_pcm_substream *substream,
+			       struct snd_soc_dai *dai)
+{
+	struct sdw_stream_runtime *sdw_stream = snd_soc_dai_get_dma_data(dai, substream);
+	struct tac5xx2_prv *tac_dev = snd_soc_component_get_drvdata(dai->component);
+	int pde_entity, function_id;
+	int ret;
+
+	sdw_stream_remove_slave(tac_dev->sdw_peripheral, sdw_stream);
+
+	switch (dai->id) {
+	case TAC5XX2_DMIC:
+		pde_entity = TAC_SDCA_ENT_PDE11;
+		function_id = TAC_FUNCTION_ID_SM;
+		break;
+	case TAC5XX2_UAJ:
+		function_id = TAC_FUNCTION_ID_UAJ;
+		pde_entity = substream->stream == SNDRV_PCM_STREAM_PLAYBACK ?
+				TAC_SDCA_ENT_PDE47 : TAC_SDCA_ENT_PDE34;
+		break;
+	case TAC5XX2_SPK:
+		function_id = TAC_FUNCTION_ID_SA;
+		pde_entity = TAC_SDCA_ENT_PDE23;
+		break;
+	default:
+		dev_err(tac_dev->dev, "unhandled dai %d for power down\n", dai->id);
+		return -EINVAL;
+	}
+
+	ret = regmap_write(tac_dev->regmap, SDW_SDCA_CTL(function_id, pde_entity,
+							 TAC_SDCA_REQUESTED_PS, 0),
+			   SDCA_PDE_PS3);
+	if (ret) {
+		dev_err(tac_dev->dev,
+			"failed to set func %d, entity %d's requested PS to 3: %d\n",
+			function_id, pde_entity, ret);
+		return ret;
+	}
+
+	ret = sdca_asoc_pde_poll_actual_ps(tac_dev->dev, tac_dev->regmap, function_id,
+					   pde_entity, SDCA_PDE_PS0, SDCA_PDE_PS3,
+					   NULL, 0);
+	if (ret)
+		dev_err(tac_dev->dev,
+			"failed to transition func %d, pde %d from PS0 -> PS3, err=%d\n",
+			function_id, pde_entity, ret);
+
+	return ret;
+}
+
+static const struct snd_soc_dai_ops tac_dai_ops = {
+	.hw_params = tac_sdw_hw_params,
+	.hw_free = tac_sdw_pcm_hw_free,
+	.set_stream = tac_set_sdw_stream,
+	.shutdown = tac_sdw_shutdown,
+};
+
+static int tac5xx2_sdca_btn_type(unsigned char *buffer, struct tac5xx2_prv *tac_dev)
+{
+	switch (*buffer) {
+	case 1: /* play pause */
+		return SND_JACK_BTN_0;
+	case 10: /* vol down */
+		return SND_JACK_BTN_3;
+	case 8: /* vol up */
+		return SND_JACK_BTN_2;
+	case 4: /* long press */
+		return SND_JACK_BTN_1;
+	case 2: /* next song */
+	case 32: /* next song */
+		return SND_JACK_BTN_4;
+	default:
+		return 0;
+	}
+}
+
+static int tac5xx2_sdca_button_detect(struct tac5xx2_prv *tac_dev)
+{
+	unsigned int btn_type, offset, idx;
+	int ret, value, owner;
+	u8 buf[2];
+
+	ret = regmap_read(tac_dev->regmap,
+			  SDW_SDCA_CTL(TAC_FUNCTION_ID_HID, TAC_SDCA_ENT_HID1,
+				       TAC_SDCA_CTL_HIDTX_CURRENT_OWNER, 0), &owner);
+	if (ret) {
+		dev_err(tac_dev->dev,
+			"Failed to read current UMP message owner 0x%x", ret);
+		return ret;
+	}
+
+	if (owner == SDCA_UMP_OWNER_DEVICE) {
+		dev_dbg(tac_dev->dev, "skip button detect as current owner is not host\n");
+		return 0;
+	}
+
+	ret = regmap_read(tac_dev->regmap,
+			  SDW_SDCA_CTL(TAC_FUNCTION_ID_HID, TAC_SDCA_ENT_HID1,
+				       TAC_SDCA_CTL_HIDTX_MESSAGE_OFFSET, 0), &offset);
+	if (ret) {
+		dev_err(tac_dev->dev,
+			"Failed to read current UMP message offset: %d", ret);
+		goto end_btn_det;
+	}
+
+	dev_dbg(tac_dev->dev, "button detect: message offset = %x", offset);
+
+	for (idx = 0; idx < sizeof(buf); idx++) {
+		ret = regmap_read(tac_dev->regmap,
+				  TAC_BUF_ADDR_HID1 + offset + idx, &value);
+		if (ret) {
+			dev_err(tac_dev->dev,
+				"Failed to read HID buffer: %d", ret);
+			goto end_btn_det;
+		}
+		buf[idx] = value & 0xff;
+	}
+
+	if (buf[0] == 0x1) {
+		btn_type = tac5xx2_sdca_btn_type(&buf[1], tac_dev);
+		ret = btn_type;
+	}
+
+end_btn_det:
+	regmap_write(tac_dev->regmap,
+		     SDW_SDCA_CTL(TAC_FUNCTION_ID_HID, TAC_SDCA_ENT_HID1,
+				  TAC_SDCA_CTL_HIDTX_CURRENT_OWNER, 0), 0x01);
+
+	return ret;
+}
+
+static int tac5xx2_sdca_headset_detect(struct tac5xx2_prv *tac_dev)
+{
+	int val, ret;
+
+	ret = regmap_read(tac_dev->regmap,
+			  SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_GE35,
+				       TAC_SDCA_CTL_DET_MODE, 0), &val);
+	if (ret) {
+		dev_err(tac_dev->dev, "Failed to read the detect mode");
+		return ret;
+	}
+
+	switch (val) {
+	case 4:
+		tac_dev->jack_type = SND_JACK_MICROPHONE;
+		break;
+	case 5:
+		tac_dev->jack_type = SND_JACK_HEADPHONE;
+		break;
+	case 6:
+		tac_dev->jack_type = SND_JACK_HEADSET;
+		break;
+	case 0:
+	default:
+		tac_dev->jack_type = 0;
+		break;
+	}
+
+	ret = regmap_write(tac_dev->regmap,
+			   SDW_SDCA_CTL(TAC_FUNCTION_ID_UAJ, TAC_SDCA_ENT_GE35,
+					TAC_SDCA_CTL_SEL_MODE, 0), val);
+	if (ret)
+		dev_err(tac_dev->dev, "Failed to update the jack type to device");
+
+	return 0;
+}
+
+static int tac5xx2_jack_init(struct tac5xx2_prv *tac_dev)
+{
+	int ret = 0;
+
+	if (!tac_dev->hs_jack)
+		goto disable_interrupts;
+
+	ret = regmap_write(tac_dev->regmap, SDW_SCP_SDCA_INTMASK2,
+			   SDW_SCP_SDCA_INTMASK_SDCA_11);
+	if (ret) {
+		dev_err(tac_dev->dev,
+			"Failed to register jack detection interrupt: %d\n", ret);
+		goto disable_interrupts;
+	}
+
+	ret = regmap_write(tac_dev->regmap, SDW_SCP_SDCA_INTMASK3,
+			   SDW_SCP_SDCA_INTMASK_SDCA_16);
+	if (ret) {
+		dev_err(tac_dev->dev,
+			"Failed to register for button detect interrupt: %d\n", ret);
+		goto disable_interrupts;
+	}
+
+	return 0;
+
+disable_interrupts:
+	/* ignore errors while disabling interrupts */
+	regmap_write(tac_dev->regmap, SDW_SCP_SDCA_INTMASK2, 0);
+	regmap_write(tac_dev->regmap, SDW_SCP_SDCA_INTMASK3, 0);
+
+	return ret;
+}
+
+static int tac5xx2_set_jack(struct snd_soc_component *component,
+			    struct snd_soc_jack *hs_jack, void *data)
+{
+	struct tac5xx2_prv *tac_dev = snd_soc_component_get_drvdata(component);
+	int ret;
+
+	tac_dev->hs_jack = hs_jack;
+
+	/* resume can happen only after first hw_init */
+	if (!tac_dev->first_hw_init_done)
+		return 0;
+
+	ret = pm_runtime_resume_and_get(component->dev);
+	if (ret < 0) {
+		if (ret != -EACCES) {
+			dev_err(component->dev,
+				"%s: failed to resume %d\n", __func__, ret);
+			return ret;
+		}
+
+		/* pm_runtime not enabled yet */
+		dev_dbg(component->dev,
+			"%s: skipping jack init for now\n", __func__);
+		return 0;
+	}
+
+	ret = tac5xx2_jack_init(tac_dev);
+	if (ret)
+		dev_err(tac_dev->dev, "jack init failed, err=%d\n", ret);
+
+	pm_runtime_mark_last_busy(component->dev);
+	pm_runtime_put_autosuspend(component->dev);
+
+	return ret;
+}
+
+static int tac_interrupt_callback(struct sdw_slave *slave,
+				  struct sdw_slave_intr_status *status)
+{
+	unsigned int sdca_int2, sdca_int3, jack_report_mask = 0;
+	struct tac5xx2_prv *tac_dev = dev_get_drvdata(&slave->dev);
+	struct device *dev = &slave->dev;
+	int btn_type = 0;
+	int ret = 0;
+
+	if (status->control_port) {
+		if (status->control_port & SDW_SCP_INT1_PARITY)
+			dev_warn(dev, "SCP: Parity error interrupt");
+		if (status->control_port & SDW_SCP_INT1_BUS_CLASH)
+			dev_warn(dev, "SCP: Bus clash interrupt");
+	}
+
+	if (!tac_has_uaj_support(tac_dev))
+		return 0;
+
+	ret = regmap_read(tac_dev->regmap, SDW_SCP_SDCA_INT2, &sdca_int2);
+	if (ret) {
+		dev_err(dev, "Failed to read UAJ Interrupt, reg:%#x err=%d\n",
+			SDW_SCP_SDCA_INT2, ret);
+		return ret;
+	}
+
+	ret = regmap_read(tac_dev->regmap, SDW_SCP_SDCA_INT3, &sdca_int3);
+	if (ret) {
+		dev_err(dev, "Failed to read HID interrupt reg=%#x: err=%d",
+			SDW_SCP_SDCA_INT3, ret);
+		return ret;
+	}
+
+	dev_dbg(dev, "SDCA_INT2: 0x%02x, SDCA_INT3: 0x%02x\n",
+		sdca_int2, sdca_int3);
+
+	if (sdca_int2 & SDW_SCP_SDCA_INT_SDCA_11) {
+		ret = tac5xx2_sdca_headset_detect(tac_dev);
+		if (ret < 0)
+			goto clear;
+		jack_report_mask |= SND_JACK_HEADSET;
+	}
+
+	if (sdca_int3 & SDW_SCP_SDCA_INT_SDCA_16) {
+		btn_type = tac5xx2_sdca_button_detect(tac_dev);
+		if (btn_type < 0)
+			btn_type = 0;
+		jack_report_mask |= SND_JACK_BTN_0 | SND_JACK_BTN_1 |
+			SND_JACK_BTN_2 | SND_JACK_BTN_3 | SND_JACK_BTN_4;
+	}
+
+	if (tac_dev->jack_type == 0)
+		btn_type = 0;
+
+	dev_dbg(tac_dev->dev, "in %s, jack_type=%d\n", __func__, tac_dev->jack_type);
+	dev_dbg(tac_dev->dev, "in %s, btn_type=0x%x\n", __func__, btn_type);
+
+	if (!tac_dev->hs_jack)
+		goto clear;
+
+	snd_soc_jack_report(tac_dev->hs_jack, tac_dev->jack_type | btn_type,
+			    jack_report_mask);
+
+clear:
+	if (sdca_int2) {
+		ret = regmap_write(tac_dev->regmap, SDW_SCP_SDCA_INT2, sdca_int2);
+		if (ret)
+			dev_dbg(tac_dev->dev, "Failed to clear jack interrupt\n");
+	}
+
+	if (sdca_int3) {
+		ret = regmap_write(tac_dev->regmap, SDW_SCP_SDCA_INT3, sdca_int3);
+		if (ret)
+			dev_dbg(tac_dev->dev, "failed to clear hid interrupt\n");
+	}
+
+	return 0;
+}
+
+static struct snd_soc_dai_driver tac5572_dai_driver[] = {
+	{
+		.name = "tac5xx2-aif1",
+		.id = TAC5XX2_SPK,
+		.playback = {
+			.stream_name = "DP1 Speaker Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+	},
+	{
+		.name = "tac5xx2-aif2",
+		.id = TAC5XX2_DMIC,
+		.capture = {
+			.stream_name = "DP3 Mic Capture",
+			.channels_min = 1,
+			.channels_max = 4,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+	},
+	{
+		.name = "tac5xx2-aif3",
+		.id = TAC5XX2_UAJ,
+		.playback = {
+			.stream_name = "DP4 UAJ Speaker Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.capture = {
+			.stream_name = "DP7 UAJ Mic Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+	},
+};
+
+static struct snd_soc_dai_driver tac5672_dai_driver[] = {
+	{
+		.name = "tac5xx2-aif1",
+		.id = TAC5XX2_SPK,
+		.playback = {
+			.stream_name = "DP1 Speaker Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.capture = {
+			.stream_name = "DP8 IV Sense Capture",
+			.channels_min = 1,
+			.channels_max = 4,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+		.symmetric_rate = 1,
+	},
+	{
+		.name = "tac5xx2-aif2",
+		.id = TAC5XX2_DMIC,
+		.capture = {
+			.stream_name = "DP3 Mic Capture",
+			.channels_min = 1,
+			.channels_max = 4,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+	},
+	{
+		.name = "tac5xx2-aif3",
+		.id = TAC5XX2_UAJ,
+		.playback = {
+			.stream_name = "DP4 UAJ Speaker Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.capture = {
+			.stream_name = "DP7 UAJ Mic Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+	},
+};
+
+static struct snd_soc_dai_driver tac5682_dai_driver[] = {
+	{
+		.name = "tac5xx2-aif1",
+		.id = TAC5XX2_SPK,
+		.playback = {
+			.stream_name = "DP1 Speaker Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.capture = {
+			.stream_name = "DP2 Echo Reference Capture",
+			.channels_min = 1,
+			.channels_max = 4,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+		.symmetric_rate = 1,
+	},
+	{
+		.name = "tac5xx2-aif2",
+		.id = TAC5XX2_DMIC,
+		.capture = {
+			.stream_name = "DP3 Mic Capture",
+			.channels_min = 1,
+			.channels_max = 4,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+	},
+	{
+		.name = "tac5xx2-aif3",
+		.id = TAC5XX2_UAJ,
+		.playback = {
+			.stream_name = "DP4 UAJ Speaker Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.capture = {
+			.stream_name = "DP7 UAJ Mic Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+	},
+};
+
+static struct snd_soc_dai_driver tas2883_dai_driver[] = {
+	{
+		.name = "tac5xx2-aif1",
+		.id = TAC5XX2_SPK,
+		.playback = {
+			.stream_name = "DP1 Speaker Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+		.symmetric_rate = 1,
+	},
+	{
+		.name = "tac5xx2-aif2",
+		.id = TAC5XX2_DMIC,
+		.capture = {
+			.stream_name = "DP3 Mic Capture",
+			.channels_min = 1,
+			.channels_max = 4,
+			.rates = TAC5XX2_DEVICE_RATES,
+			.formats = TAC5XX2_DEVICE_FORMATS,
+		},
+		.ops = &tac_dai_ops,
+	},
+};
+
+static s32 tac_component_probe(struct snd_soc_component *component)
+{
+	struct tac5xx2_prv *tac_dev = snd_soc_component_get_drvdata(component);
+	int ret;
+
+	ret = pm_runtime_resume(component->dev);
+	if (ret < 0 && ret != -EACCES)
+		return ret;
+
+	if (!tac_has_uaj_support(tac_dev))
+		goto done_comp_probe;
+
+	ret = snd_soc_dapm_new_controls(snd_soc_component_to_dapm(component),
+					tac_uaj_widgets,
+					ARRAY_SIZE(tac_uaj_widgets));
+	if (ret) {
+		dev_err(component->dev, "Failed to add UAJ widgets: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_dapm_add_routes(snd_soc_component_to_dapm(component),
+				      tac_uaj_routes, ARRAY_SIZE(tac_uaj_routes));
+	if (ret) {
+		dev_err(component->dev, "Failed to add UAJ routes: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_add_component_controls(component, tac_uaj_controls,
+					     ARRAY_SIZE(tac_uaj_controls));
+	if (ret) {
+		dev_err(component->dev, "Failed to add UAJ controls: %d\n", ret);
+			return ret;
+	}
+
+done_comp_probe:
+	tac_dev->component = component;
+	return 0;
+}
+
+static void tac_component_remove(struct snd_soc_component *codec)
+{
+	struct tac5xx2_prv *tac_dev = snd_soc_component_get_drvdata(codec);
+
+	tac_dev->component = NULL;
+}
+
+static const struct snd_soc_component_driver soc_codec_driver_tacdevice = {
+	.probe = tac_component_probe,
+	.remove = tac_component_remove,
+	.controls = tac5xx2_snd_controls,
+	.num_controls = ARRAY_SIZE(tac5xx2_snd_controls),
+	.dapm_widgets = tac5xx2_common_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(tac5xx2_common_widgets),
+	.dapm_routes = tac5xx2_common_routes,
+	.num_dapm_routes = ARRAY_SIZE(tac5xx2_common_routes),
+	.idle_bias_on = 0,
+	.endianness = 1,
+};
+
+static s32 tac_init(struct tac5xx2_prv *tac_dev)
+{
+	struct snd_soc_component_driver *component_driver;
+	struct snd_soc_dai_driver *dai_drv;
+	int num_dais;
+	s32 ret;
+
+	dev_set_drvdata(tac_dev->dev, tac_dev);
+
+	switch (tac_dev->part_id) {
+	case 0x5572:
+		dai_drv = tac5572_dai_driver;
+		num_dais = ARRAY_SIZE(tac5572_dai_driver);
+		break;
+	case 0x5672:
+		dai_drv = tac5672_dai_driver;
+		num_dais = ARRAY_SIZE(tac5672_dai_driver);
+		break;
+	case 0x5682:
+		dai_drv = tac5682_dai_driver;
+		num_dais = ARRAY_SIZE(tac5682_dai_driver);
+		break;
+	case 0x2883:
+		dai_drv = tas2883_dai_driver;
+		num_dais = ARRAY_SIZE(tas2883_dai_driver);
+		break;
+	default:
+		dev_err(tac_dev->dev, "Unsupported device: 0x%x\n",
+			tac_dev->part_id);
+		return -EINVAL;
+	}
+
+	component_driver = devm_kzalloc(tac_dev->dev, sizeof(*component_driver),
+					GFP_KERNEL);
+	if (!component_driver)
+		return -ENOMEM;
+
+	memcpy(component_driver, &soc_codec_driver_tacdevice, sizeof(*component_driver));
+	if (tac_has_uaj_support(tac_dev))
+		component_driver->set_jack = tac5xx2_set_jack;
+
+	ret = devm_snd_soc_register_component(tac_dev->dev, component_driver,
+					      dai_drv, num_dais);
+	if (ret) {
+		dev_err(tac_dev->dev, "%s: codec register error:%d.\n",
+			__func__, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static s32 tac5xx2_sdca_dev_suspend(struct device *dev)
+{
+	struct tac5xx2_prv *tac_dev = dev_get_drvdata(dev);
+
+	if (!tac_dev->hw_init)
+		return 0;
+
+	regcache_cache_only(tac_dev->regmap, true);
+	return 0;
+}
+
+static s32 tac5xx2_sdca_dev_system_suspend(struct device *dev)
+{
+	return tac5xx2_sdca_dev_suspend(dev);
+}
+
+static s32 tac5xx2_sdca_dev_resume(struct device *dev)
+{
+	struct tac5xx2_prv *tac_dev = dev_get_drvdata(dev);
+	struct sdw_slave *slave = dev_to_sdw_dev(dev);
+	int ret;
+
+	if (!tac_dev->first_hw_init_done) {
+		dev_dbg(dev, "Device not initialized yet, skipping resume sync\n");
+		return 0;
+	}
+
+	ret = sdw_slave_wait_for_init(slave, TAC5XX2_PROBE_TIMEOUT_MS);
+	if (ret)
+		return ret;
+
+	regcache_cache_only(tac_dev->regmap, false);
+	regcache_mark_dirty(tac_dev->regmap);
+	ret = regcache_sync(tac_dev->regmap);
+	if (ret < 0)
+		dev_warn(dev, "Failed to sync regcache: %d\n", ret);
+
+	/* Detect and set jack type for UAJ path before playback.
+	 * This is required as jack detection does not trigger interrupt
+	 * when device is in runtime_pm suspend with bus in clock stop mode.
+	 */
+	if (tac_has_uaj_support(tac_dev))
+		tac5xx2_sdca_headset_detect(tac_dev);
+
+	return 0;
+}
+
+static const struct dev_pm_ops tac5xx2_sdca_pm = {
+	SYSTEM_SLEEP_PM_OPS(tac5xx2_sdca_dev_system_suspend, tac5xx2_sdca_dev_resume)
+	RUNTIME_PM_OPS(tac5xx2_sdca_dev_suspend, tac5xx2_sdca_dev_resume, NULL)
+};
+
+static s32 tac_fw_read_hdr(const u8 *data, struct tac_fw_hdr *hdr)
+{
+	hdr->size = get_unaligned_le32(data);
+	hdr->version_offset = get_unaligned_le32(data + 4);
+	hdr->plt_id = get_unaligned_le32(data + 8);
+	hdr->ppc3_ver = get_unaligned_le32(data + 12);
+	memcpy(hdr->ddc_name, data + 16, 64);
+	hdr->ddc_name[63] = 0;
+	hdr->timestamp = get_unaligned_le64(data + 80);
+
+	return TAC_FW_HDR_SIZE;
+}
+
+static s32 tac_fw_get_next_file(const u8 *data, size_t data_size, struct tac_fw_file *file)
+{
+	u32 file_length;
+
+	/* Validate file header size */
+	if (data_size < TAC_FW_FILE_HDR)
+		return -EINVAL;
+
+	file->vendor_id = get_unaligned_le32(&data[0]);
+	file->file_id = get_unaligned_le32(&data[4]);
+	file->version = get_unaligned_le32(&data[8]);
+	file->length = get_unaligned_le32(&data[12]);
+	file->dest_addr = get_unaligned_le32(&data[16]);
+	file_length = file->length;
+
+	/* Validate file payload exists */
+	if (data_size < TAC_FW_FILE_HDR + file_length)
+		return -EINVAL;
+
+	file->fw_data = (u8 *)&data[20];
+
+	return file_length + sizeof(u32) * 5;
+}
+
+static void tac5xx2_fw_ready(const struct firmware *fmw, void *context)
+{
+	struct tac5xx2_prv *tac_dev = context;
+	struct tac_fw_file *files;
+	u32 fw_hdr_size;
+	u32 num_files = 0;
+	struct tac_fw_hdr hdr;
+	struct tm tm_time;
+	size_t img_sz;
+	u32 offset;
+	s32 ret = 0;
+	u8 *buf;
+
+	if (!fmw || !fmw->data || fmw->size == 0 || fmw->size < TAC_FW_HDR_SIZE + TAC_FW_FILE_HDR) {
+		dev_err(tac_dev->dev, "fw file: %s is empty or invalid\n",
+			tac_dev->fw_binaryname);
+		goto out;
+	}
+
+	/* Verify firmware size from header */
+	fw_hdr_size = get_unaligned_le32(fmw->data);
+	if (fw_hdr_size != fmw->size) {
+		dev_err(tac_dev->dev, "firmware size mismatch: hdr=%u, actual=%zu\n",
+			fw_hdr_size, fmw->size);
+		goto out;
+	}
+
+	files = devm_kzalloc(tac_dev->dev, sizeof(*files) * TAC_MAX_FW_CHUNKS, GFP_KERNEL);
+	buf = devm_kmemdup(tac_dev->dev, fmw->data, fmw->size, GFP_KERNEL);
+	if (!files || !buf)
+		goto out;
+
+	/* validate the cache the firmware */
+	img_sz = fmw->size;
+	offset = tac_fw_read_hdr(buf, &hdr);
+	while (offset < img_sz && num_files < TAC_MAX_FW_CHUNKS) {
+		u32 file_length;
+
+		if (offset + TAC_FW_FILE_HDR > img_sz) {
+			dev_warn(tac_dev->dev, "Incomplete block header at offset %d\n",
+				 offset);
+			goto out;
+		}
+		/* Validate that the file payload doesn't exceed buffer */
+		file_length = get_unaligned_le32(&buf[offset + 12]);
+		/* Check for integer overflow and buffer bounds */
+		if (file_length > img_sz || offset > img_sz - TAC_FW_FILE_HDR ||
+		    file_length > img_sz - offset - TAC_FW_FILE_HDR) {
+			dev_warn(tac_dev->dev, "File at offset %d exceeds buffer: length=%u, available=%zu\n",
+				 offset, file_length, img_sz - offset - TAC_FW_FILE_HDR);
+			goto out;
+		}
+		ret = tac_fw_get_next_file(&buf[offset], img_sz - offset, &files[num_files]);
+		if (ret < 0) {
+			dev_err(tac_dev->dev, "Failed to parse file at offset %d\n", offset);
+			goto out;
+		}
+		offset += ret;
+		num_files++;
+	}
+
+	if (num_files == 0) {
+		dev_err(tac_dev->dev, "firmware with no files\n");
+		goto out;
+	}
+
+	/* cache ready to use validated firmware */
+	tac_dev->fw_file_cnt = num_files;
+	tac_dev->fw_files = files;
+
+	time64_to_tm(hdr.timestamp, 0, &tm_time);
+	dev_dbg(tac_dev->dev, "fw file: %s, num_files=%u, ts:%04ld-%02d-%02d %02d:%02d\n",
+		tac_dev->fw_binaryname, tac_dev->fw_file_cnt,
+		tm_time.tm_year + 1900, tm_time.tm_mon + 1, tm_time.tm_mday,
+		tm_time.tm_hour, tm_time.tm_min);
+	dev_dbg(tac_dev->dev, "fw file: DDC Name: %s\n", hdr.ddc_name);
+	dev_dbg(tac_dev->dev, "fw file: PPC3 Version: 3.%ld.%ld.%ld\n",
+		FIELD_GET(GENMASK(31, 24), hdr.ppc3_ver),
+		FIELD_GET(GENMASK(23, 16), hdr.ppc3_ver),
+		FIELD_GET(GENMASK(15, 8), hdr.ppc3_ver) & 0x3f);
+
+out:
+	complete_all(&tac_dev->fw_caching_complete);
+	if (fmw)
+		release_firmware(fmw);
+}
+
+static int tac_load_and_cache_firmware_async(struct tac5xx2_prv *tac_dev)
+{
+	tac_dev->fw_file_cnt = 0;
+	tac_dev->fw_files = NULL; /* ready to download files */
+
+	return request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT,
+				       tac_dev->fw_binaryname, tac_dev->dev,
+				       GFP_KERNEL, tac_dev, tac5xx2_fw_ready);
+}
+
+static int tac_download(struct tac5xx2_prv *tac_dev)
+{
+	struct tac_fw_file *files = tac_dev->fw_files;
+	u32 num_files = tac_dev->fw_file_cnt;
+	u32 i;
+	int ret = 0;
+
+	for (i = 0; i < num_files; i++) {
+		ret = sdw_nwrite_no_pm(tac_dev->sdw_peripheral, files[i].dest_addr,
+				       files[i].length, files[i].fw_data);
+		if (ret < 0) {
+			dev_dbg(tac_dev->dev,
+				"FW write failed at addr 0x%x: %d\n",
+				files[i].dest_addr, ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+/*
+ * tac5xx2 uses custom firmware binary fw.
+ * This is not using UMP File Download.
+ */
+static s32 tac_download_fw_to_hw(struct tac5xx2_prv *tac_dev)
+{
+	int ret;
+
+	ret = tac_download(tac_dev);
+	if (ret < 0) {
+		dev_err(tac_dev->dev, "Firmware download failed: %d\n", ret);
+		return ret;
+	}
+
+	dev_dbg(tac_dev->dev, "Firmware download complete: %d chunks\n",
+		tac_dev->fw_file_cnt);
+	tac_dev->fw_dl_success = true;
+
+	return 0;
+}
+
+static struct pci_dev *tac_get_pci_dev(struct sdw_slave *peripheral)
+{
+	struct device *dev = &peripheral->dev;
+
+	for (; dev; dev = dev->parent) {
+		if (dev_is_pci(dev))
+			return to_pci_dev(dev);
+	}
+
+	return NULL;
+}
+
+static void tac_generate_fw_name(struct sdw_slave *slave, char *name, size_t size)
+{
+	struct sdw_bus *bus = slave->bus;
+	u16 part_id = slave->id.part_id;
+	u8 unique_id = slave->id.unique_id;
+	struct pci_dev *pci = tac_get_pci_dev(slave);
+
+	if (pci)
+		scnprintf(name, size, "%04X-%04X-%1X-%1X.bin", part_id,
+			  pci->subsystem_device, bus->link_id, unique_id);
+	else
+		/* Default firmware name based on part ID */
+		scnprintf(name, size, "%s%04x-%1X-%1X.bin",
+			  part_id == 0x2883 ? "tas" : "tac",
+			  part_id, bus->link_id, unique_id);
+}
+
+static int tac_io_init(struct device *dev, struct sdw_slave *slave, bool first)
+{
+	struct tac5xx2_prv *tac_dev = dev_get_drvdata(dev);
+	u64 time;
+	int ret;
+
+	if (tac_dev->hw_init) {
+		dev_dbg(dev, "early return hw_init already done..");
+		return 0;
+	}
+
+	time = wait_for_completion_timeout(&tac_dev->fw_caching_complete,
+					   msecs_to_jiffies(TAC5XX2_FW_CACHE_TIMEOUT_MS));
+	if (!time) {
+		ret = -ETIMEDOUT;
+		dev_warn(tac_dev->dev, "%s: fw caching timeout\n", __func__);
+		goto io_init_err;
+	}
+
+	if (tac_dev->fw_files && tac_dev->fw_file_cnt > 0) {
+		ret = tac_download_fw_to_hw(tac_dev);
+		if (ret) {
+			dev_err(tac_dev->dev, "FW download failed, fw: %d\n", ret);
+			goto io_init_err;
+		}
+	}
+
+	if (tac_dev->sa_func_data) {
+		ret = sdca_regmap_write_init(dev, tac_dev->regmap,
+					     tac_dev->sa_func_data);
+		if (ret) {
+			dev_err(dev, "smartamp init table update failed\n");
+			goto io_init_err;
+		}
+		dev_dbg(dev, "smartamp init done\n");
+
+		if (first) {
+			ret = regmap_multi_reg_write(tac_dev->regmap, tac_spk_seq,
+						     ARRAY_SIZE(tac_spk_seq));
+			if (ret) {
+				dev_err(dev, "init writes failed, err=%d", ret);
+				goto io_init_err;
+			}
+		}
+	}
+
+	if (tac_dev->sm_func_data) {
+		ret = sdca_regmap_write_init(dev, tac_dev->regmap,
+					     tac_dev->sm_func_data);
+		if (ret) {
+			dev_err(dev, "smartmic init table update failed\n");
+			goto io_init_err;
+		}
+		dev_dbg(dev, "smartmic init done\n");
+
+		if (first) {
+			ret = regmap_multi_reg_write(tac_dev->regmap, tac_sm_seq,
+						     ARRAY_SIZE(tac_sm_seq));
+			if (ret) {
+				dev_err(tac_dev->dev,
+					"init writes failed, err=%d", ret);
+				goto io_init_err;
+			}
+		}
+	}
+
+	if (tac_dev->uaj_func_data) {
+		ret = sdca_regmap_write_init(dev, tac_dev->regmap,
+					     tac_dev->uaj_func_data);
+		if (ret) {
+			dev_err(dev, "uaj init table update failed\n");
+			goto io_init_err;
+		}
+		dev_dbg(dev, "uaj init done\n");
+
+		if (first) {
+			ret = regmap_multi_reg_write(tac_dev->regmap, tac_uaj_seq,
+						     ARRAY_SIZE(tac_uaj_seq));
+			if (ret) {
+				dev_err(tac_dev->dev,
+					"init writes failed, err=%d", ret);
+				goto io_init_err;
+			}
+
+			if (tac_dev->hs_jack) {
+				ret = tac5xx2_jack_init(tac_dev);
+				if (ret) {
+					dev_err(tac_dev->dev, "jack init failed");
+					goto io_init_err;
+				}
+			}
+		}
+	}
+
+	if (tac_dev->hid_func_data) {
+		ret = sdca_regmap_write_init(dev, tac_dev->regmap,
+					     tac_dev->hid_func_data);
+		if (ret) {
+			dev_err(dev, "hid init table update failed\n");
+			goto io_init_err;
+		}
+		dev_dbg(dev, "hid init done\n");
+	}
+
+	tac_dev->hw_init = true;
+
+	return 0;
+
+io_init_err:
+	dev_err(dev, "init writes failed, err=%d", ret);
+	return ret;
+}
+
+static int tac_update_status(struct sdw_slave *slave,
+			     enum sdw_slave_status status)
+{
+	struct tac5xx2_prv *tac_dev = dev_get_drvdata(&slave->dev);
+	struct device *dev = &slave->dev;
+	bool first = false;
+	int ret;
+
+	tac_dev->status = status;
+	if (status == SDW_SLAVE_UNATTACHED) {
+		tac_dev->hw_init = false;
+		tac_dev->fw_dl_success = false;
+	}
+
+	if (tac_dev->hw_init || tac_dev->status != SDW_SLAVE_ATTACHED) {
+		dev_dbg(dev, "%s: early return, hw_init=%d, status=%d",
+			__func__, tac_dev->hw_init, tac_dev->status);
+		return 0;
+	}
+
+	if (!tac_dev->first_hw_init_done) {
+		pm_runtime_set_active(tac_dev->dev);
+		tac_dev->first_hw_init_done = true;
+		first = true;
+	}
+
+	pm_runtime_get_noresume(tac_dev->dev);
+
+	regcache_mark_dirty(tac_dev->regmap);
+	regcache_cache_only(tac_dev->regmap, false);
+	ret = tac_io_init(&slave->dev, slave, first);
+	if (ret) {
+		dev_err(dev, "Device initialization failed: %d\n", ret);
+		goto err_out;
+	}
+
+	ret = regcache_sync(tac_dev->regmap);
+	if (ret)
+		dev_warn(dev, "Failed to sync regcache after init: %d\n", ret);
+
+err_out:
+	pm_runtime_mark_last_busy(tac_dev->dev);
+	pm_runtime_put_autosuspend(tac_dev->dev);
+
+	return ret;
+}
+
+static int tac5xx2_sdw_read_prop(struct sdw_slave *peripheral)
+{
+	struct device *dev = &peripheral->dev;
+	int ret;
+
+	ret = sdw_slave_read_prop(peripheral);
+	if (ret) {
+		dev_err(dev, "sdw_slave_read_prop failed: %d", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int tac_port_prep(struct sdw_slave *slave, struct sdw_prepare_ch *prep_ch,
+			 enum sdw_port_prep_ops pre_ops)
+{
+	struct device *dev = &slave->dev;
+	struct tac5xx2_prv *tac_dev = dev_get_drvdata(dev);
+	unsigned int val;
+	int ret;
+
+	if (pre_ops != SDW_OPS_PORT_POST_PREP)
+		return 0;
+
+	if (!tac_dev->fw_dl_success)
+		return 0;
+
+	ret = regmap_read(tac_dev->regmap, TAC_DSP_ALGO_STATUS, &val);
+	if (ret) {
+		dev_err(dev, "Failed to read algo status: %d\n", ret);
+		return ret;
+	}
+
+	if (val != TAC_DSP_ALGO_STATUS_RUNNING) {
+		dev_dbg(dev, "Algo not running (0x%02x), re-enabling\n", val);
+		ret = regmap_write(tac_dev->regmap, TAC_DSP_ALGO_STATUS,
+				   TAC_DSP_ALGO_STATUS_RUNNING);
+		if (ret) {
+			dev_err(dev, "Failed to re-enable algo: %d\n", ret);
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static const struct sdw_slave_ops tac_sdw_ops = {
+	.read_prop = tac5xx2_sdw_read_prop,
+	.update_status = tac_update_status,
+	.interrupt_callback = tac_interrupt_callback,
+	.port_prep = tac_port_prep,
+};
+
+static s32 tac_sdw_probe(struct sdw_slave *peripheral,
+			 const struct sdw_device_id *id)
+{
+	struct sdca_function_data *function_data = NULL;
+	struct device *dev = &peripheral->dev;
+	struct tac5xx2_prv *tac_dev;
+	struct regmap *regmap;
+	int ret, i;
+
+	tac_dev = devm_kzalloc(dev, sizeof(*tac_dev), GFP_KERNEL);
+	if (!tac_dev)
+		return dev_err_probe(dev, -ENOMEM,
+				     "Failed devm_kzalloc");
+
+	if (peripheral->sdca_data.num_functions > 0) {
+		dev_dbg(dev, "SDCA functions found: %d",
+			peripheral->sdca_data.num_functions);
+
+		for (i = 0; i < peripheral->sdca_data.num_functions; i++) {
+			struct sdca_function_data **func_ptr;
+			const char *func_name;
+
+			switch (peripheral->sdca_data.function[i].type) {
+			case SDCA_FUNCTION_TYPE_SMART_AMP:
+				func_ptr = &tac_dev->sa_func_data;
+				func_name = "smartamp";
+				break;
+			case SDCA_FUNCTION_TYPE_SMART_MIC:
+				func_ptr = &tac_dev->sm_func_data;
+				func_name = "smartmic";
+				break;
+			case SDCA_FUNCTION_TYPE_UAJ:
+				func_ptr = &tac_dev->uaj_func_data;
+				func_name = "uaj";
+				break;
+			case SDCA_FUNCTION_TYPE_HID:
+				func_ptr = &tac_dev->hid_func_data;
+				func_name = "hid";
+				break;
+			default:
+				continue;
+			}
+
+			function_data = devm_kzalloc(dev, sizeof(*function_data),
+						     GFP_KERNEL);
+			if (!function_data)
+				return dev_err_probe(dev, -ENOMEM,
+						     "failed to allocate %s function data",
+						     func_name);
+			function_data->desc = &peripheral->sdca_data.function[i];
+			ret = sdca_parse_function(dev, peripheral, function_data);
+			if (!ret)
+				*func_ptr = function_data;
+			else
+				devm_kfree(dev, function_data);
+		}
+	}
+
+	dev_dbg(dev, "SDCA functions enabled: SA=%s SM=%s UAJ=%s HID=%s",
+		tac_dev->sa_func_data ? "yes" : "no",
+		tac_dev->sm_func_data ? "yes" : "no",
+		tac_dev->uaj_func_data ? "yes" : "no",
+		tac_dev->hid_func_data ? "yes" : "no");
+
+	tac_dev->dev = dev;
+	tac_dev->sdw_peripheral = peripheral;
+	tac_dev->hw_init = false;
+	tac_dev->first_hw_init_done = false;
+	tac_dev->part_id = id->part_id;
+	dev_set_drvdata(dev, tac_dev);
+
+	regmap = devm_regmap_init_sdw_mbq_cfg(&peripheral->dev, peripheral,
+					      &tac_regmap, &tac_mbq_cfg);
+	if (IS_ERR(regmap))
+		return dev_err_probe(dev, PTR_ERR(regmap),
+				     "Failed devm_regmap_init_sdw\n");
+
+	regcache_cache_only(regmap, true);
+	tac_dev->regmap = regmap;
+	tac_dev->jack_type = 0;
+	init_completion(&tac_dev->fw_caching_complete);
+
+	if (tac_has_dsp_algo(tac_dev)) {
+		tac_generate_fw_name(peripheral, tac_dev->fw_binaryname,
+				     sizeof(tac_dev->fw_binaryname));
+
+		ret = tac_load_and_cache_firmware_async(tac_dev);
+		if (ret) {
+			complete_all(&tac_dev->fw_caching_complete);
+			dev_dbg(dev, "failed to load fw: %d, use rom mode\n", ret);
+		}
+	} else {
+		complete_all(&tac_dev->fw_caching_complete);
+	}
+
+	ret = tac_init(tac_dev);
+	if (ret)
+		return dev_err_probe(dev, ret,
+				     "failed to initialize tac device\n");
+
+	/* set autosuspend parameters */
+	pm_runtime_set_autosuspend_delay(dev, 3000);
+	pm_runtime_use_autosuspend(dev);
+
+	/* make sure the device does not suspend immediately */
+	pm_runtime_mark_last_busy(dev);
+
+	pm_runtime_enable(dev);
+	/* the device is still not in active */
+
+	return 0;
+}
+
+static void tac_sdw_remove(struct sdw_slave *peripheral)
+{
+	struct tac5xx2_prv *tac_dev = dev_get_drvdata(&peripheral->dev);
+
+	pm_runtime_disable(tac_dev->dev);
+
+	dev_set_drvdata(&peripheral->dev, NULL);
+}
+
+static const struct sdw_device_id tac_sdw_id[] = {
+	SDW_SLAVE_ENTRY(0x0102, 0x5572, 0),
+	SDW_SLAVE_ENTRY(0x0102, 0x5672, 0),
+	SDW_SLAVE_ENTRY(0x0102, 0x5682, 0),
+	SDW_SLAVE_ENTRY(0x0102, 0x2883, 0),
+	{},
+};
+MODULE_DEVICE_TABLE(sdw, tac_sdw_id);
+
+static struct sdw_driver tac_sdw_driver = {
+	.driver = {
+		.name = "slave-tac5xx2",
+		.pm = pm_ptr(&tac5xx2_sdca_pm),
+	},
+	.probe = tac_sdw_probe,
+	.remove = tac_sdw_remove,
+	.ops = &tac_sdw_ops,
+	.id_table = tac_sdw_id,
+};
+module_sdw_driver(tac_sdw_driver);
+
+MODULE_IMPORT_NS("SND_SOC_SDCA");
+MODULE_AUTHOR("Texas Instruments Inc.");
+MODULE_DESCRIPTION("ASoC TAC5XX2 SoundWire Driver");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/tac5xx2.h b/sound/soc/codecs/tac5xx2.h
new file mode 100644
index 0000000000000..eed8e6cf3498b
--- /dev/null
+++ b/sound/soc/codecs/tac5xx2.h
@@ -0,0 +1,259 @@
+/* SPDX-License-Identifier: GPL-2.0
+ *
+ * ALSA SoC Texas Instruments TAC5XX2 Audio Smart Amplifier
+ *
+ * Copyright (C) 2025 Texas Instruments Incorporated
+ * https://www.ti.com
+ *
+ * This the header file for TAC5XX2 family of devices
+ * which includes TAC5572, TAC5672, TAC5682 and TAS2883
+ *
+ * Author: Niranjan H Y <niranjanhy@ti.com>
+ */
+#ifndef __RGL_TAC5XX2_H__
+#define __RGL_TAC5XX2_H__
+
+/* for soundwire */
+#define TAC_REG_SDW(book, page, reg) (((book) * 256 * 128) + \
+				      0x3000000 + ((page) * 128) + (reg))
+
+/* page 0 registers */
+#define TAC_SW_RESET		TAC_REG_SDW(0, 0, 1)
+#define TAC_SLEEP_MODEZ		TAC_REG_SDW(0, 0, 2)
+#define TAC_FEATURE_PDZ		TAC_REG_SDW(0, 0, 3)
+#define TAC_TX_CH_EN		TAC_REG_SDW(0, 0, 4)
+#define TAC_RX_CH_PD		TAC_REG_SDW(0, 0, 5)
+#define TAC_SHDNZ_CFG		TAC_REG_SDW(0, 0, 6)
+#define TAC_MISC_CFG0		TAC_REG_SDW(0, 0, 7)
+#define TAC_MISC_CFG1		TAC_REG_SDW(0, 0, 8)
+#define TAC_GPIO1_CFG0		TAC_REG_SDW(0, 0, 9)
+#define TAC_GPIO2_CFG0		TAC_REG_SDW(0, 0, 10)
+#define TAC_GPIO3_CFG0		TAC_REG_SDW(0, 0, 11)
+#define TAC_GPIO4_CFG0		TAC_REG_SDW(0, 0, 12)
+#define TAC_GPIO5_CFG0		TAC_REG_SDW(0, 0, 13)
+#define TAC_GPIO6_CFG0		TAC_REG_SDW(0, 0, 14)
+#define TAC_INTF_CFG1		TAC_REG_SDW(0, 0, 15)
+#define TAC_INTF_CFG5		TAC_REG_SDW(0, 0, 16)
+#define TAC_PASI_BCLK_CFG0	TAC_REG_SDW(0, 0, 17)
+#define TAC_PASI_FSYNC_CFG0	TAC_REG_SDW(0, 0, 18)
+#define TAC_PASI_DIN1_CFG0	TAC_REG_SDW(0, 0, 19)
+#define TAC_PASI_DIN2_CFG0	TAC_REG_SDW(0, 0, 20)
+#define TAC_PDM_DIN1_CFG0	TAC_REG_SDW(0, 0, 21)
+#define TAC_PDM_DIN2_CFG0	TAC_REG_SDW(0, 0, 22)
+#define TAC_MCLK_SEL		TAC_REG_SDW(0, 0, 23)
+#define TAC_I2C2_CFG0		TAC_REG_SDW(0, 0, 24)
+#define TAC_SDW_IO_CFG0		TAC_REG_SDW(0, 0, 25)
+#define TAC_SDW_CLK_CFG0	TAC_REG_SDW(0, 0, 26)
+#define TAC_PASI_CFG0		TAC_REG_SDW(0, 0, 27)
+#define TAC_PASI_CFG1		TAC_REG_SDW(0, 0, 28)
+#define TAC_PASI_TX_CFG0	TAC_REG_SDW(0, 0, 29)
+#define TAC_PASI_TX_CFG1	TAC_REG_SDW(0, 0, 30)
+#define TAC_PASI_TX_CFG2	TAC_REG_SDW(0, 0, 31)
+#define TAC_PASI_TX_CFG3	TAC_REG_SDW(0, 0, 32)
+#define TAC_PASI_TX_CH1_CFG0	TAC_REG_SDW(0, 0, 33)
+#define TAC_PASI_TX_CH2_CFG0	TAC_REG_SDW(0, 0, 34)
+#define TAC_PASI_TX_CH3_CFG0	TAC_REG_SDW(0, 0, 35)
+#define TAC_PASI_TX_CH4_CFG0	TAC_REG_SDW(0, 0, 36)
+#define TAC_PASI_TX_CH5_CFG0	TAC_REG_SDW(0, 0, 37)
+#define TAC_PASI_TX_CH6_CFG0	TAC_REG_SDW(0, 0, 38)
+#define TAC_PASI_TX_CH7_CFG0	TAC_REG_SDW(0, 0, 39)
+#define TAC_PASI_TX_CH8_CFG0	TAC_REG_SDW(0, 0, 40)
+#define TAC_PASI_RX_CFG0	TAC_REG_SDW(0, 0, 41)
+#define TAC_PASI_RX_CFG1	TAC_REG_SDW(0, 0, 42)
+#define TAC_PASI_RX_CFG2	TAC_REG_SDW(0, 0, 43)
+#define TAC_PASI_RX_CH1_CFG0	TAC_REG_SDW(0, 0, 44)
+#define TAC_PASI_RX_CH2_CFG0	TAC_REG_SDW(0, 0, 45)
+#define TAC_PASI_RX_CH3_CFG0	TAC_REG_SDW(0, 0, 46)
+#define TAC_PASI_RX_CH4_CFG0	TAC_REG_SDW(0, 0, 47)
+#define TAC_PASI_RX_CH5_CFG0	TAC_REG_SDW(0, 0, 48)
+#define TAC_PASI_RX_CH6_CFG0	TAC_REG_SDW(0, 0, 49)
+#define TAC_PASI_RX_CH7_CFG0	TAC_REG_SDW(0, 0, 50)
+#define TAC_PASI_RX_CH8_CFG0	TAC_REG_SDW(0, 0, 51)
+#define TAC_ADC_CH1_CFG0	TAC_REG_SDW(0, 0, 52)
+#define TAC_ADC_DVOL_CFG0	TAC_REG_SDW(0, 0, 53)
+#define TAC_ADC_CH1_FGAIN	TAC_REG_SDW(0, 0, 54)
+#define TAC_ADC_CH1_CFG1	TAC_REG_SDW(0, 0, 55)
+#define TAC_ADC_CH2_CFG0	TAC_REG_SDW(0, 0, 57)
+#define TAC_ADC_DVOL_CFG1	TAC_REG_SDW(0, 0, 58)
+#define TAC_ADC_CH2_FGAIN	TAC_REG_SDW(0, 0, 59)
+#define TAC_ADC_CH2_CFG1	TAC_REG_SDW(0, 0, 60)
+#define TAC_ADC_CFG1		TAC_REG_SDW(0, 0, 62)
+#define TAC_PDM_CH1_DVOL	TAC_REG_SDW(0, 0, 63)
+#define TAC_PDM_CH1_FGAIN	TAC_REG_SDW(0, 0, 64)
+#define TAC_PDM_CH1_CFG0	TAC_REG_SDW(0, 0, 65)
+#define TAC_PDM_CH2_DVOL	TAC_REG_SDW(0, 0, 67)
+#define TAC_PDM_CH2_FGAIN	TAC_REG_SDW(0, 0, 68)
+#define TAC_PDM_CH2_CFG2	TAC_REG_SDW(0, 0, 69)
+#define TAC_PDM_CH3_DVOL	TAC_REG_SDW(0, 0, 71)
+#define TAC_PDM_CH3_FGAIN	TAC_REG_SDW(0, 0, 72)
+#define TAC_PDM_CH3_CFG0	TAC_REG_SDW(0, 0, 73)
+#define TAC_PDM_CH4_DVOL	TAC_REG_SDW(0, 0, 75)
+#define TAC_PDM_CH4_FGAIN	TAC_REG_SDW(0, 0, 76)
+#define TAC_PDM_CH4_CFG0	TAC_REG_SDW(0, 0, 77)
+#define TAC_MICBIAS_CFG0	TAC_REG_SDW(0, 0, 79)
+#define TAC_MICPREAMP_CFG	TAC_REG_SDW(0, 0, 80)
+#define TAC_MICBIAS_CFG1	TAC_REG_SDW(0, 0, 81)
+#define TAC_CLASSD_CH1_DVOL	TAC_REG_SDW(0, 0, 82)
+#define TAC_CLASSD_CH1_FGAIN	TAC_REG_SDW(0, 0, 83)
+#define TAC_CLASSD_CH2_DVOL	TAC_REG_SDW(0, 0, 85)
+#define TAC_CLASSD_CH2_FGAIN	TAC_REG_SDW(0, 0, 86)
+#define TAC_GCHP_CH1_DVOL	TAC_REG_SDW(0, 0, 88)
+#define TAC_GCHP_CH1_FGAIN	TAC_REG_SDW(0, 0, 89)
+#define TAC_GCHP_CH2_DVOL	TAC_REG_SDW(0, 0, 91)
+#define TAC_GCHP_CH2_FGAIN	TAC_REG_SDW(0, 0, 92)
+#define TAC_AMP_LVL_CFG0	TAC_REG_SDW(0, 0, 94)
+#define TAC_AMP_LVL_CFG1	TAC_REG_SDW(0, 0, 95)
+#define TAC_AMP_LVL_CFG2	TAC_REG_SDW(0, 0, 96)
+#define TAC_AMP_LVL_CFG3	TAC_REG_SDW(0, 0, 97)
+#define TAC_EFF_MODE_CFG0	TAC_REG_SDW(0, 0, 98)
+#define TAC_EFF_MODE_CFG1	TAC_REG_SDW(0, 0, 99)
+#define TAC_CLASSD_CFG0		TAC_REG_SDW(0, 0, 100)
+#define TAC_CLASSD_CFG1		TAC_REG_SDW(0, 0, 101)
+#define TAC_CLASSD_CFG3		TAC_REG_SDW(0, 0, 102)
+#define TAC_CLASSD_CFG4		TAC_REG_SDW(0, 0, 103)
+#define TAC_CLASSD_CFG5		TAC_REG_SDW(0, 0, 104)
+#define TAC_CLASSD_CFG6		TAC_REG_SDW(0, 0, 105)
+#define TAC_CLASSD_CFG8		TAC_REG_SDW(0, 0, 106)
+#define TAC_ISNS_CFG		TAC_REG_SDW(0, 0, 107)
+#define TAC_DSP_CFG0		TAC_REG_SDW(0, 0, 108)
+#define TAC_DSP_CFG1		TAC_REG_SDW(0, 0, 109)
+#define TAC_DSP_CFG2		TAC_REG_SDW(0, 0, 110)
+#define TAC_DSP_CFG3		TAC_REG_SDW(0, 0, 111)
+#define TAC_JACK_DET_CFG1	TAC_REG_SDW(0, 0, 112)
+#define TAC_JACK_DET_CFG2	TAC_REG_SDW(0, 0, 113)
+#define TAC_JACK_DET_CFG3	TAC_REG_SDW(0, 0, 114)
+#define TAC_JACK_DET_CFG4	TAC_REG_SDW(0, 0, 115)
+#define TAC_JACK_DET_CFG7	TAC_REG_SDW(0, 0, 116)
+#define TAC_UJ_IMPEDANCE_L	TAC_REG_SDW(0, 0, 117)
+#define TAC_UJ_IMPEDANCE_R	TAC_REG_SDW(0, 0, 118)
+#define UJ_IMPEDANCE_L		TAC_REG_SDW(0, 0, 119)
+#define UJ_IMPEDANCE_R		TAC_REG_SDW(0, 0, 120)
+#define TAC_GP_ANA_STS		TAC_REG_SDW(0, 0, 123)
+#define TAC_DEV_ID		TAC_REG_SDW(0, 0, 124)
+#define TAC_REV_ID		TAC_REG_SDW(0, 0, 125)
+#define TAC_I2C_CKSUM		TAC_REG_SDW(0, 0, 126)
+#define TAC_BOOK		TAC_REG_SDW(0, 0, 127)
+
+#define TAC_INT_CFG		TAC_REG_SDW(0, 2, 1)
+#define TAC_INT_CFG_CLR_REG	BIT(3)
+
+/* smartamp function */
+#define TAC_FUNCTION_ID_SA	0x1
+
+#define TAC_SDCA_ENT_ENT0	0x0
+#define TAC_SDCA_ENT_PPU21	0x1
+#define TAC_SDCA_ENT_FU21	0x2
+#define TAC_SDCA_ENT_FU26	0x3
+#define TAC_SDCA_ENT_XU22	0x4
+#define TAC_SDCA_ENT_CS24	0x5
+#define TAC_SDCA_ENT_CS21	0x6
+#define TAC_SDCA_ENT_CS25	0x7
+#define TAC_SDCA_ENT_CS26	0x8
+#define TAC_SDCA_ENT_CS28	0x9
+#define TAC_SDCA_ENT_PPU26	0xa
+#define TAC_SDCA_ENT_FU23	0xb
+#define TAC_SDCA_ENT_PDE23	0xc
+#define TAC_SDCA_ENT_TG23	0x12
+#define TAC_SDCA_ENT_IT21	0x13
+#define TAC_SDCA_ENT_IT29	0x14
+#define TAC_SDCA_ENT_IT26	0x15
+#define TAC_SDCA_ENT_IT28	0x16
+#define TAC_SDCA_ENT_OT24	0x17
+#define TAC_SDCA_ENT_OT23	0x18
+#define TAC_SDCA_ENT_OT25	0x19
+#define TAC_SDCA_ENT_OT28	0x1a
+#define TAC_SDCA_ENT_OT27	0x1c
+#define TAC_SDCA_ENT_SPE199	0x21
+#define TAC_SDCA_ENT_OT20	0x24
+#define TAC_SDCA_ENT_FU27	0x26
+#define TAC_SDCA_ENT_FU20	0x27
+#define TAC_SDCA_ENT_PDE24	0x2e
+#define TAC_SDCA_ENT_PDE27	0x2f
+#define TAC_SDCA_ENT_PDE28	0x30
+#define TAC_SDCA_ENT_PDE20	0x31
+#define TAC_SDCA_ENT_SAPU29	0x35
+
+/* Control selector definitions */
+#define TAC_SDCA_MASTER_GAIN	0x0B
+#define TAC_SDCA_MASTER_MUTE	0x01
+#define TAC_SDCA_CHANNEL_MUTE	0x01
+#define TAC_SDCA_CHANNEL_GAIN	0x02
+#define TAC_SDCA_POSTURENUMBER	0x10
+#define TAC_SDCA_REQUESTED_PS	0x01
+#define TAC_SDCA_ACTUAL_PS	0x10
+#define TAC_SDCA_CHANNEL_VOLUME	0x02
+
+/* 2. smart mic function */
+#define TAC_FUNCTION_ID_SM	0x2
+
+#define TAC_SDCA_ENT_IT11   0x1
+#define TAC_SDCA_ENT_OT113	0x2
+#define TAC_SDCA_ENT_CS11	0x3
+#define TAC_SDCA_ENT_CS18	0x4
+#define TAC_SDCA_ENT_FU113	0x5
+#define TAC_SDCA_ENT_FU13	0x6
+#define TAC_SDCA_ENT_FU11	0x8
+#define TAC_SDCA_ENT_XU12	0xa
+#define TAC_SDCA_ENT_CS113	0xc
+#define TAC_SDCA_ENT_CX11	0xf
+#define TAC_SDCA_ENT_PDE11	0x12
+#define TAC_SDCA_ENT_PPU11	0x9
+
+/* controls */
+#define TAC_SDCA_CTL_USAGE	0x04
+#define TAC_SDCA_CTL_IT_CLUSTER	0x10
+#define TAC_SDCA_CTL_OT_DP_SEL	0x11
+#define TAC_SDCA_CTL_XU_BYPASS	0x01
+/* cx */
+#define TAC_SDCA_CTL_CX_CLK_SEL		0x01
+/* cs */
+#define TAC_SDCA_CTL_CS_CLKVLD		0x02
+#define TAC_SDCA_CTL_CS_SAMP_RATE_IDX	0x10
+/* cs113 end */
+/* ppu */
+#define TAC_SDCA_CTL_PPU_POSTURE_NUM	0x10
+
+/* 3. UAJ function */
+#define TAC_FUNCTION_ID_UAJ	0x3
+#define TAC_SDCA_ENT_PDE47	0x35
+#define TAC_SDCA_ENT_PDE34	0x32
+#define TAC_SDCA_ENT_FU41	0x26 /* user */
+#define TAC_SDCA_ENT_IT41	0x07
+#define TAC_SDCA_ENT_XU42	0x2C
+#define TAC_SDCA_ENT_CS41	0x30
+#define TAC_SDCA_ENT_OT45	0x0E
+#define TAC_SDCA_ENT_IT33	0x03
+#define TAC_SDCA_ENT_OT36	0x0A
+#define TAC_SDCA_ENT_FU36	0x28
+#define TAC_SDCA_ENT_CS36	0x2E
+#define TAC_SDCA_ENT_GE35	0x3B /* 59 */
+
+#define TAC_SDCA_CTL_SEL_MODE	0x1
+#define TAC_SDCA_CTL_DET_MODE	0x2
+
+/* 4. HID function */
+#define TAC_FUNCTION_ID_HID	0x4
+#define TAC_SDCA_ENT_HID1	0x1
+/* HID Control Selectors */
+#define TAC_SDCA_CTL_HIDTX_CURRENT_OWNER	0x10
+#define TAC_SDCA_CTL_HIDTX_MESSAGE_OFFSET	0x12
+#define TAC_SDCA_CTL_HIDTX_MESSAGE_LENGTH	0x13
+#define TAC_SDCA_CTL_DETECTED_MODE	0x10
+#define TAC_SDCA_CTL_SELECTED_MODE	0x11
+
+#define TAC_BUF_ADDR_HID1	0x44007F80
+
+/* DAI interfaces */
+#define TAC5XX2_SPK	0
+#define TAC5XX2_DMIC	2
+#define TAC5XX2_UAJ	3
+
+/* Port numbers for DAIs */
+#define TAC_SDW_PORT_NUM_SPK_PLAYBACK	1
+#define TAC_SDW_PORT_NUM_SPK_CAPTURE	2
+#define TAC_SDW_PORT_NUM_DMIC		3
+#define TAC_SDW_PORT_NUM_UAJ_PLAYBACK	4
+#define TAC_SDW_PORT_NUM_UAJ_CAPTURE	7
+#define TAC_SDW_PORT_NUM_IV_SENSE	8
+
+#endif
diff --git a/sound/soc/codecs/tas2783-sdw.c b/sound/soc/codecs/tas2783-sdw.c
index 90008d2d06e2c..69d03ddc7a0ba 100644
--- a/sound/soc/codecs/tas2783-sdw.c
+++ b/sound/soc/codecs/tas2783-sdw.c
@@ -1082,22 +1082,12 @@ static s32 tas2783_sdca_dev_resume(struct device *dev)
 {
 	struct sdw_slave *slave = dev_to_sdw_dev(dev);
 	struct tas2783_prv *tas_dev = dev_get_drvdata(dev);
-	unsigned long t;
-
-	if (!slave->unattach_request)
-		goto regmap_sync;
-
-	t = wait_for_completion_timeout(&slave->initialization_complete,
-					msecs_to_jiffies(TAS2783_PROBE_TIMEOUT));
-	if (!t) {
-		dev_err(&slave->dev, "resume: initialization timed out\n");
-		sdw_show_ping_status(slave->bus, true);
-		return -ETIMEDOUT;
-	}
+	int ret;
 
-	slave->unattach_request = 0;
+	ret = sdw_slave_wait_for_init(slave, TAS2783_PROBE_TIMEOUT);
+	if (ret)
+		return ret;
 
-regmap_sync:
 	regcache_cache_only(tas_dev->regmap, false);
 	regcache_sync(tas_dev->regmap);
 	return 0;
@@ -1310,10 +1300,10 @@ static s32 tas_sdw_probe(struct sdw_slave *peripheral,
 			return dev_err_probe(dev, -ENOMEM,
 					     "failed to parse sdca functions");
 
+		function_data->desc = &peripheral->sdca_data.function[i];
+
 		/* Parse the function */
-		ret = sdca_parse_function(dev, peripheral,
-					  &peripheral->sdca_data.function[i],
-					  function_data);
+		ret = sdca_parse_function(dev, peripheral, function_data);
 		if (!ret)
 			tas_dev->sa_func_data = function_data;
 		else
diff --git a/sound/soc/codecs/tas675x.c b/sound/soc/codecs/tas675x.c
new file mode 100644
index 0000000000000..6f89a422f3c62
--- /dev/null
+++ b/sound/soc/codecs/tas675x.c
@@ -0,0 +1,2194 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * ALSA SoC Texas Instruments TAS67524 Quad-Channel Audio Amplifier
+ *
+ * Copyright (C) 2026 Texas Instruments Incorporated - https://www.ti.com/
+ *	Author: Sen Wang <sen@ti.com>
+ */
+
+#include <linux/bitfield.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/i2c.h>
+#include <linux/regmap.h>
+#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
+#include <linux/delay.h>
+#include <linux/property.h>
+#include <linux/interrupt.h>
+#include <linux/workqueue.h>
+#include <linux/pm_runtime.h>
+#include <linux/iopoll.h>
+#include <sound/soc.h>
+#include <sound/tlv.h>
+#include <sound/pcm_params.h>
+
+#include "tas675x.h"
+
+#define TAS675X_FAULT_CHECK_INTERVAL_MS  200
+
+enum tas675x_type {
+	TAS67524,
+};
+
+struct tas675x_reg_param {
+	u8  page;
+	u8  reg;
+	u32 val;
+};
+
+struct tas675x_priv {
+	struct device *dev;
+	struct regmap *regmap;
+	enum tas675x_type dev_type;
+	/* Custom regmap lock; protects writes across books */
+	struct mutex io_lock;
+
+	struct gpio_desc *pd_gpio;
+	struct gpio_desc *stby_gpio;
+	struct regulator_bulk_data supplies[2];
+	struct regulator *vbat;
+	bool fast_boot;
+
+	int audio_slot;
+	int llp_slot;
+	int vpredict_slot;
+	int isense_slot;
+	int bclk_offset;
+	int slot_width;
+	unsigned int tx_mask;
+
+	int gpio1_func;
+	int gpio2_func;
+
+	unsigned long active_playback_dais;
+	unsigned long active_capture_dais;
+	unsigned int rate;
+	unsigned int saved_rtldg_en;
+#define TAS675X_DSP_PARAM_NUM	2
+	struct tas675x_reg_param dsp_params[TAS675X_DSP_PARAM_NUM];
+
+	/* Fault monitor, disabled when Fault IRQ is used */
+	struct delayed_work fault_check_work;
+#define TAS675X_FAULT_REGS_NUM	9
+	unsigned int last_status[TAS675X_FAULT_REGS_NUM];
+};
+
+static const char * const tas675x_supply_names[] = {
+	"dvdd",		/* Digital power supply */
+	"pvdd",		/* Output powerstage supply */
+};
+
+/* Page 1 setup initialization defaults */
+static const struct reg_sequence tas675x_page1_init[] = {
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0xC8), 0x20),	/* Charge pump clock */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0x2F), 0x90),	/* VBAT idle */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0x29), 0x40),	/* OC/CBC threshold */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0x2E), 0x0C),	/* OC/CBC config */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0xC5), 0x02),	/* OC/CBC config */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0xC6), 0x10),	/* OC/CBC config */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0x1F), 0x20),	/* OC/CBC config */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0x16), 0x01),	/* OC/CBC config */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0x1E), 0x04),	/* OC/CBC config */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0xC1), 0x00),	/* CH1 DC fault */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0xC2), 0x04),	/* CH2 DC fault */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0xC3), 0x00),	/* CH3 DC fault */
+	REG_SEQ0(TAS675X_PAGE_REG(1, 0xC4), 0x00),	/* CH4 DC fault */
+};
+
+static inline const char *tas675x_state_name(unsigned int state)
+{
+	switch (state & 0x0F) {
+	case TAS675X_STATE_DEEPSLEEP:	return "DEEPSLEEP";
+	case TAS675X_STATE_LOAD_DIAG:	return "LOAD_DIAG";
+	case TAS675X_STATE_SLEEP:	return "SLEEP";
+	case TAS675X_STATE_HIZ:		return "HIZ";
+	case TAS675X_STATE_PLAY:	return "PLAY";
+	case TAS675X_STATE_FAULT:	return "FAULT";
+	case TAS675X_STATE_AUTOREC:	return "AUTOREC";
+	default:			return "UNKNOWN";
+	}
+}
+
+static inline int tas675x_set_state_all(struct tas675x_priv *tas, u8 state)
+{
+	const struct reg_sequence seq[] = {
+		REG_SEQ0(TAS675X_STATE_CTRL_CH1_CH2_REG, state),
+		REG_SEQ0(TAS675X_STATE_CTRL_CH3_CH4_REG, state),
+	};
+
+	return regmap_multi_reg_write(tas->regmap, seq, ARRAY_SIZE(seq));
+}
+
+static inline int tas675x_select_book(struct regmap *regmap, u8 book)
+{
+	int ret;
+
+	/* Reset page to 0 before switching books */
+	ret = regmap_write(regmap, TAS675X_PAGE_CTRL_REG, 0x00);
+	if (!ret)
+		ret = regmap_write(regmap, TAS675X_BOOK_CTRL_REG, book);
+
+	return ret;
+}
+
+/* Raw I2C version of tas675x_select_book, must be called with io_lock held */
+static inline int __tas675x_select_book(struct tas675x_priv *tas, u8 book)
+{
+	struct i2c_client *client = to_i2c_client(tas->dev);
+	int ret;
+
+	/* Reset page to 0 before switching books */
+	ret = i2c_smbus_write_byte_data(client, TAS675X_PAGE_CTRL_REG, 0x00);
+	if (ret)
+		return ret;
+
+	return i2c_smbus_write_byte_data(client, TAS675X_BOOK_CTRL_REG, book);
+}
+
+static int tas675x_dsp_mem_write(struct tas675x_priv *tas, u8 page, u8 reg, u32 val)
+{
+	struct i2c_client *client = to_i2c_client(tas->dev);
+	u8 buf[4];
+	int ret;
+
+	/* DSP registers are 32 bit big-endian */
+	buf[0] = (val >> 24) & 0xFF;
+	buf[1] = (val >> 16) & 0xFF;
+	buf[2] = (val >> 8) & 0xFF;
+	buf[3] = val & 0xFF;
+
+	/*
+	 * DSP regs in a different book, therefore block
+	 * regmap access before completion.
+	 */
+	mutex_lock(&tas->io_lock);
+
+	ret = __tas675x_select_book(tas, TAS675X_BOOK_DSP);
+	if (ret)
+		goto out;
+
+	ret = i2c_smbus_write_byte_data(client, TAS675X_PAGE_CTRL_REG, page);
+	if (ret)
+		goto out;
+
+	ret = i2c_smbus_write_i2c_block_data(client, reg, sizeof(buf), buf);
+
+out:
+	__tas675x_select_book(tas, TAS675X_BOOK_DEFAULT);
+	mutex_unlock(&tas->io_lock);
+
+	return ret;
+}
+
+static int tas675x_dsp_mem_read(struct tas675x_priv *tas, u8 page, u8 reg, u32 *val)
+{
+	struct i2c_client *client = to_i2c_client(tas->dev);
+	u8 buf[4];
+	int ret;
+
+	/*
+	 * DSP regs in a different book, therefore block
+	 * regmap access before completion.
+	 */
+	mutex_lock(&tas->io_lock);
+
+	ret = __tas675x_select_book(tas, TAS675X_BOOK_DSP);
+	if (ret)
+		goto out;
+
+	ret = i2c_smbus_write_byte_data(client, TAS675X_PAGE_CTRL_REG, page);
+	if (ret)
+		goto out;
+
+	ret = i2c_smbus_read_i2c_block_data(client, reg, sizeof(buf), buf);
+	if (ret == sizeof(buf)) {
+		*val = (buf[0] << 24) | (buf[1] << 16) | (buf[2] << 8) | buf[3];
+		ret = 0;
+	} else if (ret >= 0) {
+		ret = -EIO;
+	}
+
+out:
+	__tas675x_select_book(tas, TAS675X_BOOK_DEFAULT);
+	mutex_unlock(&tas->io_lock);
+
+	return ret;
+}
+
+static const struct {
+	const char *name;
+	int val;
+} tas675x_gpio_func_map[] = {
+	/* Output functions */
+	{ "low",            TAS675X_GPIO_SEL_LOW },
+	{ "auto-mute",      TAS675X_GPIO_SEL_AUTO_MUTE_ALL },
+	{ "auto-mute-ch4",  TAS675X_GPIO_SEL_AUTO_MUTE_CH4 },
+	{ "auto-mute-ch3",  TAS675X_GPIO_SEL_AUTO_MUTE_CH3 },
+	{ "auto-mute-ch2",  TAS675X_GPIO_SEL_AUTO_MUTE_CH2 },
+	{ "auto-mute-ch1",  TAS675X_GPIO_SEL_AUTO_MUTE_CH1 },
+	{ "sdout2",         TAS675X_GPIO_SEL_SDOUT2 },
+	{ "sdout1",         TAS675X_GPIO_SEL_SDOUT1 },
+	{ "warn",           TAS675X_GPIO_SEL_WARN },
+	{ "fault",          TAS675X_GPIO_SEL_FAULT },
+	{ "clock-sync",     TAS675X_GPIO_SEL_CLOCK_SYNC },
+	{ "invalid-clock",  TAS675X_GPIO_SEL_INVALID_CLK },
+	{ "high",           TAS675X_GPIO_SEL_HIGH },
+	/* Input functions */
+	{ "mute",           TAS675X_GPIO_IN_MUTE },
+	{ "phase-sync",     TAS675X_GPIO_IN_PHASE_SYNC },
+	{ "sdin2",          TAS675X_GPIO_IN_SDIN2 },
+	{ "deep-sleep",     TAS675X_GPIO_IN_DEEP_SLEEP },
+	{ "hiz",            TAS675X_GPIO_IN_HIZ },
+	{ "play",           TAS675X_GPIO_IN_PLAY },
+	{ "sleep",          TAS675X_GPIO_IN_SLEEP },
+};
+
+static int tas675x_gpio_func_parse(struct device *dev, const char *propname)
+{
+	const char *str;
+	int i, ret;
+
+	ret = device_property_read_string(dev, propname, &str);
+	if (ret)
+		return -1;
+
+	for (i = 0; i < ARRAY_SIZE(tas675x_gpio_func_map); i++) {
+		if (!strcmp(str, tas675x_gpio_func_map[i].name))
+			return tas675x_gpio_func_map[i].val;
+	}
+
+	dev_warn(dev, "Invalid %s value '%s'\n", propname, str);
+	return -1;
+}
+
+static const struct {
+	unsigned int reg;
+	unsigned int mask;
+} tas675x_gpio_input_table[TAS675X_GPIO_IN_NUM] = {
+	[TAS675X_GPIO_IN_ID_MUTE] = {
+		TAS675X_GPIO_INPUT_MUTE_REG, TAS675X_GPIO_IN_MUTE_MASK },
+	[TAS675X_GPIO_IN_ID_PHASE_SYNC] = {
+		TAS675X_GPIO_INPUT_SYNC_REG, TAS675X_GPIO_IN_SYNC_MASK },
+	[TAS675X_GPIO_IN_ID_SDIN2] = {
+		TAS675X_GPIO_INPUT_SDIN2_REG, TAS675X_GPIO_IN_SDIN2_MASK },
+	[TAS675X_GPIO_IN_ID_DEEP_SLEEP] = {
+		TAS675X_GPIO_INPUT_SLEEP_HIZ_REG, TAS675X_GPIO_IN_DEEP_SLEEP_MASK },
+	[TAS675X_GPIO_IN_ID_HIZ] = {
+		TAS675X_GPIO_INPUT_SLEEP_HIZ_REG, TAS675X_GPIO_IN_HIZ_MASK },
+	[TAS675X_GPIO_IN_ID_PLAY] = {
+		TAS675X_GPIO_INPUT_PLAY_SLEEP_REG, TAS675X_GPIO_IN_PLAY_MASK },
+	[TAS675X_GPIO_IN_ID_SLEEP] = {
+		TAS675X_GPIO_INPUT_PLAY_SLEEP_REG, TAS675X_GPIO_IN_SLEEP_MASK },
+};
+
+static void tas675x_config_gpio_pin(struct regmap *regmap, int func_id,
+				    unsigned int out_sel_reg,
+				    unsigned int pin_idx,
+				    unsigned int *gpio_ctrl)
+{
+	int id;
+
+	if (func_id < 0)
+		return;
+
+	if (func_id & TAS675X_GPIO_FUNC_INPUT) {
+		/* 3-bit mux: 0 = disabled, 0b1 = GPIO1, 0b10 = GPIO2 */
+		id = func_id & ~TAS675X_GPIO_FUNC_INPUT;
+		regmap_update_bits(regmap,
+				   tas675x_gpio_input_table[id].reg,
+				   tas675x_gpio_input_table[id].mask,
+				   (pin_idx + 1) << __ffs(tas675x_gpio_input_table[id].mask));
+	} else {
+		/* Output GPIO, update selection register and enable bit */
+		regmap_write(regmap, out_sel_reg, func_id);
+		*gpio_ctrl |= pin_idx ? TAS675X_GPIO2_OUTPUT_EN : TAS675X_GPIO1_OUTPUT_EN;
+	}
+}
+
+static int tas675x_rtldg_thresh_info(struct snd_kcontrol *kcontrol,
+				     struct snd_ctl_elem_info *uinfo)
+{
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+	uinfo->count = 1;
+	uinfo->value.integer.min = 0;
+	/* threshold reg ranges up to 24bit */
+	uinfo->value.integer.max = 0x00FFFFFF;
+	return 0;
+}
+
+static int tas675x_set_rtldg_thresh(struct snd_kcontrol *kcontrol,
+				    struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(comp);
+	const struct tas675x_reg_param *t =
+		(const struct tas675x_reg_param *)kcontrol->private_value;
+	u32 val = ucontrol->value.integer.value[0];
+	int ret;
+
+	ret = tas675x_dsp_mem_write(tas, t->page, t->reg, val);
+
+	/* Cache the value */
+	if (!ret) {
+		int i;
+
+		for (i = 0; i < ARRAY_SIZE(tas->dsp_params); i++) {
+			if (tas->dsp_params[i].page == t->page &&
+			    tas->dsp_params[i].reg == t->reg) {
+				tas->dsp_params[i].val = val;
+				break;
+			}
+		}
+	}
+
+	/* Return 1 to notify change, or propagate error */
+	return ret ? ret : 1;
+}
+
+static int tas675x_get_rtldg_thresh(struct snd_kcontrol *kcontrol,
+				    struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(comp);
+	const struct tas675x_reg_param *t =
+		(const struct tas675x_reg_param *)kcontrol->private_value;
+	u32 val = 0;
+	int ret;
+
+	ret = tas675x_dsp_mem_read(tas, t->page, t->reg, &val);
+	if (!ret)
+		ucontrol->value.integer.value[0] = val;
+
+	return ret;
+}
+
+static const struct tas675x_reg_param tas675x_dsp_defaults[] = {
+	[TAS675X_DSP_PARAM_ID_OL_THRESH] = {
+		TAS675X_DSP_PAGE_RTLDG, TAS675X_DSP_RTLDG_OL_THRESH_REG },
+	[TAS675X_DSP_PARAM_ID_SL_THRESH] = {
+		TAS675X_DSP_PAGE_RTLDG, TAS675X_DSP_RTLDG_SL_THRESH_REG },
+};
+
+static_assert(ARRAY_SIZE(tas675x_dsp_defaults) == TAS675X_DSP_PARAM_NUM);
+
+static int tas675x_set_dcldg_trigger(struct snd_kcontrol *kcontrol,
+				     struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(comp);
+	unsigned int state, state34;
+	int ret;
+
+	if (!ucontrol->value.integer.value[0])
+		return 0;
+
+	if (snd_soc_component_active(comp))
+		return -EBUSY;
+
+	ret = pm_runtime_resume_and_get(tas->dev);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * Abort automatic DC LDG retry loops (startup or init-after-fault)
+	 * and clear faults before manual diagnostics.
+	 */
+	regmap_update_bits(tas->regmap, TAS675X_DC_LDG_CTRL_REG,
+			   TAS675X_LDG_ABORT_BIT | TAS675X_LDG_BYPASS_BIT,
+			   TAS675X_LDG_ABORT_BIT | TAS675X_LDG_BYPASS_BIT);
+	regmap_write(tas->regmap, TAS675X_RESET_REG, TAS675X_FAULT_CLEAR);
+
+	/* Wait for LOAD_DIAG to exit */
+	ret = regmap_read_poll_timeout(tas->regmap, TAS675X_STATE_REPORT_CH1_CH2_REG,
+				       state, (state & 0x0F) != TAS675X_STATE_LOAD_DIAG &&
+					      (state >> 4) != TAS675X_STATE_LOAD_DIAG,
+				       TAS675X_POLL_INTERVAL_US,
+				       TAS675X_STATE_TRANSITION_TIMEOUT_US);
+	ret |= regmap_read_poll_timeout(tas->regmap, TAS675X_STATE_REPORT_CH3_CH4_REG,
+					state34, (state34 & 0x0F) != TAS675X_STATE_LOAD_DIAG &&
+						 (state34 >> 4) != TAS675X_STATE_LOAD_DIAG,
+					TAS675X_POLL_INTERVAL_US,
+					TAS675X_STATE_TRANSITION_TIMEOUT_US);
+	if (ret) {
+		dev_err(tas->dev,
+			"DC LDG: abort timeout (CH1/2=0x%02x [%s/%s], CH3/4=0x%02x [%s/%s])\n",
+			state, tas675x_state_name(state), tas675x_state_name(state >> 4),
+			state34, tas675x_state_name(state34), tas675x_state_name(state34 >> 4));
+		goto out_restore_ldg_ctrl;
+	}
+
+	/* Transition to HIZ state */
+	ret = tas675x_set_state_all(tas, TAS675X_STATE_HIZ_BOTH);
+	if (ret)
+		goto out_restore_ldg_ctrl;
+
+	/* Set LOAD_DIAG state for manual DC LDG */
+	ret = tas675x_set_state_all(tas, TAS675X_STATE_LOAD_DIAG_BOTH);
+	if (ret)
+		goto out_restore_ldg_ctrl;
+
+	/* Wait for device to transition to LOAD_DIAG state */
+	ret = regmap_read_poll_timeout(tas->regmap, TAS675X_STATE_REPORT_CH1_CH2_REG,
+				       state, state == TAS675X_STATE_LOAD_DIAG_BOTH,
+				       TAS675X_POLL_INTERVAL_US,
+				       TAS675X_STATE_TRANSITION_TIMEOUT_US);
+	ret |= regmap_read_poll_timeout(tas->regmap, TAS675X_STATE_REPORT_CH3_CH4_REG,
+					state34, state34 == TAS675X_STATE_LOAD_DIAG_BOTH,
+					TAS675X_POLL_INTERVAL_US,
+					TAS675X_STATE_TRANSITION_TIMEOUT_US);
+	if (ret) {
+		dev_err(tas->dev,
+			"DC LDG: LOAD_DIAG timeout (CH1/2=0x%02x [%s/%s], CH3/4=0x%02x [%s/%s])\n",
+			state, tas675x_state_name(state), tas675x_state_name(state >> 4),
+			state34, tas675x_state_name(state34), tas675x_state_name(state34 >> 4));
+		goto out_restore_hiz;
+	}
+
+	/* Clear ABORT and BYPASS bits to enable manual DC LDG */
+	ret = regmap_update_bits(tas->regmap, TAS675X_DC_LDG_CTRL_REG,
+				 TAS675X_LDG_ABORT_BIT | TAS675X_LDG_BYPASS_BIT,
+				 0);
+	if (ret)
+		goto out_restore_hiz;
+
+	dev_dbg(tas->dev, "DC LDG: Started\n");
+
+	/* Poll all channels for SLEEP state */
+	ret = regmap_read_poll_timeout(tas->regmap, TAS675X_STATE_REPORT_CH1_CH2_REG,
+				       state, state == TAS675X_STATE_SLEEP_BOTH,
+				       TAS675X_POLL_INTERVAL_US,
+				       TAS675X_DC_LDG_TIMEOUT_US);
+	ret |= regmap_read_poll_timeout(tas->regmap, TAS675X_STATE_REPORT_CH3_CH4_REG,
+				       state34, state34 == TAS675X_STATE_SLEEP_BOTH,
+				       TAS675X_POLL_INTERVAL_US,
+				       TAS675X_DC_LDG_TIMEOUT_US);
+	if (ret) {
+		dev_err(tas->dev,
+			"DC LDG: SLEEP timeout (CH1/2=0x%02x [%s/%s], CH3/4=0x%02x [%s/%s])\n",
+			state, tas675x_state_name(state), tas675x_state_name(state >> 4),
+			state34, tas675x_state_name(state34), tas675x_state_name(state34 >> 4));
+		goto out_restore_hiz;
+	}
+
+	dev_dbg(tas->dev, "DC LDG: Completed successfully (CH1/2=0x%02x, CH3/4=0x%02x)\n",
+		state, state34);
+
+out_restore_hiz:
+	tas675x_set_state_all(tas, TAS675X_STATE_HIZ_BOTH);
+
+out_restore_ldg_ctrl:
+	regmap_update_bits(tas->regmap, TAS675X_DC_LDG_CTRL_REG,
+			   TAS675X_LDG_ABORT_BIT | TAS675X_LDG_BYPASS_BIT,
+			   0);
+
+	pm_runtime_mark_last_busy(tas->dev);
+	pm_runtime_put_autosuspend(tas->dev);
+
+	return ret;
+}
+
+static int tas675x_set_acldg_trigger(struct snd_kcontrol *kcontrol,
+				     struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(comp);
+	unsigned int state, state34;
+	int ret;
+
+	if (!ucontrol->value.integer.value[0])
+		return 0;
+
+	if (snd_soc_component_active(comp))
+		return -EBUSY;
+
+	ret = pm_runtime_resume_and_get(tas->dev);
+	if (ret < 0)
+		return ret;
+
+	/* AC Load Diagnostics requires SLEEP state */
+	ret = tas675x_set_state_all(tas, TAS675X_STATE_SLEEP_BOTH);
+	if (ret) {
+		dev_err(tas->dev, "AC LDG: Failed to set SLEEP state: %d\n", ret);
+		goto out;
+	}
+
+	/* Start AC LDG on all 4 channels (0x0F) */
+	ret = regmap_write(tas->regmap, TAS675X_AC_LDG_CTRL_REG, 0x0F);
+	if (ret) {
+		dev_err(tas->dev, "AC LDG: Failed to start: %d\n", ret);
+		goto out;
+	}
+
+	dev_dbg(tas->dev, "AC LDG: Started\n");
+
+	/* Poll all channels for SLEEP state */
+	ret = regmap_read_poll_timeout(tas->regmap, TAS675X_STATE_REPORT_CH1_CH2_REG,
+				       state, (state == TAS675X_STATE_SLEEP_BOTH),
+				       TAS675X_POLL_INTERVAL_US,
+				       TAS675X_AC_LDG_TIMEOUT_US);
+	if (ret) {
+		dev_err(tas->dev,
+			"AC LDG: CH1/CH2 timeout: %d (state=0x%02x [%s/%s])\n",
+			ret, state, tas675x_state_name(state),
+			tas675x_state_name(state >> 4));
+		regmap_write(tas->regmap, TAS675X_AC_LDG_CTRL_REG, 0x00);
+		goto out;
+	}
+
+	ret = regmap_read_poll_timeout(tas->regmap, TAS675X_STATE_REPORT_CH3_CH4_REG,
+				       state34, (state34 == TAS675X_STATE_SLEEP_BOTH),
+				       TAS675X_POLL_INTERVAL_US,
+				       TAS675X_AC_LDG_TIMEOUT_US);
+	if (ret) {
+		dev_err(tas->dev,
+			"AC LDG: CH3/CH4 timeout: %d (state=0x%02x [%s/%s])\n",
+			ret, state34, tas675x_state_name(state34),
+			tas675x_state_name(state34 >> 4));
+		regmap_write(tas->regmap, TAS675X_AC_LDG_CTRL_REG, 0x00);
+		goto out;
+	}
+
+	dev_dbg(tas->dev, "AC LDG: Completed successfully (CH1/2=0x%02x, CH3/4=0x%02x)\n",
+		state, state34);
+	regmap_write(tas->regmap, TAS675X_AC_LDG_CTRL_REG, 0x00);
+
+out:
+	pm_runtime_mark_last_busy(tas->dev);
+	pm_runtime_put_autosuspend(tas->dev);
+
+	return ret;
+}
+
+static int tas675x_rtldg_impedance_info(struct snd_kcontrol *kcontrol,
+					struct snd_ctl_elem_info *uinfo)
+{
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+	uinfo->count = 1;
+	uinfo->value.integer.min = 0;
+	uinfo->value.integer.max = 0xFFFF;
+	return 0;
+}
+
+static int tas675x_get_rtldg_impedance(struct snd_kcontrol *kcontrol,
+				       struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(comp);
+	unsigned int msb_reg = (unsigned int)kcontrol->private_value;
+	u8 buf[2];
+	int ret;
+
+	ret = regmap_bulk_read(tas->regmap, msb_reg, buf, 2);
+	if (ret)
+		return ret;
+
+	ucontrol->value.integer.value[0] = (buf[0] << 8) | buf[1];
+	return 0;
+}
+
+static int tas675x_dc_resistance_info(struct snd_kcontrol *kcontrol,
+				      struct snd_ctl_elem_info *uinfo)
+{
+	/* 10-bit: 2-bit MSB + 8-bit LSB, 0.1 ohm/code, 0-102.3 ohm */
+	uinfo->type = SNDRV_CTL_ELEM_TYPE_INTEGER;
+	uinfo->count = 1;
+	uinfo->value.integer.min = 0;
+	uinfo->value.integer.max = 1023;
+	return 0;
+}
+
+static int tas675x_get_dc_resistance(struct snd_kcontrol *kcontrol,
+				     struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(comp);
+	unsigned int lsb_reg = (unsigned int)kcontrol->private_value;
+	unsigned int msb, lsb, shift;
+	int ret;
+
+	ret = regmap_read(tas->regmap, TAS675X_DC_LDG_DCR_MSB_REG, &msb);
+	if (ret)
+		return ret;
+
+	ret = regmap_read(tas->regmap, lsb_reg, &lsb);
+	if (ret)
+		return ret;
+
+	/* 2-bit MSB: CH1=[7:6], CH2=[5:4], CH3=[3:2], CH4=[1:0] */
+	shift = 6 - (lsb_reg - TAS675X_CH1_DC_LDG_DCR_LSB_REG) * 2;
+	msb = (msb >> shift) & 0x3;
+
+	ucontrol->value.integer.value[0] = (msb << 8) | lsb;
+	return 0;
+}
+
+/* Counterparts with read-only access */
+#define SOC_SINGLE_RO(xname, xreg, xshift, xmax) \
+{	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, \
+	.name = xname, \
+	.access = SNDRV_CTL_ELEM_ACCESS_READ | SNDRV_CTL_ELEM_ACCESS_VOLATILE, \
+	.info = snd_soc_info_volsw, \
+	.get = snd_soc_get_volsw, \
+	.private_value = SOC_SINGLE_VALUE(xreg, xshift, 0, xmax, 0, 0) }
+#define SOC_DC_RESIST_RO(xname, xlsb_reg) \
+{	.name = xname, \
+	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, \
+	.access = SNDRV_CTL_ELEM_ACCESS_READ | SNDRV_CTL_ELEM_ACCESS_VOLATILE, \
+	.info = tas675x_dc_resistance_info, \
+	.get = tas675x_get_dc_resistance, \
+	.private_value = (xlsb_reg) }
+#define SOC_RTLDG_IMP_RO(xname, xreg) \
+{	.name = xname, \
+	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, \
+	.access = SNDRV_CTL_ELEM_ACCESS_READ | SNDRV_CTL_ELEM_ACCESS_VOLATILE, \
+	.info = tas675x_rtldg_impedance_info, \
+	.get = tas675x_get_rtldg_impedance, \
+	.private_value = (xreg) }
+
+#define SOC_DSP_THRESH_EXT(xname, xthresh) \
+{	.name = xname, \
+	.iface = SNDRV_CTL_ELEM_IFACE_MIXER, \
+	.info = tas675x_rtldg_thresh_info, \
+	.get = tas675x_get_rtldg_thresh, \
+	.put = tas675x_set_rtldg_thresh, \
+	.private_value = (unsigned long)&(xthresh) }
+
+/*
+ * DAC digital volumes. From -103 to 0 dB in 0.5 dB steps, -103.5 dB means mute.
+ * DAC analog gain. From -15.5 to 0 dB in 0.5 dB steps, no mute.
+ */
+static const DECLARE_TLV_DB_SCALE(tas675x_dig_vol_tlv, -10350, 50, 1);
+static const DECLARE_TLV_DB_SCALE(tas675x_ana_gain_tlv, -1550, 50, 0);
+
+static const char * const tas675x_ss_texts[] = {
+	"Disabled", "Triangle", "Random", "Triangle and Random"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ss_enum, TAS675X_SS_CTRL_REG, 0, tas675x_ss_texts);
+
+static const char * const tas675x_ss_tri_range_texts[] = {
+	"6.5%", "13.5%", "5%", "10%"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ss_tri_range_enum,
+			    TAS675X_SS_RANGE_CTRL_REG, 0,
+			    tas675x_ss_tri_range_texts);
+
+static const char * const tas675x_ss_rdm_range_texts[] = {
+	"0.83%", "2.50%", "5.83%", "12.50%", "25.83%"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ss_rdm_range_enum,
+			    TAS675X_SS_RANGE_CTRL_REG, 4,
+			    tas675x_ss_rdm_range_texts);
+
+static const char * const tas675x_ss_rdm_dwell_texts[] = {
+	"1/FSS to 2/FSS", "1/FSS to 4/FSS", "1/FSS to 8/FSS", "1/FSS to 15/FSS"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ss_rdm_dwell_enum,
+			    TAS675X_SS_RANGE_CTRL_REG, 2,
+			    tas675x_ss_rdm_dwell_texts);
+
+static const char * const tas675x_oc_limit_texts[] = {
+	"Level 4", "Level 3", "Level 2", "Level 1"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_oc_limit_enum, TAS675X_CURRENT_LIMIT_CTRL_REG,
+			    0, tas675x_oc_limit_texts);
+
+static const char * const tas675x_otw_texts[] = {
+	"Disabled", ">95C", ">110C", ">125C", ">135C", ">145C", ">155C", ">165C"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ch1_otw_enum,
+			    TAS675X_OTW_CTRL_CH1_CH2_REG, 4,
+			    tas675x_otw_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ch2_otw_enum,
+			    TAS675X_OTW_CTRL_CH1_CH2_REG, 0,
+			    tas675x_otw_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ch3_otw_enum,
+			    TAS675X_OTW_CTRL_CH3_CH4_REG, 4,
+			    tas675x_otw_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ch4_otw_enum,
+			    TAS675X_OTW_CTRL_CH3_CH4_REG, 0,
+			    tas675x_otw_texts);
+
+static const char * const tas675x_dc_ldg_sl_texts[] = {
+	"0.5 Ohm", "1 Ohm", "1.5 Ohm", "2 Ohm", "2.5 Ohm",
+	"3 Ohm", "3.5 Ohm", "4 Ohm", "4.5 Ohm", "5 Ohm"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ch1_dc_ldg_sl_enum,
+			    TAS675X_DC_LDG_SL_CH1_CH2_CTRL_REG, 4,
+			    tas675x_dc_ldg_sl_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ch2_dc_ldg_sl_enum,
+			    TAS675X_DC_LDG_SL_CH1_CH2_CTRL_REG, 0,
+			    tas675x_dc_ldg_sl_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ch3_dc_ldg_sl_enum,
+			    TAS675X_DC_LDG_SL_CH3_CH4_CTRL_REG, 4,
+			    tas675x_dc_ldg_sl_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ch4_dc_ldg_sl_enum,
+			    TAS675X_DC_LDG_SL_CH3_CH4_CTRL_REG, 0,
+			    tas675x_dc_ldg_sl_texts);
+
+static const char * const tas675x_dc_slol_ramp_texts[] = {
+	"15 ms", "30 ms", "10 ms", "20 ms"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_dc_slol_ramp_enum,
+			    TAS675X_DC_LDG_TIME_CTRL_REG, 6,
+			    tas675x_dc_slol_ramp_texts);
+
+static const char * const tas675x_dc_slol_settling_texts[] = {
+	"10 ms", "5 ms", "20 ms", "15 ms"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_dc_slol_settling_enum,
+			    TAS675X_DC_LDG_TIME_CTRL_REG, 4,
+			    tas675x_dc_slol_settling_texts);
+
+static const char * const tas675x_dc_s2pg_ramp_texts[] = {
+	"5 ms", "2.5 ms", "10 ms", "15 ms"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_dc_s2pg_ramp_enum,
+			    TAS675X_DC_LDG_TIME_CTRL_REG, 2,
+			    tas675x_dc_s2pg_ramp_texts);
+
+static const char * const tas675x_dc_s2pg_settling_texts[] = {
+	"10 ms", "5 ms", "20 ms", "30 ms"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_dc_s2pg_settling_enum,
+			    TAS675X_DC_LDG_TIME_CTRL_REG, 0,
+			    tas675x_dc_s2pg_settling_texts);
+
+static const char * const tas675x_dsp_mode_texts[] = {
+	"Normal", "LLP", "FFLP"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_dsp_mode_enum,
+			    TAS675X_LL_EN_REG, 0,
+			    tas675x_dsp_mode_texts);
+
+static const char * const tas675x_ana_ramp_texts[] = {
+	"15us", "60us", "200us", "400us"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ana_ramp_enum,
+			    TAS675X_ANALOG_GAIN_RAMP_CTRL_REG, 2,
+			    tas675x_ana_ramp_texts);
+
+static const char * const tas675x_ramp_rate_texts[] = {
+	"4 FS", "16 FS", "32 FS", "Instant"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ramp_down_rate_enum,
+			    TAS675X_DIG_VOL_RAMP_CTRL_REG, 6,
+			    tas675x_ramp_rate_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ramp_up_rate_enum,
+			    TAS675X_DIG_VOL_RAMP_CTRL_REG, 2,
+			    tas675x_ramp_rate_texts);
+
+static const char * const tas675x_ramp_step_texts[] = {
+	"4dB", "2dB", "1dB", "0.5dB"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ramp_down_step_enum,
+			    TAS675X_DIG_VOL_RAMP_CTRL_REG, 4,
+			    tas675x_ramp_step_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ramp_up_step_enum,
+			    TAS675X_DIG_VOL_RAMP_CTRL_REG, 0,
+			    tas675x_ramp_step_texts);
+
+static const char * const tas675x_vol_combine_ch12_texts[] = {
+	"Independent", "CH2 follows CH1", "CH1 follows CH2"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_vol_combine_ch12_enum,
+			    TAS675X_DIG_VOL_COMBINE_CTRL_REG, 0,
+			    tas675x_vol_combine_ch12_texts);
+
+static const char * const tas675x_vol_combine_ch34_texts[] = {
+	"Independent", "CH4 follows CH3", "CH3 follows CH4"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_vol_combine_ch34_enum,
+			    TAS675X_DIG_VOL_COMBINE_CTRL_REG, 2,
+			    tas675x_vol_combine_ch34_texts);
+
+static const char * const tas675x_auto_mute_time_texts[] = {
+	"11.5ms", "53ms", "106.5ms", "266.5ms",
+	"535ms", "1065ms", "2665ms", "5330ms"
+};
+
+static SOC_ENUM_SINGLE_DECL(tas675x_ch1_mute_time_enum,
+			    TAS675X_AUTO_MUTE_TIMING_CH1_CH2_REG, 4,
+			    tas675x_auto_mute_time_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ch2_mute_time_enum,
+			    TAS675X_AUTO_MUTE_TIMING_CH1_CH2_REG, 0,
+			    tas675x_auto_mute_time_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ch3_mute_time_enum,
+			    TAS675X_AUTO_MUTE_TIMING_CH3_CH4_REG, 4,
+			    tas675x_auto_mute_time_texts);
+static SOC_ENUM_SINGLE_DECL(tas675x_ch4_mute_time_enum,
+			    TAS675X_AUTO_MUTE_TIMING_CH3_CH4_REG, 0,
+			    tas675x_auto_mute_time_texts);
+
+/*
+ * ALSA Mixer Controls
+ *
+ * For detailed documentation of each control see:
+ * Documentation/sound/codecs/tas675x.rst
+ */
+static const struct snd_kcontrol_new tas675x_snd_controls[] = {
+	/* Volume & Gain Control */
+	SOC_DOUBLE_R_TLV("Analog Playback Volume", TAS675X_ANALOG_GAIN_CH1_CH2_REG,
+			 TAS675X_ANALOG_GAIN_CH3_CH4_REG, 1, 0x1F, 1, tas675x_ana_gain_tlv),
+	SOC_ENUM("Analog Gain Ramp Step", tas675x_ana_ramp_enum),
+	SOC_SINGLE_RANGE_TLV("CH1 Digital Playback Volume",
+			     TAS675X_DIG_VOL_CH1_REG, 0, 0x30, 0xFF, 1,
+			     tas675x_dig_vol_tlv),
+	SOC_SINGLE_RANGE_TLV("CH2 Digital Playback Volume",
+			     TAS675X_DIG_VOL_CH2_REG, 0, 0x30, 0xFF, 1,
+			     tas675x_dig_vol_tlv),
+	SOC_SINGLE_RANGE_TLV("CH3 Digital Playback Volume",
+			     TAS675X_DIG_VOL_CH3_REG, 0, 0x30, 0xFF, 1,
+			     tas675x_dig_vol_tlv),
+	SOC_SINGLE_RANGE_TLV("CH4 Digital Playback Volume",
+			     TAS675X_DIG_VOL_CH4_REG, 0, 0x30, 0xFF, 1,
+			     tas675x_dig_vol_tlv),
+	SOC_ENUM("Volume Ramp Down Rate", tas675x_ramp_down_rate_enum),
+	SOC_ENUM("Volume Ramp Down Step", tas675x_ramp_down_step_enum),
+	SOC_ENUM("Volume Ramp Up Rate", tas675x_ramp_up_rate_enum),
+	SOC_ENUM("Volume Ramp Up Step", tas675x_ramp_up_step_enum),
+	SOC_ENUM("CH1/2 Volume Combine", tas675x_vol_combine_ch12_enum),
+	SOC_ENUM("CH3/4 Volume Combine", tas675x_vol_combine_ch34_enum),
+
+	/* Auto Mute & Silence Detection */
+	SOC_SINGLE("CH1 Auto Mute Switch", TAS675X_AUTO_MUTE_EN_REG, 0, 1, 0),
+	SOC_SINGLE("CH2 Auto Mute Switch", TAS675X_AUTO_MUTE_EN_REG, 1, 1, 0),
+	SOC_SINGLE("CH3 Auto Mute Switch", TAS675X_AUTO_MUTE_EN_REG, 2, 1, 0),
+	SOC_SINGLE("CH4 Auto Mute Switch", TAS675X_AUTO_MUTE_EN_REG, 3, 1, 0),
+	SOC_SINGLE("Auto Mute Combine Switch", TAS675X_AUTO_MUTE_EN_REG, 4, 1, 0),
+	SOC_ENUM("CH1 Auto Mute Time", tas675x_ch1_mute_time_enum),
+	SOC_ENUM("CH2 Auto Mute Time", tas675x_ch2_mute_time_enum),
+	SOC_ENUM("CH3 Auto Mute Time", tas675x_ch3_mute_time_enum),
+	SOC_ENUM("CH4 Auto Mute Time", tas675x_ch4_mute_time_enum),
+
+	/* Clock & EMI Management */
+	SOC_ENUM("Spread Spectrum Mode", tas675x_ss_enum),
+	SOC_ENUM("SS Triangle Range", tas675x_ss_tri_range_enum),
+	SOC_ENUM("SS Random Range", tas675x_ss_rdm_range_enum),
+	SOC_ENUM("SS Random Dwell Range", tas675x_ss_rdm_dwell_enum),
+	SOC_SINGLE("SS Triangle Dwell Min", TAS675X_SS_DWELL_CTRL_REG, 4, 15, 0),
+	SOC_SINGLE("SS Triangle Dwell Max", TAS675X_SS_DWELL_CTRL_REG, 0, 15, 0),
+
+	/* Hardware Protection */
+	SOC_SINGLE("OTSD Auto Recovery Switch", TAS675X_OTSD_RECOVERY_EN_REG, 1, 1, 0),
+	SOC_ENUM("Overcurrent Limit Level", tas675x_oc_limit_enum),
+	SOC_ENUM("CH1 OTW Threshold", tas675x_ch1_otw_enum),
+	SOC_ENUM("CH2 OTW Threshold", tas675x_ch2_otw_enum),
+	SOC_ENUM("CH3 OTW Threshold", tas675x_ch3_otw_enum),
+	SOC_ENUM("CH4 OTW Threshold", tas675x_ch4_otw_enum),
+
+	/* DSP Signal Path & Mode */
+	SOC_ENUM("DSP Signal Path Mode", tas675x_dsp_mode_enum),
+
+	/* DC Load Diagnostics */
+	{
+		.iface = SNDRV_CTL_ELEM_IFACE_MIXER,
+		.name = "DC LDG Trigger",
+		.access = SNDRV_CTL_ELEM_ACCESS_WRITE,
+		.info = snd_ctl_boolean_mono_info,
+		.put = tas675x_set_dcldg_trigger,
+	},
+	SOC_SINGLE("DC LDG Auto Diagnostics Switch", TAS675X_DC_LDG_CTRL_REG, 0, 1, 1),
+	SOC_SINGLE("CH1 LO LDG Switch", TAS675X_DC_LDG_LO_CTRL_REG, 3, 1, 0),
+	SOC_SINGLE("CH2 LO LDG Switch", TAS675X_DC_LDG_LO_CTRL_REG, 2, 1, 0),
+	SOC_SINGLE("CH3 LO LDG Switch", TAS675X_DC_LDG_LO_CTRL_REG, 1, 1, 0),
+	SOC_SINGLE("CH4 LO LDG Switch", TAS675X_DC_LDG_LO_CTRL_REG, 0, 1, 0),
+	SOC_ENUM("DC LDG SLOL Ramp Time", tas675x_dc_slol_ramp_enum),
+	SOC_ENUM("DC LDG SLOL Settling Time", tas675x_dc_slol_settling_enum),
+	SOC_ENUM("DC LDG S2PG Ramp Time", tas675x_dc_s2pg_ramp_enum),
+	SOC_ENUM("DC LDG S2PG Settling Time", tas675x_dc_s2pg_settling_enum),
+	SOC_ENUM("CH1 DC LDG SL Threshold", tas675x_ch1_dc_ldg_sl_enum),
+	SOC_ENUM("CH2 DC LDG SL Threshold", tas675x_ch2_dc_ldg_sl_enum),
+	SOC_ENUM("CH3 DC LDG SL Threshold", tas675x_ch3_dc_ldg_sl_enum),
+	SOC_ENUM("CH4 DC LDG SL Threshold", tas675x_ch4_dc_ldg_sl_enum),
+	SOC_SINGLE_RO("DC LDG Result", TAS675X_DC_LDG_RESULT_REG, 0, 0xFF),
+	SOC_SINGLE_RO("CH1 DC LDG Report", TAS675X_DC_LDG_REPORT_CH1_CH2_REG, 4, 0x0F),
+	SOC_SINGLE_RO("CH2 DC LDG Report", TAS675X_DC_LDG_REPORT_CH1_CH2_REG, 0, 0x0F),
+	SOC_SINGLE_RO("CH3 DC LDG Report", TAS675X_DC_LDG_REPORT_CH3_CH4_REG, 4, 0x0F),
+	SOC_SINGLE_RO("CH4 DC LDG Report", TAS675X_DC_LDG_REPORT_CH3_CH4_REG, 0, 0x0F),
+	SOC_SINGLE_RO("CH1 LO LDG Report", TAS675X_DC_LDG_RESULT_REG, 7, 1),
+	SOC_SINGLE_RO("CH2 LO LDG Report", TAS675X_DC_LDG_RESULT_REG, 6, 1),
+	SOC_SINGLE_RO("CH3 LO LDG Report", TAS675X_DC_LDG_RESULT_REG, 5, 1),
+	SOC_SINGLE_RO("CH4 LO LDG Report", TAS675X_DC_LDG_RESULT_REG, 4, 1),
+	SOC_DC_RESIST_RO("CH1 DC Resistance", TAS675X_CH1_DC_LDG_DCR_LSB_REG),
+	SOC_DC_RESIST_RO("CH2 DC Resistance", TAS675X_CH2_DC_LDG_DCR_LSB_REG),
+	SOC_DC_RESIST_RO("CH3 DC Resistance", TAS675X_CH3_DC_LDG_DCR_LSB_REG),
+	SOC_DC_RESIST_RO("CH4 DC Resistance", TAS675X_CH4_DC_LDG_DCR_LSB_REG),
+
+	/* AC Load Diagnostics */
+	{
+		.iface = SNDRV_CTL_ELEM_IFACE_MIXER,
+		.name = "AC LDG Trigger",
+		.access = SNDRV_CTL_ELEM_ACCESS_WRITE,
+		.info = snd_ctl_boolean_mono_info,
+		.put = tas675x_set_acldg_trigger,
+	},
+	SOC_SINGLE("AC LDG Gain", TAS675X_AC_LDG_CTRL_REG, 4, 1, 0),
+	SOC_SINGLE("AC LDG Test Frequency", TAS675X_AC_LDG_FREQ_CTRL_REG, 0, 0xFF, 0),
+	SOC_SINGLE_RO("CH1 AC LDG Real", TAS675X_AC_LDG_REPORT_CH1_R_REG, 0, 0xFF),
+	SOC_SINGLE_RO("CH1 AC LDG Imag", TAS675X_AC_LDG_REPORT_CH1_I_REG, 0, 0xFF),
+	SOC_SINGLE_RO("CH2 AC LDG Real", TAS675X_AC_LDG_REPORT_CH2_R_REG, 0, 0xFF),
+	SOC_SINGLE_RO("CH2 AC LDG Imag", TAS675X_AC_LDG_REPORT_CH2_I_REG, 0, 0xFF),
+	SOC_SINGLE_RO("CH3 AC LDG Real", TAS675X_AC_LDG_REPORT_CH3_R_REG, 0, 0xFF),
+	SOC_SINGLE_RO("CH3 AC LDG Imag", TAS675X_AC_LDG_REPORT_CH3_I_REG, 0, 0xFF),
+	SOC_SINGLE_RO("CH4 AC LDG Real", TAS675X_AC_LDG_REPORT_CH4_R_REG, 0, 0xFF),
+	SOC_SINGLE_RO("CH4 AC LDG Imag", TAS675X_AC_LDG_REPORT_CH4_I_REG, 0, 0xFF),
+
+	/* Temperature and Voltage Monitoring */
+	SOC_SINGLE_RO("PVDD Sense", TAS675X_PVDD_SENSE_REG, 0, 0xFF),
+	SOC_SINGLE_RO("Global Temperature", TAS675X_TEMP_GLOBAL_REG, 0, 0xFF),
+	SOC_SINGLE_RO("CH1 Temperature Range", TAS675X_TEMP_CH1_CH2_REG, 6, 3),
+	SOC_SINGLE_RO("CH2 Temperature Range", TAS675X_TEMP_CH1_CH2_REG, 4, 3),
+	SOC_SINGLE_RO("CH3 Temperature Range", TAS675X_TEMP_CH3_CH4_REG, 2, 3),
+	SOC_SINGLE_RO("CH4 Temperature Range", TAS675X_TEMP_CH3_CH4_REG, 0, 3),
+
+	/* Speaker Protection & Detection */
+	SOC_SINGLE("Tweeter Detection Switch", TAS675X_TWEETER_DETECT_CTRL_REG, 0, 1, 1),
+	SOC_SINGLE("Tweeter Detect Threshold", TAS675X_TWEETER_DETECT_THRESH_REG, 0, 0xFF, 0),
+	SOC_SINGLE_RO("CH1 Tweeter Detect Report", TAS675X_TWEETER_REPORT_REG, 3, 1),
+	SOC_SINGLE_RO("CH2 Tweeter Detect Report", TAS675X_TWEETER_REPORT_REG, 2, 1),
+	SOC_SINGLE_RO("CH3 Tweeter Detect Report", TAS675X_TWEETER_REPORT_REG, 1, 1),
+	SOC_SINGLE_RO("CH4 Tweeter Detect Report", TAS675X_TWEETER_REPORT_REG, 0, 1),
+
+	/*
+	 * Unavailable in LLP, available in Normal & FFLP
+	 */
+	SOC_SINGLE("Thermal Foldback Switch", TAS675X_DSP_CTRL_REG, 0, 1, 0),
+	SOC_SINGLE("PVDD Foldback Switch", TAS675X_DSP_CTRL_REG, 4, 1, 0),
+	SOC_SINGLE("DC Blocker Bypass Switch", TAS675X_DC_BLOCK_BYP_REG, 0, 1, 0),
+	SOC_SINGLE("Clip Detect Switch", TAS675X_CLIP_DETECT_CTRL_REG, 6, 1, 0),
+	SOC_SINGLE("Audio SDOUT Switch", TAS675X_DSP_CTRL_REG, 5, 1, 0),
+
+	/*
+	 * Unavailable in both FFLP and LLP, Normal mode only
+	 */
+	/* Real-Time Load Diagnostics */
+	SOC_SINGLE("CH1 RTLDG Switch", TAS675X_RTLDG_EN_REG, 3, 1, 0),
+	SOC_SINGLE("CH2 RTLDG Switch", TAS675X_RTLDG_EN_REG, 2, 1, 0),
+	SOC_SINGLE("CH3 RTLDG Switch", TAS675X_RTLDG_EN_REG, 1, 1, 0),
+	SOC_SINGLE("CH4 RTLDG Switch", TAS675X_RTLDG_EN_REG, 0, 1, 0),
+	SOC_SINGLE("RTLDG Clip Mask Switch", TAS675X_RTLDG_EN_REG, 4, 1, 0),
+	SOC_SINGLE("ISENSE Calibration Switch", TAS675X_ISENSE_CAL_REG, 3, 1, 0),
+	SOC_DSP_THRESH_EXT("RTLDG Open Load Threshold",
+			   tas675x_dsp_defaults[TAS675X_DSP_PARAM_ID_OL_THRESH]),
+	SOC_DSP_THRESH_EXT("RTLDG Short Load Threshold",
+			   tas675x_dsp_defaults[TAS675X_DSP_PARAM_ID_SL_THRESH]),
+	SOC_RTLDG_IMP_RO("CH1 RTLDG Impedance", TAS675X_CH1_RTLDG_IMP_MSB_REG),
+	SOC_RTLDG_IMP_RO("CH2 RTLDG Impedance", TAS675X_CH2_RTLDG_IMP_MSB_REG),
+	SOC_RTLDG_IMP_RO("CH3 RTLDG Impedance", TAS675X_CH3_RTLDG_IMP_MSB_REG),
+	SOC_RTLDG_IMP_RO("CH4 RTLDG Impedance", TAS675X_CH4_RTLDG_IMP_MSB_REG),
+};
+
+static const struct snd_kcontrol_new tas675x_audio_path_switch =
+	SOC_DAPM_SINGLE("Switch", SND_SOC_NOPM, 0, 1, 1);
+
+static const struct snd_kcontrol_new tas675x_anc_path_switch =
+	SOC_DAPM_SINGLE("Switch", SND_SOC_NOPM, 0, 1, 1);
+
+static const struct snd_soc_dapm_widget tas675x_dapm_widgets[] = {
+	SND_SOC_DAPM_SUPPLY("Analog Core", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("SDOUT Vpredict", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_SUPPLY("SDOUT Isense", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	SND_SOC_DAPM_DAC("Audio DAC", "Playback", SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_DAC("ANC DAC", "ANC Playback", SND_SOC_NOPM, 0, 0),
+	SND_SOC_DAPM_ADC("Feedback ADC", "Feedback Capture", SND_SOC_NOPM, 0, 0),
+
+	SND_SOC_DAPM_SWITCH("Audio Path", SND_SOC_NOPM, 0, 0,
+			    &tas675x_audio_path_switch),
+	SND_SOC_DAPM_SWITCH("ANC Path", SND_SOC_NOPM, 0, 0,
+			    &tas675x_anc_path_switch),
+
+	/*
+	 * Even though all channels are coupled in terms of power control,
+	 * use logical outputs for each channel to allow independent routing
+	 * and DAPM controls if needed.
+	 */
+	SND_SOC_DAPM_OUTPUT("OUT_CH1"),
+	SND_SOC_DAPM_OUTPUT("OUT_CH2"),
+	SND_SOC_DAPM_OUTPUT("OUT_CH3"),
+	SND_SOC_DAPM_OUTPUT("OUT_CH4"),
+	SND_SOC_DAPM_INPUT("SPEAKER_LOAD"),
+};
+
+static const struct snd_soc_dapm_route tas675x_dapm_routes[] = {
+	{ "Audio DAC", NULL, "Analog Core" },
+	{ "Audio Path", "Switch", "Audio DAC" },
+	{ "OUT_CH1", NULL, "Audio Path" },
+	{ "OUT_CH2", NULL, "Audio Path" },
+	{ "OUT_CH3", NULL, "Audio Path" },
+	{ "OUT_CH4", NULL, "Audio Path" },
+
+	{ "ANC DAC", NULL, "Analog Core" },
+	{ "ANC Path", "Switch", "ANC DAC" },
+	{ "OUT_CH1", NULL, "ANC Path" },
+	{ "OUT_CH2", NULL, "ANC Path" },
+	{ "OUT_CH3", NULL, "ANC Path" },
+	{ "OUT_CH4", NULL, "ANC Path" },
+
+	{ "Feedback ADC", NULL, "Analog Core" },
+	{ "Feedback ADC", NULL, "SDOUT Vpredict" },
+	{ "Feedback ADC", NULL, "SDOUT Isense" },
+	{ "Feedback ADC", NULL, "SPEAKER_LOAD" },
+};
+
+static void tas675x_program_slot_offsets(struct tas675x_priv *tas,
+					 int dai_id, int slot_width)
+{
+	int offset = 0;
+
+	switch (dai_id) {
+	case 0:
+	/* Standard Audio on SDIN */
+		if (tas->audio_slot >= 0)
+			offset = tas->audio_slot * slot_width;
+		else if (tas->tx_mask)
+			offset = __ffs(tas->tx_mask) * slot_width;
+		else
+			return;
+		offset += tas->bclk_offset;
+		regmap_update_bits(tas->regmap, TAS675X_SDIN_OFFSET_MSB_REG,
+				   TAS675X_SDIN_AUDIO_OFF_MSB_MASK,
+				   FIELD_PREP(TAS675X_SDIN_AUDIO_OFF_MSB_MASK, offset >> 8));
+		regmap_write(tas->regmap, TAS675X_SDIN_AUDIO_OFFSET_REG,
+			     offset & 0xFF);
+		break;
+	case 1:
+	/*
+	 * Low-Latency Playback on SDIN, **only** enabled in LLP mode
+	 * and to be mixed with main audio before output amplification
+	 * to achieve ANC/RNC.
+	 */
+		if (tas->llp_slot >= 0)
+			offset = tas->llp_slot * slot_width;
+		else if (tas->tx_mask)
+			offset = __ffs(tas->tx_mask) * slot_width;
+		else
+			return;
+		offset += tas->bclk_offset;
+		regmap_update_bits(tas->regmap, TAS675X_SDIN_OFFSET_MSB_REG,
+				   TAS675X_SDIN_LL_OFF_MSB_MASK,
+				   FIELD_PREP(TAS675X_SDIN_LL_OFF_MSB_MASK, offset >> 8));
+		regmap_write(tas->regmap, TAS675X_SDIN_LL_OFFSET_REG,
+			     offset & 0xFF);
+		break;
+	case 2:
+	/* SDOUT Data Output (Vpredict + Isense feedback) */
+		if (!tas->slot_width)
+			break;
+		if (tas->vpredict_slot >= 0) {
+			offset = tas->vpredict_slot * slot_width;
+			offset += tas->bclk_offset;
+			regmap_update_bits(tas->regmap, TAS675X_SDOUT_OFFSET_MSB_REG,
+					   TAS675X_SDOUT_VP_OFF_MSB_MASK,
+					   FIELD_PREP(TAS675X_SDOUT_VP_OFF_MSB_MASK, offset >> 8));
+			regmap_write(tas->regmap, TAS675X_VPREDICT_OFFSET_REG,
+				     offset & 0xFF);
+		}
+		if (tas->isense_slot >= 0) {
+			offset = tas->isense_slot * slot_width;
+			offset += tas->bclk_offset;
+			regmap_update_bits(tas->regmap, TAS675X_SDOUT_OFFSET_MSB_REG,
+					   TAS675X_SDOUT_IS_OFF_MSB_MASK,
+					   FIELD_PREP(TAS675X_SDOUT_IS_OFF_MSB_MASK, offset >> 8));
+			regmap_write(tas->regmap, TAS675X_ISENSE_OFFSET_REG,
+				     offset & 0xFF);
+		}
+		break;
+	}
+
+	if (offset > 511)
+		dev_warn(tas->dev,
+			 "DAI %d slot offset %d exceeds 511 SCLK limit\n",
+			 dai_id, offset);
+}
+
+static int tas675x_hw_params(struct snd_pcm_substream *substream,
+			     struct snd_pcm_hw_params *params,
+			     struct snd_soc_dai *dai)
+{
+	struct snd_soc_component *component = dai->component;
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(component);
+	unsigned int rate = params_rate(params);
+	u8 word_length;
+
+	/*
+	 * Single clock domain: SDIN and SDOUT share one SCLK/FSYNC pair,
+	 * so all active DAIs must use the same sample rate.
+	 */
+	if ((tas->active_playback_dais || tas->active_capture_dais) &&
+	    tas->rate && tas->rate != rate) {
+		dev_err(component->dev,
+			"Rate %u conflicts with active rate %u\n",
+			rate, tas->rate);
+		return -EINVAL;
+	}
+
+	switch (params_width(params)) {
+	case 16:
+		word_length = TAS675X_WL_16BIT;
+		break;
+	case 20:
+		word_length = TAS675X_WL_20BIT;
+		break;
+	case 24:
+		word_length = TAS675X_WL_24BIT;
+		break;
+	case 32:
+		word_length = TAS675X_WL_32BIT;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) {
+		/*
+		 * RTLDG is not supported above 96kHz. Auto-disable to
+		 * prevent DSP overload and restore when rate drops back.
+		 */
+		if (rate > 96000) {
+			unsigned int val;
+
+			regmap_read(component->regmap, TAS675X_RTLDG_EN_REG,
+				    &val);
+			if (val & TAS675X_RTLDG_CH_EN_MASK) {
+				tas->saved_rtldg_en = val;
+				dev_dbg(component->dev,
+					"Sample rate %dHz > 96kHz: Auto-disabling RTLDG\n",
+					rate);
+				regmap_update_bits(component->regmap,
+						   TAS675X_RTLDG_EN_REG,
+						   TAS675X_RTLDG_CH_EN_MASK,
+						   0x00);
+			}
+		} else if (tas->saved_rtldg_en) {
+			unsigned int cur;
+
+			/*
+			 * Respect overrides and only restore if RTLDG is still auto-disabled
+			 */
+			regmap_read(component->regmap, TAS675X_RTLDG_EN_REG,
+				    &cur);
+			if (!(cur & TAS675X_RTLDG_CH_EN_MASK)) {
+				dev_dbg(component->dev,
+					"Restoring RTLDG config after high-rate stream\n");
+				regmap_update_bits(component->regmap,
+						   TAS675X_RTLDG_EN_REG,
+						   TAS675X_RTLDG_CH_EN_MASK,
+						   TAS675X_RTLDG_CH_EN_MASK &
+							tas->saved_rtldg_en);
+			}
+			tas->saved_rtldg_en = 0;
+		}
+
+		/* Set SDIN word length (audio path + low-latency path) */
+		regmap_update_bits(component->regmap, TAS675X_SDIN_CTRL_REG,
+				   TAS675X_SDIN_WL_MASK,
+				   FIELD_PREP(TAS675X_SDIN_AUDIO_WL_MASK, word_length) |
+				   FIELD_PREP(TAS675X_SDIN_LL_WL_MASK, word_length));
+	} else {
+		/* Set SDOUT word length (VPREDICT + ISENSE) for capture */
+		regmap_update_bits(component->regmap, TAS675X_SDOUT_CTRL_REG,
+				   TAS675X_SDOUT_WL_MASK,
+				   FIELD_PREP(TAS675X_SDOUT_VP_WL_MASK, word_length) |
+				   FIELD_PREP(TAS675X_SDOUT_IS_WL_MASK, word_length));
+	}
+
+	tas675x_program_slot_offsets(tas, dai->id,
+				     tas->slot_width ?: params_width(params));
+
+	tas->rate = rate;
+
+	return 0;
+}
+
+static int tas675x_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	struct snd_soc_component *component = dai->component;
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(component);
+	bool tdm_mode = false, i2s_mode = false;
+
+	/* Enforce Clocking Direction (Codec is strictly a consumer) */
+	switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) {
+	case SND_SOC_DAIFMT_BC_FC:
+		break;
+	default:
+		dev_err(component->dev, "Unsupported clock provider format\n");
+		return -EINVAL;
+	}
+
+	/* SCLK polarity: NB_NF or IB_NF only (no FSYNC inversion support) */
+	switch (fmt & SND_SOC_DAIFMT_INV_MASK) {
+	case SND_SOC_DAIFMT_NB_NF:
+		regmap_update_bits(component->regmap, TAS675X_SCLK_INV_CTRL_REG,
+				   TAS675X_SCLK_INV_MASK, 0x00);
+		break;
+	case SND_SOC_DAIFMT_IB_NF:
+		regmap_update_bits(component->regmap, TAS675X_SCLK_INV_CTRL_REG,
+				   TAS675X_SCLK_INV_MASK, TAS675X_SCLK_INV_MASK);
+		break;
+	default:
+		dev_err(component->dev, "Unsupported clock inversion\n");
+		return -EINVAL;
+	}
+
+	/* Configure Audio Format and TDM Enable */
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		i2s_mode = true;
+		tas->bclk_offset = 0;
+		regmap_update_bits(component->regmap, TAS675X_AUDIO_IF_CTRL_REG,
+				   TAS675X_TDM_EN_BIT | TAS675X_SAP_FMT_MASK |
+				   TAS675X_FS_PULSE_MASK,
+				   TAS675X_SAP_FMT_I2S);
+		regmap_update_bits(component->regmap, TAS675X_SDOUT_CTRL_REG,
+				   TAS675X_SDOUT_SELECT_MASK,
+				   TAS675X_SDOUT_SELECT_NON_TDM);
+		break;
+	case SND_SOC_DAIFMT_RIGHT_J:
+		tas->bclk_offset = 0;
+		regmap_update_bits(component->regmap, TAS675X_AUDIO_IF_CTRL_REG,
+				   TAS675X_TDM_EN_BIT | TAS675X_SAP_FMT_MASK |
+				   TAS675X_FS_PULSE_MASK,
+				   TAS675X_SAP_FMT_RIGHT_J);
+		regmap_update_bits(component->regmap, TAS675X_SDOUT_CTRL_REG,
+				   TAS675X_SDOUT_SELECT_MASK,
+				   TAS675X_SDOUT_SELECT_NON_TDM);
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+		tas->bclk_offset = 0;
+		regmap_update_bits(component->regmap, TAS675X_AUDIO_IF_CTRL_REG,
+				   TAS675X_TDM_EN_BIT | TAS675X_SAP_FMT_MASK |
+				   TAS675X_FS_PULSE_MASK,
+				   TAS675X_SAP_FMT_LEFT_J);
+		regmap_update_bits(component->regmap, TAS675X_SDOUT_CTRL_REG,
+				   TAS675X_SDOUT_SELECT_MASK,
+				   TAS675X_SDOUT_SELECT_NON_TDM);
+		break;
+	case SND_SOC_DAIFMT_DSP_A:
+		tdm_mode = true;
+		tas->bclk_offset = 1;
+		regmap_update_bits(component->regmap, TAS675X_AUDIO_IF_CTRL_REG,
+				   TAS675X_TDM_EN_BIT | TAS675X_SAP_FMT_MASK |
+				   TAS675X_FS_PULSE_MASK,
+				   TAS675X_TDM_EN_BIT | TAS675X_SAP_FMT_TDM |
+				   TAS675X_FS_PULSE_SHORT);
+		regmap_update_bits(component->regmap, TAS675X_SDOUT_CTRL_REG,
+				   TAS675X_SDOUT_SELECT_MASK,
+				   TAS675X_SDOUT_SELECT_TDM_SDOUT1);
+		break;
+	case SND_SOC_DAIFMT_DSP_B:
+		tdm_mode = true;
+		tas->bclk_offset = 0;
+		regmap_update_bits(component->regmap, TAS675X_AUDIO_IF_CTRL_REG,
+				   TAS675X_TDM_EN_BIT | TAS675X_SAP_FMT_MASK |
+				   TAS675X_FS_PULSE_MASK,
+				   TAS675X_TDM_EN_BIT | TAS675X_SAP_FMT_TDM |
+				   TAS675X_FS_PULSE_SHORT);
+		regmap_update_bits(component->regmap, TAS675X_SDOUT_CTRL_REG,
+				   TAS675X_SDOUT_SELECT_MASK,
+				   TAS675X_SDOUT_SELECT_TDM_SDOUT1);
+		break;
+	default:
+		dev_err(component->dev, "Unsupported DAI format\n");
+		return -EINVAL;
+	}
+
+	/* Setup Vpredict and Isense outputs */
+	if (dai->id == 2) {
+		unsigned int sdout_en = 0;
+
+		if (tdm_mode) {
+			/* TDM: Vpredict and Isense may coexist on separate slots */
+			if (tas->vpredict_slot >= 0)
+				sdout_en |= TAS675X_SDOUT_EN_VPREDICT;
+			if (tas->isense_slot >= 0)
+				sdout_en |= TAS675X_SDOUT_EN_ISENSE;
+			regmap_update_bits(component->regmap,
+					   TAS675X_SDOUT_EN_REG,
+					   TAS675X_SDOUT_EN_VPREDICT |
+					   TAS675X_SDOUT_EN_ISENSE,
+					   sdout_en);
+			if (tas->vpredict_slot >= 0 && tas->isense_slot >= 0 &&
+			    abs(tas->vpredict_slot - tas->isense_slot) < 4)
+				dev_warn(component->dev,
+					 "ti,vpredict-slot-no and ti,isense-slot-no overlaps (each occupies 4 consecutive slots)\n");
+		} else if (i2s_mode) {
+			/* I2S: only one source at a time; Vpredict takes priority */
+			if (tas->vpredict_slot >= 0)
+				sdout_en = TAS675X_SDOUT_NON_TDM_SEL_VPREDICT |
+					   TAS675X_SDOUT_EN_NON_TDM_ALL;
+			else if (tas->isense_slot >= 0)
+				sdout_en = TAS675X_SDOUT_NON_TDM_SEL_ISENSE |
+					   TAS675X_SDOUT_EN_NON_TDM_ALL;
+			regmap_update_bits(component->regmap,
+					   TAS675X_SDOUT_EN_REG,
+					   TAS675X_SDOUT_NON_TDM_SEL_MASK |
+					   TAS675X_SDOUT_EN_NON_TDM_ALL,
+					   sdout_en);
+			if (sdout_en &&
+			    tas->gpio1_func != TAS675X_GPIO_SEL_SDOUT2 &&
+			    tas->gpio2_func != TAS675X_GPIO_SEL_SDOUT2)
+				dev_warn(component->dev,
+					 "sdout enabled in I2S mode but no GPIO configured as SDOUT2; Ch3/Ch4 will be absent\n");
+		}
+	}
+
+	return 0;
+}
+
+static int tas675x_set_tdm_slot(struct snd_soc_dai *dai, unsigned int tx_mask,
+				unsigned int rx_mask, int slots, int slot_width)
+{
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(dai->component);
+
+	if (slots == 0) {
+		tas->slot_width = 0;
+		tas->tx_mask = 0;
+		return 0;
+	}
+
+	/* No rx_mask as hardware does not support channel muxing for capture */
+	tas->slot_width = slot_width;
+	tas->tx_mask = tx_mask;
+	return 0;
+}
+
+static int tas675x_mute_stream(struct snd_soc_dai *dai, int mute, int direction)
+{
+	struct snd_soc_component *component = dai->component;
+	struct tas675x_priv *tas = snd_soc_component_get_drvdata(component);
+	unsigned int discard;
+	int ret;
+
+	if (direction == SNDRV_PCM_STREAM_CAPTURE) {
+		if (mute)
+			clear_bit(dai->id, &tas->active_capture_dais);
+		else
+			set_bit(dai->id, &tas->active_capture_dais);
+		return 0;
+	}
+
+	/*
+	 * Track which playback DAIs are active.
+	 * The TAS675x has two playback DAIs (main audio and LLP).
+	 * Only transition to SLEEP when ALL are muted.
+	 */
+	if (mute)
+		clear_bit(dai->id, &tas->active_playback_dais);
+	else
+		set_bit(dai->id, &tas->active_playback_dais);
+
+	/* Last playback stream */
+	if (mute && !tas->active_playback_dais) {
+		ret = tas675x_set_state_all(tas, TAS675X_STATE_SLEEP_BOTH);
+		regmap_read(tas->regmap, TAS675X_CLK_FAULT_LATCHED_REG, &discard);
+		return ret;
+	}
+
+	return tas675x_set_state_all(tas,
+				     tas->active_playback_dais ?
+					TAS675X_STATE_PLAY_BOTH :
+					TAS675X_STATE_SLEEP_BOTH);
+}
+
+static const struct snd_soc_dai_ops tas675x_dai_ops = {
+	.hw_params	= tas675x_hw_params,
+	.set_fmt	= tas675x_set_fmt,
+	.set_tdm_slot	= tas675x_set_tdm_slot,
+	.mute_stream	= tas675x_mute_stream,
+};
+
+static struct snd_soc_dai_driver tas675x_dais[] = {
+	{
+		.name = "tas675x-audio",
+		.id = 0,
+		.playback = {
+			.stream_name = "Playback",
+			.channels_min = 2,
+			.channels_max = 4,
+			.rates = SNDRV_PCM_RATE_44100 | SNDRV_PCM_RATE_48000 |
+				 SNDRV_PCM_RATE_96000 | SNDRV_PCM_RATE_192000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_LE |
+				   SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE,
+		},
+		.ops = &tas675x_dai_ops,
+	},
+	/* Only available when Low Latency Path (LLP) is enabled */
+	{
+		.name = "tas675x-anc",
+		.id = 1,
+		.playback = {
+			.stream_name = "ANC Playback",
+			.channels_min = 2,
+			.channels_max = 4,
+			.rates = SNDRV_PCM_RATE_48000 | SNDRV_PCM_RATE_96000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_LE |
+				   SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE,
+		},
+		.ops = &tas675x_dai_ops,
+	},
+	{
+		.name = "tas675x-feedback",
+		.id = 2,
+		.capture = {
+			.stream_name = "Feedback Capture",
+			.channels_min = 2,
+			.channels_max = 8,
+			.rates = SNDRV_PCM_RATE_48000,
+			.formats = SNDRV_PCM_FMTBIT_S16_LE | SNDRV_PCM_FMTBIT_S20_LE |
+				   SNDRV_PCM_FMTBIT_S24_LE | SNDRV_PCM_FMTBIT_S32_LE,
+		},
+		.ops = &tas675x_dai_ops,
+	}
+};
+
+/*
+ * Enable regulators and release hardware reset GPIOs.
+ * The device is not I2C-accessible until this returns.
+ */
+static int tas675x_hw_enable(struct tas675x_priv *tas)
+{
+	int ret;
+
+	ret = regulator_bulk_enable(ARRAY_SIZE(tas->supplies), tas->supplies);
+	if (ret) {
+		dev_err(tas->dev, "Failed to enable regulators: %d\n", ret);
+		return ret;
+	}
+
+	if (!IS_ERR(tas->vbat)) {
+		ret = regulator_enable(tas->vbat);
+		if (ret) {
+			dev_err(tas->dev, "Failed to enable vbat: %d\n", ret);
+			regulator_bulk_disable(ARRAY_SIZE(tas->supplies), tas->supplies);
+			return ret;
+		}
+	}
+
+	if (tas->pd_gpio && tas->stby_gpio) {
+		/*
+		 * Independent Pin Control
+		 * Deassert PD first to boot digital, then STBY for analog.
+		 */
+		/* Min 4ms digital boot wait */
+		gpiod_set_value_cansleep(tas->pd_gpio, 0);
+		usleep_range(4000, 5000);
+
+		/* ~2ms analog stabilization */
+		gpiod_set_value_cansleep(tas->stby_gpio, 0);
+		usleep_range(2000, 3000);
+	} else if (tas->pd_gpio) {
+		/*
+		 * Simultaneous Pin Release
+		 * STBY tied to PD or hardwired HIGH.
+		 */
+		/* 6ms wait for simultaneous release transition */
+		gpiod_set_value_cansleep(tas->pd_gpio, 0);
+		usleep_range(6000, 7000);
+	} else {
+		/*
+		 * PD hardwired, device in DEEP_SLEEP.
+		 * Digital core already booted, I2C active. Deassert STBY
+		 * to bring up the analog output stage.
+		 */
+		/* ~2ms analog stabilization */
+		gpiod_set_value_cansleep(tas->stby_gpio, 0);
+		usleep_range(2000, 3000);
+	}
+
+	return 0;
+}
+
+static void tas675x_hw_disable(struct tas675x_priv *tas)
+{
+	if (tas->stby_gpio)
+		gpiod_set_value_cansleep(tas->stby_gpio, 1);
+
+	if (tas->pd_gpio)
+		gpiod_set_value_cansleep(tas->pd_gpio, 1);
+
+	/*
+	 * Hold PD/STBY asserted for at least 10ms
+	 * before removing PVDD, VBAT or DVDD.
+	 */
+	usleep_range(10000, 11000);
+
+	if (!IS_ERR(tas->vbat))
+		regulator_disable(tas->vbat);
+
+	regulator_bulk_disable(ARRAY_SIZE(tas->supplies), tas->supplies);
+}
+
+/*
+ * Write device start-up defaults.
+ * Must be called after tas675x_hw_enable() and after regcache is enabled.
+ */
+static int tas675x_init_device(struct tas675x_priv *tas)
+{
+	struct regmap *regmap = tas->regmap;
+	unsigned int val;
+	int ret, i;
+
+	/* Clear POR fault flag to prevent IRQ storm */
+	regmap_read(regmap, TAS675X_POWER_FAULT_LATCHED_REG, &val);
+
+	/* Bypass DC Load Diagnostics for fast boot */
+	if (tas->fast_boot)
+		regmap_update_bits(regmap, TAS675X_DC_LDG_CTRL_REG,
+				   TAS675X_LDG_ABORT_BIT | TAS675X_LDG_BYPASS_BIT,
+				   TAS675X_LDG_ABORT_BIT | TAS675X_LDG_BYPASS_BIT);
+
+	tas675x_select_book(regmap, TAS675X_BOOK_DEFAULT);
+
+	/* Enter setup mode */
+	ret = regmap_write(regmap, TAS675X_SETUP_REG1, TAS675X_SETUP_ENTER_VAL1);
+	if (ret)
+		goto err;
+	ret = regmap_write(regmap, TAS675X_SETUP_REG2, TAS675X_SETUP_ENTER_VAL2);
+	if (ret)
+		goto err;
+
+	/* Set all channels to Sleep (required before Page 1 config) */
+	tas675x_set_state_all(tas, TAS675X_STATE_SLEEP_BOTH);
+
+	/* Set DAC clock per TRM startup script */
+	regmap_write(regmap, TAS675X_DAC_CLK_REG, 0x00);
+
+	/*
+	 * Switch to Page 1 for safety-critical OC/CBC configuration,
+	 * while bypassing regcache. (Page 1 not accessible post setup)
+	 */
+	regcache_cache_bypass(regmap, true);
+	ret = regmap_multi_reg_write(regmap, tas675x_page1_init,
+				     ARRAY_SIZE(tas675x_page1_init));
+	regcache_cache_bypass(regmap, false);
+	if (ret)
+		goto err_setup;
+
+	/* Resync regmap's cached page selector */
+	regmap_write(regmap, TAS675X_PAGE_CTRL_REG, 0x00);
+
+	/* Exit setup mode */
+	regmap_write(regmap, TAS675X_SETUP_REG1, TAS675X_SETUP_EXIT_VAL);
+	regmap_write(regmap, TAS675X_SETUP_REG2, TAS675X_SETUP_EXIT_VAL);
+
+	/* Write DSP parameters if cached */
+	for (i = 0; i < ARRAY_SIZE(tas->dsp_params); i++) {
+		if (tas->dsp_params[i].val)
+			tas675x_dsp_mem_write(tas,
+					      tas->dsp_params[i].page,
+					      tas->dsp_params[i].reg,
+					      tas->dsp_params[i].val);
+	}
+
+	/*
+	 * Configure fault and warning event routing:
+	 *
+	 * ROUTING_1: CP fault/UVLO latch, OUTM soft short latch
+	 * ROUTING_2: CBC latch, OTSD latch, OTSD, power fault
+	 * ROUTING_3: CBC latch, OTSD latch, power latch, DC LDG,
+	 *            OTSD, power warnings
+	 * ROUTING_4: OC latch, DC latch, protection shutdown
+	 *            OTW latch, OTW, clip latch
+	 * ROUTING_5: clock latch+non-latch, RTLDG latch
+	 *            CBC warning, clip warning
+	 */
+	regmap_write(regmap, TAS675X_REPORT_ROUTING_1_REG, 0x70);
+	regmap_write(regmap, TAS675X_REPORT_ROUTING_2_REG, 0xA3);
+	regmap_write(regmap, TAS675X_REPORT_ROUTING_3_REG, 0xBB);
+	regmap_write(regmap, TAS675X_REPORT_ROUTING_4_REG, 0x7E);
+	regmap_write(regmap, TAS675X_REPORT_ROUTING_5_REG, 0xF3);
+
+	/* Configure GPIO pins if specified in DT */
+	if (tas->gpio1_func >= 0 || tas->gpio2_func >= 0) {
+		unsigned int gpio_ctrl = TAS675X_GPIO_CTRL_RSTVAL;
+
+		tas675x_config_gpio_pin(regmap, tas->gpio1_func,
+					TAS675X_GPIO1_OUTPUT_SEL_REG,
+					0, &gpio_ctrl);
+		tas675x_config_gpio_pin(regmap, tas->gpio2_func,
+					TAS675X_GPIO2_OUTPUT_SEL_REG,
+					1, &gpio_ctrl);
+		regmap_write(regmap, TAS675X_GPIO_CTRL_REG, gpio_ctrl);
+	}
+
+	/* Clear fast boot bits */
+	if (tas->fast_boot)
+		regmap_update_bits(regmap, TAS675X_DC_LDG_CTRL_REG,
+				   TAS675X_LDG_ABORT_BIT | TAS675X_LDG_BYPASS_BIT,
+				   0);
+
+	/* Clear any stale faults from the boot sequence */
+	regmap_read(regmap, TAS675X_POWER_FAULT_STATUS_1_REG, &val);
+	regmap_read(regmap, TAS675X_POWER_FAULT_LATCHED_REG, &val);
+	regmap_read(regmap, TAS675X_CLK_FAULT_LATCHED_REG, &val);
+	regmap_write(regmap, TAS675X_RESET_REG, TAS675X_FAULT_CLEAR);
+
+	return 0;
+
+err_setup:
+	regmap_write(regmap, TAS675X_SETUP_REG1, TAS675X_SETUP_EXIT_VAL);
+	regmap_write(regmap, TAS675X_SETUP_REG2, TAS675X_SETUP_EXIT_VAL);
+err:
+	dev_err(tas->dev, "Init device failed: %d\n", ret);
+	return ret;
+}
+
+static void tas675x_power_off(struct tas675x_priv *tas)
+{
+	regcache_cache_only(tas->regmap, true);
+	regcache_mark_dirty(tas->regmap);
+	tas675x_hw_disable(tas);
+}
+
+static int tas675x_power_on(struct tas675x_priv *tas)
+{
+	int ret;
+
+	ret = tas675x_hw_enable(tas);
+	if (ret)
+		return ret;
+
+	regcache_cache_only(tas->regmap, false);
+	regcache_mark_dirty(tas->regmap);
+
+	ret = tas675x_init_device(tas);
+	if (ret)
+		goto err_disable;
+
+	ret = regcache_sync(tas->regmap);
+	if (ret) {
+		dev_err(tas->dev, "Failed to sync regcache: %d\n", ret);
+		goto err_disable;
+	}
+
+	/* Reset fault tracking */
+	memset(tas->last_status, 0, sizeof(tas->last_status));
+
+	return 0;
+
+err_disable:
+	tas675x_power_off(tas);
+	return ret;
+}
+
+static int tas675x_runtime_suspend(struct device *dev)
+{
+	struct tas675x_priv *tas = dev_get_drvdata(dev);
+
+	disable_delayed_work_sync(&tas->fault_check_work);
+	tas675x_set_state_all(tas, TAS675X_STATE_SLEEP_BOTH);
+
+	return 0;
+}
+
+static int tas675x_runtime_resume(struct device *dev)
+{
+	struct tas675x_priv *tas = dev_get_drvdata(dev);
+
+	tas675x_set_state_all(tas, TAS675X_STATE_SLEEP_BOTH);
+
+	if (!to_i2c_client(dev)->irq) {
+		enable_delayed_work(&tas->fault_check_work);
+		schedule_delayed_work(&tas->fault_check_work,
+				      msecs_to_jiffies(TAS675X_FAULT_CHECK_INTERVAL_MS));
+	}
+
+	return 0;
+}
+
+static int tas675x_system_suspend(struct device *dev)
+{
+	struct tas675x_priv *tas = dev_get_drvdata(dev);
+	int ret;
+
+	ret = tas675x_runtime_suspend(dev);
+	if (ret)
+		return ret;
+
+	if (to_i2c_client(dev)->irq)
+		disable_irq(to_i2c_client(dev)->irq);
+
+	tas675x_power_off(tas);
+	return 0;
+}
+
+static int tas675x_system_resume(struct device *dev)
+{
+	struct tas675x_priv *tas = dev_get_drvdata(dev);
+	int ret;
+
+	ret = tas675x_power_on(tas);
+	if (ret)
+		return ret;
+
+	if (to_i2c_client(dev)->irq)
+		enable_irq(to_i2c_client(dev)->irq);
+
+	return tas675x_runtime_resume(dev);
+}
+
+static const struct snd_soc_component_driver soc_codec_dev_tas675x = {
+	.controls		= tas675x_snd_controls,
+	.num_controls		= ARRAY_SIZE(tas675x_snd_controls),
+	.dapm_widgets		= tas675x_dapm_widgets,
+	.num_dapm_widgets	= ARRAY_SIZE(tas675x_dapm_widgets),
+	.dapm_routes		= tas675x_dapm_routes,
+	.num_dapm_routes	= ARRAY_SIZE(tas675x_dapm_routes),
+	.endianness		= 1,
+};
+
+/* Fault register flags */
+#define TAS675X_FAULT_CRITICAL	BIT(0)	/* causes FAULT state, FAULT_CLEAR required */
+#define TAS675X_FAULT_TRACK	BIT(1)	/* track last value, only log on change */
+#define TAS675X_FAULT_ACTIVE	BIT(2)	/* skip when no stream is active */
+
+struct tas675x_fault_reg {
+	unsigned int reg;
+	unsigned int flags;
+	const char *name;
+};
+
+static const struct tas675x_fault_reg tas675x_fault_table[] = {
+	/* Critical */
+	{ TAS675X_OTSD_LATCHED_REG,              TAS675X_FAULT_CRITICAL | TAS675X_FAULT_TRACK,
+	  "Overtemperature Shutdown" },
+	{ TAS675X_OC_DC_FAULT_LATCHED_REG,       TAS675X_FAULT_CRITICAL | TAS675X_FAULT_TRACK,
+	  "Overcurrent / DC Fault" },
+	{ TAS675X_RTLDG_OL_SL_FAULT_LATCHED_REG, TAS675X_FAULT_CRITICAL | TAS675X_FAULT_TRACK,
+	  "Real-Time Load Diagnostic Fault" },
+	{ TAS675X_CBC_FAULT_WARN_LATCHED_REG,    TAS675X_FAULT_CRITICAL | TAS675X_FAULT_TRACK,
+	  "CBC Fault/Warning" },
+	/* Warning */
+	{ TAS675X_POWER_FAULT_STATUS_1_REG,      TAS675X_FAULT_TRACK,
+	  "CP / OUTM Fault" },
+	{ TAS675X_POWER_FAULT_LATCHED_REG,       TAS675X_FAULT_TRACK,
+	  "Power Fault" },
+	{ TAS675X_CLK_FAULT_LATCHED_REG,         TAS675X_FAULT_TRACK | TAS675X_FAULT_ACTIVE,
+	  "Clock Fault" },
+	{ TAS675X_OTW_LATCHED_REG,               TAS675X_FAULT_TRACK,
+	  "Overtemperature Warning" },
+	{ TAS675X_CLIP_WARN_LATCHED_REG,         TAS675X_FAULT_ACTIVE,
+	  "Clip Warning" },
+};
+
+static_assert(ARRAY_SIZE(tas675x_fault_table) == TAS675X_FAULT_REGS_NUM);
+
+/*
+ * Read and log all latched fault registers.
+ * Shared by both the polled fault_check_work and IRQ handler paths
+ * (which are mutually exclusive, only one is active per device).
+ * Returns true if any fault register needs to be cleared.
+ *
+ * For deciphering fault messages, see "Fault Monitoring" in
+ * Documentation/sound/codecs/tas675x.rst
+ */
+static bool tas675x_check_faults(struct tas675x_priv *tas)
+{
+	struct device *dev = tas->dev;
+	bool needs_clear = false;
+	unsigned int reg;
+	int i, ret;
+
+	for (i = 0; i < ARRAY_SIZE(tas675x_fault_table); i++) {
+		const struct tas675x_fault_reg *f = &tas675x_fault_table[i];
+
+		ret = regmap_read(tas->regmap, f->reg, &reg);
+		if (ret) {
+			if (f->flags & TAS675X_FAULT_CRITICAL) {
+				dev_err(dev, "failed to read %s: %d\n", f->name, ret);
+				return needs_clear;
+			}
+			continue;
+		}
+
+		if (reg)
+			needs_clear = true;
+
+		/* Skip logging stream-dependent events when no stream is active */
+		if ((f->flags & TAS675X_FAULT_ACTIVE) &&
+		    !READ_ONCE(tas->active_playback_dais) &&
+		    !READ_ONCE(tas->active_capture_dais))
+			continue;
+
+		/* Log on change or on every non-zero read */
+		if (reg && (!(f->flags & TAS675X_FAULT_TRACK) ||
+			    reg != tas->last_status[i])) {
+			if (f->flags & TAS675X_FAULT_CRITICAL)
+				dev_crit(dev, "%s Latched: 0x%02x\n", f->name, reg);
+			else
+				dev_warn(dev, "%s Latched: 0x%02x\n", f->name, reg);
+		}
+
+		if (f->flags & TAS675X_FAULT_TRACK)
+			tas->last_status[i] = reg;
+	}
+
+	return needs_clear;
+}
+
+static void tas675x_fault_check_work(struct work_struct *work)
+{
+	struct tas675x_priv *tas = container_of(work, struct tas675x_priv,
+						fault_check_work.work);
+
+	if (tas675x_check_faults(tas))
+		regmap_write(tas->regmap, TAS675X_RESET_REG, TAS675X_FAULT_CLEAR);
+
+	schedule_delayed_work(&tas->fault_check_work,
+			      msecs_to_jiffies(TAS675X_FAULT_CHECK_INTERVAL_MS));
+}
+
+static irqreturn_t tas675x_irq_handler(int irq, void *data)
+{
+	struct tas675x_priv *tas = data;
+	irqreturn_t ret = IRQ_NONE;
+
+	if (pm_runtime_resume_and_get(tas->dev) < 0)
+		return IRQ_NONE;
+
+	if (tas675x_check_faults(tas)) {
+		regmap_write(tas->regmap, TAS675X_RESET_REG, TAS675X_FAULT_CLEAR);
+		ret = IRQ_HANDLED;
+	}
+
+	pm_runtime_mark_last_busy(tas->dev);
+	pm_runtime_put_autosuspend(tas->dev);
+	return ret;
+}
+
+static const struct reg_default tas675x_reg_defaults[] = {
+	{ TAS675X_PAGE_CTRL_REG,           0x00 },
+	{ TAS675X_OUTPUT_CTRL_REG,         0x00 },
+	{ TAS675X_STATE_CTRL_CH1_CH2_REG,  TAS675X_STATE_SLEEP_BOTH },
+	{ TAS675X_STATE_CTRL_CH3_CH4_REG,  TAS675X_STATE_SLEEP_BOTH },
+	{ TAS675X_ISENSE_CTRL_REG,         0x0F },
+	{ TAS675X_DC_DETECT_CTRL_REG,      0x00 },
+	{ TAS675X_SCLK_INV_CTRL_REG,       0x00 },
+	{ TAS675X_AUDIO_IF_CTRL_REG,       0x00 },
+	{ TAS675X_SDIN_CTRL_REG,           0x0A },
+	{ TAS675X_SDOUT_CTRL_REG,          0x1A },
+	{ TAS675X_SDIN_OFFSET_MSB_REG,     0x00 },
+	{ TAS675X_SDIN_AUDIO_OFFSET_REG,   0x00 },
+	{ TAS675X_SDIN_LL_OFFSET_REG,      0x60 },
+	{ TAS675X_SDIN_CH_SWAP_REG,        0x00 },
+	{ TAS675X_SDOUT_OFFSET_MSB_REG,    0xCF },
+	{ TAS675X_VPREDICT_OFFSET_REG,     0xFF },
+	{ TAS675X_ISENSE_OFFSET_REG,       0x00 },
+	{ TAS675X_SDOUT_EN_REG,            0x00 },
+	{ TAS675X_LL_EN_REG,               0x00 },
+	{ TAS675X_RTLDG_EN_REG,            0x10 },
+	{ TAS675X_DC_BLOCK_BYP_REG,        0x00 },
+	{ TAS675X_DSP_CTRL_REG,            0x00 },
+	{ TAS675X_PAGE_AUTO_INC_REG,       0x00 },
+	{ TAS675X_DIG_VOL_CH1_REG,         0x30 },
+	{ TAS675X_DIG_VOL_CH2_REG,         0x30 },
+	{ TAS675X_DIG_VOL_CH3_REG,         0x30 },
+	{ TAS675X_DIG_VOL_CH4_REG,         0x30 },
+	{ TAS675X_DIG_VOL_RAMP_CTRL_REG,   0x77 },
+	{ TAS675X_DIG_VOL_COMBINE_CTRL_REG, 0x00 },
+	{ TAS675X_AUTO_MUTE_EN_REG,        0x00 },
+	{ TAS675X_AUTO_MUTE_TIMING_CH1_CH2_REG, 0x00 },
+	{ TAS675X_AUTO_MUTE_TIMING_CH3_CH4_REG, 0x00 },
+	{ TAS675X_ANALOG_GAIN_CH1_CH2_REG, 0x00 },
+	{ TAS675X_ANALOG_GAIN_CH3_CH4_REG, 0x00 },
+	{ TAS675X_ANALOG_GAIN_RAMP_CTRL_REG, 0x00 },
+	{ TAS675X_PULSE_INJECTION_EN_REG,  0x03 },
+	{ TAS675X_CBC_CTRL_REG,            0x07 },
+	{ TAS675X_CURRENT_LIMIT_CTRL_REG,  0x00 },
+	{ TAS675X_ISENSE_CAL_REG,          0x00 },
+	{ TAS675X_PWM_PHASE_CTRL_REG,      0x00 },
+	{ TAS675X_SS_CTRL_REG,             0x00 },
+	{ TAS675X_SS_RANGE_CTRL_REG,       0x00 },
+	{ TAS675X_SS_DWELL_CTRL_REG,       0x00 },
+	{ TAS675X_RAMP_PHASE_CTRL_GPO_REG, 0x00 },
+	{ TAS675X_PWM_PHASE_M_CTRL_CH1_REG, 0x00 },
+	{ TAS675X_PWM_PHASE_M_CTRL_CH2_REG, 0x00 },
+	{ TAS675X_PWM_PHASE_M_CTRL_CH3_REG, 0x00 },
+	{ TAS675X_PWM_PHASE_M_CTRL_CH4_REG, 0x00 },
+	{ TAS675X_DC_LDG_CTRL_REG,         0x00 },
+	{ TAS675X_DC_LDG_LO_CTRL_REG,      0x00 },
+	{ TAS675X_DC_LDG_TIME_CTRL_REG,    0x00 },
+	{ TAS675X_DC_LDG_SL_CH1_CH2_CTRL_REG, 0x11 },
+	{ TAS675X_DC_LDG_SL_CH3_CH4_CTRL_REG, 0x11 },
+	{ TAS675X_AC_LDG_CTRL_REG,         0x10 },
+	{ TAS675X_TWEETER_DETECT_CTRL_REG, 0x08 },
+	{ TAS675X_TWEETER_DETECT_THRESH_REG, 0x00 },
+	{ TAS675X_AC_LDG_FREQ_CTRL_REG,    0xC8 },
+	{ TAS675X_REPORT_ROUTING_1_REG,    0x00 },
+	{ TAS675X_OTSD_RECOVERY_EN_REG,    0x00 },
+	{ TAS675X_REPORT_ROUTING_2_REG,    0xA2 },
+	{ TAS675X_REPORT_ROUTING_3_REG,    0x00 },
+	{ TAS675X_REPORT_ROUTING_4_REG,    0x06 },
+	{ TAS675X_CLIP_DETECT_CTRL_REG,    0x00 },
+	{ TAS675X_REPORT_ROUTING_5_REG,    0x00 },
+	{ TAS675X_GPIO1_OUTPUT_SEL_REG,    0x00 },
+	{ TAS675X_GPIO2_OUTPUT_SEL_REG,    0x00 },
+	{ TAS675X_GPIO_CTRL_REG,           TAS675X_GPIO_CTRL_RSTVAL },
+	{ TAS675X_OTW_CTRL_CH1_CH2_REG,    0x11 },
+	{ TAS675X_OTW_CTRL_CH3_CH4_REG,    0x11 },
+};
+
+static bool tas675x_is_readable_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case TAS675X_RESET_REG:
+		return false;
+	default:
+		return true;
+	}
+}
+
+static bool tas675x_is_volatile_register(struct device *dev, unsigned int reg)
+{
+	switch (reg) {
+	case TAS675X_RESET_REG:
+	case TAS675X_BOOK_CTRL_REG:
+	case TAS675X_AUTO_MUTE_STATUS_REG:
+	case TAS675X_STATE_REPORT_CH1_CH2_REG:
+	case TAS675X_STATE_REPORT_CH3_CH4_REG:
+	case TAS675X_PVDD_SENSE_REG:
+	case TAS675X_TEMP_GLOBAL_REG:
+	case TAS675X_TEMP_CH1_CH2_REG:
+	case TAS675X_TEMP_CH3_CH4_REG:
+	case TAS675X_FS_MON_REG:
+	case TAS675X_SCLK_MON_REG:
+	case TAS675X_POWER_FAULT_STATUS_1_REG:
+	case TAS675X_POWER_FAULT_STATUS_2_REG:
+	case TAS675X_OT_FAULT_REG:
+	case TAS675X_OTW_STATUS_REG:
+	case TAS675X_CLIP_WARN_STATUS_REG:
+	case TAS675X_CBC_WARNING_STATUS_REG:
+	case TAS675X_POWER_FAULT_LATCHED_REG:
+	case TAS675X_OTSD_LATCHED_REG:
+	case TAS675X_OTW_LATCHED_REG:
+	case TAS675X_CLIP_WARN_LATCHED_REG:
+	case TAS675X_CLK_FAULT_LATCHED_REG:
+	case TAS675X_RTLDG_OL_SL_FAULT_LATCHED_REG:
+	case TAS675X_CBC_FAULT_WARN_LATCHED_REG:
+	case TAS675X_OC_DC_FAULT_LATCHED_REG:
+	case TAS675X_WARN_OT_MAX_FLAG_REG:
+	case TAS675X_DC_LDG_REPORT_CH1_CH2_REG ... TAS675X_TWEETER_REPORT_REG:
+	case TAS675X_CH1_RTLDG_IMP_MSB_REG ... TAS675X_CH4_DC_LDG_DCR_LSB_REG:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static const struct regmap_range_cfg tas675x_ranges[] = {
+	{
+		.name = "Pages",
+		.range_min = 0,
+		.range_max = TAS675X_PAGE_SIZE * TAS675X_PAGE_SIZE - 1,
+		.selector_reg = TAS675X_PAGE_CTRL_REG,
+		.selector_mask = 0xff,
+		.selector_shift = 0,
+		.window_start = 0,
+		.window_len = TAS675X_PAGE_SIZE,
+	},
+};
+
+static void tas675x_regmap_lock(void *lock_arg)
+{
+	struct tas675x_priv *tas = lock_arg;
+
+	mutex_lock(&tas->io_lock);
+}
+
+static void tas675x_regmap_unlock(void *lock_arg)
+{
+	struct tas675x_priv *tas = lock_arg;
+
+	mutex_unlock(&tas->io_lock);
+}
+
+static const struct regmap_config tas675x_regmap_config = {
+	.reg_bits         = 8,
+	.val_bits         = 8,
+	.max_register     = TAS675X_PAGE_SIZE * TAS675X_PAGE_SIZE - 1,
+	.ranges           = tas675x_ranges,
+	.num_ranges       = ARRAY_SIZE(tas675x_ranges),
+	.cache_type       = REGCACHE_MAPLE,
+	.reg_defaults     = tas675x_reg_defaults,
+	.num_reg_defaults = ARRAY_SIZE(tas675x_reg_defaults),
+	.readable_reg     = tas675x_is_readable_register,
+	.volatile_reg     = tas675x_is_volatile_register,
+};
+
+static int tas675x_i2c_probe(struct i2c_client *client)
+{
+	struct regmap_config cfg = tas675x_regmap_config;
+	struct tas675x_priv *tas;
+	u32 val;
+	int i, ret;
+
+	tas = devm_kzalloc(&client->dev, sizeof(*tas), GFP_KERNEL);
+	if (!tas)
+		return -ENOMEM;
+
+	tas->dev = &client->dev;
+	i2c_set_clientdata(client, tas);
+
+	mutex_init(&tas->io_lock);
+	cfg.lock     = tas675x_regmap_lock;
+	cfg.unlock   = tas675x_regmap_unlock;
+	cfg.lock_arg = tas;
+
+	memcpy(tas->dsp_params, tas675x_dsp_defaults, sizeof(tas->dsp_params));
+	INIT_DELAYED_WORK(&tas->fault_check_work, tas675x_fault_check_work);
+
+	tas->regmap = devm_regmap_init_i2c(client, &cfg);
+	if (IS_ERR(tas->regmap))
+		return PTR_ERR(tas->regmap);
+
+	/* Keep regmap cache-only until hardware is powered on */
+	regcache_cache_only(tas->regmap, true);
+
+	tas->dev_type = (enum tas675x_type)(unsigned long)device_get_match_data(tas->dev);
+	tas->fast_boot = device_property_read_bool(tas->dev, "ti,fast-boot");
+
+	tas->audio_slot = -1;
+	tas->llp_slot = -1;
+	tas->vpredict_slot = -1;
+	tas->isense_slot = -1;
+	if (!device_property_read_u32(tas->dev, "ti,audio-slot-no", &val))
+		tas->audio_slot = val;
+	if (!device_property_read_u32(tas->dev, "ti,llp-slot-no", &val))
+		tas->llp_slot = val;
+	if (!device_property_read_u32(tas->dev, "ti,vpredict-slot-no", &val))
+		tas->vpredict_slot = val;
+	if (!device_property_read_u32(tas->dev, "ti,isense-slot-no", &val))
+		tas->isense_slot = val;
+
+	tas->gpio1_func = tas675x_gpio_func_parse(tas->dev, "ti,gpio1-function");
+	tas->gpio2_func = tas675x_gpio_func_parse(tas->dev, "ti,gpio2-function");
+
+	for (i = 0; i < ARRAY_SIZE(tas675x_supply_names); i++)
+		tas->supplies[i].supply = tas675x_supply_names[i];
+
+	ret = devm_regulator_bulk_get(tas->dev, ARRAY_SIZE(tas->supplies), tas->supplies);
+	if (ret)
+		return dev_err_probe(tas->dev, ret, "Failed to request supplies\n");
+
+	tas->vbat = devm_regulator_get_optional(tas->dev, "vbat");
+	if (IS_ERR(tas->vbat) && PTR_ERR(tas->vbat) != -ENODEV)
+		return dev_err_probe(tas->dev, PTR_ERR(tas->vbat),
+				     "Failed to get vbat supply\n");
+
+	tas->pd_gpio = devm_gpiod_get_optional(tas->dev, "powerdown", GPIOD_OUT_HIGH);
+	if (IS_ERR(tas->pd_gpio))
+		return dev_err_probe(tas->dev, PTR_ERR(tas->pd_gpio), "Failed powerdown-gpios\n");
+
+	tas->stby_gpio = devm_gpiod_get_optional(tas->dev, "standby", GPIOD_OUT_HIGH);
+	if (IS_ERR(tas->stby_gpio))
+		return dev_err_probe(tas->dev, PTR_ERR(tas->stby_gpio), "Failed standby-gpios\n");
+
+	if (!tas->pd_gpio && !tas->stby_gpio)
+		return dev_err_probe(tas->dev, -EINVAL,
+				     "At least one of powerdown-gpios or standby-gpios is required\n");
+
+	ret = tas675x_power_on(tas);
+	if (ret)
+		return ret;
+
+	if (client->irq) {
+		ret = devm_request_threaded_irq(tas->dev, client->irq, NULL,
+						tas675x_irq_handler,
+						IRQF_ONESHOT | IRQF_TRIGGER_FALLING,
+						"tas675x-fault", tas);
+		if (ret) {
+			tas675x_power_off(tas);
+			return dev_err_probe(tas->dev, ret, "Failed to request IRQ\n");
+		}
+	} else {
+		/* Schedule delayed work for fault checking at probe and runtime resume */
+		schedule_delayed_work(&tas->fault_check_work,
+				      msecs_to_jiffies(TAS675X_FAULT_CHECK_INTERVAL_MS));
+	}
+
+	/* Enable runtime PM with 2s autosuspend */
+	pm_runtime_set_autosuspend_delay(tas->dev, 2000);
+	pm_runtime_use_autosuspend(tas->dev);
+	pm_runtime_set_active(tas->dev);
+	pm_runtime_mark_last_busy(tas->dev);
+	pm_runtime_enable(tas->dev);
+
+	ret = devm_snd_soc_register_component(tas->dev, &soc_codec_dev_tas675x,
+					      tas675x_dais, ARRAY_SIZE(tas675x_dais));
+	if (ret)
+		goto err_pm_disable;
+
+	return 0;
+
+err_pm_disable:
+	pm_runtime_force_suspend(tas->dev);
+	pm_runtime_disable(tas->dev);
+	tas675x_power_off(tas);
+	return ret;
+}
+
+static void tas675x_i2c_remove(struct i2c_client *client)
+{
+	struct tas675x_priv *tas = dev_get_drvdata(&client->dev);
+
+	disable_delayed_work_sync(&tas->fault_check_work);
+	if (client->irq)
+		disable_irq(client->irq);
+
+	pm_runtime_force_suspend(&client->dev);
+	pm_runtime_disable(&client->dev);
+	tas675x_power_off(tas);
+}
+
+static const struct dev_pm_ops tas675x_pm_ops = {
+	SYSTEM_SLEEP_PM_OPS(tas675x_system_suspend, tas675x_system_resume)
+	RUNTIME_PM_OPS(tas675x_runtime_suspend, tas675x_runtime_resume, NULL)
+};
+
+static const struct of_device_id tas675x_of_match[] = {
+	{ .compatible = "ti,tas67524", .data = (void *)TAS67524 },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, tas675x_of_match);
+
+static const struct i2c_device_id tas675x_i2c_id[] = {
+	{ "tas67524",  TAS67524 },
+	{ }
+};
+MODULE_DEVICE_TABLE(i2c, tas675x_i2c_id);
+
+static struct i2c_driver tas675x_i2c_driver = {
+	.driver = {
+		.name = "tas675x",
+		.of_match_table = tas675x_of_match,
+		.pm = pm_ptr(&tas675x_pm_ops),
+	},
+	.probe = tas675x_i2c_probe,
+	.remove = tas675x_i2c_remove,
+	.id_table = tas675x_i2c_id,
+};
+
+module_i2c_driver(tas675x_i2c_driver);
+
+MODULE_AUTHOR("Sen Wang <sen@ti.com>");
+MODULE_DESCRIPTION("ASoC TAS675x Audio Amplifier Driver");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/codecs/tas675x.h b/sound/soc/codecs/tas675x.h
new file mode 100644
index 0000000000000..db29bb3773362
--- /dev/null
+++ b/sound/soc/codecs/tas675x.h
@@ -0,0 +1,367 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * ALSA SoC Texas Instruments TAS675x Quad-Channel Audio Amplifier
+ *
+ * Copyright (C) 2026 Texas Instruments Incorporated - https://www.ti.com/
+ *	Author: Sen Wang <sen@ti.com>
+ */
+
+#ifndef __TAS675X_H__
+#define __TAS675X_H__
+
+/*
+ * Book 0, Page 0 — Register Addresses
+ */
+
+#define TAS675X_PAGE_SIZE                    256
+#define TAS675X_PAGE_REG(page, reg)  ((page) * TAS675X_PAGE_SIZE + (reg))
+
+/* Page Control & Basic Config */
+#define TAS675X_PAGE_CTRL_REG                0x00
+#define TAS675X_RESET_REG                    0x01
+#define TAS675X_OUTPUT_CTRL_REG              0x02
+#define TAS675X_STATE_CTRL_CH1_CH2_REG       0x03
+#define TAS675X_STATE_CTRL_CH3_CH4_REG       0x04
+#define TAS675X_ISENSE_CTRL_REG              0x05
+#define TAS675X_DC_DETECT_CTRL_REG           0x06
+
+/* Serial Audio Port */
+#define TAS675X_SCLK_INV_CTRL_REG            0x20
+#define TAS675X_AUDIO_IF_CTRL_REG            0x21
+#define TAS675X_SDIN_CTRL_REG                0x23
+#define TAS675X_SDOUT_CTRL_REG               0x25
+#define TAS675X_SDIN_OFFSET_MSB_REG          0x27
+#define TAS675X_SDIN_AUDIO_OFFSET_REG        0x28
+#define TAS675X_SDIN_LL_OFFSET_REG           0x29
+#define TAS675X_SDIN_CH_SWAP_REG             0x2A
+#define TAS675X_SDOUT_OFFSET_MSB_REG         0x2C
+#define TAS675X_VPREDICT_OFFSET_REG          0x2D
+#define TAS675X_ISENSE_OFFSET_REG            0x2E
+#define TAS675X_SDOUT_EN_REG                 0x31
+#define TAS675X_LL_EN_REG                    0x32
+
+/* DSP & Core Audio Control */
+#define TAS675X_RTLDG_EN_REG                 0x37
+#define TAS675X_DC_BLOCK_BYP_REG             0x39
+#define TAS675X_DSP_CTRL_REG                 0x3A
+#define TAS675X_PAGE_AUTO_INC_REG            0x3B
+
+/* Volume & Mute */
+#define TAS675X_DIG_VOL_CH1_REG              0x40
+#define TAS675X_DIG_VOL_CH2_REG              0x41
+#define TAS675X_DIG_VOL_CH3_REG              0x42
+#define TAS675X_DIG_VOL_CH4_REG              0x43
+#define TAS675X_DIG_VOL_RAMP_CTRL_REG        0x44
+#define TAS675X_DIG_VOL_COMBINE_CTRL_REG     0x46
+#define TAS675X_AUTO_MUTE_EN_REG             0x47
+#define TAS675X_AUTO_MUTE_TIMING_CH1_CH2_REG 0x48
+#define TAS675X_AUTO_MUTE_TIMING_CH3_CH4_REG 0x49
+
+/* Analog Gain & Power Stage */
+#define TAS675X_ANALOG_GAIN_CH1_CH2_REG      0x4A
+#define TAS675X_ANALOG_GAIN_CH3_CH4_REG      0x4B
+#define TAS675X_ANALOG_GAIN_RAMP_CTRL_REG    0x4E
+#define TAS675X_PULSE_INJECTION_EN_REG       0x52
+#define TAS675X_CBC_CTRL_REG                 0x54
+#define TAS675X_CURRENT_LIMIT_CTRL_REG       0x55
+#define TAS675X_DAC_CLK_REG                  0x5A
+#define TAS675X_ISENSE_CAL_REG               0x5B
+
+/* Spread Spectrum & PWM Phase */
+#define TAS675X_PWM_PHASE_CTRL_REG           0x60
+#define TAS675X_SS_CTRL_REG                  0x61
+#define TAS675X_SS_RANGE_CTRL_REG            0x62
+#define TAS675X_SS_DWELL_CTRL_REG            0x66
+#define TAS675X_RAMP_PHASE_CTRL_GPO_REG      0x68
+#define TAS675X_PWM_PHASE_M_CTRL_CH1_REG     0x69
+#define TAS675X_PWM_PHASE_M_CTRL_CH2_REG     0x6A
+#define TAS675X_PWM_PHASE_M_CTRL_CH3_REG     0x6B
+#define TAS675X_PWM_PHASE_M_CTRL_CH4_REG     0x6C
+
+/* Status & Reporting */
+#define TAS675X_AUTO_MUTE_STATUS_REG         0x71
+#define TAS675X_STATE_REPORT_CH1_CH2_REG     0x72
+#define TAS675X_STATE_REPORT_CH3_CH4_REG     0x73
+#define TAS675X_PVDD_SENSE_REG               0x74
+#define TAS675X_TEMP_GLOBAL_REG              0x75
+#define TAS675X_FS_MON_REG                   0x76
+#define TAS675X_SCLK_MON_REG                 0x77
+#define TAS675X_REPORT_ROUTING_1_REG         0x7C
+
+/* Memory Paging & Book Control */
+#define TAS675X_SETUP_REG1                   0x7D
+#define TAS675X_SETUP_REG2                   0x7E
+#define TAS675X_BOOK_CTRL_REG                0x7F
+
+/* Fault Status */
+#define TAS675X_POWER_FAULT_STATUS_1_REG     0x7D
+#define TAS675X_POWER_FAULT_STATUS_2_REG     0x80
+#define TAS675X_OT_FAULT_REG                 0x81
+#define TAS675X_OTW_STATUS_REG               0x82
+#define TAS675X_CLIP_WARN_STATUS_REG         0x83
+#define TAS675X_CBC_WARNING_STATUS_REG       0x85
+
+/* Latched Fault Registers */
+#define TAS675X_POWER_FAULT_LATCHED_REG      0x86
+#define TAS675X_OTSD_LATCHED_REG             0x87
+#define TAS675X_OTW_LATCHED_REG              0x88
+#define TAS675X_CLIP_WARN_LATCHED_REG        0x89
+#define TAS675X_CLK_FAULT_LATCHED_REG        0x8A
+#define TAS675X_RTLDG_OL_SL_FAULT_LATCHED_REG 0x8B
+#define TAS675X_CBC_FAULT_WARN_LATCHED_REG   0x8D
+#define TAS675X_OC_DC_FAULT_LATCHED_REG      0x8E
+#define TAS675X_OTSD_RECOVERY_EN_REG         0x8F
+
+/* Protection & Routing Controls */
+#define TAS675X_REPORT_ROUTING_2_REG         0x90
+#define TAS675X_REPORT_ROUTING_3_REG         0x91
+#define TAS675X_REPORT_ROUTING_4_REG         0x92
+#define TAS675X_CLIP_DETECT_CTRL_REG         0x93
+#define TAS675X_REPORT_ROUTING_5_REG         0x94
+
+/* GPIO Pin Configuration */
+#define TAS675X_GPIO1_OUTPUT_SEL_REG         0x95
+#define TAS675X_GPIO2_OUTPUT_SEL_REG         0x96
+#define TAS675X_GPIO_INPUT_SLEEP_HIZ_REG     0x9B
+#define TAS675X_GPIO_INPUT_PLAY_SLEEP_REG    0x9C
+#define TAS675X_GPIO_INPUT_MUTE_REG          0x9D
+#define TAS675X_GPIO_INPUT_SYNC_REG          0x9E
+#define TAS675X_GPIO_INPUT_SDIN2_REG         0x9F
+#define TAS675X_GPIO_CTRL_REG                0xA0
+#define TAS675X_GPIO_INVERT_REG              0xA1
+
+/* Load Diagnostics Config */
+#define TAS675X_DC_LDG_CTRL_REG              0xB0
+#define TAS675X_DC_LDG_LO_CTRL_REG           0xB1
+#define TAS675X_DC_LDG_TIME_CTRL_REG         0xB2
+#define TAS675X_DC_LDG_SL_CH1_CH2_CTRL_REG   0xB3
+#define TAS675X_DC_LDG_SL_CH3_CH4_CTRL_REG   0xB4
+#define TAS675X_AC_LDG_CTRL_REG              0xB5
+#define TAS675X_TWEETER_DETECT_CTRL_REG      0xB6
+#define TAS675X_TWEETER_DETECT_THRESH_REG    0xB7
+#define TAS675X_AC_LDG_FREQ_CTRL_REG         0xB8
+#define TAS675X_TEMP_CH1_CH2_REG             0xBB
+#define TAS675X_TEMP_CH3_CH4_REG             0xBC
+#define TAS675X_WARN_OT_MAX_FLAG_REG         0xBD
+
+/* DC Load Diagnostic Reports */
+#define TAS675X_DC_LDG_REPORT_CH1_CH2_REG    0xC0
+#define TAS675X_DC_LDG_REPORT_CH3_CH4_REG    0xC1
+#define TAS675X_DC_LDG_RESULT_REG            0xC2
+#define TAS675X_AC_LDG_REPORT_CH1_R_REG      0xC3
+#define TAS675X_AC_LDG_REPORT_CH1_I_REG      0xC4
+#define TAS675X_AC_LDG_REPORT_CH2_R_REG      0xC5
+#define TAS675X_AC_LDG_REPORT_CH2_I_REG      0xC6
+#define TAS675X_AC_LDG_REPORT_CH3_R_REG      0xC7
+#define TAS675X_AC_LDG_REPORT_CH3_I_REG      0xC8
+#define TAS675X_AC_LDG_REPORT_CH4_R_REG      0xC9
+#define TAS675X_AC_LDG_REPORT_CH4_I_REG      0xCA
+#define TAS675X_TWEETER_REPORT_REG           0xCB
+
+/* RTLDG Impedance */
+#define TAS675X_CH1_RTLDG_IMP_MSB_REG        0xD1
+#define TAS675X_CH1_RTLDG_IMP_LSB_REG        0xD2
+#define TAS675X_CH2_RTLDG_IMP_MSB_REG        0xD3
+#define TAS675X_CH2_RTLDG_IMP_LSB_REG        0xD4
+#define TAS675X_CH3_RTLDG_IMP_MSB_REG        0xD5
+#define TAS675X_CH3_RTLDG_IMP_LSB_REG        0xD6
+#define TAS675X_CH4_RTLDG_IMP_MSB_REG        0xD7
+#define TAS675X_CH4_RTLDG_IMP_LSB_REG        0xD8
+
+/* DC Load Diagnostic Resistance */
+#define TAS675X_DC_LDG_DCR_MSB_REG           0xD9
+#define TAS675X_CH1_DC_LDG_DCR_LSB_REG       0xDA
+#define TAS675X_CH2_DC_LDG_DCR_LSB_REG       0xDB
+#define TAS675X_CH3_DC_LDG_DCR_LSB_REG       0xDC
+#define TAS675X_CH4_DC_LDG_DCR_LSB_REG       0xDD
+
+/* Over-Temperature Warning */
+#define TAS675X_OTW_CTRL_CH1_CH2_REG         0xE2
+#define TAS675X_OTW_CTRL_CH3_CH4_REG         0xE3
+
+/* RESET_REG (all bits auto-clear) */
+#define TAS675X_DEVICE_RESET                 BIT(4)
+#define TAS675X_FAULT_CLEAR                  BIT(3)
+#define TAS675X_REGISTER_RESET               BIT(0)
+
+/* STATE_CTRL and STATE_REPORT — Channel state values */
+#define TAS675X_STATE_DEEPSLEEP              0x00
+#define TAS675X_STATE_LOAD_DIAG              0x01
+#define TAS675X_STATE_SLEEP                  0x02
+#define TAS675X_STATE_HIZ                    0x03
+#define TAS675X_STATE_PLAY                   0x04
+
+/* Additional STATE_REPORT values */
+#define TAS675X_STATE_FAULT                  0x05
+#define TAS675X_STATE_AUTOREC                0x06
+
+/* Combined values for both channel pairs in one register */
+#define TAS675X_STATE_DEEPSLEEP_BOTH \
+	(TAS675X_STATE_DEEPSLEEP | (TAS675X_STATE_DEEPSLEEP << 4))
+#define TAS675X_STATE_LOAD_DIAG_BOTH \
+	(TAS675X_STATE_LOAD_DIAG | (TAS675X_STATE_LOAD_DIAG << 4))
+#define TAS675X_STATE_SLEEP_BOTH \
+	(TAS675X_STATE_SLEEP | (TAS675X_STATE_SLEEP << 4))
+#define TAS675X_STATE_HIZ_BOTH \
+	(TAS675X_STATE_HIZ | (TAS675X_STATE_HIZ << 4))
+#define TAS675X_STATE_PLAY_BOTH \
+	(TAS675X_STATE_PLAY | (TAS675X_STATE_PLAY << 4))
+#define TAS675X_STATE_FAULT_BOTH \
+	(TAS675X_STATE_FAULT | (TAS675X_STATE_FAULT << 4))
+
+/* STATE_CTRL_CH1_CH2 / STATE_CTRL_CH3_CH4 — mute bits */
+#define TAS675X_CH1_MUTE_BIT                 BIT(7)
+#define TAS675X_CH2_MUTE_BIT                 BIT(3)
+#define TAS675X_CH_MUTE_BOTH                 (TAS675X_CH1_MUTE_BIT | TAS675X_CH2_MUTE_BIT)
+
+/* SCLK_INV_CTRL_REG */
+#define TAS675X_SCLK_INV_TX_BIT             BIT(5)
+#define TAS675X_SCLK_INV_RX_BIT             BIT(4)
+#define TAS675X_SCLK_INV_MASK               (TAS675X_SCLK_INV_TX_BIT | TAS675X_SCLK_INV_RX_BIT)
+
+/* AUDIO_IF_CTRL_REG */
+#define TAS675X_TDM_EN_BIT                   BIT(4)
+#define TAS675X_SAP_FMT_MASK                 GENMASK(3, 2)
+#define TAS675X_SAP_FMT_I2S                  (0x00 << 2)
+#define TAS675X_SAP_FMT_TDM                  (0x01 << 2)
+#define TAS675X_SAP_FMT_RIGHT_J              (0x02 << 2)
+#define TAS675X_SAP_FMT_LEFT_J               (0x03 << 2)
+#define TAS675X_FS_PULSE_MASK                GENMASK(1, 0)
+#define TAS675X_FS_PULSE_SHORT               0x01
+
+/* SDIN_CTRL_REG */
+#define TAS675X_SDIN_AUDIO_WL_MASK           GENMASK(3, 2)
+#define TAS675X_SDIN_LL_WL_MASK              GENMASK(1, 0)
+#define TAS675X_SDIN_WL_MASK                 (TAS675X_SDIN_AUDIO_WL_MASK | TAS675X_SDIN_LL_WL_MASK)
+
+/* SDOUT_CTRL_REG */
+#define TAS675X_SDOUT_SELECT_MASK            GENMASK(7, 4)
+#define TAS675X_SDOUT_SELECT_TDM_SDOUT1      0x00
+#define TAS675X_SDOUT_SELECT_NON_TDM         0x10
+#define TAS675X_SDOUT_VP_WL_MASK             GENMASK(3, 2)
+#define TAS675X_SDOUT_IS_WL_MASK             GENMASK(1, 0)
+#define TAS675X_SDOUT_WL_MASK                (TAS675X_SDOUT_VP_WL_MASK | TAS675X_SDOUT_IS_WL_MASK)
+
+/* SDOUT_EN_REG */
+#define TAS675X_SDOUT_NON_TDM_SEL_MASK      GENMASK(5, 4)
+#define TAS675X_SDOUT_NON_TDM_SEL_VPREDICT  (0x0 << 4)
+#define TAS675X_SDOUT_NON_TDM_SEL_ISENSE    (0x1 << 4)
+#define TAS675X_SDOUT_EN_VPREDICT           BIT(0)
+#define TAS675X_SDOUT_EN_ISENSE             BIT(1)
+#define TAS675X_SDOUT_EN_NON_TDM_ALL        GENMASK(1, 0)
+
+/* Word length values (shared by SDIN_CTRL and SDOUT_CTRL) */
+#define TAS675X_WL_16BIT                     0x00
+#define TAS675X_WL_20BIT                     0x01
+#define TAS675X_WL_24BIT                     0x02
+#define TAS675X_WL_32BIT                     0x03
+
+/* SDIN_OFFSET_MSB_REG */
+#define TAS675X_SDIN_AUDIO_OFF_MSB_MASK      GENMASK(7, 6)
+#define TAS675X_SDIN_LL_OFF_MSB_MASK         GENMASK(5, 4)
+
+/* SDOUT_OFFSET_MSB_REG */
+#define TAS675X_SDOUT_VP_OFF_MSB_MASK        GENMASK(7, 6)
+#define TAS675X_SDOUT_IS_OFF_MSB_MASK        GENMASK(5, 4)
+
+/* RTLDG_EN_REG */
+#define TAS675X_RTLDG_CLIP_MASK_BIT          BIT(4)
+#define TAS675X_RTLDG_CH_EN_MASK             GENMASK(3, 0)
+
+/* DC_LDG_CTRL_REG */
+#define TAS675X_LDG_ABORT_BIT                BIT(7)
+#define TAS675X_LDG_BUFFER_WAIT_MASK         GENMASK(6, 5)
+#define TAS675X_LDG_WAIT_BYPASS_BIT          BIT(2)
+#define TAS675X_SLOL_DISABLE_BIT             BIT(1)
+#define TAS675X_LDG_BYPASS_BIT               BIT(0)
+
+/* DC_LDG_TIME_CTRL_REG */
+#define TAS675X_LDG_RAMP_SLOL_MASK           GENMASK(7, 6)
+#define TAS675X_LDG_SETTLING_SLOL_MASK       GENMASK(5, 4)
+#define TAS675X_LDG_RAMP_S2PG_MASK           GENMASK(3, 2)
+#define TAS675X_LDG_SETTLING_S2PG_MASK       GENMASK(1, 0)
+
+/* AC_LDG_CTRL_REG */
+#define TAS675X_AC_DIAG_GAIN_BIT             BIT(4)
+#define TAS675X_AC_DIAG_START_MASK           GENMASK(3, 0)
+
+/* DC_LDG_RESULT_REG */
+#define TAS675X_DC_LDG_LO_RESULT_MASK        GENMASK(7, 4)
+#define TAS675X_DC_LDG_PASS_MASK             GENMASK(3, 0)
+
+/* Load Diagnostics Timing Constants */
+#define TAS675X_POLL_INTERVAL_US             10000
+#define TAS675X_STATE_TRANSITION_TIMEOUT_US  50000
+#define TAS675X_DC_LDG_TIMEOUT_US            300000
+#define TAS675X_AC_LDG_TIMEOUT_US            400000
+
+/* GPIO_CTRL_REG */
+#define TAS675X_GPIO1_OUTPUT_EN              BIT(7)
+#define TAS675X_GPIO2_OUTPUT_EN              BIT(6)
+#define TAS675X_GPIO_CTRL_RSTVAL             0x22
+
+/* GPIO output select values */
+#define TAS675X_GPIO_SEL_LOW                 0x00
+#define TAS675X_GPIO_SEL_AUTO_MUTE_ALL       0x02
+#define TAS675X_GPIO_SEL_AUTO_MUTE_CH4       0x03
+#define TAS675X_GPIO_SEL_AUTO_MUTE_CH3       0x04
+#define TAS675X_GPIO_SEL_AUTO_MUTE_CH2       0x05
+#define TAS675X_GPIO_SEL_AUTO_MUTE_CH1       0x06
+#define TAS675X_GPIO_SEL_SDOUT2              0x08
+#define TAS675X_GPIO_SEL_SDOUT1              0x09
+#define TAS675X_GPIO_SEL_WARN                0x0A
+#define TAS675X_GPIO_SEL_FAULT               0x0B
+#define TAS675X_GPIO_SEL_CLOCK_SYNC          0x0E
+#define TAS675X_GPIO_SEL_INVALID_CLK         0x0F
+#define TAS675X_GPIO_SEL_HIGH                0x13
+
+/* GPIO input function encoding (flag bit | function ID) */
+#define TAS675X_GPIO_FUNC_INPUT              0x100
+
+/* Input Function IDs */
+#define TAS675X_GPIO_IN_ID_MUTE              0
+#define TAS675X_GPIO_IN_ID_PHASE_SYNC        1
+#define TAS675X_GPIO_IN_ID_SDIN2             2
+#define TAS675X_GPIO_IN_ID_DEEP_SLEEP        3
+#define TAS675X_GPIO_IN_ID_HIZ               4
+#define TAS675X_GPIO_IN_ID_PLAY              5
+#define TAS675X_GPIO_IN_ID_SLEEP             6
+#define TAS675X_GPIO_IN_NUM                  7
+
+#define TAS675X_GPIO_IN_MUTE                 (TAS675X_GPIO_FUNC_INPUT | TAS675X_GPIO_IN_ID_MUTE)
+#define TAS675X_GPIO_IN_PHASE_SYNC \
+	(TAS675X_GPIO_FUNC_INPUT | TAS675X_GPIO_IN_ID_PHASE_SYNC)
+#define TAS675X_GPIO_IN_SDIN2                (TAS675X_GPIO_FUNC_INPUT | TAS675X_GPIO_IN_ID_SDIN2)
+#define TAS675X_GPIO_IN_DEEP_SLEEP \
+	(TAS675X_GPIO_FUNC_INPUT | TAS675X_GPIO_IN_ID_DEEP_SLEEP)
+#define TAS675X_GPIO_IN_HIZ                  (TAS675X_GPIO_FUNC_INPUT | TAS675X_GPIO_IN_ID_HIZ)
+#define TAS675X_GPIO_IN_PLAY                 (TAS675X_GPIO_FUNC_INPUT | TAS675X_GPIO_IN_ID_PLAY)
+#define TAS675X_GPIO_IN_SLEEP                (TAS675X_GPIO_FUNC_INPUT | TAS675X_GPIO_IN_ID_SLEEP)
+
+/* GPIO input 3-bit mux field masks */
+#define TAS675X_GPIO_IN_MUTE_MASK            GENMASK(2, 0)
+#define TAS675X_GPIO_IN_SYNC_MASK            GENMASK(2, 0)
+#define TAS675X_GPIO_IN_SDIN2_MASK           GENMASK(6, 4)
+#define TAS675X_GPIO_IN_DEEP_SLEEP_MASK      GENMASK(6, 4)
+#define TAS675X_GPIO_IN_HIZ_MASK             GENMASK(2, 0)
+#define TAS675X_GPIO_IN_PLAY_MASK            GENMASK(6, 4)
+#define TAS675X_GPIO_IN_SLEEP_MASK           GENMASK(2, 0)
+
+/* Book addresses for tas675x_select_book() */
+#define TAS675X_BOOK_DEFAULT                 0x00
+#define TAS675X_BOOK_DSP                     0x8C
+
+/* DSP memory addresses (DSP Book) */
+#define TAS675X_DSP_PAGE_RTLDG               0x22
+#define TAS675X_DSP_RTLDG_OL_THRESH_REG      0x98
+#define TAS675X_DSP_RTLDG_SL_THRESH_REG      0x9C
+
+#define TAS675X_DSP_PARAM_ID_OL_THRESH       0
+#define TAS675X_DSP_PARAM_ID_SL_THRESH       1
+
+/* Setup Mode Entry/Exit*/
+#define TAS675X_SETUP_ENTER_VAL1             0x11
+#define TAS675X_SETUP_ENTER_VAL2             0xFF
+#define TAS675X_SETUP_EXIT_VAL               0x00
+
+#endif /* __TAS675X_H__ */
diff --git a/sound/soc/codecs/tlv320aic3x.c b/sound/soc/codecs/tlv320aic3x.c
index ce22298b43efc..b12c1952823ba 100644
--- a/sound/soc/codecs/tlv320aic3x.c
+++ b/sound/soc/codecs/tlv320aic3x.c
@@ -1689,8 +1689,15 @@ static int aic3x_component_probe(struct snd_soc_component *component)
 	return 0;
 }
 
+static int aic3x_of_xlate_dai_id(struct snd_soc_component *component,
+				   struct device_node *endpoint)
+{
+	return 0;
+}
+
 static const struct snd_soc_component_driver soc_component_dev_aic3x = {
 	.set_bias_level		= aic3x_set_bias_level,
+	.of_xlate_dai_id	= aic3x_of_xlate_dai_id,
 	.probe			= aic3x_component_probe,
 	.controls		= aic3x_snd_controls,
 	.num_controls		= ARRAY_SIZE(aic3x_snd_controls),
diff --git a/sound/soc/codecs/uda1380.c b/sound/soc/codecs/uda1380.c
index 55c83d95bfba8..4df2d01cd5fbd 100644
--- a/sound/soc/codecs/uda1380.c
+++ b/sound/soc/codecs/uda1380.c
@@ -38,9 +38,9 @@ struct uda1380_priv {
 	unsigned int dac_clk;
 	struct work_struct work;
 	struct i2c_client *i2c;
-	u16 *reg_cache;
 	struct gpio_desc *power;
 	struct gpio_desc *reset;
+	u16 reg_cache[];
 };
 
 /*
@@ -767,11 +767,15 @@ static int uda1380_i2c_probe(struct i2c_client *i2c)
 	struct uda1380_priv *uda1380;
 	int ret;
 
-	uda1380 = devm_kzalloc(&i2c->dev, sizeof(struct uda1380_priv),
+	uda1380 = devm_kzalloc(&i2c->dev,
+			       struct_size(uda1380, reg_cache, ARRAY_SIZE(uda1380_reg)),
 			       GFP_KERNEL);
 	if (uda1380 == NULL)
 		return -ENOMEM;
 
+	memcpy(uda1380->reg_cache, uda1380_reg,
+	       ARRAY_SIZE(uda1380_reg) * sizeof(*uda1380->reg_cache));
+
 	uda1380->reset = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
 	if (IS_ERR(uda1380->reset))
 		return dev_err_probe(dev, PTR_ERR(uda1380->reset),
@@ -789,11 +793,6 @@ static int uda1380_i2c_probe(struct i2c_client *i2c)
 	if (device_property_match_string(dev, "dac-clk", "wspll") >= 0)
 		uda1380->dac_clk = UDA1380_DAC_CLK_WSPLL;
 
-	uda1380->reg_cache = devm_kmemdup_array(&i2c->dev, uda1380_reg, ARRAY_SIZE(uda1380_reg),
-						sizeof(uda1380_reg[0]), GFP_KERNEL);
-	if (!uda1380->reg_cache)
-		return -ENOMEM;
-
 	i2c_set_clientdata(i2c, uda1380);
 	uda1380->i2c = i2c;
 
diff --git a/sound/soc/codecs/wsa881x.c b/sound/soc/codecs/wsa881x.c
index 2fc234adca5f7..d15fda648dada 100644
--- a/sound/soc/codecs/wsa881x.c
+++ b/sound/soc/codecs/wsa881x.c
@@ -3,7 +3,6 @@
 // Copyright (c) 2019, Linaro Limited
 
 #include <linux/bitops.h>
-#include <linux/gpio.h>
 #include <linux/gpio/consumer.h>
 #include <linux/module.h>
 #include <linux/regmap.h>
@@ -672,11 +671,6 @@ struct wsa881x_priv {
 	struct sdw_stream_runtime *sruntime;
 	struct sdw_port_config port_config[WSA881X_MAX_SWR_PORTS];
 	struct gpio_desc *sd_n;
-	/*
-	 * Logical state for SD_N GPIO: high for shutdown, low for enable.
-	 * For backwards compatibility.
-	 */
-	unsigned int sd_n_val;
 	int active_ports;
 	bool hw_init;
 	bool port_prepared[WSA881X_MAX_SWR_PORTS];
@@ -1121,31 +1115,11 @@ static int wsa881x_probe(struct sdw_slave *pdev,
 	if (!wsa881x)
 		return -ENOMEM;
 
-	wsa881x->sd_n = devm_gpiod_get_optional(dev, "powerdown", 0);
+	wsa881x->sd_n = devm_gpiod_get_optional(dev, "powerdown", GPIOD_OUT_LOW);
 	if (IS_ERR(wsa881x->sd_n))
 		return dev_err_probe(dev, PTR_ERR(wsa881x->sd_n),
 				     "Shutdown Control GPIO not found\n");
 
-	/*
-	 * Backwards compatibility work-around.
-	 *
-	 * The SD_N GPIO is active low, however upstream DTS used always active
-	 * high.  Changing the flag in driver and DTS will break backwards
-	 * compatibility, so add a simple value inversion to work with both old
-	 * and new DTS.
-	 *
-	 * This won't work properly with DTS using the flags properly in cases:
-	 * 1. Old DTS with proper ACTIVE_LOW, however such case was broken
-	 *    before as the driver required the active high.
-	 * 2. New DTS with proper ACTIVE_HIGH (intended), which is rare case
-	 *    (not existing upstream) but possible. This is the price of
-	 *    backwards compatibility, therefore this hack should be removed at
-	 *    some point.
-	 */
-	wsa881x->sd_n_val = gpiod_is_active_low(wsa881x->sd_n);
-	if (!wsa881x->sd_n_val)
-		dev_warn(dev, "Using ACTIVE_HIGH for shutdown GPIO. Your DTB might be outdated or you use unsupported configuration for the GPIO.");
-
 	dev_set_drvdata(dev, wsa881x);
 	wsa881x->slave = pdev;
 	wsa881x->dev = dev;
@@ -1158,7 +1132,6 @@ static int wsa881x_probe(struct sdw_slave *pdev,
 	pdev->prop.sink_dpn_prop = wsa_sink_dpn_prop;
 	pdev->prop.scp_int1_mask = SDW_SCP_INT1_BUS_CLASH | SDW_SCP_INT1_PARITY;
 	pdev->prop.clk_stop_mode1 = true;
-	gpiod_direction_output(wsa881x->sd_n, !wsa881x->sd_n_val);
 
 	wsa881x->regmap = devm_regmap_init_sdw(pdev, &wsa881x_regmap_config);
 	if (IS_ERR(wsa881x->regmap))
@@ -1181,7 +1154,7 @@ static int wsa881x_runtime_suspend(struct device *dev)
 	struct regmap *regmap = dev_get_regmap(dev, NULL);
 	struct wsa881x_priv *wsa881x = dev_get_drvdata(dev);
 
-	gpiod_direction_output(wsa881x->sd_n, wsa881x->sd_n_val);
+	gpiod_direction_output(wsa881x->sd_n, 1);
 
 	regcache_cache_only(regmap, true);
 	regcache_mark_dirty(regmap);
@@ -1196,13 +1169,13 @@ static int wsa881x_runtime_resume(struct device *dev)
 	struct wsa881x_priv *wsa881x = dev_get_drvdata(dev);
 	unsigned long time;
 
-	gpiod_direction_output(wsa881x->sd_n, !wsa881x->sd_n_val);
+	gpiod_direction_output(wsa881x->sd_n, 0);
 
 	time = wait_for_completion_timeout(&slave->initialization_complete,
 					   msecs_to_jiffies(WSA881X_PROBE_TIMEOUT));
 	if (!time) {
 		dev_err(dev, "Initialization not complete, timed out\n");
-		gpiod_direction_output(wsa881x->sd_n, wsa881x->sd_n_val);
+		gpiod_direction_output(wsa881x->sd_n, 1);
 		return -ETIMEDOUT;
 	}
 
diff --git a/sound/soc/fsl/Kconfig b/sound/soc/fsl/Kconfig
index 828524c90f171..ae8774e11873d 100644
--- a/sound/soc/fsl/Kconfig
+++ b/sound/soc/fsl/Kconfig
@@ -165,7 +165,7 @@ config SND_SOC_IMX_AUDMUX
 
 config SND_POWERPC_SOC
 	tristate "SoC Audio for Freescale PowerPC CPUs"
-	depends on FSL_SOC || PPC_MPC52xx
+	depends on FSL_SOC || PPC_MPC52xx || (PPC && COMPILE_TEST)
 	help
 	  Say Y or M if you want to add support for codecs attached to
 	  the PowerPC CPUs.
@@ -226,7 +226,7 @@ config SND_SOC_P1022_RDK
 
 config SND_SOC_MPC5200_I2S
 	tristate "Freescale MPC5200 PSC in I2S mode driver"
-	depends on PPC_MPC52xx && PPC_BESTCOMM
+	depends on PPC_BESTCOMM
 	select SND_MPC52xx_DMA
 	select PPC_BESTCOMM_GEN_BD
 	help
@@ -234,7 +234,7 @@ config SND_SOC_MPC5200_I2S
 
 config SND_SOC_MPC5200_AC97
 	tristate "Freescale MPC5200 PSC in AC97 mode driver"
-	depends on PPC_MPC52xx && PPC_BESTCOMM
+	depends on PPC_BESTCOMM
 	select SND_SOC_AC97_BUS
 	select SND_MPC52xx_DMA
 	select PPC_BESTCOMM_GEN_BD
diff --git a/sound/soc/fsl/eukrea-tlv320.c b/sound/soc/fsl/eukrea-tlv320.c
index 6be074ea0b3f0..5bb31a5cdf23c 100644
--- a/sound/soc/fsl/eukrea-tlv320.c
+++ b/sound/soc/fsl/eukrea-tlv320.c
@@ -19,7 +19,6 @@
 #include <sound/core.h>
 #include <sound/pcm.h>
 #include <sound/soc.h>
-#include <asm/mach-types.h>
 
 #include "../codecs/tlv320aic23.h"
 #include "imx-ssi.h"
@@ -142,7 +141,7 @@ static int eukrea_tlv320_probe(struct platform_device *pdev)
 		eukrea_tlv320.name = "cpuimx-audio";
 	}
 
-	if (machine_is_eukrea_cpuimx27() ||
+	if (of_machine_is_compatible("eukrea,cpuimx27") ||
 	    (tmp_np = of_find_compatible_node(NULL, NULL, "fsl,imx21-audmux"))) {
 		imx_audmux_v1_configure_port(MX27_AUDMUX_HPCR1_SSI0,
 			IMX_AUDMUX_V1_PCR_SYN |
@@ -159,12 +158,12 @@ static int eukrea_tlv320_probe(struct platform_device *pdev)
 			IMX_AUDMUX_V1_PCR_RXDSEL(MX27_AUDMUX_HPCR1_SSI0)
 		);
 		of_node_put(tmp_np);
-	} else if (machine_is_eukrea_cpuimx25sd() ||
-		   machine_is_eukrea_cpuimx35sd() ||
-		   machine_is_eukrea_cpuimx51sd() ||
+	} else if (of_machine_is_compatible("eukrea,cpuimx25") ||
+		   of_machine_is_compatible("eukrea,cpuimx35") ||
+		   of_machine_is_compatible("eukrea,cpuimx51") ||
 		   (tmp_np = of_find_compatible_node(NULL, NULL, "fsl,imx31-audmux"))) {
 		if (!np)
-			ext_port = machine_is_eukrea_cpuimx25sd() ?
+			ext_port = of_machine_is_compatible("eukrea,cpuimx25") ?
 				4 : 3;
 
 		imx_audmux_v2_configure_port(int_port,
diff --git a/sound/soc/fsl/fsl-asoc-card.c b/sound/soc/fsl/fsl-asoc-card.c
index 70a6159430ed3..709543308fe9e 100644
--- a/sound/soc/fsl/fsl-asoc-card.c
+++ b/sound/soc/fsl/fsl-asoc-card.c
@@ -40,6 +40,33 @@
 /* Default DAI format without Master and Slave flag */
 #define DAI_FMT_BASE (SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF)
 
+static const u32 cs42888_rates_48k[] = {
+	48000, 96000, 192000,
+};
+
+static const u32 cs42888_rates_44k[] = {
+	44100, 88200, 176400,
+};
+
+static const u32 cs42888_channels[] = {
+	1, 2, 4, 6, 8,
+};
+
+static const struct snd_pcm_hw_constraint_list cs42888_rate_48k_constraints = {
+	.list = cs42888_rates_48k,
+	.count = ARRAY_SIZE(cs42888_rates_48k),
+};
+
+static const struct snd_pcm_hw_constraint_list cs42888_rate_44k_constraints = {
+	.list = cs42888_rates_44k,
+	.count = ARRAY_SIZE(cs42888_rates_44k),
+};
+
+static const struct snd_pcm_hw_constraint_list cs42888_channel_constraints = {
+	.list = cs42888_channels,
+	.count = ARRAY_SIZE(cs42888_channels),
+};
+
 /**
  * struct codec_priv - CODEC private data
  * @mclk: Main clock of the CODEC
@@ -48,6 +75,9 @@
  * @mclk_id: MCLK (or main clock) id for set_sysclk()
  * @fll_id: FLL (or secordary clock) id for set_sysclk()
  * @pll_id: PLL id for set_pll()
+ * @pll_ratio_s24: PLL output ratio for S24_LE format (PLL_freq = sample_rate × ratio)
+ *                 Default is 384, but some codecs (e.g., WM8904) require lower values
+ *                 to stay within PLL frequency limits
  */
 struct codec_priv {
 	struct clk *mclk;
@@ -56,6 +86,7 @@ struct codec_priv {
 	u32 mclk_id;
 	int fll_id;
 	int pll_id;
+	int pll_ratio_s24;
 };
 
 /**
@@ -87,12 +118,15 @@ struct cpu_priv {
  * @codec_priv: CODEC private data
  * @cpu_priv: CPU private data
  * @card: ASoC card structure
+ * @constraint_rates: array of supported rates
+ * @constraint_channels: array of supported channels
  * @streams: Mask of current active streams
  * @sample_rate: Current sample rate
  * @sample_format: Current sample format
  * @asrc_rate: ASRC sample rate used by Back-Ends
  * @asrc_format: ASRC sample format used by Back-Ends
  * @dai_fmt: DAI format between CPU and CODEC
+ * @exclude_format: excluded format;
  * @name: Card name
  */
 
@@ -104,12 +138,15 @@ struct fsl_asoc_card_priv {
 	struct codec_priv codec_priv[2];
 	struct cpu_priv cpu_priv;
 	struct snd_soc_card card;
+	const struct snd_pcm_hw_constraint_list *constraint_rates;
+	const struct snd_pcm_hw_constraint_list *constraint_channels;
 	u8 streams;
 	u32 sample_rate;
 	snd_pcm_format_t sample_format;
 	u32 asrc_rate;
 	snd_pcm_format_t asrc_format;
 	u32 dai_fmt;
+	u64 exclude_format;
 	char name[32];
 };
 
@@ -222,7 +259,7 @@ static int fsl_asoc_card_hw_params(struct snd_pcm_substream *substream,
 
 		if (codec_priv->pll_id >= 0 && codec_priv->fll_id >= 0) {
 			if (priv->sample_format == SNDRV_PCM_FORMAT_S24_LE)
-				pll_out = priv->sample_rate * 384;
+				pll_out = priv->sample_rate * codec_priv->pll_ratio_s24;
 			else
 				pll_out = priv->sample_rate * 256;
 
@@ -291,7 +328,47 @@ static int fsl_asoc_card_hw_free(struct snd_pcm_substream *substream)
 	return 0;
 }
 
+static int fsl_asoc_card_startup(struct snd_pcm_substream *substream)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct fsl_asoc_card_priv *priv = snd_soc_card_get_drvdata(rtd->card);
+	struct snd_pcm_runtime *runtime = substream->runtime;
+	int ret;
+
+	if (priv->exclude_format && !rtd->dai_link->no_pcm) {
+		ret = snd_pcm_hw_constraint_mask64(runtime,
+						   SNDRV_PCM_HW_PARAM_FORMAT,
+						   ~priv->exclude_format);
+		if (ret)
+			return ret;
+	}
+
+	if (priv->constraint_channels) {
+		ret = snd_pcm_hw_constraint_list(runtime, 0,
+						 SNDRV_PCM_HW_PARAM_CHANNELS,
+						 priv->constraint_channels);
+		if (ret)
+			return ret;
+	}
+
+	/*
+	 * Apply rate constraints only to frontend DAI links (no_pcm = 0).
+	 * Skip DPCM backend (no_pcm = 1) as rate is fixed by be_hw_params_fixup()
+	 * and ASRC frontend handles rate conversion.
+	 */
+	if (priv->constraint_rates && !rtd->dai_link->no_pcm) {
+		ret = snd_pcm_hw_constraint_list(runtime, 0,
+						 SNDRV_PCM_HW_PARAM_RATE,
+						 priv->constraint_rates);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
 static const struct snd_soc_ops fsl_asoc_card_ops = {
+	.startup = fsl_asoc_card_startup,
 	.hw_params = fsl_asoc_card_hw_params,
 	.hw_free = fsl_asoc_card_hw_free,
 };
@@ -742,6 +819,7 @@ static int fsl_asoc_card_probe(struct platform_device *pdev)
 	for (codec_idx = 0; codec_idx < 2; codec_idx++) {
 		priv->codec_priv[codec_idx].fll_id = -1;
 		priv->codec_priv[codec_idx].pll_id = -1;
+		priv->codec_priv[codec_idx].pll_ratio_s24 = 384;
 	}
 
 	/* Diversify the card configurations */
@@ -753,6 +831,14 @@ static int fsl_asoc_card_probe(struct platform_device *pdev)
 		priv->cpu_priv.sysclk_dir[RX] = SND_SOC_CLOCK_OUT;
 		priv->cpu_priv.slot_width = 32;
 		priv->dai_fmt |= SND_SOC_DAIFMT_CBC_CFC;
+		priv->constraint_channels = &cs42888_channel_constraints;
+		if (priv->codec_priv[0].mclk_freq % 12288000 == 0)
+			priv->constraint_rates  = &cs42888_rate_48k_constraints;
+		else if (priv->codec_priv[0].mclk_freq % 11289600 == 0)
+			priv->constraint_rates = &cs42888_rate_44k_constraints;
+		else
+			dev_warn(&pdev->dev, "Unknown MCLK frequency %lu, no rate constraints\n",
+				 priv->codec_priv[0].mclk_freq);
 	} else if (of_device_is_compatible(np, "fsl,imx-audio-cs427x")) {
 		codec_dai_name[0] = "cs4271-hifi";
 		priv->codec_priv[0].mclk_id = CS427x_SYSCLK_MCLK;
@@ -779,11 +865,30 @@ static int fsl_asoc_card_probe(struct platform_device *pdev)
 		priv->codec_priv[0].fll_id = WM8962_SYSCLK_FLL;
 		priv->codec_priv[0].pll_id = WM8962_FLL;
 		priv->dai_fmt |= SND_SOC_DAIFMT_CBP_CFP;
+		/*
+		 * WM8962 has same BCLK generation limitations as WM8960.
+		 * See WM8960 section for detailed explanation.
+		 */
+		if (of_node_name_eq(cpu_np, "sai"))
+			priv->exclude_format = SNDRV_PCM_FMTBIT_S20_3LE;
 	} else if (of_device_is_compatible(np, "fsl,imx-audio-wm8960")) {
 		codec_dai_name[0] = "wm8960-hifi";
 		priv->codec_priv[0].fll_id = WM8960_SYSCLK_AUTO;
 		priv->codec_priv[0].pll_id = WM8960_SYSCLK_AUTO;
 		priv->dai_fmt |= SND_SOC_DAIFMT_CBP_CFP;
+		/*
+		 * WM8960 in master mode cannot generate exact 1.92 MHz BCLK
+		 * required for S20_3LE (48kHz × 2ch × 20bit). Closest available
+		 * is 2.048 MHz (SYSCLK/6), which causes right channel corruption.
+		 *
+		 * In SAI master mode, SAI derive BCLK from MCLK using integer
+		 * dividers only. S20_3LE requires non-integer divider ratios
+		 * with standard MCLK frequencies. For example, 48kHz stereo
+		 * needs 1.920 MHz BCLK, which requires a divider of 6.4 from
+		 * 12.288 MHz MCLK (not an integer).
+		 */
+		if (of_node_name_eq(cpu_np, "sai"))
+			priv->exclude_format = SNDRV_PCM_FMTBIT_S20_3LE;
 	} else if (of_device_is_compatible(np, "fsl,imx-audio-ac97")) {
 		codec_dai_name[0] = "ac97-hifi";
 		priv->dai_fmt = SND_SOC_DAIFMT_AC97;
@@ -835,6 +940,7 @@ static int fsl_asoc_card_probe(struct platform_device *pdev)
 		priv->codec_priv[0].mclk_id = WM8904_FLL_MCLK;
 		priv->codec_priv[0].fll_id = WM8904_CLK_FLL;
 		priv->codec_priv[0].pll_id = WM8904_FLL_MCLK;
+		priv->codec_priv[0].pll_ratio_s24 = 192;
 		priv->dai_fmt |= SND_SOC_DAIFMT_CBP_CFP;
 	} else if (of_device_is_compatible(np, "fsl,imx-audio-spdif")) {
 		ret = fsl_asoc_card_spdif_init(codec_np, cpu_np, codec_dai_name, priv);
@@ -989,6 +1095,8 @@ static int fsl_asoc_card_probe(struct platform_device *pdev)
 
 	if (asrc_pdev) {
 		/* DPCM DAI Links only if ASRC exists */
+		priv->dai_link[1].dpcm_merged_chan = 1;
+		priv->dai_link[1].ignore_pmdown_time = 1;
 		priv->dai_link[1].cpus->of_node = asrc_np;
 		priv->dai_link[1].platforms->of_node = asrc_np;
 		for_each_link_codecs((&(priv->dai_link[2])), codec_idx, codec_comp) {
@@ -998,6 +1106,7 @@ static int fsl_asoc_card_probe(struct platform_device *pdev)
 		}
 		priv->dai_link[2].cpus->of_node = cpu_np;
 		priv->dai_link[2].dai_fmt = priv->dai_fmt;
+		priv->dai_link[2].ignore_pmdown_time = 1;
 		priv->card.num_links = 3;
 
 		ret = of_property_read_u32(asrc_np, "fsl,asrc-rate",
diff --git a/sound/soc/fsl/fsl_micfil.c b/sound/soc/fsl/fsl_micfil.c
index 2e887f1f1f361..60ac8eabab9da 100644
--- a/sound/soc/fsl/fsl_micfil.c
+++ b/sound/soc/fsl/fsl_micfil.c
@@ -74,6 +74,7 @@ struct fsl_micfil {
 	int irq[MICFIL_IRQ_LINES];
 	enum quality quality;
 	int dc_remover;
+	int dc_out_remover;
 	int vad_init_mode;
 	int vad_enabled;
 	int vad_detected;
@@ -347,6 +348,11 @@ static const char * const micfil_dc_remover_texts[] = {
 	"Cut-off @152Hz", "Bypass",
 };
 
+static const char * const micfil_dc_out_remover_texts[] = {
+	"Cut-off @20Hz", "Cut-off @13.3Hz",
+	"Cut-off @40Hz", "Bypass",
+};
+
 static const struct soc_enum hwvad_enable_enum =
 	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(micfil_hwvad_enable),
 			    micfil_hwvad_enable);
@@ -360,6 +366,9 @@ static const struct soc_enum hwvad_hpf_enum =
 static const struct soc_enum fsl_micfil_dc_remover_enum =
 	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(micfil_dc_remover_texts),
 			    micfil_dc_remover_texts);
+static const struct soc_enum fsl_micfil_dc_out_remover_enum =
+	SOC_ENUM_SINGLE_EXT(ARRAY_SIZE(micfil_dc_out_remover_texts),
+			    micfil_dc_out_remover_texts);
 
 static int micfil_put_dc_remover_state(struct snd_kcontrol *kcontrol,
 				       struct snd_ctl_elem_value *ucontrol)
@@ -405,6 +414,50 @@ static int micfil_get_dc_remover_state(struct snd_kcontrol *kcontrol,
 	return 0;
 }
 
+static int micfil_put_dc_out_remover_state(struct snd_kcontrol *kcontrol,
+					   struct snd_ctl_elem_value *ucontrol)
+{
+	struct soc_enum *e = (struct soc_enum *)kcontrol->private_value;
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct fsl_micfil *micfil = snd_soc_component_get_drvdata(comp);
+	unsigned int *item = ucontrol->value.enumerated.item;
+	int val = snd_soc_enum_item_to_val(e, item[0]);
+	int i = 0, ret = 0;
+	u32 reg_val = 0;
+
+	if (val < 0 || val > 3)
+		return -EINVAL;
+
+	ret = pm_runtime_resume_and_get(comp->dev);
+	if (ret)
+		return ret;
+
+	micfil->dc_out_remover = val;
+
+	/* Calculate total value for all channels */
+	for (i = 0; i < MICFIL_OUTPUT_CHANNELS; i++)
+		reg_val |= val << MICFIL_DC_CHX_SHIFT(i);
+
+	/* Update DC Remover mode for all channels */
+	ret = snd_soc_component_update_bits(comp, REG_MICFIL_DC_OUT_CTRL,
+					    MICFIL_DC_CTRL_CONFIG, reg_val);
+
+	pm_runtime_put_autosuspend(comp->dev);
+
+	return ret;
+}
+
+static int micfil_get_dc_out_remover_state(struct snd_kcontrol *kcontrol,
+					   struct snd_ctl_elem_value *ucontrol)
+{
+	struct snd_soc_component *comp = snd_kcontrol_chip(kcontrol);
+	struct fsl_micfil *micfil = snd_soc_component_get_drvdata(comp);
+
+	ucontrol->value.enumerated.item[0] = micfil->dc_out_remover;
+
+	return 0;
+}
+
 static int hwvad_put_enable(struct snd_kcontrol *kcontrol,
 			    struct snd_ctl_elem_value *ucontrol)
 {
@@ -525,6 +578,11 @@ static const struct snd_kcontrol_new fsl_micfil_volume_sx_controls[] = {
 			  MICFIL_OUTGAIN_CHX_SHIFT(7), 0x8, 0xF, gain_tlv),
 };
 
+static const struct snd_kcontrol_new fsl_micfil_dc_out_controls[] = {
+	SOC_ENUM_EXT("MICFIL DC Out Remover Control", fsl_micfil_dc_out_remover_enum,
+		     micfil_get_dc_out_remover_state, micfil_put_dc_out_remover_state),
+};
+
 static const struct snd_kcontrol_new fsl_micfil_snd_controls[] = {
 	SOC_ENUM_EXT("MICFIL Quality Select",
 		     fsl_micfil_quality_enum,
@@ -1047,6 +1105,19 @@ static int fsl_micfil_dai_probe(struct snd_soc_dai *cpu_dai)
 	}
 	micfil->dc_remover = MICFIL_DC_BYPASS;
 
+	if (micfil->soc->use_verid) {
+		val = 0;
+		for (i = 0; i < MICFIL_OUTPUT_CHANNELS; i++)
+			val |= MICFIL_DC_BYPASS << MICFIL_DC_CHX_SHIFT(i);
+		ret = regmap_update_bits(micfil->regmap, REG_MICFIL_DC_OUT_CTRL,
+					 MICFIL_DC_CTRL_CONFIG, val);
+		if (ret) {
+			dev_err(dev, "failed to set DC OUT Remover mode bits\n");
+			return ret;
+		}
+		micfil->dc_out_remover = MICFIL_DC_BYPASS;
+	}
+
 	snd_soc_dai_init_dma_data(cpu_dai, NULL,
 				  &micfil->dma_params_rx);
 
@@ -1071,6 +1142,10 @@ static int fsl_micfil_component_probe(struct snd_soc_component *component)
 		snd_soc_add_component_controls(component, fsl_micfil_range_controls,
 					       ARRAY_SIZE(fsl_micfil_range_controls));
 
+	if (micfil->soc->use_verid)
+		snd_soc_add_component_controls(component, fsl_micfil_dc_out_controls,
+					       ARRAY_SIZE(fsl_micfil_dc_out_controls));
+
 	return 0;
 }
 
@@ -1117,6 +1192,7 @@ static const struct reg_default fsl_micfil_reg_defaults[] = {
 	{REG_MICFIL_DATACH6,		0x00000000},
 	{REG_MICFIL_DATACH7,		0x00000000},
 	{REG_MICFIL_DC_CTRL,		0x00000000},
+	{REG_MICFIL_DC_OUT_CTRL,	0x00000000},
 	{REG_MICFIL_OUT_CTRL,		0x00000000},
 	{REG_MICFIL_OUT_STAT,		0x00000000},
 	{REG_MICFIL_VAD0_CTRL1,		0x00000000},
@@ -1143,6 +1219,7 @@ static const struct reg_default fsl_micfil_reg_defaults_v2[] = {
 	{REG_MICFIL_DATACH6 - 0x4,	0x00000000},
 	{REG_MICFIL_DATACH7 - 0x4,	0x00000000},
 	{REG_MICFIL_DC_CTRL,		0x00000000},
+	{REG_MICFIL_DC_OUT_CTRL,	0x00000000},
 	{REG_MICFIL_OUT_CTRL,		0x00000000},
 	{REG_MICFIL_OUT_STAT,		0x00000000},
 	{REG_MICFIL_VAD0_CTRL1,		0x00000000},
@@ -1179,6 +1256,7 @@ static bool fsl_micfil_readable_reg(struct device *dev, unsigned int reg)
 	case REG_MICFIL_VAD0_NDATA:
 	case REG_MICFIL_VAD0_ZCD:
 		return true;
+	case REG_MICFIL_DC_OUT_CTRL:
 	case REG_MICFIL_FSYNC_CTRL:
 	case REG_MICFIL_VERID:
 	case REG_MICFIL_PARAM:
@@ -1210,6 +1288,7 @@ static bool fsl_micfil_writeable_reg(struct device *dev, unsigned int reg)
 	case REG_MICFIL_VAD0_NCONFIG:
 	case REG_MICFIL_VAD0_ZCD:
 		return true;
+	case REG_MICFIL_DC_OUT_CTRL:
 	case REG_MICFIL_FSYNC_CTRL:
 		if (micfil->soc->use_verid)
 			return true;
diff --git a/sound/soc/fsl/fsl_micfil.h b/sound/soc/fsl/fsl_micfil.h
index fdfe4e7125bc5..e271c6073f42e 100644
--- a/sound/soc/fsl/fsl_micfil.h
+++ b/sound/soc/fsl/fsl_micfil.h
@@ -22,6 +22,7 @@
 #define REG_MICFIL_DATACH6		0x3C
 #define REG_MICFIL_DATACH7		0x40
 #define REG_MICFIL_DC_CTRL		0x64
+#define REG_MICFIL_DC_OUT_CTRL		0x68
 #define REG_MICFIL_OUT_CTRL		0x74
 #define REG_MICFIL_OUT_STAT		0x7C
 #define REG_MICFIL_FSYNC_CTRL		0x80
diff --git a/sound/soc/fsl/fsl_sai.c b/sound/soc/fsl/fsl_sai.c
index bd336d2e4cb38..d6dd95680892b 100644
--- a/sound/soc/fsl/fsl_sai.c
+++ b/sound/soc/fsl/fsl_sai.c
@@ -355,6 +355,9 @@ static int fsl_sai_set_dai_fmt_tr(struct snd_soc_dai *cpu_dai,
 	unsigned int ofs = sai->soc_data->reg_offset;
 	u32 val_cr2 = 0, val_cr4 = 0;
 
+	if (sai->is_bit_clock_swap)
+		val_cr2 |= FSL_SAI_CR2_BCS;
+
 	if (!sai->is_lsb_first)
 		val_cr4 |= FSL_SAI_CR4_MF;
 
@@ -453,7 +456,8 @@ static int fsl_sai_set_dai_fmt_tr(struct snd_soc_dai *cpu_dai,
 	}
 
 	regmap_update_bits(sai->regmap, FSL_SAI_xCR2(tx, ofs),
-			   FSL_SAI_CR2_BCP | FSL_SAI_CR2_BCD_MSTR, val_cr2);
+			   FSL_SAI_CR2_BCS | FSL_SAI_CR2_BCP | FSL_SAI_CR2_BCD_MSTR,
+			   val_cr2);
 	regmap_update_bits(sai->regmap, FSL_SAI_xCR4(tx, ofs),
 			   FSL_SAI_CR4_MF | FSL_SAI_CR4_FSE |
 			   FSL_SAI_CR4_FSP | FSL_SAI_CR4_FSD_MSTR, val_cr4);
@@ -1370,6 +1374,31 @@ static int fsl_sai_check_version(struct device *dev)
 	return 0;
 }
 
+static int fsl_sai_reset_hw(struct device *dev)
+{
+	struct fsl_sai *sai = dev_get_drvdata(dev);
+	unsigned char ofs = sai->soc_data->reg_offset;
+	int ret;
+
+	/*
+	 * Clear TCSR/RCSR to reset SAI and disable all interrupts.
+	 * Bootloader may leave SAI running causing interrupt storm.
+	 */
+	ret = regmap_write(sai->regmap, FSL_SAI_TCSR(ofs), 0);
+	if (ret) {
+		dev_err(dev, "Failed to clear TCSR: %d\n", ret);
+		return ret;
+	}
+
+	ret = regmap_write(sai->regmap, FSL_SAI_RCSR(ofs), 0);
+	if (ret) {
+		dev_err(dev, "Failed to clear RCSR: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 /*
  * Calculate the offset between first two datalines, don't
  * different offset in one case.
@@ -1507,6 +1536,7 @@ static int fsl_sai_probe(struct platform_device *pdev)
 	sai->soc_data = of_device_get_match_data(dev);
 
 	sai->is_lsb_first = of_property_read_bool(np, "lsb-first");
+	sai->is_bit_clock_swap = of_property_read_bool(np, "fsl,sai-bit-clock-swap");
 
 	base = devm_platform_get_and_ioremap_resource(pdev, 0, &sai->res);
 	if (IS_ERR(base))
@@ -1575,13 +1605,6 @@ static int fsl_sai_probe(struct platform_device *pdev)
 	if (irq < 0)
 		return irq;
 
-	ret = devm_request_irq(dev, irq, fsl_sai_isr, IRQF_SHARED,
-			       np->name, sai);
-	if (ret) {
-		dev_err(dev, "failed to claim irq %u\n", irq);
-		return ret;
-	}
-
 	memcpy(&sai->cpu_dai_drv, fsl_sai_dai_template,
 	       sizeof(*fsl_sai_dai_template) * ARRAY_SIZE(fsl_sai_dai_template));
 
@@ -1656,6 +1679,10 @@ static int fsl_sai_probe(struct platform_device *pdev)
 	if (ret < 0)
 		dev_warn(dev, "Error reading SAI version: %d\n", ret);
 
+	ret = fsl_sai_reset_hw(dev);
+	if (ret < 0)
+		dev_warn(dev, "Failed to reset hardware: %d\n", ret);
+
 	/* Select MCLK direction */
 	if (sai->mclk_direction_output &&
 	    sai->soc_data->max_register >= FSL_SAI_MCTL) {
@@ -1667,6 +1694,13 @@ static int fsl_sai_probe(struct platform_device *pdev)
 	if (ret < 0 && ret != -ENOSYS)
 		goto err_pm_get_sync;
 
+	ret = devm_request_irq(dev, irq, fsl_sai_isr, IRQF_SHARED,
+			       np->name, sai);
+	if (ret) {
+		dev_err(dev, "failed to claim irq %u\n", irq);
+		goto err_pm_get_sync;
+	}
+
 	if (of_device_is_compatible(np, "fsl,imx952-sai") &&
 	    !of_property_read_string(np, "fsl,sai-amix-mode", &str)) {
 		if (!strcmp(str, "bypass"))
diff --git a/sound/soc/fsl/fsl_sai.h b/sound/soc/fsl/fsl_sai.h
index af967833b6eda..6d84e5ff2258e 100644
--- a/sound/soc/fsl/fsl_sai.h
+++ b/sound/soc/fsl/fsl_sai.h
@@ -118,6 +118,7 @@
 
 /* SAI Transmit and Receive Configuration 2 Register */
 #define FSL_SAI_CR2_SYNC	BIT(30)
+#define FSL_SAI_CR2_BCS		BIT(29)
 #define FSL_SAI_CR2_BCI		BIT(28)
 #define FSL_SAI_CR2_MSEL_MASK	(0x3 << 26)
 #define FSL_SAI_CR2_MSEL_BUS	0
@@ -301,6 +302,7 @@ struct fsl_sai {
 	struct fsl_sai_dl_cfg *dl_cfg;
 	unsigned int dl_cfg_cnt;
 	bool mclk_direction_output;
+	bool is_bit_clock_swap;
 
 	unsigned int mclk_id[2];
 	unsigned int mclk_streams;
diff --git a/sound/soc/intel/boards/Kconfig b/sound/soc/intel/boards/Kconfig
index d53af8f7e55b8..cddbd2aa424eb 100644
--- a/sound/soc/intel/boards/Kconfig
+++ b/sound/soc/intel/boards/Kconfig
@@ -532,6 +532,7 @@ config SND_SOC_INTEL_SOUNDWIRE_SOF_MACH
 	select MFD_CS42L43_SDW
 	select SND_SOC_CS35L56_SPI
 	select SND_SOC_CS35L56_SDW
+	select SND_SOC_ES9356
 	select SND_SOC_DMIC
 	select SND_SOC_INTEL_HDA_DSP_COMMON
 	imply SND_SOC_SDW_MOCKUP
diff --git a/sound/soc/intel/boards/cht_bsw_rt5672.c b/sound/soc/intel/boards/cht_bsw_rt5672.c
index fd4cefd298d21..eddb179eaa210 100644
--- a/sound/soc/intel/boards/cht_bsw_rt5672.c
+++ b/sound/soc/intel/boards/cht_bsw_rt5672.c
@@ -63,13 +63,11 @@ static int platform_clock_control(struct snd_soc_dapm_widget *w,
 	}
 
 	if (SND_SOC_DAPM_EVENT_ON(event)) {
-		if (ctx->mclk) {
-			ret = clk_prepare_enable(ctx->mclk);
-			if (ret < 0) {
-				dev_err(card->dev,
-					"could not configure MCLK state: %d\n", ret);
-				return ret;
-			}
+		ret = clk_prepare_enable(ctx->mclk);
+		if (ret < 0) {
+			dev_err(card->dev,
+				"could not configure MCLK state: %d\n", ret);
+			return ret;
 		}
 
 		/* set codec PLL source to the 19.2MHz platform clock (MCLK) */
@@ -77,8 +75,7 @@ static int platform_clock_control(struct snd_soc_dapm_widget *w,
 				CHT_PLAT_CLK_3_HZ, 48000 * 512);
 		if (ret < 0) {
 			dev_err(card->dev, "can't set codec pll: %d\n", ret);
-			if (ctx->mclk)
-				clk_disable_unprepare(ctx->mclk);
+			clk_disable_unprepare(ctx->mclk);
 			return ret;
 		}
 
@@ -87,8 +84,7 @@ static int platform_clock_control(struct snd_soc_dapm_widget *w,
 			48000 * 512, SND_SOC_CLOCK_IN);
 		if (ret < 0) {
 			dev_err(card->dev, "can't set codec sysclk: %d\n", ret);
-			if (ctx->mclk)
-				clk_disable_unprepare(ctx->mclk);
+			clk_disable_unprepare(ctx->mclk);
 			return ret;
 		}
 	} else {
@@ -104,8 +100,7 @@ static int platform_clock_control(struct snd_soc_dapm_widget *w,
 			return ret;
 		}
 
-		if (ctx->mclk)
-			clk_disable_unprepare(ctx->mclk);
+		clk_disable_unprepare(ctx->mclk);
 	}
 	return 0;
 }
@@ -244,28 +239,25 @@ static int cht_codec_init(struct snd_soc_pcm_runtime *runtime)
 	snd_jack_set_key(ctx->headset.jack, SND_JACK_BTN_2, KEY_VOLUMEDOWN);
 
 	rt5670_set_jack_detect(component, &ctx->headset);
-	if (ctx->mclk) {
-		/*
-		 * The firmware might enable the clock at
-		 * boot (this information may or may not
-		 * be reflected in the enable clock register).
-		 * To change the rate we must disable the clock
-		 * first to cover these cases. Due to common
-		 * clock framework restrictions that do not allow
-		 * to disable a clock that has not been enabled,
-		 * we need to enable the clock first.
-		 */
-		ret = clk_prepare_enable(ctx->mclk);
-		if (!ret)
-			clk_disable_unprepare(ctx->mclk);
 
-		ret = clk_set_rate(ctx->mclk, CHT_PLAT_CLK_3_HZ);
+	/*
+	 * The firmware might enable the clock at boot (this information
+	 * may or may not be reflected in the enable clock register).
+	 * To change the rate we must disable the clock first to cover
+	 * these cases. Due to Common Clock Framework restrictions that
+	 * do not allow to disable a clock that has not been enabled, we
+	 * need to enable the clock first.
+	 */
+	ret = clk_prepare_enable(ctx->mclk);
+	if (!ret)
+		clk_disable_unprepare(ctx->mclk);
 
-		if (ret) {
-			dev_err(runtime->dev, "unable to set MCLK rate\n");
-			return ret;
-		}
+	ret = clk_set_rate(ctx->mclk, CHT_PLAT_CLK_3_HZ);
+	if (ret) {
+		dev_err(runtime->dev, "unable to set MCLK rate\n");
+		return ret;
 	}
+
 	return 0;
 }
 
@@ -454,12 +446,13 @@ static int snd_cht_mc_probe(struct platform_device *pdev)
 	struct cht_mc_private *drv;
 	struct snd_soc_acpi_mach *mach = pdev->dev.platform_data;
 	const char *platform_name;
+	struct device *dev = &pdev->dev;
 	struct acpi_device *adev;
 	bool sof_parent;
 	int dai_index = 0;
 	int i;
 
-	drv = devm_kzalloc(&pdev->dev, sizeof(*drv), GFP_KERNEL);
+	drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);
 	if (!drv)
 		return -ENOMEM;
 
@@ -481,7 +474,7 @@ static int snd_cht_mc_probe(struct platform_device *pdev)
 			 "i2c-%s", acpi_dev_name(adev));
 		cht_dailink[dai_index].codecs->name = drv->codec_name;
 	}  else {
-		dev_err(&pdev->dev, "Error cannot find '%s' dev\n", mach->id);
+		dev_err(dev, "Error cannot find '%s' dev\n", mach->id);
 		return -ENOENT;
 	}
 
@@ -494,7 +487,7 @@ static int snd_cht_mc_probe(struct platform_device *pdev)
 	}
 
 	/* override platform name, if required */
-	snd_soc_card_cht.dev = &pdev->dev;
+	snd_soc_card_cht.dev = dev;
 	platform_name = mach->mach_params.platform;
 
 	ret_val = snd_soc_fixup_dai_links_platform_name(&snd_soc_card_cht,
@@ -504,16 +497,16 @@ static int snd_cht_mc_probe(struct platform_device *pdev)
 
 	snd_soc_card_cht.components = rt5670_components();
 
-	drv->mclk = devm_clk_get(&pdev->dev, "pmc_plt_clk_3");
+	drv->mclk = devm_clk_get(dev, "pmc_plt_clk_3");
 	if (IS_ERR(drv->mclk)) {
-		dev_err(&pdev->dev,
+		dev_err(dev,
 			"Failed to get MCLK from pmc_plt_clk_3: %ld\n",
 			PTR_ERR(drv->mclk));
 		return PTR_ERR(drv->mclk);
 	}
 	snd_soc_card_set_drvdata(&snd_soc_card_cht, drv);
 
-	sof_parent = snd_soc_acpi_sof_parent(&pdev->dev);
+	sof_parent = snd_soc_acpi_sof_parent(dev);
 
 	/* set card and driver name */
 	if (sof_parent) {
@@ -529,9 +522,9 @@ static int snd_cht_mc_probe(struct platform_device *pdev)
 		pdev->dev.driver->pm = &snd_soc_pm_ops;
 
 	/* register the soc card */
-	ret_val = devm_snd_soc_register_card(&pdev->dev, &snd_soc_card_cht);
+	ret_val = devm_snd_soc_register_card(dev, &snd_soc_card_cht);
 	if (ret_val) {
-		dev_err(&pdev->dev,
+		dev_err(dev,
 			"snd_soc_register_card failed %d\n", ret_val);
 		return ret_val;
 	}
diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c
index c18ec607e0296..5c4682f9f7c9b 100644
--- a/sound/soc/intel/boards/sof_sdw.c
+++ b/sound/soc/intel/boards/sof_sdw.c
@@ -901,10 +901,16 @@ static int create_sdw_dailink(struct snd_soc_card *card,
 		}
 	}
 
+	/*
+	 * The dai_type is used to select function topologies. Since the topology stream name
+	 * and DAI link name use partial matching, unconditionally appending the dai_type provides
+	 * necessary selection metadata without breaking existing topologies. Although
+	 * ctx->append_dai_type is not checked here, we overwrite it to ensure consistency in case
+	 * it is referenced elsewhere.
+	 */
+	ctx->append_dai_type = true;
 	for_each_pcm_streams(stream) {
 		static const char * const sdw_stream_name[] = {
-			"SDW%d-Playback",
-			"SDW%d-Capture",
 			"SDW%d-Playback-%s",
 			"SDW%d-Capture-%s",
 		};
@@ -932,15 +938,10 @@ static int create_sdw_dailink(struct snd_soc_card *card,
 		}
 
 		/* create stream name according to first link id */
-		if (ctx->append_dai_type)
-			name = devm_kasprintf(dev, GFP_KERNEL,
-					      sdw_stream_name[stream + 2],
-					      ffs(sof_end->link_mask) - 1,
-					      type_strings[sof_end->dai_info->dai_type]);
-		else
-			name = devm_kasprintf(dev, GFP_KERNEL,
-					      sdw_stream_name[stream],
-					      ffs(sof_end->link_mask) - 1);
+		name = devm_kasprintf(dev, GFP_KERNEL,
+				      sdw_stream_name[stream],
+				      ffs(sof_end->link_mask) - 1,
+				      type_strings[sof_end->dai_info->dai_type]);
 		if (!name)
 			return -ENOMEM;
 
diff --git a/sound/soc/intel/common/soc-acpi-intel-arl-match.c b/sound/soc/intel/common/soc-acpi-intel-arl-match.c
index c952f7d2b2c0e..59bfd5248819f 100644
--- a/sound/soc/intel/common/soc-acpi-intel-arl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-arl-match.c
@@ -8,6 +8,7 @@
 #include <sound/soc-acpi.h>
 #include <sound/soc-acpi-intel-match.h>
 #include <sound/soc-acpi-intel-ssp-common.h>
+#include "soc-acpi-intel-sdca-quirks.h"
 #include "sof-function-topology-lib.h"
 
 static const struct snd_soc_acpi_endpoint single_endpoint = {
@@ -192,6 +193,42 @@ static const struct snd_soc_acpi_endpoint cs42l43_endpoints[] = {
 	},
 };
 
+static const struct snd_soc_acpi_endpoint es9356_endpoints[] = {
+	{ /* Jack Playback Endpoint */
+		.num = 0,
+		.aggregated = 0,
+		.group_position = 0,
+		.group_id = 0,
+	},
+	{ /* DMIC Capture Endpoint */
+		.num = 1,
+		.aggregated = 0,
+		.group_position = 0,
+		.group_id = 0,
+	},
+	{ /* Jack Capture Endpoint */
+		.num = 2,
+		.aggregated = 0,
+		.group_position = 0,
+		.group_id = 0,
+	},
+	{ /* Speaker Playback Endpoint */
+		.num = 3,
+		.aggregated = 0,
+		.group_position = 0,
+		.group_id = 0,
+	},
+};
+
+static const struct snd_soc_acpi_adr_device es9356_adr[] = {
+	{
+		.adr = 0x00013004b3935601ull,
+		.num_endpoints = ARRAY_SIZE(es9356_endpoints),
+		.endpoints = es9356_endpoints,
+		.name_prefix = "es9356"
+	}
+};
+
 static const struct snd_soc_acpi_adr_device cs42l43_0_adr[] = {
 	{
 		.adr = 0x00003001FA424301ull,
@@ -237,6 +274,15 @@ static const struct snd_soc_acpi_adr_device rt722_0_agg_adr[] = {
 	}
 };
 
+static const struct snd_soc_acpi_adr_device rt712_0_agg_adr[] = {
+	{
+		.adr = 0x000030025D071201ull,
+		.num_endpoints = ARRAY_SIZE(jack_amp_g1_dmic_endpoints),
+		.endpoints = jack_amp_g1_dmic_endpoints,
+		.name_prefix = "rt712"
+	}
+};
+
 static const struct snd_soc_acpi_adr_device rt1316_3_single_adr[] = {
 	{
 		.adr = 0x000330025D131601ull,
@@ -255,6 +301,24 @@ static const struct snd_soc_acpi_adr_device rt1320_2_single_adr[] = {
 	}
 };
 
+static const struct snd_soc_acpi_link_adr arl_n_mrd_es9356_link1[] = {
+	{
+		.mask = BIT(1),
+		.num_adr = ARRAY_SIZE(es9356_adr),
+		.adr_d = es9356_adr,
+	},
+	{}
+};
+
+static const struct snd_soc_acpi_adr_device rt1320_3_group1_adr[] = {
+	{
+		.adr = 0x000330025D132001ull,
+		.num_endpoints = 1,
+		.endpoints = &spk_r_endpoint,
+		.name_prefix = "rt1320-1"
+	}
+};
+
 static const struct snd_soc_acpi_link_adr arl_cs42l43_l0[] = {
 	{
 		.mask = BIT(0),
@@ -404,6 +468,20 @@ static const struct snd_soc_acpi_link_adr arl_rt722_l0_rt1320_l2[] = {
 	{}
 };
 
+static const struct snd_soc_acpi_link_adr arl_rt712_l0_rt1320_l3[] = {
+	{
+		.mask = BIT(0),
+		.num_adr = ARRAY_SIZE(rt712_0_agg_adr),
+		.adr_d = rt712_0_agg_adr,
+	},
+	{
+		.mask = BIT(3),
+		.num_adr = ARRAY_SIZE(rt1320_3_group1_adr),
+		.adr_d = rt1320_3_group1_adr,
+	},
+	{}
+};
+
 static const struct snd_soc_acpi_codecs arl_essx_83x6 = {
 	.num_codecs = 3,
 	.codecs = { "ESSX8316", "ESSX8326", "ESSX8336"},
@@ -483,31 +561,39 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_arl_sdw_machines[] = {
 		.get_function_tplg_files = sof_sdw_get_tplg_files,
 	},
 	{
-		.link_mask = BIT(0),
-		.links = arl_cs42l43_l0,
+		.link_mask = BIT(0) | BIT(2),
+		.links = arl_rt722_l0_rt1320_l2,
 		.drv_name = "sof_sdw",
-		.sof_tplg_filename = "sof-arl-cs42l43-l0.tplg",
+		.sof_tplg_filename = "sof-arl-rt722-l0_rt1320-l2.tplg",
 		.get_function_tplg_files = sof_sdw_get_tplg_files,
 	},
 	{
-		.link_mask = BIT(2) | BIT(3),
-		.links = arl_cs42l43_l2_cs35l56_l3,
+		.link_mask = BIT(0) | BIT(3),
+		.links = arl_rt711_l0_rt1316_l3,
 		.drv_name = "sof_sdw",
-		.sof_tplg_filename = "sof-arl-cs42l43-l2-cs35l56-l3.tplg",
+		.sof_tplg_filename = "sof-arl-rt711-l0-rt1316-l3.tplg",
+	},
+	{
+		.link_mask = BIT(0) | BIT(3),
+		.links = arl_rt712_l0_rt1320_l3,
+		.drv_name = "sof_sdw",
+		.machine_check = snd_soc_acpi_intel_sdca_is_device_rt712_vb,
+		.sof_tplg_filename = "sof-arl-rt712-l0-rt1320-l3.tplg",
 		.get_function_tplg_files = sof_sdw_get_tplg_files,
 	},
 	{
-		.link_mask = BIT(2),
-		.links = arl_cs42l43_l2,
+		.link_mask = BIT(2) | BIT(3),
+		.links = arl_cs42l43_l2_cs35l56_l3,
 		.drv_name = "sof_sdw",
-		.sof_tplg_filename = "sof-arl-cs42l43-l2.tplg",
+		.sof_tplg_filename = "sof-arl-cs42l43-l2-cs35l56-l3.tplg",
 		.get_function_tplg_files = sof_sdw_get_tplg_files,
 	},
 	{
-		.link_mask = BIT(0) | BIT(3),
-		.links = arl_rt711_l0_rt1316_l3,
+		.link_mask = BIT(0),
+		.links = arl_cs42l43_l0,
 		.drv_name = "sof_sdw",
-		.sof_tplg_filename = "sof-arl-rt711-l0-rt1316-l3.tplg",
+		.sof_tplg_filename = "sof-arl-cs42l43-l0.tplg",
+		.get_function_tplg_files = sof_sdw_get_tplg_files,
 	},
 	{
 		.link_mask = 0x1, /* link0 required */
@@ -522,10 +608,17 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_arl_sdw_machines[] = {
 		.sof_tplg_filename = "sof-arl-rt711-l0.tplg",
 	},
 	{
-		.link_mask = BIT(0) | BIT(2),
-		.links = arl_rt722_l0_rt1320_l2,
+		.link_mask = BIT(2),
+		.links = arl_cs42l43_l2,
 		.drv_name = "sof_sdw",
-		.sof_tplg_filename = "sof-arl-rt722-l0_rt1320-l2.tplg",
+		.sof_tplg_filename = "sof-arl-cs42l43-l2.tplg",
+		.get_function_tplg_files = sof_sdw_get_tplg_files,
+	},
+	{
+		.link_mask = BIT(1),
+		.links = arl_n_mrd_es9356_link1,
+		.drv_name = "sof_sdw",
+		.sof_tplg_filename = "sof-arl-es9356.tplg",
 		.get_function_tplg_files = sof_sdw_get_tplg_files,
 	},
 	{},
diff --git a/sound/soc/intel/common/soc-acpi-intel-mtl-match.c b/sound/soc/intel/common/soc-acpi-intel-mtl-match.c
index 72c35e73078e3..2e4222456f27f 100644
--- a/sound/soc/intel/common/soc-acpi-intel-mtl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-mtl-match.c
@@ -122,6 +122,42 @@ static const struct snd_soc_acpi_endpoint spk_r_endpoint = {
 	.group_id = 1,
 };
 
+static const struct snd_soc_acpi_endpoint tac5xx2_endpoints[] = {
+	{ /* Playback Endpoint */
+		.num = 0,
+		.aggregated = 0,
+		.group_position = 0,
+		.group_id = 0,
+	},
+	{ /* Mic Capture Endpoint */
+		.num = 1,
+		.aggregated = 0,
+		.group_position = 0,
+		.group_id = 0,
+	},
+	{ /* UAJ-HP with Mic Endpoint */
+		.num = 2,
+		.aggregated = 0,
+		.group_position = 0,
+		.group_id = 0,
+	},
+};
+
+static const struct snd_soc_acpi_endpoint tas2883_endpoints[] = {
+	{ /* Playback Endpoint */
+		.num = 0,
+		.aggregated = 0,
+		.group_position = 0,
+		.group_id = 0,
+	},
+	{ /* Mic Capture Endpoint */
+		.num = 1,
+		.aggregated = 0,
+		.group_position = 0,
+		.group_id = 0,
+	},
+};
+
 static const struct snd_soc_acpi_endpoint rt712_endpoints[] = {
 	{
 		.num = 0,
@@ -1011,6 +1047,33 @@ static const struct snd_soc_acpi_adr_device cs42l42_0_adr[] = {
 	}
 };
 
+static const struct snd_soc_acpi_adr_device tac5572_0_adr[] = {
+	{
+		.adr = 0x0000300102557201ull,
+		.num_endpoints = ARRAY_SIZE(tac5xx2_endpoints),
+		.endpoints = tac5xx2_endpoints,
+		.name_prefix = "tac5572"
+	}
+};
+
+static const struct snd_soc_acpi_adr_device tac5672_0_adr[] = {
+	{
+		.adr = 0x0000300102567201ull,
+		.num_endpoints = ARRAY_SIZE(tac5xx2_endpoints),
+		.endpoints = tac5xx2_endpoints,
+		.name_prefix = "tac5672"
+	}
+};
+
+static const struct snd_soc_acpi_adr_device tac5682_0_adr[] = {
+	{
+		.adr = 0x0000300102568201ull,
+		.num_endpoints = ARRAY_SIZE(tac5xx2_endpoints),
+		.endpoints = tac5xx2_endpoints,
+		.name_prefix = "tac5682"
+	}
+};
+
 static const struct snd_soc_acpi_adr_device tas2783_0_adr[] = {
 	{
 		.adr = 0x00003c0102000001ull,
@@ -1035,9 +1098,45 @@ static const struct snd_soc_acpi_adr_device tas2783_0_adr[] = {
 		.num_endpoints = 1,
 		.endpoints = &spk_r_endpoint,
 		.name_prefix = "tas2783-4"
+	},
+};
+
+static const struct snd_soc_acpi_adr_device tas2883_0_adr[] = {
+	{
+		.adr = 0x0000300102288301ull,
+		.num_endpoints = ARRAY_SIZE(tas2883_endpoints),
+		.endpoints = tas2883_endpoints,
+		.name_prefix = "tas2883"
 	}
 };
 
+static const struct snd_soc_acpi_link_adr tac5572_l0[] = {
+	{
+		.mask = BIT(0),
+		.num_adr = ARRAY_SIZE(tac5572_0_adr),
+		.adr_d = tac5572_0_adr,
+	},
+	{}
+};
+
+static const struct snd_soc_acpi_link_adr tac5672_l0[] = {
+	{
+		.mask = BIT(0),
+		.num_adr = ARRAY_SIZE(tac5672_0_adr),
+		.adr_d = tac5672_0_adr,
+	},
+	{}
+};
+
+static const struct snd_soc_acpi_link_adr tac5682_l0[] = {
+	{
+		.mask = BIT(0),
+		.num_adr = ARRAY_SIZE(tac5682_0_adr),
+		.adr_d = tac5682_0_adr,
+	},
+	{}
+};
+
 static const struct snd_soc_acpi_link_adr tas2783_link0[] = {
 	{
 		.mask = BIT(0),
@@ -1047,6 +1146,15 @@ static const struct snd_soc_acpi_link_adr tas2783_link0[] = {
 	{}
 };
 
+static const struct snd_soc_acpi_link_adr tas2883_l0[] = {
+	{
+		.mask = BIT(0),
+		.num_adr = ARRAY_SIZE(tas2883_0_adr),
+		.adr_d = tas2883_0_adr,
+	},
+	{}
+};
+
 static const struct snd_soc_acpi_link_adr cs42l42_link0_max98363_link2[] = {
 	/* Expected order: jack -> amp */
 	{
@@ -1208,12 +1316,36 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_mtl_sdw_machines[] = {
 		.drv_name = "sof_sdw",
 		.sof_tplg_filename = "sof-mtl-rt715-rt711-rt1308-mono.tplg",
 	},
+	{
+		.link_mask = BIT(0),
+		.links = tac5572_l0,
+		.drv_name = "sof_sdw",
+		.sof_tplg_filename = "sof-mtl-tac5572.tplg",
+	},
+	{
+		.link_mask = BIT(0),
+		.links = tac5672_l0,
+		.drv_name = "sof_sdw",
+		.sof_tplg_filename = "sof-mtl-tac5672.tplg",
+	},
+	{
+		.link_mask = BIT(0),
+		.links = tac5682_l0,
+		.drv_name = "sof_sdw",
+		.sof_tplg_filename = "sof-mtl-tac5682.tplg",
+	},
 	{
 		.link_mask = BIT(0),
 		.links = tas2783_link0,
 		.drv_name = "sof_sdw",
 		.sof_tplg_filename = "sof-mtl-tas2783.tplg",
 	},
+	{
+		.link_mask = BIT(0),
+		.links = tas2883_l0,
+		.drv_name = "sof_sdw",
+		.sof_tplg_filename = "sof-mtl-tas2883.tplg",
+	},
 	{
 		.link_mask = GENMASK(3, 0),
 		.links = mtl_rt713_l0_rt1316_l12_rt1713_l3,
diff --git a/sound/soc/intel/common/soc-acpi-intel-ptl-match.c b/sound/soc/intel/common/soc-acpi-intel-ptl-match.c
index 3b7818355ff64..ad3af8834e431 100644
--- a/sound/soc/intel/common/soc-acpi-intel-ptl-match.c
+++ b/sound/soc/intel/common/soc-acpi-intel-ptl-match.c
@@ -493,6 +493,20 @@ static const struct snd_soc_acpi_link_adr ptl_sdw_rt713_vb_l3_rt1320_l12[] = {
 	{}
 };
 
+static const struct snd_soc_acpi_link_adr ptl_sdw_rt713_vb_l3_rt1320_l1[] = {
+	{
+		.mask = BIT(3),
+		.num_adr = ARRAY_SIZE(rt713_vb_3_adr),
+		.adr_d = rt713_vb_3_adr,
+	},
+	{
+		.mask = BIT(1),
+		.num_adr = ARRAY_SIZE(rt1320_1_group2_adr),
+		.adr_d = rt1320_1_group2_adr,
+	},
+	{}
+};
+
 static const struct snd_soc_acpi_link_adr ptl_sdw_rt712_vb_l2_rt1320_l1[] = {
 	{
 		.mask = BIT(2),
@@ -578,6 +592,13 @@ struct snd_soc_acpi_mach snd_soc_acpi_intel_ptl_sdw_machines[] = {
 		.sof_tplg_filename = "sof-ptl-rt713-l3-rt1320-l12.tplg",
 		.get_function_tplg_files = sof_sdw_get_tplg_files,
 	},
+	{
+		.link_mask = BIT(1) | BIT(3),
+		.links = ptl_sdw_rt713_vb_l3_rt1320_l1,
+		.drv_name = "sof_sdw",
+		.sof_tplg_filename = "sof-ptl-rt713-l3-rt1320-l1.tplg",
+		.get_function_tplg_files = sof_sdw_get_tplg_files,
+	},
 	{
 		.link_mask = BIT(1) | BIT(2) | BIT(3),
 		.links = ptl_cs42l43_l2_cs35l56x6_l13,
diff --git a/sound/soc/intel/common/sof-function-topology-lib.c b/sound/soc/intel/common/sof-function-topology-lib.c
index 0daa7d83808be..2f2c902ef90ce 100644
--- a/sound/soc/intel/common/sof-function-topology-lib.c
+++ b/sound/soc/intel/common/sof-function-topology-lib.c
@@ -19,6 +19,7 @@ enum tplg_device_id {
 	TPLG_DEVICE_SDCA_MIC,
 	TPLG_DEVICE_INTEL_PCH_DMIC,
 	TPLG_DEVICE_HDMI,
+	TPLG_DEVICE_LOOPBACK_VIRTUAL,
 	TPLG_DEVICE_MAX
 };
 
@@ -81,7 +82,15 @@ int sof_sdw_get_tplg_files(struct snd_soc_card *card, const struct snd_soc_acpi_
 		} else if (strstr(dai_link->name, "iDisp")) {
 			tplg_dev = TPLG_DEVICE_HDMI;
 			tplg_dev_name = "hdmi-pcm5";
-
+		} else if (strstr(dai_link->name, "Loopback_Virtual")) {
+			tplg_dev = TPLG_DEVICE_LOOPBACK_VIRTUAL;
+			/*
+			 * Mark the LOOPBACK_VIRTUAL device but no need to create the
+			 * LOOPBACK_VIRTUAL topology. Just to avoid the dai_link is not supported
+			 * error.
+			 */
+			tplg_mask |= BIT(tplg_dev);
+			continue;
 		} else {
 			/* The dai link is not supported by separated tplg yet */
 			dev_dbg(card->dev,
diff --git a/sound/soc/mediatek/Kconfig b/sound/soc/mediatek/Kconfig
index 3a1e1fa3fe5cc..4af7bbb58010d 100644
--- a/sound/soc/mediatek/Kconfig
+++ b/sound/soc/mediatek/Kconfig
@@ -26,6 +26,16 @@ config SND_SOC_MT2701_CS42448
 	  Select Y if you have such device.
 	  If unsure select "N".
 
+config SND_SOC_MT2701_HDMI
+	tristate "ASoC Audio driver for MT2701 with on-chip HDMI codec"
+	depends on SND_SOC_MT2701
+	select SND_SOC_HDMI_CODEC
+	help
+	  This adds the ASoC machine driver for MediaTek MT2701 and
+	  MT7623N boards routing the AFE I2S back-end to the on-chip
+	  HDMI transmitter via the generic HDMI codec.
+	  If unsure select "N".
+
 config SND_SOC_MT2701_WM8960
 	tristate "ASoc Audio driver for MT2701 with WM8960 codec"
 	depends on SND_SOC_MT2701 && I2C
@@ -353,4 +363,34 @@ config SND_SOC_MT8365_MT6357
 	  Select Y if you have such device.
 	  If unsure select "N".
 
+config SND_SOC_MT8196
+	tristate "ASoC support for Mediatek MT8196 chip"
+	depends on ARCH_MEDIATEK
+	select SND_SOC_MEDIATEK
+	help
+	  This adds ASoC driver for Mediatek MT8196 boards
+	  that can be used with other codecs.
+	  Select Y if you have such device.
+	  If unsure select "N".
+
+config SND_SOC_MT8196_NAU8825
+	tristate "ASoc Audio driver for MT8196 with NAU8825 and I2S codec"
+	depends on SND_SOC_MT8196
+	depends on I2C
+	select SND_SOC_HDMI_CODEC
+	select SND_SOC_DMIC
+	select SND_SOC_NAU8315
+	select SND_SOC_NAU8825
+	select SND_SOC_RT5645
+	select SND_SOC_RT5682_I2C
+	select SND_SOC_RT5682S
+	select SND_SOC_TAS2781_COMLIB
+	select SND_SOC_TAS2781_FMWLIB
+	select SND_SOC_TAS2781_I2C
+	help
+	  This adds support for ASoC machine driver for MediaTek MT8196
+	  boards with the NAU8825 and other I2S audio codecs.
+	  Select Y if you have such device.
+	  If unsure select "N".
+
 endmenu
diff --git a/sound/soc/mediatek/Makefile b/sound/soc/mediatek/Makefile
index 7cd67bce92e9a..a6815a3c59885 100644
--- a/sound/soc/mediatek/Makefile
+++ b/sound/soc/mediatek/Makefile
@@ -11,3 +11,4 @@ obj-$(CONFIG_SND_SOC_MT8192) += mt8192/
 obj-$(CONFIG_SND_SOC_MT8195) += mt8195/
 obj-$(CONFIG_SND_SOC_MT8365) += mt8365/
 obj-$(CONFIG_SND_SOC_MT8189) += mt8189/
+obj-$(CONFIG_SND_SOC_MT8196) += mt8196/
diff --git a/sound/soc/mediatek/common/mtk-afe-platform-driver.c b/sound/soc/mediatek/common/mtk-afe-platform-driver.c
index f2b39fc9ec81e..477284e12396d 100644
--- a/sound/soc/mediatek/common/mtk-afe-platform-driver.c
+++ b/sound/soc/mediatek/common/mtk-afe-platform-driver.c
@@ -87,29 +87,49 @@ snd_pcm_uframes_t mtk_afe_pcm_pointer(struct snd_soc_component *component,
 	const struct mtk_base_memif_data *memif_data = memif->data;
 	struct regmap *regmap = afe->regmap;
 	struct device *dev = afe->dev;
-	int reg_ofs_base = memif_data->reg_ofs_base;
-	int reg_ofs_cur = memif_data->reg_ofs_cur;
-	unsigned int hw_ptr = 0, hw_base = 0;
-	int ret, pcm_ptr_bytes;
-
-	ret = regmap_read(regmap, reg_ofs_cur, &hw_ptr);
-	if (ret || hw_ptr == 0) {
-		dev_err(dev, "%s hw_ptr err\n", __func__);
-		pcm_ptr_bytes = 0;
-		goto POINTER_RETURN_FRAMES;
+	unsigned int hw_ptr_lower32 = 0, hw_ptr_upper32 = 0;
+	unsigned int hw_base_lower32 = 0, hw_base_upper32 = 0;
+	unsigned long long hw_ptr = 0, hw_base = 0;
+	int ret;
+	unsigned long long pcm_ptr_bytes = 0;
+
+	ret = regmap_read(regmap, memif_data->reg_ofs_cur, &hw_ptr_lower32);
+	if (ret) {
+		dev_err(dev, "%s hw_ptr_lower32 err\n", __func__);
+		return 0;
 	}
 
-	ret = regmap_read(regmap, reg_ofs_base, &hw_base);
-	if (ret || hw_base == 0) {
-		dev_err(dev, "%s hw_ptr err\n", __func__);
-		pcm_ptr_bytes = 0;
-		goto POINTER_RETURN_FRAMES;
+	if (memif_data->reg_ofs_cur_msb) {
+		ret = regmap_read(regmap, memif_data->reg_ofs_cur_msb, &hw_ptr_upper32);
+		if (ret) {
+			dev_err(dev, "%s hw_ptr_upper32 err\n", __func__);
+			return 0;
+		}
 	}
 
-	pcm_ptr_bytes = hw_ptr - hw_base;
+	ret = regmap_read(regmap, memif_data->reg_ofs_base, &hw_base_lower32);
+	if (ret) {
+		dev_err(dev, "%s hw_base_lower32 err\n", __func__);
+		return 0;
+	}
+	if (memif_data->reg_ofs_base_msb) {
+		ret = regmap_read(regmap, memif_data->reg_ofs_base_msb, &hw_base_upper32);
+		if (ret) {
+			dev_err(dev, "%s hw_base_upper32 err\n", __func__);
+			return 0;
+		}
+	}
+
+	hw_ptr = ((unsigned long long)hw_ptr_upper32 << 32) | hw_ptr_lower32;
+	hw_base = ((unsigned long long)hw_base_upper32 << 32) | hw_base_lower32;
+
+	if (!hw_ptr || !hw_base) {
+		dev_err(dev, "hw_ptr or hw_base = 0 err\n");
+		return 0;
+	}
 
-POINTER_RETURN_FRAMES:
-	return bytes_to_frames(substream->runtime, pcm_ptr_bytes);
+	pcm_ptr_bytes = MTK_ALIGN_16BYTES(hw_ptr - hw_base);
+	return bytes_to_frames(substream->runtime, (ssize_t)pcm_ptr_bytes);
 }
 EXPORT_SYMBOL_GPL(mtk_afe_pcm_pointer);
 
diff --git a/sound/soc/mediatek/common/mtk-afe-platform-driver.h b/sound/soc/mediatek/common/mtk-afe-platform-driver.h
index fcc923b88f124..71070b26f8f89 100644
--- a/sound/soc/mediatek/common/mtk-afe-platform-driver.h
+++ b/sound/soc/mediatek/common/mtk-afe-platform-driver.h
@@ -12,6 +12,8 @@
 #define AFE_PCM_NAME "mtk-afe-pcm"
 extern const struct snd_soc_component_driver mtk_afe_pcm_platform;
 
+#define MTK_ALIGN_16BYTES(x) ((x) & GENMASK_ULL(39, 4))
+
 struct mtk_base_afe;
 struct snd_pcm;
 struct snd_soc_component;
diff --git a/sound/soc/mediatek/mt2701/Makefile b/sound/soc/mediatek/mt2701/Makefile
index 507fa26c39452..59623d3d3a038 100644
--- a/sound/soc/mediatek/mt2701/Makefile
+++ b/sound/soc/mediatek/mt2701/Makefile
@@ -5,4 +5,5 @@ obj-$(CONFIG_SND_SOC_MT2701) += snd-soc-mt2701-afe.o
 
 # machine driver
 obj-$(CONFIG_SND_SOC_MT2701_CS42448) += mt2701-cs42448.o
+obj-$(CONFIG_SND_SOC_MT2701_HDMI) += mt2701-hdmi.o
 obj-$(CONFIG_SND_SOC_MT2701_WM8960) += mt2701-wm8960.o
diff --git a/sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.c b/sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.c
index ae620890bb3ac..5a2bcf027b4fb 100644
--- a/sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.c
+++ b/sound/soc/mediatek/mt2701/mt2701-afe-clock-ctrl.c
@@ -95,6 +95,28 @@ int mt2701_init_clock(struct mtk_base_afe *afe)
 		afe_priv->mrgif_ck = NULL;
 	}
 
+	/*
+	 * Optional HDMI audio clocks. Platforms that do not wire up the
+	 * HDMI output (e.g. MT2701 devkits using only the I2S BE DAIs)
+	 * may omit these; in that case the HDMI BE DAI simply cannot be
+	 * enabled, but the rest of the AFE still probes.
+	 */
+	afe_priv->hadds2pll_ck = devm_clk_get_optional(afe->dev, "hadds2pll_294m");
+	if (IS_ERR(afe_priv->hadds2pll_ck))
+		return PTR_ERR(afe_priv->hadds2pll_ck);
+
+	afe_priv->audio_hdmi_ck = devm_clk_get_optional(afe->dev, "audio_hdmi_pd");
+	if (IS_ERR(afe_priv->audio_hdmi_ck))
+		return PTR_ERR(afe_priv->audio_hdmi_ck);
+
+	afe_priv->audio_spdf_ck = devm_clk_get_optional(afe->dev, "audio_spdf_pd");
+	if (IS_ERR(afe_priv->audio_spdf_ck))
+		return PTR_ERR(afe_priv->audio_spdf_ck);
+
+	afe_priv->audio_apll_ck = devm_clk_get_optional(afe->dev, "audio_apll_pd");
+	if (IS_ERR(afe_priv->audio_apll_ck))
+		return PTR_ERR(afe_priv->audio_apll_ck);
+
 	return 0;
 }
 
diff --git a/sound/soc/mediatek/mt2701/mt2701-afe-common.h b/sound/soc/mediatek/mt2701/mt2701-afe-common.h
index 32bef5e2a56d9..8b6f3a200048a 100644
--- a/sound/soc/mediatek/mt2701/mt2701-afe-common.h
+++ b/sound/soc/mediatek/mt2701/mt2701-afe-common.h
@@ -33,6 +33,7 @@ enum {
 	MT2701_MEMIF_UL5,
 	MT2701_MEMIF_DLBT,
 	MT2701_MEMIF_ULBT,
+	MT2701_MEMIF_HDMI,
 	MT2701_MEMIF_NUM,
 	MT2701_IO_I2S = MT2701_MEMIF_NUM,
 	MT2701_IO_2ND_I2S,
@@ -41,6 +42,7 @@ enum {
 	MT2701_IO_5TH_I2S,
 	MT2701_IO_6TH_I2S,
 	MT2701_IO_MRG,
+	MT2701_IO_HDMI,
 };
 
 enum {
@@ -90,6 +92,10 @@ struct mt2701_afe_private {
 	struct mt2701_i2s_path *i2s_path;
 	struct clk *base_ck[MT2701_BASE_CLK_NUM];
 	struct clk *mrgif_ck;
+	struct clk *hadds2pll_ck;
+	struct clk *audio_hdmi_ck;
+	struct clk *audio_spdf_ck;
+	struct clk *audio_apll_ck;
 	bool mrg_enable[MTK_STREAM_NUM];
 
 	const struct mt2701_soc_variants *soc;
diff --git a/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c b/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c
index fcae38135d93f..bb459faa6e054 100644
--- a/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c
+++ b/sound/soc/mediatek/mt2701/mt2701-afe-pcm.c
@@ -13,6 +13,7 @@
 #include <linux/mfd/syscon.h>
 #include <linux/of.h>
 #include <linux/pm_runtime.h>
+#include <sound/pcm_params.h>
 
 #include "mt2701-afe-common.h"
 #include "mt2701-afe-clock-ctrl.h"
@@ -60,6 +61,7 @@ static const struct mt2701_afe_rate mt2701_afe_i2s_rates[] = {
 
 static const unsigned int mt2701_afe_backup_list[] = {
 	AUDIO_TOP_CON0,
+	AUDIO_TOP_CON3,
 	AUDIO_TOP_CON4,
 	AUDIO_TOP_CON5,
 	ASYS_TOP_CON,
@@ -77,6 +79,9 @@ static const unsigned int mt2701_afe_backup_list[] = {
 	AFE_CONN22,
 	AFE_DAC_CON0,
 	AFE_MEMIF_PBUF_SIZE,
+	AFE_HDMI_OUT_CON0,
+	AFE_HDMI_CONN0,
+	AFE_8CH_I2S_OUT_CON,
 };
 
 static int mt2701_dai_num_to_i2s(struct mtk_base_afe *afe, int num)
@@ -542,6 +547,220 @@ static const struct snd_soc_dai_ops mt2701_btmrg_ops = {
 	.hw_params = mt2701_btmrg_hw_params,
 };
 
+/*
+ * HDMI BE DAI -- drives the on-SoC 8-channel I2S engine whose output
+ * feeds the HDMI transmitter audio port.
+ *
+ * The HDMI audio hardware path is:
+ *   HDMI memif DMA (AFE_HDMI_OUT_*) -> interconnect mux (AFE_HDMI_CONN0)
+ *   -> 8-channel I2S engine (AFE_8CH_I2S_OUT_CON) -> HDMI TX audio port
+ *
+ * The I2S3 clock tree provides the bit/master clocks; we set its
+ * mclk_rate to 128*fs (matching HDMI_AUD_MCLK_128FS) and let
+ * mt2701_mclk_configuration program the PLL/divider path.
+ */
+#define MT2701_HDMI_I2S_PATH	3
+
+static int mt2701_afe_hdmi_startup(struct snd_pcm_substream *substream,
+				   struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct mt2701_afe_private *afe_priv = afe->platform_priv;
+	int ret;
+
+	if (!afe_priv->hadds2pll_ck || !afe_priv->audio_hdmi_ck) {
+		dev_err(afe->dev, "HDMI audio clocks not available\n");
+		return -ENODEV;
+	}
+
+	ret = clk_prepare_enable(afe_priv->hadds2pll_ck);
+	if (ret)
+		return ret;
+
+	ret = clk_prepare_enable(afe_priv->audio_hdmi_ck);
+	if (ret)
+		goto err_hdmi;
+
+	if (afe_priv->audio_spdf_ck) {
+		ret = clk_prepare_enable(afe_priv->audio_spdf_ck);
+		if (ret)
+			goto err_spdf;
+	}
+
+	if (afe_priv->audio_apll_ck) {
+		ret = clk_prepare_enable(afe_priv->audio_apll_ck);
+		if (ret)
+			goto err_apll;
+	}
+
+	ret = mt2701_afe_enable_mclk(afe, MT2701_HDMI_I2S_PATH);
+	if (ret)
+		goto err_mclk;
+
+	return 0;
+
+err_mclk:
+	if (afe_priv->audio_apll_ck)
+		clk_disable_unprepare(afe_priv->audio_apll_ck);
+err_apll:
+	if (afe_priv->audio_spdf_ck)
+		clk_disable_unprepare(afe_priv->audio_spdf_ck);
+err_spdf:
+	clk_disable_unprepare(afe_priv->audio_hdmi_ck);
+err_hdmi:
+	clk_disable_unprepare(afe_priv->hadds2pll_ck);
+	return ret;
+}
+
+static void mt2701_afe_hdmi_shutdown(struct snd_pcm_substream *substream,
+				     struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct mt2701_afe_private *afe_priv = afe->platform_priv;
+
+	mt2701_afe_disable_mclk(afe, MT2701_HDMI_I2S_PATH);
+	if (afe_priv->audio_apll_ck)
+		clk_disable_unprepare(afe_priv->audio_apll_ck);
+	if (afe_priv->audio_spdf_ck)
+		clk_disable_unprepare(afe_priv->audio_spdf_ck);
+	clk_disable_unprepare(afe_priv->audio_hdmi_ck);
+	clk_disable_unprepare(afe_priv->hadds2pll_ck);
+}
+
+static int mt2701_afe_hdmi_hw_params(struct snd_pcm_substream *substream,
+				     struct snd_pcm_hw_params *params,
+				     struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct mt2701_afe_private *afe_priv = afe->platform_priv;
+	unsigned int channels = params_channels(params);
+	unsigned int rate = params_rate(params);
+	unsigned int divp1;
+	unsigned int val;
+	unsigned int i;
+	int ret;
+
+	/*
+	 * Compute AUDIO_TOP_CON3.HDMI_BCK_DIV up front. The divider
+	 * drives an internal reference for the HDMI transmitter's
+	 * audio packet engine; it must scale with the sample rate so
+	 * that the packet engine's timing matches the data flowing in
+	 * from the AFE memif/I2S3 side. Empirically, with audpll_sel
+	 * parented to hadds2pll_98m (98.304 MHz), the correct value at
+	 * 48 kHz is div = 44 (i.e. (div+1) = 45), giving 1.0923 MHz.
+	 * Scaling inversely with rate: (div + 1) = 45 * 48000 / rate.
+	 * Integer rounding introduces small (<1%) errors at 32 kHz;
+	 * 44.1 kHz is nearly exact via round-to-nearest. Reject rates
+	 * that fall outside the 6-bit divider range before touching
+	 * any hardware so no side effects are left behind on error.
+	 */
+	divp1 = (45U * 48000U + rate / 2) / rate;
+	if (divp1 == 0 || divp1 > 64)
+		return -EINVAL;
+
+	/*
+	 * Park the I2S3 clock tree at 128*fs -- this is the MCLK that
+	 * the ASYS I2S3 engine uses to derive its BCK/LRCK. The engine
+	 * outputs BCK = 64*fs (stereo, 32-bit word length).
+	 */
+	afe_priv->i2s_path[MT2701_HDMI_I2S_PATH].mclk_rate = rate * 128;
+	ret = mt2701_mclk_configuration(afe, MT2701_HDMI_I2S_PATH);
+	if (ret)
+		return ret;
+
+	/* Program and start the ASYS I2S3 engine (FS, I2S mode, enable). */
+	mt2701_i2s_path_enable(afe,
+			       &afe_priv->i2s_path[MT2701_HDMI_I2S_PATH],
+			       SNDRV_PCM_STREAM_PLAYBACK, rate);
+
+	regmap_update_bits(afe->regmap, AUDIO_TOP_CON3,
+			   AUDIO_TOP_CON3_HDMI_BCK_DIV_MASK,
+			   AUDIO_TOP_CON3_HDMI_BCK_DIV(divp1 - 1));
+
+	/*
+	 * HDMI output memif: set channel count and confirm 16-bit
+	 * sample width. Both fields must be written together so that
+	 * stale reset-default or prior-stream values in BIT_WIDTH
+	 * cannot persist.
+	 */
+	regmap_update_bits(afe->regmap, AFE_HDMI_OUT_CON0,
+			   AFE_HDMI_OUT_CON0_CH_NUM_MASK |
+			   AFE_HDMI_OUT_CON0_BIT_WIDTH_MASK,
+			   AFE_HDMI_OUT_CON0_CH_NUM(channels) |
+			   AFE_HDMI_OUT_CON0_BIT_WIDTH_16);
+
+	/*
+	 * Interconnect mux -- map DMA input slots to HDMI output slots.
+	 * Each output takes a 3-bit field at shift (i*3). Swap the first
+	 * two inputs so that the DMA's interleaved L/R pair lands on the
+	 * correct HDMI L/R output slots. Remaining slots are identity.
+	 */
+	val = (1 << 0) | (0 << 3);  /* O20 <- I21, O21 <- I20 */
+	for (i = 2; i < 8; i++)
+		val |= ((i & 0x7) << (i * 3));
+	regmap_write(afe->regmap, AFE_HDMI_CONN0, val);
+
+	/*
+	 * 8-channel I2S framing: standard I2S, 32-bit slots,
+	 * LRCK/BCK inverted. The wire protocol is fixed.
+	 */
+	regmap_update_bits(afe->regmap, AFE_8CH_I2S_OUT_CON,
+			   AFE_8CH_I2S_OUT_CON_WLEN_MASK |
+			   AFE_8CH_I2S_OUT_CON_I2S_DELAY |
+			   AFE_8CH_I2S_OUT_CON_LRCK_INV |
+			   AFE_8CH_I2S_OUT_CON_BCK_INV,
+			   AFE_8CH_I2S_OUT_CON_WLEN_32BIT |
+			   AFE_8CH_I2S_OUT_CON_I2S_DELAY |
+			   AFE_8CH_I2S_OUT_CON_LRCK_INV |
+			   AFE_8CH_I2S_OUT_CON_BCK_INV);
+	return 0;
+}
+
+static int mt2701_afe_hdmi_trigger(struct snd_pcm_substream *substream, int cmd,
+				   struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+		/* Enable HDMI output memif. */
+		regmap_update_bits(afe->regmap, AFE_HDMI_OUT_CON0, 0x1, 0x1);
+		/* Enable 8-channel I2S engine. */
+		regmap_update_bits(afe->regmap, AFE_8CH_I2S_OUT_CON,
+				   AFE_8CH_I2S_OUT_CON_EN,
+				   AFE_8CH_I2S_OUT_CON_EN);
+		return 0;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+		regmap_update_bits(afe->regmap, AFE_8CH_I2S_OUT_CON,
+				   AFE_8CH_I2S_OUT_CON_EN, 0);
+		regmap_update_bits(afe->regmap, AFE_HDMI_OUT_CON0, 0x1, 0);
+		return 0;
+	}
+	return -EINVAL;
+}
+
+static int mt2701_afe_hdmi_hw_free(struct snd_pcm_substream *substream,
+				   struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct mt2701_afe_private *afe_priv = afe->platform_priv;
+
+	mt2701_afe_i2s_path_disable(afe,
+				    &afe_priv->i2s_path[MT2701_HDMI_I2S_PATH],
+				    SNDRV_PCM_STREAM_PLAYBACK);
+	return 0;
+}
+
+static const struct snd_soc_dai_ops mt2701_afe_hdmi_ops = {
+	.startup	= mt2701_afe_hdmi_startup,
+	.shutdown	= mt2701_afe_hdmi_shutdown,
+	.hw_params	= mt2701_afe_hdmi_hw_params,
+	.hw_free	= mt2701_afe_hdmi_hw_free,
+	.trigger	= mt2701_afe_hdmi_trigger,
+};
+
 static struct snd_soc_dai_driver mt2701_afe_pcm_dais[] = {
 	/* FE DAIs: memory intefaces to CPU */
 	{
@@ -628,6 +847,19 @@ static struct snd_soc_dai_driver mt2701_afe_pcm_dais[] = {
 		},
 		.ops = &mt2701_single_memif_dai_ops,
 	},
+	{
+		.name = "PCM_HDMI",
+		.id = MT2701_MEMIF_HDMI,
+		.playback = {
+			.stream_name = "HDMI Multich",
+			.channels_min = 2,
+			.channels_max = 8,
+			.rates = (SNDRV_PCM_RATE_44100 |
+				  SNDRV_PCM_RATE_48000),
+			.formats = SNDRV_PCM_FMTBIT_S16_LE,
+		},
+		.ops = &mt2701_single_memif_dai_ops,
+	},
 	/* BE DAIs */
 	{
 		.name = "I2S0",
@@ -748,7 +980,20 @@ static struct snd_soc_dai_driver mt2701_afe_pcm_dais[] = {
 		},
 		.ops = &mt2701_btmrg_ops,
 		.symmetric_rate = 1,
-	}
+	},
+	{
+		.name = "HDMI I2S",
+		.id = MT2701_IO_HDMI,
+		.playback = {
+			.stream_name = "HDMI 8CH I2S Playback",
+			.channels_min = 2,
+			.channels_max = 8,
+			.rates = (SNDRV_PCM_RATE_44100 |
+				  SNDRV_PCM_RATE_48000),
+			.formats = SNDRV_PCM_FMTBIT_S16_LE,
+		},
+		.ops = &mt2701_afe_hdmi_ops,
+	},
 };
 
 static const struct snd_kcontrol_new mt2701_afe_o00_mix[] = {
@@ -927,6 +1172,14 @@ static const struct snd_soc_dapm_route mt2701_afe_pcm_routes[] = {
 	{"I16I17", "Multich I2S2 Out Switch", "DLM"},
 	{"I18I19", "Multich I2S3 Out Switch", "DLM"},
 
+	/*
+	 * HDMI FE -> BE direct route. The HDMI memif has its own DMA
+	 * path that feeds the 8-channel internal I2S straight into the
+	 * HDMI transmitter; no mixer/interconnect selection is exposed
+	 * to the user.
+	 */
+	{"HDMI 8CH I2S Playback", NULL, "HDMI Multich"},
+
 	{ "I12", NULL, "I12I13" },
 	{ "I13", NULL, "I12I13" },
 	{ "I14", NULL, "I14I15" },
@@ -1207,6 +1460,35 @@ static const struct mtk_base_memif_data memif_data_array[MT2701_MEMIF_NUM] = {
 		.agent_disable_shift = 16,
 		.msb_reg = -1,
 	},
+	{
+		/*
+		 * HDMI memif feeds the on-SoC 8-channel internal I2S that
+		 * drives the HDMI transmitter audio port. Unlike the
+		 * standard memifs, the enable bit, channel count and bit
+		 * width all live in AFE_HDMI_OUT_CON0, so mono/fs/hd/agent
+		 * fields are left at -1 and programmed from the BE DAI ops
+		 * instead.
+		 */
+		.name = "HDMI",
+		.id = MT2701_MEMIF_HDMI,
+		.reg_ofs_base = AFE_HDMI_OUT_BASE,
+		.reg_ofs_cur = AFE_HDMI_OUT_CUR,
+		.reg_ofs_end = AFE_HDMI_OUT_END,
+		.fs_reg = -1,
+		.fs_shift = -1,
+		.fs_maskbit = 0,
+		.mono_reg = -1,
+		.mono_shift = -1,
+		.enable_reg = AFE_HDMI_OUT_CON0,
+		.enable_shift = 0,
+		.hd_reg = -1,
+		.hd_shift = -1,
+		.hd_align_reg = -1,
+		.hd_align_mshift = 0,
+		.agent_disable_reg = -1,
+		.agent_disable_shift = 0,
+		.msb_reg = -1,
+	},
 };
 
 static const struct mtk_base_irq_data irq_data[MT2701_IRQ_ASYS_END] = {
diff --git a/sound/soc/mediatek/mt2701/mt2701-hdmi.c b/sound/soc/mediatek/mt2701/mt2701-hdmi.c
new file mode 100644
index 0000000000000..a84907879c04e
--- /dev/null
+++ b/sound/soc/mediatek/mt2701/mt2701-hdmi.c
@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * mt2701-hdmi.c -- MT2701 HDMI ALSA SoC machine driver
+ *
+ * Copyright (c) 2026 Daniel Golle <daniel@makrotopia.org>
+ *
+ * Based on mt2701-cs42448.c
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
+#include <sound/soc.h>
+
+enum {
+	DAI_LINK_FE_HDMI_OUT,
+	DAI_LINK_BE_HDMI_I2S,
+};
+
+SND_SOC_DAILINK_DEFS(fe_hdmi_out,
+	DAILINK_COMP_ARRAY(COMP_CPU("PCM_HDMI")),
+	DAILINK_COMP_ARRAY(COMP_DUMMY()),
+	DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+SND_SOC_DAILINK_DEFS(be_hdmi_i2s,
+	DAILINK_COMP_ARRAY(COMP_CPU("HDMI I2S")),
+	DAILINK_COMP_ARRAY(COMP_CODEC(NULL, "i2s-hifi")),
+	DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+static struct snd_soc_dai_link mt2701_hdmi_dai_links[] = {
+	[DAI_LINK_FE_HDMI_OUT] = {
+		.name = "HDMI Playback",
+		.stream_name = "HDMI Playback",
+		.trigger = { SND_SOC_DPCM_TRIGGER_POST,
+			     SND_SOC_DPCM_TRIGGER_POST },
+		.dynamic = 1,
+		.playback_only = 1,
+		SND_SOC_DAILINK_REG(fe_hdmi_out),
+	},
+	[DAI_LINK_BE_HDMI_I2S] = {
+		.name = "HDMI BE",
+		.no_pcm = 1,
+		.playback_only = 1,
+		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_NB_NF |
+			   SND_SOC_DAIFMT_CBC_CFC,
+		SND_SOC_DAILINK_REG(be_hdmi_i2s),
+	},
+};
+
+static struct snd_soc_card mt2701_hdmi_soc_card = {
+	.name = "mt2701-hdmi",
+	.owner = THIS_MODULE,
+	.dai_link = mt2701_hdmi_dai_links,
+	.num_links = ARRAY_SIZE(mt2701_hdmi_dai_links),
+};
+
+static int mt2701_hdmi_machine_probe(struct platform_device *pdev)
+{
+	struct snd_soc_card *card = &mt2701_hdmi_soc_card;
+	struct device *dev = &pdev->dev;
+	struct device_node *platform_node;
+	struct device_node *codec_node;
+	struct snd_soc_dai_link *dai_link;
+	int ret;
+	int i;
+
+	platform_node = of_parse_phandle(dev->of_node, "mediatek,platform", 0);
+	if (!platform_node)
+		return dev_err_probe(dev, -EINVAL,
+				     "Property 'mediatek,platform' missing\n");
+
+	for_each_card_prelinks(card, i, dai_link) {
+		if (dai_link->platforms->name)
+			continue;
+		dai_link->platforms->of_node = platform_node;
+	}
+
+	codec_node = of_parse_phandle(dev->of_node, "mediatek,audio-codec", 0);
+	if (!codec_node) {
+		of_node_put(platform_node);
+		return dev_err_probe(dev, -EINVAL,
+				     "Property 'mediatek,audio-codec' missing\n");
+	}
+	mt2701_hdmi_dai_links[DAI_LINK_BE_HDMI_I2S].codecs->of_node = codec_node;
+
+	card->dev = dev;
+
+	ret = devm_snd_soc_register_card(dev, card);
+
+	of_node_put(platform_node);
+	of_node_put(codec_node);
+	return ret;
+}
+
+static const struct of_device_id mt2701_hdmi_machine_dt_match[] = {
+	{ .compatible = "mediatek,mt2701-hdmi-audio" },
+	{ .compatible = "mediatek,mt7623n-hdmi-audio" },
+	{}
+};
+MODULE_DEVICE_TABLE(of, mt2701_hdmi_machine_dt_match);
+
+static struct platform_driver mt2701_hdmi_machine = {
+	.driver = {
+		.name = "mt2701-hdmi",
+		.of_match_table = mt2701_hdmi_machine_dt_match,
+	},
+	.probe = mt2701_hdmi_machine_probe,
+};
+module_platform_driver(mt2701_hdmi_machine);
+
+MODULE_DESCRIPTION("MT2701 HDMI ALSA SoC machine driver");
+MODULE_AUTHOR("Daniel Golle <daniel@makrotopia.org>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:mt2701-hdmi");
diff --git a/sound/soc/mediatek/mt2701/mt2701-reg.h b/sound/soc/mediatek/mt2701/mt2701-reg.h
index c84d14cdd7ae8..ca914df02c960 100644
--- a/sound/soc/mediatek/mt2701/mt2701-reg.h
+++ b/sound/soc/mediatek/mt2701/mt2701-reg.h
@@ -10,10 +10,17 @@
 #define _MT2701_REG_H_
 
 #define AUDIO_TOP_CON0 0x0000
+#define AUDIO_TOP_CON3 0x000c
 #define AUDIO_TOP_CON4 0x0010
 #define AUDIO_TOP_CON5 0x0014
 #define AFE_DAIBT_CON0 0x001c
 #define AFE_MRGIF_CON 0x003c
+#define AFE_HDMI_OUT_CON0 0x0370
+#define AFE_HDMI_OUT_BASE 0x0374
+#define AFE_HDMI_OUT_CUR  0x0378
+#define AFE_HDMI_OUT_END  0x037c
+#define AFE_HDMI_CONN0    0x0390
+#define AFE_8CH_I2S_OUT_CON 0x0394
 #define ASMI_TIMING_CON1 0x0100
 #define ASMO_TIMING_CON1 0x0104
 #define PWR1_ASM_CON1 0x0108
@@ -125,6 +132,28 @@
 #define AFE_MEMIF_PBUF_SIZE_DLM_BYTE_MASK	(0x3 << 12)
 #define AFE_MEMIF_PBUF_SIZE_DLM_32BYTES		(0x1 << 12)
 
+/* AUDIO_TOP_CON3 (0x000c) -- HDMI BCK divider */
+#define AUDIO_TOP_CON3_HDMI_BCK_DIV_MASK	(0x3f << 8)
+#define AUDIO_TOP_CON3_HDMI_BCK_DIV(x)		(((x) & 0x3f) << 8)
+
+/* AFE_HDMI_OUT_CON0 (0x0370) */
+#define AFE_HDMI_OUT_CON0_OUT_ON		(0x1 << 0)
+#define AFE_HDMI_OUT_CON0_BIT_WIDTH_MASK	(0x1 << 1)
+#define AFE_HDMI_OUT_CON0_BIT_WIDTH_16		(0x0 << 1)
+#define AFE_HDMI_OUT_CON0_BIT_WIDTH_32		(0x1 << 1)
+#define AFE_HDMI_OUT_CON0_CH_NUM_MASK		(0xf << 4)
+#define AFE_HDMI_OUT_CON0_CH_NUM(x)		(((x) & 0xf) << 4)
+
+/* AFE_8CH_I2S_OUT_CON (0x0394) -- on-SoC 8-channel I2S that feeds HDMI TX */
+#define AFE_8CH_I2S_OUT_CON_EN			(0x1 << 0)
+#define AFE_8CH_I2S_OUT_CON_BCK_INV		(0x1 << 1)
+#define AFE_8CH_I2S_OUT_CON_LRCK_INV		(0x1 << 2)
+#define AFE_8CH_I2S_OUT_CON_I2S_DELAY		(0x1 << 3)
+#define AFE_8CH_I2S_OUT_CON_WLEN_MASK		(0x3 << 4)
+#define AFE_8CH_I2S_OUT_CON_WLEN_16BIT		(0x1 << 4)
+#define AFE_8CH_I2S_OUT_CON_WLEN_24BIT		(0x2 << 4)
+#define AFE_8CH_I2S_OUT_CON_WLEN_32BIT		(0x3 << 4)
+
 /* I2S in/out register bit control */
 #define ASYS_I2S_CON_FS			(0x1f << 8)
 #define ASYS_I2S_CON_FS_SET(x)		((x) << 8)
diff --git a/sound/soc/mediatek/mt8196/Makefile b/sound/soc/mediatek/mt8196/Makefile
new file mode 100644
index 0000000000000..91de200071d78
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/Makefile
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0
+
+# platform driver
+snd-soc-mt8196-afe-objs += \
+	mt8196-afe-pcm.o \
+	mt8196-afe-clk.o \
+	mt8196-dai-adda.o \
+	mt8196-dai-i2s.o \
+	mt8196-dai-tdm.o
+
+obj-$(CONFIG_SND_SOC_MT8196) += snd-soc-mt8196-afe.o
+
+# machine driver
+obj-$(CONFIG_SND_SOC_MT8196_NAU8825) += mt8196-nau8825.o
diff --git a/sound/soc/mediatek/mt8196/mt8196-afe-clk.c b/sound/soc/mediatek/mt8196/mt8196-afe-clk.c
new file mode 100644
index 0000000000000..286e39f53ae05
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-afe-clk.c
@@ -0,0 +1,581 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  mt8196-afe-clk.c  --  Mediatek 8196 afe clock ctrl
+ *
+ *  Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#include <linux/clk.h>
+#include <linux/regmap.h>
+#include <linux/mfd/syscon.h>
+
+#include "mt8196-afe-clk.h"
+#include "mt8196-afe-common.h"
+
+static const char *aud_clks[MT8196_CLK_NUM] = {
+	/* vlp clk */
+	[MT8196_CLK_VLP_MUX_AUDIOINTBUS] = "top_aud_intbus",
+	[MT8196_CLK_VLP_MUX_AUD_ENG1] = "top_aud_eng1",
+	[MT8196_CLK_VLP_MUX_AUD_ENG2] = "top_aud_eng2",
+	[MT8196_CLK_VLP_MUX_AUDIO_H] = "top_aud_h",
+	/* pll */
+	[MT8196_CLK_TOP_APLL1_CK] = "apll1",
+	[MT8196_CLK_TOP_APLL2_CK] = "apll2",
+	/* divider */
+	[MT8196_CLK_TOP_APLL12_DIV_I2SIN0] = "apll12_div_i2sin0",
+	[MT8196_CLK_TOP_APLL12_DIV_I2SIN1] = "apll12_div_i2sin1",
+	[MT8196_CLK_TOP_APLL12_DIV_FMI2S] = "apll12_div_fmi2s",
+	[MT8196_CLK_TOP_APLL12_DIV_TDMOUT_M] = "apll12_div_tdmout_m",
+	[MT8196_CLK_TOP_APLL12_DIV_TDMOUT_B] = "apll12_div_tdmout_b",
+	/* mux */
+	[MT8196_CLK_TOP_ADSP_SEL] = "top_adsp",
+};
+
+int mt8196_afe_enable_clk(struct mtk_base_afe *afe, struct clk *clk)
+{
+	int ret;
+
+	ret = clk_prepare_enable(clk);
+	if (ret) {
+		dev_err(afe->dev, "failed to enable clk\n");
+		return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(mt8196_afe_enable_clk);
+
+void mt8196_afe_disable_clk(struct mtk_base_afe *afe, struct clk *clk)
+{
+	if (clk)
+		clk_disable_unprepare(clk);
+	else
+		dev_err(afe->dev, "NULL clk\n");
+}
+EXPORT_SYMBOL_GPL(mt8196_afe_disable_clk);
+
+static int mt8196_afe_set_clk_rate(struct mtk_base_afe *afe, struct clk *clk,
+				   unsigned int rate)
+{
+	int ret;
+
+	if (clk) {
+		ret = clk_set_rate(clk, rate);
+		if (ret) {
+			dev_err(afe->dev, "failed to set clk rate\n");
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
+static unsigned int get_top_cg_reg(unsigned int cg_type)
+{
+	switch (cg_type) {
+	case MT8196_AUDIO_26M_EN_ON:
+	case MT8196_AUDIO_F3P25M_EN_ON:
+	case MT8196_AUDIO_APLL1_EN_ON:
+	case MT8196_AUDIO_APLL2_EN_ON:
+		return AUDIO_ENGEN_CON0;
+	case MT8196_CG_AUDIO_HOPPING_CK:
+	case MT8196_CG_AUDIO_F26M_CK:
+	case MT8196_CG_APLL1_CK:
+	case MT8196_CG_APLL2_CK:
+	case MT8196_PDN_APLL_TUNER2:
+	case MT8196_PDN_APLL_TUNER1:
+		return AUDIO_TOP_CON4;
+	default:
+		return 0;
+	}
+}
+
+static unsigned int get_top_cg_mask(unsigned int cg_type)
+{
+	switch (cg_type) {
+	case MT8196_AUDIO_26M_EN_ON:
+		return AUDIO_26M_EN_ON_MASK_SFT;
+	case MT8196_AUDIO_F3P25M_EN_ON:
+		return AUDIO_F3P25M_EN_ON_MASK_SFT;
+	case MT8196_AUDIO_APLL1_EN_ON:
+		return AUDIO_APLL1_EN_ON_MASK_SFT;
+	case MT8196_AUDIO_APLL2_EN_ON:
+		return AUDIO_APLL2_EN_ON_MASK_SFT;
+	case MT8196_CG_AUDIO_HOPPING_CK:
+		return CG_AUDIO_HOPPING_CK_MASK_SFT;
+	case MT8196_CG_AUDIO_F26M_CK:
+		return CG_AUDIO_F26M_CK_MASK_SFT;
+	case MT8196_CG_APLL1_CK:
+		return CG_APLL1_CK_MASK_SFT;
+	case MT8196_CG_APLL2_CK:
+		return CG_APLL2_CK_MASK_SFT;
+	case MT8196_PDN_APLL_TUNER2:
+		return PDN_APLL_TUNER2_MASK_SFT;
+	case MT8196_PDN_APLL_TUNER1:
+		return PDN_APLL_TUNER1_MASK_SFT;
+	default:
+		return 0;
+	}
+}
+
+static unsigned int get_top_cg_on_val(unsigned int cg_type)
+{
+	switch (cg_type) {
+	case MT8196_AUDIO_26M_EN_ON:
+	case MT8196_AUDIO_F3P25M_EN_ON:
+	case MT8196_AUDIO_APLL1_EN_ON:
+	case MT8196_AUDIO_APLL2_EN_ON:
+		return get_top_cg_mask(cg_type);
+	case MT8196_CG_AUDIO_HOPPING_CK:
+	case MT8196_CG_AUDIO_F26M_CK:
+	case MT8196_CG_APLL1_CK:
+	case MT8196_CG_APLL2_CK:
+	case MT8196_PDN_APLL_TUNER2:
+	case MT8196_PDN_APLL_TUNER1:
+		return 0;
+	default:
+		return 0;
+	}
+}
+
+static unsigned int get_top_cg_off_val(unsigned int cg_type)
+{
+	switch (cg_type) {
+	case MT8196_AUDIO_26M_EN_ON:
+	case MT8196_AUDIO_F3P25M_EN_ON:
+	case MT8196_AUDIO_APLL1_EN_ON:
+	case MT8196_AUDIO_APLL2_EN_ON:
+		return 0;
+	case MT8196_CG_AUDIO_HOPPING_CK:
+	case MT8196_CG_AUDIO_F26M_CK:
+	case MT8196_CG_APLL1_CK:
+	case MT8196_CG_APLL2_CK:
+	case MT8196_PDN_APLL_TUNER2:
+	case MT8196_PDN_APLL_TUNER1:
+		return get_top_cg_mask(cg_type);
+	default:
+		return get_top_cg_mask(cg_type);
+	}
+}
+
+static int mt8196_afe_enable_top_cg(struct mtk_base_afe *afe, unsigned int cg_type)
+{
+	int ret;
+	unsigned int reg = get_top_cg_reg(cg_type);
+	unsigned int mask = get_top_cg_mask(cg_type);
+	unsigned int val = get_top_cg_on_val(cg_type);
+
+	if (!afe->regmap) {
+		dev_err(afe->dev, "afe regmap is null !!!\n");
+		return 0;
+	}
+
+	dev_dbg(afe->dev, "reg: 0x%x, mask: 0x%x, val: 0x%x\n", reg, mask, val);
+
+	ret = regmap_update_bits(afe->regmap, reg, mask, val);
+	if (ret)
+		dev_err(afe->dev, "regmap_update_bits failed: %d\n", ret);
+
+	return ret;
+}
+
+static int mt8196_afe_disable_top_cg(struct mtk_base_afe *afe, unsigned int cg_type)
+{
+	int ret;
+	unsigned int reg = get_top_cg_reg(cg_type);
+	unsigned int mask = get_top_cg_mask(cg_type);
+	unsigned int val = get_top_cg_off_val(cg_type);
+
+	if (!afe->regmap) {
+		dev_err(afe->dev, "afe regmap is null !!!\n");
+		return 0;
+	}
+
+	dev_dbg(afe->dev, "reg: 0x%x, mask: 0x%x, val: 0x%x\n", reg, mask, val);
+
+	ret = regmap_update_bits(afe->regmap, reg, mask, val);
+	if (ret)
+		dev_err(afe->dev, "regmap_update_bits failed: %d\n", ret);
+
+	return ret;
+}
+
+static int apll1_mux_setting(struct mtk_base_afe *afe, bool enable)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int apll_rate;
+	int ret;
+
+	dev_dbg(afe->dev, "enable: %d\n", enable);
+
+	if (enable) {
+		apll_rate = mt8196_get_apll_rate(afe, MT8196_APLL1);
+
+		/* 180.6336 / 4 = 45.1584MHz */
+		ret = mt8196_afe_enable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUD_ENG1]);
+		if (ret)
+			return ret;
+
+		ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUD_ENG1],
+					      MT8196_AUD_ENG1_CLK);
+		if (ret)
+			return ret;
+
+		ret = mt8196_afe_enable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H]);
+		if (ret)
+			return ret;
+
+		ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H],
+					      apll_rate);
+		if (ret)
+			return ret;
+	} else {
+		ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUD_ENG1],
+					      MT8196_AFE_26M);
+		if (ret)
+			return ret;
+
+		mt8196_afe_disable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUD_ENG1]);
+
+		ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H],
+					      MT8196_AFE_26M);
+		if (ret)
+			return ret;
+
+		mt8196_afe_disable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H]);
+	}
+
+	return 0;
+}
+
+static int apll2_mux_setting(struct mtk_base_afe *afe, bool enable)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int apll_rate;
+	int ret;
+
+	dev_dbg(afe->dev, "enable: %d\n", enable);
+
+	if (enable) {
+		apll_rate = mt8196_get_apll_rate(afe, MT8196_APLL2);
+
+		/* 196.608 / 4 = 49.152MHz */
+		ret = mt8196_afe_enable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUD_ENG2]);
+		if (ret)
+			return ret;
+
+		ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUD_ENG2],
+					      MT8196_AUD_ENG2_CLK);
+		if (ret)
+			return ret;
+
+		ret = mt8196_afe_enable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H]);
+		if (ret)
+			return ret;
+
+		ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H],
+					      apll_rate);
+		if (ret)
+			return ret;
+	} else {
+		ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUD_ENG2],
+					      MT8196_AFE_26M);
+		if (ret)
+			return ret;
+
+		mt8196_afe_disable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUD_ENG2]);
+
+		ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H],
+					      MT8196_AFE_26M);
+		if (ret)
+			return ret;
+
+		mt8196_afe_disable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H]);
+	}
+
+	return 0;
+}
+
+int mt8196_apll1_enable(struct mtk_base_afe *afe)
+{
+	int ret;
+
+	/* setting for APLL */
+	apll1_mux_setting(afe, true);
+
+	ret = mt8196_afe_enable_top_cg(afe, MT8196_CG_APLL1_CK);
+	if (ret)
+		goto err_clk_apll1;
+
+	ret = mt8196_afe_enable_top_cg(afe, MT8196_PDN_APLL_TUNER1);
+	if (ret)
+		goto err_clk_apll1_tuner;
+
+	/* sel 44.1kHz:1, apll_div:7, upper bound:3 */
+	regmap_update_bits(afe->regmap, AFE_APLL1_TUNER_CFG,
+			   XTAL_EN_128FS_SEL_MASK_SFT | APLL_DIV_MASK_SFT | UPPER_BOUND_MASK_SFT,
+			   (0x1 << XTAL_EN_128FS_SEL_SFT) | (7 << APLL_DIV_SFT) |
+			   (3 << UPPER_BOUND_SFT));
+
+	/* apll1 freq tuner enable */
+	regmap_update_bits(afe->regmap, AFE_APLL1_TUNER_CFG,
+			   FREQ_TUNER_EN_MASK_SFT,
+			   0x1 << FREQ_TUNER_EN_SFT);
+
+	/* audio apll1 on */
+	mt8196_afe_enable_top_cg(afe, MT8196_AUDIO_APLL1_EN_ON);
+
+	return 0;
+
+err_clk_apll1_tuner:
+	mt8196_afe_disable_top_cg(afe, MT8196_PDN_APLL_TUNER1);
+err_clk_apll1:
+	mt8196_afe_disable_top_cg(afe, MT8196_CG_APLL1_CK);
+	return ret;
+}
+
+void mt8196_apll1_disable(struct mtk_base_afe *afe)
+{
+	/* audio apll1 off */
+	mt8196_afe_disable_top_cg(afe, MT8196_AUDIO_APLL1_EN_ON);
+
+	/* apll1 freq tuner disable */
+	regmap_update_bits(afe->regmap, AFE_APLL1_TUNER_CFG,
+			   FREQ_TUNER_EN_MASK_SFT,
+			   0x0);
+
+	mt8196_afe_disable_top_cg(afe, MT8196_PDN_APLL_TUNER1);
+	mt8196_afe_disable_top_cg(afe, MT8196_CG_APLL1_CK);
+	apll1_mux_setting(afe, false);
+}
+
+int mt8196_apll2_enable(struct mtk_base_afe *afe)
+{
+	int ret;
+
+	/* setting for APLL */
+	apll2_mux_setting(afe, true);
+
+	ret = mt8196_afe_enable_top_cg(afe, MT8196_CG_APLL2_CK);
+	if (ret)
+		goto err_clk_apll2;
+
+	ret = mt8196_afe_enable_top_cg(afe, MT8196_PDN_APLL_TUNER2);
+	if (ret)
+		goto err_clk_apll2_tuner;
+
+	/* sel 48kHz: 2, apll_div: 7, upper bound: 3*/
+	regmap_update_bits(afe->regmap, AFE_APLL2_TUNER_CFG,
+			   XTAL_EN_128FS_SEL_MASK_SFT | APLL_DIV_MASK_SFT | UPPER_BOUND_MASK_SFT,
+			   (0x2 << XTAL_EN_128FS_SEL_SFT) | (7 << APLL_DIV_SFT) |
+			   (3 << UPPER_BOUND_SFT));
+
+	/* apll2 freq tuner enable */
+	regmap_update_bits(afe->regmap, AFE_APLL2_TUNER_CFG,
+			   FREQ_TUNER_EN_MASK_SFT,
+			   0x1 << FREQ_TUNER_EN_SFT);
+
+	/* audio apll2 on */
+	mt8196_afe_enable_top_cg(afe, MT8196_AUDIO_APLL2_EN_ON);
+	return 0;
+
+err_clk_apll2_tuner:
+	mt8196_afe_disable_top_cg(afe, MT8196_PDN_APLL_TUNER2);
+err_clk_apll2:
+	mt8196_afe_disable_top_cg(afe, MT8196_CG_APLL2_CK);
+	return 0;
+}
+
+void mt8196_apll2_disable(struct mtk_base_afe *afe)
+{
+	/* audio apll2 off */
+	mt8196_afe_disable_top_cg(afe, MT8196_AUDIO_APLL2_EN_ON);
+
+	/* apll2 freq tuner disable */
+	regmap_update_bits(afe->regmap, AFE_APLL2_TUNER_CFG,
+			   FREQ_TUNER_EN_MASK_SFT,
+			   0x0);
+
+	mt8196_afe_disable_top_cg(afe, MT8196_PDN_APLL_TUNER2);
+	mt8196_afe_disable_top_cg(afe, MT8196_CG_APLL2_CK);
+	apll2_mux_setting(afe, false);
+}
+
+int mt8196_get_apll_rate(struct mtk_base_afe *afe, int apll)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int clk_id = 0;
+
+	if (apll < MT8196_APLL1 || apll > MT8196_APLL2) {
+		dev_warn(afe->dev, "invalid clk id %d\n", apll);
+		return 0;
+	}
+
+	if (apll == MT8196_APLL1)
+		clk_id = MT8196_CLK_TOP_APLL1_CK;
+	else
+		clk_id = MT8196_CLK_TOP_APLL2_CK;
+
+	return clk_get_rate(afe_priv->clk[clk_id]);
+}
+
+/* 48K: select APLL2; 44.1k: select APLL1 */
+int mt8196_get_apll_by_rate(struct mtk_base_afe *afe, int rate)
+{
+	return (rate % 8000) ? MT8196_APLL1 : MT8196_APLL2;
+}
+
+int mt8196_get_apll_by_name(struct mtk_base_afe *afe, const char *name)
+{
+	if (strcmp(name, APLL1_W_NAME) == 0)
+		return MT8196_APLL1;
+
+	return MT8196_APLL2;
+}
+
+static const int mck_div[MT8196_MCK_NUM] = {
+	[MT8196_I2SIN0_MCK] = MT8196_CLK_TOP_APLL12_DIV_I2SIN0,
+	[MT8196_I2SIN1_MCK] = MT8196_CLK_TOP_APLL12_DIV_I2SIN1,
+	[MT8196_FMI2S_MCK] = MT8196_CLK_TOP_APLL12_DIV_FMI2S,
+	[MT8196_TDMOUT_MCK] = MT8196_CLK_TOP_APLL12_DIV_TDMOUT_M,
+	[MT8196_TDMOUT_BCK] = MT8196_CLK_TOP_APLL12_DIV_TDMOUT_B,
+};
+
+int mt8196_mck_enable(struct mtk_base_afe *afe, int mck_id, int rate)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int div_clk_id;
+	int ret;
+
+	dev_dbg(afe->dev, "mck_id: %d, rate: %d\n", mck_id, rate);
+
+	if (mck_id >= MT8196_MCK_NUM || mck_id < 0)
+		return -EINVAL;
+
+	div_clk_id = mck_div[mck_id];
+
+	/* enable div, set rate */
+	if (div_clk_id < 0) {
+		dev_err(afe->dev, "invalid div_clk_id %d\n", div_clk_id);
+		return -EINVAL;
+	}
+
+	if (div_clk_id == MT8196_CLK_TOP_APLL12_DIV_TDMOUT_B)
+		rate *= 16;
+
+	ret = mt8196_afe_enable_clk(afe, afe_priv->clk[div_clk_id]);
+	if (ret)
+		return ret;
+
+	ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[div_clk_id], rate);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+int mt8196_mck_disable(struct mtk_base_afe *afe, int mck_id)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int div_clk_id;
+	int ret;
+
+	dev_dbg(afe->dev, "mck_id: %d.\n", mck_id);
+
+	if (mck_id < 0) {
+		dev_err(afe->dev, "mck_id = %d < 0\n", mck_id);
+		return -EINVAL;
+	}
+
+	div_clk_id = mck_div[mck_id];
+
+	if (div_clk_id < 0) {
+		dev_err(afe->dev, "div_clk_id = %d < 0\n",
+			div_clk_id);
+		return -EINVAL;
+	}
+
+	ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[div_clk_id], MT8196_AFE_26M);
+	if (ret)
+		return ret;
+
+	mt8196_afe_disable_clk(afe, afe_priv->clk[div_clk_id]);
+
+	return 0;
+}
+
+int mt8196_afe_enable_reg_rw_clk(struct mtk_base_afe *afe)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int ret;
+
+	/* bus clock for AFE external access, like DRAM */
+	mt8196_afe_enable_clk(afe, afe_priv->clk[MT8196_CLK_TOP_ADSP_SEL]);
+
+	/* bus clock for AFE internal access, like AFE SRAM */
+	mt8196_afe_enable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIOINTBUS]);
+	ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIOINTBUS],
+				      MT8196_AFE_26M);
+	if (ret)
+		return ret;
+
+	/* enable audio h clock */
+	mt8196_afe_enable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H]);
+	ret = mt8196_afe_set_clk_rate(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H],
+				      MT8196_AFE_26M);
+	if (ret)
+		return ret;
+
+	/* AFE hw clock */
+	/* IPM2.0: USE HOPPING & 26M */
+	/* set in the regmap_register_patch */
+	return 0;
+}
+
+int mt8196_afe_disable_reg_rw_clk(struct mtk_base_afe *afe)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+
+	/* IPM2.0: Use HOPPING & 26M */
+	/* set in the regmap_register_patch */
+
+	mt8196_afe_disable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIO_H]);
+	mt8196_afe_disable_clk(afe, afe_priv->clk[MT8196_CLK_VLP_MUX_AUDIOINTBUS]);
+	mt8196_afe_disable_clk(afe, afe_priv->clk[MT8196_CLK_TOP_ADSP_SEL]);
+	return 0;
+}
+
+int mt8196_afe_enable_main_clock(struct mtk_base_afe *afe)
+{
+	mt8196_afe_enable_top_cg(afe, MT8196_AUDIO_26M_EN_ON);
+	return 0;
+}
+
+int mt8196_afe_disable_main_clock(struct mtk_base_afe *afe)
+{
+	mt8196_afe_disable_top_cg(afe, MT8196_AUDIO_26M_EN_ON);
+	return 0;
+}
+
+int mt8196_init_clock(struct mtk_base_afe *afe)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int i;
+
+	afe_priv->clk = devm_kcalloc(afe->dev, MT8196_CLK_NUM, sizeof(*afe_priv->clk),
+				     GFP_KERNEL);
+	if (!afe_priv->clk)
+		return -ENOMEM;
+
+	for (i = 0; i < MT8196_CLK_NUM; i++) {
+		afe_priv->clk[i] = devm_clk_get(afe->dev, aud_clks[i]);
+		if (IS_ERR(afe_priv->clk[i])) {
+			dev_err(afe->dev, "devm_clk_get %s fail\n", aud_clks[i]);
+			return PTR_ERR(afe_priv->clk[i]);
+		}
+	}
+
+	return 0;
+}
+
diff --git a/sound/soc/mediatek/mt8196/mt8196-afe-clk.h b/sound/soc/mediatek/mt8196/mt8196-afe-clk.h
new file mode 100644
index 0000000000000..7d47dcff768b2
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-afe-clk.h
@@ -0,0 +1,64 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * mt8196-afe-clk.h  --  Mediatek MT8196 AFE Clock Control definitions
+ *
+ * Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#ifndef _MT8196_AFE_CLOCK_CTRL_H_
+#define _MT8196_AFE_CLOCK_CTRL_H_
+
+#define MT8196_AFE_26M 26000000
+#define MT8196_AUD_ENG1_CLK 45158400
+#define MT8196_AUD_ENG2_CLK 49152000
+
+/* APLL */
+#define APLL1_W_NAME "APLL1"
+#define APLL2_W_NAME "APLL2"
+
+enum {
+	MT8196_APLL1 = 0,
+	MT8196_APLL2,
+};
+
+enum {
+	/* vlp clk */
+	MT8196_CLK_VLP_MUX_AUDIOINTBUS,
+	MT8196_CLK_VLP_MUX_AUD_ENG1,
+	MT8196_CLK_VLP_MUX_AUD_ENG2,
+	MT8196_CLK_VLP_MUX_AUDIO_H,
+	/* pll */
+	MT8196_CLK_TOP_APLL1_CK,
+	MT8196_CLK_TOP_APLL2_CK,
+	/* divider */
+	MT8196_CLK_TOP_APLL12_DIV_I2SIN0,
+	MT8196_CLK_TOP_APLL12_DIV_I2SIN1,
+	MT8196_CLK_TOP_APLL12_DIV_FMI2S,
+	MT8196_CLK_TOP_APLL12_DIV_TDMOUT_M,
+	MT8196_CLK_TOP_APLL12_DIV_TDMOUT_B,
+	/* mux */
+	MT8196_CLK_TOP_ADSP_SEL,
+	MT8196_CLK_NUM,
+};
+
+struct mtk_base_afe;
+
+int mt8196_mck_enable(struct mtk_base_afe *afe, int mck_id, int rate);
+int mt8196_mck_disable(struct mtk_base_afe *afe, int mck_id);
+int mt8196_get_apll_rate(struct mtk_base_afe *afe, int apll);
+int mt8196_get_apll_by_rate(struct mtk_base_afe *afe, int rate);
+int mt8196_get_apll_by_name(struct mtk_base_afe *afe, const char *name);
+int mt8196_init_clock(struct mtk_base_afe *afe);
+int mt8196_afe_enable_clk(struct mtk_base_afe *afe, struct clk *clk);
+void mt8196_afe_disable_clk(struct mtk_base_afe *afe, struct clk *clk);
+int mt8196_apll1_enable(struct mtk_base_afe *afe);
+void mt8196_apll1_disable(struct mtk_base_afe *afe);
+int mt8196_apll2_enable(struct mtk_base_afe *afe);
+void mt8196_apll2_disable(struct mtk_base_afe *afe);
+int mt8196_afe_enable_main_clock(struct mtk_base_afe *afe);
+int mt8196_afe_disable_main_clock(struct mtk_base_afe *afe);
+int mt8196_afe_enable_reg_rw_clk(struct mtk_base_afe *afe);
+int mt8196_afe_disable_reg_rw_clk(struct mtk_base_afe *afe);
+
+#endif
diff --git a/sound/soc/mediatek/mt8196/mt8196-afe-common.h b/sound/soc/mediatek/mt8196/mt8196-afe-common.h
new file mode 100644
index 0000000000000..4951a6f32abbf
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-afe-common.h
@@ -0,0 +1,205 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * mt8196-afe-common.h  --  Mediatek 8196 audio driver definitions
+ *
+ * Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#ifndef _MT_8196_AFE_COMMON_H_
+#define _MT_8196_AFE_COMMON_H_
+#include <sound/pcm.h>
+#include <sound/soc.h>
+
+#include "mt8196-reg.h"
+#include "../common/mtk-base-afe.h"
+
+/* HW IPM 2.0 */
+enum {
+	MTK_AFE_IPM2P0_RATE_8K = 0x0,
+	MTK_AFE_IPM2P0_RATE_11K = 0x1,
+	MTK_AFE_IPM2P0_RATE_12K = 0x2,
+	MTK_AFE_IPM2P0_RATE_16K = 0x4,
+	MTK_AFE_IPM2P0_RATE_22K = 0x5,
+	MTK_AFE_IPM2P0_RATE_24K = 0x6,
+	MTK_AFE_IPM2P0_RATE_32K = 0x8,
+	MTK_AFE_IPM2P0_RATE_44K = 0x9,
+	MTK_AFE_IPM2P0_RATE_48K = 0xa,
+	MTK_AFE_IPM2P0_RATE_88K = 0xd,
+	MTK_AFE_IPM2P0_RATE_96K = 0xe,
+	MTK_AFE_IPM2P0_RATE_176K = 0x11,
+	MTK_AFE_IPM2P0_RATE_192K = 0x12,
+	MTK_AFE_IPM2P0_RATE_352K = 0x15,
+	MTK_AFE_IPM2P0_RATE_384K = 0x16,
+};
+
+enum {
+	MTKAIF_PROTOCOL_1,
+	MTKAIF_PROTOCOL_2,
+	MTKAIF_PROTOCOL_2_CLK_P2,
+};
+
+enum {
+	MT8196_MEMIF_DL0,
+	MT8196_MEMIF_DL1,
+	MT8196_MEMIF_DL2,
+	MT8196_MEMIF_DL3,
+	MT8196_MEMIF_DL4,
+	MT8196_MEMIF_DL5,
+	MT8196_MEMIF_DL6,
+	MT8196_MEMIF_DL7,
+	MT8196_MEMIF_DL8,
+	MT8196_MEMIF_DL23,
+	MT8196_MEMIF_DL24,
+	MT8196_MEMIF_DL25,
+	MT8196_MEMIF_DL26,
+	MT8196_MEMIF_DL_4CH,
+	MT8196_MEMIF_DL_24CH,
+	MT8196_MEMIF_VUL0,
+	MT8196_MEMIF_VUL1,
+	MT8196_MEMIF_VUL2,
+	MT8196_MEMIF_VUL3,
+	MT8196_MEMIF_VUL4,
+	MT8196_MEMIF_VUL5,
+	MT8196_MEMIF_VUL6,
+	MT8196_MEMIF_VUL7,
+	MT8196_MEMIF_VUL8,
+	MT8196_MEMIF_VUL9,
+	MT8196_MEMIF_VUL10,
+	MT8196_MEMIF_VUL24,
+	MT8196_MEMIF_VUL25,
+	MT8196_MEMIF_VUL26,
+	MT8196_MEMIF_VUL_CM0,
+	MT8196_MEMIF_VUL_CM1,
+	MT8196_MEMIF_VUL_CM2,
+	MT8196_MEMIF_ETDM_IN0,
+	MT8196_MEMIF_ETDM_IN1,
+	MT8196_MEMIF_ETDM_IN2,
+	MT8196_MEMIF_ETDM_IN3,
+	MT8196_MEMIF_ETDM_IN4,
+	MT8196_MEMIF_ETDM_IN6,
+	MT8196_MEMIF_HDMI,
+	MT8196_MEMIF_NUM,
+	MT8196_DAI_ADDA = MT8196_MEMIF_NUM,
+	MT8196_DAI_ADDA_CH34,
+	MT8196_DAI_ADDA_CH56,
+	MT8196_DAI_AP_DMIC,
+	MT8196_DAI_AP_DMIC_CH34,
+	MT8196_DAI_AP_DMIC_MULTICH,
+	MT8196_DAI_I2S_IN0,
+	MT8196_DAI_I2S_IN1,
+	MT8196_DAI_I2S_IN2,
+	MT8196_DAI_I2S_IN3,
+	MT8196_DAI_I2S_IN4,
+	MT8196_DAI_I2S_IN6,
+	MT8196_DAI_I2S_OUT0,
+	MT8196_DAI_I2S_OUT1,
+	MT8196_DAI_I2S_OUT2,
+	MT8196_DAI_I2S_OUT3,
+	MT8196_DAI_I2S_OUT4,
+	MT8196_DAI_I2S_OUT6,
+	MT8196_DAI_FM_I2S_MASTER,
+	MT8196_DAI_TDM,
+	MT8196_DAI_TDM_DPTX,
+	MT8196_DAI_NUM,
+};
+
+#define MT8196_DAI_I2S_MAX_NUM 13 //depends each platform's max i2s num
+
+/* update irq ID (= enum) from AFE_IRQ_MCU_STATUS */
+enum {
+	MT8196_IRQ_0,
+	MT8196_IRQ_1,
+	MT8196_IRQ_2,
+	MT8196_IRQ_3,
+	MT8196_IRQ_4,
+	MT8196_IRQ_5,
+	MT8196_IRQ_6,
+	MT8196_IRQ_7,
+	MT8196_IRQ_8,
+	MT8196_IRQ_9,
+	MT8196_IRQ_10,
+	MT8196_IRQ_11,
+	MT8196_IRQ_12,
+	MT8196_IRQ_13,
+	MT8196_IRQ_14,
+	MT8196_IRQ_15,
+	MT8196_IRQ_16,
+	MT8196_IRQ_17,
+	MT8196_IRQ_18,
+	MT8196_IRQ_19,
+	MT8196_IRQ_20,
+	MT8196_IRQ_21,
+	MT8196_IRQ_22,
+	MT8196_IRQ_23,
+	MT8196_IRQ_24,
+	MT8196_IRQ_25,
+	MT8196_IRQ_26,
+	MT8196_IRQ_31,  /* used only for TDM */
+	MT8196_IRQ_NUM,
+};
+
+/* update irq ID (= enum) from AFE_IRQ_MCU_STATUS */
+enum {
+	MT8196_CUS_IRQ_TDM,  /* used only for TDM */
+	MT8196_CUS_IRQ_NUM,
+};
+
+enum {
+	/* AUDIO_ENGEN_CON0 */
+	MT8196_AUDIO_26M_EN_ON,
+	MT8196_AUDIO_F3P25M_EN_ON,
+	MT8196_AUDIO_APLL1_EN_ON,
+	MT8196_AUDIO_APLL2_EN_ON,
+	MT8196_AUDIO_F26M_EN_RST,
+	MT8196_MULTI_USER_RST,
+	MT8196_MULTI_USER_BYPASS,
+	/* AUDIO_TOP_CON4 */
+	MT8196_CG_AUDIO_HOPPING_CK,
+	MT8196_CG_AUDIO_F26M_CK,
+	MT8196_CG_APLL1_CK,
+	MT8196_CG_APLL2_CK,
+	MT8196_PDN_APLL_TUNER2,
+	MT8196_PDN_APLL_TUNER1,
+	MT8196_AUDIO_CG_NUM,
+};
+
+/* MCLK */
+enum {
+	MT8196_I2SIN0_MCK,
+	MT8196_I2SIN1_MCK,
+	MT8196_FMI2S_MCK,
+	MT8196_TDMOUT_MCK,
+	MT8196_TDMOUT_BCK,
+	MT8196_MCK_NUM,
+};
+
+/* CM*/
+enum {
+	CM0,
+	CM1,
+	CM2,
+	CM_NUM,
+};
+
+struct clk;
+struct mtk_base_afe;
+
+struct mt8196_afe_private {
+	struct clk **clk;
+	/* dai */
+	void *dai_priv[MT8196_DAI_NUM];
+	/* mck */
+	int mck_rate[MT8196_MCK_NUM];
+	/* channel merge */
+	u32 cm_rate[CM_NUM];
+	u32 cm_channels;
+};
+
+int mt8196_dai_adda_register(struct mtk_base_afe *afe);
+int mt8196_dai_i2s_register(struct mtk_base_afe *afe);
+int mt8196_dai_tdm_register(struct mtk_base_afe *afe);
+int mt8196_dai_set_priv(struct mtk_base_afe *afe, int id,
+			int priv_size, const void *priv_data);
+
+#endif
diff --git a/sound/soc/mediatek/mt8196/mt8196-afe-pcm.c b/sound/soc/mediatek/mt8196/mt8196-afe-pcm.c
new file mode 100644
index 0000000000000..a1ae8322d8b6c
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-afe-pcm.c
@@ -0,0 +1,2519 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  Mediatek ALSA SoC AFE platform driver for 8196
+ *
+ *  Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_device.h>
+#include <linux/of_reserved_mem.h>
+#include <linux/pm_runtime.h>
+#include <linux/regmap.h>
+
+#include <sound/pcm.h>
+#include <sound/soc.h>
+
+#include "mt8196-afe-clk.h"
+#include "mt8196-afe-common.h"
+#include "mt8196-interconnection.h"
+
+#include "../common/mtk-afe-fe-dai.h"
+#include "../common/mtk-afe-platform-driver.h"
+
+static const struct snd_pcm_hardware mt8196_afe_hardware = {
+	.info = (SNDRV_PCM_INFO_MMAP |
+		 SNDRV_PCM_INFO_NO_PERIOD_WAKEUP |
+		 SNDRV_PCM_INFO_INTERLEAVED |
+		 SNDRV_PCM_INFO_MMAP_VALID),
+	.formats = (SNDRV_PCM_FMTBIT_S16_LE |
+		    SNDRV_PCM_FMTBIT_S24_LE |
+		    SNDRV_PCM_FMTBIT_S32_LE),
+	.period_bytes_min = 96,
+	.period_bytes_max = 4 * 48 * 1024,
+	.periods_min = 2,
+	.periods_max = 256,
+	.buffer_bytes_max = 256 * 1024,
+	.fifo_size = 0,
+};
+
+static unsigned int mt8196_rate_transform(struct device *dev,
+					  unsigned int rate)
+{
+	switch (rate) {
+	case 8000:
+		return MTK_AFE_IPM2P0_RATE_8K;
+	case 11025:
+		return MTK_AFE_IPM2P0_RATE_11K;
+	case 12000:
+		return MTK_AFE_IPM2P0_RATE_12K;
+	case 16000:
+		return MTK_AFE_IPM2P0_RATE_16K;
+	case 22050:
+		return MTK_AFE_IPM2P0_RATE_22K;
+	case 24000:
+		return MTK_AFE_IPM2P0_RATE_24K;
+	case 32000:
+		return MTK_AFE_IPM2P0_RATE_32K;
+	case 44100:
+		return MTK_AFE_IPM2P0_RATE_44K;
+	case 48000:
+		return MTK_AFE_IPM2P0_RATE_48K;
+	case 88200:
+		return MTK_AFE_IPM2P0_RATE_88K;
+	case 96000:
+		return MTK_AFE_IPM2P0_RATE_96K;
+	case 176400:
+		return MTK_AFE_IPM2P0_RATE_176K;
+	case 192000:
+		return MTK_AFE_IPM2P0_RATE_192K;
+	/* not support 260K */
+	case 352800:
+		return MTK_AFE_IPM2P0_RATE_352K;
+	case 384000:
+		return MTK_AFE_IPM2P0_RATE_384K;
+	default:
+		dev_err(dev, "rate %u invalid, use %d!!!\n",
+			rate, MTK_AFE_IPM2P0_RATE_48K);
+		return MTK_AFE_IPM2P0_RATE_48K;
+	}
+}
+
+static int mt8196_set_cm(struct mtk_base_afe *afe, int id,
+			 bool update, bool swap, unsigned int ch)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	unsigned int rate = afe_priv->cm_rate[id];
+	unsigned int rate_val = mt8196_rate_transform(afe->dev, rate);
+	unsigned int ch_pair = ch / 2;
+	unsigned int update_val;
+	int reg = AFE_CM0_CON0 + 0x10 * id;
+
+	if (update && ch_pair)
+		update_val = (26000000 / rate - 10) / ch_pair - 1;
+	else
+		update_val = 0x64;
+
+	dev_dbg(afe->dev, "CM%d, rate %d, update %d, swap %d, ch %d, update_val: %d\n",
+		id, rate, update, swap, ch, update_val);
+
+	/* update cnt */
+	regmap_update_bits(afe->regmap, reg,
+			   AFE_CM_UPDATE_CNT_MASK << AFE_CM_UPDATE_CNT_SFT,
+			   update_val << AFE_CM_UPDATE_CNT_SFT);
+
+	/* rate */
+	regmap_update_bits(afe->regmap, reg,
+			   AFE_CM_1X_EN_SEL_FS_MASK << AFE_CM_1X_EN_SEL_FS_SFT,
+			   rate_val << AFE_CM_1X_EN_SEL_FS_SFT);
+
+	/* ch num */
+	ch = ch - 1;
+	regmap_update_bits(afe->regmap, reg,
+			   AFE_CM_CH_NUM_MASK << AFE_CM_CH_NUM_SFT,
+			   ch << AFE_CM_CH_NUM_SFT);
+
+	/* swap */
+	regmap_update_bits(afe->regmap, reg,
+			   AFE_CM_BYTE_SWAP_MASK << AFE_CM_BYTE_SWAP_SFT,
+			   swap << AFE_CM_BYTE_SWAP_SFT);
+
+	return 0;
+}
+
+static int mt8196_enable_cm_bypass(struct mtk_base_afe *afe, int id, bool en)
+{
+	return regmap_update_bits(afe->regmap,
+				  AFE_CM0_CON0 + 0x10 * id,
+				  AFE_CM_BYPASS_MODE_MASK << AFE_CM_BYPASS_MODE_SFT,
+				  en << AFE_CM_BYPASS_MODE_SFT);
+}
+
+static int mt8196_fe_startup(struct snd_pcm_substream *substream,
+			     struct snd_soc_dai *dai)
+{
+	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct snd_pcm_runtime *runtime = substream->runtime;
+	struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0);
+	int memif_num = cpu_dai->id;
+	struct mtk_base_afe_memif *memif = &afe->memif[memif_num];
+	const struct snd_pcm_hardware *mtk_afe_hardware = afe->mtk_afe_hardware;
+	int ret;
+
+	dev_dbg(afe->dev, "memif_num: %d.\n", memif_num);
+
+	memif->substream = substream;
+
+	snd_pcm_hw_constraint_step(substream->runtime, 0,
+				   SNDRV_PCM_HW_PARAM_BUFFER_BYTES, 16);
+
+	if (memif_num == MT8196_MEMIF_VUL_CM0)
+		snd_pcm_hw_constraint_step(substream->runtime, 0,
+					   SNDRV_PCM_HW_PARAM_PERIOD_BYTES, 16);
+
+	snd_soc_set_runtime_hwparams(substream, mtk_afe_hardware);
+
+	ret = snd_pcm_hw_constraint_integer(runtime,
+					    SNDRV_PCM_HW_PARAM_PERIODS);
+	if (ret < 0)
+		dev_warn(afe->dev, "snd_pcm_hw_constraint_integer failed\n");
+
+	/* dynamic allocate irq to memif */
+	if (memif->irq_usage < 0) {
+		int irq_id = mtk_dynamic_irq_acquire(afe);
+
+		if (irq_id != afe->irqs_size) {
+			/* link */
+			memif->irq_usage = irq_id;
+		} else {
+			dev_err(afe->dev, "no more asys irq\n");
+			ret = -EBUSY;
+		}
+	}
+	return ret;
+}
+
+static void mt8196_fe_shutdown(struct snd_pcm_substream *substream,
+			       struct snd_soc_dai *dai)
+{
+	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0);
+	int memif_num = cpu_dai->id;
+	struct mtk_base_afe_memif *memif = &afe->memif[memif_num];
+	int irq_id = memif->irq_usage;
+
+	dev_dbg(afe->dev, "memif_num: %d.\n", memif_num);
+
+	memif->substream = NULL;
+
+	if (!memif->const_irq) {
+		mtk_dynamic_irq_release(afe, irq_id);
+		memif->irq_usage = -1;
+		memif->substream = NULL;
+	}
+}
+
+static int mt8196_fe_hw_params(struct snd_pcm_substream *substream,
+			       struct snd_pcm_hw_params *params,
+			       struct snd_soc_dai *dai)
+{
+	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	unsigned int channels = params_channels(params);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int id = snd_soc_rtd_to_cpu(rtd, 0)->id;
+	struct mtk_base_afe_memif *memif = &afe->memif[id];
+	const struct mtk_base_memif_data *data = memif->data;
+	int cm;
+
+	afe_priv->cm_channels = channels;
+
+	/* set channels */
+	if (data->ch_num_shift >= 0) {
+		regmap_update_bits(afe->regmap, data->ch_num_reg,
+				   data->ch_num_maskbit << data->ch_num_shift,
+				   channels << data->ch_num_shift);
+	}
+
+	switch (id) {
+	case MT8196_MEMIF_VUL8:
+	case MT8196_MEMIF_VUL_CM0:
+		cm = CM0;
+		break;
+	case MT8196_MEMIF_VUL9:
+	case MT8196_MEMIF_VUL_CM1:
+		cm = CM1;
+		break;
+	case MT8196_MEMIF_VUL10:
+	case MT8196_MEMIF_VUL_CM2:
+		cm = CM2;
+		break;
+	default:
+		cm = CM0;
+		break;
+	}
+
+	afe_priv->cm_rate[cm] = params_rate(params);
+
+	return mtk_afe_fe_hw_params(substream, params, dai);
+}
+
+static int mt8196_fe_trigger(struct snd_pcm_substream *substream, int cmd,
+			     struct snd_soc_dai *dai)
+{
+	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
+	struct snd_pcm_runtime *const runtime = substream->runtime;
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	int id = snd_soc_rtd_to_cpu(rtd, 0)->id;
+	struct mtk_base_afe_memif *memif = &afe->memif[id];
+	int irq_id = memif->irq_usage;
+	struct mtk_base_afe_irq *irqs = &afe->irqs[irq_id];
+	const struct mtk_base_irq_data *irq_data = irqs->irq_data;
+	unsigned int counter = runtime->period_size;
+	unsigned int rate = runtime->rate;
+	unsigned int tmp_reg;
+	int fs;
+	int ret;
+
+	dev_dbg(afe->dev, "%s cmd %d, irq_id %d\n", memif->data->name, cmd, irq_id);
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+		dev_dbg(afe->dev, "%s cmd %d, id %d\n", memif->data->name, cmd, id);
+
+		ret = mtk_memif_set_enable(afe, id);
+		if (ret) {
+			dev_err(afe->dev, "id %d, memif enable fail.\n", id);
+			return ret;
+		}
+
+		/*
+		 * for small latency record
+		 * ul memif need read some data before irq enable.
+		 * the context of this ops triger is atmoic, so it cannot sleep.
+		 */
+		if (substream->stream == SNDRV_PCM_STREAM_CAPTURE)
+			if ((runtime->period_size * 1000) / rate <= 10)
+				udelay(300);
+
+		regmap_update_bits(afe->regmap,
+				   irq_data->irq_cnt_reg,
+				   irq_data->irq_cnt_maskbit << irq_data->irq_cnt_shift,
+				   counter << irq_data->irq_cnt_shift);
+
+		/* set irq fs */
+		fs = afe->irq_fs(substream, rate);
+		if (fs < 0)
+			return -EINVAL;
+
+		if (irq_data->irq_fs_reg >= 0)
+			regmap_update_bits(afe->regmap,
+					   irq_data->irq_fs_reg,
+					   irq_data->irq_fs_maskbit << irq_data->irq_fs_shift,
+					   fs << irq_data->irq_fs_shift);
+
+		/* enable interrupt */
+		regmap_update_bits(afe->regmap,
+				   irq_data->irq_en_reg,
+				   1 << irq_data->irq_en_shift,
+				   1 << irq_data->irq_en_shift);
+
+		return 0;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+		ret = mtk_memif_set_disable(afe, id);
+		if (ret)
+			dev_warn(afe->dev, "id %d, memif disable fail\n", id);
+
+		/* disable interrupt */
+		regmap_update_bits(afe->regmap,
+				   irq_data->irq_en_reg,
+				   1 << irq_data->irq_en_shift,
+				   0 << irq_data->irq_en_shift);
+
+		/*
+		 * clear pending IRQ, if the register read as one, there is no need to write
+		 * one to clear operaton.
+		 */
+		regmap_read(afe->regmap, irq_data->irq_clr_reg, &tmp_reg);
+		regmap_update_bits(afe->regmap, irq_data->irq_clr_reg,
+				   AFE_IRQ_CLR_CFG_MASK_SFT | AFE_IRQ_MISS_FLAG_CLR_CFG_MASK_SFT,
+				   tmp_reg ^ (AFE_IRQ_CLR_CFG_MASK_SFT |
+				   AFE_IRQ_MISS_FLAG_CLR_CFG_MASK_SFT));
+
+		return ret;
+	default:
+		return -EINVAL;
+	}
+}
+
+static int mt8196_memif_fs(struct snd_pcm_substream *substream,
+			   unsigned int rate)
+{
+	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = NULL;
+	unsigned int rate_reg;
+
+	if (!component)
+		return -EINVAL;
+
+	afe = snd_soc_component_get_drvdata(component);
+	if (!afe)
+		return -EINVAL;
+
+	rate_reg = mt8196_rate_transform(afe->dev, rate);
+
+	return rate_reg;
+}
+
+static int mt8196_get_dai_fs(struct mtk_base_afe *afe,
+			     int dai_id, unsigned int rate)
+{
+	return mt8196_rate_transform(afe->dev, rate);
+}
+
+static int mt8196_irq_fs(struct snd_pcm_substream *substream, unsigned int rate)
+{
+	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
+	struct snd_soc_component *component =
+		snd_soc_rtdcom_lookup(rtd, AFE_PCM_NAME);
+	struct mtk_base_afe *afe = NULL;
+
+	if (!component)
+		return -EINVAL;
+	afe = snd_soc_component_get_drvdata(component);
+	return mt8196_rate_transform(afe->dev, rate);
+}
+
+static int mt8196_get_memif_pbuf_size(struct snd_pcm_substream *substream)
+{
+	struct snd_pcm_runtime *runtime = substream->runtime;
+
+	if ((runtime->period_size * 1000) / runtime->rate > 10)
+		return MT8196_MEMIF_PBUF_SIZE_256_BYTES;
+	else
+		return MT8196_MEMIF_PBUF_SIZE_32_BYTES;
+}
+
+/* FE DAIs */
+static const struct snd_soc_dai_ops mt8196_memif_dai_ops = {
+	.startup        = mt8196_fe_startup,
+	.shutdown       = mt8196_fe_shutdown,
+	.hw_params      = mt8196_fe_hw_params,
+	.hw_free        = mtk_afe_fe_hw_free,
+	.prepare        = mtk_afe_fe_prepare,
+	.trigger        = mt8196_fe_trigger,
+};
+
+#define MTK_PCM_RATES (SNDRV_PCM_RATE_8000_48000 |\
+		       SNDRV_PCM_RATE_88200 |\
+		       SNDRV_PCM_RATE_96000 |\
+		       SNDRV_PCM_RATE_176400 |\
+		       SNDRV_PCM_RATE_192000)
+
+#define MTK_PCM_DAI_RATES (SNDRV_PCM_RATE_8000 |\
+			   SNDRV_PCM_RATE_16000 |\
+			   SNDRV_PCM_RATE_32000 |\
+			   SNDRV_PCM_RATE_48000)
+
+#define MTK_PCM_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\
+			 SNDRV_PCM_FMTBIT_S24_LE |\
+			 SNDRV_PCM_FMTBIT_S32_LE)
+
+#define MT8196_FE_DAI(_name, _id, max_ch, dir) \
+{ \
+	.name = #_name, \
+	.id = _id, \
+	.dir = { \
+		.stream_name = #_name, \
+		.channels_min = 1, \
+		.channels_max = max_ch, \
+		.rates = MTK_PCM_RATES, \
+		.formats = MTK_PCM_FORMATS, \
+	}, \
+	.ops = &mt8196_memif_dai_ops, \
+}
+
+static struct snd_soc_dai_driver mt8196_memif_dai_driver[] = {
+	/* FE DAIs: memory intefaces to CPU */
+	/* Playback */
+	MT8196_FE_DAI(DL0, MT8196_MEMIF_DL0, 2, playback),
+	MT8196_FE_DAI(DL1, MT8196_MEMIF_DL1, 2, playback),
+	MT8196_FE_DAI(DL2, MT8196_MEMIF_DL2, 2, playback),
+	MT8196_FE_DAI(DL3, MT8196_MEMIF_DL3, 2, playback),
+	MT8196_FE_DAI(DL4, MT8196_MEMIF_DL4, 2, playback),
+	MT8196_FE_DAI(DL5, MT8196_MEMIF_DL5, 2, playback),
+	MT8196_FE_DAI(DL6, MT8196_MEMIF_DL6, 2, playback),
+	MT8196_FE_DAI(DL7, MT8196_MEMIF_DL7, 2, playback),
+	MT8196_FE_DAI(DL8, MT8196_MEMIF_DL8, 2, playback),
+	MT8196_FE_DAI(DL23, MT8196_MEMIF_DL23, 2, playback),
+	MT8196_FE_DAI(DL24, MT8196_MEMIF_DL24, 2, playback),
+	MT8196_FE_DAI(DL25, MT8196_MEMIF_DL25, 2, playback),
+	MT8196_FE_DAI(DL26, MT8196_MEMIF_DL26, 2, playback),
+	MT8196_FE_DAI(DL_4CH, MT8196_MEMIF_DL_4CH, 4, playback),
+	MT8196_FE_DAI(DL_24CH, MT8196_MEMIF_DL_24CH, 8, playback),
+	MT8196_FE_DAI(HDMI, MT8196_MEMIF_HDMI, 8, playback),
+	/* Capture */
+	MT8196_FE_DAI(UL0, MT8196_MEMIF_VUL0, 2, capture),
+	MT8196_FE_DAI(UL1, MT8196_MEMIF_VUL1, 2, capture),
+	MT8196_FE_DAI(UL2, MT8196_MEMIF_VUL2, 2, capture),
+	MT8196_FE_DAI(UL3, MT8196_MEMIF_VUL3, 2, capture),
+	MT8196_FE_DAI(UL4, MT8196_MEMIF_VUL4, 2, capture),
+	MT8196_FE_DAI(UL5, MT8196_MEMIF_VUL5, 2, capture),
+	MT8196_FE_DAI(UL6, MT8196_MEMIF_VUL6, 2, capture),
+	MT8196_FE_DAI(UL7, MT8196_MEMIF_VUL7, 2, capture),
+	MT8196_FE_DAI(UL8, MT8196_MEMIF_VUL8, 2, capture),
+	MT8196_FE_DAI(UL9, MT8196_MEMIF_VUL9, 16, capture),
+	MT8196_FE_DAI(UL10, MT8196_MEMIF_VUL10, 2, capture),
+	MT8196_FE_DAI(UL24, MT8196_MEMIF_VUL24, 2, capture),
+	MT8196_FE_DAI(UL25, MT8196_MEMIF_VUL25, 2, capture),
+	MT8196_FE_DAI(UL26, MT8196_MEMIF_VUL26, 2, capture),
+	MT8196_FE_DAI(UL_CM0, MT8196_MEMIF_VUL_CM0, 8, capture),
+	MT8196_FE_DAI(UL_CM1, MT8196_MEMIF_VUL_CM1, 16, capture),
+	MT8196_FE_DAI(UL_CM2, MT8196_MEMIF_VUL_CM2, 32, capture),
+	MT8196_FE_DAI(UL_ETDM_IN0, MT8196_MEMIF_ETDM_IN0, 2, capture),
+	MT8196_FE_DAI(UL_ETDM_IN1, MT8196_MEMIF_ETDM_IN1, 2, capture),
+	MT8196_FE_DAI(UL_ETDM_IN2, MT8196_MEMIF_ETDM_IN2, 2, capture),
+	MT8196_FE_DAI(UL_ETDM_IN3, MT8196_MEMIF_ETDM_IN3, 2, capture),
+	MT8196_FE_DAI(UL_ETDM_IN4, MT8196_MEMIF_ETDM_IN4, 2, capture),
+	MT8196_FE_DAI(UL_ETDM_IN6, MT8196_MEMIF_ETDM_IN6, 2, capture),
+};
+
+static int ul_cm0_event(struct snd_soc_dapm_widget *w,
+			struct snd_kcontrol *kcontrol,
+			int event)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(w->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	unsigned int channels = afe_priv->cm_channels;
+
+	dev_dbg(afe->dev, "event 0x%x, name %s, channels %u\n",
+		event, w->name, channels);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		mt8196_enable_cm_bypass(afe, CM0, false);
+		mt8196_set_cm(afe, CM0, true, false, channels);
+		regmap_update_bits(afe->regmap, AUDIO_TOP_CON0,
+				   PDN_CM0_MASK_SFT, 0 << PDN_CM0_SFT);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		mt8196_enable_cm_bypass(afe, CM0, true);
+		regmap_update_bits(afe->regmap, AUDIO_TOP_CON0,
+				   PDN_CM0_MASK_SFT, 1 << PDN_CM0_SFT);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int ul_cm1_event(struct snd_soc_dapm_widget *w,
+			struct snd_kcontrol *kcontrol,
+			int event)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(w->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	unsigned int channels = afe_priv->cm_channels;
+
+	dev_dbg(afe->dev, "event 0x%x, name %s, channels %u\n",
+		event, w->name, channels);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		mt8196_enable_cm_bypass(afe, CM1, false);
+		mt8196_set_cm(afe, CM1, true, false, channels);
+		regmap_update_bits(afe->regmap, AUDIO_TOP_CON0,
+				   PDN_CM1_MASK_SFT, 0 << PDN_CM1_SFT);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		mt8196_enable_cm_bypass(afe, CM1, true);
+		regmap_update_bits(afe->regmap, AUDIO_TOP_CON0,
+				   PDN_CM1_MASK_SFT, 1 << PDN_CM1_SFT);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int ul_cm2_event(struct snd_soc_dapm_widget *w,
+			struct snd_kcontrol *kcontrol,
+			int event)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(w->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	unsigned int channels = afe_priv->cm_channels;
+
+	dev_dbg(afe->dev, "event 0x%x, name %s, channels %u\n",
+		event, w->name, channels);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		mt8196_enable_cm_bypass(afe, CM2, false);
+		mt8196_set_cm(afe, CM2, true, false, channels);
+		regmap_update_bits(afe->regmap, AUDIO_TOP_CON0,
+				   PDN_CM2_MASK_SFT, 0 << PDN_CM2_SFT);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		mt8196_enable_cm_bypass(afe, CM2, true);
+		regmap_update_bits(afe->regmap, AUDIO_TOP_CON0,
+				   PDN_CM2_MASK_SFT, 1 << PDN_CM2_SFT);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+/*
+ * dma widget & routes
+ * The mixer controls and routes are by no means fully implemented,
+ * only the ones that are intended to be used are, as other wise a fully
+ * interconnected switch bar mixer would introduce way too many unused
+ * controls.
+ */
+static const struct snd_kcontrol_new memif_ul0_ch1_mix[] = {
+	/* Normal record */
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN018_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul0_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN019_0,
+				    I_ADDA_UL_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul1_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN4_CH1", AFE_CONN020_4,
+				    I_I2SIN4_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN6_CH1", AFE_CONN020_5,
+				    I_I2SIN6_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul1_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN4_CH2", AFE_CONN021_4,
+				    I_I2SIN4_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN6_CH2", AFE_CONN021_5,
+				    I_I2SIN6_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul2_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN022_0,
+				    I_ADDA_UL_CH3, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul2_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN023_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul3_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH1", AFE_CONN024_4,
+				    I_I2SIN0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN1_CH1", AFE_CONN024_4,
+				    I_I2SIN1_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN3_CH1", AFE_CONN024_4,
+				    I_I2SIN3_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN4_CH1", AFE_CONN024_4,
+				    I_I2SIN4_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul3_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH2", AFE_CONN025_4,
+				    I_I2SIN0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN1_CH2", AFE_CONN025_4,
+				    I_I2SIN1_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN3_CH2", AFE_CONN025_4,
+				    I_I2SIN3_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN4_CH2", AFE_CONN025_4,
+				    I_I2SIN4_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul4_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN026_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH1", AFE_CONN026_1,
+				    I_DL0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH1", AFE_CONN026_1,
+				    I_DL1_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH1", AFE_CONN026_1,
+				    I_DL6_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH1", AFE_CONN026_1,
+				    I_DL2_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH1", AFE_CONN026_1,
+				    I_DL3_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH1", AFE_CONN026_1,
+				    I_DL_24CH_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH1", AFE_CONN026_4,
+				    I_I2SIN0_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul4_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN027_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH2", AFE_CONN027_1,
+				    I_DL0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH2", AFE_CONN027_1,
+				    I_DL1_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH2", AFE_CONN027_1,
+				    I_DL6_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH2", AFE_CONN027_1,
+				    I_DL2_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH2", AFE_CONN027_1,
+				    I_DL3_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH2", AFE_CONN027_1,
+				    I_DL_24CH_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH2", AFE_CONN027_4,
+				    I_I2SIN0_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul5_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN3_CH1", AFE_CONN028_4,
+				    I_I2SIN3_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul5_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN3_CH2", AFE_CONN029_4,
+				    I_I2SIN3_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul6_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN030_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul6_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN031_0,
+				    I_ADDA_UL_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul7_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN032_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul7_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN033_0,
+				    I_ADDA_UL_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul8_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN034_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul8_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN035_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul9_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN036_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul9_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN037_0,
+				    I_ADDA_UL_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul10_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN038_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul10_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN039_0,
+				    I_ADDA_UL_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul24_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH1", AFE_CONN066_4,
+				    I_I2SIN0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN6_CH1", AFE_CONN066_5,
+				    I_I2SIN6_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul24_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH2", AFE_CONN067_4,
+				    I_I2SIN0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN6_CH2", AFE_CONN067_5,
+				    I_I2SIN6_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul25_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH1", AFE_CONN068_4,
+				    I_I2SIN0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN6_CH1", AFE_CONN068_5,
+				    I_I2SIN6_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul25_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH2", AFE_CONN069_4,
+				    I_I2SIN0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN6_CH2", AFE_CONN069_5,
+				    I_I2SIN6_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul26_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH1", AFE_CONN070_4,
+				    I_I2SIN0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN6_CH1", AFE_CONN070_5,
+				    I_I2SIN6_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul26_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN0_CH2", AFE_CONN071_4,
+				    I_I2SIN0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("I2SIN6_CH2", AFE_CONN071_5,
+				    I_I2SIN6_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm0_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN040_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm0_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN041_0,
+				    I_ADDA_UL_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm0_ch3_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN042_0,
+				    I_ADDA_UL_CH3, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm0_ch4_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN043_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm0_ch5_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN044_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm0_ch6_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN045_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm0_ch7_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN046_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm0_ch8_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN047_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN048_0,
+				    I_ADDA_UL_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN049_0,
+				    I_ADDA_UL_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch3_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN050_0,
+				    I_ADDA_UL_CH3, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch4_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN051_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch5_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN052_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN052_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN052_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN052_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch6_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN053_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN053_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN053_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN053_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch7_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN054_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN054_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN054_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN054_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch8_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN055_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN055_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN055_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN055_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch9_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN056_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN056_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN056_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN056_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch10_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN057_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN057_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN057_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN057_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch11_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN058_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN058_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN058_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN058_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch12_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN059_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN059_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN059_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN059_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch13_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN060_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN060_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN060_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN060_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch14_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN061_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN061_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN061_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN061_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch15_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN062_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN062_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN062_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN062_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm1_ch16_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN063_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN063_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN063_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN063_0,
+				    I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN064_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN064_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN064_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN064_0, I_ADDA_UL_CH4, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH5", AFE_CONN064_0, I_ADDA_UL_CH5, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH6", AFE_CONN064_0, I_ADDA_UL_CH6, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN065_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN065_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN065_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN065_0, I_ADDA_UL_CH4, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH5", AFE_CONN065_0, I_ADDA_UL_CH5, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH6", AFE_CONN065_0, I_ADDA_UL_CH6, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch3_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN066_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN066_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN066_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN066_0, I_ADDA_UL_CH4, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH5", AFE_CONN066_0, I_ADDA_UL_CH5, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH6", AFE_CONN066_0, I_ADDA_UL_CH6, 1, 0)
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch4_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN067_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN067_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN067_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN067_0, I_ADDA_UL_CH4, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH5", AFE_CONN067_0, I_ADDA_UL_CH5, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH6", AFE_CONN067_0, I_ADDA_UL_CH6, 1, 0)
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch5_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN068_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN068_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN068_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN068_0, I_ADDA_UL_CH4, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH5", AFE_CONN068_0, I_ADDA_UL_CH5, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH6", AFE_CONN068_0, I_ADDA_UL_CH6, 1, 0)
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch6_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN069_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN069_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN069_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN069_0, I_ADDA_UL_CH4, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH5", AFE_CONN069_0, I_ADDA_UL_CH5, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH6", AFE_CONN069_0, I_ADDA_UL_CH6, 1, 0)
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch7_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN070_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN070_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN070_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN070_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch8_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN071_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN071_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN071_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN071_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch9_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN072_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN072_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN072_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN072_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch10_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN073_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN073_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN073_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN073_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch11_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN074_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN074_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN074_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN074_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch12_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN075_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN075_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN075_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN075_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch13_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN076_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN076_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN076_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN076_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch14_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN077_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN077_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN077_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN077_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch15_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN078_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN078_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN078_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN078_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch16_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN079_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN079_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN079_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN079_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch17_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN080_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN080_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN080_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN080_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch18_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN081_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN081_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN081_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN081_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch19_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN082_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN082_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN082_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN082_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch20_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN083_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN083_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN083_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN083_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch21_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN084_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN084_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN084_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN084_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch22_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN085_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN085_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN085_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN085_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch23_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN086_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN086_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN086_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN086_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch24_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN087_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN087_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN087_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN087_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch25_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN088_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN088_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN088_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN088_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch26_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN089_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN089_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN089_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN089_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch27_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN090_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN090_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN090_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN090_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch28_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN091_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN091_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN091_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN091_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch29_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN092_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN092_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN092_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN092_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch30_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN093_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN093_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN093_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN093_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch31_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN094_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN094_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN094_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN094_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const struct snd_kcontrol_new memif_ul_cm2_ch32_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN095_0, I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN095_0, I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN095_0, I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH4", AFE_CONN095_0, I_ADDA_UL_CH4, 1, 0),
+};
+
+static const char * const cm0_mux_map[] = {
+	"UL8_2CH_PATH",
+	"CM0_8CH_PATH",
+};
+
+static const char * const cm1_mux_map[] = {
+	"UL9_2CH_PATH",
+	"CM1_16CH_PATH",
+};
+
+static const char * const cm2_mux_map[] = {
+	"UL10_2CH_PATH",
+	"CM2_32CH_PATH",
+};
+
+static SOC_ENUM_SINGLE_DECL(ul_cm0_mux_map_enum, AFE_CM0_CON0,
+	AFE_CM0_OUTPUT_MUX_SFT, cm0_mux_map);
+
+static SOC_ENUM_SINGLE_DECL(ul_cm1_mux_map_enum, AFE_CM1_CON0,
+	AFE_CM1_OUTPUT_MUX_SFT, cm1_mux_map);
+
+static SOC_ENUM_SINGLE_DECL(ul_cm2_mux_map_enum, AFE_CM2_CON0,
+	AFE_CM2_OUTPUT_MUX_SFT, cm2_mux_map);
+
+static const struct snd_kcontrol_new ul_cm0_mux_control =
+	SOC_DAPM_ENUM("CM0_UL_MUX Route", ul_cm0_mux_map_enum);
+
+static const struct snd_kcontrol_new ul_cm1_mux_control =
+	SOC_DAPM_ENUM("CM1_UL_MUX Route", ul_cm1_mux_map_enum);
+
+static const struct snd_kcontrol_new ul_cm2_mux_control =
+	SOC_DAPM_ENUM("CM2_UL_MUX Route", ul_cm2_mux_map_enum);
+
+static const struct snd_soc_dapm_widget mt8196_memif_widgets[] = {
+	/* inter-connections */
+	SND_SOC_DAPM_MIXER("UL0_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul0_ch1_mix, ARRAY_SIZE(memif_ul0_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL0_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul0_ch2_mix, ARRAY_SIZE(memif_ul0_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL1_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul1_ch1_mix, ARRAY_SIZE(memif_ul1_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL1_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul1_ch2_mix, ARRAY_SIZE(memif_ul1_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL2_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul2_ch1_mix, ARRAY_SIZE(memif_ul2_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL2_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul2_ch2_mix, ARRAY_SIZE(memif_ul2_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL3_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul3_ch1_mix, ARRAY_SIZE(memif_ul3_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL3_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul3_ch2_mix, ARRAY_SIZE(memif_ul3_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL4_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul4_ch1_mix, ARRAY_SIZE(memif_ul4_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL4_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul4_ch2_mix, ARRAY_SIZE(memif_ul4_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL5_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul5_ch1_mix, ARRAY_SIZE(memif_ul5_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL5_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul5_ch2_mix, ARRAY_SIZE(memif_ul5_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL6_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul6_ch1_mix, ARRAY_SIZE(memif_ul6_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL6_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul6_ch2_mix, ARRAY_SIZE(memif_ul6_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL7_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul7_ch1_mix, ARRAY_SIZE(memif_ul7_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL7_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul7_ch2_mix, ARRAY_SIZE(memif_ul7_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL8_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul8_ch1_mix, ARRAY_SIZE(memif_ul8_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL8_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul8_ch2_mix, ARRAY_SIZE(memif_ul8_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL9_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul9_ch1_mix, ARRAY_SIZE(memif_ul9_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL9_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul9_ch2_mix, ARRAY_SIZE(memif_ul9_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL10_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul10_ch1_mix, ARRAY_SIZE(memif_ul10_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL10_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul10_ch2_mix, ARRAY_SIZE(memif_ul10_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL24_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul24_ch1_mix, ARRAY_SIZE(memif_ul24_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL24_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul24_ch2_mix, ARRAY_SIZE(memif_ul24_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL25_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul25_ch1_mix, ARRAY_SIZE(memif_ul25_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL25_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul25_ch2_mix, ARRAY_SIZE(memif_ul25_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL26_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul26_ch1_mix, ARRAY_SIZE(memif_ul26_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL26_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul26_ch2_mix, ARRAY_SIZE(memif_ul26_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("UL_CM0_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm0_ch1_mix, ARRAY_SIZE(memif_ul_cm0_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM0_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm0_ch2_mix, ARRAY_SIZE(memif_ul_cm0_ch2_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM0_CH3", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm0_ch3_mix, ARRAY_SIZE(memif_ul_cm0_ch3_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM0_CH4", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm0_ch4_mix, ARRAY_SIZE(memif_ul_cm0_ch4_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM0_CH5", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm0_ch5_mix, ARRAY_SIZE(memif_ul_cm0_ch5_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM0_CH6", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm0_ch6_mix, ARRAY_SIZE(memif_ul_cm0_ch6_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM0_CH7", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm0_ch7_mix, ARRAY_SIZE(memif_ul_cm0_ch7_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM0_CH8", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm0_ch8_mix, ARRAY_SIZE(memif_ul_cm0_ch8_mix)),
+	SND_SOC_DAPM_MUX("CM0_UL_MUX", SND_SOC_NOPM, 0, 0,
+			 &ul_cm0_mux_control),
+
+	SND_SOC_DAPM_MIXER("UL_CM1_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch1_mix, ARRAY_SIZE(memif_ul_cm1_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch2_mix, ARRAY_SIZE(memif_ul_cm1_ch2_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH3", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch3_mix, ARRAY_SIZE(memif_ul_cm1_ch3_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH4", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch4_mix, ARRAY_SIZE(memif_ul_cm1_ch4_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH5", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch5_mix, ARRAY_SIZE(memif_ul_cm1_ch5_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH6", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch6_mix, ARRAY_SIZE(memif_ul_cm1_ch6_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH7", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch7_mix, ARRAY_SIZE(memif_ul_cm1_ch7_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH8", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch8_mix, ARRAY_SIZE(memif_ul_cm1_ch8_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH9", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch9_mix, ARRAY_SIZE(memif_ul_cm1_ch9_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH10", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch10_mix, ARRAY_SIZE(memif_ul_cm1_ch10_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH11", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch11_mix, ARRAY_SIZE(memif_ul_cm1_ch11_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH12", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch12_mix, ARRAY_SIZE(memif_ul_cm1_ch12_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH13", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch13_mix, ARRAY_SIZE(memif_ul_cm1_ch13_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH14", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch14_mix, ARRAY_SIZE(memif_ul_cm1_ch14_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH15", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch15_mix, ARRAY_SIZE(memif_ul_cm1_ch15_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM1_CH16", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm1_ch16_mix, ARRAY_SIZE(memif_ul_cm1_ch16_mix)),
+	SND_SOC_DAPM_MUX("CM1_UL_MUX", SND_SOC_NOPM, 0, 0,
+			 &ul_cm1_mux_control),
+
+	SND_SOC_DAPM_MIXER("UL_CM2_CH1", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch1_mix, ARRAY_SIZE(memif_ul_cm2_ch1_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH2", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch2_mix, ARRAY_SIZE(memif_ul_cm2_ch2_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH3", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch3_mix, ARRAY_SIZE(memif_ul_cm2_ch3_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH4", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch4_mix, ARRAY_SIZE(memif_ul_cm2_ch4_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH5", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch5_mix, ARRAY_SIZE(memif_ul_cm2_ch5_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH6", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch6_mix, ARRAY_SIZE(memif_ul_cm2_ch6_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH7", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch7_mix, ARRAY_SIZE(memif_ul_cm2_ch7_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH8", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch8_mix, ARRAY_SIZE(memif_ul_cm2_ch8_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH9", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch9_mix, ARRAY_SIZE(memif_ul_cm2_ch9_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH10", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch10_mix, ARRAY_SIZE(memif_ul_cm2_ch10_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH11", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch11_mix, ARRAY_SIZE(memif_ul_cm2_ch11_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH12", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch12_mix, ARRAY_SIZE(memif_ul_cm2_ch12_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH13", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch13_mix, ARRAY_SIZE(memif_ul_cm2_ch13_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH14", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch14_mix, ARRAY_SIZE(memif_ul_cm2_ch14_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH15", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch15_mix, ARRAY_SIZE(memif_ul_cm2_ch15_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH16", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch16_mix, ARRAY_SIZE(memif_ul_cm2_ch16_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH17", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch17_mix, ARRAY_SIZE(memif_ul_cm2_ch17_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH18", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch18_mix, ARRAY_SIZE(memif_ul_cm2_ch18_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH19", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch19_mix, ARRAY_SIZE(memif_ul_cm2_ch19_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH20", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch20_mix, ARRAY_SIZE(memif_ul_cm2_ch20_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH21", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch21_mix, ARRAY_SIZE(memif_ul_cm2_ch21_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH22", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch22_mix, ARRAY_SIZE(memif_ul_cm2_ch22_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH23", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch23_mix, ARRAY_SIZE(memif_ul_cm2_ch23_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH24", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch24_mix, ARRAY_SIZE(memif_ul_cm2_ch24_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH25", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch25_mix, ARRAY_SIZE(memif_ul_cm2_ch25_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH26", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch26_mix, ARRAY_SIZE(memif_ul_cm2_ch26_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH27", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch27_mix, ARRAY_SIZE(memif_ul_cm2_ch27_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH28", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch28_mix, ARRAY_SIZE(memif_ul_cm2_ch28_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH29", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch29_mix, ARRAY_SIZE(memif_ul_cm2_ch29_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH30", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch30_mix, ARRAY_SIZE(memif_ul_cm2_ch30_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH31", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch31_mix, ARRAY_SIZE(memif_ul_cm2_ch31_mix)),
+	SND_SOC_DAPM_MIXER("UL_CM2_CH32", SND_SOC_NOPM, 0, 0,
+			   memif_ul_cm2_ch32_mix, ARRAY_SIZE(memif_ul_cm2_ch32_mix)),
+	SND_SOC_DAPM_MUX("CM2_UL_MUX", SND_SOC_NOPM, 0, 0,
+			 &ul_cm2_mux_control),
+
+	SND_SOC_DAPM_SUPPLY("CM0_Enable",
+			    AFE_CM0_CON0, AFE_CM0_ON_SFT, 0,
+			    ul_cm0_event,
+			    SND_SOC_DAPM_PRE_PMU |
+			    SND_SOC_DAPM_PRE_PMD),
+
+	SND_SOC_DAPM_SUPPLY("CM1_Enable",
+			    AFE_CM1_CON0, AFE_CM1_ON_SFT, 0,
+			    ul_cm1_event,
+			    SND_SOC_DAPM_PRE_PMU |
+			    SND_SOC_DAPM_PRE_PMD),
+
+	SND_SOC_DAPM_SUPPLY("CM2_Enable",
+			    AFE_CM2_CON0, AFE_CM2_ON_SFT, 0,
+			    ul_cm2_event,
+			    SND_SOC_DAPM_PRE_PMU |
+			    SND_SOC_DAPM_PRE_PMD),
+
+	/* dynamic pinctrl */
+	SND_SOC_DAPM_PINCTRL("I2S3_PIN", "aud-gpio-i2s3-on", "aud-gpio-i2s3-off"),
+	SND_SOC_DAPM_PINCTRL("I2S4_PIN", "aud-gpio-i2s4-on", "aud-gpio-i2s4-off"),
+	SND_SOC_DAPM_PINCTRL("I2S6_PIN", "aud-gpio-i2s6-on", "aud-gpio-i2s6-off"),
+	SND_SOC_DAPM_PINCTRL("AP_DMIC0_PIN", "aud-gpio-ap-dmic-on", "aud-gpio-ap-dmic-off"),
+	SND_SOC_DAPM_PINCTRL("AP_DMIC1_PIN", "aud-gpio-ap-dmic1-on", "aud-gpio-ap-dmic1-off"),
+};
+
+static const struct snd_soc_dapm_route mt8196_memif_routes[] = {
+	{"UL0", NULL, "UL0_CH1"},
+	{"UL0", NULL, "UL0_CH2"},
+	/* Normal record */
+	{"UL0_CH1", "ADDA_UL_CH1", "ADDA_UL_Mux"},
+	{"UL0_CH2", "ADDA_UL_CH2", "ADDA_UL_Mux"},
+
+	{"UL1", NULL, "UL1_CH1"},
+	{"UL1", NULL, "UL1_CH2"},
+	{"UL1_CH1", "I2SIN4_CH1", "I2SIN4"},
+	{"UL1_CH2", "I2SIN4_CH2", "I2SIN4"},
+	{"UL1_CH1", "I2SIN6_CH1", "I2SIN6"},
+	{"UL1_CH2", "I2SIN6_CH2", "I2SIN6"},
+
+	{"UL2", NULL, "UL2_CH1"},
+	{"UL2", NULL, "UL2_CH2"},
+	{"UL2_CH1", "ADDA_UL_CH3", "ADDA_CH34_UL_Mux"},
+	{"UL2_CH2", "ADDA_UL_CH4", "ADDA_CH34_UL_Mux"},
+
+	{"UL3", NULL, "UL3_CH1"},
+	{"UL3", NULL, "UL3_CH2"},
+	{"UL3_CH1", "I2SIN0_CH1", "I2SIN0"},
+	{"UL3_CH2", "I2SIN0_CH2", "I2SIN0"},
+	{"UL3_CH1", "I2SIN1_CH1", "I2SIN1"},
+	{"UL3_CH2", "I2SIN1_CH2", "I2SIN1"},
+	{"UL3_CH1", "I2SIN3_CH1", "I2SIN3"},
+	{"UL3_CH2", "I2SIN3_CH2", "I2SIN3"},
+	{"UL3_CH1", "I2SIN4_CH1", "I2SIN4"},
+	{"UL3_CH2", "I2SIN4_CH2", "I2SIN4"},
+
+	{"UL4", NULL, "UL4_CH1"},
+	{"UL4", NULL, "UL4_CH2"},
+	{"UL4_CH1", "ADDA_UL_CH1", "ADDA_UL_Mux"},
+	{"UL4_CH2", "ADDA_UL_CH2", "ADDA_UL_Mux"},
+	{"UL4_CH1", "I2SIN0_CH1", "I2SIN0"},
+	{"UL4_CH2", "I2SIN0_CH2", "I2SIN0"},
+
+	{"UL5", NULL, "UL5_CH1"},
+	{"UL5", NULL, "UL5_CH2"},
+	{"UL5_CH1", "I2SIN3_CH1", "I2SIN3"},
+	{"UL5_CH2", "I2SIN3_CH2", "I2SIN3"},
+
+	{"UL6", NULL, "UL6_CH1"},
+	{"UL6", NULL, "UL6_CH2"},
+	{"UL6_CH1", "ADDA_UL_CH1", "ADDA_UL_Mux"},
+	{"UL6_CH2", "ADDA_UL_CH2", "ADDA_UL_Mux"},
+
+	{"UL7", NULL, "UL7_CH1"},
+	{"UL7", NULL, "UL7_CH2"},
+	{"UL7_CH1", "ADDA_UL_CH1", "ADDA_UL_Mux"},
+	{"UL7_CH2", "ADDA_UL_CH2", "ADDA_UL_Mux"},
+
+	{"UL8", NULL, "CM0_UL_MUX"},
+	{"CM0_UL_MUX", "UL8_2CH_PATH", "UL8_CH1"},
+	{"CM0_UL_MUX", "UL8_2CH_PATH", "UL8_CH2"},
+	{"CM0_UL_MUX", "CM0_8CH_PATH", "UL_CM0_CH1"},
+	{"CM0_UL_MUX", "CM0_8CH_PATH", "UL_CM0_CH2"},
+	{"CM0_UL_MUX", "CM0_8CH_PATH", "UL_CM0_CH3"},
+	{"CM0_UL_MUX", "CM0_8CH_PATH", "UL_CM0_CH4"},
+	{"CM0_UL_MUX", "CM0_8CH_PATH", "UL_CM0_CH5"},
+	{"CM0_UL_MUX", "CM0_8CH_PATH", "UL_CM0_CH6"},
+	{"CM0_UL_MUX", "CM0_8CH_PATH", "UL_CM0_CH7"},
+	{"CM0_UL_MUX", "CM0_8CH_PATH", "UL_CM0_CH8"},
+
+	{"UL_CM0", NULL, "CM0_Enable"},
+
+	/* UL9 */
+	{"UL9", NULL, "CM1_UL_MUX"},
+	{"CM1_UL_MUX", "UL9_2CH_PATH", "UL9_CH1"},
+	{"CM1_UL_MUX", "UL9_2CH_PATH", "UL9_CH2"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH1"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH2"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH3"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH4"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH5"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH6"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH7"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH8"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH9"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH10"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH11"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH12"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH13"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH14"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH15"},
+	{"CM1_UL_MUX", "CM1_16CH_PATH", "UL_CM1_CH16"},
+
+	{"UL_CM1", NULL, "CM1_Enable"},
+
+	{"UL9_CH1", "ADDA_UL_CH1", "ADDA_UL_Mux"},
+	{"UL9_CH2", "ADDA_UL_CH2", "ADDA_UL_Mux"},
+
+	{"UL10", NULL, "CM2_UL_MUX"},
+	{"CM2_UL_MUX", "UL10_2CH_PATH", "UL10_CH1"},
+	{"CM2_UL_MUX", "UL10_2CH_PATH", "UL10_CH2"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH1"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH2"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH3"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH4"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH5"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH6"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH7"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH8"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH9"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH10"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH11"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH12"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH13"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH14"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH15"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH16"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH17"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH18"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH19"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH20"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH21"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH22"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH23"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH24"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH25"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH26"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH27"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH28"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH29"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH30"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH31"},
+	{"CM2_UL_MUX", "CM2_32CH_PATH", "UL_CM2_CH32"},
+
+	{"UL_CM2", NULL, "CM2_Enable"},
+
+	{"UL10_CH1", "ADDA_UL_CH1", "ADDA_UL_Mux"},
+	{"UL10_CH2", "ADDA_UL_CH2", "ADDA_UL_Mux"},
+
+	{"UL24", NULL, "UL24_CH1"},
+	{"UL24", NULL, "UL24_CH2"},
+	{"UL24_CH1", "I2SIN6_CH1", "I2SIN6"},
+	{"UL24_CH2", "I2SIN6_CH2", "I2SIN6"},
+	{"UL24_CH1", "I2SIN0_CH1", "I2SIN0"},
+	{"UL24_CH2", "I2SIN0_CH2", "I2SIN0"},
+
+	{"UL25", NULL, "UL25_CH1"},
+	{"UL25", NULL, "UL25_CH2"},
+	{"UL25_CH1", "I2SIN6_CH1", "I2SIN6"},
+	{"UL25_CH2", "I2SIN6_CH2", "I2SIN6"},
+	{"UL25_CH1", "I2SIN0_CH1", "I2SIN0"},
+	{"UL25_CH2", "I2SIN0_CH2", "I2SIN0"},
+
+	{"UL26", NULL, "UL26_CH1"},
+	{"UL26", NULL, "UL26_CH2"},
+	{"UL26_CH1", "I2SIN6_CH1", "I2SIN6"},
+	{"UL26_CH2", "I2SIN6_CH2", "I2SIN6"},
+	{"UL26_CH1", "I2SIN0_CH1", "I2SIN0"},
+	{"UL26_CH2", "I2SIN0_CH2", "I2SIN0"},
+
+	{"UL_CM0", NULL, "UL_CM0_CH1"},
+	{"UL_CM0", NULL, "UL_CM0_CH2"},
+	{"UL_CM0", NULL, "UL_CM0_CH3"},
+	{"UL_CM0", NULL, "UL_CM0_CH4"},
+	{"UL_CM0", NULL, "UL_CM0_CH5"},
+	{"UL_CM0", NULL, "UL_CM0_CH6"},
+	{"UL_CM0", NULL, "UL_CM0_CH7"},
+	{"UL_CM0", NULL, "UL_CM0_CH8"},
+	{"UL_CM0_CH1", "ADDA_UL_CH1", "ADDA_UL_Mux"},
+	{"UL_CM0_CH2", "ADDA_UL_CH2", "ADDA_UL_Mux"},
+	{"UL_CM0_CH3", "ADDA_UL_CH3", "ADDA_CH34_UL_Mux"},
+	{"UL_CM0_CH4", "ADDA_UL_CH4", "ADDA_CH34_UL_Mux"},
+
+	{"UL_CM1", NULL, "UL_CM1_CH1"},
+	{"UL_CM1", NULL, "UL_CM1_CH2"},
+	{"UL_CM1", NULL, "UL_CM1_CH3"},
+	{"UL_CM1", NULL, "UL_CM1_CH4"},
+	{"UL_CM1", NULL, "UL_CM1_CH5"},
+	{"UL_CM1", NULL, "UL_CM1_CH6"},
+	{"UL_CM1", NULL, "UL_CM1_CH7"},
+	{"UL_CM1", NULL, "UL_CM1_CH8"},
+	{"UL_CM1", NULL, "UL_CM1_CH9"},
+	{"UL_CM1", NULL, "UL_CM1_CH10"},
+	{"UL_CM1", NULL, "UL_CM1_CH11"},
+	{"UL_CM1", NULL, "UL_CM1_CH12"},
+	{"UL_CM1", NULL, "UL_CM1_CH13"},
+	{"UL_CM1", NULL, "UL_CM1_CH14"},
+	{"UL_CM1", NULL, "UL_CM1_CH15"},
+	{"UL_CM1", NULL, "UL_CM1_CH16"},
+	{"UL_CM1_CH1", "ADDA_UL_CH1", "ADDA_UL_Mux"},
+	{"UL_CM1_CH2", "ADDA_UL_CH2", "ADDA_UL_Mux"},
+	{"UL_CM1_CH3", "ADDA_UL_CH3", "ADDA_CH34_UL_Mux"},
+	{"UL_CM1_CH4", "ADDA_UL_CH4", "ADDA_CH34_UL_Mux"},
+
+	{"UL_CM2", NULL, "UL_CM2_CH1"},
+	{"UL_CM2", NULL, "UL_CM2_CH2"},
+	{"UL_CM2", NULL, "UL_CM2_CH3"},
+	{"UL_CM2", NULL, "UL_CM2_CH4"},
+	{"UL_CM2", NULL, "UL_CM2_CH5"},
+	{"UL_CM2", NULL, "UL_CM2_CH6"},
+	{"UL_CM2", NULL, "UL_CM2_CH7"},
+	{"UL_CM2", NULL, "UL_CM2_CH8"},
+	{"UL_CM2", NULL, "UL_CM2_CH9"},
+	{"UL_CM2", NULL, "UL_CM2_CH10"},
+	{"UL_CM2", NULL, "UL_CM2_CH11"},
+	{"UL_CM2", NULL, "UL_CM2_CH12"},
+	{"UL_CM2", NULL, "UL_CM2_CH13"},
+	{"UL_CM2", NULL, "UL_CM2_CH14"},
+	{"UL_CM2", NULL, "UL_CM2_CH15"},
+	{"UL_CM2", NULL, "UL_CM2_CH16"},
+	{"UL_CM2", NULL, "UL_CM2_CH17"},
+	{"UL_CM2", NULL, "UL_CM2_CH18"},
+	{"UL_CM2", NULL, "UL_CM2_CH19"},
+	{"UL_CM2", NULL, "UL_CM2_CH20"},
+	{"UL_CM2", NULL, "UL_CM2_CH21"},
+	{"UL_CM2", NULL, "UL_CM2_CH22"},
+	{"UL_CM2", NULL, "UL_CM2_CH23"},
+	{"UL_CM2", NULL, "UL_CM2_CH24"},
+	{"UL_CM2", NULL, "UL_CM2_CH25"},
+	{"UL_CM2", NULL, "UL_CM2_CH26"},
+	{"UL_CM2", NULL, "UL_CM2_CH27"},
+	{"UL_CM2", NULL, "UL_CM2_CH28"},
+	{"UL_CM2", NULL, "UL_CM2_CH29"},
+	{"UL_CM2", NULL, "UL_CM2_CH30"},
+	{"UL_CM2", NULL, "UL_CM2_CH31"},
+	{"UL_CM2", NULL, "UL_CM2_CH32"},
+	{"UL_CM2_CH1", "ADDA_UL_CH1", "ADDA_UL_Mux"},
+	{"UL_CM2_CH2", "ADDA_UL_CH2", "ADDA_UL_Mux"},
+	{"UL_CM2_CH3", "ADDA_UL_CH3", "ADDA_CH34_UL_Mux"},
+	{"UL_CM2_CH4", "ADDA_UL_CH4", "ADDA_CH34_UL_Mux"},
+
+	/* Audio Pin */
+	{"I2SOUT4", NULL, "I2S4_PIN"},
+	{"I2SIN4", NULL, "I2S4_PIN"},
+	{"I2SOUT6", NULL, "I2S6_PIN"},
+	{"I2SIN6", NULL, "I2S6_PIN"},
+	{"I2SOUT3", NULL, "I2S3_PIN"},
+	{"I2SIN3", NULL, "I2S3_PIN"},
+	{"AP DMIC Capture", NULL, "AP_DMIC0_PIN"},
+	{"AP DMIC CH34 Capture", NULL, "AP_DMIC1_PIN"},
+};
+
+#define MT8196_DL_MEMIF(_id) \
+	[MT8196_MEMIF_##_id] = { \
+		.name = #_id, \
+		.id = MT8196_MEMIF_##_id, \
+		.reg_ofs_base = AFE_##_id##_BASE, \
+		.reg_ofs_cur = AFE_##_id##_CUR, \
+		.reg_ofs_end = AFE_##_id##_END, \
+		.reg_ofs_base_msb = AFE_##_id##_BASE_MSB, \
+		.reg_ofs_cur_msb = AFE_##_id##_CUR_MSB, \
+		.reg_ofs_end_msb = AFE_##_id##_END_MSB, \
+		.fs_reg = AFE_##_id##_CON0, \
+		.fs_shift = _id##_SEL_FS_SFT, \
+		.fs_maskbit = _id##_SEL_FS_MASK, \
+		.mono_reg = AFE_##_id##_CON0, \
+		.mono_shift = _id##_MONO_SFT, \
+		.enable_reg = AFE_##_id##_CON0, \
+		.enable_shift = _id##_ON_SFT, \
+		.hd_reg = AFE_##_id##_CON0, \
+		.hd_shift = _id##_HD_MODE_SFT, \
+		.hd_align_reg = AFE_##_id##_CON0, \
+		.hd_align_mshift = _id##_HALIGN_SFT, \
+		.agent_disable_reg = -1, \
+		.agent_disable_shift = -1, \
+		.msb_reg = -1, \
+		.msb_shift = -1, \
+		.pbuf_reg = AFE_##_id##_CON0, \
+		.pbuf_mask = _id##_PBUF_SIZE_MASK, \
+		.pbuf_shift = _id##_PBUF_SIZE_SFT, \
+		.minlen_reg = AFE_##_id##_CON0, \
+		.minlen_mask = _id##_MINLEN_MASK, \
+		.minlen_shift = _id##_MINLEN_SFT, \
+}
+
+#define MT8196_MULTI_DL_MEMIF(_id) \
+	[MT8196_MEMIF_##_id] = { \
+		.name = #_id, \
+		.id = MT8196_MEMIF_##_id, \
+		.reg_ofs_base = AFE_##_id##_BASE, \
+		.reg_ofs_cur = AFE_##_id##_CUR, \
+		.reg_ofs_end = AFE_##_id##_END, \
+		.reg_ofs_base_msb = AFE_##_id##_BASE_MSB, \
+		.reg_ofs_cur_msb = AFE_##_id##_CUR_MSB, \
+		.reg_ofs_end_msb = AFE_##_id##_END_MSB, \
+		.fs_reg = AFE_##_id##_CON0, \
+		.fs_shift = _id##_SEL_FS_SFT, \
+		.fs_maskbit = _id##_SEL_FS_MASK, \
+		.mono_reg = -1, \
+		.mono_shift = -1, \
+		.enable_reg = AFE_##_id##_CON0, \
+		.enable_shift = _id##_ON_SFT, \
+		.hd_reg = AFE_##_id##_CON0, \
+		.hd_shift = _id##_HD_MODE_SFT, \
+		.hd_align_reg = AFE_##_id##_CON0, \
+		.hd_align_mshift = _id##_HALIGN_SFT, \
+		.agent_disable_reg = -1, \
+		.agent_disable_shift = -1, \
+		.msb_reg = -1, \
+		.msb_shift = -1, \
+		.pbuf_reg = AFE_##_id##_CON0, \
+		.pbuf_mask = _id##_PBUF_SIZE_MASK, \
+		.pbuf_shift = _id##_PBUF_SIZE_SFT, \
+		.minlen_reg = AFE_##_id##_CON0, \
+		.minlen_mask = _id##_MINLEN_MASK, \
+		.minlen_shift = _id##_MINLEN_SFT, \
+		.ch_num_reg = AFE_##_id##_CON0, \
+		.ch_num_maskbit = _id##_NUM_MASK, \
+		.ch_num_shift = _id##_NUM_SFT, \
+}
+
+#define MT8196_UL_MEMIF(_id, _fs_shift, _fs_maskbit, _mono_shift) \
+	[MT8196_MEMIF_##_id] = { \
+		.name = #_id, \
+		.id = MT8196_MEMIF_##_id, \
+		.reg_ofs_base = AFE_##_id##_BASE, \
+		.reg_ofs_cur = AFE_##_id##_CUR, \
+		.reg_ofs_end = AFE_##_id##_END, \
+		.reg_ofs_base_msb = AFE_##_id##_BASE_MSB, \
+		.reg_ofs_cur_msb = AFE_##_id##_CUR_MSB, \
+		.reg_ofs_end_msb = AFE_##_id##_END_MSB, \
+		.fs_reg = AFE_##_id##_CON0, \
+		.fs_shift = _fs_shift, \
+		.fs_maskbit = _fs_maskbit, \
+		.mono_reg = AFE_##_id##_CON0, \
+		.mono_shift = _mono_shift, \
+		.enable_reg = AFE_##_id##_CON0, \
+		.enable_shift = _id##_ON_SFT, \
+		.hd_reg = AFE_##_id##_CON0, \
+		.hd_shift = _id##_HD_MODE_SFT, \
+		.hd_align_reg = AFE_##_id##_CON0, \
+		.hd_align_mshift = _id##_HALIGN_SFT, \
+		.agent_disable_reg = -1, \
+		.agent_disable_shift = -1, \
+		.msb_reg = -1, \
+		.msb_shift = -1, \
+	}
+
+/* For convenience with macros: missing register fields */
+#define HDMI_SEL_FS_SFT			-1
+#define HDMI_SEL_FS_MASK		-1
+
+/* For convenience with macros: register name differences */
+#define AFE_HDMI_BASE			AFE_HDMI_OUT_BASE
+#define AFE_HDMI_CUR			AFE_HDMI_OUT_CUR
+#define AFE_HDMI_END			AFE_HDMI_OUT_END
+#define AFE_HDMI_BASE_MSB		AFE_HDMI_OUT_BASE_MSB
+#define AFE_HDMI_CUR_MSB		AFE_HDMI_OUT_CUR_MSB
+#define AFE_HDMI_END_MSB		AFE_HDMI_OUT_END_MSB
+#define AFE_HDMI_CON0			AFE_HDMI_OUT_CON0
+#define HDMI_ON_SFT				HDMI_OUT_ON_SFT
+#define HDMI_HD_MODE_SFT		HDMI_OUT_HD_MODE_SFT
+#define HDMI_HALIGN_SFT			HDMI_OUT_HALIGN_SFT
+#define HDMI_PBUF_SIZE_MASK		HDMI_OUT_PBUF_SIZE_MASK
+#define HDMI_PBUF_SIZE_SFT		HDMI_OUT_PBUF_SIZE_SFT
+#define HDMI_MINLEN_MASK		HDMI_OUT_MINLEN_MASK
+#define HDMI_MINLEN_SFT			HDMI_OUT_MINLEN_SFT
+#define HDMI_NUM_MASK			HDMI_CH_NUM_MASK
+#define HDMI_NUM_SFT			HDMI_CH_NUM_SFT
+
+static const struct mtk_base_memif_data memif_data[MT8196_MEMIF_NUM] = {
+	MT8196_DL_MEMIF(DL0),
+	MT8196_DL_MEMIF(DL1),
+	MT8196_DL_MEMIF(DL2),
+	MT8196_DL_MEMIF(DL3),
+	MT8196_DL_MEMIF(DL4),
+	MT8196_DL_MEMIF(DL5),
+	MT8196_DL_MEMIF(DL6),
+	MT8196_DL_MEMIF(DL7),
+	MT8196_DL_MEMIF(DL8),
+	MT8196_DL_MEMIF(DL23),
+	MT8196_DL_MEMIF(DL24),
+	MT8196_DL_MEMIF(DL25),
+	MT8196_DL_MEMIF(DL26),
+	MT8196_MULTI_DL_MEMIF(DL_4CH),
+	MT8196_MULTI_DL_MEMIF(DL_24CH),
+	MT8196_MULTI_DL_MEMIF(HDMI),
+	MT8196_UL_MEMIF(VUL0, VUL0_SEL_FS_SFT, VUL0_SEL_FS_MASK, VUL0_MONO_SFT),
+	MT8196_UL_MEMIF(VUL1, VUL1_SEL_FS_SFT, VUL1_SEL_FS_MASK, VUL1_MONO_SFT),
+	MT8196_UL_MEMIF(VUL2, VUL2_SEL_FS_SFT, VUL2_SEL_FS_MASK, VUL2_MONO_SFT),
+	MT8196_UL_MEMIF(VUL3, VUL3_SEL_FS_SFT, VUL3_SEL_FS_MASK, VUL3_MONO_SFT),
+	MT8196_UL_MEMIF(VUL4, VUL4_SEL_FS_SFT, VUL4_SEL_FS_MASK, VUL4_MONO_SFT),
+	MT8196_UL_MEMIF(VUL5, VUL5_SEL_FS_SFT, VUL5_SEL_FS_MASK, VUL5_MONO_SFT),
+	MT8196_UL_MEMIF(VUL6, VUL6_SEL_FS_SFT, VUL6_SEL_FS_MASK, VUL6_MONO_SFT),
+	MT8196_UL_MEMIF(VUL7, VUL7_SEL_FS_SFT, VUL7_SEL_FS_MASK, VUL7_MONO_SFT),
+	MT8196_UL_MEMIF(VUL8, VUL8_SEL_FS_SFT, VUL8_SEL_FS_MASK, VUL8_MONO_SFT),
+	MT8196_UL_MEMIF(VUL9, VUL9_SEL_FS_SFT, VUL9_SEL_FS_MASK, VUL9_MONO_SFT),
+	MT8196_UL_MEMIF(VUL10, VUL10_SEL_FS_SFT, VUL10_SEL_FS_MASK, VUL10_MONO_SFT),
+	MT8196_UL_MEMIF(VUL24, VUL24_SEL_FS_SFT, VUL24_SEL_FS_MASK, VUL24_MONO_SFT),
+	MT8196_UL_MEMIF(VUL25, VUL25_SEL_FS_SFT, VUL25_SEL_FS_MASK, VUL25_MONO_SFT),
+	MT8196_UL_MEMIF(VUL26, VUL26_SEL_FS_SFT, VUL26_SEL_FS_MASK, VUL26_MONO_SFT),
+	MT8196_UL_MEMIF(VUL_CM0, -1, -1, -1),
+	MT8196_UL_MEMIF(VUL_CM1, -1, -1, -1),
+	MT8196_UL_MEMIF(VUL_CM2, -1, -1, -1),
+	MT8196_UL_MEMIF(ETDM_IN0, REG_FS_TIMING_SEL_SFT, REG_FS_TIMING_SEL_MASK, -1),
+	MT8196_UL_MEMIF(ETDM_IN1, REG_FS_TIMING_SEL_SFT, REG_FS_TIMING_SEL_MASK, -1),
+	MT8196_UL_MEMIF(ETDM_IN2, REG_FS_TIMING_SEL_SFT, REG_FS_TIMING_SEL_MASK, -1),
+	MT8196_UL_MEMIF(ETDM_IN3, REG_FS_TIMING_SEL_SFT, REG_FS_TIMING_SEL_MASK, -1),
+	MT8196_UL_MEMIF(ETDM_IN4, REG_FS_TIMING_SEL_SFT, REG_FS_TIMING_SEL_MASK, -1),
+	MT8196_UL_MEMIF(ETDM_IN6, REG_FS_TIMING_SEL_SFT, REG_FS_TIMING_SEL_MASK, -1),
+};
+
+#define MT8196_AFE_IRQ(_id)	\
+	[MT8196_IRQ_##_id] = { \
+		.id = MT8196_IRQ_##_id, \
+		.irq_cnt_reg = AFE_IRQ##_id##_MCU_CFG1, \
+		.irq_cnt_shift = AFE_IRQ_CNT_SHIFT, \
+		.irq_cnt_maskbit = AFE_IRQ_CNT_MASK, \
+		.irq_fs_reg = AFE_IRQ##_id##_MCU_CFG0, \
+		.irq_fs_shift = AFE_IRQ##_id##_MCU_FS_SFT, \
+		.irq_fs_maskbit = AFE_IRQ##_id##_MCU_FS_MASK, \
+		.irq_en_reg = AFE_IRQ##_id##_MCU_CFG0, \
+		.irq_en_shift = AFE_IRQ##_id##_MCU_ON_SFT, \
+		.irq_clr_reg = AFE_IRQ##_id##_MCU_CFG1, \
+		.irq_clr_shift = AFE_IRQ##_id##_CLR_CFG_SFT, \
+	}
+
+#define MT8196_AFE_TDM_IRQ(_id)	\
+	[MT8196_IRQ_##_id] = { \
+		.id = MT8196_CUS_IRQ_TDM, \
+		.irq_cnt_reg = AFE_CUSTOM_IRQ0_MCU_CFG1, \
+		.irq_cnt_shift = AFE_CUSTOM_IRQ0_MCU_CNT_SFT, \
+		.irq_cnt_maskbit = AFE_CUSTOM_IRQ0_MCU_CNT_MASK, \
+		.irq_fs_reg = -1, \
+		.irq_fs_shift = -1, \
+		.irq_fs_maskbit = -1, \
+		.irq_en_reg = AFE_CUSTOM_IRQ0_MCU_CFG0, \
+		.irq_en_shift = AFE_CUSTOM_IRQ0_MCU_ON_SFT, \
+		.irq_clr_reg = AFE_CUSTOM_IRQ0_MCU_CFG1, \
+		.irq_clr_shift = AFE_CUSTOM_IRQ0_CLR_CFG_SFT, \
+	}
+
+static const struct mtk_base_irq_data irq_data[MT8196_IRQ_NUM] = {
+	MT8196_AFE_IRQ(0),
+	MT8196_AFE_IRQ(1),
+	MT8196_AFE_IRQ(2),
+	MT8196_AFE_IRQ(3),
+	MT8196_AFE_IRQ(4),
+	MT8196_AFE_IRQ(5),
+	MT8196_AFE_IRQ(6),
+	MT8196_AFE_IRQ(7),
+	MT8196_AFE_IRQ(8),
+	MT8196_AFE_IRQ(9),
+	MT8196_AFE_IRQ(10),
+	MT8196_AFE_IRQ(11),
+	MT8196_AFE_IRQ(12),
+	MT8196_AFE_IRQ(13),
+	MT8196_AFE_IRQ(14),
+	MT8196_AFE_IRQ(15),
+	MT8196_AFE_IRQ(16),
+	MT8196_AFE_IRQ(17),
+	MT8196_AFE_IRQ(18),
+	MT8196_AFE_IRQ(19),
+	MT8196_AFE_IRQ(20),
+	MT8196_AFE_IRQ(21),
+	MT8196_AFE_IRQ(22),
+	MT8196_AFE_IRQ(23),
+	MT8196_AFE_IRQ(24),
+	MT8196_AFE_IRQ(25),
+	MT8196_AFE_IRQ(26),
+	MT8196_AFE_TDM_IRQ(31),
+};
+
+static const int memif_irq_usage[MT8196_MEMIF_NUM] = {
+	/* TODO: verify each memif & irq */
+	[MT8196_MEMIF_DL0] = MT8196_IRQ_0,
+	[MT8196_MEMIF_DL1] = MT8196_IRQ_1,
+	[MT8196_MEMIF_DL2] = MT8196_IRQ_2,
+	[MT8196_MEMIF_DL3] = MT8196_IRQ_3,
+	[MT8196_MEMIF_DL4] = MT8196_IRQ_4,
+	[MT8196_MEMIF_DL5] = MT8196_IRQ_5,
+	[MT8196_MEMIF_DL6] = MT8196_IRQ_6,
+	[MT8196_MEMIF_DL7] = MT8196_IRQ_7,
+	[MT8196_MEMIF_DL8] = MT8196_IRQ_8,
+	[MT8196_MEMIF_DL23] = MT8196_IRQ_9,
+	[MT8196_MEMIF_DL24] = MT8196_IRQ_10,
+	[MT8196_MEMIF_DL25] = MT8196_IRQ_11,
+	[MT8196_MEMIF_DL26] = MT8196_IRQ_0,
+	[MT8196_MEMIF_DL_4CH] = MT8196_IRQ_0,
+	[MT8196_MEMIF_DL_24CH] = MT8196_IRQ_12,
+	[MT8196_MEMIF_VUL0] = MT8196_IRQ_13,
+	[MT8196_MEMIF_VUL1] = MT8196_IRQ_14,
+	[MT8196_MEMIF_VUL2] = MT8196_IRQ_15,
+	[MT8196_MEMIF_VUL3] = MT8196_IRQ_16,
+	[MT8196_MEMIF_VUL4] = MT8196_IRQ_17,
+	[MT8196_MEMIF_VUL5] = MT8196_IRQ_18,
+	[MT8196_MEMIF_VUL6] = MT8196_IRQ_19,
+	[MT8196_MEMIF_VUL7] = MT8196_IRQ_20,
+	[MT8196_MEMIF_VUL8] = MT8196_IRQ_21,
+	[MT8196_MEMIF_VUL9] = MT8196_IRQ_22,
+	[MT8196_MEMIF_VUL10] = MT8196_IRQ_23,
+	[MT8196_MEMIF_VUL24] = MT8196_IRQ_24,
+	[MT8196_MEMIF_VUL25] = MT8196_IRQ_25,
+	[MT8196_MEMIF_VUL26] = MT8196_IRQ_0,
+	[MT8196_MEMIF_VUL_CM0] = MT8196_IRQ_26,
+	[MT8196_MEMIF_VUL_CM1] = MT8196_IRQ_0,
+	[MT8196_MEMIF_VUL_CM2] = MT8196_IRQ_0,
+	[MT8196_MEMIF_ETDM_IN0] = MT8196_IRQ_0,
+	[MT8196_MEMIF_ETDM_IN1] = MT8196_IRQ_0,
+	[MT8196_MEMIF_ETDM_IN2] = MT8196_IRQ_0,
+	[MT8196_MEMIF_ETDM_IN3] = MT8196_IRQ_0,
+	[MT8196_MEMIF_ETDM_IN4] = MT8196_IRQ_0,
+	[MT8196_MEMIF_ETDM_IN6] = MT8196_IRQ_0,
+	[MT8196_MEMIF_HDMI] = MT8196_IRQ_31
+};
+
+static bool mt8196_is_volatile_reg(struct device *dev, unsigned int reg)
+{
+	/* these auto-gen reg has read-only bit, so put it as volatile */
+	/* volatile reg cannot be cached, so cannot be set when power off */
+	switch (reg) {
+	case AUDIO_TOP_CON0 ... AUDIO_TOP_CON4:
+	case AFE_APLL1_TUNER_MON0:
+	case AFE_APLL2_TUNER_MON0:
+	case AFE_SPM_CONTROL_ACK:
+	case AUDIO_TOP_IP_VERSION:
+	case AUDIO_ENGEN_CON0_MON:
+	case AUD_TOP_MON_RG:
+	case AFE_CONNSYS_I2S_IPM_VER_MON:
+	case AFE_CONNSYS_I2S_MON:
+	case AFE_PCM_INTF_MON:
+	case AFE_PCM_TOP_IP_VERSION:
+	case AFE_IRQ_MCU_STATUS:
+	case AFE_CUSTOM_IRQ_MCU_STATUS:
+	case AFE_IRQ_MCU_MON0 ... AFE_IRQ26_CNT_MON:
+	case AFE_CUSTOM_IRQ0_CNT_MON:
+	case AFE_STF_MON:
+	case AFE_STF_IP_VERSION:
+	case AFE_CM0_MON:
+	case AFE_CM0_IP_VERSION:
+	case AFE_CM1_MON:
+	case AFE_CM1_IP_VERSION:
+	case AFE_ADDA_UL0_SRC_DEBUG_MON0 ... AFE_ADDA_UL0_SRC_MON1:
+	case AFE_ADDA_UL0_IP_VERSION:
+	case AFE_ADDA_UL1_SRC_DEBUG_MON0 ... AFE_ADDA_UL1_SRC_MON1:
+	case AFE_ADDA_UL1_IP_VERSION:
+	case AFE_MTKAIF_IPM_VER_MON:
+	case AFE_MTKAIF_MON:
+	case AFE_AUD_PAD_TOP_MON:
+	case AFE_ADDA_MTKAIFV4_MON0 ... AFE_ADDA6_MTKAIFV4_MON0:
+	case ETDM_IN0_MON:
+	case ETDM_IN1_MON:
+	case ETDM_IN2_MON:
+	case ETDM_IN4_MON:
+	case ETDM_IN6_MON:
+	case ETDM_OUT0_MON:
+	case ETDM_OUT1_MON:
+	case ETDM_OUT2_MON:
+	case ETDM_OUT4_MON:
+	case ETDM_OUT6_MON:
+	case AFE_DPTX_MON:
+	case AFE_TDM_TOP_IP_VERSION:
+	case AFE_CONN_MON0 ... AFE_CONN_MON5:
+	case AFE_CBIP_SLV_DECODER_MON0 ... AFE_CBIP_SLV_MUX_MON1:
+	case AFE_DL0_CUR_MSB ... AFE_DL0_CUR:
+	case AFE_DL0_RCH_MON ... AFE_DL0_LCH_MON:
+	case AFE_DL1_CUR_MSB ... AFE_DL1_CUR:
+	case AFE_DL1_RCH_MON ... AFE_DL1_LCH_MON:
+	case AFE_DL2_CUR_MSB ... AFE_DL2_CUR:
+	case AFE_DL2_RCH_MON ... AFE_DL2_LCH_MON:
+	case AFE_DL3_CUR_MSB ... AFE_DL3_CUR:
+	case AFE_DL3_RCH_MON ... AFE_DL3_LCH_MON:
+	case AFE_DL4_CUR_MSB ... AFE_DL4_CUR:
+	case AFE_DL4_RCH_MON ... AFE_DL4_LCH_MON:
+	case AFE_DL5_CUR_MSB ... AFE_DL5_CUR:
+	case AFE_DL5_RCH_MON ... AFE_DL5_LCH_MON:
+	case AFE_DL6_CUR_MSB ... AFE_DL6_CUR:
+	case AFE_DL6_RCH_MON ... AFE_DL6_LCH_MON:
+	case AFE_DL7_CUR_MSB ... AFE_DL7_CUR:
+	case AFE_DL7_RCH_MON ... AFE_DL7_LCH_MON:
+	case AFE_DL8_CUR_MSB ... AFE_DL8_CUR:
+	case AFE_DL8_RCH_MON ... AFE_DL8_LCH_MON:
+	case AFE_DL_24CH_CUR_MSB ... AFE_DL_24CH_CUR:
+	case AFE_DL_4CH_CUR_MSB ... AFE_DL_4CH_CUR:
+	case AFE_DL23_CUR_MSB ... AFE_DL23_CUR:
+	case AFE_DL23_RCH_MON ... AFE_DL23_LCH_MON:
+	case AFE_DL24_CUR_MSB ... AFE_DL24_CUR:
+	case AFE_DL24_RCH_MON ... AFE_DL24_LCH_MON:
+	case AFE_DL25_CUR_MSB ... AFE_DL25_CUR:
+	case AFE_DL25_RCH_MON ... AFE_DL25_LCH_MON:
+	case AFE_DL26_CUR_MSB ... AFE_DL26_CUR:
+	case AFE_DL26_RCH_MON ... AFE_DL26_LCH_MON:
+	case AFE_VUL0_CUR_MSB ... AFE_VUL0_CUR:
+	case AFE_VUL1_CUR_MSB ... AFE_VUL1_CUR:
+	case AFE_VUL2_CUR_MSB ... AFE_VUL2_CUR:
+	case AFE_VUL3_CUR_MSB ... AFE_VUL3_CUR:
+	case AFE_VUL4_CUR_MSB ... AFE_VUL4_CUR:
+	case AFE_VUL5_CUR_MSB ... AFE_VUL5_CUR:
+	case AFE_VUL6_CUR_MSB ... AFE_VUL6_CUR:
+	case AFE_VUL7_CUR_MSB ... AFE_VUL7_CUR:
+	case AFE_VUL8_CUR_MSB ... AFE_VUL8_CUR:
+	case AFE_VUL9_CUR_MSB ... AFE_VUL9_CUR:
+	case AFE_VUL10_CUR_MSB ... AFE_VUL10_CUR:
+	case AFE_VUL24_CUR_MSB ... AFE_VUL24_CUR:
+	case AFE_VUL25_CUR_MSB ... AFE_VUL25_CUR:
+	case AFE_VUL25_RCH_MON ... AFE_VUL25_LCH_MON:
+	case AFE_VUL26_CUR_MSB ... AFE_VUL26_CUR:
+	case AFE_VUL_CM0_BASE_MSB ... AFE_VUL_CM0_CON0:
+	case AFE_VUL_CM1_CUR_MSB ... AFE_VUL_CM1_CUR:
+	case AFE_VUL_CM2_CUR_MSB ... AFE_VUL_CM2_CUR:
+	case AFE_ETDM_IN0_CUR_MSB ... AFE_ETDM_IN0_CUR:
+	case AFE_ETDM_IN1_CUR_MSB ... AFE_ETDM_IN1_CUR:
+	case AFE_ETDM_IN2_CUR_MSB ... AFE_ETDM_IN2_CUR:
+	case AFE_ETDM_IN3_CUR_MSB ... AFE_ETDM_IN3_CUR:
+	case AFE_ETDM_IN4_CUR_MSB ... AFE_ETDM_IN4_CUR:
+	case AFE_ETDM_IN6_CUR_MSB ... AFE_ETDM_IN6_CUR:
+	case AFE_HDMI_OUT_CUR_MSB ... AFE_HDMI_OUT_CUR:
+	case AFE_HDMI_OUT_END:
+	case AFE_PROT_SIDEBAND0_MON ... AFE_DOMAIN_SIDEBAND9_MON:
+	case AFE_PCM0_INTF_CON1_MASK_MON ... AFE_ADDA_UL1_SRC_CON0_MASK_MON:
+	case AFE_IRQ_MCU_EN ... AFE_IRQ_MCU_DSP2_EN:
+	case AFE_CUSTOM_IRQ_MCU_EN:
+	case AFE_DL5_CON0:
+	case AFE_DL6_CON0:
+	case AFE_DL23_CON0:
+	case AFE_DL_24CH_CON0:
+	case AFE_VUL1_CON0:
+	case AFE_VUL3_CON0:
+	case AFE_VUL4_CON0:
+	case AFE_VUL5_CON0:
+	case AFE_VUL9_CON0:
+	case AFE_VUL25_CON0:
+	case AFE_IRQ0_MCU_CFG0 ... AFE_IRQ26_MCU_CFG1:
+		return true;
+	default:
+		return false;
+	};
+}
+
+static const struct regmap_config mt8196_afe_regmap_config = {
+	.reg_bits = 32,
+	.reg_stride = 4,
+	.val_bits = 32,
+
+	.volatile_reg = mt8196_is_volatile_reg,
+
+	.max_register = AFE_MAX_REGISTER,
+	.num_reg_defaults_raw = AFE_MAX_REGISTER,
+
+	.cache_type = REGCACHE_FLAT,
+};
+
+static irqreturn_t mt8196_afe_irq_handler(int irq_id, void *dev)
+{
+	struct mtk_base_afe *afe = dev;
+	struct mtk_base_afe_irq *irq;
+	u32 status;
+	u32 status_mcu;
+	u32 mcu_en;
+	u32 cus_status;
+	u32 cus_status_mcu;
+	u32 cus_mcu_en;
+	u32 tmp_reg;
+	int ret, cus_ret;
+	int i;
+	struct timespec64 ts64;
+	u64 t1, t2;
+	/* one interrupt period = 5ms */
+	const u64 timeout_limit = 5000000;
+
+	/* get irq that is sent to MCU */
+	regmap_read(afe->regmap, AFE_IRQ_MCU_EN, &mcu_en);
+	regmap_read(afe->regmap, AFE_CUSTOM_IRQ_MCU_EN, &cus_mcu_en);
+
+	ret = regmap_read(afe->regmap, AFE_IRQ_MCU_STATUS, &status);
+	cus_ret = regmap_read(afe->regmap, AFE_CUSTOM_IRQ_MCU_STATUS, &cus_status);
+	/* only care IRQ which is sent to MCU */
+	status_mcu = status & mcu_en & AFE_IRQ_STATUS_BITS;
+	cus_status_mcu = cus_status & cus_mcu_en & AFE_IRQ_STATUS_BITS;
+	if ((ret || !status_mcu) && (cus_ret || !cus_status_mcu)) {
+		dev_err(afe->dev, "ret %d, sat 0x%x, en 0x%x,csat 0x%x, cen 0x%x\n",
+			ret, status, mcu_en, cus_status_mcu, cus_mcu_en);
+		return IRQ_NONE;
+	}
+
+	ktime_get_ts64(&ts64);
+	t1 = ktime_get_ns();
+
+	for (i = 0; i < MT8196_MEMIF_NUM; i++) {
+		struct mtk_base_afe_memif *memif = &afe->memif[i];
+
+		if (!memif->substream)
+			continue;
+
+		if (memif->irq_usage < 0)
+			continue;
+		irq = &afe->irqs[memif->irq_usage];
+
+		if (i == MT8196_MEMIF_HDMI) {
+			if (cus_status_mcu & BIT(irq->irq_data->id))
+				snd_pcm_period_elapsed(memif->substream);
+		} else {
+			if (status_mcu & BIT(irq->irq_data->id))
+				snd_pcm_period_elapsed(memif->substream);
+		}
+	}
+
+	ktime_get_ts64(&ts64);
+	t2 = ktime_get_ns();
+	t2 = t2 - t1; /* in ns (10^9) */
+
+	if (t2 > timeout_limit)
+		dev_warn(afe->dev, "IRQ handler exceeded time limit by %llu ns\n",
+			 t2 - timeout_limit);
+
+	/* clear irq */
+	for (i = 0; i < MT8196_IRQ_NUM; ++i) {
+		/* cus_status_mcu only bit0 is used for TDM */
+		if ((status_mcu & BIT(i)) || (cus_status_mcu & 0x1)) {
+			regmap_read(afe->regmap, irq_data[i].irq_clr_reg, &tmp_reg);
+			regmap_update_bits(afe->regmap, irq_data[i].irq_clr_reg,
+					   AFE_IRQ_CLR_CFG_MASK_SFT |
+					   AFE_IRQ_MISS_FLAG_CLR_CFG_MASK_SFT,
+					   tmp_reg ^ (AFE_IRQ_CLR_CFG_MASK_SFT |
+					   AFE_IRQ_MISS_FLAG_CLR_CFG_MASK_SFT));
+		}
+	}
+
+	return IRQ_HANDLED;
+}
+
+static int mt8196_afe_runtime_suspend(struct device *dev)
+{
+	struct mtk_base_afe *afe = dev_get_drvdata(dev);
+	unsigned int value;
+	unsigned int tmp_reg;
+	int ret, i;
+
+	if (!afe->regmap) {
+		dev_err(afe->dev, "skip regmap\n");
+		goto skip_regmap;
+	}
+
+	/* disable AFE */
+	mt8196_afe_disable_main_clock(afe);
+
+	ret = regmap_read_poll_timeout(afe->regmap,
+				       AUDIO_ENGEN_CON0_MON,
+				       value,
+				       (value & AUDIO_ENGEN_MON_SFT) == 0,
+				       20,
+				       1 * 1000 * 1000);
+	dev_dbg(afe->dev, "read_poll ret %d\n", ret);
+	if (ret)
+		dev_warn(afe->dev, "ret %d\n", ret);
+
+	/* make sure all irq status are cleared */
+	for (i = 0; i < MT8196_IRQ_NUM; ++i) {
+		regmap_read(afe->regmap, irq_data[i].irq_clr_reg, &tmp_reg);
+		regmap_update_bits(afe->regmap, irq_data[i].irq_clr_reg,
+				   AFE_IRQ_CLR_CFG_MASK_SFT | AFE_IRQ_MISS_FLAG_CLR_CFG_MASK_SFT,
+				   tmp_reg ^ (AFE_IRQ_CLR_CFG_MASK_SFT |
+				   AFE_IRQ_MISS_FLAG_CLR_CFG_MASK_SFT));
+	}
+
+	/* reset audio 26M request */
+	regmap_update_bits(afe->regmap,
+			   AFE_SPM_CONTROL_REQ, 0x1, 0x0);
+
+	/* cache only */
+	regcache_cache_only(afe->regmap, true);
+	regcache_mark_dirty(afe->regmap);
+
+skip_regmap:
+	mt8196_afe_disable_reg_rw_clk(afe);
+	return 0;
+}
+
+static int mt8196_afe_runtime_resume(struct device *dev)
+{
+	struct mtk_base_afe *afe = dev_get_drvdata(dev);
+	int ret = 0;
+
+	ret = mt8196_afe_enable_reg_rw_clk(afe);
+	if (ret)
+		return ret;
+
+	if (!afe->regmap) {
+		dev_warn(afe->dev, "skip regmap\n");
+		goto skip_regmap;
+	}
+	regcache_cache_only(afe->regmap, false);
+	regcache_sync(afe->regmap);
+
+	/* set audio 26M request */
+	regmap_update_bits(afe->regmap, AFE_SPM_CONTROL_REQ, 0x1, 0x1);
+	regmap_update_bits(afe->regmap, AFE_CBIP_CFG0, 0x1, 0x1);
+
+	/* force cpu use 8_24 format when writing 32bit data */
+	regmap_update_bits(afe->regmap, AFE_MEMIF_CON0,
+			   CPU_HD_ALIGN_MASK_SFT, 0 << CPU_HD_ALIGN_SFT);
+
+	/* enable AFE */
+	mt8196_afe_enable_main_clock(afe);
+
+skip_regmap:
+	return 0;
+}
+
+static int mt8196_afe_component_probe(struct snd_soc_component *component)
+{
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(component);
+	int ret;
+
+	/* enable clock for regcache get default value from hw */
+	ret = pm_runtime_resume_and_get(afe->dev);
+	if (ret)
+		return dev_err_probe(afe->dev, ret, "failed to resume device\n");
+
+	mtk_afe_add_sub_dai_control(component);
+	pm_runtime_put_sync(afe->dev);
+
+	return 0;
+}
+
+static int mt8196_afe_pcm_open(struct snd_soc_component *component,
+			       struct snd_pcm_substream *substream)
+{
+	/* set the wait_for_avail to 2 sec*/
+	substream->wait_time = msecs_to_jiffies(2 * 1000);
+
+	return 0;
+}
+
+static void mt8196_afe_pcm_free(struct snd_soc_component *component, struct snd_pcm *pcm)
+{
+	snd_pcm_lib_preallocate_free_for_all(pcm);
+}
+
+static const struct snd_soc_component_driver mt8196_afe_component = {
+	.name = AFE_PCM_NAME,
+	.probe = mt8196_afe_component_probe,
+	.pcm_new = mtk_afe_pcm_new,
+	.pcm_free = mt8196_afe_pcm_free,
+	.open = mt8196_afe_pcm_open,
+	.pointer = mtk_afe_pcm_pointer,
+};
+
+static int mt8196_dai_memif_register(struct mtk_base_afe *afe)
+{
+	struct mtk_base_afe_dai *dai;
+
+	dai = devm_kzalloc(afe->dev, sizeof(*dai), GFP_KERNEL);
+	if (!dai)
+		return -ENOMEM;
+
+	list_add(&dai->list, &afe->sub_dais);
+
+	dai->dai_drivers = mt8196_memif_dai_driver;
+	dai->num_dai_drivers = ARRAY_SIZE(mt8196_memif_dai_driver);
+	dai->dapm_widgets = mt8196_memif_widgets;
+	dai->num_dapm_widgets = ARRAY_SIZE(mt8196_memif_widgets);
+	dai->dapm_routes = mt8196_memif_routes;
+	dai->num_dapm_routes = ARRAY_SIZE(mt8196_memif_routes);
+	return 0;
+}
+
+typedef int (*dai_register_cb)(struct mtk_base_afe *);
+static const dai_register_cb dai_register_cbs[] = {
+	mt8196_dai_adda_register,
+	mt8196_dai_i2s_register,
+	mt8196_dai_tdm_register,
+	mt8196_dai_memif_register,
+};
+
+static const struct reg_sequence mt8196_cg_patch[] = {
+	{ AUDIO_TOP_CON4, 0x361c },
+};
+
+static void mt8196_afe_release_reserved_mem(void *data)
+{
+	of_reserved_mem_device_release(data);
+}
+
+static int mt8196_afe_pcm_dev_probe(struct platform_device *pdev)
+{
+	int ret, i;
+	unsigned int tmp_reg;
+	int irq_id;
+	struct mtk_base_afe *afe;
+	struct mt8196_afe_private *afe_priv;
+	struct device *dev = &pdev->dev;
+
+	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(34));
+	if (ret)
+		return ret;
+
+	ret = of_reserved_mem_device_init(dev);
+	if (ret) {
+		dev_err(dev, "failed to assign memory region: %d\n", ret);
+	} else {
+		ret = devm_add_action_or_reset(dev, mt8196_afe_release_reserved_mem, dev);
+		if (ret)
+			return ret;
+	}
+
+	afe = devm_kzalloc(dev, sizeof(*afe), GFP_KERNEL);
+	if (!afe)
+		return -ENOMEM;
+
+	platform_set_drvdata(pdev, afe);
+
+	afe->platform_priv = devm_kzalloc(dev, sizeof(*afe_priv),
+					  GFP_KERNEL);
+	if (!afe->platform_priv)
+		return -ENOMEM;
+
+	afe_priv = afe->platform_priv;
+	afe->dev = dev;
+
+	afe->base_addr = devm_platform_ioremap_resource(pdev, 0);
+	if (IS_ERR(afe->base_addr))
+		return dev_err_probe(dev, PTR_ERR(afe->base_addr),
+				     "AFE base_addr not found\n");
+
+	/* init audio related clock */
+	ret = mt8196_init_clock(afe);
+	if (ret)
+		return dev_err_probe(dev, ret, "init clock error.\n");
+
+	/* init memif */
+	/* IPM2.0 no need banding */
+	afe->memif_32bit_supported = 1;
+	afe->memif_size = MT8196_MEMIF_NUM;
+	afe->memif = devm_kcalloc(dev, afe->memif_size, sizeof(*afe->memif),
+				  GFP_KERNEL);
+
+	if (!afe->memif)
+		return -ENOMEM;
+
+	for (i = 0; i < afe->memif_size; i++) {
+		afe->memif[i].data = &memif_data[i];
+		afe->memif[i].irq_usage = memif_irq_usage[i];
+		afe->memif[i].const_irq = 1;
+	}
+
+	mutex_init(&afe->irq_alloc_lock);
+
+	/* init irq */
+	afe->irqs_size = MT8196_IRQ_NUM;
+	afe->irqs = devm_kcalloc(dev, afe->irqs_size, sizeof(*afe->irqs),
+				 GFP_KERNEL);
+
+	if (!afe->irqs)
+		return -ENOMEM;
+
+	for (i = 0; i < afe->irqs_size; i++)
+		afe->irqs[i].irq_data = &irq_data[i];
+
+	/* request irq */
+	irq_id = platform_get_irq(pdev, 0);
+	if (irq_id < 0)
+		return dev_err_probe(dev, irq_id, "no irq found");
+
+	ret = devm_request_irq(dev, irq_id, mt8196_afe_irq_handler,
+			       IRQF_TRIGGER_NONE,
+			       "Afe_ISR_Handle", afe);
+	if (ret)
+		return dev_err_probe(dev, ret, "could not request_irq for Afe_ISR_Handle\n");
+
+	/* init sub_dais */
+	INIT_LIST_HEAD(&afe->sub_dais);
+
+	for (i = 0; i < ARRAY_SIZE(dai_register_cbs); i++) {
+		ret = dai_register_cbs[i](afe);
+		if (ret)
+			return dev_err_probe(dev, ret, "dai register i %d fail\n", i);
+	}
+
+	/* init dai_driver and component_driver */
+	ret = mtk_afe_combine_sub_dai(afe);
+	if (ret)
+		return dev_err_probe(dev, ret, "mtk_afe_combine_sub_dai fail\n");
+
+	/* others */
+	afe->mtk_afe_hardware = &mt8196_afe_hardware;
+	afe->memif_fs = mt8196_memif_fs;
+	afe->irq_fs = mt8196_irq_fs;
+	afe->get_dai_fs = mt8196_get_dai_fs;
+	afe->get_memif_pbuf_size = mt8196_get_memif_pbuf_size;
+
+	afe->runtime_resume = mt8196_afe_runtime_resume;
+	afe->runtime_suspend = mt8196_afe_runtime_suspend;
+
+	ret = devm_pm_runtime_enable(dev);
+	if (ret)
+		return ret;
+
+/*
+ * Audio device is part of genpd. Registering it as a syscore device ensure
+ * the proper power-on sequence of the AFE device.
+ */
+	dev_pm_syscore_device(dev, true);
+
+	/* enable clock for regcache get default value from hw */
+	ret = pm_runtime_resume_and_get(dev);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to resume device\n");
+
+	afe->regmap = devm_regmap_init_mmio(dev, afe->base_addr,
+					    &mt8196_afe_regmap_config);
+	if (IS_ERR(afe->regmap)) {
+		ret = PTR_ERR(afe->regmap);
+		goto err_pm_put;
+	}
+
+	ret = regmap_register_patch(afe->regmap, mt8196_cg_patch,
+				    ARRAY_SIZE(mt8196_cg_patch));
+	if (ret < 0) {
+		dev_err(dev, "Failed to apply cg patch\n");
+		goto err_pm_put;
+	}
+
+	regmap_read(afe->regmap, AFE_IRQ_MCU_EN, &tmp_reg);
+	regmap_write(afe->regmap, AFE_IRQ_MCU_EN, 0xffffffff);
+	regmap_read(afe->regmap, AFE_IRQ_MCU_EN, &tmp_reg);
+
+	pm_runtime_put_sync(dev);
+
+	regcache_cache_only(afe->regmap, true);
+	regcache_mark_dirty(afe->regmap);
+
+	/* register component */
+	ret = devm_snd_soc_register_component(dev,
+					      &mt8196_afe_component,
+					      afe->dai_drivers,
+					      afe->num_dai_drivers);
+	if (ret) {
+		dev_err(dev, "afe component err\n");
+		return ret;
+	}
+
+	return 0;
+
+err_pm_put:
+	pm_runtime_put_sync(dev);
+	return ret;
+}
+
+static void mt8196_afe_pcm_dev_remove(struct platform_device *pdev)
+{
+	struct mtk_base_afe *afe = platform_get_drvdata(pdev);
+	struct device *dev = &pdev->dev;
+
+	if (!pm_runtime_status_suspended(dev))
+		mt8196_afe_runtime_suspend(dev);
+
+	mt8196_afe_disable_main_clock(afe);
+	/* disable afe clock */
+	mt8196_afe_disable_reg_rw_clk(afe);
+}
+
+static const struct of_device_id mt8196_afe_pcm_dt_match[] = {
+	{ .compatible = "mediatek,mt8196-afe", },
+	{ /* sentinel */ },
+};
+MODULE_DEVICE_TABLE(of, mt8196_afe_pcm_dt_match);
+
+static const struct dev_pm_ops mt8196_afe_pm_ops = {
+	SET_RUNTIME_PM_OPS(mt8196_afe_runtime_suspend,
+			   mt8196_afe_runtime_resume, NULL)
+};
+
+static struct platform_driver mt8196_afe_pcm_driver = {
+	.driver = {
+		.name = "mt8196-afe",
+		.of_match_table = mt8196_afe_pcm_dt_match,
+		.pm = &mt8196_afe_pm_ops,
+	},
+	.probe = mt8196_afe_pcm_dev_probe,
+	.remove = mt8196_afe_pcm_dev_remove,
+};
+module_platform_driver(mt8196_afe_pcm_driver);
+
+MODULE_DESCRIPTION("Mediatek ALSA SoC AFE platform driver for 8196");
+MODULE_AUTHOR("Darren Ye <darren.ye@mediatek.com>");
+MODULE_LICENSE("GPL");
diff --git a/sound/soc/mediatek/mt8196/mt8196-dai-adda.c b/sound/soc/mediatek/mt8196/mt8196-dai-adda.c
new file mode 100644
index 0000000000000..9a91db4e79ae8
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-dai-adda.c
@@ -0,0 +1,845 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  MediaTek ALSA SoC Audio DAI ADDA Control
+ *
+ *  Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#include <linux/delay.h>
+#include <linux/regmap.h>
+
+#include "mt8196-afe-clk.h"
+#include "mt8196-afe-common.h"
+#include "mt8196-interconnection.h"
+
+enum {
+	UL_IIR_SW,
+	UL_IIR_5HZ,
+	UL_IIR_10HZ,
+	UL_IIR_25HZ,
+	UL_IIR_50HZ,
+	UL_IIR_75HZ,
+};
+
+enum {
+	MTK_AFE_ADDA_UL_RATE_8K,
+	MTK_AFE_ADDA_UL_RATE_16K,
+	MTK_AFE_ADDA_UL_RATE_32K,
+	MTK_AFE_ADDA_UL_RATE_48K,
+	MTK_AFE_ADDA_UL_RATE_96K,
+	MTK_AFE_ADDA_UL_RATE_192K,
+	MTK_AFE_ADDA_UL_RATE_48K_HD,
+};
+
+enum {
+	MTK_AFE_MTKAIF_RATE_8K,
+	MTK_AFE_MTKAIF_RATE_12K,
+	MTK_AFE_MTKAIF_RATE_16K,
+	MTK_AFE_MTKAIF_RATE_24K,
+	MTK_AFE_MTKAIF_RATE_32K,
+	MTK_AFE_MTKAIF_RATE_48K,
+	MTK_AFE_MTKAIF_RATE_64K,
+	MTK_AFE_MTKAIF_RATE_96K,
+	MTK_AFE_MTKAIF_RATE_128K,
+	MTK_AFE_MTKAIF_RATE_192K,
+	MTK_AFE_MTKAIF_RATE_256K,
+	MTK_AFE_MTKAIF_RATE_384K,
+	MTK_AFE_MTKAIF_RATE_11K = 0x10,
+	MTK_AFE_MTKAIF_RATE_22K,
+	MTK_AFE_MTKAIF_RATE_44K,
+	MTK_AFE_MTKAIF_RATE_88K,
+	MTK_AFE_MTKAIF_RATE_176K,
+	MTK_AFE_MTKAIF_RATE_352K,
+};
+
+struct mtk_afe_adda_priv {
+	int dl_rate;
+	int ul_rate;
+};
+
+static unsigned int adda_ul_rate_transform(struct mtk_base_afe *afe,
+					   unsigned int rate)
+{
+	switch (rate) {
+	case 8000:
+		return MTK_AFE_ADDA_UL_RATE_8K;
+	case 16000:
+		return MTK_AFE_ADDA_UL_RATE_16K;
+	case 32000:
+		return MTK_AFE_ADDA_UL_RATE_32K;
+	case 48000:
+		return MTK_AFE_ADDA_UL_RATE_48K;
+	case 96000:
+		return MTK_AFE_ADDA_UL_RATE_96K;
+	case 192000:
+		return MTK_AFE_ADDA_UL_RATE_192K;
+	default:
+		dev_warn(afe->dev, "rate %d invalid, use 48kHz!!!\n", rate);
+		return MTK_AFE_ADDA_UL_RATE_48K;
+	}
+}
+
+static unsigned int mtkaif_rate_transform(struct mtk_base_afe *afe,
+					  unsigned int rate)
+{
+	switch (rate) {
+	case 8000:
+		return MTK_AFE_MTKAIF_RATE_8K;
+	case 11025:
+		return MTK_AFE_MTKAIF_RATE_11K;
+	case 12000:
+		return MTK_AFE_MTKAIF_RATE_12K;
+	case 16000:
+		return MTK_AFE_MTKAIF_RATE_16K;
+	case 22050:
+		return MTK_AFE_MTKAIF_RATE_22K;
+	case 24000:
+		return MTK_AFE_MTKAIF_RATE_24K;
+	case 32000:
+		return MTK_AFE_MTKAIF_RATE_32K;
+	case 44100:
+		return MTK_AFE_MTKAIF_RATE_44K;
+	case 48000:
+		return MTK_AFE_MTKAIF_RATE_48K;
+	case 96000:
+		return MTK_AFE_MTKAIF_RATE_96K;
+	case 192000:
+		return MTK_AFE_MTKAIF_RATE_192K;
+	default:
+		dev_warn(afe->dev, "rate %d invalid, use 48kHz!!!\n", rate);
+		return MTK_AFE_MTKAIF_RATE_48K;
+	}
+}
+
+enum {
+	SUPPLY_SEQ_ADDA_AFE_ON,
+	SUPPLY_SEQ_ADDA_FIFO,
+	SUPPLY_SEQ_ADDA_AP_DMIC,
+	SUPPLY_SEQ_ADDA_UL_ON,
+};
+
+static int mtk_adda_ul_src_set_dmic_phase_sync(struct mtk_base_afe *afe)
+{
+	dev_dbg(afe->dev, "set dmic phase sync\n");
+	// ul0~1
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON1,
+			   UL0_PHASE_SYNC_HCLK_SET_MASK_SFT,
+			   0x1 << UL0_PHASE_SYNC_HCLK_SET_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON1,
+			   UL0_PHASE_SYNC_FCLK_SET_MASK_SFT,
+			   0x1 << UL0_PHASE_SYNC_FCLK_SET_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON1,
+			   UL1_PHASE_SYNC_HCLK_SET_MASK_SFT,
+			   0x1 << UL1_PHASE_SYNC_HCLK_SET_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON1,
+			   UL1_PHASE_SYNC_FCLK_SET_MASK_SFT,
+			   0x1 << UL1_PHASE_SYNC_FCLK_SET_SFT);
+	// dmic 0
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON1,
+			   DMIC0_PHASE_SYNC_FCLK_SET_MASK_SFT,
+			   0x1 << DMIC0_PHASE_SYNC_FCLK_SET_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON1,
+			   DMIC0_PHASE_SYNC_HCLK_SET_MASK_SFT,
+			   0x1 << DMIC0_PHASE_SYNC_HCLK_SET_SFT);
+	// dmic 1
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON1,
+			   DMIC1_PHASE_SYNC_FCLK_SET_MASK_SFT,
+			   0x1 << DMIC1_PHASE_SYNC_FCLK_SET_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON1,
+			   DMIC1_PHASE_SYNC_HCLK_SET_MASK_SFT,
+			   0x1 << DMIC1_PHASE_SYNC_HCLK_SET_SFT);
+	// ul0~1 phase sync clock
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   DMIC1_PHASE_HCLK_SEL_MASK_SFT,
+			   0x1 << DMIC1_PHASE_HCLK_SEL_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   DMIC1_PHASE_FCLK_SEL_MASK_SFT,
+			   0x1 << DMIC1_PHASE_FCLK_SEL_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   DMIC0_PHASE_HCLK_SEL_MASK_SFT,
+			   0x1 << DMIC0_PHASE_HCLK_SEL_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   DMIC0_PHASE_FCLK_SEL_MASK_SFT,
+			   0x1 << DMIC0_PHASE_FCLK_SEL_SFT);
+	// dmic 0
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   UL1_PHASE_HCLK_SEL_MASK_SFT,
+			   0x2 << UL1_PHASE_HCLK_SEL_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   UL1_PHASE_FCLK_SEL_MASK_SFT,
+			   0x2 << UL1_PHASE_FCLK_SEL_SFT);
+	// dmic 1
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   UL0_PHASE_HCLK_SEL_MASK_SFT,
+			   0x2 << UL0_PHASE_HCLK_SEL_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   UL0_PHASE_FCLK_SEL_MASK_SFT,
+			   0x2 << UL0_PHASE_FCLK_SEL_SFT);
+
+	return 0;
+}
+
+static int mtk_adda_ul_src_set_dmic_phase_sync_clock(struct mtk_base_afe *afe)
+{
+	dev_dbg(afe->dev, "dmic turn on phase sync clk\n");
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   UL_PHASE_SYNC_HCLK_1_ON_MASK_SFT,
+			   0x1 << UL_PHASE_SYNC_HCLK_1_ON_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   UL_PHASE_SYNC_HCLK_0_ON_MASK_SFT,
+			   0x1 << UL_PHASE_SYNC_HCLK_0_ON_SFT);
+
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   UL_PHASE_SYNC_FCLK_1_ON_MASK_SFT,
+			   0x1 << UL_PHASE_SYNC_FCLK_1_ON_SFT);
+	regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON0,
+			   UL_PHASE_SYNC_FCLK_0_ON_MASK_SFT,
+			   0x1 << UL_PHASE_SYNC_FCLK_0_ON_SFT);
+
+	return 0;
+}
+
+static int mtk_adda_ul_src_enable_dmic(struct mtk_base_afe *afe, int id)
+{
+	unsigned int reg_con0 = 0, reg_con1 = 0;
+
+	dev_dbg(afe->dev, "id: %d\n", id);
+
+	switch (id) {
+	case MT8196_DAI_ADDA:
+	case MT8196_DAI_AP_DMIC:
+		reg_con0 = AFE_ADDA_UL0_SRC_CON0;
+		reg_con1 = AFE_ADDA_UL0_SRC_CON1;
+		break;
+	case MT8196_DAI_ADDA_CH34:
+	case MT8196_DAI_AP_DMIC_CH34:
+		reg_con0 = AFE_ADDA_UL1_SRC_CON0;
+		reg_con1 = AFE_ADDA_UL1_SRC_CON1;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	switch (id) {
+	case MT8196_DAI_AP_DMIC:
+		dev_dbg(afe->dev, "clear mtkaifv4 ul ch1ch2 mux\n");
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_UL_CH1CH2_IN_EN_SEL_MASK_SFT,
+				   0x0 << MTKAIFV4_UL_CH1CH2_IN_EN_SEL_SFT);
+		break;
+	case MT8196_DAI_AP_DMIC_CH34:
+		dev_dbg(afe->dev, "clear mtkaifv4 ul ch3ch4 mux\n");
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_UL_CH3CH4_IN_EN_SEL_MASK_SFT,
+				   0x0 << MTKAIFV4_UL_CH3CH4_IN_EN_SEL_SFT);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	/* choose Phase */
+	regmap_update_bits(afe->regmap, reg_con0,
+			   UL_DMIC_PHASE_SEL_CH1_MASK_SFT,
+			   0x0 << UL_DMIC_PHASE_SEL_CH1_SFT);
+	regmap_update_bits(afe->regmap, reg_con0,
+			   UL_DMIC_PHASE_SEL_CH2_MASK_SFT,
+			   0x4 << UL_DMIC_PHASE_SEL_CH2_SFT);
+
+	/* dmic mode, 3.25M*/
+	regmap_update_bits(afe->regmap, reg_con0,
+			   DIGMIC_3P25M_1P625M_SEL_CTL_MASK_SFT,
+			   0x0);
+	regmap_update_bits(afe->regmap, reg_con0,
+			   DMIC_LOW_POWER_MODE_CTL_MASK_SFT,
+			   0x0);
+
+	/* turn on dmic, ch1, ch2 */
+	regmap_update_bits(afe->regmap, reg_con0,
+			   UL_SDM_3_LEVEL_CTL_MASK_SFT,
+			   0x1 << UL_SDM_3_LEVEL_CTL_SFT);
+	regmap_update_bits(afe->regmap, reg_con0,
+			   UL_MODE_3P25M_CH1_CTL_MASK_SFT,
+			   0x1 << UL_MODE_3P25M_CH1_CTL_SFT);
+	regmap_update_bits(afe->regmap, reg_con0,
+			   UL_MODE_3P25M_CH2_CTL_MASK_SFT,
+			   0x1 << UL_MODE_3P25M_CH2_CTL_SFT);
+
+	/* ul gain:  gain = 0x7fff/positive_gain = 0x0/gain_mode = 0x10 */
+	regmap_update_bits(afe->regmap, reg_con1,
+			   ADDA_UL_GAIN_VALUE_MASK_SFT,
+			   0x7fff << ADDA_UL_GAIN_VALUE_SFT);
+	regmap_update_bits(afe->regmap, reg_con1,
+			   ADDA_UL_POSTIVEGAIN_MASK_SFT,
+			   0x0 << ADDA_UL_POSTIVEGAIN_SFT);
+	/* gain_mode = 0x10: Add 0.5 gain at CIC output */
+	regmap_update_bits(afe->regmap, reg_con1,
+			   GAIN_MODE_MASK_SFT,
+			   0x02 << GAIN_MODE_SFT);
+	return 0;
+}
+
+static int mtk_adda_sleep_on_pmd_event(struct snd_soc_dapm_widget *w,
+				       struct snd_kcontrol *kcontrol, int event)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(w->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+
+	dev_dbg(afe->dev, "name %s, event 0x%x\n", w->name, event);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		/* should delayed 1/fs(smallest is 8k) = 125us before afe off */
+		usleep_range(120, 130);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+/* ADDA UL MUX */
+#define ADDA_UL_MUX_MASK 0x3
+enum {
+	ADDA_UL_MUX_MTKAIF = 0,
+	ADDA_UL_MUX_AP_DMIC,
+	ADDA_UL_MUX_AP_DMIC_MULTICH,
+};
+
+static const char *const adda_ul_mux_map[] = {
+	"MTKAIF", "AP_DMIC", "AP_DMIC_MULTI_CH",
+};
+
+static int adda_ul_map_value[] = {
+	ADDA_UL_MUX_MTKAIF,
+	ADDA_UL_MUX_AP_DMIC,
+	ADDA_UL_MUX_AP_DMIC_MULTICH,
+};
+
+static SOC_VALUE_ENUM_SINGLE_DECL(adda_ul_mux_map_enum,
+				  SND_SOC_NOPM,
+				  0,
+				  ADDA_UL_MUX_MASK,
+				  adda_ul_mux_map,
+				  adda_ul_map_value);
+
+static const struct snd_kcontrol_new adda_ul_mux_control =
+	SOC_DAPM_ENUM("ADDA_UL_MUX Select", adda_ul_mux_map_enum);
+
+static const struct snd_kcontrol_new adda_ch34_ul_mux_control =
+	SOC_DAPM_ENUM("ADDA_CH34_UL_MUX Select", adda_ul_mux_map_enum);
+
+static const struct snd_soc_dapm_widget mtk_dai_adda_widgets[] = {
+	/* inter-connections */
+	SND_SOC_DAPM_SUPPLY_S("ADDA Enable", SUPPLY_SEQ_ADDA_AFE_ON,
+			      AUDIO_ENGEN_CON0, AUDIO_F3P25M_EN_ON_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("ADDA Capture Enable", SUPPLY_SEQ_ADDA_UL_ON,
+			      AFE_ADDA_UL0_SRC_CON0,
+			      UL_SRC_ON_TMP_CTL_SFT, 0,
+			      mtk_adda_sleep_on_pmd_event,
+			      SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S("ADDA CH34 Capture Enable", SUPPLY_SEQ_ADDA_UL_ON,
+			      AFE_ADDA_UL1_SRC_CON0,
+			      UL_SRC_ON_TMP_CTL_SFT, 0,
+			      mtk_adda_sleep_on_pmd_event,
+			      SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S("AP_DMIC_EN", SUPPLY_SEQ_ADDA_AP_DMIC,
+			      AFE_ADDA_UL0_SRC_CON0,
+			      UL_AP_DMIC_ON_SFT, 0,
+			      mtk_adda_sleep_on_pmd_event,
+			      SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S("AP_DMIC_CH34_EN", SUPPLY_SEQ_ADDA_AP_DMIC,
+			      AFE_ADDA_UL1_SRC_CON0,
+			      UL_AP_DMIC_ON_SFT, 0,
+			      mtk_adda_sleep_on_pmd_event,
+			      SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S("ADDA_FIFO", SUPPLY_SEQ_ADDA_FIFO,
+			      AFE_ADDA_UL0_SRC_CON1,
+			      FIFO_SOFT_RST_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("ADDA_CH34_FIFO", SUPPLY_SEQ_ADDA_FIFO,
+			      AFE_ADDA_UL1_SRC_CON1,
+			      FIFO_SOFT_RST_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_MUX("ADDA_UL_Mux", SND_SOC_NOPM, 0, 0,
+			 &adda_ul_mux_control),
+	SND_SOC_DAPM_MUX("ADDA_CH34_UL_Mux", SND_SOC_NOPM, 0, 0,
+			 &adda_ch34_ul_mux_control),
+	SND_SOC_DAPM_INPUT("AP_DMIC_INPUT"),
+};
+
+static const struct snd_soc_dapm_route mtk_dai_adda_routes[] = {
+	/* capture */
+	{"ADDA_UL_Mux", "MTKAIF", "ADDA Capture"},
+	{"ADDA_UL_Mux", "AP_DMIC", "AP DMIC Capture"},
+	{"ADDA_UL_Mux", "AP_DMIC_MULTI_CH", "AP DMIC MULTICH Capture"},
+
+	{"ADDA_CH34_UL_Mux", "MTKAIF", "ADDA CH34 Capture"},
+	{"ADDA_CH34_UL_Mux", "AP_DMIC", "AP DMIC CH34 Capture"},
+	{"ADDA_CH34_UL_Mux", "AP_DMIC_MULTI_CH", "AP DMIC MULTICH Capture"},
+
+	{"AP DMIC Capture", NULL, "ADDA Enable"},
+	{"AP DMIC Capture", NULL, "ADDA Capture Enable"},
+	{"AP DMIC Capture", NULL, "ADDA_FIFO"},
+	{"AP DMIC Capture", NULL, "AP_DMIC_EN"},
+
+	{"AP DMIC CH34 Capture", NULL, "ADDA Enable"},
+	{"AP DMIC CH34 Capture", NULL, "ADDA CH34 Capture Enable"},
+	{"AP DMIC CH34 Capture", NULL, "ADDA_CH34_FIFO"},
+	{"AP DMIC CH34 Capture", NULL, "AP_DMIC_CH34_EN"},
+
+	{"AP DMIC MULTICH Capture", NULL, "ADDA Enable"},
+	{"AP DMIC MULTICH Capture", NULL, "ADDA Capture Enable"},
+	{"AP DMIC MULTICH Capture", NULL, "ADDA CH34 Capture Enable"},
+	{"AP DMIC MULTICH Capture", NULL, "ADDA_FIFO"},
+	{"AP DMIC MULTICH Capture", NULL, "ADDA_CH34_FIFO"},
+	{"AP DMIC MULTICH Capture", NULL, "AP_DMIC_EN"},
+	{"AP DMIC MULTICH Capture", NULL, "AP_DMIC_CH34_EN"},
+
+	{"AP DMIC Capture", NULL, "AP_DMIC_INPUT"},
+	{"AP DMIC CH34 Capture", NULL, "AP_DMIC_INPUT"},
+	{"AP DMIC MULTICH Capture", NULL, "AP_DMIC_INPUT"},
+};
+
+/* dai ops */
+static int set_playback_hw_params(struct snd_pcm_hw_params *params,
+				  struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	unsigned int rate = params_rate(params);
+	struct mtk_afe_adda_priv *adda_priv;
+	unsigned int mtkaif_rate = 0;
+	int id = dai->id;
+
+	adda_priv = afe_priv->dai_priv[id];
+	if (!adda_priv)
+		return -EINVAL;
+
+	adda_priv->dl_rate = rate;
+
+	/* get mtkaif dl rate */
+	mtkaif_rate = mtkaif_rate_transform(afe, adda_priv->dl_rate);
+
+	if (id == MT8196_DAI_ADDA) {
+		/* MTKAIF sample rate config */
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_TX_CFG0,
+				   MTKAIFV4_TXIF_INPUT_MODE_MASK_SFT,
+				   mtkaif_rate << MTKAIFV4_TXIF_INPUT_MODE_SFT);
+		/* AFE_ADDA_MTKAIFV4_TX_CFG0 */
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_TX_CFG0,
+				   MTKAIFV4_TXIF_FOUR_CHANNEL_MASK_SFT,
+				   0x0 << MTKAIFV4_TXIF_FOUR_CHANNEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_TX_CFG0,
+				   MTKAIFV4_ADDA_OUT_EN_SEL_MASK_SFT,
+				   0x1 << MTKAIFV4_ADDA_OUT_EN_SEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_TX_CFG0,
+				   MTKAIFV4_ADDA6_OUT_EN_SEL_MASK_SFT,
+				   0x1 << MTKAIFV4_ADDA6_OUT_EN_SEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_TX_CFG0,
+				   MTKAIFV4_TXIF_V4_MASK_SFT,
+				   0x1 << MTKAIFV4_TXIF_V4_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_TX_CFG0,
+				   MTKAIFV4_TXIF_EN_SEL_MASK_SFT,
+				   0x0 << MTKAIFV4_TXIF_EN_SEL_SFT);
+		/* clean predistortion */
+	} else {
+		/* MTKAIF sample rate config */
+		regmap_update_bits(afe->regmap, AFE_ADDA6_MTKAIFV4_TX_CFG0,
+				   ADDA6_MTKAIFV4_TXIF_INPUT_MODE_MASK_SFT,
+				   mtkaif_rate << ADDA6_MTKAIFV4_TXIF_INPUT_MODE_SFT);
+		/* AFE_ADDA6_MTKAIFV4_TX_CFG0 */
+		regmap_update_bits(afe->regmap, AFE_ADDA6_MTKAIFV4_TX_CFG0,
+				   ADDA6_MTKAIFV4_TXIF_FOUR_CHANNEL_MASK_SFT,
+				   0x0 << ADDA6_MTKAIFV4_TXIF_FOUR_CHANNEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA6_MTKAIFV4_TX_CFG0,
+				   ADDA6_MTKAIFV4_TXIF_EN_SEL_MASK_SFT,
+				   0x1 << ADDA6_MTKAIFV4_TXIF_EN_SEL_SFT);
+	}
+
+	return 0;
+};
+
+static int set_capture_hw_params(struct snd_pcm_hw_params *params,
+				 struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	unsigned int rate = params_rate(params);
+	struct mtk_afe_adda_priv *adda_priv;
+	unsigned int voice_mode = 0;
+	unsigned int ul_src_con0 = 0;
+	unsigned int mtkaif_rate = 0;
+	int id = dai->id;
+
+	adda_priv = afe_priv->dai_priv[id];
+	if (!adda_priv)
+		return -EINVAL;
+
+	adda_priv->ul_rate = rate;
+
+	/* get mtkaif dl rate */
+	mtkaif_rate = mtkaif_rate_transform(afe, adda_priv->ul_rate);
+
+	voice_mode = adda_ul_rate_transform(afe, rate);
+
+	ul_src_con0 |= (voice_mode << 17) & (0x7 << 17);
+
+	/* enable iir */
+	ul_src_con0 |= (1 << UL_IIR_ON_TMP_CTL_SFT) &
+			UL_IIR_ON_TMP_CTL_MASK_SFT;
+	ul_src_con0 |= (UL_IIR_SW << UL_IIRMODE_CTL_SFT) &
+			UL_IIRMODE_CTL_MASK_SFT;
+
+	regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+			   MTKAIFV4_RXIF_INPUT_MODE_MASK_SFT,
+			   mtkaif_rate << MTKAIFV4_RXIF_INPUT_MODE_SFT);
+
+	regmap_update_bits(afe->regmap, AFE_ADDA6_MTKAIFV4_RX_CFG0,
+			   ADDA6_MTKAIFV4_RXIF_INPUT_MODE_MASK_SFT,
+			   mtkaif_rate << ADDA6_MTKAIFV4_RXIF_INPUT_MODE_SFT);
+
+	switch (id) {
+	case MT8196_DAI_ADDA:
+	case MT8196_DAI_AP_DMIC:
+	case MT8196_DAI_AP_DMIC_MULTICH:
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_RXIF_INPUT_MODE_MASK_SFT,
+				   mtkaif_rate << MTKAIFV4_RXIF_INPUT_MODE_SFT);
+		/* AFE_ADDA_MTKAIFV4_RX_CFG0 */
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_RXIF_FOUR_CHANNEL_MASK_SFT,
+				   0x1 << MTKAIFV4_RXIF_FOUR_CHANNEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_RXIF_EN_SEL_MASK_SFT,
+				   0x0 << MTKAIFV4_RXIF_EN_SEL_SFT);
+		/* [28] loopback mode
+		 * 0: loopback adda tx to adda rx
+		 * 1: loopback adda6 tx to adda rx
+		 */
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_TXIF_EN_SEL_MASK_SFT,
+				   0x0 << MTKAIFV4_TXIF_EN_SEL_SFT);
+
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_UL_CH1CH2_IN_EN_SEL_MASK_SFT,
+				   0x1 << MTKAIFV4_UL_CH1CH2_IN_EN_SEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_UL_CH3CH4_IN_EN_SEL_MASK_SFT,
+				   0x1 << MTKAIFV4_UL_CH3CH4_IN_EN_SEL_SFT);
+
+		/* 35Hz @ 48k */
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_02_01, 0x00000000);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_04_03, 0x00003FB8);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_06_05, 0x3FB80000);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_08_07, 0x3FB80000);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_10_09, 0x0000C048);
+
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_SRC_CON0, ul_src_con0);
+
+		/* mtkaif_rxif_data_mode = 0, amic */
+		regmap_update_bits(afe->regmap,
+				   AFE_MTKAIF1_RX_CFG0,
+				   0x1 << 0,
+				   0x0 << 0);
+
+		/* 35Hz @ 48k */
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL0_IIR_COEF_02_01, 0x00000000);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL0_IIR_COEF_04_03, 0x00003FB8);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL0_IIR_COEF_06_05, 0x3FB80000);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL0_IIR_COEF_08_07, 0x3FB80000);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL0_IIR_COEF_10_09, 0x0000C048);
+
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL0_SRC_CON0, ul_src_con0);
+
+		/* mtkaif_rxif_data_mode = 0, amic */
+		regmap_update_bits(afe->regmap,
+				   AFE_MTKAIF0_RX_CFG0,
+				   0x1 << 0,
+				   0x0 << 0);
+		break;
+	case MT8196_DAI_ADDA_CH34:
+	case MT8196_DAI_AP_DMIC_CH34:
+		/* AFE_ADDA_MTKAIFV4_RX_CFG0 */
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_RXIF_FOUR_CHANNEL_MASK_SFT,
+				   0x1 << MTKAIFV4_RXIF_FOUR_CHANNEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_RXIF_EN_SEL_MASK_SFT,
+				   0x0 << MTKAIFV4_RXIF_EN_SEL_SFT);
+
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_UL_CH1CH2_IN_EN_SEL_MASK_SFT,
+				   0x1 << MTKAIFV4_UL_CH1CH2_IN_EN_SEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_UL_CH3CH4_IN_EN_SEL_MASK_SFT,
+				   0x1 << MTKAIFV4_UL_CH3CH4_IN_EN_SEL_SFT);
+
+		/* 35Hz @ 48k */
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_02_01, 0x00000000);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_04_03, 0x00003FB8);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_06_05, 0x3FB80000);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_08_07, 0x3FB80000);
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_IIR_COEF_10_09, 0x0000C048);
+
+		regmap_write(afe->regmap,
+			     AFE_ADDA_UL1_SRC_CON0, ul_src_con0);
+
+		/* mtkaif_rxif_data_mode = 0, amic */
+		regmap_update_bits(afe->regmap,
+				   AFE_MTKAIF1_RX_CFG0,
+				   0x1 << 0,
+				   0x0 << 0);
+		break;
+	case MT8196_DAI_ADDA_CH56:
+		regmap_update_bits(afe->regmap, AFE_ADDA6_MTKAIFV4_RX_CFG0,
+				   ADDA6_MTKAIFV4_RXIF_INPUT_MODE_MASK_SFT,
+				   mtkaif_rate << ADDA6_MTKAIFV4_RXIF_INPUT_MODE_SFT);
+		/* AFE_ADDA6_MTKAIFV4_RX_CFG0 */
+		regmap_update_bits(afe->regmap, AFE_ADDA6_MTKAIFV4_RX_CFG0,
+				   ADDA6_MTKAIFV4_RXIF_FOUR_CHANNEL_MASK_SFT,
+				   0x1 << ADDA6_MTKAIFV4_RXIF_FOUR_CHANNEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA_MTKAIFV4_RX_CFG0,
+				   MTKAIFV4_UL_CH5CH6_IN_EN_SEL_MASK_SFT,
+				   0x1 << MTKAIFV4_UL_CH5CH6_IN_EN_SEL_SFT);
+		regmap_update_bits(afe->regmap, AFE_ADDA6_MTKAIFV4_RX_CFG0,
+				   ADDA6_MTKAIFV4_RXIF_EN_SEL_MASK_SFT,
+				   0x1 << ADDA6_MTKAIFV4_RXIF_EN_SEL_SFT);
+		break;
+	default:
+		break;
+	}
+
+	/* ap dmic */
+	switch (id) {
+	case MT8196_DAI_AP_DMIC:
+	case MT8196_DAI_AP_DMIC_CH34:
+		mtk_adda_ul_src_enable_dmic(afe, id);
+		break;
+	case MT8196_DAI_AP_DMIC_MULTICH:
+		regmap_update_bits(afe->regmap, AFE_ADDA_ULSRC_PHASE_CON1,
+				   DMIC_CLK_PHASE_SYNC_SET_MASK_SFT,
+				   0x1 << DMIC_CLK_PHASE_SYNC_SET_SFT);
+		mtk_adda_ul_src_set_dmic_phase_sync(afe);
+		mtk_adda_ul_src_enable_dmic(afe, MT8196_DAI_AP_DMIC);
+		mtk_adda_ul_src_enable_dmic(afe, MT8196_DAI_AP_DMIC_CH34);
+		mtk_adda_ul_src_set_dmic_phase_sync_clock(afe);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+};
+
+static int mtk_dai_adda_hw_params(struct snd_pcm_substream *substream,
+				  struct snd_pcm_hw_params *params,
+				  struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	int id = dai->id;
+
+	if (id >= MT8196_DAI_NUM || id < 0)
+		return -EINVAL;
+
+	dev_dbg(afe->dev, "id %d, stream %d, rate %d\n",
+		id, substream->stream, params_rate(params));
+
+	if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK)
+		return set_playback_hw_params(params, dai);
+	else
+		return set_capture_hw_params(params, dai);
+
+	return 0;
+}
+
+static const struct snd_soc_dai_ops mtk_dai_adda_ops = {
+	.hw_params = mtk_dai_adda_hw_params,
+};
+
+/* dai driver */
+#define MTK_ADDA_PLAYBACK_RATES (SNDRV_PCM_RATE_8000_48000 |\
+				 SNDRV_PCM_RATE_96000 |\
+				 SNDRV_PCM_RATE_192000)
+
+#define MTK_ADDA_CAPTURE_RATES (SNDRV_PCM_RATE_8000 |\
+				SNDRV_PCM_RATE_16000 |\
+				SNDRV_PCM_RATE_32000 |\
+				SNDRV_PCM_RATE_48000 |\
+				SNDRV_PCM_RATE_96000 |\
+				SNDRV_PCM_RATE_192000)
+
+#define MTK_ADDA_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\
+			  SNDRV_PCM_FMTBIT_S24_LE |\
+			  SNDRV_PCM_FMTBIT_S32_LE)
+
+static struct snd_soc_dai_driver mtk_dai_adda_driver[] = {
+	{
+		.name = "ADDA",
+		.id = MT8196_DAI_ADDA,
+		.playback = {
+			.stream_name = "ADDA Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = MTK_ADDA_PLAYBACK_RATES,
+			.formats = MTK_ADDA_FORMATS,
+		},
+		.capture = {
+			.stream_name = "ADDA Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = MTK_ADDA_CAPTURE_RATES,
+			.formats = MTK_ADDA_FORMATS,
+		},
+		.ops = &mtk_dai_adda_ops,
+	},
+	{
+		.name = "ADDA_CH34",
+		.id = MT8196_DAI_ADDA_CH34,
+		.playback = {
+			.stream_name = "ADDA CH34 Playback",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = MTK_ADDA_PLAYBACK_RATES,
+			.formats = MTK_ADDA_FORMATS,
+		},
+		.capture = {
+			.stream_name = "ADDA CH34 Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = MTK_ADDA_CAPTURE_RATES,
+			.formats = MTK_ADDA_FORMATS,
+		},
+		.ops = &mtk_dai_adda_ops,
+	},
+	{
+		.name = "ADDA_CH56",
+		.id = MT8196_DAI_ADDA_CH56,
+		.capture = {
+			.stream_name = "ADDA CH56 Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = MTK_ADDA_CAPTURE_RATES,
+			.formats = MTK_ADDA_FORMATS,
+		},
+		.ops = &mtk_dai_adda_ops,
+	},
+	{
+		.name = "AP_DMIC",
+		.id = MT8196_DAI_AP_DMIC,
+		.capture = {
+			.stream_name = "AP DMIC Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = MTK_ADDA_CAPTURE_RATES,
+			.formats = MTK_ADDA_FORMATS,
+		},
+		.ops = &mtk_dai_adda_ops,
+	},
+	{
+		.name = "AP_DMIC_CH34",
+		.id = MT8196_DAI_AP_DMIC_CH34,
+		.capture = {
+			.stream_name = "AP DMIC CH34 Capture",
+			.channels_min = 1,
+			.channels_max = 2,
+			.rates = MTK_ADDA_CAPTURE_RATES,
+			.formats = MTK_ADDA_FORMATS,
+		},
+		.ops = &mtk_dai_adda_ops,
+	},
+	/*
+	 * Multich DMIC combines two DMIC controllers for use together,
+	 * so AP_DMIC and Multich DMIC cannot be used at the same time.
+	 */
+	{
+		.name = "AP_DMIC_MULTICH",
+		.id = MT8196_DAI_AP_DMIC_MULTICH,
+		.capture = {
+			.stream_name = "AP DMIC MULTICH Capture",
+			.channels_min = 1,
+			.channels_max = 4,
+			.rates = MTK_ADDA_CAPTURE_RATES,
+			.formats = MTK_ADDA_FORMATS,
+		},
+		.ops = &mtk_dai_adda_ops,
+	},
+};
+
+static int init_adda_priv_data(struct mtk_base_afe *afe)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	struct mtk_afe_adda_priv *adda_priv;
+	static const int adda_dai_list[] = {
+		MT8196_DAI_ADDA,
+		MT8196_DAI_ADDA_CH34,
+		MT8196_DAI_ADDA_CH56,
+		MT8196_DAI_AP_DMIC_MULTICH
+	};
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(adda_dai_list); i++) {
+		adda_priv = devm_kzalloc(afe->dev,
+					 sizeof(struct mtk_afe_adda_priv),
+					 GFP_KERNEL);
+		if (!adda_priv)
+			return -ENOMEM;
+
+		afe_priv->dai_priv[adda_dai_list[i]] = adda_priv;
+	}
+
+	/* ap dmic priv share with adda */
+	afe_priv->dai_priv[MT8196_DAI_AP_DMIC] =
+		afe_priv->dai_priv[MT8196_DAI_ADDA];
+	afe_priv->dai_priv[MT8196_DAI_AP_DMIC_CH34] =
+		afe_priv->dai_priv[MT8196_DAI_ADDA_CH34];
+
+	return 0;
+}
+
+int mt8196_dai_adda_register(struct mtk_base_afe *afe)
+{
+	struct mtk_base_afe_dai *dai;
+	int ret;
+
+	dai = devm_kzalloc(afe->dev, sizeof(*dai), GFP_KERNEL);
+	if (!dai)
+		return -ENOMEM;
+
+	dai->dai_drivers = mtk_dai_adda_driver;
+	dai->num_dai_drivers = ARRAY_SIZE(mtk_dai_adda_driver);
+	dai->dapm_widgets = mtk_dai_adda_widgets;
+	dai->num_dapm_widgets = ARRAY_SIZE(mtk_dai_adda_widgets);
+	dai->dapm_routes = mtk_dai_adda_routes;
+	dai->num_dapm_routes = ARRAY_SIZE(mtk_dai_adda_routes);
+
+	ret = init_adda_priv_data(afe);
+	if (ret)
+		return ret;
+
+	list_add(&dai->list, &afe->sub_dais);
+
+	return 0;
+}
+
diff --git a/sound/soc/mediatek/mt8196/mt8196-dai-i2s.c b/sound/soc/mediatek/mt8196/mt8196-dai-i2s.c
new file mode 100644
index 0000000000000..ef5cde0ba829e
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-dai-i2s.c
@@ -0,0 +1,2613 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  MediaTek ALSA SoC Audio DAI I2S Control
+ *
+ *  Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#include <linux/bitops.h>
+#include <linux/regmap.h>
+
+#include <sound/pcm_params.h>
+
+#include "mt8196-afe-clk.h"
+#include "mt8196-afe-common.h"
+#include "mt8196-interconnection.h"
+
+#include "../common/mtk-afe-fe-dai.h"
+
+#define ETDM_22M_CLOCK_THRES 11289600
+
+enum {
+	ETDM_CLK_SOURCE_H26M,
+	ETDM_CLK_SOURCE_APLL,
+	ETDM_CLK_SOURCE_SPDIF,
+	ETDM_CLK_SOURCE_HDMI,
+	ETDM_CLK_SOURCE_EARC,
+	ETDM_CLK_SOURCE_LINEIN,
+};
+
+enum {
+	ETDM_RELATCH_SEL_H26M,
+	ETDM_RELATCH_SEL_APLL,
+};
+
+enum {
+	ETDM_RATE_8K,
+	ETDM_RATE_12K,
+	ETDM_RATE_16K,
+	ETDM_RATE_24K,
+	ETDM_RATE_32K,
+	ETDM_RATE_48K,
+	ETDM_RATE_64K,
+	ETDM_RATE_96K,
+	ETDM_RATE_128K,
+	ETDM_RATE_192K,
+	ETDM_RATE_256K,
+	ETDM_RATE_384K,
+	ETDM_RATE_11025 = 16,
+	ETDM_RATE_22050,
+	ETDM_RATE_44100,
+	ETDM_RATE_88200,
+	ETDM_RATE_176400,
+	ETDM_RATE_352800,
+};
+
+enum {
+	ETDM_CONN_8K,
+	ETDM_CONN_11K,
+	ETDM_CONN_12K,
+	ETDM_CONN_16K = 4,
+	ETDM_CONN_22K,
+	ETDM_CONN_24K,
+	ETDM_CONN_32K = 8,
+	ETDM_CONN_44K,
+	ETDM_CONN_48K,
+	ETDM_CONN_88K = 13,
+	ETDM_CONN_96K,
+	ETDM_CONN_176K = 17,
+	ETDM_CONN_192K,
+	ETDM_CONN_352K = 21,
+	ETDM_CONN_384K,
+};
+
+enum {
+	ETDM_WLEN_8_BIT = 0x7,
+	ETDM_WLEN_16_BIT = 0xf,
+	ETDM_WLEN_32_BIT = 0x1f,
+};
+
+enum {
+	ETDM_SLAVE_SEL_ETDMIN0_MASTER,
+	ETDM_SLAVE_SEL_ETDMIN0_SLAVE,
+	ETDM_SLAVE_SEL_ETDMIN1_MASTER,
+	ETDM_SLAVE_SEL_ETDMIN1_SLAVE,
+	ETDM_SLAVE_SEL_ETDMIN2_MASTER,
+	ETDM_SLAVE_SEL_ETDMIN2_SLAVE,
+	ETDM_SLAVE_SEL_ETDMIN3_MASTER,
+	ETDM_SLAVE_SEL_ETDMIN3_SLAVE,
+	ETDM_SLAVE_SEL_ETDMOUT0_MASTER,
+	ETDM_SLAVE_SEL_ETDMOUT0_SLAVE,
+	ETDM_SLAVE_SEL_ETDMOUT1_MASTER,
+	ETDM_SLAVE_SEL_ETDMOUT1_SLAVE,
+	ETDM_SLAVE_SEL_ETDMOUT2_MASTER,
+	ETDM_SLAVE_SEL_ETDMOUT2_SLAVE,
+	ETDM_SLAVE_SEL_ETDMOUT3_MASTER,
+	ETDM_SLAVE_SEL_ETDMOUT3_SLAVE,
+};
+
+enum {
+	ETDM_SLAVE_SEL_ETDMIN4_MASTER,
+	ETDM_SLAVE_SEL_ETDMIN4_SLAVE,
+	ETDM_SLAVE_SEL_ETDMIN5_MASTER,
+	ETDM_SLAVE_SEL_ETDMIN5_SLAVE,
+	ETDM_SLAVE_SEL_ETDMIN6_MASTER,
+	ETDM_SLAVE_SEL_ETDMIN6_SLAVE,
+	ETDM_SLAVE_SEL_ETDMIN7_MASTER,
+	ETDM_SLAVE_SEL_ETDMIN7_SLAVE,
+	ETDM_SLAVE_SEL_ETDMOUT4_MASTER,
+	ETDM_SLAVE_SEL_ETDMOUT4_SLAVE,
+	ETDM_SLAVE_SEL_ETDMOUT5_MASTER,
+	ETDM_SLAVE_SEL_ETDMOUT5_SLAVE,
+	ETDM_SLAVE_SEL_ETDMOUT6_MASTER,
+	ETDM_SLAVE_SEL_ETDMOUT6_SLAVE,
+	ETDM_SLAVE_SEL_ETDMOUT7_MASTER,
+	ETDM_SLAVE_SEL_ETDMOUT7_SLAVE,
+};
+
+enum {
+	MTK_DAI_ETDM_FORMAT_I2S,
+	MTK_DAI_ETDM_FORMAT_LJ,
+	MTK_DAI_ETDM_FORMAT_RJ,
+	MTK_DAI_ETDM_FORMAT_EIAJ,
+	MTK_DAI_ETDM_FORMAT_DSPA,
+	MTK_DAI_ETDM_FORMAT_DSPB,
+};
+
+static unsigned int get_etdm_wlen(snd_pcm_format_t format)
+{
+	return snd_pcm_format_physical_width(format) < 16 ?
+		ETDM_WLEN_16_BIT : ETDM_WLEN_32_BIT;
+}
+
+static unsigned int get_etdm_lrck_width(snd_pcm_format_t format)
+{
+	/* The valid data bit number should be large than 7 due to hardware limitation. */
+	return snd_pcm_format_physical_width(format) - 1;
+}
+
+static unsigned int get_etdm_rate(unsigned int rate)
+{
+	switch (rate) {
+	case 8000:
+		return ETDM_RATE_8K;
+	case 12000:
+		return ETDM_RATE_12K;
+	case 16000:
+		return ETDM_RATE_16K;
+	case 24000:
+		return ETDM_RATE_24K;
+	case 32000:
+		return ETDM_RATE_32K;
+	case 48000:
+		return ETDM_RATE_48K;
+	case 64000:
+		return ETDM_RATE_64K;
+	case 96000:
+		return ETDM_RATE_96K;
+	case 128000:
+		return ETDM_RATE_128K;
+	case 192000:
+		return ETDM_RATE_192K;
+	case 256000:
+		return ETDM_RATE_256K;
+	case 384000:
+		return ETDM_RATE_384K;
+	case 11025:
+		return ETDM_RATE_11025;
+	case 22050:
+		return ETDM_RATE_22050;
+	case 44100:
+		return ETDM_RATE_44100;
+	case 88200:
+		return ETDM_RATE_88200;
+	case 176400:
+		return ETDM_RATE_176400;
+	case 352800:
+		return ETDM_RATE_352800;
+	default:
+		return 0;
+	}
+}
+
+static unsigned int get_etdm_inconn_rate(unsigned int rate)
+{
+	switch (rate) {
+	case 8000:
+		return ETDM_CONN_8K;
+	case 12000:
+		return ETDM_CONN_12K;
+	case 16000:
+		return ETDM_CONN_16K;
+	case 24000:
+		return ETDM_CONN_24K;
+	case 32000:
+		return ETDM_CONN_32K;
+	case 48000:
+		return ETDM_CONN_48K;
+	case 96000:
+		return ETDM_CONN_96K;
+	case 192000:
+		return ETDM_CONN_192K;
+	case 384000:
+		return ETDM_CONN_384K;
+	case 11025:
+		return ETDM_CONN_11K;
+	case 22050:
+		return ETDM_CONN_22K;
+	case 44100:
+		return ETDM_CONN_44K;
+	case 88200:
+		return ETDM_CONN_88K;
+	case 176400:
+		return ETDM_CONN_176K;
+	case 352800:
+		return ETDM_CONN_352K;
+	default:
+		return 0;
+	}
+}
+
+struct mtk_afe_i2s_priv {
+	u8 id;
+	u32 rate; /* for determine which apll to use */
+	int low_jitter_en;
+	const char *share_property_name;
+	int share_i2s_id;
+	u32 mclk_rate;
+	u8 mclk_id;
+	u8 mclk_apll;
+	u8 ch_num;
+	u8 sync;
+	u8 ip_mode;
+	u8 format;
+};
+
+/* this enum is merely for mtk_afe_i2s_priv & mtk_base_etdm_data declare */
+enum {
+	DAI_I2SIN0,
+	DAI_I2SIN1,
+	DAI_I2SIN2,
+	DAI_I2SIN3,
+	DAI_I2SIN4,
+	DAI_I2SIN6,
+	DAI_I2SOUT0,
+	DAI_I2SOUT1,
+	DAI_I2SOUT2,
+	DAI_I2SOUT3,
+	DAI_I2SOUT4,
+	DAI_I2SOUT6,
+	DAI_FMI2S_MASTER,
+	DAI_I2S_NUM,
+};
+
+static bool is_etdm_in_pad_top(unsigned int dai_num)
+{
+	switch (dai_num) {
+	case DAI_I2SOUT4:
+	case DAI_I2SIN4:
+		return true;
+	default:
+		return false;
+	}
+}
+
+struct mtk_base_etdm_data {
+	u16 enable_reg;
+	u16 enable_mask;
+	u8 enable_shift;
+	u16 sync_reg;
+	u16 sync_mask;
+	u8 sync_shift;
+	u16 ch_reg;
+	u16 ch_mask;
+	u8 ch_shift;
+	u16 ip_mode_reg;
+	u16 ip_mode_mask;
+	u8 ip_mode_shift;
+	u16 init_count_reg;
+	u16 init_count_mask;
+	u8 init_count_shift;
+	u16 init_point_reg;
+	u16 init_point_mask;
+	u8 init_point_shift;
+	u16 lrck_reset_reg;
+	u16 lrck_reset_mask;
+	u8 lrck_reset_shift;
+	u16 clk_source_reg;
+	u16 clk_source_mask;
+	u8 clk_source_shift;
+	u16 ck_en_sel_reg;
+	u16 ck_en_sel_mask;
+	u8 ck_en_sel_shift;
+	u16 fs_timing_reg;
+	u16 fs_timing_mask;
+	u8 fs_timing_shift;
+	u16 relatch_en_sel_reg;
+	u16 relatch_en_sel_mask;
+	u8 relatch_en_sel_shift;
+	u16 use_afifo_reg;
+	u16 use_afifo_mask;
+	u8 use_afifo_shift;
+	u16 afifo_mode_reg;
+	u16 afifo_mode_mask;
+	u8 afifo_mode_shift;
+	u16 almost_end_ch_reg;
+	u16 almost_end_ch_mask;
+	u8 almost_end_ch_shift;
+	u16 almost_end_bit_reg;
+	u16 almost_end_bit_mask;
+	u8 almost_end_bit_shift;
+	u16 out2latch_time_reg;
+	u16 out2latch_time_mask;
+	u8 out2latch_time_shift;
+	u16 tdm_mode_reg;
+	u16 tdm_mode_mask;
+	u8 tdm_mode_shift;
+	u16 relatch_domain_sel_reg;
+	u16 relatch_domain_sel_mask;
+	u8 relatch_domain_sel_shift;
+	u16 bit_length_reg;
+	u16 bit_length_mask;
+	u8 bit_length_shift;
+	u16 word_length_reg;
+	u16 word_length_mask;
+	u8 word_length_shift;
+	u16 cowork_reg;
+	u16 cowork_mask;
+	u8 cowork_shift;
+	u32 cowork_val;
+	u16 in2latch_time_reg;
+	u16 in2latch_time_mask;
+	u8 in2latch_time_shift;
+	u16 pad_top_ck_en_reg;
+	u16 pad_top_ck_en_mask;
+	u8 pad_top_ck_en_shift;
+	u16 master_latch_reg;
+	u16 master_latch_mask;
+	u8 master_latch_shift;
+};
+
+/*
+ * _cfg_vlp_reg should be a variable or constant, not an expression
+ * with side effects.
+ */
+#define MTK_ETDM_IN_DATA(_id, _cowork, _cfg_vlp_reg) \
+	[DAI_I2SIN##_id] = { \
+		.enable_reg = ETDM_IN##_id##_CON0, \
+		.enable_mask = REG_ETDM_IN_EN_MASK, \
+		.enable_shift = REG_ETDM_IN_EN_SFT, \
+		.sync_reg = ETDM_IN##_id##_CON0, \
+		.sync_mask = REG_SYNC_MODE_MASK, \
+		.sync_shift = REG_SYNC_MODE_SFT, \
+		.ch_reg = ETDM_IN##_id##_CON0, \
+		.ch_mask = REG_CH_NUM_MASK, \
+		.ch_shift = REG_CH_NUM_SFT, \
+		.ip_mode_reg = ETDM_IN##_id##_CON2, \
+		.ip_mode_mask = REG_MULTI_IP_MODE_MASK, \
+		.ip_mode_shift = REG_MULTI_IP_MODE_SFT, \
+		.init_count_reg = ETDM_IN##_id##_CON1, \
+		.init_count_mask = REG_INITIAL_COUNT_MASK, \
+		.init_count_shift = REG_INITIAL_COUNT_SFT, \
+		.init_point_reg = ETDM_IN##_id##_CON1, \
+		.init_point_mask = REG_INITIAL_POINT_MASK, \
+		.init_point_shift = REG_INITIAL_POINT_SFT, \
+		.lrck_reset_reg = ETDM_IN##_id##_CON1, \
+		.lrck_reset_mask = REG_LRCK_RESET_MASK, \
+		.lrck_reset_shift = REG_LRCK_RESET_SFT, \
+		.clk_source_reg = ETDM_IN##_id##_CON2, \
+		.clk_source_mask = REG_CLOCK_SOURCE_SEL_MASK, \
+		.clk_source_shift = REG_CLOCK_SOURCE_SEL_SFT, \
+		.ck_en_sel_reg = ETDM_IN##_id##_CON2, \
+		.ck_en_sel_mask = REG_CK_EN_SEL_AUTO_MASK, \
+		.ck_en_sel_shift = REG_CK_EN_SEL_AUTO_SFT, \
+		.fs_timing_reg = ETDM_IN##_id##_CON3, \
+		.fs_timing_mask = REG_FS_TIMING_SEL_MASK, \
+		.fs_timing_shift = REG_FS_TIMING_SEL_SFT, \
+		.relatch_en_sel_reg = ETDM_IN##_id##_CON4, \
+		.relatch_en_sel_mask = REG_RELATCH_1X_EN_SEL_MASK, \
+		.relatch_en_sel_shift = REG_RELATCH_1X_EN_SEL_SFT, \
+		.use_afifo_reg = ETDM_IN##_id##_CON8, \
+		.use_afifo_mask = REG_ETDM_USE_AFIFO_MASK, \
+		.use_afifo_shift = REG_ETDM_USE_AFIFO_SFT, \
+		.afifo_mode_reg = ETDM_IN##_id##_CON8, \
+		.afifo_mode_mask = REG_AFIFO_MODE_MASK, \
+		.afifo_mode_shift = REG_AFIFO_MODE_SFT, \
+		.almost_end_ch_reg = ETDM_IN##_id##_CON9, \
+		.almost_end_ch_mask = REG_ALMOST_END_CH_COUNT_MASK, \
+		.almost_end_ch_shift = REG_ALMOST_END_CH_COUNT_SFT, \
+		.almost_end_bit_reg = ETDM_IN##_id##_CON9, \
+		.almost_end_bit_mask = REG_ALMOST_END_BIT_COUNT_MASK, \
+		.almost_end_bit_shift = REG_ALMOST_END_BIT_COUNT_SFT, \
+		.out2latch_time_reg = ETDM_IN##_id##_CON9, \
+		.out2latch_time_mask = REG_OUT2LATCH_TIME_MASK, \
+		.out2latch_time_shift = REG_OUT2LATCH_TIME_SFT, \
+		.tdm_mode_reg = ETDM_IN##_id##_CON0, \
+		.tdm_mode_mask = REG_FMT_MASK, \
+		.tdm_mode_shift = REG_FMT_SFT, \
+		.relatch_domain_sel_reg = ETDM_IN##_id##_CON0, \
+		.relatch_domain_sel_mask = REG_RELATCH_1X_EN_DOMAIN_SEL_MASK, \
+		.relatch_domain_sel_shift = REG_RELATCH_1X_EN_DOMAIN_SEL_SFT, \
+		.bit_length_reg = ETDM_IN##_id##_CON0, \
+		.bit_length_mask = REG_BIT_LENGTH_MASK, \
+		.bit_length_shift = REG_BIT_LENGTH_SFT, \
+		.word_length_reg = ETDM_IN##_id##_CON0, \
+		.word_length_mask = REG_WORD_LENGTH_MASK, \
+		.word_length_shift = REG_WORD_LENGTH_SFT, \
+		.cowork_reg = _cowork, \
+		.cowork_mask = ETDM_IN##_id##_SLAVE_SEL_MASK, \
+		.cowork_shift = ETDM_IN##_id##_SLAVE_SEL_SFT, \
+		.cowork_val = ETDM_SLAVE_SEL_ETDMOUT##_id##_MASTER, \
+		.pad_top_ck_en_reg = _cfg_vlp_reg, \
+		.pad_top_ck_en_mask = RG_I2S4_PAD_TOP_CK_EN_MASK, \
+		.pad_top_ck_en_shift = RG_I2S4_PAD_TOP_CK_EN_SFT, \
+		.master_latch_reg = _cfg_vlp_reg, \
+		.master_latch_mask = RG_I2S4_IN_BCK_NEG_EG_LATCH_MASK, \
+		.master_latch_shift = RG_I2S4_IN_BCK_NEG_EG_LATCH_SFT, \
+	}
+
+/*
+ * _cfg_vlp_reg should be a variable or constant, not an expression
+ * with side effects.
+ */
+#define MTK_ETDM_OUT_DATA(_id, _cowork, _cfg_vlp_reg) \
+		[DAI_I2SOUT##_id] = { \
+			.enable_reg = ETDM_OUT##_id##_CON0, \
+			.enable_mask = OUT_REG_ETDM_OUT_EN_MASK, \
+			.enable_shift = OUT_REG_ETDM_OUT_EN_SFT, \
+			.sync_reg = ETDM_OUT##_id##_CON0, \
+			.sync_mask = REG_SYNC_MODE_MASK, \
+			.sync_shift = REG_SYNC_MODE_SFT, \
+			.ch_reg = ETDM_OUT##_id##_CON0, \
+			.ch_mask = REG_CH_NUM_MASK, \
+			.ch_shift = REG_CH_NUM_SFT, \
+			.init_count_reg = ETDM_OUT##_id##_CON1, \
+			.init_count_mask = OUT_REG_INITIAL_COUNT_MASK, \
+			.init_count_shift = OUT_REG_INITIAL_COUNT_SFT, \
+			.init_point_reg = ETDM_OUT##_id##_CON1, \
+			.init_point_mask = OUT_REG_INITIAL_POINT_MASK, \
+			.init_point_shift = OUT_REG_INITIAL_POINT_SFT, \
+			.lrck_reset_reg = ETDM_OUT##_id##_CON1, \
+			.lrck_reset_mask = OUT_REG_LRCK_RESET_MASK, \
+			.lrck_reset_shift = OUT_REG_LRCK_RESET_SFT, \
+			.clk_source_reg = ETDM_OUT##_id##_CON4, \
+			.clk_source_mask = OUT_REG_CLOCK_SOURCE_SEL_MASK, \
+			.clk_source_shift = OUT_REG_CLOCK_SOURCE_SEL_SFT, \
+			.fs_timing_reg = ETDM_OUT##_id##_CON4, \
+			.fs_timing_mask = OUT_REG_FS_TIMING_SEL_MASK, \
+			.fs_timing_shift = OUT_REG_FS_TIMING_SEL_SFT, \
+			.relatch_en_sel_reg = ETDM_OUT##_id##_CON4, \
+			.relatch_en_sel_mask = OUT_REG_RELATCH_EN_SEL_MASK, \
+			.relatch_en_sel_shift = OUT_REG_RELATCH_EN_SEL_SFT, \
+			.tdm_mode_reg = ETDM_OUT##_id##_CON0, \
+			.tdm_mode_mask = OUT_REG_FMT_MASK, \
+			.tdm_mode_shift = OUT_REG_FMT_SFT, \
+			.relatch_domain_sel_reg = ETDM_OUT##_id##_CON0, \
+			.relatch_domain_sel_mask = OUT_REG_RELATCH_DOMAIN_SEL_MASK, \
+			.relatch_domain_sel_shift = OUT_REG_RELATCH_DOMAIN_SEL_SFT, \
+			.bit_length_reg = ETDM_OUT##_id##_CON0, \
+			.bit_length_mask = OUT_REG_BIT_LENGTH_MASK, \
+			.bit_length_shift = OUT_REG_BIT_LENGTH_SFT, \
+			.word_length_reg = ETDM_OUT##_id##_CON0, \
+			.word_length_mask = OUT_REG_WORD_LENGTH_MASK, \
+			.word_length_shift = OUT_REG_WORD_LENGTH_SFT, \
+			.cowork_reg = _cowork, \
+			.cowork_mask = ETDM_OUT##_id##_SLAVE_SEL_MASK, \
+			.cowork_shift = ETDM_OUT##_id##_SLAVE_SEL_SFT, \
+			.cowork_val = ETDM_SLAVE_SEL_ETDMIN##_id##_MASTER, \
+			.in2latch_time_reg = ETDM_OUT##_id##_CON2, \
+			.in2latch_time_mask = OUT_REG_IN2LATCH_TIME_MASK, \
+			.in2latch_time_shift = OUT_REG_IN2LATCH_TIME_SFT, \
+			.pad_top_ck_en_reg = _cfg_vlp_reg, \
+			.pad_top_ck_en_mask = RG_I2S4_PAD_TOP_CK_EN_MASK, \
+			.pad_top_ck_en_shift = RG_I2S4_PAD_TOP_CK_EN_SFT, \
+			.master_latch_reg = _cfg_vlp_reg, \
+			.master_latch_mask = RG_I2S4_OUT_BCK_NEG_EG_LATCH_MASK, \
+			.master_latch_shift = RG_I2S4_OUT_BCK_NEG_EG_LATCH_SFT, \
+		}
+
+static const struct mtk_base_etdm_data mtk_etdm_data[DAI_I2S_NUM] = {
+	MTK_ETDM_IN_DATA(0, ETDM_0_3_COWORK_CON0, 0),
+	MTK_ETDM_IN_DATA(1, ETDM_0_3_COWORK_CON1, 0),
+	MTK_ETDM_IN_DATA(2, ETDM_0_3_COWORK_CON2, 0),
+	MTK_ETDM_IN_DATA(3, ETDM_0_3_COWORK_CON3, 0),
+	MTK_ETDM_IN_DATA(4, ETDM_4_7_COWORK_CON0, AUD_TOP_CFG_VLP_RG),
+	MTK_ETDM_IN_DATA(6, ETDM_4_7_COWORK_CON2, 0),
+
+	MTK_ETDM_OUT_DATA(0, ETDM_0_3_COWORK_CON0, 0),
+	MTK_ETDM_OUT_DATA(1, ETDM_0_3_COWORK_CON0, 0),
+	MTK_ETDM_OUT_DATA(2, ETDM_0_3_COWORK_CON2, 0),
+	MTK_ETDM_OUT_DATA(3, ETDM_0_3_COWORK_CON2, 0),
+	MTK_ETDM_OUT_DATA(4, ETDM_4_7_COWORK_CON0, AUD_TOP_CFG_VLP_RG),
+	MTK_ETDM_OUT_DATA(6, ETDM_4_7_COWORK_CON2, 0),
+};
+
+enum {
+	I2S_FMT_EIAJ,
+	I2S_FMT_I2S,
+};
+
+enum {
+	I2S_WLEN_16_BIT,
+	I2S_WLEN_32_BIT,
+};
+
+enum {
+	I2S_IN_PAD_CONNSYS,
+	I2S_IN_PAD_IO_MUX,
+};
+
+static unsigned int get_i2s_wlen(snd_pcm_format_t format)
+{
+	return snd_pcm_format_physical_width(format) <= 16 ?
+	       I2S_WLEN_16_BIT : I2S_WLEN_32_BIT;
+}
+
+#define I2SIN0_MCLK_EN_W_NAME "I2SIN0_MCLK_EN"
+#define I2SIN1_MCLK_EN_W_NAME "I2SIN1_MCLK_EN"
+#define I2SIN2_MCLK_EN_W_NAME "I2SIN2_MCLK_EN"
+#define I2SIN3_MCLK_EN_W_NAME "I2SIN3_MCLK_EN"
+#define I2SIN4_MCLK_EN_W_NAME "I2SIN4_MCLK_EN"
+#define I2SIN6_MCLK_EN_W_NAME "I2SIN6_MCLK_EN"
+#define I2SOUT0_MCLK_EN_W_NAME "I2SOUT0_MCLK_EN"
+#define I2SOUT1_MCLK_EN_W_NAME "I2SOUT1_MCLK_EN"
+#define I2SOUT2_MCLK_EN_W_NAME "I2SOUT2_MCLK_EN"
+#define I2SOUT3_MCLK_EN_W_NAME "I2SOUT3_MCLK_EN"
+#define I2SOUT4_MCLK_EN_W_NAME "I2SOUT4_MCLK_EN"
+#define I2SOUT6_MCLK_EN_W_NAME "I2SOUT6_MCLK_EN"
+#define FMI2S_MASTER_MCLK_EN_W_NAME "FMI2S_MASTER_MCLK_EN"
+
+static int get_i2s_id_by_name(struct mtk_base_afe *afe,
+			      const char *name)
+{
+	if (strncmp(name, "I2SIN0", 6) == 0)
+		return MT8196_DAI_I2S_IN0;
+	else if (strncmp(name, "I2SIN1", 6) == 0)
+		return MT8196_DAI_I2S_IN1;
+	else if (strncmp(name, "I2SIN2", 6) == 0)
+		return MT8196_DAI_I2S_IN2;
+	else if (strncmp(name, "I2SIN3", 6) == 0)
+		return MT8196_DAI_I2S_IN3;
+	else if (strncmp(name, "I2SIN4", 6) == 0)
+		return MT8196_DAI_I2S_IN4;
+	else if (strncmp(name, "I2SIN6", 6) == 0)
+		return MT8196_DAI_I2S_IN6;
+	else if (strncmp(name, "I2SOUT0", 7) == 0)
+		return MT8196_DAI_I2S_OUT0;
+	else if (strncmp(name, "I2SOUT1", 7) == 0)
+		return MT8196_DAI_I2S_OUT1;
+	else if (strncmp(name, "I2SOUT2", 7) == 0)
+		return MT8196_DAI_I2S_OUT2;
+	else if (strncmp(name, "I2SOUT3", 7) == 0)
+		return MT8196_DAI_I2S_OUT3;
+	else if (strncmp(name, "I2SOUT4", 7) == 0)
+		return MT8196_DAI_I2S_OUT4;
+	else if (strncmp(name, "I2SOUT6", 7) == 0)
+		return MT8196_DAI_I2S_OUT6;
+	else if (strncmp(name, "FMI2S_MASTER", 12) == 0)
+		return MT8196_DAI_FM_I2S_MASTER;
+	else
+		return -EINVAL;
+}
+
+static struct mtk_afe_i2s_priv *get_i2s_priv_by_name(struct mtk_base_afe *afe,
+						     const char *name)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int dai_id = get_i2s_id_by_name(afe, name);
+
+	if (dai_id < 0)
+		return NULL;
+
+	return afe_priv->dai_priv[dai_id];
+}
+
+static const char * const etdm_0_3_loopback_text[] = {
+	"etdmin0", "etdmin1",
+	"etdmin2", "etdmin3",
+	"etdmout0", "etdmout1",
+	"etdmout2", "etdmout3"
+};
+
+static const char * const etdm_4_7_loopback_text[] = {
+	"etdmin4", "etdmin5",
+	"etdmin6", "etdmin7",
+	"etdmout4", "etdmout5",
+	"etdmout6", "etdmout7"
+};
+
+static const u32 etdm_loopback_values[] = {
+	0, 2, 4, 6, 8, 10, 12, 14
+};
+
+static const struct soc_enum i2sin0_loopback_enum =
+	SOC_VALUE_ENUM_SINGLE(ETDM_0_3_COWORK_CON1, ETDM_IN0_SDATA0_SEL_SFT,
+			      ETDM_IN0_SDATA0_SEL_MASK, ARRAY_SIZE(etdm_0_3_loopback_text),
+			      etdm_0_3_loopback_text, etdm_loopback_values);
+
+static const struct soc_enum i2sin1_loopback_enum =
+	SOC_VALUE_ENUM_SINGLE(ETDM_0_3_COWORK_CON1, ETDM_IN1_SDATA0_SEL_SFT,
+			      ETDM_IN1_SDATA0_SEL_MASK, ARRAY_SIZE(etdm_0_3_loopback_text),
+			      etdm_0_3_loopback_text, etdm_loopback_values);
+
+static const struct soc_enum i2sin2_loopback_enum =
+	SOC_VALUE_ENUM_SINGLE(ETDM_0_3_COWORK_CON3, ETDM_IN2_SDATA0_SEL_SFT,
+			      ETDM_IN2_SDATA0_SEL_MASK, ARRAY_SIZE(etdm_0_3_loopback_text),
+			      etdm_0_3_loopback_text, etdm_loopback_values);
+
+static const struct soc_enum i2sin3_loopback_enum =
+	SOC_VALUE_ENUM_SINGLE(ETDM_0_3_COWORK_CON3, ETDM_IN3_SDATA0_SEL_SFT,
+			      ETDM_IN3_SDATA0_SEL_MASK, ARRAY_SIZE(etdm_0_3_loopback_text),
+			      etdm_0_3_loopback_text, etdm_loopback_values);
+
+static const struct soc_enum i2sin4_loopback_enum =
+	SOC_VALUE_ENUM_SINGLE(ETDM_4_7_COWORK_CON1, ETDM_IN4_SDATA0_SEL_SFT,
+			      ETDM_IN4_SDATA0_SEL_MASK, ARRAY_SIZE(etdm_4_7_loopback_text),
+			      etdm_4_7_loopback_text, etdm_loopback_values);
+
+static const struct soc_enum i2sin6_loopback_enum =
+	SOC_VALUE_ENUM_SINGLE(ETDM_4_7_COWORK_CON3, ETDM_IN6_SDATA0_SEL_SFT,
+			      ETDM_IN6_SDATA0_SEL_MASK, ARRAY_SIZE(etdm_4_7_loopback_text),
+			      etdm_4_7_loopback_text, etdm_loopback_values);
+
+static const struct snd_kcontrol_new mtk_dai_i2s_controls[] = {
+	SOC_ENUM("I2SIN0 LOOPBACK", i2sin0_loopback_enum),
+	SOC_ENUM("I2SIN1 LOOPBACK", i2sin1_loopback_enum),
+	SOC_ENUM("I2SIN2 LOOPBACK", i2sin2_loopback_enum),
+	SOC_ENUM("I2SIN3 LOOPBACK", i2sin3_loopback_enum),
+	/* The following I2S does not support multi-ip mode */
+	SOC_ENUM("I2SIN4 LOOPBACK", i2sin4_loopback_enum),
+	SOC_ENUM("I2SIN6 LOOPBACK", i2sin6_loopback_enum),
+};
+
+/* interconnection */
+static const struct snd_kcontrol_new mtk_i2sout0_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH1", AFE_CONN108_1, I_DL0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH1", AFE_CONN108_1, I_DL1_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH1", AFE_CONN108_1, I_DL2_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH1", AFE_CONN108_1, I_DL3_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH1", AFE_CONN108_1, I_DL4_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH1", AFE_CONN108_1, I_DL5_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH1", AFE_CONN108_1, I_DL6_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH1", AFE_CONN108_1, I_DL7_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH1", AFE_CONN108_1, I_DL8_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH1", AFE_CONN108_1, I_DL_24CH_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL23_CH1", AFE_CONN108_2, I_DL23_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL24_CH1", AFE_CONN108_2, I_DL24_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH1", AFE_CONN108_0,
+				    I_GAIN0_OUT_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN108_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN108_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN108_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN108_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN108_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_SRC_2_OUT_CH1", AFE_CONN108_6,
+				    I_SRC_2_OUT_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout0_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH2", AFE_CONN109_1, I_DL0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH2", AFE_CONN109_1, I_DL1_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH2", AFE_CONN109_1, I_DL2_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH2", AFE_CONN109_1, I_DL3_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH2", AFE_CONN109_1, I_DL4_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH2", AFE_CONN109_1, I_DL5_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH2", AFE_CONN109_1, I_DL6_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH2", AFE_CONN109_1, I_DL7_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH2", AFE_CONN109_1, I_DL8_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH2", AFE_CONN109_1, I_DL_24CH_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL23_CH2", AFE_CONN109_2, I_DL23_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL24_CH2", AFE_CONN109_2, I_DL24_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH2", AFE_CONN109_0,
+				    I_GAIN0_OUT_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN109_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN109_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN109_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN109_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH2", AFE_CONN109_4,
+				    I_PCM_0_CAP_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN109_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH2", AFE_CONN109_4,
+				    I_PCM_1_CAP_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_SRC_2_OUT_CH2", AFE_CONN109_6,
+				    I_SRC_2_OUT_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout1_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH1", AFE_CONN110_1, I_DL0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH1", AFE_CONN110_1, I_DL1_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH1", AFE_CONN110_1, I_DL2_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH1", AFE_CONN110_1, I_DL3_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH1", AFE_CONN110_1, I_DL4_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH1", AFE_CONN110_1, I_DL5_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH1", AFE_CONN110_1, I_DL6_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH1", AFE_CONN110_1, I_DL7_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH1", AFE_CONN110_1, I_DL8_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH1", AFE_CONN110_1, I_DL_24CH_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH1", AFE_CONN110_0,
+				    I_GAIN0_OUT_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN110_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN110_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN110_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout1_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH2", AFE_CONN111_1, I_DL0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH2", AFE_CONN111_1, I_DL1_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH2", AFE_CONN111_1, I_DL2_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH2", AFE_CONN111_1, I_DL3_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH2", AFE_CONN111_1, I_DL4_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH2", AFE_CONN111_1, I_DL5_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH2", AFE_CONN111_1, I_DL6_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH2", AFE_CONN111_1, I_DL7_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH2", AFE_CONN111_1, I_DL8_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH2", AFE_CONN111_1, I_DL_24CH_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH2", AFE_CONN111_0,
+				    I_GAIN0_OUT_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN111_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN111_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH2", AFE_CONN111_4,
+				    I_PCM_0_CAP_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN111_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH2", AFE_CONN111_4,
+				    I_PCM_1_CAP_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout2_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH1", AFE_CONN112_1, I_DL0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH1", AFE_CONN112_1, I_DL1_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH1", AFE_CONN112_1, I_DL2_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH1", AFE_CONN112_1, I_DL3_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH1", AFE_CONN112_1, I_DL4_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH1", AFE_CONN112_1, I_DL5_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH1", AFE_CONN112_1, I_DL6_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH1", AFE_CONN112_1, I_DL7_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH1", AFE_CONN112_1, I_DL8_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH1", AFE_CONN112_1, I_DL_24CH_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH1", AFE_CONN112_0,
+				    I_GAIN0_OUT_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN112_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN112_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN112_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout2_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH2", AFE_CONN113_1, I_DL0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH2", AFE_CONN113_1, I_DL1_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH2", AFE_CONN113_1, I_DL2_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH2", AFE_CONN113_1, I_DL3_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH2", AFE_CONN113_1, I_DL4_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH2", AFE_CONN113_1, I_DL5_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH2", AFE_CONN113_1, I_DL6_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH2", AFE_CONN113_1, I_DL7_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH2", AFE_CONN113_1, I_DL8_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH2", AFE_CONN113_1, I_DL_24CH_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH2", AFE_CONN113_0,
+				    I_GAIN0_OUT_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN113_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN113_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH2", AFE_CONN113_4,
+				    I_PCM_0_CAP_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN113_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH2", AFE_CONN113_4,
+				    I_PCM_1_CAP_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout3_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH1", AFE_CONN114_1, I_DL0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH1", AFE_CONN114_1, I_DL1_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH1", AFE_CONN114_1, I_DL2_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH1", AFE_CONN114_1, I_DL3_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH1", AFE_CONN114_1, I_DL4_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH1", AFE_CONN114_1, I_DL5_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH1", AFE_CONN114_1, I_DL6_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH1", AFE_CONN114_1, I_DL7_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH1", AFE_CONN114_1, I_DL8_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH1", AFE_CONN114_1, I_DL_24CH_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH1", AFE_CONN114_0,
+				    I_GAIN0_OUT_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN114_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN114_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN114_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout3_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH2", AFE_CONN115_1, I_DL0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH2", AFE_CONN115_1, I_DL1_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH2", AFE_CONN115_1, I_DL2_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH2", AFE_CONN115_1, I_DL3_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH2", AFE_CONN115_1, I_DL4_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH2", AFE_CONN115_1, I_DL5_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH2", AFE_CONN115_1, I_DL6_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH2", AFE_CONN115_1, I_DL7_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH2", AFE_CONN115_1, I_DL8_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH2", AFE_CONN115_1, I_DL_24CH_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH2", AFE_CONN115_0,
+				    I_GAIN0_OUT_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN115_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH2", AFE_CONN115_4,
+				    I_PCM_0_CAP_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH2", AFE_CONN115_4,
+				    I_PCM_1_CAP_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout4_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH1", AFE_CONN116_1, I_DL0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH1", AFE_CONN116_1, I_DL1_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH1", AFE_CONN116_1, I_DL2_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH1", AFE_CONN116_1, I_DL3_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH1", AFE_CONN116_1, I_DL4_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH1", AFE_CONN116_1, I_DL5_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH1", AFE_CONN116_1, I_DL6_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH1", AFE_CONN116_1, I_DL7_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH1", AFE_CONN116_1, I_DL8_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH1", AFE_CONN116_1, I_DL_24CH_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL24_CH1", AFE_CONN116_2, I_DL24_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH1", AFE_CONN116_0,
+				    I_GAIN0_OUT_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN116_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN116_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN116_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN116_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN116_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_SRC_2_OUT_CH1", AFE_CONN116_6,
+				    I_SRC_2_OUT_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout4_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH2", AFE_CONN117_1, I_DL0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH2", AFE_CONN117_1, I_DL1_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH2", AFE_CONN117_1, I_DL2_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH2", AFE_CONN117_1, I_DL3_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH2", AFE_CONN117_1, I_DL4_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH2", AFE_CONN117_1, I_DL5_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH2", AFE_CONN117_1, I_DL6_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH2", AFE_CONN117_1, I_DL7_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH2", AFE_CONN117_1, I_DL8_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH2", AFE_CONN117_1, I_DL_24CH_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL24_CH2", AFE_CONN117_2, I_DL24_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH2", AFE_CONN117_0,
+				    I_GAIN0_OUT_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN117_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN117_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH3", AFE_CONN117_0,
+				    I_ADDA_UL_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN117_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH2", AFE_CONN117_4,
+				    I_PCM_0_CAP_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN117_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH2", AFE_CONN117_4,
+				    I_PCM_1_CAP_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_SRC_2_OUT_CH2", AFE_CONN117_6,
+				    I_SRC_2_OUT_CH2, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout4_ch3_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH3", AFE_CONN118_1, I_DL_24CH_CH3, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN118_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN118_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout4_ch4_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH4", AFE_CONN119_1, I_DL_24CH_CH4, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN118_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN118_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout4_ch5_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH5", AFE_CONN120_1, I_DL_24CH_CH5, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout4_ch6_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH6", AFE_CONN121_1, I_DL_24CH_CH6, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout4_ch7_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH7", AFE_CONN122_1, I_DL_24CH_CH7, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout4_ch8_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH8", AFE_CONN123_1, I_DL_24CH_CH8, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout6_ch1_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH1", AFE_CONN148_1, I_DL0_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH1", AFE_CONN148_1, I_DL1_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH1", AFE_CONN148_1, I_DL2_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH1", AFE_CONN148_1, I_DL3_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH1", AFE_CONN148_1, I_DL4_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH1", AFE_CONN148_1, I_DL5_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH1", AFE_CONN148_1, I_DL6_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH1", AFE_CONN148_1, I_DL7_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH1", AFE_CONN148_1, I_DL8_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL23_CH1", AFE_CONN148_2, I_DL23_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH1", AFE_CONN148_1, I_DL_24CH_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH1", AFE_CONN148_0,
+				    I_GAIN0_OUT_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH1", AFE_CONN148_0,
+				    I_ADDA_UL_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN148_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN148_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_SRC_1_OUT_CH1", AFE_CONN148_6,
+				    I_SRC_1_OUT_CH1, 1, 0),
+};
+
+static const struct snd_kcontrol_new mtk_i2sout6_ch2_mix[] = {
+	SOC_DAPM_SINGLE_AUTODISABLE("DL0_CH2", AFE_CONN149_1, I_DL0_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL1_CH2", AFE_CONN149_1, I_DL1_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL2_CH2", AFE_CONN149_1, I_DL2_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL3_CH2", AFE_CONN149_1, I_DL3_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL4_CH2", AFE_CONN149_1, I_DL4_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL5_CH2", AFE_CONN149_1, I_DL5_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL6_CH2", AFE_CONN149_1, I_DL6_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL7_CH2", AFE_CONN149_1, I_DL7_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL8_CH2", AFE_CONN149_1, I_DL8_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL23_CH2", AFE_CONN149_2, I_DL23_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("DL_24CH_CH2", AFE_CONN149_1, I_DL_24CH_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_GAIN0_OUT_CH2", AFE_CONN149_0,
+				    I_GAIN0_OUT_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("ADDA_UL_CH2", AFE_CONN149_0,
+				    I_ADDA_UL_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH1", AFE_CONN149_4,
+				    I_PCM_0_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_0_CAP_CH2", AFE_CONN149_4,
+				    I_PCM_0_CAP_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH1", AFE_CONN149_4,
+				    I_PCM_1_CAP_CH1, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("PCM_1_CAP_CH2", AFE_CONN149_4,
+				    I_PCM_1_CAP_CH2, 1, 0),
+	SOC_DAPM_SINGLE_AUTODISABLE("HW_SRC_1_OUT_CH2", AFE_CONN148_6,
+				    I_SRC_1_OUT_CH2, 1, 0),
+};
+
+enum {
+	SUPPLY_SEQ_APLL,
+	SUPPLY_SEQ_I2S_MCLK_EN,
+	SUPPLY_SEQ_I2S_CG_EN,
+	SUPPLY_SEQ_I2S_EN,
+};
+
+static int mtk_apll_event(struct snd_soc_dapm_widget *w,
+			  struct snd_kcontrol *kcontrol,
+			  int event)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(w->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+
+	dev_dbg(cmpnt->dev, "name %s, event 0x%x\n", w->name, event);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		if (strcmp(w->name, APLL1_W_NAME) == 0)
+			mt8196_apll1_enable(afe);
+		else
+			mt8196_apll2_enable(afe);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		if (strcmp(w->name, APLL1_W_NAME) == 0)
+			mt8196_apll1_disable(afe);
+		else
+			mt8196_apll2_disable(afe);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int mtk_mclk_en_event(struct snd_soc_dapm_widget *w,
+			     struct snd_kcontrol *kcontrol,
+			     int event)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(w->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mtk_afe_i2s_priv *i2s_priv;
+
+	dev_dbg(cmpnt->dev, "name %s, event 0x%x\n", w->name, event);
+
+	i2s_priv = get_i2s_priv_by_name(afe, w->name);
+
+	if (!i2s_priv)
+		return -EINVAL;
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		mt8196_mck_enable(afe, i2s_priv->mclk_id, i2s_priv->mclk_rate);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		i2s_priv->mclk_rate = 0;
+		mt8196_mck_disable(afe, i2s_priv->mclk_id);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dapm_widget mtk_dai_i2s_widgets[] = {
+	SND_SOC_DAPM_MIXER("I2SOUT0_CH1", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout0_ch1_mix,
+			   ARRAY_SIZE(mtk_i2sout0_ch1_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT0_CH2", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout0_ch2_mix,
+			   ARRAY_SIZE(mtk_i2sout0_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("I2SOUT1_CH1", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout1_ch1_mix,
+			   ARRAY_SIZE(mtk_i2sout1_ch1_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT1_CH2", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout1_ch2_mix,
+			   ARRAY_SIZE(mtk_i2sout1_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("I2SOUT2_CH1", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout2_ch1_mix,
+			   ARRAY_SIZE(mtk_i2sout2_ch1_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT2_CH2", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout2_ch2_mix,
+			   ARRAY_SIZE(mtk_i2sout2_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("I2SOUT3_CH1", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout3_ch1_mix,
+			   ARRAY_SIZE(mtk_i2sout3_ch1_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT3_CH2", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout3_ch2_mix,
+			   ARRAY_SIZE(mtk_i2sout3_ch2_mix)),
+
+	SND_SOC_DAPM_MIXER("I2SOUT4_CH1", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout4_ch1_mix,
+			   ARRAY_SIZE(mtk_i2sout4_ch1_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT4_CH2", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout4_ch2_mix,
+			   ARRAY_SIZE(mtk_i2sout4_ch2_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT4_CH3", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout4_ch3_mix,
+			   ARRAY_SIZE(mtk_i2sout4_ch3_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT4_CH4", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout4_ch4_mix,
+			   ARRAY_SIZE(mtk_i2sout4_ch4_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT4_CH5", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout4_ch5_mix,
+			   ARRAY_SIZE(mtk_i2sout4_ch5_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT4_CH6", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout4_ch6_mix,
+			   ARRAY_SIZE(mtk_i2sout4_ch6_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT4_CH7", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout4_ch7_mix,
+			   ARRAY_SIZE(mtk_i2sout4_ch7_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT4_CH8", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout4_ch8_mix,
+			   ARRAY_SIZE(mtk_i2sout4_ch8_mix)),
+
+	SND_SOC_DAPM_MIXER("I2SOUT6_CH1", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout6_ch1_mix,
+			   ARRAY_SIZE(mtk_i2sout6_ch1_mix)),
+	SND_SOC_DAPM_MIXER("I2SOUT6_CH2", SND_SOC_NOPM, 0, 0,
+			   mtk_i2sout6_ch2_mix,
+			   ARRAY_SIZE(mtk_i2sout6_ch2_mix)),
+
+	/* i2s en*/
+	SND_SOC_DAPM_SUPPLY_S("I2SIN0_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_IN0_CON0, REG_ETDM_IN_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN1_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_IN1_CON0, REG_ETDM_IN_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN2_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_IN2_CON0, REG_ETDM_IN_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN3_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_IN3_CON0, REG_ETDM_IN_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN4_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_IN4_CON0, REG_ETDM_IN_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN6_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_IN6_CON0, REG_ETDM_IN_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT0_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_OUT0_CON0, OUT_REG_ETDM_OUT_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT1_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_OUT1_CON0, OUT_REG_ETDM_OUT_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT2_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_OUT2_CON0, OUT_REG_ETDM_OUT_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT3_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_OUT3_CON0, OUT_REG_ETDM_OUT_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT4_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_OUT4_CON0, OUT_REG_ETDM_OUT_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT6_EN", SUPPLY_SEQ_I2S_EN,
+			      ETDM_OUT6_CON0, OUT_REG_ETDM_OUT_EN_SFT, 0,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("FMI2S_MASTER_EN", SUPPLY_SEQ_I2S_EN,
+			      AFE_CONNSYS_I2S_CON, I2S_EN_SFT, 0,
+			      NULL, 0),
+
+	/* i2s mclk en */
+	SND_SOC_DAPM_SUPPLY_S(I2SIN0_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SIN1_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SIN2_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SIN3_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SIN4_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SIN6_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SOUT0_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SOUT1_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SOUT2_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SOUT3_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SOUT4_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(I2SOUT6_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(FMI2S_MASTER_MCLK_EN_W_NAME, SUPPLY_SEQ_I2S_MCLK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_mclk_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+
+	/* cg */
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT0_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_OUT0_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT1_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_OUT1_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT2_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_OUT2_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT3_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_OUT3_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT4_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_OUT4_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SOUT6_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_OUT6_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN0_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_IN0_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN1_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_IN1_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN2_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_IN2_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN3_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_IN3_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN4_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_IN4_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("I2SIN6_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON2, PDN_ETDM_IN6_SFT, 1,
+			      NULL, 0),
+	SND_SOC_DAPM_SUPPLY_S("FMI2S_MASTER_CG", SUPPLY_SEQ_I2S_CG_EN,
+			      AUDIO_TOP_CON0, PDN_FM_I2S_SFT, 1,
+			      NULL, 0),
+
+	/* apll */
+	SND_SOC_DAPM_SUPPLY_S(APLL1_W_NAME, SUPPLY_SEQ_APLL,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_apll_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+	SND_SOC_DAPM_SUPPLY_S(APLL2_W_NAME, SUPPLY_SEQ_APLL,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_apll_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+
+	SND_SOC_DAPM_MIXER("SOF_DMA_DL_24CH", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("SOF_DMA_DL1", SND_SOC_NOPM, 0, 0, NULL, 0),
+};
+
+static int mtk_afe_i2s_share_connect(struct snd_soc_dapm_widget *source,
+				     struct snd_soc_dapm_widget *sink)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(sink->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mtk_afe_i2s_priv *i2s_priv;
+
+	i2s_priv = get_i2s_priv_by_name(afe, sink->name);
+	if (!i2s_priv)
+		return 0;
+
+	if (i2s_priv->share_i2s_id < 0)
+		return 0;
+
+	return i2s_priv->share_i2s_id == get_i2s_id_by_name(afe, source->name);
+}
+
+static int mtk_afe_i2s_apll_connect(struct snd_soc_dapm_widget *source,
+				    struct snd_soc_dapm_widget *sink)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(sink->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mtk_afe_i2s_priv *i2s_priv;
+	int cur_apll;
+	int needed_apll;
+
+	i2s_priv = get_i2s_priv_by_name(afe, sink->name);
+	if (!i2s_priv)
+		return 0;
+
+	/* which apll */
+	cur_apll = mt8196_get_apll_by_name(afe, source->name);
+
+	/* choose APLL from i2s rate */
+	needed_apll = mt8196_get_apll_by_rate(afe, i2s_priv->rate);
+
+	return needed_apll == cur_apll;
+}
+
+static int mtk_afe_i2s_mclk_connect(struct snd_soc_dapm_widget *source,
+				    struct snd_soc_dapm_widget *sink)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(sink->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mtk_afe_i2s_priv *i2s_priv;
+	int i2s_num;
+
+	i2s_priv = get_i2s_priv_by_name(afe, sink->name);
+	if (!i2s_priv)
+		return 0;
+
+	i2s_num = get_i2s_id_by_name(afe, source->name);
+	if (get_i2s_id_by_name(afe, sink->name) == i2s_num)
+		return i2s_priv->mclk_rate > 0;
+
+	/* check if share i2s need mclk */
+	if (i2s_priv->share_i2s_id < 0)
+		return 0;
+
+	if (i2s_priv->share_i2s_id == i2s_num)
+		return i2s_priv->mclk_rate > 0;
+
+	return 0;
+}
+
+static int mtk_afe_mclk_apll_connect(struct snd_soc_dapm_widget *source,
+				     struct snd_soc_dapm_widget *sink)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(sink->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mtk_afe_i2s_priv *i2s_priv;
+	int cur_apll;
+
+	i2s_priv = get_i2s_priv_by_name(afe, sink->name);
+	if (!i2s_priv)
+		return 0;
+
+	/* which apll */
+	cur_apll = mt8196_get_apll_by_name(afe, source->name);
+
+	return i2s_priv->mclk_apll == cur_apll;
+}
+
+static const struct snd_soc_dapm_route mtk_dai_i2s_routes[] = {
+	/* i2sin0 */
+	{"I2SIN0", NULL, "I2SIN0_EN"},
+	{"I2SIN0", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN0", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SIN0", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SIN0", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SIN0", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN0", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+
+	{I2SIN0_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SIN0_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SIN0", NULL, "I2SIN0_CG"},
+
+	/* i2sin1 */
+	{"I2SIN1", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SIN1_EN"},
+	{"I2SIN1", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN1", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SIN1", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SIN1", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SIN1", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN1", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SIN1_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SIN1_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SIN1", NULL, "I2SIN1_CG"},
+
+	/* i2sin2 */
+	{"I2SIN2", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SIN2_EN"},
+	{"I2SIN2", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN2", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SIN2", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SIN2", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SIN2", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN2", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SIN2_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SIN2_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SIN2", NULL, "I2SIN2_CG"},
+
+	/* i2sin3 */
+	{"I2SIN3", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SIN3_EN"},
+	{"I2SIN3", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN3", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SIN3", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SIN3", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SIN3", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN3", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SIN3_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SIN3_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SIN3", NULL, "I2SIN3_CG"},
+
+	/* i2sin4 */
+	{"I2SIN4", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SIN4_EN"},
+	{"I2SIN4", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN4", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SIN4", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SIN4", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SIN4", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN4", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SIN4_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SIN4_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SIN4", NULL, "I2SIN4_CG"},
+
+	/* i2sin6 */
+	{"I2SIN6", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SIN6_EN"},
+	{"I2SIN6", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SIN6", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SIN6", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SIN6", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SIN6", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SIN6", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SIN6_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SIN6_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SIN6", NULL, "I2SIN6_CG"},
+	{"I2SIN6", NULL, "I2SOUT6_CG"},
+
+	/* i2sout0 */
+	{"I2SOUT0_CH1", "DL0_CH1", "DL0"},
+	{"I2SOUT0_CH2", "DL0_CH2", "DL0"},
+	{"I2SOUT0_CH1", "DL1_CH1", "DL1"},
+	{"I2SOUT0_CH2", "DL1_CH2", "DL1"},
+	{"I2SOUT0_CH1", "DL2_CH1", "DL2"},
+	{"I2SOUT0_CH2", "DL2_CH2", "DL2"},
+	{"I2SOUT0_CH1", "DL3_CH1", "DL3"},
+	{"I2SOUT0_CH2", "DL3_CH2", "DL3"},
+	{"I2SOUT0_CH1", "DL4_CH1", "DL4"},
+	{"I2SOUT0_CH2", "DL4_CH2", "DL4"},
+	{"I2SOUT0_CH1", "DL5_CH1", "DL5"},
+	{"I2SOUT0_CH2", "DL5_CH2", "DL5"},
+	{"I2SOUT0_CH1", "DL6_CH1", "DL6"},
+	{"I2SOUT0_CH2", "DL6_CH2", "DL6"},
+	{"I2SOUT0_CH1", "DL7_CH1", "DL7"},
+	{"I2SOUT0_CH2", "DL7_CH2", "DL7"},
+	{"I2SOUT0_CH1", "DL8_CH1", "DL8"},
+	{"I2SOUT0_CH2", "DL8_CH2", "DL8"},
+	{"I2SOUT0_CH1", "DL23_CH1", "DL23"},
+	{"I2SOUT0_CH2", "DL23_CH2", "DL23"},
+	{"I2SOUT0_CH1", "DL_24CH_CH1", "DL_24CH"},
+	{"I2SOUT0_CH2", "DL_24CH_CH2", "DL_24CH"},
+
+	{"I2SOUT0_CH1", "DL24_CH1", "DL24"},
+	{"I2SOUT0_CH2", "DL24_CH2", "DL24"},
+
+	{"I2SOUT0", NULL, "I2SOUT0_CH1"},
+	{"I2SOUT0", NULL, "I2SOUT0_CH2"},
+
+	{"I2SOUT0", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SOUT0_EN"},
+	{"I2SOUT0", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT0", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SOUT0", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SOUT0", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SOUT0", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT0", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SOUT0_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SOUT0_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SOUT0", NULL, "I2SOUT0_CG"},
+	{"I2SOUT0", NULL, "I2SIN0_CG"},
+
+	/* i2sout1 */
+	{"I2SOUT1_CH1", "DL0_CH1", "DL0"},
+	{"I2SOUT1_CH2", "DL0_CH2", "DL0"},
+	{"I2SOUT1_CH1", "DL1_CH1", "DL1"},
+	{"I2SOUT1_CH2", "DL1_CH2", "DL1"},
+	{"I2SOUT1_CH1", "DL2_CH1", "DL2"},
+	{"I2SOUT1_CH2", "DL2_CH2", "DL2"},
+	{"I2SOUT1_CH1", "DL3_CH1", "DL3"},
+	{"I2SOUT1_CH2", "DL3_CH2", "DL3"},
+	{"I2SOUT1_CH1", "DL4_CH1", "DL4"},
+	{"I2SOUT1_CH2", "DL4_CH2", "DL4"},
+	{"I2SOUT1_CH1", "DL5_CH1", "DL5"},
+	{"I2SOUT1_CH2", "DL5_CH2", "DL5"},
+	{"I2SOUT1_CH1", "DL6_CH1", "DL6"},
+	{"I2SOUT1_CH2", "DL6_CH2", "DL6"},
+	{"I2SOUT1_CH1", "DL7_CH1", "DL7"},
+	{"I2SOUT1_CH2", "DL7_CH2", "DL7"},
+	{"I2SOUT1_CH1", "DL8_CH1", "DL8"},
+	{"I2SOUT1_CH2", "DL8_CH2", "DL8"},
+	{"I2SOUT1_CH1", "DL_24CH_CH1", "DL_24CH"},
+	{"I2SOUT1_CH2", "DL_24CH_CH2", "DL_24CH"},
+
+	{"I2SOUT1", NULL, "I2SOUT1_CH1"},
+	{"I2SOUT1", NULL, "I2SOUT1_CH2"},
+
+	{"I2SOUT1", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SOUT1_EN"},
+	{"I2SOUT1", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT1", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SOUT1", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SOUT1", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SOUT1", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT1", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SOUT1_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SOUT1_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SOUT1", NULL, "I2SOUT1_CG"},
+	{"I2SOUT1", NULL, "I2SIN1_CG"},
+
+	/* i2sout2 */
+	{"I2SOUT2_CH1", "DL0_CH1", "DL0"},
+	{"I2SOUT2_CH2", "DL0_CH2", "DL0"},
+	{"I2SOUT2_CH1", "DL1_CH1", "DL1"},
+	{"I2SOUT2_CH2", "DL1_CH2", "DL1"},
+	{"I2SOUT2_CH1", "DL2_CH1", "DL2"},
+	{"I2SOUT2_CH2", "DL2_CH2", "DL2"},
+	{"I2SOUT2_CH1", "DL3_CH1", "DL3"},
+	{"I2SOUT2_CH2", "DL3_CH2", "DL3"},
+	{"I2SOUT2_CH1", "DL4_CH1", "DL4"},
+	{"I2SOUT2_CH2", "DL4_CH2", "DL4"},
+	{"I2SOUT2_CH1", "DL5_CH1", "DL5"},
+	{"I2SOUT2_CH2", "DL5_CH2", "DL5"},
+	{"I2SOUT2_CH1", "DL6_CH1", "DL6"},
+	{"I2SOUT2_CH2", "DL6_CH2", "DL6"},
+	{"I2SOUT2_CH1", "DL7_CH1", "DL7"},
+	{"I2SOUT2_CH2", "DL7_CH2", "DL7"},
+	{"I2SOUT2_CH1", "DL8_CH1", "DL8"},
+	{"I2SOUT2_CH2", "DL8_CH2", "DL8"},
+	{"I2SOUT2_CH1", "DL_24CH_CH1", "DL_24CH"},
+	{"I2SOUT2_CH2", "DL_24CH_CH2", "DL_24CH"},
+
+	{"I2SOUT2", NULL, "I2SOUT2_CH1"},
+	{"I2SOUT2", NULL, "I2SOUT2_CH2"},
+
+	{"I2SOUT2", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SOUT2_EN"},
+	{"I2SOUT2", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT2", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SOUT2", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SOUT2", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SOUT2", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT2", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SOUT2_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SOUT2_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SOUT2", NULL, "I2SOUT2_CG"},
+	{"I2SOUT2", NULL, "I2SIN2_CG"},
+
+	/* i2sout3 */
+	{"I2SOUT3_CH1", "DL0_CH1", "DL0"},
+	{"I2SOUT3_CH2", "DL0_CH2", "DL0"},
+	{"I2SOUT3_CH1", "DL1_CH1", "DL1"},
+	{"I2SOUT3_CH2", "DL1_CH2", "DL1"},
+	{"I2SOUT3_CH1", "DL2_CH1", "DL2"},
+	{"I2SOUT3_CH2", "DL2_CH2", "DL2"},
+	{"I2SOUT3_CH1", "DL3_CH1", "DL3"},
+	{"I2SOUT3_CH2", "DL3_CH2", "DL3"},
+	{"I2SOUT3_CH1", "DL4_CH1", "DL4"},
+	{"I2SOUT3_CH2", "DL4_CH2", "DL4"},
+	{"I2SOUT3_CH1", "DL5_CH1", "DL5"},
+	{"I2SOUT3_CH2", "DL5_CH2", "DL5"},
+	{"I2SOUT3_CH1", "DL6_CH1", "DL6"},
+	{"I2SOUT3_CH2", "DL6_CH2", "DL6"},
+	{"I2SOUT3_CH1", "DL7_CH1", "DL7"},
+	{"I2SOUT3_CH2", "DL7_CH2", "DL7"},
+	{"I2SOUT3_CH1", "DL8_CH1", "DL8"},
+	{"I2SOUT3_CH2", "DL8_CH2", "DL8"},
+	{"I2SOUT3_CH1", "DL_24CH_CH1", "DL_24CH"},
+	{"I2SOUT3_CH2", "DL_24CH_CH2", "DL_24CH"},
+
+	{"I2SOUT3", NULL, "I2SOUT3_CH1"},
+	{"I2SOUT3", NULL, "I2SOUT3_CH2"},
+
+	{"I2SOUT3", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SOUT3_EN"},
+	{"I2SOUT3", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT3", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SOUT3", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SOUT3", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SOUT3", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT3", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SOUT3_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SOUT3_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	{"I2SOUT3", NULL, "I2SOUT3_CG"},
+	{"I2SOUT3", NULL, "I2SIN3_CG"},
+
+	/* i2sout4 */
+	{"I2SOUT4_CH1", "DL0_CH1", "DL0"},
+	{"I2SOUT4_CH2", "DL0_CH2", "DL0"},
+	{"I2SOUT4_CH1", "DL1_CH1", "DL1"},
+	{"I2SOUT4_CH2", "DL1_CH2", "DL1"},
+	{"I2SOUT4_CH1", "DL2_CH1", "DL2"},
+	{"I2SOUT4_CH2", "DL2_CH2", "DL2"},
+	{"I2SOUT4_CH1", "DL3_CH1", "DL3"},
+	{"I2SOUT4_CH2", "DL3_CH2", "DL3"},
+	{"I2SOUT4_CH1", "DL4_CH1", "DL4"},
+	{"I2SOUT4_CH2", "DL4_CH2", "DL4"},
+	{"I2SOUT4_CH1", "DL5_CH1", "DL5"},
+	{"I2SOUT4_CH2", "DL5_CH2", "DL5"},
+	{"I2SOUT4_CH1", "DL6_CH1", "DL6"},
+	{"I2SOUT4_CH2", "DL6_CH2", "DL6"},
+	{"I2SOUT4_CH1", "DL7_CH1", "DL7"},
+	{"I2SOUT4_CH2", "DL7_CH2", "DL7"},
+	{"I2SOUT4_CH1", "DL8_CH1", "DL8"},
+	{"I2SOUT4_CH2", "DL8_CH2", "DL8"},
+	{"I2SOUT4_CH1", "DL_24CH_CH1", "DL_24CH"},
+	{"I2SOUT4_CH2", "DL_24CH_CH2", "DL_24CH"},
+	{"I2SOUT4_CH3", "DL_24CH_CH3", "DL_24CH"},
+	{"I2SOUT4_CH4", "DL_24CH_CH4", "DL_24CH"},
+	{"I2SOUT4_CH5", "DL_24CH_CH5", "DL_24CH"},
+	{"I2SOUT4_CH6", "DL_24CH_CH6", "DL_24CH"},
+	{"I2SOUT4_CH7", "DL_24CH_CH7", "DL_24CH"},
+	{"I2SOUT4_CH8", "DL_24CH_CH8", "DL_24CH"},
+	{"I2SOUT4_CH1", "DL24_CH1", "DL24"},
+	{"I2SOUT4_CH2", "DL24_CH2", "DL24"},
+
+	/* SOF Downlink */
+	{"I2SOUT4_CH1", "DL_24CH_CH1", "SOF_DMA_DL_24CH"},
+	{"I2SOUT4_CH2", "DL_24CH_CH2", "SOF_DMA_DL_24CH"},
+	{"I2SOUT4_CH3", "DL_24CH_CH3", "SOF_DMA_DL_24CH"},
+	{"I2SOUT4_CH4", "DL_24CH_CH4", "SOF_DMA_DL_24CH"},
+
+	{"I2SOUT4", NULL, "I2SOUT4_CH1"},
+	{"I2SOUT4", NULL, "I2SOUT4_CH2"},
+	{"I2SOUT4", NULL, "I2SOUT4_CH3"},
+	{"I2SOUT4", NULL, "I2SOUT4_CH4"},
+	{"I2SOUT4", NULL, "I2SOUT4_CH5"},
+	{"I2SOUT4", NULL, "I2SOUT4_CH6"},
+	{"I2SOUT4", NULL, "I2SOUT4_CH7"},
+	{"I2SOUT4", NULL, "I2SOUT4_CH8"},
+
+	{"I2SOUT4", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "I2SOUT4_EN"},
+	{"I2SOUT4", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT4", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SOUT4", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SOUT4", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SOUT4", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT4", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SOUT4_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SOUT4_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	/* CG */
+	{"I2SOUT4", NULL, "I2SOUT4_CG"},
+	{"I2SOUT4", NULL, "I2SIN4_CG"},
+
+	/* i2sout6 */
+	{"I2SOUT6_CH1", "DL0_CH1", "DL0"},
+	{"I2SOUT6_CH2", "DL0_CH2", "DL0"},
+	{"I2SOUT6_CH1", "DL1_CH1", "DL1"},
+	{"I2SOUT6_CH2", "DL1_CH2", "DL1"},
+	{"I2SOUT6_CH1", "DL2_CH1", "DL2"},
+	{"I2SOUT6_CH2", "DL2_CH2", "DL2"},
+	{"I2SOUT6_CH1", "DL3_CH1", "DL3"},
+	{"I2SOUT6_CH2", "DL3_CH2", "DL3"},
+	{"I2SOUT6_CH1", "DL4_CH1", "DL4"},
+	{"I2SOUT6_CH2", "DL4_CH2", "DL4"},
+	{"I2SOUT6_CH1", "DL5_CH1", "DL5"},
+	{"I2SOUT6_CH2", "DL5_CH2", "DL5"},
+	{"I2SOUT6_CH1", "DL6_CH1", "DL6"},
+	{"I2SOUT6_CH2", "DL6_CH2", "DL6"},
+	{"I2SOUT6_CH1", "DL7_CH1", "DL7"},
+	{"I2SOUT6_CH2", "DL7_CH2", "DL7"},
+	{"I2SOUT6_CH1", "DL8_CH1", "DL8"},
+	{"I2SOUT6_CH2", "DL8_CH2", "DL8"},
+	{"I2SOUT6_CH1", "DL23_CH1", "DL23"},
+	{"I2SOUT6_CH2", "DL23_CH2", "DL23"},
+	{"I2SOUT6_CH1", "DL_24CH_CH1", "DL_24CH"},
+	{"I2SOUT6_CH2", "DL_24CH_CH2", "DL_24CH"},
+
+	/* SOF Downlink */
+	{"I2SOUT6_CH1", "DL1_CH1", "SOF_DMA_DL1"},
+	{"I2SOUT6_CH2", "DL1_CH2", "SOF_DMA_DL1"},
+	{"I2SOUT6_CH1", "DL_24CH_CH1", "SOF_DMA_DL_24CH"},
+	{"I2SOUT6_CH2", "DL_24CH_CH2", "SOF_DMA_DL_24CH"},
+
+	{"I2SOUT6", NULL, "I2SOUT6_CH1"},
+	{"I2SOUT6", NULL, "I2SOUT6_CH2"},
+
+	{"I2SOUT6", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"I2SOUT6", NULL, "I2SOUT6_EN"},
+	{"I2SOUT6", NULL, "FMI2S_MASTER_EN", mtk_afe_i2s_share_connect},
+
+	{"I2SOUT6", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"I2SOUT6", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"I2SOUT6", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"I2SOUT6", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{I2SOUT6_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{I2SOUT6_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	/* CG */
+	{"I2SOUT6", NULL, "I2SOUT6_CG"},
+	{"I2SOUT6", NULL, "I2SIN6_CG"},
+
+	/* fmi2s */
+	{"FMI2S_MASTER", NULL, "I2SIN0_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SIN1_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SIN2_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SIN3_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SIN4_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SIN6_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SOUT0_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SOUT1_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SOUT2_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SOUT3_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SOUT4_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "I2SOUT6_EN", mtk_afe_i2s_share_connect},
+	{"FMI2S_MASTER", NULL, "FMI2S_MASTER_EN"},
+
+	{"FMI2S_MASTER", NULL, APLL1_W_NAME, mtk_afe_i2s_apll_connect},
+	{"FMI2S_MASTER", NULL, APLL2_W_NAME, mtk_afe_i2s_apll_connect},
+
+	{"FMI2S_MASTER", NULL, I2SIN0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SIN1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SIN2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SIN3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SIN4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SIN6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SOUT0_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SOUT1_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SOUT2_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SOUT3_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SOUT4_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, I2SOUT6_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{"FMI2S_MASTER", NULL, FMI2S_MASTER_MCLK_EN_W_NAME, mtk_afe_i2s_mclk_connect},
+	{FMI2S_MASTER_MCLK_EN_W_NAME, NULL, APLL1_W_NAME, mtk_afe_mclk_apll_connect},
+	{FMI2S_MASTER_MCLK_EN_W_NAME, NULL, APLL2_W_NAME, mtk_afe_mclk_apll_connect},
+	/* CG */
+	{"FMI2S_MASTER", NULL, "FMI2S_MASTER_CG"},
+};
+
+/* i2s dai ops*/
+static int mtk_dai_i2s_config(struct mtk_base_afe *afe,
+			      struct snd_pcm_hw_params *params,
+			      int i2s_id)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	struct mtk_afe_i2s_priv *i2s_priv;
+	struct mtk_afe_i2s_priv *i2sin_priv = NULL;
+	int id = i2s_id - MT8196_DAI_I2S_IN0;
+	struct mtk_base_etdm_data etdm_data;
+	unsigned int rate = params_rate(params);
+	unsigned int rate_reg = get_etdm_inconn_rate(rate);
+	snd_pcm_format_t format = params_format(params);
+	unsigned int channels = params_channels(params);
+	int ret;
+	unsigned int i2s_con;
+	int pad_top;
+
+	if (i2s_id >= MT8196_DAI_NUM || i2s_id < 0 || id < 0 || id >= DAI_I2S_NUM)
+		return -EINVAL;
+
+	i2s_priv = afe_priv->dai_priv[i2s_id];
+	if (!i2s_priv)
+		return -EINVAL;
+
+	dev_dbg(afe->dev, "id: %d, rate: %d, pcm_fmt: %d, fmt: %d, ch: %d\n",
+		i2s_id, rate, format, i2s_priv->format, channels);
+
+	i2s_priv->rate = rate;
+	etdm_data = mtk_etdm_data[id];
+
+	if (is_etdm_in_pad_top(id))
+		pad_top = 0x3;
+	else
+		pad_top = 0x5;
+
+	switch (id) {
+	case DAI_FMI2S_MASTER:
+		i2s_con = I2S_IN_PAD_IO_MUX << I2SIN_PAD_SEL_SFT;
+		i2s_con |= rate_reg << I2S_MODE_SFT;
+		i2s_con |= I2S_FMT_I2S << I2S_FMT_SFT;
+		i2s_con |= get_i2s_wlen(format) << I2S_WLEN_SFT;
+		regmap_update_bits(afe->regmap, AFE_CONNSYS_I2S_CON,
+				   0xffffeffe, i2s_con);
+		break;
+
+	case DAI_I2SIN0:
+	case DAI_I2SIN1:
+	case DAI_I2SIN2:
+	case DAI_I2SIN3:
+	case DAI_I2SIN4:
+	case DAI_I2SIN6:
+		/* ---etdm in --- */
+		regmap_update_bits(afe->regmap,
+				   etdm_data.init_count_reg,
+				   etdm_data.init_count_mask << etdm_data.init_count_shift,
+				   0x5 << etdm_data.init_count_shift);
+
+		/* 3: pad top 5: no pad top */
+		regmap_update_bits(afe->regmap,
+				   etdm_data.init_point_reg,
+				   etdm_data.init_point_mask << etdm_data.init_point_shift,
+				   pad_top << etdm_data.init_point_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.lrck_reset_reg,
+				   etdm_data.lrck_reset_mask << etdm_data.lrck_reset_shift,
+				   0x1 << etdm_data.lrck_reset_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.clk_source_reg,
+				   etdm_data.clk_source_mask << etdm_data.clk_source_shift,
+				   ETDM_CLK_SOURCE_APLL << etdm_data.clk_source_shift);
+
+		/* 0: manual 1: auto */
+		regmap_update_bits(afe->regmap,
+				   etdm_data.ck_en_sel_reg,
+				   etdm_data.ck_en_sel_mask << etdm_data.ck_en_sel_shift,
+				   0x1 << etdm_data.ck_en_sel_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.fs_timing_reg,
+				   etdm_data.fs_timing_mask << etdm_data.fs_timing_shift,
+				   get_etdm_rate(rate) << etdm_data.fs_timing_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.relatch_en_sel_reg,
+				   etdm_data.relatch_en_sel_mask << etdm_data.relatch_en_sel_shift,
+				   get_etdm_inconn_rate(rate) << etdm_data.relatch_en_sel_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.use_afifo_reg,
+				   etdm_data.use_afifo_mask << etdm_data.use_afifo_shift,
+				   0x0);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.afifo_mode_reg,
+				   etdm_data.afifo_mode_mask << etdm_data.afifo_mode_shift,
+				   0x0);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.almost_end_ch_reg,
+				   etdm_data.almost_end_ch_mask << etdm_data.almost_end_ch_shift,
+				   0x0);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.almost_end_bit_reg,
+				   etdm_data.almost_end_bit_mask << etdm_data.almost_end_bit_shift,
+				   0x0);
+
+		if (is_etdm_in_pad_top(id)) {
+			regmap_update_bits(afe->regmap,
+					   etdm_data.out2latch_time_reg,
+					   etdm_data.out2latch_time_mask <<
+					   etdm_data.out2latch_time_shift,
+					   0x6 << etdm_data.out2latch_time_shift);
+		} else {
+			regmap_update_bits(afe->regmap,
+					   etdm_data.out2latch_time_reg,
+					   etdm_data.out2latch_time_mask <<
+					   etdm_data.out2latch_time_shift,
+					   0x4 << etdm_data.out2latch_time_shift);
+		}
+
+		if (id == DAI_I2SIN4) {
+			dev_dbg(afe->dev, "i2sin4, id: %d, fmt: %d, ch: %d, ip_mode: %d, sync: %d\n",
+				id, i2s_priv->format, channels, i2s_priv->ip_mode, i2s_priv->sync);
+
+			/* Fmt Mode: 0x00 i2s, 0x04 adsp_a, DSP_A mode for multi-channel */
+			regmap_update_bits(afe->regmap,
+					   etdm_data.tdm_mode_reg,
+					   etdm_data.tdm_mode_mask << etdm_data.tdm_mode_shift,
+					   i2s_priv->format << etdm_data.tdm_mode_shift);
+
+			/* set etdm ch */
+			regmap_update_bits(afe->regmap,
+					   etdm_data.ch_reg,
+					   etdm_data.ch_mask << etdm_data.ch_shift,
+					   (channels - 1) << etdm_data.ch_shift);
+
+			/* set etdm ip mode */
+			regmap_update_bits(afe->regmap,
+					   etdm_data.ip_mode_reg,
+					   etdm_data.ip_mode_mask << etdm_data.ip_mode_shift,
+					   i2s_priv->ip_mode << etdm_data.ip_mode_shift);
+
+			/* set etdm sync */
+			regmap_update_bits(afe->regmap,
+					   etdm_data.sync_reg,
+					   etdm_data.sync_mask << etdm_data.sync_shift,
+					   i2s_priv->sync << etdm_data.sync_shift);
+		} else {
+			/* default i2s */
+			regmap_update_bits(afe->regmap,
+					   etdm_data.tdm_mode_reg,
+					   etdm_data.tdm_mode_mask << etdm_data.tdm_mode_shift,
+					   0x0 << etdm_data.tdm_mode_shift);
+
+			/* set etdm sync */
+			regmap_update_bits(afe->regmap,
+					   etdm_data.sync_reg,
+					   etdm_data.sync_mask << etdm_data.sync_shift,
+					   0x0 << etdm_data.sync_shift);
+		}
+
+		/* APLL */
+		regmap_update_bits(afe->regmap,
+				   etdm_data.relatch_domain_sel_reg,
+				   etdm_data.relatch_domain_sel_mask <<
+				   etdm_data.relatch_domain_sel_shift,
+				   ETDM_RELATCH_SEL_APLL << etdm_data.relatch_domain_sel_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.bit_length_reg,
+				   etdm_data.bit_length_mask << etdm_data.bit_length_shift,
+				   get_etdm_lrck_width(format) << etdm_data.bit_length_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.word_length_reg,
+				   etdm_data.word_length_mask << etdm_data.word_length_shift,
+				   get_etdm_wlen(format) << etdm_data.word_length_shift);
+
+		/* ---etdm cowork --- */
+		regmap_update_bits(afe->regmap,
+				   etdm_data.cowork_reg,
+				   etdm_data.cowork_mask << etdm_data.cowork_shift,
+				   etdm_data.cowork_val << etdm_data.cowork_shift);
+
+		/* i2s with pad top setting */
+		if (is_etdm_in_pad_top(id) && etdm_data.pad_top_ck_en_reg != 0) {
+			regmap_update_bits(afe->regmap,
+					   etdm_data.pad_top_ck_en_reg,
+					   etdm_data.pad_top_ck_en_mask <<
+					   etdm_data.pad_top_ck_en_shift,
+					   0x1 << etdm_data.pad_top_ck_en_shift);
+
+			regmap_update_bits(afe->regmap,
+					   etdm_data.master_latch_reg,
+					   etdm_data.master_latch_mask <<
+					   etdm_data.master_latch_shift,
+					   0x0);
+		}
+		break;
+
+	case DAI_I2SOUT0:
+	case DAI_I2SOUT1:
+	case DAI_I2SOUT2:
+	case DAI_I2SOUT3:
+	case DAI_I2SOUT4:
+	case DAI_I2SOUT6:
+		/* ---etdm out --- */
+		regmap_update_bits(afe->regmap,
+				   etdm_data.init_count_reg,
+				   etdm_data.init_count_mask << etdm_data.init_count_shift,
+				   0x5 << etdm_data.init_count_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.init_point_reg,
+				   etdm_data.init_point_mask << etdm_data.init_point_shift,
+				   0x6 << etdm_data.init_point_shift);
+
+		// clock speed > 22M need to set relatch time to avoid duplicate porint
+		if (rate * channels * (get_etdm_wlen(format) + 1) >= ETDM_22M_CLOCK_THRES &&
+		    get_etdm_wlen(format) >= 2) {
+			regmap_update_bits(afe->regmap,
+					   etdm_data.in2latch_time_reg,
+					   etdm_data.in2latch_time_mask <<
+					   etdm_data.in2latch_time_shift,
+					   (get_etdm_wlen(format) - 2) <<
+					   etdm_data.in2latch_time_shift);
+		} else {
+			regmap_update_bits(afe->regmap,
+					   etdm_data.in2latch_time_reg,
+					   etdm_data.in2latch_time_mask <<
+					   etdm_data.in2latch_time_shift,
+					   0x6 << etdm_data.in2latch_time_shift);
+		}
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.lrck_reset_reg,
+				   etdm_data.lrck_reset_mask << etdm_data.lrck_reset_shift,
+				   0x1 << etdm_data.lrck_reset_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.fs_timing_reg,
+				   etdm_data.fs_timing_mask << etdm_data.fs_timing_shift,
+				   get_etdm_rate(rate) << etdm_data.fs_timing_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.clk_source_reg,
+				   etdm_data.clk_source_mask << etdm_data.clk_source_shift,
+				   ETDM_CLK_SOURCE_APLL << etdm_data.clk_source_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.relatch_en_sel_reg,
+				   etdm_data.relatch_en_sel_mask << etdm_data.relatch_en_sel_shift,
+				   get_etdm_inconn_rate(rate) << etdm_data.relatch_en_sel_shift);
+
+		if (id == DAI_I2SOUT4) {
+			dev_dbg(afe->dev, "i2sout4, id: %d fmt: %d, ch: %d, sync: %d\n",
+				id, i2s_priv->format, channels, i2s_priv->sync);
+
+			/* Fmt Mode: 0x00 i2s, 0x04 adsp_a, DSP_A mode for multi-channel */
+			regmap_update_bits(afe->regmap,
+					   etdm_data.tdm_mode_reg,
+					   etdm_data.tdm_mode_mask << etdm_data.tdm_mode_shift,
+					   i2s_priv->format << etdm_data.tdm_mode_shift);
+
+			/* set etdm ch */
+			regmap_update_bits(afe->regmap,
+					   etdm_data.ch_reg,
+					   etdm_data.ch_mask << etdm_data.ch_shift,
+					   (channels - 1) << etdm_data.ch_shift);
+
+			/* set etdm sync */
+			regmap_update_bits(afe->regmap,
+					   etdm_data.sync_reg,
+					   etdm_data.sync_mask << etdm_data.sync_shift,
+					   i2s_priv->sync << etdm_data.sync_shift);
+		} else {
+			regmap_update_bits(afe->regmap,
+					   etdm_data.tdm_mode_reg,
+					   etdm_data.tdm_mode_mask << etdm_data.tdm_mode_shift,
+					   0x0);
+		}
+
+		/* APLL */
+		regmap_update_bits(afe->regmap,
+				   etdm_data.relatch_domain_sel_reg,
+				   etdm_data.relatch_domain_sel_mask <<
+				   etdm_data.relatch_domain_sel_shift,
+				   ETDM_RELATCH_SEL_APLL << etdm_data.relatch_domain_sel_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.bit_length_reg,
+				   etdm_data.bit_length_mask << etdm_data.bit_length_shift,
+				   get_etdm_lrck_width(format) << etdm_data.bit_length_shift);
+
+		regmap_update_bits(afe->regmap,
+				   etdm_data.word_length_reg,
+				   etdm_data.word_length_mask << etdm_data.word_length_shift,
+				   get_etdm_wlen(format) << etdm_data.word_length_shift);
+
+		/* ---etdm cowork --- */
+		regmap_update_bits(afe->regmap,
+				   etdm_data.cowork_reg,
+				   etdm_data.cowork_mask << etdm_data.cowork_shift,
+				   etdm_data.cowork_val << etdm_data.cowork_shift);
+
+		/* i2s with pad top setting */
+		if (is_etdm_in_pad_top(id) && etdm_data.pad_top_ck_en_reg != 0) {
+			regmap_update_bits(afe->regmap,
+					   etdm_data.pad_top_ck_en_reg,
+					   etdm_data.cowork_mask << etdm_data.pad_top_ck_en_shift,
+					   0x1 << etdm_data.pad_top_ck_en_shift);
+
+			regmap_update_bits(afe->regmap,
+					   etdm_data.master_latch_reg,
+					   etdm_data.master_latch_mask <<
+					   etdm_data.master_latch_shift,
+					   0x0);
+		}
+		break;
+
+	default:
+		dev_err(afe->dev, "id %d not support\n", id);
+		return -EINVAL;
+	}
+
+	/* set share i2s */
+	if (i2s_priv && i2s_priv->share_i2s_id >= 0) {
+		i2sin_priv = afe_priv->dai_priv[i2s_priv->share_i2s_id];
+		i2sin_priv->format = i2s_priv->format;
+		ret = mtk_dai_i2s_config(afe, params, i2s_priv->share_i2s_id);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+static int mtk_dai_i2s_hw_params(struct snd_pcm_substream *substream,
+				 struct snd_pcm_hw_params *params,
+				 struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+
+	return mtk_dai_i2s_config(afe, params, dai->id);
+}
+
+static int mtk_dai_i2s_set_sysclk(struct snd_soc_dai *dai,
+				  int clk_id, unsigned int freq, int dir)
+{
+	struct mtk_base_afe *afe = dev_get_drvdata(dai->dev);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	struct mtk_afe_i2s_priv *i2s_priv;
+	int apll;
+	int apll_rate;
+
+	if (dai->id >= MT8196_DAI_NUM || dai->id < 0 || dir != SND_SOC_CLOCK_OUT)
+		return -EINVAL;
+
+	i2s_priv = afe_priv->dai_priv[dai->id];
+	if (!i2s_priv)
+		return -EINVAL;
+
+	dev_dbg(afe->dev, "freq: %u\n", freq);
+
+	apll = mt8196_get_apll_by_rate(afe, freq);
+	apll_rate = mt8196_get_apll_rate(afe, apll);
+
+	if (freq > apll_rate || apll_rate % freq)
+		return -EINVAL;
+
+	i2s_priv->mclk_rate = freq;
+	i2s_priv->mclk_apll = apll;
+
+	if (i2s_priv->share_i2s_id > 0) {
+		struct mtk_afe_i2s_priv *share_i2s_priv;
+
+		share_i2s_priv = afe_priv->dai_priv[i2s_priv->share_i2s_id];
+		if (!share_i2s_priv)
+			return -EINVAL;
+
+		share_i2s_priv->mclk_rate = i2s_priv->mclk_rate;
+		share_i2s_priv->mclk_apll = i2s_priv->mclk_apll;
+	}
+
+	return 0;
+}
+
+static int mtk_dai_i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	struct mtk_afe_i2s_priv *i2s_priv;
+
+	if (dai->id >= MT8196_DAI_NUM || dai->id < 0)
+		return -EINVAL;
+
+	i2s_priv = afe_priv->dai_priv[dai->id];
+	if (!i2s_priv)
+		return -EINVAL;
+
+	dev_dbg(afe->dev, "dai->id: %d, fmt: 0x%x\n", dai->id, fmt);
+
+	switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) {
+	case SND_SOC_DAIFMT_I2S:
+		i2s_priv->format = MTK_DAI_ETDM_FORMAT_I2S;
+		break;
+	case SND_SOC_DAIFMT_LEFT_J:
+		i2s_priv->format = MTK_DAI_ETDM_FORMAT_LJ;
+		break;
+	case SND_SOC_DAIFMT_RIGHT_J:
+		i2s_priv->format = MTK_DAI_ETDM_FORMAT_RJ;
+		break;
+	case SND_SOC_DAIFMT_DSP_A:
+		i2s_priv->format = MTK_DAI_ETDM_FORMAT_DSPA;
+		break;
+	case SND_SOC_DAIFMT_DSP_B:
+		i2s_priv->format = MTK_DAI_ETDM_FORMAT_DSPB;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dai_ops mtk_dai_i2s_ops = {
+	.hw_params = mtk_dai_i2s_hw_params,
+	.set_sysclk = mtk_dai_i2s_set_sysclk,
+	.set_fmt = mtk_dai_i2s_set_fmt,
+};
+
+/* dai driver */
+#define MTK_ETDM_RATES (SNDRV_PCM_RATE_8000_384000)
+#define MTK_ETDM_FORMATS (SNDRV_PCM_FMTBIT_S8 |\
+			  SNDRV_PCM_FMTBIT_S16_LE |\
+			  SNDRV_PCM_FMTBIT_S24_LE |\
+			  SNDRV_PCM_FMTBIT_S32_LE)
+
+#define MT8196_I2S_DAI(_name, _id, max_ch, dir) \
+{ \
+	.name = #_name, \
+	.id = _id, \
+	.dir = { \
+		.stream_name = #_name, \
+		.channels_min = 1, \
+		.channels_max = max_ch, \
+		.rates = MTK_ETDM_RATES, \
+		.formats = MTK_ETDM_FORMATS, \
+	}, \
+	.ops = &mtk_dai_i2s_ops, \
+}
+
+static struct snd_soc_dai_driver mtk_dai_i2s_driver[] = {
+	/* capture */
+	MT8196_I2S_DAI(I2SIN0, MT8196_DAI_I2S_IN0, 2, capture),
+	MT8196_I2S_DAI(I2SIN1, MT8196_DAI_I2S_IN1, 2, capture),
+	MT8196_I2S_DAI(I2SIN2, MT8196_DAI_I2S_IN2, 2, capture),
+	MT8196_I2S_DAI(I2SIN3, MT8196_DAI_I2S_IN3, 2, capture),
+	MT8196_I2S_DAI(I2SIN4, MT8196_DAI_I2S_IN4, 8, capture),
+	MT8196_I2S_DAI(I2SIN6, MT8196_DAI_I2S_IN6, 2, capture),
+	MT8196_I2S_DAI(FMI2S_MASTER, MT8196_DAI_FM_I2S_MASTER, 2, capture),
+	/* playback */
+	MT8196_I2S_DAI(I2SOUT0, MT8196_DAI_I2S_OUT0, 2, playback),
+	MT8196_I2S_DAI(I2SOUT1, MT8196_DAI_I2S_OUT1, 2, playback),
+	MT8196_I2S_DAI(I2SOUT2, MT8196_DAI_I2S_OUT2, 2, playback),
+	MT8196_I2S_DAI(I2SOUT3, MT8196_DAI_I2S_OUT3, 2, playback),
+	MT8196_I2S_DAI(I2SOUT4, MT8196_DAI_I2S_OUT4, 8, playback),
+	MT8196_I2S_DAI(I2SOUT6, MT8196_DAI_I2S_OUT6, 2, playback),
+};
+
+static const struct mtk_afe_i2s_priv mt8196_i2s_priv[DAI_I2S_NUM] = {
+	[DAI_I2SIN0] = {
+		.id = MT8196_DAI_I2S_IN0,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sin0-share",
+		.share_i2s_id = -1,
+	},
+	[DAI_I2SIN1] = {
+		.id = MT8196_DAI_I2S_IN1,
+		.mclk_id = MT8196_I2SIN1_MCK,
+		.share_property_name = "i2sin1-share",
+		.share_i2s_id = -1,
+	},
+	[DAI_I2SIN2] = {
+		.id = MT8196_DAI_I2S_IN2,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sin2-share",
+		.share_i2s_id = -1,
+	},
+	[DAI_I2SIN3] = {
+		.id = MT8196_DAI_I2S_IN3,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sin3-share",
+		.share_i2s_id = -1,
+	},
+	[DAI_I2SIN4] = {
+		.id = MT8196_DAI_I2S_IN4,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sin4-share",
+		.share_i2s_id = -1,
+		.sync = 0,
+		.ip_mode = 0,
+	},
+	[DAI_I2SIN6] = {
+		.id = MT8196_DAI_I2S_IN6,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sout6-share",
+		.share_i2s_id = -1,
+	},
+	[DAI_I2SOUT0] = {
+		.id = MT8196_DAI_I2S_OUT0,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sout0-share",
+		.share_i2s_id = MT8196_DAI_I2S_IN0,
+	},
+	[DAI_I2SOUT1] = {
+		.id = MT8196_DAI_I2S_OUT1,
+		.mclk_id = MT8196_I2SIN1_MCK,
+		.share_property_name = "i2sout1-share",
+		.share_i2s_id = MT8196_DAI_I2S_IN1,
+	},
+	[DAI_I2SOUT2] = {
+		.id = MT8196_DAI_I2S_OUT2,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sout2-share",
+		.share_i2s_id = MT8196_DAI_I2S_IN2,
+	},
+	[DAI_I2SOUT3] = {
+		.id = MT8196_DAI_I2S_OUT3,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sout3-share",
+		.share_i2s_id = MT8196_DAI_I2S_IN3,
+	},
+	[DAI_I2SOUT4] = {
+		.id = MT8196_DAI_I2S_OUT4,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sout4-share",
+		.share_i2s_id = MT8196_DAI_I2S_IN4,
+		.sync = 0,
+	},
+	[DAI_I2SOUT6] = {
+		.id = MT8196_DAI_I2S_OUT6,
+		.mclk_id = MT8196_I2SIN0_MCK,
+		.share_property_name = "i2sout6-share",
+		.share_i2s_id = MT8196_DAI_I2S_IN6,
+	},
+	[DAI_FMI2S_MASTER] = {
+		.id = MT8196_DAI_FM_I2S_MASTER,
+		.mclk_id = MT8196_FMI2S_MCK,
+		.share_property_name = "fmi2s-share",
+		.share_i2s_id = -1,
+	},
+};
+
+static int mt8196_dai_i2s_get_share(struct mtk_base_afe *afe)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	const struct device_node *of_node = afe->dev->of_node;
+
+	for (int i = 0; i < DAI_I2S_NUM; i++) {
+		const char *of_str;
+		struct mtk_afe_i2s_priv *i2s_priv = afe_priv->dai_priv[mt8196_i2s_priv[i].id];
+		const char *property_name = mt8196_i2s_priv[i].share_property_name;
+
+		if (of_property_read_string(of_node, property_name, &of_str))
+			continue;
+
+		i2s_priv->share_i2s_id = get_i2s_id_by_name(afe, of_str);
+	}
+
+	return 0;
+}
+
+static int init_i2s_priv_data(struct mtk_base_afe *afe)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	struct mtk_afe_i2s_priv *i2s_priv;
+
+	for (int i = 0; i < DAI_I2S_NUM; i++) {
+		int id = mt8196_i2s_priv[i].id;
+		size_t size = sizeof(struct mtk_afe_i2s_priv);
+
+		if (id >= MT8196_DAI_NUM || id < 0)
+			return -EINVAL;
+
+		i2s_priv = devm_kzalloc(afe->dev, size, GFP_KERNEL);
+		if (!i2s_priv)
+			return -ENOMEM;
+
+		memcpy(i2s_priv, &mt8196_i2s_priv[i], size);
+
+		afe_priv->dai_priv[id] = i2s_priv;
+	}
+
+	return 0;
+}
+
+int mt8196_dai_i2s_register(struct mtk_base_afe *afe)
+{
+	struct mtk_base_afe_dai *dai;
+	int ret;
+
+	dai = devm_kzalloc(afe->dev, sizeof(*dai), GFP_KERNEL);
+	if (!dai)
+		return -ENOMEM;
+
+	dai->dai_drivers = mtk_dai_i2s_driver;
+	dai->num_dai_drivers = ARRAY_SIZE(mtk_dai_i2s_driver);
+
+	dai->controls = mtk_dai_i2s_controls;
+	dai->num_controls = ARRAY_SIZE(mtk_dai_i2s_controls);
+	dai->dapm_widgets = mtk_dai_i2s_widgets;
+	dai->num_dapm_widgets = ARRAY_SIZE(mtk_dai_i2s_widgets);
+	dai->dapm_routes = mtk_dai_i2s_routes;
+	dai->num_dapm_routes = ARRAY_SIZE(mtk_dai_i2s_routes);
+
+	/* set all dai i2s private data */
+	ret = init_i2s_priv_data(afe);
+	if (ret)
+		return ret;
+
+	/* parse share i2s */
+	ret = mt8196_dai_i2s_get_share(afe);
+	if (ret)
+		return ret;
+
+	list_add(&dai->list, &afe->sub_dais);
+
+	return 0;
+}
diff --git a/sound/soc/mediatek/mt8196/mt8196-dai-tdm.c b/sound/soc/mediatek/mt8196/mt8196-dai-tdm.c
new file mode 100644
index 0000000000000..b7aeee939d888
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-dai-tdm.c
@@ -0,0 +1,675 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  MediaTek ALSA SoC Audio DAI TDM Control
+ *
+ *  Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#include <linux/regmap.h>
+#include <sound/pcm_params.h>
+
+#include "mt8196-afe-clk.h"
+#include "mt8196-afe-common.h"
+#include "mt8196-interconnection.h"
+
+struct mtk_afe_tdm_priv {
+	int bck_id;
+	int bck_rate;
+
+	int mclk_id;
+	int mclk_multiple; /* according to sample rate */
+	int mclk_rate;
+	int mclk_apll;
+};
+
+enum {
+	TDM_WLEN_8_BIT,
+	TDM_WLEN_16_BIT,
+	TDM_WLEN_24_BIT,
+	TDM_WLEN_32_BIT,
+};
+
+enum {
+	TDM_CHANNEL_BCK_16,
+	TDM_CHANNEL_BCK_24,
+	TDM_CHANNEL_BCK_32,
+};
+
+enum {
+	TDM_CHANNEL_NUM_2,
+	TDM_CHANNEL_NUM_4,
+	TDM_CHANNEL_NUM_8,
+};
+
+enum  {
+	TDM_CH_START_O30_O31,
+	TDM_CH_START_O32_O33,
+	TDM_CH_START_O34_O35,
+	TDM_CH_START_O36_O37,
+	TDM_CH_ZERO,
+};
+
+enum {
+	DPTX_CHANNEL_2,
+	DPTX_CHANNEL_8,
+};
+
+enum {
+	DPTX_WLEN_24_BIT,
+	DPTX_WLEN_16_BIT,
+};
+
+#define DPTX_CH_EN_MASK_2CH (0x3)
+#define DPTX_CH_EN_MASK_4CH (0xf)
+#define DPTX_CH_EN_MASK_6CH (0x3f)
+#define DPTX_CH_EN_MASK_8CH (0xff)
+
+static unsigned int get_tdm_wlen(snd_pcm_format_t format)
+{
+	return (snd_pcm_format_physical_width(format) / 8) - 1;
+}
+
+static unsigned int get_tdm_channel_bck(snd_pcm_format_t format)
+{
+	return snd_pcm_format_physical_width(format) <= 16 ?
+	       TDM_CHANNEL_BCK_16 : TDM_CHANNEL_BCK_32;
+}
+
+static unsigned int get_tdm_lrck_width(snd_pcm_format_t format)
+{
+	return snd_pcm_format_physical_width(format) - 1;
+}
+
+static unsigned int get_tdm_ch(unsigned int ch)
+{
+	switch (ch) {
+	case 1:
+	case 2:
+		return TDM_CHANNEL_NUM_2;
+	case 3:
+	case 4:
+		return TDM_CHANNEL_NUM_4;
+	case 5:
+	case 6:
+	case 7:
+	case 8:
+	default:
+		return TDM_CHANNEL_NUM_8;
+	}
+}
+
+static unsigned int get_dptx_ch_enable_mask(struct device *dev, unsigned int ch)
+{
+	switch (ch) {
+	case 1:
+	case 2:
+		return DPTX_CH_EN_MASK_2CH;
+	case 3:
+	case 4:
+		return DPTX_CH_EN_MASK_4CH;
+	case 5:
+	case 6:
+		return DPTX_CH_EN_MASK_6CH;
+	case 7:
+	case 8:
+		return DPTX_CH_EN_MASK_8CH;
+	default:
+		dev_warn(dev, "invalid channel num, default use 2ch\n");
+		return DPTX_CH_EN_MASK_2CH;
+	}
+}
+
+static unsigned int get_dptx_ch(unsigned int ch)
+{
+	if (ch == 2)
+		return DPTX_CHANNEL_2;
+	else
+		return DPTX_CHANNEL_8;
+}
+
+static unsigned int get_dptx_wlen(snd_pcm_format_t format)
+{
+	return snd_pcm_format_physical_width(format) <= 16 ?
+	       DPTX_WLEN_16_BIT : DPTX_WLEN_24_BIT;
+}
+
+/* interconnection */
+enum {
+	HDMI_CONN_CH0,
+	HDMI_CONN_CH1,
+	HDMI_CONN_CH2,
+	HDMI_CONN_CH3,
+	HDMI_CONN_CH4,
+	HDMI_CONN_CH5,
+	HDMI_CONN_CH6,
+	HDMI_CONN_CH7,
+};
+
+static const char *const hdmi_conn_mux_map[] = {
+	"CH0", "CH1", "CH2", "CH3", "CH4", "CH5", "CH6", "CH7",
+};
+
+static int hdmi_conn_mux_map_value[] = {
+	HDMI_CONN_CH0, HDMI_CONN_CH1, HDMI_CONN_CH2, HDMI_CONN_CH3,
+	HDMI_CONN_CH4, HDMI_CONN_CH5, HDMI_CONN_CH6, HDMI_CONN_CH7,
+};
+
+static SOC_VALUE_ENUM_SINGLE_DECL(hdmi_ch0_mux_map_enum,
+				  AFE_HDMI_CONN0, HDMI_O_0_SFT, HDMI_O_0_MASK,
+				  hdmi_conn_mux_map, hdmi_conn_mux_map_value);
+
+static SOC_VALUE_ENUM_SINGLE_DECL(hdmi_ch1_mux_map_enum,
+				  AFE_HDMI_CONN0,
+				  HDMI_O_1_SFT,
+				  HDMI_O_1_MASK,
+				  hdmi_conn_mux_map,
+				  hdmi_conn_mux_map_value);
+
+static SOC_VALUE_ENUM_SINGLE_DECL(hdmi_ch2_mux_map_enum,
+				  AFE_HDMI_CONN0,
+				  HDMI_O_2_SFT,
+				  HDMI_O_2_MASK,
+				  hdmi_conn_mux_map,
+				  hdmi_conn_mux_map_value);
+
+static SOC_VALUE_ENUM_SINGLE_DECL(hdmi_ch3_mux_map_enum,
+				  AFE_HDMI_CONN0,
+				  HDMI_O_3_SFT,
+				  HDMI_O_3_MASK,
+				  hdmi_conn_mux_map,
+				  hdmi_conn_mux_map_value);
+
+static SOC_VALUE_ENUM_SINGLE_DECL(hdmi_ch4_mux_map_enum,
+				  AFE_HDMI_CONN0,
+				  HDMI_O_4_SFT,
+				  HDMI_O_4_MASK,
+				  hdmi_conn_mux_map,
+				  hdmi_conn_mux_map_value);
+
+static SOC_VALUE_ENUM_SINGLE_DECL(hdmi_ch5_mux_map_enum,
+				  AFE_HDMI_CONN0,
+				  HDMI_O_5_SFT,
+				  HDMI_O_5_MASK,
+				  hdmi_conn_mux_map,
+				  hdmi_conn_mux_map_value);
+
+static SOC_VALUE_ENUM_SINGLE_DECL(hdmi_ch6_mux_map_enum,
+				  AFE_HDMI_CONN0,
+				  HDMI_O_6_SFT,
+				  HDMI_O_6_MASK,
+				  hdmi_conn_mux_map,
+				  hdmi_conn_mux_map_value);
+
+static SOC_VALUE_ENUM_SINGLE_DECL(hdmi_ch7_mux_map_enum,
+				  AFE_HDMI_CONN0,
+				  HDMI_O_7_SFT,
+				  HDMI_O_7_MASK,
+				  hdmi_conn_mux_map,
+				  hdmi_conn_mux_map_value);
+
+static const struct snd_kcontrol_new mtk_dai_tdm_controls[] = {
+	SOC_ENUM("HDMI_CH0_MUX", hdmi_ch0_mux_map_enum),
+	SOC_ENUM("HDMI_CH1_MUX", hdmi_ch1_mux_map_enum),
+	SOC_ENUM("HDMI_CH2_MUX", hdmi_ch2_mux_map_enum),
+	SOC_ENUM("HDMI_CH3_MUX", hdmi_ch3_mux_map_enum),
+	SOC_ENUM("HDMI_CH4_MUX", hdmi_ch4_mux_map_enum),
+	SOC_ENUM("HDMI_CH5_MUX", hdmi_ch5_mux_map_enum),
+	SOC_ENUM("HDMI_CH6_MUX", hdmi_ch6_mux_map_enum),
+	SOC_ENUM("HDMI_CH7_MUX", hdmi_ch7_mux_map_enum),
+};
+
+static const char *const tdm_out_demux_texts[] = {
+	"NONE", "TDMOUT", "DPTXOUT",
+};
+
+static SOC_ENUM_SINGLE_DECL(tdm_out_demux_enum,
+					      SND_SOC_NOPM,
+					      0,
+					      tdm_out_demux_texts);
+static const struct snd_kcontrol_new tdm_out_demux_control =
+	SOC_DAPM_ENUM("TDM DEMUX ROUTE", tdm_out_demux_enum);
+
+enum {
+	SUPPLY_SEQ_APLL,
+	SUPPLY_SEQ_TDM_MCK_EN,
+	SUPPLY_SEQ_TDM_BCK_EN,
+	SUPPLY_SEQ_TDM_DPTX_MCK_EN,
+	SUPPLY_SEQ_TDM_DPTX_BCK_EN,
+	SUPPLY_SEQ_TDM_CG_EN,
+};
+
+static int get_tdm_id_by_name(const char *name)
+{
+	if (strstr(name, "DPTX"))
+		return MT8196_DAI_TDM_DPTX;
+	else
+		return MT8196_DAI_TDM;
+}
+
+static int mtk_tdm_bck_en_event(struct snd_soc_dapm_widget *w,
+				struct snd_kcontrol *kcontrol,
+				int event)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(w->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int dai_id = get_tdm_id_by_name(w->name);
+	struct mtk_afe_tdm_priv *tdm_priv = afe_priv->dai_priv[dai_id];
+
+	dev_dbg(cmpnt->dev, "name %s, event 0x%x, dai_id %d\n",
+		w->name, event, dai_id);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		mt8196_mck_enable(afe, tdm_priv->bck_id, tdm_priv->bck_rate);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		mt8196_mck_disable(afe, tdm_priv->bck_id);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static int mtk_tdm_mck_en_event(struct snd_soc_dapm_widget *w,
+				struct snd_kcontrol *kcontrol,
+				int event)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(w->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int dai_id = get_tdm_id_by_name(w->name);
+	struct mtk_afe_tdm_priv *tdm_priv = afe_priv->dai_priv[dai_id];
+
+	dev_dbg(cmpnt->dev, "name %s, event 0x%x, dai_id %d\n",
+		w->name, event, dai_id);
+
+	switch (event) {
+	case SND_SOC_DAPM_PRE_PMU:
+		mt8196_mck_enable(afe, tdm_priv->mclk_id, tdm_priv->mclk_rate);
+		break;
+	case SND_SOC_DAPM_POST_PMD:
+		tdm_priv->mclk_rate = 0;
+		mt8196_mck_disable(afe, tdm_priv->mclk_id);
+		break;
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_dapm_widget mtk_dai_tdm_widgets[] = {
+	SND_SOC_DAPM_DEMUX("TDM_DEMUX", SND_SOC_NOPM, 0, 0,
+			   &tdm_out_demux_control),
+
+	SND_SOC_DAPM_SUPPLY_S("TDM_BCK", SUPPLY_SEQ_TDM_BCK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_tdm_bck_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+
+	SND_SOC_DAPM_SUPPLY_S("TDM_MCK", SUPPLY_SEQ_TDM_MCK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_tdm_mck_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+
+	SND_SOC_DAPM_SUPPLY_S("TDM_DPTX_BCK", SUPPLY_SEQ_TDM_DPTX_BCK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_tdm_bck_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+
+	SND_SOC_DAPM_SUPPLY_S("TDM_DPTX_MCK", SUPPLY_SEQ_TDM_DPTX_MCK_EN,
+			      SND_SOC_NOPM, 0, 0,
+			      mtk_tdm_mck_en_event,
+			      SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD),
+
+	/* cg */
+	SND_SOC_DAPM_SUPPLY_S("TDM_CG", SUPPLY_SEQ_TDM_CG_EN,
+			      AUDIO_TOP_CON2, PDN_TDM_OUT_SFT, 1,
+			      NULL, 0),
+};
+
+static int mtk_afe_tdm_apll_connect(struct snd_soc_dapm_widget *source,
+				    struct snd_soc_dapm_widget *sink)
+{
+	struct snd_soc_component *cmpnt = snd_soc_dapm_to_component(sink->dapm);
+	struct mtk_base_afe *afe = snd_soc_component_get_drvdata(cmpnt);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int dai_id = get_tdm_id_by_name(sink->name);
+	struct mtk_afe_tdm_priv *tdm_priv = afe_priv->dai_priv[dai_id];
+	int cur_apll;
+
+	/* which apll */
+	cur_apll = mt8196_get_apll_by_name(afe, source->name);
+
+	return (tdm_priv->mclk_apll == cur_apll) ? 1 : 0;
+}
+
+static const struct snd_soc_dapm_route mtk_dai_tdm_routes[] = {
+	{"TDM_DEMUX", NULL, "HDMI"},
+
+	{"TDM", "TDMOUT", "TDM_DEMUX"},
+	{"TDM", NULL, "TDM_BCK"},
+	{"TDM", NULL, "TDM_CG"},
+
+	{"TDM_DPTX", "DPTXOUT", "TDM_DEMUX"},
+	{"TDM_DPTX", NULL, "TDM_DPTX_BCK"},
+	{"TDM_DPTX", NULL, "TDM_CG"},
+
+	{"TDM_BCK", NULL, "TDM_MCK"},
+	{"TDM_DPTX_BCK", NULL, "TDM_DPTX_MCK"},
+	{"TDM_MCK", NULL, APLL1_W_NAME, mtk_afe_tdm_apll_connect},
+	{"TDM_MCK", NULL, APLL2_W_NAME, mtk_afe_tdm_apll_connect},
+	{"TDM_DPTX_MCK", NULL, APLL1_W_NAME, mtk_afe_tdm_apll_connect},
+	{"TDM_DPTX_MCK", NULL, APLL2_W_NAME, mtk_afe_tdm_apll_connect},
+};
+
+/* dai ops */
+static int mtk_dai_tdm_cal_mclk(struct mtk_base_afe *afe,
+				struct mtk_afe_tdm_priv *tdm_priv,
+				int freq)
+{
+	int apll;
+	int apll_rate;
+
+	apll = mt8196_get_apll_by_rate(afe, freq);
+	apll_rate = mt8196_get_apll_rate(afe, apll);
+
+	if (freq > apll_rate)
+		return -EINVAL;
+
+	if (apll_rate % freq != 0)
+		return -EINVAL;
+
+	tdm_priv->mclk_rate = freq;
+	tdm_priv->mclk_apll = apll;
+
+	return 0;
+}
+
+static int mtk_dai_tdm_hw_params(struct snd_pcm_substream *substream,
+				 struct snd_pcm_hw_params *params,
+				 struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	int tdm_id = dai->id;
+	struct mtk_afe_tdm_priv *tdm_priv;
+	unsigned int rate = params_rate(params);
+	unsigned int channels = params_channels(params);
+	snd_pcm_format_t format = params_format(params);
+	unsigned int tdm_con = 0;
+
+	if (tdm_id >= MT8196_DAI_NUM || tdm_id < 0)
+		return -EINVAL;
+
+	tdm_priv = afe_priv->dai_priv[tdm_id];
+
+	if (!tdm_priv)
+		return -EINVAL;
+
+	/* calculate mclk_rate, if not set explicitly */
+	if (!tdm_priv->mclk_rate) {
+		tdm_priv->mclk_rate = rate * tdm_priv->mclk_multiple;
+		mtk_dai_tdm_cal_mclk(afe,
+				     tdm_priv,
+				     tdm_priv->mclk_rate);
+	}
+
+	/* calculate bck */
+	tdm_priv->bck_rate = rate *
+			     channels *
+			     snd_pcm_format_physical_width(format);
+
+	if (tdm_priv->bck_rate > tdm_priv->mclk_rate)
+		return -EINVAL;
+
+	if (tdm_priv->mclk_rate % tdm_priv->bck_rate != 0)
+		return -EINVAL;
+
+	dev_dbg(afe->dev, "id %d, rate %d, ch %d, fmt %d, mclk %d, bck %d\n",
+		tdm_id, rate, channels, format, tdm_priv->mclk_rate, tdm_priv->bck_rate);
+
+	/* set tdm */
+	tdm_con = 0 << BCK_INVERSE_SFT;
+	tdm_con |= 0 << LRCK_INVERSE_SFT;
+	tdm_con |= 0 << DELAY_DATA_SFT;
+	tdm_con |= 1 << LEFT_ALIGN_SFT;
+	tdm_con |= get_tdm_wlen(format) << WLEN_SFT;
+	tdm_con |= get_tdm_ch(channels) << CHANNEL_NUM_SFT;
+	tdm_con |= get_tdm_channel_bck(format) << CHANNEL_BCK_CYCLES_SFT;
+	tdm_con |= get_tdm_lrck_width(format) << LRCK_TDM_WIDTH_SFT;
+	regmap_write(afe->regmap, AFE_TDM_CON1, tdm_con);
+
+	/* set dptx */
+	if (tdm_id == MT8196_DAI_TDM_DPTX) {
+		regmap_update_bits(afe->regmap, AFE_DPTX_CON,
+				   DPTX_CHANNEL_ENABLE_MASK_SFT,
+				   get_dptx_ch_enable_mask(afe->dev, channels) <<
+				   DPTX_CHANNEL_ENABLE_SFT);
+		regmap_update_bits(afe->regmap, AFE_DPTX_CON,
+				   DPTX_CHANNEL_NUMBER_MASK_SFT,
+				   get_dptx_ch(channels) <<
+				   DPTX_CHANNEL_NUMBER_SFT);
+		regmap_update_bits(afe->regmap, AFE_DPTX_CON,
+				   DPTX_16BIT_MASK_SFT,
+				   get_dptx_wlen(format) << DPTX_16BIT_SFT);
+	}
+	switch (channels) {
+	case 1:
+	case 2:
+		tdm_con = TDM_CH_START_O30_O31 << ST_CH_PAIR_SOUT0_SFT;
+		tdm_con |= TDM_CH_ZERO << ST_CH_PAIR_SOUT1_SFT;
+		tdm_con |= TDM_CH_ZERO << ST_CH_PAIR_SOUT2_SFT;
+		tdm_con |= TDM_CH_ZERO << ST_CH_PAIR_SOUT3_SFT;
+		break;
+	case 3:
+	case 4:
+		tdm_con = TDM_CH_START_O30_O31 << ST_CH_PAIR_SOUT0_SFT;
+		tdm_con |= TDM_CH_START_O32_O33 << ST_CH_PAIR_SOUT1_SFT;
+		tdm_con |= TDM_CH_ZERO << ST_CH_PAIR_SOUT2_SFT;
+		tdm_con |= TDM_CH_ZERO << ST_CH_PAIR_SOUT3_SFT;
+		break;
+	case 5:
+	case 6:
+		tdm_con = TDM_CH_START_O30_O31 << ST_CH_PAIR_SOUT0_SFT;
+		tdm_con |= TDM_CH_START_O32_O33 << ST_CH_PAIR_SOUT1_SFT;
+		tdm_con |= TDM_CH_START_O34_O35 << ST_CH_PAIR_SOUT2_SFT;
+		tdm_con |= TDM_CH_ZERO << ST_CH_PAIR_SOUT3_SFT;
+		break;
+	case 7:
+	case 8:
+		tdm_con = TDM_CH_START_O30_O31 << ST_CH_PAIR_SOUT0_SFT;
+		tdm_con |= TDM_CH_START_O32_O33 << ST_CH_PAIR_SOUT1_SFT;
+		tdm_con |= TDM_CH_START_O34_O35 << ST_CH_PAIR_SOUT2_SFT;
+		tdm_con |= TDM_CH_START_O36_O37 << ST_CH_PAIR_SOUT3_SFT;
+		break;
+	default:
+		tdm_con = 0;
+	}
+
+	regmap_write(afe->regmap, AFE_TDM_CON2, tdm_con);
+	regmap_update_bits(afe->regmap, AFE_HDMI_OUT_CON0,
+			   HDMI_CH_NUM_MASK_SFT,
+			   channels << HDMI_CH_NUM_SFT);
+
+	return 0;
+}
+
+static int mtk_dai_tdm_trigger(struct snd_pcm_substream *substream,
+			       int cmd,
+			       struct snd_soc_dai *dai)
+{
+	struct mtk_base_afe *afe = snd_soc_dai_get_drvdata(dai);
+	int tdm_id = dai->id;
+
+	dev_dbg(afe->dev, "cmd %d, tdm_id %d\n", cmd, tdm_id);
+
+	switch (cmd) {
+	case SNDRV_PCM_TRIGGER_START:
+	case SNDRV_PCM_TRIGGER_RESUME:
+		/* enable Out control */
+		regmap_update_bits(afe->regmap, AFE_HDMI_OUT_CON0,
+				   HDMI_OUT_ON_MASK_SFT,
+				   0x1 << HDMI_OUT_ON_SFT);
+
+		/* enable dptx */
+		if (tdm_id == MT8196_DAI_TDM_DPTX) {
+			regmap_update_bits(afe->regmap, AFE_DPTX_CON,
+					   DPTX_ON_MASK_SFT, 0x1 <<
+					   DPTX_ON_SFT);
+		}
+
+		/* enable tdm */
+		regmap_update_bits(afe->regmap, AFE_TDM_CON1,
+				   TDM_EN_MASK_SFT, 0x1 << TDM_EN_SFT);
+		break;
+	case SNDRV_PCM_TRIGGER_STOP:
+	case SNDRV_PCM_TRIGGER_SUSPEND:
+		/* disable tdm */
+		regmap_update_bits(afe->regmap, AFE_TDM_CON1,
+				   TDM_EN_MASK_SFT, 0);
+
+		/* disable dptx */
+		if (tdm_id == MT8196_DAI_TDM_DPTX) {
+			regmap_update_bits(afe->regmap, AFE_DPTX_CON,
+					   DPTX_ON_MASK_SFT, 0);
+		}
+
+		/* disable Out control */
+		regmap_update_bits(afe->regmap, AFE_HDMI_OUT_CON0,
+				   HDMI_OUT_ON_MASK_SFT, 0);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static int mtk_dai_tdm_set_sysclk(struct snd_soc_dai *dai,
+				  int clk_id, unsigned int freq, int dir)
+{
+	struct mtk_base_afe *afe = dev_get_drvdata(dai->dev);
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	struct mtk_afe_tdm_priv *tdm_priv;
+
+	if (dai->id >= MT8196_DAI_NUM || dai->id < 0)
+		return -EINVAL;
+
+	tdm_priv = afe_priv->dai_priv[dai->id];
+
+	if (!tdm_priv)
+		return -EINVAL;
+
+	if (dir != SND_SOC_CLOCK_OUT)
+		return -EINVAL;
+
+	dev_dbg(afe->dev, "freq %d\n", freq);
+
+	return mtk_dai_tdm_cal_mclk(afe, tdm_priv, freq);
+}
+
+static const struct snd_soc_dai_ops mtk_dai_tdm_ops = {
+	.hw_params = mtk_dai_tdm_hw_params,
+	.trigger = mtk_dai_tdm_trigger,
+	.set_sysclk = mtk_dai_tdm_set_sysclk,
+};
+
+/* dai driver */
+#define MTK_TDM_RATES (SNDRV_PCM_RATE_8000_48000 |\
+		       SNDRV_PCM_RATE_88200 |\
+		       SNDRV_PCM_RATE_96000 |\
+		       SNDRV_PCM_RATE_176400 |\
+		       SNDRV_PCM_RATE_192000)
+
+#define MTK_TDM_FORMATS (SNDRV_PCM_FMTBIT_S16_LE |\
+			 SNDRV_PCM_FMTBIT_S24_LE |\
+			 SNDRV_PCM_FMTBIT_S32_LE)
+
+static struct snd_soc_dai_driver mtk_dai_tdm_driver[] = {
+	{
+		.name = "TDM",
+		.id = MT8196_DAI_TDM,
+		.playback = {
+			.stream_name = "TDM",
+			.channels_min = 2,
+			.channels_max = 8,
+			.rates = MTK_TDM_RATES,
+			.formats = MTK_TDM_FORMATS,
+		},
+		.ops = &mtk_dai_tdm_ops,
+	},
+	{
+		.name = "TDM_DPTX",
+		.id = MT8196_DAI_TDM_DPTX,
+		.playback = {
+			.stream_name = "TDM_DPTX",
+			.channels_min = 2,
+			.channels_max = 8,
+			.rates = MTK_TDM_RATES,
+			.formats = MTK_TDM_FORMATS,
+		},
+		.ops = &mtk_dai_tdm_ops,
+	},
+};
+
+static struct mtk_afe_tdm_priv *init_tdm_priv_data(struct mtk_base_afe *afe,
+						   int id)
+{
+	struct mtk_afe_tdm_priv *tdm_priv;
+
+	tdm_priv = devm_kzalloc(afe->dev, sizeof(struct mtk_afe_tdm_priv),
+				GFP_KERNEL);
+	if (!tdm_priv)
+		return NULL;
+
+	if (id == MT8196_DAI_TDM_DPTX)
+		tdm_priv->mclk_multiple = 256;
+	else
+		tdm_priv->mclk_multiple = 128;
+
+	tdm_priv->bck_id = MT8196_TDMOUT_BCK;
+	tdm_priv->mclk_id = MT8196_TDMOUT_MCK;
+
+	return tdm_priv;
+}
+
+int mt8196_dai_tdm_register(struct mtk_base_afe *afe)
+{
+	struct mt8196_afe_private *afe_priv = afe->platform_priv;
+	struct mtk_afe_tdm_priv *tdm_priv, *tdm_dptx_priv;
+	struct mtk_base_afe_dai *dai;
+
+	dai = devm_kzalloc(afe->dev, sizeof(*dai), GFP_KERNEL);
+	if (!dai)
+		return -ENOMEM;
+
+	dai->dai_drivers = mtk_dai_tdm_driver;
+	dai->num_dai_drivers = ARRAY_SIZE(mtk_dai_tdm_driver);
+	dai->controls = mtk_dai_tdm_controls;
+	dai->num_controls = ARRAY_SIZE(mtk_dai_tdm_controls);
+	dai->dapm_widgets = mtk_dai_tdm_widgets;
+	dai->num_dapm_widgets = ARRAY_SIZE(mtk_dai_tdm_widgets);
+	dai->dapm_routes = mtk_dai_tdm_routes;
+	dai->num_dapm_routes = ARRAY_SIZE(mtk_dai_tdm_routes);
+
+	tdm_priv = init_tdm_priv_data(afe, MT8196_DAI_TDM);
+	if (!tdm_priv)
+		return -ENOMEM;
+
+	tdm_dptx_priv = init_tdm_priv_data(afe, MT8196_DAI_TDM_DPTX);
+	if (!tdm_dptx_priv)
+		return -ENOMEM;
+
+	list_add(&dai->list, &afe->sub_dais);
+
+	afe_priv->dai_priv[MT8196_DAI_TDM] = tdm_priv;
+	afe_priv->dai_priv[MT8196_DAI_TDM_DPTX] = tdm_dptx_priv;
+
+	return 0;
+}
+
diff --git a/sound/soc/mediatek/mt8196/mt8196-interconnection.h b/sound/soc/mediatek/mt8196/mt8196-interconnection.h
new file mode 100644
index 0000000000000..acb91da3b4dbb
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-interconnection.h
@@ -0,0 +1,121 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Mediatek MT8196 audio driver interconnection definition
+ *
+ *  Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#ifndef _MT8196_INTERCONNECTION_H_
+#define _MT8196_INTERCONNECTION_H_
+
+/* in port define */
+
+#define I_CONNSYS_I2S_CH1 0
+#define I_CONNSYS_I2S_CH2 1
+#define I_GAIN0_OUT_CH1 6
+#define I_GAIN0_OUT_CH2 7
+#define I_GAIN1_OUT_CH1 8
+#define I_GAIN1_OUT_CH2 9
+#define I_GAIN2_OUT_CH1 10
+#define I_GAIN2_OUT_CH2 11
+#define I_GAIN3_OUT_CH1 12
+#define I_GAIN3_OUT_CH2 13
+#define I_STF_CH1 14
+#define I_ADDA_UL_CH1 16
+#define I_ADDA_UL_CH2 17
+#define I_ADDA_UL_CH3 18
+#define I_ADDA_UL_CH4 19
+#define I_UL_PROX_CH1 20
+#define I_UL_PROX_CH2 21
+#define I_ADDA_UL_CH5 24
+#define I_ADDA_UL_CH6 25
+#define I_DMIC0_CH1 28
+#define I_DMIC0_CH2 29
+#define I_DMIC1_CH1 30
+#define I_DMIC1_CH2 31
+
+/* in port define >= 32 */
+#define I_32_OFFSET 32
+#define I_DL0_CH1 (32 - I_32_OFFSET)
+#define I_DL0_CH2 (33 - I_32_OFFSET)
+#define I_DL1_CH1 (34 - I_32_OFFSET)
+#define I_DL1_CH2 (35 - I_32_OFFSET)
+#define I_DL2_CH1 (36 - I_32_OFFSET)
+#define I_DL2_CH2 (37 - I_32_OFFSET)
+#define I_DL3_CH1 (38 - I_32_OFFSET)
+#define I_DL3_CH2 (39 - I_32_OFFSET)
+#define I_DL4_CH1 (40 - I_32_OFFSET)
+#define I_DL4_CH2 (41 - I_32_OFFSET)
+#define I_DL5_CH1 (42 - I_32_OFFSET)
+#define I_DL5_CH2 (43 - I_32_OFFSET)
+#define I_DL6_CH1 (44 - I_32_OFFSET)
+#define I_DL6_CH2 (45 - I_32_OFFSET)
+#define I_DL7_CH1 (46 - I_32_OFFSET)
+#define I_DL7_CH2 (47 - I_32_OFFSET)
+#define I_DL8_CH1 (48 - I_32_OFFSET)
+#define I_DL8_CH2 (49 - I_32_OFFSET)
+#define I_DL_4CH_CH1 (50 - I_32_OFFSET)
+#define I_DL_4CH_CH2 (51 - I_32_OFFSET)
+#define I_DL_4CH_CH3 (52 - I_32_OFFSET)
+#define I_DL_4CH_CH4 (53 - I_32_OFFSET)
+#define I_DL_24CH_CH1 (54 - I_32_OFFSET)
+#define I_DL_24CH_CH2 (55 - I_32_OFFSET)
+#define I_DL_24CH_CH3 (56 - I_32_OFFSET)
+#define I_DL_24CH_CH4 (57 - I_32_OFFSET)
+#define I_DL_24CH_CH5 (58 - I_32_OFFSET)
+#define I_DL_24CH_CH6 (59 - I_32_OFFSET)
+#define I_DL_24CH_CH7 (60 - I_32_OFFSET)
+#define I_DL_24CH_CH8 (61 - I_32_OFFSET)
+
+/* in port define >= 64 */
+#define I_64_OFFSET 64
+#define I_DL23_CH1 (78 - I_64_OFFSET)
+#define I_DL23_CH2 (79 - I_64_OFFSET)
+#define I_DL24_CH1 (80 - I_64_OFFSET)
+#define I_DL24_CH2 (81 - I_64_OFFSET)
+#define I_DL25_CH1 (82 - I_64_OFFSET)
+#define I_DL25_CH2 (83 - I_64_OFFSET)
+#define I_DL26_CH1 (84 - I_64_OFFSET)
+#define I_DL26_CH2 (85 - I_64_OFFSET)
+
+/* in port define >= 128 */
+#define I_128_OFFSET 128
+#define I_PCM_0_CAP_CH1 (130 - I_128_OFFSET)
+#define I_PCM_0_CAP_CH2 (131 - I_128_OFFSET)
+#define I_PCM_1_CAP_CH1 (132 - I_128_OFFSET)
+#define I_PCM_1_CAP_CH2 (133 - I_128_OFFSET)
+#define I_I2SIN0_CH1 (134 - I_128_OFFSET)
+#define I_I2SIN0_CH2 (135 - I_128_OFFSET)
+#define I_I2SIN1_CH1 (136 - I_128_OFFSET)
+#define I_I2SIN1_CH2 (137 - I_128_OFFSET)
+#define I_I2SIN2_CH1 (138 - I_128_OFFSET)
+#define I_I2SIN2_CH2 (139 - I_128_OFFSET)
+#define I_I2SIN3_CH1 (140 - I_128_OFFSET)
+#define I_I2SIN3_CH2 (141 - I_128_OFFSET)
+#define I_I2SIN4_CH1 (142 - I_128_OFFSET)
+#define I_I2SIN4_CH2 (143 - I_128_OFFSET)
+#define I_I2SIN4_CH3 (144 - I_128_OFFSET)
+#define I_I2SIN4_CH4 (145 - I_128_OFFSET)
+#define I_I2SIN4_CH5 (146 - I_128_OFFSET)
+#define I_I2SIN4_CH6 (147 - I_128_OFFSET)
+#define I_I2SIN4_CH7 (148 - I_128_OFFSET)
+#define I_I2SIN4_CH8 (149 - I_128_OFFSET)
+
+/* in port define >= 160 */
+#define I_160_OFFSET 160
+#define I_I2SIN6_CH1 (166 - I_160_OFFSET)
+#define I_I2SIN6_CH2 (167 - I_160_OFFSET)
+
+/* in port define >= 192 */
+#define I_192_OFFSET 192
+#define I_SRC_0_OUT_CH1 (198 - I_192_OFFSET)
+#define I_SRC_0_OUT_CH2 (199 - I_192_OFFSET)
+#define I_SRC_1_OUT_CH1 (200 - I_192_OFFSET)
+#define I_SRC_1_OUT_CH2 (201 - I_192_OFFSET)
+#define I_SRC_2_OUT_CH1 (202 - I_192_OFFSET)
+#define I_SRC_2_OUT_CH2 (203 - I_192_OFFSET)
+#define I_SRC_3_OUT_CH1 (204 - I_192_OFFSET)
+#define I_SRC_3_OUT_CH2 (205 - I_192_OFFSET)
+
+#endif
diff --git a/sound/soc/mediatek/mt8196/mt8196-nau8825.c b/sound/soc/mediatek/mt8196/mt8196-nau8825.c
new file mode 100644
index 0000000000000..c9424786c53df
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-nau8825.c
@@ -0,0 +1,870 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ *  mt8196-nau8825.c  --  mt8196 nau8825 ALSA SoC machine driver
+ *
+ *  Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#include <linux/input.h>
+#include <linux/module.h>
+#include <linux/of_device.h>
+#include <linux/pm_runtime.h>
+
+#include <sound/soc.h>
+#include <sound/jack.h>
+#include <sound/pcm_params.h>
+
+#include "mt8196-afe-common.h"
+
+#include "../../codecs/nau8825.h"
+#include "../../codecs/rt5682s.h"
+
+#include "../common/mtk-soc-card.h"
+#include "../common/mtk-dsp-sof-common.h"
+#include "../common/mtk-soundcard-driver.h"
+#include "../common/mtk-afe-platform-driver.h"
+
+#define NAU8825_HS_PRESENT	BIT(0)
+#define RT5682S_HS_PRESENT	BIT(1)
+#define RT5650_HS_PRESENT	BIT(2)
+
+/*
+ * Nau88l25
+ */
+#define NAU8825_CODEC_DAI  "nau8825-hifi"
+
+/*
+ * Rt5682s
+ */
+#define RT5682S_CODEC_DAI     "rt5682s-aif1"
+
+/*
+ * Rt5650
+ */
+#define RT5650_CODEC_DAI     "rt5645-aif1"
+
+#define SOF_DMA_DL1 "SOF_DMA_DL1"
+#define SOF_DMA_DL_24CH "SOF_DMA_DL_24CH"
+#define SOF_DMA_UL0 "SOF_DMA_UL0"
+#define SOF_DMA_UL1 "SOF_DMA_UL1"
+#define SOF_DMA_UL2 "SOF_DMA_UL2"
+
+enum mt8196_jacks {
+	MT8196_JACK_HEADSET,
+	MT8196_JACK_DP,
+	MT8196_JACK_HDMI,
+	MT8196_JACK_MAX,
+};
+
+static struct snd_soc_jack_pin mt8196_dp_jack_pins[] = {
+	{
+		.pin = "DP",
+		.mask = SND_JACK_AVOUT,
+	},
+};
+
+static struct snd_soc_jack_pin mt8196_hdmi_jack_pins[] = {
+	{
+		.pin = "HDMI",
+		.mask = SND_JACK_AVOUT,
+	},
+};
+
+static struct snd_soc_jack_pin nau8825_jack_pins[] = {
+	{
+		.pin    = "Headphone Jack",
+		.mask   = SND_JACK_HEADPHONE,
+	},
+	{
+		.pin    = "Headset Mic",
+		.mask   = SND_JACK_MICROPHONE,
+	},
+};
+
+static const struct snd_kcontrol_new mt8196_dumb_spk_controls[] = {
+	SOC_DAPM_PIN_SWITCH("Ext Spk"),
+};
+
+static const struct snd_soc_dapm_widget mt8196_dumb_spk_widgets[] = {
+	SND_SOC_DAPM_SPK("Ext Spk", NULL),
+};
+
+static const struct snd_soc_dapm_widget mt8196_nau8825_widgets[] = {
+	SND_SOC_DAPM_HP("Headphone Jack", NULL),
+	SND_SOC_DAPM_MIC("Headset Mic", NULL),
+	SND_SOC_DAPM_SPK("Ext Spk", NULL),
+	SND_SOC_DAPM_SINK("DP"),
+};
+
+static const struct snd_kcontrol_new mt8196_nau8825_controls[] = {
+	SOC_DAPM_PIN_SWITCH("Headphone Jack"),
+	SOC_DAPM_PIN_SWITCH("Headset Mic"),
+};
+
+#define EXT_SPK_AMP_W_NAME "Ext_Speaker_Amp"
+
+static struct snd_soc_card mt8196_nau8825_soc_card;
+
+static const struct snd_soc_dapm_widget mt8196_nau8825_card_widgets[] = {
+	/* SOF Uplink */
+	SND_SOC_DAPM_MIXER("SOF_DMA_UL0", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("SOF_DMA_UL1", SND_SOC_NOPM, 0, 0, NULL, 0),
+	SND_SOC_DAPM_MIXER("SOF_DMA_UL2", SND_SOC_NOPM, 0, 0, NULL, 0),
+
+	/*
+	 * SOF Downlink
+	 * the widgets on the machine driver cannot use the parameter with kcontrol
+	 * because the widget domain is its platform driver. so sof downlink route
+	 * is written in the i2s dai driver.
+	 */
+};
+
+static const struct snd_soc_dapm_route mt8196_nau8825_card_routes[] = {
+	/* SOF Uplink */
+	{"SOF_DMA_UL0", NULL, "UL0_CH1"},
+	{"SOF_DMA_UL0", NULL, "UL0_CH2"},
+	/* SOF Uplink */
+	{"SOF_DMA_UL1", NULL, "UL1_CH1"},
+	{"SOF_DMA_UL1", NULL, "UL1_CH2"},
+	/* SOF Uplink */
+	{"SOF_DMA_UL2", NULL, "UL2_CH1"},
+	{"SOF_DMA_UL2", NULL, "UL2_CH2"},
+};
+
+static const struct snd_kcontrol_new mt8196_nau8825_card_controls[] = {
+	SOC_DAPM_PIN_SWITCH(EXT_SPK_AMP_W_NAME),
+};
+
+/*
+ * define mtk_spk_i2s_mck node in dts when need mclk,
+ * BE i2s need assign snd_soc_ops = mt8196_nau8825_i2s_ops
+ */
+static int mt8196_nau8825_i2s_hw_params(struct snd_pcm_substream *substream,
+					struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	unsigned int rate = params_rate(params);
+	unsigned int mclk_fs_ratio = 128;
+	unsigned int mclk_fs = rate * mclk_fs_ratio;
+	struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0);
+
+	return snd_soc_dai_set_sysclk(cpu_dai,
+				      0, mclk_fs, SND_SOC_CLOCK_OUT);
+}
+
+static const struct snd_soc_ops mt8196_nau8825_i2s_ops = {
+	.hw_params = mt8196_nau8825_i2s_hw_params,
+};
+
+static int mt8196_dptx_hw_params(struct snd_pcm_substream *substream,
+				 struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	unsigned int rate = params_rate(params);
+	unsigned int mclk_fs_ratio = 256;
+	unsigned int mclk_fs = rate * mclk_fs_ratio;
+	struct snd_soc_dai *dai = snd_soc_rtd_to_cpu(rtd, 0);
+
+	return snd_soc_dai_set_sysclk(dai, 0, mclk_fs, SND_SOC_CLOCK_OUT);
+}
+
+static const struct snd_soc_ops mt8196_dptx_ops = {
+	.hw_params = mt8196_dptx_hw_params,
+};
+
+static int mt8196_hw_params_fixup(struct snd_soc_pcm_runtime *rtd,
+				  struct snd_pcm_hw_params *params)
+{
+	dev_info(rtd->dev, "fix format to 32bit\n");
+
+	/* fix BE i2s format to 32bit, clean param mask first */
+	snd_mask_reset_range(hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT),
+			     0, (__force unsigned int)SNDRV_PCM_FORMAT_LAST);
+
+	params_set_format(params, SNDRV_PCM_FORMAT_S32_LE);
+	return 0;
+}
+
+static int mt8196_sof_be_hw_params(struct snd_pcm_substream *substream,
+				   struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
+	struct snd_soc_component *cmpnt_afe = NULL;
+	struct snd_soc_pcm_runtime *runtime;
+
+	/* find afe component */
+	for_each_card_rtds(rtd->card, runtime) {
+		cmpnt_afe = snd_soc_rtdcom_lookup(runtime, AFE_PCM_NAME);
+		if (cmpnt_afe) {
+			dev_info(rtd->dev, "component->name: %s\n", cmpnt_afe->name);
+			break;
+		}
+	}
+
+	if (cmpnt_afe && !pm_runtime_active(cmpnt_afe->dev)) {
+		dev_err(rtd->dev, "afe pm runtime is not active!!\n");
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_ops mt8196_sof_be_ops = {
+	.hw_params = mt8196_sof_be_hw_params,
+};
+
+static const struct sof_conn_stream g_sof_conn_streams[] = {
+	{
+		.sof_link = "AFE_SOF_DL1",
+		.sof_dma = SOF_DMA_DL1,
+		.stream_dir = SNDRV_PCM_STREAM_PLAYBACK
+	},
+	{
+		.sof_link = "AFE_SOF_DL_24CH",
+		.sof_dma = SOF_DMA_DL_24CH,
+		.stream_dir = SNDRV_PCM_STREAM_PLAYBACK
+	},
+	{
+		.sof_link = "AFE_SOF_UL0",
+		.sof_dma = SOF_DMA_UL0,
+		.stream_dir = SNDRV_PCM_STREAM_CAPTURE
+	},
+	{
+		.sof_link = "AFE_SOF_UL1",
+		.sof_dma = SOF_DMA_UL1,
+		.stream_dir = SNDRV_PCM_STREAM_CAPTURE
+	},
+	{
+		.sof_link = "AFE_SOF_UL2",
+		.sof_dma = SOF_DMA_UL2,
+		.stream_dir = SNDRV_PCM_STREAM_CAPTURE
+	},
+};
+
+/* FE */
+SND_SOC_DAILINK_DEFS(playback1,
+		     DAILINK_COMP_ARRAY(COMP_CPU("DL1")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(playback_24ch,
+		     DAILINK_COMP_ARRAY(COMP_CPU("DL_24CH")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(capture0,
+		     DAILINK_COMP_ARRAY(COMP_CPU("UL0")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(capture1,
+		     DAILINK_COMP_ARRAY(COMP_CPU("UL1")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(capture2,
+		     DAILINK_COMP_ARRAY(COMP_CPU("UL2")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(playback_hdmi,
+		     DAILINK_COMP_ARRAY(COMP_CPU("HDMI")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(playback2,
+		     DAILINK_COMP_ARRAY(COMP_CPU("DL2")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(capture_cm0,
+		     DAILINK_COMP_ARRAY(COMP_CPU("UL_CM0")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+/* BE */
+SND_SOC_DAILINK_DEFS(ap_dmic,
+		     DAILINK_COMP_ARRAY(COMP_CPU("AP_DMIC")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(ap_dmic_ch34,
+		     DAILINK_COMP_ARRAY(COMP_CPU("AP_DMIC_CH34")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(ap_dmic_multich,
+		     DAILINK_COMP_ARRAY(COMP_CPU("AP_DMIC_MULTICH")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(i2sin6,
+		     DAILINK_COMP_ARRAY(COMP_CPU("I2SIN6")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(i2sout3,
+		     DAILINK_COMP_ARRAY(COMP_CPU("I2SOUT3")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(i2sout4,
+		     DAILINK_COMP_ARRAY(COMP_CPU("I2SOUT4")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(i2sout6,
+		     DAILINK_COMP_ARRAY(COMP_CPU("I2SOUT6")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(tdm_dptx,
+		     DAILINK_COMP_ARRAY(COMP_CPU("TDM_DPTX")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(AFE_SOF_DL_24CH,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL_24CH")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(AFE_SOF_DL1,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_DL1")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(AFE_SOF_UL0,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL0")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(AFE_SOF_UL1,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL1")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+SND_SOC_DAILINK_DEFS(AFE_SOF_UL2,
+		     DAILINK_COMP_ARRAY(COMP_CPU("SOF_UL2")),
+		     DAILINK_COMP_ARRAY(COMP_DUMMY()),
+		     DAILINK_COMP_ARRAY(COMP_EMPTY()));
+
+static struct snd_soc_dai_link mt8196_nau8825_dai_links[] = {
+	/*
+	 * The SOF topology expects PCM streams 0~4 to be available
+	 * for the SOF PCM streams. Put the SOF BE definitions here
+	 * so that the PCM device numbers are skipped over.
+	 * (BE dailinks do not have PCM devices created.)
+	 */
+	{
+		.name = "AFE_SOF_DL_24CH",
+		.no_pcm = 1,
+		.playback_only = 1,
+		.ops = &mt8196_sof_be_ops,
+		SND_SOC_DAILINK_REG(AFE_SOF_DL_24CH),
+	},
+	{
+		.name = "AFE_SOF_DL1",
+		.no_pcm = 1,
+		.playback_only = 1,
+		.ops = &mt8196_sof_be_ops,
+		SND_SOC_DAILINK_REG(AFE_SOF_DL1),
+	},
+	{
+		.name = "AFE_SOF_UL0",
+		.no_pcm = 1,
+		.capture_only = 1,
+		.ops = &mt8196_sof_be_ops,
+		SND_SOC_DAILINK_REG(AFE_SOF_UL0),
+	},
+	{
+		.name = "AFE_SOF_UL1",
+		.no_pcm = 1,
+		.capture_only = 1,
+		.ops = &mt8196_sof_be_ops,
+		SND_SOC_DAILINK_REG(AFE_SOF_UL1),
+	},
+	{
+		.name = "AFE_SOF_UL2",
+		.no_pcm = 1,
+		.capture_only = 1,
+		.ops = &mt8196_sof_be_ops,
+		SND_SOC_DAILINK_REG(AFE_SOF_UL2),
+	},
+	/* Front End DAI links */
+	{
+		.name = "HDMI_FE",
+		.stream_name = "HDMI Playback",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.playback_only = 1,
+		SND_SOC_DAILINK_REG(playback_hdmi),
+	},
+	{
+		.name = "DL2_FE",
+		.stream_name = "DL2 Playback",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.playback_only = 1,
+		SND_SOC_DAILINK_REG(playback2),
+	},
+	{
+		.name = "UL_CM0_FE",
+		.stream_name = "UL_CM0 Capture",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+			    SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.capture_only = 1,
+		SND_SOC_DAILINK_REG(capture_cm0),
+	},
+	{
+		.name = "DL_24CH_FE",
+		.stream_name = "DL_24CH Playback",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+				SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.playback_only = 1,
+		SND_SOC_DAILINK_REG(playback_24ch),
+	},
+	{
+		.name = "DL1_FE",
+		.stream_name = "DL1 Playback",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+				SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.playback_only = 1,
+		SND_SOC_DAILINK_REG(playback1),
+	},
+	{
+		.name = "UL0_FE",
+		.stream_name = "UL0 Capture",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+				SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.capture_only = 1,
+		SND_SOC_DAILINK_REG(capture0),
+	},
+	{
+		.name = "UL1_FE",
+		.stream_name = "UL1 Capture",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+				SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.capture_only = 1,
+		SND_SOC_DAILINK_REG(capture1),
+	},
+	{
+		.name = "UL2_FE",
+		.stream_name = "UL2 Capture",
+		.trigger = {SND_SOC_DPCM_TRIGGER_PRE,
+				SND_SOC_DPCM_TRIGGER_PRE},
+		.dynamic = 1,
+		.capture_only = 1,
+		SND_SOC_DAILINK_REG(capture2),
+	},
+	/* Back End DAI links */
+	{
+		.name = "I2SIN6_BE",
+		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBC_CFC
+			| SND_SOC_DAIFMT_GATED,
+		.ops = &mt8196_nau8825_i2s_ops,
+		.no_pcm = 1,
+		.capture_only = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8196_hw_params_fixup,
+		SND_SOC_DAILINK_REG(i2sin6),
+	},
+	{
+		.name = "I2SOUT4_BE",
+		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBC_CFC
+			| SND_SOC_DAIFMT_GATED,
+		.ops = &mt8196_nau8825_i2s_ops,
+		.no_pcm = 1,
+		.playback_only = 1,
+		.ignore_suspend = 1,
+		.ignore_pmdown_time = 1,
+		.be_hw_params_fixup = mt8196_hw_params_fixup,
+		SND_SOC_DAILINK_REG(i2sout4),
+	},
+	{
+		.name = "I2SOUT6_BE",
+		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBC_CFC
+			| SND_SOC_DAIFMT_GATED,
+		.ops = &mt8196_nau8825_i2s_ops,
+		.no_pcm = 1,
+		.playback_only = 1,
+		.ignore_suspend = 1,
+		.be_hw_params_fixup = mt8196_hw_params_fixup,
+		SND_SOC_DAILINK_REG(i2sout6),
+	},
+	{
+		.name = "AP_DMIC_BE",
+		.no_pcm = 1,
+		.capture_only = 1,
+		.ignore_suspend = 1,
+		SND_SOC_DAILINK_REG(ap_dmic),
+	},
+	{
+		.name = "AP_DMIC_CH34_BE",
+		.no_pcm = 1,
+		.capture_only = 1,
+		.ignore_suspend = 1,
+		SND_SOC_DAILINK_REG(ap_dmic_ch34),
+	},
+	{
+		.name = "AP_DMIC_MULTICH_BE",
+		.no_pcm = 1,
+		.capture_only = 1,
+		.ignore_suspend = 1,
+		SND_SOC_DAILINK_REG(ap_dmic_multich),
+	},
+	{
+		.name = "TDM_DPTX_BE",
+		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBC_CFC
+			| SND_SOC_DAIFMT_GATED,
+		.ops = &mt8196_dptx_ops,
+		.be_hw_params_fixup = mt8196_hw_params_fixup,
+		.no_pcm = 1,
+		.playback_only = 1,
+		.ignore_suspend = 1,
+		SND_SOC_DAILINK_REG(tdm_dptx),
+	},
+	{
+		.name = "I2SOUT3_BE",
+		.dai_fmt = SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBC_CFC
+			| SND_SOC_DAIFMT_GATED,
+		.ops = &mt8196_nau8825_i2s_ops,
+		.no_pcm = 1,
+		.playback_only = 1,
+		.ignore_suspend = 1,
+		SND_SOC_DAILINK_REG(i2sout3),
+	},
+};
+
+static int mt8196_dumb_amp_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_card *card = rtd->card;
+	struct snd_soc_dapm_context *dapm = snd_soc_card_to_dapm(card);
+	int ret = 0;
+
+	ret = snd_soc_dapm_new_controls(dapm, mt8196_dumb_spk_widgets,
+					ARRAY_SIZE(mt8196_dumb_spk_widgets));
+	if (ret) {
+		dev_err(rtd->dev, "unable to add Dumb Speaker dapm, ret %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_add_card_controls(card, mt8196_dumb_spk_controls,
+					ARRAY_SIZE(mt8196_dumb_spk_controls));
+	if (ret) {
+		dev_err(rtd->dev, "unable to add Dumb card controls, ret %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int mt8196_dptx_codec_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card);
+	struct snd_soc_jack *jack = &soc_card_data->card_data->jacks[MT8196_JACK_DP];
+	struct snd_soc_component *component = snd_soc_rtd_to_codec(rtd, 0)->component;
+	int ret = 0;
+
+	ret = snd_soc_card_jack_new_pins(rtd->card, "DP Jack", SND_JACK_AVOUT,
+					 jack, mt8196_dp_jack_pins,
+					 ARRAY_SIZE(mt8196_dp_jack_pins));
+	if (ret) {
+		dev_err(rtd->dev, "new jack failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_component_set_jack(component, jack, NULL);
+	if (ret) {
+		dev_err(rtd->dev, "set jack failed on %s (ret=%d)\n",
+			component->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int mt8196_hdmi_codec_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(rtd->card);
+	struct snd_soc_jack *jack = &soc_card_data->card_data->jacks[MT8196_JACK_HDMI];
+	struct snd_soc_component *component = snd_soc_rtd_to_codec(rtd, 0)->component;
+	int ret = 0;
+
+	ret = snd_soc_card_jack_new_pins(rtd->card, "HDMI Jack", SND_JACK_AVOUT,
+					 jack, mt8196_hdmi_jack_pins,
+					 ARRAY_SIZE(mt8196_hdmi_jack_pins));
+	if (ret) {
+		dev_err(rtd->dev, "new jack failed: %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_component_set_jack(component, jack, NULL);
+	if (ret) {
+		dev_err(rtd->dev, "set jack failed on %s (ret=%d)\n",
+			component->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int mt8196_headset_codec_init(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_card *card = rtd->card;
+	struct snd_soc_dapm_context *dapm = snd_soc_card_to_dapm(card);
+	struct mtk_soc_card_data *soc_card_data = snd_soc_card_get_drvdata(card);
+	struct snd_soc_jack *jack = &soc_card_data->card_data->jacks[MT8196_JACK_HEADSET];
+	struct snd_soc_component *component = snd_soc_rtd_to_codec(rtd, 0)->component;
+	int ret;
+	int type;
+
+	ret = snd_soc_dapm_new_controls(dapm, mt8196_nau8825_widgets,
+					ARRAY_SIZE(mt8196_nau8825_widgets));
+	if (ret) {
+		dev_err(rtd->dev, "unable to add nau8825 card widget, ret %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_add_card_controls(card, mt8196_nau8825_controls,
+					ARRAY_SIZE(mt8196_nau8825_controls));
+	if (ret) {
+		dev_err(rtd->dev, "unable to add nau8825 card controls, ret %d\n", ret);
+		return ret;
+	}
+
+	ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack",
+					 SND_JACK_HEADSET | SND_JACK_BTN_0 |
+					 SND_JACK_BTN_1 | SND_JACK_BTN_2 |
+					 SND_JACK_BTN_3,
+					 jack,
+					 nau8825_jack_pins,
+					 ARRAY_SIZE(nau8825_jack_pins));
+	if (ret) {
+		dev_err(rtd->dev, "Headset Jack creation failed: %d\n", ret);
+		return ret;
+	}
+
+	snd_jack_set_key(jack->jack, SND_JACK_BTN_0, KEY_PLAYPAUSE);
+	snd_jack_set_key(jack->jack, SND_JACK_BTN_1, KEY_VOICECOMMAND);
+	snd_jack_set_key(jack->jack, SND_JACK_BTN_2, KEY_VOLUMEUP);
+	snd_jack_set_key(jack->jack, SND_JACK_BTN_3, KEY_VOLUMEDOWN);
+
+	type = SND_JACK_HEADSET | SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3;
+	ret = snd_soc_component_set_jack(component, jack, (void *)&type);
+
+	if (ret) {
+		dev_err(rtd->dev, "Headset Jack call-back failed: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+};
+
+static void mt8196_headset_codec_exit(struct snd_soc_pcm_runtime *rtd)
+{
+	struct snd_soc_component *component = snd_soc_rtd_to_codec(rtd, 0)->component;
+
+	snd_soc_component_set_jack(component, NULL, NULL);
+}
+
+static int mt8196_nau8825_hw_params(struct snd_pcm_substream *substream,
+				    struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = snd_soc_substream_to_rtd(substream);
+	struct snd_soc_dai *codec_dai = snd_soc_rtd_to_codec(rtd, 0);
+	unsigned int rate = params_rate(params);
+	unsigned int bit_width = params_width(params);
+	int clk_freq, ret;
+
+	clk_freq = rate * 2 * bit_width;
+
+	/* Configure clock for codec */
+	ret = snd_soc_dai_set_sysclk(codec_dai, NAU8825_CLK_FLL_BLK, 0,
+				     SND_SOC_CLOCK_IN);
+	if (ret < 0) {
+		dev_err(codec_dai->dev, "can't set BCLK clock %d\n", ret);
+		return ret;
+	}
+
+	/* Configure pll for codec */
+	ret = snd_soc_dai_set_pll(codec_dai, 0, 0, clk_freq,
+				  params_rate(params) * 256);
+	if (ret < 0) {
+		dev_err(codec_dai->dev, "can't set BCLK: %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static const struct snd_soc_ops mt8196_nau8825_ops = {
+	.hw_params = mt8196_nau8825_hw_params,
+};
+
+static int mt8196_rt5682s_i2s_hw_params(struct snd_pcm_substream *substream,
+					struct snd_pcm_hw_params *params)
+{
+	struct snd_soc_pcm_runtime *rtd = substream->private_data;
+	struct snd_soc_card *card = rtd->card;
+	struct snd_soc_dai *cpu_dai = snd_soc_rtd_to_cpu(rtd, 0);
+	struct snd_soc_dai *codec_dai = snd_soc_rtd_to_codec(rtd, 0);
+	unsigned int rate = params_rate(params);
+	int bitwidth;
+	int ret;
+
+	bitwidth = snd_pcm_format_width(params_format(params));
+	if (bitwidth < 0) {
+		dev_err(card->dev, "invalid bit width: %d\n", bitwidth);
+		return bitwidth;
+	}
+
+	ret = snd_soc_dai_set_tdm_slot(codec_dai, 0x00, 0x0, 0x2, bitwidth);
+	if (ret) {
+		dev_err(card->dev, "failed to set tdm slot\n");
+		return ret;
+	}
+
+	ret = snd_soc_dai_set_pll(codec_dai, RT5682S_PLL1, RT5682S_PLL_S_BCLK1,
+				  rate * 32, rate * 512);
+	if (ret) {
+		dev_err(card->dev, "failed to set pll\n");
+		return ret;
+	}
+
+	dev_info(card->dev, "%s set mclk rate: %d\n", __func__, rate * 512);
+
+	ret = snd_soc_dai_set_sysclk(codec_dai, RT5682S_SCLK_S_MCLK,
+				     rate * 512, SND_SOC_CLOCK_IN);
+	if (ret) {
+		dev_err(card->dev, "failed to set sysclk\n");
+		return ret;
+	}
+
+	return snd_soc_dai_set_sysclk(cpu_dai, 0, rate * 512,
+				      SND_SOC_CLOCK_OUT);
+}
+
+static const struct snd_soc_ops mt8196_rt5682s_i2s_ops = {
+	.hw_params = mt8196_rt5682s_i2s_hw_params,
+};
+
+static int mt8196_nau8825_soc_card_probe(struct mtk_soc_card_data *soc_card_data, bool legacy)
+{
+	struct snd_soc_card *card = soc_card_data->card_data->card;
+	struct snd_soc_dai_link *dai_link;
+	bool init_nau8825 = false;
+	bool init_rt5682s = false;
+	bool init_rt5650 = false;
+	bool init_dumb = false;
+	int i;
+
+	dev_info(card->dev, "legacy: %d\n", legacy);
+
+	for_each_card_prelinks(card, i, dai_link) {
+		if (strcmp(dai_link->name, "TDM_DPTX_BE") == 0) {
+			if (dai_link->num_codecs &&
+			    strcmp(dai_link->codecs->dai_name, "snd-soc-dummy-dai"))
+				dai_link->init = mt8196_dptx_codec_init;
+		} else if (strcmp(dai_link->name, "I2SOUT3_BE") == 0) {
+			if (dai_link->num_codecs &&
+			    strcmp(dai_link->codecs->dai_name, "snd-soc-dummy-dai"))
+				dai_link->init = mt8196_hdmi_codec_init;
+		} else if (strcmp(dai_link->name, "I2SOUT6_BE") == 0 ||
+			   strcmp(dai_link->name, "I2SIN6_BE") == 0) {
+			if (!strcmp(dai_link->codecs->dai_name, NAU8825_CODEC_DAI)) {
+				dai_link->ops = &mt8196_nau8825_ops;
+				if (!init_nau8825) {
+					dai_link->init = mt8196_headset_codec_init;
+					dai_link->exit = mt8196_headset_codec_exit;
+					init_nau8825 = true;
+				}
+			} else if (!strcmp(dai_link->codecs->dai_name, RT5682S_CODEC_DAI)) {
+				dai_link->ops = &mt8196_rt5682s_i2s_ops;
+				if (!init_rt5682s) {
+					dai_link->init = mt8196_headset_codec_init;
+					dai_link->exit = mt8196_headset_codec_exit;
+					init_rt5682s = true;
+				}
+			} else if (!strcmp(dai_link->codecs->dai_name, RT5650_CODEC_DAI)) {
+				dai_link->ops = &mt8196_rt5682s_i2s_ops;
+				if (!init_rt5650) {
+					dai_link->init = mt8196_headset_codec_init;
+					dai_link->exit = mt8196_headset_codec_exit;
+					init_rt5650 = true;
+				}
+			} else {
+				if (strcmp(dai_link->codecs->dai_name, "snd-soc-dummy-dai")) {
+					if (!init_dumb) {
+						dai_link->init = mt8196_dumb_amp_init;
+						init_dumb = true;
+					}
+				}
+			}
+		}
+	}
+
+	return 0;
+}
+
+static const struct mtk_sof_priv mt8196_sof_priv = {
+	.conn_streams = g_sof_conn_streams,
+	.num_streams = ARRAY_SIZE(g_sof_conn_streams),
+};
+
+static struct snd_soc_card mt8196_nau8825_soc_card = {
+	.owner = THIS_MODULE,
+	.dai_link = mt8196_nau8825_dai_links,
+	.num_links = ARRAY_SIZE(mt8196_nau8825_dai_links),
+	.dapm_widgets = mt8196_nau8825_card_widgets,
+	.num_dapm_widgets = ARRAY_SIZE(mt8196_nau8825_card_widgets),
+	.dapm_routes = mt8196_nau8825_card_routes,
+	.num_dapm_routes = ARRAY_SIZE(mt8196_nau8825_card_routes),
+	.controls = mt8196_nau8825_card_controls,
+	.num_controls = ARRAY_SIZE(mt8196_nau8825_card_controls),
+};
+
+static const struct mtk_soundcard_pdata mt8196_nau8825_card = {
+	.card_name = "mt8196_nau8825",
+	.card_data = &(struct mtk_platform_card_data) {
+		.card = &mt8196_nau8825_soc_card,
+		.num_jacks = MT8196_JACK_MAX,
+		.flags = NAU8825_HS_PRESENT
+	},
+	.sof_priv = &mt8196_sof_priv,
+	.soc_probe = mt8196_nau8825_soc_card_probe,
+};
+
+static const struct mtk_soundcard_pdata mt8196_rt5682s_card = {
+	.card_name = "mt8196_rt5682s",
+	.card_data = &(struct mtk_platform_card_data) {
+		.card = &mt8196_nau8825_soc_card,
+		.num_jacks = MT8196_JACK_MAX,
+		.flags = RT5682S_HS_PRESENT
+	},
+	.sof_priv = &mt8196_sof_priv,
+	.soc_probe = mt8196_nau8825_soc_card_probe,
+};
+
+static const struct mtk_soundcard_pdata mt8196_rt5650_card = {
+	.card_name = "mt8196_rt5650",
+	.card_data = &(struct mtk_platform_card_data) {
+		.card = &mt8196_nau8825_soc_card,
+		.num_jacks = MT8196_JACK_MAX,
+		.flags = RT5650_HS_PRESENT
+	},
+	.sof_priv = &mt8196_sof_priv,
+	.soc_probe = mt8196_nau8825_soc_card_probe,
+};
+
+static const struct of_device_id mt8196_nau8825_dt_match[] = {
+	{.compatible = "mediatek,mt8196-nau8825-sound", .data = &mt8196_nau8825_card,},
+	{.compatible = "mediatek,mt8196-rt5682s-sound", .data = &mt8196_rt5682s_card,},
+	{.compatible = "mediatek,mt8196-rt5650-sound", .data = &mt8196_rt5650_card,},
+	{}
+};
+MODULE_DEVICE_TABLE(of, mt8196_nau8825_dt_match);
+
+static struct platform_driver mt8196_nau8825_driver = {
+	.driver = {
+		.name = "mt8196-nau8825",
+		.of_match_table = mt8196_nau8825_dt_match,
+		.pm = &snd_soc_pm_ops,
+	},
+	.probe = mtk_soundcard_common_probe,
+};
+module_platform_driver(mt8196_nau8825_driver);
+
+/* Module information */
+MODULE_DESCRIPTION("MT8196 nau8825 ALSA SoC machine driver");
+MODULE_AUTHOR("Darren Ye <darren.ye@mediatek.com>");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("mt8196 nau8825 soc card");
+
diff --git a/sound/soc/mediatek/mt8196/mt8196-reg.h b/sound/soc/mediatek/mt8196/mt8196-reg.h
new file mode 100644
index 0000000000000..eb689d2655be4
--- /dev/null
+++ b/sound/soc/mediatek/mt8196/mt8196-reg.h
@@ -0,0 +1,12068 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * mt8196-reg.h  --  Mediatek 8196 audio driver reg definition
+ *
+ *  Copyright (c) 2025 MediaTek Inc.
+ *  Author: Darren Ye <darren.ye@mediatek.com>
+ */
+
+#ifndef _MT8196_REG_H_
+#define _MT8196_REG_H_
+
+ /* reg bit enum */
+enum {
+	MT8196_MEMIF_PBUF_SIZE_32_BYTES,
+	MT8196_MEMIF_PBUF_SIZE_64_BYTES,
+	MT8196_MEMIF_PBUF_SIZE_128_BYTES,
+	MT8196_MEMIF_PBUF_SIZE_256_BYTES,
+	MT8196_MEMIF_PBUF_SIZE_NUM,
+};
+
+enum {
+	MT8196_MEMIF_MAX_LEN_0_BYTES,
+	MT8196_MEMIF_MAX_LEN_16_BYTES,
+	MT8196_MEMIF_MAX_LEN_32_BYTES,
+	MT8196_MEMIF_MAX_LEN_64_BYTES,
+};
+
+enum {
+	MT8196_MEMIF_MIN_LEN_NOT_SUPPORT,
+	MT8196_MEMIF_MIN_LEN_16_BYTES,
+	MT8196_MEMIF_MIN_LEN_32_BYTES,
+	MT8196_MEMIF_MIN_LEN_64_BYTES,
+};
+
+/*****************************************************************************
+ * R E G I S T E R  D E F I N I T I O N
+ *****************************************************************************/
+/* AUDIO_TOP_CON0 */
+#define PDN_MTKAIFV4_SFT                                      25
+#define PDN_MTKAIFV4_MASK                                     0x1
+#define PDN_MTKAIFV4_MASK_SFT                                 (0x1 << 25)
+#define PDN_FM_I2S_SFT                                        24
+#define PDN_FM_I2S_MASK                                       0x1
+#define PDN_FM_I2S_MASK_SFT                                   (0x1 << 24)
+#define PDN_HW_GAIN01_SFT                                     21
+#define PDN_HW_GAIN01_MASK                                    0x1
+#define PDN_HW_GAIN01_MASK_SFT                                (0x1 << 21)
+#define PDN_HW_GAIN23_SFT                                     20
+#define PDN_HW_GAIN23_MASK                                    0x1
+#define PDN_HW_GAIN23_MASK_SFT                                (0x1 << 20)
+#define PDN_STF_SFT                                           19
+#define PDN_STF_MASK                                          0x1
+#define PDN_STF_MASK_SFT                                      (0x1 << 19)
+#define PDN_CM0_SFT                                           18
+#define PDN_CM0_MASK                                          0x1
+#define PDN_CM0_MASK_SFT                                      (0x1 << 18)
+#define PDN_CM1_SFT                                           17
+#define PDN_CM1_MASK                                          0x1
+#define PDN_CM1_MASK_SFT                                      (0x1 << 17)
+#define PDN_CM2_SFT                                           16
+#define PDN_CM2_MASK                                          0x1
+#define PDN_CM2_MASK_SFT                                      (0x1 << 16)
+#define PDN_PCM0_SFT                                          14
+#define PDN_PCM0_MASK                                         0x1
+#define PDN_PCM0_MASK_SFT                                     (0x1 << 14)
+#define PDN_PCM1_SFT                                          13
+#define PDN_PCM1_MASK                                         0x1
+#define PDN_PCM1_MASK_SFT                                     (0x1 << 13)
+
+/* AUDIO_TOP_CON1 */
+#define PDN_UL0_ADC_SFT                                       23
+#define PDN_UL0_ADC_MASK                                      0x1
+#define PDN_UL0_ADC_MASK_SFT                                  (0x1 << 23)
+#define PDN_UL0_TML_SFT                                       22
+#define PDN_UL0_TML_MASK                                      0x1
+#define PDN_UL0_TML_MASK_SFT                                  (0x1 << 22)
+#define PDN_UL0_ADC_HIRES_SFT                                 21
+#define PDN_UL0_ADC_HIRES_MASK                                0x1
+#define PDN_UL0_ADC_HIRES_MASK_SFT                            (0x1 << 21)
+#define PDN_UL0_ADC_HIRES_TML_SFT                             20
+#define PDN_UL0_ADC_HIRES_TML_MASK                            0x1
+#define PDN_UL0_ADC_HIRES_TML_MASK_SFT                        (0x1 << 20)
+#define PDN_UL1_ADC_SFT                                       19
+#define PDN_UL1_ADC_MASK                                      0x1
+#define PDN_UL1_ADC_MASK_SFT                                  (0x1 << 19)
+#define PDN_UL1_TML_SFT                                       18
+#define PDN_UL1_TML_MASK                                      0x1
+#define PDN_UL1_TML_MASK_SFT                                  (0x1 << 18)
+#define PDN_UL1_ADC_HIRES_SFT                                 17
+#define PDN_UL1_ADC_HIRES_MASK                                0x1
+#define PDN_UL1_ADC_HIRES_MASK_SFT                            (0x1 << 17)
+#define PDN_UL1_ADC_HIRES_TML_SFT                             16
+#define PDN_UL1_ADC_HIRES_TML_MASK                            0x1
+#define PDN_UL1_ADC_HIRES_TML_MASK_SFT                        (0x1 << 16)
+#define PDN_UL2_ADC_SFT                                       15
+#define PDN_UL2_ADC_MASK                                      0x1
+#define PDN_UL2_ADC_MASK_SFT                                  (0x1 << 15)
+#define PDN_UL2_TML_SFT                                       14
+#define PDN_UL2_TML_MASK                                      0x1
+#define PDN_UL2_TML_MASK_SFT                                  (0x1 << 14)
+#define PDN_UL2_ADC_HIRES_SFT                                 13
+#define PDN_UL2_ADC_HIRES_MASK                                0x1
+#define PDN_UL2_ADC_HIRES_MASK_SFT                            (0x1 << 13)
+#define PDN_UL2_ADC_HIRES_TML_SFT                             12
+#define PDN_UL2_ADC_HIRES_TML_MASK                            0x1
+#define PDN_UL2_ADC_HIRES_TML_MASK_SFT                        (0x1 << 12)
+
+/* AUDIO_TOP_CON2 */
+#define PDN_TDM_OUT_SFT                                       24
+#define PDN_TDM_OUT_MASK                                      0x1
+#define PDN_TDM_OUT_MASK_SFT                                  (0x1 << 24)
+#define PDN_ETDM_OUT0_SFT                                     21
+#define PDN_ETDM_OUT0_MASK                                    0x1
+#define PDN_ETDM_OUT0_MASK_SFT                                (0x1 << 21)
+#define PDN_ETDM_OUT1_SFT                                     20
+#define PDN_ETDM_OUT1_MASK                                    0x1
+#define PDN_ETDM_OUT1_MASK_SFT                                (0x1 << 20)
+#define PDN_ETDM_OUT2_SFT                                     19
+#define PDN_ETDM_OUT2_MASK                                    0x1
+#define PDN_ETDM_OUT2_MASK_SFT                                (0x1 << 19)
+#define PDN_ETDM_OUT3_SFT                                     18
+#define PDN_ETDM_OUT3_MASK                                    0x1
+#define PDN_ETDM_OUT3_MASK_SFT                                (0x1 << 18)
+#define PDN_ETDM_OUT4_SFT                                     17
+#define PDN_ETDM_OUT4_MASK                                    0x1
+#define PDN_ETDM_OUT4_MASK_SFT                                (0x1 << 17)
+#define PDN_ETDM_OUT5_SFT                                     16
+#define PDN_ETDM_OUT5_MASK                                    0x1
+#define PDN_ETDM_OUT5_MASK_SFT                                (0x1 << 16)
+#define PDN_ETDM_OUT6_SFT                                     15
+#define PDN_ETDM_OUT6_MASK                                    0x1
+#define PDN_ETDM_OUT6_MASK_SFT                                (0x1 << 15)
+#define PDN_ETDM_IN0_SFT                                      13
+#define PDN_ETDM_IN0_MASK                                     0x1
+#define PDN_ETDM_IN0_MASK_SFT                                 (0x1 << 13)
+#define PDN_ETDM_IN1_SFT                                      12
+#define PDN_ETDM_IN1_MASK                                     0x1
+#define PDN_ETDM_IN1_MASK_SFT                                 (0x1 << 12)
+#define PDN_ETDM_IN2_SFT                                      11
+#define PDN_ETDM_IN2_MASK                                     0x1
+#define PDN_ETDM_IN2_MASK_SFT                                 (0x1 << 11)
+#define PDN_ETDM_IN3_SFT                                      10
+#define PDN_ETDM_IN3_MASK                                     0x1
+#define PDN_ETDM_IN3_MASK_SFT                                 (0x1 << 10)
+#define PDN_ETDM_IN4_SFT                                      9
+#define PDN_ETDM_IN4_MASK                                     0x1
+#define PDN_ETDM_IN4_MASK_SFT                                 (0x1 << 9)
+#define PDN_ETDM_IN5_SFT                                      8
+#define PDN_ETDM_IN5_MASK                                     0x1
+#define PDN_ETDM_IN5_MASK_SFT                                 (0x1 << 8)
+#define PDN_ETDM_IN6_SFT                                      7
+#define PDN_ETDM_IN6_MASK                                     0x1
+#define PDN_ETDM_IN6_MASK_SFT                                 (0x1 << 7)
+
+/* AUDIO_TOP_CON3 */
+#define PDN_CONNSYS_I2S_ASRC_SFT                              25
+#define PDN_CONNSYS_I2S_ASRC_MASK                             0x1
+#define PDN_CONNSYS_I2S_ASRC_MASK_SFT                         (0x1 << 25)
+#define PDN_GENERAL0_ASRC_SFT                                 24
+#define PDN_GENERAL0_ASRC_MASK                                0x1
+#define PDN_GENERAL0_ASRC_MASK_SFT                            (0x1 << 24)
+#define PDN_GENERAL1_ASRC_SFT                                 23
+#define PDN_GENERAL1_ASRC_MASK                                0x1
+#define PDN_GENERAL1_ASRC_MASK_SFT                            (0x1 << 23)
+#define PDN_GENERAL2_ASRC_SFT                                 22
+#define PDN_GENERAL2_ASRC_MASK                                0x1
+#define PDN_GENERAL2_ASRC_MASK_SFT                            (0x1 << 22)
+#define PDN_GENERAL3_ASRC_SFT                                 21
+#define PDN_GENERAL3_ASRC_MASK                                0x1
+#define PDN_GENERAL3_ASRC_MASK_SFT                            (0x1 << 21)
+#define PDN_GENERAL4_ASRC_SFT                                 20
+#define PDN_GENERAL4_ASRC_MASK                                0x1
+#define PDN_GENERAL4_ASRC_MASK_SFT                            (0x1 << 20)
+#define PDN_GENERAL5_ASRC_SFT                                 19
+#define PDN_GENERAL5_ASRC_MASK                                0x1
+#define PDN_GENERAL5_ASRC_MASK_SFT                            (0x1 << 19)
+#define PDN_GENERAL6_ASRC_SFT                                 18
+#define PDN_GENERAL6_ASRC_MASK                                0x1
+#define PDN_GENERAL6_ASRC_MASK_SFT                            (0x1 << 18)
+#define PDN_GENERAL7_ASRC_SFT                                 17
+#define PDN_GENERAL7_ASRC_MASK                                0x1
+#define PDN_GENERAL7_ASRC_MASK_SFT                            (0x1 << 17)
+#define PDN_GENERAL8_ASRC_SFT                                 16
+#define PDN_GENERAL8_ASRC_MASK                                0x1
+#define PDN_GENERAL8_ASRC_MASK_SFT                            (0x1 << 16)
+#define PDN_GENERAL9_ASRC_SFT                                 15
+#define PDN_GENERAL9_ASRC_MASK                                0x1
+#define PDN_GENERAL9_ASRC_MASK_SFT                            (0x1 << 15)
+#define PDN_GENERAL10_ASRC_SFT                                14
+#define PDN_GENERAL10_ASRC_MASK                               0x1
+#define PDN_GENERAL10_ASRC_MASK_SFT                           (0x1 << 14)
+#define PDN_GENERAL11_ASRC_SFT                                13
+#define PDN_GENERAL11_ASRC_MASK                               0x1
+#define PDN_GENERAL11_ASRC_MASK_SFT                           (0x1 << 13)
+#define PDN_GENERAL12_ASRC_SFT                                12
+#define PDN_GENERAL12_ASRC_MASK                               0x1
+#define PDN_GENERAL12_ASRC_MASK_SFT                           (0x1 << 12)
+#define PDN_GENERAL13_ASRC_SFT                                11
+#define PDN_GENERAL13_ASRC_MASK                               0x1
+#define PDN_GENERAL13_ASRC_MASK_SFT                           (0x1 << 11)
+#define PDN_GENERAL14_ASRC_SFT                                10
+#define PDN_GENERAL14_ASRC_MASK                               0x1
+#define PDN_GENERAL14_ASRC_MASK_SFT                           (0x1 << 10)
+#define PDN_GENERAL15_ASRC_SFT                                9
+#define PDN_GENERAL15_ASRC_MASK                               0x1
+#define PDN_GENERAL15_ASRC_MASK_SFT                           (0x1 << 9)
+
+/* AUDIO_TOP_CON4 */
+#define PDN_APLL_TUNER1_SFT                                   13
+#define PDN_APLL_TUNER1_MASK                                  0x1
+#define PDN_APLL_TUNER1_MASK_SFT                              (0x1 << 13)
+#define PDN_APLL_TUNER2_SFT                                   12
+#define PDN_APLL_TUNER2_MASK                                  0x1
+#define PDN_APLL_TUNER2_MASK_SFT                              (0x1 << 12)
+#define CG_H208M_CK_SFT                                       4
+#define CG_H208M_CK_MASK                                      0x1
+#define CG_H208M_CK_MASK_SFT                                  (0x1 << 4)
+#define CG_APLL2_CK_SFT                                       3
+#define CG_APLL2_CK_MASK                                      0x1
+#define CG_APLL2_CK_MASK_SFT                                  (0x1 << 3)
+#define CG_APLL1_CK_SFT                                       2
+#define CG_APLL1_CK_MASK                                      0x1
+#define CG_APLL1_CK_MASK_SFT                                  (0x1 << 2)
+#define CG_AUDIO_F26M_CK_SFT                                  1
+#define CG_AUDIO_F26M_CK_MASK                                 0x1
+#define CG_AUDIO_F26M_CK_MASK_SFT                             (0x1 << 1)
+#define CG_AUDIO_HOPPING_CK_SFT                               0
+#define CG_AUDIO_HOPPING_CK_MASK                              0x1
+#define CG_AUDIO_HOPPING_CK_MASK_SFT                          (0x1 << 0)
+
+/* AUDIO_ENGEN_CON0 */
+/* AUDIO_ENGEN_CON0_USER1 */
+/* AUDIO_ENGEN_CON0_USER1 */
+#define MULTI_USER_BYPASS_SFT                                 17
+#define MULTI_USER_BYPASS_MASK                                0x1
+#define MULTI_USER_BYPASS_MASK_SFT                            (0x1 << 17)
+#define MULTI_USER_RST_SFT                                    16
+#define MULTI_USER_RST_MASK                                   0x1
+#define MULTI_USER_RST_MASK_SFT                               (0x1 << 16)
+#define AUDIO_F26M_EN_RST_SFT                                 8
+#define AUDIO_F26M_EN_RST_MASK                                0x1
+#define AUDIO_F26M_EN_RST_MASK_SFT                            (0x1 << 8)
+#define AUDIO_APLL2_EN_ON_SFT                                 3
+#define AUDIO_APLL2_EN_ON_MASK                                0x1
+#define AUDIO_APLL2_EN_ON_MASK_SFT                            (0x1 << 3)
+#define AUDIO_APLL1_EN_ON_SFT                                 2
+#define AUDIO_APLL1_EN_ON_MASK                                0x1
+#define AUDIO_APLL1_EN_ON_MASK_SFT                            (0x1 << 2)
+#define AUDIO_F3P25M_EN_ON_SFT                                1
+#define AUDIO_F3P25M_EN_ON_MASK                               0x1
+#define AUDIO_F3P25M_EN_ON_MASK_SFT                           (0x1 << 1)
+#define AUDIO_26M_EN_ON_SFT                                   0
+#define AUDIO_26M_EN_ON_MASK                                  0x1
+#define AUDIO_26M_EN_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_SINEGEN_CON0 */
+#define DAC_EN_SFT                                            26
+#define DAC_EN_MASK                                           0x1
+#define DAC_EN_MASK_SFT                                       (0x1 << 26)
+#define TIE_SW_CH2_SFT                                        25
+#define TIE_SW_CH2_MASK                                       0x1
+#define TIE_SW_CH2_MASK_SFT                                   (0x1 << 25)
+#define TIE_SW_CH1_SFT                                        24
+#define TIE_SW_CH1_MASK                                       0x1
+#define TIE_SW_CH1_MASK_SFT                                   (0x1 << 24)
+#define AMP_DIV_CH2_SFT                                       20
+#define AMP_DIV_CH2_MASK                                      0xf
+#define AMP_DIV_CH2_MASK_SFT                                  (0xf << 20)
+#define FREQ_DIV_CH2_SFT                                      12
+#define FREQ_DIV_CH2_MASK                                     0x1f
+#define FREQ_DIV_CH2_MASK_SFT                                 (0x1f << 12)
+#define AMP_DIV_CH1_SFT                                       8
+#define AMP_DIV_CH1_MASK                                      0xf
+#define AMP_DIV_CH1_MASK_SFT                                  (0xf << 8)
+#define FREQ_DIV_CH1_SFT                                      0
+#define FREQ_DIV_CH1_MASK                                     0x1f
+#define FREQ_DIV_CH1_MASK_SFT                                 (0x1f << 0)
+
+/* AFE_SINEGEN_CON1 */
+#define SINE_DOMAIN_SFT                                       20
+#define SINE_DOMAIN_MASK                                      0x7
+#define SINE_DOMAIN_MASK_SFT                                  (0x7 << 20)
+#define SINE_MODE_SFT                                         12
+#define SINE_MODE_MASK                                        0x1f
+#define SINE_MODE_MASK_SFT                                    (0x1f << 12)
+#define INNER_LOOP_BACKI_SEL_SFT                              8
+#define INNER_LOOP_BACKI_SEL_MASK                             0x1
+#define INNER_LOOP_BACKI_SEL_MASK_SFT                         (0x1 << 8)
+#define INNER_LOOP_BACK_MODE_SFT                              0
+#define INNER_LOOP_BACK_MODE_MASK                             0xff
+#define INNER_LOOP_BACK_MODE_MASK_SFT                         (0xff << 0)
+
+/* AFE_SINEGEN_CON2 */
+#define TIE_CH1_CONSTANT_SFT                                  0
+#define TIE_CH1_CONSTANT_MASK                                 0xffffffff
+#define TIE_CH1_CONSTANT_MASK_SFT                             (0xffffffff << 0)
+
+/* AFE_SINEGEN_CON3 */
+#define TIE_CH2_CONSTANT_SFT                                  0
+#define TIE_CH2_CONSTANT_MASK                                 0xffffffff
+#define TIE_CH2_CONSTANT_MASK_SFT                             (0xffffffff << 0)
+
+/* AFE_APLL1_TUNER_CFG */
+/* AFE_APLL2_TUNER_CFG */
+#define UPPER_BOUND_SFT                                       8
+#define UPPER_BOUND_MASK                                      0xff
+#define UPPER_BOUND_MASK_SFT                                  (0xff << 8)
+#define APLL_DIV_SFT                                          4
+#define APLL_DIV_MASK                                         0xf
+#define APLL_DIV_MASK_SFT                                     (0xf << 4)
+#define XTAL_EN_128FS_SEL_SFT                                 1
+#define XTAL_EN_128FS_SEL_MASK                                0x3
+#define XTAL_EN_128FS_SEL_MASK_SFT                            (0x3 << 1)
+#define FREQ_TUNER_EN_SFT                                     0
+#define FREQ_TUNER_EN_MASK                                    0x1
+#define FREQ_TUNER_EN_MASK_SFT                                (0x1 << 0)
+
+/* AFE_APLL1_TUNER_MON0 */
+/* AFE_APLL2_TUNER_MON0 */
+#define TUNER_MON_SFT                                         0
+#define TUNER_MON_MASK                                        0xffffffff
+#define TUNER_MON_MASK_SFT                                    (0xffffffff << 0)
+
+/* AUDIO_TOP_RG0 */
+/* AUDIO_TOP_RG1 */
+/* AUDIO_TOP_RG2 */
+/* AUDIO_TOP_RG3 */
+/* AUDIO_TOP_RG4 */
+#define RESERVE_RG_SFT                                        0
+#define RESERVE_RG_MASK                                       0xffffffff
+#define RESERVE_RG_MASK_SFT                                   (0xffffffff << 0)
+
+/* AFE_SPM_CONTROL_REQ */
+#define AFE_DDREN_REQ_SFT                                     4
+#define AFE_DDREN_REQ_MASK                                    0x1
+#define AFE_DDREN_REQ_MASK_SFT                                (0x1 << 4)
+#define AFE_INFRA_REQ_SFT                                     3
+#define AFE_INFRA_REQ_MASK                                    0x1
+#define AFE_INFRA_REQ_MASK_SFT                                (0x1 << 3)
+#define AFE_VRF18_REQ_SFT                                     2
+#define AFE_VRF18_REQ_MASK                                    0x1
+#define AFE_VRF18_REQ_MASK_SFT                                (0x1 << 2)
+#define AFE_APSRC_REQ_SFT                                     1
+#define AFE_APSRC_REQ_MASK                                    0x1
+#define AFE_APSRC_REQ_MASK_SFT                                (0x1 << 1)
+#define AFE_SRCCLKENA_REQ_SFT                                 0
+#define AFE_SRCCLKENA_REQ_MASK                                0x1
+#define AFE_SRCCLKENA_REQ_MASK_SFT                            (0x1 << 0)
+
+/* AFE_SPM_CONTROL_ACK */
+#define SPM_RESOURCE_CONTROL_ACK_SFT                          0
+#define SPM_RESOURCE_CONTROL_ACK_MASK                         0xffffffff
+#define SPM_RESOURCE_CONTROL_ACK_MASK_SFT                     (0xffffffff << 0)
+
+/* AUD_TOP_CFG_VCORE_RG */
+#define AUD_TOP_CFG_SFT                                       0
+#define AUD_TOP_CFG_MASK                                      0xffffffff
+#define AUD_TOP_CFG_MASK_SFT                                  (0xffffffff << 0)
+
+/* AUDIO_TOP_IP_VERSION */
+#define AUDIO_TOP_IP_VERSION_SFT                              0
+#define AUDIO_TOP_IP_VERSION_MASK                             0xffffffff
+#define AUDIO_TOP_IP_VERSION_MASK_SFT                         (0xffffffff << 0)
+
+/* AUDIO_ENGEN_CON0_MON */
+#define AUDIO_ENGEN_MON_SFT                                   0
+#define AUDIO_ENGEN_MON_MASK                                  0xffffffff
+#define AUDIO_ENGEN_MON_MASK_SFT                              (0xffffffff << 0)
+
+/* AUD_TOP_CFG_VLP_RG */
+#define I2SIN1_DAT_SEL_SFT                                    31
+#define I2SIN1_DAT_SEL_MASK                                   0x1
+#define I2SIN1_DAT_SEL_MASK_SFT                               (0x1 << 31)
+#define FMI2S_IN_SEL_SFT                                      30
+#define FMI2S_IN_SEL_MASK                                     0x1
+#define FMI2S_IN_SEL_MASK_SFT                                 (0x1 << 30)
+#define RG_I2S4_IN_BCK_NEG_EG_LATCH_SFT                       21
+#define RG_I2S4_IN_BCK_NEG_EG_LATCH_MASK                      0x1
+#define RG_I2S4_IN_BCK_NEG_EG_LATCH_MASK_SFT                  (0x1 << 21)
+#define RG_I2S4_OUT_BCK_NEG_EG_LATCH_SFT                      20
+#define RG_I2S4_OUT_BCK_NEG_EG_LATCH_MASK                     0x1
+#define RG_I2S4_OUT_BCK_NEG_EG_LATCH_MASK_SFT                 (0x1 << 20)
+#define RG_I2S4_IN_SLV_LRCK_LATCH_EDGE_SFT                    19
+#define RG_I2S4_IN_SLV_LRCK_LATCH_EDGE_MASK                   0x1
+#define RG_I2S4_IN_SLV_LRCK_LATCH_EDGE_MASK_SFT               (0x1 << 19)
+#define RG_I2S4_IN_SLV_BCK_INV_SEL_SFT                        18
+#define RG_I2S4_IN_SLV_BCK_INV_SEL_MASK                       0x1
+#define RG_I2S4_IN_SLV_BCK_INV_SEL_MASK_SFT                   (0x1 << 18)
+#define RG_I2S4_OUT_SLV_LRCK_LATCH_EDGE_SFT                   17
+#define RG_I2S4_OUT_SLV_LRCK_LATCH_EDGE_MASK                  0x1
+#define RG_I2S4_OUT_SLV_LRCK_LATCH_EDGE_MASK_SFT              (0x1 << 17)
+#define RG_I2S4_OUT_SLV_BCK_INV_SEL_SFT                       16
+#define RG_I2S4_OUT_SLV_BCK_INV_SEL_MASK                      0x1
+#define RG_I2S4_OUT_SLV_BCK_INV_SEL_MASK_SFT                  (0x1 << 16)
+#define RG_I2S5_IN_BCK_NEG_EG_LATCH_SFT                       13
+#define RG_I2S5_IN_BCK_NEG_EG_LATCH_MASK                      0x1
+#define RG_I2S5_IN_BCK_NEG_EG_LATCH_MASK_SFT                  (0x1 << 13)
+#define RG_I2S5_OUT_BCK_NEG_EG_LATCH_SFT                      12
+#define RG_I2S5_OUT_BCK_NEG_EG_LATCH_MASK                     0x1
+#define RG_I2S5_OUT_BCK_NEG_EG_LATCH_MASK_SFT                 (0x1 << 12)
+#define RG_I2S5_IN_SLV_LRCK_LATCH_EDGE_SFT                    11
+#define RG_I2S5_IN_SLV_LRCK_LATCH_EDGE_MASK                   0x1
+#define RG_I2S5_IN_SLV_LRCK_LATCH_EDGE_MASK_SFT               (0x1 << 11)
+#define RG_I2S5_IN_SLV_BCK_INV_SEL_SFT                        10
+#define RG_I2S5_IN_SLV_BCK_INV_SEL_MASK                       0x1
+#define RG_I2S5_IN_SLV_BCK_INV_SEL_MASK_SFT                   (0x1 << 10)
+#define RG_I2S5_OUT_SLV_LRCK_LATCH_EDGE_SFT                   9
+#define RG_I2S5_OUT_SLV_LRCK_LATCH_EDGE_MASK                  0x1
+#define RG_I2S5_OUT_SLV_LRCK_LATCH_EDGE_MASK_SFT              (0x1 << 9)
+#define RG_I2S5_OUT_SLV_BCK_INV_SEL_SFT                       8
+#define RG_I2S5_OUT_SLV_BCK_INV_SEL_MASK                      0x1
+#define RG_I2S5_OUT_SLV_BCK_INV_SEL_MASK_SFT                  (0x1 << 8)
+#define RG_I2S4_PAD_TOP_CK_EN_SFT                             5
+#define RG_I2S4_PAD_TOP_CK_EN_MASK                            0x1
+#define RG_I2S4_PAD_TOP_CK_EN_MASK_SFT                        (0x1 << 5)
+#define RG_I2S5_PAD_TOP_CK_EN_SFT                             4
+#define RG_I2S5_PAD_TOP_CK_EN_MASK                            0x1
+#define RG_I2S5_PAD_TOP_CK_EN_MASK_SFT                        (0x1 << 4)
+#define RG_TEST_TYPE_SFT                                      2
+#define RG_TEST_TYPE_MASK                                     0x1
+#define RG_TEST_TYPE_MASK_SFT                                 (0x1 << 2)
+#define RG_SW_RESET_SFT                                       1
+#define RG_SW_RESET_MASK                                      0x1
+#define RG_SW_RESET_MASK_SFT                                  (0x1 << 1)
+#define RG_TEST_ON_SFT                                        0
+#define RG_TEST_ON_MASK                                       0x1
+#define RG_TEST_ON_MASK_SFT                                   (0x1 << 0)
+
+/* AUD_TOP_MON_RG */
+#define AUD_TOP_MON_SFT                                       0
+#define AUD_TOP_MON_MASK                                      0xffffffff
+#define AUD_TOP_MON_MASK_SFT                                  (0xffffffff << 0)
+
+/* AUDIO_USE_DEFAULT_DELSEL0 */
+#define USE_DEFAULT_DELSEL_RG_SFT                             0
+#define USE_DEFAULT_DELSEL_RG_MASK                            0xffffffff
+#define USE_DEFAULT_DELSEL_RG_MASK_SFT                        (0xffffffff << 0)
+
+/* AUDIO_USE_DEFAULT_DELSEL1 */
+#define USE_DEFAULT_DELSEL_RG_SFT                             0
+#define USE_DEFAULT_DELSEL_RG_MASK                            0xffffffff
+#define USE_DEFAULT_DELSEL_RG_MASK_SFT                        (0xffffffff << 0)
+
+/* AUDIO_USE_DEFAULT_DELSEL2 */
+#define USE_DEFAULT_DELSEL_RG_SFT                             0
+#define USE_DEFAULT_DELSEL_RG_MASK                            0xffffffff
+#define USE_DEFAULT_DELSEL_RG_MASK_SFT                        (0xffffffff << 0)
+
+/* AFE_CONNSYS_I2S_IPM_VER_MON */
+#define RG_CONNSYS_I2S_IPM_VER_MON_SFT                        0
+#define RG_CONNSYS_I2S_IPM_VER_MON_MASK                       0xffffffff
+#define RG_CONNSYS_I2S_IPM_VER_MON_MASK_SFT                   (0xffffffff << 0)
+
+/* AFE_CONNSYS_I2S_MON_SEL */
+#define RG_CONNSYS_I2S_MON_SEL_SFT                            0
+#define RG_CONNSYS_I2S_MON_SEL_MASK                           0xff
+#define RG_CONNSYS_I2S_MON_SEL_MASK_SFT                       (0xff << 0)
+
+/* AFE_CONNSYS_I2S_MON */
+#define RG_CONNSYS_I2S_MON_SFT                                0
+#define RG_CONNSYS_I2S_MON_MASK                               0xffffffff
+#define RG_CONNSYS_I2S_MON_MASK_SFT                           (0xffffffff << 0)
+
+/* AFE_CONNSYS_I2S_CON */
+#define I2S_SOFT_RST_SFT                                      31
+#define I2S_SOFT_RST_MASK                                     0x1
+#define I2S_SOFT_RST_MASK_SFT                                 (0x1 << 31)
+#define BCK_NEG_EG_LATCH_SFT                                  30
+#define BCK_NEG_EG_LATCH_MASK                                 0x1
+#define BCK_NEG_EG_LATCH_MASK_SFT                             (0x1 << 30)
+#define BCK_INV_SFT                                           29
+#define BCK_INV_MASK                                          0x1
+#define BCK_INV_MASK_SFT                                      (0x1 << 29)
+#define I2SIN_PAD_SEL_SFT                                     28
+#define I2SIN_PAD_SEL_MASK                                    0x1
+#define I2SIN_PAD_SEL_MASK_SFT                                (0x1 << 28)
+#define I2S_LOOPBACK_SFT                                      20
+#define I2S_LOOPBACK_MASK                                     0x1
+#define I2S_LOOPBACK_MASK_SFT                                 (0x1 << 20)
+#define I2S_HDEN_SFT                                          12
+#define I2S_HDEN_MASK                                         0x1
+#define I2S_HDEN_MASK_SFT                                     (0x1 << 12)
+#define I2S_MODE_SFT                                          8
+#define I2S_MODE_MASK                                         0xf
+#define I2S_MODE_MASK_SFT                                     (0xf << 8)
+#define I2S_BYPSRC_SFT                                        6
+#define I2S_BYPSRC_MASK                                       0x1
+#define I2S_BYPSRC_MASK_SFT                                   (0x1 << 6)
+#define INV_LRCK_SFT                                          5
+#define INV_LRCK_MASK                                         0x1
+#define INV_LRCK_MASK_SFT                                     (0x1 << 5)
+#define I2S_FMT_SFT                                           3
+#define I2S_FMT_MASK                                          0x1
+#define I2S_FMT_MASK_SFT                                      (0x1 << 3)
+#define I2S_SRC_SFT                                           2
+#define I2S_SRC_MASK                                          0x1
+#define I2S_SRC_MASK_SFT                                      (0x1 << 2)
+#define I2S_WLEN_SFT                                          1
+#define I2S_WLEN_MASK                                         0x1
+#define I2S_WLEN_MASK_SFT                                     (0x1 << 1)
+#define I2S_EN_SFT                                            0
+#define I2S_EN_MASK                                           0x1
+#define I2S_EN_MASK_SFT                                       (0x1 << 0)
+
+/* AFE_PCM0_INTF_CON0 */
+#define PCM0_HDEN_SFT                                         26
+#define PCM0_HDEN_MASK                                        0x1
+#define PCM0_HDEN_MASK_SFT                                    (0x1 << 26)
+#define PCM0_SYNC_DELSEL_SFT                                  25
+#define PCM0_SYNC_DELSEL_MASK                                 0x1
+#define PCM0_SYNC_DELSEL_MASK_SFT                             (0x1 << 25)
+#define PCM0_TX_LR_SWAP_SFT                                   24
+#define PCM0_TX_LR_SWAP_MASK                                  0x1
+#define PCM0_TX_LR_SWAP_MASK_SFT                              (0x1 << 24)
+#define PCM0_SYNC_OUT_INV_SFT                                 23
+#define PCM0_SYNC_OUT_INV_MASK                                0x1
+#define PCM0_SYNC_OUT_INV_MASK_SFT                            (0x1 << 23)
+#define PCM0_BCLK_OUT_INV_SFT                                 22
+#define PCM0_BCLK_OUT_INV_MASK                                0x1
+#define PCM0_BCLK_OUT_INV_MASK_SFT                            (0x1 << 22)
+#define PCM0_SYNC_IN_INV_SFT                                  21
+#define PCM0_SYNC_IN_INV_MASK                                 0x1
+#define PCM0_SYNC_IN_INV_MASK_SFT                             (0x1 << 21)
+#define PCM0_BCLK_IN_INV_SFT                                  20
+#define PCM0_BCLK_IN_INV_MASK                                 0x1
+#define PCM0_BCLK_IN_INV_MASK_SFT                             (0x1 << 20)
+#define PCM0_TX_LCH_RPT_SFT                                   19
+#define PCM0_TX_LCH_RPT_MASK                                  0x1
+#define PCM0_TX_LCH_RPT_MASK_SFT                              (0x1 << 19)
+#define PCM0_VBT_16K_MODE_SFT                                 18
+#define PCM0_VBT_16K_MODE_MASK                                0x1
+#define PCM0_VBT_16K_MODE_MASK_SFT                            (0x1 << 18)
+#define PCM0_BIT_LENGTH_SFT                                   16
+#define PCM0_BIT_LENGTH_MASK                                  0x3
+#define PCM0_BIT_LENGTH_MASK_SFT                              (0x3 << 16)
+#define PCM0_WLEN_SFT                                         14
+#define PCM0_WLEN_MASK                                        0x3
+#define PCM0_WLEN_MASK_SFT                                    (0x3 << 14)
+#define PCM0_SYNC_LENGTH_SFT                                  9
+#define PCM0_SYNC_LENGTH_MASK                                 0x1f
+#define PCM0_SYNC_LENGTH_MASK_SFT                             (0x1f << 9)
+#define PCM0_SYNC_TYPE_SFT                                    8
+#define PCM0_SYNC_TYPE_MASK                                   0x1
+#define PCM0_SYNC_TYPE_MASK_SFT                               (0x1 << 8)
+#define PCM0_BYP_ASRC_SFT                                     7
+#define PCM0_BYP_ASRC_MASK                                    0x1
+#define PCM0_BYP_ASRC_MASK_SFT                                (0x1 << 7)
+#define PCM0_SLAVE_SFT                                         6
+#define PCM0_SLAVE_MASK                                        0x1
+#define PCM0_SLAVE_MASK_SFT                                    (0x1 << 6)
+#define PCM0_MODE_SFT                                          3
+#define PCM0_MODE_MASK                                         0x7
+#define PCM0_MODE_MASK_SFT                                     (0x7 << 3)
+#define PCM0_FMT_SFT                                           1
+#define PCM0_FMT_MASK                                          0x3
+#define PCM0_FMT_MASK_SFT                                      (0x3 << 1)
+#define PCM0_EN_SFT                                            0
+#define PCM0_EN_MASK                                           0x1
+#define PCM0_EN_MASK_SFT                                       (0x1 << 0)
+
+/* AFE_PCM0_INTF_CON1 */
+#define PCM0_TX_RX_LOOPBACK_SFT                               31
+#define PCM0_TX_RX_LOOPBACK_MASK                              0x1
+#define PCM0_TX_RX_LOOPBACK_MASK_SFT                          (0x1 << 31)
+#define PCM0_BUFFER_LOOPBACK_SFT                              30
+#define PCM0_BUFFER_LOOPBACK_MASK                             0x1
+#define PCM0_BUFFER_LOOPBACK_MASK_SFT                         (0x1 << 30)
+#define PCM0_PARALLEL_LOOPBACK_SFT                            29
+#define PCM0_PARALLEL_LOOPBACK_MASK                           0x1
+#define PCM0_PARALLEL_LOOPBACK_MASK_SFT                       (0x1 << 29)
+#define PCM0_SERIAL_LOOPBACK_SFT                              28
+#define PCM0_SERIAL_LOOPBACK_MASK                             0x1
+#define PCM0_SERIAL_LOOPBACK_MASK_SFT                         (0x1 << 28)
+#define PCM0_DAI_LOOPBACK_SFT                                 27
+#define PCM0_DAI_LOOPBACK_MASK                                0x1
+#define PCM0_DAI_LOOPBACK_MASK_SFT                            (0x1 << 27)
+#define PCM0_I2S_LOOPBACK_SFT                                 26
+#define PCM0_I2S_LOOPBACK_MASK                                0x1
+#define PCM0_I2S_LOOPBACK_MASK_SFT                            (0x1 << 26)
+#define PCM0_1X_EN_DOMAIN_SFT                                 23
+#define PCM0_1X_EN_DOMAIN_MASK                                0x7
+#define PCM0_1X_EN_DOMAIN_MASK_SFT                            (0x7 << 23)
+#define PCM0_1X_EN_MODE_SFT                                   18
+#define PCM0_1X_EN_MODE_MASK                                  0x1f
+#define PCM0_1X_EN_MODE_MASK_SFT                              (0x1f << 18)
+#define PCM0_TX3_RCH_DBG_MODE_SFT                             17
+#define PCM0_TX3_RCH_DBG_MODE_MASK                            0x1
+#define PCM0_TX3_RCH_DBG_MODE_MASK_SFT                        (0x1 << 17)
+#define PCM0_PCM1_LOOPBACK_SFT                                16
+#define PCM0_PCM1_LOOPBACK_MASK                               0x1
+#define PCM0_PCM1_LOOPBACK_MASK_SFT                           (0x1 << 16)
+#define PCM0_LOOPBACK_CH_SEL_SFT                              12
+#define PCM0_LOOPBACK_CH_SEL_MASK                             0x3
+#define PCM0_LOOPBACK_CH_SEL_MASK_SFT                         (0x3 << 12)
+#define PCM0_BT_MODE_SFT                                      11
+#define PCM0_BT_MODE_MASK                                     0x1
+#define PCM0_BT_MODE_MASK_SFT                                 (0x1 << 11)
+#define PCM0_EXT_MODEM_SFT                                    10
+#define PCM0_EXT_MODEM_MASK                                   0x1
+#define PCM0_EXT_MODEM_MASK_SFT                               (0x1 << 10)
+#define PCM0_USE_MD3_SFT                                      9
+#define PCM0_USE_MD3_MASK                                     0x1
+#define PCM0_USE_MD3_MASK_SFT                                 (0x1 << 9)
+#define PCM0_FIX_VALUE_SEL_SFT                                8
+#define PCM0_FIX_VALUE_SEL_MASK                               0x1
+#define PCM0_FIX_VALUE_SEL_MASK_SFT                           (0x1 << 8)
+#define PCM0_TX_FIX_VALUE_SFT                                 0
+#define PCM0_TX_FIX_VALUE_MASK                                0xff
+#define PCM0_TX_FIX_VALUE_MASK_SFT                            (0xff << 0)
+
+/* AFE_PCM_INTF_MON */
+#define PCM0_TX_FIFO_OV_SFT                                   5
+#define PCM0_TX_FIFO_OV_MASK                                  0x1
+#define PCM0_TX_FIFO_OV_MASK_SFT                              (0x1 << 5)
+#define PCM0_RX_FIFO_OV_SFT                                   4
+#define PCM0_RX_FIFO_OV_MASK                                  0x1
+#define PCM0_RX_FIFO_OV_MASK_SFT                              (0x1 << 4)
+#define PCM1_TX_FIFO_OV_SFT                                   3
+#define PCM1_TX_FIFO_OV_MASK                                  0x1
+#define PCM1_TX_FIFO_OV_MASK_SFT                              (0x1 << 3)
+#define PCM1_RX_FIFO_OV_SFT                                   2
+#define PCM1_RX_FIFO_OV_MASK                                  0x1
+#define PCM1_RX_FIFO_OV_MASK_SFT                              (0x1 << 2)
+#define PCM0_SYNC_GLITCH_SFT                                  1
+#define PCM0_SYNC_GLITCH_MASK                                 0x1
+#define PCM0_SYNC_GLITCH_MASK_SFT                             (0x1 << 1)
+#define PCM1_SYNC_GLITCH_SFT                                  0
+#define PCM1_SYNC_GLITCH_MASK                                 0x1
+#define PCM1_SYNC_GLITCH_MASK_SFT                             (0x1 << 0)
+
+/* AFE_PCM1_INTF_CON0 */
+#define PCM1_TX_FIX_VALUE_SFT                                 24
+#define PCM1_TX_FIX_VALUE_MASK                                0xff
+#define PCM1_TX_FIX_VALUE_MASK_SFT                            (0xff << 24)
+#define PCM1_FIX_VALUE_SEL_SFT                                23
+#define PCM1_FIX_VALUE_SEL_MASK                               0x1
+#define PCM1_FIX_VALUE_SEL_MASK_SFT                           (0x1 << 23)
+#define PCM1_BUFFER_LOOPBACK_SFT                              22
+#define PCM1_BUFFER_LOOPBACK_MASK                             0x1
+#define PCM1_BUFFER_LOOPBACK_MASK_SFT                         (0x1 << 22)
+#define PCM1_PARALLEL_LOOPBACK_SFT                            21
+#define PCM1_PARALLEL_LOOPBACK_MASK                           0x1
+#define PCM1_PARALLEL_LOOPBACK_MASK_SFT                       (0x1 << 21)
+#define PCM1_SERIAL_LOOPBACK_SFT                              20
+#define PCM1_SERIAL_LOOPBACK_MASK                             0x1
+#define PCM1_SERIAL_LOOPBACK_MASK_SFT                         (0x1 << 20)
+#define PCM1_DAI_PCM1_LOOPBACK_SFT                            19
+#define PCM1_DAI_PCM1_LOOPBACK_MASK                           0x1
+#define PCM1_DAI_PCM1_LOOPBACK_MASK_SFT                       (0x1 << 19)
+#define PCM1_I2S_PCM1_LOOPBACK_SFT                            18
+#define PCM1_I2S_PCM1_LOOPBACK_MASK                           0x1
+#define PCM1_I2S_PCM1_LOOPBACK_MASK_SFT                       (0x1 << 18)
+#define PCM1_SYNC_DELSEL_SFT                                  17
+#define PCM1_SYNC_DELSEL_MASK                                 0x1
+#define PCM1_SYNC_DELSEL_MASK_SFT                             (0x1 << 17)
+#define PCM1_TX_LR_SWAP_SFT                                   16
+#define PCM1_TX_LR_SWAP_MASK                                  0x1
+#define PCM1_TX_LR_SWAP_MASK_SFT                              (0x1 << 16)
+#define PCM1_SYNC_IN_INV_SFT                                  15
+#define PCM1_SYNC_IN_INV_MASK                                 0x1
+#define PCM1_SYNC_IN_INV_MASK_SFT                             (0x1 << 15)
+#define PCM1_BCLK_IN_INV_SFT                                  14
+#define PCM1_BCLK_IN_INV_MASK                                 0x1
+#define PCM1_BCLK_IN_INV_MASK_SFT                             (0x1 << 14)
+#define PCM1_TX_LCH_RPT_SFT                                   13
+#define PCM1_TX_LCH_RPT_MASK                                  0x1
+#define PCM1_TX_LCH_RPT_MASK_SFT                              (0x1 << 13)
+#define PCM1_VBT_16K_MODE_SFT                                 12
+#define PCM1_VBT_16K_MODE_MASK                                0x1
+#define PCM1_VBT_16K_MODE_MASK_SFT                            (0x1 << 12)
+#define PCM1_LOOPBACK_CH_SEL_SFT                              10
+#define PCM1_LOOPBACK_CH_SEL_MASK                             0x3
+#define PCM1_LOOPBACK_CH_SEL_MASK_SFT                         (0x3 << 10)
+#define PCM1_TX2_BT_MODE_SFT                                  8
+#define PCM1_TX2_BT_MODE_MASK                                 0x1
+#define PCM1_TX2_BT_MODE_MASK_SFT                             (0x1 << 8)
+#define PCM1_BT_MODE_SFT                                      7
+#define PCM1_BT_MODE_MASK                                     0x1
+#define PCM1_BT_MODE_MASK_SFT                                 (0x1 << 7)
+#define PCM1_AFIFO_SFT                                        6
+#define PCM1_AFIFO_MASK                                       0x1
+#define PCM1_AFIFO_MASK_SFT                                   (0x1 << 6)
+#define PCM1_WLEN_SFT                                         5
+#define PCM1_WLEN_MASK                                        0x1
+#define PCM1_WLEN_MASK_SFT                                    (0x1 << 5)
+#define PCM1_MODE_SFT                                         3
+#define PCM1_MODE_MASK                                        0x3
+#define PCM1_MODE_MASK_SFT                                    (0x3 << 3)
+#define PCM1_FMT_SFT                                          1
+#define PCM1_FMT_MASK                                         0x3
+#define PCM1_FMT_MASK_SFT                                     (0x3 << 1)
+#define PCM1_EN_SFT                                           0
+#define PCM1_EN_MASK                                          0x1
+#define PCM1_EN_MASK_SFT                                      (0x1 << 0)
+
+/* AFE_PCM1_INTF_CON1 */
+#define PCM1_1X_EN_DOMAIN_SFT                                 23
+#define PCM1_1X_EN_DOMAIN_MASK                                0x7
+#define PCM1_1X_EN_DOMAIN_MASK_SFT                            (0x7 << 23)
+#define PCM1_1X_EN_MODE_SFT                                   18
+#define PCM1_1X_EN_MODE_MASK                                  0x1f
+#define PCM1_1X_EN_MODE_MASK_SFT                              (0x1f << 18)
+
+/* AFE_PCM_TOP_IP_VERSION */
+#define AFE_PCM_TOP_IP_VERSION_SFT                            0
+#define AFE_PCM_TOP_IP_VERSION_MASK                           0xffffffff
+#define AFE_PCM_TOP_IP_VERSION_MASK_SFT                       (0xffffffff << 0)
+
+/* AFE_IRQ_MCU_EN */
+#define AFE_IRQ_MCU_EN_SFT                                    0
+#define AFE_IRQ_MCU_EN_MASK                                   0xffffffff
+#define AFE_IRQ_MCU_EN_MASK_SFT                               (0xffffffff << 0)
+
+/* AFE_IRQ_MCU_DSP_EN */
+#define AFE_IRQ_DSP_EN_SFT                                    0
+#define AFE_IRQ_DSP_EN_MASK                                   0xffffffff
+#define AFE_IRQ_DSP_EN_MASK_SFT                               (0xffffffff << 0)
+
+/* AFE_IRQ_MCU_DSP2_EN */
+#define AFE_IRQ_DSP2_EN_SFT                                   0
+#define AFE_IRQ_DSP2_EN_MASK                                  0xffffffff
+#define AFE_IRQ_DSP2_EN_MASK_SFT                              (0xffffffff << 0)
+
+/* AFE_IRQ_MCU_SCP_EN */
+#define IRQ31_MCU_SCP_EN_SFT                                  31
+#define IRQ30_MCU_SCP_EN_SFT                                  30
+#define IRQ29_MCU_SCP_EN_SFT                                  29
+#define IRQ28_MCU_SCP_EN_SFT                                  28
+#define IRQ27_MCU_SCP_EN_SFT                                  27
+#define IRQ26_MCU_SCP_EN_SFT                                  26
+#define IRQ25_MCU_SCP_EN_SFT                                  25
+#define IRQ24_MCU_SCP_EN_SFT                                  24
+#define IRQ23_MCU_SCP_EN_SFT                                  23
+#define IRQ22_MCU_SCP_EN_SFT                                  22
+#define IRQ21_MCU_SCP_EN_SFT                                  21
+#define IRQ20_MCU_SCP_EN_SFT                                  20
+#define IRQ19_MCU_SCP_EN_SFT                                  19
+#define IRQ18_MCU_SCP_EN_SFT                                  18
+#define IRQ17_MCU_SCP_EN_SFT                                  17
+#define IRQ16_MCU_SCP_EN_SFT                                  16
+#define IRQ15_MCU_SCP_EN_SFT                                  15
+#define IRQ14_MCU_SCP_EN_SFT                                  14
+#define IRQ13_MCU_SCP_EN_SFT                                  13
+#define IRQ12_MCU_SCP_EN_SFT                                  12
+#define IRQ11_MCU_SCP_EN_SFT                                  11
+#define IRQ10_MCU_SCP_EN_SFT                                  10
+#define IRQ9_MCU_SCP_EN_SFT                                   9
+#define IRQ8_MCU_SCP_EN_SFT                                   8
+#define IRQ7_MCU_SCP_EN_SFT                                   7
+#define IRQ6_MCU_SCP_EN_SFT                                   6
+#define IRQ5_MCU_SCP_EN_SFT                                   5
+#define IRQ4_MCU_SCP_EN_SFT                                   4
+#define IRQ3_MCU_SCP_EN_SFT                                   3
+#define IRQ2_MCU_SCP_EN_SFT                                   2
+#define IRQ1_MCU_SCP_EN_SFT                                   1
+#define IRQ0_MCU_SCP_EN_SFT                                   0
+
+/* AFE_CUSTOM_IRQ_MCU_EN */
+#define AFE_CUSTOM_IRQ_MCU_EN_SFT                             0
+#define AFE_CUSTOM_IRQ_MCU_EN_MASK                            0xffffffff
+#define AFE_CUSTOM_IRQ_MCU_EN_MASK_SFT                        (0xffffffff << 0)
+
+/* AFE_CUSTOM_IRQ_MCU_DSP_EN */
+#define AFE_CUSTOM_IRQ_DSP_EN_SFT                             0
+#define AFE_CUSTOM_IRQ_DSP_EN_MASK                            0xffffffff
+#define AFE_CUSTOM_IRQ_DSP_EN_MASK_SFT                        (0xffffffff << 0)
+
+/* AFE_CUSTOM_IRQ_MCU_DSP2_EN */
+#define AFE_CUSTOM_IRQ_DSP2_EN_SFT                            0
+#define AFE_CUSTOM_IRQ_DSP2_EN_MASK                           0xffffffff
+#define AFE_CUSTOM_IRQ_DSP2_EN_MASK_SFT                       (0xffffffff << 0)
+
+/* AFE_CUSTOM_IRQ_MCU_SCP_EN */
+#define AFE_CUSTOM_IRQ_SCP_EN_SFT                             0
+#define AFE_CUSTOM_IRQ_SCP_EN_MASK                            0xffffffff
+#define AFE_CUSTOM_IRQ_SCP_EN_MASK_SFT                        (0xffffffff << 0)
+
+/* AFE_IRQ_MCU_STATUS */
+#define IRQ26_MCU_SFT                                         26
+#define IRQ26_MCU_MASK                                        0x1
+#define IRQ26_MCU_MASK_SFT                                    (0x1 << 26)
+#define IRQ25_MCU_SFT                                         25
+#define IRQ25_MCU_MASK                                        0x1
+#define IRQ25_MCU_MASK_SFT                                    (0x1 << 25)
+#define IRQ24_MCU_SFT                                         24
+#define IRQ24_MCU_MASK                                        0x1
+#define IRQ24_MCU_MASK_SFT                                    (0x1 << 24)
+#define IRQ23_MCU_SFT                                         23
+#define IRQ23_MCU_MASK                                        0x1
+#define IRQ23_MCU_MASK_SFT                                    (0x1 << 23)
+#define IRQ22_MCU_SFT                                         22
+#define IRQ22_MCU_MASK                                        0x1
+#define IRQ22_MCU_MASK_SFT                                    (0x1 << 22)
+#define IRQ21_MCU_SFT                                         21
+#define IRQ21_MCU_MASK                                        0x1
+#define IRQ21_MCU_MASK_SFT                                    (0x1 << 21)
+#define IRQ20_MCU_SFT                                         20
+#define IRQ20_MCU_MASK                                        0x1
+#define IRQ20_MCU_MASK_SFT                                    (0x1 << 20)
+#define IRQ19_MCU_SFT                                         19
+#define IRQ19_MCU_MASK                                        0x1
+#define IRQ19_MCU_MASK_SFT                                    (0x1 << 19)
+#define IRQ18_MCU_SFT                                         18
+#define IRQ18_MCU_MASK                                        0x1
+#define IRQ18_MCU_MASK_SFT                                    (0x1 << 18)
+#define IRQ17_MCU_SFT                                         17
+#define IRQ17_MCU_MASK                                        0x1
+#define IRQ17_MCU_MASK_SFT                                    (0x1 << 17)
+#define IRQ16_MCU_SFT                                         16
+#define IRQ16_MCU_MASK                                        0x1
+#define IRQ16_MCU_MASK_SFT                                    (0x1 << 16)
+#define IRQ15_MCU_SFT                                         15
+#define IRQ15_MCU_MASK                                        0x1
+#define IRQ15_MCU_MASK_SFT                                    (0x1 << 15)
+#define IRQ14_MCU_SFT                                         14
+#define IRQ14_MCU_MASK                                        0x1
+#define IRQ14_MCU_MASK_SFT                                    (0x1 << 14)
+#define IRQ13_MCU_SFT                                         13
+#define IRQ13_MCU_MASK                                        0x1
+#define IRQ13_MCU_MASK_SFT                                    (0x1 << 13)
+#define IRQ12_MCU_SFT                                         12
+#define IRQ12_MCU_MASK                                        0x1
+#define IRQ12_MCU_MASK_SFT                                    (0x1 << 12)
+#define IRQ11_MCU_SFT                                         11
+#define IRQ11_MCU_MASK                                        0x1
+#define IRQ11_MCU_MASK_SFT                                    (0x1 << 11)
+#define IRQ10_MCU_SFT                                         10
+#define IRQ10_MCU_MASK                                        0x1
+#define IRQ10_MCU_MASK_SFT                                    (0x1 << 10)
+#define IRQ9_MCU_SFT                                          9
+#define IRQ9_MCU_MASK                                         0x1
+#define IRQ9_MCU_MASK_SFT                                     (0x1 << 9)
+#define IRQ8_MCU_SFT                                          8
+#define IRQ8_MCU_MASK                                         0x1
+#define IRQ8_MCU_MASK_SFT                                     (0x1 << 8)
+#define IRQ7_MCU_SFT                                          7
+#define IRQ7_MCU_MASK                                         0x1
+#define IRQ7_MCU_MASK_SFT                                     (0x1 << 7)
+#define IRQ6_MCU_SFT                                          6
+#define IRQ6_MCU_MASK                                         0x1
+#define IRQ6_MCU_MASK_SFT                                     (0x1 << 6)
+#define IRQ5_MCU_SFT                                          5
+#define IRQ5_MCU_MASK                                         0x1
+#define IRQ5_MCU_MASK_SFT                                     (0x1 << 5)
+#define IRQ4_MCU_SFT                                          4
+#define IRQ4_MCU_MASK                                         0x1
+#define IRQ4_MCU_MASK_SFT                                     (0x1 << 4)
+#define IRQ3_MCU_SFT                                          3
+#define IRQ3_MCU_MASK                                         0x1
+#define IRQ3_MCU_MASK_SFT                                     (0x1 << 3)
+#define IRQ2_MCU_SFT                                          2
+#define IRQ2_MCU_MASK                                         0x1
+#define IRQ2_MCU_MASK_SFT                                     (0x1 << 2)
+#define IRQ1_MCU_SFT                                          1
+#define IRQ1_MCU_MASK                                         0x1
+#define IRQ1_MCU_MASK_SFT                                     (0x1 << 1)
+#define IRQ0_MCU_SFT                                          0
+#define IRQ0_MCU_MASK                                         0x1
+#define IRQ0_MCU_MASK_SFT                                     (0x1 << 0)
+
+/* AFE_CUSTOM_IRQ_MCU_STATUS */
+#define CUSTOM_IRQ21_MCU_SFT                                  21
+#define CUSTOM_IRQ21_MCU_MASK                                 0x1
+#define CUSTOM_IRQ21_MCU_MASK_SFT                             (0x1 << 21)
+#define CUSTOM_IRQ20_MCU_SFT                                  20
+#define CUSTOM_IRQ20_MCU_MASK                                 0x1
+#define CUSTOM_IRQ20_MCU_MASK_SFT                             (0x1 << 20)
+#define CUSTOM_IRQ19_MCU_SFT                                  19
+#define CUSTOM_IRQ19_MCU_MASK                                 0x1
+#define CUSTOM_IRQ19_MCU_MASK_SFT                             (0x1 << 19)
+#define CUSTOM_IRQ18_MCU_SFT                                  18
+#define CUSTOM_IRQ18_MCU_MASK                                 0x1
+#define CUSTOM_IRQ18_MCU_MASK_SFT                             (0x1 << 18)
+#define CUSTOM_IRQ17_MCU_SFT                                  17
+#define CUSTOM_IRQ17_MCU_MASK                                 0x1
+#define CUSTOM_IRQ17_MCU_MASK_SFT                             (0x1 << 17)
+#define CUSTOM_IRQ16_MCU_SFT                                  16
+#define CUSTOM_IRQ16_MCU_MASK                                 0x1
+#define CUSTOM_IRQ16_MCU_MASK_SFT                             (0x1 << 16)
+#define CUSTOM_IRQ9_MCU_SFT                                   9
+#define CUSTOM_IRQ9_MCU_MASK                                  0x1
+#define CUSTOM_IRQ9_MCU_MASK_SFT                              (0x1 << 9)
+#define CUSTOM_IRQ8_MCU_SFT                                   8
+#define CUSTOM_IRQ8_MCU_MASK                                  0x1
+#define CUSTOM_IRQ8_MCU_MASK_SFT                              (0x1 << 8)
+#define CUSTOM_IRQ7_MCU_SFT                                   7
+#define CUSTOM_IRQ7_MCU_MASK                                  0x1
+#define CUSTOM_IRQ7_MCU_MASK_SFT                              (0x1 << 7)
+#define CUSTOM_IRQ6_MCU_SFT                                   6
+#define CUSTOM_IRQ6_MCU_MASK                                  0x1
+#define CUSTOM_IRQ6_MCU_MASK_SFT                              (0x1 << 6)
+#define CUSTOM_IRQ5_MCU_SFT                                   5
+#define CUSTOM_IRQ5_MCU_MASK                                  0x1
+#define CUSTOM_IRQ5_MCU_MASK_SFT                              (0x1 << 5)
+#define CUSTOM_IRQ4_MCU_SFT                                   4
+#define CUSTOM_IRQ4_MCU_MASK                                  0x1
+#define CUSTOM_IRQ4_MCU_MASK_SFT                              (0x1 << 4)
+#define CUSTOM_IRQ3_MCU_SFT                                   3
+#define CUSTOM_IRQ3_MCU_MASK                                  0x1
+#define CUSTOM_IRQ3_MCU_MASK_SFT                              (0x1 << 3)
+#define CUSTOM_IRQ2_MCU_SFT                                   2
+#define CUSTOM_IRQ2_MCU_MASK                                  0x1
+#define CUSTOM_IRQ2_MCU_MASK_SFT                              (0x1 << 2)
+#define CUSTOM_IRQ1_MCU_SFT                                   1
+#define CUSTOM_IRQ1_MCU_MASK                                  0x1
+#define CUSTOM_IRQ1_MCU_MASK_SFT                              (0x1 << 1)
+#define CUSTOM_IRQ0_MCU_SFT                                   0
+#define CUSTOM_IRQ0_MCU_MASK                                  0x1
+#define CUSTOM_IRQ0_MCU_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ_MCU_CFG */
+#define AFE_IRQ_CLR_CFG_SFT                                   31
+#define AFE_IRQ_CLR_CFG_MASK                                  0x1
+#define AFE_IRQ_CLR_CFG_MASK_SFT                              (0x1 << 31)
+#define AFE_IRQ_MISS_FLAG_CLR_CFG_SFT                         30
+#define AFE_IRQ_MISS_FLAG_CLR_CFG_MASK                        0x1
+#define AFE_IRQ_MISS_FLAG_CLR_CFG_MASK_SFT                    (0x1 << 30)
+#define AFE_IRQ_MCU_CNT_SFT                                   0
+#define AFE_IRQ_MCU_CNT_MASK                                  0xffffff
+#define AFE_IRQ_MCU_CNT_MASK_SFT                              (0xffffff << 0)
+
+/* AFE_IRQ0_MCU_CFG0 */
+#define AFE_IRQ0_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ0_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ0_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ0_MCU_FS_SFT                                   4
+#define AFE_IRQ0_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ0_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ0_MCU_ON_SFT                                   0
+#define AFE_IRQ0_MCU_ON_MASK                                  0x1
+#define AFE_IRQ0_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ0_MCU_CFG1 */
+#define AFE_IRQ0_CLR_CFG_SFT                                  31
+#define AFE_IRQ0_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ0_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ0_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ0_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ0_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ0_MCU_CNT_SFT                                  0
+#define AFE_IRQ0_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ0_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ1_MCU_CFG0 */
+#define AFE_IRQ1_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ1_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ1_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ1_MCU_FS_SFT                                   4
+#define AFE_IRQ1_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ1_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ1_MCU_ON_SFT                                   0
+#define AFE_IRQ1_MCU_ON_MASK                                  0x1
+#define AFE_IRQ1_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ1_MCU_CFG1 */
+#define AFE_IRQ1_CLR_CFG_SFT                                  31
+#define AFE_IRQ1_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ1_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ1_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ1_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ1_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ1_MCU_CNT_SFT                                  0
+#define AFE_IRQ1_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ1_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ2_MCU_CFG0 */
+#define AFE_IRQ2_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ2_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ2_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ2_MCU_FS_SFT                                   4
+#define AFE_IRQ2_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ2_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ2_MCU_ON_SFT                                   0
+#define AFE_IRQ2_MCU_ON_MASK                                  0x1
+#define AFE_IRQ2_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ2_MCU_CFG1 */
+#define AFE_IRQ2_CLR_CFG_SFT                                  31
+#define AFE_IRQ2_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ2_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ2_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ2_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ2_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ2_MCU_CNT_SFT                                  0
+#define AFE_IRQ2_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ2_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ3_MCU_CFG0 */
+#define AFE_IRQ3_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ3_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ3_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ3_MCU_FS_SFT                                   4
+#define AFE_IRQ3_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ3_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ3_MCU_ON_SFT                                   0
+#define AFE_IRQ3_MCU_ON_MASK                                  0x1
+#define AFE_IRQ3_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ3_MCU_CFG1 */
+#define AFE_IRQ3_CLR_CFG_SFT                                  31
+#define AFE_IRQ3_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ3_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ3_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ3_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ3_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ3_MCU_CNT_SFT                                  0
+#define AFE_IRQ3_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ3_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ4_MCU_CFG0 */
+#define AFE_IRQ4_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ4_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ4_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ4_MCU_FS_SFT                                   4
+#define AFE_IRQ4_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ4_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ4_MCU_ON_SFT                                   0
+#define AFE_IRQ4_MCU_ON_MASK                                  0x1
+#define AFE_IRQ4_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ4_MCU_CFG1 */
+#define AFE_IRQ4_CLR_CFG_SFT                                  31
+#define AFE_IRQ4_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ4_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ4_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ4_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ4_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ4_MCU_CNT_SFT                                  0
+#define AFE_IRQ4_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ4_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ5_MCU_CFG0 */
+#define AFE_IRQ5_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ5_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ5_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ5_MCU_FS_SFT                                   4
+#define AFE_IRQ5_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ5_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ5_MCU_ON_SFT                                   0
+#define AFE_IRQ5_MCU_ON_MASK                                  0x1
+#define AFE_IRQ5_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ5_MCU_CFG1 */
+#define AFE_IRQ5_CLR_CFG_SFT                                  31
+#define AFE_IRQ5_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ5_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ5_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ5_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ5_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ5_MCU_CNT_SFT                                  0
+#define AFE_IRQ5_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ5_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ6_MCU_CFG0 */
+#define AFE_IRQ6_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ6_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ6_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ6_MCU_FS_SFT                                   4
+#define AFE_IRQ6_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ6_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ6_MCU_ON_SFT                                   0
+#define AFE_IRQ6_MCU_ON_MASK                                  0x1
+#define AFE_IRQ6_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ6_MCU_CFG1 */
+#define AFE_IRQ6_CLR_CFG_SFT                                  31
+#define AFE_IRQ6_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ6_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ6_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ6_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ6_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ6_MCU_CNT_SFT                                  0
+#define AFE_IRQ6_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ6_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ7_MCU_CFG0 */
+#define AFE_IRQ7_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ7_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ7_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ7_MCU_FS_SFT                                   4
+#define AFE_IRQ7_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ7_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ7_MCU_ON_SFT                                   0
+#define AFE_IRQ7_MCU_ON_MASK                                  0x1
+#define AFE_IRQ7_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ7_MCU_CFG1 */
+#define AFE_IRQ7_CLR_CFG_SFT                                  31
+#define AFE_IRQ7_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ7_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ7_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ7_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ7_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ7_MCU_CNT_SFT                                  0
+#define AFE_IRQ7_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ7_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ8_MCU_CFG0 */
+#define AFE_IRQ8_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ8_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ8_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ8_MCU_FS_SFT                                   4
+#define AFE_IRQ8_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ8_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ8_MCU_ON_SFT                                   0
+#define AFE_IRQ8_MCU_ON_MASK                                  0x1
+#define AFE_IRQ8_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ8_MCU_CFG1 */
+#define AFE_IRQ8_CLR_CFG_SFT                                  31
+#define AFE_IRQ8_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ8_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ8_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ8_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ8_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ8_MCU_CNT_SFT                                  0
+#define AFE_IRQ8_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ8_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ9_MCU_CFG0 */
+#define AFE_IRQ9_MCU_DOMAIN_SFT                               9
+#define AFE_IRQ9_MCU_DOMAIN_MASK                              0x7
+#define AFE_IRQ9_MCU_DOMAIN_MASK_SFT                          (0x7 << 9)
+#define AFE_IRQ9_MCU_FS_SFT                                   4
+#define AFE_IRQ9_MCU_FS_MASK                                  0x1f
+#define AFE_IRQ9_MCU_FS_MASK_SFT                              (0x1f << 4)
+#define AFE_IRQ9_MCU_ON_SFT                                   0
+#define AFE_IRQ9_MCU_ON_MASK                                  0x1
+#define AFE_IRQ9_MCU_ON_MASK_SFT                              (0x1 << 0)
+
+/* AFE_IRQ9_MCU_CFG1 */
+#define AFE_IRQ9_CLR_CFG_SFT                                  31
+#define AFE_IRQ9_CLR_CFG_MASK                                 0x1
+#define AFE_IRQ9_CLR_CFG_MASK_SFT                             (0x1 << 31)
+#define AFE_IRQ9_MISS_FLAG_CLR_CFG_SFT                        30
+#define AFE_IRQ9_MISS_FLAG_CLR_CFG_MASK                       0x1
+#define AFE_IRQ9_MISS_FLAG_CLR_CFG_MASK_SFT                   (0x1 << 30)
+#define AFE_IRQ9_MCU_CNT_SFT                                  0
+#define AFE_IRQ9_MCU_CNT_MASK                                 0xffffff
+#define AFE_IRQ9_MCU_CNT_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ10_MCU_CFG0 */
+#define AFE_IRQ10_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ10_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ10_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ10_MCU_FS_SFT                                  4
+#define AFE_IRQ10_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ10_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ10_MCU_ON_SFT                                  0
+#define AFE_IRQ10_MCU_ON_MASK                                 0x1
+#define AFE_IRQ10_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ10_MCU_CFG1 */
+#define AFE_IRQ10_CLR_CFG_SFT                                 31
+#define AFE_IRQ10_CLR_CFG_MASK                                0x1
+#define AFE_IRQ10_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ10_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ10_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ10_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ10_MCU_CNT_SFT                                 0
+#define AFE_IRQ10_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ10_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ11_MCU_CFG0 */
+#define AFE_IRQ11_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ11_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ11_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ11_MCU_FS_SFT                                  4
+#define AFE_IRQ11_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ11_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ11_MCU_ON_SFT                                  0
+#define AFE_IRQ11_MCU_ON_MASK                                 0x1
+#define AFE_IRQ11_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ11_MCU_CFG1 */
+#define AFE_IRQ11_CLR_CFG_SFT                                 31
+#define AFE_IRQ11_CLR_CFG_MASK                                0x1
+#define AFE_IRQ11_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ11_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ11_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ11_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ11_MCU_CNT_SFT                                 0
+#define AFE_IRQ11_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ11_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ12_MCU_CFG0 */
+#define AFE_IRQ12_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ12_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ12_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ12_MCU_FS_SFT                                  4
+#define AFE_IRQ12_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ12_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ12_MCU_ON_SFT                                  0
+#define AFE_IRQ12_MCU_ON_MASK                                 0x1
+#define AFE_IRQ12_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ12_MCU_CFG1 */
+#define AFE_IRQ12_CLR_CFG_SFT                                 31
+#define AFE_IRQ12_CLR_CFG_MASK                                0x1
+#define AFE_IRQ12_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ12_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ12_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ12_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ12_MCU_CNT_SFT                                 0
+#define AFE_IRQ12_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ12_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ13_MCU_CFG0 */
+#define AFE_IRQ13_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ13_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ13_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ13_MCU_FS_SFT                                  4
+#define AFE_IRQ13_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ13_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ13_MCU_ON_SFT                                  0
+#define AFE_IRQ13_MCU_ON_MASK                                 0x1
+#define AFE_IRQ13_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ13_MCU_CFG1 */
+#define AFE_IRQ13_CLR_CFG_SFT                                 31
+#define AFE_IRQ13_CLR_CFG_MASK                                0x1
+#define AFE_IRQ13_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ13_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ13_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ13_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ13_MCU_CNT_SFT                                 0
+#define AFE_IRQ13_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ13_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ14_MCU_CFG0 */
+#define AFE_IRQ14_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ14_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ14_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ14_MCU_FS_SFT                                  4
+#define AFE_IRQ14_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ14_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ14_MCU_ON_SFT                                  0
+#define AFE_IRQ14_MCU_ON_MASK                                 0x1
+#define AFE_IRQ14_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ14_MCU_CFG1 */
+#define AFE_IRQ14_CLR_CFG_SFT                                 31
+#define AFE_IRQ14_CLR_CFG_MASK                                0x1
+#define AFE_IRQ14_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ14_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ14_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ14_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ14_MCU_CNT_SFT                                 0
+#define AFE_IRQ14_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ14_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ15_MCU_CFG0 */
+#define AFE_IRQ15_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ15_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ15_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ15_MCU_FS_SFT                                  4
+#define AFE_IRQ15_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ15_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ15_MCU_ON_SFT                                  0
+#define AFE_IRQ15_MCU_ON_MASK                                 0x1
+#define AFE_IRQ15_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ15_MCU_CFG1 */
+#define AFE_IRQ15_CLR_CFG_SFT                                 31
+#define AFE_IRQ15_CLR_CFG_MASK                                0x1
+#define AFE_IRQ15_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ15_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ15_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ15_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ15_MCU_CNT_SFT                                 0
+#define AFE_IRQ15_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ15_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ16_MCU_CFG0 */
+#define AFE_IRQ16_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ16_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ16_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ16_MCU_FS_SFT                                  4
+#define AFE_IRQ16_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ16_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ16_MCU_ON_SFT                                  0
+#define AFE_IRQ16_MCU_ON_MASK                                 0x1
+#define AFE_IRQ16_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ16_MCU_CFG1 */
+#define AFE_IRQ16_CLR_CFG_SFT                                 31
+#define AFE_IRQ16_CLR_CFG_MASK                                0x1
+#define AFE_IRQ16_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ16_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ16_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ16_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ16_MCU_CNT_SFT                                 0
+#define AFE_IRQ16_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ16_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ17_MCU_CFG0 */
+#define AFE_IRQ17_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ17_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ17_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ17_MCU_FS_SFT                                  4
+#define AFE_IRQ17_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ17_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ17_MCU_ON_SFT                                  0
+#define AFE_IRQ17_MCU_ON_MASK                                 0x1
+#define AFE_IRQ17_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ17_MCU_CFG1 */
+#define AFE_IRQ17_CLR_CFG_SFT                                 31
+#define AFE_IRQ17_CLR_CFG_MASK                                0x1
+#define AFE_IRQ17_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ17_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ17_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ17_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ17_MCU_CNT_SFT                                 0
+#define AFE_IRQ17_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ17_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ18_MCU_CFG0 */
+#define AFE_IRQ18_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ18_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ18_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ18_MCU_FS_SFT                                  4
+#define AFE_IRQ18_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ18_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ18_MCU_ON_SFT                                  0
+#define AFE_IRQ18_MCU_ON_MASK                                 0x1
+#define AFE_IRQ18_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ18_MCU_CFG1 */
+#define AFE_IRQ18_CLR_CFG_SFT                                 31
+#define AFE_IRQ18_CLR_CFG_MASK                                0x1
+#define AFE_IRQ18_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ18_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ18_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ18_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ18_MCU_CNT_SFT                                 0
+#define AFE_IRQ18_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ18_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ19_MCU_CFG0 */
+#define AFE_IRQ19_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ19_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ19_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ19_MCU_FS_SFT                                  4
+#define AFE_IRQ19_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ19_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ19_MCU_ON_SFT                                  0
+#define AFE_IRQ19_MCU_ON_MASK                                 0x1
+#define AFE_IRQ19_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ19_MCU_CFG1 */
+#define AFE_IRQ19_CLR_CFG_SFT                                 31
+#define AFE_IRQ19_CLR_CFG_MASK                                0x1
+#define AFE_IRQ19_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ19_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ19_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ19_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ19_MCU_CNT_SFT                                 0
+#define AFE_IRQ19_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ19_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ20_MCU_CFG0 */
+#define AFE_IRQ20_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ20_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ20_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ20_MCU_FS_SFT                                  4
+#define AFE_IRQ20_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ20_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ20_MCU_ON_SFT                                  0
+#define AFE_IRQ20_MCU_ON_MASK                                 0x1
+#define AFE_IRQ20_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ20_MCU_CFG1 */
+#define AFE_IRQ20_CLR_CFG_SFT                                 31
+#define AFE_IRQ20_CLR_CFG_MASK                                0x1
+#define AFE_IRQ20_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ20_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ20_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ20_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ20_MCU_CNT_SFT                                 0
+#define AFE_IRQ20_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ20_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ21_MCU_CFG0 */
+#define AFE_IRQ21_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ21_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ21_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ21_MCU_FS_SFT                                  4
+#define AFE_IRQ21_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ21_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ21_MCU_ON_SFT                                  0
+#define AFE_IRQ21_MCU_ON_MASK                                 0x1
+#define AFE_IRQ21_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ21_MCU_CFG1 */
+#define AFE_IRQ21_CLR_CFG_SFT                                 31
+#define AFE_IRQ21_CLR_CFG_MASK                                0x1
+#define AFE_IRQ21_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ21_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ21_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ21_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ21_MCU_CNT_SFT                                 0
+#define AFE_IRQ21_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ21_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ22_MCU_CFG0 */
+#define AFE_IRQ22_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ22_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ22_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ22_MCU_FS_SFT                                  4
+#define AFE_IRQ22_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ22_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ22_MCU_ON_SFT                                  0
+#define AFE_IRQ22_MCU_ON_MASK                                 0x1
+#define AFE_IRQ22_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ22_MCU_CFG1 */
+#define AFE_IRQ22_CLR_CFG_SFT                                 31
+#define AFE_IRQ22_CLR_CFG_MASK                                0x1
+#define AFE_IRQ22_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ22_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ22_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ22_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ22_MCU_CNT_SFT                                 0
+#define AFE_IRQ22_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ22_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ23_MCU_CFG0 */
+#define AFE_IRQ23_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ23_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ23_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ23_MCU_FS_SFT                                  4
+#define AFE_IRQ23_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ23_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ23_MCU_ON_SFT                                  0
+#define AFE_IRQ23_MCU_ON_MASK                                 0x1
+#define AFE_IRQ23_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ23_MCU_CFG1 */
+#define AFE_IRQ23_CLR_CFG_SFT                                 31
+#define AFE_IRQ23_CLR_CFG_MASK                                0x1
+#define AFE_IRQ23_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ23_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ23_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ23_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ23_MCU_CNT_SFT                                 0
+#define AFE_IRQ23_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ23_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ24_MCU_CFG0 */
+#define AFE_IRQ24_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ24_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ24_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ24_MCU_FS_SFT                                  4
+#define AFE_IRQ24_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ24_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ24_MCU_ON_SFT                                  0
+#define AFE_IRQ24_MCU_ON_MASK                                 0x1
+#define AFE_IRQ24_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ24_MCU_CFG1 */
+#define AFE_IRQ24_CLR_CFG_SFT                                 31
+#define AFE_IRQ24_CLR_CFG_MASK                                0x1
+#define AFE_IRQ24_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ24_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ24_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ24_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ24_MCU_CNT_SFT                                 0
+#define AFE_IRQ24_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ24_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ25_MCU_CFG0 */
+#define AFE_IRQ25_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ25_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ25_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ25_MCU_FS_SFT                                  4
+#define AFE_IRQ25_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ25_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ25_MCU_ON_SFT                                  0
+#define AFE_IRQ25_MCU_ON_MASK                                 0x1
+#define AFE_IRQ25_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ25_MCU_CFG1 */
+#define AFE_IRQ25_CLR_CFG_SFT                                 31
+#define AFE_IRQ25_CLR_CFG_MASK                                0x1
+#define AFE_IRQ25_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ25_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ25_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ25_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ25_MCU_CNT_SFT                                 0
+#define AFE_IRQ25_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ25_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ26_MCU_CFG0 */
+#define AFE_IRQ26_MCU_DOMAIN_SFT                              9
+#define AFE_IRQ26_MCU_DOMAIN_MASK                             0x7
+#define AFE_IRQ26_MCU_DOMAIN_MASK_SFT                         (0x7 << 9)
+#define AFE_IRQ26_MCU_FS_SFT                                  4
+#define AFE_IRQ26_MCU_FS_MASK                                 0x1f
+#define AFE_IRQ26_MCU_FS_MASK_SFT                             (0x1f << 4)
+#define AFE_IRQ26_MCU_ON_SFT                                  0
+#define AFE_IRQ26_MCU_ON_MASK                                 0x1
+#define AFE_IRQ26_MCU_ON_MASK_SFT                             (0x1 << 0)
+
+/* AFE_IRQ26_MCU_CFG1 */
+#define AFE_IRQ26_CLR_CFG_SFT                                 31
+#define AFE_IRQ26_CLR_CFG_MASK                                0x1
+#define AFE_IRQ26_CLR_CFG_MASK_SFT                            (0x1 << 31)
+#define AFE_IRQ26_MISS_FLAG_CLR_CFG_SFT                       30
+#define AFE_IRQ26_MISS_FLAG_CLR_CFG_MASK                      0x1
+#define AFE_IRQ26_MISS_FLAG_CLR_CFG_MASK_SFT                  (0x1 << 30)
+#define AFE_IRQ26_MCU_CNT_SFT                                 0
+#define AFE_IRQ26_MCU_CNT_MASK                                0xffffff
+#define AFE_IRQ26_MCU_CNT_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_CUSTOM_IRQ0_MCU_CFG0 */
+#define AFE_CUSTOM_IRQ0_MCU_ON_SFT                            0
+#define AFE_CUSTOM_IRQ0_MCU_ON_MASK                           0x1
+#define AFE_CUSTOM_IRQ0_MCU_ON_MASK_SFT                       (0x1 << 0)
+
+/* AFE_IRQ_MCU_MON0 */
+#define AFE_IRQ26_MISS_FLAG_SFT                               26
+#define AFE_IRQ26_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ26_MISS_FLAG_MASK_SFT                          (0x1 << 26)
+#define AFE_IRQ25_MISS_FLAG_SFT                               25
+#define AFE_IRQ25_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ25_MISS_FLAG_MASK_SFT                          (0x1 << 25)
+#define AFE_IRQ24_MISS_FLAG_SFT                               24
+#define AFE_IRQ24_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ24_MISS_FLAG_MASK_SFT                          (0x1 << 24)
+#define AFE_IRQ23_MISS_FLAG_SFT                               23
+#define AFE_IRQ23_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ23_MISS_FLAG_MASK_SFT                          (0x1 << 23)
+#define AFE_IRQ22_MISS_FLAG_SFT                               22
+#define AFE_IRQ22_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ22_MISS_FLAG_MASK_SFT                          (0x1 << 22)
+#define AFE_IRQ21_MISS_FLAG_SFT                               21
+#define AFE_IRQ21_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ21_MISS_FLAG_MASK_SFT                          (0x1 << 21)
+#define AFE_IRQ20_MISS_FLAG_SFT                               20
+#define AFE_IRQ20_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ20_MISS_FLAG_MASK_SFT                          (0x1 << 20)
+#define AFE_IRQ19_MISS_FLAG_SFT                               19
+#define AFE_IRQ19_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ19_MISS_FLAG_MASK_SFT                          (0x1 << 19)
+#define AFE_IRQ18_MISS_FLAG_SFT                               18
+#define AFE_IRQ18_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ18_MISS_FLAG_MASK_SFT                          (0x1 << 18)
+#define AFE_IRQ17_MISS_FLAG_SFT                               17
+#define AFE_IRQ17_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ17_MISS_FLAG_MASK_SFT                          (0x1 << 17)
+#define AFE_IRQ16_MISS_FLAG_SFT                               16
+#define AFE_IRQ16_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ16_MISS_FLAG_MASK_SFT                          (0x1 << 16)
+#define AFE_IRQ15_MISS_FLAG_SFT                               15
+#define AFE_IRQ15_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ15_MISS_FLAG_MASK_SFT                          (0x1 << 15)
+#define AFE_IRQ14_MISS_FLAG_SFT                               14
+#define AFE_IRQ14_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ14_MISS_FLAG_MASK_SFT                          (0x1 << 14)
+#define AFE_IRQ13_MISS_FLAG_SFT                               13
+#define AFE_IRQ13_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ13_MISS_FLAG_MASK_SFT                          (0x1 << 13)
+#define AFE_IRQ12_MISS_FLAG_SFT                               12
+#define AFE_IRQ12_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ12_MISS_FLAG_MASK_SFT                          (0x1 << 12)
+#define AFE_IRQ11_MISS_FLAG_SFT                               11
+#define AFE_IRQ11_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ11_MISS_FLAG_MASK_SFT                          (0x1 << 11)
+#define AFE_IRQ10_MISS_FLAG_SFT                               10
+#define AFE_IRQ10_MISS_FLAG_MASK                              0x1
+#define AFE_IRQ10_MISS_FLAG_MASK_SFT                          (0x1 << 10)
+#define AFE_IRQ9_MISS_FLAG_SFT                                9
+#define AFE_IRQ9_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ9_MISS_FLAG_MASK_SFT                           (0x1 << 9)
+#define AFE_IRQ8_MISS_FLAG_SFT                                8
+#define AFE_IRQ8_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ8_MISS_FLAG_MASK_SFT                           (0x1 << 8)
+#define AFE_IRQ7_MISS_FLAG_SFT                                7
+#define AFE_IRQ7_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ7_MISS_FLAG_MASK_SFT                           (0x1 << 7)
+#define AFE_IRQ6_MISS_FLAG_SFT                                6
+#define AFE_IRQ6_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ6_MISS_FLAG_MASK_SFT                           (0x1 << 6)
+#define AFE_IRQ5_MISS_FLAG_SFT                                5
+#define AFE_IRQ5_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ5_MISS_FLAG_MASK_SFT                           (0x1 << 5)
+#define AFE_IRQ4_MISS_FLAG_SFT                                4
+#define AFE_IRQ4_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ4_MISS_FLAG_MASK_SFT                           (0x1 << 4)
+#define AFE_IRQ3_MISS_FLAG_SFT                                3
+#define AFE_IRQ3_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ3_MISS_FLAG_MASK_SFT                           (0x1 << 3)
+#define AFE_IRQ2_MISS_FLAG_SFT                                2
+#define AFE_IRQ2_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ2_MISS_FLAG_MASK_SFT                           (0x1 << 2)
+#define AFE_IRQ1_MISS_FLAG_SFT                                1
+#define AFE_IRQ1_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ1_MISS_FLAG_MASK_SFT                           (0x1 << 1)
+#define AFE_IRQ0_MISS_FLAG_SFT                                0
+#define AFE_IRQ0_MISS_FLAG_MASK                               0x1
+#define AFE_IRQ0_MISS_FLAG_MASK_SFT                           (0x1 << 0)
+
+/* AFE_IRQ_MCU_MON1 */
+#define AFE_CUSTOM_IRQ21_MISS_FLAG_SFT                        21
+#define AFE_CUSTOM_IRQ21_MISS_FLAG_MASK                       0x1
+#define AFE_CUSTOM_IRQ21_MISS_FLAG_MASK_SFT                   (0x1 << 21)
+#define AFE_CUSTOM_IRQ20_MISS_FLAG_SFT                        20
+#define AFE_CUSTOM_IRQ20_MISS_FLAG_MASK                       0x1
+#define AFE_CUSTOM_IRQ20_MISS_FLAG_MASK_SFT                   (0x1 << 20)
+#define AFE_CUSTOM_IRQ19_MISS_FLAG_SFT                        19
+#define AFE_CUSTOM_IRQ19_MISS_FLAG_MASK                       0x1
+#define AFE_CUSTOM_IRQ19_MISS_FLAG_MASK_SFT                   (0x1 << 19)
+#define AFE_CUSTOM_IRQ18_MISS_FLAG_SFT                        18
+#define AFE_CUSTOM_IRQ18_MISS_FLAG_MASK                       0x1
+#define AFE_CUSTOM_IRQ18_MISS_FLAG_MASK_SFT                   (0x1 << 18)
+#define AFE_CUSTOM_IRQ17_MISS_FLAG_SFT                        17
+#define AFE_CUSTOM_IRQ17_MISS_FLAG_MASK                       0x1
+#define AFE_CUSTOM_IRQ17_MISS_FLAG_MASK_SFT                   (0x1 << 17)
+#define AFE_CUSTOM_IRQ16_MISS_FLAG_SFT                        16
+#define AFE_CUSTOM_IRQ16_MISS_FLAG_MASK                       0x1
+#define AFE_CUSTOM_IRQ16_MISS_FLAG_MASK_SFT                   (0x1 << 16)
+#define AFE_CUSTOM_IRQ9_MISS_FLAG_SFT                         9
+#define AFE_CUSTOM_IRQ9_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ9_MISS_FLAG_MASK_SFT                    (0x1 << 9)
+#define AFE_CUSTOM_IRQ8_MISS_FLAG_SFT                         8
+#define AFE_CUSTOM_IRQ8_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ8_MISS_FLAG_MASK_SFT                    (0x1 << 8)
+#define AFE_CUSTOM_IRQ7_MISS_FLAG_SFT                         7
+#define AFE_CUSTOM_IRQ7_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ7_MISS_FLAG_MASK_SFT                    (0x1 << 7)
+#define AFE_CUSTOM_IRQ6_MISS_FLAG_SFT                         6
+#define AFE_CUSTOM_IRQ6_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ6_MISS_FLAG_MASK_SFT                    (0x1 << 6)
+#define AFE_CUSTOM_IRQ5_MISS_FLAG_SFT                         5
+#define AFE_CUSTOM_IRQ5_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ5_MISS_FLAG_MASK_SFT                    (0x1 << 5)
+#define AFE_CUSTOM_IRQ4_MISS_FLAG_SFT                         4
+#define AFE_CUSTOM_IRQ4_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ4_MISS_FLAG_MASK_SFT                    (0x1 << 4)
+#define AFE_CUSTOM_IRQ3_MISS_FLAG_SFT                         3
+#define AFE_CUSTOM_IRQ3_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ3_MISS_FLAG_MASK_SFT                    (0x1 << 3)
+#define AFE_CUSTOM_IRQ2_MISS_FLAG_SFT                         2
+#define AFE_CUSTOM_IRQ2_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ2_MISS_FLAG_MASK_SFT                    (0x1 << 2)
+#define AFE_CUSTOM_IRQ1_MISS_FLAG_SFT                         1
+#define AFE_CUSTOM_IRQ1_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ1_MISS_FLAG_MASK_SFT                    (0x1 << 1)
+#define AFE_CUSTOM_IRQ0_MISS_FLAG_SFT                         0
+#define AFE_CUSTOM_IRQ0_MISS_FLAG_MASK                        0x1
+#define AFE_CUSTOM_IRQ0_MISS_FLAG_MASK_SFT                    (0x1 << 0)
+
+/* AFE_IRQ_MCU_MON2 */
+#define AFE_IRQ_B_R_CNT_SFT                                   8
+#define AFE_IRQ_B_R_CNT_MASK                                  0xff
+#define AFE_IRQ_B_R_CNT_MASK_SFT                              (0xff << 8)
+#define AFE_IRQ_B_F_CNT_SFT                                   0
+#define AFE_IRQ_B_F_CNT_MASK                                  0xff
+#define AFE_IRQ_B_F_CNT_MASK_SFT                              (0xff << 0)
+
+/* AFE_IRQ0_CNT_MON */
+#define AFE_IRQ0_CNT_MON_SFT                                  0
+#define AFE_IRQ0_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ0_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ1_CNT_MON */
+#define AFE_IRQ1_CNT_MON_SFT                                  0
+#define AFE_IRQ1_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ1_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ2_CNT_MON */
+#define AFE_IRQ2_CNT_MON_SFT                                  0
+#define AFE_IRQ2_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ2_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ3_CNT_MON */
+#define AFE_IRQ3_CNT_MON_SFT                                  0
+#define AFE_IRQ3_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ3_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ4_CNT_MON */
+#define AFE_IRQ4_CNT_MON_SFT                                  0
+#define AFE_IRQ4_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ4_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ5_CNT_MON */
+#define AFE_IRQ5_CNT_MON_SFT                                  0
+#define AFE_IRQ5_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ5_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ6_CNT_MON */
+#define AFE_IRQ6_CNT_MON_SFT                                  0
+#define AFE_IRQ6_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ6_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ7_CNT_MON */
+#define AFE_IRQ7_CNT_MON_SFT                                  0
+#define AFE_IRQ7_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ7_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ8_CNT_MON */
+#define AFE_IRQ8_CNT_MON_SFT                                  0
+#define AFE_IRQ8_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ8_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ9_CNT_MON */
+#define AFE_IRQ9_CNT_MON_SFT                                  0
+#define AFE_IRQ9_CNT_MON_MASK                                 0xffffff
+#define AFE_IRQ9_CNT_MON_MASK_SFT                             (0xffffff << 0)
+
+/* AFE_IRQ10_CNT_MON */
+#define AFE_IRQ10_CNT_MON_SFT                                 0
+#define AFE_IRQ10_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ10_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ11_CNT_MON */
+#define AFE_IRQ11_CNT_MON_SFT                                 0
+#define AFE_IRQ11_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ11_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ12_CNT_MON */
+#define AFE_IRQ12_CNT_MON_SFT                                 0
+#define AFE_IRQ12_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ12_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ13_CNT_MON */
+#define AFE_IRQ13_CNT_MON_SFT                                 0
+#define AFE_IRQ13_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ13_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ14_CNT_MON */
+#define AFE_IRQ14_CNT_MON_SFT                                 0
+#define AFE_IRQ14_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ14_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ15_CNT_MON */
+#define AFE_IRQ15_CNT_MON_SFT                                 0
+#define AFE_IRQ15_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ15_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ16_CNT_MON */
+#define AFE_IRQ16_CNT_MON_SFT                                 0
+#define AFE_IRQ16_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ16_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ17_CNT_MON */
+#define AFE_IRQ17_CNT_MON_SFT                                 0
+#define AFE_IRQ17_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ17_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ18_CNT_MON */
+#define AFE_IRQ18_CNT_MON_SFT                                 0
+#define AFE_IRQ18_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ18_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ19_CNT_MON */
+#define AFE_IRQ19_CNT_MON_SFT                                 0
+#define AFE_IRQ19_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ19_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ20_CNT_MON */
+#define AFE_IRQ20_CNT_MON_SFT                                 0
+#define AFE_IRQ20_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ20_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ21_CNT_MON */
+#define AFE_IRQ21_CNT_MON_SFT                                 0
+#define AFE_IRQ21_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ21_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ22_CNT_MON */
+#define AFE_IRQ22_CNT_MON_SFT                                 0
+#define AFE_IRQ22_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ22_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ23_CNT_MON */
+#define AFE_IRQ23_CNT_MON_SFT                                 0
+#define AFE_IRQ23_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ23_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ24_CNT_MON */
+#define AFE_IRQ24_CNT_MON_SFT                                 0
+#define AFE_IRQ24_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ24_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ25_CNT_MON */
+#define AFE_IRQ25_CNT_MON_SFT                                 0
+#define AFE_IRQ25_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ25_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_IRQ26_CNT_MON */
+#define AFE_IRQ26_CNT_MON_SFT                                 0
+#define AFE_IRQ26_CNT_MON_MASK                                0xffffff
+#define AFE_IRQ26_CNT_MON_MASK_SFT                            (0xffffff << 0)
+
+/* AFE_CUSTOM_IRQ0_CNT_MON */
+#define AFE_CUSTOM_IRQ0_CNT_MON_SFT                           0
+#define AFE_CUSTOM_IRQ0_CNT_MON_MASK                          0xffffff
+#define AFE_CUSTOM_IRQ0_CNT_MON_MASK_SFT                      (0xffffff << 0)
+
+/* AFE_CUSTOM_IRQ0_MCU_CFG1 */
+#define AFE_CUSTOM_IRQ0_CLR_CFG_SFT                           31
+#define AFE_CUSTOM_IRQ0_CLR_CFG_MASK                          0x1
+#define AFE_CUSTOM_IRQ0_CLR_CFG_MASK_SFT                      (0x1 << 31)
+#define AFE_CUSTOM_IRQ0_MISS_FLAG_CLR_CFG_SFT                 30
+#define AFE_CUSTOM_IRQ0_MISS_FLAG_CLR_CFG_MASK                0x1
+#define AFE_CUSTOM_IRQ0_MISS_FLAG_CLR_CFG_MASK_SFT            (0x1 << 30)
+#define AFE_CUSTOM_IRQ0_MCU_CNT_SFT                           0
+#define AFE_CUSTOM_IRQ0_MCU_CNT_MASK                          0xffffff
+#define AFE_CUSTOM_IRQ0_MCU_CNT_MASK_SFT                      (0xffffff << 0)
+
+/* AFE_GAIN0_CON1_R */
+/* AFE_GAIN1_CON1_R */
+/* AFE_GAIN2_CON1_R */
+/* AFE_GAIN3_CON1_R */
+#define GAIN_TARGET_R_SFT                                    0
+#define GAIN_TARGET_R_MASK                                   0xffffffff
+#define GAIN_TARGET_R_MASK_SFT                               (0xffffffff << 0)
+
+/* AFE_GAIN0_CON1_L */
+/* AFE_GAIN1_CON1_L */
+/* AFE_GAIN2_CON1_L */
+/* AFE_GAIN3_CON1_L */
+#define GAIN_TARGET_L_SFT                                    0
+#define GAIN_TARGET_L_MASK                                   0xffffffff
+#define GAIN_TARGET_L_MASK_SFT                               (0xffffffff << 0)
+
+/* AFE_GAIN0_CON2 */
+#define GAIN0_DOWN_STEP_SFT                                   0
+#define GAIN0_DOWN_STEP_MASK                                  0x3fffff
+#define GAIN0_DOWN_STEP_MASK_SFT                              (0x3fffff << 0)
+
+/* AFE_GAIN0_CON3 */
+#define GAIN0_UP_STEP_SFT                                     0
+#define GAIN0_UP_STEP_MASK                                    0x3fffff
+#define GAIN0_UP_STEP_MASK_SFT                                (0x3fffff << 0)
+
+/* AFE_GAIN0_CUR_R */
+/* AFE_GAIN1_CUR_R */
+/* AFE_GAIN2_CUR_R */
+/* AFE_GAIN3_CUR_R */
+#define AFE_GAIN_CUR_R_SFT                                   0
+#define AFE_GAIN_CUR_R_MASK                                  0xffffffff
+#define AFE_GAIN_CUR_R_MASK_SFT                              (0xffffffff << 0)
+
+/* AFE_GAIN0_CUR_L */
+/* AFE_GAIN1_CUR_L */
+/* AFE_GAIN2_CUR_L */
+/* AFE_GAIN3_CUR_L */
+#define AFE_GAIN_CUR_L_SFT                                   0
+#define AFE_GAIN_CUR_L_MASK                                  0xffffffff
+#define AFE_GAIN_CUR_L_MASK_SFT                              (0xffffffff << 0)
+
+/* AFE_GAIN0_CON0 */
+/* AFE_GAIN1_CON0 */
+/* AFE_GAIN2_CON0 */
+/* AFE_GAIN3_CON0 */
+#define GAIN_TARGET_SYNC_ON_SFT                              24
+#define GAIN_TARGET_SYNC_ON_MASK                             0x1
+#define GAIN_TARGET_SYNC_ON_MASK_SFT                         (0x1 << 24)
+#define GAIN_TIMEOUT_SFT                                     18
+#define GAIN_TIMEOUT_MASK                                    0x3f
+#define GAIN_TIMEOUT_MASK_SFT                                (0x3f << 18)
+#define GAIN_TRIG_SFT                                        17
+#define GAIN_TRIG_MASK                                       0x1
+#define GAIN_TRIG_MASK_SFT                                   (0x1 << 17)
+#define GAIN_ON_SFT                                          16
+#define GAIN_ON_MASK                                         0x1
+#define GAIN_ON_MASK_SFT                                     (0x1 << 16)
+#define GAIN_SAMPLE_PER_STEP_SFT                             8
+#define GAIN_SAMPLE_PER_STEP_MASK                            0xff
+#define GAIN_SAMPLE_PER_STEP_MASK_SFT                        (0xff << 8)
+#define GAIN_SEL_DOMAIN_SFT                                  5
+#define GAIN_SEL_DOMAIN_MASK                                 0x7
+#define GAIN_SEL_DOMAIN_MASK_SFT                             (0x7 << 5)
+#define GAIN_SEL_FS_SFT                                      0
+#define GAIN_SEL_FS_MASK                                     0x1f
+#define GAIN_SEL_FS_MASK_SFT                                 (0x1f << 0)
+
+/* AFE_GAIN1_CON2 */
+#define GAIN1_DOWN_STEP_SFT                                   0
+#define GAIN1_DOWN_STEP_MASK                                  0x3fffff
+#define GAIN1_DOWN_STEP_MASK_SFT                              (0x3fffff << 0)
+
+/* AFE_GAIN1_CON3 */
+#define GAIN1_UP_STEP_SFT                                     0
+#define GAIN1_UP_STEP_MASK                                    0x3fffff
+#define GAIN1_UP_STEP_MASK_SFT                                (0x3fffff << 0)
+
+/* AFE_GAIN2_CON2 */
+#define GAIN2_DOWN_STEP_SFT                                   0
+#define GAIN2_DOWN_STEP_MASK                                  0x3fffff
+#define GAIN2_DOWN_STEP_MASK_SFT                              (0x3fffff << 0)
+
+/* AFE_GAIN2_CON3 */
+#define GAIN2_UP_STEP_SFT                                     0
+#define GAIN2_UP_STEP_MASK                                    0x3fffff
+#define GAIN2_UP_STEP_MASK_SFT                                (0x3fffff << 0)
+
+/* AFE_GAIN3_CON2 */
+#define GAIN3_DOWN_STEP_SFT                                   0
+#define GAIN3_DOWN_STEP_MASK                                  0x3fffff
+#define GAIN3_DOWN_STEP_MASK_SFT                              (0x3fffff << 0)
+
+/* AFE_GAIN3_CON3 */
+#define GAIN3_UP_STEP_SFT                                     0
+#define GAIN3_UP_STEP_MASK                                    0x3fffff
+#define GAIN3_UP_STEP_MASK_SFT                                (0x3fffff << 0)
+
+/* AFE_STF_CON0 */
+#define SLT_CNT_FLAG_RESET_SFT                                28
+#define SLT_CNT_FLAG_RESET_MASK                               0x1
+#define SLT_CNT_FLAG_RESET_MASK_SFT                           (0x1 << 28)
+#define SLT_CNT_THD_SFT                                       16
+#define SLT_CNT_THD_MASK                                      0xfff
+#define SLT_CNT_THD_MASK_SFT                                  (0xfff << 16)
+#define SIDE_TONE_HALF_TAP_NUM_SFT                            4
+#define SIDE_TONE_HALF_TAP_NUM_MASK                           0x7f
+#define SIDE_TONE_HALF_TAP_NUM_MASK_SFT                       (0x7f << 4)
+#define SIDE_TONE_ODD_MODE_SFT                                1
+#define SIDE_TONE_ODD_MODE_MASK                               0x1
+#define SIDE_TONE_ODD_MODE_MASK_SFT                           (0x1 << 1)
+#define SIDE_TONE_ON_SFT                                      0
+#define SIDE_TONE_ON_MASK                                     0x1
+#define SIDE_TONE_ON_MASK_SFT                                 (0x1 << 0)
+
+/* AFE_STF_CON1 */
+#define SIDE_TONE_IN_EN_SEL_DOMAIN_SFT                        5
+#define SIDE_TONE_IN_EN_SEL_DOMAIN_MASK                       0x7
+#define SIDE_TONE_IN_EN_SEL_DOMAIN_MASK_SFT                   (0x7 << 5)
+#define SIDE_TONE_IN_EN_SEL_FS_SFT                            0
+#define SIDE_TONE_IN_EN_SEL_FS_MASK                           0x1f
+#define SIDE_TONE_IN_EN_SEL_FS_MASK_SFT                       (0x1f << 0)
+
+/* AFE_STF_COEFF */
+#define SIDE_TONE_COEFFICIENT_R_W_SEL_SFT                     24
+#define SIDE_TONE_COEFFICIENT_R_W_SEL_MASK                    0x1
+#define SIDE_TONE_COEFFICIENT_R_W_SEL_MASK_SFT                (0x1 << 24)
+#define SIDE_TONE_COEFFICIENT_ADDR_SFT                        16
+#define SIDE_TONE_COEFFICIENT_ADDR_MASK                       0x1f
+#define SIDE_TONE_COEFFICIENT_ADDR_MASK_SFT                   (0x1f << 16)
+#define SIDE_TONE_COEFFICIENT_SFT                             0
+#define SIDE_TONE_COEFFICIENT_MASK                            0xffff
+#define SIDE_TONE_COEFFICIENT_MASK_SFT                        (0xffff << 0)
+
+/* AFE_STF_GAIN */
+#define SIDE_TONE_POSITIVE_GAIN_SFT                           16
+#define SIDE_TONE_POSITIVE_GAIN_MASK                          0x7
+#define SIDE_TONE_POSITIVE_GAIN_MASK_SFT                      (0x7 << 16)
+#define SIDE_TONE_GAIN_SFT                                    0
+#define SIDE_TONE_GAIN_MASK                                   0xffff
+#define SIDE_TONE_GAIN_MASK_SFT                               (0xffff << 0)
+
+/* AFE_STF_MON */
+#define SIDE_TONE_R_RDY_SFT                                   30
+#define SIDE_TONE_R_RDY_MASK                                  0x1
+#define SIDE_TONE_R_RDY_MASK_SFT                              (0x1 << 30)
+#define SIDE_TONE_W_RDY_SFT                                   29
+#define SIDE_TONE_W_RDY_MASK                                  0x1
+#define SIDE_TONE_W_RDY_MASK_SFT                              (0x1 << 29)
+#define SLT_CNT_FLAG_SFT                                      28
+#define SLT_CNT_FLAG_MASK                                     0x1
+#define SLT_CNT_FLAG_MASK_SFT                                 (0x1 << 28)
+#define SLT_CNT_SFT                                           16
+#define SLT_CNT_MASK                                          0xfff
+#define SLT_CNT_MASK_SFT                                      (0xfff << 16)
+#define SIDE_TONE_COEFF_SFT                                   0
+#define SIDE_TONE_COEFF_MASK                                  0xffff
+#define SIDE_TONE_COEFF_MASK_SFT                              (0xffff << 0)
+
+/* AFE_STF_IP_VERSION */
+#define SIDE_TONE_IP_VERSION_SFT                              0
+#define SIDE_TONE_IP_VERSION_MASK                             0xffffffff
+#define SIDE_TONE_IP_VERSION_MASK_SFT                         (0xffffffff << 0)
+
+/* AFE_CM_REG */
+#define AFE_CM_UPDATE_CNT_SFT                                 16
+#define AFE_CM_UPDATE_CNT_MASK                                0x7fff
+#define AFE_CM_UPDATE_CNT_MASK_SFT                            (0x7fff << 16)
+#define AFE_CM_1X_EN_SEL_FS_SFT                               8
+#define AFE_CM_1X_EN_SEL_FS_MASK                              0x1f
+#define AFE_CM_1X_EN_SEL_FS_MASK_SFT                          (0x1f << 8)
+#define AFE_CM_CH_NUM_SFT                                     2
+#define AFE_CM_CH_NUM_MASK                                    0x1f
+#define AFE_CM_CH_NUM_MASK_SFT                                (0x1f << 2)
+#define AFE_CM_BYTE_SWAP_SFT                                  1
+#define AFE_CM_BYTE_SWAP_MASK                                 0x1
+#define AFE_CM_BYTE_SWAP_MASK_SFT                             (0x1 << 1)
+#define AFE_CM_BYPASS_MODE_SFT                                31
+#define AFE_CM_BYPASS_MODE_MASK                               0x1
+#define AFE_CM_BYPASS_MODE_MASK_SFT                           (0x1 << 31)
+
+/* AFE_CM0_CON0 */
+#define AFE_CM0_BYPASS_MODE_SFT                               31
+#define AFE_CM0_BYPASS_MODE_MASK                              0x1
+#define AFE_CM0_BYPASS_MODE_MASK_SFT                          (0x1 << 31)
+#define AFE_CM0_UPDATE_CNT_SFT                                16
+#define AFE_CM0_UPDATE_CNT_MASK                               0x7fff
+#define AFE_CM0_UPDATE_CNT_MASK_SFT                           (0x7fff << 16)
+#define AFE_CM0_1X_EN_SEL_DOMAIN_SFT                          13
+#define AFE_CM0_1X_EN_SEL_DOMAIN_MASK                         0x7
+#define AFE_CM0_1X_EN_SEL_DOMAIN_MASK_SFT                     (0x7 << 13)
+#define AFE_CM0_1X_EN_SEL_FS_SFT                              8
+#define AFE_CM0_1X_EN_SEL_FS_MASK                             0x1f
+#define AFE_CM0_1X_EN_SEL_FS_MASK_SFT                         (0x1f << 8)
+#define AFE_CM0_OUTPUT_MUX_SFT                                7
+#define AFE_CM0_OUTPUT_MUX_MASK                               0x1
+#define AFE_CM0_OUTPUT_MUX_MASK_SFT                           (0x1 << 7)
+#define AFE_CM0_CH_NUM_SFT                                    2
+#define AFE_CM0_CH_NUM_MASK                                   0x1f
+#define AFE_CM0_CH_NUM_MASK_SFT                               (0x1f << 2)
+#define AFE_CM0_BYTE_SWAP_SFT                                 1
+#define AFE_CM0_BYTE_SWAP_MASK                                0x1
+#define AFE_CM0_BYTE_SWAP_MASK_SFT                            (0x1 << 1)
+#define AFE_CM0_ON_SFT                                        0
+#define AFE_CM0_ON_MASK                                       0x1
+#define AFE_CM0_ON_MASK_SFT                                   (0x1 << 0)
+
+/* AFE_CM0_MON */
+#define AFE_CM0_BYPASS_MODE_MON_SFT                           31
+#define AFE_CM0_BYPASS_MODE_MON_MASK                          0x1
+#define AFE_CM0_BYPASS_MODE_MON_MASK_SFT                      (0x1 << 31)
+#define AFE_CM0_OUTPUT_CNT_MON_SFT                            16
+#define AFE_CM0_OUTPUT_CNT_MON_MASK                           0x7fff
+#define AFE_CM0_OUTPUT_CNT_MON_MASK_SFT                       (0x7fff << 16)
+#define AFE_CM0_CUR_CHSET_MON_SFT                             5
+#define AFE_CM0_CUR_CHSET_MON_MASK                            0xf
+#define AFE_CM0_CUR_CHSET_MON_MASK_SFT                        (0xf << 5)
+#define AFE_CM0_ODD_FLAG_MON_SFT                              4
+#define AFE_CM0_ODD_FLAG_MON_MASK                             0x1
+#define AFE_CM0_ODD_FLAG_MON_MASK_SFT                         (0x1 << 4)
+#define AFE_CM0_BYTE_SWAP_MON_SFT                             1
+#define AFE_CM0_BYTE_SWAP_MON_MASK                            0x1
+#define AFE_CM0_BYTE_SWAP_MON_MASK_SFT                        (0x1 << 1)
+#define AFE_CM0_ON_MON_SFT                                    0
+#define AFE_CM0_ON_MON_MASK                                   0x1
+#define AFE_CM0_ON_MON_MASK_SFT                               (0x1 << 0)
+
+/* AFE_CM0_IP_VERSION */
+#define AFE_CM0_IP_VERSION_SFT                                0
+#define AFE_CM0_IP_VERSION_MASK                               0xffffffff
+#define AFE_CM0_IP_VERSION_MASK_SFT                           (0xffffffff << 0)
+
+/* AFE_CM1_CON0 */
+#define AFE_CM1_BYPASS_MODE_SFT                               31
+#define AFE_CM1_BYPASS_MODE_MASK                              0x1
+#define AFE_CM1_BYPASS_MODE_MASK_SFT                          (0x1 << 31)
+#define AFE_CM1_UPDATE_CNT_SFT                                16
+#define AFE_CM1_UPDATE_CNT_MASK                               0x7fff
+#define AFE_CM1_UPDATE_CNT_MASK_SFT                           (0x7fff << 16)
+#define AFE_CM1_1X_EN_SEL_DOMAIN_SFT                          13
+#define AFE_CM1_1X_EN_SEL_DOMAIN_MASK                         0x7
+#define AFE_CM1_1X_EN_SEL_DOMAIN_MASK_SFT                     (0x7 << 13)
+#define AFE_CM1_1X_EN_SEL_FS_SFT                              8
+#define AFE_CM1_1X_EN_SEL_FS_MASK                             0x1f
+#define AFE_CM1_1X_EN_SEL_FS_MASK_SFT                         (0x1f << 8)
+#define AFE_CM1_OUTPUT_MUX_SFT                                7
+#define AFE_CM1_OUTPUT_MUX_MASK                               0x1
+#define AFE_CM1_OUTPUT_MUX_MASK_SFT                           (0x1 << 7)
+#define AFE_CM1_CH_NUM_SFT                                    2
+#define AFE_CM1_CH_NUM_MASK                                   0x1f
+#define AFE_CM1_CH_NUM_MASK_SFT                               (0x1f << 2)
+#define AFE_CM1_BYTE_SWAP_SFT                                 1
+#define AFE_CM1_BYTE_SWAP_MASK                                0x1
+#define AFE_CM1_BYTE_SWAP_MASK_SFT                            (0x1 << 1)
+#define AFE_CM1_ON_SFT                                        0
+#define AFE_CM1_ON_MASK                                       0x1
+#define AFE_CM1_ON_MASK_SFT                                   (0x1 << 0)
+
+/* AFE_CM1_MON */
+#define AFE_CM1_BYPASS_MODE_MON_SFT                           31
+#define AFE_CM1_BYPASS_MODE_MON_MASK                          0x1
+#define AFE_CM1_BYPASS_MODE_MON_MASK_SFT                      (0x1 << 31)
+#define AFE_CM1_OUTPUT_CNT_MON_SFT                            16
+#define AFE_CM1_OUTPUT_CNT_MON_MASK                           0x7fff
+#define AFE_CM1_OUTPUT_CNT_MON_MASK_SFT                       (0x7fff << 16)
+#define AFE_CM1_CUR_CHSET_MON_SFT                             5
+#define AFE_CM1_CUR_CHSET_MON_MASK                            0xf
+#define AFE_CM1_CUR_CHSET_MON_MASK_SFT                        (0xf << 5)
+#define AFE_CM1_ODD_FLAG_MON_SFT                              4
+#define AFE_CM1_ODD_FLAG_MON_MASK                             0x1
+#define AFE_CM1_ODD_FLAG_MON_MASK_SFT                         (0x1 << 4)
+#define AFE_CM1_BYTE_SWAP_MON_SFT                             1
+#define AFE_CM1_BYTE_SWAP_MON_MASK                            0x1
+#define AFE_CM1_BYTE_SWAP_MON_MASK_SFT                        (0x1 << 1)
+#define AFE_CM1_ON_MON_SFT                                    0
+#define AFE_CM1_ON_MON_MASK                                   0x1
+#define AFE_CM1_ON_MON_MASK_SFT                               (0x1 << 0)
+
+/* AFE_CM1_IP_VERSION */
+#define AFE_CM1_IP_VERSION_SFT                                0
+#define AFE_CM1_IP_VERSION_MASK                               0xffffffff
+#define AFE_CM1_IP_VERSION_MASK_SFT                           (0xffffffff << 0)
+
+/* AFE_CM2_CON0 */
+#define AFE_CM2_BYPASS_MODE_SFT                               31
+#define AFE_CM2_BYPASS_MODE_MASK                              0x1
+#define AFE_CM2_BYPASS_MODE_MASK_SFT                          (0x1 << 31)
+#define AFE_CM2_UPDATE_CNT_SFT                                16
+#define AFE_CM2_UPDATE_CNT_MASK                               0x7fff
+#define AFE_CM2_UPDATE_CNT_MASK_SFT                           (0x7fff << 16)
+#define AFE_CM2_1X_EN_SEL_DOMAIN_SFT                          13
+#define AFE_CM2_1X_EN_SEL_DOMAIN_MASK                         0x7
+#define AFE_CM2_1X_EN_SEL_DOMAIN_MASK_SFT                     (0x7 << 13)
+#define AFE_CM2_1X_EN_SEL_FS_SFT                              8
+#define AFE_CM2_1X_EN_SEL_FS_MASK                             0x1f
+#define AFE_CM2_1X_EN_SEL_FS_MASK_SFT                         (0x1f << 8)
+#define AFE_CM2_OUTPUT_MUX_SFT                                7
+#define AFE_CM2_OUTPUT_MUX_MASK                               0x1
+#define AFE_CM2_OUTPUT_MUX_MASK_SFT                           (0x1 << 7)
+#define AFE_CM2_CH_NUM_SFT                                    2
+#define AFE_CM2_CH_NUM_MASK                                   0x1f
+#define AFE_CM2_CH_NUM_MASK_SFT                               (0x1f << 2)
+#define AFE_CM2_BYTE_SWAP_SFT                                 1
+#define AFE_CM2_BYTE_SWAP_MASK                                0x1
+#define AFE_CM2_BYTE_SWAP_MASK_SFT                            (0x1 << 1)
+#define AFE_CM2_ON_SFT                                        0
+#define AFE_CM2_ON_MASK                                       0x1
+#define AFE_CM2_ON_MASK_SFT                                   (0x1 << 0)
+
+/* AFE_CM2_MON */
+#define AFE_CM2_BYPASS_MODE_MON_SFT                           31
+#define AFE_CM2_BYPASS_MODE_MON_MASK                          0x1
+#define AFE_CM2_BYPASS_MODE_MON_MASK_SFT                      (0x1 << 31)
+#define AFE_CM2_OUTPUT_CNT_MON_SFT                            16
+#define AFE_CM2_OUTPUT_CNT_MON_MASK                           0x7fff
+#define AFE_CM2_OUTPUT_CNT_MON_MASK_SFT                       (0x7fff << 16)
+#define AFE_CM2_CUR_CHSET_MON_SFT                             5
+#define AFE_CM2_CUR_CHSET_MON_MASK                            0xf
+#define AFE_CM2_CUR_CHSET_MON_MASK_SFT                        (0xf << 5)
+#define AFE_CM2_ODD_FLAG_MON_SFT                              4
+#define AFE_CM2_ODD_FLAG_MON_MASK                             0x1
+#define AFE_CM2_ODD_FLAG_MON_MASK_SFT                         (0x1 << 4)
+#define AFE_CM2_BYTE_SWAP_MON_SFT                             1
+#define AFE_CM2_BYTE_SWAP_MON_MASK                            0x1
+#define AFE_CM2_BYTE_SWAP_MON_MASK_SFT                        (0x1 << 1)
+#define AFE_CM2_ON_MON_SFT                                    0
+#define AFE_CM2_ON_MON_MASK                                   0x1
+#define AFE_CM2_ON_MON_MASK_SFT                               (0x1 << 0)
+
+/* AFE_CM2_IP_VERSION */
+#define AFE_CM2_IP_VERSION_SFT                                0
+#define AFE_CM2_IP_VERSION_MASK                               0xffffffff
+#define AFE_CM2_IP_VERSION_MASK_SFT                           (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_SRC_CON0 */
+#define ULCF_CFG_EN_CTL_SFT                                   31
+#define ULCF_CFG_EN_CTL_MASK                                  0x1
+#define ULCF_CFG_EN_CTL_MASK_SFT                              (0x1 << 31)
+#define UL_DMIC_PHASE_SEL_CH1_SFT                             27
+#define UL_DMIC_PHASE_SEL_CH1_MASK                            0x7
+#define UL_DMIC_PHASE_SEL_CH1_MASK_SFT                        (0x7 << 27)
+#define UL_DMIC_PHASE_SEL_CH2_SFT                             24
+#define UL_DMIC_PHASE_SEL_CH2_MASK                            0x7
+#define UL_DMIC_PHASE_SEL_CH2_MASK_SFT                        (0x7 << 24)
+#define UL_DMIC_TWO_WIRE_CTL_SFT                              23
+#define UL_DMIC_TWO_WIRE_CTL_MASK                             0x1
+#define UL_DMIC_TWO_WIRE_CTL_MASK_SFT                         (0x1 << 23)
+#define UL_MODE_3P25M_CH2_CTL_SFT                             22
+#define UL_MODE_3P25M_CH2_CTL_MASK                            0x1
+#define UL_MODE_3P25M_CH2_CTL_MASK_SFT                        (0x1 << 22)
+#define UL_MODE_3P25M_CH1_CTL_SFT                             21
+#define UL_MODE_3P25M_CH1_CTL_MASK                            0x1
+#define UL_MODE_3P25M_CH1_CTL_MASK_SFT                        (0x1 << 21)
+#define UL_VOICE_MODE_CH1_CH2_CTL_SFT                         17
+#define UL_VOICE_MODE_CH1_CH2_CTL_MASK                        0x7
+#define UL_VOICE_MODE_CH1_CH2_CTL_MASK_SFT                    (0x7 << 17)
+#define UL_AP_DMIC_ON_SFT                                     16
+#define UL_AP_DMIC_ON_MASK                                    0x1
+#define UL_AP_DMIC_ON_MASK_SFT                                (0x1 << 16)
+#define DMIC_LOW_POWER_MODE_CTL_SFT                           14
+#define DMIC_LOW_POWER_MODE_CTL_MASK                          0x3
+#define DMIC_LOW_POWER_MODE_CTL_MASK_SFT                      (0x3 << 14)
+#define UL_DISABLE_HW_CG_CTL_SFT                              12
+#define UL_DISABLE_HW_CG_CTL_MASK                             0x1
+#define UL_DISABLE_HW_CG_CTL_MASK_SFT                         (0x1 << 12)
+#define AMIC_26M_SEL_CTL_SFT                                  11
+#define AMIC_26M_SEL_CTL_MASK                                 0x1
+#define AMIC_26M_SEL_CTL_MASK_SFT                             (0x1 << 11)
+#define UL_IIR_ON_TMP_CTL_SFT                                 10
+#define UL_IIR_ON_TMP_CTL_MASK                                0x1
+#define UL_IIR_ON_TMP_CTL_MASK_SFT                            (0x1 << 10)
+#define UL_IIRMODE_CTL_SFT                                    7
+#define UL_IIRMODE_CTL_MASK                                   0x7
+#define UL_IIRMODE_CTL_MASK_SFT                               (0x7 << 7)
+#define DIGMIC_4P33M_SEL_SFT                                  6
+#define DIGMIC_4P33M_SEL_MASK                                 0x1
+#define DIGMIC_4P33M_SEL_MASK_SFT                             (0x1 << 6)
+#define DIGMIC_3P25M_1P625M_SEL_CTL_SFT                       5
+#define DIGMIC_3P25M_1P625M_SEL_CTL_MASK                      0x1
+#define DIGMIC_3P25M_1P625M_SEL_CTL_MASK_SFT                  (0x1 << 5)
+#define AMIC_6P5M_SEL_CTL_SFT                                 4
+#define AMIC_6P5M_SEL_CTL_MASK                                0x1
+#define AMIC_6P5M_SEL_CTL_MASK_SFT                            (0x1 << 4)
+#define AMIC_1P625M_SEL_CTL_SFT                               3
+#define AMIC_1P625M_SEL_CTL_MASK                              0x1
+#define AMIC_1P625M_SEL_CTL_MASK_SFT                          (0x1 << 3)
+#define UL_LOOP_BACK_MODE_CTL_SFT                             2
+#define UL_LOOP_BACK_MODE_CTL_MASK                            0x1
+#define UL_LOOP_BACK_MODE_CTL_MASK_SFT                        (0x1 << 2)
+#define UL_SDM_3_LEVEL_CTL_SFT                                1
+#define UL_SDM_3_LEVEL_CTL_MASK                               0x1
+#define UL_SDM_3_LEVEL_CTL_MASK_SFT                           (0x1 << 1)
+#define UL_SRC_ON_TMP_CTL_SFT                                 0
+#define UL_SRC_ON_TMP_CTL_MASK                                0x1
+#define UL_SRC_ON_TMP_CTL_MASK_SFT                            (0x1 << 0)
+
+/* AFE_ADDA_UL0_SRC_CON1 */
+#define ADDA_UL_GAIN_VALUE_SFT                                16
+#define ADDA_UL_GAIN_VALUE_MASK                               0xffff
+#define ADDA_UL_GAIN_VALUE_MASK_SFT                           (0xffff << 16)
+#define ADDA_UL_POSTIVEGAIN_SFT                               12
+#define ADDA_UL_POSTIVEGAIN_MASK                              0x7
+#define ADDA_UL_POSTIVEGAIN_MASK_SFT                          (0x7 << 12)
+#define ADDA_UL_ODDTAP_MODE_SFT                               11
+#define ADDA_UL_ODDTAP_MODE_MASK                              0x1
+#define ADDA_UL_ODDTAP_MODE_MASK_SFT                          (0x1 << 11)
+#define ADDA_UL_HALF_TAP_NUM_SFT                              5
+#define ADDA_UL_HALF_TAP_NUM_MASK                             0x3f
+#define ADDA_UL_HALF_TAP_NUM_MASK_SFT                         (0x3f << 5)
+#define FIFO_SOFT_RST_SFT                                     4
+#define FIFO_SOFT_RST_MASK                                    0x1
+#define FIFO_SOFT_RST_MASK_SFT                                (0x1 << 4)
+#define FIFO_SOFT_RST_EN_SFT                                  3
+#define FIFO_SOFT_RST_EN_MASK                                 0x1
+#define FIFO_SOFT_RST_EN_MASK_SFT                             (0x1 << 3)
+#define LR_SWAP_SFT                                           2
+#define LR_SWAP_MASK                                          0x1
+#define LR_SWAP_MASK_SFT                                      (0x1 << 2)
+#define GAIN_MODE_SFT                                         0
+#define GAIN_MODE_MASK                                        0x3
+#define GAIN_MODE_MASK_SFT                                    (0x3 << 0)
+
+/* AFE_ADDA_UL0_SRC_CON2 */
+#define C_DAC_EN_CTL_SFT                                      27
+#define C_DAC_EN_CTL_MASK                                     0x1
+#define C_DAC_EN_CTL_MASK_SFT                                 (0x1 << 27)
+#define C_MUTE_SW_CTL_SFT                                     26
+#define C_MUTE_SW_CTL_MASK                                    0x1
+#define C_MUTE_SW_CTL_MASK_SFT                                (0x1 << 26)
+#define C_AMP_DIV_CH2_CTL_SFT                                 21
+#define C_AMP_DIV_CH2_CTL_MASK                                0x7
+#define C_AMP_DIV_CH2_CTL_MASK_SFT                            (0x7 << 21)
+#define C_FREQ_DIV_CH2_CTL_SFT                                16
+#define C_FREQ_DIV_CH2_CTL_MASK                               0x1f
+#define C_FREQ_DIV_CH2_CTL_MASK_SFT                           (0x1f << 16)
+#define C_SINE_MODE_CH2_CTL_SFT                               12
+#define C_SINE_MODE_CH2_CTL_MASK                              0xf
+#define C_SINE_MODE_CH2_CTL_MASK_SFT                          (0xf << 12)
+#define C_AMP_DIV_CH1_CTL_SFT                                 9
+#define C_AMP_DIV_CH1_CTL_MASK                                0x7
+#define C_AMP_DIV_CH1_CTL_MASK_SFT                            (0x7 << 9)
+#define C_FREQ_DIV_CH1_CTL_SFT                                4
+#define C_FREQ_DIV_CH1_CTL_MASK                               0x1f
+#define C_FREQ_DIV_CH1_CTL_MASK_SFT                           (0x1f << 4)
+#define C_SINE_MODE_CH1_CTL_SFT                               0
+#define C_SINE_MODE_CH1_CTL_MASK                              0xf
+#define C_SINE_MODE_CH1_CTL_MASK_SFT                          (0xf << 0)
+
+/* AFE_ADDA_UL0_SRC_DEBUG */
+#define UL_SLT_CNT_FLAG_RESET_CTL_SFT                         16
+#define UL_SLT_CNT_FLAG_RESET_CTL_MASK                        0x1
+#define UL_SLT_CNT_FLAG_RESET_CTL_MASK_SFT                    (0x1 << 16)
+#define FIFO_DIGMIC_TESTIN_SFT                                12
+#define FIFO_DIGMIC_TESTIN_MASK                               0x3
+#define FIFO_DIGMIC_TESTIN_MASK_SFT                           (0x3 << 12)
+#define FIFO_DIGMIC_WDATA_TESTEN_SFT                          11
+#define FIFO_DIGMIC_WDATA_TESTEN_MASK                         0x1
+#define FIFO_DIGMIC_WDATA_TESTEN_MASK_SFT                     (0x1 << 11)
+#define SLT_CNT_THD_CTL_SFT                                   0
+#define SLT_CNT_THD_CTL_MASK                                  0x7ff
+#define SLT_CNT_THD_CTL_MASK_SFT                              (0x7ff << 0)
+
+/* AFE_ADDA_UL0_SRC_DEBUG_MON0 */
+#define SLT_CNT_FLAG_CTL_SFT                                  16
+#define SLT_CNT_FLAG_CTL_MASK                                 0x1
+#define SLT_CNT_FLAG_CTL_MASK_SFT                             (0x1 << 16)
+#define SLT_COUNTER_CTL_SFT                                   0
+#define SLT_COUNTER_CTL_MASK                                  0x7ff
+#define SLT_COUNTER_CTL_MASK_SFT                              (0x7ff << 0)
+
+/* AFE_ADDA_UL0_IIR_COEF_02_01 */
+#define ADDA_IIR_COEF_02_01_SFT                               0
+#define ADDA_IIR_COEF_02_01_MASK                              0xffffffff
+#define ADDA_IIR_COEF_02_01_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_IIR_COEF_04_03 */
+#define ADDA_IIR_COEF_04_03_SFT                               0
+#define ADDA_IIR_COEF_04_03_MASK                              0xffffffff
+#define ADDA_IIR_COEF_04_03_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_IIR_COEF_06_05 */
+#define ADDA_IIR_COEF_06_05_SFT                               0
+#define ADDA_IIR_COEF_06_05_MASK                              0xffffffff
+#define ADDA_IIR_COEF_06_05_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_IIR_COEF_08_07 */
+#define ADDA_IIR_COEF_08_07_SFT                               0
+#define ADDA_IIR_COEF_08_07_MASK                              0xffffffff
+#define ADDA_IIR_COEF_08_07_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_IIR_COEF_10_09 */
+#define ADDA_IIR_COEF_10_09_SFT                               0
+#define ADDA_IIR_COEF_10_09_MASK                              0xffffffff
+#define ADDA_IIR_COEF_10_09_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_02_01 */
+#define ADDA_ULCF_CFG_02_01_SFT                               0
+#define ADDA_ULCF_CFG_02_01_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_02_01_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_04_03 */
+#define ADDA_ULCF_CFG_04_03_SFT                               0
+#define ADDA_ULCF_CFG_04_03_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_04_03_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_06_05 */
+#define ADDA_ULCF_CFG_06_05_SFT                               0
+#define ADDA_ULCF_CFG_06_05_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_06_05_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_08_07 */
+#define ADDA_ULCF_CFG_08_07_SFT                               0
+#define ADDA_ULCF_CFG_08_07_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_08_07_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_10_09 */
+#define ADDA_ULCF_CFG_10_09_SFT                               0
+#define ADDA_ULCF_CFG_10_09_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_10_09_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_12_11 */
+#define ADDA_ULCF_CFG_12_11_SFT                               0
+#define ADDA_ULCF_CFG_12_11_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_12_11_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_14_13 */
+#define ADDA_ULCF_CFG_14_13_SFT                               0
+#define ADDA_ULCF_CFG_14_13_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_14_13_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_16_15 */
+#define ADDA_ULCF_CFG_16_15_SFT                               0
+#define ADDA_ULCF_CFG_16_15_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_16_15_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_18_17 */
+#define ADDA_ULCF_CFG_18_17_SFT                               0
+#define ADDA_ULCF_CFG_18_17_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_18_17_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_20_19 */
+#define ADDA_ULCF_CFG_20_19_SFT                               0
+#define ADDA_ULCF_CFG_20_19_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_20_19_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_22_21 */
+#define ADDA_ULCF_CFG_22_21_SFT                               0
+#define ADDA_ULCF_CFG_22_21_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_22_21_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_24_23 */
+#define ADDA_ULCF_CFG_24_23_SFT                               0
+#define ADDA_ULCF_CFG_24_23_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_24_23_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_26_25 */
+#define ADDA_ULCF_CFG_26_25_SFT                               0
+#define ADDA_ULCF_CFG_26_25_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_26_25_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_28_27 */
+#define ADDA_ULCF_CFG_28_27_SFT                               0
+#define ADDA_ULCF_CFG_28_27_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_28_27_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_30_29 */
+#define ADDA_ULCF_CFG_30_29_SFT                               0
+#define ADDA_ULCF_CFG_30_29_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_30_29_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_ULCF_CFG_32_31 */
+#define ADDA_ULCF_CFG_32_31_SFT                               0
+#define ADDA_ULCF_CFG_32_31_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_32_31_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL0_IP_VERSION */
+#define ADDA_ULCF_IP_VERSION_SFT                              0
+#define ADDA_ULCF_IP_VERSION_MASK                             0xffffffff
+#define ADDA_ULCF_IP_VERSION_MASK_SFT                         (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_SRC_CON0 */
+#define ULCF_CFG_EN_CTL_SFT                                   31
+#define ULCF_CFG_EN_CTL_MASK                                  0x1
+#define ULCF_CFG_EN_CTL_MASK_SFT                              (0x1 << 31)
+#define UL_DMIC_PHASE_SEL_CH1_SFT                             27
+#define UL_DMIC_PHASE_SEL_CH1_MASK                            0x7
+#define UL_DMIC_PHASE_SEL_CH1_MASK_SFT                        (0x7 << 27)
+#define UL_DMIC_PHASE_SEL_CH2_SFT                             24
+#define UL_DMIC_PHASE_SEL_CH2_MASK                            0x7
+#define UL_DMIC_PHASE_SEL_CH2_MASK_SFT                        (0x7 << 24)
+#define UL_DMIC_TWO_WIRE_CTL_SFT                              23
+#define UL_DMIC_TWO_WIRE_CTL_MASK                             0x1
+#define UL_DMIC_TWO_WIRE_CTL_MASK_SFT                         (0x1 << 23)
+#define UL_MODE_3P25M_CH2_CTL_SFT                             22
+#define UL_MODE_3P25M_CH2_CTL_MASK                            0x1
+#define UL_MODE_3P25M_CH2_CTL_MASK_SFT                        (0x1 << 22)
+#define UL_MODE_3P25M_CH1_CTL_SFT                             21
+#define UL_MODE_3P25M_CH1_CTL_MASK                            0x1
+#define UL_MODE_3P25M_CH1_CTL_MASK_SFT                        (0x1 << 21)
+#define UL_VOICE_MODE_CH1_CH2_CTL_SFT                         17
+#define UL_VOICE_MODE_CH1_CH2_CTL_MASK                        0x7
+#define UL_VOICE_MODE_CH1_CH2_CTL_MASK_SFT                    (0x7 << 17)
+#define UL_AP_DMIC_ON_SFT                                     16
+#define UL_AP_DMIC_ON_MASK                                    0x1
+#define UL_AP_DMIC_ON_MASK_SFT                                (0x1 << 16)
+#define DMIC_LOW_POWER_MODE_CTL_SFT                           14
+#define DMIC_LOW_POWER_MODE_CTL_MASK                          0x3
+#define DMIC_LOW_POWER_MODE_CTL_MASK_SFT                      (0x3 << 14)
+#define UL_DISABLE_HW_CG_CTL_SFT                              12
+#define UL_DISABLE_HW_CG_CTL_MASK                             0x1
+#define UL_DISABLE_HW_CG_CTL_MASK_SFT                         (0x1 << 12)
+#define AMIC_26M_SEL_CTL_SFT                                  11
+#define AMIC_26M_SEL_CTL_MASK                                 0x1
+#define AMIC_26M_SEL_CTL_MASK_SFT                             (0x1 << 11)
+#define UL_IIR_ON_TMP_CTL_SFT                                 10
+#define UL_IIR_ON_TMP_CTL_MASK                                0x1
+#define UL_IIR_ON_TMP_CTL_MASK_SFT                            (0x1 << 10)
+#define UL_IIRMODE_CTL_SFT                                    7
+#define UL_IIRMODE_CTL_MASK                                   0x7
+#define UL_IIRMODE_CTL_MASK_SFT                               (0x7 << 7)
+#define DIGMIC_4P33M_SEL_SFT                                  6
+#define DIGMIC_4P33M_SEL_MASK                                 0x1
+#define DIGMIC_4P33M_SEL_MASK_SFT                             (0x1 << 6)
+#define DIGMIC_3P25M_1P625M_SEL_CTL_SFT                       5
+#define DIGMIC_3P25M_1P625M_SEL_CTL_MASK                      0x1
+#define DIGMIC_3P25M_1P625M_SEL_CTL_MASK_SFT                  (0x1 << 5)
+#define AMIC_6P5M_SEL_CTL_SFT                                 4
+#define AMIC_6P5M_SEL_CTL_MASK                                0x1
+#define AMIC_6P5M_SEL_CTL_MASK_SFT                            (0x1 << 4)
+#define AMIC_1P625M_SEL_CTL_SFT                               3
+#define AMIC_1P625M_SEL_CTL_MASK                              0x1
+#define AMIC_1P625M_SEL_CTL_MASK_SFT                          (0x1 << 3)
+#define UL_LOOP_BACK_MODE_CTL_SFT                             2
+#define UL_LOOP_BACK_MODE_CTL_MASK                            0x1
+#define UL_LOOP_BACK_MODE_CTL_MASK_SFT                        (0x1 << 2)
+#define UL_SDM_3_LEVEL_CTL_SFT                                1
+#define UL_SDM_3_LEVEL_CTL_MASK                               0x1
+#define UL_SDM_3_LEVEL_CTL_MASK_SFT                           (0x1 << 1)
+#define UL_SRC_ON_TMP_CTL_SFT                                 0
+#define UL_SRC_ON_TMP_CTL_MASK                                0x1
+#define UL_SRC_ON_TMP_CTL_MASK_SFT                            (0x1 << 0)
+
+/* AFE_ADDA_UL1_SRC_CON1 */
+#define ADDA_UL_GAIN_VALUE_SFT                                16
+#define ADDA_UL_GAIN_VALUE_MASK                               0xffff
+#define ADDA_UL_GAIN_VALUE_MASK_SFT                           (0xffff << 16)
+#define ADDA_UL_POSTIVEGAIN_SFT                               12
+#define ADDA_UL_POSTIVEGAIN_MASK                              0x7
+#define ADDA_UL_POSTIVEGAIN_MASK_SFT                          (0x7 << 12)
+#define ADDA_UL_ODDTAP_MODE_SFT                               11
+#define ADDA_UL_ODDTAP_MODE_MASK                              0x1
+#define ADDA_UL_ODDTAP_MODE_MASK_SFT                          (0x1 << 11)
+#define ADDA_UL_HALF_TAP_NUM_SFT                              5
+#define ADDA_UL_HALF_TAP_NUM_MASK                             0x3f
+#define ADDA_UL_HALF_TAP_NUM_MASK_SFT                         (0x3f << 5)
+#define FIFO_SOFT_RST_SFT                                     4
+#define FIFO_SOFT_RST_MASK                                    0x1
+#define FIFO_SOFT_RST_MASK_SFT                                (0x1 << 4)
+#define FIFO_SOFT_RST_EN_SFT                                  3
+#define FIFO_SOFT_RST_EN_MASK                                 0x1
+#define FIFO_SOFT_RST_EN_MASK_SFT                             (0x1 << 3)
+#define LR_SWAP_SFT                                           2
+#define LR_SWAP_MASK                                          0x1
+#define LR_SWAP_MASK_SFT                                      (0x1 << 2)
+#define GAIN_MODE_SFT                                         0
+#define GAIN_MODE_MASK                                        0x3
+#define GAIN_MODE_MASK_SFT                                    (0x3 << 0)
+
+/* AFE_ADDA_UL1_SRC_CON2 */
+#define C_DAC_EN_CTL_SFT                                      27
+#define C_DAC_EN_CTL_MASK                                     0x1
+#define C_DAC_EN_CTL_MASK_SFT                                 (0x1 << 27)
+#define C_MUTE_SW_CTL_SFT                                     26
+#define C_MUTE_SW_CTL_MASK                                    0x1
+#define C_MUTE_SW_CTL_MASK_SFT                                (0x1 << 26)
+#define C_AMP_DIV_CH2_CTL_SFT                                 21
+#define C_AMP_DIV_CH2_CTL_MASK                                0x7
+#define C_AMP_DIV_CH2_CTL_MASK_SFT                            (0x7 << 21)
+#define C_FREQ_DIV_CH2_CTL_SFT                                16
+#define C_FREQ_DIV_CH2_CTL_MASK                               0x1f
+#define C_FREQ_DIV_CH2_CTL_MASK_SFT                           (0x1f << 16)
+#define C_SINE_MODE_CH2_CTL_SFT                               12
+#define C_SINE_MODE_CH2_CTL_MASK                              0xf
+#define C_SINE_MODE_CH2_CTL_MASK_SFT                          (0xf << 12)
+#define C_AMP_DIV_CH1_CTL_SFT                                 9
+#define C_AMP_DIV_CH1_CTL_MASK                                0x7
+#define C_AMP_DIV_CH1_CTL_MASK_SFT                            (0x7 << 9)
+#define C_FREQ_DIV_CH1_CTL_SFT                                4
+#define C_FREQ_DIV_CH1_CTL_MASK                               0x1f
+#define C_FREQ_DIV_CH1_CTL_MASK_SFT                           (0x1f << 4)
+#define C_SINE_MODE_CH1_CTL_SFT                               0
+#define C_SINE_MODE_CH1_CTL_MASK                              0xf
+#define C_SINE_MODE_CH1_CTL_MASK_SFT                          (0xf << 0)
+
+/* AFE_ADDA_UL1_SRC_DEBUG */
+#define UL_SLT_CNT_FLAG_RESET_CTL_SFT                         16
+#define UL_SLT_CNT_FLAG_RESET_CTL_MASK                        0x1
+#define UL_SLT_CNT_FLAG_RESET_CTL_MASK_SFT                    (0x1 << 16)
+#define FIFO_DIGMIC_TESTIN_SFT                                12
+#define FIFO_DIGMIC_TESTIN_MASK                               0x3
+#define FIFO_DIGMIC_TESTIN_MASK_SFT                           (0x3 << 12)
+#define FIFO_DIGMIC_WDATA_TESTEN_SFT                          11
+#define FIFO_DIGMIC_WDATA_TESTEN_MASK                         0x1
+#define FIFO_DIGMIC_WDATA_TESTEN_MASK_SFT                     (0x1 << 11)
+#define SLT_CNT_THD_CTL_SFT                                   0
+#define SLT_CNT_THD_CTL_MASK                                  0x7ff
+#define SLT_CNT_THD_CTL_MASK_SFT                              (0x7ff << 0)
+
+/* AFE_ADDA_UL1_SRC_DEBUG_MON0 */
+#define SLT_CNT_FLAG_CTL_SFT                                  16
+#define SLT_CNT_FLAG_CTL_MASK                                 0x1
+#define SLT_CNT_FLAG_CTL_MASK_SFT                             (0x1 << 16)
+#define SLT_COUNTER_CTL_SFT                                   0
+#define SLT_COUNTER_CTL_MASK                                  0x7ff
+#define SLT_COUNTER_CTL_MASK_SFT                              (0x7ff << 0)
+
+/* AFE_ADDA_UL1_IIR_COEF_02_01 */
+#define ADDA_IIR_COEF_02_01_SFT                               0
+#define ADDA_IIR_COEF_02_01_MASK                              0xffffffff
+#define ADDA_IIR_COEF_02_01_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_IIR_COEF_04_03 */
+#define ADDA_IIR_COEF_04_03_SFT                               0
+#define ADDA_IIR_COEF_04_03_MASK                              0xffffffff
+#define ADDA_IIR_COEF_04_03_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_IIR_COEF_06_05 */
+#define ADDA_IIR_COEF_06_05_SFT                               0
+#define ADDA_IIR_COEF_06_05_MASK                              0xffffffff
+#define ADDA_IIR_COEF_06_05_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_IIR_COEF_08_07 */
+#define ADDA_IIR_COEF_08_07_SFT                               0
+#define ADDA_IIR_COEF_08_07_MASK                              0xffffffff
+#define ADDA_IIR_COEF_08_07_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_IIR_COEF_10_09 */
+#define ADDA_IIR_COEF_10_09_SFT                               0
+#define ADDA_IIR_COEF_10_09_MASK                              0xffffffff
+#define ADDA_IIR_COEF_10_09_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_02_01 */
+#define ADDA_ULCF_CFG_02_01_SFT                               0
+#define ADDA_ULCF_CFG_02_01_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_02_01_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_04_03 */
+#define ADDA_ULCF_CFG_04_03_SFT                               0
+#define ADDA_ULCF_CFG_04_03_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_04_03_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_06_05 */
+#define ADDA_ULCF_CFG_06_05_SFT                               0
+#define ADDA_ULCF_CFG_06_05_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_06_05_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_08_07 */
+#define ADDA_ULCF_CFG_08_07_SFT                               0
+#define ADDA_ULCF_CFG_08_07_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_08_07_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_10_09 */
+#define ADDA_ULCF_CFG_10_09_SFT                               0
+#define ADDA_ULCF_CFG_10_09_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_10_09_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_12_11 */
+#define ADDA_ULCF_CFG_12_11_SFT                               0
+#define ADDA_ULCF_CFG_12_11_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_12_11_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_14_13 */
+#define ADDA_ULCF_CFG_14_13_SFT                               0
+#define ADDA_ULCF_CFG_14_13_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_14_13_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_16_15 */
+#define ADDA_ULCF_CFG_16_15_SFT                               0
+#define ADDA_ULCF_CFG_16_15_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_16_15_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_18_17 */
+#define ADDA_ULCF_CFG_18_17_SFT                               0
+#define ADDA_ULCF_CFG_18_17_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_18_17_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_20_19 */
+#define ADDA_ULCF_CFG_20_19_SFT                               0
+#define ADDA_ULCF_CFG_20_19_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_20_19_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_22_21 */
+#define ADDA_ULCF_CFG_22_21_SFT                               0
+#define ADDA_ULCF_CFG_22_21_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_22_21_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_24_23 */
+#define ADDA_ULCF_CFG_24_23_SFT                               0
+#define ADDA_ULCF_CFG_24_23_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_24_23_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_26_25 */
+#define ADDA_ULCF_CFG_26_25_SFT                               0
+#define ADDA_ULCF_CFG_26_25_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_26_25_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_28_27 */
+#define ADDA_ULCF_CFG_28_27_SFT                               0
+#define ADDA_ULCF_CFG_28_27_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_28_27_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_30_29 */
+#define ADDA_ULCF_CFG_30_29_SFT                               0
+#define ADDA_ULCF_CFG_30_29_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_30_29_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_ULCF_CFG_32_31 */
+#define ADDA_ULCF_CFG_32_31_SFT                               0
+#define ADDA_ULCF_CFG_32_31_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_32_31_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL1_IP_VERSION */
+#define ADDA_ULCF_IP_VERSION_SFT                              0
+#define ADDA_ULCF_IP_VERSION_MASK                             0xffffffff
+#define ADDA_ULCF_IP_VERSION_MASK_SFT                         (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_SRC_CON0 */
+#define ULCF_CFG_EN_CTL_SFT                                   31
+#define ULCF_CFG_EN_CTL_MASK                                  0x1
+#define ULCF_CFG_EN_CTL_MASK_SFT                              (0x1 << 31)
+#define UL_DMIC_PHASE_SEL_CH1_SFT                             27
+#define UL_DMIC_PHASE_SEL_CH1_MASK                            0x7
+#define UL_DMIC_PHASE_SEL_CH1_MASK_SFT                        (0x7 << 27)
+#define UL_DMIC_PHASE_SEL_CH2_SFT                             24
+#define UL_DMIC_PHASE_SEL_CH2_MASK                            0x7
+#define UL_DMIC_PHASE_SEL_CH2_MASK_SFT                        (0x7 << 24)
+#define UL_DMIC_TWO_WIRE_CTL_SFT                              23
+#define UL_DMIC_TWO_WIRE_CTL_MASK                             0x1
+#define UL_DMIC_TWO_WIRE_CTL_MASK_SFT                         (0x1 << 23)
+#define UL_MODE_3P25M_CH2_CTL_SFT                             22
+#define UL_MODE_3P25M_CH2_CTL_MASK                            0x1
+#define UL_MODE_3P25M_CH2_CTL_MASK_SFT                        (0x1 << 22)
+#define UL_MODE_3P25M_CH1_CTL_SFT                             21
+#define UL_MODE_3P25M_CH1_CTL_MASK                            0x1
+#define UL_MODE_3P25M_CH1_CTL_MASK_SFT                        (0x1 << 21)
+#define UL_VOICE_MODE_CH1_CH2_CTL_SFT                         17
+#define UL_VOICE_MODE_CH1_CH2_CTL_MASK                        0x7
+#define UL_VOICE_MODE_CH1_CH2_CTL_MASK_SFT                    (0x7 << 17)
+#define UL_AP_DMIC_ON_SFT                                     16
+#define UL_AP_DMIC_ON_MASK                                    0x1
+#define UL_AP_DMIC_ON_MASK_SFT                                (0x1 << 16)
+#define DMIC_LOW_POWER_MODE_CTL_SFT                           14
+#define DMIC_LOW_POWER_MODE_CTL_MASK                          0x3
+#define DMIC_LOW_POWER_MODE_CTL_MASK_SFT                      (0x3 << 14)
+#define UL_DISABLE_HW_CG_CTL_SFT                              12
+#define UL_DISABLE_HW_CG_CTL_MASK                             0x1
+#define UL_DISABLE_HW_CG_CTL_MASK_SFT                         (0x1 << 12)
+#define AMIC_26M_SEL_CTL_SFT                                  11
+#define AMIC_26M_SEL_CTL_MASK                                 0x1
+#define AMIC_26M_SEL_CTL_MASK_SFT                             (0x1 << 11)
+#define UL_IIR_ON_TMP_CTL_SFT                                 10
+#define UL_IIR_ON_TMP_CTL_MASK                                0x1
+#define UL_IIR_ON_TMP_CTL_MASK_SFT                            (0x1 << 10)
+#define UL_IIRMODE_CTL_SFT                                    7
+#define UL_IIRMODE_CTL_MASK                                   0x7
+#define UL_IIRMODE_CTL_MASK_SFT                               (0x7 << 7)
+#define DIGMIC_4P33M_SEL_SFT                                  6
+#define DIGMIC_4P33M_SEL_MASK                                 0x1
+#define DIGMIC_4P33M_SEL_MASK_SFT                             (0x1 << 6)
+#define DIGMIC_3P25M_1P625M_SEL_CTL_SFT                       5
+#define DIGMIC_3P25M_1P625M_SEL_CTL_MASK                      0x1
+#define DIGMIC_3P25M_1P625M_SEL_CTL_MASK_SFT                  (0x1 << 5)
+#define AMIC_6P5M_SEL_CTL_SFT                                 4
+#define AMIC_6P5M_SEL_CTL_MASK                                0x1
+#define AMIC_6P5M_SEL_CTL_MASK_SFT                            (0x1 << 4)
+#define AMIC_1P625M_SEL_CTL_SFT                               3
+#define AMIC_1P625M_SEL_CTL_MASK                              0x1
+#define AMIC_1P625M_SEL_CTL_MASK_SFT                          (0x1 << 3)
+#define UL_LOOP_BACK_MODE_CTL_SFT                             2
+#define UL_LOOP_BACK_MODE_CTL_MASK                            0x1
+#define UL_LOOP_BACK_MODE_CTL_MASK_SFT                        (0x1 << 2)
+#define UL_SDM_3_LEVEL_CTL_SFT                                1
+#define UL_SDM_3_LEVEL_CTL_MASK                               0x1
+#define UL_SDM_3_LEVEL_CTL_MASK_SFT                           (0x1 << 1)
+#define UL_SRC_ON_TMP_CTL_SFT                                 0
+#define UL_SRC_ON_TMP_CTL_MASK                                0x1
+#define UL_SRC_ON_TMP_CTL_MASK_SFT                            (0x1 << 0)
+
+/* AFE_ADDA_UL2_SRC_CON1 */
+#define ADDA_UL_GAIN_VALUE_SFT                                16
+#define ADDA_UL_GAIN_VALUE_MASK                               0xffff
+#define ADDA_UL_GAIN_VALUE_MASK_SFT                           (0xffff << 16)
+#define ADDA_UL_POSTIVEGAIN_SFT                               12
+#define ADDA_UL_POSTIVEGAIN_MASK                              0x7
+#define ADDA_UL_POSTIVEGAIN_MASK_SFT                          (0x7 << 12)
+#define ADDA_UL_ODDTAP_MODE_SFT                               11
+#define ADDA_UL_ODDTAP_MODE_MASK                              0x1
+#define ADDA_UL_ODDTAP_MODE_MASK_SFT                          (0x1 << 11)
+#define ADDA_UL_HALF_TAP_NUM_SFT                              5
+#define ADDA_UL_HALF_TAP_NUM_MASK                             0x3f
+#define ADDA_UL_HALF_TAP_NUM_MASK_SFT                         (0x3f << 5)
+#define FIFO_SOFT_RST_SFT                                     4
+#define FIFO_SOFT_RST_MASK                                    0x1
+#define FIFO_SOFT_RST_MASK_SFT                                (0x1 << 4)
+#define FIFO_SOFT_RST_EN_SFT                                  3
+#define FIFO_SOFT_RST_EN_MASK                                 0x1
+#define FIFO_SOFT_RST_EN_MASK_SFT                             (0x1 << 3)
+#define LR_SWAP_SFT                                           2
+#define LR_SWAP_MASK                                          0x1
+#define LR_SWAP_MASK_SFT                                      (0x1 << 2)
+#define GAIN_MODE_SFT                                         0
+#define GAIN_MODE_MASK                                        0x3
+#define GAIN_MODE_MASK_SFT                                    (0x3 << 0)
+
+/* AFE_ADDA_UL2_SRC_CON2 */
+#define C_DAC_EN_CTL_SFT                                      27
+#define C_DAC_EN_CTL_MASK                                     0x1
+#define C_DAC_EN_CTL_MASK_SFT                                 (0x1 << 27)
+#define C_MUTE_SW_CTL_SFT                                     26
+#define C_MUTE_SW_CTL_MASK                                    0x1
+#define C_MUTE_SW_CTL_MASK_SFT                                (0x1 << 26)
+#define C_AMP_DIV_CH2_CTL_SFT                                 21
+#define C_AMP_DIV_CH2_CTL_MASK                                0x7
+#define C_AMP_DIV_CH2_CTL_MASK_SFT                            (0x7 << 21)
+#define C_FREQ_DIV_CH2_CTL_SFT                                16
+#define C_FREQ_DIV_CH2_CTL_MASK                               0x1f
+#define C_FREQ_DIV_CH2_CTL_MASK_SFT                           (0x1f << 16)
+#define C_SINE_MODE_CH2_CTL_SFT                               12
+#define C_SINE_MODE_CH2_CTL_MASK                              0xf
+#define C_SINE_MODE_CH2_CTL_MASK_SFT                          (0xf << 12)
+#define C_AMP_DIV_CH1_CTL_SFT                                 9
+#define C_AMP_DIV_CH1_CTL_MASK                                0x7
+#define C_AMP_DIV_CH1_CTL_MASK_SFT                            (0x7 << 9)
+#define C_FREQ_DIV_CH1_CTL_SFT                                4
+#define C_FREQ_DIV_CH1_CTL_MASK                               0x1f
+#define C_FREQ_DIV_CH1_CTL_MASK_SFT                           (0x1f << 4)
+#define C_SINE_MODE_CH1_CTL_SFT                               0
+#define C_SINE_MODE_CH1_CTL_MASK                              0xf
+#define C_SINE_MODE_CH1_CTL_MASK_SFT                          (0xf << 0)
+
+/* AFE_ADDA_UL2_SRC_DEBUG */
+#define UL_SLT_CNT_FLAG_RESET_CTL_SFT                         16
+#define UL_SLT_CNT_FLAG_RESET_CTL_MASK                        0x1
+#define UL_SLT_CNT_FLAG_RESET_CTL_MASK_SFT                    (0x1 << 16)
+#define FIFO_DIGMIC_TESTIN_SFT                                12
+#define FIFO_DIGMIC_TESTIN_MASK                               0x3
+#define FIFO_DIGMIC_TESTIN_MASK_SFT                           (0x3 << 12)
+#define FIFO_DIGMIC_WDATA_TESTEN_SFT                          11
+#define FIFO_DIGMIC_WDATA_TESTEN_MASK                         0x1
+#define FIFO_DIGMIC_WDATA_TESTEN_MASK_SFT                     (0x1 << 11)
+#define SLT_CNT_THD_CTL_SFT                                   0
+#define SLT_CNT_THD_CTL_MASK                                  0x7ff
+#define SLT_CNT_THD_CTL_MASK_SFT                              (0x7ff << 0)
+
+/* AFE_ADDA_UL2_SRC_DEBUG_MON0 */
+#define SLT_CNT_FLAG_CTL_SFT                                  16
+#define SLT_CNT_FLAG_CTL_MASK                                 0x1
+#define SLT_CNT_FLAG_CTL_MASK_SFT                             (0x1 << 16)
+#define SLT_COUNTER_CTL_SFT                                   0
+#define SLT_COUNTER_CTL_MASK                                  0x7ff
+#define SLT_COUNTER_CTL_MASK_SFT                              (0x7ff << 0)
+
+/* AFE_ADDA_UL2_IIR_COEF_02_01 */
+#define ADDA_IIR_COEF_02_01_SFT                               0
+#define ADDA_IIR_COEF_02_01_MASK                              0xffffffff
+#define ADDA_IIR_COEF_02_01_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_IIR_COEF_04_03 */
+#define ADDA_IIR_COEF_04_03_SFT                               0
+#define ADDA_IIR_COEF_04_03_MASK                              0xffffffff
+#define ADDA_IIR_COEF_04_03_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_IIR_COEF_06_05 */
+#define ADDA_IIR_COEF_06_05_SFT                               0
+#define ADDA_IIR_COEF_06_05_MASK                              0xffffffff
+#define ADDA_IIR_COEF_06_05_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_IIR_COEF_08_07 */
+#define ADDA_IIR_COEF_08_07_SFT                               0
+#define ADDA_IIR_COEF_08_07_MASK                              0xffffffff
+#define ADDA_IIR_COEF_08_07_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_IIR_COEF_10_09 */
+#define ADDA_IIR_COEF_10_09_SFT                               0
+#define ADDA_IIR_COEF_10_09_MASK                              0xffffffff
+#define ADDA_IIR_COEF_10_09_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_02_01 */
+#define ADDA_ULCF_CFG_02_01_SFT                               0
+#define ADDA_ULCF_CFG_02_01_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_02_01_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_04_03 */
+#define ADDA_ULCF_CFG_04_03_SFT                               0
+#define ADDA_ULCF_CFG_04_03_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_04_03_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_06_05 */
+#define ADDA_ULCF_CFG_06_05_SFT                               0
+#define ADDA_ULCF_CFG_06_05_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_06_05_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_08_07 */
+#define ADDA_ULCF_CFG_08_07_SFT                               0
+#define ADDA_ULCF_CFG_08_07_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_08_07_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_10_09 */
+#define ADDA_ULCF_CFG_10_09_SFT                               0
+#define ADDA_ULCF_CFG_10_09_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_10_09_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_12_11 */
+#define ADDA_ULCF_CFG_12_11_SFT                               0
+#define ADDA_ULCF_CFG_12_11_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_12_11_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_14_13 */
+#define ADDA_ULCF_CFG_14_13_SFT                               0
+#define ADDA_ULCF_CFG_14_13_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_14_13_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_16_15 */
+#define ADDA_ULCF_CFG_16_15_SFT                               0
+#define ADDA_ULCF_CFG_16_15_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_16_15_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_18_17 */
+#define ADDA_ULCF_CFG_18_17_SFT                               0
+#define ADDA_ULCF_CFG_18_17_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_18_17_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_20_19 */
+#define ADDA_ULCF_CFG_20_19_SFT                               0
+#define ADDA_ULCF_CFG_20_19_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_20_19_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_22_21 */
+#define ADDA_ULCF_CFG_22_21_SFT                               0
+#define ADDA_ULCF_CFG_22_21_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_22_21_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_24_23 */
+#define ADDA_ULCF_CFG_24_23_SFT                               0
+#define ADDA_ULCF_CFG_24_23_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_24_23_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_26_25 */
+#define ADDA_ULCF_CFG_26_25_SFT                               0
+#define ADDA_ULCF_CFG_26_25_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_26_25_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_28_27 */
+#define ADDA_ULCF_CFG_28_27_SFT                               0
+#define ADDA_ULCF_CFG_28_27_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_28_27_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_30_29 */
+#define ADDA_ULCF_CFG_30_29_SFT                               0
+#define ADDA_ULCF_CFG_30_29_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_30_29_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_ULCF_CFG_32_31 */
+#define ADDA_ULCF_CFG_32_31_SFT                               0
+#define ADDA_ULCF_CFG_32_31_MASK                              0xffffffff
+#define ADDA_ULCF_CFG_32_31_MASK_SFT                          (0xffffffff << 0)
+
+/* AFE_ADDA_UL2_IP_VERSION */
+#define ADDA_ULCF_IP_VERSION_SFT                              0
+#define ADDA_ULCF_IP_VERSION_MASK                             0xffffffff
+#define ADDA_ULCF_IP_VERSION_MASK_SFT                         (0xffffffff << 0)
+
+/* AFE_ADDA_PROXIMITY_CON0 */
+#define PROXIMITY_CH1_ON_SFT                                  12
+#define PROXIMITY_CH1_ON_MASK                                 0x1
+#define PROXIMITY_CH1_ON_MASK_SFT                             (0x1 << 12)
+#define PROXIMITY_CH1_SEL_SFT                                 8
+#define PROXIMITY_CH1_SEL_MASK                                0xf
+#define PROXIMITY_CH1_SEL_MASK_SFT                            (0xf << 8)
+#define PROXIMITY_CH2_ON_SFT                                  4
+#define PROXIMITY_CH2_ON_MASK                                 0x1
+#define PROXIMITY_CH2_ON_MASK_SFT                             (0x1 << 4)
+#define PROXIMITY_CH2_SEL_SFT                                 0
+#define PROXIMITY_CH2_SEL_MASK                                0xf
+#define PROXIMITY_CH2_SEL_MASK_SFT                            (0xf << 0)
+
+/* AFE_ADDA_ULSRC_PHASE_CON0 */
+#define DMIC1_PHASE_FCLK_SEL_SFT                              30
+#define DMIC1_PHASE_FCLK_SEL_MASK                             0x3
+#define DMIC1_PHASE_FCLK_SEL_MASK_SFT                         (0x3 << 30)
+#define DMIC0_PHASE_FCLK_SEL_SFT                              28
+#define DMIC0_PHASE_FCLK_SEL_MASK                             0x3
+#define DMIC0_PHASE_FCLK_SEL_MASK_SFT                         (0x3 << 28)
+#define UL3_PHASE_FCLK_SEL_SFT                                26
+#define UL3_PHASE_FCLK_SEL_MASK                               0x3
+#define UL3_PHASE_FCLK_SEL_MASK_SFT                           (0x3 << 26)
+#define UL2_PHASE_FCLK_SEL_SFT                                24
+#define UL2_PHASE_FCLK_SEL_MASK                               0x3
+#define UL2_PHASE_FCLK_SEL_MASK_SFT                           (0x3 << 24)
+#define UL1_PHASE_FCLK_SEL_SFT                                22
+#define UL1_PHASE_FCLK_SEL_MASK                               0x3
+#define UL1_PHASE_FCLK_SEL_MASK_SFT                           (0x3 << 22)
+#define UL0_PHASE_FCLK_SEL_SFT                                20
+#define UL0_PHASE_FCLK_SEL_MASK                               0x3
+#define UL0_PHASE_FCLK_SEL_MASK_SFT                           (0x3 << 20)
+#define UL_PHASE_SYNC_FCLK_2_ON_SFT                           18
+#define UL_PHASE_SYNC_FCLK_2_ON_MASK                          0x1
+#define UL_PHASE_SYNC_FCLK_2_ON_MASK_SFT                      (0x1 << 18)
+#define UL_PHASE_SYNC_FCLK_1_ON_SFT                           17
+#define UL_PHASE_SYNC_FCLK_1_ON_MASK                          0x1
+#define UL_PHASE_SYNC_FCLK_1_ON_MASK_SFT                      (0x1 << 17)
+#define UL_PHASE_SYNC_FCLK_0_ON_SFT                           16
+#define UL_PHASE_SYNC_FCLK_0_ON_MASK                          0x1
+#define UL_PHASE_SYNC_FCLK_0_ON_MASK_SFT                      (0x1 << 16)
+#define DMIC1_PHASE_HCLK_SEL_SFT                              14
+#define DMIC1_PHASE_HCLK_SEL_MASK                             0x3
+#define DMIC1_PHASE_HCLK_SEL_MASK_SFT                         (0x3 << 14)
+#define DMIC0_PHASE_HCLK_SEL_SFT                              12
+#define DMIC0_PHASE_HCLK_SEL_MASK                             0x3
+#define DMIC0_PHASE_HCLK_SEL_MASK_SFT                         (0x3 << 12)
+#define UL3_PHASE_HCLK_SEL_SFT                                10
+#define UL3_PHASE_HCLK_SEL_MASK                               0x3
+#define UL3_PHASE_HCLK_SEL_MASK_SFT                           (0x3 << 10)
+#define UL2_PHASE_HCLK_SEL_SFT                                8
+#define UL2_PHASE_HCLK_SEL_MASK                               0x3
+#define UL2_PHASE_HCLK_SEL_MASK_SFT                           (0x3 << 8)
+#define UL1_PHASE_HCLK_SEL_SFT                                6
+#define UL1_PHASE_HCLK_SEL_MASK                               0x3
+#define UL1_PHASE_HCLK_SEL_MASK_SFT                           (0x3 << 6)
+#define UL0_PHASE_HCLK_SEL_SFT                                4
+#define UL0_PHASE_HCLK_SEL_MASK                               0x3
+#define UL0_PHASE_HCLK_SEL_MASK_SFT                           (0x3 << 4)
+#define UL_PHASE_SYNC_HCLK_2_ON_SFT                           2
+#define UL_PHASE_SYNC_HCLK_2_ON_MASK                          0x1
+#define UL_PHASE_SYNC_HCLK_2_ON_MASK_SFT                      (0x1 << 2)
+#define UL_PHASE_SYNC_HCLK_1_ON_SFT                           1
+#define UL_PHASE_SYNC_HCLK_1_ON_MASK                          0x1
+#define UL_PHASE_SYNC_HCLK_1_ON_MASK_SFT                      (0x1 << 1)
+#define UL_PHASE_SYNC_HCLK_0_ON_SFT                           0
+#define UL_PHASE_SYNC_HCLK_0_ON_MASK                          0x1
+#define UL_PHASE_SYNC_HCLK_0_ON_MASK_SFT                      (0x1 << 0)
+
+/* AFE_ADDA_ULSRC_PHASE_CON1 */
+#define DMIC_CLK_PHASE_SYNC_SET_SFT                           31
+#define DMIC_CLK_PHASE_SYNC_SET_MASK                          0x1
+#define DMIC_CLK_PHASE_SYNC_SET_MASK_SFT                      (0x1 << 31)
+#define DMIC1_PHASE_SYNC_FCLK_SET_SFT                         11
+#define DMIC1_PHASE_SYNC_FCLK_SET_MASK                        0x1
+#define DMIC1_PHASE_SYNC_FCLK_SET_MASK_SFT                    (0x1 << 11)
+#define DMIC1_PHASE_SYNC_HCLK_SET_SFT                         10
+#define DMIC1_PHASE_SYNC_HCLK_SET_MASK                        0x1
+#define DMIC1_PHASE_SYNC_HCLK_SET_MASK_SFT                    (0x1 << 10)
+#define DMIC0_PHASE_SYNC_FCLK_SET_SFT                         9
+#define DMIC0_PHASE_SYNC_FCLK_SET_MASK                        0x1
+#define DMIC0_PHASE_SYNC_FCLK_SET_MASK_SFT                    (0x1 << 9)
+#define DMIC0_PHASE_SYNC_HCLK_SET_SFT                         8
+#define DMIC0_PHASE_SYNC_HCLK_SET_MASK                        0x1
+#define DMIC0_PHASE_SYNC_HCLK_SET_MASK_SFT                    (0x1 << 8)
+#define UL3_PHASE_SYNC_FCLK_SET_SFT                           7
+#define UL3_PHASE_SYNC_FCLK_SET_MASK                          0x1
+#define UL3_PHASE_SYNC_FCLK_SET_MASK_SFT                      (0x1 << 7)
+#define UL3_PHASE_SYNC_HCLK_SET_SFT                           6
+#define UL3_PHASE_SYNC_HCLK_SET_MASK                          0x1
+#define UL3_PHASE_SYNC_HCLK_SET_MASK_SFT                      (0x1 << 6)
+#define UL2_PHASE_SYNC_FCLK_SET_SFT                           5
+#define UL2_PHASE_SYNC_FCLK_SET_MASK                          0x1
+#define UL2_PHASE_SYNC_FCLK_SET_MASK_SFT                      (0x1 << 5)
+#define UL2_PHASE_SYNC_HCLK_SET_SFT                           4
+#define UL2_PHASE_SYNC_HCLK_SET_MASK                          0x1
+#define UL2_PHASE_SYNC_HCLK_SET_MASK_SFT                      (0x1 << 4)
+#define UL1_PHASE_SYNC_FCLK_SET_SFT                           3
+#define UL1_PHASE_SYNC_FCLK_SET_MASK                          0x1
+#define UL1_PHASE_SYNC_FCLK_SET_MASK_SFT                      (0x1 << 3)
+#define UL1_PHASE_SYNC_HCLK_SET_SFT                           2
+#define UL1_PHASE_SYNC_HCLK_SET_MASK                          0x1
+#define UL1_PHASE_SYNC_HCLK_SET_MASK_SFT                      (0x1 << 2)
+#define UL0_PHASE_SYNC_FCLK_SET_SFT                           1
+#define UL0_PHASE_SYNC_FCLK_SET_MASK                          0x1
+#define UL0_PHASE_SYNC_FCLK_SET_MASK_SFT                      (0x1 << 1)
+#define UL0_PHASE_SYNC_HCLK_SET_SFT                           0
+#define UL0_PHASE_SYNC_HCLK_SET_MASK                          0x1
+#define UL0_PHASE_SYNC_HCLK_SET_MASK_SFT                      (0x1 << 0)
+
+/* AFE_ADDA_ULSRC_PHASE_CON2 */
+#define DMIC1_PHASE_SYNC_1X_EN_SEL_SFT                        26
+#define DMIC1_PHASE_SYNC_1X_EN_SEL_MASK                       0x3
+#define DMIC1_PHASE_SYNC_1X_EN_SEL_MASK_SFT                   (0x3 << 26)
+#define DMIC0_PHASE_SYNC_1X_EN_SEL_SFT                        24
+#define DMIC0_PHASE_SYNC_1X_EN_SEL_MASK                       0x3
+#define DMIC0_PHASE_SYNC_1X_EN_SEL_MASK_SFT                   (0x3 << 24)
+#define UL3_PHASE_SYNC_1X_EN_SEL_SFT                          22
+#define UL3_PHASE_SYNC_1X_EN_SEL_MASK                         0x3
+#define UL3_PHASE_SYNC_1X_EN_SEL_MASK_SFT                     (0x3 << 22)
+#define UL2_PHASE_SYNC_1X_EN_SEL_SFT                          20
+#define UL2_PHASE_SYNC_1X_EN_SEL_MASK                         0x3
+#define UL2_PHASE_SYNC_1X_EN_SEL_MASK_SFT                     (0x3 << 20)
+#define UL1_PHASE_SYNC_1X_EN_SEL_SFT                          18
+#define UL1_PHASE_SYNC_1X_EN_SEL_MASK                         0x3
+#define UL1_PHASE_SYNC_1X_EN_SEL_MASK_SFT                     (0x3 << 18)
+#define UL0_PHASE_SYNC_1X_EN_SEL_SFT                          16
+#define UL0_PHASE_SYNC_1X_EN_SEL_MASK                         0x3
+#define UL0_PHASE_SYNC_1X_EN_SEL_MASK_SFT                     (0x3 << 16)
+#define UL_PHASE_SYNC_FCLK_1X_EN_2_ON_SFT                     5
+#define UL_PHASE_SYNC_FCLK_1X_EN_2_ON_MASK                    0x1
+#define UL_PHASE_SYNC_FCLK_1X_EN_2_ON_MASK_SFT                (0x1 << 5)
+#define UL_PHASE_SYNC_FCLK_1X_EN_1_ON_SFT                     4
+#define UL_PHASE_SYNC_FCLK_1X_EN_1_ON_MASK                    0x1
+#define UL_PHASE_SYNC_FCLK_1X_EN_1_ON_MASK_SFT                (0x1 << 4)
+#define UL_PHASE_SYNC_FCLK_1X_EN_0_ON_SFT                     3
+#define UL_PHASE_SYNC_FCLK_1X_EN_0_ON_MASK                    0x1
+#define UL_PHASE_SYNC_FCLK_1X_EN_0_ON_MASK_SFT                (0x1 << 3)
+#define UL_PHASE_SYNC_HCLK_1X_EN_2_ON_SFT                     2
+#define UL_PHASE_SYNC_HCLK_1X_EN_2_ON_MASK                    0x1
+#define UL_PHASE_SYNC_HCLK_1X_EN_2_ON_MASK_SFT                (0x1 << 2)
+#define UL_PHASE_SYNC_HCLK_1X_EN_1_ON_SFT                     1
+#define UL_PHASE_SYNC_HCLK_1X_EN_1_ON_MASK                    0x1
+#define UL_PHASE_SYNC_HCLK_1X_EN_1_ON_MASK_SFT                (0x1 << 1)
+#define UL_PHASE_SYNC_HCLK_1X_EN_0_ON_SFT                     0
+#define UL_PHASE_SYNC_HCLK_1X_EN_0_ON_MASK                    0x1
+#define UL_PHASE_SYNC_HCLK_1X_EN_0_ON_MASK_SFT                (0x1 << 0)
+
+/* AFE_ADDA_ULSRC_PHASE_CON3 */
+#define DMIC1_PHASE_SYNC_SOFT_RST_SEL_SFT                     26
+#define DMIC1_PHASE_SYNC_SOFT_RST_SEL_MASK                    0x3
+#define DMIC1_PHASE_SYNC_SOFT_RST_SEL_MASK_SFT                (0x3 << 26)
+#define DMIC0_PHASE_SYNC_SOFT_RST_SEL_SFT                     24
+#define DMIC0_PHASE_SYNC_SOFT_RST_SEL_MASK                    0x3
+#define DMIC0_PHASE_SYNC_SOFT_RST_SEL_MASK_SFT                (0x3 << 24)
+#define UL3_PHASE_SYNC_SOFT_RST_SEL_SFT                       22
+#define UL3_PHASE_SYNC_SOFT_RST_SEL_MASK                      0x3
+#define UL3_PHASE_SYNC_SOFT_RST_SEL_MASK_SFT                  (0x3 << 22)
+#define UL2_PHASE_SYNC_SOFT_RST_SEL_SFT                       20
+#define UL2_PHASE_SYNC_SOFT_RST_SEL_MASK                      0x3
+#define UL2_PHASE_SYNC_SOFT_RST_SEL_MASK_SFT                  (0x3 << 20)
+#define UL1_PHASE_SYNC_SOFT_RST_SEL_SFT                       18
+#define UL1_PHASE_SYNC_SOFT_RST_SEL_MASK                      0x3
+#define UL1_PHASE_SYNC_SOFT_RST_SEL_MASK_SFT                  (0x3 << 18)
+#define UL0_PHASE_SYNC_SOFT_RST_SEL_SFT                       16
+#define UL0_PHASE_SYNC_SOFT_RST_SEL_MASK                      0x3
+#define UL0_PHASE_SYNC_SOFT_RST_SEL_MASK_SFT                  (0x3 << 16)
+#define DMIC1_PHASE_SYNC_CH1_FIFO_SEL_SFT                     13
+#define DMIC1_PHASE_SYNC_CH1_FIFO_SEL_MASK                    0x1
+#define DMIC1_PHASE_SYNC_CH1_FIFO_SEL_MASK_SFT                (0x1 << 13)
+#define DMIC0_PHASE_SYNC_CH1_FIFO_SEL_SFT                     12
+#define DMIC0_PHASE_SYNC_CH1_FIFO_SEL_MASK                    0x1
+#define DMIC0_PHASE_SYNC_CH1_FIFO_SEL_MASK_SFT                (0x1 << 12)
+#define UL3_PHASE_SYNC_CH1_FIFO_SEL_SFT                       11
+#define UL3_PHASE_SYNC_CH1_FIFO_SEL_MASK                      0x1
+#define UL3_PHASE_SYNC_CH1_FIFO_SEL_MASK_SFT                  (0x1 << 11)
+#define UL2_PHASE_SYNC_CH1_FIFO_SEL_SFT                       10
+#define UL2_PHASE_SYNC_CH1_FIFO_SEL_MASK                      0x1
+#define UL2_PHASE_SYNC_CH1_FIFO_SEL_MASK_SFT                  (0x1 << 10)
+#define UL1_PHASE_SYNC_CH1_FIFO_SEL_SFT                       9
+#define UL1_PHASE_SYNC_CH1_FIFO_SEL_MASK                      0x1
+#define UL1_PHASE_SYNC_CH1_FIFO_SEL_MASK_SFT                  (0x1 << 9)
+#define UL0_PHASE_SYNC_CH1_FIFO_SEL_SFT                       8
+#define UL0_PHASE_SYNC_CH1_FIFO_SEL_MASK                      0x1
+#define UL0_PHASE_SYNC_CH1_FIFO_SEL_MASK_SFT                  (0x1 << 8)
+#define UL_PHASE_SYNC_SOFT_RST_EN_2_ON_SFT                    5
+#define UL_PHASE_SYNC_SOFT_RST_EN_2_ON_MASK                   0x1
+#define UL_PHASE_SYNC_SOFT_RST_EN_2_ON_MASK_SFT               (0x1 << 5)
+#define UL_PHASE_SYNC_SOFT_RST_EN_1_ON_SFT                    4
+#define UL_PHASE_SYNC_SOFT_RST_EN_1_ON_MASK                   0x1
+#define UL_PHASE_SYNC_SOFT_RST_EN_1_ON_MASK_SFT               (0x1 << 4)
+#define UL_PHASE_SYNC_SOFT_RST_EN_0_ON_SFT                    3
+#define UL_PHASE_SYNC_SOFT_RST_EN_0_ON_MASK                   0x1
+#define UL_PHASE_SYNC_SOFT_RST_EN_0_ON_MASK_SFT               (0x1 << 3)
+#define UL_PHASE_SYNC_SOFT_RST_2_ON_SFT                       2
+#define UL_PHASE_SYNC_SOFT_RST_2_ON_MASK                      0x1
+#define UL_PHASE_SYNC_SOFT_RST_2_ON_MASK_SFT                  (0x1 << 2)
+#define UL_PHASE_SYNC_SOFT_RST_1_ON_SFT                       1
+#define UL_PHASE_SYNC_SOFT_RST_1_ON_MASK                      0x1
+#define UL_PHASE_SYNC_SOFT_RST_1_ON_MASK_SFT                  (0x1 << 1)
+#define UL_PHASE_SYNC_SOFT_RST_0_ON_SFT                       0
+#define UL_PHASE_SYNC_SOFT_RST_0_ON_MASK                      0x1
+#define UL_PHASE_SYNC_SOFT_RST_0_ON_MASK_SFT                  (0x1 << 0)
+
+/* AFE_MTKAIF_IPM_VER_MON */
+#define RG_MTKAIF_IPM_VER_MON_SFT                             0
+#define RG_MTKAIF_IPM_VER_MON_MASK                            0xffffffff
+#define RG_MTKAIF_IPM_VER_MON_MASK_SFT                        (0xffffffff << 0)
+
+/* AFE_MTKAIF_MON_SEL */
+#define RG_MTKAIF_MON_SEL_SFT                                 0
+#define RG_MTKAIF_MON_SEL_MASK                                0xff
+#define RG_MTKAIF_MON_SEL_MASK_SFT                            (0xff << 0)
+
+/* AFE_MTKAIF_MON */
+#define RG_MTKAIF_MON_SFT                                     0
+#define RG_MTKAIF_MON_MASK                                    0xffffffff
+#define RG_MTKAIF_MON_MASK_SFT                                (0xffffffff << 0)
+
+/* AFE_MTKAIF0_CFG0 */
+#define RG_MTKAIF0_RXIF_CLKINV_SFT                            31
+#define RG_MTKAIF0_RXIF_CLKINV_MASK                           0x1
+#define RG_MTKAIF0_RXIF_CLKINV_MASK_SFT                       (0x1 << 31)
+#define RG_MTKAIF0_RXIF_BYPASS_SRC_SFT                        17
+#define RG_MTKAIF0_RXIF_BYPASS_SRC_MASK                       0x1
+#define RG_MTKAIF0_RXIF_BYPASS_SRC_MASK_SFT                   (0x1 << 17)
+#define RG_MTKAIF0_RXIF_PROTOCOL2_SFT                         16
+#define RG_MTKAIF0_RXIF_PROTOCOL2_MASK                        0x1
+#define RG_MTKAIF0_RXIF_PROTOCOL2_MASK_SFT                    (0x1 << 16)
+#define RG_MTKAIF0_TXIF_NLE_DEBUG_SFT                         8
+#define RG_MTKAIF0_TXIF_NLE_DEBUG_MASK                        0x1
+#define RG_MTKAIF0_TXIF_NLE_DEBUG_MASK_SFT                    (0x1 << 8)
+#define RG_MTKAIF0_TXIF_BYPASS_SRC_SFT                        5
+#define RG_MTKAIF0_TXIF_BYPASS_SRC_MASK                       0x1
+#define RG_MTKAIF0_TXIF_BYPASS_SRC_MASK_SFT                   (0x1 << 5)
+#define RG_MTKAIF0_TXIF_PROTOCOL2_SFT                         4
+#define RG_MTKAIF0_TXIF_PROTOCOL2_MASK                        0x1
+#define RG_MTKAIF0_TXIF_PROTOCOL2_MASK_SFT                    (0x1 << 4)
+#define RG_MTKAIF0_TXIF_8TO5_SFT                              2
+#define RG_MTKAIF0_TXIF_8TO5_MASK                             0x1
+#define RG_MTKAIF0_TXIF_8TO5_MASK_SFT                         (0x1 << 2)
+#define RG_MTKAIF0_RXIF_8TO5_SFT                              1
+#define RG_MTKAIF0_RXIF_8TO5_MASK                             0x1
+#define RG_MTKAIF0_RXIF_8TO5_MASK_SFT                         (0x1 << 1)
+#define RG_MTKAIF0_TX2RX_LOOPBACK1_SFT                        0
+#define RG_MTKAIF0_TX2RX_LOOPBACK1_MASK                       0x1
+#define RG_MTKAIF0_TX2RX_LOOPBACK1_MASK_SFT                   (0x1 << 0)
+
+/* AFE_MTKAIF0_TX_CFG0 */
+#define RG_MTKAIF0_TXIF_NLE_FIFO_SWAP_SFT                     23
+#define RG_MTKAIF0_TXIF_NLE_FIFO_SWAP_MASK                    0x1
+#define RG_MTKAIF0_TXIF_NLE_FIFO_SWAP_MASK_SFT                (0x1 << 23)
+#define RG_MTKAIF0_TXIF_NLE_FIFO_RSP_SFT                      20
+#define RG_MTKAIF0_TXIF_NLE_FIFO_RSP_MASK                     0x7
+#define RG_MTKAIF0_TXIF_NLE_FIFO_RSP_MASK_SFT                 (0x7 << 20)
+#define RG_MTKAIF0_TXIF_FIFO_SWAP_SFT                         15
+#define RG_MTKAIF0_TXIF_FIFO_SWAP_MASK                        0x1
+#define RG_MTKAIF0_TXIF_FIFO_SWAP_MASK_SFT                    (0x1 << 15)
+#define RG_MTKAIF0_TXIF_FIFO_RSP_SFT                          12
+#define RG_MTKAIF0_TXIF_FIFO_RSP_MASK                         0x7
+#define RG_MTKAIF0_TXIF_FIFO_RSP_MASK_SFT                     (0x7 << 12)
+#define RG_MTKAIF0_TXIF_SYNC_WORD1_SFT                        4
+#define RG_MTKAIF0_TXIF_SYNC_WORD1_MASK                       0x7
+#define RG_MTKAIF0_TXIF_SYNC_WORD1_MASK_SFT                   (0x7 << 4)
+#define RG_MTKAIF0_TXIF_SYNC_WORD0_SFT                        0
+#define RG_MTKAIF0_TXIF_SYNC_WORD0_MASK                       0x7
+#define RG_MTKAIF0_TXIF_SYNC_WORD0_MASK_SFT                   (0x7 << 0)
+
+/* AFE_MTKAIF0_RX_CFG0 */
+#define RG_MTKAIF0_RXIF_VOICE_MODE_SFT                        20
+#define RG_MTKAIF0_RXIF_VOICE_MODE_MASK                       0xf
+#define RG_MTKAIF0_RXIF_VOICE_MODE_MASK_SFT                   (0xf << 20)
+#define RG_MTKAIF0_RXIF_DETECT_ON_SFT                         16
+#define RG_MTKAIF0_RXIF_DETECT_ON_MASK                        0x1
+#define RG_MTKAIF0_RXIF_DETECT_ON_MASK_SFT                    (0x1 << 16)
+#define RG_MTKAIF0_RXIF_DATA_BIT_SFT                          8
+#define RG_MTKAIF0_RXIF_DATA_BIT_MASK                         0x7
+#define RG_MTKAIF0_RXIF_DATA_BIT_MASK_SFT                     (0x7 << 8)
+#define RG_MTKAIF0_RXIF_FIFO_RSP_SFT                          4
+#define RG_MTKAIF0_RXIF_FIFO_RSP_MASK                         0x7
+#define RG_MTKAIF0_RXIF_FIFO_RSP_MASK_SFT                     (0x7 << 4)
+#define RG_MTKAIF0_RXIF_DATA_MODE_SFT                         0
+#define RG_MTKAIF0_RXIF_DATA_MODE_MASK                        0x1
+#define RG_MTKAIF0_RXIF_DATA_MODE_MASK_SFT                    (0x1 << 0)
+
+/* AFE_MTKAIF0_RX_CFG1 */
+#define RG_MTKAIF0_RXIF_CLEAR_SYNC_FAIL_SFT                   28
+#define RG_MTKAIF0_RXIF_CLEAR_SYNC_FAIL_MASK                  0x1
+#define RG_MTKAIF0_RXIF_CLEAR_SYNC_FAIL_MASK_SFT              (0x1 << 28)
+#define RG_MTKAIF0_RXIF_SYNC_CNT_TABLE_SFT                    16
+#define RG_MTKAIF0_RXIF_SYNC_CNT_TABLE_MASK                   0xfff
+#define RG_MTKAIF0_RXIF_SYNC_CNT_TABLE_MASK_SFT               (0xfff << 16)
+#define RG_MTKAIF0_RXIF_SYNC_SEARCH_TABLE_SFT                 12
+#define RG_MTKAIF0_RXIF_SYNC_SEARCH_TABLE_MASK                0xf
+#define RG_MTKAIF0_RXIF_SYNC_SEARCH_TABLE_MASK_SFT            (0xf << 12)
+#define RG_MTKAIF0_RXIF_INVALID_SYNC_CHECK_ROUND_SFT          8
+#define RG_MTKAIF0_RXIF_INVALID_SYNC_CHECK_ROUND_MASK         0xf
+#define RG_MTKAIF0_RXIF_INVALID_SYNC_CHECK_ROUND_MASK_SFT     (0xf << 8)
+#define RG_MTKAIF0_RXIF_SYNC_CHECK_ROUND_SFT                  4
+#define RG_MTKAIF0_RXIF_SYNC_CHECK_ROUND_MASK                 0xf
+#define RG_MTKAIF0_RXIF_SYNC_CHECK_ROUND_MASK_SFT             (0xf << 4)
+
+/* AFE_MTKAIF0_RX_CFG2 */
+#define RG_MTKAIF0_RXIF_SYNC_WORD1_DISABLE_SFT                27
+#define RG_MTKAIF0_RXIF_SYNC_WORD1_DISABLE_MASK               0x1
+#define RG_MTKAIF0_RXIF_SYNC_WORD1_DISABLE_MASK_SFT           (0x1 << 27)
+#define RG_MTKAIF0_RXIF_SYNC_WORD1_SFT                        24
+#define RG_MTKAIF0_RXIF_SYNC_WORD1_MASK                       0x7
+#define RG_MTKAIF0_RXIF_SYNC_WORD1_MASK_SFT                   (0x7 << 24)
+#define RG_MTKAIF0_RXIF_SYNC_WORD0_DISABLE_SFT                23
+#define RG_MTKAIF0_RXIF_SYNC_WORD0_DISABLE_MASK               0x1
+#define RG_MTKAIF0_RXIF_SYNC_WORD0_DISABLE_MASK_SFT           (0x1 << 23)
+#define RG_MTKAIF0_RXIF_SYNC_WORD0_SFT                        20
+#define RG_MTKAIF0_RXIF_SYNC_WORD0_MASK                       0x7
+#define RG_MTKAIF0_RXIF_SYNC_WORD0_MASK_SFT                   (0x7 << 20)
+#define RG_MTKAIF0_RXIF_DELAY_CYCLE_SFT                       12
+#define RG_MTKAIF0_RXIF_DELAY_CYCLE_MASK                      0xf
+#define RG_MTKAIF0_RXIF_DELAY_CYCLE_MASK_SFT                  (0xf << 12)
+#define RG_MTKAIF0_RXIF_DELAY_DATA_SFT                        8
+#define RG_MTKAIF0_RXIF_DELAY_DATA_MASK                       0x1
+#define RG_MTKAIF0_RXIF_DELAY_DATA_MASK_SFT                   (0x1 << 8)
+
+/* AFE_MTKAIF1_CFG0 */
+#define RG_MTKAIF1_RXIF_CLKINV_ADC_SFT                        31
+#define RG_MTKAIF1_RXIF_CLKINV_ADC_MASK                       0x1
+#define RG_MTKAIF1_RXIF_CLKINV_ADC_MASK_SFT                   (0x1 << 31)
+#define RG_MTKAIF1_RXIF_BYPASS_SRC_SFT                        17
+#define RG_MTKAIF1_RXIF_BYPASS_SRC_MASK                       0x1
+#define RG_MTKAIF1_RXIF_BYPASS_SRC_MASK_SFT                   (0x1 << 17)
+#define RG_MTKAIF1_RXIF_PROTOCOL2_SFT                         16
+#define RG_MTKAIF1_RXIF_PROTOCOL2_MASK                        0x1
+#define RG_MTKAIF1_RXIF_PROTOCOL2_MASK_SFT                    (0x1 << 16)
+#define RG_MTKAIF1_TXIF_NLE_DEBUG_SFT                         8
+#define RG_MTKAIF1_TXIF_NLE_DEBUG_MASK                        0x1
+#define RG_MTKAIF1_TXIF_NLE_DEBUG_MASK_SFT                    (0x1 << 8)
+#define RG_MTKAIF1_TXIF_BYPASS_SRC_SFT                        5
+#define RG_MTKAIF1_TXIF_BYPASS_SRC_MASK                       0x1
+#define RG_MTKAIF1_TXIF_BYPASS_SRC_MASK_SFT                   (0x1 << 5)
+#define RG_MTKAIF1_TXIF_PROTOCOL2_SFT                         4
+#define RG_MTKAIF1_TXIF_PROTOCOL2_MASK                        0x1
+#define RG_MTKAIF1_TXIF_PROTOCOL2_MASK_SFT                    (0x1 << 4)
+#define RG_MTKAIF1_TXIF_8TO5_SFT                              2
+#define RG_MTKAIF1_TXIF_8TO5_MASK                             0x1
+#define RG_MTKAIF1_TXIF_8TO5_MASK_SFT                         (0x1 << 2)
+#define RG_MTKAIF1_RXIF_8TO5_SFT                              1
+#define RG_MTKAIF1_RXIF_8TO5_MASK                             0x1
+#define RG_MTKAIF1_RXIF_8TO5_MASK_SFT                         (0x1 << 1)
+#define RG_MTKAIF1_IF_LOOPBACK1_SFT                           0
+#define RG_MTKAIF1_IF_LOOPBACK1_MASK                          0x1
+#define RG_MTKAIF1_IF_LOOPBACK1_MASK_SFT                      (0x1 << 0)
+
+/* AFE_MTKAIF1_TX_CFG0 */
+#define RG_MTKAIF1_TXIF_NLE_FIFO_SWAP_SFT                     23
+#define RG_MTKAIF1_TXIF_NLE_FIFO_SWAP_MASK                    0x1
+#define RG_MTKAIF1_TXIF_NLE_FIFO_SWAP_MASK_SFT                (0x1 << 23)
+#define RG_MTKAIF1_TXIF_NLE_FIFO_RSP_SFT                      20
+#define RG_MTKAIF1_TXIF_NLE_FIFO_RSP_MASK                     0x7
+#define RG_MTKAIF1_TXIF_NLE_FIFO_RSP_MASK_SFT                 (0x7 << 20)
+#define RG_MTKAIF1_TXIF_FIFO_SWAP_SFT                         15
+#define RG_MTKAIF1_TXIF_FIFO_SWAP_MASK                        0x1
+#define RG_MTKAIF1_TXIF_FIFO_SWAP_MASK_SFT                    (0x1 << 15)
+#define RG_MTKAIF1_TXIF_FIFO_RSP_SFT                          12
+#define RG_MTKAIF1_TXIF_FIFO_RSP_MASK                         0x7
+#define RG_MTKAIF1_TXIF_FIFO_RSP_MASK_SFT                     (0x7 << 12)
+#define RG_MTKAIF1_TXIF_SYNC_WORD1_SFT                        4
+#define RG_MTKAIF1_TXIF_SYNC_WORD1_MASK                       0x7
+#define RG_MTKAIF1_TXIF_SYNC_WORD1_MASK_SFT                   (0x7 << 4)
+#define RG_MTKAIF1_TXIF_SYNC_WORD0_SFT                        0
+#define RG_MTKAIF1_TXIF_SYNC_WORD0_MASK                       0x7
+#define RG_MTKAIF1_TXIF_SYNC_WORD0_MASK_SFT                   (0x7 << 0)
+
+/* AFE_MTKAIF1_RX_CFG0 */
+#define RG_MTKAIF1_RXIF_VOICE_MODE_SFT                        20
+#define RG_MTKAIF1_RXIF_VOICE_MODE_MASK                       0xf
+#define RG_MTKAIF1_RXIF_VOICE_MODE_MASK_SFT                   (0xf << 20)
+#define RG_MTKAIF1_RXIF_DETECT_ON_SFT                         16
+#define RG_MTKAIF1_RXIF_DETECT_ON_MASK                        0x1
+#define RG_MTKAIF1_RXIF_DETECT_ON_MASK_SFT                    (0x1 << 16)
+#define RG_MTKAIF1_RXIF_DATA_BIT_SFT                          8
+#define RG_MTKAIF1_RXIF_DATA_BIT_MASK                         0x7
+#define RG_MTKAIF1_RXIF_DATA_BIT_MASK_SFT                     (0x7 << 8)
+#define RG_MTKAIF1_RXIF_FIFO_RSP_SFT                          4
+#define RG_MTKAIF1_RXIF_FIFO_RSP_MASK                         0x7
+#define RG_MTKAIF1_RXIF_FIFO_RSP_MASK_SFT                     (0x7 << 4)
+#define RG_MTKAIF1_RXIF_DATA_MODE_SFT                         0
+#define RG_MTKAIF1_RXIF_DATA_MODE_MASK                        0x1
+#define RG_MTKAIF1_RXIF_DATA_MODE_MASK_SFT                    (0x1 << 0)
+
+/* AFE_MTKAIF1_RX_CFG1 */
+#define RG_MTKAIF1_RXIF_CLEAR_SYNC_FAIL_SFT                   28
+#define RG_MTKAIF1_RXIF_CLEAR_SYNC_FAIL_MASK                  0x1
+#define RG_MTKAIF1_RXIF_CLEAR_SYNC_FAIL_MASK_SFT              (0x1 << 28)
+#define RG_MTKAIF1_RXIF_SYNC_CNT_TABLE_SFT                    16
+#define RG_MTKAIF1_RXIF_SYNC_CNT_TABLE_MASK                   0xfff
+#define RG_MTKAIF1_RXIF_SYNC_CNT_TABLE_MASK_SFT               (0xfff << 16)
+#define RG_MTKAIF1_RXIF_SYNC_SEARCH_TABLE_SFT                 12
+#define RG_MTKAIF1_RXIF_SYNC_SEARCH_TABLE_MASK                0xf
+#define RG_MTKAIF1_RXIF_SYNC_SEARCH_TABLE_MASK_SFT            (0xf << 12)
+#define RG_MTKAIF1_RXIF_INVALID_SYNC_CHECK_ROUND_SFT          8
+#define RG_MTKAIF1_RXIF_INVALID_SYNC_CHECK_ROUND_MASK         0xf
+#define RG_MTKAIF1_RXIF_INVALID_SYNC_CHECK_ROUND_MASK_SFT     (0xf << 8)
+#define RG_MTKAIF1_RXIF_SYNC_CHECK_ROUND_SFT                  4
+#define RG_MTKAIF1_RXIF_SYNC_CHECK_ROUND_MASK                 0xf
+#define RG_MTKAIF1_RXIF_SYNC_CHECK_ROUND_MASK_SFT             (0xf << 4)
+
+/* AFE_MTKAIF1_RX_CFG2 */
+#define RG_MTKAIF1_RXIF_SYNC_WORD1_DISABLE_SFT                27
+#define RG_MTKAIF1_RXIF_SYNC_WORD1_DISABLE_MASK               0x1
+#define RG_MTKAIF1_RXIF_SYNC_WORD1_DISABLE_MASK_SFT           (0x1 << 27)
+#define RG_MTKAIF1_RXIF_SYNC_WORD1_SFT                        24
+#define RG_MTKAIF1_RXIF_SYNC_WORD1_MASK                       0x7
+#define RG_MTKAIF1_RXIF_SYNC_WORD1_MASK_SFT                   (0x7 << 24)
+#define RG_MTKAIF1_RXIF_SYNC_WORD0_DISABLE_SFT                23
+#define RG_MTKAIF1_RXIF_SYNC_WORD0_DISABLE_MASK               0x1
+#define RG_MTKAIF1_RXIF_SYNC_WORD0_DISABLE_MASK_SFT           (0x1 << 23)
+#define RG_MTKAIF1_RXIF_SYNC_WORD0_SFT                        20
+#define RG_MTKAIF1_RXIF_SYNC_WORD0_MASK                       0x7
+#define RG_MTKAIF1_RXIF_SYNC_WORD0_MASK_SFT                   (0x7 << 20)
+#define RG_MTKAIF1_RXIF_DELAY_CYCLE_SFT                       12
+#define RG_MTKAIF1_RXIF_DELAY_CYCLE_MASK                      0xf
+#define RG_MTKAIF1_RXIF_DELAY_CYCLE_MASK_SFT                  (0xf << 12)
+#define RG_MTKAIF1_RXIF_DELAY_DATA_SFT                        8
+#define RG_MTKAIF1_RXIF_DELAY_DATA_MASK                       0x1
+#define RG_MTKAIF1_RXIF_DELAY_DATA_MASK_SFT                   (0x1 << 8)
+
+/* AFE_AUD_PAD_TOP_CFG0 */
+#define AUD_PAD_TOP_FIFO_RSP_SFT                              4
+#define AUD_PAD_TOP_FIFO_RSP_MASK                             0xf
+#define AUD_PAD_TOP_FIFO_RSP_MASK_SFT                         (0xf << 4)
+#define RG_RX_PROTOCOL2_SFT                                   3
+#define RG_RX_PROTOCOL2_MASK                                  0x1
+#define RG_RX_PROTOCOL2_MASK_SFT                              (0x1 << 3)
+#define RG_RX_FIFO_ON_SFT                                     0
+#define RG_RX_FIFO_ON_MASK                                    0x1
+#define RG_RX_FIFO_ON_MASK_SFT                                (0x1 << 0)
+
+/* AFE_AUD_PAD_TOP_MON */
+#define AUD_PAD_TOP_MON_SFT                                   0
+#define AUD_PAD_TOP_MON_MASK                                  0xffff
+#define AUD_PAD_TOP_MON_MASK_SFT                              (0xffff << 0)
+
+/* AFE_ADDA_MTKAIFV4_TX_CFG0 */
+#define MTKAIFV4_TXIF_EN_SEL_SFT                              12
+#define MTKAIFV4_TXIF_EN_SEL_MASK                             0x1
+#define MTKAIFV4_TXIF_EN_SEL_MASK_SFT                         (0x1 << 12)
+#define MTKAIFV4_TXIF_V4_SFT                                  11
+#define MTKAIFV4_TXIF_V4_MASK                                 0x1
+#define MTKAIFV4_TXIF_V4_MASK_SFT                             (0x1 << 11)
+#define MTKAIFV4_ADDA6_OUT_EN_SEL_SFT                         10
+#define MTKAIFV4_ADDA6_OUT_EN_SEL_MASK                        0x1
+#define MTKAIFV4_ADDA6_OUT_EN_SEL_MASK_SFT                    (0x1 << 10)
+#define MTKAIFV4_ADDA_OUT_EN_SEL_SFT                          9
+#define MTKAIFV4_ADDA_OUT_EN_SEL_MASK                         0x1
+#define MTKAIFV4_ADDA_OUT_EN_SEL_MASK_SFT                     (0x1 << 9)
+#define MTKAIFV4_TXIF_INPUT_MODE_SFT                          4
+#define MTKAIFV4_TXIF_INPUT_MODE_MASK                         0x1f
+#define MTKAIFV4_TXIF_INPUT_MODE_MASK_SFT                     (0x1f << 4)
+#define MTKAIFV4_TXIF_FOUR_CHANNEL_SFT                        1
+#define MTKAIFV4_TXIF_FOUR_CHANNEL_MASK                       0x1
+#define MTKAIFV4_TXIF_FOUR_CHANNEL_MASK_SFT                   (0x1 << 1)
+#define MTKAIFV4_TXIF_AFE_ON_SFT                              0
+#define MTKAIFV4_TXIF_AFE_ON_MASK                             0x1
+#define MTKAIFV4_TXIF_AFE_ON_MASK_SFT                         (0x1 << 0)
+
+/* AFE_ADDA6_MTKAIFV4_TX_CFG0 */
+#define ADDA6_MTKAIFV4_TXIF_EN_SEL_SFT                        12
+#define ADDA6_MTKAIFV4_TXIF_EN_SEL_MASK                       0x1
+#define ADDA6_MTKAIFV4_TXIF_EN_SEL_MASK_SFT                   (0x1 << 12)
+#define ADDA6_MTKAIFV4_TXIF_INPUT_MODE_SFT                    4
+#define ADDA6_MTKAIFV4_TXIF_INPUT_MODE_MASK                   0x1f
+#define ADDA6_MTKAIFV4_TXIF_INPUT_MODE_MASK_SFT               (0x1f << 4)
+#define ADDA6_MTKAIFV4_TXIF_FOUR_CHANNEL_SFT                  1
+#define ADDA6_MTKAIFV4_TXIF_FOUR_CHANNEL_MASK                 0x1
+#define ADDA6_MTKAIFV4_TXIF_FOUR_CHANNEL_MASK_SFT             (0x1 << 1)
+#define ADDA6_MTKAIFV4_TXIF_AFE_ON_SFT                        0
+#define ADDA6_MTKAIFV4_TXIF_AFE_ON_MASK                       0x1
+#define ADDA6_MTKAIFV4_TXIF_AFE_ON_MASK_SFT                   (0x1 << 0)
+
+/* AFE_ADDA_MTKAIFV4_RX_CFG0 */
+#define MTKAIFV4_RXIF_CLKINV_SFT                              31
+#define MTKAIFV4_RXIF_CLKINV_MASK                             0x1
+#define MTKAIFV4_RXIF_CLKINV_MASK_SFT                         (0x1 << 31)
+#define MTKAIFV4_RXIF_LOOPBACK_MODE_SFT                       28
+#define MTKAIFV4_RXIF_LOOPBACK_MODE_MASK                      0x1
+#define MTKAIFV4_RXIF_LOOPBACK_MODE_MASK_SFT                  (0x1 << 28)
+#define MTKAIFV4_UL_CH7CH8_IN_EN_SEL_SFT                      19
+#define MTKAIFV4_UL_CH7CH8_IN_EN_SEL_MASK                     0x1
+#define MTKAIFV4_UL_CH7CH8_IN_EN_SEL_MASK_SFT                 (0x1 << 19)
+#define MTKAIFV4_UL_CH5CH6_IN_EN_SEL_SFT                      18
+#define MTKAIFV4_UL_CH5CH6_IN_EN_SEL_MASK                     0x1
+#define MTKAIFV4_UL_CH5CH6_IN_EN_SEL_MASK_SFT                 (0x1 << 18)
+#define MTKAIFV4_UL_CH3CH4_IN_EN_SEL_SFT                      17
+#define MTKAIFV4_UL_CH3CH4_IN_EN_SEL_MASK                     0x1
+#define MTKAIFV4_UL_CH3CH4_IN_EN_SEL_MASK_SFT                 (0x1 << 17)
+#define MTKAIFV4_UL_CH1CH2_IN_EN_SEL_SFT                      16
+#define MTKAIFV4_UL_CH1CH2_IN_EN_SEL_MASK                     0x1
+#define MTKAIFV4_UL_CH1CH2_IN_EN_SEL_MASK_SFT                 (0x1 << 16)
+#define MTKAIFV4_RXIF_EN_SEL_SFT                              12
+#define MTKAIFV4_RXIF_EN_SEL_MASK                             0x1
+#define MTKAIFV4_RXIF_EN_SEL_MASK_SFT                         (0x1 << 12)
+#define MTKAIFV4_RXIF_INPUT_MODE_SFT                          4
+#define MTKAIFV4_RXIF_INPUT_MODE_MASK                         0x1f
+#define MTKAIFV4_RXIF_INPUT_MODE_MASK_SFT                     (0x1f << 4)
+#define MTKAIFV4_RXIF_FOUR_CHANNEL_SFT                        1
+#define MTKAIFV4_RXIF_FOUR_CHANNEL_MASK                       0x1
+#define MTKAIFV4_RXIF_FOUR_CHANNEL_MASK_SFT                   (0x1 << 1)
+#define MTKAIFV4_RXIF_AFE_ON_SFT                              0
+#define MTKAIFV4_RXIF_AFE_ON_MASK                             0x1
+#define MTKAIFV4_RXIF_AFE_ON_MASK_SFT                         (0x1 << 0)
+
+/* AFE_ADDA_MTKAIFV4_RX_CFG1 */
+#define MTKAIFV4_RXIF_SYNC_CNT_TABLE_SFT                      17
+#define MTKAIFV4_RXIF_SYNC_CNT_TABLE_MASK                     0xfff
+#define MTKAIFV4_RXIF_SYNC_CNT_TABLE_MASK_SFT                 (0xfff << 17)
+#define MTKAIFV4_RXIF_SYNC_SEARCH_TABLE_SFT                   12
+#define MTKAIFV4_RXIF_SYNC_SEARCH_TABLE_MASK                  0x1f
+#define MTKAIFV4_RXIF_SYNC_SEARCH_TABLE_MASK_SFT              (0x1f << 12)
+#define MTKAIFV4_RXIF_INVAILD_SYNC_CHECK_ROUND_SFT            8
+#define MTKAIFV4_RXIF_INVAILD_SYNC_CHECK_ROUND_MASK           0xf
+#define MTKAIFV4_RXIF_INVAILD_SYNC_CHECK_ROUND_MASK_SFT       (0xf << 8)
+#define MTKAIFV4_RXIF_SYNC_CHECK_ROUND_SFT                    4
+#define MTKAIFV4_RXIF_SYNC_CHECK_ROUND_MASK                   0xf
+#define MTKAIFV4_RXIF_SYNC_CHECK_ROUND_MASK_SFT               (0xf << 4)
+#define MTKAIFV4_RXIF_FIFO_RSP_SFT                            1
+#define MTKAIFV4_RXIF_FIFO_RSP_MASK                           0x7
+#define MTKAIFV4_RXIF_FIFO_RSP_MASK_SFT                       (0x7 << 1)
+#define MTKAIFV4_RXIF_SELF_DEFINE_TABLE_SFT                   0
+#define MTKAIFV4_RXIF_SELF_DEFINE_TABLE_MASK                  0x1
+#define MTKAIFV4_RXIF_SELF_DEFINE_TABLE_MASK_SFT              (0x1 << 0)
+
+/* AFE_ADDA6_MTKAIFV4_RX_CFG0 */
+#define ADDA6_MTKAIFV4_RXIF_CLKINV_SFT                        31
+#define ADDA6_MTKAIFV4_RXIF_CLKINV_MASK                       0x1
+#define ADDA6_MTKAIFV4_RXIF_CLKINV_MASK_SFT                   (0x1 << 31)
+#define ADDA6_MTKAIFV4_RXIF_LOOPBACK_MODE_SFT                 28
+#define ADDA6_MTKAIFV4_RXIF_LOOPBACK_MODE_MASK                0x1
+#define ADDA6_MTKAIFV4_RXIF_LOOPBACK_MODE_MASK_SFT            (0x1 << 28)
+#define ADDA6_MTKAIFV4_RXIF_EN_SEL_SFT                        12
+#define ADDA6_MTKAIFV4_RXIF_EN_SEL_MASK                       0x1
+#define ADDA6_MTKAIFV4_RXIF_EN_SEL_MASK_SFT                   (0x1 << 12)
+#define ADDA6_MTKAIFV4_RXIF_INPUT_MODE_SFT                    4
+#define ADDA6_MTKAIFV4_RXIF_INPUT_MODE_MASK                   0x1f
+#define ADDA6_MTKAIFV4_RXIF_INPUT_MODE_MASK_SFT               (0x1f << 4)
+#define ADDA6_MTKAIFV4_RXIF_FOUR_CHANNEL_SFT                  1
+#define ADDA6_MTKAIFV4_RXIF_FOUR_CHANNEL_MASK                 0x1
+#define ADDA6_MTKAIFV4_RXIF_FOUR_CHANNEL_MASK_SFT             (0x1 << 1)
+#define ADDA6_MTKAIFV4_RXIF_AFE_ON_SFT                        0
+#define ADDA6_MTKAIFV4_RXIF_AFE_ON_MASK                       0x1
+#define ADDA6_MTKAIFV4_RXIF_AFE_ON_MASK_SFT                   (0x1 << 0)
+
+/* AFE_ADDA6_MTKAIFV4_RX_CFG1 */
+#define ADDA6_MTKAIFV4_RXIF_SYNC_CNT_TABLE_SFT                17
+#define ADDA6_MTKAIFV4_RXIF_SYNC_CNT_TABLE_MASK               0xfff
+#define ADDA6_MTKAIFV4_RXIF_SYNC_CNT_TABLE_MASK_SFT           (0xfff << 17)
+#define ADDA6_MTKAIFV4_RXIF_SYNC_SEARCH_TABLE_SFT             12
+#define ADDA6_MTKAIFV4_RXIF_SYNC_SEARCH_TABLE_MASK            0x1f
+#define ADDA6_MTKAIFV4_RXIF_SYNC_SEARCH_TABLE_MASK_SFT        (0x1f << 12)
+#define ADDA6_MTKAIFV4_RXIF_INVAILD_SYNC_CHECK_ROUND_SFT      8
+#define ADDA6_MTKAIFV4_RXIF_INVAILD_SYNC_CHECK_ROUND_MASK     0xf
+#define ADDA6_MTKAIFV4_RXIF_INVAILD_SYNC_CHECK_ROUND_MASK_SFT (0xf << 8)
+#define ADDA6_MTKAIFV4_RXIF_SYNC_CHECK_ROUND_SFT              4
+#define ADDA6_MTKAIFV4_RXIF_SYNC_CHECK_ROUND_MASK             0xf
+#define ADDA6_MTKAIFV4_RXIF_SYNC_CHECK_ROUND_MASK_SFT         (0xf << 4)
+#define ADDA6_MTKAIFV4_RXIF_FIFO_RSP_SFT                      1
+#define ADDA6_MTKAIFV4_RXIF_FIFO_RSP_MASK                     0x7
+#define ADDA6_MTKAIFV4_RXIF_FIFO_RSP_MASK_SFT                 (0x7 << 1)
+#define ADDA6_MTKAIFV4_RXIF_SELF_DEFINE_TABLE_SFT             0
+#define ADDA6_MTKAIFV4_RXIF_SELF_DEFINE_TABLE_MASK            0x1
+#define ADDA6_MTKAIFV4_RXIF_SELF_DEFINE_TABLE_MASK_SFT        (0x1 << 0)
+
+/* AFE_ADDA_MTKAIFV4_TX_SYNCWORD_CFG */
+#define ADDA6_MTKAIFV4_TXIF_SYNCWORD_SFT                      16
+#define ADDA6_MTKAIFV4_TXIF_SYNCWORD_MASK                     0xffff
+#define ADDA6_MTKAIFV4_TXIF_SYNCWORD_MASK_SFT                 (0xffff << 16)
+#define ADDA_MTKAIFV4_TXIF_SYNCWORD_SFT                       0
+#define ADDA_MTKAIFV4_TXIF_SYNCWORD_MASK                      0xffff
+#define ADDA_MTKAIFV4_TXIF_SYNCWORD_MASK_SFT                  (0xffff << 0)
+
+/* AFE_ADDA_MTKAIFV4_RX_SYNCWORD_CFG */
+#define ADDA6_MTKAIFV4_RXIF_SYNCWORD_SFT                      16
+#define ADDA6_MTKAIFV4_RXIF_SYNCWORD_MASK                     0xffff
+#define ADDA6_MTKAIFV4_RXIF_SYNCWORD_MASK_SFT                 (0xffff << 16)
+#define ADDA_MTKAIFV4_RXIF_SYNCWORD_SFT                       0
+#define ADDA_MTKAIFV4_RXIF_SYNCWORD_MASK                      0xffff
+#define ADDA_MTKAIFV4_RXIF_SYNCWORD_MASK_SFT                  (0xffff << 0)
+
+/* AFE_ADDA_MTKAIFV4_MON0 */
+#define MTKAIFV4_TXIF_SDATA_OUT_SFT                           23
+#define MTKAIFV4_TXIF_SDATA_OUT_MASK                          0x1
+#define MTKAIFV4_TXIF_SDATA_OUT_MASK_SFT                      (0x1 << 23)
+#define MTKAIFV4_RXIF_SDATA_IN_SFT                            22
+#define MTKAIFV4_RXIF_SDATA_IN_MASK                           0x1
+#define MTKAIFV4_RXIF_SDATA_IN_MASK_SFT                       (0x1 << 22)
+#define MTKAIFV4_RXIF_SEARCH_FAIL_FLAG_SFT                    21
+#define MTKAIFV4_RXIF_SEARCH_FAIL_FLAG_MASK                   0x1
+#define MTKAIFV4_RXIF_SEARCH_FAIL_FLAG_MASK_SFT               (0x1 << 21)
+#define MTKAIFV4_RXIF_ADC_FIFO_STATUS_SFT                     0
+#define MTKAIFV4_RXIF_ADC_FIFO_STATUS_MASK                    0xfff
+#define MTKAIFV4_RXIF_ADC_FIFO_STATUS_MASK_SFT                (0xfff << 0)
+
+/* AFE_ADDA_MTKAIFV4_MON1 */
+#define MTKAIFV4_RXIF_OUT_CH4_SFT                             24
+#define MTKAIFV4_RXIF_OUT_CH4_MASK                            0xff
+#define MTKAIFV4_RXIF_OUT_CH4_MASK_SFT                        (0xff << 24)
+#define MTKAIFV4_RXIF_OUT_CH3_SFT                             16
+#define MTKAIFV4_RXIF_OUT_CH3_MASK                            0xff
+#define MTKAIFV4_RXIF_OUT_CH3_MASK_SFT                        (0xff << 16)
+#define MTKAIFV4_RXIF_OUT_CH2_SFT                             8
+#define MTKAIFV4_RXIF_OUT_CH2_MASK                            0xff
+#define MTKAIFV4_RXIF_OUT_CH2_MASK_SFT                        (0xff << 8)
+#define MTKAIFV4_RXIF_OUT_CH1_SFT                             0
+#define MTKAIFV4_RXIF_OUT_CH1_MASK                            0xff
+#define MTKAIFV4_RXIF_OUT_CH1_MASK_SFT                        (0xff << 0)
+
+/* AFE_ADDA6_MTKAIFV4_MON0 */
+#define ADDA6_MTKAIFV4_TXIF_SDATA_OUT_SFT                     23
+#define ADDA6_MTKAIFV4_TXIF_SDATA_OUT_MASK                    0x1
+#define ADDA6_MTKAIFV4_TXIF_SDATA_OUT_MASK_SFT                (0x1 << 23)
+#define ADDA6_MTKAIFV4_RXIF_SDATA_IN_SFT                      22
+#define ADDA6_MTKAIFV4_RXIF_SDATA_IN_MASK                     0x1
+#define ADDA6_MTKAIFV4_RXIF_SDATA_IN_MASK_SFT                 (0x1 << 22)
+#define ADDA6_MTKAIFV4_RXIF_SEARCH_FAIL_FLAG_SFT              21
+#define ADDA6_MTKAIFV4_RXIF_SEARCH_FAIL_FLAG_MASK             0x1
+#define ADDA6_MTKAIFV4_RXIF_SEARCH_FAIL_FLAG_MASK_SFT         (0x1 << 21)
+#define ADDA6_MTKAIFV3P3_RXIF_ADC_FIFO_STATUS_SFT             0
+#define ADDA6_MTKAIFV3P3_RXIF_ADC_FIFO_STATUS_MASK            0xfff
+#define ADDA6_MTKAIFV3P3_RXIF_ADC_FIFO_STATUS_MASK_SFT        (0xfff << 0)
+
+/* ETDM_IN0_CON0 */
+#define REG_ETDM_IN_EN_SFT                                    0
+#define REG_ETDM_IN_EN_MASK                                   0x1
+#define REG_ETDM_IN_EN_MASK_SFT                               (0x1 << 0)
+#define REG_SYNC_MODE_SFT                                     1
+#define REG_SYNC_MODE_MASK                                    0x1
+#define REG_SYNC_MODE_MASK_SFT                                (0x1 << 1)
+#define REG_LSB_FIRST_SFT                                     3
+#define REG_LSB_FIRST_MASK                                    0x1
+#define REG_LSB_FIRST_MASK_SFT                                (0x1 << 3)
+#define REG_SOFT_RST_SFT                                      4
+#define REG_SOFT_RST_MASK                                     0x1
+#define REG_SOFT_RST_MASK_SFT                                 (0x1 << 4)
+#define REG_SLAVE_MODE_SFT                                    5
+#define REG_SLAVE_MODE_MASK                                   0x1
+#define REG_SLAVE_MODE_MASK_SFT                               (0x1 << 5)
+#define REG_FMT_SFT                                           6
+#define REG_FMT_MASK                                          0x7
+#define REG_FMT_MASK_SFT                                      (0x7 << 6)
+#define REG_LRCK_EDGE_SEL_SFT                                 10
+#define REG_LRCK_EDGE_SEL_MASK                                0x1
+#define REG_LRCK_EDGE_SEL_MASK_SFT                            (0x1 << 10)
+#define REG_BIT_LENGTH_SFT                                    11
+#define REG_BIT_LENGTH_MASK                                   0x1f
+#define REG_BIT_LENGTH_MASK_SFT                               (0x1f << 11)
+#define REG_WORD_LENGTH_SFT                                   16
+#define REG_WORD_LENGTH_MASK                                  0x1f
+#define REG_WORD_LENGTH_MASK_SFT                              (0x1f << 16)
+#define REG_CH_NUM_SFT                                        23
+#define REG_CH_NUM_MASK                                       0x1f
+#define REG_CH_NUM_MASK_SFT                                   (0x1f << 23)
+#define REG_RELATCH_1X_EN_DOMAIN_SEL_SFT                      28
+#define REG_RELATCH_1X_EN_DOMAIN_SEL_MASK                     0x7
+#define REG_RELATCH_1X_EN_DOMAIN_SEL_MASK_SFT                 (0x7 << 28)
+#define REG_VALID_TOGETHER_SFT                                31
+#define REG_VALID_TOGETHER_MASK                               0x1
+#define REG_VALID_TOGETHER_MASK_SFT                           (0x1 << 31)
+
+/* ETDM_IN0_CON1 */
+/* ETDM_IN1_CON1 */
+/* ETDM_IN2_CON1 */
+/* ETDM_IN3_CON1 */
+/* ETDM_IN4_CON1 */
+/* ETDM_IN5_CON1 */
+/* ETDM_IN6_CON1 */
+#define REG_INITIAL_COUNT_SFT                                 0
+#define REG_INITIAL_COUNT_MASK                                0x1f
+#define REG_INITIAL_COUNT_MASK_SFT                            (0x1f << 0)
+#define REG_INITIAL_POINT_SFT                                 5
+#define REG_INITIAL_POINT_MASK                                0x1f
+#define REG_INITIAL_POINT_MASK_SFT                            (0x1f << 5)
+#define REG_LRCK_AUTO_OFF_SFT                                 10
+#define REG_LRCK_AUTO_OFF_MASK                                0x1
+#define REG_LRCK_AUTO_OFF_MASK_SFT                            (0x1 << 10)
+#define REG_BCK_AUTO_OFF_SFT                                  11
+#define REG_BCK_AUTO_OFF_MASK                                 0x1
+#define REG_BCK_AUTO_OFF_MASK_SFT                             (0x1 << 11)
+#define REG_INITIAL_LRCK_SFT                                  13
+#define REG_INITIAL_LRCK_MASK                                 0x1
+#define REG_INITIAL_LRCK_MASK_SFT                             (0x1 << 13)
+#define REG_NO_ALIGN_1X_EN_SFT                                14
+#define REG_NO_ALIGN_1X_EN_MASK                               0x1
+#define REG_NO_ALIGN_1X_EN_MASK_SFT                           (0x1 << 14)
+#define REG_LRCK_RESET_SFT                                    15
+#define REG_LRCK_RESET_MASK                                   0x1
+#define REG_LRCK_RESET_MASK_SFT                               (0x1 << 15)
+#define PINMUX_MCLK_CTRL_OE_SFT                               16
+#define PINMUX_MCLK_CTRL_OE_MASK                              0x1
+#define PINMUX_MCLK_CTRL_OE_MASK_SFT                          (0x1 << 16)
+#define REG_OUTPUT_CR_EN_SFT                                  18
+#define REG_OUTPUT_CR_EN_MASK                                 0x1
+#define REG_OUTPUT_CR_EN_MASK_SFT                             (0x1 << 18)
+#define REG_LR_ALIGN_SFT                                      19
+#define REG_LR_ALIGN_MASK                                     0x1
+#define REG_LR_ALIGN_MASK_SFT                                 (0x1 << 19)
+#define REG_LRCK_WIDTH_SFT                                    20
+#define REG_LRCK_WIDTH_MASK                                   0x3ff
+#define REG_LRCK_WIDTH_MASK_SFT                               (0x3ff << 20)
+#define REG_DIRECT_INPUT_MASTER_BCK_SFT                       30
+#define REG_DIRECT_INPUT_MASTER_BCK_MASK                      0x1
+#define REG_DIRECT_INPUT_MASTER_BCK_MASK_SFT                  (0x1 << 30)
+#define REG_LRCK_AUTO_MODE_SFT                                31
+#define REG_LRCK_AUTO_MODE_MASK                               0x1
+#define REG_LRCK_AUTO_MODE_MASK_SFT                           (0x1 << 31)
+
+/* ETDM_IN0_CON2 */
+/* ETDM_IN1_CON2 */
+/* ETDM_IN2_CON2 */
+/* ETDM_IN3_CON2 */
+/* ETDM_IN4_CON2 */
+/* ETDM_IN5_CON2 */
+/* ETDM_IN6_CON2 */
+#define REG_UPDATE_POINT_SFT                                  0
+#define REG_UPDATE_POINT_MASK                                 0x1f
+#define REG_UPDATE_POINT_MASK_SFT                             (0x1f << 0)
+#define REG_UPDATE_GAP_SFT                                    5
+#define REG_UPDATE_GAP_MASK                                   0x1f
+#define REG_UPDATE_GAP_MASK_SFT                               (0x1f << 5)
+#define REG_CLOCK_SOURCE_SEL_SFT                              10
+#define REG_CLOCK_SOURCE_SEL_MASK                             0x7
+#define REG_CLOCK_SOURCE_SEL_MASK_SFT                         (0x7 << 10)
+#define REG_CK_EN_SEL_AUTO_SFT                                14
+#define REG_CK_EN_SEL_AUTO_MASK                               0x1
+#define REG_CK_EN_SEL_AUTO_MASK_SFT                           (0x1 << 14)
+#define REG_MULTI_IP_TOTAL_CHNUM_SFT                          15
+#define REG_MULTI_IP_TOTAL_CHNUM_MASK                         0x1f
+#define REG_MULTI_IP_TOTAL_CHNUM_MASK_SFT                     (0x1f << 15)
+#define REG_MASK_AUTO_SFT                                     20
+#define REG_MASK_AUTO_MASK                                    0x1
+#define REG_MASK_AUTO_MASK_SFT                                (0x1 << 20)
+#define REG_MASK_NUM_SFT                                      21
+#define REG_MASK_NUM_MASK                                     0x1f
+#define REG_MASK_NUM_MASK_SFT                                 (0x1f << 21)
+#define REG_UPDATE_POINT_AUTO_SFT                             26
+#define REG_UPDATE_POINT_AUTO_MASK                            0x1
+#define REG_UPDATE_POINT_AUTO_MASK_SFT                        (0x1 << 26)
+#define REG_SDATA_DELAY_0P5T_EN_SFT                           27
+#define REG_SDATA_DELAY_0P5T_EN_MASK                          0x1
+#define REG_SDATA_DELAY_0P5T_EN_MASK_SFT                      (0x1 << 27)
+#define REG_SDATA_DELAY_BCK_INV_SFT                           28
+#define REG_SDATA_DELAY_BCK_INV_MASK                          0x1
+#define REG_SDATA_DELAY_BCK_INV_MASK_SFT                      (0x1 << 28)
+#define REG_LRCK_DELAY_0P5T_EN_SFT                            29
+#define REG_LRCK_DELAY_0P5T_EN_MASK                           0x1
+#define REG_LRCK_DELAY_0P5T_EN_MASK_SFT                       (0x1 << 29)
+#define REG_LRCK_DELAY_BCK_INV_SFT                            30
+#define REG_LRCK_DELAY_BCK_INV_MASK                           0x1
+#define REG_LRCK_DELAY_BCK_INV_MASK_SFT                       (0x1 << 30)
+#define REG_MULTI_IP_MODE_SFT                                 31
+#define REG_MULTI_IP_MODE_MASK                                0x1
+#define REG_MULTI_IP_MODE_MASK_SFT                            (0x1 << 31)
+
+/* ETDM_IN0_CON3 */
+/* ETDM_IN1_CON3 */
+/* ETDM_IN2_CON3 */
+/* ETDM_IN3_CON3 */
+/* ETDM_IN4_CON3 */
+/* ETDM_IN5_CON3 */
+/* ETDM_IN6_CON3 */
+#define REG_DISABLE_OUT_SFT                                   0
+#define REG_DISABLE_OUT_MASK                                  0xffff
+#define REG_DISABLE_OUT_MASK_SFT                              (0xffff << 0)
+#define REG_RJ_DATA_RIGHT_ALIGN_SFT                           16
+#define REG_RJ_DATA_RIGHT_ALIGN_MASK                          0x1
+#define REG_RJ_DATA_RIGHT_ALIGN_MASK_SFT                      (0x1 << 16)
+#define REG_MONITOR_SEL_SFT                                   17
+#define REG_MONITOR_SEL_MASK                                  0x3
+#define REG_MONITOR_SEL_MASK_SFT                              (0x3 << 17)
+#define REG_CNT_UPPER_LIMIT_SFT                               19
+#define REG_CNT_UPPER_LIMIT_MASK                              0x3f
+#define REG_CNT_UPPER_LIMIT_MASK_SFT                          (0x3f << 19)
+#define REG_COMPACT_SAMPLE_END_DIS_SFT                        25
+#define REG_COMPACT_SAMPLE_END_DIS_MASK                       0x1
+#define REG_COMPACT_SAMPLE_END_DIS_MASK_SFT                   (0x1 << 25)
+#define REG_FS_TIMING_SEL_SFT                                 26
+#define REG_FS_TIMING_SEL_MASK                                0x1f
+#define REG_FS_TIMING_SEL_MASK_SFT                            (0x1f << 26)
+#define REG_SAMPLE_END_MODE_SFT                               31
+#define REG_SAMPLE_END_MODE_MASK                              0x1
+#define REG_SAMPLE_END_MODE_MASK_SFT                          (0x1 << 31)
+
+/* ETDM_IN0_CON4 */
+/* ETDM_IN1_CON4 */
+/* ETDM_IN2_CON4 */
+/* ETDM_IN3_CON4 */
+/* ETDM_IN4_CON4 */
+/* ETDM_IN5_CON4 */
+/* ETDM_IN6_CON4 */
+#define REG_ALWAYS_OPEN_1X_EN_SFT                             31
+#define REG_ALWAYS_OPEN_1X_EN_MASK                            0x1
+#define REG_ALWAYS_OPEN_1X_EN_MASK_SFT                        (0x1 << 31)
+#define REG_WAIT_LAST_SAMPLE_SFT                              30
+#define REG_WAIT_LAST_SAMPLE_MASK                             0x1
+#define REG_WAIT_LAST_SAMPLE_MASK_SFT                         (0x1 << 30)
+#define REG_SAMPLE_END_POINT_SFT                              25
+#define REG_SAMPLE_END_POINT_MASK                             0x1f
+#define REG_SAMPLE_END_POINT_MASK_SFT                         (0x1f << 25)
+#define REG_RELATCH_1X_EN_SEL_SFT                             20
+#define REG_RELATCH_1X_EN_SEL_MASK                            0x1f
+#define REG_RELATCH_1X_EN_SEL_MASK_SFT                        (0x1f << 20)
+#define REG_MASTER_WS_INV_SFT                                 19
+#define REG_MASTER_WS_INV_MASK                                0x1
+#define REG_MASTER_WS_INV_MASK_SFT                            (0x1 << 19)
+#define REG_MASTER_BCK_INV_SFT                                18
+#define REG_MASTER_BCK_INV_MASK                               0x1
+#define REG_MASTER_BCK_INV_MASK_SFT                           (0x1 << 18)
+#define REG_SLAVE_LRCK_INV_SFT                                17
+#define REG_SLAVE_LRCK_INV_MASK                               0x1
+#define REG_SLAVE_LRCK_INV_MASK_SFT                           (0x1 << 17)
+#define REG_SLAVE_BCK_INV_SFT                                 16
+#define REG_SLAVE_BCK_INV_MASK                                0x1
+#define REG_SLAVE_BCK_INV_MASK_SFT                            (0x1 << 16)
+#define REG_REPACK_CHNUM_SFT                                  12
+#define REG_REPACK_CHNUM_MASK                                 0xf
+#define REG_REPACK_CHNUM_MASK_SFT                             (0xf << 12)
+#define REG_ASYNC_RESET_SFT                                   11
+#define REG_ASYNC_RESET_MASK                                  0x1
+#define REG_ASYNC_RESET_MASK_SFT                              (0x1 << 11)
+#define REG_REPACK_WORD_LENGTH_SFT                            9
+#define REG_REPACK_WORD_LENGTH_MASK                           0x3
+#define REG_REPACK_WORD_LENGTH_MASK_SFT                       (0x3 << 9)
+#define REG_REPACK_AUTO_MODE_SFT                              8
+#define REG_REPACK_AUTO_MODE_MASK                             0x1
+#define REG_REPACK_AUTO_MODE_MASK_SFT                         (0x1 << 8)
+#define REG_REPACK_MODE_SFT                                   0
+#define REG_REPACK_MODE_MASK                                  0x3f
+#define REG_REPACK_MODE_MASK_SFT                              (0x3f << 0)
+
+/* ETDM_IN0_CON5 */
+/* ETDM_IN1_CON5 */
+/* ETDM_IN2_CON5 */
+/* ETDM_IN3_CON5 */
+/* ETDM_IN4_CON5 */
+/* ETDM_IN5_CON5 */
+/* ETDM_IN6_CON5 */
+#define REG_LR_SWAP_SFT                                       16
+#define REG_LR_SWAP_MASK                                      0xffff
+#define REG_LR_SWAP_MASK_SFT                                  (0xffff << 16)
+#define REG_ODD_FLAG_EN_SFT                                   0
+#define REG_ODD_FLAG_EN_MASK                                  0xffff
+#define REG_ODD_FLAG_EN_MASK_SFT                              (0xffff << 0)
+
+/* ETDM_IN0_CON6 */
+/* ETDM_IN1_CON6 */
+/* ETDM_IN2_CON6 */
+/* ETDM_IN3_CON6 */
+/* ETDM_IN4_CON6 */
+/* ETDM_IN5_CON6 */
+/* ETDM_IN6_CON6 */
+#define LCH_DATA_REG_SFT                                      0
+#define LCH_DATA_REG_MASK                                     0xffffffff
+#define LCH_DATA_REG_MASK_SFT                                 (0xffffffff << 0)
+
+/* ETDM_IN0_CON7 */
+/* ETDM_IN1_CON7 */
+/* ETDM_IN2_CON7 */
+/* ETDM_IN3_CON7 */
+/* ETDM_IN4_CON7 */
+/* ETDM_IN5_CON7 */
+/* ETDM_IN6_CON7 */
+#define RCH_DATA_REG_SFT                                      0
+#define RCH_DATA_REG_MASK                                     0xffffffff
+#define RCH_DATA_REG_MASK_SFT                                 (0xffffffff << 0)
+
+/* ETDM_IN0_CON8 */
+/* ETDM_IN1_CON8 */
+/* ETDM_IN2_CON8 */
+/* ETDM_IN3_CON8 */
+/* ETDM_IN4_CON8 */
+/* ETDM_IN5_CON8 */
+/* ETDM_IN6_CON8 */
+#define REG_AFIFO_THRESHOLD_SFT                               29
+#define REG_AFIFO_THRESHOLD_MASK                              0x3
+#define REG_AFIFO_THRESHOLD_MASK_SFT                          (0x3 << 29)
+#define REG_CK_EN_SEL_MANUAL_SFT                              16
+#define REG_CK_EN_SEL_MANUAL_MASK                             0x3ff
+#define REG_CK_EN_SEL_MANUAL_MASK_SFT                         (0x3ff << 16)
+#define REG_AFIFO_SW_RESET_SFT                                15
+#define REG_AFIFO_SW_RESET_MASK                               0x1
+#define REG_AFIFO_SW_RESET_MASK_SFT                           (0x1 << 15)
+#define REG_AFIFO_RESET_SEL_SFT                               14
+#define REG_AFIFO_RESET_SEL_MASK                              0x1
+#define REG_AFIFO_RESET_SEL_MASK_SFT                          (0x1 << 14)
+#define REG_AFIFO_AUTO_RESET_DIS_SFT                          9
+#define REG_AFIFO_AUTO_RESET_DIS_MASK                         0x1
+#define REG_AFIFO_AUTO_RESET_DIS_MASK_SFT                     (0x1 << 9)
+#define REG_ETDM_USE_AFIFO_SFT                                8
+#define REG_ETDM_USE_AFIFO_MASK                               0x1
+#define REG_ETDM_USE_AFIFO_MASK_SFT                           (0x1 << 8)
+#define REG_AFIFO_CLOCK_DOMAIN_SEL_SFT                        5
+#define REG_AFIFO_CLOCK_DOMAIN_SEL_MASK                       0x7
+#define REG_AFIFO_CLOCK_DOMAIN_SEL_MASK_SFT                   (0x7 << 5)
+#define REG_AFIFO_MODE_SFT                                    0
+#define REG_AFIFO_MODE_MASK                                   0x1f
+#define REG_AFIFO_MODE_MASK_SFT                               (0x1f << 0)
+
+/* ETDM_IN0_CON9 */
+/* ETDM_IN1_CON9 */
+/* ETDM_IN2_CON9 */
+/* ETDM_IN3_CON9 */
+/* ETDM_IN4_CON9 */
+/* ETDM_IN5_CON9 */
+/* ETDM_IN6_CON9 */
+#define REG_OUT2LATCH_TIME_SFT                                10
+#define REG_OUT2LATCH_TIME_MASK                               0x1f
+#define REG_OUT2LATCH_TIME_MASK_SFT                           (0x1f << 10)
+#define REG_ALMOST_END_BIT_COUNT_SFT                          5
+#define REG_ALMOST_END_BIT_COUNT_MASK                         0x1f
+#define REG_ALMOST_END_BIT_COUNT_MASK_SFT                     (0x1f << 5)
+#define REG_ALMOST_END_CH_COUNT_SFT                           0
+#define REG_ALMOST_END_CH_COUNT_MASK                          0x1f
+#define REG_ALMOST_END_CH_COUNT_MASK_SFT                      (0x1f << 0)
+
+/* ETDM_IN0_MON */
+/* ETDM_IN1_MON */
+/* ETDM_IN2_MON */
+/* ETDM_IN3_MON */
+/* ETDM_IN4_MON */
+/* ETDM_IN5_MON */
+/* ETDM_IN6_MON */
+#define LRCK_INV_SFT                                          30
+#define LRCK_INV_MASK                                         0x1
+#define LRCK_INV_MASK_SFT                                     (0x1 << 30)
+#define EN_SYNC_OUT_SFT                                       29
+#define EN_SYNC_OUT_MASK                                      0x1
+#define EN_SYNC_OUT_MASK_SFT                                  (0x1 << 29)
+#define HOPPING_EN_SYNC_OUT_PRE_SFT                           28
+#define HOPPING_EN_SYNC_OUT_PRE_MASK                          0x1
+#define HOPPING_EN_SYNC_OUT_PRE_MASK_SFT                      (0x1 << 28)
+#define WFULL_SFT                                             27
+#define WFULL_MASK                                            0x1
+#define WFULL_MASK_SFT                                        (0x1 << 27)
+#define REMPTY_SFT                                            26
+#define REMPTY_MASK                                           0x1
+#define REMPTY_MASK_SFT                                       (0x1 << 26)
+#define ETDM_2X_CK_EN_SFT                                     25
+#define ETDM_2X_CK_EN_MASK                                    0x1
+#define ETDM_2X_CK_EN_MASK_SFT                                (0x1 << 25)
+#define ETDM_1X_CK_EN_SFT                                     24
+#define ETDM_1X_CK_EN_MASK                                    0x1
+#define ETDM_1X_CK_EN_MASK_SFT                                (0x1 << 24)
+#define SDATA0_SFT                                            23
+#define SDATA0_MASK                                           0x1
+#define SDATA0_MASK_SFT                                       (0x1 << 23)
+#define CURRENT_STATUS_SFT                                    21
+#define CURRENT_STATUS_MASK                                   0x3
+#define CURRENT_STATUS_MASK_SFT                               (0x3 << 21)
+#define BIT_POINT_SFT                                         16
+#define BIT_POINT_MASK                                        0x1f
+#define BIT_POINT_MASK_SFT                                    (0x1f << 16)
+#define BIT_CH_COUNT_SFT                                      10
+#define BIT_CH_COUNT_MASK                                     0x3f
+#define BIT_CH_COUNT_MASK_SFT                                 (0x3f << 10)
+#define BIT_COUNT_SFT                                         5
+#define BIT_COUNT_MASK                                        0x1f
+#define BIT_COUNT_MASK_SFT                                    (0x1f << 5)
+#define CH_COUNT_SFT                                          0
+#define CH_COUNT_MASK                                         0x1f
+#define CH_COUNT_MASK_SFT                                     (0x1f << 0)
+
+/* ETDM_OUT0_CON0 */
+/* ETDM_OUT1_CON0 */
+/* ETDM_OUT2_CON0 */
+/* ETDM_OUT3_CON0 */
+/* ETDM_OUT4_CON0 */
+/* ETDM_OUT5_CON0 */
+/* ETDM_OUT6_CON0 */
+#define OUT_REG_ETDM_OUT_EN_SFT                                   0
+#define OUT_REG_ETDM_OUT_EN_MASK                                  0x1
+#define OUT_REG_ETDM_OUT_EN_MASK_SFT                              (0x1 << 0)
+#define OUT_REG_SYNC_MODE_SFT                                     1
+#define OUT_REG_SYNC_MODE_MASK                                    0x1
+#define OUT_REG_SYNC_MODE_MASK_SFT                                (0x1 << 1)
+#define OUT_REG_LSB_FIRST_SFT                                     3
+#define OUT_REG_LSB_FIRST_MASK                                    0x1
+#define OUT_REG_LSB_FIRST_MASK_SFT                                (0x1 << 3)
+#define OUT_REG_SOFT_RST_SFT                                      4
+#define OUT_REG_SOFT_RST_MASK                                     0x1
+#define OUT_REG_SOFT_RST_MASK_SFT                                 (0x1 << 4)
+#define OUT_REG_SLAVE_MODE_SFT                                    5
+#define OUT_REG_SLAVE_MODE_MASK                                   0x1
+#define OUT_REG_SLAVE_MODE_MASK_SFT                               (0x1 << 5)
+#define OUT_REG_FMT_SFT                                           6
+#define OUT_REG_FMT_MASK                                          0x7
+#define OUT_REG_FMT_MASK_SFT                                      (0x7 << 6)
+#define OUT_REG_LRCK_EDGE_SEL_SFT                                 10
+#define OUT_REG_LRCK_EDGE_SEL_MASK                                0x1
+#define OUT_REG_LRCK_EDGE_SEL_MASK_SFT                            (0x1 << 10)
+#define OUT_REG_BIT_LENGTH_SFT                                    11
+#define OUT_REG_BIT_LENGTH_MASK                                   0x1f
+#define OUT_REG_BIT_LENGTH_MASK_SFT                               (0x1f << 11)
+#define OUT_REG_WORD_LENGTH_SFT                                   16
+#define OUT_REG_WORD_LENGTH_MASK                                  0x1f
+#define OUT_REG_WORD_LENGTH_MASK_SFT                              (0x1f << 16)
+#define OUT_REG_CH_NUM_SFT                                        23
+#define OUT_REG_CH_NUM_MASK                                       0x1f
+#define OUT_REG_CH_NUM_MASK_SFT                                   (0x1f << 23)
+#define OUT_REG_RELATCH_DOMAIN_SEL_SFT                            28
+#define OUT_REG_RELATCH_DOMAIN_SEL_MASK                           0x7
+#define OUT_REG_RELATCH_DOMAIN_SEL_MASK_SFT                       (0x7 << 28)
+#define OUT_REG_VALID_TOGETHER_SFT                                31
+#define OUT_REG_VALID_TOGETHER_MASK                               0x1
+#define OUT_REG_VALID_TOGETHER_MASK_SFT                           (0x1 << 31)
+
+/* ETDM_OUT0_CON1 */
+/* ETDM_OUT1_CON1 */
+/* ETDM_OUT2_CON1 */
+/* ETDM_OUT3_CON1 */
+/* ETDM_OUT4_CON1 */
+/* ETDM_OUT5_CON1 */
+/* ETDM_OUT6_CON1 */
+#define OUT_REG_INITIAL_COUNT_SFT                                 0
+#define OUT_REG_INITIAL_COUNT_MASK                                0x1f
+#define OUT_REG_INITIAL_COUNT_MASK_SFT                            (0x1f << 0)
+#define OUT_REG_INITIAL_POINT_SFT                                 5
+#define OUT_REG_INITIAL_POINT_MASK                                0x1f
+#define OUT_REG_INITIAL_POINT_MASK_SFT                            (0x1f << 5)
+#define OUT_REG_LRCK_AUTO_OFF_SFT                                 10
+#define OUT_REG_LRCK_AUTO_OFF_MASK                                0x1
+#define OUT_REG_LRCK_AUTO_OFF_MASK_SFT                            (0x1 << 10)
+#define OUT_REG_BCK_AUTO_OFF_SFT                                  11
+#define OUT_REG_BCK_AUTO_OFF_MASK                                 0x1
+#define OUT_REG_BCK_AUTO_OFF_MASK_SFT                             (0x1 << 11)
+#define OUT_REG_INITIAL_LRCK_SFT                                  13
+#define OUT_REG_INITIAL_LRCK_MASK                                 0x1
+#define OUT_REG_INITIAL_LRCK_MASK_SFT                             (0x1 << 13)
+#define OUT_REG_NO_ALIGN_1X_EN_SFT                                14
+#define OUT_REG_NO_ALIGN_1X_EN_MASK                               0x1
+#define OUT_REG_NO_ALIGN_1X_EN_MASK_SFT                           (0x1 << 14)
+#define OUT_REG_LRCK_RESET_SFT                                    15
+#define OUT_REG_LRCK_RESET_MASK                                   0x1
+#define OUT_REG_LRCK_RESET_MASK_SFT                               (0x1 << 15)
+#define OUT_PINMUX_MCLK_CTRL_OE_SFT                               16
+#define OUT_PINMUX_MCLK_CTRL_OE_MASK                              0x1
+#define OUT_PINMUX_MCLK_CTRL_OE_MASK_SFT                          (0x1 << 16)
+#define OUT_REG_OUTPUT_CR_EN_SFT                                  18
+#define OUT_REG_OUTPUT_CR_EN_MASK                                 0x1
+#define OUT_REG_OUTPUT_CR_EN_MASK_SFT                             (0x1 << 18)
+#define OUT_REG_LRCK_WIDTH_SFT                                    19
+#define OUT_REG_LRCK_WIDTH_MASK                                   0x3ff
+#define OUT_REG_LRCK_WIDTH_MASK_SFT                               (0x3ff << 19)
+#define OUT_REG_LRCK_AUTO_MODE_SFT                                29
+#define OUT_REG_LRCK_AUTO_MODE_MASK                               0x1
+#define OUT_REG_LRCK_AUTO_MODE_MASK_SFT                           (0x1 << 29)
+#define OUT_REG_DIRECT_INPUT_MASTER_BCK_SFT                       30
+#define OUT_REG_DIRECT_INPUT_MASTER_BCK_MASK                      0x1
+#define OUT_REG_DIRECT_INPUT_MASTER_BCK_MASK_SFT                  (0x1 << 30)
+#define OUT_REG_16B_COMPACT_MODE_SFT                              31
+#define OUT_REG_16B_COMPACT_MODE_MASK                             0x1
+#define OUT_REG_16B_COMPACT_MODE_MASK_SFT                         (0x1 << 31)
+
+/* ETDM_OUT0_CON2 */
+/* ETDM_OUT1_CON2 */
+/* ETDM_OUT2_CON2 */
+/* ETDM_OUT3_CON2 */
+/* ETDM_OUT4_CON2 */
+/* ETDM_OUT5_CON2 */
+/* ETDM_OUT6_CON2 */
+#define OUT_REG_IN2LATCH_TIME_SFT                                 0
+#define OUT_REG_IN2LATCH_TIME_MASK                                0x1f
+#define OUT_REG_IN2LATCH_TIME_MASK_SFT                            (0x1f << 0)
+#define OUT_REG_MASK_NUM_SFT                                      5
+#define OUT_REG_MASK_NUM_MASK                                     0x1f
+#define OUT_REG_MASK_NUM_MASK_SFT                                 (0x1f << 5)
+#define OUT_REG_MASK_AUTO_SFT                                     10
+#define OUT_REG_MASK_AUTO_MASK                                    0x1
+#define OUT_REG_MASK_AUTO_MASK_SFT                                (0x1 << 10)
+#define OUT_REG_SDATA_SHIFT_SFT                                   11
+#define OUT_REG_SDATA_SHIFT_MASK                                  0x3
+#define OUT_REG_SDATA_SHIFT_MASK_SFT                              (0x3 << 11)
+#define OUT_REG_ALMOST_END_BIT_COUNT_SFT                          13
+#define OUT_REG_ALMOST_END_BIT_COUNT_MASK                         0x1f
+#define OUT_REG_ALMOST_END_BIT_COUNT_MASK_SFT                     (0x1f << 13)
+#define OUT_REG_SDATA_CON_SFT                                     18
+#define OUT_REG_SDATA_CON_MASK                                    0x3
+#define OUT_REG_SDATA_CON_MASK_SFT                                (0x3 << 18)
+#define OUT_REG_REDUNDANT_0_SFT                                   20
+#define OUT_REG_REDUNDANT_0_MASK                                  0x1
+#define OUT_REG_REDUNDANT_0_MASK_SFT                              (0x1 << 20)
+#define OUT_REG_SDATA_AUTO_OFF_SFT                                21
+#define OUT_REG_SDATA_AUTO_OFF_MASK                               0x1
+#define OUT_REG_SDATA_AUTO_OFF_MASK_SFT                           (0x1 << 21)
+#define OUT_REG_BCK_OFF_TIME_SFT                                  22
+#define OUT_REG_BCK_OFF_TIME_MASK                                 0x3
+#define OUT_REG_BCK_OFF_TIME_MASK_SFT                             (0x3 << 22)
+#define OUT_REG_MONITOR_SEL_SFT                                   24
+#define OUT_REG_MONITOR_SEL_MASK                                  0x3
+#define OUT_REG_MONITOR_SEL_MASK_SFT                              (0x3 << 24)
+#define OUT_REG_SHIFT_AUTO_SFT                                    26
+#define OUT_REG_SHIFT_AUTO_MASK                                   0x1
+#define OUT_REG_SHIFT_AUTO_MASK_SFT                               (0x1 << 26)
+#define OUT_REG_SDATA_DELAY_0P5T_EN_SFT                           27
+#define OUT_REG_SDATA_DELAY_0P5T_EN_MASK                          0x1
+#define OUT_REG_SDATA_DELAY_0P5T_EN_MASK_SFT                      (0x1 << 27)
+#define OUT_REG_SDATA_DELAY_BCK_INV_SFT                           28
+#define OUT_REG_SDATA_DELAY_BCK_INV_MASK                          0x1
+#define OUT_REG_SDATA_DELAY_BCK_INV_MASK_SFT                      (0x1 << 28)
+#define OUT_REG_LRCK_DELAY_0P5T_EN_SFT                            29
+#define OUT_REG_LRCK_DELAY_0P5T_EN_MASK                           0x1
+#define OUT_REG_LRCK_DELAY_0P5T_EN_MASK_SFT                       (0x1 << 29)
+#define OUT_REG_LRCK_DELAY_BCK_INV_SFT                            30
+#define OUT_REG_LRCK_DELAY_BCK_INV_MASK                           0x1
+#define OUT_REG_LRCK_DELAY_BCK_INV_MASK_SFT                       (0x1 << 30)
+#define OUT_REG_OFF_CR_EN_SFT                                     31
+#define OUT_REG_OFF_CR_EN_MASK                                    0x1
+#define OUT_REG_OFF_CR_EN_MASK_SFT                                (0x1 << 31)
+
+/* ETDM_OUT0_CON3 */
+/* ETDM_OUT1_CON3 */
+/* ETDM_OUT2_CON3 */
+/* ETDM_OUT3_CON3 */
+/* ETDM_OUT4_CON3 */
+/* ETDM_OUT5_CON3 */
+/* ETDM_OUT6_CON3 */
+#define OUT_REG_START_CH_PAIR0_SFT                                0
+#define OUT_REG_START_CH_PAIR0_MASK                               0xf
+#define OUT_REG_START_CH_PAIR0_MASK_SFT                           (0xf << 0)
+#define OUT_REG_START_CH_PAIR1_SFT                                4
+#define OUT_REG_START_CH_PAIR1_MASK                               0xf
+#define OUT_REG_START_CH_PAIR1_MASK_SFT                           (0xf << 4)
+#define OUT_REG_START_CH_PAIR2_SFT                                8
+#define OUT_REG_START_CH_PAIR2_MASK                               0xf
+#define OUT_REG_START_CH_PAIR2_MASK_SFT                           (0xf << 8)
+#define OUT_REG_START_CH_PAIR3_SFT                                12
+#define OUT_REG_START_CH_PAIR3_MASK                               0xf
+#define OUT_REG_START_CH_PAIR3_MASK_SFT                           (0xf << 12)
+#define OUT_REG_START_CH_PAIR4_SFT                                16
+#define OUT_REG_START_CH_PAIR4_MASK                               0xf
+#define OUT_REG_START_CH_PAIR4_MASK_SFT                           (0xf << 16)
+#define OUT_REG_START_CH_PAIR5_SFT                                20
+#define OUT_REG_START_CH_PAIR5_MASK                               0xf
+#define OUT_REG_START_CH_PAIR5_MASK_SFT                           (0xf << 20)
+#define OUT_REG_START_CH_PAIR6_SFT                                24
+#define OUT_REG_START_CH_PAIR6_MASK                               0xf
+#define OUT_REG_START_CH_PAIR6_MASK_SFT                           (0xf << 24)
+#define OUT_REG_START_CH_PAIR7_SFT                                28
+#define OUT_REG_START_CH_PAIR7_MASK                               0xf
+#define OUT_REG_START_CH_PAIR7_MASK_SFT                           (0xf << 28)
+
+/* ETDM_OUT0_CON4 */
+/* ETDM_OUT1_CON4 */
+/* ETDM_OUT2_CON4 */
+/* ETDM_OUT3_CON4 */
+/* ETDM_OUT4_CON4 */
+/* ETDM_OUT5_CON4 */
+/* ETDM_OUT6_CON4 */
+#define OUT_REG_FS_TIMING_SEL_SFT                                 0
+#define OUT_REG_FS_TIMING_SEL_MASK                                0x1f
+#define OUT_REG_FS_TIMING_SEL_MASK_SFT                            (0x1f << 0)
+#define OUT_REG_CLOCK_SOURCE_SEL_SFT                              6
+#define OUT_REG_CLOCK_SOURCE_SEL_MASK                             0x7
+#define OUT_REG_CLOCK_SOURCE_SEL_MASK_SFT                         (0x7 << 6)
+#define OUT_REG_CK_EN_SEL_AUTO_SFT                                10
+#define OUT_REG_CK_EN_SEL_AUTO_MASK                               0x1
+#define OUT_REG_CK_EN_SEL_AUTO_MASK_SFT                           (0x1 << 10)
+#define OUT_REG_ASYNC_RESET_SFT                                   11
+#define OUT_REG_ASYNC_RESET_MASK                                  0x1
+#define OUT_REG_ASYNC_RESET_MASK_SFT                              (0x1 << 11)
+#define OUT_REG_CK_EN_SEL_MANUAL_SFT                              14
+#define OUT_REG_CK_EN_SEL_MANUAL_MASK                             0x3ff
+#define OUT_REG_CK_EN_SEL_MANUAL_MASK_SFT                         (0x3ff << 14)
+#define OUT_REG_RELATCH_EN_SEL_SFT                                24
+#define OUT_REG_RELATCH_EN_SEL_MASK                               0x1f
+#define OUT_REG_RELATCH_EN_SEL_MASK_SFT                           (0x1f << 24)
+#define OUT_REG_WAIT_LAST_SAMPLE_SFT                              30
+#define OUT_REG_WAIT_LAST_SAMPLE_MASK                             0x1
+#define OUT_REG_WAIT_LAST_SAMPLE_MASK_SFT                         (0x1 << 30)
+#define OUT_REG_ALWAYS_OPEN_1X_EN_SFT                             31
+#define OUT_REG_ALWAYS_OPEN_1X_EN_MASK                            0x1
+#define OUT_REG_ALWAYS_OPEN_1X_EN_MASK_SFT                        (0x1 << 31)
+
+/* ETDM_OUT0_CON5 */
+/* ETDM_OUT1_CON5 */
+/* ETDM_OUT2_CON5 */
+/* ETDM_OUT3_CON5 */
+/* ETDM_OUT4_CON5 */
+/* ETDM_OUT5_CON5 */
+/* ETDM_OUT6_CON5 */
+#define OUT_REG_REPACK_BITNUM_SFT                                 0
+#define OUT_REG_REPACK_BITNUM_MASK                                0x3
+#define OUT_REG_REPACK_BITNUM_MASK_SFT                            (0x3 << 0)
+#define OUT_REG_REPACK_CHNUM_SFT                                  2
+#define OUT_REG_REPACK_CHNUM_MASK                                 0xf
+#define OUT_REG_REPACK_CHNUM_MASK_SFT                             (0xf << 2)
+#define OUT_REG_SLAVE_BCK_INV_SFT                                 7
+#define OUT_REG_SLAVE_BCK_INV_MASK                                0x1
+#define OUT_REG_SLAVE_BCK_INV_MASK_SFT                            (0x1 << 7)
+#define OUT_REG_SLAVE_LRCK_INV_SFT                                8
+#define OUT_REG_SLAVE_LRCK_INV_MASK                               0x1
+#define OUT_REG_SLAVE_LRCK_INV_MASK_SFT                           (0x1 << 8)
+#define OUT_REG_MASTER_BCK_INV_SFT                                9
+#define OUT_REG_MASTER_BCK_INV_MASK                               0x1
+#define OUT_REG_MASTER_BCK_INV_MASK_SFT                           (0x1 << 9)
+#define OUT_REG_MASTER_WS_INV_SFT                                 10
+#define OUT_REG_MASTER_WS_INV_MASK                                0x1
+#define OUT_REG_MASTER_WS_INV_MASK_SFT                            (0x1 << 10)
+#define OUT_REG_REPACK_24B_MSB_ALIGN_SFT                          11
+#define OUT_REG_REPACK_24B_MSB_ALIGN_MASK                         0x1
+#define OUT_REG_REPACK_24B_MSB_ALIGN_MASK_SFT                     (0x1 << 11)
+#define OUT_REG_LR_SWAP_SFT                                       16
+#define OUT_REG_LR_SWAP_MASK                                      0xffff
+#define OUT_REG_LR_SWAP_MASK_SFT                                  (0xffff << 16)
+
+/* ETDM_OUT0_CON6 */
+/* ETDM_OUT1_CON6 */
+/* ETDM_OUT2_CON6 */
+/* ETDM_OUT3_CON6 */
+/* ETDM_OUT4_CON6 */
+/* ETDM_OUT5_CON6 */
+/* ETDM_OUT6_CON6 */
+#define OUT_LCH_DATA_REG_SFT                                      0
+#define OUT_LCH_DATA_REG_MASK                                     0xffffffff
+#define OUT_LCH_DATA_REG_MASK_SFT                                 (0xffffffff << 0)
+
+/* ETDM_OUT0_CON7 */
+/* ETDM_OUT1_CON7 */
+/* ETDM_OUT2_CON7 */
+/* ETDM_OUT3_CON7 */
+/* ETDM_OUT4_CON7 */
+/* ETDM_OUT5_CON7 */
+/* ETDM_OUT6_CON7 */
+#define OUT_RCH_DATA_REG_SFT                                      0
+#define OUT_RCH_DATA_REG_MASK                                     0xffffffff
+#define OUT_RCH_DATA_REG_MASK_SFT                                 (0xffffffff << 0)
+
+/* ETDM_OUT0_CON8 */
+/* ETDM_OUT1_CON8 */
+/* ETDM_OUT2_CON8 */
+/* ETDM_OUT3_CON8 */
+/* ETDM_OUT4_CON8 */
+/* ETDM_OUT5_CON8 */
+/* ETDM_OUT6_CON8 */
+#define OUT_REG_START_CH_PAIR8_SFT                                0
+#define OUT_REG_START_CH_PAIR8_MASK                               0xf
+#define OUT_REG_START_CH_PAIR8_MASK_SFT                           (0xf << 0)
+#define OUT_REG_START_CH_PAIR9_SFT                                4
+#define OUT_REG_START_CH_PAIR9_MASK                               0xf
+#define OUT_REG_START_CH_PAIR9_MASK_SFT                           (0xf << 4)
+#define OUT_REG_START_CH_PAIR10_SFT                               8
+#define OUT_REG_START_CH_PAIR10_MASK                              0xf
+#define OUT_REG_START_CH_PAIR10_MASK_SFT                          (0xf << 8)
+#define OUT_REG_START_CH_PAIR11_SFT                               12
+#define OUT_REG_START_CH_PAIR11_MASK                              0xf
+#define OUT_REG_START_CH_PAIR11_MASK_SFT                          (0xf << 12)
+#define OUT_REG_START_CH_PAIR12_SFT                               16
+#define OUT_REG_START_CH_PAIR12_MASK                              0xf
+#define OUT_REG_START_CH_PAIR12_MASK_SFT                          (0xf << 16)
+#define OUT_REG_START_CH_PAIR13_SFT                               20
+#define OUT_REG_START_CH_PAIR13_MASK                              0xf
+#define OUT_REG_START_CH_PAIR13_MASK_SFT                          (0xf << 20)
+#define OUT_REG_START_CH_PAIR14_SFT                               24
+#define OUT_REG_START_CH_PAIR14_MASK                              0xf
+#define OUT_REG_START_CH_PAIR14_MASK_SFT                          (0xf << 24)
+#define OUT_REG_START_CH_PAIR15_SFT                               28
+#define OUT_REG_START_CH_PAIR15_MASK                              0xf
+#define OUT_REG_START_CH_PAIR15_MASK_SFT                          (0xf << 28)
+
+/* ETDM_OUT0_CON9 */
+/* ETDM_OUT1_CON9 */
+/* ETDM_OUT2_CON9 */
+/* ETDM_OUT3_CON9 */
+/* ETDM_OUT4_CON9 */
+/* ETDM_OUT5_CON9 */
+/* ETDM_OUT6_CON9 */
+#define OUT_REG_AFIFO_THRESHOLD_SFT                               29
+#define OUT_REG_AFIFO_THRESHOLD_MASK                              0x3
+#define OUT_REG_AFIFO_THRESHOLD_MASK_SFT                          (0x3 << 29)
+#define OUT_REG_AFIFO_SW_RESET_SFT                                15
+#define OUT_REG_AFIFO_SW_RESET_MASK                               0x1
+#define OUT_REG_AFIFO_SW_RESET_MASK_SFT                           (0x1 << 15)
+#define OUT_REG_AFIFO_RESET_SEL_SFT                               14
+#define OUT_REG_AFIFO_RESET_SEL_MASK                              0x1
+#define OUT_REG_AFIFO_RESET_SEL_MASK_SFT                          (0x1 << 14)
+#define OUT_REG_AFIFO_AUTO_RESET_DIS_SFT                          9
+#define OUT_REG_AFIFO_AUTO_RESET_DIS_MASK                         0x1
+#define OUT_REG_AFIFO_AUTO_RESET_DIS_MASK_SFT                     (0x1 << 9)
+#define OUT_REG_ETDM_USE_AFIFO_SFT                                8
+#define OUT_REG_ETDM_USE_AFIFO_MASK                               0x1
+#define OUT_REG_ETDM_USE_AFIFO_MASK_SFT                           (0x1 << 8)
+#define OUT_REG_AFIFO_CLOCK_DOMAIN_SEL_SFT                        5
+#define OUT_REG_AFIFO_CLOCK_DOMAIN_SEL_MASK                       0x7
+#define OUT_REG_AFIFO_CLOCK_DOMAIN_SEL_MASK_SFT                   (0x7 << 5)
+#define OUT_REG_AFIFO_MODE_SFT                                    0
+#define OUT_REG_AFIFO_MODE_MASK                                   0x1f
+#define OUT_REG_AFIFO_MODE_MASK_SFT                               (0x1f << 0)
+
+/* ETDM_OUT0_MON */
+/* ETDM_OUT1_MON */
+/* ETDM_OUT2_MON */
+/* ETDM_OUT3_MON */
+/* ETDM_OUT4_MON */
+/* ETDM_OUT5_MON */
+/* ETDM_OUT6_MON */
+#define LRCK_INV_SFT                                          30
+#define LRCK_INV_MASK                                         0x1
+#define LRCK_INV_MASK_SFT                                     (0x1 << 30)
+#define EN_SYNC_OUT_SFT                                       29
+#define EN_SYNC_OUT_MASK                                      0x1
+#define EN_SYNC_OUT_MASK_SFT                                  (0x1 << 29)
+#define HOPPING_EN_SYNC_OUT_PRE_SFT                           28
+#define HOPPING_EN_SYNC_OUT_PRE_MASK                          0x1
+#define HOPPING_EN_SYNC_OUT_PRE_MASK_SFT                      (0x1 << 28)
+#define ETDM_2X_CK_EN_SFT                                     25
+#define ETDM_2X_CK_EN_MASK                                    0x1
+#define ETDM_2X_CK_EN_MASK_SFT                                (0x1 << 25)
+#define ETDM_1X_CK_EN_SFT                                     24
+#define ETDM_1X_CK_EN_MASK                                    0x1
+#define ETDM_1X_CK_EN_MASK_SFT                                (0x1 << 24)
+#define SDATA0_SFT                                            23
+#define SDATA0_MASK                                           0x1
+#define SDATA0_MASK_SFT                                       (0x1 << 23)
+#define CURRENT_STATUS_SFT                                    21
+#define CURRENT_STATUS_MASK                                   0x3
+#define CURRENT_STATUS_MASK_SFT                               (0x3 << 21)
+#define BIT_POINT_SFT                                         16
+#define BIT_POINT_MASK                                        0x1f
+#define BIT_POINT_MASK_SFT                                    (0x1f << 16)
+#define BIT_CH_COUNT_SFT                                      10
+#define BIT_CH_COUNT_MASK                                     0x3f
+#define BIT_CH_COUNT_MASK_SFT                                 (0x3f << 10)
+#define BIT_COUNT_SFT                                         5
+#define BIT_COUNT_MASK                                        0x1f
+#define BIT_COUNT_MASK_SFT                                    (0x1f << 5)
+#define CH_COUNT_SFT                                          0
+#define CH_COUNT_MASK                                         0x1f
+#define CH_COUNT_MASK_SFT                                     (0x1f << 0)
+
+/* ETDM_0_3_COWORK_CON0 */
+#define ETDM_OUT0_DATA_SEL_SFT                                0
+#define ETDM_OUT0_DATA_SEL_MASK                               0xf
+#define ETDM_OUT0_DATA_SEL_MASK_SFT                           (0xf << 0)
+#define ETDM_OUT0_SYNC_SEL_SFT                                4
+#define ETDM_OUT0_SYNC_SEL_MASK                               0xf
+#define ETDM_OUT0_SYNC_SEL_MASK_SFT                           (0xf << 4)
+#define ETDM_OUT0_SLAVE_SEL_SFT                               8
+#define ETDM_OUT0_SLAVE_SEL_MASK                              0xf
+#define ETDM_OUT0_SLAVE_SEL_MASK_SFT                          (0xf << 8)
+#define ETDM_OUT1_DATA_SEL_SFT                                12
+#define ETDM_OUT1_DATA_SEL_MASK                               0xf
+#define ETDM_OUT1_DATA_SEL_MASK_SFT                           (0xf << 12)
+#define ETDM_OUT1_SYNC_SEL_SFT                                16
+#define ETDM_OUT1_SYNC_SEL_MASK                               0xf
+#define ETDM_OUT1_SYNC_SEL_MASK_SFT                           (0xf << 16)
+#define ETDM_OUT1_SLAVE_SEL_SFT                               20
+#define ETDM_OUT1_SLAVE_SEL_MASK                              0xf
+#define ETDM_OUT1_SLAVE_SEL_MASK_SFT                          (0xf << 20)
+#define ETDM_IN0_SLAVE_SEL_SFT                                24
+#define ETDM_IN0_SLAVE_SEL_MASK                               0xf
+#define ETDM_IN0_SLAVE_SEL_MASK_SFT                           (0xf << 24)
+#define ETDM_IN0_SYNC_SEL_SFT                                 28
+#define ETDM_IN0_SYNC_SEL_MASK                                0xf
+#define ETDM_IN0_SYNC_SEL_MASK_SFT                            (0xf << 28)
+
+/* ETDM_0_3_COWORK_CON1 */
+#define ETDM_IN0_SDATA0_SEL_SFT                               0
+#define ETDM_IN0_SDATA0_SEL_MASK                              0xf
+#define ETDM_IN0_SDATA0_SEL_MASK_SFT                          (0xf << 0)
+#define ETDM_IN0_SDATA1_15_SEL_SFT                            4
+#define ETDM_IN0_SDATA1_15_SEL_MASK                           0xf
+#define ETDM_IN0_SDATA1_15_SEL_MASK_SFT                       (0xf << 4)
+#define ETDM_IN1_SLAVE_SEL_SFT                                8
+#define ETDM_IN1_SLAVE_SEL_MASK                               0xf
+#define ETDM_IN1_SLAVE_SEL_MASK_SFT                           (0xf << 8)
+#define ETDM_IN1_SYNC_SEL_SFT                                 12
+#define ETDM_IN1_SYNC_SEL_MASK                                0xf
+#define ETDM_IN1_SYNC_SEL_MASK_SFT                            (0xf << 12)
+#define ETDM_IN1_SDATA0_SEL_SFT                               16
+#define ETDM_IN1_SDATA0_SEL_MASK                              0xf
+#define ETDM_IN1_SDATA0_SEL_MASK_SFT                          (0xf << 16)
+#define ETDM_IN1_SDATA1_15_SEL_SFT                            20
+#define ETDM_IN1_SDATA1_15_SEL_MASK                           0xf
+#define ETDM_IN1_SDATA1_15_SEL_MASK_SFT                       (0xf << 20)
+
+/* ETDM_0_3_COWORK_CON2 */
+#define ETDM_OUT2_DATA_SEL_SFT                                0
+#define ETDM_OUT2_DATA_SEL_MASK                               0xf
+#define ETDM_OUT2_DATA_SEL_MASK_SFT                           (0xf << 0)
+#define ETDM_OUT2_SYNC_SEL_SFT                                4
+#define ETDM_OUT2_SYNC_SEL_MASK                               0xf
+#define ETDM_OUT2_SYNC_SEL_MASK_SFT                           (0xf << 4)
+#define ETDM_OUT2_SLAVE_SEL_SFT                               8
+#define ETDM_OUT2_SLAVE_SEL_MASK                              0xf
+#define ETDM_OUT2_SLAVE_SEL_MASK_SFT                          (0xf << 8)
+#define ETDM_OUT3_DATA_SEL_SFT                                12
+#define ETDM_OUT3_DATA_SEL_MASK                               0xf
+#define ETDM_OUT3_DATA_SEL_MASK_SFT                           (0xf << 12)
+#define ETDM_OUT3_SYNC_SEL_SFT                                16
+#define ETDM_OUT3_SYNC_SEL_MASK                               0xf
+#define ETDM_OUT3_SYNC_SEL_MASK_SFT                           (0xf << 16)
+#define ETDM_OUT3_SLAVE_SEL_SFT                               20
+#define ETDM_OUT3_SLAVE_SEL_MASK                              0xf
+#define ETDM_OUT3_SLAVE_SEL_MASK_SFT                          (0xf << 20)
+#define ETDM_IN2_SLAVE_SEL_SFT                                24
+#define ETDM_IN2_SLAVE_SEL_MASK                               0xf
+#define ETDM_IN2_SLAVE_SEL_MASK_SFT                           (0xf << 24)
+#define ETDM_IN2_SYNC_SEL_SFT                                 28
+#define ETDM_IN2_SYNC_SEL_MASK                                0xf
+#define ETDM_IN2_SYNC_SEL_MASK_SFT                            (0xf << 28)
+
+/* ETDM_0_3_COWORK_CON3 */
+#define ETDM_IN2_SDATA0_SEL_SFT                               0
+#define ETDM_IN2_SDATA0_SEL_MASK                              0xf
+#define ETDM_IN2_SDATA0_SEL_MASK_SFT                          (0xf << 0)
+#define ETDM_IN2_SDATA1_15_SEL_SFT                            4
+#define ETDM_IN2_SDATA1_15_SEL_MASK                           0xf
+#define ETDM_IN2_SDATA1_15_SEL_MASK_SFT                       (0xf << 4)
+#define ETDM_IN3_SLAVE_SEL_SFT                                8
+#define ETDM_IN3_SLAVE_SEL_MASK                               0xf
+#define ETDM_IN3_SLAVE_SEL_MASK_SFT                           (0xf << 8)
+#define ETDM_IN3_SYNC_SEL_SFT                                 12
+#define ETDM_IN3_SYNC_SEL_MASK                                0xf
+#define ETDM_IN3_SYNC_SEL_MASK_SFT                            (0xf << 12)
+#define ETDM_IN3_SDATA0_SEL_SFT                               16
+#define ETDM_IN3_SDATA0_SEL_MASK                              0xf
+#define ETDM_IN3_SDATA0_SEL_MASK_SFT                          (0xf << 16)
+#define ETDM_IN3_SDATA1_15_SEL_SFT                            20
+#define ETDM_IN3_SDATA1_15_SEL_MASK                           0xf
+#define ETDM_IN3_SDATA1_15_SEL_MASK_SFT                       (0xf << 20)
+
+/* ETDM_4_7_COWORK_CON0 */
+#define ETDM_OUT4_DATA_SEL_SFT                                0
+#define ETDM_OUT4_DATA_SEL_MASK                               0xf
+#define ETDM_OUT4_DATA_SEL_MASK_SFT                           (0xf << 0)
+#define ETDM_OUT4_SYNC_SEL_SFT                                4
+#define ETDM_OUT4_SYNC_SEL_MASK                               0xf
+#define ETDM_OUT4_SYNC_SEL_MASK_SFT                           (0xf << 4)
+#define ETDM_OUT4_SLAVE_SEL_SFT                               8
+#define ETDM_OUT4_SLAVE_SEL_MASK                              0xf
+#define ETDM_OUT4_SLAVE_SEL_MASK_SFT                          (0xf << 8)
+#define ETDM_OUT5_DATA_SEL_SFT                                12
+#define ETDM_OUT5_DATA_SEL_MASK                               0xf
+#define ETDM_OUT5_DATA_SEL_MASK_SFT                           (0xf << 12)
+#define ETDM_OUT5_SYNC_SEL_SFT                                16
+#define ETDM_OUT5_SYNC_SEL_MASK                               0xf
+#define ETDM_OUT5_SYNC_SEL_MASK_SFT                           (0xf << 16)
+#define ETDM_OUT5_SLAVE_SEL_SFT                               20
+#define ETDM_OUT5_SLAVE_SEL_MASK                              0xf
+#define ETDM_OUT5_SLAVE_SEL_MASK_SFT                          (0xf << 20)
+#define ETDM_IN4_SLAVE_SEL_SFT                                24
+#define ETDM_IN4_SLAVE_SEL_MASK                               0xf
+#define ETDM_IN4_SLAVE_SEL_MASK_SFT                           (0xf << 24)
+#define ETDM_IN4_SYNC_SEL_SFT                                 28
+#define ETDM_IN4_SYNC_SEL_MASK                                0xf
+#define ETDM_IN4_SYNC_SEL_MASK_SFT                            (0xf << 28)
+
+/* ETDM_4_7_COWORK_CON1 */
+#define ETDM_IN4_SDATA0_SEL_SFT                               0
+#define ETDM_IN4_SDATA0_SEL_MASK                              0xf
+#define ETDM_IN4_SDATA0_SEL_MASK_SFT                          (0xf << 0)
+#define ETDM_IN4_SDATA1_15_SEL_SFT                            4
+#define ETDM_IN4_SDATA1_15_SEL_MASK                           0xf
+#define ETDM_IN4_SDATA1_15_SEL_MASK_SFT                       (0xf << 4)
+#define ETDM_IN5_SLAVE_SEL_SFT                                8
+#define ETDM_IN5_SLAVE_SEL_MASK                               0xf
+#define ETDM_IN5_SLAVE_SEL_MASK_SFT                           (0xf << 8)
+#define ETDM_IN5_SYNC_SEL_SFT                                 12
+#define ETDM_IN5_SYNC_SEL_MASK                                0xf
+#define ETDM_IN5_SYNC_SEL_MASK_SFT                            (0xf << 12)
+#define ETDM_IN5_SDATA0_SEL_SFT                               16
+#define ETDM_IN5_SDATA0_SEL_MASK                              0xf
+#define ETDM_IN5_SDATA0_SEL_MASK_SFT                          (0xf << 16)
+#define ETDM_IN5_SDATA1_15_SEL_SFT                            20
+#define ETDM_IN5_SDATA1_15_SEL_MASK                           0xf
+#define ETDM_IN5_SDATA1_15_SEL_MASK_SFT                       (0xf << 20)
+
+/* ETDM_4_7_COWORK_CON2 */
+#define ETDM_OUT6_DATA_SEL_SFT                                0
+#define ETDM_OUT6_DATA_SEL_MASK                               0xf
+#define ETDM_OUT6_DATA_SEL_MASK_SFT                           (0xf << 0)
+#define ETDM_OUT6_SYNC_SEL_SFT                                4
+#define ETDM_OUT6_SYNC_SEL_MASK                               0xf
+#define ETDM_OUT6_SYNC_SEL_MASK_SFT                           (0xf << 4)
+#define ETDM_OUT6_SLAVE_SEL_SFT                               8
+#define ETDM_OUT6_SLAVE_SEL_MASK                              0xf
+#define ETDM_OUT6_SLAVE_SEL_MASK_SFT                          (0xf << 8)
+#define ETDM_OUT7_DATA_SEL_SFT                                12
+#define ETDM_OUT7_DATA_SEL_MASK                               0xf
+#define ETDM_OUT7_DATA_SEL_MASK_SFT                           (0xf << 12)
+#define ETDM_OUT7_SYNC_SEL_SFT                                16
+#define ETDM_OUT7_SYNC_SEL_MASK                               0xf
+#define ETDM_OUT7_SYNC_SEL_MASK_SFT                           (0xf << 16)
+#define ETDM_OUT7_SLAVE_SEL_SFT                               20
+#define ETDM_OUT7_SLAVE_SEL_MASK                              0xf
+#define ETDM_OUT7_SLAVE_SEL_MASK_SFT                          (0xf << 20)
+#define ETDM_IN6_SLAVE_SEL_SFT                                24
+#define ETDM_IN6_SLAVE_SEL_MASK                               0xf
+#define ETDM_IN6_SLAVE_SEL_MASK_SFT                           (0xf << 24)
+#define ETDM_IN6_SYNC_SEL_SFT                                 28
+#define ETDM_IN6_SYNC_SEL_MASK                                0xf
+#define ETDM_IN6_SYNC_SEL_MASK_SFT                            (0xf << 28)
+
+/* ETDM_4_7_COWORK_CON3 */
+#define ETDM_IN6_SDATA0_SEL_SFT                               0
+#define ETDM_IN6_SDATA0_SEL_MASK                              0xf
+#define ETDM_IN6_SDATA0_SEL_MASK_SFT                          (0xf << 0)
+#define ETDM_IN6_SDATA1_15_SEL_SFT                            4
+#define ETDM_IN6_SDATA1_15_SEL_MASK                           0xf
+#define ETDM_IN6_SDATA1_15_SEL_MASK_SFT                       (0xf << 4)
+#define ETDM_IN7_SLAVE_SEL_SFT                                8
+#define ETDM_IN7_SLAVE_SEL_MASK                               0xf
+#define ETDM_IN7_SLAVE_SEL_MASK_SFT                           (0xf << 8)
+#define ETDM_IN7_SYNC_SEL_SFT                                 12
+#define ETDM_IN7_SYNC_SEL_MASK                                0xf
+#define ETDM_IN7_SYNC_SEL_MASK_SFT                            (0xf << 12)
+#define ETDM_IN7_SDATA0_SEL_SFT                               16
+#define ETDM_IN7_SDATA0_SEL_MASK                              0xf
+#define ETDM_IN7_SDATA0_SEL_MASK_SFT                          (0xf << 16)
+#define ETDM_IN7_SDATA1_15_SEL_SFT                            20
+#define ETDM_IN7_SDATA1_15_SEL_MASK                           0xf
+#define ETDM_IN7_SDATA1_15_SEL_MASK_SFT                       (0xf << 20)
+
+/* AFE_DPTX_CON */
+#define DPTX_CHANNEL_ENABLE_SFT                               8
+#define DPTX_CHANNEL_ENABLE_MASK                              0xff
+#define DPTX_CHANNEL_ENABLE_MASK_SFT                          (0xff << 8)
+#define DPTX_REGISTER_MONITOR_SELECT_SFT                      3
+#define DPTX_REGISTER_MONITOR_SELECT_MASK                     0xf
+#define DPTX_REGISTER_MONITOR_SELECT_MASK_SFT                 (0xf << 3)
+#define DPTX_16BIT_SFT                                        2
+#define DPTX_16BIT_MASK                                       0x1
+#define DPTX_16BIT_MASK_SFT                                   (0x1 << 2)
+#define DPTX_CHANNEL_NUMBER_SFT                               1
+#define DPTX_CHANNEL_NUMBER_MASK                              0x1
+#define DPTX_CHANNEL_NUMBER_MASK_SFT                          (0x1 << 1)
+#define DPTX_ON_SFT                                           0
+#define DPTX_ON_MASK                                          0x1
+#define DPTX_ON_MASK_SFT                                      (0x1 << 0)
+
+/* AFE_DPTX_MON */
+#define AFE_DPTX_MON0_SFT                                     0
+#define AFE_DPTX_MON0_MASK                                    0xffffffff
+#define AFE_DPTX_MON0_MASK_SFT                                (0xffffffff << 0)
+
+/* AFE_TDM_CON1 */
+#define TDM_EN_SFT                                            0
+#define TDM_EN_MASK                                           0x1
+#define TDM_EN_MASK_SFT                                       (0x1 << 0)
+#define BCK_INVERSE_SFT                                       1
+#define BCK_INVERSE_MASK                                      0x1
+#define BCK_INVERSE_MASK_SFT                                  (0x1 << 1)
+#define LRCK_INVERSE_SFT                                      2
+#define LRCK_INVERSE_MASK                                     0x1
+#define LRCK_INVERSE_MASK_SFT                                 (0x1 << 2)
+#define DELAY_DATA_SFT                                        3
+#define DELAY_DATA_MASK                                       0x1
+#define DELAY_DATA_MASK_SFT                                   (0x1 << 3)
+#define LEFT_ALIGN_SFT                                        4
+#define LEFT_ALIGN_MASK                                       0x1
+#define LEFT_ALIGN_MASK_SFT                                   (0x1 << 4)
+#define TDM_LRCK_D0P5T_SFT                                    5
+#define TDM_LRCK_D0P5T_MASK                                   0x1
+#define TDM_LRCK_D0P5T_MASK_SFT                               (0x1 << 5)
+#define TDM_SDATA_D0P5T_SFT                                   6
+#define TDM_SDATA_D0P5T_MASK                                  0x1
+#define TDM_SDATA_D0P5T_MASK_SFT                              (0x1 << 6)
+#define WLEN_SFT                                              8
+#define WLEN_MASK                                             0x3
+#define WLEN_MASK_SFT                                         (0x3 << 8)
+#define CHANNEL_NUM_SFT                                       10
+#define CHANNEL_NUM_MASK                                      0x3
+#define CHANNEL_NUM_MASK_SFT                                  (0x3 << 10)
+#define CHANNEL_BCK_CYCLES_SFT                                12
+#define CHANNEL_BCK_CYCLES_MASK                               0x3
+#define CHANNEL_BCK_CYCLES_MASK_SFT                           (0x3 << 12)
+#define HDMI_CLK_INV_SEL_SFT                                  15
+#define HDMI_CLK_INV_SEL_MASK                                 0x1
+#define HDMI_CLK_INV_SEL_MASK_SFT                             (0x1 << 15)
+#define DAC_BIT_NUM_SFT                                       16
+#define DAC_BIT_NUM_MASK                                      0x1f
+#define DAC_BIT_NUM_MASK_SFT                                  (0x1f << 16)
+#define LRCK_TDM_WIDTH_SFT                                    24
+#define LRCK_TDM_WIDTH_MASK                                   0xff
+#define LRCK_TDM_WIDTH_MASK_SFT                               (0xff << 24)
+
+/* AFE_TDM_CON2 */
+#define ST_CH_PAIR_SOUT0_SFT                                  0
+#define ST_CH_PAIR_SOUT0_MASK                                 0x7
+#define ST_CH_PAIR_SOUT0_MASK_SFT                             (0x7 << 0)
+#define ST_CH_PAIR_SOUT1_SFT                                  4
+#define ST_CH_PAIR_SOUT1_MASK                                 0x7
+#define ST_CH_PAIR_SOUT1_MASK_SFT                             (0x7 << 4)
+#define ST_CH_PAIR_SOUT2_SFT                                  8
+#define ST_CH_PAIR_SOUT2_MASK                                 0x7
+#define ST_CH_PAIR_SOUT2_MASK_SFT                             (0x7 << 8)
+#define ST_CH_PAIR_SOUT3_SFT                                  12
+#define ST_CH_PAIR_SOUT3_MASK                                 0x7
+#define ST_CH_PAIR_SOUT3_MASK_SFT                             (0x7 << 12)
+#define TDM_FIX_VALUE_SEL_SFT                                 16
+#define TDM_FIX_VALUE_SEL_MASK                                0x1
+#define TDM_FIX_VALUE_SEL_MASK_SFT                            (0x1 << 16)
+#define TDM_I2S_LOOPBACK_SFT                                  20
+#define TDM_I2S_LOOPBACK_MASK                                 0x1
+#define TDM_I2S_LOOPBACK_MASK_SFT                             (0x1 << 20)
+#define TDM_I2S_LOOPBACK_CH_SFT                               21
+#define TDM_I2S_LOOPBACK_CH_MASK                              0x3
+#define TDM_I2S_LOOPBACK_CH_MASK_SFT                          (0x3 << 21)
+#define TDM_USE_SINEGEN_INPUT_SFT                             23
+#define TDM_USE_SINEGEN_INPUT_MASK                            0x1
+#define TDM_USE_SINEGEN_INPUT_MASK_SFT                        (0x1 << 23)
+#define TDM_FIX_VALUE_SFT                                     24
+#define TDM_FIX_VALUE_MASK                                    0xff
+#define TDM_FIX_VALUE_MASK_SFT                                (0xff << 24)
+
+/* AFE_TDM_CON3 */
+#define TDM_OUT_SEL_DOMAIN_SFT                                29
+#define TDM_OUT_SEL_DOMAIN_MASK                               0x7
+#define TDM_OUT_SEL_DOMAIN_MASK_SFT                           (0x7 << 29)
+#define TDM_OUT_SEL_FS_SFT                                    24
+#define TDM_OUT_SEL_FS_MASK                                   0x1f
+#define TDM_OUT_SEL_FS_MASK_SFT                               (0x1f << 24)
+#define TDM_OUT_MON_SEL_SFT                                   3
+#define TDM_OUT_MON_SEL_MASK                                  0x1
+#define TDM_OUT_MON_SEL_MASK_SFT                              (0x1 << 3)
+#define RG_TDM_OUT_ASYNC_FIFO_SOFT_RST_EN_SFT                 2
+#define RG_TDM_OUT_ASYNC_FIFO_SOFT_RST_EN_MASK                0x1
+#define RG_TDM_OUT_ASYNC_FIFO_SOFT_RST_EN_MASK_SFT            (0x1 << 2)
+#define RG_TDM_OUT_ASYNC_FIFO_SOFT_RST_SFT                    1
+#define RG_TDM_OUT_ASYNC_FIFO_SOFT_RST_MASK                   0x1
+#define RG_TDM_OUT_ASYNC_FIFO_SOFT_RST_MASK_SFT               (0x1 << 1)
+#define TDM_UPDATE_EN_SEL_SFT                                 0
+#define TDM_UPDATE_EN_SEL_MASK                                0x1
+#define TDM_UPDATE_EN_SEL_MASK_SFT                            (0x1 << 0)
+
+/* AFE_TDM_OUT_MON */
+#define AFE_TDM_OUT_MON_SFT                                   0
+#define AFE_TDM_OUT_MON_MASK                                  0xffffffff
+#define AFE_TDM_OUT_MON_MASK_SFT                              (0xffffffff << 0)
+
+/* AFE_HDMI_CONN0 */
+#define HDMI_O_7_SFT                                          21
+#define HDMI_O_7_MASK                                         0x7
+#define HDMI_O_7_MASK_SFT                                     (0x7 << 21)
+#define HDMI_O_6_SFT                                          18
+#define HDMI_O_6_MASK                                         0x7
+#define HDMI_O_6_MASK_SFT                                     (0x7 << 18)
+#define HDMI_O_5_SFT                                          15
+#define HDMI_O_5_MASK                                         0x7
+#define HDMI_O_5_MASK_SFT                                     (0x7 << 15)
+#define HDMI_O_4_SFT                                          12
+#define HDMI_O_4_MASK                                         0x7
+#define HDMI_O_4_MASK_SFT                                     (0x7 << 12)
+#define HDMI_O_3_SFT                                          9
+#define HDMI_O_3_MASK                                         0x7
+#define HDMI_O_3_MASK_SFT                                     (0x7 << 9)
+#define HDMI_O_2_SFT                                          6
+#define HDMI_O_2_MASK                                         0x7
+#define HDMI_O_2_MASK_SFT                                     (0x7 << 6)
+#define HDMI_O_1_SFT                                          3
+#define HDMI_O_1_MASK                                         0x7
+#define HDMI_O_1_MASK_SFT                                     (0x7 << 3)
+#define HDMI_O_0_SFT                                          0
+#define HDMI_O_0_MASK                                         0x7
+#define HDMI_O_0_MASK_SFT                                     (0x7 << 0)
+
+/* AFE_TDM_TOP_IP_VERSION */
+#define AFE_TDM_TOP_IP_VERSION_SFT                            0
+#define AFE_TDM_TOP_IP_VERSION_MASK                           0xffffffff
+#define AFE_TDM_TOP_IP_VERSION_MASK_SFT                       (0xffffffff << 0)
+
+/* AFE_CBIP_CFG0 */
+#define CBIP_TOP_SLV_MUX_WAY_EN_SFT                           16
+#define CBIP_TOP_SLV_MUX_WAY_EN_MASK                          0xffff
+#define CBIP_TOP_SLV_MUX_WAY_EN_MASK_SFT                      (0xffff << 16)
+#define RESERVED_04_SFT                                       15
+#define RESERVED_04_MASK                                      0x1
+#define RESERVED_04_MASK_SFT                                  (0x1 << 15)
+#define CBIP_ASYNC_MST_RG_FIFO_THRE_SFT                       13
+#define CBIP_ASYNC_MST_RG_FIFO_THRE_MASK                      0x3
+#define CBIP_ASYNC_MST_RG_FIFO_THRE_MASK_SFT                  (0x3 << 13)
+#define CBIP_ASYNC_MST_POSTWRITE_DIS_SFT                      12
+#define CBIP_ASYNC_MST_POSTWRITE_DIS_MASK                     0x1
+#define CBIP_ASYNC_MST_POSTWRITE_DIS_MASK_SFT                 (0x1 << 12)
+#define RESERVED_03_SFT                                       11
+#define RESERVED_03_MASK                                      0x1
+#define RESERVED_03_MASK_SFT                                  (0x1 << 11)
+#define CBIP_ASYNC_SLV_RG_FIFO_THRE_SFT                       9
+#define CBIP_ASYNC_SLV_RG_FIFO_THRE_MASK                      0x3
+#define CBIP_ASYNC_SLV_RG_FIFO_THRE_MASK_SFT                  (0x3 << 9)
+#define CBIP_ASYNC_SLV_POSTWRITE_DIS_SFT                      8
+#define CBIP_ASYNC_SLV_POSTWRITE_DIS_MASK                     0x1
+#define CBIP_ASYNC_SLV_POSTWRITE_DIS_MASK_SFT                 (0x1 << 8)
+#define AUDIOSYS_BUSY_SFT                                     7
+#define AUDIOSYS_BUSY_MASK                                    0x1
+#define AUDIOSYS_BUSY_MASK_SFT                                (0x1 << 7)
+#define CBIP_SLV_DECODER_ERR_FLAG_EN_SFT                      6
+#define CBIP_SLV_DECODER_ERR_FLAG_EN_MASK                     0x1
+#define CBIP_SLV_DECODER_ERR_FLAG_EN_MASK_SFT                 (0x1 << 6)
+#define CBIP_SLV_DECODER_SLAVE_WAY_EN_SFT                     5
+#define CBIP_SLV_DECODER_SLAVE_WAY_EN_MASK                    0x1
+#define CBIP_SLV_DECODER_SLAVE_WAY_EN_MASK_SFT                (0x1 << 5)
+#define APB_R2T_SFT                                           3
+#define APB_R2T_MASK                                          0x1
+#define APB_R2T_MASK_SFT                                      (0x1 << 3)
+#define APB_W2T_SFT                                           2
+#define APB_W2T_MASK                                          0x1
+#define APB_W2T_MASK_SFT                                      (0x1 << 2)
+#define AHB_IDLE_EN_INT_SFT                                   1
+#define AHB_IDLE_EN_INT_MASK                                  0x1
+#define AHB_IDLE_EN_INT_MASK_SFT                              (0x1 << 1)
+#define AHB_IDLE_EN_EXT_SFT                                   0
+#define AHB_IDLE_EN_EXT_MASK                                  0x1
+#define AHB_IDLE_EN_EXT_MASK_SFT                              (0x1 << 0)
+
+/* AFE_CBIP_SLV_DECODER_MON0 */
+#define CBIP_SLV_DECODER_ERR_DOMAIN_SFT                       4
+#define CBIP_SLV_DECODER_ERR_DOMAIN_MASK                      0x1
+#define CBIP_SLV_DECODER_ERR_DOMAIN_MASK_SFT                  (0x1 << 4)
+#define CBIP_SLV_DECODER_ERR_ID_SFT                           3
+#define CBIP_SLV_DECODER_ERR_ID_MASK                          0x1
+#define CBIP_SLV_DECODER_ERR_ID_MASK_SFT                      (0x1 << 3)
+#define CBIP_SLV_DECODER_ERR_RW_SFT                           2
+#define CBIP_SLV_DECODER_ERR_RW_MASK                          0x1
+#define CBIP_SLV_DECODER_ERR_RW_MASK_SFT                      (0x1 << 2)
+#define CBIP_SLV_DECODER_ERR_DECERR_SFT                       1
+#define CBIP_SLV_DECODER_ERR_DECERR_MASK                      0x1
+#define CBIP_SLV_DECODER_ERR_DECERR_MASK_SFT                  (0x1 << 1)
+#define CBIP_SLV_DECODER_CTRL_UPDATE_STATUS_SFT               0
+#define CBIP_SLV_DECODER_CTRL_UPDATE_STATUS_MASK              0x1
+#define CBIP_SLV_DECODER_CTRL_UPDATE_STATUS_MASK_SFT          (0x1 << 0)
+
+/* AFE_CBIP_SLV_DECODER_MON1 */
+#define CBIP_SLV_DECODER_ERR_ADDR_SFT                         0
+#define CBIP_SLV_DECODER_ERR_ADDR_MASK                        0xffffffff
+#define CBIP_SLV_DECODER_ERR_ADDR_MASK_SFT                    (0xffffffff << 0)
+
+/* AFE_CBIP_SLV_MUX_MON_CFG */
+#define CBIP_SLV_MUX_ERR_FLAG_EN_SFT                          3
+#define CBIP_SLV_MUX_ERR_FLAG_EN_MASK                         0x1
+#define CBIP_SLV_MUX_ERR_FLAG_EN_MASK_SFT                     (0x1 << 3)
+#define CBIP_SLV_MUX_REG_SLAVE_WAY_EN_SFT                     2
+#define CBIP_SLV_MUX_REG_SLAVE_WAY_EN_MASK                    0x1
+#define CBIP_SLV_MUX_REG_SLAVE_WAY_EN_MASK_SFT                (0x1 << 2)
+#define CBIP_SLV_MUX_REG_LAYER_WAY_EN_SFT                     0
+#define CBIP_SLV_MUX_REG_LAYER_WAY_EN_MASK                    0x3
+#define CBIP_SLV_MUX_REG_LAYER_WAY_EN_MASK_SFT                (0x3 << 0)
+
+/* AFE_CBIP_SLV_MUX_MON0 */
+#define CBIP_SLV_MUX_ERR_DOMAIN_SFT                           8
+#define CBIP_SLV_MUX_ERR_DOMAIN_MASK                          0x1
+#define CBIP_SLV_MUX_ERR_DOMAIN_MASK_SFT                      (0x1 << 8)
+#define CBIP_SLV_MUX_ERR_ID_SFT                               7
+#define CBIP_SLV_MUX_ERR_ID_MASK                              0x1
+#define CBIP_SLV_MUX_ERR_ID_MASK_SFT                          (0x1 << 7)
+#define CBIP_SLV_MUX_ERR_RD_SFT                               6
+#define CBIP_SLV_MUX_ERR_RD_MASK                              0x1
+#define CBIP_SLV_MUX_ERR_RD_MASK_SFT                          (0x1 << 6)
+#define CBIP_SLV_MUX_ERR_WR_SFT                               5
+#define CBIP_SLV_MUX_ERR_WR_MASK                              0x1
+#define CBIP_SLV_MUX_ERR_WR_MASK_SFT                          (0x1 << 5)
+#define CBIP_SLV_MUX_ERR_EN_SLV_SFT                           4
+#define CBIP_SLV_MUX_ERR_EN_SLV_MASK                          0x1
+#define CBIP_SLV_MUX_ERR_EN_SLV_MASK_SFT                      (0x1 << 4)
+#define CBIP_SLV_MUX_ERR_EN_MST_SFT                           2
+#define CBIP_SLV_MUX_ERR_EN_MST_MASK                          0x3
+#define CBIP_SLV_MUX_ERR_EN_MST_MASK_SFT                      (0x3 << 2)
+#define CBIP_SLV_MUX_CTRL_UPDATE_STATUS_SFT                   0
+#define CBIP_SLV_MUX_CTRL_UPDATE_STATUS_MASK                  0x3
+#define CBIP_SLV_MUX_CTRL_UPDATE_STATUS_MASK_SFT              (0x3 << 0)
+
+/* AFE_CBIP_SLV_MUX_MON1 */
+#define CBIP_SLV_MUX_ERR_ADDR_SFT                             0
+#define CBIP_SLV_MUX_ERR_ADDR_MASK                            0xffffffff
+#define CBIP_SLV_MUX_ERR_ADDR_MASK_SFT                        (0xffffffff << 0)
+
+/* AFE_MEMIF_CON0 */
+#define CPU_COMPACT_MODE_SFT                                  2
+#define CPU_COMPACT_MODE_MASK                                 0x1
+#define CPU_COMPACT_MODE_MASK_SFT                             (0x1 << 2)
+#define CPU_HD_ALIGN_SFT                                      1
+#define CPU_HD_ALIGN_MASK                                     0x1
+#define CPU_HD_ALIGN_MASK_SFT                                 (0x1 << 1)
+#define SYSRAM_SIGN_SFT                                       0
+#define SYSRAM_SIGN_MASK                                      0x1
+#define SYSRAM_SIGN_MASK_SFT                                  (0x1 << 0)
+
+/* AFE_MEMIF_ONE_HEART */
+#define DL_ONE_HEART_ON_2_SFT                                 2
+#define DL_ONE_HEART_ON_2_MASK                                0x1
+#define DL_ONE_HEART_ON_2_MASK_SFT                            (0x1 << 2)
+#define DL_ONE_HEART_ON_1_SFT                                 1
+#define DL_ONE_HEART_ON_1_MASK                                0x1
+#define DL_ONE_HEART_ON_1_MASK_SFT                            (0x1 << 1)
+#define DL_ONE_HEART_ON_0_SFT                                 0
+#define DL_ONE_HEART_ON_0_MASK                                0x1
+#define DL_ONE_HEART_ON_0_MASK_SFT                            (0x1 << 0)
+
+/* AFE_DL0_BASE_MSB */
+#define DL0_BASE_ADDR_MSB_SFT                                 0
+#define DL0_BASE_ADDR_MSB_MASK                                0x1ff
+#define DL0_BASE_ADDR_MSB_MASK_SFT                            (0x1ff << 0)
+
+/* AFE_DL0_BASE */
+#define DL0_BASE_ADDR_SFT                                     4
+#define DL0_BASE_ADDR_MASK                                    0xfffffff
+#define DL0_BASE_ADDR_MASK_SFT                                (0xfffffff << 4)
+
+/* AFE_DL0_CUR_MSB */
+#define DL0_CUR_PTR_MSB_SFT                                   0
+#define DL0_CUR_PTR_MSB_MASK                                  0x1ff
+#define DL0_CUR_PTR_MSB_MASK_SFT                              (0x1ff << 0)
+
+/* AFE_DL0_CUR */
+#define DL0_CUR_PTR_SFT                                       0
+#define DL0_CUR_PTR_MASK                                      0xffffffff
+#define DL0_CUR_PTR_MASK_SFT                                  (0xffffffff << 0)
+
+/* AFE_DL0_END_MSB */
+#define DL0_END_ADDR_MSB_SFT                                  0
+#define DL0_END_ADDR_MSB_MASK                                 0x1ff
+#define DL0_END_ADDR_MSB_MASK_SFT                             (0x1ff << 0)
+
+/* AFE_DL0_END */
+#define DL0_END_ADDR_SFT                                      4
+#define DL0_END_ADDR_MASK                                     0xfffffff
+#define DL0_END_ADDR_MASK_SFT                                 (0xfffffff << 4)
+
+/* AFE_DL0_RCH_MON */
+#define DL0_RCH_DATA_SFT                                      0
+#define DL0_RCH_DATA_MASK                                     0xffffffff
+#define DL0_RCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL0_LCH_MON */
+#define DL0_LCH_DATA_SFT                                      0
+#define DL0_LCH_DATA_MASK                                     0xffffffff
+#define DL0_LCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL0_CON0 */
+#define DL0_ON_SFT                                            28
+#define DL0_ON_MASK                                           0x1
+#define DL0_ON_MASK_SFT                                       (0x1 << 28)
+#define DL0_ONE_HEART_SEL_SFT                                 22
+#define DL0_ONE_HEART_SEL_MASK                                0x3
+#define DL0_ONE_HEART_SEL_MASK_SFT                            (0x3 << 22)
+#define DL0_MINLEN_SFT                                        20
+#define DL0_MINLEN_MASK                                       0x3
+#define DL0_MINLEN_MASK_SFT                                   (0x3 << 20)
+#define DL0_MAXLEN_SFT                                        16
+#define DL0_MAXLEN_MASK                                       0x3
+#define DL0_MAXLEN_MASK_SFT                                   (0x3 << 16)
+#define DL0_SEL_DOMAIN_SFT                                    13
+#define DL0_SEL_DOMAIN_MASK                                   0x7
+#define DL0_SEL_DOMAIN_MASK_SFT                               (0x7 << 13)
+#define DL0_SEL_FS_SFT                                        8
+#define DL0_SEL_FS_MASK                                       0x1f
+#define DL0_SEL_FS_MASK_SFT                                   (0x1f << 8)
+#define DL0_SW_CLEAR_BUF_EMPTY_SFT                            7
+#define DL0_SW_CLEAR_BUF_EMPTY_MASK                           0x1
+#define DL0_SW_CLEAR_BUF_EMPTY_MASK_SFT                       (0x1 << 7)
+#define DL0_PBUF_SIZE_SFT                                     5
+#define DL0_PBUF_SIZE_MASK                                    0x3
+#define DL0_PBUF_SIZE_MASK_SFT                                (0x3 << 5)
+#define DL0_MONO_SFT                                          4
+#define DL0_MONO_MASK                                         0x1
+#define DL0_MONO_MASK_SFT                                     (0x1 << 4)
+#define DL0_NORMAL_MODE_SFT                                   3
+#define DL0_NORMAL_MODE_MASK                                  0x1
+#define DL0_NORMAL_MODE_MASK_SFT                              (0x1 << 3)
+#define DL0_HALIGN_SFT                                        2
+#define DL0_HALIGN_MASK                                       0x1
+#define DL0_HALIGN_MASK_SFT                                   (0x1 << 2)
+#define DL0_HD_MODE_SFT                                       0
+#define DL0_HD_MODE_MASK                                      0x3
+#define DL0_HD_MODE_MASK_SFT                                  (0x3 << 0)
+
+/* AFE_DL1_BASE_MSB */
+#define DL1_BASE_ADDR_MSB_SFT                                 0
+#define DL1_BASE_ADDR_MSB_MASK                                0x1ff
+#define DL1_BASE_ADDR_MSB_MASK_SFT                            (0x1ff << 0)
+
+/* AFE_DL1_BASE */
+#define DL1_BASE_ADDR_SFT                                     4
+#define DL1_BASE_ADDR_MASK                                    0xfffffff
+#define DL1_BASE_ADDR_MASK_SFT                                (0xfffffff << 4)
+
+/* AFE_DL1_CUR_MSB */
+#define DL1_CUR_PTR_MSB_SFT                                   0
+#define DL1_CUR_PTR_MSB_MASK                                  0x1ff
+#define DL1_CUR_PTR_MSB_MASK_SFT                              (0x1ff << 0)
+
+/* AFE_DL1_CUR */
+#define DL1_CUR_PTR_SFT                                       0
+#define DL1_CUR_PTR_MASK                                      0xffffffff
+#define DL1_CUR_PTR_MASK_SFT                                  (0xffffffff << 0)
+
+/* AFE_DL1_END_MSB */
+#define DL1_END_ADDR_MSB_SFT                                  0
+#define DL1_END_ADDR_MSB_MASK                                 0x1ff
+#define DL1_END_ADDR_MSB_MASK_SFT                             (0x1ff << 0)
+
+/* AFE_DL1_END */
+#define DL1_END_ADDR_SFT                                      4
+#define DL1_END_ADDR_MASK                                     0xfffffff
+#define DL1_END_ADDR_MASK_SFT                                 (0xfffffff << 4)
+
+/* AFE_DL1_RCH_MON */
+#define DL1_RCH_DATA_SFT                                      0
+#define DL1_RCH_DATA_MASK                                     0xffffffff
+#define DL1_RCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL1_LCH_MON */
+#define DL1_LCH_DATA_SFT                                      0
+#define DL1_LCH_DATA_MASK                                     0xffffffff
+#define DL1_LCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL1_CON0 */
+#define DL1_ON_SFT                                            28
+#define DL1_ON_MASK                                           0x1
+#define DL1_ON_MASK_SFT                                       (0x1 << 28)
+#define DL1_ONE_HEART_SEL_SFT                                 22
+#define DL1_ONE_HEART_SEL_MASK                                0x3
+#define DL1_ONE_HEART_SEL_MASK_SFT                            (0x3 << 22)
+#define DL1_MINLEN_SFT                                        20
+#define DL1_MINLEN_MASK                                       0x3
+#define DL1_MINLEN_MASK_SFT                                   (0x3 << 20)
+#define DL1_MAXLEN_SFT                                        16
+#define DL1_MAXLEN_MASK                                       0x3
+#define DL1_MAXLEN_MASK_SFT                                   (0x3 << 16)
+#define DL1_SEL_DOMAIN_SFT                                    13
+#define DL1_SEL_DOMAIN_MASK                                   0x7
+#define DL1_SEL_DOMAIN_MASK_SFT                               (0x7 << 13)
+#define DL1_SEL_FS_SFT                                        8
+#define DL1_SEL_FS_MASK                                       0x1f
+#define DL1_SEL_FS_MASK_SFT                                   (0x1f << 8)
+#define DL1_SW_CLEAR_BUF_EMPTY_SFT                            7
+#define DL1_SW_CLEAR_BUF_EMPTY_MASK                           0x1
+#define DL1_SW_CLEAR_BUF_EMPTY_MASK_SFT                       (0x1 << 7)
+#define DL1_PBUF_SIZE_SFT                                     5
+#define DL1_PBUF_SIZE_MASK                                    0x3
+#define DL1_PBUF_SIZE_MASK_SFT                                (0x3 << 5)
+#define DL1_MONO_SFT                                          4
+#define DL1_MONO_MASK                                         0x1
+#define DL1_MONO_MASK_SFT                                     (0x1 << 4)
+#define DL1_NORMAL_MODE_SFT                                   3
+#define DL1_NORMAL_MODE_MASK                                  0x1
+#define DL1_NORMAL_MODE_MASK_SFT                              (0x1 << 3)
+#define DL1_HALIGN_SFT                                        2
+#define DL1_HALIGN_MASK                                       0x1
+#define DL1_HALIGN_MASK_SFT                                   (0x1 << 2)
+#define DL1_HD_MODE_SFT                                       0
+#define DL1_HD_MODE_MASK                                      0x3
+#define DL1_HD_MODE_MASK_SFT                                  (0x3 << 0)
+
+/* AFE_DL2_BASE_MSB */
+#define DL2_BASE__ADDR_MSB_SFT                                0
+#define DL2_BASE__ADDR_MSB_MASK                               0x1ff
+#define DL2_BASE__ADDR_MSB_MASK_SFT                           (0x1ff << 0)
+
+/* AFE_DL2_BASE */
+#define DL2_BASE_ADDR_SFT                                     4
+#define DL2_BASE_ADDR_MASK                                    0xfffffff
+#define DL2_BASE_ADDR_MASK_SFT                                (0xfffffff << 4)
+
+/* AFE_DL2_CUR_MSB */
+#define DL2_CUR_PTR_MSB_SFT                                   0
+#define DL2_CUR_PTR_MSB_MASK                                  0x1ff
+#define DL2_CUR_PTR_MSB_MASK_SFT                              (0x1ff << 0)
+
+/* AFE_DL2_CUR */
+#define DL2_CUR_PTR_SFT                                       0
+#define DL2_CUR_PTR_MASK                                      0xffffffff
+#define DL2_CUR_PTR_MASK_SFT                                  (0xffffffff << 0)
+
+/* AFE_DL2_END_MSB */
+#define DL2_END_ADDR_MSB_SFT                                  0
+#define DL2_END_ADDR_MSB_MASK                                 0x1ff
+#define DL2_END_ADDR_MSB_MASK_SFT                             (0x1ff << 0)
+
+/* AFE_DL2_END */
+#define DL2_END_ADDR_SFT                                      4
+#define DL2_END_ADDR_MASK                                     0xfffffff
+#define DL2_END_ADDR_MASK_SFT                                 (0xfffffff << 4)
+
+/* AFE_DL2_RCH_MON */
+#define DL2_RCH_DATA_SFT                                      0
+#define DL2_RCH_DATA_MASK                                     0xffffffff
+#define DL2_RCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL2_LCH_MON */
+#define DL2_LCH_DATA_SFT                                      0
+#define DL2_LCH_DATA_MASK                                     0xffffffff
+#define DL2_LCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL2_CON0 */
+#define DL2_ON_SFT                                            28
+#define DL2_ON_MASK                                           0x1
+#define DL2_ON_MASK_SFT                                       (0x1 << 28)
+#define DL2_ONE_HEART_SEL_SFT                                 22
+#define DL2_ONE_HEART_SEL_MASK                                0x3
+#define DL2_ONE_HEART_SEL_MASK_SFT                            (0x3 << 22)
+#define DL2_MINLEN_SFT                                        20
+#define DL2_MINLEN_MASK                                       0x3
+#define DL2_MINLEN_MASK_SFT                                   (0x3 << 20)
+#define DL2_MAXLEN_SFT                                        16
+#define DL2_MAXLEN_MASK                                       0x3
+#define DL2_MAXLEN_MASK_SFT                                   (0x3 << 16)
+#define DL2_SEL_DOMAIN_SFT                                    13
+#define DL2_SEL_DOMAIN_MASK                                   0x7
+#define DL2_SEL_DOMAIN_MASK_SFT                               (0x7 << 13)
+#define DL2_SEL_FS_SFT                                        8
+#define DL2_SEL_FS_MASK                                       0x1f
+#define DL2_SEL_FS_MASK_SFT                                   (0x1f << 8)
+#define DL2_SW_CLEAR_BUF_EMPTY_SFT                            7
+#define DL2_SW_CLEAR_BUF_EMPTY_MASK                           0x1
+#define DL2_SW_CLEAR_BUF_EMPTY_MASK_SFT                       (0x1 << 7)
+#define DL2_PBUF_SIZE_SFT                                     5
+#define DL2_PBUF_SIZE_MASK                                    0x3
+#define DL2_PBUF_SIZE_MASK_SFT                                (0x3 << 5)
+#define DL2_MONO_SFT                                          4
+#define DL2_MONO_MASK                                         0x1
+#define DL2_MONO_MASK_SFT                                     (0x1 << 4)
+#define DL2_NORMAL_MODE_SFT                                   3
+#define DL2_NORMAL_MODE_MASK                                  0x1
+#define DL2_NORMAL_MODE_MASK_SFT                              (0x1 << 3)
+#define DL2_HALIGN_SFT                                        2
+#define DL2_HALIGN_MASK                                       0x1
+#define DL2_HALIGN_MASK_SFT                                   (0x1 << 2)
+#define DL2_HD_MODE_SFT                                       0
+#define DL2_HD_MODE_MASK                                      0x3
+#define DL2_HD_MODE_MASK_SFT                                  (0x3 << 0)
+
+/* AFE_DL3_BASE_MSB */
+#define DL3_BASE__ADDR_MSB_SFT                                0
+#define DL3_BASE__ADDR_MSB_MASK                               0x1ff
+#define DL3_BASE__ADDR_MSB_MASK_SFT                           (0x1ff << 0)
+
+/* AFE_DL3_BASE */
+#define DL3_BASE_ADDR_SFT                                     4
+#define DL3_BASE_ADDR_MASK                                    0xfffffff
+#define DL3_BASE_ADDR_MASK_SFT                                (0xfffffff << 4)
+
+/* AFE_DL3_CUR_MSB */
+#define DL3_CUR_PTR_MSB_SFT                                   0
+#define DL3_CUR_PTR_MSB_MASK                                  0x1ff
+#define DL3_CUR_PTR_MSB_MASK_SFT                              (0x1ff << 0)
+
+/* AFE_DL3_CUR */
+#define DL3_CUR_PTR_SFT                                       0
+#define DL3_CUR_PTR_MASK                                      0xffffffff
+#define DL3_CUR_PTR_MASK_SFT                                  (0xffffffff << 0)
+
+/* AFE_DL3_END_MSB */
+#define DL3_END_ADDR_MSB_SFT                                  0
+#define DL3_END_ADDR_MSB_MASK                                 0x1ff
+#define DL3_END_ADDR_MSB_MASK_SFT                             (0x1ff << 0)
+
+/* AFE_DL3_END */
+#define DL3_END_ADDR_SFT                                      4
+#define DL3_END_ADDR_MASK                                     0xfffffff
+#define DL3_END_ADDR_MASK_SFT                                 (0xfffffff << 4)
+
+/* AFE_DL3_RCH_MON */
+#define DL3_RCH_DATA_SFT                                      0
+#define DL3_RCH_DATA_MASK                                     0xffffffff
+#define DL3_RCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL3_LCH_MON */
+#define DL3_LCH_DATA_SFT                                      0
+#define DL3_LCH_DATA_MASK                                     0xffffffff
+#define DL3_LCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL3_CON0 */
+#define DL3_ON_SFT                                            28
+#define DL3_ON_MASK                                           0x1
+#define DL3_ON_MASK_SFT                                       (0x1 << 28)
+#define DL3_ONE_HEART_SEL_SFT                                 22
+#define DL3_ONE_HEART_SEL_MASK                                0x3
+#define DL3_ONE_HEART_SEL_MASK_SFT                            (0x3 << 22)
+#define DL3_MINLEN_SFT                                        20
+#define DL3_MINLEN_MASK                                       0x3
+#define DL3_MINLEN_MASK_SFT                                   (0x3 << 20)
+#define DL3_MAXLEN_SFT                                        16
+#define DL3_MAXLEN_MASK                                       0x3
+#define DL3_MAXLEN_MASK_SFT                                   (0x3 << 16)
+#define DL3_SEL_DOMAIN_SFT                                    13
+#define DL3_SEL_DOMAIN_MASK                                   0x7
+#define DL3_SEL_DOMAIN_MASK_SFT                               (0x7 << 13)
+#define DL3_SEL_FS_SFT                                        8
+#define DL3_SEL_FS_MASK                                       0x1f
+#define DL3_SEL_FS_MASK_SFT                                   (0x1f << 8)
+#define DL3_SW_CLEAR_BUF_EMPTY_SFT                            7
+#define DL3_SW_CLEAR_BUF_EMPTY_MASK                           0x1
+#define DL3_SW_CLEAR_BUF_EMPTY_MASK_SFT                       (0x1 << 7)
+#define DL3_PBUF_SIZE_SFT                                     5
+#define DL3_PBUF_SIZE_MASK                                    0x3
+#define DL3_PBUF_SIZE_MASK_SFT                                (0x3 << 5)
+#define DL3_MONO_SFT                                          4
+#define DL3_MONO_MASK                                         0x1
+#define DL3_MONO_MASK_SFT                                     (0x1 << 4)
+#define DL3_NORMAL_MODE_SFT                                   3
+#define DL3_NORMAL_MODE_MASK                                  0x1
+#define DL3_NORMAL_MODE_MASK_SFT                              (0x1 << 3)
+#define DL3_HALIGN_SFT                                        2
+#define DL3_HALIGN_MASK                                       0x1
+#define DL3_HALIGN_MASK_SFT                                   (0x1 << 2)
+#define DL3_HD_MODE_SFT                                       0
+#define DL3_HD_MODE_MASK                                      0x3
+#define DL3_HD_MODE_MASK_SFT                                  (0x3 << 0)
+
+/* AFE_DL4_BASE_MSB */
+#define DL4_BASE__ADDR_MSB_SFT                                0
+#define DL4_BASE__ADDR_MSB_MASK                               0x1ff
+#define DL4_BASE__ADDR_MSB_MASK_SFT                           (0x1ff << 0)
+
+/* AFE_DL4_BASE */
+#define DL4_BASE_ADDR_SFT                                     4
+#define DL4_BASE_ADDR_MASK                                    0xfffffff
+#define DL4_BASE_ADDR_MASK_SFT                                (0xfffffff << 4)
+
+/* AFE_DL4_CUR_MSB */
+#define DL4_CUR_PTR_MSB_SFT                                   0
+#define DL4_CUR_PTR_MSB_MASK                                  0x1ff
+#define DL4_CUR_PTR_MSB_MASK_SFT                              (0x1ff << 0)
+
+/* AFE_DL4_CUR */
+#define DL4_CUR_PTR_SFT                                       0
+#define DL4_CUR_PTR_MASK                                      0xffffffff
+#define DL4_CUR_PTR_MASK_SFT                                  (0xffffffff << 0)
+
+/* AFE_DL4_END_MSB */
+#define DL4_END_ADDR_MSB_SFT                                  0
+#define DL4_END_ADDR_MSB_MASK                                 0x1ff
+#define DL4_END_ADDR_MSB_MASK_SFT                             (0x1ff << 0)
+
+/* AFE_DL4_END */
+#define DL4_END_ADDR_SFT                                      4
+#define DL4_END_ADDR_MASK                                     0xfffffff
+#define DL4_END_ADDR_MASK_SFT                                 (0xfffffff << 4)
+
+/* AFE_DL4_RCH_MON */
+#define DL4_RCH_DATA_SFT                                      0
+#define DL4_RCH_DATA_MASK                                     0xffffffff
+#define DL4_RCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL4_LCH_MON */
+#define DL4_LCH_DATA_SFT                                      0
+#define DL4_LCH_DATA_MASK                                     0xffffffff
+#define DL4_LCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL4_CON0 */
+#define DL4_ON_SFT                                            28
+#define DL4_ON_MASK                                           0x1
+#define DL4_ON_MASK_SFT                                       (0x1 << 28)
+#define DL4_ONE_HEART_SEL_SFT                                 22
+#define DL4_ONE_HEART_SEL_MASK                                0x3
+#define DL4_ONE_HEART_SEL_MASK_SFT                            (0x3 << 22)
+#define DL4_MINLEN_SFT                                        20
+#define DL4_MINLEN_MASK                                       0x3
+#define DL4_MINLEN_MASK_SFT                                   (0x3 << 20)
+#define DL4_MAXLEN_SFT                                        16
+#define DL4_MAXLEN_MASK                                       0x3
+#define DL4_MAXLEN_MASK_SFT                                   (0x3 << 16)
+#define DL4_SEL_DOMAIN_SFT                                    13
+#define DL4_SEL_DOMAIN_MASK                                   0x7
+#define DL4_SEL_DOMAIN_MASK_SFT                               (0x7 << 13)
+#define DL4_SEL_FS_SFT                                        8
+#define DL4_SEL_FS_MASK                                       0x1f
+#define DL4_SEL_FS_MASK_SFT                                   (0x1f << 8)
+#define DL4_SW_CLEAR_BUF_EMPTY_SFT                            7
+#define DL4_SW_CLEAR_BUF_EMPTY_MASK                           0x1
+#define DL4_SW_CLEAR_BUF_EMPTY_MASK_SFT                       (0x1 << 7)
+#define DL4_PBUF_SIZE_SFT                                     5
+#define DL4_PBUF_SIZE_MASK                                    0x3
+#define DL4_PBUF_SIZE_MASK_SFT                                (0x3 << 5)
+#define DL4_MONO_SFT                                          4
+#define DL4_MONO_MASK                                         0x1
+#define DL4_MONO_MASK_SFT                                     (0x1 << 4)
+#define DL4_NORMAL_MODE_SFT                                   3
+#define DL4_NORMAL_MODE_MASK                                  0x1
+#define DL4_NORMAL_MODE_MASK_SFT                              (0x1 << 3)
+#define DL4_HALIGN_SFT                                        2
+#define DL4_HALIGN_MASK                                       0x1
+#define DL4_HALIGN_MASK_SFT                                   (0x1 << 2)
+#define DL4_HD_MODE_SFT                                       0
+#define DL4_HD_MODE_MASK                                      0x3
+#define DL4_HD_MODE_MASK_SFT                                  (0x3 << 0)
+
+/* AFE_DL5_BASE_MSB */
+#define DL5_BASE__ADDR_MSB_SFT                                0
+#define DL5_BASE__ADDR_MSB_MASK                               0x1ff
+#define DL5_BASE__ADDR_MSB_MASK_SFT                           (0x1ff << 0)
+
+/* AFE_DL5_BASE */
+#define DL5_BASE_ADDR_SFT                                     4
+#define DL5_BASE_ADDR_MASK                                    0xfffffff
+#define DL5_BASE_ADDR_MASK_SFT                                (0xfffffff << 4)
+
+/* AFE_DL5_CUR_MSB */
+#define DL5_CUR_PTR_MSB_SFT                                   0
+#define DL5_CUR_PTR_MSB_MASK                                  0x1ff
+#define DL5_CUR_PTR_MSB_MASK_SFT                              (0x1ff << 0)
+
+/* AFE_DL5_CUR */
+#define DL5_CUR_PTR_SFT                                       0
+#define DL5_CUR_PTR_MASK                                      0xffffffff
+#define DL5_CUR_PTR_MASK_SFT                                  (0xffffffff << 0)
+
+/* AFE_DL5_END_MSB */
+#define DL5_END_ADDR_MSB_SFT                                  0
+#define DL5_END_ADDR_MSB_MASK                                 0x1ff
+#define DL5_END_ADDR_MSB_MASK_SFT                             (0x1ff << 0)
+
+/* AFE_DL5_END */
+#define DL5_END_ADDR_SFT                                      4
+#define DL5_END_ADDR_MASK                                     0xfffffff
+#define DL5_END_ADDR_MASK_SFT                                 (0xfffffff << 4)
+
+/* AFE_DL5_RCH_MON */
+#define DL5_RCH_DATA_SFT                                      0
+#define DL5_RCH_DATA_MASK                                     0xffffffff
+#define DL5_RCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL5_LCH_MON */
+#define DL5_LCH_DATA_SFT                                      0
+#define DL5_LCH_DATA_MASK                                     0xffffffff
+#define DL5_LCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL5_CON0 */
+#define DL5_ON_SFT                                            28
+#define DL5_ON_MASK                                           0x1
+#define DL5_ON_MASK_SFT                                       (0x1 << 28)
+#define DL5_ONE_HEART_SEL_SFT                                 22
+#define DL5_ONE_HEART_SEL_MASK                                0x3
+#define DL5_ONE_HEART_SEL_MASK_SFT                            (0x3 << 22)
+#define DL5_MINLEN_SFT                                        20
+#define DL5_MINLEN_MASK                                       0x3
+#define DL5_MINLEN_MASK_SFT                                   (0x3 << 20)
+#define DL5_MAXLEN_SFT                                        16
+#define DL5_MAXLEN_MASK                                       0x3
+#define DL5_MAXLEN_MASK_SFT                                   (0x3 << 16)
+#define DL5_SEL_DOMAIN_SFT                                    13
+#define DL5_SEL_DOMAIN_MASK                                   0x7
+#define DL5_SEL_DOMAIN_MASK_SFT                               (0x7 << 13)
+#define DL5_SEL_FS_SFT                                        8
+#define DL5_SEL_FS_MASK                                       0x1f
+#define DL5_SEL_FS_MASK_SFT                                   (0x1f << 8)
+#define DL5_SW_CLEAR_BUF_EMPTY_SFT                            7
+#define DL5_SW_CLEAR_BUF_EMPTY_MASK                           0x1
+#define DL5_SW_CLEAR_BUF_EMPTY_MASK_SFT                       (0x1 << 7)
+#define DL5_PBUF_SIZE_SFT                                     5
+#define DL5_PBUF_SIZE_MASK                                    0x3
+#define DL5_PBUF_SIZE_MASK_SFT                                (0x3 << 5)
+#define DL5_MONO_SFT                                          4
+#define DL5_MONO_MASK                                         0x1
+#define DL5_MONO_MASK_SFT                                     (0x1 << 4)
+#define DL5_NORMAL_MODE_SFT                                   3
+#define DL5_NORMAL_MODE_MASK                                  0x1
+#define DL5_NORMAL_MODE_MASK_SFT                              (0x1 << 3)
+#define DL5_HALIGN_SFT                                        2
+#define DL5_HALIGN_MASK                                       0x1
+#define DL5_HALIGN_MASK_SFT                                   (0x1 << 2)
+#define DL5_HD_MODE_SFT                                       0
+#define DL5_HD_MODE_MASK                                      0x3
+#define DL5_HD_MODE_MASK_SFT                                  (0x3 << 0)
+
+/* AFE_DL6_BASE_MSB */
+#define DL6_BASE__ADDR_MSB_SFT                                0
+#define DL6_BASE__ADDR_MSB_MASK                               0x1ff
+#define DL6_BASE__ADDR_MSB_MASK_SFT                           (0x1ff << 0)
+
+/* AFE_DL6_BASE */
+#define DL6_BASE_ADDR_SFT                                     4
+#define DL6_BASE_ADDR_MASK                                    0xfffffff
+#define DL6_BASE_ADDR_MASK_SFT                                (0xfffffff << 4)
+
+/* AFE_DL6_CUR_MSB */
+#define DL6_CUR_PTR_MSB_SFT                                   0
+#define DL6_CUR_PTR_MSB_MASK                                  0x1ff
+#define DL6_CUR_PTR_MSB_MASK_SFT                              (0x1ff << 0)
+
+/* AFE_DL6_CUR */
+#define DL6_CUR_PTR_SFT                                       0
+#define DL6_CUR_PTR_MASK                                      0xffffffff
+#define DL6_CUR_PTR_MASK_SFT                                  (0xffffffff << 0)
+
+/* AFE_DL6_END_MSB */
+#define DL6_END_ADDR_MSB_SFT                                  0
+#define DL6_END_ADDR_MSB_MASK                                 0x1ff
+#define DL6_END_ADDR_MSB_MASK_SFT                             (0x1ff << 0)
+
+/* AFE_DL6_END */
+#define DL6_END_ADDR_SFT                                      4
+#define DL6_END_ADDR_MASK                                     0xfffffff
+#define DL6_END_ADDR_MASK_SFT                                 (0xfffffff << 4)
+
+/* AFE_DL6_RCH_MON */
+#define DL6_RCH_DATA_SFT                                      0
+#define DL6_RCH_DATA_MASK                                     0xffffffff
+#define DL6_RCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL6_LCH_MON */
+#define DL6_LCH_DATA_SFT                                      0
+#define DL6_LCH_DATA_MASK                                     0xffffffff
+#define DL6_LCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL6_CON0 */
+#define DL6_ON_SFT                                            28
+#define DL6_ON_MASK                                           0x1
+#define DL6_ON_MASK_SFT                                       (0x1 << 28)
+#define DL6_ONE_HEART_SEL_SFT                                 22
+#define DL6_ONE_HEART_SEL_MASK                                0x3
+#define DL6_ONE_HEART_SEL_MASK_SFT                            (0x3 << 22)
+#define DL6_MINLEN_SFT                                        20
+#define DL6_MINLEN_MASK                                       0x3
+#define DL6_MINLEN_MASK_SFT                                   (0x3 << 20)
+#define DL6_MAXLEN_SFT                                        16
+#define DL6_MAXLEN_MASK                                       0x3
+#define DL6_MAXLEN_MASK_SFT                                   (0x3 << 16)
+#define DL6_SEL_DOMAIN_SFT                                    13
+#define DL6_SEL_DOMAIN_MASK                                   0x7
+#define DL6_SEL_DOMAIN_MASK_SFT                               (0x7 << 13)
+#define DL6_SEL_FS_SFT                                        8
+#define DL6_SEL_FS_MASK                                       0x1f
+#define DL6_SEL_FS_MASK_SFT                                   (0x1f << 8)
+#define DL6_SW_CLEAR_BUF_EMPTY_SFT                            7
+#define DL6_SW_CLEAR_BUF_EMPTY_MASK                           0x1
+#define DL6_SW_CLEAR_BUF_EMPTY_MASK_SFT                       (0x1 << 7)
+#define DL6_PBUF_SIZE_SFT                                     5
+#define DL6_PBUF_SIZE_MASK                                    0x3
+#define DL6_PBUF_SIZE_MASK_SFT                                (0x3 << 5)
+#define DL6_MONO_SFT                                          4
+#define DL6_MONO_MASK                                         0x1
+#define DL6_MONO_MASK_SFT                                     (0x1 << 4)
+#define DL6_NORMAL_MODE_SFT                                   3
+#define DL6_NORMAL_MODE_MASK                                  0x1
+#define DL6_NORMAL_MODE_MASK_SFT                              (0x1 << 3)
+#define DL6_HALIGN_SFT                                        2
+#define DL6_HALIGN_MASK                                       0x1
+#define DL6_HALIGN_MASK_SFT                                   (0x1 << 2)
+#define DL6_HD_MODE_SFT                                       0
+#define DL6_HD_MODE_MASK                                      0x3
+#define DL6_HD_MODE_MASK_SFT                                  (0x3 << 0)
+
+/* AFE_DL7_BASE_MSB */
+#define DL7_BASE__ADDR_MSB_SFT                                0
+#define DL7_BASE__ADDR_MSB_MASK                               0x1ff
+#define DL7_BASE__ADDR_MSB_MASK_SFT                           (0x1ff << 0)
+
+/* AFE_DL7_BASE */
+#define DL7_BASE_ADDR_SFT                                     4
+#define DL7_BASE_ADDR_MASK                                    0xfffffff
+#define DL7_BASE_ADDR_MASK_SFT                                (0xfffffff << 4)
+
+/* AFE_DL7_CUR_MSB */
+#define DL7_CUR_PTR_MSB_SFT                                   0
+#define DL7_CUR_PTR_MSB_MASK                                  0x1ff
+#define DL7_CUR_PTR_MSB_MASK_SFT                              (0x1ff << 0)
+
+/* AFE_DL7_CUR */
+#define DL7_CUR_PTR_SFT                                       0
+#define DL7_CUR_PTR_MASK                                      0xffffffff
+#define DL7_CUR_PTR_MASK_SFT                                  (0xffffffff << 0)
+
+/* AFE_DL7_END_MSB */
+#define DL7_END_ADDR_MSB_SFT                                  0
+#define DL7_END_ADDR_MSB_MASK                                 0x1ff
+#define DL7_END_ADDR_MSB_MASK_SFT                             (0x1ff << 0)
+
+/* AFE_DL7_END */
+#define DL7_END_ADDR_SFT                                      4
+#define DL7_END_ADDR_MASK                                     0xfffffff
+#define DL7_END_ADDR_MASK_SFT                                 (0xfffffff << 4)
+
+/* AFE_DL7_RCH_MON */
+#define DL7_RCH_DATA_SFT                                      0
+#define DL7_RCH_DATA_MASK                                     0xffffffff
+#define DL7_RCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL7_LCH_MON */
+#define DL7_LCH_DATA_SFT                                      0
+#define DL7_LCH_DATA_MASK                                     0xffffffff
+#define DL7_LCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL7_CON0 */
+#define DL7_ON_SFT                                            28
+#define DL7_ON_MASK                                           0x1
+#define DL7_ON_MASK_SFT                                       (0x1 << 28)
+#define DL7_ONE_HEART_SEL_SFT                                 22
+#define DL7_ONE_HEART_SEL_MASK                                0x3
+#define DL7_ONE_HEART_SEL_MASK_SFT                            (0x3 << 22)
+#define DL7_MINLEN_SFT                                        20
+#define DL7_MINLEN_MASK                                       0x3
+#define DL7_MINLEN_MASK_SFT                                   (0x3 << 20)
+#define DL7_MAXLEN_SFT                                        16
+#define DL7_MAXLEN_MASK                                       0x3
+#define DL7_MAXLEN_MASK_SFT                                   (0x3 << 16)
+#define DL7_SEL_DOMAIN_SFT                                    13
+#define DL7_SEL_DOMAIN_MASK                                   0x7
+#define DL7_SEL_DOMAIN_MASK_SFT                               (0x7 << 13)
+#define DL7_SEL_FS_SFT                                        8
+#define DL7_SEL_FS_MASK                                       0x1f
+#define DL7_SEL_FS_MASK_SFT                                   (0x1f << 8)
+#define DL7_SW_CLEAR_BUF_EMPTY_SFT                            7
+#define DL7_SW_CLEAR_BUF_EMPTY_MASK                           0x1
+#define DL7_SW_CLEAR_BUF_EMPTY_MASK_SFT                       (0x1 << 7)
+#define DL7_PBUF_SIZE_SFT                                     5
+#define DL7_PBUF_SIZE_MASK                                    0x3
+#define DL7_PBUF_SIZE_MASK_SFT                                (0x3 << 5)
+#define DL7_MONO_SFT                                          4
+#define DL7_MONO_MASK                                         0x1
+#define DL7_MONO_MASK_SFT                                     (0x1 << 4)
+#define DL7_NORMAL_MODE_SFT                                   3
+#define DL7_NORMAL_MODE_MASK                                  0x1
+#define DL7_NORMAL_MODE_MASK_SFT                              (0x1 << 3)
+#define DL7_HALIGN_SFT                                        2
+#define DL7_HALIGN_MASK                                       0x1
+#define DL7_HALIGN_MASK_SFT                                   (0x1 << 2)
+#define DL7_HD_MODE_SFT                                       0
+#define DL7_HD_MODE_MASK                                      0x3
+#define DL7_HD_MODE_MASK_SFT                                  (0x3 << 0)
+
+/* AFE_DL8_BASE_MSB */
+#define DL8_BASE__ADDR_MSB_SFT                                0
+#define DL8_BASE__ADDR_MSB_MASK                               0x1ff
+#define DL8_BASE__ADDR_MSB_MASK_SFT                           (0x1ff << 0)
+
+/* AFE_DL8_BASE */
+#define DL8_BASE_ADDR_SFT                                     4
+#define DL8_BASE_ADDR_MASK                                    0xfffffff
+#define DL8_BASE_ADDR_MASK_SFT                                (0xfffffff << 4)
+
+/* AFE_DL8_CUR_MSB */
+#define DL8_CUR_PTR_MSB_SFT                                   0
+#define DL8_CUR_PTR_MSB_MASK                                  0x1ff
+#define DL8_CUR_PTR_MSB_MASK_SFT                              (0x1ff << 0)
+
+/* AFE_DL8_CUR */
+#define DL8_CUR_PTR_SFT                                       0
+#define DL8_CUR_PTR_MASK                                      0xffffffff
+#define DL8_CUR_PTR_MASK_SFT                                  (0xffffffff << 0)
+
+/* AFE_DL8_END_MSB */
+#define DL8_END_ADDR_MSB_SFT                                  0
+#define DL8_END_ADDR_MSB_MASK                                 0x1ff
+#define DL8_END_ADDR_MSB_MASK_SFT                             (0x1ff << 0)
+
+/* AFE_DL8_END */
+#define DL8_END_ADDR_SFT                                      4
+#define DL8_END_ADDR_MASK                                     0xfffffff
+#define DL8_END_ADDR_MASK_SFT                                 (0xfffffff << 4)
+
+/* AFE_DL8_RCH_MON */
+#define DL8_RCH_DATA_SFT                                      0
+#define DL8_RCH_DATA_MASK                                     0xffffffff
+#define DL8_RCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL8_LCH_MON */
+#define DL8_LCH_DATA_SFT                                      0
+#define DL8_LCH_DATA_MASK                                     0xffffffff
+#define DL8_LCH_DATA_MASK_SFT                                 (0xffffffff << 0)
+
+/* AFE_DL8_CON0 */
+#define DL8_ON_SFT                                            28
+#define DL8_ON_MASK                                           0x1
+#define DL8_ON_MASK_SFT                                       (0x1 << 28)
+#define DL8_ONE_HEART_SEL_SFT                                 22
+#define DL8_ONE_HEART_SEL_MASK                                0x3
+#define DL8_ONE_HEART_SEL_MASK_SFT                            (0x3 << 22)
+#define DL8_MINLEN_SFT                                        20
+#define DL8_MINLEN_MASK                                       0x3
+#define DL8_MINLEN_MASK_SFT                                   (0x3 << 20)
+#define DL8_MAXLEN_SFT                                        16
+#define DL8_MAXLEN_MASK                                       0x3
+#define DL8_MAXLEN_MASK_SFT                                   (0x3 << 16)
+#define DL8_SEL_DOMAIN_SFT                                    13
+#define DL8_SEL_DOMAIN_MASK                                   0x7
+#define DL8_SEL_DOMAIN_MASK_SFT                               (0x7 << 13)
+#define DL8_SEL_FS_SFT                                        8
+#define DL8_SEL_FS_MASK                                       0x1f
+#define DL8_SEL_FS_MASK_SFT                                   (0x1f << 8)
+#define DL8_SW_CLEAR_BUF_EMPTY_SFT                            7
+#define DL8_SW_CLEAR_BUF_EMPTY_MASK                           0x1
+#define DL8_SW_CLEAR_BUF_EMPTY_MASK_SFT                       (0x1 << 7)
+#define DL8_PBUF_SIZE_SFT                                     5
+#define DL8_PBUF_SIZE_MASK                                    0x3
+#define DL8_PBUF_SIZE_MASK_SFT                                (0x3 << 5)
+#define DL8_MONO_SFT                                          4
+#define DL8_MONO_MASK                                         0x1
+#define DL8_MONO_MASK_SFT                                     (0x1 << 4)
+#define DL8_NORMAL_MODE_SFT                                   3
+#define DL8_NORMAL_MODE_MASK                                  0x1
+#define DL8_NORMAL_MODE_MASK_SFT                              (0x1 << 3)
+#define DL8_HALIGN_SFT                                        2
+#define DL8_HALIGN_MASK                                       0x1
+#define DL8_HALIGN_MASK_SFT                                   (0x1 << 2)
+#define DL8_HD_MODE_SFT                                       0
+#define DL8_HD_MODE_MASK                                      0x3
+#define DL8_HD_MODE_MASK_SFT                                  (0x3 << 0)
+
+/* AFE_DL_4CH_BASE_MSB */
+#define DL_4CH_BASE__ADDR_MSB_SFT                             0
+#define DL_4CH_BASE__ADDR_MSB_MASK                            0x1ff
+#define DL_4CH_BASE__ADDR_MSB_MASK_SFT                        (0x1ff << 0)
+
+/* AFE_DL_4CH_BASE */
+#define DL_4CH_BASE_ADDR_SFT                                  4
+#define DL_4CH_BASE_ADDR_MASK                                 0xfffffff
+#define DL_4CH_BASE_ADDR_MASK_SFT                             (0xfffffff << 4)
+
+/* AFE_DL_4CH_CUR_MSB */
+#define DL_4CH_CUR_PTR_MSB_SFT                                0
+#define DL_4CH_CUR_PTR_MSB_MASK                               0x1ff
+#define DL_4CH_CUR_PTR_MSB_MASK_SFT                           (0x1ff << 0)
+
+/* AFE_DL_4CH_CUR */
+#define DL_4CH_CUR_PTR_SFT                                    0
+#define DL_4CH_CUR_PTR_MASK                                   0xffffffff
+#define DL_4CH_CUR_PTR_MASK_SFT                               (0xffffffff << 0)
+
+/* AFE_DL_4CH_END_MSB */
+#define DL_4CH_END_ADDR_MSB_SFT                               0
+#define DL_4CH_END_ADDR_MSB_MASK                              0x1ff
+#define DL_4CH_END_ADDR_MSB_MASK_SFT                          (0x1ff << 0)
+
+/* AFE_DL_4CH_END */
+#define DL_4CH_END_ADDR_SFT                                   4
+#define DL_4CH_END_ADDR_MASK                                  0xfffffff
+#define DL_4CH_END_ADDR_MASK_SFT                              (0xfffffff << 4)
+
+/* AFE_DL_4CH_CON0 */
+#define DL_4CH_ON_SFT                                         31
+#define DL_4CH_ON_MASK                                        0x1
+#define DL_4CH_ON_MASK_SFT                                    (0x1 << 31)
+#define DL_4CH_NUM_SFT                                        24
+#define DL_4CH_NUM_MAS